diff --git a/.asf.yaml b/.asf.yaml
index c7267ca5f346a..60bc1df21a48a 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# https://cwiki.apache.org/confluence/display/INFRA/.asf.yaml+features+for+git+repositories
+# https://cwiki.apache.org/confluence/display/INFRA/git+-+.asf.yaml+features
 ---
 github:
   description: "Apache Spark - A unified analytics engine for large-scale data processing"
diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE
index fcd3588cf81ee..66c28ee9666ee 100644
--- a/.github/PULL_REQUEST_TEMPLATE
+++ b/.github/PULL_REQUEST_TEMPLATE
@@ -8,6 +8,8 @@ Thanks for sending a pull request!  Here are some tips for you:
   6. If possible, provide a concise example to reproduce the issue for a faster review.
   7. If you want to add a new configuration, please read the guideline first for naming configurations in
      'core/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala'.
+  8. If you want to add or modify an error message, please read the guideline first:
+     https://spark.apache.org/error-message-guidelines.html
 -->
 
 ### What changes were proposed in this pull request?
diff --git a/.github/autolabeler.yml b/.github/autolabeler.yml
deleted file mode 100644
index 3bca01f89950a..0000000000000
--- a/.github/autolabeler.yml
+++ /dev/null
@@ -1,133 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Bot page: https://github.com/apps/probot-autolabeler
-# The matching patterns follow the .gitignore spec.
-# See: https://git-scm.com/docs/gitignore#_pattern_format
-# Also, note that the plugin uses 'ignore' package. See also
-# https://github.com/kaelzhang/node-ignore
-INFRA:
-  - ".github/"
-  - "appveyor.yml"
-  - "/tools/"
-  - "/dev/create-release/"
-  - ".asf.yaml"
-  - ".gitattributes"
-  - ".gitignore"
-  - "/dev/github_jira_sync.py"
-  - "/dev/merge_spark_pr.py"
-  - "/dev/run-tests-jenkins*"
-BUILD:
-  - "/dev/"
-  - "!/dev/github_jira_sync.py"
-  - "!/dev/merge_spark_pr.py"
-  - "!/dev/run-tests-jenkins*"
-  - "!/dev/.rat-excludes"
-  - "/build/"
-  - "/project/"
-  - "/assembly/"
-  - "*pom.xml"
-  - "/bin/docker-image-tool.sh"
-  - "/bin/find-spark-home*"
-  - "scalastyle-config.xml"
-DOCS:
-  - "docs/"
-  - "/README.md"
-  - "/CONTRIBUTING.md"
-EXAMPLES:
-  - "examples/"
-  - "/bin/run-example*"
-CORE:
-  - "/core/"
-  - "!UI.scala"
-  - "!ui/"
-  - "/common/kvstore/"
-  - "/common/network-common/"
-  - "/common/network-shuffle/"
-  - "/python/pyspark/*.py"
-  - "/python/pyspark/tests/*.py"
-SPARK SUBMIT:
-  - "/bin/spark-submit*"
-SPARK SHELL:
-  - "/repl/"
-  - "/bin/spark-shell*"
-SQL:
-  - "sql/"
-  - "/common/unsafe/"
-  - "!/python/pyspark/sql/avro/"
-  - "!/python/pyspark/sql/streaming.py"
-  - "!/python/pyspark/sql/tests/test_streaming.py"
-  - "/bin/spark-sql*"
-  - "/bin/beeline*"
-  - "/sbin/*thriftserver*.sh"
-  - "*SQL*.R"
-  - "DataFrame.R"
-  - "WindowSpec.R"
-  - "catalog.R"
-  - "column.R"
-  - "functions.R"
-  - "group.R"
-  - "schema.R"
-  - "types.R"
-AVRO:
-  - "/external/avro/"
-  - "/python/pyspark/sql/avro/"
-DSTREAM:
-  - "/streaming/"
-  - "/data/streaming/"
-  - "/external/flume*"
-  - "/external/kinesis*"
-  - "/external/kafka*"
-  - "/python/pyspark/streaming/"
-GRAPHX:
-  - "/graphx/"
-  - "/data/graphx/"
-ML:
-  - "ml/"
-  - "*mllib_*.R"
-MLLIB:
-  - "spark/mllib/"
-  - "/mllib-local/"
-  - "/python/pyspark/mllib/"
-STRUCTURED STREAMING:
-  - "sql/**/streaming/"
-  - "/external/kafka-0-10-sql/"
-  - "/python/pyspark/sql/streaming.py"
-  - "/python/pyspark/sql/tests/test_streaming.py"
-  - "*streaming.R"
-PYTHON:
-  - "/bin/pyspark*"
-  - "python/"
-R:
-  - "r/"
-  - "R/"
-  - "/bin/sparkR*"
-YARN:
-  - "/resource-managers/yarn/"
-MESOS:
-  - "/resource-managers/mesos/"
-  - "/sbin/*mesos*.sh"
-KUBERNETES:
-  - "/resource-managers/kubernetes/"
-WINDOWS:
-  - "*.cmd"
-  - "/R/pkg/tests/fulltests/test_Windows.R"
-WEB UI:
-  - "ui/"
-  - "UI.scala"
-DEPLOY:
-  - "/sbin/"
diff --git a/.github/labeler.yml b/.github/labeler.yml
new file mode 100644
index 0000000000000..bd61902925e33
--- /dev/null
+++ b/.github/labeler.yml
@@ -0,0 +1,152 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+#
+# Pull Request Labeler Github Action Configuration: https://github.com/marketplace/actions/labeler
+#
+# Note that we currently cannot use the negatioon operator  (i.e. `!`)  for miniglob matches as they
+# would match any file that doesn't touch them. What's needed is the concept of `any `, which takes a
+# list of constraints / globs and then matches all of the constraints for either `any` of the files or
+# `all` of the files in the change set.
+#
+# However, `any`/`all` are not supported in a released version and testing off of the `main` branch
+# resulted in some other errors when testing.
+#
+# An issue has been opened upstream requesting that a release be cut that has support for all/any:
+#   - https://github.com/actions/labeler/issues/111
+#
+# While we wait for this issue to be handled upstream, we can remove
+# the negated / `!` matches for now and at least have labels again.
+#
+INFRA:
+  - ".github/**/*"
+  - "appveyor.yml"
+  - "tools/**/*"
+  - "dev/create-release/**/*"
+  - ".asf.yaml"
+  - ".gitattributes"
+  - ".gitignore"
+  - "dev/github_jira_sync.py"
+  - "dev/merge_spark_pr.py"
+  - "dev/run-tests-jenkins*"
+BUILD:
+ # Can be supported when a stable release with correct all/any is released
+ #- any: ['dev/**/*', '!dev/github_jira_sync.py', '!dev/merge_spark_pr.py', '!dev/.rat-excludes']
+ - "dev/**/*"
+ - "build/**/*"
+ - "project/**/*"
+ - "assembly/**/*"
+ - "**/*pom.xml"
+ - "bin/docker-image-tool.sh"
+ - "bin/find-spark-home*"
+ - "scalastyle-config.xml"
+ # These can be added in the above `any` clause (and the /dev/**/* glob removed) when
+ # `any`/`all` support is released
+ # - "!dev/github_jira_sync.py"
+ # - "!dev/merge_spark_pr.py"
+ # - "!dev/run-tests-jenkins*"
+ # - "!dev/.rat-excludes"
+DOCS:
+  - "docs/**/*"
+  - "**/README.md"
+  - "**/CONTRIBUTING.md"
+EXAMPLES:
+  - "examples/**/*"
+  - "bin/run-example*"
+# CORE needs to be updated when all/any are released upstream.
+CORE:
+  # - any: ["core/**/*", "!**/*UI.scala", "!**/ui/**/*"] # If any file matches all of the globs defined in the list started by `any`, label is applied.
+  - "core/**/*"
+  - "common/kvstore/**/*"
+  - "common/network-common/**/*"
+  - "common/network-shuffle/**/*"
+  - "python/pyspark/**/*.py"
+  - "python/pyspark/tests/**/*.py"
+SPARK SUBMIT:
+  - "bin/spark-submit*"
+SPARK SHELL:
+  - "repl/**/*"
+  - "bin/spark-shell*"
+SQL:
+#- any: ["**/sql/**/*", "!python/pyspark/sql/avro/**/*", "!python/pyspark/sql/streaming.py", "!python/pyspark/sql/tests/test_streaming.py"]
+  - "**/sql/**/*"
+  - "common/unsafe/**/*"
+  #- "!python/pyspark/sql/avro/**/*"
+  #- "!python/pyspark/sql/streaming.py"
+  #- "!python/pyspark/sql/tests/test_streaming.py"
+  - "bin/spark-sql*"
+  - "bin/beeline*"
+  - "sbin/*thriftserver*.sh"
+  - "**/*SQL*.R"
+  - "**/DataFrame.R"
+  - "**/*WindowSpec.R"
+  - "**/*catalog.R"
+  - "**/*column.R"
+  - "**/*functions.R"
+  - "**/*group.R"
+  - "**/*schema.R"
+  - "**/*types.R"
+AVRO:
+  - "external/avro/**/*"
+  - "python/pyspark/sql/avro/**/*"
+DSTREAM:
+  - "streaming/**/*"
+  - "data/streaming/**/*"
+  - "external/kinesis*"
+  - "external/kafka*"
+  - "python/pyspark/streaming/**/*"
+GRAPHX:
+  - "graphx/**/*"
+  - "data/graphx/**/*"
+ML:
+  - "**/ml/**/*"
+  - "**/*mllib_*.R"
+MLLIB:
+  - "**/spark/mllib/**/*"
+  - "mllib-local/**/*"
+  - "python/pyspark/mllib/**/*"
+STRUCTURED STREAMING:
+  - "**/sql/**/streaming/**/*"
+  - "external/kafka-0-10-sql/**/*"
+  - "python/pyspark/sql/streaming.py"
+  - "python/pyspark/sql/tests/test_streaming.py"
+  - "**/*streaming.R"
+PYTHON:
+  - "bin/pyspark*"
+  - "**/python/**/*"
+R:
+  - "**/r/**/*"
+  - "**/R/**/*"
+  - "bin/sparkR*"
+YARN:
+  - "resource-managers/yarn/**/*"
+MESOS:
+  - "resource-managers/mesos/**/*"
+  - "sbin/*mesos*.sh"
+KUBERNETES:
+  - "resource-managers/kubernetes/**/*"
+WINDOWS:
+  - "**/*.cmd"
+  - "R/pkg/tests/fulltests/test_Windows.R"
+WEB UI:
+  - "**/ui/**/*"
+  - "**/*UI.scala"
+DEPLOY:
+  - "sbin/**/*"
+
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
new file mode 100644
index 0000000000000..76ae152f2307f
--- /dev/null
+++ b/.github/workflows/benchmark.yml
@@ -0,0 +1,100 @@
+name: Run benchmarks
+
+on:
+  workflow_dispatch:
+    inputs:
+      class:
+        description: 'Benchmark class'
+        required: true
+        default: '*'
+      jdk:
+        description: 'JDK version: 8 or 11'
+        required: true
+        default: '8'
+      failfast:
+        description: 'Failfast: true or false'
+        required: true
+        default: 'true'
+      num-splits:
+        description: 'Number of job splits'
+        required: true
+        default: '1'
+
+jobs:
+  matrix-gen:
+    name: Generate matrix for job splits
+    runs-on: ubuntu-20.04
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    env:
+      SPARK_BENCHMARK_NUM_SPLITS: ${{ github.event.inputs.num-splits }}
+    steps:
+    - name: Generate matrix
+      id: set-matrix
+      run: echo "::set-output name=matrix::["`seq -s, 1 $SPARK_BENCHMARK_NUM_SPLITS`"]"
+
+  benchmark:
+    name: "Run benchmarks: ${{ github.event.inputs.class }} (JDK ${{ github.event.inputs.jdk }}, ${{ matrix.split }} out of ${{ github.event.inputs.num-splits }} splits)"
+    needs: matrix-gen
+    # Ubuntu 20.04 is the latest LTS. The next LTS is 22.04.
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        split: ${{fromJSON(needs.matrix-gen.outputs.matrix)}}
+    env:
+      SPARK_BENCHMARK_FAILFAST: ${{ github.event.inputs.failfast }}
+      SPARK_BENCHMARK_NUM_SPLITS: ${{ github.event.inputs.num-splits }}
+      SPARK_BENCHMARK_CUR_SPLIT: ${{ matrix.split }}
+      SPARK_GENERATE_BENCHMARK_FILES: 1
+      SPARK_LOCAL_IP: localhost
+    steps:
+    - name: Checkout Spark repository
+      uses: actions/checkout@v2
+      # In order to get diff files
+      with:
+        fetch-depth: 0
+    - name: Cache Scala, SBT and Maven
+      uses: actions/cache@v2
+      with:
+        path: |
+          build/apache-maven-*
+          build/scala-*
+          build/*.jar
+          ~/.sbt
+        key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
+        restore-keys: |
+          build-
+    - name: Cache Coursier local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.cache/coursier
+        key: benchmark-coursier-${{ github.event.inputs.jdk }}-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
+        restore-keys: |
+          benchmark-coursier-${{ github.event.inputs.jdk }}
+    - name: Install Java ${{ github.event.inputs.jdk }}
+      uses: actions/setup-java@v1
+      with:
+        java-version: ${{ github.event.inputs.jdk }}
+    - name: Run benchmarks
+      run: |
+        ./build/sbt -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -Phadoop-cloud -Pkinesis-asl -Pspark-ganglia-lgpl test:package
+        # Make less noisy
+        cp conf/log4j.properties.template conf/log4j.properties
+        sed -i 's/log4j.rootCategory=INFO, console/log4j.rootCategory=WARN, console/g' conf/log4j.properties
+        # In benchmark, we use local as master so set driver memory only. Note that GitHub Actions has 7 GB memory limit.
+        bin/spark-submit \
+          --driver-memory 6g --class org.apache.spark.benchmark.Benchmarks \
+          --jars "`find . -name '*-SNAPSHOT-tests.jar' -o -name '*avro*-SNAPSHOT.jar' | paste -sd ',' -`" \
+          "`find . -name 'spark-core*-SNAPSHOT-tests.jar'`" \
+          "${{ github.event.inputs.class }}"
+        # To keep the directory structure and file permissions, tar them
+        # See also https://github.com/actions/upload-artifact#maintaining-file-permissions-and-case-sensitive-files
+        echo "Preparing the benchmark results:"
+        tar -cvf benchmark-results-${{ github.event.inputs.jdk }}.tar `git diff --name-only` `git ls-files --others --exclude-standard`
+    - name: Upload benchmark results
+      uses: actions/upload-artifact@v2
+      with:
+        name: benchmark-results-${{ github.event.inputs.jdk }}-${{ matrix.split }}
+        path: benchmark-results-${{ github.event.inputs.jdk }}.tar
+
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
new file mode 100644
index 0000000000000..be5978e7ecb97
--- /dev/null
+++ b/.github/workflows/build_and_test.yml
@@ -0,0 +1,561 @@
+name: Build and test
+
+on:
+  push:
+    branches:
+    - '**'
+    - '!branch-*.*'
+
+jobs:
+  # Build: build Spark and run the tests for specified modules.
+  build:
+    name: "Build modules: ${{ matrix.modules }} ${{ matrix.comment }} (JDK ${{ matrix.java }}, ${{ matrix.hadoop }}, ${{ matrix.hive }})"
+    # Ubuntu 20.04 is the latest LTS. The next LTS is 22.04.
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        java:
+          - 8
+        hadoop:
+          - hadoop3.2
+        hive:
+          - hive2.3
+        # TODO(SPARK-32246): We don't test 'streaming-kinesis-asl' for now.
+        # Kinesis tests depends on external Amazon kinesis service.
+        # Note that the modules below are from sparktestsupport/modules.py.
+        modules:
+          - >-
+            core, unsafe, kvstore, avro,
+            network-common, network-shuffle, repl, launcher,
+            examples, sketch, graphx
+          - >-
+            catalyst, hive-thriftserver
+          - >-
+            streaming, sql-kafka-0-10, streaming-kafka-0-10,
+            mllib-local, mllib,
+            yarn, mesos, kubernetes, hadoop-cloud, spark-ganglia-lgpl
+        # Here, we split Hive and SQL tests into some of slow ones and the rest of them.
+        included-tags: [""]
+        excluded-tags: [""]
+        comment: [""]
+        include:
+          # Hive tests
+          - modules: hive
+            java: 8
+            hadoop: hadoop3.2
+            hive: hive2.3
+            included-tags: org.apache.spark.tags.SlowHiveTest
+            comment: "- slow tests"
+          - modules: hive
+            java: 8
+            hadoop: hadoop3.2
+            hive: hive2.3
+            excluded-tags: org.apache.spark.tags.SlowHiveTest
+            comment: "- other tests"
+          # SQL tests
+          - modules: sql
+            java: 8
+            hadoop: hadoop3.2
+            hive: hive2.3
+            included-tags: org.apache.spark.tags.ExtendedSQLTest
+            comment: "- slow tests"
+          - modules: sql
+            java: 8
+            hadoop: hadoop3.2
+            hive: hive2.3
+            excluded-tags: org.apache.spark.tags.ExtendedSQLTest
+            comment: "- other tests"
+    env:
+      MODULES_TO_TEST: ${{ matrix.modules }}
+      EXCLUDED_TAGS: ${{ matrix.excluded-tags }}
+      INCLUDED_TAGS: ${{ matrix.included-tags }}
+      HADOOP_PROFILE: ${{ matrix.hadoop }}
+      HIVE_PROFILE: ${{ matrix.hive }}
+      GITHUB_PREV_SHA: ${{ github.event.before }}
+      SPARK_LOCAL_IP: localhost
+    steps:
+    - name: Checkout Spark repository
+      uses: actions/checkout@v2
+      # In order to fetch changed files
+      with:
+        fetch-depth: 0
+        repository: apache/spark
+        ref: master
+    - name: Sync the current branch with the latest in Apache Spark
+      if: github.repository != 'apache/spark'
+      id: sync-branch
+      run: |
+        apache_spark_ref=`git rev-parse HEAD`
+        git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/}
+        git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD
+        git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit"
+        echo "::set-output name=APACHE_SPARK_REF::$apache_spark_ref"
+    # Cache local repositories. Note that GitHub Actions cache has a 2G limit.
+    - name: Cache Scala, SBT and Maven
+      uses: actions/cache@v2
+      with:
+        path: |
+          build/apache-maven-*
+          build/scala-*
+          build/*.jar
+          ~/.sbt
+        key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
+        restore-keys: |
+          build-
+    - name: Cache Coursier local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.cache/coursier
+        key: ${{ matrix.java }}-${{ matrix.hadoop }}-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
+        restore-keys: |
+          ${{ matrix.java }}-${{ matrix.hadoop }}-coursier-
+    - name: Install Java ${{ matrix.java }}
+      uses: actions/setup-java@v1
+      with:
+        java-version: ${{ matrix.java }}
+    - name: Install Python 3.8
+      uses: actions/setup-python@v2
+      # We should install one Python that is higher then 3+ for SQL and Yarn because:
+      # - SQL component also has Python related tests, for example, IntegratedUDFTestUtils.
+      # - Yarn has a Python specific test too, for example, YarnClusterSuite.
+      if: contains(matrix.modules, 'yarn') || (contains(matrix.modules, 'sql') && !contains(matrix.modules, 'sql-'))
+      with:
+        python-version: 3.8
+        architecture: x64
+    - name: Install Python packages (Python 3.8)
+      if: (contains(matrix.modules, 'sql') && !contains(matrix.modules, 'sql-'))
+      run: |
+        python3.8 -m pip install numpy 'pyarrow<3.0.0' pandas scipy xmlrunner
+        python3.8 -m pip list
+    # Run the tests.
+    - name: Run tests
+      run: |
+        export APACHE_SPARK_REF=${{ steps.sync-branch.outputs.APACHE_SPARK_REF }}
+        # Hive and SQL tests become flaky when running in parallel as it's too intensive.
+        if [[ "$MODULES_TO_TEST" == "hive" ]] || [[ "$MODULES_TO_TEST" == "sql" ]]; then export SERIAL_SBT_TESTS=1; fi
+        ./dev/run-tests --parallelism 2 --modules "$MODULES_TO_TEST" --included-tags "$INCLUDED_TAGS" --excluded-tags "$EXCLUDED_TAGS"
+    - name: Upload test results to report
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: test-results-${{ matrix.modules }}-${{ matrix.comment }}-${{ matrix.java }}-${{ matrix.hadoop }}-${{ matrix.hive }}
+        path: "**/target/test-reports/*.xml"
+    - name: Upload unit tests log files
+      if: failure()
+      uses: actions/upload-artifact@v2
+      with:
+        name: unit-tests-log-${{ matrix.modules }}-${{ matrix.comment }}-${{ matrix.java }}-${{ matrix.hadoop }}-${{ matrix.hive }}
+        path: "**/target/unit-tests.log"
+
+  pyspark:
+    name: "Build modules: ${{ matrix.modules }}"
+    runs-on: ubuntu-20.04
+    container:
+      image: dongjoon/apache-spark-github-action-image:20201025
+    strategy:
+      fail-fast: false
+      matrix:
+        modules:
+          - >-
+            pyspark-sql, pyspark-mllib, pyspark-resource
+          - >-
+            pyspark-core, pyspark-streaming, pyspark-ml
+          - >-
+            pyspark-pandas
+    env:
+      MODULES_TO_TEST: ${{ matrix.modules }}
+      HADOOP_PROFILE: hadoop3.2
+      HIVE_PROFILE: hive2.3
+      GITHUB_PREV_SHA: ${{ github.event.before }}
+      SPARK_LOCAL_IP: localhost
+    steps:
+    - name: Checkout Spark repository
+      uses: actions/checkout@v2
+      # In order to fetch changed files
+      with:
+        fetch-depth: 0
+        repository: apache/spark
+        ref: master
+    - name: Sync the current branch with the latest in Apache Spark
+      if: github.repository != 'apache/spark'
+      id: sync-branch
+      run: |
+        apache_spark_ref=`git rev-parse HEAD`
+        git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/}
+        git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD
+        git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit"
+        echo "::set-output name=APACHE_SPARK_REF::$apache_spark_ref"
+    # Cache local repositories. Note that GitHub Actions cache has a 2G limit.
+    - name: Cache Scala, SBT and Maven
+      uses: actions/cache@v2
+      with:
+        path: |
+          build/apache-maven-*
+          build/scala-*
+          build/*.jar
+          ~/.sbt
+        key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
+        restore-keys: |
+          build-
+    - name: Cache Coursier local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.cache/coursier
+        key: pyspark-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
+        restore-keys: |
+          pyspark-coursier-
+    - name: Install Python 3.6
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.6
+        architecture: x64
+    # This step takes much less time (~30s) than other Python versions so it is not included
+    # in the Docker image being used. There is also a technical issue to install Python 3.6 on
+    # Ubuntu 20.04. See also SPARK-33162.
+    - name: Install Python packages (Python 3.6)
+      run: |
+        python3.6 -m pip install numpy 'pyarrow<3.0.0' pandas scipy xmlrunner
+        python3.6 -m pip list
+    - name: Install Conda for pip packaging test
+      run: |
+        curl -s https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh > miniconda.sh
+        bash miniconda.sh -b -p $HOME/miniconda
+    # Run the tests.
+    - name: Run tests
+      run: |
+        export APACHE_SPARK_REF=${{ steps.sync-branch.outputs.APACHE_SPARK_REF }}
+        export PATH=$PATH:$HOME/miniconda/bin
+        ./dev/run-tests --parallelism 2 --modules "$MODULES_TO_TEST"
+    - name: Upload test results to report
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: test-results-${{ matrix.modules }}--8-hadoop3.2-hive2.3
+        path: "**/target/test-reports/*.xml"
+    - name: Upload unit tests log files
+      if: failure()
+      uses: actions/upload-artifact@v2
+      with:
+        name: unit-tests-log-${{ matrix.modules }}--8-hadoop3.2-hive2.3
+        path: "**/target/unit-tests.log"
+
+  sparkr:
+    name: "Build modules: sparkr"
+    runs-on: ubuntu-20.04
+    container:
+      image: dongjoon/apache-spark-github-action-image:20201025
+    env:
+      HADOOP_PROFILE: hadoop3.2
+      HIVE_PROFILE: hive2.3
+      GITHUB_PREV_SHA: ${{ github.event.before }}
+      SPARK_LOCAL_IP: localhost
+    steps:
+    - name: Checkout Spark repository
+      uses: actions/checkout@v2
+      # In order to fetch changed files
+      with:
+        fetch-depth: 0
+        repository: apache/spark
+        ref: master
+    - name: Sync the current branch with the latest in Apache Spark
+      if: github.repository != 'apache/spark'
+      id: sync-branch
+      run: |
+        apache_spark_ref=`git rev-parse HEAD`
+        git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/}
+        git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD
+        git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit"
+        echo "::set-output name=APACHE_SPARK_REF::$apache_spark_ref"
+    # Cache local repositories. Note that GitHub Actions cache has a 2G limit.
+    - name: Cache Scala, SBT and Maven
+      uses: actions/cache@v2
+      with:
+        path: |
+          build/apache-maven-*
+          build/scala-*
+          build/*.jar
+          ~/.sbt
+        key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
+        restore-keys: |
+          build-
+    - name: Cache Coursier local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.cache/coursier
+        key: sparkr-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
+        restore-keys: |
+          sparkr-coursier-
+    - name: Run tests
+      run: |
+        # The followings are also used by `r-lib/actions/setup-r` to avoid
+        # R issues at docker environment
+        export TZ=UTC
+        export _R_CHECK_SYSTEM_CLOCK_=FALSE
+        export APACHE_SPARK_REF=${{ steps.sync-branch.outputs.APACHE_SPARK_REF }}
+        ./dev/run-tests --parallelism 2 --modules sparkr
+    - name: Upload test results to report
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: test-results-sparkr--8-hadoop3.2-hive2.3
+        path: "**/target/test-reports/*.xml"
+
+  # Static analysis, and documentation build
+  lint:
+    name: Linters, licenses, dependencies and documentation generation
+    runs-on: ubuntu-20.04
+    env:
+      LC_ALL: C.UTF-8
+      LANG: C.UTF-8
+    container:
+      image: dongjoon/apache-spark-github-action-image:20201025
+    steps:
+    - name: Checkout Spark repository
+      uses: actions/checkout@v2
+    # Cache local repositories. Note that GitHub Actions cache has a 2G limit.
+    - name: Cache Scala, SBT and Maven
+      uses: actions/cache@v2
+      with:
+        path: |
+          build/apache-maven-*
+          build/scala-*
+          build/*.jar
+          ~/.sbt
+        key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
+        restore-keys: |
+          build-
+    - name: Cache Coursier local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.cache/coursier
+        key: docs-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
+        restore-keys: |
+          docs-coursier-
+    - name: Cache Maven local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.m2/repository
+        key: docs-maven-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          docs-maven-
+    - name: Install Python 3.6
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.6
+        architecture: x64
+    - name: Install Python linter dependencies
+      run: |
+        # TODO(SPARK-32407): Sphinx 3.1+ does not correctly index nested classes.
+        #   See also https://github.com/sphinx-doc/sphinx/issues/7551.
+        # Jinja2 3.0.0+ causes error when building with Sphinx.
+        #   See also https://issues.apache.org/jira/browse/SPARK-35375.
+        python3.6 -m pip install flake8 'sphinx<3.1.0' numpy pydata_sphinx_theme ipython nbsphinx mypy numpydoc 'jinja2<3.0.0'
+    - name: Install R linter dependencies and SparkR
+      run: |
+        apt-get install -y libcurl4-openssl-dev libgit2-dev libssl-dev libxml2-dev
+        Rscript -e "install.packages(c('devtools'), repos='https://cloud.r-project.org/')"
+        Rscript -e "devtools::install_github('jimhester/lintr@v2.0.1')"
+        ./R/install-dev.sh
+    - name: Instll JavaScript linter dependencies
+      run: |
+        apt update
+        apt-get install -y nodejs npm
+    - name: Install dependencies for documentation generation
+      run: |
+        # pandoc is required to generate PySpark APIs as well in nbsphinx.
+        apt-get install -y libcurl4-openssl-dev pandoc
+        # TODO(SPARK-32407): Sphinx 3.1+ does not correctly index nested classes.
+        #   See also https://github.com/sphinx-doc/sphinx/issues/7551.
+        # Jinja2 3.0.0+ causes error when building with Sphinx.
+        #   See also https://issues.apache.org/jira/browse/SPARK-35375.
+        python3.6 -m pip install 'sphinx<3.1.0' mkdocs numpy pydata_sphinx_theme ipython nbsphinx numpydoc 'jinja2<3.0.0'
+        apt-get update -y
+        apt-get install -y ruby ruby-dev
+        Rscript -e "install.packages(c('devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2'), repos='https://cloud.r-project.org/')"
+        gem install bundler
+        cd docs
+        bundle install
+    - name: Scala linter
+      run: ./dev/lint-scala
+    - name: Java linter
+      run: ./dev/lint-java
+    - name: Python linter
+      run: ./dev/lint-python
+    - name: R linter
+      run: ./dev/lint-r
+    - name: JS linter
+      run: ./dev/lint-js
+    - name: License test
+      run: ./dev/check-license
+    - name: Dependencies test
+      run: ./dev/test-dependencies.sh
+    - name: Run documentation build
+      run: |
+        cd docs
+        bundle exec jekyll build
+
+  java-11:
+    name: Java 11 build with Maven
+    runs-on: ubuntu-20.04
+    steps:
+    - name: Checkout Spark repository
+      uses: actions/checkout@v2
+    - name: Cache Scala, SBT and Maven
+      uses: actions/cache@v2
+      with:
+        path: |
+          build/apache-maven-*
+          build/scala-*
+          build/*.jar
+          ~/.sbt
+        key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
+        restore-keys: |
+          build-
+    - name: Cache Maven local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.m2/repository
+        key: java11-maven-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          java11-maven-
+    - name: Install Java 11
+      uses: actions/setup-java@v1
+      with:
+        java-version: 11
+    - name: Build with Maven
+      run: |
+        export MAVEN_OPTS="-Xss256m -Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
+        export MAVEN_CLI_OPTS="--no-transfer-progress"
+        # It uses Maven's 'install' intentionally, see https://github.com/apache/spark/pull/26414.
+        ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -Phadoop-cloud -Djava.version=11 install
+        rm -rf ~/.m2/repository/org/apache/spark
+
+  scala-213:
+    name: Scala 2.13 build with SBT
+    runs-on: ubuntu-20.04
+    steps:
+    - name: Checkout Spark repository
+      uses: actions/checkout@v2
+    - name: Cache Scala, SBT and Maven
+      uses: actions/cache@v2
+      with:
+        path: |
+          build/apache-maven-*
+          build/scala-*
+          build/*.jar
+          ~/.sbt
+        key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
+        restore-keys: |
+          build-
+    - name: Cache Coursier local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.cache/coursier
+        key: scala-213-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
+        restore-keys: |
+          scala-213-coursier-
+    - name: Install Java 8
+      uses: actions/setup-java@v1
+      with:
+        java-version: 8
+    - name: Build with SBT
+      run: |
+        ./dev/change-scala-version.sh 2.13
+        ./build/sbt -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -Phadoop-cloud -Pkinesis-asl -Pdocker-integration-tests -Pkubernetes-integration-tests -Pspark-ganglia-lgpl -Pscala-2.13 compile test:compile
+
+  hadoop-2:
+    name: Hadoop 2 build with SBT
+    runs-on: ubuntu-20.04
+    steps:
+    - name: Checkout Spark repository
+      uses: actions/checkout@v2
+    - name: Cache Scala, SBT and Maven
+      uses: actions/cache@v2
+      with:
+        path: |
+          build/apache-maven-*
+          build/scala-*
+          build/*.jar
+          ~/.sbt
+        key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
+        restore-keys: |
+          build-
+    - name: Cache Coursier local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.cache/coursier
+        key: hadoop-2-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
+        restore-keys: |
+          hadoop-2-coursier-
+    - name: Install Java 8
+      uses: actions/setup-java@v1
+      with:
+        java-version: 8
+    - name: Build with SBT
+      run: |
+        ./build/sbt -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -Phadoop-cloud -Pkinesis-asl -Phadoop-2.7 compile test:compile
+
+  tpcds-1g:
+    name: Run TPC-DS queries with SF=1
+    runs-on: ubuntu-20.04
+    env:
+      SPARK_LOCAL_IP: localhost
+    steps:
+    - name: Checkout Spark repository
+      uses: actions/checkout@v2
+    - name: Cache Scala, SBT and Maven
+      uses: actions/cache@v2
+      with:
+        path: |
+          build/apache-maven-*
+          build/scala-*
+          build/*.jar
+          ~/.sbt
+        key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
+        restore-keys: |
+          build-
+    - name: Cache Coursier local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.cache/coursier
+        key: tpcds-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
+        restore-keys: |
+          tpcds-coursier-
+    - name: Install Java 8
+      uses: actions/setup-java@v1
+      with:
+        java-version: 8
+    - name: Cache TPC-DS generated data
+      id: cache-tpcds-sf-1
+      uses: actions/cache@v2
+      with:
+        path: ./tpcds-sf-1
+        key: tpcds-${{ hashFiles('.github/workflows/build_and_test.yml', 'sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala') }}
+    - name: Checkout tpcds-kit repository
+      if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'
+      uses: actions/checkout@v2
+      with:
+        repository: databricks/tpcds-kit
+        path: ./tpcds-kit
+    - name: Build tpcds-kit
+      if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'
+      run: cd tpcds-kit/tools && make OS=LINUX
+    - name: Generate TPC-DS (SF=1) table data
+      if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'
+      run: build/sbt "sql/test:runMain org.apache.spark.sql.GenTPCDSData --dsdgenDir `pwd`/tpcds-kit/tools --location `pwd`/tpcds-sf-1 --scaleFactor 1 --numPartitions 1 --overwrite"
+    - name: Run TPC-DS queries
+      run: |
+        SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 build/sbt "sql/testOnly org.apache.spark.sql.TPCDSQueryTestSuite"
+    - name: Upload test results to report
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: test-results-tpcds--8-hadoop3.2-hive2.3
+        path: "**/target/test-reports/*.xml"
+    - name: Upload unit tests log files
+      if: failure()
+      uses: actions/upload-artifact@v2
+      with:
+        name: unit-tests-log-tpcds--8-hadoop3.2-hive2.3
+        path: "**/target/unit-tests.log"
diff --git a/.github/workflows/cancel_duplicate_workflow_runs.yml b/.github/workflows/cancel_duplicate_workflow_runs.yml
new file mode 100644
index 0000000000000..b20fc947f6f13
--- /dev/null
+++ b/.github/workflows/cancel_duplicate_workflow_runs.yml
@@ -0,0 +1,19 @@
+name: Cancelling Duplicates
+on:
+  workflow_run:
+    workflows: 
+      - 'Build and test'
+    types: ['requested']
+
+jobs:
+  cancel-duplicate-workflow-runs:
+    name: "Cancel duplicate workflow runs"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # @master
+        name: "Cancel duplicate workflow runs"
+        with:
+          cancelMode: allDuplicates
+          token: ${{ secrets.GITHUB_TOKEN }}
+          sourceRunId: ${{ github.event.workflow_run.id }}
+          skipEventTypes: '["push", "schedule"]'
diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml
new file mode 100644
index 0000000000000..98855f4668b45
--- /dev/null
+++ b/.github/workflows/labeler.yml
@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Intentionally has a general name.
+# because the test status check created in GitHub Actions
+# currently randomly picks any associated workflow.
+# So, the name was changed to make sense in that context too.
+# See also https://github.community/t/specify-check-suite-when-creating-a-checkrun/118380/10
+
+name: "On pull requests"
+on: pull_request_target
+
+jobs:
+  label:
+    name: Label pull requests
+    runs-on: ubuntu-latest
+    steps:
+    # In order to get back the negated matches like in the old config,
+    # we need the actinons/labeler concept of `all` and `any` which matches
+    # all of the given constraints / glob patterns for either `all`
+    # files or `any` file in the change set.
+    #
+    # Github issue which requests a timeline for a release with any/all support:
+    #     - https://github.com/actions/labeler/issues/111
+    # This issue also references the issue that mentioned that any/all are only
+    # supported on main branch (previously called master):
+    #    - https://github.com/actions/labeler/issues/73#issuecomment-639034278
+    #
+    # However, these are not in a published release and the current `main` branch
+    # has some issues upon testing.
+    - uses: actions/labeler@2.2.0
+      with:
+        repo-token: "${{ secrets.GITHUB_TOKEN }}"
+        sync-labels: true
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
deleted file mode 100644
index 4282504cc3984..0000000000000
--- a/.github/workflows/master.yml
+++ /dev/null
@@ -1,156 +0,0 @@
-name: master
-
-on:
-  push:
-    branches:
-    - master
-  pull_request:
-    branches:
-    - master
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        java: [ '1.8', '11' ]
-        hadoop: [ 'hadoop-2.7', 'hadoop-3.2' ]
-        hive: [ 'hive-1.2', 'hive-2.3' ]
-        exclude:
-        - java: '11'
-          hive: 'hive-1.2'
-        - hadoop: 'hadoop-3.2'
-          hive: 'hive-1.2'
-    name: Build Spark - JDK${{ matrix.java }}/${{ matrix.hadoop }}/${{ matrix.hive }}
-
-    steps:
-    - uses: actions/checkout@master
-    # We split caches because GitHub Action Cache has a 400MB-size limit.
-    - uses: actions/cache@v1
-      with:
-        path: build
-        key: build-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          build-
-    - uses: actions/cache@v1
-      with:
-        path: ~/.m2/repository/com
-        key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-com-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          ${{ matrix.java }}-${{ matrix.hadoop }}-maven-com-
-    - uses: actions/cache@v1
-      with:
-        path: ~/.m2/repository/org
-        key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-org-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          ${{ matrix.java }}-${{ matrix.hadoop }}-maven-org-
-    - uses: actions/cache@v1
-      with:
-        path: ~/.m2/repository/net
-        key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-net-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          ${{ matrix.java }}-${{ matrix.hadoop }}-maven-net-
-    - uses: actions/cache@v1
-      with:
-        path: ~/.m2/repository/io
-        key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-io-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          ${{ matrix.java }}-${{ matrix.hadoop }}-maven-io-
-    - name: Set up JDK ${{ matrix.java }}
-      uses: actions/setup-java@v1
-      with:
-        java-version: ${{ matrix.java }}
-    - name: Build with Maven
-      run: |
-        export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
-        export MAVEN_CLI_OPTS="--no-transfer-progress"
-        mkdir -p ~/.m2
-        ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -P${{ matrix.hive }} -Phive-thriftserver -P${{ matrix.hadoop }} -Phadoop-cloud -Djava.version=${{ matrix.java }} install
-        rm -rf ~/.m2/repository/org/apache/spark
-
-
-  lint:
-    runs-on: ubuntu-latest
-    name: Linters (Java/Scala/Python), licenses, dependencies
-    steps:
-    - uses: actions/checkout@master
-    - uses: actions/setup-java@v1
-      with:
-        java-version: '11'
-    - uses: actions/setup-python@v1
-      with:
-        python-version: '3.x'
-        architecture: 'x64'
-    - name: Scala
-      run: ./dev/lint-scala
-    - name: Java
-      run: ./dev/lint-java
-    - name: Python
-      run: |
-        pip install flake8 sphinx numpy
-        ./dev/lint-python
-    - name: License
-      run: ./dev/check-license
-    - name: Dependencies
-      run: ./dev/test-dependencies.sh
-
-  lintr:
-    runs-on: ubuntu-latest
-    name: Linter (R)
-    steps:
-    - uses: actions/checkout@master
-    - uses: actions/setup-java@v1
-      with:
-        java-version: '11'
-    - uses: r-lib/actions/setup-r@v1
-      with:
-        r-version: '3.6.2'
-    - name: Install lib
-      run: |
-        sudo apt-get install -y libcurl4-openssl-dev
-    - name: install R packages
-      run: |
-        sudo Rscript -e "install.packages(c('curl', 'xml2', 'httr', 'devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2', 'e1071', 'survival'), repos='https://cloud.r-project.org/')"
-        sudo Rscript -e "devtools::install_github('jimhester/lintr@v2.0.0')"
-    - name: package and install SparkR
-      run: ./R/install-dev.sh
-    - name: lint-r
-      run: ./dev/lint-r
-
-  docs:
-    runs-on: ubuntu-latest
-    name: Generate documents
-    steps:
-    - uses: actions/checkout@master
-    - uses: actions/cache@v1
-      with:
-        path: ~/.m2/repository
-        key: docs-maven-repo-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          docs-maven-repo-
-    - uses: actions/setup-java@v1
-      with:
-        java-version: '1.8'
-    - uses: actions/setup-python@v1
-      with:
-        python-version: '3.x'
-        architecture: 'x64'
-    - uses: actions/setup-ruby@v1
-      with:
-        ruby-version: '2.7'
-    - uses: r-lib/actions/setup-r@v1
-      with:
-        r-version: '3.6.2'
-    - name: Install lib and pandoc
-      run: |
-        sudo apt-get install -y libcurl4-openssl-dev pandoc
-    - name: Install packages
-      run: |
-        pip install sphinx mkdocs numpy
-        gem install jekyll jekyll-redirect-from rouge
-        sudo Rscript -e "install.packages(c('curl', 'xml2', 'httr', 'devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2', 'e1071', 'survival'), repos='https://cloud.r-project.org/')"
-    - name: Run jekyll build
-      run: |
-        cd docs
-        jekyll build
diff --git a/.github/workflows/notify_test_workflow.yml b/.github/workflows/notify_test_workflow.yml
new file mode 100644
index 0000000000000..cc2b7a254e3f5
--- /dev/null
+++ b/.github/workflows/notify_test_workflow.yml
@@ -0,0 +1,125 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Intentionally has a general name.
+# because the test status check created in GitHub Actions
+# currently randomly picks any associated workflow.
+# So, the name was changed to make sense in that context too.
+# See also https://github.community/t/specify-check-suite-when-creating-a-checkrun/118380/10
+name: On pull request update
+on:
+  pull_request_target:
+    types: [opened, reopened, synchronize]
+
+jobs:
+  notify:
+    name: Notify test workflow
+    runs-on: ubuntu-20.04
+    steps:
+      - name: "Notify test workflow"
+        uses: actions/github-script@v3
+        if: ${{ github.base_ref == 'master' }}
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const endpoint = 'GET /repos/:owner/:repo/actions/workflows/:id/runs?&branch=:branch'
+
+            // TODO: Should use pull_request.user and pull_request.user.repos_url?
+            // If a different person creates a commit to another forked repo,
+            // it wouldn't be able to detect.
+            const params = {
+              owner: context.payload.pull_request.head.repo.owner.login,
+              repo: context.payload.pull_request.head.repo.name,
+              id: 'build_and_test.yml',
+              branch: context.payload.pull_request.head.ref,
+            }
+
+            console.log('Ref: ' + context.payload.pull_request.head.ref)
+            console.log('SHA: ' + context.payload.pull_request.head.sha)
+
+            // Wait 3 seconds to make sure the fork repository triggered a workflow.
+            await new Promise(r => setTimeout(r, 3000))
+
+            let runs
+            try {
+              runs = await github.request(endpoint, params)
+            } catch (error) {
+              console.error(error)
+              // Assume that runs were not found.
+            }
+
+            const name = 'Build and test'
+            const head_sha = context.payload.pull_request.head.sha
+            let status = 'queued'
+
+            if (!runs || runs.data.workflow_runs.length === 0) {
+              status = 'completed'
+              const conclusion = 'action_required'
+
+              github.checks.create({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                name: name,
+                head_sha: head_sha,
+                status: status,
+                conclusion: conclusion,
+                output: {
+                  title: 'Workflow run detection failed',
+                  summary: `
+            Unable to detect the workflow run for testing the changes in your PR.
+
+            1. If you did not enable GitHub Actions in your forked repository, please enable it. See also [Disabling or limiting GitHub Actions for a repository](https://docs.github.com/en/github/administering-a-repository/disabling-or-limiting-github-actions-for-a-repository) for more details.
+            2. It is possible your branch is based on the old \`master\` branch in Apache Spark, please sync your branch to the latest master branch. For example as below:
+                \`\`\`bash
+                git fetch upstream
+                git rebase upstream/master
+                git push origin YOUR_BRANCH --force
+                \`\`\``
+                }
+              })
+            } else {
+              const runID = runs.data.workflow_runs[0].id
+
+              if (runs.data.workflow_runs[0].head_sha != context.payload.pull_request.head.sha) {
+                throw new Error('There was a new unsynced commit pushed. Please retrigger the workflow.');
+              }
+
+              const runUrl = 'https://github.com/'
+                + context.payload.pull_request.head.repo.full_name
+                + '/actions/runs/'
+                + runID
+
+              github.checks.create({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                name: name,
+                head_sha: head_sha,
+                status: status,
+                output: {
+                  title: 'Test results',
+                  summary: '[See test results](' + runUrl + ')',
+                  text: JSON.stringify({
+                    owner: context.payload.pull_request.head.repo.owner.login,
+                    repo: context.payload.pull_request.head.repo.name,
+                    run_id: runID
+                  })
+                },
+                details_url: runUrl,
+              })
+            }
diff --git a/.github/workflows/publish_snapshot.yml b/.github/workflows/publish_snapshot.yml
new file mode 100644
index 0000000000000..c5dbc8d057964
--- /dev/null
+++ b/.github/workflows/publish_snapshot.yml
@@ -0,0 +1,40 @@
+name: Publish Snapshot
+
+on:
+  schedule:
+  - cron: '0 0 * * *'
+
+jobs:
+  publish-snapshot:
+    if: github.repository == 'apache/spark'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        branch:
+          - master
+          - branch-3.1
+    steps:
+    - name: Checkout Spark repository
+      uses: actions/checkout@master
+      with:
+        ref: ${{ matrix.branch }}
+    - name: Cache Maven local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.m2/repository
+        key: snapshot-maven-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          snapshot-maven-
+    - name: Install Java 8
+      uses: actions/setup-java@v1
+      with:
+        java-version: 8
+    - name: Publish snapshot
+      env:
+        ASF_USERNAME: ${{ secrets.NEXUS_USER }}
+        ASF_PASSWORD: ${{ secrets.NEXUS_PW }}
+        GPG_KEY: "not_used"
+        GPG_PASSPHRASE: "not_used"
+        GIT_REF: ${{ matrix.branch }}
+      run: ./dev/create-release/release-build.sh publish-snapshot
diff --git a/.github/workflows/test_report.yml b/.github/workflows/test_report.yml
new file mode 100644
index 0000000000000..6d70f29b2efa2
--- /dev/null
+++ b/.github/workflows/test_report.yml
@@ -0,0 +1,25 @@
+name: Report test results
+on:
+  workflow_run:
+    workflows: ["Build and test"]
+    types:
+      - completed
+
+jobs:
+  test_report:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Download test results to report
+      uses: dawidd6/action-download-artifact@v2
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        workflow: ${{ github.event.workflow_run.workflow_id }}
+        commit: ${{ github.event.workflow_run.head_commit.id }}
+        workflow_conclusion: completed
+    - name: Publish test report
+      uses: scacap/action-surefire-report@v1
+      with:
+        check_name: Report test results
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        report_paths: "**/target/test-reports/*.xml"
+        commit: ${{ github.event.workflow_run.head_commit.id }}
diff --git a/.github/workflows/update_build_status.yml b/.github/workflows/update_build_status.yml
new file mode 100644
index 0000000000000..16fbe3a6f66c7
--- /dev/null
+++ b/.github/workflows/update_build_status.yml
@@ -0,0 +1,97 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Update build status workflow
+
+on:
+  schedule:
+  - cron: "*/15 * * * *"
+
+jobs:
+  update:
+    name: Update build status
+    runs-on: ubuntu-20.04
+    steps:
+      - name: "Update build status"
+        uses: actions/github-script@v3
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const endpoint = 'GET /repos/:owner/:repo/pulls?state=:state'
+            const params = {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              state: 'open'
+            }
+
+            // See https://docs.github.com/en/graphql/reference/enums#mergestatestatus
+            const maybeReady = ['behind', 'clean', 'draft', 'has_hooks', 'unknown', 'unstable'];
+
+            // Iterate open PRs
+            for await (const prs of github.paginate.iterator(endpoint,params)) {
+              // Each page
+              for await (const pr of prs.data) {
+                console.log('SHA: ' + pr.head.sha)
+                console.log('  Mergeable status: ' + pr.mergeable_state)
+                if (pr.mergeable_state == null || maybeReady.includes(pr.mergeable_state)) {
+                  const checkRuns = await github.request('GET /repos/{owner}/{repo}/commits/{ref}/check-runs', {
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    ref: pr.head.sha
+                  })
+
+                  // Iterator GitHub Checks in the PR
+                  for await (const cr of checkRuns.data.check_runs) {
+                    if (cr.name == 'Build and test' && cr.conclusion != "action_required") {
+                      // text contains parameters to make request in JSON.
+                      const params = JSON.parse(cr.output.text)
+
+                      // Get the workflow run in the forked repository
+                      const run = await github.request('GET /repos/{owner}/{repo}/actions/runs/{run_id}', params)
+
+                      // Keep syncing the status of the checks
+                      if (run.data.status == 'completed') {
+                        console.log('    Run ' + cr.id + ': set status (' + run.data.status + ') and conclusion (' + run.data.conclusion + ')')
+                        const response = await github.request('PATCH /repos/{owner}/{repo}/check-runs/{check_run_id}', {
+                          owner: context.repo.owner,
+                          repo: context.repo.repo,
+                          check_run_id: cr.id,
+                          output: cr.output,
+                          status: run.data.status,
+                          conclusion: run.data.conclusion,
+                          details_url: run.data.details_url
+                        })
+                      } else {
+                        console.log('    Run ' + cr.id + ': set status (' + run.data.status + ')')
+                        const response = await github.request('PATCH /repos/{owner}/{repo}/check-runs/{check_run_id}', {
+                          owner: context.repo.owner,
+                          repo: context.repo.repo,
+                          check_run_id: cr.id,
+                          output: cr.output,
+                          status: run.data.status,
+                          details_url: run.data.details_url
+                        })
+                      }
+
+                      break
+                    }
+                  }
+                }
+              }
+            }
diff --git a/.gitignore b/.gitignore
index 198fdee39be95..0411eb736c11b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,13 +8,17 @@
 *.swp
 *~
 .DS_Store
+.bsp/
 .cache
 .classpath
 .ensime
 .ensime_cache/
 .ensime_lucene
 .generated-mima*
-.idea/
+# The star is required for further !.idea/ to work, see https://git-scm.com/docs/gitignore
+.idea/*
+!.idea/vcs.xml
+python/.idea
 .idea_modules/
 .project
 .pydevproject
@@ -29,7 +33,6 @@ R/pkg/tests/fulltests/Rplots.pdf
 build/*.jar
 build/apache-maven*
 build/scala*
-build/zinc*
 cache
 checkpoint
 conf/*.cmd
@@ -47,10 +50,12 @@ dev/pr-deps/
 dist/
 docs/_site/
 docs/api
+docs/.local_ruby_bundle
 sql/docs
 sql/site
 lib_managed/
 lint-r-report.log
+lint-js-report.log
 log/
 logs/
 out/
@@ -64,9 +69,11 @@ python/lib/pyspark.zip
 python/.eggs/
 python/deps
 python/docs/_site/
+python/docs/source/reference/api/
 python/test_coverage/coverage_data
 python/test_coverage/htmlcov
 python/pyspark/python
+.mypy_cache/
 reports/
 scalastyle-on-compile.generated.xml
 scalastyle-output.xml
@@ -79,6 +86,7 @@ target/
 unit-tests.log
 work/
 docs/.jekyll-metadata
+docs/.jekyll-cache
 
 # For Hive
 TempStatsStore/
@@ -99,3 +107,6 @@ spark-warehouse/
 
 # For SBT
 .jvmopts
+
+# For Node.js
+node_modules
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 0000000000000..28fd3fcdf10ea
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+-->
+<project version="4">
+  <component name="IssueNavigationConfiguration">
+    <option name="links">
+      <list>
+        <IssueNavigationLink>
+          <option name="issueRegexp" value="[A-Z]+\-\d+" />
+          <option name="linkRegexp" value="https://issues.apache.org/jira/browse/$0" />
+        </IssueNavigationLink>
+        <IssueNavigationLink>
+          <option name="issueRegexp" value="#(\d+)" />
+          <option name="linkRegexp" value="https://github.com/apache/spark/pull/$1" />
+        </IssueNavigationLink>
+      </list>
+    </option>
+  </component>
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>
diff --git a/LICENSE b/LICENSE
index 6b169b1447f14..df6bed16f4471 100644
--- a/LICENSE
+++ b/LICENSE
@@ -222,14 +222,13 @@ external/spark-ganglia-lgpl/src/main/java/com/codahale/metrics/ganglia/GangliaRe
 Python Software Foundation License
 ----------------------------------
 
-pyspark/heapq3.py
-python/docs/_static/copybutton.js
+python/docs/source/_static/copybutton.js
 
 BSD 3-Clause
 ------------
 
 python/lib/py4j-*-src.zip
-python/pyspark/cloudpickle.py
+python/pyspark/cloudpickle/*.py
 python/pyspark/join.py
 core/src/main/resources/org/apache/spark/ui/static/d3.min.js
 
diff --git a/LICENSE-binary b/LICENSE-binary
index b50da6be4e697..32c7fc315d1a3 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -218,7 +218,6 @@ javax.jdo:jdo-api
 joda-time:joda-time
 net.sf.opencsv:opencsv
 org.apache.derby:derby
-org.ehcache:ehcache
 org.objenesis:objenesis
 org.roaringbitmap:RoaringBitmap
 org.scalanlp:breeze-macros_2.12
@@ -261,7 +260,6 @@ net.sf.supercsv:super-csv
 org.apache.arrow:arrow-format
 org.apache.arrow:arrow-memory
 org.apache.arrow:arrow-vector
-org.apache.commons:commons-configuration2
 org.apache.commons:commons-crypto
 org.apache.commons:commons-lang3
 org.apache.hadoop:hadoop-annotations
@@ -296,7 +294,6 @@ org.apache.kerby:kerby-config
 org.apache.kerby:kerby-pkix
 org.apache.kerby:kerby-util
 org.apache.kerby:kerby-xdr
-org.apache.kerby:token-provider
 org.apache.orc:orc-core
 org.apache.orc:orc-mapreduce
 org.mortbay.jetty:jetty
@@ -316,19 +313,15 @@ com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider
 com.fasterxml.jackson.module:jackson-module-jaxb-annotations
 com.fasterxml.jackson.module:jackson-module-paranamer
 com.fasterxml.jackson.module:jackson-module-scala_2.12
-com.fasterxml.woodstox:woodstox-core
 com.github.mifmif:generex
-com.github.stephenc.jcip:jcip-annotations
 com.google.code.findbugs:jsr305
 com.google.code.gson:gson
 com.google.flatbuffers:flatbuffers-java
 com.google.guava:guava
 com.google.inject:guice
 com.google.inject.extensions:guice-servlet
-com.nimbusds:nimbus-jose-jwt
 com.twitter:parquet-hadoop-bundle
 commons-cli:commons-cli
-commons-daemon:commons-daemon
 commons-dbcp:commons-dbcp
 commons-io:commons-io
 commons-lang:commons-lang
@@ -340,8 +333,6 @@ javax.inject:javax.inject
 javax.validation:validation-api
 log4j:apache-log4j-extras
 log4j:log4j
-net.minidev:accessors-smart
-net.minidev:json-smart
 net.sf.jpam:jpam
 org.apache.avro:avro
 org.apache.avro:avro-ipc
@@ -357,7 +348,6 @@ org.apache.directory.server:apacheds-i18n
 org.apache.directory.server:apacheds-kerberos-codec
 org.apache.htrace:htrace-core
 org.apache.ivy:ivy
-org.apache.geronimo.specs:geronimo-jcache_1.0_spec
 org.apache.mesos:mesos
 org.apache.parquet:parquet-column
 org.apache.parquet:parquet-common
@@ -432,7 +422,6 @@ BSD 2-Clause
 ------------
 
 com.github.luben:zstd-jni
-dnsjava:dnsjava
 javolution:javolution
 com.esotericsoftware:kryo-shaded
 com.esotericsoftware:minlog
@@ -440,7 +429,6 @@ com.esotericsoftware:reflectasm
 com.google.protobuf:protobuf-java
 org.codehaus.janino:commons-compiler
 org.codehaus.janino:janino
-org.codehaus.woodstox:stax2-api
 jline:jline
 org.jodd:jodd-core
 com.github.wendykierp:JTransforms
@@ -456,8 +444,6 @@ org.antlr:ST4
 org.antlr:stringtemplate
 org.antlr:antlr4-runtime
 antlr:antlr
-com.github.fommil.netlib:core
-com.google.re2j:re2j
 com.thoughtworks.paranamer:paranamer
 org.scala-lang:scala-compiler
 org.scala-lang:scala-library
@@ -498,6 +484,9 @@ org.slf4j:jul-to-slf4j
 org.slf4j:slf4j-api
 org.slf4j:slf4j-log4j12
 com.github.scopt:scopt_2.12
+dev.ludovic.netlib:blas
+dev.ludovic.netlib:arpack
+dev.ludovic.netlib:lapack
 
 core/src/main/resources/org/apache/spark/ui/static/dagre-d3.min.js
 core/src/main/resources/org/apache/spark/ui/static/*dataTables*
@@ -521,7 +510,6 @@ Common Development and Distribution License (CDDL) 1.1
 ------------------------------------------------------
 
 javax.el:javax.el-api	https://javaee.github.io/uel-ri/
-javax.servlet:javax.servlet-api   https://javaee.github.io/servlet-spec/
 javax.servlet.jsp:jsp-api
 javax.transaction:jta http://www.oracle.com/technetwork/java/index.html
 javax.xml.bind:jaxb-api    https://github.com/javaee/jaxb-v2
@@ -553,16 +541,11 @@ Eclipse Public License (EPL) 2.0
 --------------------------------
 
 jakarta.annotation:jakarta-annotation-api https://projects.eclipse.org/projects/ee4j.ca
+jakarta.servlet:jakarta.servlet-api https://projects.eclipse.org/projects/ee4j.servlet
 jakarta.ws.rs:jakarta.ws.rs-api https://github.com/eclipse-ee4j/jaxrs-api
 org.glassfish.hk2.external:jakarta.inject
 
 
-Python Software Foundation License
-----------------------------------
-
-pyspark/heapq3.py
-
-
 Public Domain
 -------------
 
diff --git a/R/CRAN_RELEASE.md b/R/CRAN_RELEASE.md
index 4d9b6416c01cb..2f410cf8bfd94 100644
--- a/R/CRAN_RELEASE.md
+++ b/R/CRAN_RELEASE.md
@@ -25,7 +25,7 @@ To release SparkR as a package to CRAN, we would use the `devtools` package. Ple
 
 First, check that the `Version:` field in the `pkg/DESCRIPTION` file is updated. Also, check for stale files not under source control.
 
-Note that while `run-tests.sh` runs `check-cran.sh` (which runs `R CMD check`), it is doing so with `--no-manual --no-vignettes`, which skips a few vignettes or PDF checks - therefore it will be preferred to run `R CMD check` on the source package built manually before uploading a release. Also note that for CRAN checks for pdf vignettes to success, `qpdf` tool must be there (to install it, eg. `yum -q -y install qpdf`).
+Note that while `run-tests.sh` runs `check-cran.sh` (which runs `R CMD check`), it is doing so with `--no-manual --no-vignettes`, which skips a few vignettes or PDF checks - therefore it will be preferred to run `R CMD check` on the source package built manually before uploading a release. Also note that for CRAN checks for pdf vignettes to success, `qpdf` tool must be there (to install it, e.g. `yum -q -y install qpdf`).
 
 To upload a release, we would need to update the `cran-comments.md`. This should generally contain the results from running the `check-cran.sh` script along with comments on status of all `WARNING` (should not be any) or `NOTE`. As a part of `check-cran.sh` and the release process, the vignettes is build - make sure `SPARK_HOME` is set and Spark jars are accessible.
 
diff --git a/R/DOCUMENTATION.md b/R/DOCUMENTATION.md
index fd0c7644189db..d4ff3f5f4875b 100644
--- a/R/DOCUMENTATION.md
+++ b/R/DOCUMENTATION.md
@@ -19,7 +19,7 @@ license: |
 # SparkR Documentation
 
 SparkR documentation is generated by using in-source comments and annotated by using
-[`roxygen2`](https://cran.r-project.org/web/packages/roxygen2/index.html). After making changes to the documentation and generating man pages,
+[`roxygen2`](https://cran.r-project.org/package=roxygen2). After making changes to the documentation and generating man pages,
 you can run the following from an R console in the SparkR home directory
 ```R
 library(devtools)
diff --git a/R/README.md b/R/README.md
index 31174c73526f2..da9f042b4fded 100644
--- a/R/README.md
+++ b/R/README.md
@@ -17,10 +17,14 @@ export R_HOME=/home/username/R
 
 #### Build Spark
 
-Build Spark with [Maven](https://spark.apache.org/docs/latest/building-spark.html#buildmvn) and include the `-Psparkr` profile to build the R package. For example to use the default Hadoop versions you can run
+Build Spark with [Maven](https://spark.apache.org/docs/latest/building-spark.html#buildmvn) or [SBT](https://spark.apache.org/docs/latest/building-spark.html#building-with-sbt), and include the `-Psparkr` profile to build the R package. For example to use the default Hadoop versions you can run
 
 ```bash
+# Maven
 ./build/mvn -DskipTests -Psparkr package
+
+# SBT
+./build/sbt -Psparkr package
 ```
 
 #### Running sparkR
diff --git a/R/WINDOWS.md b/R/WINDOWS.md
index dbc27178bdb8c..9fe4a22bf22b2 100644
--- a/R/WINDOWS.md
+++ b/R/WINDOWS.md
@@ -22,8 +22,8 @@ To build SparkR on Windows, the following steps are required
 
 1. Make sure `bash` is available and in `PATH` if you already have a built-in `bash` on Windows. If you do not have, install [Cygwin](https://www.cygwin.com/).
 
-2. Install R (>= 3.1) and [Rtools](https://cloud.r-project.org/bin/windows/Rtools/). Make sure to
-include Rtools and R in `PATH`. Note that support for R prior to version 3.4 is deprecated as of Spark 3.0.0.
+2. Install R (>= 3.5) and [Rtools](https://cloud.r-project.org/bin/windows/Rtools/). Make sure to
+include Rtools and R in `PATH`.
 
 3. Install JDK that SparkR supports (see `R/pkg/DESCRIPTION`), and set `JAVA_HOME` in the system environment variables.
 
diff --git a/R/install-dev.bat b/R/install-dev.bat
index ed1c91ae3a0ff..ae5aa589a19d1 100644
--- a/R/install-dev.bat
+++ b/R/install-dev.bat
@@ -24,7 +24,13 @@ set SPARK_HOME=%~dp0..
 
 MKDIR %SPARK_HOME%\R\lib
 
-R.exe CMD INSTALL --library="%SPARK_HOME%\R\lib"  %SPARK_HOME%\R\pkg\
+rem When you pass the package path directly as an argument to R CMD INSTALL,
+rem it takes the path as 'C:\projects\spark\R\..\R\pkg"' as an example at
+rem R 4.0. To work around this, directly go to the directory and install it.
+rem See also SPARK-32074
+pushd %SPARK_HOME%\R\pkg\
+R.exe CMD INSTALL --library="%SPARK_HOME%\R\lib" .
+popd
 
 rem Zip the SparkR package so that it can be distributed to worker nodes on YARN
 pushd %SPARK_HOME%\R\lib
diff --git a/R/pkg/DESCRIPTION b/R/pkg/DESCRIPTION
index 52d7e1f4daa53..f55286da25de1 100644
--- a/R/pkg/DESCRIPTION
+++ b/R/pkg/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: SparkR
 Type: Package
-Version: 3.1.0
+Version: 3.2.0
 Title: R Front End for 'Apache Spark'
 Description: Provides an R Front end for 'Apache Spark' <https://spark.apache.org>.
 Authors@R: c(person("Shivaram", "Venkataraman", role = c("aut", "cre"),
@@ -11,19 +11,20 @@ Authors@R: c(person("Shivaram", "Venkataraman", role = c("aut", "cre"),
                     email = "felixcheung@apache.org"),
              person(family = "The Apache Software Foundation", role = c("aut", "cph")))
 License: Apache License (== 2.0)
-URL: https://www.apache.org/ https://spark.apache.org/
+URL: https://www.apache.org https://spark.apache.org
 BugReports: https://spark.apache.org/contributing.html
 SystemRequirements: Java (>= 8, < 12)
 Depends:
-    R (>= 3.1),
+    R (>= 3.5),
     methods
 Suggests:
     knitr,
     rmarkdown,
+    markdown,
     testthat,
     e1071,
     survival,
-    arrow (>= 0.15.1)
+    arrow (>= 1.0.0)
 Collate:
     'schema.R'
     'generics.R'
@@ -59,7 +60,7 @@ Collate:
     'types.R'
     'utils.R'
     'window.R'
-RoxygenNote: 5.0.1
+RoxygenNote: 7.1.1
 VignetteBuilder: knitr
 NeedsCompilation: no
 Encoding: UTF-8
diff --git a/R/pkg/NAMESPACE b/R/pkg/NAMESPACE
index 53a0b7856567e..1f0b69db65151 100644
--- a/R/pkg/NAMESPACE
+++ b/R/pkg/NAMESPACE
@@ -202,6 +202,7 @@ exportMethods("%<=>%",
               "%in%",
               "abs",
               "acos",
+              "acosh",
               "add_months",
               "alias",
               "approx_count_distinct",
@@ -222,20 +223,27 @@ exportMethods("%<=>%",
               "array_remove",
               "array_repeat",
               "array_sort",
+              "array_to_vector",
               "array_transform",
               "arrays_overlap",
               "array_union",
               "arrays_zip",
               "arrays_zip_with",
               "asc",
+              "asc_nulls_first",
+              "asc_nulls_last",
               "ascii",
               "asin",
+              "asinh",
+              "assert_true",
               "atan",
+              "atanh",
               "atan2",
               "avg",
               "base64",
               "between",
               "bin",
+              "bitwise_not",
               "bitwiseNOT",
               "bround",
               "cast",
@@ -252,6 +260,7 @@ exportMethods("%<=>%",
               "cos",
               "cosh",
               "count",
+              "count_distinct",
               "countDistinct",
               "crc32",
               "create_array",
@@ -272,6 +281,9 @@ exportMethods("%<=>%",
               "degrees",
               "dense_rank",
               "desc",
+              "desc_nulls_first",
+              "desc_nulls_last",
+              "dropFields",
               "element_at",
               "encode",
               "endsWith",
@@ -286,6 +298,7 @@ exportMethods("%<=>%",
               "floor",
               "format_number",
               "format_string",
+              "from_avro",
               "from_csv",
               "from_json",
               "from_unixtime",
@@ -348,6 +361,7 @@ exportMethods("%<=>%",
               "negate",
               "next_day",
               "not",
+              "nth_value",
               "ntile",
               "otherwise",
               "over",
@@ -357,8 +371,10 @@ exportMethods("%<=>%",
               "pmod",
               "posexplode",
               "posexplode_outer",
+              "product",
               "quarter",
               "radians",
+              "raise_error",
               "rand",
               "randn",
               "rank",
@@ -378,8 +394,11 @@ exportMethods("%<=>%",
               "sha1",
               "sha2",
               "shiftLeft",
+              "shiftleft",
               "shiftRight",
+              "shiftright",
               "shiftRightUnsigned",
+              "shiftrightunsigned",
               "shuffle",
               "sd",
               "sign",
@@ -402,11 +421,14 @@ exportMethods("%<=>%",
               "substr",
               "substring_index",
               "sum",
+              "sum_distinct",
               "sumDistinct",
               "tan",
               "tanh",
+              "timestamp_seconds",
               "toDegrees",
               "toRadians",
+              "to_avro",
               "to_csv",
               "to_date",
               "to_json",
@@ -425,9 +447,11 @@ exportMethods("%<=>%",
               "variance",
               "var_pop",
               "var_samp",
+              "vector_to_array",
               "weekofyear",
               "when",
               "window",
+              "withField",
               "xxhash64",
               "year")
 
diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R
index 15b3ce2935427..72d96151f6371 100644
--- a/R/pkg/R/DataFrame.R
+++ b/R/pkg/R/DataFrame.R
@@ -880,7 +880,7 @@ setMethod("toJSON",
 
 #' Save the contents of SparkDataFrame as a JSON file
 #'
-#' Save the contents of a SparkDataFrame as a JSON file (\href{http://jsonlines.org/}{
+#' Save the contents of a SparkDataFrame as a JSON file (\href{https://jsonlines.org/}{
 #' JSON Lines text format or newline-delimited JSON}). Files written out
 #' with this method can be read back in as a SparkDataFrame using read.json().
 #'
@@ -1233,14 +1233,10 @@ setMethod("collect",
                   port = port, blocking = TRUE, open = "wb", timeout = connectionTimeout)
                 output <- tryCatch({
                   doServerAuth(conn, authSecret)
-                  arrowTable <- arrow::read_arrow(readRaw(conn))
-                  # Arrow drops `as_tibble` since 0.14.0, see ARROW-5190.
-                  if (exists("as_tibble", envir = asNamespace("arrow"))) {
-                    as.data.frame(arrow::as_tibble(arrowTable), stringsAsFactors = stringsAsFactors)
-                  } else {
-                    as.data.frame(arrowTable, stringsAsFactors = stringsAsFactors)
-                  }
-                }, finally = {
+                  arrowTable <- arrow::read_ipc_stream(readRaw(conn))
+                  as.data.frame(arrowTable, stringsAsFactors = stringsAsFactors)
+                },
+                finally = {
                   close(conn)
                 })
                 return(output)
@@ -2281,16 +2277,17 @@ setMethod("mutate",
 
             # For named arguments, use the names for arguments as the column names
             # For unnamed arguments, use the argument symbols as the column names
-            args <- sapply(substitute(list(...))[-1], deparse)
             ns <- names(cols)
-            if (!is.null(ns)) {
-              lapply(seq_along(args), function(i) {
-                if (ns[[i]] != "") {
-                  args[[i]] <<- ns[[i]]
-                }
+            if (is.null(ns)) ns <- rep("", length(cols))
+            named_idx <- nzchar(ns)
+            if (!all(named_idx)) {
+              # SPARK-31517: deparse uses width.cutoff on wide input and the
+              #   output is length>1, so need to collapse it to scalar
+              colsub <- substitute(list(...))[-1L]
+              ns[!named_idx] <- sapply(which(!named_idx), function(ii) {
+                paste(gsub("^\\s*|\\s*$", "", deparse(colsub[[ii]])), collapse = " ")
               })
             }
-            ns <- args
 
             # The last column of the same name in the specific columns takes effect
             deDupCols <- list()
@@ -2776,7 +2773,7 @@ setMethod("merge",
 #' Creates a list of columns by replacing the intersected ones with aliases
 #'
 #' Creates a list of columns by replacing the intersected ones with aliases.
-#' The name of the alias column is formed by concatanating the original column name and a suffix.
+#' The name of the alias column is formed by concatenating the original column name and a suffix.
 #'
 #' @param x a SparkDataFrame
 #' @param intersectedColNames a list of intersected column names of the SparkDataFrame
@@ -2867,11 +2864,18 @@ setMethod("unionAll",
 #' \code{UNION ALL} and \code{UNION DISTINCT} in SQL as column positions are not taken
 #' into account. Input SparkDataFrames can have different data types in the schema.
 #'
+#' When the parameter allowMissingColumns is `TRUE`, the set of column names
+#' in x and y can differ; missing columns will be filled as null.
+#' Further, the missing columns of x will be added at the end
+#' in the schema of the union result.
+#'
 #' Note: This does not remove duplicate rows across the two SparkDataFrames.
 #' This function resolves columns by name (not by position).
 #'
 #' @param x A SparkDataFrame
 #' @param y A SparkDataFrame
+#' @param allowMissingColumns logical
+#' @param ... further arguments to be passed to or from other methods.
 #' @return A SparkDataFrame containing the result of the union.
 #' @family SparkDataFrame functions
 #' @rdname unionByName
@@ -2884,12 +2888,15 @@ setMethod("unionAll",
 #' df1 <- select(createDataFrame(mtcars), "carb", "am", "gear")
 #' df2 <- select(createDataFrame(mtcars), "am", "gear", "carb")
 #' head(unionByName(df1, df2))
+#'
+#' df3 <- select(createDataFrame(mtcars), "carb")
+#' head(unionByName(df1, df3, allowMissingColumns = TRUE))
 #' }
 #' @note unionByName since 2.3.0
 setMethod("unionByName",
           signature(x = "SparkDataFrame", y = "SparkDataFrame"),
-          function(x, y) {
-            unioned <- callJMethod(x@sdf, "unionByName", y@sdf)
+          function(x, y, allowMissingColumns=FALSE) {
+            unioned <- callJMethod(x@sdf, "unionByName", y@sdf, allowMissingColumns)
             dataFrame(unioned)
           })
 
@@ -3225,7 +3232,7 @@ setMethod("describe",
 #' \item stddev
 #' \item min
 #' \item max
-#' \item arbitrary approximate percentiles specified as a percentage (eg, "75\%")
+#' \item arbitrary approximate percentiles specified as a percentage (e.g., "75\%")
 #' }
 #' If no statistics are given, this function computes count, mean, stddev, min,
 #' approximate quartiles (percentiles at 25\%, 50\%, and 75\%), and max.
@@ -3438,7 +3445,8 @@ setMethod("as.data.frame",
 #' @note attach since 1.6.0
 setMethod("attach",
           signature(what = "SparkDataFrame"),
-          function(what, pos = 2L, name = deparse(substitute(what), backtick = FALSE),
+          function(what, pos = 2L,
+                   name = paste(deparse(substitute(what), backtick = FALSE), collapse = " "),
                    warn.conflicts = TRUE) {
             args <- as.list(environment()) # capture all parameters - this must be the first line
             newEnv <- assignNewEnv(args$what)
@@ -3737,7 +3745,7 @@ setMethod("histogram",
 #'
 #' @param x a SparkDataFrame.
 #' @param url JDBC database url of the form \code{jdbc:subprotocol:subname}.
-#' @param tableName yhe name of the table in the external database.
+#' @param tableName the name of the table in the external database.
 #' @param mode one of 'append', 'overwrite', 'error', 'errorifexists', 'ignore'
 #'             save mode (it is 'error' by default)
 #' @param ... additional JDBC database connection properties.
diff --git a/R/pkg/R/RDD.R b/R/pkg/R/RDD.R
index 7a1d157bb8a36..408a3ff25b2b2 100644
--- a/R/pkg/R/RDD.R
+++ b/R/pkg/R/RDD.R
@@ -970,7 +970,7 @@ setMethod("takeSample", signature(x = "RDD", withReplacement = "logical",
                                                                   MAXINT)))))
             # If the first sample didn't turn out large enough, keep trying to
             # take samples; this shouldn't happen often because we use a big
-            # multiplier for thei initial size
+            # multiplier for the initial size
             while (length(samples) < total)
               samples <- collectRDD(sampleRDD(x, withReplacement, fraction,
                                            as.integer(ceiling(stats::runif(1,
@@ -1512,7 +1512,7 @@ setMethod("glom",
 #'
 #' @param x An RDD.
 #' @param y An RDD.
-#' @return a new RDD created by performing the simple union (witout removing
+#' @return a new RDD created by performing the simple union (without removing
 #' duplicates) of two input RDDs.
 #' @examples
 #'\dontrun{
diff --git a/R/pkg/R/SQLContext.R b/R/pkg/R/SQLContext.R
index c0ac68332ec41..14262e1a74ab0 100644
--- a/R/pkg/R/SQLContext.R
+++ b/R/pkg/R/SQLContext.R
@@ -203,7 +203,7 @@ getSchema <- function(schema, firstRow = NULL, rdd = NULL) {
       })
     }
 
-    # SPAKR-SQL does not support '.' in column name, so replace it with '_'
+    # SPARK-SQL does not support '.' in column name, so replace it with '_'
     # TODO(davies): remove this once SPARK-2775 is fixed
     names <- lapply(names, function(n) {
       nn <- gsub(".", "_", n, fixed = TRUE)
@@ -374,7 +374,7 @@ setMethod("toDF", signature(x = "RDD"),
 #' Create a SparkDataFrame from a JSON file.
 #'
 #' Loads a JSON file, returning the result as a SparkDataFrame
-#' By default, (\href{http://jsonlines.org/}{JSON Lines text format or newline-delimited JSON}
+#' By default, (\href{https://jsonlines.org/}{JSON Lines text format or newline-delimited JSON}
 #' ) is supported. For JSON (one record per file), set a named property \code{multiLine} to
 #' \code{TRUE}.
 #' It goes through the entire dataset once to determine the schema.
diff --git a/R/pkg/R/WindowSpec.R b/R/pkg/R/WindowSpec.R
index 037809cd0923e..be47d0117ed7f 100644
--- a/R/pkg/R/WindowSpec.R
+++ b/R/pkg/R/WindowSpec.R
@@ -54,7 +54,7 @@ setMethod("show", "WindowSpec",
 #' Defines the partitioning columns in a WindowSpec.
 #'
 #' @param x a WindowSpec.
-#' @param col a column to partition on (desribed by the name or Column).
+#' @param col a column to partition on (described by the name or Column).
 #' @param ... additional column(s) to partition on.
 #' @return A WindowSpec.
 #' @rdname partitionBy
@@ -231,7 +231,7 @@ setMethod("rangeBetween",
 #' @rdname over
 #' @name over
 #' @aliases over,Column,WindowSpec-method
-#' @family colum_func
+#' @family column_func
 #' @examples
 #' \dontrun{
 #'   df <- createDataFrame(mtcars)
diff --git a/R/pkg/R/column.R b/R/pkg/R/column.R
index 7926a9a2467ee..9fa117ccb6281 100644
--- a/R/pkg/R/column.R
+++ b/R/pkg/R/column.R
@@ -67,7 +67,11 @@ operators <- list(
   # we can not override `&&` and `||`, so use `&` and `|` instead
   "&" = "and", "|" = "or", "^" = "pow"
 )
-column_functions1 <- c("asc", "desc", "isNaN", "isNull", "isNotNull")
+column_functions1 <- c(
+  "asc", "asc_nulls_first", "asc_nulls_last",
+  "desc", "desc_nulls_first", "desc_nulls_last",
+  "isNaN", "isNull", "isNotNull"
+)
 column_functions2 <- c("like", "rlike", "getField", "getItem", "contains")
 
 createOperator <- function(op) {
@@ -131,7 +135,7 @@ createMethods()
 #' @rdname alias
 #' @name alias
 #' @aliases alias,Column-method
-#' @family colum_func
+#' @family column_func
 #' @examples
 #' \dontrun{
 #' df <- createDataFrame(iris)
@@ -157,7 +161,7 @@ setMethod("alias",
 #'
 #' @rdname substr
 #' @name substr
-#' @family colum_func
+#' @family column_func
 #' @aliases substr,Column-method
 #'
 #' @param x a Column.
@@ -183,7 +187,7 @@ setMethod("substr", signature(x = "Column"),
 #'
 #' @rdname startsWith
 #' @name startsWith
-#' @family colum_func
+#' @family column_func
 #' @aliases startsWith,Column-method
 #'
 #' @param x vector of character string whose "starts" are considered
@@ -202,7 +206,7 @@ setMethod("startsWith", signature(x = "Column"),
 #'
 #' @rdname endsWith
 #' @name endsWith
-#' @family colum_func
+#' @family column_func
 #' @aliases endsWith,Column-method
 #'
 #' @param x vector of character string whose "ends" are considered
@@ -220,7 +224,7 @@ setMethod("endsWith", signature(x = "Column"),
 #'
 #' @rdname between
 #' @name between
-#' @family colum_func
+#' @family column_func
 #' @aliases between,Column-method
 #'
 #' @param x a Column
@@ -247,7 +251,7 @@ setMethod("between", signature(x = "Column"),
 # nolint end
 #' @rdname cast
 #' @name cast
-#' @family colum_func
+#' @family column_func
 #' @aliases cast,Column-method
 #'
 #' @examples
@@ -296,7 +300,7 @@ setMethod("%in%",
 #'              Can be a single value or a Column.
 #' @rdname otherwise
 #' @name otherwise
-#' @family colum_func
+#' @family column_func
 #' @aliases otherwise,Column-method
 #' @note otherwise since 1.5.0
 setMethod("otherwise",
@@ -356,3 +360,103 @@ setMethod("%<=>%",
 #' }
 #' @note ! since 2.3.0
 setMethod("!", signature(x = "Column"), function(x) not(x))
+
+#' withField
+#'
+#' Adds/replaces field in a struct \code{Column} by name.
+#'
+#' @param x a Column
+#' @param fieldName a character
+#' @param col a Column expression
+#'
+#' @rdname withField
+#' @aliases withField withField,Column-method
+#' @examples
+#' \dontrun{
+#' df <- withColumn(
+#'   createDataFrame(iris),
+#'   "sepal",
+#'    struct(column("Sepal_Width"), column("Sepal_Length"))
+#' )
+#'
+#' head(select(
+#'   df,
+#'   withField(df$sepal, "product", df$Sepal_Length * df$Sepal_Width)
+#' ))
+#' }
+#' @note withField since 3.1.0
+setMethod("withField",
+          signature(x = "Column", fieldName = "character", col = "Column"),
+          function(x, fieldName, col) {
+            jc <- callJMethod(x@jc, "withField", fieldName, col@jc)
+            column(jc)
+          })
+
+#' dropFields
+#'
+#' Drops fields in a struct \code{Column} by name.
+#'
+#' @param x a Column
+#' @param ... names of the fields to be dropped.
+#'
+#' @rdname dropFields
+#' @aliases dropFields dropFields,Column-method
+#' @examples
+#' \dontrun{
+#' df <- select(
+#'   createDataFrame(iris),
+#'   alias(
+#'     struct(
+#'       column("Sepal_Width"), column("Sepal_Length"),
+#'       alias(
+#'         struct(
+#'           column("Petal_Width"), column("Petal_Length"),
+#'           alias(
+#'             column("Petal_Width") * column("Petal_Length"),
+#'             "Petal_Product"
+#'           )
+#'         ),
+#'         "Petal"
+#'       )
+#'     ),
+#'     "dimensions"
+#'   )
+#' )
+#' head(withColumn(df, "dimensions", dropFields(df$dimensions, "Petal")))
+#'
+#' head(
+#'   withColumn(
+#'     df, "dimensions",
+#'     dropFields(df$dimensions, "Sepal_Width", "Sepal_Length")
+#'   )
+#' )
+#'
+#' # This method supports dropping multiple nested fields directly e.g.
+#' head(
+#'   withColumn(
+#'     df, "dimensions",
+#'     dropFields(df$dimensions, "Petal.Petal_Width", "Petal.Petal_Length")
+#'   )
+#' )
+#'
+#' # However, if you are going to add/replace multiple nested fields,
+#' # it is preferred to extract out the nested struct before
+#' # adding/replacing multiple fields e.g.
+#' head(
+#'   withColumn(
+#'     df, "dimensions",
+#'     withField(
+#'       column("dimensions"),
+#'       "Petal",
+#'       dropFields(column("dimensions.Petal"), "Petal_Width", "Petal_Length")
+#'     )
+#'   )
+#' )
+#' }
+#' @note dropFields since 3.1.0
+setMethod("dropFields",
+          signature(x = "Column"),
+          function(x, ...) {
+            jc <- callJMethod(x@jc, "dropFields", list(...))
+            column(jc)
+          })
diff --git a/R/pkg/R/context.R b/R/pkg/R/context.R
index e3c9d9f8793d6..cca6c2c817de9 100644
--- a/R/pkg/R/context.R
+++ b/R/pkg/R/context.R
@@ -86,7 +86,7 @@ makeSplits <- function(numSerializedSlices, length) {
   # For instance, for numSerializedSlices of 22, length of 50
   #  [1]  0  0  2  2  4  4  6  6  6  9  9 11 11 13 13 15 15 15 18 18 20 20 22 22 22
   # [26] 25 25 27 27 29 29 31 31 31 34 34 36 36 38 38 40 40 40 43 43 45 45 47 47 47
-  # Notice the slice group with 3 slices (ie. 6, 15, 22) are roughly evenly spaced.
+  # Notice the slice group with 3 slices (i.e. 6, 15, 22) are roughly evenly spaced.
   # We are trying to reimplement the calculation in the positions method in ParallelCollectionRDD
   if (numSerializedSlices > 0) {
     unlist(lapply(0: (numSerializedSlices - 1), function(x) {
@@ -116,7 +116,7 @@ makeSplits <- function(numSerializedSlices, length) {
 #' This change affects both createDataFrame and spark.lapply.
 #' In the specific one case that it is used to convert R native object into SparkDataFrame, it has
 #' always been kept at the default of 1. In the case the object is large, we are explicitly setting
-#' the parallism to numSlices (which is still 1).
+#' the parallelism to numSlices (which is still 1).
 #'
 #' Specifically, we are changing to split positions to match the calculation in positions() of
 #' ParallelCollectionRDD in Spark.
diff --git a/R/pkg/R/deserialize.R b/R/pkg/R/deserialize.R
index 3e7c456bd548d..89a8fbecd36b0 100644
--- a/R/pkg/R/deserialize.R
+++ b/R/pkg/R/deserialize.R
@@ -233,24 +233,13 @@ readMultipleObjectsWithKeys <- function(inputCon) {
 
 readDeserializeInArrow <- function(inputCon) {
   if (requireNamespace("arrow", quietly = TRUE)) {
-    # Arrow drops `as_tibble` since 0.14.0, see ARROW-5190.
-    useAsTibble <- exists("as_tibble", envir = asNamespace("arrow"))
-
-
     # Currently, there looks no way to read batch by batch by socket connection in R side,
     # See ARROW-4512. Therefore, it reads the whole Arrow streaming-formatted binary at once
     # for now.
     dataLen <- readInt(inputCon)
     arrowData <- readBin(inputCon, raw(), as.integer(dataLen), endian = "big")
     batches <- arrow::RecordBatchStreamReader$create(arrowData)$batches()
-
-    if (useAsTibble) {
-      as_tibble <- get("as_tibble", envir = asNamespace("arrow"))
-      # Read all groupped batches. Tibble -> data.frame is cheap.
-      lapply(batches, function(batch) as.data.frame(as_tibble(batch)))
-    } else {
-      lapply(batches, function(batch) as.data.frame(batch))
-    }
+    lapply(batches, function(batch) as.data.frame(batch))
   } else {
     stop("'arrow' package should be installed.")
   }
@@ -261,7 +250,7 @@ readDeserializeWithKeysInArrow <- function(inputCon) {
 
   keys <- readMultipleObjects(inputCon)
 
-  # Read keys to map with each groupped batch later.
+  # Read keys to map with each grouped batch later.
   list(keys = keys, data = data)
 }
 
diff --git a/R/pkg/R/functions.R b/R/pkg/R/functions.R
index da9ef1d6674bd..3ee1cd5b50b48 100644
--- a/R/pkg/R/functions.R
+++ b/R/pkg/R/functions.R
@@ -144,7 +144,7 @@ NULL
 #' @param y Column to compute on.
 #' @param pos In \itemize{
 #'                \item \code{locate}: a start position of search.
-#'                \item \code{overlay}: a start postiton for replacement.
+#'                \item \code{overlay}: a start position for replacement.
 #'                }
 #' @param len In \itemize{
 #'               \item \code{lpad} the maximum length of each output result.
@@ -247,7 +247,7 @@ NULL
 #'            used to transform the data.  The first argument is the key, the second argument
 #'            is the value.
 #'          }
-#' @param zero a \code{Column} used as the initial value in \code{array_aggregate}
+#' @param initialValue a \code{Column} used as the initial value in \code{array_aggregate}
 #' @param merge a \code{function} a binary function \code{(Column, Column) -> Column}
 #'          used in \code{array_aggregate}to merge values (the second argument)
 #'          into accumulator (the first argument).
@@ -338,12 +338,79 @@ NULL
 #' tmp <- mutate(df, dist = over(cume_dist(), ws), dense_rank = over(dense_rank(), ws),
 #'               lag = over(lag(df$mpg), ws), lead = over(lead(df$mpg, 1), ws),
 #'               percent_rank = over(percent_rank(), ws),
-#'               rank = over(rank(), ws), row_number = over(row_number(), ws))
+#'               rank = over(rank(), ws), row_number = over(row_number(), ws),
+#'               nth_value = over(nth_value(df$mpg, 3), ws))
 #' # Get ntile group id (1-4) for hp
 #' tmp <- mutate(tmp, ntile = over(ntile(4), ws))
 #' head(tmp)}
 NULL
 
+#' ML functions for Column operations
+#'
+#' ML functions defined for \code{Column}.
+#'
+#' @param x Column to compute on.
+#' @param ... additional argument(s).
+#' @name column_ml_functions
+#' @rdname column_ml_functions
+#' @family ml functions
+#' @examples
+#' \dontrun{
+#' df <- read.df("data/mllib/sample_libsvm_data.txt", source = "libsvm")
+#' head(
+#'   withColumn(
+#'     withColumn(df, "array", vector_to_array(df$features)),
+#'     "vector",
+#'     array_to_vector(column("array"))
+#'   )
+#' )
+#' }
+NULL
+
+#' Avro processing functions for Column operations
+#'
+#' Avro processing functions defined for \code{Column}.
+#'
+#' @param x Column to compute on.
+#' @param jsonFormatSchema character Avro schema in JSON string format
+#' @param ... additional argument(s) passed as parser options.
+#' @name column_avro_functions
+#' @rdname column_avro_functions
+#' @family avro functions
+#' @note Avro is built-in but external data source module since Spark 2.4.
+#'   Please deploy the application as per
+#'   \href{https://spark.apache.org/docs/latest/sql-data-sources-avro.html#deploying}{
+#'     the deployment section
+#'   } of "Apache Avro Data Source Guide".
+#' @examples
+#' \dontrun{
+#' df <- createDataFrame(iris)
+#' schema <- paste(
+#'   c(
+#'     '{"type": "record", "namespace": "example.avro", "name": "Iris", "fields": [',
+#'     '{"type": ["double", "null"], "name": "Sepal_Length"},',
+#'     '{"type": ["double", "null"], "name": "Sepal_Width"},',
+#'     '{"type": ["double", "null"], "name": "Petal_Length"},',
+#'     '{"type": ["double", "null"], "name": "Petal_Width"},',
+#'     '{"type": ["string", "null"], "name": "Species"}]}'
+#'   ),
+#'   collapse="\\n"
+#' )
+#'
+#' df_serialized <- select(
+#'   df,
+#'   alias(to_avro(alias(struct(column("*")), "fields")), "payload")
+#' )
+#'
+#' df_deserialized <- select(
+#'   df_serialized,
+#'   from_avro(df_serialized$payload, schema)
+#' )
+#'
+#' head(df_deserialized)
+#' }
+NULL
+
 #' @details
 #' \code{lit}: A new Column is created to represent the literal value.
 #' If the parameter is a Column, it is returned unchanged.
@@ -394,6 +461,19 @@ setMethod("acos",
             column(jc)
           })
 
+#' @details
+#' \code{acosh}: Computes inverse hyperbolic cosine of the input column.
+#'
+#' @rdname column_math_functions
+#' @aliases acosh acosh,Column-method
+#' @note acosh since 3.1.0
+setMethod("acosh",
+          signature(x = "Column"),
+          function(x) {
+            jc <- callJStatic("org.apache.spark.sql.functions", "acosh", x@jc)
+            column(jc)
+          })
+
 #' @details
 #' \code{approx_count_distinct}: Returns the approximate number of distinct items in a group.
 #'
@@ -404,7 +484,7 @@ setMethod("acos",
 #' \dontrun{
 #' head(select(df, approx_count_distinct(df$gear)))
 #' head(select(df, approx_count_distinct(df$gear, 0.02)))
-#' head(select(df, countDistinct(df$gear, df$cyl)))
+#' head(select(df, count_distinct(df$gear, df$cyl)))
 #' head(select(df, n_distinct(df$gear)))
 #' head(distinct(select(df, "gear")))}
 #' @note approx_count_distinct(Column) since 3.0.0
@@ -461,6 +541,19 @@ setMethod("asin",
             column(jc)
           })
 
+#' @details
+#' \code{asinh}: Computes inverse hyperbolic sine of the input column.
+#'
+#' @rdname column_math_functions
+#' @aliases asinh asinh,Column-method
+#' @note asinh since 3.1.0
+setMethod("asinh",
+          signature(x = "Column"),
+          function(x) {
+            jc <- callJStatic("org.apache.spark.sql.functions", "asinh", x@jc)
+            column(jc)
+          })
+
 #' @details
 #' \code{atan}: Returns the inverse tangent of the given value,
 #' as if computed by \code{java.lang.Math.atan()}
@@ -475,6 +568,19 @@ setMethod("atan",
             column(jc)
           })
 
+#' @details
+#' \code{atanh}: Computes inverse hyperbolic tangent of the input column.
+#'
+#' @rdname column_math_functions
+#' @aliases atanh atanh,Column-method
+#' @note atanh since 3.1.0
+setMethod("atanh",
+          signature(x = "Column"),
+          function(x) {
+            jc <- callJStatic("org.apache.spark.sql.functions", "atanh", x@jc)
+            column(jc)
+          })
+
 #' avg
 #'
 #' Aggregate function: returns the average of the values in a group.
@@ -530,20 +636,33 @@ setMethod("bin",
           })
 
 #' @details
-#' \code{bitwiseNOT}: Computes bitwise NOT.
+#' \code{bitwise_not}: Computes bitwise NOT.
 #'
 #' @rdname column_nonaggregate_functions
-#' @aliases bitwiseNOT bitwiseNOT,Column-method
+#' @aliases bitwise_not bitwise_not,Column-method
 #' @examples
 #'
 #' \dontrun{
-#' head(select(df, bitwiseNOT(cast(df$vs, "int"))))}
+#' head(select(df, bitwise_not(cast(df$vs, "int"))))}
+#' @note bitwise_not since 3.2.0
+setMethod("bitwise_not",
+          signature(x = "Column"),
+          function(x) {
+            jc <- callJStatic("org.apache.spark.sql.functions", "bitwise_not", x@jc)
+            column(jc)
+          })
+
+#' @details
+#' \code{bitwiseNOT}: Computes bitwise NOT.
+#'
+#' @rdname column_nonaggregate_functions
+#' @aliases bitwiseNOT bitwiseNOT,Column-method
 #' @note bitwiseNOT since 1.5.0
 setMethod("bitwiseNOT",
           signature(x = "Column"),
           function(x) {
-            jc <- callJStatic("org.apache.spark.sql.functions", "bitwiseNOT", x@jc)
-            column(jc)
+            .Deprecated("bitwise_not")
+            bitwise_not(x)
           })
 
 #' @details
@@ -809,6 +928,57 @@ setMethod("xxhash64",
             column(jc)
           })
 
+#' @details
+#' \code{assert_true}: Returns null if the input column is true; throws an exception
+#' with the provided error message otherwise.
+#'
+#' @param errMsg (optional) The error message to be thrown.
+#'
+#' @rdname column_misc_functions
+#' @aliases assert_true assert_true,Column-method
+#' @examples
+#' \dontrun{
+#' tmp <- mutate(df, v1 = assert_true(df$vs < 2),
+#'                   v2 = assert_true(df$vs < 2, "custom error message"),
+#'                   v3 = assert_true(df$vs < 2, df$vs))
+#' head(tmp)}
+#' @note assert_true since 3.1.0
+setMethod("assert_true",
+          signature(x = "Column"),
+          function(x, errMsg = NULL) {
+            jc <- if (is.null(errMsg)) {
+              callJStatic("org.apache.spark.sql.functions", "assert_true", x@jc)
+            } else {
+              if (is.character(errMsg)) {
+                stopifnot(length(errMsg) == 1)
+                errMsg <- lit(errMsg)
+              }
+              callJStatic("org.apache.spark.sql.functions", "assert_true", x@jc, errMsg@jc)
+            }
+            column(jc)
+          })
+
+#' @details
+#' \code{raise_error}: Throws an exception with the provided error message.
+#'
+#' @rdname column_misc_functions
+#' @aliases raise_error raise_error,characterOrColumn-method
+#' @examples
+#' \dontrun{
+#' tmp <- mutate(df, v1 = raise_error("error message"))
+#' head(tmp)}
+#' @note raise_error since 3.1.0
+setMethod("raise_error",
+          signature(x = "characterOrColumn"),
+          function(x) {
+            if (is.character(x)) {
+              stopifnot(length(x) == 1)
+              x <- lit(x)
+            }
+            jc <- callJStatic("org.apache.spark.sql.functions", "raise_error", x@jc)
+            column(jc)
+          })
+
 #' @details
 #' \code{dayofmonth}: Extracts the day of the month as an integer from a
 #' given date/timestamp/string.
@@ -1403,6 +1573,19 @@ setMethod("overlay",
     column(jc)
   })
 
+#' @details
+#' \code{product}: Returns the product of the values in a group.
+#'
+#' @rdname column_aggregate_functions
+#' @aliases product product,Column-method
+#' @note product since 3.2.0
+setMethod("product",
+          signature(x = "Column"),
+          function(x) {
+            jc <- callJStatic("org.apache.spark.sql.functions", "product", x@jc)
+            column(jc)
+          })
+
 #' @details
 #' \code{quarter}: Extracts the quarter as an integer from a given date/timestamp/string.
 #'
@@ -1417,8 +1600,10 @@ setMethod("quarter",
           })
 
 #' @details
-#' \code{percentile_approx} Returns the approximate percentile value of
-#' numeric column at the given percentage.
+#' \code{percentile_approx} Returns the approximate \code{percentile} of the numeric column
+#' \code{col} which is the smallest value in the ordered \code{col} values (sorted from least to
+#' greatest) such that no more than \code{percentage} of \code{col} values is less than the value
+#' or equal to that value.
 #'
 #' @param percentage Numeric percentage at which percentile should be computed
 #'                   All values should be between 0 and 1.
@@ -1778,21 +1963,34 @@ setMethod("sum",
           })
 
 #' @details
-#' \code{sumDistinct}: Returns the sum of distinct values in the expression.
+#' \code{sum_distinct}: Returns the sum of distinct values in the expression.
 #'
 #' @rdname column_aggregate_functions
-#' @aliases sumDistinct sumDistinct,Column-method
+#' @aliases sum_distinct sum_distinct,Column-method
 #' @examples
 #'
 #' \dontrun{
-#' head(select(df, sumDistinct(df$gear)))
+#' head(select(df, sum_distinct(df$gear)))
 #' head(distinct(select(df, "gear")))}
+#' @note sum_distinct since 3.2.0
+setMethod("sum_distinct",
+          signature(x = "Column"),
+          function(x) {
+            jc <- callJStatic("org.apache.spark.sql.functions", "sum_distinct", x@jc)
+            column(jc)
+          })
+
+#' @details
+#' \code{sumDistinct}: Returns the sum of distinct values in the expression.
+#'
+#' @rdname column_aggregate_functions
+#' @aliases sumDistinct sumDistinct,Column-method
 #' @note sumDistinct since 1.4.0
 setMethod("sumDistinct",
           signature(x = "Column"),
           function(x) {
-            jc <- callJStatic("org.apache.spark.sql.functions", "sumDistinct", x@jc)
-            column(jc)
+            .Deprecated("sum_distinct")
+            sum_distinct(x)
           })
 
 #' @details
@@ -2286,7 +2484,7 @@ setMethod("pmod", signature(y = "Column"),
             column(jc)
           })
 
-#' @param rsd maximum estimation error allowed (default = 0.05).
+#' @param rsd maximum relative standard deviation allowed (default = 0.05).
 #'
 #' @rdname column_aggregate_functions
 #' @aliases approx_count_distinct,Column-method
@@ -2310,23 +2508,37 @@ setMethod("approxCountDistinct",
           })
 
 #' @details
-#' \code{countDistinct}: Returns the number of distinct items in a group.
+#' \code{count_distinct}: Returns the number of distinct items in a group.
 #'
 #' @rdname column_aggregate_functions
-#' @aliases countDistinct countDistinct,Column-method
-#' @note countDistinct since 1.4.0
-setMethod("countDistinct",
+#' @aliases count_distinct count_distinct,Column-method
+#' @note count_distinct since 3.2.0
+setMethod("count_distinct",
           signature(x = "Column"),
           function(x, ...) {
             jcols <- lapply(list(...), function(x) {
               stopifnot(class(x) == "Column")
               x@jc
             })
-            jc <- callJStatic("org.apache.spark.sql.functions", "countDistinct", x@jc,
+            jc <- callJStatic("org.apache.spark.sql.functions", "count_distinct", x@jc,
                               jcols)
             column(jc)
           })
 
+#' @details
+#' \code{countDistinct}: Returns the number of distinct items in a group.
+#'
+#' An alias of \code{count_distinct}, and it is encouraged to use \code{count_distinct} directly.
+#'
+#' @rdname column_aggregate_functions
+#' @aliases countDistinct countDistinct,Column-method
+#' @note countDistinct since 1.4.0
+setMethod("countDistinct",
+          signature(x = "Column"),
+          function(x, ...) {
+            count_distinct(x, ...)
+          })
+
 #' @details
 #' \code{concat}: Concatenates multiple input columns together into a single column.
 #' The function works with strings, binary and compatible array columns.
@@ -2391,7 +2603,7 @@ setMethod("least",
 #' @note n_distinct since 1.4.0
 setMethod("n_distinct", signature(x = "Column"),
           function(x, ...) {
-            countDistinct(x, ...)
+            count_distinct(x, ...)
           })
 
 #' @rdname count
@@ -2734,6 +2946,21 @@ setMethod("sha2", signature(y = "Column", x = "numeric"),
             column(jc)
           })
 
+#' @details
+#' \code{shiftleft}: Shifts the given value numBits left. If the given value is a long value,
+#' this function will return a long value else it will return an integer value.
+#'
+#' @rdname column_math_functions
+#' @aliases shiftleft shiftleft,Column,numeric-method
+#' @note shiftleft since 3.2.0
+setMethod("shiftleft", signature(y = "Column", x = "numeric"),
+          function(y, x) {
+            jc <- callJStatic("org.apache.spark.sql.functions",
+                              "shiftleft",
+                              y@jc, as.integer(x))
+            column(jc)
+          })
+
 #' @details
 #' \code{shiftLeft}: Shifts the given value numBits left. If the given value is a long value,
 #' this function will return a long value else it will return an integer value.
@@ -2742,9 +2969,22 @@ setMethod("sha2", signature(y = "Column", x = "numeric"),
 #' @aliases shiftLeft shiftLeft,Column,numeric-method
 #' @note shiftLeft since 1.5.0
 setMethod("shiftLeft", signature(y = "Column", x = "numeric"),
+          function(y, x) {
+            .Deprecated("shiftleft")
+            shiftleft(y, x)
+          })
+
+#' @details
+#' \code{shiftright}: (Signed) shifts the given value numBits right. If the given value is a long
+#' value, it will return a long value else it will return an integer value.
+#'
+#' @rdname column_math_functions
+#' @aliases shiftright shiftright,Column,numeric-method
+#' @note shiftright since 3.2.0
+setMethod("shiftright", signature(y = "Column", x = "numeric"),
           function(y, x) {
             jc <- callJStatic("org.apache.spark.sql.functions",
-                              "shiftLeft",
+                              "shiftright",
                               y@jc, as.integer(x))
             column(jc)
           })
@@ -2757,15 +2997,28 @@ setMethod("shiftLeft", signature(y = "Column", x = "numeric"),
 #' @aliases shiftRight shiftRight,Column,numeric-method
 #' @note shiftRight since 1.5.0
 setMethod("shiftRight", signature(y = "Column", x = "numeric"),
+          function(y, x) {
+            .Deprecated("shiftright")
+            shiftright(y, x)
+          })
+
+#' @details
+#' \code{shiftrightunsigned}: (Unsigned) shifts the given value numBits right. If the given value is
+#' a long value, it will return a long value else it will return an integer value.
+#'
+#' @rdname column_math_functions
+#' @aliases shiftrightunsigned shiftrightunsigned,Column,numeric-method
+#' @note shiftrightunsigned since 3.2.0
+setMethod("shiftrightunsigned", signature(y = "Column", x = "numeric"),
           function(y, x) {
             jc <- callJStatic("org.apache.spark.sql.functions",
-                              "shiftRight",
+                              "shiftrightunsigned",
                               y@jc, as.integer(x))
             column(jc)
           })
 
 #' @details
-#' \code{shiftRightUnsigned}: (Unigned) shifts the given value numBits right. If the given value is
+#' \code{shiftRightUnsigned}: (Unsigned) shifts the given value numBits right. If the given value is
 #' a long value, it will return a long value else it will return an integer value.
 #'
 #' @rdname column_math_functions
@@ -2773,10 +3026,8 @@ setMethod("shiftRight", signature(y = "Column", x = "numeric"),
 #' @note shiftRightUnsigned since 1.5.0
 setMethod("shiftRightUnsigned", signature(y = "Column", x = "numeric"),
           function(y, x) {
-            jc <- callJStatic("org.apache.spark.sql.functions",
-                              "shiftRightUnsigned",
-                              y@jc, as.integer(x))
-            column(jc)
+            .Deprecated("shiftrightunsigned")
+            shiftrightunsigned(y, x)
           })
 
 #' @details
@@ -3296,6 +3547,37 @@ setMethod("lead",
             column(jc)
           })
 
+#' @details
+#' \code{nth_value}: Window function: returns the value that is the \code{offset}th
+#' row of the window frame# (counting from 1), and \code{null} if the size of window
+#' frame is less than \code{offset} rows.
+#'
+#' @param offset a numeric indicating number of row to use as the value
+#' @param na.rm a logical which indicates that the Nth value should skip null in the
+#'        determination of which row to use
+#'
+#' @rdname column_window_functions
+#' @aliases nth_value nth_value,characterOrColumn-method
+#' @note nth_value since 3.1.0
+setMethod("nth_value",
+          signature(x = "characterOrColumn", offset = "numeric"),
+          function(x, offset, na.rm = FALSE) {
+            x <- if (is.character(x)) {
+              column(x)
+            } else {
+              x
+            }
+            offset <- as.integer(offset)
+            jc <- callJStatic(
+              "org.apache.spark.sql.functions",
+              "nth_value",
+              x@jc,
+              offset,
+              na.rm
+            )
+            column(jc)
+          })
+
 #' @details
 #' \code{ntile}: Returns the ntile group id (from 1 to n inclusive) in an ordered window
 #' partition. For example, if n is 4, the first quarter of the rows will get value 1, the second
@@ -3388,7 +3670,12 @@ unresolved_named_lambda_var <- function(...) {
     "org.apache.spark.sql.Column",
     newJObject(
       "org.apache.spark.sql.catalyst.expressions.UnresolvedNamedLambdaVariable",
-      list(...)
+      lapply(list(...), function(x) {
+        handledCallJStatic(
+          "org.apache.spark.sql.catalyst.expressions.UnresolvedNamedLambdaVariable",
+          "freshVarName",
+          x)
+      })
     )
   )
   column(jc)
@@ -3476,11 +3763,11 @@ invoke_higher_order_function <- function(name, cols, funs) {
 #' @aliases array_aggregate array_aggregate,characterOrColumn,Column,function-method
 #' @note array_aggregate since 3.1.0
 setMethod("array_aggregate",
-          signature(x = "characterOrColumn", zero = "Column", merge = "function"),
-          function(x, zero, merge, finish = NULL) {
+          signature(x = "characterOrColumn", initialValue = "Column", merge = "function"),
+          function(x, initialValue, merge, finish = NULL) {
             invoke_higher_order_function(
               "ArrayAggregate",
-              cols = list(x, zero),
+              cols = list(x, initialValue),
               funs = if (is.null(finish)) {
                 list(merge)
               } else {
@@ -4380,7 +4667,8 @@ setMethod("date_trunc",
           })
 
 #' @details
-#' \code{current_date}: Returns the current date as a date column.
+#' \code{current_date}: Returns the current date at the start of query evaluation as a date column.
+#' All calls of current_date within the same query return the same value.
 #'
 #' @rdname column_datetime_functions
 #' @aliases current_date current_date,missing-method
@@ -4396,7 +4684,8 @@ setMethod("current_date",
           })
 
 #' @details
-#' \code{current_timestamp}: Returns the current timestamp as a timestamp column.
+#' \code{current_timestamp}: Returns the current timestamp at the start of query evaluation as
+#' a timestamp column. All calls of current_timestamp within the same query return the same value.
 #'
 #' @rdname column_datetime_functions
 #' @aliases current_timestamp current_timestamp,missing-method
@@ -4407,3 +4696,115 @@ setMethod("current_timestamp",
             jc <- callJStatic("org.apache.spark.sql.functions", "current_timestamp")
             column(jc)
           })
+
+#' @details
+#' \code{timestamp_seconds}: Creates timestamp from the number of seconds since UTC epoch.
+#'
+#' @rdname column_datetime_functions
+#' @aliases timestamp_seconds timestamp_seconds,Column-method
+#' @note timestamp_seconds since 3.1.0
+setMethod("timestamp_seconds",
+          signature(x = "Column"),
+          function(x) {
+            jc <- callJStatic(
+              "org.apache.spark.sql.functions", "timestamp_seconds", x@jc
+            )
+            column(jc)
+          })
+
+#' @details
+#' \code{array_to_vector} Converts a column of array of numeric type into
+#' a column of dense vectors in MLlib
+#'
+#' @rdname column_ml_functions
+#' @aliases array_to_vector array_to_vector,Column-method
+#' @note array_to_vector since 3.1.0
+setMethod("array_to_vector",
+          signature(x = "Column"),
+          function(x) {
+            jc <- callJStatic(
+              "org.apache.spark.ml.functions",
+              "array_to_vector",
+              x@jc
+            )
+            column(jc)
+          })
+
+#' @details
+#' \code{vector_to_array} Converts a column of MLlib sparse/dense vectors into
+#' a column of dense arrays.
+#'
+#' @param dtype The data type of the output array. Valid values: "float64" or "float32".
+#'
+#' @rdname column_ml_functions
+#' @aliases vector_to_array vector_to_array,Column-method
+#' @note vector_to_array since 3.1.0
+setMethod("vector_to_array",
+          signature(x = "Column"),
+          function(x, dtype = c("float64", "float32")) {
+            dtype <- match.arg(dtype)
+            jc <- callJStatic(
+              "org.apache.spark.ml.functions",
+              "vector_to_array",
+              x@jc,
+              dtype
+            )
+            column(jc)
+          })
+
+#' @details
+#' \code{from_avro} Converts a binary column of Avro format into its corresponding catalyst value.
+#' The specified schema must match the read data, otherwise the behavior is undefined:
+#' it may fail or return arbitrary result.
+#' To deserialize the data with a compatible and evolved schema, the expected Avro schema can be
+#' set via the option avroSchema.
+#'
+#' @rdname column_avro_functions
+#' @aliases from_avro from_avro,Column-method
+#' @note from_avro since 3.1.0
+setMethod("from_avro",
+          signature(x = "characterOrColumn"),
+          function(x, jsonFormatSchema, ...) {
+            x <- if (is.character(x)) {
+              column(x)
+            } else {
+              x
+            }
+
+            options <- varargsToStrEnv(...)
+            jc <- callJStatic(
+              "org.apache.spark.sql.avro.functions", "from_avro",
+              x@jc,
+              jsonFormatSchema,
+              options
+            )
+            column(jc)
+          })
+
+#' @details
+#' \code{to_avro} Converts a column into binary of Avro format.
+#'
+#' @rdname column_avro_functions
+#' @aliases to_avro to_avro,Column-method
+#' @note to_avro since 3.1.0
+setMethod("to_avro",
+          signature(x = "characterOrColumn"),
+          function(x, jsonFormatSchema = NULL) {
+            x <- if (is.character(x)) {
+              column(x)
+            } else {
+              x
+            }
+
+            jc <- if (is.null(jsonFormatSchema)) {
+              callJStatic("org.apache.spark.sql.avro.functions", "to_avro", x@jc)
+            } else {
+              callJStatic(
+                "org.apache.spark.sql.avro.functions",
+                "to_avro",
+                x@jc,
+                jsonFormatSchema
+              )
+            }
+            column(jc)
+          })
diff --git a/R/pkg/R/generics.R b/R/pkg/R/generics.R
index 839c00cf21aeb..38ad5f742ca68 100644
--- a/R/pkg/R/generics.R
+++ b/R/pkg/R/generics.R
@@ -638,7 +638,7 @@ setGeneric("union", function(x, y) { standardGeneric("union") })
 setGeneric("unionAll", function(x, y) { standardGeneric("unionAll") })
 
 #' @rdname unionByName
-setGeneric("unionByName", function(x, y) { standardGeneric("unionByName") })
+setGeneric("unionByName", function(x, y, ...) { standardGeneric("unionByName") })
 
 #' @rdname unpersist
 setGeneric("unpersist", function(x, ...) { standardGeneric("unpersist") })
@@ -675,6 +675,12 @@ setGeneric("broadcast", function(x) { standardGeneric("broadcast") })
 #' @rdname columnfunctions
 setGeneric("asc", function(x) { standardGeneric("asc") })
 
+#' @rdname columnfunctions
+setGeneric("asc_nulls_first", function(x) { standardGeneric("asc_nulls_first") })
+
+#' @rdname columnfunctions
+setGeneric("asc_nulls_last", function(x) { standardGeneric("asc_nulls_last") })
+
 #' @rdname between
 setGeneric("between", function(x, bounds) { standardGeneric("between") })
 
@@ -689,6 +695,12 @@ setGeneric("contains", function(x, ...) { standardGeneric("contains") })
 #' @rdname columnfunctions
 setGeneric("desc", function(x) { standardGeneric("desc") })
 
+#' @rdname columnfunctions
+setGeneric("desc_nulls_first", function(x) { standardGeneric("desc_nulls_first") })
+
+#' @rdname columnfunctions
+setGeneric("desc_nulls_last", function(x) { standardGeneric("desc_nulls_last") })
+
 #' @rdname endsWith
 setGeneric("endsWith", function(x, suffix) { standardGeneric("endsWith") })
 
@@ -729,6 +741,12 @@ setGeneric("over", function(x, window) { standardGeneric("over") })
 #' @rdname eq_null_safe
 setGeneric("%<=>%", function(x, value) { standardGeneric("%<=>%") })
 
+#' @rdname withField
+setGeneric("withField", function(x, fieldName, col) { standardGeneric("withField") })
+
+#' @rdname dropFields
+setGeneric("dropFields", function(x, ...) { standardGeneric("dropFields") })
+
 ###################### WindowSpec Methods ##########################
 
 #' @rdname partitionBy
@@ -762,7 +780,8 @@ setGeneric("approxCountDistinct", function(x, ...) { standardGeneric("approxCoun
 
 #' @rdname column_collection_functions
 #' @name NULL
-setGeneric("array_aggregate", function(x, zero, merge, ...) { standardGeneric("array_aggregate") })
+setGeneric("array_aggregate",
+           function(x, initialValue, merge, ...) { standardGeneric("array_aggregate") })
 
 #' @rdname column_collection_functions
 #' @name NULL
@@ -820,6 +839,10 @@ setGeneric("array_repeat", function(x, count) { standardGeneric("array_repeat")
 #' @name NULL
 setGeneric("array_sort", function(x) { standardGeneric("array_sort") })
 
+#' @rdname column_ml_functions
+#' @name NULL
+setGeneric("array_to_vector", function(x) { standardGeneric("array_to_vector") })
+
 #' @rdname column_collection_functions
 #' @name NULL
 setGeneric("array_transform", function(x, f) { standardGeneric("array_transform") })
@@ -844,6 +867,10 @@ setGeneric("arrays_zip_with", function(x, y, f) { standardGeneric("arrays_zip_wi
 #' @name NULL
 setGeneric("ascii", function(x) { standardGeneric("ascii") })
 
+#' @rdname column_misc_functions
+#' @name NULL
+setGeneric("assert_true", function(x, errMsg = NULL) { standardGeneric("assert_true") })
+
 #' @param x Column to compute on or a GroupedData object.
 #' @param ... additional argument(s) when \code{x} is a GroupedData object.
 #' @rdname avg
@@ -857,6 +884,10 @@ setGeneric("base64", function(x) { standardGeneric("base64") })
 #' @name NULL
 setGeneric("bin", function(x) { standardGeneric("bin") })
 
+#' @rdname column_nonaggregate_functions
+#' @name NULL
+setGeneric("bitwise_not", function(x) { standardGeneric("bitwise_not") })
+
 #' @rdname column_nonaggregate_functions
 #' @name NULL
 setGeneric("bitwiseNOT", function(x) { standardGeneric("bitwiseNOT") })
@@ -896,6 +927,10 @@ setGeneric("concat_ws", function(sep, x, ...) { standardGeneric("concat_ws") })
 #' @name NULL
 setGeneric("conv", function(x, fromBase, toBase) { standardGeneric("conv") })
 
+#' @rdname column_aggregate_functions
+#' @name NULL
+setGeneric("count_distinct", function(x, ...) { standardGeneric("count_distinct") })
+
 #' @rdname column_aggregate_functions
 #' @name NULL
 setGeneric("countDistinct", function(x, ...) { standardGeneric("countDistinct") })
@@ -928,7 +963,6 @@ setGeneric("current_date", function(x = "missing") { standardGeneric("current_da
 #' @name NULL
 setGeneric("current_timestamp", function(x = "missing") { standardGeneric("current_timestamp") })
 
-
 #' @rdname column_datetime_diff_functions
 #' @name NULL
 setGeneric("datediff", function(y, x) { standardGeneric("datediff") })
@@ -993,6 +1027,10 @@ setGeneric("expr", function(x) { standardGeneric("expr") })
 #' @name NULL
 setGeneric("flatten", function(x) { standardGeneric("flatten") })
 
+#' @rdname column_avro_functions
+#' @name NULL
+setGeneric("from_avro", function(x, ...) { standardGeneric("from_avro") })
+
 #' @rdname column_datetime_diff_functions
 #' @name NULL
 setGeneric("from_utc_timestamp", function(y, x) { standardGeneric("from_utc_timestamp") })
@@ -1161,6 +1199,10 @@ setGeneric("months_between", function(y, x, ...) { standardGeneric("months_betwe
 #' @rdname count
 setGeneric("n", function(x) { standardGeneric("n") })
 
+#' @rdname column_window_functions
+#' @name NULL
+setGeneric("nth_value", function(x, offset, ...) { standardGeneric("nth_value") })
+
 #' @rdname column_nonaggregate_functions
 #' @name NULL
 setGeneric("nanvl", function(y, x) { standardGeneric("nanvl") })
@@ -1209,10 +1251,18 @@ setGeneric("posexplode", function(x) { standardGeneric("posexplode") })
 #' @name NULL
 setGeneric("posexplode_outer", function(x) { standardGeneric("posexplode_outer") })
 
+#' @rdname column_aggregate_functions
+#' @name NULL
+setGeneric("product", function(x) { standardGeneric("product") })
+
 #' @rdname column_datetime_functions
 #' @name NULL
 setGeneric("quarter", function(x) { standardGeneric("quarter") })
 
+#' @rdname column_misc_functions
+#' @name NULL
+setGeneric("raise_error", function(x) { standardGeneric("raise_error") })
+
 #' @rdname column_nonaggregate_functions
 #' @name NULL
 setGeneric("rand", function(seed) { standardGeneric("rand") })
@@ -1286,14 +1336,26 @@ setGeneric("sha2", function(y, x) { standardGeneric("sha2") })
 #' @name NULL
 setGeneric("shiftLeft", function(y, x) { standardGeneric("shiftLeft") })
 
+#' @rdname column_math_functions
+#' @name NULL
+setGeneric("shiftleft", function(y, x) { standardGeneric("shiftleft") })
+
 #' @rdname column_math_functions
 #' @name NULL
 setGeneric("shiftRight", function(y, x) { standardGeneric("shiftRight") })
 
+#' @rdname column_math_functions
+#' @name NULL
+setGeneric("shiftright", function(y, x) { standardGeneric("shiftright") })
+
 #' @rdname column_math_functions
 #' @name NULL
 setGeneric("shiftRightUnsigned", function(y, x) { standardGeneric("shiftRightUnsigned") })
 
+#' @rdname column_math_functions
+#' @name NULL
+setGeneric("shiftrightunsigned", function(y, x) { standardGeneric("shiftrightunsigned") })
+
 #' @rdname column_collection_functions
 #' @name NULL
 setGeneric("shuffle", function(x) { standardGeneric("shuffle") })
@@ -1350,10 +1412,22 @@ setGeneric("struct", function(x, ...) { standardGeneric("struct") })
 #' @name NULL
 setGeneric("substring_index", function(x, delim, count) { standardGeneric("substring_index") })
 
+#' @rdname column_aggregate_functions
+#' @name NULL
+setGeneric("sum_distinct", function(x) { standardGeneric("sum_distinct") })
+
 #' @rdname column_aggregate_functions
 #' @name NULL
 setGeneric("sumDistinct", function(x) { standardGeneric("sumDistinct") })
 
+#' @rdname column_datetime_functions
+#' @name timestamp_seconds
+setGeneric("timestamp_seconds", function(x) { standardGeneric("timestamp_seconds") })
+
+#' @rdname column_avro_functions
+#' @name NULL
+setGeneric("to_avro", function(x, ...) { standardGeneric("to_avro") })
+
 #' @rdname column_collection_functions
 #' @name NULL
 setGeneric("transform_keys", function(x, f) {  standardGeneric("transform_keys") })
@@ -1438,6 +1512,10 @@ setGeneric("var_pop", function(x) { standardGeneric("var_pop") })
 #' @name NULL
 setGeneric("var_samp", function(x) { standardGeneric("var_samp") })
 
+#' @rdname column_ml_functions
+#' @name NULL
+setGeneric("vector_to_array", function(x, ...) { standardGeneric("vector_to_array") })
+
 #' @rdname column_datetime_functions
 #' @name NULL
 setGeneric("weekofyear", function(x) { standardGeneric("weekofyear") })
diff --git a/R/pkg/R/install.R b/R/pkg/R/install.R
index ea2c0b4c0f42f..bbb9188cd083f 100644
--- a/R/pkg/R/install.R
+++ b/R/pkg/R/install.R
@@ -39,11 +39,11 @@
 #'                      version number in the format of "x.y" where x and y are integer.
 #'                      If \code{hadoopVersion = "without"}, "Hadoop free" build is installed.
 #'                      See
-#'                      \href{http://spark.apache.org/docs/latest/hadoop-provided.html}{
+#'                      \href{https://spark.apache.org/docs/latest/hadoop-provided.html}{
 #'                      "Hadoop Free" Build} for more information.
 #'                      Other patched version names can also be used, e.g. \code{"cdh4"}
 #' @param mirrorUrl base URL of the repositories to use. The directory layout should follow
-#'                  \href{http://www.apache.org/dyn/closer.lua/spark/}{Apache mirrors}.
+#'                  \href{https://www.apache.org/dyn/closer.lua/spark/}{Apache mirrors}.
 #' @param localDir a local directory where Spark is installed. The directory contains
 #'                 version-specific folders of Spark packages. Default is path to
 #'                 the cache directory:
@@ -64,7 +64,7 @@
 #'}
 #' @note install.spark since 2.1.0
 #' @seealso See available Hadoop versions:
-#'          \href{http://spark.apache.org/downloads.html}{Apache Spark}
+#'          \href{https://spark.apache.org/downloads.html}{Apache Spark}
 install.spark <- function(hadoopVersion = "2.7", mirrorUrl = NULL,
                           localDir = NULL, overwrite = FALSE) {
   sparkHome <- Sys.getenv("SPARK_HOME")
@@ -289,7 +289,7 @@ sparkCachePath <- function() {
 }
 
 # Length of the Spark cache specific relative path segments for each platform
-# eg. "Apache\Spark\Cache" is 3 in Windows, or "spark" is 1 in unix
+# e.g. "Apache\Spark\Cache" is 3 in Windows, or "spark" is 1 in unix
 # Must match sparkCachePath() exactly.
 sparkCacheRelPathLength <- function() {
   if (is_windows()) {
diff --git a/R/pkg/R/mllib_classification.R b/R/pkg/R/mllib_classification.R
index ec83b6bd406a7..093467ecf7d28 100644
--- a/R/pkg/R/mllib_classification.R
+++ b/R/pkg/R/mllib_classification.R
@@ -425,7 +425,7 @@ setMethod("write.ml", signature(object = "LogisticRegressionModel", path = "char
 #' predictions on new data, and \code{write.ml}/\code{read.ml} to save/load fitted models.
 #' Only categorical data is supported.
 #' For more details, see
-#' \href{http://spark.apache.org/docs/latest/ml-classification-regression.html}{
+#' \href{https://spark.apache.org/docs/latest/ml-classification-regression.html}{
 #'   Multilayer Perceptron}
 #'
 #' @param data a \code{SparkDataFrame} of observations and labels for model fitting.
diff --git a/R/pkg/R/mllib_fpm.R b/R/pkg/R/mllib_fpm.R
index 0cc7a16c302dc..65a43514930f0 100644
--- a/R/pkg/R/mllib_fpm.R
+++ b/R/pkg/R/mllib_fpm.R
@@ -122,11 +122,12 @@ setMethod("spark.freqItemsets", signature(object = "FPGrowthModel"),
 # Get association rules.
 
 #' @return A \code{SparkDataFrame} with association rules.
-#'         The \code{SparkDataFrame} contains four columns:
+#'         The \code{SparkDataFrame} contains five columns:
 #'         \code{antecedent} (an array of the same type as the input column),
 #'         \code{consequent} (an array of the same type as the input column),
-#'         \code{condfidence} (confidence for the rule)
-#'         and \code{lift} (lift for the rule)
+#'         \code{confidence} (confidence for the rule)
+#'         \code{lift} (lift for the rule)
+#'         and \code{support} (support for the rule)
 #' @rdname spark.fpGrowth
 #' @aliases associationRules,FPGrowthModel-method
 #' @note spark.associationRules(FPGrowthModel) since 2.2.0
diff --git a/R/pkg/R/mllib_recommendation.R b/R/pkg/R/mllib_recommendation.R
index d238ff93ed245..87a1bc991f812 100644
--- a/R/pkg/R/mllib_recommendation.R
+++ b/R/pkg/R/mllib_recommendation.R
@@ -30,7 +30,7 @@ setClass("ALSModel", representation(jobj = "jobj"))
 #' to make predictions on new data, and \code{write.ml}/\code{read.ml} to save/load fitted models.
 #'
 #' For more details, see
-#' \href{http://spark.apache.org/docs/latest/ml-collaborative-filtering.html}{MLlib:
+#' \href{https://spark.apache.org/docs/latest/ml-collaborative-filtering.html}{MLlib:
 #' Collaborative Filtering}.
 #'
 #' @param data a SparkDataFrame for training.
diff --git a/R/pkg/R/mllib_stat.R b/R/pkg/R/mllib_stat.R
index 6db4d5d4831dd..f82fb589bb5a5 100644
--- a/R/pkg/R/mllib_stat.R
+++ b/R/pkg/R/mllib_stat.R
@@ -49,7 +49,7 @@ setClass("KSTest", representation(jobj = "jobj"))
 #' @rdname spark.kstest
 #' @aliases spark.kstest,SparkDataFrame-method
 #' @name spark.kstest
-#' @seealso \href{http://spark.apache.org/docs/latest/mllib-statistics.html#hypothesis-testing}{
+#' @seealso \href{https://spark.apache.org/docs/latest/mllib-statistics.html#hypothesis-testing}{
 #'          MLlib: Hypothesis Testing}
 #' @examples
 #' \dontrun{
diff --git a/R/pkg/R/mllib_tree.R b/R/pkg/R/mllib_tree.R
index f6aa48f5fa04a..f3192ee9b1382 100644
--- a/R/pkg/R/mllib_tree.R
+++ b/R/pkg/R/mllib_tree.R
@@ -53,7 +53,7 @@ setClass("DecisionTreeRegressionModel", representation(jobj = "jobj"))
 #' @note DecisionTreeClassificationModel since 2.3.0
 setClass("DecisionTreeClassificationModel", representation(jobj = "jobj"))
 
-# Create the summary of a tree ensemble model (eg. Random Forest, GBT)
+# Create the summary of a tree ensemble model (e.g. Random Forest, GBT)
 summary.treeEnsemble <- function(model) {
   jobj <- model@jobj
   formula <- callJMethod(jobj, "formula")
@@ -73,7 +73,7 @@ summary.treeEnsemble <- function(model) {
        jobj = jobj)
 }
 
-# Prints the summary of tree ensemble models (eg. Random Forest, GBT)
+# Prints the summary of tree ensemble models (e.g. Random Forest, GBT)
 print.summary.treeEnsemble <- function(x) {
   jobj <- x$jobj
   cat("Formula: ", x$formula)
@@ -127,9 +127,9 @@ print.summary.decisionTree <- function(x) {
 #' \code{write.ml}/\code{read.ml} to save/load fitted models.
 #' For more details, see
 # nolint start
-#' \href{http://spark.apache.org/docs/latest/ml-classification-regression.html#gradient-boosted-tree-regression}{
+#' \href{https://spark.apache.org/docs/latest/ml-classification-regression.html#gradient-boosted-tree-regression}{
 #' GBT Regression} and
-#' \href{http://spark.apache.org/docs/latest/ml-classification-regression.html#gradient-boosted-tree-classifier}{
+#' \href{https://spark.apache.org/docs/latest/ml-classification-regression.html#gradient-boosted-tree-classifier}{
 #' GBT Classification}
 # nolint end
 #'
@@ -343,9 +343,9 @@ setMethod("write.ml", signature(object = "GBTClassificationModel", path = "chara
 #' save/load fitted models.
 #' For more details, see
 # nolint start
-#' \href{http://spark.apache.org/docs/latest/ml-classification-regression.html#random-forest-regression}{
+#' \href{https://spark.apache.org/docs/latest/ml-classification-regression.html#random-forest-regression}{
 #' Random Forest Regression} and
-#' \href{http://spark.apache.org/docs/latest/ml-classification-regression.html#random-forest-classifier}{
+#' \href{https://spark.apache.org/docs/latest/ml-classification-regression.html#random-forest-classifier}{
 #' Random Forest Classification}
 # nolint end
 #'
@@ -568,9 +568,9 @@ setMethod("write.ml", signature(object = "RandomForestClassificationModel", path
 #' save/load fitted models.
 #' For more details, see
 # nolint start
-#' \href{http://spark.apache.org/docs/latest/ml-classification-regression.html#decision-tree-regression}{
+#' \href{https://spark.apache.org/docs/latest/ml-classification-regression.html#decision-tree-regression}{
 #' Decision Tree Regression} and
-#' \href{http://spark.apache.org/docs/latest/ml-classification-regression.html#decision-tree-classifier}{
+#' \href{https://spark.apache.org/docs/latest/ml-classification-regression.html#decision-tree-classifier}{
 #' Decision Tree Classification}
 # nolint end
 #'
diff --git a/R/pkg/R/mllib_utils.R b/R/pkg/R/mllib_utils.R
index f38f1ac3a6b4c..d943d8d0ab4c0 100644
--- a/R/pkg/R/mllib_utils.R
+++ b/R/pkg/R/mllib_utils.R
@@ -18,7 +18,7 @@
 # mllib_utils.R: Utilities for MLlib integration
 
 # Integration with R's standard functions.
-# Most of MLlib's argorithms are provided in two flavours:
+# Most of MLlib's algorithms are provided in two flavours:
 # - a specialization of the default R methods (glm). These methods try to respect
 #   the inputs and the outputs of R's method to the largest extent, but some small differences
 #   may exist.
diff --git a/R/pkg/R/pairRDD.R b/R/pkg/R/pairRDD.R
index b29381bb900fb..41676be03e951 100644
--- a/R/pkg/R/pairRDD.R
+++ b/R/pkg/R/pairRDD.R
@@ -239,7 +239,7 @@ setMethod("partitionByRDD",
             javaPairRDD <- callJMethod(javaPairRDD, "partitionBy", rPartitioner)
 
             # Call .values() on the result to get back the final result, the
-            # shuffled acutal content key-val pairs.
+            # shuffled actual content key-val pairs.
             r <- callJMethod(javaPairRDD, "values")
 
             RDD(r, serializedMode = "byte")
@@ -411,7 +411,7 @@ setMethod("reduceByKeyLocally",
 #' \itemize{
 #'   \item createCombiner, which turns a V into a C (e.g., creates a one-element list)
 #'   \item mergeValue, to merge a V into a C (e.g., adds it to the end of a list) -
-#'   \item mergeCombiners, to combine two C's into a single one (e.g., concatentates
+#'   \item mergeCombiners, to combine two C's into a single one (e.g., concatenates
 #'    two lists).
 #' }
 #'
diff --git a/R/pkg/R/stats.R b/R/pkg/R/stats.R
index 7252351ebebb2..0aabceef226e3 100644
--- a/R/pkg/R/stats.R
+++ b/R/pkg/R/stats.R
@@ -109,7 +109,8 @@ setMethod("corr",
 #'
 #' Finding frequent items for columns, possibly with false positives.
 #' Using the frequent element count algorithm described in
-#' \url{https://doi.org/10.1145/762471.762473}, proposed by Karp, Schenker, and Papadimitriou.
+#' \url{https://dl.acm.org/doi/10.1145/762471.762473}, proposed by Karp, Schenker,
+#' and Papadimitriou.
 #'
 #' @param x A SparkDataFrame.
 #' @param cols A vector column names to search frequent items in.
diff --git a/R/pkg/R/streaming.R b/R/pkg/R/streaming.R
index 5eccbdc9d3818..2bcfb363f9d24 100644
--- a/R/pkg/R/streaming.R
+++ b/R/pkg/R/streaming.R
@@ -93,7 +93,7 @@ setMethod("explain",
 
 #' lastProgress
 #'
-#' Prints the most recent progess update of this streaming query in JSON format.
+#' Prints the most recent progress update of this streaming query in JSON format.
 #'
 #' @param x a StreamingQuery.
 #' @rdname lastProgress
diff --git a/R/pkg/R/types.R b/R/pkg/R/types.R
index 5d48a9eee2799..dfa83c35665ce 100644
--- a/R/pkg/R/types.R
+++ b/R/pkg/R/types.R
@@ -68,7 +68,7 @@ rToSQLTypes <- as.environment(list(
   "character" = "string",
   "logical" = "boolean"))
 
-# Helper function of coverting decimal type. When backend returns column type in the
+# Helper function of converting decimal type. When backend returns column type in the
 # format of decimal(,) (e.g., decimal(10, 0)), this function coverts the column type
 # as double type. This function converts backend returned types that are not the key
 # of PRIMITIVE_TYPES, but should be treated as PRIMITIVE_TYPES.
diff --git a/R/pkg/R/utils.R b/R/pkg/R/utils.R
index 65db9c21d9dbb..264cbfc9ba929 100644
--- a/R/pkg/R/utils.R
+++ b/R/pkg/R/utils.R
@@ -376,6 +376,7 @@ varargsToStrEnv <- function(...) {
 
 getStorageLevel <- function(newLevel = c("DISK_ONLY",
                                          "DISK_ONLY_2",
+                                         "DISK_ONLY_3",
                                          "MEMORY_AND_DISK",
                                          "MEMORY_AND_DISK_2",
                                          "MEMORY_AND_DISK_SER",
@@ -390,6 +391,7 @@ getStorageLevel <- function(newLevel = c("DISK_ONLY",
   storageLevel <- switch(newLevel,
                          "DISK_ONLY" = callJStatic(storageLevelClass, "DISK_ONLY"),
                          "DISK_ONLY_2" = callJStatic(storageLevelClass, "DISK_ONLY_2"),
+                         "DISK_ONLY_3" = callJStatic(storageLevelClass, "DISK_ONLY_3"),
                          "MEMORY_AND_DISK" = callJStatic(storageLevelClass, "MEMORY_AND_DISK"),
                          "MEMORY_AND_DISK_2" = callJStatic(storageLevelClass, "MEMORY_AND_DISK_2"),
                          "MEMORY_AND_DISK_SER" = callJStatic(storageLevelClass,
@@ -415,6 +417,8 @@ storageLevelToString <- function(levelObj) {
     "DISK_ONLY"
   } else if (useDisk && !useMemory && !useOffHeap && !deserialized && replication == 2) {
     "DISK_ONLY_2"
+  } else if (useDisk && !useMemory && !useOffHeap && !deserialized && replication == 3) {
+    "DISK_ONLY_3"
   } else if (!useDisk && useMemory && !useOffHeap && deserialized && replication == 1) {
     "MEMORY_ONLY"
   } else if (!useDisk && useMemory && !useOffHeap && deserialized && replication == 2) {
@@ -529,7 +533,10 @@ processClosure <- function(node, oldEnv, defVars, checkedFuncs, newEnv) {
         # Namespaces other than "SparkR" will not be searched.
         if (!isNamespace(func.env) ||
             (getNamespaceName(func.env) == "SparkR" &&
-               !(nodeChar %in% getNamespaceExports("SparkR")))) {
+               !(nodeChar %in% getNamespaceExports("SparkR")) &&
+                  # Note that generic S4 methods should not be set to the environment of
+                  # cleaned closure. It does not work with R 4.0.0+. See also SPARK-31918.
+                  nodeChar != "" && !methods::isGeneric(nodeChar, func.env))) {
           # Only include SparkR internals.
 
           # Set parameter 'inherits' to FALSE since we do not need to search in
@@ -923,7 +930,7 @@ getOne <- function(x, envir, inherits = TRUE, ifnotfound = NULL) {
 }
 
 # Returns a vector of parent directories, traversing up count times, starting with a full path
-# eg. traverseParentDirs("/Users/user/Library/Caches/spark/spark2.2", 1) should return
+# e.g. traverseParentDirs("/Users/user/Library/Caches/spark/spark2.2", 1) should return
 # this "/Users/user/Library/Caches/spark/spark2.2"
 # and  "/Users/user/Library/Caches/spark"
 traverseParentDirs <- function(x, count) {
diff --git a/R/pkg/inst/profile/general.R b/R/pkg/inst/profile/general.R
index 3efb460846fc2..8c75c19ca7ac3 100644
--- a/R/pkg/inst/profile/general.R
+++ b/R/pkg/inst/profile/general.R
@@ -16,10 +16,6 @@
 #
 
 .First <- function() {
-  if (utils::compareVersion(paste0(R.version$major, ".", R.version$minor), "3.4.0") == -1) {
-    warning("Support for R prior to version 3.4 is deprecated since Spark 3.0.0")
-  }
-
   packageDir <- Sys.getenv("SPARKR_PACKAGE_DIR")
   dirs <- strsplit(packageDir, ",")[[1]]
   .libPaths(c(dirs, .libPaths()))
diff --git a/R/pkg/inst/profile/shell.R b/R/pkg/inst/profile/shell.R
index e4e0d032997de..ffedb3038fd53 100644
--- a/R/pkg/inst/profile/shell.R
+++ b/R/pkg/inst/profile/shell.R
@@ -16,10 +16,6 @@
 #
 
 .First <- function() {
-  if (utils::compareVersion(paste0(R.version$major, ".", R.version$minor), "3.4.0") == -1) {
-    warning("Support for R prior to version 3.4 is deprecated since Spark 3.0.0")
-  }
-
   home <- Sys.getenv("SPARK_HOME")
   .libPaths(c(file.path(home, "R", "lib"), .libPaths()))
   Sys.setenv(NOAWT = 1)
@@ -47,5 +43,7 @@
   cat("      /_/", "\n")
   cat("\n")
 
-  cat("\nSparkSession available as 'spark'.\n")
+  cat("\nSparkSession Web UI available at", SparkR::sparkR.uiWebUrl())
+  cat("\nSparkSession available as 'spark'(master = ", unlist(SparkR::sparkR.conf("spark.master")),
+    ", app id = ", unlist(SparkR::sparkR.conf("spark.app.id")), ").", "\n", sep = "")
 }
diff --git a/R/pkg/inst/worker/daemon.R b/R/pkg/inst/worker/daemon.R
index fb9db63b07cd0..4589bb9c6ad1b 100644
--- a/R/pkg/inst/worker/daemon.R
+++ b/R/pkg/inst/worker/daemon.R
@@ -32,7 +32,7 @@ inputCon <- socketConnection(
 
 SparkR:::doServerAuth(inputCon, Sys.getenv("SPARKR_WORKER_SECRET"))
 
-# Waits indefinitely for a socket connecion by default.
+# Waits indefinitely for a socket connection by default.
 selectTimeout <- NULL
 
 while (TRUE) {
@@ -72,7 +72,7 @@ while (TRUE) {
       }
     })
   } else if (is.null(children)) {
-    # If it is NULL, there are no children. Waits indefinitely for a socket connecion.
+    # If it is NULL, there are no children. Waits indefinitely for a socket connection.
     selectTimeout <- NULL
   }
 
diff --git a/R/pkg/inst/worker/worker.R b/R/pkg/inst/worker/worker.R
index 1ef05ea621e83..7fc4680bad10e 100644
--- a/R/pkg/inst/worker/worker.R
+++ b/R/pkg/inst/worker/worker.R
@@ -85,7 +85,7 @@ outputResult <- function(serializer, output, outputCon) {
 }
 
 # Constants
-specialLengths <- list(END_OF_STERAM = 0L, TIMING_DATA = -1L)
+specialLengths <- list(END_OF_STREAM = 0L, TIMING_DATA = -1L)
 
 # Timing R process boot
 bootTime <- currentTimeSecs()
@@ -180,7 +180,7 @@ if (isEmpty != 0) {
     } else if (deserializer == "arrow" && mode == 1) {
       data <- SparkR:::readDeserializeInArrow(inputCon)
       # See https://stat.ethz.ch/pipermail/r-help/2010-September/252046.html
-      # rbind.fill might be an anternative to make it faster if plyr is installed.
+      # rbind.fill might be an alternative to make it faster if plyr is installed.
       # Also, note that, 'dapply' applies a function to each partition.
       data <- do.call("rbind", data)
     }
@@ -196,7 +196,7 @@ if (isEmpty != 0) {
         outputs <- list()
         for (i in seq_len(length(data))) {
           # Timing reading input data for execution
-          inputElap <- elapsedSecs()
+          computeStart <- elapsedSecs()
           output <- compute(mode, partition, serializer, deserializer, keys[[i]],
                       colNames, computeFunc, data[[i]])
           computeElap <- elapsedSecs()
@@ -204,17 +204,18 @@ if (isEmpty != 0) {
             outputs[[length(outputs) + 1L]] <- output
           } else {
             outputResult(serializer, output, outputCon)
+            outputComputeElapsDiff <- outputComputeElapsDiff + (elapsedSecs() - computeElap)
           }
-          outputElap <- elapsedSecs()
-          computeInputElapsDiff <-  computeInputElapsDiff + (computeElap - inputElap)
-          outputComputeElapsDiff <- outputComputeElapsDiff + (outputElap - computeElap)
+          computeInputElapsDiff <- computeInputElapsDiff + (computeElap - computeStart)
         }
 
         if (serializer == "arrow") {
           # See https://stat.ethz.ch/pipermail/r-help/2010-September/252046.html
-          # rbind.fill might be an anternative to make it faster if plyr is installed.
+          # rbind.fill might be an alternative to make it faster if plyr is installed.
+          outputStart <- elapsedSecs()
           combined <- do.call("rbind", outputs)
           SparkR:::writeSerializeInArrow(outputCon, combined)
+          outputComputeElapsDiff <- elapsedSecs() - outputStart
         }
       }
     } else {
@@ -285,7 +286,7 @@ SparkR:::writeDouble(outputCon, computeInputElapsDiff)    # compute
 SparkR:::writeDouble(outputCon, outputComputeElapsDiff)   # output
 
 # End of output
-SparkR:::writeInt(outputCon, specialLengths$END_OF_STERAM)
+SparkR:::writeInt(outputCon, specialLengths$END_OF_STREAM)
 
 close(outputCon)
 close(inputCon)
diff --git a/R/pkg/tests/fulltests/test_Serde.R b/R/pkg/tests/fulltests/test_Serde.R
index e01f6ee005218..a52289e43ca5e 100644
--- a/R/pkg/tests/fulltests/test_Serde.R
+++ b/R/pkg/tests/fulltests/test_Serde.R
@@ -125,7 +125,7 @@ test_that("SerDe of list of lists", {
 
 sparkR.session.stop()
 
-# Note that this test should be at the end of tests since the configruations used here are not
+# Note that this test should be at the end of tests since the configurations used here are not
 # specific to sessions, and the Spark context is restarted.
 test_that("createDataFrame large objects", {
   for (encryptionEnabled in list("true", "false")) {
diff --git a/R/pkg/tests/fulltests/test_context.R b/R/pkg/tests/fulltests/test_context.R
index 6be04b321e985..1add5a9fdde44 100644
--- a/R/pkg/tests/fulltests/test_context.R
+++ b/R/pkg/tests/fulltests/test_context.R
@@ -26,7 +26,9 @@ test_that("Check masked functions", {
                      "colnames", "colnames<-", "intersect", "rank", "rbind", "sample", "subset",
                      "summary", "transform", "drop", "window", "as.data.frame", "union", "not")
   version <- packageVersion("base")
-  if (as.numeric(version$major) >= 3 && as.numeric(version$minor) >= 3) {
+  is33Above <- as.numeric(version$major) >= 3 && as.numeric(version$minor) >= 3
+  is40Above <- as.numeric(version$major) >= 4
+  if (is33Above || is40Above) {
     namesOfMasked <- c("endsWith", "startsWith", namesOfMasked)
   }
   masked <- conflicts(detail = TRUE)$`package:SparkR`
@@ -137,7 +139,7 @@ test_that("utility function can be called", {
   expect_true(TRUE)
 })
 
-test_that("getClientModeSparkSubmitOpts() returns spark-submit args from whitelist", {
+test_that("getClientModeSparkSubmitOpts() returns spark-submit args from allowList", {
   e <- new.env()
   e[["spark.driver.memory"]] <- "512m"
   ops <- getClientModeSparkSubmitOpts("sparkrmain", e)
diff --git a/R/pkg/tests/fulltests/test_jvm_api.R b/R/pkg/tests/fulltests/test_jvm_api.R
index 8b3b4f73de170..3bf6ae556c079 100644
--- a/R/pkg/tests/fulltests/test_jvm_api.R
+++ b/R/pkg/tests/fulltests/test_jvm_api.R
@@ -20,11 +20,11 @@ context("JVM API")
 sparkSession <- sparkR.session(master = sparkRTestMaster, enableHiveSupport = FALSE)
 
 test_that("Create and call methods on object", {
-  jarr <- sparkR.newJObject("java.util.ArrayList")
+  jarray <- sparkR.newJObject("java.util.ArrayList")
   # Add an element to the array
-  sparkR.callJMethod(jarr, "add", 1L)
+  sparkR.callJMethod(jarray, "add", 1L)
   # Check if get returns the same element
-  expect_equal(sparkR.callJMethod(jarr, "get", 0L), 1L)
+  expect_equal(sparkR.callJMethod(jarray, "get", 0L), 1L)
 })
 
 test_that("Call static methods", {
diff --git a/R/pkg/tests/fulltests/test_mllib_classification.R b/R/pkg/tests/fulltests/test_mllib_classification.R
index 9dd275a173089..20339c947d7bf 100644
--- a/R/pkg/tests/fulltests/test_mllib_classification.R
+++ b/R/pkg/tests/fulltests/test_mllib_classification.R
@@ -34,18 +34,18 @@ test_that("spark.svmLinear", {
   summary <- summary(model)
 
   # test summary coefficients return matrix type
-  expect_true(class(summary$coefficients) == "matrix")
+  expect_true(any(class(summary$coefficients) == "matrix"))
   expect_true(class(summary$coefficients[, 1]) == "numeric")
 
   coefs <- summary$coefficients[, "Estimate"]
-  expected_coefs <- c(-0.06004978, -0.1563083, -0.460648, 0.2276626, 1.055085)
+  expected_coefs <- c(-6.8823988, -0.6154984, -1.5135447, 1.9694126, 3.3736856)
   expect_true(all(abs(coefs - expected_coefs) < 0.1))
 
   # Test prediction with string label
   prediction <- predict(model, training)
   expect_equal(typeof(take(select(prediction, "prediction"), 1)$prediction), "character")
-  expected <- c("versicolor", "versicolor", "versicolor", "virginica",  "virginica",
-                "virginica",  "virginica",  "virginica",  "virginica",  "virginica")
+  expected <- c("versicolor", "versicolor", "versicolor", "versicolor",  "versicolor",
+                "versicolor",  "versicolor",  "versicolor",  "versicolor",  "versicolor")
   expect_equal(sort(as.list(take(select(prediction, "prediction"), 10))[[1]]), expected)
 
   # Test model save and load
@@ -130,7 +130,7 @@ test_that("spark.logit", {
   summary <- summary(model)
 
   # test summary coefficients return matrix type
-  expect_true(class(summary$coefficients) == "matrix")
+  expect_true(any(class(summary$coefficients) == "matrix"))
   expect_true(class(summary$coefficients[, 1]) == "numeric")
 
   versicolorCoefsR <- c(1.52, 0.03, -0.53, 0.04, 0.00)
@@ -242,8 +242,8 @@ test_that("spark.logit", {
   # Test binomial logistic regression against two classes with upperBoundsOnCoefficients
   # and upperBoundsOnIntercepts
   u <- matrix(c(1.0, 0.0, 1.0, 0.0), nrow = 1, ncol = 4)
-  model <- spark.logit(training, Species ~ ., upperBoundsOnCoefficients = u,
-                       upperBoundsOnIntercepts = 1.0)
+  model <- suppressWarnings(spark.logit(training, Species ~ ., upperBoundsOnCoefficients = u,
+                                        upperBoundsOnIntercepts = 1.0))
   summary <- summary(model)
   coefsR <- c(-11.13331, 1.00000, 0.00000, 1.00000, 0.00000)
   coefs <- summary$coefficients[, "Estimate"]
@@ -255,8 +255,8 @@ test_that("spark.logit", {
   # Test binomial logistic regression against two classes with lowerBoundsOnCoefficients
   # and lowerBoundsOnIntercepts
   l <- matrix(c(0.0, -1.0, 0.0, -1.0), nrow = 1, ncol = 4)
-  model <- spark.logit(training, Species ~ ., lowerBoundsOnCoefficients = l,
-                       lowerBoundsOnIntercepts = 0.0)
+  model <- suppressWarnings(spark.logit(training, Species ~ ., lowerBoundsOnCoefficients = l,
+                                        lowerBoundsOnIntercepts = 0.0))
   summary <- summary(model)
   coefsR <- c(0, 0, -1, 0, 1.902192)
   coefs <- summary$coefficients[, "Estimate"]
@@ -268,9 +268,9 @@ test_that("spark.logit", {
   # Test multinomial logistic regression with lowerBoundsOnCoefficients
   # and lowerBoundsOnIntercepts
   l <- matrix(c(0.0, -1.0, 0.0, -1.0, 0.0, -1.0, 0.0, -1.0), nrow = 2, ncol = 4)
-  model <- spark.logit(training, Species ~ ., family = "multinomial",
-                       lowerBoundsOnCoefficients = l,
-                       lowerBoundsOnIntercepts = as.array(c(0.0, 0.0)))
+  model <- suppressWarnings(spark.logit(training, Species ~ ., family = "multinomial",
+                                        lowerBoundsOnCoefficients = l,
+                                        lowerBoundsOnIntercepts = as.array(c(0.0, 0.0))))
   summary <- summary(model)
   versicolorCoefsR <- c(42.639465, 7.258104, 14.330814, 16.298243, 11.716429)
   virginicaCoefsR <- c(0.0002970796, 4.79274, 7.65047, 25.72793, 30.0021)
diff --git a/R/pkg/tests/fulltests/test_mllib_clustering.R b/R/pkg/tests/fulltests/test_mllib_clustering.R
index 028ad574b8134..f180aeea28150 100644
--- a/R/pkg/tests/fulltests/test_mllib_clustering.R
+++ b/R/pkg/tests/fulltests/test_mllib_clustering.R
@@ -171,7 +171,7 @@ test_that("spark.kmeans", {
   expect_equal(sort(collect(distinct(select(cluster, "prediction")))$prediction), c(0, 1))
 
   # test summary coefficients return matrix type
-  expect_true(class(summary.model$coefficients) == "matrix")
+  expect_true(any(class(summary.model$coefficients) == "matrix"))
   expect_true(class(summary.model$coefficients[1, ]) == "numeric")
 
   # Test model save/load
diff --git a/R/pkg/tests/fulltests/test_mllib_fpm.R b/R/pkg/tests/fulltests/test_mllib_fpm.R
index bc1e17538d41a..78d26d3324473 100644
--- a/R/pkg/tests/fulltests/test_mllib_fpm.R
+++ b/R/pkg/tests/fulltests/test_mllib_fpm.R
@@ -45,7 +45,8 @@ test_that("spark.fpGrowth", {
     antecedent = I(list(list("2"), list("3"))),
     consequent = I(list(list("1"), list("1"))),
     confidence = c(1, 1),
-    lift = c(1, 1)
+    lift = c(1, 1),
+    support = c(0.75, 0.5)
   )
 
   expect_equivalent(expected_association_rules, collect(spark.associationRules(model)))
diff --git a/R/pkg/tests/fulltests/test_mllib_regression.R b/R/pkg/tests/fulltests/test_mllib_regression.R
index 0f2a62b7229ca..b281cd6235ef0 100644
--- a/R/pkg/tests/fulltests/test_mllib_regression.R
+++ b/R/pkg/tests/fulltests/test_mllib_regression.R
@@ -116,7 +116,7 @@ test_that("spark.glm summary", {
   rStats <- summary(glm(Sepal.Width ~ Sepal.Length + Species, data = dataset))
 
   # test summary coefficients return matrix type
-  expect_true(class(stats$coefficients) == "matrix")
+  expect_true(any(class(stats$coefficients) == "matrix"))
   expect_true(class(stats$coefficients[, 1]) == "numeric")
 
   coefs <- stats$coefficients
diff --git a/R/pkg/tests/fulltests/test_sparkSQL.R b/R/pkg/tests/fulltests/test_sparkSQL.R
index 611d9057c0f13..30daa2064355d 100644
--- a/R/pkg/tests/fulltests/test_sparkSQL.R
+++ b/R/pkg/tests/fulltests/test_sparkSQL.R
@@ -1397,7 +1397,8 @@ test_that("column operators", {
 test_that("column functions", {
   c <- column("a")
   c1 <- abs(c) + acos(c) + approx_count_distinct(c) + ascii(c) + asin(c) + atan(c)
-  c2 <- avg(c) + base64(c) + bin(c) + bitwiseNOT(c) + cbrt(c) + ceil(c) + cos(c)
+  c2 <- avg(c) + base64(c) + bin(c) + suppressWarnings(bitwiseNOT(c)) +
+    bitwise_not(c) + cbrt(c) + ceil(c) + cos(c)
   c3 <- cosh(c) + count(c) + crc32(c) + hash(c) + exp(c)
   c4 <- explode(c) + expm1(c) + factorial(c) + first(c) + floor(c) + hex(c)
   c5 <- hour(c) + initcap(c) + last(c) + last_day(c) + length(c)
@@ -1405,7 +1406,8 @@ test_that("column functions", {
   c7 <- mean(c) + min(c) + month(c) + negate(c) + posexplode(c) + quarter(c)
   c8 <- reverse(c) + rint(c) + round(c) + rtrim(c) + sha1(c) + monotonically_increasing_id()
   c9 <- signum(c) + sin(c) + sinh(c) + size(c) + stddev(c) + soundex(c) + sqrt(c) + sum(c)
-  c10 <- sumDistinct(c) + tan(c) + tanh(c) + degrees(c) + radians(c)
+  c10 <- suppressWarnings(sumDistinct(c)) + sum_distinct(c) + tan(c) + tanh(c) +
+    degrees(c) + radians(c)
   c11 <- to_date(c) + trim(c) + unbase64(c) + unhex(c) + upper(c)
   c12 <- variance(c) + xxhash64(c) + ltrim(c, "a") + rtrim(c, "b") + trim(c, "c")
   c13 <- lead("col", 1) + lead(c, 1) + lag("col", 1) + lag(c, 1)
@@ -1424,6 +1426,15 @@ test_that("column functions", {
     date_trunc("quarter", c) + current_date() + current_timestamp()
   c25 <- overlay(c1, c2, c3, c3) + overlay(c1, c2, c3) + overlay(c1, c2, 1) +
     overlay(c1, c2, 3, 4)
+  c26 <- timestamp_seconds(c1) + vector_to_array(c) +
+    vector_to_array(c, "float32") + vector_to_array(c, "float64") +
+    array_to_vector(c)
+  c27 <- nth_value("x", 1L) + nth_value("y", 2, TRUE) +
+    nth_value(column("v"), 3) + nth_value(column("z"), 4L, FALSE)
+  c28 <- asc_nulls_first(c1) + asc_nulls_last(c1) +
+    desc_nulls_first(c1) + desc_nulls_last(c1)
+  c29 <- acosh(c1) + asinh(c1) + atanh(c1)
+  c30 <- product(c1) + product(c1 * 0.5)
 
   # Test if base::is.nan() is exposed
   expect_equal(is.nan(c("a", "b")), c(FALSE, FALSE))
@@ -1449,6 +1460,8 @@ test_that("column functions", {
   expect_equal(collect(df3)[[2, 1]], FALSE)
   expect_equal(collect(df3)[[3, 1]], TRUE)
 
+  df4 <- select(df, count_distinct(df$age, df$name))
+  expect_equal(collect(df4)[[1, 1]], 2)
   df4 <- select(df, countDistinct(df$age, df$name))
   expect_equal(collect(df4)[[1, 1]], 2)
 
@@ -1676,9 +1689,9 @@ test_that("column functions", {
 
   df <- as.DataFrame(list(list("col" = "1")))
   c <- collect(select(df, schema_of_csv("Amsterdam,2018")))
-  expect_equal(c[[1]], "struct<_c0:string,_c1:int>")
+  expect_equal(c[[1]], "STRUCT<`_c0`: STRING, `_c1`: INT>")
   c <- collect(select(df, schema_of_csv(lit("Amsterdam,2018"))))
-  expect_equal(c[[1]], "struct<_c0:string,_c1:int>")
+  expect_equal(c[[1]], "STRUCT<`_c0`: STRING, `_c1`: INT>")
 
   # Test to_json(), from_json(), schema_of_json()
   df <- sql("SELECT array(named_struct('name', 'Bob'), named_struct('name', 'Alice')) as people")
@@ -1711,9 +1724,9 @@ test_that("column functions", {
 
   df <- as.DataFrame(list(list("col" = "1")))
   c <- collect(select(df, schema_of_json('{"name":"Bob"}')))
-  expect_equal(c[[1]], "struct<name:string>")
+  expect_equal(c[[1]], "STRUCT<`name`: STRING>")
   c <- collect(select(df, schema_of_json(lit('{"name":"Bob"}'))))
-  expect_equal(c[[1]], "struct<name:string>")
+  expect_equal(c[[1]], "STRUCT<`name`: STRING>")
 
   # Test to_json() supports arrays of primitive types and arrays
   df <- sql("SELECT array(19, 42, 70) as age")
@@ -1803,6 +1816,62 @@ test_that("column functions", {
   )
 
   expect_equal(actual, expected)
+
+  # Test withField
+  lines <- c("{\"Person\": {\"name\":\"Bob\", \"age\":24, \"height\": 170}}")
+  jsonPath <- tempfile(pattern = "sparkr-test", fileext = ".tmp")
+  writeLines(lines, jsonPath)
+  df <- read.df(jsonPath, "json")
+  result <- collect(
+      select(
+          select(df, alias(withField(df$Person, "dummy", lit(42)), "Person")),
+          "Person.dummy"
+      )
+  )
+  expect_equal(result, data.frame(dummy = 42))
+
+  # Test dropFields
+  expect_setequal(
+    colnames(select(
+      withColumn(df, "Person", dropFields(df$Person, "age")),
+      column("Person.*")
+    )),
+    c("name", "height")
+  )
+
+  expect_equal(
+    colnames(select(
+      withColumn(df, "Person", dropFields(df$Person, "height", "name")),
+      column("Person.*")
+    )),
+    "age"
+  )
+})
+
+test_that("avro column functions", {
+  skip_if_not(
+    grepl("spark-avro", sparkR.conf("spark.jars", "")),
+    "spark-avro jar not present"
+  )
+
+  schema <- '{"namespace": "example.avro",
+    "type": "record",
+    "name": "User",
+    "fields": [
+      {"name": "name", "type": "string"},
+      {"name": "favorite_color", "type": ["string", "null"]}
+    ]
+  }'
+
+  c0 <- column("foo")
+  c1 <- from_avro(c0, schema)
+  expect_s4_class(c1, "Column")
+  c2 <- from_avro("foo", schema)
+  expect_s4_class(c2, "Column")
+  c3 <- to_avro(c1)
+  expect_s4_class(c3, "Column")
+  c4 <- to_avro(c1, schema)
+  expect_s4_class(c4, "Column")
 })
 
 test_that("column binary mathfunctions", {
@@ -1823,9 +1892,12 @@ test_that("column binary mathfunctions", {
   expect_equal(collect(select(df, hypot(df$a, df$b)))[3, "HYPOT(a, b)"], sqrt(3^2 + 7^2))
   expect_equal(collect(select(df, hypot(df$a, df$b)))[4, "HYPOT(a, b)"], sqrt(4^2 + 8^2))
   ## nolint end
-  expect_equal(collect(select(df, shiftLeft(df$b, 1)))[4, 1], 16)
-  expect_equal(collect(select(df, shiftRight(df$b, 1)))[4, 1], 4)
-  expect_equal(collect(select(df, shiftRightUnsigned(df$b, 1)))[4, 1], 4)
+  expect_equal(collect(select(df, shiftleft(df$b, 1)))[4, 1], 16)
+  expect_equal(collect(select(df, shiftright(df$b, 1)))[4, 1], 4)
+  expect_equal(collect(select(df, shiftrightunsigned(df$b, 1)))[4, 1], 4)
+  expect_equal(collect(select(df, suppressWarnings(shiftLeft(df$b, 1))))[4, 1], 16)
+  expect_equal(collect(select(df, suppressWarnings(shiftRight(df$b, 1))))[4, 1], 4)
+  expect_equal(collect(select(df, suppressWarnings(shiftRightUnsigned(df$b, 1))))[4, 1], 4)
   expect_equal(class(collect(select(df, rand()))[2, 1]), "numeric")
   expect_equal(collect(select(df, rand(1)))[1, 1], 0.636, tolerance = 0.01)
   expect_equal(class(collect(select(df, randn()))[2, 1]), "numeric")
@@ -2030,7 +2102,7 @@ test_that("higher order functions", {
     createDataFrame(data.frame(id = 1)),
     expr("CAST(array(1.0, 2.0, -3.0, -4.0) AS array<double>) xs"),
     expr("CAST(array(0.0, 3.0, 48.0) AS array<double>) ys"),
-    expr("array('FAILED', 'SUCCEDED') as vs"),
+    expr("array('FAILED', 'SUCCEEDED') as vs"),
     expr("map('foo', 1, 'bar', 2) as mx"),
     expr("map('foo', 42, 'bar', -1, 'baz', 0) as my")
   )
@@ -2089,6 +2161,20 @@ test_that("higher order functions", {
   expect_error(array_transform("xs", function(...) 42))
 })
 
+test_that("SPARK-34794: lambda vars must be resolved properly in nested higher order functions", {
+  df <- sql("SELECT array(1, 2, 3) as numbers, array('a', 'b', 'c') as letters")
+  ret <- first(select(
+    df,
+    array_transform("numbers", function(number) {
+      array_transform("letters", function(latter) {
+        struct(alias(number, "n"), alias(latter, "l"))
+      })
+    })
+  ))
+
+  expect_equal(1, ret[[1]][[1]][[1]][[1]]$n)
+})
+
 test_that("group by, agg functions", {
   df <- read.json(jsonPath)
   df1 <- agg(df, name = "max", age = "sum")
@@ -2113,7 +2199,7 @@ test_that("group by, agg functions", {
   df3 <- agg(gd, age = "stddev")
   expect_is(df3, "SparkDataFrame")
   df3_local <- collect(df3)
-  expect_true(is.nan(df3_local[df3_local$name == "Andy", ][1, 2]))
+  expect_true(is.na(df3_local[df3_local$name == "Andy", ][1, 2]))
 
   df4 <- agg(gd, sumAge = sum(df$age))
   expect_is(df4, "SparkDataFrame")
@@ -2144,7 +2230,7 @@ test_that("group by, agg functions", {
   df7 <- agg(gd2, value = "stddev")
   df7_local <- collect(df7)
   expect_true(abs(df7_local[df7_local$name == "ID1", ][1, 2] - 6.928203) < 1e-6)
-  expect_true(is.nan(df7_local[df7_local$name == "ID2", ][1, 2]))
+  expect_true(is.na(df7_local[df7_local$name == "ID2", ][1, 2]))
 
   mockLines3 <- c("{\"name\":\"Andy\", \"age\":30}",
                   "{\"name\":\"Andy\", \"age\":30}",
@@ -2696,6 +2782,19 @@ test_that("union(), unionByName(), rbind(), except(), and intersect() on a DataF
   expect_error(rbind(df, df2, df3),
                "Names of input data frames are different.")
 
+
+  df4 <- unionByName(df2, select(df2, "age"), TRUE)
+
+  expect_equal(
+      sum(collect(
+          select(df4, alias(isNull(df4$name), "missing_name")
+      ))$missing_name),
+      3
+  )
+
+  testthat::expect_error(unionByName(df2, select(df2, "age"), FALSE))
+  testthat::expect_error(unionByName(df2, select(df2, "age")))
+
   excepted <- arrange(except(df, df2), desc(df$age))
   expect_is(unioned, "SparkDataFrame")
   expect_equal(count(excepted), 2)
@@ -2807,6 +2906,15 @@ test_that("mutate(), transform(), rename() and names()", {
   expect_equal(nrow(result), 153)
   expect_equal(ncol(result), 2)
   detach(airquality)
+
+  # ensure long inferred names are handled without error (SPARK-26199)
+  #   test implicitly assumes eval(formals(deparse)$width.cutoff) = 60
+  #   (which has always been true as of 2020-11-15)
+  newDF <- mutate(
+    df,
+    df$age + 12345678901234567890 + 12345678901234567890 + 12345678901234
+  )
+  expect_match(tail(columns(newDF), 1L), "234567890", fixed = TRUE)
 })
 
 test_that("read/write ORC files", {
@@ -3196,6 +3304,12 @@ test_that("attach() on a DataFrame", {
   stat3 <- summary(df[, "age", drop = F])
   expect_equal(collect(stat3)[8, "age"], "30")
   expect_error(age)
+
+  # attach method uses deparse(); ensure no errors from a very long input
+  abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnop <- df # nolint
+  attach(abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnop)
+  expect_true(any(grepl("abcdefghijklmnopqrstuvwxyz", search())))
+  detach("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnop")
 })
 
 test_that("with() on a DataFrame", {
@@ -3591,7 +3705,7 @@ test_that("gapply() and gapplyCollect() on a DataFrame", {
     }
 
     # Computes the arithmetic mean of the second column by grouping
-    # on the first and third columns. Output the groupping value and the average.
+    # on the first and third columns. Output the grouping value and the average.
     schema <-  structType(structField("a", "integer"), structField("c", "string"),
                           structField("avg", "double"))
     df3 <- gapply(
@@ -3889,15 +4003,34 @@ test_that("catalog APIs, listTables, listColumns, listFunctions", {
                paste("Error in listFunctions : analysis error - Database",
                      "'zxwtyswklpf_db' does not exist"))
 
-  # recoverPartitions does not work with tempory view
+  # recoverPartitions does not work with temporary view
   expect_error(recoverPartitions("cars"),
-               "no such table - Table or view 'cars' not found in database 'default'")
+               paste("Error in recoverPartitions : analysis error - cars is a temp view.",
+                     "'recoverPartitions()' expects a table"), fixed = TRUE)
   expect_error(refreshTable("cars"), NA)
   expect_error(refreshByPath("/"), NA)
 
   dropTempView("cars")
 })
 
+test_that("assert_true, raise_error", {
+  df <- read.json(jsonPath)
+  filtered <- filter(df, "age < 20")
+
+  expect_equal(collect(select(filtered, assert_true(filtered$age < 20)))$age, c(NULL))
+  expect_equal(collect(select(filtered, assert_true(filtered$age < 20, "error message")))$age,
+               c(NULL))
+  expect_equal(collect(select(filtered, assert_true(filtered$age < 20, filtered$name)))$age,
+               c(NULL))
+  expect_error(collect(select(df, assert_true(df$age < 20))), "is not true!")
+  expect_error(collect(select(df, assert_true(df$age < 20, "error message"))),
+               "error message")
+  expect_error(collect(select(df, assert_true(df$age < 20, df$name))), "Michael")
+
+  expect_error(collect(select(filtered, raise_error("error message"))), "error message")
+  expect_error(collect(select(filtered, raise_error(filtered$name))), "Justin")
+})
+
 compare_list <- function(list1, list2) {
   # get testthat to show the diff by first making the 2 lists equal in length
   expect_equal(length(list1), length(list2))
@@ -3921,14 +4054,14 @@ test_that("No extra files are created in SPARK_HOME by starting session and maki
   # before creating a SparkSession with enableHiveSupport = T at the top of this test file
   # (filesBefore). The test here is to compare that (filesBefore) against the list of files before
   # any test is run in run-all.R (sparkRFilesBefore).
-  # sparkRWhitelistSQLDirs is also defined in run-all.R, and should contain only 2 whitelisted dirs,
+  # sparkRAllowedSQLDirs is also defined in run-all.R, and should contain only 2 allowed dirs,
   # here allow the first value, spark-warehouse, in the diff, everything else should be exactly the
   # same as before any test is run.
-  compare_list(sparkRFilesBefore, setdiff(filesBefore, sparkRWhitelistSQLDirs[[1]]))
+  compare_list(sparkRFilesBefore, setdiff(filesBefore, sparkRAllowedSQLDirs[[1]]))
   # third, ensure only spark-warehouse and metastore_db are created when enableHiveSupport = T
   # note: as the note above, after running all tests in this file while enableHiveSupport = T, we
-  # check the list of files again. This time we allow both whitelisted dirs to be in the diff.
-  compare_list(sparkRFilesBefore, setdiff(filesAfter, sparkRWhitelistSQLDirs))
+  # check the list of files again. This time we allow both dirs to be in the diff.
+  compare_list(sparkRFilesBefore, setdiff(filesAfter, sparkRAllowedSQLDirs))
 })
 
 unlink(parquetPath)
diff --git a/R/pkg/tests/fulltests/test_sparkSQL_arrow.R b/R/pkg/tests/fulltests/test_sparkSQL_arrow.R
index 97972753a78fa..06743488fdf11 100644
--- a/R/pkg/tests/fulltests/test_sparkSQL_arrow.R
+++ b/R/pkg/tests/fulltests/test_sparkSQL_arrow.R
@@ -19,7 +19,10 @@ library(testthat)
 
 context("SparkSQL Arrow optimization")
 
-sparkSession <- sparkR.session(master = sparkRTestMaster, enableHiveSupport = FALSE)
+sparkSession <- sparkR.session(
+  master = sparkRTestMaster,
+  enableHiveSupport = FALSE,
+  sparkConfig = list(spark.sql.execution.arrow.sparkr.enabled = "true"))
 
 test_that("createDataFrame/collect Arrow optimization", {
   skip_if_not_installed("arrow")
@@ -35,29 +38,13 @@ test_that("createDataFrame/collect Arrow optimization", {
     callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
   })
 
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    expect_equal(collect(createDataFrame(mtcars)), expected)
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  expect_equal(collect(createDataFrame(mtcars)), expected)
 })
 
 test_that("createDataFrame/collect Arrow optimization - many partitions (partition order test)", {
   skip_if_not_installed("arrow")
-
-  conf <- callJMethod(sparkSession, "conf")
-  arrowEnabled <- sparkR.conf("spark.sql.execution.arrow.sparkr.enabled")[[1]]
-
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    expect_equal(collect(createDataFrame(mtcars, numPartitions = 32)),
-                 collect(createDataFrame(mtcars, numPartitions = 1)))
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  expect_equal(collect(createDataFrame(mtcars, numPartitions = 32)),
+               collect(createDataFrame(mtcars, numPartitions = 1)))
 })
 
 test_that("createDataFrame/collect Arrow optimization - type specification", {
@@ -81,13 +68,7 @@ test_that("createDataFrame/collect Arrow optimization - type specification", {
     callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
   })
 
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    expect_equal(collect(createDataFrame(rdf)), expected)
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  expect_equal(collect(createDataFrame(rdf)), expected)
 })
 
 test_that("dapply() Arrow optimization", {
@@ -98,36 +79,30 @@ test_that("dapply() Arrow optimization", {
   arrowEnabled <- sparkR.conf("spark.sql.execution.arrow.sparkr.enabled")[[1]]
 
   callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "false")
-  tryCatch({
-    ret <- dapply(df,
-    function(rdf) {
-      stopifnot(is.data.frame(rdf))
-      rdf
-    },
-    schema(df))
-    expected <- collect(ret)
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
-
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
   tryCatch({
     ret <- dapply(df,
                   function(rdf) {
                     stopifnot(is.data.frame(rdf))
-                    # mtcars' hp is more then 50.
-                    stopifnot(all(rdf$hp > 50))
                     rdf
                   },
                   schema(df))
-    actual <- collect(ret)
-    expect_equal(actual, expected)
-    expect_equal(count(ret), nrow(mtcars))
+    expected <- collect(ret)
   },
   finally = {
     callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
   })
+
+  ret <- dapply(df,
+                function(rdf) {
+                  stopifnot(is.data.frame(rdf))
+                  # mtcars' hp is more then 50.
+                  stopifnot(all(rdf$hp > 50))
+                  rdf
+                },
+                schema(df))
+  actual <- collect(ret)
+  expect_equal(actual, expected)
+  expect_equal(count(ret), nrow(mtcars))
 })
 
 test_that("dapply() Arrow optimization - type specification", {
@@ -154,15 +129,9 @@ test_that("dapply() Arrow optimization - type specification", {
     callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
   })
 
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    ret <- dapply(df, function(rdf) { rdf }, schema(df))
-    actual <- collect(ret)
-    expect_equal(actual, expected)
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  ret <- dapply(df, function(rdf) { rdf }, schema(df))
+  actual <- collect(ret)
+  expect_equal(actual, expected)
 })
 
 test_that("dapply() Arrow optimization - type specification (date and timestamp)", {
@@ -170,18 +139,8 @@ test_that("dapply() Arrow optimization - type specification (date and timestamp)
   rdf <- data.frame(list(list(a = as.Date("1990-02-24"),
                               b = as.POSIXct("1990-02-24 12:34:56"))))
   df <- createDataFrame(rdf)
-
-  conf <- callJMethod(sparkSession, "conf")
-  arrowEnabled <- sparkR.conf("spark.sql.execution.arrow.sparkr.enabled")[[1]]
-
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    ret <- dapply(df, function(rdf) { rdf }, schema(df))
-    expect_equal(collect(ret), rdf)
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  ret <- dapply(df, function(rdf) { rdf }, schema(df))
+  expect_equal(collect(ret), rdf)
 })
 
 test_that("gapply() Arrow optimization", {
@@ -209,28 +168,22 @@ test_that("gapply() Arrow optimization", {
     callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
   })
 
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    ret <- gapply(df,
-                 "gear",
-                 function(key, grouped) {
-                   if (length(key) > 0) {
-                     stopifnot(is.numeric(key[[1]]))
-                   }
-                   stopifnot(is.data.frame(grouped))
-                   stopifnot(length(colnames(grouped)) == 11)
-                   # mtcars' hp is more then 50.
-                   stopifnot(all(grouped$hp > 50))
-                   grouped
-                 },
-                 schema(df))
-    actual <- collect(ret)
-    expect_equal(actual, expected)
-    expect_equal(count(ret), nrow(mtcars))
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  ret <- gapply(df,
+               "gear",
+               function(key, grouped) {
+                 if (length(key) > 0) {
+                   stopifnot(is.numeric(key[[1]]))
+                 }
+                 stopifnot(is.data.frame(grouped))
+                 stopifnot(length(colnames(grouped)) == 11)
+                 # mtcars' hp is more then 50.
+                 stopifnot(all(grouped$hp > 50))
+                 grouped
+               },
+               schema(df))
+  actual <- collect(ret)
+  expect_equal(actual, expected)
+  expect_equal(count(ret), nrow(mtcars))
 })
 
 test_that("gapply() Arrow optimization - type specification", {
@@ -250,26 +203,19 @@ test_that("gapply() Arrow optimization - type specification", {
   callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "false")
   tryCatch({
     ret <- gapply(df,
-                 "a",
-                 function(key, grouped) { grouped }, schema(df))
+                  "a",
+                  function(key, grouped) { grouped }, schema(df))
     expected <- collect(ret)
   },
   finally = {
     callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
   })
 
-
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    ret <- gapply(df,
-                 "a",
-                 function(key, grouped) { grouped }, schema(df))
-    actual <- collect(ret)
-    expect_equal(actual, expected)
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  ret <- gapply(df,
+                "a",
+                function(key, grouped) { grouped }, schema(df))
+  actual <- collect(ret)
+  expect_equal(actual, expected)
 })
 
 test_that("gapply() Arrow optimization - type specification (date and timestamp)", {
@@ -277,39 +223,30 @@ test_that("gapply() Arrow optimization - type specification (date and timestamp)
   rdf <- data.frame(list(list(a = as.Date("1990-02-24"),
                               b = as.POSIXct("1990-02-24 12:34:56"))))
   df <- createDataFrame(rdf)
+  ret <- gapply(df,
+                "a",
+                function(key, grouped) { grouped }, schema(df))
+  expect_equal(collect(ret), rdf)
+})
 
-  conf <- callJMethod(sparkSession, "conf")
-  arrowEnabled <- sparkR.conf("spark.sql.execution.arrow.sparkr.enabled")[[1]]
+test_that("Arrow optimization - unsupported types", {
+  skip_if_not_installed("arrow")
 
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    ret <- gapply(df,
-                  "a",
-                  function(key, grouped) { grouped }, schema(df))
-    expect_equal(collect(ret), rdf)
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  expect_error(checkSchemaInArrow(structType("a FLOAT")), "not support float type")
+  expect_error(checkSchemaInArrow(structType("a BINARY")), "not support binary type")
+  expect_error(checkSchemaInArrow(structType("a ARRAY<INT>")), "not support array type")
+  expect_error(checkSchemaInArrow(structType("a MAP<INT, INT>")), "not support map type")
+  expect_error(checkSchemaInArrow(structType("a STRUCT<a: INT>")),
+               "not support nested struct type")
 })
 
-test_that("Arrow optimization - unsupported types", {
+test_that("SPARK-32478: gapply() Arrow optimization - error message for schema mismatch", {
   skip_if_not_installed("arrow")
+  df <- createDataFrame(list(list(a = 1L, b = "a")))
 
-  conf <- callJMethod(sparkSession, "conf")
-  arrowEnabled <- sparkR.conf("spark.sql.execution.arrow.sparkr.enabled")[[1]]
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    expect_error(checkSchemaInArrow(structType("a FLOAT")), "not support float type")
-    expect_error(checkSchemaInArrow(structType("a BINARY")), "not support binary type")
-    expect_error(checkSchemaInArrow(structType("a ARRAY<INT>")), "not support array type")
-    expect_error(checkSchemaInArrow(structType("a MAP<INT, INT>")), "not support map type")
-    expect_error(checkSchemaInArrow(structType("a STRUCT<a: INT>")),
-                 "not support nested struct type")
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  expect_error(
+    count(gapply(df, "a", function(key, group) { group }, structType("a int, b int"))),
+    "expected IntegerType, IntegerType, got IntegerType, StringType")
 })
 
 sparkR.session.stop()
diff --git a/R/pkg/tests/fulltests/test_utils.R b/R/pkg/tests/fulltests/test_utils.R
index c3fb9046fcda4..6c83a137cfb7b 100644
--- a/R/pkg/tests/fulltests/test_utils.R
+++ b/R/pkg/tests/fulltests/test_utils.R
@@ -116,7 +116,7 @@ test_that("cleanClosure on R functions", {
   actual <- get("y", envir = env, inherits = FALSE)
   expect_equal(actual, y)
 
-  # Test for combination for nested and sequenctial functions in a closure
+  # Test for combination for nested and sequential functions in a closure
   f1 <- function(x) x + 1
   f2 <- function(x) f1(x) + 2
   userFunc <- function(x) { f1(x); f2(x) }
diff --git a/R/pkg/tests/run-all.R b/R/pkg/tests/run-all.R
index bf02ecdad66ff..f9e266eb4e014 100644
--- a/R/pkg/tests/run-all.R
+++ b/R/pkg/tests/run-all.R
@@ -35,8 +35,8 @@ if (identical(Sys.getenv("NOT_CRAN"), "true")) {
   install.spark(overwrite = TRUE)
 
   sparkRDir <- file.path(Sys.getenv("SPARK_HOME"), "R")
-  sparkRWhitelistSQLDirs <- c("spark-warehouse", "metastore_db")
-  invisible(lapply(sparkRWhitelistSQLDirs,
+  sparkRAllowedSQLDirs <- c("spark-warehouse", "metastore_db")
+  invisible(lapply(sparkRAllowedSQLDirs,
                    function(x) { unlink(file.path(sparkRDir, x), recursive = TRUE, force = TRUE)}))
   sparkRFilesBefore <- list.files(path = sparkRDir, all.files = TRUE)
 
@@ -60,22 +60,37 @@ if (identical(Sys.getenv("NOT_CRAN"), "true")) {
     # set random seed for predictable results. mostly for base's sample() in tree and classification
     set.seed(42)
 
-    # TODO (SPARK-30663) To be removed once testthat 1.x is removed from all builds
-    if (grepl("^1\\..*", packageVersion("testthat"))) {
-      # testthat 1.x
-      test_runner <- testthat:::run_tests
-      reporter <- "summary"
+    if (packageVersion("testthat")$major <= 1) stop("testhat 1.x is not supported")
 
+    test_runner <- if (packageVersion("testthat")$major == 2) {
+      # testthat >= 2.0.0, < 3.0.0
+      function(path, package, reporter, filter) {
+        testthat:::test_package_dir(
+          test_path = path,
+          package = package,
+          filter = filter,
+          reporter = reporter
+        )
+      }
     } else {
-      # testthat >= 2.0.0
-      test_runner <- testthat:::test_package_dir
-      reporter <- testthat::default_reporter()
+      # testthat >= 3.0.0
+      testthat::test_dir
     }
 
-    test_runner("SparkR",
-                file.path(sparkRDir, "pkg", "tests", "fulltests"),
-                NULL,
-                reporter)
+    dir.create("target/test-reports", showWarnings = FALSE)
+    reporter <- MultiReporter$new(list(
+      SummaryReporter$new(),
+      JunitReporter$new(
+        file = file.path(getwd(), "target/test-reports/test-results.xml")
+      )
+    ))
+
+    test_runner(
+      path = file.path(sparkRDir, "pkg", "tests", "fulltests"),
+      package = "SparkR",
+      reporter = reporter,
+      filter = NULL
+    )
   }
 
   SparkR:::uninstallDownloadedSpark()
diff --git a/R/pkg/vignettes/sparkr-vignettes.Rmd b/R/pkg/vignettes/sparkr-vignettes.Rmd
index 3713e6c784855..0ed0028eb5173 100644
--- a/R/pkg/vignettes/sparkr-vignettes.Rmd
+++ b/R/pkg/vignettes/sparkr-vignettes.Rmd
@@ -146,7 +146,7 @@ sparkR.session.stop()
 
 Different from many other R packages, to use SparkR, you need an additional installation of Apache Spark. The Spark installation will be used to run a backend process that will compile and execute SparkR programs.
 
-After installing the SparkR package, you can call `sparkR.session` as explained in the previous section to start and it will check for the Spark installation. If you are working with SparkR from an interactive shell (eg. R, RStudio) then Spark is downloaded and cached automatically if it is not found. Alternatively, we provide an easy-to-use function `install.spark` for running this manually. If you don't have Spark installed on the computer, you may download it from [Apache Spark Website](https://spark.apache.org/downloads.html).
+After installing the SparkR package, you can call `sparkR.session` as explained in the previous section to start and it will check for the Spark installation. If you are working with SparkR from an interactive shell (e.g. R, RStudio) then Spark is downloaded and cached automatically if it is not found. Alternatively, we provide an easy-to-use function `install.spark` for running this manually. If you don't have Spark installed on the computer, you may download it from [Apache Spark Website](https://spark.apache.org/downloads.html).
 
 ```{r, eval=FALSE}
 install.spark()
@@ -331,7 +331,7 @@ A common flow of grouping and aggregation is
 
 2. Feed the `GroupedData` object to `agg` or `summarize` functions, with some provided aggregation functions to compute a number within each group.
 
-A number of widely used functions are supported to aggregate data after grouping, including `avg`, `countDistinct`, `count`, `first`, `kurtosis`, `last`, `max`, `mean`, `min`, `sd`, `skewness`, `stddev_pop`, `stddev_samp`, `sumDistinct`, `sum`, `var_pop`, `var_samp`, `var`. See the [API doc for aggregate functions](https://spark.apache.org/docs/latest/api/R/column_aggregate_functions.html) linked there.
+A number of widely used functions are supported to aggregate data after grouping, including `avg`, `count_distinct`, `count`, `first`, `kurtosis`, `last`, `max`, `mean`, `min`, `sd`, `skewness`, `stddev_pop`, `stddev_samp`, `sum_distinct`, `sum`, `var_pop`, `var_samp`, `var`. See the [API doc for aggregate functions](https://spark.apache.org/docs/latest/api/R/column_aggregate_functions.html) linked there.
 
 For example we can compute a histogram of the number of cylinders in the `mtcars` dataset as shown below.
 
@@ -1007,7 +1007,7 @@ perplexity
 
 #### Alternating Least Squares
 
-`spark.als` learns latent factors in [collaborative filtering](https://en.wikipedia.org/wiki/Recommender_system#Collaborative_filtering) via [alternating least squares](https://dl.acm.org/citation.cfm?id=1608614).
+`spark.als` learns latent factors in [collaborative filtering](https://en.wikipedia.org/wiki/Recommender_system#Collaborative_filtering) via [alternating least squares](https://dl.acm.org/doi/10.1109/MC.2009.263).
 
 There are multiple options that can be configured in `spark.als`, including `rank`, `reg`, and `nonnegative`. For a complete list, refer to the help file.
 
diff --git a/R/run-tests.sh b/R/run-tests.sh
index 51ca7d600caf0..edc2b2b60b60e 100755
--- a/R/run-tests.sh
+++ b/R/run-tests.sh
@@ -23,7 +23,18 @@ FAILED=0
 LOGFILE=$FWDIR/unit-tests.out
 rm -f $LOGFILE
 
-SPARK_TESTING=1 NOT_CRAN=true $FWDIR/../bin/spark-submit --driver-java-options "-Dlog4j.configuration=file:$FWDIR/log4j.properties" --conf spark.hadoop.fs.defaultFS="file:///" --conf spark.driver.extraJavaOptions="-Dio.netty.tryReflectionSetAccessible=true" --conf spark.executor.extraJavaOptions="-Dio.netty.tryReflectionSetAccessible=true" $FWDIR/pkg/tests/run-all.R 2>&1 | tee -a $LOGFILE
+SPARK_AVRO_JAR_PATH=$(find $FWDIR/../external/avro/ -name "spark-avro*jar" -print | egrep -v "tests.jar|test-sources.jar|sources.jar|javadoc.jar")
+
+if [[ $(echo $SPARK_AVRO_JAR_PATH | wc -l) -eq 1 ]]; then
+  SPARK_JARS=$SPARK_AVRO_JAR_PATH
+fi
+
+if [ -z "$SPARK_JARS" ]; then
+  SPARK_TESTING=1 NOT_CRAN=true $FWDIR/../bin/spark-submit --driver-java-options "-Dlog4j.configuration=file:$FWDIR/log4j.properties" --conf spark.hadoop.fs.defaultFS="file:///" --conf spark.driver.extraJavaOptions="-Dio.netty.tryReflectionSetAccessible=true" --conf spark.executor.extraJavaOptions="-Dio.netty.tryReflectionSetAccessible=true" $FWDIR/pkg/tests/run-all.R 2>&1 | tee -a $LOGFILE
+else
+  SPARK_TESTING=1 NOT_CRAN=true $FWDIR/../bin/spark-submit --jars $SPARK_JARS --driver-java-options "-Dlog4j.configuration=file:$FWDIR/log4j.properties" --conf spark.hadoop.fs.defaultFS="file:///" --conf spark.driver.extraJavaOptions="-Dio.netty.tryReflectionSetAccessible=true" --conf spark.executor.extraJavaOptions="-Dio.netty.tryReflectionSetAccessible=true" $FWDIR/pkg/tests/run-all.R 2>&1 | tee -a $LOGFILE
+fi
+
 FAILED=$((PIPESTATUS[0]||$FAILED))
 
 NUM_TEST_WARNING="$(grep -c -e 'Warnings ----------------' $LOGFILE)"
diff --git a/README.md b/README.md
index d7931263b0fc7..aa7d1dd338be0 100644
--- a/README.md
+++ b/README.md
@@ -9,7 +9,7 @@ and Structured Streaming for stream processing.
 
 <https://spark.apache.org/>
 
-[![Jenkins Build](https://amplab.cs.berkeley.edu/jenkins/job/spark-master-test-sbt-hadoop-2.7-hive-2.3/badge/icon)](https://amplab.cs.berkeley.edu/jenkins/job/spark-master-test-sbt-hadoop-2.7-hive-2.3)
+[![Jenkins Build](https://amplab.cs.berkeley.edu/jenkins/job/spark-master-test-sbt-hadoop-3.2/badge/icon)](https://amplab.cs.berkeley.edu/jenkins/job/spark-master-test-sbt-hadoop-3.2)
 [![AppVeyor Build](https://img.shields.io/appveyor/ci/ApacheSoftwareFoundation/spark/master.svg?style=plastic&logo=appveyor)](https://ci.appveyor.com/project/ApacheSoftwareFoundation/spark)
 [![PySpark Coverage](https://img.shields.io/badge/dynamic/xml.svg?label=pyspark%20coverage&url=https%3A%2F%2Fspark-test.github.io%2Fpyspark-coverage-site&query=%2Fhtml%2Fbody%2Fdiv%5B1%5D%2Fdiv%2Fh1%2Fspan&colorB=brightgreen&style=plastic)](https://spark-test.github.io/pyspark-coverage-site)
 
diff --git a/appveyor.yml b/appveyor.yml
index a4da5f9040ded..c40b23c8341eb 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -41,9 +41,9 @@ cache:
 install:
   # Install maven and dependencies
   - ps: .\dev\appveyor-install-dependencies.ps1
-  # Required package for R unit tests
-  - cmd: R -e "install.packages(c('knitr', 'rmarkdown', 'testthat', 'e1071', 'survival', 'arrow'), repos='https://cloud.r-project.org/')"
-  - cmd: R -e "packageVersion('knitr'); packageVersion('rmarkdown'); packageVersion('testthat'); packageVersion('e1071'); packageVersion('survival'); packageVersion('arrow')"
+  # Required package for R unit tests. xml2 is required to use jUnit reporter in testthat.
+  - cmd: Rscript -e "install.packages(c('knitr', 'rmarkdown', 'testthat', 'e1071', 'survival', 'arrow', 'xml2'), repos='https://cloud.r-project.org/')"
+  - cmd: Rscript -e "pkg_list <- as.data.frame(installed.packages()[,c(1, 3:4)]); pkg_list[is.na(pkg_list$Priority), 1:2, drop = FALSE]"
 
 build_script:
   # '-Djna.nosys=true' is required to avoid kernel32.dll load failure.
diff --git a/assembly/pom.xml b/assembly/pom.xml
index d17abe857ade5..d662aae96c4af 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
@@ -136,6 +136,10 @@
           <artifactId>spark-yarn_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
         </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+        </dependency>
       </dependencies>
     </profile>
     <profile>
diff --git a/bin/docker-image-tool.sh b/bin/docker-image-tool.sh
index 57b86254ab424..83b13b83341d0 100755
--- a/bin/docker-image-tool.sh
+++ b/bin/docker-image-tool.sh
@@ -172,6 +172,7 @@ function build {
   local BASEDOCKERFILE=${BASEDOCKERFILE:-"kubernetes/dockerfiles/spark/Dockerfile"}
   local PYDOCKERFILE=${PYDOCKERFILE:-false}
   local RDOCKERFILE=${RDOCKERFILE:-false}
+  local ARCHS=${ARCHS:-"--platform linux/amd64,linux/arm64"}
 
   (cd $(img_ctx_dir base) && docker build $NOCACHEARG "${BUILD_ARGS[@]}" \
     -t $(image_ref spark) \
@@ -179,6 +180,11 @@ function build {
   if [ $? -ne 0 ]; then
     error "Failed to build Spark JVM Docker image, please refer to Docker build output for details."
   fi
+  if [ "${CROSS_BUILD}" != "false" ]; then
+  (cd $(img_ctx_dir base) && docker buildx build $ARCHS $NOCACHEARG "${BUILD_ARGS[@]}" --push \
+    -t $(image_ref spark) \
+    -f "$BASEDOCKERFILE" .)
+  fi
 
   if [ "${PYDOCKERFILE}" != "false" ]; then
     (cd $(img_ctx_dir pyspark) && docker build $NOCACHEARG "${BINDING_BUILD_ARGS[@]}" \
@@ -187,6 +193,11 @@ function build {
       if [ $? -ne 0 ]; then
         error "Failed to build PySpark Docker image, please refer to Docker build output for details."
       fi
+      if [ "${CROSS_BUILD}" != "false" ]; then
+        (cd $(img_ctx_dir pyspark) && docker buildx build $ARCHS $NOCACHEARG "${BINDING_BUILD_ARGS[@]}" --push \
+          -t $(image_ref spark-py) \
+          -f "$PYDOCKERFILE" .)
+      fi
   fi
 
   if [ "${RDOCKERFILE}" != "false" ]; then
@@ -196,6 +207,11 @@ function build {
     if [ $? -ne 0 ]; then
       error "Failed to build SparkR Docker image, please refer to Docker build output for details."
     fi
+    if [ "${CROSS_BUILD}" != "false" ]; then
+      (cd $(img_ctx_dir sparkr) && docker buildx build $ARCHS $NOCACHEARG "${BINDING_BUILD_ARGS[@]}" --push \
+        -t $(image_ref spark-r) \
+        -f "$RDOCKERFILE" .)
+    fi
   fi
 }
 
@@ -227,6 +243,8 @@ Options:
   -n                    Build docker image with --no-cache
   -u uid                UID to use in the USER directive to set the user the main Spark process runs as inside the
                         resulting container
+  -X                    Use docker buildx to cross build. Automatically pushes.
+                        See https://docs.docker.com/buildx/working-with-buildx/ for steps to setup buildx.
   -b arg                Build arg to build or push the image. For multiple build args, this option needs to
                         be used separately for each build arg.
 
@@ -252,6 +270,12 @@ Examples:
   - Build and push JDK11-based image with tag "v3.0.0" to docker.io/myrepo
     $0 -r docker.io/myrepo -t v3.0.0 -b java_image_tag=11-jre-slim build
     $0 -r docker.io/myrepo -t v3.0.0 push
+
+  - Build and push JDK11-based image for multiple archs to docker.io/myrepo
+    $0 -r docker.io/myrepo -t v3.0.0 -X -b java_image_tag=11-jre-slim build
+    # Note: buildx, which does cross building, needs to do the push during build
+    # So there is no separate push step with -X
+
 EOF
 }
 
@@ -268,7 +292,8 @@ RDOCKERFILE=
 NOCACHEARG=
 BUILD_PARAMS=
 SPARK_UID=
-while getopts f:p:R:mr:t:nb:u: option
+CROSS_BUILD="false"
+while getopts f:p:R:mr:t:Xnb:u: option
 do
  case "${option}"
  in
@@ -279,6 +304,7 @@ do
  t) TAG=${OPTARG};;
  n) NOCACHEARG="--no-cache";;
  b) BUILD_PARAMS=${BUILD_PARAMS}" --build-arg "${OPTARG};;
+ X) CROSS_BUILD=1;;
  m)
    if ! which minikube 1>/dev/null; then
      error "Cannot find minikube."
diff --git a/bin/find-spark-home b/bin/find-spark-home
index 617dbaa4fff86..462b538b00a04 100755
--- a/bin/find-spark-home
+++ b/bin/find-spark-home
@@ -33,9 +33,9 @@ elif [ ! -f "$FIND_SPARK_HOME_PYTHON_SCRIPT" ]; then
   export SPARK_HOME="$(cd "$(dirname "$0")"/..; pwd)"
 else
   # We are pip installed, use the Python script to resolve a reasonable SPARK_HOME
-  # Default to standard python interpreter unless told otherwise
+  # Default to standard python3 interpreter unless told otherwise
   if [[ -z "$PYSPARK_DRIVER_PYTHON" ]]; then
-     PYSPARK_DRIVER_PYTHON="${PYSPARK_PYTHON:-"python"}"
+     PYSPARK_DRIVER_PYTHON="${PYSPARK_PYTHON:-"python3"}"
   fi
   export SPARK_HOME=$($PYSPARK_DRIVER_PYTHON "$FIND_SPARK_HOME_PYTHON_SCRIPT")
 fi
diff --git a/bin/find-spark-home.cmd b/bin/find-spark-home.cmd
index 6025f67c38de4..3149d05039ba4 100644
--- a/bin/find-spark-home.cmd
+++ b/bin/find-spark-home.cmd
@@ -20,8 +20,8 @@ rem
 rem Path to Python script finding SPARK_HOME
 set FIND_SPARK_HOME_PYTHON_SCRIPT=%~dp0find_spark_home.py
 
-rem Default to standard python interpreter unless told otherwise
-set PYTHON_RUNNER=python
+rem Default to standard python3 interpreter unless told otherwise
+set PYTHON_RUNNER=python3
 rem If PYSPARK_DRIVER_PYTHON is set, it overwrites the python version
 if not "x%PYSPARK_DRIVER_PYTHON%"=="x" (
   set PYTHON_RUNNER=%PYSPARK_DRIVER_PYTHON%
@@ -55,6 +55,6 @@ if "x%SPARK_HOME%"=="x" (
     set SPARK_HOME=%~dp0..
   ) else (
     rem We are pip installed, use the Python script to resolve a reasonable SPARK_HOME
-    for /f "delims=" %%i in ('%PYTHON_RUNNER% %FIND_SPARK_HOME_PYTHON_SCRIPT%') do set SPARK_HOME=%%i
+    for /f "delims=" %%i in ('%PYTHON_RUNNER% "%FIND_SPARK_HOME_PYTHON_SCRIPT%"') do set SPARK_HOME=%%i
   )
 )
diff --git a/bin/load-spark-env.cmd b/bin/load-spark-env.cmd
index 5f98cc34b6bab..5692af529fb66 100644
--- a/bin/load-spark-env.cmd
+++ b/bin/load-spark-env.cmd
@@ -21,42 +21,42 @@ rem This script loads spark-env.cmd if it exists, and ensures it is only loaded
 rem spark-env.cmd is loaded from SPARK_CONF_DIR if set, or within the current directory's
 rem conf\ subdirectory.
 
-set SPARK_ENV_CMD=spark-env.cmd
-if [%SPARK_ENV_LOADED%] == [] (
+if not defined SPARK_ENV_LOADED (
   set SPARK_ENV_LOADED=1
 
-  if [%SPARK_CONF_DIR%] == [] (
+  if not defined SPARK_CONF_DIR (
     set SPARK_CONF_DIR=%~dp0..\conf
   )
 
-  set SPARK_ENV_CMD=%SPARK_CONF_DIR%\%SPARK_ENV_CMD%
-  if exist %SPARK_ENV_CMD% (
-    call %SPARK_ENV_CMD%
-  )
+  call :LoadSparkEnv
 )
 
 rem Setting SPARK_SCALA_VERSION if not already set.
 
-rem TODO: revisit for Scala 2.13 support
-set SPARK_SCALA_VERSION=2.12
-rem if [%SPARK_SCALA_VERSION%] == [] (
-rem   set SCALA_VERSION_1=2.12
-rem   set SCALA_VERSION_2=2.11
-rem
-rem   set ASSEMBLY_DIR1=%SPARK_HOME%\assembly\target\scala-%SCALA_VERSION_1%
-rem   set ASSEMBLY_DIR2=%SPARK_HOME%\assembly\target\scala-%SCALA_VERSION_2%
-rem   set ENV_VARIABLE_DOC=https://spark.apache.org/docs/latest/configuration.html#environment-variables
-rem   if exist %ASSEMBLY_DIR2% if exist %ASSEMBLY_DIR1% (
-rem     echo "Presence of build for multiple Scala versions detected (%ASSEMBLY_DIR1% and %ASSEMBLY_DIR2%)."
-rem     echo "Remove one of them or, set SPARK_SCALA_VERSION=%SCALA_VERSION_1% in %SPARK_ENV_CMD%."
-rem     echo "Visit %ENV_VARIABLE_DOC% for more details about setting environment variables in spark-env.cmd."
-rem     echo "Either clean one of them or, set SPARK_SCALA_VERSION in spark-env.cmd."
-rem     exit 1
-rem   )
-rem   if exist %ASSEMBLY_DIR1% (
-rem     set SPARK_SCALA_VERSION=%SCALA_VERSION_1%
-rem   ) else (
-rem     set SPARK_SCALA_VERSION=%SCALA_VERSION_2%
-rem   )
-rem )
+set SCALA_VERSION_1=2.13
+set SCALA_VERSION_2=2.12
+
+set ASSEMBLY_DIR1="%SPARK_HOME%\assembly\target\scala-%SCALA_VERSION_1%"
+set ASSEMBLY_DIR2="%SPARK_HOME%\assembly\target\scala-%SCALA_VERSION_2%"
+set ENV_VARIABLE_DOC=https://spark.apache.org/docs/latest/configuration.html#environment-variables
+
+if not defined SPARK_SCALA_VERSION (
+  if exist %ASSEMBLY_DIR2% if exist %ASSEMBLY_DIR1% (
+    echo Presence of build for multiple Scala versions detected ^(%ASSEMBLY_DIR1% and %ASSEMBLY_DIR2%^).
+    echo Remove one of them or, set SPARK_SCALA_VERSION=%SCALA_VERSION_1% in spark-env.cmd.
+    echo Visit %ENV_VARIABLE_DOC% for more details about setting environment variables in spark-env.cmd.
+    echo Either clean one of them or, set SPARK_SCALA_VERSION in spark-env.cmd.
+    exit 1
+  )
+  if exist %ASSEMBLY_DIR1% (
+    set SPARK_SCALA_VERSION=%SCALA_VERSION_1%
+  ) else (
+    set SPARK_SCALA_VERSION=%SCALA_VERSION_2%
+  )
+)
 exit /b 0
+
+:LoadSparkEnv
+if exist "%SPARK_CONF_DIR%\spark-env.cmd" (
+  call "%SPARK_CONF_DIR%\spark-env.cmd"
+)
diff --git a/bin/load-spark-env.sh b/bin/load-spark-env.sh
index 107e7991c28bc..04adaeed7ac61 100644
--- a/bin/load-spark-env.sh
+++ b/bin/load-spark-env.sh
@@ -43,25 +43,23 @@ fi
 
 # Setting SPARK_SCALA_VERSION if not already set.
 
-# TODO: revisit for Scala 2.13 support
-export SPARK_SCALA_VERSION=2.12
-#if [ -z "$SPARK_SCALA_VERSION" ]; then
-#  SCALA_VERSION_1=2.12
-#  SCALA_VERSION_2=2.11
-#
-#  ASSEMBLY_DIR_1="${SPARK_HOME}/assembly/target/scala-${SCALA_VERSION_1}"
-#  ASSEMBLY_DIR_2="${SPARK_HOME}/assembly/target/scala-${SCALA_VERSION_2}"
-#  ENV_VARIABLE_DOC="https://spark.apache.org/docs/latest/configuration.html#environment-variables"
-#  if [[ -d "$ASSEMBLY_DIR_1" && -d "$ASSEMBLY_DIR_2" ]]; then
-#    echo "Presence of build for multiple Scala versions detected ($ASSEMBLY_DIR_1 and $ASSEMBLY_DIR_2)." 1>&2
-#    echo "Remove one of them or, export SPARK_SCALA_VERSION=$SCALA_VERSION_1 in ${SPARK_ENV_SH}." 1>&2
-#    echo "Visit ${ENV_VARIABLE_DOC} for more details about setting environment variables in spark-env.sh." 1>&2
-#    exit 1
-#  fi
-#
-#  if [[ -d "$ASSEMBLY_DIR_1" ]]; then
-#    export SPARK_SCALA_VERSION=${SCALA_VERSION_1}
-#  else
-#    export SPARK_SCALA_VERSION=${SCALA_VERSION_2}
-#  fi
-#fi
+if [ -z "$SPARK_SCALA_VERSION" ]; then
+  SCALA_VERSION_1=2.13
+  SCALA_VERSION_2=2.12
+
+  ASSEMBLY_DIR_1="${SPARK_HOME}/assembly/target/scala-${SCALA_VERSION_1}"
+  ASSEMBLY_DIR_2="${SPARK_HOME}/assembly/target/scala-${SCALA_VERSION_2}"
+  ENV_VARIABLE_DOC="https://spark.apache.org/docs/latest/configuration.html#environment-variables"
+  if [[ -d "$ASSEMBLY_DIR_1" && -d "$ASSEMBLY_DIR_2" ]]; then
+    echo "Presence of build for multiple Scala versions detected ($ASSEMBLY_DIR_1 and $ASSEMBLY_DIR_2)." 1>&2
+    echo "Remove one of them or, export SPARK_SCALA_VERSION=$SCALA_VERSION_1 in ${SPARK_ENV_SH}." 1>&2
+    echo "Visit ${ENV_VARIABLE_DOC} for more details about setting environment variables in spark-env.sh." 1>&2
+    exit 1
+  fi
+
+  if [[ -d "$ASSEMBLY_DIR_1" ]]; then
+    export SPARK_SCALA_VERSION=${SCALA_VERSION_1}
+  else
+    export SPARK_SCALA_VERSION=${SCALA_VERSION_2}
+  fi
+fi
diff --git a/bin/pyspark b/bin/pyspark
index ad4132fb59eb0..38ebe51c8d555 100755
--- a/bin/pyspark
+++ b/bin/pyspark
@@ -37,9 +37,9 @@ if [[ -n "$IPYTHON" || -n "$IPYTHON_OPTS" ]]; then
   exit 1
 fi
 
-# Default to standard python interpreter unless told otherwise
+# Default to standard python3 interpreter unless told otherwise
 if [[ -z "$PYSPARK_PYTHON" ]]; then
-  PYSPARK_PYTHON=python
+  PYSPARK_PYTHON=python3
 fi
 if [[ -z "$PYSPARK_DRIVER_PYTHON" ]]; then
   PYSPARK_DRIVER_PYTHON=$PYSPARK_PYTHON
@@ -50,7 +50,7 @@ export PYSPARK_DRIVER_PYTHON_OPTS
 
 # Add the PySpark classes to the Python path:
 export PYTHONPATH="${SPARK_HOME}/python/:$PYTHONPATH"
-export PYTHONPATH="${SPARK_HOME}/python/lib/py4j-0.10.9-src.zip:$PYTHONPATH"
+export PYTHONPATH="${SPARK_HOME}/python/lib/py4j-0.10.9.2-src.zip:$PYTHONPATH"
 
 # Load the PySpark shell.py script when ./pyspark is used interactively:
 export OLD_PYTHONSTARTUP="$PYTHONSTARTUP"
diff --git a/bin/pyspark2.cmd b/bin/pyspark2.cmd
index dc34be1a41706..f5f9fad8dfb1b 100644
--- a/bin/pyspark2.cmd
+++ b/bin/pyspark2.cmd
@@ -30,7 +30,7 @@ if "x%PYSPARK_DRIVER_PYTHON%"=="x" (
 )
 
 set PYTHONPATH=%SPARK_HOME%\python;%PYTHONPATH%
-set PYTHONPATH=%SPARK_HOME%\python\lib\py4j-0.10.9-src.zip;%PYTHONPATH%
+set PYTHONPATH=%SPARK_HOME%\python\lib\py4j-0.10.9.2-src.zip;%PYTHONPATH%
 
 set OLD_PYTHONSTARTUP=%PYTHONSTARTUP%
 set PYTHONSTARTUP=%SPARK_HOME%\python\pyspark\shell.py
diff --git a/bin/spark-class2.cmd b/bin/spark-class2.cmd
old mode 100644
new mode 100755
index 34d04c9856d2c..68b271d1d05d9
--- a/bin/spark-class2.cmd
+++ b/bin/spark-class2.cmd
@@ -30,12 +30,12 @@ if "x%1"=="x" (
 
 rem Find Spark jars.
 if exist "%SPARK_HOME%\jars" (
-  set SPARK_JARS_DIR="%SPARK_HOME%\jars"
+  set SPARK_JARS_DIR=%SPARK_HOME%\jars
 ) else (
-  set SPARK_JARS_DIR="%SPARK_HOME%\assembly\target\scala-%SPARK_SCALA_VERSION%\jars"
+  set SPARK_JARS_DIR=%SPARK_HOME%\assembly\target\scala-%SPARK_SCALA_VERSION%\jars
 )
 
-if not exist "%SPARK_JARS_DIR%"\ (
+if not exist "%SPARK_JARS_DIR%" (
   echo Failed to find Spark jars directory.
   echo You need to build Spark before running this program.
   exit /b 1
diff --git a/binder/apt.txt b/binder/apt.txt
new file mode 100644
index 0000000000000..385f5b0fba754
--- /dev/null
+++ b/binder/apt.txt
@@ -0,0 +1 @@
+openjdk-8-jre
diff --git a/binder/postBuild b/binder/postBuild
new file mode 100644
index 0000000000000..42bb3514c5a2e
--- /dev/null
+++ b/binder/postBuild
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This file is used for Binder integration to install PySpark available in
+# Jupyter notebook.
+
+VERSION=$(python -c "exec(open('python/pyspark/version.py').read()); print(__version__)")
+pip install "pyspark[sql,ml,mllib]<=$VERSION"
diff --git a/build/mvn b/build/mvn
index 53a77f6e81f2b..9e63cc2ff9ca9 100755
--- a/build/mvn
+++ b/build/mvn
@@ -26,36 +26,67 @@ _COMPILE_JVM_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g"
 
 # Installs any application tarball given a URL, the expected tarball name,
 # and, optionally, a checkable binary path to determine if the binary has
-# already been installed
-## Arg1 - URL
-## Arg2 - Tarball Name
-## Arg3 - Checkable Binary
+# already been installed. Arguments:
+# 1 - Mirror host
+# 2 - URL path on host
+# 3 - URL query string
+# 4 - checksum suffix
+# 5 - Tarball Name
+# 6 - Checkable Binary
 install_app() {
-  local remote_tarball="$1/$2"
-  local local_tarball="${_DIR}/$2"
-  local binary="${_DIR}/$3"
+  local mirror_host="$1"
+  local url_path="$2"
+  local url_query="$3"
+  local checksum_suffix="$4"
+  local local_tarball="${_DIR}/$5"
+  local binary="${_DIR}/$6"
+  local remote_tarball="${mirror_host}/${url_path}${url_query}"
+  local local_checksum="${local_tarball}.${checksum_suffix}"
+  local remote_checksum="https://archive.apache.org/dist/${url_path}.${checksum_suffix}"
 
   local curl_opts="--silent --show-error -L"
   local wget_opts="--no-verbose"
 
-  if [ -z "$3" -o ! -f "$binary" ]; then
+  if [ ! -f "$binary" ]; then
     # check if we already have the tarball
     # check if we have curl installed
     # download application
-    [ ! -f "${local_tarball}" ] && [ $(command -v curl) ] && \
-      echo "exec: curl ${curl_opts} ${remote_tarball}" 1>&2 && \
+    if [ ! -f "${local_tarball}" -a $(command -v curl) ]; then
+      echo "exec: curl ${curl_opts} ${remote_tarball}" 1>&2
       curl ${curl_opts} "${remote_tarball}" > "${local_tarball}"
+      if [ ! -z "${checksum_suffix}" ]; then
+        echo "exec: curl ${curl_opts} ${remote_checksum}" 1>&2
+        curl ${curl_opts} "${remote_checksum}" > "${local_checksum}"
+      fi
+    fi
     # if the file still doesn't exist, lets try `wget` and cross our fingers
-    [ ! -f "${local_tarball}" ] && [ $(command -v wget) ] && \
-      echo "exec: wget ${wget_opts} ${remote_tarball}" 1>&2 && \
+    if [ ! -f "${local_tarball}" -a $(command -v wget) ]; then
+      echo "exec: wget ${wget_opts} ${remote_tarball}" 1>&2
       wget ${wget_opts} -O "${local_tarball}" "${remote_tarball}"
+      if [ ! -z "${checksum_suffix}" ]; then
+        echo "exec: wget ${wget_opts} ${remote_checksum}" 1>&2
+        wget ${wget_opts} -O "${local_checksum}" "${remote_checksum}"
+      fi
+    fi
     # if both were unsuccessful, exit
-    [ ! -f "${local_tarball}" ] && \
-      echo -n "ERROR: Cannot download $2 with cURL or wget; " && \
-      echo "please install manually and try again." && \
+    if [ ! -f "${local_tarball}" ]; then
+      echo -n "ERROR: Cannot download ${remote_tarball} with cURL or wget; please install manually and try again."
       exit 2
-    cd "${_DIR}" && tar -xzf "$2"
-    rm -rf "$local_tarball"
+    fi
+    # Checksum may not have been specified; don't check if doesn't exist
+    if [ -f "${local_checksum}" ]; then
+      echo "  ${local_tarball}" >> ${local_checksum} # two spaces + file are important!
+      # Assuming SHA512 here for now
+      echo "Veryfing checksum from ${local_checksum}" 1>&2
+      if ! shasum -a 512 -q -c "${local_checksum}" ; then
+        echo "Bad checksum from ${remote_checksum}"
+        exit 2
+      fi
+    fi
+
+    cd "${_DIR}" && tar -xzf "${local_tarball}"
+    rm -rf "${local_tarball}"
+    rm -f "${local_checksum}"
   fi
 }
 
@@ -71,47 +102,32 @@ install_mvn() {
     local MVN_DETECTED_VERSION="$(mvn --version | head -n1 | awk '{print $3}')"
   fi
   if [ $(version $MVN_DETECTED_VERSION) -lt $(version $MVN_VERSION) ]; then
-    local APACHE_MIRROR=${APACHE_MIRROR:-'https://www.apache.org/dyn/closer.lua?action=download&filename='}
-        
+    local MVN_TARBALL="apache-maven-${MVN_VERSION}-bin.tar.gz"
+    local FILE_PATH="maven/maven-3/${MVN_VERSION}/binaries/${MVN_TARBALL}"
+    local APACHE_MIRROR=${APACHE_MIRROR:-'https://www.apache.org/dyn/closer.lua'}
+    local MIRROR_URL_QUERY="?action=download"
+
     if [ $(command -v curl) ]; then
-      local TEST_MIRROR_URL="${APACHE_MIRROR}/maven/maven-3/${MVN_VERSION}/binaries/apache-maven-${MVN_VERSION}-bin.tar.gz"
-      if ! curl -L --output /dev/null --silent --head --fail "$TEST_MIRROR_URL" ; then
+      if ! curl -L --output /dev/null --silent --head --fail "${APACHE_MIRROR}/${FILE_PATH}${MIRROR_URL_QUERY}" ; then
         # Fall back to archive.apache.org for older Maven
         echo "Falling back to archive.apache.org to download Maven"
         APACHE_MIRROR="https://archive.apache.org/dist"
+        MIRROR_URL_QUERY=""
       fi
     fi
 
     install_app \
-      "${APACHE_MIRROR}/maven/maven-3/${MVN_VERSION}/binaries" \
-      "apache-maven-${MVN_VERSION}-bin.tar.gz" \
+      "${APACHE_MIRROR}" \
+      "${FILE_PATH}" \
+      "${MIRROR_URL_QUERY}" \
+      "sha512" \
+      "${MVN_TARBALL}" \
       "apache-maven-${MVN_VERSION}/bin/mvn"
 
     MVN_BIN="${_DIR}/apache-maven-${MVN_VERSION}/bin/mvn"
   fi
 }
 
-# Install zinc under the build/ folder
-install_zinc() {
-  local ZINC_VERSION=0.3.15
-  ZINC_BIN="$(command -v zinc)"
-  if [ "$ZINC_BIN" ]; then
-    local ZINC_DETECTED_VERSION="$(zinc -version | head -n1 | awk '{print $5}')"
-  fi
-
-  if [ $(version $ZINC_DETECTED_VERSION) -lt $(version $ZINC_VERSION) ]; then
-    local zinc_path="zinc-${ZINC_VERSION}/bin/zinc"
-    [ ! -f "${_DIR}/${zinc_path}" ] && ZINC_INSTALL_FLAG=1
-    local TYPESAFE_MIRROR=${TYPESAFE_MIRROR:-https://downloads.lightbend.com}
-
-    install_app \
-      "${TYPESAFE_MIRROR}/zinc/${ZINC_VERSION}" \
-      "zinc-${ZINC_VERSION}.tgz" \
-      "${zinc_path}"
-    ZINC_BIN="${_DIR}/${zinc_path}"
-  fi
-}
-
 # Determine the Scala version from the root pom.xml file, set the Scala URL,
 # and, with that, download the specific version of Scala necessary under
 # the build/ folder
@@ -121,39 +137,26 @@ install_scala() {
   local scala_version=`grep "scala.version" "${_DIR}/../pom.xml" | grep ${scala_binary_version} | head -n1 | awk -F '[<>]' '{print $3}'`
   local scala_bin="${_DIR}/scala-${scala_version}/bin/scala"
   local TYPESAFE_MIRROR=${TYPESAFE_MIRROR:-https://downloads.lightbend.com}
+  local SCALA_TARBALL="scala-${scala_version}.tgz"
 
   install_app \
-    "${TYPESAFE_MIRROR}/scala/${scala_version}" \
-    "scala-${scala_version}.tgz" \
+    "${TYPESAFE_MIRROR}" \
+    "scala/${scala_version}/${SCALA_TARBALL}" \
+    "" \
+    "" \
+    ${SCALA_TARBALL} \
     "scala-${scala_version}/bin/scala"
 
   SCALA_COMPILER="$(cd "$(dirname "${scala_bin}")/../lib" && pwd)/scala-compiler.jar"
   SCALA_LIBRARY="$(cd "$(dirname "${scala_bin}")/../lib" && pwd)/scala-library.jar"
 }
 
-# Setup healthy defaults for the Zinc port if none were provided from
-# the environment
-ZINC_PORT=${ZINC_PORT:-"3030"}
-
-# Install the proper version of Scala, Zinc and Maven for the build
-install_zinc
 install_scala
 install_mvn
 
 # Reset the current working directory
 cd "${_CALLING_DIR}"
 
-# Now that zinc is ensured to be installed, check its status and, if its
-# not running or just installed, start it
-if [ -n "${ZINC_INSTALL_FLAG}" -o -z "`"${ZINC_BIN}" -status -port ${ZINC_PORT}`" ]; then
-  export ZINC_OPTS=${ZINC_OPTS:-"$_COMPILE_JVM_OPTS"}
-  "${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
-  "${ZINC_BIN}" -start -port ${ZINC_PORT} \
-    -server 127.0.0.1 -idle-timeout 3h \
-    -scala-compiler "${SCALA_COMPILER}" \
-    -scala-library "${SCALA_LIBRARY}" &>/dev/null
-fi
-
 # Set any `mvn` options if not already present
 export MAVEN_OPTS=${MAVEN_OPTS:-"$_COMPILE_JVM_OPTS"}
 
@@ -161,10 +164,7 @@ echo "Using \`mvn\` from path: $MVN_BIN" 1>&2
 
 # call the `mvn` command as usual
 # SPARK-25854
-"${MVN_BIN}" -DzincPort=${ZINC_PORT} "$@"
+"${MVN_BIN}" "$@"
 MVN_RETCODE=$?
 
-# Try to shut down zinc explicitly if the server is still running.
-"${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
-
 exit $MVN_RETCODE
diff --git a/build/sbt b/build/sbt
index 475dfd3b20b43..ae9ca93fc9ca9 100755
--- a/build/sbt
+++ b/build/sbt
@@ -53,6 +53,7 @@ realpath () {
 declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy"
 declare -r sbt_opts_file=".sbtopts"
 declare -r etc_sbt_opts_file="/etc/sbt/sbtopts"
+declare -r default_sbt_opts="-Xss4m"
 
 usage() {
  cat <<EOM
diff --git a/build/sbt-launch-lib.bash b/build/sbt-launch-lib.bash
index 162bfbf2257c7..8fb6672bddc4d 100755
--- a/build/sbt-launch-lib.bash
+++ b/build/sbt-launch-lib.bash
@@ -17,7 +17,7 @@ declare -a java_args
 declare -a scalac_args
 declare -a sbt_commands
 declare -a maven_profiles
-declare sbt_default_mem=2048
+declare sbt_default_mem=4096
 
 if test -x "$JAVA_HOME/bin/java"; then
     echo -e "Using $JAVA_HOME as default JAVA_HOME."
@@ -39,7 +39,11 @@ dlog () {
 
 acquire_sbt_jar () {
   SBT_VERSION=`awk -F "=" '/sbt\.version/ {print $2}' ./project/build.properties`
-  URL1=https://dl.bintray.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
+  # DEFAULT_ARTIFACT_REPOSITORY env variable can be used to only fetch
+  # artifacts from internal repos only.
+  # Ex:
+  #   DEFAULT_ARTIFACT_REPOSITORY=https://artifacts.internal.com/libs-release/
+  URL1=${DEFAULT_ARTIFACT_REPOSITORY:-https://repo1.maven.org/maven2/}org/scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch-${SBT_VERSION}.jar
   JAR=build/sbt-launch-${SBT_VERSION}.jar
 
   sbt_jar=$JAR
diff --git a/build/spark-build-info b/build/spark-build-info
index ad0ec67f455cb..eb0e3d730e23e 100755
--- a/build/spark-build-info
+++ b/build/spark-build-info
@@ -32,7 +32,7 @@ echo_build_properties() {
   echo revision=$(git rev-parse HEAD)
   echo branch=$(git rev-parse --abbrev-ref HEAD)
   echo date=$(date -u +%Y-%m-%dT%H:%M:%SZ)
-  echo url=$(git config --get remote.origin.url)
+  echo url=$(git config --get remote.origin.url |  sed 's|https://\(.*\)@\(.*\)|https://\2|')
 }
 
 echo_build_properties $2 > "$SPARK_BUILD_INFO"
diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml
index 39cdc6d6d6cd3..4ade8c2032b24 100644
--- a/common/kvstore/pom.xml
+++ b/common/kvstore/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
index 42e090bc83ed1..431c7e42774e4 100644
--- a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
+++ b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
@@ -164,8 +164,9 @@ public void clear() {
   }
 
   /**
-   * An alias class for the type "ConcurrentHashMap<Comparable<Object>, Boolean>", which is used
-   * as a concurrent hashset for storing natural keys and the boolean value doesn't matter.
+   * An alias class for the type "{@literal ConcurrentHashMap<Comparable<Object>, Boolean>}",
+   * which is used as a concurrent hashset for storing natural keys
+   * and the boolean value doesn't matter.
    */
   private static class NaturalKeys extends ConcurrentHashMap<Comparable<Object>, Boolean> {}
 
diff --git a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/LevelDB.java b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/LevelDB.java
index 2ca4b0b2cb9f9..121dfbd4f6838 100644
--- a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/LevelDB.java
+++ b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/LevelDB.java
@@ -19,10 +19,13 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.lang.ref.SoftReference;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -64,6 +67,13 @@ public class LevelDB implements KVStore {
   private final ConcurrentMap<String, byte[]> typeAliases;
   private final ConcurrentMap<Class<?>, LevelDBTypeInfo> types;
 
+  /**
+   * Trying to close a JNI LevelDB handle with a closed DB causes JVM crashes. This is used to
+   * ensure that all iterators are correctly closed before LevelDB is closed. Use soft reference
+   * to ensure that the iterator can be GCed, when it is only referenced here.
+   */
+  private final ConcurrentLinkedQueue<SoftReference<LevelDBIterator<?>>> iteratorTracker;
+
   public LevelDB(File path) throws Exception {
     this(path, new KVStoreSerializer());
   }
@@ -94,6 +104,8 @@ public LevelDB(File path, KVStoreSerializer serializer) throws Exception {
       aliases = new HashMap<>();
     }
     typeAliases = new ConcurrentHashMap<>(aliases);
+
+    iteratorTracker = new ConcurrentLinkedQueue<>();
   }
 
   @Override
@@ -142,24 +154,72 @@ public void write(Object value) throws Exception {
     try (WriteBatch batch = db().createWriteBatch()) {
       byte[] data = serializer.serialize(value);
       synchronized (ti) {
-        Object existing;
-        try {
-          existing = get(ti.naturalIndex().entityKey(null, value), value.getClass());
-        } catch (NoSuchElementException e) {
-          existing = null;
-        }
+        updateBatch(batch, value, data, value.getClass(), ti.naturalIndex(), ti.indices());
+        db().write(batch);
+      }
+    }
+  }
+
+  public void writeAll(List<?> values) throws Exception {
+    Preconditions.checkArgument(values != null && !values.isEmpty(),
+      "Non-empty values required.");
+
+    // Group by class, in case there are values from different classes in the values
+    // Typical usecase is for this to be a single class.
+    // A NullPointerException will be thrown if values contain null object.
+    for (Map.Entry<? extends Class<?>, ? extends List<?>> entry :
+        values.stream().collect(Collectors.groupingBy(Object::getClass)).entrySet()) {
+
+      final Iterator<?> valueIter = entry.getValue().iterator();
+      final Iterator<byte[]> serializedValueIter;
+
+      // Deserialize outside synchronized block
+      List<byte[]> list = new ArrayList<>(entry.getValue().size());
+      for (Object value : values) {
+        list.add(serializer.serialize(value));
+      }
+      serializedValueIter = list.iterator();
+
+      final Class<?> klass = entry.getKey();
+      final LevelDBTypeInfo ti = getTypeInfo(klass);
+
+      synchronized (ti) {
+        final LevelDBTypeInfo.Index naturalIndex = ti.naturalIndex();
+        final Collection<LevelDBTypeInfo.Index> indices = ti.indices();
 
-        PrefixCache cache = new PrefixCache(value);
-        byte[] naturalKey = ti.naturalIndex().toKey(ti.naturalIndex().getValue(value));
-        for (LevelDBTypeInfo.Index idx : ti.indices()) {
-          byte[] prefix = cache.getPrefix(idx);
-          idx.add(batch, value, existing, data, naturalKey, prefix);
+        try (WriteBatch batch = db().createWriteBatch()) {
+          while (valueIter.hasNext()) {
+            updateBatch(batch, valueIter.next(), serializedValueIter.next(), klass,
+              naturalIndex, indices);
+          }
+          db().write(batch);
         }
-        db().write(batch);
       }
     }
   }
 
+  private void updateBatch(
+      WriteBatch batch,
+      Object value,
+      byte[] data,
+      Class<?> klass,
+      LevelDBTypeInfo.Index naturalIndex,
+      Collection<LevelDBTypeInfo.Index> indices) throws Exception {
+    Object existing;
+    try {
+      existing = get(naturalIndex.entityKey(null, value), klass);
+    } catch (NoSuchElementException e) {
+      existing = null;
+    }
+
+    PrefixCache cache = new PrefixCache(value);
+    byte[] naturalKey = naturalIndex.toKey(naturalIndex.getValue(value));
+    for (LevelDBTypeInfo.Index idx : indices) {
+      byte[] prefix = cache.getPrefix(idx);
+      idx.add(batch, value, existing, data, naturalKey, prefix);
+    }
+  }
+
   @Override
   public void delete(Class<?> type, Object naturalKey) throws Exception {
     Preconditions.checkArgument(naturalKey != null, "Null keys are not allowed.");
@@ -189,7 +249,9 @@ public <T> KVStoreView<T> view(Class<T> type) throws Exception {
       @Override
       public Iterator<T> iterator() {
         try {
-          return new LevelDBIterator<>(type, LevelDB.this, this);
+          LevelDBIterator<T> it = new LevelDBIterator<>(type, LevelDB.this, this);
+          iteratorTracker.add(new SoftReference<>(it));
+          return it;
         } catch (Exception e) {
           throw Throwables.propagate(e);
         }
@@ -238,6 +300,14 @@ public void close() throws IOException {
       }
 
       try {
+        if (iteratorTracker != null) {
+          for (SoftReference<LevelDBIterator<?>> ref: iteratorTracker) {
+            LevelDBIterator<?> it = ref.get();
+            if (it != null) {
+              it.close();
+            }
+          }
+        }
         _db.close();
       } catch (IOException ioe) {
         throw ioe;
@@ -252,6 +322,7 @@ public void close() throws IOException {
    * with a closed DB can cause JVM crashes, so this ensures that situation does not happen.
    */
   void closeIterator(LevelDBIterator<?> it) throws IOException {
+    notifyIteratorClosed(it);
     synchronized (this._db) {
       DB _db = this._db.get();
       if (_db != null) {
@@ -260,6 +331,14 @@ void closeIterator(LevelDBIterator<?> it) throws IOException {
     }
   }
 
+  /**
+   * Remove iterator from iterator tracker. `LevelDBIterator` calls it to notify
+   * iterator is closed.
+   */
+  void notifyIteratorClosed(LevelDBIterator<?> it) {
+    iteratorTracker.removeIf(ref -> it.equals(ref.get()));
+  }
+
   /** Returns metadata about indices for the given type. */
   LevelDBTypeInfo getTypeInfo(Class<?> type) throws Exception {
     LevelDBTypeInfo ti = types.get(type);
diff --git a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/LevelDBIterator.java b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/LevelDBIterator.java
index 94e8c9fc5796c..e8fb4fac5ba17 100644
--- a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/LevelDBIterator.java
+++ b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/LevelDBIterator.java
@@ -185,6 +185,7 @@ public boolean skip(long n) {
 
   @Override
   public synchronized void close() throws IOException {
+    db.notifyIteratorClosed(this);
     if (!closed) {
       it.close();
       closed = true;
diff --git a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/LevelDBTypeInfo.java b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/LevelDBTypeInfo.java
index d7423537ddfcf..4d7f76f673865 100644
--- a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/LevelDBTypeInfo.java
+++ b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/LevelDBTypeInfo.java
@@ -133,7 +133,7 @@ class LevelDBTypeInfo {
 
     // First create the parent indices, then the child indices.
     ti.indices().forEach(idx -> {
-      // In LevelDB, there is no parent index for the NUTURAL INDEX.
+      // In LevelDB, there is no parent index for the NATURAL INDEX.
       if (idx.parent().isEmpty() || idx.value().equals(KVIndex.NATURAL_INDEX_NAME)) {
         indices.put(idx.value(), new Index(idx, ti.getAccessor(idx.value()), null));
       }
diff --git a/common/kvstore/src/test/java/org/apache/spark/util/kvstore/LevelDBSuite.java b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/LevelDBSuite.java
index 0b755ba0e8000..f6566617765d4 100644
--- a/common/kvstore/src/test/java/org/apache/spark/util/kvstore/LevelDBSuite.java
+++ b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/LevelDBSuite.java
@@ -19,6 +19,7 @@
 
 import java.io.File;
 import java.util.Arrays;
+import java.util.Iterator;
 import java.util.List;
 import java.util.NoSuchElementException;
 import java.util.stream.Collectors;
@@ -276,6 +277,41 @@ public void testNegativeIndexValues() throws Exception {
     assertEquals(expected, results);
   }
 
+  @Test
+  public void testCloseLevelDBIterator() throws Exception {
+    // SPARK-31929: test when LevelDB.close() is called, related LevelDBIterators
+    // are closed. And files opened by iterators are also closed.
+    File dbPathForCloseTest = File
+      .createTempFile(
+        "test_db_close.",
+        ".ldb");
+    dbPathForCloseTest.delete();
+    LevelDB dbForCloseTest = new LevelDB(dbPathForCloseTest);
+    for (int i = 0; i < 8192; i++) {
+      dbForCloseTest.write(createCustomType1(i));
+    }
+    String key = dbForCloseTest
+      .view(CustomType1.class).iterator().next().key;
+    assertEquals("key0", key);
+    Iterator<CustomType1> it0 = dbForCloseTest
+      .view(CustomType1.class).max(1).iterator();
+    while (it0.hasNext()) {
+      it0.next();
+    }
+    System.gc();
+    Iterator<CustomType1> it1 = dbForCloseTest
+      .view(CustomType1.class).iterator();
+    assertEquals("key0", it1.next().key);
+    try (KVStoreIterator<CustomType1> it2 = dbForCloseTest
+      .view(CustomType1.class).closeableIterator()) {
+      assertEquals("key0", it2.next().key);
+    }
+    dbForCloseTest.close();
+    assertTrue(dbPathForCloseTest.exists());
+    FileUtils.deleteQuietly(dbPathForCloseTest);
+    assertTrue(!dbPathForCloseTest.exists());
+  }
+
   private CustomType1 createCustomType1(int i) {
     CustomType1 t = new CustomType1();
     t.key = "key" + i;
diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
index 9d5bc9aae0719..0318f60d546e7 100644
--- a/common/network-common/pom.xml
+++ b/common/network-common/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
@@ -91,6 +91,10 @@
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-crypto</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.roaringbitmap</groupId>
+      <artifactId>RoaringBitmap</artifactId>
+    </dependency>
 
     <!-- Test dependencies -->
     <dependency>
diff --git a/common/network-common/src/main/java/org/apache/spark/network/client/TransportClient.java b/common/network-common/src/main/java/org/apache/spark/network/client/TransportClient.java
index 6dcc703e92669..eb2882074d7c7 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/client/TransportClient.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/client/TransportClient.java
@@ -303,7 +303,7 @@ public void close() {
   @Override
   public String toString() {
     return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE)
-      .append("remoteAdress", channel.remoteAddress())
+      .append("remoteAddress", channel.remoteAddress())
       .append("clientId", clientId)
       .append("isActive", isActive())
       .toString();
diff --git a/common/network-common/src/main/java/org/apache/spark/network/client/TransportClientFactory.java b/common/network-common/src/main/java/org/apache/spark/network/client/TransportClientFactory.java
index 24c436a504fa8..43408d43e577e 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/client/TransportClientFactory.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/client/TransportClientFactory.java
@@ -254,7 +254,7 @@ TransportClient createClient(InetSocketAddress address)
       // Disable Nagle's Algorithm since we don't want packets to wait
       .option(ChannelOption.TCP_NODELAY, true)
       .option(ChannelOption.SO_KEEPALIVE, true)
-      .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, conf.connectionTimeoutMs())
+      .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, conf.connectionCreationTimeoutMs())
       .option(ChannelOption.ALLOCATOR, pooledAllocator);
 
     if (conf.receiveBuf() > 0) {
@@ -280,9 +280,10 @@ public void initChannel(SocketChannel ch) {
     // Connect to the remote server
     long preConnect = System.nanoTime();
     ChannelFuture cf = bootstrap.connect(address);
-    if (!cf.await(conf.connectionTimeoutMs())) {
+    if (!cf.await(conf.connectionCreationTimeoutMs())) {
       throw new IOException(
-        String.format("Connecting to %s timed out (%s ms)", address, conf.connectionTimeoutMs()));
+        String.format("Connecting to %s timed out (%s ms)",
+          address, conf.connectionCreationTimeoutMs()));
     } else if (cf.cause() != null) {
       throw new IOException(String.format("Failed to connect to %s", address), cf.cause());
     }
diff --git a/common/network-common/src/main/java/org/apache/spark/network/client/TransportResponseHandler.java b/common/network-common/src/main/java/org/apache/spark/network/client/TransportResponseHandler.java
index 2f143f77fa4ae..3aac2d2441d2a 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/client/TransportResponseHandler.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/client/TransportResponseHandler.java
@@ -188,6 +188,7 @@ public void handle(ResponseMessage message) throws Exception {
       if (listener == null) {
         logger.warn("Ignoring response for RPC {} from {} ({} bytes) since it is not outstanding",
           resp.requestId, getRemoteAddress(channel), resp.body().size());
+        resp.body().release();
       } else {
         outstandingRpcs.remove(resp.requestId);
         try {
diff --git a/common/network-common/src/main/java/org/apache/spark/network/crypto/AuthEngine.java b/common/network-common/src/main/java/org/apache/spark/network/crypto/AuthEngine.java
index 64fdb32a67ada..c2b2edc7f07d5 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/crypto/AuthEngine.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/crypto/AuthEngine.java
@@ -287,7 +287,7 @@ private byte[] doCipherOp(int mode, byte[] in, boolean isFinal)
         }
       }
     } catch (InternalError ie) {
-      // SPARK-25535. The commons-cryto library will throw InternalError if something goes wrong,
+      // SPARK-25535. The commons-crypto library will throw InternalError if something goes wrong,
       // and leave bad state behind in the Java wrappers, so it's not safe to use them afterwards.
       if (mode == Cipher.ENCRYPT_MODE) {
         this.encryptor = null;
diff --git a/common/network-common/src/main/java/org/apache/spark/network/crypto/README.md b/common/network-common/src/main/java/org/apache/spark/network/crypto/README.md
index 14df703270498..7a9fa3a91d143 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/crypto/README.md
+++ b/common/network-common/src/main/java/org/apache/spark/network/crypto/README.md
@@ -155,4 +155,4 @@ server will be able to understand. This will cause the server to close the conne
 attacker tries to send any command to the server. The attacker can just hold the channel open for
 some time, which will be closed when the server times out the channel. These issues could be
 separately mitigated by adding a shorter timeout for the first message after authentication, and
-potentially by adding host blacklists if a possible attack is detected from a particular host.
+potentially by adding host reject-lists if a possible attack is detected from a particular host.
diff --git a/common/network-common/src/main/java/org/apache/spark/network/protocol/Encoders.java b/common/network-common/src/main/java/org/apache/spark/network/protocol/Encoders.java
index 490915f6de4b3..8bab808ad6864 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/protocol/Encoders.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/protocol/Encoders.java
@@ -17,9 +17,12 @@
 
 package org.apache.spark.network.protocol;
 
+import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 
 import io.netty.buffer.ByteBuf;
+import org.roaringbitmap.RoaringBitmap;
 
 /** Provides a canonical set of Encoders for simple types. */
 public class Encoders {
@@ -44,6 +47,51 @@ public static String decode(ByteBuf buf) {
     }
   }
 
+  /**
+   * Bitmaps are encoded with their serialization length followed by the serialization bytes.
+   *
+   * @since 3.1.0
+   */
+  public static class Bitmaps {
+    public static int encodedLength(RoaringBitmap b) {
+      // Compress the bitmap before serializing it. Note that since BlockTransferMessage
+      // needs to invoke encodedLength first to figure out the length for the ByteBuf, it
+      // guarantees that the bitmap will always be compressed before being serialized.
+      b.trim();
+      b.runOptimize();
+      return b.serializedSizeInBytes();
+    }
+
+    /**
+     * The input ByteBuf for this encoder should have enough write capacity to fit the serialized
+     * bitmap. Other encoders which use {@link io.netty.buffer.AbstractByteBuf#writeBytes(byte[])}
+     * to write can expand the buf as writeBytes calls {@link ByteBuf#ensureWritable} internally.
+     * However, this encoder doesn't rely on netty's writeBytes and will fail if the input buf
+     * doesn't have enough write capacity.
+     */
+    public static void encode(ByteBuf buf, RoaringBitmap b) {
+      // RoaringBitmap requires nio ByteBuffer for serde. We expose the netty ByteBuf as a nio
+      // ByteBuffer. Here, we need to explicitly manage the index so we can write into the
+      // ByteBuffer, and the write is reflected in the underneath ByteBuf.
+      ByteBuffer byteBuffer = buf.nioBuffer(buf.writerIndex(), buf.writableBytes());
+      b.serialize(byteBuffer);
+      buf.writerIndex(buf.writerIndex() + byteBuffer.position());
+    }
+
+    public static RoaringBitmap decode(ByteBuf buf) {
+      RoaringBitmap bitmap = new RoaringBitmap();
+      try {
+        bitmap.deserialize(buf.nioBuffer());
+        // RoaringBitmap deserialize does not advance the reader index of the underlying ByteBuf.
+        // Manually update the index here.
+        buf.readerIndex(buf.readerIndex() + bitmap.serializedSizeInBytes());
+      } catch (IOException e) {
+        throw new RuntimeException("Exception while decoding bitmap", e);
+      }
+      return bitmap;
+    }
+  }
+
   /** Byte arrays are encoded with their length followed by bytes. */
   public static class ByteArrays {
     public static int encodedLength(byte[] arr) {
@@ -135,4 +183,35 @@ public static long[] decode(ByteBuf buf) {
       return longs;
     }
   }
+
+  /**
+   * Bitmap arrays are encoded with the number of bitmaps followed by per-Bitmap encoding.
+   *
+   * @since 3.1.0
+   */
+  public static class BitmapArrays {
+    public static int encodedLength(RoaringBitmap[] bitmaps) {
+      int totalLength = 4;
+      for (RoaringBitmap b : bitmaps) {
+        totalLength += Bitmaps.encodedLength(b);
+      }
+      return totalLength;
+    }
+
+    public static void encode(ByteBuf buf, RoaringBitmap[] bitmaps) {
+      buf.writeInt(bitmaps.length);
+      for (RoaringBitmap b : bitmaps) {
+        Bitmaps.encode(buf, b);
+      }
+    }
+
+    public static RoaringBitmap[] decode(ByteBuf buf) {
+      int numBitmaps = buf.readInt();
+      RoaringBitmap[] bitmaps = new RoaringBitmap[numBitmaps];
+      for (int i = 0; i < bitmaps.length; i ++) {
+        bitmaps[i] = Bitmaps.decode(buf);
+      }
+      return bitmaps;
+    }
+  }
 }
diff --git a/common/network-common/src/main/java/org/apache/spark/network/server/ChunkFetchRequestHandler.java b/common/network-common/src/main/java/org/apache/spark/network/server/ChunkFetchRequestHandler.java
index 82810dacdad84..9a71cf593e28c 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/server/ChunkFetchRequestHandler.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/server/ChunkFetchRequestHandler.java
@@ -88,12 +88,14 @@ public void processFetchRequest(
       logger.trace("Received req from {} to fetch block {}", getRemoteAddress(channel),
         msg.streamChunkId);
     }
-    long chunksBeingTransferred = streamManager.chunksBeingTransferred();
-    if (chunksBeingTransferred >= maxChunksBeingTransferred) {
-      logger.warn("The number of chunks being transferred {} is above {}, close the connection.",
-        chunksBeingTransferred, maxChunksBeingTransferred);
-      channel.close();
-      return;
+    if (maxChunksBeingTransferred < Long.MAX_VALUE) {
+      long chunksBeingTransferred = streamManager.chunksBeingTransferred();
+      if (chunksBeingTransferred >= maxChunksBeingTransferred) {
+        logger.warn("The number of chunks being transferred {} is above {}, close the connection.",
+          chunksBeingTransferred, maxChunksBeingTransferred);
+        channel.close();
+        return;
+      }
     }
     ManagedBuffer buf;
     try {
diff --git a/common/network-common/src/main/java/org/apache/spark/network/server/TransportChannelHandler.java b/common/network-common/src/main/java/org/apache/spark/network/server/TransportChannelHandler.java
index e53a0c1a0852e..2d439473bc226 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/server/TransportChannelHandler.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/server/TransportChannelHandler.java
@@ -165,8 +165,9 @@ public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exc
           if (hasInFlightRequests) {
             String address = getRemoteAddress(ctx.channel());
             logger.error("Connection to {} has been quiet for {} ms while there are outstanding " +
-              "requests. Assuming connection is dead; please adjust spark.network.timeout if " +
-              "this is wrong.", address, requestTimeoutNs / 1000 / 1000);
+              "requests. Assuming connection is dead; please adjust" +
+              " spark.{}.io.connectionTimeout if this is wrong.",
+              address, requestTimeoutNs / 1000 / 1000, transportContext.getConf().getModuleName());
             client.timeOut();
             ctx.close();
           } else if (closeIdleConnections) {
diff --git a/common/network-common/src/main/java/org/apache/spark/network/server/TransportRequestHandler.java b/common/network-common/src/main/java/org/apache/spark/network/server/TransportRequestHandler.java
index f178928006902..4a30f8de07827 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/server/TransportRequestHandler.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/server/TransportRequestHandler.java
@@ -124,12 +124,14 @@ private void processStreamRequest(final StreamRequest req) {
         req.streamId);
     }
 
-    long chunksBeingTransferred = streamManager.chunksBeingTransferred();
-    if (chunksBeingTransferred >= maxChunksBeingTransferred) {
-      logger.warn("The number of chunks being transferred {} is above {}, close the connection.",
-        chunksBeingTransferred, maxChunksBeingTransferred);
-      channel.close();
-      return;
+    if (maxChunksBeingTransferred < Long.MAX_VALUE) {
+      long chunksBeingTransferred = streamManager.chunksBeingTransferred();
+      if (chunksBeingTransferred >= maxChunksBeingTransferred) {
+        logger.warn("The number of chunks being transferred {} is above {}, close the connection.",
+          chunksBeingTransferred, maxChunksBeingTransferred);
+        channel.close();
+        return;
+      }
     }
     ManagedBuffer buf;
     try {
diff --git a/common/network-common/src/main/java/org/apache/spark/network/util/NettyUtils.java b/common/network-common/src/main/java/org/apache/spark/network/util/NettyUtils.java
index 423cc0c70ea02..ffbc54a0e0ad8 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/util/NettyUtils.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/util/NettyUtils.java
@@ -162,7 +162,6 @@ public static PooledByteBufAllocator createPooledByteBufAllocator(
       Math.min(PooledByteBufAllocator.defaultNumDirectArena(), allowDirectBufs ? numCores : 0),
       PooledByteBufAllocator.defaultPageSize(),
       PooledByteBufAllocator.defaultMaxOrder(),
-      allowCache ? PooledByteBufAllocator.defaultTinyCacheSize() : 0,
       allowCache ? PooledByteBufAllocator.defaultSmallCacheSize() : 0,
       allowCache ? PooledByteBufAllocator.defaultNormalCacheSize() : 0,
       allowCache ? PooledByteBufAllocator.defaultUseCacheForAllThreads() : false
diff --git a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
index 6c37f9a382376..f051042a7adb4 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
@@ -19,6 +19,7 @@
 
 import java.util.Locale;
 import java.util.Properties;
+import java.util.concurrent.TimeUnit;
 
 import com.google.common.primitives.Ints;
 import io.netty.util.NettyRuntime;
@@ -31,6 +32,7 @@ public class TransportConf {
   private final String SPARK_NETWORK_IO_MODE_KEY;
   private final String SPARK_NETWORK_IO_PREFERDIRECTBUFS_KEY;
   private final String SPARK_NETWORK_IO_CONNECTIONTIMEOUT_KEY;
+  private final String SPARK_NETWORK_IO_CONNECTIONCREATIONTIMEOUT_KEY;
   private final String SPARK_NETWORK_IO_BACKLOG_KEY;
   private final String SPARK_NETWORK_IO_NUMCONNECTIONSPERPEER_KEY;
   private final String SPARK_NETWORK_IO_SERVERTHREADS_KEY;
@@ -54,6 +56,7 @@ public TransportConf(String module, ConfigProvider conf) {
     SPARK_NETWORK_IO_MODE_KEY = getConfKey("io.mode");
     SPARK_NETWORK_IO_PREFERDIRECTBUFS_KEY = getConfKey("io.preferDirectBufs");
     SPARK_NETWORK_IO_CONNECTIONTIMEOUT_KEY = getConfKey("io.connectionTimeout");
+    SPARK_NETWORK_IO_CONNECTIONCREATIONTIMEOUT_KEY = getConfKey("io.connectionCreationTimeout");
     SPARK_NETWORK_IO_BACKLOG_KEY = getConfKey("io.backLog");
     SPARK_NETWORK_IO_NUMCONNECTIONSPERPEER_KEY =  getConfKey("io.numConnectionsPerPeer");
     SPARK_NETWORK_IO_SERVERTHREADS_KEY = getConfKey("io.serverThreads");
@@ -94,7 +97,7 @@ public boolean preferDirectBufs() {
     return conf.getBoolean(SPARK_NETWORK_IO_PREFERDIRECTBUFS_KEY, true);
   }
 
-  /** Connect timeout in milliseconds. Default 120 secs. */
+  /** Connection idle timeout in milliseconds. Default 120 secs. */
   public int connectionTimeoutMs() {
     long defaultNetworkTimeoutS = JavaUtils.timeStringAsSec(
       conf.get("spark.network.timeout", "120s"));
@@ -103,6 +106,14 @@ public int connectionTimeoutMs() {
     return (int) defaultTimeoutMs;
   }
 
+  /** Connect creation timeout in milliseconds. Default 30 secs. */
+  public int connectionCreationTimeoutMs() {
+    long connectionTimeoutS = TimeUnit.MILLISECONDS.toSeconds(connectionTimeoutMs());
+    long defaultTimeoutMs = JavaUtils.timeStringAsSec(
+      conf.get(SPARK_NETWORK_IO_CONNECTIONCREATIONTIMEOUT_KEY,  connectionTimeoutS + "s")) * 1000;
+    return (int) defaultTimeoutMs;
+  }
+
   /** Number of concurrent connections between two nodes for fetching data. */
   public int numConnectionsPerPeer() {
     return conf.getInt(SPARK_NETWORK_IO_NUMCONNECTIONSPERPEER_KEY, 1);
@@ -290,7 +301,7 @@ public boolean sharedByteBufAllocators() {
   }
 
   /**
-  * If enabled then off-heap byte buffers will be prefered for the shared ByteBuf allocators.
+  * If enabled then off-heap byte buffers will be preferred for the shared ByteBuf allocators.
   */
   public boolean preferDirectBufsForSharedByteBufAllocators() {
     return conf.getBoolean("spark.network.io.preferDirectBufs", true);
@@ -363,4 +374,49 @@ public boolean useOldFetchProtocol() {
     return conf.getBoolean("spark.shuffle.useOldFetchProtocol", false);
   }
 
+  /**
+   * Class name of the implementation of MergedShuffleFileManager that merges the blocks
+   * pushed to it when push-based shuffle is enabled. By default, push-based shuffle is disabled at
+   * a cluster level because this configuration is set to
+   * 'org.apache.spark.network.shuffle.ExternalBlockHandler$NoOpMergedShuffleFileManager'.
+   * To turn on push-based shuffle at a cluster level, set the configuration to
+   * 'org.apache.spark.network.shuffle.RemoteBlockPushResolver'.
+   */
+  public String mergedShuffleFileManagerImpl() {
+    return conf.get("spark.shuffle.server.mergedShuffleFileManagerImpl",
+      "org.apache.spark.network.shuffle.ExternalBlockHandler$NoOpMergedShuffleFileManager");
+  }
+
+  /**
+   * The minimum size of a chunk when dividing a merged shuffle file into multiple chunks during
+   * push-based shuffle.
+   * A merged shuffle file consists of multiple small shuffle blocks. Fetching the
+   * complete merged shuffle file in a single response increases the memory requirements for the
+   * clients. Instead of serving the entire merged file, the shuffle service serves the
+   * merged file in `chunks`. A `chunk` constitutes few shuffle blocks in entirety and this
+   * configuration controls how big a chunk can get. A corresponding index file for each merged
+   * shuffle file will be generated indicating chunk boundaries.
+   */
+  public int minChunkSizeInMergedShuffleFile() {
+    return Ints.checkedCast(JavaUtils.byteStringAsBytes(
+      conf.get("spark.shuffle.server.minChunkSizeInMergedShuffleFile", "2m")));
+  }
+
+  /**
+   * The size of cache in memory which is used in push-based shuffle for storing merged index files.
+   */
+  public long mergedIndexCacheSize() {
+    return JavaUtils.byteStringAsBytes(
+      conf.get("spark.shuffle.server.mergedIndexCacheSize", "100m"));
+  }
+
+  /**
+   * The threshold for number of IOExceptions while merging shuffle blocks to a shuffle partition.
+   * When the number of IOExceptions while writing to merged shuffle data/index/meta file exceed
+   * this threshold then the shuffle server will respond back to client to stop pushing shuffle
+   * blocks for this shuffle partition.
+   */
+  public int ioExceptionsThresholdDuringMerge() {
+    return conf.getInt("spark.shuffle.server.ioExceptionsThresholdDuringMerge", 4);
+  }
 }
diff --git a/common/network-common/src/test/java/org/apache/spark/network/TestUtils.java b/common/network-common/src/test/java/org/apache/spark/network/TestUtils.java
index 56a2b805f154c..c2c5ffa43e0ed 100644
--- a/common/network-common/src/test/java/org/apache/spark/network/TestUtils.java
+++ b/common/network-common/src/test/java/org/apache/spark/network/TestUtils.java
@@ -22,7 +22,9 @@
 public class TestUtils {
   public static String getLocalHost() {
     try {
-      return InetAddress.getLocalHost().getHostAddress();
+      return (System.getenv().containsKey("SPARK_LOCAL_IP"))?
+        System.getenv("SPARK_LOCAL_IP"):
+          InetAddress.getLocalHost().getHostAddress();
     } catch (Exception e) {
       throw new RuntimeException(e);
     }
diff --git a/common/network-common/src/test/java/org/apache/spark/network/client/TransportClientFactorySuite.java b/common/network-common/src/test/java/org/apache/spark/network/client/TransportClientFactorySuite.java
index ea0ac51589dc3..277ff85db7bf5 100644
--- a/common/network-common/src/test/java/org/apache/spark/network/client/TransportClientFactorySuite.java
+++ b/common/network-common/src/test/java/org/apache/spark/network/client/TransportClientFactorySuite.java
@@ -29,9 +29,7 @@
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
-import org.junit.Rule;
 import org.junit.Test;
-import org.junit.rules.ExpectedException;
 
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotSame;
@@ -227,11 +225,8 @@ public void closeFactoryBeforeCreateClient() throws IOException, InterruptedExce
     factory.createClient(TestUtils.getLocalHost(), server1.getPort());
   }
 
-  @Rule
-  public ExpectedException expectedException = ExpectedException.none();
-
   @Test
-  public void fastFailConnectionInTimeWindow() throws IOException, InterruptedException {
+  public void fastFailConnectionInTimeWindow() {
     TransportClientFactory factory = context.createClientFactory();
     TransportServer server = context.createServer();
     int unreachablePort = server.getPort();
@@ -241,9 +236,7 @@ public void fastFailConnectionInTimeWindow() throws IOException, InterruptedExce
     } catch (Exception e) {
       assert(e instanceof IOException);
     }
-    expectedException.expect(IOException.class);
-    expectedException.expectMessage("fail this connection directly");
-    factory.createClient(TestUtils.getLocalHost(), unreachablePort, true);
-    expectedException = ExpectedException.none();
+    Assert.assertThrows("fail this connection directly", IOException.class,
+      () -> factory.createClient(TestUtils.getLocalHost(), unreachablePort, true));
   }
 }
diff --git a/common/network-common/src/test/java/org/apache/spark/network/crypto/AuthEngineSuite.java b/common/network-common/src/test/java/org/apache/spark/network/crypto/AuthEngineSuite.java
index 0790f0079c2bd..1c2061699a128 100644
--- a/common/network-common/src/test/java/org/apache/spark/network/crypto/AuthEngineSuite.java
+++ b/common/network-common/src/test/java/org/apache/spark/network/crypto/AuthEngineSuite.java
@@ -150,8 +150,8 @@ public void testEncryptedMessage() throws Exception {
 
       ByteArrayWritableChannel channel = new ByteArrayWritableChannel(data.length);
       TransportCipher.EncryptedMessage emsg = handler.createEncryptedMessage(buf);
-      while (emsg.transfered() < emsg.count()) {
-        emsg.transferTo(channel, emsg.transfered());
+      while (emsg.transferred() < emsg.count()) {
+        emsg.transferTo(channel, emsg.transferred());
       }
       assertEquals(data.length, channel.length());
     } finally {
@@ -196,9 +196,9 @@ public Long answer(InvocationOnMock invocationOnMock) throws Throwable {
       TransportCipher.EncryptedMessage emsg = handler.createEncryptedMessage(region);
       ByteArrayWritableChannel channel = new ByteArrayWritableChannel(testDataLength);
       // "transferTo" should act correctly when the underlying FileRegion transfers 0 bytes.
-      assertEquals(0L, emsg.transferTo(channel, emsg.transfered()));
-      assertEquals(testDataLength, emsg.transferTo(channel, emsg.transfered()));
-      assertEquals(emsg.transfered(), emsg.count());
+      assertEquals(0L, emsg.transferTo(channel, emsg.transferred()));
+      assertEquals(testDataLength, emsg.transferTo(channel, emsg.transferred()));
+      assertEquals(emsg.transferred(), emsg.count());
       assertEquals(4, channel.length());
     } finally {
       client.close();
diff --git a/common/network-common/src/test/java/org/apache/spark/network/crypto/TransportCipherSuite.java b/common/network-common/src/test/java/org/apache/spark/network/crypto/TransportCipherSuite.java
index 6b2186f73cd0c..e62b8cb24e0ed 100644
--- a/common/network-common/src/test/java/org/apache/spark/network/crypto/TransportCipherSuite.java
+++ b/common/network-common/src/test/java/org/apache/spark/network/crypto/TransportCipherSuite.java
@@ -29,11 +29,11 @@
 import org.apache.spark.network.util.MapConfigProvider;
 import org.apache.spark.network.util.TransportConf;
 import org.hamcrest.CoreMatchers;
+import org.hamcrest.MatcherAssert;
 import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertThat;
 import static org.junit.Assert.fail;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyInt;
@@ -81,7 +81,7 @@ CryptoInputStream createInputStream(ReadableByteChannel ch) throws IOException {
       channel.writeInbound(buffer2);
       fail("Should have raised an exception");
     } catch (Throwable expected) {
-      assertThat(expected, CoreMatchers.instanceOf(IOException.class));
+      MatcherAssert.assertThat(expected, CoreMatchers.instanceOf(IOException.class));
       assertEquals(0, buffer2.refCnt());
     }
 
diff --git a/common/network-common/src/test/java/org/apache/spark/network/protocol/EncodersSuite.java b/common/network-common/src/test/java/org/apache/spark/network/protocol/EncodersSuite.java
new file mode 100644
index 0000000000000..6e89702c04396
--- /dev/null
+++ b/common/network-common/src/test/java/org/apache/spark/network/protocol/EncodersSuite.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.network.protocol;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import org.junit.Test;
+import org.roaringbitmap.RoaringBitmap;
+
+import static org.junit.Assert.*;
+
+/**
+ * Tests for {@link Encoders}.
+ */
+public class EncodersSuite {
+
+  @Test
+  public void testRoaringBitmapEncodeDecode() {
+    RoaringBitmap bitmap = new RoaringBitmap();
+    bitmap.add(1, 2, 3);
+    ByteBuf buf = Unpooled.buffer(Encoders.Bitmaps.encodedLength(bitmap));
+    Encoders.Bitmaps.encode(buf, bitmap);
+    RoaringBitmap decodedBitmap = Encoders.Bitmaps.decode(buf);
+    assertEquals(bitmap, decodedBitmap);
+  }
+
+  @Test (expected = java.nio.BufferOverflowException.class)
+  public void testRoaringBitmapEncodeShouldFailWhenBufferIsSmall() {
+    RoaringBitmap bitmap = new RoaringBitmap();
+    bitmap.add(1, 2, 3);
+    ByteBuf buf = Unpooled.buffer(4);
+    Encoders.Bitmaps.encode(buf, bitmap);
+  }
+
+  @Test
+  public void testBitmapArraysEncodeDecode() {
+    RoaringBitmap[] bitmaps = new RoaringBitmap[] {
+      new RoaringBitmap(),
+      new RoaringBitmap(),
+      new RoaringBitmap(), // empty
+      new RoaringBitmap(),
+      new RoaringBitmap()
+    };
+    bitmaps[0].add(1, 2, 3);
+    bitmaps[1].add(1, 2, 4);
+    bitmaps[3].add(7L, 9L);
+    bitmaps[4].add(1L, 100L);
+    ByteBuf buf = Unpooled.buffer(Encoders.BitmapArrays.encodedLength(bitmaps));
+    Encoders.BitmapArrays.encode(buf, bitmaps);
+    RoaringBitmap[] decodedBitmaps = Encoders.BitmapArrays.decode(buf);
+    assertArrayEquals(bitmaps, decodedBitmaps);
+  }
+}
diff --git a/common/network-common/src/test/java/org/apache/spark/network/protocol/MessageWithHeaderSuite.java b/common/network-common/src/test/java/org/apache/spark/network/protocol/MessageWithHeaderSuite.java
index 3bff34e210e3c..af1c2878672c0 100644
--- a/common/network-common/src/test/java/org/apache/spark/network/protocol/MessageWithHeaderSuite.java
+++ b/common/network-common/src/test/java/org/apache/spark/network/protocol/MessageWithHeaderSuite.java
@@ -129,8 +129,8 @@ private void testFileRegionBody(int totalWrites, int writesPerCall) throws Excep
   private ByteBuf doWrite(MessageWithHeader msg, int minExpectedWrites) throws Exception {
     int writes = 0;
     ByteArrayWritableChannel channel = new ByteArrayWritableChannel((int) msg.count());
-    while (msg.transfered() < msg.count()) {
-      msg.transferTo(channel, msg.transfered());
+    while (msg.transferred() < msg.count()) {
+      msg.transferTo(channel, msg.transferred());
       writes++;
     }
     assertTrue("Not enough writes!", minExpectedWrites <= writes);
diff --git a/common/network-common/src/test/java/org/apache/spark/network/sasl/SparkSaslSuite.java b/common/network-common/src/test/java/org/apache/spark/network/sasl/SparkSaslSuite.java
index ecaeec98da182..32c9acd327213 100644
--- a/common/network-common/src/test/java/org/apache/spark/network/sasl/SparkSaslSuite.java
+++ b/common/network-common/src/test/java/org/apache/spark/network/sasl/SparkSaslSuite.java
@@ -191,28 +191,28 @@ public void testEncryptedMessage() throws Exception {
 
       SaslEncryption.EncryptedMessage emsg =
         new SaslEncryption.EncryptedMessage(backend, msg, 1024);
-      long count = emsg.transferTo(channel, emsg.transfered());
+      long count = emsg.transferTo(channel, emsg.transferred());
       assertTrue(count < data.length);
       assertTrue(count > 0);
 
       // Here, the output buffer is full so nothing should be transferred.
-      assertEquals(0, emsg.transferTo(channel, emsg.transfered()));
+      assertEquals(0, emsg.transferTo(channel, emsg.transferred()));
 
       // Now there's room in the buffer, but not enough to transfer all the remaining data,
       // so the dummy count should be returned.
       channel.reset();
-      assertEquals(1, emsg.transferTo(channel, emsg.transfered()));
+      assertEquals(1, emsg.transferTo(channel, emsg.transferred()));
 
       // Eventually, the whole message should be transferred.
       for (int i = 0; i < data.length / 32 - 2; i++) {
         channel.reset();
-        assertEquals(1, emsg.transferTo(channel, emsg.transfered()));
+        assertEquals(1, emsg.transferTo(channel, emsg.transferred()));
       }
 
       channel.reset();
-      count = emsg.transferTo(channel, emsg.transfered());
+      count = emsg.transferTo(channel, emsg.transferred());
       assertTrue("Unexpected count: " + count, count > 1 && count < data.length);
-      assertEquals(data.length, emsg.transfered());
+      assertEquals(data.length, emsg.transferred());
     } finally {
       msg.release();
     }
@@ -237,9 +237,9 @@ public void testEncryptedMessageChunking() throws Exception {
         new SaslEncryption.EncryptedMessage(backend, msg.convertToNetty(), data.length / 8);
 
       ByteArrayWritableChannel channel = new ByteArrayWritableChannel(data.length);
-      while (emsg.transfered() < emsg.count()) {
+      while (emsg.transferred() < emsg.count()) {
         channel.reset();
-        emsg.transferTo(channel, emsg.transfered());
+        emsg.transferTo(channel, emsg.transferred());
       }
 
       verify(backend, times(8)).wrap(any(byte[].class), anyInt(), anyInt());
diff --git a/common/network-common/src/test/java/org/apache/spark/network/server/OneForOneStreamManagerSuite.java b/common/network-common/src/test/java/org/apache/spark/network/server/OneForOneStreamManagerSuite.java
index 45e1836da641f..634b40ed450ee 100644
--- a/common/network-common/src/test/java/org/apache/spark/network/server/OneForOneStreamManagerSuite.java
+++ b/common/network-common/src/test/java/org/apache/spark/network/server/OneForOneStreamManagerSuite.java
@@ -72,7 +72,7 @@ public void testMissingChunk() {
     Assert.assertNotNull(getChunk(manager, streamId, 2));
     manager.connectionTerminated(dummyChannel);
 
-    // loaded buffers are not released yet as in production a MangedBuffer returned by getChunk()
+    // loaded buffers are not released yet as in production a ManagedBuffer returned by getChunk()
     // would only be released by Netty after it is written to the network
     Mockito.verify(buffer1, Mockito.never()).release();
     Mockito.verify(buffer2, Mockito.never()).release();
diff --git a/common/network-common/src/test/java/org/apache/spark/network/util/TransportFrameDecoderSuite.java b/common/network-common/src/test/java/org/apache/spark/network/util/TransportFrameDecoderSuite.java
index 4b67aa80351d2..163c52b023822 100644
--- a/common/network-common/src/test/java/org/apache/spark/network/util/TransportFrameDecoderSuite.java
+++ b/common/network-common/src/test/java/org/apache/spark/network/util/TransportFrameDecoderSuite.java
@@ -98,7 +98,7 @@ public void testConsolidationPerf() throws Exception {
             writtenBytes += pieceBytes;
           }
           logger.info("Writing 300MiB frame buf with consolidation of threshold " + threshold
-              + " took " + totalTime + " milis");
+              + " took " + totalTime + " millis");
         } finally {
           for (ByteBuf buf : retained) {
             release(buf);
diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml
index 00f1defbb0093..6be6df993478d 100644
--- a/common/network-shuffle/pom.xml
+++ b/common/network-shuffle/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
@@ -47,6 +47,11 @@
       <artifactId>metrics-core</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-tags_${scala.binary.version}</artifactId>
+    </dependency>
+
     <!-- Provided dependencies -->
     <dependency>
       <groupId>org.slf4j</groupId>
@@ -57,6 +62,10 @@
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.roaringbitmap</groupId>
+      <artifactId>RoaringBitmap</artifactId>
+    </dependency>
 
     <!-- Test dependencies -->
     <dependency>
@@ -66,11 +75,6 @@
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>org.apache.spark</groupId>
-      <artifactId>spark-tags_${scala.binary.version}</artifactId>
-      <scope>test</scope>
-    </dependency>
 
     <!--
       This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude
@@ -93,6 +97,11 @@
       <artifactId>mockito-core</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>
diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/BlockStoreClient.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/BlockStoreClient.java
index fbbe8ac0f1f9b..a6bdc13e93234 100644
--- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/BlockStoreClient.java
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/BlockStoreClient.java
@@ -18,15 +18,34 @@
 package org.apache.spark.network.shuffle;
 
 import java.io.Closeable;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
 
 import com.codahale.metrics.MetricSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.spark.network.buffer.ManagedBuffer;
+import org.apache.spark.network.client.RpcResponseCallback;
+import org.apache.spark.network.client.TransportClient;
+import org.apache.spark.network.client.TransportClientFactory;
+import org.apache.spark.network.shuffle.protocol.BlockTransferMessage;
+import org.apache.spark.network.shuffle.protocol.GetLocalDirsForExecutors;
+import org.apache.spark.network.shuffle.protocol.LocalDirsForExecutors;
 
 /**
  * Provides an interface for reading both shuffle files and RDD blocks, either from an Executor
  * or external service.
  */
 public abstract class BlockStoreClient implements Closeable {
+  protected final Logger logger = LoggerFactory.getLogger(this.getClass());
+
+  protected volatile TransportClientFactory clientFactory;
+  protected String appId;
 
   /**
    * Fetch a sequence of blocks from a remote node asynchronously,
@@ -61,4 +80,102 @@ public MetricSet shuffleMetrics() {
     // Return an empty MetricSet by default.
     return () -> Collections.emptyMap();
   }
+
+  protected void checkInit() {
+    assert appId != null : "Called before init()";
+  }
+
+  /**
+   * Request the local disk directories for executors which are located at the same host with
+   * the current BlockStoreClient(it can be ExternalBlockStoreClient or NettyBlockTransferService).
+   *
+   * @param host the host of BlockManager or ExternalShuffleService. It should be the same host
+   *             with current BlockStoreClient.
+   * @param port the port of BlockManager or ExternalShuffleService.
+   * @param execIds a collection of executor Ids, which specifies the target executors that we
+   *                want to get their local directories. There could be multiple executor Ids if
+   *                BlockStoreClient is implemented by ExternalBlockStoreClient since the request
+   *                handler, ExternalShuffleService, can serve multiple executors on the same node.
+   *                Or, only one executor Id if BlockStoreClient is implemented by
+   *                NettyBlockTransferService.
+   * @param hostLocalDirsCompletable a CompletableFuture which contains a map from executor Id
+   *                                 to its local directories if the request handler replies
+   *                                 successfully. Otherwise, it contains a specific error.
+   */
+  public void getHostLocalDirs(
+      String host,
+      int port,
+      String[] execIds,
+      CompletableFuture<Map<String, String[]>> hostLocalDirsCompletable) {
+    checkInit();
+    GetLocalDirsForExecutors getLocalDirsMessage = new GetLocalDirsForExecutors(appId, execIds);
+    try {
+      TransportClient client = clientFactory.createClient(host, port);
+      client.sendRpc(getLocalDirsMessage.toByteBuffer(), new RpcResponseCallback() {
+        @Override
+        public void onSuccess(ByteBuffer response) {
+          try {
+            BlockTransferMessage msgObj = BlockTransferMessage.Decoder.fromByteBuffer(response);
+            hostLocalDirsCompletable.complete(
+              ((LocalDirsForExecutors) msgObj).getLocalDirsByExec());
+          } catch (Throwable t) {
+            logger.warn("Error while trying to get the host local dirs for " +
+              Arrays.toString(getLocalDirsMessage.execIds), t.getCause());
+            hostLocalDirsCompletable.completeExceptionally(t);
+          }
+        }
+
+        @Override
+        public void onFailure(Throwable t) {
+          logger.warn("Error while trying to get the host local dirs for " +
+            Arrays.toString(getLocalDirsMessage.execIds), t.getCause());
+          hostLocalDirsCompletable.completeExceptionally(t);
+        }
+      });
+    } catch (IOException | InterruptedException e) {
+      hostLocalDirsCompletable.completeExceptionally(e);
+    }
+  }
+
+  /**
+   * Push a sequence of shuffle blocks in a best-effort manner to a remote node asynchronously.
+   * These shuffle blocks, along with blocks pushed by other clients, will be merged into
+   * per-shuffle partition merged shuffle files on the destination node.
+   *
+   * @param host the host of the remote node.
+   * @param port the port of the remote node.
+   * @param blockIds block ids to be pushed
+   * @param buffers buffers to be pushed
+   * @param listener the listener to receive block push status.
+   *
+   * @since 3.1.0
+   */
+  public void pushBlocks(
+      String host,
+      int port,
+      String[] blockIds,
+      ManagedBuffer[] buffers,
+      BlockFetchingListener listener) {
+    throw new UnsupportedOperationException();
+  }
+
+  /**
+   * Invoked by Spark driver to notify external shuffle services to finalize the shuffle merge
+   * for a given shuffle. This allows the driver to start the shuffle reducer stage after properly
+   * finishing the shuffle merge process associated with the shuffle mapper stage.
+   *
+   * @param host host of shuffle server
+   * @param port port of shuffle server.
+   * @param shuffleId shuffle ID of the shuffle to be finalized
+   * @param listener the listener to receive MergeStatuses
+   *
+   * @since 3.1.0
+   */
+  public void finalizeShuffleMerge(
+      String host,
+      int port,
+      int shuffleId,
+      MergeFinalizerListener listener) {
+    throw new UnsupportedOperationException();
+  }
 }
diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ErrorHandler.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ErrorHandler.java
new file mode 100644
index 0000000000000..968777fba785d
--- /dev/null
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ErrorHandler.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.network.shuffle;
+
+import java.net.ConnectException;
+
+import com.google.common.base.Throwables;
+
+import org.apache.spark.annotation.Evolving;
+
+/**
+ * Plugs into {@link RetryingBlockFetcher} to further control when an exception should be retried
+ * and logged.
+ * Note: {@link RetryingBlockFetcher} will delegate the exception to this handler only when
+ * - remaining retries < max retries
+ * - exception is an IOException
+ *
+ * @since 3.1.0
+ */
+@Evolving
+public interface ErrorHandler {
+
+  boolean shouldRetryError(Throwable t);
+
+  default boolean shouldLogError(Throwable t) {
+    return true;
+  }
+
+  /**
+   * A no-op error handler instance.
+   */
+  ErrorHandler NOOP_ERROR_HANDLER = t -> true;
+
+  /**
+   * The error handler for pushing shuffle blocks to remote shuffle services.
+   *
+   * @since 3.1.0
+   */
+  class BlockPushErrorHandler implements ErrorHandler {
+    /**
+     * String constant used for generating exception messages indicating a block to be merged
+     * arrives too late on the server side, and also for later checking such exceptions on the
+     * client side. When we get a block push failure because of the block arrives too late, we
+     * will not retry pushing the block nor log the exception on the client side.
+     */
+    public static final String TOO_LATE_MESSAGE_SUFFIX =
+      "received after merged shuffle is finalized";
+
+    /**
+     * String constant used for generating exception messages indicating the server couldn't
+     * append a block after all available attempts due to collision with other blocks belonging
+     * to the same shuffle partition, and also for later checking such exceptions on the client
+     * side. When we get a block push failure because of the block couldn't be written due to
+     * this reason, we will not log the exception on the client side.
+     */
+    public static final String BLOCK_APPEND_COLLISION_DETECTED_MSG_PREFIX =
+      "Couldn't find an opportunity to write block";
+
+    /**
+     * String constant used for generating exception messages indicating the server encountered
+     * IOExceptions multiple times, greater than the configured threshold, while trying to merged
+     * shuffle blocks of the same shuffle partition. When the client receives this this response,
+     * it will stop pushing any more blocks for the same shuffle partition.
+     */
+    public static final String IOEXCEPTIONS_EXCEEDED_THRESHOLD_PREFIX =
+      "IOExceptions exceeded the threshold";
+
+    @Override
+    public boolean shouldRetryError(Throwable t) {
+      // If it is a connection time out or a connection closed exception, no need to retry.
+      if (t.getCause() != null && t.getCause() instanceof ConnectException) {
+        return false;
+      }
+      // If the block is too late, there is no need to retry it
+      return !Throwables.getStackTraceAsString(t).contains(TOO_LATE_MESSAGE_SUFFIX);
+    }
+
+    @Override
+    public boolean shouldLogError(Throwable t) {
+      String errorStackTrace = Throwables.getStackTraceAsString(t);
+      return !errorStackTrace.contains(BLOCK_APPEND_COLLISION_DETECTED_MSG_PREFIX) &&
+        !errorStackTrace.contains(TOO_LATE_MESSAGE_SUFFIX);
+    }
+  }
+}
diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExecutorDiskUtils.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExecutorDiskUtils.java
index 13f6046dd856b..e5e61aae92d2f 100644
--- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExecutorDiskUtils.java
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExecutorDiskUtils.java
@@ -18,17 +18,11 @@
 package org.apache.spark.network.shuffle;
 
 import java.io.File;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import com.google.common.annotations.VisibleForTesting;
 
 import org.apache.spark.network.util.JavaUtils;
 
 public class ExecutorDiskUtils {
 
-  private static final Pattern MULTIPLE_SEPARATORS = Pattern.compile(File.separator + "{2,}");
-
   /**
    * Hashes a filename into the corresponding local directory, in a manner consistent with
    * Spark's DiskBlockManager.getFile().
@@ -37,30 +31,16 @@ public static File getFile(String[] localDirs, int subDirsPerLocalDir, String fi
     int hash = JavaUtils.nonNegativeHash(filename);
     String localDir = localDirs[hash % localDirs.length];
     int subDirId = (hash / localDirs.length) % subDirsPerLocalDir;
-    return new File(createNormalizedInternedPathname(
-        localDir, String.format("%02x", subDirId), filename));
-  }
-
-  /**
-   * This method is needed to avoid the situation when multiple File instances for the
-   * same pathname "foo/bar" are created, each with a separate copy of the "foo/bar" String.
-   * According to measurements, in some scenarios such duplicate strings may waste a lot
-   * of memory (~ 10% of the heap). To avoid that, we intern the pathname, and before that
-   * we make sure that it's in a normalized form (contains no "//", "///" etc.) Otherwise,
-   * the internal code in java.io.File would normalize it later, creating a new "foo/bar"
-   * String copy. Unfortunately, we cannot just reuse the normalization code that java.io.File
-   * uses, since it is in the package-private class java.io.FileSystem.
-   */
-  @VisibleForTesting
-  static String createNormalizedInternedPathname(String dir1, String dir2, String fname) {
-    String pathname = dir1 + File.separator + dir2 + File.separator + fname;
-    Matcher m = MULTIPLE_SEPARATORS.matcher(pathname);
-    pathname = m.replaceAll("/");
-    // A single trailing slash needs to be taken care of separately
-    if (pathname.length() > 1 && pathname.endsWith("/")) {
-      pathname = pathname.substring(0, pathname.length() - 1);
-    }
-    return pathname.intern();
+    final String notNormalizedPath =
+      localDir + File.separator + String.format("%02x", subDirId) + File.separator + filename;
+    // Interning the normalized path as according to measurements, in some scenarios such
+    // duplicate strings may waste a lot of memory (~ 10% of the heap).
+    // Unfortunately, we cannot just call the normalization code that java.io.File
+    // uses, since it is in the package-private class java.io.FileSystem.
+    // So we are creating a File just to get the normalized path back to intern it.
+    // Finally a new File is built and returned with this interned normalized path.
+    final String normalizedInternedPath = new File(notNormalizedPath).getPath().intern();
+    return new File(normalizedInternedPath);
   }
 
 }
diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockHandler.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockHandler.java
index 33865a21ea914..688ee1c01b343 100644
--- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockHandler.java
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockHandler.java
@@ -32,6 +32,7 @@
 import com.codahale.metrics.Timer;
 import com.codahale.metrics.Counter;
 import com.google.common.annotations.VisibleForTesting;
+import org.apache.spark.network.client.StreamCallbackWithID;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -61,11 +62,21 @@ public class ExternalBlockHandler extends RpcHandler {
   final ExternalShuffleBlockResolver blockManager;
   private final OneForOneStreamManager streamManager;
   private final ShuffleMetrics metrics;
+  private final MergedShuffleFileManager mergeManager;
 
   public ExternalBlockHandler(TransportConf conf, File registeredExecutorFile)
     throws IOException {
     this(new OneForOneStreamManager(),
-      new ExternalShuffleBlockResolver(conf, registeredExecutorFile));
+      new ExternalShuffleBlockResolver(conf, registeredExecutorFile),
+      new NoOpMergedShuffleFileManager(conf));
+  }
+
+  public ExternalBlockHandler(
+      TransportConf conf,
+      File registeredExecutorFile,
+      MergedShuffleFileManager mergeManager) throws IOException {
+    this(new OneForOneStreamManager(),
+      new ExternalShuffleBlockResolver(conf, registeredExecutorFile), mergeManager);
   }
 
   @VisibleForTesting
@@ -78,9 +89,19 @@ public ExternalShuffleBlockResolver getBlockResolver() {
   public ExternalBlockHandler(
       OneForOneStreamManager streamManager,
       ExternalShuffleBlockResolver blockManager) {
+    this(streamManager, blockManager, new NoOpMergedShuffleFileManager(null));
+  }
+
+  /** Enables mocking out the StreamManager, BlockManager, and MergeManager. */
+  @VisibleForTesting
+  public ExternalBlockHandler(
+      OneForOneStreamManager streamManager,
+      ExternalShuffleBlockResolver blockManager,
+      MergedShuffleFileManager mergeManager) {
     this.metrics = new ShuffleMetrics();
     this.streamManager = streamManager;
     this.blockManager = blockManager;
+    this.mergeManager = mergeManager;
   }
 
   @Override
@@ -89,6 +110,21 @@ public void receive(TransportClient client, ByteBuffer message, RpcResponseCallb
     handleMessage(msgObj, client, callback);
   }
 
+  @Override
+  public StreamCallbackWithID receiveStream(
+      TransportClient client,
+      ByteBuffer messageHeader,
+      RpcResponseCallback callback) {
+    BlockTransferMessage msgObj = BlockTransferMessage.Decoder.fromByteBuffer(messageHeader);
+    if (msgObj instanceof PushBlockStream) {
+      PushBlockStream message = (PushBlockStream) msgObj;
+      checkAuth(client, message.appId);
+      return mergeManager.receiveBlockDataAsStream(message);
+    } else {
+      throw new UnsupportedOperationException("Unexpected message with #receiveStream: " + msgObj);
+    }
+  }
+
   protected void handleMessage(
       BlockTransferMessage msgObj,
       TransportClient client,
@@ -139,6 +175,7 @@ protected void handleMessage(
         RegisterExecutor msg = (RegisterExecutor) msgObj;
         checkAuth(client, msg.appId);
         blockManager.registerExecutor(msg.appId, msg.execId, msg.executorInfo);
+        mergeManager.registerExecutor(msg.appId, msg.executorInfo);
         callback.onSuccess(ByteBuffer.wrap(new byte[0]));
       } finally {
         responseDelayContext.stop();
@@ -156,6 +193,20 @@ protected void handleMessage(
       Map<String, String[]> localDirs = blockManager.getLocalDirs(msg.appId, msg.execIds);
       callback.onSuccess(new LocalDirsForExecutors(localDirs).toByteBuffer());
 
+    } else if (msgObj instanceof FinalizeShuffleMerge) {
+      final Timer.Context responseDelayContext =
+          metrics.finalizeShuffleMergeLatencyMillis.time();
+      FinalizeShuffleMerge msg = (FinalizeShuffleMerge) msgObj;
+      try {
+        checkAuth(client, msg.appId);
+        MergeStatuses statuses = mergeManager.finalizeShuffleMerge(msg);
+        callback.onSuccess(statuses.toByteBuffer());
+      } catch(IOException e) {
+        throw new RuntimeException(String.format("Error while finalizing shuffle merge "
+          + "for application %s shuffle %d", msg.appId, msg.shuffleId), e);
+      } finally {
+        responseDelayContext.stop();
+      }
     } else {
       throw new UnsupportedOperationException("Unexpected message: " + msgObj);
     }
@@ -181,6 +232,7 @@ public StreamManager getStreamManager() {
    */
   public void applicationRemoved(String appId, boolean cleanupLocalDirs) {
     blockManager.applicationRemoved(appId, cleanupLocalDirs);
+    mergeManager.applicationRemoved(appId, cleanupLocalDirs);
   }
 
   /**
@@ -225,6 +277,8 @@ public class ShuffleMetrics implements MetricSet {
     private final Timer openBlockRequestLatencyMillis = new Timer();
     // Time latency for executor registration latency in ms
     private final Timer registerExecutorRequestLatencyMillis = new Timer();
+    // Time latency for processing finalize shuffle merge request latency in ms
+    private final Timer finalizeShuffleMergeLatencyMillis = new Timer();
     // Block transfer rate in byte per second
     private final Meter blockTransferRateBytes = new Meter();
     // Number of active connections to the shuffle service
@@ -236,6 +290,7 @@ public ShuffleMetrics() {
       allMetrics = new HashMap<>();
       allMetrics.put("openBlockRequestLatencyMillis", openBlockRequestLatencyMillis);
       allMetrics.put("registerExecutorRequestLatencyMillis", registerExecutorRequestLatencyMillis);
+      allMetrics.put("finalizeShuffleMergeLatencyMillis", finalizeShuffleMergeLatencyMillis);
       allMetrics.put("blockTransferRateBytes", blockTransferRateBytes);
       allMetrics.put("registeredExecutorsSize",
                      (Gauge<Integer>) () -> blockManager.getRegisteredExecutorsSize());
@@ -373,6 +428,56 @@ public ManagedBuffer next() {
     }
   }
 
+  /**
+   * Dummy implementation of merged shuffle file manager. Suitable for when push-based shuffle
+   * is not enabled.
+   *
+   * @since 3.1.0
+   */
+  public static class NoOpMergedShuffleFileManager implements MergedShuffleFileManager {
+
+    // This constructor is needed because we use this constructor to instantiate an implementation
+    // of MergedShuffleFileManager using reflection.
+    // See YarnShuffleService#newMergedShuffleFileManagerInstance.
+    public NoOpMergedShuffleFileManager(TransportConf transportConf) {}
+
+    @Override
+    public StreamCallbackWithID receiveBlockDataAsStream(PushBlockStream msg) {
+      throw new UnsupportedOperationException("Cannot handle shuffle block merge");
+    }
+
+    @Override
+    public MergeStatuses finalizeShuffleMerge(FinalizeShuffleMerge msg) throws IOException {
+      throw new UnsupportedOperationException("Cannot handle shuffle block merge");
+    }
+
+    @Override
+    public void registerExecutor(String appId, ExecutorShuffleInfo executorInfo) {
+      // No-Op. Do nothing.
+    }
+
+    @Override
+    public void applicationRemoved(String appId, boolean cleanupLocalDirs) {
+      // No-Op. Do nothing.
+    }
+
+    @Override
+    public ManagedBuffer getMergedBlockData(
+        String appId, int shuffleId, int reduceId, int chunkId) {
+      throw new UnsupportedOperationException("Cannot handle shuffle block merge");
+    }
+
+    @Override
+    public MergedBlockMeta getMergedBlockMeta(String appId, int shuffleId, int reduceId) {
+      throw new UnsupportedOperationException("Cannot handle shuffle block merge");
+    }
+
+    @Override
+    public String[] getMergedBlockDirs(String appId) {
+      throw new UnsupportedOperationException("Cannot handle shuffle block merge");
+    }
+  }
+
   @Override
   public void channelActive(TransportClient client) {
     metrics.activeConnections.inc();
diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockStoreClient.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockStoreClient.java
index 51dc3337261b9..56c06e640acda 100644
--- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockStoreClient.java
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockStoreClient.java
@@ -20,6 +20,7 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
@@ -27,18 +28,16 @@
 
 import com.codahale.metrics.MetricSet;
 import com.google.common.collect.Lists;
+
+import org.apache.spark.network.TransportContext;
+import org.apache.spark.network.buffer.ManagedBuffer;
 import org.apache.spark.network.client.RpcResponseCallback;
 import org.apache.spark.network.client.TransportClient;
 import org.apache.spark.network.client.TransportClientBootstrap;
-import org.apache.spark.network.client.TransportClientFactory;
-import org.apache.spark.network.shuffle.protocol.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.spark.network.TransportContext;
 import org.apache.spark.network.crypto.AuthClientBootstrap;
 import org.apache.spark.network.sasl.SecretKeyHolder;
 import org.apache.spark.network.server.NoOpRpcHandler;
+import org.apache.spark.network.shuffle.protocol.*;
 import org.apache.spark.network.util.TransportConf;
 
 /**
@@ -47,16 +46,13 @@
  * (via BlockTransferService), which has the downside of losing the data if we lose the executors.
  */
 public class ExternalBlockStoreClient extends BlockStoreClient {
-  private static final Logger logger = LoggerFactory.getLogger(ExternalBlockStoreClient.class);
+  private static final ErrorHandler PUSH_ERROR_HANDLER = new ErrorHandler.BlockPushErrorHandler();
 
   private final TransportConf conf;
   private final boolean authEnabled;
   private final SecretKeyHolder secretKeyHolder;
   private final long registrationTimeoutMs;
 
-  protected volatile TransportClientFactory clientFactory;
-  protected String appId;
-
   /**
    * Creates an external shuffle client, with SASL optionally enabled. If SASL is not enabled,
    * then secretKeyHolder may be null.
@@ -72,10 +68,6 @@ public ExternalBlockStoreClient(
     this.registrationTimeoutMs = registrationTimeoutMs;
   }
 
-  protected void checkInit() {
-    assert appId != null : "Called before init()";
-  }
-
   /**
    * Initializes the BlockStoreClient, specifying this Executor's appId.
    * Must be called before any other method on the BlockStoreClient.
@@ -103,12 +95,12 @@ public void fetchBlocks(
     try {
       int maxRetries = conf.maxIORetries();
       RetryingBlockFetcher.BlockFetchStarter blockFetchStarter =
-          (blockIds1, listener1) -> {
+          (inputBlockId, inputListener) -> {
             // Unless this client is closed.
             if (clientFactory != null) {
               TransportClient client = clientFactory.createClient(host, port, maxRetries > 0);
               new OneForOneBlockFetcher(client, appId, execId,
-                blockIds1, listener1, conf, downloadFileManager).start();
+                inputBlockId, inputListener, conf, downloadFileManager).start();
             } else {
               logger.info("This clientFactory was closed. Skipping further block fetch retries.");
             }
@@ -129,6 +121,72 @@ public void fetchBlocks(
     }
   }
 
+  @Override
+  public void pushBlocks(
+      String host,
+      int port,
+      String[] blockIds,
+      ManagedBuffer[] buffers,
+      BlockFetchingListener listener) {
+    checkInit();
+    assert blockIds.length == buffers.length : "Number of block ids and buffers do not match.";
+
+    Map<String, ManagedBuffer> buffersWithId = new HashMap<>();
+    for (int i = 0; i < blockIds.length; i++) {
+      buffersWithId.put(blockIds[i], buffers[i]);
+    }
+    logger.debug("Push {} shuffle blocks to {}:{}", blockIds.length, host, port);
+    try {
+      RetryingBlockFetcher.BlockFetchStarter blockPushStarter =
+          (inputBlockId, inputListener) -> {
+            TransportClient client = clientFactory.createClient(host, port);
+            new OneForOneBlockPusher(client, appId, inputBlockId, inputListener, buffersWithId)
+              .start();
+          };
+      int maxRetries = conf.maxIORetries();
+      if (maxRetries > 0) {
+        new RetryingBlockFetcher(
+          conf, blockPushStarter, blockIds, listener, PUSH_ERROR_HANDLER).start();
+      } else {
+        blockPushStarter.createAndStart(blockIds, listener);
+      }
+    } catch (Exception e) {
+      logger.error("Exception while beginning pushBlocks", e);
+      for (String blockId : blockIds) {
+        listener.onBlockFetchFailure(blockId, e);
+      }
+    }
+  }
+
+  @Override
+  public void finalizeShuffleMerge(
+      String host,
+      int port,
+      int shuffleId,
+      MergeFinalizerListener listener) {
+    checkInit();
+    try {
+      TransportClient client = clientFactory.createClient(host, port);
+      ByteBuffer finalizeShuffleMerge = new FinalizeShuffleMerge(appId, shuffleId).toByteBuffer();
+      client.sendRpc(finalizeShuffleMerge, new RpcResponseCallback() {
+        @Override
+        public void onSuccess(ByteBuffer response) {
+          listener.onShuffleMergeSuccess(
+            (MergeStatuses) BlockTransferMessage.Decoder.fromByteBuffer(response));
+        }
+
+        @Override
+        public void onFailure(Throwable e) {
+          listener.onShuffleMergeFailure(e);
+        }
+      });
+    } catch (Exception e) {
+      logger.error("Exception while sending finalizeShuffleMerge request to {}:{}",
+        host, port, e);
+      listener.onShuffleMergeFailure(e);
+    }
+  }
+
   @Override
   public MetricSet shuffleMetrics() {
     checkInit();
@@ -175,8 +233,6 @@ public void onSuccess(ByteBuffer response) {
           logger.warn("Error trying to remove RDD blocks " + Arrays.toString(blockIds) +
             " via external shuffle service from executor: " + execId, t);
           numRemovedBlocksFuture.complete(0);
-        } finally {
-          client.close();
         }
       }
 
@@ -185,52 +241,11 @@ public void onFailure(Throwable e) {
         logger.warn("Error trying to remove RDD blocks " + Arrays.toString(blockIds) +
           " via external shuffle service from executor: " + execId, e);
         numRemovedBlocksFuture.complete(0);
-        client.close();
       }
     });
     return numRemovedBlocksFuture;
   }
 
-  public void getHostLocalDirs(
-      String host,
-      int port,
-      String[] execIds,
-      CompletableFuture<Map<String, String[]>> hostLocalDirsCompletable) {
-    checkInit();
-    GetLocalDirsForExecutors getLocalDirsMessage = new GetLocalDirsForExecutors(appId, execIds);
-    try {
-      TransportClient client = clientFactory.createClient(host, port);
-      client.sendRpc(getLocalDirsMessage.toByteBuffer(), new RpcResponseCallback() {
-        @Override
-        public void onSuccess(ByteBuffer response) {
-          try {
-            BlockTransferMessage msgObj = BlockTransferMessage.Decoder.fromByteBuffer(response);
-            hostLocalDirsCompletable.complete(
-              ((LocalDirsForExecutors) msgObj).getLocalDirsByExec());
-          } catch (Throwable t) {
-            logger.warn("Error trying to get the host local dirs for " +
-              Arrays.toString(getLocalDirsMessage.execIds) + " via external shuffle service",
-              t.getCause());
-            hostLocalDirsCompletable.completeExceptionally(t);
-          } finally {
-            client.close();
-          }
-        }
-
-        @Override
-        public void onFailure(Throwable t) {
-          logger.warn("Error trying to get the host local dirs for " +
-            Arrays.toString(getLocalDirsMessage.execIds) + " via external shuffle service",
-            t.getCause());
-          hostLocalDirsCompletable.completeExceptionally(t);
-          client.close();
-        }
-      });
-    } catch (IOException | InterruptedException e) {
-      hostLocalDirsCompletable.completeExceptionally(e);
-    }
-  }
-
   @Override
   public void close() {
     checkInit();
diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java
index ba1a17bf7e5ea..a095bf2723418 100644
--- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java
@@ -24,7 +24,6 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executor;
 import java.util.concurrent.Executors;
-import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.builder.ToStringBuilder;
@@ -71,8 +70,6 @@ public class ExternalShuffleBlockResolver {
   private static final String APP_KEY_PREFIX = "AppExecShuffleInfo";
   private static final StoreVersion CURRENT_VERSION = new StoreVersion(1, 0);
 
-  private static final Pattern MULTIPLE_SEPARATORS = Pattern.compile(File.separator + "{2,}");
-
   // Map containing all registered executors' metadata.
   @VisibleForTesting
   final ConcurrentMap<AppExecId, ExecutorShuffleInfo> executors;
@@ -95,10 +92,6 @@ public class ExternalShuffleBlockResolver {
   @VisibleForTesting
   final DB db;
 
-  private final List<String> knownManagers = Arrays.asList(
-    "org.apache.spark.shuffle.sort.SortShuffleManager",
-    "org.apache.spark.shuffle.unsafe.UnsafeShuffleManager");
-
   public ExternalShuffleBlockResolver(TransportConf conf, File registeredExecutorFile)
       throws IOException {
     this(conf, registeredExecutorFile, Executors.newSingleThreadExecutor(
@@ -151,10 +144,6 @@ public void registerExecutor(
       ExecutorShuffleInfo executorInfo) {
     AppExecId fullId = new AppExecId(appId, execId);
     logger.info("Registered executor {} with {}", fullId, executorInfo);
-    if (!knownManagers.contains(executorInfo.shuffleManager)) {
-      throw new UnsupportedOperationException(
-        "Unsupported shuffle manager of executor: " + executorInfo);
-    }
     try {
       if (db != null) {
         byte[] key = dbAppExecKey(fullId);
diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/MergeFinalizerListener.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/MergeFinalizerListener.java
new file mode 100644
index 0000000000000..08e13eea9f40d
--- /dev/null
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/MergeFinalizerListener.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.network.shuffle;
+
+import java.util.EventListener;
+
+import org.apache.spark.network.shuffle.protocol.MergeStatuses;
+
+/**
+ * :: DeveloperApi ::
+ *
+ * Listener providing a callback function to invoke when driver receives the response for the
+ * finalize shuffle merge request sent to remote shuffle service.
+ *
+ * @since 3.1.0
+ */
+public interface MergeFinalizerListener extends EventListener {
+  /**
+   * Called once upon successful response on finalize shuffle merge on a remote shuffle service.
+   * The returned {@link MergeStatuses} is passed to the listener for further processing
+   */
+  void onShuffleMergeSuccess(MergeStatuses statuses);
+
+  /**
+   * Called once upon failure response on finalize shuffle merge on a remote shuffle service.
+   */
+  void onShuffleMergeFailure(Throwable e);
+}
diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/MergedBlockMeta.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/MergedBlockMeta.java
new file mode 100644
index 0000000000000..5541b7460ac96
--- /dev/null
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/MergedBlockMeta.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.network.shuffle;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.google.common.base.Preconditions;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import org.roaringbitmap.RoaringBitmap;
+
+import org.apache.spark.network.buffer.ManagedBuffer;
+import org.apache.spark.network.protocol.Encoders;
+
+/**
+ * Contains meta information for a merged block. Currently this information constitutes:
+ * 1. Number of chunks in a merged shuffle block.
+ * 2. Bitmaps for each chunk in the merged block. A chunk bitmap contains all the mapIds that were
+ *    merged to that merged block chunk.
+ *
+ * @since 3.1.0
+ */
+public class MergedBlockMeta {
+  private final int numChunks;
+  private final ManagedBuffer chunksBitmapBuffer;
+
+  public MergedBlockMeta(int numChunks, ManagedBuffer chunksBitmapBuffer) {
+    this.numChunks = numChunks;
+    this.chunksBitmapBuffer = Preconditions.checkNotNull(chunksBitmapBuffer);
+  }
+
+  public int getNumChunks() {
+    return numChunks;
+  }
+
+  public ManagedBuffer getChunksBitmapBuffer() {
+    return chunksBitmapBuffer;
+  }
+
+  public RoaringBitmap[] readChunkBitmaps() throws IOException {
+    ByteBuf buf = Unpooled.wrappedBuffer(chunksBitmapBuffer.nioByteBuffer());
+    List<RoaringBitmap> bitmaps = new ArrayList<>();
+    while(buf.isReadable()) {
+      bitmaps.add(Encoders.Bitmaps.decode(buf));
+    }
+    assert (bitmaps.size() == numChunks);
+    return bitmaps.toArray(new RoaringBitmap[0]);
+  }
+}
diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/MergedShuffleFileManager.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/MergedShuffleFileManager.java
new file mode 100644
index 0000000000000..4ce6a478ffb61
--- /dev/null
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/MergedShuffleFileManager.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.network.shuffle;
+
+import java.io.IOException;
+
+import org.apache.spark.annotation.Evolving;
+import org.apache.spark.network.buffer.ManagedBuffer;
+import org.apache.spark.network.client.StreamCallbackWithID;
+import org.apache.spark.network.shuffle.protocol.ExecutorShuffleInfo;
+import org.apache.spark.network.shuffle.protocol.FinalizeShuffleMerge;
+import org.apache.spark.network.shuffle.protocol.MergeStatuses;
+import org.apache.spark.network.shuffle.protocol.PushBlockStream;
+
+/**
+ * The MergedShuffleFileManager is used to process push based shuffle when enabled. It works
+ * along side {@link ExternalBlockHandler} and serves as an RPCHandler for
+ * {@link org.apache.spark.network.server.RpcHandler#receiveStream}, where it processes the
+ * remotely pushed streams of shuffle blocks to merge them into merged shuffle files. Right
+ * now, support for push based shuffle is only implemented for external shuffle service in
+ * YARN mode.
+ *
+ * @since 3.1.0
+ */
+@Evolving
+public interface MergedShuffleFileManager {
+  /**
+   * Provides the stream callback used to process a remotely pushed block. The callback is
+   * used by the {@link org.apache.spark.network.client.StreamInterceptor} installed on the
+   * channel to process the block data in the channel outside of the message frame.
+   *
+   * @param msg metadata of the remotely pushed blocks. This is processed inside the message frame
+   * @return A stream callback to process the block data in streaming fashion as it arrives
+   */
+  StreamCallbackWithID receiveBlockDataAsStream(PushBlockStream msg);
+
+  /**
+   * Handles the request to finalize shuffle merge for a given shuffle.
+   *
+   * @param msg contains appId and shuffleId to uniquely identify a shuffle to be finalized
+   * @return The statuses of the merged shuffle partitions for the given shuffle on this
+   *         shuffle service
+   * @throws IOException
+   */
+  MergeStatuses finalizeShuffleMerge(FinalizeShuffleMerge msg) throws IOException;
+
+  /**
+   * Registers an executor with MergedShuffleFileManager. This executor-info provides
+   * the directories and number of sub-dirs per dir so that MergedShuffleFileManager knows where to
+   * store and look for shuffle data for a given application. It is invoked by the RPC call when
+   * executor tries to register with the local shuffle service.
+   *
+   * @param appId application ID
+   * @param executorInfo The list of local dirs that this executor gets granted from NodeManager
+   */
+  void registerExecutor(String appId, ExecutorShuffleInfo executorInfo);
+
+  /**
+   * Invoked when an application finishes. This cleans up any remaining metadata associated with
+   * this application, and optionally deletes the application specific directory path.
+   *
+   * @param appId application ID
+   * @param cleanupLocalDirs flag indicating whether MergedShuffleFileManager should handle
+   *                         deletion of local dirs itself.
+   */
+  void applicationRemoved(String appId, boolean cleanupLocalDirs);
+
+  /**
+   * Get the buffer for a given merged shuffle chunk when serving merged shuffle to reducers
+   *
+   * @param appId application ID
+   * @param shuffleId shuffle ID
+   * @param reduceId reducer ID
+   * @param chunkId merged shuffle file chunk ID
+   * @return The {@link ManagedBuffer} for the given merged shuffle chunk
+   */
+  ManagedBuffer getMergedBlockData(String appId, int shuffleId, int reduceId, int chunkId);
+
+  /**
+   * Get the meta information of a merged block.
+   *
+   * @param appId application ID
+   * @param shuffleId shuffle ID
+   * @param reduceId reducer ID
+   * @return meta information of a merged block
+   */
+  MergedBlockMeta getMergedBlockMeta(String appId, int shuffleId, int reduceId);
+
+  /**
+   * Get the local directories which stores the merged shuffle files.
+   *
+   * @param appId application ID
+   */
+  String[] getMergedBlockDirs(String appId);
+}
diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/OneForOneBlockFetcher.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/OneForOneBlockFetcher.java
index ec2e3dce661d9..0b7eaa6225a41 100644
--- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/OneForOneBlockFetcher.java
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/OneForOneBlockFetcher.java
@@ -21,7 +21,7 @@
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 
 import com.google.common.primitives.Ints;
 import com.google.common.primitives.Longs;
@@ -81,7 +81,6 @@ public OneForOneBlockFetcher(
       TransportConf transportConf,
       DownloadFileManager downloadFileManager) {
     this.client = client;
-    this.blockIds = blockIds;
     this.listener = listener;
     this.chunkCallback = new ChunkCallback();
     this.transportConf = transportConf;
@@ -90,8 +89,10 @@ public OneForOneBlockFetcher(
       throw new IllegalArgumentException("Zero-sized blockIds array");
     }
     if (!transportConf.useOldFetchProtocol() && isShuffleBlocks(blockIds)) {
-      this.message = createFetchShuffleBlocksMsg(appId, execId, blockIds);
+      this.blockIds = new String[blockIds.length];
+      this.message = createFetchShuffleBlocksMsgAndBuildBlockIds(appId, execId, blockIds);
     } else {
+      this.blockIds = blockIds;
       this.message = new OpenBlocks(appId, execId, blockIds);
     }
   }
@@ -106,17 +107,16 @@ private boolean isShuffleBlocks(String[] blockIds) {
   }
 
   /**
-   * Analyze the pass in blockIds and create FetchShuffleBlocks message.
-   * The blockIds has been sorted by mapId and reduceId. It's produced in
-   * org.apache.spark.MapOutputTracker.convertMapStatuses.
+   * Create FetchShuffleBlocks message and rebuild internal blockIds by
+   * analyzing the pass in blockIds.
    */
-  private FetchShuffleBlocks createFetchShuffleBlocksMsg(
+  private FetchShuffleBlocks createFetchShuffleBlocksMsgAndBuildBlockIds(
       String appId, String execId, String[] blockIds) {
     String[] firstBlock = splitBlockId(blockIds[0]);
     int shuffleId = Integer.parseInt(firstBlock[1]);
     boolean batchFetchEnabled = firstBlock.length == 5;
 
-    HashMap<Long, ArrayList<Integer>> mapIdToReduceIds = new HashMap<>();
+    LinkedHashMap<Long, BlocksInfo> mapIdToBlocksInfo = new LinkedHashMap<>();
     for (String blockId : blockIds) {
       String[] blockIdParts = splitBlockId(blockId);
       if (Integer.parseInt(blockIdParts[1]) != shuffleId) {
@@ -124,23 +124,36 @@ private FetchShuffleBlocks createFetchShuffleBlocksMsg(
           ", got:" + blockId);
       }
       long mapId = Long.parseLong(blockIdParts[2]);
-      if (!mapIdToReduceIds.containsKey(mapId)) {
-        mapIdToReduceIds.put(mapId, new ArrayList<>());
+      if (!mapIdToBlocksInfo.containsKey(mapId)) {
+        mapIdToBlocksInfo.put(mapId, new BlocksInfo());
       }
-      mapIdToReduceIds.get(mapId).add(Integer.parseInt(blockIdParts[3]));
+      BlocksInfo blocksInfoByMapId = mapIdToBlocksInfo.get(mapId);
+      blocksInfoByMapId.blockIds.add(blockId);
+      blocksInfoByMapId.reduceIds.add(Integer.parseInt(blockIdParts[3]));
       if (batchFetchEnabled) {
         // When we read continuous shuffle blocks in batch, we will reuse reduceIds in
         // FetchShuffleBlocks to store the start and end reduce id for range
         // [startReduceId, endReduceId).
         assert(blockIdParts.length == 5);
-        mapIdToReduceIds.get(mapId).add(Integer.parseInt(blockIdParts[4]));
+        blocksInfoByMapId.reduceIds.add(Integer.parseInt(blockIdParts[4]));
       }
     }
-    long[] mapIds = Longs.toArray(mapIdToReduceIds.keySet());
+    long[] mapIds = Longs.toArray(mapIdToBlocksInfo.keySet());
     int[][] reduceIdArr = new int[mapIds.length][];
+    int blockIdIndex = 0;
     for (int i = 0; i < mapIds.length; i++) {
-      reduceIdArr[i] = Ints.toArray(mapIdToReduceIds.get(mapIds[i]));
+      BlocksInfo blocksInfoByMapId = mapIdToBlocksInfo.get(mapIds[i]);
+      reduceIdArr[i] = Ints.toArray(blocksInfoByMapId.reduceIds);
+
+      // The `blockIds`'s order must be same with the read order specified in in FetchShuffleBlocks
+      // because the shuffle data's return order should match the `blockIds`'s order to ensure
+      // blockId and data match.
+      for (int j = 0; j < blocksInfoByMapId.blockIds.size(); j++) {
+        this.blockIds[blockIdIndex++] = blocksInfoByMapId.blockIds.get(j);
+      }
     }
+    assert(blockIdIndex == this.blockIds.length);
+
     return new FetchShuffleBlocks(
       appId, execId, shuffleId, mapIds, reduceIdArr, batchFetchEnabled);
   }
@@ -157,6 +170,18 @@ private String[] splitBlockId(String blockId) {
     return blockIdParts;
   }
 
+  /** The reduceIds and blocks in a single mapId */
+  private class BlocksInfo {
+
+    final ArrayList<Integer> reduceIds;
+    final ArrayList<String> blockIds;
+
+    BlocksInfo() {
+      this.reduceIds = new ArrayList<>();
+      this.blockIds = new ArrayList<>();
+    }
+  }
+
   /** Callback invoked on receipt of each chunk. We equate a single chunk to a single block. */
   private class ChunkCallback implements ChunkReceivedCallback {
     @Override
diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/OneForOneBlockPusher.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/OneForOneBlockPusher.java
new file mode 100644
index 0000000000000..6ee95ef0dea01
--- /dev/null
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/OneForOneBlockPusher.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.network.shuffle;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.spark.network.buffer.ManagedBuffer;
+import org.apache.spark.network.buffer.NioManagedBuffer;
+import org.apache.spark.network.client.RpcResponseCallback;
+import org.apache.spark.network.client.TransportClient;
+import org.apache.spark.network.shuffle.protocol.PushBlockStream;
+
+/**
+ * Similar to {@link OneForOneBlockFetcher}, but for pushing blocks to remote shuffle service to
+ * be merged instead of for fetching them from remote shuffle services. This is used by
+ * ShuffleWriter when the block push process is initiated. The supplied BlockFetchingListener
+ * is used to handle the success or failure in pushing each blocks.
+ *
+ * @since 3.1.0
+ */
+public class OneForOneBlockPusher {
+  private static final Logger logger = LoggerFactory.getLogger(OneForOneBlockPusher.class);
+  private static final ErrorHandler PUSH_ERROR_HANDLER = new ErrorHandler.BlockPushErrorHandler();
+  public static final String SHUFFLE_PUSH_BLOCK_PREFIX = "shufflePush";
+
+  private final TransportClient client;
+  private final String appId;
+  private final String[] blockIds;
+  private final BlockFetchingListener listener;
+  private final Map<String, ManagedBuffer> buffers;
+
+  public OneForOneBlockPusher(
+      TransportClient client,
+      String appId,
+      String[] blockIds,
+      BlockFetchingListener listener,
+      Map<String, ManagedBuffer> buffers) {
+    this.client = client;
+    this.appId = appId;
+    this.blockIds = blockIds;
+    this.listener = listener;
+    this.buffers = buffers;
+  }
+
+  private class BlockPushCallback implements RpcResponseCallback {
+
+    private int index;
+    private String blockId;
+
+    BlockPushCallback(int index, String blockId) {
+      this.index = index;
+      this.blockId = blockId;
+    }
+
+    @Override
+    public void onSuccess(ByteBuffer response) {
+      // On receipt of a successful block push
+      listener.onBlockFetchSuccess(blockId, new NioManagedBuffer(ByteBuffer.allocate(0)));
+    }
+
+    @Override
+    public void onFailure(Throwable e) {
+      // Since block push is best effort, i.e., if we encountered a block push failure that's not
+      // retriable or exceeding the max retires, we should not fail all remaining block pushes.
+      // The best effort nature makes block push tolerable of a partial completion. Thus, we only
+      // fail the block that's actually failed. Not that, on the RetryingBlockFetcher side, once
+      // retry is initiated, it would still invalidate the previous active retry listener, and
+      // retry all outstanding blocks. We are preventing forwarding unnecessary block push failures
+      // to the parent listener of the retry listener. The only exceptions would be if the block
+      // push failure is due to block arriving on the server side after merge finalization, or the
+      // client fails to establish connection to the server side. In both cases, we would fail all
+      // remaining blocks.
+      if (PUSH_ERROR_HANDLER.shouldRetryError(e)) {
+        String[] targetBlockId = Arrays.copyOfRange(blockIds, index, index + 1);
+        failRemainingBlocks(targetBlockId, e);
+      } else {
+        String[] targetBlockId = Arrays.copyOfRange(blockIds, index, blockIds.length);
+        failRemainingBlocks(targetBlockId, e);
+      }
+    }
+  }
+
+  private void failRemainingBlocks(String[] failedBlockIds, Throwable e) {
+    for (String blockId : failedBlockIds) {
+      try {
+        listener.onBlockFetchFailure(blockId, e);
+      } catch (Exception e2) {
+        logger.error("Error in block push failure callback", e2);
+      }
+    }
+  }
+
+  /**
+   * Begins the block pushing process, calling the listener with every block pushed.
+   */
+  public void start() {
+    logger.debug("Start pushing {} blocks", blockIds.length);
+    for (int i = 0; i < blockIds.length; i++) {
+      assert buffers.containsKey(blockIds[i]) : "Could not find the block buffer for block "
+        + blockIds[i];
+      String[] blockIdParts = blockIds[i].split("_");
+      if (blockIdParts.length != 4 || !blockIdParts[0].equals(SHUFFLE_PUSH_BLOCK_PREFIX)) {
+        throw new IllegalArgumentException(
+          "Unexpected shuffle push block id format: " + blockIds[i]);
+      }
+      ByteBuffer header = new PushBlockStream(appId, Integer.parseInt(blockIdParts[1]),
+        Integer.parseInt(blockIdParts[2]), Integer.parseInt(blockIdParts[3]) , i).toByteBuffer();
+      client.uploadStream(new NioManagedBuffer(header), buffers.get(blockIds[i]),
+        new BlockPushCallback(i, blockIds[i]));
+    }
+  }
+}
diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/RemoteBlockPushResolver.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/RemoteBlockPushResolver.java
new file mode 100644
index 0000000000000..9363efc58d7c6
--- /dev/null
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/RemoteBlockPushResolver.java
@@ -0,0 +1,1066 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.network.shuffle;
+
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.cache.Weigher;
+import com.google.common.collect.Maps;
+import com.google.common.primitives.Ints;
+import com.google.common.primitives.Longs;
+import org.roaringbitmap.RoaringBitmap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.spark.network.buffer.FileSegmentManagedBuffer;
+import org.apache.spark.network.buffer.ManagedBuffer;
+import org.apache.spark.network.client.StreamCallbackWithID;
+import org.apache.spark.network.shuffle.protocol.ExecutorShuffleInfo;
+import org.apache.spark.network.shuffle.protocol.FinalizeShuffleMerge;
+import org.apache.spark.network.shuffle.protocol.MergeStatuses;
+import org.apache.spark.network.shuffle.protocol.PushBlockStream;
+import org.apache.spark.network.util.JavaUtils;
+import org.apache.spark.network.util.NettyUtils;
+import org.apache.spark.network.util.TransportConf;
+
+/**
+ * An implementation of {@link MergedShuffleFileManager} that provides the most essential shuffle
+ * service processing logic to support push based shuffle.
+ *
+ * @since 3.1.0
+ */
+public class RemoteBlockPushResolver implements MergedShuffleFileManager {
+
+  private static final Logger logger = LoggerFactory.getLogger(RemoteBlockPushResolver.class);
+  @VisibleForTesting
+  static final String MERGE_MANAGER_DIR = "merge_manager";
+
+  private final ConcurrentMap<String, AppPathsInfo> appsPathInfo;
+  private final ConcurrentMap<AppShuffleId, Map<Integer, AppShufflePartitionInfo>> partitions;
+
+  private final Executor directoryCleaner;
+  private final TransportConf conf;
+  private final int minChunkSize;
+  private final int ioExceptionsThresholdDuringMerge;
+  private final ErrorHandler.BlockPushErrorHandler errorHandler;
+
+  @SuppressWarnings("UnstableApiUsage")
+  private final LoadingCache<File, ShuffleIndexInformation> indexCache;
+
+  @SuppressWarnings("UnstableApiUsage")
+  public RemoteBlockPushResolver(TransportConf conf) {
+    this.conf = conf;
+    this.partitions = Maps.newConcurrentMap();
+    this.appsPathInfo = Maps.newConcurrentMap();
+    this.directoryCleaner = Executors.newSingleThreadExecutor(
+      // Add `spark` prefix because it will run in NM in Yarn mode.
+      NettyUtils.createThreadFactory("spark-shuffle-merged-shuffle-directory-cleaner"));
+    this.minChunkSize = conf.minChunkSizeInMergedShuffleFile();
+    this.ioExceptionsThresholdDuringMerge = conf.ioExceptionsThresholdDuringMerge();
+    CacheLoader<File, ShuffleIndexInformation> indexCacheLoader =
+      new CacheLoader<File, ShuffleIndexInformation>() {
+        public ShuffleIndexInformation load(File file) throws IOException {
+          return new ShuffleIndexInformation(file);
+        }
+      };
+    indexCache = CacheBuilder.newBuilder()
+      .maximumWeight(conf.mergedIndexCacheSize())
+      .weigher((Weigher<File, ShuffleIndexInformation>) (file, indexInfo) -> indexInfo.getSize())
+      .build(indexCacheLoader);
+    this.errorHandler = new ErrorHandler.BlockPushErrorHandler();
+  }
+
+  /**
+   * Given the appShuffleId and reduceId that uniquely identifies a given shuffle partition of an
+   * application, retrieves the associated metadata. If not present and the corresponding merged
+   * shuffle does not exist, initializes the metadata.
+   */
+  private AppShufflePartitionInfo getOrCreateAppShufflePartitionInfo(
+      AppShuffleId appShuffleId,
+      int reduceId) {
+    File dataFile = getMergedShuffleDataFile(appShuffleId, reduceId);
+    if (!partitions.containsKey(appShuffleId) && dataFile.exists()) {
+      // If this partition is already finalized then the partitions map will not contain
+      // the appShuffleId but the data file would exist. In that case the block is considered late.
+      return null;
+    }
+    Map<Integer, AppShufflePartitionInfo> shufflePartitions =
+      partitions.computeIfAbsent(appShuffleId, id -> Maps.newConcurrentMap());
+    return shufflePartitions.computeIfAbsent(reduceId, key -> {
+      // It only gets here when the key is not present in the map. This could either
+      // be the first time the merge manager receives a pushed block for a given application
+      // shuffle partition, or after the merged shuffle file is finalized. We handle these
+      // two cases accordingly by checking if the file already exists.
+      File indexFile = getMergedShuffleIndexFile(appShuffleId, reduceId);
+      File metaFile = getMergedShuffleMetaFile(appShuffleId, reduceId);
+      try {
+        if (dataFile.exists()) {
+          return null;
+        } else {
+          return newAppShufflePartitionInfo(appShuffleId, reduceId, dataFile, indexFile, metaFile);
+        }
+      } catch (IOException e) {
+        logger.error(
+          "Cannot create merged shuffle partition with data file {}, index file {}, and "
+            + "meta file {}", dataFile.getAbsolutePath(),
+            indexFile.getAbsolutePath(), metaFile.getAbsolutePath());
+        throw new RuntimeException(
+          String.format("Cannot initialize merged shuffle partition for appId %s shuffleId %s "
+          + "reduceId %s", appShuffleId.appId, appShuffleId.shuffleId, reduceId), e);
+      }
+    });
+  }
+
+  @VisibleForTesting
+  AppShufflePartitionInfo newAppShufflePartitionInfo(
+      AppShuffleId appShuffleId,
+      int reduceId,
+      File dataFile,
+      File indexFile,
+      File metaFile) throws IOException {
+    return new AppShufflePartitionInfo(appShuffleId, reduceId, dataFile,
+      new MergeShuffleFile(indexFile), new MergeShuffleFile(metaFile));
+  }
+
+  @Override
+  public MergedBlockMeta getMergedBlockMeta(String appId, int shuffleId, int reduceId) {
+    AppShuffleId appShuffleId = new AppShuffleId(appId, shuffleId);
+    File indexFile = getMergedShuffleIndexFile(appShuffleId, reduceId);
+    if (!indexFile.exists()) {
+      throw new RuntimeException(String.format(
+        "Merged shuffle index file %s not found", indexFile.getPath()));
+    }
+    int size = (int) indexFile.length();
+    // First entry is the zero offset
+    int numChunks = (size / Long.BYTES) - 1;
+    File metaFile = getMergedShuffleMetaFile(appShuffleId, reduceId);
+    if (!metaFile.exists()) {
+      throw new RuntimeException(String.format("Merged shuffle meta file %s not found",
+        metaFile.getPath()));
+    }
+    FileSegmentManagedBuffer chunkBitMaps =
+      new FileSegmentManagedBuffer(conf, metaFile, 0L, metaFile.length());
+    logger.trace(
+      "{} shuffleId {} reduceId {} num chunks {}", appId, shuffleId, reduceId, numChunks);
+    return new MergedBlockMeta(numChunks, chunkBitMaps);
+  }
+
+  @SuppressWarnings("UnstableApiUsage")
+  @Override
+  public ManagedBuffer getMergedBlockData(String appId, int shuffleId, int reduceId, int chunkId) {
+    AppShuffleId appShuffleId = new AppShuffleId(appId, shuffleId);
+    File dataFile = getMergedShuffleDataFile(appShuffleId, reduceId);
+    if (!dataFile.exists()) {
+      throw new RuntimeException(String.format("Merged shuffle data file %s not found",
+        dataFile.getPath()));
+    }
+    File indexFile = getMergedShuffleIndexFile(appShuffleId, reduceId);
+    try {
+      // If we get here, the merged shuffle file should have been properly finalized. Thus we can
+      // use the file length to determine the size of the merged shuffle block.
+      ShuffleIndexInformation shuffleIndexInformation = indexCache.get(indexFile);
+      ShuffleIndexRecord shuffleIndexRecord = shuffleIndexInformation.getIndex(chunkId);
+      return new FileSegmentManagedBuffer(
+        conf, dataFile, shuffleIndexRecord.getOffset(), shuffleIndexRecord.getLength());
+    } catch (ExecutionException e) {
+      throw new RuntimeException(String.format(
+        "Failed to open merged shuffle index file %s", indexFile.getPath()), e);
+    }
+  }
+
+  /**
+   * The logic here is consistent with
+   * org.apache.spark.storage.DiskBlockManager#getMergedShuffleFile
+   */
+  private File getFile(String appId, String filename) {
+    // TODO: [SPARK-33236] Change the message when this service is able to handle NM restart
+    AppPathsInfo appPathsInfo = Preconditions.checkNotNull(appsPathInfo.get(appId),
+      "application " + appId + " is not registered or NM was restarted.");
+    File targetFile = ExecutorDiskUtils.getFile(appPathsInfo.activeLocalDirs,
+      appPathsInfo.subDirsPerLocalDir, filename);
+    logger.debug("Get merged file {}", targetFile.getAbsolutePath());
+    return targetFile;
+  }
+
+  private File getMergedShuffleDataFile(AppShuffleId appShuffleId, int reduceId) {
+    String fileName = String.format("%s.data", generateFileName(appShuffleId, reduceId));
+    return getFile(appShuffleId.appId, fileName);
+  }
+
+  private File getMergedShuffleIndexFile(AppShuffleId appShuffleId, int reduceId) {
+    String indexName = String.format("%s.index", generateFileName(appShuffleId, reduceId));
+    return getFile(appShuffleId.appId, indexName);
+  }
+
+  private File getMergedShuffleMetaFile(AppShuffleId appShuffleId, int reduceId) {
+    String metaName = String.format("%s.meta", generateFileName(appShuffleId, reduceId));
+    return getFile(appShuffleId.appId, metaName);
+  }
+
+  @Override
+  public String[] getMergedBlockDirs(String appId) {
+    AppPathsInfo appPathsInfo = Preconditions.checkNotNull(appsPathInfo.get(appId),
+      "application " + appId + " is not registered or NM was restarted.");
+    String[] activeLocalDirs = Preconditions.checkNotNull(appPathsInfo.activeLocalDirs,
+      "application " + appId
+      + " active local dirs list has not been updated by any executor registration");
+    return activeLocalDirs;
+  }
+
+  @Override
+  public void applicationRemoved(String appId, boolean cleanupLocalDirs) {
+    logger.info("Application {} removed, cleanupLocalDirs = {}", appId, cleanupLocalDirs);
+    // TODO: [SPARK-33236] Change the message when this service is able to handle NM restart
+    AppPathsInfo appPathsInfo = Preconditions.checkNotNull(appsPathInfo.remove(appId),
+      "application " + appId + " is not registered or NM was restarted.");
+    Iterator<Map.Entry<AppShuffleId, Map<Integer, AppShufflePartitionInfo>>> iterator =
+      partitions.entrySet().iterator();
+    while (iterator.hasNext()) {
+      Map.Entry<AppShuffleId, Map<Integer, AppShufflePartitionInfo>> entry = iterator.next();
+      AppShuffleId appShuffleId = entry.getKey();
+      if (appId.equals(appShuffleId.appId)) {
+        iterator.remove();
+        for (AppShufflePartitionInfo partitionInfo : entry.getValue().values()) {
+          partitionInfo.closeAllFiles();
+        }
+      }
+    }
+    if (cleanupLocalDirs) {
+      Path[] dirs = Arrays.stream(appPathsInfo.activeLocalDirs)
+        .map(dir -> Paths.get(dir)).toArray(Path[]::new);
+      directoryCleaner.execute(() -> deleteExecutorDirs(dirs));
+    }
+  }
+
+  /**
+   * Serially delete local dirs, executed in a separate thread.
+   */
+  @VisibleForTesting
+  void deleteExecutorDirs(Path[] dirs) {
+    for (Path localDir : dirs) {
+      try {
+        if (Files.exists(localDir)) {
+          JavaUtils.deleteRecursively(localDir.toFile());
+          logger.debug("Successfully cleaned up directory: {}", localDir);
+        }
+      } catch (Exception e) {
+        logger.error("Failed to delete directory: {}", localDir, e);
+      }
+    }
+  }
+
+  @Override
+  public StreamCallbackWithID receiveBlockDataAsStream(PushBlockStream msg) {
+    // Retrieve merged shuffle file metadata
+    AppShuffleId appShuffleId = new AppShuffleId(msg.appId, msg.shuffleId);
+    AppShufflePartitionInfo partitionInfoBeforeCheck =
+      getOrCreateAppShufflePartitionInfo(appShuffleId, msg.reduceId);
+    // Here partitionInfo will be null in 2 cases:
+    // 1) The request is received for a block that has already been merged, this is possible due
+    // to the retry logic.
+    // 2) The request is received after the merged shuffle is finalized, thus is too late.
+    //
+    // For case 1, we will drain the data in the channel and just respond success
+    // to the client. This is required because the response of the previously merged
+    // block will be ignored by the client, per the logic in RetryingBlockFetcher.
+    // Note that the netty server should receive data for a given block id only from 1 channel
+    // at any time. The block should be pushed only from successful maps, thus there should be
+    // only 1 source for a given block at any time. Although the netty client might retry sending
+    // this block to the server multiple times, the data of the same block always arrives from the
+    // same channel thus the server should have already processed the previous request of this
+    // block before seeing it again in the channel. This guarantees that we can simply just
+    // check the bitmap to determine if a block is a duplicate or not.
+    //
+    // For case 2, we will also drain the data in the channel, but throw an exception in
+    // {@link org.apache.spark.network.client.StreamCallback#onComplete(String)}. This way,
+    // the client will be notified of the failure but the channel will remain active. Keeping
+    // the channel alive is important because the same channel could be reused by multiple map
+    // tasks in the executor JVM, which belongs to different stages. While one of the shuffles
+    // in these stages is finalized, the others might still be active. Tearing down the channel
+    // on the server side will disrupt these other on-going shuffle merges. It's also important
+    // to notify the client of the failure, so that it can properly halt pushing the remaining
+    // blocks upon receiving such failures to preserve resources on the server/client side.
+    //
+    // Speculative execution would also raise a possible scenario with duplicate blocks. Although
+    // speculative execution would kill the slower task attempt, leading to only 1 task attempt
+    // succeeding in the end, there is no guarantee that only one copy of the block will be
+    // pushed. This is due to our handling of block push process outside of the map task, thus
+    // it is possible for the speculative task attempt to initiate the block push process before
+    // getting killed. When this happens, we need to distinguish the duplicate blocks as they
+    // arrive. More details on this is explained in later comments.
+
+    // Track if the block is received after shuffle merge finalize
+    final boolean isTooLate = partitionInfoBeforeCheck == null;
+    // Check if the given block is already merged by checking the bitmap against the given map index
+    final AppShufflePartitionInfo partitionInfo = partitionInfoBeforeCheck != null
+      && partitionInfoBeforeCheck.mapTracker.contains(msg.mapIndex) ? null
+        : partitionInfoBeforeCheck;
+    final String streamId = String.format("%s_%d_%d_%d",
+      OneForOneBlockPusher.SHUFFLE_PUSH_BLOCK_PREFIX, appShuffleId.shuffleId, msg.mapIndex,
+      msg.reduceId);
+    if (partitionInfo != null) {
+      return new PushBlockStreamCallback(this, streamId, partitionInfo, msg.mapIndex);
+    } else {
+      // For a duplicate block or a block which is late, respond back with a callback that handles
+      // them differently.
+      return new StreamCallbackWithID() {
+        @Override
+        public String getID() {
+          return streamId;
+        }
+
+        @Override
+        public void onData(String streamId, ByteBuffer buf) {
+          // Ignore the requests. It reaches here either when a request is received after the
+          // shuffle file is finalized or when a request is for a duplicate block.
+        }
+
+        @Override
+        public void onComplete(String streamId) {
+          if (isTooLate) {
+            // Throw an exception here so the block data is drained from channel and server
+            // responds RpcFailure to the client.
+            throw new RuntimeException(String.format("Block %s %s", streamId,
+              ErrorHandler.BlockPushErrorHandler.TOO_LATE_MESSAGE_SUFFIX));
+          }
+          // For duplicate block that is received before the shuffle merge finalizes, the
+          // server should respond success to the client.
+        }
+
+        @Override
+        public void onFailure(String streamId, Throwable cause) {
+        }
+      };
+    }
+  }
+
+  @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
+  @Override
+  public MergeStatuses finalizeShuffleMerge(FinalizeShuffleMerge msg) throws IOException {
+    logger.info("Finalizing shuffle {} from Application {}.", msg.shuffleId, msg.appId);
+    AppShuffleId appShuffleId = new AppShuffleId(msg.appId, msg.shuffleId);
+    Map<Integer, AppShufflePartitionInfo> shufflePartitions = partitions.get(appShuffleId);
+    MergeStatuses mergeStatuses;
+    if (shufflePartitions == null || shufflePartitions.isEmpty()) {
+      mergeStatuses =
+        new MergeStatuses(msg.shuffleId, new RoaringBitmap[0], new int[0], new long[0]);
+    } else {
+      Collection<AppShufflePartitionInfo> partitionsToFinalize = shufflePartitions.values();
+      List<RoaringBitmap> bitmaps = new ArrayList<>(partitionsToFinalize.size());
+      List<Integer> reduceIds = new ArrayList<>(partitionsToFinalize.size());
+      List<Long> sizes = new ArrayList<>(partitionsToFinalize.size());
+      Iterator<AppShufflePartitionInfo> partitionsIter = partitionsToFinalize.iterator();
+      while (partitionsIter.hasNext()) {
+        AppShufflePartitionInfo partition = partitionsIter.next();
+        synchronized (partition) {
+          try {
+            // This can throw IOException which will marks this shuffle partition as not merged.
+            partition.finalizePartition();
+            bitmaps.add(partition.mapTracker);
+            reduceIds.add(partition.reduceId);
+            sizes.add(partition.getLastChunkOffset());
+          } catch (IOException ioe) {
+            logger.warn("Exception while finalizing shuffle partition {} {} {}", msg.appId,
+              msg.shuffleId, partition.reduceId, ioe);
+          } finally {
+            partition.closeAllFiles();
+            // The partition should be removed after the files are written so that any new stream
+            // for the same reduce partition will see that the data file exists.
+            partitionsIter.remove();
+          }
+        }
+      }
+      mergeStatuses = new MergeStatuses(msg.shuffleId,
+        bitmaps.toArray(new RoaringBitmap[bitmaps.size()]), Ints.toArray(reduceIds),
+        Longs.toArray(sizes));
+    }
+    partitions.remove(appShuffleId);
+    logger.info("Finalized shuffle {} from Application {}.", msg.shuffleId, msg.appId);
+    return mergeStatuses;
+  }
+
+  @Override
+  public void registerExecutor(String appId, ExecutorShuffleInfo executorInfo) {
+    if (logger.isDebugEnabled()) {
+      logger.debug("register executor with RemoteBlockPushResolver {} local-dirs {} "
+        + "num sub-dirs {}", appId, Arrays.toString(executorInfo.localDirs),
+          executorInfo.subDirsPerLocalDir);
+    }
+    appsPathInfo.computeIfAbsent(appId, id -> new AppPathsInfo(appId, executorInfo.localDirs,
+      executorInfo.subDirsPerLocalDir));
+  }
+  private static String generateFileName(AppShuffleId appShuffleId, int reduceId) {
+    return String.format("mergedShuffle_%s_%d_%d", appShuffleId.appId, appShuffleId.shuffleId,
+      reduceId);
+  }
+
+  /**
+   * Callback for push stream that handles blocks which are not already merged.
+   */
+  static class PushBlockStreamCallback implements StreamCallbackWithID {
+
+    private final RemoteBlockPushResolver mergeManager;
+    private final String streamId;
+    private final int mapIndex;
+    private final AppShufflePartitionInfo partitionInfo;
+    private int length = 0;
+    // This indicates that this stream got the opportunity to write the blocks to the merged file.
+    // Once this is set to true and the stream encounters a failure then it will unset the
+    // currentMapId of the partition so that another stream can start merging the blocks to the
+    // partition. This is reset to false when the stream completes.
+    private boolean isWriting = false;
+    // Use on-heap instead of direct ByteBuffer since these buffers will be GC'ed very quickly
+    private List<ByteBuffer> deferredBufs;
+
+    private PushBlockStreamCallback(
+        RemoteBlockPushResolver mergeManager,
+        String streamId,
+        AppShufflePartitionInfo partitionInfo,
+        int mapIndex) {
+      this.mergeManager = Preconditions.checkNotNull(mergeManager);
+      this.streamId = streamId;
+      this.partitionInfo = Preconditions.checkNotNull(partitionInfo);
+      this.mapIndex = mapIndex;
+      abortIfNecessary();
+    }
+
+    @Override
+    public String getID() {
+      return streamId;
+    }
+
+    /**
+     * Write a ByteBuffer to the merged shuffle file. Here we keep track of the length of the
+     * block data written to file. In case of failure during writing block to file, we use the
+     * information tracked in partitionInfo to overwrite the corrupt block when writing the new
+     * block.
+     */
+    private void writeBuf(ByteBuffer buf) throws IOException {
+      while (buf.hasRemaining()) {
+        long updatedPos = partitionInfo.getDataFilePos() + length;
+        logger.debug("{} shuffleId {} reduceId {} current pos {} updated pos {}",
+          partitionInfo.appShuffleId.appId, partitionInfo.appShuffleId.shuffleId,
+          partitionInfo.reduceId, partitionInfo.getDataFilePos(), updatedPos);
+        length += partitionInfo.dataChannel.write(buf, updatedPos);
+      }
+    }
+
+    /**
+     * There will be multiple streams of map blocks belonging to the same reduce partition. At any
+     * given point of time, only a single map stream can write its data to the merged file. Until
+     * this stream is completed, the other streams defer writing. This prevents corruption of
+     * merged data. This returns whether this stream is the active stream that can write to the
+     * merged file.
+     */
+    private boolean allowedToWrite() {
+      return partitionInfo.getCurrentMapIndex() < 0
+        || partitionInfo.getCurrentMapIndex() == mapIndex;
+    }
+
+    /**
+     * Returns if this is a duplicate block generated by speculative tasks. With speculative
+     * tasks, we could receive the same block from 2 different sources at the same time. One of
+     * them is going to be the first to set the currentMapIndex. When that block does so, it's
+     * going to see the currentMapIndex initially as -1. After it sets the currentMapIndex, it's
+     * going to write some data to disk, thus increasing the length counter. The other duplicate
+     * block is going to see the currentMapIndex already set to its mapIndex. However, it hasn't
+     * written any data yet. If the first block gets written completely and resets the
+     * currentMapIndex to -1 before the processing for the second block finishes, we can just
+     * check the bitmap to identify the second as a duplicate.
+     */
+    private boolean isDuplicateBlock() {
+      return (partitionInfo.getCurrentMapIndex() == mapIndex && length == 0)
+        || partitionInfo.mapTracker.contains(mapIndex);
+    }
+
+    /**
+     * This is only invoked when the stream is able to write. The stream first writes any deferred
+     * block parts buffered in memory.
+     */
+    private void writeDeferredBufs() throws IOException {
+      for (ByteBuffer deferredBuf : deferredBufs) {
+        writeBuf(deferredBuf);
+      }
+      deferredBufs = null;
+    }
+
+    /**
+     * This throws RuntimeException if the number of IOExceptions have exceeded threshold.
+     */
+    private void abortIfNecessary() {
+      if (partitionInfo.shouldAbort(mergeManager.ioExceptionsThresholdDuringMerge)) {
+        deferredBufs = null;
+        throw new RuntimeException(String.format("%s when merging %s",
+          ErrorHandler.BlockPushErrorHandler.IOEXCEPTIONS_EXCEEDED_THRESHOLD_PREFIX,
+          streamId));
+      }
+    }
+
+    /**
+     * This increments the number of IOExceptions and throws RuntimeException if it exceeds the
+     * threshold which will abort the merge of a particular shuffle partition.
+     */
+    private void incrementIOExceptionsAndAbortIfNecessary() {
+      // Update the count of IOExceptions
+      partitionInfo.incrementIOExceptions();
+      abortIfNecessary();
+    }
+
+    @Override
+    public void onData(String streamId, ByteBuffer buf) throws IOException {
+      // When handling the block data using StreamInterceptor, it can help to reduce the amount
+      // of data that needs to be buffered in memory since it does not wait till the completion
+      // of the frame before handling the message, thus releasing the ByteBuf earlier. However,
+      // this also means it would chunk a block into multiple buffers. Here, we want to preserve
+      // the benefit of handling the block data using StreamInterceptor as much as possible while
+      // providing the guarantee that one block would be continuously written to the merged
+      // shuffle file before the next block starts. For each shuffle partition, we would track
+      // the current map index to make sure only block matching the map index can be written to
+      // disk. If one server thread sees the block being handled is the current block, it would
+      // directly write the block to disk. Otherwise, it would buffer the block chunks in memory.
+      // If the block becomes the current block before we see the end of it, we would then dump
+      // all buffered block data to disk and write the remaining portions of the block directly
+      // to disk as well. This way, we avoid having to buffer the entirety of every blocks in
+      // memory, while still providing the necessary guarantee.
+      synchronized (partitionInfo) {
+        Map<Integer, AppShufflePartitionInfo> shufflePartitions =
+          mergeManager.partitions.get(partitionInfo.appShuffleId);
+        // If the partitionInfo corresponding to (appId, shuffleId, reduceId) is no longer present
+        // then it means that the shuffle merge has already been finalized. We should thus ignore
+        // the data and just drain the remaining bytes of this message. This check should be
+        // placed inside the synchronized block to make sure that checking the key is still
+        // present and processing the data is atomic.
+        if (shufflePartitions == null || !shufflePartitions.containsKey(partitionInfo.reduceId)) {
+          deferredBufs = null;
+          return;
+        }
+        // Check whether we can write to disk
+        if (allowedToWrite()) {
+          // Identify duplicate block generated by speculative tasks. We respond success to
+          // the client in cases of duplicate even though no data is written.
+          if (isDuplicateBlock()) {
+            deferredBufs = null;
+            return;
+          }
+          abortIfNecessary();
+          logger.trace("{} shuffleId {} reduceId {} onData writable",
+            partitionInfo.appShuffleId.appId, partitionInfo.appShuffleId.shuffleId,
+            partitionInfo.reduceId);
+          if (partitionInfo.getCurrentMapIndex() < 0) {
+            partitionInfo.setCurrentMapIndex(mapIndex);
+          }
+
+          // If we got here, it's safe to write the block data to the merged shuffle file. We
+          // first write any deferred block.
+          isWriting = true;
+          try {
+            if (deferredBufs != null && !deferredBufs.isEmpty()) {
+              writeDeferredBufs();
+            }
+            writeBuf(buf);
+          } catch (IOException ioe) {
+            incrementIOExceptionsAndAbortIfNecessary();
+            // If the above doesn't throw a RuntimeException, then we propagate the IOException
+            // back to the client so the block could be retried.
+            throw ioe;
+          }
+        } else {
+          logger.trace("{} shuffleId {} reduceId {} onData deferred",
+            partitionInfo.appShuffleId.appId, partitionInfo.appShuffleId.shuffleId,
+            partitionInfo.reduceId);
+          // If we cannot write to disk, we buffer the current block chunk in memory so it could
+          // potentially be written to disk later. We take our best effort without guarantee
+          // that the block will be written to disk. If the block data is divided into multiple
+          // chunks during TCP transportation, each #onData invocation is an attempt to write
+          // the block to disk. If the block is still not written to disk after all #onData
+          // invocations, the final #onComplete invocation is the last attempt to write the
+          // block to disk. If we still couldn't write this block to disk after this, we give up
+          // on this block push request and respond failure to client. We could potentially
+          // buffer the block longer or wait for a few iterations inside #onData or #onComplete
+          // to increase the chance of writing the block to disk, however this would incur more
+          // memory footprint or decrease the server processing throughput for the shuffle
+          // service. In addition, during test we observed that by randomizing the order in
+          // which clients sends block push requests batches, only ~0.5% blocks failed to be
+          // written to disk due to this reason. We thus decide to optimize for server
+          // throughput and memory usage.
+          if (deferredBufs == null) {
+            deferredBufs = new ArrayList<>();
+          }
+          // Write the buffer to the in-memory deferred cache. Since buf is a slice of a larger
+          // byte buffer, we cache only the relevant bytes not the entire large buffer to save
+          // memory.
+          ByteBuffer deferredBuf = ByteBuffer.allocate(buf.remaining());
+          deferredBuf.put(buf);
+          deferredBuf.flip();
+          deferredBufs.add(deferredBuf);
+        }
+      }
+    }
+
+    @Override
+    public void onComplete(String streamId) throws IOException {
+      synchronized (partitionInfo) {
+        logger.trace("{} shuffleId {} reduceId {} onComplete invoked",
+          partitionInfo.appShuffleId.appId, partitionInfo.appShuffleId.shuffleId,
+          partitionInfo.reduceId);
+        Map<Integer, AppShufflePartitionInfo> shufflePartitions =
+          mergeManager.partitions.get(partitionInfo.appShuffleId);
+        // When this request initially got to the server, the shuffle merge finalize request
+        // was not received yet. By the time we finish reading this message, the shuffle merge
+        // however is already finalized. We should thus respond RpcFailure to the client.
+        if (shufflePartitions == null || !shufflePartitions.containsKey(partitionInfo.reduceId)) {
+          deferredBufs = null;
+          throw new RuntimeException(String.format("Block %s %s", streamId,
+            ErrorHandler.BlockPushErrorHandler.TOO_LATE_MESSAGE_SUFFIX));
+        }
+        // Check if we can commit this block
+        if (allowedToWrite()) {
+          // Identify duplicate block generated by speculative tasks. We respond success to
+          // the client in cases of duplicate even though no data is written.
+          if (isDuplicateBlock()) {
+            deferredBufs = null;
+            return;
+          }
+          if (partitionInfo.getCurrentMapIndex() < 0) {
+            try {
+              if (deferredBufs != null && !deferredBufs.isEmpty()) {
+                abortIfNecessary();
+                isWriting = true;
+                writeDeferredBufs();
+              }
+            } catch (IOException ioe) {
+              incrementIOExceptionsAndAbortIfNecessary();
+              // If the above doesn't throw a RuntimeException, then we propagate the IOException
+              // back to the client so the block could be retried.
+              throw ioe;
+            }
+          }
+          long updatedPos = partitionInfo.getDataFilePos() + length;
+          boolean indexUpdated = false;
+          if (updatedPos - partitionInfo.getLastChunkOffset() >= mergeManager.minChunkSize) {
+            try {
+              partitionInfo.updateChunkInfo(updatedPos, mapIndex);
+              indexUpdated = true;
+            } catch (IOException ioe) {
+              incrementIOExceptionsAndAbortIfNecessary();
+              // If the above doesn't throw a RuntimeException, then we do not propagate the
+              // IOException to the client. This may increase the chunk size however the increase is
+              // still limited because of the limit on the number of IOExceptions for a
+              // particular shuffle partition.
+            }
+          }
+          partitionInfo.setDataFilePos(updatedPos);
+          partitionInfo.setCurrentMapIndex(-1);
+
+          // update merged results
+          partitionInfo.blockMerged(mapIndex);
+          if (indexUpdated) {
+            partitionInfo.resetChunkTracker();
+          }
+        } else {
+          deferredBufs = null;
+          throw new RuntimeException(String.format("%s %s to merged shuffle",
+            ErrorHandler.BlockPushErrorHandler.BLOCK_APPEND_COLLISION_DETECTED_MSG_PREFIX,
+            streamId));
+        }
+      }
+      isWriting = false;
+    }
+
+    @Override
+    public void onFailure(String streamId, Throwable throwable) throws IOException {
+      if (mergeManager.errorHandler.shouldLogError(throwable)) {
+        logger.error("Encountered issue when merging {}", streamId, throwable);
+      } else {
+        logger.debug("Encountered issue when merging {}", streamId, throwable);
+      }
+      // Only update partitionInfo if the failure corresponds to a valid request. If the
+      // request is too late, i.e. received after shuffle merge finalize, #onFailure will
+      // also be triggered, and we can just ignore. Also, if we couldn't find an opportunity
+      // to write the block data to disk, we should also ignore here.
+      if (isWriting) {
+        synchronized (partitionInfo) {
+          Map<Integer, AppShufflePartitionInfo> shufflePartitions =
+            mergeManager.partitions.get(partitionInfo.appShuffleId);
+          if (shufflePartitions != null && shufflePartitions.containsKey(partitionInfo.reduceId)) {
+            logger.debug("{} shuffleId {} reduceId {} encountered failure",
+              partitionInfo.appShuffleId.appId, partitionInfo.appShuffleId.shuffleId,
+              partitionInfo.reduceId);
+            partitionInfo.setCurrentMapIndex(-1);
+          }
+        }
+      }
+      isWriting = false;
+    }
+
+    @VisibleForTesting
+    AppShufflePartitionInfo getPartitionInfo() {
+      return partitionInfo;
+    }
+  }
+
+  /**
+   * ID that uniquely identifies a shuffle for an application. This is used as a key in
+   * {@link #partitions}.
+   */
+  public static class AppShuffleId {
+    public final String appId;
+    public final int shuffleId;
+
+    AppShuffleId(String appId, int shuffleId) {
+      this.appId = appId;
+      this.shuffleId = shuffleId;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) {
+        return true;
+      }
+      if (o == null || getClass() != o.getClass()) {
+        return false;
+      }
+      AppShuffleId that = (AppShuffleId) o;
+      return shuffleId == that.shuffleId && Objects.equal(appId, that.appId);
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hashCode(appId, shuffleId);
+    }
+
+    @Override
+    public String toString() {
+      return Objects.toStringHelper(this)
+        .add("appId", appId)
+        .add("shuffleId", shuffleId)
+        .toString();
+    }
+  }
+
+  /** Metadata tracked for an actively merged shuffle partition */
+  public static class AppShufflePartitionInfo {
+
+    private final AppShuffleId appShuffleId;
+    private final int reduceId;
+    // The merged shuffle data file channel
+    public FileChannel dataChannel;
+    // Location offset of the last successfully merged block for this shuffle partition
+    private long dataFilePos;
+    // Track the map index whose block is being merged for this shuffle partition
+    private int currentMapIndex;
+    // Bitmap tracking which mapper's blocks have been merged for this shuffle partition
+    private RoaringBitmap mapTracker;
+    // The index file for a particular merged shuffle contains the chunk offsets.
+    private MergeShuffleFile indexFile;
+    // The meta file for a particular merged shuffle contains all the map indices that belong to
+    // every chunk. The entry per chunk is a serialized bitmap.
+    private MergeShuffleFile metaFile;
+    // The offset for the last chunk tracked in the index file for this shuffle partition
+    private long lastChunkOffset;
+    private int lastMergedMapIndex = -1;
+    // Bitmap tracking which mapper's blocks are in the current shuffle chunk
+    private RoaringBitmap chunkTracker;
+    private int numIOExceptions = 0;
+    private boolean indexMetaUpdateFailed;
+
+    AppShufflePartitionInfo(
+        AppShuffleId appShuffleId,
+        int reduceId,
+        File dataFile,
+        MergeShuffleFile indexFile,
+        MergeShuffleFile metaFile) throws IOException {
+      this.appShuffleId = Preconditions.checkNotNull(appShuffleId, "app shuffle id");
+      this.reduceId = reduceId;
+      this.dataChannel = new FileOutputStream(dataFile).getChannel();
+      this.indexFile = indexFile;
+      this.metaFile = metaFile;
+      this.currentMapIndex = -1;
+      // Writing 0 offset so that we can reuse ShuffleIndexInformation.getIndex()
+      updateChunkInfo(0L, -1);
+      this.dataFilePos = 0;
+      this.mapTracker = new RoaringBitmap();
+      this.chunkTracker = new RoaringBitmap();
+    }
+
+    public long getDataFilePos() {
+      return dataFilePos;
+    }
+
+    public void setDataFilePos(long dataFilePos) {
+      logger.trace("{} shuffleId {} reduceId {} current pos {} update pos {}", appShuffleId.appId,
+        appShuffleId.shuffleId, reduceId, this.dataFilePos, dataFilePos);
+      this.dataFilePos = dataFilePos;
+    }
+
+    int getCurrentMapIndex() {
+      return currentMapIndex;
+    }
+
+    void setCurrentMapIndex(int mapIndex) {
+      logger.trace("{} shuffleId {} reduceId {} updated mapIndex {} current mapIndex {}",
+        appShuffleId.appId, appShuffleId.shuffleId, reduceId, currentMapIndex, mapIndex);
+      this.currentMapIndex = mapIndex;
+    }
+
+    long getLastChunkOffset() {
+      return lastChunkOffset;
+    }
+
+    void blockMerged(int mapIndex) {
+      logger.debug("{} shuffleId {} reduceId {} updated merging mapIndex {}", appShuffleId.appId,
+        appShuffleId.shuffleId, reduceId, mapIndex);
+      mapTracker.add(mapIndex);
+      chunkTracker.add(mapIndex);
+      lastMergedMapIndex = mapIndex;
+    }
+
+    void resetChunkTracker() {
+      chunkTracker.clear();
+    }
+
+    /**
+     * Appends the chunk offset to the index file and adds the map index to the chunk tracker.
+     *
+     * @param chunkOffset the offset of the chunk in the data file.
+     * @param mapIndex the map index to be added to chunk tracker.
+     */
+    void updateChunkInfo(long chunkOffset, int mapIndex) throws IOException {
+      try {
+        logger.trace("{} shuffleId {} reduceId {} index current {} updated {}",
+          appShuffleId.appId, appShuffleId.shuffleId, reduceId, this.lastChunkOffset, chunkOffset);
+        if (indexMetaUpdateFailed) {
+          indexFile.getChannel().position(indexFile.getPos());
+        }
+        indexFile.getDos().writeLong(chunkOffset);
+        // Chunk bitmap should be written to the meta file after the index file because if there are
+        // any exceptions during writing the offset to the index file, meta file should not be
+        // updated. If the update to the index file is successful but the update to meta file isn't
+        // then the index file position is not updated.
+        writeChunkTracker(mapIndex);
+        indexFile.updatePos(8);
+        this.lastChunkOffset = chunkOffset;
+        indexMetaUpdateFailed = false;
+      } catch (IOException ioe) {
+        logger.warn("{} shuffleId {} reduceId {} update to index/meta failed", appShuffleId.appId,
+          appShuffleId.shuffleId, reduceId);
+        indexMetaUpdateFailed = true;
+        // Any exception here is propagated to the caller and the caller can decide whether to
+        // abort or not.
+        throw ioe;
+      }
+    }
+
+    private void writeChunkTracker(int mapIndex) throws IOException {
+      if (mapIndex == -1) {
+        return;
+      }
+      chunkTracker.add(mapIndex);
+      logger.trace("{} shuffleId {} reduceId {} mapIndex {} write chunk to meta file",
+        appShuffleId.appId, appShuffleId.shuffleId, reduceId, mapIndex);
+      if (indexMetaUpdateFailed) {
+        metaFile.getChannel().position(metaFile.getPos());
+      }
+      chunkTracker.serialize(metaFile.getDos());
+      metaFile.updatePos(metaFile.getChannel().position() - metaFile.getPos());
+    }
+
+    private void incrementIOExceptions() {
+      numIOExceptions++;
+    }
+
+    private boolean shouldAbort(int ioExceptionsThresholdDuringMerge) {
+      return numIOExceptions > ioExceptionsThresholdDuringMerge;
+    }
+
+    private void finalizePartition() throws IOException {
+      if (dataFilePos != lastChunkOffset) {
+        try {
+          updateChunkInfo(dataFilePos, lastMergedMapIndex);
+        } catch (IOException ioe) {
+          // Any exceptions here while updating the meta files can be ignored. If the files
+          // aren't successfully updated they will be truncated.
+        }
+      }
+      // Get rid of any partial block data at the end of the file. This could either
+      // be due to failure, or a request still being processed when the shuffle
+      // merge gets finalized, or any exceptions while updating index/meta files.
+      dataChannel.truncate(lastChunkOffset);
+      indexFile.getChannel().truncate(indexFile.getPos());
+      metaFile.getChannel().truncate(metaFile.getPos());
+    }
+
+    void closeAllFiles() {
+      if (dataChannel != null) {
+        try {
+          dataChannel.close();
+        } catch (IOException ioe) {
+          logger.warn("Error closing data channel for {} shuffleId {} reduceId {}",
+            appShuffleId.appId, appShuffleId.shuffleId, reduceId);
+        } finally {
+          dataChannel = null;
+        }
+      }
+      if (metaFile != null) {
+        try {
+          metaFile.close();
+        } catch (IOException ioe) {
+          logger.warn("Error closing meta file for {} shuffleId {} reduceId {}",
+            appShuffleId.appId, appShuffleId.shuffleId, reduceId);
+        } finally {
+          metaFile = null;
+        }
+      }
+      if (indexFile != null) {
+        try {
+          indexFile.close();
+        } catch (IOException ioe) {
+          logger.warn("Error closing index file for {} shuffleId {} reduceId {}",
+            appShuffleId.appId, appShuffleId.shuffleId, reduceId);
+        } finally {
+          indexFile = null;
+        }
+      }
+    }
+
+    @Override
+    protected void finalize() throws Throwable {
+      closeAllFiles();
+    }
+
+    @VisibleForTesting
+    MergeShuffleFile getIndexFile() {
+      return indexFile;
+    }
+
+    @VisibleForTesting
+    MergeShuffleFile getMetaFile() {
+      return metaFile;
+    }
+
+    @VisibleForTesting
+    FileChannel getDataChannel() {
+      return dataChannel;
+    }
+
+    @VisibleForTesting
+    int getNumIOExceptions() {
+      return numIOExceptions;
+    }
+  }
+
+  /**
+   * Wraps all the information related to the merge directory of an application.
+   */
+  private static class AppPathsInfo {
+
+    private final String[] activeLocalDirs;
+    private final int subDirsPerLocalDir;
+
+    private AppPathsInfo(
+        String appId,
+        String[] localDirs,
+        int subDirsPerLocalDir) {
+      activeLocalDirs = Arrays.stream(localDirs)
+        .map(localDir ->
+          // Merge directory is created at the same level as block-manager directory. The list of
+          // local directories that we get from executorShuffleInfo are paths of each
+          // block-manager directory. To find out the merge directory location, we first find the
+          // parent dir and then append the "merger_manager" directory to it.
+          Paths.get(localDir).getParent().resolve(MERGE_MANAGER_DIR).toFile().getPath())
+        .toArray(String[]::new);
+      this.subDirsPerLocalDir = subDirsPerLocalDir;
+      if (logger.isInfoEnabled()) {
+        logger.info("Updated active local dirs {} and sub dirs {} for application {}",
+          Arrays.toString(activeLocalDirs),subDirsPerLocalDir, appId);
+      }
+    }
+  }
+
+  @VisibleForTesting
+  static class MergeShuffleFile {
+    private FileChannel channel;
+    private DataOutputStream dos;
+    private long pos;
+
+    @VisibleForTesting
+    MergeShuffleFile(File file) throws IOException {
+      FileOutputStream fos = new FileOutputStream(file);
+      channel = fos.getChannel();
+      dos = new DataOutputStream(fos);
+    }
+
+    @VisibleForTesting
+    MergeShuffleFile(FileChannel channel, DataOutputStream dos) {
+      this.channel = channel;
+      this.dos = dos;
+    }
+
+    private void updatePos(long numBytes) {
+      pos += numBytes;
+    }
+
+    void close() throws IOException {
+      try {
+        dos.close();
+      } finally {
+        dos = null;
+        channel = null;
+      }
+    }
+
+    @VisibleForTesting
+    DataOutputStream getDos() {
+      return dos;
+    }
+
+    @VisibleForTesting
+    FileChannel getChannel() {
+      return channel;
+    }
+
+    @VisibleForTesting
+    long getPos() {
+      return pos;
+    }
+  }
+}
diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/RetryingBlockFetcher.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/RetryingBlockFetcher.java
index 6bf3da94030d4..43bde1610e41e 100644
--- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/RetryingBlockFetcher.java
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/RetryingBlockFetcher.java
@@ -99,11 +99,14 @@ void createAndStart(String[] blockIds, BlockFetchingListener listener)
    */
   private RetryingBlockFetchListener currentListener;
 
+  private final ErrorHandler errorHandler;
+
   public RetryingBlockFetcher(
       TransportConf conf,
       RetryingBlockFetcher.BlockFetchStarter fetchStarter,
       String[] blockIds,
-      BlockFetchingListener listener) {
+      BlockFetchingListener listener,
+      ErrorHandler errorHandler) {
     this.fetchStarter = fetchStarter;
     this.listener = listener;
     this.maxRetries = conf.maxIORetries();
@@ -111,6 +114,15 @@ public RetryingBlockFetcher(
     this.outstandingBlocksIds = Sets.newLinkedHashSet();
     Collections.addAll(outstandingBlocksIds, blockIds);
     this.currentListener = new RetryingBlockFetchListener();
+    this.errorHandler = errorHandler;
+  }
+
+  public RetryingBlockFetcher(
+      TransportConf conf,
+      BlockFetchStarter fetchStarter,
+      String[] blockIds,
+      BlockFetchingListener listener) {
+    this(conf, fetchStarter, blockIds, listener, ErrorHandler.NOOP_ERROR_HANDLER);
   }
 
   /**
@@ -178,7 +190,7 @@ private synchronized boolean shouldRetry(Throwable e) {
     boolean isIOException = e instanceof IOException
       || (e.getCause() != null && e.getCause() instanceof IOException);
     boolean hasRemainingRetries = retryCount < maxRetries;
-    return isIOException && hasRemainingRetries;
+    return isIOException && hasRemainingRetries && errorHandler.shouldRetryError(e);
   }
 
   /**
@@ -215,8 +227,15 @@ public void onBlockFetchFailure(String blockId, Throwable exception) {
           if (shouldRetry(exception)) {
             initiateRetry();
           } else {
-            logger.error(String.format("Failed to fetch block %s, and will not retry (%s retries)",
-              blockId, retryCount), exception);
+            if (errorHandler.shouldLogError(exception)) {
+              logger.error(
+                String.format("Failed to fetch block %s, and will not retry (%s retries)",
+                  blockId, retryCount), exception);
+            } else {
+              logger.debug(
+                String.format("Failed to fetch block %s, and will not retry (%s retries)",
+                  blockId, retryCount), exception);
+            }
             outstandingBlocksIds.remove(blockId);
             shouldForwardFailure = true;
           }
diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/SimpleDownloadFile.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/SimpleDownloadFile.java
index 670612fd6f66a..97ecaa627b66c 100644
--- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/SimpleDownloadFile.java
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/SimpleDownloadFile.java
@@ -32,7 +32,7 @@
  * A DownloadFile that does not take any encryption settings into account for reading and
  * writing data.
  *
- * This does *not* mean the data in the file is un-encrypted -- it could be that the data is
+ * This does *not* mean the data in the file is unencrypted -- it could be that the data is
  * already encrypted when its written, and subsequent layer is responsible for decrypting.
  */
 public class SimpleDownloadFile implements DownloadFile {
diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/BlockTransferMessage.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/BlockTransferMessage.java
index 89d8dfe8716b8..7f5058124988f 100644
--- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/BlockTransferMessage.java
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/BlockTransferMessage.java
@@ -47,7 +47,8 @@ public abstract class BlockTransferMessage implements Encodable {
   public enum Type {
     OPEN_BLOCKS(0), UPLOAD_BLOCK(1), REGISTER_EXECUTOR(2), STREAM_HANDLE(3), REGISTER_DRIVER(4),
     HEARTBEAT(5), UPLOAD_BLOCK_STREAM(6), REMOVE_BLOCKS(7), BLOCKS_REMOVED(8),
-    FETCH_SHUFFLE_BLOCKS(9), GET_LOCAL_DIRS_FOR_EXECUTORS(10), LOCAL_DIRS_FOR_EXECUTORS(11);
+    FETCH_SHUFFLE_BLOCKS(9), GET_LOCAL_DIRS_FOR_EXECUTORS(10), LOCAL_DIRS_FOR_EXECUTORS(11),
+    PUSH_BLOCK_STREAM(12), FINALIZE_SHUFFLE_MERGE(13), MERGE_STATUSES(14);
 
     private final byte id;
 
@@ -78,6 +79,9 @@ public static BlockTransferMessage fromByteBuffer(ByteBuffer msg) {
         case 9: return FetchShuffleBlocks.decode(buf);
         case 10: return GetLocalDirsForExecutors.decode(buf);
         case 11: return LocalDirsForExecutors.decode(buf);
+        case 12: return PushBlockStream.decode(buf);
+        case 13: return FinalizeShuffleMerge.decode(buf);
+        case 14: return MergeStatuses.decode(buf);
         default: throw new IllegalArgumentException("Unknown message type: " + type);
       }
     }
diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/FinalizeShuffleMerge.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/FinalizeShuffleMerge.java
new file mode 100644
index 0000000000000..8427837d1525b
--- /dev/null
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/FinalizeShuffleMerge.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.network.shuffle.protocol;
+
+import com.google.common.base.Objects;
+import io.netty.buffer.ByteBuf;
+
+import org.apache.spark.network.protocol.Encoders;
+
+/**
+ * Request to finalize merge for a given shuffle.
+ * Returns {@link MergeStatuses}
+ *
+ * @since 3.1.0
+ */
+public class FinalizeShuffleMerge extends BlockTransferMessage {
+  public final String appId;
+  public final int shuffleId;
+
+  public FinalizeShuffleMerge(
+      String appId,
+      int shuffleId) {
+    this.appId = appId;
+    this.shuffleId = shuffleId;
+  }
+
+  @Override
+  protected BlockTransferMessage.Type type() {
+    return Type.FINALIZE_SHUFFLE_MERGE;
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hashCode(appId, shuffleId);
+  }
+
+  @Override
+  public String toString() {
+    return Objects.toStringHelper(this)
+      .add("appId", appId)
+      .add("shuffleId", shuffleId)
+      .toString();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other != null && other instanceof FinalizeShuffleMerge) {
+      FinalizeShuffleMerge o = (FinalizeShuffleMerge) other;
+      return Objects.equal(appId, o.appId)
+        && shuffleId == o.shuffleId;
+    }
+    return false;
+  }
+
+  @Override
+  public int encodedLength() {
+    return Encoders.Strings.encodedLength(appId) + 4;
+  }
+
+  @Override
+  public void encode(ByteBuf buf) {
+    Encoders.Strings.encode(buf, appId);
+    buf.writeInt(shuffleId);
+  }
+
+  public static FinalizeShuffleMerge decode(ByteBuf buf) {
+    String appId = Encoders.Strings.decode(buf);
+    int shuffleId = buf.readInt();
+    return new FinalizeShuffleMerge(appId, shuffleId);
+  }
+}
diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/MergeStatuses.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/MergeStatuses.java
new file mode 100644
index 0000000000000..d506d9eb2b784
--- /dev/null
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/MergeStatuses.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.network.shuffle.protocol;
+
+import java.util.Arrays;
+
+import com.google.common.base.Objects;
+import io.netty.buffer.ByteBuf;
+import org.roaringbitmap.RoaringBitmap;
+
+import org.apache.spark.network.protocol.Encoders;
+
+/**
+ * Result returned by an ExternalShuffleService to the DAGScheduler. This represents the result
+ * of all the remote shuffle block merge operations performed by an ExternalShuffleService
+ * for a given shuffle ID. It includes the shuffle ID, an array of bitmaps each representing
+ * the set of mapper partition blocks that are merged for a given reducer partition, an array
+ * of reducer IDs, and an array of merged shuffle partition sizes. The 3 arrays list information
+ * about all the reducer partitions merged by the ExternalShuffleService in the same order.
+ *
+ * @since 3.1.0
+ */
+public class MergeStatuses extends BlockTransferMessage {
+  /** Shuffle ID **/
+  public final int shuffleId;
+  /**
+   * Array of bitmaps tracking the set of mapper partition blocks merged for each
+   * reducer partition
+   */
+  public final RoaringBitmap[] bitmaps;
+  /** Array of reducer IDs **/
+  public final int[] reduceIds;
+  /**
+   * Array of merged shuffle partition block size. Each represents the total size of all
+   * merged shuffle partition blocks for one reducer partition.
+   * **/
+  public final long[] sizes;
+
+  public MergeStatuses(
+      int shuffleId,
+      RoaringBitmap[] bitmaps,
+      int[] reduceIds,
+      long[] sizes) {
+    this.shuffleId = shuffleId;
+    this.bitmaps = bitmaps;
+    this.reduceIds = reduceIds;
+    this.sizes = sizes;
+  }
+
+  @Override
+  protected Type type() {
+    return Type.MERGE_STATUSES;
+  }
+
+  @Override
+  public int hashCode() {
+    int objectHashCode = Objects.hashCode(shuffleId);
+    return (objectHashCode * 41 + Arrays.hashCode(reduceIds) * 41
+      + Arrays.hashCode(bitmaps) * 41 + Arrays.hashCode(sizes));
+  }
+
+  @Override
+  public String toString() {
+    return Objects.toStringHelper(this)
+      .add("shuffleId", shuffleId)
+      .add("reduceId size", reduceIds.length)
+      .toString();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other != null && other instanceof MergeStatuses) {
+      MergeStatuses o = (MergeStatuses) other;
+      return Objects.equal(shuffleId, o.shuffleId)
+        && Arrays.equals(bitmaps, o.bitmaps)
+        && Arrays.equals(reduceIds, o.reduceIds)
+        && Arrays.equals(sizes, o.sizes);
+    }
+    return false;
+  }
+
+  @Override
+  public int encodedLength() {
+    return 4 // int
+      + Encoders.BitmapArrays.encodedLength(bitmaps)
+      + Encoders.IntArrays.encodedLength(reduceIds)
+      + Encoders.LongArrays.encodedLength(sizes);
+  }
+
+  @Override
+  public void encode(ByteBuf buf) {
+    buf.writeInt(shuffleId);
+    Encoders.BitmapArrays.encode(buf, bitmaps);
+    Encoders.IntArrays.encode(buf, reduceIds);
+    Encoders.LongArrays.encode(buf, sizes);
+  }
+
+  public static MergeStatuses decode(ByteBuf buf) {
+    int shuffleId = buf.readInt();
+    RoaringBitmap[] bitmaps = Encoders.BitmapArrays.decode(buf);
+    int[] reduceIds = Encoders.IntArrays.decode(buf);
+    long[] sizes = Encoders.LongArrays.decode(buf);
+    return new MergeStatuses(shuffleId, bitmaps, reduceIds, sizes);
+  }
+}
diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/PushBlockStream.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/PushBlockStream.java
new file mode 100644
index 0000000000000..83fc7b23ac444
--- /dev/null
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/PushBlockStream.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.network.shuffle.protocol;
+
+import com.google.common.base.Objects;
+import io.netty.buffer.ByteBuf;
+
+import org.apache.spark.network.protocol.Encoders;
+
+// Needed by ScalaDoc. See SPARK-7726
+
+/**
+ * Request to push a block to a remote shuffle service to be merged in push based shuffle.
+ * The remote shuffle service will also include this message when responding the push requests.
+ *
+ * @since 3.1.0
+ */
+public class PushBlockStream extends BlockTransferMessage {
+  public final String appId;
+  public final int shuffleId;
+  public final int mapIndex;
+  public final int reduceId;
+  // Similar to the chunkIndex in StreamChunkId, indicating the index of a block in a batch of
+  // blocks to be pushed.
+  public final int index;
+
+  public PushBlockStream(String appId, int shuffleId, int mapIndex, int reduceId, int index) {
+    this.appId = appId;
+    this.shuffleId = shuffleId;
+    this.mapIndex = mapIndex;
+    this.reduceId = reduceId;
+    this.index = index;
+  }
+
+  @Override
+  protected Type type() {
+    return Type.PUSH_BLOCK_STREAM;
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hashCode(appId, shuffleId, mapIndex , reduceId, index);
+  }
+
+  @Override
+  public String toString() {
+    return Objects.toStringHelper(this)
+      .add("appId", appId)
+      .add("shuffleId", shuffleId)
+      .add("mapIndex", mapIndex)
+      .add("reduceId", reduceId)
+      .add("index", index)
+      .toString();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other != null && other instanceof PushBlockStream) {
+      PushBlockStream o = (PushBlockStream) other;
+      return Objects.equal(appId, o.appId)
+        && shuffleId == o.shuffleId
+        && mapIndex == o.mapIndex
+        && reduceId == o.reduceId
+        && index == o.index;
+    }
+    return false;
+  }
+
+  @Override
+  public int encodedLength() {
+    return Encoders.Strings.encodedLength(appId) + 16;
+  }
+
+  @Override
+  public void encode(ByteBuf buf) {
+    Encoders.Strings.encode(buf, appId);
+    buf.writeInt(shuffleId);
+    buf.writeInt(mapIndex);
+    buf.writeInt(reduceId);
+    buf.writeInt(index);
+  }
+
+  public static PushBlockStream decode(ByteBuf buf) {
+    String appId = Encoders.Strings.decode(buf);
+    int shuffleId = buf.readInt();
+    int mapIdx = buf.readInt();
+    int reduceId = buf.readInt();
+    int index = buf.readInt();
+    return new PushBlockStream(appId, shuffleId, mapIdx, reduceId, index);
+  }
+}
diff --git a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ErrorHandlerSuite.java b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ErrorHandlerSuite.java
new file mode 100644
index 0000000000000..992e7762c5a54
--- /dev/null
+++ b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ErrorHandlerSuite.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.network.shuffle;
+
+import java.net.ConnectException;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * Test suite for {@link ErrorHandler}
+ */
+public class ErrorHandlerSuite {
+
+  @Test
+  public void testPushErrorRetry() {
+    ErrorHandler.BlockPushErrorHandler handler = new ErrorHandler.BlockPushErrorHandler();
+    assertFalse(handler.shouldRetryError(new RuntimeException(new IllegalArgumentException(
+      ErrorHandler.BlockPushErrorHandler.TOO_LATE_MESSAGE_SUFFIX))));
+    assertFalse(handler.shouldRetryError(new RuntimeException(new ConnectException())));
+    assertTrue(handler.shouldRetryError(new RuntimeException(new IllegalArgumentException(
+      ErrorHandler.BlockPushErrorHandler.BLOCK_APPEND_COLLISION_DETECTED_MSG_PREFIX))));
+    assertTrue(handler.shouldRetryError(new Throwable()));
+  }
+
+  @Test
+  public void testPushErrorLogging() {
+    ErrorHandler.BlockPushErrorHandler handler = new ErrorHandler.BlockPushErrorHandler();
+    assertFalse(handler.shouldLogError(new RuntimeException(new IllegalArgumentException(
+      ErrorHandler.BlockPushErrorHandler.TOO_LATE_MESSAGE_SUFFIX))));
+    assertFalse(handler.shouldLogError(new RuntimeException(new IllegalArgumentException(
+      ErrorHandler.BlockPushErrorHandler.BLOCK_APPEND_COLLISION_DETECTED_MSG_PREFIX))));
+    assertTrue(handler.shouldLogError(new Throwable()));
+  }
+}
diff --git a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalBlockHandlerSuite.java b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalBlockHandlerSuite.java
index 455351fcf767c..f06e7cb047f1a 100644
--- a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalBlockHandlerSuite.java
+++ b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalBlockHandlerSuite.java
@@ -17,6 +17,7 @@
 
 package org.apache.spark.network.shuffle;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Iterator;
 
@@ -25,6 +26,7 @@
 import org.junit.Before;
 import org.junit.Test;
 import org.mockito.ArgumentCaptor;
+import org.roaringbitmap.RoaringBitmap;
 
 import static org.junit.Assert.*;
 import static org.mockito.ArgumentMatchers.any;
@@ -39,6 +41,8 @@
 import org.apache.spark.network.shuffle.protocol.BlockTransferMessage;
 import org.apache.spark.network.shuffle.protocol.ExecutorShuffleInfo;
 import org.apache.spark.network.shuffle.protocol.FetchShuffleBlocks;
+import org.apache.spark.network.shuffle.protocol.FinalizeShuffleMerge;
+import org.apache.spark.network.shuffle.protocol.MergeStatuses;
 import org.apache.spark.network.shuffle.protocol.OpenBlocks;
 import org.apache.spark.network.shuffle.protocol.RegisterExecutor;
 import org.apache.spark.network.shuffle.protocol.StreamHandle;
@@ -50,6 +54,7 @@ public class ExternalBlockHandlerSuite {
   OneForOneStreamManager streamManager;
   ExternalShuffleBlockResolver blockResolver;
   RpcHandler handler;
+  MergedShuffleFileManager mergedShuffleManager;
   ManagedBuffer[] blockMarkers = {
     new NioManagedBuffer(ByteBuffer.wrap(new byte[3])),
     new NioManagedBuffer(ByteBuffer.wrap(new byte[7]))
@@ -59,17 +64,20 @@ public class ExternalBlockHandlerSuite {
   public void beforeEach() {
     streamManager = mock(OneForOneStreamManager.class);
     blockResolver = mock(ExternalShuffleBlockResolver.class);
-    handler = new ExternalBlockHandler(streamManager, blockResolver);
+    mergedShuffleManager = mock(MergedShuffleFileManager.class);
+    handler = new ExternalBlockHandler(streamManager, blockResolver, mergedShuffleManager);
   }
 
   @Test
   public void testRegisterExecutor() {
     RpcResponseCallback callback = mock(RpcResponseCallback.class);
 
-    ExecutorShuffleInfo config = new ExecutorShuffleInfo(new String[] {"/a", "/b"}, 16, "sort");
+    String[] localDirs = new String[] {"/a", "/b"};
+    ExecutorShuffleInfo config = new ExecutorShuffleInfo(localDirs, 16, "sort");
     ByteBuffer registerMessage = new RegisterExecutor("app0", "exec1", config).toByteBuffer();
     handler.receive(client, registerMessage, callback);
     verify(blockResolver, times(1)).registerExecutor("app0", "exec1", config);
+    verify(mergedShuffleManager, times(1)).registerExecutor("app0", config);
 
     verify(callback, times(1)).onSuccess(any(ByteBuffer.class));
     verify(callback, never()).onFailure(any(Throwable.class));
@@ -222,4 +230,32 @@ public void testBadMessages() {
     verify(callback, never()).onSuccess(any(ByteBuffer.class));
     verify(callback, never()).onFailure(any(Throwable.class));
   }
+
+  @Test
+  public void testFinalizeShuffleMerge() throws IOException {
+    RpcResponseCallback callback = mock(RpcResponseCallback.class);
+
+    FinalizeShuffleMerge req = new FinalizeShuffleMerge("app0", 0);
+    RoaringBitmap bitmap = RoaringBitmap.bitmapOf(0, 1, 2);
+    MergeStatuses statuses = new MergeStatuses(0, new RoaringBitmap[]{bitmap},
+      new int[]{3}, new long[]{30});
+    when(mergedShuffleManager.finalizeShuffleMerge(req)).thenReturn(statuses);
+
+    ByteBuffer reqBuf = req.toByteBuffer();
+    handler.receive(client, reqBuf, callback);
+    verify(mergedShuffleManager, times(1)).finalizeShuffleMerge(req);
+    ArgumentCaptor<ByteBuffer> response = ArgumentCaptor.forClass(ByteBuffer.class);
+    verify(callback, times(1)).onSuccess(response.capture());
+    verify(callback, never()).onFailure(any());
+
+    MergeStatuses mergeStatuses =
+      (MergeStatuses) BlockTransferMessage.Decoder.fromByteBuffer(response.getValue());
+    assertEquals(mergeStatuses, statuses);
+
+    Timer finalizeShuffleMergeLatencyMillis = (Timer) ((ExternalBlockHandler) handler)
+        .getAllMetrics()
+        .getMetrics()
+        .get("finalizeShuffleMergeLatencyMillis");
+    assertEquals(1, finalizeShuffleMergeLatencyMillis.getCount());
+  }
 }
diff --git a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolverSuite.java b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolverSuite.java
index 09b31430b1eb9..04d4bdf92bae7 100644
--- a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolverSuite.java
+++ b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolverSuite.java
@@ -17,7 +17,6 @@
 
 package org.apache.spark.network.shuffle;
 
-import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
@@ -72,15 +71,6 @@ public void testBadRequests() throws IOException {
       assertTrue("Bad error message: " + e, e.getMessage().contains("not registered"));
     }
 
-    // Invalid shuffle manager
-    try {
-      resolver.registerExecutor("app0", "exec2", dataContext.createExecutorInfo("foobar"));
-      resolver.getBlockData("app0", "exec2", 1, 1, 0);
-      fail("Should have failed");
-    } catch (UnsupportedOperationException e) {
-      // pass
-    }
-
     // Nonexistent shuffle block
     resolver.registerExecutor("app0", "exec3",
       dataContext.createExecutorInfo(SORT_MANAGER));
@@ -144,22 +134,4 @@ public void jsonSerializationOfExecutorRegistration() throws IOException {
     assertEquals(shuffleInfo, mapper.readValue(legacyShuffleJson, ExecutorShuffleInfo.class));
   }
 
-  @Test
-  public void testNormalizeAndInternPathname() {
-    assertPathsMatch("/foo", "bar", "baz", "/foo/bar/baz");
-    assertPathsMatch("//foo/", "bar/", "//baz", "/foo/bar/baz");
-    assertPathsMatch("foo", "bar", "baz///", "foo/bar/baz");
-    assertPathsMatch("/foo/", "/bar//", "/baz", "/foo/bar/baz");
-    assertPathsMatch("/", "", "", "/");
-    assertPathsMatch("/", "/", "/", "/");
-  }
-
-  private void assertPathsMatch(String p1, String p2, String p3, String expectedPathname) {
-    String normPathname =
-      ExecutorDiskUtils.createNormalizedInternedPathname(p1, p2, p3);
-    assertEquals(expectedPathname, normPathname);
-    File file = new File(normPathname);
-    String returnedPath = file.getPath();
-    assertTrue(normPathname == returnedPath);
-  }
 }
diff --git a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleIntegrationSuite.java b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleIntegrationSuite.java
index 9d398e372056b..49d02e5dc6fb4 100644
--- a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleIntegrationSuite.java
+++ b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleIntegrationSuite.java
@@ -233,9 +233,9 @@ public void testFetchThreeSort() throws Exception {
     exec0Fetch.releaseBuffers();
   }
 
-  @Test (expected = RuntimeException.class)
-  public void testRegisterInvalidExecutor() throws Exception {
-    registerExecutor("exec-1", dataContext0.createExecutorInfo("unknown sort manager"));
+  @Test
+  public void testRegisterWithCustomShuffleManager() throws Exception {
+    registerExecutor("exec-1", dataContext0.createExecutorInfo("custom shuffle manager"));
   }
 
   @Test
diff --git a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/OneForOneBlockFetcherSuite.java b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/OneForOneBlockFetcherSuite.java
index 285eedb39c65c..a7eb59d366966 100644
--- a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/OneForOneBlockFetcherSuite.java
+++ b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/OneForOneBlockFetcherSuite.java
@@ -201,6 +201,48 @@ public void testEmptyBlockFetch() {
     }
   }
 
+  @Test
+  public void testFetchShuffleBlocksOrder() {
+    LinkedHashMap<String, ManagedBuffer> blocks = Maps.newLinkedHashMap();
+    blocks.put("shuffle_0_0_0", new NioManagedBuffer(ByteBuffer.wrap(new byte[1])));
+    blocks.put("shuffle_0_2_1", new NioManagedBuffer(ByteBuffer.wrap(new byte[2])));
+    blocks.put("shuffle_0_10_2", new NettyManagedBuffer(Unpooled.wrappedBuffer(new byte[3])));
+    String[] blockIds = blocks.keySet().toArray(new String[blocks.size()]);
+
+    BlockFetchingListener listener = fetchBlocks(
+      blocks,
+      blockIds,
+      new FetchShuffleBlocks("app-id", "exec-id", 0,
+              new long[]{0, 2, 10}, new int[][]{{0}, {1}, {2}}, false),
+      conf);
+
+    for (int chunkIndex = 0; chunkIndex < blockIds.length; chunkIndex++) {
+      String blockId = blockIds[chunkIndex];
+      verify(listener).onBlockFetchSuccess(blockId, blocks.get(blockId));
+    }
+  }
+
+  @Test
+  public void testBatchFetchShuffleBlocksOrder() {
+    LinkedHashMap<String, ManagedBuffer> blocks = Maps.newLinkedHashMap();
+    blocks.put("shuffle_0_0_1_2", new NioManagedBuffer(ByteBuffer.wrap(new byte[1])));
+    blocks.put("shuffle_0_2_2_3", new NioManagedBuffer(ByteBuffer.wrap(new byte[2])));
+    blocks.put("shuffle_0_10_3_4", new NettyManagedBuffer(Unpooled.wrappedBuffer(new byte[3])));
+    String[] blockIds = blocks.keySet().toArray(new String[blocks.size()]);
+
+    BlockFetchingListener listener = fetchBlocks(
+      blocks,
+      blockIds,
+      new FetchShuffleBlocks("app-id", "exec-id", 0,
+              new long[]{0, 2, 10}, new int[][]{{1, 2}, {2, 3}, {3, 4}}, true),
+      conf);
+
+    for (int chunkIndex = 0; chunkIndex < blockIds.length; chunkIndex++) {
+      String blockId = blockIds[chunkIndex];
+      verify(listener).onBlockFetchSuccess(blockId, blocks.get(blockId));
+    }
+  }
+
   /**
    * Begins a fetch on the given set of blocks by mocking out the server side of the RPC which
    * simply returns the given (BlockId, Block) pairs.
diff --git a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/OneForOneBlockPusherSuite.java b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/OneForOneBlockPusherSuite.java
new file mode 100644
index 0000000000000..46a0f6cf420eb
--- /dev/null
+++ b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/OneForOneBlockPusherSuite.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.network.shuffle;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+import io.netty.buffer.Unpooled;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+import static org.mockito.AdditionalMatchers.*;
+import static org.mockito.Mockito.*;
+
+import org.apache.spark.network.buffer.ManagedBuffer;
+import org.apache.spark.network.buffer.NettyManagedBuffer;
+import org.apache.spark.network.buffer.NioManagedBuffer;
+import org.apache.spark.network.client.RpcResponseCallback;
+import org.apache.spark.network.client.TransportClient;
+import org.apache.spark.network.shuffle.protocol.BlockTransferMessage;
+import org.apache.spark.network.shuffle.protocol.PushBlockStream;
+
+
+public class OneForOneBlockPusherSuite {
+
+  @Test
+  public void testPushOne() {
+    LinkedHashMap<String, ManagedBuffer> blocks = Maps.newLinkedHashMap();
+    blocks.put("shufflePush_0_0_0", new NioManagedBuffer(ByteBuffer.wrap(new byte[1])));
+    String[] blockIds = blocks.keySet().toArray(new String[blocks.size()]);
+
+    BlockFetchingListener listener = pushBlocks(
+      blocks,
+      blockIds,
+      Arrays.asList(new PushBlockStream("app-id", 0, 0, 0, 0)));
+
+    verify(listener).onBlockFetchSuccess(eq("shufflePush_0_0_0"), any());
+  }
+
+  @Test
+  public void testPushThree() {
+    LinkedHashMap<String, ManagedBuffer> blocks = Maps.newLinkedHashMap();
+    blocks.put("shufflePush_0_0_0", new NioManagedBuffer(ByteBuffer.wrap(new byte[12])));
+    blocks.put("shufflePush_0_1_0", new NioManagedBuffer(ByteBuffer.wrap(new byte[23])));
+    blocks.put("shufflePush_0_2_0", new NettyManagedBuffer(Unpooled.wrappedBuffer(new byte[23])));
+    String[] blockIds = blocks.keySet().toArray(new String[blocks.size()]);
+
+    BlockFetchingListener listener = pushBlocks(
+      blocks,
+      blockIds,
+      Arrays.asList(new PushBlockStream("app-id", 0, 0, 0, 0),
+        new PushBlockStream("app-id", 0, 1, 0, 1),
+        new PushBlockStream("app-id", 0, 2, 0, 2)));
+
+    verify(listener, times(1)).onBlockFetchSuccess(eq("shufflePush_0_0_0"), any());
+    verify(listener, times(1)).onBlockFetchSuccess(eq("shufflePush_0_1_0"), any());
+    verify(listener, times(1)).onBlockFetchSuccess(eq("shufflePush_0_2_0"), any());
+  }
+
+  @Test
+  public void testServerFailures() {
+    LinkedHashMap<String, ManagedBuffer> blocks = Maps.newLinkedHashMap();
+    blocks.put("shufflePush_0_0_0", new NioManagedBuffer(ByteBuffer.wrap(new byte[12])));
+    blocks.put("shufflePush_0_1_0", new NioManagedBuffer(ByteBuffer.wrap(new byte[0])));
+    blocks.put("shufflePush_0_2_0", new NioManagedBuffer(ByteBuffer.wrap(new byte[0])));
+    String[] blockIds = blocks.keySet().toArray(new String[blocks.size()]);
+
+    BlockFetchingListener listener = pushBlocks(
+      blocks,
+      blockIds,
+      Arrays.asList(new PushBlockStream("app-id", 0, 0, 0, 0),
+        new PushBlockStream("app-id", 0, 1, 0, 1),
+        new PushBlockStream("app-id", 0, 2, 0, 2)));
+
+    verify(listener, times(1)).onBlockFetchSuccess(eq("shufflePush_0_0_0"), any());
+    verify(listener, times(1)).onBlockFetchFailure(eq("shufflePush_0_1_0"), any());
+    verify(listener, times(1)).onBlockFetchFailure(eq("shufflePush_0_2_0"), any());
+  }
+
+  @Test
+  public void testHandlingRetriableFailures() {
+    LinkedHashMap<String, ManagedBuffer> blocks = Maps.newLinkedHashMap();
+    blocks.put("shufflePush_0_0_0", new NioManagedBuffer(ByteBuffer.wrap(new byte[12])));
+    blocks.put("shufflePush_0_1_0", null);
+    blocks.put("shufflePush_0_2_0", new NioManagedBuffer(ByteBuffer.wrap(new byte[0])));
+    String[] blockIds = blocks.keySet().toArray(new String[blocks.size()]);
+
+    BlockFetchingListener listener = pushBlocks(
+      blocks,
+      blockIds,
+      Arrays.asList(new PushBlockStream("app-id", 0, 0, 0, 0),
+        new PushBlockStream("app-id", 0, 1, 0, 1),
+        new PushBlockStream("app-id", 0, 2, 0, 2)));
+
+    verify(listener, times(1)).onBlockFetchSuccess(eq("shufflePush_0_0_0"), any());
+    verify(listener, times(0)).onBlockFetchSuccess(not(eq("shufflePush_0_0_0")), any());
+    verify(listener, times(0)).onBlockFetchFailure(eq("shufflePush_0_0_0"), any());
+    verify(listener, times(1)).onBlockFetchFailure(eq("shufflePush_0_1_0"), any());
+    verify(listener, times(2)).onBlockFetchFailure(eq("shufflePush_0_2_0"), any());
+  }
+
+  /**
+   * Begins a push on the given set of blocks by mocking the response from server side.
+   * If a block is an empty byte, a server side retriable exception will be thrown.
+   * If a block is null, a non-retriable exception will be thrown.
+   */
+  private static BlockFetchingListener pushBlocks(
+      LinkedHashMap<String, ManagedBuffer> blocks,
+      String[] blockIds,
+      Iterable<BlockTransferMessage> expectMessages) {
+    TransportClient client = mock(TransportClient.class);
+    BlockFetchingListener listener = mock(BlockFetchingListener.class);
+    OneForOneBlockPusher pusher =
+      new OneForOneBlockPusher(client, "app-id", blockIds, listener, blocks);
+
+    Iterator<Map.Entry<String, ManagedBuffer>> blockIterator = blocks.entrySet().iterator();
+    Iterator<BlockTransferMessage> msgIterator = expectMessages.iterator();
+    doAnswer(invocation -> {
+      ByteBuffer header = ((ManagedBuffer) invocation.getArguments()[0]).nioByteBuffer();
+      BlockTransferMessage message = BlockTransferMessage.Decoder.fromByteBuffer(header);
+      RpcResponseCallback callback = (RpcResponseCallback) invocation.getArguments()[2];
+      Map.Entry<String, ManagedBuffer> entry = blockIterator.next();
+      ManagedBuffer block = entry.getValue();
+      if (block != null && block.nioByteBuffer().capacity() > 0) {
+        callback.onSuccess(header);
+      } else if (block != null) {
+        callback.onFailure(new RuntimeException("Failed " + entry.getKey()
+          + ErrorHandler.BlockPushErrorHandler.BLOCK_APPEND_COLLISION_DETECTED_MSG_PREFIX));
+      } else {
+        callback.onFailure(new RuntimeException("Quick fail " + entry.getKey()
+          + ErrorHandler.BlockPushErrorHandler.TOO_LATE_MESSAGE_SUFFIX));
+      }
+      assertEquals(msgIterator.next(), message);
+      return null;
+    }).when(client).uploadStream(any(ManagedBuffer.class), any(), any(RpcResponseCallback.class));
+
+    pusher.start();
+    return listener;
+  }
+}
diff --git a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/RemoteBlockPushResolverSuite.java b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/RemoteBlockPushResolverSuite.java
new file mode 100644
index 0000000000000..565d433ff3203
--- /dev/null
+++ b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/RemoteBlockPushResolverSuite.java
@@ -0,0 +1,957 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.network.shuffle;
+
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.ThreadLocalRandom;
+
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableMap;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.roaringbitmap.RoaringBitmap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.*;
+
+import org.apache.spark.network.buffer.FileSegmentManagedBuffer;
+import org.apache.spark.network.client.StreamCallbackWithID;
+import org.apache.spark.network.shuffle.RemoteBlockPushResolver.MergeShuffleFile;
+import org.apache.spark.network.shuffle.protocol.ExecutorShuffleInfo;
+import org.apache.spark.network.shuffle.protocol.FinalizeShuffleMerge;
+import org.apache.spark.network.shuffle.protocol.MergeStatuses;
+import org.apache.spark.network.shuffle.protocol.PushBlockStream;
+import org.apache.spark.network.util.MapConfigProvider;
+import org.apache.spark.network.util.TransportConf;
+
+/**
+ * Tests for {@link RemoteBlockPushResolver}.
+ */
+public class RemoteBlockPushResolverSuite {
+
+  private static final Logger log = LoggerFactory.getLogger(RemoteBlockPushResolverSuite.class);
+  private final String TEST_APP = "testApp";
+  private final String BLOCK_MANAGER_DIR = "blockmgr-193d8401";
+
+  private TransportConf conf;
+  private RemoteBlockPushResolver pushResolver;
+  private Path[] localDirs;
+
+  @Before
+  public void before() throws IOException {
+    localDirs = createLocalDirs(2);
+    MapConfigProvider provider = new MapConfigProvider(
+      ImmutableMap.of("spark.shuffle.server.minChunkSizeInMergedShuffleFile", "4"));
+    conf = new TransportConf("shuffle", provider);
+    pushResolver = new RemoteBlockPushResolver(conf);
+    registerExecutor(TEST_APP, prepareLocalDirs(localDirs));
+  }
+
+  @After
+  public void after() {
+    try {
+      for (Path local : localDirs) {
+        FileUtils.deleteDirectory(local.toFile());
+      }
+      removeApplication(TEST_APP);
+    } catch (Exception e) {
+      // don't fail if clean up doesn't succeed.
+      log.debug("Error while tearing down", e);
+    }
+  }
+
+  @Test(expected = RuntimeException.class)
+  public void testNoIndexFile() {
+    try {
+      pushResolver.getMergedBlockMeta(TEST_APP, 0, 0);
+    } catch (Throwable t) {
+      assertTrue(t.getMessage().startsWith("Merged shuffle index file"));
+      Throwables.propagate(t);
+    }
+  }
+
+  @Test
+  public void testBasicBlockMerge() throws IOException {
+    PushBlock[] pushBlocks = new PushBlock[] {
+      new PushBlock(0, 0, 0, ByteBuffer.wrap(new byte[4])),
+      new PushBlock(0, 1, 0, ByteBuffer.wrap(new byte[5]))
+    };
+    pushBlockHelper(TEST_APP, pushBlocks);
+    MergeStatuses statuses = pushResolver.finalizeShuffleMerge(
+      new FinalizeShuffleMerge(TEST_APP, 0));
+    validateMergeStatuses(statuses, new int[] {0}, new long[] {9});
+    MergedBlockMeta blockMeta = pushResolver.getMergedBlockMeta(TEST_APP, 0, 0);
+    validateChunks(TEST_APP, 0, 0, blockMeta, new int[]{4, 5}, new int[][]{{0}, {1}});
+  }
+
+  @Test
+  public void testDividingMergedBlocksIntoChunks() throws IOException {
+    PushBlock[] pushBlocks = new PushBlock[] {
+      new PushBlock(0, 0, 0, ByteBuffer.wrap(new byte[2])),
+      new PushBlock(0, 1, 0, ByteBuffer.wrap(new byte[3])),
+      new PushBlock(0, 2, 0, ByteBuffer.wrap(new byte[5])),
+      new PushBlock(0, 3, 0, ByteBuffer.wrap(new byte[3]))
+    };
+    pushBlockHelper(TEST_APP, pushBlocks);
+    MergeStatuses statuses = pushResolver.finalizeShuffleMerge(
+      new FinalizeShuffleMerge(TEST_APP, 0));
+    validateMergeStatuses(statuses, new int[] {0}, new long[] {13});
+    MergedBlockMeta meta = pushResolver.getMergedBlockMeta(TEST_APP, 0, 0);
+    validateChunks(TEST_APP, 0, 0, meta, new int[]{5, 5, 3}, new int[][]{{0, 1}, {2}, {3}});
+  }
+
+  @Test
+  public void testFinalizeWithMultipleReducePartitions() throws IOException {
+    PushBlock[] pushBlocks = new PushBlock[] {
+      new PushBlock(0, 0, 0, ByteBuffer.wrap(new byte[2])),
+      new PushBlock(0, 1, 0, ByteBuffer.wrap(new byte[3])),
+      new PushBlock(0, 0, 1, ByteBuffer.wrap(new byte[5])),
+      new PushBlock(0, 1, 1, ByteBuffer.wrap(new byte[3]))
+    };
+    pushBlockHelper(TEST_APP, pushBlocks);
+    MergeStatuses statuses = pushResolver.finalizeShuffleMerge(
+      new FinalizeShuffleMerge(TEST_APP, 0));
+    validateMergeStatuses(statuses, new int[] {0, 1}, new long[] {5, 8});
+    MergedBlockMeta meta = pushResolver.getMergedBlockMeta(TEST_APP, 0, 0);
+    validateChunks(TEST_APP, 0, 0, meta, new int[]{5}, new int[][]{{0, 1}});
+  }
+
+  @Test
+  public void testDeferredBufsAreWrittenDuringOnData() throws IOException {
+    StreamCallbackWithID stream1 =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 0, 0, 0));
+    stream1.onData(stream1.getID(), ByteBuffer.wrap(new byte[2]));
+    StreamCallbackWithID stream2 =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 1, 0, 0));
+    // This should be deferred
+    stream2.onData(stream2.getID(), ByteBuffer.wrap(new byte[3]));
+    // stream 1 now completes
+    stream1.onData(stream1.getID(), ByteBuffer.wrap(new byte[2]));
+    stream1.onComplete(stream1.getID());
+    // stream 2 has more data and then completes
+    stream2.onData(stream2.getID(), ByteBuffer.wrap(new byte[3]));
+    stream2.onComplete(stream2.getID());
+    pushResolver.finalizeShuffleMerge(new FinalizeShuffleMerge(TEST_APP, 0));
+    MergedBlockMeta blockMeta = pushResolver.getMergedBlockMeta(TEST_APP, 0, 0);
+    validateChunks(TEST_APP, 0, 0, blockMeta, new int[]{4, 6}, new int[][]{{0}, {1}});
+  }
+
+  @Test
+  public void testDeferredBufsAreWrittenDuringOnComplete() throws IOException {
+    StreamCallbackWithID stream1 =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 0, 0, 0));
+    stream1.onData(stream1.getID(), ByteBuffer.wrap(new byte[2]));
+    StreamCallbackWithID stream2 =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 1, 0, 0));
+    // This should be deferred
+    stream2.onData(stream2.getID(), ByteBuffer.wrap(new byte[3]));
+    stream2.onData(stream2.getID(), ByteBuffer.wrap(new byte[3]));
+    // stream 1 now completes
+    stream1.onData(stream1.getID(), ByteBuffer.wrap(new byte[2]));
+    stream1.onComplete(stream1.getID());
+    // stream 2 now completes completes
+    stream2.onComplete(stream2.getID());
+    pushResolver.finalizeShuffleMerge(new FinalizeShuffleMerge(TEST_APP, 0));
+    MergedBlockMeta blockMeta = pushResolver.getMergedBlockMeta(TEST_APP, 0, 0);
+    validateChunks(TEST_APP, 0, 0, blockMeta, new int[]{4, 6}, new int[][]{{0}, {1}});
+  }
+
+  @Test
+  public void testDuplicateBlocksAreIgnoredWhenPrevStreamHasCompleted() throws IOException {
+    StreamCallbackWithID stream1 =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 0, 0, 0));
+    stream1.onData(stream1.getID(), ByteBuffer.wrap(new byte[2]));
+    stream1.onData(stream1.getID(), ByteBuffer.wrap(new byte[2]));
+    stream1.onComplete(stream1.getID());
+    StreamCallbackWithID stream2 =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 0, 0, 0));
+    // This should be ignored
+    stream2.onData(stream2.getID(), ByteBuffer.wrap(new byte[2]));
+    stream2.onData(stream2.getID(), ByteBuffer.wrap(new byte[2]));
+    stream2.onComplete(stream2.getID());
+    pushResolver.finalizeShuffleMerge(new FinalizeShuffleMerge(TEST_APP, 0));
+    MergedBlockMeta blockMeta = pushResolver.getMergedBlockMeta(TEST_APP, 0, 0);
+    validateChunks(TEST_APP, 0, 0, blockMeta, new int[]{4}, new int[][]{{0}});
+  }
+
+  @Test
+  public void testDuplicateBlocksAreIgnoredWhenPrevStreamIsInProgress() throws IOException {
+    StreamCallbackWithID stream1 =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 0, 0, 0));
+    stream1.onData(stream1.getID(), ByteBuffer.wrap(new byte[2]));
+    StreamCallbackWithID stream2 =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 0, 0, 0));
+    // This should be ignored
+    stream2.onData(stream2.getID(), ByteBuffer.wrap(new byte[2]));
+    stream2.onData(stream2.getID(), ByteBuffer.wrap(new byte[2]));
+    // stream 1 now completes
+    stream1.onData(stream1.getID(), ByteBuffer.wrap(new byte[2]));
+    stream1.onComplete(stream1.getID());
+    // stream 2 now completes completes
+    stream2.onComplete(stream2.getID());
+    pushResolver.finalizeShuffleMerge(new FinalizeShuffleMerge(TEST_APP, 0));
+    MergedBlockMeta blockMeta = pushResolver.getMergedBlockMeta(TEST_APP, 0, 0);
+    validateChunks(TEST_APP, 0, 0, blockMeta, new int[]{4}, new int[][]{{0}});
+  }
+
+  @Test
+  public void testFailureAfterData() throws IOException {
+    StreamCallbackWithID stream =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 0, 0, 0));
+    stream.onData(stream.getID(), ByteBuffer.wrap(new byte[4]));
+    stream.onFailure(stream.getID(), new RuntimeException("Forced Failure"));
+    pushResolver.finalizeShuffleMerge(new FinalizeShuffleMerge(TEST_APP, 0));
+    MergedBlockMeta blockMeta = pushResolver.getMergedBlockMeta(TEST_APP, 0, 0);
+    assertEquals("num-chunks", 0, blockMeta.getNumChunks());
+  }
+
+  @Test
+  public void testFailureAfterMultipleDataBlocks() throws IOException {
+    StreamCallbackWithID stream =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 0, 0, 0));
+    stream.onData(stream.getID(), ByteBuffer.wrap(new byte[2]));
+    stream.onData(stream.getID(), ByteBuffer.wrap(new byte[3]));
+    stream.onData(stream.getID(), ByteBuffer.wrap(new byte[4]));
+    stream.onFailure(stream.getID(), new RuntimeException("Forced Failure"));
+    pushResolver.finalizeShuffleMerge(new FinalizeShuffleMerge(TEST_APP, 0));
+    MergedBlockMeta blockMeta = pushResolver.getMergedBlockMeta(TEST_APP, 0, 0);
+    assertEquals("num-chunks", 0, blockMeta.getNumChunks());
+  }
+
+  @Test
+  public void testFailureAfterComplete() throws IOException {
+    StreamCallbackWithID stream =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 0, 0, 0));
+    stream.onData(stream.getID(), ByteBuffer.wrap(new byte[2]));
+    stream.onData(stream.getID(), ByteBuffer.wrap(new byte[3]));
+    stream.onData(stream.getID(), ByteBuffer.wrap(new byte[4]));
+    stream.onComplete(stream.getID());
+    stream.onFailure(stream.getID(), new RuntimeException("Forced Failure"));
+    pushResolver.finalizeShuffleMerge(new FinalizeShuffleMerge(TEST_APP, 0));
+    MergedBlockMeta blockMeta = pushResolver.getMergedBlockMeta(TEST_APP, 0, 0);
+    validateChunks(TEST_APP, 0, 0, blockMeta, new int[]{9}, new int[][]{{0}});
+  }
+
+  @Test (expected = RuntimeException.class)
+  public void testTooLateArrival() throws IOException {
+    ByteBuffer[] blocks = new ByteBuffer[]{
+      ByteBuffer.wrap(new byte[4]),
+      ByteBuffer.wrap(new byte[5])
+    };
+    StreamCallbackWithID stream = pushResolver.receiveBlockDataAsStream(
+      new PushBlockStream(TEST_APP, 0, 0, 0, 0));
+    for (ByteBuffer block : blocks) {
+      stream.onData(stream.getID(), block);
+    }
+    stream.onComplete(stream.getID());
+    pushResolver.finalizeShuffleMerge(new FinalizeShuffleMerge(TEST_APP, 0));
+    StreamCallbackWithID stream1 = pushResolver.receiveBlockDataAsStream(
+      new PushBlockStream(TEST_APP, 0, 1, 0, 0));
+    stream1.onData(stream1.getID(), ByteBuffer.wrap(new byte[4]));
+    try {
+      stream1.onComplete(stream1.getID());
+    } catch (RuntimeException re) {
+      assertEquals(
+        "Block shufflePush_0_1_0 received after merged shuffle is finalized",
+          re.getMessage());
+      MergedBlockMeta blockMeta = pushResolver.getMergedBlockMeta(TEST_APP, 0, 0);
+      validateChunks(TEST_APP, 0, 0, blockMeta, new int[]{9}, new int[][]{{0}});
+      throw re;
+    }
+  }
+
+  @Test
+  public void testIncompleteStreamsAreOverwritten() throws IOException {
+    registerExecutor(TEST_APP, prepareLocalDirs(localDirs));
+    byte[] expectedBytes = new byte[4];
+    ThreadLocalRandom.current().nextBytes(expectedBytes);
+
+    StreamCallbackWithID stream1 =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 0, 0, 0));
+    byte[] data = new byte[10];
+    ThreadLocalRandom.current().nextBytes(data);
+    stream1.onData(stream1.getID(), ByteBuffer.wrap(data));
+    // There is a failure
+    stream1.onFailure(stream1.getID(), new RuntimeException("forced error"));
+    StreamCallbackWithID stream2 =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 1, 0, 0));
+    ByteBuffer nextBuf= ByteBuffer.wrap(expectedBytes, 0, 2);
+    stream2.onData(stream2.getID(), nextBuf);
+    stream2.onComplete(stream2.getID());
+    StreamCallbackWithID stream3 =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 2, 0, 0));
+    nextBuf =  ByteBuffer.wrap(expectedBytes, 2, 2);
+    stream3.onData(stream3.getID(), nextBuf);
+    stream3.onComplete(stream3.getID());
+    pushResolver.finalizeShuffleMerge(new FinalizeShuffleMerge(TEST_APP, 0));
+    MergedBlockMeta blockMeta = pushResolver.getMergedBlockMeta(TEST_APP, 0, 0);
+    validateChunks(TEST_APP, 0, 0, blockMeta, new int[]{4}, new int[][]{{1, 2}});
+    FileSegmentManagedBuffer mb =
+      (FileSegmentManagedBuffer) pushResolver.getMergedBlockData(TEST_APP, 0, 0, 0);
+    assertArrayEquals(expectedBytes, mb.nioByteBuffer().array());
+  }
+
+  @Test (expected = RuntimeException.class)
+  public void testCollision() throws IOException {
+    StreamCallbackWithID stream1 =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 0, 0, 0));
+    stream1.onData(stream1.getID(), ByteBuffer.wrap(new byte[2]));
+    StreamCallbackWithID stream2 =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 1, 0, 0));
+    // This should be deferred
+    stream2.onData(stream2.getID(), ByteBuffer.wrap(new byte[5]));
+    // Since stream2 didn't get any opportunity it will throw couldn't find opportunity error
+    try {
+      stream2.onComplete(stream2.getID());
+    } catch (RuntimeException re) {
+      assertEquals(
+        "Couldn't find an opportunity to write block shufflePush_0_1_0 to merged shuffle",
+        re.getMessage());
+      throw re;
+    }
+  }
+
+  @Test (expected = RuntimeException.class)
+  public void testFailureInAStreamDoesNotInterfereWithStreamWhichIsWriting() throws IOException {
+    StreamCallbackWithID stream1 =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 0, 0, 0));
+    stream1.onData(stream1.getID(), ByteBuffer.wrap(new byte[2]));
+    StreamCallbackWithID stream2 =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 1, 0, 0));
+    // There is a failure with stream2
+    stream2.onFailure(stream2.getID(), new RuntimeException("forced error"));
+    StreamCallbackWithID stream3 =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 2, 0, 0));
+    // This should be deferred
+    stream3.onData(stream3.getID(), ByteBuffer.wrap(new byte[5]));
+    // Since this stream didn't get any opportunity it will throw couldn't find opportunity error
+    RuntimeException failedEx = null;
+    try {
+      stream3.onComplete(stream3.getID());
+    } catch (RuntimeException re) {
+      assertEquals(
+        "Couldn't find an opportunity to write block shufflePush_0_2_0 to merged shuffle",
+        re.getMessage());
+      failedEx = re;
+    }
+    // stream 1 now completes
+    stream1.onData(stream1.getID(), ByteBuffer.wrap(new byte[2]));
+    stream1.onComplete(stream1.getID());
+
+    pushResolver.finalizeShuffleMerge(new FinalizeShuffleMerge(TEST_APP, 0));
+    MergedBlockMeta blockMeta = pushResolver.getMergedBlockMeta(TEST_APP, 0, 0);
+    validateChunks(TEST_APP, 0, 0, blockMeta, new int[] {4}, new int[][] {{0}});
+    if (failedEx != null) {
+      throw failedEx;
+    }
+  }
+
+  @Test(expected = NullPointerException.class)
+  public void testUpdateLocalDirsOnlyOnce() throws IOException {
+    String testApp = "updateLocalDirsOnlyOnceTest";
+    Path[] activeLocalDirs = createLocalDirs(1);
+    registerExecutor(testApp, prepareLocalDirs(activeLocalDirs));
+    assertEquals(pushResolver.getMergedBlockDirs(testApp).length, 1);
+    assertTrue(pushResolver.getMergedBlockDirs(testApp)[0].contains(
+      activeLocalDirs[0].toFile().getPath()));
+    // Any later executor register from the same application should not change the active local
+    // dirs list
+    Path[] updatedLocalDirs = localDirs;
+    registerExecutor(testApp, prepareLocalDirs(updatedLocalDirs));
+    assertEquals(pushResolver.getMergedBlockDirs(testApp).length, 1);
+    assertTrue(pushResolver.getMergedBlockDirs(testApp)[0].contains(
+      activeLocalDirs[0].toFile().getPath()));
+    removeApplication(testApp);
+    try {
+      pushResolver.getMergedBlockDirs(testApp);
+    } catch (Throwable e) {
+      assertTrue(e.getMessage()
+        .startsWith("application " + testApp + " is not registered or NM was restarted."));
+      Throwables.propagate(e);
+    }
+  }
+
+  @Test
+  public void testCleanUpDirectory() throws IOException, InterruptedException {
+    String testApp = "cleanUpDirectory";
+    Semaphore deleted = new Semaphore(0);
+    pushResolver = new RemoteBlockPushResolver(conf) {
+      @Override
+      void deleteExecutorDirs(Path[] dirs) {
+        super.deleteExecutorDirs(dirs);
+        deleted.release();
+      }
+    };
+    Path[] activeDirs = createLocalDirs(1);
+    registerExecutor(testApp, prepareLocalDirs(activeDirs));
+    PushBlock[] pushBlocks = new PushBlock[] {
+      new PushBlock(0, 0, 0, ByteBuffer.wrap(new byte[4]))};
+    pushBlockHelper(testApp, pushBlocks);
+    pushResolver.finalizeShuffleMerge(new FinalizeShuffleMerge(testApp, 0));
+    MergedBlockMeta blockMeta = pushResolver.getMergedBlockMeta(testApp, 0, 0);
+    validateChunks(testApp, 0, 0, blockMeta, new int[]{4}, new int[][]{{0}});
+    String[] mergeDirs = pushResolver.getMergedBlockDirs(testApp);
+    pushResolver.applicationRemoved(testApp,  true);
+    // Since the cleanup happen in a different thread, check few times to see if the merge dirs gets
+    // deleted.
+    deleted.acquire();
+    for (String mergeDir : mergeDirs) {
+      Assert.assertFalse(Files.exists(Paths.get(mergeDir)));
+    }
+  }
+
+  @Test
+  public void testRecoverIndexFileAfterIOExceptions() throws IOException {
+    useTestFiles(true, false);
+    RemoteBlockPushResolver.PushBlockStreamCallback callback1 =
+      (RemoteBlockPushResolver.PushBlockStreamCallback) pushResolver.receiveBlockDataAsStream(
+        new PushBlockStream(TEST_APP, 0, 0, 0, 0));
+    callback1.onData(callback1.getID(), ByteBuffer.wrap(new byte[4]));
+    callback1.onComplete(callback1.getID());
+    RemoteBlockPushResolver.AppShufflePartitionInfo partitionInfo = callback1.getPartitionInfo();
+    // Close the index stream so it throws IOException
+    TestMergeShuffleFile testIndexFile = (TestMergeShuffleFile) partitionInfo.getIndexFile();
+    testIndexFile.close();
+    StreamCallbackWithID callback2 = pushResolver.receiveBlockDataAsStream(
+      new PushBlockStream(TEST_APP, 0, 1, 0, 0));
+    callback2.onData(callback2.getID(), ByteBuffer.wrap(new byte[5]));
+    // This will complete without any IOExceptions because number of IOExceptions are less than
+    // the threshold but the update to index file will be unsuccessful.
+    callback2.onComplete(callback2.getID());
+    assertEquals("index position", 16, testIndexFile.getPos());
+    // Restore the index stream so it can write successfully again.
+    testIndexFile.restore();
+    StreamCallbackWithID callback3 = pushResolver.receiveBlockDataAsStream(
+      new PushBlockStream(TEST_APP, 0, 2, 0, 0));
+    callback3.onData(callback3.getID(), ByteBuffer.wrap(new byte[2]));
+    callback3.onComplete(callback3.getID());
+    assertEquals("index position", 24, testIndexFile.getPos());
+    MergeStatuses statuses = pushResolver.finalizeShuffleMerge(
+      new FinalizeShuffleMerge(TEST_APP, 0));
+    validateMergeStatuses(statuses, new int[] {0}, new long[] {11});
+    MergedBlockMeta blockMeta = pushResolver.getMergedBlockMeta(TEST_APP, 0, 0);
+    validateChunks(TEST_APP, 0, 0, blockMeta, new int[] {4, 7}, new int[][] {{0}, {1, 2}});
+  }
+
+  @Test
+  public void testRecoverIndexFileAfterIOExceptionsInFinalize() throws IOException {
+    useTestFiles(true, false);
+    RemoteBlockPushResolver.PushBlockStreamCallback callback1 =
+      (RemoteBlockPushResolver.PushBlockStreamCallback) pushResolver.receiveBlockDataAsStream(
+        new PushBlockStream(TEST_APP, 0, 0, 0, 0));
+    callback1.onData(callback1.getID(), ByteBuffer.wrap(new byte[4]));
+    callback1.onComplete(callback1.getID());
+    RemoteBlockPushResolver.AppShufflePartitionInfo partitionInfo = callback1.getPartitionInfo();
+    // Close the index stream so it throws IOException
+    TestMergeShuffleFile testIndexFile = (TestMergeShuffleFile) partitionInfo.getIndexFile();
+    testIndexFile.close();
+    StreamCallbackWithID callback2 = pushResolver.receiveBlockDataAsStream(
+      new PushBlockStream(TEST_APP, 0, 1, 0, 0));
+    callback2.onData(callback2.getID(), ByteBuffer.wrap(new byte[5]));
+    // This will complete without any IOExceptions because number of IOExceptions are less than
+    // the threshold but the update to index file will be unsuccessful.
+    callback2.onComplete(callback2.getID());
+    assertEquals("index position", 16, testIndexFile.getPos());
+    // The last update to index was unsuccessful however any further updates will be successful.
+    // Restore the index stream so it can write successfully again.
+    testIndexFile.restore();
+    MergeStatuses statuses = pushResolver.finalizeShuffleMerge(
+      new FinalizeShuffleMerge(TEST_APP, 0));
+    assertEquals("index position", 24, testIndexFile.getPos());
+    validateMergeStatuses(statuses, new int[] {0}, new long[] {9});
+    MergedBlockMeta blockMeta = pushResolver.getMergedBlockMeta(TEST_APP, 0, 0);
+    validateChunks(TEST_APP, 0, 0, blockMeta, new int[] {4, 5}, new int[][] {{0}, {1}});
+  }
+
+  @Test
+  public void testRecoverMetaFileAfterIOExceptions() throws IOException {
+    useTestFiles(false, true);
+    RemoteBlockPushResolver.PushBlockStreamCallback callback1 =
+      (RemoteBlockPushResolver.PushBlockStreamCallback) pushResolver.receiveBlockDataAsStream(
+        new PushBlockStream(TEST_APP, 0, 0, 0, 0));
+    callback1.onData(callback1.getID(), ByteBuffer.wrap(new byte[4]));
+    callback1.onComplete(callback1.getID());
+    RemoteBlockPushResolver.AppShufflePartitionInfo partitionInfo = callback1.getPartitionInfo();
+    // Close the meta stream so it throws IOException
+    TestMergeShuffleFile testMetaFile = (TestMergeShuffleFile) partitionInfo.getMetaFile();
+    long metaPosBeforeClose = testMetaFile.getPos();
+    testMetaFile.close();
+    StreamCallbackWithID callback2 = pushResolver.receiveBlockDataAsStream(
+      new PushBlockStream(TEST_APP, 0, 1, 0, 0));
+    callback2.onData(callback2.getID(), ByteBuffer.wrap(new byte[5]));
+    // This will complete without any IOExceptions because number of IOExceptions are less than
+    // the threshold but the update to index and meta file will be unsuccessful.
+    callback2.onComplete(callback2.getID());
+    assertEquals("index position", 16, partitionInfo.getIndexFile().getPos());
+    assertEquals("meta position", metaPosBeforeClose, testMetaFile.getPos());
+    // Restore the meta stream so it can write successfully again.
+    testMetaFile.restore();
+    StreamCallbackWithID callback3 = pushResolver.receiveBlockDataAsStream(
+      new PushBlockStream(TEST_APP, 0, 2, 0, 0));
+    callback3.onData(callback3.getID(), ByteBuffer.wrap(new byte[2]));
+    callback3.onComplete(callback3.getID());
+    assertEquals("index position", 24, partitionInfo.getIndexFile().getPos());
+    assertTrue("meta position", testMetaFile.getPos() > metaPosBeforeClose);
+    MergeStatuses statuses = pushResolver.finalizeShuffleMerge(
+      new FinalizeShuffleMerge(TEST_APP, 0));
+    validateMergeStatuses(statuses, new int[] {0}, new long[] {11});
+    MergedBlockMeta blockMeta = pushResolver.getMergedBlockMeta(TEST_APP, 0, 0);
+    validateChunks(TEST_APP, 0, 0, blockMeta, new int[] {4, 7}, new int[][] {{0}, {1, 2}});
+  }
+
+  @Test
+  public void testRecoverMetaFileAfterIOExceptionsInFinalize() throws IOException {
+    useTestFiles(false, true);
+    RemoteBlockPushResolver.PushBlockStreamCallback callback1 =
+      (RemoteBlockPushResolver.PushBlockStreamCallback) pushResolver.receiveBlockDataAsStream(
+        new PushBlockStream(TEST_APP, 0, 0, 0, 0));
+    callback1.onData(callback1.getID(), ByteBuffer.wrap(new byte[4]));
+    callback1.onComplete(callback1.getID());
+    RemoteBlockPushResolver.AppShufflePartitionInfo partitionInfo = callback1.getPartitionInfo();
+    // Close the meta stream so it throws IOException
+    TestMergeShuffleFile testMetaFile = (TestMergeShuffleFile) partitionInfo.getMetaFile();
+    long metaPosBeforeClose = testMetaFile.getPos();
+    testMetaFile.close();
+    StreamCallbackWithID callback2 = pushResolver.receiveBlockDataAsStream(
+      new PushBlockStream(TEST_APP, 0, 1, 0, 0));
+    callback2.onData(callback2.getID(), ByteBuffer.wrap(new byte[5]));
+    // This will complete without any IOExceptions because number of IOExceptions are less than
+    // the threshold but the update to index and meta file will be unsuccessful.
+    callback2.onComplete(callback2.getID());
+    MergeShuffleFile indexFile = partitionInfo.getIndexFile();
+    assertEquals("index position", 16, indexFile.getPos());
+    assertEquals("meta position", metaPosBeforeClose, testMetaFile.getPos());
+    // Restore the meta stream so it can write successfully again.
+    testMetaFile.restore();
+    MergeStatuses statuses = pushResolver.finalizeShuffleMerge(
+      new FinalizeShuffleMerge(TEST_APP, 0));
+    assertEquals("index position", 24, indexFile.getPos());
+    assertTrue("meta position", testMetaFile.getPos() > metaPosBeforeClose);
+    validateMergeStatuses(statuses, new int[] {0}, new long[] {9});
+    MergedBlockMeta blockMeta = pushResolver.getMergedBlockMeta(TEST_APP, 0, 0);
+    validateChunks(TEST_APP, 0, 0, blockMeta, new int[] {4, 5}, new int[][] {{0}, {1}});
+  }
+
+  @Test (expected = RuntimeException.class)
+  public void testIOExceptionsExceededThreshold() throws IOException {
+    RemoteBlockPushResolver.PushBlockStreamCallback callback =
+      (RemoteBlockPushResolver.PushBlockStreamCallback) pushResolver.receiveBlockDataAsStream(
+        new PushBlockStream(TEST_APP, 0, 0, 0, 0));
+    RemoteBlockPushResolver.AppShufflePartitionInfo partitionInfo = callback.getPartitionInfo();
+    callback.onData(callback.getID(), ByteBuffer.wrap(new byte[4]));
+    callback.onComplete(callback.getID());
+    // Close the data stream so it throws continuous IOException
+    partitionInfo.getDataChannel().close();
+    for (int i = 1; i < 5; i++) {
+      RemoteBlockPushResolver.PushBlockStreamCallback callback1 =
+        (RemoteBlockPushResolver.PushBlockStreamCallback) pushResolver.receiveBlockDataAsStream(
+          new PushBlockStream(TEST_APP, 0, i, 0, 0));
+      try {
+        callback1.onData(callback1.getID(), ByteBuffer.wrap(new byte[2]));
+      } catch (IOException ioe) {
+        // this will throw IOException so the client can retry.
+        callback1.onFailure(callback1.getID(), ioe);
+      }
+    }
+    assertEquals(4, partitionInfo.getNumIOExceptions());
+    // After 4 IOException, the server will respond with IOExceptions exceeded threshold
+    try {
+      RemoteBlockPushResolver.PushBlockStreamCallback callback2 =
+        (RemoteBlockPushResolver.PushBlockStreamCallback) pushResolver.receiveBlockDataAsStream(
+          new PushBlockStream(TEST_APP, 0, 5, 0, 0));
+      callback2.onData(callback.getID(), ByteBuffer.wrap(new byte[1]));
+    } catch (Throwable t) {
+      assertEquals("IOExceptions exceeded the threshold when merging shufflePush_0_5_0",
+        t.getMessage());
+      throw t;
+    }
+  }
+
+  @Test (expected = RuntimeException.class)
+  public void testIOExceptionsDuringMetaUpdateIncreasesExceptionCount() throws IOException {
+    useTestFiles(true, false);
+    RemoteBlockPushResolver.PushBlockStreamCallback callback =
+      (RemoteBlockPushResolver.PushBlockStreamCallback) pushResolver.receiveBlockDataAsStream(
+        new PushBlockStream(TEST_APP, 0, 0, 0, 0));
+    RemoteBlockPushResolver.AppShufflePartitionInfo partitionInfo = callback.getPartitionInfo();
+    callback.onData(callback.getID(), ByteBuffer.wrap(new byte[4]));
+    callback.onComplete(callback.getID());
+    TestMergeShuffleFile testIndexFile = (TestMergeShuffleFile) partitionInfo.getIndexFile();
+    testIndexFile.close();
+    for (int i = 1; i < 5; i++) {
+      RemoteBlockPushResolver.PushBlockStreamCallback callback1 =
+        (RemoteBlockPushResolver.PushBlockStreamCallback) pushResolver.receiveBlockDataAsStream(
+          new PushBlockStream(TEST_APP, 0, i, 0, 0));
+      callback1.onData(callback1.getID(), ByteBuffer.wrap(new byte[5]));
+      // This will complete without any exceptions but the exception count is increased.
+      callback1.onComplete(callback1.getID());
+    }
+    assertEquals(4, partitionInfo.getNumIOExceptions());
+    // After 4 IOException, the server will respond with IOExceptions exceeded threshold for any
+    // new request for this partition.
+    try {
+      RemoteBlockPushResolver.PushBlockStreamCallback callback2 =
+      (RemoteBlockPushResolver.PushBlockStreamCallback) pushResolver.receiveBlockDataAsStream(
+        new PushBlockStream(TEST_APP, 0, 5, 0, 0));
+      callback2.onData(callback2.getID(), ByteBuffer.wrap(new byte[4]));
+      callback2.onComplete(callback2.getID());
+    } catch (Throwable t) {
+      assertEquals("IOExceptions exceeded the threshold when merging shufflePush_0_5_0",
+        t.getMessage());
+      throw t;
+    }
+  }
+
+  @Test (expected = RuntimeException.class)
+  public void testRequestForAbortedShufflePartitionThrowsException() {
+    try {
+      testIOExceptionsDuringMetaUpdateIncreasesExceptionCount();
+    } catch (Throwable t) {
+      // No more blocks can be merged to this partition.
+    }
+    try {
+      pushResolver.receiveBlockDataAsStream(
+        new PushBlockStream(TEST_APP, 0, 10, 0, 0));
+    } catch (Throwable t) {
+      assertEquals("IOExceptions exceeded the threshold when merging shufflePush_0_10_0",
+        t.getMessage());
+      throw t;
+    }
+  }
+
+  @Test (expected = RuntimeException.class)
+  public void testPendingBlockIsAbortedImmediately() throws IOException {
+    useTestFiles(true, false);
+    RemoteBlockPushResolver.PushBlockStreamCallback callback =
+      (RemoteBlockPushResolver.PushBlockStreamCallback) pushResolver.receiveBlockDataAsStream(
+        new PushBlockStream(TEST_APP, 0, 0, 0, 0));
+    RemoteBlockPushResolver.AppShufflePartitionInfo partitionInfo = callback.getPartitionInfo();
+    TestMergeShuffleFile testIndexFile = (TestMergeShuffleFile) partitionInfo.getIndexFile();
+    testIndexFile.close();
+    for (int i = 1; i < 6; i++) {
+      RemoteBlockPushResolver.PushBlockStreamCallback callback1 =
+        (RemoteBlockPushResolver.PushBlockStreamCallback) pushResolver.receiveBlockDataAsStream(
+          new PushBlockStream(TEST_APP, 0, i, 0, 0));
+      try {
+        callback1.onData(callback1.getID(), ByteBuffer.wrap(new byte[5]));
+        // This will complete without any exceptions but the exception count is increased.
+        callback1.onComplete(callback1.getID());
+      } catch (Throwable t) {
+        callback1.onFailure(callback1.getID(), t);
+      }
+    }
+    assertEquals(5, partitionInfo.getNumIOExceptions());
+    // The server will respond with IOExceptions exceeded threshold for any additional attempts
+    // to write.
+    try {
+      callback.onData(callback.getID(), ByteBuffer.wrap(new byte[4]));
+    } catch (Throwable t) {
+      assertEquals("IOExceptions exceeded the threshold when merging shufflePush_0_0_0",
+        t.getMessage());
+      throw t;
+    }
+  }
+
+  @Test (expected = RuntimeException.class)
+  public void testWritingPendingBufsIsAbortedImmediatelyDuringComplete() throws IOException {
+    useTestFiles(true, false);
+    RemoteBlockPushResolver.PushBlockStreamCallback callback =
+      (RemoteBlockPushResolver.PushBlockStreamCallback) pushResolver.receiveBlockDataAsStream(
+        new PushBlockStream(TEST_APP, 0, 0, 0, 0));
+    RemoteBlockPushResolver.AppShufflePartitionInfo partitionInfo = callback.getPartitionInfo();
+    TestMergeShuffleFile testIndexFile = (TestMergeShuffleFile) partitionInfo.getIndexFile();
+    testIndexFile.close();
+    for (int i = 1; i < 5; i++) {
+      RemoteBlockPushResolver.PushBlockStreamCallback callback1 =
+        (RemoteBlockPushResolver.PushBlockStreamCallback) pushResolver.receiveBlockDataAsStream(
+          new PushBlockStream(TEST_APP, 0, i, 0, 0));
+      try {
+        callback1.onData(callback1.getID(), ByteBuffer.wrap(new byte[5]));
+        // This will complete without any exceptions but the exception count is increased.
+        callback1.onComplete(callback1.getID());
+      } catch (Throwable t) {
+        callback1.onFailure(callback1.getID(), t);
+      }
+    }
+    assertEquals(4, partitionInfo.getNumIOExceptions());
+    RemoteBlockPushResolver.PushBlockStreamCallback callback2 =
+      (RemoteBlockPushResolver.PushBlockStreamCallback) pushResolver.receiveBlockDataAsStream(
+        new PushBlockStream(TEST_APP, 0, 5, 0, 0));
+    callback2.onData(callback2.getID(), ByteBuffer.wrap(new byte[5]));
+    // This is deferred
+    callback.onData(callback.getID(), ByteBuffer.wrap(new byte[4]));
+    // Callback2 completes which will throw another exception.
+    try {
+      callback2.onComplete(callback2.getID());
+    } catch (Throwable t) {
+      callback2.onFailure(callback2.getID(), t);
+    }
+    assertEquals(5, partitionInfo.getNumIOExceptions());
+    // Restore index file so that any further writes to it are successful and any exceptions are
+    // due to IOExceptions exceeding threshold.
+    testIndexFile.restore();
+    try {
+      callback.onComplete(callback.getID());
+    } catch (Throwable t) {
+      assertEquals("IOExceptions exceeded the threshold when merging shufflePush_0_0_0",
+        t.getMessage());
+      throw t;
+    }
+  }
+
+  @Test
+  public void testFailureWhileTruncatingFiles() throws IOException {
+    useTestFiles(true, false);
+    PushBlock[] pushBlocks = new PushBlock[] {
+      new PushBlock(0, 0, 0, ByteBuffer.wrap(new byte[2])),
+      new PushBlock(0, 1, 0, ByteBuffer.wrap(new byte[3])),
+      new PushBlock(0, 0, 1, ByteBuffer.wrap(new byte[5])),
+      new PushBlock(0, 1, 1, ByteBuffer.wrap(new byte[3]))
+    };
+    pushBlockHelper(TEST_APP, pushBlocks);
+    RemoteBlockPushResolver.PushBlockStreamCallback callback =
+      (RemoteBlockPushResolver.PushBlockStreamCallback) pushResolver.receiveBlockDataAsStream(
+        new PushBlockStream(TEST_APP, 0, 2, 0, 0));
+    callback.onData(callback.getID(), ByteBuffer.wrap(new byte[2]));
+    callback.onComplete(callback.getID());
+    RemoteBlockPushResolver.AppShufflePartitionInfo partitionInfo = callback.getPartitionInfo();
+    TestMergeShuffleFile testIndexFile = (TestMergeShuffleFile) partitionInfo.getIndexFile();
+    // Close the index file so truncate throws IOException
+    testIndexFile.close();
+    MergeStatuses statuses = pushResolver.finalizeShuffleMerge(
+      new FinalizeShuffleMerge(TEST_APP, 0));
+    validateMergeStatuses(statuses, new int[] {1}, new long[] {8});
+    MergedBlockMeta meta = pushResolver.getMergedBlockMeta(TEST_APP, 0, 1);
+    validateChunks(TEST_APP, 0, 1, meta, new int[]{5, 3}, new int[][]{{0},{1}});
+  }
+
+  @Test
+  public void testOnFailureInvokedMoreThanOncePerBlock() throws IOException {
+    StreamCallbackWithID stream1 =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 0, 0, 0));
+    stream1.onData(stream1.getID(), ByteBuffer.wrap(new byte[2]));
+    stream1.onFailure(stream1.getID(), new RuntimeException("forced error"));
+    StreamCallbackWithID stream2 =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 1, 0, 0));
+    stream2.onData(stream2.getID(), ByteBuffer.wrap(new byte[5]));
+    // On failure on stream1 gets invoked again and should cause no interference
+    stream1.onFailure(stream1.getID(), new RuntimeException("2nd forced error"));
+    StreamCallbackWithID stream3 =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 3, 0, 0));
+    // This should be deferred as stream 2 is still the active stream
+    stream3.onData(stream3.getID(), ByteBuffer.wrap(new byte[2]));
+    // Stream 2 writes more and completes
+    stream2.onData(stream2.getID(), ByteBuffer.wrap(new byte[4]));
+    stream2.onComplete(stream2.getID());
+    stream3.onComplete(stream3.getID());
+    pushResolver.finalizeShuffleMerge(new FinalizeShuffleMerge(TEST_APP, 0));
+    MergedBlockMeta blockMeta = pushResolver.getMergedBlockMeta(TEST_APP, 0, 0);
+    validateChunks(TEST_APP, 0, 0, blockMeta, new int[] {9, 2}, new int[][] {{1},{3}});
+    removeApplication(TEST_APP);
+  }
+
+  @Test (expected = RuntimeException.class)
+  public void testFailureAfterDuplicateBlockDoesNotInterfereActiveStream() throws IOException {
+    StreamCallbackWithID stream1 =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 0, 0, 0));
+    StreamCallbackWithID stream1Duplicate =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 0, 0, 0));
+    stream1.onData(stream1.getID(), ByteBuffer.wrap(new byte[2]));
+    stream1.onComplete(stream1.getID());
+    stream1Duplicate.onData(stream1.getID(), ByteBuffer.wrap(new byte[2]));
+
+    StreamCallbackWithID stream2 =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 1, 0, 0));
+    stream2.onData(stream2.getID(), ByteBuffer.wrap(new byte[5]));
+    // Should not change the current map id of the reduce partition
+    stream1Duplicate.onFailure(stream2.getID(), new RuntimeException("forced error"));
+
+    StreamCallbackWithID stream3 =
+      pushResolver.receiveBlockDataAsStream(new PushBlockStream(TEST_APP, 0, 2, 0, 0));
+    // This should be deferred as stream 2 is still the active stream
+    stream3.onData(stream3.getID(), ByteBuffer.wrap(new byte[2]));
+    RuntimeException failedEx = null;
+    try {
+      stream3.onComplete(stream3.getID());
+    } catch (RuntimeException re) {
+      assertEquals(
+        "Couldn't find an opportunity to write block shufflePush_0_2_0 to merged shuffle",
+        re.getMessage());
+      failedEx = re;
+    }
+    // Stream 2 writes more and completes
+    stream2.onData(stream2.getID(), ByteBuffer.wrap(new byte[4]));
+    stream2.onComplete(stream2.getID());
+    pushResolver.finalizeShuffleMerge(new FinalizeShuffleMerge(TEST_APP, 0));
+    MergedBlockMeta blockMeta = pushResolver.getMergedBlockMeta(TEST_APP, 0, 0);
+    validateChunks(TEST_APP, 0, 0, blockMeta, new int[] {11}, new int[][] {{0, 1}});
+    removeApplication(TEST_APP);
+    if (failedEx != null) {
+      throw failedEx;
+    }
+  }
+
+  private void useTestFiles(boolean useTestIndexFile, boolean useTestMetaFile) throws IOException {
+    pushResolver = new RemoteBlockPushResolver(conf) {
+      @Override
+      AppShufflePartitionInfo newAppShufflePartitionInfo(AppShuffleId appShuffleId, int reduceId,
+        File dataFile, File indexFile, File metaFile) throws IOException {
+        MergeShuffleFile mergedIndexFile = useTestIndexFile ? new TestMergeShuffleFile(indexFile)
+          : new MergeShuffleFile(indexFile);
+        MergeShuffleFile mergedMetaFile = useTestMetaFile ? new TestMergeShuffleFile(metaFile) :
+          new MergeShuffleFile(metaFile);
+        return new AppShufflePartitionInfo(appShuffleId, reduceId, dataFile, mergedIndexFile,
+          mergedMetaFile);
+      }
+    };
+    registerExecutor(TEST_APP, prepareLocalDirs(localDirs));
+  }
+
+  private Path[] createLocalDirs(int numLocalDirs) throws IOException {
+    Path[] localDirs = new Path[numLocalDirs];
+    for (int i = 0; i < localDirs.length; i++) {
+      localDirs[i] = Files.createTempDirectory("shuffleMerge");
+      localDirs[i].toFile().deleteOnExit();
+    }
+    return localDirs;
+  }
+
+  private void registerExecutor(String appId, String[] localDirs) throws IOException {
+    ExecutorShuffleInfo shuffleInfo = new ExecutorShuffleInfo(localDirs, 1, "mergedShuffle");
+    pushResolver.registerExecutor(appId, shuffleInfo);
+  }
+
+  private String[] prepareLocalDirs(Path[] localDirs) throws IOException {
+    String[] blockMgrDirs = new String[localDirs.length];
+    for (int i = 0; i< localDirs.length; i++) {
+      Files.createDirectories(localDirs[i].resolve(
+        RemoteBlockPushResolver.MERGE_MANAGER_DIR + File.separator + "00"));
+      blockMgrDirs[i] = localDirs[i].toFile().getPath() + File.separator + BLOCK_MANAGER_DIR;
+    }
+    return blockMgrDirs;
+  }
+
+  private void removeApplication(String appId) {
+    // PushResolver cleans up the local dirs in a different thread which can conflict with the test
+    // data of other tests, since they are using the same Application Id.
+    pushResolver.applicationRemoved(appId,  false);
+  }
+
+  private void validateMergeStatuses(
+      MergeStatuses mergeStatuses,
+      int[] expectedReduceIds,
+      long[] expectedSizes) {
+    assertArrayEquals(expectedReduceIds, mergeStatuses.reduceIds);
+    assertArrayEquals(expectedSizes, mergeStatuses.sizes);
+  }
+
+  private void validateChunks(
+      String appId,
+      int shuffleId,
+      int reduceId,
+      MergedBlockMeta meta,
+      int[] expectedSizes,
+      int[][] expectedMapsPerChunk) throws IOException {
+    assertEquals("num chunks", expectedSizes.length, meta.getNumChunks());
+    RoaringBitmap[] bitmaps = meta.readChunkBitmaps();
+    assertEquals("num of bitmaps", meta.getNumChunks(), bitmaps.length);
+    for (int i = 0; i < meta.getNumChunks(); i++) {
+      RoaringBitmap chunkBitmap = bitmaps[i];
+      Arrays.stream(expectedMapsPerChunk[i]).forEach(x -> assertTrue(chunkBitmap.contains(x)));
+    }
+    for (int i = 0; i < meta.getNumChunks(); i++) {
+      FileSegmentManagedBuffer mb =
+        (FileSegmentManagedBuffer) pushResolver.getMergedBlockData(appId, shuffleId, reduceId, i);
+      assertEquals(expectedSizes[i], mb.getLength());
+    }
+  }
+
+  private void pushBlockHelper(
+      String appId,
+      PushBlock[] blocks) throws IOException {
+    for (int i = 0; i < blocks.length; i++) {
+      StreamCallbackWithID stream = pushResolver.receiveBlockDataAsStream(
+        new PushBlockStream(appId, blocks[i].shuffleId, blocks[i].mapIndex, blocks[i].reduceId, 0));
+      stream.onData(stream.getID(), blocks[i].buffer);
+      stream.onComplete(stream.getID());
+    }
+  }
+
+  private static class PushBlock {
+    private final int shuffleId;
+    private final int mapIndex;
+    private final int reduceId;
+    private final ByteBuffer buffer;
+    PushBlock(int shuffleId, int mapIndex, int reduceId, ByteBuffer buffer) {
+      this.shuffleId = shuffleId;
+      this.mapIndex = mapIndex;
+      this.reduceId = reduceId;
+      this.buffer = buffer;
+    }
+  }
+
+  private static class TestMergeShuffleFile extends MergeShuffleFile {
+    private DataOutputStream activeDos;
+    private File file;
+    private FileChannel channel;
+
+    private TestMergeShuffleFile(File file) throws IOException {
+      super(null, null);
+      this.file = file;
+      FileOutputStream fos = new FileOutputStream(file);
+      channel = fos.getChannel();
+      activeDos = new DataOutputStream(fos);
+    }
+
+    @Override
+    DataOutputStream getDos() {
+      return activeDos;
+    }
+
+    @Override
+    FileChannel getChannel() {
+      return channel;
+    }
+
+    @Override
+    void close() throws IOException {
+      activeDos.close();
+    }
+
+    void restore() throws IOException {
+      FileOutputStream fos = new FileOutputStream(file, true);
+      channel = fos.getChannel();
+      activeDos = new DataOutputStream(fos);
+    }
+  }
+}
diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
index 0225db81925c5..5036e05b52fad 100644
--- a/common/network-yarn/pom.xml
+++ b/common/network-yarn/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
@@ -65,7 +65,13 @@
     <!-- Provided dependencies -->
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-client</artifactId>
+      <artifactId>${hadoop-client-api.artifact}</artifactId>
+      <version>${hadoop.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>${hadoop-client-runtime.artifact}</artifactId>
+      <version>${hadoop.version}</version>
     </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
diff --git a/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java
index 3d14318bf90f0..cb6d5d0ca2037 100644
--- a/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java
+++ b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java
@@ -19,6 +19,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.nio.ByteBuffer;
 import java.util.List;
@@ -41,6 +42,7 @@
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.server.api.*;
+import org.apache.spark.network.shuffle.MergedShuffleFileManager;
 import org.apache.spark.network.util.LevelDBProvider;
 import org.iq80.leveldb.DB;
 import org.iq80.leveldb.DBIterator;
@@ -74,6 +76,20 @@
  * is because an application running on the same Yarn cluster may choose to not use the external
  * shuffle service, in which case its setting of `spark.authenticate` should be independent of
  * the service's.
+ *
+ * The shuffle service will produce metrics via the YARN NodeManager's {@code metrics2} system
+ * under a namespace specified by the {@value SPARK_SHUFFLE_SERVICE_METRICS_NAMESPACE_KEY} config.
+ *
+ * By default, all configurations for the shuffle service will be taken directly from the
+ * Hadoop {@link Configuration} passed by the YARN NodeManager. It is also possible to configure
+ * the shuffle service by placing a resource named
+ * {@value SHUFFLE_SERVICE_CONF_OVERLAY_RESOURCE_NAME} into the classpath, which should be an
+ * XML file in the standard Hadoop Configuration resource format. Note that when the shuffle
+ * service is loaded in the default manner, without configuring
+ * {@code yarn.nodemanager.aux-services.<service>.classpath}, this file must be on the classpath
+ * of the NodeManager itself. When using the {@code classpath} configuration, it can be present
+ * either on the NodeManager's classpath, or specified in the classpath configuration.
+ * This {@code classpath} configuration is only supported on YARN versions >= 2.9.0.
  */
 public class YarnShuffleService extends AuxiliaryService {
   private static final Logger logger = LoggerFactory.getLogger(YarnShuffleService.class);
@@ -82,6 +98,14 @@ public class YarnShuffleService extends AuxiliaryService {
   private static final String SPARK_SHUFFLE_SERVICE_PORT_KEY = "spark.shuffle.service.port";
   private static final int DEFAULT_SPARK_SHUFFLE_SERVICE_PORT = 7337;
 
+  /**
+   * The namespace to use for the metrics record which will contain all metrics produced by the
+   * shuffle service.
+   */
+  static final String SPARK_SHUFFLE_SERVICE_METRICS_NAMESPACE_KEY =
+      "spark.yarn.shuffle.service.metrics.namespace";
+  private static final String DEFAULT_SPARK_SHUFFLE_SERVICE_METRICS_NAME = "sparkShuffleService";
+
   // Whether the shuffle server should authenticate fetch requests
   private static final String SPARK_AUTHENTICATE_KEY = "spark.authenticate";
   private static final boolean DEFAULT_SPARK_AUTHENTICATE = false;
@@ -102,6 +126,13 @@ public class YarnShuffleService extends AuxiliaryService {
   private static final LevelDBProvider.StoreVersion CURRENT_VERSION = new LevelDBProvider
       .StoreVersion(1, 0);
 
+  /**
+   * The name of the resource to search for on the classpath to find a shuffle service-specific
+   * configuration overlay. If found, this will be parsed as a standard Hadoop
+   * {@link Configuration config} file and will override the configs passed from the NodeManager.
+   */
+  static final String SHUFFLE_SERVICE_CONF_OVERLAY_RESOURCE_NAME = "spark-shuffle-site.xml";
+
   // just for integration tests that want to look at this file -- in general not sensible as
   // a static
   @VisibleForTesting
@@ -138,6 +169,13 @@ public class YarnShuffleService extends AuxiliaryService {
   private DB db;
 
   public YarnShuffleService() {
+    // The name of the auxiliary service configured within the NodeManager
+    // (`yarn.nodemanager.aux-services`) is treated as the source-of-truth, so this one can be
+    // arbitrary. The NodeManager will log a warning if the configured name doesn't match this name,
+    // to inform operators of a potential misconfiguration, but this name is otherwise not used.
+    // It is hard-coded instead of using the value of the `spark.shuffle.service.name` configuration
+    // because at this point in instantiation there is no Configuration object; it is not passed
+    // until `serviceInit` is called, at which point it's too late to adjust the name.
     super("spark_shuffle");
     logger.info("Initializing YARN shuffle service for Spark");
     instance = this;
@@ -156,10 +194,18 @@ private boolean isAuthenticationEnabled() {
    * Start the shuffle server with the given configuration.
    */
   @Override
-  protected void serviceInit(Configuration conf) throws Exception {
-    _conf = conf;
+  protected void serviceInit(Configuration externalConf) throws Exception {
+    _conf = new Configuration(externalConf);
+    URL confOverlayUrl = Thread.currentThread().getContextClassLoader()
+        .getResource(SHUFFLE_SERVICE_CONF_OVERLAY_RESOURCE_NAME);
+    if (confOverlayUrl != null) {
+      logger.info("Initializing Spark YARN shuffle service with configuration overlay from {}",
+          confOverlayUrl);
+      _conf.addResource(confOverlayUrl);
+    }
+    super.serviceInit(_conf);
 
-    boolean stopOnFailure = conf.getBoolean(STOP_ON_FAILURE_KEY, DEFAULT_STOP_ON_FAILURE);
+    boolean stopOnFailure = _conf.getBoolean(STOP_ON_FAILURE_KEY, DEFAULT_STOP_ON_FAILURE);
 
     try {
       // In case this NM was killed while there were running spark applications, we need to restore
@@ -171,13 +217,16 @@ protected void serviceInit(Configuration conf) throws Exception {
         registeredExecutorFile = initRecoveryDb(RECOVERY_FILE_NAME);
       }
 
-      TransportConf transportConf = new TransportConf("shuffle", new HadoopConfigProvider(conf));
-      blockHandler = new ExternalBlockHandler(transportConf, registeredExecutorFile);
+      TransportConf transportConf = new TransportConf("shuffle", new HadoopConfigProvider(_conf));
+      MergedShuffleFileManager shuffleMergeManager = newMergedShuffleFileManagerInstance(
+        transportConf);
+      blockHandler = new ExternalBlockHandler(
+        transportConf, registeredExecutorFile, shuffleMergeManager);
 
       // If authentication is enabled, set up the shuffle server to use a
       // special RPC handler that filters out unauthenticated fetch requests
       List<TransportServerBootstrap> bootstraps = Lists.newArrayList();
-      boolean authEnabled = conf.getBoolean(SPARK_AUTHENTICATE_KEY, DEFAULT_SPARK_AUTHENTICATE);
+      boolean authEnabled = _conf.getBoolean(SPARK_AUTHENTICATE_KEY, DEFAULT_SPARK_AUTHENTICATE);
       if (authEnabled) {
         secretManager = new ShuffleSecretManager();
         if (_recoveryPath != null) {
@@ -186,7 +235,7 @@ protected void serviceInit(Configuration conf) throws Exception {
         bootstraps.add(new AuthServerBootstrap(transportConf, secretManager));
       }
 
-      int port = conf.getInt(
+      int port = _conf.getInt(
         SPARK_SHUFFLE_SERVICE_PORT_KEY, DEFAULT_SPARK_SHUFFLE_SERVICE_PORT);
       transportContext = new TransportContext(transportConf, blockHandler, true);
       shuffleServer = transportContext.createServer(port, bootstraps);
@@ -199,13 +248,16 @@ protected void serviceInit(Configuration conf) throws Exception {
       blockHandler.getAllMetrics().getMetrics().put("numRegisteredConnections",
           shuffleServer.getRegisteredConnections());
       blockHandler.getAllMetrics().getMetrics().putAll(shuffleServer.getAllMetrics().getMetrics());
+      String metricsNamespace = _conf.get(SPARK_SHUFFLE_SERVICE_METRICS_NAMESPACE_KEY,
+          DEFAULT_SPARK_SHUFFLE_SERVICE_METRICS_NAME);
       YarnShuffleServiceMetrics serviceMetrics =
-          new YarnShuffleServiceMetrics(blockHandler.getAllMetrics());
+          new YarnShuffleServiceMetrics(metricsNamespace, blockHandler.getAllMetrics());
 
       MetricsSystemImpl metricsSystem = (MetricsSystemImpl) DefaultMetricsSystem.instance();
       metricsSystem.register(
-          "sparkShuffleService", "Metrics on the Spark Shuffle Service", serviceMetrics);
-      logger.info("Registered metrics with Hadoop's DefaultMetricsSystem");
+          metricsNamespace, "Metrics on the Spark Shuffle Service", serviceMetrics);
+      logger.info("Registered metrics with Hadoop's DefaultMetricsSystem using namespace '{}'",
+          metricsNamespace);
 
       logger.info("Started YARN shuffle service for Spark on port {}. " +
         "Authentication is {}.  Registered executor file is {}", port, authEnabledString,
@@ -219,6 +271,23 @@ protected void serviceInit(Configuration conf) throws Exception {
     }
   }
 
+  @VisibleForTesting
+  static MergedShuffleFileManager newMergedShuffleFileManagerInstance(TransportConf conf) {
+    String mergeManagerImplClassName = conf.mergedShuffleFileManagerImpl();
+    try {
+      Class<?> mergeManagerImplClazz = Class.forName(
+        mergeManagerImplClassName, true, Thread.currentThread().getContextClassLoader());
+      Class<? extends MergedShuffleFileManager> mergeManagerSubClazz =
+        mergeManagerImplClazz.asSubclass(MergedShuffleFileManager.class);
+      // The assumption is that all the custom implementations just like the RemoteBlockPushResolver
+      // will also need the transport configuration.
+      return mergeManagerSubClazz.getConstructor(TransportConf.class).newInstance(conf);
+    } catch (Exception e) {
+      logger.error("Unable to create an instance of {}", mergeManagerImplClassName);
+      return new ExternalBlockHandler.NoOpMergedShuffleFileManager(conf);
+    }
+  }
+
   private void loadSecretsFromDb() throws IOException {
     secretsFile = initRecoveryDb(SECRETS_RECOVERY_FILE_NAME);
 
diff --git a/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleServiceMetrics.java b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleServiceMetrics.java
index 81be6e8036ffe..f30abbd0f7fcd 100644
--- a/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleServiceMetrics.java
+++ b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleServiceMetrics.java
@@ -32,9 +32,11 @@
  */
 class YarnShuffleServiceMetrics implements MetricsSource {
 
+  private final String metricsNamespace;
   private final MetricSet metricSet;
 
-  YarnShuffleServiceMetrics(MetricSet metricSet) {
+  YarnShuffleServiceMetrics(String metricsNamespace, MetricSet metricSet) {
+    this.metricsNamespace = metricsNamespace;
     this.metricSet = metricSet;
   }
 
@@ -46,7 +48,7 @@ class YarnShuffleServiceMetrics implements MetricsSource {
    */
   @Override
   public void getMetrics(MetricsCollector collector, boolean all) {
-    MetricsRecordBuilder metricsRecordBuilder = collector.addRecord("sparkShuffleService");
+    MetricsRecordBuilder metricsRecordBuilder = collector.addRecord(metricsNamespace);
 
     for (Map.Entry<String, Metric> entry : metricSet.getMetrics().entrySet()) {
       collectMetric(metricsRecordBuilder, entry.getKey(), entry.getValue());
diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml
index 72a2c4ceb43b6..b5a6775366a47 100644
--- a/common/sketch/pom.xml
+++ b/common/sketch/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/sketch/src/main/java/org/apache/spark/util/sketch/Murmur3_x86_32.java b/common/sketch/src/main/java/org/apache/spark/util/sketch/Murmur3_x86_32.java
index e83b331391e39..61cd2cec1a34b 100644
--- a/common/sketch/src/main/java/org/apache/spark/util/sketch/Murmur3_x86_32.java
+++ b/common/sketch/src/main/java/org/apache/spark/util/sketch/Murmur3_x86_32.java
@@ -17,12 +17,16 @@
 
 package org.apache.spark.util.sketch;
 
+import java.nio.ByteOrder;
+
 /**
  * 32-bit Murmur3 hasher.  This is based on Guava's Murmur3_32HashFunction.
  */
 // This class is duplicated from `org.apache.spark.unsafe.hash.Murmur3_x86_32` to make sure
 // spark-sketch has no external dependencies.
 final class Murmur3_x86_32 {
+  private static final boolean isBigEndian = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN);
+
   private static final int C1 = 0xcc9e2d51;
   private static final int C2 = 0x1b873593;
 
@@ -92,8 +96,10 @@ private static int hashBytesByInt(Object base, long offset, int lengthInBytes, i
     int h1 = seed;
     for (int i = 0; i < lengthInBytes; i += 4) {
       int halfWord = Platform.getInt(base, offset + i);
-      int k1 = mixK1(halfWord);
-      h1 = mixH1(h1, k1);
+      if (isBigEndian) {
+        halfWord = Integer.reverseBytes(halfWord);
+      }
+      h1 = mixH1(h1, mixK1(halfWord));
     }
     return h1;
   }
diff --git a/common/sketch/src/test/scala/org/apache/spark/util/sketch/BitArraySuite.scala b/common/sketch/src/test/scala/org/apache/spark/util/sketch/BitArraySuite.scala
index ff728f0ebcb85..4c535a8dd0411 100644
--- a/common/sketch/src/test/scala/org/apache/spark/util/sketch/BitArraySuite.scala
+++ b/common/sketch/src/test/scala/org/apache/spark/util/sketch/BitArraySuite.scala
@@ -19,9 +19,9 @@ package org.apache.spark.util.sketch
 
 import scala.util.Random
 
-import org.scalatest.FunSuite // scalastyle:ignore funsuite
+import org.scalatest.funsuite.AnyFunSuite // scalastyle:ignore funsuite
 
-class BitArraySuite extends FunSuite { // scalastyle:ignore funsuite
+class BitArraySuite extends AnyFunSuite { // scalastyle:ignore funsuite
 
   test("error case when create BitArray") {
     intercept[IllegalArgumentException](new BitArray(0))
diff --git a/common/sketch/src/test/scala/org/apache/spark/util/sketch/BloomFilterSuite.scala b/common/sketch/src/test/scala/org/apache/spark/util/sketch/BloomFilterSuite.scala
index a0408d2da4dff..8b289fc86af0f 100644
--- a/common/sketch/src/test/scala/org/apache/spark/util/sketch/BloomFilterSuite.scala
+++ b/common/sketch/src/test/scala/org/apache/spark/util/sketch/BloomFilterSuite.scala
@@ -22,9 +22,9 @@ import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
 import scala.reflect.ClassTag
 import scala.util.Random
 
-import org.scalatest.FunSuite // scalastyle:ignore funsuite
+import org.scalatest.funsuite.AnyFunSuite // scalastyle:ignore funsuite
 
-class BloomFilterSuite extends FunSuite { // scalastyle:ignore funsuite
+class BloomFilterSuite extends AnyFunSuite { // scalastyle:ignore funsuite
   private final val EPSILON = 0.01
 
   // Serializes and deserializes a given `BloomFilter`, then checks whether the deserialized
diff --git a/common/sketch/src/test/scala/org/apache/spark/util/sketch/CountMinSketchSuite.scala b/common/sketch/src/test/scala/org/apache/spark/util/sketch/CountMinSketchSuite.scala
index 174eb01986c4f..087dae26047ef 100644
--- a/common/sketch/src/test/scala/org/apache/spark/util/sketch/CountMinSketchSuite.scala
+++ b/common/sketch/src/test/scala/org/apache/spark/util/sketch/CountMinSketchSuite.scala
@@ -22,9 +22,9 @@ import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
 import scala.reflect.ClassTag
 import scala.util.Random
 
-import org.scalatest.FunSuite // scalastyle:ignore funsuite
+import org.scalatest.funsuite.AnyFunSuite // scalastyle:ignore funsuite
 
-class CountMinSketchSuite extends FunSuite { // scalastyle:ignore funsuite
+class CountMinSketchSuite extends AnyFunSuite { // scalastyle:ignore funsuite
   private val epsOfTotalCount = 0.01
 
   private val confidence = 0.9
diff --git a/common/tags/pom.xml b/common/tags/pom.xml
index ea16dadca40cb..e51357d97faab 100644
--- a/common/tags/pom.xml
+++ b/common/tags/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/tags/src/test/java/org/apache/spark/tags/ChromeUITest.java b/common/tags/src/test/java/org/apache/spark/tags/ChromeUITest.java
new file mode 100644
index 0000000000000..e3fed3d656d20
--- /dev/null
+++ b/common/tags/src/test/java/org/apache/spark/tags/ChromeUITest.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.tags;
+
+import java.lang.annotation.*;
+
+import org.scalatest.TagAnnotation;
+
+@TagAnnotation
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.METHOD, ElementType.TYPE})
+public @interface ChromeUITest { }
diff --git a/common/tags/src/test/java/org/apache/spark/tags/SlowHiveTest.java b/common/tags/src/test/java/org/apache/spark/tags/SlowHiveTest.java
new file mode 100644
index 0000000000000..a7e6f352667d7
--- /dev/null
+++ b/common/tags/src/test/java/org/apache/spark/tags/SlowHiveTest.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.tags;
+
+import org.scalatest.TagAnnotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@TagAnnotation
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.METHOD, ElementType.TYPE})
+public @interface SlowHiveTest { }
diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
index 769e2518b1fd4..b22400575dd02 100644
--- a/common/unsafe/pom.xml
+++ b/common/unsafe/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/hash/Murmur3_x86_32.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/hash/Murmur3_x86_32.java
index d239de6083ad0..0b9d9ced312a1 100644
--- a/common/unsafe/src/main/java/org/apache/spark/unsafe/hash/Murmur3_x86_32.java
+++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/hash/Murmur3_x86_32.java
@@ -17,12 +17,16 @@
 
 package org.apache.spark.unsafe.hash;
 
+import java.nio.ByteOrder;
+
 import org.apache.spark.unsafe.Platform;
 
 /**
  * 32-bit Murmur3 hasher.  This is based on Guava's Murmur3_32HashFunction.
  */
 public final class Murmur3_x86_32 {
+  private static final boolean isBigEndian = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN);
+
   private static final int C1 = 0xcc9e2d51;
   private static final int C2 = 0x1b873593;
 
@@ -92,8 +96,10 @@ private static int hashBytesByInt(Object base, long offset, int lengthInBytes, i
     int h1 = seed;
     for (int i = 0; i < lengthInBytes; i += 4) {
       int halfWord = Platform.getInt(base, offset + i);
-      int k1 = mixK1(halfWord);
-      h1 = mixH1(h1, k1);
+      if (isBigEndian) {
+        halfWord = Integer.reverseBytes(halfWord);
+      }
+      h1 = mixH1(h1, mixK1(halfWord));
     }
     return h1;
   }
diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
index 186597fa64780..db52f77481761 100644
--- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
+++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
@@ -29,7 +29,6 @@
 import com.esotericsoftware.kryo.KryoSerializable;
 import com.esotericsoftware.kryo.io.Input;
 import com.esotericsoftware.kryo.io.Output;
-import com.google.common.primitives.Ints;
 
 import org.apache.spark.unsafe.Platform;
 import org.apache.spark.unsafe.UTF8StringBuilder;
@@ -341,8 +340,17 @@ public UTF8String substringSQL(int pos, int length) {
     // to the -ith element before the end of the sequence. If a start index i is 0, it
     // refers to the first element.
     int len = numChars();
+    // `len + pos` does not overflow as `len >= 0`.
     int start = (pos > 0) ? pos -1 : ((pos < 0) ? len + pos : 0);
-    int end = (length == Integer.MAX_VALUE) ? len : start + length;
+
+    int end;
+    if ((long) start + length > Integer.MAX_VALUE) {
+      end = Integer.MAX_VALUE;
+    } else if ((long) start + length < Integer.MIN_VALUE) {
+      end = Integer.MIN_VALUE;
+    } else {
+      end = start + length;
+    }
     return substring(start, end);
   }
 
@@ -554,7 +562,7 @@ public UTF8String trim() {
   }
 
   /**
-   * Trims whitespaces (<= ASCII 32) from both ends of this string.
+   * Trims whitespaces ({@literal <=} ASCII 32) from both ends of this string.
    *
    * Note that, this method is the same as java's {@link String#trim}, and different from
    * {@link UTF8String#trim()} which remove only spaces(= ASCII 32) from both ends.
@@ -566,14 +574,14 @@ public UTF8String trim() {
   public UTF8String trimAll() {
     int s = 0;
     // skip all of the whitespaces (<=0x20) in the left side
-    while (s < this.numBytes && getByte(s) <= ' ') s++;
+    while (s < this.numBytes && Character.isWhitespace(getByte(s))) s++;
     if (s == this.numBytes) {
       // Everything trimmed
       return EMPTY_UTF8;
     }
     // skip all of the whitespaces (<=0x20) in the right side
     int e = this.numBytes - 1;
-    while (e > s && getByte(e) <= ' ') e--;
+    while (e > s && Character.isWhitespace(getByte(e))) e--;
     if (s == 0 && e == numBytes - 1) {
       // Nothing trimmed
       return this;
@@ -626,13 +634,13 @@ public UTF8String trimLeft() {
   public UTF8String trimLeft(UTF8String trimString) {
     if (trimString == null) return null;
     // the searching byte position in the source string
-    int srchIdx = 0;
+    int searchIdx = 0;
     // the first beginning byte position of a non-matching character
     int trimIdx = 0;
 
-    while (srchIdx < numBytes) {
+    while (searchIdx < numBytes) {
       UTF8String searchChar = copyUTF8String(
-          srchIdx, srchIdx + numBytesForFirstByte(this.getByte(srchIdx)) - 1);
+          searchIdx, searchIdx + numBytesForFirstByte(this.getByte(searchIdx)) - 1);
       int searchCharBytes = searchChar.numBytes;
       // try to find the matching for the searchChar in the trimString set
       if (trimString.find(searchChar, 0) >= 0) {
@@ -641,9 +649,9 @@ public UTF8String trimLeft(UTF8String trimString) {
         // no matching, exit the search
         break;
       }
-      srchIdx += searchCharBytes;
+      searchIdx += searchCharBytes;
     }
-    if (srchIdx == 0) {
+    if (searchIdx == 0) {
       // Nothing trimmed
       return this;
     }
@@ -674,6 +682,17 @@ public UTF8String trimRight() {
     return copyUTF8String(0, e);
   }
 
+  /**
+   * Trims at most `numSpaces` space characters (ASCII 32) from the end of this string.
+   */
+  public UTF8String trimTrailingSpaces(int numSpaces) {
+    assert numSpaces > 0;
+    int endIdx = numBytes - 1;
+    int trimTo = numBytes - numSpaces;
+    while (endIdx >= trimTo && getByte(endIdx) == 0x20) endIdx--;
+    return copyUTF8String(0, endIdx);
+  }
+
   /**
    * Trims instances of the given trim string from the end of this string.
    *
@@ -744,7 +763,7 @@ public UTF8String repeat(int times) {
       return EMPTY_UTF8;
     }
 
-    byte[] newBytes = new byte[numBytes * times];
+    byte[] newBytes = new byte[Math.multiplyExact(numBytes, times)];
     copyMemory(this.base, this.offset, newBytes, BYTE_ARRAY_OFFSET, numBytes);
 
     int copied = 1;
@@ -887,7 +906,8 @@ public UTF8String rpad(int len, UTF8String pad) {
       // the partial string of the padding
       UTF8String remain = pad.substring(0, spaces - padChars * count);
 
-      byte[] data = new byte[this.numBytes + pad.numBytes * count + remain.numBytes];
+      int resultSize = Math.toIntExact((long)numBytes + pad.numBytes * count + remain.numBytes);
+      byte[] data = new byte[resultSize];
       copyMemory(this.base, this.offset, data, BYTE_ARRAY_OFFSET, this.numBytes);
       int offset = this.numBytes;
       int idx = 0;
@@ -919,7 +939,8 @@ public UTF8String lpad(int len, UTF8String pad) {
       // the partial string of the padding
       UTF8String remain = pad.substring(0, spaces - padChars * count);
 
-      byte[] data = new byte[this.numBytes + pad.numBytes * count + remain.numBytes];
+      int resultSize = Math.toIntExact((long)numBytes + pad.numBytes * count + remain.numBytes);
+      byte[] data = new byte[resultSize];
 
       int offset = 0;
       int idx = 0;
@@ -951,7 +972,7 @@ public static UTF8String concat(UTF8String... inputs) {
     }
 
     // Allocate a new byte array, and copy the inputs one by one into it.
-    final byte[] result = new byte[Ints.checkedCast(totalLength)];
+    final byte[] result = new byte[Math.toIntExact(totalLength)];
     int offset = 0;
     for (int i = 0; i < inputs.length; i++) {
       int len = inputs[i].numBytes;
@@ -973,7 +994,7 @@ public static UTF8String concatWs(UTF8String separator, UTF8String... inputs) {
       return null;
     }
 
-    int numInputBytes = 0;  // total number of bytes from the inputs
+    long numInputBytes = 0L;  // total number of bytes from the inputs
     int numInputs = 0;      // number of non-null inputs
     for (int i = 0; i < inputs.length; i++) {
       if (inputs[i] != null) {
@@ -989,7 +1010,8 @@ public static UTF8String concatWs(UTF8String separator, UTF8String... inputs) {
 
     // Allocate a new byte array, and copy the inputs one by one into it.
     // The size of the new array is the size of all inputs, plus the separators.
-    final byte[] result = new byte[numInputBytes + (numInputs - 1) * separator.numBytes];
+    int resultSize = Math.toIntExact(numInputBytes + (numInputs - 1) * (long)separator.numBytes);
+    final byte[] result = new byte[resultSize];
     int offset = 0;
 
     for (int i = 0, j = 0; i < inputs.length; i++) {
@@ -1056,16 +1078,20 @@ public UTF8String replace(UTF8String search, UTF8String replace) {
     return buf.build();
   }
 
-  // TODO: Need to use `Code Point` here instead of Char in case the character longer than 2 bytes
-  public UTF8String translate(Map<Character, Character> dict) {
+  public UTF8String translate(Map<String, String> dict) {
     String srcStr = this.toString();
 
     StringBuilder sb = new StringBuilder();
-    for(int k = 0; k< srcStr.length(); k++) {
-      if (null == dict.get(srcStr.charAt(k))) {
-        sb.append(srcStr.charAt(k));
-      } else if ('\0' != dict.get(srcStr.charAt(k))){
-        sb.append(dict.get(srcStr.charAt(k)));
+    int charCount = 0;
+    for (int k = 0; k < srcStr.length(); k += charCount) {
+      int codePoint = srcStr.codePointAt(k);
+      charCount = Character.charCount(codePoint);
+      String subStr = srcStr.substring(k, k + charCount);
+      String translated = dict.get(subStr);
+      if (null == translated) {
+        sb.append(subStr);
+      } else if (!"\0".equals(translated)) {
+        sb.append(translated);
       }
     }
     return fromString(sb.toString());
@@ -1110,11 +1136,11 @@ public boolean toLong(LongWrapper toLongResult) {
 
   private boolean toLong(LongWrapper toLongResult, boolean allowDecimal) {
     int offset = 0;
-    while (offset < this.numBytes && getByte(offset) <= ' ') offset++;
+    while (offset < this.numBytes && Character.isWhitespace(getByte(offset))) offset++;
     if (offset == this.numBytes) return false;
 
     int end = this.numBytes - 1;
-    while (end > offset && getByte(end) <= ' ') end--;
+    while (end > offset && Character.isWhitespace(getByte(end))) end--;
 
     byte b = getByte(offset);
     final boolean negative = b == '-';
@@ -1207,11 +1233,11 @@ public boolean toInt(IntWrapper intWrapper) {
 
   private boolean toInt(IntWrapper intWrapper, boolean allowDecimal) {
     int offset = 0;
-    while (offset < this.numBytes && getByte(offset) <= ' ') offset++;
+    while (offset < this.numBytes && Character.isWhitespace(getByte(offset))) offset++;
     if (offset == this.numBytes) return false;
 
     int end = this.numBytes - 1;
-    while (end > offset && getByte(end) <= ' ') end--;
+    while (end > offset && Character.isWhitespace(getByte(end))) end--;
 
     byte b = getByte(offset);
     final boolean negative = b == '-';
diff --git a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java
index 8f933877f82e6..ba3e4269e9a46 100644
--- a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java
+++ b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java
@@ -390,6 +390,10 @@ public void substringSQL() {
     assertEquals(fromString("example"), e.substringSQL(0, Integer.MAX_VALUE));
     assertEquals(fromString("example"), e.substringSQL(1, Integer.MAX_VALUE));
     assertEquals(fromString("xample"), e.substringSQL(2, Integer.MAX_VALUE));
+    assertEquals(EMPTY_UTF8, e.substringSQL(-100, -100));
+    assertEquals(EMPTY_UTF8, e.substringSQL(-1207959552, -1207959552));
+    assertEquals(fromString("pl"), e.substringSQL(-3, 2));
+    assertEquals(EMPTY_UTF8, e.substringSQL(Integer.MIN_VALUE, 6));
   }
 
   @Test
@@ -461,10 +465,10 @@ public void translate() {
     assertEquals(
       fromString("1a2s3ae"),
       fromString("translate").translate(ImmutableMap.of(
-        'r', '1',
-        'n', '2',
-        'l', '3',
-        't', '\0'
+        "r", "1",
+        "n", "2",
+        "l", "3",
+        "t", "\0"
       )));
     assertEquals(
       fromString("translate"),
@@ -472,16 +476,16 @@ public void translate() {
     assertEquals(
       fromString("asae"),
       fromString("translate").translate(ImmutableMap.of(
-        'r', '\0',
-        'n', '\0',
-        'l', '\0',
-        't', '\0'
+        "r", "\0",
+        "n", "\0",
+        "l", "\0",
+        "t", "\0"
       )));
     assertEquals(
       fromString("aa世b"),
       fromString("花花世界").translate(ImmutableMap.of(
-        '花', 'a',
-        '界', 'b'
+        "花", "a",
+        "界", "b"
       )));
   }
 
diff --git a/common/unsafe/src/test/scala/org/apache/spark/unsafe/types/UTF8StringPropertyCheckSuite.scala b/common/unsafe/src/test/scala/org/apache/spark/unsafe/types/UTF8StringPropertyCheckSuite.scala
index 72aa682bb95bc..ab488e18ba3f4 100644
--- a/common/unsafe/src/test/scala/org/apache/spark/unsafe/types/UTF8StringPropertyCheckSuite.scala
+++ b/common/unsafe/src/test/scala/org/apache/spark/unsafe/types/UTF8StringPropertyCheckSuite.scala
@@ -21,14 +21,15 @@ import org.apache.commons.text.similarity.LevenshteinDistance
 import org.scalacheck.{Arbitrary, Gen}
 import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks
 // scalastyle:off
-import org.scalatest.{FunSuite, Matchers}
+import org.scalatest.funsuite.AnyFunSuite
+import org.scalatest.matchers.must.Matchers
 
 import org.apache.spark.unsafe.types.UTF8String.{fromString => toUTF8}
 
 /**
  * This TestSuite utilize ScalaCheck to generate randomized inputs for UTF8String testing.
  */
-class UTF8StringPropertyCheckSuite extends FunSuite with ScalaCheckDrivenPropertyChecks with Matchers {
+class UTF8StringPropertyCheckSuite extends AnyFunSuite with ScalaCheckDrivenPropertyChecks with Matchers {
 // scalastyle:on
 
   test("toString") {
@@ -191,7 +192,7 @@ class UTF8StringPropertyCheckSuite extends FunSuite with ScalaCheckDrivenPropert
     }
   }
 
-  val nullalbeSeq = Gen.listOf(Gen.oneOf[String](null: String, randomString))
+  val nullableSeq = Gen.listOf(Gen.oneOf[String](null: String, randomString))
 
   test("concat") {
     def concat(origin: Seq[String]): String =
@@ -200,7 +201,7 @@ class UTF8StringPropertyCheckSuite extends FunSuite with ScalaCheckDrivenPropert
     forAll { (inputs: Seq[String]) =>
       assert(UTF8String.concat(inputs.map(toUTF8): _*) === toUTF8(inputs.mkString))
     }
-    forAll (nullalbeSeq) { (inputs: Seq[String]) =>
+    forAll (nullableSeq) { (inputs: Seq[String]) =>
       assert(UTF8String.concat(inputs.map(toUTF8): _*) === toUTF8(concat(inputs)))
     }
   }
@@ -215,7 +216,7 @@ class UTF8StringPropertyCheckSuite extends FunSuite with ScalaCheckDrivenPropert
       assert(UTF8String.concatWs(toUTF8(sep), inputs.map(toUTF8): _*) ===
         toUTF8(inputs.mkString(sep)))
     }
-    forAll(randomString, nullalbeSeq) {(sep: String, inputs: Seq[String]) =>
+    forAll(randomString, nullableSeq) {(sep: String, inputs: Seq[String]) =>
       assert(UTF8String.concatWs(toUTF8(sep), inputs.map(toUTF8): _*) ===
         toUTF8(concatWs(sep, inputs)))
     }
diff --git a/conf/log4j.properties.template b/conf/log4j.properties.template
index e91595dd324b0..5db8c5c295d84 100644
--- a/conf/log4j.properties.template
+++ b/conf/log4j.properties.template
@@ -22,10 +22,12 @@ log4j.appender.console.target=System.err
 log4j.appender.console.layout=org.apache.log4j.PatternLayout
 log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
 
-# Set the default spark-shell log level to WARN. When running the spark-shell, the
-# log level for this class is used to overwrite the root logger's log level, so that
-# the user can have different defaults for the shell and regular Spark apps.
+# Set the default spark-shell/spark-sql log level to WARN. When running the
+# spark-shell/spark-sql, the log level for these classes is used to overwrite
+# the root logger's log level, so that the user can have different defaults
+# for the shell and regular Spark apps.
 log4j.logger.org.apache.spark.repl.Main=WARN
+log4j.logger.org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver=WARN
 
 # Settings to quiet third party logs that are too verbose
 log4j.logger.org.sparkproject.jetty=WARN
@@ -38,3 +40,9 @@ log4j.logger.parquet=ERROR
 # SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support
 log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL
 log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR
+
+# For deploying Spark ThriftServer
+# SPARK-34128：Suppress undesirable TTransportException warnings involved in THRIFT-4805
+log4j.appender.console.filter.1=org.apache.log4j.varia.StringMatchFilter
+log4j.appender.console.filter.1.StringToMatch=Thrift error occurred during processing of message
+log4j.appender.console.filter.1.AcceptOnMatch=false
diff --git a/conf/spark-env.sh.template b/conf/spark-env.sh.template
index df39ad8b0dcc2..3c003f45ed27a 100755
--- a/conf/spark-env.sh.template
+++ b/conf/spark-env.sh.template
@@ -62,6 +62,7 @@
 # Generic options for the daemons used in the standalone deploy mode
 # - SPARK_CONF_DIR      Alternate conf dir. (Default: ${SPARK_HOME}/conf)
 # - SPARK_LOG_DIR       Where log files are stored.  (Default: ${SPARK_HOME}/logs)
+# - SPARK_LOG_MAX_FILES Max log files of Spark daemons can rotate to. Default is 5.
 # - SPARK_PID_DIR       Where the pid file is stored. (Default: /tmp)
 # - SPARK_IDENT_STRING  A string representing this instance of spark. (Default: $USER)
 # - SPARK_NICENESS      The scheduling priority for daemons. (Default: 0)
diff --git a/conf/slaves.template b/conf/workers.template
similarity index 100%
rename from conf/slaves.template
rename to conf/workers.template
diff --git a/core/benchmarks/CoalescedRDDBenchmark-jdk11-results.txt b/core/benchmarks/CoalescedRDDBenchmark-jdk11-results.txt
index e944111ff9e93..10ac28b46091c 100644
--- a/core/benchmarks/CoalescedRDDBenchmark-jdk11-results.txt
+++ b/core/benchmarks/CoalescedRDDBenchmark-jdk11-results.txt
@@ -2,39 +2,39 @@
 Coalesced RDD , large scale
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Coalesced RDD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Coalesce Num Partitions: 100 Num Hosts: 1            344            360          14          0.3        3441.4       1.0X
-Coalesce Num Partitions: 100 Num Hosts: 5            283            301          22          0.4        2825.1       1.2X
-Coalesce Num Partitions: 100 Num Hosts: 10            270            271           2          0.4        2700.5       1.3X
-Coalesce Num Partitions: 100 Num Hosts: 20            272            273           1          0.4        2721.1       1.3X
-Coalesce Num Partitions: 100 Num Hosts: 40            271            272           1          0.4        2710.0       1.3X
-Coalesce Num Partitions: 100 Num Hosts: 80            266            267           2          0.4        2656.3       1.3X
-Coalesce Num Partitions: 500 Num Hosts: 1            609            619          15          0.2        6089.0       0.6X
-Coalesce Num Partitions: 500 Num Hosts: 5            338            343           6          0.3        3383.0       1.0X
-Coalesce Num Partitions: 500 Num Hosts: 10            303            306           3          0.3        3029.4       1.1X
-Coalesce Num Partitions: 500 Num Hosts: 20            286            288           2          0.4        2855.9       1.2X
-Coalesce Num Partitions: 500 Num Hosts: 40            279            282           4          0.4        2793.3       1.2X
-Coalesce Num Partitions: 500 Num Hosts: 80            273            275           3          0.4        2725.9       1.3X
-Coalesce Num Partitions: 1000 Num Hosts: 1            951            955           4          0.1        9514.1       0.4X
-Coalesce Num Partitions: 1000 Num Hosts: 5            421            429           8          0.2        4211.3       0.8X
-Coalesce Num Partitions: 1000 Num Hosts: 10            347            352           4          0.3        3473.5       1.0X
-Coalesce Num Partitions: 1000 Num Hosts: 20            309            312           5          0.3        3087.5       1.1X
-Coalesce Num Partitions: 1000 Num Hosts: 40            290            294           6          0.3        2896.4       1.2X
-Coalesce Num Partitions: 1000 Num Hosts: 80            281            286           5          0.4        2811.3       1.2X
-Coalesce Num Partitions: 5000 Num Hosts: 1           3928           3950          27          0.0       39278.0       0.1X
-Coalesce Num Partitions: 5000 Num Hosts: 5           1373           1389          27          0.1       13725.2       0.3X
-Coalesce Num Partitions: 5000 Num Hosts: 10            812            827          13          0.1        8123.3       0.4X
-Coalesce Num Partitions: 5000 Num Hosts: 20            530            540           9          0.2        5299.1       0.6X
-Coalesce Num Partitions: 5000 Num Hosts: 40            421            425           5          0.2        4210.5       0.8X
-Coalesce Num Partitions: 5000 Num Hosts: 80            335            344          12          0.3        3353.7       1.0X
-Coalesce Num Partitions: 10000 Num Hosts: 1           7116           7120           4          0.0       71159.0       0.0X
-Coalesce Num Partitions: 10000 Num Hosts: 5           2539           2598          51          0.0       25390.1       0.1X
-Coalesce Num Partitions: 10000 Num Hosts: 10           1393           1432          34          0.1       13928.1       0.2X
-Coalesce Num Partitions: 10000 Num Hosts: 20            833           1009         303          0.1        8329.2       0.4X
-Coalesce Num Partitions: 10000 Num Hosts: 40            562            563           3          0.2        5615.2       0.6X
-Coalesce Num Partitions: 10000 Num Hosts: 80            420            426           7          0.2        4204.0       0.8X
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+Coalesced RDD:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+----------------------------------------------------------------------------------------------------------------------------
+Coalesce Num Partitions: 100 Num Hosts: 1               365            454         104          0.3        3652.8       1.0X
+Coalesce Num Partitions: 100 Num Hosts: 5               271            374          90          0.4        2708.0       1.3X
+Coalesce Num Partitions: 100 Num Hosts: 10              265            337          88          0.4        2651.9       1.4X
+Coalesce Num Partitions: 100 Num Hosts: 20              277            338          87          0.4        2772.8       1.3X
+Coalesce Num Partitions: 100 Num Hosts: 40              313            320           6          0.3        3127.8       1.2X
+Coalesce Num Partitions: 100 Num Hosts: 80              291            330          60          0.3        2909.5       1.3X
+Coalesce Num Partitions: 500 Num Hosts: 1               974           1035          59          0.1        9741.6       0.4X
+Coalesce Num Partitions: 500 Num Hosts: 5               443            450           6          0.2        4427.7       0.8X
+Coalesce Num Partitions: 500 Num Hosts: 10              382            455          64          0.3        3816.6       1.0X
+Coalesce Num Partitions: 500 Num Hosts: 20              359            402          39          0.3        3589.9       1.0X
+Coalesce Num Partitions: 500 Num Hosts: 40              317            350          54          0.3        3165.5       1.2X
+Coalesce Num Partitions: 500 Num Hosts: 80              267            337          81          0.4        2668.5       1.4X
+Coalesce Num Partitions: 1000 Num Hosts: 1             1621           1712          84          0.1       16205.9       0.2X
+Coalesce Num Partitions: 1000 Num Hosts: 5              582            638          85          0.2        5817.7       0.6X
+Coalesce Num Partitions: 1000 Num Hosts: 10             398            446          74          0.3        3980.8       0.9X
+Coalesce Num Partitions: 1000 Num Hosts: 20             332            410         108          0.3        3315.1       1.1X
+Coalesce Num Partitions: 1000 Num Hosts: 40             345            377          54          0.3        3453.1       1.1X
+Coalesce Num Partitions: 1000 Num Hosts: 80             294            353          69          0.3        2940.5       1.2X
+Coalesce Num Partitions: 5000 Num Hosts: 1             6818           6906         151          0.0       68183.8       0.1X
+Coalesce Num Partitions: 5000 Num Hosts: 5             1606           1719         138          0.1       16061.1       0.2X
+Coalesce Num Partitions: 5000 Num Hosts: 10             965           1031          94          0.1        9653.8       0.4X
+Coalesce Num Partitions: 5000 Num Hosts: 20             643            722          98          0.2        6427.4       0.6X
+Coalesce Num Partitions: 5000 Num Hosts: 40             445            517          97          0.2        4448.8       0.8X
+Coalesce Num Partitions: 5000 Num Hosts: 80             369            432          93          0.3        3688.6       1.0X
+Coalesce Num Partitions: 10000 Num Hosts: 1           12850          12953          90          0.0      128503.9       0.0X
+Coalesce Num Partitions: 10000 Num Hosts: 5            3431           3463          49          0.0       34307.1       0.1X
+Coalesce Num Partitions: 10000 Num Hosts: 10           1687           1795         106          0.1       16874.8       0.2X
+Coalesce Num Partitions: 10000 Num Hosts: 20           1084           1125          58          0.1       10844.2       0.3X
+Coalesce Num Partitions: 10000 Num Hosts: 40            667            722          73          0.1        6674.4       0.5X
+Coalesce Num Partitions: 10000 Num Hosts: 80            488            542          82          0.2        4883.9       0.7X
 
 
diff --git a/core/benchmarks/CoalescedRDDBenchmark-results.txt b/core/benchmarks/CoalescedRDDBenchmark-results.txt
index f1b867951a074..829e43530dd5f 100644
--- a/core/benchmarks/CoalescedRDDBenchmark-results.txt
+++ b/core/benchmarks/CoalescedRDDBenchmark-results.txt
@@ -2,39 +2,39 @@
 Coalesced RDD , large scale
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Coalesced RDD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Coalesce Num Partitions: 100 Num Hosts: 1            395            401           9          0.3        3952.3       1.0X
-Coalesce Num Partitions: 100 Num Hosts: 5            296            344          42          0.3        2963.2       1.3X
-Coalesce Num Partitions: 100 Num Hosts: 10            294            308          15          0.3        2941.7       1.3X
-Coalesce Num Partitions: 100 Num Hosts: 20            316            328          13          0.3        3155.2       1.3X
-Coalesce Num Partitions: 100 Num Hosts: 40            294            316          36          0.3        2940.3       1.3X
-Coalesce Num Partitions: 100 Num Hosts: 80            292            324          30          0.3        2922.2       1.4X
-Coalesce Num Partitions: 500 Num Hosts: 1            629            687          61          0.2        6292.4       0.6X
-Coalesce Num Partitions: 500 Num Hosts: 5            354            378          42          0.3        3541.7       1.1X
-Coalesce Num Partitions: 500 Num Hosts: 10            318            338          29          0.3        3179.8       1.2X
-Coalesce Num Partitions: 500 Num Hosts: 20            306            317          11          0.3        3059.2       1.3X
-Coalesce Num Partitions: 500 Num Hosts: 40            294            311          28          0.3        2941.6       1.3X
-Coalesce Num Partitions: 500 Num Hosts: 80            288            309          34          0.3        2883.9       1.4X
-Coalesce Num Partitions: 1000 Num Hosts: 1            956            978          20          0.1        9562.2       0.4X
-Coalesce Num Partitions: 1000 Num Hosts: 5            431            452          36          0.2        4306.2       0.9X
-Coalesce Num Partitions: 1000 Num Hosts: 10            358            379          23          0.3        3581.1       1.1X
-Coalesce Num Partitions: 1000 Num Hosts: 20            324            347          20          0.3        3236.7       1.2X
-Coalesce Num Partitions: 1000 Num Hosts: 40            312            333          20          0.3        3116.8       1.3X
-Coalesce Num Partitions: 1000 Num Hosts: 80            307            342          32          0.3        3068.4       1.3X
-Coalesce Num Partitions: 5000 Num Hosts: 1           3895           3906          12          0.0       38946.8       0.1X
-Coalesce Num Partitions: 5000 Num Hosts: 5           1388           1401          19          0.1       13881.7       0.3X
-Coalesce Num Partitions: 5000 Num Hosts: 10            806            839          57          0.1        8063.7       0.5X
-Coalesce Num Partitions: 5000 Num Hosts: 20            546            573          44          0.2        5462.6       0.7X
-Coalesce Num Partitions: 5000 Num Hosts: 40            413            418           5          0.2        4134.7       1.0X
-Coalesce Num Partitions: 5000 Num Hosts: 80            345            365          23          0.3        3448.1       1.1X
-Coalesce Num Partitions: 10000 Num Hosts: 1           6933           6966          55          0.0       69328.8       0.1X
-Coalesce Num Partitions: 10000 Num Hosts: 5           2455           2499          69          0.0       24551.7       0.2X
-Coalesce Num Partitions: 10000 Num Hosts: 10           1352           1392          34          0.1       13520.2       0.3X
-Coalesce Num Partitions: 10000 Num Hosts: 20            815            853          50          0.1        8147.5       0.5X
-Coalesce Num Partitions: 10000 Num Hosts: 40            558            581          28          0.2        5578.0       0.7X
-Coalesce Num Partitions: 10000 Num Hosts: 80            416            423           5          0.2        4163.3       0.9X
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+Coalesced RDD:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+----------------------------------------------------------------------------------------------------------------------------
+Coalesce Num Partitions: 100 Num Hosts: 1               394            423          25          0.3        3942.7       1.0X
+Coalesce Num Partitions: 100 Num Hosts: 5               317            339          23          0.3        3171.6       1.2X
+Coalesce Num Partitions: 100 Num Hosts: 10              310            324          20          0.3        3095.9       1.3X
+Coalesce Num Partitions: 100 Num Hosts: 20              285            292           8          0.4        2849.0       1.4X
+Coalesce Num Partitions: 100 Num Hosts: 40              271            292          22          0.4        2712.7       1.5X
+Coalesce Num Partitions: 100 Num Hosts: 80              287            300          18          0.3        2874.2       1.4X
+Coalesce Num Partitions: 500 Num Hosts: 1               839            872          33          0.1        8388.1       0.5X
+Coalesce Num Partitions: 500 Num Hosts: 5               389            395           5          0.3        3892.4       1.0X
+Coalesce Num Partitions: 500 Num Hosts: 10              352            356           3          0.3        3522.0       1.1X
+Coalesce Num Partitions: 500 Num Hosts: 20              315            322          11          0.3        3145.0       1.3X
+Coalesce Num Partitions: 500 Num Hosts: 40              304            324          22          0.3        3036.4       1.3X
+Coalesce Num Partitions: 500 Num Hosts: 80              286            301          23          0.4        2855.7       1.4X
+Coalesce Num Partitions: 1000 Num Hosts: 1             1390           1422          28          0.1       13898.0       0.3X
+Coalesce Num Partitions: 1000 Num Hosts: 5              535            562          26          0.2        5347.4       0.7X
+Coalesce Num Partitions: 1000 Num Hosts: 10             419            425           8          0.2        4193.8       0.9X
+Coalesce Num Partitions: 1000 Num Hosts: 20             358            364           9          0.3        3575.8       1.1X
+Coalesce Num Partitions: 1000 Num Hosts: 40             323            329           5          0.3        3234.8       1.2X
+Coalesce Num Partitions: 1000 Num Hosts: 80             306            316          11          0.3        3060.3       1.3X
+Coalesce Num Partitions: 5000 Num Hosts: 1             5967           6744        1289          0.0       59666.6       0.1X
+Coalesce Num Partitions: 5000 Num Hosts: 5             2010           2032          19          0.0       20104.1       0.2X
+Coalesce Num Partitions: 5000 Num Hosts: 10            1132           1161          25          0.1       11324.9       0.3X
+Coalesce Num Partitions: 5000 Num Hosts: 20             687            714          39          0.1        6874.2       0.6X
+Coalesce Num Partitions: 5000 Num Hosts: 40             507            511           3          0.2        5070.1       0.8X
+Coalesce Num Partitions: 5000 Num Hosts: 80             375            383          12          0.3        3749.7       1.1X
+Coalesce Num Partitions: 10000 Num Hosts: 1           10782          10833          63          0.0      107816.6       0.0X
+Coalesce Num Partitions: 10000 Num Hosts: 5            3819           3901          72          0.0       38185.3       0.1X
+Coalesce Num Partitions: 10000 Num Hosts: 10           2021           2030          12          0.0       20212.9       0.2X
+Coalesce Num Partitions: 10000 Num Hosts: 20           1138           1168          44          0.1       11378.0       0.3X
+Coalesce Num Partitions: 10000 Num Hosts: 40            716            723           9          0.1        7157.2       0.6X
+Coalesce Num Partitions: 10000 Num Hosts: 80            504            514           9          0.2        5039.9       0.8X
 
 
diff --git a/core/benchmarks/KryoBenchmark-jdk11-results.txt b/core/benchmarks/KryoBenchmark-jdk11-results.txt
index 27f0b8f59f47a..22b776c6808c8 100644
--- a/core/benchmarks/KryoBenchmark-jdk11-results.txt
+++ b/core/benchmarks/KryoBenchmark-jdk11-results.txt
@@ -2,27 +2,27 @@
 Benchmark Kryo Unsafe vs safe Serialization
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Benchmark Kryo Unsafe vs safe Serialization:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-basicTypes: Int with unsafe:true                    275            288          14          3.6         275.2       1.0X
-basicTypes: Long with unsafe:true                   331            336          13          3.0         330.9       0.8X
-basicTypes: Float with unsafe:true                  304            305           1          3.3         304.4       0.9X
-basicTypes: Double with unsafe:true                 328            332           3          3.0         328.1       0.8X
-Array: Int with unsafe:true                           4              4           0        252.8           4.0      69.6X
-Array: Long with unsafe:true                          6              6           0        161.5           6.2      44.5X
-Array: Float with unsafe:true                         4              4           0        264.6           3.8      72.8X
-Array: Double with unsafe:true                        6              7           0        160.5           6.2      44.2X
-Map of string->Double  with unsafe:true              52             52           0         19.3          51.8       5.3X
-basicTypes: Int with unsafe:false                   344            345           1          2.9         344.3       0.8X
-basicTypes: Long with unsafe:false                  372            373           1          2.7         372.3       0.7X
-basicTypes: Float with unsafe:false                 333            334           1          3.0         333.4       0.8X
-basicTypes: Double with unsafe:false                344            345           0          2.9         344.3       0.8X
-Array: Int with unsafe:false                         25             25           0         40.8          24.5      11.2X
-Array: Long with unsafe:false                        37             37           1         27.3          36.7       7.5X
-Array: Float with unsafe:false                       11             11           0         92.1          10.9      25.4X
-Array: Double with unsafe:false                      17             18           0         58.3          17.2      16.0X
-Map of string->Double  with unsafe:false             51             52           1         19.4          51.5       5.3X
+---------------------------------------------------------------------------------------------------------------------------
+basicTypes: Int with unsafe:true                       331            346          11          3.0         330.7       1.0X
+basicTypes: Long with unsafe:true                      392            401           9          2.6         392.0       0.8X
+basicTypes: Float with unsafe:true                     394            404           9          2.5         394.5       0.8X
+basicTypes: Double with unsafe:true                    404            414           7          2.5         404.0       0.8X
+Array: Int with unsafe:true                              5              6           1        195.3           5.1      64.6X
+Array: Long with unsafe:true                             8              9           1        124.1           8.1      41.0X
+Array: Float with unsafe:true                            5              6           1        186.1           5.4      61.5X
+Array: Double with unsafe:true                           8              9           1        126.1           7.9      41.7X
+Map of string->Double  with unsafe:true                 54             59           4         18.4          54.4       6.1X
+basicTypes: Int with unsafe:false                      417            428           8          2.4         416.6       0.8X
+basicTypes: Long with unsafe:false                     452            466           9          2.2         451.8       0.7X
+basicTypes: Float with unsafe:false                    410            421           6          2.4         410.0       0.8X
+basicTypes: Double with unsafe:false                   429            444          10          2.3         429.2       0.8X
+Array: Int with unsafe:false                            25             27           2         39.2          25.5      13.0X
+Array: Long with unsafe:false                           40             43           2         25.0          40.0       8.3X
+Array: Float with unsafe:false                          11             12           1         90.7          11.0      30.0X
+Array: Double with unsafe:false                         18             20           1         54.2          18.5      17.9X
+Map of string->Double  with unsafe:false                55             59           2         18.0          55.5       6.0X
 
 
diff --git a/core/benchmarks/KryoBenchmark-results.txt b/core/benchmarks/KryoBenchmark-results.txt
index 49791e6e87e3a..5c38ff9cabe0e 100644
--- a/core/benchmarks/KryoBenchmark-results.txt
+++ b/core/benchmarks/KryoBenchmark-results.txt
@@ -2,27 +2,27 @@
 Benchmark Kryo Unsafe vs safe Serialization
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Benchmark Kryo Unsafe vs safe Serialization:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-basicTypes: Int with unsafe:true                    269            290          23          3.7         269.0       1.0X
-basicTypes: Long with unsafe:true                   294            295           1          3.4         293.8       0.9X
-basicTypes: Float with unsafe:true                  300            301           1          3.3         300.4       0.9X
-basicTypes: Double with unsafe:true                 304            305           1          3.3         304.0       0.9X
-Array: Int with unsafe:true                           5              6           1        193.5           5.2      52.0X
-Array: Long with unsafe:true                          8              9           1        131.2           7.6      35.3X
-Array: Float with unsafe:true                         6              6           0        163.5           6.1      44.0X
-Array: Double with unsafe:true                        9             10           0        108.8           9.2      29.3X
-Map of string->Double  with unsafe:true              54             54           1         18.7          53.6       5.0X
-basicTypes: Int with unsafe:false                   326            327           1          3.1         326.2       0.8X
-basicTypes: Long with unsafe:false                  353            354           1          2.8         353.3       0.8X
-basicTypes: Float with unsafe:false                 325            327           1          3.1         325.1       0.8X
-basicTypes: Double with unsafe:false                335            336           1          3.0         335.0       0.8X
-Array: Int with unsafe:false                         27             28           1         36.7          27.2       9.9X
-Array: Long with unsafe:false                        40             41           1         25.0          40.0       6.7X
-Array: Float with unsafe:false                       12             13           1         80.8          12.4      21.7X
-Array: Double with unsafe:false                      21             21           1         48.6          20.6      13.1X
-Map of string->Double  with unsafe:false             56             57           1         17.8          56.1       4.8X
+---------------------------------------------------------------------------------------------------------------------------
+basicTypes: Int with unsafe:true                       286            295           6          3.5         285.6       1.0X
+basicTypes: Long with unsafe:true                      320            327           5          3.1         319.8       0.9X
+basicTypes: Float with unsafe:true                     314            318           2          3.2         313.5       0.9X
+basicTypes: Double with unsafe:true                    310            319           6          3.2         309.9       0.9X
+Array: Int with unsafe:true                              5              6           1        217.9           4.6      62.2X
+Array: Long with unsafe:true                             8              9           1        121.5           8.2      34.7X
+Array: Float with unsafe:true                            5              6           0        217.5           4.6      62.1X
+Array: Double with unsafe:true                           9              9           1        117.3           8.5      33.5X
+Map of string->Double  with unsafe:true                 51             52           1         19.7          50.9       5.6X
+basicTypes: Int with unsafe:false                      373            384          12          2.7         373.3       0.8X
+basicTypes: Long with unsafe:false                     373            391           9          2.7         372.6       0.8X
+basicTypes: Float with unsafe:false                    349            362          11          2.9         349.2       0.8X
+basicTypes: Double with unsafe:false                   358            372          10          2.8         358.0       0.8X
+Array: Int with unsafe:false                            25             29           1         40.0          25.0      11.4X
+Array: Long with unsafe:false                           35             39           1         28.2          35.5       8.1X
+Array: Float with unsafe:false                          10             12           1        100.2          10.0      28.6X
+Array: Double with unsafe:false                         17             20           1         58.3          17.1      16.7X
+Map of string->Double  with unsafe:false                48             53           2         20.6          48.5       5.9X
 
 
diff --git a/core/benchmarks/KryoSerializerBenchmark-jdk11-results.txt b/core/benchmarks/KryoSerializerBenchmark-jdk11-results.txt
index 6b148bde12d36..41ac5386754a4 100644
--- a/core/benchmarks/KryoSerializerBenchmark-jdk11-results.txt
+++ b/core/benchmarks/KryoSerializerBenchmark-jdk11-results.txt
@@ -2,11 +2,11 @@
 Benchmark KryoPool vs old"pool of 1" implementation
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Benchmark KryoPool vs old"pool of 1" implementation:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-KryoPool:true                                      6208           8374         NaN          0.0    12416876.6       1.0X
-KryoPool:false                                     9084          11577         724          0.0    18168947.4       0.7X
+-----------------------------------------------------------------------------------------------------------------------------------
+KryoPool:true                                                10053          13422         983          0.0    20106810.4       1.0X
+KryoPool:false                                               16854          19164         NaN          0.0    33708260.0       0.6X
 
 
diff --git a/core/benchmarks/KryoSerializerBenchmark-results.txt b/core/benchmarks/KryoSerializerBenchmark-results.txt
index 609f3298cbc00..c10f7dc415982 100644
--- a/core/benchmarks/KryoSerializerBenchmark-results.txt
+++ b/core/benchmarks/KryoSerializerBenchmark-results.txt
@@ -2,11 +2,11 @@
 Benchmark KryoPool vs old"pool of 1" implementation
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Benchmark KryoPool vs old"pool of 1" implementation:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-KryoPool:true                                      6012           7586         NaN          0.0    12023020.2       1.0X
-KryoPool:false                                     9289          11566         909          0.0    18578683.1       0.6X
+-----------------------------------------------------------------------------------------------------------------------------------
+KryoPool:true                                                 7893          10556         NaN          0.0    15785307.8       1.0X
+KryoPool:false                                               12778          15741         426          0.0    25555753.8       0.6X
 
 
diff --git a/core/benchmarks/MapStatusesSerDeserBenchmark-jdk11-results.txt b/core/benchmarks/MapStatusesSerDeserBenchmark-jdk11-results.txt
index db23cf5c12ea7..29699a2fdcf2a 100644
--- a/core/benchmarks/MapStatusesSerDeserBenchmark-jdk11-results.txt
+++ b/core/benchmarks/MapStatusesSerDeserBenchmark-jdk11-results.txt
@@ -1,64 +1,64 @@
-OpenJDK 64-Bit Server VM 11.0.4+11-post-Ubuntu-1ubuntu218.04.3 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 200000 MapOutputs, 10 blocks w/ broadcast:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Serialization                                       170            178           9          1.2         849.7       1.0X
-Deserialization                                     530            535           9          0.4        2651.1       0.3X
+-------------------------------------------------------------------------------------------------------------------------
+Serialization                                        179            194           9          1.1         897.4       1.0X
+Deserialization                                      254            321          74          0.8        1271.0       0.7X
 
-Compressed Serialized MapStatus sizes: 411 bytes
+Compressed Serialized MapStatus sizes: 409 bytes
 Compressed Serialized Broadcast MapStatus sizes: 2 MB
 
 
-OpenJDK 64-Bit Server VM 11.0.4+11-post-Ubuntu-1ubuntu218.04.3 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 200000 MapOutputs, 10 blocks w/o broadcast:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Serialization                                       157            165           7          1.3         785.4       1.0X
-Deserialization                                     495            588          79          0.4        2476.7       0.3X
+--------------------------------------------------------------------------------------------------------------------------
+Serialization                                         160            166           7          1.2         801.2       1.0X
+Deserialization                                       256            323          69          0.8        1278.9       0.6X
 
 Compressed Serialized MapStatus sizes: 2 MB
 Compressed Serialized Broadcast MapStatus sizes: 0 bytes
 
 
-OpenJDK 64-Bit Server VM 11.0.4+11-post-Ubuntu-1ubuntu218.04.3 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 200000 MapOutputs, 100 blocks w/ broadcast:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Serialization                                       344            351           4          0.6        1720.4       1.0X
-Deserialization                                     527            579          99          0.4        2635.9       0.7X
+--------------------------------------------------------------------------------------------------------------------------
+Serialization                                         341            349           7          0.6        1707.3       1.0X
+Deserialization                                       286            370          84          0.7        1431.4       1.2X
 
-Compressed Serialized MapStatus sizes: 427 bytes
+Compressed Serialized MapStatus sizes: 426 bytes
 Compressed Serialized Broadcast MapStatus sizes: 13 MB
 
 
-OpenJDK 64-Bit Server VM 11.0.4+11-post-Ubuntu-1ubuntu218.04.3 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 200000 MapOutputs, 100 blocks w/o broadcast:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Serialization                                       317            321           4          0.6        1583.8       1.0X
-Deserialization                                     530            540          15          0.4        2648.3       0.6X
+---------------------------------------------------------------------------------------------------------------------------
+Serialization                                          309            319          11          0.6        1543.6       1.0X
+Deserialization                                        286            373         117          0.7        1429.5       1.1X
 
 Compressed Serialized MapStatus sizes: 13 MB
 Compressed Serialized Broadcast MapStatus sizes: 0 bytes
 
 
-OpenJDK 64-Bit Server VM 11.0.4+11-post-Ubuntu-1ubuntu218.04.3 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 200000 MapOutputs, 1000 blocks w/ broadcast:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Serialization                                      1738           1849         156          0.1        8692.0       1.0X
-Deserialization                                     946            977          33          0.2        4730.2       1.8X
+---------------------------------------------------------------------------------------------------------------------------
+Serialization                                         1619           1627          12          0.1        8092.6       1.0X
+Deserialization                                        864            883          26          0.2        4319.9       1.9X
 
-Compressed Serialized MapStatus sizes: 556 bytes
+Compressed Serialized MapStatus sizes: 557 bytes
 Compressed Serialized Broadcast MapStatus sizes: 121 MB
 
 
-OpenJDK 64-Bit Server VM 11.0.4+11-post-Ubuntu-1ubuntu218.04.3 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 200000 MapOutputs, 1000 blocks w/o broadcast:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Serialization                                      1379           1432          76          0.1        6892.6       1.0X
-Deserialization                                     929            941          19          0.2        4645.5       1.5X
+----------------------------------------------------------------------------------------------------------------------------
+Serialization                                          1449           1456           9          0.1        7246.8       1.0X
+Deserialization                                         853            888          46          0.2        4263.7       1.7X
 
 Compressed Serialized MapStatus sizes: 121 MB
 Compressed Serialized Broadcast MapStatus sizes: 0 bytes
diff --git a/core/benchmarks/MapStatusesSerDeserBenchmark-results.txt b/core/benchmarks/MapStatusesSerDeserBenchmark-results.txt
index 053f4bf771923..96fa3a01a8f6d 100644
--- a/core/benchmarks/MapStatusesSerDeserBenchmark-results.txt
+++ b/core/benchmarks/MapStatusesSerDeserBenchmark-results.txt
@@ -1,64 +1,64 @@
-OpenJDK 64-Bit Server VM 1.8.0_222-8u222-b10-1ubuntu1~18.04.1-b10 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 200000 MapOutputs, 10 blocks w/ broadcast:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Serialization                                       178            187          15          1.1         887.5       1.0X
-Deserialization                                     530            558          32          0.4        2647.5       0.3X
+-------------------------------------------------------------------------------------------------------------------------
+Serialization                                        135            161          56          1.5         673.9       1.0X
+Deserialization                                      213            235          26          0.9        1065.6       0.6X
 
-Compressed Serialized MapStatus sizes: 411 bytes
+Compressed Serialized MapStatus sizes: 409 bytes
 Compressed Serialized Broadcast MapStatus sizes: 2 MB
 
 
-OpenJDK 64-Bit Server VM 1.8.0_222-8u222-b10-1ubuntu1~18.04.1-b10 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 200000 MapOutputs, 10 blocks w/o broadcast:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Serialization                                       167            175           7          1.2         835.7       1.0X
-Deserialization                                     523            537          22          0.4        2616.2       0.3X
+--------------------------------------------------------------------------------------------------------------------------
+Serialization                                         130            137           5          1.5         650.8       1.0X
+Deserialization                                       211            230          20          0.9        1056.5       0.6X
 
 Compressed Serialized MapStatus sizes: 2 MB
 Compressed Serialized Broadcast MapStatus sizes: 0 bytes
 
 
-OpenJDK 64-Bit Server VM 1.8.0_222-8u222-b10-1ubuntu1~18.04.1-b10 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 200000 MapOutputs, 100 blocks w/ broadcast:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Serialization                                       351            416         147          0.6        1754.4       1.0X
-Deserialization                                     546            551           8          0.4        2727.6       0.6X
+--------------------------------------------------------------------------------------------------------------------------
+Serialization                                         281            324          86          0.7        1406.7       1.0X
+Deserialization                                       240            267          32          0.8        1200.5       1.2X
 
-Compressed Serialized MapStatus sizes: 427 bytes
+Compressed Serialized MapStatus sizes: 426 bytes
 Compressed Serialized Broadcast MapStatus sizes: 13 MB
 
 
-OpenJDK 64-Bit Server VM 1.8.0_222-8u222-b10-1ubuntu1~18.04.1-b10 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 200000 MapOutputs, 100 blocks w/o broadcast:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Serialization                                       320            321           1          0.6        1598.0       1.0X
-Deserialization                                     542            549           7          0.4        2709.0       0.6X
+---------------------------------------------------------------------------------------------------------------------------
+Serialization                                          265            273           6          0.8        1324.5       1.0X
+Deserialization                                        247            276          33          0.8        1236.1       1.1X
 
 Compressed Serialized MapStatus sizes: 13 MB
 Compressed Serialized Broadcast MapStatus sizes: 0 bytes
 
 
-OpenJDK 64-Bit Server VM 1.8.0_222-8u222-b10-1ubuntu1~18.04.1-b10 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 200000 MapOutputs, 1000 blocks w/ broadcast:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Serialization                                      1671           1877         290          0.1        8357.3       1.0X
-Deserialization                                     943            970          32          0.2        4715.8       1.8X
+---------------------------------------------------------------------------------------------------------------------------
+Serialization                                         1333           1592         366          0.2        6666.0       1.0X
+Deserialization                                        560            585          22          0.4        2799.1       2.4X
 
-Compressed Serialized MapStatus sizes: 556 bytes
+Compressed Serialized MapStatus sizes: 558 bytes
 Compressed Serialized Broadcast MapStatus sizes: 121 MB
 
 
-OpenJDK 64-Bit Server VM 1.8.0_222-8u222-b10-1ubuntu1~18.04.1-b10 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 200000 MapOutputs, 1000 blocks w/o broadcast:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Serialization                                      1373           1436          89          0.1        6865.0       1.0X
-Deserialization                                     940            970          37          0.2        4699.1       1.5X
+----------------------------------------------------------------------------------------------------------------------------
+Serialization                                          1222           1260          54          0.2        6111.7       1.0X
+Deserialization                                         539            568          42          0.4        2695.3       2.3X
 
 Compressed Serialized MapStatus sizes: 121 MB
 Compressed Serialized Broadcast MapStatus sizes: 0 bytes
diff --git a/core/benchmarks/PropertiesCloneBenchmark-jdk11-results.txt b/core/benchmarks/PropertiesCloneBenchmark-jdk11-results.txt
index 605b856d53382..f1d6b1ff372bc 100644
--- a/core/benchmarks/PropertiesCloneBenchmark-jdk11-results.txt
+++ b/core/benchmarks/PropertiesCloneBenchmark-jdk11-results.txt
@@ -2,39 +2,39 @@
 Properties Cloning
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Empty Properties:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone                              0              0           0          0.1       11539.0       1.0X
-Utils.cloneProperties                                 0              0           0          1.7         572.0      20.2X
+SerializationUtils.clone                              0              0           0          0.2        4800.0       1.0X
+Utils.cloneProperties                                 0              0           0     Infinity           0.0  InfinityX
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 System Properties:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone                              0              0           0          0.0      217514.0       1.0X
-Utils.cloneProperties                                 0              0           0          0.2        5387.0      40.4X
+SerializationUtils.clone                              0              0           0          0.0      202203.0       1.0X
+Utils.cloneProperties                                 0              0           0          0.1        6700.0      30.2X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Small Properties:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone                              1              1           0          0.0      634574.0       1.0X
-Utils.cloneProperties                                 0              0           0          0.3        3082.0     205.9X
+SerializationUtils.clone                              1              1           0          0.0      588099.0       1.0X
+Utils.cloneProperties                                 0              0           0          0.2        5699.0     103.2X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Medium Properties:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone                              3              3           0          0.0     2576565.0       1.0X
-Utils.cloneProperties                                 0              0           0          0.1       16071.0     160.3X
+SerializationUtils.clone                              2              3           0          0.0     2357927.0       1.0X
+Utils.cloneProperties                                 0              0           0          0.0       31901.0      73.9X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Large Properties:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone                              5              5           0          0.0     5027248.0       1.0X
-Utils.cloneProperties                                 0              0           0          0.0       31842.0     157.9X
+SerializationUtils.clone                              5              5           0          0.0     4636068.0       1.0X
+Utils.cloneProperties                                 0              0           0          0.0       64701.0      71.7X
 
 
diff --git a/core/benchmarks/PropertiesCloneBenchmark-results.txt b/core/benchmarks/PropertiesCloneBenchmark-results.txt
index 5d332a147c698..81ccff6e638c8 100644
--- a/core/benchmarks/PropertiesCloneBenchmark-results.txt
+++ b/core/benchmarks/PropertiesCloneBenchmark-results.txt
@@ -2,39 +2,39 @@
 Properties Cloning
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 Empty Properties:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone                              0              0           0          0.1       13640.0       1.0X
-Utils.cloneProperties                                 0              0           0          1.6         608.0      22.4X
+SerializationUtils.clone                              0              0           0          0.2        5599.0       1.0X
+Utils.cloneProperties                                 0              0           0     Infinity           0.0  InfinityX
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 System Properties:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone                              0              0           0          0.0      238968.0       1.0X
-Utils.cloneProperties                                 0              0           0          0.4        2318.0     103.1X
+SerializationUtils.clone                              0              0           0          0.0      339506.0       1.0X
+Utils.cloneProperties                                 0              0           0          0.5        1900.0     178.7X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 Small Properties:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone                              1              1           0          0.0      725849.0       1.0X
-Utils.cloneProperties                                 0              0           0          0.3        2900.0     250.3X
+SerializationUtils.clone                              1              1           0          0.0      604411.0       1.0X
+Utils.cloneProperties                                 0              0           0          0.2        5100.0     118.5X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 Medium Properties:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone                              3              3           0          0.0     2999676.0       1.0X
-Utils.cloneProperties                                 0              0           0          0.1       11734.0     255.6X
+SerializationUtils.clone                              2              2           0          0.0     2378345.0       1.0X
+Utils.cloneProperties                                 0              0           0          0.0       25100.0      94.8X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 Large Properties:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone                              6              6           1          0.0     5846410.0       1.0X
-Utils.cloneProperties                                 0              0           0          0.0       22405.0     260.9X
+SerializationUtils.clone                              4              4           0          0.0     4035677.0       1.0X
+Utils.cloneProperties                                 0              0           0          0.0       43300.0      93.2X
 
 
diff --git a/core/benchmarks/XORShiftRandomBenchmark-jdk11-results.txt b/core/benchmarks/XORShiftRandomBenchmark-jdk11-results.txt
index 9aa10e4835a2f..0b13462c59b51 100644
--- a/core/benchmarks/XORShiftRandomBenchmark-jdk11-results.txt
+++ b/core/benchmarks/XORShiftRandomBenchmark-jdk11-results.txt
@@ -2,43 +2,43 @@
 Pseudo random
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 nextInt:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-java.util.Random                                   1362           1362           0         73.4          13.6       1.0X
-XORShiftRandom                                      227            227           0        440.6           2.3       6.0X
+java.util.Random                                   1414           1423          10         70.7          14.1       1.0X
+XORShiftRandom                                      234            238           3        426.9           2.3       6.0X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 nextLong:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-java.util.Random                                   2725           2726           1         36.7          27.3       1.0X
-XORShiftRandom                                      694            694           1        144.1           6.9       3.9X
+java.util.Random                                   2669           2699          26         37.5          26.7       1.0X
+XORShiftRandom                                      622            633          12        160.7           6.2       4.3X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 nextDouble:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-java.util.Random                                   2727           2728           0         36.7          27.3       1.0X
-XORShiftRandom                                      693            694           0        144.2           6.9       3.9X
+java.util.Random                                   2613           2712          98         38.3          26.1       1.0X
+XORShiftRandom                                      667            675           7        149.9           6.7       3.9X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 nextGaussian:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-java.util.Random                                   7012           7016           4         14.3          70.1       1.0X
-XORShiftRandom                                     6065           6067           1         16.5          60.7       1.2X
+java.util.Random                                   6687           6757          61         15.0          66.9       1.0X
+XORShiftRandom                                     4882           4892           9         20.5          48.8       1.4X
 
 
 ================================================================================================
 hash seed
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Hash seed:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-XORShiftRandom.hashSeed                              36             37           1        276.5           3.6       1.0X
+XORShiftRandom.hashSeed                              40             42           3        247.4           4.0       1.0X
 
 
diff --git a/core/benchmarks/XORShiftRandomBenchmark-results.txt b/core/benchmarks/XORShiftRandomBenchmark-results.txt
index 4b069878b2e9b..87093eaabd7ea 100644
--- a/core/benchmarks/XORShiftRandomBenchmark-results.txt
+++ b/core/benchmarks/XORShiftRandomBenchmark-results.txt
@@ -2,43 +2,43 @@
 Pseudo random
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 nextInt:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-java.util.Random                                   1362           1396          59         73.4          13.6       1.0X
-XORShiftRandom                                      227            227           0        440.7           2.3       6.0X
+java.util.Random                                   1192           1217          24         83.9          11.9       1.0X
+XORShiftRandom                                      193            196           4        518.9           1.9       6.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 nextLong:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-java.util.Random                                   2732           2732           1         36.6          27.3       1.0X
-XORShiftRandom                                      630            630           1        158.7           6.3       4.3X
+java.util.Random                                   2391           2402          10         41.8          23.9       1.0X
+XORShiftRandom                                      518            528          11        193.1           5.2       4.6X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 nextDouble:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-java.util.Random                                   2731           2732           1         36.6          27.3       1.0X
-XORShiftRandom                                      630            630           0        158.8           6.3       4.3X
+java.util.Random                                   2318           2363          39         43.1          23.2       1.0X
+XORShiftRandom                                      488            496          13        205.0           4.9       4.8X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 nextGaussian:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-java.util.Random                                   8895           8899           4         11.2          88.9       1.0X
-XORShiftRandom                                     5049           5052           5         19.8          50.5       1.8X
+java.util.Random                                   6102           6156          77         16.4          61.0       1.0X
+XORShiftRandom                                     3685           3758          84         27.1          36.9       1.7X
 
 
 ================================================================================================
 hash seed
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Hash seed:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-XORShiftRandom.hashSeed                              67             68           1        148.8           6.7       1.0X
+XORShiftRandom.hashSeed                              65             66           2        155.0           6.5       1.0X
 
 
diff --git a/core/benchmarks/ZStandardBenchmark-jdk11-results.txt b/core/benchmarks/ZStandardBenchmark-jdk11-results.txt
new file mode 100644
index 0000000000000..3895e7bb6f27d
--- /dev/null
+++ b/core/benchmarks/ZStandardBenchmark-jdk11-results.txt
@@ -0,0 +1,27 @@
+================================================================================================
+Benchmark ZStandardCompressionCodec
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+Benchmark ZStandardCompressionCodec:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+--------------------------------------------------------------------------------------------------------------------------------------
+Compression 10000 times at level 1 without buffer pool            606            614           6          0.0       60645.3       1.0X
+Compression 10000 times at level 2 without buffer pool            686            693           7          0.0       68594.9       0.9X
+Compression 10000 times at level 3 without buffer pool            906            920          14          0.0       90642.7       0.7X
+Compression 10000 times at level 1 with buffer pool               389            403          20          0.0       38901.4       1.6X
+Compression 10000 times at level 2 with buffer pool               450            466          13          0.0       45032.0       1.3X
+Compression 10000 times at level 3 with buffer pool               680            682           2          0.0       68004.2       0.9X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+Benchmark ZStandardCompressionCodec:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------------------------
+Decompression 10000 times from level 1 without buffer pool           1209           1226          25          0.0      120862.8       1.0X
+Decompression 10000 times from level 2 without buffer pool           1191           1193           3          0.0      119064.9       1.0X
+Decompression 10000 times from level 3 without buffer pool           1188           1193           6          0.0      118843.3       1.0X
+Decompression 10000 times from level 1 with buffer pool               998           1004           9          0.0       99754.7       1.2X
+Decompression 10000 times from level 2 with buffer pool               990           1001          11          0.0       99043.8       1.2X
+Decompression 10000 times from level 3 with buffer pool               983            999          20          0.0       98269.5       1.2X
+
+
diff --git a/core/benchmarks/ZStandardBenchmark-results.txt b/core/benchmarks/ZStandardBenchmark-results.txt
new file mode 100644
index 0000000000000..6990c28690072
--- /dev/null
+++ b/core/benchmarks/ZStandardBenchmark-results.txt
@@ -0,0 +1,27 @@
+================================================================================================
+Benchmark ZStandardCompressionCodec
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
+Benchmark ZStandardCompressionCodec:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+--------------------------------------------------------------------------------------------------------------------------------------
+Compression 10000 times at level 1 without buffer pool            670            681           9          0.0       67011.0       1.0X
+Compression 10000 times at level 2 without buffer pool            569            571           2          0.0       56932.0       1.2X
+Compression 10000 times at level 3 without buffer pool            748            751           2          0.0       74813.8       0.9X
+Compression 10000 times at level 1 with buffer pool               336            337           1          0.0       33630.6       2.0X
+Compression 10000 times at level 2 with buffer pool               395            397           2          0.0       39472.6       1.7X
+Compression 10000 times at level 3 with buffer pool               563            567           4          0.0       56272.8       1.2X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
+Benchmark ZStandardCompressionCodec:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------------------------
+Decompression 10000 times from level 1 without buffer pool           1029           1031           3          0.0      102887.4       1.0X
+Decompression 10000 times from level 2 without buffer pool           1028           1031           4          0.0      102847.8       1.0X
+Decompression 10000 times from level 3 without buffer pool           1029           1029           0          0.0      102941.0       1.0X
+Decompression 10000 times from level 1 with buffer pool               798            799           0          0.0       79838.0       1.3X
+Decompression 10000 times from level 2 with buffer pool               799            799           0          0.0       79852.9       1.3X
+Decompression 10000 times from level 3 with buffer pool               796            798           2          0.0       79630.5       1.3X
+
+
diff --git a/core/pom.xml b/core/pom.xml
index b0f68880f1d8a..ec80807f880e2 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
@@ -35,10 +35,6 @@
   </properties>
   
   <dependencies>
-    <dependency>
-      <groupId>com.thoughtworks.paranamer</groupId>
-      <artifactId>paranamer</artifactId>
-    </dependency>
     <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro</artifactId>
@@ -46,7 +42,6 @@
     <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro-mapred</artifactId>
-      <classifier>${avro.mapred.classifier}</classifier>
     </dependency>
     <dependency>
       <groupId>com.google.guava</groupId>
@@ -66,7 +61,13 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-client</artifactId>
+      <artifactId>${hadoop-client-api.artifact}</artifactId>
+      <version>${hadoop.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>${hadoop-client-runtime.artifact}</artifactId>
+      <version>${hadoop.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
@@ -161,9 +162,13 @@
       <scope>compile</scope>
     </dependency>
     <dependency>
-      <groupId>javax.servlet</groupId>
-      <artifactId>javax.servlet-api</artifactId>
-      <version>${javaxservlet.version}</version>
+      <groupId>jakarta.servlet</groupId>
+      <artifactId>jakarta.servlet-api</artifactId>
+      <version>${jakartaservlet.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>commons-codec</groupId>
+      <artifactId>commons-codec</artifactId>
     </dependency>
     <dependency>
       <groupId>org.apache.commons</groupId>
@@ -177,6 +182,14 @@
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-text</artifactId>
     </dependency>
+    <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>commons-collections</groupId>
+      <artifactId>commons-collections</artifactId>
+    </dependency>
     <dependency>
       <groupId>com.google.code.findbugs</groupId>
       <artifactId>jsr305</artifactId>
@@ -334,7 +347,7 @@
     </dependency>
     <dependency>
       <groupId>org.seleniumhq.selenium</groupId>
-      <artifactId>selenium-htmlunit-driver</artifactId>
+      <artifactId>htmlunit-driver</artifactId>
       <scope>test</scope>
     </dependency>
     <!-- Coerce sbt into honoring these dependency updates: -->
@@ -414,7 +427,7 @@
     <dependency>
       <groupId>net.sf.py4j</groupId>
       <artifactId>py4j</artifactId>
-      <version>0.10.9</version>
+      <version>0.10.9.2</version>
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
@@ -447,6 +460,47 @@
       <scope>test</scope>
     </dependency>
 
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-aws</artifactId>
+      <version>${hadoop.version}</version>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-common</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.codehaus.jackson</groupId>
+          <artifactId>jackson-mapper-asl</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.codehaus.jackson</groupId>
+          <artifactId>jackson-core-asl</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-core</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-databind</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-annotations</artifactId>
+        </exclusion>
+        <!-- Keep old SDK out of the assembly to avoid conflict with Kinesis module -->
+        <exclusion>
+          <groupId>com.amazonaws</groupId>
+          <artifactId>aws-java-sdk</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-crypto</artifactId>
diff --git a/core/src/main/java/org/apache/spark/SparkFirehoseListener.java b/core/src/main/java/org/apache/spark/SparkFirehoseListener.java
index 731f6fc767dfd..7cb2455affe48 100644
--- a/core/src/main/java/org/apache/spark/SparkFirehoseListener.java
+++ b/core/src/main/java/org/apache/spark/SparkFirehoseListener.java
@@ -17,6 +17,7 @@
 
 package org.apache.spark;
 
+import org.apache.spark.annotation.DeveloperApi;
 import org.apache.spark.scheduler.*;
 
 /**
@@ -27,7 +28,11 @@
  * new methods to SparkListener: forgetting to add a method will result in a compilation error (if
  * this was a concrete Scala class, default implementations of new event handlers would be inherited
  * from the SparkListener trait).
+ *
+ * Please note until Spark 3.1.0 this was missing the DevelopApi annotation, this needs to be
+ * taken into account if changing this API before a major release.
  */
+@DeveloperApi
 public class SparkFirehoseListener implements SparkListenerInterface {
 
   public void onEvent(SparkListenerEvent event) { }
@@ -124,34 +129,67 @@ public final void onExecutorBlacklisted(SparkListenerExecutorBlacklisted executo
     onEvent(executorBlacklisted);
   }
 
+  @Override
+  public final void onExecutorExcluded(SparkListenerExecutorExcluded executorExcluded) {
+    onEvent(executorExcluded);
+  }
+
   @Override
   public void onExecutorBlacklistedForStage(
       SparkListenerExecutorBlacklistedForStage executorBlacklistedForStage) {
     onEvent(executorBlacklistedForStage);
   }
 
+  @Override
+  public void onExecutorExcludedForStage(
+      SparkListenerExecutorExcludedForStage executorExcludedForStage) {
+    onEvent(executorExcludedForStage);
+  }
+
   @Override
   public void onNodeBlacklistedForStage(
       SparkListenerNodeBlacklistedForStage nodeBlacklistedForStage) {
     onEvent(nodeBlacklistedForStage);
   }
 
+  @Override
+  public void onNodeExcludedForStage(
+      SparkListenerNodeExcludedForStage nodeExcludedForStage) {
+    onEvent(nodeExcludedForStage);
+  }
+
   @Override
   public final void onExecutorUnblacklisted(
       SparkListenerExecutorUnblacklisted executorUnblacklisted) {
     onEvent(executorUnblacklisted);
   }
 
+  @Override
+  public final void onExecutorUnexcluded(
+      SparkListenerExecutorUnexcluded executorUnexcluded) {
+    onEvent(executorUnexcluded);
+  }
+
   @Override
   public final void onNodeBlacklisted(SparkListenerNodeBlacklisted nodeBlacklisted) {
     onEvent(nodeBlacklisted);
   }
 
+  @Override
+  public final void onNodeExcluded(SparkListenerNodeExcluded nodeExcluded) {
+    onEvent(nodeExcluded);
+  }
+
   @Override
   public final void onNodeUnblacklisted(SparkListenerNodeUnblacklisted nodeUnblacklisted) {
     onEvent(nodeUnblacklisted);
   }
 
+  @Override
+  public final void onNodeUnexcluded(SparkListenerNodeUnexcluded nodeUnexcluded) {
+    onEvent(nodeUnexcluded);
+  }
+
   @Override
   public void onBlockUpdated(SparkListenerBlockUpdated blockUpdated) {
     onEvent(blockUpdated);
@@ -162,6 +200,21 @@ public void onSpeculativeTaskSubmitted(SparkListenerSpeculativeTaskSubmitted spe
     onEvent(speculativeTask);
   }
 
+  public void onUnschedulableTaskSetAdded(
+      SparkListenerUnschedulableTaskSetAdded unschedulableTaskSetAdded) {
+    onEvent(unschedulableTaskSetAdded);
+  }
+
+  public void onUnschedulableTaskSetRemoved(
+      SparkListenerUnschedulableTaskSetRemoved unschedulableTaskSetRemoved) {
+    onEvent(unschedulableTaskSetRemoved);
+  }
+
+  @Override
+  public void onResourceProfileAdded(SparkListenerResourceProfileAdded event) {
+    onEvent(event);
+  }
+
   @Override
   public void onOtherEvent(SparkListenerEvent event) {
     onEvent(event);
diff --git a/core/src/main/java/org/apache/spark/api/java/StorageLevels.java b/core/src/main/java/org/apache/spark/api/java/StorageLevels.java
index 3fcb52f615834..b51cde48e632b 100644
--- a/core/src/main/java/org/apache/spark/api/java/StorageLevels.java
+++ b/core/src/main/java/org/apache/spark/api/java/StorageLevels.java
@@ -26,6 +26,7 @@ public class StorageLevels {
   public static final StorageLevel NONE = create(false, false, false, false, 1);
   public static final StorageLevel DISK_ONLY = create(true, false, false, false, 1);
   public static final StorageLevel DISK_ONLY_2 = create(true, false, false, false, 2);
+  public static final StorageLevel DISK_ONLY_3 = create(true, false, false, false, 3);
   public static final StorageLevel MEMORY_ONLY = create(false, true, false, true, 1);
   public static final StorageLevel MEMORY_ONLY_2 = create(false, true, false, true, 2);
   public static final StorageLevel MEMORY_ONLY_SER = create(false, true, false, false, 1);
diff --git a/core/src/main/java/org/apache/spark/api/plugin/DriverPlugin.java b/core/src/main/java/org/apache/spark/api/plugin/DriverPlugin.java
index 0c0d0df8ae682..1d676ff781c70 100644
--- a/core/src/main/java/org/apache/spark/api/plugin/DriverPlugin.java
+++ b/core/src/main/java/org/apache/spark/api/plugin/DriverPlugin.java
@@ -41,7 +41,7 @@ public interface DriverPlugin {
    * initialization.
    * <p>
    * It's recommended that plugins be careful about what operations are performed in this call,
-   * preferrably performing expensive operations in a separate thread, or postponing them until
+   * preferably performing expensive operations in a separate thread, or postponing them until
    * the application has fully started.
    *
    * @param sc The SparkContext loading the plugin.
diff --git a/core/src/main/java/org/apache/spark/api/plugin/ExecutorPlugin.java b/core/src/main/java/org/apache/spark/api/plugin/ExecutorPlugin.java
index 4961308035163..481bf985f1c6c 100644
--- a/core/src/main/java/org/apache/spark/api/plugin/ExecutorPlugin.java
+++ b/core/src/main/java/org/apache/spark/api/plugin/ExecutorPlugin.java
@@ -19,6 +19,7 @@
 
 import java.util.Map;
 
+import org.apache.spark.TaskFailedReason;
 import org.apache.spark.annotation.DeveloperApi;
 
 /**
@@ -54,4 +55,45 @@ default void init(PluginContext ctx, Map<String, String> extraConf) {}
    */
   default void shutdown() {}
 
+  /**
+   * Perform any action before the task is run.
+   * <p>
+   * This method is invoked from the same thread the task will be executed.
+   * Task-specific information can be accessed via {@link org.apache.spark.TaskContext#get}.
+   * <p>
+   * Plugin authors should avoid expensive operations here, as this method will be called
+   * on every task, and doing something expensive can significantly slow down a job.
+   * It is not recommended for a user to call a remote service, for example.
+   * <p>
+   * Exceptions thrown from this method do not propagate - they're caught,
+   * logged, and suppressed. Therefore exceptions when executing this method won't
+   * make the job fail.
+   *
+   * @since 3.1.0
+   */
+  default void onTaskStart() {}
+
+  /**
+   * Perform an action after tasks completes without exceptions.
+   * <p>
+   * As {@link #onTaskStart() onTaskStart} exceptions are suppressed, this method
+   * will still be invoked even if the corresponding {@link #onTaskStart} call for this
+   * task failed.
+   * <p>
+   * Same warnings of {@link #onTaskStart() onTaskStart} apply here.
+   *
+   * @since 3.1.0
+   */
+  default void onTaskSucceeded() {}
+
+  /**
+   * Perform an action after tasks completes with exceptions.
+   * <p>
+   * Same warnings of {@link #onTaskStart() onTaskStart} apply here.
+   *
+   * @param failureReason the exception thrown from the failed task.
+   *
+   * @since 3.1.0
+   */
+  default void onTaskFailed(TaskFailedReason failureReason) {}
 }
diff --git a/core/src/main/java/org/apache/spark/memory/MemoryConsumer.java b/core/src/main/java/org/apache/spark/memory/MemoryConsumer.java
index 9a9d0c7946549..1d361aeb24efc 100644
--- a/core/src/main/java/org/apache/spark/memory/MemoryConsumer.java
+++ b/core/src/main/java/org/apache/spark/memory/MemoryConsumer.java
@@ -40,8 +40,8 @@ protected MemoryConsumer(TaskMemoryManager taskMemoryManager, long pageSize, Mem
     this.mode = mode;
   }
 
-  protected MemoryConsumer(TaskMemoryManager taskMemoryManager) {
-    this(taskMemoryManager, taskMemoryManager.pageSizeBytes(), MemoryMode.ON_HEAP);
+  protected MemoryConsumer(TaskMemoryManager taskMemoryManager, MemoryMode mode) {
+    this(taskMemoryManager, taskMemoryManager.pageSizeBytes(), mode);
   }
 
   /**
diff --git a/core/src/main/java/org/apache/spark/shuffle/api/ShuffleMapOutputWriter.java b/core/src/main/java/org/apache/spark/shuffle/api/ShuffleMapOutputWriter.java
index 21abe9a57cd25..0167002ceedb8 100644
--- a/core/src/main/java/org/apache/spark/shuffle/api/ShuffleMapOutputWriter.java
+++ b/core/src/main/java/org/apache/spark/shuffle/api/ShuffleMapOutputWriter.java
@@ -20,6 +20,7 @@
 import java.io.IOException;
 
 import org.apache.spark.annotation.Private;
+import org.apache.spark.shuffle.api.metadata.MapOutputCommitMessage;
 
 /**
  * :: Private ::
@@ -60,10 +61,15 @@ public interface ShuffleMapOutputWriter {
    * <p>
    * This can also close any resources and clean up temporary state if necessary.
    * <p>
-   * The returned array should contain, for each partition from (0) to (numPartitions - 1), the
-   * number of bytes written by the partition writer for that partition id.
+   * The returned commit message is a structure with two components:
+   * <p>
+   * 1) An array of longs, which should contain, for each partition from (0) to
+   *    (numPartitions - 1), the number of bytes written by the partition writer
+   *    for that partition id.
+   * <p>
+   * 2) An optional metadata blob that can be used by shuffle readers.
    */
-  long[] commitAllPartitions() throws IOException;
+  MapOutputCommitMessage commitAllPartitions() throws IOException;
 
   /**
    * Abort all of the writes done by any writers returned by {@link #getPartitionWriter(int)}.
diff --git a/core/src/main/java/org/apache/spark/shuffle/api/WritableByteChannelWrapper.java b/core/src/main/java/org/apache/spark/shuffle/api/WritableByteChannelWrapper.java
index a204903008a51..04a75dde97cb4 100644
--- a/core/src/main/java/org/apache/spark/shuffle/api/WritableByteChannelWrapper.java
+++ b/core/src/main/java/org/apache/spark/shuffle/api/WritableByteChannelWrapper.java
@@ -24,7 +24,6 @@
 
 /**
  * :: Private ::
- *
  * A thin wrapper around a {@link WritableByteChannel}.
  * <p>
  * This is primarily provided for the local disk shuffle implementation to provide a
diff --git a/core/src/main/java/org/apache/spark/shuffle/api/metadata/MapOutputCommitMessage.java b/core/src/main/java/org/apache/spark/shuffle/api/metadata/MapOutputCommitMessage.java
new file mode 100644
index 0000000000000..c5ded5e75a2d7
--- /dev/null
+++ b/core/src/main/java/org/apache/spark/shuffle/api/metadata/MapOutputCommitMessage.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.shuffle.api.metadata;
+
+import java.util.Optional;
+
+import org.apache.spark.annotation.Private;
+
+/**
+ * :: Private ::
+ * Represents the result of writing map outputs for a shuffle map task.
+ * <p>
+ * Partition lengths represents the length of each block written in the map task. This can
+ * be used for downstream readers to allocate resources, such as in-memory buffers.
+ * <p>
+ * Map output writers can choose to attach arbitrary metadata tags to register with a
+ * shuffle output tracker (a module that is currently yet to be built in a future
+ * iteration of the shuffle storage APIs).
+ */
+@Private
+public final class MapOutputCommitMessage {
+
+  private final long[] partitionLengths;
+  private final Optional<MapOutputMetadata> mapOutputMetadata;
+
+  private MapOutputCommitMessage(
+      long[] partitionLengths, Optional<MapOutputMetadata> mapOutputMetadata) {
+    this.partitionLengths = partitionLengths;
+    this.mapOutputMetadata = mapOutputMetadata;
+  }
+
+  public static MapOutputCommitMessage of(long[] partitionLengths) {
+    return new MapOutputCommitMessage(partitionLengths, Optional.empty());
+  }
+
+  public static MapOutputCommitMessage of(
+      long[] partitionLengths, MapOutputMetadata mapOutputMetadata) {
+    return new MapOutputCommitMessage(partitionLengths, Optional.of(mapOutputMetadata));
+  }
+
+  public long[] getPartitionLengths() {
+    return partitionLengths;
+  }
+
+  public Optional<MapOutputMetadata> getMapOutputMetadata() {
+    return mapOutputMetadata;
+  }
+}
diff --git a/core/src/main/java/org/apache/spark/shuffle/api/metadata/MapOutputMetadata.java b/core/src/main/java/org/apache/spark/shuffle/api/metadata/MapOutputMetadata.java
new file mode 100644
index 0000000000000..f509686621602
--- /dev/null
+++ b/core/src/main/java/org/apache/spark/shuffle/api/metadata/MapOutputMetadata.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.shuffle.api.metadata;
+
+import java.io.Serializable;
+
+/**
+ * :: Private ::
+ * An opaque metadata tag for registering the result of committing the output of a
+ * shuffle map task.
+ * <p>
+ * All implementations must be serializable since this is sent from the executors to
+ * the driver.
+ */
+public interface MapOutputMetadata extends Serializable {}
diff --git a/core/src/main/java/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriter.java b/core/src/main/java/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriter.java
index dc157eaa3b253..3dbee1b13d287 100644
--- a/core/src/main/java/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriter.java
+++ b/core/src/main/java/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriter.java
@@ -31,7 +31,6 @@
 import scala.Tuple2;
 import scala.collection.Iterator;
 
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.io.Closeables;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -130,7 +129,7 @@ public void write(Iterator<Product2<K, V>> records) throws IOException {
         .createMapOutputWriter(shuffleId, mapId, numPartitions);
     try {
       if (!records.hasNext()) {
-        partitionLengths = mapOutputWriter.commitAllPartitions();
+        partitionLengths = mapOutputWriter.commitAllPartitions().getPartitionLengths();
         mapStatus = MapStatus$.MODULE$.apply(
           blockManager.shuffleServerId(), partitionLengths, mapId);
         return;
@@ -178,8 +177,8 @@ public void write(Iterator<Product2<K, V>> records) throws IOException {
     }
   }
 
-  @VisibleForTesting
-  long[] getPartitionLengths() {
+  @Override
+  public long[] getPartitionLengths() {
     return partitionLengths;
   }
 
@@ -219,7 +218,7 @@ private long[] writePartitionedData(ShuffleMapOutputWriter mapOutputWriter) thro
       }
       partitionWriters = null;
     }
-    return mapOutputWriter.commitAllPartitions();
+    return mapOutputWriter.commitAllPartitions().getPartitionLengths();
   }
 
   private void writePartitionedDataWithChannel(
diff --git a/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java b/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java
index d09282e61a9c7..e8f94ba8ffeee 100644
--- a/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java
+++ b/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java
@@ -18,6 +18,7 @@
 package org.apache.spark.shuffle.sort;
 
 import java.nio.channels.Channels;
+import java.util.Arrays;
 import java.util.Optional;
 import javax.annotation.Nullable;
 import java.io.*;
@@ -87,6 +88,7 @@ public class UnsafeShuffleWriter<K, V> extends ShuffleWriter<K, V> {
 
   @Nullable private MapStatus mapStatus;
   @Nullable private ShuffleExternalSorter sorter;
+  @Nullable private long[] partitionLengths;
   private long peakMemoryUsedBytes = 0;
 
   /** Subclass of ByteArrayOutputStream that exposes `buf` directly. */
@@ -218,7 +220,6 @@ void closeAndWriteOutput() throws IOException {
     serOutputStream = null;
     final SpillInfo[] spills = sorter.closeAndGetSpills();
     sorter = null;
-    final long[] partitionLengths;
     try {
       partitionLengths = mergeSpills(spills);
     } finally {
@@ -266,7 +267,7 @@ private long[] mergeSpills(SpillInfo[] spills) throws IOException {
     if (spills.length == 0) {
       final ShuffleMapOutputWriter mapWriter = shuffleExecutorComponents
           .createMapOutputWriter(shuffleId, mapId, partitioner.numPartitions());
-      return mapWriter.commitAllPartitions();
+      return mapWriter.commitAllPartitions().getPartitionLengths();
     } else if (spills.length == 1) {
       Optional<SingleSpillShuffleMapOutputWriter> maybeSingleFileWriter =
           shuffleExecutorComponents.createSingleFileMapOutputWriter(shuffleId, mapId);
@@ -274,6 +275,8 @@ private long[] mergeSpills(SpillInfo[] spills) throws IOException {
         // Here, we don't need to perform any metrics updates because the bytes written to this
         // output file would have already been counted as shuffle bytes written.
         partitionLengths = spills[0].partitionLengths;
+        logger.debug("Merge shuffle spills for mapId {} with length {}", mapId,
+            partitionLengths.length);
         maybeSingleFileWriter.get().transferMapSpillFile(spills[0].file, partitionLengths);
       } else {
         partitionLengths = mergeSpillsUsingStandardWriter(spills);
@@ -327,7 +330,7 @@ private long[] mergeSpillsUsingStandardWriter(SpillInfo[] spills) throws IOExcep
       // to be counted as shuffle write, but this will lead to double-counting of the final
       // SpillInfo's bytes.
       writeMetrics.decBytesWritten(spills[spills.length - 1].file.length());
-      partitionLengths = mapWriter.commitAllPartitions();
+      partitionLengths = mapWriter.commitAllPartitions().getPartitionLengths();
     } catch (Exception e) {
       try {
         mapWriter.abort(e);
@@ -360,6 +363,7 @@ private void mergeSpillsWithFileStream(
       SpillInfo[] spills,
       ShuffleMapOutputWriter mapWriter,
       @Nullable CompressionCodec compressionCodec) throws IOException {
+    logger.debug("Merge shuffle spills with FileStream for mapId {}", mapId);
     final int numPartitions = partitioner.numPartitions();
     final InputStream[] spillInputStreams = new InputStream[spills.length];
 
@@ -369,6 +373,11 @@ private void mergeSpillsWithFileStream(
         spillInputStreams[i] = new NioBufferedFileInputStream(
           spills[i].file,
           inputBufferSizeInBytes);
+        // Only convert the partitionLengths when debug level is enabled.
+        if (logger.isDebugEnabled()) {
+          logger.debug("Partition lengths for mapId {} in Spill {}: {}", mapId, i,
+              Arrays.toString(spills[i].partitionLengths));
+        }
       }
       for (int partition = 0; partition < numPartitions; partition++) {
         boolean copyThrewException = true;
@@ -431,6 +440,7 @@ private void mergeSpillsWithFileStream(
   private void mergeSpillsWithTransferTo(
       SpillInfo[] spills,
       ShuffleMapOutputWriter mapWriter) throws IOException {
+    logger.debug("Merge shuffle spills with TransferTo for mapId {}", mapId);
     final int numPartitions = partitioner.numPartitions();
     final FileChannel[] spillInputChannels = new FileChannel[spills.length];
     final long[] spillInputChannelPositions = new long[spills.length];
@@ -439,6 +449,11 @@ private void mergeSpillsWithTransferTo(
     try {
       for (int i = 0; i < spills.length; i++) {
         spillInputChannels[i] = new FileInputStream(spills[i].file).getChannel();
+        // Only convert the partitionLengths when debug level is enabled.
+        if (logger.isDebugEnabled()) {
+          logger.debug("Partition lengths for mapId {} in Spill {}: {}", mapId, i,
+              Arrays.toString(spills[i].partitionLengths));
+        }
       }
       for (int partition = 0; partition < numPartitions; partition++) {
         boolean copyThrewException = true;
@@ -528,4 +543,9 @@ public void close() throws IOException {
       channel.close();
     }
   }
+
+  @Override
+  public long[] getPartitionLengths() {
+    return partitionLengths;
+  }
 }
diff --git a/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriter.java b/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriter.java
index a6529fd76188a..0b286264be43d 100644
--- a/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriter.java
+++ b/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriter.java
@@ -35,6 +35,7 @@
 import org.apache.spark.shuffle.api.WritableByteChannelWrapper;
 import org.apache.spark.internal.config.package$;
 import org.apache.spark.shuffle.IndexShuffleBlockResolver;
+import org.apache.spark.shuffle.api.metadata.MapOutputCommitMessage;
 import org.apache.spark.util.Utils;
 
 /**
@@ -97,7 +98,7 @@ public ShufflePartitionWriter getPartitionWriter(int reducePartitionId) throws I
   }
 
   @Override
-  public long[] commitAllPartitions() throws IOException {
+  public MapOutputCommitMessage commitAllPartitions() throws IOException {
     // Check the position after transferTo loop to see if it is in the right position and raise a
     // exception if it is incorrect. The position will not be increased to the expected length
     // after calling transferTo in kernel version 2.6.32. This issue is described at
@@ -112,8 +113,10 @@ public long[] commitAllPartitions() throws IOException {
     }
     cleanUp();
     File resolvedTmp = outputTempFile != null && outputTempFile.isFile() ? outputTempFile : null;
+    log.debug("Writing shuffle index file for mapId {} with length {}", mapId,
+        partitionLengths.length);
     blockResolver.writeIndexFileAndCommit(shuffleId, mapId, partitionLengths, resolvedTmp);
-    return partitionLengths;
+    return MapOutputCommitMessage.of(partitionLengths);
   }
 
   @Override
@@ -210,14 +213,14 @@ public long getNumBytesWritten() {
 
   private class PartitionWriterStream extends OutputStream {
     private final int partitionId;
-    private int count = 0;
+    private long count = 0;
     private boolean isClosed = false;
 
     PartitionWriterStream(int partitionId) {
       this.partitionId = partitionId;
     }
 
-    public int getCount() {
+    public long getCount() {
       return count;
     }
 
diff --git a/core/src/main/java/org/apache/spark/status/api/v1/TaskStatus.java b/core/src/main/java/org/apache/spark/status/api/v1/TaskStatus.java
new file mode 100644
index 0000000000000..dec9c31321839
--- /dev/null
+++ b/core/src/main/java/org/apache/spark/status/api/v1/TaskStatus.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.status.api.v1;
+
+import org.apache.spark.util.EnumUtil;
+
+public enum TaskStatus {
+  RUNNING,
+  KILLED,
+  FAILED,
+  SUCCESS,
+  UNKNOWN;
+
+  public static TaskStatus fromString(String str) {
+    return EnumUtil.parseIgnoreCase(TaskStatus.class, str);
+  }
+}
diff --git a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
index 6e028886f2318..f474c30b8b3d8 100644
--- a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
+++ b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
@@ -393,10 +393,12 @@ public void remove() {
     }
 
     private void handleFailedDelete() {
-      // remove the spill file from disk
-      File file = spillWriters.removeFirst().getFile();
-      if (file != null && file.exists() && !file.delete()) {
-        logger.error("Was unable to delete spill file {}", file.getAbsolutePath());
+      if (spillWriters.size() > 0) {
+        // remove the spill file from disk
+        File file = spillWriters.removeFirst().getFile();
+        if (file != null && file.exists() && !file.delete()) {
+          logger.error("Was unable to delete spill file {}", file.getAbsolutePath());
+        }
       }
     }
   }
@@ -428,6 +430,68 @@ public MapIterator destructiveIterator() {
     return new MapIterator(numValues, new Location(), true);
   }
 
+  /**
+   * Iterator for the entries of this map. This is to first iterate over key indices in
+   * `longArray` then accessing values in `dataPages`. NOTE: this is different from `MapIterator`
+   * in the sense that key index is preserved here
+   * (See `UnsafeHashedRelation` for example of usage).
+   */
+  public final class MapIteratorWithKeyIndex implements Iterator<Location> {
+
+    /**
+     * The index in `longArray` where the key is stored.
+     */
+    private int keyIndex = 0;
+
+    private int numRecords;
+    private final Location loc;
+
+    private MapIteratorWithKeyIndex() {
+      this.numRecords = numValues;
+      this.loc = new Location();
+    }
+
+    @Override
+    public boolean hasNext() {
+      return numRecords > 0;
+    }
+
+    @Override
+    public Location next() {
+      if (!loc.isDefined() || !loc.nextValue()) {
+        while (longArray.get(keyIndex * 2) == 0) {
+          keyIndex++;
+        }
+        loc.with(keyIndex, 0, true);
+        keyIndex++;
+      }
+      numRecords--;
+      return loc;
+    }
+  }
+
+  /**
+   * Returns an iterator for iterating over the entries of this map,
+   * by first iterating over the key index inside hash map's `longArray`.
+   *
+   * For efficiency, all calls to `next()` will return the same {@link Location} object.
+   *
+   * The returned iterator is NOT thread-safe. If the map is modified while iterating over it,
+   * the behavior of the returned iterator is undefined.
+   */
+  public MapIteratorWithKeyIndex iteratorWithKeyIndex() {
+    return new MapIteratorWithKeyIndex();
+  }
+
+  /**
+   * The maximum number of allowed keys index.
+   *
+   * The value of allowed keys index is in the range of [0, maxNumKeysIndex - 1].
+   */
+  public int maxNumKeysIndex() {
+    return (int) (longArray.size() / 2);
+  }
+
   /**
    * Looks up a key, and return a {@link Location} handle that can be used to test existence
    * and read/write values.
@@ -601,6 +665,14 @@ public boolean isDefined() {
       return isDefined;
     }
 
+    /**
+     * Returns index for key.
+     */
+    public int getKeyIndex() {
+      assert (isDefined);
+      return pos;
+    }
+
     /**
      * Returns the base object for key.
      */
@@ -738,12 +810,21 @@ public boolean append(Object kbase, long koff, int klen, Object vbase, long voff
         longArray.set(pos * 2 + 1, keyHashcode);
         isDefined = true;
 
-        // We use two array entries per key, so the array size is twice the capacity.
-        // We should compare the current capacity of the array, instead of its size.
-        if (numKeys >= growthThreshold && longArray.size() / 2 < MAX_CAPACITY) {
-          try {
-            growAndRehash();
-          } catch (SparkOutOfMemoryError oom) {
+        // If the map has reached its growth threshold, try to grow it.
+        if (numKeys >= growthThreshold) {
+          // We use two array entries per key, so the array size is twice the capacity.
+          // We should compare the current capacity of the array, instead of its size.
+          if (longArray.size() / 2 < MAX_CAPACITY) {
+            try {
+              growAndRehash();
+            } catch (SparkOutOfMemoryError oom) {
+              canGrowArray = false;
+            }
+          } else {
+            // The map is already at MAX_CAPACITY and cannot grow. Instead, we prevent it from
+            // accepting any more new elements to make sure we don't exceed the load factor. If we
+            // need to spill later, this allows UnsafeKVExternalSorter to reuse the array for
+            // sorting.
             canGrowArray = false;
           }
         }
diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java
index 55e4e609c3c7b..c38327cae8ce3 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java
@@ -104,11 +104,14 @@ public static UnsafeExternalSorter createWithExistingInMemorySorter(
       int initialSize,
       long pageSizeBytes,
       int numElementsForSpillThreshold,
-      UnsafeInMemorySorter inMemorySorter) throws IOException {
+      UnsafeInMemorySorter inMemorySorter,
+      long existingMemoryConsumption) throws IOException {
     UnsafeExternalSorter sorter = new UnsafeExternalSorter(taskMemoryManager, blockManager,
       serializerManager, taskContext, recordComparatorSupplier, prefixComparator, initialSize,
         pageSizeBytes, numElementsForSpillThreshold, inMemorySorter, false /* ignored */);
     sorter.spill(Long.MAX_VALUE, sorter);
+    taskContext.taskMetrics().incMemoryBytesSpilled(existingMemoryConsumption);
+    sorter.totalSpillBytes += existingMemoryConsumption;
     // The external sorter will be used to insert records, in-memory sorter is not needed.
     sorter.inMemSorter = null;
     return sorter;
@@ -203,6 +206,10 @@ public long spill(long size, MemoryConsumer trigger) throws IOException {
     }
 
     if (inMemSorter == null || inMemSorter.numRecords() <= 0) {
+      // There could still be some memory allocated when there are no records in the in-memory
+      // sorter. We will not spill it however, to ensure that we can always process at least one
+      // record before spilling. See the comments in `allocateMemoryForRecordIfNecessary` for why
+      // this is necessary.
       return 0L;
     }
 
@@ -224,7 +231,7 @@ public long spill(long size, MemoryConsumer trigger) throws IOException {
     // Note that this is more-or-less going to be a multiple of the page size, so wasted space in
     // pages will currently be counted as memory spilled even though that space isn't actually
     // written to disk. This also counts the space needed to store the sorter's pointer array.
-    inMemSorter.reset();
+    inMemSorter.freeMemory();
     // Reset the in-memory sorter's pointer array only after freeing up the memory pages holding the
     // records. Otherwise, if the task is over allocated memory, then without freeing the memory
     // pages, we might not be able to get memory for the pointer array.
@@ -325,7 +332,7 @@ public void cleanupResources() {
       deleteSpillFiles();
       freeMemory();
       if (inMemSorter != null) {
-        inMemSorter.free();
+        inMemSorter.freeMemory();
         inMemSorter = null;
       }
     }
@@ -339,40 +346,53 @@ public void cleanupResources() {
   private void growPointerArrayIfNecessary() throws IOException {
     assert(inMemSorter != null);
     if (!inMemSorter.hasSpaceForAnotherRecord()) {
+      if (inMemSorter.numRecords() <= 0) {
+        // Spilling was triggered just before this method was called. The pointer array was freed
+        // during the spill, so a new pointer array needs to be allocated here.
+        LongArray array = allocateArray(inMemSorter.getInitialSize());
+        inMemSorter.expandPointerArray(array);
+        return;
+      }
+
       long used = inMemSorter.getMemoryUsage();
-      LongArray array;
+      LongArray array = null;
       try {
         // could trigger spilling
         array = allocateArray(used / 8 * 2);
       } catch (TooLargePageException e) {
         // The pointer array is too big to fix in a single page, spill.
         spill();
-        return;
       } catch (SparkOutOfMemoryError e) {
-        // should have trigger spilling
-        if (!inMemSorter.hasSpaceForAnotherRecord()) {
+        if (inMemSorter.numRecords() > 0) {
           logger.error("Unable to grow the pointer array");
           throw e;
         }
-        return;
+        // The new array could not be allocated, but that is not an issue as it is longer needed,
+        // as all records were spilled.
       }
-      // check if spilling is triggered or not
-      if (inMemSorter.hasSpaceForAnotherRecord()) {
-        freeArray(array);
-      } else {
-        inMemSorter.expandPointerArray(array);
+
+      if (inMemSorter.numRecords() <= 0) {
+        // Spilling was triggered while trying to allocate the new array.
+        if (array != null) {
+          // We succeeded in allocating the new array, but, since all records were spilled, a
+          // smaller array would also suffice.
+          freeArray(array);
+        }
+        // The pointer array was freed during the spill, so a new pointer array needs to be
+        // allocated here.
+        array = allocateArray(inMemSorter.getInitialSize());
       }
+      inMemSorter.expandPointerArray(array);
     }
   }
 
   /**
-   * Allocates more memory in order to insert an additional record. This will request additional
-   * memory from the memory manager and spill if the requested memory can not be obtained.
+   * Allocates an additional page in order to insert an additional record. This will request
+   * additional memory from the memory manager and spill if the requested memory can not be
+   * obtained.
    *
    * @param required the required space in the data page, in bytes, including space for storing
-   *                      the record size. This must be less than or equal to the page size (records
-   *                      that exceed the page size are handled via a different code path which uses
-   *                      special overflow pages).
+   *                 the record size.
    */
   private void acquireNewPageIfNecessary(int required) {
     if (currentPage == null ||
@@ -384,6 +404,37 @@ private void acquireNewPageIfNecessary(int required) {
     }
   }
 
+  /**
+   * Allocates more memory in order to insert an additional record. This will request additional
+   * memory from the memory manager and spill if the requested memory can not be obtained.
+   *
+   * @param required the required space in the data page, in bytes, including space for storing
+   *                 the record size.
+   */
+  private void allocateMemoryForRecordIfNecessary(int required) throws IOException {
+    // Step 1:
+    // Ensure that the pointer array has space for another record. This may cause a spill.
+    growPointerArrayIfNecessary();
+    // Step 2:
+    // Ensure that the last page has space for another record. This may cause a spill.
+    acquireNewPageIfNecessary(required);
+    // Step 3:
+    // The allocation in step 2 could have caused a spill, which would have freed the pointer
+    // array allocated in step 1. Therefore we need to check again whether we have to allocate
+    // a new pointer array.
+    //
+    // If the allocation in this step causes a spill event then it will not cause the page
+    // allocated in the previous step to be freed. The function `spill` only frees memory if at
+    // least one record has been inserted in the in-memory sorter. This will not be the case if
+    // we have spilled in the previous step.
+    //
+    // If we did not spill in the previous step then `growPointerArrayIfNecessary` will be a
+    // no-op that does not allocate any memory, and therefore can't cause a spill event.
+    //
+    // Thus there is no need to call `acquireNewPageIfNecessary` again after this step.
+    growPointerArrayIfNecessary();
+  }
+
   /**
    * Write a record to the sorter.
    */
@@ -398,11 +449,10 @@ public void insertRecord(
       spill();
     }
 
-    growPointerArrayIfNecessary();
-    int uaoSize = UnsafeAlignedOffset.getUaoSize();
+    final int uaoSize = UnsafeAlignedOffset.getUaoSize();
     // Need 4 or 8 bytes to store the record length.
     final int required = length + uaoSize;
-    acquireNewPageIfNecessary(required);
+    allocateMemoryForRecordIfNecessary(required);
 
     final Object base = currentPage.getBaseObject();
     final long recordAddress = taskMemoryManager.encodePageNumberAndOffset(currentPage, pageCursor);
@@ -425,10 +475,9 @@ public void insertKVRecord(Object keyBase, long keyOffset, int keyLen,
       Object valueBase, long valueOffset, int valueLen, long prefix, boolean prefixIsNull)
     throws IOException {
 
-    growPointerArrayIfNecessary();
-    int uaoSize = UnsafeAlignedOffset.getUaoSize();
+    final int uaoSize = UnsafeAlignedOffset.getUaoSize();
     final int required = keyLen + valueLen + (2 * uaoSize);
-    acquireNewPageIfNecessary(required);
+    allocateMemoryForRecordIfNecessary(required);
 
     final Object base = currentPage.getBaseObject();
     final long recordAddress = taskMemoryManager.encodePageNumberAndOffset(currentPage, pageCursor);
@@ -450,6 +499,7 @@ public void insertKVRecord(Object keyBase, long keyOffset, int keyLen,
    */
   public void merge(UnsafeExternalSorter other) throws IOException {
     other.spill();
+    totalSpillBytes += other.totalSpillBytes;
     spillWriters.addAll(other.spillWriters);
     // remove them from `spillWriters`, or the files will be deleted in `cleanupResources`.
     other.spillWriters.clear();
@@ -501,10 +551,14 @@ private static void spillIterator(UnsafeSorterIterator inMemIterator,
    */
   class SpillableIterator extends UnsafeSorterIterator {
     private UnsafeSorterIterator upstream;
-    private UnsafeSorterIterator nextUpstream = null;
     private MemoryBlock lastPage = null;
     private boolean loaded = false;
-    private int numRecords = 0;
+    private int numRecords;
+
+    private Object currentBaseObject;
+    private long currentBaseOffset;
+    private int currentRecordLength;
+    private long currentKeyPrefix;
 
     SpillableIterator(UnsafeSorterIterator inMemIterator) {
       this.upstream = inMemIterator;
@@ -516,23 +570,32 @@ public int getNumRecords() {
       return numRecords;
     }
 
+    @Override
+    public long getCurrentPageNumber() {
+      throw new UnsupportedOperationException();
+    }
+
     public long spill() throws IOException {
       synchronized (this) {
-        if (!(upstream instanceof UnsafeInMemorySorter.SortedIterator && nextUpstream == null
-          && numRecords > 0)) {
+        if (inMemSorter == null) {
           return 0L;
         }
 
-        UnsafeInMemorySorter.SortedIterator inMemIterator =
-          ((UnsafeInMemorySorter.SortedIterator) upstream).clone();
+        long currentPageNumber = upstream.getCurrentPageNumber();
 
-       ShuffleWriteMetrics writeMetrics = new ShuffleWriteMetrics();
-        // Iterate over the records that have not been returned and spill them.
-        final UnsafeSorterSpillWriter spillWriter =
-          new UnsafeSorterSpillWriter(blockManager, fileBufferSizeBytes, writeMetrics, numRecords);
-        spillIterator(inMemIterator, spillWriter);
-        spillWriters.add(spillWriter);
-        nextUpstream = spillWriter.getReader(serializerManager);
+        ShuffleWriteMetrics writeMetrics = new ShuffleWriteMetrics();
+        if (numRecords > 0) {
+          // Iterate over the records that have not been returned and spill them.
+          final UnsafeSorterSpillWriter spillWriter = new UnsafeSorterSpillWriter(
+                  blockManager, fileBufferSizeBytes, writeMetrics, numRecords);
+          spillIterator(upstream, spillWriter);
+          spillWriters.add(spillWriter);
+          upstream = spillWriter.getReader(serializerManager);
+        } else {
+          // Nothing to spill as all records have been read already, but do not return yet, as the
+          // memory still has to be freed.
+          upstream = null;
+        }
 
         long released = 0L;
         synchronized (UnsafeExternalSorter.this) {
@@ -540,8 +603,7 @@ public long spill() throws IOException {
           // is accessing the current record. We free this page in that caller's next loadNext()
           // call.
           for (MemoryBlock page : allocatedPages) {
-            if (!loaded || page.pageNumber !=
-                    ((UnsafeInMemorySorter.SortedIterator)upstream).getCurrentPageNumber()) {
+            if (!loaded || page.pageNumber != currentPageNumber) {
               released += page.size();
               freePage(page);
             } else {
@@ -549,13 +611,18 @@ public long spill() throws IOException {
             }
           }
           allocatedPages.clear();
+          if (lastPage != null) {
+            // Add the last page back to the list of allocated pages to make sure it gets freed in
+            // case loadNext() never gets called again.
+            allocatedPages.add(lastPage);
+          }
         }
 
         // in-memory sorter will not be used after spilling
         assert(inMemSorter != null);
         released += inMemSorter.getMemoryUsage();
         totalSortTimeNanos += inMemSorter.getSortTimeNanos();
-        inMemSorter.free();
+        inMemSorter.freeMemory();
         inMemSorter = null;
         taskContext.taskMetrics().incMemoryBytesSpilled(released);
         taskContext.taskMetrics().incDiskBytesSpilled(writeMetrics.bytesWritten());
@@ -571,26 +638,32 @@ public boolean hasNext() {
 
     @Override
     public void loadNext() throws IOException {
+      assert upstream != null;
       MemoryBlock pageToFree = null;
       try {
         synchronized (this) {
           loaded = true;
-          if (nextUpstream != null) {
-            // Just consumed the last record from in memory iterator
-            if(lastPage != null) {
-              // Do not free the page here, while we are locking `SpillableIterator`. The `freePage`
-              // method locks the `TaskMemoryManager`, and it's a bad idea to lock 2 objects in
-              // sequence. We may hit dead lock if another thread locks `TaskMemoryManager` and
-              // `SpillableIterator` in sequence, which may happen in
-              // `TaskMemoryManager.acquireExecutionMemory`.
-              pageToFree = lastPage;
-              lastPage = null;
-            }
-            upstream = nextUpstream;
-            nextUpstream = null;
+          // Just consumed the last record from the in-memory iterator.
+          if (lastPage != null) {
+            // Do not free the page here, while we are locking `SpillableIterator`. The `freePage`
+            // method locks the `TaskMemoryManager`, and it's a bad idea to lock 2 objects in
+            // sequence. We may hit dead lock if another thread locks `TaskMemoryManager` and
+            // `SpillableIterator` in sequence, which may happen in
+            // `TaskMemoryManager.acquireExecutionMemory`.
+            pageToFree = lastPage;
+            allocatedPages.clear();
+            lastPage = null;
           }
           numRecords--;
           upstream.loadNext();
+
+          // Keep track of the current base object, base offset, record length, and key prefix,
+          // so that the current record can still be read in case a spill is triggered and we
+          // switch to the spill writer's iterator.
+          currentBaseObject = upstream.getBaseObject();
+          currentBaseOffset = upstream.getBaseOffset();
+          currentRecordLength = upstream.getRecordLength();
+          currentKeyPrefix = upstream.getKeyPrefix();
         }
       } finally {
         if (pageToFree != null) {
@@ -601,22 +674,22 @@ public void loadNext() throws IOException {
 
     @Override
     public Object getBaseObject() {
-      return upstream.getBaseObject();
+      return currentBaseObject;
     }
 
     @Override
     public long getBaseOffset() {
-      return upstream.getBaseOffset();
+      return currentBaseOffset;
     }
 
     @Override
     public int getRecordLength() {
-      return upstream.getRecordLength();
+      return currentRecordLength;
     }
 
     @Override
     public long getKeyPrefix() {
-      return upstream.getKeyPrefix();
+      return currentKeyPrefix;
     }
   }
 
@@ -646,7 +719,7 @@ public UnsafeSorterIterator getIterator(int startIndex) throws IOException {
         }
         i += spillWriter.recordsSpilled();
       }
-      if (inMemSorter != null) {
+      if (inMemSorter != null && inMemSorter.numRecords() > 0) {
         UnsafeSorterIterator iter = inMemSorter.getSortedIterator();
         moveOver(iter, startIndex - i);
         queue.add(iter);
@@ -693,6 +766,11 @@ public int getNumRecords() {
       return numRecords;
     }
 
+    @Override
+    public long getCurrentPageNumber() {
+      return current.getCurrentPageNumber();
+    }
+
     @Override
     public boolean hasNext() {
       while (!current.hasNext() && !iterators.isEmpty()) {
diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java
index 660eb790a550b..765ee035855d6 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java
@@ -20,7 +20,7 @@
 import java.util.Comparator;
 import java.util.LinkedList;
 
-import org.apache.avro.reflect.Nullable;
+import javax.annotation.Nullable;
 
 import org.apache.spark.TaskContext;
 import org.apache.spark.memory.MemoryConsumer;
@@ -159,32 +159,26 @@ private int getUsableCapacity() {
     return (int) (array.size() / (radixSortSupport != null ? 2 : 1.5));
   }
 
+  public long getInitialSize() {
+    return initialSize;
+  }
+
   /**
    * Free the memory used by pointer array.
    */
-  public void free() {
+  public void freeMemory() {
     if (consumer != null) {
       if (array != null) {
         consumer.freeArray(array);
       }
-      array = null;
-    }
-  }
 
-  public void reset() {
-    if (consumer != null) {
-      consumer.freeArray(array);
-      // the call to consumer.allocateArray may trigger a spill which in turn access this instance
-      // and eventually re-enter this method and try to free the array again.  by setting the array
-      // to null and its length to 0 we effectively make the spill code-path a no-op.  setting the
-      // array to null also indicates that it has already been de-allocated which prevents a double
-      // de-allocation in free().
+      // Set the array to null instead of allocating a new array. Allocating an array could have
+      // triggered another spill and this method already is called from UnsafeExternalSorter when
+      // spilling. Attempting to allocate while spilling is dangerous, as we could be holding onto
+      // a large partially complete allocation, which may prevent other memory from being allocated.
+      // Instead we will allocate the new array when it is necessary.
       array = null;
       usableCapacity = 0;
-      pos = 0;
-      nullBoundaryPos = 0;
-      array = consumer.allocateArray(initialSize);
-      usableCapacity = getUsableCapacity();
     }
     pos = 0;
     nullBoundaryPos = 0;
@@ -217,18 +211,20 @@ public boolean hasSpaceForAnotherRecord() {
   }
 
   public void expandPointerArray(LongArray newArray) {
-    if (newArray.size() < array.size()) {
-      // checkstyle.off: RegexpSinglelineJava
-      throw new SparkOutOfMemoryError("Not enough memory to grow pointer array");
-      // checkstyle.on: RegexpSinglelineJava
+    if (array != null) {
+      if (newArray.size() < array.size()) {
+        // checkstyle.off: RegexpSinglelineJava
+        throw new SparkOutOfMemoryError("Not enough memory to grow pointer array");
+        // checkstyle.on: RegexpSinglelineJava
+      }
+      Platform.copyMemory(
+        array.getBaseObject(),
+        array.getBaseOffset(),
+        newArray.getBaseObject(),
+        newArray.getBaseOffset(),
+        pos * 8L);
+      consumer.freeArray(array);
     }
-    Platform.copyMemory(
-      array.getBaseObject(),
-      array.getBaseOffset(),
-      newArray.getBaseObject(),
-      newArray.getBaseOffset(),
-      pos * 8L);
-    consumer.freeArray(array);
     array = newArray;
     usableCapacity = getUsableCapacity();
   }
@@ -330,6 +326,7 @@ public void loadNext() {
     @Override
     public long getBaseOffset() { return baseOffset; }
 
+    @Override
     public long getCurrentPageNumber() {
       return currentPageNumber;
     }
@@ -346,6 +343,11 @@ public long getCurrentPageNumber() {
    * {@code next()} will return the same mutable object.
    */
   public UnsafeSorterIterator getSortedIterator() {
+    if (numRecords() == 0) {
+      // `array` might be null, so make sure that it is not accessed by returning early.
+      return new SortedIterator(0, 0);
+    }
+
     int offset = 0;
     long start = System.nanoTime();
     if (sortComparator != null) {
diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterIterator.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterIterator.java
index 1b3167fcc250c..d9f22311d07c2 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterIterator.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterIterator.java
@@ -34,4 +34,6 @@ public abstract class UnsafeSorterIterator {
   public abstract long getKeyPrefix();
 
   public abstract int getNumRecords();
+
+  public abstract long getCurrentPageNumber();
 }
diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillMerger.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillMerger.java
index ab800288dcb43..f8603c5799e9b 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillMerger.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillMerger.java
@@ -70,6 +70,11 @@ public int getNumRecords() {
         return numRecords;
       }
 
+      @Override
+      public long getCurrentPageNumber() {
+        throw new UnsupportedOperationException();
+      }
+
       @Override
       public boolean hasNext() {
         return !priorityQueue.isEmpty() || (spillReader != null && spillReader.hasNext());
diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java
index a524c4790407d..db79efd008530 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java
@@ -89,6 +89,11 @@ public int getNumRecords() {
     return numRecords;
   }
 
+  @Override
+  public long getCurrentPageNumber() {
+    throw new UnsupportedOperationException();
+  }
+
   @Override
   public boolean hasNext() {
     return (numRecordsRemaining > 0);
diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html b/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html
index 0b26bfc5b2d82..37d56a06ded7f 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html
+++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html
@@ -56,8 +56,8 @@ <h4 class="title-table">Summary</h4>
         </th>
         <th>
           <span data-toggle="tooltip" data-placement="top"
-                title="Number of executors blacklisted by the scheduler due to task failures.">
-            Blacklisted</span>
+                title="Number of executors excluded by the scheduler due to task failures.">
+            Excluded</span>
         </th>
           </tr>
         </thead>
@@ -86,9 +86,26 @@ <h4 class="title-table">Executors</h4>
             <span data-toggle="tooltip" data-placement="top"
                   title="Memory used / total available memory for off heap storage of data like RDD partitions cached in memory.">
               Off Heap Storage Memory</span></th>
+          <th>
+            <span data-toggle="tooltip" data-placement="top"
+                  title="Peak onHeap / offHeap memory used by JVM.">
+              Peak JVM Memory OnHeap / OffHeap</span></th>
+          <th>
+            <span data-toggle="tooltip" data-placement="top"
+                  title="Peak onHeap / offHeap memory used for execution. This refers to memory used for computation in shuffles, joins, user data structures, etc. See the Memory Management Overview documentation for more details.">
+              Peak Execution Memory OnHeap / OffHeap</span></th>
+          <th>
+            <span data-toggle="tooltip" data-placement="top"
+            title="Peak storage onHeap / offHeap memory used for storage of data like RDD partitions cached in memory.">
+              Peak Storage Memory OnHeap / OffHeap</span></th>
+          <th>
+            <span data-toggle="tooltip" data-placement="top"
+                  title="Peak direct byte buffer / memory-mapped buffer pool memory used by JVM. This refers to BufferPoolMXBean with form 'java.nio:type=BufferPool,name=direct' and 'java.nio:type=BufferPool,name=mapped'.">
+              Peak Pool Memory Direct / Mapped</span></th>
           <th>Disk Used</th>
           <th>Cores</th>
-          <th>Resources</th>
+          <th><span data-toggle="tooltip" data-placement="top" title="Resources currently used by each executor, such as GPU, FPGA, etc.">Resources</span></th>
+          <th>Resource Profile Id</th>
           <th><span data-toggle="tooltip" data-placement="top" title="Number of tasks currently executing. Darker shading highlights executors with more active tasks.">Active Tasks</span></th>
           <th><span data-toggle="tooltip" data-placement="top" title="Number of tasks that have failed on this executor. Darker shading highlights executors with a high proportion of failed tasks.">Failed Tasks</span></th>
           <th>Complete Tasks</th>
@@ -111,10 +128,28 @@ <h4 class="title-table">Executors</h4>
               Shuffle Write</span></th>
           <th>Logs</th>
           <th>Thread Dump</th>
+          <th>Exec Loss Reason</th>
         </tr>
         </thead>
         <tbody>
         </tbody>
       </table>
   </div>
+
+  <div class="container-fluid active-process-container">
+    <h4 class="title-table">Miscellaneous Process</h4>
+    <table id="active-process-table" class="table table-striped compact cell-border" style="width: 100%">
+      <thead>
+      <tr>
+        <th>Process ID</th>
+        <th>Address</th>
+        <th>Status</th>
+        <th>Cores</th>
+        <th>Logs</th>
+      </tr>
+      </thead>
+      <tbody>
+      </tbody>
+    </table>
+  </div>
 </script>
diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
index ec57797ba0909..ab412a8589a28 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
@@ -15,607 +15,759 @@
  * limitations under the License.
  */
 
+/* global $, Mustache, createRESTEndPointForExecutorsPage, createRESTEndPointForMiscellaneousProcess, */
+/* global createTemplateURI, formatBytes, formatDuration, formatLogsCells, getStandAloneAppId, */
+/* global jQuery, setDataTableDefaults */
+
 var threadDumpEnabled = false;
 
+/* eslint-disable no-unused-vars */
 function setThreadDumpEnabled(val) {
-    threadDumpEnabled = val;
+  threadDumpEnabled = val;
 }
+/* eslint-enable no-unused-vars */
 
 function getThreadDumpEnabled() {
-    return threadDumpEnabled;
+  return threadDumpEnabled;
+}
+
+function formatLossReason(removeReason) {
+  if (removeReason) {
+    return removeReason
+  } else {
+    return ""
+  }
 }
 
 function formatStatus(status, type, row) {
-    if (row.isBlacklisted) {
-        return "Blacklisted";
-    }
+  if (row.isExcluded) {
+    return "Excluded";
+  }
 
-    if (status) {
-        if (row.blacklistedInStages.length == 0) {
-            return "Active"
-        }
-        return "Active (Blacklisted in Stages: [" + row.blacklistedInStages.join(", ") + "])";
+  if (status) {
+    if (row.excludedInStages.length == 0) {
+      return "Active"
     }
-    return "Dead"
+    return "Active (Excluded in Stages: [" + row.excludedInStages.join(", ") + "])";
+  }
+  return "Dead"
+}
+
+function formatProcessStatus(activeStatus) {
+  if (activeStatus) {
+    return "Active"
+  }
+  return "Dead"
 }
 
 function formatResourceCells(resources) {
-    var result = ""
-    var count = 0
-    $.each(resources, function (name, resInfo) {
-        if (count > 0) {
-            result += ", "
-        }
-        result += name + ': [' + resInfo.addresses.join(", ") + ']'
-        count += 1
-    });
-    return result
+  var result = ""
+  var count = 0
+  $.each(resources, function (name, resInfo) {
+    if (count > 0) {
+      result += ", "
+    }
+    result += name + ': [' + resInfo.addresses.join(", ") + ']';
+    count += 1
+  });
+  return result
 }
 
 jQuery.extend(jQuery.fn.dataTableExt.oSort, {
-    "title-numeric-pre": function (a) {
-        var x = a.match(/title="*(-?[0-9\.]+)/)[1];
-        return parseFloat(x);
-    },
-
-    "title-numeric-asc": function (a, b) {
-        return ((a < b) ? -1 : ((a > b) ? 1 : 0));
-    },
-
-    "title-numeric-desc": function (a, b) {
-        return ((a < b) ? 1 : ((a > b) ? -1 : 0));
-    }
+  "title-numeric-pre": function (a) {
+    var x = a.match(/title="*(-?[0-9.]+)/)[1];
+    return parseFloat(x);
+  },
+
+  "title-numeric-asc": function (a, b) {
+    return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+  },
+
+  "title-numeric-desc": function (a, b) {
+    return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+  }
 });
 
 $(document).ajaxStop($.unblockUI);
 $(document).ajaxStart(function () {
-    $.blockUI({message: '<h3>Loading Executors Page...</h3>'});
+  $.blockUI({message: '<h3>Loading Executors Page...</h3>'});
 });
 
 function logsExist(execs) {
-    return execs.some(function(exec) {
-        return !($.isEmptyObject(exec["executorLogs"]));
-    });
+  return execs.some(function(exec) {
+    return !($.isEmptyObject(exec["executorLogs"]));
+  });
 }
 
 // Determine Color Opacity from 0.5-1
 // activeTasks range from 0 to maxTasks
 function activeTasksAlpha(activeTasks, maxTasks) {
-    return maxTasks > 0 ? ((activeTasks / maxTasks) * 0.5 + 0.5) : 1;
+  return maxTasks > 0 ? ((activeTasks / maxTasks) * 0.5 + 0.5) : 1;
 }
 
 function activeTasksStyle(activeTasks, maxTasks) {
-    return activeTasks > 0 ? ("hsla(240, 100%, 50%, " + activeTasksAlpha(activeTasks, maxTasks) + ")") : "";
+  return activeTasks > 0 ? ("hsla(240, 100%, 50%, " + activeTasksAlpha(activeTasks, maxTasks) + ")") : "";
 }
 
 // failedTasks range max at 10% failure, alpha max = 1
 function failedTasksAlpha(failedTasks, totalTasks) {
-    return totalTasks > 0 ?
-        (Math.min(10 * failedTasks / totalTasks, 1) * 0.5 + 0.5) : 1;
+  return totalTasks > 0 ?
+    (Math.min(10 * failedTasks / totalTasks, 1) * 0.5 + 0.5) : 1;
 }
 
 function failedTasksStyle(failedTasks, totalTasks) {
-    return failedTasks > 0 ?
-        ("hsla(0, 100%, 50%, " + failedTasksAlpha(failedTasks, totalTasks) + ")") : "";
+  return failedTasks > 0 ?
+    ("hsla(0, 100%, 50%, " + failedTasksAlpha(failedTasks, totalTasks) + ")") : "";
 }
 
 // totalDuration range from 0 to 50% GC time, alpha max = 1
 function totalDurationAlpha(totalGCTime, totalDuration) {
-    return totalDuration > 0 ?
-        (Math.min(totalGCTime / totalDuration + 0.5, 1)) : 1;
+  return totalDuration > 0 ?
+    (Math.min(totalGCTime / totalDuration + 0.5, 1)) : 1;
 }
 
 // When GCTimePercent is edited change ToolTips.TASK_TIME to match
 var GCTimePercent = 0.1;
 
 function totalDurationStyle(totalGCTime, totalDuration) {
-    // Red if GC time over GCTimePercent of total time
-    return (totalGCTime > GCTimePercent * totalDuration) ?
-        ("hsla(0, 100%, 50%, " + totalDurationAlpha(totalGCTime, totalDuration) + ")") : "";
+  // Red if GC time over GCTimePercent of total time
+  return (totalGCTime > GCTimePercent * totalDuration) ?
+    ("hsla(0, 100%, 50%, " + totalDurationAlpha(totalGCTime, totalDuration) + ")") : "";
 }
 
 function totalDurationColor(totalGCTime, totalDuration) {
-    return (totalGCTime > GCTimePercent * totalDuration) ? "white" : "black";
+  return (totalGCTime > GCTimePercent * totalDuration) ? "white" : "black";
 }
 
 var sumOptionalColumns = [3, 4];
-var execOptionalColumns = [5, 6, 9];
+var execOptionalColumns = [5, 6, 7, 8, 9, 10, 13, 14, 15];
 var execDataTable;
 var sumDataTable;
 
 function reselectCheckboxesBasedOnTaskTableState() {
-    var allChecked = true;
-    if (typeof execDataTable !== "undefined") {
-        for (var k = 0; k < execOptionalColumns.length; k++) {
-            if (execDataTable.column(execOptionalColumns[k]).visible()) {
-                $("[data-exec-col-idx=" + execOptionalColumns[k] + "]").prop("checked", true);
-            } else {
-                allChecked = false;
-            }
-        }
-    }
-    if (allChecked) {
-        $("#select-all-box").prop("checked", true);
+  var allChecked = true;
+  if (typeof execDataTable !== "undefined") {
+    for (var k = 0; k < execOptionalColumns.length; k++) {
+      if (execDataTable.column(execOptionalColumns[k]).visible()) {
+        $("[data-exec-col-idx=" + execOptionalColumns[k] + "]").prop("checked", true);
+      } else {
+        allChecked = false;
+      }
     }
+  }
+  if (allChecked) {
+    $("#select-all-box").prop("checked", true);
+  }
 }
 
 $(document).ready(function () {
-    setDataTableDefaults();
-
-    var executorsSummary = $("#active-executors");
-
-    getStandAloneAppId(function (appId) {
-
-        var endPoint = createRESTEndPointForExecutorsPage(appId);
-        $.getJSON(endPoint, function (response, status, jqXHR) {
-            var allExecCnt = 0;
-            var allRDDBlocks = 0;
-            var allMemoryUsed = 0;
-            var allMaxMemory = 0;
-            var allOnHeapMemoryUsed = 0;
-            var allOnHeapMaxMemory = 0;
-            var allOffHeapMemoryUsed = 0;
-            var allOffHeapMaxMemory = 0;
-            var allDiskUsed = 0;
-            var allTotalCores = 0;
-            var allMaxTasks = 0;
-            var allActiveTasks = 0;
-            var allFailedTasks = 0;
-            var allCompletedTasks = 0;
-            var allTotalTasks = 0;
-            var allTotalDuration = 0;
-            var allTotalGCTime = 0;
-            var allTotalInputBytes = 0;
-            var allTotalShuffleRead = 0;
-            var allTotalShuffleWrite = 0;
-            var allTotalBlacklisted = 0;
-
-            var activeExecCnt = 0;
-            var activeRDDBlocks = 0;
-            var activeMemoryUsed = 0;
-            var activeMaxMemory = 0;
-            var activeOnHeapMemoryUsed = 0;
-            var activeOnHeapMaxMemory = 0;
-            var activeOffHeapMemoryUsed = 0;
-            var activeOffHeapMaxMemory = 0;
-            var activeDiskUsed = 0;
-            var activeTotalCores = 0;
-            var activeMaxTasks = 0;
-            var activeActiveTasks = 0;
-            var activeFailedTasks = 0;
-            var activeCompletedTasks = 0;
-            var activeTotalTasks = 0;
-            var activeTotalDuration = 0;
-            var activeTotalGCTime = 0;
-            var activeTotalInputBytes = 0;
-            var activeTotalShuffleRead = 0;
-            var activeTotalShuffleWrite = 0;
-            var activeTotalBlacklisted = 0;
-
-            var deadExecCnt = 0;
-            var deadRDDBlocks = 0;
-            var deadMemoryUsed = 0;
-            var deadMaxMemory = 0;
-            var deadOnHeapMemoryUsed = 0;
-            var deadOnHeapMaxMemory = 0;
-            var deadOffHeapMemoryUsed = 0;
-            var deadOffHeapMaxMemory = 0;
-            var deadDiskUsed = 0;
-            var deadTotalCores = 0;
-            var deadMaxTasks = 0;
-            var deadActiveTasks = 0;
-            var deadFailedTasks = 0;
-            var deadCompletedTasks = 0;
-            var deadTotalTasks = 0;
-            var deadTotalDuration = 0;
-            var deadTotalGCTime = 0;
-            var deadTotalInputBytes = 0;
-            var deadTotalShuffleRead = 0;
-            var deadTotalShuffleWrite = 0;
-            var deadTotalBlacklisted = 0;
-
-            response.forEach(function (exec) {
-                var memoryMetrics = {
-                    usedOnHeapStorageMemory: 0,
-                    usedOffHeapStorageMemory: 0,
-                    totalOnHeapStorageMemory: 0,
-                    totalOffHeapStorageMemory: 0
-                };
-
-                exec.memoryMetrics = exec.hasOwnProperty('memoryMetrics') ? exec.memoryMetrics : memoryMetrics;
-            });
-
-            response.forEach(function (exec) {
-                allExecCnt += 1;
-                allRDDBlocks += exec.rddBlocks;
-                allMemoryUsed += exec.memoryUsed;
-                allMaxMemory += exec.maxMemory;
-                allOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
-                allOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
-                allOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
-                allOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
-                allDiskUsed += exec.diskUsed;
-                allTotalCores += exec.totalCores;
-                allMaxTasks += exec.maxTasks;
-                allActiveTasks += exec.activeTasks;
-                allFailedTasks += exec.failedTasks;
-                allCompletedTasks += exec.completedTasks;
-                allTotalTasks += exec.totalTasks;
-                allTotalDuration += exec.totalDuration;
-                allTotalGCTime += exec.totalGCTime;
-                allTotalInputBytes += exec.totalInputBytes;
-                allTotalShuffleRead += exec.totalShuffleRead;
-                allTotalShuffleWrite += exec.totalShuffleWrite;
-                allTotalBlacklisted += exec.isBlacklisted ? 1 : 0;
-                if (exec.isActive) {
-                    activeExecCnt += 1;
-                    activeRDDBlocks += exec.rddBlocks;
-                    activeMemoryUsed += exec.memoryUsed;
-                    activeMaxMemory += exec.maxMemory;
-                    activeOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
-                    activeOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
-                    activeOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
-                    activeOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
-                    activeDiskUsed += exec.diskUsed;
-                    activeTotalCores += exec.totalCores;
-                    activeMaxTasks += exec.maxTasks;
-                    activeActiveTasks += exec.activeTasks;
-                    activeFailedTasks += exec.failedTasks;
-                    activeCompletedTasks += exec.completedTasks;
-                    activeTotalTasks += exec.totalTasks;
-                    activeTotalDuration += exec.totalDuration;
-                    activeTotalGCTime += exec.totalGCTime;
-                    activeTotalInputBytes += exec.totalInputBytes;
-                    activeTotalShuffleRead += exec.totalShuffleRead;
-                    activeTotalShuffleWrite += exec.totalShuffleWrite;
-                    activeTotalBlacklisted += exec.isBlacklisted ? 1 : 0;
+  setDataTableDefaults();
+
+  var executorsSummary = $("#active-executors");
+
+  getStandAloneAppId(function (appId) {
+
+    var endPoint = createRESTEndPointForExecutorsPage(appId);
+    $.getJSON(endPoint, function (response, _ignored_status, _ignored_jqXHR) {
+      var allExecCnt = 0;
+      var allRDDBlocks = 0;
+      var allMemoryUsed = 0;
+      var allMaxMemory = 0;
+      var allOnHeapMemoryUsed = 0;
+      var allOnHeapMaxMemory = 0;
+      var allOffHeapMemoryUsed = 0;
+      var allOffHeapMaxMemory = 0;
+      var allDiskUsed = 0;
+      var allTotalCores = 0;
+      var allMaxTasks = 0;
+      var allActiveTasks = 0;
+      var allFailedTasks = 0;
+      var allCompletedTasks = 0;
+      var allTotalTasks = 0;
+      var allTotalDuration = 0;
+      var allTotalGCTime = 0;
+      var allTotalInputBytes = 0;
+      var allTotalShuffleRead = 0;
+      var allTotalShuffleWrite = 0;
+      var allTotalExcluded = 0;
+
+      var activeExecCnt = 0;
+      var activeRDDBlocks = 0;
+      var activeMemoryUsed = 0;
+      var activeMaxMemory = 0;
+      var activeOnHeapMemoryUsed = 0;
+      var activeOnHeapMaxMemory = 0;
+      var activeOffHeapMemoryUsed = 0;
+      var activeOffHeapMaxMemory = 0;
+      var activeDiskUsed = 0;
+      var activeTotalCores = 0;
+      var activeMaxTasks = 0;
+      var activeActiveTasks = 0;
+      var activeFailedTasks = 0;
+      var activeCompletedTasks = 0;
+      var activeTotalTasks = 0;
+      var activeTotalDuration = 0;
+      var activeTotalGCTime = 0;
+      var activeTotalInputBytes = 0;
+      var activeTotalShuffleRead = 0;
+      var activeTotalShuffleWrite = 0;
+      var activeTotalExcluded = 0;
+
+      var deadExecCnt = 0;
+      var deadRDDBlocks = 0;
+      var deadMemoryUsed = 0;
+      var deadMaxMemory = 0;
+      var deadOnHeapMemoryUsed = 0;
+      var deadOnHeapMaxMemory = 0;
+      var deadOffHeapMemoryUsed = 0;
+      var deadOffHeapMaxMemory = 0;
+      var deadDiskUsed = 0;
+      var deadTotalCores = 0;
+      var deadMaxTasks = 0;
+      var deadActiveTasks = 0;
+      var deadFailedTasks = 0;
+      var deadCompletedTasks = 0;
+      var deadTotalTasks = 0;
+      var deadTotalDuration = 0;
+      var deadTotalGCTime = 0;
+      var deadTotalInputBytes = 0;
+      var deadTotalShuffleRead = 0;
+      var deadTotalShuffleWrite = 0;
+      var deadTotalExcluded = 0;
+
+      response.forEach(function (exec) {
+        var memoryMetrics = {
+          usedOnHeapStorageMemory: 0,
+          usedOffHeapStorageMemory: 0,
+          totalOnHeapStorageMemory: 0,
+          totalOffHeapStorageMemory: 0
+        };
+
+        // TODO: Replace hasOwnProperty with prototype.hasOwnProperty after we find it's safe to do.
+        /* eslint-disable no-prototype-builtins */
+        exec.memoryMetrics = exec.hasOwnProperty('memoryMetrics') ? exec.memoryMetrics : memoryMetrics;
+      });
+
+      response.forEach(function (exec) {
+        allExecCnt += 1;
+        allRDDBlocks += exec.rddBlocks;
+        allMemoryUsed += exec.memoryUsed;
+        allMaxMemory += exec.maxMemory;
+        allOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
+        allOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
+        allOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
+        allOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
+        allDiskUsed += exec.diskUsed;
+        allTotalCores += exec.totalCores;
+        allMaxTasks += exec.maxTasks;
+        allActiveTasks += exec.activeTasks;
+        allFailedTasks += exec.failedTasks;
+        allCompletedTasks += exec.completedTasks;
+        allTotalTasks += exec.totalTasks;
+        allTotalDuration += exec.totalDuration;
+        allTotalGCTime += exec.totalGCTime;
+        allTotalInputBytes += exec.totalInputBytes;
+        allTotalShuffleRead += exec.totalShuffleRead;
+        allTotalShuffleWrite += exec.totalShuffleWrite;
+        allTotalExcluded += exec.isExcluded ? 1 : 0;
+        if (exec.isActive) {
+          activeExecCnt += 1;
+          activeRDDBlocks += exec.rddBlocks;
+          activeMemoryUsed += exec.memoryUsed;
+          activeMaxMemory += exec.maxMemory;
+          activeOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
+          activeOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
+          activeOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
+          activeOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
+          activeDiskUsed += exec.diskUsed;
+          activeTotalCores += exec.totalCores;
+          activeMaxTasks += exec.maxTasks;
+          activeActiveTasks += exec.activeTasks;
+          activeFailedTasks += exec.failedTasks;
+          activeCompletedTasks += exec.completedTasks;
+          activeTotalTasks += exec.totalTasks;
+          activeTotalDuration += exec.totalDuration;
+          activeTotalGCTime += exec.totalGCTime;
+          activeTotalInputBytes += exec.totalInputBytes;
+          activeTotalShuffleRead += exec.totalShuffleRead;
+          activeTotalShuffleWrite += exec.totalShuffleWrite;
+          activeTotalExcluded += exec.isExcluded ? 1 : 0;
+        } else {
+          deadExecCnt += 1;
+          deadRDDBlocks += exec.rddBlocks;
+          deadMemoryUsed += exec.memoryUsed;
+          deadMaxMemory += exec.maxMemory;
+          deadOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
+          deadOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
+          deadOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
+          deadOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
+          deadDiskUsed += exec.diskUsed;
+          deadTotalCores += exec.totalCores;
+          deadMaxTasks += exec.maxTasks;
+          deadActiveTasks += exec.activeTasks;
+          deadFailedTasks += exec.failedTasks;
+          deadCompletedTasks += exec.completedTasks;
+          deadTotalTasks += exec.totalTasks;
+          deadTotalDuration += exec.totalDuration;
+          deadTotalGCTime += exec.totalGCTime;
+          deadTotalInputBytes += exec.totalInputBytes;
+          deadTotalShuffleRead += exec.totalShuffleRead;
+          deadTotalShuffleWrite += exec.totalShuffleWrite;
+          deadTotalExcluded += exec.isExcluded ? 1 : 0; // todo - TEST BACKWARDS compatibility history?
+        }
+      });
+
+      var totalSummary = {
+        "execCnt": ( "Total(" + allExecCnt + ")"),
+        "allRDDBlocks": allRDDBlocks,
+        "allMemoryUsed": allMemoryUsed,
+        "allMaxMemory": allMaxMemory,
+        "allOnHeapMemoryUsed": allOnHeapMemoryUsed,
+        "allOnHeapMaxMemory": allOnHeapMaxMemory,
+        "allOffHeapMemoryUsed": allOffHeapMemoryUsed,
+        "allOffHeapMaxMemory": allOffHeapMaxMemory,
+        "allDiskUsed": allDiskUsed,
+        "allTotalCores": allTotalCores,
+        "allMaxTasks": allMaxTasks,
+        "allActiveTasks": allActiveTasks,
+        "allFailedTasks": allFailedTasks,
+        "allCompletedTasks": allCompletedTasks,
+        "allTotalTasks": allTotalTasks,
+        "allTotalDuration": allTotalDuration,
+        "allTotalGCTime": allTotalGCTime,
+        "allTotalInputBytes": allTotalInputBytes,
+        "allTotalShuffleRead": allTotalShuffleRead,
+        "allTotalShuffleWrite": allTotalShuffleWrite,
+        "allTotalExcluded": allTotalExcluded
+      };
+      var activeSummary = {
+        "execCnt": ( "Active(" + activeExecCnt + ")"),
+        "allRDDBlocks": activeRDDBlocks,
+        "allMemoryUsed": activeMemoryUsed,
+        "allMaxMemory": activeMaxMemory,
+        "allOnHeapMemoryUsed": activeOnHeapMemoryUsed,
+        "allOnHeapMaxMemory": activeOnHeapMaxMemory,
+        "allOffHeapMemoryUsed": activeOffHeapMemoryUsed,
+        "allOffHeapMaxMemory": activeOffHeapMaxMemory,
+        "allDiskUsed": activeDiskUsed,
+        "allTotalCores": activeTotalCores,
+        "allMaxTasks": activeMaxTasks,
+        "allActiveTasks": activeActiveTasks,
+        "allFailedTasks": activeFailedTasks,
+        "allCompletedTasks": activeCompletedTasks,
+        "allTotalTasks": activeTotalTasks,
+        "allTotalDuration": activeTotalDuration,
+        "allTotalGCTime": activeTotalGCTime,
+        "allTotalInputBytes": activeTotalInputBytes,
+        "allTotalShuffleRead": activeTotalShuffleRead,
+        "allTotalShuffleWrite": activeTotalShuffleWrite,
+        "allTotalExcluded": activeTotalExcluded
+      };
+      var deadSummary = {
+        "execCnt": ( "Dead(" + deadExecCnt + ")" ),
+        "allRDDBlocks": deadRDDBlocks,
+        "allMemoryUsed": deadMemoryUsed,
+        "allMaxMemory": deadMaxMemory,
+        "allOnHeapMemoryUsed": deadOnHeapMemoryUsed,
+        "allOnHeapMaxMemory": deadOnHeapMaxMemory,
+        "allOffHeapMemoryUsed": deadOffHeapMemoryUsed,
+        "allOffHeapMaxMemory": deadOffHeapMaxMemory,
+        "allDiskUsed": deadDiskUsed,
+        "allTotalCores": deadTotalCores,
+        "allMaxTasks": deadMaxTasks,
+        "allActiveTasks": deadActiveTasks,
+        "allFailedTasks": deadFailedTasks,
+        "allCompletedTasks": deadCompletedTasks,
+        "allTotalTasks": deadTotalTasks,
+        "allTotalDuration": deadTotalDuration,
+        "allTotalGCTime": deadTotalGCTime,
+        "allTotalInputBytes": deadTotalInputBytes,
+        "allTotalShuffleRead": deadTotalShuffleRead,
+        "allTotalShuffleWrite": deadTotalShuffleWrite,
+        "allTotalExcluded": deadTotalExcluded
+      };
+
+      var data = {executors: response, "execSummary": [activeSummary, deadSummary, totalSummary]};
+      $.get(createTemplateURI(appId, "executorspage"), function (template) {
+
+        executorsSummary.append(Mustache.render($(template).filter("#executors-summary-template").html(), data));
+        var selector = "#active-executors-table";
+        var conf = {
+          "data": response,
+          "columns": [
+            {
+              data: function (row, type) {
+                return type !== 'display' ? (isNaN(row.id) ? 0 : row.id ) : row.id;
+              }
+            },
+            {data: 'hostPort'},
+            {
+              data: 'isActive',
+              render: function (data, type, row) {
+                return formatStatus (data, type, row);
+              }
+            },
+            {data: 'rddBlocks'},
+            {
+              data: function (row, type) {
+                if (type !== 'display')
+                  return row.memoryUsed;
+                else
+                  return (formatBytes(row.memoryUsed, type) + ' / ' +
+                    formatBytes(row.maxMemory, type));
+              }
+            },
+            {
+              data: function (row, type) {
+                if (type !== 'display')
+                  return row.memoryMetrics.usedOnHeapStorageMemory;
+                else
+                  return (formatBytes(row.memoryMetrics.usedOnHeapStorageMemory, type) + ' / ' +
+                    formatBytes(row.memoryMetrics.totalOnHeapStorageMemory, type));
+              }
+            },
+            {
+              data: function (row, type) {
+                if (type !== 'display')
+                  return row.memoryMetrics.usedOffHeapStorageMemory;
+                else
+                  return (formatBytes(row.memoryMetrics.usedOffHeapStorageMemory, type) + ' / ' +
+                    formatBytes(row.memoryMetrics.totalOffHeapStorageMemory, type));
+              }
+            },
+            {
+              data: function (row, type) {
+                var peakMemoryMetrics = row.peakMemoryMetrics;
+                if (typeof peakMemoryMetrics !== 'undefined') {
+                  if (type !== 'display')
+                    return peakMemoryMetrics.JVMHeapMemory;
+                  else
+                    return (formatBytes(peakMemoryMetrics.JVMHeapMemory, type) + ' / ' +
+                      formatBytes(peakMemoryMetrics.JVMOffHeapMemory, type));
                 } else {
-                    deadExecCnt += 1;
-                    deadRDDBlocks += exec.rddBlocks;
-                    deadMemoryUsed += exec.memoryUsed;
-                    deadMaxMemory += exec.maxMemory;
-                    deadOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
-                    deadOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
-                    deadOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
-                    deadOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
-                    deadDiskUsed += exec.diskUsed;
-                    deadTotalCores += exec.totalCores;
-                    deadMaxTasks += exec.maxTasks;
-                    deadActiveTasks += exec.activeTasks;
-                    deadFailedTasks += exec.failedTasks;
-                    deadCompletedTasks += exec.completedTasks;
-                    deadTotalTasks += exec.totalTasks;
-                    deadTotalDuration += exec.totalDuration;
-                    deadTotalGCTime += exec.totalGCTime;
-                    deadTotalInputBytes += exec.totalInputBytes;
-                    deadTotalShuffleRead += exec.totalShuffleRead;
-                    deadTotalShuffleWrite += exec.totalShuffleWrite;
-                    deadTotalBlacklisted += exec.isBlacklisted ? 1 : 0;
+                  if (type !== 'display') {
+                    return 0;
+                  } else {
+                    return '0.0 B / 0.0 B';
+                  }
                 }
-            });
-
-            var totalSummary = {
-                "execCnt": ( "Total(" + allExecCnt + ")"),
-                "allRDDBlocks": allRDDBlocks,
-                "allMemoryUsed": allMemoryUsed,
-                "allMaxMemory": allMaxMemory,
-                "allOnHeapMemoryUsed": allOnHeapMemoryUsed,
-                "allOnHeapMaxMemory": allOnHeapMaxMemory,
-                "allOffHeapMemoryUsed": allOffHeapMemoryUsed,
-                "allOffHeapMaxMemory": allOffHeapMaxMemory,
-                "allDiskUsed": allDiskUsed,
-                "allTotalCores": allTotalCores,
-                "allMaxTasks": allMaxTasks,
-                "allActiveTasks": allActiveTasks,
-                "allFailedTasks": allFailedTasks,
-                "allCompletedTasks": allCompletedTasks,
-                "allTotalTasks": allTotalTasks,
-                "allTotalDuration": allTotalDuration,
-                "allTotalGCTime": allTotalGCTime,
-                "allTotalInputBytes": allTotalInputBytes,
-                "allTotalShuffleRead": allTotalShuffleRead,
-                "allTotalShuffleWrite": allTotalShuffleWrite,
-                "allTotalBlacklisted": allTotalBlacklisted
-            };
-            var activeSummary = {
-                "execCnt": ( "Active(" + activeExecCnt + ")"),
-                "allRDDBlocks": activeRDDBlocks,
-                "allMemoryUsed": activeMemoryUsed,
-                "allMaxMemory": activeMaxMemory,
-                "allOnHeapMemoryUsed": activeOnHeapMemoryUsed,
-                "allOnHeapMaxMemory": activeOnHeapMaxMemory,
-                "allOffHeapMemoryUsed": activeOffHeapMemoryUsed,
-                "allOffHeapMaxMemory": activeOffHeapMaxMemory,
-                "allDiskUsed": activeDiskUsed,
-                "allTotalCores": activeTotalCores,
-                "allMaxTasks": activeMaxTasks,
-                "allActiveTasks": activeActiveTasks,
-                "allFailedTasks": activeFailedTasks,
-                "allCompletedTasks": activeCompletedTasks,
-                "allTotalTasks": activeTotalTasks,
-                "allTotalDuration": activeTotalDuration,
-                "allTotalGCTime": activeTotalGCTime,
-                "allTotalInputBytes": activeTotalInputBytes,
-                "allTotalShuffleRead": activeTotalShuffleRead,
-                "allTotalShuffleWrite": activeTotalShuffleWrite,
-                "allTotalBlacklisted": activeTotalBlacklisted
-            };
-            var deadSummary = {
-                "execCnt": ( "Dead(" + deadExecCnt + ")" ),
-                "allRDDBlocks": deadRDDBlocks,
-                "allMemoryUsed": deadMemoryUsed,
-                "allMaxMemory": deadMaxMemory,
-                "allOnHeapMemoryUsed": deadOnHeapMemoryUsed,
-                "allOnHeapMaxMemory": deadOnHeapMaxMemory,
-                "allOffHeapMemoryUsed": deadOffHeapMemoryUsed,
-                "allOffHeapMaxMemory": deadOffHeapMaxMemory,
-                "allDiskUsed": deadDiskUsed,
-                "allTotalCores": deadTotalCores,
-                "allMaxTasks": deadMaxTasks,
-                "allActiveTasks": deadActiveTasks,
-                "allFailedTasks": deadFailedTasks,
-                "allCompletedTasks": deadCompletedTasks,
-                "allTotalTasks": deadTotalTasks,
-                "allTotalDuration": deadTotalDuration,
-                "allTotalGCTime": deadTotalGCTime,
-                "allTotalInputBytes": deadTotalInputBytes,
-                "allTotalShuffleRead": deadTotalShuffleRead,
-                "allTotalShuffleWrite": deadTotalShuffleWrite,
-                "allTotalBlacklisted": deadTotalBlacklisted
+              }
+            },
+            {
+              data: function (row, type) {
+                var peakMemoryMetrics = row.peakMemoryMetrics;
+                if (typeof peakMemoryMetrics !== 'undefined') {
+                  if (type !== 'display')
+                    return peakMemoryMetrics.OnHeapExecutionMemory;
+                  else
+                    return (formatBytes(peakMemoryMetrics.OnHeapExecutionMemory, type) + ' / ' +
+                      formatBytes(peakMemoryMetrics.OffHeapExecutionMemory, type));
+                } else {
+                  if (type !== 'display') {
+                    return 0;
+                  } else {
+                    return '0.0 B / 0.0 B';
+                  }
+                }
+              }
+            },
+            {
+              data: function (row, type) {
+                var peakMemoryMetrics = row.peakMemoryMetrics;
+                if (typeof peakMemoryMetrics !== 'undefined') {
+                  if (type !== 'display')
+                    return peakMemoryMetrics.OnHeapStorageMemory;
+                  else
+                    return (formatBytes(peakMemoryMetrics.OnHeapStorageMemory, type) + ' / ' +
+                      formatBytes(peakMemoryMetrics.OffHeapStorageMemory, type));
+                } else {
+                  if (type !== 'display') {
+                    return 0;
+                  } else {
+                    return '0.0 B / 0.0 B';
+                  }
+                }
+              }
+            },
+            {
+              data: function (row, type) {
+                var peakMemoryMetrics = row.peakMemoryMetrics;
+                if (typeof peakMemoryMetrics !== 'undefined') {
+                  if (type !== 'display')
+                    return peakMemoryMetrics.DirectPoolMemory;
+                  else
+                    return (formatBytes(peakMemoryMetrics.DirectPoolMemory, type) + ' / ' +
+                      formatBytes(peakMemoryMetrics.MappedPoolMemory, type));
+                } else {
+                  if (type !== 'display') {
+                    return 0;
+                  } else {
+                    return '0.0 B / 0.0 B';
+                  }
+                }
+              }
+            },
+            {data: 'diskUsed', render: formatBytes},
+            {data: 'totalCores'},
+            {name: 'resourcesCol', data: 'resources', render: formatResourceCells, orderable: false},
+            {name: 'resourceProfileIdCol', data: 'resourceProfileId'},
+            {
+              data: 'activeTasks',
+              "fnCreatedCell": function (nTd, sData, oData, _ignored_iRow, _ignored_iCol) {
+                if (sData > 0) {
+                  $(nTd).css('color', 'white');
+                  $(nTd).css('background', activeTasksStyle(oData.activeTasks, oData.maxTasks));
+                }
+              }
+            },
+            {
+              data: 'failedTasks',
+              "fnCreatedCell": function (nTd, sData, oData, _ignored_iRow, _ignored_iCol) {
+                if (sData > 0) {
+                  $(nTd).css('color', 'white');
+                  $(nTd).css('background', failedTasksStyle(oData.failedTasks, oData.totalTasks));
+                }
+              }
+            },
+            {data: 'completedTasks'},
+            {data: 'totalTasks'},
+            {
+              data: function (row, type) {
+                return type === 'display' ? (formatDuration(row.totalDuration) + ' (' + formatDuration(row.totalGCTime) + ')') : row.totalDuration
+              },
+              "fnCreatedCell": function (nTd, sData, oData, _ignored_iRow, _ignored_iCol) {
+                if (oData.totalDuration > 0) {
+                  $(nTd).css('color', totalDurationColor(oData.totalGCTime, oData.totalDuration));
+                  $(nTd).css('background', totalDurationStyle(oData.totalGCTime, oData.totalDuration));
+                }
+              }
+            },
+            {data: 'totalInputBytes', render: formatBytes},
+            {data: 'totalShuffleRead', render: formatBytes},
+            {data: 'totalShuffleWrite', render: formatBytes},
+            {name: 'executorLogsCol', data: 'executorLogs', render: formatLogsCells},
+            {
+              name: 'threadDumpCol',
+              data: 'id', render: function (data, type) {
+                return type === 'display' ? ("<a href='threadDump/?executorId=" + data + "'>Thread Dump</a>" ) : data;
+              }
+            },
+            {
+              data: 'removeReason',
+              render: formatLossReason
+            }
+          ],
+          "order": [[0, "asc"]],
+          "columnDefs": [
+            {"visible": false, "targets": 5},
+            {"visible": false, "targets": 6},
+            {"visible": false, "targets": 7},
+            {"visible": false, "targets": 8},
+            {"visible": false, "targets": 9},
+            {"visible": false, "targets": 10},
+            {"visible": false, "targets": 13},
+            {"visible": false, "targets": 14}
+          ],
+          "deferRender": true
+        };
+
+        execDataTable = $(selector).DataTable(conf);
+        execDataTable.column('executorLogsCol:name').visible(logsExist(response));
+        execDataTable.column('threadDumpCol:name').visible(getThreadDumpEnabled());
+        $('#active-executors [data-toggle="tooltip"]').tooltip();
+    
+        // This section should be visible once API gives the response.
+        $('.active-process-container').hide();
+        var endPoint = createRESTEndPointForMiscellaneousProcess(appId);
+        $.getJSON(endPoint, function( response, _ignored_status, _ignored_jqXHR ) {
+          if (response.length) {
+            var processSummaryResponse = response;
+            var processSummaryConf = {
+              "data": processSummaryResponse,
+              "columns": [{
+                data: "id"
+              },
+              {
+                data: "hostPort"
+              },
+              {
+                data: function(row) {
+                  return formatProcessStatus(row.isActive);
+                }
+              },
+              {
+                data: "totalCores"
+              },
+              {
+                data: "processLogs",
+                render: formatLogsCells
+              },
+              ],
+              "deferRender": true,
+              "order": [
+                [0, "asc"]
+              ],
+              "bAutoWidth": false,
+              "oLanguage": {
+                "sEmptyTable": "No data to show yet"
+              }
             };
+            $("#active-process-table").DataTable(processSummaryConf);
+            $('.active-process-container').show()
+          }
+        });
 
-            var data = {executors: response, "execSummary": [activeSummary, deadSummary, totalSummary]};
-            $.get(createTemplateURI(appId, "executorspage"), function (template) {
-
-                executorsSummary.append(Mustache.render($(template).filter("#executors-summary-template").html(), data));
-                var selector = "#active-executors-table";
-                var conf = {
-                    "data": response,
-                    "columns": [
-                        {
-                            data: function (row, type) {
-                                return type !== 'display' ? (isNaN(row.id) ? 0 : row.id ) : row.id;
-                            }
-                        },
-                        {data: 'hostPort'},
-                        {
-                            data: 'isActive',
-                            render: function (data, type, row) {
-                                return formatStatus (data, type, row);
-                            }
-                        },
-                        {data: 'rddBlocks'},
-                        {
-                            data: function (row, type) {
-                                if (type !== 'display')
-                                    return row.memoryUsed;
-                                else
-                                    return (formatBytes(row.memoryUsed, type) + ' / ' +
-                                        formatBytes(row.maxMemory, type));
-                            }
-                        },
-                        {
-                            data: function (row, type) {
-                                if (type !== 'display')
-                                    return row.memoryMetrics.usedOnHeapStorageMemory;
-                                else
-                                    return (formatBytes(row.memoryMetrics.usedOnHeapStorageMemory, type) + ' / ' +
-                                        formatBytes(row.memoryMetrics.totalOnHeapStorageMemory, type));
-                            }
-                        },
-                        {
-                            data: function (row, type) {
-                                if (type !== 'display')
-                                    return row.memoryMetrics.usedOffHeapStorageMemory;
-                                else
-                                    return (formatBytes(row.memoryMetrics.usedOffHeapStorageMemory, type) + ' / ' +
-                                        formatBytes(row.memoryMetrics.totalOffHeapStorageMemory, type));
-                            }
-                        },
-                        {data: 'diskUsed', render: formatBytes},
-                        {data: 'totalCores'},
-                        {name: 'resourcesCol', data: 'resources', render: formatResourceCells, orderable: false},
-                        {
-                            data: 'activeTasks',
-                            "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
-                                if (sData > 0) {
-                                    $(nTd).css('color', 'white');
-                                    $(nTd).css('background', activeTasksStyle(oData.activeTasks, oData.maxTasks));
-                                }
-                            }
-                        },
-                        {
-                            data: 'failedTasks',
-                            "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
-                                if (sData > 0) {
-                                    $(nTd).css('color', 'white');
-                                    $(nTd).css('background', failedTasksStyle(oData.failedTasks, oData.totalTasks));
-                                }
-                            }
-                        },
-                        {data: 'completedTasks'},
-                        {data: 'totalTasks'},
-                        {
-                            data: function (row, type) {
-                                return type === 'display' ? (formatDuration(row.totalDuration) + ' (' + formatDuration(row.totalGCTime) + ')') : row.totalDuration
-                            },
-                            "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
-                                if (oData.totalDuration > 0) {
-                                    $(nTd).css('color', totalDurationColor(oData.totalGCTime, oData.totalDuration));
-                                    $(nTd).css('background', totalDurationStyle(oData.totalGCTime, oData.totalDuration));
-                                }
-                            }
-                        },
-                        {data: 'totalInputBytes', render: formatBytes},
-                        {data: 'totalShuffleRead', render: formatBytes},
-                        {data: 'totalShuffleWrite', render: formatBytes},
-                        {name: 'executorLogsCol', data: 'executorLogs', render: formatLogsCells},
-                        {
-                            name: 'threadDumpCol',
-                            data: 'id', render: function (data, type) {
-                                return type === 'display' ? ("<a href='threadDump/?executorId=" + data + "'>Thread Dump</a>" ) : data;
-                            }
-                        }
-                    ],
-                    "order": [[0, "asc"]],
-                    "columnDefs": [
-                        {"visible": false, "targets": 5},
-                        {"visible": false, "targets": 6},
-                        {"visible": false, "targets": 9}
-                    ],
-                    "deferRender": true
-                };
-
-                execDataTable = $(selector).DataTable(conf);
-                execDataTable.column('executorLogsCol:name').visible(logsExist(response));
-                execDataTable.column('threadDumpCol:name').visible(getThreadDumpEnabled());
-                $('#active-executors [data-toggle="tooltip"]').tooltip();
-    
-                var sumSelector = "#summary-execs-table";
-                var sumConf = {
-                    "data": [activeSummary, deadSummary, totalSummary],
-                    "columns": [
-                        {
-                            data: 'execCnt',
-                            "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
-                                $(nTd).css('font-weight', 'bold');
-                            }
-                        },
-                        {data: 'allRDDBlocks'},
-                        {
-                            data: function (row, type) {
-                                if (type !== 'display')
-                                    return row.allMemoryUsed
-                                else
-                                    return (formatBytes(row.allMemoryUsed, type) + ' / ' +
-                                        formatBytes(row.allMaxMemory, type));
-                            }
-                        },
-                        {
-                            data: function (row, type) {
-                                if (type !== 'display')
-                                    return row.allOnHeapMemoryUsed;
-                                else
-                                    return (formatBytes(row.allOnHeapMemoryUsed, type) + ' / ' +
-                                        formatBytes(row.allOnHeapMaxMemory, type));
-                            }
-                        },
-                        {
-                            data: function (row, type) {
-                                if (type !== 'display')
-                                    return row.allOffHeapMemoryUsed;
-                                else
-                                    return (formatBytes(row.allOffHeapMemoryUsed, type) + ' / ' +
-                                        formatBytes(row.allOffHeapMaxMemory, type));
-                            }
-                        },
-                        {data: 'allDiskUsed', render: formatBytes},
-                        {data: 'allTotalCores'},
-                        {
-                            data: 'allActiveTasks',
-                            "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
-                                if (sData > 0) {
-                                    $(nTd).css('color', 'white');
-                                    $(nTd).css('background', activeTasksStyle(oData.allActiveTasks, oData.allMaxTasks));
-                                }
-                            }
-                        },
-                        {
-                            data: 'allFailedTasks',
-                            "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
-                                if (sData > 0) {
-                                    $(nTd).css('color', 'white');
-                                    $(nTd).css('background', failedTasksStyle(oData.allFailedTasks, oData.allTotalTasks));
-                                }
-                            }
-                        },
-                        {data: 'allCompletedTasks'},
-                        {data: 'allTotalTasks'},
-                        {
-                            data: function (row, type) {
-                                return type === 'display' ? (formatDuration(row.allTotalDuration) + ' (' + formatDuration(row.allTotalGCTime) + ')') : row.allTotalDuration
-                            },
-                            "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
-                                if (oData.allTotalDuration > 0) {
-                                    $(nTd).css('color', totalDurationColor(oData.allTotalGCTime, oData.allTotalDuration));
-                                    $(nTd).css('background', totalDurationStyle(oData.allTotalGCTime, oData.allTotalDuration));
-                                }
-                            }
-                        },
-                        {data: 'allTotalInputBytes', render: formatBytes},
-                        {data: 'allTotalShuffleRead', render: formatBytes},
-                        {data: 'allTotalShuffleWrite', render: formatBytes},
-                        {data: 'allTotalBlacklisted'}
-                    ],
-                    "paging": false,
-                    "searching": false,
-                    "info": false,
-                    "columnDefs": [
-                        {"visible": false, "targets": 3},
-                        {"visible": false, "targets": 4}
-                    ]
-
-                };
-    
-                sumDataTable = $(sumSelector).DataTable(sumConf);
-                $('#execSummary [data-toggle="tooltip"]').tooltip();
-
-                $("#showAdditionalMetrics").append(
-                    "<div><a id='additionalMetrics' class='collapse-table'>" +
-                    "<span class='expand-input-rate-arrow arrow-closed' id='arrowtoggle-optional-metrics'></span>" +
-                    "Show Additional Metrics" +
-                    "</a></div>" +
-                    "<div class='container-fluid-div ml-4 d-none' id='toggle-metrics'>" +
-                    "<div><input type='checkbox' class='toggle-vis' id='select-all-box'>Select All</div>" +
-                    "<div id='on_heap_memory' class='on-heap-memory-checkbox-div'><input type='checkbox' class='toggle-vis' data-sum-col-idx='3' data-exec-col-idx='5'>On Heap Memory</div>" +
-                    "<div id='off_heap_memory' class='off-heap-memory-checkbox-div'><input type='checkbox' class='toggle-vis' data-sum-col-idx='4' data-exec-col-idx='6'>Off Heap Memory</div>" +
-                    "<div id='extra_resources' class='resources-checkbox-div'><input type='checkbox' class='toggle-vis' data-sum-col-idx='' data-exec-col-idx='9'>Resources</div>" +
-                    "</div>");
-
-                reselectCheckboxesBasedOnTaskTableState();
-
-                $("#additionalMetrics").click(function() {
-                    $("#arrowtoggle-optional-metrics").toggleClass("arrow-open arrow-closed");
-                    $("#toggle-metrics").toggleClass("d-none");
-                    if (window.localStorage) {
-                        window.localStorage.setItem("arrowtoggle-optional-metrics-class", $("#arrowtoggle-optional-metrics").attr('class'));
-                    }
-                });
-
-                $(".toggle-vis").on("click", function() {
-                    var thisBox = $(this);
-                    if (thisBox.is("#select-all-box")) {
-                        var sumColumn = sumDataTable.columns(sumOptionalColumns);
-                        var execColumn = execDataTable.columns(execOptionalColumns);
-                        if (thisBox.is(":checked")) {
-                            $(".toggle-vis").prop("checked", true);
-                            sumColumn.visible(true);
-                            execColumn.visible(true);
-                        } else {
-                            $(".toggle-vis").prop("checked", false);
-                            sumColumn.visible(false);
-                            execColumn.visible(false);
-                        }
-                    } else {
-                        var execColIdx = thisBox.attr("data-exec-col-idx");
-                        var execCol = execDataTable.column(execColIdx);
-                        execCol.visible(!execCol.visible());
-                        var sumColIdx = thisBox.attr("data-sum-col-idx");
-                        if (sumColIdx) {
-                            var sumCol = sumDataTable.column(sumColIdx);
-                            sumCol.visible(!sumCol.visible());
-                        }
-                    }
-                });
-
-                if (window.localStorage) {
-                    if (window.localStorage.getItem("arrowtoggle-optional-metrics-class") != null &&
-                        window.localStorage.getItem("arrowtoggle-optional-metrics-class").includes("arrow-open")) {
-                        $("#arrowtoggle-optional-metrics").toggleClass("arrow-open arrow-closed");
-                        $("#toggle-metrics").toggleClass("d-none");
-                    }
+        var sumSelector = "#summary-execs-table";
+        var sumConf = {
+          "data": [activeSummary, deadSummary, totalSummary],
+          "columns": [
+            {
+              data: 'execCnt',
+              "fnCreatedCell": function (nTd, _ignored_sData, _ignored_oData, _ignored_iRow, _ignored_iCol) {
+                $(nTd).css('font-weight', 'bold');
+              }
+            },
+            {data: 'allRDDBlocks'},
+            {
+              data: function (row, type) {
+                if (type !== 'display')
+                  return row.allMemoryUsed;
+                else
+                  return (formatBytes(row.allMemoryUsed, type) + ' / ' +
+                  formatBytes(row.allMaxMemory, type));
+              }
+            },
+            {
+              data: function (row, type) {
+                if (type !== 'display')
+                  return row.allOnHeapMemoryUsed;
+                else
+                  return (formatBytes(row.allOnHeapMemoryUsed, type) + ' / ' +
+                    formatBytes(row.allOnHeapMaxMemory, type));
+              }
+            },
+            {
+              data: function (row, type) {
+                if (type !== 'display')
+                  return row.allOffHeapMemoryUsed;
+                else
+                  return (formatBytes(row.allOffHeapMemoryUsed, type) + ' / ' +
+                    formatBytes(row.allOffHeapMaxMemory, type));
+              }
+            },
+            {data: 'allDiskUsed', render: formatBytes},
+            {data: 'allTotalCores'},
+            {
+              data: 'allActiveTasks',
+              "fnCreatedCell": function (nTd, sData, oData, _ignored_iRow, _ignored_iCol) {
+                if (sData > 0) {
+                  $(nTd).css('color', 'white');
+                  $(nTd).css('background', activeTasksStyle(oData.allActiveTasks, oData.allMaxTasks));
+                }
+              }
+            },
+            {
+              data: 'allFailedTasks',
+              "fnCreatedCell": function (nTd, sData, oData, _ignored_iRow, _ignored_iCol) {
+                if (sData > 0) {
+                  $(nTd).css('color', 'white');
+                  $(nTd).css('background', failedTasksStyle(oData.allFailedTasks, oData.allTotalTasks));
                 }
-            });
+              }
+            },
+            {data: 'allCompletedTasks'},
+            {data: 'allTotalTasks'},
+            {
+              data: function (row, type) {
+                return type === 'display' ? (formatDuration(row.allTotalDuration) + ' (' + formatDuration(row.allTotalGCTime) + ')') : row.allTotalDuration
+              },
+              "fnCreatedCell": function (nTd, sData, oData, _ignored_iRow, _ignored_iCol) {
+                if (oData.allTotalDuration > 0) {
+                  $(nTd).css('color', totalDurationColor(oData.allTotalGCTime, oData.allTotalDuration));
+                  $(nTd).css('background', totalDurationStyle(oData.allTotalGCTime, oData.allTotalDuration));
+                }
+              }
+            },
+            {data: 'allTotalInputBytes', render: formatBytes},
+            {data: 'allTotalShuffleRead', render: formatBytes},
+            {data: 'allTotalShuffleWrite', render: formatBytes},
+            {data: 'allTotalExcluded'}
+          ],
+          "paging": false,
+          "searching": false,
+          "info": false,
+          "columnDefs": [
+            {"visible": false, "targets": 3},
+            {"visible": false, "targets": 4}
+          ]
+
+        };
+
+        sumDataTable = $(sumSelector).DataTable(sumConf);
+        $('#execSummary [data-toggle="tooltip"]').tooltip();
+  
+        $("#showAdditionalMetrics").append(
+          "<div><a id='additionalMetrics' class='collapse-table'>" +
+          "<span class='expand-input-rate-arrow arrow-closed' id='arrowtoggle-optional-metrics'></span>" +
+          "Show Additional Metrics" +
+          "</a></div>" +
+          "<div class='container-fluid-div ml-4 d-none' id='toggle-metrics'>" +
+          "<div><input type='checkbox' class='toggle-vis' id='select-all-box'> Select All</div>" +
+          "<div id='on_heap_memory' class='on-heap-memory-checkbox-div'><input type='checkbox' class='toggle-vis' data-sum-col-idx='3' data-exec-col-idx='5'> On Heap Memory</div>" +
+          "<div id='off_heap_memory' class='off-heap-memory-checkbox-div'><input type='checkbox' class='toggle-vis' data-sum-col-idx='4' data-exec-col-idx='6'> Off Heap Memory</div>" +
+          "<div id='jvm_on_off_heap_memory' class='jvm_on_off_heap_memory-checkbox-div'><input type='checkbox' class='toggle-vis' data-sum-col-idx='' data-exec-col-idx='7'> Peak JVM Memory OnHeap / OffHeap</div>" +
+          "<div id='on_off_heap_execution_memory' class='on_off_heap_execution_memory-checkbox-div'><input type='checkbox' class='toggle-vis' data-sum-col-idx='' data-exec-col-idx='8'> Peak Execution Memory OnHeap / OffHeap</div>" +
+          "<div id='on_off_heap_storage_memory' class='on_off_heap_storage_memory'><input type='checkbox' class='toggle-vis' data-sum-col-idx='' data-exec-col-idx='9'> Peak Storage Memory OnHeap / OffHeap</div>" +
+          "<div id='direct_mapped_pool_memory' class='direct_mapped_pool_memory-checkbox-div'><input type='checkbox' class='toggle-vis' data-sum-col-idx='' data-exec-col-idx='10'> Peak Pool Memory Direct / Mapped</div>" +
+          "<div id='extra_resources' class='resources-checkbox-div'><input type='checkbox' class='toggle-vis' data-sum-col-idx='' data-exec-col-idx='13'> Resources</div>" +
+          "<div id='resource_prof_id' class='resource-prof-id-checkbox-div'><input type='checkbox' class='toggle-vis' data-sum-col-idx='' data-exec-col-idx='14'> Resource Profile Id</div>" +
+          "<div id='exec_loss_reason' class='exec-loss-reason-checkbox-div'><input type='checkbox' class='toggle-vis' data-sum-col-idx='' data-exec-col-idx='15'> Exec Loss Reason</div>" +
+          "</div>");
+
+        reselectCheckboxesBasedOnTaskTableState();
+
+        $("#additionalMetrics").click(function() {
+          $("#arrowtoggle-optional-metrics").toggleClass("arrow-open arrow-closed");
+          $("#toggle-metrics").toggleClass("d-none");
+          if (window.localStorage) {
+            window.localStorage.setItem("arrowtoggle-optional-metrics-class", $("#arrowtoggle-optional-metrics").attr('class'));
+          }
         });
+
+        $(".toggle-vis").on("click", function() {
+          var thisBox = $(this);
+          if (thisBox.is("#select-all-box")) {
+            var sumColumn = sumDataTable.columns(sumOptionalColumns);
+            var execColumn = execDataTable.columns(execOptionalColumns);
+            if (thisBox.is(":checked")) {
+              $(".toggle-vis").prop("checked", true);
+              sumColumn.visible(true);
+              execColumn.visible(true);
+            } else {
+              $(".toggle-vis").prop("checked", false);
+              sumColumn.visible(false);
+              execColumn.visible(false);
+            }
+          } else {
+            var execColIdx = thisBox.attr("data-exec-col-idx");
+            var execCol = execDataTable.column(execColIdx);
+            execCol.visible(!execCol.visible());
+            var sumColIdx = thisBox.attr("data-sum-col-idx");
+            if (sumColIdx) {
+              var sumCol = sumDataTable.column(sumColIdx);
+              sumCol.visible(!sumCol.visible());
+            }
+          }
+        });
+
+        if (window.localStorage) {
+          if (window.localStorage.getItem("arrowtoggle-optional-metrics-class") != null &&
+            window.localStorage.getItem("arrowtoggle-optional-metrics-class").includes("arrow-open")) {
+            $("#arrowtoggle-optional-metrics").toggleClass("arrow-open arrow-closed");
+            $("#toggle-metrics").toggleClass("d-none");
+          }
+        }
+      });
     });
+  });
 });
diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage-common.js b/core/src/main/resources/org/apache/spark/ui/static/historypage-common.js
index 4cfe46ec914ae..cd8cf098ef1c0 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/historypage-common.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/historypage-common.js
@@ -15,6 +15,8 @@
  * limitations under the License.
  */
 
+/* global $, formatTimeMillis, getTimeZone */
+
 $(document).ready(function() {
   if ($('#last-updated').length) {
     var lastUpdatedMillis = Number($('#last-updated').text());
diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html
index 33eb7bfa671bc..5427125cb3b54 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html
+++ b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html
@@ -75,26 +75,6 @@
       </th>
   </thead>
   <tbody>
-  {{#applications}}
-    <tr>
-      <td {{#hasMultipleAttempts}}style="background-color:#fff"{{/hasMultipleAttempts}}>{{version}}</td>
-      <td {{#hasMultipleAttempts}}style="background-color:#fff"{{/hasMultipleAttempts}}><span title="{{id}}"><a href="{{uiroot}}/history/{{id}}/{{num}}/jobs/">{{id}}</a></span></td>
-      <td {{#hasMultipleAttempts}}style="background-color:#fff"{{/hasMultipleAttempts}}>{{name}}</td>
-      {{#attempts}}
-      {{#hasMultipleAttempts}}
-      <td><a href="{{uiroot}}/history/{{id}}/{{attemptId}}/jobs/">{{attemptId}}</a></td>
-      {{/hasMultipleAttempts}}
-      <td>{{startTime}}</td>
-      {{#showCompletedColumns}}
-      <td>{{endTime}}</td>
-      <td><span title="{{durationMillisec}}">{{duration}}</span></td>
-      {{/showCompletedColumns}}
-      <td>{{sparkUser}}</td>
-      <td>{{lastUpdated}}</td>
-      <td><a href="{{log}}" class="btn btn-info btn-mini">Download</a></td>
-      {{/attempts}}
-    </tr>
-  {{/applications}}
   </tbody>
 </table>
 </script>
diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js b/core/src/main/resources/org/apache/spark/ui/static/historypage.js
index 4df5f07f077d7..b334bceb5a039 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js
@@ -15,11 +15,15 @@
  * limitations under the License.
  */
 
+/* global $, Mustache, formatDuration, formatTimeMillis, jQuery, uiRoot */
+
 var appLimit = -1;
 
+/* eslint-disable no-unused-vars */
 function setAppLimit(val) {
-    appLimit = val;
+  appLimit = val;
 }
+/* eslint-enable no-unused-vars*/
 
 function makeIdNumeric(id) {
   var strs = id.split("_");
@@ -30,8 +34,8 @@ function makeIdNumeric(id) {
   var resl = strs[0] + "_" + strs[1] + "_";
   var diff = 10 - appSeqNum.length;
   while (diff > 0) {
-      resl += "0"; // padding 0 before the app sequence number to make sure it has 10 characters
-      diff--;
+    resl += "0"; // padding 0 before the app sequence number to make sure it has 10 characters
+    diff--;
   }
   resl += appSeqNum;
   return resl;
@@ -39,7 +43,7 @@ function makeIdNumeric(id) {
 
 function getParameterByName(name, searchString) {
   var regex = new RegExp("[\\?&]" + name + "=([^&#]*)"),
-  results = regex.exec(searchString);
+    results = regex.exec(searchString);
   return results === null ? "" : decodeURIComponent(results[1].replace(/\+/g, " "));
 }
 
@@ -56,154 +60,185 @@ function getColumnIndex(columns, columnName) {
 }
 
 jQuery.extend( jQuery.fn.dataTableExt.oSort, {
-    "title-numeric-pre": function ( a ) {
-        var x = a.match(/title="*(-?[0-9\.]+)/)[1];
-        return parseFloat( x );
-    },
+  "title-numeric-pre": function ( a ) {
+    var x = a.match(/title="*(-?[0-9.]+)/)[1];
+    return parseFloat( x );
+  },
 
-    "title-numeric-asc": function ( a, b ) {
-        return ((a < b) ? -1 : ((a > b) ? 1 : 0));
-    },
+  "title-numeric-asc": function ( a, b ) {
+    return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+  },
 
-    "title-numeric-desc": function ( a, b ) {
-        return ((a < b) ? 1 : ((a > b) ? -1 : 0));
-    }
-} );
+  "title-numeric-desc": function ( a, b ) {
+    return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+  }
+});
 
 jQuery.extend( jQuery.fn.dataTableExt.oSort, {
-    "appid-numeric-pre": function ( a ) {
-        var x = a.match(/title="*(-?[0-9a-zA-Z\-\_]+)/)[1];
-        return makeIdNumeric(x);
-    },
+  "appid-numeric-pre": function ( a ) {
+    var x = a.match(/title="*(-?[0-9a-zA-Z\-_]+)/)[1];
+    return makeIdNumeric(x);
+  },
 
-    "appid-numeric-asc": function ( a, b ) {
-        return ((a < b) ? -1 : ((a > b) ? 1 : 0));
-    },
+  "appid-numeric-asc": function ( a, b ) {
+    return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+  },
 
-    "appid-numeric-desc": function ( a, b ) {
-        return ((a < b) ? 1 : ((a > b) ? -1 : 0));
-    }
-} );
+  "appid-numeric-desc": function ( a, b ) {
+    return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+  }
+});
 
 jQuery.extend( jQuery.fn.dataTableExt.ofnSearch, {
-    "appid-numeric": function ( a ) {
-        return a.replace(/[\r\n]/g, " ").replace(/<.*?>/g, "");
-    }
-} );
+  "appid-numeric": function ( a ) {
+    return a.replace(/[\r\n]/g, " ").replace(/<.*?>/g, "");
+  }
+});
 
 $(document).ajaxStop($.unblockUI);
 $(document).ajaxStart(function(){
-    $.blockUI({ message: '<h3>Loading history summary...</h3>'});
+  $.blockUI({ message: '<h3>Loading history summary...</h3>'});
 });
 
 $(document).ready(function() {
-    $.extend( $.fn.dataTable.defaults, {
-      stateSave: true,
-      lengthMenu: [[20,40,60,100,-1], [20, 40, 60, 100, "All"]],
-      pageLength: 20
-    });
+  $.extend( $.fn.dataTable.defaults, {
+    stateSave: true,
+    lengthMenu: [[20,40,60,100,-1], [20, 40, 60, 100, "All"]],
+    pageLength: 20
+  });
+
+  var historySummary = $("#history-summary");
+  var searchString = window.location.search;
+  var requestedIncomplete = getParameterByName("showIncomplete", searchString);
+  requestedIncomplete = (requestedIncomplete == "true" ? true : false);
+
+  var appParams = {
+    limit: appLimit,
+    status: (requestedIncomplete ? "running" : "completed")
+  };
+
+  $.getJSON(uiRoot + "/api/v1/applications", appParams, function(response, _ignored_status, _ignored_jqXHR) {
+    var array = [];
+    var hasMultipleAttempts = false;
+    for (var i in response) {
+      var app = response[i];
+      if (app["attempts"][0]["completed"] == requestedIncomplete) {
+        continue; // if we want to show for Incomplete, we skip the completed apps; otherwise skip incomplete ones.
+      }
+      var version = "Unknown"
+      if (app["attempts"].length > 0) {
+        version = app["attempts"][0]["appSparkVersion"]
+      }
+      var id = app["id"];
+      var name = app["name"];
+      if (app["attempts"].length > 1) {
+        hasMultipleAttempts = true;
+      }
 
-    var historySummary = $("#history-summary");
-    var searchString = window.location.search;
-    var requestedIncomplete = getParameterByName("showIncomplete", searchString);
-    requestedIncomplete = (requestedIncomplete == "true" ? true : false);
+      // TODO: Replace hasOwnProperty with prototype.hasOwnProperty after we find it's safe to do.
+      /* eslint-disable no-prototype-builtins */
+      for (var j in app["attempts"]) {
+        var attempt = app["attempts"][j];
+        attempt["startTime"] = formatTimeMillis(attempt["startTimeEpoch"]);
+        attempt["endTime"] = formatTimeMillis(attempt["endTimeEpoch"]);
+        attempt["lastUpdated"] = formatTimeMillis(attempt["lastUpdatedEpoch"]);
+        attempt["log"] = uiRoot + "/api/v1/applications/" + id + "/" +
+          (attempt.hasOwnProperty("attemptId") ? attempt["attemptId"] + "/" : "") + "logs";
+        attempt["durationMillisec"] = attempt["duration"];
+        attempt["duration"] = formatDuration(attempt["duration"]);
+        attempt["id"] = id;
+        attempt["name"] = name;
+        attempt["version"] = version;
+        attempt["attemptUrl"] = uiRoot + "/history/" + id + "/" +
+          (attempt.hasOwnProperty("attemptId") ? attempt["attemptId"] + "/" : "") + "jobs/";
+        array.push(attempt);
+      }
+      /* eslint-enable no-prototype-builtins */
+    }
+    if(array.length < 20) {
+      $.fn.dataTable.defaults.paging = false;
+    }
 
-    var appParams = {
-      limit: appLimit,
-      status: (requestedIncomplete ? "running" : "completed")
+    var data = {
+      "uiroot": uiRoot,
+      "applications": array,
+      "hasMultipleAttempts": hasMultipleAttempts,
+      "showCompletedColumns": !requestedIncomplete,
     };
 
-    $.getJSON(uiRoot + "/api/v1/applications", appParams, function(response,status,jqXHR) {
-      var array = [];
-      var hasMultipleAttempts = false;
-      for (var i in response) {
-        var app = response[i];
-        if (app["attempts"][0]["completed"] == requestedIncomplete) {
-          continue; // if we want to show for Incomplete, we skip the completed apps; otherwise skip incomplete ones.
-        }
-        var version = "Unknown"
-        if (app["attempts"].length > 0) {
-            version = app["attempts"][0]["appSparkVersion"]
-        }
-        var id = app["id"];
-        var name = app["name"];
-        if (app["attempts"].length > 1) {
-            hasMultipleAttempts = true;
-        }
-        var num = app["attempts"].length;
-        for (var j in app["attempts"]) {
-          var attempt = app["attempts"][j];
-          attempt["startTime"] = formatTimeMillis(attempt["startTimeEpoch"]);
-          attempt["endTime"] = formatTimeMillis(attempt["endTimeEpoch"]);
-          attempt["lastUpdated"] = formatTimeMillis(attempt["lastUpdatedEpoch"]);
-          attempt["log"] = uiRoot + "/api/v1/applications/" + id + "/" +
-            (attempt.hasOwnProperty("attemptId") ? attempt["attemptId"] + "/" : "") + "logs";
-          attempt["durationMillisec"] = attempt["duration"];
-          attempt["duration"] = formatDuration(attempt["duration"]);
-          var app_clone = {"id" : id, "name" : name, "version": version, "num" : num, "attempts" : [attempt]};
-          array.push(app_clone);
-        }
-      }
-      if(array.length < 20) {
-        $.fn.dataTable.defaults.paging = false;
-      }
-
-      var data = {
-        "uiroot": uiRoot,
-        "applications": array,
-        "hasMultipleAttempts": hasMultipleAttempts,
-        "showCompletedColumns": !requestedIncomplete,
+    $.get(uiRoot + "/static/historypage-template.html", function(template) {
+      var sibling = historySummary.prev();
+      historySummary.detach();
+      var apps = $(Mustache.render($(template).filter("#history-summary-template").html(),data));
+      var attemptIdColumnName = 'attemptId';
+      var startedColumnName = 'started';
+      var completedColumnName = 'completed';
+      var durationColumnName = 'duration';
+      var conf = {
+        "data": array,
+        "columns": [
+          {name: 'version', data: 'version' },
+          {
+            name: 'appId',
+            type: "appid-numeric",
+            data: 'id',
+            render:  (id, type, row) => `<span title="${id}"><a href="${row.attemptUrl}">${id}</a></span>`
+          },
+          {name: 'appName', data: 'name' },
+          {
+            name: attemptIdColumnName,
+            data: 'attemptId',
+            render: (attemptId, type, row) => (attemptId ? `<a href="${row.attemptUrl}">${attemptId}</a>` : '')
+          },
+          {name: startedColumnName, data: 'startTime' },
+          {name: completedColumnName, data: 'endTime' },
+          {name: durationColumnName, type: "title-numeric", data: 'duration' },
+          {name: 'user', data: 'sparkUser' },
+          {name: 'lastUpdated', data: 'lastUpdated' },
+          {
+            name: 'eventLog',
+            data: 'log',
+            render: (log, _ignored_type, _ignored_row) => `<a href="${log}" class="btn btn-info btn-mini">Download</a>`
+          },
+        ],
+        "aoColumnDefs": [
+          {
+            aTargets: [0, 1, 2],
+            fnCreatedCell: (nTd, _ignored_sData, _ignored_oData, _ignored_iRow, _ignored_iCol) => {
+              if (hasMultipleAttempts) {
+                $(nTd).css('background-color', '#fff');
+              }
+            }
+          },
+        ],
+        "autoWidth": false,
+        "deferRender": true
       };
 
-      $.get(uiRoot + "/static/historypage-template.html", function(template) {
-        var sibling = historySummary.prev();
-        historySummary.detach();
-        var apps = $(Mustache.render($(template).filter("#history-summary-template").html(),data));
-        var attemptIdColumnName = 'attemptId';
-        var startedColumnName = 'started';
-        var completedColumnName = 'completed';
-        var durationColumnName = 'duration';
-        var conf = {
-          "columns": [
-            {name: 'version'},
-            {name: 'appId', type: "appid-numeric"},
-            {name: 'appName'},
-            {name: attemptIdColumnName},
-            {name: startedColumnName},
-            {name: completedColumnName},
-            {name: durationColumnName, type: "title-numeric"},
-            {name: 'user'},
-            {name: 'lastUpdated'},
-            {name: 'eventLog'},
-          ],
-          "autoWidth": false,
-          "deferRender": true
-        };
-
-        if (hasMultipleAttempts) {
-          conf.rowsGroup = [
-            'appId:name',
-            'version:name',
-            'appName:name'
-          ];
-        } else {
-          conf.columns = removeColumnByName(conf.columns, attemptIdColumnName);
-        }
-
-        var defaultSortColumn = completedColumnName;
-        if (requestedIncomplete) {
-          defaultSortColumn = startedColumnName;
-          conf.columns = removeColumnByName(conf.columns, completedColumnName);
-          conf.columns = removeColumnByName(conf.columns, durationColumnName);
-        }
-        conf.order = [[ getColumnIndex(conf.columns, defaultSortColumn), "desc" ]];
-        conf.columnDefs = [
-          {"searchable": false, "targets": [getColumnIndex(conf.columns, durationColumnName)]}
+      if (hasMultipleAttempts) {
+        conf.rowsGroup = [
+          'appId:name',
+          'version:name',
+          'appName:name'
         ];
-        historySummary.append(apps);
-        apps.DataTable(conf);
-        sibling.after(historySummary);
-        $('#history-summary [data-toggle="tooltip"]').tooltip();
-      });
+      } else {
+        conf.columns = removeColumnByName(conf.columns, attemptIdColumnName);
+      }
+
+      var defaultSortColumn = completedColumnName;
+      if (requestedIncomplete) {
+        defaultSortColumn = startedColumnName;
+        conf.columns = removeColumnByName(conf.columns, completedColumnName);
+        conf.columns = removeColumnByName(conf.columns, durationColumnName);
+      }
+      conf.order = [[ getColumnIndex(conf.columns, defaultSortColumn), "desc" ]];
+      conf.columnDefs = [
+        {"searchable": false, "targets": [getColumnIndex(conf.columns, durationColumnName)]}
+      ];
+      historySummary.append(apps);
+      apps.DataTable(conf);
+      sibling.after(historySummary);
+      $('#history-summary [data-toggle="tooltip"]').tooltip();
     });
+  });
 });
diff --git a/core/src/main/resources/org/apache/spark/ui/static/initialize-tooltips.js b/core/src/main/resources/org/apache/spark/ui/static/initialize-tooltips.js
index 70f355dfb49cb..b273a19783f8f 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/initialize-tooltips.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/initialize-tooltips.js
@@ -15,7 +15,9 @@
  * limitations under the License.
  */
 
+/* global $ */
+
 $(document).ready(function(){
-   $("[data-toggle=tooltip]").tooltip({container: 'body'});
+  $("[data-toggle=tooltip]").tooltip({container: 'body'});
 });
 
diff --git a/core/src/main/resources/org/apache/spark/ui/static/jquery-3.4.1.min.js b/core/src/main/resources/org/apache/spark/ui/static/jquery-3.4.1.min.js
deleted file mode 100644
index 07c00cd227da0..0000000000000
--- a/core/src/main/resources/org/apache/spark/ui/static/jquery-3.4.1.min.js
+++ /dev/null
@@ -1,2 +0,0 @@
-/*! jQuery v3.4.1 | (c) JS Foundation and other contributors | jquery.org/license */
-!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(C,e){"use strict";var t=[],E=C.document,r=Object.getPrototypeOf,s=t.slice,g=t.concat,u=t.push,i=t.indexOf,n={},o=n.toString,v=n.hasOwnProperty,a=v.toString,l=a.call(Object),y={},m=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType},x=function(e){return null!=e&&e===e.window},c={type:!0,src:!0,nonce:!0,noModule:!0};function b(e,t,n){var r,i,o=(n=n||E).createElement("script");if(o.text=e,t)for(r in c)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function w(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[o.call(e)]||"object":typeof e}var f="3.4.1",k=function(e,t){return new k.fn.init(e,t)},p=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g;function d(e){var t=!!e&&"length"in e&&e.length,n=w(e);return!m(e)&&!x(e)&&("array"===n||0===t||"number"==typeof t&&0<t&&t-1 in e)}k.fn=k.prototype={jquery:f,constructor:k,length:0,toArray:function(){return s.call(this)},get:function(e){return null==e?s.call(this):e<0?this[e+this.length]:this[e]},pushStack:function(e){var t=k.merge(this.constructor(),e);return t.prevObject=this,t},each:function(e){return k.each(this,e)},map:function(n){return this.pushStack(k.map(this,function(e,t){return n.call(e,t,e)}))},slice:function(){return this.pushStack(s.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(e){var t=this.length,n=+e+(e<0?t:0);return this.pushStack(0<=n&&n<t?[this[n]]:[])},end:function(){return this.prevObject||this.constructor()},push:u,sort:t.sort,splice:t.splice},k.extend=k.fn.extend=function(){var e,t,n,r,i,o,a=arguments[0]||{},s=1,u=arguments.length,l=!1;for("boolean"==typeof a&&(l=a,a=arguments[s]||{},s++),"object"==typeof a||m(a)||(a={}),s===u&&(a=this,s--);s<u;s++)if(null!=(e=arguments[s]))for(t in e)r=e[t],"__proto__"!==t&&a!==r&&(l&&r&&(k.isPlainObject(r)||(i=Array.isArray(r)))?(n=a[t],o=i&&!Array.isArray(n)?[]:i||k.isPlainObject(n)?n:{},i=!1,a[t]=k.extend(l,o,r)):void 0!==r&&(a[t]=r));return a},k.extend({expando:"jQuery"+(f+Math.random()).replace(/\D/g,""),isReady:!0,error:function(e){throw new Error(e)},noop:function(){},isPlainObject:function(e){var t,n;return!(!e||"[object Object]"!==o.call(e))&&(!(t=r(e))||"function"==typeof(n=v.call(t,"constructor")&&t.constructor)&&a.call(n)===l)},isEmptyObject:function(e){var t;for(t in e)return!1;return!0},globalEval:function(e,t){b(e,{nonce:t&&t.nonce})},each:function(e,t){var n,r=0;if(d(e)){for(n=e.length;r<n;r++)if(!1===t.call(e[r],r,e[r]))break}else for(r in e)if(!1===t.call(e[r],r,e[r]))break;return e},trim:function(e){return null==e?"":(e+"").replace(p,"")},makeArray:function(e,t){var n=t||[];return null!=e&&(d(Object(e))?k.merge(n,"string"==typeof e?[e]:e):u.call(n,e)),n},inArray:function(e,t,n){return null==t?-1:i.call(t,e,n)},merge:function(e,t){for(var n=+t.length,r=0,i=e.length;r<n;r++)e[i++]=t[r];return e.length=i,e},grep:function(e,t,n){for(var r=[],i=0,o=e.length,a=!n;i<o;i++)!t(e[i],i)!==a&&r.push(e[i]);return r},map:function(e,t,n){var r,i,o=0,a=[];if(d(e))for(r=e.length;o<r;o++)null!=(i=t(e[o],o,n))&&a.push(i);else for(o in e)null!=(i=t(e[o],o,n))&&a.push(i);return g.apply([],a)},guid:1,support:y}),"function"==typeof Symbol&&(k.fn[Symbol.iterator]=t[Symbol.iterator]),k.each("Boolean Number String Function Array Date RegExp Object Error Symbol".split(" "),function(e,t){n["[object "+t+"]"]=t.toLowerCase()});var h=function(n){var e,d,b,o,i,h,f,g,w,u,l,T,C,a,E,v,s,c,y,k="sizzle"+1*new Date,m=n.document,S=0,r=0,p=ue(),x=ue(),N=ue(),A=ue(),D=function(e,t){return e===t&&(l=!0),0},j={}.hasOwnProperty,t=[],q=t.pop,L=t.push,H=t.push,O=t.slice,P=function(e,t){for(var n=0,r=e.length;n<r;n++)if(e[n]===t)return n;return-1},R="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",M="[\\x20\\t\\r\\n\\f]",I="(?:\\\\.|[\\w-]|[^\0-\\xa0])+",W="\\["+M+"*("+I+")(?:"+M+"*([*^$|!~]?=)"+M+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+I+"))|)"+M+"*\\]",$=":("+I+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+W+")*)|.*)\\)|)",F=new RegExp(M+"+","g"),B=new RegExp("^"+M+"+|((?:^|[^\\\\])(?:\\\\.)*)"+M+"+$","g"),_=new RegExp("^"+M+"*,"+M+"*"),z=new RegExp("^"+M+"*([>+~]|"+M+")"+M+"*"),U=new RegExp(M+"|>"),X=new RegExp($),V=new RegExp("^"+I+"$"),G={ID:new RegExp("^#("+I+")"),CLASS:new RegExp("^\\.("+I+")"),TAG:new RegExp("^("+I+"|[*])"),ATTR:new RegExp("^"+W),PSEUDO:new RegExp("^"+$),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+R+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+M+"*((?:-\\d)?\\d*)"+M+"*\\)|)(?=[^-]|$)","i")},Y=/HTML$/i,Q=/^(?:input|select|textarea|button)$/i,J=/^h\d$/i,K=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ee=/[+~]/,te=new RegExp("\\\\([\\da-f]{1,6}"+M+"?|("+M+")|.)","ig"),ne=function(e,t,n){var r="0x"+t-65536;return r!=r||n?t:r<0?String.fromCharCode(r+65536):String.fromCharCode(r>>10|55296,1023&r|56320)},re=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ie=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},oe=function(){T()},ae=be(function(e){return!0===e.disabled&&"fieldset"===e.nodeName.toLowerCase()},{dir:"parentNode",next:"legend"});try{H.apply(t=O.call(m.childNodes),m.childNodes),t[m.childNodes.length].nodeType}catch(e){H={apply:t.length?function(e,t){L.apply(e,O.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function se(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,p=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==p&&9!==p&&11!==p)return n;if(!r&&((e?e.ownerDocument||e:m)!==C&&T(e),e=e||C,E)){if(11!==p&&(u=Z.exec(t)))if(i=u[1]){if(9===p){if(!(a=e.getElementById(i)))return n;if(a.id===i)return n.push(a),n}else if(f&&(a=f.getElementById(i))&&y(e,a)&&a.id===i)return n.push(a),n}else{if(u[2])return H.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&d.getElementsByClassName&&e.getElementsByClassName)return H.apply(n,e.getElementsByClassName(i)),n}if(d.qsa&&!A[t+" "]&&(!v||!v.test(t))&&(1!==p||"object"!==e.nodeName.toLowerCase())){if(c=t,f=e,1===p&&U.test(t)){(s=e.getAttribute("id"))?s=s.replace(re,ie):e.setAttribute("id",s=k),o=(l=h(t)).length;while(o--)l[o]="#"+s+" "+xe(l[o]);c=l.join(","),f=ee.test(t)&&ye(e.parentNode)||e}try{return H.apply(n,f.querySelectorAll(c)),n}catch(e){A(t,!0)}finally{s===k&&e.removeAttribute("id")}}}return g(t.replace(B,"$1"),e,n,r)}function ue(){var r=[];return function e(t,n){return r.push(t+" ")>b.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function le(e){return e[k]=!0,e}function ce(e){var t=C.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){var n=e.split("|"),r=n.length;while(r--)b.attrHandle[n[r]]=t}function pe(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function de(t){return function(e){return"input"===e.nodeName.toLowerCase()&&e.type===t}}function he(n){return function(e){var t=e.nodeName.toLowerCase();return("input"===t||"button"===t)&&e.type===n}}function ge(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&ae(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function ve(a){return le(function(o){return o=+o,le(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function ye(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}for(e in d=se.support={},i=se.isXML=function(e){var t=e.namespaceURI,n=(e.ownerDocument||e).documentElement;return!Y.test(t||n&&n.nodeName||"HTML")},T=se.setDocument=function(e){var t,n,r=e?e.ownerDocument||e:m;return r!==C&&9===r.nodeType&&r.documentElement&&(a=(C=r).documentElement,E=!i(C),m!==C&&(n=C.defaultView)&&n.top!==n&&(n.addEventListener?n.addEventListener("unload",oe,!1):n.attachEvent&&n.attachEvent("onunload",oe)),d.attributes=ce(function(e){return e.className="i",!e.getAttribute("className")}),d.getElementsByTagName=ce(function(e){return e.appendChild(C.createComment("")),!e.getElementsByTagName("*").length}),d.getElementsByClassName=K.test(C.getElementsByClassName),d.getById=ce(function(e){return a.appendChild(e).id=k,!C.getElementsByName||!C.getElementsByName(k).length}),d.getById?(b.filter.ID=function(e){var t=e.replace(te,ne);return function(e){return e.getAttribute("id")===t}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n=t.getElementById(e);return n?[n]:[]}}):(b.filter.ID=function(e){var n=e.replace(te,ne);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),b.find.TAG=d.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):d.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},b.find.CLASS=d.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&E)return t.getElementsByClassName(e)},s=[],v=[],(d.qsa=K.test(C.querySelectorAll))&&(ce(function(e){a.appendChild(e).innerHTML="<a id='"+k+"'></a><select id='"+k+"-\r\\' msallowcapture=''><option selected=''></option></select>",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+M+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+M+"*(?:value|"+R+")"),e.querySelectorAll("[id~="+k+"-]").length||v.push("~="),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+k+"+*").length||v.push(".#.+[+~]")}),ce(function(e){e.innerHTML="<a href='' disabled='disabled'></a><select disabled='disabled'><option/></select>";var t=C.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+M+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")})),(d.matchesSelector=K.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce(function(e){d.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",$)}),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=K.test(a.compareDocumentPosition),y=t||K.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},D=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)===(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!d.sortDetached&&t.compareDocumentPosition(e)===n?e===C||e.ownerDocument===m&&y(m,e)?-1:t===C||t.ownerDocument===m&&y(m,t)?1:u?P(u,e)-P(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e===C?-1:t===C?1:i?-1:o?1:u?P(u,e)-P(u,t):0;if(i===o)return pe(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?pe(a[r],s[r]):a[r]===m?-1:s[r]===m?1:0}),C},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if((e.ownerDocument||e)!==C&&T(e),d.matchesSelector&&E&&!A[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||d.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){A(t,!0)}return 0<se(t,C,null,[e]).length},se.contains=function(e,t){return(e.ownerDocument||e)!==C&&T(e),y(e,t)},se.attr=function(e,t){(e.ownerDocument||e)!==C&&T(e);var n=b.attrHandle[t.toLowerCase()],r=n&&j.call(b.attrHandle,t.toLowerCase())?n(e,t,!E):void 0;return void 0!==r?r:d.attributes||!E?e.getAttribute(t):(r=e.getAttributeNode(t))&&r.specified?r.value:null},se.escape=function(e){return(e+"").replace(re,ie)},se.error=function(e){throw new Error("Syntax error, unrecognized expression: "+e)},se.uniqueSort=function(e){var t,n=[],r=0,i=0;if(l=!d.detectDuplicates,u=!d.sortStable&&e.slice(0),e.sort(D),l){while(t=e[i++])t===e[i]&&(r=n.push(i));while(r--)e.splice(n[r],1)}return u=null,e},o=se.getText=function(e){var t,n="",r=0,i=e.nodeType;if(i){if(1===i||9===i||11===i){if("string"==typeof e.textContent)return e.textContent;for(e=e.firstChild;e;e=e.nextSibling)n+=o(e)}else if(3===i||4===i)return e.nodeValue}else while(t=e[r++])n+=o(t);return n},(b=se.selectors={cacheLength:50,createPseudo:le,match:G,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||"").replace(te,ne),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return G.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&X.test(n)&&(t=h(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=p[e+" "];return t||(t=new RegExp("(^|"+M+")"+e+"("+M+"|$)"))&&p(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=se.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1<t.indexOf(i):"$="===r?i&&t.slice(-i.length)===i:"~="===r?-1<(" "+t.replace(F," ")+" ").indexOf(i):"|="===r&&(t===i||t.slice(0,i.length+1)===i+"-"))}},CHILD:function(h,e,t,g,v){var y="nth"!==h.slice(0,3),m="last"!==h.slice(-4),x="of-type"===e;return 1===g&&0===v?function(e){return!!e.parentNode}:function(e,t,n){var r,i,o,a,s,u,l=y!==m?"nextSibling":"previousSibling",c=e.parentNode,f=x&&e.nodeName.toLowerCase(),p=!n&&!x,d=!1;if(c){if(y){while(l){a=e;while(a=a[l])if(x?a.nodeName.toLowerCase()===f:1===a.nodeType)return!1;u=l="only"===h&&!u&&"nextSibling"}return!0}if(u=[m?c.firstChild:c.lastChild],m&&p){d=(s=(r=(i=(o=(a=c)[k]||(a[k]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]||[])[0]===S&&r[1])&&r[2],a=s&&c.childNodes[s];while(a=++s&&a&&a[l]||(d=s=0)||u.pop())if(1===a.nodeType&&++d&&a===e){i[h]=[S,s,d];break}}else if(p&&(d=s=(r=(i=(o=(a=e)[k]||(a[k]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]||[])[0]===S&&r[1]),!1===d)while(a=++s&&a&&a[l]||(d=s=0)||u.pop())if((x?a.nodeName.toLowerCase()===f:1===a.nodeType)&&++d&&(p&&((i=(o=a[k]||(a[k]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]=[S,d]),a===e))break;return(d-=v)===g||d%g==0&&0<=d/g}}},PSEUDO:function(e,o){var t,a=b.pseudos[e]||b.setFilters[e.toLowerCase()]||se.error("unsupported pseudo: "+e);return a[k]?a(o):1<a.length?(t=[e,e,"",o],b.setFilters.hasOwnProperty(e.toLowerCase())?le(function(e,t){var n,r=a(e,o),i=r.length;while(i--)e[n=P(e,r[i])]=!(t[n]=r[i])}):function(e){return a(e,0,t)}):a}},pseudos:{not:le(function(e){var r=[],i=[],s=f(e.replace(B,"$1"));return s[k]?le(function(e,t,n,r){var i,o=s(e,null,r,[]),a=e.length;while(a--)(i=o[a])&&(e[a]=!(t[a]=i))}):function(e,t,n){return r[0]=e,s(r,null,n,i),r[0]=null,!i.pop()}}),has:le(function(t){return function(e){return 0<se(t,e).length}}),contains:le(function(t){return t=t.replace(te,ne),function(e){return-1<(e.textContent||o(e)).indexOf(t)}}),lang:le(function(n){return V.test(n||"")||se.error("unsupported lang: "+n),n=n.replace(te,ne).toLowerCase(),function(e){var t;do{if(t=E?e.lang:e.getAttribute("xml:lang")||e.getAttribute("lang"))return(t=t.toLowerCase())===n||0===t.indexOf(n+"-")}while((e=e.parentNode)&&1===e.nodeType);return!1}}),target:function(e){var t=n.location&&n.location.hash;return t&&t.slice(1)===e.id},root:function(e){return e===a},focus:function(e){return e===C.activeElement&&(!C.hasFocus||C.hasFocus())&&!!(e.type||e.href||~e.tabIndex)},enabled:ge(!1),disabled:ge(!0),checked:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&!!e.checked||"option"===t&&!!e.selected},selected:function(e){return e.parentNode&&e.parentNode.selectedIndex,!0===e.selected},empty:function(e){for(e=e.firstChild;e;e=e.nextSibling)if(e.nodeType<6)return!1;return!0},parent:function(e){return!b.pseudos.empty(e)},header:function(e){return J.test(e.nodeName)},input:function(e){return Q.test(e.nodeName)},button:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&"button"===e.type||"button"===t},text:function(e){var t;return"input"===e.nodeName.toLowerCase()&&"text"===e.type&&(null==(t=e.getAttribute("type"))||"text"===t.toLowerCase())},first:ve(function(){return[0]}),last:ve(function(e,t){return[t-1]}),eq:ve(function(e,t,n){return[n<0?n+t:n]}),even:ve(function(e,t){for(var n=0;n<t;n+=2)e.push(n);return e}),odd:ve(function(e,t){for(var n=1;n<t;n+=2)e.push(n);return e}),lt:ve(function(e,t,n){for(var r=n<0?n+t:t<n?t:n;0<=--r;)e.push(r);return e}),gt:ve(function(e,t,n){for(var r=n<0?n+t:n;++r<t;)e.push(r);return e})}}).pseudos.nth=b.pseudos.eq,{radio:!0,checkbox:!0,file:!0,password:!0,image:!0})b.pseudos[e]=de(e);for(e in{submit:!0,reset:!0})b.pseudos[e]=he(e);function me(){}function xe(e){for(var t=0,n=e.length,r="";t<n;t++)r+=e[t].value;return r}function be(s,e,t){var u=e.dir,l=e.next,c=l||u,f=t&&"parentNode"===c,p=r++;return e.first?function(e,t,n){while(e=e[u])if(1===e.nodeType||f)return s(e,t,n);return!1}:function(e,t,n){var r,i,o,a=[S,p];if(n){while(e=e[u])if((1===e.nodeType||f)&&s(e,t,n))return!0}else while(e=e[u])if(1===e.nodeType||f)if(i=(o=e[k]||(e[k]={}))[e.uniqueID]||(o[e.uniqueID]={}),l&&l===e.nodeName.toLowerCase())e=e[u]||e;else{if((r=i[c])&&r[0]===S&&r[1]===p)return a[2]=r[2];if((i[c]=a)[2]=s(e,t,n))return!0}return!1}}function we(i){return 1<i.length?function(e,t,n){var r=i.length;while(r--)if(!i[r](e,t,n))return!1;return!0}:i[0]}function Te(e,t,n,r,i){for(var o,a=[],s=0,u=e.length,l=null!=t;s<u;s++)(o=e[s])&&(n&&!n(o,r,i)||(a.push(o),l&&t.push(s)));return a}function Ce(d,h,g,v,y,e){return v&&!v[k]&&(v=Ce(v)),y&&!y[k]&&(y=Ce(y,e)),le(function(e,t,n,r){var i,o,a,s=[],u=[],l=t.length,c=e||function(e,t,n){for(var r=0,i=t.length;r<i;r++)se(e,t[r],n);return n}(h||"*",n.nodeType?[n]:n,[]),f=!d||!e&&h?c:Te(c,s,d,n,r),p=g?y||(e?d:l||v)?[]:t:f;if(g&&g(f,p,n,r),v){i=Te(p,u),v(i,[],n,r),o=i.length;while(o--)(a=i[o])&&(p[u[o]]=!(f[u[o]]=a))}if(e){if(y||d){if(y){i=[],o=p.length;while(o--)(a=p[o])&&i.push(f[o]=a);y(null,p=[],i,r)}o=p.length;while(o--)(a=p[o])&&-1<(i=y?P(e,a):s[o])&&(e[i]=!(t[i]=a))}}else p=Te(p===t?p.splice(l,p.length):p),y?y(null,t,p,r):H.apply(t,p)})}function Ee(e){for(var i,t,n,r=e.length,o=b.relative[e[0].type],a=o||b.relative[" "],s=o?1:0,u=be(function(e){return e===i},a,!0),l=be(function(e){return-1<P(i,e)},a,!0),c=[function(e,t,n){var r=!o&&(n||t!==w)||((i=t).nodeType?u(e,t,n):l(e,t,n));return i=null,r}];s<r;s++)if(t=b.relative[e[s].type])c=[be(we(c),t)];else{if((t=b.filter[e[s].type].apply(null,e[s].matches))[k]){for(n=++s;n<r;n++)if(b.relative[e[n].type])break;return Ce(1<s&&we(c),1<s&&xe(e.slice(0,s-1).concat({value:" "===e[s-2].type?"*":""})).replace(B,"$1"),t,s<n&&Ee(e.slice(s,n)),n<r&&Ee(e=e.slice(n)),n<r&&xe(e))}c.push(t)}return we(c)}return me.prototype=b.filters=b.pseudos,b.setFilters=new me,h=se.tokenize=function(e,t){var n,r,i,o,a,s,u,l=x[e+" "];if(l)return t?0:l.slice(0);a=e,s=[],u=b.preFilter;while(a){for(o in n&&!(r=_.exec(a))||(r&&(a=a.slice(r[0].length)||a),s.push(i=[])),n=!1,(r=z.exec(a))&&(n=r.shift(),i.push({value:n,type:r[0].replace(B," ")}),a=a.slice(n.length)),b.filter)!(r=G[o].exec(a))||u[o]&&!(r=u[o](r))||(n=r.shift(),i.push({value:n,type:o,matches:r}),a=a.slice(n.length));if(!n)break}return t?a.length:a?se.error(e):x(e,s).slice(0)},f=se.compile=function(e,t){var n,v,y,m,x,r,i=[],o=[],a=N[e+" "];if(!a){t||(t=h(e)),n=t.length;while(n--)(a=Ee(t[n]))[k]?i.push(a):o.push(a);(a=N(e,(v=o,m=0<(y=i).length,x=0<v.length,r=function(e,t,n,r,i){var o,a,s,u=0,l="0",c=e&&[],f=[],p=w,d=e||x&&b.find.TAG("*",i),h=S+=null==p?1:Math.random()||.1,g=d.length;for(i&&(w=t===C||t||i);l!==g&&null!=(o=d[l]);l++){if(x&&o){a=0,t||o.ownerDocument===C||(T(o),n=!E);while(s=v[a++])if(s(o,t||C,n)){r.push(o);break}i&&(S=h)}m&&((o=!s&&o)&&u--,e&&c.push(o))}if(u+=l,m&&l!==u){a=0;while(s=y[a++])s(c,f,t,n);if(e){if(0<u)while(l--)c[l]||f[l]||(f[l]=q.call(r));f=Te(f)}H.apply(r,f),i&&!e&&0<f.length&&1<u+y.length&&se.uniqueSort(r)}return i&&(S=h,w=p),c},m?le(r):r))).selector=e}return a},g=se.select=function(e,t,n,r){var i,o,a,s,u,l="function"==typeof e&&e,c=!r&&h(e=l.selector||e);if(n=n||[],1===c.length){if(2<(o=c[0]=c[0].slice(0)).length&&"ID"===(a=o[0]).type&&9===t.nodeType&&E&&b.relative[o[1].type]){if(!(t=(b.find.ID(a.matches[0].replace(te,ne),t)||[])[0]))return n;l&&(t=t.parentNode),e=e.slice(o.shift().value.length)}i=G.needsContext.test(e)?0:o.length;while(i--){if(a=o[i],b.relative[s=a.type])break;if((u=b.find[s])&&(r=u(a.matches[0].replace(te,ne),ee.test(o[0].type)&&ye(t.parentNode)||t))){if(o.splice(i,1),!(e=r.length&&xe(o)))return H.apply(n,r),n;break}}}return(l||f(e,c))(r,t,!E,n,!t||ee.test(e)&&ye(t.parentNode)||t),n},d.sortStable=k.split("").sort(D).join("")===k,d.detectDuplicates=!!l,T(),d.sortDetached=ce(function(e){return 1&e.compareDocumentPosition(C.createElement("fieldset"))}),ce(function(e){return e.innerHTML="<a href='#'></a>","#"===e.firstChild.getAttribute("href")})||fe("type|href|height|width",function(e,t,n){if(!n)return e.getAttribute(t,"type"===t.toLowerCase()?1:2)}),d.attributes&&ce(function(e){return e.innerHTML="<input/>",e.firstChild.setAttribute("value",""),""===e.firstChild.getAttribute("value")})||fe("value",function(e,t,n){if(!n&&"input"===e.nodeName.toLowerCase())return e.defaultValue}),ce(function(e){return null==e.getAttribute("disabled")})||fe(R,function(e,t,n){var r;if(!n)return!0===e[t]?t.toLowerCase():(r=e.getAttributeNode(t))&&r.specified?r.value:null}),se}(C);k.find=h,k.expr=h.selectors,k.expr[":"]=k.expr.pseudos,k.uniqueSort=k.unique=h.uniqueSort,k.text=h.getText,k.isXMLDoc=h.isXML,k.contains=h.contains,k.escapeSelector=h.escape;var T=function(e,t,n){var r=[],i=void 0!==n;while((e=e[t])&&9!==e.nodeType)if(1===e.nodeType){if(i&&k(e).is(n))break;r.push(e)}return r},S=function(e,t){for(var n=[];e;e=e.nextSibling)1===e.nodeType&&e!==t&&n.push(e);return n},N=k.expr.match.needsContext;function A(e,t){return e.nodeName&&e.nodeName.toLowerCase()===t.toLowerCase()}var D=/^<([a-z][^\/\0>:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function j(e,n,r){return m(n)?k.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?k.grep(e,function(e){return e===n!==r}):"string"!=typeof n?k.grep(e,function(e){return-1<i.call(n,e)!==r}):k.filter(n,e,r)}k.filter=function(e,t,n){var r=t[0];return n&&(e=":not("+e+")"),1===t.length&&1===r.nodeType?k.find.matchesSelector(r,e)?[r]:[]:k.find.matches(e,k.grep(t,function(e){return 1===e.nodeType}))},k.fn.extend({find:function(e){var t,n,r=this.length,i=this;if("string"!=typeof e)return this.pushStack(k(e).filter(function(){for(t=0;t<r;t++)if(k.contains(i[t],this))return!0}));for(n=this.pushStack([]),t=0;t<r;t++)k.find(e,i[t],n);return 1<r?k.uniqueSort(n):n},filter:function(e){return this.pushStack(j(this,e||[],!1))},not:function(e){return this.pushStack(j(this,e||[],!0))},is:function(e){return!!j(this,"string"==typeof e&&N.test(e)?k(e):e||[],!1).length}});var q,L=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]+))$/;(k.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||q,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:L.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof k?t[0]:t,k.merge(this,k.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:E,!0)),D.test(r[1])&&k.isPlainObject(t))for(r in t)m(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=E.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):m(e)?void 0!==n.ready?n.ready(e):e(k):k.makeArray(e,this)}).prototype=k.fn,q=k(E);var H=/^(?:parents|prev(?:Until|All))/,O={children:!0,contents:!0,next:!0,prev:!0};function P(e,t){while((e=e[t])&&1!==e.nodeType);return e}k.fn.extend({has:function(e){var t=k(e,this),n=t.length;return this.filter(function(){for(var e=0;e<n;e++)if(k.contains(this,t[e]))return!0})},closest:function(e,t){var n,r=0,i=this.length,o=[],a="string"!=typeof e&&k(e);if(!N.test(e))for(;r<i;r++)for(n=this[r];n&&n!==t;n=n.parentNode)if(n.nodeType<11&&(a?-1<a.index(n):1===n.nodeType&&k.find.matchesSelector(n,e))){o.push(n);break}return this.pushStack(1<o.length?k.uniqueSort(o):o)},index:function(e){return e?"string"==typeof e?i.call(k(e),this[0]):i.call(this,e.jquery?e[0]:e):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(e,t){return this.pushStack(k.uniqueSort(k.merge(this.get(),k(e,t))))},addBack:function(e){return this.add(null==e?this.prevObject:this.prevObject.filter(e))}}),k.each({parent:function(e){var t=e.parentNode;return t&&11!==t.nodeType?t:null},parents:function(e){return T(e,"parentNode")},parentsUntil:function(e,t,n){return T(e,"parentNode",n)},next:function(e){return P(e,"nextSibling")},prev:function(e){return P(e,"previousSibling")},nextAll:function(e){return T(e,"nextSibling")},prevAll:function(e){return T(e,"previousSibling")},nextUntil:function(e,t,n){return T(e,"nextSibling",n)},prevUntil:function(e,t,n){return T(e,"previousSibling",n)},siblings:function(e){return S((e.parentNode||{}).firstChild,e)},children:function(e){return S(e.firstChild)},contents:function(e){return"undefined"!=typeof e.contentDocument?e.contentDocument:(A(e,"template")&&(e=e.content||e),k.merge([],e.childNodes))}},function(r,i){k.fn[r]=function(e,t){var n=k.map(this,i,e);return"Until"!==r.slice(-5)&&(t=e),t&&"string"==typeof t&&(n=k.filter(t,n)),1<this.length&&(O[r]||k.uniqueSort(n),H.test(r)&&n.reverse()),this.pushStack(n)}});var R=/[^\x20\t\r\n\f]+/g;function M(e){return e}function I(e){throw e}function W(e,t,n,r){var i;try{e&&m(i=e.promise)?i.call(e).done(t).fail(n):e&&m(i=e.then)?i.call(e,t,n):t.apply(void 0,[e].slice(r))}catch(e){n.apply(void 0,[e])}}k.Callbacks=function(r){var e,n;r="string"==typeof r?(e=r,n={},k.each(e.match(R)||[],function(e,t){n[t]=!0}),n):k.extend({},r);var i,t,o,a,s=[],u=[],l=-1,c=function(){for(a=a||r.once,o=i=!0;u.length;l=-1){t=u.shift();while(++l<s.length)!1===s[l].apply(t[0],t[1])&&r.stopOnFalse&&(l=s.length,t=!1)}r.memory||(t=!1),i=!1,a&&(s=t?[]:"")},f={add:function(){return s&&(t&&!i&&(l=s.length-1,u.push(t)),function n(e){k.each(e,function(e,t){m(t)?r.unique&&f.has(t)||s.push(t):t&&t.length&&"string"!==w(t)&&n(t)})}(arguments),t&&!i&&c()),this},remove:function(){return k.each(arguments,function(e,t){var n;while(-1<(n=k.inArray(t,s,n)))s.splice(n,1),n<=l&&l--}),this},has:function(e){return e?-1<k.inArray(e,s):0<s.length},empty:function(){return s&&(s=[]),this},disable:function(){return a=u=[],s=t="",this},disabled:function(){return!s},lock:function(){return a=u=[],t||i||(s=t=""),this},locked:function(){return!!a},fireWith:function(e,t){return a||(t=[e,(t=t||[]).slice?t.slice():t],u.push(t),i||c()),this},fire:function(){return f.fireWith(this,arguments),this},fired:function(){return!!o}};return f},k.extend({Deferred:function(e){var o=[["notify","progress",k.Callbacks("memory"),k.Callbacks("memory"),2],["resolve","done",k.Callbacks("once memory"),k.Callbacks("once memory"),0,"resolved"],["reject","fail",k.Callbacks("once memory"),k.Callbacks("once memory"),1,"rejected"]],i="pending",a={state:function(){return i},always:function(){return s.done(arguments).fail(arguments),this},"catch":function(e){return a.then(null,e)},pipe:function(){var i=arguments;return k.Deferred(function(r){k.each(o,function(e,t){var n=m(i[t[4]])&&i[t[4]];s[t[1]](function(){var e=n&&n.apply(this,arguments);e&&m(e.promise)?e.promise().progress(r.notify).done(r.resolve).fail(r.reject):r[t[0]+"With"](this,n?[e]:arguments)})}),i=null}).promise()},then:function(t,n,r){var u=0;function l(i,o,a,s){return function(){var n=this,r=arguments,e=function(){var e,t;if(!(i<u)){if((e=a.apply(n,r))===o.promise())throw new TypeError("Thenable self-resolution");t=e&&("object"==typeof e||"function"==typeof e)&&e.then,m(t)?s?t.call(e,l(u,o,M,s),l(u,o,I,s)):(u++,t.call(e,l(u,o,M,s),l(u,o,I,s),l(u,o,M,o.notifyWith))):(a!==M&&(n=void 0,r=[e]),(s||o.resolveWith)(n,r))}},t=s?e:function(){try{e()}catch(e){k.Deferred.exceptionHook&&k.Deferred.exceptionHook(e,t.stackTrace),u<=i+1&&(a!==I&&(n=void 0,r=[e]),o.rejectWith(n,r))}};i?t():(k.Deferred.getStackHook&&(t.stackTrace=k.Deferred.getStackHook()),C.setTimeout(t))}}return k.Deferred(function(e){o[0][3].add(l(0,e,m(r)?r:M,e.notifyWith)),o[1][3].add(l(0,e,m(t)?t:M)),o[2][3].add(l(0,e,m(n)?n:I))}).promise()},promise:function(e){return null!=e?k.extend(e,a):a}},s={};return k.each(o,function(e,t){var n=t[2],r=t[5];a[t[1]]=n.add,r&&n.add(function(){i=r},o[3-e][2].disable,o[3-e][3].disable,o[0][2].lock,o[0][3].lock),n.add(t[3].fire),s[t[0]]=function(){return s[t[0]+"With"](this===s?void 0:this,arguments),this},s[t[0]+"With"]=n.fireWith}),a.promise(s),e&&e.call(s,s),s},when:function(e){var n=arguments.length,t=n,r=Array(t),i=s.call(arguments),o=k.Deferred(),a=function(t){return function(e){r[t]=this,i[t]=1<arguments.length?s.call(arguments):e,--n||o.resolveWith(r,i)}};if(n<=1&&(W(e,o.done(a(t)).resolve,o.reject,!n),"pending"===o.state()||m(i[t]&&i[t].then)))return o.then();while(t--)W(i[t],a(t),o.reject);return o.promise()}});var $=/^(Eval|Internal|Range|Reference|Syntax|Type|URI)Error$/;k.Deferred.exceptionHook=function(e,t){C.console&&C.console.warn&&e&&$.test(e.name)&&C.console.warn("jQuery.Deferred exception: "+e.message,e.stack,t)},k.readyException=function(e){C.setTimeout(function(){throw e})};var F=k.Deferred();function B(){E.removeEventListener("DOMContentLoaded",B),C.removeEventListener("load",B),k.ready()}k.fn.ready=function(e){return F.then(e)["catch"](function(e){k.readyException(e)}),this},k.extend({isReady:!1,readyWait:1,ready:function(e){(!0===e?--k.readyWait:k.isReady)||(k.isReady=!0)!==e&&0<--k.readyWait||F.resolveWith(E,[k])}}),k.ready.then=F.then,"complete"===E.readyState||"loading"!==E.readyState&&!E.documentElement.doScroll?C.setTimeout(k.ready):(E.addEventListener("DOMContentLoaded",B),C.addEventListener("load",B));var _=function(e,t,n,r,i,o,a){var s=0,u=e.length,l=null==n;if("object"===w(n))for(s in i=!0,n)_(e,t,s,n[s],!0,o,a);else if(void 0!==r&&(i=!0,m(r)||(a=!0),l&&(a?(t.call(e,r),t=null):(l=t,t=function(e,t,n){return l.call(k(e),n)})),t))for(;s<u;s++)t(e[s],n,a?r:r.call(e[s],s,t(e[s],n)));return i?e:l?t.call(e):u?t(e[0],n):o},z=/^-ms-/,U=/-([a-z])/g;function X(e,t){return t.toUpperCase()}function V(e){return e.replace(z,"ms-").replace(U,X)}var G=function(e){return 1===e.nodeType||9===e.nodeType||!+e.nodeType};function Y(){this.expando=k.expando+Y.uid++}Y.uid=1,Y.prototype={cache:function(e){var t=e[this.expando];return t||(t={},G(e)&&(e.nodeType?e[this.expando]=t:Object.defineProperty(e,this.expando,{value:t,configurable:!0}))),t},set:function(e,t,n){var r,i=this.cache(e);if("string"==typeof t)i[V(t)]=n;else for(r in t)i[V(r)]=t[r];return i},get:function(e,t){return void 0===t?this.cache(e):e[this.expando]&&e[this.expando][V(t)]},access:function(e,t,n){return void 0===t||t&&"string"==typeof t&&void 0===n?this.get(e,t):(this.set(e,t,n),void 0!==n?n:t)},remove:function(e,t){var n,r=e[this.expando];if(void 0!==r){if(void 0!==t){n=(t=Array.isArray(t)?t.map(V):(t=V(t))in r?[t]:t.match(R)||[]).length;while(n--)delete r[t[n]]}(void 0===t||k.isEmptyObject(r))&&(e.nodeType?e[this.expando]=void 0:delete e[this.expando])}},hasData:function(e){var t=e[this.expando];return void 0!==t&&!k.isEmptyObject(t)}};var Q=new Y,J=new Y,K=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,Z=/[A-Z]/g;function ee(e,t,n){var r,i;if(void 0===n&&1===e.nodeType)if(r="data-"+t.replace(Z,"-$&").toLowerCase(),"string"==typeof(n=e.getAttribute(r))){try{n="true"===(i=n)||"false"!==i&&("null"===i?null:i===+i+""?+i:K.test(i)?JSON.parse(i):i)}catch(e){}J.set(e,t,n)}else n=void 0;return n}k.extend({hasData:function(e){return J.hasData(e)||Q.hasData(e)},data:function(e,t,n){return J.access(e,t,n)},removeData:function(e,t){J.remove(e,t)},_data:function(e,t,n){return Q.access(e,t,n)},_removeData:function(e,t){Q.remove(e,t)}}),k.fn.extend({data:function(n,e){var t,r,i,o=this[0],a=o&&o.attributes;if(void 0===n){if(this.length&&(i=J.get(o),1===o.nodeType&&!Q.get(o,"hasDataAttrs"))){t=a.length;while(t--)a[t]&&0===(r=a[t].name).indexOf("data-")&&(r=V(r.slice(5)),ee(o,r,i[r]));Q.set(o,"hasDataAttrs",!0)}return i}return"object"==typeof n?this.each(function(){J.set(this,n)}):_(this,function(e){var t;if(o&&void 0===e)return void 0!==(t=J.get(o,n))?t:void 0!==(t=ee(o,n))?t:void 0;this.each(function(){J.set(this,n,e)})},null,e,1<arguments.length,null,!0)},removeData:function(e){return this.each(function(){J.remove(this,e)})}}),k.extend({queue:function(e,t,n){var r;if(e)return t=(t||"fx")+"queue",r=Q.get(e,t),n&&(!r||Array.isArray(n)?r=Q.access(e,t,k.makeArray(n)):r.push(n)),r||[]},dequeue:function(e,t){t=t||"fx";var n=k.queue(e,t),r=n.length,i=n.shift(),o=k._queueHooks(e,t);"inprogress"===i&&(i=n.shift(),r--),i&&("fx"===t&&n.unshift("inprogress"),delete o.stop,i.call(e,function(){k.dequeue(e,t)},o)),!r&&o&&o.empty.fire()},_queueHooks:function(e,t){var n=t+"queueHooks";return Q.get(e,n)||Q.access(e,n,{empty:k.Callbacks("once memory").add(function(){Q.remove(e,[t+"queue",n])})})}}),k.fn.extend({queue:function(t,n){var e=2;return"string"!=typeof t&&(n=t,t="fx",e--),arguments.length<e?k.queue(this[0],t):void 0===n?this:this.each(function(){var e=k.queue(this,t,n);k._queueHooks(this,t),"fx"===t&&"inprogress"!==e[0]&&k.dequeue(this,t)})},dequeue:function(e){return this.each(function(){k.dequeue(this,e)})},clearQueue:function(e){return this.queue(e||"fx",[])},promise:function(e,t){var n,r=1,i=k.Deferred(),o=this,a=this.length,s=function(){--r||i.resolveWith(o,[o])};"string"!=typeof e&&(t=e,e=void 0),e=e||"fx";while(a--)(n=Q.get(o[a],e+"queueHooks"))&&n.empty&&(r++,n.empty.add(s));return s(),i.promise(t)}});var te=/[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/.source,ne=new RegExp("^(?:([+-])=|)("+te+")([a-z%]*)$","i"),re=["Top","Right","Bottom","Left"],ie=E.documentElement,oe=function(e){return k.contains(e.ownerDocument,e)},ae={composed:!0};ie.getRootNode&&(oe=function(e){return k.contains(e.ownerDocument,e)||e.getRootNode(ae)===e.ownerDocument});var se=function(e,t){return"none"===(e=t||e).style.display||""===e.style.display&&oe(e)&&"none"===k.css(e,"display")},ue=function(e,t,n,r){var i,o,a={};for(o in t)a[o]=e.style[o],e.style[o]=t[o];for(o in i=n.apply(e,r||[]),t)e.style[o]=a[o];return i};function le(e,t,n,r){var i,o,a=20,s=r?function(){return r.cur()}:function(){return k.css(e,t,"")},u=s(),l=n&&n[3]||(k.cssNumber[t]?"":"px"),c=e.nodeType&&(k.cssNumber[t]||"px"!==l&&+u)&&ne.exec(k.css(e,t));if(c&&c[3]!==l){u/=2,l=l||c[3],c=+u||1;while(a--)k.style(e,t,c+l),(1-o)*(1-(o=s()/u||.5))<=0&&(a=0),c/=o;c*=2,k.style(e,t,c+l),n=n||[]}return n&&(c=+c||+u||0,i=n[1]?c+(n[1]+1)*n[2]:+n[2],r&&(r.unit=l,r.start=c,r.end=i)),i}var ce={};function fe(e,t){for(var n,r,i,o,a,s,u,l=[],c=0,f=e.length;c<f;c++)(r=e[c]).style&&(n=r.style.display,t?("none"===n&&(l[c]=Q.get(r,"display")||null,l[c]||(r.style.display="")),""===r.style.display&&se(r)&&(l[c]=(u=a=o=void 0,a=(i=r).ownerDocument,s=i.nodeName,(u=ce[s])||(o=a.body.appendChild(a.createElement(s)),u=k.css(o,"display"),o.parentNode.removeChild(o),"none"===u&&(u="block"),ce[s]=u)))):"none"!==n&&(l[c]="none",Q.set(r,"display",n)));for(c=0;c<f;c++)null!=l[c]&&(e[c].style.display=l[c]);return e}k.fn.extend({show:function(){return fe(this,!0)},hide:function(){return fe(this)},toggle:function(e){return"boolean"==typeof e?e?this.show():this.hide():this.each(function(){se(this)?k(this).show():k(this).hide()})}});var pe=/^(?:checkbox|radio)$/i,de=/<([a-z][^\/\0>\x20\t\r\n\f]*)/i,he=/^$|^module$|\/(?:java|ecma)script/i,ge={option:[1,"<select multiple='multiple'>","</select>"],thead:[1,"<table>","</table>"],col:[2,"<table><colgroup>","</colgroup></table>"],tr:[2,"<table><tbody>","</tbody></table>"],td:[3,"<table><tbody><tr>","</tr></tbody></table>"],_default:[0,"",""]};function ve(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&A(e,t)?k.merge([e],n):n}function ye(e,t){for(var n=0,r=e.length;n<r;n++)Q.set(e[n],"globalEval",!t||Q.get(t[n],"globalEval"))}ge.optgroup=ge.option,ge.tbody=ge.tfoot=ge.colgroup=ge.caption=ge.thead,ge.th=ge.td;var me,xe,be=/<|&#?\w+;/;function we(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),p=[],d=0,h=e.length;d<h;d++)if((o=e[d])||0===o)if("object"===w(o))k.merge(p,o.nodeType?[o]:o);else if(be.test(o)){a=a||f.appendChild(t.createElement("div")),s=(de.exec(o)||["",""])[1].toLowerCase(),u=ge[s]||ge._default,a.innerHTML=u[1]+k.htmlPrefilter(o)+u[2],c=u[0];while(c--)a=a.lastChild;k.merge(p,a.childNodes),(a=f.firstChild).textContent=""}else p.push(t.createTextNode(o));f.textContent="",d=0;while(o=p[d++])if(r&&-1<k.inArray(o,r))i&&i.push(o);else if(l=oe(o),a=ve(f.appendChild(o),"script"),l&&ye(a),n){c=0;while(o=a[c++])he.test(o.type||"")&&n.push(o)}return f}me=E.createDocumentFragment().appendChild(E.createElement("div")),(xe=E.createElement("input")).setAttribute("type","radio"),xe.setAttribute("checked","checked"),xe.setAttribute("name","t"),me.appendChild(xe),y.checkClone=me.cloneNode(!0).cloneNode(!0).lastChild.checked,me.innerHTML="<textarea>x</textarea>",y.noCloneChecked=!!me.cloneNode(!0).lastChild.defaultValue;var Te=/^key/,Ce=/^(?:mouse|pointer|contextmenu|drag|drop)|click/,Ee=/^([^.]*)(?:\.(.+)|)/;function ke(){return!0}function Se(){return!1}function Ne(e,t){return e===function(){try{return E.activeElement}catch(e){}}()==("focus"===t)}function Ae(e,t,n,r,i,o){var a,s;if("object"==typeof t){for(s in"string"!=typeof n&&(r=r||n,n=void 0),t)Ae(e,s,n,r,t[s],o);return e}if(null==r&&null==i?(i=n,r=n=void 0):null==i&&("string"==typeof n?(i=r,r=void 0):(i=r,r=n,n=void 0)),!1===i)i=Se;else if(!i)return e;return 1===o&&(a=i,(i=function(e){return k().off(e),a.apply(this,arguments)}).guid=a.guid||(a.guid=k.guid++)),e.each(function(){k.event.add(this,t,i,r,n)})}function De(e,i,o){o?(Q.set(e,i,!1),k.event.add(e,i,{namespace:!1,handler:function(e){var t,n,r=Q.get(this,i);if(1&e.isTrigger&&this[i]){if(r.length)(k.event.special[i]||{}).delegateType&&e.stopPropagation();else if(r=s.call(arguments),Q.set(this,i,r),t=o(this,i),this[i](),r!==(n=Q.get(this,i))||t?Q.set(this,i,!1):n={},r!==n)return e.stopImmediatePropagation(),e.preventDefault(),n.value}else r.length&&(Q.set(this,i,{value:k.event.trigger(k.extend(r[0],k.Event.prototype),r.slice(1),this)}),e.stopImmediatePropagation())}})):void 0===Q.get(e,i)&&k.event.add(e,i,ke)}k.event={global:{},add:function(t,e,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Q.get(t);if(v){n.handler&&(n=(o=n).handler,i=o.selector),i&&k.find.matchesSelector(ie,i),n.guid||(n.guid=k.guid++),(u=v.events)||(u=v.events={}),(a=v.handle)||(a=v.handle=function(e){return"undefined"!=typeof k&&k.event.triggered!==e.type?k.event.dispatch.apply(t,arguments):void 0}),l=(e=(e||"").match(R)||[""]).length;while(l--)d=g=(s=Ee.exec(e[l])||[])[1],h=(s[2]||"").split(".").sort(),d&&(f=k.event.special[d]||{},d=(i?f.delegateType:f.bindType)||d,f=k.event.special[d]||{},c=k.extend({type:d,origType:g,data:r,handler:n,guid:n.guid,selector:i,needsContext:i&&k.expr.match.needsContext.test(i),namespace:h.join(".")},o),(p=u[d])||((p=u[d]=[]).delegateCount=0,f.setup&&!1!==f.setup.call(t,r,h,a)||t.addEventListener&&t.addEventListener(d,a)),f.add&&(f.add.call(t,c),c.handler.guid||(c.handler.guid=n.guid)),i?p.splice(p.delegateCount++,0,c):p.push(c),k.event.global[d]=!0)}},remove:function(e,t,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Q.hasData(e)&&Q.get(e);if(v&&(u=v.events)){l=(t=(t||"").match(R)||[""]).length;while(l--)if(d=g=(s=Ee.exec(t[l])||[])[1],h=(s[2]||"").split(".").sort(),d){f=k.event.special[d]||{},p=u[d=(r?f.delegateType:f.bindType)||d]||[],s=s[2]&&new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"),a=o=p.length;while(o--)c=p[o],!i&&g!==c.origType||n&&n.guid!==c.guid||s&&!s.test(c.namespace)||r&&r!==c.selector&&("**"!==r||!c.selector)||(p.splice(o,1),c.selector&&p.delegateCount--,f.remove&&f.remove.call(e,c));a&&!p.length&&(f.teardown&&!1!==f.teardown.call(e,h,v.handle)||k.removeEvent(e,d,v.handle),delete u[d])}else for(d in u)k.event.remove(e,d+t[l],n,r,!0);k.isEmptyObject(u)&&Q.remove(e,"handle events")}},dispatch:function(e){var t,n,r,i,o,a,s=k.event.fix(e),u=new Array(arguments.length),l=(Q.get(this,"events")||{})[s.type]||[],c=k.event.special[s.type]||{};for(u[0]=s,t=1;t<arguments.length;t++)u[t]=arguments[t];if(s.delegateTarget=this,!c.preDispatch||!1!==c.preDispatch.call(this,s)){a=k.event.handlers.call(this,s,l),t=0;while((i=a[t++])&&!s.isPropagationStopped()){s.currentTarget=i.elem,n=0;while((o=i.handlers[n++])&&!s.isImmediatePropagationStopped())s.rnamespace&&!1!==o.namespace&&!s.rnamespace.test(o.namespace)||(s.handleObj=o,s.data=o.data,void 0!==(r=((k.event.special[o.origType]||{}).handle||o.handler).apply(i.elem,u))&&!1===(s.result=r)&&(s.preventDefault(),s.stopPropagation()))}return c.postDispatch&&c.postDispatch.call(this,s),s.result}},handlers:function(e,t){var n,r,i,o,a,s=[],u=t.delegateCount,l=e.target;if(u&&l.nodeType&&!("click"===e.type&&1<=e.button))for(;l!==this;l=l.parentNode||this)if(1===l.nodeType&&("click"!==e.type||!0!==l.disabled)){for(o=[],a={},n=0;n<u;n++)void 0===a[i=(r=t[n]).selector+" "]&&(a[i]=r.needsContext?-1<k(i,this).index(l):k.find(i,this,null,[l]).length),a[i]&&o.push(r);o.length&&s.push({elem:l,handlers:o})}return l=this,u<t.length&&s.push({elem:l,handlers:t.slice(u)}),s},addProp:function(t,e){Object.defineProperty(k.Event.prototype,t,{enumerable:!0,configurable:!0,get:m(e)?function(){if(this.originalEvent)return e(this.originalEvent)}:function(){if(this.originalEvent)return this.originalEvent[t]},set:function(e){Object.defineProperty(this,t,{enumerable:!0,configurable:!0,writable:!0,value:e})}})},fix:function(e){return e[k.expando]?e:new k.Event(e)},special:{load:{noBubble:!0},click:{setup:function(e){var t=this||e;return pe.test(t.type)&&t.click&&A(t,"input")&&De(t,"click",ke),!1},trigger:function(e){var t=this||e;return pe.test(t.type)&&t.click&&A(t,"input")&&De(t,"click"),!0},_default:function(e){var t=e.target;return pe.test(t.type)&&t.click&&A(t,"input")&&Q.get(t,"click")||A(t,"a")}},beforeunload:{postDispatch:function(e){void 0!==e.result&&e.originalEvent&&(e.originalEvent.returnValue=e.result)}}}},k.removeEvent=function(e,t,n){e.removeEventListener&&e.removeEventListener(t,n)},k.Event=function(e,t){if(!(this instanceof k.Event))return new k.Event(e,t);e&&e.type?(this.originalEvent=e,this.type=e.type,this.isDefaultPrevented=e.defaultPrevented||void 0===e.defaultPrevented&&!1===e.returnValue?ke:Se,this.target=e.target&&3===e.target.nodeType?e.target.parentNode:e.target,this.currentTarget=e.currentTarget,this.relatedTarget=e.relatedTarget):this.type=e,t&&k.extend(this,t),this.timeStamp=e&&e.timeStamp||Date.now(),this[k.expando]=!0},k.Event.prototype={constructor:k.Event,isDefaultPrevented:Se,isPropagationStopped:Se,isImmediatePropagationStopped:Se,isSimulated:!1,preventDefault:function(){var e=this.originalEvent;this.isDefaultPrevented=ke,e&&!this.isSimulated&&e.preventDefault()},stopPropagation:function(){var e=this.originalEvent;this.isPropagationStopped=ke,e&&!this.isSimulated&&e.stopPropagation()},stopImmediatePropagation:function(){var e=this.originalEvent;this.isImmediatePropagationStopped=ke,e&&!this.isSimulated&&e.stopImmediatePropagation(),this.stopPropagation()}},k.each({altKey:!0,bubbles:!0,cancelable:!0,changedTouches:!0,ctrlKey:!0,detail:!0,eventPhase:!0,metaKey:!0,pageX:!0,pageY:!0,shiftKey:!0,view:!0,"char":!0,code:!0,charCode:!0,key:!0,keyCode:!0,button:!0,buttons:!0,clientX:!0,clientY:!0,offsetX:!0,offsetY:!0,pointerId:!0,pointerType:!0,screenX:!0,screenY:!0,targetTouches:!0,toElement:!0,touches:!0,which:function(e){var t=e.button;return null==e.which&&Te.test(e.type)?null!=e.charCode?e.charCode:e.keyCode:!e.which&&void 0!==t&&Ce.test(e.type)?1&t?1:2&t?3:4&t?2:0:e.which}},k.event.addProp),k.each({focus:"focusin",blur:"focusout"},function(e,t){k.event.special[e]={setup:function(){return De(this,e,Ne),!1},trigger:function(){return De(this,e),!0},delegateType:t}}),k.each({mouseenter:"mouseover",mouseleave:"mouseout",pointerenter:"pointerover",pointerleave:"pointerout"},function(e,i){k.event.special[e]={delegateType:i,bindType:i,handle:function(e){var t,n=e.relatedTarget,r=e.handleObj;return n&&(n===this||k.contains(this,n))||(e.type=r.origType,t=r.handler.apply(this,arguments),e.type=i),t}}}),k.fn.extend({on:function(e,t,n,r){return Ae(this,e,t,n,r)},one:function(e,t,n,r){return Ae(this,e,t,n,r,1)},off:function(e,t,n){var r,i;if(e&&e.preventDefault&&e.handleObj)return r=e.handleObj,k(e.delegateTarget).off(r.namespace?r.origType+"."+r.namespace:r.origType,r.selector,r.handler),this;if("object"==typeof e){for(i in e)this.off(i,t,e[i]);return this}return!1!==t&&"function"!=typeof t||(n=t,t=void 0),!1===n&&(n=Se),this.each(function(){k.event.remove(this,e,n,t)})}});var je=/<(?!area|br|col|embed|hr|img|input|link|meta|param)(([a-z][^\/\0>\x20\t\r\n\f]*)[^>]*)\/>/gi,qe=/<script|<style|<link/i,Le=/checked\s*(?:[^=]|=\s*.checked.)/i,He=/^\s*<!(?:\[CDATA\[|--)|(?:\]\]|--)>\s*$/g;function Oe(e,t){return A(e,"table")&&A(11!==t.nodeType?t:t.firstChild,"tr")&&k(e).children("tbody")[0]||e}function Pe(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function Re(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Me(e,t){var n,r,i,o,a,s,u,l;if(1===t.nodeType){if(Q.hasData(e)&&(o=Q.access(e),a=Q.set(t,o),l=o.events))for(i in delete a.handle,a.events={},l)for(n=0,r=l[i].length;n<r;n++)k.event.add(t,i,l[i][n]);J.hasData(e)&&(s=J.access(e),u=k.extend({},s),J.set(t,u))}}function Ie(n,r,i,o){r=g.apply([],r);var e,t,a,s,u,l,c=0,f=n.length,p=f-1,d=r[0],h=m(d);if(h||1<f&&"string"==typeof d&&!y.checkClone&&Le.test(d))return n.each(function(e){var t=n.eq(e);h&&(r[0]=d.call(this,e,t.html())),Ie(t,r,i,o)});if(f&&(t=(e=we(r,n[0].ownerDocument,!1,n,o)).firstChild,1===e.childNodes.length&&(e=t),t||o)){for(s=(a=k.map(ve(e,"script"),Pe)).length;c<f;c++)u=e,c!==p&&(u=k.clone(u,!0,!0),s&&k.merge(a,ve(u,"script"))),i.call(n[c],u,c);if(s)for(l=a[a.length-1].ownerDocument,k.map(a,Re),c=0;c<s;c++)u=a[c],he.test(u.type||"")&&!Q.access(u,"globalEval")&&k.contains(l,u)&&(u.src&&"module"!==(u.type||"").toLowerCase()?k._evalUrl&&!u.noModule&&k._evalUrl(u.src,{nonce:u.nonce||u.getAttribute("nonce")}):b(u.textContent.replace(He,""),u,l))}return n}function We(e,t,n){for(var r,i=t?k.filter(t,e):e,o=0;null!=(r=i[o]);o++)n||1!==r.nodeType||k.cleanData(ve(r)),r.parentNode&&(n&&oe(r)&&ye(ve(r,"script")),r.parentNode.removeChild(r));return e}k.extend({htmlPrefilter:function(e){return e.replace(je,"<$1></$2>")},clone:function(e,t,n){var r,i,o,a,s,u,l,c=e.cloneNode(!0),f=oe(e);if(!(y.noCloneChecked||1!==e.nodeType&&11!==e.nodeType||k.isXMLDoc(e)))for(a=ve(c),r=0,i=(o=ve(e)).length;r<i;r++)s=o[r],u=a[r],void 0,"input"===(l=u.nodeName.toLowerCase())&&pe.test(s.type)?u.checked=s.checked:"input"!==l&&"textarea"!==l||(u.defaultValue=s.defaultValue);if(t)if(n)for(o=o||ve(e),a=a||ve(c),r=0,i=o.length;r<i;r++)Me(o[r],a[r]);else Me(e,c);return 0<(a=ve(c,"script")).length&&ye(a,!f&&ve(e,"script")),c},cleanData:function(e){for(var t,n,r,i=k.event.special,o=0;void 0!==(n=e[o]);o++)if(G(n)){if(t=n[Q.expando]){if(t.events)for(r in t.events)i[r]?k.event.remove(n,r):k.removeEvent(n,r,t.handle);n[Q.expando]=void 0}n[J.expando]&&(n[J.expando]=void 0)}}}),k.fn.extend({detach:function(e){return We(this,e,!0)},remove:function(e){return We(this,e)},text:function(e){return _(this,function(e){return void 0===e?k.text(this):this.empty().each(function(){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||(this.textContent=e)})},null,e,arguments.length)},append:function(){return Ie(this,arguments,function(e){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||Oe(this,e).appendChild(e)})},prepend:function(){return Ie(this,arguments,function(e){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var t=Oe(this,e);t.insertBefore(e,t.firstChild)}})},before:function(){return Ie(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this)})},after:function(){return Ie(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this.nextSibling)})},empty:function(){for(var e,t=0;null!=(e=this[t]);t++)1===e.nodeType&&(k.cleanData(ve(e,!1)),e.textContent="");return this},clone:function(e,t){return e=null!=e&&e,t=null==t?e:t,this.map(function(){return k.clone(this,e,t)})},html:function(e){return _(this,function(e){var t=this[0]||{},n=0,r=this.length;if(void 0===e&&1===t.nodeType)return t.innerHTML;if("string"==typeof e&&!qe.test(e)&&!ge[(de.exec(e)||["",""])[1].toLowerCase()]){e=k.htmlPrefilter(e);try{for(;n<r;n++)1===(t=this[n]||{}).nodeType&&(k.cleanData(ve(t,!1)),t.innerHTML=e);t=0}catch(e){}}t&&this.empty().append(e)},null,e,arguments.length)},replaceWith:function(){var n=[];return Ie(this,arguments,function(e){var t=this.parentNode;k.inArray(this,n)<0&&(k.cleanData(ve(this)),t&&t.replaceChild(e,this))},n)}}),k.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(e,a){k.fn[e]=function(e){for(var t,n=[],r=k(e),i=r.length-1,o=0;o<=i;o++)t=o===i?this:this.clone(!0),k(r[o])[a](t),u.apply(n,t.get());return this.pushStack(n)}});var $e=new RegExp("^("+te+")(?!px)[a-z%]+$","i"),Fe=function(e){var t=e.ownerDocument.defaultView;return t&&t.opener||(t=C),t.getComputedStyle(e)},Be=new RegExp(re.join("|"),"i");function _e(e,t,n){var r,i,o,a,s=e.style;return(n=n||Fe(e))&&(""!==(a=n.getPropertyValue(t)||n[t])||oe(e)||(a=k.style(e,t)),!y.pixelBoxStyles()&&$e.test(a)&&Be.test(t)&&(r=s.width,i=s.minWidth,o=s.maxWidth,s.minWidth=s.maxWidth=s.width=a,a=n.width,s.width=r,s.minWidth=i,s.maxWidth=o)),void 0!==a?a+"":a}function ze(e,t){return{get:function(){if(!e())return(this.get=t).apply(this,arguments);delete this.get}}}!function(){function e(){if(u){s.style.cssText="position:absolute;left:-11111px;width:60px;margin-top:1px;padding:0;border:0",u.style.cssText="position:relative;display:block;box-sizing:border-box;overflow:scroll;margin:auto;border:1px;padding:1px;width:60%;top:1%",ie.appendChild(s).appendChild(u);var e=C.getComputedStyle(u);n="1%"!==e.top,a=12===t(e.marginLeft),u.style.right="60%",o=36===t(e.right),r=36===t(e.width),u.style.position="absolute",i=12===t(u.offsetWidth/3),ie.removeChild(s),u=null}}function t(e){return Math.round(parseFloat(e))}var n,r,i,o,a,s=E.createElement("div"),u=E.createElement("div");u.style&&(u.style.backgroundClip="content-box",u.cloneNode(!0).style.backgroundClip="",y.clearCloneStyle="content-box"===u.style.backgroundClip,k.extend(y,{boxSizingReliable:function(){return e(),r},pixelBoxStyles:function(){return e(),o},pixelPosition:function(){return e(),n},reliableMarginLeft:function(){return e(),a},scrollboxSize:function(){return e(),i}}))}();var Ue=["Webkit","Moz","ms"],Xe=E.createElement("div").style,Ve={};function Ge(e){var t=k.cssProps[e]||Ve[e];return t||(e in Xe?e:Ve[e]=function(e){var t=e[0].toUpperCase()+e.slice(1),n=Ue.length;while(n--)if((e=Ue[n]+t)in Xe)return e}(e)||e)}var Ye=/^(none|table(?!-c[ea]).+)/,Qe=/^--/,Je={position:"absolute",visibility:"hidden",display:"block"},Ke={letterSpacing:"0",fontWeight:"400"};function Ze(e,t,n){var r=ne.exec(t);return r?Math.max(0,r[2]-(n||0))+(r[3]||"px"):t}function et(e,t,n,r,i,o){var a="width"===t?1:0,s=0,u=0;if(n===(r?"border":"content"))return 0;for(;a<4;a+=2)"margin"===n&&(u+=k.css(e,n+re[a],!0,i)),r?("content"===n&&(u-=k.css(e,"padding"+re[a],!0,i)),"margin"!==n&&(u-=k.css(e,"border"+re[a]+"Width",!0,i))):(u+=k.css(e,"padding"+re[a],!0,i),"padding"!==n?u+=k.css(e,"border"+re[a]+"Width",!0,i):s+=k.css(e,"border"+re[a]+"Width",!0,i));return!r&&0<=o&&(u+=Math.max(0,Math.ceil(e["offset"+t[0].toUpperCase()+t.slice(1)]-o-u-s-.5))||0),u}function tt(e,t,n){var r=Fe(e),i=(!y.boxSizingReliable()||n)&&"border-box"===k.css(e,"boxSizing",!1,r),o=i,a=_e(e,t,r),s="offset"+t[0].toUpperCase()+t.slice(1);if($e.test(a)){if(!n)return a;a="auto"}return(!y.boxSizingReliable()&&i||"auto"===a||!parseFloat(a)&&"inline"===k.css(e,"display",!1,r))&&e.getClientRects().length&&(i="border-box"===k.css(e,"boxSizing",!1,r),(o=s in e)&&(a=e[s])),(a=parseFloat(a)||0)+et(e,t,n||(i?"border":"content"),o,r,a)+"px"}function nt(e,t,n,r,i){return new nt.prototype.init(e,t,n,r,i)}k.extend({cssHooks:{opacity:{get:function(e,t){if(t){var n=_e(e,"opacity");return""===n?"1":n}}}},cssNumber:{animationIterationCount:!0,columnCount:!0,fillOpacity:!0,flexGrow:!0,flexShrink:!0,fontWeight:!0,gridArea:!0,gridColumn:!0,gridColumnEnd:!0,gridColumnStart:!0,gridRow:!0,gridRowEnd:!0,gridRowStart:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{},style:function(e,t,n,r){if(e&&3!==e.nodeType&&8!==e.nodeType&&e.style){var i,o,a,s=V(t),u=Qe.test(t),l=e.style;if(u||(t=Ge(s)),a=k.cssHooks[t]||k.cssHooks[s],void 0===n)return a&&"get"in a&&void 0!==(i=a.get(e,!1,r))?i:l[t];"string"===(o=typeof n)&&(i=ne.exec(n))&&i[1]&&(n=le(e,t,i),o="number"),null!=n&&n==n&&("number"!==o||u||(n+=i&&i[3]||(k.cssNumber[s]?"":"px")),y.clearCloneStyle||""!==n||0!==t.indexOf("background")||(l[t]="inherit"),a&&"set"in a&&void 0===(n=a.set(e,n,r))||(u?l.setProperty(t,n):l[t]=n))}},css:function(e,t,n,r){var i,o,a,s=V(t);return Qe.test(t)||(t=Ge(s)),(a=k.cssHooks[t]||k.cssHooks[s])&&"get"in a&&(i=a.get(e,!0,n)),void 0===i&&(i=_e(e,t,r)),"normal"===i&&t in Ke&&(i=Ke[t]),""===n||n?(o=parseFloat(i),!0===n||isFinite(o)?o||0:i):i}}),k.each(["height","width"],function(e,u){k.cssHooks[u]={get:function(e,t,n){if(t)return!Ye.test(k.css(e,"display"))||e.getClientRects().length&&e.getBoundingClientRect().width?tt(e,u,n):ue(e,Je,function(){return tt(e,u,n)})},set:function(e,t,n){var r,i=Fe(e),o=!y.scrollboxSize()&&"absolute"===i.position,a=(o||n)&&"border-box"===k.css(e,"boxSizing",!1,i),s=n?et(e,u,n,a,i):0;return a&&o&&(s-=Math.ceil(e["offset"+u[0].toUpperCase()+u.slice(1)]-parseFloat(i[u])-et(e,u,"border",!1,i)-.5)),s&&(r=ne.exec(t))&&"px"!==(r[3]||"px")&&(e.style[u]=t,t=k.css(e,u)),Ze(0,t,s)}}}),k.cssHooks.marginLeft=ze(y.reliableMarginLeft,function(e,t){if(t)return(parseFloat(_e(e,"marginLeft"))||e.getBoundingClientRect().left-ue(e,{marginLeft:0},function(){return e.getBoundingClientRect().left}))+"px"}),k.each({margin:"",padding:"",border:"Width"},function(i,o){k.cssHooks[i+o]={expand:function(e){for(var t=0,n={},r="string"==typeof e?e.split(" "):[e];t<4;t++)n[i+re[t]+o]=r[t]||r[t-2]||r[0];return n}},"margin"!==i&&(k.cssHooks[i+o].set=Ze)}),k.fn.extend({css:function(e,t){return _(this,function(e,t,n){var r,i,o={},a=0;if(Array.isArray(t)){for(r=Fe(e),i=t.length;a<i;a++)o[t[a]]=k.css(e,t[a],!1,r);return o}return void 0!==n?k.style(e,t,n):k.css(e,t)},e,t,1<arguments.length)}}),((k.Tween=nt).prototype={constructor:nt,init:function(e,t,n,r,i,o){this.elem=e,this.prop=n,this.easing=i||k.easing._default,this.options=t,this.start=this.now=this.cur(),this.end=r,this.unit=o||(k.cssNumber[n]?"":"px")},cur:function(){var e=nt.propHooks[this.prop];return e&&e.get?e.get(this):nt.propHooks._default.get(this)},run:function(e){var t,n=nt.propHooks[this.prop];return this.options.duration?this.pos=t=k.easing[this.easing](e,this.options.duration*e,0,1,this.options.duration):this.pos=t=e,this.now=(this.end-this.start)*t+this.start,this.options.step&&this.options.step.call(this.elem,this.now,this),n&&n.set?n.set(this):nt.propHooks._default.set(this),this}}).init.prototype=nt.prototype,(nt.propHooks={_default:{get:function(e){var t;return 1!==e.elem.nodeType||null!=e.elem[e.prop]&&null==e.elem.style[e.prop]?e.elem[e.prop]:(t=k.css(e.elem,e.prop,""))&&"auto"!==t?t:0},set:function(e){k.fx.step[e.prop]?k.fx.step[e.prop](e):1!==e.elem.nodeType||!k.cssHooks[e.prop]&&null==e.elem.style[Ge(e.prop)]?e.elem[e.prop]=e.now:k.style(e.elem,e.prop,e.now+e.unit)}}}).scrollTop=nt.propHooks.scrollLeft={set:function(e){e.elem.nodeType&&e.elem.parentNode&&(e.elem[e.prop]=e.now)}},k.easing={linear:function(e){return e},swing:function(e){return.5-Math.cos(e*Math.PI)/2},_default:"swing"},k.fx=nt.prototype.init,k.fx.step={};var rt,it,ot,at,st=/^(?:toggle|show|hide)$/,ut=/queueHooks$/;function lt(){it&&(!1===E.hidden&&C.requestAnimationFrame?C.requestAnimationFrame(lt):C.setTimeout(lt,k.fx.interval),k.fx.tick())}function ct(){return C.setTimeout(function(){rt=void 0}),rt=Date.now()}function ft(e,t){var n,r=0,i={height:e};for(t=t?1:0;r<4;r+=2-t)i["margin"+(n=re[r])]=i["padding"+n]=e;return t&&(i.opacity=i.width=e),i}function pt(e,t,n){for(var r,i=(dt.tweeners[t]||[]).concat(dt.tweeners["*"]),o=0,a=i.length;o<a;o++)if(r=i[o].call(n,t,e))return r}function dt(o,e,t){var n,a,r=0,i=dt.prefilters.length,s=k.Deferred().always(function(){delete u.elem}),u=function(){if(a)return!1;for(var e=rt||ct(),t=Math.max(0,l.startTime+l.duration-e),n=1-(t/l.duration||0),r=0,i=l.tweens.length;r<i;r++)l.tweens[r].run(n);return s.notifyWith(o,[l,n,t]),n<1&&i?t:(i||s.notifyWith(o,[l,1,0]),s.resolveWith(o,[l]),!1)},l=s.promise({elem:o,props:k.extend({},e),opts:k.extend(!0,{specialEasing:{},easing:k.easing._default},t),originalProperties:e,originalOptions:t,startTime:rt||ct(),duration:t.duration,tweens:[],createTween:function(e,t){var n=k.Tween(o,l.opts,e,t,l.opts.specialEasing[e]||l.opts.easing);return l.tweens.push(n),n},stop:function(e){var t=0,n=e?l.tweens.length:0;if(a)return this;for(a=!0;t<n;t++)l.tweens[t].run(1);return e?(s.notifyWith(o,[l,1,0]),s.resolveWith(o,[l,e])):s.rejectWith(o,[l,e]),this}}),c=l.props;for(!function(e,t){var n,r,i,o,a;for(n in e)if(i=t[r=V(n)],o=e[n],Array.isArray(o)&&(i=o[1],o=e[n]=o[0]),n!==r&&(e[r]=o,delete e[n]),(a=k.cssHooks[r])&&"expand"in a)for(n in o=a.expand(o),delete e[r],o)n in e||(e[n]=o[n],t[n]=i);else t[r]=i}(c,l.opts.specialEasing);r<i;r++)if(n=dt.prefilters[r].call(l,o,c,l.opts))return m(n.stop)&&(k._queueHooks(l.elem,l.opts.queue).stop=n.stop.bind(n)),n;return k.map(c,pt,l),m(l.opts.start)&&l.opts.start.call(o,l),l.progress(l.opts.progress).done(l.opts.done,l.opts.complete).fail(l.opts.fail).always(l.opts.always),k.fx.timer(k.extend(u,{elem:o,anim:l,queue:l.opts.queue})),l}k.Animation=k.extend(dt,{tweeners:{"*":[function(e,t){var n=this.createTween(e,t);return le(n.elem,e,ne.exec(t),n),n}]},tweener:function(e,t){m(e)?(t=e,e=["*"]):e=e.match(R);for(var n,r=0,i=e.length;r<i;r++)n=e[r],dt.tweeners[n]=dt.tweeners[n]||[],dt.tweeners[n].unshift(t)},prefilters:[function(e,t,n){var r,i,o,a,s,u,l,c,f="width"in t||"height"in t,p=this,d={},h=e.style,g=e.nodeType&&se(e),v=Q.get(e,"fxshow");for(r in n.queue||(null==(a=k._queueHooks(e,"fx")).unqueued&&(a.unqueued=0,s=a.empty.fire,a.empty.fire=function(){a.unqueued||s()}),a.unqueued++,p.always(function(){p.always(function(){a.unqueued--,k.queue(e,"fx").length||a.empty.fire()})})),t)if(i=t[r],st.test(i)){if(delete t[r],o=o||"toggle"===i,i===(g?"hide":"show")){if("show"!==i||!v||void 0===v[r])continue;g=!0}d[r]=v&&v[r]||k.style(e,r)}if((u=!k.isEmptyObject(t))||!k.isEmptyObject(d))for(r in f&&1===e.nodeType&&(n.overflow=[h.overflow,h.overflowX,h.overflowY],null==(l=v&&v.display)&&(l=Q.get(e,"display")),"none"===(c=k.css(e,"display"))&&(l?c=l:(fe([e],!0),l=e.style.display||l,c=k.css(e,"display"),fe([e]))),("inline"===c||"inline-block"===c&&null!=l)&&"none"===k.css(e,"float")&&(u||(p.done(function(){h.display=l}),null==l&&(c=h.display,l="none"===c?"":c)),h.display="inline-block")),n.overflow&&(h.overflow="hidden",p.always(function(){h.overflow=n.overflow[0],h.overflowX=n.overflow[1],h.overflowY=n.overflow[2]})),u=!1,d)u||(v?"hidden"in v&&(g=v.hidden):v=Q.access(e,"fxshow",{display:l}),o&&(v.hidden=!g),g&&fe([e],!0),p.done(function(){for(r in g||fe([e]),Q.remove(e,"fxshow"),d)k.style(e,r,d[r])})),u=pt(g?v[r]:0,r,p),r in v||(v[r]=u.start,g&&(u.end=u.start,u.start=0))}],prefilter:function(e,t){t?dt.prefilters.unshift(e):dt.prefilters.push(e)}}),k.speed=function(e,t,n){var r=e&&"object"==typeof e?k.extend({},e):{complete:n||!n&&t||m(e)&&e,duration:e,easing:n&&t||t&&!m(t)&&t};return k.fx.off?r.duration=0:"number"!=typeof r.duration&&(r.duration in k.fx.speeds?r.duration=k.fx.speeds[r.duration]:r.duration=k.fx.speeds._default),null!=r.queue&&!0!==r.queue||(r.queue="fx"),r.old=r.complete,r.complete=function(){m(r.old)&&r.old.call(this),r.queue&&k.dequeue(this,r.queue)},r},k.fn.extend({fadeTo:function(e,t,n,r){return this.filter(se).css("opacity",0).show().end().animate({opacity:t},e,n,r)},animate:function(t,e,n,r){var i=k.isEmptyObject(t),o=k.speed(e,n,r),a=function(){var e=dt(this,k.extend({},t),o);(i||Q.get(this,"finish"))&&e.stop(!0)};return a.finish=a,i||!1===o.queue?this.each(a):this.queue(o.queue,a)},stop:function(i,e,o){var a=function(e){var t=e.stop;delete e.stop,t(o)};return"string"!=typeof i&&(o=e,e=i,i=void 0),e&&!1!==i&&this.queue(i||"fx",[]),this.each(function(){var e=!0,t=null!=i&&i+"queueHooks",n=k.timers,r=Q.get(this);if(t)r[t]&&r[t].stop&&a(r[t]);else for(t in r)r[t]&&r[t].stop&&ut.test(t)&&a(r[t]);for(t=n.length;t--;)n[t].elem!==this||null!=i&&n[t].queue!==i||(n[t].anim.stop(o),e=!1,n.splice(t,1));!e&&o||k.dequeue(this,i)})},finish:function(a){return!1!==a&&(a=a||"fx"),this.each(function(){var e,t=Q.get(this),n=t[a+"queue"],r=t[a+"queueHooks"],i=k.timers,o=n?n.length:0;for(t.finish=!0,k.queue(this,a,[]),r&&r.stop&&r.stop.call(this,!0),e=i.length;e--;)i[e].elem===this&&i[e].queue===a&&(i[e].anim.stop(!0),i.splice(e,1));for(e=0;e<o;e++)n[e]&&n[e].finish&&n[e].finish.call(this);delete t.finish})}}),k.each(["toggle","show","hide"],function(e,r){var i=k.fn[r];k.fn[r]=function(e,t,n){return null==e||"boolean"==typeof e?i.apply(this,arguments):this.animate(ft(r,!0),e,t,n)}}),k.each({slideDown:ft("show"),slideUp:ft("hide"),slideToggle:ft("toggle"),fadeIn:{opacity:"show"},fadeOut:{opacity:"hide"},fadeToggle:{opacity:"toggle"}},function(e,r){k.fn[e]=function(e,t,n){return this.animate(r,e,t,n)}}),k.timers=[],k.fx.tick=function(){var e,t=0,n=k.timers;for(rt=Date.now();t<n.length;t++)(e=n[t])()||n[t]!==e||n.splice(t--,1);n.length||k.fx.stop(),rt=void 0},k.fx.timer=function(e){k.timers.push(e),k.fx.start()},k.fx.interval=13,k.fx.start=function(){it||(it=!0,lt())},k.fx.stop=function(){it=null},k.fx.speeds={slow:600,fast:200,_default:400},k.fn.delay=function(r,e){return r=k.fx&&k.fx.speeds[r]||r,e=e||"fx",this.queue(e,function(e,t){var n=C.setTimeout(e,r);t.stop=function(){C.clearTimeout(n)}})},ot=E.createElement("input"),at=E.createElement("select").appendChild(E.createElement("option")),ot.type="checkbox",y.checkOn=""!==ot.value,y.optSelected=at.selected,(ot=E.createElement("input")).value="t",ot.type="radio",y.radioValue="t"===ot.value;var ht,gt=k.expr.attrHandle;k.fn.extend({attr:function(e,t){return _(this,k.attr,e,t,1<arguments.length)},removeAttr:function(e){return this.each(function(){k.removeAttr(this,e)})}}),k.extend({attr:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return"undefined"==typeof e.getAttribute?k.prop(e,t,n):(1===o&&k.isXMLDoc(e)||(i=k.attrHooks[t.toLowerCase()]||(k.expr.match.bool.test(t)?ht:void 0)),void 0!==n?null===n?void k.removeAttr(e,t):i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:(e.setAttribute(t,n+""),n):i&&"get"in i&&null!==(r=i.get(e,t))?r:null==(r=k.find.attr(e,t))?void 0:r)},attrHooks:{type:{set:function(e,t){if(!y.radioValue&&"radio"===t&&A(e,"input")){var n=e.value;return e.setAttribute("type",t),n&&(e.value=n),t}}}},removeAttr:function(e,t){var n,r=0,i=t&&t.match(R);if(i&&1===e.nodeType)while(n=i[r++])e.removeAttribute(n)}}),ht={set:function(e,t,n){return!1===t?k.removeAttr(e,n):e.setAttribute(n,n),n}},k.each(k.expr.match.bool.source.match(/\w+/g),function(e,t){var a=gt[t]||k.find.attr;gt[t]=function(e,t,n){var r,i,o=t.toLowerCase();return n||(i=gt[o],gt[o]=r,r=null!=a(e,t,n)?o:null,gt[o]=i),r}});var vt=/^(?:input|select|textarea|button)$/i,yt=/^(?:a|area)$/i;function mt(e){return(e.match(R)||[]).join(" ")}function xt(e){return e.getAttribute&&e.getAttribute("class")||""}function bt(e){return Array.isArray(e)?e:"string"==typeof e&&e.match(R)||[]}k.fn.extend({prop:function(e,t){return _(this,k.prop,e,t,1<arguments.length)},removeProp:function(e){return this.each(function(){delete this[k.propFix[e]||e]})}}),k.extend({prop:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return 1===o&&k.isXMLDoc(e)||(t=k.propFix[t]||t,i=k.propHooks[t]),void 0!==n?i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:e[t]=n:i&&"get"in i&&null!==(r=i.get(e,t))?r:e[t]},propHooks:{tabIndex:{get:function(e){var t=k.find.attr(e,"tabindex");return t?parseInt(t,10):vt.test(e.nodeName)||yt.test(e.nodeName)&&e.href?0:-1}}},propFix:{"for":"htmlFor","class":"className"}}),y.optSelected||(k.propHooks.selected={get:function(e){var t=e.parentNode;return t&&t.parentNode&&t.parentNode.selectedIndex,null},set:function(e){var t=e.parentNode;t&&(t.selectedIndex,t.parentNode&&t.parentNode.selectedIndex)}}),k.each(["tabIndex","readOnly","maxLength","cellSpacing","cellPadding","rowSpan","colSpan","useMap","frameBorder","contentEditable"],function(){k.propFix[this.toLowerCase()]=this}),k.fn.extend({addClass:function(t){var e,n,r,i,o,a,s,u=0;if(m(t))return this.each(function(e){k(this).addClass(t.call(this,e,xt(this)))});if((e=bt(t)).length)while(n=this[u++])if(i=xt(n),r=1===n.nodeType&&" "+mt(i)+" "){a=0;while(o=e[a++])r.indexOf(" "+o+" ")<0&&(r+=o+" ");i!==(s=mt(r))&&n.setAttribute("class",s)}return this},removeClass:function(t){var e,n,r,i,o,a,s,u=0;if(m(t))return this.each(function(e){k(this).removeClass(t.call(this,e,xt(this)))});if(!arguments.length)return this.attr("class","");if((e=bt(t)).length)while(n=this[u++])if(i=xt(n),r=1===n.nodeType&&" "+mt(i)+" "){a=0;while(o=e[a++])while(-1<r.indexOf(" "+o+" "))r=r.replace(" "+o+" "," ");i!==(s=mt(r))&&n.setAttribute("class",s)}return this},toggleClass:function(i,t){var o=typeof i,a="string"===o||Array.isArray(i);return"boolean"==typeof t&&a?t?this.addClass(i):this.removeClass(i):m(i)?this.each(function(e){k(this).toggleClass(i.call(this,e,xt(this),t),t)}):this.each(function(){var e,t,n,r;if(a){t=0,n=k(this),r=bt(i);while(e=r[t++])n.hasClass(e)?n.removeClass(e):n.addClass(e)}else void 0!==i&&"boolean"!==o||((e=xt(this))&&Q.set(this,"__className__",e),this.setAttribute&&this.setAttribute("class",e||!1===i?"":Q.get(this,"__className__")||""))})},hasClass:function(e){var t,n,r=0;t=" "+e+" ";while(n=this[r++])if(1===n.nodeType&&-1<(" "+mt(xt(n))+" ").indexOf(t))return!0;return!1}});var wt=/\r/g;k.fn.extend({val:function(n){var r,e,i,t=this[0];return arguments.length?(i=m(n),this.each(function(e){var t;1===this.nodeType&&(null==(t=i?n.call(this,e,k(this).val()):n)?t="":"number"==typeof t?t+="":Array.isArray(t)&&(t=k.map(t,function(e){return null==e?"":e+""})),(r=k.valHooks[this.type]||k.valHooks[this.nodeName.toLowerCase()])&&"set"in r&&void 0!==r.set(this,t,"value")||(this.value=t))})):t?(r=k.valHooks[t.type]||k.valHooks[t.nodeName.toLowerCase()])&&"get"in r&&void 0!==(e=r.get(t,"value"))?e:"string"==typeof(e=t.value)?e.replace(wt,""):null==e?"":e:void 0}}),k.extend({valHooks:{option:{get:function(e){var t=k.find.attr(e,"value");return null!=t?t:mt(k.text(e))}},select:{get:function(e){var t,n,r,i=e.options,o=e.selectedIndex,a="select-one"===e.type,s=a?null:[],u=a?o+1:i.length;for(r=o<0?u:a?o:0;r<u;r++)if(((n=i[r]).selected||r===o)&&!n.disabled&&(!n.parentNode.disabled||!A(n.parentNode,"optgroup"))){if(t=k(n).val(),a)return t;s.push(t)}return s},set:function(e,t){var n,r,i=e.options,o=k.makeArray(t),a=i.length;while(a--)((r=i[a]).selected=-1<k.inArray(k.valHooks.option.get(r),o))&&(n=!0);return n||(e.selectedIndex=-1),o}}}}),k.each(["radio","checkbox"],function(){k.valHooks[this]={set:function(e,t){if(Array.isArray(t))return e.checked=-1<k.inArray(k(e).val(),t)}},y.checkOn||(k.valHooks[this].get=function(e){return null===e.getAttribute("value")?"on":e.value})}),y.focusin="onfocusin"in C;var Tt=/^(?:focusinfocus|focusoutblur)$/,Ct=function(e){e.stopPropagation()};k.extend(k.event,{trigger:function(e,t,n,r){var i,o,a,s,u,l,c,f,p=[n||E],d=v.call(e,"type")?e.type:e,h=v.call(e,"namespace")?e.namespace.split("."):[];if(o=f=a=n=n||E,3!==n.nodeType&&8!==n.nodeType&&!Tt.test(d+k.event.triggered)&&(-1<d.indexOf(".")&&(d=(h=d.split(".")).shift(),h.sort()),u=d.indexOf(":")<0&&"on"+d,(e=e[k.expando]?e:new k.Event(d,"object"==typeof e&&e)).isTrigger=r?2:3,e.namespace=h.join("."),e.rnamespace=e.namespace?new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,e.result=void 0,e.target||(e.target=n),t=null==t?[e]:k.makeArray(t,[e]),c=k.event.special[d]||{},r||!c.trigger||!1!==c.trigger.apply(n,t))){if(!r&&!c.noBubble&&!x(n)){for(s=c.delegateType||d,Tt.test(s+d)||(o=o.parentNode);o;o=o.parentNode)p.push(o),a=o;a===(n.ownerDocument||E)&&p.push(a.defaultView||a.parentWindow||C)}i=0;while((o=p[i++])&&!e.isPropagationStopped())f=o,e.type=1<i?s:c.bindType||d,(l=(Q.get(o,"events")||{})[e.type]&&Q.get(o,"handle"))&&l.apply(o,t),(l=u&&o[u])&&l.apply&&G(o)&&(e.result=l.apply(o,t),!1===e.result&&e.preventDefault());return e.type=d,r||e.isDefaultPrevented()||c._default&&!1!==c._default.apply(p.pop(),t)||!G(n)||u&&m(n[d])&&!x(n)&&((a=n[u])&&(n[u]=null),k.event.triggered=d,e.isPropagationStopped()&&f.addEventListener(d,Ct),n[d](),e.isPropagationStopped()&&f.removeEventListener(d,Ct),k.event.triggered=void 0,a&&(n[u]=a)),e.result}},simulate:function(e,t,n){var r=k.extend(new k.Event,n,{type:e,isSimulated:!0});k.event.trigger(r,null,t)}}),k.fn.extend({trigger:function(e,t){return this.each(function(){k.event.trigger(e,t,this)})},triggerHandler:function(e,t){var n=this[0];if(n)return k.event.trigger(e,t,n,!0)}}),y.focusin||k.each({focus:"focusin",blur:"focusout"},function(n,r){var i=function(e){k.event.simulate(r,e.target,k.event.fix(e))};k.event.special[r]={setup:function(){var e=this.ownerDocument||this,t=Q.access(e,r);t||e.addEventListener(n,i,!0),Q.access(e,r,(t||0)+1)},teardown:function(){var e=this.ownerDocument||this,t=Q.access(e,r)-1;t?Q.access(e,r,t):(e.removeEventListener(n,i,!0),Q.remove(e,r))}}});var Et=C.location,kt=Date.now(),St=/\?/;k.parseXML=function(e){var t;if(!e||"string"!=typeof e)return null;try{t=(new C.DOMParser).parseFromString(e,"text/xml")}catch(e){t=void 0}return t&&!t.getElementsByTagName("parsererror").length||k.error("Invalid XML: "+e),t};var Nt=/\[\]$/,At=/\r?\n/g,Dt=/^(?:submit|button|image|reset|file)$/i,jt=/^(?:input|select|textarea|keygen)/i;function qt(n,e,r,i){var t;if(Array.isArray(e))k.each(e,function(e,t){r||Nt.test(n)?i(n,t):qt(n+"["+("object"==typeof t&&null!=t?e:"")+"]",t,r,i)});else if(r||"object"!==w(e))i(n,e);else for(t in e)qt(n+"["+t+"]",e[t],r,i)}k.param=function(e,t){var n,r=[],i=function(e,t){var n=m(t)?t():t;r[r.length]=encodeURIComponent(e)+"="+encodeURIComponent(null==n?"":n)};if(null==e)return"";if(Array.isArray(e)||e.jquery&&!k.isPlainObject(e))k.each(e,function(){i(this.name,this.value)});else for(n in e)qt(n,e[n],t,i);return r.join("&")},k.fn.extend({serialize:function(){return k.param(this.serializeArray())},serializeArray:function(){return this.map(function(){var e=k.prop(this,"elements");return e?k.makeArray(e):this}).filter(function(){var e=this.type;return this.name&&!k(this).is(":disabled")&&jt.test(this.nodeName)&&!Dt.test(e)&&(this.checked||!pe.test(e))}).map(function(e,t){var n=k(this).val();return null==n?null:Array.isArray(n)?k.map(n,function(e){return{name:t.name,value:e.replace(At,"\r\n")}}):{name:t.name,value:n.replace(At,"\r\n")}}).get()}});var Lt=/%20/g,Ht=/#.*$/,Ot=/([?&])_=[^&]*/,Pt=/^(.*?):[ \t]*([^\r\n]*)$/gm,Rt=/^(?:GET|HEAD)$/,Mt=/^\/\//,It={},Wt={},$t="*/".concat("*"),Ft=E.createElement("a");function Bt(o){return function(e,t){"string"!=typeof e&&(t=e,e="*");var n,r=0,i=e.toLowerCase().match(R)||[];if(m(t))while(n=i[r++])"+"===n[0]?(n=n.slice(1)||"*",(o[n]=o[n]||[]).unshift(t)):(o[n]=o[n]||[]).push(t)}}function _t(t,i,o,a){var s={},u=t===Wt;function l(e){var r;return s[e]=!0,k.each(t[e]||[],function(e,t){var n=t(i,o,a);return"string"!=typeof n||u||s[n]?u?!(r=n):void 0:(i.dataTypes.unshift(n),l(n),!1)}),r}return l(i.dataTypes[0])||!s["*"]&&l("*")}function zt(e,t){var n,r,i=k.ajaxSettings.flatOptions||{};for(n in t)void 0!==t[n]&&((i[n]?e:r||(r={}))[n]=t[n]);return r&&k.extend(!0,e,r),e}Ft.href=Et.href,k.extend({active:0,lastModified:{},etag:{},ajaxSettings:{url:Et.href,type:"GET",isLocal:/^(?:about|app|app-storage|.+-extension|file|res|widget):$/.test(Et.protocol),global:!0,processData:!0,async:!0,contentType:"application/x-www-form-urlencoded; charset=UTF-8",accepts:{"*":$t,text:"text/plain",html:"text/html",xml:"application/xml, text/xml",json:"application/json, text/javascript"},contents:{xml:/\bxml\b/,html:/\bhtml/,json:/\bjson\b/},responseFields:{xml:"responseXML",text:"responseText",json:"responseJSON"},converters:{"* text":String,"text html":!0,"text json":JSON.parse,"text xml":k.parseXML},flatOptions:{url:!0,context:!0}},ajaxSetup:function(e,t){return t?zt(zt(e,k.ajaxSettings),t):zt(k.ajaxSettings,e)},ajaxPrefilter:Bt(It),ajaxTransport:Bt(Wt),ajax:function(e,t){"object"==typeof e&&(t=e,e=void 0),t=t||{};var c,f,p,n,d,r,h,g,i,o,v=k.ajaxSetup({},t),y=v.context||v,m=v.context&&(y.nodeType||y.jquery)?k(y):k.event,x=k.Deferred(),b=k.Callbacks("once memory"),w=v.statusCode||{},a={},s={},u="canceled",T={readyState:0,getResponseHeader:function(e){var t;if(h){if(!n){n={};while(t=Pt.exec(p))n[t[1].toLowerCase()+" "]=(n[t[1].toLowerCase()+" "]||[]).concat(t[2])}t=n[e.toLowerCase()+" "]}return null==t?null:t.join(", ")},getAllResponseHeaders:function(){return h?p:null},setRequestHeader:function(e,t){return null==h&&(e=s[e.toLowerCase()]=s[e.toLowerCase()]||e,a[e]=t),this},overrideMimeType:function(e){return null==h&&(v.mimeType=e),this},statusCode:function(e){var t;if(e)if(h)T.always(e[T.status]);else for(t in e)w[t]=[w[t],e[t]];return this},abort:function(e){var t=e||u;return c&&c.abort(t),l(0,t),this}};if(x.promise(T),v.url=((e||v.url||Et.href)+"").replace(Mt,Et.protocol+"//"),v.type=t.method||t.type||v.method||v.type,v.dataTypes=(v.dataType||"*").toLowerCase().match(R)||[""],null==v.crossDomain){r=E.createElement("a");try{r.href=v.url,r.href=r.href,v.crossDomain=Ft.protocol+"//"+Ft.host!=r.protocol+"//"+r.host}catch(e){v.crossDomain=!0}}if(v.data&&v.processData&&"string"!=typeof v.data&&(v.data=k.param(v.data,v.traditional)),_t(It,v,t,T),h)return T;for(i in(g=k.event&&v.global)&&0==k.active++&&k.event.trigger("ajaxStart"),v.type=v.type.toUpperCase(),v.hasContent=!Rt.test(v.type),f=v.url.replace(Ht,""),v.hasContent?v.data&&v.processData&&0===(v.contentType||"").indexOf("application/x-www-form-urlencoded")&&(v.data=v.data.replace(Lt,"+")):(o=v.url.slice(f.length),v.data&&(v.processData||"string"==typeof v.data)&&(f+=(St.test(f)?"&":"?")+v.data,delete v.data),!1===v.cache&&(f=f.replace(Ot,"$1"),o=(St.test(f)?"&":"?")+"_="+kt+++o),v.url=f+o),v.ifModified&&(k.lastModified[f]&&T.setRequestHeader("If-Modified-Since",k.lastModified[f]),k.etag[f]&&T.setRequestHeader("If-None-Match",k.etag[f])),(v.data&&v.hasContent&&!1!==v.contentType||t.contentType)&&T.setRequestHeader("Content-Type",v.contentType),T.setRequestHeader("Accept",v.dataTypes[0]&&v.accepts[v.dataTypes[0]]?v.accepts[v.dataTypes[0]]+("*"!==v.dataTypes[0]?", "+$t+"; q=0.01":""):v.accepts["*"]),v.headers)T.setRequestHeader(i,v.headers[i]);if(v.beforeSend&&(!1===v.beforeSend.call(y,T,v)||h))return T.abort();if(u="abort",b.add(v.complete),T.done(v.success),T.fail(v.error),c=_t(Wt,v,t,T)){if(T.readyState=1,g&&m.trigger("ajaxSend",[T,v]),h)return T;v.async&&0<v.timeout&&(d=C.setTimeout(function(){T.abort("timeout")},v.timeout));try{h=!1,c.send(a,l)}catch(e){if(h)throw e;l(-1,e)}}else l(-1,"No Transport");function l(e,t,n,r){var i,o,a,s,u,l=t;h||(h=!0,d&&C.clearTimeout(d),c=void 0,p=r||"",T.readyState=0<e?4:0,i=200<=e&&e<300||304===e,n&&(s=function(e,t,n){var r,i,o,a,s=e.contents,u=e.dataTypes;while("*"===u[0])u.shift(),void 0===r&&(r=e.mimeType||t.getResponseHeader("Content-Type"));if(r)for(i in s)if(s[i]&&s[i].test(r)){u.unshift(i);break}if(u[0]in n)o=u[0];else{for(i in n){if(!u[0]||e.converters[i+" "+u[0]]){o=i;break}a||(a=i)}o=o||a}if(o)return o!==u[0]&&u.unshift(o),n[o]}(v,T,n)),s=function(e,t,n,r){var i,o,a,s,u,l={},c=e.dataTypes.slice();if(c[1])for(a in e.converters)l[a.toLowerCase()]=e.converters[a];o=c.shift();while(o)if(e.responseFields[o]&&(n[e.responseFields[o]]=t),!u&&r&&e.dataFilter&&(t=e.dataFilter(t,e.dataType)),u=o,o=c.shift())if("*"===o)o=u;else if("*"!==u&&u!==o){if(!(a=l[u+" "+o]||l["* "+o]))for(i in l)if((s=i.split(" "))[1]===o&&(a=l[u+" "+s[0]]||l["* "+s[0]])){!0===a?a=l[i]:!0!==l[i]&&(o=s[0],c.unshift(s[1]));break}if(!0!==a)if(a&&e["throws"])t=a(t);else try{t=a(t)}catch(e){return{state:"parsererror",error:a?e:"No conversion from "+u+" to "+o}}}return{state:"success",data:t}}(v,s,T,i),i?(v.ifModified&&((u=T.getResponseHeader("Last-Modified"))&&(k.lastModified[f]=u),(u=T.getResponseHeader("etag"))&&(k.etag[f]=u)),204===e||"HEAD"===v.type?l="nocontent":304===e?l="notmodified":(l=s.state,o=s.data,i=!(a=s.error))):(a=l,!e&&l||(l="error",e<0&&(e=0))),T.status=e,T.statusText=(t||l)+"",i?x.resolveWith(y,[o,l,T]):x.rejectWith(y,[T,l,a]),T.statusCode(w),w=void 0,g&&m.trigger(i?"ajaxSuccess":"ajaxError",[T,v,i?o:a]),b.fireWith(y,[T,l]),g&&(m.trigger("ajaxComplete",[T,v]),--k.active||k.event.trigger("ajaxStop")))}return T},getJSON:function(e,t,n){return k.get(e,t,n,"json")},getScript:function(e,t){return k.get(e,void 0,t,"script")}}),k.each(["get","post"],function(e,i){k[i]=function(e,t,n,r){return m(t)&&(r=r||n,n=t,t=void 0),k.ajax(k.extend({url:e,type:i,dataType:r,data:t,success:n},k.isPlainObject(e)&&e))}}),k._evalUrl=function(e,t){return k.ajax({url:e,type:"GET",dataType:"script",cache:!0,async:!1,global:!1,converters:{"text script":function(){}},dataFilter:function(e){k.globalEval(e,t)}})},k.fn.extend({wrapAll:function(e){var t;return this[0]&&(m(e)&&(e=e.call(this[0])),t=k(e,this[0].ownerDocument).eq(0).clone(!0),this[0].parentNode&&t.insertBefore(this[0]),t.map(function(){var e=this;while(e.firstElementChild)e=e.firstElementChild;return e}).append(this)),this},wrapInner:function(n){return m(n)?this.each(function(e){k(this).wrapInner(n.call(this,e))}):this.each(function(){var e=k(this),t=e.contents();t.length?t.wrapAll(n):e.append(n)})},wrap:function(t){var n=m(t);return this.each(function(e){k(this).wrapAll(n?t.call(this,e):t)})},unwrap:function(e){return this.parent(e).not("body").each(function(){k(this).replaceWith(this.childNodes)}),this}}),k.expr.pseudos.hidden=function(e){return!k.expr.pseudos.visible(e)},k.expr.pseudos.visible=function(e){return!!(e.offsetWidth||e.offsetHeight||e.getClientRects().length)},k.ajaxSettings.xhr=function(){try{return new C.XMLHttpRequest}catch(e){}};var Ut={0:200,1223:204},Xt=k.ajaxSettings.xhr();y.cors=!!Xt&&"withCredentials"in Xt,y.ajax=Xt=!!Xt,k.ajaxTransport(function(i){var o,a;if(y.cors||Xt&&!i.crossDomain)return{send:function(e,t){var n,r=i.xhr();if(r.open(i.type,i.url,i.async,i.username,i.password),i.xhrFields)for(n in i.xhrFields)r[n]=i.xhrFields[n];for(n in i.mimeType&&r.overrideMimeType&&r.overrideMimeType(i.mimeType),i.crossDomain||e["X-Requested-With"]||(e["X-Requested-With"]="XMLHttpRequest"),e)r.setRequestHeader(n,e[n]);o=function(e){return function(){o&&(o=a=r.onload=r.onerror=r.onabort=r.ontimeout=r.onreadystatechange=null,"abort"===e?r.abort():"error"===e?"number"!=typeof r.status?t(0,"error"):t(r.status,r.statusText):t(Ut[r.status]||r.status,r.statusText,"text"!==(r.responseType||"text")||"string"!=typeof r.responseText?{binary:r.response}:{text:r.responseText},r.getAllResponseHeaders()))}},r.onload=o(),a=r.onerror=r.ontimeout=o("error"),void 0!==r.onabort?r.onabort=a:r.onreadystatechange=function(){4===r.readyState&&C.setTimeout(function(){o&&a()})},o=o("abort");try{r.send(i.hasContent&&i.data||null)}catch(e){if(o)throw e}},abort:function(){o&&o()}}}),k.ajaxPrefilter(function(e){e.crossDomain&&(e.contents.script=!1)}),k.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},contents:{script:/\b(?:java|ecma)script\b/},converters:{"text script":function(e){return k.globalEval(e),e}}}),k.ajaxPrefilter("script",function(e){void 0===e.cache&&(e.cache=!1),e.crossDomain&&(e.type="GET")}),k.ajaxTransport("script",function(n){var r,i;if(n.crossDomain||n.scriptAttrs)return{send:function(e,t){r=k("<script>").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),E.head.appendChild(r[0])},abort:function(){i&&i()}}});var Vt,Gt=[],Yt=/(=)\?(?=&|$)|\?\?/;k.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=Gt.pop()||k.expando+"_"+kt++;return this[e]=!0,e}}),k.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Yt.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Yt.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=m(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Yt,"$1"+r):!1!==e.jsonp&&(e.url+=(St.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||k.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=C[r],C[r]=function(){o=arguments},n.always(function(){void 0===i?k(C).removeProp(r):C[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,Gt.push(r)),o&&m(i)&&i(o[0]),o=i=void 0}),"script"}),y.createHTMLDocument=((Vt=E.implementation.createHTMLDocument("").body).innerHTML="<form></form><form></form>",2===Vt.childNodes.length),k.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(y.createHTMLDocument?((r=(t=E.implementation.createHTMLDocument("")).createElement("base")).href=E.location.href,t.head.appendChild(r)):t=E),o=!n&&[],(i=D.exec(e))?[t.createElement(i[1])]:(i=we([e],t,o),o&&o.length&&k(o).remove(),k.merge([],i.childNodes)));var r,i,o},k.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1<s&&(r=mt(e.slice(s)),e=e.slice(0,s)),m(t)?(n=t,t=void 0):t&&"object"==typeof t&&(i="POST"),0<a.length&&k.ajax({url:e,type:i||"GET",dataType:"html",data:t}).done(function(e){o=arguments,a.html(r?k("<div>").append(k.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},k.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){k.fn[t]=function(e){return this.on(t,e)}}),k.expr.pseudos.animated=function(t){return k.grep(k.timers,function(e){return t===e.elem}).length},k.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=k.css(e,"position"),c=k(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=k.css(e,"top"),u=k.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),m(t)&&(t=t.call(e,n,k.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):c.css(f)}},k.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){k.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===k.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===k.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=k(e).offset()).top+=k.css(e,"borderTopWidth",!0),i.left+=k.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-k.css(r,"marginTop",!0),left:t.left-i.left-k.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===k.css(e,"position"))e=e.offsetParent;return e||ie})}}),k.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;k.fn[t]=function(e){return _(this,function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),k.each(["top","left"],function(e,n){k.cssHooks[n]=ze(y.pixelPosition,function(e,t){if(t)return t=_e(e,n),$e.test(t)?k(e).position()[n]+"px":t})}),k.each({Height:"height",Width:"width"},function(a,s){k.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){k.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return _(this,function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?k.css(e,t,i):k.style(e,t,n,i)},s,n?e:void 0,n)}})}),k.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){k.fn[n]=function(e,t){return 0<arguments.length?this.on(n,null,e,t):this.trigger(n)}}),k.fn.extend({hover:function(e,t){return this.mouseenter(e).mouseleave(t||e)}}),k.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)}}),k.proxy=function(e,t){var n,r,i;if("string"==typeof t&&(n=e[t],t=e,e=n),m(e))return r=s.call(arguments,2),(i=function(){return e.apply(t||this,r.concat(s.call(arguments)))}).guid=e.guid=e.guid||k.guid++,i},k.holdReady=function(e){e?k.readyWait++:k.ready(!0)},k.isArray=Array.isArray,k.parseJSON=JSON.parse,k.nodeName=A,k.isFunction=m,k.isWindow=x,k.camelCase=V,k.type=w,k.now=Date.now,k.isNumeric=function(e){var t=k.type(e);return("number"===t||"string"===t)&&!isNaN(e-parseFloat(e))},"function"==typeof define&&define.amd&&define("jquery",[],function(){return k});var Qt=C.jQuery,Jt=C.$;return k.noConflict=function(e){return C.$===k&&(C.$=Jt),e&&C.jQuery===k&&(C.jQuery=Qt),k},e||(C.jQuery=C.$=k),k});
\ No newline at end of file
diff --git a/core/src/main/resources/org/apache/spark/ui/static/jquery-3.5.1.min.js b/core/src/main/resources/org/apache/spark/ui/static/jquery-3.5.1.min.js
new file mode 100644
index 0000000000000..b0614034ad3a9
--- /dev/null
+++ b/core/src/main/resources/org/apache/spark/ui/static/jquery-3.5.1.min.js
@@ -0,0 +1,2 @@
+/*! jQuery v3.5.1 | (c) JS Foundation and other contributors | jquery.org/license */
+!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(C,e){"use strict";var t=[],r=Object.getPrototypeOf,s=t.slice,g=t.flat?function(e){return t.flat.call(e)}:function(e){return t.concat.apply([],e)},u=t.push,i=t.indexOf,n={},o=n.toString,v=n.hasOwnProperty,a=v.toString,l=a.call(Object),y={},m=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType},x=function(e){return null!=e&&e===e.window},E=C.document,c={type:!0,src:!0,nonce:!0,noModule:!0};function b(e,t,n){var r,i,o=(n=n||E).createElement("script");if(o.text=e,t)for(r in c)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function w(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[o.call(e)]||"object":typeof e}var f="3.5.1",S=function(e,t){return new S.fn.init(e,t)};function p(e){var t=!!e&&"length"in e&&e.length,n=w(e);return!m(e)&&!x(e)&&("array"===n||0===t||"number"==typeof t&&0<t&&t-1 in e)}S.fn=S.prototype={jquery:f,constructor:S,length:0,toArray:function(){return s.call(this)},get:function(e){return null==e?s.call(this):e<0?this[e+this.length]:this[e]},pushStack:function(e){var t=S.merge(this.constructor(),e);return t.prevObject=this,t},each:function(e){return S.each(this,e)},map:function(n){return this.pushStack(S.map(this,function(e,t){return n.call(e,t,e)}))},slice:function(){return this.pushStack(s.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},even:function(){return this.pushStack(S.grep(this,function(e,t){return(t+1)%2}))},odd:function(){return this.pushStack(S.grep(this,function(e,t){return t%2}))},eq:function(e){var t=this.length,n=+e+(e<0?t:0);return this.pushStack(0<=n&&n<t?[this[n]]:[])},end:function(){return this.prevObject||this.constructor()},push:u,sort:t.sort,splice:t.splice},S.extend=S.fn.extend=function(){var e,t,n,r,i,o,a=arguments[0]||{},s=1,u=arguments.length,l=!1;for("boolean"==typeof a&&(l=a,a=arguments[s]||{},s++),"object"==typeof a||m(a)||(a={}),s===u&&(a=this,s--);s<u;s++)if(null!=(e=arguments[s]))for(t in e)r=e[t],"__proto__"!==t&&a!==r&&(l&&r&&(S.isPlainObject(r)||(i=Array.isArray(r)))?(n=a[t],o=i&&!Array.isArray(n)?[]:i||S.isPlainObject(n)?n:{},i=!1,a[t]=S.extend(l,o,r)):void 0!==r&&(a[t]=r));return a},S.extend({expando:"jQuery"+(f+Math.random()).replace(/\D/g,""),isReady:!0,error:function(e){throw new Error(e)},noop:function(){},isPlainObject:function(e){var t,n;return!(!e||"[object Object]"!==o.call(e))&&(!(t=r(e))||"function"==typeof(n=v.call(t,"constructor")&&t.constructor)&&a.call(n)===l)},isEmptyObject:function(e){var t;for(t in e)return!1;return!0},globalEval:function(e,t,n){b(e,{nonce:t&&t.nonce},n)},each:function(e,t){var n,r=0;if(p(e)){for(n=e.length;r<n;r++)if(!1===t.call(e[r],r,e[r]))break}else for(r in e)if(!1===t.call(e[r],r,e[r]))break;return e},makeArray:function(e,t){var n=t||[];return null!=e&&(p(Object(e))?S.merge(n,"string"==typeof e?[e]:e):u.call(n,e)),n},inArray:function(e,t,n){return null==t?-1:i.call(t,e,n)},merge:function(e,t){for(var n=+t.length,r=0,i=e.length;r<n;r++)e[i++]=t[r];return e.length=i,e},grep:function(e,t,n){for(var r=[],i=0,o=e.length,a=!n;i<o;i++)!t(e[i],i)!==a&&r.push(e[i]);return r},map:function(e,t,n){var r,i,o=0,a=[];if(p(e))for(r=e.length;o<r;o++)null!=(i=t(e[o],o,n))&&a.push(i);else for(o in e)null!=(i=t(e[o],o,n))&&a.push(i);return g(a)},guid:1,support:y}),"function"==typeof Symbol&&(S.fn[Symbol.iterator]=t[Symbol.iterator]),S.each("Boolean Number String Function Array Date RegExp Object Error Symbol".split(" "),function(e,t){n["[object "+t+"]"]=t.toLowerCase()});var d=function(n){var e,d,b,o,i,h,f,g,w,u,l,T,C,a,E,v,s,c,y,S="sizzle"+1*new Date,p=n.document,k=0,r=0,m=ue(),x=ue(),A=ue(),N=ue(),D=function(e,t){return e===t&&(l=!0),0},j={}.hasOwnProperty,t=[],q=t.pop,L=t.push,H=t.push,O=t.slice,P=function(e,t){for(var n=0,r=e.length;n<r;n++)if(e[n]===t)return n;return-1},R="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",M="[\\x20\\t\\r\\n\\f]",I="(?:\\\\[\\da-fA-F]{1,6}"+M+"?|\\\\[^\\r\\n\\f]|[\\w-]|[^\0-\\x7f])+",W="\\["+M+"*("+I+")(?:"+M+"*([*^$|!~]?=)"+M+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+I+"))|)"+M+"*\\]",F=":("+I+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+W+")*)|.*)\\)|)",B=new RegExp(M+"+","g"),$=new RegExp("^"+M+"+|((?:^|[^\\\\])(?:\\\\.)*)"+M+"+$","g"),_=new RegExp("^"+M+"*,"+M+"*"),z=new RegExp("^"+M+"*([>+~]|"+M+")"+M+"*"),U=new RegExp(M+"|>"),X=new RegExp(F),V=new RegExp("^"+I+"$"),G={ID:new RegExp("^#("+I+")"),CLASS:new RegExp("^\\.("+I+")"),TAG:new RegExp("^("+I+"|[*])"),ATTR:new RegExp("^"+W),PSEUDO:new RegExp("^"+F),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+R+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+M+"*((?:-\\d)?\\d*)"+M+"*\\)|)(?=[^-]|$)","i")},Y=/HTML$/i,Q=/^(?:input|select|textarea|button)$/i,J=/^h\d$/i,K=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ee=/[+~]/,te=new RegExp("\\\\[\\da-fA-F]{1,6}"+M+"?|\\\\([^\\r\\n\\f])","g"),ne=function(e,t){var n="0x"+e.slice(1)-65536;return t||(n<0?String.fromCharCode(n+65536):String.fromCharCode(n>>10|55296,1023&n|56320))},re=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ie=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},oe=function(){T()},ae=be(function(e){return!0===e.disabled&&"fieldset"===e.nodeName.toLowerCase()},{dir:"parentNode",next:"legend"});try{H.apply(t=O.call(p.childNodes),p.childNodes),t[p.childNodes.length].nodeType}catch(e){H={apply:t.length?function(e,t){L.apply(e,O.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function se(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,p=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==p&&9!==p&&11!==p)return n;if(!r&&(T(e),e=e||C,E)){if(11!==p&&(u=Z.exec(t)))if(i=u[1]){if(9===p){if(!(a=e.getElementById(i)))return n;if(a.id===i)return n.push(a),n}else if(f&&(a=f.getElementById(i))&&y(e,a)&&a.id===i)return n.push(a),n}else{if(u[2])return H.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&d.getElementsByClassName&&e.getElementsByClassName)return H.apply(n,e.getElementsByClassName(i)),n}if(d.qsa&&!N[t+" "]&&(!v||!v.test(t))&&(1!==p||"object"!==e.nodeName.toLowerCase())){if(c=t,f=e,1===p&&(U.test(t)||z.test(t))){(f=ee.test(t)&&ye(e.parentNode)||e)===e&&d.scope||((s=e.getAttribute("id"))?s=s.replace(re,ie):e.setAttribute("id",s=S)),o=(l=h(t)).length;while(o--)l[o]=(s?"#"+s:":scope")+" "+xe(l[o]);c=l.join(",")}try{return H.apply(n,f.querySelectorAll(c)),n}catch(e){N(t,!0)}finally{s===S&&e.removeAttribute("id")}}}return g(t.replace($,"$1"),e,n,r)}function ue(){var r=[];return function e(t,n){return r.push(t+" ")>b.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function le(e){return e[S]=!0,e}function ce(e){var t=C.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){var n=e.split("|"),r=n.length;while(r--)b.attrHandle[n[r]]=t}function pe(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function de(t){return function(e){return"input"===e.nodeName.toLowerCase()&&e.type===t}}function he(n){return function(e){var t=e.nodeName.toLowerCase();return("input"===t||"button"===t)&&e.type===n}}function ge(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&ae(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function ve(a){return le(function(o){return o=+o,le(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function ye(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}for(e in d=se.support={},i=se.isXML=function(e){var t=e.namespaceURI,n=(e.ownerDocument||e).documentElement;return!Y.test(t||n&&n.nodeName||"HTML")},T=se.setDocument=function(e){var t,n,r=e?e.ownerDocument||e:p;return r!=C&&9===r.nodeType&&r.documentElement&&(a=(C=r).documentElement,E=!i(C),p!=C&&(n=C.defaultView)&&n.top!==n&&(n.addEventListener?n.addEventListener("unload",oe,!1):n.attachEvent&&n.attachEvent("onunload",oe)),d.scope=ce(function(e){return a.appendChild(e).appendChild(C.createElement("div")),"undefined"!=typeof e.querySelectorAll&&!e.querySelectorAll(":scope fieldset div").length}),d.attributes=ce(function(e){return e.className="i",!e.getAttribute("className")}),d.getElementsByTagName=ce(function(e){return e.appendChild(C.createComment("")),!e.getElementsByTagName("*").length}),d.getElementsByClassName=K.test(C.getElementsByClassName),d.getById=ce(function(e){return a.appendChild(e).id=S,!C.getElementsByName||!C.getElementsByName(S).length}),d.getById?(b.filter.ID=function(e){var t=e.replace(te,ne);return function(e){return e.getAttribute("id")===t}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n=t.getElementById(e);return n?[n]:[]}}):(b.filter.ID=function(e){var n=e.replace(te,ne);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),b.find.TAG=d.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):d.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},b.find.CLASS=d.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&E)return t.getElementsByClassName(e)},s=[],v=[],(d.qsa=K.test(C.querySelectorAll))&&(ce(function(e){var t;a.appendChild(e).innerHTML="<a id='"+S+"'></a><select id='"+S+"-\r\\' msallowcapture=''><option selected=''></option></select>",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+M+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+M+"*(?:value|"+R+")"),e.querySelectorAll("[id~="+S+"-]").length||v.push("~="),(t=C.createElement("input")).setAttribute("name",""),e.appendChild(t),e.querySelectorAll("[name='']").length||v.push("\\["+M+"*name"+M+"*="+M+"*(?:''|\"\")"),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+S+"+*").length||v.push(".#.+[+~]"),e.querySelectorAll("\\\f"),v.push("[\\r\\n\\f]")}),ce(function(e){e.innerHTML="<a href='' disabled='disabled'></a><select disabled='disabled'><option/></select>";var t=C.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+M+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")})),(d.matchesSelector=K.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce(function(e){d.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",F)}),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=K.test(a.compareDocumentPosition),y=t||K.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},D=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)==(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!d.sortDetached&&t.compareDocumentPosition(e)===n?e==C||e.ownerDocument==p&&y(p,e)?-1:t==C||t.ownerDocument==p&&y(p,t)?1:u?P(u,e)-P(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e==C?-1:t==C?1:i?-1:o?1:u?P(u,e)-P(u,t):0;if(i===o)return pe(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?pe(a[r],s[r]):a[r]==p?-1:s[r]==p?1:0}),C},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if(T(e),d.matchesSelector&&E&&!N[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||d.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){N(t,!0)}return 0<se(t,C,null,[e]).length},se.contains=function(e,t){return(e.ownerDocument||e)!=C&&T(e),y(e,t)},se.attr=function(e,t){(e.ownerDocument||e)!=C&&T(e);var n=b.attrHandle[t.toLowerCase()],r=n&&j.call(b.attrHandle,t.toLowerCase())?n(e,t,!E):void 0;return void 0!==r?r:d.attributes||!E?e.getAttribute(t):(r=e.getAttributeNode(t))&&r.specified?r.value:null},se.escape=function(e){return(e+"").replace(re,ie)},se.error=function(e){throw new Error("Syntax error, unrecognized expression: "+e)},se.uniqueSort=function(e){var t,n=[],r=0,i=0;if(l=!d.detectDuplicates,u=!d.sortStable&&e.slice(0),e.sort(D),l){while(t=e[i++])t===e[i]&&(r=n.push(i));while(r--)e.splice(n[r],1)}return u=null,e},o=se.getText=function(e){var t,n="",r=0,i=e.nodeType;if(i){if(1===i||9===i||11===i){if("string"==typeof e.textContent)return e.textContent;for(e=e.firstChild;e;e=e.nextSibling)n+=o(e)}else if(3===i||4===i)return e.nodeValue}else while(t=e[r++])n+=o(t);return n},(b=se.selectors={cacheLength:50,createPseudo:le,match:G,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||"").replace(te,ne),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return G.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&X.test(n)&&(t=h(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=m[e+" "];return t||(t=new RegExp("(^|"+M+")"+e+"("+M+"|$)"))&&m(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=se.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1<t.indexOf(i):"$="===r?i&&t.slice(-i.length)===i:"~="===r?-1<(" "+t.replace(B," ")+" ").indexOf(i):"|="===r&&(t===i||t.slice(0,i.length+1)===i+"-"))}},CHILD:function(h,e,t,g,v){var y="nth"!==h.slice(0,3),m="last"!==h.slice(-4),x="of-type"===e;return 1===g&&0===v?function(e){return!!e.parentNode}:function(e,t,n){var r,i,o,a,s,u,l=y!==m?"nextSibling":"previousSibling",c=e.parentNode,f=x&&e.nodeName.toLowerCase(),p=!n&&!x,d=!1;if(c){if(y){while(l){a=e;while(a=a[l])if(x?a.nodeName.toLowerCase()===f:1===a.nodeType)return!1;u=l="only"===h&&!u&&"nextSibling"}return!0}if(u=[m?c.firstChild:c.lastChild],m&&p){d=(s=(r=(i=(o=(a=c)[S]||(a[S]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]||[])[0]===k&&r[1])&&r[2],a=s&&c.childNodes[s];while(a=++s&&a&&a[l]||(d=s=0)||u.pop())if(1===a.nodeType&&++d&&a===e){i[h]=[k,s,d];break}}else if(p&&(d=s=(r=(i=(o=(a=e)[S]||(a[S]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]||[])[0]===k&&r[1]),!1===d)while(a=++s&&a&&a[l]||(d=s=0)||u.pop())if((x?a.nodeName.toLowerCase()===f:1===a.nodeType)&&++d&&(p&&((i=(o=a[S]||(a[S]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]=[k,d]),a===e))break;return(d-=v)===g||d%g==0&&0<=d/g}}},PSEUDO:function(e,o){var t,a=b.pseudos[e]||b.setFilters[e.toLowerCase()]||se.error("unsupported pseudo: "+e);return a[S]?a(o):1<a.length?(t=[e,e,"",o],b.setFilters.hasOwnProperty(e.toLowerCase())?le(function(e,t){var n,r=a(e,o),i=r.length;while(i--)e[n=P(e,r[i])]=!(t[n]=r[i])}):function(e){return a(e,0,t)}):a}},pseudos:{not:le(function(e){var r=[],i=[],s=f(e.replace($,"$1"));return s[S]?le(function(e,t,n,r){var i,o=s(e,null,r,[]),a=e.length;while(a--)(i=o[a])&&(e[a]=!(t[a]=i))}):function(e,t,n){return r[0]=e,s(r,null,n,i),r[0]=null,!i.pop()}}),has:le(function(t){return function(e){return 0<se(t,e).length}}),contains:le(function(t){return t=t.replace(te,ne),function(e){return-1<(e.textContent||o(e)).indexOf(t)}}),lang:le(function(n){return V.test(n||"")||se.error("unsupported lang: "+n),n=n.replace(te,ne).toLowerCase(),function(e){var t;do{if(t=E?e.lang:e.getAttribute("xml:lang")||e.getAttribute("lang"))return(t=t.toLowerCase())===n||0===t.indexOf(n+"-")}while((e=e.parentNode)&&1===e.nodeType);return!1}}),target:function(e){var t=n.location&&n.location.hash;return t&&t.slice(1)===e.id},root:function(e){return e===a},focus:function(e){return e===C.activeElement&&(!C.hasFocus||C.hasFocus())&&!!(e.type||e.href||~e.tabIndex)},enabled:ge(!1),disabled:ge(!0),checked:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&!!e.checked||"option"===t&&!!e.selected},selected:function(e){return e.parentNode&&e.parentNode.selectedIndex,!0===e.selected},empty:function(e){for(e=e.firstChild;e;e=e.nextSibling)if(e.nodeType<6)return!1;return!0},parent:function(e){return!b.pseudos.empty(e)},header:function(e){return J.test(e.nodeName)},input:function(e){return Q.test(e.nodeName)},button:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&"button"===e.type||"button"===t},text:function(e){var t;return"input"===e.nodeName.toLowerCase()&&"text"===e.type&&(null==(t=e.getAttribute("type"))||"text"===t.toLowerCase())},first:ve(function(){return[0]}),last:ve(function(e,t){return[t-1]}),eq:ve(function(e,t,n){return[n<0?n+t:n]}),even:ve(function(e,t){for(var n=0;n<t;n+=2)e.push(n);return e}),odd:ve(function(e,t){for(var n=1;n<t;n+=2)e.push(n);return e}),lt:ve(function(e,t,n){for(var r=n<0?n+t:t<n?t:n;0<=--r;)e.push(r);return e}),gt:ve(function(e,t,n){for(var r=n<0?n+t:n;++r<t;)e.push(r);return e})}}).pseudos.nth=b.pseudos.eq,{radio:!0,checkbox:!0,file:!0,password:!0,image:!0})b.pseudos[e]=de(e);for(e in{submit:!0,reset:!0})b.pseudos[e]=he(e);function me(){}function xe(e){for(var t=0,n=e.length,r="";t<n;t++)r+=e[t].value;return r}function be(s,e,t){var u=e.dir,l=e.next,c=l||u,f=t&&"parentNode"===c,p=r++;return e.first?function(e,t,n){while(e=e[u])if(1===e.nodeType||f)return s(e,t,n);return!1}:function(e,t,n){var r,i,o,a=[k,p];if(n){while(e=e[u])if((1===e.nodeType||f)&&s(e,t,n))return!0}else while(e=e[u])if(1===e.nodeType||f)if(i=(o=e[S]||(e[S]={}))[e.uniqueID]||(o[e.uniqueID]={}),l&&l===e.nodeName.toLowerCase())e=e[u]||e;else{if((r=i[c])&&r[0]===k&&r[1]===p)return a[2]=r[2];if((i[c]=a)[2]=s(e,t,n))return!0}return!1}}function we(i){return 1<i.length?function(e,t,n){var r=i.length;while(r--)if(!i[r](e,t,n))return!1;return!0}:i[0]}function Te(e,t,n,r,i){for(var o,a=[],s=0,u=e.length,l=null!=t;s<u;s++)(o=e[s])&&(n&&!n(o,r,i)||(a.push(o),l&&t.push(s)));return a}function Ce(d,h,g,v,y,e){return v&&!v[S]&&(v=Ce(v)),y&&!y[S]&&(y=Ce(y,e)),le(function(e,t,n,r){var i,o,a,s=[],u=[],l=t.length,c=e||function(e,t,n){for(var r=0,i=t.length;r<i;r++)se(e,t[r],n);return n}(h||"*",n.nodeType?[n]:n,[]),f=!d||!e&&h?c:Te(c,s,d,n,r),p=g?y||(e?d:l||v)?[]:t:f;if(g&&g(f,p,n,r),v){i=Te(p,u),v(i,[],n,r),o=i.length;while(o--)(a=i[o])&&(p[u[o]]=!(f[u[o]]=a))}if(e){if(y||d){if(y){i=[],o=p.length;while(o--)(a=p[o])&&i.push(f[o]=a);y(null,p=[],i,r)}o=p.length;while(o--)(a=p[o])&&-1<(i=y?P(e,a):s[o])&&(e[i]=!(t[i]=a))}}else p=Te(p===t?p.splice(l,p.length):p),y?y(null,t,p,r):H.apply(t,p)})}function Ee(e){for(var i,t,n,r=e.length,o=b.relative[e[0].type],a=o||b.relative[" "],s=o?1:0,u=be(function(e){return e===i},a,!0),l=be(function(e){return-1<P(i,e)},a,!0),c=[function(e,t,n){var r=!o&&(n||t!==w)||((i=t).nodeType?u(e,t,n):l(e,t,n));return i=null,r}];s<r;s++)if(t=b.relative[e[s].type])c=[be(we(c),t)];else{if((t=b.filter[e[s].type].apply(null,e[s].matches))[S]){for(n=++s;n<r;n++)if(b.relative[e[n].type])break;return Ce(1<s&&we(c),1<s&&xe(e.slice(0,s-1).concat({value:" "===e[s-2].type?"*":""})).replace($,"$1"),t,s<n&&Ee(e.slice(s,n)),n<r&&Ee(e=e.slice(n)),n<r&&xe(e))}c.push(t)}return we(c)}return me.prototype=b.filters=b.pseudos,b.setFilters=new me,h=se.tokenize=function(e,t){var n,r,i,o,a,s,u,l=x[e+" "];if(l)return t?0:l.slice(0);a=e,s=[],u=b.preFilter;while(a){for(o in n&&!(r=_.exec(a))||(r&&(a=a.slice(r[0].length)||a),s.push(i=[])),n=!1,(r=z.exec(a))&&(n=r.shift(),i.push({value:n,type:r[0].replace($," ")}),a=a.slice(n.length)),b.filter)!(r=G[o].exec(a))||u[o]&&!(r=u[o](r))||(n=r.shift(),i.push({value:n,type:o,matches:r}),a=a.slice(n.length));if(!n)break}return t?a.length:a?se.error(e):x(e,s).slice(0)},f=se.compile=function(e,t){var n,v,y,m,x,r,i=[],o=[],a=A[e+" "];if(!a){t||(t=h(e)),n=t.length;while(n--)(a=Ee(t[n]))[S]?i.push(a):o.push(a);(a=A(e,(v=o,m=0<(y=i).length,x=0<v.length,r=function(e,t,n,r,i){var o,a,s,u=0,l="0",c=e&&[],f=[],p=w,d=e||x&&b.find.TAG("*",i),h=k+=null==p?1:Math.random()||.1,g=d.length;for(i&&(w=t==C||t||i);l!==g&&null!=(o=d[l]);l++){if(x&&o){a=0,t||o.ownerDocument==C||(T(o),n=!E);while(s=v[a++])if(s(o,t||C,n)){r.push(o);break}i&&(k=h)}m&&((o=!s&&o)&&u--,e&&c.push(o))}if(u+=l,m&&l!==u){a=0;while(s=y[a++])s(c,f,t,n);if(e){if(0<u)while(l--)c[l]||f[l]||(f[l]=q.call(r));f=Te(f)}H.apply(r,f),i&&!e&&0<f.length&&1<u+y.length&&se.uniqueSort(r)}return i&&(k=h,w=p),c},m?le(r):r))).selector=e}return a},g=se.select=function(e,t,n,r){var i,o,a,s,u,l="function"==typeof e&&e,c=!r&&h(e=l.selector||e);if(n=n||[],1===c.length){if(2<(o=c[0]=c[0].slice(0)).length&&"ID"===(a=o[0]).type&&9===t.nodeType&&E&&b.relative[o[1].type]){if(!(t=(b.find.ID(a.matches[0].replace(te,ne),t)||[])[0]))return n;l&&(t=t.parentNode),e=e.slice(o.shift().value.length)}i=G.needsContext.test(e)?0:o.length;while(i--){if(a=o[i],b.relative[s=a.type])break;if((u=b.find[s])&&(r=u(a.matches[0].replace(te,ne),ee.test(o[0].type)&&ye(t.parentNode)||t))){if(o.splice(i,1),!(e=r.length&&xe(o)))return H.apply(n,r),n;break}}}return(l||f(e,c))(r,t,!E,n,!t||ee.test(e)&&ye(t.parentNode)||t),n},d.sortStable=S.split("").sort(D).join("")===S,d.detectDuplicates=!!l,T(),d.sortDetached=ce(function(e){return 1&e.compareDocumentPosition(C.createElement("fieldset"))}),ce(function(e){return e.innerHTML="<a href='#'></a>","#"===e.firstChild.getAttribute("href")})||fe("type|href|height|width",function(e,t,n){if(!n)return e.getAttribute(t,"type"===t.toLowerCase()?1:2)}),d.attributes&&ce(function(e){return e.innerHTML="<input/>",e.firstChild.setAttribute("value",""),""===e.firstChild.getAttribute("value")})||fe("value",function(e,t,n){if(!n&&"input"===e.nodeName.toLowerCase())return e.defaultValue}),ce(function(e){return null==e.getAttribute("disabled")})||fe(R,function(e,t,n){var r;if(!n)return!0===e[t]?t.toLowerCase():(r=e.getAttributeNode(t))&&r.specified?r.value:null}),se}(C);S.find=d,S.expr=d.selectors,S.expr[":"]=S.expr.pseudos,S.uniqueSort=S.unique=d.uniqueSort,S.text=d.getText,S.isXMLDoc=d.isXML,S.contains=d.contains,S.escapeSelector=d.escape;var h=function(e,t,n){var r=[],i=void 0!==n;while((e=e[t])&&9!==e.nodeType)if(1===e.nodeType){if(i&&S(e).is(n))break;r.push(e)}return r},T=function(e,t){for(var n=[];e;e=e.nextSibling)1===e.nodeType&&e!==t&&n.push(e);return n},k=S.expr.match.needsContext;function A(e,t){return e.nodeName&&e.nodeName.toLowerCase()===t.toLowerCase()}var N=/^<([a-z][^\/\0>:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function D(e,n,r){return m(n)?S.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?S.grep(e,function(e){return e===n!==r}):"string"!=typeof n?S.grep(e,function(e){return-1<i.call(n,e)!==r}):S.filter(n,e,r)}S.filter=function(e,t,n){var r=t[0];return n&&(e=":not("+e+")"),1===t.length&&1===r.nodeType?S.find.matchesSelector(r,e)?[r]:[]:S.find.matches(e,S.grep(t,function(e){return 1===e.nodeType}))},S.fn.extend({find:function(e){var t,n,r=this.length,i=this;if("string"!=typeof e)return this.pushStack(S(e).filter(function(){for(t=0;t<r;t++)if(S.contains(i[t],this))return!0}));for(n=this.pushStack([]),t=0;t<r;t++)S.find(e,i[t],n);return 1<r?S.uniqueSort(n):n},filter:function(e){return this.pushStack(D(this,e||[],!1))},not:function(e){return this.pushStack(D(this,e||[],!0))},is:function(e){return!!D(this,"string"==typeof e&&k.test(e)?S(e):e||[],!1).length}});var j,q=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]+))$/;(S.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||j,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:q.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof S?t[0]:t,S.merge(this,S.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:E,!0)),N.test(r[1])&&S.isPlainObject(t))for(r in t)m(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=E.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):m(e)?void 0!==n.ready?n.ready(e):e(S):S.makeArray(e,this)}).prototype=S.fn,j=S(E);var L=/^(?:parents|prev(?:Until|All))/,H={children:!0,contents:!0,next:!0,prev:!0};function O(e,t){while((e=e[t])&&1!==e.nodeType);return e}S.fn.extend({has:function(e){var t=S(e,this),n=t.length;return this.filter(function(){for(var e=0;e<n;e++)if(S.contains(this,t[e]))return!0})},closest:function(e,t){var n,r=0,i=this.length,o=[],a="string"!=typeof e&&S(e);if(!k.test(e))for(;r<i;r++)for(n=this[r];n&&n!==t;n=n.parentNode)if(n.nodeType<11&&(a?-1<a.index(n):1===n.nodeType&&S.find.matchesSelector(n,e))){o.push(n);break}return this.pushStack(1<o.length?S.uniqueSort(o):o)},index:function(e){return e?"string"==typeof e?i.call(S(e),this[0]):i.call(this,e.jquery?e[0]:e):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(e,t){return this.pushStack(S.uniqueSort(S.merge(this.get(),S(e,t))))},addBack:function(e){return this.add(null==e?this.prevObject:this.prevObject.filter(e))}}),S.each({parent:function(e){var t=e.parentNode;return t&&11!==t.nodeType?t:null},parents:function(e){return h(e,"parentNode")},parentsUntil:function(e,t,n){return h(e,"parentNode",n)},next:function(e){return O(e,"nextSibling")},prev:function(e){return O(e,"previousSibling")},nextAll:function(e){return h(e,"nextSibling")},prevAll:function(e){return h(e,"previousSibling")},nextUntil:function(e,t,n){return h(e,"nextSibling",n)},prevUntil:function(e,t,n){return h(e,"previousSibling",n)},siblings:function(e){return T((e.parentNode||{}).firstChild,e)},children:function(e){return T(e.firstChild)},contents:function(e){return null!=e.contentDocument&&r(e.contentDocument)?e.contentDocument:(A(e,"template")&&(e=e.content||e),S.merge([],e.childNodes))}},function(r,i){S.fn[r]=function(e,t){var n=S.map(this,i,e);return"Until"!==r.slice(-5)&&(t=e),t&&"string"==typeof t&&(n=S.filter(t,n)),1<this.length&&(H[r]||S.uniqueSort(n),L.test(r)&&n.reverse()),this.pushStack(n)}});var P=/[^\x20\t\r\n\f]+/g;function R(e){return e}function M(e){throw e}function I(e,t,n,r){var i;try{e&&m(i=e.promise)?i.call(e).done(t).fail(n):e&&m(i=e.then)?i.call(e,t,n):t.apply(void 0,[e].slice(r))}catch(e){n.apply(void 0,[e])}}S.Callbacks=function(r){var e,n;r="string"==typeof r?(e=r,n={},S.each(e.match(P)||[],function(e,t){n[t]=!0}),n):S.extend({},r);var i,t,o,a,s=[],u=[],l=-1,c=function(){for(a=a||r.once,o=i=!0;u.length;l=-1){t=u.shift();while(++l<s.length)!1===s[l].apply(t[0],t[1])&&r.stopOnFalse&&(l=s.length,t=!1)}r.memory||(t=!1),i=!1,a&&(s=t?[]:"")},f={add:function(){return s&&(t&&!i&&(l=s.length-1,u.push(t)),function n(e){S.each(e,function(e,t){m(t)?r.unique&&f.has(t)||s.push(t):t&&t.length&&"string"!==w(t)&&n(t)})}(arguments),t&&!i&&c()),this},remove:function(){return S.each(arguments,function(e,t){var n;while(-1<(n=S.inArray(t,s,n)))s.splice(n,1),n<=l&&l--}),this},has:function(e){return e?-1<S.inArray(e,s):0<s.length},empty:function(){return s&&(s=[]),this},disable:function(){return a=u=[],s=t="",this},disabled:function(){return!s},lock:function(){return a=u=[],t||i||(s=t=""),this},locked:function(){return!!a},fireWith:function(e,t){return a||(t=[e,(t=t||[]).slice?t.slice():t],u.push(t),i||c()),this},fire:function(){return f.fireWith(this,arguments),this},fired:function(){return!!o}};return f},S.extend({Deferred:function(e){var o=[["notify","progress",S.Callbacks("memory"),S.Callbacks("memory"),2],["resolve","done",S.Callbacks("once memory"),S.Callbacks("once memory"),0,"resolved"],["reject","fail",S.Callbacks("once memory"),S.Callbacks("once memory"),1,"rejected"]],i="pending",a={state:function(){return i},always:function(){return s.done(arguments).fail(arguments),this},"catch":function(e){return a.then(null,e)},pipe:function(){var i=arguments;return S.Deferred(function(r){S.each(o,function(e,t){var n=m(i[t[4]])&&i[t[4]];s[t[1]](function(){var e=n&&n.apply(this,arguments);e&&m(e.promise)?e.promise().progress(r.notify).done(r.resolve).fail(r.reject):r[t[0]+"With"](this,n?[e]:arguments)})}),i=null}).promise()},then:function(t,n,r){var u=0;function l(i,o,a,s){return function(){var n=this,r=arguments,e=function(){var e,t;if(!(i<u)){if((e=a.apply(n,r))===o.promise())throw new TypeError("Thenable self-resolution");t=e&&("object"==typeof e||"function"==typeof e)&&e.then,m(t)?s?t.call(e,l(u,o,R,s),l(u,o,M,s)):(u++,t.call(e,l(u,o,R,s),l(u,o,M,s),l(u,o,R,o.notifyWith))):(a!==R&&(n=void 0,r=[e]),(s||o.resolveWith)(n,r))}},t=s?e:function(){try{e()}catch(e){S.Deferred.exceptionHook&&S.Deferred.exceptionHook(e,t.stackTrace),u<=i+1&&(a!==M&&(n=void 0,r=[e]),o.rejectWith(n,r))}};i?t():(S.Deferred.getStackHook&&(t.stackTrace=S.Deferred.getStackHook()),C.setTimeout(t))}}return S.Deferred(function(e){o[0][3].add(l(0,e,m(r)?r:R,e.notifyWith)),o[1][3].add(l(0,e,m(t)?t:R)),o[2][3].add(l(0,e,m(n)?n:M))}).promise()},promise:function(e){return null!=e?S.extend(e,a):a}},s={};return S.each(o,function(e,t){var n=t[2],r=t[5];a[t[1]]=n.add,r&&n.add(function(){i=r},o[3-e][2].disable,o[3-e][3].disable,o[0][2].lock,o[0][3].lock),n.add(t[3].fire),s[t[0]]=function(){return s[t[0]+"With"](this===s?void 0:this,arguments),this},s[t[0]+"With"]=n.fireWith}),a.promise(s),e&&e.call(s,s),s},when:function(e){var n=arguments.length,t=n,r=Array(t),i=s.call(arguments),o=S.Deferred(),a=function(t){return function(e){r[t]=this,i[t]=1<arguments.length?s.call(arguments):e,--n||o.resolveWith(r,i)}};if(n<=1&&(I(e,o.done(a(t)).resolve,o.reject,!n),"pending"===o.state()||m(i[t]&&i[t].then)))return o.then();while(t--)I(i[t],a(t),o.reject);return o.promise()}});var W=/^(Eval|Internal|Range|Reference|Syntax|Type|URI)Error$/;S.Deferred.exceptionHook=function(e,t){C.console&&C.console.warn&&e&&W.test(e.name)&&C.console.warn("jQuery.Deferred exception: "+e.message,e.stack,t)},S.readyException=function(e){C.setTimeout(function(){throw e})};var F=S.Deferred();function B(){E.removeEventListener("DOMContentLoaded",B),C.removeEventListener("load",B),S.ready()}S.fn.ready=function(e){return F.then(e)["catch"](function(e){S.readyException(e)}),this},S.extend({isReady:!1,readyWait:1,ready:function(e){(!0===e?--S.readyWait:S.isReady)||(S.isReady=!0)!==e&&0<--S.readyWait||F.resolveWith(E,[S])}}),S.ready.then=F.then,"complete"===E.readyState||"loading"!==E.readyState&&!E.documentElement.doScroll?C.setTimeout(S.ready):(E.addEventListener("DOMContentLoaded",B),C.addEventListener("load",B));var $=function(e,t,n,r,i,o,a){var s=0,u=e.length,l=null==n;if("object"===w(n))for(s in i=!0,n)$(e,t,s,n[s],!0,o,a);else if(void 0!==r&&(i=!0,m(r)||(a=!0),l&&(a?(t.call(e,r),t=null):(l=t,t=function(e,t,n){return l.call(S(e),n)})),t))for(;s<u;s++)t(e[s],n,a?r:r.call(e[s],s,t(e[s],n)));return i?e:l?t.call(e):u?t(e[0],n):o},_=/^-ms-/,z=/-([a-z])/g;function U(e,t){return t.toUpperCase()}function X(e){return e.replace(_,"ms-").replace(z,U)}var V=function(e){return 1===e.nodeType||9===e.nodeType||!+e.nodeType};function G(){this.expando=S.expando+G.uid++}G.uid=1,G.prototype={cache:function(e){var t=e[this.expando];return t||(t={},V(e)&&(e.nodeType?e[this.expando]=t:Object.defineProperty(e,this.expando,{value:t,configurable:!0}))),t},set:function(e,t,n){var r,i=this.cache(e);if("string"==typeof t)i[X(t)]=n;else for(r in t)i[X(r)]=t[r];return i},get:function(e,t){return void 0===t?this.cache(e):e[this.expando]&&e[this.expando][X(t)]},access:function(e,t,n){return void 0===t||t&&"string"==typeof t&&void 0===n?this.get(e,t):(this.set(e,t,n),void 0!==n?n:t)},remove:function(e,t){var n,r=e[this.expando];if(void 0!==r){if(void 0!==t){n=(t=Array.isArray(t)?t.map(X):(t=X(t))in r?[t]:t.match(P)||[]).length;while(n--)delete r[t[n]]}(void 0===t||S.isEmptyObject(r))&&(e.nodeType?e[this.expando]=void 0:delete e[this.expando])}},hasData:function(e){var t=e[this.expando];return void 0!==t&&!S.isEmptyObject(t)}};var Y=new G,Q=new G,J=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,K=/[A-Z]/g;function Z(e,t,n){var r,i;if(void 0===n&&1===e.nodeType)if(r="data-"+t.replace(K,"-$&").toLowerCase(),"string"==typeof(n=e.getAttribute(r))){try{n="true"===(i=n)||"false"!==i&&("null"===i?null:i===+i+""?+i:J.test(i)?JSON.parse(i):i)}catch(e){}Q.set(e,t,n)}else n=void 0;return n}S.extend({hasData:function(e){return Q.hasData(e)||Y.hasData(e)},data:function(e,t,n){return Q.access(e,t,n)},removeData:function(e,t){Q.remove(e,t)},_data:function(e,t,n){return Y.access(e,t,n)},_removeData:function(e,t){Y.remove(e,t)}}),S.fn.extend({data:function(n,e){var t,r,i,o=this[0],a=o&&o.attributes;if(void 0===n){if(this.length&&(i=Q.get(o),1===o.nodeType&&!Y.get(o,"hasDataAttrs"))){t=a.length;while(t--)a[t]&&0===(r=a[t].name).indexOf("data-")&&(r=X(r.slice(5)),Z(o,r,i[r]));Y.set(o,"hasDataAttrs",!0)}return i}return"object"==typeof n?this.each(function(){Q.set(this,n)}):$(this,function(e){var t;if(o&&void 0===e)return void 0!==(t=Q.get(o,n))?t:void 0!==(t=Z(o,n))?t:void 0;this.each(function(){Q.set(this,n,e)})},null,e,1<arguments.length,null,!0)},removeData:function(e){return this.each(function(){Q.remove(this,e)})}}),S.extend({queue:function(e,t,n){var r;if(e)return t=(t||"fx")+"queue",r=Y.get(e,t),n&&(!r||Array.isArray(n)?r=Y.access(e,t,S.makeArray(n)):r.push(n)),r||[]},dequeue:function(e,t){t=t||"fx";var n=S.queue(e,t),r=n.length,i=n.shift(),o=S._queueHooks(e,t);"inprogress"===i&&(i=n.shift(),r--),i&&("fx"===t&&n.unshift("inprogress"),delete o.stop,i.call(e,function(){S.dequeue(e,t)},o)),!r&&o&&o.empty.fire()},_queueHooks:function(e,t){var n=t+"queueHooks";return Y.get(e,n)||Y.access(e,n,{empty:S.Callbacks("once memory").add(function(){Y.remove(e,[t+"queue",n])})})}}),S.fn.extend({queue:function(t,n){var e=2;return"string"!=typeof t&&(n=t,t="fx",e--),arguments.length<e?S.queue(this[0],t):void 0===n?this:this.each(function(){var e=S.queue(this,t,n);S._queueHooks(this,t),"fx"===t&&"inprogress"!==e[0]&&S.dequeue(this,t)})},dequeue:function(e){return this.each(function(){S.dequeue(this,e)})},clearQueue:function(e){return this.queue(e||"fx",[])},promise:function(e,t){var n,r=1,i=S.Deferred(),o=this,a=this.length,s=function(){--r||i.resolveWith(o,[o])};"string"!=typeof e&&(t=e,e=void 0),e=e||"fx";while(a--)(n=Y.get(o[a],e+"queueHooks"))&&n.empty&&(r++,n.empty.add(s));return s(),i.promise(t)}});var ee=/[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/.source,te=new RegExp("^(?:([+-])=|)("+ee+")([a-z%]*)$","i"),ne=["Top","Right","Bottom","Left"],re=E.documentElement,ie=function(e){return S.contains(e.ownerDocument,e)},oe={composed:!0};re.getRootNode&&(ie=function(e){return S.contains(e.ownerDocument,e)||e.getRootNode(oe)===e.ownerDocument});var ae=function(e,t){return"none"===(e=t||e).style.display||""===e.style.display&&ie(e)&&"none"===S.css(e,"display")};function se(e,t,n,r){var i,o,a=20,s=r?function(){return r.cur()}:function(){return S.css(e,t,"")},u=s(),l=n&&n[3]||(S.cssNumber[t]?"":"px"),c=e.nodeType&&(S.cssNumber[t]||"px"!==l&&+u)&&te.exec(S.css(e,t));if(c&&c[3]!==l){u/=2,l=l||c[3],c=+u||1;while(a--)S.style(e,t,c+l),(1-o)*(1-(o=s()/u||.5))<=0&&(a=0),c/=o;c*=2,S.style(e,t,c+l),n=n||[]}return n&&(c=+c||+u||0,i=n[1]?c+(n[1]+1)*n[2]:+n[2],r&&(r.unit=l,r.start=c,r.end=i)),i}var ue={};function le(e,t){for(var n,r,i,o,a,s,u,l=[],c=0,f=e.length;c<f;c++)(r=e[c]).style&&(n=r.style.display,t?("none"===n&&(l[c]=Y.get(r,"display")||null,l[c]||(r.style.display="")),""===r.style.display&&ae(r)&&(l[c]=(u=a=o=void 0,a=(i=r).ownerDocument,s=i.nodeName,(u=ue[s])||(o=a.body.appendChild(a.createElement(s)),u=S.css(o,"display"),o.parentNode.removeChild(o),"none"===u&&(u="block"),ue[s]=u)))):"none"!==n&&(l[c]="none",Y.set(r,"display",n)));for(c=0;c<f;c++)null!=l[c]&&(e[c].style.display=l[c]);return e}S.fn.extend({show:function(){return le(this,!0)},hide:function(){return le(this)},toggle:function(e){return"boolean"==typeof e?e?this.show():this.hide():this.each(function(){ae(this)?S(this).show():S(this).hide()})}});var ce,fe,pe=/^(?:checkbox|radio)$/i,de=/<([a-z][^\/\0>\x20\t\r\n\f]*)/i,he=/^$|^module$|\/(?:java|ecma)script/i;ce=E.createDocumentFragment().appendChild(E.createElement("div")),(fe=E.createElement("input")).setAttribute("type","radio"),fe.setAttribute("checked","checked"),fe.setAttribute("name","t"),ce.appendChild(fe),y.checkClone=ce.cloneNode(!0).cloneNode(!0).lastChild.checked,ce.innerHTML="<textarea>x</textarea>",y.noCloneChecked=!!ce.cloneNode(!0).lastChild.defaultValue,ce.innerHTML="<option></option>",y.option=!!ce.lastChild;var ge={thead:[1,"<table>","</table>"],col:[2,"<table><colgroup>","</colgroup></table>"],tr:[2,"<table><tbody>","</tbody></table>"],td:[3,"<table><tbody><tr>","</tr></tbody></table>"],_default:[0,"",""]};function ve(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&A(e,t)?S.merge([e],n):n}function ye(e,t){for(var n=0,r=e.length;n<r;n++)Y.set(e[n],"globalEval",!t||Y.get(t[n],"globalEval"))}ge.tbody=ge.tfoot=ge.colgroup=ge.caption=ge.thead,ge.th=ge.td,y.option||(ge.optgroup=ge.option=[1,"<select multiple='multiple'>","</select>"]);var me=/<|&#?\w+;/;function xe(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),p=[],d=0,h=e.length;d<h;d++)if((o=e[d])||0===o)if("object"===w(o))S.merge(p,o.nodeType?[o]:o);else if(me.test(o)){a=a||f.appendChild(t.createElement("div")),s=(de.exec(o)||["",""])[1].toLowerCase(),u=ge[s]||ge._default,a.innerHTML=u[1]+S.htmlPrefilter(o)+u[2],c=u[0];while(c--)a=a.lastChild;S.merge(p,a.childNodes),(a=f.firstChild).textContent=""}else p.push(t.createTextNode(o));f.textContent="",d=0;while(o=p[d++])if(r&&-1<S.inArray(o,r))i&&i.push(o);else if(l=ie(o),a=ve(f.appendChild(o),"script"),l&&ye(a),n){c=0;while(o=a[c++])he.test(o.type||"")&&n.push(o)}return f}var be=/^key/,we=/^(?:mouse|pointer|contextmenu|drag|drop)|click/,Te=/^([^.]*)(?:\.(.+)|)/;function Ce(){return!0}function Ee(){return!1}function Se(e,t){return e===function(){try{return E.activeElement}catch(e){}}()==("focus"===t)}function ke(e,t,n,r,i,o){var a,s;if("object"==typeof t){for(s in"string"!=typeof n&&(r=r||n,n=void 0),t)ke(e,s,n,r,t[s],o);return e}if(null==r&&null==i?(i=n,r=n=void 0):null==i&&("string"==typeof n?(i=r,r=void 0):(i=r,r=n,n=void 0)),!1===i)i=Ee;else if(!i)return e;return 1===o&&(a=i,(i=function(e){return S().off(e),a.apply(this,arguments)}).guid=a.guid||(a.guid=S.guid++)),e.each(function(){S.event.add(this,t,i,r,n)})}function Ae(e,i,o){o?(Y.set(e,i,!1),S.event.add(e,i,{namespace:!1,handler:function(e){var t,n,r=Y.get(this,i);if(1&e.isTrigger&&this[i]){if(r.length)(S.event.special[i]||{}).delegateType&&e.stopPropagation();else if(r=s.call(arguments),Y.set(this,i,r),t=o(this,i),this[i](),r!==(n=Y.get(this,i))||t?Y.set(this,i,!1):n={},r!==n)return e.stopImmediatePropagation(),e.preventDefault(),n.value}else r.length&&(Y.set(this,i,{value:S.event.trigger(S.extend(r[0],S.Event.prototype),r.slice(1),this)}),e.stopImmediatePropagation())}})):void 0===Y.get(e,i)&&S.event.add(e,i,Ce)}S.event={global:{},add:function(t,e,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Y.get(t);if(V(t)){n.handler&&(n=(o=n).handler,i=o.selector),i&&S.find.matchesSelector(re,i),n.guid||(n.guid=S.guid++),(u=v.events)||(u=v.events=Object.create(null)),(a=v.handle)||(a=v.handle=function(e){return"undefined"!=typeof S&&S.event.triggered!==e.type?S.event.dispatch.apply(t,arguments):void 0}),l=(e=(e||"").match(P)||[""]).length;while(l--)d=g=(s=Te.exec(e[l])||[])[1],h=(s[2]||"").split(".").sort(),d&&(f=S.event.special[d]||{},d=(i?f.delegateType:f.bindType)||d,f=S.event.special[d]||{},c=S.extend({type:d,origType:g,data:r,handler:n,guid:n.guid,selector:i,needsContext:i&&S.expr.match.needsContext.test(i),namespace:h.join(".")},o),(p=u[d])||((p=u[d]=[]).delegateCount=0,f.setup&&!1!==f.setup.call(t,r,h,a)||t.addEventListener&&t.addEventListener(d,a)),f.add&&(f.add.call(t,c),c.handler.guid||(c.handler.guid=n.guid)),i?p.splice(p.delegateCount++,0,c):p.push(c),S.event.global[d]=!0)}},remove:function(e,t,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Y.hasData(e)&&Y.get(e);if(v&&(u=v.events)){l=(t=(t||"").match(P)||[""]).length;while(l--)if(d=g=(s=Te.exec(t[l])||[])[1],h=(s[2]||"").split(".").sort(),d){f=S.event.special[d]||{},p=u[d=(r?f.delegateType:f.bindType)||d]||[],s=s[2]&&new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"),a=o=p.length;while(o--)c=p[o],!i&&g!==c.origType||n&&n.guid!==c.guid||s&&!s.test(c.namespace)||r&&r!==c.selector&&("**"!==r||!c.selector)||(p.splice(o,1),c.selector&&p.delegateCount--,f.remove&&f.remove.call(e,c));a&&!p.length&&(f.teardown&&!1!==f.teardown.call(e,h,v.handle)||S.removeEvent(e,d,v.handle),delete u[d])}else for(d in u)S.event.remove(e,d+t[l],n,r,!0);S.isEmptyObject(u)&&Y.remove(e,"handle events")}},dispatch:function(e){var t,n,r,i,o,a,s=new Array(arguments.length),u=S.event.fix(e),l=(Y.get(this,"events")||Object.create(null))[u.type]||[],c=S.event.special[u.type]||{};for(s[0]=u,t=1;t<arguments.length;t++)s[t]=arguments[t];if(u.delegateTarget=this,!c.preDispatch||!1!==c.preDispatch.call(this,u)){a=S.event.handlers.call(this,u,l),t=0;while((i=a[t++])&&!u.isPropagationStopped()){u.currentTarget=i.elem,n=0;while((o=i.handlers[n++])&&!u.isImmediatePropagationStopped())u.rnamespace&&!1!==o.namespace&&!u.rnamespace.test(o.namespace)||(u.handleObj=o,u.data=o.data,void 0!==(r=((S.event.special[o.origType]||{}).handle||o.handler).apply(i.elem,s))&&!1===(u.result=r)&&(u.preventDefault(),u.stopPropagation()))}return c.postDispatch&&c.postDispatch.call(this,u),u.result}},handlers:function(e,t){var n,r,i,o,a,s=[],u=t.delegateCount,l=e.target;if(u&&l.nodeType&&!("click"===e.type&&1<=e.button))for(;l!==this;l=l.parentNode||this)if(1===l.nodeType&&("click"!==e.type||!0!==l.disabled)){for(o=[],a={},n=0;n<u;n++)void 0===a[i=(r=t[n]).selector+" "]&&(a[i]=r.needsContext?-1<S(i,this).index(l):S.find(i,this,null,[l]).length),a[i]&&o.push(r);o.length&&s.push({elem:l,handlers:o})}return l=this,u<t.length&&s.push({elem:l,handlers:t.slice(u)}),s},addProp:function(t,e){Object.defineProperty(S.Event.prototype,t,{enumerable:!0,configurable:!0,get:m(e)?function(){if(this.originalEvent)return e(this.originalEvent)}:function(){if(this.originalEvent)return this.originalEvent[t]},set:function(e){Object.defineProperty(this,t,{enumerable:!0,configurable:!0,writable:!0,value:e})}})},fix:function(e){return e[S.expando]?e:new S.Event(e)},special:{load:{noBubble:!0},click:{setup:function(e){var t=this||e;return pe.test(t.type)&&t.click&&A(t,"input")&&Ae(t,"click",Ce),!1},trigger:function(e){var t=this||e;return pe.test(t.type)&&t.click&&A(t,"input")&&Ae(t,"click"),!0},_default:function(e){var t=e.target;return pe.test(t.type)&&t.click&&A(t,"input")&&Y.get(t,"click")||A(t,"a")}},beforeunload:{postDispatch:function(e){void 0!==e.result&&e.originalEvent&&(e.originalEvent.returnValue=e.result)}}}},S.removeEvent=function(e,t,n){e.removeEventListener&&e.removeEventListener(t,n)},S.Event=function(e,t){if(!(this instanceof S.Event))return new S.Event(e,t);e&&e.type?(this.originalEvent=e,this.type=e.type,this.isDefaultPrevented=e.defaultPrevented||void 0===e.defaultPrevented&&!1===e.returnValue?Ce:Ee,this.target=e.target&&3===e.target.nodeType?e.target.parentNode:e.target,this.currentTarget=e.currentTarget,this.relatedTarget=e.relatedTarget):this.type=e,t&&S.extend(this,t),this.timeStamp=e&&e.timeStamp||Date.now(),this[S.expando]=!0},S.Event.prototype={constructor:S.Event,isDefaultPrevented:Ee,isPropagationStopped:Ee,isImmediatePropagationStopped:Ee,isSimulated:!1,preventDefault:function(){var e=this.originalEvent;this.isDefaultPrevented=Ce,e&&!this.isSimulated&&e.preventDefault()},stopPropagation:function(){var e=this.originalEvent;this.isPropagationStopped=Ce,e&&!this.isSimulated&&e.stopPropagation()},stopImmediatePropagation:function(){var e=this.originalEvent;this.isImmediatePropagationStopped=Ce,e&&!this.isSimulated&&e.stopImmediatePropagation(),this.stopPropagation()}},S.each({altKey:!0,bubbles:!0,cancelable:!0,changedTouches:!0,ctrlKey:!0,detail:!0,eventPhase:!0,metaKey:!0,pageX:!0,pageY:!0,shiftKey:!0,view:!0,"char":!0,code:!0,charCode:!0,key:!0,keyCode:!0,button:!0,buttons:!0,clientX:!0,clientY:!0,offsetX:!0,offsetY:!0,pointerId:!0,pointerType:!0,screenX:!0,screenY:!0,targetTouches:!0,toElement:!0,touches:!0,which:function(e){var t=e.button;return null==e.which&&be.test(e.type)?null!=e.charCode?e.charCode:e.keyCode:!e.which&&void 0!==t&&we.test(e.type)?1&t?1:2&t?3:4&t?2:0:e.which}},S.event.addProp),S.each({focus:"focusin",blur:"focusout"},function(e,t){S.event.special[e]={setup:function(){return Ae(this,e,Se),!1},trigger:function(){return Ae(this,e),!0},delegateType:t}}),S.each({mouseenter:"mouseover",mouseleave:"mouseout",pointerenter:"pointerover",pointerleave:"pointerout"},function(e,i){S.event.special[e]={delegateType:i,bindType:i,handle:function(e){var t,n=e.relatedTarget,r=e.handleObj;return n&&(n===this||S.contains(this,n))||(e.type=r.origType,t=r.handler.apply(this,arguments),e.type=i),t}}}),S.fn.extend({on:function(e,t,n,r){return ke(this,e,t,n,r)},one:function(e,t,n,r){return ke(this,e,t,n,r,1)},off:function(e,t,n){var r,i;if(e&&e.preventDefault&&e.handleObj)return r=e.handleObj,S(e.delegateTarget).off(r.namespace?r.origType+"."+r.namespace:r.origType,r.selector,r.handler),this;if("object"==typeof e){for(i in e)this.off(i,t,e[i]);return this}return!1!==t&&"function"!=typeof t||(n=t,t=void 0),!1===n&&(n=Ee),this.each(function(){S.event.remove(this,e,n,t)})}});var Ne=/<script|<style|<link/i,De=/checked\s*(?:[^=]|=\s*.checked.)/i,je=/^\s*<!(?:\[CDATA\[|--)|(?:\]\]|--)>\s*$/g;function qe(e,t){return A(e,"table")&&A(11!==t.nodeType?t:t.firstChild,"tr")&&S(e).children("tbody")[0]||e}function Le(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function He(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Oe(e,t){var n,r,i,o,a,s;if(1===t.nodeType){if(Y.hasData(e)&&(s=Y.get(e).events))for(i in Y.remove(t,"handle events"),s)for(n=0,r=s[i].length;n<r;n++)S.event.add(t,i,s[i][n]);Q.hasData(e)&&(o=Q.access(e),a=S.extend({},o),Q.set(t,a))}}function Pe(n,r,i,o){r=g(r);var e,t,a,s,u,l,c=0,f=n.length,p=f-1,d=r[0],h=m(d);if(h||1<f&&"string"==typeof d&&!y.checkClone&&De.test(d))return n.each(function(e){var t=n.eq(e);h&&(r[0]=d.call(this,e,t.html())),Pe(t,r,i,o)});if(f&&(t=(e=xe(r,n[0].ownerDocument,!1,n,o)).firstChild,1===e.childNodes.length&&(e=t),t||o)){for(s=(a=S.map(ve(e,"script"),Le)).length;c<f;c++)u=e,c!==p&&(u=S.clone(u,!0,!0),s&&S.merge(a,ve(u,"script"))),i.call(n[c],u,c);if(s)for(l=a[a.length-1].ownerDocument,S.map(a,He),c=0;c<s;c++)u=a[c],he.test(u.type||"")&&!Y.access(u,"globalEval")&&S.contains(l,u)&&(u.src&&"module"!==(u.type||"").toLowerCase()?S._evalUrl&&!u.noModule&&S._evalUrl(u.src,{nonce:u.nonce||u.getAttribute("nonce")},l):b(u.textContent.replace(je,""),u,l))}return n}function Re(e,t,n){for(var r,i=t?S.filter(t,e):e,o=0;null!=(r=i[o]);o++)n||1!==r.nodeType||S.cleanData(ve(r)),r.parentNode&&(n&&ie(r)&&ye(ve(r,"script")),r.parentNode.removeChild(r));return e}S.extend({htmlPrefilter:function(e){return e},clone:function(e,t,n){var r,i,o,a,s,u,l,c=e.cloneNode(!0),f=ie(e);if(!(y.noCloneChecked||1!==e.nodeType&&11!==e.nodeType||S.isXMLDoc(e)))for(a=ve(c),r=0,i=(o=ve(e)).length;r<i;r++)s=o[r],u=a[r],void 0,"input"===(l=u.nodeName.toLowerCase())&&pe.test(s.type)?u.checked=s.checked:"input"!==l&&"textarea"!==l||(u.defaultValue=s.defaultValue);if(t)if(n)for(o=o||ve(e),a=a||ve(c),r=0,i=o.length;r<i;r++)Oe(o[r],a[r]);else Oe(e,c);return 0<(a=ve(c,"script")).length&&ye(a,!f&&ve(e,"script")),c},cleanData:function(e){for(var t,n,r,i=S.event.special,o=0;void 0!==(n=e[o]);o++)if(V(n)){if(t=n[Y.expando]){if(t.events)for(r in t.events)i[r]?S.event.remove(n,r):S.removeEvent(n,r,t.handle);n[Y.expando]=void 0}n[Q.expando]&&(n[Q.expando]=void 0)}}}),S.fn.extend({detach:function(e){return Re(this,e,!0)},remove:function(e){return Re(this,e)},text:function(e){return $(this,function(e){return void 0===e?S.text(this):this.empty().each(function(){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||(this.textContent=e)})},null,e,arguments.length)},append:function(){return Pe(this,arguments,function(e){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||qe(this,e).appendChild(e)})},prepend:function(){return Pe(this,arguments,function(e){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var t=qe(this,e);t.insertBefore(e,t.firstChild)}})},before:function(){return Pe(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this)})},after:function(){return Pe(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this.nextSibling)})},empty:function(){for(var e,t=0;null!=(e=this[t]);t++)1===e.nodeType&&(S.cleanData(ve(e,!1)),e.textContent="");return this},clone:function(e,t){return e=null!=e&&e,t=null==t?e:t,this.map(function(){return S.clone(this,e,t)})},html:function(e){return $(this,function(e){var t=this[0]||{},n=0,r=this.length;if(void 0===e&&1===t.nodeType)return t.innerHTML;if("string"==typeof e&&!Ne.test(e)&&!ge[(de.exec(e)||["",""])[1].toLowerCase()]){e=S.htmlPrefilter(e);try{for(;n<r;n++)1===(t=this[n]||{}).nodeType&&(S.cleanData(ve(t,!1)),t.innerHTML=e);t=0}catch(e){}}t&&this.empty().append(e)},null,e,arguments.length)},replaceWith:function(){var n=[];return Pe(this,arguments,function(e){var t=this.parentNode;S.inArray(this,n)<0&&(S.cleanData(ve(this)),t&&t.replaceChild(e,this))},n)}}),S.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(e,a){S.fn[e]=function(e){for(var t,n=[],r=S(e),i=r.length-1,o=0;o<=i;o++)t=o===i?this:this.clone(!0),S(r[o])[a](t),u.apply(n,t.get());return this.pushStack(n)}});var Me=new RegExp("^("+ee+")(?!px)[a-z%]+$","i"),Ie=function(e){var t=e.ownerDocument.defaultView;return t&&t.opener||(t=C),t.getComputedStyle(e)},We=function(e,t,n){var r,i,o={};for(i in t)o[i]=e.style[i],e.style[i]=t[i];for(i in r=n.call(e),t)e.style[i]=o[i];return r},Fe=new RegExp(ne.join("|"),"i");function Be(e,t,n){var r,i,o,a,s=e.style;return(n=n||Ie(e))&&(""!==(a=n.getPropertyValue(t)||n[t])||ie(e)||(a=S.style(e,t)),!y.pixelBoxStyles()&&Me.test(a)&&Fe.test(t)&&(r=s.width,i=s.minWidth,o=s.maxWidth,s.minWidth=s.maxWidth=s.width=a,a=n.width,s.width=r,s.minWidth=i,s.maxWidth=o)),void 0!==a?a+"":a}function $e(e,t){return{get:function(){if(!e())return(this.get=t).apply(this,arguments);delete this.get}}}!function(){function e(){if(l){u.style.cssText="position:absolute;left:-11111px;width:60px;margin-top:1px;padding:0;border:0",l.style.cssText="position:relative;display:block;box-sizing:border-box;overflow:scroll;margin:auto;border:1px;padding:1px;width:60%;top:1%",re.appendChild(u).appendChild(l);var e=C.getComputedStyle(l);n="1%"!==e.top,s=12===t(e.marginLeft),l.style.right="60%",o=36===t(e.right),r=36===t(e.width),l.style.position="absolute",i=12===t(l.offsetWidth/3),re.removeChild(u),l=null}}function t(e){return Math.round(parseFloat(e))}var n,r,i,o,a,s,u=E.createElement("div"),l=E.createElement("div");l.style&&(l.style.backgroundClip="content-box",l.cloneNode(!0).style.backgroundClip="",y.clearCloneStyle="content-box"===l.style.backgroundClip,S.extend(y,{boxSizingReliable:function(){return e(),r},pixelBoxStyles:function(){return e(),o},pixelPosition:function(){return e(),n},reliableMarginLeft:function(){return e(),s},scrollboxSize:function(){return e(),i},reliableTrDimensions:function(){var e,t,n,r;return null==a&&(e=E.createElement("table"),t=E.createElement("tr"),n=E.createElement("div"),e.style.cssText="position:absolute;left:-11111px",t.style.height="1px",n.style.height="9px",re.appendChild(e).appendChild(t).appendChild(n),r=C.getComputedStyle(t),a=3<parseInt(r.height),re.removeChild(e)),a}}))}();var _e=["Webkit","Moz","ms"],ze=E.createElement("div").style,Ue={};function Xe(e){var t=S.cssProps[e]||Ue[e];return t||(e in ze?e:Ue[e]=function(e){var t=e[0].toUpperCase()+e.slice(1),n=_e.length;while(n--)if((e=_e[n]+t)in ze)return e}(e)||e)}var Ve=/^(none|table(?!-c[ea]).+)/,Ge=/^--/,Ye={position:"absolute",visibility:"hidden",display:"block"},Qe={letterSpacing:"0",fontWeight:"400"};function Je(e,t,n){var r=te.exec(t);return r?Math.max(0,r[2]-(n||0))+(r[3]||"px"):t}function Ke(e,t,n,r,i,o){var a="width"===t?1:0,s=0,u=0;if(n===(r?"border":"content"))return 0;for(;a<4;a+=2)"margin"===n&&(u+=S.css(e,n+ne[a],!0,i)),r?("content"===n&&(u-=S.css(e,"padding"+ne[a],!0,i)),"margin"!==n&&(u-=S.css(e,"border"+ne[a]+"Width",!0,i))):(u+=S.css(e,"padding"+ne[a],!0,i),"padding"!==n?u+=S.css(e,"border"+ne[a]+"Width",!0,i):s+=S.css(e,"border"+ne[a]+"Width",!0,i));return!r&&0<=o&&(u+=Math.max(0,Math.ceil(e["offset"+t[0].toUpperCase()+t.slice(1)]-o-u-s-.5))||0),u}function Ze(e,t,n){var r=Ie(e),i=(!y.boxSizingReliable()||n)&&"border-box"===S.css(e,"boxSizing",!1,r),o=i,a=Be(e,t,r),s="offset"+t[0].toUpperCase()+t.slice(1);if(Me.test(a)){if(!n)return a;a="auto"}return(!y.boxSizingReliable()&&i||!y.reliableTrDimensions()&&A(e,"tr")||"auto"===a||!parseFloat(a)&&"inline"===S.css(e,"display",!1,r))&&e.getClientRects().length&&(i="border-box"===S.css(e,"boxSizing",!1,r),(o=s in e)&&(a=e[s])),(a=parseFloat(a)||0)+Ke(e,t,n||(i?"border":"content"),o,r,a)+"px"}function et(e,t,n,r,i){return new et.prototype.init(e,t,n,r,i)}S.extend({cssHooks:{opacity:{get:function(e,t){if(t){var n=Be(e,"opacity");return""===n?"1":n}}}},cssNumber:{animationIterationCount:!0,columnCount:!0,fillOpacity:!0,flexGrow:!0,flexShrink:!0,fontWeight:!0,gridArea:!0,gridColumn:!0,gridColumnEnd:!0,gridColumnStart:!0,gridRow:!0,gridRowEnd:!0,gridRowStart:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{},style:function(e,t,n,r){if(e&&3!==e.nodeType&&8!==e.nodeType&&e.style){var i,o,a,s=X(t),u=Ge.test(t),l=e.style;if(u||(t=Xe(s)),a=S.cssHooks[t]||S.cssHooks[s],void 0===n)return a&&"get"in a&&void 0!==(i=a.get(e,!1,r))?i:l[t];"string"===(o=typeof n)&&(i=te.exec(n))&&i[1]&&(n=se(e,t,i),o="number"),null!=n&&n==n&&("number"!==o||u||(n+=i&&i[3]||(S.cssNumber[s]?"":"px")),y.clearCloneStyle||""!==n||0!==t.indexOf("background")||(l[t]="inherit"),a&&"set"in a&&void 0===(n=a.set(e,n,r))||(u?l.setProperty(t,n):l[t]=n))}},css:function(e,t,n,r){var i,o,a,s=X(t);return Ge.test(t)||(t=Xe(s)),(a=S.cssHooks[t]||S.cssHooks[s])&&"get"in a&&(i=a.get(e,!0,n)),void 0===i&&(i=Be(e,t,r)),"normal"===i&&t in Qe&&(i=Qe[t]),""===n||n?(o=parseFloat(i),!0===n||isFinite(o)?o||0:i):i}}),S.each(["height","width"],function(e,u){S.cssHooks[u]={get:function(e,t,n){if(t)return!Ve.test(S.css(e,"display"))||e.getClientRects().length&&e.getBoundingClientRect().width?Ze(e,u,n):We(e,Ye,function(){return Ze(e,u,n)})},set:function(e,t,n){var r,i=Ie(e),o=!y.scrollboxSize()&&"absolute"===i.position,a=(o||n)&&"border-box"===S.css(e,"boxSizing",!1,i),s=n?Ke(e,u,n,a,i):0;return a&&o&&(s-=Math.ceil(e["offset"+u[0].toUpperCase()+u.slice(1)]-parseFloat(i[u])-Ke(e,u,"border",!1,i)-.5)),s&&(r=te.exec(t))&&"px"!==(r[3]||"px")&&(e.style[u]=t,t=S.css(e,u)),Je(0,t,s)}}}),S.cssHooks.marginLeft=$e(y.reliableMarginLeft,function(e,t){if(t)return(parseFloat(Be(e,"marginLeft"))||e.getBoundingClientRect().left-We(e,{marginLeft:0},function(){return e.getBoundingClientRect().left}))+"px"}),S.each({margin:"",padding:"",border:"Width"},function(i,o){S.cssHooks[i+o]={expand:function(e){for(var t=0,n={},r="string"==typeof e?e.split(" "):[e];t<4;t++)n[i+ne[t]+o]=r[t]||r[t-2]||r[0];return n}},"margin"!==i&&(S.cssHooks[i+o].set=Je)}),S.fn.extend({css:function(e,t){return $(this,function(e,t,n){var r,i,o={},a=0;if(Array.isArray(t)){for(r=Ie(e),i=t.length;a<i;a++)o[t[a]]=S.css(e,t[a],!1,r);return o}return void 0!==n?S.style(e,t,n):S.css(e,t)},e,t,1<arguments.length)}}),((S.Tween=et).prototype={constructor:et,init:function(e,t,n,r,i,o){this.elem=e,this.prop=n,this.easing=i||S.easing._default,this.options=t,this.start=this.now=this.cur(),this.end=r,this.unit=o||(S.cssNumber[n]?"":"px")},cur:function(){var e=et.propHooks[this.prop];return e&&e.get?e.get(this):et.propHooks._default.get(this)},run:function(e){var t,n=et.propHooks[this.prop];return this.options.duration?this.pos=t=S.easing[this.easing](e,this.options.duration*e,0,1,this.options.duration):this.pos=t=e,this.now=(this.end-this.start)*t+this.start,this.options.step&&this.options.step.call(this.elem,this.now,this),n&&n.set?n.set(this):et.propHooks._default.set(this),this}}).init.prototype=et.prototype,(et.propHooks={_default:{get:function(e){var t;return 1!==e.elem.nodeType||null!=e.elem[e.prop]&&null==e.elem.style[e.prop]?e.elem[e.prop]:(t=S.css(e.elem,e.prop,""))&&"auto"!==t?t:0},set:function(e){S.fx.step[e.prop]?S.fx.step[e.prop](e):1!==e.elem.nodeType||!S.cssHooks[e.prop]&&null==e.elem.style[Xe(e.prop)]?e.elem[e.prop]=e.now:S.style(e.elem,e.prop,e.now+e.unit)}}}).scrollTop=et.propHooks.scrollLeft={set:function(e){e.elem.nodeType&&e.elem.parentNode&&(e.elem[e.prop]=e.now)}},S.easing={linear:function(e){return e},swing:function(e){return.5-Math.cos(e*Math.PI)/2},_default:"swing"},S.fx=et.prototype.init,S.fx.step={};var tt,nt,rt,it,ot=/^(?:toggle|show|hide)$/,at=/queueHooks$/;function st(){nt&&(!1===E.hidden&&C.requestAnimationFrame?C.requestAnimationFrame(st):C.setTimeout(st,S.fx.interval),S.fx.tick())}function ut(){return C.setTimeout(function(){tt=void 0}),tt=Date.now()}function lt(e,t){var n,r=0,i={height:e};for(t=t?1:0;r<4;r+=2-t)i["margin"+(n=ne[r])]=i["padding"+n]=e;return t&&(i.opacity=i.width=e),i}function ct(e,t,n){for(var r,i=(ft.tweeners[t]||[]).concat(ft.tweeners["*"]),o=0,a=i.length;o<a;o++)if(r=i[o].call(n,t,e))return r}function ft(o,e,t){var n,a,r=0,i=ft.prefilters.length,s=S.Deferred().always(function(){delete u.elem}),u=function(){if(a)return!1;for(var e=tt||ut(),t=Math.max(0,l.startTime+l.duration-e),n=1-(t/l.duration||0),r=0,i=l.tweens.length;r<i;r++)l.tweens[r].run(n);return s.notifyWith(o,[l,n,t]),n<1&&i?t:(i||s.notifyWith(o,[l,1,0]),s.resolveWith(o,[l]),!1)},l=s.promise({elem:o,props:S.extend({},e),opts:S.extend(!0,{specialEasing:{},easing:S.easing._default},t),originalProperties:e,originalOptions:t,startTime:tt||ut(),duration:t.duration,tweens:[],createTween:function(e,t){var n=S.Tween(o,l.opts,e,t,l.opts.specialEasing[e]||l.opts.easing);return l.tweens.push(n),n},stop:function(e){var t=0,n=e?l.tweens.length:0;if(a)return this;for(a=!0;t<n;t++)l.tweens[t].run(1);return e?(s.notifyWith(o,[l,1,0]),s.resolveWith(o,[l,e])):s.rejectWith(o,[l,e]),this}}),c=l.props;for(!function(e,t){var n,r,i,o,a;for(n in e)if(i=t[r=X(n)],o=e[n],Array.isArray(o)&&(i=o[1],o=e[n]=o[0]),n!==r&&(e[r]=o,delete e[n]),(a=S.cssHooks[r])&&"expand"in a)for(n in o=a.expand(o),delete e[r],o)n in e||(e[n]=o[n],t[n]=i);else t[r]=i}(c,l.opts.specialEasing);r<i;r++)if(n=ft.prefilters[r].call(l,o,c,l.opts))return m(n.stop)&&(S._queueHooks(l.elem,l.opts.queue).stop=n.stop.bind(n)),n;return S.map(c,ct,l),m(l.opts.start)&&l.opts.start.call(o,l),l.progress(l.opts.progress).done(l.opts.done,l.opts.complete).fail(l.opts.fail).always(l.opts.always),S.fx.timer(S.extend(u,{elem:o,anim:l,queue:l.opts.queue})),l}S.Animation=S.extend(ft,{tweeners:{"*":[function(e,t){var n=this.createTween(e,t);return se(n.elem,e,te.exec(t),n),n}]},tweener:function(e,t){m(e)?(t=e,e=["*"]):e=e.match(P);for(var n,r=0,i=e.length;r<i;r++)n=e[r],ft.tweeners[n]=ft.tweeners[n]||[],ft.tweeners[n].unshift(t)},prefilters:[function(e,t,n){var r,i,o,a,s,u,l,c,f="width"in t||"height"in t,p=this,d={},h=e.style,g=e.nodeType&&ae(e),v=Y.get(e,"fxshow");for(r in n.queue||(null==(a=S._queueHooks(e,"fx")).unqueued&&(a.unqueued=0,s=a.empty.fire,a.empty.fire=function(){a.unqueued||s()}),a.unqueued++,p.always(function(){p.always(function(){a.unqueued--,S.queue(e,"fx").length||a.empty.fire()})})),t)if(i=t[r],ot.test(i)){if(delete t[r],o=o||"toggle"===i,i===(g?"hide":"show")){if("show"!==i||!v||void 0===v[r])continue;g=!0}d[r]=v&&v[r]||S.style(e,r)}if((u=!S.isEmptyObject(t))||!S.isEmptyObject(d))for(r in f&&1===e.nodeType&&(n.overflow=[h.overflow,h.overflowX,h.overflowY],null==(l=v&&v.display)&&(l=Y.get(e,"display")),"none"===(c=S.css(e,"display"))&&(l?c=l:(le([e],!0),l=e.style.display||l,c=S.css(e,"display"),le([e]))),("inline"===c||"inline-block"===c&&null!=l)&&"none"===S.css(e,"float")&&(u||(p.done(function(){h.display=l}),null==l&&(c=h.display,l="none"===c?"":c)),h.display="inline-block")),n.overflow&&(h.overflow="hidden",p.always(function(){h.overflow=n.overflow[0],h.overflowX=n.overflow[1],h.overflowY=n.overflow[2]})),u=!1,d)u||(v?"hidden"in v&&(g=v.hidden):v=Y.access(e,"fxshow",{display:l}),o&&(v.hidden=!g),g&&le([e],!0),p.done(function(){for(r in g||le([e]),Y.remove(e,"fxshow"),d)S.style(e,r,d[r])})),u=ct(g?v[r]:0,r,p),r in v||(v[r]=u.start,g&&(u.end=u.start,u.start=0))}],prefilter:function(e,t){t?ft.prefilters.unshift(e):ft.prefilters.push(e)}}),S.speed=function(e,t,n){var r=e&&"object"==typeof e?S.extend({},e):{complete:n||!n&&t||m(e)&&e,duration:e,easing:n&&t||t&&!m(t)&&t};return S.fx.off?r.duration=0:"number"!=typeof r.duration&&(r.duration in S.fx.speeds?r.duration=S.fx.speeds[r.duration]:r.duration=S.fx.speeds._default),null!=r.queue&&!0!==r.queue||(r.queue="fx"),r.old=r.complete,r.complete=function(){m(r.old)&&r.old.call(this),r.queue&&S.dequeue(this,r.queue)},r},S.fn.extend({fadeTo:function(e,t,n,r){return this.filter(ae).css("opacity",0).show().end().animate({opacity:t},e,n,r)},animate:function(t,e,n,r){var i=S.isEmptyObject(t),o=S.speed(e,n,r),a=function(){var e=ft(this,S.extend({},t),o);(i||Y.get(this,"finish"))&&e.stop(!0)};return a.finish=a,i||!1===o.queue?this.each(a):this.queue(o.queue,a)},stop:function(i,e,o){var a=function(e){var t=e.stop;delete e.stop,t(o)};return"string"!=typeof i&&(o=e,e=i,i=void 0),e&&this.queue(i||"fx",[]),this.each(function(){var e=!0,t=null!=i&&i+"queueHooks",n=S.timers,r=Y.get(this);if(t)r[t]&&r[t].stop&&a(r[t]);else for(t in r)r[t]&&r[t].stop&&at.test(t)&&a(r[t]);for(t=n.length;t--;)n[t].elem!==this||null!=i&&n[t].queue!==i||(n[t].anim.stop(o),e=!1,n.splice(t,1));!e&&o||S.dequeue(this,i)})},finish:function(a){return!1!==a&&(a=a||"fx"),this.each(function(){var e,t=Y.get(this),n=t[a+"queue"],r=t[a+"queueHooks"],i=S.timers,o=n?n.length:0;for(t.finish=!0,S.queue(this,a,[]),r&&r.stop&&r.stop.call(this,!0),e=i.length;e--;)i[e].elem===this&&i[e].queue===a&&(i[e].anim.stop(!0),i.splice(e,1));for(e=0;e<o;e++)n[e]&&n[e].finish&&n[e].finish.call(this);delete t.finish})}}),S.each(["toggle","show","hide"],function(e,r){var i=S.fn[r];S.fn[r]=function(e,t,n){return null==e||"boolean"==typeof e?i.apply(this,arguments):this.animate(lt(r,!0),e,t,n)}}),S.each({slideDown:lt("show"),slideUp:lt("hide"),slideToggle:lt("toggle"),fadeIn:{opacity:"show"},fadeOut:{opacity:"hide"},fadeToggle:{opacity:"toggle"}},function(e,r){S.fn[e]=function(e,t,n){return this.animate(r,e,t,n)}}),S.timers=[],S.fx.tick=function(){var e,t=0,n=S.timers;for(tt=Date.now();t<n.length;t++)(e=n[t])()||n[t]!==e||n.splice(t--,1);n.length||S.fx.stop(),tt=void 0},S.fx.timer=function(e){S.timers.push(e),S.fx.start()},S.fx.interval=13,S.fx.start=function(){nt||(nt=!0,st())},S.fx.stop=function(){nt=null},S.fx.speeds={slow:600,fast:200,_default:400},S.fn.delay=function(r,e){return r=S.fx&&S.fx.speeds[r]||r,e=e||"fx",this.queue(e,function(e,t){var n=C.setTimeout(e,r);t.stop=function(){C.clearTimeout(n)}})},rt=E.createElement("input"),it=E.createElement("select").appendChild(E.createElement("option")),rt.type="checkbox",y.checkOn=""!==rt.value,y.optSelected=it.selected,(rt=E.createElement("input")).value="t",rt.type="radio",y.radioValue="t"===rt.value;var pt,dt=S.expr.attrHandle;S.fn.extend({attr:function(e,t){return $(this,S.attr,e,t,1<arguments.length)},removeAttr:function(e){return this.each(function(){S.removeAttr(this,e)})}}),S.extend({attr:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return"undefined"==typeof e.getAttribute?S.prop(e,t,n):(1===o&&S.isXMLDoc(e)||(i=S.attrHooks[t.toLowerCase()]||(S.expr.match.bool.test(t)?pt:void 0)),void 0!==n?null===n?void S.removeAttr(e,t):i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:(e.setAttribute(t,n+""),n):i&&"get"in i&&null!==(r=i.get(e,t))?r:null==(r=S.find.attr(e,t))?void 0:r)},attrHooks:{type:{set:function(e,t){if(!y.radioValue&&"radio"===t&&A(e,"input")){var n=e.value;return e.setAttribute("type",t),n&&(e.value=n),t}}}},removeAttr:function(e,t){var n,r=0,i=t&&t.match(P);if(i&&1===e.nodeType)while(n=i[r++])e.removeAttribute(n)}}),pt={set:function(e,t,n){return!1===t?S.removeAttr(e,n):e.setAttribute(n,n),n}},S.each(S.expr.match.bool.source.match(/\w+/g),function(e,t){var a=dt[t]||S.find.attr;dt[t]=function(e,t,n){var r,i,o=t.toLowerCase();return n||(i=dt[o],dt[o]=r,r=null!=a(e,t,n)?o:null,dt[o]=i),r}});var ht=/^(?:input|select|textarea|button)$/i,gt=/^(?:a|area)$/i;function vt(e){return(e.match(P)||[]).join(" ")}function yt(e){return e.getAttribute&&e.getAttribute("class")||""}function mt(e){return Array.isArray(e)?e:"string"==typeof e&&e.match(P)||[]}S.fn.extend({prop:function(e,t){return $(this,S.prop,e,t,1<arguments.length)},removeProp:function(e){return this.each(function(){delete this[S.propFix[e]||e]})}}),S.extend({prop:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return 1===o&&S.isXMLDoc(e)||(t=S.propFix[t]||t,i=S.propHooks[t]),void 0!==n?i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:e[t]=n:i&&"get"in i&&null!==(r=i.get(e,t))?r:e[t]},propHooks:{tabIndex:{get:function(e){var t=S.find.attr(e,"tabindex");return t?parseInt(t,10):ht.test(e.nodeName)||gt.test(e.nodeName)&&e.href?0:-1}}},propFix:{"for":"htmlFor","class":"className"}}),y.optSelected||(S.propHooks.selected={get:function(e){var t=e.parentNode;return t&&t.parentNode&&t.parentNode.selectedIndex,null},set:function(e){var t=e.parentNode;t&&(t.selectedIndex,t.parentNode&&t.parentNode.selectedIndex)}}),S.each(["tabIndex","readOnly","maxLength","cellSpacing","cellPadding","rowSpan","colSpan","useMap","frameBorder","contentEditable"],function(){S.propFix[this.toLowerCase()]=this}),S.fn.extend({addClass:function(t){var e,n,r,i,o,a,s,u=0;if(m(t))return this.each(function(e){S(this).addClass(t.call(this,e,yt(this)))});if((e=mt(t)).length)while(n=this[u++])if(i=yt(n),r=1===n.nodeType&&" "+vt(i)+" "){a=0;while(o=e[a++])r.indexOf(" "+o+" ")<0&&(r+=o+" ");i!==(s=vt(r))&&n.setAttribute("class",s)}return this},removeClass:function(t){var e,n,r,i,o,a,s,u=0;if(m(t))return this.each(function(e){S(this).removeClass(t.call(this,e,yt(this)))});if(!arguments.length)return this.attr("class","");if((e=mt(t)).length)while(n=this[u++])if(i=yt(n),r=1===n.nodeType&&" "+vt(i)+" "){a=0;while(o=e[a++])while(-1<r.indexOf(" "+o+" "))r=r.replace(" "+o+" "," ");i!==(s=vt(r))&&n.setAttribute("class",s)}return this},toggleClass:function(i,t){var o=typeof i,a="string"===o||Array.isArray(i);return"boolean"==typeof t&&a?t?this.addClass(i):this.removeClass(i):m(i)?this.each(function(e){S(this).toggleClass(i.call(this,e,yt(this),t),t)}):this.each(function(){var e,t,n,r;if(a){t=0,n=S(this),r=mt(i);while(e=r[t++])n.hasClass(e)?n.removeClass(e):n.addClass(e)}else void 0!==i&&"boolean"!==o||((e=yt(this))&&Y.set(this,"__className__",e),this.setAttribute&&this.setAttribute("class",e||!1===i?"":Y.get(this,"__className__")||""))})},hasClass:function(e){var t,n,r=0;t=" "+e+" ";while(n=this[r++])if(1===n.nodeType&&-1<(" "+vt(yt(n))+" ").indexOf(t))return!0;return!1}});var xt=/\r/g;S.fn.extend({val:function(n){var r,e,i,t=this[0];return arguments.length?(i=m(n),this.each(function(e){var t;1===this.nodeType&&(null==(t=i?n.call(this,e,S(this).val()):n)?t="":"number"==typeof t?t+="":Array.isArray(t)&&(t=S.map(t,function(e){return null==e?"":e+""})),(r=S.valHooks[this.type]||S.valHooks[this.nodeName.toLowerCase()])&&"set"in r&&void 0!==r.set(this,t,"value")||(this.value=t))})):t?(r=S.valHooks[t.type]||S.valHooks[t.nodeName.toLowerCase()])&&"get"in r&&void 0!==(e=r.get(t,"value"))?e:"string"==typeof(e=t.value)?e.replace(xt,""):null==e?"":e:void 0}}),S.extend({valHooks:{option:{get:function(e){var t=S.find.attr(e,"value");return null!=t?t:vt(S.text(e))}},select:{get:function(e){var t,n,r,i=e.options,o=e.selectedIndex,a="select-one"===e.type,s=a?null:[],u=a?o+1:i.length;for(r=o<0?u:a?o:0;r<u;r++)if(((n=i[r]).selected||r===o)&&!n.disabled&&(!n.parentNode.disabled||!A(n.parentNode,"optgroup"))){if(t=S(n).val(),a)return t;s.push(t)}return s},set:function(e,t){var n,r,i=e.options,o=S.makeArray(t),a=i.length;while(a--)((r=i[a]).selected=-1<S.inArray(S.valHooks.option.get(r),o))&&(n=!0);return n||(e.selectedIndex=-1),o}}}}),S.each(["radio","checkbox"],function(){S.valHooks[this]={set:function(e,t){if(Array.isArray(t))return e.checked=-1<S.inArray(S(e).val(),t)}},y.checkOn||(S.valHooks[this].get=function(e){return null===e.getAttribute("value")?"on":e.value})}),y.focusin="onfocusin"in C;var bt=/^(?:focusinfocus|focusoutblur)$/,wt=function(e){e.stopPropagation()};S.extend(S.event,{trigger:function(e,t,n,r){var i,o,a,s,u,l,c,f,p=[n||E],d=v.call(e,"type")?e.type:e,h=v.call(e,"namespace")?e.namespace.split("."):[];if(o=f=a=n=n||E,3!==n.nodeType&&8!==n.nodeType&&!bt.test(d+S.event.triggered)&&(-1<d.indexOf(".")&&(d=(h=d.split(".")).shift(),h.sort()),u=d.indexOf(":")<0&&"on"+d,(e=e[S.expando]?e:new S.Event(d,"object"==typeof e&&e)).isTrigger=r?2:3,e.namespace=h.join("."),e.rnamespace=e.namespace?new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,e.result=void 0,e.target||(e.target=n),t=null==t?[e]:S.makeArray(t,[e]),c=S.event.special[d]||{},r||!c.trigger||!1!==c.trigger.apply(n,t))){if(!r&&!c.noBubble&&!x(n)){for(s=c.delegateType||d,bt.test(s+d)||(o=o.parentNode);o;o=o.parentNode)p.push(o),a=o;a===(n.ownerDocument||E)&&p.push(a.defaultView||a.parentWindow||C)}i=0;while((o=p[i++])&&!e.isPropagationStopped())f=o,e.type=1<i?s:c.bindType||d,(l=(Y.get(o,"events")||Object.create(null))[e.type]&&Y.get(o,"handle"))&&l.apply(o,t),(l=u&&o[u])&&l.apply&&V(o)&&(e.result=l.apply(o,t),!1===e.result&&e.preventDefault());return e.type=d,r||e.isDefaultPrevented()||c._default&&!1!==c._default.apply(p.pop(),t)||!V(n)||u&&m(n[d])&&!x(n)&&((a=n[u])&&(n[u]=null),S.event.triggered=d,e.isPropagationStopped()&&f.addEventListener(d,wt),n[d](),e.isPropagationStopped()&&f.removeEventListener(d,wt),S.event.triggered=void 0,a&&(n[u]=a)),e.result}},simulate:function(e,t,n){var r=S.extend(new S.Event,n,{type:e,isSimulated:!0});S.event.trigger(r,null,t)}}),S.fn.extend({trigger:function(e,t){return this.each(function(){S.event.trigger(e,t,this)})},triggerHandler:function(e,t){var n=this[0];if(n)return S.event.trigger(e,t,n,!0)}}),y.focusin||S.each({focus:"focusin",blur:"focusout"},function(n,r){var i=function(e){S.event.simulate(r,e.target,S.event.fix(e))};S.event.special[r]={setup:function(){var e=this.ownerDocument||this.document||this,t=Y.access(e,r);t||e.addEventListener(n,i,!0),Y.access(e,r,(t||0)+1)},teardown:function(){var e=this.ownerDocument||this.document||this,t=Y.access(e,r)-1;t?Y.access(e,r,t):(e.removeEventListener(n,i,!0),Y.remove(e,r))}}});var Tt=C.location,Ct={guid:Date.now()},Et=/\?/;S.parseXML=function(e){var t;if(!e||"string"!=typeof e)return null;try{t=(new C.DOMParser).parseFromString(e,"text/xml")}catch(e){t=void 0}return t&&!t.getElementsByTagName("parsererror").length||S.error("Invalid XML: "+e),t};var St=/\[\]$/,kt=/\r?\n/g,At=/^(?:submit|button|image|reset|file)$/i,Nt=/^(?:input|select|textarea|keygen)/i;function Dt(n,e,r,i){var t;if(Array.isArray(e))S.each(e,function(e,t){r||St.test(n)?i(n,t):Dt(n+"["+("object"==typeof t&&null!=t?e:"")+"]",t,r,i)});else if(r||"object"!==w(e))i(n,e);else for(t in e)Dt(n+"["+t+"]",e[t],r,i)}S.param=function(e,t){var n,r=[],i=function(e,t){var n=m(t)?t():t;r[r.length]=encodeURIComponent(e)+"="+encodeURIComponent(null==n?"":n)};if(null==e)return"";if(Array.isArray(e)||e.jquery&&!S.isPlainObject(e))S.each(e,function(){i(this.name,this.value)});else for(n in e)Dt(n,e[n],t,i);return r.join("&")},S.fn.extend({serialize:function(){return S.param(this.serializeArray())},serializeArray:function(){return this.map(function(){var e=S.prop(this,"elements");return e?S.makeArray(e):this}).filter(function(){var e=this.type;return this.name&&!S(this).is(":disabled")&&Nt.test(this.nodeName)&&!At.test(e)&&(this.checked||!pe.test(e))}).map(function(e,t){var n=S(this).val();return null==n?null:Array.isArray(n)?S.map(n,function(e){return{name:t.name,value:e.replace(kt,"\r\n")}}):{name:t.name,value:n.replace(kt,"\r\n")}}).get()}});var jt=/%20/g,qt=/#.*$/,Lt=/([?&])_=[^&]*/,Ht=/^(.*?):[ \t]*([^\r\n]*)$/gm,Ot=/^(?:GET|HEAD)$/,Pt=/^\/\//,Rt={},Mt={},It="*/".concat("*"),Wt=E.createElement("a");function Ft(o){return function(e,t){"string"!=typeof e&&(t=e,e="*");var n,r=0,i=e.toLowerCase().match(P)||[];if(m(t))while(n=i[r++])"+"===n[0]?(n=n.slice(1)||"*",(o[n]=o[n]||[]).unshift(t)):(o[n]=o[n]||[]).push(t)}}function Bt(t,i,o,a){var s={},u=t===Mt;function l(e){var r;return s[e]=!0,S.each(t[e]||[],function(e,t){var n=t(i,o,a);return"string"!=typeof n||u||s[n]?u?!(r=n):void 0:(i.dataTypes.unshift(n),l(n),!1)}),r}return l(i.dataTypes[0])||!s["*"]&&l("*")}function $t(e,t){var n,r,i=S.ajaxSettings.flatOptions||{};for(n in t)void 0!==t[n]&&((i[n]?e:r||(r={}))[n]=t[n]);return r&&S.extend(!0,e,r),e}Wt.href=Tt.href,S.extend({active:0,lastModified:{},etag:{},ajaxSettings:{url:Tt.href,type:"GET",isLocal:/^(?:about|app|app-storage|.+-extension|file|res|widget):$/.test(Tt.protocol),global:!0,processData:!0,async:!0,contentType:"application/x-www-form-urlencoded; charset=UTF-8",accepts:{"*":It,text:"text/plain",html:"text/html",xml:"application/xml, text/xml",json:"application/json, text/javascript"},contents:{xml:/\bxml\b/,html:/\bhtml/,json:/\bjson\b/},responseFields:{xml:"responseXML",text:"responseText",json:"responseJSON"},converters:{"* text":String,"text html":!0,"text json":JSON.parse,"text xml":S.parseXML},flatOptions:{url:!0,context:!0}},ajaxSetup:function(e,t){return t?$t($t(e,S.ajaxSettings),t):$t(S.ajaxSettings,e)},ajaxPrefilter:Ft(Rt),ajaxTransport:Ft(Mt),ajax:function(e,t){"object"==typeof e&&(t=e,e=void 0),t=t||{};var c,f,p,n,d,r,h,g,i,o,v=S.ajaxSetup({},t),y=v.context||v,m=v.context&&(y.nodeType||y.jquery)?S(y):S.event,x=S.Deferred(),b=S.Callbacks("once memory"),w=v.statusCode||{},a={},s={},u="canceled",T={readyState:0,getResponseHeader:function(e){var t;if(h){if(!n){n={};while(t=Ht.exec(p))n[t[1].toLowerCase()+" "]=(n[t[1].toLowerCase()+" "]||[]).concat(t[2])}t=n[e.toLowerCase()+" "]}return null==t?null:t.join(", ")},getAllResponseHeaders:function(){return h?p:null},setRequestHeader:function(e,t){return null==h&&(e=s[e.toLowerCase()]=s[e.toLowerCase()]||e,a[e]=t),this},overrideMimeType:function(e){return null==h&&(v.mimeType=e),this},statusCode:function(e){var t;if(e)if(h)T.always(e[T.status]);else for(t in e)w[t]=[w[t],e[t]];return this},abort:function(e){var t=e||u;return c&&c.abort(t),l(0,t),this}};if(x.promise(T),v.url=((e||v.url||Tt.href)+"").replace(Pt,Tt.protocol+"//"),v.type=t.method||t.type||v.method||v.type,v.dataTypes=(v.dataType||"*").toLowerCase().match(P)||[""],null==v.crossDomain){r=E.createElement("a");try{r.href=v.url,r.href=r.href,v.crossDomain=Wt.protocol+"//"+Wt.host!=r.protocol+"//"+r.host}catch(e){v.crossDomain=!0}}if(v.data&&v.processData&&"string"!=typeof v.data&&(v.data=S.param(v.data,v.traditional)),Bt(Rt,v,t,T),h)return T;for(i in(g=S.event&&v.global)&&0==S.active++&&S.event.trigger("ajaxStart"),v.type=v.type.toUpperCase(),v.hasContent=!Ot.test(v.type),f=v.url.replace(qt,""),v.hasContent?v.data&&v.processData&&0===(v.contentType||"").indexOf("application/x-www-form-urlencoded")&&(v.data=v.data.replace(jt,"+")):(o=v.url.slice(f.length),v.data&&(v.processData||"string"==typeof v.data)&&(f+=(Et.test(f)?"&":"?")+v.data,delete v.data),!1===v.cache&&(f=f.replace(Lt,"$1"),o=(Et.test(f)?"&":"?")+"_="+Ct.guid+++o),v.url=f+o),v.ifModified&&(S.lastModified[f]&&T.setRequestHeader("If-Modified-Since",S.lastModified[f]),S.etag[f]&&T.setRequestHeader("If-None-Match",S.etag[f])),(v.data&&v.hasContent&&!1!==v.contentType||t.contentType)&&T.setRequestHeader("Content-Type",v.contentType),T.setRequestHeader("Accept",v.dataTypes[0]&&v.accepts[v.dataTypes[0]]?v.accepts[v.dataTypes[0]]+("*"!==v.dataTypes[0]?", "+It+"; q=0.01":""):v.accepts["*"]),v.headers)T.setRequestHeader(i,v.headers[i]);if(v.beforeSend&&(!1===v.beforeSend.call(y,T,v)||h))return T.abort();if(u="abort",b.add(v.complete),T.done(v.success),T.fail(v.error),c=Bt(Mt,v,t,T)){if(T.readyState=1,g&&m.trigger("ajaxSend",[T,v]),h)return T;v.async&&0<v.timeout&&(d=C.setTimeout(function(){T.abort("timeout")},v.timeout));try{h=!1,c.send(a,l)}catch(e){if(h)throw e;l(-1,e)}}else l(-1,"No Transport");function l(e,t,n,r){var i,o,a,s,u,l=t;h||(h=!0,d&&C.clearTimeout(d),c=void 0,p=r||"",T.readyState=0<e?4:0,i=200<=e&&e<300||304===e,n&&(s=function(e,t,n){var r,i,o,a,s=e.contents,u=e.dataTypes;while("*"===u[0])u.shift(),void 0===r&&(r=e.mimeType||t.getResponseHeader("Content-Type"));if(r)for(i in s)if(s[i]&&s[i].test(r)){u.unshift(i);break}if(u[0]in n)o=u[0];else{for(i in n){if(!u[0]||e.converters[i+" "+u[0]]){o=i;break}a||(a=i)}o=o||a}if(o)return o!==u[0]&&u.unshift(o),n[o]}(v,T,n)),!i&&-1<S.inArray("script",v.dataTypes)&&(v.converters["text script"]=function(){}),s=function(e,t,n,r){var i,o,a,s,u,l={},c=e.dataTypes.slice();if(c[1])for(a in e.converters)l[a.toLowerCase()]=e.converters[a];o=c.shift();while(o)if(e.responseFields[o]&&(n[e.responseFields[o]]=t),!u&&r&&e.dataFilter&&(t=e.dataFilter(t,e.dataType)),u=o,o=c.shift())if("*"===o)o=u;else if("*"!==u&&u!==o){if(!(a=l[u+" "+o]||l["* "+o]))for(i in l)if((s=i.split(" "))[1]===o&&(a=l[u+" "+s[0]]||l["* "+s[0]])){!0===a?a=l[i]:!0!==l[i]&&(o=s[0],c.unshift(s[1]));break}if(!0!==a)if(a&&e["throws"])t=a(t);else try{t=a(t)}catch(e){return{state:"parsererror",error:a?e:"No conversion from "+u+" to "+o}}}return{state:"success",data:t}}(v,s,T,i),i?(v.ifModified&&((u=T.getResponseHeader("Last-Modified"))&&(S.lastModified[f]=u),(u=T.getResponseHeader("etag"))&&(S.etag[f]=u)),204===e||"HEAD"===v.type?l="nocontent":304===e?l="notmodified":(l=s.state,o=s.data,i=!(a=s.error))):(a=l,!e&&l||(l="error",e<0&&(e=0))),T.status=e,T.statusText=(t||l)+"",i?x.resolveWith(y,[o,l,T]):x.rejectWith(y,[T,l,a]),T.statusCode(w),w=void 0,g&&m.trigger(i?"ajaxSuccess":"ajaxError",[T,v,i?o:a]),b.fireWith(y,[T,l]),g&&(m.trigger("ajaxComplete",[T,v]),--S.active||S.event.trigger("ajaxStop")))}return T},getJSON:function(e,t,n){return S.get(e,t,n,"json")},getScript:function(e,t){return S.get(e,void 0,t,"script")}}),S.each(["get","post"],function(e,i){S[i]=function(e,t,n,r){return m(t)&&(r=r||n,n=t,t=void 0),S.ajax(S.extend({url:e,type:i,dataType:r,data:t,success:n},S.isPlainObject(e)&&e))}}),S.ajaxPrefilter(function(e){var t;for(t in e.headers)"content-type"===t.toLowerCase()&&(e.contentType=e.headers[t]||"")}),S._evalUrl=function(e,t,n){return S.ajax({url:e,type:"GET",dataType:"script",cache:!0,async:!1,global:!1,converters:{"text script":function(){}},dataFilter:function(e){S.globalEval(e,t,n)}})},S.fn.extend({wrapAll:function(e){var t;return this[0]&&(m(e)&&(e=e.call(this[0])),t=S(e,this[0].ownerDocument).eq(0).clone(!0),this[0].parentNode&&t.insertBefore(this[0]),t.map(function(){var e=this;while(e.firstElementChild)e=e.firstElementChild;return e}).append(this)),this},wrapInner:function(n){return m(n)?this.each(function(e){S(this).wrapInner(n.call(this,e))}):this.each(function(){var e=S(this),t=e.contents();t.length?t.wrapAll(n):e.append(n)})},wrap:function(t){var n=m(t);return this.each(function(e){S(this).wrapAll(n?t.call(this,e):t)})},unwrap:function(e){return this.parent(e).not("body").each(function(){S(this).replaceWith(this.childNodes)}),this}}),S.expr.pseudos.hidden=function(e){return!S.expr.pseudos.visible(e)},S.expr.pseudos.visible=function(e){return!!(e.offsetWidth||e.offsetHeight||e.getClientRects().length)},S.ajaxSettings.xhr=function(){try{return new C.XMLHttpRequest}catch(e){}};var _t={0:200,1223:204},zt=S.ajaxSettings.xhr();y.cors=!!zt&&"withCredentials"in zt,y.ajax=zt=!!zt,S.ajaxTransport(function(i){var o,a;if(y.cors||zt&&!i.crossDomain)return{send:function(e,t){var n,r=i.xhr();if(r.open(i.type,i.url,i.async,i.username,i.password),i.xhrFields)for(n in i.xhrFields)r[n]=i.xhrFields[n];for(n in i.mimeType&&r.overrideMimeType&&r.overrideMimeType(i.mimeType),i.crossDomain||e["X-Requested-With"]||(e["X-Requested-With"]="XMLHttpRequest"),e)r.setRequestHeader(n,e[n]);o=function(e){return function(){o&&(o=a=r.onload=r.onerror=r.onabort=r.ontimeout=r.onreadystatechange=null,"abort"===e?r.abort():"error"===e?"number"!=typeof r.status?t(0,"error"):t(r.status,r.statusText):t(_t[r.status]||r.status,r.statusText,"text"!==(r.responseType||"text")||"string"!=typeof r.responseText?{binary:r.response}:{text:r.responseText},r.getAllResponseHeaders()))}},r.onload=o(),a=r.onerror=r.ontimeout=o("error"),void 0!==r.onabort?r.onabort=a:r.onreadystatechange=function(){4===r.readyState&&C.setTimeout(function(){o&&a()})},o=o("abort");try{r.send(i.hasContent&&i.data||null)}catch(e){if(o)throw e}},abort:function(){o&&o()}}}),S.ajaxPrefilter(function(e){e.crossDomain&&(e.contents.script=!1)}),S.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},contents:{script:/\b(?:java|ecma)script\b/},converters:{"text script":function(e){return S.globalEval(e),e}}}),S.ajaxPrefilter("script",function(e){void 0===e.cache&&(e.cache=!1),e.crossDomain&&(e.type="GET")}),S.ajaxTransport("script",function(n){var r,i;if(n.crossDomain||n.scriptAttrs)return{send:function(e,t){r=S("<script>").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),E.head.appendChild(r[0])},abort:function(){i&&i()}}});var Ut,Xt=[],Vt=/(=)\?(?=&|$)|\?\?/;S.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=Xt.pop()||S.expando+"_"+Ct.guid++;return this[e]=!0,e}}),S.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Vt.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Vt.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=m(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Vt,"$1"+r):!1!==e.jsonp&&(e.url+=(Et.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||S.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=C[r],C[r]=function(){o=arguments},n.always(function(){void 0===i?S(C).removeProp(r):C[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,Xt.push(r)),o&&m(i)&&i(o[0]),o=i=void 0}),"script"}),y.createHTMLDocument=((Ut=E.implementation.createHTMLDocument("").body).innerHTML="<form></form><form></form>",2===Ut.childNodes.length),S.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(y.createHTMLDocument?((r=(t=E.implementation.createHTMLDocument("")).createElement("base")).href=E.location.href,t.head.appendChild(r)):t=E),o=!n&&[],(i=N.exec(e))?[t.createElement(i[1])]:(i=xe([e],t,o),o&&o.length&&S(o).remove(),S.merge([],i.childNodes)));var r,i,o},S.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1<s&&(r=vt(e.slice(s)),e=e.slice(0,s)),m(t)?(n=t,t=void 0):t&&"object"==typeof t&&(i="POST"),0<a.length&&S.ajax({url:e,type:i||"GET",dataType:"html",data:t}).done(function(e){o=arguments,a.html(r?S("<div>").append(S.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},S.expr.pseudos.animated=function(t){return S.grep(S.timers,function(e){return t===e.elem}).length},S.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=S.css(e,"position"),c=S(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=S.css(e,"top"),u=S.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),m(t)&&(t=t.call(e,n,S.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):("number"==typeof f.top&&(f.top+="px"),"number"==typeof f.left&&(f.left+="px"),c.css(f))}},S.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){S.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===S.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===S.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=S(e).offset()).top+=S.css(e,"borderTopWidth",!0),i.left+=S.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-S.css(r,"marginTop",!0),left:t.left-i.left-S.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===S.css(e,"position"))e=e.offsetParent;return e||re})}}),S.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;S.fn[t]=function(e){return $(this,function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),S.each(["top","left"],function(e,n){S.cssHooks[n]=$e(y.pixelPosition,function(e,t){if(t)return t=Be(e,n),Me.test(t)?S(e).position()[n]+"px":t})}),S.each({Height:"height",Width:"width"},function(a,s){S.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){S.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return $(this,function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?S.css(e,t,i):S.style(e,t,n,i)},s,n?e:void 0,n)}})}),S.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){S.fn[t]=function(e){return this.on(t,e)}}),S.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.mouseenter(e).mouseleave(t||e)}}),S.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){S.fn[n]=function(e,t){return 0<arguments.length?this.on(n,null,e,t):this.trigger(n)}});var Gt=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g;S.proxy=function(e,t){var n,r,i;if("string"==typeof t&&(n=e[t],t=e,e=n),m(e))return r=s.call(arguments,2),(i=function(){return e.apply(t||this,r.concat(s.call(arguments)))}).guid=e.guid=e.guid||S.guid++,i},S.holdReady=function(e){e?S.readyWait++:S.ready(!0)},S.isArray=Array.isArray,S.parseJSON=JSON.parse,S.nodeName=A,S.isFunction=m,S.isWindow=x,S.camelCase=X,S.type=w,S.now=Date.now,S.isNumeric=function(e){var t=S.type(e);return("number"===t||"string"===t)&&!isNaN(e-parseFloat(e))},S.trim=function(e){return null==e?"":(e+"").replace(Gt,"")},"function"==typeof define&&define.amd&&define("jquery",[],function(){return S});var Yt=C.jQuery,Qt=C.$;return S.noConflict=function(e){return C.$===S&&(C.$=Qt),e&&C.jQuery===S&&(C.jQuery=Yt),S},"undefined"==typeof e&&(C.jQuery=C.$=S),S});
diff --git a/core/src/main/resources/org/apache/spark/ui/static/log-view.js b/core/src/main/resources/org/apache/spark/ui/static/log-view.js
index b5c43e5788bc3..1a4f9254742ff 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/log-view.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/log-view.js
@@ -15,6 +15,8 @@
  * limitations under the License.
  */
 
+/* global $ */
+
 var baseParams;
 
 var curLogLength;
@@ -59,11 +61,12 @@ function getRESTEndPoint() {
   var words = document.baseURI.split('/');
   var ind = words.indexOf("proxy");
   if (ind > 0) {
-      return words.slice(0, ind + 2).join('/') + "/log";
+    return words.slice(0, ind + 2).join('/') + "/log";
   }
   return "/log"
 }
 
+/* eslint-disable no-unused-vars */
 function loadMore() {
   var offset = Math.max(startByte - byteLength, 0);
   var moreByteLength = Math.min(byteLength, startByte);
@@ -139,4 +142,5 @@ function initLogPage(params, logLen, start, end, totLogLen, defaultLen) {
   if (startByte == 0) {
     disableMoreButton();
   }
-}
\ No newline at end of file
+}
+/* eslint-enable no-unused-vars */
\ No newline at end of file
diff --git a/core/src/main/resources/org/apache/spark/ui/static/sorttable.js b/core/src/main/resources/org/apache/spark/ui/static/sorttable.js
index ecd580e5c64aa..3f98a0379dc3c 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/sorttable.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/sorttable.js
@@ -99,12 +99,12 @@ sorttable = {
                                                     'sorttable_sorted_reverse');
             rowlists = this.parentNode.getElementsByTagName("span");
             for (var j=0; j < rowlists.length; j++) {
-                if (rowlists[j].className.search(/\bsorttable_sortfwdind\b/)) {
+                if (rowlists[j].className.search(/\bsorttable_sortfwdind\b/) != -1) {
                     rowlists[j].parentNode.removeChild(rowlists[j]);
                 }
             }
             sortrevind = document.createElement('span');
-            sortrevind.class = "sorttable_sortrevind";
+            sortrevind.className = "sorttable_sortrevind";
             sortrevind.innerHTML = stIsIE ? '&nbsp<font face="webdings">5</font>' : '&nbsp;&#x25BE;';
             this.appendChild(sortrevind);
             return;
@@ -117,12 +117,12 @@ sorttable = {
                                                   'sorttable_sorted');
             rowlists = this.parentNode.getElementsByTagName("span");
             for (var j=0; j < rowlists.length; j++) {
-                if (rowlists[j].className.search(/\sorttable_sortrevind\b/)) {
+                if (rowlists[j].className.search(/\bsorttable_sortrevind\b/) != -1) {
                     rowlists[j].parentNode.removeChild(rowlists[j]);
                 }
             }
             sortfwdind = document.createElement('span');
-            sortfwdind.class = "sorttable_sortfwdind";
+            sortfwdind.className = "sorttable_sortfwdind";
             sortfwdind.innerHTML = stIsIE ? '&nbsp<font face="webdings">6</font>' : '&nbsp;&#x25B4;';
             this.appendChild(sortfwdind);
             return;
@@ -138,15 +138,15 @@ sorttable = {
           });
           rowlists = this.parentNode.getElementsByTagName("span");
           for (var j=0; j < rowlists.length; j++) {
-              if (rowlists[j].className.search(/\bsorttable_sortfwdind\b/)
-                  || rowlists[j].className.search(/\sorttable_sortrevind\b/) ) {
+              if (rowlists[j].className.search(/\bsorttable_sortfwdind\b/) != -1
+                  || rowlists[j].className.search(/\bsorttable_sortrevind\b/) != -1) {
                   rowlists[j].parentNode.removeChild(rowlists[j]);
               }
           }
 
           this.className += ' sorttable_sorted';
           sortfwdind = document.createElement('span');
-          sortfwdind.class = "sorttable_sortfwdind";
+          sortfwdind.className = "sorttable_sortfwdind";
           sortfwdind.innerHTML = stIsIE ? '&nbsp<font face="webdings">6</font>' : '&nbsp;&#x25B4;';
           this.appendChild(sortfwdind);
 
diff --git a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.css b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.css
index e44a724c202f7..d6ddb860389be 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.css
+++ b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.css
@@ -96,6 +96,12 @@
   stroke-width: 2px;
 }
 
+#dag-viz-graph svg.job g.node.indeterminate circle {
+  fill: #f6b187;
+  stroke: #D97536;
+  stroke-width: 2px;
+}
+
 /* Stage page specific styles */
 
 #dag-viz-graph svg.stage g.cluster rect {
@@ -136,6 +142,12 @@
   stroke-width: 2px;
 }
 
+#dag-viz-graph svg.stage g.node.indeterminate rect {
+  fill: #F09c67;
+  stroke: #D97536;
+  stroke-width: 2px;
+}
+
 .tooltip-inner {
   white-space: pre-wrap;
 }
diff --git a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
index ae02defd9bb9c..6a0cd012146ec 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
@@ -51,6 +51,8 @@
  * since it was forked (commit 101503833a8ce5fe369547f6addf3e71172ce10b).
  */
 
+/* global $, appBasePath, d3, dagreD3, graphlibDot, uiRoot */
+
 var VizConstants = {
   svgMarginX: 16,
   svgMarginY: 16,
@@ -166,16 +168,24 @@ function renderDagViz(forJob) {
   }
 
   // Find cached RDDs and mark them as such
-  metadataContainer().selectAll(".cached-rdd").each(function(v) {
+  metadataContainer().selectAll(".cached-rdd").each(function(_ignored_v) {
     var rddId = d3.select(this).text().trim();
     var nodeId = VizConstants.nodePrefix + rddId;
     svg.selectAll("g." + nodeId).classed("cached", true);
   });
 
   metadataContainer().selectAll(".barrier-rdd").each(function() {
+    var opId = d3.select(this).text().trim();
+    var opClusterId = VizConstants.clusterPrefix + opId;
+    var stageId = $(this).parents(".stage-metadata").attr("stage-id");
+    var stageClusterId = VizConstants.graphPrefix + stageId;
+    svg.selectAll("g[id=" + stageClusterId + "] g." + opClusterId).classed("barrier", true)
+  });
+
+  metadataContainer().selectAll(".indeterminate-rdd").each(function(_ignored_v) {
     var rddId = d3.select(this).text().trim();
-    var clusterId = VizConstants.clusterPrefix + rddId;
-    svg.selectAll("g." + clusterId).classed("barrier", true)
+    var nodeId = VizConstants.nodePrefix + rddId;
+    svg.selectAll("g." + nodeId).classed("indeterminate", true);
   });
 
   resizeSvg(svg);
@@ -216,7 +226,7 @@ function renderDagVizForJob(svgContainer) {
     var dot = metadata.select(".dot-file").text();
     var stageId = metadata.attr("stage-id");
     var containerId = VizConstants.graphPrefix + stageId;
-    var isSkipped = metadata.attr("skipped") == "true";
+    var isSkipped = metadata.attr("skipped") === "true";
     var container;
     if (isSkipped) {
       container = svgContainer
@@ -225,11 +235,8 @@ function renderDagVizForJob(svgContainer) {
         .attr("skipped", "true");
     } else {
       // Link each graph to the corresponding stage page (TODO: handle stage attempts)
-      // Use the link from the stage table so it also works for the history server
       var attemptId = 0;
-      var stageLink = d3.select("#stage-" + stageId + "-" + attemptId)
-        .select("a.name-link")
-        .attr("href");
+      var stageLink = uiRoot + appBasePath + "/stages/stage/?id=" + stageId + "&attempt=" + attemptId;
       container = svgContainer
         .append("a")
         .attr("xlink:href", stageLink)
@@ -270,7 +277,7 @@ function renderDagVizForJob(svgContainer) {
     // If there are any incoming edges into this graph, keep track of them to render
     // them separately later. Note that we cannot draw them now because we need to
     // put these edges in a separate container that is on top of all stage graphs.
-    metadata.selectAll(".incoming-edge").each(function(v) {
+    metadata.selectAll(".incoming-edge").each(function(_ignored_v) {
       var edge = d3.select(this).text().trim().split(","); // e.g. 3,4 => [3, 4]
       crossStageEdges.push(edge);
     });
@@ -335,7 +342,7 @@ function preprocessGraphLayout(g, forJob) {
 }
 
 /*
- * Helper function to size the SVG appropriately such that all elements are displyed.
+ * Helper function to size the SVG appropriately such that all elements are displayed.
  * This assumes that all outermost elements are clusters (rectangles).
  */
 function resizeSvg(svg) {
diff --git a/core/src/main/resources/org/apache/spark/ui/static/stagepage.js b/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
index 93b37c296271b..fea595d0a6216 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
@@ -15,251 +15,291 @@
  * limitations under the License.
  */
 
+/* global $, ConvertDurationString, Mustache, createRESTEndPointForExecutorsPage */
+/* global createTemplateURI, formatBytes, formatDate, formatDuration, formatLogsCells */
+/* global getStandAloneAppId, setDataTableDefaults, uiRoot */
+
 var shouldBlockUI = true;
 
 $(document).ajaxStop(function () {
-    if (shouldBlockUI) {
-        $.unblockUI();
-        shouldBlockUI = false;
-    }
+  if (shouldBlockUI) {
+    $.unblockUI();
+    shouldBlockUI = false;
+  }
 });
 
 $(document).ajaxStart(function () {
-    if (shouldBlockUI) {
-        $.blockUI({message: '<h3>Loading Stage Page...</h3>'});
-    }
+  if (shouldBlockUI) {
+    $.blockUI({message: '<h3>Loading Stage Page...</h3>'});
+  }
 });
 
 $.extend( $.fn.dataTable.ext.type.order, {
-    "duration-pre": ConvertDurationString,
-
-    "duration-asc": function ( a, b ) {
-        a = ConvertDurationString( a );
-        b = ConvertDurationString( b );
-        return ((a < b) ? -1 : ((a > b) ? 1 : 0));
-    },
-
-    "duration-desc": function ( a, b ) {
-        a = ConvertDurationString( a );
-        b = ConvertDurationString( b );
-        return ((a < b) ? 1 : ((a > b) ? -1 : 0));
-    }
-} );
+  "duration-pre": ConvertDurationString,
+
+  "duration-asc": function ( a, b ) {
+    a = ConvertDurationString( a );
+    b = ConvertDurationString( b );
+    return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+  },
+
+  "duration-desc": function ( a, b ) {
+    a = ConvertDurationString( a );
+    b = ConvertDurationString( b );
+    return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+  },
+
+  "size-pre": function (data) {
+    var floatValue = parseFloat(data)
+    return isNaN(floatValue) ? 0 : floatValue;
+  },
+
+  "size-asc": function (a, b) {
+    a = parseFloat(a);
+    b = parseFloat(b);
+    return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+  },
+
+  "size-desc": function (a, b) {
+    a = parseFloat(a);
+    b = parseFloat(b);
+    return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+  }
+});
 
 // This function will only parse the URL under certain format
 // e.g. (history) https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4&attempt=1
 // e.g. (proxy) https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4&attempt=1
 function stageEndPoint(appId) {
-    var queryString = document.baseURI.split('?');
-    var words = document.baseURI.split('/');
-    var indexOfProxy = words.indexOf("proxy");
-    var stageId = queryString[1].split("&").filter(word => word.includes("id="))[0].split("=")[1];
-    if (indexOfProxy > 0) {
-        var appId = words[indexOfProxy + 1];
-        var newBaseURI = words.slice(0, words.indexOf("proxy") + 2).join('/');
-        return newBaseURI + "/api/v1/applications/" + appId + "/stages/" + stageId;
-    }
-    var indexOfHistory = words.indexOf("history");
-    if (indexOfHistory > 0) {
-        var appId = words[indexOfHistory + 1];
-        var appAttemptId = words[indexOfHistory + 2];
-        var newBaseURI = words.slice(0, words.indexOf("history")).join('/');
-        if (isNaN(appAttemptId) || appAttemptId == "0") {
-            return newBaseURI + "/api/v1/applications/" + appId + "/stages/" + stageId;
-        } else {
-            return newBaseURI + "/api/v1/applications/" + appId + "/" + appAttemptId + "/stages/" + stageId;
-        }
+  var queryString = document.baseURI.split('?');
+  var words = document.baseURI.split('/');
+  var indexOfProxy = words.indexOf("proxy");
+  var stageId = queryString[1].split("&").filter(word => word.includes("id="))[0].split("=")[1];
+  var newBaseURI;
+  if (indexOfProxy > 0) {
+    appId = words[indexOfProxy + 1];
+    newBaseURI = words.slice(0, words.indexOf("proxy") + 2).join('/');
+    return newBaseURI + "/api/v1/applications/" + appId + "/stages/" + stageId;
+  }
+  var indexOfHistory = words.indexOf("history");
+  if (indexOfHistory > 0) {
+    appId = words[indexOfHistory + 1];
+    var appAttemptId = words[indexOfHistory + 2];
+    newBaseURI = words.slice(0, words.indexOf("history")).join('/');
+    if (isNaN(appAttemptId) || appAttemptId == "0") {
+      return newBaseURI + "/api/v1/applications/" + appId + "/stages/" + stageId;
+    } else {
+      return newBaseURI + "/api/v1/applications/" + appId + "/" + appAttemptId + "/stages/" + stageId;
     }
-    return location.origin + "/api/v1/applications/" + appId + "/stages/" + stageId;
+  }
+  return uiRoot + "/api/v1/applications/" + appId + "/stages/" + stageId;
 }
 
 function getColumnNameForTaskMetricSummary(columnKey) {
-    switch(columnKey) {
-        case "executorRunTime":
-            return "Duration";
+  switch(columnKey) {
+    case "executorRunTime":
+      return "Duration";
 
-        case "jvmGcTime":
-            return "GC Time";
+    case "jvmGcTime":
+      return "GC Time";
 
-        case "gettingResultTime":
-            return "Getting Result Time";
+    case "gettingResultTime":
+      return "Getting Result Time";
 
-        case "inputMetrics":
-            return "Input Size / Records";
+    case "inputMetrics":
+      return "Input Size / Records";
 
-        case "outputMetrics":
-            return "Output Size / Records";
+    case "outputMetrics":
+      return "Output Size / Records";
 
-        case "peakExecutionMemory":
-            return "Peak Execution Memory";
+    case "peakExecutionMemory":
+      return "Peak Execution Memory";
 
-        case "resultSerializationTime":
-            return "Result Serialization Time";
+    case "resultSerializationTime":
+      return "Result Serialization Time";
 
-        case "schedulerDelay":
-            return "Scheduler Delay";
+    case "schedulerDelay":
+      return "Scheduler Delay";
 
-        case "diskBytesSpilled":
-            return "Spill (disk)";
+    case "diskBytesSpilled":
+      return "Spill (disk)";
 
-        case "memoryBytesSpilled":
-            return "Spill (memory)";
+    case "memoryBytesSpilled":
+      return "Spill (memory)";
 
-        case "shuffleReadMetrics":
-            return "Shuffle Read Size / Records";
+    case "shuffleReadMetrics":
+      return "Shuffle Read Size / Records";
 
-        case "shuffleWriteMetrics":
-            return "Shuffle Write Size / Records";
+    case "shuffleWriteMetrics":
+      return "Shuffle Write Size / Records";
 
-        case "executorDeserializeTime":
-            return "Task Deserialization Time";
+    case "executorDeserializeTime":
+      return "Task Deserialization Time";
 
-        case "shuffleReadBlockedTime":
-            return "Shuffle Read Blocked Time";
+    case "shuffleReadBlockedTime":
+      return "Shuffle Read Blocked Time";
 
-        case "shuffleRemoteReads":
-            return "Shuffle Remote Reads";
+    case "shuffleRemoteReads":
+      return "Shuffle Remote Reads";
 
-        case "shuffleWriteTime":
-            return "Shuffle Write Time";
+    case "shuffleWriteTime":
+      return "Shuffle Write Time";
 
-        default:
-            return "NA";
-    }
+    default:
+      return "NA";
+  }
 }
 
 function displayRowsForSummaryMetricsTable(row, type, columnIndex) {
-    switch(row.columnKey) {
-        case 'inputMetrics':
-            var str = formatBytes(row.data.bytesRead[columnIndex], type) + " / " +
-              row.data.recordsRead[columnIndex];
-            return str;
-
-        case 'outputMetrics':
-            var str = formatBytes(row.data.bytesWritten[columnIndex], type) + " / " +
-              row.data.recordsWritten[columnIndex];
-            return str;
-
-        case 'shuffleReadMetrics':
-            var str = formatBytes(row.data.readBytes[columnIndex], type) + " / " +
-              row.data.readRecords[columnIndex];
-            return str;
-
-        case 'shuffleReadBlockedTime':
-            var str = formatDuration(row.data.fetchWaitTime[columnIndex]);
-            return str;
-
-        case 'shuffleRemoteReads':
-            var str = formatBytes(row.data.remoteBytesRead[columnIndex], type);
-            return str;
-
-        case 'shuffleWriteMetrics':
-            var str = formatBytes(row.data.writeBytes[columnIndex], type) + " / " +
-              row.data.writeRecords[columnIndex];
-            return str;
-
-        case 'shuffleWriteTime':
-            var str = formatDuration(row.data.writeTime[columnIndex] / 1000000.0);
-            return str;
-
-        default:
-            return (row.columnKey == 'peakExecutionMemory' || row.columnKey == 'memoryBytesSpilled'
-                    || row.columnKey == 'diskBytesSpilled') ? formatBytes(
-                    row.data[columnIndex], type) : (formatDuration(row.data[columnIndex]));
-
-    }
+  var str;
+  switch(row.columnKey) {
+    case 'inputMetrics':
+      str = formatBytes(row.data.bytesRead[columnIndex], type) + " / " +
+        row.data.recordsRead[columnIndex];
+      return str;
+
+    case 'outputMetrics':
+      str = formatBytes(row.data.bytesWritten[columnIndex], type) + " / " +
+        row.data.recordsWritten[columnIndex];
+      return str;
+ 
+    case 'shuffleReadMetrics':
+      str = formatBytes(row.data.readBytes[columnIndex], type) + " / " +
+        row.data.readRecords[columnIndex];
+      return str;
+ 
+    case 'shuffleReadBlockedTime':
+      str = formatDuration(row.data.fetchWaitTime[columnIndex]);
+      return str;
+ 
+    case 'shuffleRemoteReads':
+      str = formatBytes(row.data.remoteBytesRead[columnIndex], type);
+      return str;
+ 
+    case 'shuffleWriteMetrics':
+      str = formatBytes(row.data.writeBytes[columnIndex], type) + " / " +
+        row.data.writeRecords[columnIndex];
+      return str;
+ 
+    case 'shuffleWriteTime':
+      str = formatDuration(row.data.writeTime[columnIndex] / 1000000.0);
+      return str;
+ 
+    default:
+      return (row.columnKey == 'peakExecutionMemory' || row.columnKey == 'memoryBytesSpilled'
+        || row.columnKey == 'diskBytesSpilled') ? formatBytes(
+          row.data[columnIndex], type) : (formatDuration(row.data[columnIndex]));
+
+  }
 }
 
 function createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTable) {
-    var taskMetricsTable = "#summary-metrics-table";
-    if ($.fn.dataTable.isDataTable(taskMetricsTable)) {
-        taskSummaryMetricsDataTable.clear().draw();
-        taskSummaryMetricsDataTable.rows.add(taskSummaryMetricsTable).draw();
-    } else {
-        var taskConf = {
-            "data": taskSummaryMetricsTable,
-            "columns": [
-                {data : 'metric'},
-                // Min
-                {
-                    data: function (row, type) {
-                        return displayRowsForSummaryMetricsTable(row, type, 0);
-                    }
-                },
-                // 25th percentile
-                {
-                    data: function (row, type) {
-                        return displayRowsForSummaryMetricsTable(row, type, 1);
-                    }
-                },
-                // Median
-                {
-                    data: function (row, type) {
-                        return displayRowsForSummaryMetricsTable(row, type, 2);
-                    }
-                },
-                // 75th percentile
-                {
-                    data: function (row, type) {
-                        return displayRowsForSummaryMetricsTable(row, type, 3);
-                    }
-                },
-                // Max
-                {
-                    data: function (row, type) {
-                        return displayRowsForSummaryMetricsTable(row, type, 4);
-                    }
-                }
-            ],
-            "columnDefs": [
-                { "type": "duration", "targets": 1 },
-                { "type": "duration", "targets": 2 },
-                { "type": "duration", "targets": 3 },
-                { "type": "duration", "targets": 4 },
-                { "type": "duration", "targets": 5 }
-            ],
-            "paging": false,
-            "searching": false,
-            "order": [[0, "asc"]],
-            "bSort": false,
-            "bAutoWidth": false,
-            "oLanguage": {
-                "sEmptyTable": "No tasks have reported metrics yet"
-            }
-        };
-        taskSummaryMetricsDataTable = $(taskMetricsTable).DataTable(taskConf);
-    }
-    taskSummaryMetricsTableCurrentStateArray = taskSummaryMetricsTable.slice();
+  var taskMetricsTable = "#summary-metrics-table";
+  if ($.fn.dataTable.isDataTable(taskMetricsTable)) {
+    taskSummaryMetricsDataTable.clear().draw();
+    taskSummaryMetricsDataTable.rows.add(taskSummaryMetricsTable).draw();
+  } else {
+    var taskConf = {
+      "data": taskSummaryMetricsTable,
+      "columns": [
+        {data : 'metric'},
+        // Min
+        {
+          data: function (row, type) {
+            return displayRowsForSummaryMetricsTable(row, type, 0);
+          }
+        },
+        // 25th percentile
+        {
+          data: function (row, type) {
+            return displayRowsForSummaryMetricsTable(row, type, 1);
+          }
+        },
+        // Median
+        {
+          data: function (row, type) {
+            return displayRowsForSummaryMetricsTable(row, type, 2);
+          }
+        },
+        // 75th percentile
+        {
+          data: function (row, type) {
+            return displayRowsForSummaryMetricsTable(row, type, 3);
+          }
+        },
+        // Max
+        {
+          data: function (row, type) {
+            return displayRowsForSummaryMetricsTable(row, type, 4);
+          }
+        }
+      ],
+      "columnDefs": [
+        { "type": "duration", "targets": 1 },
+        { "type": "duration", "targets": 2 },
+        { "type": "duration", "targets": 3 },
+        { "type": "duration", "targets": 4 },
+        { "type": "duration", "targets": 5 }
+      ],
+      "paging": false,
+      "searching": false,
+      "order": [[0, "asc"]],
+      "bSort": false,
+      "bAutoWidth": false,
+      "oLanguage": {
+        "sEmptyTable": "No tasks have reported metrics yet"
+      }
+    };
+    taskSummaryMetricsDataTable = $(taskMetricsTable).DataTable(taskConf);
+  }
+  taskSummaryMetricsTableCurrentStateArray = taskSummaryMetricsTable.slice();
 }
 
 function createRowMetadataForColumn(colKey, data, checkboxId) {
   var row = {
-      "metric": getColumnNameForTaskMetricSummary(colKey),
-      "data": data,
-      "checkboxId": checkboxId,
-      "columnKey": colKey
+    "metric": getColumnNameForTaskMetricSummary(colKey),
+    "data": data,
+    "checkboxId": checkboxId,
+    "columnKey": colKey
   };
   return row;
 }
 
 function reselectCheckboxesBasedOnTaskTableState() {
-    var allChecked = true;
-    var taskSummaryMetricsTableCurrentFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice();
-    if (typeof taskTableSelector !== 'undefined' && taskSummaryMetricsTableCurrentStateArray.length > 0) {
-        for (var k = 0; k < optionalColumns.length; k++) {
-            if (taskTableSelector.column(optionalColumns[k]).visible()) {
-                $("#box-"+optionalColumns[k]).prop('checked', true);
-                taskSummaryMetricsTableCurrentStateArray.push(taskSummaryMetricsTableArray.filter(row => (row.checkboxId).toString() == optionalColumns[k])[0]);
-                taskSummaryMetricsTableCurrentFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice();
-            } else {
-                allChecked = false;
-            }
-        }
-        if (allChecked) {
-            $("#box-0").prop('checked', true);
-        }
-        createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableCurrentFilteredArray);
+  var taskSummaryHasSelected = false;
+  var executorSummaryHasSelected = false;
+  var allTaskSummaryChecked = true;
+  var allExecutorSummaryChecked = true;
+  var taskSummaryMetricsTableCurrentFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice();
+  var k;
+  if (typeof taskTableSelector !== 'undefined' && taskSummaryMetricsTableCurrentStateArray.length > 0) {
+    for (k = 0; k < optionalColumns.length; k++) {
+      if (taskTableSelector.column(optionalColumns[k]).visible()) {
+        taskSummaryHasSelected = true;
+        $("#box-"+optionalColumns[k]).prop('checked', true);
+        taskSummaryMetricsTableCurrentStateArray.push(taskSummaryMetricsTableArray.filter(row => (row.checkboxId).toString() == optionalColumns[k])[0]);
+        taskSummaryMetricsTableCurrentFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice();
+      } else {
+        allTaskSummaryChecked = false;
+      }
     }
+    createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableCurrentFilteredArray);
+  }
+
+  if (typeof executorSummaryTableSelector !== 'undefined') {
+    for (k = 0; k < executorOptionalColumns.length; k++) {
+      if (executorSummaryTableSelector.column(executorOptionalColumns[k]).visible()) {
+        executorSummaryHasSelected = true;
+        $("#executor-box-"+executorOptionalColumns[k]).prop('checked', true);
+      } else {
+        allExecutorSummaryChecked = false;
+      }
+    }
+  }
+
+  if ((taskSummaryHasSelected || executorSummaryHasSelected) && allTaskSummaryChecked && allExecutorSummaryChecked) {
+    $("#box-0").prop('checked', true);
+  }
 }
 
 function getStageAttemptId() {
@@ -268,7 +308,7 @@ function getStageAttemptId() {
   // We are using regex here to extract the stage attempt id as there might be certain url's with format
   // like /proxy/application_1539986433979_27115/stages/stage/?id=0&attempt=0#tasksTitle
   var stgAttemptId = words[1].split("&").filter(
-      word => word.includes("attempt="))[0].split("=")[1].match(digitsRegex);
+    word => word.includes("attempt="))[0].split("=")[1].match(digitsRegex);
   return stgAttemptId;
 }
 
@@ -278,711 +318,837 @@ var taskSummaryMetricsDataTable;
 var optionalColumns = [11, 12, 13, 14, 15, 16, 17, 21];
 var taskTableSelector;
 
+var executorOptionalColumns = [15, 16, 17, 18];
+var executorSummaryTableSelector;
+
 $(document).ready(function () {
-    setDataTableDefaults();
-
-    $("#showAdditionalMetrics").append(
-        "<div><a id='additionalMetrics' class='collapse-table'>" +
-        "<span class='expand-input-rate-arrow arrow-closed' id='arrowtoggle1'></span>" +
-        " Show Additional Metrics" +
-        "</a></div>" +
-        "<div class='container-fluid-div ml-4 d-none' id='toggle-metrics'>" +
-        "<div id='select_all' class='select-all-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-0' data-column='0'> Select All</div>" +
-        "<div id='scheduler_delay' class='scheduler-delay-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-11' data-column='11'> Scheduler Delay</div>" +
-        "<div id='task_deserialization_time' class='task-deserialization-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-12' data-column='12'> Task Deserialization Time</div>" +
-        "<div id='shuffle_read_blocked_time' class='shuffle-read-blocked-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-13' data-column='13'> Shuffle Read Blocked Time</div>" +
-        "<div id='shuffle_remote_reads' class='shuffle-remote-reads-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-14' data-column='14'> Shuffle Remote Reads</div>" +
-        "<div id='shuffle_write_time' class='shuffle-write-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-21' data-column='21'> Shuffle Write Time</div>" +
-        "<div id='result_serialization_time' class='result-serialization-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-15' data-column='15'> Result Serialization Time</div>" +
-        "<div id='getting_result_time' class='getting-result-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-16' data-column='16'> Getting Result Time</div>" +
-        "<div id='peak_execution_memory' class='peak-execution-memory-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-17' data-column='17'> Peak Execution Memory</div>" +
-        "</div>");
-
-    $('#scheduler_delay').attr("data-toggle", "tooltip")
-        .attr("data-placement", "top")
-        .attr("title", "Scheduler delay includes time to ship the task from the scheduler to the executor, and time to send " +
-            "the task result from the executor to the scheduler. If scheduler delay is large, consider decreasing the size of tasks or decreasing the size of task results.");
-    $('#task_deserialization_time').attr("data-toggle", "tooltip")
-        .attr("data-placement", "top")
-        .attr("title", "Time spent deserializing the task closure on the executor, including the time to read the broadcasted task.");
-    $('#shuffle_read_blocked_time').attr("data-toggle", "tooltip")
-        .attr("data-placement", "top")
-        .attr("title", "Time that the task spent blocked waiting for shuffle data to be read from remote machines.");
-    $('#shuffle_remote_reads').attr("data-toggle", "tooltip")
-        .attr("data-placement", "top")
-        .attr("title", "Total shuffle bytes read from remote executors. This is a subset of the shuffle read bytes; the remaining shuffle data is read locally. ");
-    $('#shuffle_write_time').attr("data-toggle", "tooltip")
-        .attr("data-placement", "top")
-        .attr("title", "Time that the task spent writing shuffle data.");
-    $('#result_serialization_time').attr("data-toggle", "tooltip")
-            .attr("data-placement", "top")
-            .attr("title", "Time spent serializing the task result on the executor before sending it back to the driver.");
-    $('#getting_result_time').attr("data-toggle", "tooltip")
-            .attr("data-placement", "top")
-            .attr("title", "Time that the driver spends fetching task results from workers. If this is large, consider decreasing the amount of data returned from each task.");
-    $('#peak_execution_memory').attr("data-toggle", "tooltip")
-            .attr("data-placement", "top")
-            .attr("title", "Execution memory refers to the memory used by internal data structures created during " +
-                "shuffles, aggregations and joins when Tungsten is enabled. The value of this accumulator " +
-                "should be approximately the sum of the peak sizes across all such data structures created " +
-                "in this task. For SQL jobs, this only tracks all unsafe operators, broadcast joins, and " +
-                "external sort.");
-    $('[data-toggle="tooltip"]').tooltip();
-    var tasksSummary = $("#parent-container");
-    getStandAloneAppId(function (appId) {
-        // rendering the UI page
-        $.get(createTemplateURI(appId, "stagespage"), function(template) {
-          tasksSummary.append(Mustache.render($(template).filter("#stages-summary-template").html()));
-
-          $("#additionalMetrics").click(function(){
-              $("#arrowtoggle1").toggleClass("arrow-open arrow-closed");
-              $("#toggle-metrics").toggleClass("d-none");
-              if (window.localStorage) {
-                  window.localStorage.setItem("arrowtoggle1class", $("#arrowtoggle1").attr('class'));
-              }
+  setDataTableDefaults();
+
+  $("#showAdditionalMetrics").append(
+    "<div><a id='additionalMetrics' class='collapse-table'>" +
+    "<span class='expand-input-rate-arrow arrow-closed' id='arrowtoggle1'></span>" +
+    " Show Additional Metrics" +
+    "</a></div>" +
+    "<div class='container-fluid-div ml-4 d-none' id='toggle-metrics'>" +
+    "<div id='select_all' class='select-all-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-0' data-column='0'> Select All</div>" +
+    "<div id='scheduler_delay' class='scheduler-delay-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-11' data-column='11' data-metrics-type='task'> Scheduler Delay</div>" +
+    "<div id='task_deserialization_time' class='task-deserialization-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-12' data-column='12' data-metrics-type='task'> Task Deserialization Time</div>" +
+    "<div id='shuffle_read_blocked_time' class='shuffle-read-blocked-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-13' data-column='13' data-metrics-type='task'> Shuffle Read Blocked Time</div>" +
+    "<div id='shuffle_remote_reads' class='shuffle-remote-reads-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-14' data-column='14' data-metrics-type='task'> Shuffle Remote Reads</div>" +
+    "<div id='shuffle_write_time' class='shuffle-write-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-21' data-column='21' data-metrics-type='task'> Shuffle Write Time</div>" +
+    "<div id='result_serialization_time' class='result-serialization-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-15' data-column='15' data-metrics-type='task'> Result Serialization Time</div>" +
+    "<div id='getting_result_time' class='getting-result-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-16' data-column='16' data-metrics-type='task'> Getting Result Time</div>" +
+    "<div id='peak_execution_memory' class='peak-execution-memory-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-17' data-column='17' data-metrics-type='task'> Peak Execution Memory</div>" +
+    "<div id='executor_jvm_on_off_heap_memory' class='executor-jvm-metrics-checkbox-div'><input type='checkbox' class='toggle-vis' id='executor-box-15'  data-column='15' data-metrics-type='executor'> Peak JVM Memory OnHeap / OffHeap</div>" +
+    "<div id='executor_on_off_heap_execution_memory' class='executor-jvm-metrics-checkbox-div'><input type='checkbox' class='toggle-vis' id='executor-box-16' data-column='16' data-metrics-type='executor'> Peak Execution Memory OnHeap / OffHeap</div>" +
+    "<div id='executor_on_off_heap_storage_memory' class='executor-jvm-metrics-checkbox-div'><input type='checkbox' class='toggle-vis' id='executor-box-17' data-column='17' data-metrics-type='executor'> Peak Storage Memory OnHeap / OffHeap</div>" +
+    "<div id='executor_direct_mapped_pool_memory' class='executor-jvm-metrics-checkbox-div'><input type='checkbox' class='toggle-vis' id='executor-box-18' data-column='18' data-metrics-type='executor'> Peak Pool Memory Direct / Mapped</div>" +
+    "</div>");
+
+  $('#scheduler_delay').attr("data-toggle", "tooltip")
+    .attr("data-placement", "top")
+    .attr("title", "Scheduler delay includes time to ship the task from the scheduler to the executor, and time to send " +
+      "the task result from the executor to the scheduler. If scheduler delay is large, consider decreasing the size of tasks or decreasing the size of task results.");
+  $('#task_deserialization_time').attr("data-toggle", "tooltip")
+    .attr("data-placement", "top")
+    .attr("title", "Time spent deserializing the task closure on the executor, including the time to read the broadcasted task.");
+  $('#shuffle_read_blocked_time').attr("data-toggle", "tooltip")
+    .attr("data-placement", "top")
+    .attr("title", "Time that the task spent blocked waiting for shuffle data to be read from remote machines.");
+  $('#shuffle_remote_reads').attr("data-toggle", "tooltip")
+    .attr("data-placement", "top")
+    .attr("title", "Total shuffle bytes read from remote executors. This is a subset of the shuffle read bytes; the remaining shuffle data is read locally. ");
+  $('#shuffle_write_time').attr("data-toggle", "tooltip")
+    .attr("data-placement", "top")
+    .attr("title", "Time that the task spent writing shuffle data.");
+  $('#result_serialization_time').attr("data-toggle", "tooltip")
+    .attr("data-placement", "top")
+    .attr("title", "Time spent serializing the task result on the executor before sending it back to the driver.");
+  $('#getting_result_time').attr("data-toggle", "tooltip")
+    .attr("data-placement", "top")
+    .attr("title", "Time that the driver spends fetching task results from workers. If this is large, consider decreasing the amount of data returned from each task.");
+  $('#peak_execution_memory').attr("data-toggle", "tooltip")
+    .attr("data-placement", "top")
+    .attr("title", "Execution memory refers to the memory used by internal data structures created during " +
+      "shuffles, aggregations and joins when Tungsten is enabled. The value of this accumulator " +
+      "should be approximately the sum of the peak sizes across all such data structures created " +
+      "in this task. For SQL jobs, this only tracks all unsafe operators, broadcast joins, and " +
+      "external sort.");
+  $('[data-toggle="tooltip"]').tooltip();
+  var tasksSummary = $("#parent-container");
+  getStandAloneAppId(function (appId) {
+    // rendering the UI page
+    $.get(createTemplateURI(appId, "stagespage"), function(template) {
+      tasksSummary.append(Mustache.render($(template).filter("#stages-summary-template").html()));
+
+      $("#additionalMetrics").click(function(){
+        $("#arrowtoggle1").toggleClass("arrow-open arrow-closed");
+        $("#toggle-metrics").toggleClass("d-none");
+        if (window.localStorage) {
+          window.localStorage.setItem("arrowtoggle1class", $("#arrowtoggle1").attr('class'));
+        }
+      });
+
+      $("#aggregatedMetrics").click(function(){
+        $("#arrowtoggle2").toggleClass("arrow-open arrow-closed");
+        $("#toggle-aggregatedMetrics").toggleClass("d-none");
+        if (window.localStorage) {
+          window.localStorage.setItem("arrowtoggle2class", $("#arrowtoggle2").attr('class'));
+        }
+      });
+
+      var endPoint = stageEndPoint(appId);
+      var stageAttemptId = getStageAttemptId();
+      $.getJSON(endPoint + "/" + stageAttemptId, function(response, _ignored_status, _ignored_jqXHR) {
+  
+        var responseBody = response;
+        var dataToShow = {};
+        dataToShow.showInputData = responseBody.inputBytes > 0;
+        dataToShow.showOutputData = responseBody.outputBytes > 0;
+        dataToShow.showShuffleReadData = responseBody.shuffleReadBytes > 0;
+        dataToShow.showShuffleWriteData = responseBody.shuffleWriteBytes > 0;
+        dataToShow.showBytesSpilledData =
+          (responseBody.diskBytesSpilled > 0 || responseBody.memoryBytesSpilled > 0);
+  
+        var columnIndicesToRemove = [];
+        if (!dataToShow.showShuffleReadData) {
+          $('#shuffle_read_blocked_time').remove();
+          $('#shuffle_remote_reads').remove();
+          columnIndicesToRemove.push(2);
+          columnIndicesToRemove.push(3);
+        }
+  
+        if (!dataToShow.showShuffleWriteData) {
+          $('#shuffle_write_time').remove();
+          columnIndicesToRemove.push(7);
+        }
+  
+        if (columnIndicesToRemove.length > 0) {
+          columnIndicesToRemove.sort(function(a, b) { return b - a; });
+          columnIndicesToRemove.forEach(function(idx) {
+            optionalColumns.splice(idx, 1);
           });
+        }
+  
+        // prepare data for executor summary table
+        var stageExecutorSummaryInfoKeys = Object.keys(responseBody.executorSummary);
+        $.getJSON(createRESTEndPointForExecutorsPage(appId),
+          function(executorSummaryResponse, _ignored_status, _ignored_jqXHR) {
+            var executorDetailsMap = {};
+            executorSummaryResponse.forEach(function (executorDetail) {
+              executorDetailsMap[executorDetail.id] = executorDetail;
+            });
+  
+            var executorSummaryTable = [];
+            stageExecutorSummaryInfoKeys.forEach(function (columnKeyIndex) {
+              var executorSummary = responseBody.executorSummary[columnKeyIndex];
+              var executorDetail = executorDetailsMap[columnKeyIndex.toString()];
+              executorSummary.id = columnKeyIndex;
+              executorSummary.executorLogs = {};
+              executorSummary.hostPort = "CANNOT FIND ADDRESS";
+  
+              if (executorDetail) {
+                if (executorDetail["executorLogs"]) {
+                  responseBody.executorSummary[columnKeyIndex].executorLogs =
+                    executorDetail["executorLogs"];
+                }
+                if (executorDetail["hostPort"]) {
+                  responseBody.executorSummary[columnKeyIndex].hostPort =
+                    executorDetail["hostPort"];
+                }
+              }
+              executorSummaryTable.push(responseBody.executorSummary[columnKeyIndex]);
+            });
+            // building task aggregated metrics by executor table
+            var executorSummaryConf = {
+              "data": executorSummaryTable,
+              "columns": [
+                {data : "id"},
+                {data : "executorLogs", render: formatLogsCells},
+                {data : "hostPort"},
+                {
+                  data : function (row, type) {
+                    return type === 'display' ? formatDuration(row.taskTime) : row.taskTime;
+                  }
+                },
+                {
+                  data : function (row, type) {
+                    var totaltasks = row.succeededTasks + row.failedTasks + row.killedTasks;
+                    return type === 'display' ? totaltasks : totaltasks.toString();
+                  }
+                },
+                {data : "failedTasks"},
+                {data : "killedTasks"},
+                {data : "succeededTasks"},
+                {data : "isExcludedForStage"},
+                {
+                  data : function (row, type) {
+                    return row.inputRecords != 0 ? formatBytes(row.inputBytes, type) + " / " + row.inputRecords : "";
+                  }
+                },
+                {
+                  data : function (row, type) {
+                    return row.outputRecords != 0 ? formatBytes(row.outputBytes, type) + " / " + row.outputRecords : "";
+                  }
+                },
+                {
+                  data : function (row, type) {
+                    return row.shuffleReadRecords != 0 ? formatBytes(row.shuffleRead, type) + " / " + row.shuffleReadRecords : "";
+                  }
+                },
+                {
+                  data : function (row, type) {
+                    return row.shuffleWriteRecords != 0 ? formatBytes(row.shuffleWrite, type) + " / " + row.shuffleWriteRecords : "";
+                  }
+                },
+                {
+                  data : function (row, type) {
+                    return typeof row.memoryBytesSpilled != 'undefined' ? formatBytes(row.memoryBytesSpilled, type) : "";
+                  }
+                },
+                {
+                  data : function (row, type) {
+                    return typeof row.diskBytesSpilled != 'undefined' ? formatBytes(row.diskBytesSpilled, type) : "";
+                  }
+                },
+                {
+                  data : function (row, type) {
+                    var peakMemoryMetrics = row.peakMemoryMetrics;
+                    if (typeof peakMemoryMetrics !== 'undefined') {
+                      if (type !== 'display')
+                        return peakMemoryMetrics.JVMHeapMemory;
+                      else
+                        return (formatBytes(peakMemoryMetrics.JVMHeapMemory, type) + ' / ' +
+                          formatBytes(peakMemoryMetrics.JVMOffHeapMemory, type));
+                    } else {
+                      if (type !== 'display') {
+                        return 0;
+                      } else {
+                        return '0.0 B / 0.0 B';
+                      }
+                    }
 
-          $("#aggregatedMetrics").click(function(){
-              $("#arrowtoggle2").toggleClass("arrow-open arrow-closed");
-              $("#toggle-aggregatedMetrics").toggleClass("d-none");
-              if (window.localStorage) {
-                  window.localStorage.setItem("arrowtoggle2class", $("#arrowtoggle2").attr('class'));
+                  }
+                },
+                {
+                  data : function (row, type) {
+                    var peakMemoryMetrics = row.peakMemoryMetrics
+                    if (typeof peakMemoryMetrics !== 'undefined') {
+                      if (type !== 'display')
+                        return peakMemoryMetrics.OnHeapExecutionMemory;
+                      else
+                        return (formatBytes(peakMemoryMetrics.OnHeapExecutionMemory, type) + ' / ' +
+                          formatBytes(peakMemoryMetrics.OffHeapExecutionMemory, type));
+                    } else {
+                      if (type !== 'display') {
+                        return 0;
+                      } else {
+                        return '0.0 B / 0.0 B';
+                      }
+                    }
+                  }
+                },
+                {
+                  data : function (row, type) {
+                    var peakMemoryMetrics = row.peakMemoryMetrics
+                    if (typeof peakMemoryMetrics !== 'undefined') {
+                      if (type !== 'display')
+                        return peakMemoryMetrics.OnHeapStorageMemory;
+                      else
+                        return (formatBytes(peakMemoryMetrics.OnHeapStorageMemory, type) + ' / ' +
+                          formatBytes(peakMemoryMetrics.OffHeapStorageMemory, type));
+                    } else {
+                      if (type !== 'display') {
+                        return 0;
+                      } else {
+                        return '0.0 B / 0.0 B';
+                      }
+                    }
+                  }
+                },
+                {
+                  data : function (row, type) {
+                    var peakMemoryMetrics = row.peakMemoryMetrics
+                    if (typeof peakMemoryMetrics !== 'undefined') {
+                      if (type !== 'display')
+                        return peakMemoryMetrics.DirectPoolMemory;
+                      else
+                        return (formatBytes(peakMemoryMetrics.DirectPoolMemory, type) + ' / ' +
+                          formatBytes(peakMemoryMetrics.MappedPoolMemory, type));
+                    } else {
+                      if (type !== 'display') {
+                        return 0;
+                      } else {
+                        return '0.0 B / 0.0 B';
+                      }
+                    }
+                  }
+                }
+              ],
+              "columnDefs": [
+                // SPARK-35087 [type:size] means String with structures like : 'size / records',
+                // they should be sorted as numerical-order instead of lexicographical-order by default.
+                // The targets: $id represents column id which comes from stagespage-template.html
+                // #summary-executor-table.If the relative position of the columns in the table
+                // #summary-executor-table has changed,please be careful to adjust the column index here
+                // Input Size / Records
+                {"type": "size", "targets": 9},
+                // Output Size / Records
+                {"type": "size", "targets": 10},
+                // Shuffle Read Size / Records
+                {"type": "size", "targets": 11},
+                // Shuffle Write Size / Records
+                {"type": "size", "targets": 12},
+                // Peak JVM Memory OnHeap / OffHeap
+                {"visible": false, "targets": 15},
+                // Peak Execution Memory OnHeap / OffHeap
+                {"visible": false, "targets": 16},
+                // Peak Storage Memory OnHeap / OffHeap
+                {"visible": false, "targets": 17},
+                // Peak Pool Memory Direct / Mapped
+                {"visible": false, "targets": 18}
+              ],
+              "deferRender": true,
+              "order": [[0, "asc"]],
+              "bAutoWidth": false,
+              "oLanguage": {
+                "sEmptyTable": "No data to show yet"
               }
+            };
+            executorSummaryTableSelector =
+              $("#summary-executor-table").DataTable(executorSummaryConf);
+            $('#parent-container [data-toggle="tooltip"]').tooltip();
+  
+            executorSummaryTableSelector.column(9).visible(dataToShow.showInputData);
+            if (dataToShow.showInputData) {
+              $('#executor-summary-input').attr("data-toggle", "tooltip")
+                .attr("data-placement", "top")
+                .attr("title", "Bytes and records read from Hadoop or from Spark storage.");
+              $('#executor-summary-input').tooltip(true);
+            }
+            executorSummaryTableSelector.column(10).visible(dataToShow.showOutputData);
+            if (dataToShow.showOutputData) {
+              $('#executor-summary-output').attr("data-toggle", "tooltip")
+                .attr("data-placement", "top")
+                .attr("title", "Bytes and records written to Hadoop.");
+              $('#executor-summary-output').tooltip(true);
+            }
+            executorSummaryTableSelector.column(11).visible(dataToShow.showShuffleReadData);
+            if (dataToShow.showShuffleReadData) {
+              $('#executor-summary-shuffle-read').attr("data-toggle", "tooltip")
+                .attr("data-placement", "top")
+                .attr("title", "Total shuffle bytes and records read (includes both data read locally and data read from remote executors).");
+              $('#executor-summary-shuffle-read').tooltip(true);
+            }
+            executorSummaryTableSelector.column(12).visible(dataToShow.showShuffleWriteData);
+            if (dataToShow.showShuffleWriteData) {
+              $('#executor-summary-shuffle-write').attr("data-toggle", "tooltip")
+                .attr("data-placement", "top")
+                .attr("title", "Bytes and records written to disk in order to be read by a shuffle in a future stage.");
+              $('#executor-summary-shuffle-write').tooltip(true);
+            }
+            executorSummaryTableSelector.column(13).visible(dataToShow.showBytesSpilledData);
+            executorSummaryTableSelector.column(14).visible(dataToShow.showBytesSpilledData);
           });
 
-        var endPoint = stageEndPoint(appId);
-        var stageAttemptId = getStageAttemptId();
-        $.getJSON(endPoint + "/" + stageAttemptId, function(response, status, jqXHR) {
-
-            var responseBody = response;
-            var dataToShow = {};
-            dataToShow.showInputData = responseBody.inputBytes > 0;
-            dataToShow.showOutputData = responseBody.outputBytes > 0;
-            dataToShow.showShuffleReadData = responseBody.shuffleReadBytes > 0;
-            dataToShow.showShuffleWriteData = responseBody.shuffleWriteBytes > 0;
-            dataToShow.showBytesSpilledData =
-                (responseBody.diskBytesSpilled > 0 || responseBody.memoryBytesSpilled > 0);
-
-            var columnIndicesToRemove = [];
-            if (!dataToShow.showShuffleReadData) {
-                $('#shuffle_read_blocked_time').remove();
-                $('#shuffle_remote_reads').remove();
-                columnIndicesToRemove.push(2);
-                columnIndicesToRemove.push(3);
-            }
+        // prepare data for accumulatorUpdates
+        var accumulatorTable = responseBody.accumulatorUpdates.filter(accumUpdate =>
+          !(accumUpdate.name).toString().includes("internal."));
+  
+        var quantiles = "0,0.25,0.5,0.75,1.0";
+        $.getJSON(endPoint + "/" + stageAttemptId + "/taskSummary?quantiles=" + quantiles,
+          function(taskMetricsResponse, _ignored_status, _ignored_jqXHR) {
+            var taskMetricKeys = Object.keys(taskMetricsResponse);
+            taskMetricKeys.forEach(function (columnKey) {
+              var row;
+              var row1;
+              var row2;
+              var row3;
+              switch(columnKey) {
+                case "shuffleReadMetrics":
+                  row1 = createRowMetadataForColumn(
+                    columnKey, taskMetricsResponse[columnKey], 3);
+                  row2 = createRowMetadataForColumn(
+                    "shuffleReadBlockedTime", taskMetricsResponse[columnKey], 13);
+                  row3 = createRowMetadataForColumn(
+                    "shuffleRemoteReads", taskMetricsResponse[columnKey], 14);
+                  if (dataToShow.showShuffleReadData) {
+                    taskSummaryMetricsTableArray.push(row1);
+                    taskSummaryMetricsTableArray.push(row2);
+                    taskSummaryMetricsTableArray.push(row3);
+                  }
+                  break;
+
+                case "schedulerDelay":
+                  row = createRowMetadataForColumn(
+                    columnKey, taskMetricsResponse[columnKey], 11);
+                  taskSummaryMetricsTableArray.push(row);
+                  break;
+
+                case "executorDeserializeTime":
+                  row = createRowMetadataForColumn(
+                    columnKey, taskMetricsResponse[columnKey], 12);
+                  taskSummaryMetricsTableArray.push(row);
+                  break;
+
+                case "resultSerializationTime":
+                  row = createRowMetadataForColumn(
+                    columnKey, taskMetricsResponse[columnKey], 15);
+                  taskSummaryMetricsTableArray.push(row);
+                  break;
+
+                case "gettingResultTime":
+                  row = createRowMetadataForColumn(
+                    columnKey, taskMetricsResponse[columnKey], 16);
+                  taskSummaryMetricsTableArray.push(row);
+                  break;
+
+                case "peakExecutionMemory":
+                  row = createRowMetadataForColumn(
+                    columnKey, taskMetricsResponse[columnKey], 17);
+                  taskSummaryMetricsTableArray.push(row);
+                  break;
+
+                case "inputMetrics":
+                  row = createRowMetadataForColumn(
+                    columnKey, taskMetricsResponse[columnKey], 1);
+                  if (dataToShow.showInputData) {
+                    taskSummaryMetricsTableArray.push(row);
+                  }
+                  break;
 
-            if (!dataToShow.showShuffleWriteData) {
-                $('#shuffle_write_time').remove();
-                columnIndicesToRemove.push(7);
-            }
+                case "outputMetrics":
+                  row = createRowMetadataForColumn(
+                    columnKey, taskMetricsResponse[columnKey], 2);
+                  if (dataToShow.showOutputData) {
+                    taskSummaryMetricsTableArray.push(row);
+                  }
+                  break;
+
+                case "shuffleWriteMetrics":
+                  row1 = createRowMetadataForColumn(
+                    columnKey, taskMetricsResponse[columnKey], 4);
+                  row2 = createRowMetadataForColumn(
+                    "shuffleWriteTime", taskMetricsResponse[columnKey], 21);
+                  if (dataToShow.showShuffleWriteData) {
+                    taskSummaryMetricsTableArray.push(row1);
+                    taskSummaryMetricsTableArray.push(row2);
+                  }
+                  break;
 
-            if (columnIndicesToRemove.length > 0) {
-                columnIndicesToRemove.sort(function(a, b) { return b - a; });
-                columnIndicesToRemove.forEach(function(idx) {
-                   optionalColumns.splice(idx, 1);
-                });
-            }
+                case "diskBytesSpilled":
+                  row = createRowMetadataForColumn(
+                    columnKey, taskMetricsResponse[columnKey], 5);
+                  if (dataToShow.showBytesSpilledData) {
+                    taskSummaryMetricsTableArray.push(row);
+                  }
+                  break;
 
-            // prepare data for executor summary table
-            var stageExecutorSummaryInfoKeys = Object.keys(responseBody.executorSummary);
-            $.getJSON(createRESTEndPointForExecutorsPage(appId),
-              function(executorSummaryResponse, status, jqXHR) {
-                var executorDetailsMap = {};
-                executorSummaryResponse.forEach(function (executorDetail) {
-                    executorDetailsMap[executorDetail.id] = executorDetail;
-                });
-
-                var executorSummaryTable = [];
-                stageExecutorSummaryInfoKeys.forEach(function (columnKeyIndex) {
-                    var executorSummary = responseBody.executorSummary[columnKeyIndex];
-                    var executorDetail = executorDetailsMap[columnKeyIndex.toString()];
-                    executorSummary.id = columnKeyIndex;
-                    executorSummary.executorLogs = {};
-                    executorSummary.hostPort = "CANNOT FIND ADDRESS";
-
-                    if (executorDetail) {
-                        if (executorDetail["executorLogs"]) {
-                            responseBody.executorSummary[columnKeyIndex].executorLogs =
-                                executorDetail["executorLogs"];
-                            }
-                        if (executorDetail["hostPort"]) {
-                            responseBody.executorSummary[columnKeyIndex].hostPort =
-                                executorDetail["hostPort"];
-                        }
-                    }
-                    executorSummaryTable.push(responseBody.executorSummary[columnKeyIndex]);
-                });
-                // building task aggregated metrics by executor table
-                var executorSummaryConf = {
-                    "data": executorSummaryTable,
-                    "columns": [
-                        {data : "id"},
-                        {data : "executorLogs", render: formatLogsCells},
-                        {data : "hostPort"},
-                        {
-                            data : function (row, type) {
-                                return type === 'display' ? formatDuration(row.taskTime) : row.taskTime;
-                            }
-                        },
-                        {
-                            data : function (row, type) {
-                                var totaltasks = row.succeededTasks + row.failedTasks + row.killedTasks;
-                                return type === 'display' ? totaltasks : totaltasks.toString();
-                            }
-                        },
-                        {data : "failedTasks"},
-                        {data : "killedTasks"},
-                        {data : "succeededTasks"},
-                        {data : "isBlacklistedForStage"},
-                        {
-                            data : function (row, type) {
-                                return row.inputRecords != 0 ? formatBytes(row.inputBytes, type) + " / " + row.inputRecords : "";
-                            }
-                        },
-                        {
-                            data : function (row, type) {
-                                return row.outputRecords != 0 ? formatBytes(row.outputBytes, type) + " / " + row.outputRecords : "";
-                            }
-                        },
-                        {
-                            data : function (row, type) {
-                                return row.shuffleReadRecords != 0 ? formatBytes(row.shuffleRead, type) + " / " + row.shuffleReadRecords : "";
-                            }
-                        },
-                        {
-                            data : function (row, type) {
-                                return row.shuffleWriteRecords != 0 ? formatBytes(row.shuffleWrite, type) + " / " + row.shuffleWriteRecords : "";
-                            }
-                        },
-                        {
-                            data : function (row, type) {
-                                return typeof row.memoryBytesSpilled != 'undefined' ? formatBytes(row.memoryBytesSpilled, type) : "";
-                            }
-                        },
-                        {
-                            data : function (row, type) {
-                                return typeof row.diskBytesSpilled != 'undefined' ? formatBytes(row.diskBytesSpilled, type) : "";
-                            }
-                        }
-                    ],
-                    "order": [[0, "asc"]],
-                    "bAutoWidth": false,
-                    "oLanguage": {
-                        "sEmptyTable": "No data to show yet"
-                    }
-                };
-                var executorSummaryTableSelector =
-                    $("#summary-executor-table").DataTable(executorSummaryConf);
-                $('#parent-container [data-toggle="tooltip"]').tooltip();
-
-                executorSummaryTableSelector.column(9).visible(dataToShow.showInputData);
-                if (dataToShow.showInputData) {
-                    $('#executor-summary-input').attr("data-toggle", "tooltip")
-                        .attr("data-placement", "top")
-                        .attr("title", "Bytes and records read from Hadoop or from Spark storage.");
-                    $('#executor-summary-input').tooltip(true);
+                case "memoryBytesSpilled":
+                  row = createRowMetadataForColumn(
+                    columnKey, taskMetricsResponse[columnKey], 6);
+                  if (dataToShow.showBytesSpilledData) {
+                    taskSummaryMetricsTableArray.push(row);
+                  }
+                  break;
+
+                default:
+                  if (getColumnNameForTaskMetricSummary(columnKey) != "NA") {
+                    row = createRowMetadataForColumn(
+                      columnKey, taskMetricsResponse[columnKey], 0);
+                    taskSummaryMetricsTableArray.push(row);
+                  }
+                  break;
+              }
+            });
+            var taskSummaryMetricsTableFilteredArray =
+              taskSummaryMetricsTableArray.filter(row => row.checkboxId < 11);
+            taskSummaryMetricsTableCurrentStateArray = taskSummaryMetricsTableFilteredArray.slice();
+            reselectCheckboxesBasedOnTaskTableState();
+          });
+
+        // building accumulator update table
+        var accumulatorConf = {
+          "data": accumulatorTable,
+          "columns": [
+            {data : "id"},
+            {data : "name"},
+            {data : "value"}
+          ],
+          "paging": false,
+          "searching": false,
+          "order": [[0, "asc"]],
+          "bAutoWidth": false
+        };
+        $("#accumulator-table").DataTable(accumulatorConf);
+
+        // building tasks table that uses server side functionality
+        var totalTasksToShow = responseBody.numCompleteTasks + responseBody.numActiveTasks +
+          responseBody.numKilledTasks + responseBody.numFailedTasks;
+        var taskTable = "#active-tasks-table";
+        var taskConf = {
+          "serverSide": true,
+          "paging": true,
+          "info": true,
+          "processing": true,
+          "lengthMenu": [[20, 40, 60, 100, -1], [20, 40, 60, 100, "All"]],
+          "orderMulti": false,
+          "bAutoWidth": false,
+          "ajax": {
+            "url": endPoint + "/" + stageAttemptId + "/taskTable",
+            "data": function (data) {
+              var columnIndexToSort = 0;
+              var columnNameToSort = "Index";
+              if (data.order[0].column && data.order[0].column != "") {
+                columnIndexToSort = parseInt(data.order[0].column);
+                columnNameToSort = data.columns[columnIndexToSort].name;
+              }
+              delete data.columns;
+              data.numTasks = totalTasksToShow;
+              data.columnIndexToSort = columnIndexToSort;
+              data.columnNameToSort = columnNameToSort;
+              if (data.length === -1) {
+                data.length = totalTasksToShow;
+              }
+            },
+            "dataSrc": function (jsons) {
+              var jsonStr = JSON.stringify(jsons);
+              var tasksToShow = JSON.parse(jsonStr);
+              return tasksToShow.aaData;
+            },
+            "error": function (_ignored_jqXHR, _ignored_textStatus, _ignored_errorThrown) {
+              alert("Unable to connect to the server. Looks like the Spark " +
+                "application must have ended. Please Switch to the history UI.");
+              $("#active-tasks-table_processing").css("display","none");
+            }
+          },
+          "columns": [
+            {
+              data: function (row, type) {
+                return type !== 'display' ? (isNaN(row.index) ? 0 : row.index ) : row.index;
+              },
+              name: "Index"
+            },
+            {data : "taskId", name: "ID"},
+            {data : "attempt", name: "Attempt"},
+            {data : "status", name: "Status"},
+            {data : "taskLocality", name: "Locality Level"},
+            {data : "executorId", name: "Executor ID"},
+            {data : "host", name: "Host"},
+            {data : "executorLogs", name: "Logs", render: formatLogsCells},
+            {data : "launchTime", name: "Launch Time", render: formatDate},
+            {
+              data : function (row, type) {
+                if (row.taskMetrics && row.taskMetrics.executorRunTime) {
+                  return type === 'display' ? formatDuration(row.taskMetrics.executorRunTime) : row.taskMetrics.executorRunTime;
+                } else {
+                  return "";
                 }
-                executorSummaryTableSelector.column(10).visible(dataToShow.showOutputData);
-                if (dataToShow.showOutputData) {
-                    $('#executor-summary-output').attr("data-toggle", "tooltip")
-                        .attr("data-placement", "top")
-                        .attr("title", "Bytes and records written to Hadoop.");
-                    $('#executor-summary-output').tooltip(true);
+              },
+              name: "Duration"
+            },
+            {
+              data : function (row, type) {
+                if (row.taskMetrics && row.taskMetrics.jvmGcTime) {
+                  return type === 'display' ? formatDuration(row.taskMetrics.jvmGcTime) : row.taskMetrics.jvmGcTime;
+                } else {
+                  return "";
                 }
-                executorSummaryTableSelector.column(11).visible(dataToShow.showShuffleReadData);
-                if (dataToShow.showShuffleReadData) {
-                    $('#executor-summary-shuffle-read').attr("data-toggle", "tooltip")
-                        .attr("data-placement", "top")
-                        .attr("title", "Total shuffle bytes and records read (includes both data read locally and data read from remote executors).");
-                    $('#executor-summary-shuffle-read').tooltip(true);
+              },
+              name: "GC Time"
+            },
+            {
+              data : function (row, type) {
+                if (row.schedulerDelay) {
+                  return type === 'display' ? formatDuration(row.schedulerDelay) : row.schedulerDelay;
+                } else {
+                  return "";
                 }
-                executorSummaryTableSelector.column(12).visible(dataToShow.showShuffleWriteData);
-                if (dataToShow.showShuffleWriteData) {
-                    $('#executor-summary-shuffle-write').attr("data-toggle", "tooltip")
-                        .attr("data-placement", "top")
-                        .attr("title", "Bytes and records written to disk in order to be read by a shuffle in a future stage.");
-                    $('#executor-summary-shuffle-write').tooltip(true);
+              },
+              name: "Scheduler Delay"
+            },
+            {
+              data : function (row, type) {
+                if (row.taskMetrics && row.taskMetrics.executorDeserializeTime) {
+                  return type === 'display' ? formatDuration(row.taskMetrics.executorDeserializeTime) : row.taskMetrics.executorDeserializeTime;
+                } else {
+                  return "";
                 }
-                executorSummaryTableSelector.column(13).visible(dataToShow.showBytesSpilledData);
-                executorSummaryTableSelector.column(14).visible(dataToShow.showBytesSpilledData);
-            });
-
-            // prepare data for accumulatorUpdates
-            var accumulatorTable = responseBody.accumulatorUpdates.filter(accumUpdate =>
-                !(accumUpdate.name).toString().includes("internal."));
-
-                var quantiles = "0,0.25,0.5,0.75,1.0";
-                $.getJSON(endPoint + "/" + stageAttemptId + "/taskSummary?quantiles=" + quantiles,
-                  function(taskMetricsResponse, status, jqXHR) {
-                    var taskMetricKeys = Object.keys(taskMetricsResponse);
-                    taskMetricKeys.forEach(function (columnKey) {
-                        switch(columnKey) {
-                            case "shuffleReadMetrics":
-                                var row1 = createRowMetadataForColumn(
-                                    columnKey, taskMetricsResponse[columnKey], 3);
-                                var row2 = createRowMetadataForColumn(
-                                    "shuffleReadBlockedTime", taskMetricsResponse[columnKey], 13);
-                                var row3 = createRowMetadataForColumn(
-                                    "shuffleRemoteReads", taskMetricsResponse[columnKey], 14);
-                                if (dataToShow.showShuffleReadData) {
-                                    taskSummaryMetricsTableArray.push(row1);
-                                    taskSummaryMetricsTableArray.push(row2);
-                                    taskSummaryMetricsTableArray.push(row3);
-                                }
-                                break;
-
-                            case "schedulerDelay":
-                                var row = createRowMetadataForColumn(
-                                    columnKey, taskMetricsResponse[columnKey], 11);
-                                taskSummaryMetricsTableArray.push(row);
-                                break;
-
-                            case "executorDeserializeTime":
-                                var row = createRowMetadataForColumn(
-                                    columnKey, taskMetricsResponse[columnKey], 12);
-                                taskSummaryMetricsTableArray.push(row);
-                                break;
-
-                            case "resultSerializationTime":
-                                var row = createRowMetadataForColumn(
-                                    columnKey, taskMetricsResponse[columnKey], 15);
-                                taskSummaryMetricsTableArray.push(row);
-                                break;
-
-                            case "gettingResultTime":
-                                var row = createRowMetadataForColumn(
-                                    columnKey, taskMetricsResponse[columnKey], 16);
-                                taskSummaryMetricsTableArray.push(row);
-                                break;
-
-                            case "peakExecutionMemory":
-                                var row = createRowMetadataForColumn(
-                                    columnKey, taskMetricsResponse[columnKey], 17);
-                                taskSummaryMetricsTableArray.push(row);
-                                break;
-
-                            case "inputMetrics":
-                                var row = createRowMetadataForColumn(
-                                    columnKey, taskMetricsResponse[columnKey], 1);
-                                if (dataToShow.showInputData) {
-                                    taskSummaryMetricsTableArray.push(row);
-                                }
-                                break;
-
-                            case "outputMetrics":
-                                var row = createRowMetadataForColumn(
-                                    columnKey, taskMetricsResponse[columnKey], 2);
-                                if (dataToShow.showOutputData) {
-                                    taskSummaryMetricsTableArray.push(row);
-                                }
-                                break;
-
-                            case "shuffleWriteMetrics":
-                                var row1 = createRowMetadataForColumn(
-                                    columnKey, taskMetricsResponse[columnKey], 4);
-                                var row2 = createRowMetadataForColumn(
-                                    "shuffleWriteTime", taskMetricsResponse[columnKey], 21);
-                                if (dataToShow.showShuffleWriteData) {
-                                    taskSummaryMetricsTableArray.push(row1);
-                                    taskSummaryMetricsTableArray.push(row2);
-                                }
-                                break;
-
-                            case "diskBytesSpilled":
-                                var row = createRowMetadataForColumn(
-                                    columnKey, taskMetricsResponse[columnKey], 5);
-                                if (dataToShow.showBytesSpilledData) {
-                                    taskSummaryMetricsTableArray.push(row);
-                                }
-                                break;
-
-                            case "memoryBytesSpilled":
-                                var row = createRowMetadataForColumn(
-                                    columnKey, taskMetricsResponse[columnKey], 6);
-                                if (dataToShow.showBytesSpilledData) {
-                                    taskSummaryMetricsTableArray.push(row);
-                                }
-                                break;
-
-                            default:
-                                if (getColumnNameForTaskMetricSummary(columnKey) != "NA") {
-                                    var row = createRowMetadataForColumn(
-                                        columnKey, taskMetricsResponse[columnKey], 0);
-                                    taskSummaryMetricsTableArray.push(row);
-                                }
-                                break;
-                        }
-                    });
-                    var taskSummaryMetricsTableFilteredArray =
-                        taskSummaryMetricsTableArray.filter(row => row.checkboxId < 11);
-                    taskSummaryMetricsTableCurrentStateArray = taskSummaryMetricsTableFilteredArray.slice();
-                    reselectCheckboxesBasedOnTaskTableState();
-                });
-
-                // building accumulator update table
-                var accumulatorConf = {
-                    "data": accumulatorTable,
-                    "columns": [
-                        {data : "id"},
-                        {data : "name"},
-                        {data : "value"}
-                    ],
-                    "paging": false,
-                    "searching": false,
-                    "order": [[0, "asc"]],
-                    "bAutoWidth": false
-                };
-                $("#accumulator-table").DataTable(accumulatorConf);
-
-                // building tasks table that uses server side functionality
-                var totalTasksToShow = responseBody.numCompleteTasks + responseBody.numActiveTasks +
-                    responseBody.numKilledTasks + responseBody.numFailedTasks;
-                var taskTable = "#active-tasks-table";
-                var taskConf = {
-                    "serverSide": true,
-                    "paging": true,
-                    "info": true,
-                    "processing": true,
-                    "lengthMenu": [[20, 40, 60, 100, totalTasksToShow], [20, 40, 60, 100, "All"]],
-                    "orderMulti": false,
-                    "bAutoWidth": false,
-                    "ajax": {
-                        "url": endPoint + "/" + stageAttemptId + "/taskTable",
-                        "data": function (data) {
-                            var columnIndexToSort = 0;
-                            var columnNameToSort = "Index";
-                            if (data.order[0].column && data.order[0].column != "") {
-                                columnIndexToSort = parseInt(data.order[0].column);
-                                columnNameToSort = data.columns[columnIndexToSort].name;
-                            }
-                            delete data.columns;
-                            data.numTasks = totalTasksToShow;
-                            data.columnIndexToSort = columnIndexToSort;
-                            data.columnNameToSort = columnNameToSort;
-                        },
-                        "dataSrc": function (jsons) {
-                            var jsonStr = JSON.stringify(jsons);
-                            var tasksToShow = JSON.parse(jsonStr);
-                            return tasksToShow.aaData;
-                        },
-                        "error": function (jqXHR, textStatus, errorThrown) {
-                            alert("Unable to connect to the server. Looks like the Spark " +
-                              "application must have ended. Please Switch to the history UI.");
-                            $("#active-tasks-table_processing").css("display","none");
-                        }
-                    },
-                    "columns": [
-                        {data: function (row, type) {
-                            return type !== 'display' ? (isNaN(row.index) ? 0 : row.index ) : row.index;
-                            },
-                            name: "Index"
-                        },
-                        {data : "taskId", name: "ID"},
-                        {data : "attempt", name: "Attempt"},
-                        {data : "status", name: "Status"},
-                        {data : "taskLocality", name: "Locality Level"},
-                        {data : "executorId", name: "Executor ID"},
-                        {data : "host", name: "Host"},
-                        {data : "executorLogs", name: "Logs", render: formatLogsCells},
-                        {data : "launchTime", name: "Launch Time", render: formatDate},
-                        {
-                            data : function (row, type) {
-                                if (row.taskMetrics && row.taskMetrics.executorRunTime) {
-                                    return type === 'display' ? formatDuration(row.taskMetrics.executorRunTime) : row.taskMetrics.executorRunTime;
-                                } else {
-                                    return "";
-                                }
-                            },
-                            name: "Duration"
-                        },
-                        {
-                            data : function (row, type) {
-                                if (row.taskMetrics && row.taskMetrics.jvmGcTime) {
-                                    return type === 'display' ? formatDuration(row.taskMetrics.jvmGcTime) : row.taskMetrics.jvmGcTime;
-                                } else {
-                                    return "";
-                                }
-                            },
-                            name: "GC Time"
-                        },
-                        {
-                            data : function (row, type) {
-                                if (row.schedulerDelay) {
-                                    return type === 'display' ? formatDuration(row.schedulerDelay) : row.schedulerDelay;
-                                } else {
-                                    return "";
-                                }
-                            },
-                            name: "Scheduler Delay"
-                        },
-                        {
-                            data : function (row, type) {
-                                if (row.taskMetrics && row.taskMetrics.executorDeserializeTime) {
-                                    return type === 'display' ? formatDuration(row.taskMetrics.executorDeserializeTime) : row.taskMetrics.executorDeserializeTime;
-                                } else {
-                                    return "";
-                                }
-                            },
-                            name: "Task Deserialization Time"
-                        },
-                        {
-                            data : function (row, type) {
-                                if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics) {
-                                    return type === 'display' ? formatDuration(row.taskMetrics.shuffleReadMetrics.fetchWaitTime) : row.taskMetrics.shuffleReadMetrics.fetchWaitTime;
-                                } else {
-                                    return "";
-                                }
-                            },
-                            name: "Shuffle Read Blocked Time"
-                        },
-                        {
-                            data : function (row, type) {
-                                if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics) {
-                                    return type === 'display' ? formatBytes(row.taskMetrics.shuffleReadMetrics.remoteBytesRead, type) : row.taskMetrics.shuffleReadMetrics.remoteBytesRead;
-                                } else {
-                                    return "";
-                                }
-                            },
-                            name: "Shuffle Remote Reads"
-                        },
-                        {
-                            data : function (row, type) {
-                                if (row.taskMetrics && row.taskMetrics.resultSerializationTime) {
-                                    return type === 'display' ? formatDuration(row.taskMetrics.resultSerializationTime) : row.taskMetrics.resultSerializationTime;
-                                } else {
-                                    return "";
-                                }
-                            },
-                            name: "Result Serialization Time"
-                        },
-                        {
-                            data : function (row, type) {
-                                if (row.gettingResultTime) {
-                                    return type === 'display' ? formatDuration(row.gettingResultTime) : row.gettingResultTime;
-                                } else {
-                                    return "";
-                                }
-                            },
-                            name: "Getting Result Time"
-                        },
-                        {
-                            data : function (row, type) {
-                                if (row.taskMetrics && row.taskMetrics.peakExecutionMemory) {
-                                    return type === 'display' ? formatBytes(row.taskMetrics.peakExecutionMemory, type) : row.taskMetrics.peakExecutionMemory;
-                                } else {
-                                    return "";
-                                }
-                            },
-                            name: "Peak Execution Memory"
-                        },
-                        {
-                            data : function (row, type) {
-                                if (accumulatorTable.length > 0 && row.accumulatorUpdates.length > 0) {
-                                    var allAccums = "";
-                                    row.accumulatorUpdates.forEach(function(accumulator) {
-                                        allAccums += accumulator.name + ': ' + accumulator.update + "<BR>";
-                                    });
-                                    return allAccums;
-                                } else {
-                                    return "";
-                                }
-                            },
-                            name: "Accumulators"
-                        },
-                        {
-                            data : function (row, type) {
-                                if (row.taskMetrics && row.taskMetrics.inputMetrics && row.taskMetrics.inputMetrics.bytesRead > 0) {
-                                    if (type === 'display') {
-                                        return formatBytes(row.taskMetrics.inputMetrics.bytesRead, type) + " / " + row.taskMetrics.inputMetrics.recordsRead;
-                                    } else {
-                                        return row.taskMetrics.inputMetrics.bytesRead + " / " + row.taskMetrics.inputMetrics.recordsRead;
-                                    }
-                                } else {
-                                    return "";
-                                }
-                            },
-                            name: "Input Size / Records"
-                        },
-                        {
-                            data : function (row, type) {
-                                if (row.taskMetrics && row.taskMetrics.outputMetrics && row.taskMetrics.outputMetrics.bytesWritten > 0) {
-                                    if (type === 'display') {
-                                        return formatBytes(row.taskMetrics.outputMetrics.bytesWritten, type) + " / " + row.taskMetrics.outputMetrics.recordsWritten;
-                                    } else {
-                                        return row.taskMetrics.outputMetrics.bytesWritten + " / " + row.taskMetrics.outputMetrics.recordsWritten;
-                                    }
-                                } else {
-                                    return "";
-                                }
-                            },
-                            name: "Output Size / Records"
-                        },
-                        {
-                            data : function (row, type) {
-                                if (row.taskMetrics && row.taskMetrics.shuffleWriteMetrics && row.taskMetrics.shuffleWriteMetrics.writeTime > 0) {
-                                    return type === 'display' ? formatDuration(parseInt(row.taskMetrics.shuffleWriteMetrics.writeTime) / 1000000.0) : row.taskMetrics.shuffleWriteMetrics.writeTime;
-                                } else {
-                                    return "";
-                                }
-                            },
-                            name: "Shuffle Write Time"
-                        },
-                        {
-                            data : function (row, type) {
-                                if (row.taskMetrics && row.taskMetrics.shuffleWriteMetrics && row.taskMetrics.shuffleWriteMetrics.bytesWritten > 0) {
-                                    if (type === 'display') {
-                                        return formatBytes(row.taskMetrics.shuffleWriteMetrics.bytesWritten, type) + " / " + row.taskMetrics.shuffleWriteMetrics.recordsWritten;
-                                    } else {
-                                        return row.taskMetrics.shuffleWriteMetrics.bytesWritten + " / " + row.taskMetrics.shuffleWriteMetrics.recordsWritten;
-                                    }
-                                } else {
-                                    return "";
-                                }
-                            },
-                            name: "Shuffle Write Size / Records"
-                        },
-                        {
-                            data : function (row, type) {
-                                if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics && row.taskMetrics.shuffleReadMetrics.localBytesRead > 0) {
-                                    var totalBytesRead = parseInt(row.taskMetrics.shuffleReadMetrics.localBytesRead) + parseInt(row.taskMetrics.shuffleReadMetrics.remoteBytesRead);
-                                    if (type === 'display') {
-                                        return formatBytes(totalBytesRead, type) + " / " + row.taskMetrics.shuffleReadMetrics.recordsRead;
-                                    } else {
-                                        return totalBytesRead + " / " + row.taskMetrics.shuffleReadMetrics.recordsRead;
-                                    }
-                                } else {
-                                    return "";
-                                }
-                            },
-                            name: "Shuffle Read Size / Records"
-                        },
-                        {
-                            data : function (row, type) {
-                                if (row.taskMetrics && row.taskMetrics.memoryBytesSpilled && row.taskMetrics.memoryBytesSpilled > 0) {
-                                    return type === 'display' ? formatBytes(row.taskMetrics.memoryBytesSpilled, type) : row.taskMetrics.memoryBytesSpilled;
-                                } else {
-                                    return "";
-                                }
-                            },
-                            name: "Spill (Memory)"
-                        },
-                        {
-                            data : function (row, type) {
-                                if (row.taskMetrics && row.taskMetrics.diskBytesSpilled && row.taskMetrics.diskBytesSpilled > 0) {
-                                    return type === 'display' ? formatBytes(row.taskMetrics.diskBytesSpilled, type) : row.taskMetrics.diskBytesSpilled;
-                                } else {
-                                    return "";
-                                }
-                            },
-                            name: "Spill (Disk)"
-                        },
-                        {
-                            data : function (row, type) {
-                                var msg = row.errorMessage;
-                                if (typeof msg === 'undefined') {
-                                    return "";
-                                } else {
-                                    var formHead = msg.substring(0, msg.indexOf("at"));
-                                    var form = "<span onclick=\"this.parentNode.querySelector('.stacktrace-details').classList.toggle('collapsed')\" class=\"expand-details\">+details</span>";
-                                    var formMsg = "<div class=\"stacktrace-details collapsed\"><pre>" + row.errorMessage + "</pre></div>";
-                                    return formHead + form + formMsg;
-                                }
-                            },
-                            name: "Errors"
-                        }
-                    ],
-                    "columnDefs": [
-                        { "visible": false, "targets": 11 },
-                        { "visible": false, "targets": 12 },
-                        { "visible": false, "targets": 13 },
-                        { "visible": false, "targets": 14 },
-                        { "visible": false, "targets": 15 },
-                        { "visible": false, "targets": 16 },
-                        { "visible": false, "targets": 17 },
-                        { "visible": false, "targets": 18 },
-                        { "visible": false, "targets": 21 }
-                    ],
-                    "deferRender": true
-                };
-                taskTableSelector = $(taskTable).DataTable(taskConf);
-                $('#active-tasks-table_filter input').unbind();
-                var searchEvent;
-                $('#active-tasks-table_filter input').bind('keyup', function(e) {
-                  if (typeof searchEvent !== 'undefined') {
-                    window.clearTimeout(searchEvent);
+              },
+              name: "Task Deserialization Time"
+            },
+            {
+              data : function (row, type) {
+                if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics) {
+                  return type === 'display' ? formatDuration(row.taskMetrics.shuffleReadMetrics.fetchWaitTime) : row.taskMetrics.shuffleReadMetrics.fetchWaitTime;
+                } else {
+                  return "";
+                }
+              },
+              name: "Shuffle Read Blocked Time"
+            },
+            {
+              data : function (row, type) {
+                if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics) {
+                  return type === 'display' ? formatBytes(row.taskMetrics.shuffleReadMetrics.remoteBytesRead, type) : row.taskMetrics.shuffleReadMetrics.remoteBytesRead;
+                } else {
+                  return "";
+                }
+              },
+              name: "Shuffle Remote Reads"
+            },
+            {
+              data : function (row, type) {
+                if (row.taskMetrics && row.taskMetrics.resultSerializationTime) {
+                  return type === 'display' ? formatDuration(row.taskMetrics.resultSerializationTime) : row.taskMetrics.resultSerializationTime;
+                } else {
+                  return "";
+                }
+              },
+              name: "Result Serialization Time"
+            },
+            {
+              data : function (row, type) {
+                if (row.gettingResultTime) {
+                  return type === 'display' ? formatDuration(row.gettingResultTime) : row.gettingResultTime;
+                } else {
+                  return "";
+                }
+              },
+              name: "Getting Result Time"
+            },
+            {
+              data : function (row, type) {
+                if (row.taskMetrics && row.taskMetrics.peakExecutionMemory) {
+                  return type === 'display' ? formatBytes(row.taskMetrics.peakExecutionMemory, type) : row.taskMetrics.peakExecutionMemory;
+                } else {
+                  return "";
+                }
+              },
+              name: "Peak Execution Memory"
+            },
+            {
+              data : function (row, _ignored_type) {
+                if (accumulatorTable.length > 0 && row.accumulatorUpdates.length > 0) {
+                  var allAccums = "";
+                  row.accumulatorUpdates.forEach(function(accumulator) {
+                    allAccums += accumulator.name + ': ' + accumulator.update + "<BR>";
+                  });
+                  return allAccums;
+                } else {
+                  return "";
+                }
+              },
+              name: "Accumulators"
+            },
+            {
+              data : function (row, type) {
+                if (row.taskMetrics && row.taskMetrics.inputMetrics && row.taskMetrics.inputMetrics.bytesRead > 0) {
+                  if (type === 'display') {
+                    return formatBytes(row.taskMetrics.inputMetrics.bytesRead, type) + " / " + row.taskMetrics.inputMetrics.recordsRead;
+                  } else {
+                    return row.taskMetrics.inputMetrics.bytesRead + " / " + row.taskMetrics.inputMetrics.recordsRead;
                   }
-                  var value = this.value;
-                  searchEvent = window.setTimeout(function(){
-                    taskTableSelector.search( value ).draw();}, 500);
-                });
-                reselectCheckboxesBasedOnTaskTableState();
-
-                // hide or show columns dynamically event
-                $('input.toggle-vis').on('click', function(e){
-                    // Get the column
-                    var para = $(this).attr('data-column');
-                    if (para == "0") {
-                        var allColumns = taskTableSelector.columns(optionalColumns);
-                        if ($(this).is(":checked")) {
-                            $(".toggle-vis").prop('checked', true);
-                            allColumns.visible(true);
-                            createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableArray);
-                        } else {
-                            $(".toggle-vis").prop('checked', false);
-                            allColumns.visible(false);
-                            var taskSummaryMetricsTableFilteredArray =
-                                taskSummaryMetricsTableArray.filter(row => row.checkboxId < 11);
-                            createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableFilteredArray);
-                        }
-                    } else {
-                        var column = taskTableSelector.column(para);
-                        // Toggle the visibility
-                        column.visible(!column.visible());
-                        var taskSummaryMetricsTableFilteredArray = [];
-                        if ($(this).is(":checked")) {
-                            taskSummaryMetricsTableCurrentStateArray.push(taskSummaryMetricsTableArray.filter(row => (row.checkboxId).toString() == para)[0]);
-                            taskSummaryMetricsTableFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice();
-                        } else {
-                            taskSummaryMetricsTableFilteredArray =
-                                taskSummaryMetricsTableCurrentStateArray.filter(row => (row.checkboxId).toString() != para);
-                        }
-                        createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableFilteredArray);
-                    }
-                });
-
-                // title number and toggle list
-                $("#summaryMetricsTitle").html("Summary Metrics for " + "<a href='#tasksTitle'>" + responseBody.numCompleteTasks + " Completed Tasks" + "</a>");
-                $("#tasksTitle").html("Tasks (" + totalTasksToShow + ")");
-
-                // hide or show the accumulate update table
-                if (accumulatorTable.length == 0) {
-                    $("#accumulator-update-table").hide();
                 } else {
-                    taskTableSelector.column(18).visible(true);
-                    $("#accumulator-update-table").show();
+                  return "";
                 }
-                // Showing relevant stage data depending on stage type for task table and executor
-                // summary table
-                taskTableSelector.column(19).visible(dataToShow.showInputData);
-                taskTableSelector.column(20).visible(dataToShow.showOutputData);
-                taskTableSelector.column(22).visible(dataToShow.showShuffleWriteData);
-                taskTableSelector.column(23).visible(dataToShow.showShuffleReadData);
-                taskTableSelector.column(24).visible(dataToShow.showBytesSpilledData);
-                taskTableSelector.column(25).visible(dataToShow.showBytesSpilledData);
-
-                if (window.localStorage) {
-                    if (window.localStorage.getItem("arrowtoggle1class") !== null &&
-                        window.localStorage.getItem("arrowtoggle1class").includes("arrow-open")) {
-                        $("#arrowtoggle1").toggleClass("arrow-open arrow-closed");
-                        $("#toggle-metrics").toggleClass("d-none");
-                    }
-                    if (window.localStorage.getItem("arrowtoggle2class") !== null &&
-                        window.localStorage.getItem("arrowtoggle2class").includes("arrow-open")) {
-                        $("#arrowtoggle2").toggleClass("arrow-open arrow-closed");
-                        $("#toggle-aggregatedMetrics").toggleClass("d-none");
-                    }
+              },
+              name: "Input Size / Records"
+            },
+            {
+              data : function (row, type) {
+                if (row.taskMetrics && row.taskMetrics.outputMetrics && row.taskMetrics.outputMetrics.bytesWritten > 0) {
+                  if (type === 'display') {
+                    return formatBytes(row.taskMetrics.outputMetrics.bytesWritten, type) + " / " + row.taskMetrics.outputMetrics.recordsWritten;
+                  } else {
+                    return row.taskMetrics.outputMetrics.bytesWritten + " / " + row.taskMetrics.outputMetrics.recordsWritten;
+                  }
+                } else {
+                  return "";
                 }
-            });
+              },
+              name: "Output Size / Records"
+            },
+            {
+              data : function (row, type) {
+                if (row.taskMetrics && row.taskMetrics.shuffleWriteMetrics && row.taskMetrics.shuffleWriteMetrics.writeTime > 0) {
+                  return type === 'display' ? formatDuration(parseInt(row.taskMetrics.shuffleWriteMetrics.writeTime) / 1000000.0) : row.taskMetrics.shuffleWriteMetrics.writeTime;
+                } else {
+                  return "";
+                }
+              },
+              name: "Shuffle Write Time"
+            },
+            {
+              data : function (row, type) {
+                if (row.taskMetrics && row.taskMetrics.shuffleWriteMetrics && row.taskMetrics.shuffleWriteMetrics.bytesWritten > 0) {
+                  if (type === 'display') {
+                    return formatBytes(row.taskMetrics.shuffleWriteMetrics.bytesWritten, type) + " / " + row.taskMetrics.shuffleWriteMetrics.recordsWritten;
+                  } else {
+                    return row.taskMetrics.shuffleWriteMetrics.bytesWritten + " / " + row.taskMetrics.shuffleWriteMetrics.recordsWritten;
+                  }
+                } else {
+                  return "";
+                }
+              },
+              name: "Shuffle Write Size / Records"
+            },
+            {
+              data : function (row, type) {
+                if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics &&
+                  (row.taskMetrics.shuffleReadMetrics.localBytesRead > 0 || row.taskMetrics.shuffleReadMetrics.remoteBytesRead > 0)) {
+                  var totalBytesRead = parseInt(row.taskMetrics.shuffleReadMetrics.localBytesRead) + parseInt(row.taskMetrics.shuffleReadMetrics.remoteBytesRead);
+                  if (type === 'display') {
+                    return formatBytes(totalBytesRead, type) + " / " + row.taskMetrics.shuffleReadMetrics.recordsRead;
+                  } else {
+                    return totalBytesRead + " / " + row.taskMetrics.shuffleReadMetrics.recordsRead;
+                  }
+                } else {
+                  return "";
+                }
+              },
+              name: "Shuffle Read Size / Records"
+            },
+            {
+              data : function (row, type) {
+                if (row.taskMetrics && row.taskMetrics.memoryBytesSpilled && row.taskMetrics.memoryBytesSpilled > 0) {
+                  return type === 'display' ? formatBytes(row.taskMetrics.memoryBytesSpilled, type) : row.taskMetrics.memoryBytesSpilled;
+                } else {
+                  return "";
+                }
+              },
+              name: "Spill (Memory)"
+            },
+            {
+              data : function (row, type) {
+                if (row.taskMetrics && row.taskMetrics.diskBytesSpilled && row.taskMetrics.diskBytesSpilled > 0) {
+                  return type === 'display' ? formatBytes(row.taskMetrics.diskBytesSpilled, type) : row.taskMetrics.diskBytesSpilled;
+                } else {
+                  return "";
+                }
+              },
+              name: "Spill (Disk)"
+            },
+            {
+              data : function (row, _ignored_type) {
+                var msg = row.errorMessage;
+                if (typeof msg === 'undefined') {
+                  return "";
+                } else {
+                  var indexOfLineSeparator = msg.indexOf("\n");
+                  var formHead = indexOfLineSeparator > 0 ? msg.substring(0, indexOfLineSeparator) : (msg.length > 100 ? msg.substring(0, 100) : msg);
+                  var form = "<span onclick=\"this.parentNode.querySelector('.stacktrace-details').classList.toggle('collapsed')\" class=\"expand-details\">+details</span>";
+                  var formMsg = "<div class=\"stacktrace-details collapsed\"><pre>" + row.errorMessage + "</pre></div>";
+                  return formHead + form + formMsg;
+                }
+              },
+              name: "Errors"
+            }
+          ],
+          "columnDefs": [
+            { "visible": false, "targets": 11 },
+            { "visible": false, "targets": 12 },
+            { "visible": false, "targets": 13 },
+            { "visible": false, "targets": 14 },
+            { "visible": false, "targets": 15 },
+            { "visible": false, "targets": 16 },
+            { "visible": false, "targets": 17 },
+            { "visible": false, "targets": 18 },
+            { "visible": false, "targets": 21 }
+          ],
+          "deferRender": true
+        };
+        taskTableSelector = $(taskTable).DataTable(taskConf);
+        $('#active-tasks-table_filter input').unbind();
+        var searchEvent;
+        $('#active-tasks-table_filter input').bind('keyup', function(_ignored_e) {
+          if (typeof searchEvent !== 'undefined') {
+            window.clearTimeout(searchEvent);
+          }
+          var value = this.value;
+          searchEvent = window.setTimeout(function(){
+            taskTableSelector.search( value ).draw();}, 500);
         });
+        reselectCheckboxesBasedOnTaskTableState();
+
+        // hide or show columns dynamically event
+        $('input.toggle-vis').on('click', function(_ignored_e){
+          var taskSummaryMetricsTableFilteredArray;
+          // Get the column
+          var para = $(this).attr('data-column');
+          if (para == "0") {
+            var allColumns = taskTableSelector.columns(optionalColumns);
+            var executorAllColumns = executorSummaryTableSelector.columns(executorOptionalColumns);
+            if ($(this).is(":checked")) {
+              $(".toggle-vis").prop('checked', true);
+              allColumns.visible(true);
+              executorAllColumns.visible(true);
+              createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableArray);
+            } else {
+              $(".toggle-vis").prop('checked', false);
+              allColumns.visible(false);
+              executorAllColumns.visible(false);
+              taskSummaryMetricsTableFilteredArray =
+                taskSummaryMetricsTableArray.filter(row => row.checkboxId < 11);
+              createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableFilteredArray);
+            }
+          } else {
+            var dataMetricsType = $(this).attr("data-metrics-type");
+            var column;
+            if (dataMetricsType === 'task') {
+              column = taskTableSelector.column(para);
+              // Toggle the visibility
+              column.visible(!column.visible());
+              taskSummaryMetricsTableFilteredArray = [];
+              if ($(this).is(":checked")) {
+                taskSummaryMetricsTableCurrentStateArray.push(taskSummaryMetricsTableArray.filter(row => (row.checkboxId).toString() == para)[0]);
+                taskSummaryMetricsTableFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice();
+              } else {
+                taskSummaryMetricsTableFilteredArray =
+                  taskSummaryMetricsTableCurrentStateArray.filter(row => (row.checkboxId).toString() != para);
+              }
+              createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableFilteredArray);
+            }
+            if (dataMetricsType === "executor") {
+              column = executorSummaryTableSelector.column(para);
+              column.visible(!column.visible());
+            }
+          }
+        });
+
+        // title number and toggle list
+        $("#summaryMetricsTitle").html("Summary Metrics for " + "<a href='#tasksTitle'>" + responseBody.numCompleteTasks + " Completed Tasks" + "</a>");
+        $("#tasksTitle").html("Tasks (" + totalTasksToShow + ")");
+
+        // hide or show the accumulate update table
+        if (accumulatorTable.length == 0) {
+          $("#accumulator-update-table").hide();
+        } else {
+          taskTableSelector.column(18).visible(true);
+          $("#accumulator-update-table").show();
+        }
+        // Showing relevant stage data depending on stage type for task table and executor
+        // summary table
+        taskTableSelector.column(19).visible(dataToShow.showInputData);
+        taskTableSelector.column(20).visible(dataToShow.showOutputData);
+        taskTableSelector.column(22).visible(dataToShow.showShuffleWriteData);
+        taskTableSelector.column(23).visible(dataToShow.showShuffleReadData);
+        taskTableSelector.column(24).visible(dataToShow.showBytesSpilledData);
+        taskTableSelector.column(25).visible(dataToShow.showBytesSpilledData);
+
+        if (window.localStorage) {
+          if (window.localStorage.getItem("arrowtoggle1class") !== null &&
+            window.localStorage.getItem("arrowtoggle1class").includes("arrow-open")) {
+            $("#arrowtoggle1").toggleClass("arrow-open arrow-closed");
+            $("#toggle-metrics").toggleClass("d-none");
+          }
+          if (window.localStorage.getItem("arrowtoggle2class") !== null &&
+            window.localStorage.getItem("arrowtoggle2class").includes("arrow-open")) {
+            $("#arrowtoggle2").toggleClass("arrow-open arrow-closed");
+            $("#toggle-aggregatedMetrics").toggleClass("d-none");
+          }
+        }
+      });
     });
+  });
 });
diff --git a/core/src/main/resources/org/apache/spark/ui/static/stagespage-template.html b/core/src/main/resources/org/apache/spark/ui/static/stagespage-template.html
index 77ea70e4ad966..b938158b77027 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/stagespage-template.html
+++ b/core/src/main/resources/org/apache/spark/ui/static/stagespage-template.html
@@ -50,8 +50,8 @@ <h4 class="title-table">Aggregated Metrics by Executor</h4>
                 <th>Succeeded Tasks</th>
                 <th>
           <span data-toggle="tooltip" data-placement="top"
-                title="Shows if this executor has been blacklisted by the scheduler due to task failures.">
-            Blacklisted</span>
+                title="Shows if this executor has been excluded by the scheduler due to task failures.">
+            Excluded</span>
                 </th>
                 <th><span id="executor-summary-input">Input Size / Records</span></th>
                 <th><span id="executor-summary-output">Output Size / Records</span></th>
@@ -59,6 +59,10 @@ <h4 class="title-table">Aggregated Metrics by Executor</h4>
                 <th><span id="executor-summary-shuffle-write">Shuffle Write Size / Records</span></th>
                 <th>Spill (Memory) </th>
                 <th>Spill (Disk) </th>
+                <th>Peak JVM Memory OnHeap / OffHeap</th>
+                <th>Peak Execution Memory OnHeap / OffHeap</th>
+                <th>Peak Storage Memory OnHeap / OffHeap</th>
+                <th>Peak Pool Memory Direct / Mapped</th>
                 </tr>
             </thead>
             <tbody>
diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js
index 6d4c8d94b4288..9f366025a8b55 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js
@@ -15,6 +15,7 @@
  * limitations under the License.
  */
 
+/* global $, d3, timeFormat, timeTipStrings */
 
 // timeFormat: StreamingPage.scala will generate a global "timeFormat" dictionary to store the time
 // and its formatted string. Because we cannot specify a timezone in JavaScript, to make sure the
@@ -37,87 +38,90 @@ var onClickTimeline = function() {};
 
 // Show a tooltip "text" for "node"
 function showBootstrapTooltip(node, text) {
-    $(node).tooltip({title: text, trigger: "manual", container: "body"});
-    $(node).tooltip("show");
+  $(node).tooltip({title: text, trigger: "manual", container: "body"});
+  $(node).tooltip("show");
 }
 
 // Hide the tooltip for "node"
 function hideBootstrapTooltip(node) {
-    $(node).tooltip("dispose");
+  $(node).tooltip("dispose");
 }
 
+/* eslint-disable no-unused-vars */
 // Return the function to scroll to the corresponding
 // row on clicking a point of batch in the timeline.
 function getOnClickTimelineFunction() {
-    // If the user click one point in the graphs, jump to the batch row and highlight it. And
-    // recovery the batch row after 3 seconds if necessary.
-    // We need to remember the last clicked batch so that we can recovery it.
-    var lastClickedBatch = null;
-    var lastTimeout = null;
-
-    return function(d) {
-        var batchSelector = $("#batch-" + d.x);
-        // If there is a corresponding batch row, scroll down to it and highlight it.
-        if (batchSelector.length > 0) {
-            if (lastTimeout != null) {
-                window.clearTimeout(lastTimeout);
-            }
-            if (lastClickedBatch != null) {
-                clearBatchRow(lastClickedBatch);
-                lastClickedBatch = null;
-            }
-            lastClickedBatch = d.x;
-            highlightBatchRow(lastClickedBatch);
-            lastTimeout = window.setTimeout(function () {
-                lastTimeout = null;
-                if (lastClickedBatch != null) {
-                    clearBatchRow(lastClickedBatch);
-                    lastClickedBatch = null;
-                }
-            }, 3000); // Clean up after 3 seconds
-
-            var topOffset = batchSelector.offset().top - 15;
-            if (topOffset < 0) {
-                topOffset = 0;
-            }
-            $('html,body').animate({scrollTop: topOffset}, 200);
+  // If the user click one point in the graphs, jump to the batch row and highlight it. And
+  // recovery the batch row after 3 seconds if necessary.
+  // We need to remember the last clicked batch so that we can recovery it.
+  var lastClickedBatch = null;
+  var lastTimeout = null;
+
+  return function(d) {
+    var batchSelector = $("#batch-" + d.x);
+    // If there is a corresponding batch row, scroll down to it and highlight it.
+    if (batchSelector.length > 0) {
+      if (lastTimeout != null) {
+        window.clearTimeout(lastTimeout);
+      }
+      if (lastClickedBatch != null) {
+        clearBatchRow(lastClickedBatch);
+        lastClickedBatch = null;
+      }
+      lastClickedBatch = d.x;
+      highlightBatchRow(lastClickedBatch);
+      lastTimeout = window.setTimeout(function () {
+        lastTimeout = null;
+        if (lastClickedBatch != null) {
+          clearBatchRow(lastClickedBatch);
+          lastClickedBatch = null;
         }
+      }, 3000); // Clean up after 3 seconds
+
+      var topOffset = batchSelector.offset().top - 15;
+      if (topOffset < 0) {
+        topOffset = 0;
+      }
+      $('html,body').animate({scrollTop: topOffset}, 200);
     }
+  }
 }
 
 // Register a timeline graph. All timeline graphs should be register before calling any
 // "drawTimeline" so that we can determine the max margin left for all timeline graphs.
 function registerTimeline(minY, maxY) {
-    var numOfChars = yValueFormat(maxY).length;
-    // A least width for "maxY" in the graph
-    var pxForMaxY = numOfChars * 8 + 10;
-    // Make sure we have enough space to show the ticks in the y axis of timeline
-    maxMarginLeftForTimeline = pxForMaxY > maxMarginLeftForTimeline? pxForMaxY : maxMarginLeftForTimeline;
+  var numOfChars = yValueFormat(maxY).length;
+  // A least width for "maxY" in the graph
+  var pxForMaxY = numOfChars * 8 + 10;
+  // Make sure we have enough space to show the ticks in the y axis of timeline
+  maxMarginLeftForTimeline = pxForMaxY > maxMarginLeftForTimeline? pxForMaxY : maxMarginLeftForTimeline;
 }
 
 // Register a histogram graph. All histogram graphs should be register before calling any
 // "drawHistogram" so that we can determine the max X value for histograms.
 function registerHistogram(values, minY, maxY) {
-    var data = d3.layout.histogram().range([minY, maxY]).bins(histogramBinCount)(values);
-    // d.x is the y values while d.y is the x values
-    var maxX = d3.max(data, function(d) { return d.y; });
-    maxXForHistogram = maxX > maxXForHistogram ? maxX : maxXForHistogram;
+  var data = d3.layout.histogram().range([minY, maxY]).bins(histogramBinCount)(values);
+  // d.x is the y values while d.y is the x values
+  var maxX = d3.max(data, function(d) { return d.y; });
+  maxXForHistogram = maxX > maxXForHistogram ? maxX : maxXForHistogram;
 }
+/* eslint-enable no-unused-vars */
 
 // Draw a line between (x1, y1) and (x2, y2)
 function drawLine(svg, xFunc, yFunc, x1, y1, x2, y2) {
-    var line = d3.svg.line()
-        .x(function(d) { return xFunc(d.x); })
-        .y(function(d) { return yFunc(d.y); });
-    var data = [{x: x1, y: y1}, {x: x2, y: y2}];
-    svg.append("path")
-        .datum(data)
-        .style("stroke-dasharray", ("6, 6"))
-        .style("stroke", "lightblue")
-        .attr("class", "line")
-        .attr("d", line);
+  var line = d3.svg.line()
+    .x(function(d) { return xFunc(d.x); })
+    .y(function(d) { return yFunc(d.y); });
+  var data = [{x: x1, y: y1}, {x: x2, y: y2}];
+  svg.append("path")
+    .datum(data)
+    .style("stroke-dasharray", ("6, 6"))
+    .style("stroke", "lightblue")
+    .attr("class", "line")
+    .attr("d", line);
 }
 
+/* eslint-disable no-unused-vars */
 /**
  * @param id the `id` used in the html `div` tag
  * @param data the data for the timeline graph
@@ -129,108 +133,105 @@ function drawLine(svg, xFunc, yFunc, x1, y1, x2, y2) {
  * @param batchInterval if "batchInterval" is specified, we will draw a line for "batchInterval" in the graph
  */
 function drawTimeline(id, data, minX, maxX, minY, maxY, unitY, batchInterval) {
-    // Hide the right border of "<td>". We cannot use "css" directly, or "sorttable.js" will override them.
-    d3.select(d3.select(id).node().parentNode)
-        .style("padding", "8px 0 8px 8px")
-        .style("border-right", "0px solid white");
-
-    var margin = {top: 20, right: 27, bottom: 30, left: maxMarginLeftForTimeline};
-    var width = 500 - margin.left - margin.right;
-    var height = 150 - margin.top - margin.bottom;
-
-    var x = d3.scale.linear().domain([minX, maxX]).range([0, width]);
-    var y = d3.scale.linear().domain([minY, maxY]).range([height, 0]);
-
-    var xAxis = d3.svg.axis().scale(x).orient("bottom").tickFormat(function(d) {
-        var formattedDate = timeFormat[d];
-        var dotIndex = formattedDate.indexOf('.');
-        if (dotIndex >= 0) {
-            // Remove milliseconds
-            return formattedDate.substring(0, dotIndex);
-        } else {
-            return formattedDate;
-        }
-    });
-    var formatYValue = d3.format(",.2f");
-    var yAxis = d3.svg.axis().scale(y).orient("left").ticks(5).tickFormat(formatYValue);
-
-    var line = d3.svg.line()
-        .x(function(d) { return x(d.x); })
-        .y(function(d) { return y(d.y); });
-
-    var svg = d3.select(id).append("svg")
-        .attr("width", width + margin.left + margin.right)
-        .attr("height", height + margin.top + margin.bottom)
-        .append("g")
-            .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
-
-    // Only show the first and last time in the graph
-    xAxis.tickValues(x.domain());
-
-    svg.append("g")
-        .attr("class", "x axis")
-        .attr("transform", "translate(0," + height + ")")
-        .call(xAxis);
-
-    svg.append("g")
-        .attr("class", "y axis")
-        .call(yAxis)
-        .append("text")
-            .attr("transform", "translate(0," + unitLabelYOffset + ")")
-            .text(unitY);
-
-
-    if (batchInterval && batchInterval <= maxY) {
-        drawLine(svg, x, y, minX, batchInterval, maxX, batchInterval);
-    }
-
-    svg.append("path")
-        .datum(data)
-        .attr("class", "line")
-        .attr("d", line);
-
-    // If the user click one point in the graphs, jump to the batch row and highlight it. And
-    // recovery the batch row after 3 seconds if necessary.
-    // We need to remember the last clicked batch so that we can recovery it.
-    var lastClickedBatch = null;
-    var lastTimeout = null;
-
-    function isFailedBatch(batchTime) {
-        return $("#batch-" + batchTime).attr("isFailed") == "true";
+  // Hide the right border of "<td>". We cannot use "css" directly, or "sorttable.js" will override them.
+  d3.select(d3.select(id).node().parentNode)
+    .style("padding", "8px 0 8px 8px")
+    .style("border-right", "0px solid white");
+  var margin = {top: 20, right: 27, bottom: 30, left: maxMarginLeftForTimeline};
+  var width = 500 - margin.left - margin.right;
+  var height = 150 - margin.top - margin.bottom;
+
+  var x = d3.scale.linear().domain([minX, maxX]).range([0, width]);
+  var y = d3.scale.linear().domain([minY, maxY]).range([height, 0]);
+
+  var xAxis = d3.svg.axis().scale(x).orient("bottom").tickFormat(function(d) {
+    var formattedDate = timeFormat[d];
+    var dotIndex = formattedDate.indexOf('.');
+    if (dotIndex >= 0) {
+      // Remove milliseconds
+      return formattedDate.substring(0, dotIndex);
+    } else {
+      return formattedDate;
     }
-
-    // Add points to the line. However, we make it invisible at first. But when the user moves mouse
-    // over a point, it will be displayed with its detail.
-    svg.selectAll(".point")
-        .data(data)
-        .enter().append("circle")
-            .attr("stroke", function(d) { return isFailedBatch(d.x) ? "red" : "white";}) // white and opacity = 0 make it invisible
-            .attr("fill", function(d) { return isFailedBatch(d.x) ? "red" : "white";})
-            .attr("opacity", function(d) { return isFailedBatch(d.x) ? "1" : "0";})
-            .style("cursor", "pointer")
-            .attr("cx", function(d) { return x(d.x); })
-            .attr("cy", function(d) { return y(d.y); })
-            .attr("r", function(d) { return isFailedBatch(d.x) ? "2" : "3";})
-            .on('mouseover', function(d) {
-                var tip = formatYValue(d.y) + " " + unitY + " at " + timeTipStrings[d.x];
-                showBootstrapTooltip(d3.select(this).node(), tip);
-                // show the point
-                d3.select(this)
-                    .attr("stroke", function(d) { return isFailedBatch(d.x) ? "red" : "steelblue";})
-                    .attr("fill", function(d) { return isFailedBatch(d.x) ? "red" : "steelblue";})
-                    .attr("opacity", "1")
-                    .attr("r", "3");
-            })
-            .on('mouseout',  function() {
-                hideBootstrapTooltip(d3.select(this).node());
-                // hide the point
-                d3.select(this)
-                    .attr("stroke", function(d) { return isFailedBatch(d.x) ? "red" : "white";})
-                    .attr("fill", function(d) { return isFailedBatch(d.x) ? "red" : "white";})
-                    .attr("opacity", function(d) { return isFailedBatch(d.x) ? "1" : "0";})
-                    .attr("r", function(d) { return isFailedBatch(d.x) ? "2" : "3";});
-            })
-            .on("click", onClickTimeline);
+  });
+  var formatYValue = d3.format(",.2f");
+  var yAxis = d3.svg.axis().scale(y).orient("left").ticks(5).tickFormat(formatYValue);
+
+  var line = d3.svg.line()
+    .x(function(d) { return x(d.x); })
+    .y(function(d) { return y(d.y); });
+
+  var svg = d3.select(id).append("svg")
+    .attr("width", width + margin.left + margin.right)
+    .attr("height", height + margin.top + margin.bottom)
+    .append("g")
+    .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
+
+  // Only show the first and last time in the graph
+  xAxis.tickValues(x.domain());
+
+  svg.append("g")
+    .attr("class", "x axis")
+    .attr("transform", "translate(0," + height + ")")
+    .call(xAxis);
+
+  svg.append("g")
+    .attr("class", "y axis")
+    .call(yAxis)
+    .append("text")
+    .attr("transform", "translate(0," + unitLabelYOffset + ")")
+    .text(unitY);
+
+  if (batchInterval && batchInterval <= maxY) {
+    drawLine(svg, x, y, minX, batchInterval, maxX, batchInterval);
+  }
+
+  svg.append("path")
+    .datum(data)
+    .attr("class", "line")
+    .attr("d", line);
+  // If the user click one point in the graphs, jump to the batch row and highlight it. And
+  // recovery the batch row after 3 seconds if necessary.
+  // We need to remember the last clicked batch so that we can recovery it.
+  var lastClickedBatch = null;
+  var lastTimeout = null;
+
+  function isFailedBatch(batchTime) {
+    return $("#batch-" + batchTime).attr("isFailed") == "true";
+  }
+
+  // Add points to the line. However, we make it invisible at first. But when the user moves mouse
+  // over a point, it will be displayed with its detail.
+  svg.selectAll(".point")
+    .data(data)
+    .enter().append("circle")
+    .attr("stroke", function(d) { return isFailedBatch(d.x) ? "red" : "white";}) // white and opacity = 0 make it invisible
+    .attr("fill", function(d) { return isFailedBatch(d.x) ? "red" : "white";})
+    .attr("opacity", function(d) { return isFailedBatch(d.x) ? "1" : "0";})
+    .style("cursor", "pointer")
+    .attr("cx", function(d) { return x(d.x); })
+    .attr("cy", function(d) { return y(d.y); })
+    .attr("r", function(d) { return isFailedBatch(d.x) ? "2" : "3";})
+    .on('mouseover', function(d) {
+      var tip = formatYValue(d.y) + " " + unitY + " at " + timeTipStrings[d.x];
+      showBootstrapTooltip(d3.select(this).node(), tip);
+      // show the point
+      d3.select(this)
+        .attr("stroke", function(d) { return isFailedBatch(d.x) ? "red" : "steelblue";})
+        .attr("fill", function(d) { return isFailedBatch(d.x) ? "red" : "steelblue";})
+        .attr("opacity", "1")
+        .attr("r", "3");
+    })
+    .on('mouseout',  function() {
+      hideBootstrapTooltip(d3.select(this).node());
+      // hide the point
+      d3.select(this)
+        .attr("stroke", function(d) { return isFailedBatch(d.x) ? "red" : "white";})
+        .attr("fill", function(d) { return isFailedBatch(d.x) ? "red" : "white";})
+        .attr("opacity", function(d) { return isFailedBatch(d.x) ? "1" : "0";})
+        .attr("r", function(d) { return isFailedBatch(d.x) ? "2" : "3";});
+    })
+    .on("click", onClickTimeline);
 }
 
 /**
@@ -242,105 +243,106 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY, batchInterval) {
  * @param batchInterval if "batchInterval" is specified, we will draw a line for "batchInterval" in the graph
  */
 function drawHistogram(id, values, minY, maxY, unitY, batchInterval) {
-    // Hide the left border of "<td>". We cannot use "css" directly, or "sorttable.js" will override them.
-    d3.select(d3.select(id).node().parentNode)
-        .style("padding", "8px 8px 8px 0")
-        .style("border-left", "0px solid white");
-
-    var margin = {top: 20, right: 30, bottom: 30, left: 10};
-    var width = 350 - margin.left - margin.right;
-    var height = 150 - margin.top - margin.bottom;
-
-    var x = d3.scale.linear().domain([0, maxXForHistogram]).range([0, width - 50]);
-    var y = d3.scale.linear().domain([minY, maxY]).range([height, 0]);
-
-    var xAxis = d3.svg.axis().scale(x).orient("top").ticks(5);
-    var yAxis = d3.svg.axis().scale(y).orient("left").ticks(0).tickFormat(function(d) { return ""; });
-
-    var data = d3.layout.histogram().range([minY, maxY]).bins(histogramBinCount)(values);
-
-    var svg = d3.select(id).append("svg")
-        .attr("width", width + margin.left + margin.right)
-        .attr("height", height + margin.top + margin.bottom)
-        .append("g")
-            .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
-
-    if (batchInterval && batchInterval <= maxY) {
-        drawLine(svg, x, y, 0, batchInterval, maxXForHistogram, batchInterval);
-    }
+  // Hide the left border of "<td>". We cannot use "css" directly, or "sorttable.js" will override them.
+  d3.select(d3.select(id).node().parentNode)
+    .style("padding", "8px 8px 8px 0")
+    .style("border-left", "0px solid white");
+
+  var margin = {top: 20, right: 30, bottom: 30, left: 10};
+  var width = 350 - margin.left - margin.right;
+  var height = 150 - margin.top - margin.bottom;
+
+  var x = d3.scale.linear().domain([0, maxXForHistogram]).range([0, width - 50]);
+  var y = d3.scale.linear().domain([minY, maxY]).range([height, 0]);
+
+  var xAxis = d3.svg.axis().scale(x).orient("top").ticks(5);
+  var yAxis = d3.svg.axis().scale(y).orient("left").ticks(0).tickFormat(function(d) { return ""; });
+
+  var data = d3.layout.histogram().range([minY, maxY]).bins(histogramBinCount)(values);
+
+  var svg = d3.select(id).append("svg")
+    .attr("width", width + margin.left + margin.right)
+    .attr("height", height + margin.top + margin.bottom)
+    .append("g")
+    .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
+
+  if (batchInterval && batchInterval <= maxY) {
+    drawLine(svg, x, y, 0, batchInterval, maxXForHistogram, batchInterval);
+  }
+
+  svg.append("g")
+    .attr("class", "x axis")
+    .call(xAxis)
+    .append("text")
+    .attr("transform", "translate(" + (margin.left + width - 45) + ", " + unitLabelYOffset + ")")
+    .text("#batches");
+
+  svg.append("g")
+    .attr("class", "y axis")
+    .call(yAxis);
+
+  svg.selectAll(".bar")
+    .data(data)
+    .enter()
+    .append("g")
+    .attr("transform", function(d) { return "translate(0," + (y(d.x) - height + y(d.dx))  + ")";})
+    .attr("class", "bar").append("rect")
+    .attr("width", function(d) { return x(d.y); })
+    .attr("height", function(d) { return height - y(d.dx); })
+    .on('mouseover', function(d) {
+      var percent = yValueFormat(d.y * 100.0 / values.length) + "%";
+      var tip = d.y + " batches (" + percent + ") between " + yValueFormat(d.x) + " and " + yValueFormat(d.x + d.dx) + " " + unitY;
+      showBootstrapTooltip(d3.select(this).node(), tip);
+    })
+    .on('mouseout',  function() {
+      hideBootstrapTooltip(d3.select(this).node());
+    });
 
-    svg.append("g")
-        .attr("class", "x axis")
-        .call(xAxis)
-        .append("text")
-            .attr("transform", "translate(" + (margin.left + width - 45) + ", " + unitLabelYOffset + ")")
-            .text("#batches");
-
-    svg.append("g")
-        .attr("class", "y axis")
-        .call(yAxis);
-
-    svg.selectAll(".bar")
-        .data(data)
-        .enter()
-        .append("g")
-            .attr("transform", function(d) { return "translate(0," + (y(d.x) - height + y(d.dx))  + ")";})
-            .attr("class", "bar").append("rect")
-            .attr("width", function(d) { return x(d.y); })
-            .attr("height", function(d) { return height - y(d.dx); })
-            .on('mouseover', function(d) {
-                var percent = yValueFormat(d.y * 100.0 / values.length) + "%";
-                var tip = d.y + " batches (" + percent + ") between " + yValueFormat(d.x) + " and " + yValueFormat(d.x + d.dx) + " " + unitY;
-                showBootstrapTooltip(d3.select(this).node(), tip);
-            })
-            .on('mouseout',  function() {
-                hideBootstrapTooltip(d3.select(this).node());
-            });
-
-    if (batchInterval && batchInterval <= maxY) {
-        // Add the "stable" text to the graph below the batch interval line.
-        var stableXOffset = x(maxXForHistogram) - 20;
-        var stableYOffset = y(batchInterval) + 15;
-        svg.append("text")
-            .style("fill", "lightblue")
-            .attr("class", "stable-text")
-            .attr("text-anchor", "middle")
-            .attr("transform", "translate(" + stableXOffset + "," + stableYOffset + ")")
-            .text("stable")
-            .on('mouseover', function(d) {
-              var tip = "Processing Time <= Batch Interval (" + yValueFormat(batchInterval) +" " + unitY +")";
-              showBootstrapTooltip(d3.select(this).node(), tip);
-            })
-            .on('mouseout',  function() {
-              hideBootstrapTooltip(d3.select(this).node());
-            });
-    }
+  if (batchInterval && batchInterval <= maxY) {
+    // Add the "stable" text to the graph below the batch interval line.
+    var stableXOffset = x(maxXForHistogram) - 20;
+    var stableYOffset = y(batchInterval) + 15;
+    svg.append("text")
+      .style("fill", "lightblue")
+      .attr("class", "stable-text")
+      .attr("text-anchor", "middle")
+      .attr("transform", "translate(" + stableXOffset + "," + stableYOffset + ")")
+      .text("stable")
+      .on('mouseover', function(d) {
+        var tip = "Processing Time <= Batch Interval (" + yValueFormat(batchInterval) +" " + unitY +")";
+        showBootstrapTooltip(d3.select(this).node(), tip);
+      })
+      .on('mouseout',  function() {
+        hideBootstrapTooltip(d3.select(this).node());
+      });
+  }
 }
+/* eslint-enable no-unused-vars */
 
 $(function() {
-    var status = window.localStorage && window.localStorage.getItem("show-streams-detail") == "true";
-
-    $("span.expand-input-rate").click(function() {
-        status = !status;
-        $("#inputs-table").toggle('collapsed');
-        // Toggle the class of the arrow between open and closed
-        $(this).find('.expand-input-rate-arrow').toggleClass('arrow-open').toggleClass('arrow-closed');
-        if (window.localStorage) {
-            window.localStorage.setItem("show-streams-detail", "" + status);
-        }
-    });
-
-    if (status) {
-        $("#inputs-table").toggle('collapsed');
-        // Toggle the class of the arrow between open and closed
-        $(this).find('.expand-input-rate-arrow').toggleClass('arrow-open').toggleClass('arrow-closed');
+  var status = window.localStorage && window.localStorage.getItem("show-streams-detail") == "true";
+
+  $("span.expand-input-rate").click(function() {
+    status = !status;
+    $("#inputs-table").toggle('collapsed');
+    // Toggle the class of the arrow between open and closed
+    $(this).find('.expand-input-rate-arrow').toggleClass('arrow-open').toggleClass('arrow-closed');
+    if (window.localStorage) {
+      window.localStorage.setItem("show-streams-detail", "" + status);
     }
+  });
+
+  if (status) {
+    $("#inputs-table").toggle('collapsed');
+    // Toggle the class of the arrow between open and closed
+    $(this).find('.expand-input-rate-arrow').toggleClass('arrow-open').toggleClass('arrow-closed');
+  }
 });
 
 function highlightBatchRow(batch) {
-    $("#batch-" + batch).parent().addClass("batch-table-cell-highlight");
+  $("#batch-" + batch).parent().addClass("batch-table-cell-highlight");
 }
 
 function clearBatchRow(batch) {
-    $("#batch-" + batch).parent().removeClass("batch-table-cell-highlight");
+  $("#batch-" + batch).parent().removeClass("batch-table-cell-highlight");
 }
diff --git a/core/src/main/resources/org/apache/spark/ui/static/structured-streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/structured-streaming-page.js
index c92226b408b6c..9701f5a57e170 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/structured-streaming-page.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/structured-streaming-page.js
@@ -15,157 +15,158 @@
  * limitations under the License.
  */
 
+/* global d3, formattedTimeTipStrings, formattedTimeToValues, hideBootstrapTooltip, maxMarginLeftForTimeline, showBootstrapTooltip, unitLabelYOffset */
 // pre-define some colors for legends.
 var colorPool = ["#F8C471", "#F39C12", "#B9770E", "#73C6B6", "#16A085", "#117A65", "#B2BABB", "#7F8C8D", "#616A6B"];
 
+/* eslint-disable no-unused-vars */
 function drawAreaStack(id, labels, values, minX, maxX, minY, maxY) {
-    d3.select(d3.select(id).node().parentNode)
-        .style("padding", "8px 0 8px 8px")
-        .style("border-right", "0px solid white");
-
-    // Setup svg using Bostock's margin convention
-    var margin = {top: 20, right: 40, bottom: 30, left: maxMarginLeftForTimeline};
-    var width = 850 - margin.left - margin.right;
-    var height = 300 - margin.top - margin.bottom;
-
-    var svg = d3.select(id)
-        .append("svg")
-        .attr("width", width + margin.left + margin.right)
-        .attr("height", height + margin.top + margin.bottom)
-        .append("g")
-        .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
-
-    var data = values;
-
-    var parse = d3.time.format("%H:%M:%S.%L").parse;
-
-    // Transpose the data into layers
-    var dataset = d3.layout.stack()(labels.map(function(fruit) {
-        return data.map(function(d) {
-            return {_x: d.x, x: parse(d.x), y: +d[fruit]};
-        });
-    }));
-
-
-    // Set x, y and colors
-    var x = d3.scale.ordinal()
-        .domain(dataset[0].map(function(d) { return d.x; }))
-        .rangeRoundBands([10, width-10], 0.02);
-
-    var y = d3.scale.linear()
-        .domain([0, d3.max(dataset, function(d) {  return d3.max(d, function(d) { return d.y0 + d.y; });  })])
-        .range([height, 0]);
-
-    var colors = colorPool.slice(0, labels.length)
-
-    // Define and draw axes
-    var yAxis = d3.svg.axis()
-        .scale(y)
-        .orient("left")
-        .ticks(7)
-        .tickFormat( function(d) { return d } );
-
-    var xAxis = d3.svg.axis()
-        .scale(x)
-        .orient("bottom")
-        .tickFormat(d3.time.format("%H:%M:%S.%L"));
-
-    // Only show the first and last time in the graph
-    var xline = []
-    xline.push(x.domain()[0])
-    xline.push(x.domain()[x.domain().length - 1])
-    xAxis.tickValues(xline);
-
-    svg.append("g")
-        .attr("class", "y axis")
-        .call(yAxis)
-        .append("text")
-            .attr("transform", "translate(0," + unitLabelYOffset + ")")
-            .text("ms");
-
-    svg.append("g")
-        .attr("class", "x axis")
-        .attr("transform", "translate(0," + height + ")")
-        .call(xAxis);
-
-    // Create groups for each series, rects for each segment
-    var groups = svg.selectAll("g.cost")
-        .data(dataset)
-        .enter().append("g")
-        .attr("class", "cost")
-        .style("fill", function(d, i) { return colors[i]; });
-
-    var rect = groups.selectAll("rect")
-        .data(function(d) { return d; })
-        .enter()
-        .append("rect")
-        .attr("x", function(d) { return x(d.x); })
-        .attr("y", function(d) { return y(d.y0 + d.y); })
-        .attr("height", function(d) { return y(d.y0) - y(d.y0 + d.y); })
-        .attr("width", x.rangeBand())
-        .on('mouseover', function(d) {
-            var tip = '';
-            var idx = 0;
-            var _values = formattedTimeToValues[d._x];
-            _values.forEach(function (k) {
-                tip += labels[idx] + ': ' + k + '   ';
-                idx += 1;
-            });
-            tip += " at " + formattedTimeTipStrings[d._x];
-            showBootstrapTooltip(d3.select(this).node(), tip);
-        })
-        .on('mouseout',  function() {
-            hideBootstrapTooltip(d3.select(this).node());
-        })
-        .on("mousemove", function(d) {
-            var xPosition = d3.mouse(this)[0] - 15;
-            var yPosition = d3.mouse(this)[1] - 25;
-            tooltip.attr("transform", "translate(" + xPosition + "," + yPosition + ")");
-            tooltip.select("text").text(d.y);
-        });
-
-
-    // Draw legend
-    var legend = svg.selectAll(".legend")
-        .data(colors)
-        .enter().append("g")
-        .attr("class", "legend")
-        .attr("transform", function(d, i) { return "translate(30," + i * 19 + ")"; });
-
-    legend.append("rect")
-        .attr("x", width - 20)
-        .attr("width", 18)
-        .attr("height", 18)
-        .style("fill", function(d, i) {return colors.slice().reverse()[i];})
-        .on('mouseover', function(d, i) {
-            var len = labels.length
-            showBootstrapTooltip(d3.select(this).node(), labels[len - 1 - i]);
-        })
-        .on('mouseout',  function() {
-            hideBootstrapTooltip(d3.select(this).node());
-        })
-        .on("mousemove", function(d) {
-            var xPosition = d3.mouse(this)[0] - 15;
-            var yPosition = d3.mouse(this)[1] - 25;
-            tooltip.attr("transform", "translate(" + xPosition + "," + yPosition + ")");
-            tooltip.select("text").text(d.y);
-        });
-
-    // Prep the tooltip bits, initial display is hidden
-    var tooltip = svg.append("g")
-        .attr("class", "tooltip")
-        .style("display", "none");
-
-    tooltip.append("rect")
-        .attr("width", 30)
-        .attr("height", 20)
-        .attr("fill", "white")
-        .style("opacity", 0.5);
-
-    tooltip.append("text")
-        .attr("x", 15)
-        .attr("dy", "1.2em")
-        .style("text-anchor", "middle")
-        .attr("font-size", "12px")
-        .attr("font-weight", "bold");
+  d3.select(d3.select(id).node().parentNode)
+    .style("padding", "8px 0 8px 8px")
+    .style("border-right", "0px solid white");
+
+  // Setup svg using Bostock's margin convention
+  var margin = {top: 20, right: 40, bottom: 30, left: maxMarginLeftForTimeline};
+  var width = 850 - margin.left - margin.right;
+  var height = 300 - margin.top - margin.bottom;
+
+  var svg = d3.select(id)
+    .append("svg")
+    .attr("width", width + margin.left + margin.right)
+    .attr("height", height + margin.top + margin.bottom)
+    .append("g")
+    .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
+
+  var data = values;
+
+  var parse = d3.time.format("%H:%M:%S.%L").parse;
+
+  // Transpose the data into layers
+  var dataset = d3.layout.stack()(labels.map(function(fruit) {
+    return data.map(function(d) {
+      return {_x: d.x, x: parse(d.x), y: +d[fruit]};
+    });
+  }));
+
+  // Set x, y and colors
+  var x = d3.scale.ordinal()
+    .domain(dataset[0].map(function(d) { return d.x; }))
+    .rangeRoundBands([10, width-10], 0.02);
+
+  var y = d3.scale.linear()
+    .domain([0, d3.max(dataset, function(d) {  return d3.max(d, function(d) { return d.y0 + d.y; });  })])
+    .range([height, 0]);
+
+  var colors = colorPool.slice(0, labels.length);
+
+  // Define and draw axes
+  var yAxis = d3.svg.axis()
+    .scale(y)
+    .orient("left")
+    .ticks(7)
+    .tickFormat( function(d) { return d } );
+
+  var xAxis = d3.svg.axis()
+    .scale(x)
+    .orient("bottom")
+    .tickFormat(d3.time.format("%H:%M:%S.%L"));
+
+  // Only show the first and last time in the graph
+  var xline = [];
+  xline.push(x.domain()[0]);
+  xline.push(x.domain()[x.domain().length - 1]);
+  xAxis.tickValues(xline);
+
+  svg.append("g")
+    .attr("class", "y axis")
+    .call(yAxis)
+    .append("text")
+    .attr("transform", "translate(0," + unitLabelYOffset + ")")
+    .text("ms");
+
+  svg.append("g")
+    .attr("class", "x axis")
+    .attr("transform", "translate(0," + height + ")")
+    .call(xAxis);
+
+  // Create groups for each series, rects for each segment
+  var groups = svg.selectAll("g.cost")
+    .data(dataset)
+    .enter().append("g")
+    .attr("class", "cost")
+    .style("fill", function(d, i) { return colors[i]; });
+
+  var rect = groups.selectAll("rect")
+    .data(function(d) { return d; })
+    .enter()
+    .append("rect")
+    .attr("x", function(d) { return x(d.x); })
+    .attr("y", function(d) { return y(d.y0 + d.y); })
+    .attr("height", function(d) { return y(d.y0) - y(d.y0 + d.y); })
+    .attr("width", x.rangeBand())
+    .on('mouseover', function(d) {
+      var tip = '';
+      var idx = 0;
+      var _values = formattedTimeToValues[d._x];
+      _values.forEach(function (k) {
+        tip += labels[idx] + ': ' + k + '   ';
+        idx += 1;
+      });
+      tip += " at " + formattedTimeTipStrings[d._x];
+      showBootstrapTooltip(d3.select(this).node(), tip);
+    })
+    .on('mouseout',  function() {
+      hideBootstrapTooltip(d3.select(this).node());
+    })
+    .on("mousemove", function(d) {
+      var xPosition = d3.mouse(this)[0] - 15;
+      var yPosition = d3.mouse(this)[1] - 25;
+      tooltip.attr("transform", "translate(" + xPosition + "," + yPosition + ")");
+      tooltip.select("text").text(d.y);
+    });
+
+  // Draw legend
+  var legend = svg.selectAll(".legend")
+    .data(colors)
+    .enter().append("g")
+    .attr("class", "legend")
+    .attr("transform", function(d, i) { return "translate(30," + i * 19 + ")"; });
+
+  legend.append("rect")
+    .attr("x", width - 20)
+    .attr("width", 18)
+    .attr("height", 18)
+    .style("fill", function(d, i) {return colors.slice().reverse()[i];})
+    .on('mouseover', function(d, i) {
+      var len = labels.length;
+      showBootstrapTooltip(d3.select(this).node(), labels[len - 1 - i]);
+    })
+    .on('mouseout',  function() {
+      hideBootstrapTooltip(d3.select(this).node());
+    })
+    .on("mousemove", function(d) {
+      var xPosition = d3.mouse(this)[0] - 15;
+      var yPosition = d3.mouse(this)[1] - 25;
+      tooltip.attr("transform", "translate(" + xPosition + "," + yPosition + ")");
+      tooltip.select("text").text(d.y);
+    });
+
+  // Prep the tooltip bits, initial display is hidden
+  var tooltip = svg.append("g")
+    .attr("class", "tooltip")
+    .style("display", "none");
+
+  tooltip.append("rect")
+    .attr("width", 30)
+    .attr("height", 20)
+    .attr("fill", "white")
+    .style("opacity", 0.5);
+
+  tooltip.append("text")
+    .attr("x", 15)
+    .attr("dy", "1.2em")
+    .style("text-anchor", "middle")
+    .attr("font-size", "12px")
+    .attr("font-weight", "bold");
 }
+/* eslint-enable no-unused-vars */
\ No newline at end of file
diff --git a/core/src/main/resources/org/apache/spark/ui/static/table.js b/core/src/main/resources/org/apache/spark/ui/static/table.js
index 32b7a6522d050..0203748cf7dbc 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/table.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/table.js
@@ -15,6 +15,8 @@
  * limitations under the License.
  */
 
+/* global $ */
+/* eslint-disable no-unused-vars */
 /* Adds background colors to stripe table rows in the summary table (on the stage page). This is
  * necessary (instead of using css or the table striping provided by bootstrap) because the summary
  * table has hidden rows.
@@ -22,84 +24,88 @@
  * An ID selector (rather than a class selector) is used to ensure this runs quickly even on pages
  * with thousands of task rows (ID selectors are much faster than class selectors). */
 function stripeSummaryTable() {
-    $("#task-summary-table").find("tr:not(:hidden)").each(function (index) {
-       if (index % 2 == 1) {
-         $(this).css("background-color", "#f9f9f9");
-       } else {
-         $(this).css("background-color", "#ffffff");
-       }
-    });
+  $("#task-summary-table").find("tr:not(:hidden)").each(function (index) {
+    if (index % 2 == 1) {
+      $(this).css("background-color", "#f9f9f9");
+    } else {
+      $(this).css("background-color", "#ffffff");
+    }
+  });
 }
+/* eslint-enable no-unused-vars */
 
 function toggleThreadStackTrace(threadId, forceAdd) {
-    var stackTrace = $("#" + threadId + "_stacktrace")
-    if (stackTrace.length == 0) {
-        var stackTraceText = $('#' + threadId + "_td_stacktrace").html()
-        var threadCell = $("#thread_" + threadId + "_tr")
-        threadCell.after("<tr id=\"" + threadId +"_stacktrace\" class=\"accordion-body\"><td colspan=\"4\"><pre>" +
-            stackTraceText +  "</pre></td></tr>")
-    } else {
-        if (!forceAdd) {
-            stackTrace.remove()
-        }
+  var stackTrace = $("#" + threadId + "_stacktrace");
+  if (stackTrace.length == 0) {
+    var stackTraceText = $('#' + threadId + "_td_stacktrace").html();
+    var threadCell = $("#thread_" + threadId + "_tr");
+    threadCell.after("<tr id=\"" + threadId +"_stacktrace\" class=\"accordion-body\"><td colspan=\"4\"><pre>" +
+      stackTraceText +  "</pre></td></tr>")
+  } else {
+    if (!forceAdd) {
+      stackTrace.remove()
     }
+  }
 }
 
+/* eslint-disable no-unused-vars */
 function expandAllThreadStackTrace(toggleButton) {
-    $('.accordion-heading').each(function() {
-        //get thread ID
-        if (!$(this).hasClass("d-none")) {
-            var trId = $(this).attr('id').match(/thread_([0-9]+)_tr/m)[1]
-            toggleThreadStackTrace(trId, true)
-        }
-    })
-    if (toggleButton) {
-        $('.expandbutton').toggleClass('d-none')
+  $('.accordion-heading').each(function() {
+    //get thread ID
+    if (!$(this).hasClass("d-none")) {
+      var trId = $(this).attr('id').match(/thread_([0-9]+)_tr/m)[1];
+      toggleThreadStackTrace(trId, true)
     }
+  });
+  if (toggleButton) {
+    $('.expandbutton').toggleClass('d-none')
+  }
 }
+/* eslint-enable no-unused-vars */
 
 function collapseAllThreadStackTrace(toggleButton) {
-    $('.accordion-body').each(function() {
-        $(this).remove()
-    })
-    if (toggleButton) {
-        $('.expandbutton').toggleClass('d-none');
-    }
+  $('.accordion-body').each(function() {
+    $(this).remove()
+  });
+  if (toggleButton) {
+    $('.expandbutton').toggleClass('d-none');
+  }
 }
 
-
+/* eslint-disable no-unused-vars */
 // inOrOut - true: over, false: out
 function onMouseOverAndOut(threadId) {
-    $("#" + threadId + "_td_id").toggleClass("threaddump-td-mouseover");
-    $("#" + threadId + "_td_name").toggleClass("threaddump-td-mouseover");
-    $("#" + threadId + "_td_state").toggleClass("threaddump-td-mouseover");
-    $("#" + threadId + "_td_locking").toggleClass("threaddump-td-mouseover");
+  $("#" + threadId + "_td_id").toggleClass("threaddump-td-mouseover");
+  $("#" + threadId + "_td_name").toggleClass("threaddump-td-mouseover");
+  $("#" + threadId + "_td_state").toggleClass("threaddump-td-mouseover");
+  $("#" + threadId + "_td_locking").toggleClass("threaddump-td-mouseover");
 }
 
 function onSearchStringChange() {
-    var searchString = $('#search').val().toLowerCase();
-    //remove the stacktrace
-    collapseAllThreadStackTrace(false)
-    if (searchString.length == 0) {
-        $('tr').each(function() {
-            $(this).removeClass('d-none')
-        })
-    } else {
-        $('tr').each(function(){
-            if($(this).attr('id') && $(this).attr('id').match(/thread_[0-9]+_tr/) ) {
-                var children = $(this).children()
-                var found = false
-                for (var i = 0; i < children.length; i++) {
-                    if (children.eq(i).text().toLowerCase().indexOf(searchString) >= 0) {
-                        found = true
-                    }
-                }
-                if (found) {
-                    $(this).removeClass('d-none')
-                } else {
-                    $(this).addClass('d-none')
-                }
-            }
-        });
-    }
+  var searchString = $('#search').val().toLowerCase();
+  //remove the stacktrace
+  collapseAllThreadStackTrace(false);
+  if (searchString.length == 0) {
+    $('tr').each(function() {
+      $(this).removeClass('d-none')
+    })
+  } else {
+    $('tr').each(function(){
+      if($(this).attr('id') && $(this).attr('id').match(/thread_[0-9]+_tr/) ) {
+        var children = $(this).children();
+        var found = false;
+        for (var i = 0; i < children.length; i++) {
+          if (children.eq(i).text().toLowerCase().indexOf(searchString) >= 0) {
+            found = true;
+          }
+        }
+        if (found) {
+          $(this).removeClass('d-none')
+        } else {
+          $(this).addClass('d-none')
+        }
+      }
+    });
+  }
 }
+/* eslint-enable no-unused-vars */
diff --git a/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js b/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js
index 5be8cffd1f8db..274848d5e62b5 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js
@@ -15,6 +15,8 @@
  * limitations under the License.
  */
 
+/* global $, vis, uiRoot, appBasePath */
+/* eslint-disable no-unused-vars */
 function drawApplicationTimeline(groupArray, eventObjArray, startTime, offset) {
   var groups = new vis.DataSet(groupArray);
   var items = new vis.DataSet(eventObjArray);
@@ -42,26 +44,31 @@ function drawApplicationTimeline(groupArray, eventObjArray, startTime, offset) {
   setupZoomable("#application-timeline-zoom-lock", applicationTimeline);
   setupExecutorEventAction();
 
+  function getIdForJobEntry(baseElem) {
+    var jobIdText = $($(baseElem).find(".application-timeline-content")[0]).text();
+    var jobId = jobIdText.match("\\(Job (\\d+)\\)$")[1];
+    return jobId;
+  }
+
+  function getSelectorForJobEntry(jobId) {
+    return "#job-" + jobId;
+  }
+
   function setupJobEventAction() {
     $(".vis-item.vis-range.job.application-timeline-object").each(function() {
-      var getSelectorForJobEntry = function(baseElem) {
-        var jobIdText = $($(baseElem).find(".application-timeline-content")[0]).text();
-        var jobId = jobIdText.match("\\(Job (\\d+)\\)$")[1];
-       return "#job-" + jobId;
-      };
-
       $(this).click(function() {
-        var jobPagePath = $(getSelectorForJobEntry(this)).find("a.name-link").attr("href");
-          window.location.href = jobPagePath
+        var jobId = getIdForJobEntry(this);
+        var jobPagePath = uiRoot + appBasePath + "/jobs/job/?id=" + jobId;
+        window.location.href = jobPagePath;
       });
 
       $(this).hover(
         function() {
-          $(getSelectorForJobEntry(this)).addClass("corresponding-item-hover");
+          $(getSelectorForJobEntry(getIdForJobEntry(this))).addClass("corresponding-item-hover");
           $($(this).find("div.application-timeline-content")[0]).tooltip("show");
         },
         function() {
-          $(getSelectorForJobEntry(this)).removeClass("corresponding-item-hover");
+          $(getSelectorForJobEntry(getIdForJobEntry(this))).removeClass("corresponding-item-hover");
           $($(this).find("div.application-timeline-content")[0]).tooltip("hide");
         }
       );
@@ -125,26 +132,34 @@ function drawJobTimeline(groupArray, eventObjArray, startTime, offset) {
   setupZoomable("#job-timeline-zoom-lock", jobTimeline);
   setupExecutorEventAction();
 
+  function getStageIdAndAttemptForStageEntry(baseElem) {
+    var stageIdText = $($(baseElem).find(".job-timeline-content")[0]).text();
+    var stageIdAndAttempt = stageIdText.match("\\(Stage (\\d+\\.\\d+)\\)$")[1].split(".");
+    return stageIdAndAttempt;
+  }
+
+  function getSelectorForStageEntry(stageIdAndAttempt) {
+    return "#stage-" + stageIdAndAttempt[0] + "-" + stageIdAndAttempt[1];
+  }
+
   function setupStageEventAction() {
     $(".vis-item.vis-range.stage.job-timeline-object").each(function() {
-      var getSelectorForStageEntry = function(baseElem) {
-        var stageIdText = $($(baseElem).find(".job-timeline-content")[0]).text();
-        var stageIdAndAttempt = stageIdText.match("\\(Stage (\\d+\\.\\d+)\\)$")[1].split(".");
-        return "#stage-" + stageIdAndAttempt[0] + "-" + stageIdAndAttempt[1];
-      };
-
       $(this).click(function() {
-        var stagePagePath = $(getSelectorForStageEntry(this)).find("a.name-link").attr("href")
-        window.location.href = stagePagePath
+        var stageIdAndAttempt = getStageIdAndAttemptForStageEntry(this);
+        var stagePagePath = uiRoot + appBasePath +
+          "/stages/stage/?id=" + stageIdAndAttempt[0] + "&attempt=" + stageIdAndAttempt[1];
+        window.location.href = stagePagePath;
       });
 
       $(this).hover(
         function() {
-          $(getSelectorForStageEntry(this)).addClass("corresponding-item-hover");
+          $(getSelectorForStageEntry(getStageIdAndAttemptForStageEntry(this)))
+            .addClass("corresponding-item-hover");
           $($(this).find("div.job-timeline-content")[0]).tooltip("show");
         },
         function() {
-          $(getSelectorForStageEntry(this)).removeClass("corresponding-item-hover");
+          $(getSelectorForStageEntry(getStageIdAndAttemptForStageEntry(this)))
+            .removeClass("corresponding-item-hover");
           $($(this).find("div.job-timeline-content")[0]).tooltip("hide");
         }
       );
@@ -236,7 +251,7 @@ function drawTaskAssignmentTimeline(groupArray, eventObjArray, minLaunchTime, ma
     var visibilityState = status ? "" : "none";
     $("#task-assignment-timeline").css("display", visibilityState);
 
-     // Switch the class of the arrow from open to closed.
+    // Switch the class of the arrow from open to closed.
     $(this).find(".expand-task-assignment-timeline-arrow").toggleClass("arrow-open");
     $(this).find(".expand-task-assignment-timeline-arrow").toggleClass("arrow-closed");
 
@@ -267,6 +282,7 @@ function setupExecutorEventAction() {
     );
   });
 }
+/* eslint-enable no-unused-vars */
 
 function setupZoomable(id, timeline) {
   $(id + ' > input[type="checkbox"]').click(function() {
diff --git a/core/src/main/resources/org/apache/spark/ui/static/utils.js b/core/src/main/resources/org/apache/spark/ui/static/utils.js
index 6fc34a9e1f7ea..a2d4f5568cb26 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/utils.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/utils.js
@@ -15,62 +15,72 @@
  * limitations under the License.
  */
 
+/* global $, uiRoot */
+/* eslint-disable no-unused-vars */
 // this function works exactly the same as UIUtils.formatDuration
 function formatDuration(milliseconds) {
-    if (milliseconds < 100) {
-        return parseInt(milliseconds).toFixed(1) + " ms";
-    }
-    var seconds = milliseconds * 1.0 / 1000;
-    if (seconds < 1) {
-        return seconds.toFixed(1) + " s";
-    }
-    if (seconds < 60) {
-        return seconds.toFixed(0) + " s";
-    }
-    var minutes = seconds / 60;
-    if (minutes < 10) {
-        return minutes.toFixed(1) + " min";
-    } else if (minutes < 60) {
-        return minutes.toFixed(0) + " min";
-    }
-    var hours = minutes / 60;
-    return hours.toFixed(1) + " h";
+  if (milliseconds < 100) {
+    return parseInt(milliseconds).toFixed(1) + " ms";
+  }
+  var seconds = milliseconds * 1.0 / 1000;
+  if (seconds < 1) {
+    return seconds.toFixed(1) + " s";
+  }
+  if (seconds < 60) {
+    return seconds.toFixed(0) + " s";
+  }
+  var minutes = seconds / 60;
+  if (minutes < 10) {
+    return minutes.toFixed(1) + " min";
+  } else if (minutes < 60) {
+    return minutes.toFixed(0) + " min";
+  }
+  var hours = minutes / 60;
+  return hours.toFixed(1) + " h";
 }
 
 function formatBytes(bytes, type) {
-    if (type !== 'display') return bytes;
-    if (bytes == 0) return '0.0 B';
-    var k = 1024;
-    var dm = 1;
-    var sizes = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'];
-    var i = Math.floor(Math.log(bytes) / Math.log(k));
-    return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
+  if (type !== 'display') return bytes;
+  if (bytes <= 0) return '0.0 B';
+  var k = 1024;
+  var dm = 1;
+  var sizes = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'];
+  var i = Math.floor(Math.log(bytes) / Math.log(k));
+  return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
 }
+/* eslint-enable no-unused-vars */
 
 function padZeroes(num) {
   return ("0" + num).slice(-2);
 }
 
+/* eslint-disable no-unused-vars */
 function formatTimeMillis(timeMillis) {
   if (timeMillis <= 0) {
     return "-";
   } else {
     var dt = new Date(timeMillis);
-    return dt.getFullYear() + "-" +
-      padZeroes(dt.getMonth() + 1) + "-" +
-      padZeroes(dt.getDate()) + " " +
-      padZeroes(dt.getHours()) + ":" +
-      padZeroes(dt.getMinutes()) + ":" +
-      padZeroes(dt.getSeconds());
+    return formatDateString(dt);
   }
 }
+/* eslint-enable no-unused-vars */
+
+function formatDateString(dt) {
+  return dt.getFullYear() + "-" +
+    padZeroes(dt.getMonth() + 1) + "-" +
+    padZeroes(dt.getDate()) + " " +
+    padZeroes(dt.getHours()) + ":" +
+    padZeroes(dt.getMinutes()) + ":" +
+    padZeroes(dt.getSeconds());
+}
 
+/* eslint-disable no-unused-vars */
 function getTimeZone() {
   try {
     return Intl.DateTimeFormat().resolvedOptions().timeZone;
   } catch(ex) {
     // Get time zone from a string representing the date,
-    // eg. "Thu Nov 16 2017 01:13:32 GMT+0800 (CST)" -> "CST"
+    // e.g. "Thu Nov 16 2017 01:13:32 GMT+0800 (CST)" -> "CST"
     return new Date().toString().match(/\((.*)\)/)[1];
   }
 }
@@ -88,20 +98,21 @@ function formatLogsCells(execLogs, type) {
 function getStandAloneAppId(cb) {
   var words = document.baseURI.split('/');
   var ind = words.indexOf("proxy");
+  var appId;
   if (ind > 0) {
-    var appId = words[ind + 1];
+    appId = words[ind + 1];
     cb(appId);
     return;
   }
   ind = words.indexOf("history");
   if (ind > 0) {
-    var appId = words[ind + 1];
+    appId = words[ind + 1];
     cb(appId);
     return;
   }
   // Looks like Web UI is running in standalone mode
   // Let's get application-id using REST End Point
-  $.getJSON(location.origin + "/api/v1/applications", function(response, status, jqXHR) {
+  $.getJSON(uiRoot + "/api/v1/applications", function(response, status, jqXHR) {
     if (response && response.length > 0) {
       var appId = response[0].id;
       cb(appId);
@@ -115,7 +126,7 @@ function getStandAloneAppId(cb) {
 // It will convert the string into integer for correct ordering
 function ConvertDurationString(data) {
   data = data.toString();
-  var units = data.replace(/[\d\.]/g, '' )
+  var units = data.replace(/[\d.]/g, '' )
                   .replace(' ', '')
                   .toLowerCase();
   var multiplier = 1;
@@ -139,21 +150,25 @@ function ConvertDurationString(data) {
 function createTemplateURI(appId, templateName) {
   var words = document.baseURI.split('/');
   var ind = words.indexOf("proxy");
+  var baseURI;
   if (ind > 0) {
-    var baseURI = words.slice(0, ind + 1).join('/') + '/' + appId + '/static/' + templateName + '-template.html';
+    baseURI = words.slice(0, ind + 1).join('/') + '/' + appId + '/static/' + templateName + '-template.html';
     return baseURI;
   }
   ind = words.indexOf("history");
   if(ind > 0) {
-    var baseURI = words.slice(0, ind).join('/') + '/static/' + templateName + '-template.html';
+    baseURI = words.slice(0, ind).join('/') + '/static/' + templateName + '-template.html';
     return baseURI;
   }
-  return location.origin + "/static/" + templateName + "-template.html";
+  return uiRoot + "/static/" + templateName + "-template.html";
 }
 
 function setDataTableDefaults() {
   $.extend($.fn.dataTable.defaults, {
     stateSave: true,
+    stateSaveParams: function(_, data) {
+      data.search.search = "";
+    },
     lengthMenu: [[20, 40, 60, 100, -1], [20, 40, 60, 100, "All"]],
     pageLength: 20
   });
@@ -161,27 +176,55 @@ function setDataTableDefaults() {
 
 function formatDate(date) {
   if (date <= 0) return "-";
-  else return date.split(".")[0].replace("T", " ");
+  else {
+    var dt = new Date(date.replace("GMT", "Z"));
+    return formatDateString(dt);
+  }
 }
 
 function createRESTEndPointForExecutorsPage(appId) {
-    var words = document.baseURI.split('/');
-    var ind = words.indexOf("proxy");
-    if (ind > 0) {
-        var appId = words[ind + 1];
-        var newBaseURI = words.slice(0, ind + 2).join('/');
-        return newBaseURI + "/api/v1/applications/" + appId + "/allexecutors";
+  var words = document.baseURI.split('/');
+  var ind = words.indexOf("proxy");
+  var newBaseURI;
+  if (ind > 0) {
+    appId = words[ind + 1];
+    newBaseURI = words.slice(0, ind + 2).join('/');
+    return newBaseURI + "/api/v1/applications/" + appId + "/allexecutors";
+  }
+  ind = words.indexOf("history");
+  if (ind > 0) {
+    appId = words[ind + 1];
+    var attemptId = words[ind + 2];
+    newBaseURI = words.slice(0, ind).join('/');
+    if (isNaN(attemptId)) {
+      return newBaseURI + "/api/v1/applications/" + appId + "/allexecutors";
+    } else {
+      return newBaseURI + "/api/v1/applications/" + appId + "/" + attemptId + "/allexecutors";
     }
-    ind = words.indexOf("history");
-    if (ind > 0) {
-        var appId = words[ind + 1];
-        var attemptId = words[ind + 2];
-        var newBaseURI = words.slice(0, ind).join('/');
-        if (isNaN(attemptId)) {
-            return newBaseURI + "/api/v1/applications/" + appId + "/allexecutors";
-        } else {
-            return newBaseURI + "/api/v1/applications/" + appId + "/" + attemptId + "/allexecutors";
-        }
+  }
+  return uiRoot + "/api/v1/applications/" + appId + "/allexecutors";
+}
+
+function createRESTEndPointForMiscellaneousProcess(appId) {
+  var words = document.baseURI.split('/');
+  var ind = words.indexOf("proxy");
+  var newBaseURI;
+  if (ind > 0) {
+    appId = words[ind + 1];
+    newBaseURI = words.slice(0, ind + 2).join('/');
+    return newBaseURI + "/api/v1/applications/" + appId + "/allmiscellaneousprocess";
+  }
+  ind = words.indexOf("history");
+  if (ind > 0) {
+    appId = words[ind + 1];
+    var attemptId = words[ind + 2];
+    newBaseURI = words.slice(0, ind).join('/');
+    if (isNaN(attemptId)) {
+      return newBaseURI + "/api/v1/applications/" + appId + "/allmiscellaneousprocess";
+    } else {
+      return newBaseURI + "/api/v1/applications/" + appId + "/" + attemptId + "/allmiscellaneousprocess";
     }
-    return location.origin + "/api/v1/applications/" + appId + "/allexecutors";
+  }
+  return uiRoot + "/api/v1/applications/" + appId + "/allmiscellaneousprocess";
 }
+/* eslint-enable no-unused-vars */
\ No newline at end of file
diff --git a/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css b/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css
index d22a9ac801891..eeacbd802d305 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css
+++ b/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css
@@ -1,2 +1 @@
-.vis [class*=span]{min-height:0;width:auto}.vis .overlay{position:absolute;top:0;left:0;width:100%;height:100%;z-index:10}.vis-active{box-shadow:0 0 10px #86d5f8}div.vis-configuration{position:relative;display:block;float:left;font-size:12px}div.vis-configuration-wrapper{display:block;width:700px}div.vis-configuration-wrapper:after{clear:both;content:"";display:block}div.vis-configuration.vis-config-option-container{display:block;width:495px;background-color:#fff;border:2px solid #f7f8fa;border-radius:4px;margin-top:20px;left:10px;padding-left:5px}div.vis-configuration.vis-config-button{display:block;width:495px;height:25px;vertical-align:middle;line-height:25px;background-color:#f7f8fa;border:2px solid #ceced0;border-radius:4px;margin-top:20px;left:10px;padding-left:5px;cursor:pointer;margin-bottom:30px}div.vis-configuration.vis-config-button.hover{background-color:#4588e6;border:2px solid #214373;color:#fff}div.vis-configuration.vis-config-item{display:block;float:left;width:495px;height:25px;vertical-align:middle;line-height:25px}div.vis-configuration.vis-config-item.vis-config-s2{left:10px;background-color:#f7f8fa;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-item.vis-config-s3{left:20px;background-color:#e4e9f0;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-item.vis-config-s4{left:30px;background-color:#cfd8e6;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-header{font-size:18px;font-weight:700}div.vis-configuration.vis-config-label{width:120px;height:25px;line-height:25px}div.vis-configuration.vis-config-label.vis-config-s3{width:110px}div.vis-configuration.vis-config-label.vis-config-s4{width:100px}div.vis-configuration.vis-config-colorBlock{top:1px;width:30px;height:19px;border:1px solid #444;border-radius:2px;padding:0;margin:0;cursor:pointer}input.vis-configuration.vis-config-checkbox{left:-5px}input.vis-configuration.vis-config-rangeinput{position:relative;top:-5px;width:60px;padding:1px;margin:0;pointer-events:none}input.vis-configuration.vis-config-range{-webkit-appearance:none;border:0 solid #fff;background-color:transparent;width:300px;height:20px}input.vis-configuration.vis-config-range::-webkit-slider-runnable-track{width:300px;height:5px;background:#dedede;background:-moz-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#dedede),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#dedede,#c8c8c8 99%);background:-o-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:linear-gradient(180deg,#dedede 0,#c8c8c8 99%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#dedede",endColorstr="#c8c8c8",GradientType=0);border:1px solid #999;box-shadow:0 0 3px 0 #aaa;border-radius:3px}input.vis-configuration.vis-config-range::-webkit-slider-thumb{-webkit-appearance:none;border:1px solid #14334b;height:17px;width:17px;border-radius:50%;background:#3876c2;background:-moz-linear-gradient(top,#3876c2 0,#385380 100%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#3876c2),color-stop(100%,#385380));background:-webkit-linear-gradient(top,#3876c2,#385380);background:-o-linear-gradient(top,#3876c2 0,#385380 100%);background:-ms-linear-gradient(top,#3876c2 0,#385380 100%);background:linear-gradient(180deg,#3876c2 0,#385380);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#3876c2",endColorstr="#385380",GradientType=0);box-shadow:0 0 1px 0 #111927;margin-top:-7px}input.vis-configuration.vis-config-range:focus{outline:none}input.vis-configuration.vis-config-range:focus::-webkit-slider-runnable-track{background:#9d9d9d;background:-moz-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#9d9d9d),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#9d9d9d,#c8c8c8 99%);background:-o-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:linear-gradient(180deg,#9d9d9d 0,#c8c8c8 99%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#9d9d9d",endColorstr="#c8c8c8",GradientType=0)}input.vis-configuration.vis-config-range::-moz-range-track{width:300px;height:10px;background:#dedede;background:-moz-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#dedede),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#dedede,#c8c8c8 99%);background:-o-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:linear-gradient(180deg,#dedede 0,#c8c8c8 99%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#dedede",endColorstr="#c8c8c8",GradientType=0);border:1px solid #999;box-shadow:0 0 3px 0 #aaa;border-radius:3px}input.vis-configuration.vis-config-range::-moz-range-thumb{border:none;height:16px;width:16px;border-radius:50%;background:#385380}input.vis-configuration.vis-config-range:-moz-focusring{outline:1px solid #fff;outline-offset:-1px}input.vis-configuration.vis-config-range::-ms-track{width:300px;height:5px;background:transparent;border-color:transparent;border-width:6px 0;color:transparent}input.vis-configuration.vis-config-range::-ms-fill-lower{background:#777;border-radius:10px}input.vis-configuration.vis-config-range::-ms-fill-upper{background:#ddd;border-radius:10px}input.vis-configuration.vis-config-range::-ms-thumb{border:none;height:16px;width:16px;border-radius:50%;background:#385380}input.vis-configuration.vis-config-range:focus::-ms-fill-lower{background:#888}input.vis-configuration.vis-config-range:focus::-ms-fill-upper{background:#ccc}.vis-configuration-popup{position:absolute;background:rgba(57,76,89,.85);border:2px solid #f2faff;line-height:30px;height:30px;width:150px;text-align:center;color:#fff;font-size:14px;border-radius:4px;-webkit-transition:opacity .3s ease-in-out;-moz-transition:opacity .3s ease-in-out;transition:opacity .3s ease-in-out}.vis-configuration-popup:after,.vis-configuration-popup:before{left:100%;top:50%;border:solid transparent;content:" ";height:0;width:0;position:absolute;pointer-events:none}.vis-configuration-popup:after{border-color:rgba(136,183,213,0) rgba(136,183,213,0) rgba(136,183,213,0) rgba(57,76,89,.85);border-width:8px;margin-top:-8px}.vis-configuration-popup:before{border-color:rgba(194,225,245,0) rgba(194,225,245,0) rgba(194,225,245,0) #f2faff;border-width:12px;margin-top:-12px}div.vis-tooltip{position:absolute;visibility:hidden;padding:5px;white-space:nowrap;font-family:verdana;font-size:14px;color:#000;background-color:#f5f4ed;-moz-border-radius:3px;-webkit-border-radius:3px;border-radius:3px;border:1px solid #808074;box-shadow:3px 3px 10px rgba(0,0,0,.2);pointer-events:none;z-index:5}.vis-panel{position:absolute;padding:0;margin:0;box-sizing:border-box}.vis-panel.vis-bottom,.vis-panel.vis-center,.vis-panel.vis-left,.vis-panel.vis-right,.vis-panel.vis-top{border:1px #bfbfbf}.vis-panel.vis-center,.vis-panel.vis-left,.vis-panel.vis-right{border-top-style:solid;border-bottom-style:solid;overflow:hidden}.vis-left.vis-panel.vis-vertical-scroll,.vis-right.vis-panel.vis-vertical-scroll{height:100%;overflow-x:hidden;overflow-y:scroll}.vis-left.vis-panel.vis-vertical-scroll{direction:rtl}.vis-left.vis-panel.vis-vertical-scroll .vis-content,.vis-right.vis-panel.vis-vertical-scroll{direction:ltr}.vis-right.vis-panel.vis-vertical-scroll .vis-content{direction:rtl}.vis-panel.vis-bottom,.vis-panel.vis-center,.vis-panel.vis-top{border-left-style:solid;border-right-style:solid}.vis-background{overflow:hidden}.vis-panel>.vis-content{position:relative}.vis-panel .vis-shadow{position:absolute;width:100%;height:1px;box-shadow:0 0 10px rgba(0,0,0,.8)}.vis-panel .vis-shadow.vis-top{top:-1px;left:0}.vis-panel .vis-shadow.vis-bottom{bottom:-1px;left:0}.vis-current-time{background-color:#ff7f6e;width:2px;z-index:1;pointer-events:none}.vis-rolling-mode-btn{height:40px;width:40px;position:absolute;top:7px;right:20px;border-radius:50%;font-size:28px;cursor:pointer;opacity:.8;color:#fff;font-weight:700;text-align:center;background:#3876c2}.vis-rolling-mode-btn:before{content:"\26F6"}.vis-rolling-mode-btn:hover{opacity:1}.vis-custom-time{background-color:#6e94ff;width:2px;cursor:move;z-index:1}.vis-custom-time>.vis-custom-time-marker{background-color:inherit;color:#fff;font-size:12px;white-space:nowrap;padding:3px 5px;top:0;cursor:auto;z-index:inherit}.vis-timeline{position:relative;border:1px solid #bfbfbf;overflow:hidden;padding:0;margin:0;box-sizing:border-box}.vis-loading-screen{width:100%;height:100%;position:absolute;top:0;left:0}.vis-graph-group0{fill:#4f81bd;fill-opacity:0;stroke-width:2px;stroke:#4f81bd}.vis-graph-group1{fill:#f79646;fill-opacity:0;stroke-width:2px;stroke:#f79646}.vis-graph-group2{fill:#8c51cf;fill-opacity:0;stroke-width:2px;stroke:#8c51cf}.vis-graph-group3{fill:#75c841;fill-opacity:0;stroke-width:2px;stroke:#75c841}.vis-graph-group4{fill:#ff0100;fill-opacity:0;stroke-width:2px;stroke:#ff0100}.vis-graph-group5{fill:#37d8e6;fill-opacity:0;stroke-width:2px;stroke:#37d8e6}.vis-graph-group6{fill:#042662;fill-opacity:0;stroke-width:2px;stroke:#042662}.vis-graph-group7{fill:#00ff26;fill-opacity:0;stroke-width:2px;stroke:#00ff26}.vis-graph-group8{fill:#f0f;fill-opacity:0;stroke-width:2px;stroke:#f0f}.vis-graph-group9{fill:#8f3938;fill-opacity:0;stroke-width:2px;stroke:#8f3938}.vis-timeline .vis-fill{fill-opacity:.1;stroke:none}.vis-timeline .vis-bar{fill-opacity:.5;stroke-width:1px}.vis-timeline .vis-point{stroke-width:2px;fill-opacity:1}.vis-timeline .vis-legend-background{stroke-width:1px;fill-opacity:.9;fill:#fff;stroke:#c2c2c2}.vis-timeline .vis-outline{stroke-width:1px;fill-opacity:1;fill:#fff;stroke:#e5e5e5}.vis-timeline .vis-icon-fill{fill-opacity:.3;stroke:none}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-horizontal{position:absolute;width:100%;height:0;border-bottom:1px solid}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-minor{border-color:#e5e5e5}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-major{border-color:#bfbfbf}.vis-data-axis .vis-y-axis.vis-major{width:100%;position:absolute;color:#4d4d4d;white-space:nowrap}.vis-data-axis .vis-y-axis.vis-major.vis-measure{padding:0;margin:0;border:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-minor{position:absolute;width:100%;color:#bebebe;white-space:nowrap}.vis-data-axis .vis-y-axis.vis-minor.vis-measure{padding:0;margin:0;border:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-title{position:absolute;color:#4d4d4d;white-space:nowrap;bottom:20px;text-align:center}.vis-data-axis .vis-y-axis.vis-title.vis-measure{padding:0;margin:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-title.vis-left{bottom:0;-webkit-transform-origin:left top;-moz-transform-origin:left top;-ms-transform-origin:left top;-o-transform-origin:left top;transform-origin:left bottom;-webkit-transform:rotate(-90deg);-moz-transform:rotate(-90deg);-ms-transform:rotate(-90deg);-o-transform:rotate(-90deg);transform:rotate(-90deg)}.vis-data-axis .vis-y-axis.vis-title.vis-right{bottom:0;-webkit-transform-origin:right bottom;-moz-transform-origin:right bottom;-ms-transform-origin:right bottom;-o-transform-origin:right bottom;transform-origin:right bottom;-webkit-transform:rotate(90deg);-moz-transform:rotate(90deg);-ms-transform:rotate(90deg);-o-transform:rotate(90deg);transform:rotate(90deg)}.vis-legend{background-color:rgba(247,252,255,.65);padding:5px;border:1px solid #b3b3b3;box-shadow:2px 2px 10px hsla(0,0%,60.4%,.55)}.vis-legend-text{white-space:nowrap;display:inline-block}.vis-itemset{position:relative;padding:0;margin:0;box-sizing:border-box}.vis-itemset .vis-background,.vis-itemset .vis-foreground{position:absolute;width:100%;height:100%;overflow:visible}.vis-axis{position:absolute;width:100%;height:0;left:0;z-index:1}.vis-foreground .vis-group{position:relative;box-sizing:border-box;border-bottom:1px solid #bfbfbf}.vis-foreground .vis-group:last-child{border-bottom:none}.vis-nesting-group{cursor:pointer}.vis-label.vis-nested-group.vis-group-level-unknown-but-gte1{background:#f5f5f5}.vis-label.vis-nested-group.vis-group-level-0{background-color:#fff}.vis-ltr .vis-label.vis-nested-group.vis-group-level-0 .vis-inner{padding-left:0}.vis-rtl .vis-label.vis-nested-group.vis-group-level-0 .vis-inner{padding-right:0}.vis-label.vis-nested-group.vis-group-level-1{background-color:rgba(0,0,0,.05)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-1 .vis-inner{padding-left:15px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-1 .vis-inner{padding-right:15px}.vis-label.vis-nested-group.vis-group-level-2{background-color:rgba(0,0,0,.1)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-2 .vis-inner{padding-left:30px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-2 .vis-inner{padding-right:30px}.vis-label.vis-nested-group.vis-group-level-3{background-color:rgba(0,0,0,.15)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-3 .vis-inner{padding-left:45px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-3 .vis-inner{padding-right:45px}.vis-label.vis-nested-group.vis-group-level-4{background-color:rgba(0,0,0,.2)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-4 .vis-inner{padding-left:60px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-4 .vis-inner{padding-right:60px}.vis-label.vis-nested-group.vis-group-level-5{background-color:rgba(0,0,0,.25)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-5 .vis-inner{padding-left:75px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-5 .vis-inner{padding-right:75px}.vis-label.vis-nested-group.vis-group-level-6{background-color:rgba(0,0,0,.3)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-6 .vis-inner{padding-left:90px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-6 .vis-inner{padding-right:90px}.vis-label.vis-nested-group.vis-group-level-7{background-color:rgba(0,0,0,.35)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-7 .vis-inner{padding-left:105px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-7 .vis-inner{padding-right:105px}.vis-label.vis-nested-group.vis-group-level-8{background-color:rgba(0,0,0,.4)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-8 .vis-inner{padding-left:120px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-8 .vis-inner{padding-right:120px}.vis-label.vis-nested-group.vis-group-level-9{background-color:rgba(0,0,0,.45)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-9 .vis-inner{padding-left:135px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-9 .vis-inner{padding-right:135px}.vis-label.vis-nested-group{background-color:rgba(0,0,0,.5)}.vis-ltr .vis-label.vis-nested-group .vis-inner{padding-left:150px}.vis-rtl .vis-label.vis-nested-group .vis-inner{padding-right:150px}.vis-group-level-unknown-but-gte1{border:1px solid red}.vis-label.vis-nesting-group:before{display:inline-block;width:15px}.vis-label.vis-nesting-group.expanded:before{content:"\25BC"}.vis-label.vis-nesting-group.collapsed:before{content:"\25B6"}.vis-rtl .vis-label.vis-nesting-group.collapsed:before{content:"\25C0"}.vis-ltr .vis-label:not(.vis-nesting-group):not(.vis-group-level-0){padding-left:15px}.vis-rtl .vis-label:not(.vis-nesting-group):not(.vis-group-level-0){padding-right:15px}.vis-overlay{position:absolute;top:0;left:0;width:100%;height:100%;z-index:10}.vis-labelset{overflow:hidden}.vis-labelset,.vis-labelset .vis-label{position:relative;box-sizing:border-box}.vis-labelset .vis-label{left:0;top:0;width:100%;color:#4d4d4d;border-bottom:1px solid #bfbfbf}.vis-labelset .vis-label.draggable{cursor:pointer}.vis-group-is-dragging{background:rgba(0,0,0,.1)}.vis-labelset .vis-label:last-child{border-bottom:none}.vis-labelset .vis-label .vis-inner{display:inline-block;padding:5px}.vis-labelset .vis-label .vis-inner.vis-hidden{padding:0}.vis-time-axis{position:relative;overflow:hidden}.vis-time-axis.vis-foreground{top:0;left:0;width:100%}.vis-time-axis.vis-background{position:absolute;top:0;left:0;width:100%;height:100%}.vis-time-axis .vis-text{position:absolute;color:#4d4d4d;padding:3px;overflow:hidden;box-sizing:border-box;white-space:nowrap}.vis-time-axis .vis-text.vis-measure{position:absolute;padding-left:0;padding-right:0;margin-left:0;margin-right:0;visibility:hidden}.vis-time-axis .vis-grid.vis-vertical{position:absolute;border-left:1px solid}.vis-time-axis .vis-grid.vis-vertical-rtl{position:absolute;border-right:1px solid}.vis-time-axis .vis-grid.vis-minor{border-color:#e5e5e5}.vis-time-axis .vis-grid.vis-major{border-color:#bfbfbf}.vis-item{position:absolute;color:#1a1a1a;border-color:#97b0f8;border-width:1px;background-color:#d5ddf6;display:inline-block;z-index:1}.vis-item.vis-selected{border-color:#ffc200;background-color:#fff785;z-index:2}.vis-editable.vis-selected{cursor:move}.vis-item.vis-point.vis-selected{background-color:#fff785}.vis-item.vis-box{text-align:center;border-style:solid;border-radius:2px}.vis-item.vis-point{background:none}.vis-item.vis-dot{position:absolute;padding:0;border-width:4px;border-style:solid;border-radius:4px}.vis-item.vis-range{border-style:solid;border-radius:2px;box-sizing:border-box}.vis-item.vis-background{border:none;background-color:rgba(213,221,246,.4);box-sizing:border-box;padding:0;margin:0}.vis-item .vis-item-overflow{position:relative;width:100%;height:100%;padding:0;margin:0;overflow:hidden}.vis-item-visible-frame{white-space:nowrap}.vis-item.vis-range .vis-item-content{position:relative;display:inline-block}.vis-item.vis-background .vis-item-content{position:absolute;display:inline-block}.vis-item.vis-line{padding:0;position:absolute;width:0;border-left-width:1px;border-left-style:solid}.vis-item .vis-item-content{white-space:nowrap;box-sizing:border-box;padding:5px}.vis-item .vis-onUpdateTime-tooltip{position:absolute;background:#4f81bd;color:#fff;width:200px;text-align:center;white-space:nowrap;padding:5px;border-radius:1px;transition:.4s;-o-transition:.4s;-moz-transition:.4s;-webkit-transition:.4s}.vis-item .vis-delete,.vis-item .vis-delete-rtl{position:absolute;top:0;width:24px;height:24px;box-sizing:border-box;padding:0 5px;cursor:pointer;-webkit-transition:background .2s linear;-moz-transition:background .2s linear;-ms-transition:background .2s linear;-o-transition:background .2s linear;transition:background .2s linear}.vis-item .vis-delete{right:-24px}.vis-item .vis-delete-rtl{left:-24px}.vis-item .vis-delete-rtl:after,.vis-item .vis-delete:after{content:"\00D7";color:red;font-family:arial,sans-serif;font-size:22px;font-weight:700;-webkit-transition:color .2s linear;-moz-transition:color .2s linear;-ms-transition:color .2s linear;-o-transition:color .2s linear;transition:color .2s linear}.vis-item .vis-delete-rtl:hover,.vis-item .vis-delete:hover{background:red}.vis-item .vis-delete-rtl:hover:after,.vis-item .vis-delete:hover:after{color:#fff}.vis-item .vis-drag-center{position:absolute;width:100%;height:100%;top:0;left:0;cursor:move}.vis-item.vis-range .vis-drag-left{left:-4px;cursor:w-resize}.vis-item.vis-range .vis-drag-left,.vis-item.vis-range .vis-drag-right{position:absolute;width:24px;max-width:20%;min-width:2px;height:100%;top:0}.vis-item.vis-range .vis-drag-right{right:-4px;cursor:e-resize}.vis-range.vis-item.vis-readonly .vis-drag-left,.vis-range.vis-item.vis-readonly .vis-drag-right{cursor:auto}.vis-item.vis-cluster{vertical-align:center;text-align:center;border-style:solid;border-radius:2px}.vis-item.vis-cluster-line{padding:0;position:absolute;width:0;border-left-width:1px;border-left-style:solid}.vis-item.vis-cluster-dot{position:absolute;padding:0;border-width:4px;border-style:solid;border-radius:4px}
-/*# sourceMappingURL=vis-timeline-graph2d.min.css.map */
\ No newline at end of file
+.vis .overlay{position:absolute;top:0;left:0;width:100%;height:100%;z-index:10}.vis-active{box-shadow:0 0 10px #86d5f8}.vis [class*=span]{min-height:0;width:auto}div.vis-configuration{position:relative;display:block;float:left;font-size:12px}div.vis-configuration-wrapper{display:block;width:700px}div.vis-configuration-wrapper::after{clear:both;content:"";display:block}div.vis-configuration.vis-config-option-container{display:block;width:495px;background-color:#fff;border:2px solid #f7f8fa;border-radius:4px;margin-top:20px;left:10px;padding-left:5px}div.vis-configuration.vis-config-button{display:block;width:495px;height:25px;vertical-align:middle;line-height:25px;background-color:#f7f8fa;border:2px solid #ceced0;border-radius:4px;margin-top:20px;left:10px;padding-left:5px;cursor:pointer;margin-bottom:30px}div.vis-configuration.vis-config-button.hover{background-color:#4588e6;border:2px solid #214373;color:#fff}div.vis-configuration.vis-config-item{display:block;float:left;width:495px;height:25px;vertical-align:middle;line-height:25px}div.vis-configuration.vis-config-item.vis-config-s2{left:10px;background-color:#f7f8fa;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-item.vis-config-s3{left:20px;background-color:#e4e9f0;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-item.vis-config-s4{left:30px;background-color:#cfd8e6;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-header{font-size:18px;font-weight:700}div.vis-configuration.vis-config-label{width:120px;height:25px;line-height:25px}div.vis-configuration.vis-config-label.vis-config-s3{width:110px}div.vis-configuration.vis-config-label.vis-config-s4{width:100px}div.vis-configuration.vis-config-colorBlock{top:1px;width:30px;height:19px;border:1px solid #444;border-radius:2px;padding:0;margin:0;cursor:pointer}input.vis-configuration.vis-config-checkbox{left:-5px}input.vis-configuration.vis-config-rangeinput{position:relative;top:-5px;width:60px;padding:1px;margin:0;pointer-events:none}input.vis-configuration.vis-config-range{-webkit-appearance:none;border:0 solid #fff;background-color:rgba(0,0,0,0);width:300px;height:20px}input.vis-configuration.vis-config-range::-webkit-slider-runnable-track{width:300px;height:5px;background:#dedede;background:-moz-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#dedede),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-o-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:linear-gradient(to bottom,#dedede 0,#c8c8c8 99%);border:1px solid #999;box-shadow:#aaa 0 0 3px 0;border-radius:3px}input.vis-configuration.vis-config-range::-webkit-slider-thumb{-webkit-appearance:none;border:1px solid #14334b;height:17px;width:17px;border-radius:50%;background:#3876c2;background:-moz-linear-gradient(top,#3876c2 0,#385380 100%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#3876c2),color-stop(100%,#385380));background:-webkit-linear-gradient(top,#3876c2 0,#385380 100%);background:-o-linear-gradient(top,#3876c2 0,#385380 100%);background:-ms-linear-gradient(top,#3876c2 0,#385380 100%);background:linear-gradient(to bottom,#3876c2 0,#385380 100%);box-shadow:#111927 0 0 1px 0;margin-top:-7px}input.vis-configuration.vis-config-range:focus{outline:0}input.vis-configuration.vis-config-range:focus::-webkit-slider-runnable-track{background:#9d9d9d;background:-moz-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#9d9d9d),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-o-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:linear-gradient(to bottom,#9d9d9d 0,#c8c8c8 99%)}input.vis-configuration.vis-config-range::-moz-range-track{width:300px;height:10px;background:#dedede;background:-moz-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#dedede),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-o-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:linear-gradient(to bottom,#dedede 0,#c8c8c8 99%);border:1px solid #999;box-shadow:#aaa 0 0 3px 0;border-radius:3px}input.vis-configuration.vis-config-range::-moz-range-thumb{border:none;height:16px;width:16px;border-radius:50%;background:#385380}input.vis-configuration.vis-config-range:-moz-focusring{outline:1px solid #fff;outline-offset:-1px}input.vis-configuration.vis-config-range::-ms-track{width:300px;height:5px;background:0 0;border-color:transparent;border-width:6px 0;color:transparent}input.vis-configuration.vis-config-range::-ms-fill-lower{background:#777;border-radius:10px}input.vis-configuration.vis-config-range::-ms-fill-upper{background:#ddd;border-radius:10px}input.vis-configuration.vis-config-range::-ms-thumb{border:none;height:16px;width:16px;border-radius:50%;background:#385380}input.vis-configuration.vis-config-range:focus::-ms-fill-lower{background:#888}input.vis-configuration.vis-config-range:focus::-ms-fill-upper{background:#ccc}.vis-configuration-popup{position:absolute;background:rgba(57,76,89,.85);border:2px solid #f2faff;line-height:30px;height:30px;width:150px;text-align:center;color:#fff;font-size:14px;border-radius:4px;-webkit-transition:opacity .3s ease-in-out;-moz-transition:opacity .3s ease-in-out;transition:opacity .3s ease-in-out}.vis-configuration-popup:after,.vis-configuration-popup:before{left:100%;top:50%;border:solid transparent;content:" ";height:0;width:0;position:absolute;pointer-events:none}.vis-configuration-popup:after{border-color:rgba(136,183,213,0);border-left-color:rgba(57,76,89,.85);border-width:8px;margin-top:-8px}.vis-configuration-popup:before{border-color:rgba(194,225,245,0);border-left-color:#f2faff;border-width:12px;margin-top:-12px}div.vis-tooltip{position:absolute;visibility:hidden;padding:5px;white-space:nowrap;font-family:verdana;font-size:14px;color:#000;background-color:#f5f4ed;-moz-border-radius:3px;-webkit-border-radius:3px;border-radius:3px;border:1px solid #808074;box-shadow:3px 3px 10px rgba(0,0,0,.2);pointer-events:none;z-index:5}.vis-current-time{background-color:#ff7f6e;width:2px;z-index:1;pointer-events:none}.vis-rolling-mode-btn{height:40px;width:40px;position:absolute;top:7px;right:20px;border-radius:50%;font-size:28px;cursor:pointer;opacity:.8;color:#fff;font-weight:700;text-align:center;background:#3876c2}.vis-rolling-mode-btn:before{content:"\26F6"}.vis-rolling-mode-btn:hover{opacity:1}.vis-custom-time{background-color:#6e94ff;width:2px;cursor:move;z-index:1}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-horizontal{position:absolute;width:100%;height:0;border-bottom:1px solid}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-minor{border-color:#e5e5e5}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-major{border-color:#bfbfbf}.vis-data-axis .vis-y-axis.vis-major{width:100%;position:absolute;color:#4d4d4d;white-space:nowrap}.vis-data-axis .vis-y-axis.vis-major.vis-measure{padding:0;margin:0;border:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-minor{position:absolute;width:100%;color:#bebebe;white-space:nowrap}.vis-data-axis .vis-y-axis.vis-minor.vis-measure{padding:0;margin:0;border:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-title{position:absolute;color:#4d4d4d;white-space:nowrap;bottom:20px;text-align:center}.vis-data-axis .vis-y-axis.vis-title.vis-measure{padding:0;margin:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-title.vis-left{bottom:0;-webkit-transform-origin:left top;-moz-transform-origin:left top;-ms-transform-origin:left top;-o-transform-origin:left top;transform-origin:left bottom;-webkit-transform:rotate(-90deg);-moz-transform:rotate(-90deg);-ms-transform:rotate(-90deg);-o-transform:rotate(-90deg);transform:rotate(-90deg)}.vis-data-axis .vis-y-axis.vis-title.vis-right{bottom:0;-webkit-transform-origin:right bottom;-moz-transform-origin:right bottom;-ms-transform-origin:right bottom;-o-transform-origin:right bottom;transform-origin:right bottom;-webkit-transform:rotate(90deg);-moz-transform:rotate(90deg);-ms-transform:rotate(90deg);-o-transform:rotate(90deg);transform:rotate(90deg)}.vis-legend{background-color:rgba(247,252,255,.65);padding:5px;border:1px solid #b3b3b3;box-shadow:2px 2px 10px rgba(154,154,154,.55)}.vis-legend-text{white-space:nowrap;display:inline-block}.vis-item{position:absolute;color:#1a1a1a;border-color:#97b0f8;border-width:1px;background-color:#d5ddf6;display:inline-block;z-index:1}.vis-item.vis-selected{border-color:#ffc200;background-color:#fff785;z-index:2}.vis-editable.vis-selected{cursor:move}.vis-item.vis-point.vis-selected{background-color:#fff785}.vis-item.vis-box{text-align:center;border-style:solid;border-radius:2px}.vis-item.vis-point{background:0 0}.vis-item.vis-dot{position:absolute;padding:0;border-width:4px;border-style:solid;border-radius:4px}.vis-item.vis-range{border-style:solid;border-radius:2px;box-sizing:border-box}.vis-item.vis-background{border:none;background-color:rgba(213,221,246,.4);box-sizing:border-box;padding:0;margin:0}.vis-item .vis-item-overflow{position:relative;width:100%;height:100%;padding:0;margin:0;overflow:hidden}.vis-item-visible-frame{white-space:nowrap}.vis-item.vis-range .vis-item-content{position:relative;display:inline-block}.vis-item.vis-background .vis-item-content{position:absolute;display:inline-block}.vis-item.vis-line{padding:0;position:absolute;width:0;border-left-width:1px;border-left-style:solid}.vis-item .vis-item-content{white-space:nowrap;box-sizing:border-box;padding:5px}.vis-item .vis-onUpdateTime-tooltip{position:absolute;background:#4f81bd;color:#fff;width:200px;text-align:center;white-space:nowrap;padding:5px;border-radius:1px;transition:.4s;-o-transition:.4s;-moz-transition:.4s;-webkit-transition:.4s}.vis-item .vis-delete,.vis-item .vis-delete-rtl{position:absolute;top:0;width:24px;height:24px;box-sizing:border-box;padding:0 5px;cursor:pointer;-webkit-transition:background .2s linear;-moz-transition:background .2s linear;-ms-transition:background .2s linear;-o-transition:background .2s linear;transition:background .2s linear}.vis-item .vis-delete{right:-24px}.vis-item .vis-delete-rtl{left:-24px}.vis-item .vis-delete-rtl:after,.vis-item .vis-delete:after{content:"\00D7";color:red;font-family:arial,sans-serif;font-size:22px;font-weight:700;-webkit-transition:color .2s linear;-moz-transition:color .2s linear;-ms-transition:color .2s linear;-o-transition:color .2s linear;transition:color .2s linear}.vis-item .vis-delete-rtl:hover,.vis-item .vis-delete:hover{background:red}.vis-item .vis-delete-rtl:hover:after,.vis-item .vis-delete:hover:after{color:#fff}.vis-item .vis-drag-center{position:absolute;width:100%;height:100%;top:0;left:0;cursor:move}.vis-item.vis-range .vis-drag-left{position:absolute;width:24px;max-width:20%;min-width:2px;height:100%;top:0;left:-4px;cursor:w-resize}.vis-item.vis-range .vis-drag-right{position:absolute;width:24px;max-width:20%;min-width:2px;height:100%;top:0;right:-4px;cursor:e-resize}.vis-range.vis-item.vis-readonly .vis-drag-left,.vis-range.vis-item.vis-readonly .vis-drag-right{cursor:auto}.vis-itemset{position:relative;padding:0;margin:0;box-sizing:border-box}.vis-itemset .vis-background,.vis-itemset .vis-foreground{position:absolute;width:100%;height:100%;overflow:visible}.vis-axis{position:absolute;width:100%;height:0;left:0;z-index:1}.vis-foreground .vis-group{position:relative;box-sizing:border-box;border-bottom:1px solid #bfbfbf}.vis-foreground .vis-group:last-child{border-bottom:none}.vis-nesting-group{cursor:pointer}.vis-nested-group{background:#f5f5f5}.vis-label.vis-nesting-group.expanded:before{content:"\25BC"}.vis-label.vis-nesting-group.collapsed-rtl:before{content:"\25C0"}.vis-label.vis-nesting-group.collapsed:before{content:"\25B6"}.vis-overlay{position:absolute;top:0;left:0;width:100%;height:100%;z-index:10}.vis-labelset{position:relative;overflow:hidden;box-sizing:border-box}.vis-labelset .vis-label{position:relative;left:0;top:0;width:100%;color:#4d4d4d;box-sizing:border-box}.vis-labelset .vis-label{border-bottom:1px solid #bfbfbf}.vis-labelset .vis-label.draggable{cursor:pointer}.vis-labelset .vis-label:last-child{border-bottom:none}.vis-labelset .vis-label .vis-inner{display:inline-block;padding:5px}.vis-labelset .vis-label .vis-inner.vis-hidden{padding:0}.vis-panel{position:absolute;padding:0;margin:0;box-sizing:border-box}.vis-panel.vis-bottom,.vis-panel.vis-center,.vis-panel.vis-left,.vis-panel.vis-right,.vis-panel.vis-top{border:1px #bfbfbf}.vis-panel.vis-center,.vis-panel.vis-left,.vis-panel.vis-right{border-top-style:solid;border-bottom-style:solid;overflow:hidden}.vis-left.vis-panel.vis-vertical-scroll,.vis-right.vis-panel.vis-vertical-scroll{height:100%;overflow-x:hidden;overflow-y:scroll}.vis-left.vis-panel.vis-vertical-scroll{direction:rtl}.vis-left.vis-panel.vis-vertical-scroll .vis-content{direction:ltr}.vis-right.vis-panel.vis-vertical-scroll{direction:ltr}.vis-right.vis-panel.vis-vertical-scroll .vis-content{direction:rtl}.vis-panel.vis-bottom,.vis-panel.vis-center,.vis-panel.vis-top{border-left-style:solid;border-right-style:solid}.vis-background{overflow:hidden}.vis-panel>.vis-content{position:relative}.vis-panel .vis-shadow{position:absolute;width:100%;height:1px;box-shadow:0 0 10px rgba(0,0,0,.8)}.vis-panel .vis-shadow.vis-top{top:-1px;left:0}.vis-panel .vis-shadow.vis-bottom{bottom:-1px;left:0}.vis-graph-group0{fill:#4f81bd;fill-opacity:0;stroke-width:2px;stroke:#4f81bd}.vis-graph-group1{fill:#f79646;fill-opacity:0;stroke-width:2px;stroke:#f79646}.vis-graph-group2{fill:#8c51cf;fill-opacity:0;stroke-width:2px;stroke:#8c51cf}.vis-graph-group3{fill:#75c841;fill-opacity:0;stroke-width:2px;stroke:#75c841}.vis-graph-group4{fill:#ff0100;fill-opacity:0;stroke-width:2px;stroke:#ff0100}.vis-graph-group5{fill:#37d8e6;fill-opacity:0;stroke-width:2px;stroke:#37d8e6}.vis-graph-group6{fill:#042662;fill-opacity:0;stroke-width:2px;stroke:#042662}.vis-graph-group7{fill:#00ff26;fill-opacity:0;stroke-width:2px;stroke:#00ff26}.vis-graph-group8{fill:#f0f;fill-opacity:0;stroke-width:2px;stroke:#f0f}.vis-graph-group9{fill:#8f3938;fill-opacity:0;stroke-width:2px;stroke:#8f3938}.vis-timeline .vis-fill{fill-opacity:.1;stroke:none}.vis-timeline .vis-bar{fill-opacity:.5;stroke-width:1px}.vis-timeline .vis-point{stroke-width:2px;fill-opacity:1}.vis-timeline .vis-legend-background{stroke-width:1px;fill-opacity:.9;fill:#fff;stroke:#c2c2c2}.vis-timeline .vis-outline{stroke-width:1px;fill-opacity:1;fill:#fff;stroke:#e5e5e5}.vis-timeline .vis-icon-fill{fill-opacity:.3;stroke:none}.vis-time-axis{position:relative;overflow:hidden}.vis-time-axis.vis-foreground{top:0;left:0;width:100%}.vis-time-axis.vis-background{position:absolute;top:0;left:0;width:100%;height:100%}.vis-time-axis .vis-text{position:absolute;color:#4d4d4d;padding:3px;overflow:hidden;box-sizing:border-box;white-space:nowrap}.vis-time-axis .vis-text.vis-measure{position:absolute;padding-left:0;padding-right:0;margin-left:0;margin-right:0;visibility:hidden}.vis-time-axis .vis-grid.vis-vertical{position:absolute;border-left:1px solid}.vis-time-axis .vis-grid.vis-vertical-rtl{position:absolute;border-right:1px solid}.vis-time-axis .vis-grid.vis-minor{border-color:#e5e5e5}.vis-time-axis .vis-grid.vis-major{border-color:#bfbfbf}.vis-timeline{position:relative;border:1px solid #bfbfbf;overflow:hidden;padding:0;margin:0;box-sizing:border-box}
\ No newline at end of file
diff --git a/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css.map b/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css.map
deleted file mode 100644
index 3c165a792d0f3..0000000000000
--- a/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css.map
+++ /dev/null
@@ -1 +0,0 @@
-{"version":3,"sources":["bootstrap.css","activator.css","configuration.css","tooltip.css","panel.css","currenttime.css","customtime.css","timeline.css","pathStyles.css","dataaxis.css","itemset.css","labelset.css","timeaxis.css","item.css"],"names":[],"mappings":"AAEA,mBACE,YAAa,CACb,UACF,CCLA,cACE,iBAAkB,CAClB,KAAM,CACN,MAAO,CACP,UAAW,CACX,WAAY,CAGZ,UACF,CAEA,YACE,2BACF,CCbA,sBACI,iBAAiB,CACjB,aAAa,CACb,UAAU,CACV,cACJ,CAEA,8BACI,aAAa,CACb,WACJ,CAEA,oCACE,UAAW,CACX,UAAW,CACX,aACF,CAEA,kDACI,aAAa,CACb,WAAW,CACX,qBAAyB,CACzB,wBAAwB,CACxB,iBAAiB,CACjB,eAAe,CACf,SAAS,CACT,gBACJ,CAEA,wCACI,aAAa,CACb,WAAW,CACX,WAAW,CACX,qBAAsB,CACtB,gBAAgB,CAChB,wBAAyB,CACzB,wBAAwB,CACxB,iBAAiB,CACjB,eAAe,CACf,SAAS,CACT,gBAAgB,CAChB,cAAe,CACf,kBACJ,CAEA,8CACI,wBAAyB,CACzB,wBAAwB,CACxB,UACJ,CAEA,sCACI,aAAa,CACb,UAAU,CACV,WAAW,CACX,WAAW,CACX,qBAAsB,CACtB,gBACJ,CAGA,oDACI,SAAS,CACT,wBAAyB,CACzB,gBAAgB,CAChB,iBACJ,CACA,oDACI,SAAS,CACT,wBAAyB,CACzB,gBAAgB,CAChB,iBACJ,CACA,oDACI,SAAS,CACT,wBAAyB,CACzB,gBAAgB,CAChB,iBACJ,CAEA,wCACI,cAAc,CACd,eACJ,CAEA,uCACI,WAAW,CACX,WAAW,CACX,gBACJ,CAEA,qDACI,WACJ,CACA,qDACI,WACJ,CAEA,4CACI,OAAO,CACP,UAAU,CACV,WAAW,CACX,qBAAwB,CACxB,iBAAiB,CACjB,SAAW,CACX,QAAU,CACV,cACJ,CAEA,4CACI,SACJ,CAGA,8CACI,iBAAiB,CACjB,QAAQ,CACR,UAAU,CAEV,WAAW,CACX,QAAQ,CACR,mBACJ,CAEA,yCAEI,uBAAwB,CAGxB,mBAAuB,CACvB,4BAA8B,CAG9B,WAAY,CACZ,WACJ,CACA,wEACI,WAAY,CACZ,UAAW,CACX,kBAAmB,CACnB,0DAA+D,CAC/D,sGAA4G,CAC5G,2DAAiE,CACjE,wDAA4D,CAC5D,yDAA6D,CAC7D,wDAA+D,CAC/D,+GAAmH,CAEnH,qBAAyB,CACzB,yBAAmC,CACnC,iBACJ,CACA,+DACI,uBAAwB,CACxB,wBAAyB,CACzB,WAAY,CACZ,UAAW,CACX,iBAAkB,CAClB,kBAAmB,CACnB,2DAAgE,CAChE,uGAA6G,CAC7G,uDAAkE,CAClE,yDAA6D,CAC7D,0DAA8D,CAC9D,oDAAgE,CAChE,+GAAmH,CACnH,4BAAmC,CACnC,eACJ,CACA,+CACI,YACJ,CACA,8EACI,kBAAmB,CACnB,0DAA8D,CAC9D,sGAA4G,CAC5G,2DAAiE,CACjE,wDAA4D,CAC5D,yDAA6D,CAC7D,wDAA+D,CAC/D,+GACJ,CAEA,2DACI,WAAY,CACZ,WAAY,CACZ,kBAAmB,CACnB,0DAA+D,CAC/D,sGAA4G,CAC5G,2DAAiE,CACjE,wDAA4D,CAC5D,yDAA6D,CAC7D,wDAA+D,CAC/D,+GAAmH,CAEnH,qBAAyB,CACzB,yBAAmC,CACnC,iBACJ,CACA,2DACI,WAAY,CACZ,WAAY,CACZ,UAAW,CAEX,iBAAkB,CAClB,kBACJ,CAGA,wDACI,sBAAwB,CACxB,mBACJ,CAEA,oDACI,WAAY,CACZ,UAAW,CAGX,sBAAuB,CAGvB,wBAAyB,CACzB,kBAAmB,CAGnB,iBACJ,CACA,yDACI,eAAgB,CAChB,kBACJ,CACA,yDACI,eAAgB,CAChB,kBACJ,CACA,oDACI,WAAY,CACZ,WAAY,CACZ,UAAW,CACX,iBAAkB,CAClB,kBACJ,CACA,+DACI,eACJ,CACA,+DACI,eACJ,CAEA,yBACI,iBAAkB,CAClB,6BAAkC,CAClC,wBAAyB,CACzB,gBAAgB,CAChB,WAAW,CACX,WAAW,CACX,iBAAiB,CACjB,UAAc,CACd,cAAc,CACd,iBAAiB,CACjB,0CAA4C,CAC5C,uCAAyC,CACzC,kCACJ,CACA,+DACI,SAAU,CACV,OAAQ,CACR,wBAAyB,CACzB,WAAY,CACZ,QAAS,CACT,OAAQ,CACR,iBAAkB,CAClB,mBACJ,CAEA,+BAEI,2FAAyC,CACzC,gBAAiB,CACjB,eACJ,CACA,gCAEI,gFAA0B,CAC1B,iBAAkB,CAClB,gBACJ,CC/RA,gBACE,iBAAkB,CAClB,iBAAkB,CAClB,WAAY,CACZ,kBAAmB,CAEnB,mBAAoB,CACpB,cAAc,CACd,UAAa,CACb,wBAAyB,CAEzB,sBAAuB,CACvB,yBAA0B,CAC1B,iBAAkB,CAClB,wBAAyB,CAEzB,sCAA2C,CAC3C,mBAAoB,CAEpB,SACF,CCpBA,WACE,iBAAkB,CAElB,SAAU,CACV,QAAS,CAET,qBACF,CAEA,wGAKE,kBACF,CAEA,+DAGE,sBAAuB,CACvB,yBAA0B,CAC1B,eACF,CAEA,iFACE,WAAY,CACZ,iBAAkB,CAClB,iBACF,CAEA,wCACE,aACF,CAMA,8FACE,aACF,CAEA,sDACE,aACF,CAEA,+DAGE,uBAAwB,CACxB,wBACF,CAEA,gBACE,eACF,CAEA,wBACE,iBACF,CAEA,uBACE,iBAAkB,CAClB,UAAW,CACX,UAAW,CACX,kCAIF,CAEA,+BACE,QAAS,CACT,MACF,CAEA,kCACE,WAAY,CACZ,MACF,CChFA,kBACE,wBAAyB,CACzB,SAAU,CACV,SAAU,CACV,mBACF,CAEA,sBACE,WAAY,CACZ,UAAW,CACX,iBAAkB,CAClB,OAAQ,CACR,UAAW,CACX,iBAAkB,CAClB,cAAe,CACf,cAAe,CACf,UAAY,CACZ,UAAY,CACZ,eAAiB,CACjB,iBAAkB,CAClB,kBACF,CACA,6BACE,eACF,CAEA,4BACE,SACF,CC5BA,iBACE,wBAAyB,CACzB,SAAU,CACV,WAAY,CACZ,SACF,CAEA,yCACE,wBAAyB,CACzB,UAAY,CACZ,cAAe,CACf,kBAAmB,CACnB,eAAgB,CAChB,KAAQ,CACR,WAAe,CACf,eACF,CCfA,cACE,iBAAkB,CAClB,wBAAyB,CACzB,eAAgB,CAChB,SAAU,CACV,QAAS,CACT,qBACF,CAEA,oBACE,UAAW,CACX,WAAY,CACZ,iBAAkB,CAClB,KAAM,CACN,MACF,CChBA,kBACI,YAAY,CACZ,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,kBACI,YAAY,CACZ,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,kBACI,YAAa,CACb,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,kBACI,YAAa,CACb,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,kBACI,YAAa,CACb,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,kBACI,YAAa,CACb,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,kBACI,YAAa,CACb,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,kBACI,YAAY,CACZ,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,kBACI,SAAY,CACZ,cAAc,CACd,gBAAgB,CAChB,WACJ,CAEA,kBACI,YAAa,CACb,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,wBACI,eAAgB,CAChB,WACJ,CAGA,uBACI,eAAgB,CAChB,gBACJ,CAEA,yBACI,gBAAgB,CAChB,cACJ,CAGA,qCACI,gBAAgB,CAChB,eAAgB,CAChB,SAAa,CACb,cACJ,CAGA,2BACI,gBAAgB,CAChB,cAAc,CACd,SAAa,CACb,cACJ,CAEA,6BACI,eAAgB,CAChB,WACJ,CCxGA,kEACE,iBAAkB,CAClB,UAAW,CACX,QAAS,CACT,uBACF,CAEA,6DACE,oBACF,CAEA,6DACE,oBACF,CAGA,qCACE,UAAW,CACX,iBAAkB,CAClB,aAAc,CACd,kBACF,CAEA,iDACE,SAAU,CACV,QAAS,CACT,QAAS,CACT,iBAAkB,CAClB,UACF,CAGA,qCACE,iBAAkB,CAClB,UAAW,CACX,aAAc,CACd,kBACF,CAEA,iDACE,SAAU,CACV,QAAS,CACT,QAAS,CACT,iBAAkB,CAClB,UACF,CAEA,qCACE,iBAAkB,CAClB,aAAc,CACd,kBAAmB,CACnB,WAAY,CACZ,iBACF,CAEA,iDACE,SAAU,CACV,QAAS,CACT,iBAAkB,CAClB,UACF,CAEA,8CACE,QAAS,CACT,iCAAkC,CAClC,8BAA+B,CAC/B,6BAA8B,CAC9B,4BAA6B,CAC7B,4BAA6B,CAC7B,gCAAiC,CACjC,6BAA8B,CAC9B,4BAA6B,CAC7B,2BAA4B,CAC5B,wBACF,CAEA,+CACE,QAAS,CACT,qCAAsC,CACtC,kCAAmC,CACnC,iCAAkC,CAClC,gCAAiC,CACjC,6BAA8B,CAC9B,+BAAgC,CAChC,4BAA6B,CAC7B,2BAA4B,CAC5B,0BAA2B,CAC3B,uBACF,CAEA,YACE,sCAA2C,CAC3C,WAAY,CACZ,wBAAyB,CACzB,4CACF,CAEA,iBAEE,kBAAmB,CACnB,oBACF,CCrGA,aACE,iBAAkB,CAClB,SAAU,CACV,QAAS,CAET,qBACF,CAEA,0DAEE,iBAAkB,CAClB,UAAW,CACX,WAAY,CACZ,gBACF,CAEA,UACE,iBAAkB,CAClB,UAAW,CACX,QAAS,CACT,MAAO,CACP,SACF,CAEA,2BACE,iBAAkB,CAClB,qBAAsB,CACtB,+BACF,CAEA,sCACE,kBACF,CAEA,mBACE,cACF,CAEA,6DACE,kBACF,CACA,8CACE,qBACF,CACA,kEACE,cACF,CACA,kEACE,eACF,CACA,8CACE,gCACF,CACA,kEACE,iBACF,CACA,kEACE,kBACF,CACA,8CACE,+BACF,CACA,kEACE,iBACF,CACA,kEACE,kBACF,CACA,8CACE,gCACF,CACA,kEACE,iBACF,CACA,kEACE,kBACF,CACA,8CACE,+BACF,CACA,kEACE,iBACF,CACA,kEACE,kBACF,CACA,8CACE,gCACF,CACA,kEACE,iBACF,CACA,kEACE,kBACF,CACA,8CACE,+BACF,CACA,kEACE,iBACF,CACA,kEACE,kBACF,CACA,8CACE,gCACF,CACA,kEACE,kBACF,CACA,kEACE,mBACF,CACA,8CACE,+BACF,CACA,kEACE,kBACF,CACA,kEACE,mBACF,CACA,8CACE,gCACF,CACA,kEACE,kBACF,CACA,kEACE,mBACF,CAGA,4BACE,+BACF,CACA,gDACE,kBACF,CACA,gDACE,mBACF,CAEA,kCACE,oBACF,CAGA,oCAEE,oBAAqB,CACrB,UACF,CACA,6CACE,eACF,CACA,8CACE,eACF,CACA,uDACE,eACF,CAEA,oEACE,iBACF,CACA,oEACE,kBACF,CAEA,aACE,iBAAkB,CAClB,KAAM,CACN,MAAO,CACP,UAAW,CACX,WAAY,CACZ,UACF,CCjLA,cAGE,eAGF,CAEA,uCAPE,iBAAkB,CAIlB,qBAWF,CARA,yBAEE,MAAO,CACP,KAAM,CACN,UAAW,CACX,aAAc,CAMd,+BAHF,CAMA,mCACE,cACF,CAEA,uBACE,yBACF,CAEA,oCACE,kBACF,CAEA,oCACE,oBAAqB,CACrB,WACF,CAEA,+CACE,SACF,CC1CA,eACE,iBAAkB,CAClB,eACF,CAEA,8BACE,KAAM,CACN,MAAO,CACP,UACF,CAEA,8BACE,iBAAkB,CAClB,KAAM,CACN,MAAO,CACP,UAAW,CACX,WACF,CAEA,yBACE,iBAAkB,CAClB,aAAc,CACd,WAAY,CACZ,eAAgB,CAChB,qBAAsB,CAEtB,kBACF,CAEA,qCACE,iBAAkB,CAClB,cAAe,CACf,eAAgB,CAChB,aAAc,CACd,cAAe,CACf,iBACF,CAEA,sCACE,iBAAkB,CAClB,qBACF,CAEA,0CACE,iBAAkB,CAClB,sBACF,CAEA,mCACE,oBACF,CAEA,mCACE,oBACF,CCrDA,UACE,iBAAkB,CAClB,aAAc,CACd,oBAAqB,CACrB,gBAAiB,CACjB,wBAAyB,CACzB,oBAAqB,CACrB,SAEF,CAEA,uBACE,oBAAqB,CACrB,wBAAyB,CAGzB,SACF,CAEA,2BACE,WACF,CAEA,iCACE,wBACF,CAEA,kBACE,iBAAkB,CAClB,kBAAmB,CACnB,iBACF,CAEA,oBACE,eACF,CAEA,kBACE,iBAAkB,CAClB,SAAU,CACV,gBAAiB,CACjB,kBAAmB,CACnB,iBACF,CAEA,oBACE,kBAAmB,CACnB,iBAAkB,CAClB,qBACF,CAEA,yBACE,WAAY,CACZ,qCAA0C,CAC1C,qBAAsB,CACtB,SAAU,CACV,QACF,CAEA,6BACE,iBAAkB,CAClB,UAAW,CACX,WAAY,CACZ,SAAU,CACV,QAAS,CACT,eACF,CAEA,wBACE,kBACF,CAEA,sCACE,iBAAkB,CAClB,oBACF,CAEA,2CACE,iBAAkB,CAClB,oBACF,CAEA,mBACE,SAAU,CACV,iBAAkB,CAClB,OAAQ,CACR,qBAAsB,CACtB,uBACF,CAEA,4BACE,kBAAmB,CACnB,qBAAsB,CACtB,WACF,CAEA,oCACE,iBAAkB,CAClB,kBAAmB,CACnB,UAAY,CACZ,WAAY,CACZ,iBAAkB,CAClB,kBAAmB,CACnB,WAAY,CACZ,iBAAkB,CAClB,cAAgB,CAChB,iBAAmB,CACnB,mBAAqB,CACrB,sBACF,CAEA,gDACE,iBAAkB,CAClB,KAAQ,CACR,UAAW,CACX,WAAY,CACZ,qBAAsB,CACtB,aAAgB,CAChB,cAAe,CAEf,wCAA0C,CAC1C,qCAAuC,CACvC,oCAAsC,CACtC,mCAAqC,CACrC,gCACF,CAEA,sBACE,WACF,CAEA,0BACE,UACF,CAEA,4DACE,eAAgB,CAChB,SAAU,CACV,4BAA8B,CAC9B,cAAe,CACf,eAAiB,CAEjB,mCAAqC,CACrC,gCAAkC,CAClC,+BAAiC,CACjC,8BAAgC,CAChC,2BACF,CAEA,4DACE,cACF,CAEA,wEACE,UACF,CAEA,2BACE,iBAAkB,CAClB,UAAW,CACX,WAAY,CACZ,KAAM,CACN,MAAS,CACT,WACF,CAEA,mCAOE,SAAU,CAEV,eACF,CAEA,uEAXE,iBAAkB,CAClB,UAAW,CACX,aAAc,CACd,aAAc,CACd,WAAY,CACZ,KAgBF,CAVA,oCAOE,UAAW,CAEX,eACF,CAEA,iGAEE,WACF,CAEA,sBACE,qBAAsB,CACtB,iBAAkB,CAClB,kBAAmB,CACnB,iBACF,CAEA,2BACE,SAAU,CACV,iBAAkB,CAClB,OAAQ,CACR,qBAAsB,CACtB,uBACF,CAEA,0BACE,iBAAkB,CAClB,SAAU,CACV,gBAAiB,CACjB,kBAAmB,CACnB,iBACF","file":"vis-timeline-graph2d.min.css","sourcesContent":["/* override some bootstrap styles screwing up the timelines css */\n\n.vis [class*=\"span\"] {\n  min-height: 0;\n  width: auto;\n}\n",".vis .overlay {\n  position: absolute;\n  top: 0;\n  left: 0;\n  width: 100%;\n  height: 100%;\n\n  /* Must be displayed above for example selected Timeline items */\n  z-index: 10;\n}\n\n.vis-active {\n  box-shadow: 0 0 10px #86d5f8;\n}\n","div.vis-configuration {\n    position:relative;\n    display:block;\n    float:left;\n    font-size:12px;\n}\n\ndiv.vis-configuration-wrapper {\n    display:block;\n    width:700px;\n}\n\ndiv.vis-configuration-wrapper::after {\n  clear: both;\n  content: \"\";\n  display: block;\n}\n\ndiv.vis-configuration.vis-config-option-container{\n    display:block;\n    width:495px;\n    background-color: #ffffff;\n    border:2px solid #f7f8fa;\n    border-radius:4px;\n    margin-top:20px;\n    left:10px;\n    padding-left:5px;\n}\n\ndiv.vis-configuration.vis-config-button{\n    display:block;\n    width:495px;\n    height:25px;\n    vertical-align: middle;\n    line-height:25px;\n    background-color: #f7f8fa;\n    border:2px solid #ceced0;\n    border-radius:4px;\n    margin-top:20px;\n    left:10px;\n    padding-left:5px;\n    cursor: pointer;\n    margin-bottom:30px;\n}\n\ndiv.vis-configuration.vis-config-button.hover{\n    background-color: #4588e6;\n    border:2px solid #214373;\n    color:#ffffff;\n}\n\ndiv.vis-configuration.vis-config-item{\n    display:block;\n    float:left;\n    width:495px;\n    height:25px;\n    vertical-align: middle;\n    line-height:25px;\n}\n\n\ndiv.vis-configuration.vis-config-item.vis-config-s2{\n    left:10px;\n    background-color: #f7f8fa;\n    padding-left:5px;\n    border-radius:3px;\n}\ndiv.vis-configuration.vis-config-item.vis-config-s3{\n    left:20px;\n    background-color: #e4e9f0;\n    padding-left:5px;\n    border-radius:3px;\n}\ndiv.vis-configuration.vis-config-item.vis-config-s4{\n    left:30px;\n    background-color: #cfd8e6;\n    padding-left:5px;\n    border-radius:3px;\n}\n\ndiv.vis-configuration.vis-config-header{\n    font-size:18px;\n    font-weight: bold;\n}\n\ndiv.vis-configuration.vis-config-label{\n    width:120px;\n    height:25px;\n    line-height: 25px;\n}\n\ndiv.vis-configuration.vis-config-label.vis-config-s3{\n    width:110px;\n}\ndiv.vis-configuration.vis-config-label.vis-config-s4{\n    width:100px;\n}\n\ndiv.vis-configuration.vis-config-colorBlock{\n    top:1px;\n    width:30px;\n    height:19px;\n    border:1px solid #444444;\n    border-radius:2px;\n    padding:0px;\n    margin:0px;\n    cursor:pointer;\n}\n\ninput.vis-configuration.vis-config-checkbox {\n    left:-5px;\n}\n\n\ninput.vis-configuration.vis-config-rangeinput{\n    position:relative;\n    top:-5px;\n    width:60px;\n    /*height:13px;*/\n    padding:1px;\n    margin:0;\n    pointer-events:none;\n}\n\ninput.vis-configuration.vis-config-range{\n    /*removes default webkit styles*/\n    -webkit-appearance: none;\n\n    /*fix for FF unable to apply focus style bug */\n    border: 0px solid white;\n    background-color:rgba(0,0,0,0);\n\n    /*required for proper track sizing in FF*/\n    width: 300px;\n    height:20px;\n}\ninput.vis-configuration.vis-config-range::-webkit-slider-runnable-track {\n    width: 300px;\n    height: 5px;\n    background: #dedede; /* Old browsers */\n    background: -moz-linear-gradient(top,  #dedede 0%, #c8c8c8 99%); /* FF3.6+ */\n    background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#dedede), color-stop(99%,#c8c8c8)); /* Chrome,Safari4+ */\n    background: -webkit-linear-gradient(top,  #dedede 0%,#c8c8c8 99%); /* Chrome10+,Safari5.1+ */\n    background: -o-linear-gradient(top, #dedede 0%, #c8c8c8 99%); /* Opera 11.10+ */\n    background: -ms-linear-gradient(top,  #dedede 0%,#c8c8c8 99%); /* IE10+ */\n    background: linear-gradient(to bottom,  #dedede 0%,#c8c8c8 99%); /* W3C */\n    filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#dedede', endColorstr='#c8c8c8',GradientType=0 ); /* IE6-9 */\n\n    border: 1px solid #999999;\n    box-shadow: #aaaaaa 0px 0px 3px 0px;\n    border-radius: 3px;\n}\ninput.vis-configuration.vis-config-range::-webkit-slider-thumb {\n    -webkit-appearance: none;\n    border: 1px solid #14334b;\n    height: 17px;\n    width: 17px;\n    border-radius: 50%;\n    background: #3876c2; /* Old browsers */\n    background: -moz-linear-gradient(top,  #3876c2 0%, #385380 100%); /* FF3.6+ */\n    background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#3876c2), color-stop(100%,#385380)); /* Chrome,Safari4+ */\n    background: -webkit-linear-gradient(top,  #3876c2 0%,#385380 100%); /* Chrome10+,Safari5.1+ */\n    background: -o-linear-gradient(top,  #3876c2 0%,#385380 100%); /* Opera 11.10+ */\n    background: -ms-linear-gradient(top,  #3876c2 0%,#385380 100%); /* IE10+ */\n    background: linear-gradient(to bottom,  #3876c2 0%,#385380 100%); /* W3C */\n    filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#3876c2', endColorstr='#385380',GradientType=0 ); /* IE6-9 */\n    box-shadow: #111927 0px 0px 1px 0px;\n    margin-top: -7px;\n}\ninput.vis-configuration.vis-config-range:focus {\n    outline: none;\n}\ninput.vis-configuration.vis-config-range:focus::-webkit-slider-runnable-track {\n    background: #9d9d9d; /* Old browsers */\n    background: -moz-linear-gradient(top, #9d9d9d 0%, #c8c8c8 99%); /* FF3.6+ */\n    background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#9d9d9d), color-stop(99%,#c8c8c8)); /* Chrome,Safari4+ */\n    background: -webkit-linear-gradient(top,  #9d9d9d 0%,#c8c8c8 99%); /* Chrome10+,Safari5.1+ */\n    background: -o-linear-gradient(top,  #9d9d9d 0%,#c8c8c8 99%); /* Opera 11.10+ */\n    background: -ms-linear-gradient(top,  #9d9d9d 0%,#c8c8c8 99%); /* IE10+ */\n    background: linear-gradient(to bottom,  #9d9d9d 0%,#c8c8c8 99%); /* W3C */\n    filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#9d9d9d', endColorstr='#c8c8c8',GradientType=0 ); /* IE6-9 */\n}\n\ninput.vis-configuration.vis-config-range::-moz-range-track {\n    width: 300px;\n    height: 10px;\n    background: #dedede; /* Old browsers */\n    background: -moz-linear-gradient(top,  #dedede 0%, #c8c8c8 99%); /* FF3.6+ */\n    background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#dedede), color-stop(99%,#c8c8c8)); /* Chrome,Safari4+ */\n    background: -webkit-linear-gradient(top,  #dedede 0%,#c8c8c8 99%); /* Chrome10+,Safari5.1+ */\n    background: -o-linear-gradient(top, #dedede 0%, #c8c8c8 99%); /* Opera 11.10+ */\n    background: -ms-linear-gradient(top,  #dedede 0%,#c8c8c8 99%); /* IE10+ */\n    background: linear-gradient(to bottom,  #dedede 0%,#c8c8c8 99%); /* W3C */\n    filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#dedede', endColorstr='#c8c8c8',GradientType=0 ); /* IE6-9 */\n\n    border: 1px solid #999999;\n    box-shadow: #aaaaaa 0px 0px 3px 0px;\n    border-radius: 3px;\n}\ninput.vis-configuration.vis-config-range::-moz-range-thumb {\n    border: none;\n    height: 16px;\n    width: 16px;\n\n    border-radius: 50%;\n    background:  #385380;\n}\n\n/*hide the outline behind the border*/\ninput.vis-configuration.vis-config-range:-moz-focusring{\n    outline: 1px solid white;\n    outline-offset: -1px;\n}\n\ninput.vis-configuration.vis-config-range::-ms-track {\n    width: 300px;\n    height: 5px;\n\n    /*remove bg colour from the track, we'll use ms-fill-lower and ms-fill-upper instead */\n    background: transparent;\n\n    /*leave room for the larger thumb to overflow with a transparent border */\n    border-color: transparent;\n    border-width: 6px 0;\n\n    /*remove default tick marks*/\n    color: transparent;\n}\ninput.vis-configuration.vis-config-range::-ms-fill-lower {\n    background: #777;\n    border-radius: 10px;\n}\ninput.vis-configuration.vis-config-range::-ms-fill-upper {\n    background: #ddd;\n    border-radius: 10px;\n}\ninput.vis-configuration.vis-config-range::-ms-thumb {\n    border: none;\n    height: 16px;\n    width: 16px;\n    border-radius: 50%;\n    background:  #385380;\n}\ninput.vis-configuration.vis-config-range:focus::-ms-fill-lower {\n    background: #888;\n}\ninput.vis-configuration.vis-config-range:focus::-ms-fill-upper {\n    background: #ccc;\n}\n\n.vis-configuration-popup {\n    position: absolute;\n    background: rgba(57, 76, 89, 0.85);\n    border: 2px solid #f2faff;\n    line-height:30px;\n    height:30px;\n    width:150px;\n    text-align:center;\n    color: #ffffff;\n    font-size:14px;\n    border-radius:4px;\n    -webkit-transition: opacity 0.3s ease-in-out;\n    -moz-transition: opacity 0.3s ease-in-out;\n    transition: opacity 0.3s ease-in-out;\n}\n.vis-configuration-popup:after, .vis-configuration-popup:before {\n    left: 100%;\n    top: 50%;\n    border: solid transparent;\n    content: \" \";\n    height: 0;\n    width: 0;\n    position: absolute;\n    pointer-events: none;\n}\n\n.vis-configuration-popup:after {\n    border-color: rgba(136, 183, 213, 0);\n    border-left-color: rgba(57, 76, 89, 0.85);\n    border-width: 8px;\n    margin-top: -8px;\n}\n.vis-configuration-popup:before {\n    border-color: rgba(194, 225, 245, 0);\n    border-left-color: #f2faff;\n    border-width: 12px;\n    margin-top: -12px;\n}","div.vis-tooltip {\n  position: absolute;\n  visibility: hidden;\n  padding: 5px;\n  white-space: nowrap;\n\n  font-family: verdana;\n  font-size:14px;\n  color:#000000;\n  background-color: #f5f4ed;\n\n  -moz-border-radius: 3px;\n  -webkit-border-radius: 3px;\n  border-radius: 3px;\n  border: 1px solid #808074;\n\n  box-shadow: 3px 3px 10px rgba(0, 0, 0, 0.2);\n  pointer-events: none;\n\n  z-index: 5;\n}\n",".vis-panel {\n  position: absolute;\n\n  padding: 0;\n  margin: 0;\n\n  box-sizing: border-box;\n}\n\n.vis-panel.vis-center,\n.vis-panel.vis-left,\n.vis-panel.vis-right,\n.vis-panel.vis-top,\n.vis-panel.vis-bottom {\n  border: 1px #bfbfbf;\n}\n\n.vis-panel.vis-center,\n.vis-panel.vis-left,\n.vis-panel.vis-right {\n  border-top-style: solid;\n  border-bottom-style: solid;\n  overflow: hidden;\n}\n\n.vis-left.vis-panel.vis-vertical-scroll, .vis-right.vis-panel.vis-vertical-scroll {\n  height: 100%;\n  overflow-x: hidden;\n  overflow-y: scroll;\n} \n\n.vis-left.vis-panel.vis-vertical-scroll {\n  direction: rtl;\n}\n\n.vis-left.vis-panel.vis-vertical-scroll .vis-content {\n  direction: ltr;\n}\n\n.vis-right.vis-panel.vis-vertical-scroll {\n  direction: ltr;\n}\n\n.vis-right.vis-panel.vis-vertical-scroll .vis-content {\n  direction: rtl;\n}\n\n.vis-panel.vis-center,\n.vis-panel.vis-top,\n.vis-panel.vis-bottom {\n  border-left-style: solid;\n  border-right-style: solid;\n}\n\n.vis-background {\n  overflow: hidden;\n}\n\n.vis-panel > .vis-content {\n  position: relative;\n}\n\n.vis-panel .vis-shadow {\n  position: absolute;\n  width: 100%;\n  height: 1px;\n  box-shadow: 0 0 10px rgba(0,0,0,0.8);\n  /* TODO: find a nice way to ensure vis-shadows are drawn on top of items\n  z-index: 1;\n  */\n}\n\n.vis-panel .vis-shadow.vis-top {\n  top: -1px;\n  left: 0;\n}\n\n.vis-panel .vis-shadow.vis-bottom {\n  bottom: -1px;\n  left: 0;\n}",".vis-current-time {\n  background-color: #FF7F6E;\n  width: 2px;\n  z-index: 1;\n  pointer-events: none;\n}\n\n.vis-rolling-mode-btn {\n  height: 40px;\n  width: 40px;\n  position: absolute;\n  top: 7px;\n  right: 20px;\n  border-radius: 50%;\n  font-size: 28px;\n  cursor: pointer;\n  opacity: 0.8;\n  color: white;\n  font-weight: bold;\n  text-align: center;\n  background: #3876c2;\n}\n.vis-rolling-mode-btn:before {\n  content: \"\\26F6\";\n}\n\n.vis-rolling-mode-btn:hover {\n  opacity: 1;\n}",".vis-custom-time {\n  background-color: #6E94FF;\n  width: 2px;\n  cursor: move;\n  z-index: 1;\n}\n\n.vis-custom-time > .vis-custom-time-marker {\n  background-color: inherit;\n  color: white;\n  font-size: 12px;\n  white-space: nowrap;\n  padding: 3px 5px;\n  top: 0px;\n  cursor: initial;\n  z-index: inherit;\n}","\n.vis-timeline {\n  position: relative;\n  border: 1px solid #bfbfbf;\n  overflow: hidden;\n  padding: 0;\n  margin: 0;\n  box-sizing: border-box;\n}\n\n.vis-loading-screen {\n  width: 100%;\n  height: 100%;\n  position: absolute;\n  top: 0;\n  left: 0;\n}",".vis-graph-group0 {\n    fill:#4f81bd;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #4f81bd;\n}\n\n.vis-graph-group1 {\n    fill:#f79646;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #f79646;\n}\n\n.vis-graph-group2 {\n    fill: #8c51cf;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #8c51cf;\n}\n\n.vis-graph-group3 {\n    fill: #75c841;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #75c841;\n}\n\n.vis-graph-group4 {\n    fill: #ff0100;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #ff0100;\n}\n\n.vis-graph-group5 {\n    fill: #37d8e6;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #37d8e6;\n}\n\n.vis-graph-group6 {\n    fill: #042662;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #042662;\n}\n\n.vis-graph-group7 {\n    fill:#00ff26;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #00ff26;\n}\n\n.vis-graph-group8 {\n    fill:#ff00ff;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #ff00ff;\n}\n\n.vis-graph-group9 {\n    fill: #8f3938;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #8f3938;\n}\n\n.vis-timeline .vis-fill {\n    fill-opacity:0.1;\n    stroke: none;\n}\n\n\n.vis-timeline .vis-bar {\n    fill-opacity:0.5;\n    stroke-width:1px;\n}\n\n.vis-timeline .vis-point {\n    stroke-width:2px;\n    fill-opacity:1.0;\n}\n\n\n.vis-timeline .vis-legend-background {\n    stroke-width:1px;\n    fill-opacity:0.9;\n    fill: #ffffff;\n    stroke: #c2c2c2;\n}\n\n\n.vis-timeline .vis-outline {\n    stroke-width:1px;\n    fill-opacity:1;\n    fill: #ffffff;\n    stroke: #e5e5e5;\n}\n\n.vis-timeline .vis-icon-fill {\n    fill-opacity:0.3;\n    stroke: none;\n}\n","\n.vis-panel.vis-background.vis-horizontal .vis-grid.vis-horizontal {\n  position: absolute;\n  width: 100%;\n  height: 0;\n  border-bottom: 1px solid;\n}\n\n.vis-panel.vis-background.vis-horizontal .vis-grid.vis-minor {\n  border-color: #e5e5e5;\n}\n\n.vis-panel.vis-background.vis-horizontal .vis-grid.vis-major {\n  border-color: #bfbfbf;\n}\n\n\n.vis-data-axis .vis-y-axis.vis-major {\n  width: 100%;\n  position: absolute;\n  color: #4d4d4d;\n  white-space: nowrap;\n}\n\n.vis-data-axis .vis-y-axis.vis-major.vis-measure {\n  padding: 0;\n  margin: 0;\n  border: 0;\n  visibility: hidden;\n  width: auto;\n}\n\n\n.vis-data-axis .vis-y-axis.vis-minor {\n  position: absolute;\n  width: 100%;\n  color: #bebebe;\n  white-space: nowrap;\n}\n\n.vis-data-axis .vis-y-axis.vis-minor.vis-measure {\n  padding: 0;\n  margin: 0;\n  border: 0;\n  visibility: hidden;\n  width: auto;\n}\n\n.vis-data-axis .vis-y-axis.vis-title {\n  position: absolute;\n  color: #4d4d4d;\n  white-space: nowrap;\n  bottom: 20px;\n  text-align: center;\n}\n\n.vis-data-axis .vis-y-axis.vis-title.vis-measure {\n  padding: 0;\n  margin: 0;\n  visibility: hidden;\n  width: auto;\n}\n\n.vis-data-axis .vis-y-axis.vis-title.vis-left {\n  bottom: 0;\n  -webkit-transform-origin: left top;\n  -moz-transform-origin: left top;\n  -ms-transform-origin: left top;\n  -o-transform-origin: left top;\n  transform-origin: left bottom;\n  -webkit-transform: rotate(-90deg);\n  -moz-transform: rotate(-90deg);\n  -ms-transform: rotate(-90deg);\n  -o-transform: rotate(-90deg);\n  transform: rotate(-90deg);\n}\n\n.vis-data-axis .vis-y-axis.vis-title.vis-right {\n  bottom: 0;\n  -webkit-transform-origin: right bottom;\n  -moz-transform-origin: right bottom;\n  -ms-transform-origin: right bottom;\n  -o-transform-origin: right bottom;\n  transform-origin: right bottom;\n  -webkit-transform: rotate(90deg);\n  -moz-transform: rotate(90deg);\n  -ms-transform: rotate(90deg);\n  -o-transform: rotate(90deg);\n  transform: rotate(90deg);\n}\n\n.vis-legend {\n  background-color: rgba(247, 252, 255, 0.65);\n  padding: 5px;\n  border: 1px solid #b3b3b3;\n  box-shadow: 2px 2px 10px rgba(154, 154, 154, 0.55);\n}\n\n.vis-legend-text {\n  /*font-size: 10px;*/\n  white-space: nowrap;\n  display: inline-block\n}","\n.vis-itemset {\n  position: relative;\n  padding: 0;\n  margin: 0;\n\n  box-sizing: border-box;\n}\n\n.vis-itemset .vis-background,\n.vis-itemset .vis-foreground {\n  position: absolute;\n  width: 100%;\n  height: 100%;\n  overflow: visible;\n}\n\n.vis-axis {\n  position: absolute;\n  width: 100%;\n  height: 0;\n  left: 0;\n  z-index: 1;\n}\n\n.vis-foreground .vis-group {\n  position: relative;\n  box-sizing: border-box;\n  border-bottom: 1px solid #bfbfbf;\n}\n\n.vis-foreground .vis-group:last-child {\n  border-bottom: none;\n}\n\n.vis-nesting-group {\n  cursor: pointer;\n}\n\n.vis-label.vis-nested-group.vis-group-level-unknown-but-gte1 {\n  background: #f5f5f5;\n}\n.vis-label.vis-nested-group.vis-group-level-0 {\n  background-color: #ffffff;\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-0 .vis-inner {\n  padding-left: 0;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-0 .vis-inner {\n  padding-right: 0;\n}\n.vis-label.vis-nested-group.vis-group-level-1 {\n  background-color: rgba(0, 0, 0, 0.05);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-1 .vis-inner {\n  padding-left: 15px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-1 .vis-inner {\n  padding-right: 15px;\n}\n.vis-label.vis-nested-group.vis-group-level-2 {\n  background-color: rgba(0, 0, 0, 0.1);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-2 .vis-inner {\n  padding-left: 30px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-2 .vis-inner {\n  padding-right: 30px;\n}\n.vis-label.vis-nested-group.vis-group-level-3 {\n  background-color: rgba(0, 0, 0, 0.15);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-3 .vis-inner {\n  padding-left: 45px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-3 .vis-inner {\n  padding-right: 45px;\n}\n.vis-label.vis-nested-group.vis-group-level-4 {\n  background-color: rgba(0, 0, 0, 0.2);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-4 .vis-inner {\n  padding-left: 60px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-4 .vis-inner {\n  padding-right: 60px;\n}\n.vis-label.vis-nested-group.vis-group-level-5 {\n  background-color: rgba(0, 0, 0, 0.25);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-5 .vis-inner {\n  padding-left: 75px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-5 .vis-inner {\n  padding-right: 75px;\n}\n.vis-label.vis-nested-group.vis-group-level-6 {\n  background-color: rgba(0, 0, 0, 0.3);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-6 .vis-inner {\n  padding-left: 90px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-6 .vis-inner {\n  padding-right: 90px;\n}\n.vis-label.vis-nested-group.vis-group-level-7 {\n  background-color: rgba(0, 0, 0, 0.35);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-7 .vis-inner {\n  padding-left: 105px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-7 .vis-inner {\n  padding-right: 105px;\n}\n.vis-label.vis-nested-group.vis-group-level-8 {\n  background-color: rgba(0, 0, 0, 0.4);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-8 .vis-inner {\n  padding-left: 120px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-8 .vis-inner {\n  padding-right: 120px;\n}\n.vis-label.vis-nested-group.vis-group-level-9 {\n  background-color: rgba(0, 0, 0, 0.45);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-9 .vis-inner {\n  padding-left: 135px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-9 .vis-inner {\n  padding-right: 135px;\n}\n/* default takes over beginning with level-10 (thats why we add .vis-nested-group\n  to the selectors above, to have higher specifity than these rules for the defaults) */\n.vis-label.vis-nested-group {\n  background-color: rgba(0, 0, 0, 0.5);\n}\n.vis-ltr .vis-label.vis-nested-group .vis-inner {\n  padding-left: 150px;\n}\n.vis-rtl .vis-label.vis-nested-group .vis-inner {\n  padding-right: 150px;\n}\n\n.vis-group-level-unknown-but-gte1 {\n  border: 1px solid red;\n}\n\n/* expanded/collapsed indicators */\n.vis-label.vis-nesting-group:before,\n.vis-label.vis-nesting-group:before {\n  display: inline-block;\n  width: 15px;\n}\n.vis-label.vis-nesting-group.expanded:before {\n  content: \"\\25BC\";\n}\n.vis-label.vis-nesting-group.collapsed:before {\n  content: \"\\25B6\";\n}\n.vis-rtl .vis-label.vis-nesting-group.collapsed:before {\n  content: \"\\25C0\";\n}\n/* compensate missing expanded/collapsed indicator, but only at levels > 0 */\n.vis-ltr .vis-label:not(.vis-nesting-group):not(.vis-group-level-0) {\n  padding-left: 15px;\n}\n.vis-rtl .vis-label:not(.vis-nesting-group):not(.vis-group-level-0) {\n  padding-right: 15px;\n}\n\n.vis-overlay {\n  position: absolute;\n  top: 0;\n  left: 0;\n  width: 100%;\n  height: 100%;\n  z-index: 10;\n}","\n.vis-labelset {\n  position: relative;\n\n  overflow: hidden;\n\n  box-sizing: border-box;\n}\n\n.vis-labelset .vis-label {\n  position: relative;\n  left: 0;\n  top: 0;\n  width: 100%;\n  color: #4d4d4d;\n\n  box-sizing: border-box;\n}\n\n.vis-labelset .vis-label {\n  border-bottom: 1px solid #bfbfbf;\n}\n\n.vis-labelset .vis-label.draggable {\n  cursor: pointer;\n}\n\n.vis-group-is-dragging {\n  background: rgba(0, 0, 0, .1);\n}\n\n.vis-labelset .vis-label:last-child {\n  border-bottom: none;\n}\n\n.vis-labelset .vis-label .vis-inner {\n  display: inline-block;\n  padding: 5px;\n}\n\n.vis-labelset .vis-label .vis-inner.vis-hidden {\n  padding: 0;\n}\n",".vis-time-axis {\n  position: relative;\n  overflow: hidden;\n}\n\n.vis-time-axis.vis-foreground {\n  top: 0;\n  left: 0;\n  width: 100%;\n}\n\n.vis-time-axis.vis-background {\n  position: absolute;\n  top: 0;\n  left: 0;\n  width: 100%;\n  height: 100%;\n}\n\n.vis-time-axis .vis-text {\n  position: absolute;\n  color: #4d4d4d;\n  padding: 3px;\n  overflow: hidden;\n  box-sizing: border-box;\n\n  white-space: nowrap;\n}\n\n.vis-time-axis .vis-text.vis-measure {\n  position: absolute;\n  padding-left: 0;\n  padding-right: 0;\n  margin-left: 0;\n  margin-right: 0;\n  visibility: hidden;\n}\n\n.vis-time-axis .vis-grid.vis-vertical {\n  position: absolute;\n  border-left: 1px solid;\n}\n\n.vis-time-axis .vis-grid.vis-vertical-rtl {\n  position: absolute;\n  border-right: 1px solid;\n}\n\n.vis-time-axis .vis-grid.vis-minor {\n  border-color: #e5e5e5;\n}\n\n.vis-time-axis .vis-grid.vis-major {\n  border-color: #bfbfbf;\n}\n","\n.vis-item {\n  position: absolute;\n  color: #1A1A1A;\n  border-color: #97B0F8;\n  border-width: 1px;\n  background-color: #D5DDF6;\n  display: inline-block;\n  z-index: 1;\n  /*overflow: hidden;*/\n}\n\n.vis-item.vis-selected {\n  border-color: #FFC200;\n  background-color: #FFF785;\n\n  /* z-index must be higher than the z-index of custom time bar and current time bar */\n  z-index: 2;\n}\n\n.vis-editable.vis-selected {\n  cursor: move;\n}\n\n.vis-item.vis-point.vis-selected {\n  background-color: #FFF785;\n}\n\n.vis-item.vis-box {\n  text-align: center;\n  border-style: solid;\n  border-radius: 2px;\n}\n\n.vis-item.vis-point {\n  background: none;\n}\n\n.vis-item.vis-dot {\n  position: absolute;\n  padding: 0;\n  border-width: 4px;\n  border-style: solid;\n  border-radius: 4px;\n}\n\n.vis-item.vis-range {\n  border-style: solid;\n  border-radius: 2px;\n  box-sizing: border-box;\n}\n\n.vis-item.vis-background {\n  border: none;\n  background-color: rgba(213, 221, 246, 0.4);\n  box-sizing: border-box;\n  padding: 0;\n  margin: 0;\n}\n\n.vis-item .vis-item-overflow {\n  position: relative;\n  width: 100%;\n  height: 100%;\n  padding: 0;\n  margin: 0;\n  overflow: hidden;\n}\n\n.vis-item-visible-frame {\n  white-space: nowrap;\n}\n\n.vis-item.vis-range .vis-item-content {\n  position: relative;\n  display: inline-block;\n}\n\n.vis-item.vis-background .vis-item-content {\n  position: absolute;\n  display: inline-block;\n}\n\n.vis-item.vis-line {\n  padding: 0;\n  position: absolute;\n  width: 0;\n  border-left-width: 1px;\n  border-left-style: solid;\n}\n\n.vis-item .vis-item-content {\n  white-space: nowrap;\n  box-sizing: border-box;\n  padding: 5px;\n}\n\n.vis-item .vis-onUpdateTime-tooltip {\n  position: absolute;\n  background: #4f81bd;\n  color: white;\n  width: 200px;\n  text-align: center;\n  white-space: nowrap;\n  padding: 5px;\n  border-radius: 1px;\n  transition: 0.4s;\n  -o-transition: 0.4s;\n  -moz-transition: 0.4s;\n  -webkit-transition: 0.4s;\n}\n\n.vis-item .vis-delete, .vis-item .vis-delete-rtl {\n  position: absolute;\n  top: 0px;\n  width: 24px;\n  height: 24px;\n  box-sizing: border-box;\n  padding: 0px 5px;\n  cursor: pointer;\n\n  -webkit-transition: background 0.2s linear;\n  -moz-transition: background 0.2s linear;\n  -ms-transition: background 0.2s linear;\n  -o-transition: background 0.2s linear;\n  transition: background 0.2s linear;\n}\n\n.vis-item .vis-delete {\n  right: -24px;\n}\n\n.vis-item .vis-delete-rtl {\n  left: -24px;\n}\n\n.vis-item .vis-delete:after, .vis-item .vis-delete-rtl:after {\n  content: \"\\00D7\"; /* MULTIPLICATION SIGN */\n  color: red;\n  font-family: arial, sans-serif;\n  font-size: 22px;\n  font-weight: bold;\n\n  -webkit-transition: color 0.2s linear;\n  -moz-transition: color 0.2s linear;\n  -ms-transition: color 0.2s linear;\n  -o-transition: color 0.2s linear;\n  transition: color 0.2s linear;\n}\n\n.vis-item .vis-delete:hover, .vis-item .vis-delete-rtl:hover {\n  background: red;\n}\n\n.vis-item .vis-delete:hover:after, .vis-item .vis-delete-rtl:hover:after {\n  color: white;\n}\n\n.vis-item .vis-drag-center {\n  position: absolute;\n  width: 100%;\n  height: 100%;\n  top: 0;\n  left: 0px;\n  cursor: move;\n}\n\n.vis-item.vis-range .vis-drag-left {\n  position: absolute;\n  width: 24px;\n  max-width: 20%;\n  min-width: 2px;\n  height: 100%;\n  top: 0;\n  left: -4px;\n\n  cursor: w-resize;\n}\n\n.vis-item.vis-range .vis-drag-right {\n  position: absolute;\n  width: 24px;\n  max-width: 20%;\n  min-width: 2px;\n  height: 100%;\n  top: 0;\n  right: -4px;\n\n  cursor: e-resize;\n}\n\n.vis-range.vis-item.vis-readonly .vis-drag-left,\n.vis-range.vis-item.vis-readonly .vis-drag-right {\n  cursor: auto;\n}\n\n.vis-item.vis-cluster {\n  vertical-align: center;\n  text-align: center;\n  border-style: solid;\n  border-radius: 2px;\n}\n\n.vis-item.vis-cluster-line {\n  padding: 0;\n  position: absolute;\n  width: 0;\n  border-left-width: 1px;\n  border-left-style: solid;\n}\n\n.vis-item.vis-cluster-dot {\n  position: absolute;\n  padding: 0;\n  border-width: 4px;\n  border-style: solid;\n  border-radius: 4px;\n}"]}
\ No newline at end of file
diff --git a/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js b/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js
index df730409ee491..5cfa0da3d4f38 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js
@@ -1,60 +1,30 @@
 /**
- * vis-timeline and vis-graph2d
- * https://visjs.github.io/vis-timeline/
+ * vis.js
+ * https://github.com/almende/vis
  *
- * Create a fully customizable, interactive timeline with items and ranges.
+ * A dynamic, browser-based visualization library.
  *
- * @version 7.3.4
- * @date    2020-03-18T17:03:58.105Z
- *
- * @copyright (c) 2011-2017 Almende B.V, http://almende.com
- * @copyright (c) 2017-2019 visjs contributors, https://github.com/visjs
+ * @version 4.21.0
+ * @date    2017-10-12
  *
  * @license
- * vis.js is dual licensed under both
+ * Copyright (C) 2011-2017 Almende B.V, http://almende.com
+ *
+ * Vis.js is dual licensed under both
  *
- *   1. The Apache 2.0 License
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * * The Apache 2.0 License
+ *   http://www.apache.org/licenses/LICENSE-2.0
  *
- *   and
+ * and
  *
- *   2. The MIT License
- *      http://opensource.org/licenses/MIT
+ * * The MIT License
+ *   http://opensource.org/licenses/MIT
  *
- * vis.js may be distributed under either license.
+ * Vis.js may be distributed under either license.
  */
-!function(t,e){"object"==typeof exports&&"undefined"!=typeof module?e(exports):"function"==typeof define&&define.amd?define(["exports"],e):e((t=t||self).vis=t.vis||{})}(this,(function(t){"use strict";var e="undefined"!=typeof globalThis?globalThis:"undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof self?self:{};function i(){throw new Error("Dynamic requires are not currently supported by @rollup/plugin-commonjs")}function n(t,e){return t(e={exports:{}},e.exports),e.exports}function o(t){return t&&t.default||t}var s=n((function(t,e){t.exports=function(){var e,n;function o(){return e.apply(null,arguments)}function s(t){return t instanceof Array||"[object Array]"===Object.prototype.toString.call(t)}function r(t){return null!=t&&"[object Object]"===Object.prototype.toString.call(t)}function a(t){return void 0===t}function l(t){return"number"==typeof t||"[object Number]"===Object.prototype.toString.call(t)}function h(t){return t instanceof Date||"[object Date]"===Object.prototype.toString.call(t)}function d(t,e){var i,n=[];for(i=0;i<t.length;++i)n.push(e(t[i],i));return n}function u(t,e){return Object.prototype.hasOwnProperty.call(t,e)}function c(t,e){for(var i in e)u(e,i)&&(t[i]=e[i]);return u(e,"toString")&&(t.toString=e.toString),u(e,"valueOf")&&(t.valueOf=e.valueOf),t}function p(t,e,i,n){return De(t,e,i,n,!0).utc()}function m(t){return null==t._pf&&(t._pf={empty:!1,unusedTokens:[],unusedInput:[],overflow:-2,charsLeftOver:0,nullInput:!1,invalidMonth:null,invalidFormat:!1,userInvalidated:!1,iso:!1,parsedDateParts:[],meridiem:null,rfc2822:!1,weekdayMismatch:!1}),t._pf}function f(t){if(null==t._isValid){var e=m(t),i=n.call(e.parsedDateParts,(function(t){return null!=t})),o=!isNaN(t._d.getTime())&&e.overflow<0&&!e.empty&&!e.invalidMonth&&!e.invalidWeekday&&!e.weekdayMismatch&&!e.nullInput&&!e.invalidFormat&&!e.userInvalidated&&(!e.meridiem||e.meridiem&&i);if(t._strict&&(o=o&&0===e.charsLeftOver&&0===e.unusedTokens.length&&void 0===e.bigHour),null!=Object.isFrozen&&Object.isFrozen(t))return o;t._isValid=o}return t._isValid}function g(t){var e=p(NaN);return null!=t?c(m(e),t):m(e).userInvalidated=!0,e}n=Array.prototype.some?Array.prototype.some:function(t){for(var e=Object(this),i=e.length>>>0,n=0;n<i;n++)if(n in e&&t.call(this,e[n],n,e))return!0;return!1};var v=o.momentProperties=[];function b(t,e){var i,n,o;if(a(e._isAMomentObject)||(t._isAMomentObject=e._isAMomentObject),a(e._i)||(t._i=e._i),a(e._f)||(t._f=e._f),a(e._l)||(t._l=e._l),a(e._strict)||(t._strict=e._strict),a(e._tzm)||(t._tzm=e._tzm),a(e._isUTC)||(t._isUTC=e._isUTC),a(e._offset)||(t._offset=e._offset),a(e._pf)||(t._pf=m(e)),a(e._locale)||(t._locale=e._locale),v.length>0)for(i=0;i<v.length;i++)a(o=e[n=v[i]])||(t[n]=o);return t}var y=!1;function x(t){b(this,t),this._d=new Date(null!=t._d?t._d.getTime():NaN),this.isValid()||(this._d=new Date(NaN)),!1===y&&(y=!0,o.updateOffset(this),y=!1)}function w(t){return t instanceof x||null!=t&&null!=t._isAMomentObject}function _(t){return t<0?Math.ceil(t)||0:Math.floor(t)}function k(t){var e=+t,i=0;return 0!==e&&isFinite(e)&&(i=_(e)),i}function D(t,e,i){var n,o=Math.min(t.length,e.length),s=Math.abs(t.length-e.length),r=0;for(n=0;n<o;n++)(i&&t[n]!==e[n]||!i&&k(t[n])!==k(e[n]))&&r++;return r+s}function I(t){!1===o.suppressDeprecationWarnings&&"undefined"!=typeof console&&console.warn&&console.warn("Deprecation warning: "+t)}function C(t,e){var i=!0;return c((function(){if(null!=o.deprecationHandler&&o.deprecationHandler(null,t),i){for(var n,s=[],r=0;r<arguments.length;r++){if(n="","object"==typeof arguments[r]){for(var a in n+="\n["+r+"] ",arguments[0])n+=a+": "+arguments[0][a]+", ";n=n.slice(0,-2)}else n=arguments[r];s.push(n)}I(t+"\nArguments: "+Array.prototype.slice.call(s).join("")+"\n"+(new Error).stack),i=!1}return e.apply(this,arguments)}),e)}var L,G={};function M(t,e){null!=o.deprecationHandler&&o.deprecationHandler(t,e),G[t]||(I(e),G[t]=!0)}function S(t){return t instanceof Function||"[object Function]"===Object.prototype.toString.call(t)}function Q(t,e){var i,n=c({},t);for(i in e)u(e,i)&&(r(t[i])&&r(e[i])?(n[i]={},c(n[i],t[i]),c(n[i],e[i])):null!=e[i]?n[i]=e[i]:delete n[i]);for(i in t)u(t,i)&&!u(e,i)&&r(t[i])&&(n[i]=c({},n[i]));return n}function N(t){null!=t&&this.set(t)}o.suppressDeprecationWarnings=!1,o.deprecationHandler=null,L=Object.keys?Object.keys:function(t){var e,i=[];for(e in t)u(t,e)&&i.push(e);return i};var F={};function Z(t,e){var i=t.toLowerCase();F[i]=F[i+"s"]=F[e]=t}function B(t){return"string"==typeof t?F[t]||F[t.toLowerCase()]:void 0}function O(t){var e,i,n={};for(i in t)u(t,i)&&(e=B(i))&&(n[e]=t[i]);return n}var W={};function A(t,e){W[t]=e}function X(t,e,i){var n=""+Math.abs(t),o=e-n.length;return(t>=0?i?"+":"":"-")+Math.pow(10,Math.max(0,o)).toString().substr(1)+n}var T=/(\[[^\[]*\])|(\\)?([Hh]mm(ss)?|Mo|MM?M?M?|Do|DDDo|DD?D?D?|ddd?d?|do?|w[o|w]?|W[o|W]?|Qo?|YYYYYY|YYYYY|YYYY|YY|gg(ggg?)?|GG(GGG?)?|e|E|a|A|hh?|HH?|kk?|mm?|ss?|S{1,9}|x|X|zz?|ZZ?|.)/g,Y=/(\[[^\[]*\])|(\\)?(LTS|LT|LL?L?L?|l{1,4})/g,E={},R={};function U(t,e,i,n){var o=n;"string"==typeof n&&(o=function(){return this[n]()}),t&&(R[t]=o),e&&(R[e[0]]=function(){return X(o.apply(this,arguments),e[1],e[2])}),i&&(R[i]=function(){return this.localeData().ordinal(o.apply(this,arguments),t)})}function z(t,e){return t.isValid()?(e=V(e,t.localeData()),E[e]=E[e]||function(t){var e,i,n,o=t.match(T);for(e=0,i=o.length;e<i;e++)R[o[e]]?o[e]=R[o[e]]:o[e]=(n=o[e]).match(/\[[\s\S]/)?n.replace(/^\[|\]$/g,""):n.replace(/\\/g,"");return function(e){var n,s="";for(n=0;n<i;n++)s+=S(o[n])?o[n].call(e,t):o[n];return s}}(e),E[e](t)):t.localeData().invalidDate()}function V(t,e){var i=5;function n(t){return e.longDateFormat(t)||t}for(Y.lastIndex=0;i>=0&&Y.test(t);)t=t.replace(Y,n),Y.lastIndex=0,i-=1;return t}var j=/\d/,H=/\d\d/,P=/\d{3}/,J=/\d{4}/,K=/[+-]?\d{6}/,q=/\d\d?/,$=/\d\d\d\d?/,tt=/\d\d\d\d\d\d?/,et=/\d{1,3}/,it=/\d{1,4}/,nt=/[+-]?\d{1,6}/,ot=/\d+/,st=/[+-]?\d+/,rt=/Z|[+-]\d\d:?\d\d/gi,at=/Z|[+-]\d\d(?::?\d\d)?/gi,lt=/[0-9]{0,256}['a-z\u00A0-\u05FF\u0700-\uD7FF\uF900-\uFDCF\uFDF0-\uFF07\uFF10-\uFFEF]{1,256}|[\u0600-\u06FF\/]{1,256}(\s*?[\u0600-\u06FF]{1,256}){1,2}/i,ht={};function dt(t,e,i){ht[t]=S(e)?e:function(t,n){return t&&i?i:e}}function ut(t,e){return u(ht,t)?ht[t](e._strict,e._locale):new RegExp(ct(t.replace("\\","").replace(/\\(\[)|\\(\])|\[([^\]\[]*)\]|\\(.)/g,(function(t,e,i,n,o){return e||i||n||o}))))}function ct(t){return t.replace(/[-\/\\^$*+?.()|[\]{}]/g,"\\$&")}var pt={};function mt(t,e){var i,n=e;for("string"==typeof t&&(t=[t]),l(e)&&(n=function(t,i){i[e]=k(t)}),i=0;i<t.length;i++)pt[t[i]]=n}function ft(t,e){mt(t,(function(t,i,n,o){n._w=n._w||{},e(t,n._w,n,o)}))}function gt(t,e,i){null!=e&&u(pt,t)&&pt[t](e,i._a,i,t)}function vt(t){return bt(t)?366:365}function bt(t){return t%4==0&&t%100!=0||t%400==0}U("Y",0,0,(function(){var t=this.year();return t<=9999?""+t:"+"+t})),U(0,["YY",2],0,(function(){return this.year()%100})),U(0,["YYYY",4],0,"year"),U(0,["YYYYY",5],0,"year"),U(0,["YYYYYY",6,!0],0,"year"),Z("year","y"),A("year",1),dt("Y",st),dt("YY",q,H),dt("YYYY",it,J),dt("YYYYY",nt,K),dt("YYYYYY",nt,K),mt(["YYYYY","YYYYYY"],0),mt("YYYY",(function(t,e){e[0]=2===t.length?o.parseTwoDigitYear(t):k(t)})),mt("YY",(function(t,e){e[0]=o.parseTwoDigitYear(t)})),mt("Y",(function(t,e){e[0]=parseInt(t,10)})),o.parseTwoDigitYear=function(t){return k(t)+(k(t)>68?1900:2e3)};var yt,xt=wt("FullYear",!0);function wt(t,e){return function(i){return null!=i?(kt(this,t,i),o.updateOffset(this,e),this):_t(this,t)}}function _t(t,e){return t.isValid()?t._d["get"+(t._isUTC?"UTC":"")+e]():NaN}function kt(t,e,i){t.isValid()&&!isNaN(i)&&("FullYear"===e&&bt(t.year())&&1===t.month()&&29===t.date()?t._d["set"+(t._isUTC?"UTC":"")+e](i,t.month(),Dt(i,t.month())):t._d["set"+(t._isUTC?"UTC":"")+e](i))}function Dt(t,e){if(isNaN(t)||isNaN(e))return NaN;var i,n=(e%(i=12)+i)%i;return t+=(e-n)/12,1===n?bt(t)?29:28:31-n%7%2}yt=Array.prototype.indexOf?Array.prototype.indexOf:function(t){var e;for(e=0;e<this.length;++e)if(this[e]===t)return e;return-1},U("M",["MM",2],"Mo",(function(){return this.month()+1})),U("MMM",0,0,(function(t){return this.localeData().monthsShort(this,t)})),U("MMMM",0,0,(function(t){return this.localeData().months(this,t)})),Z("month","M"),A("month",8),dt("M",q),dt("MM",q,H),dt("MMM",(function(t,e){return e.monthsShortRegex(t)})),dt("MMMM",(function(t,e){return e.monthsRegex(t)})),mt(["M","MM"],(function(t,e){e[1]=k(t)-1})),mt(["MMM","MMMM"],(function(t,e,i,n){var o=i._locale.monthsParse(t,n,i._strict);null!=o?e[1]=o:m(i).invalidMonth=t}));var It=/D[oD]?(\[[^\[\]]*\]|\s)+MMMM?/,Ct="January_February_March_April_May_June_July_August_September_October_November_December".split("_"),Lt="Jan_Feb_Mar_Apr_May_Jun_Jul_Aug_Sep_Oct_Nov_Dec".split("_");function Gt(t,e,i){var n,o,s,r=t.toLocaleLowerCase();if(!this._monthsParse)for(this._monthsParse=[],this._longMonthsParse=[],this._shortMonthsParse=[],n=0;n<12;++n)s=p([2e3,n]),this._shortMonthsParse[n]=this.monthsShort(s,"").toLocaleLowerCase(),this._longMonthsParse[n]=this.months(s,"").toLocaleLowerCase();return i?"MMM"===e?-1!==(o=yt.call(this._shortMonthsParse,r))?o:null:-1!==(o=yt.call(this._longMonthsParse,r))?o:null:"MMM"===e?-1!==(o=yt.call(this._shortMonthsParse,r))||-1!==(o=yt.call(this._longMonthsParse,r))?o:null:-1!==(o=yt.call(this._longMonthsParse,r))||-1!==(o=yt.call(this._shortMonthsParse,r))?o:null}function Mt(t,e){var i;if(!t.isValid())return t;if("string"==typeof e)if(/^\d+$/.test(e))e=k(e);else if(!l(e=t.localeData().monthsParse(e)))return t;return i=Math.min(t.date(),Dt(t.year(),e)),t._d["set"+(t._isUTC?"UTC":"")+"Month"](e,i),t}function St(t){return null!=t?(Mt(this,t),o.updateOffset(this,!0),this):_t(this,"Month")}var Qt=lt,Nt=lt;function Ft(){function t(t,e){return e.length-t.length}var e,i,n=[],o=[],s=[];for(e=0;e<12;e++)i=p([2e3,e]),n.push(this.monthsShort(i,"")),o.push(this.months(i,"")),s.push(this.months(i,"")),s.push(this.monthsShort(i,""));for(n.sort(t),o.sort(t),s.sort(t),e=0;e<12;e++)n[e]=ct(n[e]),o[e]=ct(o[e]);for(e=0;e<24;e++)s[e]=ct(s[e]);this._monthsRegex=new RegExp("^("+s.join("|")+")","i"),this._monthsShortRegex=this._monthsRegex,this._monthsStrictRegex=new RegExp("^("+o.join("|")+")","i"),this._monthsShortStrictRegex=new RegExp("^("+n.join("|")+")","i")}function Zt(t,e,i,n,o,s,r){var a;return t<100&&t>=0?(a=new Date(t+400,e,i,n,o,s,r),isFinite(a.getFullYear())&&a.setFullYear(t)):a=new Date(t,e,i,n,o,s,r),a}function Bt(t){var e;if(t<100&&t>=0){var i=Array.prototype.slice.call(arguments);i[0]=t+400,e=new Date(Date.UTC.apply(null,i)),isFinite(e.getUTCFullYear())&&e.setUTCFullYear(t)}else e=new Date(Date.UTC.apply(null,arguments));return e}function Ot(t,e,i){var n=7+e-i;return-(7+Bt(t,0,n).getUTCDay()-e)%7+n-1}function Wt(t,e,i,n,o){var s,r,a=1+7*(e-1)+(7+i-n)%7+Ot(t,n,o);return a<=0?r=vt(s=t-1)+a:a>vt(t)?(s=t+1,r=a-vt(t)):(s=t,r=a),{year:s,dayOfYear:r}}function At(t,e,i){var n,o,s=Ot(t.year(),e,i),r=Math.floor((t.dayOfYear()-s-1)/7)+1;return r<1?n=r+Xt(o=t.year()-1,e,i):r>Xt(t.year(),e,i)?(n=r-Xt(t.year(),e,i),o=t.year()+1):(o=t.year(),n=r),{week:n,year:o}}function Xt(t,e,i){var n=Ot(t,e,i),o=Ot(t+1,e,i);return(vt(t)-n+o)/7}function Tt(t,e){return t.slice(e,7).concat(t.slice(0,e))}U("w",["ww",2],"wo","week"),U("W",["WW",2],"Wo","isoWeek"),Z("week","w"),Z("isoWeek","W"),A("week",5),A("isoWeek",5),dt("w",q),dt("ww",q,H),dt("W",q),dt("WW",q,H),ft(["w","ww","W","WW"],(function(t,e,i,n){e[n.substr(0,1)]=k(t)})),U("d",0,"do","day"),U("dd",0,0,(function(t){return this.localeData().weekdaysMin(this,t)})),U("ddd",0,0,(function(t){return this.localeData().weekdaysShort(this,t)})),U("dddd",0,0,(function(t){return this.localeData().weekdays(this,t)})),U("e",0,0,"weekday"),U("E",0,0,"isoWeekday"),Z("day","d"),Z("weekday","e"),Z("isoWeekday","E"),A("day",11),A("weekday",11),A("isoWeekday",11),dt("d",q),dt("e",q),dt("E",q),dt("dd",(function(t,e){return e.weekdaysMinRegex(t)})),dt("ddd",(function(t,e){return e.weekdaysShortRegex(t)})),dt("dddd",(function(t,e){return e.weekdaysRegex(t)})),ft(["dd","ddd","dddd"],(function(t,e,i,n){var o=i._locale.weekdaysParse(t,n,i._strict);null!=o?e.d=o:m(i).invalidWeekday=t})),ft(["d","e","E"],(function(t,e,i,n){e[n]=k(t)}));var Yt="Sunday_Monday_Tuesday_Wednesday_Thursday_Friday_Saturday".split("_"),Et="Sun_Mon_Tue_Wed_Thu_Fri_Sat".split("_"),Rt="Su_Mo_Tu_We_Th_Fr_Sa".split("_");function Ut(t,e,i){var n,o,s,r=t.toLocaleLowerCase();if(!this._weekdaysParse)for(this._weekdaysParse=[],this._shortWeekdaysParse=[],this._minWeekdaysParse=[],n=0;n<7;++n)s=p([2e3,1]).day(n),this._minWeekdaysParse[n]=this.weekdaysMin(s,"").toLocaleLowerCase(),this._shortWeekdaysParse[n]=this.weekdaysShort(s,"").toLocaleLowerCase(),this._weekdaysParse[n]=this.weekdays(s,"").toLocaleLowerCase();return i?"dddd"===e?-1!==(o=yt.call(this._weekdaysParse,r))?o:null:"ddd"===e?-1!==(o=yt.call(this._shortWeekdaysParse,r))?o:null:-1!==(o=yt.call(this._minWeekdaysParse,r))?o:null:"dddd"===e?-1!==(o=yt.call(this._weekdaysParse,r))||-1!==(o=yt.call(this._shortWeekdaysParse,r))||-1!==(o=yt.call(this._minWeekdaysParse,r))?o:null:"ddd"===e?-1!==(o=yt.call(this._shortWeekdaysParse,r))||-1!==(o=yt.call(this._weekdaysParse,r))||-1!==(o=yt.call(this._minWeekdaysParse,r))?o:null:-1!==(o=yt.call(this._minWeekdaysParse,r))||-1!==(o=yt.call(this._weekdaysParse,r))||-1!==(o=yt.call(this._shortWeekdaysParse,r))?o:null}var zt=lt,Vt=lt,jt=lt;function Ht(){function t(t,e){return e.length-t.length}var e,i,n,o,s,r=[],a=[],l=[],h=[];for(e=0;e<7;e++)i=p([2e3,1]).day(e),n=this.weekdaysMin(i,""),o=this.weekdaysShort(i,""),s=this.weekdays(i,""),r.push(n),a.push(o),l.push(s),h.push(n),h.push(o),h.push(s);for(r.sort(t),a.sort(t),l.sort(t),h.sort(t),e=0;e<7;e++)a[e]=ct(a[e]),l[e]=ct(l[e]),h[e]=ct(h[e]);this._weekdaysRegex=new RegExp("^("+h.join("|")+")","i"),this._weekdaysShortRegex=this._weekdaysRegex,this._weekdaysMinRegex=this._weekdaysRegex,this._weekdaysStrictRegex=new RegExp("^("+l.join("|")+")","i"),this._weekdaysShortStrictRegex=new RegExp("^("+a.join("|")+")","i"),this._weekdaysMinStrictRegex=new RegExp("^("+r.join("|")+")","i")}function Pt(){return this.hours()%12||12}function Jt(t,e){U(t,0,0,(function(){return this.localeData().meridiem(this.hours(),this.minutes(),e)}))}function Kt(t,e){return e._meridiemParse}U("H",["HH",2],0,"hour"),U("h",["hh",2],0,Pt),U("k",["kk",2],0,(function(){return this.hours()||24})),U("hmm",0,0,(function(){return""+Pt.apply(this)+X(this.minutes(),2)})),U("hmmss",0,0,(function(){return""+Pt.apply(this)+X(this.minutes(),2)+X(this.seconds(),2)})),U("Hmm",0,0,(function(){return""+this.hours()+X(this.minutes(),2)})),U("Hmmss",0,0,(function(){return""+this.hours()+X(this.minutes(),2)+X(this.seconds(),2)})),Jt("a",!0),Jt("A",!1),Z("hour","h"),A("hour",13),dt("a",Kt),dt("A",Kt),dt("H",q),dt("h",q),dt("k",q),dt("HH",q,H),dt("hh",q,H),dt("kk",q,H),dt("hmm",$),dt("hmmss",tt),dt("Hmm",$),dt("Hmmss",tt),mt(["H","HH"],3),mt(["k","kk"],(function(t,e,i){var n=k(t);e[3]=24===n?0:n})),mt(["a","A"],(function(t,e,i){i._isPm=i._locale.isPM(t),i._meridiem=t})),mt(["h","hh"],(function(t,e,i){e[3]=k(t),m(i).bigHour=!0})),mt("hmm",(function(t,e,i){var n=t.length-2;e[3]=k(t.substr(0,n)),e[4]=k(t.substr(n)),m(i).bigHour=!0})),mt("hmmss",(function(t,e,i){var n=t.length-4,o=t.length-2;e[3]=k(t.substr(0,n)),e[4]=k(t.substr(n,2)),e[5]=k(t.substr(o)),m(i).bigHour=!0})),mt("Hmm",(function(t,e,i){var n=t.length-2;e[3]=k(t.substr(0,n)),e[4]=k(t.substr(n))})),mt("Hmmss",(function(t,e,i){var n=t.length-4,o=t.length-2;e[3]=k(t.substr(0,n)),e[4]=k(t.substr(n,2)),e[5]=k(t.substr(o))}));var qt,$t=wt("Hours",!0),te={calendar:{sameDay:"[Today at] LT",nextDay:"[Tomorrow at] LT",nextWeek:"dddd [at] LT",lastDay:"[Yesterday at] LT",lastWeek:"[Last] dddd [at] LT",sameElse:"L"},longDateFormat:{LTS:"h:mm:ss A",LT:"h:mm A",L:"MM/DD/YYYY",LL:"MMMM D, YYYY",LLL:"MMMM D, YYYY h:mm A",LLLL:"dddd, MMMM D, YYYY h:mm A"},invalidDate:"Invalid date",ordinal:"%d",dayOfMonthOrdinalParse:/\d{1,2}/,relativeTime:{future:"in %s",past:"%s ago",s:"a few seconds",ss:"%d seconds",m:"a minute",mm:"%d minutes",h:"an hour",hh:"%d hours",d:"a day",dd:"%d days",M:"a month",MM:"%d months",y:"a year",yy:"%d years"},months:Ct,monthsShort:Lt,week:{dow:0,doy:6},weekdays:Yt,weekdaysMin:Rt,weekdaysShort:Et,meridiemParse:/[ap]\.?m?\.?/i},ee={},ie={};function ne(t){return t?t.toLowerCase().replace("_","-"):t}function oe(e){var n=null;if(!ee[e]&&t&&t.exports)try{n=qt._abbr,i(),se(n)}catch(t){}return ee[e]}function se(t,e){var i;return t&&((i=a(e)?ae(t):re(t,e))?qt=i:"undefined"!=typeof console&&console.warn&&console.warn("Locale "+t+" not found. Did you forget to load it?")),qt._abbr}function re(t,e){if(null!==e){var i,n=te;if(e.abbr=t,null!=ee[t])M("defineLocaleOverride","use moment.updateLocale(localeName, config) to change an existing locale. moment.defineLocale(localeName, config) should only be used for creating a new locale See http://momentjs.com/guides/#/warnings/define-locale/ for more info."),n=ee[t]._config;else if(null!=e.parentLocale)if(null!=ee[e.parentLocale])n=ee[e.parentLocale]._config;else{if(null==(i=oe(e.parentLocale)))return ie[e.parentLocale]||(ie[e.parentLocale]=[]),ie[e.parentLocale].push({name:t,config:e}),null;n=i._config}return ee[t]=new N(Q(n,e)),ie[t]&&ie[t].forEach((function(t){re(t.name,t.config)})),se(t),ee[t]}return delete ee[t],null}function ae(t){var e;if(t&&t._locale&&t._locale._abbr&&(t=t._locale._abbr),!t)return qt;if(!s(t)){if(e=oe(t))return e;t=[t]}return function(t){for(var e,i,n,o,s=0;s<t.length;){for(e=(o=ne(t[s]).split("-")).length,i=(i=ne(t[s+1]))?i.split("-"):null;e>0;){if(n=oe(o.slice(0,e).join("-")))return n;if(i&&i.length>=e&&D(o,i,!0)>=e-1)break;e--}s++}return qt}(t)}function le(t){var e,i=t._a;return i&&-2===m(t).overflow&&(e=i[1]<0||i[1]>11?1:i[2]<1||i[2]>Dt(i[0],i[1])?2:i[3]<0||i[3]>24||24===i[3]&&(0!==i[4]||0!==i[5]||0!==i[6])?3:i[4]<0||i[4]>59?4:i[5]<0||i[5]>59?5:i[6]<0||i[6]>999?6:-1,m(t)._overflowDayOfYear&&(e<0||e>2)&&(e=2),m(t)._overflowWeeks&&-1===e&&(e=7),m(t)._overflowWeekday&&-1===e&&(e=8),m(t).overflow=e),t}function he(t,e,i){return null!=t?t:null!=e?e:i}function de(t){var e,i,n,s,r,a=[];if(!t._d){for(n=function(t){var e=new Date(o.now());return t._useUTC?[e.getUTCFullYear(),e.getUTCMonth(),e.getUTCDate()]:[e.getFullYear(),e.getMonth(),e.getDate()]}(t),t._w&&null==t._a[2]&&null==t._a[1]&&function(t){var e,i,n,o,s,r,a,l;if(null!=(e=t._w).GG||null!=e.W||null!=e.E)s=1,r=4,i=he(e.GG,t._a[0],At(Ie(),1,4).year),n=he(e.W,1),((o=he(e.E,1))<1||o>7)&&(l=!0);else{s=t._locale._week.dow,r=t._locale._week.doy;var h=At(Ie(),s,r);i=he(e.gg,t._a[0],h.year),n=he(e.w,h.week),null!=e.d?((o=e.d)<0||o>6)&&(l=!0):null!=e.e?(o=e.e+s,(e.e<0||e.e>6)&&(l=!0)):o=s}n<1||n>Xt(i,s,r)?m(t)._overflowWeeks=!0:null!=l?m(t)._overflowWeekday=!0:(a=Wt(i,n,o,s,r),t._a[0]=a.year,t._dayOfYear=a.dayOfYear)}(t),null!=t._dayOfYear&&(r=he(t._a[0],n[0]),(t._dayOfYear>vt(r)||0===t._dayOfYear)&&(m(t)._overflowDayOfYear=!0),i=Bt(r,0,t._dayOfYear),t._a[1]=i.getUTCMonth(),t._a[2]=i.getUTCDate()),e=0;e<3&&null==t._a[e];++e)t._a[e]=a[e]=n[e];for(;e<7;e++)t._a[e]=a[e]=null==t._a[e]?2===e?1:0:t._a[e];24===t._a[3]&&0===t._a[4]&&0===t._a[5]&&0===t._a[6]&&(t._nextDay=!0,t._a[3]=0),t._d=(t._useUTC?Bt:Zt).apply(null,a),s=t._useUTC?t._d.getUTCDay():t._d.getDay(),null!=t._tzm&&t._d.setUTCMinutes(t._d.getUTCMinutes()-t._tzm),t._nextDay&&(t._a[3]=24),t._w&&void 0!==t._w.d&&t._w.d!==s&&(m(t).weekdayMismatch=!0)}}var ue=/^\s*((?:[+-]\d{6}|\d{4})-(?:\d\d-\d\d|W\d\d-\d|W\d\d|\d\d\d|\d\d))(?:(T| )(\d\d(?::\d\d(?::\d\d(?:[.,]\d+)?)?)?)([\+\-]\d\d(?::?\d\d)?|\s*Z)?)?$/,ce=/^\s*((?:[+-]\d{6}|\d{4})(?:\d\d\d\d|W\d\d\d|W\d\d|\d\d\d|\d\d))(?:(T| )(\d\d(?:\d\d(?:\d\d(?:[.,]\d+)?)?)?)([\+\-]\d\d(?::?\d\d)?|\s*Z)?)?$/,pe=/Z|[+-]\d\d(?::?\d\d)?/,me=[["YYYYYY-MM-DD",/[+-]\d{6}-\d\d-\d\d/],["YYYY-MM-DD",/\d{4}-\d\d-\d\d/],["GGGG-[W]WW-E",/\d{4}-W\d\d-\d/],["GGGG-[W]WW",/\d{4}-W\d\d/,!1],["YYYY-DDD",/\d{4}-\d{3}/],["YYYY-MM",/\d{4}-\d\d/,!1],["YYYYYYMMDD",/[+-]\d{10}/],["YYYYMMDD",/\d{8}/],["GGGG[W]WWE",/\d{4}W\d{3}/],["GGGG[W]WW",/\d{4}W\d{2}/,!1],["YYYYDDD",/\d{7}/]],fe=[["HH:mm:ss.SSSS",/\d\d:\d\d:\d\d\.\d+/],["HH:mm:ss,SSSS",/\d\d:\d\d:\d\d,\d+/],["HH:mm:ss",/\d\d:\d\d:\d\d/],["HH:mm",/\d\d:\d\d/],["HHmmss.SSSS",/\d\d\d\d\d\d\.\d+/],["HHmmss,SSSS",/\d\d\d\d\d\d,\d+/],["HHmmss",/\d\d\d\d\d\d/],["HHmm",/\d\d\d\d/],["HH",/\d\d/]],ge=/^\/?Date\((\-?\d+)/i;function ve(t){var e,i,n,o,s,r,a=t._i,l=ue.exec(a)||ce.exec(a);if(l){for(m(t).iso=!0,e=0,i=me.length;e<i;e++)if(me[e][1].exec(l[1])){o=me[e][0],n=!1!==me[e][2];break}if(null==o)return void(t._isValid=!1);if(l[3]){for(e=0,i=fe.length;e<i;e++)if(fe[e][1].exec(l[3])){s=(l[2]||" ")+fe[e][0];break}if(null==s)return void(t._isValid=!1)}if(!n&&null!=s)return void(t._isValid=!1);if(l[4]){if(!pe.exec(l[4]))return void(t._isValid=!1);r="Z"}t._f=o+(s||"")+(r||""),_e(t)}else t._isValid=!1}var be=/^(?:(Mon|Tue|Wed|Thu|Fri|Sat|Sun),?\s)?(\d{1,2})\s(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s(\d{2,4})\s(\d\d):(\d\d)(?::(\d\d))?\s(?:(UT|GMT|[ECMP][SD]T)|([Zz])|([+-]\d{4}))$/;function ye(t){var e=parseInt(t,10);return e<=49?2e3+e:e<=999?1900+e:e}var xe={UT:0,GMT:0,EDT:-240,EST:-300,CDT:-300,CST:-360,MDT:-360,MST:-420,PDT:-420,PST:-480};function we(t){var e,i,n,o,s,r,a,l=be.exec(t._i.replace(/\([^)]*\)|[\n\t]/g," ").replace(/(\s\s+)/g," ").replace(/^\s\s*/,"").replace(/\s\s*$/,""));if(l){var h=(e=l[4],i=l[3],n=l[2],o=l[5],s=l[6],r=l[7],a=[ye(e),Lt.indexOf(i),parseInt(n,10),parseInt(o,10),parseInt(s,10)],r&&a.push(parseInt(r,10)),a);if(!function(t,e,i){return!t||Et.indexOf(t)===new Date(e[0],e[1],e[2]).getDay()||(m(i).weekdayMismatch=!0,i._isValid=!1,!1)}(l[1],h,t))return;t._a=h,t._tzm=function(t,e,i){if(t)return xe[t];if(e)return 0;var n=parseInt(i,10),o=n%100;return(n-o)/100*60+o}(l[8],l[9],l[10]),t._d=Bt.apply(null,t._a),t._d.setUTCMinutes(t._d.getUTCMinutes()-t._tzm),m(t).rfc2822=!0}else t._isValid=!1}function _e(t){if(t._f!==o.ISO_8601)if(t._f!==o.RFC_2822){t._a=[],m(t).empty=!0;var e,i,n,s,r,a=""+t._i,l=a.length,h=0;for(n=V(t._f,t._locale).match(T)||[],e=0;e<n.length;e++)s=n[e],(i=(a.match(ut(s,t))||[])[0])&&((r=a.substr(0,a.indexOf(i))).length>0&&m(t).unusedInput.push(r),a=a.slice(a.indexOf(i)+i.length),h+=i.length),R[s]?(i?m(t).empty=!1:m(t).unusedTokens.push(s),gt(s,i,t)):t._strict&&!i&&m(t).unusedTokens.push(s);m(t).charsLeftOver=l-h,a.length>0&&m(t).unusedInput.push(a),t._a[3]<=12&&!0===m(t).bigHour&&t._a[3]>0&&(m(t).bigHour=void 0),m(t).parsedDateParts=t._a.slice(0),m(t).meridiem=t._meridiem,t._a[3]=function(t,e,i){var n;return null==i?e:null!=t.meridiemHour?t.meridiemHour(e,i):null!=t.isPM?((n=t.isPM(i))&&e<12&&(e+=12),n||12!==e||(e=0),e):e}(t._locale,t._a[3],t._meridiem),de(t),le(t)}else we(t);else ve(t)}function ke(t){var e=t._i,i=t._f;return t._locale=t._locale||ae(t._l),null===e||void 0===i&&""===e?g({nullInput:!0}):("string"==typeof e&&(t._i=e=t._locale.preparse(e)),w(e)?new x(le(e)):(h(e)?t._d=e:s(i)?function(t){var e,i,n,o,s;if(0===t._f.length)return m(t).invalidFormat=!0,void(t._d=new Date(NaN));for(o=0;o<t._f.length;o++)s=0,e=b({},t),null!=t._useUTC&&(e._useUTC=t._useUTC),e._f=t._f[o],_e(e),f(e)&&(s+=m(e).charsLeftOver,s+=10*m(e).unusedTokens.length,m(e).score=s,(null==n||s<n)&&(n=s,i=e));c(t,i||e)}(t):i?_e(t):function(t){var e=t._i;a(e)?t._d=new Date(o.now()):h(e)?t._d=new Date(e.valueOf()):"string"==typeof e?function(t){var e=ge.exec(t._i);null===e?(ve(t),!1===t._isValid&&(delete t._isValid,we(t),!1===t._isValid&&(delete t._isValid,o.createFromInputFallback(t)))):t._d=new Date(+e[1])}(t):s(e)?(t._a=d(e.slice(0),(function(t){return parseInt(t,10)})),de(t)):r(e)?function(t){if(!t._d){var e=O(t._i);t._a=d([e.year,e.month,e.day||e.date,e.hour,e.minute,e.second,e.millisecond],(function(t){return t&&parseInt(t,10)})),de(t)}}(t):l(e)?t._d=new Date(e):o.createFromInputFallback(t)}(t),f(t)||(t._d=null),t))}function De(t,e,i,n,o){var a,l={};return!0!==i&&!1!==i||(n=i,i=void 0),(r(t)&&function(t){if(Object.getOwnPropertyNames)return 0===Object.getOwnPropertyNames(t).length;var e;for(e in t)if(t.hasOwnProperty(e))return!1;return!0}(t)||s(t)&&0===t.length)&&(t=void 0),l._isAMomentObject=!0,l._useUTC=l._isUTC=o,l._l=i,l._i=t,l._f=e,l._strict=n,(a=new x(le(ke(l))))._nextDay&&(a.add(1,"d"),a._nextDay=void 0),a}function Ie(t,e,i,n){return De(t,e,i,n,!1)}o.createFromInputFallback=C("value provided is not in a recognized RFC2822 or ISO format. moment construction falls back to js Date(), which is not reliable across all browsers and versions. Non RFC2822/ISO date formats are discouraged and will be removed in an upcoming major release. Please refer to http://momentjs.com/guides/#/warnings/js-date/ for more info.",(function(t){t._d=new Date(t._i+(t._useUTC?" UTC":""))})),o.ISO_8601=function(){},o.RFC_2822=function(){};var Ce=C("moment().min is deprecated, use moment.max instead. http://momentjs.com/guides/#/warnings/min-max/",(function(){var t=Ie.apply(null,arguments);return this.isValid()&&t.isValid()?t<this?this:t:g()})),Le=C("moment().max is deprecated, use moment.min instead. http://momentjs.com/guides/#/warnings/min-max/",(function(){var t=Ie.apply(null,arguments);return this.isValid()&&t.isValid()?t>this?this:t:g()}));function Ge(t,e){var i,n;if(1===e.length&&s(e[0])&&(e=e[0]),!e.length)return Ie();for(i=e[0],n=1;n<e.length;++n)e[n].isValid()&&!e[n][t](i)||(i=e[n]);return i}var Me=["year","quarter","month","week","day","hour","minute","second","millisecond"];function Se(t){var e=O(t),i=e.year||0,n=e.quarter||0,o=e.month||0,s=e.week||e.isoWeek||0,r=e.day||0,a=e.hour||0,l=e.minute||0,h=e.second||0,d=e.millisecond||0;this._isValid=function(t){for(var e in t)if(-1===yt.call(Me,e)||null!=t[e]&&isNaN(t[e]))return!1;for(var i=!1,n=0;n<Me.length;++n)if(t[Me[n]]){if(i)return!1;parseFloat(t[Me[n]])!==k(t[Me[n]])&&(i=!0)}return!0}(e),this._milliseconds=+d+1e3*h+6e4*l+1e3*a*60*60,this._days=+r+7*s,this._months=+o+3*n+12*i,this._data={},this._locale=ae(),this._bubble()}function Qe(t){return t instanceof Se}function Ne(t){return t<0?-1*Math.round(-1*t):Math.round(t)}function Fe(t,e){U(t,0,0,(function(){var t=this.utcOffset(),i="+";return t<0&&(t=-t,i="-"),i+X(~~(t/60),2)+e+X(~~t%60,2)}))}Fe("Z",":"),Fe("ZZ",""),dt("Z",at),dt("ZZ",at),mt(["Z","ZZ"],(function(t,e,i){i._useUTC=!0,i._tzm=Be(at,t)}));var Ze=/([\+\-]|\d\d)/gi;function Be(t,e){var i=(e||"").match(t);if(null===i)return null;var n=((i[i.length-1]||[])+"").match(Ze)||["-",0,0],o=60*n[1]+k(n[2]);return 0===o?0:"+"===n[0]?o:-o}function Oe(t,e){var i,n;return e._isUTC?(i=e.clone(),n=(w(t)||h(t)?t.valueOf():Ie(t).valueOf())-i.valueOf(),i._d.setTime(i._d.valueOf()+n),o.updateOffset(i,!1),i):Ie(t).local()}function We(t){return 15*-Math.round(t._d.getTimezoneOffset()/15)}function Ae(){return!!this.isValid()&&this._isUTC&&0===this._offset}o.updateOffset=function(){};var Xe=/^(\-|\+)?(?:(\d*)[. ])?(\d+)\:(\d+)(?:\:(\d+)(\.\d*)?)?$/,Te=/^(-|\+)?P(?:([-+]?[0-9,.]*)Y)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)W)?(?:([-+]?[0-9,.]*)D)?(?:T(?:([-+]?[0-9,.]*)H)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)S)?)?$/;function Ye(t,e){var i,n,o,s,r,a,h=t,d=null;return Qe(t)?h={ms:t._milliseconds,d:t._days,M:t._months}:l(t)?(h={},e?h[e]=t:h.milliseconds=t):(d=Xe.exec(t))?(i="-"===d[1]?-1:1,h={y:0,d:k(d[2])*i,h:k(d[3])*i,m:k(d[4])*i,s:k(d[5])*i,ms:k(Ne(1e3*d[6]))*i}):(d=Te.exec(t))?(i="-"===d[1]?-1:1,h={y:Ee(d[2],i),M:Ee(d[3],i),w:Ee(d[4],i),d:Ee(d[5],i),h:Ee(d[6],i),m:Ee(d[7],i),s:Ee(d[8],i)}):null==h?h={}:"object"==typeof h&&("from"in h||"to"in h)&&(s=Ie(h.from),r=Ie(h.to),o=s.isValid()&&r.isValid()?(r=Oe(r,s),s.isBefore(r)?a=Re(s,r):((a=Re(r,s)).milliseconds=-a.milliseconds,a.months=-a.months),a):{milliseconds:0,months:0},(h={}).ms=o.milliseconds,h.M=o.months),n=new Se(h),Qe(t)&&u(t,"_locale")&&(n._locale=t._locale),n}function Ee(t,e){var i=t&&parseFloat(t.replace(",","."));return(isNaN(i)?0:i)*e}function Re(t,e){var i={};return i.months=e.month()-t.month()+12*(e.year()-t.year()),t.clone().add(i.months,"M").isAfter(e)&&--i.months,i.milliseconds=+e-+t.clone().add(i.months,"M"),i}function Ue(t,e){return function(i,n){var o;return null===n||isNaN(+n)||(M(e,"moment()."+e+"(period, number) is deprecated. Please use moment()."+e+"(number, period). See http://momentjs.com/guides/#/warnings/add-inverted-param/ for more info."),o=i,i=n,n=o),ze(this,Ye(i="string"==typeof i?+i:i,n),t),this}}function ze(t,e,i,n){var s=e._milliseconds,r=Ne(e._days),a=Ne(e._months);t.isValid()&&(n=null==n||n,a&&Mt(t,_t(t,"Month")+a*i),r&&kt(t,"Date",_t(t,"Date")+r*i),s&&t._d.setTime(t._d.valueOf()+s*i),n&&o.updateOffset(t,r||a))}Ye.fn=Se.prototype,Ye.invalid=function(){return Ye(NaN)};var Ve=Ue(1,"add"),je=Ue(-1,"subtract");function He(t,e){var i=12*(e.year()-t.year())+(e.month()-t.month()),n=t.clone().add(i,"months");return-(i+(e-n<0?(e-n)/(n-t.clone().add(i-1,"months")):(e-n)/(t.clone().add(i+1,"months")-n)))||0}function Pe(t){var e;return void 0===t?this._locale._abbr:(null!=(e=ae(t))&&(this._locale=e),this)}o.defaultFormat="YYYY-MM-DDTHH:mm:ssZ",o.defaultFormatUtc="YYYY-MM-DDTHH:mm:ss[Z]";var Je=C("moment().lang() is deprecated. Instead, use moment().localeData() to get the language configuration. Use moment().locale() to change languages.",(function(t){return void 0===t?this.localeData():this.locale(t)}));function Ke(){return this._locale}function qe(t,e){return(t%e+e)%e}function $e(t,e,i){return t<100&&t>=0?new Date(t+400,e,i)-126227808e5:new Date(t,e,i).valueOf()}function ti(t,e,i){return t<100&&t>=0?Date.UTC(t+400,e,i)-126227808e5:Date.UTC(t,e,i)}function ei(t,e){U(0,[t,t.length],0,e)}function ii(t,e,i,n,o){var s;return null==t?At(this,n,o).year:(e>(s=Xt(t,n,o))&&(e=s),ni.call(this,t,e,i,n,o))}function ni(t,e,i,n,o){var s=Wt(t,e,i,n,o),r=Bt(s.year,0,s.dayOfYear);return this.year(r.getUTCFullYear()),this.month(r.getUTCMonth()),this.date(r.getUTCDate()),this}U(0,["gg",2],0,(function(){return this.weekYear()%100})),U(0,["GG",2],0,(function(){return this.isoWeekYear()%100})),ei("gggg","weekYear"),ei("ggggg","weekYear"),ei("GGGG","isoWeekYear"),ei("GGGGG","isoWeekYear"),Z("weekYear","gg"),Z("isoWeekYear","GG"),A("weekYear",1),A("isoWeekYear",1),dt("G",st),dt("g",st),dt("GG",q,H),dt("gg",q,H),dt("GGGG",it,J),dt("gggg",it,J),dt("GGGGG",nt,K),dt("ggggg",nt,K),ft(["gggg","ggggg","GGGG","GGGGG"],(function(t,e,i,n){e[n.substr(0,2)]=k(t)})),ft(["gg","GG"],(function(t,e,i,n){e[n]=o.parseTwoDigitYear(t)})),U("Q",0,"Qo","quarter"),Z("quarter","Q"),A("quarter",7),dt("Q",j),mt("Q",(function(t,e){e[1]=3*(k(t)-1)})),U("D",["DD",2],"Do","date"),Z("date","D"),A("date",9),dt("D",q),dt("DD",q,H),dt("Do",(function(t,e){return t?e._dayOfMonthOrdinalParse||e._ordinalParse:e._dayOfMonthOrdinalParseLenient})),mt(["D","DD"],2),mt("Do",(function(t,e){e[2]=k(t.match(q)[0])}));var oi=wt("Date",!0);U("DDD",["DDDD",3],"DDDo","dayOfYear"),Z("dayOfYear","DDD"),A("dayOfYear",4),dt("DDD",et),dt("DDDD",P),mt(["DDD","DDDD"],(function(t,e,i){i._dayOfYear=k(t)})),U("m",["mm",2],0,"minute"),Z("minute","m"),A("minute",14),dt("m",q),dt("mm",q,H),mt(["m","mm"],4);var si=wt("Minutes",!1);U("s",["ss",2],0,"second"),Z("second","s"),A("second",15),dt("s",q),dt("ss",q,H),mt(["s","ss"],5);var ri,ai=wt("Seconds",!1);for(U("S",0,0,(function(){return~~(this.millisecond()/100)})),U(0,["SS",2],0,(function(){return~~(this.millisecond()/10)})),U(0,["SSS",3],0,"millisecond"),U(0,["SSSS",4],0,(function(){return 10*this.millisecond()})),U(0,["SSSSS",5],0,(function(){return 100*this.millisecond()})),U(0,["SSSSSS",6],0,(function(){return 1e3*this.millisecond()})),U(0,["SSSSSSS",7],0,(function(){return 1e4*this.millisecond()})),U(0,["SSSSSSSS",8],0,(function(){return 1e5*this.millisecond()})),U(0,["SSSSSSSSS",9],0,(function(){return 1e6*this.millisecond()})),Z("millisecond","ms"),A("millisecond",16),dt("S",et,j),dt("SS",et,H),dt("SSS",et,P),ri="SSSS";ri.length<=9;ri+="S")dt(ri,ot);function li(t,e){e[6]=k(1e3*("0."+t))}for(ri="S";ri.length<=9;ri+="S")mt(ri,li);var hi=wt("Milliseconds",!1);U("z",0,0,"zoneAbbr"),U("zz",0,0,"zoneName");var di=x.prototype;function ui(t){return t}di.add=Ve,di.calendar=function(t,e){var i=t||Ie(),n=Oe(i,this).startOf("day"),s=o.calendarFormat(this,n)||"sameElse",r=e&&(S(e[s])?e[s].call(this,i):e[s]);return this.format(r||this.localeData().calendar(s,this,Ie(i)))},di.clone=function(){return new x(this)},di.diff=function(t,e,i){var n,o,s;if(!this.isValid())return NaN;if(!(n=Oe(t,this)).isValid())return NaN;switch(o=6e4*(n.utcOffset()-this.utcOffset()),e=B(e)){case"year":s=He(this,n)/12;break;case"month":s=He(this,n);break;case"quarter":s=He(this,n)/3;break;case"second":s=(this-n)/1e3;break;case"minute":s=(this-n)/6e4;break;case"hour":s=(this-n)/36e5;break;case"day":s=(this-n-o)/864e5;break;case"week":s=(this-n-o)/6048e5;break;default:s=this-n}return i?s:_(s)},di.endOf=function(t){var e;if(void 0===(t=B(t))||"millisecond"===t||!this.isValid())return this;var i=this._isUTC?ti:$e;switch(t){case"year":e=i(this.year()+1,0,1)-1;break;case"quarter":e=i(this.year(),this.month()-this.month()%3+3,1)-1;break;case"month":e=i(this.year(),this.month()+1,1)-1;break;case"week":e=i(this.year(),this.month(),this.date()-this.weekday()+7)-1;break;case"isoWeek":e=i(this.year(),this.month(),this.date()-(this.isoWeekday()-1)+7)-1;break;case"day":case"date":e=i(this.year(),this.month(),this.date()+1)-1;break;case"hour":e=this._d.valueOf(),e+=36e5-qe(e+(this._isUTC?0:6e4*this.utcOffset()),36e5)-1;break;case"minute":e=this._d.valueOf(),e+=6e4-qe(e,6e4)-1;break;case"second":e=this._d.valueOf(),e+=1e3-qe(e,1e3)-1}return this._d.setTime(e),o.updateOffset(this,!0),this},di.format=function(t){t||(t=this.isUtc()?o.defaultFormatUtc:o.defaultFormat);var e=z(this,t);return this.localeData().postformat(e)},di.from=function(t,e){return this.isValid()&&(w(t)&&t.isValid()||Ie(t).isValid())?Ye({to:this,from:t}).locale(this.locale()).humanize(!e):this.localeData().invalidDate()},di.fromNow=function(t){return this.from(Ie(),t)},di.to=function(t,e){return this.isValid()&&(w(t)&&t.isValid()||Ie(t).isValid())?Ye({from:this,to:t}).locale(this.locale()).humanize(!e):this.localeData().invalidDate()},di.toNow=function(t){return this.to(Ie(),t)},di.get=function(t){return S(this[t=B(t)])?this[t]():this},di.invalidAt=function(){return m(this).overflow},di.isAfter=function(t,e){var i=w(t)?t:Ie(t);return!(!this.isValid()||!i.isValid())&&("millisecond"===(e=B(e)||"millisecond")?this.valueOf()>i.valueOf():i.valueOf()<this.clone().startOf(e).valueOf())},di.isBefore=function(t,e){var i=w(t)?t:Ie(t);return!(!this.isValid()||!i.isValid())&&("millisecond"===(e=B(e)||"millisecond")?this.valueOf()<i.valueOf():this.clone().endOf(e).valueOf()<i.valueOf())},di.isBetween=function(t,e,i,n){var o=w(t)?t:Ie(t),s=w(e)?e:Ie(e);return!!(this.isValid()&&o.isValid()&&s.isValid())&&("("===(n=n||"()")[0]?this.isAfter(o,i):!this.isBefore(o,i))&&(")"===n[1]?this.isBefore(s,i):!this.isAfter(s,i))},di.isSame=function(t,e){var i,n=w(t)?t:Ie(t);return!(!this.isValid()||!n.isValid())&&("millisecond"===(e=B(e)||"millisecond")?this.valueOf()===n.valueOf():(i=n.valueOf(),this.clone().startOf(e).valueOf()<=i&&i<=this.clone().endOf(e).valueOf()))},di.isSameOrAfter=function(t,e){return this.isSame(t,e)||this.isAfter(t,e)},di.isSameOrBefore=function(t,e){return this.isSame(t,e)||this.isBefore(t,e)},di.isValid=function(){return f(this)},di.lang=Je,di.locale=Pe,di.localeData=Ke,di.max=Le,di.min=Ce,di.parsingFlags=function(){return c({},m(this))},di.set=function(t,e){if("object"==typeof t)for(var i=function(t){var e=[];for(var i in t)e.push({unit:i,priority:W[i]});return e.sort((function(t,e){return t.priority-e.priority})),e}(t=O(t)),n=0;n<i.length;n++)this[i[n].unit](t[i[n].unit]);else if(S(this[t=B(t)]))return this[t](e);return this},di.startOf=function(t){var e;if(void 0===(t=B(t))||"millisecond"===t||!this.isValid())return this;var i=this._isUTC?ti:$e;switch(t){case"year":e=i(this.year(),0,1);break;case"quarter":e=i(this.year(),this.month()-this.month()%3,1);break;case"month":e=i(this.year(),this.month(),1);break;case"week":e=i(this.year(),this.month(),this.date()-this.weekday());break;case"isoWeek":e=i(this.year(),this.month(),this.date()-(this.isoWeekday()-1));break;case"day":case"date":e=i(this.year(),this.month(),this.date());break;case"hour":e=this._d.valueOf(),e-=qe(e+(this._isUTC?0:6e4*this.utcOffset()),36e5);break;case"minute":e=this._d.valueOf(),e-=qe(e,6e4);break;case"second":e=this._d.valueOf(),e-=qe(e,1e3)}return this._d.setTime(e),o.updateOffset(this,!0),this},di.subtract=je,di.toArray=function(){var t=this;return[t.year(),t.month(),t.date(),t.hour(),t.minute(),t.second(),t.millisecond()]},di.toObject=function(){var t=this;return{years:t.year(),months:t.month(),date:t.date(),hours:t.hours(),minutes:t.minutes(),seconds:t.seconds(),milliseconds:t.milliseconds()}},di.toDate=function(){return new Date(this.valueOf())},di.toISOString=function(t){if(!this.isValid())return null;var e=!0!==t,i=e?this.clone().utc():this;return i.year()<0||i.year()>9999?z(i,e?"YYYYYY-MM-DD[T]HH:mm:ss.SSS[Z]":"YYYYYY-MM-DD[T]HH:mm:ss.SSSZ"):S(Date.prototype.toISOString)?e?this.toDate().toISOString():new Date(this.valueOf()+60*this.utcOffset()*1e3).toISOString().replace("Z",z(i,"Z")):z(i,e?"YYYY-MM-DD[T]HH:mm:ss.SSS[Z]":"YYYY-MM-DD[T]HH:mm:ss.SSSZ")},di.inspect=function(){if(!this.isValid())return"moment.invalid(/* "+this._i+" */)";var t="moment",e="";this.isLocal()||(t=0===this.utcOffset()?"moment.utc":"moment.parseZone",e="Z");var i="["+t+'("]',n=0<=this.year()&&this.year()<=9999?"YYYY":"YYYYYY",o=e+'[")]';return this.format(i+n+"-MM-DD[T]HH:mm:ss.SSS"+o)},di.toJSON=function(){return this.isValid()?this.toISOString():null},di.toString=function(){return this.clone().locale("en").format("ddd MMM DD YYYY HH:mm:ss [GMT]ZZ")},di.unix=function(){return Math.floor(this.valueOf()/1e3)},di.valueOf=function(){return this._d.valueOf()-6e4*(this._offset||0)},di.creationData=function(){return{input:this._i,format:this._f,locale:this._locale,isUTC:this._isUTC,strict:this._strict}},di.year=xt,di.isLeapYear=function(){return bt(this.year())},di.weekYear=function(t){return ii.call(this,t,this.week(),this.weekday(),this.localeData()._week.dow,this.localeData()._week.doy)},di.isoWeekYear=function(t){return ii.call(this,t,this.isoWeek(),this.isoWeekday(),1,4)},di.quarter=di.quarters=function(t){return null==t?Math.ceil((this.month()+1)/3):this.month(3*(t-1)+this.month()%3)},di.month=St,di.daysInMonth=function(){return Dt(this.year(),this.month())},di.week=di.weeks=function(t){var e=this.localeData().week(this);return null==t?e:this.add(7*(t-e),"d")},di.isoWeek=di.isoWeeks=function(t){var e=At(this,1,4).week;return null==t?e:this.add(7*(t-e),"d")},di.weeksInYear=function(){var t=this.localeData()._week;return Xt(this.year(),t.dow,t.doy)},di.isoWeeksInYear=function(){return Xt(this.year(),1,4)},di.date=oi,di.day=di.days=function(t){if(!this.isValid())return null!=t?this:NaN;var e=this._isUTC?this._d.getUTCDay():this._d.getDay();return null!=t?(t=function(t,e){return"string"!=typeof t?t:isNaN(t)?"number"==typeof(t=e.weekdaysParse(t))?t:null:parseInt(t,10)}(t,this.localeData()),this.add(t-e,"d")):e},di.weekday=function(t){if(!this.isValid())return null!=t?this:NaN;var e=(this.day()+7-this.localeData()._week.dow)%7;return null==t?e:this.add(t-e,"d")},di.isoWeekday=function(t){if(!this.isValid())return null!=t?this:NaN;if(null!=t){var e=function(t,e){return"string"==typeof t?e.weekdaysParse(t)%7||7:isNaN(t)?null:t}(t,this.localeData());return this.day(this.day()%7?e:e-7)}return this.day()||7},di.dayOfYear=function(t){var e=Math.round((this.clone().startOf("day")-this.clone().startOf("year"))/864e5)+1;return null==t?e:this.add(t-e,"d")},di.hour=di.hours=$t,di.minute=di.minutes=si,di.second=di.seconds=ai,di.millisecond=di.milliseconds=hi,di.utcOffset=function(t,e,i){var n,s=this._offset||0;if(!this.isValid())return null!=t?this:NaN;if(null!=t){if("string"==typeof t){if(null===(t=Be(at,t)))return this}else Math.abs(t)<16&&!i&&(t*=60);return!this._isUTC&&e&&(n=We(this)),this._offset=t,this._isUTC=!0,null!=n&&this.add(n,"m"),s!==t&&(!e||this._changeInProgress?ze(this,Ye(t-s,"m"),1,!1):this._changeInProgress||(this._changeInProgress=!0,o.updateOffset(this,!0),this._changeInProgress=null)),this}return this._isUTC?s:We(this)},di.utc=function(t){return this.utcOffset(0,t)},di.local=function(t){return this._isUTC&&(this.utcOffset(0,t),this._isUTC=!1,t&&this.subtract(We(this),"m")),this},di.parseZone=function(){if(null!=this._tzm)this.utcOffset(this._tzm,!1,!0);else if("string"==typeof this._i){var t=Be(rt,this._i);null!=t?this.utcOffset(t):this.utcOffset(0,!0)}return this},di.hasAlignedHourOffset=function(t){return!!this.isValid()&&(t=t?Ie(t).utcOffset():0,(this.utcOffset()-t)%60==0)},di.isDST=function(){return this.utcOffset()>this.clone().month(0).utcOffset()||this.utcOffset()>this.clone().month(5).utcOffset()},di.isLocal=function(){return!!this.isValid()&&!this._isUTC},di.isUtcOffset=function(){return!!this.isValid()&&this._isUTC},di.isUtc=Ae,di.isUTC=Ae,di.zoneAbbr=function(){return this._isUTC?"UTC":""},di.zoneName=function(){return this._isUTC?"Coordinated Universal Time":""},di.dates=C("dates accessor is deprecated. Use date instead.",oi),di.months=C("months accessor is deprecated. Use month instead",St),di.years=C("years accessor is deprecated. Use year instead",xt),di.zone=C("moment().zone is deprecated, use moment().utcOffset instead. http://momentjs.com/guides/#/warnings/zone/",(function(t,e){return null!=t?("string"!=typeof t&&(t=-t),this.utcOffset(t,e),this):-this.utcOffset()})),di.isDSTShifted=C("isDSTShifted is deprecated. See http://momentjs.com/guides/#/warnings/dst-shifted/ for more information",(function(){if(!a(this._isDSTShifted))return this._isDSTShifted;var t={};if(b(t,this),(t=ke(t))._a){var e=t._isUTC?p(t._a):Ie(t._a);this._isDSTShifted=this.isValid()&&D(t._a,e.toArray())>0}else this._isDSTShifted=!1;return this._isDSTShifted}));var ci=N.prototype;function pi(t,e,i,n){var o=ae(),s=p().set(n,e);return o[i](s,t)}function mi(t,e,i){if(l(t)&&(e=t,t=void 0),t=t||"",null!=e)return pi(t,e,i,"month");var n,o=[];for(n=0;n<12;n++)o[n]=pi(t,n,i,"month");return o}function fi(t,e,i,n){"boolean"==typeof t?(l(e)&&(i=e,e=void 0),e=e||""):(i=e=t,t=!1,l(e)&&(i=e,e=void 0),e=e||"");var o,s=ae(),r=t?s._week.dow:0;if(null!=i)return pi(e,(i+r)%7,n,"day");var a=[];for(o=0;o<7;o++)a[o]=pi(e,(o+r)%7,n,"day");return a}ci.calendar=function(t,e,i){var n=this._calendar[t]||this._calendar.sameElse;return S(n)?n.call(e,i):n},ci.longDateFormat=function(t){var e=this._longDateFormat[t],i=this._longDateFormat[t.toUpperCase()];return e||!i?e:(this._longDateFormat[t]=i.replace(/MMMM|MM|DD|dddd/g,(function(t){return t.slice(1)})),this._longDateFormat[t])},ci.invalidDate=function(){return this._invalidDate},ci.ordinal=function(t){return this._ordinal.replace("%d",t)},ci.preparse=ui,ci.postformat=ui,ci.relativeTime=function(t,e,i,n){var o=this._relativeTime[i];return S(o)?o(t,e,i,n):o.replace(/%d/i,t)},ci.pastFuture=function(t,e){var i=this._relativeTime[t>0?"future":"past"];return S(i)?i(e):i.replace(/%s/i,e)},ci.set=function(t){var e,i;for(i in t)S(e=t[i])?this[i]=e:this["_"+i]=e;this._config=t,this._dayOfMonthOrdinalParseLenient=new RegExp((this._dayOfMonthOrdinalParse.source||this._ordinalParse.source)+"|"+/\d{1,2}/.source)},ci.months=function(t,e){return t?s(this._months)?this._months[t.month()]:this._months[(this._months.isFormat||It).test(e)?"format":"standalone"][t.month()]:s(this._months)?this._months:this._months.standalone},ci.monthsShort=function(t,e){return t?s(this._monthsShort)?this._monthsShort[t.month()]:this._monthsShort[It.test(e)?"format":"standalone"][t.month()]:s(this._monthsShort)?this._monthsShort:this._monthsShort.standalone},ci.monthsParse=function(t,e,i){var n,o,s;if(this._monthsParseExact)return Gt.call(this,t,e,i);for(this._monthsParse||(this._monthsParse=[],this._longMonthsParse=[],this._shortMonthsParse=[]),n=0;n<12;n++){if(o=p([2e3,n]),i&&!this._longMonthsParse[n]&&(this._longMonthsParse[n]=new RegExp("^"+this.months(o,"").replace(".","")+"$","i"),this._shortMonthsParse[n]=new RegExp("^"+this.monthsShort(o,"").replace(".","")+"$","i")),i||this._monthsParse[n]||(s="^"+this.months(o,"")+"|^"+this.monthsShort(o,""),this._monthsParse[n]=new RegExp(s.replace(".",""),"i")),i&&"MMMM"===e&&this._longMonthsParse[n].test(t))return n;if(i&&"MMM"===e&&this._shortMonthsParse[n].test(t))return n;if(!i&&this._monthsParse[n].test(t))return n}},ci.monthsRegex=function(t){return this._monthsParseExact?(u(this,"_monthsRegex")||Ft.call(this),t?this._monthsStrictRegex:this._monthsRegex):(u(this,"_monthsRegex")||(this._monthsRegex=Nt),this._monthsStrictRegex&&t?this._monthsStrictRegex:this._monthsRegex)},ci.monthsShortRegex=function(t){return this._monthsParseExact?(u(this,"_monthsRegex")||Ft.call(this),t?this._monthsShortStrictRegex:this._monthsShortRegex):(u(this,"_monthsShortRegex")||(this._monthsShortRegex=Qt),this._monthsShortStrictRegex&&t?this._monthsShortStrictRegex:this._monthsShortRegex)},ci.week=function(t){return At(t,this._week.dow,this._week.doy).week},ci.firstDayOfYear=function(){return this._week.doy},ci.firstDayOfWeek=function(){return this._week.dow},ci.weekdays=function(t,e){var i=s(this._weekdays)?this._weekdays:this._weekdays[t&&!0!==t&&this._weekdays.isFormat.test(e)?"format":"standalone"];return!0===t?Tt(i,this._week.dow):t?i[t.day()]:i},ci.weekdaysMin=function(t){return!0===t?Tt(this._weekdaysMin,this._week.dow):t?this._weekdaysMin[t.day()]:this._weekdaysMin},ci.weekdaysShort=function(t){return!0===t?Tt(this._weekdaysShort,this._week.dow):t?this._weekdaysShort[t.day()]:this._weekdaysShort},ci.weekdaysParse=function(t,e,i){var n,o,s;if(this._weekdaysParseExact)return Ut.call(this,t,e,i);for(this._weekdaysParse||(this._weekdaysParse=[],this._minWeekdaysParse=[],this._shortWeekdaysParse=[],this._fullWeekdaysParse=[]),n=0;n<7;n++){if(o=p([2e3,1]).day(n),i&&!this._fullWeekdaysParse[n]&&(this._fullWeekdaysParse[n]=new RegExp("^"+this.weekdays(o,"").replace(".","\\.?")+"$","i"),this._shortWeekdaysParse[n]=new RegExp("^"+this.weekdaysShort(o,"").replace(".","\\.?")+"$","i"),this._minWeekdaysParse[n]=new RegExp("^"+this.weekdaysMin(o,"").replace(".","\\.?")+"$","i")),this._weekdaysParse[n]||(s="^"+this.weekdays(o,"")+"|^"+this.weekdaysShort(o,"")+"|^"+this.weekdaysMin(o,""),this._weekdaysParse[n]=new RegExp(s.replace(".",""),"i")),i&&"dddd"===e&&this._fullWeekdaysParse[n].test(t))return n;if(i&&"ddd"===e&&this._shortWeekdaysParse[n].test(t))return n;if(i&&"dd"===e&&this._minWeekdaysParse[n].test(t))return n;if(!i&&this._weekdaysParse[n].test(t))return n}},ci.weekdaysRegex=function(t){return this._weekdaysParseExact?(u(this,"_weekdaysRegex")||Ht.call(this),t?this._weekdaysStrictRegex:this._weekdaysRegex):(u(this,"_weekdaysRegex")||(this._weekdaysRegex=zt),this._weekdaysStrictRegex&&t?this._weekdaysStrictRegex:this._weekdaysRegex)},ci.weekdaysShortRegex=function(t){return this._weekdaysParseExact?(u(this,"_weekdaysRegex")||Ht.call(this),t?this._weekdaysShortStrictRegex:this._weekdaysShortRegex):(u(this,"_weekdaysShortRegex")||(this._weekdaysShortRegex=Vt),this._weekdaysShortStrictRegex&&t?this._weekdaysShortStrictRegex:this._weekdaysShortRegex)},ci.weekdaysMinRegex=function(t){return this._weekdaysParseExact?(u(this,"_weekdaysRegex")||Ht.call(this),t?this._weekdaysMinStrictRegex:this._weekdaysMinRegex):(u(this,"_weekdaysMinRegex")||(this._weekdaysMinRegex=jt),this._weekdaysMinStrictRegex&&t?this._weekdaysMinStrictRegex:this._weekdaysMinRegex)},ci.isPM=function(t){return"p"===(t+"").toLowerCase().charAt(0)},ci.meridiem=function(t,e,i){return t>11?i?"pm":"PM":i?"am":"AM"},se("en",{dayOfMonthOrdinalParse:/\d{1,2}(th|st|nd|rd)/,ordinal:function(t){var e=t%10;return t+(1===k(t%100/10)?"th":1===e?"st":2===e?"nd":3===e?"rd":"th")}}),o.lang=C("moment.lang is deprecated. Use moment.locale instead.",se),o.langData=C("moment.langData is deprecated. Use moment.localeData instead.",ae);var gi=Math.abs;function vi(t,e,i,n){var o=Ye(e,i);return t._milliseconds+=n*o._milliseconds,t._days+=n*o._days,t._months+=n*o._months,t._bubble()}function bi(t){return t<0?Math.floor(t):Math.ceil(t)}function yi(t){return 4800*t/146097}function xi(t){return 146097*t/4800}function wi(t){return function(){return this.as(t)}}var _i=wi("ms"),ki=wi("s"),Di=wi("m"),Ii=wi("h"),Ci=wi("d"),Li=wi("w"),Gi=wi("M"),Mi=wi("Q"),Si=wi("y");function Qi(t){return function(){return this.isValid()?this._data[t]:NaN}}var Ni=Qi("milliseconds"),Fi=Qi("seconds"),Zi=Qi("minutes"),Bi=Qi("hours"),Oi=Qi("days"),Wi=Qi("months"),Ai=Qi("years"),Xi=Math.round,Ti={ss:44,s:45,m:45,h:22,d:26,M:11};function Yi(t,e,i,n,o){return o.relativeTime(e||1,!!i,t,n)}function Ei(t,e,i){var n=Ye(t).abs(),o=Xi(n.as("s")),s=Xi(n.as("m")),r=Xi(n.as("h")),a=Xi(n.as("d")),l=Xi(n.as("M")),h=Xi(n.as("y")),d=o<=Ti.ss&&["s",o]||o<Ti.s&&["ss",o]||s<=1&&["m"]||s<Ti.m&&["mm",s]||r<=1&&["h"]||r<Ti.h&&["hh",r]||a<=1&&["d"]||a<Ti.d&&["dd",a]||l<=1&&["M"]||l<Ti.M&&["MM",l]||h<=1&&["y"]||["yy",h];return d[2]=e,d[3]=+t>0,d[4]=i,Yi.apply(null,d)}var Ri=Math.abs;function Ui(t){return(t>0)-(t<0)||+t}function zi(){if(!this.isValid())return this.localeData().invalidDate();var t,e,i=Ri(this._milliseconds)/1e3,n=Ri(this._days),o=Ri(this._months);t=_(i/60),e=_(t/60),i%=60,t%=60;var s=_(o/12),r=o%=12,a=n,l=e,h=t,d=i?i.toFixed(3).replace(/\.?0+$/,""):"",u=this.asSeconds();if(!u)return"P0D";var c=u<0?"-":"",p=Ui(this._months)!==Ui(u)?"-":"",m=Ui(this._days)!==Ui(u)?"-":"",f=Ui(this._milliseconds)!==Ui(u)?"-":"";return c+"P"+(s?p+s+"Y":"")+(r?p+r+"M":"")+(a?m+a+"D":"")+(l||h||d?"T":"")+(l?f+l+"H":"")+(h?f+h+"M":"")+(d?f+d+"S":"")}var Vi=Se.prototype;return Vi.isValid=function(){return this._isValid},Vi.abs=function(){var t=this._data;return this._milliseconds=gi(this._milliseconds),this._days=gi(this._days),this._months=gi(this._months),t.milliseconds=gi(t.milliseconds),t.seconds=gi(t.seconds),t.minutes=gi(t.minutes),t.hours=gi(t.hours),t.months=gi(t.months),t.years=gi(t.years),this},Vi.add=function(t,e){return vi(this,t,e,1)},Vi.subtract=function(t,e){return vi(this,t,e,-1)},Vi.as=function(t){if(!this.isValid())return NaN;var e,i,n=this._milliseconds;if("month"===(t=B(t))||"quarter"===t||"year"===t)switch(e=this._days+n/864e5,i=this._months+yi(e),t){case"month":return i;case"quarter":return i/3;case"year":return i/12}else switch(e=this._days+Math.round(xi(this._months)),t){case"week":return e/7+n/6048e5;case"day":return e+n/864e5;case"hour":return 24*e+n/36e5;case"minute":return 1440*e+n/6e4;case"second":return 86400*e+n/1e3;case"millisecond":return Math.floor(864e5*e)+n;default:throw new Error("Unknown unit "+t)}},Vi.asMilliseconds=_i,Vi.asSeconds=ki,Vi.asMinutes=Di,Vi.asHours=Ii,Vi.asDays=Ci,Vi.asWeeks=Li,Vi.asMonths=Gi,Vi.asQuarters=Mi,Vi.asYears=Si,Vi.valueOf=function(){return this.isValid()?this._milliseconds+864e5*this._days+this._months%12*2592e6+31536e6*k(this._months/12):NaN},Vi._bubble=function(){var t,e,i,n,o,s=this._milliseconds,r=this._days,a=this._months,l=this._data;return s>=0&&r>=0&&a>=0||s<=0&&r<=0&&a<=0||(s+=864e5*bi(xi(a)+r),r=0,a=0),l.milliseconds=s%1e3,t=_(s/1e3),l.seconds=t%60,e=_(t/60),l.minutes=e%60,i=_(e/60),l.hours=i%24,r+=_(i/24),o=_(yi(r)),a+=o,r-=bi(xi(o)),n=_(a/12),a%=12,l.days=r,l.months=a,l.years=n,this},Vi.clone=function(){return Ye(this)},Vi.get=function(t){return t=B(t),this.isValid()?this[t+"s"]():NaN},Vi.milliseconds=Ni,Vi.seconds=Fi,Vi.minutes=Zi,Vi.hours=Bi,Vi.days=Oi,Vi.weeks=function(){return _(this.days()/7)},Vi.months=Wi,Vi.years=Ai,Vi.humanize=function(t){if(!this.isValid())return this.localeData().invalidDate();var e=this.localeData(),i=Ei(this,!t,e);return t&&(i=e.pastFuture(+this,i)),e.postformat(i)},Vi.toISOString=zi,Vi.toString=zi,Vi.toJSON=zi,Vi.locale=Pe,Vi.localeData=Ke,Vi.toIsoString=C("toIsoString() is deprecated. Please use toISOString() instead (notice the capitals)",zi),Vi.lang=Je,U("X",0,0,"unix"),U("x",0,0,"valueOf"),dt("x",st),dt("X",/[+-]?\d+(\.\d{1,3})?/),mt("X",(function(t,e,i){i._d=new Date(1e3*parseFloat(t,10))})),mt("x",(function(t,e,i){i._d=new Date(k(t))})),o.version="2.24.0",e=Ie,o.fn=di,o.min=function(){var t=[].slice.call(arguments,0);return Ge("isBefore",t)},o.max=function(){var t=[].slice.call(arguments,0);return Ge("isAfter",t)},o.now=function(){return Date.now?Date.now():+new Date},o.utc=p,o.unix=function(t){return Ie(1e3*t)},o.months=function(t,e){return mi(t,e,"months")},o.isDate=h,o.locale=se,o.invalid=g,o.duration=Ye,o.isMoment=w,o.weekdays=function(t,e,i){return fi(t,e,i,"weekdays")},o.parseZone=function(){return Ie.apply(null,arguments).parseZone()},o.localeData=ae,o.isDuration=Qe,o.monthsShort=function(t,e){return mi(t,e,"monthsShort")},o.weekdaysMin=function(t,e,i){return fi(t,e,i,"weekdaysMin")},o.defineLocale=re,o.updateLocale=function(t,e){if(null!=e){var i,n,o=te;null!=(n=oe(t))&&(o=n._config),(i=new N(e=Q(o,e))).parentLocale=ee[t],ee[t]=i,se(t)}else null!=ee[t]&&(null!=ee[t].parentLocale?ee[t]=ee[t].parentLocale:null!=ee[t]&&delete ee[t]);return ee[t]},o.locales=function(){return L(ee)},o.weekdaysShort=function(t,e,i){return fi(t,e,i,"weekdaysShort")},o.normalizeUnits=B,o.relativeTimeRounding=function(t){return void 0===t?Xi:"function"==typeof t&&(Xi=t,!0)},o.relativeTimeThreshold=function(t,e){return void 0!==Ti[t]&&(void 0===e?Ti[t]:(Ti[t]=e,"s"===t&&(Ti.ss=e-1),!0))},o.calendarFormat=function(t,e){var i=t.diff(e,"days",!0);return i<-6?"sameElse":i<-1?"lastWeek":i<0?"lastDay":i<1?"sameDay":i<2?"nextDay":i<7?"nextWeek":"sameElse"},o.prototype=di,o.HTML5_FMT={DATETIME_LOCAL:"YYYY-MM-DDTHH:mm",DATETIME_LOCAL_SECONDS:"YYYY-MM-DDTHH:mm:ss",DATETIME_LOCAL_MS:"YYYY-MM-DDTHH:mm:ss.SSS",DATE:"YYYY-MM-DD",TIME:"HH:mm",TIME_SECONDS:"HH:mm:ss",TIME_MS:"HH:mm:ss.SSS",WEEK:"GGGG-[W]WW",MONTH:"YYYY-MM"},o}()})),r=(n((function(t,e){(function(t){function e(t,e,i,n){var o={m:["eine Minute","einer Minute"],h:["eine Stunde","einer Stunde"],d:["ein Tag","einem Tag"],dd:[t+" Tage",t+" Tagen"],M:["ein Monat","einem Monat"],MM:[t+" Monate",t+" Monaten"],y:["ein Jahr","einem Jahr"],yy:[t+" Jahre",t+" Jahren"]};return e?o[i][0]:o[i][1]}t.defineLocale("de",{months:"Januar_Februar_März_April_Mai_Juni_Juli_August_September_Oktober_November_Dezember".split("_"),monthsShort:"Jan._Feb._März_Apr._Mai_Juni_Juli_Aug._Sep._Okt._Nov._Dez.".split("_"),monthsParseExact:!0,weekdays:"Sonntag_Montag_Dienstag_Mittwoch_Donnerstag_Freitag_Samstag".split("_"),weekdaysShort:"So._Mo._Di._Mi._Do._Fr._Sa.".split("_"),weekdaysMin:"So_Mo_Di_Mi_Do_Fr_Sa".split("_"),weekdaysParseExact:!0,longDateFormat:{LT:"HH:mm",LTS:"HH:mm:ss",L:"DD.MM.YYYY",LL:"D. MMMM YYYY",LLL:"D. MMMM YYYY HH:mm",LLLL:"dddd, D. MMMM YYYY HH:mm"},calendar:{sameDay:"[heute um] LT [Uhr]",sameElse:"L",nextDay:"[morgen um] LT [Uhr]",nextWeek:"dddd [um] LT [Uhr]",lastDay:"[gestern um] LT [Uhr]",lastWeek:"[letzten] dddd [um] LT [Uhr]"},relativeTime:{future:"in %s",past:"vor %s",s:"ein paar Sekunden",ss:"%d Sekunden",m:e,mm:"%d Minuten",h:e,hh:"%d Stunden",d:e,dd:e,M:e,MM:e,y:e,yy:e},dayOfMonthOrdinalParse:/\d{1,2}\./,ordinal:"%d.",week:{dow:1,doy:4}})})(s)})),n((function(t,e){(function(t){var e="ene._feb._mar._abr._may._jun._jul._ago._sep._oct._nov._dic.".split("_"),i="ene_feb_mar_abr_may_jun_jul_ago_sep_oct_nov_dic".split("_"),n=[/^ene/i,/^feb/i,/^mar/i,/^abr/i,/^may/i,/^jun/i,/^jul/i,/^ago/i,/^sep/i,/^oct/i,/^nov/i,/^dic/i],o=/^(enero|febrero|marzo|abril|mayo|junio|julio|agosto|septiembre|octubre|noviembre|diciembre|ene\.?|feb\.?|mar\.?|abr\.?|may\.?|jun\.?|jul\.?|ago\.?|sep\.?|oct\.?|nov\.?|dic\.?)/i;t.defineLocale("es",{months:"enero_febrero_marzo_abril_mayo_junio_julio_agosto_septiembre_octubre_noviembre_diciembre".split("_"),monthsShort:function(t,n){return t?/-MMM-/.test(n)?i[t.month()]:e[t.month()]:e},monthsRegex:o,monthsShortRegex:o,monthsStrictRegex:/^(enero|febrero|marzo|abril|mayo|junio|julio|agosto|septiembre|octubre|noviembre|diciembre)/i,monthsShortStrictRegex:/^(ene\.?|feb\.?|mar\.?|abr\.?|may\.?|jun\.?|jul\.?|ago\.?|sep\.?|oct\.?|nov\.?|dic\.?)/i,monthsParse:n,longMonthsParse:n,shortMonthsParse:n,weekdays:"domingo_lunes_martes_miércoles_jueves_viernes_sábado".split("_"),weekdaysShort:"dom._lun._mar._mié._jue._vie._sáb.".split("_"),weekdaysMin:"do_lu_ma_mi_ju_vi_sá".split("_"),weekdaysParseExact:!0,longDateFormat:{LT:"H:mm",LTS:"H:mm:ss",L:"DD/MM/YYYY",LL:"D [de] MMMM [de] YYYY",LLL:"D [de] MMMM [de] YYYY H:mm",LLLL:"dddd, D [de] MMMM [de] YYYY H:mm"},calendar:{sameDay:function(){return"[hoy a la"+(1!==this.hours()?"s":"")+"] LT"},nextDay:function(){return"[mañana a la"+(1!==this.hours()?"s":"")+"] LT"},nextWeek:function(){return"dddd [a la"+(1!==this.hours()?"s":"")+"] LT"},lastDay:function(){return"[ayer a la"+(1!==this.hours()?"s":"")+"] LT"},lastWeek:function(){return"[el] dddd [pasado a la"+(1!==this.hours()?"s":"")+"] LT"},sameElse:"L"},relativeTime:{future:"en %s",past:"hace %s",s:"unos segundos",ss:"%d segundos",m:"un minuto",mm:"%d minutos",h:"una hora",hh:"%d horas",d:"un día",dd:"%d días",M:"un mes",MM:"%d meses",y:"un año",yy:"%d años"},dayOfMonthOrdinalParse:/\d{1,2}º/,ordinal:"%dº",week:{dow:1,doy:4}})})(s)})),n((function(t,e){(function(t){t.defineLocale("fr",{months:"janvier_février_mars_avril_mai_juin_juillet_août_septembre_octobre_novembre_décembre".split("_"),monthsShort:"janv._févr._mars_avr._mai_juin_juil._août_sept._oct._nov._déc.".split("_"),monthsParseExact:!0,weekdays:"dimanche_lundi_mardi_mercredi_jeudi_vendredi_samedi".split("_"),weekdaysShort:"dim._lun._mar._mer._jeu._ven._sam.".split("_"),weekdaysMin:"di_lu_ma_me_je_ve_sa".split("_"),weekdaysParseExact:!0,longDateFormat:{LT:"HH:mm",LTS:"HH:mm:ss",L:"DD/MM/YYYY",LL:"D MMMM YYYY",LLL:"D MMMM YYYY HH:mm",LLLL:"dddd D MMMM YYYY HH:mm"},calendar:{sameDay:"[Aujourd’hui à] LT",nextDay:"[Demain à] LT",nextWeek:"dddd [à] LT",lastDay:"[Hier à] LT",lastWeek:"dddd [dernier à] LT",sameElse:"L"},relativeTime:{future:"dans %s",past:"il y a %s",s:"quelques secondes",ss:"%d secondes",m:"une minute",mm:"%d minutes",h:"une heure",hh:"%d heures",d:"un jour",dd:"%d jours",M:"un mois",MM:"%d mois",y:"un an",yy:"%d ans"},dayOfMonthOrdinalParse:/\d{1,2}(er|)/,ordinal:function(t,e){switch(e){case"D":return t+(1===t?"er":"");default:case"M":case"Q":case"DDD":case"d":return t+(1===t?"er":"e");case"w":case"W":return t+(1===t?"re":"e")}},week:{dow:1,doy:4}})})(s)})),n((function(t,e){(function(t){t.defineLocale("it",{months:"gennaio_febbraio_marzo_aprile_maggio_giugno_luglio_agosto_settembre_ottobre_novembre_dicembre".split("_"),monthsShort:"gen_feb_mar_apr_mag_giu_lug_ago_set_ott_nov_dic".split("_"),weekdays:"domenica_lunedì_martedì_mercoledì_giovedì_venerdì_sabato".split("_"),weekdaysShort:"dom_lun_mar_mer_gio_ven_sab".split("_"),weekdaysMin:"do_lu_ma_me_gi_ve_sa".split("_"),longDateFormat:{LT:"HH:mm",LTS:"HH:mm:ss",L:"DD/MM/YYYY",LL:"D MMMM YYYY",LLL:"D MMMM YYYY HH:mm",LLLL:"dddd D MMMM YYYY HH:mm"},calendar:{sameDay:"[Oggi alle] LT",nextDay:"[Domani alle] LT",nextWeek:"dddd [alle] LT",lastDay:"[Ieri alle] LT",lastWeek:function(){switch(this.day()){case 0:return"[la scorsa] dddd [alle] LT";default:return"[lo scorso] dddd [alle] LT"}},sameElse:"L"},relativeTime:{future:function(t){return(/^[0-9].+$/.test(t)?"tra":"in")+" "+t},past:"%s fa",s:"alcuni secondi",ss:"%d secondi",m:"un minuto",mm:"%d minuti",h:"un'ora",hh:"%d ore",d:"un giorno",dd:"%d giorni",M:"un mese",MM:"%d mesi",y:"un anno",yy:"%d anni"},dayOfMonthOrdinalParse:/\d{1,2}º/,ordinal:"%dº",week:{dow:1,doy:4}})})(s)})),n((function(t,e){(function(t){t.defineLocale("ja",{months:"一月_二月_三月_四月_五月_六月_七月_八月_九月_十月_十一月_十二月".split("_"),monthsShort:"1月_2月_3月_4月_5月_6月_7月_8月_9月_10月_11月_12月".split("_"),weekdays:"日曜日_月曜日_火曜日_水曜日_木曜日_金曜日_土曜日".split("_"),weekdaysShort:"日_月_火_水_木_金_土".split("_"),weekdaysMin:"日_月_火_水_木_金_土".split("_"),longDateFormat:{LT:"HH:mm",LTS:"HH:mm:ss",L:"YYYY/MM/DD",LL:"YYYY年M月D日",LLL:"YYYY年M月D日 HH:mm",LLLL:"YYYY年M月D日 dddd HH:mm",l:"YYYY/MM/DD",ll:"YYYY年M月D日",lll:"YYYY年M月D日 HH:mm",llll:"YYYY年M月D日(ddd) HH:mm"},meridiemParse:/午前|午後/i,isPM:function(t){return"午後"===t},meridiem:function(t,e,i){return t<12?"午前":"午後"},calendar:{sameDay:"[今日] LT",nextDay:"[明日] LT",nextWeek:function(t){return t.week()<this.week()?"[来週]dddd LT":"dddd LT"},lastDay:"[昨日] LT",lastWeek:function(t){return this.week()<t.week()?"[先週]dddd LT":"dddd LT"},sameElse:"L"},dayOfMonthOrdinalParse:/\d{1,2}日/,ordinal:function(t,e){switch(e){case"d":case"D":case"DDD":return t+"日";default:return t}},relativeTime:{future:"%s後",past:"%s前",s:"数秒",ss:"%d秒",m:"1分",mm:"%d分",h:"1時間",hh:"%d時間",d:"1日",dd:"%d日",M:"1ヶ月",MM:"%dヶ月",y:"1年",yy:"%d年"}})})(s)})),n((function(t,e){(function(t){var e="jan._feb._mrt._apr._mei_jun._jul._aug._sep._okt._nov._dec.".split("_"),i="jan_feb_mrt_apr_mei_jun_jul_aug_sep_okt_nov_dec".split("_"),n=[/^jan/i,/^feb/i,/^maart|mrt.?$/i,/^apr/i,/^mei$/i,/^jun[i.]?$/i,/^jul[i.]?$/i,/^aug/i,/^sep/i,/^okt/i,/^nov/i,/^dec/i],o=/^(januari|februari|maart|april|mei|ju[nl]i|augustus|september|oktober|november|december|jan\.?|feb\.?|mrt\.?|apr\.?|ju[nl]\.?|aug\.?|sep\.?|okt\.?|nov\.?|dec\.?)/i;t.defineLocale("nl",{months:"januari_februari_maart_april_mei_juni_juli_augustus_september_oktober_november_december".split("_"),monthsShort:function(t,n){return t?/-MMM-/.test(n)?i[t.month()]:e[t.month()]:e},monthsRegex:o,monthsShortRegex:o,monthsStrictRegex:/^(januari|februari|maart|april|mei|ju[nl]i|augustus|september|oktober|november|december)/i,monthsShortStrictRegex:/^(jan\.?|feb\.?|mrt\.?|apr\.?|mei|ju[nl]\.?|aug\.?|sep\.?|okt\.?|nov\.?|dec\.?)/i,monthsParse:n,longMonthsParse:n,shortMonthsParse:n,weekdays:"zondag_maandag_dinsdag_woensdag_donderdag_vrijdag_zaterdag".split("_"),weekdaysShort:"zo._ma._di._wo._do._vr._za.".split("_"),weekdaysMin:"zo_ma_di_wo_do_vr_za".split("_"),weekdaysParseExact:!0,longDateFormat:{LT:"HH:mm",LTS:"HH:mm:ss",L:"DD-MM-YYYY",LL:"D MMMM YYYY",LLL:"D MMMM YYYY HH:mm",LLLL:"dddd D MMMM YYYY HH:mm"},calendar:{sameDay:"[vandaag om] LT",nextDay:"[morgen om] LT",nextWeek:"dddd [om] LT",lastDay:"[gisteren om] LT",lastWeek:"[afgelopen] dddd [om] LT",sameElse:"L"},relativeTime:{future:"over %s",past:"%s geleden",s:"een paar seconden",ss:"%d seconden",m:"één minuut",mm:"%d minuten",h:"één uur",hh:"%d uur",d:"één dag",dd:"%d dagen",M:"één maand",MM:"%d maanden",y:"één jaar",yy:"%d jaar"},dayOfMonthOrdinalParse:/\d{1,2}(ste|de)/,ordinal:function(t){return t+(1===t||8===t||t>=20?"ste":"de")},week:{dow:1,doy:4}})})(s)})),n((function(t,e){(function(t){var e="styczeń_luty_marzec_kwiecień_maj_czerwiec_lipiec_sierpień_wrzesień_październik_listopad_grudzień".split("_"),i="stycznia_lutego_marca_kwietnia_maja_czerwca_lipca_sierpnia_września_października_listopada_grudnia".split("_");function n(t){return t%10<5&&t%10>1&&~~(t/10)%10!=1}function o(t,e,i){var o=t+" ";switch(i){case"ss":return o+(n(t)?"sekundy":"sekund");case"m":return e?"minuta":"minutę";case"mm":return o+(n(t)?"minuty":"minut");case"h":return e?"godzina":"godzinę";case"hh":return o+(n(t)?"godziny":"godzin");case"MM":return o+(n(t)?"miesiące":"miesięcy");case"yy":return o+(n(t)?"lata":"lat")}}t.defineLocale("pl",{months:function(t,n){return t?""===n?"("+i[t.month()]+"|"+e[t.month()]+")":/D MMMM/.test(n)?i[t.month()]:e[t.month()]:e},monthsShort:"sty_lut_mar_kwi_maj_cze_lip_sie_wrz_paź_lis_gru".split("_"),weekdays:"niedziela_poniedziałek_wtorek_środa_czwartek_piątek_sobota".split("_"),weekdaysShort:"ndz_pon_wt_śr_czw_pt_sob".split("_"),weekdaysMin:"Nd_Pn_Wt_Śr_Cz_Pt_So".split("_"),longDateFormat:{LT:"HH:mm",LTS:"HH:mm:ss",L:"DD.MM.YYYY",LL:"D MMMM YYYY",LLL:"D MMMM YYYY HH:mm",LLLL:"dddd, D MMMM YYYY HH:mm"},calendar:{sameDay:"[Dziś o] LT",nextDay:"[Jutro o] LT",nextWeek:function(){switch(this.day()){case 0:return"[W niedzielę o] LT";case 2:return"[We wtorek o] LT";case 3:return"[W środę o] LT";case 6:return"[W sobotę o] LT";default:return"[W] dddd [o] LT"}},lastDay:"[Wczoraj o] LT",lastWeek:function(){switch(this.day()){case 0:return"[W zeszłą niedzielę o] LT";case 3:return"[W zeszłą środę o] LT";case 6:return"[W zeszłą sobotę o] LT";default:return"[W zeszły] dddd [o] LT"}},sameElse:"L"},relativeTime:{future:"za %s",past:"%s temu",s:"kilka sekund",ss:o,m:o,mm:o,h:o,hh:o,d:"1 dzień",dd:"%d dni",M:"miesiąc",MM:o,y:"rok",yy:o},dayOfMonthOrdinalParse:/\d{1,2}\./,ordinal:"%d.",week:{dow:1,doy:4}})})(s)})),n((function(t,e){(function(t){function e(t,e,i){var n,o;return"m"===i?e?"минута":"минуту":t+" "+(n=+t,o={ss:e?"секунда_секунды_секунд":"секунду_секунды_секунд",mm:e?"минута_минуты_минут":"минуту_минуты_минут",hh:"час_часа_часов",dd:"день_дня_дней",MM:"месяц_месяца_месяцев",yy:"год_года_лет"}[i].split("_"),n%10==1&&n%100!=11?o[0]:n%10>=2&&n%10<=4&&(n%100<10||n%100>=20)?o[1]:o[2])}var i=[/^янв/i,/^фев/i,/^мар/i,/^апр/i,/^ма[йя]/i,/^июн/i,/^июл/i,/^авг/i,/^сен/i,/^окт/i,/^ноя/i,/^дек/i];t.defineLocale("ru",{months:{format:"января_февраля_марта_апреля_мая_июня_июля_августа_сентября_октября_ноября_декабря".split("_"),standalone:"январь_февраль_март_апрель_май_июнь_июль_август_сентябрь_октябрь_ноябрь_декабрь".split("_")},monthsShort:{format:"янв._февр._мар._апр._мая_июня_июля_авг._сент._окт._нояб._дек.".split("_"),standalone:"янв._февр._март_апр._май_июнь_июль_авг._сент._окт._нояб._дек.".split("_")},weekdays:{standalone:"воскресенье_понедельник_вторник_среда_четверг_пятница_суббота".split("_"),format:"воскресенье_понедельник_вторник_среду_четверг_пятницу_субботу".split("_"),isFormat:/\[ ?[Вв] ?(?:прошлую|следующую|эту)? ?\] ?dddd/},weekdaysShort:"вс_пн_вт_ср_чт_пт_сб".split("_"),weekdaysMin:"вс_пн_вт_ср_чт_пт_сб".split("_"),monthsParse:i,longMonthsParse:i,shortMonthsParse:i,monthsRegex:/^(январ[ья]|янв\.?|феврал[ья]|февр?\.?|марта?|мар\.?|апрел[ья]|апр\.?|ма[йя]|июн[ья]|июн\.?|июл[ья]|июл\.?|августа?|авг\.?|сентябр[ья]|сент?\.?|октябр[ья]|окт\.?|ноябр[ья]|нояб?\.?|декабр[ья]|дек\.?)/i,monthsShortRegex:/^(январ[ья]|янв\.?|феврал[ья]|февр?\.?|марта?|мар\.?|апрел[ья]|апр\.?|ма[йя]|июн[ья]|июн\.?|июл[ья]|июл\.?|августа?|авг\.?|сентябр[ья]|сент?\.?|октябр[ья]|окт\.?|ноябр[ья]|нояб?\.?|декабр[ья]|дек\.?)/i,monthsStrictRegex:/^(январ[яь]|феврал[яь]|марта?|апрел[яь]|ма[яй]|июн[яь]|июл[яь]|августа?|сентябр[яь]|октябр[яь]|ноябр[яь]|декабр[яь])/i,monthsShortStrictRegex:/^(янв\.|февр?\.|мар[т.]|апр\.|ма[яй]|июн[ья.]|июл[ья.]|авг\.|сент?\.|окт\.|нояб?\.|дек\.)/i,longDateFormat:{LT:"H:mm",LTS:"H:mm:ss",L:"DD.MM.YYYY",LL:"D MMMM YYYY г.",LLL:"D MMMM YYYY г., H:mm",LLLL:"dddd, D MMMM YYYY г., H:mm"},calendar:{sameDay:"[Сегодня, в] LT",nextDay:"[Завтра, в] LT",lastDay:"[Вчера, в] LT",nextWeek:function(t){if(t.week()===this.week())return 2===this.day()?"[Во] dddd, [в] LT":"[В] dddd, [в] LT";switch(this.day()){case 0:return"[В следующее] dddd, [в] LT";case 1:case 2:case 4:return"[В следующий] dddd, [в] LT";case 3:case 5:case 6:return"[В следующую] dddd, [в] LT"}},lastWeek:function(t){if(t.week()===this.week())return 2===this.day()?"[Во] dddd, [в] LT":"[В] dddd, [в] LT";switch(this.day()){case 0:return"[В прошлое] dddd, [в] LT";case 1:case 2:case 4:return"[В прошлый] dddd, [в] LT";case 3:case 5:case 6:return"[В прошлую] dddd, [в] LT"}},sameElse:"L"},relativeTime:{future:"через %s",past:"%s назад",s:"несколько секунд",ss:e,m:e,mm:e,h:"час",hh:e,d:"день",dd:e,M:"месяц",MM:e,y:"год",yy:e},meridiemParse:/ночи|утра|дня|вечера/i,isPM:function(t){return/^(дня|вечера)$/.test(t)},meridiem:function(t,e,i){return t<4?"ночи":t<12?"утра":t<17?"дня":"вечера"},dayOfMonthOrdinalParse:/\d{1,2}-(й|го|я)/,ordinal:function(t,e){switch(e){case"M":case"d":case"DDD":return t+"-й";case"D":return t+"-го";case"w":case"W":return t+"-я";default:return t}},week:{dow:1,doy:4}})})(s)})),n((function(t,e){(function(t){function e(t,e,i){var n,o;return"m"===i?e?"хвилина":"хвилину":"h"===i?e?"година":"годину":t+" "+(n=+t,o={ss:e?"секунда_секунди_секунд":"секунду_секунди_секунд",mm:e?"хвилина_хвилини_хвилин":"хвилину_хвилини_хвилин",hh:e?"година_години_годин":"годину_години_годин",dd:"день_дні_днів",MM:"місяць_місяці_місяців",yy:"рік_роки_років"}[i].split("_"),n%10==1&&n%100!=11?o[0]:n%10>=2&&n%10<=4&&(n%100<10||n%100>=20)?o[1]:o[2])}function i(t){return function(){return t+"о"+(11===this.hours()?"б":"")+"] LT"}}t.defineLocale("uk",{months:{format:"січня_лютого_березня_квітня_травня_червня_липня_серпня_вересня_жовтня_листопада_грудня".split("_"),standalone:"січень_лютий_березень_квітень_травень_червень_липень_серпень_вересень_жовтень_листопад_грудень".split("_")},monthsShort:"січ_лют_бер_квіт_трав_черв_лип_серп_вер_жовт_лист_груд".split("_"),weekdays:function(t,e){var i={nominative:"неділя_понеділок_вівторок_середа_четвер_п’ятниця_субота".split("_"),accusative:"неділю_понеділок_вівторок_середу_четвер_п’ятницю_суботу".split("_"),genitive:"неділі_понеділка_вівторка_середи_четверга_п’ятниці_суботи".split("_")};return!0===t?i.nominative.slice(1,7).concat(i.nominative.slice(0,1)):t?i[/(\[[ВвУу]\]) ?dddd/.test(e)?"accusative":/\[?(?:минулої|наступної)? ?\] ?dddd/.test(e)?"genitive":"nominative"][t.day()]:i.nominative},weekdaysShort:"нд_пн_вт_ср_чт_пт_сб".split("_"),weekdaysMin:"нд_пн_вт_ср_чт_пт_сб".split("_"),longDateFormat:{LT:"HH:mm",LTS:"HH:mm:ss",L:"DD.MM.YYYY",LL:"D MMMM YYYY р.",LLL:"D MMMM YYYY р., HH:mm",LLLL:"dddd, D MMMM YYYY р., HH:mm"},calendar:{sameDay:i("[Сьогодні "),nextDay:i("[Завтра "),lastDay:i("[Вчора "),nextWeek:i("[У] dddd ["),lastWeek:function(){switch(this.day()){case 0:case 3:case 5:case 6:return i("[Минулої] dddd [").call(this);case 1:case 2:case 4:return i("[Минулого] dddd [").call(this)}},sameElse:"L"},relativeTime:{future:"за %s",past:"%s тому",s:"декілька секунд",ss:e,m:e,mm:e,h:"годину",hh:e,d:"день",dd:e,M:"місяць",MM:e,y:"рік",yy:e},meridiemParse:/ночі|ранку|дня|вечора/,isPM:function(t){return/^(дня|вечора)$/.test(t)},meridiem:function(t,e,i){return t<4?"ночі":t<12?"ранку":t<17?"дня":"вечора"},dayOfMonthOrdinalParse:/\d{1,2}-(й|го)/,ordinal:function(t,e){switch(e){case"M":case"d":case"DDD":case"w":case"W":return t+"-й";case"D":return t+"-го";default:return t}},week:{dow:1,doy:7}})})(s)})),function(t){return t&&t.Math==Math&&t}),a=r("object"==typeof globalThis&&globalThis)||r("object"==typeof window&&window)||r("object"==typeof self&&self)||r("object"==typeof e&&e)||Function("return this")(),l=function(t){try{return!!t()}catch(t){return!0}},h=!l((function(){return 7!=Object.defineProperty({},1,{get:function(){return 7}})[1]})),d={}.propertyIsEnumerable,u=Object.getOwnPropertyDescriptor,c={f:u&&!d.call({1:2},1)?function(t){var e=u(this,t);return!!e&&e.enumerable}:d},p=function(t,e){return{enumerable:!(1&t),configurable:!(2&t),writable:!(4&t),value:e}},m={}.toString,f=function(t){return m.call(t).slice(8,-1)},g="".split,v=l((function(){return!Object("z").propertyIsEnumerable(0)}))?function(t){return"String"==f(t)?g.call(t,""):Object(t)}:Object,b=function(t){if(null==t)throw TypeError("Can't call method on "+t);return t},y=function(t){return v(b(t))},x=function(t){return"object"==typeof t?null!==t:"function"==typeof t},w=function(t,e){if(!x(t))return t;var i,n;if(e&&"function"==typeof(i=t.toString)&&!x(n=i.call(t)))return n;if("function"==typeof(i=t.valueOf)&&!x(n=i.call(t)))return n;if(!e&&"function"==typeof(i=t.toString)&&!x(n=i.call(t)))return n;throw TypeError("Can't convert object to primitive value")},_={}.hasOwnProperty,k=function(t,e){return _.call(t,e)},D=a.document,I=x(D)&&x(D.createElement),C=function(t){return I?D.createElement(t):{}},L=!h&&!l((function(){return 7!=Object.defineProperty(C("div"),"a",{get:function(){return 7}}).a})),G=Object.getOwnPropertyDescriptor,M={f:h?G:function(t,e){if(t=y(t),e=w(e,!0),L)try{return G(t,e)}catch(t){}if(k(t,e))return p(!c.f.call(t,e),t[e])}},S=/#|\.prototype\./,Q=function(t,e){var i=F[N(t)];return i==B||i!=Z&&("function"==typeof e?l(e):!!e)},N=Q.normalize=function(t){return String(t).replace(S,".").toLowerCase()},F=Q.data={},Z=Q.NATIVE="N",B=Q.POLYFILL="P",O=Q,W={},A=function(t){if("function"!=typeof t)throw TypeError(String(t)+" is not a function");return t},X=function(t,e,i){if(A(t),void 0===e)return t;switch(i){case 0:return function(){return t.call(e)};case 1:return function(i){return t.call(e,i)};case 2:return function(i,n){return t.call(e,i,n)};case 3:return function(i,n,o){return t.call(e,i,n,o)}}return function(){return t.apply(e,arguments)}},T=function(t){if(!x(t))throw TypeError(String(t)+" is not an object");return t},Y=Object.defineProperty,E={f:h?Y:function(t,e,i){if(T(t),e=w(e,!0),T(i),L)try{return Y(t,e,i)}catch(t){}if("get"in i||"set"in i)throw TypeError("Accessors not supported");return"value"in i&&(t[e]=i.value),t}},R=h?function(t,e,i){return E.f(t,e,p(1,i))}:function(t,e,i){return t[e]=i,t},U=M.f,z=function(t){var e=function(e,i,n){if(this instanceof t){switch(arguments.length){case 0:return new t;case 1:return new t(e);case 2:return new t(e,i)}return new t(e,i,n)}return t.apply(this,arguments)};return e.prototype=t.prototype,e},V=function(t,e){var i,n,o,s,r,l,h,d,u=t.target,c=t.global,p=t.stat,m=t.proto,f=c?a:p?a[u]:(a[u]||{}).prototype,g=c?W:W[u]||(W[u]={}),v=g.prototype;for(o in e)i=!O(c?o:u+(p?".":"#")+o,t.forced)&&f&&k(f,o),r=g[o],i&&(l=t.noTargetGet?(d=U(f,o))&&d.value:f[o]),s=i&&l?l:e[o],i&&typeof r==typeof s||(h=t.bind&&i?X(s,a):t.wrap&&i?z(s):m&&"function"==typeof s?X(Function.call,s):s,(t.sham||s&&s.sham||r&&r.sham)&&R(h,"sham",!0),g[o]=h,m&&(k(W,n=u+"Prototype")||R(W,n,{}),W[n][o]=s,t.real&&v&&!v[o]&&R(v,o,s)))};V({target:"Object",stat:!0,forced:!h,sham:!h},{defineProperty:E.f});var j=n((function(t){var e=W.Object,i=t.exports=function(t,i,n){return e.defineProperty(t,i,n)};e.defineProperty.sham&&(i.sham=!0)})),H=j,P=Math.ceil,J=Math.floor,K=function(t){return isNaN(t=+t)?0:(t>0?J:P)(t)},q=Math.min,$=function(t){return t>0?q(K(t),9007199254740991):0},tt=Math.max,et=Math.min,it=function(t,e){var i=K(t);return i<0?tt(i+e,0):et(i,e)},nt=function(t){return function(e,i,n){var o,s=y(e),r=$(s.length),a=it(n,r);if(t&&i!=i){for(;r>a;)if((o=s[a++])!=o)return!0}else for(;r>a;a++)if((t||a in s)&&s[a]===i)return t||a||0;return!t&&-1}},ot={includes:nt(!0),indexOf:nt(!1)},st={},rt=ot.indexOf,at=function(t,e){var i,n=y(t),o=0,s=[];for(i in n)!k(st,i)&&k(n,i)&&s.push(i);for(;e.length>o;)k(n,i=e[o++])&&(~rt(s,i)||s.push(i));return s},lt=["constructor","hasOwnProperty","isPrototypeOf","propertyIsEnumerable","toLocaleString","toString","valueOf"],ht=Object.keys||function(t){return at(t,lt)},dt=h?Object.defineProperties:function(t,e){T(t);for(var i,n=ht(e),o=n.length,s=0;o>s;)E.f(t,i=n[s++],e[i]);return t};V({target:"Object",stat:!0,forced:!h,sham:!h},{defineProperties:dt});var ut=n((function(t){var e=W.Object,i=t.exports=function(t,i){return e.defineProperties(t,i)};e.defineProperties.sham&&(i.sham=!0)})),ct=function(t){return"function"==typeof t?t:void 0},pt=function(t,e){return arguments.length<2?ct(W[t])||ct(a[t]):W[t]&&W[t][e]||a[t]&&a[t][e]},mt=lt.concat("length","prototype"),ft={f:Object.getOwnPropertyNames||function(t){return at(t,mt)}},gt={f:Object.getOwnPropertySymbols},vt=pt("Reflect","ownKeys")||function(t){var e=ft.f(T(t)),i=gt.f;return i?e.concat(i(t)):e},bt=function(t,e,i){var n=w(e);n in t?E.f(t,n,p(0,i)):t[n]=i};V({target:"Object",stat:!0,sham:!h},{getOwnPropertyDescriptors:function(t){for(var e,i,n=y(t),o=M.f,s=vt(n),r={},a=0;s.length>a;)void 0!==(i=o(n,e=s[a++]))&&bt(r,e,i);return r}});var yt=W.Object.getOwnPropertyDescriptors,xt=M.f,wt=l((function(){xt(1)}));V({target:"Object",stat:!0,forced:!h||wt,sham:!h},{getOwnPropertyDescriptor:function(t,e){return xt(y(t),e)}});var _t,kt=n((function(t){var e=W.Object,i=t.exports=function(t,i){return e.getOwnPropertyDescriptor(t,i)};e.getOwnPropertyDescriptor.sham&&(i.sham=!0)})),Dt=!!Object.getOwnPropertySymbols&&!l((function(){return!String(Symbol())})),It=Dt&&!Symbol.sham&&"symbol"==typeof Symbol.iterator,Ct=Array.isArray||function(t){return"Array"==f(t)},Lt=function(t){return Object(b(t))},Gt=pt("document","documentElement"),Mt=a["__core-js_shared__"]||function(t,e){try{R(a,t,e)}catch(i){a[t]=e}return e}("__core-js_shared__",{}),St=n((function(t){(t.exports=function(t,e){return Mt[t]||(Mt[t]=void 0!==e?e:{})})("versions",[]).push({version:"3.6.4",mode:"pure",copyright:"© 2020 Denis Pushkarev (zloirock.ru)"})})),Qt=0,Nt=Math.random(),Ft=function(t){return"Symbol("+String(void 0===t?"":t)+")_"+(++Qt+Nt).toString(36)},Zt=St("keys"),Bt=function(t){return Zt[t]||(Zt[t]=Ft(t))},Ot=Bt("IE_PROTO"),Wt=function(){},At=function(t){return"<script>"+t+"<\/script>"},Xt=function(){try{_t=document.domain&&new ActiveXObject("htmlfile")}catch(t){}var t,e;Xt=_t?function(t){t.write(At("")),t.close();var e=t.parentWindow.Object;return t=null,e}(_t):((e=C("iframe")).style.display="none",Gt.appendChild(e),e.src=String("javascript:"),(t=e.contentWindow.document).open(),t.write(At("document.F=Object")),t.close(),t.F);for(var i=lt.length;i--;)delete Xt.prototype[lt[i]];return Xt()};st[Ot]=!0;var Tt=Object.create||function(t,e){var i;return null!==t?(Wt.prototype=T(t),i=new Wt,Wt.prototype=null,i[Ot]=t):i=Xt(),void 0===e?i:dt(i,e)},Yt=ft.f,Et={}.toString,Rt="object"==typeof window&&window&&Object.getOwnPropertyNames?Object.getOwnPropertyNames(window):[],Ut={f:function(t){return Rt&&"[object Window]"==Et.call(t)?function(t){try{return Yt(t)}catch(t){return Rt.slice()}}(t):Yt(y(t))}},zt=function(t,e,i,n){n&&n.enumerable?t[e]=i:R(t,e,i)},Vt=St("wks"),jt=a.Symbol,Ht=It?jt:jt&&jt.withoutSetter||Ft,Pt=function(t){return k(Vt,t)||(Dt&&k(jt,t)?Vt[t]=jt[t]:Vt[t]=Ht("Symbol."+t)),Vt[t]},Jt={f:Pt},Kt=E.f,qt=function(t){var e=W.Symbol||(W.Symbol={});k(e,t)||Kt(e,t,{value:Jt.f(t)})},$t={};$t[Pt("toStringTag")]="z";var te="[object z]"===String($t),ee=Pt("toStringTag"),ie="Arguments"==f(function(){return arguments}()),ne=te?f:function(t){var e,i,n;return void 0===t?"Undefined":null===t?"Null":"string"==typeof(i=function(t,e){try{return t[e]}catch(t){}}(e=Object(t),ee))?i:ie?f(e):"Object"==(n=f(e))&&"function"==typeof e.callee?"Arguments":n},oe=te?{}.toString:function(){return"[object "+ne(this)+"]"},se=E.f,re=Pt("toStringTag"),ae=function(t,e,i,n){if(t){var o=i?t:t.prototype;k(o,re)||se(o,re,{configurable:!0,value:e}),n&&!te&&R(o,"toString",oe)}},le=Function.toString;"function"!=typeof Mt.inspectSource&&(Mt.inspectSource=function(t){return le.call(t)});var he,de,ue,ce=Mt.inspectSource,pe=a.WeakMap,me="function"==typeof pe&&/native code/.test(ce(pe)),fe=a.WeakMap;if(me){var ge=new fe,ve=ge.get,be=ge.has,ye=ge.set;he=function(t,e){return ye.call(ge,t,e),e},de=function(t){return ve.call(ge,t)||{}},ue=function(t){return be.call(ge,t)}}else{var xe=Bt("state");st[xe]=!0,he=function(t,e){return R(t,xe,e),e},de=function(t){return k(t,xe)?t[xe]:{}},ue=function(t){return k(t,xe)}}var we={set:he,get:de,has:ue,enforce:function(t){return ue(t)?de(t):he(t,{})},getterFor:function(t){return function(e){var i;if(!x(e)||(i=de(e)).type!==t)throw TypeError("Incompatible receiver, "+t+" required");return i}}},_e=Pt("species"),ke=function(t,e){var i;return Ct(t)&&("function"!=typeof(i=t.constructor)||i!==Array&&!Ct(i.prototype)?x(i)&&null===(i=i[_e])&&(i=void 0):i=void 0),new(void 0===i?Array:i)(0===e?0:e)},De=[].push,Ie=function(t){var e=1==t,i=2==t,n=3==t,o=4==t,s=6==t,r=5==t||s;return function(a,l,h,d){for(var u,c,p=Lt(a),m=v(p),f=X(l,h,3),g=$(m.length),b=0,y=d||ke,x=e?y(a,g):i?y(a,0):void 0;g>b;b++)if((r||b in m)&&(c=f(u=m[b],b,p),t))if(e)x[b]=c;else if(c)switch(t){case 3:return!0;case 5:return u;case 6:return b;case 2:De.call(x,u)}else if(o)return!1;return s?-1:n||o?o:x}},Ce={forEach:Ie(0),map:Ie(1),filter:Ie(2),some:Ie(3),every:Ie(4),find:Ie(5),findIndex:Ie(6)},Le=Ce.forEach,Ge=Bt("hidden"),Me=Pt("toPrimitive"),Se=we.set,Qe=we.getterFor("Symbol"),Ne=Object.prototype,Fe=a.Symbol,Ze=pt("JSON","stringify"),Be=M.f,Oe=E.f,We=Ut.f,Ae=c.f,Xe=St("symbols"),Te=St("op-symbols"),Ye=St("string-to-symbol-registry"),Ee=St("symbol-to-string-registry"),Re=St("wks"),Ue=a.QObject,ze=!Ue||!Ue.prototype||!Ue.prototype.findChild,Ve=h&&l((function(){return 7!=Tt(Oe({},"a",{get:function(){return Oe(this,"a",{value:7}).a}})).a}))?function(t,e,i){var n=Be(Ne,e);n&&delete Ne[e],Oe(t,e,i),n&&t!==Ne&&Oe(Ne,e,n)}:Oe,je=function(t,e){var i=Xe[t]=Tt(Fe.prototype);return Se(i,{type:"Symbol",tag:t,description:e}),h||(i.description=e),i},He=It?function(t){return"symbol"==typeof t}:function(t){return Object(t)instanceof Fe},Pe=function(t,e,i){t===Ne&&Pe(Te,e,i),T(t);var n=w(e,!0);return T(i),k(Xe,n)?(i.enumerable?(k(t,Ge)&&t[Ge][n]&&(t[Ge][n]=!1),i=Tt(i,{enumerable:p(0,!1)})):(k(t,Ge)||Oe(t,Ge,p(1,{})),t[Ge][n]=!0),Ve(t,n,i)):Oe(t,n,i)},Je=function(t,e){T(t);var i=y(e),n=ht(i).concat(ti(i));return Le(n,(function(e){h&&!Ke.call(i,e)||Pe(t,e,i[e])})),t},Ke=function(t){var e=w(t,!0),i=Ae.call(this,e);return!(this===Ne&&k(Xe,e)&&!k(Te,e))&&(!(i||!k(this,e)||!k(Xe,e)||k(this,Ge)&&this[Ge][e])||i)},qe=function(t,e){var i=y(t),n=w(e,!0);if(i!==Ne||!k(Xe,n)||k(Te,n)){var o=Be(i,n);return!o||!k(Xe,n)||k(i,Ge)&&i[Ge][n]||(o.enumerable=!0),o}},$e=function(t){var e=We(y(t)),i=[];return Le(e,(function(t){k(Xe,t)||k(st,t)||i.push(t)})),i},ti=function(t){var e=t===Ne,i=We(e?Te:y(t)),n=[];return Le(i,(function(t){!k(Xe,t)||e&&!k(Ne,t)||n.push(Xe[t])})),n};if(Dt||(zt((Fe=function(){if(this instanceof Fe)throw TypeError("Symbol is not a constructor");var t=arguments.length&&void 0!==arguments[0]?String(arguments[0]):void 0,e=Ft(t),i=function(t){this===Ne&&i.call(Te,t),k(this,Ge)&&k(this[Ge],e)&&(this[Ge][e]=!1),Ve(this,e,p(1,t))};return h&&ze&&Ve(Ne,e,{configurable:!0,set:i}),je(e,t)}).prototype,"toString",(function(){return Qe(this).tag})),zt(Fe,"withoutSetter",(function(t){return je(Ft(t),t)})),c.f=Ke,E.f=Pe,M.f=qe,ft.f=Ut.f=$e,gt.f=ti,Jt.f=function(t){return je(Pt(t),t)},h&&Oe(Fe.prototype,"description",{configurable:!0,get:function(){return Qe(this).description}})),V({global:!0,wrap:!0,forced:!Dt,sham:!Dt},{Symbol:Fe}),Le(ht(Re),(function(t){qt(t)})),V({target:"Symbol",stat:!0,forced:!Dt},{for:function(t){var e=String(t);if(k(Ye,e))return Ye[e];var i=Fe(e);return Ye[e]=i,Ee[i]=e,i},keyFor:function(t){if(!He(t))throw TypeError(t+" is not a symbol");if(k(Ee,t))return Ee[t]},useSetter:function(){ze=!0},useSimple:function(){ze=!1}}),V({target:"Object",stat:!0,forced:!Dt,sham:!h},{create:function(t,e){return void 0===e?Tt(t):Je(Tt(t),e)},defineProperty:Pe,defineProperties:Je,getOwnPropertyDescriptor:qe}),V({target:"Object",stat:!0,forced:!Dt},{getOwnPropertyNames:$e,getOwnPropertySymbols:ti}),V({target:"Object",stat:!0,forced:l((function(){gt.f(1)}))},{getOwnPropertySymbols:function(t){return gt.f(Lt(t))}}),Ze){var ei=!Dt||l((function(){var t=Fe();return"[null]"!=Ze([t])||"{}"!=Ze({a:t})||"{}"!=Ze(Object(t))}));V({target:"JSON",stat:!0,forced:ei},{stringify:function(t,e,i){for(var n,o=[t],s=1;arguments.length>s;)o.push(arguments[s++]);if(n=e,(x(e)||void 0!==t)&&!He(t))return Ct(e)||(e=function(t,e){if("function"==typeof n&&(e=n.call(this,t,e)),!He(e))return e}),o[1]=e,Ze.apply(null,o)}})}Fe.prototype[Me]||R(Fe.prototype,Me,Fe.prototype.valueOf),ae(Fe,"Symbol"),st[Ge]=!0;var ii,ni,oi,si=W.Object.getOwnPropertySymbols,ri={},ai=!l((function(){function t(){}return t.prototype.constructor=null,Object.getPrototypeOf(new t)!==t.prototype})),li=Bt("IE_PROTO"),hi=Object.prototype,di=ai?Object.getPrototypeOf:function(t){return t=Lt(t),k(t,li)?t[li]:"function"==typeof t.constructor&&t instanceof t.constructor?t.constructor.prototype:t instanceof Object?hi:null},ui=(Pt("iterator"),!1);[].keys&&("next"in(oi=[].keys())?(ni=di(di(oi)))!==Object.prototype&&(ii=ni):ui=!0),null==ii&&(ii={});var ci={IteratorPrototype:ii,BUGGY_SAFARI_ITERATORS:ui},pi=ci.IteratorPrototype,mi=function(){return this},fi=Object.setPrototypeOf||("__proto__"in{}?function(){var t,e=!1,i={};try{(t=Object.getOwnPropertyDescriptor(Object.prototype,"__proto__").set).call(i,[]),e=i instanceof Array}catch(t){}return function(i,n){return T(i),function(t){if(!x(t)&&null!==t)throw TypeError("Can't set "+String(t)+" as a prototype")}(n),e?t.call(i,n):i.__proto__=n,i}}():void 0),gi=ci.IteratorPrototype,vi=ci.BUGGY_SAFARI_ITERATORS,bi=Pt("iterator"),yi=function(){return this},xi=function(t,e,i,n,o,s,r){!function(t,e,i){var n=e+" Iterator";t.prototype=Tt(pi,{next:p(1,i)}),ae(t,n,!1,!0),ri[n]=mi}(i,e,n);var a,l,h,d=function(t){if(t===o&&g)return g;if(!vi&&t in m)return m[t];switch(t){case"keys":case"values":case"entries":return function(){return new i(this,t)}}return function(){return new i(this)}},u=e+" Iterator",c=!1,m=t.prototype,f=m[bi]||m["@@iterator"]||o&&m[o],g=!vi&&f||d(o),v="Array"==e&&m.entries||f;if(v&&(a=di(v.call(new t)),gi!==Object.prototype&&a.next&&(ae(a,u,!0,!0),ri[u]=yi)),"values"==o&&f&&"values"!==f.name&&(c=!0,g=function(){return f.call(this)}),r&&m[bi]!==g&&R(m,bi,g),ri[e]=g,o)if(l={values:d("values"),keys:s?g:d("keys"),entries:d("entries")},r)for(h in l)!vi&&!c&&h in m||zt(m,h,l[h]);else V({target:e,proto:!0,forced:vi||c},l);return l},wi=we.set,_i=we.getterFor("Array Iterator");xi(Array,"Array",(function(t,e){wi(this,{type:"Array Iterator",target:y(t),index:0,kind:e})}),(function(){var t=_i(this),e=t.target,i=t.kind,n=t.index++;return!e||n>=e.length?(t.target=void 0,{value:void 0,done:!0}):"keys"==i?{value:n,done:!1}:"values"==i?{value:e[n],done:!1}:{value:[n,e[n]],done:!1}}),"values");ri.Arguments=ri.Array;var ki=Pt("toStringTag");for(var Di in{CSSRuleList:0,CSSStyleDeclaration:0,CSSValueList:0,ClientRectList:0,DOMRectList:0,DOMStringList:0,DOMTokenList:1,DataTransferItemList:0,FileList:0,HTMLAllCollection:0,HTMLCollection:0,HTMLFormElement:0,HTMLSelectElement:0,MediaList:0,MimeTypeArray:0,NamedNodeMap:0,NodeList:1,PaintRequestList:0,Plugin:0,PluginArray:0,SVGLengthList:0,SVGNumberList:0,SVGPathSegList:0,SVGPointList:0,SVGStringList:0,SVGTransformList:0,SourceBufferList:0,StyleSheetList:0,TextTrackCueList:0,TextTrackList:0,TouchList:0}){var Ii=a[Di],Ci=Ii&&Ii.prototype;Ci&&ne(Ci)!==ki&&R(Ci,ki,Di),ri[Di]=ri.Array}var Li=function(t){return function(e,i){var n,o,s=String(b(e)),r=K(i),a=s.length;return r<0||r>=a?t?"":void 0:(n=s.charCodeAt(r))<55296||n>56319||r+1===a||(o=s.charCodeAt(r+1))<56320||o>57343?t?s.charAt(r):n:t?s.slice(r,r+2):o-56320+(n-55296<<10)+65536}},Gi={codeAt:Li(!1),charAt:Li(!0)}.charAt,Mi=we.set,Si=we.getterFor("String Iterator");xi(String,"String",(function(t){Mi(this,{type:"String Iterator",string:String(t),index:0})}),(function(){var t,e=Si(this),i=e.string,n=e.index;return n>=i.length?{value:void 0,done:!0}:(t=Gi(i,n),e.index+=t.length,{value:t,done:!1})}));var Qi=Pt("iterator"),Ni=function(t){if(null!=t)return t[Qi]||t["@@iterator"]||ri[ne(t)]},Fi=function(t){var e=Ni(t);if("function"!=typeof e)throw TypeError(String(t)+" is not iterable");return T(e.call(t))};V({target:"Object",stat:!0,sham:!h},{create:Tt});var Zi=W.Object,Bi=function(t,e){return Zi.create(t,e)},Oi=Bi,Wi=j;var Ai=function(t,e,i){return e in t?Wi(t,e,{value:i,enumerable:!0,configurable:!0,writable:!0}):t[e]=i,t},Xi=l((function(){ht(1)}));V({target:"Object",stat:!0,forced:Xi},{keys:function(t){return ht(Lt(t))}});var Ti,Yi=W.Object.keys,Ei="\t\n\v\f\r                　\u2028\u2029\ufeff",Ri="["+Ei+"]",Ui=RegExp("^"+Ri+Ri+"*"),zi=RegExp(Ri+Ri+"*$"),Vi=function(t){return function(e){var i=String(b(e));return 1&t&&(i=i.replace(Ui,"")),2&t&&(i=i.replace(zi,"")),i}},ji={start:Vi(1),end:Vi(2),trim:Vi(3)},Hi=ji.trim;V({target:"String",proto:!0,forced:(Ti="trim",l((function(){return!!Ei[Ti]()||"​…᠎"!="​…᠎"[Ti]()||Ei[Ti].name!==Ti})))},{trim:function(){return Hi(this)}});var Pi=function(t){return W[t+"Prototype"]},Ji=Pi("String").trim,Ki=String.prototype,qi=function(t){var e=t.trim;return"string"==typeof t||t===Ki||t instanceof String&&e===Ki.trim?Ji:e},$i=function(t,e){var i=[][t];return!!i&&l((function(){i.call(null,e||function(){throw 1},1)}))},tn=Object.defineProperty,en={},nn=function(t){throw t},on=function(t,e){if(k(en,t))return en[t];e||(e={});var i=[][t],n=!!k(e,"ACCESSORS")&&e.ACCESSORS,o=k(e,0)?e[0]:nn,s=k(e,1)?e[1]:void 0;return en[t]=!!i&&!l((function(){if(n&&!h)return!0;var t={length:-1};n?tn(t,1,{enumerable:!0,get:nn}):t[1]=1,i.call(t,o,s)}))},sn=Ce.forEach,rn=$i("forEach"),an=on("forEach"),ln=rn&&an?[].forEach:function(t){return sn(this,t,arguments.length>1?arguments[1]:void 0)};V({target:"Array",proto:!0,forced:[].forEach!=ln},{forEach:ln});var hn,dn,un=Pi("Array").forEach,cn=Array.prototype,pn={DOMTokenList:!0,NodeList:!0},mn=function(t){var e=t.forEach;return t===cn||t instanceof Array&&e===cn.forEach||pn.hasOwnProperty(ne(t))?un:e},fn=pt("navigator","userAgent")||"",gn=a.process,vn=gn&&gn.versions,bn=vn&&vn.v8;bn?dn=(hn=bn.split("."))[0]+hn[1]:fn&&(!(hn=fn.match(/Edge\/(\d+)/))||hn[1]>=74)&&(hn=fn.match(/Chrome\/(\d+)/))&&(dn=hn[1]);var yn=dn&&+dn,xn=Pt("species"),wn=function(t){return yn>=51||!l((function(){var e=[];return(e.constructor={})[xn]=function(){return{foo:1}},1!==e[t](Boolean).foo}))},_n=Ce.map,kn=wn("map"),Dn=on("map");V({target:"Array",proto:!0,forced:!kn||!Dn},{map:function(t){return _n(this,t,arguments.length>1?arguments[1]:void 0)}});var In=Pi("Array").map,Cn=Array.prototype,Ln=function(t){var e=t.map;return t===Cn||t instanceof Array&&e===Cn.map?In:e},Gn=ji.trim,Mn=a.parseInt,Sn=/^[+-]?0[Xx]/,Qn=8!==Mn(Ei+"08")||22!==Mn(Ei+"0x16")?function(t,e){var i=Gn(String(t));return Mn(i,e>>>0||(Sn.test(i)?16:10))}:Mn;V({global:!0,forced:parseInt!=Qn},{parseInt:Qn});var Nn=W.parseInt,Fn=c.f,Zn=function(t){return function(e){for(var i,n=y(e),o=ht(n),s=o.length,r=0,a=[];s>r;)i=o[r++],h&&!Fn.call(n,i)||a.push(t?[i,n[i]]:n[i]);return a}},Bn={entries:Zn(!0),values:Zn(!1)}.values;V({target:"Object",stat:!0},{values:function(t){return Bn(t)}});var On=W.Object.values,Wn=Ce.filter,An=wn("filter"),Xn=on("filter");V({target:"Array",proto:!0,forced:!An||!Xn},{filter:function(t){return Wn(this,t,arguments.length>1?arguments[1]:void 0)}});var Tn=Pi("Array").filter,Yn=Array.prototype,En=function(t){var e=t.filter;return t===Yn||t instanceof Array&&e===Yn.filter?Tn:e},Rn=Pt("isConcatSpreadable"),Un=yn>=51||!l((function(){var t=[];return t[Rn]=!1,t.concat()[0]!==t})),zn=wn("concat"),Vn=function(t){if(!x(t))return!1;var e=t[Rn];return void 0!==e?!!e:Ct(t)};V({target:"Array",proto:!0,forced:!Un||!zn},{concat:function(t){var e,i,n,o,s,r=Lt(this),a=ke(r,0),l=0;for(e=-1,n=arguments.length;e<n;e++)if(s=-1===e?r:arguments[e],Vn(s)){if(l+(o=$(s.length))>9007199254740991)throw TypeError("Maximum allowed index exceeded");for(i=0;i<o;i++,l++)i in s&&bt(a,l,s[i])}else{if(l>=9007199254740991)throw TypeError("Maximum allowed index exceeded");bt(a,l++,s)}return a.length=l,a}});var jn=Pi("Array").concat,Hn=Array.prototype,Pn=function(t){var e=t.concat;return t===Hn||t instanceof Array&&e===Hn.concat?jn:e};V({target:"Array",stat:!0},{isArray:Ct});var Jn=W.Array.isArray,Kn=Jn;var qn=function(t){if(Kn(t)){for(var e=0,i=new Array(t.length);e<t.length;e++)i[e]=t[e];return i}},$n=function(t,e,i,n){try{return n?e(T(i)[0],i[1]):e(i)}catch(e){var o=t.return;throw void 0!==o&&T(o.call(t)),e}},to=Pt("iterator"),eo=Array.prototype,io=function(t){return void 0!==t&&(ri.Array===t||eo[to]===t)},no=Pt("iterator"),oo=!1;try{var so=0,ro={next:function(){return{done:!!so++}},return:function(){oo=!0}};ro[no]=function(){return this},Array.from(ro,(function(){throw 2}))}catch(t){}var ao=!function(t,e){if(!e&&!oo)return!1;var i=!1;try{var n={};n[no]=function(){return{next:function(){return{done:i=!0}}}},t(n)}catch(t){}return i}((function(t){Array.from(t)}));V({target:"Array",stat:!0,forced:ao},{from:function(t){var e,i,n,o,s,r,a=Lt(t),l="function"==typeof this?this:Array,h=arguments.length,d=h>1?arguments[1]:void 0,u=void 0!==d,c=Ni(a),p=0;if(u&&(d=X(d,h>2?arguments[2]:void 0,2)),null==c||l==Array&&io(c))for(i=new l(e=$(a.length));e>p;p++)r=u?d(a[p],p):a[p],bt(i,p,r);else for(s=(o=c.call(a)).next,i=new l;!(n=s.call(o)).done;p++)r=u?$n(o,d,[n.value,p],!0):n.value,bt(i,p,r);return i.length=p,i}});var lo=W.Array.from,ho=Pt("iterator"),uo=function(t){var e=Object(t);return void 0!==e[ho]||"@@iterator"in e||ri.hasOwnProperty(ne(e))};var co=function(t){if(uo(Object(t))||"[object Arguments]"===Object.prototype.toString.call(t))return lo(t)};var po=function(){throw new TypeError("Invalid attempt to spread non-iterable instance")};var mo=function(t){return qn(t)||co(t)||po()},fo=wn("slice"),go=on("slice",{ACCESSORS:!0,0:0,1:2}),vo=Pt("species"),bo=[].slice,yo=Math.max;V({target:"Array",proto:!0,forced:!fo||!go},{slice:function(t,e){var i,n,o,s=y(this),r=$(s.length),a=it(t,r),l=it(void 0===e?r:e,r);if(Ct(s)&&("function"!=typeof(i=s.constructor)||i!==Array&&!Ct(i.prototype)?x(i)&&null===(i=i[vo])&&(i=void 0):i=void 0,i===Array||void 0===i))return bo.call(s,a,l);for(n=new(void 0===i?Array:i)(yo(l-a,0)),o=0;a<l;a++,o++)a in s&&bt(n,o,s[a]);return n.length=o,n}});var xo=Pi("Array").slice,wo=Array.prototype,_o=function(t){var e=t.slice;return t===wo||t instanceof Array&&e===wo.slice?xo:e},ko=l((function(){di(1)}));V({target:"Object",stat:!0,forced:ko,sham:!ai},{getPrototypeOf:function(t){return di(Lt(t))}});var Do=W.Object.getPrototypeOf,Io=Do,Co=ot.indexOf,Lo=[].indexOf,Go=!!Lo&&1/[1].indexOf(1,-0)<0,Mo=$i("indexOf"),So=on("indexOf",{ACCESSORS:!0,1:0});V({target:"Array",proto:!0,forced:Go||!Mo||!So},{indexOf:function(t){return Go?Lo.apply(this,arguments)||0:Co(this,t,arguments.length>1?arguments[1]:void 0)}});var Qo=Pi("Array").indexOf,No=Array.prototype,Fo=function(t){var e=t.indexOf;return t===No||t instanceof Array&&e===No.indexOf?Qo:e},Zo=Jn,Bo=Object.assign,Oo=Object.defineProperty,Wo=!Bo||l((function(){if(h&&1!==Bo({b:1},Bo(Oo({},"a",{enumerable:!0,get:function(){Oo(this,"b",{value:3,enumerable:!1})}}),{b:2})).b)return!0;var t={},e={},i=Symbol();return t[i]=7,"abcdefghijklmnopqrst".split("").forEach((function(t){e[t]=t})),7!=Bo({},t)[i]||"abcdefghijklmnopqrst"!=ht(Bo({},e)).join("")}))?function(t,e){for(var i=Lt(t),n=arguments.length,o=1,s=gt.f,r=c.f;n>o;)for(var a,l=v(arguments[o++]),d=s?ht(l).concat(s(l)):ht(l),u=d.length,p=0;u>p;)a=d[p++],h&&!r.call(l,a)||(i[a]=l[a]);return i}:Bo;V({target:"Object",stat:!0,forced:Object.assign!==Wo},{assign:Wo});var Ao=W.Object.assign;qt("iterator");var Xo=Jt.f("iterator"),To=Xo,Yo=Object.freeze({__proto__:null});qt("asyncIterator");var Eo=Object.freeze({__proto__:null});qt("hasInstance"),qt("isConcatSpreadable"),qt("match"),qt("matchAll"),qt("replace"),qt("search"),qt("species"),qt("split"),qt("toPrimitive"),qt("toStringTag"),qt("unscopables"),ae(Math,"Math",!0),ae(a.JSON,"JSON",!0),o(Yo),o(Eo);var Ro=W.Symbol;qt("asyncDispose"),qt("dispose"),qt("observable"),qt("patternMatch"),qt("replaceAll");var Uo=Ro,zo=n((function(t){function e(i){return t.exports=e="function"==typeof Uo&&"symbol"==typeof To?function(t){return typeof t}:function(t){return t&&"function"==typeof Uo&&t.constructor===Uo&&t!==Uo.prototype?"symbol":typeof t},e(i)}t.exports=e}));var Vo=function(t){if(Kn(t))return t};var jo=function(t,e){if(uo(Object(t))||"[object Arguments]"===Object.prototype.toString.call(t)){var i=[],n=!0,o=!1,s=void 0;try{for(var r,a=Fi(t);!(n=(r=a.next()).done)&&(i.push(r.value),!e||i.length!==e);n=!0);}catch(t){o=!0,s=t}finally{try{n||null==a.return||a.return()}finally{if(o)throw s}}return i}};var Ho=function(){throw new TypeError("Invalid attempt to destructure non-iterable instance")};var Po=function(t,e){return Vo(t)||jo(t,e)||Ho()};V({target:"Date",stat:!0},{now:function(){return(new Date).getTime()}});var Jo=W.Date.now;function Ko(t,e){var i=Yi(t);if(si){var n=si(t);e&&(n=En(n).call(n,(function(e){return kt(t,e).enumerable}))),i.push.apply(i,n)}return i}function qo(t){var e=function(){for(var t=$o(),e=t(" "),i=t(" "),n=t(" "),o=0;o<arguments.length;o++)(e-=t(o<0||arguments.length<=o?void 0:arguments[o]))<0&&(e+=1),(i-=t(o<0||arguments.length<=o?void 0:arguments[o]))<0&&(i+=1),(n-=t(o<0||arguments.length<=o?void 0:arguments[o]))<0&&(n+=1);return[e,i,n]}(t),i=Po(e,3),n=i[0],o=i[1],s=i[2],r=1,a=function(){var t=2091639*n+2.3283064365386963e-10*r;return n=o,o=s,s=t-(r=0|t)};return a.uint32=function(){return 4294967296*a()},a.fract53=function(){return a()+11102230246251565e-32*(2097152*a()|0)},a.algorithm="Alea",a.seed=t,a.version="0.9",a}function $o(){var t=4022871197;return function(e){for(var i=e.toString(),n=0;n<i.length;n++){var o=.02519603282416938*(t+=i.charCodeAt(n));o-=t=o>>>0,t=(o*=t)>>>0,t+=4294967296*(o-=t)}return 2.3283064365386963e-10*(t>>>0)}}var ts=/^\/?Date\((-?\d+)/i,es=/^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i,is=/^#?([a-f\d])([a-f\d])([a-f\d])$/i,ns=/^rgb\( *(1?\d{1,2}|2[0-4]\d|25[0-5]) *, *(1?\d{1,2}|2[0-4]\d|25[0-5]) *, *(1?\d{1,2}|2[0-4]\d|25[0-5]) *\)$/i,os=/^rgba\( *(1?\d{1,2}|2[0-4]\d|25[0-5]) *, *(1?\d{1,2}|2[0-4]\d|25[0-5]) *, *(1?\d{1,2}|2[0-4]\d|25[0-5]) *, *([01]|0?\.\d+) *\)$/i;function ss(t){return t instanceof Number||"number"==typeof t}function rs(t){return t instanceof String||"string"==typeof t}function as(t){return"object"===zo(t)&&null!==t}function ls(t,e,i,n){var o=!1;!0===n&&(o=null===e[i]&&void 0!==t[i]),o?delete t[i]:t[i]=e[i]}var hs=Ao;function ds(t,e){var i=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=arguments.length>3&&void 0!==arguments[3]&&arguments[3];for(var o in e)if(Object.prototype.hasOwnProperty.call(e,o)||!0===i)if("object"===zo(e[o])&&null!==e[o]&&Io(e[o])===Object.prototype)void 0===t[o]?t[o]=ds({},e[o],i):"object"===zo(t[o])&&null!==t[o]&&Io(t[o])===Object.prototype?ds(t[o],e[o],i):ls(t,e,o,n);else if(Zo(e[o])){var s;t[o]=_o(s=e[o]).call(s)}else ls(t,e,o,n);return t}function us(t){var e=zo(t);return"object"===e?null===t?"null":t instanceof Boolean?"Boolean":t instanceof Number?"Number":t instanceof String?"String":Zo(t)?"Array":t instanceof Date?"Date":"Object":"number"===e?"Number":"boolean"===e?"Boolean":"string"===e?"String":void 0===e?"undefined":e}var cs=On;var ps={asBoolean:function(t,e){return"function"==typeof t&&(t=t()),null!=t?0!=t:e||null},asNumber:function(t,e){return"function"==typeof t&&(t=t()),null!=t?Number(t)||e||null:e||null},asString:function(t,e){return"function"==typeof t&&(t=t()),null!=t?String(t):e||null},asSize:function(t,e){return"function"==typeof t&&(t=t()),rs(t)?t:ss(t)?t+"px":e||null},asElement:function(t,e){return"function"==typeof t&&(t=t()),t||e||null}};function ms(t){var e;switch(t.length){case 3:case 4:return(e=is.exec(t))?{r:Nn(e[1]+e[1],16),g:Nn(e[2]+e[2],16),b:Nn(e[3]+e[3],16)}:null;case 6:case 7:return(e=es.exec(t))?{r:Nn(e[1],16),g:Nn(e[2],16),b:Nn(e[3],16)}:null;default:return null}}function fs(t,e,i){var n;return"#"+_o(n=((1<<24)+(t<<16)+(e<<8)+i).toString(16)).call(n,1)}function gs(t,e,i){t/=255,e/=255,i/=255;var n=Math.min(t,Math.min(e,i)),o=Math.max(t,Math.max(e,i));return n===o?{h:0,s:0,v:n}:{h:60*((t===n?3:i===n?1:5)-(t===n?e-i:i===n?t-e:i-t)/(o-n))/360,s:(o-n)/o,v:o}}var vs=function(t){var e,i={};return mn(e=t.split(";")).call(e,(function(t){if(""!=qi(t).call(t)){var e,n,o=t.split(":"),s=qi(e=o[0]).call(e),r=qi(n=o[1]).call(n);i[s]=r}})),i},bs=function(t){var e;return Ln(e=Yi(t)).call(e,(function(e){return e+": "+t[e]})).join("; ")};function ys(t,e,i){var n,o,s,r=Math.floor(6*t),a=6*t-r,l=i*(1-e),h=i*(1-a*e),d=i*(1-(1-a)*e);switch(r%6){case 0:n=i,o=d,s=l;break;case 1:n=h,o=i,s=l;break;case 2:n=l,o=i,s=d;break;case 3:n=l,o=h,s=i;break;case 4:n=d,o=l,s=i;break;case 5:n=i,o=l,s=h}return{r:Math.floor(255*n),g:Math.floor(255*o),b:Math.floor(255*s)}}function xs(t,e,i){var n=ys(t,e,i);return fs(n.r,n.g,n.b)}function ws(t){var e=ms(t);if(!e)throw new TypeError("'".concat(t,"' is not a valid color."));return gs(e.r,e.g,e.b)}function _s(t){return/(^#[0-9A-F]{6}$)|(^#[0-9A-F]{3}$)/i.test(t)}function ks(t){return ns.test(t)}function Ds(t){if(null===t||"object"!==zo(t))return null;if(t instanceof Element)return t;var e=Oi(t);for(var i in t)Object.prototype.hasOwnProperty.call(t,i)&&"object"==zo(t[i])&&(e[i]=Ds(t[i]));return e}var Is=Object.freeze({__proto__:null,Alea:
-/**
-	 * vis-util
-	 * https://github.com/visjs/vis-util
-	 *
-	 * utilitie collection for visjs
-	 *
-	 * @version 4.0.0
-	 * @date    2020-02-29T15:35:13.178Z
-	 *
-	 * @copyright (c) 2011-2017 Almende B.V, http://almende.com
-	 * @copyright (c) 2017-2019 visjs contributors, https://github.com/visjs
-	 *
-	 * @license
-	 * vis.js is dual licensed under both
-	 *
-	 *   1. The Apache 2.0 License
-	 *      http://www.apache.org/licenses/LICENSE-2.0
-	 *
-	 *   and
-	 *
-	 *   2. The MIT License
-	 *      http://opensource.org/licenses/MIT
-	 *
-	 * vis.js may be distributed under either license.
-	 */
-function(){for(var t=arguments.length,e=new Array(t),i=0;i<t;i++)e[i]=arguments[i];return qo(e.length?e:[Jo()])},HSVToHex:xs,HSVToRGB:ys,RGBToHSV:gs,RGBToHex:fs,addClassName:function(t,e){var i=t.className.split(" "),n=e.split(" ");i=Pn(i).call(i,En(n).call(n,(function(t){return Fo(i).call(i,t)<0}))),t.className=i.join(" ")},addCssText:function(t,e){var i=function(t){for(var e=1;e<arguments.length;e++){var i,n=null!=arguments[e]?arguments[e]:{};if(e%2)mn(i=Ko(Object(n),!0)).call(i,(function(e){Ai(t,e,n[e])}));else if(yt)ut(t,yt(n));else{var o;mn(o=Ko(Object(n))).call(o,(function(e){H(t,e,kt(n,e))}))}}return t}({},vs(t.style.cssText),{},vs(e));t.style.cssText=bs(i)},addEventListener:function(t,e,i,n){var o;t.addEventListener?(void 0===n&&(n=!1),"mousewheel"===e&&Fo(o=navigator.userAgent).call(o,"Firefox")>=0&&(e="DOMMouseScroll"),t.addEventListener(e,i,n)):t.attachEvent("on"+e,i)},binarySearchCustom:function(t,e,i,n){for(var o=0,s=0,r=t.length-1;s<=r&&o<1e4;){var a=Math.floor((s+r)/2),l=t[a],h=e(void 0===n?l[i]:l[i][n]);if(0==h)return a;-1==h?s=a+1:r=a-1,o++}return-1},binarySearchValue:function(t,e,i,n,o){var s,r,a,l,h=0,d=0,u=t.length-1;for(o=null!=o?o:function(t,e){return t==e?0:t<e?-1:1};d<=u&&h<1e4;){if(l=Math.floor(.5*(u+d)),s=t[Math.max(0,l-1)][i],r=t[l][i],a=t[Math.min(t.length-1,l+1)][i],0==o(r,e))return l;if(o(s,e)<0&&o(r,e)>0)return"before"==n?Math.max(0,l-1):l;if(o(r,e)<0&&o(a,e)>0)return"before"==n?l:Math.min(t.length-1,l+1);o(r,e)<0?d=l+1:u=l-1,h++}return-1},bridgeObject:Ds,copyAndExtendArray:function(t,e){var i;return Pn(i=[]).call(i,mo(t),[e])},copyArray:function(t){return _o(t).call(t)},deepExtend:ds,easingFunctions:{linear:function(t){return t},easeInQuad:function(t){return t*t},easeOutQuad:function(t){return t*(2-t)},easeInOutQuad:function(t){return t<.5?2*t*t:(4-2*t)*t-1},easeInCubic:function(t){return t*t*t},easeOutCubic:function(t){return--t*t*t+1},easeInOutCubic:function(t){return t<.5?4*t*t*t:(t-1)*(2*t-2)*(2*t-2)+1},easeInQuart:function(t){return t*t*t*t},easeOutQuart:function(t){return 1- --t*t*t*t},easeInOutQuart:function(t){return t<.5?8*t*t*t*t:1-8*--t*t*t*t},easeInQuint:function(t){return t*t*t*t*t},easeOutQuint:function(t){return 1+--t*t*t*t*t},easeInOutQuint:function(t){return t<.5?16*t*t*t*t*t:1+16*--t*t*t*t*t}},equalArray:function(t,e){if(t.length!==e.length)return!1;for(var i=0,n=t.length;i<n;i++)if(t[i]!=e[i])return!1;return!0},extend:hs,fillIfDefined:function t(e,i){var n=arguments.length>2&&void 0!==arguments[2]&&arguments[2];for(var o in e)if(void 0!==i[o])if(null===i[o]||"object"!==zo(i[o]))ls(e,i,o,n);else{var s=e[o],r=i[o];as(s)&&as(r)&&t(s,r,n)}},forEach:function(t,e){if(Zo(t))for(var i=t.length,n=0;n<i;n++)e(t[n],n,t);else for(var o in t)Object.prototype.hasOwnProperty.call(t,o)&&e(t[o],o,t)},getAbsoluteLeft:function(t){return t.getBoundingClientRect().left},getAbsoluteRight:function(t){return t.getBoundingClientRect().right},getAbsoluteTop:function(t){return t.getBoundingClientRect().top},getScrollBarWidth:function(){var t=document.createElement("p");t.style.width="100%",t.style.height="200px";var e=document.createElement("div");e.style.position="absolute",e.style.top="0px",e.style.left="0px",e.style.visibility="hidden",e.style.width="200px",e.style.height="150px",e.style.overflow="hidden",e.appendChild(t),document.body.appendChild(e);var i=t.offsetWidth;e.style.overflow="scroll";var n=t.offsetWidth;return i==n&&(n=e.clientWidth),document.body.removeChild(e),i-n},getTarget:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:window.event,e=null;return t&&(t.target?e=t.target:t.srcElement&&(e=t.srcElement)),e instanceof Element&&(null==e.nodeType||3!=e.nodeType||(e=e.parentNode)instanceof Element)?e:null},getType:us,hasParent:function(t,e){for(var i=t;i;){if(i===e)return!0;if(!i.parentNode)return!1;i=i.parentNode}return!1},hexToHSV:ws,hexToRGB:ms,insertSort:function(t,e){for(var i=0;i<t.length;i++){var n=t[i],o=void 0;for(o=i;o>0&&e(n,t[o-1])<0;o--)t[o]=t[o-1];t[o]=n}return t},isDate:function(t){if(t instanceof Date)return!0;if(rs(t)){if(ts.exec(t))return!0;if(!isNaN(Date.parse(t)))return!0}return!1},isNumber:ss,isObject:as,isString:rs,isValidHex:_s,isValidRGB:ks,isValidRGBA:function(t){return os.test(t)},mergeOptions:function(t,e,i){var n=arguments.length>3&&void 0!==arguments[3]?arguments[3]:{},o=function(t){return null!=t},s=function(t){return null!==t&&"object"===zo(t)},r=function(t){for(var e in t)if(Object.prototype.hasOwnProperty.call(t,e))return!1;return!0};if(!s(t))throw new Error("Parameter mergeTarget must be an object");if(!s(e))throw new Error("Parameter options must be an object");if(!o(i))throw new Error("Parameter option must have a value");if(!s(n))throw new Error("Parameter globalOptions must be an object");var a=function(t,e,i){s(t[i])||(t[i]={});var n=e[i],o=t[i];for(var r in n)Object.prototype.hasOwnProperty.call(n,r)&&(o[r]=n[r])},l=e[i],h=s(n)&&!r(n),d=h?n[i]:void 0,u=d?d.enabled:void 0;if(void 0!==l){if("boolean"==typeof l)return s(t[i])||(t[i]={}),void(t[i].enabled=l);if(null===l&&!s(t[i])){if(!o(d))return;t[i]=Oi(d)}if(s(l)){var c=!0;void 0!==l.enabled?c=l.enabled:void 0!==u&&(c=d.enabled),a(t,e,i),t[i].enabled=c}}},option:ps,overrideOpacity:function(t,e){if(-1!==Fo(t).call(t,"rgba"))return t;if(-1!==Fo(t).call(t,"rgb")){var i=t.substr(Fo(t).call(t,"(")+1).replace(")","").split(",");return"rgba("+i[0]+","+i[1]+","+i[2]+","+e+")"}var n=ms(t);return null==n?t:"rgba("+n.r+","+n.g+","+n.b+","+e+")"},parseColor:function(t,e){if(rs(t)){var i=t;if(ks(i)){var n,o=Ln(n=i.substr(4).substr(0,i.length-5).split(",")).call(n,(function(t){return Nn(t)}));i=fs(o[0],o[1],o[2])}if(!0===_s(i)){var s=ws(i),r={h:s.h,s:.8*s.s,v:Math.min(1,1.02*s.v)},a={h:s.h,s:Math.min(1,1.25*s.s),v:.8*s.v},l=xs(a.h,a.s,a.v),h=xs(r.h,r.s,r.v);return{background:i,border:l,highlight:{background:h,border:l},hover:{background:h,border:l}}}return{background:i,border:i,highlight:{background:i,border:i},hover:{background:i,border:i}}}return e?{background:t.background||e.background,border:t.border||e.border,highlight:rs(t.highlight)?{border:t.highlight,background:t.highlight}:{background:t.highlight&&t.highlight.background||e.highlight.background,border:t.highlight&&t.highlight.border||e.highlight.border},hover:rs(t.hover)?{border:t.hover,background:t.hover}:{border:t.hover&&t.hover.border||e.hover.border,background:t.hover&&t.hover.background||e.hover.background}}:{background:t.background||void 0,border:t.border||void 0,highlight:rs(t.highlight)?{border:t.highlight,background:t.highlight}:{background:t.highlight&&t.highlight.background||void 0,border:t.highlight&&t.highlight.border||void 0},hover:rs(t.hover)?{border:t.hover,background:t.hover}:{border:t.hover&&t.hover.border||void 0,background:t.hover&&t.hover.background||void 0}}},preventDefault:function(t){t||(t=window.event),t&&(t.preventDefault?t.preventDefault():t.returnValue=!1)},recursiveDOMDelete:function t(e){if(e)for(;!0===e.hasChildNodes();){var i=e.firstChild;i&&(t(i),e.removeChild(i))}},removeClassName:function(t,e){var i=t.className.split(" "),n=e.split(" ");i=En(i).call(i,(function(t){return Fo(n).call(n,t)<0})),t.className=i.join(" ")},removeCssText:function(t,e){var i=vs(t.style.cssText),n=vs(e);for(var o in n)Object.prototype.hasOwnProperty.call(n,o)&&delete i[o];t.style.cssText=bs(i)},removeEventListener:function(t,e,i,n){var o;t.removeEventListener?(void 0===n&&(n=!1),"mousewheel"===e&&Fo(o=navigator.userAgent).call(o,"Firefox")>=0&&(e="DOMMouseScroll"),t.removeEventListener(e,i,n)):t.detachEvent("on"+e,i)},selectiveBridgeObject:function(t,e){if(null!==e&&"object"===zo(e)){for(var i=Oi(e),n=0;n<t.length;n++)Object.prototype.hasOwnProperty.call(e,t[n])&&"object"==zo(e[t[n]])&&(i[t[n]]=Ds(e[t[n]]));return i}return null},selectiveDeepExtend:function(t,e,i){var n=arguments.length>3&&void 0!==arguments[3]&&arguments[3];if(Zo(i))throw new TypeError("Arrays are not supported by deepExtend");for(var o=0;o<t.length;o++){var s=t[o];if(Object.prototype.hasOwnProperty.call(i,s))if(i[s]&&i[s].constructor===Object)void 0===e[s]&&(e[s]={}),e[s].constructor===Object?ds(e[s],i[s],!1,n):ls(e,i,s,n);else{if(Zo(i[s]))throw new TypeError("Arrays are not supported by deepExtend");ls(e,i,s,n)}}return e},selectiveExtend:function(t,e){if(!Zo(t))throw new Error("Array with property names expected as first argument");for(var i=arguments.length,n=new Array(i>2?i-2:0),o=2;o<i;o++)n[o-2]=arguments[o];for(var s=0,r=n;s<r.length;s++)for(var a=r[s],l=0;l<t.length;l++){var h=t[l];a&&Object.prototype.hasOwnProperty.call(a,h)&&(e[h]=a[h])}return e},selectiveNotDeepExtend:function(t,e,i){var n=arguments.length>3&&void 0!==arguments[3]&&arguments[3];if(Zo(i))throw new TypeError("Arrays are not supported by deepExtend");for(var o in i)if(Object.prototype.hasOwnProperty.call(i,o)&&-1===Fo(t).call(t,o))if(i[o]&&i[o].constructor===Object)void 0===e[o]&&(e[o]={}),e[o].constructor===Object?ds(e[o],i[o]):ls(e,i,o,n);else if(Zo(i[o])){e[o]=[];for(var s=0;s<i[o].length;s++)e[o].push(i[o][s])}else ls(e,i,o,n);return e},throttle:function(t){var e=!1;return function(){e||(e=!0,requestAnimationFrame((function(){e=!1,t()})))}},toArray:cs,topMost:function(t,e){var i;Zo(e)||(e=[e]);var n=!0,o=!1,s=void 0;try{for(var r,a=Fi(t);!(n=(r=a.next()).done);n=!0){var l=r.value;if(l){i=l[e[0]];for(var h=1;h<e.length;h++)i&&(i=i[e[h]]);if(void 0!==i)break}}}catch(t){o=!0,s=t}finally{try{n||null==a.return||a.return()}finally{if(o)throw s}}return i},updateProperty:function(t,e,i){return t[e]!==i&&(t[e]=i,!0)}});function Cs(t){for(var e in t)t.hasOwnProperty(e)&&(t[e].redundant=t[e].used,t[e].used=[])}function Ls(t){for(var e in t)if(t.hasOwnProperty(e)&&t[e].redundant){for(var i=0;i<t[e].redundant.length;i++)t[e].redundant[i].parentNode.removeChild(t[e].redundant[i]);t[e].redundant=[]}}function Gs(t){Cs(t),Ls(t),Cs(t)}function Ms(t,e,i){var n;return e.hasOwnProperty(t)?e[t].redundant.length>0?(n=e[t].redundant[0],e[t].redundant.shift()):(n=document.createElementNS("http://www.w3.org/2000/svg",t),i.appendChild(n)):(n=document.createElementNS("http://www.w3.org/2000/svg",t),e[t]={used:[],redundant:[]},i.appendChild(n)),e[t].used.push(n),n}function Ss(t,e,i,n){var o;return e.hasOwnProperty(t)?e[t].redundant.length>0?(o=e[t].redundant[0],e[t].redundant.shift()):(o=document.createElement(t),void 0!==n?i.insertBefore(o,n):i.appendChild(o)):(o=document.createElement(t),e[t]={used:[],redundant:[]},void 0!==n?i.insertBefore(o,n):i.appendChild(o)),e[t].used.push(o),o}function Qs(t,e,i,n,o,s){var r;if("circle"==i.style?((r=Ms("circle",n,o)).setAttributeNS(null,"cx",t),r.setAttributeNS(null,"cy",e),r.setAttributeNS(null,"r",.5*i.size)):((r=Ms("rect",n,o)).setAttributeNS(null,"x",t-.5*i.size),r.setAttributeNS(null,"y",e-.5*i.size),r.setAttributeNS(null,"width",i.size),r.setAttributeNS(null,"height",i.size)),void 0!==i.styles&&r.setAttributeNS(null,"style",i.styles),r.setAttributeNS(null,"class",i.className+" vis-point"),s){var a=Ms("text",n,o);s.xOffset&&(t+=s.xOffset),s.yOffset&&(e+=s.yOffset),s.content&&(a.textContent=s.content),s.className&&a.setAttributeNS(null,"class",s.className+" vis-label"),a.setAttributeNS(null,"x",t),a.setAttributeNS(null,"y",e)}return r}function Ns(t,e,i,n,o,s,r,a){if(0!=n){n<0&&(e-=n*=-1);var l=Ms("rect",s,r);l.setAttributeNS(null,"x",t-.5*i),l.setAttributeNS(null,"y",e),l.setAttributeNS(null,"width",i),l.setAttributeNS(null,"height",n),l.setAttributeNS(null,"class",o),a&&l.setAttributeNS(null,"style",a)}}function Fs(){try{return navigator?navigator.languages&&navigator.languages.length?navigator.languages:navigator.userLanguage||navigator.language||navigator.browserLanguage||"en":"en"}catch(t){return"en"}}var Zs=Object.freeze({__proto__:null,prepareElements:Cs,cleanupElements:Ls,resetElements:Gs,getSVGElement:Ms,getDOMElement:Ss,drawPoint:Qs,drawBar:Ns,getNavigatorLanguage:Fs});var Bs=function(t){if(void 0===t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return t},Os=Pi("Array").entries,Ws=Array.prototype,As={DOMTokenList:!0,NodeList:!0},Xs=function(t){var e=t.entries;return t===Ws||t instanceof Array&&e===Ws.entries||As.hasOwnProperty(ne(t))?Os:e},Ts=pt("JSON","stringify"),Ys=/[\uD800-\uDFFF]/g,Es=/^[\uD800-\uDBFF]$/,Rs=/^[\uDC00-\uDFFF]$/,Us=function(t,e,i){var n=i.charAt(e-1),o=i.charAt(e+1);return Es.test(t)&&!Rs.test(o)||Rs.test(t)&&!Es.test(n)?"\\u"+t.charCodeAt(0).toString(16):t},zs=l((function(){return'"\\udf06\\ud834"'!==Ts("\udf06\ud834")||'"\\udead"'!==Ts("\udead")}));Ts&&V({target:"JSON",stat:!0,forced:zs},{stringify:function(t,e,i){var n=Ts.apply(null,arguments);return"string"==typeof n?n.replace(Ys,Us):n}}),W.JSON||(W.JSON={stringify:JSON.stringify});var Vs=function(t,e,i){return W.JSON.stringify.apply(null,arguments)},js=Pi("Array").values,Hs=Array.prototype,Ps={DOMTokenList:!0,NodeList:!0},Js=function(t){var e=t.values;return t===Hs||t instanceof Array&&e===Hs.values||Ps.hasOwnProperty(ne(t))?js:e},Ks=Pi("Array").keys,qs=Array.prototype,$s={DOMTokenList:!0,NodeList:!0},tr=function(t){var e=t.keys;return t===qs||t instanceof Array&&e===qs.keys||$s.hasOwnProperty(ne(t))?Ks:e},er=Ce.some,ir=$i("some"),nr=on("some");V({target:"Array",proto:!0,forced:!ir||!nr},{some:function(t){return er(this,t,arguments.length>1?arguments[1]:void 0)}});var or=Pi("Array").some,sr=Array.prototype,rr=function(t){var e=t.some;return t===sr||t instanceof Array&&e===sr.some?or:e};var ar=function(t,e){return!e||"object"!==zo(e)&&"function"!=typeof e?Bs(t):e},lr=Do;V({target:"Object",stat:!0},{setPrototypeOf:fi});var hr=W.Object.setPrototypeOf,dr=n((function(t){function e(i){return t.exports=e=hr?lr:function(t){return t.__proto__||lr(t)},e(i)}t.exports=e})),ur=Bi,cr=n((function(t){function e(i,n){return t.exports=e=hr||function(t,e){return t.__proto__=e,t},e(i,n)}t.exports=e}));var pr=function(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function");t.prototype=ur(e&&e.prototype,{constructor:{value:t,writable:!0,configurable:!0}}),e&&cr(t,e)},mr=[],fr=mr.sort,gr=l((function(){mr.sort(void 0)})),vr=l((function(){mr.sort(null)})),br=$i("sort");V({target:"Array",proto:!0,forced:gr||!vr||!br},{sort:function(t){return void 0===t?fr.call(Lt(this)):fr.call(Lt(this),A(t))}});var yr=Pi("Array").sort,xr=Array.prototype,wr=function(t){var e=t.sort;return t===xr||t instanceof Array&&e===xr.sort?yr:e},_r=!l((function(){return Object.isExtensible(Object.preventExtensions({}))})),kr=n((function(t){var e=E.f,i=Ft("meta"),n=0,o=Object.isExtensible||function(){return!0},s=function(t){e(t,i,{value:{objectID:"O"+ ++n,weakData:{}}})},r=t.exports={REQUIRED:!1,fastKey:function(t,e){if(!x(t))return"symbol"==typeof t?t:("string"==typeof t?"S":"P")+t;if(!k(t,i)){if(!o(t))return"F";if(!e)return"E";s(t)}return t[i].objectID},getWeakData:function(t,e){if(!k(t,i)){if(!o(t))return!0;if(!e)return!1;s(t)}return t[i].weakData},onFreeze:function(t){return _r&&r.REQUIRED&&o(t)&&!k(t,i)&&s(t),t}};st[i]=!0})),Dr=(kr.REQUIRED,kr.fastKey,kr.getWeakData,kr.onFreeze,n((function(t){var e=function(t,e){this.stopped=t,this.result=e};(t.exports=function(t,i,n,o,s){var r,a,l,h,d,u,c,p=X(i,n,o?2:1);if(s)r=t;else{if("function"!=typeof(a=Ni(t)))throw TypeError("Target is not iterable");if(io(a)){for(l=0,h=$(t.length);h>l;l++)if((d=o?p(T(c=t[l])[0],c[1]):p(t[l]))&&d instanceof e)return d;return new e(!1)}r=a.call(t)}for(u=r.next;!(c=u.call(r)).done;)if("object"==typeof(d=$n(r,p,c.value,o))&&d&&d instanceof e)return d;return new e(!1)}).stop=function(t){return new e(!0,t)}}))),Ir=function(t,e,i){if(!(t instanceof e))throw TypeError("Incorrect "+(i?i+" ":"")+"invocation");return t},Cr=E.f,Lr=Ce.forEach,Gr=we.set,Mr=we.getterFor,Sr=function(t,e,i){var n,o=-1!==t.indexOf("Map"),s=-1!==t.indexOf("Weak"),r=o?"set":"add",d=a[t],u=d&&d.prototype,c={};if(h&&"function"==typeof d&&(s||u.forEach&&!l((function(){(new d).entries().next()})))){n=e((function(e,i){Gr(Ir(e,n,t),{type:t,collection:new d}),null!=i&&Dr(i,e[r],e,o)}));var p=Mr(t);Lr(["add","clear","delete","forEach","get","has","set","keys","values","entries"],(function(t){var e="add"==t||"set"==t;t in u&&(!s||"clear"!=t)&&R(n.prototype,t,(function(i,n){var o=p(this).collection;if(!e&&s&&!x(i))return"get"==t&&void 0;var r=o[t](0===i?0:i,n);return e?this:r}))})),s||Cr(n.prototype,"size",{configurable:!0,get:function(){return p(this).collection.size}})}else n=i.getConstructor(e,t,o,r),kr.REQUIRED=!0;return ae(n,t,!1,!0),c[t]=n,V({global:!0,forced:!0},c),s||i.setStrong(n,t,o),n},Qr=function(t,e,i){for(var n in e)i&&i.unsafe&&t[n]?t[n]=e[n]:zt(t,n,e[n],i);return t},Nr=Pt("species"),Fr=E.f,Zr=kr.fastKey,Br=we.set,Or=we.getterFor,Wr={getConstructor:function(t,e,i,n){var o=t((function(t,s){Ir(t,o,e),Br(t,{type:e,index:Tt(null),first:void 0,last:void 0,size:0}),h||(t.size=0),null!=s&&Dr(s,t[n],t,i)})),s=Or(e),r=function(t,e,i){var n,o,r=s(t),l=a(t,e);return l?l.value=i:(r.last=l={index:o=Zr(e,!0),key:e,value:i,previous:n=r.last,next:void 0,removed:!1},r.first||(r.first=l),n&&(n.next=l),h?r.size++:t.size++,"F"!==o&&(r.index[o]=l)),t},a=function(t,e){var i,n=s(t),o=Zr(e);if("F"!==o)return n.index[o];for(i=n.first;i;i=i.next)if(i.key==e)return i};return Qr(o.prototype,{clear:function(){for(var t=s(this),e=t.index,i=t.first;i;)i.removed=!0,i.previous&&(i.previous=i.previous.next=void 0),delete e[i.index],i=i.next;t.first=t.last=void 0,h?t.size=0:this.size=0},delete:function(t){var e=s(this),i=a(this,t);if(i){var n=i.next,o=i.previous;delete e.index[i.index],i.removed=!0,o&&(o.next=n),n&&(n.previous=o),e.first==i&&(e.first=n),e.last==i&&(e.last=o),h?e.size--:this.size--}return!!i},forEach:function(t){for(var e,i=s(this),n=X(t,arguments.length>1?arguments[1]:void 0,3);e=e?e.next:i.first;)for(n(e.value,e.key,this);e&&e.removed;)e=e.previous},has:function(t){return!!a(this,t)}}),Qr(o.prototype,i?{get:function(t){var e=a(this,t);return e&&e.value},set:function(t,e){return r(this,0===t?0:t,e)}}:{add:function(t){return r(this,t=0===t?0:t,t)}}),h&&Fr(o.prototype,"size",{get:function(){return s(this).size}}),o},setStrong:function(t,e,i){var n=e+" Iterator",o=Or(e),s=Or(n);xi(t,e,(function(t,e){Br(this,{type:n,target:t,state:o(t),kind:e,last:void 0})}),(function(){for(var t=s(this),e=t.kind,i=t.last;i&&i.removed;)i=i.previous;return t.target&&(t.last=i=i?i.next:t.state.first)?"keys"==e?{value:i.key,done:!1}:"values"==e?{value:i.value,done:!1}:{value:[i.key,i.value],done:!1}:(t.target=void 0,{value:void 0,done:!0})}),i?"entries":"values",!i,!0),function(t){var e=pt(t),i=E.f;h&&e&&!e[Nr]&&i(e,Nr,{configurable:!0,get:function(){return this}})}(e)}},Ar=(Sr("Set",(function(t){return function(){return t(this,arguments.length?arguments[0]:void 0)}}),Wr),W.Set),Xr=(Sr("Map",(function(t){return function(){return t(this,arguments.length?arguments[0]:void 0)}}),Wr),W.Map),Tr=n((function(t){var e=function(t){var e=Object.prototype,i=e.hasOwnProperty,n="function"==typeof Symbol?Symbol:{},o=n.iterator||"@@iterator",s=n.asyncIterator||"@@asyncIterator",r=n.toStringTag||"@@toStringTag";function a(t,e,i,n){var o=e&&e.prototype instanceof d?e:d,s=Object.create(o.prototype),r=new _(n||[]);return s._invoke=function(t,e,i){var n="suspendedStart";return function(o,s){if("executing"===n)throw new Error("Generator is already running");if("completed"===n){if("throw"===o)throw s;return D()}for(i.method=o,i.arg=s;;){var r=i.delegate;if(r){var a=y(r,i);if(a){if(a===h)continue;return a}}if("next"===i.method)i.sent=i._sent=i.arg;else if("throw"===i.method){if("suspendedStart"===n)throw n="completed",i.arg;i.dispatchException(i.arg)}else"return"===i.method&&i.abrupt("return",i.arg);n="executing";var d=l(t,e,i);if("normal"===d.type){if(n=i.done?"completed":"suspendedYield",d.arg===h)continue;return{value:d.arg,done:i.done}}"throw"===d.type&&(n="completed",i.method="throw",i.arg=d.arg)}}}(t,i,r),s}function l(t,e,i){try{return{type:"normal",arg:t.call(e,i)}}catch(t){return{type:"throw",arg:t}}}t.wrap=a;var h={};function d(){}function u(){}function c(){}var p={};p[o]=function(){return this};var m=Object.getPrototypeOf,f=m&&m(m(k([])));f&&f!==e&&i.call(f,o)&&(p=f);var g=c.prototype=d.prototype=Object.create(p);function v(t){["next","throw","return"].forEach((function(e){t[e]=function(t){return this._invoke(e,t)}}))}function b(t,e){var n;this._invoke=function(o,s){function r(){return new e((function(n,r){!function n(o,s,r,a){var h=l(t[o],t,s);if("throw"!==h.type){var d=h.arg,u=d.value;return u&&"object"==typeof u&&i.call(u,"__await")?e.resolve(u.__await).then((function(t){n("next",t,r,a)}),(function(t){n("throw",t,r,a)})):e.resolve(u).then((function(t){d.value=t,r(d)}),(function(t){return n("throw",t,r,a)}))}a(h.arg)}(o,s,n,r)}))}return n=n?n.then(r,r):r()}}function y(t,e){var i=t.iterator[e.method];if(void 0===i){if(e.delegate=null,"throw"===e.method){if(t.iterator.return&&(e.method="return",e.arg=void 0,y(t,e),"throw"===e.method))return h;e.method="throw",e.arg=new TypeError("The iterator does not provide a 'throw' method")}return h}var n=l(i,t.iterator,e.arg);if("throw"===n.type)return e.method="throw",e.arg=n.arg,e.delegate=null,h;var o=n.arg;return o?o.done?(e[t.resultName]=o.value,e.next=t.nextLoc,"return"!==e.method&&(e.method="next",e.arg=void 0),e.delegate=null,h):o:(e.method="throw",e.arg=new TypeError("iterator result is not an object"),e.delegate=null,h)}function x(t){var e={tryLoc:t[0]};1 in t&&(e.catchLoc=t[1]),2 in t&&(e.finallyLoc=t[2],e.afterLoc=t[3]),this.tryEntries.push(e)}function w(t){var e=t.completion||{};e.type="normal",delete e.arg,t.completion=e}function _(t){this.tryEntries=[{tryLoc:"root"}],t.forEach(x,this),this.reset(!0)}function k(t){if(t){var e=t[o];if(e)return e.call(t);if("function"==typeof t.next)return t;if(!isNaN(t.length)){var n=-1,s=function e(){for(;++n<t.length;)if(i.call(t,n))return e.value=t[n],e.done=!1,e;return e.value=void 0,e.done=!0,e};return s.next=s}}return{next:D}}function D(){return{value:void 0,done:!0}}return u.prototype=g.constructor=c,c.constructor=u,c[r]=u.displayName="GeneratorFunction",t.isGeneratorFunction=function(t){var e="function"==typeof t&&t.constructor;return!!e&&(e===u||"GeneratorFunction"===(e.displayName||e.name))},t.mark=function(t){return Object.setPrototypeOf?Object.setPrototypeOf(t,c):(t.__proto__=c,r in t||(t[r]="GeneratorFunction")),t.prototype=Object.create(g),t},t.awrap=function(t){return{__await:t}},v(b.prototype),b.prototype[s]=function(){return this},t.AsyncIterator=b,t.async=function(e,i,n,o,s){void 0===s&&(s=Promise);var r=new b(a(e,i,n,o),s);return t.isGeneratorFunction(i)?r:r.next().then((function(t){return t.done?t.value:r.next()}))},v(g),g[r]="Generator",g[o]=function(){return this},g.toString=function(){return"[object Generator]"},t.keys=function(t){var e=[];for(var i in t)e.push(i);return e.reverse(),function i(){for(;e.length;){var n=e.pop();if(n in t)return i.value=n,i.done=!1,i}return i.done=!0,i}},t.values=k,_.prototype={constructor:_,reset:function(t){if(this.prev=0,this.next=0,this.sent=this._sent=void 0,this.done=!1,this.delegate=null,this.method="next",this.arg=void 0,this.tryEntries.forEach(w),!t)for(var e in this)"t"===e.charAt(0)&&i.call(this,e)&&!isNaN(+e.slice(1))&&(this[e]=void 0)},stop:function(){this.done=!0;var t=this.tryEntries[0].completion;if("throw"===t.type)throw t.arg;return this.rval},dispatchException:function(t){if(this.done)throw t;var e=this;function n(i,n){return r.type="throw",r.arg=t,e.next=i,n&&(e.method="next",e.arg=void 0),!!n}for(var o=this.tryEntries.length-1;o>=0;--o){var s=this.tryEntries[o],r=s.completion;if("root"===s.tryLoc)return n("end");if(s.tryLoc<=this.prev){var a=i.call(s,"catchLoc"),l=i.call(s,"finallyLoc");if(a&&l){if(this.prev<s.catchLoc)return n(s.catchLoc,!0);if(this.prev<s.finallyLoc)return n(s.finallyLoc)}else if(a){if(this.prev<s.catchLoc)return n(s.catchLoc,!0)}else{if(!l)throw new Error("try statement without catch or finally");if(this.prev<s.finallyLoc)return n(s.finallyLoc)}}}},abrupt:function(t,e){for(var n=this.tryEntries.length-1;n>=0;--n){var o=this.tryEntries[n];if(o.tryLoc<=this.prev&&i.call(o,"finallyLoc")&&this.prev<o.finallyLoc){var s=o;break}}s&&("break"===t||"continue"===t)&&s.tryLoc<=e&&e<=s.finallyLoc&&(s=null);var r=s?s.completion:{};return r.type=t,r.arg=e,s?(this.method="next",this.next=s.finallyLoc,h):this.complete(r)},complete:function(t,e){if("throw"===t.type)throw t.arg;return"break"===t.type||"continue"===t.type?this.next=t.arg:"return"===t.type?(this.rval=this.arg=t.arg,this.method="return",this.next="end"):"normal"===t.type&&e&&(this.next=e),h},finish:function(t){for(var e=this.tryEntries.length-1;e>=0;--e){var i=this.tryEntries[e];if(i.finallyLoc===t)return this.complete(i.completion,i.afterLoc),w(i),h}},catch:function(t){for(var e=this.tryEntries.length-1;e>=0;--e){var i=this.tryEntries[e];if(i.tryLoc===t){var n=i.completion;if("throw"===n.type){var o=n.arg;w(i)}return o}}throw new Error("illegal catch attempt")},delegateYield:function(t,e,i){return this.delegate={iterator:k(t),resultName:e,nextLoc:i},"next"===this.method&&(this.arg=void 0),h}},t}(t.exports);try{regeneratorRuntime=e}catch(t){Function("r","regeneratorRuntime = r")(e)}})),Yr=Xo,Er=wn("splice"),Rr=on("splice",{ACCESSORS:!0,0:0,1:2}),Ur=Math.max,zr=Math.min;V({target:"Array",proto:!0,forced:!Er||!Rr},{splice:function(t,e){var i,n,o,s,r,a,l=Lt(this),h=$(l.length),d=it(t,h),u=arguments.length;if(0===u?i=n=0:1===u?(i=0,n=h-d):(i=u-2,n=zr(Ur(K(e),0),h-d)),h+i-n>9007199254740991)throw TypeError("Maximum allowed length exceeded");for(o=ke(l,n),s=0;s<n;s++)(r=d+s)in l&&bt(o,s,l[r]);if(o.length=n,i<n){for(s=d;s<h-n;s++)a=s+i,(r=s+n)in l?l[a]=l[r]:delete l[a];for(s=h;s>h-n+i;s--)delete l[s-1]}else if(i>n)for(s=h-n;s>d;s--)a=s+i-1,(r=s+n-1)in l?l[a]=l[r]:delete l[a];for(s=0;s<i;s++)l[s+d]=arguments[s+2];return l.length=h-n+i,o}});var Vr=Pi("Array").splice,jr=Array.prototype,Hr=function(t){var e=t.splice;return t===jr||t instanceof Array&&e===jr.splice?Vr:e},Pr=[].slice,Jr=/MSIE .\./.test(fn),Kr=function(t){return function(e,i){var n=arguments.length>2,o=n?Pr.call(arguments,2):void 0;return t(n?function(){("function"==typeof e?e:Function(e)).apply(this,o)}:e,i)}};V({global:!0,bind:!0,forced:Jr},{setTimeout:Kr(a.setTimeout),setInterval:Kr(a.setInterval)});var qr=W.setTimeout,$r=function(t,e,i,n,o,s,r,a){for(var l,h=o,d=0,u=!!r&&X(r,a,3);d<n;){if(d in i){if(l=u?u(i[d],d,e):i[d],s>0&&Ct(l))h=$r(t,e,l,$(l.length),h,s-1)-1;else{if(h>=9007199254740991)throw TypeError("Exceed the acceptable array length");t[h]=l}h++}d++}return h},ta=$r;V({target:"Array",proto:!0},{flatMap:function(t){var e,i=Lt(this),n=$(i.length);return A(t),(e=ke(i,0)).length=ta(e,i,i,n,0,1,t,arguments.length>1?arguments[1]:void 0),e}});var ea=Pi("Array").flatMap,ia=Array.prototype,na=function(t){var e=t.flatMap;return t===ia||t instanceof Array&&e===ia.flatMap?ea:e},oa=function(t){return function(e,i,n,o){A(i);var s=Lt(e),r=v(s),a=$(s.length),l=t?a-1:0,h=t?-1:1;if(n<2)for(;;){if(l in r){o=r[l],l+=h;break}if(l+=h,t?l<0:a<=l)throw TypeError("Reduce of empty array with no initial value")}for(;t?l>=0:a>l;l+=h)l in r&&(o=i(o,r[l],l,s));return o}},sa={left:oa(!1),right:oa(!0)}.left,ra=$i("reduce"),aa=on("reduce",{1:0});V({target:"Array",proto:!0,forced:!ra||!aa},{reduce:function(t){return sa(this,t,arguments.length,arguments.length>1?arguments[1]:void 0)}});var la=Pi("Array").reduce,ha=Array.prototype,da=function(t){var e=t.reduce;return t===ha||t instanceof Array&&e===ha.reduce?la:e},ua=[].slice,ca={},pa=function(t,e,i){if(!(e in ca)){for(var n=[],o=0;o<e;o++)n[o]="a["+o+"]";ca[e]=Function("C,a","return new C("+n.join(",")+")")}return ca[e](t,i)},ma=Function.bind||function(t){var e=A(this),i=ua.call(arguments,1),n=function(){var o=i.concat(ua.call(arguments));return this instanceof n?pa(e,o.length,o):e.apply(t,o)};return x(e.prototype)&&(n.prototype=e.prototype),n};V({target:"Function",proto:!0},{bind:ma});var fa=Pi("Function").bind,ga=Function.prototype,va=function(t){var e=t.bind;return t===ga||t instanceof Function&&e===ga.bind?fa:e};var ba=function(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")};function ya(t,e){for(var i=0;i<e.length;i++){var n=e[i];n.enumerable=n.enumerable||!1,n.configurable=!0,"value"in n&&(n.writable=!0),Wi(t,n.key,n)}}var xa=function(t,e,i){return e&&ya(t.prototype,e),i&&ya(t,i),t};function wa(t,e){var i=Yi(t);if(si){var n=si(t);e&&(n=En(n).call(n,(function(e){return kt(t,e).enumerable}))),i.push.apply(i,n)}return i}function _a(t){return new Da(t)}var ka=function(){function t(e,i,n){var o,s,r;ba(this,t),this._source=e,this._transformers=i,this._target=n,this._listeners={add:va(o=this._add).call(o,this),remove:va(s=this._remove).call(s,this),update:va(r=this._update).call(r,this)}}return xa(t,[{key:"all",value:function(){return this._target.update(this._transformItems(this._source.get())),this}},{key:"start",value:function(){return this._source.on("add",this._listeners.add),this._source.on("remove",this._listeners.remove),this._source.on("update",this._listeners.update),this}},{key:"stop",value:function(){return this._source.off("add",this._listeners.add),this._source.off("remove",this._listeners.remove),this._source.off("update",this._listeners.update),this}},{key:"_transformItems",value:function(t){var e;return da(e=this._transformers).call(e,(function(t,e){return e(t)}),t)}},{key:"_add",value:function(t,e){null!=e&&this._target.add(this._transformItems(this._source.get(e.items)))}},{key:"_update",value:function(t,e){null!=e&&this._target.update(this._transformItems(this._source.get(e.items)))}},{key:"_remove",value:function(t,e){null!=e&&this._target.remove(this._transformItems(e.oldData))}}]),t}(),Da=function(){function t(e){ba(this,t),this._source=e,this._transformers=[]}return xa(t,[{key:"filter",value:function(t){return this._transformers.push((function(e){return En(e).call(e,t)})),this}},{key:"map",value:function(t){return this._transformers.push((function(e){return Ln(e).call(e,t)})),this}},{key:"flatMap",value:function(t){return this._transformers.push((function(e){return na(e).call(e,t)})),this}},{key:"to",value:function(t){return new ka(this._source,this._transformers,t)}}]),t}();for(var Ia=function(t,e){return t(e={exports:{}},e.exports),e.exports}((function(t){var e,i,n="undefined"!=typeof crypto&&crypto.getRandomValues&&va(e=crypto.getRandomValues).call(e,crypto)||"undefined"!=typeof msCrypto&&"function"==typeof window.msCrypto.getRandomValues&&va(i=msCrypto.getRandomValues).call(i,msCrypto);if(n){var o=new Uint8Array(16);t.exports=function(){return n(o),o}}else{var s=new Array(16);t.exports=function(){for(var t,e=0;e<16;e++)0==(3&e)&&(t=4294967296*Math.random()),s[e]=t>>>((3&e)<<3)&255;return s}}})),Ca=[],La=0;La<256;++La)Ca[La]=(La+256).toString(16).substr(1);var Ga=function(t,e){var i=e||0,n=Ca;return[n[t[i++]],n[t[i++]],n[t[i++]],n[t[i++]],"-",n[t[i++]],n[t[i++]],"-",n[t[i++]],n[t[i++]],"-",n[t[i++]],n[t[i++]],"-",n[t[i++]],n[t[i++]],n[t[i++]],n[t[i++]],n[t[i++]],n[t[i++]]].join("")};var Ma=function(t,e,i){var n=e&&i||0;"string"==typeof t&&(e="binary"===t?new Array(16):null,t=null);var o=(t=t||{}).random||(t.rng||Ia)();if(o[6]=15&o[6]|64,o[8]=63&o[8]|128,e)for(var s=0;s<16;++s)e[n+s]=o[s];return e||Ga(o)},Sa=/^\/?Date\((-?\d+)/i;function Qa(t){return s.isMoment(t)}function Na(t,e){var i;if(void 0!==t){if(null===t)return null;if(!e)return t;if("string"!=typeof e&&!(e instanceof String))throw new Error("Type must be a string");switch(e){case"boolean":case"Boolean":return Boolean(t);case"number":case"Number":return rs(t)&&!isNaN(Date.parse(t))?s(t).valueOf():Number(t.valueOf());case"string":case"String":return String(t);case"Date":if(ss(t))return new Date(t);if(t instanceof Date)return new Date(t.valueOf());if(Qa(t))return new Date(t.valueOf());if(rs(t))return(i=Sa.exec(t))?new Date(Number(i[1])):s(new Date(t)).toDate();throw new Error("Cannot convert object of type "+us(t)+" to type Date");case"Moment":if(ss(t))return s(t);if(t instanceof Date)return s(t.valueOf());if(Qa(t))return s(t);if(rs(t))return i=Sa.exec(t),s(i?Number(i[1]):t);throw new Error("Cannot convert object of type "+us(t)+" to type Date");case"ISODate":if(ss(t))return new Date(t);if(t instanceof Date)return t.toISOString();if(Qa(t))return t.toDate().toISOString();if(rs(t))return(i=Sa.exec(t))?new Date(Number(i[1])).toISOString():s(t).format();throw new Error("Cannot convert object of type "+us(t)+" to type ISODate");case"ASPDate":if(ss(t))return"/Date("+t+")/";if(t instanceof Date||Qa(t))return"/Date("+t.valueOf()+")/";if(rs(t))return"/Date("+((i=Sa.exec(t))?new Date(Number(i[1])).valueOf():new Date(t).valueOf())+")/";throw new Error("Cannot convert object of type "+us(t)+" to type ASPDate");default:throw new Error("Unknown type ".concat(e))}}}function Fa(t){return"string"==typeof t||"number"==typeof t}var Za=function(){function t(e){ba(this,t),this._queue=[],this._timeout=null,this._extended=null,this.delay=null,this.max=1/0,this.setOptions(e)}return xa(t,[{key:"setOptions",value:function(t){t&&void 0!==t.delay&&(this.delay=t.delay),t&&void 0!==t.max&&(this.max=t.max),this._flushIfNeeded()}},{key:"destroy",value:function(){if(this.flush(),this._extended){for(var t=this._extended.object,e=this._extended.methods,i=0;i<e.length;i++){var n=e[i];n.original?t[n.name]=n.original:delete t[n.name]}this._extended=null}}},{key:"replace",value:function(t,e){var i=this,n=t[e];if(!n)throw new Error("Method "+e+" undefined");t[e]=function(){for(var t=arguments.length,e=new Array(t),o=0;o<t;o++)e[o]=arguments[o];i.queue({args:e,fn:n,context:this})}}},{key:"queue",value:function(t){"function"==typeof t?this._queue.push({fn:t}):this._queue.push(t),this._flushIfNeeded()}},{key:"_flushIfNeeded",value:function(){var t=this;this._queue.length>this.max&&this.flush(),null!=this._timeout&&(clearTimeout(this._timeout),this._timeout=null),this.queue.length>0&&"number"==typeof this.delay&&(this._timeout=qr((function(){t.flush()}),this.delay))}},{key:"flush",value:function(){var t,e;mn(t=Hr(e=this._queue).call(e,0)).call(t,(function(t){t.fn.apply(t.context||t.fn,t.args||[])}))}}],[{key:"extend",value:function(e,i){var n=new t(i);if(void 0!==e.flush)throw new Error("Target object already has a property flush");e.flush=function(){n.flush()};var o=[{name:"flush",original:void 0}];if(i&&i.replace)for(var s=0;s<i.replace.length;s++){var r=i.replace[s];o.push({name:r,original:e[r]}),n.replace(e,r)}return n._extended={object:e,methods:o},n}}]),t}(),Ba=function(){function t(){ba(this,t),this._subscribers={"*":[],add:[],remove:[],update:[]},this.subscribe=t.prototype.on,this.unsubscribe=t.prototype.off}return xa(t,[{key:"_trigger",value:function(t,e,i){var n,o;if("*"===t)throw new Error("Cannot trigger event *");mn(n=Pn(o=[]).call(o,mo(this._subscribers[t]),mo(this._subscribers["*"]))).call(n,(function(n){n(t,e,null!=i?i:null)}))}},{key:"on",value:function(t,e){"function"==typeof e&&this._subscribers[t].push(e)}},{key:"off",value:function(t,e){var i;this._subscribers[t]=En(i=this._subscribers[t]).call(i,(function(t){return t!==e}))}}]),t}(),Oa=function(){function t(e){ba(this,t),this._pairs=e}return xa(t,[{key:Yr,value:Tr.mark((function t(){var e,i,n,o,s,r,a,l;return Tr.wrap((function(t){for(;;)switch(t.prev=t.next){case 0:e=!0,i=!1,n=void 0,t.prev=3,o=Fi(this._pairs);case 5:if(e=(s=o.next()).done){t.next=12;break}return r=Po(s.value,2),a=r[0],l=r[1],t.next=9,[a,l];case 9:e=!0,t.next=5;break;case 12:t.next=18;break;case 14:t.prev=14,t.t0=t.catch(3),i=!0,n=t.t0;case 18:t.prev=18,t.prev=19,e||null==o.return||o.return();case 21:if(t.prev=21,!i){t.next=24;break}throw n;case 24:return t.finish(21);case 25:return t.finish(18);case 26:case"end":return t.stop()}}),t,this,[[3,14,18,26],[19,,21,25]])}))},{key:"entries",value:Tr.mark((function t(){var e,i,n,o,s,r,a,l;return Tr.wrap((function(t){for(;;)switch(t.prev=t.next){case 0:e=!0,i=!1,n=void 0,t.prev=3,o=Fi(this._pairs);case 5:if(e=(s=o.next()).done){t.next=12;break}return r=Po(s.value,2),a=r[0],l=r[1],t.next=9,[a,l];case 9:e=!0,t.next=5;break;case 12:t.next=18;break;case 14:t.prev=14,t.t0=t.catch(3),i=!0,n=t.t0;case 18:t.prev=18,t.prev=19,e||null==o.return||o.return();case 21:if(t.prev=21,!i){t.next=24;break}throw n;case 24:return t.finish(21);case 25:return t.finish(18);case 26:case"end":return t.stop()}}),t,this,[[3,14,18,26],[19,,21,25]])}))},{key:"keys",value:Tr.mark((function t(){var e,i,n,o,s,r,a;return Tr.wrap((function(t){for(;;)switch(t.prev=t.next){case 0:e=!0,i=!1,n=void 0,t.prev=3,o=Fi(this._pairs);case 5:if(e=(s=o.next()).done){t.next=12;break}return r=Po(s.value,1),a=r[0],t.next=9,a;case 9:e=!0,t.next=5;break;case 12:t.next=18;break;case 14:t.prev=14,t.t0=t.catch(3),i=!0,n=t.t0;case 18:t.prev=18,t.prev=19,e||null==o.return||o.return();case 21:if(t.prev=21,!i){t.next=24;break}throw n;case 24:return t.finish(21);case 25:return t.finish(18);case 26:case"end":return t.stop()}}),t,this,[[3,14,18,26],[19,,21,25]])}))},{key:"values",value:Tr.mark((function t(){var e,i,n,o,s,r,a;return Tr.wrap((function(t){for(;;)switch(t.prev=t.next){case 0:e=!0,i=!1,n=void 0,t.prev=3,o=Fi(this._pairs);case 5:if(e=(s=o.next()).done){t.next=12;break}return r=Po(s.value,2),a=r[1],t.next=9,a;case 9:e=!0,t.next=5;break;case 12:t.next=18;break;case 14:t.prev=14,t.t0=t.catch(3),i=!0,n=t.t0;case 18:t.prev=18,t.prev=19,e||null==o.return||o.return();case 21:if(t.prev=21,!i){t.next=24;break}throw n;case 24:return t.finish(21);case 25:return t.finish(18);case 26:case"end":return t.stop()}}),t,this,[[3,14,18,26],[19,,21,25]])}))},{key:"toIdArray",value:function(){var t;return Ln(t=mo(this._pairs)).call(t,(function(t){return t[0]}))}},{key:"toItemArray",value:function(){var t;return Ln(t=mo(this._pairs)).call(t,(function(t){return t[1]}))}},{key:"toEntryArray",value:function(){return mo(this._pairs)}},{key:"toObjectMap",value:function(){var t=Oi(null),e=!0,i=!1,n=void 0;try{for(var o,s=Fi(this._pairs);!(e=(o=s.next()).done);e=!0){var r=Po(o.value,2),a=r[0],l=r[1];t[a]=l}}catch(t){i=!0,n=t}finally{try{e||null==s.return||s.return()}finally{if(i)throw n}}return t}},{key:"toMap",value:function(){return new Xr(this._pairs)}},{key:"toIdSet",value:function(){return new Ar(this.toIdArray())}},{key:"toItemSet",value:function(){return new Ar(this.toItemArray())}},{key:"cache",value:function(){return new t(mo(this._pairs))}},{key:"distinct",value:function(t){var e=new Ar,i=!0,n=!1,o=void 0;try{for(var s,r=Fi(this._pairs);!(i=(s=r.next()).done);i=!0){var a=Po(s.value,2),l=a[0],h=a[1];e.add(t(h,l))}}catch(t){n=!0,o=t}finally{try{i||null==r.return||r.return()}finally{if(n)throw o}}return e}},{key:"filter",value:function(e){var i=this._pairs;return new t(Ai({},Yr,Tr.mark((function t(){var n,o,s,r,a,l,h,d;return Tr.wrap((function(t){for(;;)switch(t.prev=t.next){case 0:n=!0,o=!1,s=void 0,t.prev=3,r=Fi(i);case 5:if(n=(a=r.next()).done){t.next=13;break}if(l=Po(a.value,2),h=l[0],d=l[1],!e(d,h)){t.next=10;break}return t.next=10,[h,d];case 10:n=!0,t.next=5;break;case 13:t.next=19;break;case 15:t.prev=15,t.t0=t.catch(3),o=!0,s=t.t0;case 19:t.prev=19,t.prev=20,n||null==r.return||r.return();case 22:if(t.prev=22,!o){t.next=25;break}throw s;case 25:return t.finish(22);case 26:return t.finish(19);case 27:case"end":return t.stop()}}),t,null,[[3,15,19,27],[20,,22,26]])}))))}},{key:"forEach",value:function(t){var e=!0,i=!1,n=void 0;try{for(var o,s=Fi(this._pairs);!(e=(o=s.next()).done);e=!0){var r=Po(o.value,2),a=r[0];t(r[1],a)}}catch(t){i=!0,n=t}finally{try{e||null==s.return||s.return()}finally{if(i)throw n}}}},{key:"map",value:function(e){var i=this._pairs;return new t(Ai({},Yr,Tr.mark((function t(){var n,o,s,r,a,l,h,d;return Tr.wrap((function(t){for(;;)switch(t.prev=t.next){case 0:n=!0,o=!1,s=void 0,t.prev=3,r=Fi(i);case 5:if(n=(a=r.next()).done){t.next=12;break}return l=Po(a.value,2),h=l[0],d=l[1],t.next=9,[h,e(d,h)];case 9:n=!0,t.next=5;break;case 12:t.next=18;break;case 14:t.prev=14,t.t0=t.catch(3),o=!0,s=t.t0;case 18:t.prev=18,t.prev=19,n||null==r.return||r.return();case 21:if(t.prev=21,!o){t.next=24;break}throw s;case 24:return t.finish(21);case 25:return t.finish(18);case 26:case"end":return t.stop()}}),t,null,[[3,14,18,26],[19,,21,25]])}))))}},{key:"max",value:function(t){var e=Fi(this._pairs),i=e.next();if(i.done)return null;for(var n=i.value[1],o=t(i.value[1],i.value[0]);!(i=e.next()).done;){var s=Po(i.value,2),r=s[0],a=s[1],l=t(a,r);l>o&&(o=l,n=a)}return n}},{key:"min",value:function(t){var e=Fi(this._pairs),i=e.next();if(i.done)return null;for(var n=i.value[1],o=t(i.value[1],i.value[0]);!(i=e.next()).done;){var s=Po(i.value,2),r=s[0],a=s[1],l=t(a,r);l<o&&(o=l,n=a)}return n}},{key:"reduce",value:function(t,e){var i=!0,n=!1,o=void 0;try{for(var s,r=Fi(this._pairs);!(i=(s=r.next()).done);i=!0){var a=Po(s.value,2),l=a[0];e=t(e,a[1],l)}}catch(t){n=!0,o=t}finally{try{i||null==r.return||r.return()}finally{if(n)throw o}}return e}},{key:"sort",value:function(e){var i=this;return new t(Ai({},Yr,(function(){var t;return Fi(wr(t=mo(i._pairs)).call(t,(function(t,i){var n=Po(t,2),o=n[0],s=n[1],r=Po(i,2),a=r[0],l=r[1];return e(s,l,o,a)})))})))}}]),t}(),Wa=function(){console.warn("Type coercion has been deprecated. Please, use data pipes instead. See https://visjs.github.io/vis-data/data/datapipe.html#TypeCoercion for more details with working migration example.")},Aa=function(t){function e(t,i){var n;if(ba(this,e),n=ar(this,dr(e).call(this)),t&&!Zo(t)&&(i=t,t=[]),n._options=i||{},n._data=new Xr,n.length=0,n._idProp=n._options.fieldId||"id",n._type={},n._options.type){Wa();for(var o=Yi(n._options.type),s=0,r=o.length;s<r;s++){var a=o[s],l=n._options.type[a];n._type[a]="Date"==l||"ISODate"==l||"ASPDate"==l?"Date":l}}return t&&t.length&&n.add(t),n.setOptions(i),n}return pr(e,t),xa(e,[{key:"setOptions",value:function(t){t&&void 0!==t.queue&&(!1===t.queue?this._queue&&(this._queue.destroy(),delete this._queue):(this._queue||(this._queue=Za.extend(this,{replace:["add","update","remove"]})),t.queue&&"object"===zo(t.queue)&&this._queue.setOptions(t.queue)))}},{key:"add",value:function(t,e){var i,n=this,o=[];if(Zo(t)){var s=Ln(t).call(t,(function(t){return t[n._idProp]}));if(rr(s).call(s,(function(t){return n._data.has(t)})))throw new Error("A duplicate id was found in the parameter array.");for(var r=0,a=t.length;r<a;r++)i=this._addItem(t[r]),o.push(i)}else{if(!t||"object"!==zo(t))throw new Error("Unknown dataType");i=this._addItem(t),o.push(i)}return o.length&&this._trigger("add",{items:o},e),o}},{key:"update",value:function(t,e){var i=this,n=[],o=[],s=[],r=[],a=this._idProp,l=function(t){var e=t[a];if(null!=e&&i._data.has(e)){var l=t,h=Ao({},i._data.get(e)),d=i._updateItem(l);o.push(d),r.push(l),s.push(h)}else{var u=i._addItem(t);n.push(u)}};if(Zo(t))for(var h=0,d=t.length;h<d;h++)t[h]&&"object"===zo(t[h])?l(t[h]):console.warn("Ignoring input item, which is not an object at index "+h);else{if(!t||"object"!==zo(t))throw new Error("Unknown dataType");l(t)}if(n.length&&this._trigger("add",{items:n},e),o.length){var u={items:o,oldData:s,data:r};this._trigger("update",u,e)}return Pn(n).call(n,o)}},{key:"updateOnly",value:function(t,e){var i,n=this;Zo(t)||(t=[t]);var o=Ln(i=Ln(t).call(t,(function(t){var e=n._data.get(t[n._idProp]);if(null==e)throw new Error("Updating non-existent items is not allowed.");return{oldData:e,update:t}}))).call(i,(function(t){var e=t.oldData,i=t.update,o=e[n._idProp],s=ds(ds({},e),i);return n._data.set(o,s),{id:o,oldData:e,updatedData:s}}));if(o.length){var s={items:Ln(o).call(o,(function(t){return t.id})),oldData:Ln(o).call(o,(function(t){return t.oldData})),data:Ln(o).call(o,(function(t){return t.updatedData}))};return this._trigger("update",s,e),s.items}return[]}},{key:"get",value:function(t,e){var i=void 0,n=void 0,o=void 0;Fa(t)?(i=t,o=e):Zo(t)?(n=t,o=e):o=t;var s=o&&"Object"===o.returnType?"Object":"Array",r=o&&o.type||this._options.type,a=o&&En(o),l=[],h=null,d=null,u=null;if(null!=i)(h=this._getItem(i,r))&&a&&!a(h)&&(h=null);else if(null!=n)for(var c=0,p=n.length;c<p;c++)null==(h=this._getItem(n[c],r))||a&&!a(h)||l.push(h);else for(var m,f=0,g=(d=mo(tr(m=this._data).call(m))).length;f<g;f++)u=d[f],null==(h=this._getItem(u,r))||a&&!a(h)||l.push(h);if(o&&o.order&&null==i&&this._sort(l,o.order),o&&o.fields){var v=o.fields;if(null!=i&&null!=h)h=this._filterFields(h,v);else for(var b=0,y=l.length;b<y;b++)l[b]=this._filterFields(l[b],v)}if("Object"==s){for(var x={},w=0,_=l.length;w<_;w++){var k=l[w];x[k[this._idProp]]=k}return x}return null!=i?h:l}},{key:"getIds",value:function(t){var e,i,n=this._data,o=t&&En(t),s=t&&t.order,r=t&&t.type||this._options.type,a=mo(tr(n).call(n)),l=[];if(o)if(s){i=[];for(var h=0,d=a.length;h<d;h++){var u=a[h];o(e=this._getItem(u,r))&&i.push(e)}this._sort(i,s);for(var c=0,p=i.length;c<p;c++)l.push(i[c][this._idProp])}else for(var m=0,f=a.length;m<f;m++){var g=a[m];o(e=this._getItem(g,r))&&l.push(e[this._idProp])}else if(s){i=[];for(var v=0,b=a.length;v<b;v++){var y=a[v];i.push(n.get(y))}this._sort(i,s);for(var x=0,w=i.length;x<w;x++)l.push(i[x][this._idProp])}else for(var _=0,k=a.length;_<k;_++){var D=a[_];e=n.get(D),l.push(e[this._idProp])}return l}},{key:"getDataSet",value:function(){return this}},{key:"forEach",value:function(t,e){var i=e&&En(e),n=e&&e.type||this._options.type,o=this._data,s=mo(tr(o).call(o));if(e&&e.order)for(var r=this.get(e),a=0,l=r.length;a<l;a++){var h=r[a];t(h,h[this._idProp])}else for(var d=0,u=s.length;d<u;d++){var c=s[d],p=this._getItem(c,n);i&&!i(p)||t(p,c)}}},{key:"map",value:function(t,e){for(var i=e&&En(e),n=e&&e.type||this._options.type,o=[],s=this._data,r=mo(tr(s).call(s)),a=0,l=r.length;a<l;a++){var h=r[a],d=this._getItem(h,n);i&&!i(d)||o.push(t(d,h))}return e&&e.order&&this._sort(o,e.order),o}},{key:"_filterFields",value:function(t,e){var i;return t?da(i=Zo(e)?e:Yi(e)).call(i,(function(e,i){return e[i]=t[i],e}),{}):t}},{key:"_sort",value:function(t,e){if("string"==typeof e){var i=e;wr(t).call(t,(function(t,e){var n=t[i],o=e[i];return n>o?1:n<o?-1:0}))}else{if("function"!=typeof e)throw new TypeError("Order must be a function or a string");wr(t).call(t,e)}}},{key:"remove",value:function(t,e){for(var i=[],n=[],o=Zo(t)?t:[t],s=0,r=o.length;s<r;s++){var a=this._remove(o[s]);if(a){var l=a[this._idProp];null!=l&&(i.push(l),n.push(a))}}return i.length&&this._trigger("remove",{items:i,oldData:n},e),i}},{key:"_remove",value:function(t){var e;if(Fa(t)?e=t:t&&"object"===zo(t)&&(e=t[this._idProp]),null!=e&&this._data.has(e)){var i=this._data.get(e)||null;return this._data.delete(e),--this.length,i}return null}},{key:"clear",value:function(t){for(var e,i=mo(tr(e=this._data).call(e)),n=[],o=0,s=i.length;o<s;o++)n.push(this._data.get(i[o]));return this._data.clear(),this.length=0,this._trigger("remove",{items:i,oldData:n},t),i}},{key:"max",value:function(t){var e=null,i=null,n=!0,o=!1,s=void 0;try{for(var r,a=Fi(Js(l=this._data).call(l));!(n=(r=a.next()).done);n=!0){var l,h=r.value,d=h[t];"number"==typeof d&&(null==i||d>i)&&(e=h,i=d)}}catch(t){o=!0,s=t}finally{try{n||null==a.return||a.return()}finally{if(o)throw s}}return e||null}},{key:"min",value:function(t){var e=null,i=null,n=!0,o=!1,s=void 0;try{for(var r,a=Fi(Js(l=this._data).call(l));!(n=(r=a.next()).done);n=!0){var l,h=r.value,d=h[t];"number"==typeof d&&(null==i||d<i)&&(e=h,i=d)}}catch(t){o=!0,s=t}finally{try{n||null==a.return||a.return()}finally{if(o)throw s}}return e||null}},{key:"distinct",value:function(t){for(var e=this._data,i=mo(tr(e).call(e)),n=[],o=this._options.type&&this._options.type[t]||null,s=0,r=0,a=i.length;r<a;r++){for(var l=i[r],h=e.get(l)[t],d=!1,u=0;u<s;u++)if(n[u]==h){d=!0;break}d||void 0===h||(n[s]=h,s++)}if(o)for(var c=0,p=n.length;c<p;c++)n[c]=Na(n[c],o);return n}},{key:"_addItem",value:function(t){var e=t[this._idProp];if(null!=e){if(this._data.has(e))throw new Error("Cannot add item: item with id "+e+" already exists")}else e=Ma(),t[this._idProp]=e;for(var i={},n=Yi(t),o=0,s=n.length;o<s;o++){var r=n[o],a=this._type[r];i[r]=Na(t[r],a)}return this._data.set(e,i),++this.length,e}},{key:"_getItem",value:function(t,e){var i,n=this._data.get(t);if(!n)return null;var o=Yi(n);if(e){Wa(),i={};for(var s=0,r=o.length;s<r;s++){var a=o[s],l=n[a];i[a]=Na(l,e[a])}}else i=function(t){for(var e=1;e<arguments.length;e++){var i,n=null!=arguments[e]?arguments[e]:{};if(e%2)mn(i=wa(Object(n),!0)).call(i,(function(e){Ai(t,e,n[e])}));else if(yt)ut(t,yt(n));else{var o;mn(o=wa(Object(n))).call(o,(function(e){H(t,e,kt(n,e))}))}}return t}({},n);return null==i[this._idProp]&&(i[this._idProp]=n.id),i}},{key:"_updateItem",value:function(t){var e=t[this._idProp];if(null==e)throw new Error("Cannot update item: item has no id (item: "+Vs(t)+")");var i=this._data.get(e);if(!i)throw new Error("Cannot update item: no item with id "+e+" found");for(var n=Yi(t),o=0,s=n.length;o<s;o++){var r=n[o],a=this._type[r];i[r]=Na(t[r],a)}return e}},{key:"stream",value:function(t){if(t){var e=this._data;return new Oa(Ai({},Yr,Tr.mark((function i(){var n,o,s,r,a,l,h;return Tr.wrap((function(i){for(;;)switch(i.prev=i.next){case 0:n=!0,o=!1,s=void 0,i.prev=3,r=Fi(t);case 5:if(n=(a=r.next()).done){i.next=14;break}if(l=a.value,null==(h=e.get(l))){i.next=11;break}return i.next=11,[l,h];case 11:n=!0,i.next=5;break;case 14:i.next=20;break;case 16:i.prev=16,i.t0=i.catch(3),o=!0,s=i.t0;case 20:i.prev=20,i.prev=21,n||null==r.return||r.return();case 23:if(i.prev=23,!o){i.next=26;break}throw s;case 26:return i.finish(23);case 27:return i.finish(20);case 28:case"end":return i.stop()}}),i,null,[[3,16,20,28],[21,,23,27]])}))))}var i;return new Oa(Ai({},Yr,va(i=Xs(this._data)).call(i,this._data)))}}]),e}(Ba),Xa=function(t){function e(t,i){var n,o;return ba(this,e),(o=ar(this,dr(e).call(this))).length=0,o._ids=new Ar,o._options=i||{},o._listener=va(n=o._onEvent).call(n,Bs(o)),o.setData(t),o}return pr(e,t),xa(e,[{key:"setData",value:function(t){if(this._data){this._data.off&&this._data.off("*",this._listener);var e=this._data.getIds({filter:En(this._options)}),i=this._data.get(e);this._ids.clear(),this.length=0,this._trigger("remove",{items:e,oldData:i})}if(null!=t){this._data=t;for(var n=this._data.getIds({filter:En(this._options)}),o=0,s=n.length;o<s;o++){var r=n[o];this._ids.add(r)}this.length=n.length,this._trigger("add",{items:n})}else this._data=new Aa;this._data.on&&this._data.on("*",this._listener)}},{key:"refresh",value:function(){for(var t=this._data.getIds({filter:En(this._options)}),e=mo(this._ids),i={},n=[],o=[],s=[],r=0,a=t.length;r<a;r++){var l=t[r];i[l]=!0,this._ids.has(l)||(n.push(l),this._ids.add(l))}for(var h=0,d=e.length;h<d;h++){var u=e[h],c=this._data.get(u);null==c?console.error("If you see this, report it please."):i[u]||(o.push(u),s.push(c),this._ids.delete(u))}this.length+=n.length-o.length,n.length&&this._trigger("add",{items:n}),o.length&&this._trigger("remove",{items:o,oldData:s})}},{key:"get",value:function(t,e){if(null==this._data)return null;var i,n=null;Fa(t)||Zo(t)?(n=t,i=e):i=t;var o=Ao({},this._options,i),s=En(this._options),r=i&&En(i);return s&&r&&(o.filter=function(t){return s(t)&&r(t)}),null==n?this._data.get(o):this._data.get(n,o)}},{key:"getIds",value:function(t){if(this._data.length){var e,i=En(this._options),n=null!=t?En(t):null;return e=n?i?function(t){return i(t)&&n(t)}:n:i,this._data.getIds({filter:e,order:t&&t.order})}return[]}},{key:"forEach",value:function(t,e){if(this._data){var i,n,o=En(this._options),s=e&&En(e);n=s?o?function(t){return o(t)&&s(t)}:s:o,mn(i=this._data).call(i,t,{filter:n,order:e&&e.order})}}},{key:"map",value:function(t,e){if(this._data){var i,n,o=En(this._options),s=e&&En(e);return n=s?o?function(t){return o(t)&&s(t)}:s:o,Ln(i=this._data).call(i,t,{filter:n,order:e&&e.order})}return[]}},{key:"getDataSet",value:function(){return this._data.getDataSet()}},{key:"stream",value:function(t){var e;return this._data.stream(t||Ai({},Yr,va(e=tr(this._ids)).call(e,this._ids)))}},{key:"dispose",value:function(){var t;(null===(t=this._data)||void 0===t?void 0:t.off)&&this._data.off("*",this._listener);var e="This data view has already been disposed of.";H(this,"_data",{get:function(){throw new Error(e)},set:function(){throw new Error(e)},configurable:!1})}},{key:"_onEvent",value:function(t,e,i){if(e&&e.items&&this._data){var n=e.items,o=[],s=[],r=[],a=[],l=[],h=[];switch(t){case"add":for(var d=0,u=n.length;d<u;d++){var c=n[d];this.get(c)&&(this._ids.add(c),o.push(c))}break;case"update":for(var p=0,m=n.length;p<m;p++){var f=n[p];this.get(f)?this._ids.has(f)?(s.push(f),l.push(e.data[p]),a.push(e.oldData[p])):(this._ids.add(f),o.push(f)):this._ids.has(f)&&(this._ids.delete(f),r.push(f),h.push(e.oldData[p]))}break;case"remove":for(var g=0,v=n.length;g<v;g++){var b=n[g];this._ids.has(b)&&(this._ids.delete(b),r.push(b),h.push(e.oldData[g]))}}this.length+=o.length-r.length,o.length&&this._trigger("add",{items:o},i),s.length&&this._trigger("update",{items:s,oldData:a,data:l},i),r.length&&this._trigger("remove",{items:r,oldData:h},i)}}}]),e}(Ba),Ta="undefined"!=typeof window&&window.moment||s;function Ya(t,e){var i=Yi(t);if(si){var n=si(t);e&&(n=En(n).call(n,(function(e){return kt(t,e).enumerable}))),i.push.apply(i,n)}return i}var Ea=/^\/?Date\((-?\d+)/i;function Ra(t,e){var i;if(void 0!==t){if(null===t)return null;if(!e)return t;if("string"!=typeof e&&!(e instanceof String))throw new Error("Type must be a string");switch(e){case"boolean":case"Boolean":return Boolean(t);case"number":case"Number":return rs(t)&&!isNaN(Date.parse(t))?s(t).valueOf():Number(t.valueOf());case"string":case"String":return String(t);case"Date":if(ss(t))return new Date(t);if(t instanceof Date)return new Date(t.valueOf());if(s.isMoment(t))return new Date(t.valueOf());if(rs(t))return(i=Ea.exec(t))?new Date(Number(i[1])):s(new Date(t)).toDate();throw new Error("Cannot convert object of type "+us(t)+" to type Date");case"Moment":if(ss(t))return s(t);if(t instanceof Date)return s(t.valueOf());if(s.isMoment(t))return s(t);if(rs(t))return i=Ea.exec(t),s(i?Number(i[1]):t);throw new Error("Cannot convert object of type "+us(t)+" to type Date");case"ISODate":if(ss(t))return new Date(t);if(t instanceof Date)return t.toISOString();if(s.isMoment(t))return t.toDate().toISOString();if(rs(t))return(i=Ea.exec(t))?new Date(Number(i[1])).toISOString():s(t).format();throw new Error("Cannot convert object of type "+us(t)+" to type ISODate");case"ASPDate":if(ss(t))return"/Date("+t+")/";if(t instanceof Date||s.isMoment(t))return"/Date("+t.valueOf()+")/";if(rs(t))return"/Date("+((i=Ea.exec(t))?new Date(Number(i[1])).valueOf():new Date(t).valueOf())+")/";throw new Error("Cannot convert object of type "+us(t)+" to type ASPDate");default:throw new Error("Unknown type ".concat(e))}}}function Ua(t){var e,i,n,o,s,r,a=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{start:"Date",end:"Date"},l=t._idProp,h=new Aa({fieldId:l}),d=Ln(e=_a(t)).call(e,(function(t){var e;return da(e=Yi(t)).call(e,(function(e,i){return e[i]=Ra(t[i],a[i]),e}),{})})).to(h);return d.all().start(),{add:function(){var e;return(e=t.getDataSet()).add.apply(e,arguments)},remove:function(){var e;return(e=t.getDataSet()).remove.apply(e,arguments)},update:function(){var e;return(e=t.getDataSet()).update.apply(e,arguments)},updateOnly:function(){var e;return(e=t.getDataSet()).updateOnly.apply(e,arguments)},forEach:va(i=mn(h)).call(i,h),get:va(n=h.get).call(n,h),getIds:va(o=h.getIds).call(o,h),off:va(s=h.off).call(s,h),on:va(r=h.on).call(r,h),get length(){return h.length},idProp:l,type:a,rawDS:t,coercedDS:h,dispose:function(){return d.stop()}}}var za=function(t){for(var e=1;e<arguments.length;e++){var i,n=null!=arguments[e]?arguments[e]:{};if(e%2)mn(i=Ya(Object(n),!0)).call(i,(function(e){Ai(t,e,n[e])}));else if(yt)ut(t,yt(n));else{var o;mn(o=Ya(Object(n))).call(o,(function(e){H(t,e,kt(n,e))}))}}return t}({},Is,{convert:Ra}),Va=ji.trim,ja=a.parseFloat,Ha=1/ja(Ei+"-0")!=-1/0?function(t){var e=Va(String(t)),i=ja(e);return 0===i&&"-"==e.charAt(0)?-0:i}:ja;V({global:!0,forced:parseFloat!=Ha},{parseFloat:Ha});var Pa=W.parseFloat,Ja=function(){function t(e,i){ba(this,t),this.options=null,this.props=null}return xa(t,[{key:"setOptions",value:function(t){t&&za.extend(this.options,t)}},{key:"redraw",value:function(){return!1}},{key:"destroy",value:function(){}},{key:"_isResized",value:function(){var t=this.props._previousWidth!==this.props.width||this.props._previousHeight!==this.props.height;return this.props._previousWidth=this.props.width,this.props._previousHeight=this.props.height,t}}]),t}();V({target:"String",proto:!0},{repeat:"".repeat||function(t){var e=String(b(this)),i="",n=K(t);if(n<0||n==1/0)throw RangeError("Wrong number of repetitions");for(;n>0;(n>>>=1)&&(e+=e))1&n&&(i+=e);return i}});var Ka=Pi("String").repeat,qa=String.prototype,$a=function(t){var e=t.repeat;return"string"==typeof t||t===qa||t instanceof String&&e===qa.repeat?Ka:e};function tl(t,e,i){if(i&&!Zo(i))return tl(t,e,[i]);if(e.hiddenDates=[],i&&1==Zo(i)){for(var n,o=0;o<i.length;o++)if(void 0===$a(i[o])){var s={};s.start=t(i[o].start).toDate().valueOf(),s.end=t(i[o].end).toDate().valueOf(),e.hiddenDates.push(s)}wr(n=e.hiddenDates).call(n,(function(t,e){return t.start-e.start}))}}function el(t,e,i){if(i&&!Zo(i))return el(t,e,[i]);if(i&&void 0!==e.domProps.centerContainer.width){tl(t,e,i);for(var n=t(e.range.start),o=t(e.range.end),s=(e.range.end-e.range.start)/e.domProps.centerContainer.width,r=0;r<i.length;r++)if(void 0!==$a(i[r])){var a=t(i[r].start),l=t(i[r].end);if("Invalid Date"==a._d)throw new Error("Supplied start date is not valid: ".concat(i[r].start));if("Invalid Date"==l._d)throw new Error("Supplied end date is not valid: ".concat(i[r].end));if(l-a>=4*s){var h=0,d=o.clone();switch($a(i[r])){case"daily":a.day()!=l.day()&&(h=1),a.dayOfYear(n.dayOfYear()),a.year(n.year()),a.subtract(7,"days"),l.dayOfYear(n.dayOfYear()),l.year(n.year()),l.subtract(7-h,"days"),d.add(1,"weeks");break;case"weekly":var u=l.diff(a,"days"),c=a.day();a.date(n.date()),a.month(n.month()),a.year(n.year()),l=a.clone(),a.day(c),l.day(c),l.add(u,"days"),a.subtract(1,"weeks"),l.subtract(1,"weeks"),d.add(1,"weeks");break;case"monthly":a.month()!=l.month()&&(h=1),a.month(n.month()),a.year(n.year()),a.subtract(1,"months"),l.month(n.month()),l.year(n.year()),l.subtract(1,"months"),l.add(h,"months"),d.add(1,"months");break;case"yearly":a.year()!=l.year()&&(h=1),a.year(n.year()),a.subtract(1,"years"),l.year(n.year()),l.subtract(1,"years"),l.add(h,"years"),d.add(1,"years");break;default:return void console.log("Wrong repeat format, allowed are: daily, weekly, monthly, yearly. Given:",$a(i[r]))}for(;a<d;)switch(e.hiddenDates.push({start:a.valueOf(),end:l.valueOf()}),$a(i[r])){case"daily":a.add(1,"days"),l.add(1,"days");break;case"weekly":a.add(1,"weeks"),l.add(1,"weeks");break;case"monthly":a.add(1,"months"),l.add(1,"months");break;case"yearly":a.add(1,"y"),l.add(1,"y");break;default:return void console.log("Wrong repeat format, allowed are: daily, weekly, monthly, yearly. Given:",$a(i[r]))}e.hiddenDates.push({start:a.valueOf(),end:l.valueOf()})}}il(e);var p=cl(e.range.start,e.hiddenDates),m=cl(e.range.end,e.hiddenDates),f=e.range.start,g=e.range.end;1==p.hidden&&(f=1==e.range.startToFront?p.startDate-1:p.endDate+1),1==m.hidden&&(g=1==e.range.endToFront?m.startDate-1:m.endDate+1),1!=p.hidden&&1!=m.hidden||e.range._applyRange(f,g)}}function il(t){for(var e,i=t.hiddenDates,n=[],o=0;o<i.length;o++)for(var s=0;s<i.length;s++)o!=s&&1!=i[s].remove&&1!=i[o].remove&&(i[s].start>=i[o].start&&i[s].end<=i[o].end?i[s].remove=!0:i[s].start>=i[o].start&&i[s].start<=i[o].end?(i[o].end=i[s].end,i[s].remove=!0):i[s].end>=i[o].start&&i[s].end<=i[o].end&&(i[o].start=i[s].start,i[s].remove=!0));for(o=0;o<i.length;o++)!0!==i[o].remove&&n.push(i[o]);t.hiddenDates=n,wr(e=t.hiddenDates).call(e,(function(t,e){return t.start-e.start}))}function nl(t,e,i){for(var n=!1,o=e.current.valueOf(),s=0;s<e.hiddenDates.length;s++){var r=e.hiddenDates[s].start,a=e.hiddenDates[s].end;if(o>=r&&o<a){n=!0;break}}if(1==n&&o<e._end.valueOf()&&o!=i){var l=t(i),h=t(a);l.year()!=h.year()?e.switchedYear=!0:l.month()!=h.month()?e.switchedMonth=!0:l.dayOfYear()!=h.dayOfYear()&&(e.switchedDay=!0),e.current=h}}function ol(t,e,i){var n;if(0==t.body.hiddenDates.length)return n=t.range.conversion(i),(e.valueOf()-n.offset)*n.scale;var o=cl(e,t.body.hiddenDates);1==o.hidden&&(e=o.startDate);var s=rl(t.body.hiddenDates,t.range.start,t.range.end);if(e<t.range.start){n=t.range.conversion(i,s);var r=al(t.body.hiddenDates,e,n.offset);return e=t.options.moment(e).toDate().valueOf(),e+=r,-(n.offset-e.valueOf())*n.scale}if(e>t.range.end){var a={start:t.range.start,end:e};return e=ll(t.options.moment,t.body.hiddenDates,a,e),n=t.range.conversion(i,s),(e.valueOf()-n.offset)*n.scale}return e=ll(t.options.moment,t.body.hiddenDates,t.range,e),n=t.range.conversion(i,s),(e.valueOf()-n.offset)*n.scale}function sl(t,e,i){if(0==t.body.hiddenDates.length){var n=t.range.conversion(i);return new Date(e/n.scale+n.offset)}var o=rl(t.body.hiddenDates,t.range.start,t.range.end),s=(t.range.end-t.range.start-o)*e/i,r=dl(t.body.hiddenDates,t.range,s);return new Date(r+s+t.range.start)}function rl(t,e,i){for(var n=0,o=0;o<t.length;o++){var s=t[o].start,r=t[o].end;s>=e&&r<i&&(n+=r-s)}return n}function al(t,e,i){for(var n=0,o=0;o<t.length;o++){var s=t[o].start,r=t[o].end;s>=e&&r<=i&&(n+=r-s)}return n}function ll(t,e,i,n){return n=t(n).toDate().valueOf(),n-=hl(t,e,i,n)}function hl(t,e,i,n){var o=0;n=t(n).toDate().valueOf();for(var s=0;s<e.length;s++){var r=e[s].start,a=e[s].end;r>=i.start&&a<i.end&&n>=a&&(o+=a-r)}return o}function dl(t,e,i){for(var n=0,o=0,s=e.start,r=0;r<t.length;r++){var a=t[r].start,l=t[r].end;if(a>=e.start&&l<e.end){if(o+=a-s,s=l,o>=i)break;n+=l-a}}return n}function ul(t,e,i,n){var o=cl(e,t);return 1==o.hidden?i<0?1==n?o.startDate-(o.endDate-e)-1:o.startDate-1:1==n?o.endDate+(e-o.startDate)+1:o.endDate+1:e}function cl(t,e){for(var i=0;i<e.length;i++){var n=e[i].start,o=e[i].end;if(t>=n&&t<o)return{hidden:!0,startDate:n,endDate:o}}return{hidden:!1,startDate:n,endDate:o}}var pl=Object.freeze({__proto__:null,convertHiddenOptions:tl,updateHiddenDates:el,removeDuplicates:il,printDates:function(t){for(var e=0;e<t.length;e++)console.log(e,new Date(t[e].start),new Date(t[e].end),t[e].start,t[e].end,t[e].remove)},stepOverHiddenDates:nl,toScreen:ol,toTime:sl,getHiddenDurationBetween:rl,getHiddenDurationBeforeStart:al,correctTimeForHidden:ll,getHiddenDurationBefore:hl,getAccumulatedHiddenDuration:dl,snapAwayFromHidden:ul,getIsHidden:cl}),ml=function(t){function e(t,i){var n,o,s,r,a,l,h,d;ba(this,e),d=ar(this,dr(e).call(this));var u=Ta().hours(0).minutes(0).seconds(0).milliseconds(0),c=u.clone().add(-3,"days").valueOf(),p=u.clone().add(3,"days").valueOf();return d.millisecondsPerPixelCache=void 0,void 0===i?(d.start=c,d.end=p):(d.start=i.start||c,d.end=i.end||p),d.rolling=!1,d.body=t,d.deltaDifference=0,d.scaleOffset=0,d.startToFront=!1,d.endToFront=!0,d.defaultOptions={rtl:!1,start:null,end:null,moment:Ta,direction:"horizontal",moveable:!0,zoomable:!0,min:null,max:null,zoomMin:10,zoomMax:31536e10,rollingMode:{follow:!1,offset:.5}},d.options=za.extend({},d.defaultOptions),d.props={touch:{}},d.animationTimer=null,d.body.emitter.on("panstart",va(n=d._onDragStart).call(n,Bs(d))),d.body.emitter.on("panmove",va(o=d._onDrag).call(o,Bs(d))),d.body.emitter.on("panend",va(s=d._onDragEnd).call(s,Bs(d))),d.body.emitter.on("mousewheel",va(r=d._onMouseWheel).call(r,Bs(d))),d.body.emitter.on("touch",va(a=d._onTouch).call(a,Bs(d))),d.body.emitter.on("pinch",va(l=d._onPinch).call(l,Bs(d))),d.body.dom.rollingModeBtn.addEventListener("click",va(h=d.startRolling).call(h,Bs(d))),d.setOptions(i),d}return pr(e,t),xa(e,[{key:"setOptions",value:function(t){if(t){za.selectiveExtend(["animation","direction","min","max","zoomMin","zoomMax","moveable","zoomable","moment","activate","hiddenDates","zoomKey","zoomFriction","rtl","showCurrentTime","rollingMode","horizontalScroll"],this.options,t),t.rollingMode&&t.rollingMode.follow&&this.startRolling(),("start"in t||"end"in t)&&this.setRange(t.start,t.end)}}},{key:"startRolling",value:function(){var t=this;!function e(){t.stopRolling(),t.rolling=!0;var i=t.end-t.start,n=za.convert(new Date,"Date").valueOf(),o=t.options.rollingMode&&t.options.rollingMode.offset||.5,s=n-i*o,r=n+i*(1-o);t.setRange(s,r,{animation:!1}),(i=1/t.conversion(t.body.domProps.center.width).scale/10)<30&&(i=30),i>1e3&&(i=1e3),t.body.dom.rollingModeBtn.style.visibility="hidden",t.currentTimeTimer=qr(e,i)}()}},{key:"stopRolling",value:function(){void 0!==this.currentTimeTimer&&(clearTimeout(this.currentTimeTimer),this.rolling=!1,this.body.dom.rollingModeBtn.style.visibility="visible")}},{key:"setRange",value:function(t,e,i,n,o){i||(i={}),!0!==i.byUser&&(i.byUser=!1);var s=this,r=null!=t?za.convert(t,"Date").valueOf():null,a=null!=e?za.convert(e,"Date").valueOf():null;if(this._cancelAnimation(),this.millisecondsPerPixelCache=void 0,i.animation){var l,h=this.start,d=this.end,u="object"===zo(i.animation)&&"duration"in i.animation?i.animation.duration:500,c="object"===zo(i.animation)&&"easingFunction"in i.animation?i.animation.easingFunction:"easeInOutQuad",p=za.easingFunctions[c];if(!p)throw new Error(Pn(l="Unknown easing function ".concat(Vs(c),". Choose from: ")).call(l,Yi(za.easingFunctions).join(", ")));var m=Jo(),f=!1;return function t(){if(!s.props.touch.dragging){var e=Jo()-m,l=p(e/u),c=e>u,v=c||null===r?r:h+(r-h)*l,b=c||null===a?a:d+(a-d)*l;g=s._applyRange(v,b),el(s.options.moment,s.body,s.options.hiddenDates),f=f||g;var y={start:new Date(s.start),end:new Date(s.end),byUser:i.byUser,event:i.event};if(o&&o(l,g,c),g&&s.body.emitter.emit("rangechange",y),c){if(f&&(s.body.emitter.emit("rangechanged",y),n))return n()}else s.animationTimer=qr(t,20)}}()}var g=this._applyRange(r,a);if(el(this.options.moment,this.body,this.options.hiddenDates),g){var v={start:new Date(this.start),end:new Date(this.end),byUser:i.byUser,event:i.event};if(this.body.emitter.emit("rangechange",v),clearTimeout(s.timeoutID),s.timeoutID=qr((function(){s.body.emitter.emit("rangechanged",v)}),200),n)return n()}}},{key:"getMillisecondsPerPixel",value:function(){return void 0===this.millisecondsPerPixelCache&&(this.millisecondsPerPixelCache=(this.end-this.start)/this.body.dom.center.clientWidth),this.millisecondsPerPixelCache}},{key:"_cancelAnimation",value:function(){this.animationTimer&&(clearTimeout(this.animationTimer),this.animationTimer=null)}},{key:"_applyRange",value:function(t,e){var i,n=null!=t?za.convert(t,"Date").valueOf():this.start,o=null!=e?za.convert(e,"Date").valueOf():this.end,s=null!=this.options.max?za.convert(this.options.max,"Date").valueOf():null,r=null!=this.options.min?za.convert(this.options.min,"Date").valueOf():null;if(isNaN(n)||null===n)throw new Error('Invalid start "'.concat(t,'"'));if(isNaN(o)||null===o)throw new Error('Invalid end "'.concat(e,'"'));if(o<n&&(o=n),null!==r&&n<r&&(n+=i=r-n,o+=i,null!=s&&o>s&&(o=s)),null!==s&&o>s&&(n-=i=o-s,o-=i,null!=r&&n<r&&(n=r)),null!==this.options.zoomMin){var a=Pa(this.options.zoomMin);if(a<0&&(a=0),o-n<a){this.end-this.start===a&&n>=this.start-.5&&o<=this.end?(n=this.start,o=this.end):(n-=(i=a-(o-n))/2,o+=i/2)}}if(null!==this.options.zoomMax){var l=Pa(this.options.zoomMax);l<0&&(l=0),o-n>l&&(this.end-this.start===l&&n<this.start&&o>this.end?(n=this.start,o=this.end):(n+=(i=o-n-l)/2,o-=i/2))}var h=this.start!=n||this.end!=o;return n>=this.start&&n<=this.end||o>=this.start&&o<=this.end||this.start>=n&&this.start<=o||this.end>=n&&this.end<=o||this.body.emitter.emit("checkRangedItems"),this.start=n,this.end=o,h}},{key:"getRange",value:function(){return{start:this.start,end:this.end}}},{key:"conversion",value:function(t,i){return e.conversion(this.start,this.end,t,i)}},{key:"_onDragStart",value:function(t){this.deltaDifference=0,this.previousDelta=0,this.options.moveable&&this._isInsideRange(t)&&this.props.touch.allowDragging&&(this.stopRolling(),this.props.touch.start=this.start,this.props.touch.end=this.end,this.props.touch.dragging=!0,this.body.dom.root&&(this.body.dom.root.style.cursor="move"))}},{key:"_onDrag",value:function(t){if(t&&this.props.touch.dragging&&this.options.moveable&&this.props.touch.allowDragging){var e=this.options.direction;fl(e);var i="horizontal"==e?t.deltaX:t.deltaY;i-=this.deltaDifference;var n=this.props.touch.end-this.props.touch.start;n-=rl(this.body.hiddenDates,this.start,this.end);var o,s="horizontal"==e?this.body.domProps.center.width:this.body.domProps.center.height;o=this.options.rtl?i/s*n:-i/s*n;var r=this.props.touch.start+o,a=this.props.touch.end+o,l=ul(this.body.hiddenDates,r,this.previousDelta-i,!0),h=ul(this.body.hiddenDates,a,this.previousDelta-i,!0);if(l!=r||h!=a)return this.deltaDifference+=i,this.props.touch.start=l,this.props.touch.end=h,void this._onDrag(t);this.previousDelta=i,this._applyRange(r,a);var d=new Date(this.start),u=new Date(this.end);this.body.emitter.emit("rangechange",{start:d,end:u,byUser:!0,event:t}),this.body.emitter.emit("panmove")}}},{key:"_onDragEnd",value:function(t){this.props.touch.dragging&&this.options.moveable&&this.props.touch.allowDragging&&(this.props.touch.dragging=!1,this.body.dom.root&&(this.body.dom.root.style.cursor="auto"),this.body.emitter.emit("rangechanged",{start:new Date(this.start),end:new Date(this.end),byUser:!0,event:t}))}},{key:"_onMouseWheel",value:function(t){var e=0;if(t.wheelDelta?e=t.wheelDelta/120:t.detail?e=-t.detail/3:t.deltaY&&(e=-t.deltaY/3),!(this.options.zoomKey&&!t[this.options.zoomKey]&&this.options.zoomable||!this.options.zoomable&&this.options.moveable)&&this.options.zoomable&&this.options.moveable&&this._isInsideRange(t)&&e){var i,n,o=this.options.zoomFriction||5;if(i=e<0?1-e/o:1/(1+e/o),this.rolling){var s=this.options.rollingMode&&this.options.rollingMode.offset||.5;n=this.start+(this.end-this.start)*s}else{var r=this.getPointer({x:t.clientX,y:t.clientY},this.body.dom.center);n=this._pointerToDate(r)}this.zoom(i,n,e,t),t.preventDefault()}}},{key:"_onTouch",value:function(t){this.props.touch.start=this.start,this.props.touch.end=this.end,this.props.touch.allowDragging=!0,this.props.touch.center=null,this.props.touch.centerDate=null,this.scaleOffset=0,this.deltaDifference=0,za.preventDefault(t)}},{key:"_onPinch",value:function(t){if(this.options.zoomable&&this.options.moveable){za.preventDefault(t),this.props.touch.allowDragging=!1,this.props.touch.center||(this.props.touch.center=this.getPointer(t.center,this.body.dom.center),this.props.touch.centerDate=this._pointerToDate(this.props.touch.center)),this.stopRolling();var e=1/(t.scale+this.scaleOffset),i=this.props.touch.centerDate,n=rl(this.body.hiddenDates,this.start,this.end),o=hl(this.options.moment,this.body.hiddenDates,this,i),s=n-o,r=i-o+(this.props.touch.start-(i-o))*e,a=i+s+(this.props.touch.end-(i+s))*e;this.startToFront=1-e<=0,this.endToFront=e-1<=0;var l=ul(this.body.hiddenDates,r,1-e,!0),h=ul(this.body.hiddenDates,a,e-1,!0);l==r&&h==a||(this.props.touch.start=l,this.props.touch.end=h,this.scaleOffset=1-t.scale,r=l,a=h);var d={animation:!1,byUser:!0,event:t};this.setRange(r,a,d),this.startToFront=!1,this.endToFront=!0}}},{key:"_isInsideRange",value:function(t){var e=t.center?t.center.x:t.clientX,i=this.body.dom.centerContainer.getBoundingClientRect(),n=this.options.rtl?e-i.left:i.right-e,o=this.body.util.toTime(n);return o>=this.start&&o<=this.end}},{key:"_pointerToDate",value:function(t){var e,i=this.options.direction;if(fl(i),"horizontal"==i)return this.body.util.toTime(t.x).valueOf();var n=this.body.domProps.center.height;return e=this.conversion(n),t.y/e.scale+e.offset}},{key:"getPointer",value:function(t,e){var i=e.getBoundingClientRect();return this.options.rtl?{x:i.right-t.x,y:t.y-i.top}:{x:t.x-i.left,y:t.y-i.top}}},{key:"zoom",value:function(t,e,i,n){null==e&&(e=(this.start+this.end)/2);var o=rl(this.body.hiddenDates,this.start,this.end),s=hl(this.options.moment,this.body.hiddenDates,this,e),r=o-s,a=e-s+(this.start-(e-s))*t,l=e+r+(this.end-(e+r))*t;this.startToFront=!(i>0),this.endToFront=!(-i>0);var h=ul(this.body.hiddenDates,a,i,!0),d=ul(this.body.hiddenDates,l,-i,!0);h==a&&d==l||(a=h,l=d);var u={animation:!1,byUser:!0,event:n};this.setRange(a,l,u),this.startToFront=!1,this.endToFront=!0}},{key:"move",value:function(t){var e=this.end-this.start,i=this.start+e*t,n=this.end+e*t;this.start=i,this.end=n}},{key:"moveTo",value:function(t){var e=(this.start+this.end)/2-t,i=this.start-e,n=this.end-e;this.setRange(i,n,{animation:!1,byUser:!0,event:null})}}],[{key:"conversion",value:function(t,e,i,n){return void 0===n&&(n=0),0!=i&&e-t!=0?{offset:t,scale:i/(e-t-n)}:{offset:0,scale:1}}}]),e}(Ja);function fl(t){if("horizontal"!=t&&"vertical"!=t)throw new TypeError('Unknown direction "'.concat(t,'". Choose "horizontal" or "vertical".'))}var gl=W.setInterval,vl=n((function(t){function e(t){if(t)return function(t){for(var i in e.prototype)t[i]=e.prototype[i];return t}(t)}t.exports=e,e.prototype.on=e.prototype.addEventListener=function(t,e){return this._callbacks=this._callbacks||{},(this._callbacks["$"+t]=this._callbacks["$"+t]||[]).push(e),this},e.prototype.once=function(t,e){function i(){this.off(t,i),e.apply(this,arguments)}return i.fn=e,this.on(t,i),this},e.prototype.off=e.prototype.removeListener=e.prototype.removeAllListeners=e.prototype.removeEventListener=function(t,e){if(this._callbacks=this._callbacks||{},0==arguments.length)return this._callbacks={},this;var i,n=this._callbacks["$"+t];if(!n)return this;if(1==arguments.length)return delete this._callbacks["$"+t],this;for(var o=0;o<n.length;o++)if((i=n[o])===e||i.fn===e){n.splice(o,1);break}return 0===n.length&&delete this._callbacks["$"+t],this},e.prototype.emit=function(t){this._callbacks=this._callbacks||{};for(var e=new Array(arguments.length-1),i=this._callbacks["$"+t],n=1;n<arguments.length;n++)e[n-1]=arguments[n];if(i){n=0;for(var o=(i=i.slice(0)).length;n<o;++n)i[n].apply(this,e)}return this},e.prototype.listeners=function(t){return this._callbacks=this._callbacks||{},this._callbacks["$"+t]||[]},e.prototype.hasListeners=function(t){return!!this.listeners(t).length}})),bl=n((function(t,e){var i;t.exports=(i=null,function t(e,n){var o=n||{preventDefault:!1};if(e.Manager){var s=e,r=function(e,i){var n=Object.create(o);return i&&s.assign(n,i),t(new s(e,n),n)};return s.assign(r,s),r.Manager=function(e,i){var n=Object.create(o);return i&&s.assign(n,i),t(new s.Manager(e,n),n)},r}var a=Object.create(e),l=e.element;function h(t){return t.match(/[^ ]+/g)}function d(t){if("hammer.input"!==t.type){if(t.srcEvent._handled||(t.srcEvent._handled={}),t.srcEvent._handled[t.type])return;t.srcEvent._handled[t.type]=!0}var e=!1;t.stopPropagation=function(){e=!0};var n=t.srcEvent.stopPropagation.bind(t.srcEvent);"function"==typeof n&&(t.srcEvent.stopPropagation=function(){n(),t.stopPropagation()}),t.firstTarget=i;for(var o=i;o&&!e;){var s=o.hammer;if(s)for(var r,a=0;a<s.length;a++)if(r=s[a]._handlers[t.type])for(var l=0;l<r.length&&!e;l++)r[l](t);o=o.parentNode}}return l.hammer||(l.hammer=[]),l.hammer.push(a),e.on("hammer.input",(function(t){!0!==o.preventDefault&&o.preventDefault!==t.pointerType||t.preventDefault(),t.isFirst&&(i=t.target)})),a._handlers={},a.on=function(t,i){return h(t).forEach((function(t){var n=a._handlers[t];n||(a._handlers[t]=n=[],e.on(t,d)),n.push(i)})),a},a.off=function(t,i){return h(t).forEach((function(t){var n=a._handlers[t];n&&((n=i?n.filter((function(t){return t!==i})):[]).length>0?a._handlers[t]=n:(e.off(t,d),delete a._handlers[t]))})),a},a.emit=function(t,n){i=n.target,e.emit(t,n)},a.destroy=function(){var t=e.element.hammer,i=t.indexOf(a);-1!==i&&t.splice(i,1),t.length||delete e.element.hammer,a._handlers={},e.destroy()},a})}));
-/*! Hammer.JS - v2.0.17-rc - 2019-12-16
-	 * http://naver.github.io/egjs
-	 *
-	 * Forked By Naver egjs
-	 * Copyright (c) hammerjs
-	 * Licensed under the MIT license */
-function yl(){return(yl=Object.assign||function(t){for(var e=1;e<arguments.length;e++){var i=arguments[e];for(var n in i)Object.prototype.hasOwnProperty.call(i,n)&&(t[n]=i[n])}return t}).apply(this,arguments)}function xl(t,e){t.prototype=Object.create(e.prototype),t.prototype.constructor=t,t.__proto__=e}function wl(t){if(void 0===t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return t}var _l,kl="function"!=typeof Object.assign?function(t){if(null==t)throw new TypeError("Cannot convert undefined or null to object");for(var e=Object(t),i=1;i<arguments.length;i++){var n=arguments[i];if(null!=n)for(var o in n)n.hasOwnProperty(o)&&(e[o]=n[o])}return e}:Object.assign,Dl=["","webkit","Moz","MS","ms","o"],Il="undefined"==typeof document?{style:{}}:document.createElement("div"),Cl=Math.round,Ll=Math.abs,Gl=Date.now;function Ml(t,e){for(var i,n,o=e[0].toUpperCase()+e.slice(1),s=0;s<Dl.length;){if((n=(i=Dl[s])?i+o:e)in t)return n;s++}}_l="undefined"==typeof window?{}:window;var Sl=Ml(Il.style,"touchAction"),Ql=void 0!==Sl;var Nl=function(){if(!Ql)return!1;var t={},e=_l.CSS&&_l.CSS.supports;return["auto","manipulation","pan-y","pan-x","pan-x pan-y","none"].forEach((function(i){return t[i]=!e||_l.CSS.supports("touch-action",i)})),t}(),Fl="ontouchstart"in _l,Zl=void 0!==Ml(_l,"PointerEvent"),Bl=Fl&&/mobile|tablet|ip(ad|hone|od)|android/i.test(navigator.userAgent),Ol=["x","y"],Wl=["clientX","clientY"];function Al(t,e,i){var n;if(t)if(t.forEach)t.forEach(e,i);else if(void 0!==t.length)for(n=0;n<t.length;)e.call(i,t[n],n,t),n++;else for(n in t)t.hasOwnProperty(n)&&e.call(i,t[n],n,t)}function Xl(t,e){return"function"==typeof t?t.apply(e&&e[0]||void 0,e):t}function Tl(t,e){return t.indexOf(e)>-1}var Yl=function(){function t(t,e){this.manager=t,this.set(e)}var e=t.prototype;return e.set=function(t){"compute"===t&&(t=this.compute()),Ql&&this.manager.element.style&&Nl[t]&&(this.manager.element.style[Sl]=t),this.actions=t.toLowerCase().trim()},e.update=function(){this.set(this.manager.options.touchAction)},e.compute=function(){var t=[];return Al(this.manager.recognizers,(function(e){Xl(e.options.enable,[e])&&(t=t.concat(e.getTouchAction()))})),function(t){if(Tl(t,"none"))return"none";var e=Tl(t,"pan-x"),i=Tl(t,"pan-y");return e&&i?"none":e||i?e?"pan-x":"pan-y":Tl(t,"manipulation")?"manipulation":"auto"}(t.join(" "))},e.preventDefaults=function(t){var e=t.srcEvent,i=t.offsetDirection;if(this.manager.session.prevented)e.preventDefault();else{var n=this.actions,o=Tl(n,"none")&&!Nl.none,s=Tl(n,"pan-y")&&!Nl["pan-y"],r=Tl(n,"pan-x")&&!Nl["pan-x"];if(o){var a=1===t.pointers.length,l=t.distance<2,h=t.deltaTime<250;if(a&&l&&h)return}if(!r||!s)return o||s&&6&i||r&&24&i?this.preventSrc(e):void 0}},e.preventSrc=function(t){this.manager.session.prevented=!0,t.preventDefault()},t}();function El(t,e){for(;t;){if(t===e)return!0;t=t.parentNode}return!1}function Rl(t){var e=t.length;if(1===e)return{x:Cl(t[0].clientX),y:Cl(t[0].clientY)};for(var i=0,n=0,o=0;o<e;)i+=t[o].clientX,n+=t[o].clientY,o++;return{x:Cl(i/e),y:Cl(n/e)}}function Ul(t){for(var e=[],i=0;i<t.pointers.length;)e[i]={clientX:Cl(t.pointers[i].clientX),clientY:Cl(t.pointers[i].clientY)},i++;return{timeStamp:Gl(),pointers:e,center:Rl(e),deltaX:t.deltaX,deltaY:t.deltaY}}function zl(t,e,i){i||(i=Ol);var n=e[i[0]]-t[i[0]],o=e[i[1]]-t[i[1]];return Math.sqrt(n*n+o*o)}function Vl(t,e,i){i||(i=Ol);var n=e[i[0]]-t[i[0]],o=e[i[1]]-t[i[1]];return 180*Math.atan2(o,n)/Math.PI}function jl(t,e){return t===e?1:Ll(t)>=Ll(e)?t<0?2:4:e<0?8:16}function Hl(t,e,i){return{x:e/t||0,y:i/t||0}}function Pl(t,e){var i=t.session,n=e.pointers,o=n.length;i.firstInput||(i.firstInput=Ul(e)),o>1&&!i.firstMultiple?i.firstMultiple=Ul(e):1===o&&(i.firstMultiple=!1);var s=i.firstInput,r=i.firstMultiple,a=r?r.center:s.center,l=e.center=Rl(n);e.timeStamp=Gl(),e.deltaTime=e.timeStamp-s.timeStamp,e.angle=Vl(a,l),e.distance=zl(a,l),function(t,e){var i=e.center,n=t.offsetDelta||{},o=t.prevDelta||{},s=t.prevInput||{};1!==e.eventType&&4!==s.eventType||(o=t.prevDelta={x:s.deltaX||0,y:s.deltaY||0},n=t.offsetDelta={x:i.x,y:i.y}),e.deltaX=o.x+(i.x-n.x),e.deltaY=o.y+(i.y-n.y)}(i,e),e.offsetDirection=jl(e.deltaX,e.deltaY);var h,d,u=Hl(e.deltaTime,e.deltaX,e.deltaY);e.overallVelocityX=u.x,e.overallVelocityY=u.y,e.overallVelocity=Ll(u.x)>Ll(u.y)?u.x:u.y,e.scale=r?(h=r.pointers,zl((d=n)[0],d[1],Wl)/zl(h[0],h[1],Wl)):1,e.rotation=r?function(t,e){return Vl(e[1],e[0],Wl)+Vl(t[1],t[0],Wl)}(r.pointers,n):0,e.maxPointers=i.prevInput?e.pointers.length>i.prevInput.maxPointers?e.pointers.length:i.prevInput.maxPointers:e.pointers.length,function(t,e){var i,n,o,s,r=t.lastInterval||e,a=e.timeStamp-r.timeStamp;if(8!==e.eventType&&(a>25||void 0===r.velocity)){var l=e.deltaX-r.deltaX,h=e.deltaY-r.deltaY,d=Hl(a,l,h);n=d.x,o=d.y,i=Ll(d.x)>Ll(d.y)?d.x:d.y,s=jl(l,h),t.lastInterval=e}else i=r.velocity,n=r.velocityX,o=r.velocityY,s=r.direction;e.velocity=i,e.velocityX=n,e.velocityY=o,e.direction=s}(i,e);var c,p=t.element,m=e.srcEvent;El(c=m.composedPath?m.composedPath()[0]:m.path?m.path[0]:m.target,p)&&(p=c),e.target=p}function Jl(t,e,i){var n=i.pointers.length,o=i.changedPointers.length,s=1&e&&n-o==0,r=12&e&&n-o==0;i.isFirst=!!s,i.isFinal=!!r,s&&(t.session={}),i.eventType=e,Pl(t,i),t.emit("hammer.input",i),t.recognize(i),t.session.prevInput=i}function Kl(t){return t.trim().split(/\s+/g)}function ql(t,e,i){Al(Kl(e),(function(e){t.addEventListener(e,i,!1)}))}function $l(t,e,i){Al(Kl(e),(function(e){t.removeEventListener(e,i,!1)}))}function th(t){var e=t.ownerDocument||t;return e.defaultView||e.parentWindow||window}var eh=function(){function t(t,e){var i=this;this.manager=t,this.callback=e,this.element=t.element,this.target=t.options.inputTarget,this.domHandler=function(e){Xl(t.options.enable,[t])&&i.handler(e)},this.init()}var e=t.prototype;return e.handler=function(){},e.init=function(){this.evEl&&ql(this.element,this.evEl,this.domHandler),this.evTarget&&ql(this.target,this.evTarget,this.domHandler),this.evWin&&ql(th(this.element),this.evWin,this.domHandler)},e.destroy=function(){this.evEl&&$l(this.element,this.evEl,this.domHandler),this.evTarget&&$l(this.target,this.evTarget,this.domHandler),this.evWin&&$l(th(this.element),this.evWin,this.domHandler)},t}();function ih(t,e,i){if(t.indexOf&&!i)return t.indexOf(e);for(var n=0;n<t.length;){if(i&&t[n][i]==e||!i&&t[n]===e)return n;n++}return-1}var nh={pointerdown:1,pointermove:2,pointerup:4,pointercancel:8,pointerout:8},oh={2:"touch",3:"pen",4:"mouse",5:"kinect"},sh="pointerdown",rh="pointermove pointerup pointercancel";_l.MSPointerEvent&&!_l.PointerEvent&&(sh="MSPointerDown",rh="MSPointerMove MSPointerUp MSPointerCancel");var ah=function(t){function e(){var i,n=e.prototype;return n.evEl=sh,n.evWin=rh,(i=t.apply(this,arguments)||this).store=i.manager.session.pointerEvents=[],i}return xl(e,t),e.prototype.handler=function(t){var e=this.store,i=!1,n=t.type.toLowerCase().replace("ms",""),o=nh[n],s=oh[t.pointerType]||t.pointerType,r="touch"===s,a=ih(e,t.pointerId,"pointerId");1&o&&(0===t.button||r)?a<0&&(e.push(t),a=e.length-1):12&o&&(i=!0),a<0||(e[a]=t,this.callback(this.manager,o,{pointers:e,changedPointers:[t],pointerType:s,srcEvent:t}),i&&e.splice(a,1))},e}(eh);function lh(t){return Array.prototype.slice.call(t,0)}function hh(t,e,i){for(var n=[],o=[],s=0;s<t.length;){var r=e?t[s][e]:t[s];ih(o,r)<0&&n.push(t[s]),o[s]=r,s++}return i&&(n=e?n.sort((function(t,i){return t[e]>i[e]})):n.sort()),n}var dh={touchstart:1,touchmove:2,touchend:4,touchcancel:8},uh=function(t){function e(){var i;return e.prototype.evTarget="touchstart touchmove touchend touchcancel",(i=t.apply(this,arguments)||this).targetIds={},i}return xl(e,t),e.prototype.handler=function(t){var e=dh[t.type],i=ch.call(this,t,e);i&&this.callback(this.manager,e,{pointers:i[0],changedPointers:i[1],pointerType:"touch",srcEvent:t})},e}(eh);function ch(t,e){var i,n,o=lh(t.touches),s=this.targetIds;if(3&e&&1===o.length)return s[o[0].identifier]=!0,[o,o];var r=lh(t.changedTouches),a=[],l=this.target;if(n=o.filter((function(t){return El(t.target,l)})),1===e)for(i=0;i<n.length;)s[n[i].identifier]=!0,i++;for(i=0;i<r.length;)s[r[i].identifier]&&a.push(r[i]),12&e&&delete s[r[i].identifier],i++;return a.length?[hh(n.concat(a),"identifier",!0),a]:void 0}var ph={mousedown:1,mousemove:2,mouseup:4},mh=function(t){function e(){var i,n=e.prototype;return n.evEl="mousedown",n.evWin="mousemove mouseup",(i=t.apply(this,arguments)||this).pressed=!1,i}return xl(e,t),e.prototype.handler=function(t){var e=ph[t.type];1&e&&0===t.button&&(this.pressed=!0),2&e&&1!==t.which&&(e=4),this.pressed&&(4&e&&(this.pressed=!1),this.callback(this.manager,e,{pointers:[t],changedPointers:[t],pointerType:"mouse",srcEvent:t}))},e}(eh);function fh(t){var e=t.changedPointers[0];if(e.identifier===this.primaryTouch){var i={x:e.clientX,y:e.clientY},n=this.lastTouches;this.lastTouches.push(i);setTimeout((function(){var t=n.indexOf(i);t>-1&&n.splice(t,1)}),2500)}}function gh(t,e){1&t?(this.primaryTouch=e.changedPointers[0].identifier,fh.call(this,e)):12&t&&fh.call(this,e)}function vh(t){for(var e=t.srcEvent.clientX,i=t.srcEvent.clientY,n=0;n<this.lastTouches.length;n++){var o=this.lastTouches[n],s=Math.abs(e-o.x),r=Math.abs(i-o.y);if(s<=25&&r<=25)return!0}return!1}var bh=function(){return function(t){function e(e,i){var n;return(n=t.call(this,e,i)||this).handler=function(t,e,i){var o="touch"===i.pointerType,s="mouse"===i.pointerType;if(!(s&&i.sourceCapabilities&&i.sourceCapabilities.firesTouchEvents)){if(o)gh.call(wl(wl(n)),e,i);else if(s&&vh.call(wl(wl(n)),i))return;n.callback(t,e,i)}},n.touch=new uh(n.manager,n.handler),n.mouse=new mh(n.manager,n.handler),n.primaryTouch=null,n.lastTouches=[],n}return xl(e,t),e.prototype.destroy=function(){this.touch.destroy(),this.mouse.destroy()},e}(eh)}();function yh(t,e,i){return!!Array.isArray(t)&&(Al(t,i[e],i),!0)}var xh=1;function wh(t,e){var i=e.manager;return i?i.get(t):t}function _h(t){return 16&t?"cancel":8&t?"end":4&t?"move":2&t?"start":""}var kh=function(){function t(t){void 0===t&&(t={}),this.options=yl({enable:!0},t),this.id=xh++,this.manager=null,this.state=1,this.simultaneous={},this.requireFail=[]}var e=t.prototype;return e.set=function(t){return kl(this.options,t),this.manager&&this.manager.touchAction.update(),this},e.recognizeWith=function(t){if(yh(t,"recognizeWith",this))return this;var e=this.simultaneous;return e[(t=wh(t,this)).id]||(e[t.id]=t,t.recognizeWith(this)),this},e.dropRecognizeWith=function(t){return yh(t,"dropRecognizeWith",this)||(t=wh(t,this),delete this.simultaneous[t.id]),this},e.requireFailure=function(t){if(yh(t,"requireFailure",this))return this;var e=this.requireFail;return-1===ih(e,t=wh(t,this))&&(e.push(t),t.requireFailure(this)),this},e.dropRequireFailure=function(t){if(yh(t,"dropRequireFailure",this))return this;t=wh(t,this);var e=ih(this.requireFail,t);return e>-1&&this.requireFail.splice(e,1),this},e.hasRequireFailures=function(){return this.requireFail.length>0},e.canRecognizeWith=function(t){return!!this.simultaneous[t.id]},e.emit=function(t){var e=this,i=this.state;function n(i){e.manager.emit(i,t)}i<8&&n(e.options.event+_h(i)),n(e.options.event),t.additionalEvent&&n(t.additionalEvent),i>=8&&n(e.options.event+_h(i))},e.tryEmit=function(t){if(this.canEmit())return this.emit(t);this.state=32},e.canEmit=function(){for(var t=0;t<this.requireFail.length;){if(!(33&this.requireFail[t].state))return!1;t++}return!0},e.recognize=function(t){var e=kl({},t);if(!Xl(this.options.enable,[this,e]))return this.reset(),void(this.state=32);56&this.state&&(this.state=1),this.state=this.process(e),30&this.state&&this.tryEmit(e)},e.process=function(t){},e.getTouchAction=function(){},e.reset=function(){},t}(),Dh=function(t){function e(e){var i;return void 0===e&&(e={}),(i=t.call(this,yl({event:"tap",pointers:1,taps:1,interval:300,time:250,threshold:9,posThreshold:10},e))||this).pTime=!1,i.pCenter=!1,i._timer=null,i._input=null,i.count=0,i}xl(e,t);var i=e.prototype;return i.getTouchAction=function(){return["manipulation"]},i.process=function(t){var e=this,i=this.options,n=t.pointers.length===i.pointers,o=t.distance<i.threshold,s=t.deltaTime<i.time;if(this.reset(),1&t.eventType&&0===this.count)return this.failTimeout();if(o&&s&&n){if(4!==t.eventType)return this.failTimeout();var r=!this.pTime||t.timeStamp-this.pTime<i.interval,a=!this.pCenter||zl(this.pCenter,t.center)<i.posThreshold;if(this.pTime=t.timeStamp,this.pCenter=t.center,a&&r?this.count+=1:this.count=1,this._input=t,0===this.count%i.taps)return this.hasRequireFailures()?(this._timer=setTimeout((function(){e.state=8,e.tryEmit()}),i.interval),2):8}return 32},i.failTimeout=function(){var t=this;return this._timer=setTimeout((function(){t.state=32}),this.options.interval),32},i.reset=function(){clearTimeout(this._timer)},i.emit=function(){8===this.state&&(this._input.tapCount=this.count,this.manager.emit(this.options.event,this._input))},e}(kh),Ih=function(t){function e(e){return void 0===e&&(e={}),t.call(this,yl({pointers:1},e))||this}xl(e,t);var i=e.prototype;return i.attrTest=function(t){var e=this.options.pointers;return 0===e||t.pointers.length===e},i.process=function(t){var e=this.state,i=t.eventType,n=6&e,o=this.attrTest(t);return n&&(8&i||!o)?16|e:n||o?4&i?8|e:2&e?4|e:2:32},e}(kh);function Ch(t){return 16===t?"down":8===t?"up":2===t?"left":4===t?"right":""}var Lh=function(t){function e(e){var i;return void 0===e&&(e={}),(i=t.call(this,yl({event:"pan",threshold:10,pointers:1,direction:30},e))||this).pX=null,i.pY=null,i}xl(e,t);var i=e.prototype;return i.getTouchAction=function(){var t=this.options.direction,e=[];return 6&t&&e.push("pan-y"),24&t&&e.push("pan-x"),e},i.directionTest=function(t){var e=this.options,i=!0,n=t.distance,o=t.direction,s=t.deltaX,r=t.deltaY;return o&e.direction||(6&e.direction?(o=0===s?1:s<0?2:4,i=s!==this.pX,n=Math.abs(t.deltaX)):(o=0===r?1:r<0?8:16,i=r!==this.pY,n=Math.abs(t.deltaY))),t.direction=o,i&&n>e.threshold&&o&e.direction},i.attrTest=function(t){return Ih.prototype.attrTest.call(this,t)&&(2&this.state||!(2&this.state)&&this.directionTest(t))},i.emit=function(e){this.pX=e.deltaX,this.pY=e.deltaY;var i=Ch(e.direction);i&&(e.additionalEvent=this.options.event+i),t.prototype.emit.call(this,e)},e}(Ih),Gh=function(t){function e(e){return void 0===e&&(e={}),t.call(this,yl({event:"swipe",threshold:10,velocity:.3,direction:30,pointers:1},e))||this}xl(e,t);var i=e.prototype;return i.getTouchAction=function(){return Lh.prototype.getTouchAction.call(this)},i.attrTest=function(e){var i,n=this.options.direction;return 30&n?i=e.overallVelocity:6&n?i=e.overallVelocityX:24&n&&(i=e.overallVelocityY),t.prototype.attrTest.call(this,e)&&n&e.offsetDirection&&e.distance>this.options.threshold&&e.maxPointers===this.options.pointers&&Ll(i)>this.options.velocity&&4&e.eventType},i.emit=function(t){var e=Ch(t.offsetDirection);e&&this.manager.emit(this.options.event+e,t),this.manager.emit(this.options.event,t)},e}(Ih),Mh=function(t){function e(e){return void 0===e&&(e={}),t.call(this,yl({event:"pinch",threshold:0,pointers:2},e))||this}xl(e,t);var i=e.prototype;return i.getTouchAction=function(){return["none"]},i.attrTest=function(e){return t.prototype.attrTest.call(this,e)&&(Math.abs(e.scale-1)>this.options.threshold||2&this.state)},i.emit=function(e){if(1!==e.scale){var i=e.scale<1?"in":"out";e.additionalEvent=this.options.event+i}t.prototype.emit.call(this,e)},e}(Ih),Sh=function(t){function e(e){return void 0===e&&(e={}),t.call(this,yl({event:"rotate",threshold:0,pointers:2},e))||this}xl(e,t);var i=e.prototype;return i.getTouchAction=function(){return["none"]},i.attrTest=function(e){return t.prototype.attrTest.call(this,e)&&(Math.abs(e.rotation)>this.options.threshold||2&this.state)},e}(Ih),Qh=function(t){function e(e){var i;return void 0===e&&(e={}),(i=t.call(this,yl({event:"press",pointers:1,time:251,threshold:9},e))||this)._timer=null,i._input=null,i}xl(e,t);var i=e.prototype;return i.getTouchAction=function(){return["auto"]},i.process=function(t){var e=this,i=this.options,n=t.pointers.length===i.pointers,o=t.distance<i.threshold,s=t.deltaTime>i.time;if(this._input=t,!o||!n||12&t.eventType&&!s)this.reset();else if(1&t.eventType)this.reset(),this._timer=setTimeout((function(){e.state=8,e.tryEmit()}),i.time);else if(4&t.eventType)return 8;return 32},i.reset=function(){clearTimeout(this._timer)},i.emit=function(t){8===this.state&&(t&&4&t.eventType?this.manager.emit(this.options.event+"up",t):(this._input.timeStamp=Gl(),this.manager.emit(this.options.event,this._input)))},e}(kh),Nh={domEvents:!1,touchAction:"compute",enable:!0,inputTarget:null,inputClass:null,cssProps:{userSelect:"none",touchSelect:"none",touchCallout:"none",contentZooming:"none",userDrag:"none",tapHighlightColor:"rgba(0,0,0,0)"}},Fh=[[Sh,{enable:!1}],[Mh,{enable:!1},["rotate"]],[Gh,{direction:6}],[Lh,{direction:6},["swipe"]],[Dh],[Dh,{event:"doubletap",taps:2},["tap"]],[Qh]];function Zh(t,e){var i,n=t.element;n.style&&(Al(t.options.cssProps,(function(o,s){i=Ml(n.style,s),e?(t.oldCssProps[i]=n.style[i],n.style[i]=o):n.style[i]=t.oldCssProps[i]||""})),e||(t.oldCssProps={}))}var Bh=function(){function t(t,e){var i,n=this;this.options=kl({},Nh,e||{}),this.options.inputTarget=this.options.inputTarget||t,this.handlers={},this.session={},this.recognizers=[],this.oldCssProps={},this.element=t,this.input=new((i=this).options.inputClass||(Zl?ah:Bl?uh:Fl?bh:mh))(i,Jl),this.touchAction=new Yl(this,this.options.touchAction),Zh(this,!0),Al(this.options.recognizers,(function(t){var e=n.add(new t[0](t[1]));t[2]&&e.recognizeWith(t[2]),t[3]&&e.requireFailure(t[3])}),this)}var e=t.prototype;return e.set=function(t){return kl(this.options,t),t.touchAction&&this.touchAction.update(),t.inputTarget&&(this.input.destroy(),this.input.target=t.inputTarget,this.input.init()),this},e.stop=function(t){this.session.stopped=t?2:1},e.recognize=function(t){var e=this.session;if(!e.stopped){var i;this.touchAction.preventDefaults(t);var n=this.recognizers,o=e.curRecognizer;(!o||o&&8&o.state)&&(e.curRecognizer=null,o=null);for(var s=0;s<n.length;)i=n[s],2===e.stopped||o&&i!==o&&!i.canRecognizeWith(o)?i.reset():i.recognize(t),!o&&14&i.state&&(e.curRecognizer=i,o=i),s++}},e.get=function(t){if(t instanceof kh)return t;for(var e=this.recognizers,i=0;i<e.length;i++)if(e[i].options.event===t)return e[i];return null},e.add=function(t){if(yh(t,"add",this))return this;var e=this.get(t.options.event);return e&&this.remove(e),this.recognizers.push(t),t.manager=this,this.touchAction.update(),t},e.remove=function(t){if(yh(t,"remove",this))return this;var e=this.get(t);if(t){var i=this.recognizers,n=ih(i,e);-1!==n&&(i.splice(n,1),this.touchAction.update())}return this},e.on=function(t,e){if(void 0===t||void 0===e)return this;var i=this.handlers;return Al(Kl(t),(function(t){i[t]=i[t]||[],i[t].push(e)})),this},e.off=function(t,e){if(void 0===t)return this;var i=this.handlers;return Al(Kl(t),(function(t){e?i[t]&&i[t].splice(ih(i[t],e),1):delete i[t]})),this},e.emit=function(t,e){this.options.domEvents&&function(t,e){var i=document.createEvent("Event");i.initEvent(t,!0,!0),i.gesture=e,e.target.dispatchEvent(i)}(t,e);var i=this.handlers[t]&&this.handlers[t].slice();if(i&&i.length){e.type=t,e.preventDefault=function(){e.srcEvent.preventDefault()};for(var n=0;n<i.length;)i[n](e),n++}},e.destroy=function(){this.element&&Zh(this,!1),this.handlers={},this.session={},this.input.destroy(),this.element=null},t}(),Oh={touchstart:1,touchmove:2,touchend:4,touchcancel:8},Wh=function(t){function e(){var i,n=e.prototype;return n.evTarget="touchstart",n.evWin="touchstart touchmove touchend touchcancel",(i=t.apply(this,arguments)||this).started=!1,i}return xl(e,t),e.prototype.handler=function(t){var e=Oh[t.type];if(1===e&&(this.started=!0),this.started){var i=Ah.call(this,t,e);12&e&&i[0].length-i[1].length==0&&(this.started=!1),this.callback(this.manager,e,{pointers:i[0],changedPointers:i[1],pointerType:"touch",srcEvent:t})}},e}(eh);function Ah(t,e){var i=lh(t.touches),n=lh(t.changedTouches);return 12&e&&(i=hh(i.concat(n),"identifier",!0)),[i,n]}function Xh(t,e,i){var n="DEPRECATED METHOD: "+e+"\n"+i+" AT \n";return function(){var e=new Error("get-stack-trace"),i=e&&e.stack?e.stack.replace(/^[^\(]+?[\n$]/gm,"").replace(/^\s+at\s+/gm,"").replace(/^Object.<anonymous>\s*\(/gm,"{anonymous}()@"):"Unknown Stack Trace",o=window.console&&(window.console.warn||window.console.log);return o&&o.call(window.console,n,i),t.apply(this,arguments)}}var Th=Xh((function(t,e,i){for(var n=Object.keys(e),o=0;o<n.length;)(!i||i&&void 0===t[n[o]])&&(t[n[o]]=e[n[o]]),o++;return t}),"extend","Use `assign`."),Yh=Xh((function(t,e){return Th(t,e,!0)}),"merge","Use `assign`.");function Eh(t,e,i){var n,o=e.prototype;(n=t.prototype=Object.create(o)).constructor=t,n._super=o,i&&kl(n,i)}function Rh(t,e){return function(){return t.apply(e,arguments)}}var Uh,zh=function(){var t=function(t,e){return void 0===e&&(e={}),new Bh(t,yl({recognizers:Fh.concat()},e))};return t.VERSION="2.0.17-rc",t.DIRECTION_ALL=30,t.DIRECTION_DOWN=16,t.DIRECTION_LEFT=2,t.DIRECTION_RIGHT=4,t.DIRECTION_UP=8,t.DIRECTION_HORIZONTAL=6,t.DIRECTION_VERTICAL=24,t.DIRECTION_NONE=1,t.DIRECTION_DOWN=16,t.INPUT_START=1,t.INPUT_MOVE=2,t.INPUT_END=4,t.INPUT_CANCEL=8,t.STATE_POSSIBLE=1,t.STATE_BEGAN=2,t.STATE_CHANGED=4,t.STATE_ENDED=8,t.STATE_RECOGNIZED=8,t.STATE_CANCELLED=16,t.STATE_FAILED=32,t.Manager=Bh,t.Input=eh,t.TouchAction=Yl,t.TouchInput=uh,t.MouseInput=mh,t.PointerEventInput=ah,t.TouchMouseInput=bh,t.SingleTouchInput=Wh,t.Recognizer=kh,t.AttrRecognizer=Ih,t.Tap=Dh,t.Pan=Lh,t.Swipe=Gh,t.Pinch=Mh,t.Rotate=Sh,t.Press=Qh,t.on=ql,t.off=$l,t.each=Al,t.merge=Yh,t.extend=Th,t.bindFn=Rh,t.assign=kl,t.inherit=Eh,t.bindFn=Rh,t.prefixed=Ml,t.toArray=lh,t.inArray=ih,t.uniqueArray=hh,t.splitStr=Kl,t.boolOrFn=Xl,t.hasParent=El,t.addEventListeners=ql,t.removeEventListeners=$l,t.defaults=kl({},Nh,{preset:Fh}),t}();"undefined"!=typeof window?Uh=bl(window.Hammer||zh,{preventDefault:"mouse"}):Uh=function(){return{on:t=function(){},off:t,destroy:t,emit:t,get:function(e){return{set:t}}};var t};var Vh=Uh;function jh(t,e){e.inputHandler=function(t){t.isFirst&&e(t)},t.on("hammer.input",e.inputHandler)}var Hh=function(){function t(e,i,n,o,s){ba(this,t),this.moment=s&&s.moment||Ta,this.options=s||{},this.current=this.moment(),this._start=this.moment(),this._end=this.moment(),this.autoScale=!0,this.scale="day",this.step=1,this.setRange(e,i,n),this.switchedDay=!1,this.switchedMonth=!1,this.switchedYear=!1,Zo(o)?this.hiddenDates=o:this.hiddenDates=null!=o?[o]:[],this.format=t.FORMAT}return xa(t,[{key:"setMoment",value:function(t){this.moment=t,this.current=this.moment(this.current.valueOf()),this._start=this.moment(this._start.valueOf()),this._end=this.moment(this._end.valueOf())}},{key:"setFormat",value:function(e){var i=za.deepExtend({},t.FORMAT);this.format=za.deepExtend(i,e)}},{key:"setRange",value:function(t,e,i){if(!(t instanceof Date&&e instanceof Date))throw"No legal start or end date in method setRange";this._start=null!=t?this.moment(t.valueOf()):Jo(),this._end=null!=e?this.moment(e.valueOf()):Jo(),this.autoScale&&this.setMinimumStep(i)}},{key:"start",value:function(){this.current=this._start.clone(),this.roundToMinor()}},{key:"roundToMinor",value:function(){switch("week"==this.scale&&this.current.weekday(0),this.scale){case"year":this.current.year(this.step*Math.floor(this.current.year()/this.step)),this.current.month(0);case"month":this.current.date(1);case"week":case"day":case"weekday":this.current.hours(0);case"hour":this.current.minutes(0);case"minute":this.current.seconds(0);case"second":this.current.milliseconds(0)}if(1!=this.step){var t=this.current.clone();switch(this.scale){case"millisecond":this.current.subtract(this.current.milliseconds()%this.step,"milliseconds");break;case"second":this.current.subtract(this.current.seconds()%this.step,"seconds");break;case"minute":this.current.subtract(this.current.minutes()%this.step,"minutes");break;case"hour":this.current.subtract(this.current.hours()%this.step,"hours");break;case"weekday":case"day":this.current.subtract((this.current.date()-1)%this.step,"day");break;case"week":this.current.subtract(this.current.week()%this.step,"week");break;case"month":this.current.subtract(this.current.month()%this.step,"month");break;case"year":this.current.subtract(this.current.year()%this.step,"year")}t.isSame(this.current)||(this.current=this.moment(ul(this.hiddenDates,this.current.valueOf(),-1,!0)))}}},{key:"hasNext",value:function(){return this.current.valueOf()<=this._end.valueOf()}},{key:"next",value:function(){var t=this.current.valueOf();switch(this.scale){case"millisecond":this.current.add(this.step,"millisecond");break;case"second":this.current.add(this.step,"second");break;case"minute":this.current.add(this.step,"minute");break;case"hour":this.current.add(this.step,"hour"),this.current.month()<6?this.current.subtract(this.current.hours()%this.step,"hour"):this.current.hours()%this.step!=0&&this.current.add(this.step-this.current.hours()%this.step,"hour");break;case"weekday":case"day":this.current.add(this.step,"day");break;case"week":if(0!==this.current.weekday())this.current.weekday(0),this.current.add(this.step,"week");else if(!1===this.options.showMajorLabels)this.current.add(this.step,"week");else{var e=this.current.clone();e.add(1,"week"),e.isSame(this.current,"month")?this.current.add(this.step,"week"):(this.current.add(this.step,"week"),this.current.date(1))}break;case"month":this.current.add(this.step,"month");break;case"year":this.current.add(this.step,"year")}if(1!=this.step)switch(this.scale){case"millisecond":this.current.milliseconds()>0&&this.current.milliseconds()<this.step&&this.current.milliseconds(0);break;case"second":this.current.seconds()>0&&this.current.seconds()<this.step&&this.current.seconds(0);break;case"minute":this.current.minutes()>0&&this.current.minutes()<this.step&&this.current.minutes(0);break;case"hour":this.current.hours()>0&&this.current.hours()<this.step&&this.current.hours(0);break;case"weekday":case"day":this.current.date()<this.step+1&&this.current.date(1);break;case"week":this.current.week()<this.step&&this.current.week(1);break;case"month":this.current.month()<this.step&&this.current.month(0)}this.current.valueOf()==t&&(this.current=this._end.clone()),this.switchedDay=!1,this.switchedMonth=!1,this.switchedYear=!1,nl(this.moment,this,t)}},{key:"getCurrent",value:function(){return this.current.clone()}},{key:"setScale",value:function(t){t&&"string"==typeof t.scale&&(this.scale=t.scale,this.step=t.step>0?t.step:1,this.autoScale=!1)}},{key:"setAutoScale",value:function(t){this.autoScale=t}},{key:"setMinimumStep",value:function(t){if(null!=t){31104e9>t&&(this.scale="year",this.step=1e3),15552e9>t&&(this.scale="year",this.step=500),31104e8>t&&(this.scale="year",this.step=100),15552e8>t&&(this.scale="year",this.step=50),31104e7>t&&(this.scale="year",this.step=10),15552e7>t&&(this.scale="year",this.step=5),31104e6>t&&(this.scale="year",this.step=1),7776e6>t&&(this.scale="month",this.step=3),2592e6>t&&(this.scale="month",this.step=1),6048e5>t&&this.options.showWeekScale&&(this.scale="week",this.step=1),1728e5>t&&(this.scale="day",this.step=2),864e5>t&&(this.scale="day",this.step=1),432e5>t&&(this.scale="weekday",this.step=1),144e5>t&&(this.scale="hour",this.step=4),36e5>t&&(this.scale="hour",this.step=1),9e5>t&&(this.scale="minute",this.step=15),6e5>t&&(this.scale="minute",this.step=10),3e5>t&&(this.scale="minute",this.step=5),6e4>t&&(this.scale="minute",this.step=1),15e3>t&&(this.scale="second",this.step=15),1e4>t&&(this.scale="second",this.step=10),5e3>t&&(this.scale="second",this.step=5),1e3>t&&(this.scale="second",this.step=1),200>t&&(this.scale="millisecond",this.step=200),100>t&&(this.scale="millisecond",this.step=100),50>t&&(this.scale="millisecond",this.step=50),10>t&&(this.scale="millisecond",this.step=10),5>t&&(this.scale="millisecond",this.step=5),1>t&&(this.scale="millisecond",this.step=1)}}},{key:"isMajor",value:function(){if(1==this.switchedYear)switch(this.scale){case"year":case"month":case"week":case"weekday":case"day":case"hour":case"minute":case"second":case"millisecond":return!0;default:return!1}else if(1==this.switchedMonth)switch(this.scale){case"week":case"weekday":case"day":case"hour":case"minute":case"second":case"millisecond":return!0;default:return!1}else if(1==this.switchedDay)switch(this.scale){case"millisecond":case"second":case"minute":case"hour":return!0;default:return!1}var t=this.moment(this.current);switch(this.scale){case"millisecond":return 0==t.milliseconds();case"second":return 0==t.seconds();case"minute":return 0==t.hours()&&0==t.minutes();case"hour":return 0==t.hours();case"weekday":case"day":case"week":return 1==t.date();case"month":return 0==t.month();case"year":default:return!1}}},{key:"getLabelMinor",value:function(t){if(null==t&&(t=this.current),t instanceof Date&&(t=this.moment(t)),"function"==typeof this.format.minorLabels)return this.format.minorLabels(t,this.scale,this.step);var e=this.format.minorLabels[this.scale];switch(this.scale){case"week":if(1===t.date()&&0!==t.weekday())return"";default:return e&&e.length>0?this.moment(t).format(e):""}}},{key:"getLabelMajor",value:function(t){if(null==t&&(t=this.current),t instanceof Date&&(t=this.moment(t)),"function"==typeof this.format.majorLabels)return this.format.majorLabels(t,this.scale,this.step);var e=this.format.majorLabels[this.scale];return e&&e.length>0?this.moment(t).format(e):""}},{key:"getClassName",value:function(){var t,e=this.moment,i=this.moment(this.current),n=i.locale?i.locale("en"):i.lang("en"),o=this.step,s=[];function r(t){return t/o%2==0?" vis-even":" vis-odd"}function a(t){return t.isSame(Jo(),"day")?" vis-today":t.isSame(e().add(1,"day"),"day")?" vis-tomorrow":t.isSame(e().add(-1,"day"),"day")?" vis-yesterday":""}function l(t){return t.isSame(Jo(),"week")?" vis-current-week":""}function h(t){return t.isSame(Jo(),"month")?" vis-current-month":""}switch(this.scale){case"millisecond":s.push(a(n)),s.push(r(n.milliseconds()));break;case"second":s.push(a(n)),s.push(r(n.seconds()));break;case"minute":s.push(a(n)),s.push(r(n.minutes()));break;case"hour":s.push(Pn(t="vis-h".concat(n.hours())).call(t,4==this.step?"-h"+(n.hours()+4):"")),s.push(a(n)),s.push(r(n.hours()));break;case"weekday":s.push("vis-".concat(n.format("dddd").toLowerCase())),s.push(a(n)),s.push(l(n)),s.push(r(n.date()));break;case"day":s.push("vis-day".concat(n.date())),s.push("vis-".concat(n.format("MMMM").toLowerCase())),s.push(a(n)),s.push(h(n)),s.push(this.step<=2?a(n):""),s.push(this.step<=2?"vis-".concat(n.format("dddd").toLowerCase()):""),s.push(r(n.date()-1));break;case"week":s.push("vis-week".concat(n.format("w"))),s.push(l(n)),s.push(r(n.week()));break;case"month":s.push("vis-".concat(n.format("MMMM").toLowerCase())),s.push(h(n)),s.push(r(n.month()));break;case"year":s.push("vis-year".concat(n.year())),s.push(function(t){return t.isSame(Jo(),"year")?" vis-current-year":""}(n)),s.push(r(n.year()))}return En(s).call(s,String).join(" ")}}],[{key:"snap",value:function(t,e,i){var n=Ta(t);if("year"==e){var o=n.year()+Math.round(n.month()/12);n.year(Math.round(o/i)*i),n.month(0),n.date(0),n.hours(0),n.minutes(0),n.seconds(0),n.milliseconds(0)}else if("month"==e)n.date()>15?(n.date(1),n.add(1,"month")):n.date(1),n.hours(0),n.minutes(0),n.seconds(0),n.milliseconds(0);else if("week"==e)n.weekday()>2?(n.weekday(0),n.add(1,"week")):n.weekday(0),n.hours(0),n.minutes(0),n.seconds(0),n.milliseconds(0);else if("day"==e){switch(i){case 5:case 2:n.hours(24*Math.round(n.hours()/24));break;default:n.hours(12*Math.round(n.hours()/12))}n.minutes(0),n.seconds(0),n.milliseconds(0)}else if("weekday"==e){switch(i){case 5:case 2:n.hours(12*Math.round(n.hours()/12));break;default:n.hours(6*Math.round(n.hours()/6))}n.minutes(0),n.seconds(0),n.milliseconds(0)}else if("hour"==e){switch(i){case 4:n.minutes(60*Math.round(n.minutes()/60));break;default:n.minutes(30*Math.round(n.minutes()/30))}n.seconds(0),n.milliseconds(0)}else if("minute"==e){switch(i){case 15:case 10:n.minutes(5*Math.round(n.minutes()/5)),n.seconds(0);break;case 5:n.seconds(60*Math.round(n.seconds()/60));break;default:n.seconds(30*Math.round(n.seconds()/30))}n.milliseconds(0)}else if("second"==e)switch(i){case 15:case 10:n.seconds(5*Math.round(n.seconds()/5)),n.milliseconds(0);break;case 5:n.milliseconds(1e3*Math.round(n.milliseconds()/1e3));break;default:n.milliseconds(500*Math.round(n.milliseconds()/500))}else if("millisecond"==e){var s=i>5?i/2:1;n.milliseconds(Math.round(n.milliseconds()/s)*s)}return n}}]),t}();function Ph(t,e){void 0===e&&(e={});var i=e.insertAt;if(t&&"undefined"!=typeof document){var n=document.head||document.getElementsByTagName("head")[0],o=document.createElement("style");o.type="text/css","top"===i&&n.firstChild?n.insertBefore(o,n.firstChild):n.appendChild(o),o.styleSheet?o.styleSheet.cssText=t:o.appendChild(document.createTextNode(t))}}Hh.FORMAT={minorLabels:{millisecond:"SSS",second:"s",minute:"HH:mm",hour:"HH:mm",weekday:"ddd D",day:"D",week:"w",month:"MMM",year:"YYYY"},majorLabels:{millisecond:"HH:mm:ss",second:"D MMMM HH:mm",minute:"ddd D MMMM",hour:"ddd D MMMM",weekday:"MMMM YYYY",day:"MMMM YYYY",week:"MMMM YYYY",month:"YYYY",year:""}};Ph(".vis-time-axis{position:relative;overflow:hidden}.vis-time-axis.vis-foreground{top:0;left:0;width:100%}.vis-time-axis.vis-background{position:absolute;top:0;left:0;width:100%;height:100%}.vis-time-axis .vis-text{position:absolute;color:#4d4d4d;padding:3px;overflow:hidden;box-sizing:border-box;white-space:nowrap}.vis-time-axis .vis-text.vis-measure{position:absolute;padding-left:0;padding-right:0;margin-left:0;margin-right:0;visibility:hidden}.vis-time-axis .vis-grid.vis-vertical{position:absolute;border-left:1px solid}.vis-time-axis .vis-grid.vis-vertical-rtl{position:absolute;border-right:1px solid}.vis-time-axis .vis-grid.vis-minor{border-color:#e5e5e5}.vis-time-axis .vis-grid.vis-major{border-color:#bfbfbf}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbInRpbWVheGlzLmNzcyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxlQUNFLGlCQUFrQixDQUNsQixlQUNGLENBRUEsOEJBQ0UsS0FBTSxDQUNOLE1BQU8sQ0FDUCxVQUNGLENBRUEsOEJBQ0UsaUJBQWtCLENBQ2xCLEtBQU0sQ0FDTixNQUFPLENBQ1AsVUFBVyxDQUNYLFdBQ0YsQ0FFQSx5QkFDRSxpQkFBa0IsQ0FDbEIsYUFBYyxDQUNkLFdBQVksQ0FDWixlQUFnQixDQUNoQixxQkFBc0IsQ0FFdEIsa0JBQ0YsQ0FFQSxxQ0FDRSxpQkFBa0IsQ0FDbEIsY0FBZSxDQUNmLGVBQWdCLENBQ2hCLGFBQWMsQ0FDZCxjQUFlLENBQ2YsaUJBQ0YsQ0FFQSxzQ0FDRSxpQkFBa0IsQ0FDbEIscUJBQ0YsQ0FFQSwwQ0FDRSxpQkFBa0IsQ0FDbEIsc0JBQ0YsQ0FFQSxtQ0FDRSxvQkFDRixDQUVBLG1DQUNFLG9CQUNGIiwiZmlsZSI6InRpbWVheGlzLmNzcyIsInNvdXJjZXNDb250ZW50IjpbIi52aXMtdGltZS1heGlzIHtcbiAgcG9zaXRpb246IHJlbGF0aXZlO1xuICBvdmVyZmxvdzogaGlkZGVuO1xufVxuXG4udmlzLXRpbWUtYXhpcy52aXMtZm9yZWdyb3VuZCB7XG4gIHRvcDogMDtcbiAgbGVmdDogMDtcbiAgd2lkdGg6IDEwMCU7XG59XG5cbi52aXMtdGltZS1heGlzLnZpcy1iYWNrZ3JvdW5kIHtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICB0b3A6IDA7XG4gIGxlZnQ6IDA7XG4gIHdpZHRoOiAxMDAlO1xuICBoZWlnaHQ6IDEwMCU7XG59XG5cbi52aXMtdGltZS1heGlzIC52aXMtdGV4dCB7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgY29sb3I6ICM0ZDRkNGQ7XG4gIHBhZGRpbmc6IDNweDtcbiAgb3ZlcmZsb3c6IGhpZGRlbjtcbiAgYm94LXNpemluZzogYm9yZGVyLWJveDtcblxuICB3aGl0ZS1zcGFjZTogbm93cmFwO1xufVxuXG4udmlzLXRpbWUtYXhpcyAudmlzLXRleHQudmlzLW1lYXN1cmUge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHBhZGRpbmctbGVmdDogMDtcbiAgcGFkZGluZy1yaWdodDogMDtcbiAgbWFyZ2luLWxlZnQ6IDA7XG4gIG1hcmdpbi1yaWdodDogMDtcbiAgdmlzaWJpbGl0eTogaGlkZGVuO1xufVxuXG4udmlzLXRpbWUtYXhpcyAudmlzLWdyaWQudmlzLXZlcnRpY2FsIHtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICBib3JkZXItbGVmdDogMXB4IHNvbGlkO1xufVxuXG4udmlzLXRpbWUtYXhpcyAudmlzLWdyaWQudmlzLXZlcnRpY2FsLXJ0bCB7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgYm9yZGVyLXJpZ2h0OiAxcHggc29saWQ7XG59XG5cbi52aXMtdGltZS1heGlzIC52aXMtZ3JpZC52aXMtbWlub3Ige1xuICBib3JkZXItY29sb3I6ICNlNWU1ZTU7XG59XG5cbi52aXMtdGltZS1heGlzIC52aXMtZ3JpZC52aXMtbWFqb3Ige1xuICBib3JkZXItY29sb3I6ICNiZmJmYmY7XG59XG4iXX0= */");var Jh=function(t){function e(t,i){var n;return ba(this,e),(n=ar(this,dr(e).call(this))).dom={foreground:null,lines:[],majorTexts:[],minorTexts:[],redundant:{lines:[],majorTexts:[],minorTexts:[]}},n.props={range:{start:0,end:0,minimumStep:0},lineTop:0},n.defaultOptions={orientation:{axis:"bottom"},showMinorLabels:!0,showMajorLabels:!0,showWeekScale:!1,maxMinorChars:7,format:Hh.FORMAT,moment:Ta,timeAxis:null},n.options=za.extend({},n.defaultOptions),n.body=t,n._create(),n.setOptions(i),n}return pr(e,t),xa(e,[{key:"setOptions",value:function(t){t&&(za.selectiveExtend(["showMinorLabels","showMajorLabels","showWeekScale","maxMinorChars","hiddenDates","timeAxis","moment","rtl"],this.options,t),za.selectiveDeepExtend(["format"],this.options,t),"orientation"in t&&("string"==typeof t.orientation?this.options.orientation.axis=t.orientation:"object"===zo(t.orientation)&&"axis"in t.orientation&&(this.options.orientation.axis=t.orientation.axis)),"locale"in t&&("function"==typeof Ta.locale?Ta.locale(t.locale):Ta.lang(t.locale)))}},{key:"_create",value:function(){this.dom.foreground=document.createElement("div"),this.dom.background=document.createElement("div"),this.dom.foreground.className="vis-time-axis vis-foreground",this.dom.background.className="vis-time-axis vis-background"}},{key:"destroy",value:function(){this.dom.foreground.parentNode&&this.dom.foreground.parentNode.removeChild(this.dom.foreground),this.dom.background.parentNode&&this.dom.background.parentNode.removeChild(this.dom.background),this.body=null}},{key:"redraw",value:function(){var t=this.props,e=this.dom.foreground,i=this.dom.background,n="top"==this.options.orientation.axis?this.body.dom.top:this.body.dom.bottom,o=e.parentNode!==n;this._calculateCharSize();var s=this.options.showMinorLabels&&"none"!==this.options.orientation.axis,r=this.options.showMajorLabels&&"none"!==this.options.orientation.axis;t.minorLabelHeight=s?t.minorCharHeight:0,t.majorLabelHeight=r?t.majorCharHeight:0,t.height=t.minorLabelHeight+t.majorLabelHeight,t.width=e.offsetWidth,t.minorLineHeight=this.body.domProps.root.height-t.majorLabelHeight-("top"==this.options.orientation.axis?this.body.domProps.bottom.height:this.body.domProps.top.height),t.minorLineWidth=1,t.majorLineHeight=t.minorLineHeight+t.majorLabelHeight,t.majorLineWidth=1;var a=e.nextSibling,l=i.nextSibling;return e.parentNode&&e.parentNode.removeChild(e),i.parentNode&&i.parentNode.removeChild(i),e.style.height="".concat(this.props.height,"px"),this._repaintLabels(),a?n.insertBefore(e,a):n.appendChild(e),l?this.body.dom.backgroundVertical.insertBefore(i,l):this.body.dom.backgroundVertical.appendChild(i),this._isResized()||o}},{key:"_repaintLabels",value:function(){var t=this.options.orientation.axis,e=za.convert(this.body.range.start,"Number"),i=za.convert(this.body.range.end,"Number"),n=this.body.util.toTime((this.props.minorCharWidth||10)*this.options.maxMinorChars).valueOf(),o=n-hl(this.options.moment,this.body.hiddenDates,this.body.range,n);o-=this.body.util.toTime(0).valueOf();var s=new Hh(new Date(e),new Date(i),o,this.body.hiddenDates,this.options);s.setMoment(this.options.moment),this.options.format&&s.setFormat(this.options.format),this.options.timeAxis&&s.setScale(this.options.timeAxis),this.step=s;var r,a,l,h,d,u,c=this.dom;c.redundant.lines=c.lines,c.redundant.majorTexts=c.majorTexts,c.redundant.minorTexts=c.minorTexts,c.lines=[],c.majorTexts=[],c.minorTexts=[];var p,m,f,g=0,v=void 0,b=0;for(s.start(),a=s.getCurrent(),h=this.body.util.toScreen(a);s.hasNext()&&b<1e3;){switch(b++,d=s.isMajor(),f=s.getClassName(),r=a,l=h,s.next(),a=s.getCurrent(),p=g,g=(h=this.body.util.toScreen(a))-l,s.scale){case"week":u=!0;break;default:u=g>=.4*p}if(this.options.showMinorLabels&&u){var y=this._repaintMinorText(l,s.getLabelMinor(r),t,f);y.style.width="".concat(g,"px")}d&&this.options.showMajorLabels?(l>0&&(null==v&&(v=l),y=this._repaintMajorText(l,s.getLabelMajor(r),t,f)),m=this._repaintMajorLine(l,g,t,f)):u?m=this._repaintMinorLine(l,g,t,f):m&&(m.style.width="".concat(Nn(m.style.width)+g,"px"))}if(1e3!==b||Kh||(console.warn("Something is wrong with the Timeline scale. Limited drawing of grid lines to ".concat(1e3," lines.")),Kh=!0),this.options.showMajorLabels){var x=this.body.util.toTime(0),w=s.getLabelMajor(x),_=w.length*(this.props.majorCharWidth||10)+10;(null==v||_<v)&&this._repaintMajorText(0,w,t,f)}mn(za).call(za,this.dom.redundant,(function(t){for(;t.length;){var e=t.pop();e&&e.parentNode&&e.parentNode.removeChild(e)}}))}},{key:"_repaintMinorText",value:function(t,e,i,n){var o=this.dom.redundant.minorTexts.shift();if(!o){var s=document.createTextNode("");(o=document.createElement("div")).appendChild(s),this.dom.foreground.appendChild(o)}this.dom.minorTexts.push(o),o.innerHTML=e;var r="top"==i?this.props.majorLabelHeight:0;return this._setXY(o,t,r),o.className="vis-text vis-minor ".concat(n),o}},{key:"_repaintMajorText",value:function(t,e,i,n){var o=this.dom.redundant.majorTexts.shift();if(!o){var s=document.createElement("div");(o=document.createElement("div")).appendChild(s),this.dom.foreground.appendChild(o)}o.childNodes[0].innerHTML=e,o.className="vis-text vis-major ".concat(n);var r="top"==i?0:this.props.minorLabelHeight;return this._setXY(o,t,r),this.dom.majorTexts.push(o),o}},{key:"_setXY",value:function(t,e,i){var n,o=this.options.rtl?-1*e:e;t.style.transform=Pn(n="translate(".concat(o,"px, ")).call(n,i,"px)")}},{key:"_repaintMinorLine",value:function(t,e,i,n){var o,s=this.dom.redundant.lines.shift();s||(s=document.createElement("div"),this.dom.background.appendChild(s)),this.dom.lines.push(s);var r=this.props;s.style.width="".concat(e,"px"),s.style.height="".concat(r.minorLineHeight,"px");var a="top"==i?r.majorLabelHeight:this.body.domProps.top.height,l=t-r.minorLineWidth/2;return this._setXY(s,l,a),s.className=Pn(o="vis-grid ".concat(this.options.rtl?"vis-vertical-rtl":"vis-vertical"," vis-minor ")).call(o,n),s}},{key:"_repaintMajorLine",value:function(t,e,i,n){var o,s=this.dom.redundant.lines.shift();s||(s=document.createElement("div"),this.dom.background.appendChild(s)),this.dom.lines.push(s);var r=this.props;s.style.width="".concat(e,"px"),s.style.height="".concat(r.majorLineHeight,"px");var a="top"==i?0:this.body.domProps.top.height,l=t-r.majorLineWidth/2;return this._setXY(s,l,a),s.className=Pn(o="vis-grid ".concat(this.options.rtl?"vis-vertical-rtl":"vis-vertical"," vis-major ")).call(o,n),s}},{key:"_calculateCharSize",value:function(){this.dom.measureCharMinor||(this.dom.measureCharMinor=document.createElement("DIV"),this.dom.measureCharMinor.className="vis-text vis-minor vis-measure",this.dom.measureCharMinor.style.position="absolute",this.dom.measureCharMinor.appendChild(document.createTextNode("0")),this.dom.foreground.appendChild(this.dom.measureCharMinor)),this.props.minorCharHeight=this.dom.measureCharMinor.clientHeight,this.props.minorCharWidth=this.dom.measureCharMinor.clientWidth,this.dom.measureCharMajor||(this.dom.measureCharMajor=document.createElement("DIV"),this.dom.measureCharMajor.className="vis-text vis-major vis-measure",this.dom.measureCharMajor.style.position="absolute",this.dom.measureCharMajor.appendChild(document.createTextNode("0")),this.dom.foreground.appendChild(this.dom.measureCharMajor)),this.props.majorCharHeight=this.dom.measureCharMajor.clientHeight,this.props.majorCharWidth=this.dom.measureCharMajor.clientWidth}}]),e}(Ja),Kh=!1,qh=n((function(t,e){t.exports=function(t){var e,i=t&&t.preventDefault||!1,n=t&&t.container||window,o={},s={keydown:{},keyup:{}},r={};for(e=97;e<=122;e++)r[String.fromCharCode(e)]={code:e-97+65,shift:!1};for(e=65;e<=90;e++)r[String.fromCharCode(e)]={code:e,shift:!0};for(e=0;e<=9;e++)r[""+e]={code:48+e,shift:!1};for(e=1;e<=12;e++)r["F"+e]={code:111+e,shift:!1};for(e=0;e<=9;e++)r["num"+e]={code:96+e,shift:!1};r["num*"]={code:106,shift:!1},r["num+"]={code:107,shift:!1},r["num-"]={code:109,shift:!1},r["num/"]={code:111,shift:!1},r["num."]={code:110,shift:!1},r.left={code:37,shift:!1},r.up={code:38,shift:!1},r.right={code:39,shift:!1},r.down={code:40,shift:!1},r.space={code:32,shift:!1},r.enter={code:13,shift:!1},r.shift={code:16,shift:void 0},r.esc={code:27,shift:!1},r.backspace={code:8,shift:!1},r.tab={code:9,shift:!1},r.ctrl={code:17,shift:!1},r.alt={code:18,shift:!1},r.delete={code:46,shift:!1},r.pageup={code:33,shift:!1},r.pagedown={code:34,shift:!1},r["="]={code:187,shift:!1},r["-"]={code:189,shift:!1},r["]"]={code:221,shift:!1},r["["]={code:219,shift:!1};var a=function(t){h(t,"keydown")},l=function(t){h(t,"keyup")},h=function(t,e){if(void 0!==s[e][t.keyCode]){for(var n=s[e][t.keyCode],o=0;o<n.length;o++)(void 0===n[o].shift||1==n[o].shift&&1==t.shiftKey||0==n[o].shift&&0==t.shiftKey)&&n[o].fn(t);1==i&&t.preventDefault()}};return o.bind=function(t,e,i){if(void 0===i&&(i="keydown"),void 0===r[t])throw new Error("unsupported key: "+t);void 0===s[i][r[t].code]&&(s[i][r[t].code]=[]),s[i][r[t].code].push({fn:e,shift:r[t].shift})},o.bindAll=function(t,e){for(var i in void 0===e&&(e="keydown"),r)r.hasOwnProperty(i)&&o.bind(i,t,e)},o.getKey=function(t){for(var e in r)if(r.hasOwnProperty(e)){if(1==t.shiftKey&&1==r[e].shift&&t.keyCode==r[e].code)return e;if(0==t.shiftKey&&0==r[e].shift&&t.keyCode==r[e].code)return e;if(t.keyCode==r[e].code&&"shift"==e)return e}return"unknown key, currently not supported"},o.unbind=function(t,e,i){if(void 0===i&&(i="keydown"),void 0===r[t])throw new Error("unsupported key: "+t);if(void 0!==e){var n=[],o=s[i][r[t].code];if(void 0!==o)for(var a=0;a<o.length;a++)o[a].fn==e&&o[a].shift==r[t].shift||n.push(s[i][r[t].code][a]);s[i][r[t].code]=n}else s[i][r[t].code]=[]},o.reset=function(){s={keydown:{},keyup:{}}},o.destroy=function(){s={keydown:{},keyup:{}},n.removeEventListener("keydown",a,!0),n.removeEventListener("keyup",l,!0)},n.addEventListener("keydown",a,!0),n.addEventListener("keyup",l,!0),o}}));function $h(t){var e,i;this.active=!1,this.dom={container:t},this.dom.overlay=document.createElement("div"),this.dom.overlay.className="vis-overlay",this.dom.container.appendChild(this.dom.overlay),this.hammer=Vh(this.dom.overlay),this.hammer.on("tap",va(e=this._onTapOverlay).call(e,this));var n=this,o=["tap","doubletap","press","pinch","pan","panstart","panmove","panend"];mn(o).call(o,(function(t){n.hammer.on(t,(function(t){t.stopPropagation()}))})),document&&document.body&&(this.onClick=function(e){(function(t,e){for(;t;){if(t===e)return!0;t=t.parentNode}return!1})(e.target,t)||n.deactivate()},document.body.addEventListener("click",this.onClick)),void 0!==this.keycharm&&this.keycharm.destroy(),this.keycharm=qh(),this.escListener=va(i=this.deactivate).call(i,this)}Ph(".vis .overlay{position:absolute;top:0;left:0;width:100%;height:100%;z-index:10}.vis-active{box-shadow:0 0 10px #86d5f8}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbImFjdGl2YXRvci5jc3MiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsY0FDRSxpQkFBa0IsQ0FDbEIsS0FBTSxDQUNOLE1BQU8sQ0FDUCxVQUFXLENBQ1gsV0FBWSxDQUdaLFVBQ0YsQ0FFQSxZQUNFLDJCQUNGIiwiZmlsZSI6ImFjdGl2YXRvci5jc3MiLCJzb3VyY2VzQ29udGVudCI6WyIudmlzIC5vdmVybGF5IHtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICB0b3A6IDA7XG4gIGxlZnQ6IDA7XG4gIHdpZHRoOiAxMDAlO1xuICBoZWlnaHQ6IDEwMCU7XG5cbiAgLyogTXVzdCBiZSBkaXNwbGF5ZWQgYWJvdmUgZm9yIGV4YW1wbGUgc2VsZWN0ZWQgVGltZWxpbmUgaXRlbXMgKi9cbiAgei1pbmRleDogMTA7XG59XG5cbi52aXMtYWN0aXZlIHtcbiAgYm94LXNoYWRvdzogMCAwIDEwcHggIzg2ZDVmODtcbn1cbiJdfQ== */"),vl($h.prototype),$h.current=null,$h.prototype.destroy=function(){this.deactivate(),this.dom.overlay.parentNode.removeChild(this.dom.overlay),this.onClick&&document.body.removeEventListener("click",this.onClick),void 0!==this.keycharm&&this.keycharm.destroy(),this.keycharm=null,this.hammer.destroy(),this.hammer=null},$h.prototype.activate=function(){var t;$h.current&&$h.current.deactivate(),$h.current=this,this.active=!0,this.dom.overlay.style.display="none",za.addClassName(this.dom.container,"vis-active"),this.emit("change"),this.emit("activate"),va(t=this.keycharm).call(t,"esc",this.escListener)},$h.prototype.deactivate=function(){$h.current===this&&($h.current=null),this.active=!1,this.dom.overlay.style.display="",za.removeClassName(this.dom.container,"vis-active"),this.keycharm.unbind("esc",this.escListener),this.emit("change"),this.emit("deactivate")},$h.prototype._onTapOverlay=function(t){this.activate(),t.stopPropagation()};var td={current:"current",time:"time",deleteSelected:"Delete selected"},ed={current:"attuale",time:"tempo",deleteSelected:"Cancella la selezione"},id={current:"huidige",time:"tijd",deleteSelected:"Selectie verwijderen"},nd={current:"Aktuelle",time:"Zeit",deleteSelected:"Lösche Auswahl"},od={current:"actuel",time:"heure",deleteSelected:"Effacer la selection"},sd={current:"corriente",time:"hora",deleteSelected:"Eliminar selección"},rd={current:"поточний",time:"час",deleteSelected:"Видалити обране"},ad={current:"текущее",time:"время",deleteSelected:"Удалить выбранное"},ld={current:"aktualny",time:"czas",deleteSelected:"Usuń wybrane"},hd={current:"現在",time:"時刻",deleteSelected:"選択されたものを削除"},dd={en:td,en_EN:td,en_US:td,it:ed,it_IT:ed,it_CH:ed,nl:id,nl_NL:id,nl_BE:id,de:nd,de_DE:nd,fr:od,fr_FR:od,fr_CA:od,fr_BE:od,es:sd,es_ES:sd,uk:rd,uk_UA:rd,ru:ad,ru_RU:ad,pl:ld,pl_PL:ld,ja:hd,ja_JP:hd};Ph(".vis-custom-time{background-color:#6e94ff;width:2px;cursor:move;z-index:1}.vis-custom-time>.vis-custom-time-marker{background-color:inherit;color:#fff;font-size:12px;white-space:nowrap;padding:3px 5px;top:0;cursor:auto;z-index:inherit}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbImN1c3RvbXRpbWUuY3NzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLGlCQUNFLHdCQUF5QixDQUN6QixTQUFVLENBQ1YsV0FBWSxDQUNaLFNBQ0YsQ0FFQSx5Q0FDRSx3QkFBeUIsQ0FDekIsVUFBWSxDQUNaLGNBQWUsQ0FDZixrQkFBbUIsQ0FDbkIsZUFBZ0IsQ0FDaEIsS0FBUSxDQUNSLFdBQWUsQ0FDZixlQUNGIiwiZmlsZSI6ImN1c3RvbXRpbWUuY3NzIiwic291cmNlc0NvbnRlbnQiOlsiLnZpcy1jdXN0b20tdGltZSB7XG4gIGJhY2tncm91bmQtY29sb3I6ICM2RTk0RkY7XG4gIHdpZHRoOiAycHg7XG4gIGN1cnNvcjogbW92ZTtcbiAgei1pbmRleDogMTtcbn1cblxuLnZpcy1jdXN0b20tdGltZSA+IC52aXMtY3VzdG9tLXRpbWUtbWFya2VyIHtcbiAgYmFja2dyb3VuZC1jb2xvcjogaW5oZXJpdDtcbiAgY29sb3I6IHdoaXRlO1xuICBmb250LXNpemU6IDEycHg7XG4gIHdoaXRlLXNwYWNlOiBub3dyYXA7XG4gIHBhZGRpbmc6IDNweCA1cHg7XG4gIHRvcDogMHB4O1xuICBjdXJzb3I6IGluaXRpYWw7XG4gIHotaW5kZXg6IGluaGVyaXQ7XG59Il19 */");var ud=function(t){function e(t,i){var n,o;ba(this,e),(o=ar(this,dr(e).call(this))).body=t,o.defaultOptions={moment:Ta,locales:dd,locale:"en",id:void 0,title:void 0},o.options=za.extend({},o.defaultOptions),o.setOptions(i),o.options.locales=za.extend({},dd,o.options.locales);var s=o.defaultOptions.locales[o.defaultOptions.locale];return mn(n=Yi(o.options.locales)).call(n,(function(t){o.options.locales[t]=za.extend({},s,o.options.locales[t])})),i&&null!=i.time?o.customTime=i.time:o.customTime=new Date,o.eventParams={},o._create(),o}return pr(e,t),xa(e,[{key:"setOptions",value:function(t){t&&za.selectiveExtend(["moment","locale","locales","id","title","rtl","snap"],this.options,t)}},{key:"_create",value:function(){var t,e,i,n=document.createElement("div");n["custom-time"]=this,n.className="vis-custom-time ".concat(this.options.id||""),n.style.position="absolute",n.style.top="0px",n.style.height="100%",this.bar=n;var o=document.createElement("div");function s(t){this.body.range._onMouseWheel(t)}o.style.position="relative",o.style.top="0px",this.options.rtl?o.style.right="-10px":o.style.left="-10px",o.style.height="100%",o.style.width="20px",o.addEventListener?(o.addEventListener("mousewheel",va(s).call(s,this),!1),o.addEventListener("DOMMouseScroll",va(s).call(s,this),!1)):o.attachEvent("onmousewheel",va(s).call(s,this)),n.appendChild(o),this.hammer=new Vh(o),this.hammer.on("panstart",va(t=this._onDragStart).call(t,this)),this.hammer.on("panmove",va(e=this._onDrag).call(e,this)),this.hammer.on("panend",va(i=this._onDragEnd).call(i,this)),this.hammer.get("pan").set({threshold:5,direction:Vh.DIRECTION_ALL})}},{key:"destroy",value:function(){this.hide(),this.hammer.destroy(),this.hammer=null,this.body=null}},{key:"redraw",value:function(){var t=this.body.dom.backgroundVertical;this.bar.parentNode!=t&&(this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar),t.appendChild(this.bar));var e=this.body.util.toScreen(this.customTime),i=this.options.locales[this.options.locale];i||(this.warned||(console.warn("WARNING: options.locales['".concat(this.options.locale,"'] not found. See https://visjs.github.io/vis-timeline/docs/timeline/#Localization")),this.warned=!0),i=this.options.locales.en);var n,o=this.options.title;void 0===o?o=(o=Pn(n="".concat(i.time,": ")).call(n,this.options.moment(this.customTime).format("dddd, MMMM Do YYYY, H:mm:ss"))).charAt(0).toUpperCase()+o.substring(1):"function"==typeof o&&(o=o.call(this,this.customTime));return this.options.rtl?this.bar.style.right="".concat(e,"px"):this.bar.style.left="".concat(e,"px"),this.bar.title=o,!1}},{key:"hide",value:function(){this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar)}},{key:"setCustomTime",value:function(t){this.customTime=za.convert(t,"Date"),this.redraw()}},{key:"getCustomTime",value:function(){return new Date(this.customTime.valueOf())}},{key:"setCustomMarker",value:function(t,e){var i,n,o=document.createElement("div");(o.className="vis-custom-time-marker",o.innerHTML=t,o.style.position="absolute",e)&&(o.setAttribute("contenteditable","true"),o.addEventListener("pointerdown",(function(){o.focus()})),o.addEventListener("input",va(i=this._onMarkerChange).call(i,this)),o.title=t,o.addEventListener("blur",va(n=function(t){this.title!=t.target.innerHTML&&(this._onMarkerChanged(t),this.title=t.target.innerHTML)}).call(n,this)));this.bar.appendChild(o)}},{key:"setCustomTitle",value:function(t){this.options.title=t}},{key:"_onDragStart",value:function(t){this.eventParams.dragging=!0,this.eventParams.customTime=this.customTime,t.stopPropagation()}},{key:"_onDrag",value:function(t){if(this.eventParams.dragging){var e=this.options.rtl?-1*t.deltaX:t.deltaX,i=this.body.util.toScreen(this.eventParams.customTime)+e,n=this.body.util.toTime(i),o=this.body.util.getScale(),s=this.body.util.getStep(),r=this.options.snap,a=r?r(n,o,s):n;this.setCustomTime(a),this.body.emitter.emit("timechange",{id:this.options.id,time:new Date(this.customTime.valueOf()),event:t}),t.stopPropagation()}}},{key:"_onDragEnd",value:function(t){this.eventParams.dragging&&(this.body.emitter.emit("timechanged",{id:this.options.id,time:new Date(this.customTime.valueOf()),event:t}),t.stopPropagation())}},{key:"_onMarkerChange",value:function(t){this.body.emitter.emit("markerchange",{id:this.options.id,title:t.target.innerHTML,event:t}),t.stopPropagation()}},{key:"_onMarkerChanged",value:function(t){this.body.emitter.emit("markerchanged",{id:this.options.id,title:t.target.innerHTML,event:t}),t.stopPropagation()}}],[{key:"customTimeFromTarget",value:function(t){for(var e=t.target;e;){if(e.hasOwnProperty("custom-time"))return e["custom-time"];e=e.parentNode}return null}}]),e}(Ja);Ph("\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IiIsImZpbGUiOiJhbmltYXRpb24uY3NzIn0= */");Ph('.vis-current-time{background-color:#ff7f6e;width:2px;z-index:1;pointer-events:none}.vis-rolling-mode-btn{height:40px;width:40px;position:absolute;top:7px;right:20px;border-radius:50%;font-size:28px;cursor:pointer;opacity:.8;color:#fff;font-weight:700;text-align:center;background:#3876c2}.vis-rolling-mode-btn:before{content:"\\26F6"}.vis-rolling-mode-btn:hover{opacity:1}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbImN1cnJlbnR0aW1lLmNzcyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxrQkFDRSx3QkFBeUIsQ0FDekIsU0FBVSxDQUNWLFNBQVUsQ0FDVixtQkFDRixDQUVBLHNCQUNFLFdBQVksQ0FDWixVQUFXLENBQ1gsaUJBQWtCLENBQ2xCLE9BQVEsQ0FDUixVQUFXLENBQ1gsaUJBQWtCLENBQ2xCLGNBQWUsQ0FDZixjQUFlLENBQ2YsVUFBWSxDQUNaLFVBQVksQ0FDWixlQUFpQixDQUNqQixpQkFBa0IsQ0FDbEIsa0JBQ0YsQ0FDQSw2QkFDRSxlQUNGLENBRUEsNEJBQ0UsU0FDRiIsImZpbGUiOiJjdXJyZW50dGltZS5jc3MiLCJzb3VyY2VzQ29udGVudCI6WyIudmlzLWN1cnJlbnQtdGltZSB7XG4gIGJhY2tncm91bmQtY29sb3I6ICNGRjdGNkU7XG4gIHdpZHRoOiAycHg7XG4gIHotaW5kZXg6IDE7XG4gIHBvaW50ZXItZXZlbnRzOiBub25lO1xufVxuXG4udmlzLXJvbGxpbmctbW9kZS1idG4ge1xuICBoZWlnaHQ6IDQwcHg7XG4gIHdpZHRoOiA0MHB4O1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHRvcDogN3B4O1xuICByaWdodDogMjBweDtcbiAgYm9yZGVyLXJhZGl1czogNTAlO1xuICBmb250LXNpemU6IDI4cHg7XG4gIGN1cnNvcjogcG9pbnRlcjtcbiAgb3BhY2l0eTogMC44O1xuICBjb2xvcjogd2hpdGU7XG4gIGZvbnQtd2VpZ2h0OiBib2xkO1xuICB0ZXh0LWFsaWduOiBjZW50ZXI7XG4gIGJhY2tncm91bmQ6ICMzODc2YzI7XG59XG4udmlzLXJvbGxpbmctbW9kZS1idG46YmVmb3JlIHtcbiAgY29udGVudDogXCJcXDI2RjZcIjtcbn1cblxuLnZpcy1yb2xsaW5nLW1vZGUtYnRuOmhvdmVyIHtcbiAgb3BhY2l0eTogMTtcbn0iXX0= */');Ph(".vis-panel{position:absolute;padding:0;margin:0;box-sizing:border-box}.vis-panel.vis-bottom,.vis-panel.vis-center,.vis-panel.vis-left,.vis-panel.vis-right,.vis-panel.vis-top{border:1px #bfbfbf}.vis-panel.vis-center,.vis-panel.vis-left,.vis-panel.vis-right{border-top-style:solid;border-bottom-style:solid;overflow:hidden}.vis-left.vis-panel.vis-vertical-scroll,.vis-right.vis-panel.vis-vertical-scroll{height:100%;overflow-x:hidden;overflow-y:scroll}.vis-left.vis-panel.vis-vertical-scroll{direction:rtl}.vis-left.vis-panel.vis-vertical-scroll .vis-content,.vis-right.vis-panel.vis-vertical-scroll{direction:ltr}.vis-right.vis-panel.vis-vertical-scroll .vis-content{direction:rtl}.vis-panel.vis-bottom,.vis-panel.vis-center,.vis-panel.vis-top{border-left-style:solid;border-right-style:solid}.vis-background{overflow:hidden}.vis-panel>.vis-content{position:relative}.vis-panel .vis-shadow{position:absolute;width:100%;height:1px;box-shadow:0 0 10px rgba(0,0,0,.8)}.vis-panel .vis-shadow.vis-top{top:-1px;left:0}.vis-panel .vis-shadow.vis-bottom{bottom:-1px;left:0}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbInBhbmVsLmNzcyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxXQUNFLGlCQUFrQixDQUVsQixTQUFVLENBQ1YsUUFBUyxDQUVULHFCQUNGLENBRUEsd0dBS0Usa0JBQ0YsQ0FFQSwrREFHRSxzQkFBdUIsQ0FDdkIseUJBQTBCLENBQzFCLGVBQ0YsQ0FFQSxpRkFDRSxXQUFZLENBQ1osaUJBQWtCLENBQ2xCLGlCQUNGLENBRUEsd0NBQ0UsYUFDRixDQU1BLDhGQUNFLGFBQ0YsQ0FFQSxzREFDRSxhQUNGLENBRUEsK0RBR0UsdUJBQXdCLENBQ3hCLHdCQUNGLENBRUEsZ0JBQ0UsZUFDRixDQUVBLHdCQUNFLGlCQUNGLENBRUEsdUJBQ0UsaUJBQWtCLENBQ2xCLFVBQVcsQ0FDWCxVQUFXLENBQ1gsa0NBSUYsQ0FFQSwrQkFDRSxRQUFTLENBQ1QsTUFDRixDQUVBLGtDQUNFLFdBQVksQ0FDWixNQUNGIiwiZmlsZSI6InBhbmVsLmNzcyIsInNvdXJjZXNDb250ZW50IjpbIi52aXMtcGFuZWwge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG5cbiAgcGFkZGluZzogMDtcbiAgbWFyZ2luOiAwO1xuXG4gIGJveC1zaXppbmc6IGJvcmRlci1ib3g7XG59XG5cbi52aXMtcGFuZWwudmlzLWNlbnRlcixcbi52aXMtcGFuZWwudmlzLWxlZnQsXG4udmlzLXBhbmVsLnZpcy1yaWdodCxcbi52aXMtcGFuZWwudmlzLXRvcCxcbi52aXMtcGFuZWwudmlzLWJvdHRvbSB7XG4gIGJvcmRlcjogMXB4ICNiZmJmYmY7XG59XG5cbi52aXMtcGFuZWwudmlzLWNlbnRlcixcbi52aXMtcGFuZWwudmlzLWxlZnQsXG4udmlzLXBhbmVsLnZpcy1yaWdodCB7XG4gIGJvcmRlci10b3Atc3R5bGU6IHNvbGlkO1xuICBib3JkZXItYm90dG9tLXN0eWxlOiBzb2xpZDtcbiAgb3ZlcmZsb3c6IGhpZGRlbjtcbn1cblxuLnZpcy1sZWZ0LnZpcy1wYW5lbC52aXMtdmVydGljYWwtc2Nyb2xsLCAudmlzLXJpZ2h0LnZpcy1wYW5lbC52aXMtdmVydGljYWwtc2Nyb2xsIHtcbiAgaGVpZ2h0OiAxMDAlO1xuICBvdmVyZmxvdy14OiBoaWRkZW47XG4gIG92ZXJmbG93LXk6IHNjcm9sbDtcbn0gXG5cbi52aXMtbGVmdC52aXMtcGFuZWwudmlzLXZlcnRpY2FsLXNjcm9sbCB7XG4gIGRpcmVjdGlvbjogcnRsO1xufVxuXG4udmlzLWxlZnQudmlzLXBhbmVsLnZpcy12ZXJ0aWNhbC1zY3JvbGwgLnZpcy1jb250ZW50IHtcbiAgZGlyZWN0aW9uOiBsdHI7XG59XG5cbi52aXMtcmlnaHQudmlzLXBhbmVsLnZpcy12ZXJ0aWNhbC1zY3JvbGwge1xuICBkaXJlY3Rpb246IGx0cjtcbn1cblxuLnZpcy1yaWdodC52aXMtcGFuZWwudmlzLXZlcnRpY2FsLXNjcm9sbCAudmlzLWNvbnRlbnQge1xuICBkaXJlY3Rpb246IHJ0bDtcbn1cblxuLnZpcy1wYW5lbC52aXMtY2VudGVyLFxuLnZpcy1wYW5lbC52aXMtdG9wLFxuLnZpcy1wYW5lbC52aXMtYm90dG9tIHtcbiAgYm9yZGVyLWxlZnQtc3R5bGU6IHNvbGlkO1xuICBib3JkZXItcmlnaHQtc3R5bGU6IHNvbGlkO1xufVxuXG4udmlzLWJhY2tncm91bmQge1xuICBvdmVyZmxvdzogaGlkZGVuO1xufVxuXG4udmlzLXBhbmVsID4gLnZpcy1jb250ZW50IHtcbiAgcG9zaXRpb246IHJlbGF0aXZlO1xufVxuXG4udmlzLXBhbmVsIC52aXMtc2hhZG93IHtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICB3aWR0aDogMTAwJTtcbiAgaGVpZ2h0OiAxcHg7XG4gIGJveC1zaGFkb3c6IDAgMCAxMHB4IHJnYmEoMCwwLDAsMC44KTtcbiAgLyogVE9ETzogZmluZCBhIG5pY2Ugd2F5IHRvIGVuc3VyZSB2aXMtc2hhZG93cyBhcmUgZHJhd24gb24gdG9wIG9mIGl0ZW1zXG4gIHotaW5kZXg6IDE7XG4gICovXG59XG5cbi52aXMtcGFuZWwgLnZpcy1zaGFkb3cudmlzLXRvcCB7XG4gIHRvcDogLTFweDtcbiAgbGVmdDogMDtcbn1cblxuLnZpcy1wYW5lbCAudmlzLXNoYWRvdy52aXMtYm90dG9tIHtcbiAgYm90dG9tOiAtMXB4O1xuICBsZWZ0OiAwO1xufSJdfQ== */");Ph(".vis-graph-group0{fill:#4f81bd;fill-opacity:0;stroke-width:2px;stroke:#4f81bd}.vis-graph-group1{fill:#f79646;fill-opacity:0;stroke-width:2px;stroke:#f79646}.vis-graph-group2{fill:#8c51cf;fill-opacity:0;stroke-width:2px;stroke:#8c51cf}.vis-graph-group3{fill:#75c841;fill-opacity:0;stroke-width:2px;stroke:#75c841}.vis-graph-group4{fill:#ff0100;fill-opacity:0;stroke-width:2px;stroke:#ff0100}.vis-graph-group5{fill:#37d8e6;fill-opacity:0;stroke-width:2px;stroke:#37d8e6}.vis-graph-group6{fill:#042662;fill-opacity:0;stroke-width:2px;stroke:#042662}.vis-graph-group7{fill:#00ff26;fill-opacity:0;stroke-width:2px;stroke:#00ff26}.vis-graph-group8{fill:#f0f;fill-opacity:0;stroke-width:2px;stroke:#f0f}.vis-graph-group9{fill:#8f3938;fill-opacity:0;stroke-width:2px;stroke:#8f3938}.vis-timeline .vis-fill{fill-opacity:.1;stroke:none}.vis-timeline .vis-bar{fill-opacity:.5;stroke-width:1px}.vis-timeline .vis-point{stroke-width:2px;fill-opacity:1}.vis-timeline .vis-legend-background{stroke-width:1px;fill-opacity:.9;fill:#fff;stroke:#c2c2c2}.vis-timeline .vis-outline{stroke-width:1px;fill-opacity:1;fill:#fff;stroke:#e5e5e5}.vis-timeline .vis-icon-fill{fill-opacity:.3;stroke:none}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbInBhdGhTdHlsZXMuY3NzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLGtCQUNJLFlBQVksQ0FDWixjQUFjLENBQ2QsZ0JBQWdCLENBQ2hCLGNBQ0osQ0FFQSxrQkFDSSxZQUFZLENBQ1osY0FBYyxDQUNkLGdCQUFnQixDQUNoQixjQUNKLENBRUEsa0JBQ0ksWUFBYSxDQUNiLGNBQWMsQ0FDZCxnQkFBZ0IsQ0FDaEIsY0FDSixDQUVBLGtCQUNJLFlBQWEsQ0FDYixjQUFjLENBQ2QsZ0JBQWdCLENBQ2hCLGNBQ0osQ0FFQSxrQkFDSSxZQUFhLENBQ2IsY0FBYyxDQUNkLGdCQUFnQixDQUNoQixjQUNKLENBRUEsa0JBQ0ksWUFBYSxDQUNiLGNBQWMsQ0FDZCxnQkFBZ0IsQ0FDaEIsY0FDSixDQUVBLGtCQUNJLFlBQWEsQ0FDYixjQUFjLENBQ2QsZ0JBQWdCLENBQ2hCLGNBQ0osQ0FFQSxrQkFDSSxZQUFZLENBQ1osY0FBYyxDQUNkLGdCQUFnQixDQUNoQixjQUNKLENBRUEsa0JBQ0ksU0FBWSxDQUNaLGNBQWMsQ0FDZCxnQkFBZ0IsQ0FDaEIsV0FDSixDQUVBLGtCQUNJLFlBQWEsQ0FDYixjQUFjLENBQ2QsZ0JBQWdCLENBQ2hCLGNBQ0osQ0FFQSx3QkFDSSxlQUFnQixDQUNoQixXQUNKLENBR0EsdUJBQ0ksZUFBZ0IsQ0FDaEIsZ0JBQ0osQ0FFQSx5QkFDSSxnQkFBZ0IsQ0FDaEIsY0FDSixDQUdBLHFDQUNJLGdCQUFnQixDQUNoQixlQUFnQixDQUNoQixTQUFhLENBQ2IsY0FDSixDQUdBLDJCQUNJLGdCQUFnQixDQUNoQixjQUFjLENBQ2QsU0FBYSxDQUNiLGNBQ0osQ0FFQSw2QkFDSSxlQUFnQixDQUNoQixXQUNKIiwiZmlsZSI6InBhdGhTdHlsZXMuY3NzIiwic291cmNlc0NvbnRlbnQiOlsiLnZpcy1ncmFwaC1ncm91cDAge1xuICAgIGZpbGw6IzRmODFiZDtcbiAgICBmaWxsLW9wYWNpdHk6MDtcbiAgICBzdHJva2Utd2lkdGg6MnB4O1xuICAgIHN0cm9rZTogIzRmODFiZDtcbn1cblxuLnZpcy1ncmFwaC1ncm91cDEge1xuICAgIGZpbGw6I2Y3OTY0NjtcbiAgICBmaWxsLW9wYWNpdHk6MDtcbiAgICBzdHJva2Utd2lkdGg6MnB4O1xuICAgIHN0cm9rZTogI2Y3OTY0Njtcbn1cblxuLnZpcy1ncmFwaC1ncm91cDIge1xuICAgIGZpbGw6ICM4YzUxY2Y7XG4gICAgZmlsbC1vcGFjaXR5OjA7XG4gICAgc3Ryb2tlLXdpZHRoOjJweDtcbiAgICBzdHJva2U6ICM4YzUxY2Y7XG59XG5cbi52aXMtZ3JhcGgtZ3JvdXAzIHtcbiAgICBmaWxsOiAjNzVjODQxO1xuICAgIGZpbGwtb3BhY2l0eTowO1xuICAgIHN0cm9rZS13aWR0aDoycHg7XG4gICAgc3Ryb2tlOiAjNzVjODQxO1xufVxuXG4udmlzLWdyYXBoLWdyb3VwNCB7XG4gICAgZmlsbDogI2ZmMDEwMDtcbiAgICBmaWxsLW9wYWNpdHk6MDtcbiAgICBzdHJva2Utd2lkdGg6MnB4O1xuICAgIHN0cm9rZTogI2ZmMDEwMDtcbn1cblxuLnZpcy1ncmFwaC1ncm91cDUge1xuICAgIGZpbGw6ICMzN2Q4ZTY7XG4gICAgZmlsbC1vcGFjaXR5OjA7XG4gICAgc3Ryb2tlLXdpZHRoOjJweDtcbiAgICBzdHJva2U6ICMzN2Q4ZTY7XG59XG5cbi52aXMtZ3JhcGgtZ3JvdXA2IHtcbiAgICBmaWxsOiAjMDQyNjYyO1xuICAgIGZpbGwtb3BhY2l0eTowO1xuICAgIHN0cm9rZS13aWR0aDoycHg7XG4gICAgc3Ryb2tlOiAjMDQyNjYyO1xufVxuXG4udmlzLWdyYXBoLWdyb3VwNyB7XG4gICAgZmlsbDojMDBmZjI2O1xuICAgIGZpbGwtb3BhY2l0eTowO1xuICAgIHN0cm9rZS13aWR0aDoycHg7XG4gICAgc3Ryb2tlOiAjMDBmZjI2O1xufVxuXG4udmlzLWdyYXBoLWdyb3VwOCB7XG4gICAgZmlsbDojZmYwMGZmO1xuICAgIGZpbGwtb3BhY2l0eTowO1xuICAgIHN0cm9rZS13aWR0aDoycHg7XG4gICAgc3Ryb2tlOiAjZmYwMGZmO1xufVxuXG4udmlzLWdyYXBoLWdyb3VwOSB7XG4gICAgZmlsbDogIzhmMzkzODtcbiAgICBmaWxsLW9wYWNpdHk6MDtcbiAgICBzdHJva2Utd2lkdGg6MnB4O1xuICAgIHN0cm9rZTogIzhmMzkzODtcbn1cblxuLnZpcy10aW1lbGluZSAudmlzLWZpbGwge1xuICAgIGZpbGwtb3BhY2l0eTowLjE7XG4gICAgc3Ryb2tlOiBub25lO1xufVxuXG5cbi52aXMtdGltZWxpbmUgLnZpcy1iYXIge1xuICAgIGZpbGwtb3BhY2l0eTowLjU7XG4gICAgc3Ryb2tlLXdpZHRoOjFweDtcbn1cblxuLnZpcy10aW1lbGluZSAudmlzLXBvaW50IHtcbiAgICBzdHJva2Utd2lkdGg6MnB4O1xuICAgIGZpbGwtb3BhY2l0eToxLjA7XG59XG5cblxuLnZpcy10aW1lbGluZSAudmlzLWxlZ2VuZC1iYWNrZ3JvdW5kIHtcbiAgICBzdHJva2Utd2lkdGg6MXB4O1xuICAgIGZpbGwtb3BhY2l0eTowLjk7XG4gICAgZmlsbDogI2ZmZmZmZjtcbiAgICBzdHJva2U6ICNjMmMyYzI7XG59XG5cblxuLnZpcy10aW1lbGluZSAudmlzLW91dGxpbmUge1xuICAgIHN0cm9rZS13aWR0aDoxcHg7XG4gICAgZmlsbC1vcGFjaXR5OjE7XG4gICAgZmlsbDogI2ZmZmZmZjtcbiAgICBzdHJva2U6ICNlNWU1ZTU7XG59XG5cbi52aXMtdGltZWxpbmUgLnZpcy1pY29uLWZpbGwge1xuICAgIGZpbGwtb3BhY2l0eTowLjM7XG4gICAgc3Ryb2tlOiBub25lO1xufVxuIl19 */");Ph(".vis-timeline{position:relative;border:1px solid #bfbfbf;overflow:hidden;padding:0;margin:0;box-sizing:border-box}.vis-loading-screen{width:100%;height:100%;position:absolute;top:0;left:0}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbInRpbWVsaW5lLmNzcyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFDQSxjQUNFLGlCQUFrQixDQUNsQix3QkFBeUIsQ0FDekIsZUFBZ0IsQ0FDaEIsU0FBVSxDQUNWLFFBQVMsQ0FDVCxxQkFDRixDQUVBLG9CQUNFLFVBQVcsQ0FDWCxXQUFZLENBQ1osaUJBQWtCLENBQ2xCLEtBQU0sQ0FDTixNQUNGIiwiZmlsZSI6InRpbWVsaW5lLmNzcyIsInNvdXJjZXNDb250ZW50IjpbIlxuLnZpcy10aW1lbGluZSB7XG4gIHBvc2l0aW9uOiByZWxhdGl2ZTtcbiAgYm9yZGVyOiAxcHggc29saWQgI2JmYmZiZjtcbiAgb3ZlcmZsb3c6IGhpZGRlbjtcbiAgcGFkZGluZzogMDtcbiAgbWFyZ2luOiAwO1xuICBib3gtc2l6aW5nOiBib3JkZXItYm94O1xufVxuXG4udmlzLWxvYWRpbmctc2NyZWVuIHtcbiAgd2lkdGg6IDEwMCU7XG4gIGhlaWdodDogMTAwJTtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICB0b3A6IDA7XG4gIGxlZnQ6IDA7XG59Il19 */");Ph(".vis [class*=span]{min-height:0;width:auto}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbImJvb3RzdHJhcC5jc3MiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBRUEsbUJBQ0UsWUFBYSxDQUNiLFVBQ0YiLCJmaWxlIjoiYm9vdHN0cmFwLmNzcyIsInNvdXJjZXNDb250ZW50IjpbIi8qIG92ZXJyaWRlIHNvbWUgYm9vdHN0cmFwIHN0eWxlcyBzY3Jld2luZyB1cCB0aGUgdGltZWxpbmVzIGNzcyAqL1xuXG4udmlzIFtjbGFzcyo9XCJzcGFuXCJdIHtcbiAgbWluLWhlaWdodDogMDtcbiAgd2lkdGg6IGF1dG87XG59XG4iXX0= */");var cd=function(){function t(){ba(this,t)}return xa(t,[{key:"_create",value:function(t){var e,i,n,o=this;this.dom={},this.dom.container=t,this.dom.container.style.position="relative",this.dom.root=document.createElement("div"),this.dom.background=document.createElement("div"),this.dom.backgroundVertical=document.createElement("div"),this.dom.backgroundHorizontal=document.createElement("div"),this.dom.centerContainer=document.createElement("div"),this.dom.leftContainer=document.createElement("div"),this.dom.rightContainer=document.createElement("div"),this.dom.center=document.createElement("div"),this.dom.left=document.createElement("div"),this.dom.right=document.createElement("div"),this.dom.top=document.createElement("div"),this.dom.bottom=document.createElement("div"),this.dom.shadowTop=document.createElement("div"),this.dom.shadowBottom=document.createElement("div"),this.dom.shadowTopLeft=document.createElement("div"),this.dom.shadowBottomLeft=document.createElement("div"),this.dom.shadowTopRight=document.createElement("div"),this.dom.shadowBottomRight=document.createElement("div"),this.dom.rollingModeBtn=document.createElement("div"),this.dom.loadingScreen=document.createElement("div"),this.dom.root.className="vis-timeline",this.dom.background.className="vis-panel vis-background",this.dom.backgroundVertical.className="vis-panel vis-background vis-vertical",this.dom.backgroundHorizontal.className="vis-panel vis-background vis-horizontal",this.dom.centerContainer.className="vis-panel vis-center",this.dom.leftContainer.className="vis-panel vis-left",this.dom.rightContainer.className="vis-panel vis-right",this.dom.top.className="vis-panel vis-top",this.dom.bottom.className="vis-panel vis-bottom",this.dom.left.className="vis-content",this.dom.center.className="vis-content",this.dom.right.className="vis-content",this.dom.shadowTop.className="vis-shadow vis-top",this.dom.shadowBottom.className="vis-shadow vis-bottom",this.dom.shadowTopLeft.className="vis-shadow vis-top",this.dom.shadowBottomLeft.className="vis-shadow vis-bottom",this.dom.shadowTopRight.className="vis-shadow vis-top",this.dom.shadowBottomRight.className="vis-shadow vis-bottom",this.dom.rollingModeBtn.className="vis-rolling-mode-btn",this.dom.loadingScreen.className="vis-loading-screen",this.dom.root.appendChild(this.dom.background),this.dom.root.appendChild(this.dom.backgroundVertical),this.dom.root.appendChild(this.dom.backgroundHorizontal),this.dom.root.appendChild(this.dom.centerContainer),this.dom.root.appendChild(this.dom.leftContainer),this.dom.root.appendChild(this.dom.rightContainer),this.dom.root.appendChild(this.dom.top),this.dom.root.appendChild(this.dom.bottom),this.dom.root.appendChild(this.dom.rollingModeBtn),this.dom.centerContainer.appendChild(this.dom.center),this.dom.leftContainer.appendChild(this.dom.left),this.dom.rightContainer.appendChild(this.dom.right),this.dom.centerContainer.appendChild(this.dom.shadowTop),this.dom.centerContainer.appendChild(this.dom.shadowBottom),this.dom.leftContainer.appendChild(this.dom.shadowTopLeft),this.dom.leftContainer.appendChild(this.dom.shadowBottomLeft),this.dom.rightContainer.appendChild(this.dom.shadowTopRight),this.dom.rightContainer.appendChild(this.dom.shadowBottomRight),this.props={root:{},background:{},centerContainer:{},leftContainer:{},rightContainer:{},center:{},left:{},right:{},top:{},bottom:{},border:{},scrollTop:0,scrollTopMin:0},this.on("rangechange",(function(){!0===o.initialDrawDone&&o._redraw()})),this.on("rangechanged",(function(){o.initialRangeChangeDone||(o.initialRangeChangeDone=!0)})),this.on("touch",va(e=this._onTouch).call(e,this)),this.on("panmove",va(i=this._onDrag).call(i,this));var s=this;this._origRedraw=va(n=this._redraw).call(n,this),this._redraw=za.throttle(this._origRedraw),this.on("_change",(function(t){s.itemSet&&s.itemSet.initialItemSetDrawn&&t&&1==t.queue?s._redraw():s._origRedraw()})),this.hammer=new Vh(this.dom.root);var r=this.hammer.get("pinch").set({enable:!0});r&&function(t){t.getTouchAction=function(){return["pan-y"]}}(r),this.hammer.get("pan").set({threshold:5,direction:Vh.DIRECTION_ALL}),this.listeners={};var a,l,h=["tap","doubletap","press","pinch","pan","panstart","panmove","panend"];function d(t){this.isActive()&&this.emit("mousewheel",t);var e=0,i=0;if("detail"in t&&(i=-1*t.detail),"wheelDelta"in t&&(i=t.wheelDelta),"wheelDeltaY"in t&&(i=t.wheelDeltaY),"wheelDeltaX"in t&&(e=-1*t.wheelDeltaX),"axis"in t&&t.axis===t.HORIZONTAL_AXIS&&(e=-1*i,i=0),"deltaY"in t&&(i=-1*t.deltaY),"deltaX"in t&&(e=t.deltaX),t.deltaMode&&(1===t.deltaMode?(e*=40,i*=40):(e*=40,i*=800)),this.options.preferZoom){if(!this.options.zoomKey||t[this.options.zoomKey])return}else if(this.options.zoomKey&&t[this.options.zoomKey])return;if(this.options.verticalScroll||this.options.horizontalScroll)if(this.options.verticalScroll&&Math.abs(i)>=Math.abs(e)){var n=this.props.scrollTop,o=n+i;if(this.isActive())this._setScrollTop(o)!==n&&(this._redraw(),this.emit("scroll",t),t.preventDefault())}else if(this.options.horizontalScroll){var s=(Math.abs(e)>=Math.abs(i)?e:i)/120*(this.range.end-this.range.start)/20,r=this.range.start+s,a=this.range.end+s,l={animation:!1,byUser:!0,event:t};this.range.setRange(r,a,l),t.preventDefault()}}mn(h).call(h,(function(t){var e=function(e){s.isActive()&&s.emit(t,e)};s.hammer.on(t,e),s.listeners[t]=e})),jh(this.hammer,(function(t){s.emit("touch",t)})),a=this.hammer,(l=function(t){s.emit("release",t)}).inputHandler=function(t){t.isFinal&&l(t)},a.on("hammer.input",l.inputHandler);var u="onwheel"in document.createElement("div")?"wheel":void 0!==document.onmousewheel?"mousewheel":this.dom.centerContainer.addEventListener?"DOMMouseScroll":"onmousewheel";function c(t){if(s.options.verticalScroll)if(s._isProgramaticallyScrolled)s._isProgramaticallyScrolled=!1;else if(t.preventDefault(),s.isActive()){var e=-t.target.scrollTop;s._setScrollTop(e),s._redraw(),s.emit("scrollSide",t)}}this.dom.top.addEventListener,this.dom.bottom.addEventListener,this.dom.centerContainer.addEventListener(u,va(d).call(d,this),!1),this.dom.top.addEventListener(u,va(d).call(d,this),!1),this.dom.bottom.addEventListener(u,va(d).call(d,this),!1),this.dom.left.parentNode.addEventListener("scroll",va(c).call(c,this)),this.dom.right.parentNode.addEventListener("scroll",va(c).call(c,this));var p=!1;function m(t){var e;if(t.preventDefault&&(s.emit("dragover",s.getEventProperties(t)),t.preventDefault()),Fo(e=t.target.className).call(e,"timeline")>-1&&!p)return t.dataTransfer.dropEffect="move",p=!0,!1}function f(t){t.preventDefault&&t.preventDefault(),t.stopPropagation&&t.stopPropagation();try{var e=JSON.parse(t.dataTransfer.getData("text"));if(!e||!e.content)return}catch(t){return!1}return p=!1,t.center={x:t.clientX,y:t.clientY},"item"!==e.target?s.itemSet._onAddItem(t):s.itemSet._onDropObjectOnItem(t),s.emit("drop",s.getEventProperties(t)),!1}if(this.dom.center.addEventListener("dragover",va(m).call(m,this),!1),this.dom.center.addEventListener("drop",va(f).call(f,this),!1),this.customTimes=[],this.touch={},this.redrawCount=0,this.initialDrawDone=!1,this.initialRangeChangeDone=!1,!t)throw new Error("No container provided");t.appendChild(this.dom.root),t.appendChild(this.dom.loadingScreen)}},{key:"setOptions",value:function(t){var e;if(t){if(za.selectiveExtend(["width","height","minHeight","maxHeight","autoResize","start","end","clickToUse","dataAttributes","hiddenDates","locale","locales","moment","preferZoom","rtl","zoomKey","horizontalScroll","verticalScroll","longSelectPressTime"],this.options,t),this.dom.rollingModeBtn.style.visibility="hidden",this.options.rtl&&(this.dom.container.style.direction="rtl",this.dom.backgroundVertical.className="vis-panel vis-background vis-vertical-rtl"),this.options.verticalScroll&&(this.options.rtl?this.dom.rightContainer.className="vis-panel vis-right vis-vertical-scroll":this.dom.leftContainer.className="vis-panel vis-left vis-vertical-scroll"),"object"!==zo(this.options.orientation)&&(this.options.orientation={item:void 0,axis:void 0}),"orientation"in t&&("string"==typeof t.orientation?this.options.orientation={item:t.orientation,axis:t.orientation}:"object"===zo(t.orientation)&&("item"in t.orientation&&(this.options.orientation.item=t.orientation.item),"axis"in t.orientation&&(this.options.orientation.axis=t.orientation.axis))),"both"===this.options.orientation.axis){if(!this.timeAxis2){var i=this.timeAxis2=new Jh(this.body);i.setOptions=function(t){var e=t?za.extend({},t):{};e.orientation="top",Jh.prototype.setOptions.call(i,e)},this.components.push(i)}}else if(this.timeAxis2){var n,o,s=Fo(n=this.components).call(n,this.timeAxis2);if(-1!==s)Hr(o=this.components).call(o,s,1);this.timeAxis2.destroy(),this.timeAxis2=null}"function"==typeof t.drawPoints&&(t.drawPoints={onRender:t.drawPoints}),"hiddenDates"in this.options&&tl(this.options.moment,this.body,this.options.hiddenDates),"clickToUse"in t&&(t.clickToUse?this.activator||(this.activator=new $h(this.dom.root)):this.activator&&(this.activator.destroy(),delete this.activator)),this._initAutoResize()}if(mn(e=this.components).call(e,(function(e){return e.setOptions(t)})),"configure"in t){var r;this.configurator||(this.configurator=this._createConfigurator()),this.configurator.setOptions(t.configure);var a=za.deepExtend({},this.options);mn(r=this.components).call(r,(function(t){za.deepExtend(a,t.options)})),this.configurator.setModuleOptions({global:a})}this._redraw()}},{key:"isActive",value:function(){return!this.activator||this.activator.active}},{key:"destroy",value:function(){var t;for(var e in this.setItems(null),this.setGroups(null),this.off(),this._stopAutoResize(),this.dom.root.parentNode&&this.dom.root.parentNode.removeChild(this.dom.root),this.dom=null,this.activator&&(this.activator.destroy(),delete this.activator),this.listeners)this.listeners.hasOwnProperty(e)&&delete this.listeners[e];this.listeners=null,this.hammer&&this.hammer.destroy(),this.hammer=null,mn(t=this.components).call(t,(function(t){return t.destroy()})),this.body=null}},{key:"setCustomTime",value:function(t,e){var i,n=En(i=this.customTimes).call(i,(function(t){return e===t.options.id}));if(0===n.length)throw new Error("No custom time bar found with id ".concat(Vs(e)));n.length>0&&n[0].setCustomTime(t)}},{key:"getCustomTime",value:function(t){var e,i=En(e=this.customTimes).call(e,(function(e){return e.options.id===t}));if(0===i.length)throw new Error("No custom time bar found with id ".concat(Vs(t)));return i[0].getCustomTime()}},{key:"setCustomTimeMarker",value:function(t,e,i){var n,o=En(n=this.customTimes).call(n,(function(t){return t.options.id===e}));if(0===o.length)throw new Error("No custom time bar found with id ".concat(Vs(e)));o.length>0&&o[0].setCustomMarker(t,i)}},{key:"setCustomTimeTitle",value:function(t,e){var i,n=En(i=this.customTimes).call(i,(function(t){return t.options.id===e}));if(0===n.length)throw new Error("No custom time bar found with id ".concat(Vs(e)));if(n.length>0)return n[0].setCustomTitle(t)}},{key:"getEventProperties",value:function(t){return{event:t}}},{key:"addCustomTime",value:function(t,e){var i,n=void 0!==t?za.convert(t,"Date"):new Date;if(rr(i=this.customTimes).call(i,(function(t){return t.options.id===e})))throw new Error("A custom time with id ".concat(Vs(e)," already exists"));var o=new ud(this.body,za.extend({},this.options,{time:n,id:e,snap:this.itemSet.options.snap}));return this.customTimes.push(o),this.components.push(o),this._redraw(),e}},{key:"removeCustomTime",value:function(t){var e,i=this,n=En(e=this.customTimes).call(e,(function(e){return e.options.id===t}));if(0===n.length)throw new Error("No custom time bar found with id ".concat(Vs(t)));mn(n).call(n,(function(t){var e,n,o,s;Hr(e=i.customTimes).call(e,Fo(n=i.customTimes).call(n,t),1),Hr(o=i.components).call(o,Fo(s=i.components).call(s,t),1),t.destroy()}))}},{key:"getVisibleItems",value:function(){return this.itemSet&&this.itemSet.getVisibleItems()||[]}},{key:"getVisibleGroups",value:function(){return this.itemSet&&this.itemSet.getVisibleGroups()||[]}},{key:"fit",value:function(t,e){var i=this.getDataRange();if(null!==i.min||null!==i.max){var n=i.max-i.min,o=new Date(i.min.valueOf()-.01*n),s=new Date(i.max.valueOf()+.01*n),r=!t||void 0===t.animation||t.animation;this.range.setRange(o,s,{animation:r},e)}}},{key:"getDataRange",value:function(){throw new Error("Cannot invoke abstract method getDataRange")}},{key:"setWindow",value:function(t,e,i,n){var o,s;"function"==typeof arguments[2]&&(n=arguments[2],i={}),1==arguments.length?(o=void 0===(s=arguments[0]).animation||s.animation,this.range.setRange(s.start,s.end,{animation:o})):2==arguments.length&&"function"==typeof arguments[1]?(s=arguments[0],n=arguments[1],o=void 0===s.animation||s.animation,this.range.setRange(s.start,s.end,{animation:o},n)):(o=!i||void 0===i.animation||i.animation,this.range.setRange(t,e,{animation:o},n))}},{key:"moveTo",value:function(t,e,i){"function"==typeof arguments[1]&&(i=arguments[1],e={});var n=this.range.end-this.range.start,o=za.convert(t,"Date").valueOf(),s=o-n/2,r=o+n/2,a=!e||void 0===e.animation||e.animation;this.range.setRange(s,r,{animation:a},i)}},{key:"getWindow",value:function(){var t=this.range.getRange();return{start:new Date(t.start),end:new Date(t.end)}}},{key:"zoomIn",value:function(t,e,i){if(!(!t||t<0||t>1)){"function"==typeof arguments[1]&&(i=arguments[1],e={});var n=this.getWindow(),o=n.start.valueOf(),s=n.end.valueOf(),r=s-o,a=r/(1+t),l=(r-a)/2,h=o+l,d=s-l;this.setWindow(h,d,e,i)}}},{key:"zoomOut",value:function(t,e,i){if(!(!t||t<0||t>1)){"function"==typeof arguments[1]&&(i=arguments[1],e={});var n=this.getWindow(),o=n.start.valueOf(),s=n.end.valueOf(),r=s-o,a=o-r*t/2,l=s+r*t/2;this.setWindow(a,l,e,i)}}},{key:"redraw",value:function(){this._redraw()}},{key:"_redraw",value:function(){var t;this.redrawCount++;var e=this.dom;if(e&&e.container&&0!=e.root.offsetWidth){var i=!1,n=this.options,o=this.props;el(this.options.moment,this.body,this.options.hiddenDates),"top"==n.orientation?(za.addClassName(e.root,"vis-top"),za.removeClassName(e.root,"vis-bottom")):(za.removeClassName(e.root,"vis-top"),za.addClassName(e.root,"vis-bottom")),n.rtl?(za.addClassName(e.root,"vis-rtl"),za.removeClassName(e.root,"vis-ltr")):(za.addClassName(e.root,"vis-ltr"),za.removeClassName(e.root,"vis-rtl")),e.root.style.maxHeight=za.option.asSize(n.maxHeight,""),e.root.style.minHeight=za.option.asSize(n.minHeight,""),e.root.style.width=za.option.asSize(n.width,"");var s=e.root.offsetWidth;o.border.left=1,o.border.right=1,o.border.top=1,o.border.bottom=1,o.center.height=e.center.offsetHeight,o.left.height=e.left.offsetHeight,o.right.height=e.right.offsetHeight,o.top.height=e.top.clientHeight||-o.border.top,o.bottom.height=e.bottom.clientHeight||-o.border.bottom;var r=Math.max(o.left.height,o.center.height,o.right.height),a=o.top.height+r+o.bottom.height+o.border.top+o.border.bottom;e.root.style.height=za.option.asSize(n.height,"".concat(a,"px")),o.root.height=e.root.offsetHeight,o.background.height=o.root.height;var l=o.root.height-o.top.height-o.bottom.height;o.centerContainer.height=l,o.leftContainer.height=l,o.rightContainer.height=o.leftContainer.height,o.root.width=s,o.background.width=o.root.width,this.initialDrawDone||(o.scrollbarWidth=za.getScrollBarWidth());var h=e.leftContainer.clientWidth,d=e.rightContainer.clientWidth;n.verticalScroll?n.rtl?(o.left.width=h||-o.border.left,o.right.width=d+o.scrollbarWidth||-o.border.right):(o.left.width=h+o.scrollbarWidth||-o.border.left,o.right.width=d||-o.border.right):(o.left.width=h||-o.border.left,o.right.width=d||-o.border.right),this._setDOM();var u=this._updateScrollTop();"top"!=n.orientation.item&&(u+=Math.max(o.centerContainer.height-o.center.height-o.border.top-o.border.bottom,0)),e.center.style.transform="translateY(".concat(u,"px)");var c=0==o.scrollTop?"hidden":"",p=o.scrollTop==o.scrollTopMin?"hidden":"";e.shadowTop.style.visibility=c,e.shadowBottom.style.visibility=p,e.shadowTopLeft.style.visibility=c,e.shadowBottomLeft.style.visibility=p,e.shadowTopRight.style.visibility=c,e.shadowBottomRight.style.visibility=p,n.verticalScroll&&(e.rightContainer.className="vis-panel vis-right vis-vertical-scroll",e.leftContainer.className="vis-panel vis-left vis-vertical-scroll",e.shadowTopRight.style.visibility="hidden",e.shadowBottomRight.style.visibility="hidden",e.shadowTopLeft.style.visibility="hidden",e.shadowBottomLeft.style.visibility="hidden",e.left.style.top="0px",e.right.style.top="0px"),(!n.verticalScroll||o.center.height<o.centerContainer.height)&&(e.left.style.top="".concat(u,"px"),e.right.style.top="".concat(u,"px"),e.rightContainer.className=e.rightContainer.className.replace(new RegExp("(?:^|\\s)vis-vertical-scroll(?:\\s|$)")," "),e.leftContainer.className=e.leftContainer.className.replace(new RegExp("(?:^|\\s)vis-vertical-scroll(?:\\s|$)")," "),o.left.width=h||-o.border.left,o.right.width=d||-o.border.right,this._setDOM());var m=o.center.height>o.centerContainer.height;this.hammer.get("pan").set({direction:m?Vh.DIRECTION_ALL:Vh.DIRECTION_HORIZONTAL}),this.hammer.get("press").set({time:this.options.longSelectPressTime}),mn(t=this.components).call(t,(function(t){i=t.redraw()||i}));if(i){if(this.redrawCount<5)return void this.body.emitter.emit("_change");console.log("WARNING: infinite loop in redraw?")}else this.redrawCount=0;this.body.emitter.emit("changed")}}},{key:"_setDOM",value:function(){var t=this.props,e=this.dom;t.leftContainer.width=t.left.width,t.rightContainer.width=t.right.width;var i=t.root.width-t.left.width-t.right.width;t.center.width=i,t.centerContainer.width=i,t.top.width=i,t.bottom.width=i,e.background.style.height="".concat(t.background.height,"px"),e.backgroundVertical.style.height="".concat(t.background.height,"px"),e.backgroundHorizontal.style.height="".concat(t.centerContainer.height,"px"),e.centerContainer.style.height="".concat(t.centerContainer.height,"px"),e.leftContainer.style.height="".concat(t.leftContainer.height,"px"),e.rightContainer.style.height="".concat(t.rightContainer.height,"px"),e.background.style.width="".concat(t.background.width,"px"),e.backgroundVertical.style.width="".concat(t.centerContainer.width,"px"),e.backgroundHorizontal.style.width="".concat(t.background.width,"px"),e.centerContainer.style.width="".concat(t.center.width,"px"),e.top.style.width="".concat(t.top.width,"px"),e.bottom.style.width="".concat(t.bottom.width,"px"),e.background.style.left="0",e.background.style.top="0",e.backgroundVertical.style.left="".concat(t.left.width+t.border.left,"px"),e.backgroundVertical.style.top="0",e.backgroundHorizontal.style.left="0",e.backgroundHorizontal.style.top="".concat(t.top.height,"px"),e.centerContainer.style.left="".concat(t.left.width,"px"),e.centerContainer.style.top="".concat(t.top.height,"px"),e.leftContainer.style.left="0",e.leftContainer.style.top="".concat(t.top.height,"px"),e.rightContainer.style.left="".concat(t.left.width+t.center.width,"px"),e.rightContainer.style.top="".concat(t.top.height,"px"),e.top.style.left="".concat(t.left.width,"px"),e.top.style.top="0",e.bottom.style.left="".concat(t.left.width,"px"),e.bottom.style.top="".concat(t.top.height+t.centerContainer.height,"px"),e.center.style.left="0",e.left.style.left="0",e.right.style.left="0"}},{key:"setCurrentTime",value:function(t){if(!this.currentTime)throw new Error("Option showCurrentTime must be true");this.currentTime.setCurrentTime(t)}},{key:"getCurrentTime",value:function(){if(!this.currentTime)throw new Error("Option showCurrentTime must be true");return this.currentTime.getCurrentTime()}},{key:"_toTime",value:function(t){return sl(this,t,this.props.center.width)}},{key:"_toGlobalTime",value:function(t){return sl(this,t,this.props.root.width)}},{key:"_toScreen",value:function(t){return ol(this,t,this.props.center.width)}},{key:"_toGlobalScreen",value:function(t){return ol(this,t,this.props.root.width)}},{key:"_initAutoResize",value:function(){1==this.options.autoResize?this._startAutoResize():this._stopAutoResize()}},{key:"_startAutoResize",value:function(){var t=this;this._stopAutoResize(),this._onResize=function(){if(1==t.options.autoResize){if(t.dom.root){var e=t.dom.root.offsetHeight,i=t.dom.root.offsetWidth;i==t.props.lastWidth&&e==t.props.lastHeight||(t.props.lastWidth=i,t.props.lastHeight=e,t.props.scrollbarWidth=za.getScrollBarWidth(),t.body.emitter.emit("_change"))}}else t._stopAutoResize()},za.addEventListener(window,"resize",this._onResize),t.dom.root&&(t.props.lastWidth=t.dom.root.offsetWidth,t.props.lastHeight=t.dom.root.offsetHeight),this.watchTimer=gl(this._onResize,1e3)}},{key:"_stopAutoResize",value:function(){this.watchTimer&&(clearInterval(this.watchTimer),this.watchTimer=void 0),this._onResize&&(za.removeEventListener(window,"resize",this._onResize),this._onResize=null)}},{key:"_onTouch",value:function(t){this.touch.allowDragging=!0,this.touch.initialScrollTop=this.props.scrollTop}},{key:"_onPinch",value:function(t){this.touch.allowDragging=!1}},{key:"_onDrag",value:function(t){if(t&&this.touch.allowDragging){var e=t.deltaY,i=this._getScrollTop(),n=this._setScrollTop(this.touch.initialScrollTop+e);this.options.verticalScroll&&(this.dom.left.parentNode.scrollTop=-this.props.scrollTop,this.dom.right.parentNode.scrollTop=-this.props.scrollTop),n!=i&&this.emit("verticalDrag")}}},{key:"_setScrollTop",value:function(t){return this.props.scrollTop=t,this._updateScrollTop(),this.props.scrollTop}},{key:"_updateScrollTop",value:function(){var t=Math.min(this.props.centerContainer.height-this.props.center.height,0);return t!=this.props.scrollTopMin&&("top"!=this.options.orientation.item&&(this.props.scrollTop+=t-this.props.scrollTopMin),this.props.scrollTopMin=t),this.props.scrollTop>0&&(this.props.scrollTop=0),this.props.scrollTop<t&&(this.props.scrollTop=t),this.options.verticalScroll&&(this.dom.left.parentNode.scrollTop=-this.props.scrollTop,this.dom.right.parentNode.scrollTop=-this.props.scrollTop),this._isProgramaticallyScrolled=!0,this.props.scrollTop}},{key:"_getScrollTop",value:function(){return this.props.scrollTop}},{key:"_createConfigurator",value:function(){throw new Error("Cannot invoke abstract method _createConfigurator")}}]),t}();vl(cd.prototype);var pd=function(t){function e(t,i){var n,o;ba(this,e),(o=ar(this,dr(e).call(this))).body=t,o.defaultOptions={rtl:!1,showCurrentTime:!0,alignCurrentTime:void 0,moment:Ta,locales:dd,locale:"en"},o.options=za.extend({},o.defaultOptions),o.setOptions(i),o.options.locales=za.extend({},dd,o.options.locales);var s=o.defaultOptions.locales[o.defaultOptions.locale];return mn(n=Yi(o.options.locales)).call(n,(function(t){o.options.locales[t]=za.extend({},s,o.options.locales[t])})),o.offset=0,o._create(),o}return pr(e,t),xa(e,[{key:"_create",value:function(){var t=document.createElement("div");t.className="vis-current-time",t.style.position="absolute",t.style.top="0px",t.style.height="100%",this.bar=t}},{key:"destroy",value:function(){this.options.showCurrentTime=!1,this.redraw(),this.body=null}},{key:"setOptions",value:function(t){t&&za.selectiveExtend(["rtl","showCurrentTime","alignCurrentTime","moment","locale","locales"],this.options,t)}},{key:"redraw",value:function(){if(this.options.showCurrentTime){var t,e,i=this.body.dom.backgroundVertical;this.bar.parentNode!=i&&(this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar),i.appendChild(this.bar),this.start());var n=this.options.moment(Jo()+this.offset);this.options.alignCurrentTime&&(n=n.startOf(this.options.alignCurrentTime));var o=this.body.util.toScreen(n),s=this.options.locales[this.options.locale];s||(this.warned||(console.warn("WARNING: options.locales['".concat(this.options.locale,"'] not found. See https://visjs.github.io/vis-timeline/docs/timeline/#Localization")),this.warned=!0),s=this.options.locales.en);var r=Pn(t=Pn(e="".concat(s.current," ")).call(e,s.time,": ")).call(t,n.format("dddd, MMMM Do YYYY, H:mm:ss"));r=r.charAt(0).toUpperCase()+r.substring(1),this.options.rtl?this.bar.style.transform="translateX(".concat(-1*o,"px)"):this.bar.style.transform="translateX(".concat(o,"px)"),this.bar.title=r}else this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar),this.stop();return!1}},{key:"start",value:function(){var t=this;!function e(){t.stop();var i=1/t.body.range.conversion(t.body.domProps.center.width).scale/10;i<30&&(i=30),i>1e3&&(i=1e3),t.redraw(),t.body.emitter.emit("currentTimeTick"),t.currentTimeTimer=qr(e,i)}()}},{key:"stop",value:function(){void 0!==this.currentTimeTimer&&(clearTimeout(this.currentTimeTimer),delete this.currentTimeTimer)}},{key:"setCurrentTime",value:function(t){var e=za.convert(t,"Date").valueOf(),i=Jo();this.offset=e-i,this.redraw()}},{key:"getCurrentTime",value:function(){return new Date(Jo()+this.offset)}}]),e}(Ja),md=Ce.find,fd=!0,gd=on("find");"find"in[]&&Array(1).find((function(){fd=!1})),V({target:"Array",proto:!0,forced:fd||!gd},{find:function(t){return md(this,t,arguments.length>1?arguments[1]:void 0)}});var vd=Pi("Array").find,bd=Array.prototype,yd=function(t){var e=t.find;return t===bd||t instanceof Array&&e===bd.find?vd:e},xd="undefined"!=typeof crypto&&crypto.getRandomValues&&crypto.getRandomValues.bind(crypto)||"undefined"!=typeof msCrypto&&"function"==typeof msCrypto.getRandomValues&&msCrypto.getRandomValues.bind(msCrypto),wd=new Uint8Array(16);function _d(){if(!xd)throw new Error("crypto.getRandomValues() not supported. See https://github.com/uuidjs/uuid#getrandomvalues-not-supported");return xd(wd)}for(var kd=[],Dd=0;Dd<256;++Dd)kd[Dd]=(Dd+256).toString(16).substr(1);function Id(t,e,i){var n=e&&i||0;"string"==typeof t&&(e="binary"===t?new Array(16):null,t=null);var o=(t=t||{}).random||(t.rng||_d)();if(o[6]=15&o[6]|64,o[8]=63&o[8]|128,e)for(var s=0;s<16;++s)e[n+s]=o[s];return e||function(t,e){var i=e||0,n=kd;return[n[t[i++]],n[t[i++]],n[t[i++]],n[t[i++]],"-",n[t[i++]],n[t[i++]],"-",n[t[i++]],n[t[i++]],"-",n[t[i++]],n[t[i++]],"-",n[t[i++]],n[t[i++]],n[t[i++]],n[t[i++]],n[t[i++]],n[t[i++]]].join("")}(o)}var Cd=ot.includes,Ld=on("indexOf",{ACCESSORS:!0,1:0});V({target:"Array",proto:!0,forced:!Ld},{includes:function(t){return Cd(this,t,arguments.length>1?arguments[1]:void 0)}});var Gd=Pi("Array").includes,Md=Pt("match"),Sd=function(t){if(function(t){var e;return x(t)&&(void 0!==(e=t[Md])?!!e:"RegExp"==f(t))}(t))throw TypeError("The method doesn't accept regular expressions");return t},Qd=Pt("match");V({target:"String",proto:!0,forced:!function(t){var e=/./;try{"/./"[t](e)}catch(i){try{return e[Qd]=!1,"/./"[t](e)}catch(t){}}return!1}("includes")},{includes:function(t){return!!~String(b(this)).indexOf(Sd(t),arguments.length>1?arguments[1]:void 0)}});var Nd=Pi("String").includes,Fd=Array.prototype,Zd=String.prototype,Bd=function(t){var e=t.includes;return t===Fd||t instanceof Array&&e===Fd.includes?Gd:"string"==typeof t||t===Zd||t instanceof String&&e===Zd.includes?Nd:e};function Od(t){wr(t).call(t,(function(t,e){return t.data.start-e.data.start}))}function Wd(t){wr(t).call(t,(function(t,e){return("end"in t.data?t.data.end:t.data.start)-("end"in e.data?e.data.end:e.data.start)}))}function Ad(t,e,i,n){if(i)for(var o=0;o<t.length;o++)t[o].top=null;for(o=0;o<t.length;o++){var s=t[o];if(s.stack&&null===s.top){s.top=e.axis;var r=!1;do{for(var a=null,l=0,h=t.length;l<h;l++){var d=t[l];if(r=n()||!1)return!0;if(null!==d.top&&d!==s&&d.stack&&Rd(s,d,e.item,d.options.rtl)){a=d;break}}null!=a&&(s.top=a.top+a.height+e.item.vertical)}while(a)}}return r}function Xd(t,e,i){for(var n=0;n<t.length;n++)t[n].top=null;var o=i.height;for(n=0;n<t.length;n++){var s=t[n];if(s.stack&&null===s.top){s.top=s.baseTop;do{for(var r=null,a=0,l=t.length;a<l;a++){var h=t[a];if(null!==h.top&&h!==s&&Rd(s,h,e.item,h.options.rtl)){r=h;break}}null!=r&&(s.top=r.top+r.height+e.item.vertical),s.top+s.height>o&&(o=s.top+s.height)}while(r)}}i.height=o-i.top+.5*e.item.vertical}function Td(t,e,i,n){for(var o=0;o<t.length;o++)if(null==t[o].data.subgroup)t[o].top=e.item.vertical;else if(void 0!==t[o].data.subgroup&&n){var s=0;for(var r in i)i.hasOwnProperty(r)&&1==i[r].visible&&i[r].index<i[t[o].data.subgroup].index&&(s+=i[r].height,i[t[o].data.subgroup].top=s);t[o].top=s+.5*e.item.vertical}n||Yd(t,e,i)}function Yd(t,e,i){for(var n in i)if(i.hasOwnProperty(n)){i[n].top=0;do{var o=null;for(var s in i)if(null!==i[s].top&&s!==n&&i[n].index>i[s].index&&Ud(i[n],i[s])){o=i[s];break}null!=o&&(i[n].top=o.top+o.height)}while(o)}for(var r=0;r<t.length;r++)void 0!==t[r].data.subgroup&&(t[r].top=i[t[r].data.subgroup].top+.5*e.item.vertical)}function Ed(t,e,i){var n=!1,o=[];for(var s in i)i[s].hasOwnProperty("index")?o[i[s].index]=s:o.push(s);for(var r=0;r<o.length;r++)if(s=o[r],i.hasOwnProperty(s)){for(var a in n=n||i[s].stack,i[s].top=0,i)i[a].visible&&i[s].index>i[a].index&&(i[s].top+=i[a].height);for(var l=t[s],h=0;h<l.length;h++)void 0!==l[h].data.subgroup&&(l[h].top=i[l[h].data.subgroup].top+.5*e.item.vertical,i[s].stack&&(l[h].baseTop=l[h].top));n&&i[s].stack&&Xd(t[s],e,i[s])}}function Rd(t,e,i,n){return n?t.right-i.horizontal+.001<e.right+e.width&&t.right+t.width+i.horizontal-.001>e.right&&t.top-i.vertical+.001<e.top+e.height&&t.top+t.height+i.vertical-.001>e.top:t.left-i.horizontal+.001<e.left+e.width&&t.left+t.width+i.horizontal-.001>e.left&&t.top-i.vertical+.001<e.top+e.height&&t.top+t.height+i.vertical-.001>e.top}function Ud(t,e){var i=t.start<e.end&&t.end>e.start,n=t.top<e.top+e.height&&t.top+t.height>e.top;return i&&n}var zd=Object.freeze({__proto__:null,orderByStart:Od,orderByEnd:Wd,stack:Ad,substack:Xd,nostack:Td,stackSubgroups:Yd,stackSubgroupsWithInnerStack:Ed,collision:Rd,collisionByTimes:Ud}),Vd="__background__",jd=function(){function t(e,i,n){var o=this;if(ba(this,t),this.groupId=e,this.subgroups={},this.subgroupStack={},this.subgroupStackAll=!1,this.subgroupVisibility={},this.doInnerStack=!1,this.shouldBailStackItems=!1,this.subgroupIndex=0,this.subgroupOrderer=i&&i.subgroupOrder,this.itemSet=n,this.isVisible=null,this.stackDirty=!0,this._disposeCallbacks=[],i&&i.nestedGroups&&(this.nestedGroups=i.nestedGroups,0==i.showNested?this.showNested=!1:this.showNested=!0),i&&i.subgroupStack)if("boolean"==typeof i.subgroupStack)this.doInnerStack=i.subgroupStack,this.subgroupStackAll=i.subgroupStack;else for(var s in i.subgroupStack)this.subgroupStack[s]=i.subgroupStack[s],this.doInnerStack=this.doInnerStack||i.subgroupStack[s];i&&i.heightMode?this.heightMode=i.heightMode:this.heightMode=n.options.groupHeightMode,this.nestedInGroup=null,this.dom={},this.props={label:{width:0,height:0}},this.className=null,this.items={},this.visibleItems=[],this.itemsInRange=[],this.orderedItems={byStart:[],byEnd:[]},this.checkRangedItems=!1;var r=function(){o.checkRangedItems=!0};this.itemSet.body.emitter.on("checkRangedItems",r),this._disposeCallbacks.push((function(){o.itemSet.body.emitter.off("checkRangedItems",r)})),this._create(),this.setData(i)}return xa(t,[{key:"_create",value:function(){var t=document.createElement("div");this.itemSet.options.groupEditable.order?t.className="vis-label draggable":t.className="vis-label",this.dom.label=t;var e=document.createElement("div");e.className="vis-inner",t.appendChild(e),this.dom.inner=e;var i=document.createElement("div");i.className="vis-group",i["vis-group"]=this,this.dom.foreground=i,this.dom.background=document.createElement("div"),this.dom.background.className="vis-group",this.dom.axis=document.createElement("div"),this.dom.axis.className="vis-group",this.dom.marker=document.createElement("div"),this.dom.marker.style.visibility="hidden",this.dom.marker.style.position="absolute",this.dom.marker.innerHTML="",this.dom.background.appendChild(this.dom.marker)}},{key:"setData",value:function(t){if(!this.itemSet.groupTouchParams.isDragging){var e,i,n;if(t&&t.subgroupVisibility)for(var o in t.subgroupVisibility)this.subgroupVisibility[o]=t.subgroupVisibility[o];if(this.itemSet.options&&this.itemSet.options.groupTemplate)e=(i=va(n=this.itemSet.options.groupTemplate).call(n,this))(t,this.dom.inner);else e=t&&t.content;if(e instanceof Element){for(;this.dom.inner.firstChild;)this.dom.inner.removeChild(this.dom.inner.firstChild);this.dom.inner.appendChild(e)}else e instanceof Object&&e.isReactComponent||(e instanceof Object?i(t,this.dom.inner):this.dom.inner.innerHTML=null!=e?e:this.groupId||"");this.dom.label.title=t&&t.title||"",this.dom.inner.firstChild?za.removeClassName(this.dom.inner,"vis-hidden"):za.addClassName(this.dom.inner,"vis-hidden"),t&&t.nestedGroups?(this.nestedGroups&&this.nestedGroups==t.nestedGroups||(this.nestedGroups=t.nestedGroups),void 0===t.showNested&&void 0!==this.showNested||(0==t.showNested?this.showNested=!1:this.showNested=!0),za.addClassName(this.dom.label,"vis-nesting-group"),this.showNested?(za.removeClassName(this.dom.label,"collapsed"),za.addClassName(this.dom.label,"expanded")):(za.removeClassName(this.dom.label,"expanded"),za.addClassName(this.dom.label,"collapsed"))):this.nestedGroups&&(this.nestedGroups=null,za.removeClassName(this.dom.label,"collapsed"),za.removeClassName(this.dom.label,"expanded"),za.removeClassName(this.dom.label,"vis-nesting-group")),t&&(t.treeLevel||t.nestedInGroup)?(za.addClassName(this.dom.label,"vis-nested-group"),t.treeLevel?za.addClassName(this.dom.label,"vis-group-level-"+t.treeLevel):za.addClassName(this.dom.label,"vis-group-level-unknown-but-gte1")):za.addClassName(this.dom.label,"vis-group-level-0");var s=t&&t.className||null;s!=this.className&&(this.className&&(za.removeClassName(this.dom.label,this.className),za.removeClassName(this.dom.foreground,this.className),za.removeClassName(this.dom.background,this.className),za.removeClassName(this.dom.axis,this.className)),za.addClassName(this.dom.label,s),za.addClassName(this.dom.foreground,s),za.addClassName(this.dom.background,s),za.addClassName(this.dom.axis,s),this.className=s),this.style&&(za.removeCssText(this.dom.label,this.style),this.style=null),t&&t.style&&(za.addCssText(this.dom.label,t.style),this.style=t.style)}}},{key:"getLabelWidth",value:function(){return this.props.label.width}},{key:"_didMarkerHeightChange",value:function(){var t=this.dom.marker.clientHeight;if(t!=this.lastMarkerHeight){this.lastMarkerHeight=t;var e={},i=0;if(mn(za).call(za,this.items,(function(t,n){if(t.dirty=!0,t.displayed){e[n]=t.redraw(!0),i=e[n].length}})),i>0)for(var n=function(t){mn(za).call(za,e,(function(e){e[t]()}))},o=0;o<i;o++)n(o);return!0}return!1}},{key:"_calculateGroupSizeAndPosition",value:function(){var t=this.dom.foreground,e=t.offsetTop,i=t.offsetLeft,n=t.offsetWidth;this.top=e,this.right=i,this.width=n}},{key:"_shouldBailItemsRedraw",value:function(){var t=this,e=this.itemSet.options.onTimeout,i={relativeBailingTime:this.itemSet.itemsSettingTime,bailTimeMs:e&&e.timeoutMs,userBailFunction:e&&e.callback,shouldBailStackItems:this.shouldBailStackItems},n=null;if(!this.itemSet.initialDrawDone){if(i.shouldBailStackItems)return!0;Math.abs(Jo()-new Date(i.relativeBailingTime))>i.bailTimeMs&&(i.userBailFunction&&null==this.itemSet.userContinueNotBail?i.userBailFunction((function(e){t.itemSet.userContinueNotBail=e,n=!e})):n=0==t.itemSet.userContinueNotBail)}return n}},{key:"_redrawItems",value:function(t,e,i,n){var o=this;if(t||this.stackDirty||this.isVisible&&!e){var s,r,a,l,h,d,u={byEnd:En(s=this.orderedItems.byEnd).call(s,(function(t){return!t.isCluster})),byStart:En(r=this.orderedItems.byStart).call(r,(function(t){return!t.isCluster}))},c={byEnd:mo(new Ar(En(a=Ln(l=this.orderedItems.byEnd).call(l,(function(t){return t.cluster}))).call(a,(function(t){return!!t})))),byStart:mo(new Ar(En(h=Ln(d=this.orderedItems.byStart).call(d,(function(t){return t.cluster}))).call(h,(function(t){return!!t}))))},p=function(){var t,e,i,s=o._updateItemsInRange(u,En(t=o.visibleItems).call(t,(function(t){return!t.isCluster})),n),r=o._updateClustersInRange(c,En(e=o.visibleItems).call(e,(function(t){return t.isCluster})),n);return Pn(i=[]).call(i,mo(s),mo(r))},m=function(t){var e={},i=function(i){var n,s=En(n=o.visibleItems).call(n,(function(t){return t.data.subgroup===i}));e[i]=t?wr(s).call(s,(function(e,i){return t(e.data,i.data)})):s};for(var n in o.subgroups)i(n);return e};if("function"==typeof this.itemSet.options.order){var f=this;if(this.doInnerStack&&this.itemSet.options.stackSubgroups){Ed(m(this.itemSet.options.order),i,this.subgroups),this.visibleItems=p(),this._updateSubGroupHeights(i)}else{var g,v,b,y;this.visibleItems=p(),this._updateSubGroupHeights(i);var x=wr(g=En(v=_o(b=this.visibleItems).call(b)).call(v,(function(t){return t.isCluster||!t.isCluster&&!t.cluster}))).call(g,(function(t,e){return f.itemSet.options.order(t.data,e.data)}));this.shouldBailStackItems=Ad(x,i,!0,va(y=this._shouldBailItemsRedraw).call(y,this))}}else{var w;if(this.visibleItems=p(),this._updateSubGroupHeights(i),this.itemSet.options.stack)if(this.doInnerStack&&this.itemSet.options.stackSubgroups)Ed(m(),i,this.subgroups);else this.shouldBailStackItems=Ad(this.visibleItems,i,!0,va(w=this._shouldBailItemsRedraw).call(w,this));else Td(this.visibleItems,i,this.subgroups,this.itemSet.options.stackSubgroups)}for(var _=0;_<this.visibleItems.length;_++)this.visibleItems[_].repositionX(),void 0!==this.subgroupVisibility[this.visibleItems[_].data.subgroup]&&(this.subgroupVisibility[this.visibleItems[_].data.subgroup]||this.visibleItems[_].hide());this.itemSet.options.cluster&&mn(za).call(za,this.items,(function(t){t.cluster&&t.displayed&&t.hide()})),this.shouldBailStackItems&&this.itemSet.body.emitter.emit("destroyTimeline"),this.stackDirty=!1}}},{key:"_didResize",value:function(t,e){t=za.updateProperty(this,"height",e)||t;var i=this.dom.inner.clientWidth,n=this.dom.inner.clientHeight;return t=za.updateProperty(this.props.label,"width",i)||t,t=za.updateProperty(this.props.label,"height",n)||t}},{key:"_applyGroupHeight",value:function(t){this.dom.background.style.height="".concat(t,"px"),this.dom.foreground.style.height="".concat(t,"px"),this.dom.label.style.height="".concat(t,"px")}},{key:"_updateItemsVerticalPosition",value:function(t){for(var e=0,i=this.visibleItems.length;e<i;e++){var n=this.visibleItems[e];n.repositionY(t),this.isVisible||this.groupId==Vd||n.displayed&&n.hide()}}},{key:"redraw",value:function(t,e,i,n){var o,s,r,a,l,h,d,u=this,c=!1,p=this.isVisible,m=[function(){i=u._didMarkerHeightChange.call(u)||i},va(o=this._updateSubGroupHeights).call(o,this,e),va(s=this._calculateGroupSizeAndPosition).call(s,this),function(){var i;u.isVisible=va(i=u._isGroupVisible).call(i,u)(t,e)},function(){var n;va(n=u._redrawItems).call(n,u)(i,p,e,t)},va(r=this._updateSubgroupsSizes).call(r,this),function(){var t;h=va(t=u._calculateHeight).call(t,u)(e)},va(a=this._calculateGroupSizeAndPosition).call(a,this),function(){var t;c=va(t=u._didResize).call(t,u)(c,h)},function(){var t;va(t=u._applyGroupHeight).call(t,u)(h)},function(){var t;va(t=u._updateItemsVerticalPosition).call(t,u)(e)},va(l=function(){return!u.isVisible&&u.height&&(c=!1),c}).call(l,this)];return n?m:(mn(m).call(m,(function(t){d=t()})),d)}},{key:"_updateSubGroupHeights",value:function(t){var e=this;if(Yi(this.subgroups).length>0){var i=this;this._resetSubgroups(),mn(za).call(za,this.visibleItems,(function(n){void 0!==n.data.subgroup&&(i.subgroups[n.data.subgroup].height=Math.max(i.subgroups[n.data.subgroup].height,n.height+t.item.vertical),i.subgroups[n.data.subgroup].visible=void 0===e.subgroupVisibility[n.data.subgroup]||Boolean(e.subgroupVisibility[n.data.subgroup]))}))}}},{key:"_isGroupVisible",value:function(t,e){return this.top<=t.body.domProps.centerContainer.height-t.body.domProps.scrollTop+e.axis&&this.top+this.height+e.axis>=-t.body.domProps.scrollTop}},{key:"_calculateHeight",value:function(t){var e,i;if((i="fixed"===this.heightMode?za.toArray(this.items):this.visibleItems).length>0){var n=i[0].top,o=i[0].top+i[0].height;if(mn(za).call(za,i,(function(t){n=Math.min(n,t.top),o=Math.max(o,t.top+t.height)})),n>t.axis){var s=n-t.axis;o-=s,mn(za).call(za,i,(function(t){t.top-=s}))}e=Math.ceil(o+t.item.vertical/2),"fitItems"!==this.heightMode&&(e=Math.max(e,this.props.label.height))}else e=this.props.label.height;return e}},{key:"show",value:function(){this.dom.label.parentNode||this.itemSet.dom.labelSet.appendChild(this.dom.label),this.dom.foreground.parentNode||this.itemSet.dom.foreground.appendChild(this.dom.foreground),this.dom.background.parentNode||this.itemSet.dom.background.appendChild(this.dom.background),this.dom.axis.parentNode||this.itemSet.dom.axis.appendChild(this.dom.axis)}},{key:"hide",value:function(){var t=this.dom.label;t.parentNode&&t.parentNode.removeChild(t);var e=this.dom.foreground;e.parentNode&&e.parentNode.removeChild(e);var i=this.dom.background;i.parentNode&&i.parentNode.removeChild(i);var n=this.dom.axis;n.parentNode&&n.parentNode.removeChild(n)}},{key:"add",value:function(t){var e;if(this.items[t.id]=t,t.setParent(this),this.stackDirty=!0,void 0!==t.data.subgroup&&(this._addToSubgroup(t),this.orderSubgroups()),!Bd(e=this.visibleItems).call(e,t)){var i=this.itemSet.body.range;this._checkIfVisible(t,this.visibleItems,i)}}},{key:"_addToSubgroup",value:function(t){var e=arguments.length>1&&void 0!==arguments[1]?arguments[1]:t.data.subgroup;null!=e&&void 0===this.subgroups[e]&&(this.subgroups[e]={height:0,top:0,start:t.data.start,end:t.data.end||t.data.start,visible:!1,index:this.subgroupIndex,items:[],stack:this.subgroupStackAll||this.subgroupStack[e]||!1},this.subgroupIndex++),new Date(t.data.start)<new Date(this.subgroups[e].start)&&(this.subgroups[e].start=t.data.start);var i=t.data.end||t.data.start;new Date(i)>new Date(this.subgroups[e].end)&&(this.subgroups[e].end=i),this.subgroups[e].items.push(t)}},{key:"_updateSubgroupsSizes",value:function(){var t=this;if(t.subgroups){var e=function(e){var i,n=t.subgroups[e].items[0].data.end||t.subgroups[e].items[0].data.start,o=t.subgroups[e].items[0].data.start,s=n-1;mn(i=t.subgroups[e].items).call(i,(function(t){new Date(t.data.start)<new Date(o)&&(o=t.data.start);var e=t.data.end||t.data.start;new Date(e)>new Date(s)&&(s=e)})),t.subgroups[e].start=o,t.subgroups[e].end=new Date(s-1)};for(var i in t.subgroups)e(i)}}},{key:"orderSubgroups",value:function(){if(void 0!==this.subgroupOrderer){var t=[];if("string"==typeof this.subgroupOrderer){for(var e in this.subgroups)t.push({subgroup:e,sortField:this.subgroups[e].items[0].data[this.subgroupOrderer]});wr(t).call(t,(function(t,e){return t.sortField-e.sortField}))}else if("function"==typeof this.subgroupOrderer){for(var i in this.subgroups)t.push(this.subgroups[i].items[0].data);wr(t).call(t,this.subgroupOrderer)}if(t.length>0)for(var n=0;n<t.length;n++)this.subgroups[t[n].subgroup].index=n}}},{key:"_resetSubgroups",value:function(){for(var t in this.subgroups)this.subgroups.hasOwnProperty(t)&&(this.subgroups[t].visible=!1,this.subgroups[t].height=0)}},{key:"remove",value:function(t){var e,i;delete this.items[t.id],t.setParent(null),this.stackDirty=!0;var n=Fo(e=this.visibleItems).call(e,t);-1!=n&&Hr(i=this.visibleItems).call(i,n,1),void 0!==t.data.subgroup&&(this._removeFromSubgroup(t),this.orderSubgroups())}},{key:"_removeFromSubgroup",value:function(t){var e=arguments.length>1&&void 0!==arguments[1]?arguments[1]:t.data.subgroup;if(null!=e){var i=this.subgroups[e];if(i){var n,o,s=Fo(n=i.items).call(n,t);if(s>=0)Hr(o=i.items).call(o,s,1),i.items.length?this._updateSubgroupsSizes():delete this.subgroups[e]}}}},{key:"removeFromDataSet",value:function(t){this.itemSet.removeItem(t.id)}},{key:"order",value:function(){for(var t=za.toArray(this.items),e=[],i=[],n=0;n<t.length;n++)void 0!==t[n].data.end&&i.push(t[n]),e.push(t[n]);this.orderedItems={byStart:e,byEnd:i},Od(this.orderedItems.byStart),Wd(this.orderedItems.byEnd)}},{key:"_updateItemsInRange",value:function(t,e,i){var n=[],o={};if(!this.isVisible&&this.groupId!=Vd){for(var s=0;s<e.length;s++){var r=e[s];r.displayed&&r.hide()}return n}var a=(i.end-i.start)/4,l=i.start-a,h=i.end+a;if(e.length>0)for(var d=0;d<e.length;d++)this._checkIfVisibleWithReference(e[d],n,o,i);var u=za.binarySearchCustom(t.byStart,(function(t){return t<l?-1:t<=h?0:1}),"data","start");if(this._traceVisible(u,t.byStart,n,o,(function(t){return t.data.start<l||t.data.start>h})),1==this.checkRangedItems){this.checkRangedItems=!1;for(var c=0;c<t.byEnd.length;c++)this._checkIfVisibleWithReference(t.byEnd[c],n,o,i)}else{var p=za.binarySearchCustom(t.byEnd,(function(t){return t<l?-1:0}),"data","end");this._traceVisible(p,t.byEnd,n,o,(function(t){return t.data.end<l||t.data.start>h}))}for(var m={},f=0,g=0;g<n.length;g++){var v=n[g];if(!v.displayed){m[g]=v.redraw(!0),f=m[g].length}}if(f>0)for(var b=function(t){mn(za).call(za,m,(function(e){e[t]()}))},y=0;y<f;y++)b(y);for(var x=0;x<n.length;x++)n[x].repositionX();return n}},{key:"_traceVisible",value:function(t,e,i,n,o){if(-1!=t){for(var s=t;s>=0;s--){var r=e[s];if(o(r))break;r.isCluster&&!r.hasItems()||r.cluster||void 0===n[r.id]&&(n[r.id]=!0,i.push(r))}for(var a=t+1;a<e.length;a++){var l=e[a];if(o(l))break;l.isCluster&&!l.hasItems()||l.cluster||void 0===n[l.id]&&(n[l.id]=!0,i.push(l))}}}},{key:"_checkIfVisible",value:function(t,e,i){t.isVisible(i)?(t.displayed||t.show(),t.repositionX(),e.push(t)):t.displayed&&t.hide()}},{key:"_checkIfVisibleWithReference",value:function(t,e,i,n){t.isVisible(n)?void 0===i[t.id]&&(i[t.id]=!0,e.push(t)):t.displayed&&t.hide()}},{key:"_updateClustersInRange",value:function(t,e,i){var n=[],o={};if(e.length>0)for(var s=0;s<e.length;s++)this._checkIfVisibleWithReference(e[s],n,o,i);for(var r=0;r<t.byStart.length;r++)this._checkIfVisibleWithReference(t.byStart[r],n,o,i);for(var a=0;a<t.byEnd.length;a++)this._checkIfVisibleWithReference(t.byEnd[a],n,o,i);for(var l={},h=0,d=0;d<n.length;d++){var u=n[d];if(!u.displayed){l[d]=u.redraw(!0),h=l[d].length}}if(h>0)for(var c=0;c<h;c++)mn(za).call(za,l,(function(t){t[c]()}));for(var p=0;p<n.length;p++)n[p].repositionX();return n}},{key:"changeSubgroup",value:function(t,e,i){this._removeFromSubgroup(t,e),this._addToSubgroup(t,i),this.orderSubgroups()}},{key:"dispose",value:function(){var t;for(this.hide();t=this._disposeCallbacks.pop();)t()}}]),t}(),Hd=function(t){function e(t,i,n){var o;return ba(this,e),(o=ar(this,dr(e).call(this,t,i,n))).width=0,o.height=0,o.top=0,o.left=0,o}return pr(e,t),xa(e,[{key:"redraw",value:function(t,e,i){this.visibleItems=this._updateItemsInRange(this.orderedItems,this.visibleItems,t),this.width=this.dom.background.offsetWidth,this.dom.background.style.height="0";for(var n=0,o=this.visibleItems.length;n<o;n++){this.visibleItems[n].repositionY(e)}return!1}},{key:"show",value:function(){this.dom.background.parentNode||this.itemSet.dom.background.appendChild(this.dom.background)}}]),e}(jd);Ph('.vis-item{position:absolute;color:#1a1a1a;border-color:#97b0f8;border-width:1px;background-color:#d5ddf6;display:inline-block;z-index:1}.vis-item.vis-selected{border-color:#ffc200;background-color:#fff785;z-index:2}.vis-editable.vis-selected{cursor:move}.vis-item.vis-point.vis-selected{background-color:#fff785}.vis-item.vis-box{text-align:center;border-style:solid;border-radius:2px}.vis-item.vis-point{background:none}.vis-item.vis-dot{position:absolute;padding:0;border-width:4px;border-style:solid;border-radius:4px}.vis-item.vis-range{border-style:solid;border-radius:2px;box-sizing:border-box}.vis-item.vis-background{border:none;background-color:rgba(213,221,246,.4);box-sizing:border-box;padding:0;margin:0}.vis-item .vis-item-overflow{position:relative;width:100%;height:100%;padding:0;margin:0;overflow:hidden}.vis-item-visible-frame{white-space:nowrap}.vis-item.vis-range .vis-item-content{position:relative;display:inline-block}.vis-item.vis-background .vis-item-content{position:absolute;display:inline-block}.vis-item.vis-line{padding:0;position:absolute;width:0;border-left-width:1px;border-left-style:solid}.vis-item .vis-item-content{white-space:nowrap;box-sizing:border-box;padding:5px}.vis-item .vis-onUpdateTime-tooltip{position:absolute;background:#4f81bd;color:#fff;width:200px;text-align:center;white-space:nowrap;padding:5px;border-radius:1px;transition:.4s;-o-transition:.4s;-moz-transition:.4s;-webkit-transition:.4s}.vis-item .vis-delete,.vis-item .vis-delete-rtl{position:absolute;top:0;width:24px;height:24px;box-sizing:border-box;padding:0 5px;cursor:pointer;-webkit-transition:background .2s linear;-moz-transition:background .2s linear;-ms-transition:background .2s linear;-o-transition:background .2s linear;transition:background .2s linear}.vis-item .vis-delete{right:-24px}.vis-item .vis-delete-rtl{left:-24px}.vis-item .vis-delete-rtl:after,.vis-item .vis-delete:after{content:"\\00D7";color:red;font-family:arial,sans-serif;font-size:22px;font-weight:700;-webkit-transition:color .2s linear;-moz-transition:color .2s linear;-ms-transition:color .2s linear;-o-transition:color .2s linear;transition:color .2s linear}.vis-item .vis-delete-rtl:hover,.vis-item .vis-delete:hover{background:red}.vis-item .vis-delete-rtl:hover:after,.vis-item .vis-delete:hover:after{color:#fff}.vis-item .vis-drag-center{position:absolute;width:100%;height:100%;top:0;left:0;cursor:move}.vis-item.vis-range .vis-drag-left{left:-4px;cursor:w-resize}.vis-item.vis-range .vis-drag-left,.vis-item.vis-range .vis-drag-right{position:absolute;width:24px;max-width:20%;min-width:2px;height:100%;top:0}.vis-item.vis-range .vis-drag-right{right:-4px;cursor:e-resize}.vis-range.vis-item.vis-readonly .vis-drag-left,.vis-range.vis-item.vis-readonly .vis-drag-right{cursor:auto}.vis-item.vis-cluster{vertical-align:center;text-align:center;border-style:solid;border-radius:2px}.vis-item.vis-cluster-line{padding:0;position:absolute;width:0;border-left-width:1px;border-left-style:solid}.vis-item.vis-cluster-dot{position:absolute;padding:0;border-width:4px;border-style:solid;border-radius:4px}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbIml0ZW0uY3NzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUNBLFVBQ0UsaUJBQWtCLENBQ2xCLGFBQWMsQ0FDZCxvQkFBcUIsQ0FDckIsZ0JBQWlCLENBQ2pCLHdCQUF5QixDQUN6QixvQkFBcUIsQ0FDckIsU0FFRixDQUVBLHVCQUNFLG9CQUFxQixDQUNyQix3QkFBeUIsQ0FHekIsU0FDRixDQUVBLDJCQUNFLFdBQ0YsQ0FFQSxpQ0FDRSx3QkFDRixDQUVBLGtCQUNFLGlCQUFrQixDQUNsQixrQkFBbUIsQ0FDbkIsaUJBQ0YsQ0FFQSxvQkFDRSxlQUNGLENBRUEsa0JBQ0UsaUJBQWtCLENBQ2xCLFNBQVUsQ0FDVixnQkFBaUIsQ0FDakIsa0JBQW1CLENBQ25CLGlCQUNGLENBRUEsb0JBQ0Usa0JBQW1CLENBQ25CLGlCQUFrQixDQUNsQixxQkFDRixDQUVBLHlCQUNFLFdBQVksQ0FDWixxQ0FBMEMsQ0FDMUMscUJBQXNCLENBQ3RCLFNBQVUsQ0FDVixRQUNGLENBRUEsNkJBQ0UsaUJBQWtCLENBQ2xCLFVBQVcsQ0FDWCxXQUFZLENBQ1osU0FBVSxDQUNWLFFBQVMsQ0FDVCxlQUNGLENBRUEsd0JBQ0Usa0JBQ0YsQ0FFQSxzQ0FDRSxpQkFBa0IsQ0FDbEIsb0JBQ0YsQ0FFQSwyQ0FDRSxpQkFBa0IsQ0FDbEIsb0JBQ0YsQ0FFQSxtQkFDRSxTQUFVLENBQ1YsaUJBQWtCLENBQ2xCLE9BQVEsQ0FDUixxQkFBc0IsQ0FDdEIsdUJBQ0YsQ0FFQSw0QkFDRSxrQkFBbUIsQ0FDbkIscUJBQXNCLENBQ3RCLFdBQ0YsQ0FFQSxvQ0FDRSxpQkFBa0IsQ0FDbEIsa0JBQW1CLENBQ25CLFVBQVksQ0FDWixXQUFZLENBQ1osaUJBQWtCLENBQ2xCLGtCQUFtQixDQUNuQixXQUFZLENBQ1osaUJBQWtCLENBQ2xCLGNBQWdCLENBQ2hCLGlCQUFtQixDQUNuQixtQkFBcUIsQ0FDckIsc0JBQ0YsQ0FFQSxnREFDRSxpQkFBa0IsQ0FDbEIsS0FBUSxDQUNSLFVBQVcsQ0FDWCxXQUFZLENBQ1oscUJBQXNCLENBQ3RCLGFBQWdCLENBQ2hCLGNBQWUsQ0FFZix3Q0FBMEMsQ0FDMUMscUNBQXVDLENBQ3ZDLG9DQUFzQyxDQUN0QyxtQ0FBcUMsQ0FDckMsZ0NBQ0YsQ0FFQSxzQkFDRSxXQUNGLENBRUEsMEJBQ0UsVUFDRixDQUVBLDREQUNFLGVBQWdCLENBQ2hCLFNBQVUsQ0FDViw0QkFBOEIsQ0FDOUIsY0FBZSxDQUNmLGVBQWlCLENBRWpCLG1DQUFxQyxDQUNyQyxnQ0FBa0MsQ0FDbEMsK0JBQWlDLENBQ2pDLDhCQUFnQyxDQUNoQywyQkFDRixDQUVBLDREQUNFLGNBQ0YsQ0FFQSx3RUFDRSxVQUNGLENBRUEsMkJBQ0UsaUJBQWtCLENBQ2xCLFVBQVcsQ0FDWCxXQUFZLENBQ1osS0FBTSxDQUNOLE1BQVMsQ0FDVCxXQUNGLENBRUEsbUNBT0UsU0FBVSxDQUVWLGVBQ0YsQ0FFQSx1RUFYRSxpQkFBa0IsQ0FDbEIsVUFBVyxDQUNYLGFBQWMsQ0FDZCxhQUFjLENBQ2QsV0FBWSxDQUNaLEtBZ0JGLENBVkEsb0NBT0UsVUFBVyxDQUVYLGVBQ0YsQ0FFQSxpR0FFRSxXQUNGLENBRUEsc0JBQ0UscUJBQXNCLENBQ3RCLGlCQUFrQixDQUNsQixrQkFBbUIsQ0FDbkIsaUJBQ0YsQ0FFQSwyQkFDRSxTQUFVLENBQ1YsaUJBQWtCLENBQ2xCLE9BQVEsQ0FDUixxQkFBc0IsQ0FDdEIsdUJBQ0YsQ0FFQSwwQkFDRSxpQkFBa0IsQ0FDbEIsU0FBVSxDQUNWLGdCQUFpQixDQUNqQixrQkFBbUIsQ0FDbkIsaUJBQ0YiLCJmaWxlIjoiaXRlbS5jc3MiLCJzb3VyY2VzQ29udGVudCI6WyJcbi52aXMtaXRlbSB7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgY29sb3I6ICMxQTFBMUE7XG4gIGJvcmRlci1jb2xvcjogIzk3QjBGODtcbiAgYm9yZGVyLXdpZHRoOiAxcHg7XG4gIGJhY2tncm91bmQtY29sb3I6ICNENURERjY7XG4gIGRpc3BsYXk6IGlubGluZS1ibG9jaztcbiAgei1pbmRleDogMTtcbiAgLypvdmVyZmxvdzogaGlkZGVuOyovXG59XG5cbi52aXMtaXRlbS52aXMtc2VsZWN0ZWQge1xuICBib3JkZXItY29sb3I6ICNGRkMyMDA7XG4gIGJhY2tncm91bmQtY29sb3I6ICNGRkY3ODU7XG5cbiAgLyogei1pbmRleCBtdXN0IGJlIGhpZ2hlciB0aGFuIHRoZSB6LWluZGV4IG9mIGN1c3RvbSB0aW1lIGJhciBhbmQgY3VycmVudCB0aW1lIGJhciAqL1xuICB6LWluZGV4OiAyO1xufVxuXG4udmlzLWVkaXRhYmxlLnZpcy1zZWxlY3RlZCB7XG4gIGN1cnNvcjogbW92ZTtcbn1cblxuLnZpcy1pdGVtLnZpcy1wb2ludC52aXMtc2VsZWN0ZWQge1xuICBiYWNrZ3JvdW5kLWNvbG9yOiAjRkZGNzg1O1xufVxuXG4udmlzLWl0ZW0udmlzLWJveCB7XG4gIHRleHQtYWxpZ246IGNlbnRlcjtcbiAgYm9yZGVyLXN0eWxlOiBzb2xpZDtcbiAgYm9yZGVyLXJhZGl1czogMnB4O1xufVxuXG4udmlzLWl0ZW0udmlzLXBvaW50IHtcbiAgYmFja2dyb3VuZDogbm9uZTtcbn1cblxuLnZpcy1pdGVtLnZpcy1kb3Qge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHBhZGRpbmc6IDA7XG4gIGJvcmRlci13aWR0aDogNHB4O1xuICBib3JkZXItc3R5bGU6IHNvbGlkO1xuICBib3JkZXItcmFkaXVzOiA0cHg7XG59XG5cbi52aXMtaXRlbS52aXMtcmFuZ2Uge1xuICBib3JkZXItc3R5bGU6IHNvbGlkO1xuICBib3JkZXItcmFkaXVzOiAycHg7XG4gIGJveC1zaXppbmc6IGJvcmRlci1ib3g7XG59XG5cbi52aXMtaXRlbS52aXMtYmFja2dyb3VuZCB7XG4gIGJvcmRlcjogbm9uZTtcbiAgYmFja2dyb3VuZC1jb2xvcjogcmdiYSgyMTMsIDIyMSwgMjQ2LCAwLjQpO1xuICBib3gtc2l6aW5nOiBib3JkZXItYm94O1xuICBwYWRkaW5nOiAwO1xuICBtYXJnaW46IDA7XG59XG5cbi52aXMtaXRlbSAudmlzLWl0ZW0tb3ZlcmZsb3cge1xuICBwb3NpdGlvbjogcmVsYXRpdmU7XG4gIHdpZHRoOiAxMDAlO1xuICBoZWlnaHQ6IDEwMCU7XG4gIHBhZGRpbmc6IDA7XG4gIG1hcmdpbjogMDtcbiAgb3ZlcmZsb3c6IGhpZGRlbjtcbn1cblxuLnZpcy1pdGVtLXZpc2libGUtZnJhbWUge1xuICB3aGl0ZS1zcGFjZTogbm93cmFwO1xufVxuXG4udmlzLWl0ZW0udmlzLXJhbmdlIC52aXMtaXRlbS1jb250ZW50IHtcbiAgcG9zaXRpb246IHJlbGF0aXZlO1xuICBkaXNwbGF5OiBpbmxpbmUtYmxvY2s7XG59XG5cbi52aXMtaXRlbS52aXMtYmFja2dyb3VuZCAudmlzLWl0ZW0tY29udGVudCB7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgZGlzcGxheTogaW5saW5lLWJsb2NrO1xufVxuXG4udmlzLWl0ZW0udmlzLWxpbmUge1xuICBwYWRkaW5nOiAwO1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHdpZHRoOiAwO1xuICBib3JkZXItbGVmdC13aWR0aDogMXB4O1xuICBib3JkZXItbGVmdC1zdHlsZTogc29saWQ7XG59XG5cbi52aXMtaXRlbSAudmlzLWl0ZW0tY29udGVudCB7XG4gIHdoaXRlLXNwYWNlOiBub3dyYXA7XG4gIGJveC1zaXppbmc6IGJvcmRlci1ib3g7XG4gIHBhZGRpbmc6IDVweDtcbn1cblxuLnZpcy1pdGVtIC52aXMtb25VcGRhdGVUaW1lLXRvb2x0aXAge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIGJhY2tncm91bmQ6ICM0ZjgxYmQ7XG4gIGNvbG9yOiB3aGl0ZTtcbiAgd2lkdGg6IDIwMHB4O1xuICB0ZXh0LWFsaWduOiBjZW50ZXI7XG4gIHdoaXRlLXNwYWNlOiBub3dyYXA7XG4gIHBhZGRpbmc6IDVweDtcbiAgYm9yZGVyLXJhZGl1czogMXB4O1xuICB0cmFuc2l0aW9uOiAwLjRzO1xuICAtby10cmFuc2l0aW9uOiAwLjRzO1xuICAtbW96LXRyYW5zaXRpb246IDAuNHM7XG4gIC13ZWJraXQtdHJhbnNpdGlvbjogMC40cztcbn1cblxuLnZpcy1pdGVtIC52aXMtZGVsZXRlLCAudmlzLWl0ZW0gLnZpcy1kZWxldGUtcnRsIHtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICB0b3A6IDBweDtcbiAgd2lkdGg6IDI0cHg7XG4gIGhlaWdodDogMjRweDtcbiAgYm94LXNpemluZzogYm9yZGVyLWJveDtcbiAgcGFkZGluZzogMHB4IDVweDtcbiAgY3Vyc29yOiBwb2ludGVyO1xuXG4gIC13ZWJraXQtdHJhbnNpdGlvbjogYmFja2dyb3VuZCAwLjJzIGxpbmVhcjtcbiAgLW1vei10cmFuc2l0aW9uOiBiYWNrZ3JvdW5kIDAuMnMgbGluZWFyO1xuICAtbXMtdHJhbnNpdGlvbjogYmFja2dyb3VuZCAwLjJzIGxpbmVhcjtcbiAgLW8tdHJhbnNpdGlvbjogYmFja2dyb3VuZCAwLjJzIGxpbmVhcjtcbiAgdHJhbnNpdGlvbjogYmFja2dyb3VuZCAwLjJzIGxpbmVhcjtcbn1cblxuLnZpcy1pdGVtIC52aXMtZGVsZXRlIHtcbiAgcmlnaHQ6IC0yNHB4O1xufVxuXG4udmlzLWl0ZW0gLnZpcy1kZWxldGUtcnRsIHtcbiAgbGVmdDogLTI0cHg7XG59XG5cbi52aXMtaXRlbSAudmlzLWRlbGV0ZTphZnRlciwgLnZpcy1pdGVtIC52aXMtZGVsZXRlLXJ0bDphZnRlciB7XG4gIGNvbnRlbnQ6IFwiXFwwMEQ3XCI7IC8qIE1VTFRJUExJQ0FUSU9OIFNJR04gKi9cbiAgY29sb3I6IHJlZDtcbiAgZm9udC1mYW1pbHk6IGFyaWFsLCBzYW5zLXNlcmlmO1xuICBmb250LXNpemU6IDIycHg7XG4gIGZvbnQtd2VpZ2h0OiBib2xkO1xuXG4gIC13ZWJraXQtdHJhbnNpdGlvbjogY29sb3IgMC4ycyBsaW5lYXI7XG4gIC1tb3otdHJhbnNpdGlvbjogY29sb3IgMC4ycyBsaW5lYXI7XG4gIC1tcy10cmFuc2l0aW9uOiBjb2xvciAwLjJzIGxpbmVhcjtcbiAgLW8tdHJhbnNpdGlvbjogY29sb3IgMC4ycyBsaW5lYXI7XG4gIHRyYW5zaXRpb246IGNvbG9yIDAuMnMgbGluZWFyO1xufVxuXG4udmlzLWl0ZW0gLnZpcy1kZWxldGU6aG92ZXIsIC52aXMtaXRlbSAudmlzLWRlbGV0ZS1ydGw6aG92ZXIge1xuICBiYWNrZ3JvdW5kOiByZWQ7XG59XG5cbi52aXMtaXRlbSAudmlzLWRlbGV0ZTpob3ZlcjphZnRlciwgLnZpcy1pdGVtIC52aXMtZGVsZXRlLXJ0bDpob3ZlcjphZnRlciB7XG4gIGNvbG9yOiB3aGl0ZTtcbn1cblxuLnZpcy1pdGVtIC52aXMtZHJhZy1jZW50ZXIge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHdpZHRoOiAxMDAlO1xuICBoZWlnaHQ6IDEwMCU7XG4gIHRvcDogMDtcbiAgbGVmdDogMHB4O1xuICBjdXJzb3I6IG1vdmU7XG59XG5cbi52aXMtaXRlbS52aXMtcmFuZ2UgLnZpcy1kcmFnLWxlZnQge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHdpZHRoOiAyNHB4O1xuICBtYXgtd2lkdGg6IDIwJTtcbiAgbWluLXdpZHRoOiAycHg7XG4gIGhlaWdodDogMTAwJTtcbiAgdG9wOiAwO1xuICBsZWZ0OiAtNHB4O1xuXG4gIGN1cnNvcjogdy1yZXNpemU7XG59XG5cbi52aXMtaXRlbS52aXMtcmFuZ2UgLnZpcy1kcmFnLXJpZ2h0IHtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICB3aWR0aDogMjRweDtcbiAgbWF4LXdpZHRoOiAyMCU7XG4gIG1pbi13aWR0aDogMnB4O1xuICBoZWlnaHQ6IDEwMCU7XG4gIHRvcDogMDtcbiAgcmlnaHQ6IC00cHg7XG5cbiAgY3Vyc29yOiBlLXJlc2l6ZTtcbn1cblxuLnZpcy1yYW5nZS52aXMtaXRlbS52aXMtcmVhZG9ubHkgLnZpcy1kcmFnLWxlZnQsXG4udmlzLXJhbmdlLnZpcy1pdGVtLnZpcy1yZWFkb25seSAudmlzLWRyYWctcmlnaHQge1xuICBjdXJzb3I6IGF1dG87XG59XG5cbi52aXMtaXRlbS52aXMtY2x1c3RlciB7XG4gIHZlcnRpY2FsLWFsaWduOiBjZW50ZXI7XG4gIHRleHQtYWxpZ246IGNlbnRlcjtcbiAgYm9yZGVyLXN0eWxlOiBzb2xpZDtcbiAgYm9yZGVyLXJhZGl1czogMnB4O1xufVxuXG4udmlzLWl0ZW0udmlzLWNsdXN0ZXItbGluZSB7XG4gIHBhZGRpbmc6IDA7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgd2lkdGg6IDA7XG4gIGJvcmRlci1sZWZ0LXdpZHRoOiAxcHg7XG4gIGJvcmRlci1sZWZ0LXN0eWxlOiBzb2xpZDtcbn1cblxuLnZpcy1pdGVtLnZpcy1jbHVzdGVyLWRvdCB7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgcGFkZGluZzogMDtcbiAgYm9yZGVyLXdpZHRoOiA0cHg7XG4gIGJvcmRlci1zdHlsZTogc29saWQ7XG4gIGJvcmRlci1yYWRpdXM6IDRweDtcbn0iXX0= */');var Pd=function(){function t(e,i,n){var o,s=this;ba(this,t),this.id=null,this.parent=null,this.data=e,this.dom=null,this.conversion=i||{},this.defaultOptions={locales:dd,locale:"en"},this.options=za.extend({},this.defaultOptions,n),this.options.locales=za.extend({},dd,this.options.locales);var r=this.defaultOptions.locales[this.defaultOptions.locale];mn(o=Yi(this.options.locales)).call(o,(function(t){s.options.locales[t]=za.extend({},r,s.options.locales[t])})),this.selected=!1,this.displayed=!1,this.groupShowing=!0,this.selectable=n&&n.selectable||!1,this.dirty=!0,this.top=null,this.right=null,this.left=null,this.width=null,this.height=null,this.setSelectability(e),this.editable=null,this._updateEditStatus()}return xa(t,[{key:"select",value:function(){this.selectable&&(this.selected=!0,this.dirty=!0,this.displayed&&this.redraw())}},{key:"unselect",value:function(){this.selected=!1,this.dirty=!0,this.displayed&&this.redraw()}},{key:"setData",value:function(t){null!=t.group&&this.data.group!=t.group&&null!=this.parent&&this.parent.itemSet._moveToGroup(this,t.group),this.setSelectability(t),this.parent&&(this.parent.stackDirty=!0),null!=t.subgroup&&this.data.subgroup!=t.subgroup&&null!=this.parent&&this.parent.changeSubgroup(this,this.data.subgroup,t.subgroup),this.data=t,this._updateEditStatus(),this.dirty=!0,this.displayed&&this.redraw()}},{key:"setSelectability",value:function(t){t&&(this.selectable=void 0===t.selectable||Boolean(t.selectable))}},{key:"setParent",value:function(t){this.displayed?(this.hide(),this.parent=t,this.parent&&this.show()):this.parent=t}},{key:"isVisible",value:function(t){return!1}},{key:"show",value:function(){return!1}},{key:"hide",value:function(){return!1}},{key:"redraw",value:function(){}},{key:"repositionX",value:function(){}},{key:"repositionY",value:function(){}},{key:"_repaintDragCenter",value:function(){if(this.selected&&this.editable.updateTime&&!this.dom.dragCenter){var t,e,i=this,n=document.createElement("div");n.className="vis-drag-center",n.dragCenterItem=this,this.hammerDragCenter=new Vh(n),this.hammerDragCenter.on("tap",(function(t){i.parent.itemSet.body.emitter.emit("click",{event:t,item:i.id})})),this.hammerDragCenter.on("doubletap",(function(t){t.stopPropagation(),i.parent.itemSet._onUpdateItem(i),i.parent.itemSet.body.emitter.emit("doubleClick",{event:t,item:i.id})})),this.hammerDragCenter.on("panstart",(function(t){t.stopPropagation(),i.parent.itemSet._onDragStart(t)})),this.hammerDragCenter.on("panmove",va(t=i.parent.itemSet._onDrag).call(t,i.parent.itemSet)),this.hammerDragCenter.on("panend",va(e=i.parent.itemSet._onDragEnd).call(e,i.parent.itemSet)),this.dom.box?this.dom.dragLeft?this.dom.box.insertBefore(n,this.dom.dragLeft):this.dom.box.appendChild(n):this.dom.point&&this.dom.point.appendChild(n),this.dom.dragCenter=n}else!this.selected&&this.dom.dragCenter&&(this.dom.dragCenter.parentNode&&this.dom.dragCenter.parentNode.removeChild(this.dom.dragCenter),this.dom.dragCenter=null,this.hammerDragCenter&&(this.hammerDragCenter.destroy(),this.hammerDragCenter=null))}},{key:"_repaintDeleteButton",value:function(t){var e=(this.options.editable.overrideItems||null==this.editable)&&this.options.editable.remove||!this.options.editable.overrideItems&&null!=this.editable&&this.editable.remove;if(this.selected&&e&&!this.dom.deleteButton){var i=this,n=document.createElement("div");this.options.rtl?n.className="vis-delete-rtl":n.className="vis-delete";var o=this.options.locales[this.options.locale];o||(this.warned||(console.warn("WARNING: options.locales['".concat(this.options.locale,"'] not found. See https://visjs.github.io/vis-timeline/docs/timeline/#Localization")),this.warned=!0),o=this.options.locales.en),n.title=o.deleteSelected,this.hammerDeleteButton=new Vh(n).on("tap",(function(t){t.stopPropagation(),i.parent.removeFromDataSet(i)})),t.appendChild(n),this.dom.deleteButton=n}else!this.selected&&this.dom.deleteButton&&(this.dom.deleteButton.parentNode&&this.dom.deleteButton.parentNode.removeChild(this.dom.deleteButton),this.dom.deleteButton=null,this.hammerDeleteButton&&(this.hammerDeleteButton.destroy(),this.hammerDeleteButton=null))}},{key:"_repaintOnItemUpdateTimeTooltip",value:function(t){if(this.options.tooltipOnItemUpdateTime){var e=(this.options.editable.updateTime||!0===this.data.editable)&&!1!==this.data.editable;if(this.selected&&e&&!this.dom.onItemUpdateTimeTooltip){var i=document.createElement("div");i.className="vis-onUpdateTime-tooltip",t.appendChild(i),this.dom.onItemUpdateTimeTooltip=i}else!this.selected&&this.dom.onItemUpdateTimeTooltip&&(this.dom.onItemUpdateTimeTooltip.parentNode&&this.dom.onItemUpdateTimeTooltip.parentNode.removeChild(this.dom.onItemUpdateTimeTooltip),this.dom.onItemUpdateTimeTooltip=null);if(this.dom.onItemUpdateTimeTooltip){this.dom.onItemUpdateTimeTooltip.style.visibility=this.parent.itemSet.touchParams.itemIsDragging?"visible":"hidden",this.dom.onItemUpdateTimeTooltip.style.transform="translateX(-50%)",this.dom.onItemUpdateTimeTooltip.style.left="50%";var n,o,s=this.parent.itemSet.body.domProps.scrollTop;if(("top"==this.options.orientation.item?this.top:this.parent.height-this.top-this.height)+this.parent.top-50<-s?(this.dom.onItemUpdateTimeTooltip.style.bottom="",this.dom.onItemUpdateTimeTooltip.style.top="".concat(this.height+2,"px")):(this.dom.onItemUpdateTimeTooltip.style.top="",this.dom.onItemUpdateTimeTooltip.style.bottom="".concat(this.height+2,"px")),this.options.tooltipOnItemUpdateTime&&this.options.tooltipOnItemUpdateTime.template)n=va(o=this.options.tooltipOnItemUpdateTime.template).call(o,this)(this.data);else n="start: ".concat(Ta(this.data.start).format("MM/DD/YYYY hh:mm")),this.data.end&&(n+="<br> end: ".concat(Ta(this.data.end).format("MM/DD/YYYY hh:mm")));this.dom.onItemUpdateTimeTooltip.innerHTML=n}}}},{key:"_getItemData",value:function(){return this.parent.itemSet.itemsData.get(this.id)}},{key:"_updateContents",value:function(t){var e,i,n,o,s,r,a=this._getItemData(),l=(this.dom.box||this.dom.point).getElementsByClassName("vis-item-visible-frame")[0];this.options.visibleFrameTemplate?n=(o=va(s=this.options.visibleFrameTemplate).call(s,this))(a,l):n="";if(l)if(n instanceof Object&&!(n instanceof Element))o(a,l);else if(this._contentToString(this.itemVisibleFrameContent)!==this._contentToString(n)){if(n instanceof Element)l.innerHTML="",l.appendChild(n);else if(null!=n)l.innerHTML=n;else if("background"!=this.data.type||void 0!==this.data.content)throw new Error('Property "content" missing in item '.concat(this.id));this.itemVisibleFrameContent=n}this.options.template?e=(i=va(r=this.options.template).call(r,this))(a,t,this.data):e=this.data.content;if(e instanceof Object&&!(e instanceof Element))i(a,t);else if(this._contentToString(this.content)!==this._contentToString(e)){if(e instanceof Element)t.innerHTML="",t.appendChild(e);else if(null!=e)t.innerHTML=e;else if("background"!=this.data.type||void 0!==this.data.content)throw new Error('Property "content" missing in item '.concat(this.id));this.content=e}}},{key:"_updateDataAttributes",value:function(t){if(this.options.dataAttributes&&this.options.dataAttributes.length>0){var e=[];if(Zo(this.options.dataAttributes))e=this.options.dataAttributes;else{if("all"!=this.options.dataAttributes)return;e=Yi(this.data)}var i=!0,n=!1,o=void 0;try{for(var s,r=Fi(e);!(i=(s=r.next()).done);i=!0){var a=s.value,l=this.data[a];null!=l?t.setAttribute("data-".concat(a),l):t.removeAttribute("data-".concat(a))}}catch(t){n=!0,o=t}finally{try{i||null==r.return||r.return()}finally{if(n)throw o}}}}},{key:"_updateStyle",value:function(t){this.style&&(za.removeCssText(t,this.style),this.style=null),this.data.style&&(za.addCssText(t,this.data.style),this.style=this.data.style)}},{key:"_contentToString",value:function(t){return"string"==typeof t?t:t&&"outerHTML"in t?t.outerHTML:t}},{key:"_updateEditStatus",value:function(){this.options&&("boolean"==typeof this.options.editable?this.editable={updateTime:this.options.editable,updateGroup:this.options.editable,remove:this.options.editable}:"object"===zo(this.options.editable)&&(this.editable={},za.selectiveExtend(["updateTime","updateGroup","remove"],this.editable,this.options.editable))),this.options&&this.options.editable&&!0===this.options.editable.overrideItems||this.data&&("boolean"==typeof this.data.editable?this.editable={updateTime:this.data.editable,updateGroup:this.data.editable,remove:this.data.editable}:"object"===zo(this.data.editable)&&(this.editable={},za.selectiveExtend(["updateTime","updateGroup","remove"],this.editable,this.data.editable)))}},{key:"getWidthLeft",value:function(){return 0}},{key:"getWidthRight",value:function(){return 0}},{key:"getTitle",value:function(){var t;return this.options.tooltip&&this.options.tooltip.template?va(t=this.options.tooltip.template).call(t,this)(this._getItemData(),this.data):this.data.title}}]),t}();Pd.prototype.stack=!0;var Jd=function(t){function e(t,i,n){var o;if(ba(this,e),(o=ar(this,dr(e).call(this,t,i,n))).props={dot:{width:0,height:0},line:{width:0,height:0}},t&&null==t.start)throw new Error('Property "start" missing in item '.concat(t));return o}return pr(e,t),xa(e,[{key:"isVisible",value:function(t){if(this.cluster)return!1;var e,i=this.data.align||this.options.align,n=this.width*t.getMillisecondsPerPixel();return e="right"==i?this.data.start.getTime()>t.start&&this.data.start.getTime()-n<t.end:"left"==i?this.data.start.getTime()+n>t.start&&this.data.start.getTime()<t.end:this.data.start.getTime()+n/2>t.start&&this.data.start.getTime()-n/2<t.end,e}},{key:"_createDomElement",value:function(){this.dom||(this.dom={},this.dom.box=document.createElement("DIV"),this.dom.content=document.createElement("DIV"),this.dom.content.className="vis-item-content",this.dom.box.appendChild(this.dom.content),this.dom.line=document.createElement("DIV"),this.dom.line.className="vis-line",this.dom.dot=document.createElement("DIV"),this.dom.dot.className="vis-dot",this.dom.box["vis-item"]=this,this.dirty=!0)}},{key:"_appendDomElement",value:function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.box.parentNode){var t=this.parent.dom.foreground;if(!t)throw new Error("Cannot redraw item: parent has no foreground container element");t.appendChild(this.dom.box)}if(!this.dom.line.parentNode){var e=this.parent.dom.background;if(!e)throw new Error("Cannot redraw item: parent has no background container element");e.appendChild(this.dom.line)}if(!this.dom.dot.parentNode){var i=this.parent.dom.axis;if(!e)throw new Error("Cannot redraw item: parent has no axis container element");i.appendChild(this.dom.dot)}this.displayed=!0}},{key:"_updateDirtyDomComponents",value:function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.box),this._updateStyle(this.dom.box);var t=this.editable.updateTime||this.editable.updateGroup,e=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+(t?" vis-editable":" vis-readonly");this.dom.box.className="vis-item vis-box".concat(e),this.dom.line.className="vis-item vis-line".concat(e),this.dom.dot.className="vis-item vis-dot".concat(e)}}},{key:"_getDomComponentsSizes",value:function(){return{previous:{right:this.dom.box.style.right,left:this.dom.box.style.left},dot:{height:this.dom.dot.offsetHeight,width:this.dom.dot.offsetWidth},line:{width:this.dom.line.offsetWidth},box:{width:this.dom.box.offsetWidth,height:this.dom.box.offsetHeight}}}},{key:"_updateDomComponentsSizes",value:function(t){this.options.rtl?this.dom.box.style.right="0px":this.dom.box.style.left="0px",this.props.dot.height=t.dot.height,this.props.dot.width=t.dot.width,this.props.line.width=t.line.width,this.width=t.box.width,this.height=t.box.height,this.options.rtl?this.dom.box.style.right=t.previous.right:this.dom.box.style.left=t.previous.left,this.dirty=!1}},{key:"_repaintDomAdditionals",value:function(){this._repaintOnItemUpdateTimeTooltip(this.dom.box),this._repaintDragCenter(),this._repaintDeleteButton(this.dom.box)}},{key:"redraw",value:function(t){var e,i,n,o,s,r,a=this,l=[va(e=this._createDomElement).call(e,this),va(i=this._appendDomElement).call(i,this),va(n=this._updateDirtyDomComponents).call(n,this),function(){a.dirty&&(s=a._getDomComponentsSizes())},function(){var t;a.dirty&&va(t=a._updateDomComponentsSizes).call(t,a)(s)},va(o=this._repaintDomAdditionals).call(o,this)];return t?l:(mn(l).call(l,(function(t){r=t()})),r)}},{key:"show",value:function(t){if(!this.displayed)return this.redraw(t)}},{key:"hide",value:function(){if(this.displayed){var t=this.dom;t.box.remove?t.box.remove():t.box.parentNode&&t.box.parentNode.removeChild(t.box),t.line.remove?t.line.remove():t.line.parentNode&&t.line.parentNode.removeChild(t.line),t.dot.remove?t.dot.remove():t.dot.parentNode&&t.dot.parentNode.removeChild(t.dot),this.displayed=!1}}},{key:"repositionXY",value:function(){var t=this.options.rtl,e=function(t,e,i){var n,o=arguments.length>3&&void 0!==arguments[3]&&arguments[3];if(void 0!==e||void 0!==i){var s=o?-1*e:e;t.style.transform=void 0!==i?void 0!==e?Pn(n="translate(".concat(s,"px, ")).call(n,i,"px)"):"translateY(".concat(i,"px)"):"translateX(".concat(s,"px)")}};e(this.dom.box,this.boxX,this.boxY,t),e(this.dom.dot,this.dotX,this.dotY,t),e(this.dom.line,this.lineX,this.lineY,t)}},{key:"repositionX",value:function(){var t=this.conversion.toScreen(this.data.start),e=void 0===this.data.align?this.options.align:this.data.align,i=this.props.line.width,n=this.props.dot.width;"right"==e?(this.boxX=t-this.width,this.lineX=t-i,this.dotX=t-i/2-n/2):"left"==e?(this.boxX=t,this.lineX=t,this.dotX=t+i/2-n/2):(this.boxX=t-this.width/2,this.lineX=this.options.rtl?t-i:t-i/2,this.dotX=t-n/2),this.options.rtl?this.right=this.boxX:this.left=this.boxX,this.repositionXY()}},{key:"repositionY",value:function(){var t=this.options.orientation.item,e=this.dom.line.style;if("top"==t){var i=this.parent.top+this.top+1;this.boxY=this.top||0,e.height="".concat(i,"px"),e.bottom="",e.top="0"}else{var n=this.parent.itemSet.props.height-this.parent.top-this.parent.height+this.top;this.boxY=this.parent.height-this.top-(this.height||0),e.height="".concat(n,"px"),e.top="",e.bottom="0"}this.dotY=-this.props.dot.height/2,this.repositionXY()}},{key:"getWidthLeft",value:function(){return this.width/2}},{key:"getWidthRight",value:function(){return this.width/2}}]),e}(Pd),Kd=function(t){function e(t,i,n){var o;if(ba(this,e),(o=ar(this,dr(e).call(this,t,i,n))).props={dot:{top:0,width:0,height:0},content:{height:0,marginLeft:0,marginRight:0}},t&&null==t.start)throw new Error('Property "start" missing in item '.concat(t));return o}return pr(e,t),xa(e,[{key:"isVisible",value:function(t){if(this.cluster)return!1;var e=this.width*t.getMillisecondsPerPixel();return this.data.start.getTime()+e>t.start&&this.data.start<t.end}},{key:"_createDomElement",value:function(){this.dom||(this.dom={},this.dom.point=document.createElement("div"),this.dom.content=document.createElement("div"),this.dom.content.className="vis-item-content",this.dom.point.appendChild(this.dom.content),this.dom.dot=document.createElement("div"),this.dom.point.appendChild(this.dom.dot),this.dom.point["vis-item"]=this,this.dirty=!0)}},{key:"_appendDomElement",value:function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.point.parentNode){var t=this.parent.dom.foreground;if(!t)throw new Error("Cannot redraw item: parent has no foreground container element");t.appendChild(this.dom.point)}this.displayed=!0}},{key:"_updateDirtyDomComponents",value:function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.point),this._updateStyle(this.dom.point);var t=this.editable.updateTime||this.editable.updateGroup,e=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+(t?" vis-editable":" vis-readonly");this.dom.point.className="vis-item vis-point".concat(e),this.dom.dot.className="vis-item vis-dot".concat(e)}}},{key:"_getDomComponentsSizes",value:function(){return{dot:{width:this.dom.dot.offsetWidth,height:this.dom.dot.offsetHeight},content:{width:this.dom.content.offsetWidth,height:this.dom.content.offsetHeight},point:{width:this.dom.point.offsetWidth,height:this.dom.point.offsetHeight}}}},{key:"_updateDomComponentsSizes",value:function(t){this.props.dot.width=t.dot.width,this.props.dot.height=t.dot.height,this.props.content.height=t.content.height,this.options.rtl?this.dom.content.style.marginRight="".concat(2*this.props.dot.width,"px"):this.dom.content.style.marginLeft="".concat(2*this.props.dot.width,"px"),this.width=t.point.width,this.height=t.point.height,this.dom.dot.style.top="".concat((this.height-this.props.dot.height)/2,"px");var e=this.props.dot.width,i=this.options.rtl?e/2*-1:e/2;this.dom.dot.style.transform="translateX(".concat(i,"px"),this.dirty=!1}},{key:"_repaintDomAdditionals",value:function(){this._repaintOnItemUpdateTimeTooltip(this.dom.point),this._repaintDragCenter(),this._repaintDeleteButton(this.dom.point)}},{key:"redraw",value:function(t){var e,i,n,o,s,r,a=this,l=[va(e=this._createDomElement).call(e,this),va(i=this._appendDomElement).call(i,this),va(n=this._updateDirtyDomComponents).call(n,this),function(){a.dirty&&(s=a._getDomComponentsSizes())},function(){var t;a.dirty&&va(t=a._updateDomComponentsSizes).call(t,a)(s)},va(o=this._repaintDomAdditionals).call(o,this)];return t?l:(mn(l).call(l,(function(t){r=t()})),r)}},{key:"repositionXY",value:function(){var t=this.options.rtl,e=function(t,e,i){var n,o=arguments.length>3&&void 0!==arguments[3]&&arguments[3];if(void 0!==e||void 0!==i){var s=o?-1*e:e;t.style.transform=void 0!==i?void 0!==e?Pn(n="translate(".concat(s,"px, ")).call(n,i,"px)"):"translateY(".concat(i,"px)"):"translateX(".concat(s,"px)")}};e(this.dom.point,this.pointX,this.pointY,t)}},{key:"show",value:function(t){if(!this.displayed)return this.redraw(t)}},{key:"hide",value:function(){this.displayed&&(this.dom.point.parentNode&&this.dom.point.parentNode.removeChild(this.dom.point),this.displayed=!1)}},{key:"repositionX",value:function(){var t=this.conversion.toScreen(this.data.start);this.pointX=t,this.options.rtl?this.right=t-this.props.dot.width:this.left=t-this.props.dot.width,this.repositionXY()}},{key:"repositionY",value:function(){var t=this.options.orientation.item;this.pointY="top"==t?this.top:this.parent.height-this.top-this.height,this.repositionXY()}},{key:"getWidthLeft",value:function(){return this.props.dot.width}},{key:"getWidthRight",value:function(){return this.props.dot.width}}]),e}(Pd),qd=function(t){function e(t,i,n){var o;if(ba(this,e),(o=ar(this,dr(e).call(this,t,i,n))).props={content:{width:0}},o.overflow=!1,t){if(null==t.start)throw new Error('Property "start" missing in item '.concat(t.id));if(null==t.end)throw new Error('Property "end" missing in item '.concat(t.id))}return o}return pr(e,t),xa(e,[{key:"isVisible",value:function(t){return!this.cluster&&(this.data.start<t.end&&this.data.end>t.start)}},{key:"_createDomElement",value:function(){this.dom||(this.dom={},this.dom.box=document.createElement("div"),this.dom.frame=document.createElement("div"),this.dom.frame.className="vis-item-overflow",this.dom.box.appendChild(this.dom.frame),this.dom.visibleFrame=document.createElement("div"),this.dom.visibleFrame.className="vis-item-visible-frame",this.dom.box.appendChild(this.dom.visibleFrame),this.dom.content=document.createElement("div"),this.dom.content.className="vis-item-content",this.dom.frame.appendChild(this.dom.content),this.dom.box["vis-item"]=this,this.dirty=!0)}},{key:"_appendDomElement",value:function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.box.parentNode){var t=this.parent.dom.foreground;if(!t)throw new Error("Cannot redraw item: parent has no foreground container element");t.appendChild(this.dom.box)}this.displayed=!0}},{key:"_updateDirtyDomComponents",value:function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.box),this._updateStyle(this.dom.box);var t=this.editable.updateTime||this.editable.updateGroup,e=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+(t?" vis-editable":" vis-readonly");this.dom.box.className=this.baseClassName+e,this.dom.content.style.maxWidth="none"}}},{key:"_getDomComponentsSizes",value:function(){return this.overflow="hidden"!==window.getComputedStyle(this.dom.frame).overflow,this.whiteSpace="nowrap"!==window.getComputedStyle(this.dom.content).whiteSpace,{content:{width:this.dom.content.offsetWidth},box:{height:this.dom.box.offsetHeight}}}},{key:"_updateDomComponentsSizes",value:function(t){this.props.content.width=t.content.width,this.height=t.box.height,this.dom.content.style.maxWidth="",this.dirty=!1}},{key:"_repaintDomAdditionals",value:function(){this._repaintOnItemUpdateTimeTooltip(this.dom.box),this._repaintDeleteButton(this.dom.box),this._repaintDragCenter(),this._repaintDragLeft(),this._repaintDragRight()}},{key:"redraw",value:function(t){var e,i,n,o,s,r,a=this,l=[va(e=this._createDomElement).call(e,this),va(i=this._appendDomElement).call(i,this),va(n=this._updateDirtyDomComponents).call(n,this),function(){var t;a.dirty&&(s=va(t=a._getDomComponentsSizes).call(t,a)())},function(){var t;a.dirty&&va(t=a._updateDomComponentsSizes).call(t,a)(s)},va(o=this._repaintDomAdditionals).call(o,this)];return t?l:(mn(l).call(l,(function(t){r=t()})),r)}},{key:"show",value:function(t){if(!this.displayed)return this.redraw(t)}},{key:"hide",value:function(){if(this.displayed){var t=this.dom.box;t.parentNode&&t.parentNode.removeChild(t),this.displayed=!1}}},{key:"repositionX",value:function(t){var e,i,n=this.parent.width,o=this.conversion.toScreen(this.data.start),s=this.conversion.toScreen(this.data.end),r=void 0===this.data.align?this.options.align:this.data.align;!1===this.data.limitSize||void 0!==t&&!0!==t||(o<-n&&(o=-n),s>2*n&&(s=2*n));var a=Math.max(Math.round(1e3*(s-o))/1e3,1);switch(this.overflow?(this.options.rtl?this.right=o:this.left=o,this.width=a+this.props.content.width,i=this.props.content.width):(this.options.rtl?this.right=o:this.left=o,this.width=a,i=Math.min(s-o,this.props.content.width)),this.options.rtl?this.dom.box.style.transform="translateX(".concat(-1*this.right,"px)"):this.dom.box.style.transform="translateX(".concat(this.left,"px)"),this.dom.box.style.width="".concat(a,"px"),this.whiteSpace&&(this.height=this.dom.box.offsetHeight),r){case"left":this.dom.content.style.transform="translateX(0)";break;case"right":if(this.options.rtl){var l=-1*Math.max(a-i,0);this.dom.content.style.transform="translateX(".concat(l,"px)")}else this.dom.content.style.transform="translateX(".concat(Math.max(a-i,0),"px)");break;case"center":if(this.options.rtl){var h=-1*Math.max((a-i)/2,0);this.dom.content.style.transform="translateX(".concat(h,"px)")}else this.dom.content.style.transform="translateX(".concat(Math.max((a-i)/2,0),"px)");break;default:if(e=this.overflow?s>0?Math.max(-o,0):-i:o<0?-o:0,this.options.rtl){var d=-1*e;this.dom.content.style.transform="translateX(".concat(d,"px)")}else this.dom.content.style.transform="translateX(".concat(e,"px)")}}},{key:"repositionY",value:function(){var t=this.options.orientation.item,e=this.dom.box;e.style.top="".concat("top"==t?this.top:this.parent.height-this.top-this.height,"px")}},{key:"_repaintDragLeft",value:function(){if((this.selected||this.options.itemsAlwaysDraggable.range)&&this.editable.updateTime&&!this.dom.dragLeft){var t=document.createElement("div");t.className="vis-drag-left",t.dragLeftItem=this,this.dom.box.appendChild(t),this.dom.dragLeft=t}else this.selected||this.options.itemsAlwaysDraggable.range||!this.dom.dragLeft||(this.dom.dragLeft.parentNode&&this.dom.dragLeft.parentNode.removeChild(this.dom.dragLeft),this.dom.dragLeft=null)}},{key:"_repaintDragRight",value:function(){if((this.selected||this.options.itemsAlwaysDraggable.range)&&this.editable.updateTime&&!this.dom.dragRight){var t=document.createElement("div");t.className="vis-drag-right",t.dragRightItem=this,this.dom.box.appendChild(t),this.dom.dragRight=t}else this.selected||this.options.itemsAlwaysDraggable.range||!this.dom.dragRight||(this.dom.dragRight.parentNode&&this.dom.dragRight.parentNode.removeChild(this.dom.dragRight),this.dom.dragRight=null)}}]),e}(Pd);qd.prototype.baseClassName="vis-item vis-range";var $d=function(t){function e(t,i,n){var o;if(ba(this,e),(o=ar(this,dr(e).call(this,t,i,n))).props={content:{width:0}},o.overflow=!1,t){if(null==t.start)throw new Error('Property "start" missing in item '.concat(t.id));if(null==t.end)throw new Error('Property "end" missing in item '.concat(t.id))}return o}return pr(e,t),xa(e,[{key:"isVisible",value:function(t){return this.data.start<t.end&&this.data.end>t.start}},{key:"_createDomElement",value:function(){this.dom||(this.dom={},this.dom.box=document.createElement("div"),this.dom.frame=document.createElement("div"),this.dom.frame.className="vis-item-overflow",this.dom.box.appendChild(this.dom.frame),this.dom.content=document.createElement("div"),this.dom.content.className="vis-item-content",this.dom.frame.appendChild(this.dom.content),this.dirty=!0)}},{key:"_appendDomElement",value:function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.box.parentNode){var t=this.parent.dom.background;if(!t)throw new Error("Cannot redraw item: parent has no background container element");t.appendChild(this.dom.box)}this.displayed=!0}},{key:"_updateDirtyDomComponents",value:function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.content),this._updateStyle(this.dom.box);var t=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"");this.dom.box.className=this.baseClassName+t}}},{key:"_getDomComponentsSizes",value:function(){return this.overflow="hidden"!==window.getComputedStyle(this.dom.content).overflow,{content:{width:this.dom.content.offsetWidth}}}},{key:"_updateDomComponentsSizes",value:function(t){this.props.content.width=t.content.width,this.height=0,this.dirty=!1}},{key:"_repaintDomAdditionals",value:function(){}},{key:"redraw",value:function(t){var e,i,n,o,s,r,a=this,l=[va(e=this._createDomElement).call(e,this),va(i=this._appendDomElement).call(i,this),va(n=this._updateDirtyDomComponents).call(n,this),function(){var t;a.dirty&&(s=va(t=a._getDomComponentsSizes).call(t,a)())},function(){var t;a.dirty&&va(t=a._updateDomComponentsSizes).call(t,a)(s)},va(o=this._repaintDomAdditionals).call(o,this)];return t?l:(mn(l).call(l,(function(t){r=t()})),r)}},{key:"repositionY",value:function(t){var e,i=this.options.orientation.item;if(void 0!==this.data.subgroup){var n=this.data.subgroup;this.dom.box.style.height="".concat(this.parent.subgroups[n].height,"px"),this.dom.box.style.top="".concat("top"==i?this.parent.top+this.parent.subgroups[n].top:this.parent.top+this.parent.height-this.parent.subgroups[n].top-this.parent.subgroups[n].height,"px"),this.dom.box.style.bottom=""}else this.parent instanceof Hd?(e=Math.max(this.parent.height,this.parent.itemSet.body.domProps.center.height,this.parent.itemSet.body.domProps.centerContainer.height),this.dom.box.style.bottom="bottom"==i?"0":"",this.dom.box.style.top="top"==i?"0":""):(e=this.parent.height,this.dom.box.style.top="".concat(this.parent.top,"px"),this.dom.box.style.bottom="");this.dom.box.style.height="".concat(e,"px")}}]),e}(Pd);$d.prototype.baseClassName="vis-item vis-background",$d.prototype.stack=!1,$d.prototype.show=qd.prototype.show,$d.prototype.hide=qd.prototype.hide,$d.prototype.repositionX=qd.prototype.repositionX;Ph("div.vis-tooltip{position:absolute;visibility:hidden;padding:5px;white-space:nowrap;font-family:verdana;font-size:14px;color:#000;background-color:#f5f4ed;-moz-border-radius:3px;-webkit-border-radius:3px;border-radius:3px;border:1px solid #808074;box-shadow:3px 3px 10px rgba(0,0,0,.2);pointer-events:none;z-index:5}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbInRvb2x0aXAuY3NzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLGdCQUNFLGlCQUFrQixDQUNsQixpQkFBa0IsQ0FDbEIsV0FBWSxDQUNaLGtCQUFtQixDQUVuQixtQkFBb0IsQ0FDcEIsY0FBYyxDQUNkLFVBQWEsQ0FDYix3QkFBeUIsQ0FFekIsc0JBQXVCLENBQ3ZCLHlCQUEwQixDQUMxQixpQkFBa0IsQ0FDbEIsd0JBQXlCLENBRXpCLHNDQUEyQyxDQUMzQyxtQkFBb0IsQ0FFcEIsU0FDRiIsImZpbGUiOiJ0b29sdGlwLmNzcyIsInNvdXJjZXNDb250ZW50IjpbImRpdi52aXMtdG9vbHRpcCB7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgdmlzaWJpbGl0eTogaGlkZGVuO1xuICBwYWRkaW5nOiA1cHg7XG4gIHdoaXRlLXNwYWNlOiBub3dyYXA7XG5cbiAgZm9udC1mYW1pbHk6IHZlcmRhbmE7XG4gIGZvbnQtc2l6ZToxNHB4O1xuICBjb2xvcjojMDAwMDAwO1xuICBiYWNrZ3JvdW5kLWNvbG9yOiAjZjVmNGVkO1xuXG4gIC1tb3otYm9yZGVyLXJhZGl1czogM3B4O1xuICAtd2Via2l0LWJvcmRlci1yYWRpdXM6IDNweDtcbiAgYm9yZGVyLXJhZGl1czogM3B4O1xuICBib3JkZXI6IDFweCBzb2xpZCAjODA4MDc0O1xuXG4gIGJveC1zaGFkb3c6IDNweCAzcHggMTBweCByZ2JhKDAsIDAsIDAsIDAuMik7XG4gIHBvaW50ZXItZXZlbnRzOiBub25lO1xuXG4gIHotaW5kZXg6IDU7XG59XG4iXX0= */");var tu=function(){function t(e,i){ba(this,t),this.container=e,this.overflowMethod=i||"cap",this.x=0,this.y=0,this.padding=5,this.hidden=!1,this.frame=document.createElement("div"),this.frame.className="vis-tooltip",this.container.appendChild(this.frame)}return xa(t,[{key:"setPosition",value:function(t,e){this.x=Nn(t),this.y=Nn(e)}},{key:"setText",value:function(t){t instanceof Element?(this.frame.innerHTML="",this.frame.appendChild(t)):this.frame.innerHTML=t}},{key:"show",value:function(t){if(void 0===t&&(t=!0),!0===t){var e=this.frame.clientHeight,i=this.frame.clientWidth,n=this.frame.parentNode.clientHeight,o=this.frame.parentNode.clientWidth,s=0,r=0;if("flip"==this.overflowMethod||"none"==this.overflowMethod){var a=!1,l=!0;"flip"==this.overflowMethod&&(this.y-e<this.padding&&(l=!1),this.x+i>o-this.padding&&(a=!0)),s=a?this.x-i:this.x,r=l?this.y-e:this.y}else(r=this.y-e)+e+this.padding>n&&(r=n-e-this.padding),r<this.padding&&(r=this.padding),(s=this.x)+i+this.padding>o&&(s=o-i-this.padding),s<this.padding&&(s=this.padding);this.frame.style.left=s+"px",this.frame.style.top=r+"px",this.frame.style.visibility="visible",this.hidden=!1}else this.hide()}},{key:"hide",value:function(){this.hidden=!0,this.frame.style.left="0",this.frame.style.top="0",this.frame.style.visibility="hidden"}},{key:"destroy",value:function(){this.frame.parentNode.removeChild(this.frame)}}]),t}(),eu=Ce.every,iu=$i("every"),nu=on("every");V({target:"Array",proto:!0,forced:!iu||!nu},{every:function(t){return eu(this,t,arguments.length>1?arguments[1]:void 0)}});var ou=Pi("Array").every,su=Array.prototype,ru=function(t){var e=t.every;return t===su||t instanceof Array&&e===su.every?ou:e},au=function(t){function e(t,i,n){var o;ba(this,e);var s=Ao({},{fitOnDoubleClick:!0},n,{editable:!1});if((o=ar(this,dr(e).call(this,t,i,s))).props={content:{width:0,height:0}},!t||null==t.uiItems)throw new Error('Property "uiItems" missing in item '+t.id);return o.id=Id(),o.group=t.group,o._setupRange(),o.emitter=o.data.eventEmitter,o.range=o.data.range,o.attached=!1,o.isCluster=!0,o.data.isCluster=!0,o}return pr(e,t),xa(e,[{key:"hasItems",value:function(){return this.data.uiItems&&this.data.uiItems.length&&this.attached}},{key:"setUiItems",value:function(t){this.detach(),this.data.uiItems=t,this._setupRange(),this.attach()}},{key:"isVisible",value:function(t){var e=this.data.end?this.data.end-this.data.start:0,i=this.width*t.getMillisecondsPerPixel(),n=Math.max(e,this.data.start.getTime()+i);return this.data.start<t.end&&n>t.start&&this.hasItems()}},{key:"getData",value:function(){return{isCluster:!0,id:this.id,items:this.data.items||[],data:this.data}}},{key:"redraw",value:function(t){var e,i,n,o,s,r,a,l,h=[va(e=this._createDomElement).call(e,this),va(i=this._appendDomElement).call(i,this),va(n=this._updateDirtyDomComponents).call(n,this),va(o=function(){this.dirty&&(a=this._getDomComponentsSizes())}).call(o,this),va(s=function(){var t;this.dirty&&va(t=this._updateDomComponentsSizes).call(t,this)(a)}).call(s,this),va(r=this._repaintDomAdditionals).call(r,this)];return t?h:(mn(h).call(h,(function(t){l=t()})),l)}},{key:"show",value:function(){this.displayed||this.redraw()}},{key:"hide",value:function(){if(this.displayed){var t=this.dom;t.box.parentNode&&t.box.parentNode.removeChild(t.box),this.options.showStipes&&(t.line.parentNode&&t.line.parentNode.removeChild(t.line),t.dot.parentNode&&t.dot.parentNode.removeChild(t.dot)),this.displayed=!1}}},{key:"repositionX",value:function(){var t=this.conversion.toScreen(this.data.start),e=this.data.end?this.conversion.toScreen(this.data.end):0;if(e)this.repositionXWithRanges(t,e);else{var i=void 0===this.data.align?this.options.align:this.data.align;this.repositionXWithoutRanges(t,i)}this.options.showStipes&&(this.dom.line.style.display=this._isStipeVisible()?"block":"none",this.dom.dot.style.display=this._isStipeVisible()?"block":"none",this._isStipeVisible()&&this.repositionStype(t,e))}},{key:"repositionStype",value:function(t,e){this.dom.line.style.display="block",this.dom.dot.style.display="block";var i=this.dom.line.offsetWidth,n=this.dom.dot.offsetWidth;if(e){var o=i+t+(e-t)/2,s=o-n/2,r=this.options.rtl?-1*o:o,a=this.options.rtl?-1*s:s;this.dom.line.style.transform="translateX(".concat(r,"px)"),this.dom.dot.style.transform="translateX(".concat(a,"px)")}else{var l=this.options.rtl?-1*t:t,h=this.options.rtl?-1*(t-n/2):t-n/2;this.dom.line.style.transform="translateX(".concat(l,"px)"),this.dom.dot.style.transform="translateX(".concat(h,"px)")}}},{key:"repositionXWithoutRanges",value:function(t,e){"right"==e?this.options.rtl?(this.right=t-this.width,this.dom.box.style.right=this.right+"px"):(this.left=t-this.width,this.dom.box.style.left=this.left+"px"):"left"==e?this.options.rtl?(this.right=t,this.dom.box.style.right=this.right+"px"):(this.left=t,this.dom.box.style.left=this.left+"px"):this.options.rtl?(this.right=t-this.width/2,this.dom.box.style.right=this.right+"px"):(this.left=t-this.width/2,this.dom.box.style.left=this.left+"px")}},{key:"repositionXWithRanges",value:function(t,e){var i=Math.round(Math.max(e-t+.5,1));this.options.rtl?this.right=t:this.left=t,this.width=Math.max(i,this.minWidth||0),this.options.rtl?this.dom.box.style.right=this.right+"px":this.dom.box.style.left=this.left+"px",this.dom.box.style.width=i+"px"}},{key:"repositionY",value:function(){var t=this.options.orientation.item,e=this.dom.box;if(e.style.top="top"==t?(this.top||0)+"px":(this.parent.height-this.top-this.height||0)+"px",this.options.showStipes){if("top"==t)this.dom.line.style.top="0",this.dom.line.style.height=this.parent.top+this.top+1+"px",this.dom.line.style.bottom="";else{var i=this.parent.itemSet.props.height,n=i-this.parent.top-this.parent.height+this.top;this.dom.line.style.top=i-n+"px",this.dom.line.style.bottom="0"}this.dom.dot.style.top=-this.dom.dot.offsetHeight/2+"px"}}},{key:"getWidthLeft",value:function(){return this.width/2}},{key:"getWidthRight",value:function(){return this.width/2}},{key:"move",value:function(){this.repositionX(),this.repositionY()}},{key:"attach",value:function(){var t,e=!0,i=!1,n=void 0;try{for(var o,s=Fi(this.data.uiItems);!(e=(o=s.next()).done);e=!0){o.value.cluster=this}}catch(t){i=!0,n=t}finally{try{e||null==s.return||s.return()}finally{if(i)throw n}}this.data.items=Ln(t=this.data.uiItems).call(t,(function(t){return t.data})),this.attached=!0,this.dirty=!0}},{key:"detach",value:function(){var t=arguments.length>0&&void 0!==arguments[0]&&arguments[0];if(this.hasItems()){var e=!0,i=!1,n=void 0;try{for(var o,s=Fi(this.data.uiItems);!(e=(o=s.next()).done);e=!0){var r=o.value;delete r.cluster}}catch(t){i=!0,n=t}finally{try{e||null==s.return||s.return()}finally{if(i)throw n}}this.attached=!1,t&&this.group&&(this.group.remove(this),this.group=null),this.data.items=[],this.dirty=!0}}},{key:"_onDoubleClick",value:function(){this._fit()}},{key:"_setupRange",value:function(){var t,e,i,n=Ln(t=this.data.uiItems).call(t,(function(t){return{start:t.data.start.valueOf(),end:t.data.end?t.data.end.valueOf():t.data.start.valueOf()}}));this.data.min=Math.min.apply(Math,mo(Ln(n).call(n,(function(t){return Math.min(t.start,t.end||t.start)})))),this.data.max=Math.max.apply(Math,mo(Ln(n).call(n,(function(t){return Math.max(t.start,t.end||t.start)}))));var o=Ln(e=this.data.uiItems).call(e,(function(t){return t.center})),s=da(o).call(o,(function(t,e){return t+e}),0)/this.data.uiItems.length;rr(i=this.data.uiItems).call(i,(function(t){return t.data.end}))?(this.data.start=new Date(this.data.min),this.data.end=new Date(this.data.max)):(this.data.start=new Date(s),this.data.end=null)}},{key:"_getUiItems",value:function(){var t,e=this;return this.data.uiItems&&this.data.uiItems.length?En(t=this.data.uiItems).call(t,(function(t){return t.cluster===e})):[]}},{key:"_createDomElement",value:function(){if(!this.dom){var t;if(this.dom={},this.dom.box=document.createElement("DIV"),this.dom.content=document.createElement("DIV"),this.dom.content.className="vis-item-content",this.dom.box.appendChild(this.dom.content),this.options.showStipes&&(this.dom.line=document.createElement("DIV"),this.dom.line.className="vis-cluster-line",this.dom.line.style.display="none",this.dom.dot=document.createElement("DIV"),this.dom.dot.className="vis-cluster-dot",this.dom.dot.style.display="none"),this.options.fitOnDoubleClick)this.dom.box.ondblclick=va(t=e.prototype._onDoubleClick).call(t,this);this.dom.box["vis-item"]=this,this.dirty=!0}}},{key:"_appendDomElement",value:function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.box.parentNode){var t=this.parent.dom.foreground;if(!t)throw new Error("Cannot redraw item: parent has no foreground container element");t.appendChild(this.dom.box)}var e=this.parent.dom.background;if(this.options.showStipes){if(!this.dom.line.parentNode){if(!e)throw new Error("Cannot redraw item: parent has no background container element");e.appendChild(this.dom.line)}if(!this.dom.dot.parentNode){var i=this.parent.dom.axis;if(!e)throw new Error("Cannot redraw item: parent has no axis container element");i.appendChild(this.dom.dot)}}this.displayed=!0}},{key:"_updateDirtyDomComponents",value:function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.box),this._updateStyle(this.dom.box);var t=this.baseClassName+" "+(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+" vis-readonly";this.dom.box.className="vis-item "+t,this.options.showStipes&&(this.dom.line.className="vis-item vis-cluster-line "+(this.selected?" vis-selected":""),this.dom.dot.className="vis-item vis-cluster-dot "+(this.selected?" vis-selected":"")),this.data.end&&(this.dom.content.style.maxWidth="none")}}},{key:"_getDomComponentsSizes",value:function(){var t={previous:{right:this.dom.box.style.right,left:this.dom.box.style.left},box:{width:this.dom.box.offsetWidth,height:this.dom.box.offsetHeight}};return this.options.showStipes&&(t.dot={height:this.dom.dot.offsetHeight,width:this.dom.dot.offsetWidth},t.line={width:this.dom.line.offsetWidth}),t}},{key:"_updateDomComponentsSizes",value:function(t){this.options.rtl?this.dom.box.style.right="0px":this.dom.box.style.left="0px",this.data.end?this.minWidth=t.box.width:this.width=t.box.width,this.height=t.box.height,this.options.rtl?this.dom.box.style.right=t.previous.right:this.dom.box.style.left=t.previous.left,this.dirty=!1}},{key:"_repaintDomAdditionals",value:function(){this._repaintOnItemUpdateTimeTooltip(this.dom.box)}},{key:"_isStipeVisible",value:function(){return this.minWidth>=this.width||!this.data.end}},{key:"_getFitRange",value:function(){var t=.05*(this.data.max-this.data.min)/2;return{fitStart:this.data.min-t,fitEnd:this.data.max+t}}},{key:"_fit",value:function(){if(this.emitter){var t=this._getFitRange(),e=t.fitStart,i=t.fitEnd,n={start:new Date(e),end:new Date(i),animation:!0};this.emitter.emit("fit",n)}}},{key:"_getItemData",value:function(){return this.data}}]),e}(Pd);au.prototype.baseClassName="vis-item vis-range vis-cluster";var lu="__ungrouped__",hu=function(){function t(e){ba(this,t),this.itemSet=e,this.groups={},this.cache={},this.cache[-1]=[]}return xa(t,[{key:"createClusterItem",value:function(t,e,i){return new au(t,e,i)}},{key:"setItems",value:function(t,e){this.items=t||[],this.dataChanged=!0,this.applyOnChangedLevel=!1,e&&e.applyOnChangedLevel&&(this.applyOnChangedLevel=e.applyOnChangedLevel)}},{key:"updateData",value:function(){this.dataChanged=!0,this.applyOnChangedLevel=!1}},{key:"getClusters",value:function(t,e,i){var n="boolean"==typeof i?{}:i,o=n.maxItems,s=n.clusterCriteria;s||(s=function(){return!0}),o=o||1;var r=-1,a=0;if(e>0){if(e>=1)return[];r=Math.abs(Math.round(Math.log(100/e)/Math.log(2))),a=Math.abs(Math.pow(2,r))}if(this.dataChanged){var l=r!=this.cacheLevel;(!this.applyOnChangedLevel||l)&&(this._dropLevelsCache(),this._filterData())}this.cacheLevel=r;var h=this.cache[r];if(!h){for(var d in h=[],this.groups)if(this.groups.hasOwnProperty(d))for(var u=this.groups[d],c=u.length,p=0;p<c;){for(var m=u[p],f=1,g=p-1;g>=0&&m.center-u[g].center<a/2;)!u[g].cluster&&s(m.data,u[g].data)&&f++,g--;for(var v=p+1;v<u.length&&u[v].center-m.center<a/2;)s(m.data,u[v].data)&&f++,v++;for(var b=h.length-1;b>=0&&m.center-h[b].center<a;)m.group==h[b].group&&s(m.data,h[b].data)&&f++,b--;if(f>o){for(var y=f-o+1,x=[],w=p;x.length<y&&w<u.length;)s(u[w].data,u[w].data)&&x.push(u[w]),w++;var _=this.itemSet.getGroupId(m.data),k=this.itemSet.groups[_]||this.itemSet.groups[lu],D=this._getClusterForItems(x,k,t,i);h.push(D),p+=y}else delete m.cluster,p+=1}this.cache[r]=h}return h}},{key:"_filterData",value:function(){var t={};this.groups=t;for(var e=0,i=On(this.items);e<i.length;e++){var n=i[e],o=n.parent?n.parent.groupId:"",s=t[o];s||(s=[],t[o]=s),s.push(n),n.data.start&&(n.data.end?n.center=(n.data.start.valueOf()+n.data.end.valueOf())/2:n.center=n.data.start.valueOf())}for(var r in t){var a;if(t.hasOwnProperty(r))wr(a=t[r]).call(a,(function(t,e){return t.center-e.center}))}this.dataChanged=!1}},{key:"_getClusterForItems",value:function(t,e,i,n){var o,s,r=Ln(o=i||[]).call(o,(function(t){var e;return{cluster:t,itemsIds:new Ar(Ln(e=t.data.uiItems).call(e,(function(t){return t.id})))}}));if(r.length){var a=!0,l=!1,h=void 0;try{for(var d,u=function(){var e=d.value;if(e.itemsIds.size===t.length&&ru(t).call(t,(function(t){return e.itemsIds.has(t.id)})))return s=e.cluster,"break"},c=Fi(r);!(a=(d=c.next()).done);a=!0){if("break"===u())break}}catch(t){l=!0,h=t}finally{try{a||null==c.return||c.return()}finally{if(l)throw h}}}if(s)return s.setUiItems(t),s.group!==e&&(s.group&&s.group.remove(s),e&&(e.add(s),s.group=e)),s;var p=n.titleTemplate||"",m={toScreen:this.itemSet.body.util.toScreen,toTime:this.itemSet.body.util.toTime},f=p.replace(/{count}/,t.length),g='<div title="'+f+'">'+t.length+"</div>",v=Ao({},n,this.itemSet.options),b={content:g,title:f,group:e,uiItems:t,eventEmitter:this.itemSet.body.emitter,range:this.itemSet.body.range};return s=this.createClusterItem(b,m,v),e&&(e.add(s),s.group=e),s.attach(),s}},{key:"_dropLevelsCache",value:function(){this.cache={},this.cacheLevel=-1,this.cache[this.cacheLevel]=[]}}]),t}();Ph('.vis-itemset{position:relative;padding:0;margin:0;box-sizing:border-box}.vis-itemset .vis-background,.vis-itemset .vis-foreground{position:absolute;width:100%;height:100%;overflow:visible}.vis-axis{position:absolute;width:100%;height:0;left:0;z-index:1}.vis-foreground .vis-group{position:relative;box-sizing:border-box;border-bottom:1px solid #bfbfbf}.vis-foreground .vis-group:last-child{border-bottom:none}.vis-nesting-group{cursor:pointer}.vis-label.vis-nested-group.vis-group-level-unknown-but-gte1{background:#f5f5f5}.vis-label.vis-nested-group.vis-group-level-0{background-color:#fff}.vis-ltr .vis-label.vis-nested-group.vis-group-level-0 .vis-inner{padding-left:0}.vis-rtl .vis-label.vis-nested-group.vis-group-level-0 .vis-inner{padding-right:0}.vis-label.vis-nested-group.vis-group-level-1{background-color:rgba(0,0,0,.05)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-1 .vis-inner{padding-left:15px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-1 .vis-inner{padding-right:15px}.vis-label.vis-nested-group.vis-group-level-2{background-color:rgba(0,0,0,.1)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-2 .vis-inner{padding-left:30px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-2 .vis-inner{padding-right:30px}.vis-label.vis-nested-group.vis-group-level-3{background-color:rgba(0,0,0,.15)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-3 .vis-inner{padding-left:45px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-3 .vis-inner{padding-right:45px}.vis-label.vis-nested-group.vis-group-level-4{background-color:rgba(0,0,0,.2)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-4 .vis-inner{padding-left:60px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-4 .vis-inner{padding-right:60px}.vis-label.vis-nested-group.vis-group-level-5{background-color:rgba(0,0,0,.25)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-5 .vis-inner{padding-left:75px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-5 .vis-inner{padding-right:75px}.vis-label.vis-nested-group.vis-group-level-6{background-color:rgba(0,0,0,.3)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-6 .vis-inner{padding-left:90px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-6 .vis-inner{padding-right:90px}.vis-label.vis-nested-group.vis-group-level-7{background-color:rgba(0,0,0,.35)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-7 .vis-inner{padding-left:105px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-7 .vis-inner{padding-right:105px}.vis-label.vis-nested-group.vis-group-level-8{background-color:rgba(0,0,0,.4)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-8 .vis-inner{padding-left:120px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-8 .vis-inner{padding-right:120px}.vis-label.vis-nested-group.vis-group-level-9{background-color:rgba(0,0,0,.45)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-9 .vis-inner{padding-left:135px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-9 .vis-inner{padding-right:135px}.vis-label.vis-nested-group{background-color:rgba(0,0,0,.5)}.vis-ltr .vis-label.vis-nested-group .vis-inner{padding-left:150px}.vis-rtl .vis-label.vis-nested-group .vis-inner{padding-right:150px}.vis-group-level-unknown-but-gte1{border:1px solid red}.vis-label.vis-nesting-group:before{display:inline-block;width:15px}.vis-label.vis-nesting-group.expanded:before{content:"\\25BC"}.vis-label.vis-nesting-group.collapsed:before{content:"\\25B6"}.vis-rtl .vis-label.vis-nesting-group.collapsed:before{content:"\\25C0"}.vis-ltr .vis-label:not(.vis-nesting-group):not(.vis-group-level-0){padding-left:15px}.vis-rtl .vis-label:not(.vis-nesting-group):not(.vis-group-level-0){padding-right:15px}.vis-overlay{position:absolute;top:0;left:0;width:100%;height:100%;z-index:10}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbIml0ZW1zZXQuY3NzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUNBLGFBQ0UsaUJBQWtCLENBQ2xCLFNBQVUsQ0FDVixRQUFTLENBRVQscUJBQ0YsQ0FFQSwwREFFRSxpQkFBa0IsQ0FDbEIsVUFBVyxDQUNYLFdBQVksQ0FDWixnQkFDRixDQUVBLFVBQ0UsaUJBQWtCLENBQ2xCLFVBQVcsQ0FDWCxRQUFTLENBQ1QsTUFBTyxDQUNQLFNBQ0YsQ0FFQSwyQkFDRSxpQkFBa0IsQ0FDbEIscUJBQXNCLENBQ3RCLCtCQUNGLENBRUEsc0NBQ0Usa0JBQ0YsQ0FFQSxtQkFDRSxjQUNGLENBRUEsNkRBQ0Usa0JBQ0YsQ0FDQSw4Q0FDRSxxQkFDRixDQUNBLGtFQUNFLGNBQ0YsQ0FDQSxrRUFDRSxlQUNGLENBQ0EsOENBQ0UsZ0NBQ0YsQ0FDQSxrRUFDRSxpQkFDRixDQUNBLGtFQUNFLGtCQUNGLENBQ0EsOENBQ0UsK0JBQ0YsQ0FDQSxrRUFDRSxpQkFDRixDQUNBLGtFQUNFLGtCQUNGLENBQ0EsOENBQ0UsZ0NBQ0YsQ0FDQSxrRUFDRSxpQkFDRixDQUNBLGtFQUNFLGtCQUNGLENBQ0EsOENBQ0UsK0JBQ0YsQ0FDQSxrRUFDRSxpQkFDRixDQUNBLGtFQUNFLGtCQUNGLENBQ0EsOENBQ0UsZ0NBQ0YsQ0FDQSxrRUFDRSxpQkFDRixDQUNBLGtFQUNFLGtCQUNGLENBQ0EsOENBQ0UsK0JBQ0YsQ0FDQSxrRUFDRSxpQkFDRixDQUNBLGtFQUNFLGtCQUNGLENBQ0EsOENBQ0UsZ0NBQ0YsQ0FDQSxrRUFDRSxrQkFDRixDQUNBLGtFQUNFLG1CQUNGLENBQ0EsOENBQ0UsK0JBQ0YsQ0FDQSxrRUFDRSxrQkFDRixDQUNBLGtFQUNFLG1CQUNGLENBQ0EsOENBQ0UsZ0NBQ0YsQ0FDQSxrRUFDRSxrQkFDRixDQUNBLGtFQUNFLG1CQUNGLENBR0EsNEJBQ0UsK0JBQ0YsQ0FDQSxnREFDRSxrQkFDRixDQUNBLGdEQUNFLG1CQUNGLENBRUEsa0NBQ0Usb0JBQ0YsQ0FHQSxvQ0FFRSxvQkFBcUIsQ0FDckIsVUFDRixDQUNBLDZDQUNFLGVBQ0YsQ0FDQSw4Q0FDRSxlQUNGLENBQ0EsdURBQ0UsZUFDRixDQUVBLG9FQUNFLGlCQUNGLENBQ0Esb0VBQ0Usa0JBQ0YsQ0FFQSxhQUNFLGlCQUFrQixDQUNsQixLQUFNLENBQ04sTUFBTyxDQUNQLFVBQVcsQ0FDWCxXQUFZLENBQ1osVUFDRiIsImZpbGUiOiJpdGVtc2V0LmNzcyIsInNvdXJjZXNDb250ZW50IjpbIlxuLnZpcy1pdGVtc2V0IHtcbiAgcG9zaXRpb246IHJlbGF0aXZlO1xuICBwYWRkaW5nOiAwO1xuICBtYXJnaW46IDA7XG5cbiAgYm94LXNpemluZzogYm9yZGVyLWJveDtcbn1cblxuLnZpcy1pdGVtc2V0IC52aXMtYmFja2dyb3VuZCxcbi52aXMtaXRlbXNldCAudmlzLWZvcmVncm91bmQge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHdpZHRoOiAxMDAlO1xuICBoZWlnaHQ6IDEwMCU7XG4gIG92ZXJmbG93OiB2aXNpYmxlO1xufVxuXG4udmlzLWF4aXMge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHdpZHRoOiAxMDAlO1xuICBoZWlnaHQ6IDA7XG4gIGxlZnQ6IDA7XG4gIHotaW5kZXg6IDE7XG59XG5cbi52aXMtZm9yZWdyb3VuZCAudmlzLWdyb3VwIHtcbiAgcG9zaXRpb246IHJlbGF0aXZlO1xuICBib3gtc2l6aW5nOiBib3JkZXItYm94O1xuICBib3JkZXItYm90dG9tOiAxcHggc29saWQgI2JmYmZiZjtcbn1cblxuLnZpcy1mb3JlZ3JvdW5kIC52aXMtZ3JvdXA6bGFzdC1jaGlsZCB7XG4gIGJvcmRlci1ib3R0b206IG5vbmU7XG59XG5cbi52aXMtbmVzdGluZy1ncm91cCB7XG4gIGN1cnNvcjogcG9pbnRlcjtcbn1cblxuLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC11bmtub3duLWJ1dC1ndGUxIHtcbiAgYmFja2dyb3VuZDogI2Y1ZjVmNTtcbn1cbi52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtMCB7XG4gIGJhY2tncm91bmQtY29sb3I6ICNmZmZmZmY7XG59XG4udmlzLWx0ciAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTAgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctbGVmdDogMDtcbn1cbi52aXMtcnRsIC52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtMCAudmlzLWlubmVyIHtcbiAgcGFkZGluZy1yaWdodDogMDtcbn1cbi52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtMSB7XG4gIGJhY2tncm91bmQtY29sb3I6IHJnYmEoMCwgMCwgMCwgMC4wNSk7XG59XG4udmlzLWx0ciAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTEgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctbGVmdDogMTVweDtcbn1cbi52aXMtcnRsIC52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtMSAudmlzLWlubmVyIHtcbiAgcGFkZGluZy1yaWdodDogMTVweDtcbn1cbi52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtMiB7XG4gIGJhY2tncm91bmQtY29sb3I6IHJnYmEoMCwgMCwgMCwgMC4xKTtcbn1cbi52aXMtbHRyIC52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtMiAudmlzLWlubmVyIHtcbiAgcGFkZGluZy1sZWZ0OiAzMHB4O1xufVxuLnZpcy1ydGwgLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC0yIC52aXMtaW5uZXIge1xuICBwYWRkaW5nLXJpZ2h0OiAzMHB4O1xufVxuLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC0zIHtcbiAgYmFja2dyb3VuZC1jb2xvcjogcmdiYSgwLCAwLCAwLCAwLjE1KTtcbn1cbi52aXMtbHRyIC52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtMyAudmlzLWlubmVyIHtcbiAgcGFkZGluZy1sZWZ0OiA0NXB4O1xufVxuLnZpcy1ydGwgLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC0zIC52aXMtaW5uZXIge1xuICBwYWRkaW5nLXJpZ2h0OiA0NXB4O1xufVxuLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC00IHtcbiAgYmFja2dyb3VuZC1jb2xvcjogcmdiYSgwLCAwLCAwLCAwLjIpO1xufVxuLnZpcy1sdHIgLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC00IC52aXMtaW5uZXIge1xuICBwYWRkaW5nLWxlZnQ6IDYwcHg7XG59XG4udmlzLXJ0bCAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTQgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctcmlnaHQ6IDYwcHg7XG59XG4udmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTUge1xuICBiYWNrZ3JvdW5kLWNvbG9yOiByZ2JhKDAsIDAsIDAsIDAuMjUpO1xufVxuLnZpcy1sdHIgLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC01IC52aXMtaW5uZXIge1xuICBwYWRkaW5nLWxlZnQ6IDc1cHg7XG59XG4udmlzLXJ0bCAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTUgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctcmlnaHQ6IDc1cHg7XG59XG4udmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTYge1xuICBiYWNrZ3JvdW5kLWNvbG9yOiByZ2JhKDAsIDAsIDAsIDAuMyk7XG59XG4udmlzLWx0ciAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTYgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctbGVmdDogOTBweDtcbn1cbi52aXMtcnRsIC52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtNiAudmlzLWlubmVyIHtcbiAgcGFkZGluZy1yaWdodDogOTBweDtcbn1cbi52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtNyB7XG4gIGJhY2tncm91bmQtY29sb3I6IHJnYmEoMCwgMCwgMCwgMC4zNSk7XG59XG4udmlzLWx0ciAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTcgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctbGVmdDogMTA1cHg7XG59XG4udmlzLXJ0bCAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTcgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctcmlnaHQ6IDEwNXB4O1xufVxuLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC04IHtcbiAgYmFja2dyb3VuZC1jb2xvcjogcmdiYSgwLCAwLCAwLCAwLjQpO1xufVxuLnZpcy1sdHIgLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC04IC52aXMtaW5uZXIge1xuICBwYWRkaW5nLWxlZnQ6IDEyMHB4O1xufVxuLnZpcy1ydGwgLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC04IC52aXMtaW5uZXIge1xuICBwYWRkaW5nLXJpZ2h0OiAxMjBweDtcbn1cbi52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtOSB7XG4gIGJhY2tncm91bmQtY29sb3I6IHJnYmEoMCwgMCwgMCwgMC40NSk7XG59XG4udmlzLWx0ciAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTkgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctbGVmdDogMTM1cHg7XG59XG4udmlzLXJ0bCAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTkgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctcmlnaHQ6IDEzNXB4O1xufVxuLyogZGVmYXVsdCB0YWtlcyBvdmVyIGJlZ2lubmluZyB3aXRoIGxldmVsLTEwICh0aGF0cyB3aHkgd2UgYWRkIC52aXMtbmVzdGVkLWdyb3VwXG4gIHRvIHRoZSBzZWxlY3RvcnMgYWJvdmUsIHRvIGhhdmUgaGlnaGVyIHNwZWNpZml0eSB0aGFuIHRoZXNlIHJ1bGVzIGZvciB0aGUgZGVmYXVsdHMpICovXG4udmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAge1xuICBiYWNrZ3JvdW5kLWNvbG9yOiByZ2JhKDAsIDAsIDAsIDAuNSk7XG59XG4udmlzLWx0ciAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctbGVmdDogMTUwcHg7XG59XG4udmlzLXJ0bCAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctcmlnaHQ6IDE1MHB4O1xufVxuXG4udmlzLWdyb3VwLWxldmVsLXVua25vd24tYnV0LWd0ZTEge1xuICBib3JkZXI6IDFweCBzb2xpZCByZWQ7XG59XG5cbi8qIGV4cGFuZGVkL2NvbGxhcHNlZCBpbmRpY2F0b3JzICovXG4udmlzLWxhYmVsLnZpcy1uZXN0aW5nLWdyb3VwOmJlZm9yZSxcbi52aXMtbGFiZWwudmlzLW5lc3RpbmctZ3JvdXA6YmVmb3JlIHtcbiAgZGlzcGxheTogaW5saW5lLWJsb2NrO1xuICB3aWR0aDogMTVweDtcbn1cbi52aXMtbGFiZWwudmlzLW5lc3RpbmctZ3JvdXAuZXhwYW5kZWQ6YmVmb3JlIHtcbiAgY29udGVudDogXCJcXDI1QkNcIjtcbn1cbi52aXMtbGFiZWwudmlzLW5lc3RpbmctZ3JvdXAuY29sbGFwc2VkOmJlZm9yZSB7XG4gIGNvbnRlbnQ6IFwiXFwyNUI2XCI7XG59XG4udmlzLXJ0bCAudmlzLWxhYmVsLnZpcy1uZXN0aW5nLWdyb3VwLmNvbGxhcHNlZDpiZWZvcmUge1xuICBjb250ZW50OiBcIlxcMjVDMFwiO1xufVxuLyogY29tcGVuc2F0ZSBtaXNzaW5nIGV4cGFuZGVkL2NvbGxhcHNlZCBpbmRpY2F0b3IsIGJ1dCBvbmx5IGF0IGxldmVscyA+IDAgKi9cbi52aXMtbHRyIC52aXMtbGFiZWw6bm90KC52aXMtbmVzdGluZy1ncm91cCk6bm90KC52aXMtZ3JvdXAtbGV2ZWwtMCkge1xuICBwYWRkaW5nLWxlZnQ6IDE1cHg7XG59XG4udmlzLXJ0bCAudmlzLWxhYmVsOm5vdCgudmlzLW5lc3RpbmctZ3JvdXApOm5vdCgudmlzLWdyb3VwLWxldmVsLTApIHtcbiAgcGFkZGluZy1yaWdodDogMTVweDtcbn1cblxuLnZpcy1vdmVybGF5IHtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICB0b3A6IDA7XG4gIGxlZnQ6IDA7XG4gIHdpZHRoOiAxMDAlO1xuICBoZWlnaHQ6IDEwMCU7XG4gIHotaW5kZXg6IDEwO1xufSJdfQ== */');Ph(".vis-labelset{overflow:hidden}.vis-labelset,.vis-labelset .vis-label{position:relative;box-sizing:border-box}.vis-labelset .vis-label{left:0;top:0;width:100%;color:#4d4d4d;border-bottom:1px solid #bfbfbf}.vis-labelset .vis-label.draggable{cursor:pointer}.vis-group-is-dragging{background:rgba(0,0,0,.1)}.vis-labelset .vis-label:last-child{border-bottom:none}.vis-labelset .vis-label .vis-inner{display:inline-block;padding:5px}.vis-labelset .vis-label .vis-inner.vis-hidden{padding:0}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbImxhYmVsc2V0LmNzcyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFDQSxjQUdFLGVBR0YsQ0FFQSx1Q0FQRSxpQkFBa0IsQ0FJbEIscUJBV0YsQ0FSQSx5QkFFRSxNQUFPLENBQ1AsS0FBTSxDQUNOLFVBQVcsQ0FDWCxhQUFjLENBTWQsK0JBSEYsQ0FNQSxtQ0FDRSxjQUNGLENBRUEsdUJBQ0UseUJBQ0YsQ0FFQSxvQ0FDRSxrQkFDRixDQUVBLG9DQUNFLG9CQUFxQixDQUNyQixXQUNGLENBRUEsK0NBQ0UsU0FDRiIsImZpbGUiOiJsYWJlbHNldC5jc3MiLCJzb3VyY2VzQ29udGVudCI6WyJcbi52aXMtbGFiZWxzZXQge1xuICBwb3NpdGlvbjogcmVsYXRpdmU7XG5cbiAgb3ZlcmZsb3c6IGhpZGRlbjtcblxuICBib3gtc2l6aW5nOiBib3JkZXItYm94O1xufVxuXG4udmlzLWxhYmVsc2V0IC52aXMtbGFiZWwge1xuICBwb3NpdGlvbjogcmVsYXRpdmU7XG4gIGxlZnQ6IDA7XG4gIHRvcDogMDtcbiAgd2lkdGg6IDEwMCU7XG4gIGNvbG9yOiAjNGQ0ZDRkO1xuXG4gIGJveC1zaXppbmc6IGJvcmRlci1ib3g7XG59XG5cbi52aXMtbGFiZWxzZXQgLnZpcy1sYWJlbCB7XG4gIGJvcmRlci1ib3R0b206IDFweCBzb2xpZCAjYmZiZmJmO1xufVxuXG4udmlzLWxhYmVsc2V0IC52aXMtbGFiZWwuZHJhZ2dhYmxlIHtcbiAgY3Vyc29yOiBwb2ludGVyO1xufVxuXG4udmlzLWdyb3VwLWlzLWRyYWdnaW5nIHtcbiAgYmFja2dyb3VuZDogcmdiYSgwLCAwLCAwLCAuMSk7XG59XG5cbi52aXMtbGFiZWxzZXQgLnZpcy1sYWJlbDpsYXN0LWNoaWxkIHtcbiAgYm9yZGVyLWJvdHRvbTogbm9uZTtcbn1cblxuLnZpcy1sYWJlbHNldCAudmlzLWxhYmVsIC52aXMtaW5uZXIge1xuICBkaXNwbGF5OiBpbmxpbmUtYmxvY2s7XG4gIHBhZGRpbmc6IDVweDtcbn1cblxuLnZpcy1sYWJlbHNldCAudmlzLWxhYmVsIC52aXMtaW5uZXIudmlzLWhpZGRlbiB7XG4gIHBhZGRpbmc6IDA7XG59XG4iXX0= */");var du="__background__",uu=function(t){function e(t,i){var n;ba(this,e),(n=ar(this,dr(e).call(this))).body=t,n.defaultOptions={type:null,orientation:{item:"bottom"},align:"auto",stack:!0,stackSubgroups:!0,groupOrderSwap:function(t,e,i){var n=e.order;e.order=t.order,t.order=n},groupOrder:"order",selectable:!0,multiselect:!1,longSelectPressTime:251,itemsAlwaysDraggable:{item:!1,range:!1},editable:{updateTime:!1,updateGroup:!1,add:!1,remove:!1,overrideItems:!1},groupEditable:{order:!1,add:!1,remove:!1},snap:Hh.snap,onDropObjectOnItem:function(t,e,i){i(e)},onAdd:function(t,e){e(t)},onUpdate:function(t,e){e(t)},onMove:function(t,e){e(t)},onRemove:function(t,e){e(t)},onMoving:function(t,e){e(t)},onAddGroup:function(t,e){e(t)},onMoveGroup:function(t,e){e(t)},onRemoveGroup:function(t,e){e(t)},margin:{item:{horizontal:10,vertical:10},axis:20},showTooltips:!0,tooltip:{followMouse:!1,overflowMethod:"flip",delay:500},tooltipOnItemUpdateTime:!1},n.options=za.extend({},n.defaultOptions),n.options.rtl=i.rtl,n.options.onTimeout=i.onTimeout,n.conversion={toScreen:t.util.toScreen,toTime:t.util.toTime},n.dom={},n.props={},n.hammer=null;var o=Bs(n);return n.itemsData=null,n.groupsData=null,n.itemsSettingTime=null,n.initialItemSetDrawn=!1,n.userContinueNotBail=null,n.sequentialSelection=!1,n.itemListeners={add:function(t,e,i){o._onAdd(e.items),o.options.cluster&&o.clusterGenerator.setItems(o.items,{applyOnChangedLevel:!1}),o.redraw()},update:function(t,e,i){o._onUpdate(e.items),o.options.cluster&&o.clusterGenerator.setItems(o.items,{applyOnChangedLevel:!1}),o.redraw()},remove:function(t,e,i){o._onRemove(e.items),o.options.cluster&&o.clusterGenerator.setItems(o.items,{applyOnChangedLevel:!1}),o.redraw()}},n.groupListeners={add:function(t,e,i){if(o._onAddGroups(e.items),o.groupsData&&o.groupsData.length>0){var n,s=o.groupsData.getDataSet();mn(n=s.get()).call(n,(function(t){if(t.nestedGroups){var e;0!=t.showNested&&(t.showNested=!0);var n=[];mn(e=t.nestedGroups).call(e,(function(e){var i=s.get(e);i&&(i.nestedInGroup=t.id,0==t.showNested&&(i.visible=!1),n=Pn(n).call(n,i))})),s.update(n,i)}}))}},update:function(t,e,i){o._onUpdateGroups(e.items)},remove:function(t,e,i){o._onRemoveGroups(e.items)}},n.items={},n.groups={},n.groupIds=[],n.selection=[],n.popup=null,n.popupTimer=null,n.touchParams={},n.groupTouchParams={group:null,isDragging:!1},n._create(),n.setOptions(i),n.clusters=[],n}return pr(e,t),xa(e,[{key:"_create",value:function(){var t,e,i,n,o,s,r,a,l,h,d,u,c,p,m,f=this,g=document.createElement("div");g.className="vis-itemset",g["vis-itemset"]=this,this.dom.frame=g;var v=document.createElement("div");v.className="vis-background",g.appendChild(v),this.dom.background=v;var b=document.createElement("div");b.className="vis-foreground",g.appendChild(b),this.dom.foreground=b;var y=document.createElement("div");y.className="vis-axis",this.dom.axis=y;var x=document.createElement("div");x.className="vis-labelset",this.dom.labelSet=x,this._updateUngrouped();var w=new Hd(du,null,this);w.show(),this.groups[du]=w,this.hammer=new Vh(this.body.dom.centerContainer),this.hammer.on("hammer.input",(function(t){t.isFirst&&f._onTouch(t)})),this.hammer.on("panstart",va(t=this._onDragStart).call(t,this)),this.hammer.on("panmove",va(e=this._onDrag).call(e,this)),this.hammer.on("panend",va(i=this._onDragEnd).call(i,this)),this.hammer.get("pan").set({threshold:5,direction:Vh.ALL}),this.hammer.on("tap",va(n=this._onSelectItem).call(n,this)),this.hammer.on("press",va(o=this._onMultiSelectItem).call(o,this)),this.hammer.on("doubletap",va(s=this._onAddItem).call(s,this)),this.options.rtl?this.groupHammer=new Vh(this.body.dom.rightContainer):this.groupHammer=new Vh(this.body.dom.leftContainer),this.groupHammer.on("tap",va(r=this._onGroupClick).call(r,this)),this.groupHammer.on("panstart",va(a=this._onGroupDragStart).call(a,this)),this.groupHammer.on("panmove",va(l=this._onGroupDrag).call(l,this)),this.groupHammer.on("panend",va(h=this._onGroupDragEnd).call(h,this)),this.groupHammer.get("pan").set({threshold:5,direction:Vh.DIRECTION_VERTICAL}),this.body.dom.centerContainer.addEventListener("mouseover",va(d=this._onMouseOver).call(d,this)),this.body.dom.centerContainer.addEventListener("mouseout",va(u=this._onMouseOut).call(u,this)),this.body.dom.centerContainer.addEventListener("mousemove",va(c=this._onMouseMove).call(c,this)),this.body.dom.centerContainer.addEventListener("contextmenu",va(p=this._onDragEnd).call(p,this)),this.body.dom.centerContainer.addEventListener("mousewheel",va(m=this._onMouseWheel).call(m,this)),this.show()}},{key:"setOptions",value:function(t){var e=this;if(t){var i,n;za.selectiveExtend(["type","rtl","align","order","stack","stackSubgroups","selectable","multiselect","sequentialSelection","multiselectPerGroup","longSelectPressTime","groupOrder","dataAttributes","template","groupTemplate","visibleFrameTemplate","hide","snap","groupOrderSwap","showTooltips","tooltip","tooltipOnItemUpdateTime","groupHeightMode","onTimeout"],this.options,t),"itemsAlwaysDraggable"in t&&("boolean"==typeof t.itemsAlwaysDraggable?(this.options.itemsAlwaysDraggable.item=t.itemsAlwaysDraggable,this.options.itemsAlwaysDraggable.range=!1):"object"===zo(t.itemsAlwaysDraggable)&&(za.selectiveExtend(["item","range"],this.options.itemsAlwaysDraggable,t.itemsAlwaysDraggable),this.options.itemsAlwaysDraggable.item||(this.options.itemsAlwaysDraggable.range=!1))),"sequentialSelection"in t&&"boolean"==typeof t.sequentialSelection&&(this.options.sequentialSelection=t.sequentialSelection),"orientation"in t&&("string"==typeof t.orientation?this.options.orientation.item="top"===t.orientation?"top":"bottom":"object"===zo(t.orientation)&&"item"in t.orientation&&(this.options.orientation.item=t.orientation.item)),"margin"in t&&("number"==typeof t.margin?(this.options.margin.axis=t.margin,this.options.margin.item.horizontal=t.margin,this.options.margin.item.vertical=t.margin):"object"===zo(t.margin)&&(za.selectiveExtend(["axis"],this.options.margin,t.margin),"item"in t.margin&&("number"==typeof t.margin.item?(this.options.margin.item.horizontal=t.margin.item,this.options.margin.item.vertical=t.margin.item):"object"===zo(t.margin.item)&&za.selectiveExtend(["horizontal","vertical"],this.options.margin.item,t.margin.item)))),mn(i=["locale","locales"]).call(i,(function(i){i in t&&(e.options[i]=t[i])})),"editable"in t&&("boolean"==typeof t.editable?(this.options.editable.updateTime=t.editable,this.options.editable.updateGroup=t.editable,this.options.editable.add=t.editable,this.options.editable.remove=t.editable,this.options.editable.overrideItems=!1):"object"===zo(t.editable)&&za.selectiveExtend(["updateTime","updateGroup","add","remove","overrideItems"],this.options.editable,t.editable)),"groupEditable"in t&&("boolean"==typeof t.groupEditable?(this.options.groupEditable.order=t.groupEditable,this.options.groupEditable.add=t.groupEditable,this.options.groupEditable.remove=t.groupEditable):"object"===zo(t.groupEditable)&&za.selectiveExtend(["order","add","remove"],this.options.groupEditable,t.groupEditable));mn(n=["onDropObjectOnItem","onAdd","onUpdate","onRemove","onMove","onMoving","onAddGroup","onMoveGroup","onRemoveGroup"]).call(n,(function(i){var n=t[i];if(n){var o;if("function"!=typeof n)throw new Error(Pn(o="option ".concat(i," must be a function ")).call(o,i,"(item, callback)"));e.options[i]=n}})),t.cluster?(Ao(this.options,{cluster:t.cluster}),this.clusterGenerator||(this.clusterGenerator=new hu(this)),this.clusterGenerator.setItems(this.items,{applyOnChangedLevel:!1}),this.markDirty({refreshItems:!0,restackGroups:!0}),this.redraw()):this.clusterGenerator?(this._detachAllClusters(),this.clusters=[],this.clusterGenerator=null,this.options.cluster=void 0,this.markDirty({refreshItems:!0,restackGroups:!0}),this.redraw()):this.markDirty()}}},{key:"markDirty",value:function(t){this.groupIds=[],t&&(t.refreshItems&&mn(za).call(za,this.items,(function(t){t.dirty=!0,t.displayed&&t.redraw()})),t.restackGroups&&mn(za).call(za,this.groups,(function(t,e){e!==du&&(t.stackDirty=!0)})))}},{key:"destroy",value:function(){this.clearPopupTimer(),this.hide(),this.setItems(null),this.setGroups(null),this.hammer&&this.hammer.destroy(),this.groupHammer&&this.groupHammer.destroy(),this.hammer=null,this.body=null,this.conversion=null}},{key:"hide",value:function(){this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame),this.dom.axis.parentNode&&this.dom.axis.parentNode.removeChild(this.dom.axis),this.dom.labelSet.parentNode&&this.dom.labelSet.parentNode.removeChild(this.dom.labelSet)}},{key:"show",value:function(){this.dom.frame.parentNode||this.body.dom.center.appendChild(this.dom.frame),this.dom.axis.parentNode||this.body.dom.backgroundVertical.appendChild(this.dom.axis),this.dom.labelSet.parentNode||(this.options.rtl?this.body.dom.right.appendChild(this.dom.labelSet):this.body.dom.left.appendChild(this.dom.labelSet))}},{key:"setPopupTimer",value:function(t){if(this.clearPopupTimer(),t){var e=this.options.tooltip.delay||"number"==typeof this.options.tooltip.delay?this.options.tooltip.delay:500;this.popupTimer=qr((function(){t.show()}),e)}}},{key:"clearPopupTimer",value:function(){null!=this.popupTimer&&(clearTimeout(this.popupTimer),this.popupTimer=null)}},{key:"setSelection",value:function(t){var e;null==t&&(t=[]),Zo(t)||(t=[t]);var i=En(e=this.selection).call(e,(function(e){return-1===Fo(t).call(t,e)})),n=!0,o=!1,s=void 0;try{for(var r,a=Fi(i);!(n=(r=a.next()).done);n=!0){var l=r.value,h=this.getItemById(l);h&&h.unselect()}}catch(t){o=!0,s=t}finally{try{n||null==a.return||a.return()}finally{if(o)throw s}}this.selection=mo(t);var d=!0,u=!1,c=void 0;try{for(var p,m=Fi(t);!(d=(p=m.next()).done);d=!0){var f=p.value,g=this.getItemById(f);g&&g.select()}}catch(t){u=!0,c=t}finally{try{d||null==m.return||m.return()}finally{if(u)throw c}}}},{key:"getSelection",value:function(){var t;return Pn(t=this.selection).call(t,[])}},{key:"getVisibleItems",value:function(){var t,e,i=this.body.range.getRange();this.options.rtl?(t=this.body.util.toScreen(i.start),e=this.body.util.toScreen(i.end)):(e=this.body.util.toScreen(i.start),t=this.body.util.toScreen(i.end));var n=[];for(var o in this.groups)if(this.groups.hasOwnProperty(o)){var s=this.groups[o],r=s.isVisible?s.visibleItems:[],a=!0,l=!1,h=void 0;try{for(var d,u=Fi(r);!(a=(d=u.next()).done);a=!0){var c=d.value;this.options.rtl?c.right<e&&c.right+c.width>t&&n.push(c.id):c.left<t&&c.left+c.width>e&&n.push(c.id)}}catch(t){l=!0,h=t}finally{try{a||null==u.return||u.return()}finally{if(l)throw h}}}return n}},{key:"getVisibleGroups",value:function(){var t=[];for(var e in this.groups){if(this.groups.hasOwnProperty(e))this.groups[e].isVisible&&t.push(e)}return t}},{key:"getItemById",value:function(t){var e;return this.items[t]||yd(e=this.clusters).call(e,(function(e){return e.id===t}))}},{key:"_deselect",value:function(t){for(var e=this.selection,i=0,n=e.length;i<n;i++)if(e[i]==t){Hr(e).call(e,i,1);break}}},{key:"redraw",value:function(){var t=this,e=this.options.margin,i=this.body.range,n=za.option.asSize,o=this.options,s=o.orientation.item,r=!1,a=this.dom.frame;this.props.top=this.body.domProps.top.height+this.body.domProps.border.top,this.options.rtl?this.props.right=this.body.domProps.right.width+this.body.domProps.border.right:this.props.left=this.body.domProps.left.width+this.body.domProps.border.left,a.className="vis-itemset",this.options.cluster&&this._clusterItems(),r=this._orderGroups()||r;var l=i.end-i.start,h=l!=this.lastVisibleInterval||this.props.width!=this.props.lastWidth,d=i.start!=this.lastRangeStart,u=o.stack!=this.lastStack,c=o.stackSubgroups!=this.lastStackSubgroups,p=h||d||u||c;this.lastVisibleInterval=l,this.lastRangeStart=i.start,this.lastStack=o.stack,this.lastStackSubgroups=o.stackSubgroups,this.props.lastWidth=this.props.width;var m=this._firstGroup(),f={item:e.item,axis:e.axis},g={item:e.item,axis:e.item.vertical/2},v=0,b=e.axis+e.item.vertical;this.groups[du].redraw(i,g,p);var y={},x=0;return mn(za).call(za,this.groups,(function(t,e){if(e!==du){var n=t==m?f:g;y[e]=t.redraw(i,n,p,!0),x=y[e].length}})),x>0&&function(){for(var e={},i=function(t){mn(za).call(za,y,(function(i,n){e[n]=i[t]()}))},n=0;n<x;n++)i(n);mn(za).call(za,t.groups,(function(t,i){if(i!==du){var n=e[i];r=n||r,v+=t.height}})),v=Math.max(v,b)}(),v=Math.max(v,b),a.style.height=n(v),this.props.width=a.offsetWidth,this.props.height=v,this.dom.axis.style.top=n("top"==s?this.body.domProps.top.height+this.body.domProps.border.top:this.body.domProps.top.height+this.body.domProps.centerContainer.height),this.options.rtl?this.dom.axis.style.right="0":this.dom.axis.style.left="0",this.hammer.get("press").set({time:this.options.longSelectPressTime}),this.initialItemSetDrawn=!0,r=this._isResized()||r}},{key:"_firstGroup",value:function(){var t="top"==this.options.orientation.item?0:this.groupIds.length-1,e=this.groupIds[t];return this.groups[e]||this.groups.__ungrouped__||null}},{key:"_updateUngrouped",value:function(){var t,e,i=this.groups.__ungrouped__;if(this.groupsData){if(i)for(e in i.dispose(),delete this.groups.__ungrouped__,this.items)if(this.items.hasOwnProperty(e)){(t=this.items[e]).parent&&t.parent.remove(t);var n=this.getGroupId(t.data),o=this.groups[n];o&&o.add(t)||t.hide()}}else if(!i){for(e in i=new jd(null,null,this),this.groups.__ungrouped__=i,this.items)this.items.hasOwnProperty(e)&&(t=this.items[e],i.add(t));i.show()}}},{key:"getLabelSet",value:function(){return this.dom.labelSet}},{key:"setItems",value:function(t){this.itemsSettingTime=new Date;var e,i=this,n=this.itemsData;if(t){if(!(t instanceof Aa||t instanceof Xa))throw new TypeError("Data must be an instance of DataSet or DataView");this.itemsData=Ua(t)}else this.itemsData=null;if(n&&(mn(za).call(za,this.itemListeners,(function(t,e){n.off(e,t)})),n.dispose(),e=n.getIds(),this._onRemove(e)),this.itemsData){var o=this.id;mn(za).call(za,this.itemListeners,(function(t,e){i.itemsData.on(e,t,o)})),e=this.itemsData.getIds(),this._onAdd(e),this._updateUngrouped()}this.body.emitter.emit("_change",{queue:!0})}},{key:"getItems",value:function(){return null!=this.itemsData?this.itemsData.rawDS:null}},{key:"setGroups",value:function(t){var e,i=this;if(this.groupsData&&(mn(za).call(za,this.groupListeners,(function(t,e){i.groupsData.off(e,t)})),e=this.groupsData.getIds(),this.groupsData=null,this._onRemoveGroups(e)),t){if(!(t instanceof Aa||t instanceof Xa))throw new TypeError("Data must be an instance of DataSet or DataView");this.groupsData=t}else this.groupsData=null;if(this.groupsData){var n,o=this.groupsData;this.groupsData instanceof Xa&&(o=this.groupsData.getDataSet()),mn(n=o.get()).call(n,(function(t){var e;t.nestedGroups&&mn(e=t.nestedGroups).call(e,(function(e){var i=o.get(e);i.nestedInGroup=t.id,0==t.showNested&&(i.visible=!1),o.update(i)}))}));var s=this.id;mn(za).call(za,this.groupListeners,(function(t,e){i.groupsData.on(e,t,s)})),e=this.groupsData.getIds(),this._onAddGroups(e)}this._updateUngrouped(),this._order(),this.options.cluster&&(this.clusterGenerator.updateData(),this._clusterItems(),this.markDirty({refreshItems:!0,restackGroups:!0})),this.body.emitter.emit("_change",{queue:!0})}},{key:"getGroups",value:function(){return this.groupsData}},{key:"removeItem",value:function(t){var e=this,i=this.itemsData.get(t);i&&this.options.onRemove(i,(function(i){i&&e.itemsData.remove(t)}))}},{key:"_getType",value:function(t){return t.type||this.options.type||(t.end?"range":"box")}},{key:"getGroupId",value:function(t){return"background"==this._getType(t)&&null==t.group?du:this.groupsData?t.group:"__ungrouped__"}},{key:"_onUpdate",value:function(t){var i=this,n=this;mn(t).call(t,(function(t){var o,s=n.itemsData.get(t),r=n.items[t],a=s?n._getType(s):null,l=e.types[a];if(r&&(l&&r instanceof l?n._updateItem(r,s):(o=r.selected,n._removeItem(r),r=null)),!r&&s){if(!l)throw new TypeError('Unknown item type "'.concat(a,'"'));(r=new l(s,n.conversion,n.options)).id=t,n._addItem(r),o&&(i.selection.push(t),r.select())}})),this._order(),this.options.cluster&&(this.clusterGenerator.setItems(this.items,{applyOnChangedLevel:!1}),this._clusterItems()),this.body.emitter.emit("_change",{queue:!0})}},{key:"_onRemove",value:function(t){var e=0,i=this;mn(t).call(t,(function(t){var n=i.items[t];n&&(e++,i._removeItem(n))})),e&&(this._order(),this.body.emitter.emit("_change",{queue:!0}))}},{key:"_order",value:function(){mn(za).call(za,this.groups,(function(t){t.order()}))}},{key:"_onUpdateGroups",value:function(t){this._onAddGroups(t)}},{key:"_onAddGroups",value:function(t){var e=this;mn(t).call(t,(function(t){var i=e.groupsData.get(t),n=e.groups[t];if(n)n.setData(i);else{if("__ungrouped__"==t||t==du)throw new Error("Illegal group id. ".concat(t," is a reserved id."));var o=Oi(e.options);for(var s in za.extend(o,{height:null}),n=new jd(t,i,e),e.groups[t]=n,e.items)if(e.items.hasOwnProperty(s)){var r=e.items[s];r.data.group==t&&n.add(r)}n.order(),n.show()}})),this.body.emitter.emit("_change",{queue:!0})}},{key:"_onRemoveGroups",value:function(t){var e=this;mn(t).call(t,(function(t){var i=e.groups[t];i&&(i.dispose(),delete e.groups[t])})),this.options.cluster&&(this.clusterGenerator.updateData(),this._clusterItems()),this.markDirty({restackGroups:!!this.options.cluster}),this.body.emitter.emit("_change",{queue:!0})}},{key:"_orderGroups",value:function(){if(this.groupsData){var t=this.groupsData.getIds({order:this.options.groupOrder});t=this._orderNestedGroups(t);var e=!za.equalArray(t,this.groupIds);if(e){var i=this.groups;mn(t).call(t,(function(t){i[t].hide()})),mn(t).call(t,(function(t){i[t].show()})),this.groupIds=t}return e}return!1}},{key:"_orderNestedGroups",value:function(t){var e=this;return function t(e,i){var n=[];return mn(i).call(i,(function(i){if(n.push(i),e.groupsData.get(i).nestedGroups){var o,s=Ln(o=e.groupsData.get({filter:function(t){return t.nestedInGroup==i},order:e.options.groupOrder})).call(o,(function(t){return t.id}));n=Pn(n).call(n,t(e,s))}})),n}(this,En(t).call(t,(function(t){return!e.groupsData.get(t).nestedInGroup})))}},{key:"_addItem",value:function(t){this.items[t.id]=t;var e=this.getGroupId(t.data),i=this.groups[e];i?i&&i.data&&i.data.showNested&&(t.groupShowing=!0):t.groupShowing=!1,i&&i.add(t)}},{key:"_updateItem",value:function(t,e){t.setData(e);var i=this.getGroupId(t.data),n=this.groups[i];n?n&&n.data&&n.data.showNested&&(t.groupShowing=!0):t.groupShowing=!1}},{key:"_removeItem",value:function(t){var e,i;t.hide(),delete this.items[t.id];var n=Fo(e=this.selection).call(e,t.id);-1!=n&&Hr(i=this.selection).call(i,n,1),t.parent&&t.parent.remove(t),null!=this.popup&&this.popup.hide()}},{key:"_constructByEndArray",value:function(t){for(var e=[],i=0;i<t.length;i++)t[i]instanceof qd&&e.push(t[i]);return e}},{key:"_onTouch",value:function(t){this.touchParams.item=this.itemFromTarget(t),this.touchParams.dragLeftItem=t.target.dragLeftItem||!1,this.touchParams.dragRightItem=t.target.dragRightItem||!1,this.touchParams.itemProps=null}},{key:"_getGroupIndex",value:function(t){for(var e=0;e<this.groupIds.length;e++)if(t==this.groupIds[e])return e}},{key:"_onDragStart",value:function(t){var e=this;if(!this.touchParams.itemIsDragging){var i,n=this.touchParams.item||null,o=this;if(n&&(n.selected||this.options.itemsAlwaysDraggable.item)){if(this.options.editable.overrideItems&&!this.options.editable.updateTime&&!this.options.editable.updateGroup)return;if(null!=n.editable&&!n.editable.updateTime&&!n.editable.updateGroup&&!this.options.editable.overrideItems)return;var s=this.touchParams.dragLeftItem,r=this.touchParams.dragRightItem;if(this.touchParams.itemIsDragging=!0,this.touchParams.selectedItem=n,s)i={item:s,initialX:t.center.x,dragLeft:!0,data:this._cloneItemData(n.data)},this.touchParams.itemProps=[i];else if(r)i={item:r,initialX:t.center.x,dragRight:!0,data:this._cloneItemData(n.data)},this.touchParams.itemProps=[i];else if(this.options.editable.add&&(t.srcEvent.ctrlKey||t.srcEvent.metaKey))this._onDragStartAddItem(t);else{this.groupIds.length<1&&this.redraw();var a=this._getGroupIndex(n.data.group),l=this.options.itemsAlwaysDraggable.item&&!n.selected?[n.id]:this.getSelection();this.touchParams.itemProps=Ln(l).call(l,(function(i){var n=o.items[i],s=o._getGroupIndex(n.data.group);return{item:n,initialX:t.center.x,groupOffset:a-s,data:e._cloneItemData(n.data)}}))}t.stopPropagation()}else this.options.editable.add&&(t.srcEvent.ctrlKey||t.srcEvent.metaKey)&&this._onDragStartAddItem(t)}}},{key:"_onDragStartAddItem",value:function(t){var e=this.options.snap||null,i=this.dom.frame.getBoundingClientRect(),n=this.options.rtl?i.right-t.center.x+10:t.center.x-i.left-10,o=this.body.util.toTime(n),s=this.body.util.getScale(),r=this.body.util.getStep(),a=e?e(o,s,r):o,l={type:"range",start:a,end:a,content:"new item"},h=Id();l[this.itemsData.idProp]=h;var d=this.groupFromTarget(t);d&&(l.group=d.groupId);var u=new qd(l,this.conversion,this.options);u.id=h,u.data=this._cloneItemData(l),this._addItem(u),this.touchParams.selectedItem=u;var c={item:u,initialX:t.center.x,data:u.data};this.options.rtl?c.dragLeft=!0:c.dragRight=!0,this.touchParams.itemProps=[c],t.stopPropagation()}},{key:"_onDrag",value:function(t){var e=this;if(null!=this.popup&&this.options.showTooltips&&!this.popup.hidden){var i=this.body.dom.centerContainer,n=i.getBoundingClientRect();this.popup.setPosition(t.center.x-n.left+i.offsetLeft,t.center.y-n.top+i.offsetTop),this.popup.show()}if(this.touchParams.itemProps){var o;t.stopPropagation();var s=this,r=this.options.snap||null,a=this.body.dom.root.offsetLeft,l=this.options.rtl?a+this.body.domProps.right.width:a+this.body.domProps.left.width,h=this.body.util.getScale(),d=this.body.util.getStep(),u=this.touchParams.selectedItem,c=(this.options.editable.overrideItems||null==u.editable)&&this.options.editable.updateGroup||!this.options.editable.overrideItems&&null!=u.editable&&u.editable.updateGroup,p=null;if(c&&u&&null!=u.data.group){var m=s.groupFromTarget(t);m&&(p=this._getGroupIndex(m.groupId))}mn(o=this.touchParams.itemProps).call(o,(function(i){var n,o,a,m,f,g=s.body.util.toTime(t.center.x-l),v=s.body.util.toTime(i.initialX-l);n=e.options.rtl?-(g-v):g-v;var b=e._cloneItemData(i.item.data);if(null==i.item.editable||i.item.editable.updateTime||i.item.editable.updateGroup||s.options.editable.overrideItems){if((e.options.editable.overrideItems||null==u.editable)&&e.options.editable.updateTime||!e.options.editable.overrideItems&&null!=u.editable&&u.editable.updateTime)if(i.dragLeft)e.options.rtl?null!=b.end&&(a=za.convert(i.data.end,"Date"),f=new Date(a.valueOf()+n),b.end=r?r(f,h,d):f):null!=b.start&&(o=za.convert(i.data.start,"Date"),m=new Date(o.valueOf()+n),b.start=r?r(m,h,d):m);else if(i.dragRight)e.options.rtl?null!=b.start&&(o=za.convert(i.data.start,"Date"),m=new Date(o.valueOf()+n),b.start=r?r(m,h,d):m):null!=b.end&&(a=za.convert(i.data.end,"Date"),f=new Date(a.valueOf()+n),b.end=r?r(f,h,d):f);else if(null!=b.start)if(o=za.convert(i.data.start,"Date").valueOf(),m=new Date(o+n),null!=b.end){var y=(a=za.convert(i.data.end,"Date")).valueOf()-o.valueOf();b.start=r?r(m,h,d):m,b.end=new Date(b.start.valueOf()+y)}else b.start=r?r(m,h,d):m;if(c&&!i.dragLeft&&!i.dragRight&&null!=p&&null!=b.group){var x=p-i.groupOffset;x=Math.max(0,x),x=Math.min(s.groupIds.length-1,x),b.group=s.groupIds[x]}b=e._cloneItemData(b),s.options.onMoving(b,(function(t){t&&i.item.setData(e._cloneItemData(t,"Date"))}))}})),this.body.emitter.emit("_change")}}},{key:"_moveToGroup",value:function(t,e){var i=this.groups[e];if(i&&i.groupId!=t.data.group){var n=t.parent;n.remove(t),n.order(),t.data.group=i.groupId,i.add(t),i.order()}}},{key:"_onDragEnd",value:function(t){var e=this;if(this.touchParams.itemIsDragging=!1,this.touchParams.itemProps){t.stopPropagation();var i=this,n=this.touchParams.itemProps;this.touchParams.itemProps=null,mn(n).call(n,(function(t){var n=t.item.id;if(null!=i.itemsData.get(n)){var o=e._cloneItemData(t.item.data);i.options.onMove(o,(function(o){o?(o[e.itemsData.idProp]=n,e.itemsData.update(o)):(t.item.setData(t.data),i.body.emitter.emit("_change"))}))}else i.options.onAdd(t.item.data,(function(e){i._removeItem(t.item),e&&i.itemsData.add(e),i.body.emitter.emit("_change")}))}))}}},{key:"_onGroupClick",value:function(t){var e=this,i=this.groupFromTarget(t);qr((function(){e.toggleGroupShowNested(i)}),1)}},{key:"toggleGroupShowNested",value:function(t){var e,i=arguments.length>1&&void 0!==arguments[1]?arguments[1]:void 0;if(t&&t.nestedGroups){var n=this.groupsData.getDataSet();t.showNested=null!=i?!!i:!t.showNested;var o=n.get(t.groupId);o.showNested=t.showNested;for(var s=t.nestedGroups,r=s;r.length>0;){var a=r;r=[];for(var l=0;l<a.length;l++){var h=n.get(a[l]);h.nestedGroups&&(r=Pn(r).call(r,h.nestedGroups))}r.length>0&&(s=Pn(s).call(s,r))}var d=Ln(e=n.get(s)).call(e,(function(t){return null==t.visible&&(t.visible=!0),t.visible=!!o.showNested,t}));n.update(Pn(d).call(d,o)),o.showNested?(za.removeClassName(t.dom.label,"collapsed"),za.addClassName(t.dom.label,"expanded")):(za.removeClassName(t.dom.label,"expanded"),za.addClassName(t.dom.label,"collapsed"))}}},{key:"toggleGroupDragClassName",value:function(t){t.dom.label.classList.toggle("vis-group-is-dragging"),t.dom.foreground.classList.toggle("vis-group-is-dragging")}},{key:"_onGroupDragStart",value:function(t){this.groupTouchParams.isDragging||this.options.groupEditable.order&&(this.groupTouchParams.group=this.groupFromTarget(t),this.groupTouchParams.group&&(t.stopPropagation(),this.groupTouchParams.isDragging=!0,this.toggleGroupDragClassName(this.groupTouchParams.group),this.groupTouchParams.originalOrder=this.groupsData.getIds({order:this.options.groupOrder})))}},{key:"_onGroupDrag",value:function(t){if(this.options.groupEditable.order&&this.groupTouchParams.group){t.stopPropagation();var e=this.groupsData;this.groupsData instanceof Xa&&(e=this.groupsData.getDataSet());var i=this.groupFromTarget(t);if(i&&i.height!=this.groupTouchParams.group.height){var n=i.top<this.groupTouchParams.group.top,o=t.center?t.center.y:t.clientY,s=i.dom.foreground.getBoundingClientRect(),r=this.groupTouchParams.group.height;if(n){if(s.top+r<o)return}else{var a=i.height;if(s.top+a-r>o)return}}if(i&&i!=this.groupTouchParams.group){var l=e.get(i.groupId),h=e.get(this.groupTouchParams.group.groupId);h&&l&&(this.options.groupOrderSwap(h,l,e),e.update(h),e.update(l));var d=e.getIds({order:this.options.groupOrder});if(!za.equalArray(d,this.groupTouchParams.originalOrder))for(var u=this.groupTouchParams.originalOrder,c=this.groupTouchParams.group.groupId,p=Math.min(u.length,d.length),m=0,f=0,g=0;m<p;){for(;m+f<p&&m+g<p&&d[m+f]==u[m+g];)m++;if(m+f>=p)break;if(d[m+f]==c)f=1;else if(u[m+g]==c)g=1;else{var v=Fo(d).call(d,u[m+g]),b=e.get(d[m+f]),y=e.get(u[m+g]);this.options.groupOrderSwap(b,y,e),e.update(b),e.update(y);var x=d[m+f];d[m+f]=u[m+g],d[v]=x,m++}}}}}},{key:"_onGroupDragEnd",value:function(t){if(this.groupTouchParams.isDragging=!1,this.options.groupEditable.order&&this.groupTouchParams.group){t.stopPropagation();var e=this,i=e.groupTouchParams.group.groupId,n=e.groupsData.getDataSet(),o=za.extend({},n.get(i));e.options.onMoveGroup(o,(function(t){if(t)t[n._idProp]=i,n.update(t);else{var o=n.getIds({order:e.options.groupOrder});if(!za.equalArray(o,e.groupTouchParams.originalOrder))for(var s=e.groupTouchParams.originalOrder,r=Math.min(s.length,o.length),a=0;a<r;){for(;a<r&&o[a]==s[a];)a++;if(a>=r)break;var l=Fo(o).call(o,s[a]),h=n.get(o[a]),d=n.get(s[a]);e.options.groupOrderSwap(h,d,n),n.update(h),n.update(d);var u=o[a];o[a]=s[a],o[l]=u,a++}}})),e.body.emitter.emit("groupDragged",{groupId:i}),this.toggleGroupDragClassName(this.groupTouchParams.group),this.groupTouchParams.group=null}}},{key:"_onSelectItem",value:function(t){if(this.options.selectable){var e=t.srcEvent&&(t.srcEvent.ctrlKey||t.srcEvent.metaKey),i=t.srcEvent&&t.srcEvent.shiftKey;if(e||i)this._onMultiSelectItem(t);else{var n=this.getSelection(),o=this.itemFromTarget(t),s=o&&o.selectable?[o.id]:[];this.setSelection(s);var r=this.getSelection();(r.length>0||n.length>0)&&this.body.emitter.emit("select",{items:r,event:t})}}}},{key:"_onMouseOver",value:function(t){var e=this.itemFromTarget(t);if(e&&e!==this.itemFromRelatedTarget(t)){var i=e.getTitle();if(this.options.showTooltips&&i){null==this.popup&&(this.popup=new tu(this.body.dom.root,this.options.tooltip.overflowMethod||"flip")),this.popup.setText(i);var n=this.body.dom.centerContainer,o=n.getBoundingClientRect();this.popup.setPosition(t.clientX-o.left+n.offsetLeft,t.clientY-o.top+n.offsetTop),this.setPopupTimer(this.popup)}else this.clearPopupTimer(),null!=this.popup&&this.popup.hide();this.body.emitter.emit("itemover",{item:e.id,event:t})}}},{key:"_onMouseOut",value:function(t){var e=this.itemFromTarget(t);e&&(e!==this.itemFromRelatedTarget(t)&&(this.clearPopupTimer(),null!=this.popup&&this.popup.hide(),this.body.emitter.emit("itemout",{item:e.id,event:t})))}},{key:"_onMouseMove",value:function(t){if(this.itemFromTarget(t)&&(null!=this.popupTimer&&this.setPopupTimer(this.popup),this.options.showTooltips&&this.options.tooltip.followMouse&&this.popup&&!this.popup.hidden)){var e=this.body.dom.centerContainer,i=e.getBoundingClientRect();this.popup.setPosition(t.clientX-i.left+e.offsetLeft,t.clientY-i.top+e.offsetTop),this.popup.show()}}},{key:"_onMouseWheel",value:function(t){this.touchParams.itemIsDragging&&this._onDragEnd(t)}},{key:"_onUpdateItem",value:function(t){if(this.options.selectable&&(this.options.editable.updateTime||this.options.editable.updateGroup)){var e=this;if(t){var i=e.itemsData.get(t.id);this.options.onUpdate(i,(function(t){t&&e.itemsData.update(t)}))}}}},{key:"_onDropObjectOnItem",value:function(t){var e=this.itemFromTarget(t),i=JSON.parse(t.dataTransfer.getData("text"));this.options.onDropObjectOnItem(i,e)}},{key:"_onAddItem",value:function(t){if(this.options.selectable&&this.options.editable.add){var e,i,n=this,o=this.options.snap||null,s=this.dom.frame.getBoundingClientRect(),r=this.options.rtl?s.right-t.center.x:t.center.x-s.left,a=this.body.util.toTime(r),l=this.body.util.getScale(),h=this.body.util.getStep();"drop"==t.type?((i=JSON.parse(t.dataTransfer.getData("text"))).content=i.content?i.content:"new item",i.start=i.start?i.start:o?o(a,l,h):a,i.type=i.type||"box",i[this.itemsData.idProp]=i.id||Id(),"range"!=i.type||i.end||(e=this.body.util.toTime(r+this.props.width/5),i.end=o?o(e,l,h):e)):((i={start:o?o(a,l,h):a,content:"new item"})[this.itemsData.idProp]=Id(),"range"===this.options.type&&(e=this.body.util.toTime(r+this.props.width/5),i.end=o?o(e,l,h):e));var d=this.groupFromTarget(t);d&&(i.group=d.groupId),i=this._cloneItemData(i),this.options.onAdd(i,(function(e){e&&(n.itemsData.add(e),"drop"==t.type&&n.setSelection([e.id]))}))}}},{key:"_onMultiSelectItem",value:function(t){var i=this;if(this.options.selectable){var n=this.itemFromTarget(t);if(n){var o=this.options.multiselect?this.getSelection():[];if((t.srcEvent&&t.srcEvent.shiftKey||!1||this.options.sequentialSelection)&&this.options.multiselect){var s=this.itemsData.get(n.id).group,r=void 0;this.options.multiselectPerGroup&&o.length>0&&(r=this.itemsData.get(o[0]).group),this.options.multiselectPerGroup&&null!=r&&r!=s||o.push(n.id);var a=e._getItemRange(this.itemsData.get(o));if(!this.options.multiselectPerGroup||r==s)for(var l in o=[],this.items)if(this.items.hasOwnProperty(l)){var h=this.items[l],d=h.data.start,u=void 0!==h.data.end?h.data.end:d;!(d>=a.min&&u<=a.max)||this.options.multiselectPerGroup&&r!=this.itemsData.get(h.id).group||h instanceof $d||o.push(h.id)}}else{var c=Fo(o).call(o,n.id);-1==c?o.push(n.id):Hr(o).call(o,c,1)}var p=En(o).call(o,(function(t){return i.getItemById(t).selectable}));this.setSelection(p),this.body.emitter.emit("select",{items:this.getSelection(),event:t})}}}},{key:"itemFromElement",value:function(t){for(var e=t;e;){if(e.hasOwnProperty("vis-item"))return e["vis-item"];e=e.parentNode}return null}},{key:"itemFromTarget",value:function(t){return this.itemFromElement(t.target)}},{key:"itemFromRelatedTarget",value:function(t){return this.itemFromElement(t.relatedTarget)}},{key:"groupFromTarget",value:function(t){var e=t.center?t.center.y:t.clientY,i=this.groupIds;i.length<=0&&this.groupsData&&(i=this.groupsData.getIds({order:this.options.groupOrder}));for(var n=0;n<i.length;n++){var o=i[n],s=this.groups[o],r=s.dom.foreground,a=r.getBoundingClientRect();if(e>=a.top&&e<a.top+r.offsetHeight)return s;if("top"===this.options.orientation.item){if(n===this.groupIds.length-1&&e>a.top)return s}else if(0===n&&e<a.top+r.offset)return s}return null}},{key:"_cloneItemData",value:function(t,e){var i=za.extend({},t);return e||(e=this.itemsData.type),null!=i.start&&(i.start=za.convert(i.start,e&&e.start||"Date")),null!=i.end&&(i.end=za.convert(i.end,e&&e.end||"Date")),i}},{key:"_clusterItems",value:function(){if(this.options.cluster){var t=this.body.range.conversion(this.body.domProps.center.width).scale,e=this.clusterGenerator.getClusters(this.clusters,t,this.options.cluster);if(this.clusters!=e){if(this._detachAllClusters(),e){var i=!0,n=!1,o=void 0;try{for(var s,r=Fi(e);!(i=(s=r.next()).done);i=!0){s.value.attach()}}catch(t){n=!0,o=t}finally{try{i||null==r.return||r.return()}finally{if(n)throw o}}this.clusters=e}this._updateClusters(e)}}}},{key:"_detachAllClusters",value:function(){if(this.options.cluster&&this.clusters&&this.clusters.length){var t=!0,e=!1,i=void 0;try{for(var n,o=Fi(this.clusters);!(t=(n=o.next()).done);t=!0){n.value.detach()}}catch(t){e=!0,i=t}finally{try{t||null==o.return||o.return()}finally{if(e)throw i}}}}},{key:"_updateClusters",value:function(t){if(this.clusters&&this.clusters.length){var e,i=new Ar(Ln(t).call(t,(function(t){return t.id}))),n=En(e=this.clusters).call(e,(function(t){return!i.has(t.id)})),o=!1,s=!0,r=!1,a=void 0;try{for(var l,h=Fi(n);!(s=(l=h.next()).done);s=!0){var d,u,c=l.value,p=Fo(d=this.selection).call(d,c.id);if(-1!==p)c.unselect(),Hr(u=this.selection).call(u,p,1),o=!0}}catch(t){r=!0,a=t}finally{try{s||null==h.return||h.return()}finally{if(r)throw a}}if(o){var m=this.getSelection();this.body.emitter.emit("select",{items:m,event:event})}}this.clusters=t||[]}}],[{key:"_getItemRange",value:function(t){var e=null,i=null;return mn(t).call(t,(function(t){(null==i||t.start<i)&&(i=t.start),null!=t.end?(null==e||t.end>e)&&(e=t.end):(null==e||t.start>e)&&(e=t.start)})),{min:i,max:e}}},{key:"itemSetFromTarget",value:function(t){for(var e=t.target;e;){if(e.hasOwnProperty("vis-itemset"))return e["vis-itemset"];e=e.parentNode}return null}}]),e}(Ja);uu.types={background:$d,box:Jd,range:qd,point:Kd},uu.prototype._onAdd=uu.prototype._onUpdate;var cu,pu=!1,mu="background: #FFeeee; color: #dd0000",fu=function(){function t(){ba(this,t)}return xa(t,null,[{key:"validate",value:function(e,i,n){pu=!1,cu=i;var o=i;return void 0!==n&&(o=i[n]),t.parse(e,o,[]),pu}},{key:"parse",value:function(e,i,n){for(var o in e)e.hasOwnProperty(o)&&t.check(o,e,i,n)}},{key:"check",value:function(e,i,n,o){if(void 0!==n[e]||void 0!==n.__any__){var s=e,r=!0;void 0===n[e]&&void 0!==n.__any__&&(s="__any__",r="object"===t.getType(i[e]));var a=n[s];r&&void 0!==a.__type__&&(a=a.__type__),t.checkFields(e,i,n,s,a,o)}else t.getSuggestion(e,n,o)}},{key:"checkFields",value:function(e,i,n,o,s,r){var a=function(i){console.log("%c"+i+t.printLocation(r,e),mu)},l=t.getType(i[e]),h=s[l];void 0!==h?"array"===t.getType(h)&&-1===Fo(h).call(h,i[e])?(a('Invalid option detected in "'+e+'". Allowed values are:'+t.print(h)+' not "'+i[e]+'". '),pu=!0):"object"===l&&"__any__"!==o&&(r=za.copyAndExtendArray(r,e),t.parse(i[e],n[o],r)):void 0===s.any&&(a('Invalid type received for "'+e+'". Expected: '+t.print(Yi(s))+". Received ["+l+'] "'+i[e]+'"'),pu=!0)}},{key:"getType",value:function(t){var e=zo(t);return"object"===e?null===t?"null":t instanceof Boolean?"boolean":t instanceof Number?"number":t instanceof String?"string":Zo(t)?"array":t instanceof Date?"date":void 0!==t.nodeType?"dom":!0===t._isAMomentObject?"moment":"object":"number"===e?"number":"boolean"===e?"boolean":"string"===e?"string":void 0===e?"undefined":e}},{key:"getSuggestion",value:function(e,i,n){var o,s=t.findInOptions(e,i,n,!1),r=t.findInOptions(e,cu,[],!0);o=void 0!==s.indexMatch?" in "+t.printLocation(s.path,e,"")+'Perhaps it was incomplete? Did you mean: "'+s.indexMatch+'"?\n\n':r.distance<=4&&s.distance>r.distance?" in "+t.printLocation(s.path,e,"")+"Perhaps it was misplaced? Matching option found at: "+t.printLocation(r.path,r.closestMatch,""):s.distance<=8?'. Did you mean "'+s.closestMatch+'"?'+t.printLocation(s.path,e):". Did you mean one of these: "+t.print(Yi(i))+t.printLocation(n,e),console.log('%cUnknown option detected: "'+e+'"'+o,mu),pu=!0}},{key:"findInOptions",value:function(e,i,n){var o=arguments.length>3&&void 0!==arguments[3]&&arguments[3],s=1e9,r="",a=[],l=e.toLowerCase(),h=void 0;for(var d in i){var u=void 0;if(void 0!==i[d].__type__&&!0===o){var c=t.findInOptions(e,i[d],za.copyAndExtendArray(n,d));s>c.distance&&(r=c.closestMatch,a=c.path,s=c.distance,h=c.indexMatch)}else{var p;-1!==Fo(p=d.toLowerCase()).call(p,l)&&(h=d),s>(u=t.levenshteinDistance(e,d))&&(r=d,a=za.copyArray(n),s=u)}}return{closestMatch:r,path:a,distance:s,indexMatch:h}}},{key:"printLocation",value:function(t,e){for(var i=arguments.length>2&&void 0!==arguments[2]?arguments[2]:"Problem value found at: \n",n="\n\n"+i+"options = {\n",o=0;o<t.length;o++){for(var s=0;s<o+1;s++)n+="  ";n+=t[o]+": {\n"}for(var r=0;r<t.length+1;r++)n+="  ";n+=e+"\n";for(var a=0;a<t.length+1;a++){for(var l=0;l<t.length-a;l++)n+="  ";n+="}\n"}return n+"\n\n"}},{key:"print",value:function(t){return Vs(t).replace(/(\")|(\[)|(\])|(,"__type__")/g,"").replace(/(\,)/g,", ")}},{key:"levenshteinDistance",value:function(t,e){if(0===t.length)return e.length;if(0===e.length)return t.length;var i,n,o=[];for(i=0;i<=e.length;i++)o[i]=[i];for(n=0;n<=t.length;n++)o[0][n]=n;for(i=1;i<=e.length;i++)for(n=1;n<=t.length;n++)e.charAt(i-1)==t.charAt(n-1)?o[i][n]=o[i-1][n-1]:o[i][n]=Math.min(o[i-1][n-1]+1,Math.min(o[i][n-1]+1,o[i-1][n]+1));return o[e.length][t.length]}}]),t}(),gu="string",vu="boolean",bu="number",yu="object",xu={configure:{enabled:{boolean:vu},filter:{boolean:vu,function:"function"},container:{dom:"dom"},__type__:{object:yu,boolean:vu,function:"function"}},align:{string:gu},alignCurrentTime:{string:gu,undefined:"undefined"},rtl:{boolean:vu,undefined:"undefined"},rollingMode:{follow:{boolean:vu},offset:{number:bu,undefined:"undefined"},__type__:{object:yu}},onTimeout:{timeoutMs:{number:bu},callback:{function:"function"},__type__:{object:yu}},verticalScroll:{boolean:vu,undefined:"undefined"},horizontalScroll:{boolean:vu,undefined:"undefined"},autoResize:{boolean:vu},throttleRedraw:{number:bu},clickToUse:{boolean:vu},dataAttributes:{string:gu,array:"array"},editable:{add:{boolean:vu,undefined:"undefined"},remove:{boolean:vu,undefined:"undefined"},updateGroup:{boolean:vu,undefined:"undefined"},updateTime:{boolean:vu,undefined:"undefined"},overrideItems:{boolean:vu,undefined:"undefined"},__type__:{boolean:vu,object:yu}},end:{number:bu,date:"date",string:gu,moment:"moment"},format:{minorLabels:{millisecond:{string:gu,undefined:"undefined"},second:{string:gu,undefined:"undefined"},minute:{string:gu,undefined:"undefined"},hour:{string:gu,undefined:"undefined"},weekday:{string:gu,undefined:"undefined"},day:{string:gu,undefined:"undefined"},week:{string:gu,undefined:"undefined"},month:{string:gu,undefined:"undefined"},year:{string:gu,undefined:"undefined"},__type__:{object:yu,function:"function"}},majorLabels:{millisecond:{string:gu,undefined:"undefined"},second:{string:gu,undefined:"undefined"},minute:{string:gu,undefined:"undefined"},hour:{string:gu,undefined:"undefined"},weekday:{string:gu,undefined:"undefined"},day:{string:gu,undefined:"undefined"},week:{string:gu,undefined:"undefined"},month:{string:gu,undefined:"undefined"},year:{string:gu,undefined:"undefined"},__type__:{object:yu,function:"function"}},__type__:{object:yu}},moment:{function:"function"},groupHeightMode:{string:gu},groupOrder:{string:gu,function:"function"},groupEditable:{add:{boolean:vu,undefined:"undefined"},remove:{boolean:vu,undefined:"undefined"},order:{boolean:vu,undefined:"undefined"},__type__:{boolean:vu,object:yu}},groupOrderSwap:{function:"function"},height:{string:gu,number:bu},hiddenDates:{start:{date:"date",number:bu,string:gu,moment:"moment"},end:{date:"date",number:bu,string:gu,moment:"moment"},repeat:{string:gu},__type__:{object:yu,array:"array"}},itemsAlwaysDraggable:{item:{boolean:vu,undefined:"undefined"},range:{boolean:vu,undefined:"undefined"},__type__:{boolean:vu,object:yu}},limitSize:{boolean:vu},locale:{string:gu},locales:{__any__:{any:"any"},__type__:{object:yu}},longSelectPressTime:{number:bu},margin:{axis:{number:bu},item:{horizontal:{number:bu,undefined:"undefined"},vertical:{number:bu,undefined:"undefined"},__type__:{object:yu,number:bu}},__type__:{object:yu,number:bu}},max:{date:"date",number:bu,string:gu,moment:"moment"},maxHeight:{number:bu,string:gu},maxMinorChars:{number:bu},min:{date:"date",number:bu,string:gu,moment:"moment"},minHeight:{number:bu,string:gu},moveable:{boolean:vu},multiselect:{boolean:vu},multiselectPerGroup:{boolean:vu},onAdd:{function:"function"},onDropObjectOnItem:{function:"function"},onUpdate:{function:"function"},onMove:{function:"function"},onMoving:{function:"function"},onRemove:{function:"function"},onAddGroup:{function:"function"},onMoveGroup:{function:"function"},onRemoveGroup:{function:"function"},onInitialDrawComplete:{function:"function"},order:{function:"function"},orientation:{axis:{string:gu,undefined:"undefined"},item:{string:gu,undefined:"undefined"},__type__:{string:gu,object:yu}},selectable:{boolean:vu},sequentialSelection:{boolean:vu},showCurrentTime:{boolean:vu},showMajorLabels:{boolean:vu},showMinorLabels:{boolean:vu},showWeekScale:{boolean:vu},stack:{boolean:vu},stackSubgroups:{boolean:vu},cluster:{maxItems:{number:bu,undefined:"undefined"},titleTemplate:{string:gu,undefined:"undefined"},clusterCriteria:{function:"function",undefined:"undefined"},showStipes:{boolean:vu,undefined:"undefined"},fitOnDoubleClick:{boolean:vu,undefined:"undefined"},__type__:{boolean:vu,object:yu}},snap:{function:"function",null:"null"},start:{date:"date",number:bu,string:gu,moment:"moment"},template:{function:"function"},loadingScreenTemplate:{function:"function"},groupTemplate:{function:"function"},visibleFrameTemplate:{string:gu,function:"function"},showTooltips:{boolean:vu},tooltip:{followMouse:{boolean:vu},overflowMethod:{string:["cap","flip","none"]},delay:{number:bu},template:{function:"function"},__type__:{object:yu}},tooltipOnItemUpdateTime:{template:{function:"function"},__type__:{boolean:vu,object:yu}},timeAxis:{scale:{string:gu,undefined:"undefined"},step:{number:bu,undefined:"undefined"},__type__:{object:yu}},type:{string:gu},width:{string:gu,number:bu},preferZoom:{boolean:vu},zoomable:{boolean:vu},zoomKey:{string:["ctrlKey","altKey","shiftKey","metaKey",""]},zoomFriction:{number:bu},zoomMax:{number:bu},zoomMin:{number:bu},__type__:{object:yu}},wu={global:{align:["center","left","right"],alignCurrentTime:["none","year","month","quarter","week","isoWeek","day","date","hour","minute","second"],direction:!1,autoResize:!0,clickToUse:!1,editable:{add:!1,remove:!1,updateGroup:!1,updateTime:!1},end:"",format:{minorLabels:{millisecond:"SSS",second:"s",minute:"HH:mm",hour:"HH:mm",weekday:"ddd D",day:"D",week:"w",month:"MMM",year:"YYYY"},majorLabels:{millisecond:"HH:mm:ss",second:"D MMMM HH:mm",minute:"ddd D MMMM",hour:"ddd D MMMM",weekday:"MMMM YYYY",day:"MMMM YYYY",week:"MMMM YYYY",month:"YYYY",year:""}},groupHeightMode:["auto","fixed","fitItems"],groupsDraggable:!1,height:"",locale:"",longSelectPressTime:251,margin:{axis:[20,0,100,1],item:{horizontal:[10,0,100,1],vertical:[10,0,100,1]}},max:"",maxHeight:"",maxMinorChars:[7,0,20,1],min:"",minHeight:"",moveable:!1,multiselect:!1,multiselectPerGroup:!1,orientation:{axis:["both","bottom","top"],item:["bottom","top"]},preferZoom:!1,selectable:!0,showCurrentTime:!1,showMajorLabels:!0,showMinorLabels:!0,stack:!0,stackSubgroups:!0,cluster:!1,start:"",showTooltips:!0,tooltip:{followMouse:!1,overflowMethod:"flip",delay:[500,0,99999,100]},tooltipOnItemUpdateTime:!1,type:["box","point","range","background"],width:"100%",zoomable:!0,zoomKey:["ctrlKey","altKey","shiftKey","metaKey",""],zoomMax:[31536e10,10,31536e10,1],zoomMin:[10,10,31536e10,1]}};V({target:"Array",proto:!0},{fill:function(t){for(var e=Lt(this),i=$(e.length),n=arguments.length,o=it(n>1?arguments[1]:void 0,i),s=n>2?arguments[2]:void 0,r=void 0===s?i:it(s,i);r>o;)e[o++]=t;return e}});var _u=Pi("Array").fill,ku=Array.prototype,Du=function(t){var e=t.fill;return t===ku||t instanceof Array&&e===ku.fill?_u:e},Iu={black:"#000000",navy:"#000080",darkblue:"#00008B",mediumblue:"#0000CD",blue:"#0000FF",darkgreen:"#006400",green:"#008000",teal:"#008080",darkcyan:"#008B8B",deepskyblue:"#00BFFF",darkturquoise:"#00CED1",mediumspringgreen:"#00FA9A",lime:"#00FF00",springgreen:"#00FF7F",aqua:"#00FFFF",cyan:"#00FFFF",midnightblue:"#191970",dodgerblue:"#1E90FF",lightseagreen:"#20B2AA",forestgreen:"#228B22",seagreen:"#2E8B57",darkslategray:"#2F4F4F",limegreen:"#32CD32",mediumseagreen:"#3CB371",turquoise:"#40E0D0",royalblue:"#4169E1",steelblue:"#4682B4",darkslateblue:"#483D8B",mediumturquoise:"#48D1CC",indigo:"#4B0082",darkolivegreen:"#556B2F",cadetblue:"#5F9EA0",cornflowerblue:"#6495ED",mediumaquamarine:"#66CDAA",dimgray:"#696969",slateblue:"#6A5ACD",olivedrab:"#6B8E23",slategray:"#708090",lightslategray:"#778899",mediumslateblue:"#7B68EE",lawngreen:"#7CFC00",chartreuse:"#7FFF00",aquamarine:"#7FFFD4",maroon:"#800000",purple:"#800080",olive:"#808000",gray:"#808080",skyblue:"#87CEEB",lightskyblue:"#87CEFA",blueviolet:"#8A2BE2",darkred:"#8B0000",darkmagenta:"#8B008B",saddlebrown:"#8B4513",darkseagreen:"#8FBC8F",lightgreen:"#90EE90",mediumpurple:"#9370D8",darkviolet:"#9400D3",palegreen:"#98FB98",darkorchid:"#9932CC",yellowgreen:"#9ACD32",sienna:"#A0522D",brown:"#A52A2A",darkgray:"#A9A9A9",lightblue:"#ADD8E6",greenyellow:"#ADFF2F",paleturquoise:"#AFEEEE",lightsteelblue:"#B0C4DE",powderblue:"#B0E0E6",firebrick:"#B22222",darkgoldenrod:"#B8860B",mediumorchid:"#BA55D3",rosybrown:"#BC8F8F",darkkhaki:"#BDB76B",silver:"#C0C0C0",mediumvioletred:"#C71585",indianred:"#CD5C5C",peru:"#CD853F",chocolate:"#D2691E",tan:"#D2B48C",lightgrey:"#D3D3D3",palevioletred:"#D87093",thistle:"#D8BFD8",orchid:"#DA70D6",goldenrod:"#DAA520",crimson:"#DC143C",gainsboro:"#DCDCDC",plum:"#DDA0DD",burlywood:"#DEB887",lightcyan:"#E0FFFF",lavender:"#E6E6FA",darksalmon:"#E9967A",violet:"#EE82EE",palegoldenrod:"#EEE8AA",lightcoral:"#F08080",khaki:"#F0E68C",aliceblue:"#F0F8FF",honeydew:"#F0FFF0",azure:"#F0FFFF",sandybrown:"#F4A460",wheat:"#F5DEB3",beige:"#F5F5DC",whitesmoke:"#F5F5F5",mintcream:"#F5FFFA",ghostwhite:"#F8F8FF",salmon:"#FA8072",antiquewhite:"#FAEBD7",linen:"#FAF0E6",lightgoldenrodyellow:"#FAFAD2",oldlace:"#FDF5E6",red:"#FF0000",fuchsia:"#FF00FF",magenta:"#FF00FF",deeppink:"#FF1493",orangered:"#FF4500",tomato:"#FF6347",hotpink:"#FF69B4",coral:"#FF7F50",darkorange:"#FF8C00",lightsalmon:"#FFA07A",orange:"#FFA500",lightpink:"#FFB6C1",pink:"#FFC0CB",gold:"#FFD700",peachpuff:"#FFDAB9",navajowhite:"#FFDEAD",moccasin:"#FFE4B5",bisque:"#FFE4C4",mistyrose:"#FFE4E1",blanchedalmond:"#FFEBCD",papayawhip:"#FFEFD5",lavenderblush:"#FFF0F5",seashell:"#FFF5EE",cornsilk:"#FFF8DC",lemonchiffon:"#FFFACD",floralwhite:"#FFFAF0",snow:"#FFFAFA",yellow:"#FFFF00",lightyellow:"#FFFFE0",ivory:"#FFFFF0",white:"#FFFFFF"},Cu=function(){function t(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:1;ba(this,t),this.pixelRatio=e,this.generated=!1,this.centerCoordinates={x:144.5,y:144.5},this.r=289*.49,this.color={r:255,g:255,b:255,a:1},this.hueCircle=void 0,this.initialColor={r:255,g:255,b:255,a:1},this.previousColor=void 0,this.applied=!1,this.updateCallback=function(){},this.closeCallback=function(){},this._create()}return xa(t,[{key:"insertTo",value:function(t){void 0!==this.hammer&&(this.hammer.destroy(),this.hammer=void 0),this.container=t,this.container.appendChild(this.frame),this._bindHammer(),this._setSize()}},{key:"setUpdateCallback",value:function(t){if("function"!=typeof t)throw new Error("Function attempted to set as colorPicker update callback is not a function.");this.updateCallback=t}},{key:"setCloseCallback",value:function(t){if("function"!=typeof t)throw new Error("Function attempted to set as colorPicker closing callback is not a function.");this.closeCallback=t}},{key:"_isColorString",value:function(t){if("string"==typeof t)return Iu[t]}},{key:"setColor",value:function(t){var e=!(arguments.length>1&&void 0!==arguments[1])||arguments[1];if("none"!==t){var i,n=this._isColorString(t);if(void 0!==n&&(t=n),!0===za.isString(t)){if(!0===za.isValidRGB(t)){var o=t.substr(4).substr(0,t.length-5).split(",");i={r:o[0],g:o[1],b:o[2],a:1}}else if(!0===za.isValidRGBA(t)){var s=t.substr(5).substr(0,t.length-6).split(",");i={r:s[0],g:s[1],b:s[2],a:s[3]}}else if(!0===za.isValidHex(t)){var r=za.hexToRGB(t);i={r:r.r,g:r.g,b:r.b,a:1}}}else if(t instanceof Object&&void 0!==t.r&&void 0!==t.g&&void 0!==t.b){var a=void 0!==t.a?t.a:"1.0";i={r:t.r,g:t.g,b:t.b,a:a}}if(void 0===i)throw new Error("Unknown color passed to the colorPicker. Supported are strings: rgb, hex, rgba. Object: rgb ({r:r,g:g,b:b,[a:a]}). Supplied: "+Vs(t));this._setColor(i,e)}}},{key:"show",value:function(){void 0!==this.closeCallback&&(this.closeCallback(),this.closeCallback=void 0),this.applied=!1,this.frame.style.display="block",this._generateHueCircle()}},{key:"_hide",value:function(){var t=this,e=!(arguments.length>0&&void 0!==arguments[0])||arguments[0];!0===e&&(this.previousColor=za.extend({},this.color)),!0===this.applied&&this.updateCallback(this.initialColor),this.frame.style.display="none",qr((function(){void 0!==t.closeCallback&&(t.closeCallback(),t.closeCallback=void 0)}),0)}},{key:"_save",value:function(){this.updateCallback(this.color),this.applied=!1,this._hide()}},{key:"_apply",value:function(){this.applied=!0,this.updateCallback(this.color),this._updatePicker(this.color)}},{key:"_loadLast",value:function(){void 0!==this.previousColor?this.setColor(this.previousColor,!1):alert("There is no last color to load...")}},{key:"_setColor",value:function(t){var e=!(arguments.length>1&&void 0!==arguments[1])||arguments[1];!0===e&&(this.initialColor=za.extend({},t)),this.color=t;var i=za.RGBToHSV(t.r,t.g,t.b),n=2*Math.PI,o=this.r*i.s,s=this.centerCoordinates.x+o*Math.sin(n*i.h),r=this.centerCoordinates.y+o*Math.cos(n*i.h);this.colorPickerSelector.style.left=s-.5*this.colorPickerSelector.clientWidth+"px",this.colorPickerSelector.style.top=r-.5*this.colorPickerSelector.clientHeight+"px",this._updatePicker(t)}},{key:"_setOpacity",value:function(t){this.color.a=t/100,this._updatePicker(this.color)}},{key:"_setBrightness",value:function(t){var e=za.RGBToHSV(this.color.r,this.color.g,this.color.b);e.v=t/100;var i=za.HSVToRGB(e.h,e.s,e.v);i.a=this.color.a,this.color=i,this._updatePicker()}},{key:"_updatePicker",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:this.color,e=za.RGBToHSV(t.r,t.g,t.b),i=this.colorPickerCanvas.getContext("2d");void 0===this.pixelRation&&(this.pixelRatio=(window.devicePixelRatio||1)/(i.webkitBackingStorePixelRatio||i.mozBackingStorePixelRatio||i.msBackingStorePixelRatio||i.oBackingStorePixelRatio||i.backingStorePixelRatio||1)),i.setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0);var n=this.colorPickerCanvas.clientWidth,o=this.colorPickerCanvas.clientHeight;i.clearRect(0,0,n,o),i.putImageData(this.hueCircle,0,0),i.fillStyle="rgba(0,0,0,"+(1-e.v)+")",i.circle(this.centerCoordinates.x,this.centerCoordinates.y,this.r),Du(i).call(i),this.brightnessRange.value=100*e.v,this.opacityRange.value=100*t.a,this.initialColorDiv.style.backgroundColor="rgba("+this.initialColor.r+","+this.initialColor.g+","+this.initialColor.b+","+this.initialColor.a+")",this.newColorDiv.style.backgroundColor="rgba("+this.color.r+","+this.color.g+","+this.color.b+","+this.color.a+")"}},{key:"_setSize",value:function(){this.colorPickerCanvas.style.width="100%",this.colorPickerCanvas.style.height="100%",this.colorPickerCanvas.width=289*this.pixelRatio,this.colorPickerCanvas.height=289*this.pixelRatio}},{key:"_create",value:function(){var t,e,i,n;if(this.frame=document.createElement("div"),this.frame.className="vis-color-picker",this.colorPickerDiv=document.createElement("div"),this.colorPickerSelector=document.createElement("div"),this.colorPickerSelector.className="vis-selector",this.colorPickerDiv.appendChild(this.colorPickerSelector),this.colorPickerCanvas=document.createElement("canvas"),this.colorPickerDiv.appendChild(this.colorPickerCanvas),this.colorPickerCanvas.getContext){var o=this.colorPickerCanvas.getContext("2d");this.pixelRatio=(window.devicePixelRatio||1)/(o.webkitBackingStorePixelRatio||o.mozBackingStorePixelRatio||o.msBackingStorePixelRatio||o.oBackingStorePixelRatio||o.backingStorePixelRatio||1),this.colorPickerCanvas.getContext("2d").setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0)}else{var s=document.createElement("DIV");s.style.color="red",s.style.fontWeight="bold",s.style.padding="10px",s.innerHTML="Error: your browser does not support HTML canvas",this.colorPickerCanvas.appendChild(s)}this.colorPickerDiv.className="vis-color",this.opacityDiv=document.createElement("div"),this.opacityDiv.className="vis-opacity",this.brightnessDiv=document.createElement("div"),this.brightnessDiv.className="vis-brightness",this.arrowDiv=document.createElement("div"),this.arrowDiv.className="vis-arrow",this.opacityRange=document.createElement("input");try{this.opacityRange.type="range",this.opacityRange.min="0",this.opacityRange.max="100"}catch(t){}this.opacityRange.value="100",this.opacityRange.className="vis-range",this.brightnessRange=document.createElement("input");try{this.brightnessRange.type="range",this.brightnessRange.min="0",this.brightnessRange.max="100"}catch(t){}this.brightnessRange.value="100",this.brightnessRange.className="vis-range",this.opacityDiv.appendChild(this.opacityRange),this.brightnessDiv.appendChild(this.brightnessRange);var r=this;this.opacityRange.onchange=function(){r._setOpacity(this.value)},this.opacityRange.oninput=function(){r._setOpacity(this.value)},this.brightnessRange.onchange=function(){r._setBrightness(this.value)},this.brightnessRange.oninput=function(){r._setBrightness(this.value)},this.brightnessLabel=document.createElement("div"),this.brightnessLabel.className="vis-label vis-brightness",this.brightnessLabel.innerHTML="brightness:",this.opacityLabel=document.createElement("div"),this.opacityLabel.className="vis-label vis-opacity",this.opacityLabel.innerHTML="opacity:",this.newColorDiv=document.createElement("div"),this.newColorDiv.className="vis-new-color",this.newColorDiv.innerHTML="new",this.initialColorDiv=document.createElement("div"),this.initialColorDiv.className="vis-initial-color",this.initialColorDiv.innerHTML="initial",this.cancelButton=document.createElement("div"),this.cancelButton.className="vis-button vis-cancel",this.cancelButton.innerHTML="cancel",this.cancelButton.onclick=va(t=this._hide).call(t,this,!1),this.applyButton=document.createElement("div"),this.applyButton.className="vis-button vis-apply",this.applyButton.innerHTML="apply",this.applyButton.onclick=va(e=this._apply).call(e,this),this.saveButton=document.createElement("div"),this.saveButton.className="vis-button vis-save",this.saveButton.innerHTML="save",this.saveButton.onclick=va(i=this._save).call(i,this),this.loadButton=document.createElement("div"),this.loadButton.className="vis-button vis-load",this.loadButton.innerHTML="load last",this.loadButton.onclick=va(n=this._loadLast).call(n,this),this.frame.appendChild(this.colorPickerDiv),this.frame.appendChild(this.arrowDiv),this.frame.appendChild(this.brightnessLabel),this.frame.appendChild(this.brightnessDiv),this.frame.appendChild(this.opacityLabel),this.frame.appendChild(this.opacityDiv),this.frame.appendChild(this.newColorDiv),this.frame.appendChild(this.initialColorDiv),this.frame.appendChild(this.cancelButton),this.frame.appendChild(this.applyButton),this.frame.appendChild(this.saveButton),this.frame.appendChild(this.loadButton)}},{key:"_bindHammer",value:function(){var t=this;this.drag={},this.pinch={},this.hammer=new Vh(this.colorPickerCanvas),this.hammer.get("pinch").set({enable:!0}),jh(this.hammer,(function(e){t._moveSelector(e)})),this.hammer.on("tap",(function(e){t._moveSelector(e)})),this.hammer.on("panstart",(function(e){t._moveSelector(e)})),this.hammer.on("panmove",(function(e){t._moveSelector(e)})),this.hammer.on("panend",(function(e){t._moveSelector(e)}))}},{key:"_generateHueCircle",value:function(){if(!1===this.generated){var t=this.colorPickerCanvas.getContext("2d");void 0===this.pixelRation&&(this.pixelRatio=(window.devicePixelRatio||1)/(t.webkitBackingStorePixelRatio||t.mozBackingStorePixelRatio||t.msBackingStorePixelRatio||t.oBackingStorePixelRatio||t.backingStorePixelRatio||1)),t.setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0);var e,i,n,o,s=this.colorPickerCanvas.clientWidth,r=this.colorPickerCanvas.clientHeight;t.clearRect(0,0,s,r),this.centerCoordinates={x:.5*s,y:.5*r},this.r=.49*s;var a,l=2*Math.PI/360,h=1/this.r;for(n=0;n<360;n++)for(o=0;o<this.r;o++)e=this.centerCoordinates.x+o*Math.sin(l*n),i=this.centerCoordinates.y+o*Math.cos(l*n),a=za.HSVToRGB(n*(1/360),o*h,1),t.fillStyle="rgb("+a.r+","+a.g+","+a.b+")",t.fillRect(e-.5,i-.5,2,2);t.strokeStyle="rgba(0,0,0,1)",t.circle(this.centerCoordinates.x,this.centerCoordinates.y,this.r),t.stroke(),this.hueCircle=t.getImageData(0,0,s,r)}this.generated=!0}},{key:"_moveSelector",value:function(t){var e=this.colorPickerDiv.getBoundingClientRect(),i=t.center.x-e.left,n=t.center.y-e.top,o=.5*this.colorPickerDiv.clientHeight,s=.5*this.colorPickerDiv.clientWidth,r=i-s,a=n-o,l=Math.atan2(r,a),h=.98*Math.min(Math.sqrt(r*r+a*a),s),d=Math.cos(l)*h+o,u=Math.sin(l)*h+s;this.colorPickerSelector.style.top=d-.5*this.colorPickerSelector.clientHeight+"px",this.colorPickerSelector.style.left=u-.5*this.colorPickerSelector.clientWidth+"px";var c=l/(2*Math.PI);c=c<0?c+1:c;var p=h/this.r,m=za.RGBToHSV(this.color.r,this.color.g,this.color.b);m.h=c,m.s=p;var f=za.HSVToRGB(m.h,m.s,m.v);f.a=this.color.a,this.color=f,this.initialColorDiv.style.backgroundColor="rgba("+this.initialColor.r+","+this.initialColor.g+","+this.initialColor.b+","+this.initialColor.a+")",this.newColorDiv.style.backgroundColor="rgba("+this.color.r+","+this.color.g+","+this.color.b+","+this.color.a+")"}}]),t}();Ph('div.vis-configuration{position:relative;display:block;float:left;font-size:12px}div.vis-configuration-wrapper{display:block;width:700px}div.vis-configuration-wrapper:after{clear:both;content:"";display:block}div.vis-configuration.vis-config-option-container{display:block;width:495px;background-color:#fff;border:2px solid #f7f8fa;border-radius:4px;margin-top:20px;left:10px;padding-left:5px}div.vis-configuration.vis-config-button{display:block;width:495px;height:25px;vertical-align:middle;line-height:25px;background-color:#f7f8fa;border:2px solid #ceced0;border-radius:4px;margin-top:20px;left:10px;padding-left:5px;cursor:pointer;margin-bottom:30px}div.vis-configuration.vis-config-button.hover{background-color:#4588e6;border:2px solid #214373;color:#fff}div.vis-configuration.vis-config-item{display:block;float:left;width:495px;height:25px;vertical-align:middle;line-height:25px}div.vis-configuration.vis-config-item.vis-config-s2{left:10px;background-color:#f7f8fa;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-item.vis-config-s3{left:20px;background-color:#e4e9f0;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-item.vis-config-s4{left:30px;background-color:#cfd8e6;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-header{font-size:18px;font-weight:700}div.vis-configuration.vis-config-label{width:120px;height:25px;line-height:25px}div.vis-configuration.vis-config-label.vis-config-s3{width:110px}div.vis-configuration.vis-config-label.vis-config-s4{width:100px}div.vis-configuration.vis-config-colorBlock{top:1px;width:30px;height:19px;border:1px solid #444;border-radius:2px;padding:0;margin:0;cursor:pointer}input.vis-configuration.vis-config-checkbox{left:-5px}input.vis-configuration.vis-config-rangeinput{position:relative;top:-5px;width:60px;padding:1px;margin:0;pointer-events:none}input.vis-configuration.vis-config-range{-webkit-appearance:none;border:0 solid #fff;background-color:transparent;width:300px;height:20px}input.vis-configuration.vis-config-range::-webkit-slider-runnable-track{width:300px;height:5px;background:#dedede;background:-moz-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#dedede),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#dedede,#c8c8c8 99%);background:-o-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:linear-gradient(180deg,#dedede 0,#c8c8c8 99%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#dedede",endColorstr="#c8c8c8",GradientType=0);border:1px solid #999;box-shadow:0 0 3px 0 #aaa;border-radius:3px}input.vis-configuration.vis-config-range::-webkit-slider-thumb{-webkit-appearance:none;border:1px solid #14334b;height:17px;width:17px;border-radius:50%;background:#3876c2;background:-moz-linear-gradient(top,#3876c2 0,#385380 100%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#3876c2),color-stop(100%,#385380));background:-webkit-linear-gradient(top,#3876c2,#385380);background:-o-linear-gradient(top,#3876c2 0,#385380 100%);background:-ms-linear-gradient(top,#3876c2 0,#385380 100%);background:linear-gradient(180deg,#3876c2 0,#385380);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#3876c2",endColorstr="#385380",GradientType=0);box-shadow:0 0 1px 0 #111927;margin-top:-7px}input.vis-configuration.vis-config-range:focus{outline:none}input.vis-configuration.vis-config-range:focus::-webkit-slider-runnable-track{background:#9d9d9d;background:-moz-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#9d9d9d),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#9d9d9d,#c8c8c8 99%);background:-o-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:linear-gradient(180deg,#9d9d9d 0,#c8c8c8 99%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#9d9d9d",endColorstr="#c8c8c8",GradientType=0)}input.vis-configuration.vis-config-range::-moz-range-track{width:300px;height:10px;background:#dedede;background:-moz-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#dedede),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#dedede,#c8c8c8 99%);background:-o-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:linear-gradient(180deg,#dedede 0,#c8c8c8 99%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#dedede",endColorstr="#c8c8c8",GradientType=0);border:1px solid #999;box-shadow:0 0 3px 0 #aaa;border-radius:3px}input.vis-configuration.vis-config-range::-moz-range-thumb{border:none;height:16px;width:16px;border-radius:50%;background:#385380}input.vis-configuration.vis-config-range:-moz-focusring{outline:1px solid #fff;outline-offset:-1px}input.vis-configuration.vis-config-range::-ms-track{width:300px;height:5px;background:transparent;border-color:transparent;border-width:6px 0;color:transparent}input.vis-configuration.vis-config-range::-ms-fill-lower{background:#777;border-radius:10px}input.vis-configuration.vis-config-range::-ms-fill-upper{background:#ddd;border-radius:10px}input.vis-configuration.vis-config-range::-ms-thumb{border:none;height:16px;width:16px;border-radius:50%;background:#385380}input.vis-configuration.vis-config-range:focus::-ms-fill-lower{background:#888}input.vis-configuration.vis-config-range:focus::-ms-fill-upper{background:#ccc}.vis-configuration-popup{position:absolute;background:rgba(57,76,89,.85);border:2px solid #f2faff;line-height:30px;height:30px;width:150px;text-align:center;color:#fff;font-size:14px;border-radius:4px;-webkit-transition:opacity .3s ease-in-out;-moz-transition:opacity .3s ease-in-out;transition:opacity .3s ease-in-out}.vis-configuration-popup:after,.vis-configuration-popup:before{left:100%;top:50%;border:solid transparent;content:" ";height:0;width:0;position:absolute;pointer-events:none}.vis-configuration-popup:after{border-color:rgba(136,183,213,0) rgba(136,183,213,0) rgba(136,183,213,0) rgba(57,76,89,.85);border-width:8px;margin-top:-8px}.vis-configuration-popup:before{border-color:rgba(194,225,245,0) rgba(194,225,245,0) rgba(194,225,245,0) #f2faff;border-width:12px;margin-top:-12px}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbImNvbmZpZ3VyYXRpb24uY3NzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLHNCQUNJLGlCQUFpQixDQUNqQixhQUFhLENBQ2IsVUFBVSxDQUNWLGNBQ0osQ0FFQSw4QkFDSSxhQUFhLENBQ2IsV0FDSixDQUVBLG9DQUNFLFVBQVcsQ0FDWCxVQUFXLENBQ1gsYUFDRixDQUVBLGtEQUNJLGFBQWEsQ0FDYixXQUFXLENBQ1gscUJBQXlCLENBQ3pCLHdCQUF3QixDQUN4QixpQkFBaUIsQ0FDakIsZUFBZSxDQUNmLFNBQVMsQ0FDVCxnQkFDSixDQUVBLHdDQUNJLGFBQWEsQ0FDYixXQUFXLENBQ1gsV0FBVyxDQUNYLHFCQUFzQixDQUN0QixnQkFBZ0IsQ0FDaEIsd0JBQXlCLENBQ3pCLHdCQUF3QixDQUN4QixpQkFBaUIsQ0FDakIsZUFBZSxDQUNmLFNBQVMsQ0FDVCxnQkFBZ0IsQ0FDaEIsY0FBZSxDQUNmLGtCQUNKLENBRUEsOENBQ0ksd0JBQXlCLENBQ3pCLHdCQUF3QixDQUN4QixVQUNKLENBRUEsc0NBQ0ksYUFBYSxDQUNiLFVBQVUsQ0FDVixXQUFXLENBQ1gsV0FBVyxDQUNYLHFCQUFzQixDQUN0QixnQkFDSixDQUdBLG9EQUNJLFNBQVMsQ0FDVCx3QkFBeUIsQ0FDekIsZ0JBQWdCLENBQ2hCLGlCQUNKLENBQ0Esb0RBQ0ksU0FBUyxDQUNULHdCQUF5QixDQUN6QixnQkFBZ0IsQ0FDaEIsaUJBQ0osQ0FDQSxvREFDSSxTQUFTLENBQ1Qsd0JBQXlCLENBQ3pCLGdCQUFnQixDQUNoQixpQkFDSixDQUVBLHdDQUNJLGNBQWMsQ0FDZCxlQUNKLENBRUEsdUNBQ0ksV0FBVyxDQUNYLFdBQVcsQ0FDWCxnQkFDSixDQUVBLHFEQUNJLFdBQ0osQ0FDQSxxREFDSSxXQUNKLENBRUEsNENBQ0ksT0FBTyxDQUNQLFVBQVUsQ0FDVixXQUFXLENBQ1gscUJBQXdCLENBQ3hCLGlCQUFpQixDQUNqQixTQUFXLENBQ1gsUUFBVSxDQUNWLGNBQ0osQ0FFQSw0Q0FDSSxTQUNKLENBR0EsOENBQ0ksaUJBQWlCLENBQ2pCLFFBQVEsQ0FDUixVQUFVLENBRVYsV0FBVyxDQUNYLFFBQVEsQ0FDUixtQkFDSixDQUVBLHlDQUVJLHVCQUF3QixDQUd4QixtQkFBdUIsQ0FDdkIsNEJBQThCLENBRzlCLFdBQVksQ0FDWixXQUNKLENBQ0Esd0VBQ0ksV0FBWSxDQUNaLFVBQVcsQ0FDWCxrQkFBbUIsQ0FDbkIsMERBQStELENBQy9ELHNHQUE0RyxDQUM1RywyREFBaUUsQ0FDakUsd0RBQTRELENBQzVELHlEQUE2RCxDQUM3RCx3REFBK0QsQ0FDL0QsK0dBQW1ILENBRW5ILHFCQUF5QixDQUN6Qix5QkFBbUMsQ0FDbkMsaUJBQ0osQ0FDQSwrREFDSSx1QkFBd0IsQ0FDeEIsd0JBQXlCLENBQ3pCLFdBQVksQ0FDWixVQUFXLENBQ1gsaUJBQWtCLENBQ2xCLGtCQUFtQixDQUNuQiwyREFBZ0UsQ0FDaEUsdUdBQTZHLENBQzdHLHVEQUFrRSxDQUNsRSx5REFBNkQsQ0FDN0QsMERBQThELENBQzlELG9EQUFnRSxDQUNoRSwrR0FBbUgsQ0FDbkgsNEJBQW1DLENBQ25DLGVBQ0osQ0FDQSwrQ0FDSSxZQUNKLENBQ0EsOEVBQ0ksa0JBQW1CLENBQ25CLDBEQUE4RCxDQUM5RCxzR0FBNEcsQ0FDNUcsMkRBQWlFLENBQ2pFLHdEQUE0RCxDQUM1RCx5REFBNkQsQ0FDN0Qsd0RBQStELENBQy9ELCtHQUNKLENBRUEsMkRBQ0ksV0FBWSxDQUNaLFdBQVksQ0FDWixrQkFBbUIsQ0FDbkIsMERBQStELENBQy9ELHNHQUE0RyxDQUM1RywyREFBaUUsQ0FDakUsd0RBQTRELENBQzVELHlEQUE2RCxDQUM3RCx3REFBK0QsQ0FDL0QsK0dBQW1ILENBRW5ILHFCQUF5QixDQUN6Qix5QkFBbUMsQ0FDbkMsaUJBQ0osQ0FDQSwyREFDSSxXQUFZLENBQ1osV0FBWSxDQUNaLFVBQVcsQ0FFWCxpQkFBa0IsQ0FDbEIsa0JBQ0osQ0FHQSx3REFDSSxzQkFBd0IsQ0FDeEIsbUJBQ0osQ0FFQSxvREFDSSxXQUFZLENBQ1osVUFBVyxDQUdYLHNCQUF1QixDQUd2Qix3QkFBeUIsQ0FDekIsa0JBQW1CLENBR25CLGlCQUNKLENBQ0EseURBQ0ksZUFBZ0IsQ0FDaEIsa0JBQ0osQ0FDQSx5REFDSSxlQUFnQixDQUNoQixrQkFDSixDQUNBLG9EQUNJLFdBQVksQ0FDWixXQUFZLENBQ1osVUFBVyxDQUNYLGlCQUFrQixDQUNsQixrQkFDSixDQUNBLCtEQUNJLGVBQ0osQ0FDQSwrREFDSSxlQUNKLENBRUEseUJBQ0ksaUJBQWtCLENBQ2xCLDZCQUFrQyxDQUNsQyx3QkFBeUIsQ0FDekIsZ0JBQWdCLENBQ2hCLFdBQVcsQ0FDWCxXQUFXLENBQ1gsaUJBQWlCLENBQ2pCLFVBQWMsQ0FDZCxjQUFjLENBQ2QsaUJBQWlCLENBQ2pCLDBDQUE0QyxDQUM1Qyx1Q0FBeUMsQ0FDekMsa0NBQ0osQ0FDQSwrREFDSSxTQUFVLENBQ1YsT0FBUSxDQUNSLHdCQUF5QixDQUN6QixXQUFZLENBQ1osUUFBUyxDQUNULE9BQVEsQ0FDUixpQkFBa0IsQ0FDbEIsbUJBQ0osQ0FFQSwrQkFFSSwyRkFBeUMsQ0FDekMsZ0JBQWlCLENBQ2pCLGVBQ0osQ0FDQSxnQ0FFSSxnRkFBMEIsQ0FDMUIsaUJBQWtCLENBQ2xCLGdCQUNKIiwiZmlsZSI6ImNvbmZpZ3VyYXRpb24uY3NzIiwic291cmNlc0NvbnRlbnQiOlsiZGl2LnZpcy1jb25maWd1cmF0aW9uIHtcbiAgICBwb3NpdGlvbjpyZWxhdGl2ZTtcbiAgICBkaXNwbGF5OmJsb2NrO1xuICAgIGZsb2F0OmxlZnQ7XG4gICAgZm9udC1zaXplOjEycHg7XG59XG5cbmRpdi52aXMtY29uZmlndXJhdGlvbi13cmFwcGVyIHtcbiAgICBkaXNwbGF5OmJsb2NrO1xuICAgIHdpZHRoOjcwMHB4O1xufVxuXG5kaXYudmlzLWNvbmZpZ3VyYXRpb24td3JhcHBlcjo6YWZ0ZXIge1xuICBjbGVhcjogYm90aDtcbiAgY29udGVudDogXCJcIjtcbiAgZGlzcGxheTogYmxvY2s7XG59XG5cbmRpdi52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLW9wdGlvbi1jb250YWluZXJ7XG4gICAgZGlzcGxheTpibG9jaztcbiAgICB3aWR0aDo0OTVweDtcbiAgICBiYWNrZ3JvdW5kLWNvbG9yOiAjZmZmZmZmO1xuICAgIGJvcmRlcjoycHggc29saWQgI2Y3ZjhmYTtcbiAgICBib3JkZXItcmFkaXVzOjRweDtcbiAgICBtYXJnaW4tdG9wOjIwcHg7XG4gICAgbGVmdDoxMHB4O1xuICAgIHBhZGRpbmctbGVmdDo1cHg7XG59XG5cbmRpdi52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLWJ1dHRvbntcbiAgICBkaXNwbGF5OmJsb2NrO1xuICAgIHdpZHRoOjQ5NXB4O1xuICAgIGhlaWdodDoyNXB4O1xuICAgIHZlcnRpY2FsLWFsaWduOiBtaWRkbGU7XG4gICAgbGluZS1oZWlnaHQ6MjVweDtcbiAgICBiYWNrZ3JvdW5kLWNvbG9yOiAjZjdmOGZhO1xuICAgIGJvcmRlcjoycHggc29saWQgI2NlY2VkMDtcbiAgICBib3JkZXItcmFkaXVzOjRweDtcbiAgICBtYXJnaW4tdG9wOjIwcHg7XG4gICAgbGVmdDoxMHB4O1xuICAgIHBhZGRpbmctbGVmdDo1cHg7XG4gICAgY3Vyc29yOiBwb2ludGVyO1xuICAgIG1hcmdpbi1ib3R0b206MzBweDtcbn1cblxuZGl2LnZpcy1jb25maWd1cmF0aW9uLnZpcy1jb25maWctYnV0dG9uLmhvdmVye1xuICAgIGJhY2tncm91bmQtY29sb3I6ICM0NTg4ZTY7XG4gICAgYm9yZGVyOjJweCBzb2xpZCAjMjE0MzczO1xuICAgIGNvbG9yOiNmZmZmZmY7XG59XG5cbmRpdi52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLWl0ZW17XG4gICAgZGlzcGxheTpibG9jaztcbiAgICBmbG9hdDpsZWZ0O1xuICAgIHdpZHRoOjQ5NXB4O1xuICAgIGhlaWdodDoyNXB4O1xuICAgIHZlcnRpY2FsLWFsaWduOiBtaWRkbGU7XG4gICAgbGluZS1oZWlnaHQ6MjVweDtcbn1cblxuXG5kaXYudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1pdGVtLnZpcy1jb25maWctczJ7XG4gICAgbGVmdDoxMHB4O1xuICAgIGJhY2tncm91bmQtY29sb3I6ICNmN2Y4ZmE7XG4gICAgcGFkZGluZy1sZWZ0OjVweDtcbiAgICBib3JkZXItcmFkaXVzOjNweDtcbn1cbmRpdi52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLWl0ZW0udmlzLWNvbmZpZy1zM3tcbiAgICBsZWZ0OjIwcHg7XG4gICAgYmFja2dyb3VuZC1jb2xvcjogI2U0ZTlmMDtcbiAgICBwYWRkaW5nLWxlZnQ6NXB4O1xuICAgIGJvcmRlci1yYWRpdXM6M3B4O1xufVxuZGl2LnZpcy1jb25maWd1cmF0aW9uLnZpcy1jb25maWctaXRlbS52aXMtY29uZmlnLXM0e1xuICAgIGxlZnQ6MzBweDtcbiAgICBiYWNrZ3JvdW5kLWNvbG9yOiAjY2ZkOGU2O1xuICAgIHBhZGRpbmctbGVmdDo1cHg7XG4gICAgYm9yZGVyLXJhZGl1czozcHg7XG59XG5cbmRpdi52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLWhlYWRlcntcbiAgICBmb250LXNpemU6MThweDtcbiAgICBmb250LXdlaWdodDogYm9sZDtcbn1cblxuZGl2LnZpcy1jb25maWd1cmF0aW9uLnZpcy1jb25maWctbGFiZWx7XG4gICAgd2lkdGg6MTIwcHg7XG4gICAgaGVpZ2h0OjI1cHg7XG4gICAgbGluZS1oZWlnaHQ6IDI1cHg7XG59XG5cbmRpdi52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLWxhYmVsLnZpcy1jb25maWctczN7XG4gICAgd2lkdGg6MTEwcHg7XG59XG5kaXYudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1sYWJlbC52aXMtY29uZmlnLXM0e1xuICAgIHdpZHRoOjEwMHB4O1xufVxuXG5kaXYudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1jb2xvckJsb2Nre1xuICAgIHRvcDoxcHg7XG4gICAgd2lkdGg6MzBweDtcbiAgICBoZWlnaHQ6MTlweDtcbiAgICBib3JkZXI6MXB4IHNvbGlkICM0NDQ0NDQ7XG4gICAgYm9yZGVyLXJhZGl1czoycHg7XG4gICAgcGFkZGluZzowcHg7XG4gICAgbWFyZ2luOjBweDtcbiAgICBjdXJzb3I6cG9pbnRlcjtcbn1cblxuaW5wdXQudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1jaGVja2JveCB7XG4gICAgbGVmdDotNXB4O1xufVxuXG5cbmlucHV0LnZpcy1jb25maWd1cmF0aW9uLnZpcy1jb25maWctcmFuZ2VpbnB1dHtcbiAgICBwb3NpdGlvbjpyZWxhdGl2ZTtcbiAgICB0b3A6LTVweDtcbiAgICB3aWR0aDo2MHB4O1xuICAgIC8qaGVpZ2h0OjEzcHg7Ki9cbiAgICBwYWRkaW5nOjFweDtcbiAgICBtYXJnaW46MDtcbiAgICBwb2ludGVyLWV2ZW50czpub25lO1xufVxuXG5pbnB1dC52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLXJhbmdle1xuICAgIC8qcmVtb3ZlcyBkZWZhdWx0IHdlYmtpdCBzdHlsZXMqL1xuICAgIC13ZWJraXQtYXBwZWFyYW5jZTogbm9uZTtcblxuICAgIC8qZml4IGZvciBGRiB1bmFibGUgdG8gYXBwbHkgZm9jdXMgc3R5bGUgYnVnICovXG4gICAgYm9yZGVyOiAwcHggc29saWQgd2hpdGU7XG4gICAgYmFja2dyb3VuZC1jb2xvcjpyZ2JhKDAsMCwwLDApO1xuXG4gICAgLypyZXF1aXJlZCBmb3IgcHJvcGVyIHRyYWNrIHNpemluZyBpbiBGRiovXG4gICAgd2lkdGg6IDMwMHB4O1xuICAgIGhlaWdodDoyMHB4O1xufVxuaW5wdXQudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1yYW5nZTo6LXdlYmtpdC1zbGlkZXItcnVubmFibGUtdHJhY2sge1xuICAgIHdpZHRoOiAzMDBweDtcbiAgICBoZWlnaHQ6IDVweDtcbiAgICBiYWNrZ3JvdW5kOiAjZGVkZWRlOyAvKiBPbGQgYnJvd3NlcnMgKi9cbiAgICBiYWNrZ3JvdW5kOiAtbW96LWxpbmVhci1ncmFkaWVudCh0b3AsICAjZGVkZWRlIDAlLCAjYzhjOGM4IDk5JSk7IC8qIEZGMy42KyAqL1xuICAgIGJhY2tncm91bmQ6IC13ZWJraXQtZ3JhZGllbnQobGluZWFyLCBsZWZ0IHRvcCwgbGVmdCBib3R0b20sIGNvbG9yLXN0b3AoMCUsI2RlZGVkZSksIGNvbG9yLXN0b3AoOTklLCNjOGM4YzgpKTsgLyogQ2hyb21lLFNhZmFyaTQrICovXG4gICAgYmFja2dyb3VuZDogLXdlYmtpdC1saW5lYXItZ3JhZGllbnQodG9wLCAgI2RlZGVkZSAwJSwjYzhjOGM4IDk5JSk7IC8qIENocm9tZTEwKyxTYWZhcmk1LjErICovXG4gICAgYmFja2dyb3VuZDogLW8tbGluZWFyLWdyYWRpZW50KHRvcCwgI2RlZGVkZSAwJSwgI2M4YzhjOCA5OSUpOyAvKiBPcGVyYSAxMS4xMCsgKi9cbiAgICBiYWNrZ3JvdW5kOiAtbXMtbGluZWFyLWdyYWRpZW50KHRvcCwgICNkZWRlZGUgMCUsI2M4YzhjOCA5OSUpOyAvKiBJRTEwKyAqL1xuICAgIGJhY2tncm91bmQ6IGxpbmVhci1ncmFkaWVudCh0byBib3R0b20sICAjZGVkZWRlIDAlLCNjOGM4YzggOTklKTsgLyogVzNDICovXG4gICAgZmlsdGVyOiBwcm9naWQ6RFhJbWFnZVRyYW5zZm9ybS5NaWNyb3NvZnQuZ3JhZGllbnQoIHN0YXJ0Q29sb3JzdHI9JyNkZWRlZGUnLCBlbmRDb2xvcnN0cj0nI2M4YzhjOCcsR3JhZGllbnRUeXBlPTAgKTsgLyogSUU2LTkgKi9cblxuICAgIGJvcmRlcjogMXB4IHNvbGlkICM5OTk5OTk7XG4gICAgYm94LXNoYWRvdzogI2FhYWFhYSAwcHggMHB4IDNweCAwcHg7XG4gICAgYm9yZGVyLXJhZGl1czogM3B4O1xufVxuaW5wdXQudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1yYW5nZTo6LXdlYmtpdC1zbGlkZXItdGh1bWIge1xuICAgIC13ZWJraXQtYXBwZWFyYW5jZTogbm9uZTtcbiAgICBib3JkZXI6IDFweCBzb2xpZCAjMTQzMzRiO1xuICAgIGhlaWdodDogMTdweDtcbiAgICB3aWR0aDogMTdweDtcbiAgICBib3JkZXItcmFkaXVzOiA1MCU7XG4gICAgYmFja2dyb3VuZDogIzM4NzZjMjsgLyogT2xkIGJyb3dzZXJzICovXG4gICAgYmFja2dyb3VuZDogLW1vei1saW5lYXItZ3JhZGllbnQodG9wLCAgIzM4NzZjMiAwJSwgIzM4NTM4MCAxMDAlKTsgLyogRkYzLjYrICovXG4gICAgYmFja2dyb3VuZDogLXdlYmtpdC1ncmFkaWVudChsaW5lYXIsIGxlZnQgdG9wLCBsZWZ0IGJvdHRvbSwgY29sb3Itc3RvcCgwJSwjMzg3NmMyKSwgY29sb3Itc3RvcCgxMDAlLCMzODUzODApKTsgLyogQ2hyb21lLFNhZmFyaTQrICovXG4gICAgYmFja2dyb3VuZDogLXdlYmtpdC1saW5lYXItZ3JhZGllbnQodG9wLCAgIzM4NzZjMiAwJSwjMzg1MzgwIDEwMCUpOyAvKiBDaHJvbWUxMCssU2FmYXJpNS4xKyAqL1xuICAgIGJhY2tncm91bmQ6IC1vLWxpbmVhci1ncmFkaWVudCh0b3AsICAjMzg3NmMyIDAlLCMzODUzODAgMTAwJSk7IC8qIE9wZXJhIDExLjEwKyAqL1xuICAgIGJhY2tncm91bmQ6IC1tcy1saW5lYXItZ3JhZGllbnQodG9wLCAgIzM4NzZjMiAwJSwjMzg1MzgwIDEwMCUpOyAvKiBJRTEwKyAqL1xuICAgIGJhY2tncm91bmQ6IGxpbmVhci1ncmFkaWVudCh0byBib3R0b20sICAjMzg3NmMyIDAlLCMzODUzODAgMTAwJSk7IC8qIFczQyAqL1xuICAgIGZpbHRlcjogcHJvZ2lkOkRYSW1hZ2VUcmFuc2Zvcm0uTWljcm9zb2Z0LmdyYWRpZW50KCBzdGFydENvbG9yc3RyPScjMzg3NmMyJywgZW5kQ29sb3JzdHI9JyMzODUzODAnLEdyYWRpZW50VHlwZT0wICk7IC8qIElFNi05ICovXG4gICAgYm94LXNoYWRvdzogIzExMTkyNyAwcHggMHB4IDFweCAwcHg7XG4gICAgbWFyZ2luLXRvcDogLTdweDtcbn1cbmlucHV0LnZpcy1jb25maWd1cmF0aW9uLnZpcy1jb25maWctcmFuZ2U6Zm9jdXMge1xuICAgIG91dGxpbmU6IG5vbmU7XG59XG5pbnB1dC52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLXJhbmdlOmZvY3VzOjotd2Via2l0LXNsaWRlci1ydW5uYWJsZS10cmFjayB7XG4gICAgYmFja2dyb3VuZDogIzlkOWQ5ZDsgLyogT2xkIGJyb3dzZXJzICovXG4gICAgYmFja2dyb3VuZDogLW1vei1saW5lYXItZ3JhZGllbnQodG9wLCAjOWQ5ZDlkIDAlLCAjYzhjOGM4IDk5JSk7IC8qIEZGMy42KyAqL1xuICAgIGJhY2tncm91bmQ6IC13ZWJraXQtZ3JhZGllbnQobGluZWFyLCBsZWZ0IHRvcCwgbGVmdCBib3R0b20sIGNvbG9yLXN0b3AoMCUsIzlkOWQ5ZCksIGNvbG9yLXN0b3AoOTklLCNjOGM4YzgpKTsgLyogQ2hyb21lLFNhZmFyaTQrICovXG4gICAgYmFja2dyb3VuZDogLXdlYmtpdC1saW5lYXItZ3JhZGllbnQodG9wLCAgIzlkOWQ5ZCAwJSwjYzhjOGM4IDk5JSk7IC8qIENocm9tZTEwKyxTYWZhcmk1LjErICovXG4gICAgYmFja2dyb3VuZDogLW8tbGluZWFyLWdyYWRpZW50KHRvcCwgICM5ZDlkOWQgMCUsI2M4YzhjOCA5OSUpOyAvKiBPcGVyYSAxMS4xMCsgKi9cbiAgICBiYWNrZ3JvdW5kOiAtbXMtbGluZWFyLWdyYWRpZW50KHRvcCwgICM5ZDlkOWQgMCUsI2M4YzhjOCA5OSUpOyAvKiBJRTEwKyAqL1xuICAgIGJhY2tncm91bmQ6IGxpbmVhci1ncmFkaWVudCh0byBib3R0b20sICAjOWQ5ZDlkIDAlLCNjOGM4YzggOTklKTsgLyogVzNDICovXG4gICAgZmlsdGVyOiBwcm9naWQ6RFhJbWFnZVRyYW5zZm9ybS5NaWNyb3NvZnQuZ3JhZGllbnQoIHN0YXJ0Q29sb3JzdHI9JyM5ZDlkOWQnLCBlbmRDb2xvcnN0cj0nI2M4YzhjOCcsR3JhZGllbnRUeXBlPTAgKTsgLyogSUU2LTkgKi9cbn1cblxuaW5wdXQudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1yYW5nZTo6LW1vei1yYW5nZS10cmFjayB7XG4gICAgd2lkdGg6IDMwMHB4O1xuICAgIGhlaWdodDogMTBweDtcbiAgICBiYWNrZ3JvdW5kOiAjZGVkZWRlOyAvKiBPbGQgYnJvd3NlcnMgKi9cbiAgICBiYWNrZ3JvdW5kOiAtbW96LWxpbmVhci1ncmFkaWVudCh0b3AsICAjZGVkZWRlIDAlLCAjYzhjOGM4IDk5JSk7IC8qIEZGMy42KyAqL1xuICAgIGJhY2tncm91bmQ6IC13ZWJraXQtZ3JhZGllbnQobGluZWFyLCBsZWZ0IHRvcCwgbGVmdCBib3R0b20sIGNvbG9yLXN0b3AoMCUsI2RlZGVkZSksIGNvbG9yLXN0b3AoOTklLCNjOGM4YzgpKTsgLyogQ2hyb21lLFNhZmFyaTQrICovXG4gICAgYmFja2dyb3VuZDogLXdlYmtpdC1saW5lYXItZ3JhZGllbnQodG9wLCAgI2RlZGVkZSAwJSwjYzhjOGM4IDk5JSk7IC8qIENocm9tZTEwKyxTYWZhcmk1LjErICovXG4gICAgYmFja2dyb3VuZDogLW8tbGluZWFyLWdyYWRpZW50KHRvcCwgI2RlZGVkZSAwJSwgI2M4YzhjOCA5OSUpOyAvKiBPcGVyYSAxMS4xMCsgKi9cbiAgICBiYWNrZ3JvdW5kOiAtbXMtbGluZWFyLWdyYWRpZW50KHRvcCwgICNkZWRlZGUgMCUsI2M4YzhjOCA5OSUpOyAvKiBJRTEwKyAqL1xuICAgIGJhY2tncm91bmQ6IGxpbmVhci1ncmFkaWVudCh0byBib3R0b20sICAjZGVkZWRlIDAlLCNjOGM4YzggOTklKTsgLyogVzNDICovXG4gICAgZmlsdGVyOiBwcm9naWQ6RFhJbWFnZVRyYW5zZm9ybS5NaWNyb3NvZnQuZ3JhZGllbnQoIHN0YXJ0Q29sb3JzdHI9JyNkZWRlZGUnLCBlbmRDb2xvcnN0cj0nI2M4YzhjOCcsR3JhZGllbnRUeXBlPTAgKTsgLyogSUU2LTkgKi9cblxuICAgIGJvcmRlcjogMXB4IHNvbGlkICM5OTk5OTk7XG4gICAgYm94LXNoYWRvdzogI2FhYWFhYSAwcHggMHB4IDNweCAwcHg7XG4gICAgYm9yZGVyLXJhZGl1czogM3B4O1xufVxuaW5wdXQudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1yYW5nZTo6LW1vei1yYW5nZS10aHVtYiB7XG4gICAgYm9yZGVyOiBub25lO1xuICAgIGhlaWdodDogMTZweDtcbiAgICB3aWR0aDogMTZweDtcblxuICAgIGJvcmRlci1yYWRpdXM6IDUwJTtcbiAgICBiYWNrZ3JvdW5kOiAgIzM4NTM4MDtcbn1cblxuLypoaWRlIHRoZSBvdXRsaW5lIGJlaGluZCB0aGUgYm9yZGVyKi9cbmlucHV0LnZpcy1jb25maWd1cmF0aW9uLnZpcy1jb25maWctcmFuZ2U6LW1vei1mb2N1c3Jpbmd7XG4gICAgb3V0bGluZTogMXB4IHNvbGlkIHdoaXRlO1xuICAgIG91dGxpbmUtb2Zmc2V0OiAtMXB4O1xufVxuXG5pbnB1dC52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLXJhbmdlOjotbXMtdHJhY2sge1xuICAgIHdpZHRoOiAzMDBweDtcbiAgICBoZWlnaHQ6IDVweDtcblxuICAgIC8qcmVtb3ZlIGJnIGNvbG91ciBmcm9tIHRoZSB0cmFjaywgd2UnbGwgdXNlIG1zLWZpbGwtbG93ZXIgYW5kIG1zLWZpbGwtdXBwZXIgaW5zdGVhZCAqL1xuICAgIGJhY2tncm91bmQ6IHRyYW5zcGFyZW50O1xuXG4gICAgLypsZWF2ZSByb29tIGZvciB0aGUgbGFyZ2VyIHRodW1iIHRvIG92ZXJmbG93IHdpdGggYSB0cmFuc3BhcmVudCBib3JkZXIgKi9cbiAgICBib3JkZXItY29sb3I6IHRyYW5zcGFyZW50O1xuICAgIGJvcmRlci13aWR0aDogNnB4IDA7XG5cbiAgICAvKnJlbW92ZSBkZWZhdWx0IHRpY2sgbWFya3MqL1xuICAgIGNvbG9yOiB0cmFuc3BhcmVudDtcbn1cbmlucHV0LnZpcy1jb25maWd1cmF0aW9uLnZpcy1jb25maWctcmFuZ2U6Oi1tcy1maWxsLWxvd2VyIHtcbiAgICBiYWNrZ3JvdW5kOiAjNzc3O1xuICAgIGJvcmRlci1yYWRpdXM6IDEwcHg7XG59XG5pbnB1dC52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLXJhbmdlOjotbXMtZmlsbC11cHBlciB7XG4gICAgYmFja2dyb3VuZDogI2RkZDtcbiAgICBib3JkZXItcmFkaXVzOiAxMHB4O1xufVxuaW5wdXQudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1yYW5nZTo6LW1zLXRodW1iIHtcbiAgICBib3JkZXI6IG5vbmU7XG4gICAgaGVpZ2h0OiAxNnB4O1xuICAgIHdpZHRoOiAxNnB4O1xuICAgIGJvcmRlci1yYWRpdXM6IDUwJTtcbiAgICBiYWNrZ3JvdW5kOiAgIzM4NTM4MDtcbn1cbmlucHV0LnZpcy1jb25maWd1cmF0aW9uLnZpcy1jb25maWctcmFuZ2U6Zm9jdXM6Oi1tcy1maWxsLWxvd2VyIHtcbiAgICBiYWNrZ3JvdW5kOiAjODg4O1xufVxuaW5wdXQudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1yYW5nZTpmb2N1czo6LW1zLWZpbGwtdXBwZXIge1xuICAgIGJhY2tncm91bmQ6ICNjY2M7XG59XG5cbi52aXMtY29uZmlndXJhdGlvbi1wb3B1cCB7XG4gICAgcG9zaXRpb246IGFic29sdXRlO1xuICAgIGJhY2tncm91bmQ6IHJnYmEoNTcsIDc2LCA4OSwgMC44NSk7XG4gICAgYm9yZGVyOiAycHggc29saWQgI2YyZmFmZjtcbiAgICBsaW5lLWhlaWdodDozMHB4O1xuICAgIGhlaWdodDozMHB4O1xuICAgIHdpZHRoOjE1MHB4O1xuICAgIHRleHQtYWxpZ246Y2VudGVyO1xuICAgIGNvbG9yOiAjZmZmZmZmO1xuICAgIGZvbnQtc2l6ZToxNHB4O1xuICAgIGJvcmRlci1yYWRpdXM6NHB4O1xuICAgIC13ZWJraXQtdHJhbnNpdGlvbjogb3BhY2l0eSAwLjNzIGVhc2UtaW4tb3V0O1xuICAgIC1tb3otdHJhbnNpdGlvbjogb3BhY2l0eSAwLjNzIGVhc2UtaW4tb3V0O1xuICAgIHRyYW5zaXRpb246IG9wYWNpdHkgMC4zcyBlYXNlLWluLW91dDtcbn1cbi52aXMtY29uZmlndXJhdGlvbi1wb3B1cDphZnRlciwgLnZpcy1jb25maWd1cmF0aW9uLXBvcHVwOmJlZm9yZSB7XG4gICAgbGVmdDogMTAwJTtcbiAgICB0b3A6IDUwJTtcbiAgICBib3JkZXI6IHNvbGlkIHRyYW5zcGFyZW50O1xuICAgIGNvbnRlbnQ6IFwiIFwiO1xuICAgIGhlaWdodDogMDtcbiAgICB3aWR0aDogMDtcbiAgICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gICAgcG9pbnRlci1ldmVudHM6IG5vbmU7XG59XG5cbi52aXMtY29uZmlndXJhdGlvbi1wb3B1cDphZnRlciB7XG4gICAgYm9yZGVyLWNvbG9yOiByZ2JhKDEzNiwgMTgzLCAyMTMsIDApO1xuICAgIGJvcmRlci1sZWZ0LWNvbG9yOiByZ2JhKDU3LCA3NiwgODksIDAuODUpO1xuICAgIGJvcmRlci13aWR0aDogOHB4O1xuICAgIG1hcmdpbi10b3A6IC04cHg7XG59XG4udmlzLWNvbmZpZ3VyYXRpb24tcG9wdXA6YmVmb3JlIHtcbiAgICBib3JkZXItY29sb3I6IHJnYmEoMTk0LCAyMjUsIDI0NSwgMCk7XG4gICAgYm9yZGVyLWxlZnQtY29sb3I6ICNmMmZhZmY7XG4gICAgYm9yZGVyLXdpZHRoOiAxMnB4O1xuICAgIG1hcmdpbi10b3A6IC0xMnB4O1xufSJdfQ== */');var Lu=function(){function t(e,i,n){var o=arguments.length>3&&void 0!==arguments[3]?arguments[3]:1;ba(this,t),this.parent=e,this.changedOptions=[],this.container=i,this.allowCreation=!1,this.options={},this.initialized=!1,this.popupCounter=0,this.defaultOptions={enabled:!1,filter:!0,container:void 0,showButton:!0},za.extend(this.options,this.defaultOptions),this.configureOptions=n,this.moduleOptions={},this.domElements=[],this.popupDiv={},this.popupLimit=5,this.popupHistory={},this.colorPicker=new Cu(o),this.wrapper=void 0}return xa(t,[{key:"setOptions",value:function(t){if(void 0!==t){this.popupHistory={},this._removePopup();var e=!0;if("string"==typeof t)this.options.filter=t;else if(t instanceof Array)this.options.filter=t.join();else if("object"===zo(t)){if(null==t)throw new TypeError("options cannot be null");void 0!==t.container&&(this.options.container=t.container),void 0!==En(t)&&(this.options.filter=En(t)),void 0!==t.showButton&&(this.options.showButton=t.showButton),void 0!==t.enabled&&(e=t.enabled)}else"boolean"==typeof t?(this.options.filter=!0,e=t):"function"==typeof t&&(this.options.filter=t,e=!0);!1===En(this.options)&&(e=!1),this.options.enabled=e}this._clean()}},{key:"setModuleOptions",value:function(t){this.moduleOptions=t,!0===this.options.enabled&&(this._clean(),void 0!==this.options.container&&(this.container=this.options.container),this._create())}},{key:"_create",value:function(){this._clean(),this.changedOptions=[];var t=En(this.options),e=0,i=!1;for(var n in this.configureOptions)this.configureOptions.hasOwnProperty(n)&&(this.allowCreation=!1,i=!1,"function"==typeof t?i=(i=t(n,[]))||this._handleObject(this.configureOptions[n],[n],!0):!0!==t&&-1===Fo(t).call(t,n)||(i=!0),!1!==i&&(this.allowCreation=!0,e>0&&this._makeItem([]),this._makeHeader(n),this._handleObject(this.configureOptions[n],[n])),e++);this._makeButton(),this._push()}},{key:"_push",value:function(){this.wrapper=document.createElement("div"),this.wrapper.className="vis-configuration-wrapper",this.container.appendChild(this.wrapper);for(var t=0;t<this.domElements.length;t++)this.wrapper.appendChild(this.domElements[t]);this._showPopupIfNeeded()}},{key:"_clean",value:function(){for(var t=0;t<this.domElements.length;t++)this.wrapper.removeChild(this.domElements[t]);void 0!==this.wrapper&&(this.container.removeChild(this.wrapper),this.wrapper=void 0),this.domElements=[],this._removePopup()}},{key:"_getValue",value:function(t){for(var e=this.moduleOptions,i=0;i<t.length;i++){if(void 0===e[t[i]]){e=void 0;break}e=e[t[i]]}return e}},{key:"_makeItem",value:function(t){if(!0===this.allowCreation){var e=document.createElement("div");e.className="vis-configuration vis-config-item vis-config-s"+t.length;for(var i=arguments.length,n=new Array(i>1?i-1:0),o=1;o<i;o++)n[o-1]=arguments[o];return mn(n).call(n,(function(t){e.appendChild(t)})),this.domElements.push(e),this.domElements.length}return 0}},{key:"_makeHeader",value:function(t){var e=document.createElement("div");e.className="vis-configuration vis-config-header",e.innerHTML=t,this._makeItem([],e)}},{key:"_makeLabel",value:function(t,e){var i=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=document.createElement("div");return n.className="vis-configuration vis-config-label vis-config-s"+e.length,n.innerHTML=!0===i?"<i><b>"+t+":</b></i>":t+":",n}},{key:"_makeDropdown",value:function(t,e,i){var n=document.createElement("select");n.className="vis-configuration vis-config-select";var o=0;void 0!==e&&-1!==Fo(t).call(t,e)&&(o=Fo(t).call(t,e));for(var s=0;s<t.length;s++){var r=document.createElement("option");r.value=t[s],s===o&&(r.selected="selected"),r.innerHTML=t[s],n.appendChild(r)}var a=this;n.onchange=function(){a._update(this.value,i)};var l=this._makeLabel(i[i.length-1],i);this._makeItem(i,l,n)}},{key:"_makeRange",value:function(t,e,i){var n=t[0],o=t[1],s=t[2],r=t[3],a=document.createElement("input");a.className="vis-configuration vis-config-range";try{a.type="range",a.min=o,a.max=s}catch(t){}a.step=r;var l="",h=0;if(void 0!==e){e<0&&1.2*e<o?(a.min=Math.ceil(1.2*e),h=a.min,l="range increased"):e/1.2<o&&(a.min=Math.ceil(e/1.2),h=a.min,l="range increased"),1.2*e>s&&1!==s&&(a.max=Math.ceil(1.2*e),h=a.max,l="range increased"),a.value=e}else a.value=n;var d=document.createElement("input");d.className="vis-configuration vis-config-rangeinput",d.value=Number(a.value);var u=this;a.onchange=function(){d.value=this.value,u._update(Number(this.value),i)},a.oninput=function(){d.value=this.value};var c=this._makeLabel(i[i.length-1],i),p=this._makeItem(i,c,a,d);""!==l&&this.popupHistory[p]!==h&&(this.popupHistory[p]=h,this._setupPopup(l,p))}},{key:"_makeButton",value:function(){var t=this;if(!0===this.options.showButton){var e=document.createElement("div");e.className="vis-configuration vis-config-button",e.innerHTML="generate options",e.onclick=function(){t._printOptions()},e.onmouseover=function(){e.className="vis-configuration vis-config-button hover"},e.onmouseout=function(){e.className="vis-configuration vis-config-button"},this.optionsContainer=document.createElement("div"),this.optionsContainer.className="vis-configuration vis-config-option-container",this.domElements.push(this.optionsContainer),this.domElements.push(e)}}},{key:"_setupPopup",value:function(t,e){var i=this;if(!0===this.initialized&&!0===this.allowCreation&&this.popupCounter<this.popupLimit){var n=document.createElement("div");n.id="vis-configuration-popup",n.className="vis-configuration-popup",n.innerHTML=t,n.onclick=function(){i._removePopup()},this.popupCounter+=1,this.popupDiv={html:n,index:e}}}},{key:"_removePopup",value:function(){void 0!==this.popupDiv.html&&(this.popupDiv.html.parentNode.removeChild(this.popupDiv.html),clearTimeout(this.popupDiv.hideTimeout),clearTimeout(this.popupDiv.deleteTimeout),this.popupDiv={})}},{key:"_showPopupIfNeeded",value:function(){var t=this;if(void 0!==this.popupDiv.html){var e=this.domElements[this.popupDiv.index].getBoundingClientRect();this.popupDiv.html.style.left=e.left+"px",this.popupDiv.html.style.top=e.top-30+"px",document.body.appendChild(this.popupDiv.html),this.popupDiv.hideTimeout=qr((function(){t.popupDiv.html.style.opacity=0}),1500),this.popupDiv.deleteTimeout=qr((function(){t._removePopup()}),1800)}}},{key:"_makeCheckbox",value:function(t,e,i){var n=document.createElement("input");n.type="checkbox",n.className="vis-configuration vis-config-checkbox",n.checked=t,void 0!==e&&(n.checked=e,e!==t&&("object"===zo(t)?e!==t.enabled&&this.changedOptions.push({path:i,value:e}):this.changedOptions.push({path:i,value:e})));var o=this;n.onchange=function(){o._update(this.checked,i)};var s=this._makeLabel(i[i.length-1],i);this._makeItem(i,s,n)}},{key:"_makeTextInput",value:function(t,e,i){var n=document.createElement("input");n.type="text",n.className="vis-configuration vis-config-text",n.value=e,e!==t&&this.changedOptions.push({path:i,value:e});var o=this;n.onchange=function(){o._update(this.value,i)};var s=this._makeLabel(i[i.length-1],i);this._makeItem(i,s,n)}},{key:"_makeColorField",value:function(t,e,i){var n=this,o=t[1],s=document.createElement("div");"none"!==(e=void 0===e?o:e)?(s.className="vis-configuration vis-config-colorBlock",s.style.backgroundColor=e):s.className="vis-configuration vis-config-colorBlock none",e=void 0===e?o:e,s.onclick=function(){n._showColorPicker(e,s,i)};var r=this._makeLabel(i[i.length-1],i);this._makeItem(i,r,s)}},{key:"_showColorPicker",value:function(t,e,i){var n=this;e.onclick=function(){},this.colorPicker.insertTo(e),this.colorPicker.show(),this.colorPicker.setColor(t),this.colorPicker.setUpdateCallback((function(t){var o="rgba("+t.r+","+t.g+","+t.b+","+t.a+")";e.style.backgroundColor=o,n._update(o,i)})),this.colorPicker.setCloseCallback((function(){e.onclick=function(){n._showColorPicker(t,e,i)}}))}},{key:"_handleObject",value:function(t){var e=arguments.length>1&&void 0!==arguments[1]?arguments[1]:[],i=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=!1,o=En(this.options),s=!1;for(var r in t)if(t.hasOwnProperty(r)){n=!0;var a=t[r],l=za.copyAndExtendArray(e,r);if("function"==typeof o&&!1===(n=o(r,e))&&!(a instanceof Array)&&"string"!=typeof a&&"boolean"!=typeof a&&a instanceof Object&&(this.allowCreation=!1,n=this._handleObject(a,l,!0),this.allowCreation=!1===i),!1!==n){s=!0;var h=this._getValue(l);if(a instanceof Array)this._handleArray(a,h,l);else if("string"==typeof a)this._makeTextInput(a,h,l);else if("boolean"==typeof a)this._makeCheckbox(a,h,l);else if(a instanceof Object){var d=!0;if(-1!==Fo(e).call(e,"physics")&&this.moduleOptions.physics.solver!==r&&(d=!1),!0===d)if(void 0!==a.enabled){var u=za.copyAndExtendArray(l,"enabled"),c=this._getValue(u);if(!0===c){var p=this._makeLabel(r,l,!0);this._makeItem(l,p),s=this._handleObject(a,l)||s}else this._makeCheckbox(a,c,l)}else{var m=this._makeLabel(r,l,!0);this._makeItem(l,m),s=this._handleObject(a,l)||s}}else console.error("dont know how to handle",a,r,l)}}return s}},{key:"_handleArray",value:function(t,e,i){"string"==typeof t[0]&&"color"===t[0]?(this._makeColorField(t,e,i),t[1]!==e&&this.changedOptions.push({path:i,value:e})):"string"==typeof t[0]?(this._makeDropdown(t,e,i),t[0]!==e&&this.changedOptions.push({path:i,value:e})):"number"==typeof t[0]&&(this._makeRange(t,e,i),t[0]!==e&&this.changedOptions.push({path:i,value:Number(e)}))}},{key:"_update",value:function(t,e){var i=this._constructOptions(t,e);this.parent.body&&this.parent.body.emitter&&this.parent.body.emitter.emit&&this.parent.body.emitter.emit("configChange",i),this.initialized=!0,this.parent.setOptions(i)}},{key:"_constructOptions",value:function(t,e){var i=arguments.length>2&&void 0!==arguments[2]?arguments[2]:{},n=i;t="false"!==(t="true"===t||t)&&t;for(var o=0;o<e.length;o++)"global"!==e[o]&&(void 0===n[e[o]]&&(n[e[o]]={}),o!==e.length-1?n=n[e[o]]:n[e[o]]=t);return i}},{key:"_printOptions",value:function(){var t=this.getOptions();this.optionsContainer.innerHTML="<pre>var options = "+Vs(t,null,2)+"</pre>"}},{key:"getOptions",value:function(){for(var t={},e=0;e<this.changedOptions.length;e++)this._constructOptions(this.changedOptions[e].value,this.changedOptions[e].path,t);return t}}]),t}(),Gu=function(t){function e(t,i,n,o){var s,r,a,l,h,d,u,c;if(ba(this,e),(c=ar(this,dr(e).call(this))).initTime=new Date,c.itemsDone=!1,!(Bs(c)instanceof e))throw new SyntaxError("Constructor must be called with the new operator");if(!(Zo(n)||n instanceof Aa||n instanceof Xa)&&n instanceof Object){var p=o;o=n,n=p}o&&o.throttleRedraw&&console.warn('Timeline option "throttleRedraw" is DEPRICATED and no longer supported. It will be removed in the next MAJOR release.');var m=Bs(c);if(c.defaultOptions={autoResize:!0,longSelectPressTime:251,orientation:{axis:"bottom",item:"bottom"},moment:Ta},c.options=za.deepExtend({},c.defaultOptions),c._create(t),!o||o&&void 0===o.rtl){var f;c.dom.root.style.visibility="hidden";for(var g=c.dom.root;!f&&g;)f=window.getComputedStyle(g,null).direction,g=g.parentElement;c.options.rtl=f&&"rtl"==f.toLowerCase()}else c.options.rtl=o.rtl;o&&(o.rollingMode&&(c.options.rollingMode=o.rollingMode),o.onInitialDrawComplete&&(c.options.onInitialDrawComplete=o.onInitialDrawComplete),o.onTimeout&&(c.options.onTimeout=o.onTimeout),o.loadingScreenTemplate&&(c.options.loadingScreenTemplate=o.loadingScreenTemplate));var v=document.createElement("div");if(c.options.loadingScreenTemplate){var b,y=va(b=c.options.loadingScreenTemplate).call(b,Bs(c)),x=y(c.dom.loadingScreen);x instanceof Object&&!(x instanceof Element)?y(v):x instanceof Element?(v.innerHTML="",v.appendChild(x)):null!=x&&(v.innerHTML=x)}return c.dom.loadingScreen.appendChild(v),c.components=[],c.body={dom:c.dom,domProps:c.props,emitter:{on:va(s=c.on).call(s,Bs(c)),off:va(r=c.off).call(r,Bs(c)),emit:va(a=c.emit).call(a,Bs(c))},hiddenDates:[],util:{getScale:function(){return m.timeAxis.step.scale},getStep:function(){return m.timeAxis.step.step},toScreen:va(l=m._toScreen).call(l,m),toGlobalScreen:va(h=m._toGlobalScreen).call(h,m),toTime:va(d=m._toTime).call(d,m),toGlobalTime:va(u=m._toGlobalTime).call(u,m)}},c.range=new ml(c.body,c.options),c.components.push(c.range),c.body.range=c.range,c.timeAxis=new Jh(c.body,c.options),c.timeAxis2=null,c.components.push(c.timeAxis),c.currentTime=new pd(c.body,c.options),c.components.push(c.currentTime),c.itemSet=new uu(c.body,c.options),c.components.push(c.itemSet),c.itemsData=null,c.groupsData=null,c.dom.root.onclick=function(t){m.emit("click",m.getEventProperties(t))},c.dom.root.ondblclick=function(t){m.emit("doubleClick",m.getEventProperties(t))},c.dom.root.oncontextmenu=function(t){m.emit("contextmenu",m.getEventProperties(t))},c.dom.root.onmouseover=function(t){m.emit("mouseOver",m.getEventProperties(t))},window.PointerEvent?(c.dom.root.onpointerdown=function(t){m.emit("mouseDown",m.getEventProperties(t))},c.dom.root.onpointermove=function(t){m.emit("mouseMove",m.getEventProperties(t))},c.dom.root.onpointerup=function(t){m.emit("mouseUp",m.getEventProperties(t))}):(c.dom.root.onmousemove=function(t){m.emit("mouseMove",m.getEventProperties(t))},c.dom.root.onmousedown=function(t){m.emit("mouseDown",m.getEventProperties(t))},c.dom.root.onmouseup=function(t){m.emit("mouseUp",m.getEventProperties(t))}),c.initialFitDone=!1,c.on("changed",(function(){if(null!=m.itemsData){if(!m.initialFitDone&&!m.options.rollingMode)if(m.initialFitDone=!0,null!=m.options.start||null!=m.options.end){if(null==m.options.start||null==m.options.end)var t=m.getItemRange();var e=null!=m.options.start?m.options.start:t.min,i=null!=m.options.end?m.options.end:t.max;m.setWindow(e,i,{animation:!1})}else m.fit({animation:!1});m.initialDrawDone||!m.initialRangeChangeDone&&(m.options.start||m.options.end)&&!m.options.rollingMode||(m.initialDrawDone=!0,m.itemSet.initialDrawDone=!0,m.dom.root.style.visibility="visible",m.dom.loadingScreen.parentNode.removeChild(m.dom.loadingScreen),m.options.onInitialDrawComplete&&qr((function(){return m.options.onInitialDrawComplete()}),0))}})),c.on("destroyTimeline",(function(){m.destroy()})),o&&c.setOptions(o),c.body.emitter.on("fit",(function(t){c._onFit(t),c.redraw()})),n&&c.setGroups(n),i&&c.setItems(i),c._redraw(),c}return pr(e,t),xa(e,[{key:"_createConfigurator",value:function(){return new Lu(this,this.dom.container,wu)}},{key:"redraw",value:function(){this.itemSet&&this.itemSet.markDirty({refreshItems:!0}),this._redraw()}},{key:"setOptions",value:function(t){if(!0===fu.validate(t,xu)&&console.log("%cErrors have been found in the supplied options object.",mu),cd.prototype.setOptions.call(this,t),"type"in t&&t.type!==this.options.type){this.options.type=t.type;var e=this.itemsData;if(e){var i=this.getSelection();this.setItems(null),this.setItems(e.rawDS),this.setSelection(i)}}}},{key:"setItems",value:function(t){var e;this.itemsDone=!1,e=t?Ua(t instanceof Aa||t instanceof Xa?t:new Aa(t)):null,this.itemsData&&this.itemsData.dispose(),this.itemsData=e,this.itemSet&&this.itemSet.setItems(null!=e?e.rawDS:null)}},{key:"setGroups",value:function(t){var e;if(t){var i=function(t){return!1!==t.visible};e=t instanceof Aa||t instanceof Xa?new Xa(t,{filter:i}):new Aa(En(t).call(t,i))}else e=null;this.groupsData instanceof Xa&&this.groupsData.setData(null),this.groupsData=e,this.itemSet.setGroups(e)}},{key:"setData",value:function(t){t&&t.groups&&this.setGroups(t.groups),t&&t.items&&this.setItems(t.items)}},{key:"setSelection",value:function(t,e){this.itemSet&&this.itemSet.setSelection(t),e&&e.focus&&this.focus(t,e)}},{key:"getSelection",value:function(){return this.itemSet&&this.itemSet.getSelection()||[]}},{key:"focus",value:function(t,e){if(this.itemsData&&null!=t){var i=Zo(t)?t:[t],n=this.itemsData.get(i),o=null,s=null;if(mn(n).call(n,(function(t){var e=t.start.valueOf(),i="end"in t?t.end.valueOf():t.start.valueOf();(null===o||e<o)&&(o=e),(null===s||i>s)&&(s=i)})),null!==o&&null!==s){var r=this,a=this.itemSet.items[i[0]],l=-1*this._getScrollTop(),h=null,d=function(){var t=Qu(r,a);t.shouldScroll&&t.itemTop!=h.itemTop&&(r._setScrollTop(-t.scrollOffset),r._redraw())},u=!e||void 0===e.zoom||e.zoom,c=(o+s)/2,p=u?1.1*(s-o):Math.max(this.range.end-this.range.start,1.1*(s-o)),m=!e||void 0===e.animation||e.animation;m||(h={shouldScroll:!1,scrollOffset:-1,itemTop:-1}),this.range.setRange(c-p/2,c+p/2,{animation:m},(function(){d(),qr(d,100)}),(function(t,e,i){var n=Qu(r,a);if(!1!==n&&(h||(h=n),h.itemTop!=n.itemTop||h.shouldScroll)){h.itemTop!=n.itemTop&&n.shouldScroll&&(h=n,l=-1*r._getScrollTop());var o=l,s=h.scrollOffset,d=i?s:o+(s-o)*t;r._setScrollTop(-d),e||r._redraw()}}))}}}},{key:"fit",value:function(t,e){var i,n=!t||void 0===t.animation||t.animation;1===this.itemsData.length&&void 0===this.itemsData.get()[0].end?(i=this.getDataRange(),this.moveTo(i.min.valueOf(),{animation:n},e)):(i=this.getItemRange(),this.range.setRange(i.min,i.max,{animation:n},e))}},{key:"getItemRange",value:function(){var t=this,e=this.getDataRange(),i=null!==e.min?e.min.valueOf():null,n=null!==e.max?e.max.valueOf():null,o=null,s=null;if(null!=i&&null!=n){var r=n-i;r<=0&&(r=10);var a=r/this.props.center.width,l={},h=0;if(mn(za).call(za,this.itemSet.items,(function(t,e){if(t.groupShowing){l[e]=t.redraw(!0),h=l[e].length}})),h>0)for(var d=function(t){mn(za).call(za,l,(function(e){e[t]()}))},u=0;u<h;u++)d(u);if(mn(za).call(za,this.itemSet.items,(function(e){var r,l,h=Mu(e),d=Su(e);t.options.rtl?(r=h-(e.getWidthRight()+10)*a,l=d+(e.getWidthLeft()+10)*a):(r=h-(e.getWidthLeft()+10)*a,l=d+(e.getWidthRight()+10)*a),r<i&&(i=r,o=e),l>n&&(n=l,s=e)})),o&&s){var c=o.getWidthLeft()+10,p=s.getWidthRight()+10,m=this.props.center.width-c-p;m>0&&(this.options.rtl?(i=Mu(o)-p*r/m,n=Su(s)+c*r/m):(i=Mu(o)-c*r/m,n=Su(s)+p*r/m))}}return{min:null!=i?new Date(i):null,max:null!=n?new Date(n):null}}},{key:"getDataRange",value:function(){var t,e=null,i=null;this.itemsData&&mn(t=this.itemsData).call(t,(function(t){var n=za.convert(t.start,"Date").valueOf(),o=za.convert(null!=t.end?t.end:t.start,"Date").valueOf();(null===e||n<e)&&(e=n),(null===i||o>i)&&(i=o)}));return{min:null!=e?new Date(e):null,max:null!=i?new Date(i):null}}},{key:"getEventProperties",value:function(t){var e=t.center?t.center.x:t.clientX,i=t.center?t.center.y:t.clientY,n=this.dom.centerContainer.getBoundingClientRect(),o=this.options.rtl?n.right-e:e-n.left,s=i-n.top,r=this.itemSet.itemFromTarget(t),a=this.itemSet.groupFromTarget(t),l=ud.customTimeFromTarget(t),h=this.itemSet.options.snap||null,d=this.body.util.getScale(),u=this.body.util.getStep(),c=this._toTime(o),p=h?h(c,d,u):c,m=za.getTarget(t),f=null;return null!=r?f="item":null!=l?f="custom-time":za.hasParent(m,this.timeAxis.dom.foreground)||this.timeAxis2&&za.hasParent(m,this.timeAxis2.dom.foreground)?f="axis":za.hasParent(m,this.itemSet.dom.labelSet)?f="group-label":za.hasParent(m,this.currentTime.bar)?f="current-time":za.hasParent(m,this.dom.center)&&(f="background"),{event:t,item:r?r.id:null,isCluster:!!r&&!!r.isCluster,items:r?r.items||[]:null,group:a?a.groupId:null,customTime:l?l.options.id:null,what:f,pageX:t.srcEvent?t.srcEvent.pageX:t.pageX,pageY:t.srcEvent?t.srcEvent.pageY:t.pageY,x:o,y:s,time:c,snappedTime:p}}},{key:"toggleRollingMode",value:function(){this.range.rolling?this.range.stopRolling():(null==this.options.rollingMode&&this.setOptions(this.options),this.range.startRolling())}},{key:"_redraw",value:function(){cd.prototype._redraw.call(this)}},{key:"_onFit",value:function(t){var e=t.start,i=t.end,n=t.animation;i?this.range.setRange(e,i,{animation:n}):this.moveTo(e.valueOf(),{animation:n})}}]),e}(cd);function Mu(t){return za.convert(t.data.start,"Date").valueOf()}function Su(t){var e=null!=t.data.end?t.data.end:t.data.start;return za.convert(e,"Date").valueOf()}function Qu(t,e){if(!e.parent)return!1;var i=t.options.rtl?t.props.rightContainer.height:t.props.leftContainer.height,n=t.props.center.height,o=e.parent,s=o.top,r=!0,a=t.timeAxis.options.orientation.axis,l=function(){return"bottom"==a?o.height-e.top-e.height:e.top},h=-1*t._getScrollTop(),d=s+l(),u=e.height;return d<h?s+i<=s+l()+u&&(s+=l()-t.itemSet.options.margin.item.vertical):d+u>h+i?s+=l()+u-i+t.itemSet.options.margin.item.vertical:r=!1,{shouldScroll:r,scrollOffset:s=Math.min(s,n-i),itemTop:d}}var Nu=function(){function t(e,i,n,o,s,r){var a=arguments.length>6&&void 0!==arguments[6]&&arguments[6],l=arguments.length>7&&void 0!==arguments[7]&&arguments[7];if(ba(this,t),this.majorSteps=[1,2,5,10],this.minorSteps=[.25,.5,1,2],this.customLines=null,this.containerHeight=s,this.majorCharHeight=r,this._start=e,this._end=i,this.scale=1,this.minorStepIdx=-1,this.magnitudefactor=1,this.determineScale(),this.zeroAlign=a,this.autoScaleStart=n,this.autoScaleEnd=o,this.formattingFunction=l,n||o){var h=this,d=function(t){var e=t-t%(h.magnitudefactor*h.minorSteps[h.minorStepIdx]);return t%(h.magnitudefactor*h.minorSteps[h.minorStepIdx])>h.magnitudefactor*h.minorSteps[h.minorStepIdx]*.5?e+h.magnitudefactor*h.minorSteps[h.minorStepIdx]:e};n&&(this._start-=2*this.magnitudefactor*this.minorSteps[this.minorStepIdx],this._start=d(this._start)),o&&(this._end+=this.magnitudefactor*this.minorSteps[this.minorStepIdx],this._end=d(this._end)),this.determineScale()}}return xa(t,[{key:"setCharHeight",value:function(t){this.majorCharHeight=t}},{key:"setHeight",value:function(t){this.containerHeight=t}},{key:"determineScale",value:function(){var t=this._end-this._start;this.scale=this.containerHeight/t;var e=this.majorCharHeight/this.scale,i=t>0?Math.round(Math.log(t)/Math.LN10):0;this.minorStepIdx=-1,this.magnitudefactor=Math.pow(10,i);var n=0;i<0&&(n=i);for(var o=!1,s=n;Math.abs(s)<=Math.abs(i);s++){this.magnitudefactor=Math.pow(10,s);for(var r=0;r<this.minorSteps.length;r++){if(this.magnitudefactor*this.minorSteps[r]>=e){o=!0,this.minorStepIdx=r;break}}if(!0===o)break}}},{key:"is_major",value:function(t){return t%(this.magnitudefactor*this.majorSteps[this.minorStepIdx])==0}},{key:"getStep",value:function(){return this.magnitudefactor*this.minorSteps[this.minorStepIdx]}},{key:"getFirstMajor",value:function(){var t=this.magnitudefactor*this.majorSteps[this.minorStepIdx];return this.convertValue(this._start+(t-this._start%t)%t)}},{key:"formatValue",value:function(t){var e=t.toPrecision(5);return"function"==typeof this.formattingFunction&&(e=this.formattingFunction(t)),"number"==typeof e?"".concat(e):"string"==typeof e?e:t.toPrecision(5)}},{key:"getLines",value:function(){for(var t=[],e=this.getStep(),i=(e-this._start%e)%e,n=this._start+i;this._end-n>1e-5;n+=e)n!=this._start&&t.push({major:this.is_major(n),y:this.convertValue(n),val:this.formatValue(n)});return t}},{key:"followScale",value:function(t){var e=this.minorStepIdx,i=this._start,n=this._end,o=this,s=function(){o.magnitudefactor*=2},r=function(){o.magnitudefactor/=2};t.minorStepIdx<=1&&this.minorStepIdx<=1||t.minorStepIdx>1&&this.minorStepIdx>1||(t.minorStepIdx<this.minorStepIdx?(this.minorStepIdx=1,2==e||s(),s()):(this.minorStepIdx=2,1==e||r(),r()));for(var a=t.convertValue(0),l=t.getStep()*t.scale,h=!1,d=0;!h&&d++<5;){this.scale=l/(this.minorSteps[this.minorStepIdx]*this.magnitudefactor);var u=this.containerHeight/this.scale;this._start=i,this._end=this._start+u;var c=this._end*this.scale,p=this.magnitudefactor*this.majorSteps[this.minorStepIdx],m=this.getFirstMajor()-t.getFirstMajor();if(this.zeroAlign){var f=a-c;this._end+=f/this.scale,this._start=this._end-u}else this.autoScaleStart?(this._start-=m/this.scale,this._end=this._start+u):(this._start+=p-m/this.scale,this._end=this._start+u);if(!this.autoScaleEnd&&this._end>n+1e-5)r(),h=!1;else{if(!this.autoScaleStart&&this._start<i-1e-5){if(!(this.zeroAlign&&i>=0)){r(),h=!1;continue}console.warn("Can't adhere to given 'min' range, due to zeroalign")}this.autoScaleStart&&this.autoScaleEnd&&u<n-i?(s(),h=!1):h=!0}}}},{key:"convertValue",value:function(t){return this.containerHeight-(t-this._start)*this.scale}},{key:"screenToValue",value:function(t){return(this.containerHeight-t)/this.scale+this._start}}]),t}();Ph(".vis-panel.vis-background.vis-horizontal .vis-grid.vis-horizontal{position:absolute;width:100%;height:0;border-bottom:1px solid}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-minor{border-color:#e5e5e5}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-major{border-color:#bfbfbf}.vis-data-axis .vis-y-axis.vis-major{width:100%;position:absolute;color:#4d4d4d;white-space:nowrap}.vis-data-axis .vis-y-axis.vis-major.vis-measure{padding:0;margin:0;border:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-minor{position:absolute;width:100%;color:#bebebe;white-space:nowrap}.vis-data-axis .vis-y-axis.vis-minor.vis-measure{padding:0;margin:0;border:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-title{position:absolute;color:#4d4d4d;white-space:nowrap;bottom:20px;text-align:center}.vis-data-axis .vis-y-axis.vis-title.vis-measure{padding:0;margin:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-title.vis-left{bottom:0;-webkit-transform-origin:left top;-moz-transform-origin:left top;-ms-transform-origin:left top;-o-transform-origin:left top;transform-origin:left bottom;-webkit-transform:rotate(-90deg);-moz-transform:rotate(-90deg);-ms-transform:rotate(-90deg);-o-transform:rotate(-90deg);transform:rotate(-90deg)}.vis-data-axis .vis-y-axis.vis-title.vis-right{bottom:0;-webkit-transform-origin:right bottom;-moz-transform-origin:right bottom;-ms-transform-origin:right bottom;-o-transform-origin:right bottom;transform-origin:right bottom;-webkit-transform:rotate(90deg);-moz-transform:rotate(90deg);-ms-transform:rotate(90deg);-o-transform:rotate(90deg);transform:rotate(90deg)}.vis-legend{background-color:rgba(247,252,255,.65);padding:5px;border:1px solid #b3b3b3;box-shadow:2px 2px 10px hsla(0,0%,60.4%,.55)}.vis-legend-text{white-space:nowrap;display:inline-block}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbImRhdGFheGlzLmNzcyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFDQSxrRUFDRSxpQkFBa0IsQ0FDbEIsVUFBVyxDQUNYLFFBQVMsQ0FDVCx1QkFDRixDQUVBLDZEQUNFLG9CQUNGLENBRUEsNkRBQ0Usb0JBQ0YsQ0FHQSxxQ0FDRSxVQUFXLENBQ1gsaUJBQWtCLENBQ2xCLGFBQWMsQ0FDZCxrQkFDRixDQUVBLGlEQUNFLFNBQVUsQ0FDVixRQUFTLENBQ1QsUUFBUyxDQUNULGlCQUFrQixDQUNsQixVQUNGLENBR0EscUNBQ0UsaUJBQWtCLENBQ2xCLFVBQVcsQ0FDWCxhQUFjLENBQ2Qsa0JBQ0YsQ0FFQSxpREFDRSxTQUFVLENBQ1YsUUFBUyxDQUNULFFBQVMsQ0FDVCxpQkFBa0IsQ0FDbEIsVUFDRixDQUVBLHFDQUNFLGlCQUFrQixDQUNsQixhQUFjLENBQ2Qsa0JBQW1CLENBQ25CLFdBQVksQ0FDWixpQkFDRixDQUVBLGlEQUNFLFNBQVUsQ0FDVixRQUFTLENBQ1QsaUJBQWtCLENBQ2xCLFVBQ0YsQ0FFQSw4Q0FDRSxRQUFTLENBQ1QsaUNBQWtDLENBQ2xDLDhCQUErQixDQUMvQiw2QkFBOEIsQ0FDOUIsNEJBQTZCLENBQzdCLDRCQUE2QixDQUM3QixnQ0FBaUMsQ0FDakMsNkJBQThCLENBQzlCLDRCQUE2QixDQUM3QiwyQkFBNEIsQ0FDNUIsd0JBQ0YsQ0FFQSwrQ0FDRSxRQUFTLENBQ1QscUNBQXNDLENBQ3RDLGtDQUFtQyxDQUNuQyxpQ0FBa0MsQ0FDbEMsZ0NBQWlDLENBQ2pDLDZCQUE4QixDQUM5QiwrQkFBZ0MsQ0FDaEMsNEJBQTZCLENBQzdCLDJCQUE0QixDQUM1QiwwQkFBMkIsQ0FDM0IsdUJBQ0YsQ0FFQSxZQUNFLHNDQUEyQyxDQUMzQyxXQUFZLENBQ1osd0JBQXlCLENBQ3pCLDRDQUNGLENBRUEsaUJBRUUsa0JBQW1CLENBQ25CLG9CQUNGIiwiZmlsZSI6ImRhdGFheGlzLmNzcyIsInNvdXJjZXNDb250ZW50IjpbIlxuLnZpcy1wYW5lbC52aXMtYmFja2dyb3VuZC52aXMtaG9yaXpvbnRhbCAudmlzLWdyaWQudmlzLWhvcml6b250YWwge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHdpZHRoOiAxMDAlO1xuICBoZWlnaHQ6IDA7XG4gIGJvcmRlci1ib3R0b206IDFweCBzb2xpZDtcbn1cblxuLnZpcy1wYW5lbC52aXMtYmFja2dyb3VuZC52aXMtaG9yaXpvbnRhbCAudmlzLWdyaWQudmlzLW1pbm9yIHtcbiAgYm9yZGVyLWNvbG9yOiAjZTVlNWU1O1xufVxuXG4udmlzLXBhbmVsLnZpcy1iYWNrZ3JvdW5kLnZpcy1ob3Jpem9udGFsIC52aXMtZ3JpZC52aXMtbWFqb3Ige1xuICBib3JkZXItY29sb3I6ICNiZmJmYmY7XG59XG5cblxuLnZpcy1kYXRhLWF4aXMgLnZpcy15LWF4aXMudmlzLW1ham9yIHtcbiAgd2lkdGg6IDEwMCU7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgY29sb3I6ICM0ZDRkNGQ7XG4gIHdoaXRlLXNwYWNlOiBub3dyYXA7XG59XG5cbi52aXMtZGF0YS1heGlzIC52aXMteS1heGlzLnZpcy1tYWpvci52aXMtbWVhc3VyZSB7XG4gIHBhZGRpbmc6IDA7XG4gIG1hcmdpbjogMDtcbiAgYm9yZGVyOiAwO1xuICB2aXNpYmlsaXR5OiBoaWRkZW47XG4gIHdpZHRoOiBhdXRvO1xufVxuXG5cbi52aXMtZGF0YS1heGlzIC52aXMteS1heGlzLnZpcy1taW5vciB7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgd2lkdGg6IDEwMCU7XG4gIGNvbG9yOiAjYmViZWJlO1xuICB3aGl0ZS1zcGFjZTogbm93cmFwO1xufVxuXG4udmlzLWRhdGEtYXhpcyAudmlzLXktYXhpcy52aXMtbWlub3IudmlzLW1lYXN1cmUge1xuICBwYWRkaW5nOiAwO1xuICBtYXJnaW46IDA7XG4gIGJvcmRlcjogMDtcbiAgdmlzaWJpbGl0eTogaGlkZGVuO1xuICB3aWR0aDogYXV0bztcbn1cblxuLnZpcy1kYXRhLWF4aXMgLnZpcy15LWF4aXMudmlzLXRpdGxlIHtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICBjb2xvcjogIzRkNGQ0ZDtcbiAgd2hpdGUtc3BhY2U6IG5vd3JhcDtcbiAgYm90dG9tOiAyMHB4O1xuICB0ZXh0LWFsaWduOiBjZW50ZXI7XG59XG5cbi52aXMtZGF0YS1heGlzIC52aXMteS1heGlzLnZpcy10aXRsZS52aXMtbWVhc3VyZSB7XG4gIHBhZGRpbmc6IDA7XG4gIG1hcmdpbjogMDtcbiAgdmlzaWJpbGl0eTogaGlkZGVuO1xuICB3aWR0aDogYXV0bztcbn1cblxuLnZpcy1kYXRhLWF4aXMgLnZpcy15LWF4aXMudmlzLXRpdGxlLnZpcy1sZWZ0IHtcbiAgYm90dG9tOiAwO1xuICAtd2Via2l0LXRyYW5zZm9ybS1vcmlnaW46IGxlZnQgdG9wO1xuICAtbW96LXRyYW5zZm9ybS1vcmlnaW46IGxlZnQgdG9wO1xuICAtbXMtdHJhbnNmb3JtLW9yaWdpbjogbGVmdCB0b3A7XG4gIC1vLXRyYW5zZm9ybS1vcmlnaW46IGxlZnQgdG9wO1xuICB0cmFuc2Zvcm0tb3JpZ2luOiBsZWZ0IGJvdHRvbTtcbiAgLXdlYmtpdC10cmFuc2Zvcm06IHJvdGF0ZSgtOTBkZWcpO1xuICAtbW96LXRyYW5zZm9ybTogcm90YXRlKC05MGRlZyk7XG4gIC1tcy10cmFuc2Zvcm06IHJvdGF0ZSgtOTBkZWcpO1xuICAtby10cmFuc2Zvcm06IHJvdGF0ZSgtOTBkZWcpO1xuICB0cmFuc2Zvcm06IHJvdGF0ZSgtOTBkZWcpO1xufVxuXG4udmlzLWRhdGEtYXhpcyAudmlzLXktYXhpcy52aXMtdGl0bGUudmlzLXJpZ2h0IHtcbiAgYm90dG9tOiAwO1xuICAtd2Via2l0LXRyYW5zZm9ybS1vcmlnaW46IHJpZ2h0IGJvdHRvbTtcbiAgLW1vei10cmFuc2Zvcm0tb3JpZ2luOiByaWdodCBib3R0b207XG4gIC1tcy10cmFuc2Zvcm0tb3JpZ2luOiByaWdodCBib3R0b207XG4gIC1vLXRyYW5zZm9ybS1vcmlnaW46IHJpZ2h0IGJvdHRvbTtcbiAgdHJhbnNmb3JtLW9yaWdpbjogcmlnaHQgYm90dG9tO1xuICAtd2Via2l0LXRyYW5zZm9ybTogcm90YXRlKDkwZGVnKTtcbiAgLW1vei10cmFuc2Zvcm06IHJvdGF0ZSg5MGRlZyk7XG4gIC1tcy10cmFuc2Zvcm06IHJvdGF0ZSg5MGRlZyk7XG4gIC1vLXRyYW5zZm9ybTogcm90YXRlKDkwZGVnKTtcbiAgdHJhbnNmb3JtOiByb3RhdGUoOTBkZWcpO1xufVxuXG4udmlzLWxlZ2VuZCB7XG4gIGJhY2tncm91bmQtY29sb3I6IHJnYmEoMjQ3LCAyNTIsIDI1NSwgMC42NSk7XG4gIHBhZGRpbmc6IDVweDtcbiAgYm9yZGVyOiAxcHggc29saWQgI2IzYjNiMztcbiAgYm94LXNoYWRvdzogMnB4IDJweCAxMHB4IHJnYmEoMTU0LCAxNTQsIDE1NCwgMC41NSk7XG59XG5cbi52aXMtbGVnZW5kLXRleHQge1xuICAvKmZvbnQtc2l6ZTogMTBweDsqL1xuICB3aGl0ZS1zcGFjZTogbm93cmFwO1xuICBkaXNwbGF5OiBpbmxpbmUtYmxvY2tcbn0iXX0= */");var Fu=function(t){function e(t,i,n,o){var s;ba(this,e),(s=ar(this,dr(e).call(this))).id=Id(),s.body=t,s.defaultOptions={orientation:"left",showMinorLabels:!0,showMajorLabels:!0,showWeekScale:!1,icons:!1,majorLinesOffset:7,minorLinesOffset:4,labelOffsetX:10,labelOffsetY:2,iconWidth:20,width:"40px",visible:!0,alignZeros:!0,left:{range:{min:void 0,max:void 0},format:function(t){return"".concat(Pa(t.toPrecision(3)))},title:{text:void 0,style:void 0}},right:{range:{min:void 0,max:void 0},format:function(t){return"".concat(Pa(t.toPrecision(3)))},title:{text:void 0,style:void 0}}},s.linegraphOptions=o,s.linegraphSVG=n,s.props={},s.DOMelements={lines:{},labels:{},title:{}},s.dom={},s.scale=void 0,s.range={start:0,end:0},s.options=za.extend({},s.defaultOptions),s.conversionFactor=1,s.setOptions(i),s.width=Number("".concat(s.options.width).replace("px","")),s.minWidth=s.width,s.height=s.linegraphSVG.getBoundingClientRect().height,s.hidden=!1,s.stepPixels=25,s.zeroCrossing=-1,s.amountOfSteps=-1,s.lineOffset=0,s.master=!0,s.masterAxis=null,s.svgElements={},s.iconsRemoved=!1,s.groups={},s.amountOfGroups=0,s._create(),null==s.scale&&s._redrawLabels(),s.framework={svg:s.svg,svgElements:s.svgElements,options:s.options,groups:s.groups};var r=Bs(s);return s.body.emitter.on("verticalDrag",(function(){r.dom.lineContainer.style.top="".concat(r.body.domProps.scrollTop,"px")})),s}return pr(e,t),xa(e,[{key:"addGroup",value:function(t,e){this.groups.hasOwnProperty(t)||(this.groups[t]=e),this.amountOfGroups+=1}},{key:"updateGroup",value:function(t,e){this.groups.hasOwnProperty(t)||(this.amountOfGroups+=1),this.groups[t]=e}},{key:"removeGroup",value:function(t){this.groups.hasOwnProperty(t)&&(delete this.groups[t],this.amountOfGroups-=1)}},{key:"setOptions",value:function(t){if(t){var e=!1;this.options.orientation!=t.orientation&&void 0!==t.orientation&&(e=!0);za.selectiveDeepExtend(["orientation","showMinorLabels","showMajorLabels","icons","majorLinesOffset","minorLinesOffset","labelOffsetX","labelOffsetY","iconWidth","width","visible","left","right","alignZeros"],this.options,t),this.minWidth=Number("".concat(this.options.width).replace("px","")),!0===e&&this.dom.frame&&(this.hide(),this.show())}}},{key:"_create",value:function(){this.dom.frame=document.createElement("div"),this.dom.frame.style.width=this.options.width,this.dom.frame.style.height=this.height,this.dom.lineContainer=document.createElement("div"),this.dom.lineContainer.style.width="100%",this.dom.lineContainer.style.height=this.height,this.dom.lineContainer.style.position="relative",this.dom.lineContainer.style.visibility="visible",this.dom.lineContainer.style.display="block",this.svg=document.createElementNS("http://www.w3.org/2000/svg","svg"),this.svg.style.position="absolute",this.svg.style.top="0px",this.svg.style.height="100%",this.svg.style.width="100%",this.svg.style.display="block",this.dom.frame.appendChild(this.svg)}},{key:"_redrawGroupIcons",value:function(){var t;Cs(this.svgElements);var e=this.options.iconWidth,i=11.5;t="left"===this.options.orientation?4:this.width-e-4;var n=Yi(this.groups);wr(n).call(n,(function(t,e){return t<e?-1:1}));var o=!0,s=!1,r=void 0;try{for(var a,l=Fi(n);!(o=(a=l.next()).done);o=!0){var h=a.value;!0!==this.groups[h].visible||void 0!==this.linegraphOptions.visibility[h]&&!0!==this.linegraphOptions.visibility[h]||(this.groups[h].getLegend(e,15,this.framework,t,i),i+=19)}}catch(t){s=!0,r=t}finally{try{o||null==l.return||l.return()}finally{if(s)throw r}}Ls(this.svgElements),this.iconsRemoved=!1}},{key:"_cleanupIcons",value:function(){!1===this.iconsRemoved&&(Cs(this.svgElements),Ls(this.svgElements),this.iconsRemoved=!0)}},{key:"show",value:function(){this.hidden=!1,this.dom.frame.parentNode||("left"===this.options.orientation?this.body.dom.left.appendChild(this.dom.frame):this.body.dom.right.appendChild(this.dom.frame)),this.dom.lineContainer.parentNode||this.body.dom.backgroundHorizontal.appendChild(this.dom.lineContainer),this.dom.lineContainer.style.display="block"}},{key:"hide",value:function(){this.hidden=!0,this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame),this.dom.lineContainer.style.display="none"}},{key:"setRange",value:function(t,e){this.range.start=t,this.range.end=e}},{key:"redraw",value:function(){var t=!1,e=0;for(var i in this.dom.lineContainer.style.top="".concat(this.body.domProps.scrollTop,"px"),this.groups)this.groups.hasOwnProperty(i)&&(!0!==this.groups[i].visible||void 0!==this.linegraphOptions.visibility[i]&&!0!==this.linegraphOptions.visibility[i]||e++);if(0===this.amountOfGroups||0===e)this.hide();else{this.show(),this.height=Number(this.linegraphSVG.style.height.replace("px","")),this.dom.lineContainer.style.height="".concat(this.height,"px"),this.width=!0===this.options.visible?Number("".concat(this.options.width).replace("px","")):0;var n=this.props,o=this.dom.frame;o.className="vis-data-axis",this._calculateCharSize();var s=this.options.orientation,r=this.options.showMinorLabels,a=this.options.showMajorLabels,l=this.body.dom.backgroundHorizontal.offsetWidth;n.minorLabelHeight=r?n.minorCharHeight:0,n.majorLabelHeight=a?n.majorCharHeight:0,n.minorLineWidth=l-this.lineOffset-this.width+2*this.options.minorLinesOffset,n.minorLineHeight=1,n.majorLineWidth=l-this.lineOffset-this.width+2*this.options.majorLinesOffset,n.majorLineHeight=1,"left"===s?(o.style.top="0",o.style.left="0",o.style.bottom="",o.style.width="".concat(this.width,"px"),o.style.height="".concat(this.height,"px"),this.props.width=this.body.domProps.left.width,this.props.height=this.body.domProps.left.height):(o.style.top="",o.style.bottom="0",o.style.left="0",o.style.width="".concat(this.width,"px"),o.style.height="".concat(this.height,"px"),this.props.width=this.body.domProps.right.width,this.props.height=this.body.domProps.right.height),t=this._redrawLabels(),t=this._isResized()||t,!0===this.options.icons?this._redrawGroupIcons():this._cleanupIcons(),this._redrawTitle(s)}return t}},{key:"_redrawLabels",value:function(){var t=this,e=!1;Cs(this.DOMelements.lines),Cs(this.DOMelements.labels);var i=this.options.orientation,n=null!=this.options[i].range?this.options[i].range:{},o=!0;null!=n.max&&(this.range.end=n.max,o=!1);var s=!0;null!=n.min&&(this.range.start=n.min,s=!1),this.scale=new Nu(this.range.start,this.range.end,s,o,this.dom.frame.offsetHeight,this.props.majorCharHeight,this.options.alignZeros,this.options[i].format),!1===this.master&&null!=this.masterAxis?(this.scale.followScale(this.masterAxis.scale),this.dom.lineContainer.style.display="none"):this.dom.lineContainer.style.display="block",this.maxLabelSize=0;var r=this.scale.getLines();mn(r).call(r,(function(e){var n=e.y,o=e.major;t.options.showMinorLabels&&!1===o&&t._redrawLabel(n-2,e.val,i,"vis-y-axis vis-minor",t.props.minorCharHeight),o&&n>=0&&t._redrawLabel(n-2,e.val,i,"vis-y-axis vis-major",t.props.majorCharHeight),!0===t.master&&(o?t._redrawLine(n,i,"vis-grid vis-horizontal vis-major",t.options.majorLinesOffset,t.props.majorLineWidth):t._redrawLine(n,i,"vis-grid vis-horizontal vis-minor",t.options.minorLinesOffset,t.props.minorLineWidth))}));var a=0;void 0!==this.options[i].title&&void 0!==this.options[i].title.text&&(a=this.props.titleCharHeight);var l=!0===this.options.icons?Math.max(this.options.iconWidth,a)+this.options.labelOffsetX+15:a+this.options.labelOffsetX+15;return this.maxLabelSize>this.width-l&&!0===this.options.visible?(this.width=this.maxLabelSize+l,this.options.width="".concat(this.width,"px"),Ls(this.DOMelements.lines),Ls(this.DOMelements.labels),this.redraw(),e=!0):this.maxLabelSize<this.width-l&&!0===this.options.visible&&this.width>this.minWidth?(this.width=Math.max(this.minWidth,this.maxLabelSize+l),this.options.width="".concat(this.width,"px"),Ls(this.DOMelements.lines),Ls(this.DOMelements.labels),this.redraw(),e=!0):(Ls(this.DOMelements.lines),Ls(this.DOMelements.labels),e=!1),e}},{key:"convertValue",value:function(t){return this.scale.convertValue(t)}},{key:"screenToValue",value:function(t){return this.scale.screenToValue(t)}},{key:"_redrawLabel",value:function(t,e,i,n,o){var s=Ss("div",this.DOMelements.labels,this.dom.frame);s.className=n,s.innerHTML=e,"left"===i?(s.style.left="-".concat(this.options.labelOffsetX,"px"),s.style.textAlign="right"):(s.style.right="-".concat(this.options.labelOffsetX,"px"),s.style.textAlign="left"),s.style.top="".concat(t-.5*o+this.options.labelOffsetY,"px"),e+="";var r=Math.max(this.props.majorCharWidth,this.props.minorCharWidth);this.maxLabelSize<e.length*r&&(this.maxLabelSize=e.length*r)}},{key:"_redrawLine",value:function(t,e,i,n,o){if(!0===this.master){var s=Ss("div",this.DOMelements.lines,this.dom.lineContainer);s.className=i,s.innerHTML="","left"===e?s.style.left="".concat(this.width-n,"px"):s.style.right="".concat(this.width-n,"px"),s.style.width="".concat(o,"px"),s.style.top="".concat(t,"px")}}},{key:"_redrawTitle",value:function(t){if(Cs(this.DOMelements.title),void 0!==this.options[t].title&&void 0!==this.options[t].title.text){var e=Ss("div",this.DOMelements.title,this.dom.frame);e.className="vis-y-axis vis-title vis-".concat(t),e.innerHTML=this.options[t].title.text,void 0!==this.options[t].title.style&&za.addCssText(e,this.options[t].title.style),"left"===t?e.style.left="".concat(this.props.titleCharHeight,"px"):e.style.right="".concat(this.props.titleCharHeight,"px"),e.style.width="".concat(this.height,"px")}Ls(this.DOMelements.title)}},{key:"_calculateCharSize",value:function(){if(!("minorCharHeight"in this.props)){var t=document.createTextNode("0"),e=document.createElement("div");e.className="vis-y-axis vis-minor vis-measure",e.appendChild(t),this.dom.frame.appendChild(e),this.props.minorCharHeight=e.clientHeight,this.props.minorCharWidth=e.clientWidth,this.dom.frame.removeChild(e)}if(!("majorCharHeight"in this.props)){var i=document.createTextNode("0"),n=document.createElement("div");n.className="vis-y-axis vis-major vis-measure",n.appendChild(i),this.dom.frame.appendChild(n),this.props.majorCharHeight=n.clientHeight,this.props.majorCharWidth=n.clientWidth,this.dom.frame.removeChild(n)}if(!("titleCharHeight"in this.props)){var o=document.createTextNode("0"),s=document.createElement("div");s.className="vis-y-axis vis-title vis-measure",s.appendChild(o),this.dom.frame.appendChild(s),this.props.titleCharHeight=s.clientHeight,this.props.titleCharWidth=s.clientWidth,this.dom.frame.removeChild(s)}}}]),e}(Ja);function Zu(t,e){}function Bu(t,e){return{style:(e=void 0===e?{}:e).style||t.options.drawPoints.style,styles:e.styles||t.options.drawPoints.styles,size:e.size||t.options.drawPoints.size,className:e.className||t.className}}function Ou(t,e){}function Wu(t,e){}function Au(t,e,i,n){this.id=e;this.options=za.selectiveBridgeObject(["sampling","style","sort","yAxisOrientation","barChart","drawPoints","shaded","interpolation","zIndex","excludeFromStacking","excludeFromLegend"],i),this.usingDefaultStyle=void 0===t.className,this.groupsUsingDefaultStyles=n,this.zeroPosition=0,this.update(t),1==this.usingDefaultStyle&&(this.groupsUsingDefaultStyles[0]+=1),this.itemsData=[],this.visible=void 0===t.visible||t.visible}function Xu(t,e,i,n){this.body=t,this.defaultOptions={enabled:!1,icons:!0,iconSize:20,iconSpacing:6,left:{visible:!0,position:"top-left"},right:{visible:!0,position:"top-right"}},this.side=i,this.options=za.extend({},this.defaultOptions),this.linegraphOptions=n,this.svgElements={},this.dom={},this.groups={},this.amountOfGroups=0,this._create(),this.framework={svg:this.svg,svgElements:this.svgElements,options:this.options,groups:this.groups},this.setOptions(e)}Zu.draw=function(t,e,i,n){n=n||0;for(var o=function(t,e){var i=void 0;t.options&&t.options.drawPoints&&t.options.drawPoints.onRender&&"function"==typeof t.options.drawPoints.onRender&&(i=t.options.drawPoints.onRender);e.group.options&&e.group.options.drawPoints&&e.group.options.drawPoints.onRender&&"function"==typeof e.group.options.drawPoints.onRender&&(i=e.group.options.drawPoints.onRender);return i}(i,e),s=0;s<t.length;s++)if(o){var r=o(t[s],e);!0!==r&&"object"!==zo(r)||Qs(t[s].screen_x+n,t[s].screen_y,Bu(e,r),i.svgElements,i.svg,t[s].label)}else Qs(t[s].screen_x+n,t[s].screen_y,Bu(e),i.svgElements,i.svg,t[s].label)},Zu.drawIcon=function(t,e,i,n,o,s){var r=.5*o,a=Ms("rect",s.svgElements,s.svg);a.setAttributeNS(null,"x",e),a.setAttributeNS(null,"y",i-r),a.setAttributeNS(null,"width",n),a.setAttributeNS(null,"height",2*r),a.setAttributeNS(null,"class","vis-outline"),Qs(e+.5*n,i,Bu(t),s.svgElements,s.svg)},Ou.drawIcon=function(t,e,i,n,o,s){var r=.5*o,a=Ms("rect",s.svgElements,s.svg);a.setAttributeNS(null,"x",e),a.setAttributeNS(null,"y",i-r),a.setAttributeNS(null,"width",n),a.setAttributeNS(null,"height",2*r),a.setAttributeNS(null,"class","vis-outline");var l=Math.round(.3*n),h=t.options.barChart.width/l,d=Math.round(.4*o),u=Math.round(.75*o),c=Math.round((n-2*l)/3);if(Ns(e+.5*l+c,i+r-d-1,l,d,t.className+" vis-bar",s.svgElements,s.svg,t.style),Ns(e+1.5*l+c+2,i+r-u-1,l,u,t.className+" vis-bar",s.svgElements,s.svg,t.style),1==t.options.drawPoints.enabled){var p={style:t.options.drawPoints.style,styles:t.options.drawPoints.styles,size:t.options.drawPoints.size/h,className:t.className};Qs(e+.5*l+c,i+r-d-1,p,s.svgElements,s.svg),Qs(e+1.5*l+c+2,i+r-u-1,p,s.svgElements,s.svg)}},Ou.draw=function(t,e,i){var n,o,s,r,a,l,h=[],d={},u=0;for(a=0;a<t.length;a++)if("bar"===(r=i.groups[t[a]]).options.style&&!0===r.visible&&(void 0===i.options.groups.visibility[t[a]]||!0===i.options.groups.visibility[t[a]]))for(l=0;l<e[t[a]].length;l++)h.push({screen_x:e[t[a]][l].screen_x,screen_end:e[t[a]][l].screen_end,screen_y:e[t[a]][l].screen_y,x:e[t[a]][l].x,end:e[t[a]][l].end,y:e[t[a]][l].y,groupId:t[a],label:e[t[a]][l].label}),u+=1;if(0!==u)for(wr(h).call(h,(function(t,e){return t.screen_x===e.screen_x?t.groupId<e.groupId?-1:1:t.screen_x-e.screen_x})),Ou._getDataIntersections(d,h),a=0;a<h.length;a++){var c=null!=(r=i.groups[h[a].groupId]).options.barChart.minWidth?r.options.barChart.minWidth:.1*r.options.barChart.width,p=0;if(void 0===d[o=h[a].screen_x])a+1<h.length&&(n=Math.abs(h[a+1].screen_x-o)),s=Ou._getSafeDrawData(n,r,c);else{var m=a+(d[o].amount-d[o].resolved);m<h.length&&(n=Math.abs(h[m].screen_x-o)),s=Ou._getSafeDrawData(n,r,c),d[o].resolved+=1,!0===r.options.stack&&!0!==r.options.excludeFromStacking?h[a].screen_y<r.zeroPosition?(p=d[o].accumulatedNegative,d[o].accumulatedNegative+=r.zeroPosition-h[a].screen_y):(p=d[o].accumulatedPositive,d[o].accumulatedPositive+=r.zeroPosition-h[a].screen_y):!0===r.options.barChart.sideBySide&&(s.width=s.width/d[o].amount,s.offset+=d[o].resolved*s.width-.5*s.width*(d[o].amount+1))}var f=s.width,g=h[a].screen_x;if(null!=h[a].screen_end?g+=.5*(f=h[a].screen_end-h[a].screen_x):g+=s.offset,Ns(g,h[a].screen_y-p,f,r.zeroPosition-h[a].screen_y,r.className+" vis-bar",i.svgElements,i.svg,r.style),!0===r.options.drawPoints.enabled){var v={screen_x:h[a].screen_x,screen_y:h[a].screen_y-p,x:h[a].x,y:h[a].y,groupId:h[a].groupId,label:h[a].label};Zu.draw([v],r,i,s.offset)}}},Ou._getDataIntersections=function(t,e){for(var i,n=0;n<e.length;n++)n+1<e.length&&(i=Math.abs(e[n+1].screen_x-e[n].screen_x)),n>0&&(i=Math.min(i,Math.abs(e[n-1].screen_x-e[n].screen_x))),0===i&&(void 0===t[e[n].screen_x]&&(t[e[n].screen_x]={amount:0,resolved:0,accumulatedPositive:0,accumulatedNegative:0}),t[e[n].screen_x].amount+=1)},Ou._getSafeDrawData=function(t,e,i){var n,o;return t<e.options.barChart.width&&t>0?(n=t<i?i:t,o=0,"left"===e.options.barChart.align?o-=.5*t:"right"===e.options.barChart.align&&(o+=.5*t)):(n=e.options.barChart.width,o=0,"left"===e.options.barChart.align?o-=.5*e.options.barChart.width:"right"===e.options.barChart.align&&(o+=.5*e.options.barChart.width)),{width:n,offset:o}},Ou.getStackedYRange=function(t,e,i,n,o){if(t.length>0){wr(t).call(t,(function(t,e){return t.screen_x===e.screen_x?t.groupId<e.groupId?-1:1:t.screen_x-e.screen_x}));var s={};Ou._getDataIntersections(s,t),e[n]=Ou._getStackedYRange(s,t),e[n].yAxisOrientation=o,i.push(n)}},Ou._getStackedYRange=function(t,e){for(var i,n=e[0].screen_y,o=e[0].screen_y,s=0;s<e.length;s++)void 0===t[i=e[s].screen_x]?(n=n>e[s].screen_y?e[s].screen_y:n,o=o<e[s].screen_y?e[s].screen_y:o):e[s].screen_y<0?t[i].accumulatedNegative+=e[s].screen_y:t[i].accumulatedPositive+=e[s].screen_y;for(var r in t)t.hasOwnProperty(r)&&(n=(n=n>t[r].accumulatedNegative?t[r].accumulatedNegative:n)>t[r].accumulatedPositive?t[r].accumulatedPositive:n,o=(o=o<t[r].accumulatedNegative?t[r].accumulatedNegative:o)<t[r].accumulatedPositive?t[r].accumulatedPositive:o);return{min:n,max:o}},Wu.calcPath=function(t,e){if(null!=t&&t.length>0){return 1==e.options.interpolation.enabled?Wu._catmullRom(t,e):Wu._linear(t)}},Wu.drawIcon=function(t,e,i,n,o,s){var r,a,l=.5*o,h=Ms("rect",s.svgElements,s.svg);(h.setAttributeNS(null,"x",e),h.setAttributeNS(null,"y",i-l),h.setAttributeNS(null,"width",n),h.setAttributeNS(null,"height",2*l),h.setAttributeNS(null,"class","vis-outline"),(r=Ms("path",s.svgElements,s.svg)).setAttributeNS(null,"class",t.className),void 0!==t.style&&r.setAttributeNS(null,"style",t.style),r.setAttributeNS(null,"d","M"+e+","+i+" L"+(e+n)+","+i),1==t.options.shaded.enabled&&(a=Ms("path",s.svgElements,s.svg),"top"==t.options.shaded.orientation?a.setAttributeNS(null,"d","M"+e+", "+(i-l)+"L"+e+","+i+" L"+(e+n)+","+i+" L"+(e+n)+","+(i-l)):a.setAttributeNS(null,"d","M"+e+","+i+" L"+e+","+(i+l)+" L"+(e+n)+","+(i+l)+"L"+(e+n)+","+i),a.setAttributeNS(null,"class",t.className+" vis-icon-fill"),void 0!==t.options.shaded.style&&""!==t.options.shaded.style&&a.setAttributeNS(null,"style",t.options.shaded.style)),1==t.options.drawPoints.enabled)&&Qs(e+.5*n,i,{style:t.options.drawPoints.style,styles:t.options.drawPoints.styles,size:t.options.drawPoints.size,className:t.className},s.svgElements,s.svg)},Wu.drawShading=function(t,e,i,n){if(1==e.options.shaded.enabled){var o,s=Number(n.svg.style.height.replace("px","")),r=Ms("path",n.svgElements,n.svg),a="L";1==e.options.interpolation.enabled&&(a="C");var l=0;l="top"==e.options.shaded.orientation?0:"bottom"==e.options.shaded.orientation?s:Math.min(Math.max(0,e.zeroPosition),s),o="group"==e.options.shaded.orientation&&null!=i&&null!=i?"M"+t[0][0]+","+t[0][1]+" "+this.serializePath(t,a,!1)+" L"+i[i.length-1][0]+","+i[i.length-1][1]+" "+this.serializePath(i,a,!0)+i[0][0]+","+i[0][1]+" Z":"M"+t[0][0]+","+t[0][1]+" "+this.serializePath(t,a,!1)+" V"+l+" H"+t[0][0]+" Z",r.setAttributeNS(null,"class",e.className+" vis-fill"),void 0!==e.options.shaded.style&&r.setAttributeNS(null,"style",e.options.shaded.style),r.setAttributeNS(null,"d",o)}},Wu.draw=function(t,e,i){if(null!=t&&null!=t){var n=Ms("path",i.svgElements,i.svg);n.setAttributeNS(null,"class",e.className),void 0!==e.style&&n.setAttributeNS(null,"style",e.style);var o="L";1==e.options.interpolation.enabled&&(o="C"),n.setAttributeNS(null,"d","M"+t[0][0]+","+t[0][1]+" "+this.serializePath(t,o,!1))}},Wu.serializePath=function(t,e,i){if(t.length<2)return"";var n,o=e;if(i)for(n=t.length-2;n>0;n--)o+=t[n][0]+","+t[n][1]+" ";else for(n=1;n<t.length;n++)o+=t[n][0]+","+t[n][1]+" ";return o},Wu._catmullRomUniform=function(t){var e,i,n,o,s,r,a=[];a.push([Math.round(t[0].screen_x),Math.round(t[0].screen_y)]);for(var l=t.length,h=0;h<l-1;h++)e=0==h?t[0]:t[h-1],i=t[h],n=t[h+1],o=h+2<l?t[h+2]:n,s={screen_x:(-e.screen_x+6*i.screen_x+n.screen_x)*(1/6),screen_y:(-e.screen_y+6*i.screen_y+n.screen_y)*(1/6)},r={screen_x:(i.screen_x+6*n.screen_x-o.screen_x)*(1/6),screen_y:(i.screen_y+6*n.screen_y-o.screen_y)*(1/6)},a.push([s.screen_x,s.screen_y]),a.push([r.screen_x,r.screen_y]),a.push([n.screen_x,n.screen_y]);return a},Wu._catmullRom=function(t,e){var i=e.options.interpolation.alpha;if(0==i||void 0===i)return this._catmullRomUniform(t);var n,o,s,r,a,l,h,d,u,c,p,m,f,g,v,b,y,x,w,_=[];_.push([Math.round(t[0].screen_x),Math.round(t[0].screen_y)]);for(var k=t.length,D=0;D<k-1;D++)n=0==D?t[0]:t[D-1],o=t[D],s=t[D+1],r=D+2<k?t[D+2]:s,h=Math.sqrt(Math.pow(n.screen_x-o.screen_x,2)+Math.pow(n.screen_y-o.screen_y,2)),d=Math.sqrt(Math.pow(o.screen_x-s.screen_x,2)+Math.pow(o.screen_y-s.screen_y,2)),u=Math.sqrt(Math.pow(s.screen_x-r.screen_x,2)+Math.pow(s.screen_y-r.screen_y,2)),g=Math.pow(u,i),b=Math.pow(u,2*i),v=Math.pow(d,i),y=Math.pow(d,2*i),w=Math.pow(h,i),c=2*(x=Math.pow(h,2*i))+3*w*v+y,p=2*b+3*g*v+y,(m=3*w*(w+v))>0&&(m=1/m),(f=3*g*(g+v))>0&&(f=1/f),a={screen_x:(-y*n.screen_x+c*o.screen_x+x*s.screen_x)*m,screen_y:(-y*n.screen_y+c*o.screen_y+x*s.screen_y)*m},l={screen_x:(b*o.screen_x+p*s.screen_x-y*r.screen_x)*f,screen_y:(b*o.screen_y+p*s.screen_y-y*r.screen_y)*f},0==a.screen_x&&0==a.screen_y&&(a=o),0==l.screen_x&&0==l.screen_y&&(l=s),_.push([a.screen_x,a.screen_y]),_.push([l.screen_x,l.screen_y]),_.push([s.screen_x,s.screen_y]);return _},Wu._linear=function(t){for(var e=[],i=0;i<t.length;i++)e.push([t[i].screen_x,t[i].screen_y]);return e},Au.prototype.setItems=function(t){null!=t?(this.itemsData=t,1==wr(this.options)&&za.insertSort(this.itemsData,(function(t,e){return t.x>e.x?1:-1}))):this.itemsData=[]},Au.prototype.getItems=function(){return this.itemsData},Au.prototype.setZeroPosition=function(t){this.zeroPosition=t},Au.prototype.setOptions=function(t){if(void 0!==t){za.selectiveDeepExtend(["sampling","style","sort","yAxisOrientation","barChart","zIndex","excludeFromStacking","excludeFromLegend"],this.options,t),"function"==typeof t.drawPoints&&(t.drawPoints={onRender:t.drawPoints}),za.mergeOptions(this.options,t,"interpolation"),za.mergeOptions(this.options,t,"drawPoints"),za.mergeOptions(this.options,t,"shaded"),t.interpolation&&"object"==zo(t.interpolation)&&t.interpolation.parametrization&&("uniform"==t.interpolation.parametrization?this.options.interpolation.alpha=0:"chordal"==t.interpolation.parametrization?this.options.interpolation.alpha=1:(this.options.interpolation.parametrization="centripetal",this.options.interpolation.alpha=.5))}},Au.prototype.update=function(t){this.group=t,this.content=t.content||"graph",this.className=t.className||this.className||"vis-graph-group"+this.groupsUsingDefaultStyles[0]%10,this.visible=void 0===t.visible||t.visible,this.style=t.style,this.setOptions(t.options)},Au.prototype.getLegend=function(t,e,i,n,o){null!=i&&null!=i||(i={svg:document.createElementNS("http://www.w3.org/2000/svg","svg"),svgElements:{},options:this.options,groups:[this]});switch(null!=n&&null!=n||(n=0),null!=o&&null!=o||(o=.5*e),this.options.style){case"line":Wu.drawIcon(this,n,o,t,e,i);break;case"points":case"point":Zu.drawIcon(this,n,o,t,e,i);break;case"bar":Ou.drawIcon(this,n,o,t,e,i)}return{icon:i.svg,label:this.content,orientation:this.options.yAxisOrientation}},Au.prototype.getYRange=function(t){for(var e=t[0].y,i=t[0].y,n=0;n<t.length;n++)e=e>t[n].y?t[n].y:e,i=i<t[n].y?t[n].y:i;return{min:e,max:i,yAxisOrientation:this.options.yAxisOrientation}},Xu.prototype=new Ja,Xu.prototype.clear=function(){this.groups={},this.amountOfGroups=0},Xu.prototype.addGroup=function(t,e){1!=e.options.excludeFromLegend&&(this.groups.hasOwnProperty(t)||(this.groups[t]=e),this.amountOfGroups+=1)},Xu.prototype.updateGroup=function(t,e){this.groups[t]=e},Xu.prototype.removeGroup=function(t){this.groups.hasOwnProperty(t)&&(delete this.groups[t],this.amountOfGroups-=1)},Xu.prototype._create=function(){this.dom.frame=document.createElement("div"),this.dom.frame.className="vis-legend",this.dom.frame.style.position="absolute",this.dom.frame.style.top="10px",this.dom.frame.style.display="block",this.dom.textArea=document.createElement("div"),this.dom.textArea.className="vis-legend-text",this.dom.textArea.style.position="relative",this.dom.textArea.style.top="0px",this.svg=document.createElementNS("http://www.w3.org/2000/svg","svg"),this.svg.style.position="absolute",this.svg.style.top="0px",this.svg.style.width=this.options.iconSize+5+"px",this.svg.style.height="100%",this.dom.frame.appendChild(this.svg),this.dom.frame.appendChild(this.dom.textArea)},Xu.prototype.hide=function(){this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame)},Xu.prototype.show=function(){this.dom.frame.parentNode||this.body.dom.center.appendChild(this.dom.frame)},Xu.prototype.setOptions=function(t){za.selectiveDeepExtend(["enabled","orientation","icons","left","right"],this.options,t)},Xu.prototype.redraw=function(){var t=0,e=Yi(this.groups);wr(e).call(e,(function(t,e){return t<e?-1:1}));for(var i=0;i<e.length;i++){var n=e[i];1!=this.groups[n].visible||void 0!==this.linegraphOptions.visibility[n]&&1!=this.linegraphOptions.visibility[n]||t++}if(0==this.options[this.side].visible||0==this.amountOfGroups||0==this.options.enabled||0==t)this.hide();else{if(this.show(),"top-left"==this.options[this.side].position||"bottom-left"==this.options[this.side].position?(this.dom.frame.style.left="4px",this.dom.frame.style.textAlign="left",this.dom.textArea.style.textAlign="left",this.dom.textArea.style.left=this.options.iconSize+15+"px",this.dom.textArea.style.right="",this.svg.style.left="0px",this.svg.style.right=""):(this.dom.frame.style.right="4px",this.dom.frame.style.textAlign="right",this.dom.textArea.style.textAlign="right",this.dom.textArea.style.right=this.options.iconSize+15+"px",this.dom.textArea.style.left="",this.svg.style.right="0px",this.svg.style.left=""),"top-left"==this.options[this.side].position||"top-right"==this.options[this.side].position)this.dom.frame.style.top=4-Number(this.body.dom.center.style.top.replace("px",""))+"px",this.dom.frame.style.bottom="";else{var o=this.body.domProps.center.height-this.body.domProps.centerContainer.height;this.dom.frame.style.bottom=4+o+Number(this.body.dom.center.style.top.replace("px",""))+"px",this.dom.frame.style.top=""}0==this.options.icons?(this.dom.frame.style.width=this.dom.textArea.offsetWidth+10+"px",this.dom.textArea.style.right="",this.dom.textArea.style.left="",this.svg.style.width="0px"):(this.dom.frame.style.width=this.options.iconSize+15+this.dom.textArea.offsetWidth+10+"px",this.drawLegendIcons());var s="";for(i=0;i<e.length;i++)n=e[i],1!=this.groups[n].visible||void 0!==this.linegraphOptions.visibility[n]&&1!=this.linegraphOptions.visibility[n]||(s+=this.groups[n].content+"<br />");this.dom.textArea.innerHTML=s,this.dom.textArea.style.lineHeight=.75*this.options.iconSize+this.options.iconSpacing+"px"}},Xu.prototype.drawLegendIcons=function(){if(this.dom.frame.parentNode){var t=Yi(this.groups);wr(t).call(t,(function(t,e){return t<e?-1:1})),Gs(this.svgElements);var e=window.getComputedStyle(this.dom.frame).paddingTop,i=Number(e.replace("px","")),n=i,o=this.options.iconSize,s=.75*this.options.iconSize,r=i+.5*s+3;this.svg.style.width=o+5+i+"px";for(var a=0;a<t.length;a++){var l=t[a];1!=this.groups[l].visible||void 0!==this.linegraphOptions.visibility[l]&&1!=this.linegraphOptions.visibility[l]||(this.groups[l].getLegend(o,s,this.framework,n,r),r+=s+this.options.iconSpacing)}}};function Tu(t,e){this.id=Id(),this.body=t,this.defaultOptions={yAxisOrientation:"left",defaultGroup:"default",sort:!0,sampling:!0,stack:!1,graphHeight:"400px",shaded:{enabled:!1,orientation:"bottom"},style:"line",barChart:{width:50,sideBySide:!1,align:"center"},interpolation:{enabled:!0,parametrization:"centripetal",alpha:.5},drawPoints:{enabled:!0,size:6,style:"square"},dataAxis:{},legend:{},groups:{visibility:{}}},this.options=za.extend({},this.defaultOptions),this.dom={},this.props={},this.hammer=null,this.groups={},this.abortedGraphUpdate=!1,this.updateSVGheight=!1,this.updateSVGheightOnResize=!1,this.forceGraphUpdate=!0;var i=this;this.itemsData=null,this.groupsData=null,this.itemListeners={add:function(t,e,n){i._onAdd(e.items)},update:function(t,e,n){i._onUpdate(e.items)},remove:function(t,e,n){i._onRemove(e.items)}},this.groupListeners={add:function(t,e,n){i._onAddGroups(e.items)},update:function(t,e,n){i._onUpdateGroups(e.items)},remove:function(t,e,n){i._onRemoveGroups(e.items)}},this.items={},this.selection=[],this.lastStart=this.body.range.start,this.touchParams={},this.svgElements={},this.setOptions(e),this.groupsUsingDefaultStyles=[0],this.body.emitter.on("rangechanged",(function(){i.lastStart=i.body.range.start,i.svg.style.left=za.option.asSize(-i.props.width),i.forceGraphUpdate=!0,i.redraw.call(i)})),this._create(),this.framework={svg:this.svg,svgElements:this.svgElements,options:this.options,groups:this.groups}}Tu.prototype=new Ja,Tu.prototype._create=function(){var t=document.createElement("div");t.className="vis-line-graph",this.dom.frame=t,this.svg=document.createElementNS("http://www.w3.org/2000/svg","svg"),this.svg.style.position="relative",this.svg.style.height=(""+this.options.graphHeight).replace("px","")+"px",this.svg.style.display="block",t.appendChild(this.svg),this.options.dataAxis.orientation="left",this.yAxisLeft=new Fu(this.body,this.options.dataAxis,this.svg,this.options.groups),this.options.dataAxis.orientation="right",this.yAxisRight=new Fu(this.body,this.options.dataAxis,this.svg,this.options.groups),delete this.options.dataAxis.orientation,this.legendLeft=new Xu(this.body,this.options.legend,"left",this.options.groups),this.legendRight=new Xu(this.body,this.options.legend,"right",this.options.groups),this.show()},Tu.prototype.setOptions=function(t){if(t){void 0===t.graphHeight&&void 0!==t.height?(this.updateSVGheight=!0,this.updateSVGheightOnResize=!0):void 0!==this.body.domProps.centerContainer.height&&void 0!==t.graphHeight&&Nn((t.graphHeight+"").replace("px",""))<this.body.domProps.centerContainer.height&&(this.updateSVGheight=!0),za.selectiveDeepExtend(["sampling","defaultGroup","stack","height","graphHeight","yAxisOrientation","style","barChart","dataAxis","sort","groups"],this.options,t),za.mergeOptions(this.options,t,"interpolation"),za.mergeOptions(this.options,t,"drawPoints"),za.mergeOptions(this.options,t,"shaded"),za.mergeOptions(this.options,t,"legend"),t.interpolation&&"object"==zo(t.interpolation)&&t.interpolation.parametrization&&("uniform"==t.interpolation.parametrization?this.options.interpolation.alpha=0:"chordal"==t.interpolation.parametrization?this.options.interpolation.alpha=1:(this.options.interpolation.parametrization="centripetal",this.options.interpolation.alpha=.5)),this.yAxisLeft&&void 0!==t.dataAxis&&(this.yAxisLeft.setOptions(this.options.dataAxis),this.yAxisRight.setOptions(this.options.dataAxis)),this.legendLeft&&void 0!==t.legend&&(this.legendLeft.setOptions(this.options.legend),this.legendRight.setOptions(this.options.legend)),this.groups.hasOwnProperty("__ungrouped__")&&this.groups.__ungrouped__.setOptions(t)}this.dom.frame&&(this.forceGraphUpdate=!0,this.body.emitter.emit("_change",{queue:!0}))},Tu.prototype.hide=function(){this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame)},Tu.prototype.show=function(){this.dom.frame.parentNode||this.body.dom.center.appendChild(this.dom.frame)},Tu.prototype.setItems=function(t){var e,i=this,n=this.itemsData;if(t){if(!(t instanceof Aa||t instanceof Xa))throw new TypeError("Data must be an instance of DataSet or DataView");this.itemsData=Ua(t)}else this.itemsData=null;if(n&&(mn(za).call(za,this.itemListeners,(function(t,e){n.off(e,t)})),n.dispose(),e=n.getIds(),this._onRemove(e)),this.itemsData){var o=this.id;mn(za).call(za,this.itemListeners,(function(t,e){i.itemsData.on(e,t,o)})),e=this.itemsData.getIds(),this._onAdd(e)}},Tu.prototype.setGroups=function(t){var e,i=this;if(this.groupsData){mn(za).call(za,this.groupListeners,(function(t,e){i.groupsData.off(e,t)})),e=this.groupsData.getIds(),this.groupsData=null;for(var n=0;n<e.length;n++)this._removeGroup(e[n])}if(t){if(!(t instanceof Aa||t instanceof Xa))throw new TypeError("Data must be an instance of DataSet or DataView");this.groupsData=t}else this.groupsData=null;if(this.groupsData){var o=this.id;mn(za).call(za,this.groupListeners,(function(t,e){i.groupsData.on(e,t,o)})),e=this.groupsData.getIds(),this._onAddGroups(e)}},Tu.prototype._onUpdate=function(t){this._updateAllGroupData(t)},Tu.prototype._onAdd=function(t){this._onUpdate(t)},Tu.prototype._onRemove=function(t){this._onUpdate(t)},Tu.prototype._onUpdateGroups=function(t){this._updateAllGroupData(null,t)},Tu.prototype._onAddGroups=function(t){this._onUpdateGroups(t)},Tu.prototype._onRemoveGroups=function(t){for(var e=0;e<t.length;e++)this._removeGroup(t[e]);this.forceGraphUpdate=!0,this.body.emitter.emit("_change",{queue:!0})},Tu.prototype._removeGroup=function(t){this.groups.hasOwnProperty(t)&&("right"==this.groups[t].options.yAxisOrientation?(this.yAxisRight.removeGroup(t),this.legendRight.removeGroup(t),this.legendRight.redraw()):(this.yAxisLeft.removeGroup(t),this.legendLeft.removeGroup(t),this.legendLeft.redraw()),delete this.groups[t])},Tu.prototype._updateGroup=function(t,e){this.groups.hasOwnProperty(e)?(this.groups[e].update(t),"right"==this.groups[e].options.yAxisOrientation?(this.yAxisRight.updateGroup(e,this.groups[e]),this.legendRight.updateGroup(e,this.groups[e]),this.yAxisLeft.removeGroup(e),this.legendLeft.removeGroup(e)):(this.yAxisLeft.updateGroup(e,this.groups[e]),this.legendLeft.updateGroup(e,this.groups[e]),this.yAxisRight.removeGroup(e),this.legendRight.removeGroup(e))):(this.groups[e]=new Au(t,e,this.options,this.groupsUsingDefaultStyles),"right"==this.groups[e].options.yAxisOrientation?(this.yAxisRight.addGroup(e,this.groups[e]),this.legendRight.addGroup(e,this.groups[e])):(this.yAxisLeft.addGroup(e,this.groups[e]),this.legendLeft.addGroup(e,this.groups[e]))),this.legendLeft.redraw(),this.legendRight.redraw()},Tu.prototype._updateAllGroupData=function(t,e){if(null!=this.itemsData){var i={},n=this.itemsData.get(),o=this.itemsData.idProp,s={};t&&Ln(t).call(t,(function(t){s[t]=t}));for(var r={},a=0;a<n.length;a++){var l=n[a],h=l.group;null==h&&(h="__ungrouped__"),r.hasOwnProperty(h)?r[h]++:r[h]=1}var d={};if(!e&&t)for(h in this.groups)if(this.groups.hasOwnProperty(h)){var u=(f=this.groups[h]).getItems();i[h]=En(u).call(u,(function(t){return d[t[o]]=t[o],t[o]!==s[t[o]]}));var c=r[h];r[h]-=i[h].length,i[h].length<c&&(i[h][c-1]={})}for(a=0;a<n.length;a++)if(null==(h=(l=n[a]).group)&&(h="__ungrouped__"),e||!t||l[o]===s[l[o]]||!d.hasOwnProperty(l[o])){i.hasOwnProperty(h)||(i[h]=new Array(r[h]));var p=za.bridgeObject(l);p.x=za.convert(l.x,"Date"),p.end=za.convert(l.end,"Date"),p.orginalY=l.y,p.y=Number(l.y),p[o]=l[o];var m=i[h].length-r[h]--;i[h][m]=p}for(h in this.groups)this.groups.hasOwnProperty(h)&&(i.hasOwnProperty(h)||(i[h]=new Array(0)));for(h in i)if(i.hasOwnProperty(h))if(0==i[h].length)this.groups.hasOwnProperty(h)&&this._removeGroup(h);else{var f=void 0;null!=this.groupsData&&(f=this.groupsData.get(h)),null==f&&(f={id:h,content:this.options.defaultGroup+h}),this._updateGroup(f,h),this.groups[h].setItems(i[h])}this.forceGraphUpdate=!0,this.body.emitter.emit("_change",{queue:!0})}},Tu.prototype.redraw=function(){var t=!1;this.props.width=this.dom.frame.offsetWidth,this.props.height=this.body.domProps.centerContainer.height-this.body.domProps.border.top-this.body.domProps.border.bottom,t=this._isResized()||t;var e,i=this.body.range.end-this.body.range.start,n=i!=this.lastVisibleInterval;(this.lastVisibleInterval=i,1==t)&&(this.svg.style.width=za.option.asSize(3*this.props.width),this.svg.style.left=za.option.asSize(-this.props.width),-1==Fo(e=this.options.height+"").call(e,"%")&&1!=this.updateSVGheightOnResize||(this.updateSVGheight=!0));if(1==this.updateSVGheight?(this.options.graphHeight!=this.props.height+"px"&&(this.options.graphHeight=this.props.height+"px",this.svg.style.height=this.props.height+"px"),this.updateSVGheight=!1):this.svg.style.height=(""+this.options.graphHeight).replace("px","")+"px",1==t||1==n||1==this.abortedGraphUpdate||1==this.forceGraphUpdate)t=this._updateGraph()||t,this.forceGraphUpdate=!1;else if(0!=this.lastStart){var o=this.body.range.start-this.lastStart,s=this.body.range.end-this.body.range.start;if(0!=this.props.width){var r=o*(this.props.width/s);this.svg.style.left=-this.props.width-r+"px"}}return this.legendLeft.redraw(),this.legendRight.redraw(),t},Tu.prototype._getSortedGroupIds=function(){var t=[];for(var e in this.groups)if(this.groups.hasOwnProperty(e)){var i=this.groups[e];1!=i.visible||void 0!==this.options.groups.visibility[e]&&1!=this.options.groups.visibility[e]||t.push({id:e,zIndex:i.options.zIndex})}za.insertSort(t,(function(t,e){var i=t.zIndex,n=e.zIndex;return void 0===i&&(i=0),void 0===n&&(n=0),i==n?0:i<n?-1:1}));for(var n=new Array(t.length),o=0;o<t.length;o++)n[o]=t[o].id;return n},Tu.prototype._updateGraph=function(){if(Cs(this.svgElements),0!=this.props.width&&null!=this.itemsData){var t,e,i={},n=this.body.util.toGlobalTime(-this.body.domProps.root.width),o=this.body.util.toGlobalTime(2*this.body.domProps.root.width),s=this._getSortedGroupIds();if(s.length>0){var r={};for(this._getRelevantData(s,r,n,o),this._applySampling(s,r),e=0;e<s.length;e++)this._convertXcoordinates(r[s[e]]);if(this._getYRanges(s,r,i),1==this._updateYAxis(s,i))return Ls(this.svgElements),this.abortedGraphUpdate=!0,!0;this.abortedGraphUpdate=!1;var a=void 0;for(e=0;e<s.length;e++)t=this.groups[s[e]],!0===this.options.stack&&"line"===this.options.style&&(null!=t.options.excludeFromStacking&&t.options.excludeFromStacking||(null!=a&&(this._stack(r[t.id],r[a.id]),1==t.options.shaded.enabled&&"group"!==t.options.shaded.orientation&&("top"==t.options.shaded.orientation&&"group"!==a.options.shaded.orientation?(a.options.shaded.orientation="group",a.options.shaded.groupId=t.id):(t.options.shaded.orientation="group",t.options.shaded.groupId=a.id))),a=t)),this._convertYcoordinates(r[s[e]],t);var l={};for(e=0;e<s.length;e++)if("line"===(t=this.groups[s[e]]).options.style&&1==t.options.shaded.enabled){var h=r[s[e]];if(null==h||0==h.length)continue;if(l.hasOwnProperty(s[e])||(l[s[e]]=Wu.calcPath(h,t)),"group"===t.options.shaded.orientation){var d=t.options.shaded.groupId;if(-1===Fo(s).call(s,d)){console.log(t.id+": Unknown shading group target given:"+d);continue}l.hasOwnProperty(d)||(l[d]=Wu.calcPath(r[d],this.groups[d])),Wu.drawShading(l[s[e]],t,l[d],this.framework)}else Wu.drawShading(l[s[e]],t,void 0,this.framework)}for(Ou.draw(s,r,this.framework),e=0;e<s.length;e++)if(t=this.groups[s[e]],r[s[e]].length>0)switch(t.options.style){case"line":l.hasOwnProperty(s[e])||(l[s[e]]=Wu.calcPath(r[s[e]],t)),Wu.draw(l[s[e]],t,this.framework);case"point":case"points":"point"!=t.options.style&&"points"!=t.options.style&&1!=t.options.drawPoints.enabled||Zu.draw(r[s[e]],t,this.framework)}}}return Ls(this.svgElements),!1},Tu.prototype._stack=function(t,e){var i,n,o,s,r;i=0;for(var a=0;a<t.length;a++){s=void 0,r=void 0;for(var l=i;l<e.length;l++){if(e[l].x===t[a].x){s=e[l],r=e[l],i=l;break}if(e[l].x>t[a].x){r=e[l],s=0==l?r:e[l-1],i=l;break}}void 0===r&&(s=e[e.length-1],r=e[e.length-1]),n=r.x-s.x,o=r.y-s.y,t[a].y=0==n?t[a].orginalY+r.y:t[a].orginalY+o/n*(t[a].x-s.x)+s.y}},Tu.prototype._getRelevantData=function(t,e,i,n){var o,s,r,a;if(t.length>0)for(s=0;s<t.length;s++){var l=(o=this.groups[t[s]]).getItems();if(1==wr(o.options)){var h=function(t,e){return t.getTime()==e.getTime()?0:t<e?-1:1},d=Math.max(0,za.binarySearchValue(l,i,"x","before",h)),u=Math.min(l.length,za.binarySearchValue(l,n,"x","after",h)+1);u<=0&&(u=l.length);var c=new Array(u-d);for(r=d;r<u;r++)a=o.itemsData[r],c[r-d]=a;e[t[s]]=c}else e[t[s]]=o.itemsData}},Tu.prototype._applySampling=function(t,e){if(t.length>0)for(var i=0;i<t.length;i++)if(1==this.groups[t[i]].options.sampling){var n=e[t[i]];if(n.length>0){var o,s=n.length,r=s/(this.body.util.toGlobalScreen(n[n.length-1].x)-this.body.util.toGlobalScreen(n[0].x));o=Math.min(Math.ceil(.2*s),Math.max(1,Math.round(r)));for(var a=new Array(s),l=0;l<s;l+=o){a[Math.round(l/o)]=n[l]}e[t[i]]=Hr(a).call(a,0,Math.round(s/o))}}},Tu.prototype._getYRanges=function(t,e,i){var n,o,s,r,a=[],l=[];if(t.length>0){for(s=0;s<t.length;s++)n=e[t[s]],r=this.groups[t[s]].options,n.length>0&&(o=this.groups[t[s]],!0===r.stack&&"bar"===r.style?"left"===r.yAxisOrientation?a=Pn(a).call(a,n):l=Pn(l).call(l,n):i[t[s]]=o.getYRange(n,t[s]));Ou.getStackedYRange(a,i,t,"__barStackLeft","left"),Ou.getStackedYRange(l,i,t,"__barStackRight","right")}},Tu.prototype._updateYAxis=function(t,e){var i,n,o=!1,s=!1,r=!1,a=1e9,l=1e9,h=-1e9,d=-1e9;if(t.length>0){for(var u=0;u<t.length;u++){var c=this.groups[t[u]];c&&"right"!=c.options.yAxisOrientation?(s=!0,a=1e9,h=-1e9):c&&c.options.yAxisOrientation&&(r=!0,l=1e9,d=-1e9)}for(u=0;u<t.length;u++)e.hasOwnProperty(t[u])&&!0!==e[t[u]].ignore&&(i=e[t[u]].min,n=e[t[u]].max,"right"!=e[t[u]].yAxisOrientation?(s=!0,a=a>i?i:a,h=h<n?n:h):(r=!0,l=l>i?i:l,d=d<n?n:d));1==s&&this.yAxisLeft.setRange(a,h),1==r&&this.yAxisRight.setRange(l,d)}o=this._toggleAxisVisiblity(s,this.yAxisLeft)||o,o=this._toggleAxisVisiblity(r,this.yAxisRight)||o,1==r&&1==s?(this.yAxisLeft.drawIcons=!0,this.yAxisRight.drawIcons=!0):(this.yAxisLeft.drawIcons=!1,this.yAxisRight.drawIcons=!1),this.yAxisRight.master=!s,this.yAxisRight.masterAxis=this.yAxisLeft,0==this.yAxisRight.master?(this.yAxisLeft.lineOffset=1==r?this.yAxisRight.width:0,o=this.yAxisLeft.redraw()||o,o=this.yAxisRight.redraw()||o):o=this.yAxisRight.redraw()||o;var p=["__barStackLeft","__barStackRight","__lineStackLeft","__lineStackRight"];for(u=0;u<p.length;u++)-1!=Fo(t).call(t,p[u])&&Hr(t).call(t,Fo(t).call(t,p[u]),1);return o},Tu.prototype._toggleAxisVisiblity=function(t,e){var i=!1;return 0==t?e.dom.frame.parentNode&&0==e.hidden&&(e.hide(),i=!0):e.dom.frame.parentNode||1!=e.hidden||(e.show(),i=!0),i},Tu.prototype._convertXcoordinates=function(t){for(var e=this.body.util.toScreen,i=0;i<t.length;i++)t[i].screen_x=e(t[i].x)+this.props.width,t[i].screen_y=t[i].y,null!=t[i].end?t[i].screen_end=e(t[i].end)+this.props.width:t[i].screen_end=void 0},Tu.prototype._convertYcoordinates=function(t,e){var i=this.yAxisLeft,n=Number(this.svg.style.height.replace("px",""));"right"==e.options.yAxisOrientation&&(i=this.yAxisRight);for(var o=0;o<t.length;o++)t[o].screen_y=Math.round(i.convertValue(t[o].y));e.setZeroPosition(Math.min(n,i.convertValue(0)))};var Yu="boolean",Eu={configure:{enabled:{boolean:Yu},filter:{boolean:Yu,function:"function"},container:{dom:"dom"},__type__:{object:"object",boolean:Yu,function:"function"}},alignCurrentTime:{string:"string",undefined:"undefined"},yAxisOrientation:{string:["left","right"]},defaultGroup:{string:"string"},sort:{boolean:Yu},sampling:{boolean:Yu},stack:{boolean:Yu},graphHeight:{string:"string",number:"number"},shaded:{enabled:{boolean:Yu},orientation:{string:["bottom","top","zero","group"]},groupId:{object:"object"},__type__:{boolean:Yu,object:"object"}},style:{string:["line","bar","points"]},barChart:{width:{number:"number"},minWidth:{number:"number"},sideBySide:{boolean:Yu},align:{string:["left","center","right"]},__type__:{object:"object"}},interpolation:{enabled:{boolean:Yu},parametrization:{string:["centripetal","chordal","uniform"]},alpha:{number:"number"},__type__:{object:"object",boolean:Yu}},drawPoints:{enabled:{boolean:Yu},onRender:{function:"function"},size:{number:"number"},style:{string:["square","circle"]},__type__:{object:"object",boolean:Yu,function:"function"}},dataAxis:{showMinorLabels:{boolean:Yu},showMajorLabels:{boolean:Yu},showWeekScale:{boolean:Yu},icons:{boolean:Yu},width:{string:"string",number:"number"},visible:{boolean:Yu},alignZeros:{boolean:Yu},left:{range:{min:{number:"number",undefined:"undefined"},max:{number:"number",undefined:"undefined"},__type__:{object:"object"}},format:{function:"function"},title:{text:{string:"string",number:"number",undefined:"undefined"},style:{string:"string",undefined:"undefined"},__type__:{object:"object"}},__type__:{object:"object"}},right:{range:{min:{number:"number",undefined:"undefined"},max:{number:"number",undefined:"undefined"},__type__:{object:"object"}},format:{function:"function"},title:{text:{string:"string",number:"number",undefined:"undefined"},style:{string:"string",undefined:"undefined"},__type__:{object:"object"}},__type__:{object:"object"}},__type__:{object:"object"}},legend:{enabled:{boolean:Yu},icons:{boolean:Yu},left:{visible:{boolean:Yu},position:{string:["top-right","bottom-right","top-left","bottom-left"]},__type__:{object:"object"}},right:{visible:{boolean:Yu},position:{string:["top-right","bottom-right","top-left","bottom-left"]},__type__:{object:"object"}},__type__:{object:"object",boolean:Yu}},groups:{visibility:{any:"any"},__type__:{object:"object"}},autoResize:{boolean:Yu},throttleRedraw:{number:"number"},clickToUse:{boolean:Yu},end:{number:"number",date:"date",string:"string",moment:"moment"},format:{minorLabels:{millisecond:{string:"string",undefined:"undefined"},second:{string:"string",undefined:"undefined"},minute:{string:"string",undefined:"undefined"},hour:{string:"string",undefined:"undefined"},weekday:{string:"string",undefined:"undefined"},day:{string:"string",undefined:"undefined"},week:{string:"string",undefined:"undefined"},month:{string:"string",undefined:"undefined"},quarter:{string:"string",undefined:"undefined"},year:{string:"string",undefined:"undefined"},__type__:{object:"object"}},majorLabels:{millisecond:{string:"string",undefined:"undefined"},second:{string:"string",undefined:"undefined"},minute:{string:"string",undefined:"undefined"},hour:{string:"string",undefined:"undefined"},weekday:{string:"string",undefined:"undefined"},day:{string:"string",undefined:"undefined"},week:{string:"string",undefined:"undefined"},month:{string:"string",undefined:"undefined"},quarter:{string:"string",undefined:"undefined"},year:{string:"string",undefined:"undefined"},__type__:{object:"object"}},__type__:{object:"object"}},moment:{function:"function"},height:{string:"string",number:"number"},hiddenDates:{start:{date:"date",number:"number",string:"string",moment:"moment"},end:{date:"date",number:"number",string:"string",moment:"moment"},repeat:{string:"string"},__type__:{object:"object",array:"array"}},locale:{string:"string"},locales:{__any__:{any:"any"},__type__:{object:"object"}},max:{date:"date",number:"number",string:"string",moment:"moment"},maxHeight:{number:"number",string:"string"},maxMinorChars:{number:"number"},min:{date:"date",number:"number",string:"string",moment:"moment"},minHeight:{number:"number",string:"string"},moveable:{boolean:Yu},multiselect:{boolean:Yu},orientation:{string:"string"},showCurrentTime:{boolean:Yu},showMajorLabels:{boolean:Yu},showMinorLabels:{boolean:Yu},showWeekScale:{boolean:Yu},start:{date:"date",number:"number",string:"string",moment:"moment"},timeAxis:{scale:{string:"string",undefined:"undefined"},step:{number:"number",undefined:"undefined"},__type__:{object:"object"}},width:{string:"string",number:"number"},zoomable:{boolean:Yu},zoomKey:{string:["ctrlKey","altKey","metaKey",""]},zoomMax:{number:"number"},zoomMin:{number:"number"},zIndex:{number:"number"},__type__:{object:"object"}},Ru={global:{alignCurrentTime:["none","year","month","quarter","week","isoWeek","day","date","hour","minute","second"],sort:!0,sampling:!0,stack:!1,shaded:{enabled:!1,orientation:["zero","top","bottom","group"]},style:["line","bar","points"],barChart:{width:[50,5,100,5],minWidth:[50,5,100,5],sideBySide:!1,align:["left","center","right"]},interpolation:{enabled:!0,parametrization:["centripetal","chordal","uniform"]},drawPoints:{enabled:!0,size:[6,2,30,1],style:["square","circle"]},dataAxis:{showMinorLabels:!0,showMajorLabels:!0,showWeekScale:!1,icons:!1,width:[40,0,200,1],visible:!0,alignZeros:!0,left:{title:{text:"",style:""}},right:{title:{text:"",style:""}}},legend:{enabled:!1,icons:!0,left:{visible:!0,position:["top-right","bottom-right","top-left","bottom-left"]},right:{visible:!0,position:["top-right","bottom-right","top-left","bottom-left"]}},autoResize:!0,clickToUse:!1,end:"",format:{minorLabels:{millisecond:"SSS",second:"s",minute:"HH:mm",hour:"HH:mm",weekday:"ddd D",day:"D",week:"w",month:"MMM",quarter:"[Q]Q",year:"YYYY"},majorLabels:{millisecond:"HH:mm:ss",second:"D MMMM HH:mm",minute:"ddd D MMMM",hour:"ddd D MMMM",weekday:"MMMM YYYY",day:"MMMM YYYY",week:"MMMM YYYY",month:"YYYY",quarter:"YYYY",year:""}},height:"",locale:"",max:"",maxHeight:"",maxMinorChars:[7,0,20,1],min:"",minHeight:"",moveable:!0,orientation:["both","bottom","top"],showCurrentTime:!1,showMajorLabels:!0,showMinorLabels:!0,showWeekScale:!1,start:"",width:"100%",zoomable:!0,zoomKey:["ctrlKey","altKey","metaKey",""],zoomMax:[31536e10,10,31536e10,1],zoomMin:[10,10,31536e10,1],zIndex:0}};function Uu(t,e,i,n){var o,s,r,a,l,h,d;if(!(Zo(i)||i instanceof Aa||i instanceof Xa)&&i instanceof Object){var u=n;n=i,i=u}n&&n.throttleRedraw&&console.warn('Graph2d option "throttleRedraw" is DEPRICATED and no longer supported. It will be removed in the next MAJOR release.');var c=this;this.defaultOptions={start:null,end:null,autoResize:!0,orientation:{axis:"bottom",item:"bottom"},moment:Ta,width:null,height:null,maxHeight:null,minHeight:null},this.options=za.deepExtend({},this.defaultOptions),this._create(t),this.components=[],this.body={dom:this.dom,domProps:this.props,emitter:{on:va(o=this.on).call(o,this),off:va(s=this.off).call(s,this),emit:va(r=this.emit).call(r,this)},hiddenDates:[],util:{toScreen:va(a=c._toScreen).call(a,c),toGlobalScreen:va(l=c._toGlobalScreen).call(l,c),toTime:va(h=c._toTime).call(h,c),toGlobalTime:va(d=c._toGlobalTime).call(d,c)}},this.range=new ml(this.body),this.components.push(this.range),this.body.range=this.range,this.timeAxis=new Jh(this.body),this.components.push(this.timeAxis),this.currentTime=new pd(this.body),this.components.push(this.currentTime),this.linegraph=new Tu(this.body),this.components.push(this.linegraph),this.itemsData=null,this.groupsData=null,this.on("tap",(function(t){c.emit("click",c.getEventProperties(t))})),this.on("doubletap",(function(t){c.emit("doubleClick",c.getEventProperties(t))})),this.dom.root.oncontextmenu=function(t){c.emit("contextmenu",c.getEventProperties(t))},this.initialFitDone=!1,this.on("changed",(function(){if(null!=c.itemsData){if(!c.initialFitDone&&!c.options.rollingMode)if(c.initialFitDone=!0,null!=c.options.start||null!=c.options.end){if(null==c.options.start||null==c.options.end)var t=c.getItemRange();var e=null!=c.options.start?c.options.start:t.min,i=null!=c.options.end?c.options.end:t.max;c.setWindow(e,i,{animation:!1})}else c.fit({animation:!1});c.initialDrawDone||!c.initialRangeChangeDone&&(c.options.start||c.options.end)&&!c.options.rollingMode||(c.initialDrawDone=!0,c.dom.root.style.visibility="visible",c.dom.loadingScreen.parentNode.removeChild(c.dom.loadingScreen),c.options.onInitialDrawComplete&&qr((function(){return c.options.onInitialDrawComplete()}),0))}})),n&&this.setOptions(n),i&&this.setGroups(i),e&&this.setItems(e),this._redraw()}Uu.prototype=new cd,Uu.prototype.setOptions=function(t){!0===fu.validate(t,Eu)&&console.log("%cErrors have been found in the supplied options object.",mu),cd.prototype.setOptions.call(this,t)},Uu.prototype.setItems=function(t){var e,i=null==this.itemsData;if(e=t?Ua(t instanceof Aa||t instanceof Xa?t:new Aa(t)):null,this.itemsData&&this.itemsData.dispose(),this.itemsData=e,this.linegraph&&this.linegraph.setItems(null!=e?e.rawDS:null),i)if(null!=this.options.start||null!=this.options.end){var n=null!=this.options.start?this.options.start:null,o=null!=this.options.end?this.options.end:null;this.setWindow(n,o,{animation:!1})}else this.fit({animation:!1})},Uu.prototype.setGroups=function(t){var e;e=t?t instanceof Aa||t instanceof Xa?t:new Aa(t):null,this.groupsData=e,this.linegraph.setGroups(e)},Uu.prototype.getLegend=function(t,e,i){return void 0===e&&(e=15),void 0===i&&(i=15),void 0!==this.linegraph.groups[t]?this.linegraph.groups[t].getLegend(e,i):"cannot find group:'"+t+"'"},Uu.prototype.isGroupVisible=function(t){return void 0!==this.linegraph.groups[t]&&(this.linegraph.groups[t].visible&&(void 0===this.linegraph.options.groups.visibility[t]||1==this.linegraph.options.groups.visibility[t]))},Uu.prototype.getDataRange=function(){var t=null,e=null;for(var i in this.linegraph.groups)if(this.linegraph.groups.hasOwnProperty(i)&&1==this.linegraph.groups[i].visible)for(var n=0;n<this.linegraph.groups[i].itemsData.length;n++){var o=this.linegraph.groups[i].itemsData[n],s=za.convert(o.x,"Date").valueOf();t=null==t||t>s?s:t,e=null==e||e<s?s:e}return{min:null!=t?new Date(t):null,max:null!=e?new Date(e):null}},Uu.prototype.getEventProperties=function(t){var e=t.center?t.center.x:t.clientX,i=t.center?t.center.y:t.clientY,n=e-za.getAbsoluteLeft(this.dom.centerContainer),o=i-za.getAbsoluteTop(this.dom.centerContainer),s=this._toTime(n),r=ud.customTimeFromTarget(t),a=za.getTarget(t),l=null;za.hasParent(a,this.timeAxis.dom.foreground)||this.timeAxis2&&za.hasParent(a,this.timeAxis2.dom.foreground)?l="axis":za.hasParent(a,this.linegraph.yAxisLeft.dom.frame)||za.hasParent(a,this.linegraph.yAxisRight.dom.frame)?l="data-axis":za.hasParent(a,this.linegraph.legendLeft.dom.frame)||za.hasParent(a,this.linegraph.legendRight.dom.frame)?l="legend":null!=r?l="custom-time":za.hasParent(a,this.currentTime.bar)?l="current-time":za.hasParent(a,this.dom.center)&&(l="background");var h=[],d=this.linegraph.yAxisLeft,u=this.linegraph.yAxisRight;return!d.hidden&&this.itemsData.length>0&&h.push(d.screenToValue(o)),!u.hidden&&this.itemsData.length>0&&h.push(u.screenToValue(o)),{event:t,customTime:r?r.options.id:null,what:l,pageX:t.srcEvent?t.srcEvent.pageX:t.pageX,pageY:t.srcEvent?t.srcEvent.pageY:t.pageY,x:n,y:o,time:s,value:h}},Uu.prototype._createConfigurator=function(){return new Lu(this,this.dom.container,Ru)};var zu=Fs();Ta.locale(zu);var Vu={Core:cd,DateUtil:pl,Range:ml,stack:zd,TimeStep:Hh,components:{items:{Item:Pd,BackgroundItem:$d,BoxItem:Jd,PointItem:Kd,RangeItem:qd},BackgroundGroup:Hd,Component:Ja,CurrentTime:pd,CustomTime:ud,DataAxis:Fu,DataScale:Nu,GraphGroup:Au,Group:jd,ItemSet:uu,Legend:Xu,LineGraph:Tu,TimeAxis:Jh}};t.DOMutil=Zs,t.DataSet=Aa,t.DataView=Xa,t.Graph2d=Uu,t.Hammer=Vh,t.Queue=Za,t.Timeline=Gu,t.keycharm=qh,t.moment=Ta,t.timeline=Vu,t.util=Is,Object.defineProperty(t,"__esModule",{value:!0})}));
-//# sourceMappingURL=vis-timeline-graph2d.min.js.map
+"use strict";!function(t,e){"object"==typeof exports&&"object"==typeof module?module.exports=e():"function"==typeof define&&define.amd?define([],e):"object"==typeof exports?exports.vis=e():t.vis=e()}(this,function(){return function(t){function e(o){if(i[o])return i[o].exports;var n=i[o]={i:o,l:!1,exports:{}};return t[o].call(n.exports,n,n.exports,e),n.l=!0,n.exports}var i={};return e.m=t,e.c=i,e.d=function(t,i,o){e.o(t,i)||Object.defineProperty(t,i,{configurable:!1,enumerable:!0,get:o})},e.n=function(t){var i=t&&t.__esModule?function(){return t.default}:function(){return t};return e.d(i,"a",i),i},e.o=function(t,e){return Object.prototype.hasOwnProperty.call(t,e)},e.p="",e(e.s=85)}([function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}function n(t,e,i,o){var n=!1;!0===o&&(n=null===e[i]&&void 0!==t[i]),n?delete t[i]:t[i]=e[i]}var s=i(86),r=o(s),a=i(43),h=o(a),d=i(4),l=o(d),u=i(1),p=o(u),c=i(3),m=i(126);e.isNumber=function(t){return t instanceof Number||"number"==typeof t},e.recursiveDOMDelete=function(t){if(t)for(;!0===t.hasChildNodes();)e.recursiveDOMDelete(t.firstChild),t.removeChild(t.firstChild)},e.giveRange=function(t,e,i,o){if(e==t)return.5;var n=1/(e-t);return Math.max(0,(o-t)*n)},e.isString=function(t){return t instanceof String||"string"==typeof t},e.isDate=function(t){if(t instanceof Date)return!0;if(e.isString(t)){if(f.exec(t))return!0;if(!isNaN(Date.parse(t)))return!0}return!1},e.randomUUID=function(){return m.v4()},e.assignAllKeys=function(t,e){for(var i in t)t.hasOwnProperty(i)&&"object"!==(0,p.default)(t[i])&&(t[i]=e)},e.fillIfDefined=function(t,i){var o=arguments.length>2&&void 0!==arguments[2]&&arguments[2];for(var s in t)void 0!==i[s]&&(null===i[s]||"object"!==(0,p.default)(i[s])?n(t,i,s,o):"object"===(0,p.default)(t[s])&&e.fillIfDefined(t[s],i[s],o))},e.extend=function(t,e){for(var i=1;i<arguments.length;i++){var o=arguments[i];for(var n in o)o.hasOwnProperty(n)&&(t[n]=o[n])}return t},e.selectiveExtend=function(t,e,i){if(!Array.isArray(t))throw new Error("Array with property names expected as first argument");for(var o=2;o<arguments.length;o++)for(var n=arguments[o],s=0;s<t.length;s++){var r=t[s];n&&n.hasOwnProperty(r)&&(e[r]=n[r])}return e},e.selectiveDeepExtend=function(t,i,o){var s=arguments.length>3&&void 0!==arguments[3]&&arguments[3];if(Array.isArray(o))throw new TypeError("Arrays are not supported by deepExtend");for(var r=0;r<t.length;r++){var a=t[r];if(o.hasOwnProperty(a))if(o[a]&&o[a].constructor===Object)void 0===i[a]&&(i[a]={}),i[a].constructor===Object?e.deepExtend(i[a],o[a],!1,s):n(i,o,a,s);else{if(Array.isArray(o[a]))throw new TypeError("Arrays are not supported by deepExtend");n(i,o,a,s)}}return i},e.selectiveNotDeepExtend=function(t,i,o){var s=arguments.length>3&&void 0!==arguments[3]&&arguments[3];if(Array.isArray(o))throw new TypeError("Arrays are not supported by deepExtend");for(var r in o)if(o.hasOwnProperty(r)&&-1===t.indexOf(r))if(o[r]&&o[r].constructor===Object)void 0===i[r]&&(i[r]={}),i[r].constructor===Object?e.deepExtend(i[r],o[r]):n(i,o,r,s);else if(Array.isArray(o[r])){i[r]=[];for(var a=0;a<o[r].length;a++)i[r].push(o[r][a])}else n(i,o,r,s);return i},e.deepExtend=function(t,i){var o=arguments.length>2&&void 0!==arguments[2]&&arguments[2],s=arguments.length>3&&void 0!==arguments[3]&&arguments[3];for(var r in i)if(i.hasOwnProperty(r)||!0===o)if(i[r]&&i[r].constructor===Object)void 0===t[r]&&(t[r]={}),t[r].constructor===Object?e.deepExtend(t[r],i[r],o):n(t,i,r,s);else if(Array.isArray(i[r])){t[r]=[];for(var a=0;a<i[r].length;a++)t[r].push(i[r][a])}else n(t,i,r,s);return t},e.equalArray=function(t,e){if(t.length!=e.length)return!1;for(var i=0,o=t.length;i<o;i++)if(t[i]!=e[i])return!1;return!0},e.convert=function(t,i){var o;if(void 0!==t){if(null===t)return null;if(!i)return t;if("string"!=typeof i&&!(i instanceof String))throw new Error("Type must be a string");switch(i){case"boolean":case"Boolean":return Boolean(t);case"number":case"Number":return e.isString(t)&&!isNaN(Date.parse(t))?c(t).valueOf():Number(t.valueOf());case"string":case"String":return String(t);case"Date":if(e.isNumber(t))return new Date(t);if(t instanceof Date)return new Date(t.valueOf());if(c.isMoment(t))return new Date(t.valueOf());if(e.isString(t))return o=f.exec(t),o?new Date(Number(o[1])):c(new Date(t)).toDate();throw new Error("Cannot convert object of type "+e.getType(t)+" to type Date");case"Moment":if(e.isNumber(t))return c(t);if(t instanceof Date)return c(t.valueOf());if(c.isMoment(t))return c(t);if(e.isString(t))return o=f.exec(t),c(o?Number(o[1]):t);throw new Error("Cannot convert object of type "+e.getType(t)+" to type Date");case"ISODate":if(e.isNumber(t))return new Date(t);if(t instanceof Date)return t.toISOString();if(c.isMoment(t))return t.toDate().toISOString();if(e.isString(t))return o=f.exec(t),o?new Date(Number(o[1])).toISOString():c(t).format();throw new Error("Cannot convert object of type "+e.getType(t)+" to type ISODate");case"ASPDate":if(e.isNumber(t))return"/Date("+t+")/";if(t instanceof Date)return"/Date("+t.valueOf()+")/";if(e.isString(t)){o=f.exec(t);return"/Date("+(o?new Date(Number(o[1])).valueOf():new Date(t).valueOf())+")/"}throw new Error("Cannot convert object of type "+e.getType(t)+" to type ASPDate");default:throw new Error('Unknown type "'+i+'"')}}};var f=/^\/?Date\((\-?\d+)/i;e.getType=function(t){var e=void 0===t?"undefined":(0,p.default)(t);return"object"==e?null===t?"null":t instanceof Boolean?"Boolean":t instanceof Number?"Number":t instanceof String?"String":Array.isArray(t)?"Array":t instanceof Date?"Date":"Object":"number"==e?"Number":"boolean"==e?"Boolean":"string"==e?"String":void 0===e?"undefined":e},e.copyAndExtendArray=function(t,e){for(var i=[],o=0;o<t.length;o++)i.push(t[o]);return i.push(e),i},e.copyArray=function(t){for(var e=[],i=0;i<t.length;i++)e.push(t[i]);return e},e.getAbsoluteLeft=function(t){return t.getBoundingClientRect().left},e.getAbsoluteRight=function(t){return t.getBoundingClientRect().right},e.getAbsoluteTop=function(t){return t.getBoundingClientRect().top},e.addClassName=function(t,e){var i=t.className.split(" "),o=e.split(" ");i=i.concat(o.filter(function(t){return i.indexOf(t)<0})),t.className=i.join(" ")},e.removeClassName=function(t,e){var i=t.className.split(" "),o=e.split(" ");i=i.filter(function(t){return o.indexOf(t)<0}),t.className=i.join(" ")},e.forEach=function(t,e){var i,o;if(Array.isArray(t))for(i=0,o=t.length;i<o;i++)e(t[i],i,t);else for(i in t)t.hasOwnProperty(i)&&e(t[i],i,t)},e.toArray=function(t){var e=[];for(var i in t)t.hasOwnProperty(i)&&e.push(t[i]);return e},e.updateProperty=function(t,e,i){return t[e]!==i&&(t[e]=i,!0)},e.throttle=function(t){var e=!1;return function(){e||(e=!0,requestAnimationFrame(function(){e=!1,t()}))}},e.addEventListener=function(t,e,i,o){t.addEventListener?(void 0===o&&(o=!1),"mousewheel"===e&&navigator.userAgent.indexOf("Firefox")>=0&&(e="DOMMouseScroll"),t.addEventListener(e,i,o)):t.attachEvent("on"+e,i)},e.removeEventListener=function(t,e,i,o){t.removeEventListener?(void 0===o&&(o=!1),"mousewheel"===e&&navigator.userAgent.indexOf("Firefox")>=0&&(e="DOMMouseScroll"),t.removeEventListener(e,i,o)):t.detachEvent("on"+e,i)},e.preventDefault=function(t){t||(t=window.event),t.preventDefault?t.preventDefault():t.returnValue=!1},e.getTarget=function(t){t||(t=window.event);var e;return t.target?e=t.target:t.srcElement&&(e=t.srcElement),void 0!=e.nodeType&&3==e.nodeType&&(e=e.parentNode),e},e.hasParent=function(t,e){for(var i=t;i;){if(i===e)return!0;i=i.parentNode}return!1},e.option={},e.option.asBoolean=function(t,e){return"function"==typeof t&&(t=t()),null!=t?0!=t:e||null},e.option.asNumber=function(t,e){return"function"==typeof t&&(t=t()),null!=t?Number(t)||e||null:e||null},e.option.asString=function(t,e){return"function"==typeof t&&(t=t()),null!=t?String(t):e||null},e.option.asSize=function(t,i){return"function"==typeof t&&(t=t()),e.isString(t)?t:e.isNumber(t)?t+"px":i||null},e.option.asElement=function(t,e){return"function"==typeof t&&(t=t()),t||e||null},e.hexToRGB=function(t){var e=/^#?([a-f\d])([a-f\d])([a-f\d])$/i;t=t.replace(e,function(t,e,i,o){return e+e+i+i+o+o});var i=/^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i.exec(t);return i?{r:parseInt(i[1],16),g:parseInt(i[2],16),b:parseInt(i[3],16)}:null},e.overrideOpacity=function(t,i){var o;return-1!=t.indexOf("rgba")?t:-1!=t.indexOf("rgb")?(o=t.substr(t.indexOf("(")+1).replace(")","").split(","),"rgba("+o[0]+","+o[1]+","+o[2]+","+i+")"):(o=e.hexToRGB(t),null==o?t:"rgba("+o.r+","+o.g+","+o.b+","+i+")")},e.RGBToHex=function(t,e,i){return"#"+((1<<24)+(t<<16)+(e<<8)+i).toString(16).slice(1)},e.parseColor=function(t){var i;if(!0===e.isString(t)){if(!0===e.isValidRGB(t)){var o=t.substr(4).substr(0,t.length-5).split(",").map(function(t){return parseInt(t)});t=e.RGBToHex(o[0],o[1],o[2])}if(!0===e.isValidHex(t)){var n=e.hexToHSV(t),s={h:n.h,s:.8*n.s,v:Math.min(1,1.02*n.v)},r={h:n.h,s:Math.min(1,1.25*n.s),v:.8*n.v},a=e.HSVToHex(r.h,r.s,r.v),h=e.HSVToHex(s.h,s.s,s.v);i={background:t,border:a,highlight:{background:h,border:a},hover:{background:h,border:a}}}else i={background:t,border:t,highlight:{background:t,border:t},hover:{background:t,border:t}}}else i={},i.background=t.background||void 0,i.border=t.border||void 0,e.isString(t.highlight)?i.highlight={border:t.highlight,background:t.highlight}:(i.highlight={},i.highlight.background=t.highlight&&t.highlight.background||void 0,i.highlight.border=t.highlight&&t.highlight.border||void 0),e.isString(t.hover)?i.hover={border:t.hover,background:t.hover}:(i.hover={},i.hover.background=t.hover&&t.hover.background||void 0,i.hover.border=t.hover&&t.hover.border||void 0);return i},e.RGBToHSV=function(t,e,i){t/=255,e/=255,i/=255;var o=Math.min(t,Math.min(e,i)),n=Math.max(t,Math.max(e,i));if(o==n)return{h:0,s:0,v:o};var s=t==o?e-i:i==o?t-e:i-t;return{h:60*((t==o?3:i==o?1:5)-s/(n-o))/360,s:(n-o)/n,v:n}};var g={split:function(t){var e={};return t.split(";").forEach(function(t){if(""!=t.trim()){var i=t.split(":"),o=i[0].trim(),n=i[1].trim();e[o]=n}}),e},join:function(t){return(0,l.default)(t).map(function(e){return e+": "+t[e]}).join("; ")}};e.addCssText=function(t,i){var o=g.split(t.style.cssText),n=g.split(i),s=e.extend(o,n);t.style.cssText=g.join(s)},e.removeCssText=function(t,e){var i=g.split(t.style.cssText),o=g.split(e);for(var n in o)o.hasOwnProperty(n)&&delete i[n];t.style.cssText=g.join(i)},e.HSVToRGB=function(t,e,i){var o,n,s,r=Math.floor(6*t),a=6*t-r,h=i*(1-e),d=i*(1-a*e),l=i*(1-(1-a)*e);switch(r%6){case 0:o=i,n=l,s=h;break;case 1:o=d,n=i,s=h;break;case 2:o=h,n=i,s=l;break;case 3:o=h,n=d,s=i;break;case 4:o=l,n=h,s=i;break;case 5:o=i,n=h,s=d}return{r:Math.floor(255*o),g:Math.floor(255*n),b:Math.floor(255*s)}},e.HSVToHex=function(t,i,o){var n=e.HSVToRGB(t,i,o);return e.RGBToHex(n.r,n.g,n.b)},e.hexToHSV=function(t){var i=e.hexToRGB(t);return e.RGBToHSV(i.r,i.g,i.b)},e.isValidHex=function(t){return/(^#[0-9A-F]{6}$)|(^#[0-9A-F]{3}$)/i.test(t)},e.isValidRGB=function(t){return t=t.replace(" ",""),/rgb\((\d{1,3}),(\d{1,3}),(\d{1,3})\)/i.test(t)},e.isValidRGBA=function(t){return t=t.replace(" ",""),/rgba\((\d{1,3}),(\d{1,3}),(\d{1,3}),(.{1,3})\)/i.test(t)},e.selectiveBridgeObject=function(t,i){if(null!==i&&"object"===(void 0===i?"undefined":(0,p.default)(i))){for(var o=(0,h.default)(i),n=0;n<t.length;n++)i.hasOwnProperty(t[n])&&"object"==(0,p.default)(i[t[n]])&&(o[t[n]]=e.bridgeObject(i[t[n]]));return o}return null},e.bridgeObject=function(t){if(null!==t&&"object"===(void 0===t?"undefined":(0,p.default)(t))){var i=(0,h.default)(t);if(t instanceof Element)i=t;else{i=(0,h.default)(t);for(var o in t)t.hasOwnProperty(o)&&"object"==(0,p.default)(t[o])&&(i[o]=e.bridgeObject(t[o]))}return i}return null},e.insertSort=function(t,e){for(var i=0;i<t.length;i++){for(var o=t[i],n=i;n>0&&e(o,t[n-1])<0;n--)t[n]=t[n-1];t[n]=o}return t},e.mergeOptions=function(t,e,i){var o=arguments.length>3&&void 0!==arguments[3]?arguments[3]:{},n=function(t){return null!==t&&void 0!==t},s=function(t){return null!==t&&"object"===(void 0===t?"undefined":(0,p.default)(t))};if(!s(t))throw new Error("Parameter mergeTarget must be an object");if(!s(e))throw new Error("Parameter options must be an object");if(!n(i))throw new Error("Parameter option must have a value");if(!s(o))throw new Error("Parameter globalOptions must be an object");var r=e[i],a=s(o)&&!function(t){for(var e in t)if(t.hasOwnProperty(e))return!1;return!0}(o),d=a?o[i]:void 0,l=d?d.enabled:void 0;if(void 0!==r){if("boolean"==typeof r)return s(t[i])||(t[i]={}),void(t[i].enabled=r);if(null===r&&!s(t[i])){if(!n(d))return;t[i]=(0,h.default)(d)}if(s(r)){var u=!0;void 0!==r.enabled?u=r.enabled:void 0!==l&&(u=d.enabled),function(t,e,i){s(t[i])||(t[i]={});var o=e[i],n=t[i];for(var r in o)o.hasOwnProperty(r)&&(n[r]=o[r])}(t,e,i),t[i].enabled=u}}},e.binarySearchCustom=function(t,e,i,o){for(var n=0,s=0,r=t.length-1;s<=r&&n<1e4;){var a=Math.floor((s+r)/2),h=t[a],d=void 0===o?h[i]:h[i][o],l=e(d);if(0==l)return a;-1==l?s=a+1:r=a-1,n++}return-1},e.binarySearchValue=function(t,e,i,o,n){var s,r,a,h,d=0,l=0,u=t.length-1;for(n=void 0!=n?n:function(t,e){return t==e?0:t<e?-1:1};l<=u&&d<1e4;){if(h=Math.floor(.5*(u+l)),s=t[Math.max(0,h-1)][i],r=t[h][i],a=t[Math.min(t.length-1,h+1)][i],0==n(r,e))return h;if(n(s,e)<0&&n(r,e)>0)return"before"==o?Math.max(0,h-1):h;if(n(r,e)<0&&n(a,e)>0)return"before"==o?h:Math.min(t.length-1,h+1);n(r,e)<0?l=h+1:u=h-1,d++}return-1},e.easingFunctions={linear:function(t){return t},easeInQuad:function(t){return t*t},easeOutQuad:function(t){return t*(2-t)},easeInOutQuad:function(t){return t<.5?2*t*t:(4-2*t)*t-1},easeInCubic:function(t){return t*t*t},easeOutCubic:function(t){return--t*t*t+1},easeInOutCubic:function(t){return t<.5?4*t*t*t:(t-1)*(2*t-2)*(2*t-2)+1},easeInQuart:function(t){return t*t*t*t},easeOutQuart:function(t){return 1- --t*t*t*t},easeInOutQuart:function(t){return t<.5?8*t*t*t*t:1-8*--t*t*t*t},easeInQuint:function(t){return t*t*t*t*t},easeOutQuint:function(t){return 1+--t*t*t*t*t},easeInOutQuint:function(t){return t<.5?16*t*t*t*t*t:1+16*--t*t*t*t*t}},e.getScrollBarWidth=function(){var t=document.createElement("p");t.style.width="100%",t.style.height="200px";var e=document.createElement("div");e.style.position="absolute",e.style.top="0px",e.style.left="0px",e.style.visibility="hidden",e.style.width="200px",e.style.height="150px",e.style.overflow="hidden",e.appendChild(t),document.body.appendChild(e);var i=t.offsetWidth;e.style.overflow="scroll";var o=t.offsetWidth;return i==o&&(o=e.clientWidth),document.body.removeChild(e),i-o},e.topMost=function(t,e){var i=void 0;Array.isArray(e)||(e=[e]);var o=!0,n=!1,s=void 0;try{for(var a,h=(0,r.default)(t);!(o=(a=h.next()).done);o=!0){var d=a.value;if(d){i=d[e[0]];for(var l=1;l<e.length;l++)i&&(i=i[e[l]]);if(void 0!==i)break}}}catch(t){n=!0,s=t}finally{try{!o&&h.return&&h.return()}finally{if(n)throw s}}return i}},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}e.__esModule=!0;var n=i(110),s=o(n),r=i(112),a=o(r),h="function"==typeof a.default&&"symbol"==typeof s.default?function(t){return typeof t}:function(t){return t&&"function"==typeof a.default&&t.constructor===a.default&&t!==a.default.prototype?"symbol":typeof t};e.default="function"==typeof a.default&&"symbol"===h(s.default)?function(t){return void 0===t?"undefined":h(t)}:function(t){return t&&"function"==typeof a.default&&t.constructor===a.default&&t!==a.default.prototype?"symbol":void 0===t?"undefined":h(t)}},function(t,e){var i=t.exports={version:"2.5.1"};"number"==typeof __e&&(__e=i)},function(t,e,i){t.exports="undefined"!=typeof window&&window.moment||i(123)},function(t,e,i){t.exports={default:i(107),__esModule:!0}},function(t,e,i){function o(t,e){this.options=null,this.props=null}var n=i(0);o.prototype.setOptions=function(t){t&&n.extend(this.options,t)},o.prototype.redraw=function(){return!1},o.prototype.destroy=function(){},o.prototype._isResized=function(){var t=this.props._previousWidth!==this.props.width||this.props._previousHeight!==this.props.height;return this.props._previousWidth=this.props.width,this.props._previousHeight=this.props.height,t},t.exports=o},function(t,e){var i=t.exports="undefined"!=typeof window&&window.Math==Math?window:"undefined"!=typeof self&&self.Math==Math?self:Function("return this")();"number"==typeof __g&&(__g=i)},function(t,e,i){var o=i(40)("wks"),n=i(27),s=i(6).Symbol,r="function"==typeof s;(t.exports=function(t){return o[t]||(o[t]=r&&s[t]||(r?s:n)("Symbol."+t))}).store=o},function(t,e,i){var o=i(18),n=i(57),s=i(36),r=Object.defineProperty;e.f=i(9)?Object.defineProperty:function(t,e,i){if(o(t),e=s(e,!0),o(i),n)try{return r(t,e,i)}catch(t){}if("get"in i||"set"in i)throw TypeError("Accessors not supported!");return"value"in i&&(t[e]=i.value),t}},function(t,e,i){t.exports=!i(19)(function(){return 7!=Object.defineProperty({},"a",{get:function(){return 7}}).a})},function(t,e){var i={}.hasOwnProperty;t.exports=function(t,e){return i.call(t,e)}},function(t,e,i){e.prepareElements=function(t){for(var e in t)t.hasOwnProperty(e)&&(t[e].redundant=t[e].used,t[e].used=[])},e.cleanupElements=function(t){for(var e in t)if(t.hasOwnProperty(e)&&t[e].redundant){for(var i=0;i<t[e].redundant.length;i++)t[e].redundant[i].parentNode.removeChild(t[e].redundant[i]);t[e].redundant=[]}},e.resetElements=function(t){e.prepareElements(t),e.cleanupElements(t),e.prepareElements(t)},e.getSVGElement=function(t,e,i){var o;return e.hasOwnProperty(t)?e[t].redundant.length>0?(o=e[t].redundant[0],e[t].redundant.shift()):(o=document.createElementNS("http://www.w3.org/2000/svg",t),i.appendChild(o)):(o=document.createElementNS("http://www.w3.org/2000/svg",t),e[t]={used:[],redundant:[]},i.appendChild(o)),e[t].used.push(o),o},e.getDOMElement=function(t,e,i,o){var n;return e.hasOwnProperty(t)?e[t].redundant.length>0?(n=e[t].redundant[0],e[t].redundant.shift()):(n=document.createElement(t),void 0!==o?i.insertBefore(n,o):i.appendChild(n)):(n=document.createElement(t),e[t]={used:[],redundant:[]},void 0!==o?i.insertBefore(n,o):i.appendChild(n)),e[t].used.push(n),n},e.drawPoint=function(t,i,o,n,s,r){var a;if("circle"==o.style?(a=e.getSVGElement("circle",n,s),a.setAttributeNS(null,"cx",t),a.setAttributeNS(null,"cy",i),a.setAttributeNS(null,"r",.5*o.size)):(a=e.getSVGElement("rect",n,s),a.setAttributeNS(null,"x",t-.5*o.size),a.setAttributeNS(null,"y",i-.5*o.size),a.setAttributeNS(null,"width",o.size),a.setAttributeNS(null,"height",o.size)),void 0!==o.styles&&a.setAttributeNS(null,"style",o.styles),a.setAttributeNS(null,"class",o.className+" vis-point"),r){var h=e.getSVGElement("text",n,s);r.xOffset&&(t+=r.xOffset),r.yOffset&&(i+=r.yOffset),r.content&&(h.textContent=r.content),r.className&&h.setAttributeNS(null,"class",r.className+" vis-label"),h.setAttributeNS(null,"x",t),h.setAttributeNS(null,"y",i)}return a},e.drawBar=function(t,i,o,n,s,r,a,h){if(0!=n){n<0&&(n*=-1,i-=n);var d=e.getSVGElement("rect",r,a);d.setAttributeNS(null,"x",t-.5*o),d.setAttributeNS(null,"y",i),d.setAttributeNS(null,"width",o),d.setAttributeNS(null,"height",n),d.setAttributeNS(null,"class",s),h&&d.setAttributeNS(null,"style",h)}}},function(t,e,i){function o(){var t=function(){};return{on:t,off:t,destroy:t,emit:t,get:function(e){return{set:t}}}}if("undefined"!=typeof window){var n=i(130),s=window.Hammer||i(131);t.exports=n(s,{preventDefault:"mouse"})}else t.exports=function(){return o()}},function(t,e,i){var o=i(91),n=i(34);t.exports=function(t){return o(n(t))}},function(t,e,i){var o=i(8),n=i(25);t.exports=i(9)?function(t,e,i){return o.f(t,e,n(1,i))}:function(t,e,i){return t[e]=i,t}},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}function n(t,e){if(t&&!Array.isArray(t)&&(e=t,t=null),this._options=e||{},this._data={},this.length=0,this._fieldId=this._options.fieldId||"id",this._type={},this._options.type)for(var i=(0,l.default)(this._options.type),o=0,n=i.length;o<n;o++){var s=i[o],r=this._options.type[s];this._type[s]="Date"==r||"ISODate"==r||"ASPDate"==r?"Date":r}this._subscribers={},t&&this.add(t),this.setOptions(e)}var s=i(16),r=o(s),a=i(1),h=o(a),d=i(4),l=o(d),u=i(0),p=i(65);n.prototype.setOptions=function(t){t&&void 0!==t.queue&&(!1===t.queue?this._queue&&(this._queue.destroy(),delete this._queue):(this._queue||(this._queue=p.extend(this,{replace:["add","update","remove"]})),"object"===(0,h.default)(t.queue)&&this._queue.setOptions(t.queue)))},n.prototype.on=function(t,e){var i=this._subscribers[t];i||(i=[],this._subscribers[t]=i),i.push({callback:e})},n.prototype.off=function(t,e){var i=this._subscribers[t];i&&(this._subscribers[t]=i.filter(function(t){return t.callback!=e}))},n.prototype._trigger=function(t,e,i){if("*"==t)throw new Error("Cannot trigger event *");var o=[];t in this._subscribers&&(o=o.concat(this._subscribers[t])),"*"in this._subscribers&&(o=o.concat(this._subscribers["*"]));for(var n=0,s=o.length;n<s;n++){var r=o[n];r.callback&&r.callback(t,e,i||null)}},n.prototype.add=function(t,e){var i,o=[],n=this;if(Array.isArray(t))for(var s=0,r=t.length;s<r;s++)i=n._addItem(t[s]),o.push(i);else{if(!t||"object"!==(void 0===t?"undefined":(0,h.default)(t)))throw new Error("Unknown dataType");i=n._addItem(t),o.push(i)}return o.length&&this._trigger("add",{items:o},e),o},n.prototype.update=function(t,e){var i=[],o=[],n=[],s=[],r=this,a=r._fieldId,d=function(t){var e=t[a];if(r._data[e]){var h=u.extend({},r._data[e]);e=r._updateItem(t),o.push(e),s.push(t),n.push(h)}else e=r._addItem(t),i.push(e)};if(Array.isArray(t))for(var l=0,p=t.length;l<p;l++)t[l]&&"object"===(0,h.default)(t[l])?d(t[l]):console.warn("Ignoring input item, which is not an object at index "+l);else{if(!t||"object"!==(void 0===t?"undefined":(0,h.default)(t)))throw new Error("Unknown dataType");d(t)}if(i.length&&this._trigger("add",{items:i},e),o.length){var c={items:o,oldData:n,data:s};this._trigger("update",c,e)}return i.concat(o)},n.prototype.get=function(t){var e,i,o,n=this,s=u.getType(arguments[0]);"String"==s||"Number"==s?(e=arguments[0],o=arguments[1]):"Array"==s?(i=arguments[0],o=arguments[1]):o=arguments[0];var r;if(o&&o.returnType){r=-1==["Array","Object"].indexOf(o.returnType)?"Array":o.returnType}else r="Array";var a,h,d,p,c,m=o&&o.type||this._options.type,f=o&&o.filter,g=[];if(void 0!=e)(a=n._getItem(e,m))&&f&&!f(a)&&(a=null);else if(void 0!=i)for(p=0,c=i.length;p<c;p++)a=n._getItem(i[p],m),f&&!f(a)||g.push(a);else for(h=(0,l.default)(this._data),p=0,c=h.length;p<c;p++)d=h[p],a=n._getItem(d,m),f&&!f(a)||g.push(a);if(o&&o.order&&void 0==e&&this._sort(g,o.order),o&&o.fields){var v=o.fields;if(void 0!=e)a=this._filterFields(a,v);else for(p=0,c=g.length;p<c;p++)g[p]=this._filterFields(g[p],v)}if("Object"==r){var y,b={};for(p=0,c=g.length;p<c;p++)y=g[p],b[y.id]=y;return b}return void 0!=e?a:g},n.prototype.getIds=function(t){var e,i,o,n,s,r=this._data,a=t&&t.filter,h=t&&t.order,d=t&&t.type||this._options.type,u=(0,l.default)(r),p=[];if(a)if(h){for(s=[],e=0,i=u.length;e<i;e++)o=u[e],n=this._getItem(o,d),a(n)&&s.push(n);for(this._sort(s,h),e=0,i=s.length;e<i;e++)p.push(s[e][this._fieldId])}else for(e=0,i=u.length;e<i;e++)o=u[e],n=this._getItem(o,d),a(n)&&p.push(n[this._fieldId]);else if(h){for(s=[],e=0,i=u.length;e<i;e++)o=u[e],s.push(r[o]);for(this._sort(s,h),e=0,i=s.length;e<i;e++)p.push(s[e][this._fieldId])}else for(e=0,i=u.length;e<i;e++)o=u[e],n=r[o],p.push(n[this._fieldId]);return p},n.prototype.getDataSet=function(){return this},n.prototype.forEach=function(t,e){var i,o,n,s,r=e&&e.filter,a=e&&e.type||this._options.type,h=this._data,d=(0,l.default)(h);if(e&&e.order){var u=this.get(e);for(i=0,o=u.length;i<o;i++)n=u[i],s=n[this._fieldId],t(n,s)}else for(i=0,o=d.length;i<o;i++)s=d[i],n=this._getItem(s,a),r&&!r(n)||t(n,s)},n.prototype.map=function(t,e){var i,o,n,s,r=e&&e.filter,a=e&&e.type||this._options.type,h=[],d=this._data,u=(0,l.default)(d);for(i=0,o=u.length;i<o;i++)n=u[i],s=this._getItem(n,a),r&&!r(s)||h.push(t(s,n));return e&&e.order&&this._sort(h,e.order),h},n.prototype._filterFields=function(t,e){if(!t)return t;var i,o,n={},s=(0,l.default)(t),r=s.length;if(Array.isArray(e))for(i=0;i<r;i++)o=s[i],-1!=e.indexOf(o)&&(n[o]=t[o]);else for(i=0;i<r;i++)o=s[i],e.hasOwnProperty(o)&&(n[e[o]]=t[o]);return n},n.prototype._sort=function(t,e){if(u.isString(e)){var i=e;t.sort(function(t,e){var o=t[i],n=e[i];return o>n?1:o<n?-1:0})}else{if("function"!=typeof e)throw new TypeError("Order must be a function or a string");t.sort(e)}},n.prototype.remove=function(t,e){var i,o,n,s,r=[],a=[],h=[];for(h=Array.isArray(t)?t:[t],i=0,o=h.length;i<o;i++)(s=this._remove(h[i]))&&void 0!=(n=s[this._fieldId])&&(r.push(n),a.push(s));return r.length&&this._trigger("remove",{items:r,oldData:a},e),r},n.prototype._remove=function(t){var e,i;return u.isNumber(t)||u.isString(t)?i=t:t&&"object"===(void 0===t?"undefined":(0,h.default)(t))&&(i=t[this._fieldId]),void 0!==i&&this._data[i]?(e=this._data[i],delete this._data[i],this.length--,e):null},n.prototype.clear=function(t){var e,i,o=(0,l.default)(this._data),n=[];for(e=0,i=o.length;e<i;e++)n.push(this._data[o[e]]);return this._data={},this.length=0,this._trigger("remove",{items:o,oldData:n},t),o},n.prototype.max=function(t){var e,i,o=this._data,n=(0,l.default)(o),s=null,r=null;for(e=0,i=n.length;e<i;e++){var a=n[e],h=o[a],d=h[t];null!=d&&(!s||d>r)&&(s=h,r=d)}return s},n.prototype.min=function(t){var e,i,o=this._data,n=(0,l.default)(o),s=null,r=null;for(e=0,i=n.length;e<i;e++){var a=n[e],h=o[a],d=h[t];null!=d&&(!s||d<r)&&(s=h,r=d)}return s},n.prototype.distinct=function(t){var e,i,o,n=this._data,s=(0,l.default)(n),r=[],a=this._options.type&&this._options.type[t]||null,h=0;for(e=0,o=s.length;e<o;e++){var d=s[e],p=n[d],c=p[t],m=!1;for(i=0;i<h;i++)if(r[i]==c){m=!0;break}m||void 0===c||(r[h]=c,h++)}if(a)for(e=0,o=r.length;e<o;e++)r[e]=u.convert(r[e],a);return r},n.prototype._addItem=function(t){var e=t[this._fieldId];if(void 0!=e){if(this._data[e])throw new Error("Cannot add item: item with id "+e+" already exists")}else e=u.randomUUID(),t[this._fieldId]=e;var i,o,n={},s=(0,l.default)(t);for(i=0,o=s.length;i<o;i++){var r=s[i],a=this._type[r];n[r]=u.convert(t[r],a)}return this._data[e]=n,this.length++,e},n.prototype._getItem=function(t,e){var i,o,n,s,r=this._data[t];if(!r)return null;var a={},h=(0,l.default)(r);if(e)for(n=0,s=h.length;n<s;n++)i=h[n],o=r[i],a[i]=u.convert(o,e[i]);else for(n=0,s=h.length;n<s;n++)i=h[n],o=r[i],a[i]=o;return a[this._fieldId]||(a[this._fieldId]=r.id),a},n.prototype._updateItem=function(t){var e=t[this._fieldId];if(void 0==e)throw new Error("Cannot update item: item has no id (item: "+(0,r.default)(t)+")");var i=this._data[e];if(!i)throw new Error("Cannot update item: no item with id "+e+" found");for(var o=(0,l.default)(t),n=0,s=o.length;n<s;n++){var a=o[n],h=this._type[a];i[a]=u.convert(t[a],h)}return e},t.exports=n},function(t,e,i){t.exports={default:i(128),__esModule:!0}},function(t,e,i){var o=i(6),n=i(2),s=i(92),r=i(14),a=function(t,e,i){var h,d,l,u=t&a.F,p=t&a.G,c=t&a.S,m=t&a.P,f=t&a.B,g=t&a.W,v=p?n:n[e]||(n[e]={}),y=v.prototype,b=p?o:c?o[e]:(o[e]||{}).prototype;p&&(i=e);for(h in i)(d=!u&&b&&void 0!==b[h])&&h in v||(l=d?b[h]:i[h],v[h]=p&&"function"!=typeof b[h]?i[h]:f&&d?s(l,o):g&&b[h]==l?function(t){var e=function(e,i,o){if(this instanceof t){switch(arguments.length){case 0:return new t;case 1:return new t(e);case 2:return new t(e,i)}return new t(e,i,o)}return t.apply(this,arguments)};return e.prototype=t.prototype,e}(l):m&&"function"==typeof l?s(Function.call,l):l,m&&((v.virtual||(v.virtual={}))[h]=l,t&a.R&&y&&!y[h]&&r(y,h,l)))};a.F=1,a.G=2,a.S=4,a.P=8,a.B=16,a.W=32,a.U=64,a.R=128,t.exports=a},function(t,e,i){var o=i(24);t.exports=function(t){if(!o(t))throw TypeError(t+" is not an object!");return t}},function(t,e){t.exports=function(t){try{return!!t()}catch(t){return!0}}},function(t,e,i){function o(t,e){this._data=null,this._ids={},this.length=0,this._options=e||{},this._fieldId="id",this._subscribers={};var i=this;this.listener=function(){i._onEvent.apply(i,arguments)},this.setData(t)}var n=i(4),s=function(t){return t&&t.__esModule?t:{default:t}}(n),r=i(0),a=i(15);o.prototype.setData=function(t){var e,i,o,n,s;if(this._data){for(this._data.off&&this._data.off("*",this.listener),e=this._data.getIds({filter:this._options&&this._options.filter}),s=[],o=0,n=e.length;o<n;o++)s.push(this._data._data[e[o]]);this._ids={},this.length=0,this._trigger("remove",{items:e,oldData:s})}if(this._data=t,this._data){for(this._fieldId=this._options.fieldId||this._data&&this._data.options&&this._data.options.fieldId||"id",e=this._data.getIds({filter:this._options&&this._options.filter}),o=0,n=e.length;o<n;o++)i=e[o],this._ids[i]=!0;this.length=e.length,this._trigger("add",{items:e}),this._data.on&&this._data.on("*",this.listener)}},o.prototype.refresh=function(){var t,e,i,o=this._data.getIds({filter:this._options&&this._options.filter}),n=(0,s.default)(this._ids),r={},a=[],h=[],d=[];for(e=0,i=o.length;e<i;e++)t=o[e],r[t]=!0,this._ids[t]||(a.push(t),this._ids[t]=!0);for(e=0,i=n.length;e<i;e++)t=n[e],r[t]||(h.push(t),d.push(this._data._data[t]),delete this._ids[t]);this.length+=a.length-h.length,a.length&&this._trigger("add",{items:a}),h.length&&this._trigger("remove",{items:h,oldData:d})},o.prototype.get=function(t){var e,i,o,n=this,s=r.getType(arguments[0]);"String"==s||"Number"==s||"Array"==s?(e=arguments[0],i=arguments[1],o=arguments[2]):(i=arguments[0],o=arguments[1]);var a=r.extend({},this._options,i);this._options.filter&&i&&i.filter&&(a.filter=function(t){return n._options.filter(t)&&i.filter(t)});var h=[];return void 0!=e&&h.push(e),h.push(a),h.push(o),this._data&&this._data.get.apply(this._data,h)},o.prototype.getIds=function(t){var e;if(this._data){var i,o=this._options.filter;i=t&&t.filter?o?function(e){return o(e)&&t.filter(e)}:t.filter:o,e=this._data.getIds({filter:i,order:t&&t.order})}else e=[];return e},o.prototype.map=function(t,e){var i=[];if(this._data){var o,n=this._options.filter;o=e&&e.filter?n?function(t){return n(t)&&e.filter(t)}:e.filter:n,i=this._data.map(t,{filter:o,order:e&&e.order})}else i=[];return i},o.prototype.getDataSet=function(){for(var t=this;t instanceof o;)t=t._data;return t||null},o.prototype._onEvent=function(t,e,i){var o,n,s,r,a=e&&e.items,h=[],d=[],l=[],u=[],p=[],c=[];if(a&&this._data){switch(t){case"add":for(o=0,n=a.length;o<n;o++)s=a[o],(r=this.get(s))&&(this._ids[s]=!0,h.push(s));break;case"update":for(o=0,n=a.length;o<n;o++)s=a[o],r=this.get(s),r?this._ids[s]?(d.push(s),p.push(e.data[o]),u.push(e.oldData[o])):(this._ids[s]=!0,h.push(s)):this._ids[s]&&(delete this._ids[s],l.push(s),c.push(e.oldData[o]));break;case"remove":for(o=0,n=a.length;o<n;o++)s=a[o],this._ids[s]&&(delete this._ids[s],l.push(s),c.push(e.oldData[o]))}this.length+=h.length-l.length,h.length&&this._trigger("add",{items:h},i),d.length&&this._trigger("update",{items:d,oldData:u,data:p},i),l.length&&this._trigger("remove",{items:l,oldData:c},i)}},o.prototype.on=a.prototype.on,o.prototype.off=a.prototype.off,o.prototype._trigger=a.prototype._trigger,o.prototype.subscribe=o.prototype.on,o.prototype.unsubscribe=o.prototype.off,t.exports=o},function(t,e,i){e.convertHiddenOptions=function(t,i,o){if(o&&!Array.isArray(o))return e.convertHiddenOptions(t,i,[o]);if(i.hiddenDates=[],o&&1==Array.isArray(o)){for(var n=0;n<o.length;n++)if(void 0===o[n].repeat){var s={};s.start=t(o[n].start).toDate().valueOf(),s.end=t(o[n].end).toDate().valueOf(),i.hiddenDates.push(s)}i.hiddenDates.sort(function(t,e){return t.start-e.start})}},e.updateHiddenDates=function(t,i,o){if(o&&!Array.isArray(o))return e.updateHiddenDates(t,i,[o]);if(o&&void 0!==i.domProps.centerContainer.width){e.convertHiddenOptions(t,i,o);for(var n=t(i.range.start),s=t(i.range.end),r=i.range.end-i.range.start,a=r/i.domProps.centerContainer.width,h=0;h<o.length;h++)if(void 0!==o[h].repeat){var d=t(o[h].start),l=t(o[h].end);if("Invalid Date"==d._d)throw new Error("Supplied start date is not valid: "+o[h].start);if("Invalid Date"==l._d)throw new Error("Supplied end date is not valid: "+o[h].end);var u=l-d;if(u>=4*a){var p=0,c=s.clone();switch(o[h].repeat){case"daily":d.day()!=l.day()&&(p=1),d.dayOfYear(n.dayOfYear()),d.year(n.year()),d.subtract(7,"days"),l.dayOfYear(n.dayOfYear()),l.year(n.year()),l.subtract(7-p,"days"),c.add(1,"weeks");break;case"weekly":var m=l.diff(d,"days"),f=d.day();d.date(n.date()),d.month(n.month()),d.year(n.year()),l=d.clone(),d.day(f),l.day(f),l.add(m,"days"),d.subtract(1,"weeks"),l.subtract(1,"weeks"),c.add(1,"weeks");break;case"monthly":d.month()!=l.month()&&(p=1),d.month(n.month()),d.year(n.year()),d.subtract(1,"months"),l.month(n.month()),l.year(n.year()),l.subtract(1,"months"),l.add(p,"months"),c.add(1,"months");break;case"yearly":d.year()!=l.year()&&(p=1),d.year(n.year()),d.subtract(1,"years"),l.year(n.year()),l.subtract(1,"years"),l.add(p,"years"),c.add(1,"years");break;default:return void console.log("Wrong repeat format, allowed are: daily, weekly, monthly, yearly. Given:",o[h].repeat)}for(;d<c;)switch(i.hiddenDates.push({start:d.valueOf(),end:l.valueOf()}),o[h].repeat){case"daily":d.add(1,"days"),l.add(1,"days");break;case"weekly":d.add(1,"weeks"),l.add(1,"weeks");break;case"monthly":d.add(1,"months"),l.add(1,"months");break;case"yearly":d.add(1,"y"),l.add(1,"y");break;default:return void console.log("Wrong repeat format, allowed are: daily, weekly, monthly, yearly. Given:",o[h].repeat)}i.hiddenDates.push({start:d.valueOf(),end:l.valueOf()})}}e.removeDuplicates(i);var g=e.isHidden(i.range.start,i.hiddenDates),v=e.isHidden(i.range.end,i.hiddenDates),y=i.range.start,b=i.range.end;1==g.hidden&&(y=1==i.range.startToFront?g.startDate-1:g.endDate+1),1==v.hidden&&(b=1==i.range.endToFront?v.startDate-1:v.endDate+1),1!=g.hidden&&1!=v.hidden||i.range._applyRange(y,b)}},e.removeDuplicates=function(t){for(var e=t.hiddenDates,i=[],o=0;o<e.length;o++)for(var n=0;n<e.length;n++)o!=n&&1!=e[n].remove&&1!=e[o].remove&&(e[n].start>=e[o].start&&e[n].end<=e[o].end?e[n].remove=!0:e[n].start>=e[o].start&&e[n].start<=e[o].end?(e[o].end=e[n].end,e[n].remove=!0):e[n].end>=e[o].start&&e[n].end<=e[o].end&&(e[o].start=e[n].start,e[n].remove=!0));for(o=0;o<e.length;o++)!0!==e[o].remove&&i.push(e[o]);t.hiddenDates=i,t.hiddenDates.sort(function(t,e){return t.start-e.start})},e.printDates=function(t){for(var e=0;e<t.length;e++)console.log(e,new Date(t[e].start),new Date(t[e].end),t[e].start,t[e].end,t[e].remove)},e.stepOverHiddenDates=function(t,e,i){for(var o=!1,n=e.current.valueOf(),s=0;s<e.hiddenDates.length;s++){var r=e.hiddenDates[s].start,a=e.hiddenDates[s].end;if(n>=r&&n<a){o=!0;break}}if(1==o&&n<e._end.valueOf()&&n!=i){var h=t(i),d=t(a);h.year()!=d.year()?e.switchedYear=!0:h.month()!=d.month()?e.switchedMonth=!0:h.dayOfYear()!=d.dayOfYear()&&(e.switchedDay=!0),e.current=d}},e.toScreen=function(t,i,o){var n;if(0==t.body.hiddenDates.length)return n=t.range.conversion(o),(i.valueOf()-n.offset)*n.scale;var s=e.isHidden(i,t.body.hiddenDates);1==s.hidden&&(i=s.startDate);var r=e.getHiddenDurationBetween(t.body.hiddenDates,t.range.start,t.range.end);if(i<t.range.start){n=t.range.conversion(o,r);var a=e.getHiddenDurationBeforeStart(t.body.hiddenDates,i,n.offset);return i=t.options.moment(i).toDate().valueOf(),i+=a,-(n.offset-i.valueOf())*n.scale}if(i>t.range.end){var h={start:t.range.start,end:i};return i=e.correctTimeForHidden(t.options.moment,t.body.hiddenDates,h,i),n=t.range.conversion(o,r),(i.valueOf()-n.offset)*n.scale}return i=e.correctTimeForHidden(t.options.moment,t.body.hiddenDates,t.range,i),n=t.range.conversion(o,r),(i.valueOf()-n.offset)*n.scale},e.toTime=function(t,i,o){if(0==t.body.hiddenDates.length){var n=t.range.conversion(o);return new Date(i/n.scale+n.offset)}var s=e.getHiddenDurationBetween(t.body.hiddenDates,t.range.start,t.range.end),r=t.range.end-t.range.start-s,a=r*i/o,h=e.getAccumulatedHiddenDuration(t.body.hiddenDates,t.range,a);return new Date(h+a+t.range.start)},e.getHiddenDurationBetween=function(t,e,i){for(var o=0,n=0;n<t.length;n++){var s=t[n].start,r=t[n].end;s>=e&&r<i&&(o+=r-s)}return o},e.getHiddenDurationBeforeStart=function(t,e,i){for(var o=0,n=0;n<t.length;n++){var s=t[n].start,r=t[n].end;s>=e&&r<=i&&(o+=r-s)}return o},e.correctTimeForHidden=function(t,i,o,n){return n=t(n).toDate().valueOf(),n-=e.getHiddenDurationBefore(t,i,o,n)},e.getHiddenDurationBefore=function(t,e,i,o){var n=0;o=t(o).toDate().valueOf();for(var s=0;s<e.length;s++){var r=e[s].start,a=e[s].end;r>=i.start&&a<i.end&&o>=a&&(n+=a-r)}return n},e.getAccumulatedHiddenDuration=function(t,e,i){for(var o=0,n=0,s=e.start,r=0;r<t.length;r++){var a=t[r].start,h=t[r].end;if(a>=e.start&&h<e.end){if(n+=a-s,s=h,n>=i)break;o+=h-a}}return o},e.snapAwayFromHidden=function(t,i,o,n){var s=e.isHidden(i,t);return 1==s.hidden?o<0?1==n?s.startDate-(s.endDate-i)-1:s.startDate-1:1==n?s.endDate+(i-s.startDate)+1:s.endDate+1:i},e.isHidden=function(t,e){for(var i=0;i<e.length;i++){var o=e[i].start,n=e[i].end;if(t>=o&&t<n)return{hidden:!0,startDate:o,endDate:n}}return{hidden:!1,startDate:o,endDate:n}}},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}function n(t,e,i){this.id=null,this.parent=null,this.data=t,this.dom=null,this.conversion=e||{},this.options=i||{},this.selected=!1,this.displayed=!1,this.groupShowing=!0,this.dirty=!0,this.top=null,this.right=null,this.left=null,this.width=null,this.height=null,this.editable=null,this._updateEditStatus()}var s=i(1),r=o(s),a=i(4),h=o(a),d=i(12),l=i(0),u=i(3);n.prototype.stack=!0,n.prototype.select=function(){this.selected=!0,this.dirty=!0,this.displayed&&this.redraw()},n.prototype.unselect=function(){this.selected=!1,this.dirty=!0,this.displayed&&this.redraw()},n.prototype.setData=function(t){void 0!=t.group&&this.data.group!=t.group&&null!=this.parent&&this.parent.itemSet._moveToGroup(this,t.group),this.parent&&(this.parent.stackDirty=!0),void 0!=t.subgroup&&this.data.subgroup!=t.subgroup&&null!=this.parent&&this.parent.changeSubgroup(this,this.data.subgroup,t.subgroup),this.data=t,this._updateEditStatus(),this.dirty=!0,this.displayed&&this.redraw()},n.prototype.setParent=function(t){this.displayed?(this.hide(),this.parent=t,this.parent&&this.show()):this.parent=t},n.prototype.isVisible=function(t){return!1},n.prototype.show=function(){return!1},n.prototype.hide=function(){return!1},n.prototype.redraw=function(){},n.prototype.repositionX=function(){},n.prototype.repositionY=function(){},n.prototype._repaintDragCenter=function(){if(this.selected&&this.options.editable.updateTime&&!this.dom.dragCenter){var t=this,e=document.createElement("div");e.className="vis-drag-center",e.dragCenterItem=this;var i=new d(e);i.on("tap",function(e){t.parent.itemSet.body.emitter.emit("click",{event:e,item:t.id})}),i.on("doubletap",function(e){e.stopPropagation(),t.parent.itemSet._onUpdateItem(t),t.parent.itemSet.body.emitter.emit("doubleClick",{event:e,item:t.id})}),this.dom.box?this.dom.dragLeft?this.dom.box.insertBefore(e,this.dom.dragLeft):this.dom.box.appendChild(e):this.dom.point&&this.dom.point.appendChild(e),this.dom.dragCenter=e}else!this.selected&&this.dom.dragCenter&&(this.dom.dragCenter.parentNode&&this.dom.dragCenter.parentNode.removeChild(this.dom.dragCenter),this.dom.dragCenter=null)},n.prototype._repaintDeleteButton=function(t){var e=(this.options.editable.overrideItems||null==this.editable)&&this.options.editable.remove||!this.options.editable.overrideItems&&null!=this.editable&&this.editable.remove;if(this.selected&&e&&!this.dom.deleteButton){var i=this,o=document.createElement("div");this.options.rtl?o.className="vis-delete-rtl":o.className="vis-delete",o.title="Delete this item",new d(o).on("tap",function(t){t.stopPropagation(),i.parent.removeFromDataSet(i)}),t.appendChild(o),this.dom.deleteButton=o}else!this.selected&&this.dom.deleteButton&&(this.dom.deleteButton.parentNode&&this.dom.deleteButton.parentNode.removeChild(this.dom.deleteButton),this.dom.deleteButton=null)},n.prototype._repaintOnItemUpdateTimeTooltip=function(t){if(this.options.tooltipOnItemUpdateTime){var e=(this.options.editable.updateTime||!0===this.data.editable)&&!1!==this.data.editable;if(this.selected&&e&&!this.dom.onItemUpdateTimeTooltip){var i=document.createElement("div");i.className="vis-onUpdateTime-tooltip",t.appendChild(i),this.dom.onItemUpdateTimeTooltip=i}else!this.selected&&this.dom.onItemUpdateTimeTooltip&&(this.dom.onItemUpdateTimeTooltip.parentNode&&this.dom.onItemUpdateTimeTooltip.parentNode.removeChild(this.dom.onItemUpdateTimeTooltip),this.dom.onItemUpdateTimeTooltip=null);if(this.dom.onItemUpdateTimeTooltip){this.dom.onItemUpdateTimeTooltip.style.visibility=this.parent.itemSet.touchParams.itemIsDragging?"visible":"hidden",this.options.rtl?this.dom.onItemUpdateTimeTooltip.style.right=this.dom.content.style.right:this.dom.onItemUpdateTimeTooltip.style.left=this.dom.content.style.left;var o,n=this.parent.itemSet.body.domProps.scrollTop;o="top"==this.options.orientation.item?this.top:this.parent.height-this.top-this.height;o+this.parent.top-50<-n?(this.dom.onItemUpdateTimeTooltip.style.bottom="",this.dom.onItemUpdateTimeTooltip.style.top=this.height+2+"px"):(this.dom.onItemUpdateTimeTooltip.style.top="",this.dom.onItemUpdateTimeTooltip.style.bottom=this.height+2+"px");var s,r;this.options.tooltipOnItemUpdateTime&&this.options.tooltipOnItemUpdateTime.template?(r=this.options.tooltipOnItemUpdateTime.template.bind(this),s=r(this.data)):(s="start: "+u(this.data.start).format("MM/DD/YYYY hh:mm"),this.data.end&&(s+="<br> end: "+u(this.data.end).format("MM/DD/YYYY hh:mm"))),this.dom.onItemUpdateTimeTooltip.innerHTML=s}}},n.prototype._updateContents=function(t){var e,i,o,n,s=this.parent.itemSet.itemsData.get(this.id),r=this.dom.box||this.dom.point,a=r.getElementsByClassName("vis-item-visible-frame")[0];if(this.options.visibleFrameTemplate?(n=this.options.visibleFrameTemplate.bind(this),o=n(s,r)):o="",a)if(o instanceof Object&&!(o instanceof Element))n(s,a);else if(this._contentToString(this.itemVisibleFrameContent)!==this._contentToString(o)){if(o instanceof Element)a.innerHTML="",a.appendChild(o);else if(void 0!=o)a.innerHTML=o;else if("background"!=this.data.type||void 0!==this.data.content)throw new Error('Property "content" missing in item '+this.id);this.itemVisibleFrameContent=o}if(this.options.template?(i=this.options.template.bind(this),e=i(s,t,this.data)):e=this.data.content,e instanceof Object&&!(e instanceof Element))i(s,t);else if(this._contentToString(this.content)!==this._contentToString(e)){if(e instanceof Element)t.innerHTML="",t.appendChild(e);else if(void 0!=e)t.innerHTML=e;else if("background"!=this.data.type||void 0!==this.data.content)throw new Error('Property "content" missing in item '+this.id);this.content=e}},n.prototype._updateDataAttributes=function(t){if(this.options.dataAttributes&&this.options.dataAttributes.length>0){var e=[];if(Array.isArray(this.options.dataAttributes))e=this.options.dataAttributes;else{if("all"!=this.options.dataAttributes)return;e=(0,h.default)(this.data)}for(var i=0;i<e.length;i++){var o=e[i],n=this.data[o];null!=n?t.setAttribute("data-"+o,n):t.removeAttribute("data-"+o)}}},n.prototype._updateStyle=function(t){this.style&&(l.removeCssText(t,this.style),this.style=null),this.data.style&&(l.addCssText(t,this.data.style),this.style=this.data.style)},n.prototype._contentToString=function(t){return"string"==typeof t?t:t&&"outerHTML"in t?t.outerHTML:t},n.prototype._updateEditStatus=function(){this.options&&("boolean"==typeof this.options.editable?this.editable={updateTime:this.options.editable,updateGroup:this.options.editable,remove:this.options.editable}:"object"===(0,r.default)(this.options.editable)&&(this.editable={},l.selectiveExtend(["updateTime","updateGroup","remove"],this.editable,this.options.editable))),this.options&&this.options.editable&&!0===this.options.editable.overrideItems||this.data&&("boolean"==typeof this.data.editable?this.editable={updateTime:this.data.editable,updateGroup:this.data.editable,remove:this.data.editable}:"object"===(0,r.default)(this.data.editable)&&(this.editable={},l.selectiveExtend(["updateTime","updateGroup","remove"],this.editable,this.data.editable)))},n.prototype.getWidthLeft=function(){return 0},n.prototype.getWidthRight=function(){return 0},n.prototype.getTitle=function(){return this.data.title},t.exports=n},function(t,e){t.exports={}},function(t,e){t.exports=function(t){return"object"==typeof t?null!==t:"function"==typeof t}},function(t,e){t.exports=function(t,e){return{enumerable:!(1&t),configurable:!(2&t),writable:!(4&t),value:e}}},function(t,e,i){var o=i(60),n=i(41);t.exports=Object.keys||function(t){return o(t,n)}},function(t,e){var i=0,o=Math.random();t.exports=function(t){return"Symbol(".concat(void 0===t?"":t,")_",(++i+o).toString(36))}},function(t,e,i){function o(t,e){this.dom={foreground:null,lines:[],majorTexts:[],minorTexts:[],redundant:{lines:[],majorTexts:[],minorTexts:[]}},this.props={range:{start:0,end:0,minimumStep:0},lineTop:0},this.defaultOptions={orientation:{axis:"bottom"},showMinorLabels:!0,showMajorLabels:!0,maxMinorChars:7,format:h.FORMAT,moment:l,timeAxis:null},this.options=r.extend({},this.defaultOptions),this.body=t,this._create(),this.setOptions(e)}var n=i(1),s=function(t){return t&&t.__esModule?t:{default:t}}(n),r=i(0),a=i(5),h=i(49),d=i(21),l=i(3);o.prototype=new a,o.prototype.setOptions=function(t){t&&(r.selectiveExtend(["showMinorLabels","showMajorLabels","maxMinorChars","hiddenDates","timeAxis","moment","rtl"],this.options,t),r.selectiveDeepExtend(["format"],this.options,t),"orientation"in t&&("string"==typeof t.orientation?this.options.orientation.axis=t.orientation:"object"===(0,s.default)(t.orientation)&&"axis"in t.orientation&&(this.options.orientation.axis=t.orientation.axis)),"locale"in t&&("function"==typeof l.locale?l.locale(t.locale):l.lang(t.locale)))},o.prototype._create=function(){this.dom.foreground=document.createElement("div"),this.dom.background=document.createElement("div"),this.dom.foreground.className="vis-time-axis vis-foreground",this.dom.background.className="vis-time-axis vis-background"},o.prototype.destroy=function(){this.dom.foreground.parentNode&&this.dom.foreground.parentNode.removeChild(this.dom.foreground),this.dom.background.parentNode&&this.dom.background.parentNode.removeChild(this.dom.background),this.body=null},o.prototype.redraw=function(){var t=this.props,e=this.dom.foreground,i=this.dom.background,o="top"==this.options.orientation.axis?this.body.dom.top:this.body.dom.bottom,n=e.parentNode!==o;this._calculateCharSize();var s=this.options.showMinorLabels&&"none"!==this.options.orientation.axis,r=this.options.showMajorLabels&&"none"!==this.options.orientation.axis;t.minorLabelHeight=s?t.minorCharHeight:0,t.majorLabelHeight=r?t.majorCharHeight:0,t.height=t.minorLabelHeight+t.majorLabelHeight,t.width=e.offsetWidth,t.minorLineHeight=this.body.domProps.root.height-t.majorLabelHeight-("top"==this.options.orientation.axis?this.body.domProps.bottom.height:this.body.domProps.top.height),t.minorLineWidth=1,t.majorLineHeight=t.minorLineHeight+t.majorLabelHeight,t.majorLineWidth=1;var a=e.nextSibling,h=i.nextSibling;return e.parentNode&&e.parentNode.removeChild(e),i.parentNode&&i.parentNode.removeChild(i),e.style.height=this.props.height+"px",this._repaintLabels(),a?o.insertBefore(e,a):o.appendChild(e),h?this.body.dom.backgroundVertical.insertBefore(i,h):this.body.dom.backgroundVertical.appendChild(i),this._isResized()||n},o.prototype._repaintLabels=function(){var t=this.options.orientation.axis,e=r.convert(this.body.range.start,"Number"),i=r.convert(this.body.range.end,"Number"),o=this.body.util.toTime((this.props.minorCharWidth||10)*this.options.maxMinorChars).valueOf(),n=o-d.getHiddenDurationBefore(this.options.moment,this.body.hiddenDates,this.body.range,o);n-=this.body.util.toTime(0).valueOf();var s=new h(new Date(e),new Date(i),n,this.body.hiddenDates,this.options);s.setMoment(this.options.moment),this.options.format&&s.setFormat(this.options.format),this.options.timeAxis&&s.setScale(this.options.timeAxis),this.step=s;var a=this.dom;a.redundant.lines=a.lines,a.redundant.majorTexts=a.majorTexts,a.redundant.minorTexts=a.minorTexts,a.lines=[],a.majorTexts=[],a.minorTexts=[];var l,p,c,m,f,g,v,y,b,_=0,w=void 0,x=0;for(s.start(),l=s.getCurrent(),c=this.body.util.toScreen(l);s.hasNext()&&x<1e3;){switch(x++,m=s.isMajor(),b=s.getClassName(),y=s.getLabelMinor(),l,p=c,s.next(),l=s.getCurrent(),s.isMajor(),c=this.body.util.toScreen(l),g=_,_=c-p,s.scale){case"week":f=!0;break;default:f=_>=.4*g}if(this.options.showMinorLabels&&f){var D=this._repaintMinorText(p,y,t,b);D.style.width=_+"px"}m&&this.options.showMajorLabels?(p>0&&(void 0==w&&(w=p),D=this._repaintMajorText(p,s.getLabelMajor(),t,b)),v=this._repaintMajorLine(p,_,t,b)):f?v=this._repaintMinorLine(p,_,t,b):v&&(v.style.width=parseInt(v.style.width)+_+"px")}if(1e3!==x||u||(console.warn("Something is wrong with the Timeline scale. Limited drawing of grid lines to 1000 lines."),u=!0),this.options.showMajorLabels){var S=this.body.util.toTime(0),k=s.getLabelMajor(S),C=k.length*(this.props.majorCharWidth||10)+10;(void 0==w||C<w)&&this._repaintMajorText(0,k,t,b)}r.forEach(this.dom.redundant,function(t){for(;t.length;){var e=t.pop();e&&e.parentNode&&e.parentNode.removeChild(e)}})},o.prototype._repaintMinorText=function(t,e,i,o){var n=this.dom.redundant.minorTexts.shift();if(!n){var s=document.createTextNode("");n=document.createElement("div"),n.appendChild(s),this.dom.foreground.appendChild(n)}return this.dom.minorTexts.push(n),n.innerHTML=e,n.style.top="top"==i?this.props.majorLabelHeight+"px":"0",this.options.rtl?(n.style.left="",n.style.right=t+"px"):n.style.left=t+"px",n.className="vis-text vis-minor "+o,n},o.prototype._repaintMajorText=function(t,e,i,o){var n=this.dom.redundant.majorTexts.shift();if(!n){var s=document.createElement("div");n=document.createElement("div"),n.appendChild(s),this.dom.foreground.appendChild(n)}return n.childNodes[0].innerHTML=e,n.className="vis-text vis-major "+o,n.style.top="top"==i?"0":this.props.minorLabelHeight+"px",this.options.rtl?(n.style.left="",n.style.right=t+"px"):n.style.left=t+"px",this.dom.majorTexts.push(n),n},o.prototype._repaintMinorLine=function(t,e,i,o){var n=this.dom.redundant.lines.shift();n||(n=document.createElement("div"),this.dom.background.appendChild(n)),this.dom.lines.push(n);var s=this.props;return n.style.top="top"==i?s.majorLabelHeight+"px":this.body.domProps.top.height+"px",n.style.height=s.minorLineHeight+"px",this.options.rtl?(n.style.left="",n.style.right=t-s.minorLineWidth/2+"px",n.className="vis-grid vis-vertical-rtl vis-minor "+o):(n.style.left=t-s.minorLineWidth/2+"px",n.className="vis-grid vis-vertical vis-minor "+o),n.style.width=e+"px",n},o.prototype._repaintMajorLine=function(t,e,i,o){var n=this.dom.redundant.lines.shift();n||(n=document.createElement("div"),this.dom.background.appendChild(n)),this.dom.lines.push(n);var s=this.props;return n.style.top="top"==i?"0":this.body.domProps.top.height+"px",this.options.rtl?(n.style.left="",n.style.right=t-s.majorLineWidth/2+"px",n.className="vis-grid vis-vertical-rtl vis-major "+o):(n.style.left=t-s.majorLineWidth/2+"px",n.className="vis-grid vis-vertical vis-major "+o),n.style.height=s.majorLineHeight+"px",n.style.width=e+"px",n},o.prototype._calculateCharSize=function(){this.dom.measureCharMinor||(this.dom.measureCharMinor=document.createElement("DIV"),this.dom.measureCharMinor.className="vis-text vis-minor vis-measure",this.dom.measureCharMinor.style.position="absolute",this.dom.measureCharMinor.appendChild(document.createTextNode("0")),this.dom.foreground.appendChild(this.dom.measureCharMinor)),this.props.minorCharHeight=this.dom.measureCharMinor.clientHeight,this.props.minorCharWidth=this.dom.measureCharMinor.clientWidth,this.dom.measureCharMajor||(this.dom.measureCharMajor=document.createElement("DIV"),this.dom.measureCharMajor.className="vis-text vis-major vis-measure",this.dom.measureCharMajor.style.position="absolute",this.dom.measureCharMajor.appendChild(document.createTextNode("0")),this.dom.foreground.appendChild(this.dom.measureCharMajor)),this.props.majorCharHeight=this.dom.measureCharMajor.clientHeight,this.props.majorCharWidth=this.dom.measureCharMajor.clientWidth};var u=!1;t.exports=o},function(t,e,i){function o(t,e){this.body=t,this.defaultOptions={moment:a,locales:h,locale:"en",id:void 0,title:void 0},this.options=s.extend({},this.defaultOptions),e&&e.time?this.customTime=e.time:this.customTime=new Date,this.eventParams={},this.setOptions(e),this._create()}var n=i(12),s=i(0),r=i(5),a=i(3),h=i(69);o.prototype=new r,o.prototype.setOptions=function(t){t&&s.selectiveExtend(["moment","locale","locales","id"],this.options,t)},o.prototype._create=function(){function t(t){this.body.range._onMouseWheel(t)}var e=document.createElement("div");e["custom-time"]=this,e.className="vis-custom-time "+(this.options.id||""),e.style.position="absolute",e.style.top="0px",e.style.height="100%",this.bar=e;var i=document.createElement("div");i.style.position="relative",i.style.top="0px",i.style.left="-10px",i.style.height="100%",i.style.width="20px",i.addEventListener?(i.addEventListener("mousewheel",t.bind(this),!1),i.addEventListener("DOMMouseScroll",t.bind(this),!1)):i.attachEvent("onmousewheel",t.bind(this)),e.appendChild(i),this.hammer=new n(i),this.hammer.on("panstart",this._onDragStart.bind(this)),this.hammer.on("panmove",this._onDrag.bind(this)),this.hammer.on("panend",this._onDragEnd.bind(this)),this.hammer.get("pan").set({threshold:5,direction:n.DIRECTION_HORIZONTAL})},o.prototype.destroy=function(){this.hide(),this.hammer.destroy(),this.hammer=null,this.body=null},o.prototype.redraw=function(){var t=this.body.dom.backgroundVertical;this.bar.parentNode!=t&&(this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar),t.appendChild(this.bar));var e=this.body.util.toScreen(this.customTime),i=this.options.locales[this.options.locale];i||(this.warned||(console.log("WARNING: options.locales['"+this.options.locale+"'] not found. See http://visjs.org/docs/timeline/#Localization"),this.warned=!0),i=this.options.locales.en);var o=this.options.title;return void 0===o?(o=i.time+": "+this.options.moment(this.customTime).format("dddd, MMMM Do YYYY, H:mm:ss"),o=o.charAt(0).toUpperCase()+o.substring(1)):"function"==typeof o&&(o=o.call(this.customTime)),this.bar.style.left=e+"px",this.bar.title=o,!1},o.prototype.hide=function(){this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar)},o.prototype.setCustomTime=function(t){this.customTime=s.convert(t,"Date"),this.redraw()},o.prototype.getCustomTime=function(){return new Date(this.customTime.valueOf())},o.prototype.setCustomTitle=function(t){this.options.title=t},o.prototype._onDragStart=function(t){this.eventParams.dragging=!0,this.eventParams.customTime=this.customTime,t.stopPropagation()},o.prototype._onDrag=function(t){if(this.eventParams.dragging){var e=this.body.util.toScreen(this.eventParams.customTime)+t.deltaX,i=this.body.util.toTime(e);this.setCustomTime(i),this.body.emitter.emit("timechange",{id:this.options.id,time:new Date(this.customTime.valueOf()),event:t}),t.stopPropagation()}},o.prototype._onDragEnd=function(t){this.eventParams.dragging&&(this.body.emitter.emit("timechanged",{id:this.options.id,time:new Date(this.customTime.valueOf()),event:t}),t.stopPropagation())},o.customTimeFromTarget=function(t){for(var e=t.target;e;){if(e.hasOwnProperty("custom-time"))return e["custom-time"];e=e.parentNode}return null},t.exports=o},function(t,e,i){e.__esModule=!0,e.default=function(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}},function(t,e,i){e.__esModule=!0;var o=i(134),n=function(t){return t&&t.__esModule?t:{default:t}}(o);e.default=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),(0,n.default)(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}()},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}Object.defineProperty(e,"__esModule",{value:!0}),e.printStyle=void 0;var n=i(16),s=o(n),r=i(1),a=o(r),h=i(4),d=o(h),l=i(30),u=o(l),p=i(31),c=o(p),m=i(0),f=!1,g=void 0,v="background: #FFeeee; color: #dd0000",y=function(){function t(){(0,u.default)(this,t)}return(0,c.default)(t,null,[{key:"validate",value:function(e,i,o){f=!1,g=i;var n=i;return void 0!==o&&(n=i[o]),t.parse(e,n,[]),f}},{key:"parse",value:function(e,i,o){for(var n in e)e.hasOwnProperty(n)&&t.check(n,e,i,o)}},{key:"check",value:function(e,i,o,n){if(void 0===o[e]&&void 0===o.__any__)return void t.getSuggestion(e,o,n);var s=e,r=!0;void 0===o[e]&&void 0!==o.__any__&&(s="__any__",r="object"===t.getType(i[e]));var a=o[s];r&&void 0!==a.__type__&&(a=a.__type__),t.checkFields(e,i,o,s,a,n)}},{key:"checkFields",value:function(e,i,o,n,s,r){var a=function(i){console.log("%c"+i+t.printLocation(r,e),v)},h=t.getType(i[e]),l=s[h];void 0!==l?"array"===t.getType(l)&&-1===l.indexOf(i[e])?(a('Invalid option detected in "'+e+'". Allowed values are:'+t.print(l)+' not "'+i[e]+'". '),f=!0):"object"===h&&"__any__"!==n&&(r=m.copyAndExtendArray(r,e),t.parse(i[e],o[n],r)):void 0===s.any&&(a('Invalid type received for "'+e+'". Expected: '+t.print((0,d.default)(s))+". Received ["+h+'] "'+i[e]+'"'),f=!0)}},{key:"getType",value:function(t){var e=void 0===t?"undefined":(0,a.default)(t);return"object"===e?null===t?"null":t instanceof Boolean?"boolean":t instanceof Number?"number":t instanceof String?"string":Array.isArray(t)?"array":t instanceof Date?"date":void 0!==t.nodeType?"dom":!0===t._isAMomentObject?"moment":"object":"number"===e?"number":"boolean"===e?"boolean":"string"===e?"string":void 0===e?"undefined":e}},{key:"getSuggestion",value:function(e,i,o){var n=t.findInOptions(e,i,o,!1),s=t.findInOptions(e,g,[],!0),r=void 0;r=void 0!==n.indexMatch?" in "+t.printLocation(n.path,e,"")+'Perhaps it was incomplete? Did you mean: "'+n.indexMatch+'"?\n\n':s.distance<=4&&n.distance>s.distance?" in "+t.printLocation(n.path,e,"")+"Perhaps it was misplaced? Matching option found at: "+t.printLocation(s.path,s.closestMatch,""):n.distance<=8?'. Did you mean "'+n.closestMatch+'"?'+t.printLocation(n.path,e):". Did you mean one of these: "+t.print((0,d.default)(i))+t.printLocation(o,e),console.log('%cUnknown option detected: "'+e+'"'+r,v),f=!0}},{key:"findInOptions",value:function(e,i,o){var n=arguments.length>3&&void 0!==arguments[3]&&arguments[3],s=1e9,r="",a=[],h=e.toLowerCase(),d=void 0;for(var l in i){var u=void 0;if(void 0!==i[l].__type__&&!0===n){var p=t.findInOptions(e,i[l],m.copyAndExtendArray(o,l));s>p.distance&&(r=p.closestMatch,a=p.path,s=p.distance,d=p.indexMatch)}else-1!==l.toLowerCase().indexOf(h)&&(d=l),u=t.levenshteinDistance(e,l),s>u&&(r=l,a=m.copyArray(o),s=u)}return{closestMatch:r,path:a,distance:s,indexMatch:d}}},{key:"printLocation",value:function(t,e){for(var i=arguments.length>2&&void 0!==arguments[2]?arguments[2]:"Problem value found at: \n",o="\n\n"+i+"options = {\n",n=0;n<t.length;n++){for(var s=0;s<n+1;s++)o+="  ";o+=t[n]+": {\n"}for(var r=0;r<t.length+1;r++)o+="  ";o+=e+"\n";for(var a=0;a<t.length+1;a++){for(var h=0;h<t.length-a;h++)o+="  ";o+="}\n"}return o+"\n\n"}},{key:"print",value:function(t){return(0,s.default)(t).replace(/(\")|(\[)|(\])|(,"__type__")/g,"").replace(/(\,)/g,", ")}},{key:"levenshteinDistance",value:function(t,e){if(0===t.length)return e.length;if(0===e.length)return t.length;var i,o=[];for(i=0;i<=e.length;i++)o[i]=[i];var n;for(n=0;n<=t.length;n++)o[0][n]=n;for(i=1;i<=e.length;i++)for(n=1;n<=t.length;n++)e.charAt(i-1)==t.charAt(n-1)?o[i][n]=o[i-1][n-1]:o[i][n]=Math.min(o[i-1][n-1]+1,Math.min(o[i][n-1]+1,o[i-1][n]+1));return o[e.length][t.length]}}]),t}();e.default=y,e.printStyle=v},function(t,e){var i={}.toString;t.exports=function(t){return i.call(t).slice(8,-1)}},function(t,e){t.exports=function(t){if(void 0==t)throw TypeError("Can't call method on  "+t);return t}},function(t,e){t.exports=!0},function(t,e,i){var o=i(24);t.exports=function(t,e){if(!o(t))return t;var i,n;if(e&&"function"==typeof(i=t.toString)&&!o(n=i.call(t)))return n;if("function"==typeof(i=t.valueOf)&&!o(n=i.call(t)))return n;if(!e&&"function"==typeof(i=t.toString)&&!o(n=i.call(t)))return n;throw TypeError("Can't convert object to primitive value")}},function(t,e,i){var o=i(18),n=i(95),s=i(41),r=i(39)("IE_PROTO"),a=function(){},h=function(){var t,e=i(58)("iframe"),o=s.length;for(e.style.display="none",i(99).appendChild(e),e.src="javascript:",t=e.contentWindow.document,t.open(),t.write("<script>document.F=Object<\/script>"),t.close(),h=t.F;o--;)delete h.prototype[s[o]];return h()};t.exports=Object.create||function(t,e){var i;return null!==t?(a.prototype=o(t),i=new a,a.prototype=null,i[r]=t):i=h(),void 0===e?i:n(i,e)}},function(t,e){var i=Math.ceil,o=Math.floor;t.exports=function(t){return isNaN(t=+t)?0:(t>0?o:i)(t)}},function(t,e,i){var o=i(40)("keys"),n=i(27);t.exports=function(t){return o[t]||(o[t]=n(t))}},function(t,e,i){var o=i(6),n=o["__core-js_shared__"]||(o["__core-js_shared__"]={});t.exports=function(t){return n[t]||(n[t]={})}},function(t,e){t.exports="constructor,hasOwnProperty,isPrototypeOf,propertyIsEnumerable,toLocaleString,toString,valueOf".split(",")},function(t,e,i){var o=i(8).f,n=i(10),s=i(7)("toStringTag");t.exports=function(t,e,i){t&&!n(t=i?t:t.prototype,s)&&o(t,s,{configurable:!0,value:e})}},function(t,e,i){t.exports={default:i(105),__esModule:!0}},function(t,e,i){e.f=i(7)},function(t,e,i){var o=i(6),n=i(2),s=i(35),r=i(44),a=i(8).f;t.exports=function(t){var e=n.Symbol||(n.Symbol=s?{}:o.Symbol||{});"_"==t.charAt(0)||t in e||a(e,t,{value:r.f(t)})}},function(t,e){e.f={}.propertyIsEnumerable},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}function n(t,e){var i=c().hours(0).minutes(0).seconds(0).milliseconds(0),o=i.clone().add(-3,"days").valueOf(),n=i.clone().add(3,"days").valueOf();this.millisecondsPerPixelCache=void 0,void 0===e?(this.start=o,this.end=n):(this.start=e.start||o,this.end=e.end||n),this.rolling=!1,this.body=t,this.deltaDifference=0,this.scaleOffset=0,this.startToFront=!1,this.endToFront=!0,this.defaultOptions={rtl:!1,start:null,end:null,moment:c,direction:"horizontal",moveable:!0,zoomable:!0,min:null,max:null,zoomMin:10,zoomMax:31536e10,rollingMode:{follow:!1,offset:.5}},this.options=p.extend({},this.defaultOptions),this.props={touch:{}},this.animationTimer=null,this.body.emitter.on("panstart",this._onDragStart.bind(this)),this.body.emitter.on("panmove",this._onDrag.bind(this)),this.body.emitter.on("panend",this._onDragEnd.bind(this)),this.body.emitter.on("mousewheel",this._onMouseWheel.bind(this)),this.body.emitter.on("touch",this._onTouch.bind(this)),this.body.emitter.on("pinch",this._onPinch.bind(this)),this.body.dom.rollingModeBtn.addEventListener("click",this.startRolling.bind(this)),this.setOptions(e)}function s(t){if("horizontal"!=t&&"vertical"!=t)throw new TypeError('Unknown direction "'+t+'". Choose "horizontal" or "vertical".')}var r=i(4),a=o(r),h=i(16),d=o(h),l=i(1),u=o(l),p=i(0),c=i(3),m=i(5),f=i(21);n.prototype=new m,n.prototype.setOptions=function(t){if(t){var e=["animation","direction","min","max","zoomMin","zoomMax","moveable","zoomable","moment","activate","hiddenDates","zoomKey","rtl","showCurrentTime","rollingMode","horizontalScroll"];p.selectiveExtend(e,this.options,t),t.rollingMode&&t.rollingMode.follow&&this.startRolling(),("start"in t||"end"in t)&&this.setRange(t.start,t.end)}},n.prototype.startRolling=function(){function t(){e.stopRolling(),e.rolling=!0;var i=e.end-e.start,o=p.convert(new Date,"Date").valueOf(),n=o-i*e.options.rollingMode.offset,s=o+i*(1-e.options.rollingMode.offset),r={animation:!1};e.setRange(n,s,r),i=1/e.conversion(e.body.domProps.center.width).scale/10,i<30&&(i=30),i>1e3&&(i=1e3),e.body.dom.rollingModeBtn.style.visibility="hidden",e.currentTimeTimer=setTimeout(t,i)}var e=this;t()},n.prototype.stopRolling=function(){void 0!==this.currentTimeTimer&&(clearTimeout(this.currentTimeTimer),this.rolling=!1,this.body.dom.rollingModeBtn.style.visibility="visible")},n.prototype.setRange=function(t,e,i,o,n){i||(i={}),!0!==i.byUser&&(i.byUser=!1);var s=this,r=void 0!=t?p.convert(t,"Date").valueOf():null,h=void 0!=e?p.convert(e,"Date").valueOf():null;if(this._cancelAnimation(),this.millisecondsPerPixelCache=void 0,i.animation){var l=this.start,c=this.end,m="object"===(0,u.default)(i.animation)&&"duration"in i.animation?i.animation.duration:500,g="object"===(0,u.default)(i.animation)&&"easingFunction"in i.animation?i.animation.easingFunction:"easeInOutQuad",v=p.easingFunctions[g];if(!v)throw new Error("Unknown easing function "+(0,d.default)(g)+". Choose from: "+(0,a.default)(p.easingFunctions).join(", "));var y=(new Date).valueOf(),b=!1;return function t(){if(!s.props.touch.dragging){var e=(new Date).valueOf(),a=e-y,d=v(a/m),u=a>m,p=u||null===r?r:l+(r-l)*d,g=u||null===h?h:c+(h-c)*d;_=s._applyRange(p,g),f.updateHiddenDates(s.options.moment,s.body,s.options.hiddenDates),b=b||_;var w={start:new Date(s.start),end:new Date(s.end),byUser:i.byUser,event:i.event};if(n&&n(d,_,u),_&&s.body.emitter.emit("rangechange",w),u){if(b&&(s.body.emitter.emit("rangechanged",w),o))return o()}else s.animationTimer=setTimeout(t,20)}}()}var _=this._applyRange(r,h);if(f.updateHiddenDates(this.options.moment,this.body,this.options.hiddenDates),_){var w={start:new Date(this.start),end:new Date(this.end),byUser:i.byUser,event:i.event};if(this.body.emitter.emit("rangechange",w),clearTimeout(s.timeoutID),s.timeoutID=setTimeout(function(){s.body.emitter.emit("rangechanged",w)},200),o)return o()}},n.prototype.getMillisecondsPerPixel=function(){return void 0===this.millisecondsPerPixelCache&&(this.millisecondsPerPixelCache=(this.end-this.start)/this.body.dom.center.clientWidth),this.millisecondsPerPixelCache},n.prototype._cancelAnimation=function(){this.animationTimer&&(clearTimeout(this.animationTimer),this.animationTimer=null)},n.prototype._applyRange=function(t,e){var i,o=null!=t?p.convert(t,"Date").valueOf():this.start,n=null!=e?p.convert(e,"Date").valueOf():this.end,s=null!=this.options.max?p.convert(this.options.max,"Date").valueOf():null,r=null!=this.options.min?p.convert(this.options.min,"Date").valueOf():null;if(isNaN(o)||null===o)throw new Error('Invalid start "'+t+'"');if(isNaN(n)||null===n)throw new Error('Invalid end "'+e+'"');if(n<o&&(n=o),null!==r&&o<r&&(i=r-o,o+=i,n+=i,null!=s&&n>s&&(n=s)),null!==s&&n>s&&(i=n-s,o-=i,n-=i,null!=r&&o<r&&(o=r)),null!==this.options.zoomMin){var a=parseFloat(this.options.zoomMin);if(a<0&&(a=0),n-o<a){this.end-this.start===a&&o>=this.start-.5&&n<=this.end?(o=this.start,n=this.end):(i=a-(n-o),o-=i/2,n+=i/2)}}if(null!==this.options.zoomMax){var h=parseFloat(this.options.zoomMax);h<0&&(h=0),n-o>h&&(this.end-this.start===h&&o<this.start&&n>this.end?(o=this.start,n=this.end):(i=n-o-h,o+=i/2,n-=i/2))}var d=this.start!=o||this.end!=n;return o>=this.start&&o<=this.end||n>=this.start&&n<=this.end||this.start>=o&&this.start<=n||this.end>=o&&this.end<=n||this.body.emitter.emit("checkRangedItems"),this.start=o,this.end=n,d},n.prototype.getRange=function(){return{start:this.start,end:this.end}},n.prototype.conversion=function(t,e){return n.conversion(this.start,this.end,t,e)},n.conversion=function(t,e,i,o){return void 0===o&&(o=0),0!=i&&e-t!=0?{offset:t,scale:i/(e-t-o)}:{offset:0,scale:1}},n.prototype._onDragStart=function(t){this.deltaDifference=0,this.previousDelta=0,this.options.moveable&&this._isInsideRange(t)&&this.props.touch.allowDragging&&(this.stopRolling(),this.props.touch.start=this.start,this.props.touch.end=this.end,this.props.touch.dragging=!0,this.body.dom.root&&(this.body.dom.root.style.cursor="move"))},n.prototype._onDrag=function(t){if(t&&this.props.touch.dragging&&this.options.moveable&&this.props.touch.allowDragging){var e=this.options.direction;s(e);var i="horizontal"==e?t.deltaX:t.deltaY;i-=this.deltaDifference;var o=this.props.touch.end-this.props.touch.start;o-=f.getHiddenDurationBetween(this.body.hiddenDates,this.start,this.end);var n,r="horizontal"==e?this.body.domProps.center.width:this.body.domProps.center.height;n=this.options.rtl?i/r*o:-i/r*o;var a=this.props.touch.start+n,h=this.props.touch.end+n,d=f.snapAwayFromHidden(this.body.hiddenDates,a,this.previousDelta-i,!0),l=f.snapAwayFromHidden(this.body.hiddenDates,h,this.previousDelta-i,!0);if(d!=a||l!=h)return this.deltaDifference+=i,this.props.touch.start=d,this.props.touch.end=l,void this._onDrag(t);this.previousDelta=i,this._applyRange(a,h);var u=new Date(this.start),p=new Date(this.end);this.body.emitter.emit("rangechange",{start:u,end:p,byUser:!0,event:t}),this.body.emitter.emit("panmove")}},n.prototype._onDragEnd=function(t){this.props.touch.dragging&&this.options.moveable&&this.props.touch.allowDragging&&(this.props.touch.dragging=!1,this.body.dom.root&&(this.body.dom.root.style.cursor="auto"),this.body.emitter.emit("rangechanged",{start:new Date(this.start),end:new Date(this.end),byUser:!0,event:t}))},n.prototype._onMouseWheel=function(t){var e=0;if(t.wheelDelta?e=t.wheelDelta/120:t.detail&&(e=-t.detail/3),!(this.options.zoomKey&&!t[this.options.zoomKey]&&this.options.zoomable||!this.options.zoomable&&this.options.moveable)&&this.options.zoomable&&this.options.moveable&&this._isInsideRange(t)&&e){var i;i=e<0?1-e/5:1/(1+e/5);var o;if(this.rolling)o=this.start+(this.end-this.start)*this.options.rollingMode.offset;else{var n=this.getPointer({x:t.clientX,y:t.clientY},this.body.dom.center);o=this._pointerToDate(n)}this.zoom(i,o,e,t),t.preventDefault()}},n.prototype._onTouch=function(t){this.props.touch.start=this.start,this.props.touch.end=this.end,this.props.touch.allowDragging=!0,this.props.touch.center=null,this.scaleOffset=0,this.deltaDifference=0,p.preventDefault(t)},n.prototype._onPinch=function(t){if(this.options.zoomable&&this.options.moveable){p.preventDefault(t),this.props.touch.allowDragging=!1,this.props.touch.center||(this.props.touch.center=this.getPointer(t.center,this.body.dom.center)),this.stopRolling();var e=1/(t.scale+this.scaleOffset),i=this._pointerToDate(this.props.touch.center),o=f.getHiddenDurationBetween(this.body.hiddenDates,this.start,this.end),n=f.getHiddenDurationBefore(this.options.moment,this.body.hiddenDates,this,i),s=o-n,r=i-n+(this.props.touch.start-(i-n))*e,a=i+s+(this.props.touch.end-(i+s))*e;this.startToFront=1-e<=0,this.endToFront=e-1<=0;var h=f.snapAwayFromHidden(this.body.hiddenDates,r,1-e,!0),d=f.snapAwayFromHidden(this.body.hiddenDates,a,e-1,!0);h==r&&d==a||(this.props.touch.start=h,this.props.touch.end=d,this.scaleOffset=1-t.scale,r=h,a=d);var l={animation:!1,byUser:!0,event:t};this.setRange(r,a,l),this.startToFront=!1,this.endToFront=!0}},n.prototype._isInsideRange=function(t){var e,i=t.center?t.center.x:t.clientX;e=this.options.rtl?i-p.getAbsoluteLeft(this.body.dom.centerContainer):p.getAbsoluteRight(this.body.dom.centerContainer)-i;var o=this.body.util.toTime(e);return o>=this.start&&o<=this.end},n.prototype._pointerToDate=function(t){var e,i=this.options.direction;if(s(i),"horizontal"==i)return this.body.util.toTime(t.x).valueOf();var o=this.body.domProps.center.height;return e=this.conversion(o),t.y/e.scale+e.offset},n.prototype.getPointer=function(t,e){return this.options.rtl?{x:p.getAbsoluteRight(e)-t.x,y:t.y-p.getAbsoluteTop(e)}:{x:t.x-p.getAbsoluteLeft(e),y:t.y-p.getAbsoluteTop(e)}},n.prototype.zoom=function(t,e,i,o){null==e&&(e=(this.start+this.end)/2);var n=f.getHiddenDurationBetween(this.body.hiddenDates,this.start,this.end),s=f.getHiddenDurationBefore(this.options.moment,this.body.hiddenDates,this,e),r=n-s,a=e-s+(this.start-(e-s))*t,h=e+r+(this.end-(e+r))*t;this.startToFront=!(i>0),this.endToFront=!(-i>0);var d=f.snapAwayFromHidden(this.body.hiddenDates,a,i,!0),l=f.snapAwayFromHidden(this.body.hiddenDates,h,-i,!0);d==a&&l==h||(a=d,h=l);var u={animation:!1,byUser:!0,event:o};this.setRange(a,h,u),this.startToFront=!1,this.endToFront=!0},n.prototype.move=function(t){var e=this.end-this.start,i=this.start+e*t,o=this.end+e*t;this.start=i,this.end=o},n.prototype.moveTo=function(t){var e=(this.start+this.end)/2,i=e-t,o=this.start-i,n=this.end-i,s={animation:!1,byUser:!0,event:null};this.setRange(o,n,s)},t.exports=n},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}function n(){}var s=i(16),r=o(s),a=i(1),h=o(a),d=i(66),l=i(12),u=i(67),p=i(0),c=i(28),m=i(132),f=i(21),g=i(29);d(n.prototype),n.prototype._create=function(t){function e(t){this.isActive()&&this.emit("mousewheel",t);var e=0,i=0;if("detail"in t&&(i=-1*t.detail),"wheelDelta"in t&&(i=t.wheelDelta),"wheelDeltaY"in t&&(i=t.wheelDeltaY),"wheelDeltaX"in t&&(e=-1*t.wheelDeltaX),"axis"in t&&t.axis===t.HORIZONTAL_AXIS&&(e=-1*i,i=0),"deltaY"in t&&(i=-1*t.deltaY),"deltaX"in t&&(e=t.deltaX),this.options.zoomKey&&!t[this.options.zoomKey])if(t.preventDefault(),this.options.verticalScroll&&Math.abs(i)>=Math.abs(e)){var o=this.props.scrollTop,n=o+i;this.isActive()&&(this._setScrollTop(n),this._redraw(),this.emit("scroll",t))}else if(this.options.horizontalScroll){var s=Math.abs(e)>=Math.abs(i)?e:i,r=s/120*(this.range.end-this.range.start)/20,a=this.range.start+r,h=this.range.end+r,d={animation:!1,byUser:!0,event:t};this.range.setRange(a,h,d)}}function i(t){if(s.options.verticalScroll&&(t.preventDefault(),s.isActive())){var e=-t.target.scrollTop;s._setScrollTop(e),s._redraw(),s.emit("scrollSide",t)}}function o(t){if(t.preventDefault&&t.preventDefault(),!(!t.target.className.indexOf("vis")>-1||a))return t.dataTransfer.dropEffect="move",a=!0,!1}function n(t){t.preventDefault&&t.preventDefault(),t.stopPropagation&&t.stopPropagation();try{var e=JSON.parse(t.dataTransfer.getData("text"));if(!e||!e.content)return}catch(t){return!1}return a=!1,t.center={x:t.clientX,y:t.clientY},"item"!==e.target?s.itemSet._onAddItem(t):s.itemSet._onDropObjectOnItem(t),s.emit("drop",s.getEventProperties(t)),!1}this.dom={},this.dom.container=t,this.dom.root=document.createElement("div"),this.dom.background=document.createElement("div"),this.dom.backgroundVertical=document.createElement("div"),this.dom.backgroundHorizontal=document.createElement("div"),this.dom.centerContainer=document.createElement("div"),this.dom.leftContainer=document.createElement("div"),this.dom.rightContainer=document.createElement("div"),this.dom.center=document.createElement("div"),this.dom.left=document.createElement("div"),this.dom.right=document.createElement("div"),this.dom.top=document.createElement("div"),this.dom.bottom=document.createElement("div"),this.dom.shadowTop=document.createElement("div"),this.dom.shadowBottom=document.createElement("div"),this.dom.shadowTopLeft=document.createElement("div"),this.dom.shadowBottomLeft=document.createElement("div"),this.dom.shadowTopRight=document.createElement("div"),this.dom.shadowBottomRight=document.createElement("div"),this.dom.rollingModeBtn=document.createElement("div"),this.dom.root.className="vis-timeline",this.dom.background.className="vis-panel vis-background",this.dom.backgroundVertical.className="vis-panel vis-background vis-vertical",this.dom.backgroundHorizontal.className="vis-panel vis-background vis-horizontal",this.dom.centerContainer.className="vis-panel vis-center",this.dom.leftContainer.className="vis-panel vis-left",this.dom.rightContainer.className="vis-panel vis-right",this.dom.top.className="vis-panel vis-top",this.dom.bottom.className="vis-panel vis-bottom",this.dom.left.className="vis-content",this.dom.center.className="vis-content",this.dom.right.className="vis-content",this.dom.shadowTop.className="vis-shadow vis-top",this.dom.shadowBottom.className="vis-shadow vis-bottom",this.dom.shadowTopLeft.className="vis-shadow vis-top",this.dom.shadowBottomLeft.className="vis-shadow vis-bottom",this.dom.shadowTopRight.className="vis-shadow vis-top",this.dom.shadowBottomRight.className="vis-shadow vis-bottom",this.dom.rollingModeBtn.className="vis-rolling-mode-btn",this.dom.root.appendChild(this.dom.background),this.dom.root.appendChild(this.dom.backgroundVertical),this.dom.root.appendChild(this.dom.backgroundHorizontal),this.dom.root.appendChild(this.dom.centerContainer),this.dom.root.appendChild(this.dom.leftContainer),this.dom.root.appendChild(this.dom.rightContainer),this.dom.root.appendChild(this.dom.top),this.dom.root.appendChild(this.dom.bottom),this.dom.root.appendChild(this.dom.bottom),this.dom.root.appendChild(this.dom.rollingModeBtn),this.dom.centerContainer.appendChild(this.dom.center),this.dom.leftContainer.appendChild(this.dom.left),this.dom.rightContainer.appendChild(this.dom.right),this.dom.centerContainer.appendChild(this.dom.shadowTop),this.dom.centerContainer.appendChild(this.dom.shadowBottom),this.dom.leftContainer.appendChild(this.dom.shadowTopLeft),this.dom.leftContainer.appendChild(this.dom.shadowBottomLeft),this.dom.rightContainer.appendChild(this.dom.shadowTopRight),this.dom.rightContainer.appendChild(this.dom.shadowBottomRight),this.props={root:{},background:{},centerContainer:{},leftContainer:{},rightContainer:{},center:{},left:{},right:{},top:{},bottom:{},border:{},scrollTop:0,scrollTopMin:0},this.on("rangechange",function(){!0===this.initialDrawDone&&this._redraw()}.bind(this)),this.on("rangechanged",function(){this.initialRangeChangeDone||(this.initialRangeChangeDone=!0)}.bind(this)),this.on("touch",this._onTouch.bind(this)),this.on("panmove",this._onDrag.bind(this));var s=this;this._origRedraw=this._redraw.bind(this),this._redraw=p.throttle(this._origRedraw),this.on("_change",function(t){s.itemSet&&s.itemSet.initialItemSetDrawn&&t&&1==t.queue?s._redraw():s._origRedraw()}),this.hammer=new l(this.dom.root);var r=this.hammer.get("pinch").set({enable:!0});u.disablePreventDefaultVertically(r),this.hammer.get("pan").set({threshold:5,direction:l.DIRECTION_HORIZONTAL}),this.listeners={},["tap","doubletap","press","pinch","pan","panstart","panmove","panend"].forEach(function(t){var e=function(e){s.isActive()&&s.emit(t,e)};s.hammer.on(t,e),s.listeners[t]=e}),u.onTouch(this.hammer,function(t){s.emit("touch",t)}.bind(this)),u.onRelease(this.hammer,function(t){s.emit("release",t)}.bind(this)),this.dom.centerContainer.addEventListener?(this.dom.centerContainer.addEventListener("mousewheel",e.bind(this),!1),this.dom.centerContainer.addEventListener("DOMMouseScroll",e.bind(this),!1)):this.dom.centerContainer.attachEvent("onmousewheel",e.bind(this)),this.dom.left.parentNode.addEventListener("scroll",i.bind(this)),this.dom.right.parentNode.addEventListener("scroll",i.bind(this));var a=!1;if(this.dom.center.addEventListener("dragover",o.bind(this),!1),this.dom.center.addEventListener("drop",n.bind(this),!1),this.customTimes=[],this.touch={},this.redrawCount=0,this.initialDrawDone=!1,this.initialRangeChangeDone=!1,!t)throw new Error("No container provided");t.appendChild(this.dom.root)},n.prototype.setOptions=function(t){if(t){var e=["width","height","minHeight","maxHeight","autoResize","start","end","clickToUse","dataAttributes","hiddenDates","locale","locales","moment","rtl","zoomKey","horizontalScroll","verticalScroll"];if(p.selectiveExtend(e,this.options,t),this.dom.rollingModeBtn.style.visibility="hidden",this.options.rtl&&(this.dom.container.style.direction="rtl",this.dom.backgroundVertical.className="vis-panel vis-background vis-vertical-rtl"),this.options.verticalScroll&&(this.options.rtl?this.dom.rightContainer.className="vis-panel vis-right vis-vertical-scroll":this.dom.leftContainer.className="vis-panel vis-left vis-vertical-scroll"),"object"!==(0,h.default)(this.options.orientation)&&(this.options.orientation={item:void 0,axis:void 0}),"orientation"in t&&("string"==typeof t.orientation?this.options.orientation={item:t.orientation,axis:t.orientation}:"object"===(0,h.default)(t.orientation)&&("item"in t.orientation&&(this.options.orientation.item=t.orientation.item),"axis"in t.orientation&&(this.options.orientation.axis=t.orientation.axis))),"both"===this.options.orientation.axis){if(!this.timeAxis2){var i=this.timeAxis2=new c(this.body);i.setOptions=function(t){var e=t?p.extend({},t):{};e.orientation="top",c.prototype.setOptions.call(i,e)},this.components.push(i)}}else if(this.timeAxis2){var o=this.components.indexOf(this.timeAxis2);-1!==o&&this.components.splice(o,1),this.timeAxis2.destroy(),this.timeAxis2=null}if("function"==typeof t.drawPoints&&(t.drawPoints={onRender:t.drawPoints}),"hiddenDates"in this.options&&f.convertHiddenOptions(this.options.moment,this.body,this.options.hiddenDates),"clickToUse"in t&&(t.clickToUse?this.activator||(this.activator=new m(this.dom.root)):this.activator&&(this.activator.destroy(),delete this.activator)),"showCustomTime"in t)throw new Error("Option `showCustomTime` is deprecated. Create a custom time bar via timeline.addCustomTime(time [, id])");this._initAutoResize()}if(this.components.forEach(function(e){return e.setOptions(t)}),"configure"in t){this.configurator||(this.configurator=this._createConfigurator()),this.configurator.setOptions(t.configure);var n=p.deepExtend({},this.options);this.components.forEach(function(t){p.deepExtend(n,t.options)}),this.configurator.setModuleOptions({global:n})}this._redraw()},n.prototype.isActive=function(){return!this.activator||this.activator.active},n.prototype.destroy=function(){this.setItems(null),this.setGroups(null),this.off(),this._stopAutoResize(),this.dom.root.parentNode&&this.dom.root.parentNode.removeChild(this.dom.root),this.dom=null,this.activator&&(this.activator.destroy(),delete this.activator);for(var t in this.listeners)this.listeners.hasOwnProperty(t)&&delete this.listeners[t];this.listeners=null,this.hammer=null,this.components.forEach(function(t){return t.destroy()}),this.body=null},n.prototype.setCustomTime=function(t,e){var i=this.customTimes.filter(function(t){return e===t.options.id});if(0===i.length)throw new Error("No custom time bar found with id "+(0,r.default)(e));i.length>0&&i[0].setCustomTime(t)},n.prototype.getCustomTime=function(t){var e=this.customTimes.filter(function(e){return e.options.id===t});if(0===e.length)throw new Error("No custom time bar found with id "+(0,r.default)(t));return e[0].getCustomTime()},n.prototype.setCustomTimeTitle=function(t,e){var i=this.customTimes.filter(function(t){return t.options.id===e});if(0===i.length)throw new Error("No custom time bar found with id "+(0,r.default)(e));if(i.length>0)return i[0].setCustomTitle(t)},n.prototype.getEventProperties=function(t){return{event:t}},n.prototype.addCustomTime=function(t,e){var i=void 0!==t?p.convert(t,"Date").valueOf():new Date;if(this.customTimes.some(function(t){return t.options.id===e}))throw new Error("A custom time with id "+(0,r.default)(e)+" already exists");var o=new g(this.body,p.extend({},this.options,{time:i,id:e}));return this.customTimes.push(o),this.components.push(o),this._redraw(),e},n.prototype.removeCustomTime=function(t){var e=this.customTimes.filter(function(e){return e.options.id===t});if(0===e.length)throw new Error("No custom time bar found with id "+(0,r.default)(t));e.forEach(function(t){this.customTimes.splice(this.customTimes.indexOf(t),1),this.components.splice(this.components.indexOf(t),1),t.destroy()}.bind(this))},n.prototype.getVisibleItems=function(){return this.itemSet&&this.itemSet.getVisibleItems()||[]},n.prototype.fit=function(t,e){var i=this.getDataRange();if(null!==i.min||null!==i.max){var o=i.max-i.min,n=new Date(i.min.valueOf()-.01*o),s=new Date(i.max.valueOf()+.01*o),r=!t||void 0===t.animation||t.animation;this.range.setRange(n,s,{animation:r},e)}},n.prototype.getDataRange=function(){throw new Error("Cannot invoke abstract method getDataRange")},n.prototype.setWindow=function(t,e,i,o){"function"==typeof arguments[2]&&(o=arguments[2],i={});var n,s;1==arguments.length?(s=arguments[0],n=void 0===s.animation||s.animation,this.range.setRange(s.start,s.end,{animation:n})):2==arguments.length&&"function"==typeof arguments[1]?(s=arguments[0],o=arguments[1],n=void 0===s.animation||s.animation,this.range.setRange(s.start,s.end,{animation:n},o)):(n=!i||void 0===i.animation||i.animation,this.range.setRange(t,e,{animation:n},o))},n.prototype.moveTo=function(t,e,i){"function"==typeof arguments[1]&&(i=arguments[1],e={});var o=this.range.end-this.range.start,n=p.convert(t,"Date").valueOf(),s=n-o/2,r=n+o/2,a=!e||void 0===e.animation||e.animation;this.range.setRange(s,r,{animation:a},i)},n.prototype.getWindow=function(){var t=this.range.getRange();return{start:new Date(t.start),end:new Date(t.end)}},n.prototype.zoomIn=function(t,e,i){if(!(!t||t<0||t>1)){"function"==typeof arguments[1]&&(i=arguments[1],e={});var o=this.getWindow(),n=o.start.valueOf(),s=o.end.valueOf(),r=s-n,a=r/(1+t),h=(r-a)/2,d=n+h,l=s-h;this.setWindow(d,l,e,i)}},n.prototype.zoomOut=function(t,e,i){if(!(!t||t<0||t>1)){"function"==typeof arguments[1]&&(i=arguments[1],e={});var o=this.getWindow(),n=o.start.valueOf(),s=o.end.valueOf(),r=s-n,a=n-r*t/2,h=s+r*t/2;this.setWindow(a,h,e,i)}},n.prototype.redraw=function(){this._redraw()},n.prototype._redraw=function(){this.redrawCount++;var t=!1,e=this.options,i=this.props,o=this.dom;if(o&&o.container&&0!=o.root.offsetWidth){f.updateHiddenDates(this.options.moment,this.body,this.options.hiddenDates),"top"==e.orientation?(p.addClassName(o.root,"vis-top"),p.removeClassName(o.root,"vis-bottom")):(p.removeClassName(o.root,"vis-top"),p.addClassName(o.root,"vis-bottom")),o.root.style.maxHeight=p.option.asSize(e.maxHeight,""),o.root.style.minHeight=p.option.asSize(e.minHeight,""),o.root.style.width=p.option.asSize(e.width,""),i.border.left=(o.centerContainer.offsetWidth-o.centerContainer.clientWidth)/2,i.border.right=i.border.left,i.border.top=(o.centerContainer.offsetHeight-o.centerContainer.clientHeight)/2,i.border.bottom=i.border.top,i.borderRootHeight=o.root.offsetHeight-o.root.clientHeight,i.borderRootWidth=o.root.offsetWidth-o.root.clientWidth,0===o.centerContainer.clientHeight&&(i.border.left=i.border.top,i.border.right=i.border.left),0===o.root.clientHeight&&(i.borderRootWidth=i.borderRootHeight),i.center.height=o.center.offsetHeight,i.left.height=o.left.offsetHeight,i.right.height=o.right.offsetHeight,i.top.height=o.top.clientHeight||-i.border.top,i.bottom.height=o.bottom.clientHeight||-i.border.bottom;var n=Math.max(i.left.height,i.center.height,i.right.height),s=i.top.height+n+i.bottom.height+i.borderRootHeight+i.border.top+i.border.bottom;o.root.style.height=p.option.asSize(e.height,s+"px"),i.root.height=o.root.offsetHeight,i.background.height=i.root.height-i.borderRootHeight;var r=i.root.height-i.top.height-i.bottom.height-i.borderRootHeight;i.centerContainer.height=r,i.leftContainer.height=r,i.rightContainer.height=i.leftContainer.height,i.root.width=o.root.offsetWidth,i.background.width=i.root.width-i.borderRootWidth,this.initialDrawDone||(i.scrollbarWidth=p.getScrollBarWidth()),e.verticalScroll?e.rtl?(i.left.width=o.leftContainer.clientWidth||-i.border.left,i.right.width=o.rightContainer.clientWidth+i.scrollbarWidth||-i.border.right):(i.left.width=o.leftContainer.clientWidth+i.scrollbarWidth||-i.border.left,i.right.width=o.rightContainer.clientWidth||-i.border.right):(i.left.width=o.leftContainer.clientWidth||-i.border.left,i.right.width=o.rightContainer.clientWidth||-i.border.right),this._setDOM();var a=this._updateScrollTop();"top"!=e.orientation.item&&(a+=Math.max(i.centerContainer.height-i.center.height-i.border.top-i.border.bottom,0)),o.center.style.top=a+"px";var h=0==i.scrollTop?"hidden":"",d=i.scrollTop==i.scrollTopMin?"hidden":"";o.shadowTop.style.visibility=h,o.shadowBottom.style.visibility=d,o.shadowTopLeft.style.visibility=h,o.shadowBottomLeft.style.visibility=d,o.shadowTopRight.style.visibility=h,o.shadowBottomRight.style.visibility=d,e.verticalScroll&&(o.rightContainer.className="vis-panel vis-right vis-vertical-scroll",o.leftContainer.className="vis-panel vis-left vis-vertical-scroll",o.shadowTopRight.style.visibility="hidden",o.shadowBottomRight.style.visibility="hidden",o.shadowTopLeft.style.visibility="hidden",o.shadowBottomLeft.style.visibility="hidden",o.left.style.top="0px",o.right.style.top="0px"),(!e.verticalScroll||i.center.height<i.centerContainer.height)&&(o.left.style.top=a+"px",o.right.style.top=a+"px",o.rightContainer.className=o.rightContainer.className.replace(new RegExp("(?:^|\\s)vis-vertical-scroll(?:\\s|$)")," "),o.leftContainer.className=o.leftContainer.className.replace(new RegExp("(?:^|\\s)vis-vertical-scroll(?:\\s|$)")," "),i.left.width=o.leftContainer.clientWidth||-i.border.left,i.right.width=o.rightContainer.clientWidth||-i.border.right,this._setDOM());var u=i.center.height>i.centerContainer.height;this.hammer.get("pan").set({direction:u?l.DIRECTION_ALL:l.DIRECTION_HORIZONTAL}),this.components.forEach(function(e){t=e.redraw()||t});if(t){if(this.redrawCount<5)return void this.body.emitter.emit("_change");console.log("WARNING: infinite loop in redraw?")}else this.redrawCount=0;this.body.emitter.emit("changed")}},n.prototype._setDOM=function(){var t=this.props,e=this.dom;t.leftContainer.width=t.left.width,t.rightContainer.width=t.right.width;var i=t.root.width-t.left.width-t.right.width-t.borderRootWidth;t.center.width=i,t.centerContainer.width=i,t.top.width=i,t.bottom.width=i,e.background.style.height=t.background.height+"px",e.backgroundVertical.style.height=t.background.height+"px",e.backgroundHorizontal.style.height=t.centerContainer.height+"px",e.centerContainer.style.height=t.centerContainer.height+"px",e.leftContainer.style.height=t.leftContainer.height+"px",e.rightContainer.style.height=t.rightContainer.height+"px",e.background.style.width=t.background.width+"px",e.backgroundVertical.style.width=t.centerContainer.width+"px",e.backgroundHorizontal.style.width=t.background.width+"px",e.centerContainer.style.width=t.center.width+"px",e.top.style.width=t.top.width+"px",e.bottom.style.width=t.bottom.width+"px",e.background.style.left="0",e.background.style.top="0",e.backgroundVertical.style.left=t.left.width+t.border.left+"px",e.backgroundVertical.style.top="0",e.backgroundHorizontal.style.left="0",e.backgroundHorizontal.style.top=t.top.height+"px",e.centerContainer.style.left=t.left.width+"px",e.centerContainer.style.top=t.top.height+"px",e.leftContainer.style.left="0",e.leftContainer.style.top=t.top.height+"px",e.rightContainer.style.left=t.left.width+t.center.width+"px",e.rightContainer.style.top=t.top.height+"px",e.top.style.left=t.left.width+"px",e.top.style.top="0",e.bottom.style.left=t.left.width+"px",e.bottom.style.top=t.top.height+t.centerContainer.height+"px",e.center.style.left="0",e.left.style.left="0",e.right.style.left="0"},n.prototype.repaint=function(){throw new Error("Function repaint is deprecated. Use redraw instead.")},n.prototype.setCurrentTime=function(t){if(!this.currentTime)throw new Error("Option showCurrentTime must be true");this.currentTime.setCurrentTime(t)},n.prototype.getCurrentTime=function(){if(!this.currentTime)throw new Error("Option showCurrentTime must be true");return this.currentTime.getCurrentTime()},n.prototype._toTime=function(t){return f.toTime(this,t,this.props.center.width)},n.prototype._toGlobalTime=function(t){return f.toTime(this,t,this.props.root.width)},n.prototype._toScreen=function(t){return f.toScreen(this,t,this.props.center.width)},n.prototype._toGlobalScreen=function(t){return f.toScreen(this,t,this.props.root.width)},n.prototype._initAutoResize=function(){1==this.options.autoResize?this._startAutoResize():this._stopAutoResize()},n.prototype._startAutoResize=function(){var t=this;this._stopAutoResize(),this._onResize=function(){if(1!=t.options.autoResize)return void t._stopAutoResize();t.dom.root&&(t.dom.root.offsetWidth==t.props.lastWidth&&t.dom.root.offsetHeight==t.props.lastHeight||(t.props.lastWidth=t.dom.root.offsetWidth,t.props.lastHeight=t.dom.root.offsetHeight,t.props.scrollbarWidth=p.getScrollBarWidth(),t.body.emitter.emit("_change")))},p.addEventListener(window,"resize",this._onResize),t.dom.root&&(t.props.lastWidth=t.dom.root.offsetWidth,t.props.lastHeight=t.dom.root.offsetHeight),this.watchTimer=setInterval(this._onResize,1e3)},n.prototype._stopAutoResize=function(){this.watchTimer&&(clearInterval(this.watchTimer),this.watchTimer=void 0),this._onResize&&(p.removeEventListener(window,"resize",this._onResize),this._onResize=null)},n.prototype._onTouch=function(t){this.touch.allowDragging=!0,this.touch.initialScrollTop=this.props.scrollTop},n.prototype._onPinch=function(t){this.touch.allowDragging=!1},n.prototype._onDrag=function(t){if(t&&this.touch.allowDragging){var e=t.deltaY,i=this._getScrollTop(),o=this._setScrollTop(this.touch.initialScrollTop+e);this.options.verticalScroll&&(this.dom.left.parentNode.scrollTop=-this.props.scrollTop,this.dom.right.parentNode.scrollTop=-this.props.scrollTop),o!=i&&this.emit("verticalDrag")}},n.prototype._setScrollTop=function(t){return this.props.scrollTop=t,this._updateScrollTop(),this.props.scrollTop},n.prototype._updateScrollTop=function(){var t=Math.min(this.props.centerContainer.height-this.props.center.height,0);return t!=this.props.scrollTopMin&&("top"!=this.options.orientation.item&&(this.props.scrollTop+=t-this.props.scrollTopMin),this.props.scrollTopMin=t),this.props.scrollTop>0&&(this.props.scrollTop=0),this.props.scrollTop<t&&(this.props.scrollTop=t),this.options.verticalScroll&&(this.dom.left.parentNode.scrollTop=-this.props.scrollTop,this.dom.right.parentNode.scrollTop=-this.props.scrollTop),this.props.scrollTop},n.prototype._getScrollTop=function(){return this.props.scrollTop},n.prototype._createConfigurator=function(){throw new Error("Cannot invoke abstract method _createConfigurator")},t.exports=n},function(t,e,i){function o(t,e,i,s,r){this.moment=n,this.current=this.moment(),this._start=this.moment(),this._end=this.moment(),this.autoScale=!0,this.scale="day",this.step=1,this.setRange(t,e,i),this.switchedDay=!1,this.switchedMonth=!1,this.switchedYear=!1,Array.isArray(s)?this.hiddenDates=s:this.hiddenDates=void 0!=s?[s]:[],this.format=o.FORMAT,this.options=r||{}}var n=i(3),s=i(21),r=i(0);o.FORMAT={minorLabels:{millisecond:"SSS",second:"s",minute:"HH:mm",hour:"HH:mm",weekday:"ddd D",day:"D",week:"w",month:"MMM",year:"YYYY"},majorLabels:{millisecond:"HH:mm:ss",second:"D MMMM HH:mm",minute:"ddd D MMMM",hour:"ddd D MMMM",weekday:"MMMM YYYY",day:"MMMM YYYY",week:"MMMM YYYY",month:"YYYY",year:""}},o.prototype.setMoment=function(t){this.moment=t,this.current=this.moment(this.current.valueOf()),this._start=this.moment(this._start.valueOf()),this._end=this.moment(this._end.valueOf())},o.prototype.setFormat=function(t){var e=r.deepExtend({},o.FORMAT);this.format=r.deepExtend(e,t)},o.prototype.setRange=function(t,e,i){if(!(t instanceof Date&&e instanceof Date))throw"No legal start or end date in method setRange";this._start=void 0!=t?this.moment(t.valueOf()):new Date,this._end=void 0!=e?this.moment(e.valueOf()):new Date,this.autoScale&&this.setMinimumStep(i)},o.prototype.start=function(){this.current=this._start.clone(),this.roundToMinor()},o.prototype.roundToMinor=function(){switch("week"==this.scale&&this.current.weekday(0),this.scale){case"year":this.current.year(this.step*Math.floor(this.current.year()/this.step)),this.current.month(0);case"month":this.current.date(1);case"week":case"day":case"weekday":this.current.hours(0);case"hour":this.current.minutes(0);case"minute":this.current.seconds(0);case"second":this.current.milliseconds(0)}if(1!=this.step)switch(this.scale){case"millisecond":this.current.subtract(this.current.milliseconds()%this.step,"milliseconds");break;case"second":this.current.subtract(this.current.seconds()%this.step,"seconds");break;case"minute":this.current.subtract(this.current.minutes()%this.step,"minutes");break;case"hour":this.current.subtract(this.current.hours()%this.step,"hours");break;case"weekday":case"day":this.current.subtract((this.current.date()-1)%this.step,"day");break;case"week":this.current.subtract(this.current.week()%this.step,"week");break;case"month":this.current.subtract(this.current.month()%this.step,"month");break;case"year":this.current.subtract(this.current.year()%this.step,"year")}},o.prototype.hasNext=function(){return this.current.valueOf()<=this._end.valueOf()},o.prototype.next=function(){var t=this.current.valueOf();switch(this.scale){case"millisecond":this.current.add(this.step,"millisecond");break;case"second":this.current.add(this.step,"second");break;case"minute":this.current.add(this.step,"minute");break;case"hour":this.current.add(this.step,"hour"),this.current.month()<6?this.current.subtract(this.current.hours()%this.step,"hour"):this.current.hours()%this.step!=0&&this.current.add(this.step-this.current.hours()%this.step,"hour");break;case"weekday":case"day":this.current.add(this.step,"day");break;case"week":if(0!==this.current.weekday())this.current.weekday(0),this.current.add(this.step,"week");else if(!1===this.options.showMajorLabels)this.current.add(this.step,"week");else{var e=this.current.clone();e.add(1,"week"),e.isSame(this.current,"month")?this.current.add(this.step,"week"):(this.current.add(this.step,"week"),this.current.date(1))}break;case"month":this.current.add(this.step,"month");break;case"year":this.current.add(this.step,"year")}if(1!=this.step)switch(this.scale){case"millisecond":this.current.milliseconds()>0&&this.current.milliseconds()<this.step&&this.current.milliseconds(0);break;case"second":this.current.seconds()>0&&this.current.seconds()<this.step&&this.current.seconds(0);break;case"minute":this.current.minutes()>0&&this.current.minutes()<this.step&&this.current.minutes(0);break;case"hour":this.current.hours()>0&&this.current.hours()<this.step&&this.current.hours(0);break;case"weekday":case"day":this.current.date()<this.step+1&&this.current.date(1);break;case"week":this.current.week()<this.step&&this.current.week(1);break;case"month":this.current.month()<this.step&&this.current.month(0)}this.current.valueOf()==t&&(this.current=this._end.clone()),this.switchedDay=!1,this.switchedMonth=!1,this.switchedYear=!1,s.stepOverHiddenDates(this.moment,this,t)},o.prototype.getCurrent=function(){return this.current},o.prototype.setScale=function(t){t&&"string"==typeof t.scale&&(this.scale=t.scale,this.step=t.step>0?t.step:1,this.autoScale=!1)},o.prototype.setAutoScale=function(t){this.autoScale=t},o.prototype.setMinimumStep=function(t){if(void 0!=t){31104e9>t&&(this.scale="year",this.step=1e3),15552e9>t&&(this.scale="year",this.step=500),31104e8>t&&(this.scale="year",this.step=100),15552e8>t&&(this.scale="year",this.step=50),31104e7>t&&(this.scale="year",this.step=10),15552e7>t&&(this.scale="year",this.step=5),31104e6>t&&(this.scale="year",this.step=1),7776e6>t&&(this.scale="month",this.step=3),2592e6>t&&(this.scale="month",this.step=1),432e6>t&&(this.scale="day",this.step=5),1728e5>t&&(this.scale="day",this.step=2),864e5>t&&(this.scale="day",this.step=1),432e5>t&&(this.scale="weekday",this.step=1),144e5>t&&(this.scale="hour",this.step=4),36e5>t&&(this.scale="hour",this.step=1),9e5>t&&(this.scale="minute",this.step=15),6e5>t&&(this.scale="minute",this.step=10),3e5>t&&(this.scale="minute",this.step=5),6e4>t&&(this.scale="minute",this.step=1),15e3>t&&(this.scale="second",this.step=15),1e4>t&&(this.scale="second",this.step=10),5e3>t&&(this.scale="second",this.step=5),1e3>t&&(this.scale="second",this.step=1),200>t&&(this.scale="millisecond",this.step=200),100>t&&(this.scale="millisecond",this.step=100),50>t&&(this.scale="millisecond",this.step=50),10>t&&(this.scale="millisecond",this.step=10),5>t&&(this.scale="millisecond",this.step=5),1>t&&(this.scale="millisecond",this.step=1)}},o.snap=function(t,e,i){var o=n(t);if("year"==e){var s=o.year()+Math.round(o.month()/12);o.year(Math.round(s/i)*i),o.month(0),o.date(0),o.hours(0),o.minutes(0),o.seconds(0),o.milliseconds(0)}else if("month"==e)o.date()>15?(o.date(1),o.add(1,"month")):o.date(1),o.hours(0),o.minutes(0),o.seconds(0),o.milliseconds(0);else if("week"==e)o.weekday()>2?(o.weekday(0),o.add(1,"week")):o.weekday(0),o.hours(0),o.minutes(0),o.seconds(0),o.milliseconds(0);else if("day"==e){switch(i){case 5:case 2:o.hours(24*Math.round(o.hours()/24));break;default:o.hours(12*Math.round(o.hours()/12))}o.minutes(0),o.seconds(0),o.milliseconds(0)}else if("weekday"==e){switch(i){case 5:case 2:o.hours(12*Math.round(o.hours()/12));break;default:o.hours(6*Math.round(o.hours()/6))}o.minutes(0),o.seconds(0),o.milliseconds(0)}else if("hour"==e){switch(i){case 4:o.minutes(60*Math.round(o.minutes()/60));break;default:o.minutes(30*Math.round(o.minutes()/30))}o.seconds(0),o.milliseconds(0)}else if("minute"==e){switch(i){case 15:case 10:o.minutes(5*Math.round(o.minutes()/5)),o.seconds(0);break;case 5:o.seconds(60*Math.round(o.seconds()/60));break;default:o.seconds(30*Math.round(o.seconds()/30))}o.milliseconds(0)}else if("second"==e)switch(i){case 15:case 10:o.seconds(5*Math.round(o.seconds()/5)),o.milliseconds(0);break;case 5:o.milliseconds(1e3*Math.round(o.milliseconds()/1e3));break;default:o.milliseconds(500*Math.round(o.milliseconds()/500))}else if("millisecond"==e){var r=i>5?i/2:1;o.milliseconds(Math.round(o.milliseconds()/r)*r)}return o},o.prototype.isMajor=function(){if(1==this.switchedYear)switch(this.scale){case"year":case"month":case"week":case"weekday":case"day":case"hour":case"minute":case"second":case"millisecond":return!0;default:return!1}else if(1==this.switchedMonth)switch(this.scale){case"week":case"weekday":case"day":case"hour":case"minute":case"second":case"millisecond":return!0;default:return!1}else if(1==this.switchedDay)switch(this.scale){case"millisecond":case"second":case"minute":case"hour":return!0;default:return!1}var t=this.moment(this.current);switch(this.scale){case"millisecond":return 0==t.milliseconds();case"second":return 0==t.seconds();case"minute":return 0==t.hours()&&0==t.minutes();case"hour":return 0==t.hours();case"weekday":case"day":case"week":return 1==t.date();case"month":return 0==t.month();case"year":default:return!1}},o.prototype.getLabelMinor=function(t){if(void 0==t&&(t=this.current),t instanceof Date&&(t=this.moment(t)),"function"==typeof this.format.minorLabels)return this.format.minorLabels(t,this.scale,this.step);var e=this.format.minorLabels[this.scale];switch(this.scale){case"week":if(this.isMajor()&&0!==t.weekday())return"";default:return e&&e.length>0?this.moment(t).format(e):""}},o.prototype.getLabelMajor=function(t){if(void 0==t&&(t=this.current),t instanceof Date&&(t=this.moment(t)),"function"==typeof this.format.majorLabels)return this.format.majorLabels(t,this.scale,this.step);var e=this.format.majorLabels[this.scale];return e&&e.length>0?this.moment(t).format(e):""},o.prototype.getClassName=function(){function t(t){return t/a%2==0?" vis-even":" vis-odd"}function e(t){return t.isSame(new Date,"day")?" vis-today":t.isSame(n().add(1,"day"),"day")?" vis-tomorrow":t.isSame(n().add(-1,"day"),"day")?" vis-yesterday":""}function i(t){return t.isSame(new Date,"week")?" vis-current-week":""}function o(t){return t.isSame(new Date,"month")?" vis-current-month":""}var n=this.moment,s=this.moment(this.current),r=s.locale?s.locale("en"):s.lang("en"),a=this.step,h=[];switch(this.scale){case"millisecond":h.push(e(r)),h.push(t(r.milliseconds()));break;case"second":h.push(e(r)),h.push(t(r.seconds()));break;case"minute":h.push(e(r)),h.push(t(r.minutes()));break;case"hour":h.push("vis-h"+r.hours()+(4==this.step?"-h"+(r.hours()+4):"")),h.push(e(r)),h.push(t(r.hours()));break;case"weekday":h.push("vis-"+r.format("dddd").toLowerCase()),h.push(e(r)),h.push(i(r)),h.push(t(r.date()));break;case"day":h.push("vis-day"+r.date()),h.push("vis-"+r.format("MMMM").toLowerCase()),h.push(e(r)),h.push(o(r)),h.push(this.step<=2?e(r):""),h.push(this.step<=2?"vis-"+r.format("dddd").toLowerCase():""),h.push(t(r.date()-1));break;case"week":h.push("vis-week"+r.format("w")),h.push(i(r)),h.push(t(r.week()));break;case"month":h.push("vis-"+r.format("MMMM").toLowerCase()),h.push(o(r)),h.push(t(r.month()));break;case"year":h.push("vis-year"+r.year()),h.push(function(t){return t.isSame(new Date,"year")?" vis-current-year":""}(r)),h.push(t(r.year()))}return h.filter(String).join(" ")},t.exports=o},function(t,e,i){function o(t,e){this.body=t,this.defaultOptions={rtl:!1,showCurrentTime:!0,moment:r,locales:a,locale:"en"},this.options=n.extend({},this.defaultOptions),this.offset=0,this._create(),this.setOptions(e)}var n=i(0),s=i(5),r=i(3),a=i(69);o.prototype=new s,o.prototype._create=function(){var t=document.createElement("div");t.className="vis-current-time",t.style.position="absolute",t.style.top="0px",t.style.height="100%",this.bar=t},o.prototype.destroy=function(){this.options.showCurrentTime=!1,this.redraw(),this.body=null},o.prototype.setOptions=function(t){t&&n.selectiveExtend(["rtl","showCurrentTime","moment","locale","locales"],this.options,t)},o.prototype.redraw=function(){if(this.options.showCurrentTime){var t=this.body.dom.backgroundVertical;this.bar.parentNode!=t&&(this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar),t.appendChild(this.bar),this.start());var e=this.options.moment((new Date).valueOf()+this.offset),i=this.body.util.toScreen(e),o=this.options.locales[this.options.locale];o||(this.warned||(console.log("WARNING: options.locales['"+this.options.locale+"'] not found. See http://visjs.org/docs/timeline/#Localization"),this.warned=!0),o=this.options.locales.en);var n=o.current+" "+o.time+": "+e.format("dddd, MMMM Do YYYY, H:mm:ss");n=n.charAt(0).toUpperCase()+n.substring(1),this.options.rtl?this.bar.style.right=i+"px":this.bar.style.left=i+"px",this.bar.title=n}else this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar),this.stop();return!1},o.prototype.start=function(){function t(){e.stop();var i=e.body.range.conversion(e.body.domProps.center.width).scale,o=1/i/10;o<30&&(o=30),o>1e3&&(o=1e3),e.redraw(),e.body.emitter.emit("currentTimeTick"),e.currentTimeTimer=setTimeout(t,o)}var e=this;t()},o.prototype.stop=function(){void 0!==this.currentTimeTimer&&(clearTimeout(this.currentTimeTimer),delete this.currentTimeTimer)},o.prototype.setCurrentTime=function(t){var e=n.convert(t,"Date").valueOf(),i=(new Date).valueOf();this.offset=e-i,this.redraw()},o.prototype.getCurrentTime=function(){return new Date((new Date).valueOf()+this.offset)},t.exports=o},function(t,e,i){function o(t,e,i){if(this.groupId=t,this.subgroups={},this.subgroupStack={},this.subgroupStackAll=!1,this.doInnerStack=!1,this.subgroupIndex=0,this.subgroupOrderer=e&&e.subgroupOrder,this.itemSet=i,this.isVisible=null,this.stackDirty=!0,e&&e.nestedGroups&&(this.nestedGroups=e.nestedGroups,0==e.showNested?this.showNested=!1:this.showNested=!0),e&&e.subgroupStack)if("boolean"==typeof e.subgroupStack)this.doInnerStack=e.subgroupStack,this.subgroupStackAll=e.subgroupStack;else for(var o in e.subgroupStack)this.subgroupStack[o]=e.subgroupStack[o],this.doInnerStack=this.doInnerStack||e.subgroupStack[o];this.nestedInGroup=null,this.dom={},this.props={label:{width:0,height:0}},this.className=null,this.items={},this.visibleItems=[],this.itemsInRange=[],this.orderedItems={byStart:[],byEnd:[]},this.checkRangedItems=!1;var n=this;this.itemSet.body.emitter.on("checkRangedItems",function(){n.checkRangedItems=!0}),this._create(),this.setData(e)}var n=i(4),s=function(t){return t&&t.__esModule?t:{default:t}}(n),r=i(0),a=i(71);o.prototype._create=function(){var t=document.createElement("div");this.itemSet.options.groupEditable.order?t.className="vis-label draggable":t.className="vis-label",this.dom.label=t;var e=document.createElement("div");e.className="vis-inner",t.appendChild(e),this.dom.inner=e;var i=document.createElement("div");i.className="vis-group",i["timeline-group"]=this,this.dom.foreground=i,this.dom.background=document.createElement("div"),this.dom.background.className="vis-group",this.dom.axis=document.createElement("div"),this.dom.axis.className="vis-group",this.dom.marker=document.createElement("div"),this.dom.marker.style.visibility="hidden",this.dom.marker.style.position="absolute",this.dom.marker.innerHTML="",this.dom.background.appendChild(this.dom.marker)},o.prototype.setData=function(t){var e,i;if(this.itemSet.options&&this.itemSet.options.groupTemplate?(i=this.itemSet.options.groupTemplate.bind(this),e=i(t,this.dom.inner)):e=t&&t.content,e instanceof Element){for(this.dom.inner.appendChild(e);this.dom.inner.firstChild;)this.dom.inner.removeChild(this.dom.inner.firstChild);this.dom.inner.appendChild(e)}else e instanceof Object?i(t,this.dom.inner):this.dom.inner.innerHTML=void 0!==e&&null!==e?e:this.groupId||"";if(this.dom.label.title=t&&t.title||"",this.dom.inner.firstChild?r.removeClassName(this.dom.inner,"vis-hidden"):r.addClassName(this.dom.inner,"vis-hidden"),t&&t.nestedGroups){this.nestedGroups&&this.nestedGroups==t.nestedGroups||(this.nestedGroups=t.nestedGroups),void 0===t.showNested&&void 0!==this.showNested||(0==t.showNested?this.showNested=!1:this.showNested=!0),r.addClassName(this.dom.label,"vis-nesting-group");var o=this.itemSet.options.rtl?"collapsed-rtl":"collapsed";this.showNested?(r.removeClassName(this.dom.label,o),r.addClassName(this.dom.label,"expanded")):(r.removeClassName(this.dom.label,"expanded"),r.addClassName(this.dom.label,o))}else this.nestedGroups&&(this.nestedGroups=null,o=this.itemSet.options.rtl?"collapsed-rtl":"collapsed",r.removeClassName(this.dom.label,o),r.removeClassName(this.dom.label,"expanded"),r.removeClassName(this.dom.label,"vis-nesting-group"));t&&t.nestedInGroup&&(r.addClassName(this.dom.label,"vis-nested-group"),this.itemSet.options&&this.itemSet.options.rtl?this.dom.inner.style.paddingRight="30px":this.dom.inner.style.paddingLeft="30px");var n=t&&t.className||null;n!=this.className&&(this.className&&(r.removeClassName(this.dom.label,this.className),r.removeClassName(this.dom.foreground,this.className),r.removeClassName(this.dom.background,this.className),r.removeClassName(this.dom.axis,this.className)),r.addClassName(this.dom.label,n),r.addClassName(this.dom.foreground,n),r.addClassName(this.dom.background,n),r.addClassName(this.dom.axis,n),this.className=n),this.style&&(r.removeCssText(this.dom.label,this.style),this.style=null),t&&t.style&&(r.addCssText(this.dom.label,t.style),this.style=t.style)},o.prototype.getLabelWidth=function(){return this.props.label.width},o.prototype._didMarkerHeightChange=function(){var t=this.dom.marker.clientHeight;if(t!=this.lastMarkerHeight){this.lastMarkerHeight=t;var e={},i=0;r.forEach(this.items,function(t,o){if(t.dirty=!0,t.displayed){e[o]=t.redraw(!0),i=e[o].length}});if(i>0)for(var o=0;o<i;o++)r.forEach(e,function(t){t[o]()});return!0}},o.prototype._calculateGroupSizeAndPosition=function(){var t=this.dom.foreground.offsetTop,e=this.dom.foreground.offsetLeft,i=this.dom.foreground.offsetWidth;this.top=t,this.right=e,this.width=i},o.prototype._redrawItems=function(t,e,i,o){if(t||this.stackDirty||this.isVisible&&!e){var n={},s=null;if("function"==typeof this.itemSet.options.order){var h=this,d={},l=0;if(r.forEach(this.items,function(t,e){t.displayed||(d[e]=t.redraw(!0),l=d[e].length,h.visibleItems.push(t))}),l>0)for(var u=0;u<l;u++)r.forEach(d,function(t){t[u]()});if(r.forEach(this.items,function(t){t.repositionX(!1)}),this.doInnerStack&&this.itemSet.options.stackSubgroups){for(s in this.subgroups)n[s]=this.subgroups[s].items.slice().sort(function(t,e){return h.itemSet.options.order(t.data,e.data)});a.stackSubgroupsWithInnerStack(n,i,this.subgroups)}else{var p=this.orderedItems.byStart.slice().sort(function(t,e){return h.itemSet.options.order(t.data,e.data)});a.stack(p,i,!0)}this.visibleItems=this._updateItemsInRange(this.orderedItems,this.visibleItems,o)}else if(this.visibleItems=this._updateItemsInRange(this.orderedItems,this.visibleItems,o),this.itemSet.options.stack)if(this.doInnerStack&&this.itemSet.options.stackSubgroups){for(s in this.subgroups)n[s]=this.subgroups[s].items;a.stackSubgroupsWithInnerStack(n,i,this.subgroups)}else a.stack(this.visibleItems,i,!0);else a.nostack(this.visibleItems,i,this.subgroups,this.itemSet.options.stackSubgroups);this.stackDirty=!1}},o.prototype._didResize=function(t,e){t=r.updateProperty(this,"height",e)||t;var i=this.dom.inner.clientWidth,o=this.dom.inner.clientHeight;return t=r.updateProperty(this.props.label,"width",i)||t,t=r.updateProperty(this.props.label,"height",o)||t},o.prototype._applyGroupHeight=function(t){this.dom.background.style.height=t+"px",this.dom.foreground.style.height=t+"px",this.dom.label.style.height=t+"px"},o.prototype._updateItemsVerticalPosition=function(t){for(var e=0,i=this.visibleItems.length;e<i;e++){var o=this.visibleItems[e];o.repositionY(t),this.isVisible||"__background__"==this.groupId||o.displayed&&o.hide()}},o.prototype.redraw=function(t,e,i,o){var n,s=!1,r=this.isVisible,a=[function(){i=this._didMarkerHeightChange.bind(this)}.bind(this),this._updateSubGroupHeights.bind(this,e),this._calculateGroupSizeAndPosition.bind(this),function(){this.isVisible=this._isGroupVisible.bind(this)(t,e)}.bind(this),function(){this._redrawItems.bind(this)(i,r,e,t)}.bind(this),this._updateSubgroupsSizes.bind(this),function(){n=this._calculateHeight.bind(this)(e)}.bind(this),this._calculateGroupSizeAndPosition.bind(this),function(){s=this._didResize.bind(this)(s,n)}.bind(this),function(){this._applyGroupHeight.bind(this)(n)}.bind(this),function(){this._updateItemsVerticalPosition.bind(this)(e)}.bind(this),function(){return!this.isVisible&&this.height&&(s=!1),s}];if(o)return a;var h;return a.forEach(function(t){h=t()}),h},o.prototype._updateSubGroupHeights=function(t){if((0,s.default)(this.subgroups).length>0){var e=this;this.resetSubgroups(),r.forEach(this.visibleItems,function(i){void 0!==i.data.subgroup&&(e.subgroups[i.data.subgroup].height=Math.max(e.subgroups[i.data.subgroup].height,i.height+t.item.vertical),e.subgroups[i.data.subgroup].visible=!0)})}},o.prototype._isGroupVisible=function(t,e){return this.top<=t.body.domProps.centerContainer.height-t.body.domProps.scrollTop+e.axis&&this.top+this.height+e.axis>=-t.body.domProps.scrollTop},o.prototype._calculateHeight=function(t){var e,i=this.visibleItems;if(i.length>0){var o=i[0].top,n=i[0].top+i[0].height;if(r.forEach(i,function(t){o=Math.min(o,t.top),n=Math.max(n,t.top+t.height)}),o>t.axis){var s=o-t.axis;n-=s,r.forEach(i,function(t){t.top-=s})}e=n+t.item.vertical/2}else e=0;return e=Math.max(e,this.props.label.height)},o.prototype.show=function(){this.dom.label.parentNode||this.itemSet.dom.labelSet.appendChild(this.dom.label),this.dom.foreground.parentNode||this.itemSet.dom.foreground.appendChild(this.dom.foreground),this.dom.background.parentNode||this.itemSet.dom.background.appendChild(this.dom.background),this.dom.axis.parentNode||this.itemSet.dom.axis.appendChild(this.dom.axis)},o.prototype.hide=function(){var t=this.dom.label;t.parentNode&&t.parentNode.removeChild(t);var e=this.dom.foreground;e.parentNode&&e.parentNode.removeChild(e);var i=this.dom.background;i.parentNode&&i.parentNode.removeChild(i);var o=this.dom.axis;o.parentNode&&o.parentNode.removeChild(o)},o.prototype.add=function(t){if(this.items[t.id]=t,t.setParent(this),this.stackDirty=!0,void 0!==t.data.subgroup&&(this._addToSubgroup(t),this.orderSubgroups()),-1==this.visibleItems.indexOf(t)){var e=this.itemSet.body.range;this._checkIfVisible(t,this.visibleItems,e)}},o.prototype._addToSubgroup=function(t,e){e=e||t.data.subgroup,void 0!=e&&void 0===this.subgroups[e]&&(this.subgroups[e]={height:0,top:0,start:t.data.start,end:t.data.end||t.data.start,visible:!1,index:this.subgroupIndex,items:[],stack:this.subgroupStackAll||this.subgroupStack[e]||!1},this.subgroupIndex++),new Date(t.data.start)<new Date(this.subgroups[e].start)&&(this.subgroups[e].start=t.data.start);var i=t.data.end||t.data.start;new Date(i)>new Date(this.subgroups[e].end)&&(this.subgroups[e].end=i),this.subgroups[e].items.push(t)},o.prototype._updateSubgroupsSizes=function(){var t=this;if(t.subgroups)for(var e in t.subgroups){var i=t.subgroups[e].items[0].data.end||t.subgroups[e].items[0].data.start,o=t.subgroups[e].items[0].data.start,n=i-1;t.subgroups[e].items.forEach(function(t){new Date(t.data.start)<new Date(o)&&(o=t.data.start);var e=t.data.end||t.data.start;new Date(e)>new Date(n)&&(n=e)}),t.subgroups[e].start=o,t.subgroups[e].end=new Date(n-1)}},o.prototype.orderSubgroups=function(){if(void 0!==this.subgroupOrderer){var t,e=[];if("string"==typeof this.subgroupOrderer){for(t in this.subgroups)e.push({subgroup:t,sortField:this.subgroups[t].items[0].data[this.subgroupOrderer]});e.sort(function(t,e){return t.sortField-e.sortField})}else if("function"==typeof this.subgroupOrderer){for(t in this.subgroups)e.push(this.subgroups[t].items[0].data);e.sort(this.subgroupOrderer)}if(e.length>0)for(var i=0;i<e.length;i++)this.subgroups[e[i].subgroup].index=i}},o.prototype.resetSubgroups=function(){for(var t in this.subgroups)this.subgroups.hasOwnProperty(t)&&(this.subgroups[t].visible=!1,this.subgroups[t].height=0)},o.prototype.remove=function(t){delete this.items[t.id],t.setParent(null),this.stackDirty=!0;var e=this.visibleItems.indexOf(t);-1!=e&&this.visibleItems.splice(e,1),void 0!==t.data.subgroup&&(this._removeFromSubgroup(t),this.orderSubgroups())},o.prototype._removeFromSubgroup=function(t,e){if(void 0!=(e=e||t.data.subgroup)){var i=this.subgroups[e];if(i){var o=i.items.indexOf(t);o>=0&&(i.items.splice(o,1),i.items.length?this._updateSubgroupsSizes():delete this.subgroups[e])}}},o.prototype.removeFromDataSet=function(t){this.itemSet.removeItem(t.id)},o.prototype.order=function(){for(var t=r.toArray(this.items),e=[],i=[],o=0;o<t.length;o++)void 0!==t[o].data.end&&i.push(t[o]),e.push(t[o]);this.orderedItems={byStart:e,byEnd:i},a.orderByStart(this.orderedItems.byStart),a.orderByEnd(this.orderedItems.byEnd)},o.prototype._updateItemsInRange=function(t,e,i){var o=[],n={},s=(i.end-i.start)/4,a=i.start-s,h=i.end+s,d=function(t){return t<a?-1:t<=h?0:1};if(e.length>0)for(var l=0;l<e.length;l++)this._checkIfVisibleWithReference(e[l],o,n,i);var u=r.binarySearchCustom(t.byStart,d,"data","start");if(this._traceVisible(u,t.byStart,o,n,function(t){return t.data.start<a||t.data.start>h}),1==this.checkRangedItems)for(this.checkRangedItems=!1,l=0;l<t.byEnd.length;l++)this._checkIfVisibleWithReference(t.byEnd[l],o,n,i);else{var p=r.binarySearchCustom(t.byEnd,d,"data","end");this._traceVisible(p,t.byEnd,o,n,function(t){return t.data.end<a||t.data.end>h})}var c={},m=0;for(l=0;l<o.length;l++){var f=o[l];if(!f.displayed){c[l]=f.redraw(!0),m=c[l].length}}if(m>0)for(var g=0;g<m;g++)r.forEach(c,function(t){t[g]()});for(l=0;l<o.length;l++)o[l].repositionX();return o},o.prototype._traceVisible=function(t,e,i,o,n){if(-1!=t){var s,r;for(s=t;s>=0&&(r=e[s],!n(r));s--)void 0===o[r.id]&&(o[r.id]=!0,i.push(r));for(s=t+1;s<e.length&&(r=e[s],!n(r));s++)void 0===o[r.id]&&(o[r.id]=!0,i.push(r))}},o.prototype._checkIfVisible=function(t,e,i){t.isVisible(i)?(t.displayed||t.show(),t.repositionX(),e.push(t)):t.displayed&&t.hide()},o.prototype._checkIfVisibleWithReference=function(t,e,i,o){t.isVisible(o)?void 0===i[t.id]&&(i[t.id]=!0,e.push(t)):t.displayed&&t.hide()},o.prototype.changeSubgroup=function(t,e,i){this._removeFromSubgroup(t,e),this._addToSubgroup(t,i),this.orderSubgroups()},t.exports=o},function(t,e,i){function o(t,e,i){r.call(this,t,e,i),this.width=0,this.height=0,this.top=0,this.left=0}var n=i(43),s=function(t){return t&&t.__esModule?t:{default:t}}(n),r=i(51);o.prototype=(0,s.default)(r.prototype),o.prototype.redraw=function(t,e,i){this.visibleItems=this._updateItemsInRange(this.orderedItems,this.visibleItems,t),this.width=this.dom.background.offsetWidth,this.dom.background.style.height="0";for(var o=0,n=this.visibleItems.length;o<n;o++){this.visibleItems[o].repositionY(e)}return!1},o.prototype.show=function(){this.dom.background.parentNode||this.itemSet.dom.background.appendChild(this.dom.background)},t.exports=o},function(t,e,i){function o(t,e,i){if(this.props={content:{width:0}},this.overflow=!1,this.options=i,t){if(void 0==t.start)throw new Error('Property "start" missing in item '+t.id);if(void 0==t.end)throw new Error('Property "end" missing in item '+t.id)}n.call(this,t,e,i)}var n=i(22);o.prototype=new n(null,null,null),o.prototype.baseClassName="vis-item vis-range",o.prototype.isVisible=function(t){return this.data.start<t.end&&this.data.end>t.start},o.prototype._createDomElement=function(){this.dom||(this.dom={},this.dom.box=document.createElement("div"),this.dom.frame=document.createElement("div"),this.dom.frame.className="vis-item-overflow",this.dom.box.appendChild(this.dom.frame),this.dom.visibleFrame=document.createElement("div"),this.dom.visibleFrame.className="vis-item-visible-frame",this.dom.box.appendChild(this.dom.visibleFrame),this.dom.content=document.createElement("div"),this.dom.content.className="vis-item-content",this.dom.frame.appendChild(this.dom.content),this.dom.box["timeline-item"]=this,this.dirty=!0)},o.prototype._appendDomElement=function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.box.parentNode){var t=this.parent.dom.foreground;if(!t)throw new Error("Cannot redraw item: parent has no foreground container element");t.appendChild(this.dom.box)}this.displayed=!0},o.prototype._updateDirtyDomComponents=function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.box),this._updateStyle(this.dom.box);var t=this.editable.updateTime||this.editable.updateGroup,e=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+(t?" vis-editable":" vis-readonly");this.dom.box.className=this.baseClassName+e,this.dom.content.style.maxWidth="none"}},o.prototype._getDomComponentsSizes=function(){return this.overflow="hidden"!==window.getComputedStyle(this.dom.frame).overflow,{content:{width:this.dom.content.offsetWidth},box:{height:this.dom.box.offsetHeight}}},o.prototype._updateDomComponentsSizes=function(t){this.props.content.width=t.content.width,this.height=t.box.height,this.dom.content.style.maxWidth="",this.dirty=!1},o.prototype._repaintDomAdditionals=function(){this._repaintOnItemUpdateTimeTooltip(this.dom.box),this._repaintDeleteButton(this.dom.box),this._repaintDragCenter(),this._repaintDragLeft(),this._repaintDragRight()},o.prototype.redraw=function(t){var e,i=[this._createDomElement.bind(this),this._appendDomElement.bind(this),this._updateDirtyDomComponents.bind(this),function(){this.dirty&&(e=this._getDomComponentsSizes.bind(this)())}.bind(this),function(){this.dirty&&this._updateDomComponentsSizes.bind(this)(e)}.bind(this),this._repaintDomAdditionals.bind(this)];if(t)return i;var o;return i.forEach(function(t){o=t()}),o},o.prototype.show=function(){this.displayed||this.redraw()},o.prototype.hide=function(){if(this.displayed){var t=this.dom.box;t.parentNode&&t.parentNode.removeChild(t),this.displayed=!1}},o.prototype.repositionX=function(t){var e,i,o=this.parent.width,n=this.conversion.toScreen(this.data.start),s=this.conversion.toScreen(this.data.end),r=void 0===this.data.align?this.options.align:this.data.align;!1===this.data.limitSize||void 0!==t&&!0!==t||(n<-o&&(n=-o),s>2*o&&(s=2*o));var a=Math.max(s-n+.5,1);switch(this.overflow?(this.options.rtl?this.right=n:this.left=n,this.width=a+this.props.content.width,i=this.props.content.width):(this.options.rtl?this.right=n:this.left=n,this.width=a,i=Math.min(s-n,this.props.content.width)),this.options.rtl?this.dom.box.style.right=this.right+"px":this.dom.box.style.left=this.left+"px",this.dom.box.style.width=a+"px",r){case"left":this.options.rtl?this.dom.content.style.right="0":this.dom.content.style.left="0";break;case"right":this.options.rtl?this.dom.content.style.right=Math.max(a-i,0)+"px":this.dom.content.style.left=Math.max(a-i,0)+"px";break;case"center":this.options.rtl?this.dom.content.style.right=Math.max((a-i)/2,0)+"px":this.dom.content.style.left=Math.max((a-i)/2,0)+"px";break;default:e=this.overflow?s>0?Math.max(-n,0):-i:n<0?-n:0,this.options.rtl?this.dom.content.style.right=e+"px":(this.dom.content.style.left=e+"px",this.dom.content.style.width="calc(100% - "+e+"px)")}},o.prototype.repositionY=function(){var t=this.options.orientation.item,e=this.dom.box;e.style.top="top"==t?this.top+"px":this.parent.height-this.top-this.height+"px"},o.prototype._repaintDragLeft=function(){if((this.selected||this.options.itemsAlwaysDraggable.range)&&this.options.editable.updateTime&&!this.dom.dragLeft){var t=document.createElement("div");t.className="vis-drag-left",t.dragLeftItem=this,this.dom.box.appendChild(t),this.dom.dragLeft=t}else this.selected||this.options.itemsAlwaysDraggable.range||!this.dom.dragLeft||(this.dom.dragLeft.parentNode&&this.dom.dragLeft.parentNode.removeChild(this.dom.dragLeft),this.dom.dragLeft=null)},o.prototype._repaintDragRight=function(){if((this.selected||this.options.itemsAlwaysDraggable.range)&&this.options.editable.updateTime&&!this.dom.dragRight){var t=document.createElement("div");t.className="vis-drag-right",t.dragRightItem=this,this.dom.box.appendChild(t),this.dom.dragRight=t}else this.selected||this.options.itemsAlwaysDraggable.range||!this.dom.dragRight||(this.dom.dragRight.parentNode&&this.dom.dragRight.parentNode.removeChild(this.dom.dragRight),this.dom.dragRight=null)},t.exports=o},function(t,e,i){function o(t,e){}function n(t,e){return e=void 0===e?{}:e,{style:e.style||t.options.drawPoints.style,styles:e.styles||t.options.drawPoints.styles,size:e.size||t.options.drawPoints.size,className:e.className||t.className}}function s(t,e){var i=void 0;return t.options&&t.options.drawPoints&&t.options.drawPoints.onRender&&"function"==typeof t.options.drawPoints.onRender&&(i=t.options.drawPoints.onRender),e.group.options&&e.group.options.drawPoints&&e.group.options.drawPoints.onRender&&"function"==typeof e.group.options.drawPoints.onRender&&(i=e.group.options.drawPoints.onRender),i}var r=i(1),a=function(t){return t&&t.__esModule?t:{default:t}}(r),h=i(11);o.draw=function(t,e,i,o){o=o||0;for(var r=s(i,e),d=0;d<t.length;d++)if(r){var l=r(t[d],e);!0!==l&&"object"!==(void 0===l?"undefined":(0,a.default)(l))||h.drawPoint(t[d].screen_x+o,t[d].screen_y,n(e,l),i.svgElements,i.svg,t[d].label)}else h.drawPoint(t[d].screen_x+o,t[d].screen_y,n(e),i.svgElements,i.svg,t[d].label)},o.drawIcon=function(t,e,i,o,s,r){var a=.5*s,d=h.getSVGElement("rect",r.svgElements,r.svg);d.setAttributeNS(null,"x",e),d.setAttributeNS(null,"y",i-a),d.setAttributeNS(null,"width",o),d.setAttributeNS(null,"height",2*a),d.setAttributeNS(null,"class","vis-outline"),h.drawPoint(e+.5*o,i,n(t),r.svgElements,r.svg)},t.exports=o},function(t,e,i){i(88);for(var o=i(6),n=i(14),s=i(23),r=i(7)("toStringTag"),a="CSSRuleList,CSSStyleDeclaration,CSSValueList,ClientRectList,DOMRectList,DOMStringList,DOMTokenList,DataTransferItemList,FileList,HTMLAllCollection,HTMLCollection,HTMLFormElement,HTMLSelectElement,MediaList,MimeTypeArray,NamedNodeMap,NodeList,PaintRequestList,Plugin,PluginArray,SVGLengthList,SVGNumberList,SVGPathSegList,SVGPointList,SVGStringList,SVGTransformList,SourceBufferList,StyleSheetList,TextTrackCueList,TextTrackList,TouchList".split(","),h=0;h<a.length;h++){var d=a[h],l=o[d],u=l&&l.prototype;u&&!u[r]&&n(u,r,d),s[d]=s.Array}},function(t,e,i){var o=i(35),n=i(17),s=i(59),r=i(14),a=i(10),h=i(23),d=i(94),l=i(42),u=i(100),p=i(7)("iterator"),c=!([].keys&&"next"in[].keys()),m=function(){return this};t.exports=function(t,e,i,f,g,v,y){d(i,e,f);var b,_,w,x=function(t){if(!c&&t in C)return C[t];switch(t){case"keys":case"values":return function(){return new i(this,t)}}return function(){return new i(this,t)}},D=e+" Iterator",S="values"==g,k=!1,C=t.prototype,T=C[p]||C["@@iterator"]||g&&C[g],M=T||x(g),O=g?S?x("entries"):M:void 0,E="Array"==e?C.entries||T:T;if(E&&(w=u(E.call(new t)))!==Object.prototype&&w.next&&(l(w,D,!0),o||a(w,p)||r(w,p,m)),S&&T&&"values"!==T.name&&(k=!0,M=function(){return T.call(this)}),o&&!y||!c&&!k&&C[p]||r(C,p,M),h[e]=M,h[D]=m,g)if(b={values:S?M:x("values"),keys:v?M:x("keys"),entries:O},y)for(_ in b)_ in C||s(C,_,b[_]);else n(n.P+n.F*(c||k),e,b);return b}},function(t,e,i){t.exports=!i(9)&&!i(19)(function(){return 7!=Object.defineProperty(i(58)("div"),"a",{get:function(){return 7}}).a})},function(t,e,i){var o=i(24),n=i(6).document,s=o(n)&&o(n.createElement);t.exports=function(t){return s?n.createElement(t):{}}},function(t,e,i){t.exports=i(14)},function(t,e,i){var o=i(10),n=i(13),s=i(96)(!1),r=i(39)("IE_PROTO");t.exports=function(t,e){var i,a=n(t),h=0,d=[];for(i in a)i!=r&&o(a,i)&&d.push(i);for(;e.length>h;)o(a,i=e[h++])&&(~s(d,i)||d.push(i));return d}},function(t,e,i){var o=i(34);t.exports=function(t){return Object(o(t))}},function(t,e,i){var o=i(101)(!0);i(56)(String,"String",function(t){this._t=String(t),this._i=0},function(){var t,e=this._t,i=this._i;return i>=e.length?{value:void 0,done:!0}:(t=o(e,i),this._i+=t.length,{value:t,done:!1})})},function(t,e){e.f=Object.getOwnPropertySymbols},function(t,e,i){var o=i(60),n=i(41).concat("length","prototype");e.f=Object.getOwnPropertyNames||function(t){return o(t,n)}},function(t,e,i){function o(t){this.delay=null,this.max=1/0,this._queue=[],this._timeout=null,this._extended=null,this.setOptions(t)}o.prototype.setOptions=function(t){t&&void 0!==t.delay&&(this.delay=t.delay),t&&void 0!==t.max&&(this.max=t.max),this._flushIfNeeded()},o.extend=function(t,e){var i=new o(e);if(void 0!==t.flush)throw new Error("Target object already has a property flush");t.flush=function(){i.flush()};var n=[{name:"flush",original:void 0}];if(e&&e.replace)for(var s=0;s<e.replace.length;s++){var r=e.replace[s];n.push({name:r,original:t[r]}),i.replace(t,r)}return i._extended={object:t,methods:n},i},o.prototype.destroy=function(){if(this.flush(),this._extended){for(var t=this._extended.object,e=this._extended.methods,i=0;i<e.length;i++){var o=e[i];o.original?t[o.name]=o.original:delete t[o.name]}this._extended=null}},o.prototype.replace=function(t,e){var i=this,o=t[e];if(!o)throw new Error("Method "+e+" undefined");t[e]=function(){for(var t=[],e=0;e<arguments.length;e++)t[e]=arguments[e];i.queue({args:t,fn:o,context:this})}},o.prototype.queue=function(t){"function"==typeof t?this._queue.push({fn:t}):this._queue.push(t),this._flushIfNeeded()},o.prototype._flushIfNeeded=function(){if(this._queue.length>this.max&&this.flush(),clearTimeout(this._timeout),this.queue.length>0&&"number"==typeof this.delay){var t=this;this._timeout=setTimeout(function(){t.flush()},this.delay)}},o.prototype.flush=function(){for(;this._queue.length>0;){var t=this._queue.shift();t.fn.apply(t.context||t.fn,t.args||[])}},t.exports=o},function(t,e){function i(t){if(t)return o(t)}function o(t){for(var e in i.prototype)t[e]=i.prototype[e];return t}t.exports=i,i.prototype.on=i.prototype.addEventListener=function(t,e){return this._callbacks=this._callbacks||{},(this._callbacks[t]=this._callbacks[t]||[]).push(e),this},i.prototype.once=function(t,e){function i(){o.off(t,i),e.apply(this,arguments)}var o=this;return this._callbacks=this._callbacks||{},i.fn=e,this.on(t,i),this},i.prototype.off=i.prototype.removeListener=i.prototype.removeAllListeners=i.prototype.removeEventListener=function(t,e){if(this._callbacks=this._callbacks||{},0==arguments.length)return this._callbacks={},this;var i=this._callbacks[t];if(!i)return this;if(1==arguments.length)return delete this._callbacks[t],this;for(var o,n=0;n<i.length;n++)if((o=i[n])===e||o.fn===e){i.splice(n,1);break}return this},i.prototype.emit=function(t){this._callbacks=this._callbacks||{};var e=[].slice.call(arguments,1),i=this._callbacks[t];if(i){i=i.slice(0);for(var o=0,n=i.length;o<n;++o)i[o].apply(this,e)}return this},i.prototype.listeners=function(t){return this._callbacks=this._callbacks||{},this._callbacks[t]||[]},i.prototype.hasListeners=function(t){return!!this.listeners(t).length}},function(t,e,i){e.onTouch=function(t,e){e.inputHandler=function(t){t.isFirst&&e(t)},t.on("hammer.input",e.inputHandler)},e.onRelease=function(t,e){return e.inputHandler=function(t){t.isFinal&&e(t)},t.on("hammer.input",e.inputHandler)},e.offTouch=function(t,e){t.off("hammer.input",e.inputHandler)},e.offRelease=e.offTouch,e.disablePreventDefaultVertically=function(t){return t.getTouchAction=function(){return["pan-y"]},t}},function(t,e,i){var o,n,s;!function(i,r){n=[],o=r,void 0!==(s="function"==typeof o?o.apply(e,n):o)&&(t.exports=s)}(0,function(){function t(t){var e,i=t&&t.preventDefault||!1,o=t&&t.container||window,n={},s={keydown:{},keyup:{}},r={};for(e=97;e<=122;e++)r[String.fromCharCode(e)]={code:e-97+65,shift:!1};for(e=65;e<=90;e++)r[String.fromCharCode(e)]={code:e,shift:!0};for(e=0;e<=9;e++)r[""+e]={code:48+e,shift:!1};for(e=1;e<=12;e++)r["F"+e]={code:111+e,shift:!1};for(e=0;e<=9;e++)r["num"+e]={code:96+e,shift:!1};r["num*"]={code:106,shift:!1},r["num+"]={code:107,shift:!1},r["num-"]={code:109,shift:!1},r["num/"]={code:111,shift:!1},r["num."]={code:110,shift:!1},r.left={code:37,shift:!1},r.up={code:38,shift:!1},r.right={code:39,shift:!1},r.down={code:40,shift:!1},r.space={code:32,shift:!1},r.enter={code:13,shift:!1},r.shift={code:16,shift:void 0},r.esc={code:27,shift:!1},r.backspace={code:8,shift:!1},r.tab={code:9,shift:!1},r.ctrl={code:17,shift:!1},r.alt={code:18,shift:!1},r.delete={code:46,shift:!1},r.pageup={code:33,shift:!1},r.pagedown={code:34,shift:!1},r["="]={code:187,shift:!1},r["-"]={code:189,shift:!1},r["]"]={code:221,shift:!1},r["["]={code:219,shift:!1};var a=function(t){d(t,"keydown")},h=function(t){d(t,"keyup")},d=function(t,e){if(void 0!==s[e][t.keyCode]){for(var o=s[e][t.keyCode],n=0;n<o.length;n++)void 0===o[n].shift?o[n].fn(t):1==o[n].shift&&1==t.shiftKey?o[n].fn(t):0==o[n].shift&&0==t.shiftKey&&o[n].fn(t);1==i&&t.preventDefault()}};return n.bind=function(t,e,i){if(void 0===i&&(i="keydown"),void 0===r[t])throw new Error("unsupported key: "+t);void 0===s[i][r[t].code]&&(s[i][r[t].code]=[]),s[i][r[t].code].push({fn:e,shift:r[t].shift})},n.bindAll=function(t,e){void 0===e&&(e="keydown");for(var i in r)r.hasOwnProperty(i)&&n.bind(i,t,e)},n.getKey=function(t){for(var e in r)if(r.hasOwnProperty(e)){if(1==t.shiftKey&&1==r[e].shift&&t.keyCode==r[e].code)return e;if(0==t.shiftKey&&0==r[e].shift&&t.keyCode==r[e].code)return e;if(t.keyCode==r[e].code&&"shift"==e)return e}return"unknown key, currently not supported"},n.unbind=function(t,e,i){if(void 0===i&&(i="keydown"),void 0===r[t])throw new Error("unsupported key: "+t);if(void 0!==e){var o=[],n=s[i][r[t].code];if(void 0!==n)for(var a=0;a<n.length;a++)n[a].fn==e&&n[a].shift==r[t].shift||o.push(s[i][r[t].code][a]);s[i][r[t].code]=o}else s[i][r[t].code]=[]},n.reset=function(){s={keydown:{},keyup:{}}},n.destroy=function(){s={keydown:{},keyup:{}},o.removeEventListener("keydown",a,!0),o.removeEventListener("keyup",h,!0)},o.addEventListener("keydown",a,!0),o.addEventListener("keyup",h,!0),n}return t})},function(t,e,i){e.en={current:"current",time:"time"},e.en_EN=e.en,e.en_US=e.en,e.it={current:"attuale",time:"tempo"},e.it_IT=e.it,e.it_CH=e.it,e.nl={current:"huidige",time:"tijd"},e.nl_NL=e.nl,e.nl_BE=e.nl,e.de={current:"Aktuelle",time:"Zeit"},e.de_DE=e.de,e.fr={current:"actuel",time:"heure"},e.fr_FR=e.fr,e.fr_CA=e.fr,e.fr_BE=e.fr,e.es={current:"corriente",time:"hora"},e.es_ES=e.es},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}function n(t,e){this.body=t,this.defaultOptions={type:null,orientation:{item:"bottom"},align:"auto",stack:!0,stackSubgroups:!0,groupOrderSwap:function(t,e,i){var o=e.order;e.order=t.order,t.order=o},groupOrder:"order",selectable:!0,multiselect:!1,itemsAlwaysDraggable:{item:!1,range:!1},editable:{updateTime:!1,updateGroup:!1,add:!1,remove:!1,overrideItems:!1},groupEditable:{order:!1,add:!1,remove:!1},snap:c.snap,onDropObjectOnItem:function(t,e,i){i(e)},onAdd:function(t,e){e(t)},onUpdate:function(t,e){e(t)},onMove:function(t,e){e(t)},onRemove:function(t,e){e(t)},onMoving:function(t,e){e(t)},onAddGroup:function(t,e){e(t)},onMoveGroup:function(t,e){e(t)},onRemoveGroup:function(t,e){e(t)},margin:{item:{horizontal:10,vertical:10},axis:20},showTooltips:!0,tooltip:{followMouse:!1,overflowMethod:"flip"},tooltipOnItemUpdateTime:!1},this.options=l.extend({},this.defaultOptions),this.options.rtl=e.rtl,this.itemOptions={type:{start:"Date",end:"Date"}},this.conversion={toScreen:t.util.toScreen,toTime:t.util.toTime},this.dom={},this.props={},this.hammer=null;var i=this;this.itemsData=null,this.groupsData=null,this.itemListeners={add:function(t,e,o){i._onAdd(e.items)},update:function(t,e,o){i._onUpdate(e.items)},remove:function(t,e,o){i._onRemove(e.items)}},this.groupListeners={add:function(t,e,o){if(i._onAddGroups(e.items),i.groupsData&&i.groupsData.length>0){var n=i.groupsData.getDataSet();n.get().forEach(function(t){if(t.nestedGroups){0!=t.showNested&&(t.showNested=!0);var e=[];t.nestedGroups.forEach(function(i){var o=n.get(i);o&&(o.nestedInGroup=t.id,0==t.showNested&&(o.visible=!1),e=e.concat(o))}),n.update(e,o)}})}},update:function(t,e,o){i._onUpdateGroups(e.items)},remove:function(t,e,o){i._onRemoveGroups(e.items)}},this.items={},this.groups={},this.groupIds=[],this.selection=[],this.popup=null,this.touchParams={},this.groupTouchParams={},this._create(),this.setOptions(e)}var s=i(43),r=o(s),a=i(1),h=o(a),d=i(12),l=i(0),u=i(15),p=i(20),c=i(49),m=i(5),f=i(51),g=i(52),v=i(72),y=i(73),b=i(53),_=i(74),w=i(133).default,x="__ungrouped__",D="__background__";n.prototype=new m,n.types={background:_,box:v,range:b,point:y},n.prototype._create=function(){var t=document.createElement("div");t.className="vis-itemset",t["timeline-itemset"]=this,this.dom.frame=t;var e=document.createElement("div");e.className="vis-background",t.appendChild(e),this.dom.background=e;var i=document.createElement("div");i.className="vis-foreground",t.appendChild(i),this.dom.foreground=i;var o=document.createElement("div");o.className="vis-axis",this.dom.axis=o;var n=document.createElement("div");n.className="vis-labelset",this.dom.labelSet=n,this._updateUngrouped();var s=new g(D,null,this);s.show(),this.groups[D]=s,this.hammer=new d(this.body.dom.centerContainer),this.hammer.on("hammer.input",function(t){t.isFirst&&this._onTouch(t)}.bind(this)),this.hammer.on("panstart",this._onDragStart.bind(this)),this.hammer.on("panmove",this._onDrag.bind(this)),this.hammer.on("panend",this._onDragEnd.bind(this)),this.hammer.get("pan").set({threshold:5,direction:d.DIRECTION_HORIZONTAL}),this.hammer.on("tap",this._onSelectItem.bind(this)),this.hammer.on("press",this._onMultiSelectItem.bind(this)),this.hammer.on("doubletap",this._onAddItem.bind(this)),this.options.rtl?this.groupHammer=new d(this.body.dom.rightContainer):this.groupHammer=new d(this.body.dom.leftContainer),this.groupHammer.on("tap",this._onGroupClick.bind(this)),this.groupHammer.on("panstart",this._onGroupDragStart.bind(this)),this.groupHammer.on("panmove",this._onGroupDrag.bind(this)),this.groupHammer.on("panend",this._onGroupDragEnd.bind(this)),this.groupHammer.get("pan").set({threshold:5,direction:d.DIRECTION_VERTICAL}),this.body.dom.centerContainer.addEventListener("mouseover",this._onMouseOver.bind(this)),this.body.dom.centerContainer.addEventListener("mouseout",this._onMouseOut.bind(this)),this.body.dom.centerContainer.addEventListener("mousemove",this._onMouseMove.bind(this)),this.body.dom.centerContainer.addEventListener("contextmenu",this._onDragEnd.bind(this)),this.body.dom.centerContainer.addEventListener("mousewheel",this._onMouseWheel.bind(this)),this.show()},n.prototype.setOptions=function(t){if(t){var e=["type","rtl","align","order","stack","stackSubgroups","selectable","multiselect","multiselectPerGroup","groupOrder","dataAttributes","template","groupTemplate","visibleFrameTemplate","hide","snap","groupOrderSwap","showTooltips","tooltip","tooltipOnItemUpdateTime"];l.selectiveExtend(e,this.options,t),"itemsAlwaysDraggable"in t&&("boolean"==typeof t.itemsAlwaysDraggable?(this.options.itemsAlwaysDraggable.item=t.itemsAlwaysDraggable,this.options.itemsAlwaysDraggable.range=!1):"object"===(0,h.default)(t.itemsAlwaysDraggable)&&(l.selectiveExtend(["item","range"],this.options.itemsAlwaysDraggable,t.itemsAlwaysDraggable),this.options.itemsAlwaysDraggable.item||(this.options.itemsAlwaysDraggable.range=!1))),"orientation"in t&&("string"==typeof t.orientation?this.options.orientation.item="top"===t.orientation?"top":"bottom":"object"===(0,h.default)(t.orientation)&&"item"in t.orientation&&(this.options.orientation.item=t.orientation.item)),"margin"in t&&("number"==typeof t.margin?(this.options.margin.axis=t.margin,this.options.margin.item.horizontal=t.margin,this.options.margin.item.vertical=t.margin):"object"===(0,h.default)(t.margin)&&(l.selectiveExtend(["axis"],this.options.margin,t.margin),"item"in t.margin&&("number"==typeof t.margin.item?(this.options.margin.item.horizontal=t.margin.item,this.options.margin.item.vertical=t.margin.item):"object"===(0,h.default)(t.margin.item)&&l.selectiveExtend(["horizontal","vertical"],this.options.margin.item,t.margin.item)))),"editable"in t&&("boolean"==typeof t.editable?(this.options.editable.updateTime=t.editable,this.options.editable.updateGroup=t.editable,this.options.editable.add=t.editable,this.options.editable.remove=t.editable,this.options.editable.overrideItems=!1):"object"===(0,h.default)(t.editable)&&l.selectiveExtend(["updateTime","updateGroup","add","remove","overrideItems"],this.options.editable,t.editable)),"groupEditable"in t&&("boolean"==typeof t.groupEditable?(this.options.groupEditable.order=t.groupEditable,this.options.groupEditable.add=t.groupEditable,this.options.groupEditable.remove=t.groupEditable):"object"===(0,h.default)(t.groupEditable)&&l.selectiveExtend(["order","add","remove"],this.options.groupEditable,t.groupEditable));["onDropObjectOnItem","onAdd","onUpdate","onRemove","onMove","onMoving","onAddGroup","onMoveGroup","onRemoveGroup"].forEach(function(e){var i=t[e];if(i){if(!(i instanceof Function))throw new Error("option "+e+" must be a function "+e+"(item, callback)");this.options[e]=i}}.bind(this)),this.markDirty()}},n.prototype.markDirty=function(t){this.groupIds=[],t&&t.refreshItems&&l.forEach(this.items,function(t){t.dirty=!0,t.displayed&&t.redraw()})},n.prototype.destroy=function(){this.hide(),this.setItems(null),this.setGroups(null),this.hammer=null,this.body=null,this.conversion=null},n.prototype.hide=function(){this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame),this.dom.axis.parentNode&&this.dom.axis.parentNode.removeChild(this.dom.axis),this.dom.labelSet.parentNode&&this.dom.labelSet.parentNode.removeChild(this.dom.labelSet)},n.prototype.show=function(){this.dom.frame.parentNode||this.body.dom.center.appendChild(this.dom.frame),this.dom.axis.parentNode||this.body.dom.backgroundVertical.appendChild(this.dom.axis),this.dom.labelSet.parentNode||(this.options.rtl?this.body.dom.right.appendChild(this.dom.labelSet):this.body.dom.left.appendChild(this.dom.labelSet))},n.prototype.setSelection=function(t){var e,i,o,n;for(void 0==t&&(t=[]),Array.isArray(t)||(t=[t]),e=0,i=this.selection.length;e<i;e++)o=this.selection[e],(n=this.items[o])&&n.unselect();for(this.selection=[],e=0,i=t.length;e<i;e++)o=t[e],(n=this.items[o])&&(this.selection.push(o),n.select())},n.prototype.getSelection=function(){return this.selection.concat([])},n.prototype.getVisibleItems=function(){var t,e,i=this.body.range.getRange();this.options.rtl?(t=this.body.util.toScreen(i.start),e=this.body.util.toScreen(i.end)):(e=this.body.util.toScreen(i.start),t=this.body.util.toScreen(i.end));var o=[];for(var n in this.groups)if(this.groups.hasOwnProperty(n))for(var s=this.groups[n],r=s.isVisible?s.visibleItems:[],a=0;a<r.length;a++){var h=r[a];this.options.rtl?h.right<e&&h.right+h.width>t&&o.push(h.id):h.left<t&&h.left+h.width>e&&o.push(h.id)}return o},n.prototype._deselect=function(t){for(var e=this.selection,i=0,o=e.length;i<o;i++)if(e[i]==t){e.splice(i,1);break}},n.prototype.redraw=function(){var t=this.options.margin,e=this.body.range,i=l.option.asSize,o=this.options,n=o.orientation.item,s=!1,r=this.dom.frame;this.props.top=this.body.domProps.top.height+this.body.domProps.border.top,this.options.rtl?this.props.right=this.body.domProps.right.width+this.body.domProps.border.right:this.props.left=this.body.domProps.left.width+this.body.domProps.border.left,r.className="vis-itemset",s=this._orderGroups()||s;var a=e.end-e.start,h=a!=this.lastVisibleInterval||this.props.width!=this.props.lastWidth,d=e.start!=this.lastRangeStart,u=o.stack!=this.lastStack,p=o.stackSubgroups!=this.lastStackSubgroups,c=h||d||u||p;this.lastVisibleInterval=a,this.lastRangeStart=e.start,this.lastStack=o.stack,this.lastStackSubgroups=o.stackSubgroups,this.props.lastWidth=this.props.width;var m=this._firstGroup(),f={item:t.item,axis:t.axis},g={item:t.item,axis:t.item.vertical/2},v=0,y=t.axis+t.item.vertical;this.groups[D].redraw(e,g,c);var b={},_=0;if(l.forEach(this.groups,function(t,i){if(i!==D){var o=t==m?f:g;b[i]=t.redraw(e,o,c,!0),_=b[i].length}}),_>0){for(var w={},x=0;x<_;x++)l.forEach(b,function(t,e){w[e]=t[x]()});l.forEach(this.groups,function(t,e){if(e!==D){var i=w[e];s=i||s,v+=t.height}}),v=Math.max(v,y)}return v=Math.max(v,y),r.style.height=i(v),this.props.width=r.offsetWidth,this.props.height=v,this.dom.axis.style.top=i("top"==n?this.body.domProps.top.height+this.body.domProps.border.top:this.body.domProps.top.height+this.body.domProps.centerContainer.height),this.options.rtl?this.dom.axis.style.right="0":this.dom.axis.style.left="0",this.initialItemSetDrawn=!0,s=this._isResized()||s},n.prototype._firstGroup=function(){var t="top"==this.options.orientation.item?0:this.groupIds.length-1,e=this.groupIds[t];return this.groups[e]||this.groups[x]||null},n.prototype._updateUngrouped=function(){var t,e,i=this.groups[x];if(this.groupsData){if(i){i.hide(),delete this.groups[x];for(e in this.items)if(this.items.hasOwnProperty(e)){t=this.items[e],t.parent&&t.parent.remove(t);var o=this._getGroupId(t.data),n=this.groups[o];n&&n.add(t)||t.hide()}}}else if(!i){i=new f(null,null,this),this.groups[x]=i;for(e in this.items)this.items.hasOwnProperty(e)&&(t=this.items[e],i.add(t));i.show()}},n.prototype.getLabelSet=function(){return this.dom.labelSet},n.prototype.setItems=function(t){var e,i=this,o=this.itemsData;if(t){if(!(t instanceof u||t instanceof p))throw new TypeError("Data must be an instance of DataSet or DataView");this.itemsData=t}else this.itemsData=null;if(o&&(l.forEach(this.itemListeners,function(t,e){o.off(e,t)}),e=o.getIds(),this._onRemove(e)),this.itemsData){var n=this.id;l.forEach(this.itemListeners,function(t,e){i.itemsData.on(e,t,n)}),e=this.itemsData.getIds(),this._onAdd(e),this._updateUngrouped()}this.body.emitter.emit("_change",{queue:!0})},n.prototype.getItems=function(){return this.itemsData},n.prototype.setGroups=function(t){var e,i=this;if(this.groupsData&&(l.forEach(this.groupListeners,function(t,e){i.groupsData.off(e,t)}),e=this.groupsData.getIds(),this.groupsData=null,this._onRemoveGroups(e)),t){if(!(t instanceof u||t instanceof p))throw new TypeError("Data must be an instance of DataSet or DataView");this.groupsData=t}else this.groupsData=null;if(this.groupsData){var o=this.groupsData;this.groupsData instanceof p&&(o=this.groupsData.getDataSet()),o.get().forEach(function(t){t.nestedGroups&&t.nestedGroups.forEach(function(e){var i=o.get(e);i.nestedInGroup=t.id,0==t.showNested&&(i.visible=!1),o.update(i)})});var n=this.id;l.forEach(this.groupListeners,function(t,e){i.groupsData.on(e,t,n)}),e=this.groupsData.getIds(),this._onAddGroups(e)}this._updateUngrouped(),this._order(),this.body.emitter.emit("_change",{queue:!0})},n.prototype.getGroups=function(){return this.groupsData},n.prototype.removeItem=function(t){var e=this.itemsData.get(t),i=this.itemsData.getDataSet();e&&this.options.onRemove(e,function(e){e&&i.remove(t)})},n.prototype._getType=function(t){return t.type||this.options.type||(t.end?"range":"box")},n.prototype._getGroupId=function(t){return"background"==this._getType(t)&&void 0==t.group?D:this.groupsData?t.group:x},n.prototype._onUpdate=function(t){var e=this;t.forEach(function(t){var i,o=e.itemsData.get(t,e.itemOptions),s=e.items[t],r=o?e._getType(o):null,a=n.types[r];if(s&&(a&&s instanceof a?e._updateItem(s,o):(i=s.selected,e._removeItem(s),s=null)),!s&&o){if(!a)throw"rangeoverflow"==r?new TypeError('Item type "rangeoverflow" is deprecated. Use css styling instead: .vis-item.vis-range .vis-item-content {overflow: visible;}'):new TypeError('Unknown item type "'+r+'"');s=new a(o,e.conversion,e.options),s.id=t,e._addItem(s),i&&(this.selection.push(t),s.select())}}.bind(this)),this._order(),this.body.emitter.emit("_change",{queue:!0})},n.prototype._onAdd=n.prototype._onUpdate,n.prototype._onRemove=function(t){var e=0,i=this;t.forEach(function(t){var o=i.items[t];o&&(e++,i._removeItem(o))}),e&&(this._order(),this.body.emitter.emit("_change",{queue:!0}))},n.prototype._order=function(){l.forEach(this.groups,function(t){t.order()})},n.prototype._onUpdateGroups=function(t){this._onAddGroups(t)},n.prototype._onAddGroups=function(t){var e=this;t.forEach(function(t){var i=e.groupsData.get(t),o=e.groups[t];if(o)o.setData(i);else{if(t==x||t==D)throw new Error("Illegal group id. "+t+" is a reserved id.");var n=(0,r.default)(e.options);l.extend(n,{height:null}),o=new f(t,i,e),e.groups[t]=o;for(var s in e.items)if(e.items.hasOwnProperty(s)){var a=e.items[s];a.data.group==t&&o.add(a)}o.order(),o.show()}}),this.body.emitter.emit("_change",{queue:!0})},n.prototype._onRemoveGroups=function(t){var e=this.groups;t.forEach(function(t){var i=e[t];i&&(i.hide(),delete e[t])}),this.markDirty(),this.body.emitter.emit("_change",{queue:!0})},n.prototype._orderGroups=function(){if(this.groupsData){var t=this.groupsData.getIds({order:this.options.groupOrder});t=this._orderNestedGroups(t);var e=!l.equalArray(t,this.groupIds);if(e){var i=this.groups;t.forEach(function(t){i[t].hide()}),t.forEach(function(t){i[t].show()}),this.groupIds=t}return e}return!1},n.prototype._orderNestedGroups=function(t){var e=[];return t.forEach(function(t){var i=this.groupsData.get(t);if(i.nestedInGroup||e.push(t),i.nestedGroups){var o=this.groupsData.get({filter:function(e){return e.nestedInGroup==t},order:this.options.groupOrder}),n=o.map(function(t){return t.id});e=e.concat(n)}},this),e},n.prototype._addItem=function(t){this.items[t.id]=t;var e=this._getGroupId(t.data),i=this.groups[e];i?i&&i.data&&i.data.showNested&&(t.groupShowing=!0):t.groupShowing=!1,i&&i.add(t)},n.prototype._updateItem=function(t,e){t.setData(e);var i=this._getGroupId(t.data),o=this.groups[i];o?o&&o.data&&o.data.showNested&&(t.groupShowing=!0):t.groupShowing=!1},n.prototype._removeItem=function(t){t.hide(),delete this.items[t.id];var e=this.selection.indexOf(t.id);-1!=e&&this.selection.splice(e,1),t.parent&&t.parent.remove(t)},n.prototype._constructByEndArray=function(t){for(var e=[],i=0;i<t.length;i++)t[i]instanceof b&&e.push(t[i]);return e},n.prototype._onTouch=function(t){this.touchParams.item=this.itemFromTarget(t),this.touchParams.dragLeftItem=t.target.dragLeftItem||!1,this.touchParams.dragRightItem=t.target.dragRightItem||!1,this.touchParams.itemProps=null},n.prototype._getGroupIndex=function(t){for(var e=0;e<this.groupIds.length;e++)if(t==this.groupIds[e])return e},n.prototype._onDragStart=function(t){if(!this.touchParams.itemIsDragging){var e,i=this.touchParams.item||null,o=this;if(i&&(i.selected||this.options.itemsAlwaysDraggable.item)){if(this.options.editable.overrideItems&&!this.options.editable.updateTime&&!this.options.editable.updateGroup)return;if(null!=i.editable&&!i.editable.updateTime&&!i.editable.updateGroup&&!this.options.editable.overrideItems)return;var n=this.touchParams.dragLeftItem,s=this.touchParams.dragRightItem;if(this.touchParams.itemIsDragging=!0,this.touchParams.selectedItem=i,n)e={item:n,initialX:t.center.x,dragLeft:!0,data:this._cloneItemData(i.data)},this.touchParams.itemProps=[e];else if(s)e={item:s,initialX:t.center.x,dragRight:!0,data:this._cloneItemData(i.data)},this.touchParams.itemProps=[e];else if(this.options.editable.add&&(t.srcEvent.ctrlKey||t.srcEvent.metaKey))this._onDragStartAddItem(t);else{this.groupIds.length<1&&this.redraw();var r=this._getGroupIndex(i.data.group),a=this.options.itemsAlwaysDraggable.item&&!i.selected?[i.id]:this.getSelection();this.touchParams.itemProps=a.map(function(e){var i=o.items[e],n=o._getGroupIndex(i.data.group);return{item:i,initialX:t.center.x,groupOffset:r-n,data:this._cloneItemData(i.data)}}.bind(this))}t.stopPropagation()}else this.options.editable.add&&(t.srcEvent.ctrlKey||t.srcEvent.metaKey)&&this._onDragStartAddItem(t)}},n.prototype._onDragStartAddItem=function(t){var e,i,o=this.options.snap||null;this.options.rtl?(e=l.getAbsoluteRight(this.dom.frame),i=e-t.center.x+10):(e=l.getAbsoluteLeft(this.dom.frame),i=t.center.x-e-10);var n=this.body.util.toTime(i),s=this.body.util.getScale(),r=this.body.util.getStep(),a=o?o(n,s,r):n,h=a,d={type:"range",start:a,end:h,content:"new item"},u=l.randomUUID();d[this.itemsData._fieldId]=u;var p=this.groupFromTarget(t);p&&(d.group=p.groupId);var c=new b(d,this.conversion,this.options);c.id=u,c.data=this._cloneItemData(d),this._addItem(c),this.touchParams.selectedItem=c;var m={item:c,initialX:t.center.x,data:c.data};this.options.rtl?m.dragLeft=!0:m.dragRight=!0,this.touchParams.itemProps=[m],t.stopPropagation()},n.prototype._onDrag=function(t){if(this.touchParams.itemProps){t.stopPropagation();var e,i=this,o=this.options.snap||null;e=this.options.rtl?this.body.dom.root.offsetLeft+this.body.domProps.right.width:this.body.dom.root.offsetLeft+this.body.domProps.left.width;var n=this.body.util.getScale(),s=this.body.util.getStep(),r=this.touchParams.selectedItem,a=(this.options.editable.overrideItems||null==r.editable)&&this.options.editable.updateGroup||!this.options.editable.overrideItems&&null!=r.editable&&r.editable.updateGroup,h=null;if(a&&r&&void 0!=r.data.group){var d=i.groupFromTarget(t);d&&(h=this._getGroupIndex(d.groupId))}this.touchParams.itemProps.forEach(function(d){var u,p,c,m,f,g=i.body.util.toTime(t.center.x-e),v=i.body.util.toTime(d.initialX-e);u=this.options.rtl?-(g-v):g-v;var y=this._cloneItemData(d.item.data);if(null==d.item.editable||d.item.editable.updateTime||d.item.editable.updateGroup||i.options.editable.overrideItems){if((this.options.editable.overrideItems||null==r.editable)&&this.options.editable.updateTime||!this.options.editable.overrideItems&&null!=r.editable&&r.editable.updateTime)if(d.dragLeft)this.options.rtl?void 0!=y.end&&(c=l.convert(d.data.end,"Date"),f=new Date(c.valueOf()+u),y.end=o?o(f,n,s):f):void 0!=y.start&&(p=l.convert(d.data.start,"Date"),m=new Date(p.valueOf()+u),y.start=o?o(m,n,s):m);else if(d.dragRight)this.options.rtl?void 0!=y.start&&(p=l.convert(d.data.start,"Date"),m=new Date(p.valueOf()+u),y.start=o?o(m,n,s):m):void 0!=y.end&&(c=l.convert(d.data.end,"Date"),f=new Date(c.valueOf()+u),y.end=o?o(f,n,s):f);else if(void 0!=y.start)if(p=l.convert(d.data.start,"Date").valueOf(),m=new Date(p+u),void 0!=y.end){c=l.convert(d.data.end,"Date");var b=c.valueOf()-p.valueOf();y.start=o?o(m,n,s):m,y.end=new Date(y.start.valueOf()+b)}else y.start=o?o(m,n,s):m;if(a&&!d.dragLeft&&!d.dragRight&&null!=h&&void 0!=y.group){var _=h-d.groupOffset;_=Math.max(0,_),_=Math.min(i.groupIds.length-1,_),y.group=i.groupIds[_]}y=this._cloneItemData(y),i.options.onMoving(y,function(t){t&&d.item.setData(this._cloneItemData(t,"Date"))}.bind(this))}}.bind(this)),this.body.emitter.emit("_change")}},n.prototype._moveToGroup=function(t,e){var i=this.groups[e];if(i&&i.groupId!=t.data.group){var o=t.parent;o.remove(t),o.order(),t.data.group=i.groupId,i.add(t),i.order()}},n.prototype._onDragEnd=function(t){if(this.touchParams.itemIsDragging=!1,this.touchParams.itemProps){t.stopPropagation();var e=this,i=this.itemsData.getDataSet(),o=this.touchParams.itemProps;this.touchParams.itemProps=null,o.forEach(function(t){var o=t.item.id;if(null!=e.itemsData.get(o,e.itemOptions)){var n=this._cloneItemData(t.item.data);e.options.onMove(n,function(n){n?(n[i._fieldId]=o,i.update(n)):(t.item.setData(t.data),e.body.emitter.emit("_change"))})}else e.options.onAdd(t.item.data,function(i){e._removeItem(t.item),i&&e.itemsData.getDataSet().add(i),e.body.emitter.emit("_change")})}.bind(this))}},n.prototype._onGroupClick=function(t){var e=this.groupFromTarget(t);if(e&&e.nestedGroups){var i=this.groupsData.getDataSet(),o=i.get(e.groupId);void 0==o.showNested&&(o.showNested=!0),o.showNested=!o.showNested;var n=i.get(e.nestedGroups).map(function(t){return t.visible=o.showNested,t});if(i.update(n.concat(o)),o.showNested)l.removeClassName(e.dom.label,"collapsed"),l.addClassName(e.dom.label,"expanded");else{l.removeClassName(e.dom.label,"expanded");var s=this.options.rtl?"collapsed-rtl":"collapsed";l.addClassName(e.dom.label,s)}}},n.prototype._onGroupDragStart=function(t){this.options.groupEditable.order&&(this.groupTouchParams.group=this.groupFromTarget(t),this.groupTouchParams.group&&(t.stopPropagation(),this.groupTouchParams.originalOrder=this.groupsData.getIds({order:this.options.groupOrder})))},n.prototype._onGroupDrag=function(t){if(this.options.groupEditable.order&&this.groupTouchParams.group){t.stopPropagation();var e=this.groupsData;this.groupsData instanceof p&&(e=this.groupsData.getDataSet());var i=this.groupFromTarget(t);if(i&&i.height!=this.groupTouchParams.group.height){var o=i.top<this.groupTouchParams.group.top,n=t.center?t.center.y:t.clientY,s=l.getAbsoluteTop(i.dom.foreground),r=this.groupTouchParams.group.height;if(o){if(s+r<n)return}else{if(s+i.height-r>n)return}}if(i&&i!=this.groupTouchParams.group){var a=e.get(i.groupId),h=e.get(this.groupTouchParams.group.groupId);h&&a&&(this.options.groupOrderSwap(h,a,e),e.update(h),e.update(a));var d=e.getIds({order:this.options.groupOrder});if(!l.equalArray(d,this.groupTouchParams.originalOrder))for(var u=this.groupTouchParams.originalOrder,c=this.groupTouchParams.group.groupId,m=Math.min(u.length,d.length),f=0,g=0,v=0;f<m;){for(;f+g<m&&f+v<m&&d[f+g]==u[f+v];)f++;if(f+g>=m)break;if(d[f+g]==c)g=1;else if(u[f+v]==c)v=1;else{var y=d.indexOf(u[f+v]),b=e.get(d[f+g]),_=e.get(u[f+v]);this.options.groupOrderSwap(b,_,e),e.update(b),e.update(_);var w=d[f+g];d[f+g]=u[f+v],d[y]=w,f++}}}}},n.prototype._onGroupDragEnd=function(t){if(this.options.groupEditable.order&&this.groupTouchParams.group){t.stopPropagation();var e=this,i=e.groupTouchParams.group.groupId,o=e.groupsData.getDataSet(),n=l.extend({},o.get(i));e.options.onMoveGroup(n,function(t){if(t)t[o._fieldId]=i,o.update(t);else{var n=o.getIds({order:e.options.groupOrder});if(!l.equalArray(n,e.groupTouchParams.originalOrder))for(var s=e.groupTouchParams.originalOrder,r=Math.min(s.length,n.length),a=0;a<r;){for(;a<r&&n[a]==s[a];)a++;if(a>=r)break;var h=n.indexOf(s[a]),d=o.get(n[a]),u=o.get(s[a]);e.options.groupOrderSwap(d,u,o),o.update(d),o.update(u);var p=n[a];n[a]=s[a],n[h]=p,a++}}}),e.body.emitter.emit("groupDragged",{groupId:i})}},n.prototype._onSelectItem=function(t){if(this.options.selectable){var e=t.srcEvent&&(t.srcEvent.ctrlKey||t.srcEvent.metaKey),i=t.srcEvent&&t.srcEvent.shiftKey;if(e||i)return void this._onMultiSelectItem(t);var o=this.getSelection(),n=this.itemFromTarget(t),s=n?[n.id]:[];this.setSelection(s);var r=this.getSelection();(r.length>0||o.length>0)&&this.body.emitter.emit("select",{items:r,event:t})}},n.prototype._onMouseOver=function(t){var e=this.itemFromTarget(t);if(e){if(e!==this.itemFromRelatedTarget(t)){var i=e.getTitle();if(this.options.showTooltips&&i){null==this.popup&&(this.popup=new w(this.body.dom.root,this.options.tooltip.overflowMethod||"flip")),this.popup.setText(i);var o=this.body.dom.centerContainer;this.popup.setPosition(t.clientX-l.getAbsoluteLeft(o)+o.offsetLeft,t.clientY-l.getAbsoluteTop(o)+o.offsetTop),this.popup.show()}else null!=this.popup&&this.popup.hide();this.body.emitter.emit("itemover",{item:e.id,event:t})}}},n.prototype._onMouseOut=function(t){var e=this.itemFromTarget(t);if(e){e!==this.itemFromRelatedTarget(t)&&(null!=this.popup&&this.popup.hide(),this.body.emitter.emit("itemout",{item:e.id,event:t}))}},n.prototype._onMouseMove=function(t){if(this.itemFromTarget(t)&&this.options.showTooltips&&this.options.tooltip.followMouse&&this.popup&&!this.popup.hidden){var e=this.body.dom.centerContainer;this.popup.setPosition(t.clientX-l.getAbsoluteLeft(e)+e.offsetLeft,t.clientY-l.getAbsoluteTop(e)+e.offsetTop),this.popup.show()}},n.prototype._onMouseWheel=function(t){this.touchParams.itemIsDragging&&this._onDragEnd(t)},n.prototype._onUpdateItem=function(t){if(this.options.selectable&&this.options.editable.add){var e=this;if(t){var i=e.itemsData.get(t.id);this.options.onUpdate(i,function(t){t&&e.itemsData.getDataSet().update(t)})}}},n.prototype._onDropObjectOnItem=function(t){var e=this.itemFromTarget(t),i=JSON.parse(t.dataTransfer.getData("text"));this.options.onDropObjectOnItem(i,e)},n.prototype._onAddItem=function(t){if(this.options.selectable&&this.options.editable.add){var e,i,o=this,n=this.options.snap||null;this.options.rtl?(e=l.getAbsoluteRight(this.dom.frame),i=e-t.center.x):(e=l.getAbsoluteLeft(this.dom.frame),i=t.center.x-e);var s,r,a=this.body.util.toTime(i),h=this.body.util.getScale(),d=this.body.util.getStep();"drop"==t.type?(r=JSON.parse(t.dataTransfer.getData("text")),r.content=r.content?r.content:"new item",r.start=r.start?r.start:n?n(a,h,d):a,r.type=r.type||"box",r[this.itemsData._fieldId]=r.id||l.randomUUID(),"range"!=r.type||r.end||(s=this.body.util.toTime(i+this.props.width/5),r.end=n?n(s,h,d):s)):(r={start:n?n(a,h,d):a,content:"new item"},r[this.itemsData._fieldId]=l.randomUUID(),"range"===this.options.type&&(s=this.body.util.toTime(i+this.props.width/5),r.end=n?n(s,h,d):s));var u=this.groupFromTarget(t);u&&(r.group=u.groupId),r=this._cloneItemData(r),this.options.onAdd(r,function(e){e&&(o.itemsData.getDataSet().add(e),"drop"==t.type&&o.setSelection([e.id]))})}},n.prototype._onMultiSelectItem=function(t){if(this.options.selectable){var e=this.itemFromTarget(t);if(e){var i=this.options.multiselect?this.getSelection():[];if((t.srcEvent&&t.srcEvent.shiftKey||!1)&&this.options.multiselect){var o=this.itemsData.get(e.id).group,s=void 0;this.options.multiselectPerGroup&&i.length>0&&(s=this.itemsData.get(i[0]).group),this.options.multiselectPerGroup&&void 0!=s&&s!=o||i.push(e.id);var r=n._getItemRange(this.itemsData.get(i,this.itemOptions));if(!this.options.multiselectPerGroup||s==o){i=[];for(var a in this.items)if(this.items.hasOwnProperty(a)){var h=this.items[a],d=h.data.start,l=void 0!==h.data.end?h.data.end:d;!(d>=r.min&&l<=r.max)||this.options.multiselectPerGroup&&s!=this.itemsData.get(h.id).group||h instanceof _||i.push(h.id)}}}else{var u=i.indexOf(e.id);-1==u?i.push(e.id):i.splice(u,1)}this.setSelection(i),this.body.emitter.emit("select",{items:this.getSelection(),event:t})}}},n._getItemRange=function(t){var e=null,i=null;return t.forEach(function(t){(null==i||t.start<i)&&(i=t.start),void 0!=t.end?(null==e||t.end>e)&&(e=t.end):(null==e||t.start>e)&&(e=t.start)}),{min:i,max:e}},n.prototype.itemFromElement=function(t){for(var e=t;e;){if(e.hasOwnProperty("timeline-item"))return e["timeline-item"];e=e.parentNode}return null},n.prototype.itemFromTarget=function(t){return this.itemFromElement(t.target)},n.prototype.itemFromRelatedTarget=function(t){return this.itemFromElement(t.relatedTarget)},n.prototype.groupFromTarget=function(t){var e=t.center?t.center.y:t.clientY,i=this.groupIds;i.length<=0&&this.groupsData&&(i=this.groupsData.getIds({order:this.options.groupOrder}));for(var o=0;o<i.length;o++){var n=i[o],s=this.groups[n],r=s.dom.foreground,a=l.getAbsoluteTop(r);if(e>a&&e<a+r.offsetHeight)return s;if("top"===this.options.orientation.item){if(o===this.groupIds.length-1&&e>a)return s}else if(0===o&&e<a+r.offset)return s}return null},n.itemSetFromTarget=function(t){for(var e=t.target;e;){if(e.hasOwnProperty("timeline-itemset"))return e["timeline-itemset"];e=e.parentNode}return null},n.prototype._cloneItemData=function(t,e){var i=l.extend({},t);return e||(e=this.itemsData.getDataSet()._options.type),void 0!=i.start&&(i.start=l.convert(i.start,e&&e.start||"Date")),void 0!=i.end&&(i.end=l.convert(i.end,e&&e.end||"Date")),i},t.exports=n},function(t,e,i){e.orderByStart=function(t){t.sort(function(t,e){return t.data.start-e.data.start})},e.orderByEnd=function(t){t.sort(function(t,e){return("end"in t.data?t.data.end:t.data.start)-("end"in e.data?e.data.end:e.data.start)})},e.stack=function(t,i,o){if(o)for(var n=0;n<t.length;n++)t[n].top=null;for(var n=0;n<t.length;n++){var s=t[n];if(s.stack&&null===s.top){s.top=i.axis;do{for(var r=null,a=0,h=t.length;a<h;a++){var d=t[a];if(null!==d.top&&d!==s&&d.stack&&e.collision(s,d,i.item,d.options.rtl)){r=d;break}}null!=r&&(s.top=r.top+r.height+i.item.vertical)}while(r)}}},e.substack=function(t,i,o){for(var n=0;n<t.length;n++)t[n].top=null;var s=o.height;for(n=0;n<t.length;n++){var r=t[n];if(r.stack&&null===r.top){r.top=r.baseTop;do{for(var a=null,h=0,d=t.length;h<d;h++){var l=t[h];if(null!==l.top&&l!==r&&e.collision(r,l,i.item,l.options.rtl)){a=l;break}}null!=a&&(r.top=a.top+a.height+i.item.vertical),r.top+r.height>s&&(s=r.top+r.height)}while(a)}}o.height=s-o.top+.5*i.item.vertical},e.nostack=function(t,i,o,n){for(var s=0;s<t.length;s++)if(void 0==t[s].data.subgroup)t[s].top=i.item.vertical;else if(void 0!==t[s].data.subgroup&&n){var r=0;for(var a in o)o.hasOwnProperty(a)&&1==o[a].visible&&o[a].index<o[t[s].data.subgroup].index&&(r+=o[a].height,o[t[s].data.subgroup].top=r);t[s].top=r+.5*i.item.vertical}n||e.stackSubgroups(t,i,o)},e.stackSubgroups=function(t,i,o){for(var n in o)if(o.hasOwnProperty(n)){o[n].top=0;do{var s=null;for(var r in o)if(null!==o[r].top&&r!==n&&o[n].index>o[r].index&&e.collisionByTimes(o[n],o[r])){s=o[r];break}null!=s&&(o[n].top=s.top+s.height)}while(s)}for(var a=0;a<t.length;a++)void 0!==t[a].data.subgroup&&(t[a].top=o[t[a].data.subgroup].top+.5*i.item.vertical)},e.stackSubgroupsWithInnerStack=function(t,i,o){var n=!1,s=[];for(var r in o)o[r].hasOwnProperty("index")?s[o[r].index]=r:s.push(r);for(var a=0;a<s.length;a++)if(r=s[a],o.hasOwnProperty(r)){n=n||o[r].stack,o[r].top=0;for(var h in o)o[h].visible&&o[r].index>o[h].index&&(o[r].top+=o[h].height);for(var d=t[r],l=0;l<d.length;l++)void 0!==d[l].data.subgroup&&(d[l].top=o[d[l].data.subgroup].top+.5*i.item.vertical,o[r].stack&&(d[l].baseTop=d[l].top));n&&o[r].stack&&e.substack(t[r],i,o[r])}},e.collision=function(t,e,i,o){return o?t.right-i.horizontal+.001<e.right+e.width&&t.right+t.width+i.horizontal-.001>e.right&&t.top-i.vertical+.001<e.top+e.height&&t.top+t.height+i.vertical-.001>e.top:t.left-i.horizontal+.001<e.left+e.width&&t.left+t.width+i.horizontal-.001>e.left&&t.top-i.vertical+.001<e.top+e.height&&t.top+t.height+i.vertical-.001>e.top},e.collisionByTimes=function(t,e){return t.start<=e.start&&t.end>=e.start&&t.top<e.top+e.height&&t.top+t.height>e.top||e.start<=t.start&&e.end>=t.start&&e.top<t.top+t.height&&e.top+e.height>t.top}},function(t,e,i){function o(t,e,i){if(this.props={dot:{width:0,height:0},line:{width:0,height:0}},this.options=i,t&&void 0==t.start)throw new Error('Property "start" missing in item '+t);n.call(this,t,e,i)}var n=i(22);o.prototype=new n(null,null,null),o.prototype.isVisible=function(t){var e=this.options.align,i=this.width*t.getMillisecondsPerPixel();return"right"==e?this.data.start.getTime()>t.start&&this.data.start.getTime()-i<t.end:"left"==e?this.data.start.getTime()+i>t.start&&this.data.start.getTime()<t.end:this.data.start.getTime()+i/2>t.start&&this.data.start.getTime()-i/2<t.end},o.prototype._createDomElement=function(){this.dom||(this.dom={},this.dom.box=document.createElement("DIV"),this.dom.content=document.createElement("DIV"),this.dom.content.className="vis-item-content",this.dom.box.appendChild(this.dom.content),this.dom.line=document.createElement("DIV"),this.dom.line.className="vis-line",this.dom.dot=document.createElement("DIV"),this.dom.dot.className="vis-dot",this.dom.box["timeline-item"]=this,this.dirty=!0)},o.prototype._appendDomElement=function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.box.parentNode){var t=this.parent.dom.foreground;if(!t)throw new Error("Cannot redraw item: parent has no foreground container element");t.appendChild(this.dom.box)}if(!this.dom.line.parentNode){var e=this.parent.dom.background;if(!e)throw new Error("Cannot redraw item: parent has no background container element");e.appendChild(this.dom.line)}if(!this.dom.dot.parentNode){var i=this.parent.dom.axis;if(!e)throw new Error("Cannot redraw item: parent has no axis container element");i.appendChild(this.dom.dot)}this.displayed=!0},o.prototype._updateDirtyDomComponents=function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.box),this._updateStyle(this.dom.box);var t=this.editable.updateTime||this.editable.updateGroup,e=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+(t?" vis-editable":" vis-readonly");this.dom.box.className="vis-item vis-box"+e,this.dom.line.className="vis-item vis-line"+e,this.dom.dot.className="vis-item vis-dot"+e}},o.prototype._getDomComponentsSizes=function(){return{previous:{right:this.dom.box.style.right,left:this.dom.box.style.left},dot:{height:this.dom.dot.offsetHeight,width:this.dom.dot.offsetWidth},line:{width:this.dom.line.offsetWidth},box:{width:this.dom.box.offsetWidth,height:this.dom.box.offsetHeight}}},o.prototype._updateDomComponentsSizes=function(t){this.options.rtl?this.dom.box.style.right="0px":this.dom.box.style.left="0px",this.props.dot.height=t.dot.height,this.props.dot.width=t.dot.width,this.props.line.width=t.line.width,this.width=t.box.width,this.height=t.box.height,this.options.rtl?this.dom.box.style.right=t.previous.right:this.dom.box.style.left=t.previous.left,this.dirty=!1},o.prototype._repaintDomAdditionals=function(){this._repaintOnItemUpdateTimeTooltip(this.dom.box),this._repaintDragCenter(),this._repaintDeleteButton(this.dom.box)},o.prototype.redraw=function(t){var e,i=[this._createDomElement.bind(this),this._appendDomElement.bind(this),this._updateDirtyDomComponents.bind(this),function(){this.dirty&&(e=this._getDomComponentsSizes())}.bind(this),function(){this.dirty&&this._updateDomComponentsSizes.bind(this)(e)}.bind(this),this._repaintDomAdditionals.bind(this)];if(t)return i;var o;return i.forEach(function(t){o=t()}),o},o.prototype.show=function(){this.displayed||this.redraw()},o.prototype.hide=function(){if(this.displayed){var t=this.dom;t.box.parentNode&&t.box.parentNode.removeChild(t.box),t.line.parentNode&&t.line.parentNode.removeChild(t.line),t.dot.parentNode&&t.dot.parentNode.removeChild(t.dot),this.displayed=!1}},o.prototype.repositionX=function(){var t=this.conversion.toScreen(this.data.start),e=this.options.align;"right"==e?this.options.rtl?(this.right=t-this.width,this.dom.box.style.right=this.right+"px",this.dom.line.style.right=t-this.props.line.width+"px",this.dom.dot.style.right=t-this.props.line.width/2-this.props.dot.width/2+"px"):(this.left=t-this.width,this.dom.box.style.left=this.left+"px",this.dom.line.style.left=t-this.props.line.width+"px",this.dom.dot.style.left=t-this.props.line.width/2-this.props.dot.width/2+"px"):"left"==e?this.options.rtl?(this.right=t,this.dom.box.style.right=this.right+"px",this.dom.line.style.right=t+"px",this.dom.dot.style.right=t+this.props.line.width/2-this.props.dot.width/2+"px"):(this.left=t,this.dom.box.style.left=this.left+"px",this.dom.line.style.left=t+"px",this.dom.dot.style.left=t+this.props.line.width/2-this.props.dot.width/2+"px"):this.options.rtl?(this.right=t-this.width/2,this.dom.box.style.right=this.right+"px",this.dom.line.style.right=t-this.props.line.width+"px",this.dom.dot.style.right=t-this.props.dot.width/2+"px"):(this.left=t-this.width/2,this.dom.box.style.left=this.left+"px",this.dom.line.style.left=t-this.props.line.width/2+"px",this.dom.dot.style.left=t-this.props.dot.width/2+"px")},o.prototype.repositionY=function(){var t=this.options.orientation.item,e=this.dom.box,i=this.dom.line,o=this.dom.dot;if("top"==t)e.style.top=(this.top||0)+"px",i.style.top="0",i.style.height=this.parent.top+this.top+1+"px",i.style.bottom="";else{var n=this.parent.itemSet.props.height,s=n-this.parent.top-this.parent.height+this.top;e.style.top=(this.parent.height-this.top-this.height||0)+"px",i.style.top=n-s+"px",i.style.bottom="0"}o.style.top=-this.props.dot.height/2+"px"},o.prototype.getWidthLeft=function(){return this.width/2},o.prototype.getWidthRight=function(){return this.width/2},t.exports=o},function(t,e,i){function o(t,e,i){if(this.props={dot:{top:0,width:0,height:0},content:{height:0,marginLeft:0,marginRight:0}},this.options=i,t&&void 0==t.start)throw new Error('Property "start" missing in item '+t);n.call(this,t,e,i)}var n=i(22);o.prototype=new n(null,null,null),o.prototype.isVisible=function(t){var e=this.width*t.getMillisecondsPerPixel();return this.data.start.getTime()+e>t.start&&this.data.start<t.end},o.prototype._createDomElement=function(){this.dom||(this.dom={},this.dom.point=document.createElement("div"),this.dom.content=document.createElement("div"),this.dom.content.className="vis-item-content",this.dom.point.appendChild(this.dom.content),this.dom.dot=document.createElement("div"),this.dom.point.appendChild(this.dom.dot),this.dom.point["timeline-item"]=this,this.dirty=!0)},o.prototype._appendDomElement=function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.point.parentNode){var t=this.parent.dom.foreground;if(!t)throw new Error("Cannot redraw item: parent has no foreground container element");t.appendChild(this.dom.point)}this.displayed=!0},o.prototype._updateDirtyDomComponents=function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.point),this._updateStyle(this.dom.point);var t=this.editable.updateTime||this.editable.updateGroup,e=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+(t?" vis-editable":" vis-readonly");this.dom.point.className="vis-item vis-point"+e,this.dom.dot.className="vis-item vis-dot"+e}},o.prototype._getDomComponentsSizes=function(){return{dot:{width:this.dom.dot.offsetWidth,height:this.dom.dot.offsetHeight},content:{width:this.dom.content.offsetWidth,height:this.dom.content.offsetHeight},point:{width:this.dom.point.offsetWidth,height:this.dom.point.offsetHeight}}},o.prototype._updateDomComponentsSizes=function(t){this.props.dot.width=t.dot.width,this.props.dot.height=t.dot.height,this.props.content.height=t.content.height,this.options.rtl?this.dom.content.style.marginRight=2*this.props.dot.width+"px":this.dom.content.style.marginLeft=2*this.props.dot.width+"px",this.width=t.point.width,this.height=t.point.height,this.dom.dot.style.top=(this.height-this.props.dot.height)/2+"px",this.options.rtl?this.dom.dot.style.right=this.props.dot.width/2+"px":this.dom.dot.style.left=this.props.dot.width/2+"px",this.dirty=!1},o.prototype._repaintDomAdditionals=function(){this._repaintOnItemUpdateTimeTooltip(this.dom.point),this._repaintDragCenter(),this._repaintDeleteButton(this.dom.point)},o.prototype.redraw=function(t){var e,i=[this._createDomElement.bind(this),this._appendDomElement.bind(this),this._updateDirtyDomComponents.bind(this),function(){this.dirty&&(e=this._getDomComponentsSizes())}.bind(this),function(){this.dirty&&this._updateDomComponentsSizes.bind(this)(e)}.bind(this),this._repaintDomAdditionals.bind(this)];if(t)return i;var o;return i.forEach(function(t){o=t()}),o},o.prototype.show=function(){this.displayed||this.redraw()},o.prototype.hide=function(){this.displayed&&(this.dom.point.parentNode&&this.dom.point.parentNode.removeChild(this.dom.point),this.displayed=!1)},o.prototype.repositionX=function(){var t=this.conversion.toScreen(this.data.start);this.options.rtl?(this.right=t-this.props.dot.width,this.dom.point.style.right=this.right+"px"):(this.left=t-this.props.dot.width,this.dom.point.style.left=this.left+"px")},o.prototype.repositionY=function(){var t=this.options.orientation.item,e=this.dom.point;e.style.top="top"==t?this.top+"px":this.parent.height-this.top-this.height+"px"},o.prototype.getWidthLeft=function(){return this.props.dot.width},o.prototype.getWidthRight=function(){return this.props.dot.width},t.exports=o},function(t,e,i){function o(t,e,i){if(this.props={content:{width:0}},this.overflow=!1,t){if(void 0==t.start)throw new Error('Property "start" missing in item '+t.id);if(void 0==t.end)throw new Error('Property "end" missing in item '+t.id)}n.call(this,t,e,i)}var n=i(22),s=i(52),r=i(53);o.prototype=new n(null,null,null),o.prototype.baseClassName="vis-item vis-background",o.prototype.stack=!1,o.prototype.isVisible=function(t){return this.data.start<t.end&&this.data.end>t.start},o.prototype._createDomElement=function(){this.dom||(this.dom={},this.dom.box=document.createElement("div"),this.dom.frame=document.createElement("div"),this.dom.frame.className="vis-item-overflow",this.dom.box.appendChild(this.dom.frame),this.dom.content=document.createElement("div"),this.dom.content.className="vis-item-content",this.dom.frame.appendChild(this.dom.content),this.dirty=!0)},o.prototype._appendDomElement=function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.box.parentNode){var t=this.parent.dom.background;if(!t)throw new Error("Cannot redraw item: parent has no background container element");t.appendChild(this.dom.box)}this.displayed=!0},o.prototype._updateDirtyDomComponents=function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.content),this._updateStyle(this.dom.box);var t=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"");this.dom.box.className=this.baseClassName+t}},o.prototype._getDomComponentsSizes=function(){return this.overflow="hidden"!==window.getComputedStyle(this.dom.content).overflow,{content:{width:this.dom.content.offsetWidth}}},o.prototype._updateDomComponentsSizes=function(t){this.props.content.width=t.content.width,this.height=0,this.dirty=!1},o.prototype._repaintDomAdditionals=function(){},o.prototype.redraw=function(t){var e,i=[this._createDomElement.bind(this),this._appendDomElement.bind(this),this._updateDirtyDomComponents.bind(this),function(){this.dirty&&(e=this._getDomComponentsSizes.bind(this)())}.bind(this),function(){this.dirty&&this._updateDomComponentsSizes.bind(this)(e)}.bind(this),this._repaintDomAdditionals.bind(this)];if(t)return i;var o;return i.forEach(function(t){o=t()}),o},o.prototype.show=r.prototype.show,o.prototype.hide=r.prototype.hide,o.prototype.repositionX=r.prototype.repositionX,o.prototype.repositionY=function(t){var e,i=this.options.orientation.item;if(void 0!==this.data.subgroup){var o=this.data.subgroup;this.dom.box.style.height=this.parent.subgroups[o].height+"px",this.dom.box.style.top="top"==i?this.parent.top+this.parent.subgroups[o].top+"px":this.parent.top+this.parent.height-this.parent.subgroups[o].top-this.parent.subgroups[o].height+"px",this.dom.box.style.bottom=""}else this.parent instanceof s?(e=Math.max(this.parent.height,this.parent.itemSet.body.domProps.center.height,this.parent.itemSet.body.domProps.centerContainer.height),this.dom.box.style.bottom="bottom"==i?"0":"",this.dom.box.style.top="top"==i?"0":""):(e=this.parent.height,this.dom.box.style.top=this.parent.top+"px",this.dom.box.style.bottom="");this.dom.box.style.height=e+"px"},t.exports=o},function(t,e,i){Object.defineProperty(e,"__esModule",{value:!0});var o="string",n="boolean",s="number",r="object",a={configure:{enabled:{boolean:n},filter:{boolean:n,function:"function"},container:{dom:"dom"},__type__:{object:r,boolean:n,function:"function"}},align:{string:o},rtl:{boolean:n,undefined:"undefined"},rollingMode:{follow:{boolean:n},offset:{number:s,undefined:"undefined"},__type__:{object:r}},verticalScroll:{boolean:n,undefined:"undefined"},horizontalScroll:{boolean:n,undefined:"undefined"},autoResize:{boolean:n},throttleRedraw:{number:s},clickToUse:{boolean:n},dataAttributes:{string:o,array:"array"},editable:{add:{boolean:n,undefined:"undefined"},remove:{boolean:n,undefined:"undefined"},updateGroup:{boolean:n,undefined:"undefined"},updateTime:{boolean:n,undefined:"undefined"},overrideItems:{boolean:n,undefined:"undefined"},__type__:{boolean:n,object:r}},end:{number:s,date:"date",string:o,moment:"moment"},format:{minorLabels:{millisecond:{string:o,undefined:"undefined"},second:{string:o,undefined:"undefined"},minute:{string:o,undefined:"undefined"},hour:{string:o,undefined:"undefined"},weekday:{string:o,undefined:"undefined"},day:{string:o,undefined:"undefined"},week:{string:o,undefined:"undefined"},month:{string:o,undefined:"undefined"},year:{string:o,undefined:"undefined"},__type__:{object:r,function:"function"}},majorLabels:{millisecond:{string:o,undefined:"undefined"},second:{string:o,undefined:"undefined"},minute:{string:o,undefined:"undefined"},hour:{string:o,undefined:"undefined"},weekday:{string:o,undefined:"undefined"},day:{string:o,undefined:"undefined"},week:{string:o,undefined:"undefined"},month:{string:o,undefined:"undefined"},year:{string:o,undefined:"undefined"},__type__:{object:r,function:"function"}},__type__:{object:r}},moment:{function:"function"},groupOrder:{string:o,function:"function"},groupEditable:{add:{boolean:n,undefined:"undefined"},remove:{boolean:n,undefined:"undefined"},order:{boolean:n,undefined:"undefined"},__type__:{boolean:n,object:r}},groupOrderSwap:{function:"function"},height:{string:o,number:s},hiddenDates:{start:{date:"date",number:s,string:o,moment:"moment"},end:{date:"date",number:s,string:o,moment:"moment"},repeat:{string:o},__type__:{object:r,array:"array"}},itemsAlwaysDraggable:{item:{boolean:n,undefined:"undefined"},range:{boolean:n,undefined:"undefined"},__type__:{boolean:n,object:r}},limitSize:{boolean:n},locale:{string:o},locales:{__any__:{any:"any"},__type__:{object:r}},margin:{axis:{number:s},item:{horizontal:{number:s,undefined:"undefined"},vertical:{number:s,undefined:"undefined"},__type__:{object:r,number:s}},__type__:{object:r,number:s}},max:{date:"date",number:s,string:o,moment:"moment"},maxHeight:{number:s,string:o},maxMinorChars:{number:s},min:{date:"date",number:s,string:o,moment:"moment"},minHeight:{number:s,string:o},moveable:{boolean:n},multiselect:{boolean:n},multiselectPerGroup:{boolean:n},onAdd:{function:"function"},onDropObjectOnItem:{function:"function"},onUpdate:{function:"function"},onMove:{function:"function"},onMoving:{function:"function"},onRemove:{function:"function"},onAddGroup:{function:"function"},onMoveGroup:{function:"function"},onRemoveGroup:{function:"function"},onInitialDrawComplete:{function:"function"},order:{function:"function"},orientation:{axis:{string:o,undefined:"undefined"},item:{string:o,undefined:"undefined"},__type__:{string:o,object:r}},selectable:{boolean:n},showCurrentTime:{boolean:n},showMajorLabels:{boolean:n},showMinorLabels:{boolean:n},stack:{boolean:n},stackSubgroups:{boolean:n},snap:{function:"function",null:"null"},start:{date:"date",number:s,string:o,moment:"moment"},template:{function:"function"},groupTemplate:{function:"function"},visibleFrameTemplate:{string:o,function:"function"},showTooltips:{boolean:n},tooltip:{followMouse:{boolean:n},overflowMethod:{string:["cap","flip"]},__type__:{object:r}},tooltipOnItemUpdateTime:{template:{function:"function"},__type__:{boolean:n,object:r}},timeAxis:{scale:{string:o,undefined:"undefined"},step:{number:s,undefined:"undefined"},__type__:{object:r}},type:{string:o},width:{string:o,number:s},zoomable:{boolean:n},zoomKey:{string:["ctrlKey","altKey","metaKey",""]},zoomMax:{number:s},zoomMin:{number:s},__type__:{object:r}},h={global:{align:["center","left","right"],direction:!1,autoResize:!0,clickToUse:!1,editable:{add:!1,remove:!1,updateGroup:!1,updateTime:!1},end:"",format:{minorLabels:{millisecond:"SSS",second:"s",minute:"HH:mm",hour:"HH:mm",weekday:"ddd D",day:"D",week:"w",month:"MMM",year:"YYYY"},majorLabels:{millisecond:"HH:mm:ss",second:"D MMMM HH:mm",minute:"ddd D MMMM",hour:"ddd D MMMM",weekday:"MMMM YYYY",day:"MMMM YYYY",week:"MMMM YYYY",month:"YYYY",year:""}},groupsDraggable:!1,height:"",locale:"",margin:{axis:[20,0,100,1],item:{horizontal:[10,0,100,1],vertical:[10,0,100,1]}},max:"",maxHeight:"",maxMinorChars:[7,0,20,1],min:"",minHeight:"",moveable:!1,multiselect:!1,multiselectPerGroup:!1,orientation:{axis:["both","bottom","top"],item:["bottom","top"]},selectable:!0,showCurrentTime:!1,showMajorLabels:!0,showMinorLabels:!0,stack:!0,stackSubgroups:!0,start:"",showTooltips:!0,tooltip:{followMouse:!1,overflowMethod:"flip"},tooltipOnItemUpdateTime:!1,type:["box","point","range","background"],width:"100%",zoomable:!0,zoomKey:["ctrlKey","altKey","metaKey",""],zoomMax:[31536e10,10,31536e10,1],zoomMin:[10,10,31536e10,1]}};e.allOptions=a,e.configureOptions=h},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}Object.defineProperty(e,"__esModule",{value:!0});var n=i(16),s=o(n),r=i(1),a=o(r),h=i(30),d=o(h),l=i(31),u=o(l),p=i(0),c=i(137).default,m=function(){function t(e,i,o){var n=arguments.length>3&&void 0!==arguments[3]?arguments[3]:1;(0,d.default)(this,t),this.parent=e,this.changedOptions=[],this.container=i,this.allowCreation=!1,this.options={},this.initialized=!1,this.popupCounter=0,this.defaultOptions={enabled:!1,filter:!0,container:void 0,showButton:!0},p.extend(this.options,this.defaultOptions),this.configureOptions=o,this.moduleOptions={},this.domElements=[],this.popupDiv={},this.popupLimit=5,this.popupHistory={},this.colorPicker=new c(n),this.wrapper=void 0}return(0,u.default)(t,[{key:"setOptions",value:function(t){if(void 0!==t){this.popupHistory={},this._removePopup();var e=!0;"string"==typeof t?this.options.filter=t:t instanceof Array?this.options.filter=t.join():"object"===(void 0===t?"undefined":(0,a.default)(t))?(void 0!==t.container&&(this.options.container=t.container),void 0!==t.filter&&(this.options.filter=t.filter),void 0!==t.showButton&&(this.options.showButton=t.showButton),void 0!==t.enabled&&(e=t.enabled)):"boolean"==typeof t?(this.options.filter=!0,e=t):"function"==typeof t&&(this.options.filter=t,e=!0),!1===this.options.filter&&(e=!1),this.options.enabled=e}this._clean()}},{key:"setModuleOptions",value:function(t){this.moduleOptions=t,!0===this.options.enabled&&(this._clean(),void 0!==this.options.container&&(this.container=this.options.container),this._create())}},{key:"_create",value:function(){var t=this;this._clean(),this.changedOptions=[];var e=this.options.filter,i=0,o=!1;for(var n in this.configureOptions)this.configureOptions.hasOwnProperty(n)&&(this.allowCreation=!1,o=!1,"function"==typeof e?(o=e(n,[]),o=o||this._handleObject(this.configureOptions[n],[n],!0)):!0!==e&&-1===e.indexOf(n)||(o=!0),!1!==o&&(this.allowCreation=!0,i>0&&this._makeItem([]),this._makeHeader(n),this._handleObject(this.configureOptions[n],[n])),i++);if(!0===this.options.showButton){var s=document.createElement("div");s.className="vis-configuration vis-config-button",s.innerHTML="generate options",s.onclick=function(){t._printOptions()},s.onmouseover=function(){s.className="vis-configuration vis-config-button hover"},s.onmouseout=function(){s.className="vis-configuration vis-config-button"},this.optionsContainer=document.createElement("div"),this.optionsContainer.className="vis-configuration vis-config-option-container",this.domElements.push(this.optionsContainer),this.domElements.push(s)}this._push()}},{key:"_push",value:function(){this.wrapper=document.createElement("div"),this.wrapper.className="vis-configuration-wrapper",this.container.appendChild(this.wrapper);for(var t=0;t<this.domElements.length;t++)this.wrapper.appendChild(this.domElements[t]);this._showPopupIfNeeded()}},{key:"_clean",value:function(){for(var t=0;t<this.domElements.length;t++)this.wrapper.removeChild(this.domElements[t]);void 0!==this.wrapper&&(this.container.removeChild(this.wrapper),this.wrapper=void 0),this.domElements=[],this._removePopup()}},{key:"_getValue",value:function(t){for(var e=this.moduleOptions,i=0;i<t.length;i++){if(void 0===e[t[i]]){e=void 0;break}e=e[t[i]]}return e}},{key:"_makeItem",value:function(t){if(!0===this.allowCreation){var e=document.createElement("div");e.className="vis-configuration vis-config-item vis-config-s"+t.length;for(var i=arguments.length,o=Array(i>1?i-1:0),n=1;n<i;n++)o[n-1]=arguments[n];return o.forEach(function(t){e.appendChild(t)}),this.domElements.push(e),this.domElements.length}return 0}},{key:"_makeHeader",value:function(t){var e=document.createElement("div");e.className="vis-configuration vis-config-header",e.innerHTML=t,this._makeItem([],e)}},{key:"_makeLabel",value:function(t,e){var i=arguments.length>2&&void 0!==arguments[2]&&arguments[2],o=document.createElement("div");return o.className="vis-configuration vis-config-label vis-config-s"+e.length,o.innerHTML=!0===i?"<i><b>"+t+":</b></i>":t+":",o}},{key:"_makeDropdown",value:function(t,e,i){var o=document.createElement("select");o.className="vis-configuration vis-config-select";var n=0;void 0!==e&&-1!==t.indexOf(e)&&(n=t.indexOf(e));for(var s=0;s<t.length;s++){var r=document.createElement("option");r.value=t[s],s===n&&(r.selected="selected"),r.innerHTML=t[s],o.appendChild(r)}var a=this;o.onchange=function(){a._update(this.value,i)};var h=this._makeLabel(i[i.length-1],i);this._makeItem(i,h,o)}},{key:"_makeRange",value:function(t,e,i){var o=t[0],n=t[1],s=t[2],r=t[3],a=document.createElement("input");a.className="vis-configuration vis-config-range";try{a.type="range",a.min=n,a.max=s}catch(t){}a.step=r;var h="",d=0;if(void 0!==e){e<0&&1.2*e<n?(a.min=Math.ceil(1.2*e),d=a.min,h="range increased"):e/1.2<n&&(a.min=Math.ceil(e/1.2),d=a.min,h="range increased"),1.2*e>s&&1!==s&&(a.max=Math.ceil(1.2*e),d=a.max,h="range increased"),a.value=e}else a.value=o;var l=document.createElement("input");l.className="vis-configuration vis-config-rangeinput",l.value=a.value;var u=this;a.onchange=function(){l.value=this.value,u._update(Number(this.value),i)},a.oninput=function(){l.value=this.value};var p=this._makeLabel(i[i.length-1],i),c=this._makeItem(i,p,a,l);""!==h&&this.popupHistory[c]!==d&&(this.popupHistory[c]=d,this._setupPopup(h,c))}},{key:"_setupPopup",value:function(t,e){var i=this;if(!0===this.initialized&&!0===this.allowCreation&&this.popupCounter<this.popupLimit){var o=document.createElement("div");o.id="vis-configuration-popup",o.className="vis-configuration-popup",o.innerHTML=t,o.onclick=function(){i._removePopup()},this.popupCounter+=1,this.popupDiv={html:o,index:e}}}},{key:"_removePopup",value:function(){void 0!==this.popupDiv.html&&(this.popupDiv.html.parentNode.removeChild(this.popupDiv.html),clearTimeout(this.popupDiv.hideTimeout),clearTimeout(this.popupDiv.deleteTimeout),this.popupDiv={})}},{key:"_showPopupIfNeeded",value:function(){var t=this;if(void 0!==this.popupDiv.html){var e=this.domElements[this.popupDiv.index],i=e.getBoundingClientRect();this.popupDiv.html.style.left=i.left+"px",this.popupDiv.html.style.top=i.top-30+"px",document.body.appendChild(this.popupDiv.html),this.popupDiv.hideTimeout=setTimeout(function(){t.popupDiv.html.style.opacity=0},1500),this.popupDiv.deleteTimeout=setTimeout(function(){t._removePopup()},1800)}}},{key:"_makeCheckbox",value:function(t,e,i){var o=document.createElement("input");o.type="checkbox",o.className="vis-configuration vis-config-checkbox",o.checked=t,void 0!==e&&(o.checked=e,e!==t&&("object"===(void 0===t?"undefined":(0,a.default)(t))?e!==t.enabled&&this.changedOptions.push({path:i,value:e}):this.changedOptions.push({path:i,value:e})));var n=this;o.onchange=function(){n._update(this.checked,i)};var s=this._makeLabel(i[i.length-1],i);this._makeItem(i,s,o)}},{key:"_makeTextInput",value:function(t,e,i){var o=document.createElement("input");o.type="text",o.className="vis-configuration vis-config-text",o.value=e,e!==t&&this.changedOptions.push({path:i,value:e});var n=this;o.onchange=function(){n._update(this.value,i)};var s=this._makeLabel(i[i.length-1],i);this._makeItem(i,s,o)}},{key:"_makeColorField",value:function(t,e,i){var o=this,n=t[1],s=document.createElement("div");e=void 0===e?n:e,"none"!==e?(s.className="vis-configuration vis-config-colorBlock",s.style.backgroundColor=e):s.className="vis-configuration vis-config-colorBlock none",e=void 0===e?n:e,s.onclick=function(){o._showColorPicker(e,s,i)};var r=this._makeLabel(i[i.length-1],i);this._makeItem(i,r,s)}},{key:"_showColorPicker",value:function(t,e,i){var o=this;e.onclick=function(){},this.colorPicker.insertTo(e),this.colorPicker.show(),this.colorPicker.setColor(t),this.colorPicker.setUpdateCallback(function(t){var n="rgba("+t.r+","+t.g+","+t.b+","+t.a+")";e.style.backgroundColor=n,o._update(n,i)}),this.colorPicker.setCloseCallback(function(){e.onclick=function(){o._showColorPicker(t,e,i)}})}},{key:"_handleObject",value:function(t){var e=arguments.length>1&&void 0!==arguments[1]?arguments[1]:[],i=arguments.length>2&&void 0!==arguments[2]&&arguments[2],o=!1,n=this.options.filter,s=!1;for(var r in t)if(t.hasOwnProperty(r)){o=!0;var a=t[r],h=p.copyAndExtendArray(e,r);if("function"==typeof n&&!1===(o=n(r,e))&&!(a instanceof Array)&&"string"!=typeof a&&"boolean"!=typeof a&&a instanceof Object&&(this.allowCreation=!1,o=this._handleObject(a,h,!0),this.allowCreation=!1===i),!1!==o){s=!0;var d=this._getValue(h);if(a instanceof Array)this._handleArray(a,d,h);else if("string"==typeof a)this._makeTextInput(a,d,h);else if("boolean"==typeof a)this._makeCheckbox(a,d,h);else if(a instanceof Object){var l=!0;if(-1!==e.indexOf("physics")&&this.moduleOptions.physics.solver!==r&&(l=!1),!0===l)if(void 0!==a.enabled){var u=p.copyAndExtendArray(h,"enabled"),c=this._getValue(u);if(!0===c){var m=this._makeLabel(r,h,!0);this._makeItem(h,m),s=this._handleObject(a,h)||s}else this._makeCheckbox(a,c,h)}else{var f=this._makeLabel(r,h,!0);this._makeItem(h,f),s=this._handleObject(a,h)||s}}else console.error("dont know how to handle",a,r,h)}}return s}},{key:"_handleArray",value:function(t,e,i){"string"==typeof t[0]&&"color"===t[0]?(this._makeColorField(t,e,i),t[1]!==e&&this.changedOptions.push({path:i,value:e})):"string"==typeof t[0]?(this._makeDropdown(t,e,i),t[0]!==e&&this.changedOptions.push({path:i,value:e})):"number"==typeof t[0]&&(this._makeRange(t,e,i),t[0]!==e&&this.changedOptions.push({path:i,value:Number(e)}))}},{key:"_update",value:function(t,e){var i=this._constructOptions(t,e);this.parent.body&&this.parent.body.emitter&&this.parent.body.emitter.emit&&this.parent.body.emitter.emit("configChange",i),this.initialized=!0,this.parent.setOptions(i)}},{key:"_constructOptions",value:function(t,e){var i=arguments.length>2&&void 0!==arguments[2]?arguments[2]:{},o=i;t="true"===t||t,t="false"!==t&&t;for(var n=0;n<e.length;n++)"global"!==e[n]&&(void 0===o[e[n]]&&(o[e[n]]={}),n!==e.length-1?o=o[e[n]]:o[e[n]]=t);return i}},{key:"_printOptions",value:function(){var t=this.getOptions();this.optionsContainer.innerHTML="<pre>var options = "+(0,s.default)(t,null,2)+"</pre>"}},{key:"getOptions",value:function(){for(var t={},e=0;e<this.changedOptions.length;e++)this._constructOptions(this.changedOptions[e].value,this.changedOptions[e].path,t);return t}}]),t}();e.default=m},function(t,e,i){function o(t,e){this.id=r.randomUUID(),this.body=t,this.defaultOptions={yAxisOrientation:"left",defaultGroup:"default",sort:!0,sampling:!0,stack:!1,graphHeight:"400px",shaded:{enabled:!1,orientation:"bottom"},style:"line",barChart:{width:50,sideBySide:!1,align:"center"},interpolation:{enabled:!0,parametrization:"centripetal",alpha:.5},drawPoints:{enabled:!0,size:6,style:"square"},dataAxis:{},legend:{},groups:{visibility:{}}},this.options=r.extend({},this.defaultOptions),this.dom={},this.props={},this.hammer=null,this.groups={},this.abortedGraphUpdate=!1,this.updateSVGheight=!1,this.updateSVGheightOnResize=!1,this.forceGraphUpdate=!0;var i=this;this.itemsData=null,this.groupsData=null,this.itemListeners={add:function(t,e,o){i._onAdd(e.items)},update:function(t,e,o){i._onUpdate(e.items)},remove:function(t,e,o){i._onRemove(e.items)}},this.groupListeners={add:function(t,e,o){i._onAddGroups(e.items)},update:function(t,e,o){i._onUpdateGroups(e.items)},remove:function(t,e,o){i._onRemoveGroups(e.items)}},this.items={},this.selection=[],this.lastStart=this.body.range.start,this.touchParams={},this.svgElements={},this.setOptions(e),this.groupsUsingDefaultStyles=[0],this.body.emitter.on("rangechanged",function(){i.lastStart=i.body.range.start,i.svg.style.left=r.option.asSize(-i.props.width),i.forceGraphUpdate=!0,i.redraw.call(i)}),this._create(),this.framework={svg:this.svg,svgElements:this.svgElements,options:this.options,groups:this.groups}}var n=i(1),s=function(t){return t&&t.__esModule?t:{default:t}}(n),r=i(0),a=i(11),h=i(15),d=i(20),l=i(5),u=i(78),p=i(80),c=i(83),m=i(81),f=i(82),g=i(54);o.prototype=new l,o.prototype._create=function(){var t=document.createElement("div");t.className="vis-line-graph",this.dom.frame=t,this.svg=document.createElementNS("http://www.w3.org/2000/svg","svg"),this.svg.style.position="relative",this.svg.style.height=(""+this.options.graphHeight).replace("px","")+"px",this.svg.style.display="block",t.appendChild(this.svg),this.options.dataAxis.orientation="left",this.yAxisLeft=new u(this.body,this.options.dataAxis,this.svg,this.options.groups),this.options.dataAxis.orientation="right",this.yAxisRight=new u(this.body,this.options.dataAxis,this.svg,this.options.groups),delete this.options.dataAxis.orientation,this.legendLeft=new c(this.body,this.options.legend,"left",this.options.groups),this.legendRight=new c(this.body,this.options.legend,"right",this.options.groups),this.show()},o.prototype.setOptions=function(t){if(t){var e=["sampling","defaultGroup","stack","height","graphHeight","yAxisOrientation","style","barChart","dataAxis","sort","groups"];void 0===t.graphHeight&&void 0!==t.height?(this.updateSVGheight=!0,this.updateSVGheightOnResize=!0):void 0!==this.body.domProps.centerContainer.height&&void 0!==t.graphHeight&&parseInt((t.graphHeight+"").replace("px",""))<this.body.domProps.centerContainer.height&&(this.updateSVGheight=!0),r.selectiveDeepExtend(e,this.options,t),r.mergeOptions(this.options,t,"interpolation"),r.mergeOptions(this.options,t,"drawPoints"),r.mergeOptions(this.options,t,"shaded"),r.mergeOptions(this.options,t,"legend"),t.interpolation&&"object"==(0,s.default)(t.interpolation)&&t.interpolation.parametrization&&("uniform"==t.interpolation.parametrization?this.options.interpolation.alpha=0:"chordal"==t.interpolation.parametrization?this.options.interpolation.alpha=1:(this.options.interpolation.parametrization="centripetal",this.options.interpolation.alpha=.5)),this.yAxisLeft&&void 0!==t.dataAxis&&(this.yAxisLeft.setOptions(this.options.dataAxis),this.yAxisRight.setOptions(this.options.dataAxis)),this.legendLeft&&void 0!==t.legend&&(this.legendLeft.setOptions(this.options.legend),this.legendRight.setOptions(this.options.legend)),this.groups.hasOwnProperty("__ungrouped__")&&this.groups.__ungrouped__.setOptions(t)}this.dom.frame&&(this.forceGraphUpdate=!0,this.body.emitter.emit("_change",{queue:!0}))},o.prototype.hide=function(){this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame)},o.prototype.show=function(){this.dom.frame.parentNode||this.body.dom.center.appendChild(this.dom.frame)},o.prototype.setItems=function(t){var e,i=this,o=this.itemsData;if(t){if(!(t instanceof h||t instanceof d))throw new TypeError("Data must be an instance of DataSet or DataView");this.itemsData=t}else this.itemsData=null;if(o&&(r.forEach(this.itemListeners,function(t,e){o.off(e,t)}),e=o.getIds(),this._onRemove(e)),this.itemsData){var n=this.id;r.forEach(this.itemListeners,function(t,e){i.itemsData.on(e,t,n)}),e=this.itemsData.getIds(),this._onAdd(e)}},o.prototype.setGroups=function(t){var e,i=this;if(this.groupsData){r.forEach(this.groupListeners,function(t,e){i.groupsData.off(e,t)}),e=this.groupsData.getIds(),this.groupsData=null;for(var o=0;o<e.length;o++)this._removeGroup(e[o])}if(t){if(!(t instanceof h||t instanceof d))throw new TypeError("Data must be an instance of DataSet or DataView");this.groupsData=t}else this.groupsData=null;if(this.groupsData){var n=this.id;r.forEach(this.groupListeners,function(t,e){i.groupsData.on(e,t,n)}),e=this.groupsData.getIds(),this._onAddGroups(e)}},o.prototype._onUpdate=function(t){this._updateAllGroupData(t)},o.prototype._onAdd=function(t){this._onUpdate(t)},o.prototype._onRemove=function(t){this._onUpdate(t)},o.prototype._onUpdateGroups=function(t){this._updateAllGroupData(null,t)},o.prototype._onAddGroups=function(t){this._onUpdateGroups(t)},o.prototype._onRemoveGroups=function(t){for(var e=0;e<t.length;e++)this._removeGroup(t[e]);this.forceGraphUpdate=!0,this.body.emitter.emit("_change",{queue:!0})},o.prototype._removeGroup=function(t){this.groups.hasOwnProperty(t)&&("right"==this.groups[t].options.yAxisOrientation?(this.yAxisRight.removeGroup(t),this.legendRight.removeGroup(t),this.legendRight.redraw()):(this.yAxisLeft.removeGroup(t),this.legendLeft.removeGroup(t),this.legendLeft.redraw()),delete this.groups[t])},o.prototype._updateGroup=function(t,e){this.groups.hasOwnProperty(e)?(this.groups[e].update(t),"right"==this.groups[e].options.yAxisOrientation?(this.yAxisRight.updateGroup(e,this.groups[e]),this.legendRight.updateGroup(e,this.groups[e]),this.yAxisLeft.removeGroup(e),this.legendLeft.removeGroup(e)):(this.yAxisLeft.updateGroup(e,this.groups[e]),this.legendLeft.updateGroup(e,this.groups[e]),this.yAxisRight.removeGroup(e),this.legendRight.removeGroup(e))):(this.groups[e]=new p(t,e,this.options,this.groupsUsingDefaultStyles),"right"==this.groups[e].options.yAxisOrientation?(this.yAxisRight.addGroup(e,this.groups[e]),this.legendRight.addGroup(e,this.groups[e])):(this.yAxisLeft.addGroup(e,this.groups[e]),this.legendLeft.addGroup(e,this.groups[e]))),this.legendLeft.redraw(),this.legendRight.redraw()},o.prototype._updateAllGroupData=function(t,e){if(null!=this.itemsData){var i={},o=this.itemsData.get(),n=this.itemsData._fieldId,s={};t&&t.map(function(t){s[t]=t});for(var a={},h=0;h<o.length;h++){var d=o[h],l=d.group;null!==l&&void 0!==l||(l="__ungrouped__"),a.hasOwnProperty(l)?a[l]++:a[l]=1}var u={};if(!e&&t)for(l in this.groups)if(this.groups.hasOwnProperty(l)){g=this.groups[l];var p=g.getItems();i[l]=p.filter(function(t){return u[t[n]]=t[n],t[n]!==s[t[n]]});var c=a[l];a[l]-=i[l].length,i[l].length<c&&(i[l][c-1]={})}for(h=0;h<o.length;h++)if(d=o[h],l=d.group,null!==l&&void 0!==l||(l="__ungrouped__"),e||!t||d[n]===s[d[n]]||!u.hasOwnProperty(d[n])){i.hasOwnProperty(l)||(i[l]=new Array(a[l]));var m=r.bridgeObject(d);m.x=r.convert(d.x,"Date"),m.end=r.convert(d.end,"Date"),m.orginalY=d.y,m.y=Number(d.y),m[n]=d[n];var f=i[l].length-a[l]--;i[l][f]=m}for(l in this.groups)this.groups.hasOwnProperty(l)&&(i.hasOwnProperty(l)||(i[l]=new Array(0)));for(l in i)if(i.hasOwnProperty(l))if(0==i[l].length)this.groups.hasOwnProperty(l)&&this._removeGroup(l);else{var g=void 0;void 0!=this.groupsData&&(g=this.groupsData.get(l)),void 0==g&&(g={id:l,content:this.options.defaultGroup+l}),this._updateGroup(g,l),this.groups[l].setItems(i[l])}this.forceGraphUpdate=!0,this.body.emitter.emit("_change",{queue:!0})}},o.prototype.redraw=function(){var t=!1;this.props.width=this.dom.frame.offsetWidth,this.props.height=this.body.domProps.centerContainer.height-this.body.domProps.border.top-this.body.domProps.border.bottom,t=this._isResized()||t;var e=this.body.range.end-this.body.range.start,i=e!=this.lastVisibleInterval;if(this.lastVisibleInterval=e,1==t&&(this.svg.style.width=r.option.asSize(3*this.props.width),this.svg.style.left=r.option.asSize(-this.props.width),-1==(this.options.height+"").indexOf("%")&&1!=this.updateSVGheightOnResize||(this.updateSVGheight=!0)),1==this.updateSVGheight?(this.options.graphHeight!=this.props.height+"px"&&(this.options.graphHeight=this.props.height+"px",this.svg.style.height=this.props.height+"px"),this.updateSVGheight=!1):this.svg.style.height=(""+this.options.graphHeight).replace("px","")+"px",1==t||1==i||1==this.abortedGraphUpdate||1==this.forceGraphUpdate)t=this._updateGraph()||t,this.forceGraphUpdate=!1;else if(0!=this.lastStart){var o=this.body.range.start-this.lastStart,n=this.body.range.end-this.body.range.start;if(0!=this.props.width){var s=this.props.width/n,a=o*s;this.svg.style.left=-this.props.width-a+"px"}}return this.legendLeft.redraw(),this.legendRight.redraw(),t},o.prototype._getSortedGroupIds=function(){var t=[];for(var e in this.groups)if(this.groups.hasOwnProperty(e)){var i=this.groups[e];1!=i.visible||void 0!==this.options.groups.visibility[e]&&1!=this.options.groups.visibility[e]||t.push({id:e,zIndex:i.options.zIndex})}r.insertSort(t,function(t,e){var i=t.zIndex,o=e.zIndex;return void 0===i&&(i=0),void 0===o&&(o=0),i==o?0:i<o?-1:1});for(var o=new Array(t.length),n=0;n<t.length;n++)o[n]=t[n].id;return o},o.prototype._updateGraph=function(){if(a.prepareElements(this.svgElements),0!=this.props.width&&null!=this.itemsData){var t,e,i={},o=this.body.util.toGlobalTime(-this.body.domProps.root.width),n=this.body.util.toGlobalTime(2*this.body.domProps.root.width),s=this._getSortedGroupIds();if(s.length>0){var r={};for(this._getRelevantData(s,r,o,n),this._applySampling(s,r),e=0;e<s.length;e++)this._convertXcoordinates(r[s[e]]);if(this._getYRanges(s,r,i),1==this._updateYAxis(s,i))return a.cleanupElements(this.svgElements),this.abortedGraphUpdate=!0,!0;this.abortedGraphUpdate=!1;var h=void 0;for(e=0;e<s.length;e++)t=this.groups[s[e]],!0===this.options.stack&&"line"===this.options.style&&(void 0!=t.options.excludeFromStacking&&t.options.excludeFromStacking||(void 0!=h&&(this._stack(r[t.id],r[h.id]),1==t.options.shaded.enabled&&"group"!==t.options.shaded.orientation&&("top"==t.options.shaded.orientation&&"group"!==h.options.shaded.orientation?(h.options.shaded.orientation="group",h.options.shaded.groupId=t.id):(t.options.shaded.orientation="group",t.options.shaded.groupId=h.id))),h=t)),this._convertYcoordinates(r[s[e]],t);var d={};for(e=0;e<s.length;e++)if(t=this.groups[s[e]],"line"===t.options.style&&1==t.options.shaded.enabled){var l=r[s[e]];if(null==l||0==l.length)continue;if(d.hasOwnProperty(s[e])||(d[s[e]]=f.calcPath(l,t)),"group"===t.options.shaded.orientation){var u=t.options.shaded.groupId;if(-1===s.indexOf(u)){console.log(t.id+": Unknown shading group target given:"+u);continue}d.hasOwnProperty(u)||(d[u]=f.calcPath(r[u],this.groups[u])),f.drawShading(d[s[e]],t,d[u],this.framework)}else f.drawShading(d[s[e]],t,void 0,this.framework)}for(m.draw(s,r,this.framework),e=0;e<s.length;e++)if(t=this.groups[s[e]],r[s[e]].length>0)switch(t.options.style){case"line":d.hasOwnProperty(s[e])||(d[s[e]]=f.calcPath(r[s[e]],t)),f.draw(d[s[e]],t,this.framework);case"point":case"points":"point"!=t.options.style&&"points"!=t.options.style&&1!=t.options.drawPoints.enabled||g.draw(r[s[e]],t,this.framework)}}}return a.cleanupElements(this.svgElements),!1},o.prototype._stack=function(t,e){var i,o,n,s,r;i=0;for(var a=0;a<t.length;a++){s=void 0,r=void 0;for(var h=i;h<e.length;h++){if(e[h].x===t[a].x){s=e[h],r=e[h],i=h;break}if(e[h].x>t[a].x){r=e[h],s=0==h?r:e[h-1],i=h;break}}void 0===r&&(s=e[e.length-1],r=e[e.length-1]),o=r.x-s.x,n=r.y-s.y,t[a].y=0==o?t[a].orginalY+r.y:t[a].orginalY+n/o*(t[a].x-s.x)+s.y}},o.prototype._getRelevantData=function(t,e,i,o){var n,s,a,h;if(t.length>0)for(s=0;s<t.length;s++){n=this.groups[t[s]];var d=n.getItems();if(1==n.options.sort){var l=function(t,e){return t.getTime()==e.getTime()?0:t<e?-1:1},u=Math.max(0,r.binarySearchValue(d,i,"x","before",l)),p=Math.min(d.length,r.binarySearchValue(d,o,"x","after",l)+1);p<=0&&(p=d.length);var c=new Array(p-u);for(a=u;a<p;a++)h=n.itemsData[a],c[a-u]=h;e[t[s]]=c}else e[t[s]]=n.itemsData}},o.prototype._applySampling=function(t,e){var i;if(t.length>0)for(var o=0;o<t.length;o++)if(i=this.groups[t[o]],1==i.options.sampling){var n=e[t[o]];if(n.length>0){var s=1,r=n.length,a=this.body.util.toGlobalScreen(n[n.length-1].x)-this.body.util.toGlobalScreen(n[0].x),h=r/a;s=Math.min(Math.ceil(.2*r),Math.max(1,Math.round(h)));for(var d=new Array(r),l=0;l<r;l+=s){var u=Math.round(l/s);d[u]=n[l]}e[t[o]]=d.splice(0,Math.round(r/s))}}},o.prototype._getYRanges=function(t,e,i){var o,n,s,r,a=[],h=[];if(t.length>0){for(s=0;s<t.length;s++)o=e[t[s]],r=this.groups[t[s]].options,o.length>0&&(n=this.groups[t[s]],!0===r.stack&&"bar"===r.style?"left"===r.yAxisOrientation?a=a.concat(o):h=h.concat(o):i[t[s]]=n.getYRange(o,t[s]));m.getStackedYRange(a,i,t,"__barStackLeft","left"),m.getStackedYRange(h,i,t,"__barStackRight","right")}},o.prototype._updateYAxis=function(t,e){var i,o,n=!1,s=!1,r=!1,a=1e9,h=1e9,d=-1e9,l=-1e9;if(t.length>0){for(var u=0;u<t.length;u++){var p=this.groups[t[u]];p&&"right"!=p.options.yAxisOrientation?(s=!0,a=1e9,d=-1e9):p&&p.options.yAxisOrientation&&(r=!0,h=1e9,l=-1e9)}for(u=0;u<t.length;u++)e.hasOwnProperty(t[u])&&!0!==e[t[u]].ignore&&(i=e[t[u]].min,o=e[t[u]].max,"right"!=e[t[u]].yAxisOrientation?(s=!0,a=a>i?i:a,d=d<o?o:d):(r=!0,h=h>i?i:h,l=l<o?o:l));1==s&&this.yAxisLeft.setRange(a,d),1==r&&this.yAxisRight.setRange(h,l)}n=this._toggleAxisVisiblity(s,this.yAxisLeft)||n,n=this._toggleAxisVisiblity(r,this.yAxisRight)||n,1==r&&1==s?(this.yAxisLeft.drawIcons=!0,this.yAxisRight.drawIcons=!0):(this.yAxisLeft.drawIcons=!1,this.yAxisRight.drawIcons=!1),this.yAxisRight.master=!s,this.yAxisRight.masterAxis=this.yAxisLeft,0==this.yAxisRight.master?(this.yAxisLeft.lineOffset=1==r?this.yAxisRight.width:0,n=this.yAxisLeft.redraw()||n,n=this.yAxisRight.redraw()||n):n=this.yAxisRight.redraw()||n;var c=["__barStackLeft","__barStackRight","__lineStackLeft","__lineStackRight"];for(u=0;u<c.length;u++)-1!=t.indexOf(c[u])&&t.splice(t.indexOf(c[u]),1);return n},o.prototype._toggleAxisVisiblity=function(t,e){var i=!1;return 0==t?e.dom.frame.parentNode&&0==e.hidden&&(e.hide(),i=!0):e.dom.frame.parentNode||1!=e.hidden||(e.show(),i=!0),i},o.prototype._convertXcoordinates=function(t){for(var e=this.body.util.toScreen,i=0;i<t.length;i++)t[i].screen_x=e(t[i].x)+this.props.width,t[i].screen_y=t[i].y,void 0!=t[i].end?t[i].screen_end=e(t[i].end)+this.props.width:t[i].screen_end=void 0},o.prototype._convertYcoordinates=function(t,e){var i=this.yAxisLeft,o=Number(this.svg.style.height.replace("px",""));"right"==e.options.yAxisOrientation&&(i=this.yAxisRight);for(var n=0;n<t.length;n++)t[n].screen_y=Math.round(i.convertValue(t[n].y));e.setZeroPosition(Math.min(o,i.convertValue(0)))},t.exports=o},function(t,e,i){function o(t,e,i,o){this.id=r.randomUUID(),this.body=t,this.defaultOptions={orientation:"left",showMinorLabels:!0,showMajorLabels:!0,icons:!1,majorLinesOffset:7,minorLinesOffset:4,labelOffsetX:10,labelOffsetY:2,iconWidth:20,width:"40px",visible:!0,alignZeros:!0,left:{range:{min:void 0,max:void 0},format:function(t){return""+parseFloat(t.toPrecision(3))},title:{text:void 0,style:void 0}},right:{range:{min:void 0,max:void 0},format:function(t){return""+parseFloat(t.toPrecision(3))},title:{text:void 0,style:void 0}}},this.linegraphOptions=o,this.linegraphSVG=i,this.props={},this.DOMelements={lines:{},labels:{},title:{}},this.dom={},this.scale=void 0,this.range={start:0,end:0},this.options=r.extend({},this.defaultOptions),this.conversionFactor=1,this.setOptions(e),this.width=Number((""+this.options.width).replace("px","")),this.minWidth=this.width,this.height=this.linegraphSVG.getBoundingClientRect().height,this.hidden=!1,this.stepPixels=25,this.zeroCrossing=-1,this.amountOfSteps=-1,this.lineOffset=0,this.master=!0,this.masterAxis=null,this.svgElements={},this.iconsRemoved=!1,this.groups={},this.amountOfGroups=0,this._create(),this.framework={svg:this.svg,svgElements:this.svgElements,options:this.options,groups:this.groups};var n=this;this.body.emitter.on("verticalDrag",function(){n.dom.lineContainer.style.top=n.body.domProps.scrollTop+"px"})}var n=i(4),s=function(t){return t&&t.__esModule?t:{default:t}}(n),r=i(0),a=i(11),h=i(5),d=i(79);o.prototype=new h,o.prototype.addGroup=function(t,e){this.groups.hasOwnProperty(t)||(this.groups[t]=e),this.amountOfGroups+=1},o.prototype.updateGroup=function(t,e){this.groups.hasOwnProperty(t)||(this.amountOfGroups+=1),this.groups[t]=e},o.prototype.removeGroup=function(t){this.groups.hasOwnProperty(t)&&(delete this.groups[t],this.amountOfGroups-=1)},o.prototype.setOptions=function(t){if(t){var e=!1;this.options.orientation!=t.orientation&&void 0!==t.orientation&&(e=!0);var i=["orientation","showMinorLabels","showMajorLabels","icons","majorLinesOffset","minorLinesOffset","labelOffsetX","labelOffsetY","iconWidth","width","visible","left","right","alignZeros"];r.selectiveDeepExtend(i,this.options,t),this.minWidth=Number((""+this.options.width).replace("px","")),!0===e&&this.dom.frame&&(this.hide(),this.show())}},o.prototype._create=function(){this.dom.frame=document.createElement("div"),this.dom.frame.style.width=this.options.width,this.dom.frame.style.height=this.height,this.dom.lineContainer=document.createElement("div"),this.dom.lineContainer.style.width="100%",this.dom.lineContainer.style.height=this.height,this.dom.lineContainer.style.position="relative",this.svg=document.createElementNS("http://www.w3.org/2000/svg","svg"),this.svg.style.position="absolute",this.svg.style.top="0px",this.svg.style.height="100%",this.svg.style.width="100%",this.svg.style.display="block",this.dom.frame.appendChild(this.svg)},o.prototype._redrawGroupIcons=function(){a.prepareElements(this.svgElements);var t,e=this.options.iconWidth,i=11.5;t="left"===this.options.orientation?4:this.width-e-4;var o=(0,s.default)(this.groups);o.sort(function(t,e){return t<e?-1:1});for(var n=0;n<o.length;n++){var r=o[n];!0!==this.groups[r].visible||void 0!==this.linegraphOptions.visibility[r]&&!0!==this.linegraphOptions.visibility[r]||(this.groups[r].getLegend(e,15,this.framework,t,i),i+=19)}a.cleanupElements(this.svgElements),this.iconsRemoved=!1},o.prototype._cleanupIcons=function(){!1===this.iconsRemoved&&(a.prepareElements(this.svgElements),a.cleanupElements(this.svgElements),this.iconsRemoved=!0)},o.prototype.show=function(){this.hidden=!1,this.dom.frame.parentNode||("left"===this.options.orientation?this.body.dom.left.appendChild(this.dom.frame):this.body.dom.right.appendChild(this.dom.frame)),this.dom.lineContainer.parentNode||this.body.dom.backgroundHorizontal.appendChild(this.dom.lineContainer)},o.prototype.hide=function(){this.hidden=!0,this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame),this.dom.lineContainer.parentNode&&this.dom.lineContainer.parentNode.removeChild(this.dom.lineContainer)},o.prototype.setRange=function(t,e){this.range.start=t,this.range.end=e},o.prototype.redraw=function(){var t=!1,e=0;this.dom.lineContainer.style.top=this.body.domProps.scrollTop+"px";for(var i in this.groups)this.groups.hasOwnProperty(i)&&(!0!==this.groups[i].visible||void 0!==this.linegraphOptions.visibility[i]&&!0!==this.linegraphOptions.visibility[i]||e++);if(0===this.amountOfGroups||0===e)this.hide();else{this.show(),this.height=Number(this.linegraphSVG.style.height.replace("px","")),this.dom.lineContainer.style.height=this.height+"px",this.width=!0===this.options.visible?Number((""+this.options.width).replace("px","")):0;var o=this.props,n=this.dom.frame;n.className="vis-data-axis",this._calculateCharSize();var s=this.options.orientation,r=this.options.showMinorLabels,a=this.options.showMajorLabels;o.minorLabelHeight=r?o.minorCharHeight:0,o.majorLabelHeight=a?o.majorCharHeight:0,o.minorLineWidth=this.body.dom.backgroundHorizontal.offsetWidth-this.lineOffset-this.width+2*this.options.minorLinesOffset,o.minorLineHeight=1,o.majorLineWidth=this.body.dom.backgroundHorizontal.offsetWidth-this.lineOffset-this.width+2*this.options.majorLinesOffset,o.majorLineHeight=1,"left"===s?(n.style.top="0",n.style.left="0",n.style.bottom="",n.style.width=this.width+"px",n.style.height=this.height+"px",this.props.width=this.body.domProps.left.width,this.props.height=this.body.domProps.left.height):(n.style.top="",n.style.bottom="0",n.style.left="0",n.style.width=this.width+"px",n.style.height=this.height+"px",this.props.width=this.body.domProps.right.width,this.props.height=this.body.domProps.right.height),t=this._redrawLabels(),t=this._isResized()||t,!0===this.options.icons?this._redrawGroupIcons():this._cleanupIcons(),this._redrawTitle(s)}return t},o.prototype._redrawLabels=function(){var t=this,e=!1;a.prepareElements(this.DOMelements.lines),a.prepareElements(this.DOMelements.labels);var i=this.options.orientation,o=void 0!=this.options[i].range?this.options[i].range:{},n=!0;void 0!=o.max&&(this.range.end=o.max,n=!1);var s=!0;void 0!=o.min&&(this.range.start=o.min,s=!1),this.scale=new d(this.range.start,this.range.end,s,n,this.dom.frame.offsetHeight,this.props.majorCharHeight,this.options.alignZeros,this.options[i].format),!1===this.master&&void 0!=this.masterAxis&&this.scale.followScale(this.masterAxis.scale),this.maxLabelSize=0,this.scale.getLines().forEach(function(e){var o=e.y,n=e.major;t.options.showMinorLabels&&!1===n&&t._redrawLabel(o-2,e.val,i,"vis-y-axis vis-minor",t.props.minorCharHeight),n&&o>=0&&t._redrawLabel(o-2,e.val,i,"vis-y-axis vis-major",t.props.majorCharHeight),!0===t.master&&(n?t._redrawLine(o,i,"vis-grid vis-horizontal vis-major",t.options.majorLinesOffset,t.props.majorLineWidth):t._redrawLine(o,i,"vis-grid vis-horizontal vis-minor",t.options.minorLinesOffset,t.props.minorLineWidth))});var r=0;void 0!==this.options[i].title&&void 0!==this.options[i].title.text&&(r=this.props.titleCharHeight);var h=!0===this.options.icons?Math.max(this.options.iconWidth,r)+this.options.labelOffsetX+15:r+this.options.labelOffsetX+15;return this.maxLabelSize>this.width-h&&!0===this.options.visible?(this.width=this.maxLabelSize+h,this.options.width=this.width+"px",a.cleanupElements(this.DOMelements.lines),a.cleanupElements(this.DOMelements.labels),this.redraw(),e=!0):this.maxLabelSize<this.width-h&&!0===this.options.visible&&this.width>this.minWidth?(this.width=Math.max(this.minWidth,this.maxLabelSize+h),this.options.width=this.width+"px",a.cleanupElements(this.DOMelements.lines),a.cleanupElements(this.DOMelements.labels),this.redraw(),e=!0):(a.cleanupElements(this.DOMelements.lines),a.cleanupElements(this.DOMelements.labels),e=!1),e},o.prototype.convertValue=function(t){return this.scale.convertValue(t)},o.prototype.screenToValue=function(t){return this.scale.screenToValue(t)},o.prototype._redrawLabel=function(t,e,i,o,n){var s=a.getDOMElement("div",this.DOMelements.labels,this.dom.frame);s.className=o,s.innerHTML=e,"left"===i?(s.style.left="-"+this.options.labelOffsetX+"px",s.style.textAlign="right"):(s.style.right="-"+this.options.labelOffsetX+"px",s.style.textAlign="left"),s.style.top=t-.5*n+this.options.labelOffsetY+"px",e+="";var r=Math.max(this.props.majorCharWidth,this.props.minorCharWidth);this.maxLabelSize<e.length*r&&(this.maxLabelSize=e.length*r)},o.prototype._redrawLine=function(t,e,i,o,n){if(!0===this.master){var s=a.getDOMElement("div",this.DOMelements.lines,this.dom.lineContainer);s.className=i,s.innerHTML="","left"===e?s.style.left=this.width-o+"px":s.style.right=this.width-o+"px",s.style.width=n+"px",s.style.top=t+"px"}},o.prototype._redrawTitle=function(t){if(a.prepareElements(this.DOMelements.title),void 0!==this.options[t].title&&void 0!==this.options[t].title.text){var e=a.getDOMElement("div",this.DOMelements.title,this.dom.frame);e.className="vis-y-axis vis-title vis-"+t,e.innerHTML=this.options[t].title.text,void 0!==this.options[t].title.style&&r.addCssText(e,this.options[t].title.style),"left"===t?e.style.left=this.props.titleCharHeight+"px":e.style.right=this.props.titleCharHeight+"px",e.style.width=this.height+"px"}a.cleanupElements(this.DOMelements.title)},o.prototype._calculateCharSize=function(){if(!("minorCharHeight"in this.props)){var t=document.createTextNode("0"),e=document.createElement("div");e.className="vis-y-axis vis-minor vis-measure",e.appendChild(t),this.dom.frame.appendChild(e),this.props.minorCharHeight=e.clientHeight,this.props.minorCharWidth=e.clientWidth,this.dom.frame.removeChild(e)}if(!("majorCharHeight"in this.props)){var i=document.createTextNode("0"),o=document.createElement("div");o.className="vis-y-axis vis-major vis-measure",o.appendChild(i),this.dom.frame.appendChild(o),this.props.majorCharHeight=o.clientHeight,this.props.majorCharWidth=o.clientWidth,this.dom.frame.removeChild(o)}if(!("titleCharHeight"in this.props)){var n=document.createTextNode("0"),s=document.createElement("div");s.className="vis-y-axis vis-title vis-measure",s.appendChild(n),this.dom.frame.appendChild(s),this.props.titleCharHeight=s.clientHeight,this.props.titleCharWidth=s.clientWidth,this.dom.frame.removeChild(s)}},t.exports=o},function(t,e,i){function o(t,e,i,o,n,s){var r=arguments.length>6&&void 0!==arguments[6]&&arguments[6],a=arguments.length>7&&void 0!==arguments[7]&&arguments[7];if(this.majorSteps=[1,2,5,10],this.minorSteps=[.25,.5,1,2],this.customLines=null,this.containerHeight=n,this.majorCharHeight=s,this._start=t,this._end=e,this.scale=1,this.minorStepIdx=-1,this.magnitudefactor=1,this.determineScale(),this.zeroAlign=r,this.autoScaleStart=i,this.autoScaleEnd=o,this.formattingFunction=a,i||o){var h=this,d=function(t){var e=t-t%(h.magnitudefactor*h.minorSteps[h.minorStepIdx]);return t%(h.magnitudefactor*h.minorSteps[h.minorStepIdx])>h.magnitudefactor*h.minorSteps[h.minorStepIdx]*.5?e+h.magnitudefactor*h.minorSteps[h.minorStepIdx]:e};i&&(this._start-=2*this.magnitudefactor*this.minorSteps[this.minorStepIdx],this._start=d(this._start)),o&&(this._end+=this.magnitudefactor*this.minorSteps[this.minorStepIdx],this._end=d(this._end)),this.determineScale()}}o.prototype.setCharHeight=function(t){this.majorCharHeight=t},o.prototype.setHeight=function(t){this.containerHeight=t},o.prototype.determineScale=function(){var t=this._end-this._start;this.scale=this.containerHeight/t;var e=this.majorCharHeight/this.scale,i=t>0?Math.round(Math.log(t)/Math.LN10):0;this.minorStepIdx=-1,this.magnitudefactor=Math.pow(10,i);var o=0;i<0&&(o=i);for(var n=!1,s=o;Math.abs(s)<=Math.abs(i);s++){this.magnitudefactor=Math.pow(10,s);for(var r=0;r<this.minorSteps.length;r++){if(this.magnitudefactor*this.minorSteps[r]>=e){n=!0,this.minorStepIdx=r;break}}if(!0===n)break}},o.prototype.is_major=function(t){return t%(this.magnitudefactor*this.majorSteps[this.minorStepIdx])==0},o.prototype.getStep=function(){return this.magnitudefactor*this.minorSteps[this.minorStepIdx]},o.prototype.getFirstMajor=function(){var t=this.magnitudefactor*this.majorSteps[this.minorStepIdx];return this.convertValue(this._start+(t-this._start%t)%t)},o.prototype.formatValue=function(t){var e=t.toPrecision(5);return"function"==typeof this.formattingFunction&&(e=this.formattingFunction(t)),"number"==typeof e?""+e:"string"==typeof e?e:t.toPrecision(5)},o.prototype.getLines=function(){for(var t=[],e=this.getStep(),i=(e-this._start%e)%e,o=this._start+i;this._end-o>1e-5;o+=e)o!=this._start&&t.push({major:this.is_major(o),y:this.convertValue(o),val:this.formatValue(o)});return t},o.prototype.followScale=function(t){var e=this.minorStepIdx,i=this._start,o=this._end,n=this,s=function(){n.magnitudefactor*=2},r=function(){n.magnitudefactor/=2};t.minorStepIdx<=1&&this.minorStepIdx<=1||t.minorStepIdx>1&&this.minorStepIdx>1||(t.minorStepIdx<this.minorStepIdx?(this.minorStepIdx=1,2==e?s():(s(),s())):(this.minorStepIdx=2,1==e?r():(r(),r())));for(var a=t.convertValue(0),h=t.getStep()*t.scale,d=!1,l=0;!d&&l++<5;){this.scale=h/(this.minorSteps[this.minorStepIdx]*this.magnitudefactor);var u=this.containerHeight/this.scale;this._start=i,this._end=this._start+u;var p=this._end*this.scale,c=this.magnitudefactor*this.majorSteps[this.minorStepIdx],m=this.getFirstMajor()-t.getFirstMajor();if(this.zeroAlign){var f=a-p;this._end+=f/this.scale,this._start=this._end-u}else this.autoScaleStart?(this._start-=m/this.scale,this._end=this._start+u):(this._start+=c-m/this.scale,this._end=this._start+u);if(!this.autoScaleEnd&&this._end>o+1e-5)r(),d=!1;else{if(!this.autoScaleStart&&this._start<i-1e-5){if(!(this.zeroAlign&&i>=0)){r(),d=!1;continue}console.warn("Can't adhere to given 'min' range, due to zeroalign")}this.autoScaleStart&&this.autoScaleEnd&&u<o-i?(s(),d=!1):d=!0}}},o.prototype.convertValue=function(t){return this.containerHeight-(t-this._start)*this.scale},o.prototype.screenToValue=function(t){return(this.containerHeight-t)/this.scale+this._start},t.exports=o},function(t,e,i){function o(t,e,i,o){this.id=e;var n=["sampling","style","sort","yAxisOrientation","barChart","drawPoints","shaded","interpolation","zIndex","excludeFromStacking","excludeFromLegend"];this.options=r.selectiveBridgeObject(n,i),this.usingDefaultStyle=void 0===t.className,this.groupsUsingDefaultStyles=o,this.zeroPosition=0,this.update(t),1==this.usingDefaultStyle&&(this.groupsUsingDefaultStyles[0]+=1),this.itemsData=[],this.visible=void 0===t.visible||t.visible}var n=i(1),s=function(t){return t&&t.__esModule?t:{default:t}}(n),r=i(0),a=i(81),h=i(82),d=i(54);o.prototype.setItems=function(t){null!=t?(this.itemsData=t,1==this.options.sort&&r.insertSort(this.itemsData,function(t,e){return t.x>e.x?1:-1})):this.itemsData=[]},o.prototype.getItems=function(){return this.itemsData},o.prototype.setZeroPosition=function(t){this.zeroPosition=t},o.prototype.setOptions=function(t){if(void 0!==t){var e=["sampling","style","sort","yAxisOrientation","barChart","zIndex","excludeFromStacking","excludeFromLegend"];r.selectiveDeepExtend(e,this.options,t),"function"==typeof t.drawPoints&&(t.drawPoints={onRender:t.drawPoints}),r.mergeOptions(this.options,t,"interpolation"),r.mergeOptions(this.options,t,"drawPoints"),r.mergeOptions(this.options,t,"shaded"),t.interpolation&&"object"==(0,s.default)(t.interpolation)&&t.interpolation.parametrization&&("uniform"==t.interpolation.parametrization?this.options.interpolation.alpha=0:"chordal"==t.interpolation.parametrization?this.options.interpolation.alpha=1:(this.options.interpolation.parametrization="centripetal",this.options.interpolation.alpha=.5))}},o.prototype.update=function(t){this.group=t,this.content=t.content||"graph",this.className=t.className||this.className||"vis-graph-group"+this.groupsUsingDefaultStyles[0]%10,this.visible=void 0===t.visible||t.visible,this.style=t.style,this.setOptions(t.options)},o.prototype.getLegend=function(t,e,i,o,n){if(void 0==i||null==i){i={svg:document.createElementNS("http://www.w3.org/2000/svg","svg"),svgElements:{},options:this.options,groups:[this]}}switch(void 0!=o&&null!=o||(o=0),void 0!=n&&null!=n||(n=.5*e),this.options.style){case"line":h.drawIcon(this,o,n,t,e,i);break;case"points":case"point":d.drawIcon(this,o,n,t,e,i);break;case"bar":a.drawIcon(this,o,n,t,e,i)}return{icon:i.svg,label:this.content,orientation:this.options.yAxisOrientation}},o.prototype.getYRange=function(t){for(var e=t[0].y,i=t[0].y,o=0;o<t.length;o++)e=e>t[o].y?t[o].y:e,i=i<t[o].y?t[o].y:i;return{min:e,max:i,yAxisOrientation:this.options.yAxisOrientation}},t.exports=o},function(t,e,i){function o(t,e){}var n=i(11),s=i(54);o.drawIcon=function(t,e,i,o,s,r){var a=.5*s,h=n.getSVGElement("rect",r.svgElements,r.svg);h.setAttributeNS(null,"x",e),h.setAttributeNS(null,"y",i-a),h.setAttributeNS(null,"width",o),h.setAttributeNS(null,"height",2*a),h.setAttributeNS(null,"class","vis-outline");var d=Math.round(.3*o),l=t.options.barChart.width,u=l/d,p=Math.round(.4*s),c=Math.round(.75*s),m=Math.round((o-2*d)/3);if(n.drawBar(e+.5*d+m,i+a-p-1,d,p,t.className+" vis-bar",r.svgElements,r.svg,t.style),n.drawBar(e+1.5*d+m+2,i+a-c-1,d,c,t.className+" vis-bar",r.svgElements,r.svg,t.style),1==t.options.drawPoints.enabled){var f={style:t.options.drawPoints.style,styles:t.options.drawPoints.styles,size:t.options.drawPoints.size/u,className:t.className};n.drawPoint(e+.5*d+m,i+a-p-1,f,r.svgElements,r.svg),n.drawPoint(e+1.5*d+m+2,i+a-c-1,f,r.svgElements,r.svg)}},o.draw=function(t,e,i){var r,a,h,d,l,u,p=[],c={},m=0;for(l=0;l<t.length;l++)if(d=i.groups[t[l]],"bar"===d.options.style&&!0===d.visible&&(void 0===i.options.groups.visibility[t[l]]||!0===i.options.groups.visibility[t[l]]))for(u=0;u<e[t[l]].length;u++)p.push({screen_x:e[t[l]][u].screen_x,screen_end:e[t[l]][u].screen_end,screen_y:e[t[l]][u].screen_y,x:e[t[l]][u].x,end:e[t[l]][u].end,y:e[t[l]][u].y,groupId:t[l],label:e[t[l]][u].label}),m+=1;if(0!==m)for(p.sort(function(t,e){return t.screen_x===e.screen_x?t.groupId<e.groupId?-1:1:t.screen_x-e.screen_x}),o._getDataIntersections(c,p),l=0;l<p.length;l++){d=i.groups[p[l].groupId];var f=void 0!=d.options.barChart.minWidth?d.options.barChart.minWidth:.1*d.options.barChart.width;a=p[l].screen_x;var g=0;if(void 0===c[a])l+1<p.length&&(r=Math.abs(p[l+1].screen_x-a)),h=o._getSafeDrawData(r,d,f);else{var v=l+(c[a].amount-c[a].resolved);v<p.length&&(r=Math.abs(p[v].screen_x-a)),h=o._getSafeDrawData(r,d,f),c[a].resolved+=1,!0===d.options.stack&&!0!==d.options.excludeFromStacking?p[l].screen_y<d.zeroPosition?(g=c[a].accumulatedNegative,c[a].accumulatedNegative+=d.zeroPosition-p[l].screen_y):(g=c[a].accumulatedPositive,c[a].accumulatedPositive+=d.zeroPosition-p[l].screen_y):!0===d.options.barChart.sideBySide&&(h.width=h.width/c[a].amount,h.offset+=c[a].resolved*h.width-.5*h.width*(c[a].amount+1))}var y=h.width,b=p[l].screen_x;if(void 0!=p[l].screen_end?(y=p[l].screen_end-p[l].screen_x,b+=.5*y):b+=h.offset,n.drawBar(b,p[l].screen_y-g,y,d.zeroPosition-p[l].screen_y,d.className+" vis-bar",i.svgElements,i.svg,d.style),!0===d.options.drawPoints.enabled){var _={screen_x:p[l].screen_x,screen_y:p[l].screen_y-g,x:p[l].x,y:p[l].y,groupId:p[l].groupId,label:p[l].label};s.draw([_],d,i,h.offset)}}},o._getDataIntersections=function(t,e){for(var i,o=0;o<e.length;o++)o+1<e.length&&(i=Math.abs(e[o+1].screen_x-e[o].screen_x)),o>0&&(i=Math.min(i,Math.abs(e[o-1].screen_x-e[o].screen_x))),0===i&&(void 0===t[e[o].screen_x]&&(t[e[o].screen_x]={amount:0,resolved:0,accumulatedPositive:0,accumulatedNegative:0}),t[e[o].screen_x].amount+=1)},o._getSafeDrawData=function(t,e,i){var o,n;return t<e.options.barChart.width&&t>0?(o=t<i?i:t,n=0,"left"===e.options.barChart.align?n-=.5*t:"right"===e.options.barChart.align&&(n+=.5*t)):(o=e.options.barChart.width,n=0,"left"===e.options.barChart.align?n-=.5*e.options.barChart.width:"right"===e.options.barChart.align&&(n+=.5*e.options.barChart.width)),{width:o,offset:n}},o.getStackedYRange=function(t,e,i,n,s){if(t.length>0){t.sort(function(t,e){return t.screen_x===e.screen_x?t.groupId<e.groupId?-1:1:t.screen_x-e.screen_x});var r={};o._getDataIntersections(r,t),e[n]=o._getStackedYRange(r,t),e[n].yAxisOrientation=s,i.push(n)}},o._getStackedYRange=function(t,e){for(var i,o=e[0].screen_y,n=e[0].screen_y,s=0;s<e.length;s++)i=e[s].screen_x,void 0===t[i]?(o=o>e[s].screen_y?e[s].screen_y:o,n=n<e[s].screen_y?e[s].screen_y:n):e[s].screen_y<0?t[i].accumulatedNegative+=e[s].screen_y:t[i].accumulatedPositive+=e[s].screen_y;for(var r in t)t.hasOwnProperty(r)&&(o=o>t[r].accumulatedNegative?t[r].accumulatedNegative:o,o=o>t[r].accumulatedPositive?t[r].accumulatedPositive:o,n=n<t[r].accumulatedNegative?t[r].accumulatedNegative:n,n=n<t[r].accumulatedPositive?t[r].accumulatedPositive:n);return{min:o,max:n}},t.exports=o},function(t,e,i){function o(t,e){}var n=i(11);o.calcPath=function(t,e){if(null!=t&&t.length>0){return 1==e.options.interpolation.enabled?o._catmullRom(t,e):o._linear(t)}},o.drawIcon=function(t,e,i,o,s,r){var a,h,d=.5*s,l=n.getSVGElement("rect",r.svgElements,r.svg);if(l.setAttributeNS(null,"x",e),l.setAttributeNS(null,"y",i-d),l.setAttributeNS(null,"width",o),l.setAttributeNS(null,"height",2*d),l.setAttributeNS(null,"class","vis-outline"),a=n.getSVGElement("path",r.svgElements,r.svg),a.setAttributeNS(null,"class",t.className),void 0!==t.style&&a.setAttributeNS(null,"style",t.style),a.setAttributeNS(null,"d","M"+e+","+i+" L"+(e+o)+","+i),1==t.options.shaded.enabled&&(h=n.getSVGElement("path",r.svgElements,r.svg),"top"==t.options.shaded.orientation?h.setAttributeNS(null,"d","M"+e+", "+(i-d)+"L"+e+","+i+" L"+(e+o)+","+i+" L"+(e+o)+","+(i-d)):h.setAttributeNS(null,"d","M"+e+","+i+" L"+e+","+(i+d)+" L"+(e+o)+","+(i+d)+"L"+(e+o)+","+i),h.setAttributeNS(null,"class",t.className+" vis-icon-fill"),void 0!==t.options.shaded.style&&""!==t.options.shaded.style&&h.setAttributeNS(null,"style",t.options.shaded.style)),1==t.options.drawPoints.enabled){var u={style:t.options.drawPoints.style,styles:t.options.drawPoints.styles,size:t.options.drawPoints.size,className:t.className};n.drawPoint(e+.5*o,i,u,r.svgElements,r.svg)}},o.drawShading=function(t,e,i,o){if(1==e.options.shaded.enabled){var s=Number(o.svg.style.height.replace("px","")),r=n.getSVGElement("path",o.svgElements,o.svg),a="L";1==e.options.interpolation.enabled&&(a="C");var h,d=0;d="top"==e.options.shaded.orientation?0:"bottom"==e.options.shaded.orientation?s:Math.min(Math.max(0,e.zeroPosition),s),h="group"==e.options.shaded.orientation&&null!=i&&void 0!=i?"M"+t[0][0]+","+t[0][1]+" "+this.serializePath(t,a,!1)+" L"+i[i.length-1][0]+","+i[i.length-1][1]+" "+this.serializePath(i,a,!0)+i[0][0]+","+i[0][1]+" Z":"M"+t[0][0]+","+t[0][1]+" "+this.serializePath(t,a,!1)+" V"+d+" H"+t[0][0]+" Z",r.setAttributeNS(null,"class",e.className+" vis-fill"),void 0!==e.options.shaded.style&&r.setAttributeNS(null,"style",e.options.shaded.style),r.setAttributeNS(null,"d",h)}},o.draw=function(t,e,i){if(null!=t&&void 0!=t){var o=n.getSVGElement("path",i.svgElements,i.svg);o.setAttributeNS(null,"class",e.className),void 0!==e.style&&o.setAttributeNS(null,"style",e.style);var s="L";1==e.options.interpolation.enabled&&(s="C"),o.setAttributeNS(null,"d","M"+t[0][0]+","+t[0][1]+" "+this.serializePath(t,s,!1))}},o.serializePath=function(t,e,i){if(t.length<2)return"";var o,n=e;if(i)for(o=t.length-2;o>0;o--)n+=t[o][0]+","+t[o][1]+" ";else for(o=1;o<t.length;o++)n+=t[o][0]+","+t[o][1]+" ";return n},o._catmullRomUniform=function(t){var e,i,o,n,s,r,a=[];a.push([Math.round(t[0].screen_x),Math.round(t[0].screen_y)]);for(var h=t.length,d=0;d<h-1;d++)e=0==d?t[0]:t[d-1],i=t[d],o=t[d+1],n=d+2<h?t[d+2]:o,s={screen_x:(-e.screen_x+6*i.screen_x+o.screen_x)*(1/6),screen_y:(-e.screen_y+6*i.screen_y+o.screen_y)*(1/6)},r={screen_x:(i.screen_x+6*o.screen_x-n.screen_x)*(1/6),screen_y:(i.screen_y+6*o.screen_y-n.screen_y)*(1/6)},a.push([s.screen_x,s.screen_y]),a.push([r.screen_x,r.screen_y]),a.push([o.screen_x,o.screen_y]);return a},o._catmullRom=function(t,e){var i=e.options.interpolation.alpha;if(0==i||void 0===i)return this._catmullRomUniform(t);var o,n,s,r,a,h,d,l,u,p,c,m,f,g,v,y,b,_,w,x=[];x.push([Math.round(t[0].screen_x),Math.round(t[0].screen_y)]);for(var D=t.length,S=0;S<D-1;S++)o=0==S?t[0]:t[S-1],n=t[S],s=t[S+1],r=S+2<D?t[S+2]:s,d=Math.sqrt(Math.pow(o.screen_x-n.screen_x,2)+Math.pow(o.screen_y-n.screen_y,2)),l=Math.sqrt(Math.pow(n.screen_x-s.screen_x,2)+Math.pow(n.screen_y-s.screen_y,2)),u=Math.sqrt(Math.pow(s.screen_x-r.screen_x,2)+Math.pow(s.screen_y-r.screen_y,2)),g=Math.pow(u,i),y=Math.pow(u,2*i),v=Math.pow(l,i),b=Math.pow(l,2*i),w=Math.pow(d,i),_=Math.pow(d,2*i),p=2*_+3*w*v+b,c=2*y+3*g*v+b,m=3*w*(w+v),m>0&&(m=1/m),f=3*g*(g+v),f>0&&(f=1/f),a={screen_x:(-b*o.screen_x+p*n.screen_x+_*s.screen_x)*m,screen_y:(-b*o.screen_y+p*n.screen_y+_*s.screen_y)*m},h={screen_x:(y*n.screen_x+c*s.screen_x-b*r.screen_x)*f,screen_y:(y*n.screen_y+c*s.screen_y-b*r.screen_y)*f},0==a.screen_x&&0==a.screen_y&&(a=n),0==h.screen_x&&0==h.screen_y&&(h=s),x.push([a.screen_x,a.screen_y]),x.push([h.screen_x,h.screen_y]),x.push([s.screen_x,s.screen_y]);return x},o._linear=function(t){for(var e=[],i=0;i<t.length;i++)e.push([t[i].screen_x,t[i].screen_y]);return e},t.exports=o},function(t,e,i){function o(t,e,i,o){this.body=t,this.defaultOptions={enabled:!1,icons:!0,iconSize:20,iconSpacing:6,left:{visible:!0,position:"top-left"},right:{visible:!0,position:"top-right"}},this.side=i,this.options=r.extend({},this.defaultOptions),this.linegraphOptions=o,this.svgElements={},this.dom={},this.groups={},this.amountOfGroups=0,this._create(),this.framework={svg:this.svg,svgElements:this.svgElements,options:this.options,groups:this.groups},this.setOptions(e)}var n=i(4),s=function(t){return t&&t.__esModule?t:{default:t}}(n),r=i(0),a=i(11),h=i(5);o.prototype=new h,o.prototype.clear=function(){this.groups={},this.amountOfGroups=0},o.prototype.addGroup=function(t,e){1!=e.options.excludeFromLegend&&(this.groups.hasOwnProperty(t)||(this.groups[t]=e),this.amountOfGroups+=1)},o.prototype.updateGroup=function(t,e){this.groups[t]=e},o.prototype.removeGroup=function(t){this.groups.hasOwnProperty(t)&&(delete this.groups[t],this.amountOfGroups-=1)},o.prototype._create=function(){this.dom.frame=document.createElement("div"),this.dom.frame.className="vis-legend",this.dom.frame.style.position="absolute",this.dom.frame.style.top="10px",this.dom.frame.style.display="block",this.dom.textArea=document.createElement("div"),this.dom.textArea.className="vis-legend-text",this.dom.textArea.style.position="relative",this.dom.textArea.style.top="0px",this.svg=document.createElementNS("http://www.w3.org/2000/svg","svg"),this.svg.style.position="absolute",this.svg.style.top="0px",this.svg.style.width=this.options.iconSize+5+"px",this.svg.style.height="100%",this.dom.frame.appendChild(this.svg),this.dom.frame.appendChild(this.dom.textArea)},o.prototype.hide=function(){this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame)},o.prototype.show=function(){this.dom.frame.parentNode||this.body.dom.center.appendChild(this.dom.frame)},o.prototype.setOptions=function(t){var e=["enabled","orientation","icons","left","right"];r.selectiveDeepExtend(e,this.options,t)},o.prototype.redraw=function(){var t=0,e=(0,s.default)(this.groups);e.sort(function(t,e){return t<e?-1:1});for(var i=0;i<e.length;i++){var o=e[i];1!=this.groups[o].visible||void 0!==this.linegraphOptions.visibility[o]&&1!=this.linegraphOptions.visibility[o]||t++}if(0==this.options[this.side].visible||0==this.amountOfGroups||0==this.options.enabled||0==t)this.hide();else{if(this.show(),"top-left"==this.options[this.side].position||"bottom-left"==this.options[this.side].position?(this.dom.frame.style.left="4px",this.dom.frame.style.textAlign="left",this.dom.textArea.style.textAlign="left",this.dom.textArea.style.left=this.options.iconSize+15+"px",this.dom.textArea.style.right="",this.svg.style.left="0px",this.svg.style.right=""):(this.dom.frame.style.right="4px",this.dom.frame.style.textAlign="right",this.dom.textArea.style.textAlign="right",this.dom.textArea.style.right=this.options.iconSize+15+"px",this.dom.textArea.style.left="",this.svg.style.right="0px",this.svg.style.left=""),"top-left"==this.options[this.side].position||"top-right"==this.options[this.side].position)this.dom.frame.style.top=4-Number(this.body.dom.center.style.top.replace("px",""))+"px",this.dom.frame.style.bottom="";else{var n=this.body.domProps.center.height-this.body.domProps.centerContainer.height;this.dom.frame.style.bottom=4+n+Number(this.body.dom.center.style.top.replace("px",""))+"px",this.dom.frame.style.top=""}0==this.options.icons?(this.dom.frame.style.width=this.dom.textArea.offsetWidth+10+"px",this.dom.textArea.style.right="",this.dom.textArea.style.left="",this.svg.style.width="0px"):(this.dom.frame.style.width=this.options.iconSize+15+this.dom.textArea.offsetWidth+10+"px",this.drawLegendIcons());var r="";for(i=0;i<e.length;i++)o=e[i],1!=this.groups[o].visible||void 0!==this.linegraphOptions.visibility[o]&&1!=this.linegraphOptions.visibility[o]||(r+=this.groups[o].content+"<br />");this.dom.textArea.innerHTML=r,this.dom.textArea.style.lineHeight=.75*this.options.iconSize+this.options.iconSpacing+"px"}},o.prototype.drawLegendIcons=function(){if(this.dom.frame.parentNode){var t=(0,s.default)(this.groups);t.sort(function(t,e){return t<e?-1:1}),a.resetElements(this.svgElements);var e=window.getComputedStyle(this.dom.frame).paddingTop,i=Number(e.replace("px","")),o=i,n=this.options.iconSize,r=.75*this.options.iconSize,h=i+.5*r+3;this.svg.style.width=n+5+i+"px";for(var d=0;d<t.length;d++){var l=t[d];1!=this.groups[l].visible||void 0!==this.linegraphOptions.visibility[l]&&1!=this.linegraphOptions.visibility[l]||(this.groups[l].getLegend(n,r,this.framework,o,h),h+=r+this.options.iconSpacing)}}},t.exports=o},function(t,e,i){Object.defineProperty(e,"__esModule",{value:!0});var o="string",n="boolean",s="number",r="object",a={configure:{enabled:{boolean:n},filter:{boolean:n,function:"function"},container:{dom:"dom"},__type__:{object:r,boolean:n,function:"function"}},yAxisOrientation:{string:["left","right"]},defaultGroup:{string:o},sort:{boolean:n},sampling:{boolean:n},stack:{boolean:n},graphHeight:{string:o,number:s},shaded:{enabled:{boolean:n},orientation:{string:["bottom","top","zero","group"]},groupId:{object:r},__type__:{boolean:n,object:r}},style:{string:["line","bar","points"]},barChart:{width:{number:s},minWidth:{number:s},sideBySide:{boolean:n},align:{string:["left","center","right"]},__type__:{object:r}},interpolation:{enabled:{boolean:n},parametrization:{string:["centripetal","chordal","uniform"]},alpha:{number:s},__type__:{object:r,boolean:n}},drawPoints:{enabled:{boolean:n},onRender:{function:"function"},size:{number:s},style:{string:["square","circle"]},__type__:{object:r,boolean:n,function:"function"}},dataAxis:{showMinorLabels:{boolean:n},showMajorLabels:{boolean:n},icons:{boolean:n},width:{string:o,number:s},visible:{boolean:n},alignZeros:{boolean:n},left:{range:{min:{number:s,undefined:"undefined"},max:{number:s,undefined:"undefined"},__type__:{object:r}},format:{function:"function"},title:{text:{string:o,number:s,undefined:"undefined"},style:{string:o,undefined:"undefined"},__type__:{object:r}},__type__:{object:r}},right:{range:{min:{number:s,undefined:"undefined"},max:{number:s,undefined:"undefined"},__type__:{object:r}},format:{function:"function"},title:{text:{string:o,number:s,undefined:"undefined"},style:{string:o,undefined:"undefined"},__type__:{object:r}},__type__:{object:r}},__type__:{object:r}},legend:{enabled:{boolean:n},icons:{boolean:n},left:{visible:{boolean:n},position:{string:["top-right","bottom-right","top-left","bottom-left"]},__type__:{object:r}},right:{visible:{boolean:n},position:{string:["top-right","bottom-right","top-left","bottom-left"]},__type__:{object:r}},__type__:{object:r,boolean:n}},groups:{visibility:{any:"any"},__type__:{object:r}},autoResize:{boolean:n},throttleRedraw:{number:s},clickToUse:{boolean:n},end:{number:s,date:"date",string:o,moment:"moment"},format:{minorLabels:{millisecond:{string:o,undefined:"undefined"},second:{string:o,undefined:"undefined"},minute:{string:o,undefined:"undefined"},hour:{string:o,undefined:"undefined"},weekday:{string:o,undefined:"undefined"},day:{string:o,undefined:"undefined"},month:{string:o,undefined:"undefined"},year:{string:o,undefined:"undefined"},__type__:{object:r}},majorLabels:{millisecond:{string:o,undefined:"undefined"},second:{string:o,undefined:"undefined"},minute:{string:o,undefined:"undefined"},hour:{string:o,undefined:"undefined"},weekday:{string:o,undefined:"undefined"},day:{string:o,undefined:"undefined"},month:{string:o,undefined:"undefined"},year:{string:o,undefined:"undefined"},__type__:{object:r}},__type__:{object:r}},moment:{function:"function"},height:{string:o,number:s},hiddenDates:{start:{date:"date",number:s,string:o,moment:"moment"},end:{date:"date",number:s,string:o,moment:"moment"},repeat:{string:o},__type__:{object:r,array:"array"}},locale:{string:o},locales:{__any__:{any:"any"},__type__:{object:r}},max:{date:"date",number:s,string:o,moment:"moment"},maxHeight:{number:s,string:o},maxMinorChars:{number:s},min:{date:"date",number:s,string:o,moment:"moment"},minHeight:{number:s,string:o},moveable:{boolean:n},multiselect:{boolean:n},orientation:{string:o},showCurrentTime:{boolean:n},showMajorLabels:{boolean:n},showMinorLabels:{boolean:n},start:{date:"date",number:s,string:o,moment:"moment"},timeAxis:{scale:{string:o,undefined:"undefined"},step:{number:s,undefined:"undefined"},__type__:{object:r}},width:{string:o,number:s},zoomable:{boolean:n},zoomKey:{string:["ctrlKey","altKey","metaKey",""]},zoomMax:{number:s},zoomMin:{number:s},zIndex:{number:s},__type__:{object:r}},h={global:{sort:!0,sampling:!0,stack:!1,shaded:{enabled:!1,orientation:["zero","top","bottom","group"]},style:["line","bar","points"],barChart:{width:[50,5,100,5],minWidth:[50,5,100,5],sideBySide:!1,align:["left","center","right"]},interpolation:{enabled:!0,parametrization:["centripetal","chordal","uniform"]},drawPoints:{enabled:!0,size:[6,2,30,1],style:["square","circle"]},dataAxis:{showMinorLabels:!0,showMajorLabels:!0,icons:!1,width:[40,0,200,1],visible:!0,alignZeros:!0,left:{title:{text:"",style:""}},right:{title:{text:"",style:""}}},legend:{enabled:!1,icons:!0,left:{visible:!0,position:["top-right","bottom-right","top-left","bottom-left"]},right:{visible:!0,position:["top-right","bottom-right","top-left","bottom-left"]}},autoResize:!0,clickToUse:!1,end:"",format:{minorLabels:{millisecond:"SSS",second:"s",minute:"HH:mm",hour:"HH:mm",weekday:"ddd D",day:"D",month:"MMM",year:"YYYY"},majorLabels:{millisecond:"HH:mm:ss",second:"D MMMM HH:mm",minute:"ddd D MMMM",hour:"ddd D MMMM",weekday:"MMMM YYYY",day:"MMMM YYYY",month:"YYYY",year:""}},height:"",locale:"",max:"",maxHeight:"",maxMinorChars:[7,0,20,1],min:"",minHeight:"",moveable:!0,orientation:["both","bottom","top"],showCurrentTime:!1,showMajorLabels:!0,showMinorLabels:!0,start:"",width:"100%",zoomable:!0,zoomKey:["ctrlKey","altKey","metaKey",""],zoomMax:[31536e10,10,31536e10,1],zoomMin:[10,10,31536e10,1],zIndex:0}};e.allOptions=a,e.configureOptions=h},function(t,e,i){e.util=i(0),e.DOMutil=i(11),e.DataSet=i(15),e.DataView=i(20),e.Queue=i(65),e.Timeline=i(129),e.Graph2d=i(138),e.timeline={Core:i(48),DateUtil:i(21),Range:i(47),stack:i(71),TimeStep:i(49),components:{items:{Item:i(22),BackgroundItem:i(74),BoxItem:i(72),PointItem:i(73),RangeItem:i(53)},BackgroundGroup:i(52),Component:i(5),CurrentTime:i(50),CustomTime:i(29),DataAxis:i(78),DataScale:i(79),GraphGroup:i(80),Group:i(51),ItemSet:i(70),Legend:i(83),LineGraph:i(77),TimeAxis:i(28)}},e.moment=i(3),e.Hammer=i(12),e.keycharm=i(68)},function(t,e,i){t.exports={default:i(87),__esModule:!0}},function(t,e,i){i(55),i(62),t.exports=i(102)},function(t,e,i){var o=i(89),n=i(90),s=i(23),r=i(13);t.exports=i(56)(Array,"Array",function(t,e){this._t=r(t),this._i=0,this._k=e},function(){var t=this._t,e=this._k,i=this._i++;return!t||i>=t.length?(this._t=void 0,n(1)):"keys"==e?n(0,i):"values"==e?n(0,t[i]):n(0,[i,t[i]])},"values"),s.Arguments=s.Array,o("keys"),o("values"),o("entries")},function(t,e){t.exports=function(){}},function(t,e){t.exports=function(t,e){return{value:e,done:!!t}}},function(t,e,i){var o=i(33);t.exports=Object("z").propertyIsEnumerable(0)?Object:function(t){return"String"==o(t)?t.split(""):Object(t)}},function(t,e,i){var o=i(93);t.exports=function(t,e,i){if(o(t),void 0===e)return t;switch(i){case 1:return function(i){return t.call(e,i)};case 2:return function(i,o){return t.call(e,i,o)};case 3:return function(i,o,n){return t.call(e,i,o,n)}}return function(){return t.apply(e,arguments)}}},function(t,e){t.exports=function(t){if("function"!=typeof t)throw TypeError(t+" is not a function!");return t}},function(t,e,i){var o=i(37),n=i(25),s=i(42),r={};i(14)(r,i(7)("iterator"),function(){return this}),t.exports=function(t,e,i){t.prototype=o(r,{next:n(1,i)}),s(t,e+" Iterator")}},function(t,e,i){var o=i(8),n=i(18),s=i(26);t.exports=i(9)?Object.defineProperties:function(t,e){n(t);for(var i,r=s(e),a=r.length,h=0;a>h;)o.f(t,i=r[h++],e[i]);return t}},function(t,e,i){var o=i(13),n=i(97),s=i(98);t.exports=function(t){return function(e,i,r){var a,h=o(e),d=n(h.length),l=s(r,d);if(t&&i!=i){for(;d>l;)if((a=h[l++])!=a)return!0}else for(;d>l;l++)if((t||l in h)&&h[l]===i)return t||l||0;return!t&&-1}}},function(t,e,i){var o=i(38),n=Math.min;t.exports=function(t){return t>0?n(o(t),9007199254740991):0}},function(t,e,i){var o=i(38),n=Math.max,s=Math.min;t.exports=function(t,e){return t=o(t),t<0?n(t+e,0):s(t,e)}},function(t,e,i){var o=i(6).document;t.exports=o&&o.documentElement},function(t,e,i){var o=i(10),n=i(61),s=i(39)("IE_PROTO"),r=Object.prototype;t.exports=Object.getPrototypeOf||function(t){return t=n(t),o(t,s)?t[s]:"function"==typeof t.constructor&&t instanceof t.constructor?t.constructor.prototype:t instanceof Object?r:null}},function(t,e,i){var o=i(38),n=i(34);t.exports=function(t){return function(e,i){var s,r,a=String(n(e)),h=o(i),d=a.length;return h<0||h>=d?t?"":void 0:(s=a.charCodeAt(h),s<55296||s>56319||h+1===d||(r=a.charCodeAt(h+1))<56320||r>57343?t?a.charAt(h):s:t?a.slice(h,h+2):r-56320+(s-55296<<10)+65536)}}},function(t,e,i){var o=i(18),n=i(103);t.exports=i(2).getIterator=function(t){var e=n(t);if("function"!=typeof e)throw TypeError(t+" is not iterable!");return o(e.call(t))}},function(t,e,i){var o=i(104),n=i(7)("iterator"),s=i(23);t.exports=i(2).getIteratorMethod=function(t){if(void 0!=t)return t[n]||t["@@iterator"]||s[o(t)]}},function(t,e,i){var o=i(33),n=i(7)("toStringTag"),s="Arguments"==o(function(){return arguments}()),r=function(t,e){try{return t[e]}catch(t){}};t.exports=function(t){var e,i,a;return void 0===t?"Undefined":null===t?"Null":"string"==typeof(i=r(e=Object(t),n))?i:s?o(e):"Object"==(a=o(e))&&"function"==typeof e.callee?"Arguments":a}},function(t,e,i){i(106);var o=i(2).Object;t.exports=function(t,e){return o.create(t,e)}},function(t,e,i){var o=i(17);o(o.S,"Object",{create:i(37)})},function(t,e,i){i(108),t.exports=i(2).Object.keys},function(t,e,i){var o=i(61),n=i(26);i(109)("keys",function(){return function(t){return n(o(t))}})},function(t,e,i){var o=i(17),n=i(2),s=i(19);t.exports=function(t,e){var i=(n.Object||{})[t]||Object[t],r={};r[t]=e(i),o(o.S+o.F*s(function(){i(1)}),"Object",r)}},function(t,e,i){t.exports={default:i(111),__esModule:!0}},function(t,e,i){i(62),i(55),t.exports=i(44).f("iterator")},function(t,e,i){t.exports={default:i(113),__esModule:!0}},function(t,e,i){i(114),i(120),i(121),i(122),t.exports=i(2).Symbol},function(t,e,i){var o=i(6),n=i(10),s=i(9),r=i(17),a=i(59),h=i(115).KEY,d=i(19),l=i(40),u=i(42),p=i(27),c=i(7),m=i(44),f=i(45),g=i(116),v=i(117),y=i(18),b=i(13),_=i(36),w=i(25),x=i(37),D=i(118),S=i(119),k=i(8),C=i(26),T=S.f,M=k.f,O=D.f,E=o.Symbol,P=o.JSON,I=P&&P.stringify,N=c("_hidden"),A=c("toPrimitive"),R={}.propertyIsEnumerable,L=l("symbol-registry"),F=l("symbols"),H=l("op-symbols"),j=Object.prototype,Y="function"==typeof E,G=o.QObject,z=!G||!G.prototype||!G.prototype.findChild,W=s&&d(function(){return 7!=x(M({},"a",{get:function(){return M(this,"a",{value:7}).a}})).a})?function(t,e,i){var o=T(j,e);o&&delete j[e],M(t,e,i),o&&t!==j&&M(j,e,o)}:M,V=function(t){var e=F[t]=x(E.prototype);return e._k=t,e},B=Y&&"symbol"==typeof E.iterator?function(t){return"symbol"==typeof t}:function(t){return t instanceof E},U=function(t,e,i){return t===j&&U(H,e,i),y(t),e=_(e,!0),y(i),n(F,e)?(i.enumerable?(n(t,N)&&t[N][e]&&(t[N][e]=!1),i=x(i,{enumerable:w(0,!1)})):(n(t,N)||M(t,N,w(1,{})),t[N][e]=!0),W(t,e,i)):M(t,e,i)},q=function(t,e){y(t);for(var i,o=g(e=b(e)),n=0,s=o.length;s>n;)U(t,i=o[n++],e[i]);return t},X=function(t,e){return void 0===e?x(t):q(x(t),e)},Z=function(t){var e=R.call(this,t=_(t,!0));return!(this===j&&n(F,t)&&!n(H,t))&&(!(e||!n(this,t)||!n(F,t)||n(this,N)&&this[N][t])||e)},K=function(t,e){if(t=b(t),e=_(e,!0),t!==j||!n(F,e)||n(H,e)){var i=T(t,e);return!i||!n(F,e)||n(t,N)&&t[N][e]||(i.enumerable=!0),i}},J=function(t){for(var e,i=O(b(t)),o=[],s=0;i.length>s;)n(F,e=i[s++])||e==N||e==h||o.push(e);return o},$=function(t){for(var e,i=t===j,o=O(i?H:b(t)),s=[],r=0;o.length>r;)!n(F,e=o[r++])||i&&!n(j,e)||s.push(F[e]);return s};Y||(E=function(){if(this instanceof E)throw TypeError("Symbol is not a constructor!");var t=p(arguments.length>0?arguments[0]:void 0),e=function(i){this===j&&e.call(H,i),n(this,N)&&n(this[N],t)&&(this[N][t]=!1),W(this,t,w(1,i))};return s&&z&&W(j,t,{configurable:!0,set:e}),V(t)},a(E.prototype,"toString",function(){return this._k}),S.f=K,k.f=U,i(64).f=D.f=J,i(46).f=Z,i(63).f=$,s&&!i(35)&&a(j,"propertyIsEnumerable",Z,!0),m.f=function(t){return V(c(t))}),r(r.G+r.W+r.F*!Y,{Symbol:E});for(var Q="hasInstance,isConcatSpreadable,iterator,match,replace,search,species,split,toPrimitive,toStringTag,unscopables".split(","),tt=0;Q.length>tt;)c(Q[tt++]);for(var et=C(c.store),it=0;et.length>it;)f(et[it++]);r(r.S+r.F*!Y,"Symbol",{for:function(t){return n(L,t+="")?L[t]:L[t]=E(t)},keyFor:function(t){if(!B(t))throw TypeError(t+" is not a symbol!");for(var e in L)if(L[e]===t)return e},useSetter:function(){z=!0},useSimple:function(){z=!1}}),r(r.S+r.F*!Y,"Object",{create:X,defineProperty:U,defineProperties:q,getOwnPropertyDescriptor:K,getOwnPropertyNames:J,getOwnPropertySymbols:$}),P&&r(r.S+r.F*(!Y||d(function(){var t=E();return"[null]"!=I([t])||"{}"!=I({a:t})||"{}"!=I(Object(t))})),"JSON",{stringify:function(t){if(void 0!==t&&!B(t)){for(var e,i,o=[t],n=1;arguments.length>n;)o.push(arguments[n++]);return e=o[1],"function"==typeof e&&(i=e),!i&&v(e)||(e=function(t,e){if(i&&(e=i.call(this,t,e)),!B(e))return e}),o[1]=e,I.apply(P,o)}}}),E.prototype[A]||i(14)(E.prototype,A,E.prototype.valueOf),u(E,"Symbol"),u(Math,"Math",!0),u(o.JSON,"JSON",!0)},function(t,e,i){var o=i(27)("meta"),n=i(24),s=i(10),r=i(8).f,a=0,h=Object.isExtensible||function(){return!0},d=!i(19)(function(){return h(Object.preventExtensions({}))}),l=function(t){r(t,o,{value:{i:"O"+ ++a,w:{}}})},u=function(t,e){if(!n(t))return"symbol"==typeof t?t:("string"==typeof t?"S":"P")+t;if(!s(t,o)){if(!h(t))return"F";if(!e)return"E";l(t)}return t[o].i},p=function(t,e){if(!s(t,o)){if(!h(t))return!0;if(!e)return!1;l(t)}return t[o].w},c=function(t){return d&&m.NEED&&h(t)&&!s(t,o)&&l(t),t},m=t.exports={KEY:o,NEED:!1,fastKey:u,getWeak:p,onFreeze:c}},function(t,e,i){var o=i(26),n=i(63),s=i(46);t.exports=function(t){var e=o(t),i=n.f;if(i)for(var r,a=i(t),h=s.f,d=0;a.length>d;)h.call(t,r=a[d++])&&e.push(r);return e}},function(t,e,i){var o=i(33);t.exports=Array.isArray||function(t){return"Array"==o(t)}},function(t,e,i){var o=i(13),n=i(64).f,s={}.toString,r="object"==typeof window&&window&&Object.getOwnPropertyNames?Object.getOwnPropertyNames(window):[],a=function(t){try{return n(t)}catch(t){return r.slice()}};t.exports.f=function(t){return r&&"[object Window]"==s.call(t)?a(t):n(o(t))}},function(t,e,i){var o=i(46),n=i(25),s=i(13),r=i(36),a=i(10),h=i(57),d=Object.getOwnPropertyDescriptor;e.f=i(9)?d:function(t,e){if(t=s(t),e=r(e,!0),h)try{return d(t,e)}catch(t){}if(a(t,e))return n(!o.f.call(t,e),t[e])}},function(t,e){},function(t,e,i){i(45)("asyncIterator")},function(t,e,i){i(45)("observable")},function(t,e,i){(function(t){!function(e,i){t.exports=i()}(0,function(){function e(){return To.apply(null,arguments)}function i(t){return t instanceof Array||"[object Array]"===Object.prototype.toString.call(t)}function o(t){return null!=t&&"[object Object]"===Object.prototype.toString.call(t)}function n(t){if(Object.getOwnPropertyNames)return 0===Object.getOwnPropertyNames(t).length;var e;for(e in t)if(t.hasOwnProperty(e))return!1;return!0}function s(t){return void 0===t}function r(t){return"number"==typeof t||"[object Number]"===Object.prototype.toString.call(t)}function a(t){return t instanceof Date||"[object Date]"===Object.prototype.toString.call(t)}function h(t,e){var i,o=[];for(i=0;i<t.length;++i)o.push(e(t[i],i));return o}function d(t,e){return Object.prototype.hasOwnProperty.call(t,e)}function l(t,e){for(var i in e)d(e,i)&&(t[i]=e[i]);return d(e,"toString")&&(t.toString=e.toString),d(e,"valueOf")&&(t.valueOf=e.valueOf),t}function u(t,e,i,o){return Se(t,e,i,o,!0).utc()}function p(){return{empty:!1,unusedTokens:[],unusedInput:[],overflow:-2,charsLeftOver:0,nullInput:!1,invalidMonth:null,invalidFormat:!1,userInvalidated:!1,iso:!1,parsedDateParts:[],meridiem:null,rfc2822:!1,weekdayMismatch:!1}}function c(t){return null==t._pf&&(t._pf=p()),t._pf}function m(t){if(null==t._isValid){var e=c(t),i=Mo.call(e.parsedDateParts,function(t){return null!=t}),o=!isNaN(t._d.getTime())&&e.overflow<0&&!e.empty&&!e.invalidMonth&&!e.invalidWeekday&&!e.weekdayMismatch&&!e.nullInput&&!e.invalidFormat&&!e.userInvalidated&&(!e.meridiem||e.meridiem&&i);if(t._strict&&(o=o&&0===e.charsLeftOver&&0===e.unusedTokens.length&&void 0===e.bigHour),null!=Object.isFrozen&&Object.isFrozen(t))return o;t._isValid=o}return t._isValid}function f(t){var e=u(NaN);return null!=t?l(c(e),t):c(e).userInvalidated=!0,e}function g(t,e){var i,o,n;if(s(e._isAMomentObject)||(t._isAMomentObject=e._isAMomentObject),s(e._i)||(t._i=e._i),s(e._f)||(t._f=e._f),s(e._l)||(t._l=e._l),s(e._strict)||(t._strict=e._strict),s(e._tzm)||(t._tzm=e._tzm),s(e._isUTC)||(t._isUTC=e._isUTC),s(e._offset)||(t._offset=e._offset),s(e._pf)||(t._pf=c(e)),s(e._locale)||(t._locale=e._locale),Oo.length>0)for(i=0;i<Oo.length;i++)o=Oo[i],n=e[o],s(n)||(t[o]=n);return t}function v(t){g(this,t),this._d=new Date(null!=t._d?t._d.getTime():NaN),this.isValid()||(this._d=new Date(NaN)),!1===Eo&&(Eo=!0,e.updateOffset(this),Eo=!1)}function y(t){return t instanceof v||null!=t&&null!=t._isAMomentObject}function b(t){return t<0?Math.ceil(t)||0:Math.floor(t)}function _(t){var e=+t,i=0;return 0!==e&&isFinite(e)&&(i=b(e)),i}function w(t,e,i){var o,n=Math.min(t.length,e.length),s=Math.abs(t.length-e.length),r=0;for(o=0;o<n;o++)(i&&t[o]!==e[o]||!i&&_(t[o])!==_(e[o]))&&r++;return r+s}function x(t){!1===e.suppressDeprecationWarnings&&"undefined"!=typeof console&&console.warn&&console.warn("Deprecation warning: "+t)}function D(t,i){var o=!0;return l(function(){if(null!=e.deprecationHandler&&e.deprecationHandler(null,t),o){for(var n,s=[],r=0;r<arguments.length;r++){if(n="","object"==typeof arguments[r]){n+="\n["+r+"] ";for(var a in arguments[0])n+=a+": "+arguments[0][a]+", ";n=n.slice(0,-2)}else n=arguments[r];s.push(n)}x(t+"\nArguments: "+Array.prototype.slice.call(s).join("")+"\n"+(new Error).stack),o=!1}return i.apply(this,arguments)},i)}function S(t,i){null!=e.deprecationHandler&&e.deprecationHandler(t,i),Po[t]||(x(i),Po[t]=!0)}function k(t){return t instanceof Function||"[object Function]"===Object.prototype.toString.call(t)}function C(t){var e,i;for(i in t)e=t[i],k(e)?this[i]=e:this["_"+i]=e;this._config=t,this._dayOfMonthOrdinalParseLenient=new RegExp((this._dayOfMonthOrdinalParse.source||this._ordinalParse.source)+"|"+/\d{1,2}/.source)}function T(t,e){var i,n=l({},t);for(i in e)d(e,i)&&(o(t[i])&&o(e[i])?(n[i]={},l(n[i],t[i]),l(n[i],e[i])):null!=e[i]?n[i]=e[i]:delete n[i]);for(i in t)d(t,i)&&!d(e,i)&&o(t[i])&&(n[i]=l({},n[i]));return n}function M(t){null!=t&&this.set(t)}function O(t,e,i){var o=this._calendar[t]||this._calendar.sameElse;return k(o)?o.call(e,i):o}function E(t){var e=this._longDateFormat[t],i=this._longDateFormat[t.toUpperCase()];return e||!i?e:(this._longDateFormat[t]=i.replace(/MMMM|MM|DD|dddd/g,function(t){return t.slice(1)}),this._longDateFormat[t])}function P(){return this._invalidDate}function I(t){return this._ordinal.replace("%d",t)}function N(t,e,i,o){var n=this._relativeTime[i];return k(n)?n(t,e,i,o):n.replace(/%d/i,t)}function A(t,e){var i=this._relativeTime[t>0?"future":"past"];return k(i)?i(e):i.replace(/%s/i,e)}function R(t,e){var i=t.toLowerCase();Fo[i]=Fo[i+"s"]=Fo[e]=t}function L(t){return"string"==typeof t?Fo[t]||Fo[t.toLowerCase()]:void 0}function F(t){var e,i,o={};for(i in t)d(t,i)&&(e=L(i))&&(o[e]=t[i]);return o}function H(t,e){Ho[t]=e}function j(t){var e=[];for(var i in t)e.push({unit:i,priority:Ho[i]});return e.sort(function(t,e){return t.priority-e.priority}),e}function Y(t,e,i){var o=""+Math.abs(t),n=e-o.length;return(t>=0?i?"+":"":"-")+Math.pow(10,Math.max(0,n)).toString().substr(1)+o}function G(t,e,i,o){var n=o;"string"==typeof o&&(n=function(){return this[o]()}),t&&(zo[t]=n),e&&(zo[e[0]]=function(){return Y(n.apply(this,arguments),e[1],e[2])}),i&&(zo[i]=function(){return this.localeData().ordinal(n.apply(this,arguments),t)})}function z(t){return t.match(/\[[\s\S]/)?t.replace(/^\[|\]$/g,""):t.replace(/\\/g,"")}function W(t){var e,i,o=t.match(jo);for(e=0,i=o.length;e<i;e++)zo[o[e]]?o[e]=zo[o[e]]:o[e]=z(o[e]);return function(e){var n,s="";for(n=0;n<i;n++)s+=k(o[n])?o[n].call(e,t):o[n];return s}}function V(t,e){return t.isValid()?(e=B(e,t.localeData()),Go[e]=Go[e]||W(e),Go[e](t)):t.localeData().invalidDate()}function B(t,e){function i(t){return e.longDateFormat(t)||t}var o=5;for(Yo.lastIndex=0;o>=0&&Yo.test(t);)t=t.replace(Yo,i),Yo.lastIndex=0,o-=1;return t}function U(t,e,i){an[t]=k(e)?e:function(t,o){return t&&i?i:e}}function q(t,e){return d(an,t)?an[t](e._strict,e._locale):new RegExp(X(t))}function X(t){return Z(t.replace("\\","").replace(/\\(\[)|\\(\])|\[([^\]\[]*)\]|\\(.)/g,function(t,e,i,o,n){return e||i||o||n}))}function Z(t){return t.replace(/[-\/\\^$*+?.()|[\]{}]/g,"\\$&")}function K(t,e){var i,o=e;for("string"==typeof t&&(t=[t]),r(e)&&(o=function(t,i){i[e]=_(t)}),i=0;i<t.length;i++)hn[t[i]]=o}function J(t,e){K(t,function(t,i,o,n){o._w=o._w||{},e(t,o._w,o,n)})}function $(t,e,i){null!=e&&d(hn,t)&&hn[t](e,i._a,i,t)}function Q(t){return tt(t)?366:365}function tt(t){return t%4==0&&t%100!=0||t%400==0}function et(){return tt(this.year())}function it(t,i){return function(o){return null!=o?(nt(this,t,o),e.updateOffset(this,i),this):ot(this,t)}}function ot(t,e){return t.isValid()?t._d["get"+(t._isUTC?"UTC":"")+e]():NaN}function nt(t,e,i){t.isValid()&&!isNaN(i)&&("FullYear"===e&&tt(t.year())?t._d["set"+(t._isUTC?"UTC":"")+e](i,t.month(),ht(i,t.month())):t._d["set"+(t._isUTC?"UTC":"")+e](i))}function st(t){return t=L(t),k(this[t])?this[t]():this}function rt(t,e){if("object"==typeof t){t=F(t);for(var i=j(t),o=0;o<i.length;o++)this[i[o].unit](t[i[o].unit])}else if(t=L(t),k(this[t]))return this[t](e);return this}function at(t,e){return(t%e+e)%e}function ht(t,e){if(isNaN(t)||isNaN(e))return NaN;var i=at(e,12);return t+=(e-i)/12,1===i?tt(t)?29:28:31-i%7%2}function dt(t,e){return t?i(this._months)?this._months[t.month()]:this._months[(this._months.isFormat||_n).test(e)?"format":"standalone"][t.month()]:i(this._months)?this._months:this._months.standalone}function lt(t,e){return t?i(this._monthsShort)?this._monthsShort[t.month()]:this._monthsShort[_n.test(e)?"format":"standalone"][t.month()]:i(this._monthsShort)?this._monthsShort:this._monthsShort.standalone}function ut(t,e,i){var o,n,s,r=t.toLocaleLowerCase();if(!this._monthsParse)for(this._monthsParse=[],this._longMonthsParse=[],this._shortMonthsParse=[],o=0;o<12;++o)s=u([2e3,o]),this._shortMonthsParse[o]=this.monthsShort(s,"").toLocaleLowerCase(),this._longMonthsParse[o]=this.months(s,"").toLocaleLowerCase();return i?"MMM"===e?(n=yn.call(this._shortMonthsParse,r),-1!==n?n:null):(n=yn.call(this._longMonthsParse,r),-1!==n?n:null):"MMM"===e?-1!==(n=yn.call(this._shortMonthsParse,r))?n:(n=yn.call(this._longMonthsParse,r),-1!==n?n:null):-1!==(n=yn.call(this._longMonthsParse,r))?n:(n=yn.call(this._shortMonthsParse,r),-1!==n?n:null)}function pt(t,e,i){var o,n,s;if(this._monthsParseExact)return ut.call(this,t,e,i);for(this._monthsParse||(this._monthsParse=[],this._longMonthsParse=[],this._shortMonthsParse=[]),o=0;o<12;o++){if(n=u([2e3,o]),i&&!this._longMonthsParse[o]&&(this._longMonthsParse[o]=new RegExp("^"+this.months(n,"").replace(".","")+"$","i"),this._shortMonthsParse[o]=new RegExp("^"+this.monthsShort(n,"").replace(".","")+"$","i")),i||this._monthsParse[o]||(s="^"+this.months(n,"")+"|^"+this.monthsShort(n,""),this._monthsParse[o]=new RegExp(s.replace(".",""),"i")),i&&"MMMM"===e&&this._longMonthsParse[o].test(t))return o;if(i&&"MMM"===e&&this._shortMonthsParse[o].test(t))return o;if(!i&&this._monthsParse[o].test(t))return o}}function ct(t,e){var i;if(!t.isValid())return t;if("string"==typeof e)if(/^\d+$/.test(e))e=_(e);else if(e=t.localeData().monthsParse(e),!r(e))return t;return i=Math.min(t.date(),ht(t.year(),e)),t._d["set"+(t._isUTC?"UTC":"")+"Month"](e,i),t}function mt(t){return null!=t?(ct(this,t),e.updateOffset(this,!0),this):ot(this,"Month")}function ft(){return ht(this.year(),this.month())}function gt(t){return this._monthsParseExact?(d(this,"_monthsRegex")||yt.call(this),t?this._monthsShortStrictRegex:this._monthsShortRegex):(d(this,"_monthsShortRegex")||(this._monthsShortRegex=Dn),this._monthsShortStrictRegex&&t?this._monthsShortStrictRegex:this._monthsShortRegex)}function vt(t){return this._monthsParseExact?(d(this,"_monthsRegex")||yt.call(this),t?this._monthsStrictRegex:this._monthsRegex):(d(this,"_monthsRegex")||(this._monthsRegex=Sn),this._monthsStrictRegex&&t?this._monthsStrictRegex:this._monthsRegex)}function yt(){function t(t,e){return e.length-t.length}var e,i,o=[],n=[],s=[];for(e=0;e<12;e++)i=u([2e3,e]),o.push(this.monthsShort(i,"")),n.push(this.months(i,"")),s.push(this.months(i,"")),s.push(this.monthsShort(i,""));for(o.sort(t),n.sort(t),s.sort(t),e=0;e<12;e++)o[e]=Z(o[e]),n[e]=Z(n[e]);for(e=0;e<24;e++)s[e]=Z(s[e]);this._monthsRegex=new RegExp("^("+s.join("|")+")","i"),this._monthsShortRegex=this._monthsRegex,this._monthsStrictRegex=new RegExp("^("+n.join("|")+")","i"),this._monthsShortStrictRegex=new RegExp("^("+o.join("|")+")","i")}function bt(t,e,i,o,n,s,r){var a=new Date(t,e,i,o,n,s,r);return t<100&&t>=0&&isFinite(a.getFullYear())&&a.setFullYear(t),a}function _t(t){var e=new Date(Date.UTC.apply(null,arguments));return t<100&&t>=0&&isFinite(e.getUTCFullYear())&&e.setUTCFullYear(t),e}function wt(t,e,i){var o=7+e-i;return-(7+_t(t,0,o).getUTCDay()-e)%7+o-1}function xt(t,e,i,o,n){var s,r,a=(7+i-o)%7,h=wt(t,o,n),d=1+7*(e-1)+a+h;return d<=0?(s=t-1,r=Q(s)+d):d>Q(t)?(s=t+1,r=d-Q(t)):(s=t,r=d),{year:s,dayOfYear:r}}function Dt(t,e,i){var o,n,s=wt(t.year(),e,i),r=Math.floor((t.dayOfYear()-s-1)/7)+1;return r<1?(n=t.year()-1,o=r+St(n,e,i)):r>St(t.year(),e,i)?(o=r-St(t.year(),e,i),n=t.year()+1):(n=t.year(),o=r),{week:o,year:n}}function St(t,e,i){var o=wt(t,e,i),n=wt(t+1,e,i);return(Q(t)-o+n)/7}function kt(t){return Dt(t,this._week.dow,this._week.doy).week}function Ct(){return this._week.dow}function Tt(){return this._week.doy}function Mt(t){var e=this.localeData().week(this);return null==t?e:this.add(7*(t-e),"d")}function Ot(t){var e=Dt(this,1,4).week;return null==t?e:this.add(7*(t-e),"d")}function Et(t,e){return"string"!=typeof t?t:isNaN(t)?(t=e.weekdaysParse(t),"number"==typeof t?t:null):parseInt(t,10)}function Pt(t,e){return"string"==typeof t?e.weekdaysParse(t)%7||7:isNaN(t)?null:t}function It(t,e){return t?i(this._weekdays)?this._weekdays[t.day()]:this._weekdays[this._weekdays.isFormat.test(e)?"format":"standalone"][t.day()]:i(this._weekdays)?this._weekdays:this._weekdays.standalone}function Nt(t){return t?this._weekdaysShort[t.day()]:this._weekdaysShort}function At(t){return t?this._weekdaysMin[t.day()]:this._weekdaysMin}function Rt(t,e,i){var o,n,s,r=t.toLocaleLowerCase();if(!this._weekdaysParse)for(this._weekdaysParse=[],this._shortWeekdaysParse=[],this._minWeekdaysParse=[],o=0;o<7;++o)s=u([2e3,1]).day(o),this._minWeekdaysParse[o]=this.weekdaysMin(s,"").toLocaleLowerCase(),this._shortWeekdaysParse[o]=this.weekdaysShort(s,"").toLocaleLowerCase(),this._weekdaysParse[o]=this.weekdays(s,"").toLocaleLowerCase();return i?"dddd"===e?(n=yn.call(this._weekdaysParse,r),-1!==n?n:null):"ddd"===e?(n=yn.call(this._shortWeekdaysParse,r),-1!==n?n:null):(n=yn.call(this._minWeekdaysParse,r),-1!==n?n:null):"dddd"===e?-1!==(n=yn.call(this._weekdaysParse,r))?n:-1!==(n=yn.call(this._shortWeekdaysParse,r))?n:(n=yn.call(this._minWeekdaysParse,r),-1!==n?n:null):"ddd"===e?-1!==(n=yn.call(this._shortWeekdaysParse,r))?n:-1!==(n=yn.call(this._weekdaysParse,r))?n:(n=yn.call(this._minWeekdaysParse,r),-1!==n?n:null):-1!==(n=yn.call(this._minWeekdaysParse,r))?n:-1!==(n=yn.call(this._weekdaysParse,r))?n:(n=yn.call(this._shortWeekdaysParse,r),-1!==n?n:null)}function Lt(t,e,i){var o,n,s;if(this._weekdaysParseExact)return Rt.call(this,t,e,i);for(this._weekdaysParse||(this._weekdaysParse=[],this._minWeekdaysParse=[],this._shortWeekdaysParse=[],this._fullWeekdaysParse=[]),o=0;o<7;o++){if(n=u([2e3,1]).day(o),i&&!this._fullWeekdaysParse[o]&&(this._fullWeekdaysParse[o]=new RegExp("^"+this.weekdays(n,"").replace(".",".?")+"$","i"),this._shortWeekdaysParse[o]=new RegExp("^"+this.weekdaysShort(n,"").replace(".",".?")+"$","i"),this._minWeekdaysParse[o]=new RegExp("^"+this.weekdaysMin(n,"").replace(".",".?")+"$","i")),this._weekdaysParse[o]||(s="^"+this.weekdays(n,"")+"|^"+this.weekdaysShort(n,"")+"|^"+this.weekdaysMin(n,""),this._weekdaysParse[o]=new RegExp(s.replace(".",""),"i")),i&&"dddd"===e&&this._fullWeekdaysParse[o].test(t))return o;if(i&&"ddd"===e&&this._shortWeekdaysParse[o].test(t))return o;if(i&&"dd"===e&&this._minWeekdaysParse[o].test(t))return o;if(!i&&this._weekdaysParse[o].test(t))return o}}function Ft(t){if(!this.isValid())return null!=t?this:NaN;var e=this._isUTC?this._d.getUTCDay():this._d.getDay();return null!=t?(t=Et(t,this.localeData()),this.add(t-e,"d")):e}function Ht(t){if(!this.isValid())return null!=t?this:NaN;var e=(this.day()+7-this.localeData()._week.dow)%7;return null==t?e:this.add(t-e,"d")}function jt(t){if(!this.isValid())return null!=t?this:NaN;if(null!=t){var e=Pt(t,this.localeData());return this.day(this.day()%7?e:e-7)}return this.day()||7}function Yt(t){return this._weekdaysParseExact?(d(this,"_weekdaysRegex")||Wt.call(this),t?this._weekdaysStrictRegex:this._weekdaysRegex):(d(this,"_weekdaysRegex")||(this._weekdaysRegex=On),this._weekdaysStrictRegex&&t?this._weekdaysStrictRegex:this._weekdaysRegex)}function Gt(t){return this._weekdaysParseExact?(d(this,"_weekdaysRegex")||Wt.call(this),t?this._weekdaysShortStrictRegex:this._weekdaysShortRegex):(d(this,"_weekdaysShortRegex")||(this._weekdaysShortRegex=En),this._weekdaysShortStrictRegex&&t?this._weekdaysShortStrictRegex:this._weekdaysShortRegex)}function zt(t){return this._weekdaysParseExact?(d(this,"_weekdaysRegex")||Wt.call(this),t?this._weekdaysMinStrictRegex:this._weekdaysMinRegex):(d(this,"_weekdaysMinRegex")||(this._weekdaysMinRegex=Pn),this._weekdaysMinStrictRegex&&t?this._weekdaysMinStrictRegex:this._weekdaysMinRegex)}function Wt(){function t(t,e){return e.length-t.length}var e,i,o,n,s,r=[],a=[],h=[],d=[];for(e=0;e<7;e++)i=u([2e3,1]).day(e),o=this.weekdaysMin(i,""),n=this.weekdaysShort(i,""),s=this.weekdays(i,""),r.push(o),a.push(n),h.push(s),d.push(o),d.push(n),d.push(s);for(r.sort(t),a.sort(t),h.sort(t),d.sort(t),e=0;e<7;e++)a[e]=Z(a[e]),h[e]=Z(h[e]),d[e]=Z(d[e]);this._weekdaysRegex=new RegExp("^("+d.join("|")+")","i"),this._weekdaysShortRegex=this._weekdaysRegex,this._weekdaysMinRegex=this._weekdaysRegex,this._weekdaysStrictRegex=new RegExp("^("+h.join("|")+")","i"),this._weekdaysShortStrictRegex=new RegExp("^("+a.join("|")+")","i"),this._weekdaysMinStrictRegex=new RegExp("^("+r.join("|")+")","i")}function Vt(){return this.hours()%12||12}function Bt(){return this.hours()||24}function Ut(t,e){G(t,0,0,function(){return this.localeData().meridiem(this.hours(),this.minutes(),e)})}function qt(t,e){return e._meridiemParse}function Xt(t){return"p"===(t+"").toLowerCase().charAt(0)}function Zt(t,e,i){return t>11?i?"pm":"PM":i?"am":"AM"}function Kt(t){return t?t.toLowerCase().replace("_","-"):t}function Jt(t){for(var e,i,o,n,s=0;s<t.length;){for(n=Kt(t[s]).split("-"),e=n.length,i=Kt(t[s+1]),i=i?i.split("-"):null;e>0;){if(o=$t(n.slice(0,e).join("-")))return o;if(i&&i.length>=e&&w(n,i,!0)>=e-1)break;e--}s++}return null}function $t(e){var i=null;if(!Ln[e]&&void 0!==t&&t&&t.exports)try{i=In._abbr;!function(){var t=new Error('Cannot find module "./locale"');throw t.code="MODULE_NOT_FOUND",t}(),Qt(i)}catch(t){}return Ln[e]}function Qt(t,e){var i;return t&&(i=s(e)?ie(t):te(t,e))&&(In=i),In._abbr}function te(t,e){if(null!==e){var i=Rn;if(e.abbr=t,null!=Ln[t])S("defineLocaleOverride","use moment.updateLocale(localeName, config) to change an existing locale. moment.defineLocale(localeName, config) should only be used for creating a new locale See http://momentjs.com/guides/#/warnings/define-locale/ for more info."),i=Ln[t]._config;else if(null!=e.parentLocale){if(null==Ln[e.parentLocale])return Fn[e.parentLocale]||(Fn[e.parentLocale]=[]),Fn[e.parentLocale].push({name:t,config:e}),null;i=Ln[e.parentLocale]._config}return Ln[t]=new M(T(i,e)),Fn[t]&&Fn[t].forEach(function(t){te(t.name,t.config)}),Qt(t),Ln[t]}return delete Ln[t],null}function ee(t,e){if(null!=e){var i,o=Rn;null!=Ln[t]&&(o=Ln[t]._config),e=T(o,e),i=new M(e),i.parentLocale=Ln[t],Ln[t]=i,Qt(t)}else null!=Ln[t]&&(null!=Ln[t].parentLocale?Ln[t]=Ln[t].parentLocale:null!=Ln[t]&&delete Ln[t]);return Ln[t]}function ie(t){var e;if(t&&t._locale&&t._locale._abbr&&(t=t._locale._abbr),!t)return In;if(!i(t)){if(e=$t(t))return e;t=[t]}return Jt(t)}function oe(){return Io(Ln)}function ne(t){var e,i=t._a;return i&&-2===c(t).overflow&&(e=i[ln]<0||i[ln]>11?ln:i[un]<1||i[un]>ht(i[dn],i[ln])?un:i[pn]<0||i[pn]>24||24===i[pn]&&(0!==i[cn]||0!==i[mn]||0!==i[fn])?pn:i[cn]<0||i[cn]>59?cn:i[mn]<0||i[mn]>59?mn:i[fn]<0||i[fn]>999?fn:-1,c(t)._overflowDayOfYear&&(e<dn||e>un)&&(e=un),c(t)._overflowWeeks&&-1===e&&(e=gn),c(t)._overflowWeekday&&-1===e&&(e=vn),c(t).overflow=e),t}function se(t,e,i){return null!=t?t:null!=e?e:i}function re(t){var i=new Date(e.now());return t._useUTC?[i.getUTCFullYear(),i.getUTCMonth(),i.getUTCDate()]:[i.getFullYear(),i.getMonth(),i.getDate()]}function ae(t){var e,i,o,n,s=[];if(!t._d){for(o=re(t),t._w&&null==t._a[un]&&null==t._a[ln]&&he(t),null!=t._dayOfYear&&(n=se(t._a[dn],o[dn]),(t._dayOfYear>Q(n)||0===t._dayOfYear)&&(c(t)._overflowDayOfYear=!0),i=_t(n,0,t._dayOfYear),t._a[ln]=i.getUTCMonth(),t._a[un]=i.getUTCDate()),e=0;e<3&&null==t._a[e];++e)t._a[e]=s[e]=o[e];for(;e<7;e++)t._a[e]=s[e]=null==t._a[e]?2===e?1:0:t._a[e];24===t._a[pn]&&0===t._a[cn]&&0===t._a[mn]&&0===t._a[fn]&&(t._nextDay=!0,t._a[pn]=0),t._d=(t._useUTC?_t:bt).apply(null,s),null!=t._tzm&&t._d.setUTCMinutes(t._d.getUTCMinutes()-t._tzm),t._nextDay&&(t._a[pn]=24),t._w&&void 0!==t._w.d&&t._w.d!==t._d.getDay()&&(c(t).weekdayMismatch=!0)}}function he(t){var e,i,o,n,s,r,a,h;if(e=t._w,null!=e.GG||null!=e.W||null!=e.E)s=1,r=4,i=se(e.GG,t._a[dn],Dt(ke(),1,4).year),o=se(e.W,1),((n=se(e.E,1))<1||n>7)&&(h=!0);else{s=t._locale._week.dow,r=t._locale._week.doy;var d=Dt(ke(),s,r);i=se(e.gg,t._a[dn],d.year),o=se(e.w,d.week),null!=e.d?((n=e.d)<0||n>6)&&(h=!0):null!=e.e?(n=e.e+s,(e.e<0||e.e>6)&&(h=!0)):n=s}o<1||o>St(i,s,r)?c(t)._overflowWeeks=!0:null!=h?c(t)._overflowWeekday=!0:(a=xt(i,o,n,s,r),t._a[dn]=a.year,t._dayOfYear=a.dayOfYear)}function de(t){var e,i,o,n,s,r,a=t._i,h=Hn.exec(a)||jn.exec(a);if(h){for(c(t).iso=!0,e=0,i=Gn.length;e<i;e++)if(Gn[e][1].exec(h[1])){n=Gn[e][0],o=!1!==Gn[e][2];break}if(null==n)return void(t._isValid=!1);if(h[3]){for(e=0,i=zn.length;e<i;e++)if(zn[e][1].exec(h[3])){s=(h[2]||" ")+zn[e][0];break}if(null==s)return void(t._isValid=!1)}if(!o&&null!=s)return void(t._isValid=!1);if(h[4]){if(!Yn.exec(h[4]))return void(t._isValid=!1);r="Z"}t._f=n+(s||"")+(r||""),ve(t)}else t._isValid=!1}function le(t,e,i,o,n,s){var r=[ue(t),xn.indexOf(e),parseInt(i,10),parseInt(o,10),parseInt(n,10)];return s&&r.push(parseInt(s,10)),r}function ue(t){var e=parseInt(t,10);return e<=49?2e3+e:e<=999?1900+e:e}function pe(t){return t.replace(/\([^)]*\)|[\n\t]/g," ").replace(/(\s\s+)/g," ").trim()}function ce(t,e,i){if(t){if(Tn.indexOf(t)!==new Date(e[0],e[1],e[2]).getDay())return c(i).weekdayMismatch=!0,i._isValid=!1,!1}return!0}function me(t,e,i){if(t)return Bn[t];if(e)return 0;var o=parseInt(i,10),n=o%100;return(o-n)/100*60+n}function fe(t){var e=Vn.exec(pe(t._i));if(e){var i=le(e[4],e[3],e[2],e[5],e[6],e[7]);if(!ce(e[1],i,t))return;t._a=i,t._tzm=me(e[8],e[9],e[10]),t._d=_t.apply(null,t._a),t._d.setUTCMinutes(t._d.getUTCMinutes()-t._tzm),c(t).rfc2822=!0}else t._isValid=!1}function ge(t){var i=Wn.exec(t._i);if(null!==i)return void(t._d=new Date(+i[1]));de(t),!1===t._isValid&&(delete t._isValid,fe(t),!1===t._isValid&&(delete t._isValid,e.createFromInputFallback(t)))}function ve(t){if(t._f===e.ISO_8601)return void de(t);if(t._f===e.RFC_2822)return void fe(t);t._a=[],c(t).empty=!0;var i,o,n,s,r,a=""+t._i,h=a.length,d=0;for(n=B(t._f,t._locale).match(jo)||[],i=0;i<n.length;i++)s=n[i],o=(a.match(q(s,t))||[])[0],o&&(r=a.substr(0,a.indexOf(o)),r.length>0&&c(t).unusedInput.push(r),a=a.slice(a.indexOf(o)+o.length),d+=o.length),zo[s]?(o?c(t).empty=!1:c(t).unusedTokens.push(s),$(s,o,t)):t._strict&&!o&&c(t).unusedTokens.push(s);c(t).charsLeftOver=h-d,a.length>0&&c(t).unusedInput.push(a),t._a[pn]<=12&&!0===c(t).bigHour&&t._a[pn]>0&&(c(t).bigHour=void 0),c(t).parsedDateParts=t._a.slice(0),c(t).meridiem=t._meridiem,t._a[pn]=ye(t._locale,t._a[pn],t._meridiem),ae(t),ne(t)}function ye(t,e,i){var o;return null==i?e:null!=t.meridiemHour?t.meridiemHour(e,i):null!=t.isPM?(o=t.isPM(i),o&&e<12&&(e+=12),o||12!==e||(e=0),e):e}function be(t){var e,i,o,n,s;if(0===t._f.length)return c(t).invalidFormat=!0,void(t._d=new Date(NaN));for(n=0;n<t._f.length;n++)s=0,e=g({},t),null!=t._useUTC&&(e._useUTC=t._useUTC),e._f=t._f[n],ve(e),m(e)&&(s+=c(e).charsLeftOver,s+=10*c(e).unusedTokens.length,c(e).score=s,(null==o||s<o)&&(o=s,i=e));l(t,i||e)}function _e(t){if(!t._d){var e=F(t._i);t._a=h([e.year,e.month,e.day||e.date,e.hour,e.minute,e.second,e.millisecond],function(t){return t&&parseInt(t,10)}),ae(t)}}function we(t){var e=new v(ne(xe(t)));return e._nextDay&&(e.add(1,"d"),e._nextDay=void 0),e}function xe(t){var e=t._i,o=t._f;return t._locale=t._locale||ie(t._l),null===e||void 0===o&&""===e?f({nullInput:!0}):("string"==typeof e&&(t._i=e=t._locale.preparse(e)),y(e)?new v(ne(e)):(a(e)?t._d=e:i(o)?be(t):o?ve(t):De(t),m(t)||(t._d=null),t))}function De(t){var n=t._i;s(n)?t._d=new Date(e.now()):a(n)?t._d=new Date(n.valueOf()):"string"==typeof n?ge(t):i(n)?(t._a=h(n.slice(0),function(t){return parseInt(t,10)}),ae(t)):o(n)?_e(t):r(n)?t._d=new Date(n):e.createFromInputFallback(t)}function Se(t,e,s,r,a){var h={};return!0!==s&&!1!==s||(r=s,s=void 0),(o(t)&&n(t)||i(t)&&0===t.length)&&(t=void 0),h._isAMomentObject=!0,h._useUTC=h._isUTC=a,h._l=s,h._i=t,h._f=e,h._strict=r,we(h)}function ke(t,e,i,o){return Se(t,e,i,o,!1)}function Ce(t,e){var o,n;if(1===e.length&&i(e[0])&&(e=e[0]),!e.length)return ke();for(o=e[0],n=1;n<e.length;++n)e[n].isValid()&&!e[n][t](o)||(o=e[n]);return o}function Te(){return Ce("isBefore",[].slice.call(arguments,0))}function Me(){return Ce("isAfter",[].slice.call(arguments,0))}function Oe(t){for(var e in t)if(-1===yn.call(Zn,e)||null!=t[e]&&isNaN(t[e]))return!1;for(var i=!1,o=0;o<Zn.length;++o)if(t[Zn[o]]){if(i)return!1;parseFloat(t[Zn[o]])!==_(t[Zn[o]])&&(i=!0)}return!0}function Ee(){return this._isValid}function Pe(){return Ke(NaN)}function Ie(t){var e=F(t),i=e.year||0,o=e.quarter||0,n=e.month||0,s=e.week||0,r=e.day||0,a=e.hour||0,h=e.minute||0,d=e.second||0,l=e.millisecond||0;this._isValid=Oe(e),this._milliseconds=+l+1e3*d+6e4*h+1e3*a*60*60,this._days=+r+7*s,this._months=+n+3*o+12*i,this._data={},this._locale=ie(),this._bubble()}function Ne(t){return t instanceof Ie}function Ae(t){return t<0?-1*Math.round(-1*t):Math.round(t)}function Re(t,e){G(t,0,0,function(){var t=this.utcOffset(),i="+";return t<0&&(t=-t,i="-"),i+Y(~~(t/60),2)+e+Y(~~t%60,2)})}function Le(t,e){var i=(e||"").match(t);if(null===i)return null;var o=i[i.length-1]||[],n=(o+"").match(Kn)||["-",0,0],s=60*n[1]+_(n[2]);return 0===s?0:"+"===n[0]?s:-s}function Fe(t,i){var o,n;return i._isUTC?(o=i.clone(),n=(y(t)||a(t)?t.valueOf():ke(t).valueOf())-o.valueOf(),o._d.setTime(o._d.valueOf()+n),e.updateOffset(o,!1),o):ke(t).local()}function He(t){return 15*-Math.round(t._d.getTimezoneOffset()/15)}function je(t,i,o){var n,s=this._offset||0;if(!this.isValid())return null!=t?this:NaN;if(null!=t){if("string"==typeof t){if(null===(t=Le(nn,t)))return this}else Math.abs(t)<16&&!o&&(t*=60);return!this._isUTC&&i&&(n=He(this)),this._offset=t,this._isUTC=!0,null!=n&&this.add(n,"m"),s!==t&&(!i||this._changeInProgress?ei(this,Ke(t-s,"m"),1,!1):this._changeInProgress||(this._changeInProgress=!0,e.updateOffset(this,!0),this._changeInProgress=null)),this}return this._isUTC?s:He(this)}function Ye(t,e){return null!=t?("string"!=typeof t&&(t=-t),this.utcOffset(t,e),this):-this.utcOffset()}function Ge(t){return this.utcOffset(0,t)}function ze(t){return this._isUTC&&(this.utcOffset(0,t),this._isUTC=!1,t&&this.subtract(He(this),"m")),this}function We(){if(null!=this._tzm)this.utcOffset(this._tzm,!1,!0);else if("string"==typeof this._i){var t=Le(on,this._i);null!=t?this.utcOffset(t):this.utcOffset(0,!0)}return this}function Ve(t){return!!this.isValid()&&(t=t?ke(t).utcOffset():0,(this.utcOffset()-t)%60==0)}function Be(){return this.utcOffset()>this.clone().month(0).utcOffset()||this.utcOffset()>this.clone().month(5).utcOffset()}function Ue(){if(!s(this._isDSTShifted))return this._isDSTShifted;var t={};if(g(t,this),t=xe(t),t._a){var e=t._isUTC?u(t._a):ke(t._a);this._isDSTShifted=this.isValid()&&w(t._a,e.toArray())>0}else this._isDSTShifted=!1;return this._isDSTShifted}function qe(){return!!this.isValid()&&!this._isUTC}function Xe(){return!!this.isValid()&&this._isUTC}function Ze(){return!!this.isValid()&&(this._isUTC&&0===this._offset)}function Ke(t,e){var i,o,n,s=t,a=null;return Ne(t)?s={ms:t._milliseconds,d:t._days,M:t._months}:r(t)?(s={},e?s[e]=t:s.milliseconds=t):(a=Jn.exec(t))?(i="-"===a[1]?-1:1,s={y:0,d:_(a[un])*i,h:_(a[pn])*i,m:_(a[cn])*i,s:_(a[mn])*i,ms:_(Ae(1e3*a[fn]))*i}):(a=$n.exec(t))?(i="-"===a[1]?-1:(a[1],1),s={y:Je(a[2],i),M:Je(a[3],i),w:Je(a[4],i),d:Je(a[5],i),h:Je(a[6],i),m:Je(a[7],i),s:Je(a[8],i)}):null==s?s={}:"object"==typeof s&&("from"in s||"to"in s)&&(n=Qe(ke(s.from),ke(s.to)),s={},s.ms=n.milliseconds,s.M=n.months),o=new Ie(s),Ne(t)&&d(t,"_locale")&&(o._locale=t._locale),o}function Je(t,e){var i=t&&parseFloat(t.replace(",","."));return(isNaN(i)?0:i)*e}function $e(t,e){var i={milliseconds:0,months:0};return i.months=e.month()-t.month()+12*(e.year()-t.year()),t.clone().add(i.months,"M").isAfter(e)&&--i.months,i.milliseconds=+e-+t.clone().add(i.months,"M"),i}function Qe(t,e){var i;return t.isValid()&&e.isValid()?(e=Fe(e,t),t.isBefore(e)?i=$e(t,e):(i=$e(e,t),i.milliseconds=-i.milliseconds,i.months=-i.months),i):{milliseconds:0,months:0}}function ti(t,e){return function(i,o){var n,s;return null===o||isNaN(+o)||(S(e,"moment()."+e+"(period, number) is deprecated. Please use moment()."+e+"(number, period). See http://momentjs.com/guides/#/warnings/add-inverted-param/ for more info."),s=i,i=o,o=s),i="string"==typeof i?+i:i,n=Ke(i,o),ei(this,n,t),this}}function ei(t,i,o,n){var s=i._milliseconds,r=Ae(i._days),a=Ae(i._months);t.isValid()&&(n=null==n||n,a&&ct(t,ot(t,"Month")+a*o),r&&nt(t,"Date",ot(t,"Date")+r*o),s&&t._d.setTime(t._d.valueOf()+s*o),n&&e.updateOffset(t,r||a))}function ii(t,e){var i=t.diff(e,"days",!0);return i<-6?"sameElse":i<-1?"lastWeek":i<0?"lastDay":i<1?"sameDay":i<2?"nextDay":i<7?"nextWeek":"sameElse"}function oi(t,i){var o=t||ke(),n=Fe(o,this).startOf("day"),s=e.calendarFormat(this,n)||"sameElse",r=i&&(k(i[s])?i[s].call(this,o):i[s]);return this.format(r||this.localeData().calendar(s,this,ke(o)))}function ni(){return new v(this)}function si(t,e){var i=y(t)?t:ke(t);return!(!this.isValid()||!i.isValid())&&(e=L(s(e)?"millisecond":e),"millisecond"===e?this.valueOf()>i.valueOf():i.valueOf()<this.clone().startOf(e).valueOf())}function ri(t,e){var i=y(t)?t:ke(t);return!(!this.isValid()||!i.isValid())&&(e=L(s(e)?"millisecond":e),"millisecond"===e?this.valueOf()<i.valueOf():this.clone().endOf(e).valueOf()<i.valueOf())}function ai(t,e,i,o){return o=o||"()",("("===o[0]?this.isAfter(t,i):!this.isBefore(t,i))&&(")"===o[1]?this.isBefore(e,i):!this.isAfter(e,i))}function hi(t,e){var i,o=y(t)?t:ke(t);return!(!this.isValid()||!o.isValid())&&(e=L(e||"millisecond"),"millisecond"===e?this.valueOf()===o.valueOf():(i=o.valueOf(),this.clone().startOf(e).valueOf()<=i&&i<=this.clone().endOf(e).valueOf()))}function di(t,e){return this.isSame(t,e)||this.isAfter(t,e)}function li(t,e){return this.isSame(t,e)||this.isBefore(t,e)}function ui(t,e,i){var o,n,s;if(!this.isValid())return NaN;if(o=Fe(t,this),!o.isValid())return NaN;switch(n=6e4*(o.utcOffset()-this.utcOffset()),e=L(e)){case"year":s=pi(this,o)/12;break;case"month":s=pi(this,o);break;case"quarter":s=pi(this,o)/3;break;case"second":s=(this-o)/1e3;break;case"minute":s=(this-o)/6e4;break;case"hour":s=(this-o)/36e5;break;case"day":s=(this-o-n)/864e5;break;case"week":s=(this-o-n)/6048e5;break;default:s=this-o}return i?s:b(s)}function pi(t,e){var i,o,n=12*(e.year()-t.year())+(e.month()-t.month()),s=t.clone().add(n,"months");return e-s<0?(i=t.clone().add(n-1,"months"),o=(e-s)/(s-i)):(i=t.clone().add(n+1,"months"),o=(e-s)/(i-s)),-(n+o)||0}function ci(){return this.clone().locale("en").format("ddd MMM DD YYYY HH:mm:ss [GMT]ZZ")}function mi(){if(!this.isValid())return null;var t=this.clone().utc();return t.year()<0||t.year()>9999?V(t,"YYYYYY-MM-DD[T]HH:mm:ss.SSS[Z]"):k(Date.prototype.toISOString)?this.toDate().toISOString():V(t,"YYYY-MM-DD[T]HH:mm:ss.SSS[Z]")}function fi(){if(!this.isValid())return"moment.invalid(/* "+this._i+" */)";var t="moment",e="";this.isLocal()||(t=0===this.utcOffset()?"moment.utc":"moment.parseZone",e="Z");var i="["+t+'("]',o=0<=this.year()&&this.year()<=9999?"YYYY":"YYYYYY",n=e+'[")]';return this.format(i+o+"-MM-DD[T]HH:mm:ss.SSS"+n)}function gi(t){t||(t=this.isUtc()?e.defaultFormatUtc:e.defaultFormat);var i=V(this,t);return this.localeData().postformat(i)}function vi(t,e){return this.isValid()&&(y(t)&&t.isValid()||ke(t).isValid())?Ke({to:this,from:t}).locale(this.locale()).humanize(!e):this.localeData().invalidDate()}function yi(t){return this.from(ke(),t)}function bi(t,e){return this.isValid()&&(y(t)&&t.isValid()||ke(t).isValid())?Ke({from:this,to:t}).locale(this.locale()).humanize(!e):this.localeData().invalidDate()}function _i(t){return this.to(ke(),t)}function wi(t){var e;return void 0===t?this._locale._abbr:(e=ie(t),null!=e&&(this._locale=e),this)}function xi(){return this._locale}function Di(t){switch(t=L(t)){case"year":this.month(0);case"quarter":case"month":this.date(1);case"week":case"isoWeek":case"day":case"date":this.hours(0);case"hour":this.minutes(0);case"minute":this.seconds(0);case"second":this.milliseconds(0)}return"week"===t&&this.weekday(0),"isoWeek"===t&&this.isoWeekday(1),"quarter"===t&&this.month(3*Math.floor(this.month()/3)),this}function Si(t){return void 0===(t=L(t))||"millisecond"===t?this:("date"===t&&(t="day"),this.startOf(t).add(1,"isoWeek"===t?"week":t).subtract(1,"ms"))}function ki(){return this._d.valueOf()-6e4*(this._offset||0)}function Ci(){return Math.floor(this.valueOf()/1e3)}function Ti(){return new Date(this.valueOf())}function Mi(){var t=this;return[t.year(),t.month(),t.date(),t.hour(),t.minute(),t.second(),t.millisecond()]}function Oi(){var t=this;return{years:t.year(),months:t.month(),date:t.date(),hours:t.hours(),minutes:t.minutes(),seconds:t.seconds(),milliseconds:t.milliseconds()}}function Ei(){return this.isValid()?this.toISOString():null}function Pi(){return m(this)}function Ii(){return l({},c(this))}function Ni(){return c(this).overflow}function Ai(){return{input:this._i,format:this._f,locale:this._locale,isUTC:this._isUTC,strict:this._strict}}function Ri(t,e){G(0,[t,t.length],0,e)}function Li(t){return Yi.call(this,t,this.week(),this.weekday(),this.localeData()._week.dow,this.localeData()._week.doy)}function Fi(t){return Yi.call(this,t,this.isoWeek(),this.isoWeekday(),1,4)}function Hi(){return St(this.year(),1,4)}function ji(){var t=this.localeData()._week;return St(this.year(),t.dow,t.doy)}function Yi(t,e,i,o,n){var s;return null==t?Dt(this,o,n).year:(s=St(t,o,n),e>s&&(e=s),Gi.call(this,t,e,i,o,n))}function Gi(t,e,i,o,n){var s=xt(t,e,i,o,n),r=_t(s.year,0,s.dayOfYear);return this.year(r.getUTCFullYear()),this.month(r.getUTCMonth()),this.date(r.getUTCDate()),this}function zi(t){return null==t?Math.ceil((this.month()+1)/3):this.month(3*(t-1)+this.month()%3)}function Wi(t){var e=Math.round((this.clone().startOf("day")-this.clone().startOf("year"))/864e5)+1;return null==t?e:this.add(t-e,"d")}function Vi(t,e){e[fn]=_(1e3*("0."+t))}function Bi(){return this._isUTC?"UTC":""}function Ui(){return this._isUTC?"Coordinated Universal Time":""}function qi(t){return ke(1e3*t)}function Xi(){return ke.apply(null,arguments).parseZone()}function Zi(t){return t}function Ki(t,e,i,o){var n=ie(),s=u().set(o,e);return n[i](s,t)}function Ji(t,e,i){if(r(t)&&(e=t,t=void 0),t=t||"",null!=e)return Ki(t,e,i,"month");var o,n=[];for(o=0;o<12;o++)n[o]=Ki(t,o,i,"month");return n}function $i(t,e,i,o){"boolean"==typeof t?(r(e)&&(i=e,e=void 0),e=e||""):(e=t,i=e,t=!1,r(e)&&(i=e,e=void 0),e=e||"");var n=ie(),s=t?n._week.dow:0;if(null!=i)return Ki(e,(i+s)%7,o,"day");var a,h=[];for(a=0;a<7;a++)h[a]=Ki(e,(a+s)%7,o,"day");return h}function Qi(t,e){return Ji(t,e,"months")}function to(t,e){return Ji(t,e,"monthsShort")}function eo(t,e,i){return $i(t,e,i,"weekdays")}function io(t,e,i){return $i(t,e,i,"weekdaysShort")}function oo(t,e,i){return $i(t,e,i,"weekdaysMin")}function no(){var t=this._data;return this._milliseconds=ds(this._milliseconds),this._days=ds(this._days),this._months=ds(this._months),t.milliseconds=ds(t.milliseconds),t.seconds=ds(t.seconds),t.minutes=ds(t.minutes),t.hours=ds(t.hours),t.months=ds(t.months),t.years=ds(t.years),this}function so(t,e,i,o){var n=Ke(e,i);return t._milliseconds+=o*n._milliseconds,t._days+=o*n._days,t._months+=o*n._months,t._bubble()}function ro(t,e){return so(this,t,e,1)}function ao(t,e){return so(this,t,e,-1)}function ho(t){return t<0?Math.floor(t):Math.ceil(t)}function lo(){var t,e,i,o,n,s=this._milliseconds,r=this._days,a=this._months,h=this._data;return s>=0&&r>=0&&a>=0||s<=0&&r<=0&&a<=0||(s+=864e5*ho(po(a)+r),r=0,a=0),h.milliseconds=s%1e3,t=b(s/1e3),h.seconds=t%60,e=b(t/60),h.minutes=e%60,i=b(e/60),h.hours=i%24,r+=b(i/24),n=b(uo(r)),a+=n,r-=ho(po(n)),o=b(a/12),a%=12,h.days=r,h.months=a,h.years=o,this}function uo(t){return 4800*t/146097}function po(t){return 146097*t/4800}function co(t){if(!this.isValid())return NaN;var e,i,o=this._milliseconds;if("month"===(t=L(t))||"year"===t)return e=this._days+o/864e5,i=this._months+uo(e),"month"===t?i:i/12;switch(e=this._days+Math.round(po(this._months)),t){case"week":return e/7+o/6048e5;case"day":return e+o/864e5;case"hour":return 24*e+o/36e5;case"minute":return 1440*e+o/6e4;case"second":return 86400*e+o/1e3;case"millisecond":return Math.floor(864e5*e)+o;default:throw new Error("Unknown unit "+t)}}function mo(){return this.isValid()?this._milliseconds+864e5*this._days+this._months%12*2592e6+31536e6*_(this._months/12):NaN}function fo(t){return function(){return this.as(t)}}function go(){return Ke(this)}function vo(t){return t=L(t),this.isValid()?this[t+"s"]():NaN}function yo(t){return function(){return this.isValid()?this._data[t]:NaN}}function bo(){return b(this.days()/7)}function _o(t,e,i,o,n){return n.relativeTime(e||1,!!i,t,o)}function wo(t,e,i){var o=Ke(t).abs(),n=ks(o.as("s")),s=ks(o.as("m")),r=ks(o.as("h")),a=ks(o.as("d")),h=ks(o.as("M")),d=ks(o.as("y")),l=n<=Cs.ss&&["s",n]||n<Cs.s&&["ss",n]||s<=1&&["m"]||s<Cs.m&&["mm",s]||r<=1&&["h"]||r<Cs.h&&["hh",r]||a<=1&&["d"]||a<Cs.d&&["dd",a]||h<=1&&["M"]||h<Cs.M&&["MM",h]||d<=1&&["y"]||["yy",d];return l[2]=e,l[3]=+t>0,l[4]=i,_o.apply(null,l)}function xo(t){return void 0===t?ks:"function"==typeof t&&(ks=t,!0)}function Do(t,e){return void 0!==Cs[t]&&(void 0===e?Cs[t]:(Cs[t]=e,"s"===t&&(Cs.ss=e-1),!0))}function So(t){if(!this.isValid())return this.localeData().invalidDate();var e=this.localeData(),i=wo(this,!t,e);return t&&(i=e.pastFuture(+this,i)),e.postformat(i)}function ko(t){return(t>0)-(t<0)||+t}function Co(){if(!this.isValid())return this.localeData().invalidDate();var t,e,i,o=Ts(this._milliseconds)/1e3,n=Ts(this._days),s=Ts(this._months);t=b(o/60),e=b(t/60),o%=60,t%=60,i=b(s/12),s%=12;var r=i,a=s,h=n,d=e,l=t,u=o?o.toFixed(3).replace(/\.?0+$/,""):"",p=this.asSeconds();if(!p)return"P0D";var c=p<0?"-":"",m=ko(this._months)!==ko(p)?"-":"",f=ko(this._days)!==ko(p)?"-":"",g=ko(this._milliseconds)!==ko(p)?"-":"";return c+"P"+(r?m+r+"Y":"")+(a?m+a+"M":"")+(h?f+h+"D":"")+(d||l||u?"T":"")+(d?g+d+"H":"")+(l?g+l+"M":"")+(u?g+u+"S":"")}var To,Mo;Mo=Array.prototype.some?Array.prototype.some:function(t){for(var e=Object(this),i=e.length>>>0,o=0;o<i;o++)if(o in e&&t.call(this,e[o],o,e))return!0;return!1};var Oo=e.momentProperties=[],Eo=!1,Po={};e.suppressDeprecationWarnings=!1,e.deprecationHandler=null;var Io;Io=Object.keys?Object.keys:function(t){var e,i=[];for(e in t)d(t,e)&&i.push(e);return i};var No={sameDay:"[Today at] LT",nextDay:"[Tomorrow at] LT",nextWeek:"dddd [at] LT",lastDay:"[Yesterday at] LT",lastWeek:"[Last] dddd [at] LT",sameElse:"L"},Ao={LTS:"h:mm:ss A",LT:"h:mm A",L:"MM/DD/YYYY",LL:"MMMM D, YYYY",LLL:"MMMM D, YYYY h:mm A",LLLL:"dddd, MMMM D, YYYY h:mm A"},Ro=/\d{1,2}/,Lo={future:"in %s",past:"%s ago",s:"a few seconds",ss:"%d seconds",m:"a minute",mm:"%d minutes",h:"an hour",hh:"%d hours",d:"a day",dd:"%d days",M:"a month",MM:"%d months",y:"a year",yy:"%d years"},Fo={},Ho={},jo=/(\[[^\[]*\])|(\\)?([Hh]mm(ss)?|Mo|MM?M?M?|Do|DDDo|DD?D?D?|ddd?d?|do?|w[o|w]?|W[o|W]?|Qo?|YYYYYY|YYYYY|YYYY|YY|gg(ggg?)?|GG(GGG?)?|e|E|a|A|hh?|HH?|kk?|mm?|ss?|S{1,9}|x|X|zz?|ZZ?|.)/g,Yo=/(\[[^\[]*\])|(\\)?(LTS|LT|LL?L?L?|l{1,4})/g,Go={},zo={},Wo=/\d/,Vo=/\d\d/,Bo=/\d{3}/,Uo=/\d{4}/,qo=/[+-]?\d{6}/,Xo=/\d\d?/,Zo=/\d\d\d\d?/,Ko=/\d\d\d\d\d\d?/,Jo=/\d{1,3}/,$o=/\d{1,4}/,Qo=/[+-]?\d{1,6}/,tn=/\d+/,en=/[+-]?\d+/,on=/Z|[+-]\d\d:?\d\d/gi,nn=/Z|[+-]\d\d(?::?\d\d)?/gi,sn=/[+-]?\d+(\.\d{1,3})?/,rn=/[0-9]*['a-z\u00A0-\u05FF\u0700-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF]+|[\u0600-\u06FF\/]+(\s*?[\u0600-\u06FF]+){1,2}/i,an={},hn={},dn=0,ln=1,un=2,pn=3,cn=4,mn=5,fn=6,gn=7,vn=8;G("Y",0,0,function(){var t=this.year();return t<=9999?""+t:"+"+t}),G(0,["YY",2],0,function(){return this.year()%100}),G(0,["YYYY",4],0,"year"),G(0,["YYYYY",5],0,"year"),G(0,["YYYYYY",6,!0],0,"year"),R("year","y"),H("year",1),U("Y",en),U("YY",Xo,Vo),U("YYYY",$o,Uo),U("YYYYY",Qo,qo),U("YYYYYY",Qo,qo),K(["YYYYY","YYYYYY"],dn),K("YYYY",function(t,i){i[dn]=2===t.length?e.parseTwoDigitYear(t):_(t)}),K("YY",function(t,i){i[dn]=e.parseTwoDigitYear(t)}),K("Y",function(t,e){e[dn]=parseInt(t,10)}),e.parseTwoDigitYear=function(t){return _(t)+(_(t)>68?1900:2e3)};var yn,bn=it("FullYear",!0);yn=Array.prototype.indexOf?Array.prototype.indexOf:function(t){var e;for(e=0;e<this.length;++e)if(this[e]===t)return e;return-1},G("M",["MM",2],"Mo",function(){return this.month()+1}),G("MMM",0,0,function(t){return this.localeData().monthsShort(this,t)}),G("MMMM",0,0,function(t){return this.localeData().months(this,t)}),R("month","M"),H("month",8),U("M",Xo),U("MM",Xo,Vo),U("MMM",function(t,e){return e.monthsShortRegex(t)}),U("MMMM",function(t,e){return e.monthsRegex(t)}),K(["M","MM"],function(t,e){e[ln]=_(t)-1}),K(["MMM","MMMM"],function(t,e,i,o){var n=i._locale.monthsParse(t,o,i._strict);null!=n?e[ln]=n:c(i).invalidMonth=t});var _n=/D[oD]?(\[[^\[\]]*\]|\s)+MMMM?/,wn="January_February_March_April_May_June_July_August_September_October_November_December".split("_"),xn="Jan_Feb_Mar_Apr_May_Jun_Jul_Aug_Sep_Oct_Nov_Dec".split("_"),Dn=rn,Sn=rn;G("w",["ww",2],"wo","week"),G("W",["WW",2],"Wo","isoWeek"),R("week","w"),R("isoWeek","W"),H("week",5),H("isoWeek",5),U("w",Xo),U("ww",Xo,Vo),U("W",Xo),U("WW",Xo,Vo),J(["w","ww","W","WW"],function(t,e,i,o){e[o.substr(0,1)]=_(t)});var kn={dow:0,doy:6};G("d",0,"do","day"),G("dd",0,0,function(t){return this.localeData().weekdaysMin(this,t)}),G("ddd",0,0,function(t){return this.localeData().weekdaysShort(this,t)}),G("dddd",0,0,function(t){return this.localeData().weekdays(this,t)}),G("e",0,0,"weekday"),G("E",0,0,"isoWeekday"),R("day","d"),R("weekday","e"),R("isoWeekday","E"),H("day",11),H("weekday",11),H("isoWeekday",11),U("d",Xo),U("e",Xo),U("E",Xo),U("dd",function(t,e){return e.weekdaysMinRegex(t)}),U("ddd",function(t,e){return e.weekdaysShortRegex(t)}),U("dddd",function(t,e){return e.weekdaysRegex(t)}),J(["dd","ddd","dddd"],function(t,e,i,o){var n=i._locale.weekdaysParse(t,o,i._strict);null!=n?e.d=n:c(i).invalidWeekday=t}),J(["d","e","E"],function(t,e,i,o){e[o]=_(t)});var Cn="Sunday_Monday_Tuesday_Wednesday_Thursday_Friday_Saturday".split("_"),Tn="Sun_Mon_Tue_Wed_Thu_Fri_Sat".split("_"),Mn="Su_Mo_Tu_We_Th_Fr_Sa".split("_"),On=rn,En=rn,Pn=rn;G("H",["HH",2],0,"hour"),G("h",["hh",2],0,Vt),G("k",["kk",2],0,Bt),G("hmm",0,0,function(){return""+Vt.apply(this)+Y(this.minutes(),2)}),G("hmmss",0,0,function(){return""+Vt.apply(this)+Y(this.minutes(),2)+Y(this.seconds(),2)}),G("Hmm",0,0,function(){return""+this.hours()+Y(this.minutes(),2)}),G("Hmmss",0,0,function(){return""+this.hours()+Y(this.minutes(),2)+Y(this.seconds(),2)}),Ut("a",!0),Ut("A",!1),R("hour","h"),H("hour",13),U("a",qt),U("A",qt),U("H",Xo),U("h",Xo),U("k",Xo),U("HH",Xo,Vo),U("hh",Xo,Vo),U("kk",Xo,Vo),U("hmm",Zo),U("hmmss",Ko),U("Hmm",Zo),U("Hmmss",Ko),K(["H","HH"],pn),K(["k","kk"],function(t,e,i){var o=_(t);e[pn]=24===o?0:o}),K(["a","A"],function(t,e,i){i._isPm=i._locale.isPM(t),i._meridiem=t}),K(["h","hh"],function(t,e,i){e[pn]=_(t),c(i).bigHour=!0}),K("hmm",function(t,e,i){var o=t.length-2;e[pn]=_(t.substr(0,o)),e[cn]=_(t.substr(o)),c(i).bigHour=!0}),K("hmmss",function(t,e,i){var o=t.length-4,n=t.length-2;e[pn]=_(t.substr(0,o)),e[cn]=_(t.substr(o,2)),e[mn]=_(t.substr(n)),c(i).bigHour=!0}),K("Hmm",function(t,e,i){var o=t.length-2;e[pn]=_(t.substr(0,o)),e[cn]=_(t.substr(o))}),K("Hmmss",function(t,e,i){var o=t.length-4,n=t.length-2;e[pn]=_(t.substr(0,o)),e[cn]=_(t.substr(o,2)),e[mn]=_(t.substr(n))});var In,Nn=/[ap]\.?m?\.?/i,An=it("Hours",!0),Rn={calendar:No,longDateFormat:Ao,invalidDate:"Invalid date",ordinal:"%d",dayOfMonthOrdinalParse:Ro,relativeTime:Lo,months:wn,monthsShort:xn,week:kn,weekdays:Cn,weekdaysMin:Mn,weekdaysShort:Tn,meridiemParse:Nn},Ln={},Fn={},Hn=/^\s*((?:[+-]\d{6}|\d{4})-(?:\d\d-\d\d|W\d\d-\d|W\d\d|\d\d\d|\d\d))(?:(T| )(\d\d(?::\d\d(?::\d\d(?:[.,]\d+)?)?)?)([\+\-]\d\d(?::?\d\d)?|\s*Z)?)?$/,jn=/^\s*((?:[+-]\d{6}|\d{4})(?:\d\d\d\d|W\d\d\d|W\d\d|\d\d\d|\d\d))(?:(T| )(\d\d(?:\d\d(?:\d\d(?:[.,]\d+)?)?)?)([\+\-]\d\d(?::?\d\d)?|\s*Z)?)?$/,Yn=/Z|[+-]\d\d(?::?\d\d)?/,Gn=[["YYYYYY-MM-DD",/[+-]\d{6}-\d\d-\d\d/],["YYYY-MM-DD",/\d{4}-\d\d-\d\d/],["GGGG-[W]WW-E",/\d{4}-W\d\d-\d/],["GGGG-[W]WW",/\d{4}-W\d\d/,!1],["YYYY-DDD",/\d{4}-\d{3}/],["YYYY-MM",/\d{4}-\d\d/,!1],["YYYYYYMMDD",/[+-]\d{10}/],["YYYYMMDD",/\d{8}/],["GGGG[W]WWE",/\d{4}W\d{3}/],["GGGG[W]WW",/\d{4}W\d{2}/,!1],["YYYYDDD",/\d{7}/]],zn=[["HH:mm:ss.SSSS",/\d\d:\d\d:\d\d\.\d+/],["HH:mm:ss,SSSS",/\d\d:\d\d:\d\d,\d+/],["HH:mm:ss",/\d\d:\d\d:\d\d/],["HH:mm",/\d\d:\d\d/],["HHmmss.SSSS",/\d\d\d\d\d\d\.\d+/],["HHmmss,SSSS",/\d\d\d\d\d\d,\d+/],["HHmmss",/\d\d\d\d\d\d/],["HHmm",/\d\d\d\d/],["HH",/\d\d/]],Wn=/^\/?Date\((\-?\d+)/i,Vn=/^(?:(Mon|Tue|Wed|Thu|Fri|Sat|Sun),?\s)?(\d{1,2})\s(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s(\d{2,4})\s(\d\d):(\d\d)(?::(\d\d))?\s(?:(UT|GMT|[ECMP][SD]T)|([Zz])|([+-]\d{4}))$/,Bn={UT:0,GMT:0,EDT:-240,EST:-300,CDT:-300,CST:-360,MDT:-360,MST:-420,PDT:-420,PST:-480};e.createFromInputFallback=D("value provided is not in a recognized RFC2822 or ISO format. moment construction falls back to js Date(), which is not reliable across all browsers and versions. Non RFC2822/ISO date formats are discouraged and will be removed in an upcoming major release. Please refer to http://momentjs.com/guides/#/warnings/js-date/ for more info.",function(t){t._d=new Date(t._i+(t._useUTC?" UTC":""))}),e.ISO_8601=function(){},e.RFC_2822=function(){};var Un=D("moment().min is deprecated, use moment.max instead. http://momentjs.com/guides/#/warnings/min-max/",function(){var t=ke.apply(null,arguments);return this.isValid()&&t.isValid()?t<this?this:t:f()}),qn=D("moment().max is deprecated, use moment.min instead. http://momentjs.com/guides/#/warnings/min-max/",function(){var t=ke.apply(null,arguments);return this.isValid()&&t.isValid()?t>this?this:t:f()}),Xn=function(){return Date.now?Date.now():+new Date},Zn=["year","quarter","month","week","day","hour","minute","second","millisecond"];Re("Z",":"),Re("ZZ",""),U("Z",nn),U("ZZ",nn),K(["Z","ZZ"],function(t,e,i){i._useUTC=!0,i._tzm=Le(nn,t)});var Kn=/([\+\-]|\d\d)/gi;e.updateOffset=function(){};var Jn=/^(\-|\+)?(?:(\d*)[. ])?(\d+)\:(\d+)(?:\:(\d+)(\.\d*)?)?$/,$n=/^(-|\+)?P(?:([-+]?[0-9,.]*)Y)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)W)?(?:([-+]?[0-9,.]*)D)?(?:T(?:([-+]?[0-9,.]*)H)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)S)?)?$/;Ke.fn=Ie.prototype,Ke.invalid=Pe;var Qn=ti(1,"add"),ts=ti(-1,"subtract");e.defaultFormat="YYYY-MM-DDTHH:mm:ssZ",e.defaultFormatUtc="YYYY-MM-DDTHH:mm:ss[Z]";var es=D("moment().lang() is deprecated. Instead, use moment().localeData() to get the language configuration. Use moment().locale() to change languages.",function(t){return void 0===t?this.localeData():this.locale(t)});G(0,["gg",2],0,function(){return this.weekYear()%100}),G(0,["GG",2],0,function(){return this.isoWeekYear()%100}),Ri("gggg","weekYear"),Ri("ggggg","weekYear"),Ri("GGGG","isoWeekYear"),Ri("GGGGG","isoWeekYear"),R("weekYear","gg"),R("isoWeekYear","GG"),H("weekYear",1),H("isoWeekYear",1),U("G",en),U("g",en),U("GG",Xo,Vo),U("gg",Xo,Vo),U("GGGG",$o,Uo),U("gggg",$o,Uo),U("GGGGG",Qo,qo),U("ggggg",Qo,qo),J(["gggg","ggggg","GGGG","GGGGG"],function(t,e,i,o){e[o.substr(0,2)]=_(t)}),J(["gg","GG"],function(t,i,o,n){i[n]=e.parseTwoDigitYear(t)}),G("Q",0,"Qo","quarter"),R("quarter","Q"),H("quarter",7),U("Q",Wo),K("Q",function(t,e){e[ln]=3*(_(t)-1)}),G("D",["DD",2],"Do","date"),R("date","D"),H("date",9),U("D",Xo),U("DD",Xo,Vo),U("Do",function(t,e){return t?e._dayOfMonthOrdinalParse||e._ordinalParse:e._dayOfMonthOrdinalParseLenient}),K(["D","DD"],un),K("Do",function(t,e){e[un]=_(t.match(Xo)[0],10)});var is=it("Date",!0);G("DDD",["DDDD",3],"DDDo","dayOfYear"),R("dayOfYear","DDD"),H("dayOfYear",4),U("DDD",Jo),U("DDDD",Bo),K(["DDD","DDDD"],function(t,e,i){i._dayOfYear=_(t)}),G("m",["mm",2],0,"minute"),R("minute","m"),H("minute",14),U("m",Xo),U("mm",Xo,Vo),K(["m","mm"],cn);var os=it("Minutes",!1);G("s",["ss",2],0,"second"),R("second","s"),H("second",15),U("s",Xo),U("ss",Xo,Vo),K(["s","ss"],mn);var ns=it("Seconds",!1);G("S",0,0,function(){return~~(this.millisecond()/100)}),G(0,["SS",2],0,function(){return~~(this.millisecond()/10)}),G(0,["SSS",3],0,"millisecond"),G(0,["SSSS",4],0,function(){return 10*this.millisecond()}),G(0,["SSSSS",5],0,function(){return 100*this.millisecond()}),G(0,["SSSSSS",6],0,function(){return 1e3*this.millisecond()}),G(0,["SSSSSSS",7],0,function(){return 1e4*this.millisecond()}),G(0,["SSSSSSSS",8],0,function(){return 1e5*this.millisecond()}),G(0,["SSSSSSSSS",9],0,function(){return 1e6*this.millisecond()}),R("millisecond","ms"),H("millisecond",16),U("S",Jo,Wo),U("SS",Jo,Vo),U("SSS",Jo,Bo);var ss;for(ss="SSSS";ss.length<=9;ss+="S")U(ss,tn);for(ss="S";ss.length<=9;ss+="S")K(ss,Vi);var rs=it("Milliseconds",!1);G("z",0,0,"zoneAbbr"),G("zz",0,0,"zoneName");var as=v.prototype;as.add=Qn,as.calendar=oi,as.clone=ni,as.diff=ui,as.endOf=Si,as.format=gi,as.from=vi,as.fromNow=yi,as.to=bi,as.toNow=_i,as.get=st,as.invalidAt=Ni,as.isAfter=si,as.isBefore=ri,as.isBetween=ai,as.isSame=hi,as.isSameOrAfter=di,as.isSameOrBefore=li,as.isValid=Pi,as.lang=es,as.locale=wi,as.localeData=xi,as.max=qn,as.min=Un,as.parsingFlags=Ii,as.set=rt,as.startOf=Di,as.subtract=ts,as.toArray=Mi,as.toObject=Oi,as.toDate=Ti,as.toISOString=mi,as.inspect=fi,as.toJSON=Ei,as.toString=ci,as.unix=Ci,as.valueOf=ki,as.creationData=Ai,as.year=bn,as.isLeapYear=et,as.weekYear=Li,as.isoWeekYear=Fi,as.quarter=as.quarters=zi,as.month=mt,as.daysInMonth=ft,as.week=as.weeks=Mt,as.isoWeek=as.isoWeeks=Ot,as.weeksInYear=ji,as.isoWeeksInYear=Hi,as.date=is,as.day=as.days=Ft,as.weekday=Ht,as.isoWeekday=jt,as.dayOfYear=Wi,as.hour=as.hours=An,as.minute=as.minutes=os,as.second=as.seconds=ns,as.millisecond=as.milliseconds=rs,as.utcOffset=je,as.utc=Ge,as.local=ze,as.parseZone=We,as.hasAlignedHourOffset=Ve,as.isDST=Be,as.isLocal=qe,as.isUtcOffset=Xe,as.isUtc=Ze,as.isUTC=Ze,as.zoneAbbr=Bi,as.zoneName=Ui,as.dates=D("dates accessor is deprecated. Use date instead.",is),as.months=D("months accessor is deprecated. Use month instead",mt),as.years=D("years accessor is deprecated. Use year instead",bn),as.zone=D("moment().zone is deprecated, use moment().utcOffset instead. http://momentjs.com/guides/#/warnings/zone/",Ye),as.isDSTShifted=D("isDSTShifted is deprecated. See http://momentjs.com/guides/#/warnings/dst-shifted/ for more information",Ue);var hs=M.prototype;hs.calendar=O,hs.longDateFormat=E,hs.invalidDate=P,hs.ordinal=I,hs.preparse=Zi,hs.postformat=Zi,hs.relativeTime=N,hs.pastFuture=A,hs.set=C,hs.months=dt,hs.monthsShort=lt,hs.monthsParse=pt,hs.monthsRegex=vt,hs.monthsShortRegex=gt,hs.week=kt,hs.firstDayOfYear=Tt,hs.firstDayOfWeek=Ct,hs.weekdays=It,hs.weekdaysMin=At,hs.weekdaysShort=Nt,hs.weekdaysParse=Lt,hs.weekdaysRegex=Yt,hs.weekdaysShortRegex=Gt,hs.weekdaysMinRegex=zt,hs.isPM=Xt,hs.meridiem=Zt,Qt("en",{dayOfMonthOrdinalParse:/\d{1,2}(th|st|nd|rd)/,ordinal:function(t){var e=t%10;return t+(1===_(t%100/10)?"th":1===e?"st":2===e?"nd":3===e?"rd":"th")}}),e.lang=D("moment.lang is deprecated. Use moment.locale instead.",Qt),e.langData=D("moment.langData is deprecated. Use moment.localeData instead.",ie);var ds=Math.abs,ls=fo("ms"),us=fo("s"),ps=fo("m"),cs=fo("h"),ms=fo("d"),fs=fo("w"),gs=fo("M"),vs=fo("y"),ys=yo("milliseconds"),bs=yo("seconds"),_s=yo("minutes"),ws=yo("hours"),xs=yo("days"),Ds=yo("months"),Ss=yo("years"),ks=Math.round,Cs={ss:44,s:45,m:45,h:22,d:26,M:11},Ts=Math.abs,Ms=Ie.prototype;return Ms.isValid=Ee,Ms.abs=no,Ms.add=ro,Ms.subtract=ao,Ms.as=co,Ms.asMilliseconds=ls,Ms.asSeconds=us,Ms.asMinutes=ps,Ms.asHours=cs,Ms.asDays=ms,Ms.asWeeks=fs,Ms.asMonths=gs,Ms.asYears=vs,Ms.valueOf=mo,Ms._bubble=lo,Ms.clone=go,Ms.get=vo,Ms.milliseconds=ys,Ms.seconds=bs,Ms.minutes=_s,Ms.hours=ws,Ms.days=xs,Ms.weeks=bo,Ms.months=Ds,Ms.years=Ss,Ms.humanize=So,Ms.toISOString=Co,Ms.toString=Co,Ms.toJSON=Co,Ms.locale=wi,Ms.localeData=xi,Ms.toIsoString=D("toIsoString() is deprecated. Please use toISOString() instead (notice the capitals)",Co),Ms.lang=es,G("X",0,0,"unix"),G("x",0,0,"valueOf"),U("x",en),U("X",sn),K("X",function(t,e,i){i._d=new Date(1e3*parseFloat(t,10))}),K("x",function(t,e,i){i._d=new Date(_(t))}),e.version="2.19.1",function(t){To=t}(ke),e.fn=as,e.min=Te,e.max=Me,e.now=Xn,e.utc=u,e.unix=qi,e.months=Qi,e.isDate=a,e.locale=Qt,e.invalid=f,e.duration=Ke,e.isMoment=y,e.weekdays=eo,e.parseZone=Xi,e.localeData=ie,e.isDuration=Ne,e.monthsShort=to,e.weekdaysMin=oo,e.defineLocale=te,e.updateLocale=ee,e.locales=oe,e.weekdaysShort=io,e.normalizeUnits=L,e.relativeTimeRounding=xo,e.relativeTimeThreshold=Do,e.calendarFormat=ii,e.prototype=as,e})}).call(e,i(124)(t))},function(t,e){t.exports=function(t){return t.webpackPolyfill||(t.deprecate=function(){},t.paths=[],t.children||(t.children=[]),Object.defineProperty(t,"loaded",{enumerable:!0,get:function(){return t.l}}),Object.defineProperty(t,"id",{enumerable:!0,get:function(){return t.i}}),t.webpackPolyfill=1),t}},function(t,e){function i(t){throw new Error("Cannot find module '"+t+"'.")}i.keys=function(){return[]},i.resolve=i,t.exports=i,i.id=125},function(t,e,i){(function(e){function i(t,e,i){var o=e&&i||0,n=0;for(e=e||[],t.toLowerCase().replace(/[0-9a-f]{2}/g,function(t){n<16&&(e[o+n++]=u[t])});n<16;)e[o+n++]=0;return e}function o(t,e){var i=e||0,o=l;return o[t[i++]]+o[t[i++]]+o[t[i++]]+o[t[i++]]+"-"+o[t[i++]]+o[t[i++]]+"-"+o[t[i++]]+o[t[i++]]+"-"+o[t[i++]]+o[t[i++]]+"-"+o[t[i++]]+o[t[i++]]+o[t[i++]]+o[t[i++]]+o[t[i++]]+o[t[i++]]}function n(t,e,i){var n=e&&i||0,s=e||[];t=t||{};var r=void 0!==t.clockseq?t.clockseq:f,a=void 0!==t.msecs?t.msecs:(new Date).getTime(),h=void 0!==t.nsecs?t.nsecs:v+1,d=a-g+(h-v)/1e4;if(d<0&&void 0===t.clockseq&&(r=r+1&16383),(d<0||a>g)&&void 0===t.nsecs&&(h=0),h>=1e4)throw new Error("uuid.v1(): Can't create more than 10M uuids/sec");g=a,v=h,f=r,a+=122192928e5;var l=(1e4*(268435455&a)+h)%4294967296;s[n++]=l>>>24&255,s[n++]=l>>>16&255,s[n++]=l>>>8&255,s[n++]=255&l;var u=a/4294967296*1e4&268435455;s[n++]=u>>>8&255,s[n++]=255&u,s[n++]=u>>>24&15|16,s[n++]=u>>>16&255,s[n++]=r>>>8|128,s[n++]=255&r;for(var p=t.node||m,c=0;c<6;c++)s[n+c]=p[c];return e||o(s)}function s(t,e,i){var n=e&&i||0;"string"==typeof t&&(e="binary"==t?new Array(16):null,t=null),t=t||{};var s=t.random||(t.rng||r)();if(s[6]=15&s[6]|64,s[8]=63&s[8]|128,e)for(var a=0;a<16;a++)e[n+a]=s[a];return e||o(s)}var r,a="undefined"!=typeof window?window:void 0!==e?e:null;if(a&&a.crypto&&crypto.getRandomValues){var h=new Uint8Array(16);r=function(){return crypto.getRandomValues(h),h}}if(!r){var d=new Array(16);r=function(){for(var t,e=0;e<16;e++)0==(3&e)&&(t=4294967296*Math.random()),d[e]=t>>>((3&e)<<3)&255;return d}}for(var l=[],u={},p=0;p<256;p++)l[p]=(p+256).toString(16).substr(1),u[l[p]]=p;var c=r(),m=[1|c[0],c[1],c[2],c[3],c[4],c[5]],f=16383&(c[6]<<8|c[7]),g=0,v=0,y=s;y.v1=n,y.v4=s,y.parse=i,y.unparse=o,t.exports=y}).call(e,i(127))},function(t,e){var i;i=function(){return this}();try{i=i||Function("return this")()||(0,eval)("this")}catch(t){"object"==typeof window&&(i=window)}t.exports=i},function(t,e,i){var o=i(2),n=o.JSON||(o.JSON={stringify:JSON.stringify});t.exports=function(t){return n.stringify.apply(n,arguments)}},function(t,e,i){function o(t,e,i,n){if(!(this instanceof o))throw new SyntaxError("Constructor must be called with the new operator");if(!(Array.isArray(i)||i instanceof d||i instanceof l)&&i instanceof Object){var s=n;n=i,i=s}n&&n.throttleRedraw&&console.warn('Timeline option "throttleRedraw" is DEPRICATED and no longer supported. It will be removed in the next MAJOR release.');var r=this;if(this.defaultOptions={start:null,end:null,autoResize:!0,orientation:{axis:"bottom",item:"bottom"},moment:a,width:null,height:null,maxHeight:null,minHeight:null},this.options=h.deepExtend({},this.defaultOptions),this._create(t),!n||n&&void 0===n.rtl){this.dom.root.style.visibility="hidden";for(var p,f=this.dom.root;!p&&f;)p=window.getComputedStyle(f,null).direction,f=f.parentElement;this.options.rtl=p&&"rtl"==p.toLowerCase()}else this.options.rtl=n.rtl;this.options.rollingMode=n&&n.rollingMode,this.options.onInitialDrawComplete=n&&n.onInitialDrawComplete,this.components=[],this.body={dom:this.dom,domProps:this.props,emitter:{on:this.on.bind(this),off:this.off.bind(this),emit:this.emit.bind(this)},hiddenDates:[],util:{getScale:function(){return r.timeAxis.step.scale},getStep:function(){return r.timeAxis.step.step},toScreen:r._toScreen.bind(r),toGlobalScreen:r._toGlobalScreen.bind(r),toTime:r._toTime.bind(r),toGlobalTime:r._toGlobalTime.bind(r)}},this.range=new u(this.body,this.options),this.components.push(this.range),this.body.range=this.range,this.timeAxis=new c(this.body,this.options),this.timeAxis2=null,this.components.push(this.timeAxis),this.currentTime=new m(this.body,this.options),this.components.push(this.currentTime),this.itemSet=new g(this.body,this.options),this.components.push(this.itemSet),this.itemsData=null,this.groupsData=null,this.dom.root.onclick=function(t){r.emit("click",r.getEventProperties(t))},this.dom.root.ondblclick=function(t){r.emit("doubleClick",r.getEventProperties(t))},this.dom.root.oncontextmenu=function(t){r.emit("contextmenu",r.getEventProperties(t))},this.dom.root.onmouseover=function(t){r.emit("mouseOver",r.getEventProperties(t))},window.PointerEvent?(this.dom.root.onpointerdown=function(t){r.emit("mouseDown",r.getEventProperties(t))},this.dom.root.onpointermove=function(t){r.emit("mouseMove",r.getEventProperties(t))},this.dom.root.onpointerup=function(t){r.emit("mouseUp",r.getEventProperties(t))}):(this.dom.root.onmousemove=function(t){r.emit("mouseMove",r.getEventProperties(t))},this.dom.root.onmousedown=function(t){r.emit("mouseDown",r.getEventProperties(t))},this.dom.root.onmouseup=function(t){r.emit("mouseUp",r.getEventProperties(t))}),this.initialFitDone=!1,this.on("changed",function(){if(null!=this.itemsData&&!this.options.rollingMode){if(!r.initialFitDone)if(r.initialFitDone=!0,void 0!=r.options.start||void 0!=r.options.end){if(void 0==r.options.start||void 0==r.options.end)var t=r.getItemRange();var e=void 0!=r.options.start?r.options.start:t.min,i=void 0!=r.options.end?r.options.end:t.max;r.setWindow(e,i,{animation:!1})}else r.fit({animation:!1});!r.initialDrawDone&&r.initialRangeChangeDone&&(r.initialDrawDone=!0,r.dom.root.style.visibility="visible",r.options.onInitialDrawComplete&&setTimeout(function(){return r.options.onInitialDrawComplete()},0))}}),n&&this.setOptions(n),i&&this.setGroups(i),e&&this.setItems(e),this._redraw()}function n(t){return h.convert(t.data.start,"Date").valueOf()}function s(t){var e=void 0!=t.data.end?t.data.end:t.data.start;return h.convert(e,"Date").valueOf()}function r(t,e){var i=t.props.leftContainer.height,o=t.props.left.height,n=e.parent,s=n.top,r=!0,a=t.timeAxis.options.orientation.axis,h=function(){return"bottom"==a?n.height-e.top-e.height:e.top},d=-1*t._getScrollTop(),l=s+h(),u=e.height;return l<d?s+i<=s+h()+u&&(s+=h()-t.itemSet.options.margin.item.vertical):l+u>d+i?s+=h()+u-i+t.itemSet.options.margin.item.vertical:r=!1,s=Math.min(s,o-i),{shouldScroll:r,scrollOffset:s,itemTop:l}}var a=i(3),h=i(0),d=i(15),l=i(20),u=i(47),p=i(48),c=i(28),m=i(50),f=i(29),g=i(70),v=i(32).printStyle,y=i(75).allOptions,b=i(75).configureOptions,_=i(76).default,w=i(32).default;o.prototype=new p,o.prototype._createConfigurator=function(){return new _(this,this.dom.container,b)},o.prototype.redraw=function(){this.itemSet&&this.itemSet.markDirty({refreshItems:!0}),this._redraw()},o.prototype.setOptions=function(t){if(!0===w.validate(t,y)&&console.log("%cErrors have been found in the supplied options object.",v),p.prototype.setOptions.call(this,t),"type"in t&&t.type!==this.options.type){this.options.type=t.type;var e=this.itemsData;if(e){var i=this.getSelection();this.setItems(null),this.setItems(e),this.setSelection(i)}}},o.prototype.setItems=function(t){var e;e=t?t instanceof d||t instanceof l?t:new d(t,{type:{start:"Date",end:"Date"}}):null,this.itemsData=e,this.itemSet&&this.itemSet.setItems(e)},o.prototype.setGroups=function(t){var e;if(t){var i=function(t){return!1!==t.visible};e=t instanceof d||t instanceof l?new l(t,{filter:i}):new d(t.filter(i))}else e=null;this.groupsData=e,this.itemSet.setGroups(e)},o.prototype.setData=function(t){t&&t.groups&&this.setGroups(t.groups),t&&t.items&&this.setItems(t.items)},o.prototype.setSelection=function(t,e){this.itemSet&&this.itemSet.setSelection(t),e&&e.focus&&this.focus(t,e)},o.prototype.getSelection=function(){return this.itemSet&&this.itemSet.getSelection()||[]},o.prototype.focus=function(t,e){if(this.itemsData&&void 0!=t){var i=Array.isArray(t)?t:[t],o=this.itemsData.getDataSet().get(i,{type:{start:"Date",end:"Date"}}),n=null,s=null;if(o.forEach(function(t){var e=t.start.valueOf(),i="end"in t?t.end.valueOf():t.start.valueOf();(null===n||e<n)&&(n=e),(null===s||i>s)&&(s=i)}),null!==n&&null!==s){var a=this,h=this.itemSet.items[i[0]],d=-1*this._getScrollTop(),l=null,u=function(t,e,i){var o=r(a,h);if(l||(l=o),l.itemTop!=o.itemTop||l.shouldScroll){l.itemTop!=o.itemTop&&o.shouldScroll&&(l=o,d=-1*a._getScrollTop());var n=d,s=l.scrollOffset,u=i?s:n+(s-n)*t;a._setScrollTop(-u),e||a._redraw()}},p=function(){var t=r(a,h);t.shouldScroll&&t.itemTop!=l.itemTop&&(a._setScrollTop(-t.scrollOffset),a._redraw())},c=function(){p(),setTimeout(p,100)},m=(n+s)/2,f=Math.max(this.range.end-this.range.start,1.1*(s-n)),g=!e||void 0===e.animation||e.animation;g||(l={shouldScroll:!1,scrollOffset:-1,itemTop:-1}),this.range.setRange(m-f/2,m+f/2,{animation:g},c,u)}}},o.prototype.fit=function(t,e){var i,o=!t||void 0===t.animation||t.animation,n=this.itemsData&&this.itemsData.getDataSet();1===n.length&&void 0===n.get()[0].end?(i=this.getDataRange(),this.moveTo(i.min.valueOf(),{animation:o},e)):(i=this.getItemRange(),this.range.setRange(i.min,i.max,{animation:o},e))},o.prototype.getItemRange=function(){var t=this.getDataRange(),e=null!==t.min?t.min.valueOf():null,i=null!==t.max?t.max.valueOf():null,o=null,r=null;if(null!=e&&null!=i){var a=i-e;a<=0&&(a=10);var d=a/this.props.center.width,l={},u=0;h.forEach(this.itemSet.items,function(t,e){if(t.groupShowing){l[e]=t.redraw(!0),u=l[e].length}});if(u>0)for(var p=0;p<u;p++)h.forEach(l,function(t){t[p]()});if(h.forEach(this.itemSet.items,function(t){var a,h,l=n(t),u=s(t);this.options.rtl?(a=l-(t.getWidthRight()+10)*d,h=u+(t.getWidthLeft()+10)*d):(a=l-(t.getWidthLeft()+10)*d,h=u+(t.getWidthRight()+10)*d),a<e&&(e=a,o=t),h>i&&(i=h,r=t)}.bind(this)),o&&r){var c=o.getWidthLeft()+10,m=r.getWidthRight()+10,f=this.props.center.width-c-m;f>0&&(this.options.rtl?(e=n(o)-m*a/f,i=s(r)+c*a/f):(e=n(o)-c*a/f,i=s(r)+m*a/f))}}return{min:null!=e?new Date(e):null,max:null!=i?new Date(i):null}},o.prototype.getDataRange=function(){var t=null,e=null,i=this.itemsData&&this.itemsData.getDataSet();return i&&i.forEach(function(i){var o=h.convert(i.start,"Date").valueOf(),n=h.convert(void 0!=i.end?i.end:i.start,"Date").valueOf();(null===t||o<t)&&(t=o),(null===e||n>e)&&(e=n)}),{min:null!=t?new Date(t):null,max:null!=e?new Date(e):null}},o.prototype.getEventProperties=function(t){var e,i=t.center?t.center.x:t.clientX,o=t.center?t.center.y:t.clientY;e=this.options.rtl?h.getAbsoluteRight(this.dom.centerContainer)-i:i-h.getAbsoluteLeft(this.dom.centerContainer);var n=o-h.getAbsoluteTop(this.dom.centerContainer),s=this.itemSet.itemFromTarget(t),r=this.itemSet.groupFromTarget(t),a=f.customTimeFromTarget(t),d=this.itemSet.options.snap||null,l=this.body.util.getScale(),u=this.body.util.getStep(),p=this._toTime(e),c=d?d(p,l,u):p,m=h.getTarget(t),g=null;return null!=s?g="item":null!=a?g="custom-time":h.hasParent(m,this.timeAxis.dom.foreground)?g="axis":this.timeAxis2&&h.hasParent(m,this.timeAxis2.dom.foreground)?g="axis":h.hasParent(m,this.itemSet.dom.labelSet)?g="group-label":h.hasParent(m,this.currentTime.bar)?g="current-time":h.hasParent(m,this.dom.center)&&(g="background"),{event:t,item:s?s.id:null,group:r?r.groupId:null,what:g,pageX:t.srcEvent?t.srcEvent.pageX:t.pageX,pageY:t.srcEvent?t.srcEvent.pageY:t.pageY,x:e,y:n,time:p,snappedTime:c}},o.prototype.toggleRollingMode=function(){this.range.rolling?this.range.stopRolling():(void 0==this.options.rollingMode&&this.setOptions(this.options),this.range.startRolling())},t.exports=o},function(t,e,i){var o,n,s;!function(i){n=[],o=i,void 0!==(s="function"==typeof o?o.apply(e,n):o)&&(t.exports=s)}(function(){var t=null;return function e(i,o){function n(t){return t.match(/[^ ]+/g)}function s(e){if("hammer.input"!==e.type){if(e.srcEvent._handled||(e.srcEvent._handled={}),e.srcEvent._handled[e.type])return;e.srcEvent._handled[e.type]=!0}var i=!1;e.stopPropagation=function(){i=!0};var o=e.srcEvent.stopPropagation.bind(e.srcEvent);"function"==typeof o&&(e.srcEvent.stopPropagation=function(){o(),e.stopPropagation()}),e.firstTarget=t;for(var n=t;n&&!i;){var s=n.hammer;if(s)for(var r,a=0;a<s.length;a++)if(r=s[a]._handlers[e.type])for(var h=0;h<r.length&&!i;h++)r[h](e);n=n.parentNode}}var r=o||{preventDefault:!1};if(i.Manager){var a=i,h=function(t,i){var o=Object.create(r);return i&&a.assign(o,i),e(new a(t,o),o)};return a.assign(h,a),h.Manager=function(t,i){var o=Object.create(r);return i&&a.assign(o,i),e(new a.Manager(t,o),o)},h}var d=Object.create(i),l=i.element;return l.hammer||(l.hammer=[]),l.hammer.push(d),i.on("hammer.input",function(e){!0!==r.preventDefault&&r.preventDefault!==e.pointerType||e.preventDefault(),e.isFirst&&(t=e.target)}),d._handlers={},d.on=function(t,e){return n(t).forEach(function(t){var o=d._handlers[t];o||(d._handlers[t]=o=[],i.on(t,s)),o.push(e)}),d},d.off=function(t,e){return n(t).forEach(function(t){var o=d._handlers[t];o&&(o=e?o.filter(function(t){return t!==e}):[],o.length>0?d._handlers[t]=o:(i.off(t,s),delete d._handlers[t]))}),d},d.emit=function(e,o){t=o.target,i.emit(e,o)},d.destroy=function(){var t=i.element.hammer,e=t.indexOf(d);-1!==e&&t.splice(e,1),t.length||delete i.element.hammer,d._handlers={},i.destroy()},d}})},function(t,e,i){var o;/*! Hammer.JS - v2.0.7 - 2016-04-22
+ * http://hammerjs.github.io/
+ *
+ * Copyright (c) 2016 Jorik Tangelder;
+ * Licensed under the MIT license */
+!function(n,s,r,a){function h(t,e,i){return setTimeout(c(t,i),e)}function d(t,e,i){return!!Array.isArray(t)&&(l(t,i[e],i),!0)}function l(t,e,i){var o;if(t)if(t.forEach)t.forEach(e,i);else if(t.length!==a)for(o=0;o<t.length;)e.call(i,t[o],o,t),o++;else for(o in t)t.hasOwnProperty(o)&&e.call(i,t[o],o,t)}function u(t,e,i){var o="DEPRECATED METHOD: "+e+"\n"+i+" AT \n";return function(){var e=new Error("get-stack-trace"),i=e&&e.stack?e.stack.replace(/^[^\(]+?[\n$]/gm,"").replace(/^\s+at\s+/gm,"").replace(/^Object.<anonymous>\s*\(/gm,"{anonymous}()@"):"Unknown Stack Trace",s=n.console&&(n.console.warn||n.console.log);return s&&s.call(n.console,o,i),t.apply(this,arguments)}}function p(t,e,i){var o,n=e.prototype;o=t.prototype=Object.create(n),o.constructor=t,o._super=n,i&&mt(o,i)}function c(t,e){return function(){return t.apply(e,arguments)}}function m(t,e){return typeof t==vt?t.apply(e?e[0]||a:a,e):t}function f(t,e){return t===a?e:t}function g(t,e,i){l(_(e),function(e){t.addEventListener(e,i,!1)})}function v(t,e,i){l(_(e),function(e){t.removeEventListener(e,i,!1)})}function y(t,e){for(;t;){if(t==e)return!0;t=t.parentNode}return!1}function b(t,e){return t.indexOf(e)>-1}function _(t){return t.trim().split(/\s+/g)}function w(t,e,i){if(t.indexOf&&!i)return t.indexOf(e);for(var o=0;o<t.length;){if(i&&t[o][i]==e||!i&&t[o]===e)return o;o++}return-1}function x(t){return Array.prototype.slice.call(t,0)}function D(t,e,i){for(var o=[],n=[],s=0;s<t.length;){var r=e?t[s][e]:t[s];w(n,r)<0&&o.push(t[s]),n[s]=r,s++}return i&&(o=e?o.sort(function(t,i){return t[e]>i[e]}):o.sort()),o}function S(t,e){for(var i,o,n=e[0].toUpperCase()+e.slice(1),s=0;s<ft.length;){if(i=ft[s],(o=i?i+n:e)in t)return o;s++}return a}function k(){return Dt++}function C(t){var e=t.ownerDocument||t;return e.defaultView||e.parentWindow||n}function T(t,e){var i=this;this.manager=t,this.callback=e,this.element=t.element,this.target=t.options.inputTarget,this.domHandler=function(e){m(t.options.enable,[t])&&i.handler(e)},this.init()}function M(t){var e=t.options.inputClass;return new(e||(Ct?z:Tt?B:kt?q:G))(t,O)}function O(t,e,i){var o=i.pointers.length,n=i.changedPointers.length,s=e&Ot&&o-n==0,r=e&(Pt|It)&&o-n==0;i.isFirst=!!s,i.isFinal=!!r,s&&(t.session={}),i.eventType=e,E(t,i),t.emit("hammer.input",i),t.recognize(i),t.session.prevInput=i}function E(t,e){var i=t.session,o=e.pointers,n=o.length;i.firstInput||(i.firstInput=N(e)),n>1&&!i.firstMultiple?i.firstMultiple=N(e):1===n&&(i.firstMultiple=!1);var s=i.firstInput,r=i.firstMultiple,a=r?r.center:s.center,h=e.center=A(o);e.timeStamp=_t(),e.deltaTime=e.timeStamp-s.timeStamp,e.angle=H(a,h),e.distance=F(a,h),P(i,e),e.offsetDirection=L(e.deltaX,e.deltaY);var d=R(e.deltaTime,e.deltaX,e.deltaY);e.overallVelocityX=d.x,e.overallVelocityY=d.y,e.overallVelocity=bt(d.x)>bt(d.y)?d.x:d.y,e.scale=r?Y(r.pointers,o):1,e.rotation=r?j(r.pointers,o):0,e.maxPointers=i.prevInput?e.pointers.length>i.prevInput.maxPointers?e.pointers.length:i.prevInput.maxPointers:e.pointers.length,I(i,e);var l=t.element;y(e.srcEvent.target,l)&&(l=e.srcEvent.target),e.target=l}function P(t,e){var i=e.center,o=t.offsetDelta||{},n=t.prevDelta||{},s=t.prevInput||{};e.eventType!==Ot&&s.eventType!==Pt||(n=t.prevDelta={x:s.deltaX||0,y:s.deltaY||0},o=t.offsetDelta={x:i.x,y:i.y}),e.deltaX=n.x+(i.x-o.x),e.deltaY=n.y+(i.y-o.y)}function I(t,e){var i,o,n,s,r=t.lastInterval||e,h=e.timeStamp-r.timeStamp;if(e.eventType!=It&&(h>Mt||r.velocity===a)){var d=e.deltaX-r.deltaX,l=e.deltaY-r.deltaY,u=R(h,d,l);o=u.x,n=u.y,i=bt(u.x)>bt(u.y)?u.x:u.y,s=L(d,l),t.lastInterval=e}else i=r.velocity,o=r.velocityX,n=r.velocityY,s=r.direction;e.velocity=i,e.velocityX=o,e.velocityY=n,e.direction=s}function N(t){for(var e=[],i=0;i<t.pointers.length;)e[i]={clientX:yt(t.pointers[i].clientX),clientY:yt(t.pointers[i].clientY)},i++;return{timeStamp:_t(),pointers:e,center:A(e),deltaX:t.deltaX,deltaY:t.deltaY}}function A(t){var e=t.length;if(1===e)return{x:yt(t[0].clientX),y:yt(t[0].clientY)};for(var i=0,o=0,n=0;n<e;)i+=t[n].clientX,o+=t[n].clientY,n++;return{x:yt(i/e),y:yt(o/e)}}function R(t,e,i){return{x:e/t||0,y:i/t||0}}function L(t,e){return t===e?Nt:bt(t)>=bt(e)?t<0?At:Rt:e<0?Lt:Ft}function F(t,e,i){i||(i=Gt);var o=e[i[0]]-t[i[0]],n=e[i[1]]-t[i[1]];return Math.sqrt(o*o+n*n)}function H(t,e,i){i||(i=Gt);var o=e[i[0]]-t[i[0]],n=e[i[1]]-t[i[1]];return 180*Math.atan2(n,o)/Math.PI}function j(t,e){return H(e[1],e[0],zt)+H(t[1],t[0],zt)}function Y(t,e){return F(e[0],e[1],zt)/F(t[0],t[1],zt)}function G(){this.evEl=Vt,this.evWin=Bt,this.pressed=!1,T.apply(this,arguments)}function z(){this.evEl=Xt,this.evWin=Zt,T.apply(this,arguments),this.store=this.manager.session.pointerEvents=[]}function W(){this.evTarget=Jt,this.evWin=$t,this.started=!1,T.apply(this,arguments)}function V(t,e){var i=x(t.touches),o=x(t.changedTouches);return e&(Pt|It)&&(i=D(i.concat(o),"identifier",!0)),[i,o]}function B(){this.evTarget=te,this.targetIds={},T.apply(this,arguments)}function U(t,e){var i=x(t.touches),o=this.targetIds;if(e&(Ot|Et)&&1===i.length)return o[i[0].identifier]=!0,[i,i];var n,s,r=x(t.changedTouches),a=[],h=this.target;if(s=i.filter(function(t){return y(t.target,h)}),e===Ot)for(n=0;n<s.length;)o[s[n].identifier]=!0,n++;for(n=0;n<r.length;)o[r[n].identifier]&&a.push(r[n]),e&(Pt|It)&&delete o[r[n].identifier],n++;return a.length?[D(s.concat(a),"identifier",!0),a]:void 0}function q(){T.apply(this,arguments);var t=c(this.handler,this);this.touch=new B(this.manager,t),this.mouse=new G(this.manager,t),this.primaryTouch=null,this.lastTouches=[]}function X(t,e){t&Ot?(this.primaryTouch=e.changedPointers[0].identifier,Z.call(this,e)):t&(Pt|It)&&Z.call(this,e)}function Z(t){var e=t.changedPointers[0];if(e.identifier===this.primaryTouch){var i={x:e.clientX,y:e.clientY};this.lastTouches.push(i);var o=this.lastTouches,n=function(){var t=o.indexOf(i);t>-1&&o.splice(t,1)};setTimeout(n,ee)}}function K(t){for(var e=t.srcEvent.clientX,i=t.srcEvent.clientY,o=0;o<this.lastTouches.length;o++){var n=this.lastTouches[o],s=Math.abs(e-n.x),r=Math.abs(i-n.y);if(s<=ie&&r<=ie)return!0}return!1}function J(t,e){this.manager=t,this.set(e)}function $(t){if(b(t,ae))return ae;var e=b(t,he),i=b(t,de);return e&&i?ae:e||i?e?he:de:b(t,re)?re:se}function Q(t){this.options=mt({},this.defaults,t||{}),this.id=k(),this.manager=null,this.options.enable=f(this.options.enable,!0),this.state=ue,this.simultaneous={},this.requireFail=[]}function tt(t){return t&ge?"cancel":t&me?"end":t&ce?"move":t&pe?"start":""}function et(t){return t==Ft?"down":t==Lt?"up":t==At?"left":t==Rt?"right":""}function it(t,e){var i=e.manager;return i?i.get(t):t}function ot(){Q.apply(this,arguments)}function nt(){ot.apply(this,arguments),this.pX=null,this.pY=null}function st(){ot.apply(this,arguments)}function rt(){Q.apply(this,arguments),this._timer=null,this._input=null}function at(){ot.apply(this,arguments)}function ht(){ot.apply(this,arguments)}function dt(){Q.apply(this,arguments),this.pTime=!1,this.pCenter=!1,this._timer=null,this._input=null,this.count=0}function lt(t,e){return e=e||{},e.recognizers=f(e.recognizers,lt.defaults.preset),new ut(t,e)}function ut(t,e){this.options=mt({},lt.defaults,e||{}),this.options.inputTarget=this.options.inputTarget||t,this.handlers={},this.session={},this.recognizers=[],this.oldCssProps={},this.element=t,this.input=M(this),this.touchAction=new J(this,this.options.touchAction),pt(this,!0),l(this.options.recognizers,function(t){var e=this.add(new t[0](t[1]));t[2]&&e.recognizeWith(t[2]),t[3]&&e.requireFailure(t[3])},this)}function pt(t,e){var i=t.element;if(i.style){var o;l(t.options.cssProps,function(n,s){o=S(i.style,s),e?(t.oldCssProps[o]=i.style[o],i.style[o]=n):i.style[o]=t.oldCssProps[o]||""}),e||(t.oldCssProps={})}}function ct(t,e){var i=s.createEvent("Event");i.initEvent(t,!0,!0),i.gesture=e,e.target.dispatchEvent(i)}var mt,ft=["","webkit","Moz","MS","ms","o"],gt=s.createElement("div"),vt="function",yt=Math.round,bt=Math.abs,_t=Date.now;mt="function"!=typeof Object.assign?function(t){if(t===a||null===t)throw new TypeError("Cannot convert undefined or null to object");for(var e=Object(t),i=1;i<arguments.length;i++){var o=arguments[i];if(o!==a&&null!==o)for(var n in o)o.hasOwnProperty(n)&&(e[n]=o[n])}return e}:Object.assign;var wt=u(function(t,e,i){for(var o=Object.keys(e),n=0;n<o.length;)(!i||i&&t[o[n]]===a)&&(t[o[n]]=e[o[n]]),n++;return t},"extend","Use `assign`."),xt=u(function(t,e){return wt(t,e,!0)},"merge","Use `assign`."),Dt=1,St=/mobile|tablet|ip(ad|hone|od)|android/i,kt="ontouchstart"in n,Ct=S(n,"PointerEvent")!==a,Tt=kt&&St.test(navigator.userAgent),Mt=25,Ot=1,Et=2,Pt=4,It=8,Nt=1,At=2,Rt=4,Lt=8,Ft=16,Ht=At|Rt,jt=Lt|Ft,Yt=Ht|jt,Gt=["x","y"],zt=["clientX","clientY"];T.prototype={handler:function(){},init:function(){this.evEl&&g(this.element,this.evEl,this.domHandler),this.evTarget&&g(this.target,this.evTarget,this.domHandler),this.evWin&&g(C(this.element),this.evWin,this.domHandler)},destroy:function(){this.evEl&&v(this.element,this.evEl,this.domHandler),this.evTarget&&v(this.target,this.evTarget,this.domHandler),this.evWin&&v(C(this.element),this.evWin,this.domHandler)}};var Wt={mousedown:Ot,mousemove:Et,mouseup:Pt},Vt="mousedown",Bt="mousemove mouseup";p(G,T,{handler:function(t){var e=Wt[t.type];e&Ot&&0===t.button&&(this.pressed=!0),e&Et&&1!==t.which&&(e=Pt),this.pressed&&(e&Pt&&(this.pressed=!1),this.callback(this.manager,e,{pointers:[t],changedPointers:[t],pointerType:"mouse",srcEvent:t}))}});var Ut={pointerdown:Ot,pointermove:Et,pointerup:Pt,pointercancel:It,pointerout:It},qt={2:"touch",3:"pen",4:"mouse",5:"kinect"},Xt="pointerdown",Zt="pointermove pointerup pointercancel";n.MSPointerEvent&&!n.PointerEvent&&(Xt="MSPointerDown",Zt="MSPointerMove MSPointerUp MSPointerCancel"),p(z,T,{handler:function(t){var e=this.store,i=!1,o=t.type.toLowerCase().replace("ms",""),n=Ut[o],s=qt[t.pointerType]||t.pointerType,r="touch"==s,a=w(e,t.pointerId,"pointerId");n&Ot&&(0===t.button||r)?a<0&&(e.push(t),a=e.length-1):n&(Pt|It)&&(i=!0),a<0||(e[a]=t,this.callback(this.manager,n,{pointers:e,changedPointers:[t],pointerType:s,srcEvent:t}),i&&e.splice(a,1))}});var Kt={touchstart:Ot,touchmove:Et,touchend:Pt,touchcancel:It},Jt="touchstart",$t="touchstart touchmove touchend touchcancel";p(W,T,{handler:function(t){var e=Kt[t.type];if(e===Ot&&(this.started=!0),this.started){var i=V.call(this,t,e);e&(Pt|It)&&i[0].length-i[1].length==0&&(this.started=!1),this.callback(this.manager,e,{pointers:i[0],changedPointers:i[1],pointerType:"touch",srcEvent:t})}}});var Qt={touchstart:Ot,touchmove:Et,touchend:Pt,touchcancel:It},te="touchstart touchmove touchend touchcancel";p(B,T,{handler:function(t){var e=Qt[t.type],i=U.call(this,t,e);i&&this.callback(this.manager,e,{pointers:i[0],changedPointers:i[1],pointerType:"touch",srcEvent:t})}});var ee=2500,ie=25;p(q,T,{handler:function(t,e,i){var o="touch"==i.pointerType,n="mouse"==i.pointerType;if(!(n&&i.sourceCapabilities&&i.sourceCapabilities.firesTouchEvents)){if(o)X.call(this,e,i);else if(n&&K.call(this,i))return;this.callback(t,e,i)}},destroy:function(){this.touch.destroy(),this.mouse.destroy()}});var oe=S(gt.style,"touchAction"),ne=oe!==a,se="auto",re="manipulation",ae="none",he="pan-x",de="pan-y",le=function(){if(!ne)return!1;var t={},e=n.CSS&&n.CSS.supports;return["auto","manipulation","pan-y","pan-x","pan-x pan-y","none"].forEach(function(i){t[i]=!e||n.CSS.supports("touch-action",i)}),t}();J.prototype={set:function(t){"compute"==t&&(t=this.compute()),ne&&this.manager.element.style&&le[t]&&(this.manager.element.style[oe]=t),this.actions=t.toLowerCase().trim()},update:function(){this.set(this.manager.options.touchAction)},compute:function(){var t=[];return l(this.manager.recognizers,function(e){m(e.options.enable,[e])&&(t=t.concat(e.getTouchAction()))}),$(t.join(" "))},preventDefaults:function(t){var e=t.srcEvent,i=t.offsetDirection;if(this.manager.session.prevented)return void e.preventDefault();var o=this.actions,n=b(o,ae)&&!le[ae],s=b(o,de)&&!le[de],r=b(o,he)&&!le[he];if(n){var a=1===t.pointers.length,h=t.distance<2,d=t.deltaTime<250;if(a&&h&&d)return}return r&&s?void 0:n||s&&i&Ht||r&&i&jt?this.preventSrc(e):void 0},preventSrc:function(t){this.manager.session.prevented=!0,t.preventDefault()}};var ue=1,pe=2,ce=4,me=8,fe=me,ge=16;Q.prototype={defaults:{},set:function(t){return mt(this.options,t),this.manager&&this.manager.touchAction.update(),this},recognizeWith:function(t){if(d(t,"recognizeWith",this))return this;var e=this.simultaneous;return t=it(t,this),e[t.id]||(e[t.id]=t,t.recognizeWith(this)),this},dropRecognizeWith:function(t){return d(t,"dropRecognizeWith",this)?this:(t=it(t,this),delete this.simultaneous[t.id],this)},requireFailure:function(t){if(d(t,"requireFailure",this))return this;var e=this.requireFail;return t=it(t,this),-1===w(e,t)&&(e.push(t),t.requireFailure(this)),this},dropRequireFailure:function(t){if(d(t,"dropRequireFailure",this))return this;t=it(t,this);var e=w(this.requireFail,t);return e>-1&&this.requireFail.splice(e,1),this},hasRequireFailures:function(){return this.requireFail.length>0},canRecognizeWith:function(t){return!!this.simultaneous[t.id]},emit:function(t){function e(e){i.manager.emit(e,t)}var i=this,o=this.state;o<me&&e(i.options.event+tt(o)),e(i.options.event),t.additionalEvent&&e(t.additionalEvent),o>=me&&e(i.options.event+tt(o))},tryEmit:function(t){if(this.canEmit())return this.emit(t);this.state=32},canEmit:function(){for(var t=0;t<this.requireFail.length;){if(!(this.requireFail[t].state&(32|ue)))return!1;t++}return!0},recognize:function(t){var e=mt({},t);if(!m(this.options.enable,[this,e]))return this.reset(),void(this.state=32);this.state&(fe|ge|32)&&(this.state=ue),this.state=this.process(e),this.state&(pe|ce|me|ge)&&this.tryEmit(e)},process:function(t){},getTouchAction:function(){},reset:function(){}},p(ot,Q,{defaults:{pointers:1},attrTest:function(t){var e=this.options.pointers;return 0===e||t.pointers.length===e},process:function(t){var e=this.state,i=t.eventType,o=e&(pe|ce),n=this.attrTest(t);return o&&(i&It||!n)?e|ge:o||n?i&Pt?e|me:e&pe?e|ce:pe:32}}),p(nt,ot,{defaults:{event:"pan",threshold:10,pointers:1,direction:Yt},getTouchAction:function(){var t=this.options.direction,e=[];return t&Ht&&e.push(de),t&jt&&e.push(he),e},directionTest:function(t){var e=this.options,i=!0,o=t.distance,n=t.direction,s=t.deltaX,r=t.deltaY;return n&e.direction||(e.direction&Ht?(n=0===s?Nt:s<0?At:Rt,i=s!=this.pX,o=Math.abs(t.deltaX)):(n=0===r?Nt:r<0?Lt:Ft,i=r!=this.pY,o=Math.abs(t.deltaY))),t.direction=n,i&&o>e.threshold&&n&e.direction},attrTest:function(t){return ot.prototype.attrTest.call(this,t)&&(this.state&pe||!(this.state&pe)&&this.directionTest(t))},emit:function(t){this.pX=t.deltaX,this.pY=t.deltaY;var e=et(t.direction);e&&(t.additionalEvent=this.options.event+e),this._super.emit.call(this,t)}}),p(st,ot,{defaults:{event:"pinch",threshold:0,pointers:2},getTouchAction:function(){return[ae]},attrTest:function(t){return this._super.attrTest.call(this,t)&&(Math.abs(t.scale-1)>this.options.threshold||this.state&pe)},emit:function(t){if(1!==t.scale){var e=t.scale<1?"in":"out";t.additionalEvent=this.options.event+e}this._super.emit.call(this,t)}}),p(rt,Q,{defaults:{event:"press",pointers:1,time:251,threshold:9},getTouchAction:function(){return[se]},process:function(t){var e=this.options,i=t.pointers.length===e.pointers,o=t.distance<e.threshold,n=t.deltaTime>e.time;if(this._input=t,!o||!i||t.eventType&(Pt|It)&&!n)this.reset();else if(t.eventType&Ot)this.reset(),this._timer=h(function(){this.state=fe,this.tryEmit()},e.time,this);else if(t.eventType&Pt)return fe;return 32},reset:function(){clearTimeout(this._timer)},emit:function(t){this.state===fe&&(t&&t.eventType&Pt?this.manager.emit(this.options.event+"up",t):(this._input.timeStamp=_t(),this.manager.emit(this.options.event,this._input)))}}),p(at,ot,{defaults:{event:"rotate",threshold:0,pointers:2},getTouchAction:function(){return[ae]},attrTest:function(t){return this._super.attrTest.call(this,t)&&(Math.abs(t.rotation)>this.options.threshold||this.state&pe)}}),p(ht,ot,{defaults:{event:"swipe",threshold:10,velocity:.3,direction:Ht|jt,pointers:1},getTouchAction:function(){return nt.prototype.getTouchAction.call(this)},attrTest:function(t){var e,i=this.options.direction;return i&(Ht|jt)?e=t.overallVelocity:i&Ht?e=t.overallVelocityX:i&jt&&(e=t.overallVelocityY),this._super.attrTest.call(this,t)&&i&t.offsetDirection&&t.distance>this.options.threshold&&t.maxPointers==this.options.pointers&&bt(e)>this.options.velocity&&t.eventType&Pt},emit:function(t){var e=et(t.offsetDirection);e&&this.manager.emit(this.options.event+e,t),this.manager.emit(this.options.event,t)}}),p(dt,Q,{defaults:{event:"tap",pointers:1,taps:1,interval:300,time:250,threshold:9,posThreshold:10},getTouchAction:function(){return[re]},process:function(t){var e=this.options,i=t.pointers.length===e.pointers,o=t.distance<e.threshold,n=t.deltaTime<e.time;if(this.reset(),t.eventType&Ot&&0===this.count)return this.failTimeout();if(o&&n&&i){if(t.eventType!=Pt)return this.failTimeout();var s=!this.pTime||t.timeStamp-this.pTime<e.interval,r=!this.pCenter||F(this.pCenter,t.center)<e.posThreshold;this.pTime=t.timeStamp,this.pCenter=t.center,r&&s?this.count+=1:this.count=1,this._input=t;if(0===this.count%e.taps)return this.hasRequireFailures()?(this._timer=h(function(){this.state=fe,this.tryEmit()},e.interval,this),pe):fe}return 32},failTimeout:function(){return this._timer=h(function(){this.state=32},this.options.interval,this),32},reset:function(){clearTimeout(this._timer)},emit:function(){this.state==fe&&(this._input.tapCount=this.count,this.manager.emit(this.options.event,this._input))}}),lt.VERSION="2.0.7",lt.defaults={domEvents:!1,touchAction:"compute",enable:!0,inputTarget:null,inputClass:null,preset:[[at,{enable:!1}],[st,{enable:!1},["rotate"]],[ht,{direction:Ht}],[nt,{direction:Ht},["swipe"]],[dt],[dt,{event:"doubletap",taps:2},["tap"]],[rt]],cssProps:{userSelect:"none",touchSelect:"none",touchCallout:"none",contentZooming:"none",userDrag:"none",tapHighlightColor:"rgba(0,0,0,0)"}};ut.prototype={set:function(t){return mt(this.options,t),t.touchAction&&this.touchAction.update(),t.inputTarget&&(this.input.destroy(),this.input.target=t.inputTarget,this.input.init()),this},stop:function(t){this.session.stopped=t?2:1},recognize:function(t){var e=this.session;if(!e.stopped){this.touchAction.preventDefaults(t);var i,o=this.recognizers,n=e.curRecognizer;(!n||n&&n.state&fe)&&(n=e.curRecognizer=null);for(var s=0;s<o.length;)i=o[s],2===e.stopped||n&&i!=n&&!i.canRecognizeWith(n)?i.reset():i.recognize(t),!n&&i.state&(pe|ce|me)&&(n=e.curRecognizer=i),s++}},get:function(t){if(t instanceof Q)return t;for(var e=this.recognizers,i=0;i<e.length;i++)if(e[i].options.event==t)return e[i];return null},add:function(t){if(d(t,"add",this))return this;var e=this.get(t.options.event);return e&&this.remove(e),this.recognizers.push(t),t.manager=this,this.touchAction.update(),t},remove:function(t){if(d(t,"remove",this))return this;if(t=this.get(t)){var e=this.recognizers,i=w(e,t);-1!==i&&(e.splice(i,1),this.touchAction.update())}return this},on:function(t,e){if(t!==a&&e!==a){var i=this.handlers;return l(_(t),function(t){i[t]=i[t]||[],i[t].push(e)}),this}},off:function(t,e){if(t!==a){var i=this.handlers;return l(_(t),function(t){e?i[t]&&i[t].splice(w(i[t],e),1):delete i[t]}),this}},emit:function(t,e){this.options.domEvents&&ct(t,e);var i=this.handlers[t]&&this.handlers[t].slice();if(i&&i.length){e.type=t,e.preventDefault=function(){e.srcEvent.preventDefault()};for(var o=0;o<i.length;)i[o](e),o++}},destroy:function(){this.element&&pt(this,!1),this.handlers={},this.session={},this.input.destroy(),this.element=null}},mt(lt,{INPUT_START:Ot,INPUT_MOVE:Et,INPUT_END:Pt,INPUT_CANCEL:It,STATE_POSSIBLE:ue,STATE_BEGAN:pe,STATE_CHANGED:ce,STATE_ENDED:me,STATE_RECOGNIZED:fe,STATE_CANCELLED:ge,STATE_FAILED:32,DIRECTION_NONE:Nt,DIRECTION_LEFT:At,DIRECTION_RIGHT:Rt,DIRECTION_UP:Lt,DIRECTION_DOWN:Ft,DIRECTION_HORIZONTAL:Ht,DIRECTION_VERTICAL:jt,DIRECTION_ALL:Yt,Manager:ut,Input:T,TouchAction:J,TouchInput:B,MouseInput:G,PointerEventInput:z,TouchMouseInput:q,SingleTouchInput:W,Recognizer:Q,AttrRecognizer:ot,Tap:dt,Pan:nt,Swipe:ht,Pinch:st,Rotate:at,Press:rt,on:g,off:v,each:l,merge:xt,extend:wt,assign:mt,inherit:p,bindFn:c,prefixed:S}),(void 0!==n?n:"undefined"!=typeof self?self:{}).Hammer=lt,(o=function(){return lt}.call(e,i,e,t))!==a&&(t.exports=o)}(window,document)},function(t,e,i){function o(t){this.active=!1,this.dom={container:t},this.dom.overlay=document.createElement("div"),this.dom.overlay.className="vis-overlay",this.dom.container.appendChild(this.dom.overlay),this.hammer=a(this.dom.overlay),this.hammer.on("tap",this._onTapOverlay.bind(this));var e=this;["tap","doubletap","press","pinch","pan","panstart","panmove","panend"].forEach(function(t){e.hammer.on(t,function(t){t.stopPropagation()})}),document&&document.body&&(this.onClick=function(i){n(i.target,t)||e.deactivate()},document.body.addEventListener("click",this.onClick)),void 0!==this.keycharm&&this.keycharm.destroy(),this.keycharm=s(),this.escListener=this.deactivate.bind(this)}function n(t,e){for(;t;){if(t===e)return!0;t=t.parentNode}return!1}var s=i(68),r=i(66),a=i(12),h=i(0);r(o.prototype),o.current=null,o.prototype.destroy=function(){this.deactivate(),this.dom.overlay.parentNode.removeChild(this.dom.overlay),this.onClick&&document.body.removeEventListener("click",this.onClick),this.hammer.destroy(),this.hammer=null},o.prototype.activate=function(){o.current&&o.current.deactivate(),o.current=this,this.active=!0,this.dom.overlay.style.display="none",h.addClassName(this.dom.container,"vis-active"),this.emit("change"),this.emit("activate"),this.keycharm.bind("esc",this.escListener)},o.prototype.deactivate=function(){this.active=!1,this.dom.overlay.style.display="",h.removeClassName(this.dom.container,"vis-active"),this.keycharm.unbind("esc",this.escListener),this.emit("change"),this.emit("deactivate")},o.prototype._onTapOverlay=function(t){this.activate(),t.stopPropagation()},t.exports=o},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}Object.defineProperty(e,"__esModule",{value:!0});var n=i(30),s=o(n),r=i(31),a=o(r),h=function(){function t(e,i){(0,s.default)(this,t),this.container=e,this.overflowMethod=i||"cap",this.x=0,this.y=0,this.padding=5,this.hidden=!1,this.frame=document.createElement("div"),this.frame.className="vis-tooltip",this.container.appendChild(this.frame)}return(0,a.default)(t,[{key:"setPosition",value:function(t,e){this.x=parseInt(t),this.y=parseInt(e)}},{key:"setText",value:function(t){t instanceof Element?(this.frame.innerHTML="",this.frame.appendChild(t)):this.frame.innerHTML=t}},{key:"show",value:function(t){if(void 0===t&&(t=!0),!0===t){var e=this.frame.clientHeight,i=this.frame.clientWidth,o=this.frame.parentNode.clientHeight,n=this.frame.parentNode.clientWidth,s=0,r=0;if("flip"==this.overflowMethod){var a=!1,h=!0;this.y-e<this.padding&&(h=!1),this.x+i>n-this.padding&&(a=!0),s=a?this.x-i:this.x,r=h?this.y-e:this.y}else r=this.y-e,r+e+this.padding>o&&(r=o-e-this.padding),r<this.padding&&(r=this.padding),s=this.x,s+i+this.padding>n&&(s=n-i-this.padding),s<this.padding&&(s=this.padding);this.frame.style.left=s+"px",this.frame.style.top=r+"px",this.frame.style.visibility="visible",this.hidden=!1}else this.hide()}},{key:"hide",value:function(){this.hidden=!0,this.frame.style.left="0",this.frame.style.top="0",this.frame.style.visibility="hidden"}},{key:"destroy",value:function(){this.frame.parentNode.removeChild(this.frame)}}]),t}();e.default=h},function(t,e,i){t.exports={default:i(135),__esModule:!0}},function(t,e,i){i(136);var o=i(2).Object;t.exports=function(t,e,i){return o.defineProperty(t,e,i)}},function(t,e,i){var o=i(17);o(o.S+o.F*!i(9),"Object",{defineProperty:i(8).f})},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}Object.defineProperty(e,"__esModule",{value:!0});var n=i(16),s=o(n),r=i(30),a=o(r),h=i(31),d=o(h),l=i(12),u=i(67),p=i(0),c=function(){function t(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:1;(0,a.default)(this,t),this.pixelRatio=e,this.generated=!1,this.centerCoordinates={x:144.5,y:144.5},this.r=289*.49,this.color={r:255,g:255,b:255,a:1},this.hueCircle=void 0,this.initialColor={r:255,g:255,b:255,a:1},this.previousColor=void 0,this.applied=!1,this.updateCallback=function(){},this.closeCallback=function(){},this._create()}return(0,d.default)(t,[{key:"insertTo",value:function(t){void 0!==this.hammer&&(this.hammer.destroy(),this.hammer=void 0),this.container=t,this.container.appendChild(this.frame),this._bindHammer(),this._setSize()}},{key:"setUpdateCallback",value:function(t){if("function"!=typeof t)throw new Error("Function attempted to set as colorPicker update callback is not a function.");this.updateCallback=t}},{key:"setCloseCallback",value:function(t){if("function"!=typeof t)throw new Error("Function attempted to set as colorPicker closing callback is not a function.");this.closeCallback=t}},{key:"_isColorString",value:function(t){var e={black:"#000000",navy:"#000080",darkblue:"#00008B",mediumblue:"#0000CD",blue:"#0000FF",darkgreen:"#006400",green:"#008000",teal:"#008080",darkcyan:"#008B8B",deepskyblue:"#00BFFF",darkturquoise:"#00CED1",mediumspringgreen:"#00FA9A",lime:"#00FF00",springgreen:"#00FF7F",aqua:"#00FFFF",cyan:"#00FFFF",midnightblue:"#191970",dodgerblue:"#1E90FF",lightseagreen:"#20B2AA",forestgreen:"#228B22",seagreen:"#2E8B57",darkslategray:"#2F4F4F",limegreen:"#32CD32",mediumseagreen:"#3CB371",turquoise:"#40E0D0",royalblue:"#4169E1",steelblue:"#4682B4",darkslateblue:"#483D8B",mediumturquoise:"#48D1CC",indigo:"#4B0082",darkolivegreen:"#556B2F",cadetblue:"#5F9EA0",cornflowerblue:"#6495ED",mediumaquamarine:"#66CDAA",dimgray:"#696969",slateblue:"#6A5ACD",olivedrab:"#6B8E23",slategray:"#708090",lightslategray:"#778899",mediumslateblue:"#7B68EE",lawngreen:"#7CFC00",chartreuse:"#7FFF00",aquamarine:"#7FFFD4",maroon:"#800000",purple:"#800080",olive:"#808000",gray:"#808080",skyblue:"#87CEEB",lightskyblue:"#87CEFA",blueviolet:"#8A2BE2",darkred:"#8B0000",darkmagenta:"#8B008B",saddlebrown:"#8B4513",darkseagreen:"#8FBC8F",lightgreen:"#90EE90",mediumpurple:"#9370D8",darkviolet:"#9400D3",palegreen:"#98FB98",darkorchid:"#9932CC",yellowgreen:"#9ACD32",sienna:"#A0522D",brown:"#A52A2A",darkgray:"#A9A9A9",lightblue:"#ADD8E6",greenyellow:"#ADFF2F",paleturquoise:"#AFEEEE",lightsteelblue:"#B0C4DE",powderblue:"#B0E0E6",firebrick:"#B22222",darkgoldenrod:"#B8860B",mediumorchid:"#BA55D3",rosybrown:"#BC8F8F",darkkhaki:"#BDB76B",silver:"#C0C0C0",mediumvioletred:"#C71585",indianred:"#CD5C5C",peru:"#CD853F",chocolate:"#D2691E",tan:"#D2B48C",lightgrey:"#D3D3D3",palevioletred:"#D87093",thistle:"#D8BFD8",orchid:"#DA70D6",goldenrod:"#DAA520",crimson:"#DC143C",gainsboro:"#DCDCDC",plum:"#DDA0DD",burlywood:"#DEB887",lightcyan:"#E0FFFF",lavender:"#E6E6FA",darksalmon:"#E9967A",violet:"#EE82EE",palegoldenrod:"#EEE8AA",lightcoral:"#F08080",khaki:"#F0E68C",aliceblue:"#F0F8FF",honeydew:"#F0FFF0",azure:"#F0FFFF",sandybrown:"#F4A460",wheat:"#F5DEB3",beige:"#F5F5DC",whitesmoke:"#F5F5F5",mintcream:"#F5FFFA",ghostwhite:"#F8F8FF",salmon:"#FA8072",antiquewhite:"#FAEBD7",linen:"#FAF0E6",lightgoldenrodyellow:"#FAFAD2",oldlace:"#FDF5E6",red:"#FF0000",fuchsia:"#FF00FF",magenta:"#FF00FF",deeppink:"#FF1493",orangered:"#FF4500",tomato:"#FF6347",hotpink:"#FF69B4",coral:"#FF7F50",darkorange:"#FF8C00",lightsalmon:"#FFA07A",orange:"#FFA500",lightpink:"#FFB6C1",pink:"#FFC0CB",gold:"#FFD700",peachpuff:"#FFDAB9",navajowhite:"#FFDEAD",moccasin:"#FFE4B5",bisque:"#FFE4C4",mistyrose:"#FFE4E1",blanchedalmond:"#FFEBCD",papayawhip:"#FFEFD5",lavenderblush:"#FFF0F5",seashell:"#FFF5EE",cornsilk:"#FFF8DC",lemonchiffon:"#FFFACD",floralwhite:"#FFFAF0",snow:"#FFFAFA",yellow:"#FFFF00",lightyellow:"#FFFFE0",ivory:"#FFFFF0",white:"#FFFFFF"};if("string"==typeof t)return e[t]}},{key:"setColor",value:function(t){var e=!(arguments.length>1&&void 0!==arguments[1])||arguments[1];if("none"!==t){var i=void 0,o=this._isColorString(t);if(void 0!==o&&(t=o),!0===p.isString(t)){if(!0===p.isValidRGB(t)){var n=t.substr(4).substr(0,t.length-5).split(",");i={r:n[0],g:n[1],b:n[2],a:1}}else if(!0===p.isValidRGBA(t)){var r=t.substr(5).substr(0,t.length-6).split(",");i={r:r[0],g:r[1],b:r[2],a:r[3]}}else if(!0===p.isValidHex(t)){var a=p.hexToRGB(t);i={r:a.r,g:a.g,b:a.b,a:1}}}else if(t instanceof Object&&void 0!==t.r&&void 0!==t.g&&void 0!==t.b){var h=void 0!==t.a?t.a:"1.0";i={r:t.r,g:t.g,b:t.b,a:h}}if(void 0===i)throw new Error("Unknown color passed to the colorPicker. Supported are strings: rgb, hex, rgba. Object: rgb ({r:r,g:g,b:b,[a:a]}). Supplied: "+(0,s.default)(t));this._setColor(i,e)}}},{key:"show",value:function(){void 0!==this.closeCallback&&(this.closeCallback(),this.closeCallback=void 0),this.applied=!1,this.frame.style.display="block",this._generateHueCircle()}},{key:"_hide",value:function(){var t=this;!0===(!(arguments.length>0&&void 0!==arguments[0])||arguments[0])&&(this.previousColor=p.extend({},this.color)),!0===this.applied&&this.updateCallback(this.initialColor),this.frame.style.display="none",setTimeout(function(){void 0!==t.closeCallback&&(t.closeCallback(),t.closeCallback=void 0)},0)}},{key:"_save",value:function(){this.updateCallback(this.color),this.applied=!1,this._hide()}},{key:"_apply",value:function(){this.applied=!0,this.updateCallback(this.color),this._updatePicker(this.color)}},{key:"_loadLast",value:function(){void 0!==this.previousColor?this.setColor(this.previousColor,!1):alert("There is no last color to load...")}},{key:"_setColor",value:function(t){!0===(!(arguments.length>1&&void 0!==arguments[1])||arguments[1])&&(this.initialColor=p.extend({},t)),this.color=t;var e=p.RGBToHSV(t.r,t.g,t.b),i=2*Math.PI,o=this.r*e.s,n=this.centerCoordinates.x+o*Math.sin(i*e.h),s=this.centerCoordinates.y+o*Math.cos(i*e.h);this.colorPickerSelector.style.left=n-.5*this.colorPickerSelector.clientWidth+"px",this.colorPickerSelector.style.top=s-.5*this.colorPickerSelector.clientHeight+"px",this._updatePicker(t)}},{key:"_setOpacity",value:function(t){this.color.a=t/100,this._updatePicker(this.color)}},{key:"_setBrightness",value:function(t){var e=p.RGBToHSV(this.color.r,this.color.g,this.color.b);e.v=t/100;var i=p.HSVToRGB(e.h,e.s,e.v);i.a=this.color.a,this.color=i,this._updatePicker()}},{key:"_updatePicker",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:this.color,e=p.RGBToHSV(t.r,t.g,t.b),i=this.colorPickerCanvas.getContext("2d");void 0===this.pixelRation&&(this.pixelRatio=(window.devicePixelRatio||1)/(i.webkitBackingStorePixelRatio||i.mozBackingStorePixelRatio||i.msBackingStorePixelRatio||i.oBackingStorePixelRatio||i.backingStorePixelRatio||1)),i.setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0);var o=this.colorPickerCanvas.clientWidth,n=this.colorPickerCanvas.clientHeight;i.clearRect(0,0,o,n),i.putImageData(this.hueCircle,0,0),i.fillStyle="rgba(0,0,0,"+(1-e.v)+")",i.circle(this.centerCoordinates.x,this.centerCoordinates.y,this.r),i.fill(),this.brightnessRange.value=100*e.v,this.opacityRange.value=100*t.a,this.initialColorDiv.style.backgroundColor="rgba("+this.initialColor.r+","+this.initialColor.g+","+this.initialColor.b+","+this.initialColor.a+")",this.newColorDiv.style.backgroundColor="rgba("+this.color.r+","+this.color.g+","+this.color.b+","+this.color.a+")"}},{key:"_setSize",value:function(){this.colorPickerCanvas.style.width="100%",this.colorPickerCanvas.style.height="100%",this.colorPickerCanvas.width=289*this.pixelRatio,this.colorPickerCanvas.height=289*this.pixelRatio}},{key:"_create",value:function(){if(this.frame=document.createElement("div"),this.frame.className="vis-color-picker",this.colorPickerDiv=document.createElement("div"),this.colorPickerSelector=document.createElement("div"),this.colorPickerSelector.className="vis-selector",this.colorPickerDiv.appendChild(this.colorPickerSelector),this.colorPickerCanvas=document.createElement("canvas"),this.colorPickerDiv.appendChild(this.colorPickerCanvas),this.colorPickerCanvas.getContext){var t=this.colorPickerCanvas.getContext("2d");this.pixelRatio=(window.devicePixelRatio||1)/(t.webkitBackingStorePixelRatio||t.mozBackingStorePixelRatio||t.msBackingStorePixelRatio||t.oBackingStorePixelRatio||t.backingStorePixelRatio||1),this.colorPickerCanvas.getContext("2d").setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0)}else{var e=document.createElement("DIV");e.style.color="red",e.style.fontWeight="bold",e.style.padding="10px",e.innerHTML="Error: your browser does not support HTML canvas",this.colorPickerCanvas.appendChild(e)}this.colorPickerDiv.className="vis-color",this.opacityDiv=document.createElement("div"),this.opacityDiv.className="vis-opacity",this.brightnessDiv=document.createElement("div"),this.brightnessDiv.className="vis-brightness",this.arrowDiv=document.createElement("div"),this.arrowDiv.className="vis-arrow",this.opacityRange=document.createElement("input");try{this.opacityRange.type="range",this.opacityRange.min="0",this.opacityRange.max="100"}catch(t){}this.opacityRange.value="100",this.opacityRange.className="vis-range",this.brightnessRange=document.createElement("input");try{this.brightnessRange.type="range",this.brightnessRange.min="0",this.brightnessRange.max="100"}catch(t){}this.brightnessRange.value="100",this.brightnessRange.className="vis-range",this.opacityDiv.appendChild(this.opacityRange),this.brightnessDiv.appendChild(this.brightnessRange);var i=this;this.opacityRange.onchange=function(){i._setOpacity(this.value)},this.opacityRange.oninput=function(){i._setOpacity(this.value)},this.brightnessRange.onchange=function(){i._setBrightness(this.value)},this.brightnessRange.oninput=function(){i._setBrightness(this.value)},this.brightnessLabel=document.createElement("div"),this.brightnessLabel.className="vis-label vis-brightness",this.brightnessLabel.innerHTML="brightness:",this.opacityLabel=document.createElement("div"),this.opacityLabel.className="vis-label vis-opacity",this.opacityLabel.innerHTML="opacity:",this.newColorDiv=document.createElement("div"),this.newColorDiv.className="vis-new-color",this.newColorDiv.innerHTML="new",this.initialColorDiv=document.createElement("div"),this.initialColorDiv.className="vis-initial-color",this.initialColorDiv.innerHTML="initial",this.cancelButton=document.createElement("div"),this.cancelButton.className="vis-button vis-cancel",this.cancelButton.innerHTML="cancel",this.cancelButton.onclick=this._hide.bind(this,!1),this.applyButton=document.createElement("div"),this.applyButton.className="vis-button vis-apply",this.applyButton.innerHTML="apply",this.applyButton.onclick=this._apply.bind(this),this.saveButton=document.createElement("div"),this.saveButton.className="vis-button vis-save",this.saveButton.innerHTML="save",this.saveButton.onclick=this._save.bind(this),this.loadButton=document.createElement("div"),this.loadButton.className="vis-button vis-load",this.loadButton.innerHTML="load last",this.loadButton.onclick=this._loadLast.bind(this),this.frame.appendChild(this.colorPickerDiv),this.frame.appendChild(this.arrowDiv),this.frame.appendChild(this.brightnessLabel),this.frame.appendChild(this.brightnessDiv),this.frame.appendChild(this.opacityLabel),this.frame.appendChild(this.opacityDiv),this.frame.appendChild(this.newColorDiv),this.frame.appendChild(this.initialColorDiv),this.frame.appendChild(this.cancelButton),this.frame.appendChild(this.applyButton),this.frame.appendChild(this.saveButton),this.frame.appendChild(this.loadButton)}},{key:"_bindHammer",value:function(){var t=this;this.drag={},this.pinch={},this.hammer=new l(this.colorPickerCanvas),this.hammer.get("pinch").set({enable:!0}),u.onTouch(this.hammer,function(e){t._moveSelector(e)}),this.hammer.on("tap",function(e){t._moveSelector(e)}),this.hammer.on("panstart",function(e){t._moveSelector(e)}),this.hammer.on("panmove",function(e){t._moveSelector(e)}),this.hammer.on("panend",function(e){t._moveSelector(e)})}},{key:"_generateHueCircle",value:function(){if(!1===this.generated){var t=this.colorPickerCanvas.getContext("2d");void 0===this.pixelRation&&(this.pixelRatio=(window.devicePixelRatio||1)/(t.webkitBackingStorePixelRatio||t.mozBackingStorePixelRatio||t.msBackingStorePixelRatio||t.oBackingStorePixelRatio||t.backingStorePixelRatio||1)),t.setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0);var e=this.colorPickerCanvas.clientWidth,i=this.colorPickerCanvas.clientHeight;t.clearRect(0,0,e,i);var o=void 0,n=void 0,s=void 0,r=void 0;this.centerCoordinates={x:.5*e,y:.5*i},this.r=.49*e;var a=2*Math.PI/360,h=1/this.r,d=void 0;for(s=0;s<360;s++)for(r=0;r<this.r;r++)o=this.centerCoordinates.x+r*Math.sin(a*s),n=this.centerCoordinates.y+r*Math.cos(a*s),d=p.HSVToRGB(s*(1/360),r*h,1),t.fillStyle="rgb("+d.r+","+d.g+","+d.b+")",t.fillRect(o-.5,n-.5,2,2);t.strokeStyle="rgba(0,0,0,1)",t.circle(this.centerCoordinates.x,this.centerCoordinates.y,this.r),t.stroke(),this.hueCircle=t.getImageData(0,0,e,i)}this.generated=!0}},{key:"_moveSelector",value:function(t){var e=this.colorPickerDiv.getBoundingClientRect(),i=t.center.x-e.left,o=t.center.y-e.top,n=.5*this.colorPickerDiv.clientHeight,s=.5*this.colorPickerDiv.clientWidth,r=i-s,a=o-n,h=Math.atan2(r,a),d=.98*Math.min(Math.sqrt(r*r+a*a),s),l=Math.cos(h)*d+n,u=Math.sin(h)*d+s;this.colorPickerSelector.style.top=l-.5*this.colorPickerSelector.clientHeight+"px",this.colorPickerSelector.style.left=u-.5*this.colorPickerSelector.clientWidth+"px";var c=h/(2*Math.PI);c=c<0?c+1:c;var m=d/this.r,f=p.RGBToHSV(this.color.r,this.color.g,this.color.b);f.h=c,f.s=m;var g=p.HSVToRGB(f.h,f.s,f.v);g.a=this.color.a,this.color=g,this.initialColorDiv.style.backgroundColor="rgba("+this.initialColor.r+","+this.initialColor.g+","+this.initialColor.b+","+this.initialColor.a+")",this.newColorDiv.style.backgroundColor="rgba("+this.color.r+","+this.color.g+","+this.color.b+","+this.color.a+")"}}]),t}();e.default=c},function(t,e,i){function o(t,e,i,o){if(!(Array.isArray(i)||i instanceof r||i instanceof a)&&i instanceof Object){var d=o;o=i,i=d}o&&o.throttleRedraw&&console.warn('Graph2d option "throttleRedraw" is DEPRICATED and no longer supported. It will be removed in the next MAJOR release.');var p=this;this.defaultOptions={start:null,end:null,autoResize:!0,orientation:{axis:"bottom",item:"bottom"},moment:n,width:null,height:null,maxHeight:null,minHeight:null},this.options=s.deepExtend({},this.defaultOptions),this._create(t),this.components=[],this.body={dom:this.dom,domProps:this.props,emitter:{on:this.on.bind(this),off:this.off.bind(this),emit:this.emit.bind(this)},hiddenDates:[],util:{toScreen:p._toScreen.bind(p),toGlobalScreen:p._toGlobalScreen.bind(p),toTime:p._toTime.bind(p),toGlobalTime:p._toGlobalTime.bind(p)}},this.range=new h(this.body),this.components.push(this.range),this.body.range=this.range,this.timeAxis=new l(this.body),this.components.push(this.timeAxis),this.currentTime=new u(this.body),this.components.push(this.currentTime),this.linegraph=new c(this.body),this.components.push(this.linegraph),this.itemsData=null,this.groupsData=null,this.on("tap",function(t){p.emit("click",p.getEventProperties(t))}),this.on("doubletap",function(t){p.emit("doubleClick",p.getEventProperties(t))}),this.dom.root.oncontextmenu=function(t){p.emit("contextmenu",p.getEventProperties(t))},o&&this.setOptions(o),i&&this.setGroups(i),e&&this.setItems(e),this._redraw()}var n=i(3),s=i(0),r=i(15),a=i(20),h=i(47),d=i(48),l=i(28),u=i(50),p=i(29),c=i(77),m=i(32).printStyle,f=i(84).allOptions,g=i(84).configureOptions,v=i(76).default,y=i(32).default;o.prototype=new d,o.prototype.setOptions=function(t){!0===y.validate(t,f)&&console.log("%cErrors have been found in the supplied options object.",m),d.prototype.setOptions.call(this,t)},o.prototype.setItems=function(t){var e,i=null==this.itemsData;if(e=t?t instanceof r||t instanceof a?t:new r(t,{type:{start:"Date",end:"Date"}}):null,this.itemsData=e,this.linegraph&&this.linegraph.setItems(e),i)if(void 0!=this.options.start||void 0!=this.options.end){var o=void 0!=this.options.start?this.options.start:null,n=void 0!=this.options.end?this.options.end:null;this.setWindow(o,n,{animation:!1})}else this.fit({animation:!1})},o.prototype.setGroups=function(t){var e;e=t?t instanceof r||t instanceof a?t:new r(t):null,this.groupsData=e,this.linegraph.setGroups(e)},o.prototype.getLegend=function(t,e,i){return void 0===e&&(e=15),void 0===i&&(i=15),void 0!==this.linegraph.groups[t]?this.linegraph.groups[t].getLegend(e,i):"cannot find group:'"+t+"'"},o.prototype.isGroupVisible=function(t){return void 0!==this.linegraph.groups[t]&&(this.linegraph.groups[t].visible&&(void 0===this.linegraph.options.groups.visibility[t]||1==this.linegraph.options.groups.visibility[t]))},o.prototype.getDataRange=function(){var t=null,e=null;for(var i in this.linegraph.groups)if(this.linegraph.groups.hasOwnProperty(i)&&1==this.linegraph.groups[i].visible)for(var o=0;o<this.linegraph.groups[i].itemsData.length;o++){var n=this.linegraph.groups[i].itemsData[o],r=s.convert(n.x,"Date").valueOf();t=null==t?r:t>r?r:t,e=null==e?r:e<r?r:e}return{min:null!=t?new Date(t):null,max:null!=e?new Date(e):null}},o.prototype.getEventProperties=function(t){var e=t.center?t.center.x:t.clientX,i=t.center?t.center.y:t.clientY,o=e-s.getAbsoluteLeft(this.dom.centerContainer),n=i-s.getAbsoluteTop(this.dom.centerContainer),r=this._toTime(o),a=p.customTimeFromTarget(t),h=s.getTarget(t),d=null;s.hasParent(h,this.timeAxis.dom.foreground)?d="axis":this.timeAxis2&&s.hasParent(h,this.timeAxis2.dom.foreground)?d="axis":s.hasParent(h,this.linegraph.yAxisLeft.dom.frame)?d="data-axis":s.hasParent(h,this.linegraph.yAxisRight.dom.frame)?d="data-axis":s.hasParent(h,this.linegraph.legendLeft.dom.frame)?d="legend":s.hasParent(h,this.linegraph.legendRight.dom.frame)?d="legend":null!=a?d="custom-time":s.hasParent(h,this.currentTime.bar)?d="current-time":s.hasParent(h,this.dom.center)&&(d="background");var l=[],u=this.linegraph.yAxisLeft,c=this.linegraph.yAxisRight;return!u.hidden&&this.itemsData.length>0&&l.push(u.screenToValue(n)),!c.hidden&&this.itemsData.length>0&&l.push(c.screenToValue(n)),{event:t,what:d,pageX:t.srcEvent?t.srcEvent.pageX:t.pageX,pageY:t.srcEvent?t.srcEvent.pageY:t.pageY,x:o,y:n,time:r,value:l}},o.prototype._createConfigurator=function(){return new v(this,this.dom.container,g)},t.exports=o}])});
\ No newline at end of file
diff --git a/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js.map b/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js.map
deleted file mode 100644
index 779880dcc11af..0000000000000
--- a/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js.map
+++ /dev/null
@@ -1 +0,0 @@
-{"version":3,"file":"vis-timeline-graph2d.min.js","sources":["../../node_modules/moment/moment.js","../../node_modules/core-js-pure/internals/global.js","../../node_modules/moment/locale/de.js","../../node_modules/moment/locale/es.js","../../node_modules/moment/locale/fr.js","../../node_modules/moment/locale/it.js","../../node_modules/moment/locale/ja.js","../../node_modules/moment/locale/nl.js","../../node_modules/moment/locale/pl.js","../../node_modules/moment/locale/ru.js","../../node_modules/moment/locale/uk.js","../../node_modules/core-js-pure/internals/fails.js","../../node_modules/core-js-pure/internals/descriptors.js","../../node_modules/core-js-pure/internals/object-property-is-enumerable.js","../../node_modules/core-js-pure/internals/create-property-descriptor.js","../../node_modules/core-js-pure/internals/classof-raw.js","../../node_modules/core-js-pure/internals/indexed-object.js","../../node_modules/core-js-pure/internals/require-object-coercible.js","../../node_modules/core-js-pure/internals/to-indexed-object.js","../../node_modules/core-js-pure/internals/is-object.js","../../node_modules/core-js-pure/internals/to-primitive.js","../../node_modules/core-js-pure/internals/has.js","../../node_modules/core-js-pure/internals/document-create-element.js","../../node_modules/core-js-pure/internals/ie8-dom-define.js","../../node_modules/core-js-pure/internals/object-get-own-property-descriptor.js","../../node_modules/core-js-pure/internals/is-forced.js","../../node_modules/core-js-pure/internals/path.js","../../node_modules/core-js-pure/internals/a-function.js","../../node_modules/core-js-pure/internals/function-bind-context.js","../../node_modules/core-js-pure/internals/an-object.js","../../node_modules/core-js-pure/internals/object-define-property.js","../../node_modules/core-js-pure/internals/create-non-enumerable-property.js","../../node_modules/core-js-pure/internals/export.js","../../node_modules/core-js-pure/modules/es.object.define-property.js","../../node_modules/core-js-pure/es/object/define-property.js","../../node_modules/core-js-pure/stable/object/define-property.js","../../node_modules/core-js-pure/internals/to-integer.js","../../node_modules/core-js-pure/internals/to-length.js","../../node_modules/core-js-pure/internals/to-absolute-index.js","../../node_modules/core-js-pure/internals/array-includes.js","../../node_modules/core-js-pure/internals/hidden-keys.js","../../node_modules/core-js-pure/internals/object-keys-internal.js","../../node_modules/core-js-pure/internals/enum-bug-keys.js","../../node_modules/core-js-pure/internals/object-keys.js","../../node_modules/core-js-pure/internals/object-define-properties.js","../../node_modules/core-js-pure/modules/es.object.define-properties.js","../../node_modules/core-js-pure/es/object/define-properties.js","../../node_modules/core-js-pure/internals/get-built-in.js","../../node_modules/core-js-pure/internals/object-get-own-property-names.js","../../node_modules/core-js-pure/internals/object-get-own-property-symbols.js","../../node_modules/core-js-pure/internals/own-keys.js","../../node_modules/core-js-pure/internals/create-property.js","../../node_modules/core-js-pure/modules/es.object.get-own-property-descriptors.js","../../node_modules/core-js-pure/es/object/get-own-property-descriptors.js","../../node_modules/core-js-pure/modules/es.object.get-own-property-descriptor.js","../../node_modules/core-js-pure/internals/object-create.js","../../node_modules/core-js-pure/es/object/get-own-property-descriptor.js","../../node_modules/core-js-pure/internals/native-symbol.js","../../node_modules/core-js-pure/internals/use-symbol-as-uid.js","../../node_modules/core-js-pure/internals/is-array.js","../../node_modules/core-js-pure/internals/to-object.js","../../node_modules/core-js-pure/internals/html.js","../../node_modules/core-js-pure/internals/shared-store.js","../../node_modules/core-js-pure/internals/set-global.js","../../node_modules/core-js-pure/internals/shared.js","../../node_modules/core-js-pure/internals/uid.js","../../node_modules/core-js-pure/internals/shared-key.js","../../node_modules/core-js-pure/internals/object-get-own-property-names-external.js","../../node_modules/core-js-pure/internals/redefine.js","../../node_modules/core-js-pure/internals/well-known-symbol.js","../../node_modules/core-js-pure/internals/well-known-symbol-wrapped.js","../../node_modules/core-js-pure/internals/define-well-known-symbol.js","../../node_modules/core-js-pure/internals/to-string-tag-support.js","../../node_modules/core-js-pure/internals/classof.js","../../node_modules/core-js-pure/internals/object-to-string.js","../../node_modules/core-js-pure/internals/set-to-string-tag.js","../../node_modules/core-js-pure/internals/inspect-source.js","../../node_modules/core-js-pure/internals/internal-state.js","../../node_modules/core-js-pure/internals/native-weak-map.js","../../node_modules/core-js-pure/internals/array-species-create.js","../../node_modules/core-js-pure/internals/array-iteration.js","../../node_modules/core-js-pure/modules/es.symbol.js","../../node_modules/core-js-pure/es/object/get-own-property-symbols.js","../../node_modules/core-js-pure/internals/iterators-core.js","../../node_modules/core-js-pure/internals/iterators.js","../../node_modules/core-js-pure/internals/correct-prototype-getter.js","../../node_modules/core-js-pure/internals/object-get-prototype-of.js","../../node_modules/core-js-pure/internals/create-iterator-constructor.js","../../node_modules/core-js-pure/internals/object-set-prototype-of.js","../../node_modules/core-js-pure/internals/a-possible-prototype.js","../../node_modules/core-js-pure/internals/define-iterator.js","../../node_modules/core-js-pure/modules/es.array.iterator.js","../../node_modules/core-js-pure/internals/dom-iterables.js","../../node_modules/core-js-pure/modules/web.dom-collections.iterator.js","../../node_modules/core-js-pure/internals/string-multibyte.js","../../node_modules/core-js-pure/modules/es.string.iterator.js","../../node_modules/core-js-pure/internals/get-iterator-method.js","../../node_modules/core-js-pure/internals/get-iterator.js","../../node_modules/core-js-pure/modules/es.object.create.js","../../node_modules/core-js-pure/es/object/create.js","../../node_modules/core-js-pure/stable/object/create.js","../../node_modules/core-js-pure/features/object/define-property.js","../../node_modules/@babel/runtime-corejs3/helpers/defineProperty.js","../../node_modules/core-js-pure/modules/es.object.keys.js","../../node_modules/core-js-pure/es/object/keys.js","../../node_modules/core-js-pure/internals/string-trim-forced.js","../../node_modules/core-js-pure/internals/whitespaces.js","../../node_modules/core-js-pure/internals/string-trim.js","../../node_modules/core-js-pure/modules/es.string.trim.js","../../node_modules/core-js-pure/internals/entry-virtual.js","../../node_modules/core-js-pure/es/string/virtual/trim.js","../../node_modules/core-js-pure/es/instance/trim.js","../../node_modules/core-js-pure/internals/array-method-is-strict.js","../../node_modules/core-js-pure/internals/array-method-uses-to-length.js","../../node_modules/core-js-pure/internals/array-for-each.js","../../node_modules/core-js-pure/modules/es.array.for-each.js","../../node_modules/core-js-pure/es/array/virtual/for-each.js","../../node_modules/core-js-pure/internals/engine-v8-version.js","../../node_modules/core-js-pure/stable/instance/for-each.js","../../node_modules/core-js-pure/internals/engine-user-agent.js","../../node_modules/core-js-pure/internals/array-method-has-species-support.js","../../node_modules/core-js-pure/modules/es.array.map.js","../../node_modules/core-js-pure/es/array/virtual/map.js","../../node_modules/core-js-pure/es/instance/map.js","../../node_modules/core-js-pure/internals/number-parse-int.js","../../node_modules/core-js-pure/modules/es.parse-int.js","../../node_modules/core-js-pure/es/parse-int.js","../../node_modules/core-js-pure/internals/object-to-array.js","../../node_modules/core-js-pure/modules/es.object.values.js","../../node_modules/core-js-pure/es/object/values.js","../../node_modules/core-js-pure/modules/es.array.filter.js","../../node_modules/core-js-pure/es/array/virtual/filter.js","../../node_modules/core-js-pure/es/instance/filter.js","../../node_modules/core-js-pure/modules/es.array.concat.js","../../node_modules/core-js-pure/es/array/virtual/concat.js","../../node_modules/core-js-pure/es/instance/concat.js","../../node_modules/core-js-pure/modules/es.array.is-array.js","../../node_modules/core-js-pure/es/array/is-array.js","../../node_modules/core-js-pure/features/array/is-array.js","../../node_modules/@babel/runtime-corejs3/helpers/arrayWithoutHoles.js","../../node_modules/core-js-pure/internals/call-with-safe-iteration-closing.js","../../node_modules/core-js-pure/internals/is-array-iterator-method.js","../../node_modules/core-js-pure/internals/check-correctness-of-iteration.js","../../node_modules/core-js-pure/modules/es.array.from.js","../../node_modules/core-js-pure/internals/array-from.js","../../node_modules/core-js-pure/es/array/from.js","../../node_modules/core-js-pure/internals/is-iterable.js","../../node_modules/@babel/runtime-corejs3/helpers/iterableToArray.js","../../node_modules/@babel/runtime-corejs3/helpers/nonIterableSpread.js","../../node_modules/@babel/runtime-corejs3/helpers/toConsumableArray.js","../../node_modules/core-js-pure/modules/es.array.slice.js","../../node_modules/core-js-pure/es/array/virtual/slice.js","../../node_modules/core-js-pure/es/instance/slice.js","../../node_modules/core-js-pure/modules/es.object.get-prototype-of.js","../../node_modules/core-js-pure/es/object/get-prototype-of.js","../../node_modules/core-js-pure/stable/object/get-prototype-of.js","../../node_modules/core-js-pure/modules/es.array.index-of.js","../../node_modules/core-js-pure/es/array/virtual/index-of.js","../../node_modules/core-js-pure/es/instance/index-of.js","../../node_modules/core-js-pure/stable/array/is-array.js","../../node_modules/core-js-pure/internals/object-assign.js","../../node_modules/core-js-pure/modules/es.object.assign.js","../../node_modules/core-js-pure/es/object/assign.js","../../node_modules/core-js-pure/modules/es.symbol.iterator.js","../../node_modules/core-js-pure/es/symbol/iterator.js","../../node_modules/core-js-pure/features/symbol/iterator.js","../../node_modules/core-js-pure/modules/es.symbol.async-iterator.js","../../node_modules/core-js-pure/modules/es.symbol.has-instance.js","../../node_modules/core-js-pure/modules/es.symbol.is-concat-spreadable.js","../../node_modules/core-js-pure/modules/es.symbol.match.js","../../node_modules/core-js-pure/modules/es.symbol.match-all.js","../../node_modules/core-js-pure/modules/es.symbol.replace.js","../../node_modules/core-js-pure/modules/es.symbol.search.js","../../node_modules/core-js-pure/modules/es.symbol.species.js","../../node_modules/core-js-pure/modules/es.symbol.split.js","../../node_modules/core-js-pure/modules/es.symbol.to-primitive.js","../../node_modules/core-js-pure/modules/es.symbol.to-string-tag.js","../../node_modules/core-js-pure/modules/es.symbol.unscopables.js","../../node_modules/core-js-pure/modules/es.math.to-string-tag.js","../../node_modules/core-js-pure/modules/es.json.to-string-tag.js","../../node_modules/core-js-pure/es/symbol/index.js","../../node_modules/core-js-pure/modules/esnext.symbol.async-dispose.js","../../node_modules/core-js-pure/modules/esnext.symbol.dispose.js","../../node_modules/core-js-pure/modules/esnext.symbol.observable.js","../../node_modules/core-js-pure/modules/esnext.symbol.pattern-match.js","../../node_modules/core-js-pure/modules/esnext.symbol.replace-all.js","../../node_modules/core-js-pure/features/symbol/index.js","../../node_modules/@babel/runtime-corejs3/helpers/typeof.js","../../node_modules/@babel/runtime-corejs3/helpers/arrayWithHoles.js","../../node_modules/@babel/runtime-corejs3/helpers/iterableToArrayLimit.js","../../node_modules/@babel/runtime-corejs3/helpers/nonIterableRest.js","../../node_modules/@babel/runtime-corejs3/helpers/slicedToArray.js","../../node_modules/core-js-pure/modules/es.date.now.js","../../node_modules/core-js-pure/es/date/now.js","../../node_modules/vis-util/esnext/esm/vis-util.js","../../lib/DOMutil.js","../../node_modules/@babel/runtime-corejs3/helpers/assertThisInitialized.js","../../node_modules/core-js-pure/es/array/virtual/entries.js","../../node_modules/core-js-pure/stable/instance/entries.js","../../node_modules/core-js-pure/modules/es.json.stringify.js","../../node_modules/core-js-pure/es/json/stringify.js","../../node_modules/core-js-pure/es/array/virtual/values.js","../../node_modules/core-js-pure/stable/instance/values.js","../../node_modules/core-js-pure/es/array/virtual/keys.js","../../node_modules/core-js-pure/stable/instance/keys.js","../../node_modules/core-js-pure/modules/es.array.some.js","../../node_modules/core-js-pure/es/array/virtual/some.js","../../node_modules/core-js-pure/es/instance/some.js","../../node_modules/@babel/runtime-corejs3/helpers/possibleConstructorReturn.js","../../node_modules/core-js-pure/features/object/get-prototype-of.js","../../node_modules/core-js-pure/modules/es.object.set-prototype-of.js","../../node_modules/core-js-pure/es/object/set-prototype-of.js","../../node_modules/@babel/runtime-corejs3/helpers/getPrototypeOf.js","../../node_modules/core-js-pure/features/object/create.js","../../node_modules/@babel/runtime-corejs3/helpers/setPrototypeOf.js","../../node_modules/@babel/runtime-corejs3/helpers/inherits.js","../../node_modules/core-js-pure/modules/es.array.sort.js","../../node_modules/core-js-pure/es/array/virtual/sort.js","../../node_modules/core-js-pure/es/instance/sort.js","../../node_modules/core-js-pure/internals/freezing.js","../../node_modules/core-js-pure/internals/internal-metadata.js","../../node_modules/core-js-pure/internals/iterate.js","../../node_modules/core-js-pure/internals/an-instance.js","../../node_modules/core-js-pure/internals/collection.js","../../node_modules/core-js-pure/internals/redefine-all.js","../../node_modules/core-js-pure/internals/set-species.js","../../node_modules/core-js-pure/internals/collection-strong.js","../../node_modules/core-js-pure/modules/es.set.js","../../node_modules/core-js-pure/es/set/index.js","../../node_modules/core-js-pure/modules/es.map.js","../../node_modules/core-js-pure/es/map/index.js","../../node_modules/@babel/runtime-corejs3/node_modules/regenerator-runtime/runtime.js","../../node_modules/core-js-pure/stable/symbol/iterator.js","../../node_modules/core-js-pure/modules/es.array.splice.js","../../node_modules/core-js-pure/es/array/virtual/splice.js","../../node_modules/core-js-pure/es/instance/splice.js","../../node_modules/core-js-pure/modules/web.timers.js","../../node_modules/core-js-pure/stable/set-timeout.js","../../node_modules/core-js-pure/internals/flatten-into-array.js","../../node_modules/core-js-pure/modules/es.array.flat-map.js","../../node_modules/core-js-pure/es/array/virtual/flat-map.js","../../node_modules/core-js-pure/es/instance/flat-map.js","../../node_modules/core-js-pure/internals/array-reduce.js","../../node_modules/core-js-pure/modules/es.array.reduce.js","../../node_modules/core-js-pure/es/array/virtual/reduce.js","../../node_modules/core-js-pure/es/instance/reduce.js","../../node_modules/core-js-pure/internals/function-bind.js","../../node_modules/core-js-pure/modules/es.function.bind.js","../../node_modules/core-js-pure/es/function/virtual/bind.js","../../node_modules/core-js-pure/es/instance/bind.js","../../node_modules/@babel/runtime-corejs3/helpers/classCallCheck.js","../../node_modules/@babel/runtime-corejs3/helpers/createClass.js","../../node_modules/vis-data/esnext/esm/vis-data.js","../../lib/module/moment.js","../../lib/util.js","../../node_modules/core-js-pure/internals/number-parse-float.js","../../node_modules/core-js-pure/modules/es.parse-float.js","../../node_modules/core-js-pure/es/parse-float.js","../../lib/timeline/component/Component.js","../../node_modules/core-js-pure/modules/es.string.repeat.js","../../node_modules/core-js-pure/internals/string-repeat.js","../../node_modules/core-js-pure/es/string/virtual/repeat.js","../../node_modules/core-js-pure/es/instance/repeat.js","../../lib/timeline/DateUtil.js","../../lib/timeline/Range.js","../../node_modules/core-js-pure/stable/set-interval.js","../../node_modules/component-emitter/index.js","../../node_modules/propagating-hammerjs/propagating.js","../../node_modules/@egjs/hammerjs/dist/hammer.esm.js","../../lib/module/hammer.js","../../lib/hammerUtil.js","../../lib/timeline/TimeStep.js","../../node_modules/style-inject/dist/style-inject.es.js","../../lib/timeline/component/TimeAxis.js","../../node_modules/keycharm/keycharm.js","../../lib/shared/Activator.js","../../lib/timeline/locales.js","../../lib/timeline/component/CustomTime.js","../../lib/timeline/Core.js","../../lib/timeline/component/CurrentTime.js","../../node_modules/core-js-pure/modules/es.array.find.js","../../node_modules/core-js-pure/es/array/virtual/find.js","../../node_modules/core-js-pure/es/instance/find.js","../../node_modules/uuid/dist/esm-browser/rng.js","../../node_modules/uuid/dist/esm-browser/bytesToUuid.js","../../node_modules/uuid/dist/esm-browser/v4.js","../../node_modules/core-js-pure/modules/es.array.includes.js","../../node_modules/core-js-pure/es/array/virtual/includes.js","../../node_modules/core-js-pure/internals/is-regexp.js","../../node_modules/core-js-pure/internals/not-a-regexp.js","../../node_modules/core-js-pure/internals/correct-is-regexp-logic.js","../../node_modules/core-js-pure/modules/es.string.includes.js","../../node_modules/core-js-pure/es/string/virtual/includes.js","../../node_modules/core-js-pure/es/instance/includes.js","../../lib/timeline/Stack.js","../../lib/timeline/component/Group.js","../../lib/timeline/component/BackgroundGroup.js","../../lib/timeline/component/item/Item.js","../../lib/timeline/component/item/BoxItem.js","../../lib/timeline/component/item/PointItem.js","../../lib/timeline/component/item/RangeItem.js","../../lib/timeline/component/item/BackgroundItem.js","../../lib/shared/Popup.js","../../node_modules/core-js-pure/modules/es.array.every.js","../../node_modules/core-js-pure/es/array/virtual/every.js","../../node_modules/core-js-pure/es/instance/every.js","../../lib/timeline/component/item/ClusterItem.js","../../lib/timeline/component/ClusterGenerator.js","../../lib/timeline/component/ItemSet.js","../../lib/shared/Validator.js","../../lib/timeline/optionsTimeline.js","../../node_modules/core-js-pure/modules/es.array.fill.js","../../node_modules/core-js-pure/internals/array-fill.js","../../node_modules/core-js-pure/es/array/virtual/fill.js","../../node_modules/core-js-pure/es/instance/fill.js","../../lib/shared/ColorPicker.js","../../lib/shared/Configurator.js","../../lib/timeline/Timeline.js","../../lib/timeline/component/DataScale.js","../../lib/timeline/component/DataAxis.js","../../lib/timeline/component/graph2d_types/points.js","../../lib/timeline/component/graph2d_types/bar.js","../../lib/timeline/component/graph2d_types/line.js","../../lib/timeline/component/GraphGroup.js","../../lib/timeline/component/Legend.js","../../lib/timeline/component/LineGraph.js","../../lib/timeline/optionsGraph2d.js","../../lib/timeline/Graph2d.js","../../lib/entry-standalone.js"],"sourcesContent":["//! moment.js\n\n;(function (global, factory) {\n    typeof exports === 'object' && typeof module !== 'undefined' ? module.exports = factory() :\n    typeof define === 'function' && define.amd ? define(factory) :\n    global.moment = factory()\n}(this, (function () { 'use strict';\n\n    var hookCallback;\n\n    function hooks () {\n        return hookCallback.apply(null, arguments);\n    }\n\n    // This is done to register the method called with moment()\n    // without creating circular dependencies.\n    function setHookCallback (callback) {\n        hookCallback = callback;\n    }\n\n    function isArray(input) {\n        return input instanceof Array || Object.prototype.toString.call(input) === '[object Array]';\n    }\n\n    function isObject(input) {\n        // IE8 will treat undefined and null as object if it wasn't for\n        // input != null\n        return input != null && Object.prototype.toString.call(input) === '[object Object]';\n    }\n\n    function isObjectEmpty(obj) {\n        if (Object.getOwnPropertyNames) {\n            return (Object.getOwnPropertyNames(obj).length === 0);\n        } else {\n            var k;\n            for (k in obj) {\n                if (obj.hasOwnProperty(k)) {\n                    return false;\n                }\n            }\n            return true;\n        }\n    }\n\n    function isUndefined(input) {\n        return input === void 0;\n    }\n\n    function isNumber(input) {\n        return typeof input === 'number' || Object.prototype.toString.call(input) === '[object Number]';\n    }\n\n    function isDate(input) {\n        return input instanceof Date || Object.prototype.toString.call(input) === '[object Date]';\n    }\n\n    function map(arr, fn) {\n        var res = [], i;\n        for (i = 0; i < arr.length; ++i) {\n            res.push(fn(arr[i], i));\n        }\n        return res;\n    }\n\n    function hasOwnProp(a, b) {\n        return Object.prototype.hasOwnProperty.call(a, b);\n    }\n\n    function extend(a, b) {\n        for (var i in b) {\n            if (hasOwnProp(b, i)) {\n                a[i] = b[i];\n            }\n        }\n\n        if (hasOwnProp(b, 'toString')) {\n            a.toString = b.toString;\n        }\n\n        if (hasOwnProp(b, 'valueOf')) {\n            a.valueOf = b.valueOf;\n        }\n\n        return a;\n    }\n\n    function createUTC (input, format, locale, strict) {\n        return createLocalOrUTC(input, format, locale, strict, true).utc();\n    }\n\n    function defaultParsingFlags() {\n        // We need to deep clone this object.\n        return {\n            empty           : false,\n            unusedTokens    : [],\n            unusedInput     : [],\n            overflow        : -2,\n            charsLeftOver   : 0,\n            nullInput       : false,\n            invalidMonth    : null,\n            invalidFormat   : false,\n            userInvalidated : false,\n            iso             : false,\n            parsedDateParts : [],\n            meridiem        : null,\n            rfc2822         : false,\n            weekdayMismatch : false\n        };\n    }\n\n    function getParsingFlags(m) {\n        if (m._pf == null) {\n            m._pf = defaultParsingFlags();\n        }\n        return m._pf;\n    }\n\n    var some;\n    if (Array.prototype.some) {\n        some = Array.prototype.some;\n    } else {\n        some = function (fun) {\n            var t = Object(this);\n            var len = t.length >>> 0;\n\n            for (var i = 0; i < len; i++) {\n                if (i in t && fun.call(this, t[i], i, t)) {\n                    return true;\n                }\n            }\n\n            return false;\n        };\n    }\n\n    function isValid(m) {\n        if (m._isValid == null) {\n            var flags = getParsingFlags(m);\n            var parsedParts = some.call(flags.parsedDateParts, function (i) {\n                return i != null;\n            });\n            var isNowValid = !isNaN(m._d.getTime()) &&\n                flags.overflow < 0 &&\n                !flags.empty &&\n                !flags.invalidMonth &&\n                !flags.invalidWeekday &&\n                !flags.weekdayMismatch &&\n                !flags.nullInput &&\n                !flags.invalidFormat &&\n                !flags.userInvalidated &&\n                (!flags.meridiem || (flags.meridiem && parsedParts));\n\n            if (m._strict) {\n                isNowValid = isNowValid &&\n                    flags.charsLeftOver === 0 &&\n                    flags.unusedTokens.length === 0 &&\n                    flags.bigHour === undefined;\n            }\n\n            if (Object.isFrozen == null || !Object.isFrozen(m)) {\n                m._isValid = isNowValid;\n            }\n            else {\n                return isNowValid;\n            }\n        }\n        return m._isValid;\n    }\n\n    function createInvalid (flags) {\n        var m = createUTC(NaN);\n        if (flags != null) {\n            extend(getParsingFlags(m), flags);\n        }\n        else {\n            getParsingFlags(m).userInvalidated = true;\n        }\n\n        return m;\n    }\n\n    // Plugins that add properties should also add the key here (null value),\n    // so we can properly clone ourselves.\n    var momentProperties = hooks.momentProperties = [];\n\n    function copyConfig(to, from) {\n        var i, prop, val;\n\n        if (!isUndefined(from._isAMomentObject)) {\n            to._isAMomentObject = from._isAMomentObject;\n        }\n        if (!isUndefined(from._i)) {\n            to._i = from._i;\n        }\n        if (!isUndefined(from._f)) {\n            to._f = from._f;\n        }\n        if (!isUndefined(from._l)) {\n            to._l = from._l;\n        }\n        if (!isUndefined(from._strict)) {\n            to._strict = from._strict;\n        }\n        if (!isUndefined(from._tzm)) {\n            to._tzm = from._tzm;\n        }\n        if (!isUndefined(from._isUTC)) {\n            to._isUTC = from._isUTC;\n        }\n        if (!isUndefined(from._offset)) {\n            to._offset = from._offset;\n        }\n        if (!isUndefined(from._pf)) {\n            to._pf = getParsingFlags(from);\n        }\n        if (!isUndefined(from._locale)) {\n            to._locale = from._locale;\n        }\n\n        if (momentProperties.length > 0) {\n            for (i = 0; i < momentProperties.length; i++) {\n                prop = momentProperties[i];\n                val = from[prop];\n                if (!isUndefined(val)) {\n                    to[prop] = val;\n                }\n            }\n        }\n\n        return to;\n    }\n\n    var updateInProgress = false;\n\n    // Moment prototype object\n    function Moment(config) {\n        copyConfig(this, config);\n        this._d = new Date(config._d != null ? config._d.getTime() : NaN);\n        if (!this.isValid()) {\n            this._d = new Date(NaN);\n        }\n        // Prevent infinite loop in case updateOffset creates new moment\n        // objects.\n        if (updateInProgress === false) {\n            updateInProgress = true;\n            hooks.updateOffset(this);\n            updateInProgress = false;\n        }\n    }\n\n    function isMoment (obj) {\n        return obj instanceof Moment || (obj != null && obj._isAMomentObject != null);\n    }\n\n    function absFloor (number) {\n        if (number < 0) {\n            // -0 -> 0\n            return Math.ceil(number) || 0;\n        } else {\n            return Math.floor(number);\n        }\n    }\n\n    function toInt(argumentForCoercion) {\n        var coercedNumber = +argumentForCoercion,\n            value = 0;\n\n        if (coercedNumber !== 0 && isFinite(coercedNumber)) {\n            value = absFloor(coercedNumber);\n        }\n\n        return value;\n    }\n\n    // compare two arrays, return the number of differences\n    function compareArrays(array1, array2, dontConvert) {\n        var len = Math.min(array1.length, array2.length),\n            lengthDiff = Math.abs(array1.length - array2.length),\n            diffs = 0,\n            i;\n        for (i = 0; i < len; i++) {\n            if ((dontConvert && array1[i] !== array2[i]) ||\n                (!dontConvert && toInt(array1[i]) !== toInt(array2[i]))) {\n                diffs++;\n            }\n        }\n        return diffs + lengthDiff;\n    }\n\n    function warn(msg) {\n        if (hooks.suppressDeprecationWarnings === false &&\n                (typeof console !==  'undefined') && console.warn) {\n            console.warn('Deprecation warning: ' + msg);\n        }\n    }\n\n    function deprecate(msg, fn) {\n        var firstTime = true;\n\n        return extend(function () {\n            if (hooks.deprecationHandler != null) {\n                hooks.deprecationHandler(null, msg);\n            }\n            if (firstTime) {\n                var args = [];\n                var arg;\n                for (var i = 0; i < arguments.length; i++) {\n                    arg = '';\n                    if (typeof arguments[i] === 'object') {\n                        arg += '\\n[' + i + '] ';\n                        for (var key in arguments[0]) {\n                            arg += key + ': ' + arguments[0][key] + ', ';\n                        }\n                        arg = arg.slice(0, -2); // Remove trailing comma and space\n                    } else {\n                        arg = arguments[i];\n                    }\n                    args.push(arg);\n                }\n                warn(msg + '\\nArguments: ' + Array.prototype.slice.call(args).join('') + '\\n' + (new Error()).stack);\n                firstTime = false;\n            }\n            return fn.apply(this, arguments);\n        }, fn);\n    }\n\n    var deprecations = {};\n\n    function deprecateSimple(name, msg) {\n        if (hooks.deprecationHandler != null) {\n            hooks.deprecationHandler(name, msg);\n        }\n        if (!deprecations[name]) {\n            warn(msg);\n            deprecations[name] = true;\n        }\n    }\n\n    hooks.suppressDeprecationWarnings = false;\n    hooks.deprecationHandler = null;\n\n    function isFunction(input) {\n        return input instanceof Function || Object.prototype.toString.call(input) === '[object Function]';\n    }\n\n    function set (config) {\n        var prop, i;\n        for (i in config) {\n            prop = config[i];\n            if (isFunction(prop)) {\n                this[i] = prop;\n            } else {\n                this['_' + i] = prop;\n            }\n        }\n        this._config = config;\n        // Lenient ordinal parsing accepts just a number in addition to\n        // number + (possibly) stuff coming from _dayOfMonthOrdinalParse.\n        // TODO: Remove \"ordinalParse\" fallback in next major release.\n        this._dayOfMonthOrdinalParseLenient = new RegExp(\n            (this._dayOfMonthOrdinalParse.source || this._ordinalParse.source) +\n                '|' + (/\\d{1,2}/).source);\n    }\n\n    function mergeConfigs(parentConfig, childConfig) {\n        var res = extend({}, parentConfig), prop;\n        for (prop in childConfig) {\n            if (hasOwnProp(childConfig, prop)) {\n                if (isObject(parentConfig[prop]) && isObject(childConfig[prop])) {\n                    res[prop] = {};\n                    extend(res[prop], parentConfig[prop]);\n                    extend(res[prop], childConfig[prop]);\n                } else if (childConfig[prop] != null) {\n                    res[prop] = childConfig[prop];\n                } else {\n                    delete res[prop];\n                }\n            }\n        }\n        for (prop in parentConfig) {\n            if (hasOwnProp(parentConfig, prop) &&\n                    !hasOwnProp(childConfig, prop) &&\n                    isObject(parentConfig[prop])) {\n                // make sure changes to properties don't modify parent config\n                res[prop] = extend({}, res[prop]);\n            }\n        }\n        return res;\n    }\n\n    function Locale(config) {\n        if (config != null) {\n            this.set(config);\n        }\n    }\n\n    var keys;\n\n    if (Object.keys) {\n        keys = Object.keys;\n    } else {\n        keys = function (obj) {\n            var i, res = [];\n            for (i in obj) {\n                if (hasOwnProp(obj, i)) {\n                    res.push(i);\n                }\n            }\n            return res;\n        };\n    }\n\n    var defaultCalendar = {\n        sameDay : '[Today at] LT',\n        nextDay : '[Tomorrow at] LT',\n        nextWeek : 'dddd [at] LT',\n        lastDay : '[Yesterday at] LT',\n        lastWeek : '[Last] dddd [at] LT',\n        sameElse : 'L'\n    };\n\n    function calendar (key, mom, now) {\n        var output = this._calendar[key] || this._calendar['sameElse'];\n        return isFunction(output) ? output.call(mom, now) : output;\n    }\n\n    var defaultLongDateFormat = {\n        LTS  : 'h:mm:ss A',\n        LT   : 'h:mm A',\n        L    : 'MM/DD/YYYY',\n        LL   : 'MMMM D, YYYY',\n        LLL  : 'MMMM D, YYYY h:mm A',\n        LLLL : 'dddd, MMMM D, YYYY h:mm A'\n    };\n\n    function longDateFormat (key) {\n        var format = this._longDateFormat[key],\n            formatUpper = this._longDateFormat[key.toUpperCase()];\n\n        if (format || !formatUpper) {\n            return format;\n        }\n\n        this._longDateFormat[key] = formatUpper.replace(/MMMM|MM|DD|dddd/g, function (val) {\n            return val.slice(1);\n        });\n\n        return this._longDateFormat[key];\n    }\n\n    var defaultInvalidDate = 'Invalid date';\n\n    function invalidDate () {\n        return this._invalidDate;\n    }\n\n    var defaultOrdinal = '%d';\n    var defaultDayOfMonthOrdinalParse = /\\d{1,2}/;\n\n    function ordinal (number) {\n        return this._ordinal.replace('%d', number);\n    }\n\n    var defaultRelativeTime = {\n        future : 'in %s',\n        past   : '%s ago',\n        s  : 'a few seconds',\n        ss : '%d seconds',\n        m  : 'a minute',\n        mm : '%d minutes',\n        h  : 'an hour',\n        hh : '%d hours',\n        d  : 'a day',\n        dd : '%d days',\n        M  : 'a month',\n        MM : '%d months',\n        y  : 'a year',\n        yy : '%d years'\n    };\n\n    function relativeTime (number, withoutSuffix, string, isFuture) {\n        var output = this._relativeTime[string];\n        return (isFunction(output)) ?\n            output(number, withoutSuffix, string, isFuture) :\n            output.replace(/%d/i, number);\n    }\n\n    function pastFuture (diff, output) {\n        var format = this._relativeTime[diff > 0 ? 'future' : 'past'];\n        return isFunction(format) ? format(output) : format.replace(/%s/i, output);\n    }\n\n    var aliases = {};\n\n    function addUnitAlias (unit, shorthand) {\n        var lowerCase = unit.toLowerCase();\n        aliases[lowerCase] = aliases[lowerCase + 's'] = aliases[shorthand] = unit;\n    }\n\n    function normalizeUnits(units) {\n        return typeof units === 'string' ? aliases[units] || aliases[units.toLowerCase()] : undefined;\n    }\n\n    function normalizeObjectUnits(inputObject) {\n        var normalizedInput = {},\n            normalizedProp,\n            prop;\n\n        for (prop in inputObject) {\n            if (hasOwnProp(inputObject, prop)) {\n                normalizedProp = normalizeUnits(prop);\n                if (normalizedProp) {\n                    normalizedInput[normalizedProp] = inputObject[prop];\n                }\n            }\n        }\n\n        return normalizedInput;\n    }\n\n    var priorities = {};\n\n    function addUnitPriority(unit, priority) {\n        priorities[unit] = priority;\n    }\n\n    function getPrioritizedUnits(unitsObj) {\n        var units = [];\n        for (var u in unitsObj) {\n            units.push({unit: u, priority: priorities[u]});\n        }\n        units.sort(function (a, b) {\n            return a.priority - b.priority;\n        });\n        return units;\n    }\n\n    function zeroFill(number, targetLength, forceSign) {\n        var absNumber = '' + Math.abs(number),\n            zerosToFill = targetLength - absNumber.length,\n            sign = number >= 0;\n        return (sign ? (forceSign ? '+' : '') : '-') +\n            Math.pow(10, Math.max(0, zerosToFill)).toString().substr(1) + absNumber;\n    }\n\n    var formattingTokens = /(\\[[^\\[]*\\])|(\\\\)?([Hh]mm(ss)?|Mo|MM?M?M?|Do|DDDo|DD?D?D?|ddd?d?|do?|w[o|w]?|W[o|W]?|Qo?|YYYYYY|YYYYY|YYYY|YY|gg(ggg?)?|GG(GGG?)?|e|E|a|A|hh?|HH?|kk?|mm?|ss?|S{1,9}|x|X|zz?|ZZ?|.)/g;\n\n    var localFormattingTokens = /(\\[[^\\[]*\\])|(\\\\)?(LTS|LT|LL?L?L?|l{1,4})/g;\n\n    var formatFunctions = {};\n\n    var formatTokenFunctions = {};\n\n    // token:    'M'\n    // padded:   ['MM', 2]\n    // ordinal:  'Mo'\n    // callback: function () { this.month() + 1 }\n    function addFormatToken (token, padded, ordinal, callback) {\n        var func = callback;\n        if (typeof callback === 'string') {\n            func = function () {\n                return this[callback]();\n            };\n        }\n        if (token) {\n            formatTokenFunctions[token] = func;\n        }\n        if (padded) {\n            formatTokenFunctions[padded[0]] = function () {\n                return zeroFill(func.apply(this, arguments), padded[1], padded[2]);\n            };\n        }\n        if (ordinal) {\n            formatTokenFunctions[ordinal] = function () {\n                return this.localeData().ordinal(func.apply(this, arguments), token);\n            };\n        }\n    }\n\n    function removeFormattingTokens(input) {\n        if (input.match(/\\[[\\s\\S]/)) {\n            return input.replace(/^\\[|\\]$/g, '');\n        }\n        return input.replace(/\\\\/g, '');\n    }\n\n    function makeFormatFunction(format) {\n        var array = format.match(formattingTokens), i, length;\n\n        for (i = 0, length = array.length; i < length; i++) {\n            if (formatTokenFunctions[array[i]]) {\n                array[i] = formatTokenFunctions[array[i]];\n            } else {\n                array[i] = removeFormattingTokens(array[i]);\n            }\n        }\n\n        return function (mom) {\n            var output = '', i;\n            for (i = 0; i < length; i++) {\n                output += isFunction(array[i]) ? array[i].call(mom, format) : array[i];\n            }\n            return output;\n        };\n    }\n\n    // format date using native date object\n    function formatMoment(m, format) {\n        if (!m.isValid()) {\n            return m.localeData().invalidDate();\n        }\n\n        format = expandFormat(format, m.localeData());\n        formatFunctions[format] = formatFunctions[format] || makeFormatFunction(format);\n\n        return formatFunctions[format](m);\n    }\n\n    function expandFormat(format, locale) {\n        var i = 5;\n\n        function replaceLongDateFormatTokens(input) {\n            return locale.longDateFormat(input) || input;\n        }\n\n        localFormattingTokens.lastIndex = 0;\n        while (i >= 0 && localFormattingTokens.test(format)) {\n            format = format.replace(localFormattingTokens, replaceLongDateFormatTokens);\n            localFormattingTokens.lastIndex = 0;\n            i -= 1;\n        }\n\n        return format;\n    }\n\n    var match1         = /\\d/;            //       0 - 9\n    var match2         = /\\d\\d/;          //      00 - 99\n    var match3         = /\\d{3}/;         //     000 - 999\n    var match4         = /\\d{4}/;         //    0000 - 9999\n    var match6         = /[+-]?\\d{6}/;    // -999999 - 999999\n    var match1to2      = /\\d\\d?/;         //       0 - 99\n    var match3to4      = /\\d\\d\\d\\d?/;     //     999 - 9999\n    var match5to6      = /\\d\\d\\d\\d\\d\\d?/; //   99999 - 999999\n    var match1to3      = /\\d{1,3}/;       //       0 - 999\n    var match1to4      = /\\d{1,4}/;       //       0 - 9999\n    var match1to6      = /[+-]?\\d{1,6}/;  // -999999 - 999999\n\n    var matchUnsigned  = /\\d+/;           //       0 - inf\n    var matchSigned    = /[+-]?\\d+/;      //    -inf - inf\n\n    var matchOffset    = /Z|[+-]\\d\\d:?\\d\\d/gi; // +00:00 -00:00 +0000 -0000 or Z\n    var matchShortOffset = /Z|[+-]\\d\\d(?::?\\d\\d)?/gi; // +00 -00 +00:00 -00:00 +0000 -0000 or Z\n\n    var matchTimestamp = /[+-]?\\d+(\\.\\d{1,3})?/; // 123456789 123456789.123\n\n    // any word (or two) characters or numbers including two/three word month in arabic.\n    // includes scottish gaelic two word and hyphenated months\n    var matchWord = /[0-9]{0,256}['a-z\\u00A0-\\u05FF\\u0700-\\uD7FF\\uF900-\\uFDCF\\uFDF0-\\uFF07\\uFF10-\\uFFEF]{1,256}|[\\u0600-\\u06FF\\/]{1,256}(\\s*?[\\u0600-\\u06FF]{1,256}){1,2}/i;\n\n    var regexes = {};\n\n    function addRegexToken (token, regex, strictRegex) {\n        regexes[token] = isFunction(regex) ? regex : function (isStrict, localeData) {\n            return (isStrict && strictRegex) ? strictRegex : regex;\n        };\n    }\n\n    function getParseRegexForToken (token, config) {\n        if (!hasOwnProp(regexes, token)) {\n            return new RegExp(unescapeFormat(token));\n        }\n\n        return regexes[token](config._strict, config._locale);\n    }\n\n    // Code from http://stackoverflow.com/questions/3561493/is-there-a-regexp-escape-function-in-javascript\n    function unescapeFormat(s) {\n        return regexEscape(s.replace('\\\\', '').replace(/\\\\(\\[)|\\\\(\\])|\\[([^\\]\\[]*)\\]|\\\\(.)/g, function (matched, p1, p2, p3, p4) {\n            return p1 || p2 || p3 || p4;\n        }));\n    }\n\n    function regexEscape(s) {\n        return s.replace(/[-\\/\\\\^$*+?.()|[\\]{}]/g, '\\\\$&');\n    }\n\n    var tokens = {};\n\n    function addParseToken (token, callback) {\n        var i, func = callback;\n        if (typeof token === 'string') {\n            token = [token];\n        }\n        if (isNumber(callback)) {\n            func = function (input, array) {\n                array[callback] = toInt(input);\n            };\n        }\n        for (i = 0; i < token.length; i++) {\n            tokens[token[i]] = func;\n        }\n    }\n\n    function addWeekParseToken (token, callback) {\n        addParseToken(token, function (input, array, config, token) {\n            config._w = config._w || {};\n            callback(input, config._w, config, token);\n        });\n    }\n\n    function addTimeToArrayFromToken(token, input, config) {\n        if (input != null && hasOwnProp(tokens, token)) {\n            tokens[token](input, config._a, config, token);\n        }\n    }\n\n    var YEAR = 0;\n    var MONTH = 1;\n    var DATE = 2;\n    var HOUR = 3;\n    var MINUTE = 4;\n    var SECOND = 5;\n    var MILLISECOND = 6;\n    var WEEK = 7;\n    var WEEKDAY = 8;\n\n    // FORMATTING\n\n    addFormatToken('Y', 0, 0, function () {\n        var y = this.year();\n        return y <= 9999 ? '' + y : '+' + y;\n    });\n\n    addFormatToken(0, ['YY', 2], 0, function () {\n        return this.year() % 100;\n    });\n\n    addFormatToken(0, ['YYYY',   4],       0, 'year');\n    addFormatToken(0, ['YYYYY',  5],       0, 'year');\n    addFormatToken(0, ['YYYYYY', 6, true], 0, 'year');\n\n    // ALIASES\n\n    addUnitAlias('year', 'y');\n\n    // PRIORITIES\n\n    addUnitPriority('year', 1);\n\n    // PARSING\n\n    addRegexToken('Y',      matchSigned);\n    addRegexToken('YY',     match1to2, match2);\n    addRegexToken('YYYY',   match1to4, match4);\n    addRegexToken('YYYYY',  match1to6, match6);\n    addRegexToken('YYYYYY', match1to6, match6);\n\n    addParseToken(['YYYYY', 'YYYYYY'], YEAR);\n    addParseToken('YYYY', function (input, array) {\n        array[YEAR] = input.length === 2 ? hooks.parseTwoDigitYear(input) : toInt(input);\n    });\n    addParseToken('YY', function (input, array) {\n        array[YEAR] = hooks.parseTwoDigitYear(input);\n    });\n    addParseToken('Y', function (input, array) {\n        array[YEAR] = parseInt(input, 10);\n    });\n\n    // HELPERS\n\n    function daysInYear(year) {\n        return isLeapYear(year) ? 366 : 365;\n    }\n\n    function isLeapYear(year) {\n        return (year % 4 === 0 && year % 100 !== 0) || year % 400 === 0;\n    }\n\n    // HOOKS\n\n    hooks.parseTwoDigitYear = function (input) {\n        return toInt(input) + (toInt(input) > 68 ? 1900 : 2000);\n    };\n\n    // MOMENTS\n\n    var getSetYear = makeGetSet('FullYear', true);\n\n    function getIsLeapYear () {\n        return isLeapYear(this.year());\n    }\n\n    function makeGetSet (unit, keepTime) {\n        return function (value) {\n            if (value != null) {\n                set$1(this, unit, value);\n                hooks.updateOffset(this, keepTime);\n                return this;\n            } else {\n                return get(this, unit);\n            }\n        };\n    }\n\n    function get (mom, unit) {\n        return mom.isValid() ?\n            mom._d['get' + (mom._isUTC ? 'UTC' : '') + unit]() : NaN;\n    }\n\n    function set$1 (mom, unit, value) {\n        if (mom.isValid() && !isNaN(value)) {\n            if (unit === 'FullYear' && isLeapYear(mom.year()) && mom.month() === 1 && mom.date() === 29) {\n                mom._d['set' + (mom._isUTC ? 'UTC' : '') + unit](value, mom.month(), daysInMonth(value, mom.month()));\n            }\n            else {\n                mom._d['set' + (mom._isUTC ? 'UTC' : '') + unit](value);\n            }\n        }\n    }\n\n    // MOMENTS\n\n    function stringGet (units) {\n        units = normalizeUnits(units);\n        if (isFunction(this[units])) {\n            return this[units]();\n        }\n        return this;\n    }\n\n\n    function stringSet (units, value) {\n        if (typeof units === 'object') {\n            units = normalizeObjectUnits(units);\n            var prioritized = getPrioritizedUnits(units);\n            for (var i = 0; i < prioritized.length; i++) {\n                this[prioritized[i].unit](units[prioritized[i].unit]);\n            }\n        } else {\n            units = normalizeUnits(units);\n            if (isFunction(this[units])) {\n                return this[units](value);\n            }\n        }\n        return this;\n    }\n\n    function mod(n, x) {\n        return ((n % x) + x) % x;\n    }\n\n    var indexOf;\n\n    if (Array.prototype.indexOf) {\n        indexOf = Array.prototype.indexOf;\n    } else {\n        indexOf = function (o) {\n            // I know\n            var i;\n            for (i = 0; i < this.length; ++i) {\n                if (this[i] === o) {\n                    return i;\n                }\n            }\n            return -1;\n        };\n    }\n\n    function daysInMonth(year, month) {\n        if (isNaN(year) || isNaN(month)) {\n            return NaN;\n        }\n        var modMonth = mod(month, 12);\n        year += (month - modMonth) / 12;\n        return modMonth === 1 ? (isLeapYear(year) ? 29 : 28) : (31 - modMonth % 7 % 2);\n    }\n\n    // FORMATTING\n\n    addFormatToken('M', ['MM', 2], 'Mo', function () {\n        return this.month() + 1;\n    });\n\n    addFormatToken('MMM', 0, 0, function (format) {\n        return this.localeData().monthsShort(this, format);\n    });\n\n    addFormatToken('MMMM', 0, 0, function (format) {\n        return this.localeData().months(this, format);\n    });\n\n    // ALIASES\n\n    addUnitAlias('month', 'M');\n\n    // PRIORITY\n\n    addUnitPriority('month', 8);\n\n    // PARSING\n\n    addRegexToken('M',    match1to2);\n    addRegexToken('MM',   match1to2, match2);\n    addRegexToken('MMM',  function (isStrict, locale) {\n        return locale.monthsShortRegex(isStrict);\n    });\n    addRegexToken('MMMM', function (isStrict, locale) {\n        return locale.monthsRegex(isStrict);\n    });\n\n    addParseToken(['M', 'MM'], function (input, array) {\n        array[MONTH] = toInt(input) - 1;\n    });\n\n    addParseToken(['MMM', 'MMMM'], function (input, array, config, token) {\n        var month = config._locale.monthsParse(input, token, config._strict);\n        // if we didn't find a month name, mark the date as invalid.\n        if (month != null) {\n            array[MONTH] = month;\n        } else {\n            getParsingFlags(config).invalidMonth = input;\n        }\n    });\n\n    // LOCALES\n\n    var MONTHS_IN_FORMAT = /D[oD]?(\\[[^\\[\\]]*\\]|\\s)+MMMM?/;\n    var defaultLocaleMonths = 'January_February_March_April_May_June_July_August_September_October_November_December'.split('_');\n    function localeMonths (m, format) {\n        if (!m) {\n            return isArray(this._months) ? this._months :\n                this._months['standalone'];\n        }\n        return isArray(this._months) ? this._months[m.month()] :\n            this._months[(this._months.isFormat || MONTHS_IN_FORMAT).test(format) ? 'format' : 'standalone'][m.month()];\n    }\n\n    var defaultLocaleMonthsShort = 'Jan_Feb_Mar_Apr_May_Jun_Jul_Aug_Sep_Oct_Nov_Dec'.split('_');\n    function localeMonthsShort (m, format) {\n        if (!m) {\n            return isArray(this._monthsShort) ? this._monthsShort :\n                this._monthsShort['standalone'];\n        }\n        return isArray(this._monthsShort) ? this._monthsShort[m.month()] :\n            this._monthsShort[MONTHS_IN_FORMAT.test(format) ? 'format' : 'standalone'][m.month()];\n    }\n\n    function handleStrictParse(monthName, format, strict) {\n        var i, ii, mom, llc = monthName.toLocaleLowerCase();\n        if (!this._monthsParse) {\n            // this is not used\n            this._monthsParse = [];\n            this._longMonthsParse = [];\n            this._shortMonthsParse = [];\n            for (i = 0; i < 12; ++i) {\n                mom = createUTC([2000, i]);\n                this._shortMonthsParse[i] = this.monthsShort(mom, '').toLocaleLowerCase();\n                this._longMonthsParse[i] = this.months(mom, '').toLocaleLowerCase();\n            }\n        }\n\n        if (strict) {\n            if (format === 'MMM') {\n                ii = indexOf.call(this._shortMonthsParse, llc);\n                return ii !== -1 ? ii : null;\n            } else {\n                ii = indexOf.call(this._longMonthsParse, llc);\n                return ii !== -1 ? ii : null;\n            }\n        } else {\n            if (format === 'MMM') {\n                ii = indexOf.call(this._shortMonthsParse, llc);\n                if (ii !== -1) {\n                    return ii;\n                }\n                ii = indexOf.call(this._longMonthsParse, llc);\n                return ii !== -1 ? ii : null;\n            } else {\n                ii = indexOf.call(this._longMonthsParse, llc);\n                if (ii !== -1) {\n                    return ii;\n                }\n                ii = indexOf.call(this._shortMonthsParse, llc);\n                return ii !== -1 ? ii : null;\n            }\n        }\n    }\n\n    function localeMonthsParse (monthName, format, strict) {\n        var i, mom, regex;\n\n        if (this._monthsParseExact) {\n            return handleStrictParse.call(this, monthName, format, strict);\n        }\n\n        if (!this._monthsParse) {\n            this._monthsParse = [];\n            this._longMonthsParse = [];\n            this._shortMonthsParse = [];\n        }\n\n        // TODO: add sorting\n        // Sorting makes sure if one month (or abbr) is a prefix of another\n        // see sorting in computeMonthsParse\n        for (i = 0; i < 12; i++) {\n            // make the regex if we don't have it already\n            mom = createUTC([2000, i]);\n            if (strict && !this._longMonthsParse[i]) {\n                this._longMonthsParse[i] = new RegExp('^' + this.months(mom, '').replace('.', '') + '$', 'i');\n                this._shortMonthsParse[i] = new RegExp('^' + this.monthsShort(mom, '').replace('.', '') + '$', 'i');\n            }\n            if (!strict && !this._monthsParse[i]) {\n                regex = '^' + this.months(mom, '') + '|^' + this.monthsShort(mom, '');\n                this._monthsParse[i] = new RegExp(regex.replace('.', ''), 'i');\n            }\n            // test the regex\n            if (strict && format === 'MMMM' && this._longMonthsParse[i].test(monthName)) {\n                return i;\n            } else if (strict && format === 'MMM' && this._shortMonthsParse[i].test(monthName)) {\n                return i;\n            } else if (!strict && this._monthsParse[i].test(monthName)) {\n                return i;\n            }\n        }\n    }\n\n    // MOMENTS\n\n    function setMonth (mom, value) {\n        var dayOfMonth;\n\n        if (!mom.isValid()) {\n            // No op\n            return mom;\n        }\n\n        if (typeof value === 'string') {\n            if (/^\\d+$/.test(value)) {\n                value = toInt(value);\n            } else {\n                value = mom.localeData().monthsParse(value);\n                // TODO: Another silent failure?\n                if (!isNumber(value)) {\n                    return mom;\n                }\n            }\n        }\n\n        dayOfMonth = Math.min(mom.date(), daysInMonth(mom.year(), value));\n        mom._d['set' + (mom._isUTC ? 'UTC' : '') + 'Month'](value, dayOfMonth);\n        return mom;\n    }\n\n    function getSetMonth (value) {\n        if (value != null) {\n            setMonth(this, value);\n            hooks.updateOffset(this, true);\n            return this;\n        } else {\n            return get(this, 'Month');\n        }\n    }\n\n    function getDaysInMonth () {\n        return daysInMonth(this.year(), this.month());\n    }\n\n    var defaultMonthsShortRegex = matchWord;\n    function monthsShortRegex (isStrict) {\n        if (this._monthsParseExact) {\n            if (!hasOwnProp(this, '_monthsRegex')) {\n                computeMonthsParse.call(this);\n            }\n            if (isStrict) {\n                return this._monthsShortStrictRegex;\n            } else {\n                return this._monthsShortRegex;\n            }\n        } else {\n            if (!hasOwnProp(this, '_monthsShortRegex')) {\n                this._monthsShortRegex = defaultMonthsShortRegex;\n            }\n            return this._monthsShortStrictRegex && isStrict ?\n                this._monthsShortStrictRegex : this._monthsShortRegex;\n        }\n    }\n\n    var defaultMonthsRegex = matchWord;\n    function monthsRegex (isStrict) {\n        if (this._monthsParseExact) {\n            if (!hasOwnProp(this, '_monthsRegex')) {\n                computeMonthsParse.call(this);\n            }\n            if (isStrict) {\n                return this._monthsStrictRegex;\n            } else {\n                return this._monthsRegex;\n            }\n        } else {\n            if (!hasOwnProp(this, '_monthsRegex')) {\n                this._monthsRegex = defaultMonthsRegex;\n            }\n            return this._monthsStrictRegex && isStrict ?\n                this._monthsStrictRegex : this._monthsRegex;\n        }\n    }\n\n    function computeMonthsParse () {\n        function cmpLenRev(a, b) {\n            return b.length - a.length;\n        }\n\n        var shortPieces = [], longPieces = [], mixedPieces = [],\n            i, mom;\n        for (i = 0; i < 12; i++) {\n            // make the regex if we don't have it already\n            mom = createUTC([2000, i]);\n            shortPieces.push(this.monthsShort(mom, ''));\n            longPieces.push(this.months(mom, ''));\n            mixedPieces.push(this.months(mom, ''));\n            mixedPieces.push(this.monthsShort(mom, ''));\n        }\n        // Sorting makes sure if one month (or abbr) is a prefix of another it\n        // will match the longer piece.\n        shortPieces.sort(cmpLenRev);\n        longPieces.sort(cmpLenRev);\n        mixedPieces.sort(cmpLenRev);\n        for (i = 0; i < 12; i++) {\n            shortPieces[i] = regexEscape(shortPieces[i]);\n            longPieces[i] = regexEscape(longPieces[i]);\n        }\n        for (i = 0; i < 24; i++) {\n            mixedPieces[i] = regexEscape(mixedPieces[i]);\n        }\n\n        this._monthsRegex = new RegExp('^(' + mixedPieces.join('|') + ')', 'i');\n        this._monthsShortRegex = this._monthsRegex;\n        this._monthsStrictRegex = new RegExp('^(' + longPieces.join('|') + ')', 'i');\n        this._monthsShortStrictRegex = new RegExp('^(' + shortPieces.join('|') + ')', 'i');\n    }\n\n    function createDate (y, m, d, h, M, s, ms) {\n        // can't just apply() to create a date:\n        // https://stackoverflow.com/q/181348\n        var date;\n        // the date constructor remaps years 0-99 to 1900-1999\n        if (y < 100 && y >= 0) {\n            // preserve leap years using a full 400 year cycle, then reset\n            date = new Date(y + 400, m, d, h, M, s, ms);\n            if (isFinite(date.getFullYear())) {\n                date.setFullYear(y);\n            }\n        } else {\n            date = new Date(y, m, d, h, M, s, ms);\n        }\n\n        return date;\n    }\n\n    function createUTCDate (y) {\n        var date;\n        // the Date.UTC function remaps years 0-99 to 1900-1999\n        if (y < 100 && y >= 0) {\n            var args = Array.prototype.slice.call(arguments);\n            // preserve leap years using a full 400 year cycle, then reset\n            args[0] = y + 400;\n            date = new Date(Date.UTC.apply(null, args));\n            if (isFinite(date.getUTCFullYear())) {\n                date.setUTCFullYear(y);\n            }\n        } else {\n            date = new Date(Date.UTC.apply(null, arguments));\n        }\n\n        return date;\n    }\n\n    // start-of-first-week - start-of-year\n    function firstWeekOffset(year, dow, doy) {\n        var // first-week day -- which january is always in the first week (4 for iso, 1 for other)\n            fwd = 7 + dow - doy,\n            // first-week day local weekday -- which local weekday is fwd\n            fwdlw = (7 + createUTCDate(year, 0, fwd).getUTCDay() - dow) % 7;\n\n        return -fwdlw + fwd - 1;\n    }\n\n    // https://en.wikipedia.org/wiki/ISO_week_date#Calculating_a_date_given_the_year.2C_week_number_and_weekday\n    function dayOfYearFromWeeks(year, week, weekday, dow, doy) {\n        var localWeekday = (7 + weekday - dow) % 7,\n            weekOffset = firstWeekOffset(year, dow, doy),\n            dayOfYear = 1 + 7 * (week - 1) + localWeekday + weekOffset,\n            resYear, resDayOfYear;\n\n        if (dayOfYear <= 0) {\n            resYear = year - 1;\n            resDayOfYear = daysInYear(resYear) + dayOfYear;\n        } else if (dayOfYear > daysInYear(year)) {\n            resYear = year + 1;\n            resDayOfYear = dayOfYear - daysInYear(year);\n        } else {\n            resYear = year;\n            resDayOfYear = dayOfYear;\n        }\n\n        return {\n            year: resYear,\n            dayOfYear: resDayOfYear\n        };\n    }\n\n    function weekOfYear(mom, dow, doy) {\n        var weekOffset = firstWeekOffset(mom.year(), dow, doy),\n            week = Math.floor((mom.dayOfYear() - weekOffset - 1) / 7) + 1,\n            resWeek, resYear;\n\n        if (week < 1) {\n            resYear = mom.year() - 1;\n            resWeek = week + weeksInYear(resYear, dow, doy);\n        } else if (week > weeksInYear(mom.year(), dow, doy)) {\n            resWeek = week - weeksInYear(mom.year(), dow, doy);\n            resYear = mom.year() + 1;\n        } else {\n            resYear = mom.year();\n            resWeek = week;\n        }\n\n        return {\n            week: resWeek,\n            year: resYear\n        };\n    }\n\n    function weeksInYear(year, dow, doy) {\n        var weekOffset = firstWeekOffset(year, dow, doy),\n            weekOffsetNext = firstWeekOffset(year + 1, dow, doy);\n        return (daysInYear(year) - weekOffset + weekOffsetNext) / 7;\n    }\n\n    // FORMATTING\n\n    addFormatToken('w', ['ww', 2], 'wo', 'week');\n    addFormatToken('W', ['WW', 2], 'Wo', 'isoWeek');\n\n    // ALIASES\n\n    addUnitAlias('week', 'w');\n    addUnitAlias('isoWeek', 'W');\n\n    // PRIORITIES\n\n    addUnitPriority('week', 5);\n    addUnitPriority('isoWeek', 5);\n\n    // PARSING\n\n    addRegexToken('w',  match1to2);\n    addRegexToken('ww', match1to2, match2);\n    addRegexToken('W',  match1to2);\n    addRegexToken('WW', match1to2, match2);\n\n    addWeekParseToken(['w', 'ww', 'W', 'WW'], function (input, week, config, token) {\n        week[token.substr(0, 1)] = toInt(input);\n    });\n\n    // HELPERS\n\n    // LOCALES\n\n    function localeWeek (mom) {\n        return weekOfYear(mom, this._week.dow, this._week.doy).week;\n    }\n\n    var defaultLocaleWeek = {\n        dow : 0, // Sunday is the first day of the week.\n        doy : 6  // The week that contains Jan 6th is the first week of the year.\n    };\n\n    function localeFirstDayOfWeek () {\n        return this._week.dow;\n    }\n\n    function localeFirstDayOfYear () {\n        return this._week.doy;\n    }\n\n    // MOMENTS\n\n    function getSetWeek (input) {\n        var week = this.localeData().week(this);\n        return input == null ? week : this.add((input - week) * 7, 'd');\n    }\n\n    function getSetISOWeek (input) {\n        var week = weekOfYear(this, 1, 4).week;\n        return input == null ? week : this.add((input - week) * 7, 'd');\n    }\n\n    // FORMATTING\n\n    addFormatToken('d', 0, 'do', 'day');\n\n    addFormatToken('dd', 0, 0, function (format) {\n        return this.localeData().weekdaysMin(this, format);\n    });\n\n    addFormatToken('ddd', 0, 0, function (format) {\n        return this.localeData().weekdaysShort(this, format);\n    });\n\n    addFormatToken('dddd', 0, 0, function (format) {\n        return this.localeData().weekdays(this, format);\n    });\n\n    addFormatToken('e', 0, 0, 'weekday');\n    addFormatToken('E', 0, 0, 'isoWeekday');\n\n    // ALIASES\n\n    addUnitAlias('day', 'd');\n    addUnitAlias('weekday', 'e');\n    addUnitAlias('isoWeekday', 'E');\n\n    // PRIORITY\n    addUnitPriority('day', 11);\n    addUnitPriority('weekday', 11);\n    addUnitPriority('isoWeekday', 11);\n\n    // PARSING\n\n    addRegexToken('d',    match1to2);\n    addRegexToken('e',    match1to2);\n    addRegexToken('E',    match1to2);\n    addRegexToken('dd',   function (isStrict, locale) {\n        return locale.weekdaysMinRegex(isStrict);\n    });\n    addRegexToken('ddd',   function (isStrict, locale) {\n        return locale.weekdaysShortRegex(isStrict);\n    });\n    addRegexToken('dddd',   function (isStrict, locale) {\n        return locale.weekdaysRegex(isStrict);\n    });\n\n    addWeekParseToken(['dd', 'ddd', 'dddd'], function (input, week, config, token) {\n        var weekday = config._locale.weekdaysParse(input, token, config._strict);\n        // if we didn't get a weekday name, mark the date as invalid\n        if (weekday != null) {\n            week.d = weekday;\n        } else {\n            getParsingFlags(config).invalidWeekday = input;\n        }\n    });\n\n    addWeekParseToken(['d', 'e', 'E'], function (input, week, config, token) {\n        week[token] = toInt(input);\n    });\n\n    // HELPERS\n\n    function parseWeekday(input, locale) {\n        if (typeof input !== 'string') {\n            return input;\n        }\n\n        if (!isNaN(input)) {\n            return parseInt(input, 10);\n        }\n\n        input = locale.weekdaysParse(input);\n        if (typeof input === 'number') {\n            return input;\n        }\n\n        return null;\n    }\n\n    function parseIsoWeekday(input, locale) {\n        if (typeof input === 'string') {\n            return locale.weekdaysParse(input) % 7 || 7;\n        }\n        return isNaN(input) ? null : input;\n    }\n\n    // LOCALES\n    function shiftWeekdays (ws, n) {\n        return ws.slice(n, 7).concat(ws.slice(0, n));\n    }\n\n    var defaultLocaleWeekdays = 'Sunday_Monday_Tuesday_Wednesday_Thursday_Friday_Saturday'.split('_');\n    function localeWeekdays (m, format) {\n        var weekdays = isArray(this._weekdays) ? this._weekdays :\n            this._weekdays[(m && m !== true && this._weekdays.isFormat.test(format)) ? 'format' : 'standalone'];\n        return (m === true) ? shiftWeekdays(weekdays, this._week.dow)\n            : (m) ? weekdays[m.day()] : weekdays;\n    }\n\n    var defaultLocaleWeekdaysShort = 'Sun_Mon_Tue_Wed_Thu_Fri_Sat'.split('_');\n    function localeWeekdaysShort (m) {\n        return (m === true) ? shiftWeekdays(this._weekdaysShort, this._week.dow)\n            : (m) ? this._weekdaysShort[m.day()] : this._weekdaysShort;\n    }\n\n    var defaultLocaleWeekdaysMin = 'Su_Mo_Tu_We_Th_Fr_Sa'.split('_');\n    function localeWeekdaysMin (m) {\n        return (m === true) ? shiftWeekdays(this._weekdaysMin, this._week.dow)\n            : (m) ? this._weekdaysMin[m.day()] : this._weekdaysMin;\n    }\n\n    function handleStrictParse$1(weekdayName, format, strict) {\n        var i, ii, mom, llc = weekdayName.toLocaleLowerCase();\n        if (!this._weekdaysParse) {\n            this._weekdaysParse = [];\n            this._shortWeekdaysParse = [];\n            this._minWeekdaysParse = [];\n\n            for (i = 0; i < 7; ++i) {\n                mom = createUTC([2000, 1]).day(i);\n                this._minWeekdaysParse[i] = this.weekdaysMin(mom, '').toLocaleLowerCase();\n                this._shortWeekdaysParse[i] = this.weekdaysShort(mom, '').toLocaleLowerCase();\n                this._weekdaysParse[i] = this.weekdays(mom, '').toLocaleLowerCase();\n            }\n        }\n\n        if (strict) {\n            if (format === 'dddd') {\n                ii = indexOf.call(this._weekdaysParse, llc);\n                return ii !== -1 ? ii : null;\n            } else if (format === 'ddd') {\n                ii = indexOf.call(this._shortWeekdaysParse, llc);\n                return ii !== -1 ? ii : null;\n            } else {\n                ii = indexOf.call(this._minWeekdaysParse, llc);\n                return ii !== -1 ? ii : null;\n            }\n        } else {\n            if (format === 'dddd') {\n                ii = indexOf.call(this._weekdaysParse, llc);\n                if (ii !== -1) {\n                    return ii;\n                }\n                ii = indexOf.call(this._shortWeekdaysParse, llc);\n                if (ii !== -1) {\n                    return ii;\n                }\n                ii = indexOf.call(this._minWeekdaysParse, llc);\n                return ii !== -1 ? ii : null;\n            } else if (format === 'ddd') {\n                ii = indexOf.call(this._shortWeekdaysParse, llc);\n                if (ii !== -1) {\n                    return ii;\n                }\n                ii = indexOf.call(this._weekdaysParse, llc);\n                if (ii !== -1) {\n                    return ii;\n                }\n                ii = indexOf.call(this._minWeekdaysParse, llc);\n                return ii !== -1 ? ii : null;\n            } else {\n                ii = indexOf.call(this._minWeekdaysParse, llc);\n                if (ii !== -1) {\n                    return ii;\n                }\n                ii = indexOf.call(this._weekdaysParse, llc);\n                if (ii !== -1) {\n                    return ii;\n                }\n                ii = indexOf.call(this._shortWeekdaysParse, llc);\n                return ii !== -1 ? ii : null;\n            }\n        }\n    }\n\n    function localeWeekdaysParse (weekdayName, format, strict) {\n        var i, mom, regex;\n\n        if (this._weekdaysParseExact) {\n            return handleStrictParse$1.call(this, weekdayName, format, strict);\n        }\n\n        if (!this._weekdaysParse) {\n            this._weekdaysParse = [];\n            this._minWeekdaysParse = [];\n            this._shortWeekdaysParse = [];\n            this._fullWeekdaysParse = [];\n        }\n\n        for (i = 0; i < 7; i++) {\n            // make the regex if we don't have it already\n\n            mom = createUTC([2000, 1]).day(i);\n            if (strict && !this._fullWeekdaysParse[i]) {\n                this._fullWeekdaysParse[i] = new RegExp('^' + this.weekdays(mom, '').replace('.', '\\\\.?') + '$', 'i');\n                this._shortWeekdaysParse[i] = new RegExp('^' + this.weekdaysShort(mom, '').replace('.', '\\\\.?') + '$', 'i');\n                this._minWeekdaysParse[i] = new RegExp('^' + this.weekdaysMin(mom, '').replace('.', '\\\\.?') + '$', 'i');\n            }\n            if (!this._weekdaysParse[i]) {\n                regex = '^' + this.weekdays(mom, '') + '|^' + this.weekdaysShort(mom, '') + '|^' + this.weekdaysMin(mom, '');\n                this._weekdaysParse[i] = new RegExp(regex.replace('.', ''), 'i');\n            }\n            // test the regex\n            if (strict && format === 'dddd' && this._fullWeekdaysParse[i].test(weekdayName)) {\n                return i;\n            } else if (strict && format === 'ddd' && this._shortWeekdaysParse[i].test(weekdayName)) {\n                return i;\n            } else if (strict && format === 'dd' && this._minWeekdaysParse[i].test(weekdayName)) {\n                return i;\n            } else if (!strict && this._weekdaysParse[i].test(weekdayName)) {\n                return i;\n            }\n        }\n    }\n\n    // MOMENTS\n\n    function getSetDayOfWeek (input) {\n        if (!this.isValid()) {\n            return input != null ? this : NaN;\n        }\n        var day = this._isUTC ? this._d.getUTCDay() : this._d.getDay();\n        if (input != null) {\n            input = parseWeekday(input, this.localeData());\n            return this.add(input - day, 'd');\n        } else {\n            return day;\n        }\n    }\n\n    function getSetLocaleDayOfWeek (input) {\n        if (!this.isValid()) {\n            return input != null ? this : NaN;\n        }\n        var weekday = (this.day() + 7 - this.localeData()._week.dow) % 7;\n        return input == null ? weekday : this.add(input - weekday, 'd');\n    }\n\n    function getSetISODayOfWeek (input) {\n        if (!this.isValid()) {\n            return input != null ? this : NaN;\n        }\n\n        // behaves the same as moment#day except\n        // as a getter, returns 7 instead of 0 (1-7 range instead of 0-6)\n        // as a setter, sunday should belong to the previous week.\n\n        if (input != null) {\n            var weekday = parseIsoWeekday(input, this.localeData());\n            return this.day(this.day() % 7 ? weekday : weekday - 7);\n        } else {\n            return this.day() || 7;\n        }\n    }\n\n    var defaultWeekdaysRegex = matchWord;\n    function weekdaysRegex (isStrict) {\n        if (this._weekdaysParseExact) {\n            if (!hasOwnProp(this, '_weekdaysRegex')) {\n                computeWeekdaysParse.call(this);\n            }\n            if (isStrict) {\n                return this._weekdaysStrictRegex;\n            } else {\n                return this._weekdaysRegex;\n            }\n        } else {\n            if (!hasOwnProp(this, '_weekdaysRegex')) {\n                this._weekdaysRegex = defaultWeekdaysRegex;\n            }\n            return this._weekdaysStrictRegex && isStrict ?\n                this._weekdaysStrictRegex : this._weekdaysRegex;\n        }\n    }\n\n    var defaultWeekdaysShortRegex = matchWord;\n    function weekdaysShortRegex (isStrict) {\n        if (this._weekdaysParseExact) {\n            if (!hasOwnProp(this, '_weekdaysRegex')) {\n                computeWeekdaysParse.call(this);\n            }\n            if (isStrict) {\n                return this._weekdaysShortStrictRegex;\n            } else {\n                return this._weekdaysShortRegex;\n            }\n        } else {\n            if (!hasOwnProp(this, '_weekdaysShortRegex')) {\n                this._weekdaysShortRegex = defaultWeekdaysShortRegex;\n            }\n            return this._weekdaysShortStrictRegex && isStrict ?\n                this._weekdaysShortStrictRegex : this._weekdaysShortRegex;\n        }\n    }\n\n    var defaultWeekdaysMinRegex = matchWord;\n    function weekdaysMinRegex (isStrict) {\n        if (this._weekdaysParseExact) {\n            if (!hasOwnProp(this, '_weekdaysRegex')) {\n                computeWeekdaysParse.call(this);\n            }\n            if (isStrict) {\n                return this._weekdaysMinStrictRegex;\n            } else {\n                return this._weekdaysMinRegex;\n            }\n        } else {\n            if (!hasOwnProp(this, '_weekdaysMinRegex')) {\n                this._weekdaysMinRegex = defaultWeekdaysMinRegex;\n            }\n            return this._weekdaysMinStrictRegex && isStrict ?\n                this._weekdaysMinStrictRegex : this._weekdaysMinRegex;\n        }\n    }\n\n\n    function computeWeekdaysParse () {\n        function cmpLenRev(a, b) {\n            return b.length - a.length;\n        }\n\n        var minPieces = [], shortPieces = [], longPieces = [], mixedPieces = [],\n            i, mom, minp, shortp, longp;\n        for (i = 0; i < 7; i++) {\n            // make the regex if we don't have it already\n            mom = createUTC([2000, 1]).day(i);\n            minp = this.weekdaysMin(mom, '');\n            shortp = this.weekdaysShort(mom, '');\n            longp = this.weekdays(mom, '');\n            minPieces.push(minp);\n            shortPieces.push(shortp);\n            longPieces.push(longp);\n            mixedPieces.push(minp);\n            mixedPieces.push(shortp);\n            mixedPieces.push(longp);\n        }\n        // Sorting makes sure if one weekday (or abbr) is a prefix of another it\n        // will match the longer piece.\n        minPieces.sort(cmpLenRev);\n        shortPieces.sort(cmpLenRev);\n        longPieces.sort(cmpLenRev);\n        mixedPieces.sort(cmpLenRev);\n        for (i = 0; i < 7; i++) {\n            shortPieces[i] = regexEscape(shortPieces[i]);\n            longPieces[i] = regexEscape(longPieces[i]);\n            mixedPieces[i] = regexEscape(mixedPieces[i]);\n        }\n\n        this._weekdaysRegex = new RegExp('^(' + mixedPieces.join('|') + ')', 'i');\n        this._weekdaysShortRegex = this._weekdaysRegex;\n        this._weekdaysMinRegex = this._weekdaysRegex;\n\n        this._weekdaysStrictRegex = new RegExp('^(' + longPieces.join('|') + ')', 'i');\n        this._weekdaysShortStrictRegex = new RegExp('^(' + shortPieces.join('|') + ')', 'i');\n        this._weekdaysMinStrictRegex = new RegExp('^(' + minPieces.join('|') + ')', 'i');\n    }\n\n    // FORMATTING\n\n    function hFormat() {\n        return this.hours() % 12 || 12;\n    }\n\n    function kFormat() {\n        return this.hours() || 24;\n    }\n\n    addFormatToken('H', ['HH', 2], 0, 'hour');\n    addFormatToken('h', ['hh', 2], 0, hFormat);\n    addFormatToken('k', ['kk', 2], 0, kFormat);\n\n    addFormatToken('hmm', 0, 0, function () {\n        return '' + hFormat.apply(this) + zeroFill(this.minutes(), 2);\n    });\n\n    addFormatToken('hmmss', 0, 0, function () {\n        return '' + hFormat.apply(this) + zeroFill(this.minutes(), 2) +\n            zeroFill(this.seconds(), 2);\n    });\n\n    addFormatToken('Hmm', 0, 0, function () {\n        return '' + this.hours() + zeroFill(this.minutes(), 2);\n    });\n\n    addFormatToken('Hmmss', 0, 0, function () {\n        return '' + this.hours() + zeroFill(this.minutes(), 2) +\n            zeroFill(this.seconds(), 2);\n    });\n\n    function meridiem (token, lowercase) {\n        addFormatToken(token, 0, 0, function () {\n            return this.localeData().meridiem(this.hours(), this.minutes(), lowercase);\n        });\n    }\n\n    meridiem('a', true);\n    meridiem('A', false);\n\n    // ALIASES\n\n    addUnitAlias('hour', 'h');\n\n    // PRIORITY\n    addUnitPriority('hour', 13);\n\n    // PARSING\n\n    function matchMeridiem (isStrict, locale) {\n        return locale._meridiemParse;\n    }\n\n    addRegexToken('a',  matchMeridiem);\n    addRegexToken('A',  matchMeridiem);\n    addRegexToken('H',  match1to2);\n    addRegexToken('h',  match1to2);\n    addRegexToken('k',  match1to2);\n    addRegexToken('HH', match1to2, match2);\n    addRegexToken('hh', match1to2, match2);\n    addRegexToken('kk', match1to2, match2);\n\n    addRegexToken('hmm', match3to4);\n    addRegexToken('hmmss', match5to6);\n    addRegexToken('Hmm', match3to4);\n    addRegexToken('Hmmss', match5to6);\n\n    addParseToken(['H', 'HH'], HOUR);\n    addParseToken(['k', 'kk'], function (input, array, config) {\n        var kInput = toInt(input);\n        array[HOUR] = kInput === 24 ? 0 : kInput;\n    });\n    addParseToken(['a', 'A'], function (input, array, config) {\n        config._isPm = config._locale.isPM(input);\n        config._meridiem = input;\n    });\n    addParseToken(['h', 'hh'], function (input, array, config) {\n        array[HOUR] = toInt(input);\n        getParsingFlags(config).bigHour = true;\n    });\n    addParseToken('hmm', function (input, array, config) {\n        var pos = input.length - 2;\n        array[HOUR] = toInt(input.substr(0, pos));\n        array[MINUTE] = toInt(input.substr(pos));\n        getParsingFlags(config).bigHour = true;\n    });\n    addParseToken('hmmss', function (input, array, config) {\n        var pos1 = input.length - 4;\n        var pos2 = input.length - 2;\n        array[HOUR] = toInt(input.substr(0, pos1));\n        array[MINUTE] = toInt(input.substr(pos1, 2));\n        array[SECOND] = toInt(input.substr(pos2));\n        getParsingFlags(config).bigHour = true;\n    });\n    addParseToken('Hmm', function (input, array, config) {\n        var pos = input.length - 2;\n        array[HOUR] = toInt(input.substr(0, pos));\n        array[MINUTE] = toInt(input.substr(pos));\n    });\n    addParseToken('Hmmss', function (input, array, config) {\n        var pos1 = input.length - 4;\n        var pos2 = input.length - 2;\n        array[HOUR] = toInt(input.substr(0, pos1));\n        array[MINUTE] = toInt(input.substr(pos1, 2));\n        array[SECOND] = toInt(input.substr(pos2));\n    });\n\n    // LOCALES\n\n    function localeIsPM (input) {\n        // IE8 Quirks Mode & IE7 Standards Mode do not allow accessing strings like arrays\n        // Using charAt should be more compatible.\n        return ((input + '').toLowerCase().charAt(0) === 'p');\n    }\n\n    var defaultLocaleMeridiemParse = /[ap]\\.?m?\\.?/i;\n    function localeMeridiem (hours, minutes, isLower) {\n        if (hours > 11) {\n            return isLower ? 'pm' : 'PM';\n        } else {\n            return isLower ? 'am' : 'AM';\n        }\n    }\n\n\n    // MOMENTS\n\n    // Setting the hour should keep the time, because the user explicitly\n    // specified which hour they want. So trying to maintain the same hour (in\n    // a new timezone) makes sense. Adding/subtracting hours does not follow\n    // this rule.\n    var getSetHour = makeGetSet('Hours', true);\n\n    var baseConfig = {\n        calendar: defaultCalendar,\n        longDateFormat: defaultLongDateFormat,\n        invalidDate: defaultInvalidDate,\n        ordinal: defaultOrdinal,\n        dayOfMonthOrdinalParse: defaultDayOfMonthOrdinalParse,\n        relativeTime: defaultRelativeTime,\n\n        months: defaultLocaleMonths,\n        monthsShort: defaultLocaleMonthsShort,\n\n        week: defaultLocaleWeek,\n\n        weekdays: defaultLocaleWeekdays,\n        weekdaysMin: defaultLocaleWeekdaysMin,\n        weekdaysShort: defaultLocaleWeekdaysShort,\n\n        meridiemParse: defaultLocaleMeridiemParse\n    };\n\n    // internal storage for locale config files\n    var locales = {};\n    var localeFamilies = {};\n    var globalLocale;\n\n    function normalizeLocale(key) {\n        return key ? key.toLowerCase().replace('_', '-') : key;\n    }\n\n    // pick the locale from the array\n    // try ['en-au', 'en-gb'] as 'en-au', 'en-gb', 'en', as in move through the list trying each\n    // substring from most specific to least, but move to the next array item if it's a more specific variant than the current root\n    function chooseLocale(names) {\n        var i = 0, j, next, locale, split;\n\n        while (i < names.length) {\n            split = normalizeLocale(names[i]).split('-');\n            j = split.length;\n            next = normalizeLocale(names[i + 1]);\n            next = next ? next.split('-') : null;\n            while (j > 0) {\n                locale = loadLocale(split.slice(0, j).join('-'));\n                if (locale) {\n                    return locale;\n                }\n                if (next && next.length >= j && compareArrays(split, next, true) >= j - 1) {\n                    //the next array item is better than a shallower substring of this one\n                    break;\n                }\n                j--;\n            }\n            i++;\n        }\n        return globalLocale;\n    }\n\n    function loadLocale(name) {\n        var oldLocale = null;\n        // TODO: Find a better way to register and load all the locales in Node\n        if (!locales[name] && (typeof module !== 'undefined') &&\n                module && module.exports) {\n            try {\n                oldLocale = globalLocale._abbr;\n                var aliasedRequire = require;\n                aliasedRequire('./locale/' + name);\n                getSetGlobalLocale(oldLocale);\n            } catch (e) {}\n        }\n        return locales[name];\n    }\n\n    // This function will load locale and then set the global locale.  If\n    // no arguments are passed in, it will simply return the current global\n    // locale key.\n    function getSetGlobalLocale (key, values) {\n        var data;\n        if (key) {\n            if (isUndefined(values)) {\n                data = getLocale(key);\n            }\n            else {\n                data = defineLocale(key, values);\n            }\n\n            if (data) {\n                // moment.duration._locale = moment._locale = data;\n                globalLocale = data;\n            }\n            else {\n                if ((typeof console !==  'undefined') && console.warn) {\n                    //warn user if arguments are passed but the locale could not be set\n                    console.warn('Locale ' + key +  ' not found. Did you forget to load it?');\n                }\n            }\n        }\n\n        return globalLocale._abbr;\n    }\n\n    function defineLocale (name, config) {\n        if (config !== null) {\n            var locale, parentConfig = baseConfig;\n            config.abbr = name;\n            if (locales[name] != null) {\n                deprecateSimple('defineLocaleOverride',\n                        'use moment.updateLocale(localeName, config) to change ' +\n                        'an existing locale. moment.defineLocale(localeName, ' +\n                        'config) should only be used for creating a new locale ' +\n                        'See http://momentjs.com/guides/#/warnings/define-locale/ for more info.');\n                parentConfig = locales[name]._config;\n            } else if (config.parentLocale != null) {\n                if (locales[config.parentLocale] != null) {\n                    parentConfig = locales[config.parentLocale]._config;\n                } else {\n                    locale = loadLocale(config.parentLocale);\n                    if (locale != null) {\n                        parentConfig = locale._config;\n                    } else {\n                        if (!localeFamilies[config.parentLocale]) {\n                            localeFamilies[config.parentLocale] = [];\n                        }\n                        localeFamilies[config.parentLocale].push({\n                            name: name,\n                            config: config\n                        });\n                        return null;\n                    }\n                }\n            }\n            locales[name] = new Locale(mergeConfigs(parentConfig, config));\n\n            if (localeFamilies[name]) {\n                localeFamilies[name].forEach(function (x) {\n                    defineLocale(x.name, x.config);\n                });\n            }\n\n            // backwards compat for now: also set the locale\n            // make sure we set the locale AFTER all child locales have been\n            // created, so we won't end up with the child locale set.\n            getSetGlobalLocale(name);\n\n\n            return locales[name];\n        } else {\n            // useful for testing\n            delete locales[name];\n            return null;\n        }\n    }\n\n    function updateLocale(name, config) {\n        if (config != null) {\n            var locale, tmpLocale, parentConfig = baseConfig;\n            // MERGE\n            tmpLocale = loadLocale(name);\n            if (tmpLocale != null) {\n                parentConfig = tmpLocale._config;\n            }\n            config = mergeConfigs(parentConfig, config);\n            locale = new Locale(config);\n            locale.parentLocale = locales[name];\n            locales[name] = locale;\n\n            // backwards compat for now: also set the locale\n            getSetGlobalLocale(name);\n        } else {\n            // pass null for config to unupdate, useful for tests\n            if (locales[name] != null) {\n                if (locales[name].parentLocale != null) {\n                    locales[name] = locales[name].parentLocale;\n                } else if (locales[name] != null) {\n                    delete locales[name];\n                }\n            }\n        }\n        return locales[name];\n    }\n\n    // returns locale data\n    function getLocale (key) {\n        var locale;\n\n        if (key && key._locale && key._locale._abbr) {\n            key = key._locale._abbr;\n        }\n\n        if (!key) {\n            return globalLocale;\n        }\n\n        if (!isArray(key)) {\n            //short-circuit everything else\n            locale = loadLocale(key);\n            if (locale) {\n                return locale;\n            }\n            key = [key];\n        }\n\n        return chooseLocale(key);\n    }\n\n    function listLocales() {\n        return keys(locales);\n    }\n\n    function checkOverflow (m) {\n        var overflow;\n        var a = m._a;\n\n        if (a && getParsingFlags(m).overflow === -2) {\n            overflow =\n                a[MONTH]       < 0 || a[MONTH]       > 11  ? MONTH :\n                a[DATE]        < 1 || a[DATE]        > daysInMonth(a[YEAR], a[MONTH]) ? DATE :\n                a[HOUR]        < 0 || a[HOUR]        > 24 || (a[HOUR] === 24 && (a[MINUTE] !== 0 || a[SECOND] !== 0 || a[MILLISECOND] !== 0)) ? HOUR :\n                a[MINUTE]      < 0 || a[MINUTE]      > 59  ? MINUTE :\n                a[SECOND]      < 0 || a[SECOND]      > 59  ? SECOND :\n                a[MILLISECOND] < 0 || a[MILLISECOND] > 999 ? MILLISECOND :\n                -1;\n\n            if (getParsingFlags(m)._overflowDayOfYear && (overflow < YEAR || overflow > DATE)) {\n                overflow = DATE;\n            }\n            if (getParsingFlags(m)._overflowWeeks && overflow === -1) {\n                overflow = WEEK;\n            }\n            if (getParsingFlags(m)._overflowWeekday && overflow === -1) {\n                overflow = WEEKDAY;\n            }\n\n            getParsingFlags(m).overflow = overflow;\n        }\n\n        return m;\n    }\n\n    // Pick the first defined of two or three arguments.\n    function defaults(a, b, c) {\n        if (a != null) {\n            return a;\n        }\n        if (b != null) {\n            return b;\n        }\n        return c;\n    }\n\n    function currentDateArray(config) {\n        // hooks is actually the exported moment object\n        var nowValue = new Date(hooks.now());\n        if (config._useUTC) {\n            return [nowValue.getUTCFullYear(), nowValue.getUTCMonth(), nowValue.getUTCDate()];\n        }\n        return [nowValue.getFullYear(), nowValue.getMonth(), nowValue.getDate()];\n    }\n\n    // convert an array to a date.\n    // the array should mirror the parameters below\n    // note: all values past the year are optional and will default to the lowest possible value.\n    // [year, month, day , hour, minute, second, millisecond]\n    function configFromArray (config) {\n        var i, date, input = [], currentDate, expectedWeekday, yearToUse;\n\n        if (config._d) {\n            return;\n        }\n\n        currentDate = currentDateArray(config);\n\n        //compute day of the year from weeks and weekdays\n        if (config._w && config._a[DATE] == null && config._a[MONTH] == null) {\n            dayOfYearFromWeekInfo(config);\n        }\n\n        //if the day of the year is set, figure out what it is\n        if (config._dayOfYear != null) {\n            yearToUse = defaults(config._a[YEAR], currentDate[YEAR]);\n\n            if (config._dayOfYear > daysInYear(yearToUse) || config._dayOfYear === 0) {\n                getParsingFlags(config)._overflowDayOfYear = true;\n            }\n\n            date = createUTCDate(yearToUse, 0, config._dayOfYear);\n            config._a[MONTH] = date.getUTCMonth();\n            config._a[DATE] = date.getUTCDate();\n        }\n\n        // Default to current date.\n        // * if no year, month, day of month are given, default to today\n        // * if day of month is given, default month and year\n        // * if month is given, default only year\n        // * if year is given, don't default anything\n        for (i = 0; i < 3 && config._a[i] == null; ++i) {\n            config._a[i] = input[i] = currentDate[i];\n        }\n\n        // Zero out whatever was not defaulted, including time\n        for (; i < 7; i++) {\n            config._a[i] = input[i] = (config._a[i] == null) ? (i === 2 ? 1 : 0) : config._a[i];\n        }\n\n        // Check for 24:00:00.000\n        if (config._a[HOUR] === 24 &&\n                config._a[MINUTE] === 0 &&\n                config._a[SECOND] === 0 &&\n                config._a[MILLISECOND] === 0) {\n            config._nextDay = true;\n            config._a[HOUR] = 0;\n        }\n\n        config._d = (config._useUTC ? createUTCDate : createDate).apply(null, input);\n        expectedWeekday = config._useUTC ? config._d.getUTCDay() : config._d.getDay();\n\n        // Apply timezone offset from input. The actual utcOffset can be changed\n        // with parseZone.\n        if (config._tzm != null) {\n            config._d.setUTCMinutes(config._d.getUTCMinutes() - config._tzm);\n        }\n\n        if (config._nextDay) {\n            config._a[HOUR] = 24;\n        }\n\n        // check for mismatching day of week\n        if (config._w && typeof config._w.d !== 'undefined' && config._w.d !== expectedWeekday) {\n            getParsingFlags(config).weekdayMismatch = true;\n        }\n    }\n\n    function dayOfYearFromWeekInfo(config) {\n        var w, weekYear, week, weekday, dow, doy, temp, weekdayOverflow;\n\n        w = config._w;\n        if (w.GG != null || w.W != null || w.E != null) {\n            dow = 1;\n            doy = 4;\n\n            // TODO: We need to take the current isoWeekYear, but that depends on\n            // how we interpret now (local, utc, fixed offset). So create\n            // a now version of current config (take local/utc/offset flags, and\n            // create now).\n            weekYear = defaults(w.GG, config._a[YEAR], weekOfYear(createLocal(), 1, 4).year);\n            week = defaults(w.W, 1);\n            weekday = defaults(w.E, 1);\n            if (weekday < 1 || weekday > 7) {\n                weekdayOverflow = true;\n            }\n        } else {\n            dow = config._locale._week.dow;\n            doy = config._locale._week.doy;\n\n            var curWeek = weekOfYear(createLocal(), dow, doy);\n\n            weekYear = defaults(w.gg, config._a[YEAR], curWeek.year);\n\n            // Default to current week.\n            week = defaults(w.w, curWeek.week);\n\n            if (w.d != null) {\n                // weekday -- low day numbers are considered next week\n                weekday = w.d;\n                if (weekday < 0 || weekday > 6) {\n                    weekdayOverflow = true;\n                }\n            } else if (w.e != null) {\n                // local weekday -- counting starts from beginning of week\n                weekday = w.e + dow;\n                if (w.e < 0 || w.e > 6) {\n                    weekdayOverflow = true;\n                }\n            } else {\n                // default to beginning of week\n                weekday = dow;\n            }\n        }\n        if (week < 1 || week > weeksInYear(weekYear, dow, doy)) {\n            getParsingFlags(config)._overflowWeeks = true;\n        } else if (weekdayOverflow != null) {\n            getParsingFlags(config)._overflowWeekday = true;\n        } else {\n            temp = dayOfYearFromWeeks(weekYear, week, weekday, dow, doy);\n            config._a[YEAR] = temp.year;\n            config._dayOfYear = temp.dayOfYear;\n        }\n    }\n\n    // iso 8601 regex\n    // 0000-00-00 0000-W00 or 0000-W00-0 + T + 00 or 00:00 or 00:00:00 or 00:00:00.000 + +00:00 or +0000 or +00)\n    var extendedIsoRegex = /^\\s*((?:[+-]\\d{6}|\\d{4})-(?:\\d\\d-\\d\\d|W\\d\\d-\\d|W\\d\\d|\\d\\d\\d|\\d\\d))(?:(T| )(\\d\\d(?::\\d\\d(?::\\d\\d(?:[.,]\\d+)?)?)?)([\\+\\-]\\d\\d(?::?\\d\\d)?|\\s*Z)?)?$/;\n    var basicIsoRegex = /^\\s*((?:[+-]\\d{6}|\\d{4})(?:\\d\\d\\d\\d|W\\d\\d\\d|W\\d\\d|\\d\\d\\d|\\d\\d))(?:(T| )(\\d\\d(?:\\d\\d(?:\\d\\d(?:[.,]\\d+)?)?)?)([\\+\\-]\\d\\d(?::?\\d\\d)?|\\s*Z)?)?$/;\n\n    var tzRegex = /Z|[+-]\\d\\d(?::?\\d\\d)?/;\n\n    var isoDates = [\n        ['YYYYYY-MM-DD', /[+-]\\d{6}-\\d\\d-\\d\\d/],\n        ['YYYY-MM-DD', /\\d{4}-\\d\\d-\\d\\d/],\n        ['GGGG-[W]WW-E', /\\d{4}-W\\d\\d-\\d/],\n        ['GGGG-[W]WW', /\\d{4}-W\\d\\d/, false],\n        ['YYYY-DDD', /\\d{4}-\\d{3}/],\n        ['YYYY-MM', /\\d{4}-\\d\\d/, false],\n        ['YYYYYYMMDD', /[+-]\\d{10}/],\n        ['YYYYMMDD', /\\d{8}/],\n        // YYYYMM is NOT allowed by the standard\n        ['GGGG[W]WWE', /\\d{4}W\\d{3}/],\n        ['GGGG[W]WW', /\\d{4}W\\d{2}/, false],\n        ['YYYYDDD', /\\d{7}/]\n    ];\n\n    // iso time formats and regexes\n    var isoTimes = [\n        ['HH:mm:ss.SSSS', /\\d\\d:\\d\\d:\\d\\d\\.\\d+/],\n        ['HH:mm:ss,SSSS', /\\d\\d:\\d\\d:\\d\\d,\\d+/],\n        ['HH:mm:ss', /\\d\\d:\\d\\d:\\d\\d/],\n        ['HH:mm', /\\d\\d:\\d\\d/],\n        ['HHmmss.SSSS', /\\d\\d\\d\\d\\d\\d\\.\\d+/],\n        ['HHmmss,SSSS', /\\d\\d\\d\\d\\d\\d,\\d+/],\n        ['HHmmss', /\\d\\d\\d\\d\\d\\d/],\n        ['HHmm', /\\d\\d\\d\\d/],\n        ['HH', /\\d\\d/]\n    ];\n\n    var aspNetJsonRegex = /^\\/?Date\\((\\-?\\d+)/i;\n\n    // date from iso format\n    function configFromISO(config) {\n        var i, l,\n            string = config._i,\n            match = extendedIsoRegex.exec(string) || basicIsoRegex.exec(string),\n            allowTime, dateFormat, timeFormat, tzFormat;\n\n        if (match) {\n            getParsingFlags(config).iso = true;\n\n            for (i = 0, l = isoDates.length; i < l; i++) {\n                if (isoDates[i][1].exec(match[1])) {\n                    dateFormat = isoDates[i][0];\n                    allowTime = isoDates[i][2] !== false;\n                    break;\n                }\n            }\n            if (dateFormat == null) {\n                config._isValid = false;\n                return;\n            }\n            if (match[3]) {\n                for (i = 0, l = isoTimes.length; i < l; i++) {\n                    if (isoTimes[i][1].exec(match[3])) {\n                        // match[2] should be 'T' or space\n                        timeFormat = (match[2] || ' ') + isoTimes[i][0];\n                        break;\n                    }\n                }\n                if (timeFormat == null) {\n                    config._isValid = false;\n                    return;\n                }\n            }\n            if (!allowTime && timeFormat != null) {\n                config._isValid = false;\n                return;\n            }\n            if (match[4]) {\n                if (tzRegex.exec(match[4])) {\n                    tzFormat = 'Z';\n                } else {\n                    config._isValid = false;\n                    return;\n                }\n            }\n            config._f = dateFormat + (timeFormat || '') + (tzFormat || '');\n            configFromStringAndFormat(config);\n        } else {\n            config._isValid = false;\n        }\n    }\n\n    // RFC 2822 regex: For details see https://tools.ietf.org/html/rfc2822#section-3.3\n    var rfc2822 = /^(?:(Mon|Tue|Wed|Thu|Fri|Sat|Sun),?\\s)?(\\d{1,2})\\s(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\\s(\\d{2,4})\\s(\\d\\d):(\\d\\d)(?::(\\d\\d))?\\s(?:(UT|GMT|[ECMP][SD]T)|([Zz])|([+-]\\d{4}))$/;\n\n    function extractFromRFC2822Strings(yearStr, monthStr, dayStr, hourStr, minuteStr, secondStr) {\n        var result = [\n            untruncateYear(yearStr),\n            defaultLocaleMonthsShort.indexOf(monthStr),\n            parseInt(dayStr, 10),\n            parseInt(hourStr, 10),\n            parseInt(minuteStr, 10)\n        ];\n\n        if (secondStr) {\n            result.push(parseInt(secondStr, 10));\n        }\n\n        return result;\n    }\n\n    function untruncateYear(yearStr) {\n        var year = parseInt(yearStr, 10);\n        if (year <= 49) {\n            return 2000 + year;\n        } else if (year <= 999) {\n            return 1900 + year;\n        }\n        return year;\n    }\n\n    function preprocessRFC2822(s) {\n        // Remove comments and folding whitespace and replace multiple-spaces with a single space\n        return s.replace(/\\([^)]*\\)|[\\n\\t]/g, ' ').replace(/(\\s\\s+)/g, ' ').replace(/^\\s\\s*/, '').replace(/\\s\\s*$/, '');\n    }\n\n    function checkWeekday(weekdayStr, parsedInput, config) {\n        if (weekdayStr) {\n            // TODO: Replace the vanilla JS Date object with an indepentent day-of-week check.\n            var weekdayProvided = defaultLocaleWeekdaysShort.indexOf(weekdayStr),\n                weekdayActual = new Date(parsedInput[0], parsedInput[1], parsedInput[2]).getDay();\n            if (weekdayProvided !== weekdayActual) {\n                getParsingFlags(config).weekdayMismatch = true;\n                config._isValid = false;\n                return false;\n            }\n        }\n        return true;\n    }\n\n    var obsOffsets = {\n        UT: 0,\n        GMT: 0,\n        EDT: -4 * 60,\n        EST: -5 * 60,\n        CDT: -5 * 60,\n        CST: -6 * 60,\n        MDT: -6 * 60,\n        MST: -7 * 60,\n        PDT: -7 * 60,\n        PST: -8 * 60\n    };\n\n    function calculateOffset(obsOffset, militaryOffset, numOffset) {\n        if (obsOffset) {\n            return obsOffsets[obsOffset];\n        } else if (militaryOffset) {\n            // the only allowed military tz is Z\n            return 0;\n        } else {\n            var hm = parseInt(numOffset, 10);\n            var m = hm % 100, h = (hm - m) / 100;\n            return h * 60 + m;\n        }\n    }\n\n    // date and time from ref 2822 format\n    function configFromRFC2822(config) {\n        var match = rfc2822.exec(preprocessRFC2822(config._i));\n        if (match) {\n            var parsedArray = extractFromRFC2822Strings(match[4], match[3], match[2], match[5], match[6], match[7]);\n            if (!checkWeekday(match[1], parsedArray, config)) {\n                return;\n            }\n\n            config._a = parsedArray;\n            config._tzm = calculateOffset(match[8], match[9], match[10]);\n\n            config._d = createUTCDate.apply(null, config._a);\n            config._d.setUTCMinutes(config._d.getUTCMinutes() - config._tzm);\n\n            getParsingFlags(config).rfc2822 = true;\n        } else {\n            config._isValid = false;\n        }\n    }\n\n    // date from iso format or fallback\n    function configFromString(config) {\n        var matched = aspNetJsonRegex.exec(config._i);\n\n        if (matched !== null) {\n            config._d = new Date(+matched[1]);\n            return;\n        }\n\n        configFromISO(config);\n        if (config._isValid === false) {\n            delete config._isValid;\n        } else {\n            return;\n        }\n\n        configFromRFC2822(config);\n        if (config._isValid === false) {\n            delete config._isValid;\n        } else {\n            return;\n        }\n\n        // Final attempt, use Input Fallback\n        hooks.createFromInputFallback(config);\n    }\n\n    hooks.createFromInputFallback = deprecate(\n        'value provided is not in a recognized RFC2822 or ISO format. moment construction falls back to js Date(), ' +\n        'which is not reliable across all browsers and versions. Non RFC2822/ISO date formats are ' +\n        'discouraged and will be removed in an upcoming major release. Please refer to ' +\n        'http://momentjs.com/guides/#/warnings/js-date/ for more info.',\n        function (config) {\n            config._d = new Date(config._i + (config._useUTC ? ' UTC' : ''));\n        }\n    );\n\n    // constant that refers to the ISO standard\n    hooks.ISO_8601 = function () {};\n\n    // constant that refers to the RFC 2822 form\n    hooks.RFC_2822 = function () {};\n\n    // date from string and format string\n    function configFromStringAndFormat(config) {\n        // TODO: Move this to another part of the creation flow to prevent circular deps\n        if (config._f === hooks.ISO_8601) {\n            configFromISO(config);\n            return;\n        }\n        if (config._f === hooks.RFC_2822) {\n            configFromRFC2822(config);\n            return;\n        }\n        config._a = [];\n        getParsingFlags(config).empty = true;\n\n        // This array is used to make a Date, either with `new Date` or `Date.UTC`\n        var string = '' + config._i,\n            i, parsedInput, tokens, token, skipped,\n            stringLength = string.length,\n            totalParsedInputLength = 0;\n\n        tokens = expandFormat(config._f, config._locale).match(formattingTokens) || [];\n\n        for (i = 0; i < tokens.length; i++) {\n            token = tokens[i];\n            parsedInput = (string.match(getParseRegexForToken(token, config)) || [])[0];\n            // console.log('token', token, 'parsedInput', parsedInput,\n            //         'regex', getParseRegexForToken(token, config));\n            if (parsedInput) {\n                skipped = string.substr(0, string.indexOf(parsedInput));\n                if (skipped.length > 0) {\n                    getParsingFlags(config).unusedInput.push(skipped);\n                }\n                string = string.slice(string.indexOf(parsedInput) + parsedInput.length);\n                totalParsedInputLength += parsedInput.length;\n            }\n            // don't parse if it's not a known token\n            if (formatTokenFunctions[token]) {\n                if (parsedInput) {\n                    getParsingFlags(config).empty = false;\n                }\n                else {\n                    getParsingFlags(config).unusedTokens.push(token);\n                }\n                addTimeToArrayFromToken(token, parsedInput, config);\n            }\n            else if (config._strict && !parsedInput) {\n                getParsingFlags(config).unusedTokens.push(token);\n            }\n        }\n\n        // add remaining unparsed input length to the string\n        getParsingFlags(config).charsLeftOver = stringLength - totalParsedInputLength;\n        if (string.length > 0) {\n            getParsingFlags(config).unusedInput.push(string);\n        }\n\n        // clear _12h flag if hour is <= 12\n        if (config._a[HOUR] <= 12 &&\n            getParsingFlags(config).bigHour === true &&\n            config._a[HOUR] > 0) {\n            getParsingFlags(config).bigHour = undefined;\n        }\n\n        getParsingFlags(config).parsedDateParts = config._a.slice(0);\n        getParsingFlags(config).meridiem = config._meridiem;\n        // handle meridiem\n        config._a[HOUR] = meridiemFixWrap(config._locale, config._a[HOUR], config._meridiem);\n\n        configFromArray(config);\n        checkOverflow(config);\n    }\n\n\n    function meridiemFixWrap (locale, hour, meridiem) {\n        var isPm;\n\n        if (meridiem == null) {\n            // nothing to do\n            return hour;\n        }\n        if (locale.meridiemHour != null) {\n            return locale.meridiemHour(hour, meridiem);\n        } else if (locale.isPM != null) {\n            // Fallback\n            isPm = locale.isPM(meridiem);\n            if (isPm && hour < 12) {\n                hour += 12;\n            }\n            if (!isPm && hour === 12) {\n                hour = 0;\n            }\n            return hour;\n        } else {\n            // this is not supposed to happen\n            return hour;\n        }\n    }\n\n    // date from string and array of format strings\n    function configFromStringAndArray(config) {\n        var tempConfig,\n            bestMoment,\n\n            scoreToBeat,\n            i,\n            currentScore;\n\n        if (config._f.length === 0) {\n            getParsingFlags(config).invalidFormat = true;\n            config._d = new Date(NaN);\n            return;\n        }\n\n        for (i = 0; i < config._f.length; i++) {\n            currentScore = 0;\n            tempConfig = copyConfig({}, config);\n            if (config._useUTC != null) {\n                tempConfig._useUTC = config._useUTC;\n            }\n            tempConfig._f = config._f[i];\n            configFromStringAndFormat(tempConfig);\n\n            if (!isValid(tempConfig)) {\n                continue;\n            }\n\n            // if there is any input that was not parsed add a penalty for that format\n            currentScore += getParsingFlags(tempConfig).charsLeftOver;\n\n            //or tokens\n            currentScore += getParsingFlags(tempConfig).unusedTokens.length * 10;\n\n            getParsingFlags(tempConfig).score = currentScore;\n\n            if (scoreToBeat == null || currentScore < scoreToBeat) {\n                scoreToBeat = currentScore;\n                bestMoment = tempConfig;\n            }\n        }\n\n        extend(config, bestMoment || tempConfig);\n    }\n\n    function configFromObject(config) {\n        if (config._d) {\n            return;\n        }\n\n        var i = normalizeObjectUnits(config._i);\n        config._a = map([i.year, i.month, i.day || i.date, i.hour, i.minute, i.second, i.millisecond], function (obj) {\n            return obj && parseInt(obj, 10);\n        });\n\n        configFromArray(config);\n    }\n\n    function createFromConfig (config) {\n        var res = new Moment(checkOverflow(prepareConfig(config)));\n        if (res._nextDay) {\n            // Adding is smart enough around DST\n            res.add(1, 'd');\n            res._nextDay = undefined;\n        }\n\n        return res;\n    }\n\n    function prepareConfig (config) {\n        var input = config._i,\n            format = config._f;\n\n        config._locale = config._locale || getLocale(config._l);\n\n        if (input === null || (format === undefined && input === '')) {\n            return createInvalid({nullInput: true});\n        }\n\n        if (typeof input === 'string') {\n            config._i = input = config._locale.preparse(input);\n        }\n\n        if (isMoment(input)) {\n            return new Moment(checkOverflow(input));\n        } else if (isDate(input)) {\n            config._d = input;\n        } else if (isArray(format)) {\n            configFromStringAndArray(config);\n        } else if (format) {\n            configFromStringAndFormat(config);\n        }  else {\n            configFromInput(config);\n        }\n\n        if (!isValid(config)) {\n            config._d = null;\n        }\n\n        return config;\n    }\n\n    function configFromInput(config) {\n        var input = config._i;\n        if (isUndefined(input)) {\n            config._d = new Date(hooks.now());\n        } else if (isDate(input)) {\n            config._d = new Date(input.valueOf());\n        } else if (typeof input === 'string') {\n            configFromString(config);\n        } else if (isArray(input)) {\n            config._a = map(input.slice(0), function (obj) {\n                return parseInt(obj, 10);\n            });\n            configFromArray(config);\n        } else if (isObject(input)) {\n            configFromObject(config);\n        } else if (isNumber(input)) {\n            // from milliseconds\n            config._d = new Date(input);\n        } else {\n            hooks.createFromInputFallback(config);\n        }\n    }\n\n    function createLocalOrUTC (input, format, locale, strict, isUTC) {\n        var c = {};\n\n        if (locale === true || locale === false) {\n            strict = locale;\n            locale = undefined;\n        }\n\n        if ((isObject(input) && isObjectEmpty(input)) ||\n                (isArray(input) && input.length === 0)) {\n            input = undefined;\n        }\n        // object construction must be done this way.\n        // https://github.com/moment/moment/issues/1423\n        c._isAMomentObject = true;\n        c._useUTC = c._isUTC = isUTC;\n        c._l = locale;\n        c._i = input;\n        c._f = format;\n        c._strict = strict;\n\n        return createFromConfig(c);\n    }\n\n    function createLocal (input, format, locale, strict) {\n        return createLocalOrUTC(input, format, locale, strict, false);\n    }\n\n    var prototypeMin = deprecate(\n        'moment().min is deprecated, use moment.max instead. http://momentjs.com/guides/#/warnings/min-max/',\n        function () {\n            var other = createLocal.apply(null, arguments);\n            if (this.isValid() && other.isValid()) {\n                return other < this ? this : other;\n            } else {\n                return createInvalid();\n            }\n        }\n    );\n\n    var prototypeMax = deprecate(\n        'moment().max is deprecated, use moment.min instead. http://momentjs.com/guides/#/warnings/min-max/',\n        function () {\n            var other = createLocal.apply(null, arguments);\n            if (this.isValid() && other.isValid()) {\n                return other > this ? this : other;\n            } else {\n                return createInvalid();\n            }\n        }\n    );\n\n    // Pick a moment m from moments so that m[fn](other) is true for all\n    // other. This relies on the function fn to be transitive.\n    //\n    // moments should either be an array of moment objects or an array, whose\n    // first element is an array of moment objects.\n    function pickBy(fn, moments) {\n        var res, i;\n        if (moments.length === 1 && isArray(moments[0])) {\n            moments = moments[0];\n        }\n        if (!moments.length) {\n            return createLocal();\n        }\n        res = moments[0];\n        for (i = 1; i < moments.length; ++i) {\n            if (!moments[i].isValid() || moments[i][fn](res)) {\n                res = moments[i];\n            }\n        }\n        return res;\n    }\n\n    // TODO: Use [].sort instead?\n    function min () {\n        var args = [].slice.call(arguments, 0);\n\n        return pickBy('isBefore', args);\n    }\n\n    function max () {\n        var args = [].slice.call(arguments, 0);\n\n        return pickBy('isAfter', args);\n    }\n\n    var now = function () {\n        return Date.now ? Date.now() : +(new Date());\n    };\n\n    var ordering = ['year', 'quarter', 'month', 'week', 'day', 'hour', 'minute', 'second', 'millisecond'];\n\n    function isDurationValid(m) {\n        for (var key in m) {\n            if (!(indexOf.call(ordering, key) !== -1 && (m[key] == null || !isNaN(m[key])))) {\n                return false;\n            }\n        }\n\n        var unitHasDecimal = false;\n        for (var i = 0; i < ordering.length; ++i) {\n            if (m[ordering[i]]) {\n                if (unitHasDecimal) {\n                    return false; // only allow non-integers for smallest unit\n                }\n                if (parseFloat(m[ordering[i]]) !== toInt(m[ordering[i]])) {\n                    unitHasDecimal = true;\n                }\n            }\n        }\n\n        return true;\n    }\n\n    function isValid$1() {\n        return this._isValid;\n    }\n\n    function createInvalid$1() {\n        return createDuration(NaN);\n    }\n\n    function Duration (duration) {\n        var normalizedInput = normalizeObjectUnits(duration),\n            years = normalizedInput.year || 0,\n            quarters = normalizedInput.quarter || 0,\n            months = normalizedInput.month || 0,\n            weeks = normalizedInput.week || normalizedInput.isoWeek || 0,\n            days = normalizedInput.day || 0,\n            hours = normalizedInput.hour || 0,\n            minutes = normalizedInput.minute || 0,\n            seconds = normalizedInput.second || 0,\n            milliseconds = normalizedInput.millisecond || 0;\n\n        this._isValid = isDurationValid(normalizedInput);\n\n        // representation for dateAddRemove\n        this._milliseconds = +milliseconds +\n            seconds * 1e3 + // 1000\n            minutes * 6e4 + // 1000 * 60\n            hours * 1000 * 60 * 60; //using 1000 * 60 * 60 instead of 36e5 to avoid floating point rounding errors https://github.com/moment/moment/issues/2978\n        // Because of dateAddRemove treats 24 hours as different from a\n        // day when working around DST, we need to store them separately\n        this._days = +days +\n            weeks * 7;\n        // It is impossible to translate months into days without knowing\n        // which months you are are talking about, so we have to store\n        // it separately.\n        this._months = +months +\n            quarters * 3 +\n            years * 12;\n\n        this._data = {};\n\n        this._locale = getLocale();\n\n        this._bubble();\n    }\n\n    function isDuration (obj) {\n        return obj instanceof Duration;\n    }\n\n    function absRound (number) {\n        if (number < 0) {\n            return Math.round(-1 * number) * -1;\n        } else {\n            return Math.round(number);\n        }\n    }\n\n    // FORMATTING\n\n    function offset (token, separator) {\n        addFormatToken(token, 0, 0, function () {\n            var offset = this.utcOffset();\n            var sign = '+';\n            if (offset < 0) {\n                offset = -offset;\n                sign = '-';\n            }\n            return sign + zeroFill(~~(offset / 60), 2) + separator + zeroFill(~~(offset) % 60, 2);\n        });\n    }\n\n    offset('Z', ':');\n    offset('ZZ', '');\n\n    // PARSING\n\n    addRegexToken('Z',  matchShortOffset);\n    addRegexToken('ZZ', matchShortOffset);\n    addParseToken(['Z', 'ZZ'], function (input, array, config) {\n        config._useUTC = true;\n        config._tzm = offsetFromString(matchShortOffset, input);\n    });\n\n    // HELPERS\n\n    // timezone chunker\n    // '+10:00' > ['10',  '00']\n    // '-1530'  > ['-15', '30']\n    var chunkOffset = /([\\+\\-]|\\d\\d)/gi;\n\n    function offsetFromString(matcher, string) {\n        var matches = (string || '').match(matcher);\n\n        if (matches === null) {\n            return null;\n        }\n\n        var chunk   = matches[matches.length - 1] || [];\n        var parts   = (chunk + '').match(chunkOffset) || ['-', 0, 0];\n        var minutes = +(parts[1] * 60) + toInt(parts[2]);\n\n        return minutes === 0 ?\n          0 :\n          parts[0] === '+' ? minutes : -minutes;\n    }\n\n    // Return a moment from input, that is local/utc/zone equivalent to model.\n    function cloneWithOffset(input, model) {\n        var res, diff;\n        if (model._isUTC) {\n            res = model.clone();\n            diff = (isMoment(input) || isDate(input) ? input.valueOf() : createLocal(input).valueOf()) - res.valueOf();\n            // Use low-level api, because this fn is low-level api.\n            res._d.setTime(res._d.valueOf() + diff);\n            hooks.updateOffset(res, false);\n            return res;\n        } else {\n            return createLocal(input).local();\n        }\n    }\n\n    function getDateOffset (m) {\n        // On Firefox.24 Date#getTimezoneOffset returns a floating point.\n        // https://github.com/moment/moment/pull/1871\n        return -Math.round(m._d.getTimezoneOffset() / 15) * 15;\n    }\n\n    // HOOKS\n\n    // This function will be called whenever a moment is mutated.\n    // It is intended to keep the offset in sync with the timezone.\n    hooks.updateOffset = function () {};\n\n    // MOMENTS\n\n    // keepLocalTime = true means only change the timezone, without\n    // affecting the local hour. So 5:31:26 +0300 --[utcOffset(2, true)]-->\n    // 5:31:26 +0200 It is possible that 5:31:26 doesn't exist with offset\n    // +0200, so we adjust the time as needed, to be valid.\n    //\n    // Keeping the time actually adds/subtracts (one hour)\n    // from the actual represented time. That is why we call updateOffset\n    // a second time. In case it wants us to change the offset again\n    // _changeInProgress == true case, then we have to adjust, because\n    // there is no such time in the given timezone.\n    function getSetOffset (input, keepLocalTime, keepMinutes) {\n        var offset = this._offset || 0,\n            localAdjust;\n        if (!this.isValid()) {\n            return input != null ? this : NaN;\n        }\n        if (input != null) {\n            if (typeof input === 'string') {\n                input = offsetFromString(matchShortOffset, input);\n                if (input === null) {\n                    return this;\n                }\n            } else if (Math.abs(input) < 16 && !keepMinutes) {\n                input = input * 60;\n            }\n            if (!this._isUTC && keepLocalTime) {\n                localAdjust = getDateOffset(this);\n            }\n            this._offset = input;\n            this._isUTC = true;\n            if (localAdjust != null) {\n                this.add(localAdjust, 'm');\n            }\n            if (offset !== input) {\n                if (!keepLocalTime || this._changeInProgress) {\n                    addSubtract(this, createDuration(input - offset, 'm'), 1, false);\n                } else if (!this._changeInProgress) {\n                    this._changeInProgress = true;\n                    hooks.updateOffset(this, true);\n                    this._changeInProgress = null;\n                }\n            }\n            return this;\n        } else {\n            return this._isUTC ? offset : getDateOffset(this);\n        }\n    }\n\n    function getSetZone (input, keepLocalTime) {\n        if (input != null) {\n            if (typeof input !== 'string') {\n                input = -input;\n            }\n\n            this.utcOffset(input, keepLocalTime);\n\n            return this;\n        } else {\n            return -this.utcOffset();\n        }\n    }\n\n    function setOffsetToUTC (keepLocalTime) {\n        return this.utcOffset(0, keepLocalTime);\n    }\n\n    function setOffsetToLocal (keepLocalTime) {\n        if (this._isUTC) {\n            this.utcOffset(0, keepLocalTime);\n            this._isUTC = false;\n\n            if (keepLocalTime) {\n                this.subtract(getDateOffset(this), 'm');\n            }\n        }\n        return this;\n    }\n\n    function setOffsetToParsedOffset () {\n        if (this._tzm != null) {\n            this.utcOffset(this._tzm, false, true);\n        } else if (typeof this._i === 'string') {\n            var tZone = offsetFromString(matchOffset, this._i);\n            if (tZone != null) {\n                this.utcOffset(tZone);\n            }\n            else {\n                this.utcOffset(0, true);\n            }\n        }\n        return this;\n    }\n\n    function hasAlignedHourOffset (input) {\n        if (!this.isValid()) {\n            return false;\n        }\n        input = input ? createLocal(input).utcOffset() : 0;\n\n        return (this.utcOffset() - input) % 60 === 0;\n    }\n\n    function isDaylightSavingTime () {\n        return (\n            this.utcOffset() > this.clone().month(0).utcOffset() ||\n            this.utcOffset() > this.clone().month(5).utcOffset()\n        );\n    }\n\n    function isDaylightSavingTimeShifted () {\n        if (!isUndefined(this._isDSTShifted)) {\n            return this._isDSTShifted;\n        }\n\n        var c = {};\n\n        copyConfig(c, this);\n        c = prepareConfig(c);\n\n        if (c._a) {\n            var other = c._isUTC ? createUTC(c._a) : createLocal(c._a);\n            this._isDSTShifted = this.isValid() &&\n                compareArrays(c._a, other.toArray()) > 0;\n        } else {\n            this._isDSTShifted = false;\n        }\n\n        return this._isDSTShifted;\n    }\n\n    function isLocal () {\n        return this.isValid() ? !this._isUTC : false;\n    }\n\n    function isUtcOffset () {\n        return this.isValid() ? this._isUTC : false;\n    }\n\n    function isUtc () {\n        return this.isValid() ? this._isUTC && this._offset === 0 : false;\n    }\n\n    // ASP.NET json date format regex\n    var aspNetRegex = /^(\\-|\\+)?(?:(\\d*)[. ])?(\\d+)\\:(\\d+)(?:\\:(\\d+)(\\.\\d*)?)?$/;\n\n    // from http://docs.closure-library.googlecode.com/git/closure_goog_date_date.js.source.html\n    // somewhat more in line with 4.4.3.2 2004 spec, but allows decimal anywhere\n    // and further modified to allow for strings containing both week and day\n    var isoRegex = /^(-|\\+)?P(?:([-+]?[0-9,.]*)Y)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)W)?(?:([-+]?[0-9,.]*)D)?(?:T(?:([-+]?[0-9,.]*)H)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)S)?)?$/;\n\n    function createDuration (input, key) {\n        var duration = input,\n            // matching against regexp is expensive, do it on demand\n            match = null,\n            sign,\n            ret,\n            diffRes;\n\n        if (isDuration(input)) {\n            duration = {\n                ms : input._milliseconds,\n                d  : input._days,\n                M  : input._months\n            };\n        } else if (isNumber(input)) {\n            duration = {};\n            if (key) {\n                duration[key] = input;\n            } else {\n                duration.milliseconds = input;\n            }\n        } else if (!!(match = aspNetRegex.exec(input))) {\n            sign = (match[1] === '-') ? -1 : 1;\n            duration = {\n                y  : 0,\n                d  : toInt(match[DATE])                         * sign,\n                h  : toInt(match[HOUR])                         * sign,\n                m  : toInt(match[MINUTE])                       * sign,\n                s  : toInt(match[SECOND])                       * sign,\n                ms : toInt(absRound(match[MILLISECOND] * 1000)) * sign // the millisecond decimal point is included in the match\n            };\n        } else if (!!(match = isoRegex.exec(input))) {\n            sign = (match[1] === '-') ? -1 : 1;\n            duration = {\n                y : parseIso(match[2], sign),\n                M : parseIso(match[3], sign),\n                w : parseIso(match[4], sign),\n                d : parseIso(match[5], sign),\n                h : parseIso(match[6], sign),\n                m : parseIso(match[7], sign),\n                s : parseIso(match[8], sign)\n            };\n        } else if (duration == null) {// checks for null or undefined\n            duration = {};\n        } else if (typeof duration === 'object' && ('from' in duration || 'to' in duration)) {\n            diffRes = momentsDifference(createLocal(duration.from), createLocal(duration.to));\n\n            duration = {};\n            duration.ms = diffRes.milliseconds;\n            duration.M = diffRes.months;\n        }\n\n        ret = new Duration(duration);\n\n        if (isDuration(input) && hasOwnProp(input, '_locale')) {\n            ret._locale = input._locale;\n        }\n\n        return ret;\n    }\n\n    createDuration.fn = Duration.prototype;\n    createDuration.invalid = createInvalid$1;\n\n    function parseIso (inp, sign) {\n        // We'd normally use ~~inp for this, but unfortunately it also\n        // converts floats to ints.\n        // inp may be undefined, so careful calling replace on it.\n        var res = inp && parseFloat(inp.replace(',', '.'));\n        // apply sign while we're at it\n        return (isNaN(res) ? 0 : res) * sign;\n    }\n\n    function positiveMomentsDifference(base, other) {\n        var res = {};\n\n        res.months = other.month() - base.month() +\n            (other.year() - base.year()) * 12;\n        if (base.clone().add(res.months, 'M').isAfter(other)) {\n            --res.months;\n        }\n\n        res.milliseconds = +other - +(base.clone().add(res.months, 'M'));\n\n        return res;\n    }\n\n    function momentsDifference(base, other) {\n        var res;\n        if (!(base.isValid() && other.isValid())) {\n            return {milliseconds: 0, months: 0};\n        }\n\n        other = cloneWithOffset(other, base);\n        if (base.isBefore(other)) {\n            res = positiveMomentsDifference(base, other);\n        } else {\n            res = positiveMomentsDifference(other, base);\n            res.milliseconds = -res.milliseconds;\n            res.months = -res.months;\n        }\n\n        return res;\n    }\n\n    // TODO: remove 'name' arg after deprecation is removed\n    function createAdder(direction, name) {\n        return function (val, period) {\n            var dur, tmp;\n            //invert the arguments, but complain about it\n            if (period !== null && !isNaN(+period)) {\n                deprecateSimple(name, 'moment().' + name  + '(period, number) is deprecated. Please use moment().' + name + '(number, period). ' +\n                'See http://momentjs.com/guides/#/warnings/add-inverted-param/ for more info.');\n                tmp = val; val = period; period = tmp;\n            }\n\n            val = typeof val === 'string' ? +val : val;\n            dur = createDuration(val, period);\n            addSubtract(this, dur, direction);\n            return this;\n        };\n    }\n\n    function addSubtract (mom, duration, isAdding, updateOffset) {\n        var milliseconds = duration._milliseconds,\n            days = absRound(duration._days),\n            months = absRound(duration._months);\n\n        if (!mom.isValid()) {\n            // No op\n            return;\n        }\n\n        updateOffset = updateOffset == null ? true : updateOffset;\n\n        if (months) {\n            setMonth(mom, get(mom, 'Month') + months * isAdding);\n        }\n        if (days) {\n            set$1(mom, 'Date', get(mom, 'Date') + days * isAdding);\n        }\n        if (milliseconds) {\n            mom._d.setTime(mom._d.valueOf() + milliseconds * isAdding);\n        }\n        if (updateOffset) {\n            hooks.updateOffset(mom, days || months);\n        }\n    }\n\n    var add      = createAdder(1, 'add');\n    var subtract = createAdder(-1, 'subtract');\n\n    function getCalendarFormat(myMoment, now) {\n        var diff = myMoment.diff(now, 'days', true);\n        return diff < -6 ? 'sameElse' :\n                diff < -1 ? 'lastWeek' :\n                diff < 0 ? 'lastDay' :\n                diff < 1 ? 'sameDay' :\n                diff < 2 ? 'nextDay' :\n                diff < 7 ? 'nextWeek' : 'sameElse';\n    }\n\n    function calendar$1 (time, formats) {\n        // We want to compare the start of today, vs this.\n        // Getting start-of-today depends on whether we're local/utc/offset or not.\n        var now = time || createLocal(),\n            sod = cloneWithOffset(now, this).startOf('day'),\n            format = hooks.calendarFormat(this, sod) || 'sameElse';\n\n        var output = formats && (isFunction(formats[format]) ? formats[format].call(this, now) : formats[format]);\n\n        return this.format(output || this.localeData().calendar(format, this, createLocal(now)));\n    }\n\n    function clone () {\n        return new Moment(this);\n    }\n\n    function isAfter (input, units) {\n        var localInput = isMoment(input) ? input : createLocal(input);\n        if (!(this.isValid() && localInput.isValid())) {\n            return false;\n        }\n        units = normalizeUnits(units) || 'millisecond';\n        if (units === 'millisecond') {\n            return this.valueOf() > localInput.valueOf();\n        } else {\n            return localInput.valueOf() < this.clone().startOf(units).valueOf();\n        }\n    }\n\n    function isBefore (input, units) {\n        var localInput = isMoment(input) ? input : createLocal(input);\n        if (!(this.isValid() && localInput.isValid())) {\n            return false;\n        }\n        units = normalizeUnits(units) || 'millisecond';\n        if (units === 'millisecond') {\n            return this.valueOf() < localInput.valueOf();\n        } else {\n            return this.clone().endOf(units).valueOf() < localInput.valueOf();\n        }\n    }\n\n    function isBetween (from, to, units, inclusivity) {\n        var localFrom = isMoment(from) ? from : createLocal(from),\n            localTo = isMoment(to) ? to : createLocal(to);\n        if (!(this.isValid() && localFrom.isValid() && localTo.isValid())) {\n            return false;\n        }\n        inclusivity = inclusivity || '()';\n        return (inclusivity[0] === '(' ? this.isAfter(localFrom, units) : !this.isBefore(localFrom, units)) &&\n            (inclusivity[1] === ')' ? this.isBefore(localTo, units) : !this.isAfter(localTo, units));\n    }\n\n    function isSame (input, units) {\n        var localInput = isMoment(input) ? input : createLocal(input),\n            inputMs;\n        if (!(this.isValid() && localInput.isValid())) {\n            return false;\n        }\n        units = normalizeUnits(units) || 'millisecond';\n        if (units === 'millisecond') {\n            return this.valueOf() === localInput.valueOf();\n        } else {\n            inputMs = localInput.valueOf();\n            return this.clone().startOf(units).valueOf() <= inputMs && inputMs <= this.clone().endOf(units).valueOf();\n        }\n    }\n\n    function isSameOrAfter (input, units) {\n        return this.isSame(input, units) || this.isAfter(input, units);\n    }\n\n    function isSameOrBefore (input, units) {\n        return this.isSame(input, units) || this.isBefore(input, units);\n    }\n\n    function diff (input, units, asFloat) {\n        var that,\n            zoneDelta,\n            output;\n\n        if (!this.isValid()) {\n            return NaN;\n        }\n\n        that = cloneWithOffset(input, this);\n\n        if (!that.isValid()) {\n            return NaN;\n        }\n\n        zoneDelta = (that.utcOffset() - this.utcOffset()) * 6e4;\n\n        units = normalizeUnits(units);\n\n        switch (units) {\n            case 'year': output = monthDiff(this, that) / 12; break;\n            case 'month': output = monthDiff(this, that); break;\n            case 'quarter': output = monthDiff(this, that) / 3; break;\n            case 'second': output = (this - that) / 1e3; break; // 1000\n            case 'minute': output = (this - that) / 6e4; break; // 1000 * 60\n            case 'hour': output = (this - that) / 36e5; break; // 1000 * 60 * 60\n            case 'day': output = (this - that - zoneDelta) / 864e5; break; // 1000 * 60 * 60 * 24, negate dst\n            case 'week': output = (this - that - zoneDelta) / 6048e5; break; // 1000 * 60 * 60 * 24 * 7, negate dst\n            default: output = this - that;\n        }\n\n        return asFloat ? output : absFloor(output);\n    }\n\n    function monthDiff (a, b) {\n        // difference in months\n        var wholeMonthDiff = ((b.year() - a.year()) * 12) + (b.month() - a.month()),\n            // b is in (anchor - 1 month, anchor + 1 month)\n            anchor = a.clone().add(wholeMonthDiff, 'months'),\n            anchor2, adjust;\n\n        if (b - anchor < 0) {\n            anchor2 = a.clone().add(wholeMonthDiff - 1, 'months');\n            // linear across the month\n            adjust = (b - anchor) / (anchor - anchor2);\n        } else {\n            anchor2 = a.clone().add(wholeMonthDiff + 1, 'months');\n            // linear across the month\n            adjust = (b - anchor) / (anchor2 - anchor);\n        }\n\n        //check for negative zero, return zero if negative zero\n        return -(wholeMonthDiff + adjust) || 0;\n    }\n\n    hooks.defaultFormat = 'YYYY-MM-DDTHH:mm:ssZ';\n    hooks.defaultFormatUtc = 'YYYY-MM-DDTHH:mm:ss[Z]';\n\n    function toString () {\n        return this.clone().locale('en').format('ddd MMM DD YYYY HH:mm:ss [GMT]ZZ');\n    }\n\n    function toISOString(keepOffset) {\n        if (!this.isValid()) {\n            return null;\n        }\n        var utc = keepOffset !== true;\n        var m = utc ? this.clone().utc() : this;\n        if (m.year() < 0 || m.year() > 9999) {\n            return formatMoment(m, utc ? 'YYYYYY-MM-DD[T]HH:mm:ss.SSS[Z]' : 'YYYYYY-MM-DD[T]HH:mm:ss.SSSZ');\n        }\n        if (isFunction(Date.prototype.toISOString)) {\n            // native implementation is ~50x faster, use it when we can\n            if (utc) {\n                return this.toDate().toISOString();\n            } else {\n                return new Date(this.valueOf() + this.utcOffset() * 60 * 1000).toISOString().replace('Z', formatMoment(m, 'Z'));\n            }\n        }\n        return formatMoment(m, utc ? 'YYYY-MM-DD[T]HH:mm:ss.SSS[Z]' : 'YYYY-MM-DD[T]HH:mm:ss.SSSZ');\n    }\n\n    /**\n     * Return a human readable representation of a moment that can\n     * also be evaluated to get a new moment which is the same\n     *\n     * @link https://nodejs.org/dist/latest/docs/api/util.html#util_custom_inspect_function_on_objects\n     */\n    function inspect () {\n        if (!this.isValid()) {\n            return 'moment.invalid(/* ' + this._i + ' */)';\n        }\n        var func = 'moment';\n        var zone = '';\n        if (!this.isLocal()) {\n            func = this.utcOffset() === 0 ? 'moment.utc' : 'moment.parseZone';\n            zone = 'Z';\n        }\n        var prefix = '[' + func + '(\"]';\n        var year = (0 <= this.year() && this.year() <= 9999) ? 'YYYY' : 'YYYYYY';\n        var datetime = '-MM-DD[T]HH:mm:ss.SSS';\n        var suffix = zone + '[\")]';\n\n        return this.format(prefix + year + datetime + suffix);\n    }\n\n    function format (inputString) {\n        if (!inputString) {\n            inputString = this.isUtc() ? hooks.defaultFormatUtc : hooks.defaultFormat;\n        }\n        var output = formatMoment(this, inputString);\n        return this.localeData().postformat(output);\n    }\n\n    function from (time, withoutSuffix) {\n        if (this.isValid() &&\n                ((isMoment(time) && time.isValid()) ||\n                 createLocal(time).isValid())) {\n            return createDuration({to: this, from: time}).locale(this.locale()).humanize(!withoutSuffix);\n        } else {\n            return this.localeData().invalidDate();\n        }\n    }\n\n    function fromNow (withoutSuffix) {\n        return this.from(createLocal(), withoutSuffix);\n    }\n\n    function to (time, withoutSuffix) {\n        if (this.isValid() &&\n                ((isMoment(time) && time.isValid()) ||\n                 createLocal(time).isValid())) {\n            return createDuration({from: this, to: time}).locale(this.locale()).humanize(!withoutSuffix);\n        } else {\n            return this.localeData().invalidDate();\n        }\n    }\n\n    function toNow (withoutSuffix) {\n        return this.to(createLocal(), withoutSuffix);\n    }\n\n    // If passed a locale key, it will set the locale for this\n    // instance.  Otherwise, it will return the locale configuration\n    // variables for this instance.\n    function locale (key) {\n        var newLocaleData;\n\n        if (key === undefined) {\n            return this._locale._abbr;\n        } else {\n            newLocaleData = getLocale(key);\n            if (newLocaleData != null) {\n                this._locale = newLocaleData;\n            }\n            return this;\n        }\n    }\n\n    var lang = deprecate(\n        'moment().lang() is deprecated. Instead, use moment().localeData() to get the language configuration. Use moment().locale() to change languages.',\n        function (key) {\n            if (key === undefined) {\n                return this.localeData();\n            } else {\n                return this.locale(key);\n            }\n        }\n    );\n\n    function localeData () {\n        return this._locale;\n    }\n\n    var MS_PER_SECOND = 1000;\n    var MS_PER_MINUTE = 60 * MS_PER_SECOND;\n    var MS_PER_HOUR = 60 * MS_PER_MINUTE;\n    var MS_PER_400_YEARS = (365 * 400 + 97) * 24 * MS_PER_HOUR;\n\n    // actual modulo - handles negative numbers (for dates before 1970):\n    function mod$1(dividend, divisor) {\n        return (dividend % divisor + divisor) % divisor;\n    }\n\n    function localStartOfDate(y, m, d) {\n        // the date constructor remaps years 0-99 to 1900-1999\n        if (y < 100 && y >= 0) {\n            // preserve leap years using a full 400 year cycle, then reset\n            return new Date(y + 400, m, d) - MS_PER_400_YEARS;\n        } else {\n            return new Date(y, m, d).valueOf();\n        }\n    }\n\n    function utcStartOfDate(y, m, d) {\n        // Date.UTC remaps years 0-99 to 1900-1999\n        if (y < 100 && y >= 0) {\n            // preserve leap years using a full 400 year cycle, then reset\n            return Date.UTC(y + 400, m, d) - MS_PER_400_YEARS;\n        } else {\n            return Date.UTC(y, m, d);\n        }\n    }\n\n    function startOf (units) {\n        var time;\n        units = normalizeUnits(units);\n        if (units === undefined || units === 'millisecond' || !this.isValid()) {\n            return this;\n        }\n\n        var startOfDate = this._isUTC ? utcStartOfDate : localStartOfDate;\n\n        switch (units) {\n            case 'year':\n                time = startOfDate(this.year(), 0, 1);\n                break;\n            case 'quarter':\n                time = startOfDate(this.year(), this.month() - this.month() % 3, 1);\n                break;\n            case 'month':\n                time = startOfDate(this.year(), this.month(), 1);\n                break;\n            case 'week':\n                time = startOfDate(this.year(), this.month(), this.date() - this.weekday());\n                break;\n            case 'isoWeek':\n                time = startOfDate(this.year(), this.month(), this.date() - (this.isoWeekday() - 1));\n                break;\n            case 'day':\n            case 'date':\n                time = startOfDate(this.year(), this.month(), this.date());\n                break;\n            case 'hour':\n                time = this._d.valueOf();\n                time -= mod$1(time + (this._isUTC ? 0 : this.utcOffset() * MS_PER_MINUTE), MS_PER_HOUR);\n                break;\n            case 'minute':\n                time = this._d.valueOf();\n                time -= mod$1(time, MS_PER_MINUTE);\n                break;\n            case 'second':\n                time = this._d.valueOf();\n                time -= mod$1(time, MS_PER_SECOND);\n                break;\n        }\n\n        this._d.setTime(time);\n        hooks.updateOffset(this, true);\n        return this;\n    }\n\n    function endOf (units) {\n        var time;\n        units = normalizeUnits(units);\n        if (units === undefined || units === 'millisecond' || !this.isValid()) {\n            return this;\n        }\n\n        var startOfDate = this._isUTC ? utcStartOfDate : localStartOfDate;\n\n        switch (units) {\n            case 'year':\n                time = startOfDate(this.year() + 1, 0, 1) - 1;\n                break;\n            case 'quarter':\n                time = startOfDate(this.year(), this.month() - this.month() % 3 + 3, 1) - 1;\n                break;\n            case 'month':\n                time = startOfDate(this.year(), this.month() + 1, 1) - 1;\n                break;\n            case 'week':\n                time = startOfDate(this.year(), this.month(), this.date() - this.weekday() + 7) - 1;\n                break;\n            case 'isoWeek':\n                time = startOfDate(this.year(), this.month(), this.date() - (this.isoWeekday() - 1) + 7) - 1;\n                break;\n            case 'day':\n            case 'date':\n                time = startOfDate(this.year(), this.month(), this.date() + 1) - 1;\n                break;\n            case 'hour':\n                time = this._d.valueOf();\n                time += MS_PER_HOUR - mod$1(time + (this._isUTC ? 0 : this.utcOffset() * MS_PER_MINUTE), MS_PER_HOUR) - 1;\n                break;\n            case 'minute':\n                time = this._d.valueOf();\n                time += MS_PER_MINUTE - mod$1(time, MS_PER_MINUTE) - 1;\n                break;\n            case 'second':\n                time = this._d.valueOf();\n                time += MS_PER_SECOND - mod$1(time, MS_PER_SECOND) - 1;\n                break;\n        }\n\n        this._d.setTime(time);\n        hooks.updateOffset(this, true);\n        return this;\n    }\n\n    function valueOf () {\n        return this._d.valueOf() - ((this._offset || 0) * 60000);\n    }\n\n    function unix () {\n        return Math.floor(this.valueOf() / 1000);\n    }\n\n    function toDate () {\n        return new Date(this.valueOf());\n    }\n\n    function toArray () {\n        var m = this;\n        return [m.year(), m.month(), m.date(), m.hour(), m.minute(), m.second(), m.millisecond()];\n    }\n\n    function toObject () {\n        var m = this;\n        return {\n            years: m.year(),\n            months: m.month(),\n            date: m.date(),\n            hours: m.hours(),\n            minutes: m.minutes(),\n            seconds: m.seconds(),\n            milliseconds: m.milliseconds()\n        };\n    }\n\n    function toJSON () {\n        // new Date(NaN).toJSON() === null\n        return this.isValid() ? this.toISOString() : null;\n    }\n\n    function isValid$2 () {\n        return isValid(this);\n    }\n\n    function parsingFlags () {\n        return extend({}, getParsingFlags(this));\n    }\n\n    function invalidAt () {\n        return getParsingFlags(this).overflow;\n    }\n\n    function creationData() {\n        return {\n            input: this._i,\n            format: this._f,\n            locale: this._locale,\n            isUTC: this._isUTC,\n            strict: this._strict\n        };\n    }\n\n    // FORMATTING\n\n    addFormatToken(0, ['gg', 2], 0, function () {\n        return this.weekYear() % 100;\n    });\n\n    addFormatToken(0, ['GG', 2], 0, function () {\n        return this.isoWeekYear() % 100;\n    });\n\n    function addWeekYearFormatToken (token, getter) {\n        addFormatToken(0, [token, token.length], 0, getter);\n    }\n\n    addWeekYearFormatToken('gggg',     'weekYear');\n    addWeekYearFormatToken('ggggg',    'weekYear');\n    addWeekYearFormatToken('GGGG',  'isoWeekYear');\n    addWeekYearFormatToken('GGGGG', 'isoWeekYear');\n\n    // ALIASES\n\n    addUnitAlias('weekYear', 'gg');\n    addUnitAlias('isoWeekYear', 'GG');\n\n    // PRIORITY\n\n    addUnitPriority('weekYear', 1);\n    addUnitPriority('isoWeekYear', 1);\n\n\n    // PARSING\n\n    addRegexToken('G',      matchSigned);\n    addRegexToken('g',      matchSigned);\n    addRegexToken('GG',     match1to2, match2);\n    addRegexToken('gg',     match1to2, match2);\n    addRegexToken('GGGG',   match1to4, match4);\n    addRegexToken('gggg',   match1to4, match4);\n    addRegexToken('GGGGG',  match1to6, match6);\n    addRegexToken('ggggg',  match1to6, match6);\n\n    addWeekParseToken(['gggg', 'ggggg', 'GGGG', 'GGGGG'], function (input, week, config, token) {\n        week[token.substr(0, 2)] = toInt(input);\n    });\n\n    addWeekParseToken(['gg', 'GG'], function (input, week, config, token) {\n        week[token] = hooks.parseTwoDigitYear(input);\n    });\n\n    // MOMENTS\n\n    function getSetWeekYear (input) {\n        return getSetWeekYearHelper.call(this,\n                input,\n                this.week(),\n                this.weekday(),\n                this.localeData()._week.dow,\n                this.localeData()._week.doy);\n    }\n\n    function getSetISOWeekYear (input) {\n        return getSetWeekYearHelper.call(this,\n                input, this.isoWeek(), this.isoWeekday(), 1, 4);\n    }\n\n    function getISOWeeksInYear () {\n        return weeksInYear(this.year(), 1, 4);\n    }\n\n    function getWeeksInYear () {\n        var weekInfo = this.localeData()._week;\n        return weeksInYear(this.year(), weekInfo.dow, weekInfo.doy);\n    }\n\n    function getSetWeekYearHelper(input, week, weekday, dow, doy) {\n        var weeksTarget;\n        if (input == null) {\n            return weekOfYear(this, dow, doy).year;\n        } else {\n            weeksTarget = weeksInYear(input, dow, doy);\n            if (week > weeksTarget) {\n                week = weeksTarget;\n            }\n            return setWeekAll.call(this, input, week, weekday, dow, doy);\n        }\n    }\n\n    function setWeekAll(weekYear, week, weekday, dow, doy) {\n        var dayOfYearData = dayOfYearFromWeeks(weekYear, week, weekday, dow, doy),\n            date = createUTCDate(dayOfYearData.year, 0, dayOfYearData.dayOfYear);\n\n        this.year(date.getUTCFullYear());\n        this.month(date.getUTCMonth());\n        this.date(date.getUTCDate());\n        return this;\n    }\n\n    // FORMATTING\n\n    addFormatToken('Q', 0, 'Qo', 'quarter');\n\n    // ALIASES\n\n    addUnitAlias('quarter', 'Q');\n\n    // PRIORITY\n\n    addUnitPriority('quarter', 7);\n\n    // PARSING\n\n    addRegexToken('Q', match1);\n    addParseToken('Q', function (input, array) {\n        array[MONTH] = (toInt(input) - 1) * 3;\n    });\n\n    // MOMENTS\n\n    function getSetQuarter (input) {\n        return input == null ? Math.ceil((this.month() + 1) / 3) : this.month((input - 1) * 3 + this.month() % 3);\n    }\n\n    // FORMATTING\n\n    addFormatToken('D', ['DD', 2], 'Do', 'date');\n\n    // ALIASES\n\n    addUnitAlias('date', 'D');\n\n    // PRIORITY\n    addUnitPriority('date', 9);\n\n    // PARSING\n\n    addRegexToken('D',  match1to2);\n    addRegexToken('DD', match1to2, match2);\n    addRegexToken('Do', function (isStrict, locale) {\n        // TODO: Remove \"ordinalParse\" fallback in next major release.\n        return isStrict ?\n          (locale._dayOfMonthOrdinalParse || locale._ordinalParse) :\n          locale._dayOfMonthOrdinalParseLenient;\n    });\n\n    addParseToken(['D', 'DD'], DATE);\n    addParseToken('Do', function (input, array) {\n        array[DATE] = toInt(input.match(match1to2)[0]);\n    });\n\n    // MOMENTS\n\n    var getSetDayOfMonth = makeGetSet('Date', true);\n\n    // FORMATTING\n\n    addFormatToken('DDD', ['DDDD', 3], 'DDDo', 'dayOfYear');\n\n    // ALIASES\n\n    addUnitAlias('dayOfYear', 'DDD');\n\n    // PRIORITY\n    addUnitPriority('dayOfYear', 4);\n\n    // PARSING\n\n    addRegexToken('DDD',  match1to3);\n    addRegexToken('DDDD', match3);\n    addParseToken(['DDD', 'DDDD'], function (input, array, config) {\n        config._dayOfYear = toInt(input);\n    });\n\n    // HELPERS\n\n    // MOMENTS\n\n    function getSetDayOfYear (input) {\n        var dayOfYear = Math.round((this.clone().startOf('day') - this.clone().startOf('year')) / 864e5) + 1;\n        return input == null ? dayOfYear : this.add((input - dayOfYear), 'd');\n    }\n\n    // FORMATTING\n\n    addFormatToken('m', ['mm', 2], 0, 'minute');\n\n    // ALIASES\n\n    addUnitAlias('minute', 'm');\n\n    // PRIORITY\n\n    addUnitPriority('minute', 14);\n\n    // PARSING\n\n    addRegexToken('m',  match1to2);\n    addRegexToken('mm', match1to2, match2);\n    addParseToken(['m', 'mm'], MINUTE);\n\n    // MOMENTS\n\n    var getSetMinute = makeGetSet('Minutes', false);\n\n    // FORMATTING\n\n    addFormatToken('s', ['ss', 2], 0, 'second');\n\n    // ALIASES\n\n    addUnitAlias('second', 's');\n\n    // PRIORITY\n\n    addUnitPriority('second', 15);\n\n    // PARSING\n\n    addRegexToken('s',  match1to2);\n    addRegexToken('ss', match1to2, match2);\n    addParseToken(['s', 'ss'], SECOND);\n\n    // MOMENTS\n\n    var getSetSecond = makeGetSet('Seconds', false);\n\n    // FORMATTING\n\n    addFormatToken('S', 0, 0, function () {\n        return ~~(this.millisecond() / 100);\n    });\n\n    addFormatToken(0, ['SS', 2], 0, function () {\n        return ~~(this.millisecond() / 10);\n    });\n\n    addFormatToken(0, ['SSS', 3], 0, 'millisecond');\n    addFormatToken(0, ['SSSS', 4], 0, function () {\n        return this.millisecond() * 10;\n    });\n    addFormatToken(0, ['SSSSS', 5], 0, function () {\n        return this.millisecond() * 100;\n    });\n    addFormatToken(0, ['SSSSSS', 6], 0, function () {\n        return this.millisecond() * 1000;\n    });\n    addFormatToken(0, ['SSSSSSS', 7], 0, function () {\n        return this.millisecond() * 10000;\n    });\n    addFormatToken(0, ['SSSSSSSS', 8], 0, function () {\n        return this.millisecond() * 100000;\n    });\n    addFormatToken(0, ['SSSSSSSSS', 9], 0, function () {\n        return this.millisecond() * 1000000;\n    });\n\n\n    // ALIASES\n\n    addUnitAlias('millisecond', 'ms');\n\n    // PRIORITY\n\n    addUnitPriority('millisecond', 16);\n\n    // PARSING\n\n    addRegexToken('S',    match1to3, match1);\n    addRegexToken('SS',   match1to3, match2);\n    addRegexToken('SSS',  match1to3, match3);\n\n    var token;\n    for (token = 'SSSS'; token.length <= 9; token += 'S') {\n        addRegexToken(token, matchUnsigned);\n    }\n\n    function parseMs(input, array) {\n        array[MILLISECOND] = toInt(('0.' + input) * 1000);\n    }\n\n    for (token = 'S'; token.length <= 9; token += 'S') {\n        addParseToken(token, parseMs);\n    }\n    // MOMENTS\n\n    var getSetMillisecond = makeGetSet('Milliseconds', false);\n\n    // FORMATTING\n\n    addFormatToken('z',  0, 0, 'zoneAbbr');\n    addFormatToken('zz', 0, 0, 'zoneName');\n\n    // MOMENTS\n\n    function getZoneAbbr () {\n        return this._isUTC ? 'UTC' : '';\n    }\n\n    function getZoneName () {\n        return this._isUTC ? 'Coordinated Universal Time' : '';\n    }\n\n    var proto = Moment.prototype;\n\n    proto.add               = add;\n    proto.calendar          = calendar$1;\n    proto.clone             = clone;\n    proto.diff              = diff;\n    proto.endOf             = endOf;\n    proto.format            = format;\n    proto.from              = from;\n    proto.fromNow           = fromNow;\n    proto.to                = to;\n    proto.toNow             = toNow;\n    proto.get               = stringGet;\n    proto.invalidAt         = invalidAt;\n    proto.isAfter           = isAfter;\n    proto.isBefore          = isBefore;\n    proto.isBetween         = isBetween;\n    proto.isSame            = isSame;\n    proto.isSameOrAfter     = isSameOrAfter;\n    proto.isSameOrBefore    = isSameOrBefore;\n    proto.isValid           = isValid$2;\n    proto.lang              = lang;\n    proto.locale            = locale;\n    proto.localeData        = localeData;\n    proto.max               = prototypeMax;\n    proto.min               = prototypeMin;\n    proto.parsingFlags      = parsingFlags;\n    proto.set               = stringSet;\n    proto.startOf           = startOf;\n    proto.subtract          = subtract;\n    proto.toArray           = toArray;\n    proto.toObject          = toObject;\n    proto.toDate            = toDate;\n    proto.toISOString       = toISOString;\n    proto.inspect           = inspect;\n    proto.toJSON            = toJSON;\n    proto.toString          = toString;\n    proto.unix              = unix;\n    proto.valueOf           = valueOf;\n    proto.creationData      = creationData;\n    proto.year       = getSetYear;\n    proto.isLeapYear = getIsLeapYear;\n    proto.weekYear    = getSetWeekYear;\n    proto.isoWeekYear = getSetISOWeekYear;\n    proto.quarter = proto.quarters = getSetQuarter;\n    proto.month       = getSetMonth;\n    proto.daysInMonth = getDaysInMonth;\n    proto.week           = proto.weeks        = getSetWeek;\n    proto.isoWeek        = proto.isoWeeks     = getSetISOWeek;\n    proto.weeksInYear    = getWeeksInYear;\n    proto.isoWeeksInYear = getISOWeeksInYear;\n    proto.date       = getSetDayOfMonth;\n    proto.day        = proto.days             = getSetDayOfWeek;\n    proto.weekday    = getSetLocaleDayOfWeek;\n    proto.isoWeekday = getSetISODayOfWeek;\n    proto.dayOfYear  = getSetDayOfYear;\n    proto.hour = proto.hours = getSetHour;\n    proto.minute = proto.minutes = getSetMinute;\n    proto.second = proto.seconds = getSetSecond;\n    proto.millisecond = proto.milliseconds = getSetMillisecond;\n    proto.utcOffset            = getSetOffset;\n    proto.utc                  = setOffsetToUTC;\n    proto.local                = setOffsetToLocal;\n    proto.parseZone            = setOffsetToParsedOffset;\n    proto.hasAlignedHourOffset = hasAlignedHourOffset;\n    proto.isDST                = isDaylightSavingTime;\n    proto.isLocal              = isLocal;\n    proto.isUtcOffset          = isUtcOffset;\n    proto.isUtc                = isUtc;\n    proto.isUTC                = isUtc;\n    proto.zoneAbbr = getZoneAbbr;\n    proto.zoneName = getZoneName;\n    proto.dates  = deprecate('dates accessor is deprecated. Use date instead.', getSetDayOfMonth);\n    proto.months = deprecate('months accessor is deprecated. Use month instead', getSetMonth);\n    proto.years  = deprecate('years accessor is deprecated. Use year instead', getSetYear);\n    proto.zone   = deprecate('moment().zone is deprecated, use moment().utcOffset instead. http://momentjs.com/guides/#/warnings/zone/', getSetZone);\n    proto.isDSTShifted = deprecate('isDSTShifted is deprecated. See http://momentjs.com/guides/#/warnings/dst-shifted/ for more information', isDaylightSavingTimeShifted);\n\n    function createUnix (input) {\n        return createLocal(input * 1000);\n    }\n\n    function createInZone () {\n        return createLocal.apply(null, arguments).parseZone();\n    }\n\n    function preParsePostFormat (string) {\n        return string;\n    }\n\n    var proto$1 = Locale.prototype;\n\n    proto$1.calendar        = calendar;\n    proto$1.longDateFormat  = longDateFormat;\n    proto$1.invalidDate     = invalidDate;\n    proto$1.ordinal         = ordinal;\n    proto$1.preparse        = preParsePostFormat;\n    proto$1.postformat      = preParsePostFormat;\n    proto$1.relativeTime    = relativeTime;\n    proto$1.pastFuture      = pastFuture;\n    proto$1.set             = set;\n\n    proto$1.months            =        localeMonths;\n    proto$1.monthsShort       =        localeMonthsShort;\n    proto$1.monthsParse       =        localeMonthsParse;\n    proto$1.monthsRegex       = monthsRegex;\n    proto$1.monthsShortRegex  = monthsShortRegex;\n    proto$1.week = localeWeek;\n    proto$1.firstDayOfYear = localeFirstDayOfYear;\n    proto$1.firstDayOfWeek = localeFirstDayOfWeek;\n\n    proto$1.weekdays       =        localeWeekdays;\n    proto$1.weekdaysMin    =        localeWeekdaysMin;\n    proto$1.weekdaysShort  =        localeWeekdaysShort;\n    proto$1.weekdaysParse  =        localeWeekdaysParse;\n\n    proto$1.weekdaysRegex       =        weekdaysRegex;\n    proto$1.weekdaysShortRegex  =        weekdaysShortRegex;\n    proto$1.weekdaysMinRegex    =        weekdaysMinRegex;\n\n    proto$1.isPM = localeIsPM;\n    proto$1.meridiem = localeMeridiem;\n\n    function get$1 (format, index, field, setter) {\n        var locale = getLocale();\n        var utc = createUTC().set(setter, index);\n        return locale[field](utc, format);\n    }\n\n    function listMonthsImpl (format, index, field) {\n        if (isNumber(format)) {\n            index = format;\n            format = undefined;\n        }\n\n        format = format || '';\n\n        if (index != null) {\n            return get$1(format, index, field, 'month');\n        }\n\n        var i;\n        var out = [];\n        for (i = 0; i < 12; i++) {\n            out[i] = get$1(format, i, field, 'month');\n        }\n        return out;\n    }\n\n    // ()\n    // (5)\n    // (fmt, 5)\n    // (fmt)\n    // (true)\n    // (true, 5)\n    // (true, fmt, 5)\n    // (true, fmt)\n    function listWeekdaysImpl (localeSorted, format, index, field) {\n        if (typeof localeSorted === 'boolean') {\n            if (isNumber(format)) {\n                index = format;\n                format = undefined;\n            }\n\n            format = format || '';\n        } else {\n            format = localeSorted;\n            index = format;\n            localeSorted = false;\n\n            if (isNumber(format)) {\n                index = format;\n                format = undefined;\n            }\n\n            format = format || '';\n        }\n\n        var locale = getLocale(),\n            shift = localeSorted ? locale._week.dow : 0;\n\n        if (index != null) {\n            return get$1(format, (index + shift) % 7, field, 'day');\n        }\n\n        var i;\n        var out = [];\n        for (i = 0; i < 7; i++) {\n            out[i] = get$1(format, (i + shift) % 7, field, 'day');\n        }\n        return out;\n    }\n\n    function listMonths (format, index) {\n        return listMonthsImpl(format, index, 'months');\n    }\n\n    function listMonthsShort (format, index) {\n        return listMonthsImpl(format, index, 'monthsShort');\n    }\n\n    function listWeekdays (localeSorted, format, index) {\n        return listWeekdaysImpl(localeSorted, format, index, 'weekdays');\n    }\n\n    function listWeekdaysShort (localeSorted, format, index) {\n        return listWeekdaysImpl(localeSorted, format, index, 'weekdaysShort');\n    }\n\n    function listWeekdaysMin (localeSorted, format, index) {\n        return listWeekdaysImpl(localeSorted, format, index, 'weekdaysMin');\n    }\n\n    getSetGlobalLocale('en', {\n        dayOfMonthOrdinalParse: /\\d{1,2}(th|st|nd|rd)/,\n        ordinal : function (number) {\n            var b = number % 10,\n                output = (toInt(number % 100 / 10) === 1) ? 'th' :\n                (b === 1) ? 'st' :\n                (b === 2) ? 'nd' :\n                (b === 3) ? 'rd' : 'th';\n            return number + output;\n        }\n    });\n\n    // Side effect imports\n\n    hooks.lang = deprecate('moment.lang is deprecated. Use moment.locale instead.', getSetGlobalLocale);\n    hooks.langData = deprecate('moment.langData is deprecated. Use moment.localeData instead.', getLocale);\n\n    var mathAbs = Math.abs;\n\n    function abs () {\n        var data           = this._data;\n\n        this._milliseconds = mathAbs(this._milliseconds);\n        this._days         = mathAbs(this._days);\n        this._months       = mathAbs(this._months);\n\n        data.milliseconds  = mathAbs(data.milliseconds);\n        data.seconds       = mathAbs(data.seconds);\n        data.minutes       = mathAbs(data.minutes);\n        data.hours         = mathAbs(data.hours);\n        data.months        = mathAbs(data.months);\n        data.years         = mathAbs(data.years);\n\n        return this;\n    }\n\n    function addSubtract$1 (duration, input, value, direction) {\n        var other = createDuration(input, value);\n\n        duration._milliseconds += direction * other._milliseconds;\n        duration._days         += direction * other._days;\n        duration._months       += direction * other._months;\n\n        return duration._bubble();\n    }\n\n    // supports only 2.0-style add(1, 's') or add(duration)\n    function add$1 (input, value) {\n        return addSubtract$1(this, input, value, 1);\n    }\n\n    // supports only 2.0-style subtract(1, 's') or subtract(duration)\n    function subtract$1 (input, value) {\n        return addSubtract$1(this, input, value, -1);\n    }\n\n    function absCeil (number) {\n        if (number < 0) {\n            return Math.floor(number);\n        } else {\n            return Math.ceil(number);\n        }\n    }\n\n    function bubble () {\n        var milliseconds = this._milliseconds;\n        var days         = this._days;\n        var months       = this._months;\n        var data         = this._data;\n        var seconds, minutes, hours, years, monthsFromDays;\n\n        // if we have a mix of positive and negative values, bubble down first\n        // check: https://github.com/moment/moment/issues/2166\n        if (!((milliseconds >= 0 && days >= 0 && months >= 0) ||\n                (milliseconds <= 0 && days <= 0 && months <= 0))) {\n            milliseconds += absCeil(monthsToDays(months) + days) * 864e5;\n            days = 0;\n            months = 0;\n        }\n\n        // The following code bubbles up values, see the tests for\n        // examples of what that means.\n        data.milliseconds = milliseconds % 1000;\n\n        seconds           = absFloor(milliseconds / 1000);\n        data.seconds      = seconds % 60;\n\n        minutes           = absFloor(seconds / 60);\n        data.minutes      = minutes % 60;\n\n        hours             = absFloor(minutes / 60);\n        data.hours        = hours % 24;\n\n        days += absFloor(hours / 24);\n\n        // convert days to months\n        monthsFromDays = absFloor(daysToMonths(days));\n        months += monthsFromDays;\n        days -= absCeil(monthsToDays(monthsFromDays));\n\n        // 12 months -> 1 year\n        years = absFloor(months / 12);\n        months %= 12;\n\n        data.days   = days;\n        data.months = months;\n        data.years  = years;\n\n        return this;\n    }\n\n    function daysToMonths (days) {\n        // 400 years have 146097 days (taking into account leap year rules)\n        // 400 years have 12 months === 4800\n        return days * 4800 / 146097;\n    }\n\n    function monthsToDays (months) {\n        // the reverse of daysToMonths\n        return months * 146097 / 4800;\n    }\n\n    function as (units) {\n        if (!this.isValid()) {\n            return NaN;\n        }\n        var days;\n        var months;\n        var milliseconds = this._milliseconds;\n\n        units = normalizeUnits(units);\n\n        if (units === 'month' || units === 'quarter' || units === 'year') {\n            days = this._days + milliseconds / 864e5;\n            months = this._months + daysToMonths(days);\n            switch (units) {\n                case 'month':   return months;\n                case 'quarter': return months / 3;\n                case 'year':    return months / 12;\n            }\n        } else {\n            // handle milliseconds separately because of floating point math errors (issue #1867)\n            days = this._days + Math.round(monthsToDays(this._months));\n            switch (units) {\n                case 'week'   : return days / 7     + milliseconds / 6048e5;\n                case 'day'    : return days         + milliseconds / 864e5;\n                case 'hour'   : return days * 24    + milliseconds / 36e5;\n                case 'minute' : return days * 1440  + milliseconds / 6e4;\n                case 'second' : return days * 86400 + milliseconds / 1000;\n                // Math.floor prevents floating point math errors here\n                case 'millisecond': return Math.floor(days * 864e5) + milliseconds;\n                default: throw new Error('Unknown unit ' + units);\n            }\n        }\n    }\n\n    // TODO: Use this.as('ms')?\n    function valueOf$1 () {\n        if (!this.isValid()) {\n            return NaN;\n        }\n        return (\n            this._milliseconds +\n            this._days * 864e5 +\n            (this._months % 12) * 2592e6 +\n            toInt(this._months / 12) * 31536e6\n        );\n    }\n\n    function makeAs (alias) {\n        return function () {\n            return this.as(alias);\n        };\n    }\n\n    var asMilliseconds = makeAs('ms');\n    var asSeconds      = makeAs('s');\n    var asMinutes      = makeAs('m');\n    var asHours        = makeAs('h');\n    var asDays         = makeAs('d');\n    var asWeeks        = makeAs('w');\n    var asMonths       = makeAs('M');\n    var asQuarters     = makeAs('Q');\n    var asYears        = makeAs('y');\n\n    function clone$1 () {\n        return createDuration(this);\n    }\n\n    function get$2 (units) {\n        units = normalizeUnits(units);\n        return this.isValid() ? this[units + 's']() : NaN;\n    }\n\n    function makeGetter(name) {\n        return function () {\n            return this.isValid() ? this._data[name] : NaN;\n        };\n    }\n\n    var milliseconds = makeGetter('milliseconds');\n    var seconds      = makeGetter('seconds');\n    var minutes      = makeGetter('minutes');\n    var hours        = makeGetter('hours');\n    var days         = makeGetter('days');\n    var months       = makeGetter('months');\n    var years        = makeGetter('years');\n\n    function weeks () {\n        return absFloor(this.days() / 7);\n    }\n\n    var round = Math.round;\n    var thresholds = {\n        ss: 44,         // a few seconds to seconds\n        s : 45,         // seconds to minute\n        m : 45,         // minutes to hour\n        h : 22,         // hours to day\n        d : 26,         // days to month\n        M : 11          // months to year\n    };\n\n    // helper function for moment.fn.from, moment.fn.fromNow, and moment.duration.fn.humanize\n    function substituteTimeAgo(string, number, withoutSuffix, isFuture, locale) {\n        return locale.relativeTime(number || 1, !!withoutSuffix, string, isFuture);\n    }\n\n    function relativeTime$1 (posNegDuration, withoutSuffix, locale) {\n        var duration = createDuration(posNegDuration).abs();\n        var seconds  = round(duration.as('s'));\n        var minutes  = round(duration.as('m'));\n        var hours    = round(duration.as('h'));\n        var days     = round(duration.as('d'));\n        var months   = round(duration.as('M'));\n        var years    = round(duration.as('y'));\n\n        var a = seconds <= thresholds.ss && ['s', seconds]  ||\n                seconds < thresholds.s   && ['ss', seconds] ||\n                minutes <= 1             && ['m']           ||\n                minutes < thresholds.m   && ['mm', minutes] ||\n                hours   <= 1             && ['h']           ||\n                hours   < thresholds.h   && ['hh', hours]   ||\n                days    <= 1             && ['d']           ||\n                days    < thresholds.d   && ['dd', days]    ||\n                months  <= 1             && ['M']           ||\n                months  < thresholds.M   && ['MM', months]  ||\n                years   <= 1             && ['y']           || ['yy', years];\n\n        a[2] = withoutSuffix;\n        a[3] = +posNegDuration > 0;\n        a[4] = locale;\n        return substituteTimeAgo.apply(null, a);\n    }\n\n    // This function allows you to set the rounding function for relative time strings\n    function getSetRelativeTimeRounding (roundingFunction) {\n        if (roundingFunction === undefined) {\n            return round;\n        }\n        if (typeof(roundingFunction) === 'function') {\n            round = roundingFunction;\n            return true;\n        }\n        return false;\n    }\n\n    // This function allows you to set a threshold for relative time strings\n    function getSetRelativeTimeThreshold (threshold, limit) {\n        if (thresholds[threshold] === undefined) {\n            return false;\n        }\n        if (limit === undefined) {\n            return thresholds[threshold];\n        }\n        thresholds[threshold] = limit;\n        if (threshold === 's') {\n            thresholds.ss = limit - 1;\n        }\n        return true;\n    }\n\n    function humanize (withSuffix) {\n        if (!this.isValid()) {\n            return this.localeData().invalidDate();\n        }\n\n        var locale = this.localeData();\n        var output = relativeTime$1(this, !withSuffix, locale);\n\n        if (withSuffix) {\n            output = locale.pastFuture(+this, output);\n        }\n\n        return locale.postformat(output);\n    }\n\n    var abs$1 = Math.abs;\n\n    function sign(x) {\n        return ((x > 0) - (x < 0)) || +x;\n    }\n\n    function toISOString$1() {\n        // for ISO strings we do not use the normal bubbling rules:\n        //  * milliseconds bubble up until they become hours\n        //  * days do not bubble at all\n        //  * months bubble up until they become years\n        // This is because there is no context-free conversion between hours and days\n        // (think of clock changes)\n        // and also not between days and months (28-31 days per month)\n        if (!this.isValid()) {\n            return this.localeData().invalidDate();\n        }\n\n        var seconds = abs$1(this._milliseconds) / 1000;\n        var days         = abs$1(this._days);\n        var months       = abs$1(this._months);\n        var minutes, hours, years;\n\n        // 3600 seconds -> 60 minutes -> 1 hour\n        minutes           = absFloor(seconds / 60);\n        hours             = absFloor(minutes / 60);\n        seconds %= 60;\n        minutes %= 60;\n\n        // 12 months -> 1 year\n        years  = absFloor(months / 12);\n        months %= 12;\n\n\n        // inspired by https://github.com/dordille/moment-isoduration/blob/master/moment.isoduration.js\n        var Y = years;\n        var M = months;\n        var D = days;\n        var h = hours;\n        var m = minutes;\n        var s = seconds ? seconds.toFixed(3).replace(/\\.?0+$/, '') : '';\n        var total = this.asSeconds();\n\n        if (!total) {\n            // this is the same as C#'s (Noda) and python (isodate)...\n            // but not other JS (goog.date)\n            return 'P0D';\n        }\n\n        var totalSign = total < 0 ? '-' : '';\n        var ymSign = sign(this._months) !== sign(total) ? '-' : '';\n        var daysSign = sign(this._days) !== sign(total) ? '-' : '';\n        var hmsSign = sign(this._milliseconds) !== sign(total) ? '-' : '';\n\n        return totalSign + 'P' +\n            (Y ? ymSign + Y + 'Y' : '') +\n            (M ? ymSign + M + 'M' : '') +\n            (D ? daysSign + D + 'D' : '') +\n            ((h || m || s) ? 'T' : '') +\n            (h ? hmsSign + h + 'H' : '') +\n            (m ? hmsSign + m + 'M' : '') +\n            (s ? hmsSign + s + 'S' : '');\n    }\n\n    var proto$2 = Duration.prototype;\n\n    proto$2.isValid        = isValid$1;\n    proto$2.abs            = abs;\n    proto$2.add            = add$1;\n    proto$2.subtract       = subtract$1;\n    proto$2.as             = as;\n    proto$2.asMilliseconds = asMilliseconds;\n    proto$2.asSeconds      = asSeconds;\n    proto$2.asMinutes      = asMinutes;\n    proto$2.asHours        = asHours;\n    proto$2.asDays         = asDays;\n    proto$2.asWeeks        = asWeeks;\n    proto$2.asMonths       = asMonths;\n    proto$2.asQuarters     = asQuarters;\n    proto$2.asYears        = asYears;\n    proto$2.valueOf        = valueOf$1;\n    proto$2._bubble        = bubble;\n    proto$2.clone          = clone$1;\n    proto$2.get            = get$2;\n    proto$2.milliseconds   = milliseconds;\n    proto$2.seconds        = seconds;\n    proto$2.minutes        = minutes;\n    proto$2.hours          = hours;\n    proto$2.days           = days;\n    proto$2.weeks          = weeks;\n    proto$2.months         = months;\n    proto$2.years          = years;\n    proto$2.humanize       = humanize;\n    proto$2.toISOString    = toISOString$1;\n    proto$2.toString       = toISOString$1;\n    proto$2.toJSON         = toISOString$1;\n    proto$2.locale         = locale;\n    proto$2.localeData     = localeData;\n\n    proto$2.toIsoString = deprecate('toIsoString() is deprecated. Please use toISOString() instead (notice the capitals)', toISOString$1);\n    proto$2.lang = lang;\n\n    // Side effect imports\n\n    // FORMATTING\n\n    addFormatToken('X', 0, 0, 'unix');\n    addFormatToken('x', 0, 0, 'valueOf');\n\n    // PARSING\n\n    addRegexToken('x', matchSigned);\n    addRegexToken('X', matchTimestamp);\n    addParseToken('X', function (input, array, config) {\n        config._d = new Date(parseFloat(input, 10) * 1000);\n    });\n    addParseToken('x', function (input, array, config) {\n        config._d = new Date(toInt(input));\n    });\n\n    // Side effect imports\n\n\n    hooks.version = '2.24.0';\n\n    setHookCallback(createLocal);\n\n    hooks.fn                    = proto;\n    hooks.min                   = min;\n    hooks.max                   = max;\n    hooks.now                   = now;\n    hooks.utc                   = createUTC;\n    hooks.unix                  = createUnix;\n    hooks.months                = listMonths;\n    hooks.isDate                = isDate;\n    hooks.locale                = getSetGlobalLocale;\n    hooks.invalid               = createInvalid;\n    hooks.duration              = createDuration;\n    hooks.isMoment              = isMoment;\n    hooks.weekdays              = listWeekdays;\n    hooks.parseZone             = createInZone;\n    hooks.localeData            = getLocale;\n    hooks.isDuration            = isDuration;\n    hooks.monthsShort           = listMonthsShort;\n    hooks.weekdaysMin           = listWeekdaysMin;\n    hooks.defineLocale          = defineLocale;\n    hooks.updateLocale          = updateLocale;\n    hooks.locales               = listLocales;\n    hooks.weekdaysShort         = listWeekdaysShort;\n    hooks.normalizeUnits        = normalizeUnits;\n    hooks.relativeTimeRounding  = getSetRelativeTimeRounding;\n    hooks.relativeTimeThreshold = getSetRelativeTimeThreshold;\n    hooks.calendarFormat        = getCalendarFormat;\n    hooks.prototype             = proto;\n\n    // currently HTML5 input type only supports 24-hour formats\n    hooks.HTML5_FMT = {\n        DATETIME_LOCAL: 'YYYY-MM-DDTHH:mm',             // <input type=\"datetime-local\" />\n        DATETIME_LOCAL_SECONDS: 'YYYY-MM-DDTHH:mm:ss',  // <input type=\"datetime-local\" step=\"1\" />\n        DATETIME_LOCAL_MS: 'YYYY-MM-DDTHH:mm:ss.SSS',   // <input type=\"datetime-local\" step=\"0.001\" />\n        DATE: 'YYYY-MM-DD',                             // <input type=\"date\" />\n        TIME: 'HH:mm',                                  // <input type=\"time\" />\n        TIME_SECONDS: 'HH:mm:ss',                       // <input type=\"time\" step=\"1\" />\n        TIME_MS: 'HH:mm:ss.SSS',                        // <input type=\"time\" step=\"0.001\" />\n        WEEK: 'GGGG-[W]WW',                             // <input type=\"week\" />\n        MONTH: 'YYYY-MM'                                // <input type=\"month\" />\n    };\n\n    return hooks;\n\n})));\n","var check = function (it) {\n  return it && it.Math == Math && it;\n};\n\n// https://github.com/zloirock/core-js/issues/86#issuecomment-115759028\nmodule.exports =\n  // eslint-disable-next-line no-undef\n  check(typeof globalThis == 'object' && globalThis) ||\n  check(typeof window == 'object' && window) ||\n  check(typeof self == 'object' && self) ||\n  check(typeof global == 'object' && global) ||\n  // eslint-disable-next-line no-new-func\n  Function('return this')();\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    function processRelativeTime(number, withoutSuffix, key, isFuture) {\n        var format = {\n            'm': ['eine Minute', 'einer Minute'],\n            'h': ['eine Stunde', 'einer Stunde'],\n            'd': ['ein Tag', 'einem Tag'],\n            'dd': [number + ' Tage', number + ' Tagen'],\n            'M': ['ein Monat', 'einem Monat'],\n            'MM': [number + ' Monate', number + ' Monaten'],\n            'y': ['ein Jahr', 'einem Jahr'],\n            'yy': [number + ' Jahre', number + ' Jahren']\n        };\n        return withoutSuffix ? format[key][0] : format[key][1];\n    }\n\n    var de = moment.defineLocale('de', {\n        months : 'Januar_Februar_März_April_Mai_Juni_Juli_August_September_Oktober_November_Dezember'.split('_'),\n        monthsShort : 'Jan._Feb._März_Apr._Mai_Juni_Juli_Aug._Sep._Okt._Nov._Dez.'.split('_'),\n        monthsParseExact : true,\n        weekdays : 'Sonntag_Montag_Dienstag_Mittwoch_Donnerstag_Freitag_Samstag'.split('_'),\n        weekdaysShort : 'So._Mo._Di._Mi._Do._Fr._Sa.'.split('_'),\n        weekdaysMin : 'So_Mo_Di_Mi_Do_Fr_Sa'.split('_'),\n        weekdaysParseExact : true,\n        longDateFormat : {\n            LT: 'HH:mm',\n            LTS: 'HH:mm:ss',\n            L : 'DD.MM.YYYY',\n            LL : 'D. MMMM YYYY',\n            LLL : 'D. MMMM YYYY HH:mm',\n            LLLL : 'dddd, D. MMMM YYYY HH:mm'\n        },\n        calendar : {\n            sameDay: '[heute um] LT [Uhr]',\n            sameElse: 'L',\n            nextDay: '[morgen um] LT [Uhr]',\n            nextWeek: 'dddd [um] LT [Uhr]',\n            lastDay: '[gestern um] LT [Uhr]',\n            lastWeek: '[letzten] dddd [um] LT [Uhr]'\n        },\n        relativeTime : {\n            future : 'in %s',\n            past : 'vor %s',\n            s : 'ein paar Sekunden',\n            ss : '%d Sekunden',\n            m : processRelativeTime,\n            mm : '%d Minuten',\n            h : processRelativeTime,\n            hh : '%d Stunden',\n            d : processRelativeTime,\n            dd : processRelativeTime,\n            M : processRelativeTime,\n            MM : processRelativeTime,\n            y : processRelativeTime,\n            yy : processRelativeTime\n        },\n        dayOfMonthOrdinalParse: /\\d{1,2}\\./,\n        ordinal : '%d.',\n        week : {\n            dow : 1, // Monday is the first day of the week.\n            doy : 4  // The week that contains Jan 4th is the first week of the year.\n        }\n    });\n\n    return de;\n\n})));\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    var monthsShortDot = 'ene._feb._mar._abr._may._jun._jul._ago._sep._oct._nov._dic.'.split('_'),\n        monthsShort = 'ene_feb_mar_abr_may_jun_jul_ago_sep_oct_nov_dic'.split('_');\n\n    var monthsParse = [/^ene/i, /^feb/i, /^mar/i, /^abr/i, /^may/i, /^jun/i, /^jul/i, /^ago/i, /^sep/i, /^oct/i, /^nov/i, /^dic/i];\n    var monthsRegex = /^(enero|febrero|marzo|abril|mayo|junio|julio|agosto|septiembre|octubre|noviembre|diciembre|ene\\.?|feb\\.?|mar\\.?|abr\\.?|may\\.?|jun\\.?|jul\\.?|ago\\.?|sep\\.?|oct\\.?|nov\\.?|dic\\.?)/i;\n\n    var es = moment.defineLocale('es', {\n        months : 'enero_febrero_marzo_abril_mayo_junio_julio_agosto_septiembre_octubre_noviembre_diciembre'.split('_'),\n        monthsShort : function (m, format) {\n            if (!m) {\n                return monthsShortDot;\n            } else if (/-MMM-/.test(format)) {\n                return monthsShort[m.month()];\n            } else {\n                return monthsShortDot[m.month()];\n            }\n        },\n        monthsRegex : monthsRegex,\n        monthsShortRegex : monthsRegex,\n        monthsStrictRegex : /^(enero|febrero|marzo|abril|mayo|junio|julio|agosto|septiembre|octubre|noviembre|diciembre)/i,\n        monthsShortStrictRegex : /^(ene\\.?|feb\\.?|mar\\.?|abr\\.?|may\\.?|jun\\.?|jul\\.?|ago\\.?|sep\\.?|oct\\.?|nov\\.?|dic\\.?)/i,\n        monthsParse : monthsParse,\n        longMonthsParse : monthsParse,\n        shortMonthsParse : monthsParse,\n        weekdays : 'domingo_lunes_martes_miércoles_jueves_viernes_sábado'.split('_'),\n        weekdaysShort : 'dom._lun._mar._mié._jue._vie._sáb.'.split('_'),\n        weekdaysMin : 'do_lu_ma_mi_ju_vi_sá'.split('_'),\n        weekdaysParseExact : true,\n        longDateFormat : {\n            LT : 'H:mm',\n            LTS : 'H:mm:ss',\n            L : 'DD/MM/YYYY',\n            LL : 'D [de] MMMM [de] YYYY',\n            LLL : 'D [de] MMMM [de] YYYY H:mm',\n            LLLL : 'dddd, D [de] MMMM [de] YYYY H:mm'\n        },\n        calendar : {\n            sameDay : function () {\n                return '[hoy a la' + ((this.hours() !== 1) ? 's' : '') + '] LT';\n            },\n            nextDay : function () {\n                return '[mañana a la' + ((this.hours() !== 1) ? 's' : '') + '] LT';\n            },\n            nextWeek : function () {\n                return 'dddd [a la' + ((this.hours() !== 1) ? 's' : '') + '] LT';\n            },\n            lastDay : function () {\n                return '[ayer a la' + ((this.hours() !== 1) ? 's' : '') + '] LT';\n            },\n            lastWeek : function () {\n                return '[el] dddd [pasado a la' + ((this.hours() !== 1) ? 's' : '') + '] LT';\n            },\n            sameElse : 'L'\n        },\n        relativeTime : {\n            future : 'en %s',\n            past : 'hace %s',\n            s : 'unos segundos',\n            ss : '%d segundos',\n            m : 'un minuto',\n            mm : '%d minutos',\n            h : 'una hora',\n            hh : '%d horas',\n            d : 'un día',\n            dd : '%d días',\n            M : 'un mes',\n            MM : '%d meses',\n            y : 'un año',\n            yy : '%d años'\n        },\n        dayOfMonthOrdinalParse : /\\d{1,2}º/,\n        ordinal : '%dº',\n        week : {\n            dow : 1, // Monday is the first day of the week.\n            doy : 4  // The week that contains Jan 4th is the first week of the year.\n        }\n    });\n\n    return es;\n\n})));\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    var fr = moment.defineLocale('fr', {\n        months : 'janvier_février_mars_avril_mai_juin_juillet_août_septembre_octobre_novembre_décembre'.split('_'),\n        monthsShort : 'janv._févr._mars_avr._mai_juin_juil._août_sept._oct._nov._déc.'.split('_'),\n        monthsParseExact : true,\n        weekdays : 'dimanche_lundi_mardi_mercredi_jeudi_vendredi_samedi'.split('_'),\n        weekdaysShort : 'dim._lun._mar._mer._jeu._ven._sam.'.split('_'),\n        weekdaysMin : 'di_lu_ma_me_je_ve_sa'.split('_'),\n        weekdaysParseExact : true,\n        longDateFormat : {\n            LT : 'HH:mm',\n            LTS : 'HH:mm:ss',\n            L : 'DD/MM/YYYY',\n            LL : 'D MMMM YYYY',\n            LLL : 'D MMMM YYYY HH:mm',\n            LLLL : 'dddd D MMMM YYYY HH:mm'\n        },\n        calendar : {\n            sameDay : '[Aujourd’hui à] LT',\n            nextDay : '[Demain à] LT',\n            nextWeek : 'dddd [à] LT',\n            lastDay : '[Hier à] LT',\n            lastWeek : 'dddd [dernier à] LT',\n            sameElse : 'L'\n        },\n        relativeTime : {\n            future : 'dans %s',\n            past : 'il y a %s',\n            s : 'quelques secondes',\n            ss : '%d secondes',\n            m : 'une minute',\n            mm : '%d minutes',\n            h : 'une heure',\n            hh : '%d heures',\n            d : 'un jour',\n            dd : '%d jours',\n            M : 'un mois',\n            MM : '%d mois',\n            y : 'un an',\n            yy : '%d ans'\n        },\n        dayOfMonthOrdinalParse: /\\d{1,2}(er|)/,\n        ordinal : function (number, period) {\n            switch (period) {\n                // TODO: Return 'e' when day of month > 1. Move this case inside\n                // block for masculine words below.\n                // See https://github.com/moment/moment/issues/3375\n                case 'D':\n                    return number + (number === 1 ? 'er' : '');\n\n                // Words with masculine grammatical gender: mois, trimestre, jour\n                default:\n                case 'M':\n                case 'Q':\n                case 'DDD':\n                case 'd':\n                    return number + (number === 1 ? 'er' : 'e');\n\n                // Words with feminine grammatical gender: semaine\n                case 'w':\n                case 'W':\n                    return number + (number === 1 ? 're' : 'e');\n            }\n        },\n        week : {\n            dow : 1, // Monday is the first day of the week.\n            doy : 4  // The week that contains Jan 4th is the first week of the year.\n        }\n    });\n\n    return fr;\n\n})));\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    var it = moment.defineLocale('it', {\n        months : 'gennaio_febbraio_marzo_aprile_maggio_giugno_luglio_agosto_settembre_ottobre_novembre_dicembre'.split('_'),\n        monthsShort : 'gen_feb_mar_apr_mag_giu_lug_ago_set_ott_nov_dic'.split('_'),\n        weekdays : 'domenica_lunedì_martedì_mercoledì_giovedì_venerdì_sabato'.split('_'),\n        weekdaysShort : 'dom_lun_mar_mer_gio_ven_sab'.split('_'),\n        weekdaysMin : 'do_lu_ma_me_gi_ve_sa'.split('_'),\n        longDateFormat : {\n            LT : 'HH:mm',\n            LTS : 'HH:mm:ss',\n            L : 'DD/MM/YYYY',\n            LL : 'D MMMM YYYY',\n            LLL : 'D MMMM YYYY HH:mm',\n            LLLL : 'dddd D MMMM YYYY HH:mm'\n        },\n        calendar : {\n            sameDay: '[Oggi alle] LT',\n            nextDay: '[Domani alle] LT',\n            nextWeek: 'dddd [alle] LT',\n            lastDay: '[Ieri alle] LT',\n            lastWeek: function () {\n                switch (this.day()) {\n                    case 0:\n                        return '[la scorsa] dddd [alle] LT';\n                    default:\n                        return '[lo scorso] dddd [alle] LT';\n                }\n            },\n            sameElse: 'L'\n        },\n        relativeTime : {\n            future : function (s) {\n                return ((/^[0-9].+$/).test(s) ? 'tra' : 'in') + ' ' + s;\n            },\n            past : '%s fa',\n            s : 'alcuni secondi',\n            ss : '%d secondi',\n            m : 'un minuto',\n            mm : '%d minuti',\n            h : 'un\\'ora',\n            hh : '%d ore',\n            d : 'un giorno',\n            dd : '%d giorni',\n            M : 'un mese',\n            MM : '%d mesi',\n            y : 'un anno',\n            yy : '%d anni'\n        },\n        dayOfMonthOrdinalParse : /\\d{1,2}º/,\n        ordinal: '%dº',\n        week : {\n            dow : 1, // Monday is the first day of the week.\n            doy : 4  // The week that contains Jan 4th is the first week of the year.\n        }\n    });\n\n    return it;\n\n})));\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    var ja = moment.defineLocale('ja', {\n        months : '一月_二月_三月_四月_五月_六月_七月_八月_九月_十月_十一月_十二月'.split('_'),\n        monthsShort : '1月_2月_3月_4月_5月_6月_7月_8月_9月_10月_11月_12月'.split('_'),\n        weekdays : '日曜日_月曜日_火曜日_水曜日_木曜日_金曜日_土曜日'.split('_'),\n        weekdaysShort : '日_月_火_水_木_金_土'.split('_'),\n        weekdaysMin : '日_月_火_水_木_金_土'.split('_'),\n        longDateFormat : {\n            LT : 'HH:mm',\n            LTS : 'HH:mm:ss',\n            L : 'YYYY/MM/DD',\n            LL : 'YYYY年M月D日',\n            LLL : 'YYYY年M月D日 HH:mm',\n            LLLL : 'YYYY年M月D日 dddd HH:mm',\n            l : 'YYYY/MM/DD',\n            ll : 'YYYY年M月D日',\n            lll : 'YYYY年M月D日 HH:mm',\n            llll : 'YYYY年M月D日(ddd) HH:mm'\n        },\n        meridiemParse: /午前|午後/i,\n        isPM : function (input) {\n            return input === '午後';\n        },\n        meridiem : function (hour, minute, isLower) {\n            if (hour < 12) {\n                return '午前';\n            } else {\n                return '午後';\n            }\n        },\n        calendar : {\n            sameDay : '[今日] LT',\n            nextDay : '[明日] LT',\n            nextWeek : function (now) {\n                if (now.week() < this.week()) {\n                    return '[来週]dddd LT';\n                } else {\n                    return 'dddd LT';\n                }\n            },\n            lastDay : '[昨日] LT',\n            lastWeek : function (now) {\n                if (this.week() < now.week()) {\n                    return '[先週]dddd LT';\n                } else {\n                    return 'dddd LT';\n                }\n            },\n            sameElse : 'L'\n        },\n        dayOfMonthOrdinalParse : /\\d{1,2}日/,\n        ordinal : function (number, period) {\n            switch (period) {\n                case 'd':\n                case 'D':\n                case 'DDD':\n                    return number + '日';\n                default:\n                    return number;\n            }\n        },\n        relativeTime : {\n            future : '%s後',\n            past : '%s前',\n            s : '数秒',\n            ss : '%d秒',\n            m : '1分',\n            mm : '%d分',\n            h : '1時間',\n            hh : '%d時間',\n            d : '1日',\n            dd : '%d日',\n            M : '1ヶ月',\n            MM : '%dヶ月',\n            y : '1年',\n            yy : '%d年'\n        }\n    });\n\n    return ja;\n\n})));\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    var monthsShortWithDots = 'jan._feb._mrt._apr._mei_jun._jul._aug._sep._okt._nov._dec.'.split('_'),\n        monthsShortWithoutDots = 'jan_feb_mrt_apr_mei_jun_jul_aug_sep_okt_nov_dec'.split('_');\n\n    var monthsParse = [/^jan/i, /^feb/i, /^maart|mrt.?$/i, /^apr/i, /^mei$/i, /^jun[i.]?$/i, /^jul[i.]?$/i, /^aug/i, /^sep/i, /^okt/i, /^nov/i, /^dec/i];\n    var monthsRegex = /^(januari|februari|maart|april|mei|ju[nl]i|augustus|september|oktober|november|december|jan\\.?|feb\\.?|mrt\\.?|apr\\.?|ju[nl]\\.?|aug\\.?|sep\\.?|okt\\.?|nov\\.?|dec\\.?)/i;\n\n    var nl = moment.defineLocale('nl', {\n        months : 'januari_februari_maart_april_mei_juni_juli_augustus_september_oktober_november_december'.split('_'),\n        monthsShort : function (m, format) {\n            if (!m) {\n                return monthsShortWithDots;\n            } else if (/-MMM-/.test(format)) {\n                return monthsShortWithoutDots[m.month()];\n            } else {\n                return monthsShortWithDots[m.month()];\n            }\n        },\n\n        monthsRegex: monthsRegex,\n        monthsShortRegex: monthsRegex,\n        monthsStrictRegex: /^(januari|februari|maart|april|mei|ju[nl]i|augustus|september|oktober|november|december)/i,\n        monthsShortStrictRegex: /^(jan\\.?|feb\\.?|mrt\\.?|apr\\.?|mei|ju[nl]\\.?|aug\\.?|sep\\.?|okt\\.?|nov\\.?|dec\\.?)/i,\n\n        monthsParse : monthsParse,\n        longMonthsParse : monthsParse,\n        shortMonthsParse : monthsParse,\n\n        weekdays : 'zondag_maandag_dinsdag_woensdag_donderdag_vrijdag_zaterdag'.split('_'),\n        weekdaysShort : 'zo._ma._di._wo._do._vr._za.'.split('_'),\n        weekdaysMin : 'zo_ma_di_wo_do_vr_za'.split('_'),\n        weekdaysParseExact : true,\n        longDateFormat : {\n            LT : 'HH:mm',\n            LTS : 'HH:mm:ss',\n            L : 'DD-MM-YYYY',\n            LL : 'D MMMM YYYY',\n            LLL : 'D MMMM YYYY HH:mm',\n            LLLL : 'dddd D MMMM YYYY HH:mm'\n        },\n        calendar : {\n            sameDay: '[vandaag om] LT',\n            nextDay: '[morgen om] LT',\n            nextWeek: 'dddd [om] LT',\n            lastDay: '[gisteren om] LT',\n            lastWeek: '[afgelopen] dddd [om] LT',\n            sameElse: 'L'\n        },\n        relativeTime : {\n            future : 'over %s',\n            past : '%s geleden',\n            s : 'een paar seconden',\n            ss : '%d seconden',\n            m : 'één minuut',\n            mm : '%d minuten',\n            h : 'één uur',\n            hh : '%d uur',\n            d : 'één dag',\n            dd : '%d dagen',\n            M : 'één maand',\n            MM : '%d maanden',\n            y : 'één jaar',\n            yy : '%d jaar'\n        },\n        dayOfMonthOrdinalParse: /\\d{1,2}(ste|de)/,\n        ordinal : function (number) {\n            return number + ((number === 1 || number === 8 || number >= 20) ? 'ste' : 'de');\n        },\n        week : {\n            dow : 1, // Monday is the first day of the week.\n            doy : 4  // The week that contains Jan 4th is the first week of the year.\n        }\n    });\n\n    return nl;\n\n})));\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    var monthsNominative = 'styczeń_luty_marzec_kwiecień_maj_czerwiec_lipiec_sierpień_wrzesień_październik_listopad_grudzień'.split('_'),\n        monthsSubjective = 'stycznia_lutego_marca_kwietnia_maja_czerwca_lipca_sierpnia_września_października_listopada_grudnia'.split('_');\n    function plural(n) {\n        return (n % 10 < 5) && (n % 10 > 1) && ((~~(n / 10) % 10) !== 1);\n    }\n    function translate(number, withoutSuffix, key) {\n        var result = number + ' ';\n        switch (key) {\n            case 'ss':\n                return result + (plural(number) ? 'sekundy' : 'sekund');\n            case 'm':\n                return withoutSuffix ? 'minuta' : 'minutę';\n            case 'mm':\n                return result + (plural(number) ? 'minuty' : 'minut');\n            case 'h':\n                return withoutSuffix  ? 'godzina'  : 'godzinę';\n            case 'hh':\n                return result + (plural(number) ? 'godziny' : 'godzin');\n            case 'MM':\n                return result + (plural(number) ? 'miesiące' : 'miesięcy');\n            case 'yy':\n                return result + (plural(number) ? 'lata' : 'lat');\n        }\n    }\n\n    var pl = moment.defineLocale('pl', {\n        months : function (momentToFormat, format) {\n            if (!momentToFormat) {\n                return monthsNominative;\n            } else if (format === '') {\n                // Hack: if format empty we know this is used to generate\n                // RegExp by moment. Give then back both valid forms of months\n                // in RegExp ready format.\n                return '(' + monthsSubjective[momentToFormat.month()] + '|' + monthsNominative[momentToFormat.month()] + ')';\n            } else if (/D MMMM/.test(format)) {\n                return monthsSubjective[momentToFormat.month()];\n            } else {\n                return monthsNominative[momentToFormat.month()];\n            }\n        },\n        monthsShort : 'sty_lut_mar_kwi_maj_cze_lip_sie_wrz_paź_lis_gru'.split('_'),\n        weekdays : 'niedziela_poniedziałek_wtorek_środa_czwartek_piątek_sobota'.split('_'),\n        weekdaysShort : 'ndz_pon_wt_śr_czw_pt_sob'.split('_'),\n        weekdaysMin : 'Nd_Pn_Wt_Śr_Cz_Pt_So'.split('_'),\n        longDateFormat : {\n            LT : 'HH:mm',\n            LTS : 'HH:mm:ss',\n            L : 'DD.MM.YYYY',\n            LL : 'D MMMM YYYY',\n            LLL : 'D MMMM YYYY HH:mm',\n            LLLL : 'dddd, D MMMM YYYY HH:mm'\n        },\n        calendar : {\n            sameDay: '[Dziś o] LT',\n            nextDay: '[Jutro o] LT',\n            nextWeek: function () {\n                switch (this.day()) {\n                    case 0:\n                        return '[W niedzielę o] LT';\n\n                    case 2:\n                        return '[We wtorek o] LT';\n\n                    case 3:\n                        return '[W środę o] LT';\n\n                    case 6:\n                        return '[W sobotę o] LT';\n\n                    default:\n                        return '[W] dddd [o] LT';\n                }\n            },\n            lastDay: '[Wczoraj o] LT',\n            lastWeek: function () {\n                switch (this.day()) {\n                    case 0:\n                        return '[W zeszłą niedzielę o] LT';\n                    case 3:\n                        return '[W zeszłą środę o] LT';\n                    case 6:\n                        return '[W zeszłą sobotę o] LT';\n                    default:\n                        return '[W zeszły] dddd [o] LT';\n                }\n            },\n            sameElse: 'L'\n        },\n        relativeTime : {\n            future : 'za %s',\n            past : '%s temu',\n            s : 'kilka sekund',\n            ss : translate,\n            m : translate,\n            mm : translate,\n            h : translate,\n            hh : translate,\n            d : '1 dzień',\n            dd : '%d dni',\n            M : 'miesiąc',\n            MM : translate,\n            y : 'rok',\n            yy : translate\n        },\n        dayOfMonthOrdinalParse: /\\d{1,2}\\./,\n        ordinal : '%d.',\n        week : {\n            dow : 1, // Monday is the first day of the week.\n            doy : 4  // The week that contains Jan 4th is the first week of the year.\n        }\n    });\n\n    return pl;\n\n})));\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    function plural(word, num) {\n        var forms = word.split('_');\n        return num % 10 === 1 && num % 100 !== 11 ? forms[0] : (num % 10 >= 2 && num % 10 <= 4 && (num % 100 < 10 || num % 100 >= 20) ? forms[1] : forms[2]);\n    }\n    function relativeTimeWithPlural(number, withoutSuffix, key) {\n        var format = {\n            'ss': withoutSuffix ? 'секунда_секунды_секунд' : 'секунду_секунды_секунд',\n            'mm': withoutSuffix ? 'минута_минуты_минут' : 'минуту_минуты_минут',\n            'hh': 'час_часа_часов',\n            'dd': 'день_дня_дней',\n            'MM': 'месяц_месяца_месяцев',\n            'yy': 'год_года_лет'\n        };\n        if (key === 'm') {\n            return withoutSuffix ? 'минута' : 'минуту';\n        }\n        else {\n            return number + ' ' + plural(format[key], +number);\n        }\n    }\n    var monthsParse = [/^янв/i, /^фев/i, /^мар/i, /^апр/i, /^ма[йя]/i, /^июн/i, /^июл/i, /^авг/i, /^сен/i, /^окт/i, /^ноя/i, /^дек/i];\n\n    // http://new.gramota.ru/spravka/rules/139-prop : § 103\n    // Сокращения месяцев: http://new.gramota.ru/spravka/buro/search-answer?s=242637\n    // CLDR data:          http://www.unicode.org/cldr/charts/28/summary/ru.html#1753\n    var ru = moment.defineLocale('ru', {\n        months : {\n            format: 'января_февраля_марта_апреля_мая_июня_июля_августа_сентября_октября_ноября_декабря'.split('_'),\n            standalone: 'январь_февраль_март_апрель_май_июнь_июль_август_сентябрь_октябрь_ноябрь_декабрь'.split('_')\n        },\n        monthsShort : {\n            // по CLDR именно \"июл.\" и \"июн.\", но какой смысл менять букву на точку ?\n            format: 'янв._февр._мар._апр._мая_июня_июля_авг._сент._окт._нояб._дек.'.split('_'),\n            standalone: 'янв._февр._март_апр._май_июнь_июль_авг._сент._окт._нояб._дек.'.split('_')\n        },\n        weekdays : {\n            standalone: 'воскресенье_понедельник_вторник_среда_четверг_пятница_суббота'.split('_'),\n            format: 'воскресенье_понедельник_вторник_среду_четверг_пятницу_субботу'.split('_'),\n            isFormat: /\\[ ?[Вв] ?(?:прошлую|следующую|эту)? ?\\] ?dddd/\n        },\n        weekdaysShort : 'вс_пн_вт_ср_чт_пт_сб'.split('_'),\n        weekdaysMin : 'вс_пн_вт_ср_чт_пт_сб'.split('_'),\n        monthsParse : monthsParse,\n        longMonthsParse : monthsParse,\n        shortMonthsParse : monthsParse,\n\n        // полные названия с падежами, по три буквы, для некоторых, по 4 буквы, сокращения с точкой и без точки\n        monthsRegex: /^(январ[ья]|янв\\.?|феврал[ья]|февр?\\.?|марта?|мар\\.?|апрел[ья]|апр\\.?|ма[йя]|июн[ья]|июн\\.?|июл[ья]|июл\\.?|августа?|авг\\.?|сентябр[ья]|сент?\\.?|октябр[ья]|окт\\.?|ноябр[ья]|нояб?\\.?|декабр[ья]|дек\\.?)/i,\n\n        // копия предыдущего\n        monthsShortRegex: /^(январ[ья]|янв\\.?|феврал[ья]|февр?\\.?|марта?|мар\\.?|апрел[ья]|апр\\.?|ма[йя]|июн[ья]|июн\\.?|июл[ья]|июл\\.?|августа?|авг\\.?|сентябр[ья]|сент?\\.?|октябр[ья]|окт\\.?|ноябр[ья]|нояб?\\.?|декабр[ья]|дек\\.?)/i,\n\n        // полные названия с падежами\n        monthsStrictRegex: /^(январ[яь]|феврал[яь]|марта?|апрел[яь]|ма[яй]|июн[яь]|июл[яь]|августа?|сентябр[яь]|октябр[яь]|ноябр[яь]|декабр[яь])/i,\n\n        // Выражение, которое соотвествует только сокращённым формам\n        monthsShortStrictRegex: /^(янв\\.|февр?\\.|мар[т.]|апр\\.|ма[яй]|июн[ья.]|июл[ья.]|авг\\.|сент?\\.|окт\\.|нояб?\\.|дек\\.)/i,\n        longDateFormat : {\n            LT : 'H:mm',\n            LTS : 'H:mm:ss',\n            L : 'DD.MM.YYYY',\n            LL : 'D MMMM YYYY г.',\n            LLL : 'D MMMM YYYY г., H:mm',\n            LLLL : 'dddd, D MMMM YYYY г., H:mm'\n        },\n        calendar : {\n            sameDay: '[Сегодня, в] LT',\n            nextDay: '[Завтра, в] LT',\n            lastDay: '[Вчера, в] LT',\n            nextWeek: function (now) {\n                if (now.week() !== this.week()) {\n                    switch (this.day()) {\n                        case 0:\n                            return '[В следующее] dddd, [в] LT';\n                        case 1:\n                        case 2:\n                        case 4:\n                            return '[В следующий] dddd, [в] LT';\n                        case 3:\n                        case 5:\n                        case 6:\n                            return '[В следующую] dddd, [в] LT';\n                    }\n                } else {\n                    if (this.day() === 2) {\n                        return '[Во] dddd, [в] LT';\n                    } else {\n                        return '[В] dddd, [в] LT';\n                    }\n                }\n            },\n            lastWeek: function (now) {\n                if (now.week() !== this.week()) {\n                    switch (this.day()) {\n                        case 0:\n                            return '[В прошлое] dddd, [в] LT';\n                        case 1:\n                        case 2:\n                        case 4:\n                            return '[В прошлый] dddd, [в] LT';\n                        case 3:\n                        case 5:\n                        case 6:\n                            return '[В прошлую] dddd, [в] LT';\n                    }\n                } else {\n                    if (this.day() === 2) {\n                        return '[Во] dddd, [в] LT';\n                    } else {\n                        return '[В] dddd, [в] LT';\n                    }\n                }\n            },\n            sameElse: 'L'\n        },\n        relativeTime : {\n            future : 'через %s',\n            past : '%s назад',\n            s : 'несколько секунд',\n            ss : relativeTimeWithPlural,\n            m : relativeTimeWithPlural,\n            mm : relativeTimeWithPlural,\n            h : 'час',\n            hh : relativeTimeWithPlural,\n            d : 'день',\n            dd : relativeTimeWithPlural,\n            M : 'месяц',\n            MM : relativeTimeWithPlural,\n            y : 'год',\n            yy : relativeTimeWithPlural\n        },\n        meridiemParse: /ночи|утра|дня|вечера/i,\n        isPM : function (input) {\n            return /^(дня|вечера)$/.test(input);\n        },\n        meridiem : function (hour, minute, isLower) {\n            if (hour < 4) {\n                return 'ночи';\n            } else if (hour < 12) {\n                return 'утра';\n            } else if (hour < 17) {\n                return 'дня';\n            } else {\n                return 'вечера';\n            }\n        },\n        dayOfMonthOrdinalParse: /\\d{1,2}-(й|го|я)/,\n        ordinal: function (number, period) {\n            switch (period) {\n                case 'M':\n                case 'd':\n                case 'DDD':\n                    return number + '-й';\n                case 'D':\n                    return number + '-го';\n                case 'w':\n                case 'W':\n                    return number + '-я';\n                default:\n                    return number;\n            }\n        },\n        week : {\n            dow : 1, // Monday is the first day of the week.\n            doy : 4  // The week that contains Jan 4th is the first week of the year.\n        }\n    });\n\n    return ru;\n\n})));\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    function plural(word, num) {\n        var forms = word.split('_');\n        return num % 10 === 1 && num % 100 !== 11 ? forms[0] : (num % 10 >= 2 && num % 10 <= 4 && (num % 100 < 10 || num % 100 >= 20) ? forms[1] : forms[2]);\n    }\n    function relativeTimeWithPlural(number, withoutSuffix, key) {\n        var format = {\n            'ss': withoutSuffix ? 'секунда_секунди_секунд' : 'секунду_секунди_секунд',\n            'mm': withoutSuffix ? 'хвилина_хвилини_хвилин' : 'хвилину_хвилини_хвилин',\n            'hh': withoutSuffix ? 'година_години_годин' : 'годину_години_годин',\n            'dd': 'день_дні_днів',\n            'MM': 'місяць_місяці_місяців',\n            'yy': 'рік_роки_років'\n        };\n        if (key === 'm') {\n            return withoutSuffix ? 'хвилина' : 'хвилину';\n        }\n        else if (key === 'h') {\n            return withoutSuffix ? 'година' : 'годину';\n        }\n        else {\n            return number + ' ' + plural(format[key], +number);\n        }\n    }\n    function weekdaysCaseReplace(m, format) {\n        var weekdays = {\n            'nominative': 'неділя_понеділок_вівторок_середа_четвер_п’ятниця_субота'.split('_'),\n            'accusative': 'неділю_понеділок_вівторок_середу_четвер_п’ятницю_суботу'.split('_'),\n            'genitive': 'неділі_понеділка_вівторка_середи_четверга_п’ятниці_суботи'.split('_')\n        };\n\n        if (m === true) {\n            return weekdays['nominative'].slice(1, 7).concat(weekdays['nominative'].slice(0, 1));\n        }\n        if (!m) {\n            return weekdays['nominative'];\n        }\n\n        var nounCase = (/(\\[[ВвУу]\\]) ?dddd/).test(format) ?\n            'accusative' :\n            ((/\\[?(?:минулої|наступної)? ?\\] ?dddd/).test(format) ?\n                'genitive' :\n                'nominative');\n        return weekdays[nounCase][m.day()];\n    }\n    function processHoursFunction(str) {\n        return function () {\n            return str + 'о' + (this.hours() === 11 ? 'б' : '') + '] LT';\n        };\n    }\n\n    var uk = moment.defineLocale('uk', {\n        months : {\n            'format': 'січня_лютого_березня_квітня_травня_червня_липня_серпня_вересня_жовтня_листопада_грудня'.split('_'),\n            'standalone': 'січень_лютий_березень_квітень_травень_червень_липень_серпень_вересень_жовтень_листопад_грудень'.split('_')\n        },\n        monthsShort : 'січ_лют_бер_квіт_трав_черв_лип_серп_вер_жовт_лист_груд'.split('_'),\n        weekdays : weekdaysCaseReplace,\n        weekdaysShort : 'нд_пн_вт_ср_чт_пт_сб'.split('_'),\n        weekdaysMin : 'нд_пн_вт_ср_чт_пт_сб'.split('_'),\n        longDateFormat : {\n            LT : 'HH:mm',\n            LTS : 'HH:mm:ss',\n            L : 'DD.MM.YYYY',\n            LL : 'D MMMM YYYY р.',\n            LLL : 'D MMMM YYYY р., HH:mm',\n            LLLL : 'dddd, D MMMM YYYY р., HH:mm'\n        },\n        calendar : {\n            sameDay: processHoursFunction('[Сьогодні '),\n            nextDay: processHoursFunction('[Завтра '),\n            lastDay: processHoursFunction('[Вчора '),\n            nextWeek: processHoursFunction('[У] dddd ['),\n            lastWeek: function () {\n                switch (this.day()) {\n                    case 0:\n                    case 3:\n                    case 5:\n                    case 6:\n                        return processHoursFunction('[Минулої] dddd [').call(this);\n                    case 1:\n                    case 2:\n                    case 4:\n                        return processHoursFunction('[Минулого] dddd [').call(this);\n                }\n            },\n            sameElse: 'L'\n        },\n        relativeTime : {\n            future : 'за %s',\n            past : '%s тому',\n            s : 'декілька секунд',\n            ss : relativeTimeWithPlural,\n            m : relativeTimeWithPlural,\n            mm : relativeTimeWithPlural,\n            h : 'годину',\n            hh : relativeTimeWithPlural,\n            d : 'день',\n            dd : relativeTimeWithPlural,\n            M : 'місяць',\n            MM : relativeTimeWithPlural,\n            y : 'рік',\n            yy : relativeTimeWithPlural\n        },\n        // M. E.: those two are virtually unused but a user might want to implement them for his/her website for some reason\n        meridiemParse: /ночі|ранку|дня|вечора/,\n        isPM: function (input) {\n            return /^(дня|вечора)$/.test(input);\n        },\n        meridiem : function (hour, minute, isLower) {\n            if (hour < 4) {\n                return 'ночі';\n            } else if (hour < 12) {\n                return 'ранку';\n            } else if (hour < 17) {\n                return 'дня';\n            } else {\n                return 'вечора';\n            }\n        },\n        dayOfMonthOrdinalParse: /\\d{1,2}-(й|го)/,\n        ordinal: function (number, period) {\n            switch (period) {\n                case 'M':\n                case 'd':\n                case 'DDD':\n                case 'w':\n                case 'W':\n                    return number + '-й';\n                case 'D':\n                    return number + '-го';\n                default:\n                    return number;\n            }\n        },\n        week : {\n            dow : 1, // Monday is the first day of the week.\n            doy : 7  // The week that contains Jan 7th is the first week of the year.\n        }\n    });\n\n    return uk;\n\n})));\n","module.exports = function (exec) {\n  try {\n    return !!exec();\n  } catch (error) {\n    return true;\n  }\n};\n","var fails = require('../internals/fails');\n\n// Thank's IE8 for his funny defineProperty\nmodule.exports = !fails(function () {\n  return Object.defineProperty({}, 1, { get: function () { return 7; } })[1] != 7;\n});\n","'use strict';\nvar nativePropertyIsEnumerable = {}.propertyIsEnumerable;\nvar getOwnPropertyDescriptor = Object.getOwnPropertyDescriptor;\n\n// Nashorn ~ JDK8 bug\nvar NASHORN_BUG = getOwnPropertyDescriptor && !nativePropertyIsEnumerable.call({ 1: 2 }, 1);\n\n// `Object.prototype.propertyIsEnumerable` method implementation\n// https://tc39.github.io/ecma262/#sec-object.prototype.propertyisenumerable\nexports.f = NASHORN_BUG ? function propertyIsEnumerable(V) {\n  var descriptor = getOwnPropertyDescriptor(this, V);\n  return !!descriptor && descriptor.enumerable;\n} : nativePropertyIsEnumerable;\n","module.exports = function (bitmap, value) {\n  return {\n    enumerable: !(bitmap & 1),\n    configurable: !(bitmap & 2),\n    writable: !(bitmap & 4),\n    value: value\n  };\n};\n","var toString = {}.toString;\n\nmodule.exports = function (it) {\n  return toString.call(it).slice(8, -1);\n};\n","var fails = require('../internals/fails');\nvar classof = require('../internals/classof-raw');\n\nvar split = ''.split;\n\n// fallback for non-array-like ES3 and non-enumerable old V8 strings\nmodule.exports = fails(function () {\n  // throws an error in rhino, see https://github.com/mozilla/rhino/issues/346\n  // eslint-disable-next-line no-prototype-builtins\n  return !Object('z').propertyIsEnumerable(0);\n}) ? function (it) {\n  return classof(it) == 'String' ? split.call(it, '') : Object(it);\n} : Object;\n","// `RequireObjectCoercible` abstract operation\n// https://tc39.github.io/ecma262/#sec-requireobjectcoercible\nmodule.exports = function (it) {\n  if (it == undefined) throw TypeError(\"Can't call method on \" + it);\n  return it;\n};\n","// toObject with fallback for non-array-like ES3 strings\nvar IndexedObject = require('../internals/indexed-object');\nvar requireObjectCoercible = require('../internals/require-object-coercible');\n\nmodule.exports = function (it) {\n  return IndexedObject(requireObjectCoercible(it));\n};\n","module.exports = function (it) {\n  return typeof it === 'object' ? it !== null : typeof it === 'function';\n};\n","var isObject = require('../internals/is-object');\n\n// `ToPrimitive` abstract operation\n// https://tc39.github.io/ecma262/#sec-toprimitive\n// instead of the ES6 spec version, we didn't implement @@toPrimitive case\n// and the second argument - flag - preferred type is a string\nmodule.exports = function (input, PREFERRED_STRING) {\n  if (!isObject(input)) return input;\n  var fn, val;\n  if (PREFERRED_STRING && typeof (fn = input.toString) == 'function' && !isObject(val = fn.call(input))) return val;\n  if (typeof (fn = input.valueOf) == 'function' && !isObject(val = fn.call(input))) return val;\n  if (!PREFERRED_STRING && typeof (fn = input.toString) == 'function' && !isObject(val = fn.call(input))) return val;\n  throw TypeError(\"Can't convert object to primitive value\");\n};\n","var hasOwnProperty = {}.hasOwnProperty;\n\nmodule.exports = function (it, key) {\n  return hasOwnProperty.call(it, key);\n};\n","var global = require('../internals/global');\nvar isObject = require('../internals/is-object');\n\nvar document = global.document;\n// typeof document.createElement is 'object' in old IE\nvar EXISTS = isObject(document) && isObject(document.createElement);\n\nmodule.exports = function (it) {\n  return EXISTS ? document.createElement(it) : {};\n};\n","var DESCRIPTORS = require('../internals/descriptors');\nvar fails = require('../internals/fails');\nvar createElement = require('../internals/document-create-element');\n\n// Thank's IE8 for his funny defineProperty\nmodule.exports = !DESCRIPTORS && !fails(function () {\n  return Object.defineProperty(createElement('div'), 'a', {\n    get: function () { return 7; }\n  }).a != 7;\n});\n","var DESCRIPTORS = require('../internals/descriptors');\nvar propertyIsEnumerableModule = require('../internals/object-property-is-enumerable');\nvar createPropertyDescriptor = require('../internals/create-property-descriptor');\nvar toIndexedObject = require('../internals/to-indexed-object');\nvar toPrimitive = require('../internals/to-primitive');\nvar has = require('../internals/has');\nvar IE8_DOM_DEFINE = require('../internals/ie8-dom-define');\n\nvar nativeGetOwnPropertyDescriptor = Object.getOwnPropertyDescriptor;\n\n// `Object.getOwnPropertyDescriptor` method\n// https://tc39.github.io/ecma262/#sec-object.getownpropertydescriptor\nexports.f = DESCRIPTORS ? nativeGetOwnPropertyDescriptor : function getOwnPropertyDescriptor(O, P) {\n  O = toIndexedObject(O);\n  P = toPrimitive(P, true);\n  if (IE8_DOM_DEFINE) try {\n    return nativeGetOwnPropertyDescriptor(O, P);\n  } catch (error) { /* empty */ }\n  if (has(O, P)) return createPropertyDescriptor(!propertyIsEnumerableModule.f.call(O, P), O[P]);\n};\n","var fails = require('../internals/fails');\n\nvar replacement = /#|\\.prototype\\./;\n\nvar isForced = function (feature, detection) {\n  var value = data[normalize(feature)];\n  return value == POLYFILL ? true\n    : value == NATIVE ? false\n    : typeof detection == 'function' ? fails(detection)\n    : !!detection;\n};\n\nvar normalize = isForced.normalize = function (string) {\n  return String(string).replace(replacement, '.').toLowerCase();\n};\n\nvar data = isForced.data = {};\nvar NATIVE = isForced.NATIVE = 'N';\nvar POLYFILL = isForced.POLYFILL = 'P';\n\nmodule.exports = isForced;\n","module.exports = {};\n","module.exports = function (it) {\n  if (typeof it != 'function') {\n    throw TypeError(String(it) + ' is not a function');\n  } return it;\n};\n","var aFunction = require('../internals/a-function');\n\n// optional / simple context binding\nmodule.exports = function (fn, that, length) {\n  aFunction(fn);\n  if (that === undefined) return fn;\n  switch (length) {\n    case 0: return function () {\n      return fn.call(that);\n    };\n    case 1: return function (a) {\n      return fn.call(that, a);\n    };\n    case 2: return function (a, b) {\n      return fn.call(that, a, b);\n    };\n    case 3: return function (a, b, c) {\n      return fn.call(that, a, b, c);\n    };\n  }\n  return function (/* ...args */) {\n    return fn.apply(that, arguments);\n  };\n};\n","var isObject = require('../internals/is-object');\n\nmodule.exports = function (it) {\n  if (!isObject(it)) {\n    throw TypeError(String(it) + ' is not an object');\n  } return it;\n};\n","var DESCRIPTORS = require('../internals/descriptors');\nvar IE8_DOM_DEFINE = require('../internals/ie8-dom-define');\nvar anObject = require('../internals/an-object');\nvar toPrimitive = require('../internals/to-primitive');\n\nvar nativeDefineProperty = Object.defineProperty;\n\n// `Object.defineProperty` method\n// https://tc39.github.io/ecma262/#sec-object.defineproperty\nexports.f = DESCRIPTORS ? nativeDefineProperty : function defineProperty(O, P, Attributes) {\n  anObject(O);\n  P = toPrimitive(P, true);\n  anObject(Attributes);\n  if (IE8_DOM_DEFINE) try {\n    return nativeDefineProperty(O, P, Attributes);\n  } catch (error) { /* empty */ }\n  if ('get' in Attributes || 'set' in Attributes) throw TypeError('Accessors not supported');\n  if ('value' in Attributes) O[P] = Attributes.value;\n  return O;\n};\n","var DESCRIPTORS = require('../internals/descriptors');\nvar definePropertyModule = require('../internals/object-define-property');\nvar createPropertyDescriptor = require('../internals/create-property-descriptor');\n\nmodule.exports = DESCRIPTORS ? function (object, key, value) {\n  return definePropertyModule.f(object, key, createPropertyDescriptor(1, value));\n} : function (object, key, value) {\n  object[key] = value;\n  return object;\n};\n","'use strict';\nvar global = require('../internals/global');\nvar getOwnPropertyDescriptor = require('../internals/object-get-own-property-descriptor').f;\nvar isForced = require('../internals/is-forced');\nvar path = require('../internals/path');\nvar bind = require('../internals/function-bind-context');\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\nvar has = require('../internals/has');\n\nvar wrapConstructor = function (NativeConstructor) {\n  var Wrapper = function (a, b, c) {\n    if (this instanceof NativeConstructor) {\n      switch (arguments.length) {\n        case 0: return new NativeConstructor();\n        case 1: return new NativeConstructor(a);\n        case 2: return new NativeConstructor(a, b);\n      } return new NativeConstructor(a, b, c);\n    } return NativeConstructor.apply(this, arguments);\n  };\n  Wrapper.prototype = NativeConstructor.prototype;\n  return Wrapper;\n};\n\n/*\n  options.target      - name of the target object\n  options.global      - target is the global object\n  options.stat        - export as static methods of target\n  options.proto       - export as prototype methods of target\n  options.real        - real prototype method for the `pure` version\n  options.forced      - export even if the native feature is available\n  options.bind        - bind methods to the target, required for the `pure` version\n  options.wrap        - wrap constructors to preventing global pollution, required for the `pure` version\n  options.unsafe      - use the simple assignment of property instead of delete + defineProperty\n  options.sham        - add a flag to not completely full polyfills\n  options.enumerable  - export as enumerable property\n  options.noTargetGet - prevent calling a getter on target\n*/\nmodule.exports = function (options, source) {\n  var TARGET = options.target;\n  var GLOBAL = options.global;\n  var STATIC = options.stat;\n  var PROTO = options.proto;\n\n  var nativeSource = GLOBAL ? global : STATIC ? global[TARGET] : (global[TARGET] || {}).prototype;\n\n  var target = GLOBAL ? path : path[TARGET] || (path[TARGET] = {});\n  var targetPrototype = target.prototype;\n\n  var FORCED, USE_NATIVE, VIRTUAL_PROTOTYPE;\n  var key, sourceProperty, targetProperty, nativeProperty, resultProperty, descriptor;\n\n  for (key in source) {\n    FORCED = isForced(GLOBAL ? key : TARGET + (STATIC ? '.' : '#') + key, options.forced);\n    // contains in native\n    USE_NATIVE = !FORCED && nativeSource && has(nativeSource, key);\n\n    targetProperty = target[key];\n\n    if (USE_NATIVE) if (options.noTargetGet) {\n      descriptor = getOwnPropertyDescriptor(nativeSource, key);\n      nativeProperty = descriptor && descriptor.value;\n    } else nativeProperty = nativeSource[key];\n\n    // export native or implementation\n    sourceProperty = (USE_NATIVE && nativeProperty) ? nativeProperty : source[key];\n\n    if (USE_NATIVE && typeof targetProperty === typeof sourceProperty) continue;\n\n    // bind timers to global for call from export context\n    if (options.bind && USE_NATIVE) resultProperty = bind(sourceProperty, global);\n    // wrap global constructors for prevent changs in this version\n    else if (options.wrap && USE_NATIVE) resultProperty = wrapConstructor(sourceProperty);\n    // make static versions for prototype methods\n    else if (PROTO && typeof sourceProperty == 'function') resultProperty = bind(Function.call, sourceProperty);\n    // default case\n    else resultProperty = sourceProperty;\n\n    // add a flag to not completely full polyfills\n    if (options.sham || (sourceProperty && sourceProperty.sham) || (targetProperty && targetProperty.sham)) {\n      createNonEnumerableProperty(resultProperty, 'sham', true);\n    }\n\n    target[key] = resultProperty;\n\n    if (PROTO) {\n      VIRTUAL_PROTOTYPE = TARGET + 'Prototype';\n      if (!has(path, VIRTUAL_PROTOTYPE)) {\n        createNonEnumerableProperty(path, VIRTUAL_PROTOTYPE, {});\n      }\n      // export virtual prototype methods\n      path[VIRTUAL_PROTOTYPE][key] = sourceProperty;\n      // export real prototype methods\n      if (options.real && targetPrototype && !targetPrototype[key]) {\n        createNonEnumerableProperty(targetPrototype, key, sourceProperty);\n      }\n    }\n  }\n};\n","var $ = require('../internals/export');\nvar DESCRIPTORS = require('../internals/descriptors');\nvar objectDefinePropertyModile = require('../internals/object-define-property');\n\n// `Object.defineProperty` method\n// https://tc39.github.io/ecma262/#sec-object.defineproperty\n$({ target: 'Object', stat: true, forced: !DESCRIPTORS, sham: !DESCRIPTORS }, {\n  defineProperty: objectDefinePropertyModile.f\n});\n","require('../../modules/es.object.define-property');\nvar path = require('../../internals/path');\n\nvar Object = path.Object;\n\nvar defineProperty = module.exports = function defineProperty(it, key, desc) {\n  return Object.defineProperty(it, key, desc);\n};\n\nif (Object.defineProperty.sham) defineProperty.sham = true;\n","var parent = require('../../es/object/define-property');\n\nmodule.exports = parent;\n","var ceil = Math.ceil;\nvar floor = Math.floor;\n\n// `ToInteger` abstract operation\n// https://tc39.github.io/ecma262/#sec-tointeger\nmodule.exports = function (argument) {\n  return isNaN(argument = +argument) ? 0 : (argument > 0 ? floor : ceil)(argument);\n};\n","var toInteger = require('../internals/to-integer');\n\nvar min = Math.min;\n\n// `ToLength` abstract operation\n// https://tc39.github.io/ecma262/#sec-tolength\nmodule.exports = function (argument) {\n  return argument > 0 ? min(toInteger(argument), 0x1FFFFFFFFFFFFF) : 0; // 2 ** 53 - 1 == 9007199254740991\n};\n","var toInteger = require('../internals/to-integer');\n\nvar max = Math.max;\nvar min = Math.min;\n\n// Helper for a popular repeating case of the spec:\n// Let integer be ? ToInteger(index).\n// If integer < 0, let result be max((length + integer), 0); else let result be min(integer, length).\nmodule.exports = function (index, length) {\n  var integer = toInteger(index);\n  return integer < 0 ? max(integer + length, 0) : min(integer, length);\n};\n","var toIndexedObject = require('../internals/to-indexed-object');\nvar toLength = require('../internals/to-length');\nvar toAbsoluteIndex = require('../internals/to-absolute-index');\n\n// `Array.prototype.{ indexOf, includes }` methods implementation\nvar createMethod = function (IS_INCLUDES) {\n  return function ($this, el, fromIndex) {\n    var O = toIndexedObject($this);\n    var length = toLength(O.length);\n    var index = toAbsoluteIndex(fromIndex, length);\n    var value;\n    // Array#includes uses SameValueZero equality algorithm\n    // eslint-disable-next-line no-self-compare\n    if (IS_INCLUDES && el != el) while (length > index) {\n      value = O[index++];\n      // eslint-disable-next-line no-self-compare\n      if (value != value) return true;\n    // Array#indexOf ignores holes, Array#includes - not\n    } else for (;length > index; index++) {\n      if ((IS_INCLUDES || index in O) && O[index] === el) return IS_INCLUDES || index || 0;\n    } return !IS_INCLUDES && -1;\n  };\n};\n\nmodule.exports = {\n  // `Array.prototype.includes` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.includes\n  includes: createMethod(true),\n  // `Array.prototype.indexOf` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.indexof\n  indexOf: createMethod(false)\n};\n","module.exports = {};\n","var has = require('../internals/has');\nvar toIndexedObject = require('../internals/to-indexed-object');\nvar indexOf = require('../internals/array-includes').indexOf;\nvar hiddenKeys = require('../internals/hidden-keys');\n\nmodule.exports = function (object, names) {\n  var O = toIndexedObject(object);\n  var i = 0;\n  var result = [];\n  var key;\n  for (key in O) !has(hiddenKeys, key) && has(O, key) && result.push(key);\n  // Don't enum bug & hidden keys\n  while (names.length > i) if (has(O, key = names[i++])) {\n    ~indexOf(result, key) || result.push(key);\n  }\n  return result;\n};\n","// IE8- don't enum bug keys\nmodule.exports = [\n  'constructor',\n  'hasOwnProperty',\n  'isPrototypeOf',\n  'propertyIsEnumerable',\n  'toLocaleString',\n  'toString',\n  'valueOf'\n];\n","var internalObjectKeys = require('../internals/object-keys-internal');\nvar enumBugKeys = require('../internals/enum-bug-keys');\n\n// `Object.keys` method\n// https://tc39.github.io/ecma262/#sec-object.keys\nmodule.exports = Object.keys || function keys(O) {\n  return internalObjectKeys(O, enumBugKeys);\n};\n","var DESCRIPTORS = require('../internals/descriptors');\nvar definePropertyModule = require('../internals/object-define-property');\nvar anObject = require('../internals/an-object');\nvar objectKeys = require('../internals/object-keys');\n\n// `Object.defineProperties` method\n// https://tc39.github.io/ecma262/#sec-object.defineproperties\nmodule.exports = DESCRIPTORS ? Object.defineProperties : function defineProperties(O, Properties) {\n  anObject(O);\n  var keys = objectKeys(Properties);\n  var length = keys.length;\n  var index = 0;\n  var key;\n  while (length > index) definePropertyModule.f(O, key = keys[index++], Properties[key]);\n  return O;\n};\n","var $ = require('../internals/export');\nvar DESCRIPTORS = require('../internals/descriptors');\nvar defineProperties = require('../internals/object-define-properties');\n\n// `Object.defineProperties` method\n// https://tc39.github.io/ecma262/#sec-object.defineproperties\n$({ target: 'Object', stat: true, forced: !DESCRIPTORS, sham: !DESCRIPTORS }, {\n  defineProperties: defineProperties\n});\n","require('../../modules/es.object.define-properties');\nvar path = require('../../internals/path');\n\nvar Object = path.Object;\n\nvar defineProperties = module.exports = function defineProperties(T, D) {\n  return Object.defineProperties(T, D);\n};\n\nif (Object.defineProperties.sham) defineProperties.sham = true;\n","var path = require('../internals/path');\nvar global = require('../internals/global');\n\nvar aFunction = function (variable) {\n  return typeof variable == 'function' ? variable : undefined;\n};\n\nmodule.exports = function (namespace, method) {\n  return arguments.length < 2 ? aFunction(path[namespace]) || aFunction(global[namespace])\n    : path[namespace] && path[namespace][method] || global[namespace] && global[namespace][method];\n};\n","var internalObjectKeys = require('../internals/object-keys-internal');\nvar enumBugKeys = require('../internals/enum-bug-keys');\n\nvar hiddenKeys = enumBugKeys.concat('length', 'prototype');\n\n// `Object.getOwnPropertyNames` method\n// https://tc39.github.io/ecma262/#sec-object.getownpropertynames\nexports.f = Object.getOwnPropertyNames || function getOwnPropertyNames(O) {\n  return internalObjectKeys(O, hiddenKeys);\n};\n","exports.f = Object.getOwnPropertySymbols;\n","var getBuiltIn = require('../internals/get-built-in');\nvar getOwnPropertyNamesModule = require('../internals/object-get-own-property-names');\nvar getOwnPropertySymbolsModule = require('../internals/object-get-own-property-symbols');\nvar anObject = require('../internals/an-object');\n\n// all object keys, includes non-enumerable and symbols\nmodule.exports = getBuiltIn('Reflect', 'ownKeys') || function ownKeys(it) {\n  var keys = getOwnPropertyNamesModule.f(anObject(it));\n  var getOwnPropertySymbols = getOwnPropertySymbolsModule.f;\n  return getOwnPropertySymbols ? keys.concat(getOwnPropertySymbols(it)) : keys;\n};\n","'use strict';\nvar toPrimitive = require('../internals/to-primitive');\nvar definePropertyModule = require('../internals/object-define-property');\nvar createPropertyDescriptor = require('../internals/create-property-descriptor');\n\nmodule.exports = function (object, key, value) {\n  var propertyKey = toPrimitive(key);\n  if (propertyKey in object) definePropertyModule.f(object, propertyKey, createPropertyDescriptor(0, value));\n  else object[propertyKey] = value;\n};\n","var $ = require('../internals/export');\nvar DESCRIPTORS = require('../internals/descriptors');\nvar ownKeys = require('../internals/own-keys');\nvar toIndexedObject = require('../internals/to-indexed-object');\nvar getOwnPropertyDescriptorModule = require('../internals/object-get-own-property-descriptor');\nvar createProperty = require('../internals/create-property');\n\n// `Object.getOwnPropertyDescriptors` method\n// https://tc39.github.io/ecma262/#sec-object.getownpropertydescriptors\n$({ target: 'Object', stat: true, sham: !DESCRIPTORS }, {\n  getOwnPropertyDescriptors: function getOwnPropertyDescriptors(object) {\n    var O = toIndexedObject(object);\n    var getOwnPropertyDescriptor = getOwnPropertyDescriptorModule.f;\n    var keys = ownKeys(O);\n    var result = {};\n    var index = 0;\n    var key, descriptor;\n    while (keys.length > index) {\n      descriptor = getOwnPropertyDescriptor(O, key = keys[index++]);\n      if (descriptor !== undefined) createProperty(result, key, descriptor);\n    }\n    return result;\n  }\n});\n","require('../../modules/es.object.get-own-property-descriptors');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Object.getOwnPropertyDescriptors;\n","var $ = require('../internals/export');\nvar fails = require('../internals/fails');\nvar toIndexedObject = require('../internals/to-indexed-object');\nvar nativeGetOwnPropertyDescriptor = require('../internals/object-get-own-property-descriptor').f;\nvar DESCRIPTORS = require('../internals/descriptors');\n\nvar FAILS_ON_PRIMITIVES = fails(function () { nativeGetOwnPropertyDescriptor(1); });\nvar FORCED = !DESCRIPTORS || FAILS_ON_PRIMITIVES;\n\n// `Object.getOwnPropertyDescriptor` method\n// https://tc39.github.io/ecma262/#sec-object.getownpropertydescriptor\n$({ target: 'Object', stat: true, forced: FORCED, sham: !DESCRIPTORS }, {\n  getOwnPropertyDescriptor: function getOwnPropertyDescriptor(it, key) {\n    return nativeGetOwnPropertyDescriptor(toIndexedObject(it), key);\n  }\n});\n","var anObject = require('../internals/an-object');\nvar defineProperties = require('../internals/object-define-properties');\nvar enumBugKeys = require('../internals/enum-bug-keys');\nvar hiddenKeys = require('../internals/hidden-keys');\nvar html = require('../internals/html');\nvar documentCreateElement = require('../internals/document-create-element');\nvar sharedKey = require('../internals/shared-key');\n\nvar GT = '>';\nvar LT = '<';\nvar PROTOTYPE = 'prototype';\nvar SCRIPT = 'script';\nvar IE_PROTO = sharedKey('IE_PROTO');\n\nvar EmptyConstructor = function () { /* empty */ };\n\nvar scriptTag = function (content) {\n  return LT + SCRIPT + GT + content + LT + '/' + SCRIPT + GT;\n};\n\n// Create object with fake `null` prototype: use ActiveX Object with cleared prototype\nvar NullProtoObjectViaActiveX = function (activeXDocument) {\n  activeXDocument.write(scriptTag(''));\n  activeXDocument.close();\n  var temp = activeXDocument.parentWindow.Object;\n  activeXDocument = null; // avoid memory leak\n  return temp;\n};\n\n// Create object with fake `null` prototype: use iframe Object with cleared prototype\nvar NullProtoObjectViaIFrame = function () {\n  // Thrash, waste and sodomy: IE GC bug\n  var iframe = documentCreateElement('iframe');\n  var JS = 'java' + SCRIPT + ':';\n  var iframeDocument;\n  iframe.style.display = 'none';\n  html.appendChild(iframe);\n  // https://github.com/zloirock/core-js/issues/475\n  iframe.src = String(JS);\n  iframeDocument = iframe.contentWindow.document;\n  iframeDocument.open();\n  iframeDocument.write(scriptTag('document.F=Object'));\n  iframeDocument.close();\n  return iframeDocument.F;\n};\n\n// Check for document.domain and active x support\n// No need to use active x approach when document.domain is not set\n// see https://github.com/es-shims/es5-shim/issues/150\n// variation of https://github.com/kitcambridge/es5-shim/commit/4f738ac066346\n// avoid IE GC bug\nvar activeXDocument;\nvar NullProtoObject = function () {\n  try {\n    /* global ActiveXObject */\n    activeXDocument = document.domain && new ActiveXObject('htmlfile');\n  } catch (error) { /* ignore */ }\n  NullProtoObject = activeXDocument ? NullProtoObjectViaActiveX(activeXDocument) : NullProtoObjectViaIFrame();\n  var length = enumBugKeys.length;\n  while (length--) delete NullProtoObject[PROTOTYPE][enumBugKeys[length]];\n  return NullProtoObject();\n};\n\nhiddenKeys[IE_PROTO] = true;\n\n// `Object.create` method\n// https://tc39.github.io/ecma262/#sec-object.create\nmodule.exports = Object.create || function create(O, Properties) {\n  var result;\n  if (O !== null) {\n    EmptyConstructor[PROTOTYPE] = anObject(O);\n    result = new EmptyConstructor();\n    EmptyConstructor[PROTOTYPE] = null;\n    // add \"__proto__\" for Object.getPrototypeOf polyfill\n    result[IE_PROTO] = O;\n  } else result = NullProtoObject();\n  return Properties === undefined ? result : defineProperties(result, Properties);\n};\n","require('../../modules/es.object.get-own-property-descriptor');\nvar path = require('../../internals/path');\n\nvar Object = path.Object;\n\nvar getOwnPropertyDescriptor = module.exports = function getOwnPropertyDescriptor(it, key) {\n  return Object.getOwnPropertyDescriptor(it, key);\n};\n\nif (Object.getOwnPropertyDescriptor.sham) getOwnPropertyDescriptor.sham = true;\n","var fails = require('../internals/fails');\n\nmodule.exports = !!Object.getOwnPropertySymbols && !fails(function () {\n  // Chrome 38 Symbol has incorrect toString conversion\n  // eslint-disable-next-line no-undef\n  return !String(Symbol());\n});\n","var NATIVE_SYMBOL = require('../internals/native-symbol');\n\nmodule.exports = NATIVE_SYMBOL\n  // eslint-disable-next-line no-undef\n  && !Symbol.sham\n  // eslint-disable-next-line no-undef\n  && typeof Symbol.iterator == 'symbol';\n","var classof = require('../internals/classof-raw');\n\n// `IsArray` abstract operation\n// https://tc39.github.io/ecma262/#sec-isarray\nmodule.exports = Array.isArray || function isArray(arg) {\n  return classof(arg) == 'Array';\n};\n","var requireObjectCoercible = require('../internals/require-object-coercible');\n\n// `ToObject` abstract operation\n// https://tc39.github.io/ecma262/#sec-toobject\nmodule.exports = function (argument) {\n  return Object(requireObjectCoercible(argument));\n};\n","var getBuiltIn = require('../internals/get-built-in');\n\nmodule.exports = getBuiltIn('document', 'documentElement');\n","var global = require('../internals/global');\nvar setGlobal = require('../internals/set-global');\n\nvar SHARED = '__core-js_shared__';\nvar store = global[SHARED] || setGlobal(SHARED, {});\n\nmodule.exports = store;\n","var global = require('../internals/global');\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\n\nmodule.exports = function (key, value) {\n  try {\n    createNonEnumerableProperty(global, key, value);\n  } catch (error) {\n    global[key] = value;\n  } return value;\n};\n","var IS_PURE = require('../internals/is-pure');\nvar store = require('../internals/shared-store');\n\n(module.exports = function (key, value) {\n  return store[key] || (store[key] = value !== undefined ? value : {});\n})('versions', []).push({\n  version: '3.6.4',\n  mode: IS_PURE ? 'pure' : 'global',\n  copyright: '© 2020 Denis Pushkarev (zloirock.ru)'\n});\n","var id = 0;\nvar postfix = Math.random();\n\nmodule.exports = function (key) {\n  return 'Symbol(' + String(key === undefined ? '' : key) + ')_' + (++id + postfix).toString(36);\n};\n","var shared = require('../internals/shared');\nvar uid = require('../internals/uid');\n\nvar keys = shared('keys');\n\nmodule.exports = function (key) {\n  return keys[key] || (keys[key] = uid(key));\n};\n","var toIndexedObject = require('../internals/to-indexed-object');\nvar nativeGetOwnPropertyNames = require('../internals/object-get-own-property-names').f;\n\nvar toString = {}.toString;\n\nvar windowNames = typeof window == 'object' && window && Object.getOwnPropertyNames\n  ? Object.getOwnPropertyNames(window) : [];\n\nvar getWindowNames = function (it) {\n  try {\n    return nativeGetOwnPropertyNames(it);\n  } catch (error) {\n    return windowNames.slice();\n  }\n};\n\n// fallback for IE11 buggy Object.getOwnPropertyNames with iframe and window\nmodule.exports.f = function getOwnPropertyNames(it) {\n  return windowNames && toString.call(it) == '[object Window]'\n    ? getWindowNames(it)\n    : nativeGetOwnPropertyNames(toIndexedObject(it));\n};\n","var createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\n\nmodule.exports = function (target, key, value, options) {\n  if (options && options.enumerable) target[key] = value;\n  else createNonEnumerableProperty(target, key, value);\n};\n","var global = require('../internals/global');\nvar shared = require('../internals/shared');\nvar has = require('../internals/has');\nvar uid = require('../internals/uid');\nvar NATIVE_SYMBOL = require('../internals/native-symbol');\nvar USE_SYMBOL_AS_UID = require('../internals/use-symbol-as-uid');\n\nvar WellKnownSymbolsStore = shared('wks');\nvar Symbol = global.Symbol;\nvar createWellKnownSymbol = USE_SYMBOL_AS_UID ? Symbol : Symbol && Symbol.withoutSetter || uid;\n\nmodule.exports = function (name) {\n  if (!has(WellKnownSymbolsStore, name)) {\n    if (NATIVE_SYMBOL && has(Symbol, name)) WellKnownSymbolsStore[name] = Symbol[name];\n    else WellKnownSymbolsStore[name] = createWellKnownSymbol('Symbol.' + name);\n  } return WellKnownSymbolsStore[name];\n};\n","var wellKnownSymbol = require('../internals/well-known-symbol');\n\nexports.f = wellKnownSymbol;\n","var path = require('../internals/path');\nvar has = require('../internals/has');\nvar wrappedWellKnownSymbolModule = require('../internals/well-known-symbol-wrapped');\nvar defineProperty = require('../internals/object-define-property').f;\n\nmodule.exports = function (NAME) {\n  var Symbol = path.Symbol || (path.Symbol = {});\n  if (!has(Symbol, NAME)) defineProperty(Symbol, NAME, {\n    value: wrappedWellKnownSymbolModule.f(NAME)\n  });\n};\n","var wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar TO_STRING_TAG = wellKnownSymbol('toStringTag');\nvar test = {};\n\ntest[TO_STRING_TAG] = 'z';\n\nmodule.exports = String(test) === '[object z]';\n","var TO_STRING_TAG_SUPPORT = require('../internals/to-string-tag-support');\nvar classofRaw = require('../internals/classof-raw');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar TO_STRING_TAG = wellKnownSymbol('toStringTag');\n// ES3 wrong here\nvar CORRECT_ARGUMENTS = classofRaw(function () { return arguments; }()) == 'Arguments';\n\n// fallback for IE11 Script Access Denied error\nvar tryGet = function (it, key) {\n  try {\n    return it[key];\n  } catch (error) { /* empty */ }\n};\n\n// getting tag from ES6+ `Object.prototype.toString`\nmodule.exports = TO_STRING_TAG_SUPPORT ? classofRaw : function (it) {\n  var O, tag, result;\n  return it === undefined ? 'Undefined' : it === null ? 'Null'\n    // @@toStringTag case\n    : typeof (tag = tryGet(O = Object(it), TO_STRING_TAG)) == 'string' ? tag\n    // builtinTag case\n    : CORRECT_ARGUMENTS ? classofRaw(O)\n    // ES3 arguments fallback\n    : (result = classofRaw(O)) == 'Object' && typeof O.callee == 'function' ? 'Arguments' : result;\n};\n","'use strict';\nvar TO_STRING_TAG_SUPPORT = require('../internals/to-string-tag-support');\nvar classof = require('../internals/classof');\n\n// `Object.prototype.toString` method implementation\n// https://tc39.github.io/ecma262/#sec-object.prototype.tostring\nmodule.exports = TO_STRING_TAG_SUPPORT ? {}.toString : function toString() {\n  return '[object ' + classof(this) + ']';\n};\n","var TO_STRING_TAG_SUPPORT = require('../internals/to-string-tag-support');\nvar defineProperty = require('../internals/object-define-property').f;\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\nvar has = require('../internals/has');\nvar toString = require('../internals/object-to-string');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar TO_STRING_TAG = wellKnownSymbol('toStringTag');\n\nmodule.exports = function (it, TAG, STATIC, SET_METHOD) {\n  if (it) {\n    var target = STATIC ? it : it.prototype;\n    if (!has(target, TO_STRING_TAG)) {\n      defineProperty(target, TO_STRING_TAG, { configurable: true, value: TAG });\n    }\n    if (SET_METHOD && !TO_STRING_TAG_SUPPORT) {\n      createNonEnumerableProperty(target, 'toString', toString);\n    }\n  }\n};\n","var store = require('../internals/shared-store');\n\nvar functionToString = Function.toString;\n\n// this helper broken in `3.4.1-3.4.4`, so we can't use `shared` helper\nif (typeof store.inspectSource != 'function') {\n  store.inspectSource = function (it) {\n    return functionToString.call(it);\n  };\n}\n\nmodule.exports = store.inspectSource;\n","var NATIVE_WEAK_MAP = require('../internals/native-weak-map');\nvar global = require('../internals/global');\nvar isObject = require('../internals/is-object');\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\nvar objectHas = require('../internals/has');\nvar sharedKey = require('../internals/shared-key');\nvar hiddenKeys = require('../internals/hidden-keys');\n\nvar WeakMap = global.WeakMap;\nvar set, get, has;\n\nvar enforce = function (it) {\n  return has(it) ? get(it) : set(it, {});\n};\n\nvar getterFor = function (TYPE) {\n  return function (it) {\n    var state;\n    if (!isObject(it) || (state = get(it)).type !== TYPE) {\n      throw TypeError('Incompatible receiver, ' + TYPE + ' required');\n    } return state;\n  };\n};\n\nif (NATIVE_WEAK_MAP) {\n  var store = new WeakMap();\n  var wmget = store.get;\n  var wmhas = store.has;\n  var wmset = store.set;\n  set = function (it, metadata) {\n    wmset.call(store, it, metadata);\n    return metadata;\n  };\n  get = function (it) {\n    return wmget.call(store, it) || {};\n  };\n  has = function (it) {\n    return wmhas.call(store, it);\n  };\n} else {\n  var STATE = sharedKey('state');\n  hiddenKeys[STATE] = true;\n  set = function (it, metadata) {\n    createNonEnumerableProperty(it, STATE, metadata);\n    return metadata;\n  };\n  get = function (it) {\n    return objectHas(it, STATE) ? it[STATE] : {};\n  };\n  has = function (it) {\n    return objectHas(it, STATE);\n  };\n}\n\nmodule.exports = {\n  set: set,\n  get: get,\n  has: has,\n  enforce: enforce,\n  getterFor: getterFor\n};\n","var global = require('../internals/global');\nvar inspectSource = require('../internals/inspect-source');\n\nvar WeakMap = global.WeakMap;\n\nmodule.exports = typeof WeakMap === 'function' && /native code/.test(inspectSource(WeakMap));\n","var isObject = require('../internals/is-object');\nvar isArray = require('../internals/is-array');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar SPECIES = wellKnownSymbol('species');\n\n// `ArraySpeciesCreate` abstract operation\n// https://tc39.github.io/ecma262/#sec-arrayspeciescreate\nmodule.exports = function (originalArray, length) {\n  var C;\n  if (isArray(originalArray)) {\n    C = originalArray.constructor;\n    // cross-realm fallback\n    if (typeof C == 'function' && (C === Array || isArray(C.prototype))) C = undefined;\n    else if (isObject(C)) {\n      C = C[SPECIES];\n      if (C === null) C = undefined;\n    }\n  } return new (C === undefined ? Array : C)(length === 0 ? 0 : length);\n};\n","var bind = require('../internals/function-bind-context');\nvar IndexedObject = require('../internals/indexed-object');\nvar toObject = require('../internals/to-object');\nvar toLength = require('../internals/to-length');\nvar arraySpeciesCreate = require('../internals/array-species-create');\n\nvar push = [].push;\n\n// `Array.prototype.{ forEach, map, filter, some, every, find, findIndex }` methods implementation\nvar createMethod = function (TYPE) {\n  var IS_MAP = TYPE == 1;\n  var IS_FILTER = TYPE == 2;\n  var IS_SOME = TYPE == 3;\n  var IS_EVERY = TYPE == 4;\n  var IS_FIND_INDEX = TYPE == 6;\n  var NO_HOLES = TYPE == 5 || IS_FIND_INDEX;\n  return function ($this, callbackfn, that, specificCreate) {\n    var O = toObject($this);\n    var self = IndexedObject(O);\n    var boundFunction = bind(callbackfn, that, 3);\n    var length = toLength(self.length);\n    var index = 0;\n    var create = specificCreate || arraySpeciesCreate;\n    var target = IS_MAP ? create($this, length) : IS_FILTER ? create($this, 0) : undefined;\n    var value, result;\n    for (;length > index; index++) if (NO_HOLES || index in self) {\n      value = self[index];\n      result = boundFunction(value, index, O);\n      if (TYPE) {\n        if (IS_MAP) target[index] = result; // map\n        else if (result) switch (TYPE) {\n          case 3: return true;              // some\n          case 5: return value;             // find\n          case 6: return index;             // findIndex\n          case 2: push.call(target, value); // filter\n        } else if (IS_EVERY) return false;  // every\n      }\n    }\n    return IS_FIND_INDEX ? -1 : IS_SOME || IS_EVERY ? IS_EVERY : target;\n  };\n};\n\nmodule.exports = {\n  // `Array.prototype.forEach` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.foreach\n  forEach: createMethod(0),\n  // `Array.prototype.map` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.map\n  map: createMethod(1),\n  // `Array.prototype.filter` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.filter\n  filter: createMethod(2),\n  // `Array.prototype.some` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.some\n  some: createMethod(3),\n  // `Array.prototype.every` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.every\n  every: createMethod(4),\n  // `Array.prototype.find` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.find\n  find: createMethod(5),\n  // `Array.prototype.findIndex` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.findIndex\n  findIndex: createMethod(6)\n};\n","'use strict';\nvar $ = require('../internals/export');\nvar global = require('../internals/global');\nvar getBuiltIn = require('../internals/get-built-in');\nvar IS_PURE = require('../internals/is-pure');\nvar DESCRIPTORS = require('../internals/descriptors');\nvar NATIVE_SYMBOL = require('../internals/native-symbol');\nvar USE_SYMBOL_AS_UID = require('../internals/use-symbol-as-uid');\nvar fails = require('../internals/fails');\nvar has = require('../internals/has');\nvar isArray = require('../internals/is-array');\nvar isObject = require('../internals/is-object');\nvar anObject = require('../internals/an-object');\nvar toObject = require('../internals/to-object');\nvar toIndexedObject = require('../internals/to-indexed-object');\nvar toPrimitive = require('../internals/to-primitive');\nvar createPropertyDescriptor = require('../internals/create-property-descriptor');\nvar nativeObjectCreate = require('../internals/object-create');\nvar objectKeys = require('../internals/object-keys');\nvar getOwnPropertyNamesModule = require('../internals/object-get-own-property-names');\nvar getOwnPropertyNamesExternal = require('../internals/object-get-own-property-names-external');\nvar getOwnPropertySymbolsModule = require('../internals/object-get-own-property-symbols');\nvar getOwnPropertyDescriptorModule = require('../internals/object-get-own-property-descriptor');\nvar definePropertyModule = require('../internals/object-define-property');\nvar propertyIsEnumerableModule = require('../internals/object-property-is-enumerable');\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\nvar redefine = require('../internals/redefine');\nvar shared = require('../internals/shared');\nvar sharedKey = require('../internals/shared-key');\nvar hiddenKeys = require('../internals/hidden-keys');\nvar uid = require('../internals/uid');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\nvar wrappedWellKnownSymbolModule = require('../internals/well-known-symbol-wrapped');\nvar defineWellKnownSymbol = require('../internals/define-well-known-symbol');\nvar setToStringTag = require('../internals/set-to-string-tag');\nvar InternalStateModule = require('../internals/internal-state');\nvar $forEach = require('../internals/array-iteration').forEach;\n\nvar HIDDEN = sharedKey('hidden');\nvar SYMBOL = 'Symbol';\nvar PROTOTYPE = 'prototype';\nvar TO_PRIMITIVE = wellKnownSymbol('toPrimitive');\nvar setInternalState = InternalStateModule.set;\nvar getInternalState = InternalStateModule.getterFor(SYMBOL);\nvar ObjectPrototype = Object[PROTOTYPE];\nvar $Symbol = global.Symbol;\nvar $stringify = getBuiltIn('JSON', 'stringify');\nvar nativeGetOwnPropertyDescriptor = getOwnPropertyDescriptorModule.f;\nvar nativeDefineProperty = definePropertyModule.f;\nvar nativeGetOwnPropertyNames = getOwnPropertyNamesExternal.f;\nvar nativePropertyIsEnumerable = propertyIsEnumerableModule.f;\nvar AllSymbols = shared('symbols');\nvar ObjectPrototypeSymbols = shared('op-symbols');\nvar StringToSymbolRegistry = shared('string-to-symbol-registry');\nvar SymbolToStringRegistry = shared('symbol-to-string-registry');\nvar WellKnownSymbolsStore = shared('wks');\nvar QObject = global.QObject;\n// Don't use setters in Qt Script, https://github.com/zloirock/core-js/issues/173\nvar USE_SETTER = !QObject || !QObject[PROTOTYPE] || !QObject[PROTOTYPE].findChild;\n\n// fallback for old Android, https://code.google.com/p/v8/issues/detail?id=687\nvar setSymbolDescriptor = DESCRIPTORS && fails(function () {\n  return nativeObjectCreate(nativeDefineProperty({}, 'a', {\n    get: function () { return nativeDefineProperty(this, 'a', { value: 7 }).a; }\n  })).a != 7;\n}) ? function (O, P, Attributes) {\n  var ObjectPrototypeDescriptor = nativeGetOwnPropertyDescriptor(ObjectPrototype, P);\n  if (ObjectPrototypeDescriptor) delete ObjectPrototype[P];\n  nativeDefineProperty(O, P, Attributes);\n  if (ObjectPrototypeDescriptor && O !== ObjectPrototype) {\n    nativeDefineProperty(ObjectPrototype, P, ObjectPrototypeDescriptor);\n  }\n} : nativeDefineProperty;\n\nvar wrap = function (tag, description) {\n  var symbol = AllSymbols[tag] = nativeObjectCreate($Symbol[PROTOTYPE]);\n  setInternalState(symbol, {\n    type: SYMBOL,\n    tag: tag,\n    description: description\n  });\n  if (!DESCRIPTORS) symbol.description = description;\n  return symbol;\n};\n\nvar isSymbol = USE_SYMBOL_AS_UID ? function (it) {\n  return typeof it == 'symbol';\n} : function (it) {\n  return Object(it) instanceof $Symbol;\n};\n\nvar $defineProperty = function defineProperty(O, P, Attributes) {\n  if (O === ObjectPrototype) $defineProperty(ObjectPrototypeSymbols, P, Attributes);\n  anObject(O);\n  var key = toPrimitive(P, true);\n  anObject(Attributes);\n  if (has(AllSymbols, key)) {\n    if (!Attributes.enumerable) {\n      if (!has(O, HIDDEN)) nativeDefineProperty(O, HIDDEN, createPropertyDescriptor(1, {}));\n      O[HIDDEN][key] = true;\n    } else {\n      if (has(O, HIDDEN) && O[HIDDEN][key]) O[HIDDEN][key] = false;\n      Attributes = nativeObjectCreate(Attributes, { enumerable: createPropertyDescriptor(0, false) });\n    } return setSymbolDescriptor(O, key, Attributes);\n  } return nativeDefineProperty(O, key, Attributes);\n};\n\nvar $defineProperties = function defineProperties(O, Properties) {\n  anObject(O);\n  var properties = toIndexedObject(Properties);\n  var keys = objectKeys(properties).concat($getOwnPropertySymbols(properties));\n  $forEach(keys, function (key) {\n    if (!DESCRIPTORS || $propertyIsEnumerable.call(properties, key)) $defineProperty(O, key, properties[key]);\n  });\n  return O;\n};\n\nvar $create = function create(O, Properties) {\n  return Properties === undefined ? nativeObjectCreate(O) : $defineProperties(nativeObjectCreate(O), Properties);\n};\n\nvar $propertyIsEnumerable = function propertyIsEnumerable(V) {\n  var P = toPrimitive(V, true);\n  var enumerable = nativePropertyIsEnumerable.call(this, P);\n  if (this === ObjectPrototype && has(AllSymbols, P) && !has(ObjectPrototypeSymbols, P)) return false;\n  return enumerable || !has(this, P) || !has(AllSymbols, P) || has(this, HIDDEN) && this[HIDDEN][P] ? enumerable : true;\n};\n\nvar $getOwnPropertyDescriptor = function getOwnPropertyDescriptor(O, P) {\n  var it = toIndexedObject(O);\n  var key = toPrimitive(P, true);\n  if (it === ObjectPrototype && has(AllSymbols, key) && !has(ObjectPrototypeSymbols, key)) return;\n  var descriptor = nativeGetOwnPropertyDescriptor(it, key);\n  if (descriptor && has(AllSymbols, key) && !(has(it, HIDDEN) && it[HIDDEN][key])) {\n    descriptor.enumerable = true;\n  }\n  return descriptor;\n};\n\nvar $getOwnPropertyNames = function getOwnPropertyNames(O) {\n  var names = nativeGetOwnPropertyNames(toIndexedObject(O));\n  var result = [];\n  $forEach(names, function (key) {\n    if (!has(AllSymbols, key) && !has(hiddenKeys, key)) result.push(key);\n  });\n  return result;\n};\n\nvar $getOwnPropertySymbols = function getOwnPropertySymbols(O) {\n  var IS_OBJECT_PROTOTYPE = O === ObjectPrototype;\n  var names = nativeGetOwnPropertyNames(IS_OBJECT_PROTOTYPE ? ObjectPrototypeSymbols : toIndexedObject(O));\n  var result = [];\n  $forEach(names, function (key) {\n    if (has(AllSymbols, key) && (!IS_OBJECT_PROTOTYPE || has(ObjectPrototype, key))) {\n      result.push(AllSymbols[key]);\n    }\n  });\n  return result;\n};\n\n// `Symbol` constructor\n// https://tc39.github.io/ecma262/#sec-symbol-constructor\nif (!NATIVE_SYMBOL) {\n  $Symbol = function Symbol() {\n    if (this instanceof $Symbol) throw TypeError('Symbol is not a constructor');\n    var description = !arguments.length || arguments[0] === undefined ? undefined : String(arguments[0]);\n    var tag = uid(description);\n    var setter = function (value) {\n      if (this === ObjectPrototype) setter.call(ObjectPrototypeSymbols, value);\n      if (has(this, HIDDEN) && has(this[HIDDEN], tag)) this[HIDDEN][tag] = false;\n      setSymbolDescriptor(this, tag, createPropertyDescriptor(1, value));\n    };\n    if (DESCRIPTORS && USE_SETTER) setSymbolDescriptor(ObjectPrototype, tag, { configurable: true, set: setter });\n    return wrap(tag, description);\n  };\n\n  redefine($Symbol[PROTOTYPE], 'toString', function toString() {\n    return getInternalState(this).tag;\n  });\n\n  redefine($Symbol, 'withoutSetter', function (description) {\n    return wrap(uid(description), description);\n  });\n\n  propertyIsEnumerableModule.f = $propertyIsEnumerable;\n  definePropertyModule.f = $defineProperty;\n  getOwnPropertyDescriptorModule.f = $getOwnPropertyDescriptor;\n  getOwnPropertyNamesModule.f = getOwnPropertyNamesExternal.f = $getOwnPropertyNames;\n  getOwnPropertySymbolsModule.f = $getOwnPropertySymbols;\n\n  wrappedWellKnownSymbolModule.f = function (name) {\n    return wrap(wellKnownSymbol(name), name);\n  };\n\n  if (DESCRIPTORS) {\n    // https://github.com/tc39/proposal-Symbol-description\n    nativeDefineProperty($Symbol[PROTOTYPE], 'description', {\n      configurable: true,\n      get: function description() {\n        return getInternalState(this).description;\n      }\n    });\n    if (!IS_PURE) {\n      redefine(ObjectPrototype, 'propertyIsEnumerable', $propertyIsEnumerable, { unsafe: true });\n    }\n  }\n}\n\n$({ global: true, wrap: true, forced: !NATIVE_SYMBOL, sham: !NATIVE_SYMBOL }, {\n  Symbol: $Symbol\n});\n\n$forEach(objectKeys(WellKnownSymbolsStore), function (name) {\n  defineWellKnownSymbol(name);\n});\n\n$({ target: SYMBOL, stat: true, forced: !NATIVE_SYMBOL }, {\n  // `Symbol.for` method\n  // https://tc39.github.io/ecma262/#sec-symbol.for\n  'for': function (key) {\n    var string = String(key);\n    if (has(StringToSymbolRegistry, string)) return StringToSymbolRegistry[string];\n    var symbol = $Symbol(string);\n    StringToSymbolRegistry[string] = symbol;\n    SymbolToStringRegistry[symbol] = string;\n    return symbol;\n  },\n  // `Symbol.keyFor` method\n  // https://tc39.github.io/ecma262/#sec-symbol.keyfor\n  keyFor: function keyFor(sym) {\n    if (!isSymbol(sym)) throw TypeError(sym + ' is not a symbol');\n    if (has(SymbolToStringRegistry, sym)) return SymbolToStringRegistry[sym];\n  },\n  useSetter: function () { USE_SETTER = true; },\n  useSimple: function () { USE_SETTER = false; }\n});\n\n$({ target: 'Object', stat: true, forced: !NATIVE_SYMBOL, sham: !DESCRIPTORS }, {\n  // `Object.create` method\n  // https://tc39.github.io/ecma262/#sec-object.create\n  create: $create,\n  // `Object.defineProperty` method\n  // https://tc39.github.io/ecma262/#sec-object.defineproperty\n  defineProperty: $defineProperty,\n  // `Object.defineProperties` method\n  // https://tc39.github.io/ecma262/#sec-object.defineproperties\n  defineProperties: $defineProperties,\n  // `Object.getOwnPropertyDescriptor` method\n  // https://tc39.github.io/ecma262/#sec-object.getownpropertydescriptors\n  getOwnPropertyDescriptor: $getOwnPropertyDescriptor\n});\n\n$({ target: 'Object', stat: true, forced: !NATIVE_SYMBOL }, {\n  // `Object.getOwnPropertyNames` method\n  // https://tc39.github.io/ecma262/#sec-object.getownpropertynames\n  getOwnPropertyNames: $getOwnPropertyNames,\n  // `Object.getOwnPropertySymbols` method\n  // https://tc39.github.io/ecma262/#sec-object.getownpropertysymbols\n  getOwnPropertySymbols: $getOwnPropertySymbols\n});\n\n// Chrome 38 and 39 `Object.getOwnPropertySymbols` fails on primitives\n// https://bugs.chromium.org/p/v8/issues/detail?id=3443\n$({ target: 'Object', stat: true, forced: fails(function () { getOwnPropertySymbolsModule.f(1); }) }, {\n  getOwnPropertySymbols: function getOwnPropertySymbols(it) {\n    return getOwnPropertySymbolsModule.f(toObject(it));\n  }\n});\n\n// `JSON.stringify` method behavior with symbols\n// https://tc39.github.io/ecma262/#sec-json.stringify\nif ($stringify) {\n  var FORCED_JSON_STRINGIFY = !NATIVE_SYMBOL || fails(function () {\n    var symbol = $Symbol();\n    // MS Edge converts symbol values to JSON as {}\n    return $stringify([symbol]) != '[null]'\n      // WebKit converts symbol values to JSON as null\n      || $stringify({ a: symbol }) != '{}'\n      // V8 throws on boxed symbols\n      || $stringify(Object(symbol)) != '{}';\n  });\n\n  $({ target: 'JSON', stat: true, forced: FORCED_JSON_STRINGIFY }, {\n    // eslint-disable-next-line no-unused-vars\n    stringify: function stringify(it, replacer, space) {\n      var args = [it];\n      var index = 1;\n      var $replacer;\n      while (arguments.length > index) args.push(arguments[index++]);\n      $replacer = replacer;\n      if (!isObject(replacer) && it === undefined || isSymbol(it)) return; // IE8 returns string on undefined\n      if (!isArray(replacer)) replacer = function (key, value) {\n        if (typeof $replacer == 'function') value = $replacer.call(this, key, value);\n        if (!isSymbol(value)) return value;\n      };\n      args[1] = replacer;\n      return $stringify.apply(null, args);\n    }\n  });\n}\n\n// `Symbol.prototype[@@toPrimitive]` method\n// https://tc39.github.io/ecma262/#sec-symbol.prototype-@@toprimitive\nif (!$Symbol[PROTOTYPE][TO_PRIMITIVE]) {\n  createNonEnumerableProperty($Symbol[PROTOTYPE], TO_PRIMITIVE, $Symbol[PROTOTYPE].valueOf);\n}\n// `Symbol.prototype[@@toStringTag]` property\n// https://tc39.github.io/ecma262/#sec-symbol.prototype-@@tostringtag\nsetToStringTag($Symbol, SYMBOL);\n\nhiddenKeys[HIDDEN] = true;\n","require('../../modules/es.symbol');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Object.getOwnPropertySymbols;\n","'use strict';\nvar getPrototypeOf = require('../internals/object-get-prototype-of');\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\nvar has = require('../internals/has');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\nvar IS_PURE = require('../internals/is-pure');\n\nvar ITERATOR = wellKnownSymbol('iterator');\nvar BUGGY_SAFARI_ITERATORS = false;\n\nvar returnThis = function () { return this; };\n\n// `%IteratorPrototype%` object\n// https://tc39.github.io/ecma262/#sec-%iteratorprototype%-object\nvar IteratorPrototype, PrototypeOfArrayIteratorPrototype, arrayIterator;\n\nif ([].keys) {\n  arrayIterator = [].keys();\n  // Safari 8 has buggy iterators w/o `next`\n  if (!('next' in arrayIterator)) BUGGY_SAFARI_ITERATORS = true;\n  else {\n    PrototypeOfArrayIteratorPrototype = getPrototypeOf(getPrototypeOf(arrayIterator));\n    if (PrototypeOfArrayIteratorPrototype !== Object.prototype) IteratorPrototype = PrototypeOfArrayIteratorPrototype;\n  }\n}\n\nif (IteratorPrototype == undefined) IteratorPrototype = {};\n\n// 25.1.2.1.1 %IteratorPrototype%[@@iterator]()\nif (!IS_PURE && !has(IteratorPrototype, ITERATOR)) {\n  createNonEnumerableProperty(IteratorPrototype, ITERATOR, returnThis);\n}\n\nmodule.exports = {\n  IteratorPrototype: IteratorPrototype,\n  BUGGY_SAFARI_ITERATORS: BUGGY_SAFARI_ITERATORS\n};\n","module.exports = {};\n","var fails = require('../internals/fails');\n\nmodule.exports = !fails(function () {\n  function F() { /* empty */ }\n  F.prototype.constructor = null;\n  return Object.getPrototypeOf(new F()) !== F.prototype;\n});\n","var has = require('../internals/has');\nvar toObject = require('../internals/to-object');\nvar sharedKey = require('../internals/shared-key');\nvar CORRECT_PROTOTYPE_GETTER = require('../internals/correct-prototype-getter');\n\nvar IE_PROTO = sharedKey('IE_PROTO');\nvar ObjectPrototype = Object.prototype;\n\n// `Object.getPrototypeOf` method\n// https://tc39.github.io/ecma262/#sec-object.getprototypeof\nmodule.exports = CORRECT_PROTOTYPE_GETTER ? Object.getPrototypeOf : function (O) {\n  O = toObject(O);\n  if (has(O, IE_PROTO)) return O[IE_PROTO];\n  if (typeof O.constructor == 'function' && O instanceof O.constructor) {\n    return O.constructor.prototype;\n  } return O instanceof Object ? ObjectPrototype : null;\n};\n","'use strict';\nvar IteratorPrototype = require('../internals/iterators-core').IteratorPrototype;\nvar create = require('../internals/object-create');\nvar createPropertyDescriptor = require('../internals/create-property-descriptor');\nvar setToStringTag = require('../internals/set-to-string-tag');\nvar Iterators = require('../internals/iterators');\n\nvar returnThis = function () { return this; };\n\nmodule.exports = function (IteratorConstructor, NAME, next) {\n  var TO_STRING_TAG = NAME + ' Iterator';\n  IteratorConstructor.prototype = create(IteratorPrototype, { next: createPropertyDescriptor(1, next) });\n  setToStringTag(IteratorConstructor, TO_STRING_TAG, false, true);\n  Iterators[TO_STRING_TAG] = returnThis;\n  return IteratorConstructor;\n};\n","var anObject = require('../internals/an-object');\nvar aPossiblePrototype = require('../internals/a-possible-prototype');\n\n// `Object.setPrototypeOf` method\n// https://tc39.github.io/ecma262/#sec-object.setprototypeof\n// Works with __proto__ only. Old v8 can't work with null proto objects.\n/* eslint-disable no-proto */\nmodule.exports = Object.setPrototypeOf || ('__proto__' in {} ? function () {\n  var CORRECT_SETTER = false;\n  var test = {};\n  var setter;\n  try {\n    setter = Object.getOwnPropertyDescriptor(Object.prototype, '__proto__').set;\n    setter.call(test, []);\n    CORRECT_SETTER = test instanceof Array;\n  } catch (error) { /* empty */ }\n  return function setPrototypeOf(O, proto) {\n    anObject(O);\n    aPossiblePrototype(proto);\n    if (CORRECT_SETTER) setter.call(O, proto);\n    else O.__proto__ = proto;\n    return O;\n  };\n}() : undefined);\n","var isObject = require('../internals/is-object');\n\nmodule.exports = function (it) {\n  if (!isObject(it) && it !== null) {\n    throw TypeError(\"Can't set \" + String(it) + ' as a prototype');\n  } return it;\n};\n","'use strict';\nvar $ = require('../internals/export');\nvar createIteratorConstructor = require('../internals/create-iterator-constructor');\nvar getPrototypeOf = require('../internals/object-get-prototype-of');\nvar setPrototypeOf = require('../internals/object-set-prototype-of');\nvar setToStringTag = require('../internals/set-to-string-tag');\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\nvar redefine = require('../internals/redefine');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\nvar IS_PURE = require('../internals/is-pure');\nvar Iterators = require('../internals/iterators');\nvar IteratorsCore = require('../internals/iterators-core');\n\nvar IteratorPrototype = IteratorsCore.IteratorPrototype;\nvar BUGGY_SAFARI_ITERATORS = IteratorsCore.BUGGY_SAFARI_ITERATORS;\nvar ITERATOR = wellKnownSymbol('iterator');\nvar KEYS = 'keys';\nvar VALUES = 'values';\nvar ENTRIES = 'entries';\n\nvar returnThis = function () { return this; };\n\nmodule.exports = function (Iterable, NAME, IteratorConstructor, next, DEFAULT, IS_SET, FORCED) {\n  createIteratorConstructor(IteratorConstructor, NAME, next);\n\n  var getIterationMethod = function (KIND) {\n    if (KIND === DEFAULT && defaultIterator) return defaultIterator;\n    if (!BUGGY_SAFARI_ITERATORS && KIND in IterablePrototype) return IterablePrototype[KIND];\n    switch (KIND) {\n      case KEYS: return function keys() { return new IteratorConstructor(this, KIND); };\n      case VALUES: return function values() { return new IteratorConstructor(this, KIND); };\n      case ENTRIES: return function entries() { return new IteratorConstructor(this, KIND); };\n    } return function () { return new IteratorConstructor(this); };\n  };\n\n  var TO_STRING_TAG = NAME + ' Iterator';\n  var INCORRECT_VALUES_NAME = false;\n  var IterablePrototype = Iterable.prototype;\n  var nativeIterator = IterablePrototype[ITERATOR]\n    || IterablePrototype['@@iterator']\n    || DEFAULT && IterablePrototype[DEFAULT];\n  var defaultIterator = !BUGGY_SAFARI_ITERATORS && nativeIterator || getIterationMethod(DEFAULT);\n  var anyNativeIterator = NAME == 'Array' ? IterablePrototype.entries || nativeIterator : nativeIterator;\n  var CurrentIteratorPrototype, methods, KEY;\n\n  // fix native\n  if (anyNativeIterator) {\n    CurrentIteratorPrototype = getPrototypeOf(anyNativeIterator.call(new Iterable()));\n    if (IteratorPrototype !== Object.prototype && CurrentIteratorPrototype.next) {\n      if (!IS_PURE && getPrototypeOf(CurrentIteratorPrototype) !== IteratorPrototype) {\n        if (setPrototypeOf) {\n          setPrototypeOf(CurrentIteratorPrototype, IteratorPrototype);\n        } else if (typeof CurrentIteratorPrototype[ITERATOR] != 'function') {\n          createNonEnumerableProperty(CurrentIteratorPrototype, ITERATOR, returnThis);\n        }\n      }\n      // Set @@toStringTag to native iterators\n      setToStringTag(CurrentIteratorPrototype, TO_STRING_TAG, true, true);\n      if (IS_PURE) Iterators[TO_STRING_TAG] = returnThis;\n    }\n  }\n\n  // fix Array#{values, @@iterator}.name in V8 / FF\n  if (DEFAULT == VALUES && nativeIterator && nativeIterator.name !== VALUES) {\n    INCORRECT_VALUES_NAME = true;\n    defaultIterator = function values() { return nativeIterator.call(this); };\n  }\n\n  // define iterator\n  if ((!IS_PURE || FORCED) && IterablePrototype[ITERATOR] !== defaultIterator) {\n    createNonEnumerableProperty(IterablePrototype, ITERATOR, defaultIterator);\n  }\n  Iterators[NAME] = defaultIterator;\n\n  // export additional methods\n  if (DEFAULT) {\n    methods = {\n      values: getIterationMethod(VALUES),\n      keys: IS_SET ? defaultIterator : getIterationMethod(KEYS),\n      entries: getIterationMethod(ENTRIES)\n    };\n    if (FORCED) for (KEY in methods) {\n      if (BUGGY_SAFARI_ITERATORS || INCORRECT_VALUES_NAME || !(KEY in IterablePrototype)) {\n        redefine(IterablePrototype, KEY, methods[KEY]);\n      }\n    } else $({ target: NAME, proto: true, forced: BUGGY_SAFARI_ITERATORS || INCORRECT_VALUES_NAME }, methods);\n  }\n\n  return methods;\n};\n","'use strict';\nvar toIndexedObject = require('../internals/to-indexed-object');\nvar addToUnscopables = require('../internals/add-to-unscopables');\nvar Iterators = require('../internals/iterators');\nvar InternalStateModule = require('../internals/internal-state');\nvar defineIterator = require('../internals/define-iterator');\n\nvar ARRAY_ITERATOR = 'Array Iterator';\nvar setInternalState = InternalStateModule.set;\nvar getInternalState = InternalStateModule.getterFor(ARRAY_ITERATOR);\n\n// `Array.prototype.entries` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.entries\n// `Array.prototype.keys` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.keys\n// `Array.prototype.values` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.values\n// `Array.prototype[@@iterator]` method\n// https://tc39.github.io/ecma262/#sec-array.prototype-@@iterator\n// `CreateArrayIterator` internal method\n// https://tc39.github.io/ecma262/#sec-createarrayiterator\nmodule.exports = defineIterator(Array, 'Array', function (iterated, kind) {\n  setInternalState(this, {\n    type: ARRAY_ITERATOR,\n    target: toIndexedObject(iterated), // target\n    index: 0,                          // next index\n    kind: kind                         // kind\n  });\n// `%ArrayIteratorPrototype%.next` method\n// https://tc39.github.io/ecma262/#sec-%arrayiteratorprototype%.next\n}, function () {\n  var state = getInternalState(this);\n  var target = state.target;\n  var kind = state.kind;\n  var index = state.index++;\n  if (!target || index >= target.length) {\n    state.target = undefined;\n    return { value: undefined, done: true };\n  }\n  if (kind == 'keys') return { value: index, done: false };\n  if (kind == 'values') return { value: target[index], done: false };\n  return { value: [index, target[index]], done: false };\n}, 'values');\n\n// argumentsList[@@iterator] is %ArrayProto_values%\n// https://tc39.github.io/ecma262/#sec-createunmappedargumentsobject\n// https://tc39.github.io/ecma262/#sec-createmappedargumentsobject\nIterators.Arguments = Iterators.Array;\n\n// https://tc39.github.io/ecma262/#sec-array.prototype-@@unscopables\naddToUnscopables('keys');\naddToUnscopables('values');\naddToUnscopables('entries');\n","// iterable DOM collections\n// flag - `iterable` interface - 'entries', 'keys', 'values', 'forEach' methods\nmodule.exports = {\n  CSSRuleList: 0,\n  CSSStyleDeclaration: 0,\n  CSSValueList: 0,\n  ClientRectList: 0,\n  DOMRectList: 0,\n  DOMStringList: 0,\n  DOMTokenList: 1,\n  DataTransferItemList: 0,\n  FileList: 0,\n  HTMLAllCollection: 0,\n  HTMLCollection: 0,\n  HTMLFormElement: 0,\n  HTMLSelectElement: 0,\n  MediaList: 0,\n  MimeTypeArray: 0,\n  NamedNodeMap: 0,\n  NodeList: 1,\n  PaintRequestList: 0,\n  Plugin: 0,\n  PluginArray: 0,\n  SVGLengthList: 0,\n  SVGNumberList: 0,\n  SVGPathSegList: 0,\n  SVGPointList: 0,\n  SVGStringList: 0,\n  SVGTransformList: 0,\n  SourceBufferList: 0,\n  StyleSheetList: 0,\n  TextTrackCueList: 0,\n  TextTrackList: 0,\n  TouchList: 0\n};\n","require('./es.array.iterator');\nvar DOMIterables = require('../internals/dom-iterables');\nvar global = require('../internals/global');\nvar classof = require('../internals/classof');\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\nvar Iterators = require('../internals/iterators');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar TO_STRING_TAG = wellKnownSymbol('toStringTag');\n\nfor (var COLLECTION_NAME in DOMIterables) {\n  var Collection = global[COLLECTION_NAME];\n  var CollectionPrototype = Collection && Collection.prototype;\n  if (CollectionPrototype && classof(CollectionPrototype) !== TO_STRING_TAG) {\n    createNonEnumerableProperty(CollectionPrototype, TO_STRING_TAG, COLLECTION_NAME);\n  }\n  Iterators[COLLECTION_NAME] = Iterators.Array;\n}\n","var toInteger = require('../internals/to-integer');\nvar requireObjectCoercible = require('../internals/require-object-coercible');\n\n// `String.prototype.{ codePointAt, at }` methods implementation\nvar createMethod = function (CONVERT_TO_STRING) {\n  return function ($this, pos) {\n    var S = String(requireObjectCoercible($this));\n    var position = toInteger(pos);\n    var size = S.length;\n    var first, second;\n    if (position < 0 || position >= size) return CONVERT_TO_STRING ? '' : undefined;\n    first = S.charCodeAt(position);\n    return first < 0xD800 || first > 0xDBFF || position + 1 === size\n      || (second = S.charCodeAt(position + 1)) < 0xDC00 || second > 0xDFFF\n        ? CONVERT_TO_STRING ? S.charAt(position) : first\n        : CONVERT_TO_STRING ? S.slice(position, position + 2) : (first - 0xD800 << 10) + (second - 0xDC00) + 0x10000;\n  };\n};\n\nmodule.exports = {\n  // `String.prototype.codePointAt` method\n  // https://tc39.github.io/ecma262/#sec-string.prototype.codepointat\n  codeAt: createMethod(false),\n  // `String.prototype.at` method\n  // https://github.com/mathiasbynens/String.prototype.at\n  charAt: createMethod(true)\n};\n","'use strict';\nvar charAt = require('../internals/string-multibyte').charAt;\nvar InternalStateModule = require('../internals/internal-state');\nvar defineIterator = require('../internals/define-iterator');\n\nvar STRING_ITERATOR = 'String Iterator';\nvar setInternalState = InternalStateModule.set;\nvar getInternalState = InternalStateModule.getterFor(STRING_ITERATOR);\n\n// `String.prototype[@@iterator]` method\n// https://tc39.github.io/ecma262/#sec-string.prototype-@@iterator\ndefineIterator(String, 'String', function (iterated) {\n  setInternalState(this, {\n    type: STRING_ITERATOR,\n    string: String(iterated),\n    index: 0\n  });\n// `%StringIteratorPrototype%.next` method\n// https://tc39.github.io/ecma262/#sec-%stringiteratorprototype%.next\n}, function next() {\n  var state = getInternalState(this);\n  var string = state.string;\n  var index = state.index;\n  var point;\n  if (index >= string.length) return { value: undefined, done: true };\n  point = charAt(string, index);\n  state.index += point.length;\n  return { value: point, done: false };\n});\n","var classof = require('../internals/classof');\nvar Iterators = require('../internals/iterators');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar ITERATOR = wellKnownSymbol('iterator');\n\nmodule.exports = function (it) {\n  if (it != undefined) return it[ITERATOR]\n    || it['@@iterator']\n    || Iterators[classof(it)];\n};\n","var anObject = require('../internals/an-object');\nvar getIteratorMethod = require('../internals/get-iterator-method');\n\nmodule.exports = function (it) {\n  var iteratorMethod = getIteratorMethod(it);\n  if (typeof iteratorMethod != 'function') {\n    throw TypeError(String(it) + ' is not iterable');\n  } return anObject(iteratorMethod.call(it));\n};\n","var $ = require('../internals/export');\nvar DESCRIPTORS = require('../internals/descriptors');\nvar create = require('../internals/object-create');\n\n// `Object.create` method\n// https://tc39.github.io/ecma262/#sec-object.create\n$({ target: 'Object', stat: true, sham: !DESCRIPTORS }, {\n  create: create\n});\n","require('../../modules/es.object.create');\nvar path = require('../../internals/path');\n\nvar Object = path.Object;\n\nmodule.exports = function create(P, D) {\n  return Object.create(P, D);\n};\n","var parent = require('../../es/object/create');\n\nmodule.exports = parent;\n","var parent = require('../../es/object/define-property');\n\nmodule.exports = parent;\n","var _Object$defineProperty = require(\"../core-js/object/define-property\");\n\nfunction _defineProperty(obj, key, value) {\n  if (key in obj) {\n    _Object$defineProperty(obj, key, {\n      value: value,\n      enumerable: true,\n      configurable: true,\n      writable: true\n    });\n  } else {\n    obj[key] = value;\n  }\n\n  return obj;\n}\n\nmodule.exports = _defineProperty;","var $ = require('../internals/export');\nvar toObject = require('../internals/to-object');\nvar nativeKeys = require('../internals/object-keys');\nvar fails = require('../internals/fails');\n\nvar FAILS_ON_PRIMITIVES = fails(function () { nativeKeys(1); });\n\n// `Object.keys` method\n// https://tc39.github.io/ecma262/#sec-object.keys\n$({ target: 'Object', stat: true, forced: FAILS_ON_PRIMITIVES }, {\n  keys: function keys(it) {\n    return nativeKeys(toObject(it));\n  }\n});\n","require('../../modules/es.object.keys');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Object.keys;\n","var fails = require('../internals/fails');\nvar whitespaces = require('../internals/whitespaces');\n\nvar non = '\\u200B\\u0085\\u180E';\n\n// check that a method works with the correct list\n// of whitespaces and has a correct name\nmodule.exports = function (METHOD_NAME) {\n  return fails(function () {\n    return !!whitespaces[METHOD_NAME]() || non[METHOD_NAME]() != non || whitespaces[METHOD_NAME].name !== METHOD_NAME;\n  });\n};\n","// a string of all valid unicode whitespaces\n// eslint-disable-next-line max-len\nmodule.exports = '\\u0009\\u000A\\u000B\\u000C\\u000D\\u0020\\u00A0\\u1680\\u2000\\u2001\\u2002\\u2003\\u2004\\u2005\\u2006\\u2007\\u2008\\u2009\\u200A\\u202F\\u205F\\u3000\\u2028\\u2029\\uFEFF';\n","var requireObjectCoercible = require('../internals/require-object-coercible');\nvar whitespaces = require('../internals/whitespaces');\n\nvar whitespace = '[' + whitespaces + ']';\nvar ltrim = RegExp('^' + whitespace + whitespace + '*');\nvar rtrim = RegExp(whitespace + whitespace + '*$');\n\n// `String.prototype.{ trim, trimStart, trimEnd, trimLeft, trimRight }` methods implementation\nvar createMethod = function (TYPE) {\n  return function ($this) {\n    var string = String(requireObjectCoercible($this));\n    if (TYPE & 1) string = string.replace(ltrim, '');\n    if (TYPE & 2) string = string.replace(rtrim, '');\n    return string;\n  };\n};\n\nmodule.exports = {\n  // `String.prototype.{ trimLeft, trimStart }` methods\n  // https://tc39.github.io/ecma262/#sec-string.prototype.trimstart\n  start: createMethod(1),\n  // `String.prototype.{ trimRight, trimEnd }` methods\n  // https://tc39.github.io/ecma262/#sec-string.prototype.trimend\n  end: createMethod(2),\n  // `String.prototype.trim` method\n  // https://tc39.github.io/ecma262/#sec-string.prototype.trim\n  trim: createMethod(3)\n};\n","'use strict';\nvar $ = require('../internals/export');\nvar $trim = require('../internals/string-trim').trim;\nvar forcedStringTrimMethod = require('../internals/string-trim-forced');\n\n// `String.prototype.trim` method\n// https://tc39.github.io/ecma262/#sec-string.prototype.trim\n$({ target: 'String', proto: true, forced: forcedStringTrimMethod('trim') }, {\n  trim: function trim() {\n    return $trim(this);\n  }\n});\n","var path = require('../internals/path');\n\nmodule.exports = function (CONSTRUCTOR) {\n  return path[CONSTRUCTOR + 'Prototype'];\n};\n","require('../../../modules/es.string.trim');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('String').trim;\n","var trim = require('../string/virtual/trim');\n\nvar StringPrototype = String.prototype;\n\nmodule.exports = function (it) {\n  var own = it.trim;\n  return typeof it === 'string' || it === StringPrototype\n    || (it instanceof String && own === StringPrototype.trim) ? trim : own;\n};\n","'use strict';\nvar fails = require('../internals/fails');\n\nmodule.exports = function (METHOD_NAME, argument) {\n  var method = [][METHOD_NAME];\n  return !!method && fails(function () {\n    // eslint-disable-next-line no-useless-call,no-throw-literal\n    method.call(null, argument || function () { throw 1; }, 1);\n  });\n};\n","var DESCRIPTORS = require('../internals/descriptors');\nvar fails = require('../internals/fails');\nvar has = require('../internals/has');\n\nvar defineProperty = Object.defineProperty;\nvar cache = {};\n\nvar thrower = function (it) { throw it; };\n\nmodule.exports = function (METHOD_NAME, options) {\n  if (has(cache, METHOD_NAME)) return cache[METHOD_NAME];\n  if (!options) options = {};\n  var method = [][METHOD_NAME];\n  var ACCESSORS = has(options, 'ACCESSORS') ? options.ACCESSORS : false;\n  var argument0 = has(options, 0) ? options[0] : thrower;\n  var argument1 = has(options, 1) ? options[1] : undefined;\n\n  return cache[METHOD_NAME] = !!method && !fails(function () {\n    if (ACCESSORS && !DESCRIPTORS) return true;\n    var O = { length: -1 };\n\n    if (ACCESSORS) defineProperty(O, 1, { enumerable: true, get: thrower });\n    else O[1] = 1;\n\n    method.call(O, argument0, argument1);\n  });\n};\n","'use strict';\nvar $forEach = require('../internals/array-iteration').forEach;\nvar arrayMethodIsStrict = require('../internals/array-method-is-strict');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar STRICT_METHOD = arrayMethodIsStrict('forEach');\nvar USES_TO_LENGTH = arrayMethodUsesToLength('forEach');\n\n// `Array.prototype.forEach` method implementation\n// https://tc39.github.io/ecma262/#sec-array.prototype.foreach\nmodule.exports = (!STRICT_METHOD || !USES_TO_LENGTH) ? function forEach(callbackfn /* , thisArg */) {\n  return $forEach(this, callbackfn, arguments.length > 1 ? arguments[1] : undefined);\n} : [].forEach;\n","'use strict';\nvar $ = require('../internals/export');\nvar forEach = require('../internals/array-for-each');\n\n// `Array.prototype.forEach` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.foreach\n$({ target: 'Array', proto: true, forced: [].forEach != forEach }, {\n  forEach: forEach\n});\n","require('../../../modules/es.array.for-each');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').forEach;\n","var global = require('../internals/global');\nvar userAgent = require('../internals/engine-user-agent');\n\nvar process = global.process;\nvar versions = process && process.versions;\nvar v8 = versions && versions.v8;\nvar match, version;\n\nif (v8) {\n  match = v8.split('.');\n  version = match[0] + match[1];\n} else if (userAgent) {\n  match = userAgent.match(/Edge\\/(\\d+)/);\n  if (!match || match[1] >= 74) {\n    match = userAgent.match(/Chrome\\/(\\d+)/);\n    if (match) version = match[1];\n  }\n}\n\nmodule.exports = version && +version;\n","require('../../modules/web.dom-collections.iterator');\nvar forEach = require('../array/virtual/for-each');\nvar classof = require('../../internals/classof');\nvar ArrayPrototype = Array.prototype;\n\nvar DOMIterables = {\n  DOMTokenList: true,\n  NodeList: true\n};\n\nmodule.exports = function (it) {\n  var own = it.forEach;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.forEach)\n    // eslint-disable-next-line no-prototype-builtins\n    || DOMIterables.hasOwnProperty(classof(it)) ? forEach : own;\n};\n","var getBuiltIn = require('../internals/get-built-in');\n\nmodule.exports = getBuiltIn('navigator', 'userAgent') || '';\n","var fails = require('../internals/fails');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\nvar V8_VERSION = require('../internals/engine-v8-version');\n\nvar SPECIES = wellKnownSymbol('species');\n\nmodule.exports = function (METHOD_NAME) {\n  // We can't use this feature detection in V8 since it causes\n  // deoptimization and serious performance degradation\n  // https://github.com/zloirock/core-js/issues/677\n  return V8_VERSION >= 51 || !fails(function () {\n    var array = [];\n    var constructor = array.constructor = {};\n    constructor[SPECIES] = function () {\n      return { foo: 1 };\n    };\n    return array[METHOD_NAME](Boolean).foo !== 1;\n  });\n};\n","'use strict';\nvar $ = require('../internals/export');\nvar $map = require('../internals/array-iteration').map;\nvar arrayMethodHasSpeciesSupport = require('../internals/array-method-has-species-support');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar HAS_SPECIES_SUPPORT = arrayMethodHasSpeciesSupport('map');\n// FF49- issue\nvar USES_TO_LENGTH = arrayMethodUsesToLength('map');\n\n// `Array.prototype.map` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.map\n// with adding support of @@species\n$({ target: 'Array', proto: true, forced: !HAS_SPECIES_SUPPORT || !USES_TO_LENGTH }, {\n  map: function map(callbackfn /* , thisArg */) {\n    return $map(this, callbackfn, arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n","require('../../../modules/es.array.map');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').map;\n","var map = require('../array/virtual/map');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.map;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.map) ? map : own;\n};\n","var global = require('../internals/global');\nvar trim = require('../internals/string-trim').trim;\nvar whitespaces = require('../internals/whitespaces');\n\nvar $parseInt = global.parseInt;\nvar hex = /^[+-]?0[Xx]/;\nvar FORCED = $parseInt(whitespaces + '08') !== 8 || $parseInt(whitespaces + '0x16') !== 22;\n\n// `parseInt` method\n// https://tc39.github.io/ecma262/#sec-parseint-string-radix\nmodule.exports = FORCED ? function parseInt(string, radix) {\n  var S = trim(String(string));\n  return $parseInt(S, (radix >>> 0) || (hex.test(S) ? 16 : 10));\n} : $parseInt;\n","var $ = require('../internals/export');\nvar parseIntImplementation = require('../internals/number-parse-int');\n\n// `parseInt` method\n// https://tc39.github.io/ecma262/#sec-parseint-string-radix\n$({ global: true, forced: parseInt != parseIntImplementation }, {\n  parseInt: parseIntImplementation\n});\n","require('../modules/es.parse-int');\nvar path = require('../internals/path');\n\nmodule.exports = path.parseInt;\n","var DESCRIPTORS = require('../internals/descriptors');\nvar objectKeys = require('../internals/object-keys');\nvar toIndexedObject = require('../internals/to-indexed-object');\nvar propertyIsEnumerable = require('../internals/object-property-is-enumerable').f;\n\n// `Object.{ entries, values }` methods implementation\nvar createMethod = function (TO_ENTRIES) {\n  return function (it) {\n    var O = toIndexedObject(it);\n    var keys = objectKeys(O);\n    var length = keys.length;\n    var i = 0;\n    var result = [];\n    var key;\n    while (length > i) {\n      key = keys[i++];\n      if (!DESCRIPTORS || propertyIsEnumerable.call(O, key)) {\n        result.push(TO_ENTRIES ? [key, O[key]] : O[key]);\n      }\n    }\n    return result;\n  };\n};\n\nmodule.exports = {\n  // `Object.entries` method\n  // https://tc39.github.io/ecma262/#sec-object.entries\n  entries: createMethod(true),\n  // `Object.values` method\n  // https://tc39.github.io/ecma262/#sec-object.values\n  values: createMethod(false)\n};\n","var $ = require('../internals/export');\nvar $values = require('../internals/object-to-array').values;\n\n// `Object.values` method\n// https://tc39.github.io/ecma262/#sec-object.values\n$({ target: 'Object', stat: true }, {\n  values: function values(O) {\n    return $values(O);\n  }\n});\n","require('../../modules/es.object.values');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Object.values;\n","'use strict';\nvar $ = require('../internals/export');\nvar $filter = require('../internals/array-iteration').filter;\nvar arrayMethodHasSpeciesSupport = require('../internals/array-method-has-species-support');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar HAS_SPECIES_SUPPORT = arrayMethodHasSpeciesSupport('filter');\n// Edge 14- issue\nvar USES_TO_LENGTH = arrayMethodUsesToLength('filter');\n\n// `Array.prototype.filter` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.filter\n// with adding support of @@species\n$({ target: 'Array', proto: true, forced: !HAS_SPECIES_SUPPORT || !USES_TO_LENGTH }, {\n  filter: function filter(callbackfn /* , thisArg */) {\n    return $filter(this, callbackfn, arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n","require('../../../modules/es.array.filter');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').filter;\n","var filter = require('../array/virtual/filter');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.filter;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.filter) ? filter : own;\n};\n","'use strict';\nvar $ = require('../internals/export');\nvar fails = require('../internals/fails');\nvar isArray = require('../internals/is-array');\nvar isObject = require('../internals/is-object');\nvar toObject = require('../internals/to-object');\nvar toLength = require('../internals/to-length');\nvar createProperty = require('../internals/create-property');\nvar arraySpeciesCreate = require('../internals/array-species-create');\nvar arrayMethodHasSpeciesSupport = require('../internals/array-method-has-species-support');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\nvar V8_VERSION = require('../internals/engine-v8-version');\n\nvar IS_CONCAT_SPREADABLE = wellKnownSymbol('isConcatSpreadable');\nvar MAX_SAFE_INTEGER = 0x1FFFFFFFFFFFFF;\nvar MAXIMUM_ALLOWED_INDEX_EXCEEDED = 'Maximum allowed index exceeded';\n\n// We can't use this feature detection in V8 since it causes\n// deoptimization and serious performance degradation\n// https://github.com/zloirock/core-js/issues/679\nvar IS_CONCAT_SPREADABLE_SUPPORT = V8_VERSION >= 51 || !fails(function () {\n  var array = [];\n  array[IS_CONCAT_SPREADABLE] = false;\n  return array.concat()[0] !== array;\n});\n\nvar SPECIES_SUPPORT = arrayMethodHasSpeciesSupport('concat');\n\nvar isConcatSpreadable = function (O) {\n  if (!isObject(O)) return false;\n  var spreadable = O[IS_CONCAT_SPREADABLE];\n  return spreadable !== undefined ? !!spreadable : isArray(O);\n};\n\nvar FORCED = !IS_CONCAT_SPREADABLE_SUPPORT || !SPECIES_SUPPORT;\n\n// `Array.prototype.concat` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.concat\n// with adding support of @@isConcatSpreadable and @@species\n$({ target: 'Array', proto: true, forced: FORCED }, {\n  concat: function concat(arg) { // eslint-disable-line no-unused-vars\n    var O = toObject(this);\n    var A = arraySpeciesCreate(O, 0);\n    var n = 0;\n    var i, k, length, len, E;\n    for (i = -1, length = arguments.length; i < length; i++) {\n      E = i === -1 ? O : arguments[i];\n      if (isConcatSpreadable(E)) {\n        len = toLength(E.length);\n        if (n + len > MAX_SAFE_INTEGER) throw TypeError(MAXIMUM_ALLOWED_INDEX_EXCEEDED);\n        for (k = 0; k < len; k++, n++) if (k in E) createProperty(A, n, E[k]);\n      } else {\n        if (n >= MAX_SAFE_INTEGER) throw TypeError(MAXIMUM_ALLOWED_INDEX_EXCEEDED);\n        createProperty(A, n++, E);\n      }\n    }\n    A.length = n;\n    return A;\n  }\n});\n","require('../../../modules/es.array.concat');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').concat;\n","var concat = require('../array/virtual/concat');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.concat;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.concat) ? concat : own;\n};\n","var $ = require('../internals/export');\nvar isArray = require('../internals/is-array');\n\n// `Array.isArray` method\n// https://tc39.github.io/ecma262/#sec-array.isarray\n$({ target: 'Array', stat: true }, {\n  isArray: isArray\n});\n","require('../../modules/es.array.is-array');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Array.isArray;\n","var parent = require('../../es/array/is-array');\n\nmodule.exports = parent;\n","var _Array$isArray = require(\"../core-js/array/is-array\");\n\nfunction _arrayWithoutHoles(arr) {\n  if (_Array$isArray(arr)) {\n    for (var i = 0, arr2 = new Array(arr.length); i < arr.length; i++) {\n      arr2[i] = arr[i];\n    }\n\n    return arr2;\n  }\n}\n\nmodule.exports = _arrayWithoutHoles;","var anObject = require('../internals/an-object');\n\n// call something on iterator step with safe closing on error\nmodule.exports = function (iterator, fn, value, ENTRIES) {\n  try {\n    return ENTRIES ? fn(anObject(value)[0], value[1]) : fn(value);\n  // 7.4.6 IteratorClose(iterator, completion)\n  } catch (error) {\n    var returnMethod = iterator['return'];\n    if (returnMethod !== undefined) anObject(returnMethod.call(iterator));\n    throw error;\n  }\n};\n","var wellKnownSymbol = require('../internals/well-known-symbol');\nvar Iterators = require('../internals/iterators');\n\nvar ITERATOR = wellKnownSymbol('iterator');\nvar ArrayPrototype = Array.prototype;\n\n// check on default Array iterator\nmodule.exports = function (it) {\n  return it !== undefined && (Iterators.Array === it || ArrayPrototype[ITERATOR] === it);\n};\n","var wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar ITERATOR = wellKnownSymbol('iterator');\nvar SAFE_CLOSING = false;\n\ntry {\n  var called = 0;\n  var iteratorWithReturn = {\n    next: function () {\n      return { done: !!called++ };\n    },\n    'return': function () {\n      SAFE_CLOSING = true;\n    }\n  };\n  iteratorWithReturn[ITERATOR] = function () {\n    return this;\n  };\n  // eslint-disable-next-line no-throw-literal\n  Array.from(iteratorWithReturn, function () { throw 2; });\n} catch (error) { /* empty */ }\n\nmodule.exports = function (exec, SKIP_CLOSING) {\n  if (!SKIP_CLOSING && !SAFE_CLOSING) return false;\n  var ITERATION_SUPPORT = false;\n  try {\n    var object = {};\n    object[ITERATOR] = function () {\n      return {\n        next: function () {\n          return { done: ITERATION_SUPPORT = true };\n        }\n      };\n    };\n    exec(object);\n  } catch (error) { /* empty */ }\n  return ITERATION_SUPPORT;\n};\n","var $ = require('../internals/export');\nvar from = require('../internals/array-from');\nvar checkCorrectnessOfIteration = require('../internals/check-correctness-of-iteration');\n\nvar INCORRECT_ITERATION = !checkCorrectnessOfIteration(function (iterable) {\n  Array.from(iterable);\n});\n\n// `Array.from` method\n// https://tc39.github.io/ecma262/#sec-array.from\n$({ target: 'Array', stat: true, forced: INCORRECT_ITERATION }, {\n  from: from\n});\n","'use strict';\nvar bind = require('../internals/function-bind-context');\nvar toObject = require('../internals/to-object');\nvar callWithSafeIterationClosing = require('../internals/call-with-safe-iteration-closing');\nvar isArrayIteratorMethod = require('../internals/is-array-iterator-method');\nvar toLength = require('../internals/to-length');\nvar createProperty = require('../internals/create-property');\nvar getIteratorMethod = require('../internals/get-iterator-method');\n\n// `Array.from` method implementation\n// https://tc39.github.io/ecma262/#sec-array.from\nmodule.exports = function from(arrayLike /* , mapfn = undefined, thisArg = undefined */) {\n  var O = toObject(arrayLike);\n  var C = typeof this == 'function' ? this : Array;\n  var argumentsLength = arguments.length;\n  var mapfn = argumentsLength > 1 ? arguments[1] : undefined;\n  var mapping = mapfn !== undefined;\n  var iteratorMethod = getIteratorMethod(O);\n  var index = 0;\n  var length, result, step, iterator, next, value;\n  if (mapping) mapfn = bind(mapfn, argumentsLength > 2 ? arguments[2] : undefined, 2);\n  // if the target is not iterable or it's an array with the default iterator - use a simple case\n  if (iteratorMethod != undefined && !(C == Array && isArrayIteratorMethod(iteratorMethod))) {\n    iterator = iteratorMethod.call(O);\n    next = iterator.next;\n    result = new C();\n    for (;!(step = next.call(iterator)).done; index++) {\n      value = mapping ? callWithSafeIterationClosing(iterator, mapfn, [step.value, index], true) : step.value;\n      createProperty(result, index, value);\n    }\n  } else {\n    length = toLength(O.length);\n    result = new C(length);\n    for (;length > index; index++) {\n      value = mapping ? mapfn(O[index], index) : O[index];\n      createProperty(result, index, value);\n    }\n  }\n  result.length = index;\n  return result;\n};\n","require('../../modules/es.string.iterator');\nrequire('../../modules/es.array.from');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Array.from;\n","var classof = require('../internals/classof');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\nvar Iterators = require('../internals/iterators');\n\nvar ITERATOR = wellKnownSymbol('iterator');\n\nmodule.exports = function (it) {\n  var O = Object(it);\n  return O[ITERATOR] !== undefined\n    || '@@iterator' in O\n    // eslint-disable-next-line no-prototype-builtins\n    || Iterators.hasOwnProperty(classof(O));\n};\n","var _Array$from = require(\"../core-js/array/from\");\n\nvar _isIterable = require(\"../core-js/is-iterable\");\n\nfunction _iterableToArray(iter) {\n  if (_isIterable(Object(iter)) || Object.prototype.toString.call(iter) === \"[object Arguments]\") return _Array$from(iter);\n}\n\nmodule.exports = _iterableToArray;","function _nonIterableSpread() {\n  throw new TypeError(\"Invalid attempt to spread non-iterable instance\");\n}\n\nmodule.exports = _nonIterableSpread;","var arrayWithoutHoles = require(\"./arrayWithoutHoles\");\n\nvar iterableToArray = require(\"./iterableToArray\");\n\nvar nonIterableSpread = require(\"./nonIterableSpread\");\n\nfunction _toConsumableArray(arr) {\n  return arrayWithoutHoles(arr) || iterableToArray(arr) || nonIterableSpread();\n}\n\nmodule.exports = _toConsumableArray;","'use strict';\nvar $ = require('../internals/export');\nvar isObject = require('../internals/is-object');\nvar isArray = require('../internals/is-array');\nvar toAbsoluteIndex = require('../internals/to-absolute-index');\nvar toLength = require('../internals/to-length');\nvar toIndexedObject = require('../internals/to-indexed-object');\nvar createProperty = require('../internals/create-property');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\nvar arrayMethodHasSpeciesSupport = require('../internals/array-method-has-species-support');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar HAS_SPECIES_SUPPORT = arrayMethodHasSpeciesSupport('slice');\nvar USES_TO_LENGTH = arrayMethodUsesToLength('slice', { ACCESSORS: true, 0: 0, 1: 2 });\n\nvar SPECIES = wellKnownSymbol('species');\nvar nativeSlice = [].slice;\nvar max = Math.max;\n\n// `Array.prototype.slice` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.slice\n// fallback for not array-like ES3 strings and DOM objects\n$({ target: 'Array', proto: true, forced: !HAS_SPECIES_SUPPORT || !USES_TO_LENGTH }, {\n  slice: function slice(start, end) {\n    var O = toIndexedObject(this);\n    var length = toLength(O.length);\n    var k = toAbsoluteIndex(start, length);\n    var fin = toAbsoluteIndex(end === undefined ? length : end, length);\n    // inline `ArraySpeciesCreate` for usage native `Array#slice` where it's possible\n    var Constructor, result, n;\n    if (isArray(O)) {\n      Constructor = O.constructor;\n      // cross-realm fallback\n      if (typeof Constructor == 'function' && (Constructor === Array || isArray(Constructor.prototype))) {\n        Constructor = undefined;\n      } else if (isObject(Constructor)) {\n        Constructor = Constructor[SPECIES];\n        if (Constructor === null) Constructor = undefined;\n      }\n      if (Constructor === Array || Constructor === undefined) {\n        return nativeSlice.call(O, k, fin);\n      }\n    }\n    result = new (Constructor === undefined ? Array : Constructor)(max(fin - k, 0));\n    for (n = 0; k < fin; k++, n++) if (k in O) createProperty(result, n, O[k]);\n    result.length = n;\n    return result;\n  }\n});\n","require('../../../modules/es.array.slice');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').slice;\n","var slice = require('../array/virtual/slice');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.slice;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.slice) ? slice : own;\n};\n","var $ = require('../internals/export');\nvar fails = require('../internals/fails');\nvar toObject = require('../internals/to-object');\nvar nativeGetPrototypeOf = require('../internals/object-get-prototype-of');\nvar CORRECT_PROTOTYPE_GETTER = require('../internals/correct-prototype-getter');\n\nvar FAILS_ON_PRIMITIVES = fails(function () { nativeGetPrototypeOf(1); });\n\n// `Object.getPrototypeOf` method\n// https://tc39.github.io/ecma262/#sec-object.getprototypeof\n$({ target: 'Object', stat: true, forced: FAILS_ON_PRIMITIVES, sham: !CORRECT_PROTOTYPE_GETTER }, {\n  getPrototypeOf: function getPrototypeOf(it) {\n    return nativeGetPrototypeOf(toObject(it));\n  }\n});\n\n","require('../../modules/es.object.get-prototype-of');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Object.getPrototypeOf;\n","var parent = require('../../es/object/get-prototype-of');\n\nmodule.exports = parent;\n","'use strict';\nvar $ = require('../internals/export');\nvar $indexOf = require('../internals/array-includes').indexOf;\nvar arrayMethodIsStrict = require('../internals/array-method-is-strict');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar nativeIndexOf = [].indexOf;\n\nvar NEGATIVE_ZERO = !!nativeIndexOf && 1 / [1].indexOf(1, -0) < 0;\nvar STRICT_METHOD = arrayMethodIsStrict('indexOf');\nvar USES_TO_LENGTH = arrayMethodUsesToLength('indexOf', { ACCESSORS: true, 1: 0 });\n\n// `Array.prototype.indexOf` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.indexof\n$({ target: 'Array', proto: true, forced: NEGATIVE_ZERO || !STRICT_METHOD || !USES_TO_LENGTH }, {\n  indexOf: function indexOf(searchElement /* , fromIndex = 0 */) {\n    return NEGATIVE_ZERO\n      // convert -0 to +0\n      ? nativeIndexOf.apply(this, arguments) || 0\n      : $indexOf(this, searchElement, arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n","require('../../../modules/es.array.index-of');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').indexOf;\n","var indexOf = require('../array/virtual/index-of');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.indexOf;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.indexOf) ? indexOf : own;\n};\n","var parent = require('../../es/array/is-array');\n\nmodule.exports = parent;\n","'use strict';\nvar DESCRIPTORS = require('../internals/descriptors');\nvar fails = require('../internals/fails');\nvar objectKeys = require('../internals/object-keys');\nvar getOwnPropertySymbolsModule = require('../internals/object-get-own-property-symbols');\nvar propertyIsEnumerableModule = require('../internals/object-property-is-enumerable');\nvar toObject = require('../internals/to-object');\nvar IndexedObject = require('../internals/indexed-object');\n\nvar nativeAssign = Object.assign;\nvar defineProperty = Object.defineProperty;\n\n// `Object.assign` method\n// https://tc39.github.io/ecma262/#sec-object.assign\nmodule.exports = !nativeAssign || fails(function () {\n  // should have correct order of operations (Edge bug)\n  if (DESCRIPTORS && nativeAssign({ b: 1 }, nativeAssign(defineProperty({}, 'a', {\n    enumerable: true,\n    get: function () {\n      defineProperty(this, 'b', {\n        value: 3,\n        enumerable: false\n      });\n    }\n  }), { b: 2 })).b !== 1) return true;\n  // should work with symbols and should have deterministic property order (V8 bug)\n  var A = {};\n  var B = {};\n  // eslint-disable-next-line no-undef\n  var symbol = Symbol();\n  var alphabet = 'abcdefghijklmnopqrst';\n  A[symbol] = 7;\n  alphabet.split('').forEach(function (chr) { B[chr] = chr; });\n  return nativeAssign({}, A)[symbol] != 7 || objectKeys(nativeAssign({}, B)).join('') != alphabet;\n}) ? function assign(target, source) { // eslint-disable-line no-unused-vars\n  var T = toObject(target);\n  var argumentsLength = arguments.length;\n  var index = 1;\n  var getOwnPropertySymbols = getOwnPropertySymbolsModule.f;\n  var propertyIsEnumerable = propertyIsEnumerableModule.f;\n  while (argumentsLength > index) {\n    var S = IndexedObject(arguments[index++]);\n    var keys = getOwnPropertySymbols ? objectKeys(S).concat(getOwnPropertySymbols(S)) : objectKeys(S);\n    var length = keys.length;\n    var j = 0;\n    var key;\n    while (length > j) {\n      key = keys[j++];\n      if (!DESCRIPTORS || propertyIsEnumerable.call(S, key)) T[key] = S[key];\n    }\n  } return T;\n} : nativeAssign;\n","var $ = require('../internals/export');\nvar assign = require('../internals/object-assign');\n\n// `Object.assign` method\n// https://tc39.github.io/ecma262/#sec-object.assign\n$({ target: 'Object', stat: true, forced: Object.assign !== assign }, {\n  assign: assign\n});\n","require('../../modules/es.object.assign');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Object.assign;\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.iterator` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.iterator\ndefineWellKnownSymbol('iterator');\n","require('../../modules/es.symbol.iterator');\nrequire('../../modules/es.string.iterator');\nrequire('../../modules/web.dom-collections.iterator');\nvar WrappedWellKnownSymbolModule = require('../../internals/well-known-symbol-wrapped');\n\nmodule.exports = WrappedWellKnownSymbolModule.f('iterator');\n","var parent = require('../../es/symbol/iterator');\n\nmodule.exports = parent;\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.asyncIterator` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.asynciterator\ndefineWellKnownSymbol('asyncIterator');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.hasInstance` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.hasinstance\ndefineWellKnownSymbol('hasInstance');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.isConcatSpreadable` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.isconcatspreadable\ndefineWellKnownSymbol('isConcatSpreadable');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.match` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.match\ndefineWellKnownSymbol('match');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.matchAll` well-known symbol\ndefineWellKnownSymbol('matchAll');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.replace` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.replace\ndefineWellKnownSymbol('replace');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.search` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.search\ndefineWellKnownSymbol('search');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.species` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.species\ndefineWellKnownSymbol('species');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.split` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.split\ndefineWellKnownSymbol('split');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.toPrimitive` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.toprimitive\ndefineWellKnownSymbol('toPrimitive');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.toStringTag` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.tostringtag\ndefineWellKnownSymbol('toStringTag');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.unscopables` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.unscopables\ndefineWellKnownSymbol('unscopables');\n","var setToStringTag = require('../internals/set-to-string-tag');\n\n// Math[@@toStringTag] property\n// https://tc39.github.io/ecma262/#sec-math-@@tostringtag\nsetToStringTag(Math, 'Math', true);\n","var global = require('../internals/global');\nvar setToStringTag = require('../internals/set-to-string-tag');\n\n// JSON[@@toStringTag] property\n// https://tc39.github.io/ecma262/#sec-json-@@tostringtag\nsetToStringTag(global.JSON, 'JSON', true);\n","require('../../modules/es.array.concat');\nrequire('../../modules/es.object.to-string');\nrequire('../../modules/es.symbol');\nrequire('../../modules/es.symbol.async-iterator');\nrequire('../../modules/es.symbol.description');\nrequire('../../modules/es.symbol.has-instance');\nrequire('../../modules/es.symbol.is-concat-spreadable');\nrequire('../../modules/es.symbol.iterator');\nrequire('../../modules/es.symbol.match');\nrequire('../../modules/es.symbol.match-all');\nrequire('../../modules/es.symbol.replace');\nrequire('../../modules/es.symbol.search');\nrequire('../../modules/es.symbol.species');\nrequire('../../modules/es.symbol.split');\nrequire('../../modules/es.symbol.to-primitive');\nrequire('../../modules/es.symbol.to-string-tag');\nrequire('../../modules/es.symbol.unscopables');\nrequire('../../modules/es.math.to-string-tag');\nrequire('../../modules/es.json.to-string-tag');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Symbol;\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.asyncDispose` well-known symbol\n// https://github.com/tc39/proposal-using-statement\ndefineWellKnownSymbol('asyncDispose');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.dispose` well-known symbol\n// https://github.com/tc39/proposal-using-statement\ndefineWellKnownSymbol('dispose');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.observable` well-known symbol\n// https://github.com/tc39/proposal-observable\ndefineWellKnownSymbol('observable');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.patternMatch` well-known symbol\n// https://github.com/tc39/proposal-pattern-matching\ndefineWellKnownSymbol('patternMatch');\n","// TODO: remove from `core-js@4`\nvar defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\ndefineWellKnownSymbol('replaceAll');\n","var parent = require('../../es/symbol');\nrequire('../../modules/esnext.symbol.async-dispose');\nrequire('../../modules/esnext.symbol.dispose');\nrequire('../../modules/esnext.symbol.observable');\nrequire('../../modules/esnext.symbol.pattern-match');\n// TODO: Remove from `core-js@4`\nrequire('../../modules/esnext.symbol.replace-all');\n\nmodule.exports = parent;\n","var _Symbol$iterator = require(\"../core-js/symbol/iterator\");\n\nvar _Symbol = require(\"../core-js/symbol\");\n\nfunction _typeof(obj) {\n  \"@babel/helpers - typeof\";\n\n  if (typeof _Symbol === \"function\" && typeof _Symbol$iterator === \"symbol\") {\n    module.exports = _typeof = function _typeof(obj) {\n      return typeof obj;\n    };\n  } else {\n    module.exports = _typeof = function _typeof(obj) {\n      return obj && typeof _Symbol === \"function\" && obj.constructor === _Symbol && obj !== _Symbol.prototype ? \"symbol\" : typeof obj;\n    };\n  }\n\n  return _typeof(obj);\n}\n\nmodule.exports = _typeof;","var _Array$isArray = require(\"../core-js/array/is-array\");\n\nfunction _arrayWithHoles(arr) {\n  if (_Array$isArray(arr)) return arr;\n}\n\nmodule.exports = _arrayWithHoles;","var _getIterator = require(\"../core-js/get-iterator\");\n\nvar _isIterable = require(\"../core-js/is-iterable\");\n\nfunction _iterableToArrayLimit(arr, i) {\n  if (!(_isIterable(Object(arr)) || Object.prototype.toString.call(arr) === \"[object Arguments]\")) {\n    return;\n  }\n\n  var _arr = [];\n  var _n = true;\n  var _d = false;\n  var _e = undefined;\n\n  try {\n    for (var _i = _getIterator(arr), _s; !(_n = (_s = _i.next()).done); _n = true) {\n      _arr.push(_s.value);\n\n      if (i && _arr.length === i) break;\n    }\n  } catch (err) {\n    _d = true;\n    _e = err;\n  } finally {\n    try {\n      if (!_n && _i[\"return\"] != null) _i[\"return\"]();\n    } finally {\n      if (_d) throw _e;\n    }\n  }\n\n  return _arr;\n}\n\nmodule.exports = _iterableToArrayLimit;","function _nonIterableRest() {\n  throw new TypeError(\"Invalid attempt to destructure non-iterable instance\");\n}\n\nmodule.exports = _nonIterableRest;","var arrayWithHoles = require(\"./arrayWithHoles\");\n\nvar iterableToArrayLimit = require(\"./iterableToArrayLimit\");\n\nvar nonIterableRest = require(\"./nonIterableRest\");\n\nfunction _slicedToArray(arr, i) {\n  return arrayWithHoles(arr) || iterableToArrayLimit(arr, i) || nonIterableRest();\n}\n\nmodule.exports = _slicedToArray;","var $ = require('../internals/export');\n\n// `Date.now` method\n// https://tc39.github.io/ecma262/#sec-date.now\n$({ target: 'Date', stat: true }, {\n  now: function now() {\n    return new Date().getTime();\n  }\n});\n","require('../../modules/es.date.now');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Date.now;\n","/**\n * vis-util\n * https://github.com/visjs/vis-util\n *\n * utilitie collection for visjs\n *\n * @version 4.0.0\n * @date    2020-02-29T15:35:13.178Z\n *\n * @copyright (c) 2011-2017 Almende B.V, http://almende.com\n * @copyright (c) 2017-2019 visjs contributors, https://github.com/visjs\n *\n * @license\n * vis.js is dual licensed under both\n *\n *   1. The Apache 2.0 License\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *   and\n *\n *   2. The MIT License\n *      http://opensource.org/licenses/MIT\n *\n * vis.js may be distributed under either license.\n */\n\n/**\r\n * Seedable, fast and reasonably good (not crypto but more than okay for our\r\n * needs) random number generator.\r\n *\r\n * @remarks\r\n * Adapted from {@link https://web.archive.org/web/20110429100736/http://baagoe.com:80/en/RandomMusings/javascript}.\r\n * Original algorithm created by Johannes Baagøe \\<baagoe\\@baagoe.com\\> in 2010.\r\n */\r\n/**\r\n * Create a seeded pseudo random generator based on Alea by Johannes Baagøe.\r\n *\r\n * @param seed - All supplied arguments will be used as a seed. In case nothing\r\n * is supplied the current time will be used to seed the generator.\r\n *\r\n * @returns A ready to use seeded generator.\r\n */\r\nfunction Alea(...seed) {\r\n    return AleaImplementation(seed.length ? seed : [Date.now()]);\r\n}\r\n/**\r\n * An implementation of [[Alea]] without user input validation.\r\n *\r\n * @param seed - The data that will be used to seed the generator.\r\n *\r\n * @returns A ready to use seeded generator.\r\n */\r\nfunction AleaImplementation(seed) {\r\n    let [s0, s1, s2] = mashSeed(seed);\r\n    let c = 1;\r\n    const random = () => {\r\n        const t = 2091639 * s0 + c * 2.3283064365386963e-10; // 2^-32\r\n        s0 = s1;\r\n        s1 = s2;\r\n        return (s2 = t - (c = t | 0));\r\n    };\r\n    random.uint32 = () => random() * 0x100000000; // 2^32\r\n    random.fract53 = () => random() + ((random() * 0x200000) | 0) * 1.1102230246251565e-16; // 2^-53\r\n    random.algorithm = \"Alea\";\r\n    random.seed = seed;\r\n    random.version = \"0.9\";\r\n    return random;\r\n}\r\n/**\r\n * Turn arbitrary data into values [[AleaImplementation]] can use to generate\r\n * random numbers.\r\n *\r\n * @param seed - Arbitrary data that will be used as the seed.\r\n *\r\n * @returns Three numbers to use as initial values for [[AleaImplementation]].\r\n */\r\nfunction mashSeed(...seed) {\r\n    const mash = Mash();\r\n    let s0 = mash(\" \");\r\n    let s1 = mash(\" \");\r\n    let s2 = mash(\" \");\r\n    for (let i = 0; i < seed.length; i++) {\r\n        s0 -= mash(seed[i]);\r\n        if (s0 < 0) {\r\n            s0 += 1;\r\n        }\r\n        s1 -= mash(seed[i]);\r\n        if (s1 < 0) {\r\n            s1 += 1;\r\n        }\r\n        s2 -= mash(seed[i]);\r\n        if (s2 < 0) {\r\n            s2 += 1;\r\n        }\r\n    }\r\n    return [s0, s1, s2];\r\n}\r\n/**\r\n * Create a new mash function.\r\n *\r\n * @returns A nonpure function that takes arbitrary [[Mashable]] data and turns\r\n * them into numbers.\r\n */\r\nfunction Mash() {\r\n    let n = 0xefc8249d;\r\n    return function (data) {\r\n        const string = data.toString();\r\n        for (let i = 0; i < string.length; i++) {\r\n            n += string.charCodeAt(i);\r\n            let h = 0.02519603282416938 * n;\r\n            n = h >>> 0;\r\n            h -= n;\r\n            h *= n;\r\n            n = h >>> 0;\r\n            h -= n;\r\n            n += h * 0x100000000; // 2^32\r\n        }\r\n        return (n >>> 0) * 2.3283064365386963e-10; // 2^-32\r\n    };\r\n}\n\n// utility functions\r\n// parse ASP.Net Date pattern,\r\n// for example '/Date(1198908717056)/' or '/Date(1198908717056-0700)/'\r\n// code from http://momentjs.com/\r\nconst ASPDateRegex = /^\\/?Date\\((-?\\d+)/i;\r\n// Color REs\r\nconst fullHexRE = /^#?([a-f\\d]{2})([a-f\\d]{2})([a-f\\d]{2})$/i;\r\nconst shortHexRE = /^#?([a-f\\d])([a-f\\d])([a-f\\d])$/i;\r\nconst rgbRE = /^rgb\\( *(1?\\d{1,2}|2[0-4]\\d|25[0-5]) *, *(1?\\d{1,2}|2[0-4]\\d|25[0-5]) *, *(1?\\d{1,2}|2[0-4]\\d|25[0-5]) *\\)$/i;\r\nconst rgbaRE = /^rgba\\( *(1?\\d{1,2}|2[0-4]\\d|25[0-5]) *, *(1?\\d{1,2}|2[0-4]\\d|25[0-5]) *, *(1?\\d{1,2}|2[0-4]\\d|25[0-5]) *, *([01]|0?\\.\\d+) *\\)$/i;\r\n/**\r\n * Test whether given object is a number.\r\n *\r\n * @param value - Input value of unknown type.\r\n *\r\n * @returns True if number, false otherwise.\r\n */\r\nfunction isNumber(value) {\r\n    return value instanceof Number || typeof value === \"number\";\r\n}\r\n/**\r\n * Remove everything in the DOM object.\r\n *\r\n * @param DOMobject - Node whose child nodes will be recursively deleted.\r\n */\r\nfunction recursiveDOMDelete(DOMobject) {\r\n    if (DOMobject) {\r\n        while (DOMobject.hasChildNodes() === true) {\r\n            const child = DOMobject.firstChild;\r\n            if (child) {\r\n                recursiveDOMDelete(child);\r\n                DOMobject.removeChild(child);\r\n            }\r\n        }\r\n    }\r\n}\r\n/**\r\n * Test whether given object is a string.\r\n *\r\n * @param value - Input value of unknown type.\r\n *\r\n * @returns True if string, false otherwise.\r\n */\r\nfunction isString(value) {\r\n    return value instanceof String || typeof value === \"string\";\r\n}\r\n/**\r\n * Test whether given object is a object (not primitive or null).\r\n *\r\n * @param value - Input value of unknown type.\r\n *\r\n * @returns True if not null object, false otherwise.\r\n */\r\nfunction isObject(value) {\r\n    return typeof value === \"object\" && value !== null;\r\n}\r\n/**\r\n * Test whether given object is a Date, or a String containing a Date.\r\n *\r\n * @param value - Input value of unknown type.\r\n *\r\n * @returns True if Date instance or string date representation, false otherwise.\r\n */\r\nfunction isDate(value) {\r\n    if (value instanceof Date) {\r\n        return true;\r\n    }\r\n    else if (isString(value)) {\r\n        // test whether this string contains a date\r\n        const match = ASPDateRegex.exec(value);\r\n        if (match) {\r\n            return true;\r\n        }\r\n        else if (!isNaN(Date.parse(value))) {\r\n            return true;\r\n        }\r\n    }\r\n    return false;\r\n}\r\n/**\r\n * Copy property from b to a if property present in a.\r\n * If property in b explicitly set to null, delete it if `allowDeletion` set.\r\n *\r\n * Internal helper routine, should not be exported. Not added to `exports` for that reason.\r\n *\r\n * @param a - Target object.\r\n * @param b - Source object.\r\n * @param prop - Name of property to copy from b to a.\r\n * @param allowDeletion - If true, delete property in a if explicitly set to null in b.\r\n */\r\nfunction copyOrDelete(a, b, prop, allowDeletion) {\r\n    let doDeletion = false;\r\n    if (allowDeletion === true) {\r\n        doDeletion = b[prop] === null && a[prop] !== undefined;\r\n    }\r\n    if (doDeletion) {\r\n        delete a[prop];\r\n    }\r\n    else {\r\n        a[prop] = b[prop]; // Remember, this is a reference copy!\r\n    }\r\n}\r\n/**\r\n * Fill an object with a possibly partially defined other object.\r\n *\r\n * Only copies values for the properties already present in a.\r\n * That means an object is not created on a property if only the b object has it.\r\n *\r\n * @param a - The object that will have it's properties updated.\r\n * @param b - The object with property updates.\r\n * @param allowDeletion - If true, delete properties in a that are explicitly set to null in b.\r\n */\r\nfunction fillIfDefined(a, b, allowDeletion = false) {\r\n    // NOTE: iteration of properties of a\r\n    // NOTE: prototype properties iterated over as well\r\n    for (const prop in a) {\r\n        if (b[prop] !== undefined) {\r\n            if (b[prop] === null || typeof b[prop] !== \"object\") {\r\n                // Note: typeof null === 'object'\r\n                copyOrDelete(a, b, prop, allowDeletion);\r\n            }\r\n            else {\r\n                const aProp = a[prop];\r\n                const bProp = b[prop];\r\n                if (isObject(aProp) && isObject(bProp)) {\r\n                    fillIfDefined(aProp, bProp, allowDeletion);\r\n                }\r\n            }\r\n        }\r\n    }\r\n}\r\n/**\r\n * Copy the values of all of the enumerable own properties from one or more source objects to a\r\n * target object. Returns the target object.\r\n *\r\n * @param target - The target object to copy to.\r\n * @param source - The source object from which to copy properties.\r\n *\r\n * @returns The target object.\r\n */\r\nconst extend = Object.assign;\r\n/**\r\n * Extend object a with selected properties of object b or a series of objects.\r\n *\r\n * @remarks\r\n * Only properties with defined values are copied.\r\n *\r\n * @param props - Properties to be copied to a.\r\n * @param a - The target.\r\n * @param others - The sources.\r\n *\r\n * @returns Argument a.\r\n */\r\nfunction selectiveExtend(props, a, ...others) {\r\n    if (!Array.isArray(props)) {\r\n        throw new Error(\"Array with property names expected as first argument\");\r\n    }\r\n    for (const other of others) {\r\n        for (let p = 0; p < props.length; p++) {\r\n            const prop = props[p];\r\n            if (other && Object.prototype.hasOwnProperty.call(other, prop)) {\r\n                a[prop] = other[prop];\r\n            }\r\n        }\r\n    }\r\n    return a;\r\n}\r\n/**\r\n * Extend object a with selected properties of object b.\r\n * Only properties with defined values are copied.\r\n *\r\n * @remarks\r\n * Previous version of this routine implied that multiple source objects could\r\n * be used; however, the implementation was **wrong**. Since multiple (\\>1)\r\n * sources weren't used anywhere in the `vis.js` code, this has been removed\r\n *\r\n * @param props - Names of first-level properties to copy over.\r\n * @param a - Target object.\r\n * @param b - Source object.\r\n * @param allowDeletion - If true, delete property in a if explicitly set to null in b.\r\n *\r\n * @returns Argument a.\r\n */\r\nfunction selectiveDeepExtend(props, a, b, allowDeletion = false) {\r\n    // TODO: add support for Arrays to deepExtend\r\n    if (Array.isArray(b)) {\r\n        throw new TypeError(\"Arrays are not supported by deepExtend\");\r\n    }\r\n    for (let p = 0; p < props.length; p++) {\r\n        const prop = props[p];\r\n        if (Object.prototype.hasOwnProperty.call(b, prop)) {\r\n            if (b[prop] && b[prop].constructor === Object) {\r\n                if (a[prop] === undefined) {\r\n                    a[prop] = {};\r\n                }\r\n                if (a[prop].constructor === Object) {\r\n                    deepExtend(a[prop], b[prop], false, allowDeletion);\r\n                }\r\n                else {\r\n                    copyOrDelete(a, b, prop, allowDeletion);\r\n                }\r\n            }\r\n            else if (Array.isArray(b[prop])) {\r\n                throw new TypeError(\"Arrays are not supported by deepExtend\");\r\n            }\r\n            else {\r\n                copyOrDelete(a, b, prop, allowDeletion);\r\n            }\r\n        }\r\n    }\r\n    return a;\r\n}\r\n/**\r\n * Extend object `a` with properties of object `b`, ignoring properties which\r\n * are explicitly specified to be excluded.\r\n *\r\n * @remarks\r\n * The properties of `b` are considered for copying. Properties which are\r\n * themselves objects are are also extended. Only properties with defined\r\n * values are copied.\r\n *\r\n * @param propsToExclude - Names of properties which should *not* be copied.\r\n * @param a - Object to extend.\r\n * @param b - Object to take properties from for extension.\r\n * @param allowDeletion - If true, delete properties in a that are explicitly\r\n * set to null in b.\r\n *\r\n * @returns Argument a.\r\n */\r\nfunction selectiveNotDeepExtend(propsToExclude, a, b, allowDeletion = false) {\r\n    // TODO: add support for Arrays to deepExtend\r\n    // NOTE: array properties have an else-below; apparently, there is a problem here.\r\n    if (Array.isArray(b)) {\r\n        throw new TypeError(\"Arrays are not supported by deepExtend\");\r\n    }\r\n    for (const prop in b) {\r\n        if (!Object.prototype.hasOwnProperty.call(b, prop)) {\r\n            continue;\r\n        } // Handle local properties only\r\n        if (propsToExclude.indexOf(prop) !== -1) {\r\n            continue;\r\n        } // In exclusion list, skip\r\n        if (b[prop] && b[prop].constructor === Object) {\r\n            if (a[prop] === undefined) {\r\n                a[prop] = {};\r\n            }\r\n            if (a[prop].constructor === Object) {\r\n                deepExtend(a[prop], b[prop]); // NOTE: allowDeletion not propagated!\r\n            }\r\n            else {\r\n                copyOrDelete(a, b, prop, allowDeletion);\r\n            }\r\n        }\r\n        else if (Array.isArray(b[prop])) {\r\n            a[prop] = [];\r\n            for (let i = 0; i < b[prop].length; i++) {\r\n                a[prop].push(b[prop][i]);\r\n            }\r\n        }\r\n        else {\r\n            copyOrDelete(a, b, prop, allowDeletion);\r\n        }\r\n    }\r\n    return a;\r\n}\r\n/**\r\n * Deep extend an object a with the properties of object b.\r\n *\r\n * @param a - Target object.\r\n * @param b - Source object.\r\n * @param protoExtend - If true, the prototype values will also be extended.\r\n * (That is the options objects that inherit from others will also get the\r\n * inherited options).\r\n * @param allowDeletion - If true, the values of fields that are null will be deleted.\r\n *\r\n * @returns Argument a.\r\n */\r\nfunction deepExtend(a, b, protoExtend = false, allowDeletion = false) {\r\n    for (const prop in b) {\r\n        if (Object.prototype.hasOwnProperty.call(b, prop) || protoExtend === true) {\r\n            if (typeof b[prop] === \"object\" &&\r\n                b[prop] !== null &&\r\n                Object.getPrototypeOf(b[prop]) === Object.prototype) {\r\n                if (a[prop] === undefined) {\r\n                    a[prop] = deepExtend({}, b[prop], protoExtend); // NOTE: allowDeletion not propagated!\r\n                }\r\n                else if (typeof a[prop] === \"object\" &&\r\n                    a[prop] !== null &&\r\n                    Object.getPrototypeOf(a[prop]) === Object.prototype) {\r\n                    deepExtend(a[prop], b[prop], protoExtend); // NOTE: allowDeletion not propagated!\r\n                }\r\n                else {\r\n                    copyOrDelete(a, b, prop, allowDeletion);\r\n                }\r\n            }\r\n            else if (Array.isArray(b[prop])) {\r\n                a[prop] = b[prop].slice();\r\n            }\r\n            else {\r\n                copyOrDelete(a, b, prop, allowDeletion);\r\n            }\r\n        }\r\n    }\r\n    return a;\r\n}\r\n/**\r\n * Test whether all elements in two arrays are equal.\r\n *\r\n * @param a - First array.\r\n * @param b - Second array.\r\n *\r\n * @returns True if both arrays have the same length and same elements (1 = '1').\r\n */\r\nfunction equalArray(a, b) {\r\n    if (a.length !== b.length) {\r\n        return false;\r\n    }\r\n    for (let i = 0, len = a.length; i < len; i++) {\r\n        if (a[i] != b[i]) {\r\n            return false;\r\n        }\r\n    }\r\n    return true;\r\n}\r\n/**\r\n * Get the type of an object, for example exports.getType([]) returns 'Array'.\r\n *\r\n * @param object - Input value of unknown type.\r\n *\r\n * @returns Detected type.\r\n */\r\nfunction getType(object) {\r\n    const type = typeof object;\r\n    if (type === \"object\") {\r\n        if (object === null) {\r\n            return \"null\";\r\n        }\r\n        if (object instanceof Boolean) {\r\n            return \"Boolean\";\r\n        }\r\n        if (object instanceof Number) {\r\n            return \"Number\";\r\n        }\r\n        if (object instanceof String) {\r\n            return \"String\";\r\n        }\r\n        if (Array.isArray(object)) {\r\n            return \"Array\";\r\n        }\r\n        if (object instanceof Date) {\r\n            return \"Date\";\r\n        }\r\n        return \"Object\";\r\n    }\r\n    if (type === \"number\") {\r\n        return \"Number\";\r\n    }\r\n    if (type === \"boolean\") {\r\n        return \"Boolean\";\r\n    }\r\n    if (type === \"string\") {\r\n        return \"String\";\r\n    }\r\n    if (type === undefined) {\r\n        return \"undefined\";\r\n    }\r\n    return type;\r\n}\r\n/**\r\n * Used to extend an array and copy it. This is used to propagate paths recursively.\r\n *\r\n * @param arr - First part.\r\n * @param newValue - The value to be aadded into the array.\r\n *\r\n * @returns A new array with all items from arr and newValue (which is last).\r\n */\r\nfunction copyAndExtendArray(arr, newValue) {\r\n    return [...arr, newValue];\r\n}\r\n/**\r\n * Used to extend an array and copy it. This is used to propagate paths recursively.\r\n *\r\n * @param arr - The array to be copied.\r\n *\r\n * @returns Shallow copy of arr.\r\n */\r\nfunction copyArray(arr) {\r\n    return arr.slice();\r\n}\r\n/**\r\n * Retrieve the absolute left value of a DOM element.\r\n *\r\n * @param elem - A dom element, for example a div.\r\n *\r\n * @returns The absolute left position of this element in the browser page.\r\n */\r\nfunction getAbsoluteLeft(elem) {\r\n    return elem.getBoundingClientRect().left;\r\n}\r\n/**\r\n * Retrieve the absolute right value of a DOM element.\r\n *\r\n * @param elem - A dom element, for example a div.\r\n *\r\n * @returns The absolute right position of this element in the browser page.\r\n */\r\nfunction getAbsoluteRight(elem) {\r\n    return elem.getBoundingClientRect().right;\r\n}\r\n/**\r\n * Retrieve the absolute top value of a DOM element.\r\n *\r\n * @param elem - A dom element, for example a div.\r\n *\r\n * @returns The absolute top position of this element in the browser page.\r\n */\r\nfunction getAbsoluteTop(elem) {\r\n    return elem.getBoundingClientRect().top;\r\n}\r\n/**\r\n * Add a className to the given elements style.\r\n *\r\n * @param elem - The element to which the classes will be added.\r\n * @param classNames - Space separated list of classes.\r\n */\r\nfunction addClassName(elem, classNames) {\r\n    let classes = elem.className.split(\" \");\r\n    const newClasses = classNames.split(\" \");\r\n    classes = classes.concat(newClasses.filter(function (className) {\r\n        return classes.indexOf(className) < 0;\r\n    }));\r\n    elem.className = classes.join(\" \");\r\n}\r\n/**\r\n * Remove a className from the given elements style.\r\n *\r\n * @param elem - The element from which the classes will be removed.\r\n * @param classNames - Space separated list of classes.\r\n */\r\nfunction removeClassName(elem, classNames) {\r\n    let classes = elem.className.split(\" \");\r\n    const oldClasses = classNames.split(\" \");\r\n    classes = classes.filter(function (className) {\r\n        return oldClasses.indexOf(className) < 0;\r\n    });\r\n    elem.className = classes.join(\" \");\r\n}\r\n/**\r\n * For each method for both arrays and objects.\r\n * In case of an array, the built-in Array.forEach() is applied (**No, it's not!**).\r\n * In case of an Object, the method loops over all properties of the object.\r\n *\r\n * @param object - An Object or Array to be iterated over.\r\n * @param callback - Array.forEach-like callback.\r\n */\r\nfunction forEach(object, callback) {\r\n    if (Array.isArray(object)) {\r\n        // array\r\n        const len = object.length;\r\n        for (let i = 0; i < len; i++) {\r\n            callback(object[i], i, object);\r\n        }\r\n    }\r\n    else {\r\n        // object\r\n        for (const key in object) {\r\n            if (Object.prototype.hasOwnProperty.call(object, key)) {\r\n                callback(object[key], key, object);\r\n            }\r\n        }\r\n    }\r\n}\r\n/**\r\n * Convert an object into an array: all objects properties are put into the array. The resulting array is unordered.\r\n *\r\n * @param o - Object that contains the properties and methods.\r\n *\r\n * @returns An array of unordered values.\r\n */\r\nconst toArray = Object.values;\r\n/**\r\n * Update a property in an object.\r\n *\r\n * @param object - The object whose property will be updated.\r\n * @param key - Name of the property to be updated.\r\n * @param value - The new value to be assigned.\r\n *\r\n * @returns Whether the value was updated (true) or already strictly the same in the original object (false).\r\n */\r\nfunction updateProperty(object, key, value) {\r\n    if (object[key] !== value) {\r\n        object[key] = value;\r\n        return true;\r\n    }\r\n    else {\r\n        return false;\r\n    }\r\n}\r\n/**\r\n * Throttle the given function to be only executed once per animation frame.\r\n *\r\n * @param fn - The original function.\r\n *\r\n * @returns The throttled function.\r\n */\r\nfunction throttle(fn) {\r\n    let scheduled = false;\r\n    return () => {\r\n        if (!scheduled) {\r\n            scheduled = true;\r\n            requestAnimationFrame(() => {\r\n                scheduled = false;\r\n                fn();\r\n            });\r\n        }\r\n    };\r\n}\r\n/**\r\n * Add and event listener. Works for all browsers.\r\n *\r\n * @param element - The element to bind the event listener to.\r\n * @param action - Same as Element.addEventListener(action, —, —).\r\n * @param listener - Same as Element.addEventListener(—, listener, —).\r\n * @param useCapture - Same as Element.addEventListener(—, —, useCapture).\r\n */\r\nfunction addEventListener(element, action, listener, useCapture) {\r\n    if (element.addEventListener) {\r\n        if (useCapture === undefined) {\r\n            useCapture = false;\r\n        }\r\n        if (action === \"mousewheel\" &&\r\n            navigator.userAgent.indexOf(\"Firefox\") >= 0) {\r\n            action = \"DOMMouseScroll\"; // For Firefox\r\n        }\r\n        element.addEventListener(action, listener, useCapture);\r\n    }\r\n    else {\r\n        // @TODO: IE types? Does anyone care?\r\n        element.attachEvent(\"on\" + action, listener); // IE browsers\r\n    }\r\n}\r\n/**\r\n * Remove an event listener from an element.\r\n *\r\n * @param element - The element to bind the event listener to.\r\n * @param action - Same as Element.removeEventListener(action, —, —).\r\n * @param listener - Same as Element.removeEventListener(—, listener, —).\r\n * @param useCapture - Same as Element.removeEventListener(—, —, useCapture).\r\n */\r\nfunction removeEventListener(element, action, listener, useCapture) {\r\n    if (element.removeEventListener) {\r\n        // non-IE browsers\r\n        if (useCapture === undefined) {\r\n            useCapture = false;\r\n        }\r\n        if (action === \"mousewheel\" &&\r\n            navigator.userAgent.indexOf(\"Firefox\") >= 0) {\r\n            action = \"DOMMouseScroll\"; // For Firefox\r\n        }\r\n        element.removeEventListener(action, listener, useCapture);\r\n    }\r\n    else {\r\n        // @TODO: IE types? Does anyone care?\r\n        element.detachEvent(\"on\" + action, listener); // IE browsers\r\n    }\r\n}\r\n/**\r\n * Cancels the event's default action if it is cancelable, without stopping further propagation of the event.\r\n *\r\n * @param event - The event whose default action should be prevented.\r\n */\r\nfunction preventDefault(event) {\r\n    if (!event) {\r\n        event = window.event;\r\n    }\r\n    if (!event) ;\r\n    else if (event.preventDefault) {\r\n        event.preventDefault(); // non-IE browsers\r\n    }\r\n    else {\r\n        // @TODO: IE types? Does anyone care?\r\n        event.returnValue = false; // IE browsers\r\n    }\r\n}\r\n/**\r\n * Get HTML element which is the target of the event.\r\n *\r\n * @param event - The event.\r\n *\r\n * @returns The element or null if not obtainable.\r\n */\r\nfunction getTarget(event = window.event) {\r\n    // code from http://www.quirksmode.org/js/events_properties.html\r\n    // @TODO: EventTarget can be almost anything, is it okay to return only Elements?\r\n    let target = null;\r\n    if (!event) ;\r\n    else if (event.target) {\r\n        target = event.target;\r\n    }\r\n    else if (event.srcElement) {\r\n        target = event.srcElement;\r\n    }\r\n    if (!(target instanceof Element)) {\r\n        return null;\r\n    }\r\n    if (target.nodeType != null && target.nodeType == 3) {\r\n        // defeat Safari bug\r\n        target = target.parentNode;\r\n        if (!(target instanceof Element)) {\r\n            return null;\r\n        }\r\n    }\r\n    return target;\r\n}\r\n/**\r\n * Check if given element contains given parent somewhere in the DOM tree.\r\n *\r\n * @param element - The element to be tested.\r\n * @param parent - The ancestor (not necessarily parent) of the element.\r\n *\r\n * @returns True if parent is an ancestor of the element, false otherwise.\r\n */\r\nfunction hasParent(element, parent) {\r\n    let elem = element;\r\n    while (elem) {\r\n        if (elem === parent) {\r\n            return true;\r\n        }\r\n        else if (elem.parentNode) {\r\n            elem = elem.parentNode;\r\n        }\r\n        else {\r\n            return false;\r\n        }\r\n    }\r\n    return false;\r\n}\r\nconst option = {\r\n    /**\r\n     * Convert a value into a boolean.\r\n     *\r\n     * @param value - Value to be converted intoboolean, a function will be executed as `(() => unknown)`.\r\n     * @param defaultValue - If the value or the return value of the function == null then this will be returned.\r\n     *\r\n     * @returns Corresponding boolean value, if none then the default value, if none then null.\r\n     */\r\n    asBoolean(value, defaultValue) {\r\n        if (typeof value == \"function\") {\r\n            value = value();\r\n        }\r\n        if (value != null) {\r\n            return value != false;\r\n        }\r\n        return defaultValue || null;\r\n    },\r\n    /**\r\n     * Convert a value into a number.\r\n     *\r\n     * @param value - Value to be converted intonumber, a function will be executed as `(() => unknown)`.\r\n     * @param defaultValue - If the value or the return value of the function == null then this will be returned.\r\n     *\r\n     * @returns Corresponding **boxed** number value, if none then the default value, if none then null.\r\n     */\r\n    asNumber(value, defaultValue) {\r\n        if (typeof value == \"function\") {\r\n            value = value();\r\n        }\r\n        if (value != null) {\r\n            return Number(value) || defaultValue || null;\r\n        }\r\n        return defaultValue || null;\r\n    },\r\n    /**\r\n     * Convert a value into a string.\r\n     *\r\n     * @param value - Value to be converted intostring, a function will be executed as `(() => unknown)`.\r\n     * @param defaultValue - If the value or the return value of the function == null then this will be returned.\r\n     *\r\n     * @returns Corresponding **boxed** string value, if none then the default value, if none then null.\r\n     */\r\n    asString(value, defaultValue) {\r\n        if (typeof value == \"function\") {\r\n            value = value();\r\n        }\r\n        if (value != null) {\r\n            return String(value);\r\n        }\r\n        return defaultValue || null;\r\n    },\r\n    /**\r\n     * Convert a value into a size.\r\n     *\r\n     * @param value - Value to be converted intosize, a function will be executed as `(() => unknown)`.\r\n     * @param defaultValue - If the value or the return value of the function == null then this will be returned.\r\n     *\r\n     * @returns Corresponding string value (number + 'px'), if none then the default value, if none then null.\r\n     */\r\n    asSize(value, defaultValue) {\r\n        if (typeof value == \"function\") {\r\n            value = value();\r\n        }\r\n        if (isString(value)) {\r\n            return value;\r\n        }\r\n        else if (isNumber(value)) {\r\n            return value + \"px\";\r\n        }\r\n        else {\r\n            return defaultValue || null;\r\n        }\r\n    },\r\n    /**\r\n     * Convert a value into a DOM Element.\r\n     *\r\n     * @param value - Value to be converted into DOM Element, a function will be executed as `(() => unknown)`.\r\n     * @param defaultValue - If the value or the return value of the function == null then this will be returned.\r\n     *\r\n     * @returns The DOM Element, if none then the default value, if none then null.\r\n     */\r\n    asElement(value, defaultValue) {\r\n        if (typeof value == \"function\") {\r\n            value = value();\r\n        }\r\n        return value || defaultValue || null;\r\n    }\r\n};\r\n/**\r\n * Convert hex color string into RGB color object.\r\n *\r\n * @remarks\r\n * {@link http://stackoverflow.com/questions/5623838/rgb-to-hex-and-hex-to-rgb}\r\n *\r\n * @param hex - Hex color string (3 or 6 digits, with or without #).\r\n *\r\n * @returns RGB color object.\r\n */\r\nfunction hexToRGB(hex) {\r\n    let result;\r\n    switch (hex.length) {\r\n        case 3:\r\n        case 4:\r\n            result = shortHexRE.exec(hex);\r\n            return result\r\n                ? {\r\n                    r: parseInt(result[1] + result[1], 16),\r\n                    g: parseInt(result[2] + result[2], 16),\r\n                    b: parseInt(result[3] + result[3], 16)\r\n                }\r\n                : null;\r\n        case 6:\r\n        case 7:\r\n            result = fullHexRE.exec(hex);\r\n            return result\r\n                ? {\r\n                    r: parseInt(result[1], 16),\r\n                    g: parseInt(result[2], 16),\r\n                    b: parseInt(result[3], 16)\r\n                }\r\n                : null;\r\n        default:\r\n            return null;\r\n    }\r\n}\r\n/**\r\n * This function takes string color in hex or RGB format and adds the opacity, RGBA is passed through unchanged.\r\n *\r\n * @param color - The color string (hex, RGB, RGBA).\r\n * @param opacity - The new opacity.\r\n *\r\n * @returns RGBA string, for example 'rgba(255, 0, 127, 0.3)'.\r\n */\r\nfunction overrideOpacity(color, opacity) {\r\n    if (color.indexOf(\"rgba\") !== -1) {\r\n        return color;\r\n    }\r\n    else if (color.indexOf(\"rgb\") !== -1) {\r\n        const rgb = color\r\n            .substr(color.indexOf(\"(\") + 1)\r\n            .replace(\")\", \"\")\r\n            .split(\",\");\r\n        return \"rgba(\" + rgb[0] + \",\" + rgb[1] + \",\" + rgb[2] + \",\" + opacity + \")\";\r\n    }\r\n    else {\r\n        const rgb = hexToRGB(color);\r\n        if (rgb == null) {\r\n            return color;\r\n        }\r\n        else {\r\n            return \"rgba(\" + rgb.r + \",\" + rgb.g + \",\" + rgb.b + \",\" + opacity + \")\";\r\n        }\r\n    }\r\n}\r\n/**\r\n * Convert RGB \\<0, 255\\> into hex color string.\r\n *\r\n * @param red - Red channel.\r\n * @param green - Green channel.\r\n * @param blue - Blue channel.\r\n *\r\n * @returns Hex color string (for example: '#0acdc0').\r\n */\r\nfunction RGBToHex(red, green, blue) {\r\n    return (\"#\" + ((1 << 24) + (red << 16) + (green << 8) + blue).toString(16).slice(1));\r\n}\r\n/**\r\n * Parse a color property into an object with border, background, and highlight colors.\r\n *\r\n * @param inputColor - Shorthand color string or input color object.\r\n * @param defaultColor - Full color object to fill in missing values in inputColor.\r\n *\r\n * @returns Color object.\r\n */\r\nfunction parseColor(inputColor, defaultColor) {\r\n    if (isString(inputColor)) {\r\n        let colorStr = inputColor;\r\n        if (isValidRGB(colorStr)) {\r\n            const rgb = colorStr\r\n                .substr(4)\r\n                .substr(0, colorStr.length - 5)\r\n                .split(\",\")\r\n                .map(function (value) {\r\n                return parseInt(value);\r\n            });\r\n            colorStr = RGBToHex(rgb[0], rgb[1], rgb[2]);\r\n        }\r\n        if (isValidHex(colorStr) === true) {\r\n            const hsv = hexToHSV(colorStr);\r\n            const lighterColorHSV = {\r\n                h: hsv.h,\r\n                s: hsv.s * 0.8,\r\n                v: Math.min(1, hsv.v * 1.02)\r\n            };\r\n            const darkerColorHSV = {\r\n                h: hsv.h,\r\n                s: Math.min(1, hsv.s * 1.25),\r\n                v: hsv.v * 0.8\r\n            };\r\n            const darkerColorHex = HSVToHex(darkerColorHSV.h, darkerColorHSV.s, darkerColorHSV.v);\r\n            const lighterColorHex = HSVToHex(lighterColorHSV.h, lighterColorHSV.s, lighterColorHSV.v);\r\n            return {\r\n                background: colorStr,\r\n                border: darkerColorHex,\r\n                highlight: {\r\n                    background: lighterColorHex,\r\n                    border: darkerColorHex\r\n                },\r\n                hover: {\r\n                    background: lighterColorHex,\r\n                    border: darkerColorHex\r\n                }\r\n            };\r\n        }\r\n        else {\r\n            return {\r\n                background: colorStr,\r\n                border: colorStr,\r\n                highlight: {\r\n                    background: colorStr,\r\n                    border: colorStr\r\n                },\r\n                hover: {\r\n                    background: colorStr,\r\n                    border: colorStr\r\n                }\r\n            };\r\n        }\r\n    }\r\n    else {\r\n        if (defaultColor) {\r\n            const color = {\r\n                background: inputColor.background || defaultColor.background,\r\n                border: inputColor.border || defaultColor.border,\r\n                highlight: isString(inputColor.highlight)\r\n                    ? {\r\n                        border: inputColor.highlight,\r\n                        background: inputColor.highlight\r\n                    }\r\n                    : {\r\n                        background: (inputColor.highlight && inputColor.highlight.background) ||\r\n                            defaultColor.highlight.background,\r\n                        border: (inputColor.highlight && inputColor.highlight.border) ||\r\n                            defaultColor.highlight.border\r\n                    },\r\n                hover: isString(inputColor.hover)\r\n                    ? {\r\n                        border: inputColor.hover,\r\n                        background: inputColor.hover\r\n                    }\r\n                    : {\r\n                        border: (inputColor.hover && inputColor.hover.border) ||\r\n                            defaultColor.hover.border,\r\n                        background: (inputColor.hover && inputColor.hover.background) ||\r\n                            defaultColor.hover.background\r\n                    }\r\n            };\r\n            return color;\r\n        }\r\n        else {\r\n            const color = {\r\n                background: inputColor.background || undefined,\r\n                border: inputColor.border || undefined,\r\n                highlight: isString(inputColor.highlight)\r\n                    ? {\r\n                        border: inputColor.highlight,\r\n                        background: inputColor.highlight\r\n                    }\r\n                    : {\r\n                        background: (inputColor.highlight && inputColor.highlight.background) ||\r\n                            undefined,\r\n                        border: (inputColor.highlight && inputColor.highlight.border) ||\r\n                            undefined\r\n                    },\r\n                hover: isString(inputColor.hover)\r\n                    ? {\r\n                        border: inputColor.hover,\r\n                        background: inputColor.hover\r\n                    }\r\n                    : {\r\n                        border: (inputColor.hover && inputColor.hover.border) || undefined,\r\n                        background: (inputColor.hover && inputColor.hover.background) || undefined\r\n                    }\r\n            };\r\n            return color;\r\n        }\r\n    }\r\n}\r\n/**\r\n * Convert RGB \\<0, 255\\> into HSV object.\r\n *\r\n * @remarks\r\n * {@link http://www.javascripter.net/faq/rgb2hsv.htm}\r\n *\r\n * @param red - Red channel.\r\n * @param green - Green channel.\r\n * @param blue - Blue channel.\r\n *\r\n * @returns HSV color object.\r\n */\r\nfunction RGBToHSV(red, green, blue) {\r\n    red = red / 255;\r\n    green = green / 255;\r\n    blue = blue / 255;\r\n    const minRGB = Math.min(red, Math.min(green, blue));\r\n    const maxRGB = Math.max(red, Math.max(green, blue));\r\n    // Black-gray-white\r\n    if (minRGB === maxRGB) {\r\n        return { h: 0, s: 0, v: minRGB };\r\n    }\r\n    // Colors other than black-gray-white:\r\n    const d = red === minRGB ? green - blue : blue === minRGB ? red - green : blue - red;\r\n    const h = red === minRGB ? 3 : blue === minRGB ? 1 : 5;\r\n    const hue = (60 * (h - d / (maxRGB - minRGB))) / 360;\r\n    const saturation = (maxRGB - minRGB) / maxRGB;\r\n    const value = maxRGB;\r\n    return { h: hue, s: saturation, v: value };\r\n}\r\nconst cssUtil = {\r\n    // split a string with css styles into an object with key/values\r\n    split(cssText) {\r\n        const styles = {};\r\n        cssText.split(\";\").forEach((style) => {\r\n            if (style.trim() != \"\") {\r\n                const parts = style.split(\":\");\r\n                const key = parts[0].trim();\r\n                const value = parts[1].trim();\r\n                styles[key] = value;\r\n            }\r\n        });\r\n        return styles;\r\n    },\r\n    // build a css text string from an object with key/values\r\n    join(styles) {\r\n        return Object.keys(styles)\r\n            .map(function (key) {\r\n            return key + \": \" + styles[key];\r\n        })\r\n            .join(\"; \");\r\n    }\r\n};\r\n/**\r\n * Append a string with css styles to an element.\r\n *\r\n * @param element - The element that will receive new styles.\r\n * @param cssText - The styles to be appended.\r\n */\r\nfunction addCssText(element, cssText) {\r\n    const currentStyles = cssUtil.split(element.style.cssText);\r\n    const newStyles = cssUtil.split(cssText);\r\n    const styles = {\r\n        ...currentStyles,\r\n        ...newStyles\r\n    };\r\n    element.style.cssText = cssUtil.join(styles);\r\n}\r\n/**\r\n * Remove a string with css styles from an element.\r\n *\r\n * @param element - The element from which styles should be removed.\r\n * @param cssText - The styles to be removed.\r\n */\r\nfunction removeCssText(element, cssText) {\r\n    const styles = cssUtil.split(element.style.cssText);\r\n    const removeStyles = cssUtil.split(cssText);\r\n    for (const key in removeStyles) {\r\n        if (Object.prototype.hasOwnProperty.call(removeStyles, key)) {\r\n            delete styles[key];\r\n        }\r\n    }\r\n    element.style.cssText = cssUtil.join(styles);\r\n}\r\n/**\r\n * Convert HSV \\<0, 1\\> into RGB color object.\r\n *\r\n * @remarks\r\n * {@link https://gist.github.com/mjijackson/5311256}\r\n *\r\n * @param h - Hue.\r\n * @param s - Saturation.\r\n * @param v - Value.\r\n *\r\n * @returns RGB color object.\r\n */\r\nfunction HSVToRGB(h, s, v) {\r\n    let r;\r\n    let g;\r\n    let b;\r\n    const i = Math.floor(h * 6);\r\n    const f = h * 6 - i;\r\n    const p = v * (1 - s);\r\n    const q = v * (1 - f * s);\r\n    const t = v * (1 - (1 - f) * s);\r\n    switch (i % 6) {\r\n        case 0:\r\n            (r = v), (g = t), (b = p);\r\n            break;\r\n        case 1:\r\n            (r = q), (g = v), (b = p);\r\n            break;\r\n        case 2:\r\n            (r = p), (g = v), (b = t);\r\n            break;\r\n        case 3:\r\n            (r = p), (g = q), (b = v);\r\n            break;\r\n        case 4:\r\n            (r = t), (g = p), (b = v);\r\n            break;\r\n        case 5:\r\n            (r = v), (g = p), (b = q);\r\n            break;\r\n    }\r\n    return {\r\n        r: Math.floor(r * 255),\r\n        g: Math.floor(g * 255),\r\n        b: Math.floor(b * 255)\r\n    };\r\n}\r\n/**\r\n * Convert HSV \\<0, 1\\> into hex color string.\r\n *\r\n * @param h - Hue.\r\n * @param s - Saturation.\r\n * @param v - Value.\r\n *\r\n * @returns Hex color string.\r\n */\r\nfunction HSVToHex(h, s, v) {\r\n    const rgb = HSVToRGB(h, s, v);\r\n    return RGBToHex(rgb.r, rgb.g, rgb.b);\r\n}\r\n/**\r\n * Convert hex color string into HSV \\<0, 1\\>.\r\n *\r\n * @param hex - Hex color string.\r\n *\r\n * @returns HSV color object.\r\n */\r\nfunction hexToHSV(hex) {\r\n    const rgb = hexToRGB(hex);\r\n    if (!rgb) {\r\n        throw new TypeError(`'${hex}' is not a valid color.`);\r\n    }\r\n    return RGBToHSV(rgb.r, rgb.g, rgb.b);\r\n}\r\n/**\r\n * Validate hex color string.\r\n *\r\n * @param hex - Unknown string that may contain a color.\r\n *\r\n * @returns True if the string is valid, false otherwise.\r\n */\r\nfunction isValidHex(hex) {\r\n    const isOk = /(^#[0-9A-F]{6}$)|(^#[0-9A-F]{3}$)/i.test(hex);\r\n    return isOk;\r\n}\r\n/**\r\n * Validate RGB color string.\r\n *\r\n * @param rgb - Unknown string that may contain a color.\r\n *\r\n * @returns True if the string is valid, false otherwise.\r\n */\r\nfunction isValidRGB(rgb) {\r\n    return rgbRE.test(rgb);\r\n}\r\n/**\r\n * Validate RGBA color string.\r\n *\r\n * @param rgba - Unknown string that may contain a color.\r\n *\r\n * @returns True if the string is valid, false otherwise.\r\n */\r\nfunction isValidRGBA(rgba) {\r\n    return rgbaRE.test(rgba);\r\n}\r\n/**\r\n * This recursively redirects the prototype of JSON objects to the referenceObject.\r\n * This is used for default options.\r\n *\r\n * @param fields - Names of properties to be bridged.\r\n * @param referenceObject - The original object.\r\n *\r\n * @returns A new object inheriting from the referenceObject.\r\n */\r\nfunction selectiveBridgeObject(fields, referenceObject) {\r\n    if (referenceObject !== null && typeof referenceObject === \"object\") {\r\n        // !!! typeof null === 'object'\r\n        const objectTo = Object.create(referenceObject);\r\n        for (let i = 0; i < fields.length; i++) {\r\n            if (Object.prototype.hasOwnProperty.call(referenceObject, fields[i])) {\r\n                if (typeof referenceObject[fields[i]] == \"object\") {\r\n                    objectTo[fields[i]] = bridgeObject(referenceObject[fields[i]]);\r\n                }\r\n            }\r\n        }\r\n        return objectTo;\r\n    }\r\n    else {\r\n        return null;\r\n    }\r\n}\r\n/**\r\n * This recursively redirects the prototype of JSON objects to the referenceObject.\r\n * This is used for default options.\r\n *\r\n * @param referenceObject - The original object.\r\n *\r\n * @returns The Element if the referenceObject is an Element, or a new object inheriting from the referenceObject.\r\n */\r\nfunction bridgeObject(referenceObject) {\r\n    if (referenceObject === null || typeof referenceObject !== \"object\") {\r\n        return null;\r\n    }\r\n    if (referenceObject instanceof Element) {\r\n        // Avoid bridging DOM objects\r\n        return referenceObject;\r\n    }\r\n    const objectTo = Object.create(referenceObject);\r\n    for (const i in referenceObject) {\r\n        if (Object.prototype.hasOwnProperty.call(referenceObject, i)) {\r\n            if (typeof referenceObject[i] == \"object\") {\r\n                objectTo[i] = bridgeObject(referenceObject[i]);\r\n            }\r\n        }\r\n    }\r\n    return objectTo;\r\n}\r\n/**\r\n * This method provides a stable sort implementation, very fast for presorted data.\r\n *\r\n * @param a - The array to be sorted (in-place).\r\n * @param compare - An order comparator.\r\n *\r\n * @returns The argument a.\r\n */\r\nfunction insertSort(a, compare) {\r\n    for (let i = 0; i < a.length; i++) {\r\n        const k = a[i];\r\n        let j;\r\n        for (j = i; j > 0 && compare(k, a[j - 1]) < 0; j--) {\r\n            a[j] = a[j - 1];\r\n        }\r\n        a[j] = k;\r\n    }\r\n    return a;\r\n}\r\n/**\r\n * This is used to set the options of subobjects in the options object.\r\n *\r\n * A requirement of these subobjects is that they have an 'enabled' element\r\n * which is optional for the user but mandatory for the program.\r\n *\r\n * The added value here of the merge is that option 'enabled' is set as required.\r\n *\r\n * @param mergeTarget - Either this.options or the options used for the groups.\r\n * @param options - Options.\r\n * @param option - Option key in the options argument.\r\n * @param globalOptions - Global options, passed in to determine value of option 'enabled'.\r\n */\r\nfunction mergeOptions(mergeTarget, options, option, globalOptions = {}) {\r\n    // Local helpers\r\n    const isPresent = function (obj) {\r\n        return obj !== null && obj !== undefined;\r\n    };\r\n    const isObject = function (obj) {\r\n        return obj !== null && typeof obj === \"object\";\r\n    };\r\n    // https://stackoverflow.com/a/34491287/1223531\r\n    const isEmpty = function (obj) {\r\n        for (const x in obj) {\r\n            if (Object.prototype.hasOwnProperty.call(obj, x)) {\r\n                return false;\r\n            }\r\n        }\r\n        return true;\r\n    };\r\n    // Guards\r\n    if (!isObject(mergeTarget)) {\r\n        throw new Error(\"Parameter mergeTarget must be an object\");\r\n    }\r\n    if (!isObject(options)) {\r\n        throw new Error(\"Parameter options must be an object\");\r\n    }\r\n    if (!isPresent(option)) {\r\n        throw new Error(\"Parameter option must have a value\");\r\n    }\r\n    if (!isObject(globalOptions)) {\r\n        throw new Error(\"Parameter globalOptions must be an object\");\r\n    }\r\n    //\r\n    // Actual merge routine, separated from main logic\r\n    // Only a single level of options is merged. Deeper levels are ref'd. This may actually be an issue.\r\n    //\r\n    const doMerge = function (target, options, option) {\r\n        if (!isObject(target[option])) {\r\n            target[option] = {};\r\n        }\r\n        const src = options[option];\r\n        const dst = target[option];\r\n        for (const prop in src) {\r\n            if (Object.prototype.hasOwnProperty.call(src, prop)) {\r\n                dst[prop] = src[prop];\r\n            }\r\n        }\r\n    };\r\n    // Local initialization\r\n    const srcOption = options[option];\r\n    const globalPassed = isObject(globalOptions) && !isEmpty(globalOptions);\r\n    const globalOption = globalPassed ? globalOptions[option] : undefined;\r\n    const globalEnabled = globalOption ? globalOption.enabled : undefined;\r\n    /////////////////////////////////////////\r\n    // Main routine\r\n    /////////////////////////////////////////\r\n    if (srcOption === undefined) {\r\n        return; // Nothing to do\r\n    }\r\n    if (typeof srcOption === \"boolean\") {\r\n        if (!isObject(mergeTarget[option])) {\r\n            mergeTarget[option] = {};\r\n        }\r\n        mergeTarget[option].enabled = srcOption;\r\n        return;\r\n    }\r\n    if (srcOption === null && !isObject(mergeTarget[option])) {\r\n        // If possible, explicit copy from globals\r\n        if (isPresent(globalOption)) {\r\n            mergeTarget[option] = Object.create(globalOption);\r\n        }\r\n        else {\r\n            return; // Nothing to do\r\n        }\r\n    }\r\n    if (!isObject(srcOption)) {\r\n        return;\r\n    }\r\n    //\r\n    // Ensure that 'enabled' is properly set. It is required internally\r\n    // Note that the value from options will always overwrite the existing value\r\n    //\r\n    let enabled = true; // default value\r\n    if (srcOption.enabled !== undefined) {\r\n        enabled = srcOption.enabled;\r\n    }\r\n    else {\r\n        // Take from globals, if present\r\n        if (globalEnabled !== undefined) {\r\n            enabled = globalOption.enabled;\r\n        }\r\n    }\r\n    doMerge(mergeTarget, options, option);\r\n    mergeTarget[option].enabled = enabled;\r\n}\r\n/**\r\n * This function does a binary search for a visible item in a sorted list. If we find a visible item, the code that uses\r\n * this function will then iterate in both directions over this sorted list to find all visible items.\r\n *\r\n * @param orderedItems - Items ordered by start.\r\n * @param comparator - -1 is lower, 0 is equal, 1 is higher.\r\n * @param field - Property name on an item (That is item[field]).\r\n * @param field2 - Second property name on an item (That is item[field][field2]).\r\n *\r\n * @returns Index of the found item or -1 if nothing was found.\r\n */\r\nfunction binarySearchCustom(orderedItems, comparator, field, field2) {\r\n    const maxIterations = 10000;\r\n    let iteration = 0;\r\n    let low = 0;\r\n    let high = orderedItems.length - 1;\r\n    while (low <= high && iteration < maxIterations) {\r\n        const middle = Math.floor((low + high) / 2);\r\n        const item = orderedItems[middle];\r\n        const value = field2 === undefined ? item[field] : item[field][field2];\r\n        const searchResult = comparator(value);\r\n        if (searchResult == 0) {\r\n            // jihaa, found a visible item!\r\n            return middle;\r\n        }\r\n        else if (searchResult == -1) {\r\n            // it is too small --> increase low\r\n            low = middle + 1;\r\n        }\r\n        else {\r\n            // it is too big --> decrease high\r\n            high = middle - 1;\r\n        }\r\n        iteration++;\r\n    }\r\n    return -1;\r\n}\r\n/**\r\n * This function does a binary search for a specific value in a sorted array.\r\n * If it does not exist but is in between of two values, we return either the\r\n * one before or the one after, depending on user input If it is found, we\r\n * return the index, else -1.\r\n *\r\n * @param orderedItems - Sorted array.\r\n * @param target - The searched value.\r\n * @param field - Name of the property in items to be searched.\r\n * @param sidePreference - If the target is between two values, should the index of the before or the after be returned?\r\n * @param comparator - An optional comparator, returning -1, 0, 1 for \\<, ===, \\>.\r\n *\r\n * @returns The index of found value or -1 if nothing was found.\r\n */\r\nfunction binarySearchValue(orderedItems, target, field, sidePreference, comparator) {\r\n    const maxIterations = 10000;\r\n    let iteration = 0;\r\n    let low = 0;\r\n    let high = orderedItems.length - 1;\r\n    let prevValue;\r\n    let value;\r\n    let nextValue;\r\n    let middle;\r\n    comparator =\r\n        comparator != undefined\r\n            ? comparator\r\n            : function (a, b) {\r\n                return a == b ? 0 : a < b ? -1 : 1;\r\n            };\r\n    while (low <= high && iteration < maxIterations) {\r\n        // get a new guess\r\n        middle = Math.floor(0.5 * (high + low));\r\n        prevValue = orderedItems[Math.max(0, middle - 1)][field];\r\n        value = orderedItems[middle][field];\r\n        nextValue =\r\n            orderedItems[Math.min(orderedItems.length - 1, middle + 1)][field];\r\n        if (comparator(value, target) == 0) {\r\n            // we found the target\r\n            return middle;\r\n        }\r\n        else if (comparator(prevValue, target) < 0 &&\r\n            comparator(value, target) > 0) {\r\n            // target is in between of the previous and the current\r\n            return sidePreference == \"before\" ? Math.max(0, middle - 1) : middle;\r\n        }\r\n        else if (comparator(value, target) < 0 &&\r\n            comparator(nextValue, target) > 0) {\r\n            // target is in between of the current and the next\r\n            return sidePreference == \"before\"\r\n                ? middle\r\n                : Math.min(orderedItems.length - 1, middle + 1);\r\n        }\r\n        else {\r\n            // didnt find the target, we need to change our boundaries.\r\n            if (comparator(value, target) < 0) {\r\n                // it is too small --> increase low\r\n                low = middle + 1;\r\n            }\r\n            else {\r\n                // it is too big --> decrease high\r\n                high = middle - 1;\r\n            }\r\n        }\r\n        iteration++;\r\n    }\r\n    // didnt find anything. Return -1.\r\n    return -1;\r\n}\r\n/*\r\n * Easing Functions.\r\n * Only considering the t value for the range [0, 1] => [0, 1].\r\n *\r\n * Inspiration: from http://gizma.com/easing/\r\n * https://gist.github.com/gre/1650294\r\n */\r\nconst easingFunctions = {\r\n    /**\r\n     * Provides no easing and no acceleration.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    linear(t) {\r\n        return t;\r\n    },\r\n    /**\r\n     * Accelerate from zero velocity.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeInQuad(t) {\r\n        return t * t;\r\n    },\r\n    /**\r\n     * Decelerate to zero velocity.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeOutQuad(t) {\r\n        return t * (2 - t);\r\n    },\r\n    /**\r\n     * Accelerate until halfway, then decelerate.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeInOutQuad(t) {\r\n        return t < 0.5 ? 2 * t * t : -1 + (4 - 2 * t) * t;\r\n    },\r\n    /**\r\n     * Accelerate from zero velocity.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeInCubic(t) {\r\n        return t * t * t;\r\n    },\r\n    /**\r\n     * Decelerate to zero velocity.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeOutCubic(t) {\r\n        return --t * t * t + 1;\r\n    },\r\n    /**\r\n     * Accelerate until halfway, then decelerate.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeInOutCubic(t) {\r\n        return t < 0.5 ? 4 * t * t * t : (t - 1) * (2 * t - 2) * (2 * t - 2) + 1;\r\n    },\r\n    /**\r\n     * Accelerate from zero velocity.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeInQuart(t) {\r\n        return t * t * t * t;\r\n    },\r\n    /**\r\n     * Decelerate to zero velocity.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeOutQuart(t) {\r\n        return 1 - --t * t * t * t;\r\n    },\r\n    /**\r\n     * Accelerate until halfway, then decelerate.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeInOutQuart(t) {\r\n        return t < 0.5 ? 8 * t * t * t * t : 1 - 8 * --t * t * t * t;\r\n    },\r\n    /**\r\n     * Accelerate from zero velocity.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeInQuint(t) {\r\n        return t * t * t * t * t;\r\n    },\r\n    /**\r\n     * Decelerate to zero velocity.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeOutQuint(t) {\r\n        return 1 + --t * t * t * t * t;\r\n    },\r\n    /**\r\n     * Accelerate until halfway, then decelerate.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeInOutQuint(t) {\r\n        return t < 0.5 ? 16 * t * t * t * t * t : 1 + 16 * --t * t * t * t * t;\r\n    }\r\n};\r\n/**\r\n * Experimentaly compute the width of the scrollbar for this browser.\r\n *\r\n * @returns The width in pixels.\r\n */\r\nfunction getScrollBarWidth() {\r\n    const inner = document.createElement(\"p\");\r\n    inner.style.width = \"100%\";\r\n    inner.style.height = \"200px\";\r\n    const outer = document.createElement(\"div\");\r\n    outer.style.position = \"absolute\";\r\n    outer.style.top = \"0px\";\r\n    outer.style.left = \"0px\";\r\n    outer.style.visibility = \"hidden\";\r\n    outer.style.width = \"200px\";\r\n    outer.style.height = \"150px\";\r\n    outer.style.overflow = \"hidden\";\r\n    outer.appendChild(inner);\r\n    document.body.appendChild(outer);\r\n    const w1 = inner.offsetWidth;\r\n    outer.style.overflow = \"scroll\";\r\n    let w2 = inner.offsetWidth;\r\n    if (w1 == w2) {\r\n        w2 = outer.clientWidth;\r\n    }\r\n    document.body.removeChild(outer);\r\n    return w1 - w2;\r\n}\r\n// @TODO: This doesn't work properly.\r\n// It works only for single property objects,\r\n// otherwise it combines all of the types in a union.\r\n// export function topMost<K1 extends string, V1> (\r\n//   pile: Record<K1, undefined | V1>[],\r\n//   accessors: K1 | [K1]\r\n// ): undefined | V1\r\n// export function topMost<K1 extends string, K2 extends string, V1, V2> (\r\n//   pile: Record<K1, undefined | V1 | Record<K2, undefined | V2>>[],\r\n//   accessors: [K1, K2]\r\n// ): undefined | V1 | V2\r\n// export function topMost<K1 extends string, K2 extends string, K3 extends string, V1, V2, V3> (\r\n//   pile: Record<K1, undefined | V1 | Record<K2, undefined | V2 | Record<K3, undefined | V3>>>[],\r\n//   accessors: [K1, K2, K3]\r\n// ): undefined | V1 | V2 | V3\r\n/**\r\n * Get the top most property value from a pile of objects.\r\n *\r\n * @param pile - Array of objects, no required format.\r\n * @param accessors - Array of property names.\r\n * For example `object['foo']['bar']` → `['foo', 'bar']`.\r\n *\r\n * @returns Value of the property with given accessors path from the first pile item where it's not undefined.\r\n */\r\nfunction topMost(pile, accessors) {\r\n    let candidate;\r\n    if (!Array.isArray(accessors)) {\r\n        accessors = [accessors];\r\n    }\r\n    for (const member of pile) {\r\n        if (member) {\r\n            candidate = member[accessors[0]];\r\n            for (let i = 1; i < accessors.length; i++) {\r\n                if (candidate) {\r\n                    candidate = candidate[accessors[i]];\r\n                }\r\n            }\r\n            if (typeof candidate !== \"undefined\") {\r\n                break;\r\n            }\r\n        }\r\n    }\r\n    return candidate;\r\n}\n\nexport { Alea, HSVToHex, HSVToRGB, RGBToHSV, RGBToHex, addClassName, addCssText, addEventListener, binarySearchCustom, binarySearchValue, bridgeObject, copyAndExtendArray, copyArray, deepExtend, easingFunctions, equalArray, extend, fillIfDefined, forEach, getAbsoluteLeft, getAbsoluteRight, getAbsoluteTop, getScrollBarWidth, getTarget, getType, hasParent, hexToHSV, hexToRGB, insertSort, isDate, isNumber, isObject, isString, isValidHex, isValidRGB, isValidRGBA, mergeOptions, option, overrideOpacity, parseColor, preventDefault, recursiveDOMDelete, removeClassName, removeCssText, removeEventListener, selectiveBridgeObject, selectiveDeepExtend, selectiveExtend, selectiveNotDeepExtend, throttle, toArray, topMost, updateProperty };\n//# sourceMappingURL=vis-util.js.map\n","// DOM utility methods\n\n/**\n * this prepares the JSON container for allocating SVG elements\n * @param {Object} JSONcontainer\n * @private\n */\nexport function prepareElements(JSONcontainer) {\n  // cleanup the redundant svgElements;\n  for (var elementType in JSONcontainer) {\n    if (JSONcontainer.hasOwnProperty(elementType)) {\n      JSONcontainer[elementType].redundant = JSONcontainer[elementType].used;\n      JSONcontainer[elementType].used = [];\n    }\n  }\n}\n\n/**\n * this cleans up all the unused SVG elements. By asking for the parentNode, we only need to supply the JSON container from\n * which to remove the redundant elements.\n *\n * @param {Object} JSONcontainer\n * @private\n */\nexport function cleanupElements(JSONcontainer) {\n  // cleanup the redundant svgElements;\n  for (var elementType in JSONcontainer) {\n    if (JSONcontainer.hasOwnProperty(elementType)) {\n      if (JSONcontainer[elementType].redundant) {\n        for (var i = 0; i < JSONcontainer[elementType].redundant.length; i++) {\n          JSONcontainer[elementType].redundant[i].parentNode.removeChild(JSONcontainer[elementType].redundant[i]);\n        }\n        JSONcontainer[elementType].redundant = [];\n      }\n    }\n  }\n}\n\n/**\n * Ensures that all elements are removed first up so they can be recreated cleanly\n * @param {Object} JSONcontainer\n */\nexport function resetElements(JSONcontainer) {\n  prepareElements(JSONcontainer);\n  cleanupElements(JSONcontainer);\n  prepareElements(JSONcontainer);\n}\n\n/**\n * Allocate or generate an SVG element if needed. Store a reference to it in the JSON container and draw it in the svgContainer\n * the JSON container and the SVG container have to be supplied so other svg containers (like the legend) can use this.\n *\n * @param {string} elementType\n * @param {Object} JSONcontainer\n * @param {Object} svgContainer\n * @returns {Element}\n * @private\n */\nexport function getSVGElement(elementType, JSONcontainer, svgContainer) {\n  var element;\n  // allocate SVG element, if it doesnt yet exist, create one.\n  if (JSONcontainer.hasOwnProperty(elementType)) { // this element has been created before\n    // check if there is an redundant element\n    if (JSONcontainer[elementType].redundant.length > 0) {\n      element = JSONcontainer[elementType].redundant[0];\n      JSONcontainer[elementType].redundant.shift();\n    }\n    else {\n      // create a new element and add it to the SVG\n      element = document.createElementNS('http://www.w3.org/2000/svg', elementType);\n      svgContainer.appendChild(element);\n    }\n  }\n  else {\n    // create a new element and add it to the SVG, also create a new object in the svgElements to keep track of it.\n    element = document.createElementNS('http://www.w3.org/2000/svg', elementType);\n    JSONcontainer[elementType] = {used: [], redundant: []};\n    svgContainer.appendChild(element);\n  }\n  JSONcontainer[elementType].used.push(element);\n  return element;\n}\n\n\n/**\n * Allocate or generate an SVG element if needed. Store a reference to it in the JSON container and draw it in the svgContainer\n * the JSON container and the SVG container have to be supplied so other svg containers (like the legend) can use this.\n *\n * @param {string} elementType\n * @param {Object} JSONcontainer\n * @param {Element} DOMContainer\n * @param {Element} insertBefore\n * @returns {*}\n */\nexport function getDOMElement(elementType, JSONcontainer, DOMContainer, insertBefore) {\n  var element;\n  // allocate DOM element, if it doesnt yet exist, create one.\n  if (JSONcontainer.hasOwnProperty(elementType)) { // this element has been created before\n    // check if there is an redundant element\n    if (JSONcontainer[elementType].redundant.length > 0) {\n      element = JSONcontainer[elementType].redundant[0];\n      JSONcontainer[elementType].redundant.shift();\n    }\n    else {\n      // create a new element and add it to the SVG\n      element = document.createElement(elementType);\n      if (insertBefore !== undefined) {\n        DOMContainer.insertBefore(element, insertBefore);\n      }\n      else {\n        DOMContainer.appendChild(element);\n      }\n    }\n  }\n  else {\n    // create a new element and add it to the SVG, also create a new object in the svgElements to keep track of it.\n    element = document.createElement(elementType);\n    JSONcontainer[elementType] = {used: [], redundant: []};\n    if (insertBefore !== undefined) {\n      DOMContainer.insertBefore(element, insertBefore);\n    }\n    else {\n      DOMContainer.appendChild(element);\n    }\n  }\n  JSONcontainer[elementType].used.push(element);\n  return element;\n}\n\n\n\n\n/**\n * Draw a point object. This is a separate function because it can also be called by the legend.\n * The reason the JSONcontainer and the target SVG svgContainer have to be supplied is so the legend can use these functions\n * as well.\n *\n * @param {number} x\n * @param {number} y\n * @param {Object} groupTemplate: A template containing the necessary information to draw the datapoint e.g., {style: 'circle', size: 5, className: 'className' }\n * @param {Object} JSONcontainer\n * @param {Object} svgContainer\n * @param {Object} labelObj\n * @returns {vis.PointItem}\n */\nexport function drawPoint(x, y, groupTemplate, JSONcontainer, svgContainer, labelObj) {\n  var point;\n  if (groupTemplate.style == 'circle') {\n    point = getSVGElement('circle', JSONcontainer, svgContainer);\n    point.setAttributeNS(null, \"cx\", x);\n    point.setAttributeNS(null, \"cy\", y);\n    point.setAttributeNS(null, \"r\", 0.5 * groupTemplate.size);\n  }\n  else {\n    point = getSVGElement('rect', JSONcontainer, svgContainer);\n    point.setAttributeNS(null, \"x\", x - 0.5 * groupTemplate.size);\n    point.setAttributeNS(null, \"y\", y - 0.5 * groupTemplate.size);\n    point.setAttributeNS(null, \"width\", groupTemplate.size);\n    point.setAttributeNS(null, \"height\", groupTemplate.size);\n  }\n\n  if (groupTemplate.styles !== undefined) {\n    point.setAttributeNS(null, \"style\", groupTemplate.styles);\n  }\n  point.setAttributeNS(null, \"class\", groupTemplate.className + \" vis-point\");\n  //handle label\n\n\n  if (labelObj) {\n    var label = getSVGElement('text', JSONcontainer, svgContainer);\n    if (labelObj.xOffset) {\n      x = x + labelObj.xOffset;\n    }\n\n    if (labelObj.yOffset) {\n      y = y + labelObj.yOffset;\n    }\n    if (labelObj.content) {\n      label.textContent = labelObj.content;\n    }\n\n    if (labelObj.className) {\n      label.setAttributeNS(null, \"class\", labelObj.className  + \" vis-label\");\n    }\n    label.setAttributeNS(null, \"x\", x);\n    label.setAttributeNS(null, \"y\", y);\n  }\n\n  return point;\n}\n\n/**\n * draw a bar SVG element centered on the X coordinate\n *\n * @param {number} x\n * @param {number} y\n * @param {number} width\n * @param {number} height\n * @param {string} className\n * @param {Object} JSONcontainer\n * @param {Object} svgContainer\n * @param {string} style\n */\nexport function drawBar (x, y, width, height, className, JSONcontainer, svgContainer, style) {\n  if (height != 0) {\n    if (height < 0) {\n      height *= -1;\n      y -= height;\n    }\n    var rect = getSVGElement('rect',JSONcontainer, svgContainer);\n    rect.setAttributeNS(null, \"x\", x - 0.5 * width);\n    rect.setAttributeNS(null, \"y\", y);\n    rect.setAttributeNS(null, \"width\", width);\n    rect.setAttributeNS(null, \"height\", height);\n    rect.setAttributeNS(null, \"class\", className);\n    if (style) {\n      rect.setAttributeNS(null, \"style\", style);\n    }\n  }\n}\n\n/**\n * get default language\n * @returns {string}\n */\nexport function getNavigatorLanguage() {\n  try {\n    if (!navigator) return 'en';\n    if (navigator.languages && navigator.languages.length) {\n      return navigator.languages;\n    } else {\n      return navigator.userLanguage || navigator.language || navigator.browserLanguage || 'en';\n    }\n  } \n  catch(error) {\n    return 'en';\n  }\n}\n","function _assertThisInitialized(self) {\n  if (self === void 0) {\n    throw new ReferenceError(\"this hasn't been initialised - super() hasn't been called\");\n  }\n\n  return self;\n}\n\nmodule.exports = _assertThisInitialized;","require('../../../modules/es.array.iterator');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').entries;\n","require('../../modules/web.dom-collections.iterator');\nvar entries = require('../array/virtual/entries');\nvar classof = require('../../internals/classof');\nvar ArrayPrototype = Array.prototype;\n\nvar DOMIterables = {\n  DOMTokenList: true,\n  NodeList: true\n};\n\nmodule.exports = function (it) {\n  var own = it.entries;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.entries)\n    // eslint-disable-next-line no-prototype-builtins\n    || DOMIterables.hasOwnProperty(classof(it)) ? entries : own;\n};\n","var $ = require('../internals/export');\nvar getBuiltIn = require('../internals/get-built-in');\nvar fails = require('../internals/fails');\n\nvar $stringify = getBuiltIn('JSON', 'stringify');\nvar re = /[\\uD800-\\uDFFF]/g;\nvar low = /^[\\uD800-\\uDBFF]$/;\nvar hi = /^[\\uDC00-\\uDFFF]$/;\n\nvar fix = function (match, offset, string) {\n  var prev = string.charAt(offset - 1);\n  var next = string.charAt(offset + 1);\n  if ((low.test(match) && !hi.test(next)) || (hi.test(match) && !low.test(prev))) {\n    return '\\\\u' + match.charCodeAt(0).toString(16);\n  } return match;\n};\n\nvar FORCED = fails(function () {\n  return $stringify('\\uDF06\\uD834') !== '\"\\\\udf06\\\\ud834\"'\n    || $stringify('\\uDEAD') !== '\"\\\\udead\"';\n});\n\nif ($stringify) {\n  // https://github.com/tc39/proposal-well-formed-stringify\n  $({ target: 'JSON', stat: true, forced: FORCED }, {\n    // eslint-disable-next-line no-unused-vars\n    stringify: function stringify(it, replacer, space) {\n      var result = $stringify.apply(null, arguments);\n      return typeof result == 'string' ? result.replace(re, fix) : result;\n    }\n  });\n}\n","require('../../modules/es.json.stringify');\nvar core = require('../../internals/path');\n\nif (!core.JSON) core.JSON = { stringify: JSON.stringify };\n\n// eslint-disable-next-line no-unused-vars\nmodule.exports = function stringify(it, replacer, space) {\n  return core.JSON.stringify.apply(null, arguments);\n};\n","require('../../../modules/es.array.iterator');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').values;\n","require('../../modules/web.dom-collections.iterator');\nvar values = require('../array/virtual/values');\nvar classof = require('../../internals/classof');\nvar ArrayPrototype = Array.prototype;\n\nvar DOMIterables = {\n  DOMTokenList: true,\n  NodeList: true\n};\n\nmodule.exports = function (it) {\n  var own = it.values;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.values)\n    // eslint-disable-next-line no-prototype-builtins\n    || DOMIterables.hasOwnProperty(classof(it)) ? values : own;\n};\n","require('../../../modules/es.array.iterator');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').keys;\n","require('../../modules/web.dom-collections.iterator');\nvar keys = require('../array/virtual/keys');\nvar classof = require('../../internals/classof');\nvar ArrayPrototype = Array.prototype;\n\nvar DOMIterables = {\n  DOMTokenList: true,\n  NodeList: true\n};\n\nmodule.exports = function (it) {\n  var own = it.keys;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.keys)\n    // eslint-disable-next-line no-prototype-builtins\n    || DOMIterables.hasOwnProperty(classof(it)) ? keys : own;\n};\n","'use strict';\nvar $ = require('../internals/export');\nvar $some = require('../internals/array-iteration').some;\nvar arrayMethodIsStrict = require('../internals/array-method-is-strict');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar STRICT_METHOD = arrayMethodIsStrict('some');\nvar USES_TO_LENGTH = arrayMethodUsesToLength('some');\n\n// `Array.prototype.some` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.some\n$({ target: 'Array', proto: true, forced: !STRICT_METHOD || !USES_TO_LENGTH }, {\n  some: function some(callbackfn /* , thisArg */) {\n    return $some(this, callbackfn, arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n","require('../../../modules/es.array.some');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').some;\n","var some = require('../array/virtual/some');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.some;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.some) ? some : own;\n};\n","var _typeof = require(\"../helpers/typeof\");\n\nvar assertThisInitialized = require(\"./assertThisInitialized\");\n\nfunction _possibleConstructorReturn(self, call) {\n  if (call && (_typeof(call) === \"object\" || typeof call === \"function\")) {\n    return call;\n  }\n\n  return assertThisInitialized(self);\n}\n\nmodule.exports = _possibleConstructorReturn;","var parent = require('../../es/object/get-prototype-of');\n\nmodule.exports = parent;\n","var $ = require('../internals/export');\nvar setPrototypeOf = require('../internals/object-set-prototype-of');\n\n// `Object.setPrototypeOf` method\n// https://tc39.github.io/ecma262/#sec-object.setprototypeof\n$({ target: 'Object', stat: true }, {\n  setPrototypeOf: setPrototypeOf\n});\n","require('../../modules/es.object.set-prototype-of');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Object.setPrototypeOf;\n","var _Object$getPrototypeOf = require(\"../core-js/object/get-prototype-of\");\n\nvar _Object$setPrototypeOf = require(\"../core-js/object/set-prototype-of\");\n\nfunction _getPrototypeOf(o) {\n  module.exports = _getPrototypeOf = _Object$setPrototypeOf ? _Object$getPrototypeOf : function _getPrototypeOf(o) {\n    return o.__proto__ || _Object$getPrototypeOf(o);\n  };\n  return _getPrototypeOf(o);\n}\n\nmodule.exports = _getPrototypeOf;","var parent = require('../../es/object/create');\n\nmodule.exports = parent;\n","var _Object$setPrototypeOf = require(\"../core-js/object/set-prototype-of\");\n\nfunction _setPrototypeOf(o, p) {\n  module.exports = _setPrototypeOf = _Object$setPrototypeOf || function _setPrototypeOf(o, p) {\n    o.__proto__ = p;\n    return o;\n  };\n\n  return _setPrototypeOf(o, p);\n}\n\nmodule.exports = _setPrototypeOf;","var _Object$create = require(\"../core-js/object/create\");\n\nvar setPrototypeOf = require(\"./setPrototypeOf\");\n\nfunction _inherits(subClass, superClass) {\n  if (typeof superClass !== \"function\" && superClass !== null) {\n    throw new TypeError(\"Super expression must either be null or a function\");\n  }\n\n  subClass.prototype = _Object$create(superClass && superClass.prototype, {\n    constructor: {\n      value: subClass,\n      writable: true,\n      configurable: true\n    }\n  });\n  if (superClass) setPrototypeOf(subClass, superClass);\n}\n\nmodule.exports = _inherits;","'use strict';\nvar $ = require('../internals/export');\nvar aFunction = require('../internals/a-function');\nvar toObject = require('../internals/to-object');\nvar fails = require('../internals/fails');\nvar arrayMethodIsStrict = require('../internals/array-method-is-strict');\n\nvar test = [];\nvar nativeSort = test.sort;\n\n// IE8-\nvar FAILS_ON_UNDEFINED = fails(function () {\n  test.sort(undefined);\n});\n// V8 bug\nvar FAILS_ON_NULL = fails(function () {\n  test.sort(null);\n});\n// Old WebKit\nvar STRICT_METHOD = arrayMethodIsStrict('sort');\n\nvar FORCED = FAILS_ON_UNDEFINED || !FAILS_ON_NULL || !STRICT_METHOD;\n\n// `Array.prototype.sort` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.sort\n$({ target: 'Array', proto: true, forced: FORCED }, {\n  sort: function sort(comparefn) {\n    return comparefn === undefined\n      ? nativeSort.call(toObject(this))\n      : nativeSort.call(toObject(this), aFunction(comparefn));\n  }\n});\n","require('../../../modules/es.array.sort');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').sort;\n","var sort = require('../array/virtual/sort');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.sort;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.sort) ? sort : own;\n};\n","var fails = require('../internals/fails');\n\nmodule.exports = !fails(function () {\n  return Object.isExtensible(Object.preventExtensions({}));\n});\n","var hiddenKeys = require('../internals/hidden-keys');\nvar isObject = require('../internals/is-object');\nvar has = require('../internals/has');\nvar defineProperty = require('../internals/object-define-property').f;\nvar uid = require('../internals/uid');\nvar FREEZING = require('../internals/freezing');\n\nvar METADATA = uid('meta');\nvar id = 0;\n\nvar isExtensible = Object.isExtensible || function () {\n  return true;\n};\n\nvar setMetadata = function (it) {\n  defineProperty(it, METADATA, { value: {\n    objectID: 'O' + ++id, // object ID\n    weakData: {}          // weak collections IDs\n  } });\n};\n\nvar fastKey = function (it, create) {\n  // return a primitive with prefix\n  if (!isObject(it)) return typeof it == 'symbol' ? it : (typeof it == 'string' ? 'S' : 'P') + it;\n  if (!has(it, METADATA)) {\n    // can't set metadata to uncaught frozen object\n    if (!isExtensible(it)) return 'F';\n    // not necessary to add metadata\n    if (!create) return 'E';\n    // add missing metadata\n    setMetadata(it);\n  // return object ID\n  } return it[METADATA].objectID;\n};\n\nvar getWeakData = function (it, create) {\n  if (!has(it, METADATA)) {\n    // can't set metadata to uncaught frozen object\n    if (!isExtensible(it)) return true;\n    // not necessary to add metadata\n    if (!create) return false;\n    // add missing metadata\n    setMetadata(it);\n  // return the store of weak collections IDs\n  } return it[METADATA].weakData;\n};\n\n// add metadata on freeze-family methods calling\nvar onFreeze = function (it) {\n  if (FREEZING && meta.REQUIRED && isExtensible(it) && !has(it, METADATA)) setMetadata(it);\n  return it;\n};\n\nvar meta = module.exports = {\n  REQUIRED: false,\n  fastKey: fastKey,\n  getWeakData: getWeakData,\n  onFreeze: onFreeze\n};\n\nhiddenKeys[METADATA] = true;\n","var anObject = require('../internals/an-object');\nvar isArrayIteratorMethod = require('../internals/is-array-iterator-method');\nvar toLength = require('../internals/to-length');\nvar bind = require('../internals/function-bind-context');\nvar getIteratorMethod = require('../internals/get-iterator-method');\nvar callWithSafeIterationClosing = require('../internals/call-with-safe-iteration-closing');\n\nvar Result = function (stopped, result) {\n  this.stopped = stopped;\n  this.result = result;\n};\n\nvar iterate = module.exports = function (iterable, fn, that, AS_ENTRIES, IS_ITERATOR) {\n  var boundFunction = bind(fn, that, AS_ENTRIES ? 2 : 1);\n  var iterator, iterFn, index, length, result, next, step;\n\n  if (IS_ITERATOR) {\n    iterator = iterable;\n  } else {\n    iterFn = getIteratorMethod(iterable);\n    if (typeof iterFn != 'function') throw TypeError('Target is not iterable');\n    // optimisation for array iterators\n    if (isArrayIteratorMethod(iterFn)) {\n      for (index = 0, length = toLength(iterable.length); length > index; index++) {\n        result = AS_ENTRIES\n          ? boundFunction(anObject(step = iterable[index])[0], step[1])\n          : boundFunction(iterable[index]);\n        if (result && result instanceof Result) return result;\n      } return new Result(false);\n    }\n    iterator = iterFn.call(iterable);\n  }\n\n  next = iterator.next;\n  while (!(step = next.call(iterator)).done) {\n    result = callWithSafeIterationClosing(iterator, boundFunction, step.value, AS_ENTRIES);\n    if (typeof result == 'object' && result && result instanceof Result) return result;\n  } return new Result(false);\n};\n\niterate.stop = function (result) {\n  return new Result(true, result);\n};\n","module.exports = function (it, Constructor, name) {\n  if (!(it instanceof Constructor)) {\n    throw TypeError('Incorrect ' + (name ? name + ' ' : '') + 'invocation');\n  } return it;\n};\n","'use strict';\nvar $ = require('./export');\nvar global = require('../internals/global');\nvar InternalMetadataModule = require('../internals/internal-metadata');\nvar fails = require('../internals/fails');\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\nvar iterate = require('../internals/iterate');\nvar anInstance = require('../internals/an-instance');\nvar isObject = require('../internals/is-object');\nvar setToStringTag = require('../internals/set-to-string-tag');\nvar defineProperty = require('../internals/object-define-property').f;\nvar forEach = require('../internals/array-iteration').forEach;\nvar DESCRIPTORS = require('../internals/descriptors');\nvar InternalStateModule = require('../internals/internal-state');\n\nvar setInternalState = InternalStateModule.set;\nvar internalStateGetterFor = InternalStateModule.getterFor;\n\nmodule.exports = function (CONSTRUCTOR_NAME, wrapper, common) {\n  var IS_MAP = CONSTRUCTOR_NAME.indexOf('Map') !== -1;\n  var IS_WEAK = CONSTRUCTOR_NAME.indexOf('Weak') !== -1;\n  var ADDER = IS_MAP ? 'set' : 'add';\n  var NativeConstructor = global[CONSTRUCTOR_NAME];\n  var NativePrototype = NativeConstructor && NativeConstructor.prototype;\n  var exported = {};\n  var Constructor;\n\n  if (!DESCRIPTORS || typeof NativeConstructor != 'function'\n    || !(IS_WEAK || NativePrototype.forEach && !fails(function () { new NativeConstructor().entries().next(); }))\n  ) {\n    // create collection constructor\n    Constructor = common.getConstructor(wrapper, CONSTRUCTOR_NAME, IS_MAP, ADDER);\n    InternalMetadataModule.REQUIRED = true;\n  } else {\n    Constructor = wrapper(function (target, iterable) {\n      setInternalState(anInstance(target, Constructor, CONSTRUCTOR_NAME), {\n        type: CONSTRUCTOR_NAME,\n        collection: new NativeConstructor()\n      });\n      if (iterable != undefined) iterate(iterable, target[ADDER], target, IS_MAP);\n    });\n\n    var getInternalState = internalStateGetterFor(CONSTRUCTOR_NAME);\n\n    forEach(['add', 'clear', 'delete', 'forEach', 'get', 'has', 'set', 'keys', 'values', 'entries'], function (KEY) {\n      var IS_ADDER = KEY == 'add' || KEY == 'set';\n      if (KEY in NativePrototype && !(IS_WEAK && KEY == 'clear')) {\n        createNonEnumerableProperty(Constructor.prototype, KEY, function (a, b) {\n          var collection = getInternalState(this).collection;\n          if (!IS_ADDER && IS_WEAK && !isObject(a)) return KEY == 'get' ? undefined : false;\n          var result = collection[KEY](a === 0 ? 0 : a, b);\n          return IS_ADDER ? this : result;\n        });\n      }\n    });\n\n    IS_WEAK || defineProperty(Constructor.prototype, 'size', {\n      configurable: true,\n      get: function () {\n        return getInternalState(this).collection.size;\n      }\n    });\n  }\n\n  setToStringTag(Constructor, CONSTRUCTOR_NAME, false, true);\n\n  exported[CONSTRUCTOR_NAME] = Constructor;\n  $({ global: true, forced: true }, exported);\n\n  if (!IS_WEAK) common.setStrong(Constructor, CONSTRUCTOR_NAME, IS_MAP);\n\n  return Constructor;\n};\n","var redefine = require('../internals/redefine');\n\nmodule.exports = function (target, src, options) {\n  for (var key in src) {\n    if (options && options.unsafe && target[key]) target[key] = src[key];\n    else redefine(target, key, src[key], options);\n  } return target;\n};\n","'use strict';\nvar getBuiltIn = require('../internals/get-built-in');\nvar definePropertyModule = require('../internals/object-define-property');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\nvar DESCRIPTORS = require('../internals/descriptors');\n\nvar SPECIES = wellKnownSymbol('species');\n\nmodule.exports = function (CONSTRUCTOR_NAME) {\n  var Constructor = getBuiltIn(CONSTRUCTOR_NAME);\n  var defineProperty = definePropertyModule.f;\n\n  if (DESCRIPTORS && Constructor && !Constructor[SPECIES]) {\n    defineProperty(Constructor, SPECIES, {\n      configurable: true,\n      get: function () { return this; }\n    });\n  }\n};\n","'use strict';\nvar defineProperty = require('../internals/object-define-property').f;\nvar create = require('../internals/object-create');\nvar redefineAll = require('../internals/redefine-all');\nvar bind = require('../internals/function-bind-context');\nvar anInstance = require('../internals/an-instance');\nvar iterate = require('../internals/iterate');\nvar defineIterator = require('../internals/define-iterator');\nvar setSpecies = require('../internals/set-species');\nvar DESCRIPTORS = require('../internals/descriptors');\nvar fastKey = require('../internals/internal-metadata').fastKey;\nvar InternalStateModule = require('../internals/internal-state');\n\nvar setInternalState = InternalStateModule.set;\nvar internalStateGetterFor = InternalStateModule.getterFor;\n\nmodule.exports = {\n  getConstructor: function (wrapper, CONSTRUCTOR_NAME, IS_MAP, ADDER) {\n    var C = wrapper(function (that, iterable) {\n      anInstance(that, C, CONSTRUCTOR_NAME);\n      setInternalState(that, {\n        type: CONSTRUCTOR_NAME,\n        index: create(null),\n        first: undefined,\n        last: undefined,\n        size: 0\n      });\n      if (!DESCRIPTORS) that.size = 0;\n      if (iterable != undefined) iterate(iterable, that[ADDER], that, IS_MAP);\n    });\n\n    var getInternalState = internalStateGetterFor(CONSTRUCTOR_NAME);\n\n    var define = function (that, key, value) {\n      var state = getInternalState(that);\n      var entry = getEntry(that, key);\n      var previous, index;\n      // change existing entry\n      if (entry) {\n        entry.value = value;\n      // create new entry\n      } else {\n        state.last = entry = {\n          index: index = fastKey(key, true),\n          key: key,\n          value: value,\n          previous: previous = state.last,\n          next: undefined,\n          removed: false\n        };\n        if (!state.first) state.first = entry;\n        if (previous) previous.next = entry;\n        if (DESCRIPTORS) state.size++;\n        else that.size++;\n        // add to index\n        if (index !== 'F') state.index[index] = entry;\n      } return that;\n    };\n\n    var getEntry = function (that, key) {\n      var state = getInternalState(that);\n      // fast case\n      var index = fastKey(key);\n      var entry;\n      if (index !== 'F') return state.index[index];\n      // frozen object case\n      for (entry = state.first; entry; entry = entry.next) {\n        if (entry.key == key) return entry;\n      }\n    };\n\n    redefineAll(C.prototype, {\n      // 23.1.3.1 Map.prototype.clear()\n      // 23.2.3.2 Set.prototype.clear()\n      clear: function clear() {\n        var that = this;\n        var state = getInternalState(that);\n        var data = state.index;\n        var entry = state.first;\n        while (entry) {\n          entry.removed = true;\n          if (entry.previous) entry.previous = entry.previous.next = undefined;\n          delete data[entry.index];\n          entry = entry.next;\n        }\n        state.first = state.last = undefined;\n        if (DESCRIPTORS) state.size = 0;\n        else that.size = 0;\n      },\n      // 23.1.3.3 Map.prototype.delete(key)\n      // 23.2.3.4 Set.prototype.delete(value)\n      'delete': function (key) {\n        var that = this;\n        var state = getInternalState(that);\n        var entry = getEntry(that, key);\n        if (entry) {\n          var next = entry.next;\n          var prev = entry.previous;\n          delete state.index[entry.index];\n          entry.removed = true;\n          if (prev) prev.next = next;\n          if (next) next.previous = prev;\n          if (state.first == entry) state.first = next;\n          if (state.last == entry) state.last = prev;\n          if (DESCRIPTORS) state.size--;\n          else that.size--;\n        } return !!entry;\n      },\n      // 23.2.3.6 Set.prototype.forEach(callbackfn, thisArg = undefined)\n      // 23.1.3.5 Map.prototype.forEach(callbackfn, thisArg = undefined)\n      forEach: function forEach(callbackfn /* , that = undefined */) {\n        var state = getInternalState(this);\n        var boundFunction = bind(callbackfn, arguments.length > 1 ? arguments[1] : undefined, 3);\n        var entry;\n        while (entry = entry ? entry.next : state.first) {\n          boundFunction(entry.value, entry.key, this);\n          // revert to the last existing entry\n          while (entry && entry.removed) entry = entry.previous;\n        }\n      },\n      // 23.1.3.7 Map.prototype.has(key)\n      // 23.2.3.7 Set.prototype.has(value)\n      has: function has(key) {\n        return !!getEntry(this, key);\n      }\n    });\n\n    redefineAll(C.prototype, IS_MAP ? {\n      // 23.1.3.6 Map.prototype.get(key)\n      get: function get(key) {\n        var entry = getEntry(this, key);\n        return entry && entry.value;\n      },\n      // 23.1.3.9 Map.prototype.set(key, value)\n      set: function set(key, value) {\n        return define(this, key === 0 ? 0 : key, value);\n      }\n    } : {\n      // 23.2.3.1 Set.prototype.add(value)\n      add: function add(value) {\n        return define(this, value = value === 0 ? 0 : value, value);\n      }\n    });\n    if (DESCRIPTORS) defineProperty(C.prototype, 'size', {\n      get: function () {\n        return getInternalState(this).size;\n      }\n    });\n    return C;\n  },\n  setStrong: function (C, CONSTRUCTOR_NAME, IS_MAP) {\n    var ITERATOR_NAME = CONSTRUCTOR_NAME + ' Iterator';\n    var getInternalCollectionState = internalStateGetterFor(CONSTRUCTOR_NAME);\n    var getInternalIteratorState = internalStateGetterFor(ITERATOR_NAME);\n    // add .keys, .values, .entries, [@@iterator]\n    // 23.1.3.4, 23.1.3.8, 23.1.3.11, 23.1.3.12, 23.2.3.5, 23.2.3.8, 23.2.3.10, 23.2.3.11\n    defineIterator(C, CONSTRUCTOR_NAME, function (iterated, kind) {\n      setInternalState(this, {\n        type: ITERATOR_NAME,\n        target: iterated,\n        state: getInternalCollectionState(iterated),\n        kind: kind,\n        last: undefined\n      });\n    }, function () {\n      var state = getInternalIteratorState(this);\n      var kind = state.kind;\n      var entry = state.last;\n      // revert to the last existing entry\n      while (entry && entry.removed) entry = entry.previous;\n      // get next entry\n      if (!state.target || !(state.last = entry = entry ? entry.next : state.state.first)) {\n        // or finish the iteration\n        state.target = undefined;\n        return { value: undefined, done: true };\n      }\n      // return step by kind\n      if (kind == 'keys') return { value: entry.key, done: false };\n      if (kind == 'values') return { value: entry.value, done: false };\n      return { value: [entry.key, entry.value], done: false };\n    }, IS_MAP ? 'entries' : 'values', !IS_MAP, true);\n\n    // add [@@species], 23.1.2.2, 23.2.2.2\n    setSpecies(CONSTRUCTOR_NAME);\n  }\n};\n","'use strict';\nvar collection = require('../internals/collection');\nvar collectionStrong = require('../internals/collection-strong');\n\n// `Set` constructor\n// https://tc39.github.io/ecma262/#sec-set-objects\nmodule.exports = collection('Set', function (init) {\n  return function Set() { return init(this, arguments.length ? arguments[0] : undefined); };\n}, collectionStrong);\n","require('../../modules/es.set');\nrequire('../../modules/es.object.to-string');\nrequire('../../modules/es.string.iterator');\nrequire('../../modules/web.dom-collections.iterator');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Set;\n","'use strict';\nvar collection = require('../internals/collection');\nvar collectionStrong = require('../internals/collection-strong');\n\n// `Map` constructor\n// https://tc39.github.io/ecma262/#sec-map-objects\nmodule.exports = collection('Map', function (init) {\n  return function Map() { return init(this, arguments.length ? arguments[0] : undefined); };\n}, collectionStrong);\n","require('../../modules/es.map');\nrequire('../../modules/es.object.to-string');\nrequire('../../modules/es.string.iterator');\nrequire('../../modules/web.dom-collections.iterator');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Map;\n","/**\n * Copyright (c) 2014-present, Facebook, Inc.\n *\n * This source code is licensed under the MIT license found in the\n * LICENSE file in the root directory of this source tree.\n */\n\nvar runtime = (function (exports) {\n  \"use strict\";\n\n  var Op = Object.prototype;\n  var hasOwn = Op.hasOwnProperty;\n  var undefined; // More compressible than void 0.\n  var $Symbol = typeof Symbol === \"function\" ? Symbol : {};\n  var iteratorSymbol = $Symbol.iterator || \"@@iterator\";\n  var asyncIteratorSymbol = $Symbol.asyncIterator || \"@@asyncIterator\";\n  var toStringTagSymbol = $Symbol.toStringTag || \"@@toStringTag\";\n\n  function wrap(innerFn, outerFn, self, tryLocsList) {\n    // If outerFn provided and outerFn.prototype is a Generator, then outerFn.prototype instanceof Generator.\n    var protoGenerator = outerFn && outerFn.prototype instanceof Generator ? outerFn : Generator;\n    var generator = Object.create(protoGenerator.prototype);\n    var context = new Context(tryLocsList || []);\n\n    // The ._invoke method unifies the implementations of the .next,\n    // .throw, and .return methods.\n    generator._invoke = makeInvokeMethod(innerFn, self, context);\n\n    return generator;\n  }\n  exports.wrap = wrap;\n\n  // Try/catch helper to minimize deoptimizations. Returns a completion\n  // record like context.tryEntries[i].completion. This interface could\n  // have been (and was previously) designed to take a closure to be\n  // invoked without arguments, but in all the cases we care about we\n  // already have an existing method we want to call, so there's no need\n  // to create a new function object. We can even get away with assuming\n  // the method takes exactly one argument, since that happens to be true\n  // in every case, so we don't have to touch the arguments object. The\n  // only additional allocation required is the completion record, which\n  // has a stable shape and so hopefully should be cheap to allocate.\n  function tryCatch(fn, obj, arg) {\n    try {\n      return { type: \"normal\", arg: fn.call(obj, arg) };\n    } catch (err) {\n      return { type: \"throw\", arg: err };\n    }\n  }\n\n  var GenStateSuspendedStart = \"suspendedStart\";\n  var GenStateSuspendedYield = \"suspendedYield\";\n  var GenStateExecuting = \"executing\";\n  var GenStateCompleted = \"completed\";\n\n  // Returning this object from the innerFn has the same effect as\n  // breaking out of the dispatch switch statement.\n  var ContinueSentinel = {};\n\n  // Dummy constructor functions that we use as the .constructor and\n  // .constructor.prototype properties for functions that return Generator\n  // objects. For full spec compliance, you may wish to configure your\n  // minifier not to mangle the names of these two functions.\n  function Generator() {}\n  function GeneratorFunction() {}\n  function GeneratorFunctionPrototype() {}\n\n  // This is a polyfill for %IteratorPrototype% for environments that\n  // don't natively support it.\n  var IteratorPrototype = {};\n  IteratorPrototype[iteratorSymbol] = function () {\n    return this;\n  };\n\n  var getProto = Object.getPrototypeOf;\n  var NativeIteratorPrototype = getProto && getProto(getProto(values([])));\n  if (NativeIteratorPrototype &&\n      NativeIteratorPrototype !== Op &&\n      hasOwn.call(NativeIteratorPrototype, iteratorSymbol)) {\n    // This environment has a native %IteratorPrototype%; use it instead\n    // of the polyfill.\n    IteratorPrototype = NativeIteratorPrototype;\n  }\n\n  var Gp = GeneratorFunctionPrototype.prototype =\n    Generator.prototype = Object.create(IteratorPrototype);\n  GeneratorFunction.prototype = Gp.constructor = GeneratorFunctionPrototype;\n  GeneratorFunctionPrototype.constructor = GeneratorFunction;\n  GeneratorFunctionPrototype[toStringTagSymbol] =\n    GeneratorFunction.displayName = \"GeneratorFunction\";\n\n  // Helper for defining the .next, .throw, and .return methods of the\n  // Iterator interface in terms of a single ._invoke method.\n  function defineIteratorMethods(prototype) {\n    [\"next\", \"throw\", \"return\"].forEach(function(method) {\n      prototype[method] = function(arg) {\n        return this._invoke(method, arg);\n      };\n    });\n  }\n\n  exports.isGeneratorFunction = function(genFun) {\n    var ctor = typeof genFun === \"function\" && genFun.constructor;\n    return ctor\n      ? ctor === GeneratorFunction ||\n        // For the native GeneratorFunction constructor, the best we can\n        // do is to check its .name property.\n        (ctor.displayName || ctor.name) === \"GeneratorFunction\"\n      : false;\n  };\n\n  exports.mark = function(genFun) {\n    if (Object.setPrototypeOf) {\n      Object.setPrototypeOf(genFun, GeneratorFunctionPrototype);\n    } else {\n      genFun.__proto__ = GeneratorFunctionPrototype;\n      if (!(toStringTagSymbol in genFun)) {\n        genFun[toStringTagSymbol] = \"GeneratorFunction\";\n      }\n    }\n    genFun.prototype = Object.create(Gp);\n    return genFun;\n  };\n\n  // Within the body of any async function, `await x` is transformed to\n  // `yield regeneratorRuntime.awrap(x)`, so that the runtime can test\n  // `hasOwn.call(value, \"__await\")` to determine if the yielded value is\n  // meant to be awaited.\n  exports.awrap = function(arg) {\n    return { __await: arg };\n  };\n\n  function AsyncIterator(generator, PromiseImpl) {\n    function invoke(method, arg, resolve, reject) {\n      var record = tryCatch(generator[method], generator, arg);\n      if (record.type === \"throw\") {\n        reject(record.arg);\n      } else {\n        var result = record.arg;\n        var value = result.value;\n        if (value &&\n            typeof value === \"object\" &&\n            hasOwn.call(value, \"__await\")) {\n          return PromiseImpl.resolve(value.__await).then(function(value) {\n            invoke(\"next\", value, resolve, reject);\n          }, function(err) {\n            invoke(\"throw\", err, resolve, reject);\n          });\n        }\n\n        return PromiseImpl.resolve(value).then(function(unwrapped) {\n          // When a yielded Promise is resolved, its final value becomes\n          // the .value of the Promise<{value,done}> result for the\n          // current iteration.\n          result.value = unwrapped;\n          resolve(result);\n        }, function(error) {\n          // If a rejected Promise was yielded, throw the rejection back\n          // into the async generator function so it can be handled there.\n          return invoke(\"throw\", error, resolve, reject);\n        });\n      }\n    }\n\n    var previousPromise;\n\n    function enqueue(method, arg) {\n      function callInvokeWithMethodAndArg() {\n        return new PromiseImpl(function(resolve, reject) {\n          invoke(method, arg, resolve, reject);\n        });\n      }\n\n      return previousPromise =\n        // If enqueue has been called before, then we want to wait until\n        // all previous Promises have been resolved before calling invoke,\n        // so that results are always delivered in the correct order. If\n        // enqueue has not been called before, then it is important to\n        // call invoke immediately, without waiting on a callback to fire,\n        // so that the async generator function has the opportunity to do\n        // any necessary setup in a predictable way. This predictability\n        // is why the Promise constructor synchronously invokes its\n        // executor callback, and why async functions synchronously\n        // execute code before the first await. Since we implement simple\n        // async functions in terms of async generators, it is especially\n        // important to get this right, even though it requires care.\n        previousPromise ? previousPromise.then(\n          callInvokeWithMethodAndArg,\n          // Avoid propagating failures to Promises returned by later\n          // invocations of the iterator.\n          callInvokeWithMethodAndArg\n        ) : callInvokeWithMethodAndArg();\n    }\n\n    // Define the unified helper method that is used to implement .next,\n    // .throw, and .return (see defineIteratorMethods).\n    this._invoke = enqueue;\n  }\n\n  defineIteratorMethods(AsyncIterator.prototype);\n  AsyncIterator.prototype[asyncIteratorSymbol] = function () {\n    return this;\n  };\n  exports.AsyncIterator = AsyncIterator;\n\n  // Note that simple async functions are implemented on top of\n  // AsyncIterator objects; they just return a Promise for the value of\n  // the final result produced by the iterator.\n  exports.async = function(innerFn, outerFn, self, tryLocsList, PromiseImpl) {\n    if (PromiseImpl === void 0) PromiseImpl = Promise;\n\n    var iter = new AsyncIterator(\n      wrap(innerFn, outerFn, self, tryLocsList),\n      PromiseImpl\n    );\n\n    return exports.isGeneratorFunction(outerFn)\n      ? iter // If outerFn is a generator, return the full iterator.\n      : iter.next().then(function(result) {\n          return result.done ? result.value : iter.next();\n        });\n  };\n\n  function makeInvokeMethod(innerFn, self, context) {\n    var state = GenStateSuspendedStart;\n\n    return function invoke(method, arg) {\n      if (state === GenStateExecuting) {\n        throw new Error(\"Generator is already running\");\n      }\n\n      if (state === GenStateCompleted) {\n        if (method === \"throw\") {\n          throw arg;\n        }\n\n        // Be forgiving, per 25.3.3.3.3 of the spec:\n        // https://people.mozilla.org/~jorendorff/es6-draft.html#sec-generatorresume\n        return doneResult();\n      }\n\n      context.method = method;\n      context.arg = arg;\n\n      while (true) {\n        var delegate = context.delegate;\n        if (delegate) {\n          var delegateResult = maybeInvokeDelegate(delegate, context);\n          if (delegateResult) {\n            if (delegateResult === ContinueSentinel) continue;\n            return delegateResult;\n          }\n        }\n\n        if (context.method === \"next\") {\n          // Setting context._sent for legacy support of Babel's\n          // function.sent implementation.\n          context.sent = context._sent = context.arg;\n\n        } else if (context.method === \"throw\") {\n          if (state === GenStateSuspendedStart) {\n            state = GenStateCompleted;\n            throw context.arg;\n          }\n\n          context.dispatchException(context.arg);\n\n        } else if (context.method === \"return\") {\n          context.abrupt(\"return\", context.arg);\n        }\n\n        state = GenStateExecuting;\n\n        var record = tryCatch(innerFn, self, context);\n        if (record.type === \"normal\") {\n          // If an exception is thrown from innerFn, we leave state ===\n          // GenStateExecuting and loop back for another invocation.\n          state = context.done\n            ? GenStateCompleted\n            : GenStateSuspendedYield;\n\n          if (record.arg === ContinueSentinel) {\n            continue;\n          }\n\n          return {\n            value: record.arg,\n            done: context.done\n          };\n\n        } else if (record.type === \"throw\") {\n          state = GenStateCompleted;\n          // Dispatch the exception by looping back around to the\n          // context.dispatchException(context.arg) call above.\n          context.method = \"throw\";\n          context.arg = record.arg;\n        }\n      }\n    };\n  }\n\n  // Call delegate.iterator[context.method](context.arg) and handle the\n  // result, either by returning a { value, done } result from the\n  // delegate iterator, or by modifying context.method and context.arg,\n  // setting context.delegate to null, and returning the ContinueSentinel.\n  function maybeInvokeDelegate(delegate, context) {\n    var method = delegate.iterator[context.method];\n    if (method === undefined) {\n      // A .throw or .return when the delegate iterator has no .throw\n      // method always terminates the yield* loop.\n      context.delegate = null;\n\n      if (context.method === \"throw\") {\n        // Note: [\"return\"] must be used for ES3 parsing compatibility.\n        if (delegate.iterator[\"return\"]) {\n          // If the delegate iterator has a return method, give it a\n          // chance to clean up.\n          context.method = \"return\";\n          context.arg = undefined;\n          maybeInvokeDelegate(delegate, context);\n\n          if (context.method === \"throw\") {\n            // If maybeInvokeDelegate(context) changed context.method from\n            // \"return\" to \"throw\", let that override the TypeError below.\n            return ContinueSentinel;\n          }\n        }\n\n        context.method = \"throw\";\n        context.arg = new TypeError(\n          \"The iterator does not provide a 'throw' method\");\n      }\n\n      return ContinueSentinel;\n    }\n\n    var record = tryCatch(method, delegate.iterator, context.arg);\n\n    if (record.type === \"throw\") {\n      context.method = \"throw\";\n      context.arg = record.arg;\n      context.delegate = null;\n      return ContinueSentinel;\n    }\n\n    var info = record.arg;\n\n    if (! info) {\n      context.method = \"throw\";\n      context.arg = new TypeError(\"iterator result is not an object\");\n      context.delegate = null;\n      return ContinueSentinel;\n    }\n\n    if (info.done) {\n      // Assign the result of the finished delegate to the temporary\n      // variable specified by delegate.resultName (see delegateYield).\n      context[delegate.resultName] = info.value;\n\n      // Resume execution at the desired location (see delegateYield).\n      context.next = delegate.nextLoc;\n\n      // If context.method was \"throw\" but the delegate handled the\n      // exception, let the outer generator proceed normally. If\n      // context.method was \"next\", forget context.arg since it has been\n      // \"consumed\" by the delegate iterator. If context.method was\n      // \"return\", allow the original .return call to continue in the\n      // outer generator.\n      if (context.method !== \"return\") {\n        context.method = \"next\";\n        context.arg = undefined;\n      }\n\n    } else {\n      // Re-yield the result returned by the delegate method.\n      return info;\n    }\n\n    // The delegate iterator is finished, so forget it and continue with\n    // the outer generator.\n    context.delegate = null;\n    return ContinueSentinel;\n  }\n\n  // Define Generator.prototype.{next,throw,return} in terms of the\n  // unified ._invoke helper method.\n  defineIteratorMethods(Gp);\n\n  Gp[toStringTagSymbol] = \"Generator\";\n\n  // A Generator should always return itself as the iterator object when the\n  // @@iterator function is called on it. Some browsers' implementations of the\n  // iterator prototype chain incorrectly implement this, causing the Generator\n  // object to not be returned from this call. This ensures that doesn't happen.\n  // See https://github.com/facebook/regenerator/issues/274 for more details.\n  Gp[iteratorSymbol] = function() {\n    return this;\n  };\n\n  Gp.toString = function() {\n    return \"[object Generator]\";\n  };\n\n  function pushTryEntry(locs) {\n    var entry = { tryLoc: locs[0] };\n\n    if (1 in locs) {\n      entry.catchLoc = locs[1];\n    }\n\n    if (2 in locs) {\n      entry.finallyLoc = locs[2];\n      entry.afterLoc = locs[3];\n    }\n\n    this.tryEntries.push(entry);\n  }\n\n  function resetTryEntry(entry) {\n    var record = entry.completion || {};\n    record.type = \"normal\";\n    delete record.arg;\n    entry.completion = record;\n  }\n\n  function Context(tryLocsList) {\n    // The root entry object (effectively a try statement without a catch\n    // or a finally block) gives us a place to store values thrown from\n    // locations where there is no enclosing try statement.\n    this.tryEntries = [{ tryLoc: \"root\" }];\n    tryLocsList.forEach(pushTryEntry, this);\n    this.reset(true);\n  }\n\n  exports.keys = function(object) {\n    var keys = [];\n    for (var key in object) {\n      keys.push(key);\n    }\n    keys.reverse();\n\n    // Rather than returning an object with a next method, we keep\n    // things simple and return the next function itself.\n    return function next() {\n      while (keys.length) {\n        var key = keys.pop();\n        if (key in object) {\n          next.value = key;\n          next.done = false;\n          return next;\n        }\n      }\n\n      // To avoid creating an additional object, we just hang the .value\n      // and .done properties off the next function object itself. This\n      // also ensures that the minifier will not anonymize the function.\n      next.done = true;\n      return next;\n    };\n  };\n\n  function values(iterable) {\n    if (iterable) {\n      var iteratorMethod = iterable[iteratorSymbol];\n      if (iteratorMethod) {\n        return iteratorMethod.call(iterable);\n      }\n\n      if (typeof iterable.next === \"function\") {\n        return iterable;\n      }\n\n      if (!isNaN(iterable.length)) {\n        var i = -1, next = function next() {\n          while (++i < iterable.length) {\n            if (hasOwn.call(iterable, i)) {\n              next.value = iterable[i];\n              next.done = false;\n              return next;\n            }\n          }\n\n          next.value = undefined;\n          next.done = true;\n\n          return next;\n        };\n\n        return next.next = next;\n      }\n    }\n\n    // Return an iterator with no values.\n    return { next: doneResult };\n  }\n  exports.values = values;\n\n  function doneResult() {\n    return { value: undefined, done: true };\n  }\n\n  Context.prototype = {\n    constructor: Context,\n\n    reset: function(skipTempReset) {\n      this.prev = 0;\n      this.next = 0;\n      // Resetting context._sent for legacy support of Babel's\n      // function.sent implementation.\n      this.sent = this._sent = undefined;\n      this.done = false;\n      this.delegate = null;\n\n      this.method = \"next\";\n      this.arg = undefined;\n\n      this.tryEntries.forEach(resetTryEntry);\n\n      if (!skipTempReset) {\n        for (var name in this) {\n          // Not sure about the optimal order of these conditions:\n          if (name.charAt(0) === \"t\" &&\n              hasOwn.call(this, name) &&\n              !isNaN(+name.slice(1))) {\n            this[name] = undefined;\n          }\n        }\n      }\n    },\n\n    stop: function() {\n      this.done = true;\n\n      var rootEntry = this.tryEntries[0];\n      var rootRecord = rootEntry.completion;\n      if (rootRecord.type === \"throw\") {\n        throw rootRecord.arg;\n      }\n\n      return this.rval;\n    },\n\n    dispatchException: function(exception) {\n      if (this.done) {\n        throw exception;\n      }\n\n      var context = this;\n      function handle(loc, caught) {\n        record.type = \"throw\";\n        record.arg = exception;\n        context.next = loc;\n\n        if (caught) {\n          // If the dispatched exception was caught by a catch block,\n          // then let that catch block handle the exception normally.\n          context.method = \"next\";\n          context.arg = undefined;\n        }\n\n        return !! caught;\n      }\n\n      for (var i = this.tryEntries.length - 1; i >= 0; --i) {\n        var entry = this.tryEntries[i];\n        var record = entry.completion;\n\n        if (entry.tryLoc === \"root\") {\n          // Exception thrown outside of any try block that could handle\n          // it, so set the completion value of the entire function to\n          // throw the exception.\n          return handle(\"end\");\n        }\n\n        if (entry.tryLoc <= this.prev) {\n          var hasCatch = hasOwn.call(entry, \"catchLoc\");\n          var hasFinally = hasOwn.call(entry, \"finallyLoc\");\n\n          if (hasCatch && hasFinally) {\n            if (this.prev < entry.catchLoc) {\n              return handle(entry.catchLoc, true);\n            } else if (this.prev < entry.finallyLoc) {\n              return handle(entry.finallyLoc);\n            }\n\n          } else if (hasCatch) {\n            if (this.prev < entry.catchLoc) {\n              return handle(entry.catchLoc, true);\n            }\n\n          } else if (hasFinally) {\n            if (this.prev < entry.finallyLoc) {\n              return handle(entry.finallyLoc);\n            }\n\n          } else {\n            throw new Error(\"try statement without catch or finally\");\n          }\n        }\n      }\n    },\n\n    abrupt: function(type, arg) {\n      for (var i = this.tryEntries.length - 1; i >= 0; --i) {\n        var entry = this.tryEntries[i];\n        if (entry.tryLoc <= this.prev &&\n            hasOwn.call(entry, \"finallyLoc\") &&\n            this.prev < entry.finallyLoc) {\n          var finallyEntry = entry;\n          break;\n        }\n      }\n\n      if (finallyEntry &&\n          (type === \"break\" ||\n           type === \"continue\") &&\n          finallyEntry.tryLoc <= arg &&\n          arg <= finallyEntry.finallyLoc) {\n        // Ignore the finally entry if control is not jumping to a\n        // location outside the try/catch block.\n        finallyEntry = null;\n      }\n\n      var record = finallyEntry ? finallyEntry.completion : {};\n      record.type = type;\n      record.arg = arg;\n\n      if (finallyEntry) {\n        this.method = \"next\";\n        this.next = finallyEntry.finallyLoc;\n        return ContinueSentinel;\n      }\n\n      return this.complete(record);\n    },\n\n    complete: function(record, afterLoc) {\n      if (record.type === \"throw\") {\n        throw record.arg;\n      }\n\n      if (record.type === \"break\" ||\n          record.type === \"continue\") {\n        this.next = record.arg;\n      } else if (record.type === \"return\") {\n        this.rval = this.arg = record.arg;\n        this.method = \"return\";\n        this.next = \"end\";\n      } else if (record.type === \"normal\" && afterLoc) {\n        this.next = afterLoc;\n      }\n\n      return ContinueSentinel;\n    },\n\n    finish: function(finallyLoc) {\n      for (var i = this.tryEntries.length - 1; i >= 0; --i) {\n        var entry = this.tryEntries[i];\n        if (entry.finallyLoc === finallyLoc) {\n          this.complete(entry.completion, entry.afterLoc);\n          resetTryEntry(entry);\n          return ContinueSentinel;\n        }\n      }\n    },\n\n    \"catch\": function(tryLoc) {\n      for (var i = this.tryEntries.length - 1; i >= 0; --i) {\n        var entry = this.tryEntries[i];\n        if (entry.tryLoc === tryLoc) {\n          var record = entry.completion;\n          if (record.type === \"throw\") {\n            var thrown = record.arg;\n            resetTryEntry(entry);\n          }\n          return thrown;\n        }\n      }\n\n      // The context.catch method must only be called with a location\n      // argument that corresponds to a known catch block.\n      throw new Error(\"illegal catch attempt\");\n    },\n\n    delegateYield: function(iterable, resultName, nextLoc) {\n      this.delegate = {\n        iterator: values(iterable),\n        resultName: resultName,\n        nextLoc: nextLoc\n      };\n\n      if (this.method === \"next\") {\n        // Deliberately forget the last sent value so that we don't\n        // accidentally pass it on to the delegate.\n        this.arg = undefined;\n      }\n\n      return ContinueSentinel;\n    }\n  };\n\n  // Regardless of whether this script is executing as a CommonJS module\n  // or not, return the runtime object so that we can declare the variable\n  // regeneratorRuntime in the outer scope, which allows this module to be\n  // injected easily by `bin/regenerator --include-runtime script.js`.\n  return exports;\n\n}(\n  // If this script is executing as a CommonJS module, use module.exports\n  // as the regeneratorRuntime namespace. Otherwise create a new empty\n  // object. Either way, the resulting object will be used to initialize\n  // the regeneratorRuntime variable at the top of this file.\n  typeof module === \"object\" ? module.exports : {}\n));\n\ntry {\n  regeneratorRuntime = runtime;\n} catch (accidentalStrictMode) {\n  // This module should not be running in strict mode, so the above\n  // assignment should always work unless something is misconfigured. Just\n  // in case runtime.js accidentally runs in strict mode, we can escape\n  // strict mode using a global Function call. This could conceivably fail\n  // if a Content Security Policy forbids using Function, but in that case\n  // the proper solution is to fix the accidental strict mode problem. If\n  // you've misconfigured your bundler to force strict mode and applied a\n  // CSP to forbid Function, and you're not willing to fix either of those\n  // problems, please detail your unique predicament in a GitHub issue.\n  Function(\"r\", \"regeneratorRuntime = r\")(runtime);\n}\n","var parent = require('../../es/symbol/iterator');\n\nmodule.exports = parent;\n","'use strict';\nvar $ = require('../internals/export');\nvar toAbsoluteIndex = require('../internals/to-absolute-index');\nvar toInteger = require('../internals/to-integer');\nvar toLength = require('../internals/to-length');\nvar toObject = require('../internals/to-object');\nvar arraySpeciesCreate = require('../internals/array-species-create');\nvar createProperty = require('../internals/create-property');\nvar arrayMethodHasSpeciesSupport = require('../internals/array-method-has-species-support');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar HAS_SPECIES_SUPPORT = arrayMethodHasSpeciesSupport('splice');\nvar USES_TO_LENGTH = arrayMethodUsesToLength('splice', { ACCESSORS: true, 0: 0, 1: 2 });\n\nvar max = Math.max;\nvar min = Math.min;\nvar MAX_SAFE_INTEGER = 0x1FFFFFFFFFFFFF;\nvar MAXIMUM_ALLOWED_LENGTH_EXCEEDED = 'Maximum allowed length exceeded';\n\n// `Array.prototype.splice` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.splice\n// with adding support of @@species\n$({ target: 'Array', proto: true, forced: !HAS_SPECIES_SUPPORT || !USES_TO_LENGTH }, {\n  splice: function splice(start, deleteCount /* , ...items */) {\n    var O = toObject(this);\n    var len = toLength(O.length);\n    var actualStart = toAbsoluteIndex(start, len);\n    var argumentsLength = arguments.length;\n    var insertCount, actualDeleteCount, A, k, from, to;\n    if (argumentsLength === 0) {\n      insertCount = actualDeleteCount = 0;\n    } else if (argumentsLength === 1) {\n      insertCount = 0;\n      actualDeleteCount = len - actualStart;\n    } else {\n      insertCount = argumentsLength - 2;\n      actualDeleteCount = min(max(toInteger(deleteCount), 0), len - actualStart);\n    }\n    if (len + insertCount - actualDeleteCount > MAX_SAFE_INTEGER) {\n      throw TypeError(MAXIMUM_ALLOWED_LENGTH_EXCEEDED);\n    }\n    A = arraySpeciesCreate(O, actualDeleteCount);\n    for (k = 0; k < actualDeleteCount; k++) {\n      from = actualStart + k;\n      if (from in O) createProperty(A, k, O[from]);\n    }\n    A.length = actualDeleteCount;\n    if (insertCount < actualDeleteCount) {\n      for (k = actualStart; k < len - actualDeleteCount; k++) {\n        from = k + actualDeleteCount;\n        to = k + insertCount;\n        if (from in O) O[to] = O[from];\n        else delete O[to];\n      }\n      for (k = len; k > len - actualDeleteCount + insertCount; k--) delete O[k - 1];\n    } else if (insertCount > actualDeleteCount) {\n      for (k = len - actualDeleteCount; k > actualStart; k--) {\n        from = k + actualDeleteCount - 1;\n        to = k + insertCount - 1;\n        if (from in O) O[to] = O[from];\n        else delete O[to];\n      }\n    }\n    for (k = 0; k < insertCount; k++) {\n      O[k + actualStart] = arguments[k + 2];\n    }\n    O.length = len - actualDeleteCount + insertCount;\n    return A;\n  }\n});\n","require('../../../modules/es.array.splice');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').splice;\n","var splice = require('../array/virtual/splice');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.splice;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.splice) ? splice : own;\n};\n","var $ = require('../internals/export');\nvar global = require('../internals/global');\nvar userAgent = require('../internals/engine-user-agent');\n\nvar slice = [].slice;\nvar MSIE = /MSIE .\\./.test(userAgent); // <- dirty ie9- check\n\nvar wrap = function (scheduler) {\n  return function (handler, timeout /* , ...arguments */) {\n    var boundArgs = arguments.length > 2;\n    var args = boundArgs ? slice.call(arguments, 2) : undefined;\n    return scheduler(boundArgs ? function () {\n      // eslint-disable-next-line no-new-func\n      (typeof handler == 'function' ? handler : Function(handler)).apply(this, args);\n    } : handler, timeout);\n  };\n};\n\n// ie9- setTimeout & setInterval additional parameters fix\n// https://html.spec.whatwg.org/multipage/timers-and-user-prompts.html#timers\n$({ global: true, bind: true, forced: MSIE }, {\n  // `setTimeout` method\n  // https://html.spec.whatwg.org/multipage/timers-and-user-prompts.html#dom-settimeout\n  setTimeout: wrap(global.setTimeout),\n  // `setInterval` method\n  // https://html.spec.whatwg.org/multipage/timers-and-user-prompts.html#dom-setinterval\n  setInterval: wrap(global.setInterval)\n});\n","require('../modules/web.timers');\nvar path = require('../internals/path');\n\nmodule.exports = path.setTimeout;\n","'use strict';\nvar isArray = require('../internals/is-array');\nvar toLength = require('../internals/to-length');\nvar bind = require('../internals/function-bind-context');\n\n// `FlattenIntoArray` abstract operation\n// https://tc39.github.io/proposal-flatMap/#sec-FlattenIntoArray\nvar flattenIntoArray = function (target, original, source, sourceLen, start, depth, mapper, thisArg) {\n  var targetIndex = start;\n  var sourceIndex = 0;\n  var mapFn = mapper ? bind(mapper, thisArg, 3) : false;\n  var element;\n\n  while (sourceIndex < sourceLen) {\n    if (sourceIndex in source) {\n      element = mapFn ? mapFn(source[sourceIndex], sourceIndex, original) : source[sourceIndex];\n\n      if (depth > 0 && isArray(element)) {\n        targetIndex = flattenIntoArray(target, original, element, toLength(element.length), targetIndex, depth - 1) - 1;\n      } else {\n        if (targetIndex >= 0x1FFFFFFFFFFFFF) throw TypeError('Exceed the acceptable array length');\n        target[targetIndex] = element;\n      }\n\n      targetIndex++;\n    }\n    sourceIndex++;\n  }\n  return targetIndex;\n};\n\nmodule.exports = flattenIntoArray;\n","'use strict';\nvar $ = require('../internals/export');\nvar flattenIntoArray = require('../internals/flatten-into-array');\nvar toObject = require('../internals/to-object');\nvar toLength = require('../internals/to-length');\nvar aFunction = require('../internals/a-function');\nvar arraySpeciesCreate = require('../internals/array-species-create');\n\n// `Array.prototype.flatMap` method\n// https://github.com/tc39/proposal-flatMap\n$({ target: 'Array', proto: true }, {\n  flatMap: function flatMap(callbackfn /* , thisArg */) {\n    var O = toObject(this);\n    var sourceLen = toLength(O.length);\n    var A;\n    aFunction(callbackfn);\n    A = arraySpeciesCreate(O, 0);\n    A.length = flattenIntoArray(A, O, O, sourceLen, 0, 1, callbackfn, arguments.length > 1 ? arguments[1] : undefined);\n    return A;\n  }\n});\n","require('../../../modules/es.array.flat-map');\nrequire('../../../modules/es.array.unscopables.flat-map');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').flatMap;\n","var flatMap = require('../array/virtual/flat-map');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.flatMap;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.flatMap) ? flatMap : own;\n};\n","var aFunction = require('../internals/a-function');\nvar toObject = require('../internals/to-object');\nvar IndexedObject = require('../internals/indexed-object');\nvar toLength = require('../internals/to-length');\n\n// `Array.prototype.{ reduce, reduceRight }` methods implementation\nvar createMethod = function (IS_RIGHT) {\n  return function (that, callbackfn, argumentsLength, memo) {\n    aFunction(callbackfn);\n    var O = toObject(that);\n    var self = IndexedObject(O);\n    var length = toLength(O.length);\n    var index = IS_RIGHT ? length - 1 : 0;\n    var i = IS_RIGHT ? -1 : 1;\n    if (argumentsLength < 2) while (true) {\n      if (index in self) {\n        memo = self[index];\n        index += i;\n        break;\n      }\n      index += i;\n      if (IS_RIGHT ? index < 0 : length <= index) {\n        throw TypeError('Reduce of empty array with no initial value');\n      }\n    }\n    for (;IS_RIGHT ? index >= 0 : length > index; index += i) if (index in self) {\n      memo = callbackfn(memo, self[index], index, O);\n    }\n    return memo;\n  };\n};\n\nmodule.exports = {\n  // `Array.prototype.reduce` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.reduce\n  left: createMethod(false),\n  // `Array.prototype.reduceRight` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.reduceright\n  right: createMethod(true)\n};\n","'use strict';\nvar $ = require('../internals/export');\nvar $reduce = require('../internals/array-reduce').left;\nvar arrayMethodIsStrict = require('../internals/array-method-is-strict');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar STRICT_METHOD = arrayMethodIsStrict('reduce');\nvar USES_TO_LENGTH = arrayMethodUsesToLength('reduce', { 1: 0 });\n\n// `Array.prototype.reduce` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.reduce\n$({ target: 'Array', proto: true, forced: !STRICT_METHOD || !USES_TO_LENGTH }, {\n  reduce: function reduce(callbackfn /* , initialValue */) {\n    return $reduce(this, callbackfn, arguments.length, arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n","require('../../../modules/es.array.reduce');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').reduce;\n","var reduce = require('../array/virtual/reduce');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.reduce;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.reduce) ? reduce : own;\n};\n","'use strict';\nvar aFunction = require('../internals/a-function');\nvar isObject = require('../internals/is-object');\n\nvar slice = [].slice;\nvar factories = {};\n\nvar construct = function (C, argsLength, args) {\n  if (!(argsLength in factories)) {\n    for (var list = [], i = 0; i < argsLength; i++) list[i] = 'a[' + i + ']';\n    // eslint-disable-next-line no-new-func\n    factories[argsLength] = Function('C,a', 'return new C(' + list.join(',') + ')');\n  } return factories[argsLength](C, args);\n};\n\n// `Function.prototype.bind` method implementation\n// https://tc39.github.io/ecma262/#sec-function.prototype.bind\nmodule.exports = Function.bind || function bind(that /* , ...args */) {\n  var fn = aFunction(this);\n  var partArgs = slice.call(arguments, 1);\n  var boundFunction = function bound(/* args... */) {\n    var args = partArgs.concat(slice.call(arguments));\n    return this instanceof boundFunction ? construct(fn, args.length, args) : fn.apply(that, args);\n  };\n  if (isObject(fn.prototype)) boundFunction.prototype = fn.prototype;\n  return boundFunction;\n};\n","var $ = require('../internals/export');\nvar bind = require('../internals/function-bind');\n\n// `Function.prototype.bind` method\n// https://tc39.github.io/ecma262/#sec-function.prototype.bind\n$({ target: 'Function', proto: true }, {\n  bind: bind\n});\n","require('../../../modules/es.function.bind');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Function').bind;\n","var bind = require('../function/virtual/bind');\n\nvar FunctionPrototype = Function.prototype;\n\nmodule.exports = function (it) {\n  var own = it.bind;\n  return it === FunctionPrototype || (it instanceof Function && own === FunctionPrototype.bind) ? bind : own;\n};\n","function _classCallCheck(instance, Constructor) {\n  if (!(instance instanceof Constructor)) {\n    throw new TypeError(\"Cannot call a class as a function\");\n  }\n}\n\nmodule.exports = _classCallCheck;","var _Object$defineProperty = require(\"../core-js/object/define-property\");\n\nfunction _defineProperties(target, props) {\n  for (var i = 0; i < props.length; i++) {\n    var descriptor = props[i];\n    descriptor.enumerable = descriptor.enumerable || false;\n    descriptor.configurable = true;\n    if (\"value\" in descriptor) descriptor.writable = true;\n\n    _Object$defineProperty(target, descriptor.key, descriptor);\n  }\n}\n\nfunction _createClass(Constructor, protoProps, staticProps) {\n  if (protoProps) _defineProperties(Constructor.prototype, protoProps);\n  if (staticProps) _defineProperties(Constructor, staticProps);\n  return Constructor;\n}\n\nmodule.exports = _createClass;","/**\n * vis-data\n * http://visjs.org/\n *\n * Manage unstructured data using DataSet. Add, update, and remove data, and listen for changes in the data.\n *\n * @version 6.5.0\n * @date    2020-03-07T21:44:22.378Z\n *\n * @copyright (c) 2011-2017 Almende B.V, http://almende.com\n * @copyright (c) 2017-2019 visjs contributors, https://github.com/visjs\n *\n * @license\n * vis.js is dual licensed under both\n *\n *   1. The Apache 2.0 License\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *   and\n *\n *   2. The MIT License\n *      http://opensource.org/licenses/MIT\n *\n * vis.js may be distributed under either license.\n */\n\nimport { isNumber, isString, getType, deepExtend } from 'vis-util/esnext';\nimport moment from 'moment';\n\n/* eslint @typescript-eslint/member-ordering: [\"error\", { \"classes\": [\"field\", \"constructor\", \"method\"] }] */\r\n/**\r\n * Create new data pipe.\r\n *\r\n * @param from - The source data set or data view.\r\n *\r\n * @remarks\r\n * Example usage:\r\n * ```typescript\r\n * interface AppItem {\r\n *   whoami: string;\r\n *   appData: unknown;\r\n *   visData: VisItem;\r\n * }\r\n * interface VisItem {\r\n *   id: number;\r\n *   label: string;\r\n *   color: string;\r\n *   x: number;\r\n *   y: number;\r\n * }\r\n *\r\n * const ds1 = new DataSet<AppItem, \"whoami\">([], { fieldId: \"whoami\" });\r\n * const ds2 = new DataSet<VisItem, \"id\">();\r\n *\r\n * const pipe = createNewDataPipeFrom(ds1)\r\n *   .filter((item): boolean => item.enabled === true)\r\n *   .map<VisItem, \"id\">((item): VisItem => item.visData)\r\n *   .to(ds2);\r\n *\r\n * pipe.start();\r\n * ```\r\n *\r\n * @returns A factory whose methods can be used to configure the pipe.\r\n */\r\nfunction createNewDataPipeFrom(from) {\r\n    return new DataPipeUnderConstruction(from);\r\n}\r\n/**\r\n * Internal implementation of the pipe. This should be accessible only through\r\n * `createNewDataPipeFrom` from the outside.\r\n *\r\n * @typeparam SI - Source item type.\r\n * @typeparam SP - Source item type's id property name.\r\n * @typeparam TI - Target item type.\r\n * @typeparam TP - Target item type's id property name.\r\n */\r\nclass SimpleDataPipe {\r\n    /**\r\n     * Create a new data pipe.\r\n     *\r\n     * @param _source - The data set or data view that will be observed.\r\n     * @param _transformers - An array of transforming functions to be used to\r\n     * filter or transform the items in the pipe.\r\n     * @param _target - The data set or data view that will receive the items.\r\n     */\r\n    constructor(_source, _transformers, _target) {\r\n        this._source = _source;\r\n        this._transformers = _transformers;\r\n        this._target = _target;\r\n        /**\r\n         * Bound listeners for use with `DataInterface['on' | 'off']`.\r\n         */\r\n        this._listeners = {\r\n            add: this._add.bind(this),\r\n            remove: this._remove.bind(this),\r\n            update: this._update.bind(this)\r\n        };\r\n    }\r\n    /** @inheritdoc */\r\n    all() {\r\n        this._target.update(this._transformItems(this._source.get()));\r\n        return this;\r\n    }\r\n    /** @inheritdoc */\r\n    start() {\r\n        this._source.on(\"add\", this._listeners.add);\r\n        this._source.on(\"remove\", this._listeners.remove);\r\n        this._source.on(\"update\", this._listeners.update);\r\n        return this;\r\n    }\r\n    /** @inheritdoc */\r\n    stop() {\r\n        this._source.off(\"add\", this._listeners.add);\r\n        this._source.off(\"remove\", this._listeners.remove);\r\n        this._source.off(\"update\", this._listeners.update);\r\n        return this;\r\n    }\r\n    /**\r\n     * Apply the transformers to the items.\r\n     *\r\n     * @param items - The items to be transformed.\r\n     *\r\n     * @returns The transformed items.\r\n     */\r\n    _transformItems(items) {\r\n        return this._transformers.reduce((items, transform) => {\r\n            return transform(items);\r\n        }, items);\r\n    }\r\n    /**\r\n     * Handle an add event.\r\n     *\r\n     * @param _name - Ignored.\r\n     * @param payload - The payload containing the ids of the added items.\r\n     */\r\n    _add(_name, payload) {\r\n        if (payload == null) {\r\n            return;\r\n        }\r\n        this._target.add(this._transformItems(this._source.get(payload.items)));\r\n    }\r\n    /**\r\n     * Handle an update event.\r\n     *\r\n     * @param _name - Ignored.\r\n     * @param payload - The payload containing the ids of the updated items.\r\n     */\r\n    _update(_name, payload) {\r\n        if (payload == null) {\r\n            return;\r\n        }\r\n        this._target.update(this._transformItems(this._source.get(payload.items)));\r\n    }\r\n    /**\r\n     * Handle a remove event.\r\n     *\r\n     * @param _name - Ignored.\r\n     * @param payload - The payload containing the data of the removed items.\r\n     */\r\n    _remove(_name, payload) {\r\n        if (payload == null) {\r\n            return;\r\n        }\r\n        this._target.remove(this._transformItems(payload.oldData));\r\n    }\r\n}\r\n/**\r\n * Internal implementation of the pipe factory. This should be accessible\r\n * only through `createNewDataPipeFrom` from the outside.\r\n *\r\n * @typeparam TI - Target item type.\r\n * @typeparam TP - Target item type's id property name.\r\n */\r\nclass DataPipeUnderConstruction {\r\n    /**\r\n     * Create a new data pipe factory. This is an internal constructor that\r\n     * should never be called from outside of this file.\r\n     *\r\n     * @param _source - The source data set or data view for this pipe.\r\n     */\r\n    constructor(_source) {\r\n        this._source = _source;\r\n        /**\r\n         * Array transformers used to transform items within the pipe. This is typed\r\n         * as any for the sake of simplicity.\r\n         */\r\n        this._transformers = [];\r\n    }\r\n    /**\r\n     * Filter the items.\r\n     *\r\n     * @param callback - A filtering function that returns true if given item\r\n     * should be piped and false if not.\r\n     *\r\n     * @returns This factory for further configuration.\r\n     */\r\n    filter(callback) {\r\n        this._transformers.push((input) => input.filter(callback));\r\n        return this;\r\n    }\r\n    /**\r\n     * Map each source item to a new type.\r\n     *\r\n     * @param callback - A mapping function that takes a source item and returns\r\n     * corresponding mapped item.\r\n     *\r\n     * @typeparam TI - Target item type.\r\n     * @typeparam TP - Target item type's id property name.\r\n     *\r\n     * @returns This factory for further configuration.\r\n     */\r\n    map(callback) {\r\n        this._transformers.push((input) => input.map(callback));\r\n        return this;\r\n    }\r\n    /**\r\n     * Map each source item to zero or more items of a new type.\r\n     *\r\n     * @param callback - A mapping function that takes a source item and returns\r\n     * an array of corresponding mapped items.\r\n     *\r\n     * @typeparam TI - Target item type.\r\n     * @typeparam TP - Target item type's id property name.\r\n     *\r\n     * @returns This factory for further configuration.\r\n     */\r\n    flatMap(callback) {\r\n        this._transformers.push((input) => input.flatMap(callback));\r\n        return this;\r\n    }\r\n    /**\r\n     * Connect this pipe to given data set.\r\n     *\r\n     * @param target - The data set that will receive the items from this pipe.\r\n     *\r\n     * @returns The pipe connected between given data sets and performing\r\n     * configured transformation on the processed items.\r\n     */\r\n    to(target) {\r\n        return new SimpleDataPipe(this._source, this._transformers, target);\r\n    }\r\n}\n\nfunction createCommonjsModule(fn, module) {\n\treturn module = { exports: {} }, fn(module, module.exports), module.exports;\n}\n\nvar rngBrowser = createCommonjsModule(function (module) {\n// Unique ID creation requires a high quality random # generator.  In the\n// browser this is a little complicated due to unknown quality of Math.random()\n// and inconsistent support for the `crypto` API.  We do the best we can via\n// feature-detection\n\n// getRandomValues needs to be invoked in a context where \"this\" is a Crypto\n// implementation. Also, find the complete implementation of crypto on IE11.\nvar getRandomValues = (typeof(crypto) != 'undefined' && crypto.getRandomValues && crypto.getRandomValues.bind(crypto)) ||\n                      (typeof(msCrypto) != 'undefined' && typeof window.msCrypto.getRandomValues == 'function' && msCrypto.getRandomValues.bind(msCrypto));\n\nif (getRandomValues) {\n  // WHATWG crypto RNG - http://wiki.whatwg.org/wiki/Crypto\n  var rnds8 = new Uint8Array(16); // eslint-disable-line no-undef\n\n  module.exports = function whatwgRNG() {\n    getRandomValues(rnds8);\n    return rnds8;\n  };\n} else {\n  // Math.random()-based (RNG)\n  //\n  // If all else fails, use Math.random().  It's fast, but is of unspecified\n  // quality.\n  var rnds = new Array(16);\n\n  module.exports = function mathRNG() {\n    for (var i = 0, r; i < 16; i++) {\n      if ((i & 0x03) === 0) r = Math.random() * 0x100000000;\n      rnds[i] = r >>> ((i & 0x03) << 3) & 0xff;\n    }\n\n    return rnds;\n  };\n}\n});\n\n/**\n * Convert array of 16 byte values to UUID string format of the form:\n * XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX\n */\nvar byteToHex = [];\nfor (var i = 0; i < 256; ++i) {\n  byteToHex[i] = (i + 0x100).toString(16).substr(1);\n}\n\nfunction bytesToUuid(buf, offset) {\n  var i = offset || 0;\n  var bth = byteToHex;\n  // join used to fix memory issue caused by concatenation: https://bugs.chromium.org/p/v8/issues/detail?id=3175#c4\n  return ([\n    bth[buf[i++]], bth[buf[i++]],\n    bth[buf[i++]], bth[buf[i++]], '-',\n    bth[buf[i++]], bth[buf[i++]], '-',\n    bth[buf[i++]], bth[buf[i++]], '-',\n    bth[buf[i++]], bth[buf[i++]], '-',\n    bth[buf[i++]], bth[buf[i++]],\n    bth[buf[i++]], bth[buf[i++]],\n    bth[buf[i++]], bth[buf[i++]]\n  ]).join('');\n}\n\nvar bytesToUuid_1 = bytesToUuid;\n\nfunction v4(options, buf, offset) {\n  var i = buf && offset || 0;\n\n  if (typeof(options) == 'string') {\n    buf = options === 'binary' ? new Array(16) : null;\n    options = null;\n  }\n  options = options || {};\n\n  var rnds = options.random || (options.rng || rngBrowser)();\n\n  // Per 4.4, set bits for version and `clock_seq_hi_and_reserved`\n  rnds[6] = (rnds[6] & 0x0f) | 0x40;\n  rnds[8] = (rnds[8] & 0x3f) | 0x80;\n\n  // Copy bytes to buffer, if provided\n  if (buf) {\n    for (var ii = 0; ii < 16; ++ii) {\n      buf[i + ii] = rnds[ii];\n    }\n  }\n\n  return buf || bytesToUuid_1(rnds);\n}\n\nvar v4_1 = v4;\n\n// utility functions\r\n// parse ASP.Net Date pattern,\r\n// for example '/Date(1198908717056)/' or '/Date(1198908717056-0700)/'\r\n// code from http://momentjs.com/\r\nconst ASPDateRegex = /^\\/?Date\\((-?\\d+)/i;\r\n/**\r\n * Test whether given object is a Moment date.\r\n *\r\n * @param value - Input value of unknown type.\r\n *\r\n * @returns True if Moment instance, false otherwise.\r\n */\r\nfunction isMoment(value) {\r\n    return moment.isMoment(value);\r\n}\r\n/**\r\n * Convert an object into another type\r\n *\r\n * @param object - Value of unknown type.\r\n * @param type - Name of the desired type.\r\n *\r\n * @returns Object in the desired type.\r\n * @throws Error\r\n */\r\nfunction convert(object, type) {\r\n    let match;\r\n    if (object === undefined) {\r\n        return undefined;\r\n    }\r\n    if (object === null) {\r\n        return null;\r\n    }\r\n    if (!type) {\r\n        return object;\r\n    }\r\n    if (!(typeof type === \"string\") && !(type instanceof String)) {\r\n        throw new Error(\"Type must be a string\");\r\n    }\r\n    //noinspection FallthroughInSwitchStatementJS\r\n    switch (type) {\r\n        case \"boolean\":\r\n        case \"Boolean\":\r\n            return Boolean(object);\r\n        case \"number\":\r\n        case \"Number\":\r\n            if (isString(object) && !isNaN(Date.parse(object))) {\r\n                return moment(object).valueOf();\r\n            }\r\n            else {\r\n                // @TODO: I don't think that Number and String constructors are a good idea.\r\n                // This could also fail if the object doesn't have valueOf method or if it's redefined.\r\n                // For example: Object.create(null) or { valueOf: 7 }.\r\n                return Number(object.valueOf());\r\n            }\r\n        case \"string\":\r\n        case \"String\":\r\n            return String(object);\r\n        case \"Date\":\r\n            if (isNumber(object)) {\r\n                return new Date(object);\r\n            }\r\n            if (object instanceof Date) {\r\n                return new Date(object.valueOf());\r\n            }\r\n            else if (isMoment(object)) {\r\n                return new Date(object.valueOf());\r\n            }\r\n            if (isString(object)) {\r\n                match = ASPDateRegex.exec(object);\r\n                if (match) {\r\n                    // object is an ASP date\r\n                    return new Date(Number(match[1])); // parse number\r\n                }\r\n                else {\r\n                    return moment(new Date(object)).toDate(); // parse string\r\n                }\r\n            }\r\n            else {\r\n                throw new Error(\"Cannot convert object of type \" + getType(object) + \" to type Date\");\r\n            }\r\n        case \"Moment\":\r\n            if (isNumber(object)) {\r\n                return moment(object);\r\n            }\r\n            if (object instanceof Date) {\r\n                return moment(object.valueOf());\r\n            }\r\n            else if (isMoment(object)) {\r\n                return moment(object);\r\n            }\r\n            if (isString(object)) {\r\n                match = ASPDateRegex.exec(object);\r\n                if (match) {\r\n                    // object is an ASP date\r\n                    return moment(Number(match[1])); // parse number\r\n                }\r\n                else {\r\n                    return moment(object); // parse string\r\n                }\r\n            }\r\n            else {\r\n                throw new Error(\"Cannot convert object of type \" + getType(object) + \" to type Date\");\r\n            }\r\n        case \"ISODate\":\r\n            if (isNumber(object)) {\r\n                return new Date(object);\r\n            }\r\n            else if (object instanceof Date) {\r\n                return object.toISOString();\r\n            }\r\n            else if (isMoment(object)) {\r\n                return object.toDate().toISOString();\r\n            }\r\n            else if (isString(object)) {\r\n                match = ASPDateRegex.exec(object);\r\n                if (match) {\r\n                    // object is an ASP date\r\n                    return new Date(Number(match[1])).toISOString(); // parse number\r\n                }\r\n                else {\r\n                    return moment(object).format(); // ISO 8601\r\n                }\r\n            }\r\n            else {\r\n                throw new Error(\"Cannot convert object of type \" +\r\n                    getType(object) +\r\n                    \" to type ISODate\");\r\n            }\r\n        case \"ASPDate\":\r\n            if (isNumber(object)) {\r\n                return \"/Date(\" + object + \")/\";\r\n            }\r\n            else if (object instanceof Date || isMoment(object)) {\r\n                return \"/Date(\" + object.valueOf() + \")/\";\r\n            }\r\n            else if (isString(object)) {\r\n                match = ASPDateRegex.exec(object);\r\n                let value;\r\n                if (match) {\r\n                    // object is an ASP date\r\n                    value = new Date(Number(match[1])).valueOf(); // parse number\r\n                }\r\n                else {\r\n                    value = new Date(object).valueOf(); // parse string\r\n                }\r\n                return \"/Date(\" + value + \")/\";\r\n            }\r\n            else {\r\n                throw new Error(\"Cannot convert object of type \" +\r\n                    getType(object) +\r\n                    \" to type ASPDate\");\r\n            }\r\n        default:\r\n            const never = type;\r\n            throw new Error(`Unknown type ${never}`);\r\n    }\r\n}\n\n/**\r\n * Determine whether a value can be used as an id.\r\n *\r\n * @param value - Input value of unknown type.\r\n *\r\n * @returns True if the value is valid id, false otherwise.\r\n */\r\nfunction isId(value) {\r\n    return typeof value === \"string\" || typeof value === \"number\";\r\n}\n\n/* eslint @typescript-eslint/member-ordering: [\"error\", { \"classes\": [\"field\", \"constructor\", \"method\"] }] */\r\n/**\r\n * A queue.\r\n *\r\n * @typeParam T - The type of method names to be replaced by queued versions.\r\n */\r\nclass Queue {\r\n    /**\r\n     * Construct a new Queue.\r\n     *\r\n     * @param options - Queue configuration.\r\n     */\r\n    constructor(options) {\r\n        this._queue = [];\r\n        this._timeout = null;\r\n        this._extended = null;\r\n        // options\r\n        this.delay = null;\r\n        this.max = Infinity;\r\n        this.setOptions(options);\r\n    }\r\n    /**\r\n     * Update the configuration of the queue.\r\n     *\r\n     * @param options - Queue configuration.\r\n     */\r\n    setOptions(options) {\r\n        if (options && typeof options.delay !== \"undefined\") {\r\n            this.delay = options.delay;\r\n        }\r\n        if (options && typeof options.max !== \"undefined\") {\r\n            this.max = options.max;\r\n        }\r\n        this._flushIfNeeded();\r\n    }\r\n    /**\r\n     * Extend an object with queuing functionality.\r\n     * The object will be extended with a function flush, and the methods provided in options.replace will be replaced with queued ones.\r\n     *\r\n     * @param object - The object to be extended.\r\n     * @param options - Additional options.\r\n     *\r\n     * @returns The created queue.\r\n     */\r\n    static extend(object, options) {\r\n        const queue = new Queue(options);\r\n        if (object.flush !== undefined) {\r\n            throw new Error(\"Target object already has a property flush\");\r\n        }\r\n        object.flush = () => {\r\n            queue.flush();\r\n        };\r\n        const methods = [\r\n            {\r\n                name: \"flush\",\r\n                original: undefined\r\n            }\r\n        ];\r\n        if (options && options.replace) {\r\n            for (let i = 0; i < options.replace.length; i++) {\r\n                const name = options.replace[i];\r\n                methods.push({\r\n                    name: name,\r\n                    // @TODO: better solution?\r\n                    original: object[name]\r\n                });\r\n                // @TODO: better solution?\r\n                queue.replace(object, name);\r\n            }\r\n        }\r\n        queue._extended = {\r\n            object: object,\r\n            methods: methods\r\n        };\r\n        return queue;\r\n    }\r\n    /**\r\n     * Destroy the queue. The queue will first flush all queued actions, and in case it has extended an object, will restore the original object.\r\n     */\r\n    destroy() {\r\n        this.flush();\r\n        if (this._extended) {\r\n            const object = this._extended.object;\r\n            const methods = this._extended.methods;\r\n            for (let i = 0; i < methods.length; i++) {\r\n                const method = methods[i];\r\n                if (method.original) {\r\n                    // @TODO: better solution?\r\n                    object[method.name] = method.original;\r\n                }\r\n                else {\r\n                    // @TODO: better solution?\r\n                    delete object[method.name];\r\n                }\r\n            }\r\n            this._extended = null;\r\n        }\r\n    }\r\n    /**\r\n     * Replace a method on an object with a queued version.\r\n     *\r\n     * @param object - Object having the method.\r\n     * @param method - The method name.\r\n     */\r\n    replace(object, method) {\r\n        /* eslint-disable-next-line @typescript-eslint/no-this-alias */\r\n        const me = this;\r\n        const original = object[method];\r\n        if (!original) {\r\n            throw new Error(\"Method \" + method + \" undefined\");\r\n        }\r\n        object[method] = function (...args) {\r\n            // add this call to the queue\r\n            me.queue({\r\n                args: args,\r\n                fn: original,\r\n                context: this\r\n            });\r\n        };\r\n    }\r\n    /**\r\n     * Queue a call.\r\n     *\r\n     * @param entry - The function or entry to be queued.\r\n     */\r\n    queue(entry) {\r\n        if (typeof entry === \"function\") {\r\n            this._queue.push({ fn: entry });\r\n        }\r\n        else {\r\n            this._queue.push(entry);\r\n        }\r\n        this._flushIfNeeded();\r\n    }\r\n    /**\r\n     * Check whether the queue needs to be flushed.\r\n     */\r\n    _flushIfNeeded() {\r\n        // flush when the maximum is exceeded.\r\n        if (this._queue.length > this.max) {\r\n            this.flush();\r\n        }\r\n        // flush after a period of inactivity when a delay is configured\r\n        if (this._timeout != null) {\r\n            clearTimeout(this._timeout);\r\n            this._timeout = null;\r\n        }\r\n        if (this.queue.length > 0 && typeof this.delay === \"number\") {\r\n            this._timeout = setTimeout(() => {\r\n                this.flush();\r\n            }, this.delay);\r\n        }\r\n    }\r\n    /**\r\n     * Flush all queued calls\r\n     */\r\n    flush() {\r\n        this._queue.splice(0).forEach((entry) => {\r\n            entry.fn.apply(entry.context || entry.fn, entry.args || []);\r\n        });\r\n    }\r\n}\n\n/* eslint-disable @typescript-eslint/member-ordering */\r\n/**\r\n * [[DataSet]] code that can be reused in [[DataView]] or other similar implementations of [[DataInterface]].\r\n *\r\n * @typeParam Item - Item type that may or may not have an id.\r\n * @typeParam IdProp - Name of the property that contains the id.\r\n */\r\nclass DataSetPart {\r\n    constructor() {\r\n        this._subscribers = {\r\n            \"*\": [],\r\n            add: [],\r\n            remove: [],\r\n            update: []\r\n        };\r\n        /**\r\n         * @deprecated Use on instead (PS: DataView.subscribe === DataView.on).\r\n         */\r\n        this.subscribe = DataSetPart.prototype.on;\r\n        /**\r\n         * @deprecated Use off instead (PS: DataView.unsubscribe === DataView.off).\r\n         */\r\n        this.unsubscribe = DataSetPart.prototype.off;\r\n    }\r\n    /**\r\n     * Trigger an event\r\n     *\r\n     * @param event - Event name.\r\n     * @param payload - Event payload.\r\n     * @param senderId - Id of the sender.\r\n     */\r\n    _trigger(event, payload, senderId) {\r\n        if (event === \"*\") {\r\n            throw new Error(\"Cannot trigger event *\");\r\n        }\r\n        [...this._subscribers[event], ...this._subscribers[\"*\"]].forEach((subscriber) => {\r\n            subscriber(event, payload, senderId != null ? senderId : null);\r\n        });\r\n    }\r\n    /**\r\n     * Subscribe to an event, add an event listener.\r\n     *\r\n     * @remarks Non-function callbacks are ignored.\r\n     *\r\n     * @param event - Event name.\r\n     * @param callback - Callback method.\r\n     */\r\n    on(event, callback) {\r\n        if (typeof callback === \"function\") {\r\n            this._subscribers[event].push(callback);\r\n        }\r\n        // @TODO: Maybe throw for invalid callbacks?\r\n    }\r\n    /**\r\n     * Unsubscribe from an event, remove an event listener.\r\n     *\r\n     * @remarks If the same callback was subscribed more than once **all** occurences will be removed.\r\n     *\r\n     * @param event - Event name.\r\n     * @param callback - Callback method.\r\n     */\r\n    off(event, callback) {\r\n        this._subscribers[event] = this._subscribers[event].filter((subscriber) => subscriber !== callback);\r\n    }\r\n}\n\n/**\r\n * Data stream\r\n *\r\n * @remarks\r\n * [[DataStream]] offers an always up to date stream of items from a [[DataSet]] or [[DataView]].\r\n * That means that the stream is evaluated at the time of iteration, conversion to another data type or when [[cache]] is called, not when the [[DataStream]] was created.\r\n * Multiple invocations of for example [[toItemArray]] may yield different results (if the data source like for example [[DataSet]] gets modified).\r\n *\r\n * @typeparam Item - The item type this stream is going to work with.\r\n */\r\nclass DataStream {\r\n    /**\r\n     * Create a new data stream.\r\n     *\r\n     * @param _pairs - The id, item pairs.\r\n     */\r\n    constructor(_pairs) {\r\n        this._pairs = _pairs;\r\n    }\r\n    /**\r\n     * Return an iterable of key, value pairs for every entry in the stream.\r\n     */\r\n    *[Symbol.iterator]() {\r\n        for (const [id, item] of this._pairs) {\r\n            yield [id, item];\r\n        }\r\n    }\r\n    /**\r\n     * Return an iterable of key, value pairs for every entry in the stream.\r\n     */\r\n    *entries() {\r\n        for (const [id, item] of this._pairs) {\r\n            yield [id, item];\r\n        }\r\n    }\r\n    /**\r\n     * Return an iterable of keys in the stream.\r\n     */\r\n    *keys() {\r\n        for (const [id] of this._pairs) {\r\n            yield id;\r\n        }\r\n    }\r\n    /**\r\n     * Return an iterable of values in the stream.\r\n     */\r\n    *values() {\r\n        for (const [, item] of this._pairs) {\r\n            yield item;\r\n        }\r\n    }\r\n    /**\r\n     * Return an array containing all the ids in this stream.\r\n     *\r\n     * @remarks\r\n     * The array may contain duplicities.\r\n     *\r\n     * @returns The array with all ids from this stream.\r\n     */\r\n    toIdArray() {\r\n        return [...this._pairs].map((pair) => pair[0]);\r\n    }\r\n    /**\r\n     * Return an array containing all the items in this stream.\r\n     *\r\n     * @remarks\r\n     * The array may contain duplicities.\r\n     *\r\n     * @returns The array with all items from this stream.\r\n     */\r\n    toItemArray() {\r\n        return [...this._pairs].map((pair) => pair[1]);\r\n    }\r\n    /**\r\n     * Return an array containing all the entries in this stream.\r\n     *\r\n     * @remarks\r\n     * The array may contain duplicities.\r\n     *\r\n     * @returns The array with all entries from this stream.\r\n     */\r\n    toEntryArray() {\r\n        return [...this._pairs];\r\n    }\r\n    /**\r\n     * Return an object map containing all the items in this stream accessible by ids.\r\n     *\r\n     * @remarks\r\n     * In case of duplicate ids (coerced to string so `7 == '7'`) the last encoutered appears in the returned object.\r\n     *\r\n     * @returns The object map of all id → item pairs from this stream.\r\n     */\r\n    toObjectMap() {\r\n        const map = Object.create(null);\r\n        for (const [id, item] of this._pairs) {\r\n            map[id] = item;\r\n        }\r\n        return map;\r\n    }\r\n    /**\r\n     * Return a map containing all the items in this stream accessible by ids.\r\n     *\r\n     * @returns The map of all id → item pairs from this stream.\r\n     */\r\n    toMap() {\r\n        return new Map(this._pairs);\r\n    }\r\n    /**\r\n     * Return a set containing all the (unique) ids in this stream.\r\n     *\r\n     * @returns The set of all ids from this stream.\r\n     */\r\n    toIdSet() {\r\n        return new Set(this.toIdArray());\r\n    }\r\n    /**\r\n     * Return a set containing all the (unique) items in this stream.\r\n     *\r\n     * @returns The set of all items from this stream.\r\n     */\r\n    toItemSet() {\r\n        return new Set(this.toItemArray());\r\n    }\r\n    /**\r\n     * Cache the items from this stream.\r\n     *\r\n     * @remarks\r\n     * This method allows for items to be fetched immediatelly and used (possibly multiple times) later.\r\n     * It can also be used to optimize performance as [[DataStream]] would otherwise reevaluate everything upon each iteration.\r\n     *\r\n     * ## Example\r\n     * ```javascript\r\n     * const ds = new DataSet([…])\r\n     *\r\n     * const cachedStream = ds.stream()\r\n     *   .filter(…)\r\n     *   .sort(…)\r\n     *   .map(…)\r\n     *   .cached(…) // Data are fetched, processed and cached here.\r\n     *\r\n     * ds.clear()\r\n     * chachedStream // Still has all the items.\r\n     * ```\r\n     *\r\n     * @returns A new [[DataStream]] with cached items (detached from the original [[DataSet]]).\r\n     */\r\n    cache() {\r\n        return new DataStream([...this._pairs]);\r\n    }\r\n    /**\r\n     * Get the distinct values of given property.\r\n     *\r\n     * @param callback - The function that picks and possibly converts the property.\r\n     *\r\n     * @typeparam T - The type of the distinct value.\r\n     *\r\n     * @returns A set of all distinct properties.\r\n     */\r\n    distinct(callback) {\r\n        const set = new Set();\r\n        for (const [id, item] of this._pairs) {\r\n            set.add(callback(item, id));\r\n        }\r\n        return set;\r\n    }\r\n    /**\r\n     * Filter the items of the stream.\r\n     *\r\n     * @param callback - The function that decides whether an item will be included.\r\n     *\r\n     * @returns A new data stream with the filtered items.\r\n     */\r\n    filter(callback) {\r\n        const pairs = this._pairs;\r\n        return new DataStream({\r\n            *[Symbol.iterator]() {\r\n                for (const [id, item] of pairs) {\r\n                    if (callback(item, id)) {\r\n                        yield [id, item];\r\n                    }\r\n                }\r\n            }\r\n        });\r\n    }\r\n    /**\r\n     * Execute a callback for each item of the stream.\r\n     *\r\n     * @param callback - The function that will be invoked for each item.\r\n     */\r\n    forEach(callback) {\r\n        for (const [id, item] of this._pairs) {\r\n            callback(item, id);\r\n        }\r\n    }\r\n    /**\r\n     * Map the items into a different type.\r\n     *\r\n     * @param callback - The function that does the conversion.\r\n     *\r\n     * @typeparam Mapped - The type of the item after mapping.\r\n     *\r\n     * @returns A new data stream with the mapped items.\r\n     */\r\n    map(callback) {\r\n        const pairs = this._pairs;\r\n        return new DataStream({\r\n            *[Symbol.iterator]() {\r\n                for (const [id, item] of pairs) {\r\n                    yield [id, callback(item, id)];\r\n                }\r\n            }\r\n        });\r\n    }\r\n    /**\r\n     * Get the item with the maximum value of given property.\r\n     *\r\n     * @param callback - The function that picks and possibly converts the property.\r\n     *\r\n     * @returns The item with the maximum if found otherwise null.\r\n     */\r\n    max(callback) {\r\n        const iter = this._pairs[Symbol.iterator]();\r\n        let curr = iter.next();\r\n        if (curr.done) {\r\n            return null;\r\n        }\r\n        let maxItem = curr.value[1];\r\n        let maxValue = callback(curr.value[1], curr.value[0]);\r\n        while (!(curr = iter.next()).done) {\r\n            const [id, item] = curr.value;\r\n            const value = callback(item, id);\r\n            if (value > maxValue) {\r\n                maxValue = value;\r\n                maxItem = item;\r\n            }\r\n        }\r\n        return maxItem;\r\n    }\r\n    /**\r\n     * Get the item with the minimum value of given property.\r\n     *\r\n     * @param callback - The function that picks and possibly converts the property.\r\n     *\r\n     * @returns The item with the minimum if found otherwise null.\r\n     */\r\n    min(callback) {\r\n        const iter = this._pairs[Symbol.iterator]();\r\n        let curr = iter.next();\r\n        if (curr.done) {\r\n            return null;\r\n        }\r\n        let minItem = curr.value[1];\r\n        let minValue = callback(curr.value[1], curr.value[0]);\r\n        while (!(curr = iter.next()).done) {\r\n            const [id, item] = curr.value;\r\n            const value = callback(item, id);\r\n            if (value < minValue) {\r\n                minValue = value;\r\n                minItem = item;\r\n            }\r\n        }\r\n        return minItem;\r\n    }\r\n    /**\r\n     * Reduce the items into a single value.\r\n     *\r\n     * @param callback - The function that does the reduction.\r\n     * @param accumulator - The initial value of the accumulator.\r\n     *\r\n     * @typeparam T - The type of the accumulated value.\r\n     *\r\n     * @returns The reduced value.\r\n     */\r\n    reduce(callback, accumulator) {\r\n        for (const [id, item] of this._pairs) {\r\n            accumulator = callback(accumulator, item, id);\r\n        }\r\n        return accumulator;\r\n    }\r\n    /**\r\n     * Sort the items.\r\n     *\r\n     * @param callback - Item comparator.\r\n     *\r\n     * @returns A new stream with sorted items.\r\n     */\r\n    sort(callback) {\r\n        return new DataStream({\r\n            [Symbol.iterator]: () => [...this._pairs]\r\n                .sort(([idA, itemA], [idB, itemB]) => callback(itemA, itemB, idA, idB))[Symbol.iterator]()\r\n        });\r\n    }\r\n}\n\n/* eslint @typescript-eslint/member-ordering: [\"error\", { \"classes\": [\"field\", \"constructor\", \"method\"] }] */\r\nconst warnTypeCorectionDeprecation = () => {\r\n    console.warn(\"Type coercion has been deprecated. \" +\r\n        \"Please, use data pipes instead. \" +\r\n        \"See https://visjs.github.io/vis-data/data/datapipe.html#TypeCoercion for more details with working migration example.\");\r\n};\r\n/**\r\n * # DataSet\r\n *\r\n * Vis.js comes with a flexible DataSet, which can be used to hold and manipulate unstructured data and listen for changes in the data. The DataSet is key/value based. Data items can be added, updated and removed from the DataSet, and one can subscribe to changes in the DataSet. The data in the DataSet can be filtered and ordered, and fields (like dates) can be converted to a specific type. Data can be normalized when appending it to the DataSet as well.\r\n *\r\n * ## Example\r\n *\r\n * The following example shows how to use a DataSet.\r\n *\r\n * ```javascript\r\n * // create a DataSet\r\n * var options = {};\r\n * var data = new vis.DataSet(options);\r\n *\r\n * // add items\r\n * // note that the data items can contain different properties and data formats\r\n * data.add([\r\n *   {id: 1, text: 'item 1', date: new Date(2013, 6, 20), group: 1, first: true},\r\n *   {id: 2, text: 'item 2', date: '2013-06-23', group: 2},\r\n *   {id: 3, text: 'item 3', date: '2013-06-25', group: 2},\r\n *   {id: 4, text: 'item 4'}\r\n * ]);\r\n *\r\n * // subscribe to any change in the DataSet\r\n * data.on('*', function (event, properties, senderId) {\r\n *   console.log('event', event, properties);\r\n * });\r\n *\r\n * // update an existing item\r\n * data.update({id: 2, group: 1});\r\n *\r\n * // remove an item\r\n * data.remove(4);\r\n *\r\n * // get all ids\r\n * var ids = data.getIds();\r\n * console.log('ids', ids);\r\n *\r\n * // get a specific item\r\n * var item1 = data.get(1);\r\n * console.log('item1', item1);\r\n *\r\n * // retrieve a filtered subset of the data\r\n * var items = data.get({\r\n *   filter: function (item) {\r\n *     return item.group == 1;\r\n *   }\r\n * });\r\n * console.log('filtered items', items);\r\n * ```\r\n *\r\n * @typeParam Item - Item type that may or may not have an id.\r\n * @typeParam IdProp - Name of the property that contains the id.\r\n */\r\nclass DataSet extends DataSetPart {\r\n    /**\r\n     * Construct a new DataSet.\r\n     *\r\n     * @param data - Initial data or options.\r\n     * @param options - Options (type error if data is also options).\r\n     */\r\n    constructor(data, options) {\r\n        super();\r\n        // correctly read optional arguments\r\n        if (data && !Array.isArray(data)) {\r\n            options = data;\r\n            data = [];\r\n        }\r\n        this._options = options || {};\r\n        this._data = new Map(); // map with data indexed by id\r\n        this.length = 0; // number of items in the DataSet\r\n        this._idProp = this._options.fieldId || \"id\"; // name of the field containing id\r\n        this._type = {}; // internal field types (NOTE: this can differ from this._options.type)\r\n        // all variants of a Date are internally stored as Date, so we can convert\r\n        // from everything to everything (also from ISODate to Number for example)\r\n        if (this._options.type) {\r\n            warnTypeCorectionDeprecation();\r\n            const fields = Object.keys(this._options.type);\r\n            for (let i = 0, len = fields.length; i < len; i++) {\r\n                const field = fields[i];\r\n                const value = this._options.type[field];\r\n                if (value == \"Date\" || value == \"ISODate\" || value == \"ASPDate\") {\r\n                    this._type[field] = \"Date\";\r\n                }\r\n                else {\r\n                    this._type[field] = value;\r\n                }\r\n            }\r\n        }\r\n        // add initial data when provided\r\n        if (data && data.length) {\r\n            this.add(data);\r\n        }\r\n        this.setOptions(options);\r\n    }\r\n    /**\r\n     * Set new options.\r\n     *\r\n     * @param options - The new options.\r\n     */\r\n    setOptions(options) {\r\n        if (options && options.queue !== undefined) {\r\n            if (options.queue === false) {\r\n                // delete queue if loaded\r\n                if (this._queue) {\r\n                    this._queue.destroy();\r\n                    delete this._queue;\r\n                }\r\n            }\r\n            else {\r\n                // create queue and update its options\r\n                if (!this._queue) {\r\n                    this._queue = Queue.extend(this, {\r\n                        replace: [\"add\", \"update\", \"remove\"]\r\n                    });\r\n                }\r\n                if (options.queue && typeof options.queue === \"object\") {\r\n                    this._queue.setOptions(options.queue);\r\n                }\r\n            }\r\n        }\r\n    }\r\n    /**\r\n     * Add a data item or an array with items.\r\n     *\r\n     * After the items are added to the DataSet, the DataSet will trigger an event `add`. When a `senderId` is provided, this id will be passed with the triggered event to all subscribers.\r\n     *\r\n     * ## Example\r\n     *\r\n     * ```javascript\r\n     * // create a DataSet\r\n     * const data = new vis.DataSet()\r\n     *\r\n     * // add items\r\n     * const ids = data.add([\r\n     *   { id: 1, text: 'item 1' },\r\n     *   { id: 2, text: 'item 2' },\r\n     *   { text: 'item without an id' }\r\n     * ])\r\n     *\r\n     * console.log(ids) // [1, 2, '<UUIDv4>']\r\n     * ```\r\n     *\r\n     * @param data - Items to be added (ids will be generated if missing).\r\n     * @param senderId - Sender id.\r\n     *\r\n     * @returns addedIds - Array with the ids (generated if not present) of the added items.\r\n     *\r\n     * @throws When an item with the same id as any of the added items already exists.\r\n     */\r\n    add(data, senderId) {\r\n        const addedIds = [];\r\n        let id;\r\n        if (Array.isArray(data)) {\r\n            // Array\r\n            const idsToAdd = data.map(d => d[this._idProp]);\r\n            if (idsToAdd.some(id => this._data.has(id))) {\r\n                throw new Error(\"A duplicate id was found in the parameter array.\");\r\n            }\r\n            for (let i = 0, len = data.length; i < len; i++) {\r\n                id = this._addItem(data[i]);\r\n                addedIds.push(id);\r\n            }\r\n        }\r\n        else if (data && typeof data === \"object\") {\r\n            // Single item\r\n            id = this._addItem(data);\r\n            addedIds.push(id);\r\n        }\r\n        else {\r\n            throw new Error(\"Unknown dataType\");\r\n        }\r\n        if (addedIds.length) {\r\n            this._trigger(\"add\", { items: addedIds }, senderId);\r\n        }\r\n        return addedIds;\r\n    }\r\n    /**\r\n     * Update existing items. When an item does not exist, it will be created.\r\n     *\r\n     * @remarks\r\n     * The provided properties will be merged in the existing item. When an item does not exist, it will be created.\r\n     *\r\n     * After the items are updated, the DataSet will trigger an event `add` for the added items, and an event `update`. When a `senderId` is provided, this id will be passed with the triggered event to all subscribers.\r\n     *\r\n     * ## Example\r\n     *\r\n     * ```javascript\r\n     * // create a DataSet\r\n     * const data = new vis.DataSet([\r\n     *   { id: 1, text: 'item 1' },\r\n     *   { id: 2, text: 'item 2' },\r\n     *   { id: 3, text: 'item 3' }\r\n     * ])\r\n     *\r\n     * // update items\r\n     * const ids = data.update([\r\n     *   { id: 2, text: 'item 2 (updated)' },\r\n     *   { id: 4, text: 'item 4 (new)' }\r\n     * ])\r\n     *\r\n     * console.log(ids) // [2, 4]\r\n     * ```\r\n     *\r\n     * ## Warning for TypeScript users\r\n     * This method may introduce partial items into the data set. Use add or updateOnly instead for better type safety.\r\n     *\r\n     * @param data - Items to be updated (if the id is already present) or added (if the id is missing).\r\n     * @param senderId - Sender id.\r\n     *\r\n     * @returns updatedIds - The ids of the added (these may be newly generated if there was no id in the item from the data) or updated items.\r\n     *\r\n     * @throws When the supplied data is neither an item nor an array of items.\r\n     */\r\n    update(data, senderId) {\r\n        const addedIds = [];\r\n        const updatedIds = [];\r\n        const oldData = [];\r\n        const updatedData = [];\r\n        const idProp = this._idProp;\r\n        const addOrUpdate = (item) => {\r\n            const origId = item[idProp];\r\n            if (origId != null && this._data.has(origId)) {\r\n                const fullItem = item; // it has an id, therefore it is a fullitem\r\n                const oldItem = Object.assign({}, this._data.get(origId));\r\n                // update item\r\n                const id = this._updateItem(fullItem);\r\n                updatedIds.push(id);\r\n                updatedData.push(fullItem);\r\n                oldData.push(oldItem);\r\n            }\r\n            else {\r\n                // add new item\r\n                const id = this._addItem(item);\r\n                addedIds.push(id);\r\n            }\r\n        };\r\n        if (Array.isArray(data)) {\r\n            // Array\r\n            for (let i = 0, len = data.length; i < len; i++) {\r\n                if (data[i] && typeof data[i] === \"object\") {\r\n                    addOrUpdate(data[i]);\r\n                }\r\n                else {\r\n                    console.warn(\"Ignoring input item, which is not an object at index \" + i);\r\n                }\r\n            }\r\n        }\r\n        else if (data && typeof data === \"object\") {\r\n            // Single item\r\n            addOrUpdate(data);\r\n        }\r\n        else {\r\n            throw new Error(\"Unknown dataType\");\r\n        }\r\n        if (addedIds.length) {\r\n            this._trigger(\"add\", { items: addedIds }, senderId);\r\n        }\r\n        if (updatedIds.length) {\r\n            const props = { items: updatedIds, oldData: oldData, data: updatedData };\r\n            // TODO: remove deprecated property 'data' some day\r\n            //Object.defineProperty(props, 'data', {\r\n            //  'get': (function() {\r\n            //    console.warn('Property data is deprecated. Use DataSet.get(ids) to retrieve the new data, use the oldData property on this object to get the old data');\r\n            //    return updatedData;\r\n            //  }).bind(this)\r\n            //});\r\n            this._trigger(\"update\", props, senderId);\r\n        }\r\n        return addedIds.concat(updatedIds);\r\n    }\r\n    /**\r\n     * Update existing items. When an item does not exist, an error will be thrown.\r\n     *\r\n     * @remarks\r\n     * The provided properties will be deeply merged into the existing item.\r\n     * When an item does not exist (id not present in the data set or absent), an error will be thrown and nothing will be changed.\r\n     *\r\n     * After the items are updated, the DataSet will trigger an event `update`.\r\n     * When a `senderId` is provided, this id will be passed with the triggered event to all subscribers.\r\n     *\r\n     * ## Example\r\n     *\r\n     * ```javascript\r\n     * // create a DataSet\r\n     * const data = new vis.DataSet([\r\n     *   { id: 1, text: 'item 1' },\r\n     *   { id: 2, text: 'item 2' },\r\n     *   { id: 3, text: 'item 3' },\r\n     * ])\r\n     *\r\n     * // update items\r\n     * const ids = data.update([\r\n     *   { id: 2, text: 'item 2 (updated)' }, // works\r\n     *   // { id: 4, text: 'item 4 (new)' }, // would throw\r\n     *   // { text: 'item 4 (new)' }, // would also throw\r\n     * ])\r\n     *\r\n     * console.log(ids) // [2]\r\n     * ```\r\n     *\r\n     * @param data - Updates (the id and optionally other props) to the items in this data set.\r\n     * @param senderId - Sender id.\r\n     *\r\n     * @returns updatedIds - The ids of the updated items.\r\n     *\r\n     * @throws When the supplied data is neither an item nor an array of items, when the ids are missing.\r\n     */\r\n    updateOnly(data, senderId) {\r\n        if (!Array.isArray(data)) {\r\n            data = [data];\r\n        }\r\n        const updateEventData = data\r\n            .map((update) => {\r\n            const oldData = this._data.get(update[this._idProp]);\r\n            if (oldData == null) {\r\n                throw new Error(\"Updating non-existent items is not allowed.\");\r\n            }\r\n            return { oldData, update };\r\n        })\r\n            .map(({ oldData, update }) => {\r\n            const id = oldData[this._idProp];\r\n            const updatedData = deepExtend(deepExtend({}, oldData), update);\r\n            this._data.set(id, updatedData);\r\n            return {\r\n                id,\r\n                oldData: oldData,\r\n                updatedData\r\n            };\r\n        });\r\n        if (updateEventData.length) {\r\n            const props = {\r\n                items: updateEventData.map((value) => value.id),\r\n                oldData: updateEventData.map((value) => value.oldData),\r\n                data: updateEventData.map((value) => value.updatedData)\r\n            };\r\n            // TODO: remove deprecated property 'data' some day\r\n            //Object.defineProperty(props, 'data', {\r\n            //  'get': (function() {\r\n            //    console.warn('Property data is deprecated. Use DataSet.get(ids) to retrieve the new data, use the oldData property on this object to get the old data');\r\n            //    return updatedData;\r\n            //  }).bind(this)\r\n            //});\r\n            this._trigger(\"update\", props, senderId);\r\n            return props.items;\r\n        }\r\n        else {\r\n            return [];\r\n        }\r\n    }\r\n    /** @inheritdoc */\r\n    get(first, second) {\r\n        // @TODO: Woudn't it be better to split this into multiple methods?\r\n        // parse the arguments\r\n        let id = undefined;\r\n        let ids = undefined;\r\n        let options = undefined;\r\n        if (isId(first)) {\r\n            // get(id [, options])\r\n            id = first;\r\n            options = second;\r\n        }\r\n        else if (Array.isArray(first)) {\r\n            // get(ids [, options])\r\n            ids = first;\r\n            options = second;\r\n        }\r\n        else {\r\n            // get([, options])\r\n            options = first;\r\n        }\r\n        // determine the return type\r\n        const returnType = options && options.returnType === \"Object\" ? \"Object\" : \"Array\";\r\n        // @TODO: WTF is this? Or am I missing something?\r\n        // var returnType\r\n        // if (options && options.returnType) {\r\n        //   var allowedValues = ['Array', 'Object']\r\n        //   returnType =\r\n        //     allowedValues.indexOf(options.returnType) == -1\r\n        //       ? 'Array'\r\n        //       : options.returnType\r\n        // } else {\r\n        //   returnType = 'Array'\r\n        // }\r\n        // build options\r\n        const type = (options && options.type) || this._options.type;\r\n        const filter = options && options.filter;\r\n        const items = [];\r\n        let item = null;\r\n        let itemIds = null;\r\n        let itemId = null;\r\n        // convert items\r\n        if (id != null) {\r\n            // return a single item\r\n            item = this._getItem(id, type);\r\n            if (item && filter && !filter(item)) {\r\n                item = null;\r\n            }\r\n        }\r\n        else if (ids != null) {\r\n            // return a subset of items\r\n            for (let i = 0, len = ids.length; i < len; i++) {\r\n                item = this._getItem(ids[i], type);\r\n                if (item != null && (!filter || filter(item))) {\r\n                    items.push(item);\r\n                }\r\n            }\r\n        }\r\n        else {\r\n            // return all items\r\n            itemIds = [...this._data.keys()];\r\n            for (let i = 0, len = itemIds.length; i < len; i++) {\r\n                itemId = itemIds[i];\r\n                item = this._getItem(itemId, type);\r\n                if (item != null && (!filter || filter(item))) {\r\n                    items.push(item);\r\n                }\r\n            }\r\n        }\r\n        // order the results\r\n        if (options && options.order && id == undefined) {\r\n            this._sort(items, options.order);\r\n        }\r\n        // filter fields of the items\r\n        if (options && options.fields) {\r\n            const fields = options.fields;\r\n            if (id != undefined && item != null) {\r\n                item = this._filterFields(item, fields);\r\n            }\r\n            else {\r\n                for (let i = 0, len = items.length; i < len; i++) {\r\n                    items[i] = this._filterFields(items[i], fields);\r\n                }\r\n            }\r\n        }\r\n        // return the results\r\n        if (returnType == \"Object\") {\r\n            const result = {};\r\n            for (let i = 0, len = items.length; i < len; i++) {\r\n                const resultant = items[i];\r\n                // @TODO: Shoudn't this be this._fieldId?\r\n                // result[resultant.id] = resultant\r\n                const id = resultant[this._idProp];\r\n                result[id] = resultant;\r\n            }\r\n            return result;\r\n        }\r\n        else {\r\n            if (id != null) {\r\n                // a single item\r\n                return item;\r\n            }\r\n            else {\r\n                // just return our array\r\n                return items;\r\n            }\r\n        }\r\n    }\r\n    /** @inheritdoc */\r\n    getIds(options) {\r\n        const data = this._data;\r\n        const filter = options && options.filter;\r\n        const order = options && options.order;\r\n        const type = (options && options.type) || this._options.type;\r\n        const itemIds = [...data.keys()];\r\n        const ids = [];\r\n        let item;\r\n        let items;\r\n        if (filter) {\r\n            // get filtered items\r\n            if (order) {\r\n                // create ordered list\r\n                items = [];\r\n                for (let i = 0, len = itemIds.length; i < len; i++) {\r\n                    const id = itemIds[i];\r\n                    item = this._getItem(id, type);\r\n                    if (filter(item)) {\r\n                        items.push(item);\r\n                    }\r\n                }\r\n                this._sort(items, order);\r\n                for (let i = 0, len = items.length; i < len; i++) {\r\n                    ids.push(items[i][this._idProp]);\r\n                }\r\n            }\r\n            else {\r\n                // create unordered list\r\n                for (let i = 0, len = itemIds.length; i < len; i++) {\r\n                    const id = itemIds[i];\r\n                    item = this._getItem(id, type);\r\n                    if (filter(item)) {\r\n                        ids.push(item[this._idProp]);\r\n                    }\r\n                }\r\n            }\r\n        }\r\n        else {\r\n            // get all items\r\n            if (order) {\r\n                // create an ordered list\r\n                items = [];\r\n                for (let i = 0, len = itemIds.length; i < len; i++) {\r\n                    const id = itemIds[i];\r\n                    items.push(data.get(id));\r\n                }\r\n                this._sort(items, order);\r\n                for (let i = 0, len = items.length; i < len; i++) {\r\n                    ids.push(items[i][this._idProp]);\r\n                }\r\n            }\r\n            else {\r\n                // create unordered list\r\n                for (let i = 0, len = itemIds.length; i < len; i++) {\r\n                    const id = itemIds[i];\r\n                    item = data.get(id);\r\n                    ids.push(item[this._idProp]);\r\n                }\r\n            }\r\n        }\r\n        return ids;\r\n    }\r\n    /** @inheritdoc */\r\n    getDataSet() {\r\n        return this;\r\n    }\r\n    /** @inheritdoc */\r\n    forEach(callback, options) {\r\n        const filter = options && options.filter;\r\n        const type = (options && options.type) || this._options.type;\r\n        const data = this._data;\r\n        const itemIds = [...data.keys()];\r\n        if (options && options.order) {\r\n            // execute forEach on ordered list\r\n            const items = this.get(options);\r\n            for (let i = 0, len = items.length; i < len; i++) {\r\n                const item = items[i];\r\n                const id = item[this._idProp];\r\n                callback(item, id);\r\n            }\r\n        }\r\n        else {\r\n            // unordered\r\n            for (let i = 0, len = itemIds.length; i < len; i++) {\r\n                const id = itemIds[i];\r\n                const item = this._getItem(id, type);\r\n                if (!filter || filter(item)) {\r\n                    callback(item, id);\r\n                }\r\n            }\r\n        }\r\n    }\r\n    /** @inheritdoc */\r\n    map(callback, options) {\r\n        const filter = options && options.filter;\r\n        const type = (options && options.type) || this._options.type;\r\n        const mappedItems = [];\r\n        const data = this._data;\r\n        const itemIds = [...data.keys()];\r\n        // convert and filter items\r\n        for (let i = 0, len = itemIds.length; i < len; i++) {\r\n            const id = itemIds[i];\r\n            const item = this._getItem(id, type);\r\n            if (!filter || filter(item)) {\r\n                mappedItems.push(callback(item, id));\r\n            }\r\n        }\r\n        // order items\r\n        if (options && options.order) {\r\n            this._sort(mappedItems, options.order);\r\n        }\r\n        return mappedItems;\r\n    }\r\n    /**\r\n     * Filter the fields of an item.\r\n     *\r\n     * @param item - The item whose fields should be filtered.\r\n     * @param fields - The names of the fields that will be kept.\r\n     *\r\n     * @typeParam K - Field name type.\r\n     *\r\n     * @returns The item without any additional fields.\r\n     */\r\n    _filterFields(item, fields) {\r\n        if (!item) {\r\n            // item is null\r\n            return item;\r\n        }\r\n        return (Array.isArray(fields)\r\n            ? // Use the supplied array\r\n                fields\r\n            : // Use the keys of the supplied object\r\n                Object.keys(fields)).reduce((filteredItem, field) => {\r\n            filteredItem[field] = item[field];\r\n            return filteredItem;\r\n        }, {});\r\n    }\r\n    /**\r\n     * Sort the provided array with items.\r\n     *\r\n     * @param items - Items to be sorted in place.\r\n     * @param order - A field name or custom sort function.\r\n     *\r\n     * @typeParam T - The type of the items in the items array.\r\n     */\r\n    _sort(items, order) {\r\n        if (typeof order === \"string\") {\r\n            // order by provided field name\r\n            const name = order; // field name\r\n            items.sort((a, b) => {\r\n                // @TODO: How to treat missing properties?\r\n                const av = a[name];\r\n                const bv = b[name];\r\n                return av > bv ? 1 : av < bv ? -1 : 0;\r\n            });\r\n        }\r\n        else if (typeof order === \"function\") {\r\n            // order by sort function\r\n            items.sort(order);\r\n        }\r\n        else {\r\n            // TODO: extend order by an Object {field:string, direction:string}\r\n            //       where direction can be 'asc' or 'desc'\r\n            throw new TypeError(\"Order must be a function or a string\");\r\n        }\r\n    }\r\n    /**\r\n     * Remove an item or multiple items by “reference” (only the id is used) or by id.\r\n     *\r\n     * The method ignores removal of non-existing items, and returns an array containing the ids of the items which are actually removed from the DataSet.\r\n     *\r\n     * After the items are removed, the DataSet will trigger an event `remove` for the removed items. When a `senderId` is provided, this id will be passed with the triggered event to all subscribers.\r\n     *\r\n     * ## Example\r\n     * ```javascript\r\n     * // create a DataSet\r\n     * const data = new vis.DataSet([\r\n     *   { id: 1, text: 'item 1' },\r\n     *   { id: 2, text: 'item 2' },\r\n     *   { id: 3, text: 'item 3' }\r\n     * ])\r\n     *\r\n     * // remove items\r\n     * const ids = data.remove([2, { id: 3 }, 4])\r\n     *\r\n     * console.log(ids) // [2, 3]\r\n     * ```\r\n     *\r\n     * @param id - One or more items or ids of items to be removed.\r\n     * @param senderId - Sender id.\r\n     *\r\n     * @returns The ids of the removed items.\r\n     */\r\n    remove(id, senderId) {\r\n        const removedIds = [];\r\n        const removedItems = [];\r\n        // force everything to be an array for simplicity\r\n        const ids = Array.isArray(id) ? id : [id];\r\n        for (let i = 0, len = ids.length; i < len; i++) {\r\n            const item = this._remove(ids[i]);\r\n            if (item) {\r\n                const itemId = item[this._idProp];\r\n                if (itemId != null) {\r\n                    removedIds.push(itemId);\r\n                    removedItems.push(item);\r\n                }\r\n            }\r\n        }\r\n        if (removedIds.length) {\r\n            this._trigger(\"remove\", { items: removedIds, oldData: removedItems }, senderId);\r\n        }\r\n        return removedIds;\r\n    }\r\n    /**\r\n     * Remove an item by its id or reference.\r\n     *\r\n     * @param id - Id of an item or the item itself.\r\n     *\r\n     * @returns The removed item if removed, null otherwise.\r\n     */\r\n    _remove(id) {\r\n        // @TODO: It origianlly returned the item although the docs say id.\r\n        // The code expects the item, so probably an error in the docs.\r\n        let ident;\r\n        // confirm the id to use based on the args type\r\n        if (isId(id)) {\r\n            ident = id;\r\n        }\r\n        else if (id && typeof id === \"object\") {\r\n            ident = id[this._idProp]; // look for the identifier field using ._idProp\r\n        }\r\n        // do the removing if the item is found\r\n        if (ident != null && this._data.has(ident)) {\r\n            const item = this._data.get(ident) || null;\r\n            this._data.delete(ident);\r\n            --this.length;\r\n            return item;\r\n        }\r\n        return null;\r\n    }\r\n    /**\r\n     * Clear the entire data set.\r\n     *\r\n     * After the items are removed, the [[DataSet]] will trigger an event `remove` for all removed items. When a `senderId` is provided, this id will be passed with the triggered event to all subscribers.\r\n     *\r\n     * @param senderId - Sender id.\r\n     *\r\n     * @returns removedIds - The ids of all removed items.\r\n     */\r\n    clear(senderId) {\r\n        const ids = [...this._data.keys()];\r\n        const items = [];\r\n        for (let i = 0, len = ids.length; i < len; i++) {\r\n            items.push(this._data.get(ids[i]));\r\n        }\r\n        this._data.clear();\r\n        this.length = 0;\r\n        this._trigger(\"remove\", { items: ids, oldData: items }, senderId);\r\n        return ids;\r\n    }\r\n    /**\r\n     * Find the item with maximum value of a specified field.\r\n     *\r\n     * @param field - Name of the property that should be searched for max value.\r\n     *\r\n     * @returns Item containing max value, or null if no items.\r\n     */\r\n    max(field) {\r\n        let max = null;\r\n        let maxField = null;\r\n        for (const item of this._data.values()) {\r\n            const itemField = item[field];\r\n            if (typeof itemField === \"number\" &&\r\n                (maxField == null || itemField > maxField)) {\r\n                max = item;\r\n                maxField = itemField;\r\n            }\r\n        }\r\n        return max || null;\r\n    }\r\n    /**\r\n     * Find the item with minimum value of a specified field.\r\n     *\r\n     * @param field - Name of the property that should be searched for min value.\r\n     *\r\n     * @returns Item containing min value, or null if no items.\r\n     */\r\n    min(field) {\r\n        let min = null;\r\n        let minField = null;\r\n        for (const item of this._data.values()) {\r\n            const itemField = item[field];\r\n            if (typeof itemField === \"number\" &&\r\n                (minField == null || itemField < minField)) {\r\n                min = item;\r\n                minField = itemField;\r\n            }\r\n        }\r\n        return min || null;\r\n    }\r\n    /**\r\n     * Find all distinct values of a specified field\r\n     *\r\n     * @param prop - The property name whose distinct values should be returned.\r\n     *\r\n     * @returns Unordered array containing all distinct values. Items without specified property are ignored.\r\n     */\r\n    distinct(prop) {\r\n        const data = this._data;\r\n        const itemIds = [...data.keys()];\r\n        const values = [];\r\n        const fieldType = (this._options.type && this._options.type[prop]) || null;\r\n        let count = 0;\r\n        for (let i = 0, len = itemIds.length; i < len; i++) {\r\n            const id = itemIds[i];\r\n            const item = data.get(id);\r\n            const value = item[prop];\r\n            let exists = false;\r\n            for (let j = 0; j < count; j++) {\r\n                if (values[j] == value) {\r\n                    exists = true;\r\n                    break;\r\n                }\r\n            }\r\n            if (!exists && value !== undefined) {\r\n                values[count] = value;\r\n                count++;\r\n            }\r\n        }\r\n        if (fieldType) {\r\n            for (let i = 0, len = values.length; i < len; i++) {\r\n                values[i] = convert(values[i], fieldType);\r\n            }\r\n        }\r\n        return values;\r\n    }\r\n    /**\r\n     * Add a single item. Will fail when an item with the same id already exists.\r\n     *\r\n     * @param item - A new item to be added.\r\n     *\r\n     * @returns Added item's id. An id is generated when it is not present in the item.\r\n     */\r\n    _addItem(item) {\r\n        let id = item[this._idProp];\r\n        if (id != null) {\r\n            // check whether this id is already taken\r\n            if (this._data.has(id)) {\r\n                // item already exists\r\n                throw new Error(\"Cannot add item: item with id \" + id + \" already exists\");\r\n            }\r\n        }\r\n        else {\r\n            // generate an id\r\n            id = v4_1();\r\n            item[this._idProp] = id;\r\n        }\r\n        const d = {};\r\n        const fields = Object.keys(item);\r\n        for (let i = 0, len = fields.length; i < len; i++) {\r\n            const field = fields[i];\r\n            const fieldType = this._type[field]; // type may be undefined\r\n            d[field] = convert(item[field], fieldType);\r\n        }\r\n        this._data.set(id, d);\r\n        ++this.length;\r\n        return id;\r\n    }\r\n    /**\r\n     * Get an item. Fields can be converted to a specific type\r\n     *\r\n     * @param id - Id of the requested item.\r\n     * @param types - Property name to type name object map of type converstions.\r\n     *\r\n     * @returns The item, optionally after type conversion.\r\n     */\r\n    _getItem(id, types) {\r\n        // @TODO: I have no idea how to type this.\r\n        // get the item from the dataset\r\n        const raw = this._data.get(id);\r\n        if (!raw) {\r\n            return null;\r\n        }\r\n        // convert the items field types\r\n        let converted;\r\n        const fields = Object.keys(raw);\r\n        if (types) {\r\n            warnTypeCorectionDeprecation();\r\n            converted = {};\r\n            for (let i = 0, len = fields.length; i < len; i++) {\r\n                const field = fields[i];\r\n                const value = raw[field];\r\n                converted[field] = convert(value, types[field]);\r\n            }\r\n        }\r\n        else {\r\n            // no field types specified, no converting needed\r\n            converted = { ...raw };\r\n        }\r\n        if (converted[this._idProp] == null) {\r\n            converted[this._idProp] = raw.id;\r\n        }\r\n        return converted;\r\n    }\r\n    /**\r\n     * Update a single item: merge with existing item.\r\n     * Will fail when the item has no id, or when there does not exist an item with the same id.\r\n     *\r\n     * @param item - The new item\r\n     *\r\n     * @returns The id of the updated item.\r\n     */\r\n    _updateItem(item) {\r\n        const id = item[this._idProp];\r\n        if (id == null) {\r\n            throw new Error(\"Cannot update item: item has no id (item: \" +\r\n                JSON.stringify(item) +\r\n                \")\");\r\n        }\r\n        const d = this._data.get(id);\r\n        if (!d) {\r\n            // item doesn't exist\r\n            throw new Error(\"Cannot update item: no item with id \" + id + \" found\");\r\n        }\r\n        // merge with current item\r\n        const fields = Object.keys(item);\r\n        for (let i = 0, len = fields.length; i < len; i++) {\r\n            const field = fields[i];\r\n            const fieldType = this._type[field]; // type may be undefined\r\n            d[field] = convert(item[field], fieldType);\r\n        }\r\n        return id;\r\n    }\r\n    /** @inheritdoc */\r\n    stream(ids) {\r\n        if (ids) {\r\n            const data = this._data;\r\n            return new DataStream({\r\n                *[Symbol.iterator]() {\r\n                    for (const id of ids) {\r\n                        const item = data.get(id);\r\n                        if (item != null) {\r\n                            yield [id, item];\r\n                        }\r\n                    }\r\n                }\r\n            });\r\n        }\r\n        else {\r\n            return new DataStream({\r\n                [Symbol.iterator]: this._data.entries.bind(this._data)\r\n            });\r\n        }\r\n    }\r\n}\n\n/**\r\n * DataView\r\n *\r\n * A DataView offers a filtered and/or formatted view on a DataSet. One can subscribe to changes in a DataView, and easily get filtered or formatted data without having to specify filters and field types all the time.\r\n *\r\n * ## Example\r\n * ```javascript\r\n * // create a DataSet\r\n * var data = new vis.DataSet();\r\n * data.add([\r\n *   {id: 1, text: 'item 1', date: new Date(2013, 6, 20), group: 1, first: true},\r\n *   {id: 2, text: 'item 2', date: '2013-06-23', group: 2},\r\n *   {id: 3, text: 'item 3', date: '2013-06-25', group: 2},\r\n *   {id: 4, text: 'item 4'}\r\n * ]);\r\n *\r\n * // create a DataView\r\n * // the view will only contain items having a property group with value 1,\r\n * // and will only output fields id, text, and date.\r\n * var view = new vis.DataView(data, {\r\n *   filter: function (item) {\r\n *     return (item.group == 1);\r\n *   },\r\n *   fields: ['id', 'text', 'date']\r\n * });\r\n *\r\n * // subscribe to any change in the DataView\r\n * view.on('*', function (event, properties, senderId) {\r\n *   console.log('event', event, properties);\r\n * });\r\n *\r\n * // update an item in the data set\r\n * data.update({id: 2, group: 1});\r\n *\r\n * // get all ids in the view\r\n * var ids = view.getIds();\r\n * console.log('ids', ids); // will output [1, 2]\r\n *\r\n * // get all items in the view\r\n * var items = view.get();\r\n * ```\r\n *\r\n * @typeParam Item - Item type that may or may not have an id.\r\n * @typeParam IdProp - Name of the property that contains the id.\r\n */\r\nclass DataView extends DataSetPart {\r\n    /**\r\n     * Create a DataView.\r\n     *\r\n     * @param data - The instance containing data (directly or indirectly).\r\n     * @param options - Options to configure this data view.\r\n     */\r\n    constructor(data, options) {\r\n        super();\r\n        /** @inheritdoc */\r\n        this.length = 0;\r\n        this._ids = new Set(); // ids of the items currently in memory (just contains a boolean true)\r\n        this._options = options || {};\r\n        this._listener = this._onEvent.bind(this);\r\n        this.setData(data);\r\n    }\r\n    // TODO: implement a function .config() to dynamically update things like configured filter\r\n    // and trigger changes accordingly\r\n    /**\r\n     * Set a data source for the view.\r\n     *\r\n     * @param data - The instance containing data (directly or indirectly).\r\n     *\r\n     * @remarks\r\n     * Note that when the data view is bound to a data set it won't be garbage\r\n     * collected unless the data set is too. Use `dataView.setData(null)` or\r\n     * `dataView.dispose()` to enable garbage collection before you lose the last\r\n     * reference.\r\n     */\r\n    setData(data) {\r\n        if (this._data) {\r\n            // unsubscribe from current dataset\r\n            if (this._data.off) {\r\n                this._data.off(\"*\", this._listener);\r\n            }\r\n            // trigger a remove of all items in memory\r\n            const ids = this._data.getIds({ filter: this._options.filter });\r\n            const items = this._data.get(ids);\r\n            this._ids.clear();\r\n            this.length = 0;\r\n            this._trigger(\"remove\", { items: ids, oldData: items });\r\n        }\r\n        if (data != null) {\r\n            this._data = data;\r\n            // trigger an add of all added items\r\n            const ids = this._data.getIds({ filter: this._options.filter });\r\n            for (let i = 0, len = ids.length; i < len; i++) {\r\n                const id = ids[i];\r\n                this._ids.add(id);\r\n            }\r\n            this.length = ids.length;\r\n            this._trigger(\"add\", { items: ids });\r\n        }\r\n        else {\r\n            this._data = new DataSet();\r\n        }\r\n        // subscribe to new dataset\r\n        if (this._data.on) {\r\n            this._data.on(\"*\", this._listener);\r\n        }\r\n    }\r\n    /**\r\n     * Refresh the DataView.\r\n     * Useful when the DataView has a filter function containing a variable parameter.\r\n     */\r\n    refresh() {\r\n        const ids = this._data.getIds({\r\n            filter: this._options.filter\r\n        });\r\n        const oldIds = [...this._ids];\r\n        const newIds = {};\r\n        const addedIds = [];\r\n        const removedIds = [];\r\n        const removedItems = [];\r\n        // check for additions\r\n        for (let i = 0, len = ids.length; i < len; i++) {\r\n            const id = ids[i];\r\n            newIds[id] = true;\r\n            if (!this._ids.has(id)) {\r\n                addedIds.push(id);\r\n                this._ids.add(id);\r\n            }\r\n        }\r\n        // check for removals\r\n        for (let i = 0, len = oldIds.length; i < len; i++) {\r\n            const id = oldIds[i];\r\n            const item = this._data.get(id);\r\n            if (item == null) {\r\n                // @TODO: Investigate.\r\n                // Doesn't happen during tests or examples.\r\n                // Is it really impossible or could it eventually happen?\r\n                // How to handle it if it does? The types guarantee non-nullable items.\r\n                console.error(\"If you see this, report it please.\");\r\n            }\r\n            else if (!newIds[id]) {\r\n                removedIds.push(id);\r\n                removedItems.push(item);\r\n                this._ids.delete(id);\r\n            }\r\n        }\r\n        this.length += addedIds.length - removedIds.length;\r\n        // trigger events\r\n        if (addedIds.length) {\r\n            this._trigger(\"add\", { items: addedIds });\r\n        }\r\n        if (removedIds.length) {\r\n            this._trigger(\"remove\", { items: removedIds, oldData: removedItems });\r\n        }\r\n    }\r\n    /** @inheritdoc */\r\n    get(first, second) {\r\n        if (this._data == null) {\r\n            return null;\r\n        }\r\n        // parse the arguments\r\n        let ids = null;\r\n        let options;\r\n        if (isId(first) || Array.isArray(first)) {\r\n            ids = first;\r\n            options = second;\r\n        }\r\n        else {\r\n            options = first;\r\n        }\r\n        // extend the options with the default options and provided options\r\n        const viewOptions = Object.assign({}, this._options, options);\r\n        // create a combined filter method when needed\r\n        const thisFilter = this._options.filter;\r\n        const optionsFilter = options && options.filter;\r\n        if (thisFilter && optionsFilter) {\r\n            viewOptions.filter = (item) => {\r\n                return thisFilter(item) && optionsFilter(item);\r\n            };\r\n        }\r\n        if (ids == null) {\r\n            return this._data.get(viewOptions);\r\n        }\r\n        else {\r\n            return this._data.get(ids, viewOptions);\r\n        }\r\n    }\r\n    /** @inheritdoc */\r\n    getIds(options) {\r\n        if (this._data.length) {\r\n            const defaultFilter = this._options.filter;\r\n            const optionsFilter = options != null ? options.filter : null;\r\n            let filter;\r\n            if (optionsFilter) {\r\n                if (defaultFilter) {\r\n                    filter = (item) => {\r\n                        return defaultFilter(item) && optionsFilter(item);\r\n                    };\r\n                }\r\n                else {\r\n                    filter = optionsFilter;\r\n                }\r\n            }\r\n            else {\r\n                filter = defaultFilter;\r\n            }\r\n            return this._data.getIds({\r\n                filter: filter,\r\n                order: options && options.order\r\n            });\r\n        }\r\n        else {\r\n            return [];\r\n        }\r\n    }\r\n    /** @inheritdoc */\r\n    forEach(callback, options) {\r\n        if (this._data) {\r\n            const defaultFilter = this._options.filter;\r\n            const optionsFilter = options && options.filter;\r\n            let filter;\r\n            if (optionsFilter) {\r\n                if (defaultFilter) {\r\n                    filter = function (item) {\r\n                        return defaultFilter(item) && optionsFilter(item);\r\n                    };\r\n                }\r\n                else {\r\n                    filter = optionsFilter;\r\n                }\r\n            }\r\n            else {\r\n                filter = defaultFilter;\r\n            }\r\n            this._data.forEach(callback, {\r\n                filter: filter,\r\n                order: options && options.order\r\n            });\r\n        }\r\n    }\r\n    /** @inheritdoc */\r\n    map(callback, options) {\r\n        if (this._data) {\r\n            const defaultFilter = this._options.filter;\r\n            const optionsFilter = options && options.filter;\r\n            let filter;\r\n            if (optionsFilter) {\r\n                if (defaultFilter) {\r\n                    filter = (item) => {\r\n                        return defaultFilter(item) && optionsFilter(item);\r\n                    };\r\n                }\r\n                else {\r\n                    filter = optionsFilter;\r\n                }\r\n            }\r\n            else {\r\n                filter = defaultFilter;\r\n            }\r\n            return this._data.map(callback, {\r\n                filter: filter,\r\n                order: options && options.order\r\n            });\r\n        }\r\n        else {\r\n            return [];\r\n        }\r\n    }\r\n    /** @inheritdoc */\r\n    getDataSet() {\r\n        return this._data.getDataSet();\r\n    }\r\n    /** @inheritdoc */\r\n    stream(ids) {\r\n        return this._data.stream(ids || {\r\n            [Symbol.iterator]: this._ids.keys.bind(this._ids)\r\n        });\r\n    }\r\n    /**\r\n     * Render the instance unusable prior to garbage collection.\r\n     *\r\n     * @remarks\r\n     * The intention of this method is to help discover scenarios where the data\r\n     * view is being used when the programmer thinks it has been garbage collected\r\n     * already. It's stricter version of `dataView.setData(null)`.\r\n     */\r\n    dispose() {\r\n        var _a;\r\n        if ((_a = this._data) === null || _a === void 0 ? void 0 : _a.off) {\r\n            this._data.off(\"*\", this._listener);\r\n        }\r\n        const message = \"This data view has already been disposed of.\";\r\n        Object.defineProperty(this, \"_data\", {\r\n            get: () => {\r\n                throw new Error(message);\r\n            },\r\n            set: () => {\r\n                throw new Error(message);\r\n            },\r\n            configurable: false\r\n        });\r\n    }\r\n    /**\r\n     * Event listener. Will propagate all events from the connected data set to the subscribers of the DataView, but will filter the items and only trigger when there are changes in the filtered data set.\r\n     *\r\n     * @param event - The name of the event.\r\n     * @param params - Parameters of the event.\r\n     * @param senderId - Id supplied by the sender.\r\n     */\r\n    _onEvent(event, params, senderId) {\r\n        if (!params || !params.items || !this._data) {\r\n            return;\r\n        }\r\n        const ids = params.items;\r\n        const addedIds = [];\r\n        const updatedIds = [];\r\n        const removedIds = [];\r\n        const oldItems = [];\r\n        const updatedItems = [];\r\n        const removedItems = [];\r\n        switch (event) {\r\n            case \"add\":\r\n                // filter the ids of the added items\r\n                for (let i = 0, len = ids.length; i < len; i++) {\r\n                    const id = ids[i];\r\n                    const item = this.get(id);\r\n                    if (item) {\r\n                        this._ids.add(id);\r\n                        addedIds.push(id);\r\n                    }\r\n                }\r\n                break;\r\n            case \"update\":\r\n                // determine the event from the views viewpoint: an updated\r\n                // item can be added, updated, or removed from this view.\r\n                for (let i = 0, len = ids.length; i < len; i++) {\r\n                    const id = ids[i];\r\n                    const item = this.get(id);\r\n                    if (item) {\r\n                        if (this._ids.has(id)) {\r\n                            updatedIds.push(id);\r\n                            updatedItems.push(params.data[i]);\r\n                            oldItems.push(params.oldData[i]);\r\n                        }\r\n                        else {\r\n                            this._ids.add(id);\r\n                            addedIds.push(id);\r\n                        }\r\n                    }\r\n                    else {\r\n                        if (this._ids.has(id)) {\r\n                            this._ids.delete(id);\r\n                            removedIds.push(id);\r\n                            removedItems.push(params.oldData[i]);\r\n                        }\r\n                    }\r\n                }\r\n                break;\r\n            case \"remove\":\r\n                // filter the ids of the removed items\r\n                for (let i = 0, len = ids.length; i < len; i++) {\r\n                    const id = ids[i];\r\n                    if (this._ids.has(id)) {\r\n                        this._ids.delete(id);\r\n                        removedIds.push(id);\r\n                        removedItems.push(params.oldData[i]);\r\n                    }\r\n                }\r\n                break;\r\n        }\r\n        this.length += addedIds.length - removedIds.length;\r\n        if (addedIds.length) {\r\n            this._trigger(\"add\", { items: addedIds }, senderId);\r\n        }\r\n        if (updatedIds.length) {\r\n            this._trigger(\"update\", { items: updatedIds, oldData: oldItems, data: updatedItems }, senderId);\r\n        }\r\n        if (removedIds.length) {\r\n            this._trigger(\"remove\", { items: removedIds, oldData: removedItems }, senderId);\r\n        }\r\n    }\r\n}\n\nexport { DataSet, DataStream, DataView, Queue, createNewDataPipeFrom };\n//# sourceMappingURL=vis-data.js.map\n","// first check if moment.js is already loaded in the browser window, if so,\n// use this instance. Else, load via commonjs.\n//\n// Note: This doesn't work in ESM.\nmodule.exports = (typeof window !== 'undefined') && window['moment'] || require('moment');\n","// utility functions\n\nexport * from \"vis-util/esnext\";\nimport * as util from \"vis-util/esnext\";\nimport { getType, isNumber, isString } from \"vis-util/esnext\";\nimport { DataSet, createNewDataPipeFrom } from \"vis-data/esnext\";\n\nimport moment from \"moment\";\n\n// parse ASP.Net Date pattern,\n// for example '/Date(1198908717056)/' or '/Date(1198908717056-0700)/'\n// code from http://momentjs.com/\nconst ASPDateRegex = /^\\/?Date\\((-?\\d+)/i;\n\n/**\n * Convert an object into another type\n *\n * @param object - Value of unknown type.\n * @param type - Name of the desired type.\n *\n * @returns Object in the desired type.\n * @throws Error\n */\nexport function convert(object, type) {\n  let match;\n\n  if (object === undefined) {\n    return undefined;\n  }\n  if (object === null) {\n    return null;\n  }\n\n  if (!type) {\n    return object;\n  }\n  if (!(typeof type === \"string\") && !(type instanceof String)) {\n    throw new Error(\"Type must be a string\");\n  }\n\n  //noinspection FallthroughInSwitchStatementJS\n  switch (type) {\n    case \"boolean\":\n    case \"Boolean\":\n      return Boolean(object);\n\n    case \"number\":\n    case \"Number\":\n      if (isString(object) && !isNaN(Date.parse(object))) {\n        return moment(object).valueOf();\n      } else {\n        // @TODO: I don't think that Number and String constructors are a good idea.\n        // This could also fail if the object doesn't have valueOf method or if it's redefined.\n        // For example: Object.create(null) or { valueOf: 7 }.\n        return Number(object.valueOf());\n      }\n    case \"string\":\n    case \"String\":\n      return String(object);\n\n    case \"Date\":\n      if (isNumber(object)) {\n        return new Date(object);\n      }\n      if (object instanceof Date) {\n        return new Date(object.valueOf());\n      } else if (moment.isMoment(object)) {\n        return new Date(object.valueOf());\n      }\n      if (isString(object)) {\n        match = ASPDateRegex.exec(object);\n        if (match) {\n          // object is an ASP date\n          return new Date(Number(match[1])); // parse number\n        } else {\n          return moment(new Date(object)).toDate(); // parse string\n        }\n      } else {\n        throw new Error(\n          \"Cannot convert object of type \" + getType(object) + \" to type Date\"\n        );\n      }\n\n    case \"Moment\":\n      if (isNumber(object)) {\n        return moment(object);\n      }\n      if (object instanceof Date) {\n        return moment(object.valueOf());\n      } else if (moment.isMoment(object)) {\n        return moment(object);\n      }\n      if (isString(object)) {\n        match = ASPDateRegex.exec(object);\n        if (match) {\n          // object is an ASP date\n          return moment(Number(match[1])); // parse number\n        } else {\n          return moment(object); // parse string\n        }\n      } else {\n        throw new Error(\n          \"Cannot convert object of type \" + getType(object) + \" to type Date\"\n        );\n      }\n\n    case \"ISODate\":\n      if (isNumber(object)) {\n        return new Date(object);\n      } else if (object instanceof Date) {\n        return object.toISOString();\n      } else if (moment.isMoment(object)) {\n        return object.toDate().toISOString();\n      } else if (isString(object)) {\n        match = ASPDateRegex.exec(object);\n        if (match) {\n          // object is an ASP date\n          return new Date(Number(match[1])).toISOString(); // parse number\n        } else {\n          return moment(object).format(); // ISO 8601\n        }\n      } else {\n        throw new Error(\n          \"Cannot convert object of type \" +\n            getType(object) +\n            \" to type ISODate\"\n        );\n      }\n\n    case \"ASPDate\":\n      if (isNumber(object)) {\n        return \"/Date(\" + object + \")/\";\n      } else if (object instanceof Date || moment.isMoment(object)) {\n        return \"/Date(\" + object.valueOf() + \")/\";\n      } else if (isString(object)) {\n        match = ASPDateRegex.exec(object);\n        let value;\n        if (match) {\n          // object is an ASP date\n          value = new Date(Number(match[1])).valueOf(); // parse number\n        } else {\n          value = new Date(object).valueOf(); // parse string\n        }\n        return \"/Date(\" + value + \")/\";\n      } else {\n        throw new Error(\n          \"Cannot convert object of type \" +\n            getType(object) +\n            \" to type ASPDate\"\n        );\n      }\n\n    default:\n      throw new Error(`Unknown type ${type}`);\n  }\n}\n\n/**\n * Create a Data Set like wrapper to seamlessly coerce data types.\n *\n * @param rawDS - The Data Set with raw uncoerced data.\n * @param type - A record assigning a data type to property name.\n *\n * @remarks\n * The write operations (`add`, `remove`, `update` and `updateOnly`) write into\n * the raw (uncoerced) data set. These values are then picked up by a pipe\n * which coerces the values using the [[convert]] function and feeds them into\n * the coerced data set. When querying (`forEach`, `get`, `getIds`, `off` and\n * `on`) the values are then fetched from the coerced data set and already have\n * the required data types. The values are coerced only once when inserted and\n * then the same value is returned each time until it is updated or deleted.\n *\n * For example: `typeCoercedDataSet.add({ id: 7, start: \"2020-01-21\" })` would\n * result in `typeCoercedDataSet.get(7)` returning `{ id: 7, start: moment(new\n * Date(\"2020-01-21\")).toDate() }`.\n *\n * Use the dispose method prior to throwing a reference to this away. Otherwise\n * the pipe connecting the two Data Sets will keep the unaccessible coerced\n * Data Set alive and updated as long as the raw Data Set exists.\n *\n * @returns A Data Set like object that saves data into the raw Data Set and\n * retrieves them from the coerced Data Set.\n */\nexport function typeCoerceDataSet(\n  rawDS,\n  type = { start: \"Date\", end: \"Date\" }\n) {\n  const idProp = rawDS._idProp;\n  const coercedDS = new DataSet({ fieldId: idProp });\n\n  const pipe = createNewDataPipeFrom(rawDS)\n    .map(item =>\n      Object.keys(item).reduce((acc, key) => {\n        acc[key] = convert(item[key], type[key]);\n        return acc;\n      }, {})\n    )\n    .to(coercedDS);\n\n  pipe.all().start();\n\n  return {\n    // Write only.\n    add: (...args) => rawDS.getDataSet().add(...args),\n    remove: (...args) => rawDS.getDataSet().remove(...args),\n    update: (...args) => rawDS.getDataSet().update(...args),\n    updateOnly: (...args) => rawDS.getDataSet().updateOnly(...args),\n\n    // Read only.\n    forEach: coercedDS.forEach.bind(coercedDS),\n    get: coercedDS.get.bind(coercedDS),\n    getIds: coercedDS.getIds.bind(coercedDS),\n    off: coercedDS.off.bind(coercedDS),\n    on: coercedDS.on.bind(coercedDS),\n\n    get length() {\n      return coercedDS.length;\n    },\n\n    // Non standard.\n    idProp,\n    type,\n\n    rawDS,\n    coercedDS,\n    dispose: () => pipe.stop()\n  };\n}\n\nexport default {\n  ...util,\n  convert\n};\n","var global = require('../internals/global');\nvar trim = require('../internals/string-trim').trim;\nvar whitespaces = require('../internals/whitespaces');\n\nvar $parseFloat = global.parseFloat;\nvar FORCED = 1 / $parseFloat(whitespaces + '-0') !== -Infinity;\n\n// `parseFloat` method\n// https://tc39.github.io/ecma262/#sec-parsefloat-string\nmodule.exports = FORCED ? function parseFloat(string) {\n  var trimmedString = trim(String(string));\n  var result = $parseFloat(trimmedString);\n  return result === 0 && trimmedString.charAt(0) == '-' ? -0 : result;\n} : $parseFloat;\n","var $ = require('../internals/export');\nvar parseFloatImplementation = require('../internals/number-parse-float');\n\n// `parseFloat` method\n// https://tc39.github.io/ecma262/#sec-parsefloat-string\n$({ global: true, forced: parseFloat != parseFloatImplementation }, {\n  parseFloat: parseFloatImplementation\n});\n","require('../modules/es.parse-float');\nvar path = require('../internals/path');\n\nmodule.exports = path.parseFloat;\n","import util from '../../util';\n\n/** Prototype for visual components */\nexport default class Component {\n  /**\n * @param {{dom: Object, domProps: Object, emitter: Emitter, range: Range}} [body]\n * @param {Object} [options]\n */\n  constructor(body, options) {  // eslint-disable-line no-unused-vars\n    this.options = null;\n    this.props = null;\n  }\n\n  /**\n   * Set options for the component. The new options will be merged into the\n   * current options.\n   * @param {Object} options\n   */\n  setOptions(options) {\n    if (options) {\n      util.extend(this.options, options);\n    }\n  }\n\n  /**\n   * Repaint the component\n   * @return {boolean} Returns true if the component is resized\n   */\n  redraw() {\n    // should be implemented by the component\n    return false;\n  }\n\n  /**\n   * Destroy the component. Cleanup DOM and event listeners\n   */\n  destroy() {\n    // should be implemented by the component\n  }\n\n  /**\n   * Test whether the component is resized since the last time _isResized() was\n   * called.\n   * @return {Boolean} Returns true if the component is resized\n   * @protected\n   */\n  _isResized() {\n    const resized = (\n      this.props._previousWidth !== this.props.width ||\n      this.props._previousHeight !== this.props.height\n    );\n\n    this.props._previousWidth = this.props.width;\n    this.props._previousHeight = this.props.height;\n\n    return resized;\n  }\n}\n","var $ = require('../internals/export');\nvar repeat = require('../internals/string-repeat');\n\n// `String.prototype.repeat` method\n// https://tc39.github.io/ecma262/#sec-string.prototype.repeat\n$({ target: 'String', proto: true }, {\n  repeat: repeat\n});\n","'use strict';\nvar toInteger = require('../internals/to-integer');\nvar requireObjectCoercible = require('../internals/require-object-coercible');\n\n// `String.prototype.repeat` method implementation\n// https://tc39.github.io/ecma262/#sec-string.prototype.repeat\nmodule.exports = ''.repeat || function repeat(count) {\n  var str = String(requireObjectCoercible(this));\n  var result = '';\n  var n = toInteger(count);\n  if (n < 0 || n == Infinity) throw RangeError('Wrong number of repetitions');\n  for (;n > 0; (n >>>= 1) && (str += str)) if (n & 1) result += str;\n  return result;\n};\n","require('../../../modules/es.string.repeat');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('String').repeat;\n","var repeat = require('../string/virtual/repeat');\n\nvar StringPrototype = String.prototype;\n\nmodule.exports = function (it) {\n  var own = it.repeat;\n  return typeof it === 'string' || it === StringPrototype\n    || (it instanceof String && own === StringPrototype.repeat) ? repeat : own;\n};\n","\n/**\n * used in Core to convert the options into a volatile variable\n * \n * @param {function} moment\n * @param {Object} body\n * @param {Array | Object} hiddenDates\n * @returns {number}\n */\nexport function convertHiddenOptions(moment, body, hiddenDates) {\n  if (hiddenDates && !Array.isArray(hiddenDates)) {\n    return convertHiddenOptions(moment, body, [hiddenDates])\n  }\n\n  body.hiddenDates = [];\n  if (hiddenDates) {\n    if (Array.isArray(hiddenDates) == true) {\n      for (let i = 0; i < hiddenDates.length; i++) {\n        if (hiddenDates[i].repeat === undefined) {\n          const dateItem = {};\n          dateItem.start = moment(hiddenDates[i].start).toDate().valueOf();\n          dateItem.end = moment(hiddenDates[i].end).toDate().valueOf();\n          body.hiddenDates.push(dateItem);\n        }\n      }\n      body.hiddenDates.sort((a, b) => a.start - b.start); // sort by start time\n    }\n  }\n}\n\n/**\n * create new entrees for the repeating hidden dates\n *\n * @param {function} moment\n * @param {Object} body\n * @param {Array | Object} hiddenDates\n * @returns {null}\n */\nexport function updateHiddenDates(moment, body, hiddenDates) {\n  if (hiddenDates && !Array.isArray(hiddenDates)) {\n    return updateHiddenDates(moment, body, [hiddenDates])\n  }\n\n  if (hiddenDates && body.domProps.centerContainer.width !== undefined) {\n    convertHiddenOptions(moment, body, hiddenDates);\n\n    const start = moment(body.range.start);\n    const end = moment(body.range.end);\n\n    const totalRange = (body.range.end - body.range.start);\n    const pixelTime = totalRange / body.domProps.centerContainer.width;\n\n    for (let i = 0; i < hiddenDates.length; i++) {\n      if (hiddenDates[i].repeat !== undefined) {\n        const startDate = moment(hiddenDates[i].start);\n        let endDate = moment(hiddenDates[i].end);\n\n        if (startDate._d == \"Invalid Date\") {\n          throw new Error(`Supplied start date is not valid: ${hiddenDates[i].start}`);\n        }\n        if (endDate._d == \"Invalid Date\") {\n          throw new Error(`Supplied end date is not valid: ${hiddenDates[i].end}`);\n        }\n\n        const duration = endDate - startDate;\n        if (duration >= 4 * pixelTime) {\n\n          let offset = 0;\n          const runUntil = end.clone();\n          switch (hiddenDates[i].repeat) {\n            case \"daily\": // case of time\n              if (startDate.day() != endDate.day()) {\n                offset = 1;\n              }\n              startDate.dayOfYear(start.dayOfYear());\n              startDate.year(start.year());\n              startDate.subtract(7,'days');\n\n              endDate.dayOfYear(start.dayOfYear());\n              endDate.year(start.year());\n              endDate.subtract(7 - offset,'days');\n\n              runUntil.add(1, 'weeks');\n              break;\n            case \"weekly\": {\n              const dayOffset = endDate.diff(startDate,'days');\n              const day = startDate.day();\n\n              // set the start date to the range.start\n              startDate.date(start.date());\n              startDate.month(start.month());\n              startDate.year(start.year());\n              endDate = startDate.clone();\n\n              // force\n              startDate.day(day);\n              endDate.day(day);\n              endDate.add(dayOffset,'days');\n\n              startDate.subtract(1,'weeks');\n              endDate.subtract(1,'weeks');\n\n              runUntil.add(1, 'weeks');\n              break;\n            }\n            case \"monthly\":\n              if (startDate.month() != endDate.month()) {\n                offset = 1;\n              }\n              startDate.month(start.month());\n              startDate.year(start.year());\n              startDate.subtract(1,'months');\n\n              endDate.month(start.month());\n              endDate.year(start.year());\n              endDate.subtract(1,'months');\n              endDate.add(offset,'months');\n\n              runUntil.add(1, 'months');\n              break;\n            case \"yearly\":\n              if (startDate.year() != endDate.year()) {\n                offset = 1;\n              }\n              startDate.year(start.year());\n              startDate.subtract(1,'years');\n              endDate.year(start.year());\n              endDate.subtract(1,'years');\n              endDate.add(offset,'years');\n\n              runUntil.add(1, 'years');\n              break;\n            default:\n              console.log(\"Wrong repeat format, allowed are: daily, weekly, monthly, yearly. Given:\", hiddenDates[i].repeat);\n              return;\n          }\n          while (startDate < runUntil) {\n            body.hiddenDates.push({start: startDate.valueOf(), end: endDate.valueOf()});\n            switch (hiddenDates[i].repeat) {\n              case \"daily\":\n                startDate.add(1, 'days');\n                endDate.add(1, 'days');\n                break;\n              case \"weekly\":\n                startDate.add(1, 'weeks');\n                endDate.add(1, 'weeks');\n                break;\n              case \"monthly\":\n                startDate.add(1, 'months');\n                endDate.add(1, 'months');\n                break;\n              case \"yearly\":\n                startDate.add(1, 'y');\n                endDate.add(1, 'y');\n                break;\n              default:\n                console.log(\"Wrong repeat format, allowed are: daily, weekly, monthly, yearly. Given:\", hiddenDates[i].repeat);\n                return;\n            }\n          }\n          body.hiddenDates.push({start: startDate.valueOf(), end: endDate.valueOf()});\n        }\n      }\n    }\n    // remove duplicates, merge where possible\n    removeDuplicates(body);\n    // ensure the new positions are not on hidden dates\n    const startHidden = getIsHidden(body.range.start, body.hiddenDates);\n    const endHidden = getIsHidden(body.range.end,body.hiddenDates);\n    let rangeStart = body.range.start;\n    let rangeEnd = body.range.end;\n    if (startHidden.hidden == true) {rangeStart = body.range.startToFront == true ? startHidden.startDate - 1 : startHidden.endDate + 1;}\n    if (endHidden.hidden == true)   {rangeEnd   = body.range.endToFront == true ?   endHidden.startDate - 1   : endHidden.endDate + 1;}\n    if (startHidden.hidden == true || endHidden.hidden == true) {\n      body.range._applyRange(rangeStart, rangeEnd);\n    }\n  }\n\n}\n\n/**\n * remove duplicates from the hidden dates list. Duplicates are evil. They mess everything up.\n * Scales with N^2\n *\n * @param {Object} body\n */\nexport function removeDuplicates(body) {\n  const hiddenDates = body.hiddenDates;\n  const safeDates = [];\n  for (var i = 0; i < hiddenDates.length; i++) {\n    for (let j = 0; j < hiddenDates.length; j++) {\n      if (i != j && hiddenDates[j].remove != true && hiddenDates[i].remove != true) {\n        // j inside i\n        if (hiddenDates[j].start >= hiddenDates[i].start && hiddenDates[j].end <= hiddenDates[i].end) {\n          hiddenDates[j].remove = true;\n        }\n        // j start inside i\n        else if (hiddenDates[j].start >= hiddenDates[i].start && hiddenDates[j].start <= hiddenDates[i].end) {\n          hiddenDates[i].end = hiddenDates[j].end;\n          hiddenDates[j].remove = true;\n        }\n        // j end inside i\n        else if (hiddenDates[j].end >= hiddenDates[i].start && hiddenDates[j].end <= hiddenDates[i].end) {\n          hiddenDates[i].start = hiddenDates[j].start;\n          hiddenDates[j].remove = true;\n        }\n      }\n    }\n  }\n\n  for (i = 0; i < hiddenDates.length; i++) {\n    if (hiddenDates[i].remove !== true) {\n      safeDates.push(hiddenDates[i]);\n    }\n  }\n\n  body.hiddenDates = safeDates;\n  body.hiddenDates.sort((a, b) => a.start - b.start); // sort by start time\n}\n\n/**\n * Prints dates to console\n * @param {array} dates\n */\nexport function printDates(dates) {\n  for (let i =0; i < dates.length; i++) {\n    console.log(i, new Date(dates[i].start),new Date(dates[i].end), dates[i].start, dates[i].end, dates[i].remove);\n  }\n}\n\n/**\n * Used in TimeStep to avoid the hidden times.\n * @param {function} moment\n * @param {TimeStep} timeStep\n * @param {Date} previousTime\n */\nexport function stepOverHiddenDates(moment, timeStep, previousTime) {\n  let stepInHidden = false;\n  const currentValue = timeStep.current.valueOf();\n  for (let i = 0; i < timeStep.hiddenDates.length; i++) {\n    const startDate = timeStep.hiddenDates[i].start;\n    var endDate = timeStep.hiddenDates[i].end;\n    if (currentValue >= startDate && currentValue < endDate) {\n      stepInHidden = true;\n      break;\n    }\n  }\n\n  if (stepInHidden == true && currentValue < timeStep._end.valueOf() && currentValue != previousTime) {\n    const prevValue = moment(previousTime);\n    const newValue = moment(endDate);\n    //check if the next step should be major\n    if (prevValue.year() != newValue.year()) {timeStep.switchedYear = true;}\n    else if (prevValue.month() != newValue.month()) {timeStep.switchedMonth = true;}\n    else if (prevValue.dayOfYear() != newValue.dayOfYear()) {timeStep.switchedDay = true;}\n\n    timeStep.current = newValue;\n  }\n}\n\n///**\n// * Used in TimeStep to avoid the hidden times.\n// * @param timeStep\n// * @param previousTime\n// */\n//checkFirstStep = function(timeStep) {\n//  var stepInHidden = false;\n//  var currentValue = timeStep.current.valueOf();\n//  for (var i = 0; i < timeStep.hiddenDates.length; i++) {\n//    var startDate = timeStep.hiddenDates[i].start;\n//    var endDate = timeStep.hiddenDates[i].end;\n//    if (currentValue >= startDate && currentValue < endDate) {\n//      stepInHidden = true;\n//      break;\n//    }\n//  }\n//\n//  if (stepInHidden == true && currentValue <= timeStep._end.valueOf()) {\n//    var newValue = moment(endDate);\n//    timeStep.current = newValue.toDate();\n//  }\n//};\n\n/**\n * replaces the Core toScreen methods\n *\n * @param {timeline.Core} Core\n * @param {Date} time\n * @param {number} width\n * @returns {number}\n */\nexport function toScreen(Core, time, width) {\n  let conversion;\n  if (Core.body.hiddenDates.length == 0) {\n      conversion = Core.range.conversion(width);\n      return (time.valueOf() - conversion.offset) * conversion.scale;\n    } else {\n      const hidden = getIsHidden(time, Core.body.hiddenDates);\n      if (hidden.hidden == true) {\n        time = hidden.startDate;\n      }\n\n      const duration = getHiddenDurationBetween(Core.body.hiddenDates, Core.range.start, Core.range.end);\n      if (time < Core.range.start) {\n        conversion = Core.range.conversion(width, duration);\n        const hiddenBeforeStart = getHiddenDurationBeforeStart(Core.body.hiddenDates, time, conversion.offset);\n        time = Core.options.moment(time).toDate().valueOf();\n        time = time + hiddenBeforeStart;\n        return -(conversion.offset - time.valueOf()) * conversion.scale;\n        \n      } else if (time > Core.range.end) {\n        const rangeAfterEnd = {start: Core.range.start, end: time};\n        time = correctTimeForHidden(Core.options.moment, Core.body.hiddenDates, rangeAfterEnd, time);\n        conversion = Core.range.conversion(width, duration);\n        return (time.valueOf() - conversion.offset) * conversion.scale;\n\n      } else {\n        time = correctTimeForHidden(Core.options.moment, Core.body.hiddenDates, Core.range, time);\n        conversion = Core.range.conversion(width, duration);\n        return (time.valueOf() - conversion.offset) * conversion.scale;\n      }\n    }\n  }\n\n/**\n * Replaces the core toTime methods\n *\n * @param {timeline.Core} Core\n * @param {number} x\n * @param {number} width\n * @returns {Date}\n */\nexport function toTime(Core, x, width) {\n  if (Core.body.hiddenDates.length == 0) {\n    const conversion = Core.range.conversion(width);\n    return new Date(x / conversion.scale + conversion.offset);\n  }\n  else {\n    const hiddenDuration = getHiddenDurationBetween(Core.body.hiddenDates, Core.range.start, Core.range.end);\n    const totalDuration = Core.range.end - Core.range.start - hiddenDuration;\n    const partialDuration = totalDuration * x / width;\n    const accumulatedHiddenDuration = getAccumulatedHiddenDuration(Core.body.hiddenDates, Core.range, partialDuration);\n\n    return new Date(accumulatedHiddenDuration + partialDuration + Core.range.start);\n  }\n}\n\n/**\n * Support function\n *\n * @param {Array.<{start: Window.start, end: *}>} hiddenDates\n * @param {number} start\n * @param {number} end\n * @returns {number}\n */\nexport function getHiddenDurationBetween(hiddenDates, start, end) {\n  let duration = 0;\n  for (let i = 0; i < hiddenDates.length; i++) {\n    const startDate = hiddenDates[i].start;\n    const endDate = hiddenDates[i].end;\n    // if time after the cutout, and the\n    if (startDate >= start && endDate < end) {\n      duration += endDate - startDate;\n    }\n  }\n  return duration;\n}\n\n/**\n * Support function\n *\n * @param {Array.<{start: Window.start, end: *}>} hiddenDates\n * @param {number} start\n * @param {number} end\n * @returns {number}\n */\nexport function getHiddenDurationBeforeStart(hiddenDates, start, end) {\n  let duration = 0;\n  for (let i = 0; i < hiddenDates.length; i++) {\n    const startDate = hiddenDates[i].start;\n    const endDate = hiddenDates[i].end;\n\n    if (startDate >= start && endDate <= end) {\n      duration += endDate - startDate;\n    }\n  }\n  return duration;\n}\n\n/**\n * Support function\n * @param {function} moment\n * @param {Array.<{start: Window.start, end: *}>} hiddenDates\n * @param {{start: number, end: number}} range\n * @param {Date} time\n * @returns {number}\n */\nexport function correctTimeForHidden(moment, hiddenDates, range, time) {\n  time = moment(time).toDate().valueOf();\n  time -= getHiddenDurationBefore(moment, hiddenDates,range,time);\n  return time;\n}\n\n/**\n * Support function\n * @param {function} moment\n * @param {Array.<{start: Window.start, end: *}>} hiddenDates\n * @param {{start: number, end: number}} range\n * @param {Date} time\n * @returns {number}\n */\nexport function getHiddenDurationBefore(moment, hiddenDates, range, time) {\n  let timeOffset = 0;\n  time = moment(time).toDate().valueOf();\n\n  for (let i = 0; i < hiddenDates.length; i++) {\n    const startDate = hiddenDates[i].start;\n    const endDate = hiddenDates[i].end;\n    // if time after the cutout, and the\n    if (startDate >= range.start && endDate < range.end) {\n      if (time >= endDate) {\n        timeOffset += (endDate - startDate);\n      }\n    }\n  }\n  return timeOffset;\n}\n\n/**\n * sum the duration from start to finish, including the hidden duration,\n * until the required amount has been reached, return the accumulated hidden duration\n * @param {Array.<{start: Window.start, end: *}>} hiddenDates\n * @param {{start: number, end: number}} range\n * @param {number} [requiredDuration=0]\n * @returns {number}\n */\nexport function getAccumulatedHiddenDuration(hiddenDates, range, requiredDuration) {\n  let hiddenDuration = 0;\n  let duration = 0;\n  let previousPoint = range.start;\n  //printDates(hiddenDates)\n  for (let i = 0; i < hiddenDates.length; i++) {\n    const startDate = hiddenDates[i].start;\n    const endDate = hiddenDates[i].end;\n    // if time after the cutout, and the\n    if (startDate >= range.start && endDate < range.end) {\n      duration += startDate - previousPoint;\n      previousPoint = endDate;\n      if (duration >= requiredDuration) {\n        break;\n      }\n      else {\n        hiddenDuration += endDate - startDate;\n      }\n    }\n  }\n\n  return hiddenDuration;\n}\n\n/**\n * used to step over to either side of a hidden block. Correction is disabled on tablets, might be set to true\n * @param {Array.<{start: Window.start, end: *}>} hiddenDates\n * @param {Date} time\n * @param {number} direction\n * @param {boolean} correctionEnabled\n * @returns {Date|number}\n */\nexport function snapAwayFromHidden(hiddenDates, time, direction, correctionEnabled) {\n  const isHidden = getIsHidden(time, hiddenDates);\n  if (isHidden.hidden == true) {\n    if (direction < 0) {\n      if (correctionEnabled == true) {\n        return isHidden.startDate - (isHidden.endDate - time) - 1;\n      }\n      else {\n        return isHidden.startDate - 1;\n      }\n    }\n    else {\n      if (correctionEnabled == true) {\n        return isHidden.endDate + (time - isHidden.startDate) + 1;\n      }\n      else {\n        return isHidden.endDate + 1;\n      }\n    }\n  }\n  else {\n    return time;\n  }\n\n}\n\n/**\n * Check if a time is hidden\n *\n * @param {Date} time\n * @param {Array.<{start: Window.start, end: *}>} hiddenDates\n * @returns {{hidden: boolean, startDate: Window.start, endDate: *}}\n */\nexport function getIsHidden(time, hiddenDates) {\n  for (let i = 0; i < hiddenDates.length; i++) {\n    var startDate = hiddenDates[i].start;\n    var endDate = hiddenDates[i].end;\n\n    if (time >= startDate && time < endDate) { // if the start is entering a hidden zone\n      return {hidden: true, startDate, endDate};\n    }\n  }\n  return {hidden: false, startDate, endDate};\n}\n","import util from '../util';\nimport moment from '../module/moment';\nimport Component from './component/Component';\nimport * as DateUtil  from './DateUtil';\n\n/**\n * A Range controls a numeric range with a start and end value.\n * The Range adjusts the range based on mouse events or programmatic changes,\n * and triggers events when the range is changing or has been changed.\n */\nexport default class Range extends Component {\n  /**\n * @param {{dom: Object, domProps: Object, emitter: Emitter}} body\n * @param {Object} [options]    See description at Range.setOptions\n * @constructor Range\n * @extends Component\n */\n  constructor(body, options) {\n    super();\n    const now = moment().hours(0).minutes(0).seconds(0).milliseconds(0);\n    const start = now.clone().add(-3, 'days').valueOf();\n    const end = now.clone().add(3, 'days').valueOf(); \n    this.millisecondsPerPixelCache = undefined;\n    \n    if(options === undefined) {\n      this.start = start;\n      this.end = end;\n    } else {\n      this.start = options.start || start;\n      this.end = options.end || end\n    }\n\n    this.rolling = false;\n\n    this.body = body;\n    this.deltaDifference = 0;\n    this.scaleOffset = 0;\n    this.startToFront = false;\n    this.endToFront = true;\n\n    // default options\n    this.defaultOptions = {\n      rtl: false,\n      start: null,\n      end: null,\n      moment,\n      direction: 'horizontal', // 'horizontal' or 'vertical'\n      moveable: true,\n      zoomable: true,\n      min: null,\n      max: null,\n      zoomMin: 10,                                // milliseconds\n      zoomMax: 1000 * 60 * 60 * 24 * 365 * 10000,  // milliseconds\n      rollingMode: {\n        follow: false,\n        offset: 0.5\n      }\n    };\n    this.options = util.extend({}, this.defaultOptions);\n    this.props = {\n      touch: {}\n    };\n    this.animationTimer = null;\n\n    // drag listeners for dragging\n    this.body.emitter.on('panstart', this._onDragStart.bind(this));\n    this.body.emitter.on('panmove',  this._onDrag.bind(this));\n    this.body.emitter.on('panend',   this._onDragEnd.bind(this));\n\n    // mouse wheel for zooming\n    this.body.emitter.on('mousewheel', this._onMouseWheel.bind(this));\n\n    // pinch to zoom\n    this.body.emitter.on('touch', this._onTouch.bind(this));\n    this.body.emitter.on('pinch', this._onPinch.bind(this));\n\n    // on click of rolling mode button\n    this.body.dom.rollingModeBtn.addEventListener('click', this.startRolling.bind(this));\n\n    this.setOptions(options);\n  }\n\n  /**\n   * Set options for the range controller\n   * @param {Object} options      Available options:\n   *                              {number | Date | String} start  Start date for the range\n   *                              {number | Date | String} end    End date for the range\n   *                              {number} min    Minimum value for start\n   *                              {number} max    Maximum value for end\n   *                              {number} zoomMin    Set a minimum value for\n   *                                                  (end - start).\n   *                              {number} zoomMax    Set a maximum value for\n   *                                                  (end - start).\n   *                              {boolean} moveable Enable moving of the range\n   *                                                 by dragging. True by default\n   *                              {boolean} zoomable Enable zooming of the range\n   *                                                 by pinching/scrolling. True by default\n   */\n  setOptions(options) {\n    if (options) {\n      // copy the options that we know\n      const fields = [\n        'animation', 'direction', 'min', 'max', 'zoomMin', 'zoomMax', 'moveable', 'zoomable',\n        'moment', 'activate', 'hiddenDates', 'zoomKey', 'zoomFriction', 'rtl', 'showCurrentTime', 'rollingMode', 'horizontalScroll'\n      ];\n      util.selectiveExtend(fields, this.options, options);\n\n      if (options.rollingMode && options.rollingMode.follow) {\n        this.startRolling();\n      }\n      if ('start' in options || 'end' in options) {\n        // apply a new range. both start and end are optional\n        this.setRange(options.start, options.end);\n      }\n    }\n  }\n\n  /**\n   * Start auto refreshing the current time bar\n   */\n  startRolling() {\n    const me = this;\n\n    /**\n     *  Updates the current time.\n     */\n    function update () {\n      me.stopRolling();\n      me.rolling = true;\n\n\n      let interval = me.end - me.start;\n      const t = util.convert(new Date(), 'Date').valueOf();\n      const rollingModeOffset = me.options.rollingMode && me.options.rollingMode.offset || 0.5\n\n      const start = t - interval * (rollingModeOffset);\n      const end = t + interval * (1 - rollingModeOffset);\n\n      const options = {\n        animation: false\n      };\n      me.setRange(start, end, options);\n\n      // determine interval to refresh\n      const scale = me.conversion(me.body.domProps.center.width).scale;\n      interval = 1 / scale / 10;\n      if (interval < 30)   interval = 30;\n      if (interval > 1000) interval = 1000;\n\n      me.body.dom.rollingModeBtn.style.visibility = \"hidden\";\n      // start a renderTimer to adjust for the new time\n      me.currentTimeTimer = setTimeout(update, interval);\n    }\n\n    update();\n  }\n\n  /**\n   * Stop auto refreshing the current time bar\n   */\n  stopRolling() {\n    if (this.currentTimeTimer !== undefined) {\n      clearTimeout(this.currentTimeTimer);\n      this.rolling = false;\n      this.body.dom.rollingModeBtn.style.visibility = \"visible\";\n    }\n  }\n\n  /**\n   * Set a new start and end range\n   * @param {Date | number | string} start\n   * @param {Date | number | string} end\n   * @param {Object} options      Available options:\n   *                              {boolean | {duration: number, easingFunction: string}} [animation=false]\n   *                                    If true, the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   *                              {boolean} [byUser=false]\n   *                              {Event}  event  Mouse event\n   * @param {Function} callback     a callback function to be executed at the end of this function  \n   * @param {Function} frameCallback    a callback function executed each frame of the range animation.\n   *                                    The callback will be passed three parameters:\n   *                                    {number} easeCoefficient    an easing coefficent\n   *                                    {boolean} willDraw          If true the caller will redraw after the callback completes\n   *                                    {boolean} done              If true then animation is ending after the current frame\n   * @return {void}\n   */\n  setRange(start, end, options, callback, frameCallback) {\n    if (!options) {\n      options = {};\n    }\n    if (options.byUser !== true) {\n      options.byUser = false;\n    }\n    const me = this;\n    const finalStart = start != undefined ? util.convert(start, 'Date').valueOf() : null;\n    const finalEnd   = end != undefined   ? util.convert(end, 'Date').valueOf()   : null;\n    this._cancelAnimation();\n    this.millisecondsPerPixelCache = undefined;\n\n    if (options.animation) { // true or an Object\n      const initStart = this.start;\n      const initEnd = this.end;\n      const duration = (typeof options.animation === 'object' && 'duration' in options.animation) ? options.animation.duration : 500;\n      const easingName = (typeof options.animation === 'object' && 'easingFunction' in options.animation) ? options.animation.easingFunction : 'easeInOutQuad';\n      const easingFunction = util.easingFunctions[easingName];\n      if (!easingFunction) {\n        throw new Error(`Unknown easing function ${JSON.stringify(easingName)}. Choose from: ${Object.keys(util.easingFunctions).join(', ')}`);\n      }\n\n      const initTime = Date.now();\n      let anyChanged = false;\n\n      const next = () => {\n        if (!me.props.touch.dragging) {\n          const now = Date.now();\n          const time = now - initTime;\n          const ease = easingFunction(time / duration);\n          const done = time > duration;\n          const s = (done || finalStart === null) ? finalStart : initStart + (finalStart - initStart) * ease;\n          const e = (done || finalEnd   === null) ? finalEnd   : initEnd   + (finalEnd   - initEnd)   * ease;\n\n          changed = me._applyRange(s, e);\n          DateUtil.updateHiddenDates(me.options.moment, me.body, me.options.hiddenDates);\n          anyChanged = anyChanged || changed;\n\n          const params = {\n            start: new Date(me.start), \n            end: new Date(me.end), \n            byUser: options.byUser,\n            event: options.event\n          };\n\n          if (frameCallback) { frameCallback(ease, changed, done); }\n\n          if (changed) {          \n            me.body.emitter.emit('rangechange', params);\n          }\n\n          if (done) {\n            if (anyChanged) {\n              me.body.emitter.emit('rangechanged', params);\n              if (callback) { return callback() }\n            }\n          }\n          else {\n            // animate with as high as possible frame rate, leave 20 ms in between\n            // each to prevent the browser from blocking\n            me.animationTimer = setTimeout(next, 20);\n          }\n        }\n      };\n\n      return next();\n    }\n    else {\n      var changed = this._applyRange(finalStart, finalEnd);\n      DateUtil.updateHiddenDates(this.options.moment, this.body, this.options.hiddenDates);\n      if (changed) {\n        const params = {\n          start: new Date(this.start), \n          end: new Date(this.end), \n          byUser: options.byUser, \n          event: options.event\n        };\n\n        this.body.emitter.emit('rangechange', params);\n        clearTimeout( me.timeoutID );\n        me.timeoutID = setTimeout( () => {\n          me.body.emitter.emit('rangechanged', params);\n        }, 200 );\n        if (callback) { return callback() }\n      }\n    }\n  }\n\n  /**\n   * Get the number of milliseconds per pixel.\n   *\n   * @returns {undefined|number}\n   */\n  getMillisecondsPerPixel() {\n    if (this.millisecondsPerPixelCache === undefined) {\n      this.millisecondsPerPixelCache = (this.end - this.start) / this.body.dom.center.clientWidth;\n    }\n    return this.millisecondsPerPixelCache;\n  }\n\n  /**\n   * Stop an animation\n   * @private\n   */\n  _cancelAnimation() {\n    if (this.animationTimer) {\n      clearTimeout(this.animationTimer);\n      this.animationTimer = null;\n    }\n  }\n\n  /**\n   * Set a new start and end range. This method is the same as setRange, but\n   * does not trigger a range change and range changed event, and it returns\n   * true when the range is changed\n   * @param {number} [start]\n   * @param {number} [end]\n   * @return {boolean} changed\n   * @private\n   */\n  _applyRange(start, end) {\n    let newStart = (start != null) ? util.convert(start, 'Date').valueOf() : this.start;\n    let newEnd   = (end != null)   ? util.convert(end, 'Date').valueOf()   : this.end;\n    const max = (this.options.max != null) ? util.convert(this.options.max, 'Date').valueOf() : null;\n    const min = (this.options.min != null) ? util.convert(this.options.min, 'Date').valueOf() : null;\n    let diff;\n\n    // check for valid number\n    if (isNaN(newStart) || newStart === null) {\n      throw new Error(`Invalid start \"${start}\"`);\n    }\n    if (isNaN(newEnd) || newEnd === null) {\n      throw new Error(`Invalid end \"${end}\"`);\n    }\n\n    // prevent end < start\n    if (newEnd < newStart) {\n      newEnd = newStart;\n    }\n\n    // prevent start < min\n    if (min !== null) {\n      if (newStart < min) {\n        diff = (min - newStart);\n        newStart += diff;\n        newEnd += diff;\n\n        // prevent end > max\n        if (max != null) {\n          if (newEnd > max) {\n            newEnd = max;\n          }\n        }\n      }\n    }\n\n    // prevent end > max\n    if (max !== null) {\n      if (newEnd > max) {\n        diff = (newEnd - max);\n        newStart -= diff;\n        newEnd -= diff;\n\n        // prevent start < min\n        if (min != null) {\n          if (newStart < min) {\n            newStart = min;\n          }\n        }\n      }\n    }\n\n    // prevent (end-start) < zoomMin\n    if (this.options.zoomMin !== null) {\n      let zoomMin = parseFloat(this.options.zoomMin);\n      if (zoomMin < 0) {\n        zoomMin = 0;\n      }\n      if ((newEnd - newStart) < zoomMin) {\n        // compensate for a scale of 0.5 ms\n        const compensation = 0.5;\n        if ((this.end - this.start) === zoomMin && newStart >= this.start - compensation && newEnd <= this.end) {\n          // ignore this action, we are already zoomed to the minimum\n          newStart = this.start;\n          newEnd = this.end;\n        }\n        else {\n          // zoom to the minimum\n          diff = (zoomMin - (newEnd - newStart));\n          newStart -= diff / 2;\n          newEnd += diff / 2;\n        }\n      }\n    }\n\n    // prevent (end-start) > zoomMax\n    if (this.options.zoomMax !== null) {\n      let zoomMax = parseFloat(this.options.zoomMax);\n      if (zoomMax < 0) {\n        zoomMax = 0;\n      }\n\n      if ((newEnd - newStart) > zoomMax) {\n        if ((this.end - this.start) === zoomMax && newStart < this.start && newEnd > this.end) {\n          // ignore this action, we are already zoomed to the maximum\n          newStart = this.start;\n          newEnd = this.end;\n        }\n        else {\n          // zoom to the maximum\n          diff = ((newEnd - newStart) - zoomMax);\n          newStart += diff / 2;\n          newEnd -= diff / 2;\n        }\n      }\n    }\n\n    const changed = (this.start != newStart || this.end != newEnd);\n\n    // if the new range does NOT overlap with the old range, emit checkRangedItems to avoid not showing ranged items (ranged meaning has end time, not necessarily of type Range)\n    if (!((newStart >= this.start && newStart   <= this.end) || (newEnd   >= this.start && newEnd   <= this.end)) &&\n        !((this.start >= newStart && this.start <= newEnd)   || (this.end >= newStart   && this.end <= newEnd) )) {\n      this.body.emitter.emit('checkRangedItems');\n    }\n\n    this.start = newStart;\n    this.end = newEnd;\n    return changed;\n  }\n\n  /**\n   * Retrieve the current range.\n   * @return {Object} An object with start and end properties\n   */\n  getRange() {\n    return {\n      start: this.start,\n      end: this.end\n    };\n  }\n\n  /**\n   * Calculate the conversion offset and scale for current range, based on\n   * the provided width\n   * @param {number} width\n   * @param {number} [totalHidden=0]\n   * @returns {{offset: number, scale: number}} conversion\n   */\n  conversion(width, totalHidden) {\n    return Range.conversion(this.start, this.end, width, totalHidden);\n  }\n\n  /**\n   * Static method to calculate the conversion offset and scale for a range,\n   * based on the provided start, end, and width\n   * @param {number} start\n   * @param {number} end\n   * @param {number} width\n   * @param {number} [totalHidden=0]\n   * @returns {{offset: number, scale: number}} conversion\n   */\n  static conversion(start, end, width, totalHidden) {\n    if (totalHidden === undefined) {\n      totalHidden = 0;\n    }\n    if (width != 0 && (end - start != 0)) {\n      return {\n        offset: start,\n        scale: width / (end - start - totalHidden)\n      }\n    }\n    else {\n      return {\n        offset: 0,\n        scale: 1\n      };\n    }\n  }\n\n  /**\n   * Start dragging horizontally or vertically\n   * @param {Event} event\n   * @private\n   */\n  _onDragStart(event) {\n    this.deltaDifference = 0;\n    this.previousDelta = 0;\n\n    // only allow dragging when configured as movable\n    if (!this.options.moveable) return;\n\n    // only start dragging when the mouse is inside the current range\n    if (!this._isInsideRange(event)) return;\n\n    // refuse to drag when we where pinching to prevent the timeline make a jump\n    // when releasing the fingers in opposite order from the touch screen\n    if (!this.props.touch.allowDragging) return;\n\n    this.stopRolling();\n\n    this.props.touch.start = this.start;\n    this.props.touch.end = this.end;\n    this.props.touch.dragging = true;\n\n    if (this.body.dom.root) {\n      this.body.dom.root.style.cursor = 'move';\n    }\n  }\n\n  /**\n   * Perform dragging operation\n   * @param {Event} event\n   * @private\n   */\n  _onDrag(event) {\n    if (!event) return;\n\n    if (!this.props.touch.dragging) return;\n\n    // only allow dragging when configured as movable\n    if (!this.options.moveable) return;\n\n    // TODO: this may be redundant in hammerjs2\n    // refuse to drag when we where pinching to prevent the timeline make a jump\n    // when releasing the fingers in opposite order from the touch screen\n    if (!this.props.touch.allowDragging) return;\n\n    const direction = this.options.direction;\n    validateDirection(direction);\n    let delta = (direction == 'horizontal') ? event.deltaX : event.deltaY;\n    delta -= this.deltaDifference;\n    let interval = (this.props.touch.end - this.props.touch.start);\n\n    // normalize dragging speed if cutout is in between.\n    const duration = DateUtil.getHiddenDurationBetween(this.body.hiddenDates, this.start, this.end);\n    interval -= duration;\n\n    const width = (direction == 'horizontal') ? this.body.domProps.center.width : this.body.domProps.center.height;\n    let diffRange;\n    if (this.options.rtl) {\n      diffRange = delta / width * interval;\n    } else {\n      diffRange = -delta / width * interval;\n    }\n\n    const newStart = this.props.touch.start + diffRange;\n    const newEnd = this.props.touch.end + diffRange;\n\n    // snapping times away from hidden zones\n    const safeStart = DateUtil.snapAwayFromHidden(this.body.hiddenDates, newStart, this.previousDelta-delta, true);\n    const safeEnd = DateUtil.snapAwayFromHidden(this.body.hiddenDates, newEnd, this.previousDelta-delta, true);\n    if (safeStart != newStart || safeEnd != newEnd) {\n      this.deltaDifference += delta;\n      this.props.touch.start = safeStart;\n      this.props.touch.end = safeEnd;\n      this._onDrag(event);\n      return;\n    }\n\n    this.previousDelta = delta;\n    this._applyRange(newStart, newEnd);\n\n\n    const startDate = new Date(this.start);\n    const endDate = new Date(this.end);\n\n    // fire a rangechange event\n    this.body.emitter.emit('rangechange', {\n      start: startDate,\n      end:   endDate,\n      byUser: true,\n      event\n    });\n\n    // fire a panmove event\n    this.body.emitter.emit('panmove');\n  }\n\n  /**\n   * Stop dragging operation\n   * @param {event} event\n   * @private\n   */\n  _onDragEnd(event) {\n    if (!this.props.touch.dragging) return;\n\n    // only allow dragging when configured as movable\n    if (!this.options.moveable) return;\n\n    // TODO: this may be redundant in hammerjs2\n    // refuse to drag when we where pinching to prevent the timeline make a jump\n    // when releasing the fingers in opposite order from the touch screen\n    if (!this.props.touch.allowDragging) return;\n\n    this.props.touch.dragging = false;\n    if (this.body.dom.root) {\n      this.body.dom.root.style.cursor = 'auto';\n    }\n\n    // fire a rangechanged event\n    this.body.emitter.emit('rangechanged', {\n      start: new Date(this.start),\n      end:   new Date(this.end),\n      byUser: true,\n      event\n    });\n  }\n\n  /**\n   * Event handler for mouse wheel event, used to zoom\n   * Code from http://adomas.org/javascript-mouse-wheel/\n   * @param {Event} event\n   * @private\n   */\n  _onMouseWheel(event) {\n    // retrieve delta\n    let delta = 0;\n    if (event.wheelDelta) { /* IE/Opera. */\n      delta = event.wheelDelta / 120;\n    } else if (event.detail) { /* Mozilla case. */\n      // In Mozilla, sign of delta is different than in IE.\n      // Also, delta is multiple of 3.\n      delta = -event.detail / 3;\n    } else if (event.deltaY) {\n      delta = -event.deltaY / 3;\n    } \n\n    // don't allow zoom when the according key is pressed and the zoomKey option or not zoomable but movable\n    if ((this.options.zoomKey && !event[this.options.zoomKey] && this.options.zoomable) \n      || (!this.options.zoomable && this.options.moveable)) {\n      return;\n    }\n\n    // only allow zooming when configured as zoomable and moveable\n    if (!(this.options.zoomable && this.options.moveable)) return;\n    \n    // only zoom when the mouse is inside the current range\n    if (!this._isInsideRange(event)) return;\n\n    // If delta is nonzero, handle it.\n    // Basically, delta is now positive if wheel was scrolled up,\n    // and negative, if wheel was scrolled down.\n    if (delta) {\n      // perform the zoom action. Delta is normally 1 or -1\n\n      // adjust a negative delta such that zooming in with delta 0.1\n      // equals zooming out with a delta -0.1\n\n      const zoomFriction = this.options.zoomFriction || 5;\n      let scale;\n      if (delta < 0) {\n        scale = 1 - (delta / zoomFriction);\n      }\n      else {\n        scale = 1 / (1 + (delta / zoomFriction)) ;\n      }\n\n      // calculate center, the date to zoom around\n      let pointerDate;\n      if (this.rolling) {\n        const rollingModeOffset = this.options.rollingMode && this.options.rollingMode.offset || 0.5;\n        pointerDate = this.start + ((this.end - this.start) * rollingModeOffset);\n      } else {\n        const pointer = this.getPointer({x: event.clientX, y: event.clientY}, this.body.dom.center);\n        pointerDate = this._pointerToDate(pointer);\n      }\n      this.zoom(scale, pointerDate, delta, event);\n\n      // Prevent default actions caused by mouse wheel\n      // (else the page and timeline both scroll)\n      event.preventDefault();\n    }\n  }\n\n  /**\n   * Start of a touch gesture\n   * @param {Event} event\n   * @private\n   */\n  _onTouch(event) {  // eslint-disable-line no-unused-vars\n    this.props.touch.start = this.start;\n    this.props.touch.end = this.end;\n    this.props.touch.allowDragging = true;\n    this.props.touch.center = null;\n    this.props.touch.centerDate = null;\n    this.scaleOffset = 0;\n    this.deltaDifference = 0;\n    // Disable the browser default handling of this event.\n    util.preventDefault(event);\n  }\n\n  /**\n   * Handle pinch event\n   * @param {Event} event\n   * @private\n   */\n  _onPinch(event) {\n    // only allow zooming when configured as zoomable and moveable\n    if (!(this.options.zoomable && this.options.moveable)) return;\n\n    // Disable the browser default handling of this event.\n    util.preventDefault(event);\n\n    this.props.touch.allowDragging = false;\n\n    if (!this.props.touch.center) {\n      this.props.touch.center = this.getPointer(event.center, this.body.dom.center);\n      this.props.touch.centerDate = this._pointerToDate(this.props.touch.center);\n    }\n\n    this.stopRolling();\n    const scale = 1 / (event.scale + this.scaleOffset);\n    const centerDate = this.props.touch.centerDate;\n\n    const hiddenDuration = DateUtil.getHiddenDurationBetween(this.body.hiddenDates, this.start, this.end);\n    const hiddenDurationBefore = DateUtil.getHiddenDurationBefore(this.options.moment, this.body.hiddenDates, this, centerDate);\n    const hiddenDurationAfter = hiddenDuration - hiddenDurationBefore;\n\n    // calculate new start and end\n    let newStart = (centerDate - hiddenDurationBefore) + (this.props.touch.start - (centerDate - hiddenDurationBefore)) * scale;\n    let newEnd = (centerDate + hiddenDurationAfter) + (this.props.touch.end - (centerDate + hiddenDurationAfter)) * scale;\n\n    // snapping times away from hidden zones\n    this.startToFront = 1 - scale <= 0; // used to do the right auto correction with periodic hidden times\n    this.endToFront = scale - 1 <= 0;   // used to do the right auto correction with periodic hidden times\n    \n    const safeStart = DateUtil.snapAwayFromHidden(this.body.hiddenDates, newStart, 1 - scale, true);\n    const safeEnd = DateUtil.snapAwayFromHidden(this.body.hiddenDates, newEnd, scale - 1, true);\n    if (safeStart != newStart || safeEnd != newEnd) {\n      this.props.touch.start = safeStart;\n      this.props.touch.end = safeEnd;\n      this.scaleOffset = 1 - event.scale;\n      newStart = safeStart;\n      newEnd = safeEnd;\n    }\n    \n    const options = {\n      animation: false,\n      byUser: true,\n      event\n    };\n    this.setRange(newStart, newEnd, options);\n\n    this.startToFront = false; // revert to default\n    this.endToFront = true; // revert to default\n  }\n\n  /**\n   * Test whether the mouse from a mouse event is inside the visible window,\n   * between the current start and end date\n   * @param {Object} event\n   * @return {boolean} Returns true when inside the visible window\n   * @private\n   */\n  _isInsideRange(event) {\n    // calculate the time where the mouse is, check whether inside\n    // and no scroll action should happen.\n    const clientX = event.center ? event.center.x : event.clientX;\n    const centerContainerRect = this.body.dom.centerContainer.getBoundingClientRect();\n    const x = this.options.rtl ? clientX - centerContainerRect.left : centerContainerRect.right - clientX;\n    const time = this.body.util.toTime(x);\n\n    return time >= this.start && time <= this.end;\n  }\n\n  /**\n   * Helper function to calculate the center date for zooming\n   * @param {{x: number, y: number}} pointer\n   * @return {number} date\n   * @private\n   */\n  _pointerToDate(pointer) {\n    let conversion;\n    const direction = this.options.direction;\n\n    validateDirection(direction);\n\n    if (direction == 'horizontal') {\n      return this.body.util.toTime(pointer.x).valueOf();\n    }\n    else {\n      const height = this.body.domProps.center.height;\n      conversion = this.conversion(height);\n      return pointer.y / conversion.scale + conversion.offset;\n    }\n  }\n\n  /**\n   * Get the pointer location relative to the location of the dom element\n   * @param {{x: number, y: number}} touch\n   * @param {Element} element   HTML DOM element\n   * @return {{x: number, y: number}} pointer\n   * @private\n   */\n  getPointer(touch, element) {\n    const elementRect = element.getBoundingClientRect();\n    if (this.options.rtl) {\n      return {\n        x: elementRect.right - touch.x,\n        y: touch.y - elementRect.top\n      };\n    } else {\n      return {\n        x: touch.x - elementRect.left,\n        y: touch.y - elementRect.top\n      };\n    }\n  }\n\n  /**\n   * Zoom the range the given scale in or out. Start and end date will\n   * be adjusted, and the timeline will be redrawn. You can optionally give a\n   * date around which to zoom.\n   * For example, try scale = 0.9 or 1.1\n   * @param {number} scale      Scaling factor. Values above 1 will zoom out,\n   *                            values below 1 will zoom in.\n   * @param {number} [center]   Value representing a date around which will\n   *                            be zoomed.\n   * @param {number} delta\n   * @param {Event} event\n   */\n  zoom(scale, center, delta, event) {\n    // if centerDate is not provided, take it half between start Date and end Date\n    if (center == null) {\n      center = (this.start + this.end) / 2;\n    }\n\n    const hiddenDuration = DateUtil.getHiddenDurationBetween(this.body.hiddenDates, this.start, this.end);\n    const hiddenDurationBefore = DateUtil.getHiddenDurationBefore(this.options.moment, this.body.hiddenDates, this, center);\n    const hiddenDurationAfter = hiddenDuration - hiddenDurationBefore;\n\n    // calculate new start and end\n    let newStart = (center-hiddenDurationBefore) + (this.start - (center-hiddenDurationBefore)) * scale;\n    let newEnd   = (center+hiddenDurationAfter) + (this.end - (center+hiddenDurationAfter)) * scale;\n\n    // snapping times away from hidden zones\n    this.startToFront = delta > 0 ? false : true; // used to do the right autocorrection with periodic hidden times\n    this.endToFront = -delta  > 0 ? false : true; // used to do the right autocorrection with periodic hidden times\n    const safeStart = DateUtil.snapAwayFromHidden(this.body.hiddenDates, newStart, delta, true);\n    const safeEnd = DateUtil.snapAwayFromHidden(this.body.hiddenDates, newEnd, -delta, true);\n    if (safeStart != newStart || safeEnd != newEnd) {\n      newStart = safeStart;\n      newEnd = safeEnd;\n    }\n\n    const options = {\n      animation: false,\n      byUser: true,\n      event\n    };\n    this.setRange(newStart, newEnd, options);\n\n    this.startToFront = false; // revert to default\n    this.endToFront = true; // revert to default\n  }\n\n  /**\n   * Move the range with a given delta to the left or right. Start and end\n   * value will be adjusted. For example, try delta = 0.1 or -0.1\n   * @param {number}  delta     Moving amount. Positive value will move right,\n   *                            negative value will move left\n   */\n  move(delta) {\n    // zoom start Date and end Date relative to the centerDate\n    const diff = (this.end - this.start);\n\n    // apply new values\n    const newStart = this.start + diff * delta;\n    const newEnd = this.end + diff * delta;\n\n    // TODO: reckon with min and max range\n\n    this.start = newStart;\n    this.end = newEnd;\n  }\n\n  /**\n   * Move the range to a new center point\n   * @param {number} moveTo      New center point of the range\n   */\n  moveTo(moveTo) {\n    const center = (this.start + this.end) / 2;\n\n    const diff = center - moveTo;\n\n    // calculate new start and end\n    const newStart = this.start - diff;\n    const newEnd = this.end - diff;\n\n    const options = {\n      animation: false,\n      byUser: true,\n      event: null\n    };\n    this.setRange(newStart, newEnd, options);\n  }\n}\n\n/**\n * Test whether direction has a valid value\n * @param {string} direction    'horizontal' or 'vertical'\n */\nfunction validateDirection (direction) {\n  if (direction != 'horizontal' && direction != 'vertical') {\n    throw new TypeError(`Unknown direction \"${direction}\". Choose \"horizontal\" or \"vertical\".`);\n  }\n}\n","require('../modules/web.timers');\nvar path = require('../internals/path');\n\nmodule.exports = path.setInterval;\n","\r\n/**\r\n * Expose `Emitter`.\r\n */\r\n\r\nif (typeof module !== 'undefined') {\r\n  module.exports = Emitter;\r\n}\r\n\r\n/**\r\n * Initialize a new `Emitter`.\r\n *\r\n * @api public\r\n */\r\n\r\nfunction Emitter(obj) {\r\n  if (obj) return mixin(obj);\r\n};\r\n\r\n/**\r\n * Mixin the emitter properties.\r\n *\r\n * @param {Object} obj\r\n * @return {Object}\r\n * @api private\r\n */\r\n\r\nfunction mixin(obj) {\r\n  for (var key in Emitter.prototype) {\r\n    obj[key] = Emitter.prototype[key];\r\n  }\r\n  return obj;\r\n}\r\n\r\n/**\r\n * Listen on the given `event` with `fn`.\r\n *\r\n * @param {String} event\r\n * @param {Function} fn\r\n * @return {Emitter}\r\n * @api public\r\n */\r\n\r\nEmitter.prototype.on =\r\nEmitter.prototype.addEventListener = function(event, fn){\r\n  this._callbacks = this._callbacks || {};\r\n  (this._callbacks['$' + event] = this._callbacks['$' + event] || [])\r\n    .push(fn);\r\n  return this;\r\n};\r\n\r\n/**\r\n * Adds an `event` listener that will be invoked a single\r\n * time then automatically removed.\r\n *\r\n * @param {String} event\r\n * @param {Function} fn\r\n * @return {Emitter}\r\n * @api public\r\n */\r\n\r\nEmitter.prototype.once = function(event, fn){\r\n  function on() {\r\n    this.off(event, on);\r\n    fn.apply(this, arguments);\r\n  }\r\n\r\n  on.fn = fn;\r\n  this.on(event, on);\r\n  return this;\r\n};\r\n\r\n/**\r\n * Remove the given callback for `event` or all\r\n * registered callbacks.\r\n *\r\n * @param {String} event\r\n * @param {Function} fn\r\n * @return {Emitter}\r\n * @api public\r\n */\r\n\r\nEmitter.prototype.off =\r\nEmitter.prototype.removeListener =\r\nEmitter.prototype.removeAllListeners =\r\nEmitter.prototype.removeEventListener = function(event, fn){\r\n  this._callbacks = this._callbacks || {};\r\n\r\n  // all\r\n  if (0 == arguments.length) {\r\n    this._callbacks = {};\r\n    return this;\r\n  }\r\n\r\n  // specific event\r\n  var callbacks = this._callbacks['$' + event];\r\n  if (!callbacks) return this;\r\n\r\n  // remove all handlers\r\n  if (1 == arguments.length) {\r\n    delete this._callbacks['$' + event];\r\n    return this;\r\n  }\r\n\r\n  // remove specific handler\r\n  var cb;\r\n  for (var i = 0; i < callbacks.length; i++) {\r\n    cb = callbacks[i];\r\n    if (cb === fn || cb.fn === fn) {\r\n      callbacks.splice(i, 1);\r\n      break;\r\n    }\r\n  }\r\n\r\n  // Remove event specific arrays for event types that no\r\n  // one is subscribed for to avoid memory leak.\r\n  if (callbacks.length === 0) {\r\n    delete this._callbacks['$' + event];\r\n  }\r\n\r\n  return this;\r\n};\r\n\r\n/**\r\n * Emit `event` with the given args.\r\n *\r\n * @param {String} event\r\n * @param {Mixed} ...\r\n * @return {Emitter}\r\n */\r\n\r\nEmitter.prototype.emit = function(event){\r\n  this._callbacks = this._callbacks || {};\r\n\r\n  var args = new Array(arguments.length - 1)\r\n    , callbacks = this._callbacks['$' + event];\r\n\r\n  for (var i = 1; i < arguments.length; i++) {\r\n    args[i - 1] = arguments[i];\r\n  }\r\n\r\n  if (callbacks) {\r\n    callbacks = callbacks.slice(0);\r\n    for (var i = 0, len = callbacks.length; i < len; ++i) {\r\n      callbacks[i].apply(this, args);\r\n    }\r\n  }\r\n\r\n  return this;\r\n};\r\n\r\n/**\r\n * Return array of callbacks for `event`.\r\n *\r\n * @param {String} event\r\n * @return {Array}\r\n * @api public\r\n */\r\n\r\nEmitter.prototype.listeners = function(event){\r\n  this._callbacks = this._callbacks || {};\r\n  return this._callbacks['$' + event] || [];\r\n};\r\n\r\n/**\r\n * Check if this emitter has `event` handlers.\r\n *\r\n * @param {String} event\r\n * @return {Boolean}\r\n * @api public\r\n */\r\n\r\nEmitter.prototype.hasListeners = function(event){\r\n  return !! this.listeners(event).length;\r\n};\r\n","'use strict';\n\n(function (factory) {\n  if (typeof define === 'function' && define.amd) {\n    // AMD. Register as an anonymous module.\n    define([], factory);\n  } else if (typeof exports === 'object') {\n    // Node. Does not work with strict CommonJS, but\n    // only CommonJS-like environments that support module.exports,\n    // like Node.\n    module.exports = factory();\n  } else {\n    // Browser globals (root is window)\n    window.propagating = factory();\n  }\n}(function () {\n  var _firstTarget = null; // singleton, will contain the target element where the touch event started\n\n  /**\n   * Extend an Hammer.js instance with event propagation.\n   *\n   * Features:\n   * - Events emitted by hammer will propagate in order from child to parent\n   *   elements.\n   * - Events are extended with a function `event.stopPropagation()` to stop\n   *   propagation to parent elements.\n   * - An option `preventDefault` to stop all default browser behavior.\n   *\n   * Usage:\n   *   var hammer = propagatingHammer(new Hammer(element));\n   *   var hammer = propagatingHammer(new Hammer(element), {preventDefault: true});\n   *\n   * @param {Hammer.Manager} hammer   An hammer instance.\n   * @param {Object} [options]        Available options:\n   *                                  - `preventDefault: true | false | 'mouse' | 'touch' | 'pen'`.\n   *                                    Enforce preventing the default browser behavior.\n   *                                    Cannot be set to `false`.\n   * @return {Hammer.Manager} Returns the same hammer instance with extended\n   *                          functionality\n   */\n  return function propagating(hammer, options) {\n    var _options = options || {\n      preventDefault: false\n    };\n\n    if (hammer.Manager) {\n      // This looks like the Hammer constructor.\n      // Overload the constructors with our own.\n      var Hammer = hammer;\n\n      var PropagatingHammer = function(element, options) {\n        var o = Object.create(_options);\n        if (options) Hammer.assign(o, options);\n        return propagating(new Hammer(element, o), o);\n      };\n      Hammer.assign(PropagatingHammer, Hammer);\n\n      PropagatingHammer.Manager = function (element, options) {\n        var o = Object.create(_options);\n        if (options) Hammer.assign(o, options);\n        return propagating(new Hammer.Manager(element, o), o);\n      };\n\n      return PropagatingHammer;\n    }\n\n    // create a wrapper object which will override the functions\n    // `on`, `off`, `destroy`, and `emit` of the hammer instance\n    var wrapper = Object.create(hammer);\n\n    // attach to DOM element\n    var element = hammer.element;\n\n    if(!element.hammer) element.hammer = [];\n    element.hammer.push(wrapper);\n\n    // register an event to catch the start of a gesture and store the\n    // target in a singleton\n    hammer.on('hammer.input', function (event) {\n      if (_options.preventDefault === true || (_options.preventDefault === event.pointerType)) {\n        event.preventDefault();\n      }\n      if (event.isFirst) {\n        _firstTarget = event.target;\n      }\n    });\n\n    /** @type {Object.<String, Array.<function>>} */\n    wrapper._handlers = {};\n\n    /**\n     * Register a handler for one or multiple events\n     * @param {String} events    A space separated string with events\n     * @param {function} handler A callback function, called as handler(event)\n     * @returns {Hammer.Manager} Returns the hammer instance\n     */\n    wrapper.on = function (events, handler) {\n      // register the handler\n      split(events).forEach(function (event) {\n        var _handlers = wrapper._handlers[event];\n        if (!_handlers) {\n          wrapper._handlers[event] = _handlers = [];\n\n          // register the static, propagated handler\n          hammer.on(event, propagatedHandler);\n        }\n        _handlers.push(handler);\n      });\n\n      return wrapper;\n    };\n\n    /**\n     * Unregister a handler for one or multiple events\n     * @param {String} events      A space separated string with events\n     * @param {function} [handler] Optional. The registered handler. If not\n     *                             provided, all handlers for given events\n     *                             are removed.\n     * @returns {Hammer.Manager}   Returns the hammer instance\n     */\n    wrapper.off = function (events, handler) {\n      // unregister the handler\n      split(events).forEach(function (event) {\n        var _handlers = wrapper._handlers[event];\n        if (_handlers) {\n          _handlers = handler ? _handlers.filter(function (h) {\n            return h !== handler;\n          }) : [];\n\n          if (_handlers.length > 0) {\n            wrapper._handlers[event] = _handlers;\n          }\n          else {\n            // remove static, propagated handler\n            hammer.off(event, propagatedHandler);\n            delete wrapper._handlers[event];\n          }\n        }\n      });\n\n      return wrapper;\n    };\n\n    /**\n     * Emit to the event listeners\n     * @param {string} eventType\n     * @param {Event} event\n     */\n    wrapper.emit = function(eventType, event) {\n      _firstTarget = event.target;\n      hammer.emit(eventType, event);\n    };\n\n    wrapper.destroy = function () {\n      // Detach from DOM element\n      var hammers = hammer.element.hammer;\n      var idx = hammers.indexOf(wrapper);\n      if(idx !== -1) hammers.splice(idx,1);\n      if(!hammers.length) delete hammer.element.hammer;\n\n      // clear all handlers\n      wrapper._handlers = {};\n\n      // call original hammer destroy\n      hammer.destroy();\n    };\n\n    // split a string with space separated words\n    function split(events) {\n      return events.match(/[^ ]+/g);\n    }\n\n    /**\n     * A static event handler, applying event propagation.\n     * @param {Object} event\n     */\n    function propagatedHandler(event) {\n      // let only a single hammer instance handle this event\n      if (event.type !== 'hammer.input') {\n        // it is possible that the same srcEvent is used with multiple hammer events,\n        // we keep track on which events are handled in an object _handled\n        if (!event.srcEvent._handled) {\n          event.srcEvent._handled = {};\n        }\n\n        if (event.srcEvent._handled[event.type]) {\n          return;\n        }\n        else {\n          event.srcEvent._handled[event.type] = true;\n        }\n      }\n\n      // attach a stopPropagation function to the event\n      var stopped = false;\n      event.stopPropagation = function () {\n        stopped = true;\n      };\n\n      //wrap the srcEvent's stopPropagation to also stop hammer propagation:\n      var srcStop = event.srcEvent.stopPropagation.bind(event.srcEvent);\n      if(typeof srcStop == \"function\") {\n        event.srcEvent.stopPropagation = function(){\n          srcStop();\n          event.stopPropagation();\n        }\n      }\n\n      // attach firstTarget property to the event\n      event.firstTarget = _firstTarget;\n\n      // propagate over all elements (until stopped)\n      var elem = _firstTarget;\n      while (elem && !stopped) {\n        var elemHammer = elem.hammer;\n        if(elemHammer){\n          var _handlers;\n          for(var k = 0; k < elemHammer.length; k++){\n            _handlers = elemHammer[k]._handlers[event.type];\n            if(_handlers) for (var i = 0; i < _handlers.length && !stopped; i++) {\n              _handlers[i](event);\n            }\n          }\n        }\n        elem = elem.parentNode;\n      }\n    }\n\n    return wrapper;\n  };\n}));\n","/*! Hammer.JS - v2.0.17-rc - 2019-12-16\n * http://naver.github.io/egjs\n *\n * Forked By Naver egjs\n * Copyright (c) hammerjs\n * Licensed under the MIT license */\nfunction _extends() {\n  _extends = Object.assign || function (target) {\n    for (var i = 1; i < arguments.length; i++) {\n      var source = arguments[i];\n\n      for (var key in source) {\n        if (Object.prototype.hasOwnProperty.call(source, key)) {\n          target[key] = source[key];\n        }\n      }\n    }\n\n    return target;\n  };\n\n  return _extends.apply(this, arguments);\n}\n\nfunction _inheritsLoose(subClass, superClass) {\n  subClass.prototype = Object.create(superClass.prototype);\n  subClass.prototype.constructor = subClass;\n  subClass.__proto__ = superClass;\n}\n\nfunction _assertThisInitialized(self) {\n  if (self === void 0) {\n    throw new ReferenceError(\"this hasn't been initialised - super() hasn't been called\");\n  }\n\n  return self;\n}\n\n/**\n * @private\n * extend object.\n * means that properties in dest will be overwritten by the ones in src.\n * @param {Object} target\n * @param {...Object} objects_to_assign\n * @returns {Object} target\n */\nvar assign;\n\nif (typeof Object.assign !== 'function') {\n  assign = function assign(target) {\n    if (target === undefined || target === null) {\n      throw new TypeError('Cannot convert undefined or null to object');\n    }\n\n    var output = Object(target);\n\n    for (var index = 1; index < arguments.length; index++) {\n      var source = arguments[index];\n\n      if (source !== undefined && source !== null) {\n        for (var nextKey in source) {\n          if (source.hasOwnProperty(nextKey)) {\n            output[nextKey] = source[nextKey];\n          }\n        }\n      }\n    }\n\n    return output;\n  };\n} else {\n  assign = Object.assign;\n}\n\nvar assign$1 = assign;\n\nvar VENDOR_PREFIXES = ['', 'webkit', 'Moz', 'MS', 'ms', 'o'];\nvar TEST_ELEMENT = typeof document === \"undefined\" ? {\n  style: {}\n} : document.createElement('div');\nvar TYPE_FUNCTION = 'function';\nvar round = Math.round,\n    abs = Math.abs;\nvar now = Date.now;\n\n/**\n * @private\n * get the prefixed property\n * @param {Object} obj\n * @param {String} property\n * @returns {String|Undefined} prefixed\n */\n\nfunction prefixed(obj, property) {\n  var prefix;\n  var prop;\n  var camelProp = property[0].toUpperCase() + property.slice(1);\n  var i = 0;\n\n  while (i < VENDOR_PREFIXES.length) {\n    prefix = VENDOR_PREFIXES[i];\n    prop = prefix ? prefix + camelProp : property;\n\n    if (prop in obj) {\n      return prop;\n    }\n\n    i++;\n  }\n\n  return undefined;\n}\n\n/* eslint-disable no-new-func, no-nested-ternary */\nvar win;\n\nif (typeof window === \"undefined\") {\n  // window is undefined in node.js\n  win = {};\n} else {\n  win = window;\n}\n\nvar PREFIXED_TOUCH_ACTION = prefixed(TEST_ELEMENT.style, 'touchAction');\nvar NATIVE_TOUCH_ACTION = PREFIXED_TOUCH_ACTION !== undefined;\nfunction getTouchActionProps() {\n  if (!NATIVE_TOUCH_ACTION) {\n    return false;\n  }\n\n  var touchMap = {};\n  var cssSupports = win.CSS && win.CSS.supports;\n  ['auto', 'manipulation', 'pan-y', 'pan-x', 'pan-x pan-y', 'none'].forEach(function (val) {\n    // If css.supports is not supported but there is native touch-action assume it supports\n    // all values. This is the case for IE 10 and 11.\n    return touchMap[val] = cssSupports ? win.CSS.supports('touch-action', val) : true;\n  });\n  return touchMap;\n}\n\nvar TOUCH_ACTION_COMPUTE = 'compute';\nvar TOUCH_ACTION_AUTO = 'auto';\nvar TOUCH_ACTION_MANIPULATION = 'manipulation'; // not implemented\n\nvar TOUCH_ACTION_NONE = 'none';\nvar TOUCH_ACTION_PAN_X = 'pan-x';\nvar TOUCH_ACTION_PAN_Y = 'pan-y';\nvar TOUCH_ACTION_MAP = getTouchActionProps();\n\nvar MOBILE_REGEX = /mobile|tablet|ip(ad|hone|od)|android/i;\nvar SUPPORT_TOUCH = 'ontouchstart' in win;\nvar SUPPORT_POINTER_EVENTS = prefixed(win, 'PointerEvent') !== undefined;\nvar SUPPORT_ONLY_TOUCH = SUPPORT_TOUCH && MOBILE_REGEX.test(navigator.userAgent);\nvar INPUT_TYPE_TOUCH = 'touch';\nvar INPUT_TYPE_PEN = 'pen';\nvar INPUT_TYPE_MOUSE = 'mouse';\nvar INPUT_TYPE_KINECT = 'kinect';\nvar COMPUTE_INTERVAL = 25;\nvar INPUT_START = 1;\nvar INPUT_MOVE = 2;\nvar INPUT_END = 4;\nvar INPUT_CANCEL = 8;\nvar DIRECTION_NONE = 1;\nvar DIRECTION_LEFT = 2;\nvar DIRECTION_RIGHT = 4;\nvar DIRECTION_UP = 8;\nvar DIRECTION_DOWN = 16;\nvar DIRECTION_HORIZONTAL = DIRECTION_LEFT | DIRECTION_RIGHT;\nvar DIRECTION_VERTICAL = DIRECTION_UP | DIRECTION_DOWN;\nvar DIRECTION_ALL = DIRECTION_HORIZONTAL | DIRECTION_VERTICAL;\nvar PROPS_XY = ['x', 'y'];\nvar PROPS_CLIENT_XY = ['clientX', 'clientY'];\n\n/**\n * @private\n * walk objects and arrays\n * @param {Object} obj\n * @param {Function} iterator\n * @param {Object} context\n */\nfunction each(obj, iterator, context) {\n  var i;\n\n  if (!obj) {\n    return;\n  }\n\n  if (obj.forEach) {\n    obj.forEach(iterator, context);\n  } else if (obj.length !== undefined) {\n    i = 0;\n\n    while (i < obj.length) {\n      iterator.call(context, obj[i], i, obj);\n      i++;\n    }\n  } else {\n    for (i in obj) {\n      obj.hasOwnProperty(i) && iterator.call(context, obj[i], i, obj);\n    }\n  }\n}\n\n/**\n * @private\n * let a boolean value also be a function that must return a boolean\n * this first item in args will be used as the context\n * @param {Boolean|Function} val\n * @param {Array} [args]\n * @returns {Boolean}\n */\n\nfunction boolOrFn(val, args) {\n  if (typeof val === TYPE_FUNCTION) {\n    return val.apply(args ? args[0] || undefined : undefined, args);\n  }\n\n  return val;\n}\n\n/**\n * @private\n * small indexOf wrapper\n * @param {String} str\n * @param {String} find\n * @returns {Boolean} found\n */\nfunction inStr(str, find) {\n  return str.indexOf(find) > -1;\n}\n\n/**\n * @private\n * when the touchActions are collected they are not a valid value, so we need to clean things up. *\n * @param {String} actions\n * @returns {*}\n */\n\nfunction cleanTouchActions(actions) {\n  // none\n  if (inStr(actions, TOUCH_ACTION_NONE)) {\n    return TOUCH_ACTION_NONE;\n  }\n\n  var hasPanX = inStr(actions, TOUCH_ACTION_PAN_X);\n  var hasPanY = inStr(actions, TOUCH_ACTION_PAN_Y); // if both pan-x and pan-y are set (different recognizers\n  // for different directions, e.g. horizontal pan but vertical swipe?)\n  // we need none (as otherwise with pan-x pan-y combined none of these\n  // recognizers will work, since the browser would handle all panning\n\n  if (hasPanX && hasPanY) {\n    return TOUCH_ACTION_NONE;\n  } // pan-x OR pan-y\n\n\n  if (hasPanX || hasPanY) {\n    return hasPanX ? TOUCH_ACTION_PAN_X : TOUCH_ACTION_PAN_Y;\n  } // manipulation\n\n\n  if (inStr(actions, TOUCH_ACTION_MANIPULATION)) {\n    return TOUCH_ACTION_MANIPULATION;\n  }\n\n  return TOUCH_ACTION_AUTO;\n}\n\n/**\n * @private\n * Touch Action\n * sets the touchAction property or uses the js alternative\n * @param {Manager} manager\n * @param {String} value\n * @constructor\n */\n\nvar TouchAction =\n/*#__PURE__*/\nfunction () {\n  function TouchAction(manager, value) {\n    this.manager = manager;\n    this.set(value);\n  }\n  /**\n   * @private\n   * set the touchAction value on the element or enable the polyfill\n   * @param {String} value\n   */\n\n\n  var _proto = TouchAction.prototype;\n\n  _proto.set = function set(value) {\n    // find out the touch-action by the event handlers\n    if (value === TOUCH_ACTION_COMPUTE) {\n      value = this.compute();\n    }\n\n    if (NATIVE_TOUCH_ACTION && this.manager.element.style && TOUCH_ACTION_MAP[value]) {\n      this.manager.element.style[PREFIXED_TOUCH_ACTION] = value;\n    }\n\n    this.actions = value.toLowerCase().trim();\n  };\n  /**\n   * @private\n   * just re-set the touchAction value\n   */\n\n\n  _proto.update = function update() {\n    this.set(this.manager.options.touchAction);\n  };\n  /**\n   * @private\n   * compute the value for the touchAction property based on the recognizer's settings\n   * @returns {String} value\n   */\n\n\n  _proto.compute = function compute() {\n    var actions = [];\n    each(this.manager.recognizers, function (recognizer) {\n      if (boolOrFn(recognizer.options.enable, [recognizer])) {\n        actions = actions.concat(recognizer.getTouchAction());\n      }\n    });\n    return cleanTouchActions(actions.join(' '));\n  };\n  /**\n   * @private\n   * this method is called on each input cycle and provides the preventing of the browser behavior\n   * @param {Object} input\n   */\n\n\n  _proto.preventDefaults = function preventDefaults(input) {\n    var srcEvent = input.srcEvent;\n    var direction = input.offsetDirection; // if the touch action did prevented once this session\n\n    if (this.manager.session.prevented) {\n      srcEvent.preventDefault();\n      return;\n    }\n\n    var actions = this.actions;\n    var hasNone = inStr(actions, TOUCH_ACTION_NONE) && !TOUCH_ACTION_MAP[TOUCH_ACTION_NONE];\n    var hasPanY = inStr(actions, TOUCH_ACTION_PAN_Y) && !TOUCH_ACTION_MAP[TOUCH_ACTION_PAN_Y];\n    var hasPanX = inStr(actions, TOUCH_ACTION_PAN_X) && !TOUCH_ACTION_MAP[TOUCH_ACTION_PAN_X];\n\n    if (hasNone) {\n      // do not prevent defaults if this is a tap gesture\n      var isTapPointer = input.pointers.length === 1;\n      var isTapMovement = input.distance < 2;\n      var isTapTouchTime = input.deltaTime < 250;\n\n      if (isTapPointer && isTapMovement && isTapTouchTime) {\n        return;\n      }\n    }\n\n    if (hasPanX && hasPanY) {\n      // `pan-x pan-y` means browser handles all scrolling/panning, do not prevent\n      return;\n    }\n\n    if (hasNone || hasPanY && direction & DIRECTION_HORIZONTAL || hasPanX && direction & DIRECTION_VERTICAL) {\n      return this.preventSrc(srcEvent);\n    }\n  };\n  /**\n   * @private\n   * call preventDefault to prevent the browser's default behavior (scrolling in most cases)\n   * @param {Object} srcEvent\n   */\n\n\n  _proto.preventSrc = function preventSrc(srcEvent) {\n    this.manager.session.prevented = true;\n    srcEvent.preventDefault();\n  };\n\n  return TouchAction;\n}();\n\n/**\n * @private\n * find if a node is in the given parent\n * @method hasParent\n * @param {HTMLElement} node\n * @param {HTMLElement} parent\n * @return {Boolean} found\n */\nfunction hasParent(node, parent) {\n  while (node) {\n    if (node === parent) {\n      return true;\n    }\n\n    node = node.parentNode;\n  }\n\n  return false;\n}\n\n/**\n * @private\n * get the center of all the pointers\n * @param {Array} pointers\n * @return {Object} center contains `x` and `y` properties\n */\n\nfunction getCenter(pointers) {\n  var pointersLength = pointers.length; // no need to loop when only one touch\n\n  if (pointersLength === 1) {\n    return {\n      x: round(pointers[0].clientX),\n      y: round(pointers[0].clientY)\n    };\n  }\n\n  var x = 0;\n  var y = 0;\n  var i = 0;\n\n  while (i < pointersLength) {\n    x += pointers[i].clientX;\n    y += pointers[i].clientY;\n    i++;\n  }\n\n  return {\n    x: round(x / pointersLength),\n    y: round(y / pointersLength)\n  };\n}\n\n/**\n * @private\n * create a simple clone from the input used for storage of firstInput and firstMultiple\n * @param {Object} input\n * @returns {Object} clonedInputData\n */\n\nfunction simpleCloneInputData(input) {\n  // make a simple copy of the pointers because we will get a reference if we don't\n  // we only need clientXY for the calculations\n  var pointers = [];\n  var i = 0;\n\n  while (i < input.pointers.length) {\n    pointers[i] = {\n      clientX: round(input.pointers[i].clientX),\n      clientY: round(input.pointers[i].clientY)\n    };\n    i++;\n  }\n\n  return {\n    timeStamp: now(),\n    pointers: pointers,\n    center: getCenter(pointers),\n    deltaX: input.deltaX,\n    deltaY: input.deltaY\n  };\n}\n\n/**\n * @private\n * calculate the absolute distance between two points\n * @param {Object} p1 {x, y}\n * @param {Object} p2 {x, y}\n * @param {Array} [props] containing x and y keys\n * @return {Number} distance\n */\n\nfunction getDistance(p1, p2, props) {\n  if (!props) {\n    props = PROPS_XY;\n  }\n\n  var x = p2[props[0]] - p1[props[0]];\n  var y = p2[props[1]] - p1[props[1]];\n  return Math.sqrt(x * x + y * y);\n}\n\n/**\n * @private\n * calculate the angle between two coordinates\n * @param {Object} p1\n * @param {Object} p2\n * @param {Array} [props] containing x and y keys\n * @return {Number} angle\n */\n\nfunction getAngle(p1, p2, props) {\n  if (!props) {\n    props = PROPS_XY;\n  }\n\n  var x = p2[props[0]] - p1[props[0]];\n  var y = p2[props[1]] - p1[props[1]];\n  return Math.atan2(y, x) * 180 / Math.PI;\n}\n\n/**\n * @private\n * get the direction between two points\n * @param {Number} x\n * @param {Number} y\n * @return {Number} direction\n */\n\nfunction getDirection(x, y) {\n  if (x === y) {\n    return DIRECTION_NONE;\n  }\n\n  if (abs(x) >= abs(y)) {\n    return x < 0 ? DIRECTION_LEFT : DIRECTION_RIGHT;\n  }\n\n  return y < 0 ? DIRECTION_UP : DIRECTION_DOWN;\n}\n\nfunction computeDeltaXY(session, input) {\n  var center = input.center; // let { offsetDelta:offset = {}, prevDelta = {}, prevInput = {} } = session;\n  // jscs throwing error on defalut destructured values and without defaults tests fail\n\n  var offset = session.offsetDelta || {};\n  var prevDelta = session.prevDelta || {};\n  var prevInput = session.prevInput || {};\n\n  if (input.eventType === INPUT_START || prevInput.eventType === INPUT_END) {\n    prevDelta = session.prevDelta = {\n      x: prevInput.deltaX || 0,\n      y: prevInput.deltaY || 0\n    };\n    offset = session.offsetDelta = {\n      x: center.x,\n      y: center.y\n    };\n  }\n\n  input.deltaX = prevDelta.x + (center.x - offset.x);\n  input.deltaY = prevDelta.y + (center.y - offset.y);\n}\n\n/**\n * @private\n * calculate the velocity between two points. unit is in px per ms.\n * @param {Number} deltaTime\n * @param {Number} x\n * @param {Number} y\n * @return {Object} velocity `x` and `y`\n */\nfunction getVelocity(deltaTime, x, y) {\n  return {\n    x: x / deltaTime || 0,\n    y: y / deltaTime || 0\n  };\n}\n\n/**\n * @private\n * calculate the scale factor between two pointersets\n * no scale is 1, and goes down to 0 when pinched together, and bigger when pinched out\n * @param {Array} start array of pointers\n * @param {Array} end array of pointers\n * @return {Number} scale\n */\n\nfunction getScale(start, end) {\n  return getDistance(end[0], end[1], PROPS_CLIENT_XY) / getDistance(start[0], start[1], PROPS_CLIENT_XY);\n}\n\n/**\n * @private\n * calculate the rotation degrees between two pointersets\n * @param {Array} start array of pointers\n * @param {Array} end array of pointers\n * @return {Number} rotation\n */\n\nfunction getRotation(start, end) {\n  return getAngle(end[1], end[0], PROPS_CLIENT_XY) + getAngle(start[1], start[0], PROPS_CLIENT_XY);\n}\n\n/**\n * @private\n * velocity is calculated every x ms\n * @param {Object} session\n * @param {Object} input\n */\n\nfunction computeIntervalInputData(session, input) {\n  var last = session.lastInterval || input;\n  var deltaTime = input.timeStamp - last.timeStamp;\n  var velocity;\n  var velocityX;\n  var velocityY;\n  var direction;\n\n  if (input.eventType !== INPUT_CANCEL && (deltaTime > COMPUTE_INTERVAL || last.velocity === undefined)) {\n    var deltaX = input.deltaX - last.deltaX;\n    var deltaY = input.deltaY - last.deltaY;\n    var v = getVelocity(deltaTime, deltaX, deltaY);\n    velocityX = v.x;\n    velocityY = v.y;\n    velocity = abs(v.x) > abs(v.y) ? v.x : v.y;\n    direction = getDirection(deltaX, deltaY);\n    session.lastInterval = input;\n  } else {\n    // use latest velocity info if it doesn't overtake a minimum period\n    velocity = last.velocity;\n    velocityX = last.velocityX;\n    velocityY = last.velocityY;\n    direction = last.direction;\n  }\n\n  input.velocity = velocity;\n  input.velocityX = velocityX;\n  input.velocityY = velocityY;\n  input.direction = direction;\n}\n\n/**\n* @private\n * extend the data with some usable properties like scale, rotate, velocity etc\n * @param {Object} manager\n * @param {Object} input\n */\n\nfunction computeInputData(manager, input) {\n  var session = manager.session;\n  var pointers = input.pointers;\n  var pointersLength = pointers.length; // store the first input to calculate the distance and direction\n\n  if (!session.firstInput) {\n    session.firstInput = simpleCloneInputData(input);\n  } // to compute scale and rotation we need to store the multiple touches\n\n\n  if (pointersLength > 1 && !session.firstMultiple) {\n    session.firstMultiple = simpleCloneInputData(input);\n  } else if (pointersLength === 1) {\n    session.firstMultiple = false;\n  }\n\n  var firstInput = session.firstInput,\n      firstMultiple = session.firstMultiple;\n  var offsetCenter = firstMultiple ? firstMultiple.center : firstInput.center;\n  var center = input.center = getCenter(pointers);\n  input.timeStamp = now();\n  input.deltaTime = input.timeStamp - firstInput.timeStamp;\n  input.angle = getAngle(offsetCenter, center);\n  input.distance = getDistance(offsetCenter, center);\n  computeDeltaXY(session, input);\n  input.offsetDirection = getDirection(input.deltaX, input.deltaY);\n  var overallVelocity = getVelocity(input.deltaTime, input.deltaX, input.deltaY);\n  input.overallVelocityX = overallVelocity.x;\n  input.overallVelocityY = overallVelocity.y;\n  input.overallVelocity = abs(overallVelocity.x) > abs(overallVelocity.y) ? overallVelocity.x : overallVelocity.y;\n  input.scale = firstMultiple ? getScale(firstMultiple.pointers, pointers) : 1;\n  input.rotation = firstMultiple ? getRotation(firstMultiple.pointers, pointers) : 0;\n  input.maxPointers = !session.prevInput ? input.pointers.length : input.pointers.length > session.prevInput.maxPointers ? input.pointers.length : session.prevInput.maxPointers;\n  computeIntervalInputData(session, input); // find the correct target\n\n  var target = manager.element;\n  var srcEvent = input.srcEvent;\n  var srcEventTarget;\n\n  if (srcEvent.composedPath) {\n    srcEventTarget = srcEvent.composedPath()[0];\n  } else if (srcEvent.path) {\n    srcEventTarget = srcEvent.path[0];\n  } else {\n    srcEventTarget = srcEvent.target;\n  }\n\n  if (hasParent(srcEventTarget, target)) {\n    target = srcEventTarget;\n  }\n\n  input.target = target;\n}\n\n/**\n * @private\n * handle input events\n * @param {Manager} manager\n * @param {String} eventType\n * @param {Object} input\n */\n\nfunction inputHandler(manager, eventType, input) {\n  var pointersLen = input.pointers.length;\n  var changedPointersLen = input.changedPointers.length;\n  var isFirst = eventType & INPUT_START && pointersLen - changedPointersLen === 0;\n  var isFinal = eventType & (INPUT_END | INPUT_CANCEL) && pointersLen - changedPointersLen === 0;\n  input.isFirst = !!isFirst;\n  input.isFinal = !!isFinal;\n\n  if (isFirst) {\n    manager.session = {};\n  } // source event is the normalized value of the domEvents\n  // like 'touchstart, mouseup, pointerdown'\n\n\n  input.eventType = eventType; // compute scale, rotation etc\n\n  computeInputData(manager, input); // emit secret event\n\n  manager.emit('hammer.input', input);\n  manager.recognize(input);\n  manager.session.prevInput = input;\n}\n\n/**\n * @private\n * split string on whitespace\n * @param {String} str\n * @returns {Array} words\n */\nfunction splitStr(str) {\n  return str.trim().split(/\\s+/g);\n}\n\n/**\n * @private\n * addEventListener with multiple events at once\n * @param {EventTarget} target\n * @param {String} types\n * @param {Function} handler\n */\n\nfunction addEventListeners(target, types, handler) {\n  each(splitStr(types), function (type) {\n    target.addEventListener(type, handler, false);\n  });\n}\n\n/**\n * @private\n * removeEventListener with multiple events at once\n * @param {EventTarget} target\n * @param {String} types\n * @param {Function} handler\n */\n\nfunction removeEventListeners(target, types, handler) {\n  each(splitStr(types), function (type) {\n    target.removeEventListener(type, handler, false);\n  });\n}\n\n/**\n * @private\n * get the window object of an element\n * @param {HTMLElement} element\n * @returns {DocumentView|Window}\n */\nfunction getWindowForElement(element) {\n  var doc = element.ownerDocument || element;\n  return doc.defaultView || doc.parentWindow || window;\n}\n\n/**\n * @private\n * create new input type manager\n * @param {Manager} manager\n * @param {Function} callback\n * @returns {Input}\n * @constructor\n */\n\nvar Input =\n/*#__PURE__*/\nfunction () {\n  function Input(manager, callback) {\n    var self = this;\n    this.manager = manager;\n    this.callback = callback;\n    this.element = manager.element;\n    this.target = manager.options.inputTarget; // smaller wrapper around the handler, for the scope and the enabled state of the manager,\n    // so when disabled the input events are completely bypassed.\n\n    this.domHandler = function (ev) {\n      if (boolOrFn(manager.options.enable, [manager])) {\n        self.handler(ev);\n      }\n    };\n\n    this.init();\n  }\n  /**\n   * @private\n   * should handle the inputEvent data and trigger the callback\n   * @virtual\n   */\n\n\n  var _proto = Input.prototype;\n\n  _proto.handler = function handler() {};\n  /**\n   * @private\n   * bind the events\n   */\n\n\n  _proto.init = function init() {\n    this.evEl && addEventListeners(this.element, this.evEl, this.domHandler);\n    this.evTarget && addEventListeners(this.target, this.evTarget, this.domHandler);\n    this.evWin && addEventListeners(getWindowForElement(this.element), this.evWin, this.domHandler);\n  };\n  /**\n   * @private\n   * unbind the events\n   */\n\n\n  _proto.destroy = function destroy() {\n    this.evEl && removeEventListeners(this.element, this.evEl, this.domHandler);\n    this.evTarget && removeEventListeners(this.target, this.evTarget, this.domHandler);\n    this.evWin && removeEventListeners(getWindowForElement(this.element), this.evWin, this.domHandler);\n  };\n\n  return Input;\n}();\n\n/**\n * @private\n * find if a array contains the object using indexOf or a simple polyFill\n * @param {Array} src\n * @param {String} find\n * @param {String} [findByKey]\n * @return {Boolean|Number} false when not found, or the index\n */\nfunction inArray(src, find, findByKey) {\n  if (src.indexOf && !findByKey) {\n    return src.indexOf(find);\n  } else {\n    var i = 0;\n\n    while (i < src.length) {\n      if (findByKey && src[i][findByKey] == find || !findByKey && src[i] === find) {\n        // do not use === here, test fails\n        return i;\n      }\n\n      i++;\n    }\n\n    return -1;\n  }\n}\n\nvar POINTER_INPUT_MAP = {\n  pointerdown: INPUT_START,\n  pointermove: INPUT_MOVE,\n  pointerup: INPUT_END,\n  pointercancel: INPUT_CANCEL,\n  pointerout: INPUT_CANCEL\n}; // in IE10 the pointer types is defined as an enum\n\nvar IE10_POINTER_TYPE_ENUM = {\n  2: INPUT_TYPE_TOUCH,\n  3: INPUT_TYPE_PEN,\n  4: INPUT_TYPE_MOUSE,\n  5: INPUT_TYPE_KINECT // see https://twitter.com/jacobrossi/status/480596438489890816\n\n};\nvar POINTER_ELEMENT_EVENTS = 'pointerdown';\nvar POINTER_WINDOW_EVENTS = 'pointermove pointerup pointercancel'; // IE10 has prefixed support, and case-sensitive\n\nif (win.MSPointerEvent && !win.PointerEvent) {\n  POINTER_ELEMENT_EVENTS = 'MSPointerDown';\n  POINTER_WINDOW_EVENTS = 'MSPointerMove MSPointerUp MSPointerCancel';\n}\n/**\n * @private\n * Pointer events input\n * @constructor\n * @extends Input\n */\n\n\nvar PointerEventInput =\n/*#__PURE__*/\nfunction (_Input) {\n  _inheritsLoose(PointerEventInput, _Input);\n\n  function PointerEventInput() {\n    var _this;\n\n    var proto = PointerEventInput.prototype;\n    proto.evEl = POINTER_ELEMENT_EVENTS;\n    proto.evWin = POINTER_WINDOW_EVENTS;\n    _this = _Input.apply(this, arguments) || this;\n    _this.store = _this.manager.session.pointerEvents = [];\n    return _this;\n  }\n  /**\n   * @private\n   * handle mouse events\n   * @param {Object} ev\n   */\n\n\n  var _proto = PointerEventInput.prototype;\n\n  _proto.handler = function handler(ev) {\n    var store = this.store;\n    var removePointer = false;\n    var eventTypeNormalized = ev.type.toLowerCase().replace('ms', '');\n    var eventType = POINTER_INPUT_MAP[eventTypeNormalized];\n    var pointerType = IE10_POINTER_TYPE_ENUM[ev.pointerType] || ev.pointerType;\n    var isTouch = pointerType === INPUT_TYPE_TOUCH; // get index of the event in the store\n\n    var storeIndex = inArray(store, ev.pointerId, 'pointerId'); // start and mouse must be down\n\n    if (eventType & INPUT_START && (ev.button === 0 || isTouch)) {\n      if (storeIndex < 0) {\n        store.push(ev);\n        storeIndex = store.length - 1;\n      }\n    } else if (eventType & (INPUT_END | INPUT_CANCEL)) {\n      removePointer = true;\n    } // it not found, so the pointer hasn't been down (so it's probably a hover)\n\n\n    if (storeIndex < 0) {\n      return;\n    } // update the event in the store\n\n\n    store[storeIndex] = ev;\n    this.callback(this.manager, eventType, {\n      pointers: store,\n      changedPointers: [ev],\n      pointerType: pointerType,\n      srcEvent: ev\n    });\n\n    if (removePointer) {\n      // remove from the store\n      store.splice(storeIndex, 1);\n    }\n  };\n\n  return PointerEventInput;\n}(Input);\n\n/**\n * @private\n * convert array-like objects to real arrays\n * @param {Object} obj\n * @returns {Array}\n */\nfunction toArray(obj) {\n  return Array.prototype.slice.call(obj, 0);\n}\n\n/**\n * @private\n * unique array with objects based on a key (like 'id') or just by the array's value\n * @param {Array} src [{id:1},{id:2},{id:1}]\n * @param {String} [key]\n * @param {Boolean} [sort=False]\n * @returns {Array} [{id:1},{id:2}]\n */\n\nfunction uniqueArray(src, key, sort) {\n  var results = [];\n  var values = [];\n  var i = 0;\n\n  while (i < src.length) {\n    var val = key ? src[i][key] : src[i];\n\n    if (inArray(values, val) < 0) {\n      results.push(src[i]);\n    }\n\n    values[i] = val;\n    i++;\n  }\n\n  if (sort) {\n    if (!key) {\n      results = results.sort();\n    } else {\n      results = results.sort(function (a, b) {\n        return a[key] > b[key];\n      });\n    }\n  }\n\n  return results;\n}\n\nvar TOUCH_INPUT_MAP = {\n  touchstart: INPUT_START,\n  touchmove: INPUT_MOVE,\n  touchend: INPUT_END,\n  touchcancel: INPUT_CANCEL\n};\nvar TOUCH_TARGET_EVENTS = 'touchstart touchmove touchend touchcancel';\n/**\n * @private\n * Multi-user touch events input\n * @constructor\n * @extends Input\n */\n\nvar TouchInput =\n/*#__PURE__*/\nfunction (_Input) {\n  _inheritsLoose(TouchInput, _Input);\n\n  function TouchInput() {\n    var _this;\n\n    TouchInput.prototype.evTarget = TOUCH_TARGET_EVENTS;\n    _this = _Input.apply(this, arguments) || this;\n    _this.targetIds = {}; // this.evTarget = TOUCH_TARGET_EVENTS;\n\n    return _this;\n  }\n\n  var _proto = TouchInput.prototype;\n\n  _proto.handler = function handler(ev) {\n    var type = TOUCH_INPUT_MAP[ev.type];\n    var touches = getTouches.call(this, ev, type);\n\n    if (!touches) {\n      return;\n    }\n\n    this.callback(this.manager, type, {\n      pointers: touches[0],\n      changedPointers: touches[1],\n      pointerType: INPUT_TYPE_TOUCH,\n      srcEvent: ev\n    });\n  };\n\n  return TouchInput;\n}(Input);\n\nfunction getTouches(ev, type) {\n  var allTouches = toArray(ev.touches);\n  var targetIds = this.targetIds; // when there is only one touch, the process can be simplified\n\n  if (type & (INPUT_START | INPUT_MOVE) && allTouches.length === 1) {\n    targetIds[allTouches[0].identifier] = true;\n    return [allTouches, allTouches];\n  }\n\n  var i;\n  var targetTouches;\n  var changedTouches = toArray(ev.changedTouches);\n  var changedTargetTouches = [];\n  var target = this.target; // get target touches from touches\n\n  targetTouches = allTouches.filter(function (touch) {\n    return hasParent(touch.target, target);\n  }); // collect touches\n\n  if (type === INPUT_START) {\n    i = 0;\n\n    while (i < targetTouches.length) {\n      targetIds[targetTouches[i].identifier] = true;\n      i++;\n    }\n  } // filter changed touches to only contain touches that exist in the collected target ids\n\n\n  i = 0;\n\n  while (i < changedTouches.length) {\n    if (targetIds[changedTouches[i].identifier]) {\n      changedTargetTouches.push(changedTouches[i]);\n    } // cleanup removed touches\n\n\n    if (type & (INPUT_END | INPUT_CANCEL)) {\n      delete targetIds[changedTouches[i].identifier];\n    }\n\n    i++;\n  }\n\n  if (!changedTargetTouches.length) {\n    return;\n  }\n\n  return [// merge targetTouches with changedTargetTouches so it contains ALL touches, including 'end' and 'cancel'\n  uniqueArray(targetTouches.concat(changedTargetTouches), 'identifier', true), changedTargetTouches];\n}\n\nvar MOUSE_INPUT_MAP = {\n  mousedown: INPUT_START,\n  mousemove: INPUT_MOVE,\n  mouseup: INPUT_END\n};\nvar MOUSE_ELEMENT_EVENTS = 'mousedown';\nvar MOUSE_WINDOW_EVENTS = 'mousemove mouseup';\n/**\n * @private\n * Mouse events input\n * @constructor\n * @extends Input\n */\n\nvar MouseInput =\n/*#__PURE__*/\nfunction (_Input) {\n  _inheritsLoose(MouseInput, _Input);\n\n  function MouseInput() {\n    var _this;\n\n    var proto = MouseInput.prototype;\n    proto.evEl = MOUSE_ELEMENT_EVENTS;\n    proto.evWin = MOUSE_WINDOW_EVENTS;\n    _this = _Input.apply(this, arguments) || this;\n    _this.pressed = false; // mousedown state\n\n    return _this;\n  }\n  /**\n   * @private\n   * handle mouse events\n   * @param {Object} ev\n   */\n\n\n  var _proto = MouseInput.prototype;\n\n  _proto.handler = function handler(ev) {\n    var eventType = MOUSE_INPUT_MAP[ev.type]; // on start we want to have the left mouse button down\n\n    if (eventType & INPUT_START && ev.button === 0) {\n      this.pressed = true;\n    }\n\n    if (eventType & INPUT_MOVE && ev.which !== 1) {\n      eventType = INPUT_END;\n    } // mouse must be down\n\n\n    if (!this.pressed) {\n      return;\n    }\n\n    if (eventType & INPUT_END) {\n      this.pressed = false;\n    }\n\n    this.callback(this.manager, eventType, {\n      pointers: [ev],\n      changedPointers: [ev],\n      pointerType: INPUT_TYPE_MOUSE,\n      srcEvent: ev\n    });\n  };\n\n  return MouseInput;\n}(Input);\n\n/**\n * @private\n * Combined touch and mouse input\n *\n * Touch has a higher priority then mouse, and while touching no mouse events are allowed.\n * This because touch devices also emit mouse events while doing a touch.\n *\n * @constructor\n * @extends Input\n */\n\nvar DEDUP_TIMEOUT = 2500;\nvar DEDUP_DISTANCE = 25;\n\nfunction setLastTouch(eventData) {\n  var _eventData$changedPoi = eventData.changedPointers,\n      touch = _eventData$changedPoi[0];\n\n  if (touch.identifier === this.primaryTouch) {\n    var lastTouch = {\n      x: touch.clientX,\n      y: touch.clientY\n    };\n    var lts = this.lastTouches;\n    this.lastTouches.push(lastTouch);\n\n    var removeLastTouch = function removeLastTouch() {\n      var i = lts.indexOf(lastTouch);\n\n      if (i > -1) {\n        lts.splice(i, 1);\n      }\n    };\n\n    setTimeout(removeLastTouch, DEDUP_TIMEOUT);\n  }\n}\n\nfunction recordTouches(eventType, eventData) {\n  if (eventType & INPUT_START) {\n    this.primaryTouch = eventData.changedPointers[0].identifier;\n    setLastTouch.call(this, eventData);\n  } else if (eventType & (INPUT_END | INPUT_CANCEL)) {\n    setLastTouch.call(this, eventData);\n  }\n}\n\nfunction isSyntheticEvent(eventData) {\n  var x = eventData.srcEvent.clientX;\n  var y = eventData.srcEvent.clientY;\n\n  for (var i = 0; i < this.lastTouches.length; i++) {\n    var t = this.lastTouches[i];\n    var dx = Math.abs(x - t.x);\n    var dy = Math.abs(y - t.y);\n\n    if (dx <= DEDUP_DISTANCE && dy <= DEDUP_DISTANCE) {\n      return true;\n    }\n  }\n\n  return false;\n}\n\nvar TouchMouseInput =\n/*#__PURE__*/\nfunction () {\n  var TouchMouseInput =\n  /*#__PURE__*/\n  function (_Input) {\n    _inheritsLoose(TouchMouseInput, _Input);\n\n    function TouchMouseInput(_manager, callback) {\n      var _this;\n\n      _this = _Input.call(this, _manager, callback) || this;\n\n      _this.handler = function (manager, inputEvent, inputData) {\n        var isTouch = inputData.pointerType === INPUT_TYPE_TOUCH;\n        var isMouse = inputData.pointerType === INPUT_TYPE_MOUSE;\n\n        if (isMouse && inputData.sourceCapabilities && inputData.sourceCapabilities.firesTouchEvents) {\n          return;\n        } // when we're in a touch event, record touches to  de-dupe synthetic mouse event\n\n\n        if (isTouch) {\n          recordTouches.call(_assertThisInitialized(_assertThisInitialized(_this)), inputEvent, inputData);\n        } else if (isMouse && isSyntheticEvent.call(_assertThisInitialized(_assertThisInitialized(_this)), inputData)) {\n          return;\n        }\n\n        _this.callback(manager, inputEvent, inputData);\n      };\n\n      _this.touch = new TouchInput(_this.manager, _this.handler);\n      _this.mouse = new MouseInput(_this.manager, _this.handler);\n      _this.primaryTouch = null;\n      _this.lastTouches = [];\n      return _this;\n    }\n    /**\n     * @private\n     * handle mouse and touch events\n     * @param {Hammer} manager\n     * @param {String} inputEvent\n     * @param {Object} inputData\n     */\n\n\n    var _proto = TouchMouseInput.prototype;\n\n    /**\n     * @private\n     * remove the event listeners\n     */\n    _proto.destroy = function destroy() {\n      this.touch.destroy();\n      this.mouse.destroy();\n    };\n\n    return TouchMouseInput;\n  }(Input);\n\n  return TouchMouseInput;\n}();\n\n/**\n * @private\n * create new input type manager\n * called by the Manager constructor\n * @param {Hammer} manager\n * @returns {Input}\n */\n\nfunction createInputInstance(manager) {\n  var Type; // let inputClass = manager.options.inputClass;\n\n  var inputClass = manager.options.inputClass;\n\n  if (inputClass) {\n    Type = inputClass;\n  } else if (SUPPORT_POINTER_EVENTS) {\n    Type = PointerEventInput;\n  } else if (SUPPORT_ONLY_TOUCH) {\n    Type = TouchInput;\n  } else if (!SUPPORT_TOUCH) {\n    Type = MouseInput;\n  } else {\n    Type = TouchMouseInput;\n  }\n\n  return new Type(manager, inputHandler);\n}\n\n/**\n * @private\n * if the argument is an array, we want to execute the fn on each entry\n * if it aint an array we don't want to do a thing.\n * this is used by all the methods that accept a single and array argument.\n * @param {*|Array} arg\n * @param {String} fn\n * @param {Object} [context]\n * @returns {Boolean}\n */\n\nfunction invokeArrayArg(arg, fn, context) {\n  if (Array.isArray(arg)) {\n    each(arg, context[fn], context);\n    return true;\n  }\n\n  return false;\n}\n\nvar STATE_POSSIBLE = 1;\nvar STATE_BEGAN = 2;\nvar STATE_CHANGED = 4;\nvar STATE_ENDED = 8;\nvar STATE_RECOGNIZED = STATE_ENDED;\nvar STATE_CANCELLED = 16;\nvar STATE_FAILED = 32;\n\n/**\n * @private\n * get a unique id\n * @returns {number} uniqueId\n */\nvar _uniqueId = 1;\nfunction uniqueId() {\n  return _uniqueId++;\n}\n\n/**\n * @private\n * get a recognizer by name if it is bound to a manager\n * @param {Recognizer|String} otherRecognizer\n * @param {Recognizer} recognizer\n * @returns {Recognizer}\n */\nfunction getRecognizerByNameIfManager(otherRecognizer, recognizer) {\n  var manager = recognizer.manager;\n\n  if (manager) {\n    return manager.get(otherRecognizer);\n  }\n\n  return otherRecognizer;\n}\n\n/**\n * @private\n * get a usable string, used as event postfix\n * @param {constant} state\n * @returns {String} state\n */\n\nfunction stateStr(state) {\n  if (state & STATE_CANCELLED) {\n    return 'cancel';\n  } else if (state & STATE_ENDED) {\n    return 'end';\n  } else if (state & STATE_CHANGED) {\n    return 'move';\n  } else if (state & STATE_BEGAN) {\n    return 'start';\n  }\n\n  return '';\n}\n\n/**\n * @private\n * Recognizer flow explained; *\n * All recognizers have the initial state of POSSIBLE when a input session starts.\n * The definition of a input session is from the first input until the last input, with all it's movement in it. *\n * Example session for mouse-input: mousedown -> mousemove -> mouseup\n *\n * On each recognizing cycle (see Manager.recognize) the .recognize() method is executed\n * which determines with state it should be.\n *\n * If the recognizer has the state FAILED, CANCELLED or RECOGNIZED (equals ENDED), it is reset to\n * POSSIBLE to give it another change on the next cycle.\n *\n *               Possible\n *                  |\n *            +-----+---------------+\n *            |                     |\n *      +-----+-----+               |\n *      |           |               |\n *   Failed      Cancelled          |\n *                          +-------+------+\n *                          |              |\n *                      Recognized       Began\n *                                         |\n *                                      Changed\n *                                         |\n *                                  Ended/Recognized\n */\n\n/**\n * @private\n * Recognizer\n * Every recognizer needs to extend from this class.\n * @constructor\n * @param {Object} options\n */\n\nvar Recognizer =\n/*#__PURE__*/\nfunction () {\n  function Recognizer(options) {\n    if (options === void 0) {\n      options = {};\n    }\n\n    this.options = _extends({\n      enable: true\n    }, options);\n    this.id = uniqueId();\n    this.manager = null; // default is enable true\n\n    this.state = STATE_POSSIBLE;\n    this.simultaneous = {};\n    this.requireFail = [];\n  }\n  /**\n   * @private\n   * set options\n   * @param {Object} options\n   * @return {Recognizer}\n   */\n\n\n  var _proto = Recognizer.prototype;\n\n  _proto.set = function set(options) {\n    assign$1(this.options, options); // also update the touchAction, in case something changed about the directions/enabled state\n\n    this.manager && this.manager.touchAction.update();\n    return this;\n  };\n  /**\n   * @private\n   * recognize simultaneous with an other recognizer.\n   * @param {Recognizer} otherRecognizer\n   * @returns {Recognizer} this\n   */\n\n\n  _proto.recognizeWith = function recognizeWith(otherRecognizer) {\n    if (invokeArrayArg(otherRecognizer, 'recognizeWith', this)) {\n      return this;\n    }\n\n    var simultaneous = this.simultaneous;\n    otherRecognizer = getRecognizerByNameIfManager(otherRecognizer, this);\n\n    if (!simultaneous[otherRecognizer.id]) {\n      simultaneous[otherRecognizer.id] = otherRecognizer;\n      otherRecognizer.recognizeWith(this);\n    }\n\n    return this;\n  };\n  /**\n   * @private\n   * drop the simultaneous link. it doesnt remove the link on the other recognizer.\n   * @param {Recognizer} otherRecognizer\n   * @returns {Recognizer} this\n   */\n\n\n  _proto.dropRecognizeWith = function dropRecognizeWith(otherRecognizer) {\n    if (invokeArrayArg(otherRecognizer, 'dropRecognizeWith', this)) {\n      return this;\n    }\n\n    otherRecognizer = getRecognizerByNameIfManager(otherRecognizer, this);\n    delete this.simultaneous[otherRecognizer.id];\n    return this;\n  };\n  /**\n   * @private\n   * recognizer can only run when an other is failing\n   * @param {Recognizer} otherRecognizer\n   * @returns {Recognizer} this\n   */\n\n\n  _proto.requireFailure = function requireFailure(otherRecognizer) {\n    if (invokeArrayArg(otherRecognizer, 'requireFailure', this)) {\n      return this;\n    }\n\n    var requireFail = this.requireFail;\n    otherRecognizer = getRecognizerByNameIfManager(otherRecognizer, this);\n\n    if (inArray(requireFail, otherRecognizer) === -1) {\n      requireFail.push(otherRecognizer);\n      otherRecognizer.requireFailure(this);\n    }\n\n    return this;\n  };\n  /**\n   * @private\n   * drop the requireFailure link. it does not remove the link on the other recognizer.\n   * @param {Recognizer} otherRecognizer\n   * @returns {Recognizer} this\n   */\n\n\n  _proto.dropRequireFailure = function dropRequireFailure(otherRecognizer) {\n    if (invokeArrayArg(otherRecognizer, 'dropRequireFailure', this)) {\n      return this;\n    }\n\n    otherRecognizer = getRecognizerByNameIfManager(otherRecognizer, this);\n    var index = inArray(this.requireFail, otherRecognizer);\n\n    if (index > -1) {\n      this.requireFail.splice(index, 1);\n    }\n\n    return this;\n  };\n  /**\n   * @private\n   * has require failures boolean\n   * @returns {boolean}\n   */\n\n\n  _proto.hasRequireFailures = function hasRequireFailures() {\n    return this.requireFail.length > 0;\n  };\n  /**\n   * @private\n   * if the recognizer can recognize simultaneous with an other recognizer\n   * @param {Recognizer} otherRecognizer\n   * @returns {Boolean}\n   */\n\n\n  _proto.canRecognizeWith = function canRecognizeWith(otherRecognizer) {\n    return !!this.simultaneous[otherRecognizer.id];\n  };\n  /**\n   * @private\n   * You should use `tryEmit` instead of `emit` directly to check\n   * that all the needed recognizers has failed before emitting.\n   * @param {Object} input\n   */\n\n\n  _proto.emit = function emit(input) {\n    var self = this;\n    var state = this.state;\n\n    function emit(event) {\n      self.manager.emit(event, input);\n    } // 'panstart' and 'panmove'\n\n\n    if (state < STATE_ENDED) {\n      emit(self.options.event + stateStr(state));\n    }\n\n    emit(self.options.event); // simple 'eventName' events\n\n    if (input.additionalEvent) {\n      // additional event(panleft, panright, pinchin, pinchout...)\n      emit(input.additionalEvent);\n    } // panend and pancancel\n\n\n    if (state >= STATE_ENDED) {\n      emit(self.options.event + stateStr(state));\n    }\n  };\n  /**\n   * @private\n   * Check that all the require failure recognizers has failed,\n   * if true, it emits a gesture event,\n   * otherwise, setup the state to FAILED.\n   * @param {Object} input\n   */\n\n\n  _proto.tryEmit = function tryEmit(input) {\n    if (this.canEmit()) {\n      return this.emit(input);\n    } // it's failing anyway\n\n\n    this.state = STATE_FAILED;\n  };\n  /**\n   * @private\n   * can we emit?\n   * @returns {boolean}\n   */\n\n\n  _proto.canEmit = function canEmit() {\n    var i = 0;\n\n    while (i < this.requireFail.length) {\n      if (!(this.requireFail[i].state & (STATE_FAILED | STATE_POSSIBLE))) {\n        return false;\n      }\n\n      i++;\n    }\n\n    return true;\n  };\n  /**\n   * @private\n   * update the recognizer\n   * @param {Object} inputData\n   */\n\n\n  _proto.recognize = function recognize(inputData) {\n    // make a new copy of the inputData\n    // so we can change the inputData without messing up the other recognizers\n    var inputDataClone = assign$1({}, inputData); // is is enabled and allow recognizing?\n\n    if (!boolOrFn(this.options.enable, [this, inputDataClone])) {\n      this.reset();\n      this.state = STATE_FAILED;\n      return;\n    } // reset when we've reached the end\n\n\n    if (this.state & (STATE_RECOGNIZED | STATE_CANCELLED | STATE_FAILED)) {\n      this.state = STATE_POSSIBLE;\n    }\n\n    this.state = this.process(inputDataClone); // the recognizer has recognized a gesture\n    // so trigger an event\n\n    if (this.state & (STATE_BEGAN | STATE_CHANGED | STATE_ENDED | STATE_CANCELLED)) {\n      this.tryEmit(inputDataClone);\n    }\n  };\n  /**\n   * @private\n   * return the state of the recognizer\n   * the actual recognizing happens in this method\n   * @virtual\n   * @param {Object} inputData\n   * @returns {constant} STATE\n   */\n\n  /* jshint ignore:start */\n\n\n  _proto.process = function process(inputData) {};\n  /* jshint ignore:end */\n\n  /**\n   * @private\n   * return the preferred touch-action\n   * @virtual\n   * @returns {Array}\n   */\n\n\n  _proto.getTouchAction = function getTouchAction() {};\n  /**\n   * @private\n   * called when the gesture isn't allowed to recognize\n   * like when another is being recognized or it is disabled\n   * @virtual\n   */\n\n\n  _proto.reset = function reset() {};\n\n  return Recognizer;\n}();\n\n/**\n * @private\n * A tap is recognized when the pointer is doing a small tap/click. Multiple taps are recognized if they occur\n * between the given interval and position. The delay option can be used to recognize multi-taps without firing\n * a single tap.\n *\n * The eventData from the emitted event contains the property `tapCount`, which contains the amount of\n * multi-taps being recognized.\n * @constructor\n * @extends Recognizer\n */\n\nvar TapRecognizer =\n/*#__PURE__*/\nfunction (_Recognizer) {\n  _inheritsLoose(TapRecognizer, _Recognizer);\n\n  function TapRecognizer(options) {\n    var _this;\n\n    if (options === void 0) {\n      options = {};\n    }\n\n    _this = _Recognizer.call(this, _extends({\n      event: 'tap',\n      pointers: 1,\n      taps: 1,\n      interval: 300,\n      // max time between the multi-tap taps\n      time: 250,\n      // max time of the pointer to be down (like finger on the screen)\n      threshold: 9,\n      // a minimal movement is ok, but keep it low\n      posThreshold: 10\n    }, options)) || this; // previous time and center,\n    // used for tap counting\n\n    _this.pTime = false;\n    _this.pCenter = false;\n    _this._timer = null;\n    _this._input = null;\n    _this.count = 0;\n    return _this;\n  }\n\n  var _proto = TapRecognizer.prototype;\n\n  _proto.getTouchAction = function getTouchAction() {\n    return [TOUCH_ACTION_MANIPULATION];\n  };\n\n  _proto.process = function process(input) {\n    var _this2 = this;\n\n    var options = this.options;\n    var validPointers = input.pointers.length === options.pointers;\n    var validMovement = input.distance < options.threshold;\n    var validTouchTime = input.deltaTime < options.time;\n    this.reset();\n\n    if (input.eventType & INPUT_START && this.count === 0) {\n      return this.failTimeout();\n    } // we only allow little movement\n    // and we've reached an end event, so a tap is possible\n\n\n    if (validMovement && validTouchTime && validPointers) {\n      if (input.eventType !== INPUT_END) {\n        return this.failTimeout();\n      }\n\n      var validInterval = this.pTime ? input.timeStamp - this.pTime < options.interval : true;\n      var validMultiTap = !this.pCenter || getDistance(this.pCenter, input.center) < options.posThreshold;\n      this.pTime = input.timeStamp;\n      this.pCenter = input.center;\n\n      if (!validMultiTap || !validInterval) {\n        this.count = 1;\n      } else {\n        this.count += 1;\n      }\n\n      this._input = input; // if tap count matches we have recognized it,\n      // else it has began recognizing...\n\n      var tapCount = this.count % options.taps;\n\n      if (tapCount === 0) {\n        // no failing requirements, immediately trigger the tap event\n        // or wait as long as the multitap interval to trigger\n        if (!this.hasRequireFailures()) {\n          return STATE_RECOGNIZED;\n        } else {\n          this._timer = setTimeout(function () {\n            _this2.state = STATE_RECOGNIZED;\n\n            _this2.tryEmit();\n          }, options.interval);\n          return STATE_BEGAN;\n        }\n      }\n    }\n\n    return STATE_FAILED;\n  };\n\n  _proto.failTimeout = function failTimeout() {\n    var _this3 = this;\n\n    this._timer = setTimeout(function () {\n      _this3.state = STATE_FAILED;\n    }, this.options.interval);\n    return STATE_FAILED;\n  };\n\n  _proto.reset = function reset() {\n    clearTimeout(this._timer);\n  };\n\n  _proto.emit = function emit() {\n    if (this.state === STATE_RECOGNIZED) {\n      this._input.tapCount = this.count;\n      this.manager.emit(this.options.event, this._input);\n    }\n  };\n\n  return TapRecognizer;\n}(Recognizer);\n\n/**\n * @private\n * This recognizer is just used as a base for the simple attribute recognizers.\n * @constructor\n * @extends Recognizer\n */\n\nvar AttrRecognizer =\n/*#__PURE__*/\nfunction (_Recognizer) {\n  _inheritsLoose(AttrRecognizer, _Recognizer);\n\n  function AttrRecognizer(options) {\n    if (options === void 0) {\n      options = {};\n    }\n\n    return _Recognizer.call(this, _extends({\n      pointers: 1\n    }, options)) || this;\n  }\n  /**\n   * @private\n   * Used to check if it the recognizer receives valid input, like input.distance > 10.\n   * @memberof AttrRecognizer\n   * @param {Object} input\n   * @returns {Boolean} recognized\n   */\n\n\n  var _proto = AttrRecognizer.prototype;\n\n  _proto.attrTest = function attrTest(input) {\n    var optionPointers = this.options.pointers;\n    return optionPointers === 0 || input.pointers.length === optionPointers;\n  };\n  /**\n   * @private\n   * Process the input and return the state for the recognizer\n   * @memberof AttrRecognizer\n   * @param {Object} input\n   * @returns {*} State\n   */\n\n\n  _proto.process = function process(input) {\n    var state = this.state;\n    var eventType = input.eventType;\n    var isRecognized = state & (STATE_BEGAN | STATE_CHANGED);\n    var isValid = this.attrTest(input); // on cancel input and we've recognized before, return STATE_CANCELLED\n\n    if (isRecognized && (eventType & INPUT_CANCEL || !isValid)) {\n      return state | STATE_CANCELLED;\n    } else if (isRecognized || isValid) {\n      if (eventType & INPUT_END) {\n        return state | STATE_ENDED;\n      } else if (!(state & STATE_BEGAN)) {\n        return STATE_BEGAN;\n      }\n\n      return state | STATE_CHANGED;\n    }\n\n    return STATE_FAILED;\n  };\n\n  return AttrRecognizer;\n}(Recognizer);\n\n/**\n * @private\n * direction cons to string\n * @param {constant} direction\n * @returns {String}\n */\n\nfunction directionStr(direction) {\n  if (direction === DIRECTION_DOWN) {\n    return 'down';\n  } else if (direction === DIRECTION_UP) {\n    return 'up';\n  } else if (direction === DIRECTION_LEFT) {\n    return 'left';\n  } else if (direction === DIRECTION_RIGHT) {\n    return 'right';\n  }\n\n  return '';\n}\n\n/**\n * @private\n * Pan\n * Recognized when the pointer is down and moved in the allowed direction.\n * @constructor\n * @extends AttrRecognizer\n */\n\nvar PanRecognizer =\n/*#__PURE__*/\nfunction (_AttrRecognizer) {\n  _inheritsLoose(PanRecognizer, _AttrRecognizer);\n\n  function PanRecognizer(options) {\n    var _this;\n\n    if (options === void 0) {\n      options = {};\n    }\n\n    _this = _AttrRecognizer.call(this, _extends({\n      event: 'pan',\n      threshold: 10,\n      pointers: 1,\n      direction: DIRECTION_ALL\n    }, options)) || this;\n    _this.pX = null;\n    _this.pY = null;\n    return _this;\n  }\n\n  var _proto = PanRecognizer.prototype;\n\n  _proto.getTouchAction = function getTouchAction() {\n    var direction = this.options.direction;\n    var actions = [];\n\n    if (direction & DIRECTION_HORIZONTAL) {\n      actions.push(TOUCH_ACTION_PAN_Y);\n    }\n\n    if (direction & DIRECTION_VERTICAL) {\n      actions.push(TOUCH_ACTION_PAN_X);\n    }\n\n    return actions;\n  };\n\n  _proto.directionTest = function directionTest(input) {\n    var options = this.options;\n    var hasMoved = true;\n    var distance = input.distance;\n    var direction = input.direction;\n    var x = input.deltaX;\n    var y = input.deltaY; // lock to axis?\n\n    if (!(direction & options.direction)) {\n      if (options.direction & DIRECTION_HORIZONTAL) {\n        direction = x === 0 ? DIRECTION_NONE : x < 0 ? DIRECTION_LEFT : DIRECTION_RIGHT;\n        hasMoved = x !== this.pX;\n        distance = Math.abs(input.deltaX);\n      } else {\n        direction = y === 0 ? DIRECTION_NONE : y < 0 ? DIRECTION_UP : DIRECTION_DOWN;\n        hasMoved = y !== this.pY;\n        distance = Math.abs(input.deltaY);\n      }\n    }\n\n    input.direction = direction;\n    return hasMoved && distance > options.threshold && direction & options.direction;\n  };\n\n  _proto.attrTest = function attrTest(input) {\n    return AttrRecognizer.prototype.attrTest.call(this, input) && ( // replace with a super call\n    this.state & STATE_BEGAN || !(this.state & STATE_BEGAN) && this.directionTest(input));\n  };\n\n  _proto.emit = function emit(input) {\n    this.pX = input.deltaX;\n    this.pY = input.deltaY;\n    var direction = directionStr(input.direction);\n\n    if (direction) {\n      input.additionalEvent = this.options.event + direction;\n    }\n\n    _AttrRecognizer.prototype.emit.call(this, input);\n  };\n\n  return PanRecognizer;\n}(AttrRecognizer);\n\n/**\n * @private\n * Swipe\n * Recognized when the pointer is moving fast (velocity), with enough distance in the allowed direction.\n * @constructor\n * @extends AttrRecognizer\n */\n\nvar SwipeRecognizer =\n/*#__PURE__*/\nfunction (_AttrRecognizer) {\n  _inheritsLoose(SwipeRecognizer, _AttrRecognizer);\n\n  function SwipeRecognizer(options) {\n    if (options === void 0) {\n      options = {};\n    }\n\n    return _AttrRecognizer.call(this, _extends({\n      event: 'swipe',\n      threshold: 10,\n      velocity: 0.3,\n      direction: DIRECTION_HORIZONTAL | DIRECTION_VERTICAL,\n      pointers: 1\n    }, options)) || this;\n  }\n\n  var _proto = SwipeRecognizer.prototype;\n\n  _proto.getTouchAction = function getTouchAction() {\n    return PanRecognizer.prototype.getTouchAction.call(this);\n  };\n\n  _proto.attrTest = function attrTest(input) {\n    var direction = this.options.direction;\n    var velocity;\n\n    if (direction & (DIRECTION_HORIZONTAL | DIRECTION_VERTICAL)) {\n      velocity = input.overallVelocity;\n    } else if (direction & DIRECTION_HORIZONTAL) {\n      velocity = input.overallVelocityX;\n    } else if (direction & DIRECTION_VERTICAL) {\n      velocity = input.overallVelocityY;\n    }\n\n    return _AttrRecognizer.prototype.attrTest.call(this, input) && direction & input.offsetDirection && input.distance > this.options.threshold && input.maxPointers === this.options.pointers && abs(velocity) > this.options.velocity && input.eventType & INPUT_END;\n  };\n\n  _proto.emit = function emit(input) {\n    var direction = directionStr(input.offsetDirection);\n\n    if (direction) {\n      this.manager.emit(this.options.event + direction, input);\n    }\n\n    this.manager.emit(this.options.event, input);\n  };\n\n  return SwipeRecognizer;\n}(AttrRecognizer);\n\n/**\n * @private\n * Pinch\n * Recognized when two or more pointers are moving toward (zoom-in) or away from each other (zoom-out).\n * @constructor\n * @extends AttrRecognizer\n */\n\nvar PinchRecognizer =\n/*#__PURE__*/\nfunction (_AttrRecognizer) {\n  _inheritsLoose(PinchRecognizer, _AttrRecognizer);\n\n  function PinchRecognizer(options) {\n    if (options === void 0) {\n      options = {};\n    }\n\n    return _AttrRecognizer.call(this, _extends({\n      event: 'pinch',\n      threshold: 0,\n      pointers: 2\n    }, options)) || this;\n  }\n\n  var _proto = PinchRecognizer.prototype;\n\n  _proto.getTouchAction = function getTouchAction() {\n    return [TOUCH_ACTION_NONE];\n  };\n\n  _proto.attrTest = function attrTest(input) {\n    return _AttrRecognizer.prototype.attrTest.call(this, input) && (Math.abs(input.scale - 1) > this.options.threshold || this.state & STATE_BEGAN);\n  };\n\n  _proto.emit = function emit(input) {\n    if (input.scale !== 1) {\n      var inOut = input.scale < 1 ? 'in' : 'out';\n      input.additionalEvent = this.options.event + inOut;\n    }\n\n    _AttrRecognizer.prototype.emit.call(this, input);\n  };\n\n  return PinchRecognizer;\n}(AttrRecognizer);\n\n/**\n * @private\n * Rotate\n * Recognized when two or more pointer are moving in a circular motion.\n * @constructor\n * @extends AttrRecognizer\n */\n\nvar RotateRecognizer =\n/*#__PURE__*/\nfunction (_AttrRecognizer) {\n  _inheritsLoose(RotateRecognizer, _AttrRecognizer);\n\n  function RotateRecognizer(options) {\n    if (options === void 0) {\n      options = {};\n    }\n\n    return _AttrRecognizer.call(this, _extends({\n      event: 'rotate',\n      threshold: 0,\n      pointers: 2\n    }, options)) || this;\n  }\n\n  var _proto = RotateRecognizer.prototype;\n\n  _proto.getTouchAction = function getTouchAction() {\n    return [TOUCH_ACTION_NONE];\n  };\n\n  _proto.attrTest = function attrTest(input) {\n    return _AttrRecognizer.prototype.attrTest.call(this, input) && (Math.abs(input.rotation) > this.options.threshold || this.state & STATE_BEGAN);\n  };\n\n  return RotateRecognizer;\n}(AttrRecognizer);\n\n/**\n * @private\n * Press\n * Recognized when the pointer is down for x ms without any movement.\n * @constructor\n * @extends Recognizer\n */\n\nvar PressRecognizer =\n/*#__PURE__*/\nfunction (_Recognizer) {\n  _inheritsLoose(PressRecognizer, _Recognizer);\n\n  function PressRecognizer(options) {\n    var _this;\n\n    if (options === void 0) {\n      options = {};\n    }\n\n    _this = _Recognizer.call(this, _extends({\n      event: 'press',\n      pointers: 1,\n      time: 251,\n      // minimal time of the pointer to be pressed\n      threshold: 9\n    }, options)) || this;\n    _this._timer = null;\n    _this._input = null;\n    return _this;\n  }\n\n  var _proto = PressRecognizer.prototype;\n\n  _proto.getTouchAction = function getTouchAction() {\n    return [TOUCH_ACTION_AUTO];\n  };\n\n  _proto.process = function process(input) {\n    var _this2 = this;\n\n    var options = this.options;\n    var validPointers = input.pointers.length === options.pointers;\n    var validMovement = input.distance < options.threshold;\n    var validTime = input.deltaTime > options.time;\n    this._input = input; // we only allow little movement\n    // and we've reached an end event, so a tap is possible\n\n    if (!validMovement || !validPointers || input.eventType & (INPUT_END | INPUT_CANCEL) && !validTime) {\n      this.reset();\n    } else if (input.eventType & INPUT_START) {\n      this.reset();\n      this._timer = setTimeout(function () {\n        _this2.state = STATE_RECOGNIZED;\n\n        _this2.tryEmit();\n      }, options.time);\n    } else if (input.eventType & INPUT_END) {\n      return STATE_RECOGNIZED;\n    }\n\n    return STATE_FAILED;\n  };\n\n  _proto.reset = function reset() {\n    clearTimeout(this._timer);\n  };\n\n  _proto.emit = function emit(input) {\n    if (this.state !== STATE_RECOGNIZED) {\n      return;\n    }\n\n    if (input && input.eventType & INPUT_END) {\n      this.manager.emit(this.options.event + \"up\", input);\n    } else {\n      this._input.timeStamp = now();\n      this.manager.emit(this.options.event, this._input);\n    }\n  };\n\n  return PressRecognizer;\n}(Recognizer);\n\nvar defaults = {\n  /**\n   * @private\n   * set if DOM events are being triggered.\n   * But this is slower and unused by simple implementations, so disabled by default.\n   * @type {Boolean}\n   * @default false\n   */\n  domEvents: false,\n\n  /**\n   * @private\n   * The value for the touchAction property/fallback.\n   * When set to `compute` it will magically set the correct value based on the added recognizers.\n   * @type {String}\n   * @default compute\n   */\n  touchAction: TOUCH_ACTION_COMPUTE,\n\n  /**\n   * @private\n   * @type {Boolean}\n   * @default true\n   */\n  enable: true,\n\n  /**\n   * @private\n   * EXPERIMENTAL FEATURE -- can be removed/changed\n   * Change the parent input target element.\n   * If Null, then it is being set the to main element.\n   * @type {Null|EventTarget}\n   * @default null\n   */\n  inputTarget: null,\n\n  /**\n   * @private\n   * force an input class\n   * @type {Null|Function}\n   * @default null\n   */\n  inputClass: null,\n\n  /**\n   * @private\n   * Some CSS properties can be used to improve the working of Hammer.\n   * Add them to this method and they will be set when creating a new Manager.\n   * @namespace\n   */\n  cssProps: {\n    /**\n     * @private\n     * Disables text selection to improve the dragging gesture. Mainly for desktop browsers.\n     * @type {String}\n     * @default 'none'\n     */\n    userSelect: \"none\",\n\n    /**\n     * @private\n     * Disable the Windows Phone grippers when pressing an element.\n     * @type {String}\n     * @default 'none'\n     */\n    touchSelect: \"none\",\n\n    /**\n     * @private\n     * Disables the default callout shown when you touch and hold a touch target.\n     * On iOS, when you touch and hold a touch target such as a link, Safari displays\n     * a callout containing information about the link. This property allows you to disable that callout.\n     * @type {String}\n     * @default 'none'\n     */\n    touchCallout: \"none\",\n\n    /**\n     * @private\n     * Specifies whether zooming is enabled. Used by IE10>\n     * @type {String}\n     * @default 'none'\n     */\n    contentZooming: \"none\",\n\n    /**\n     * @private\n     * Specifies that an entire element should be draggable instead of its contents. Mainly for desktop browsers.\n     * @type {String}\n     * @default 'none'\n     */\n    userDrag: \"none\",\n\n    /**\n     * @private\n     * Overrides the highlight color shown when the user taps a link or a JavaScript\n     * clickable element in iOS. This property obeys the alpha value, if specified.\n     * @type {String}\n     * @default 'rgba(0,0,0,0)'\n     */\n    tapHighlightColor: \"rgba(0,0,0,0)\"\n  }\n};\n/**\n * @private\n * Default recognizer setup when calling `Hammer()`\n * When creating a new Manager these will be skipped.\n * This is separated with other defaults because of tree-shaking.\n * @type {Array}\n */\n\nvar preset = [[RotateRecognizer, {\n  enable: false\n}], [PinchRecognizer, {\n  enable: false\n}, ['rotate']], [SwipeRecognizer, {\n  direction: DIRECTION_HORIZONTAL\n}], [PanRecognizer, {\n  direction: DIRECTION_HORIZONTAL\n}, ['swipe']], [TapRecognizer], [TapRecognizer, {\n  event: 'doubletap',\n  taps: 2\n}, ['tap']], [PressRecognizer]];\n\nvar STOP = 1;\nvar FORCED_STOP = 2;\n/**\n * @private\n * add/remove the css properties as defined in manager.options.cssProps\n * @param {Manager} manager\n * @param {Boolean} add\n */\n\nfunction toggleCssProps(manager, add) {\n  var element = manager.element;\n\n  if (!element.style) {\n    return;\n  }\n\n  var prop;\n  each(manager.options.cssProps, function (value, name) {\n    prop = prefixed(element.style, name);\n\n    if (add) {\n      manager.oldCssProps[prop] = element.style[prop];\n      element.style[prop] = value;\n    } else {\n      element.style[prop] = manager.oldCssProps[prop] || \"\";\n    }\n  });\n\n  if (!add) {\n    manager.oldCssProps = {};\n  }\n}\n/**\n * @private\n * trigger dom event\n * @param {String} event\n * @param {Object} data\n */\n\n\nfunction triggerDomEvent(event, data) {\n  var gestureEvent = document.createEvent(\"Event\");\n  gestureEvent.initEvent(event, true, true);\n  gestureEvent.gesture = data;\n  data.target.dispatchEvent(gestureEvent);\n}\n/**\n* @private\n * Manager\n * @param {HTMLElement} element\n * @param {Object} [options]\n * @constructor\n */\n\n\nvar Manager =\n/*#__PURE__*/\nfunction () {\n  function Manager(element, options) {\n    var _this = this;\n\n    this.options = assign$1({}, defaults, options || {});\n    this.options.inputTarget = this.options.inputTarget || element;\n    this.handlers = {};\n    this.session = {};\n    this.recognizers = [];\n    this.oldCssProps = {};\n    this.element = element;\n    this.input = createInputInstance(this);\n    this.touchAction = new TouchAction(this, this.options.touchAction);\n    toggleCssProps(this, true);\n    each(this.options.recognizers, function (item) {\n      var recognizer = _this.add(new item[0](item[1]));\n\n      item[2] && recognizer.recognizeWith(item[2]);\n      item[3] && recognizer.requireFailure(item[3]);\n    }, this);\n  }\n  /**\n   * @private\n   * set options\n   * @param {Object} options\n   * @returns {Manager}\n   */\n\n\n  var _proto = Manager.prototype;\n\n  _proto.set = function set(options) {\n    assign$1(this.options, options); // Options that need a little more setup\n\n    if (options.touchAction) {\n      this.touchAction.update();\n    }\n\n    if (options.inputTarget) {\n      // Clean up existing event listeners and reinitialize\n      this.input.destroy();\n      this.input.target = options.inputTarget;\n      this.input.init();\n    }\n\n    return this;\n  };\n  /**\n   * @private\n   * stop recognizing for this session.\n   * This session will be discarded, when a new [input]start event is fired.\n   * When forced, the recognizer cycle is stopped immediately.\n   * @param {Boolean} [force]\n   */\n\n\n  _proto.stop = function stop(force) {\n    this.session.stopped = force ? FORCED_STOP : STOP;\n  };\n  /**\n   * @private\n   * run the recognizers!\n   * called by the inputHandler function on every movement of the pointers (touches)\n   * it walks through all the recognizers and tries to detect the gesture that is being made\n   * @param {Object} inputData\n   */\n\n\n  _proto.recognize = function recognize(inputData) {\n    var session = this.session;\n\n    if (session.stopped) {\n      return;\n    } // run the touch-action polyfill\n\n\n    this.touchAction.preventDefaults(inputData);\n    var recognizer;\n    var recognizers = this.recognizers; // this holds the recognizer that is being recognized.\n    // so the recognizer's state needs to be BEGAN, CHANGED, ENDED or RECOGNIZED\n    // if no recognizer is detecting a thing, it is set to `null`\n\n    var curRecognizer = session.curRecognizer; // reset when the last recognizer is recognized\n    // or when we're in a new session\n\n    if (!curRecognizer || curRecognizer && curRecognizer.state & STATE_RECOGNIZED) {\n      session.curRecognizer = null;\n      curRecognizer = null;\n    }\n\n    var i = 0;\n\n    while (i < recognizers.length) {\n      recognizer = recognizers[i]; // find out if we are allowed try to recognize the input for this one.\n      // 1.   allow if the session is NOT forced stopped (see the .stop() method)\n      // 2.   allow if we still haven't recognized a gesture in this session, or the this recognizer is the one\n      //      that is being recognized.\n      // 3.   allow if the recognizer is allowed to run simultaneous with the current recognized recognizer.\n      //      this can be setup with the `recognizeWith()` method on the recognizer.\n\n      if (session.stopped !== FORCED_STOP && ( // 1\n      !curRecognizer || recognizer === curRecognizer || // 2\n      recognizer.canRecognizeWith(curRecognizer))) {\n        // 3\n        recognizer.recognize(inputData);\n      } else {\n        recognizer.reset();\n      } // if the recognizer has been recognizing the input as a valid gesture, we want to store this one as the\n      // current active recognizer. but only if we don't already have an active recognizer\n\n\n      if (!curRecognizer && recognizer.state & (STATE_BEGAN | STATE_CHANGED | STATE_ENDED)) {\n        session.curRecognizer = recognizer;\n        curRecognizer = recognizer;\n      }\n\n      i++;\n    }\n  };\n  /**\n   * @private\n   * get a recognizer by its event name.\n   * @param {Recognizer|String} recognizer\n   * @returns {Recognizer|Null}\n   */\n\n\n  _proto.get = function get(recognizer) {\n    if (recognizer instanceof Recognizer) {\n      return recognizer;\n    }\n\n    var recognizers = this.recognizers;\n\n    for (var i = 0; i < recognizers.length; i++) {\n      if (recognizers[i].options.event === recognizer) {\n        return recognizers[i];\n      }\n    }\n\n    return null;\n  };\n  /**\n   * @private add a recognizer to the manager\n   * existing recognizers with the same event name will be removed\n   * @param {Recognizer} recognizer\n   * @returns {Recognizer|Manager}\n   */\n\n\n  _proto.add = function add(recognizer) {\n    if (invokeArrayArg(recognizer, \"add\", this)) {\n      return this;\n    } // remove existing\n\n\n    var existing = this.get(recognizer.options.event);\n\n    if (existing) {\n      this.remove(existing);\n    }\n\n    this.recognizers.push(recognizer);\n    recognizer.manager = this;\n    this.touchAction.update();\n    return recognizer;\n  };\n  /**\n   * @private\n   * remove a recognizer by name or instance\n   * @param {Recognizer|String} recognizer\n   * @returns {Manager}\n   */\n\n\n  _proto.remove = function remove(recognizer) {\n    if (invokeArrayArg(recognizer, \"remove\", this)) {\n      return this;\n    }\n\n    var targetRecognizer = this.get(recognizer); // let's make sure this recognizer exists\n\n    if (recognizer) {\n      var recognizers = this.recognizers;\n      var index = inArray(recognizers, targetRecognizer);\n\n      if (index !== -1) {\n        recognizers.splice(index, 1);\n        this.touchAction.update();\n      }\n    }\n\n    return this;\n  };\n  /**\n   * @private\n   * bind event\n   * @param {String} events\n   * @param {Function} handler\n   * @returns {EventEmitter} this\n   */\n\n\n  _proto.on = function on(events, handler) {\n    if (events === undefined || handler === undefined) {\n      return this;\n    }\n\n    var handlers = this.handlers;\n    each(splitStr(events), function (event) {\n      handlers[event] = handlers[event] || [];\n      handlers[event].push(handler);\n    });\n    return this;\n  };\n  /**\n   * @private unbind event, leave emit blank to remove all handlers\n   * @param {String} events\n   * @param {Function} [handler]\n   * @returns {EventEmitter} this\n   */\n\n\n  _proto.off = function off(events, handler) {\n    if (events === undefined) {\n      return this;\n    }\n\n    var handlers = this.handlers;\n    each(splitStr(events), function (event) {\n      if (!handler) {\n        delete handlers[event];\n      } else {\n        handlers[event] && handlers[event].splice(inArray(handlers[event], handler), 1);\n      }\n    });\n    return this;\n  };\n  /**\n   * @private emit event to the listeners\n   * @param {String} event\n   * @param {Object} data\n   */\n\n\n  _proto.emit = function emit(event, data) {\n    // we also want to trigger dom events\n    if (this.options.domEvents) {\n      triggerDomEvent(event, data);\n    } // no handlers, so skip it all\n\n\n    var handlers = this.handlers[event] && this.handlers[event].slice();\n\n    if (!handlers || !handlers.length) {\n      return;\n    }\n\n    data.type = event;\n\n    data.preventDefault = function () {\n      data.srcEvent.preventDefault();\n    };\n\n    var i = 0;\n\n    while (i < handlers.length) {\n      handlers[i](data);\n      i++;\n    }\n  };\n  /**\n   * @private\n   * destroy the manager and unbinds all events\n   * it doesn't unbind dom events, that is the user own responsibility\n   */\n\n\n  _proto.destroy = function destroy() {\n    this.element && toggleCssProps(this, false);\n    this.handlers = {};\n    this.session = {};\n    this.input.destroy();\n    this.element = null;\n  };\n\n  return Manager;\n}();\n\nvar SINGLE_TOUCH_INPUT_MAP = {\n  touchstart: INPUT_START,\n  touchmove: INPUT_MOVE,\n  touchend: INPUT_END,\n  touchcancel: INPUT_CANCEL\n};\nvar SINGLE_TOUCH_TARGET_EVENTS = 'touchstart';\nvar SINGLE_TOUCH_WINDOW_EVENTS = 'touchstart touchmove touchend touchcancel';\n/**\n * @private\n * Touch events input\n * @constructor\n * @extends Input\n */\n\nvar SingleTouchInput =\n/*#__PURE__*/\nfunction (_Input) {\n  _inheritsLoose(SingleTouchInput, _Input);\n\n  function SingleTouchInput() {\n    var _this;\n\n    var proto = SingleTouchInput.prototype;\n    proto.evTarget = SINGLE_TOUCH_TARGET_EVENTS;\n    proto.evWin = SINGLE_TOUCH_WINDOW_EVENTS;\n    _this = _Input.apply(this, arguments) || this;\n    _this.started = false;\n    return _this;\n  }\n\n  var _proto = SingleTouchInput.prototype;\n\n  _proto.handler = function handler(ev) {\n    var type = SINGLE_TOUCH_INPUT_MAP[ev.type]; // should we handle the touch events?\n\n    if (type === INPUT_START) {\n      this.started = true;\n    }\n\n    if (!this.started) {\n      return;\n    }\n\n    var touches = normalizeSingleTouches.call(this, ev, type); // when done, reset the started state\n\n    if (type & (INPUT_END | INPUT_CANCEL) && touches[0].length - touches[1].length === 0) {\n      this.started = false;\n    }\n\n    this.callback(this.manager, type, {\n      pointers: touches[0],\n      changedPointers: touches[1],\n      pointerType: INPUT_TYPE_TOUCH,\n      srcEvent: ev\n    });\n  };\n\n  return SingleTouchInput;\n}(Input);\n\nfunction normalizeSingleTouches(ev, type) {\n  var all = toArray(ev.touches);\n  var changed = toArray(ev.changedTouches);\n\n  if (type & (INPUT_END | INPUT_CANCEL)) {\n    all = uniqueArray(all.concat(changed), 'identifier', true);\n  }\n\n  return [all, changed];\n}\n\n/**\n * @private\n * wrap a method with a deprecation warning and stack trace\n * @param {Function} method\n * @param {String} name\n * @param {String} message\n * @returns {Function} A new function wrapping the supplied method.\n */\nfunction deprecate(method, name, message) {\n  var deprecationMessage = \"DEPRECATED METHOD: \" + name + \"\\n\" + message + \" AT \\n\";\n  return function () {\n    var e = new Error('get-stack-trace');\n    var stack = e && e.stack ? e.stack.replace(/^[^\\(]+?[\\n$]/gm, '').replace(/^\\s+at\\s+/gm, '').replace(/^Object.<anonymous>\\s*\\(/gm, '{anonymous}()@') : 'Unknown Stack Trace';\n    var log = window.console && (window.console.warn || window.console.log);\n\n    if (log) {\n      log.call(window.console, deprecationMessage, stack);\n    }\n\n    return method.apply(this, arguments);\n  };\n}\n\n/**\n * @private\n * extend object.\n * means that properties in dest will be overwritten by the ones in src.\n * @param {Object} dest\n * @param {Object} src\n * @param {Boolean} [merge=false]\n * @returns {Object} dest\n */\n\nvar extend = deprecate(function (dest, src, merge) {\n  var keys = Object.keys(src);\n  var i = 0;\n\n  while (i < keys.length) {\n    if (!merge || merge && dest[keys[i]] === undefined) {\n      dest[keys[i]] = src[keys[i]];\n    }\n\n    i++;\n  }\n\n  return dest;\n}, 'extend', 'Use `assign`.');\n\n/**\n * @private\n * merge the values from src in the dest.\n * means that properties that exist in dest will not be overwritten by src\n * @param {Object} dest\n * @param {Object} src\n * @returns {Object} dest\n */\n\nvar merge = deprecate(function (dest, src) {\n  return extend(dest, src, true);\n}, 'merge', 'Use `assign`.');\n\n/**\n * @private\n * simple class inheritance\n * @param {Function} child\n * @param {Function} base\n * @param {Object} [properties]\n */\n\nfunction inherit(child, base, properties) {\n  var baseP = base.prototype;\n  var childP;\n  childP = child.prototype = Object.create(baseP);\n  childP.constructor = child;\n  childP._super = baseP;\n\n  if (properties) {\n    assign$1(childP, properties);\n  }\n}\n\n/**\n * @private\n * simple function bind\n * @param {Function} fn\n * @param {Object} context\n * @returns {Function}\n */\nfunction bindFn(fn, context) {\n  return function boundFn() {\n    return fn.apply(context, arguments);\n  };\n}\n\n/**\n * @private\n * Simple way to create a manager with a default set of recognizers.\n * @param {HTMLElement} element\n * @param {Object} [options]\n * @constructor\n */\n\nvar Hammer =\n/*#__PURE__*/\nfunction () {\n  var Hammer =\n  /**\n    * @private\n    * @const {string}\n    */\n  function Hammer(element, options) {\n    if (options === void 0) {\n      options = {};\n    }\n\n    return new Manager(element, _extends({\n      recognizers: preset.concat()\n    }, options));\n  };\n\n  Hammer.VERSION = \"2.0.17-rc\";\n  Hammer.DIRECTION_ALL = DIRECTION_ALL;\n  Hammer.DIRECTION_DOWN = DIRECTION_DOWN;\n  Hammer.DIRECTION_LEFT = DIRECTION_LEFT;\n  Hammer.DIRECTION_RIGHT = DIRECTION_RIGHT;\n  Hammer.DIRECTION_UP = DIRECTION_UP;\n  Hammer.DIRECTION_HORIZONTAL = DIRECTION_HORIZONTAL;\n  Hammer.DIRECTION_VERTICAL = DIRECTION_VERTICAL;\n  Hammer.DIRECTION_NONE = DIRECTION_NONE;\n  Hammer.DIRECTION_DOWN = DIRECTION_DOWN;\n  Hammer.INPUT_START = INPUT_START;\n  Hammer.INPUT_MOVE = INPUT_MOVE;\n  Hammer.INPUT_END = INPUT_END;\n  Hammer.INPUT_CANCEL = INPUT_CANCEL;\n  Hammer.STATE_POSSIBLE = STATE_POSSIBLE;\n  Hammer.STATE_BEGAN = STATE_BEGAN;\n  Hammer.STATE_CHANGED = STATE_CHANGED;\n  Hammer.STATE_ENDED = STATE_ENDED;\n  Hammer.STATE_RECOGNIZED = STATE_RECOGNIZED;\n  Hammer.STATE_CANCELLED = STATE_CANCELLED;\n  Hammer.STATE_FAILED = STATE_FAILED;\n  Hammer.Manager = Manager;\n  Hammer.Input = Input;\n  Hammer.TouchAction = TouchAction;\n  Hammer.TouchInput = TouchInput;\n  Hammer.MouseInput = MouseInput;\n  Hammer.PointerEventInput = PointerEventInput;\n  Hammer.TouchMouseInput = TouchMouseInput;\n  Hammer.SingleTouchInput = SingleTouchInput;\n  Hammer.Recognizer = Recognizer;\n  Hammer.AttrRecognizer = AttrRecognizer;\n  Hammer.Tap = TapRecognizer;\n  Hammer.Pan = PanRecognizer;\n  Hammer.Swipe = SwipeRecognizer;\n  Hammer.Pinch = PinchRecognizer;\n  Hammer.Rotate = RotateRecognizer;\n  Hammer.Press = PressRecognizer;\n  Hammer.on = addEventListeners;\n  Hammer.off = removeEventListeners;\n  Hammer.each = each;\n  Hammer.merge = merge;\n  Hammer.extend = extend;\n  Hammer.bindFn = bindFn;\n  Hammer.assign = assign$1;\n  Hammer.inherit = inherit;\n  Hammer.bindFn = bindFn;\n  Hammer.prefixed = prefixed;\n  Hammer.toArray = toArray;\n  Hammer.inArray = inArray;\n  Hammer.uniqueArray = uniqueArray;\n  Hammer.splitStr = splitStr;\n  Hammer.boolOrFn = boolOrFn;\n  Hammer.hasParent = hasParent;\n  Hammer.addEventListeners = addEventListeners;\n  Hammer.removeEventListeners = removeEventListeners;\n  Hammer.defaults = assign$1({}, defaults, {\n    preset: preset\n  });\n  return Hammer;\n}();\n\n//  style loader but by script tag, not by the loader.\n\nvar defaults$1 = Hammer.defaults;\n\nexport default Hammer;\nexport { INPUT_START, INPUT_MOVE, INPUT_END, INPUT_CANCEL, STATE_POSSIBLE, STATE_BEGAN, STATE_CHANGED, STATE_ENDED, STATE_RECOGNIZED, STATE_CANCELLED, STATE_FAILED, DIRECTION_NONE, DIRECTION_LEFT, DIRECTION_RIGHT, DIRECTION_UP, DIRECTION_DOWN, DIRECTION_HORIZONTAL, DIRECTION_VERTICAL, DIRECTION_ALL, Manager, Input, TouchAction, TouchInput, MouseInput, PointerEventInput, TouchMouseInput, SingleTouchInput, Recognizer, AttrRecognizer, TapRecognizer as Tap, PanRecognizer as Pan, SwipeRecognizer as Swipe, PinchRecognizer as Pinch, RotateRecognizer as Rotate, PressRecognizer as Press, addEventListeners as on, removeEventListeners as off, each, merge, extend, assign$1 as assign, inherit, bindFn, prefixed, toArray, inArray, uniqueArray, splitStr, boolOrFn, hasParent, addEventListeners, removeEventListeners, defaults$1 as defaults };\n//# sourceMappingURL=hammer.esm.js.map\n","import PropagatingHammer from 'propagating-hammerjs';\nimport Hammer from '@egjs/hammerjs';\n\n/**\n * Setup a mock hammer.js object, for unit testing.\n *\n * Inspiration: https://github.com/uber/deck.gl/pull/658\n *\n * @returns {{on: noop, off: noop, destroy: noop, emit: noop, get: get}}\n */\nfunction hammerMock() {\n  const noop = () => {};\n\n  return {\n    on: noop,\n    off: noop,\n    destroy: noop,\n    emit: noop,\n\n    get(m) {\t//eslint-disable-line no-unused-vars\n      return {\n        set: noop\n      };\n    }\n  };\n}\n\nlet modifiedHammer\n\nif (typeof window !== 'undefined') {\n  const OurHammer = window['Hammer'] || Hammer;\n  modifiedHammer = PropagatingHammer(OurHammer, {\n    preventDefault: 'mouse'\n  });\n} else {\n  modifiedHammer = () => // hammer.js is only available in a browser, not in node.js. Replacing it with a mock object.\n  hammerMock()\n}\n\nexport default modifiedHammer;\n","/**\n * Register a touch event, taking place before a gesture\n * @param {Hammer} hammer       A hammer instance\n * @param {function} callback   Callback, called as callback(event)\n */\nexport function onTouch (hammer, callback) {\n  callback.inputHandler = function (event) {\n    if (event.isFirst) {\n      callback(event);\n    }\n  };\n\n  hammer.on('hammer.input', callback.inputHandler);\n}\n\n/**\n * Register a release event, taking place after a gesture\n * @param {Hammer} hammer       A hammer instance\n * @param {function} callback   Callback, called as callback(event)\n * @returns {*}\n */\nexport function onRelease (hammer, callback) {\n  callback.inputHandler = function (event) {\n    if (event.isFinal) {\n      callback(event);\n    }\n  };\n\n  return hammer.on('hammer.input', callback.inputHandler);\n}\n\n\n/**\n * Unregister a touch event, taking place before a gesture\n * @param {Hammer} hammer       A hammer instance\n * @param {function} callback   Callback, called as callback(event)\n */\nexport function offTouch (hammer, callback) {\n  hammer.off('hammer.input', callback.inputHandler);\n}\n\n/**\n * Unregister a release event, taking place before a gesture\n * @param {Hammer} hammer       A hammer instance\n * @param {function} callback   Callback, called as callback(event)\n */\n\nexport const offRelease = offTouch;\n\n/**\n * Hack the PinchRecognizer such that it doesn't prevent default behavior\n * for vertical panning.\n *\n * Yeah ... this is quite a hack ... see https://github.com/hammerjs/hammer.js/issues/932\n *\n * @param {Hammer.Pinch} pinchRecognizer\n * @return {Hammer.Pinch} returns the pinchRecognizer\n */\nexport function disablePreventDefaultVertically (pinchRecognizer) {\n  const TOUCH_ACTION_PAN_Y = 'pan-y';\n\n  pinchRecognizer.getTouchAction = function() {\n    // default method returns [TOUCH_ACTION_NONE]\n    return [TOUCH_ACTION_PAN_Y];\n  };\n\n  return pinchRecognizer;\n}\n","import moment from '../module/moment';\nimport * as DateUtil  from './DateUtil';\nimport util from '../util';\n\n/**\n * The class TimeStep is an iterator for dates. You provide a start date and an\n * end date. The class itself determines the best scale (step size) based on the\n * provided start Date, end Date, and minimumStep.\n *\n * If minimumStep is provided, the step size is chosen as close as possible\n * to the minimumStep but larger than minimumStep. If minimumStep is not\n * provided, the scale is set to 1 DAY.\n * The minimumStep should correspond with the onscreen size of about 6 characters\n *\n * Alternatively, you can set a scale by hand.\n * After creation, you can initialize the class by executing first(). Then you\n * can iterate from the start date to the end date via next(). You can check if\n * the end date is reached with the function hasNext(). After each step, you can\n * retrieve the current date via getCurrent().\n * The TimeStep has scales ranging from milliseconds, seconds, minutes, hours,\n * days, to years.\n *\n * Version: 1.2\n *\n */\nclass TimeStep {\n  /**\n    * @param {Date} [start]         The start date, for example new Date(2010, 9, 21)\n    *                               or new Date(2010, 9, 21, 23, 45, 00)\n    * @param {Date} [end]           The end date\n    * @param {number} [minimumStep] Optional. Minimum step size in milliseconds\n    * @param {Date|Array.<Date>} [hiddenDates] Optional.\n    * @param {{showMajorLabels: boolean, showWeekScale: boolean}} [options] Optional.\n    * @constructor  TimeStep\n    */\n  constructor(start, end, minimumStep, hiddenDates, options) {\n    this.moment = (options && options.moment) || moment;\n    this.options = options ? options : {};\n\n    // variables\n    this.current = this.moment();\n    this._start = this.moment();\n    this._end = this.moment();\n\n    this.autoScale  = true;\n    this.scale = 'day';\n    this.step = 1;\n\n    // initialize the range\n    this.setRange(start, end, minimumStep);\n\n    // hidden Dates options\n    this.switchedDay = false;\n    this.switchedMonth = false;\n    this.switchedYear = false;\n    if (Array.isArray(hiddenDates)) {\n      this.hiddenDates = hiddenDates;\n    }\n    else if (hiddenDates != undefined) {\n      this.hiddenDates = [hiddenDates];\n    }\n    else {\n      this.hiddenDates = [];\n    }\n\n    this.format = TimeStep.FORMAT; // default formatting\n  }\n\n  /**\n   * Set custom constructor function for moment. Can be used to set dates\n   * to UTC or to set a utcOffset.\n   * @param {function} moment\n   */\n  setMoment(moment) {\n    this.moment = moment;\n\n    // update the date properties, can have a new utcOffset\n    this.current = this.moment(this.current.valueOf());\n    this._start = this.moment(this._start.valueOf());\n    this._end = this.moment(this._end.valueOf());\n  }\n\n  /**\n   * Set custom formatting for the minor an major labels of the TimeStep.\n   * Both `minorLabels` and `majorLabels` are an Object with properties:\n   * 'millisecond', 'second', 'minute', 'hour', 'weekday', 'day', 'week', 'month', 'year'.\n   * @param {{minorLabels: Object, majorLabels: Object}} format\n   */\n  setFormat(format) {\n    const defaultFormat = util.deepExtend({}, TimeStep.FORMAT);\n    this.format = util.deepExtend(defaultFormat, format);\n  }\n\n  /**\n   * Set a new range\n   * If minimumStep is provided, the step size is chosen as close as possible\n   * to the minimumStep but larger than minimumStep. If minimumStep is not\n   * provided, the scale is set to 1 DAY.\n   * The minimumStep should correspond with the onscreen size of about 6 characters\n   * @param {Date} [start]      The start date and time.\n   * @param {Date} [end]        The end date and time.\n   * @param {int} [minimumStep] Optional. Minimum step size in milliseconds\n   */\n  setRange(start, end, minimumStep) {\n    if (!(start instanceof Date) || !(end instanceof Date)) {\n      throw  \"No legal start or end date in method setRange\";\n    }\n\n    this._start = (start != undefined) ? this.moment(start.valueOf()) : Date.now();\n    this._end = (end != undefined) ? this.moment(end.valueOf()) : Date.now();\n\n    if (this.autoScale) {\n      this.setMinimumStep(minimumStep);\n    }\n  }\n\n  /**\n   * Set the range iterator to the start date.\n   */\n  start() {\n    this.current = this._start.clone();\n    this.roundToMinor();\n  }\n\n  /**\n   * Round the current date to the first minor date value\n   * This must be executed once when the current date is set to start Date\n   */\n  roundToMinor() {\n    // round to floor\n    // to prevent year & month scales rounding down to the first day of week we perform this separately\n    if (this.scale == 'week') {\n      this.current.weekday(0);\n    }\n    // IMPORTANT: we have no breaks in this switch! (this is no bug)\n    // noinspection FallThroughInSwitchStatementJS\n    switch (this.scale) {\n      case 'year':\n        this.current.year(this.step * Math.floor(this.current.year() / this.step));\n        this.current.month(0);\n      case 'month':        this.current.date(1);          // eslint-disable-line no-fallthrough\n      case 'week':                                        // eslint-disable-line no-fallthrough\n      case 'day':                                         // eslint-disable-line no-fallthrough\n      case 'weekday':      this.current.hours(0);         // eslint-disable-line no-fallthrough\n      case 'hour':         this.current.minutes(0);       // eslint-disable-line no-fallthrough\n      case 'minute':       this.current.seconds(0);       // eslint-disable-line no-fallthrough\n      case 'second':       this.current.milliseconds(0);  // eslint-disable-line no-fallthrough\n      //case 'millisecond': // nothing to do for milliseconds\n    }\n\n    if (this.step != 1) {\n      // round down to the first minor value that is a multiple of the current step size\n      let  priorCurrent = this.current.clone();\n      switch (this.scale) {        \n        case 'millisecond':  this.current.subtract(this.current.milliseconds() % this.step, 'milliseconds');  break;\n        case 'second':       this.current.subtract(this.current.seconds() % this.step, 'seconds'); break;\n        case 'minute':       this.current.subtract(this.current.minutes() % this.step, 'minutes'); break;\n        case 'hour':         this.current.subtract(this.current.hours() % this.step, 'hours'); break;\n        case 'weekday':      // intentional fall through\n        case 'day':          this.current.subtract((this.current.date() - 1) % this.step, 'day'); break;\n        case 'week':         this.current.subtract(this.current.week() % this.step, 'week'); break;\n        case 'month':        this.current.subtract(this.current.month() % this.step, 'month');  break;\n        case 'year':         this.current.subtract(this.current.year() % this.step, 'year'); break;\n        default: break;\n      }\n      if (!priorCurrent.isSame(this.current)) {\n          this.current = this.moment(DateUtil.snapAwayFromHidden(this.hiddenDates, this.current.valueOf(), -1, true));\n      }\n    }\n  }\n\n  /**\n   * Check if the there is a next step\n   * @return {boolean}  true if the current date has not passed the end date\n   */\n  hasNext() {\n    return (this.current.valueOf() <= this._end.valueOf());\n  }\n\n  /**\n   * Do the next step\n   */\n  next() {\n    const prev = this.current.valueOf();\n\n    // Two cases, needed to prevent issues with switching daylight savings\n    // (end of March and end of October)\n    switch (this.scale) {\n      case 'millisecond':  this.current.add(this.step, 'millisecond'); break;\n      case 'second':       this.current.add(this.step, 'second'); break;\n      case 'minute':       this.current.add(this.step, 'minute'); break;\n      case 'hour':\n        this.current.add(this.step, 'hour');\n\n        if (this.current.month() < 6) {\n          this.current.subtract(this.current.hours() % this.step, 'hour');\n        } else {\n          if (this.current.hours() % this.step !== 0) {\n            this.current.add(this.step - this.current.hours() % this.step, 'hour');\n          }\n        }\n        break;\n      case 'weekday':      // intentional fall through\n      case 'day':          this.current.add(this.step, 'day'); break;\n      case 'week':\n        if (this.current.weekday() !== 0){ // we had a month break not correlating with a week's start before\n          this.current.weekday(0); // switch back to week cycles\n          this.current.add(this.step, 'week');\n        } else if(this.options.showMajorLabels === false) {\n          this.current.add(this.step, 'week'); // the default case\n        } else { // first day of the week\n          const nextWeek = this.current.clone();\n          nextWeek.add(1, 'week');\n          if(nextWeek.isSame(this.current, 'month')){ // is the first day of the next week in the same month?\n            this.current.add(this.step, 'week'); // the default case\n          } else { // inject a step at each first day of the month\n            this.current.add(this.step, 'week');\n            this.current.date(1);\n          }\n        }\n        break;\n      case 'month':        this.current.add(this.step, 'month'); break;\n      case 'year':         this.current.add(this.step, 'year'); break;\n      default: break;\n    }\n\n    if (this.step != 1) {\n      // round down to the correct major value\n      switch (this.scale) {\n        case 'millisecond':  if(this.current.milliseconds() > 0 && this.current.milliseconds() < this.step) this.current.milliseconds(0);  break;\n        case 'second':       if(this.current.seconds() > 0 && this.current.seconds() < this.step) this.current.seconds(0);  break;\n        case 'minute':       if(this.current.minutes() > 0 && this.current.minutes() < this.step) this.current.minutes(0); break;\n        case 'hour':         if(this.current.hours() > 0 && this.current.hours() < this.step) this.current.hours(0);  break;\n        case 'weekday':      // intentional fall through\n        case 'day':          if(this.current.date() < this.step+1) this.current.date(1); break;\n        case 'week':         if(this.current.week() < this.step) this.current.week(1); break; // week numbering starts at 1, not 0\n        case 'month':        if(this.current.month() < this.step) this.current.month(0);  break;\n        case 'year':         break; // nothing to do for year\n        default:             break;\n      }\n    }\n\n    // safety mechanism: if current time is still unchanged, move to the end\n    if (this.current.valueOf() == prev) {\n      this.current = this._end.clone();\n    }\n\n    // Reset switches for year, month and day. Will get set to true where appropriate in DateUtil.stepOverHiddenDates\n    this.switchedDay = false;\n    this.switchedMonth = false;\n    this.switchedYear = false;\n\n    DateUtil.stepOverHiddenDates(this.moment, this, prev);\n  }\n\n  /**\n   * Get the current datetime\n   * @return {Moment}  current The current date\n   */\n  getCurrent() {\n    return this.current.clone();\n  }\n\n  /**\n   * Set a custom scale. Autoscaling will be disabled.\n   * For example setScale('minute', 5) will result\n   * in minor steps of 5 minutes, and major steps of an hour.\n   *\n   * @param {{scale: string, step: number}} params\n   *                               An object containing two properties:\n   *                               - A string 'scale'. Choose from 'millisecond', 'second',\n   *                                 'minute', 'hour', 'weekday', 'day', 'week', 'month', 'year'.\n   *                               - A number 'step'. A step size, by default 1.\n   *                                 Choose for example 1, 2, 5, or 10.\n   */\n  setScale(params) {\n    if (params && typeof params.scale == 'string') {\n      this.scale = params.scale;\n      this.step = params.step > 0 ? params.step : 1;\n      this.autoScale = false;\n    }\n  }\n\n  /**\n   * Enable or disable autoscaling\n   * @param {boolean} enable  If true, autoascaling is set true\n   */\n  setAutoScale(enable) {\n    this.autoScale = enable;\n  }\n\n  /**\n   * Automatically determine the scale that bests fits the provided minimum step\n   * @param {number} [minimumStep]  The minimum step size in milliseconds\n   */\n  setMinimumStep(minimumStep) {\n    if (minimumStep == undefined) {\n      return;\n    }\n\n    //var b = asc + ds;\n\n    const stepYear       = (1000 * 60 * 60 * 24 * 30 * 12);\n    const stepMonth      = (1000 * 60 * 60 * 24 * 30);\n    const stepDay        = (1000 * 60 * 60 * 24);\n    const stepHour       = (1000 * 60 * 60);\n    const stepMinute     = (1000 * 60);\n    const stepSecond     = (1000);\n    const stepMillisecond= (1);\n\n    // find the smallest step that is larger than the provided minimumStep\n    if (stepYear*1000 > minimumStep)        {this.scale = 'year';        this.step = 1000;}\n    if (stepYear*500 > minimumStep)         {this.scale = 'year';        this.step = 500;}\n    if (stepYear*100 > minimumStep)         {this.scale = 'year';        this.step = 100;}\n    if (stepYear*50 > minimumStep)          {this.scale = 'year';        this.step = 50;}\n    if (stepYear*10 > minimumStep)          {this.scale = 'year';        this.step = 10;}\n    if (stepYear*5 > minimumStep)           {this.scale = 'year';        this.step = 5;}\n    if (stepYear > minimumStep)             {this.scale = 'year';        this.step = 1;}\n    if (stepMonth*3 > minimumStep)          {this.scale = 'month';       this.step = 3;}\n    if (stepMonth > minimumStep)            {this.scale = 'month';       this.step = 1;}\n    if (stepDay*7 > minimumStep && this.options.showWeekScale)            {this.scale = 'week';        this.step = 1;}\n    if (stepDay*2 > minimumStep)            {this.scale = 'day';         this.step = 2;}\n    if (stepDay > minimumStep)              {this.scale = 'day';         this.step = 1;}\n    if (stepDay/2 > minimumStep)            {this.scale = 'weekday';     this.step = 1;}\n    if (stepHour*4 > minimumStep)           {this.scale = 'hour';        this.step = 4;}\n    if (stepHour > minimumStep)             {this.scale = 'hour';        this.step = 1;}\n    if (stepMinute*15 > minimumStep)        {this.scale = 'minute';      this.step = 15;}\n    if (stepMinute*10 > minimumStep)        {this.scale = 'minute';      this.step = 10;}\n    if (stepMinute*5 > minimumStep)         {this.scale = 'minute';      this.step = 5;}\n    if (stepMinute > minimumStep)           {this.scale = 'minute';      this.step = 1;}\n    if (stepSecond*15 > minimumStep)        {this.scale = 'second';      this.step = 15;}\n    if (stepSecond*10 > minimumStep)        {this.scale = 'second';      this.step = 10;}\n    if (stepSecond*5 > minimumStep)         {this.scale = 'second';      this.step = 5;}\n    if (stepSecond > minimumStep)           {this.scale = 'second';      this.step = 1;}\n    if (stepMillisecond*200 > minimumStep)  {this.scale = 'millisecond'; this.step = 200;}\n    if (stepMillisecond*100 > minimumStep)  {this.scale = 'millisecond'; this.step = 100;}\n    if (stepMillisecond*50 > minimumStep)   {this.scale = 'millisecond'; this.step = 50;}\n    if (stepMillisecond*10 > minimumStep)   {this.scale = 'millisecond'; this.step = 10;}\n    if (stepMillisecond*5 > minimumStep)    {this.scale = 'millisecond'; this.step = 5;}\n    if (stepMillisecond > minimumStep)      {this.scale = 'millisecond'; this.step = 1;}\n  }\n\n  /**\n   * Snap a date to a rounded value.\n   * The snap intervals are dependent on the current scale and step.\n   * Static function\n   * @param {Date} date    the date to be snapped.\n   * @param {string} scale Current scale, can be 'millisecond', 'second',\n   *                       'minute', 'hour', 'weekday, 'day', 'week', 'month', 'year'.\n   * @param {number} step  Current step (1, 2, 4, 5, ...\n   * @return {Date} snappedDate\n   */\n  static snap(date, scale, step) {\n    const clone = moment(date);\n\n    if (scale == 'year') {\n      const year = clone.year() + Math.round(clone.month() / 12);\n      clone.year(Math.round(year / step) * step);\n      clone.month(0);\n      clone.date(0);\n      clone.hours(0);\n      clone.minutes(0);\n      clone.seconds(0);\n      clone.milliseconds(0);\n    }\n    else if (scale == 'month') {\n      if (clone.date() > 15) {\n        clone.date(1);\n        clone.add(1, 'month');\n        // important: first set Date to 1, after that change the month.\n      }\n      else {\n        clone.date(1);\n      }\n\n      clone.hours(0);\n      clone.minutes(0);\n      clone.seconds(0);\n      clone.milliseconds(0);\n    }\n    else if (scale == 'week') {\n        if (clone.weekday() > 2) { // doing it the momentjs locale aware way\n            clone.weekday(0);\n            clone.add(1, 'week');\n        }\n        else {\n            clone.weekday(0);\n        }\n\n        clone.hours(0);\n        clone.minutes(0);\n        clone.seconds(0);\n        clone.milliseconds(0);\n    }\n    else if (scale == 'day') {\n      //noinspection FallthroughInSwitchStatementJS\n      switch (step) {\n        case 5:\n        case 2:\n          clone.hours(Math.round(clone.hours() / 24) * 24); break;\n        default:\n          clone.hours(Math.round(clone.hours() / 12) * 12); break;\n      }\n      clone.minutes(0);\n      clone.seconds(0);\n      clone.milliseconds(0);\n    }\n    else if (scale == 'weekday') {\n      //noinspection FallthroughInSwitchStatementJS\n      switch (step) {\n        case 5:\n        case 2:\n          clone.hours(Math.round(clone.hours() / 12) * 12); break;\n        default:\n          clone.hours(Math.round(clone.hours() / 6) * 6); break;\n      }\n      clone.minutes(0);\n      clone.seconds(0);\n      clone.milliseconds(0);\n    }\n    else if (scale == 'hour') {\n      switch (step) {\n        case 4:\n          clone.minutes(Math.round(clone.minutes() / 60) * 60); break;\n        default:\n          clone.minutes(Math.round(clone.minutes() / 30) * 30); break;\n      }\n      clone.seconds(0);\n      clone.milliseconds(0);\n    } else if (scale == 'minute') {\n      //noinspection FallthroughInSwitchStatementJS\n      switch (step) {\n        case 15:\n        case 10:\n          clone.minutes(Math.round(clone.minutes() / 5) * 5);\n          clone.seconds(0);\n          break;\n        case 5:\n          clone.seconds(Math.round(clone.seconds() / 60) * 60); break;\n        default:\n          clone.seconds(Math.round(clone.seconds() / 30) * 30); break;\n      }\n      clone.milliseconds(0);\n    }\n    else if (scale == 'second') {\n      //noinspection FallthroughInSwitchStatementJS\n      switch (step) {\n        case 15:\n        case 10:\n          clone.seconds(Math.round(clone.seconds() / 5) * 5);\n          clone.milliseconds(0);\n          break;\n        case 5:\n          clone.milliseconds(Math.round(clone.milliseconds() / 1000) * 1000); break;\n        default:\n          clone.milliseconds(Math.round(clone.milliseconds() / 500) * 500); break;\n      }\n    }\n    else if (scale == 'millisecond') {\n      const _step = step > 5 ? step / 2 : 1;\n      clone.milliseconds(Math.round(clone.milliseconds() / _step) * _step);\n    }\n\n    return clone;\n  }\n\n  /**\n   * Check if the current value is a major value (for example when the step\n   * is DAY, a major value is each first day of the MONTH)\n   * @return {boolean} true if current date is major, else false.\n   */\n  isMajor() {\n    if (this.switchedYear == true) {\n      switch (this.scale) {\n        case 'year':\n        case 'month':\n        case 'week':\n        case 'weekday':\n        case 'day':\n        case 'hour':\n        case 'minute':\n        case 'second':\n        case 'millisecond':\n          return true;\n        default:\n          return false;\n      }\n    }\n    else if (this.switchedMonth == true) {\n      switch (this.scale) {\n        case 'week':\n        case 'weekday':\n        case 'day':\n        case 'hour':\n        case 'minute':\n        case 'second':\n        case 'millisecond':\n          return true;\n        default:\n          return false;\n      }\n    }\n    else if (this.switchedDay == true) {\n      switch (this.scale) {\n        case 'millisecond':\n        case 'second':\n        case 'minute':\n        case 'hour':\n          return true;\n        default:\n          return false;\n      }\n    }\n\n    const date = this.moment(this.current);\n    switch (this.scale) {\n      case 'millisecond':\n        return (date.milliseconds() == 0);\n      case 'second':\n        return (date.seconds() == 0);\n      case 'minute':\n        return (date.hours() == 0) && (date.minutes() == 0);\n      case 'hour':\n        return (date.hours() == 0);\n      case 'weekday': // intentional fall through\n      case 'day':\n        return (date.date() == 1);\n      case 'week':\n        return (date.date() == 1);\n      case 'month':\n        return (date.month() == 0);\n      case 'year':\n        return false;\n      default:\n        return false;\n    }\n  }\n\n  /**\n   * Returns formatted text for the minor axislabel, depending on the current\n   * date and the scale. For example when scale is MINUTE, the current time is\n   * formatted as \"hh:mm\".\n   * @param {Date} [date=this.current] custom date. if not provided, current date is taken\n   * @returns {String}\n   */\n  getLabelMinor(date) {\n    if (date == undefined) {\n      date = this.current;\n    }\n    if (date instanceof Date) {\n      date = this.moment(date)\n    }\n\n    if (typeof(this.format.minorLabels) === \"function\") {\n      return this.format.minorLabels(date, this.scale, this.step);\n    }\n\n    const format = this.format.minorLabels[this.scale];\n    // noinspection FallThroughInSwitchStatementJS\n    switch (this.scale) {\n      case 'week':\n        // Don't draw the minor label if this date is the first day of a month AND if it's NOT the start of the week.\n        // The 'date' variable may actually be the 'next' step when called from TimeAxis' _repaintLabels.\n        if(date.date() === 1 && date.weekday() !== 0){\n            return \"\";\n        }\n      default: // eslint-disable-line no-fallthrough\n        return (format && format.length > 0) ? this.moment(date).format(format) : '';\n    }\n  }\n\n  /**\n   * Returns formatted text for the major axis label, depending on the current\n   * date and the scale. For example when scale is MINUTE, the major scale is\n   * hours, and the hour will be formatted as \"hh\".\n   * @param {Date} [date=this.current] custom date. if not provided, current date is taken\n   * @returns {String}\n   */\n  getLabelMajor(date) {\n    if (date == undefined) {\n      date = this.current;\n    }\n    if (date instanceof Date) {\n      date = this.moment(date)\n    }\n\n    if (typeof(this.format.majorLabels) === \"function\") {\n      return this.format.majorLabels(date, this.scale, this.step);\n    }\n\n    const format = this.format.majorLabels[this.scale];\n    return (format && format.length > 0) ? this.moment(date).format(format) : '';\n  }\n\n  /**\n   * get class name\n   * @return {string} class name\n   */\n  getClassName() {\n    const _moment = this.moment;\n    const m = this.moment(this.current);\n    const current = m.locale ? m.locale('en') : m.lang('en'); // old versions of moment have .lang() function\n    const step = this.step;\n    const classNames = [];\n\n    /**\n     *\n     * @param {number} value\n     * @returns {String}\n     */\n    function even(value) {\n      return (value / step % 2 == 0) ? ' vis-even' : ' vis-odd';\n    }\n\n    /**\n     *\n     * @param {Date} date\n     * @returns {String}\n     */\n    function today(date) {\n      if (date.isSame(Date.now(), 'day')) {\n        return ' vis-today';\n      }\n      if (date.isSame(_moment().add(1, 'day'), 'day')) {\n        return ' vis-tomorrow';\n      }\n      if (date.isSame(_moment().add(-1, 'day'), 'day')) {\n        return ' vis-yesterday';\n      }\n      return '';\n    }\n\n    /**\n     *\n     * @param {Date} date\n     * @returns {String}\n     */\n    function currentWeek(date) {\n      return date.isSame(Date.now(), 'week') ? ' vis-current-week' : '';\n    }\n\n    /**\n     *\n     * @param {Date} date\n     * @returns {String}\n     */\n    function currentMonth(date) {\n      return date.isSame(Date.now(), 'month') ? ' vis-current-month' : '';\n    }\n\n    /**\n     *\n     * @param {Date} date\n     * @returns {String}\n     */\n    function currentYear(date) {\n      return date.isSame(Date.now(), 'year') ? ' vis-current-year' : '';\n    }\n\n    switch (this.scale) {\n      case 'millisecond':\n        classNames.push(today(current));\n        classNames.push(even(current.milliseconds()));\n        break;\n      case 'second':\n        classNames.push(today(current));\n        classNames.push(even(current.seconds()));\n        break;\n      case 'minute':\n        classNames.push(today(current));\n        classNames.push(even(current.minutes()));\n        break;\n      case 'hour':\n        classNames.push(`vis-h${current.hours()}${this.step == 4 ? '-h' + (current.hours() + 4) : ''}`);\n        classNames.push(today(current));\n        classNames.push(even(current.hours()));\n        break;\n      case 'weekday':\n        classNames.push(`vis-${current.format('dddd').toLowerCase()}`);\n        classNames.push(today(current));\n        classNames.push(currentWeek(current));\n        classNames.push(even(current.date()));\n        break;\n      case 'day':\n        classNames.push(`vis-day${current.date()}`);\n        classNames.push(`vis-${current.format('MMMM').toLowerCase()}`);\n        classNames.push(today(current));\n        classNames.push(currentMonth(current));\n        classNames.push(this.step <= 2 ? today(current) : '');\n        classNames.push(this.step <= 2 ? `vis-${current.format('dddd').toLowerCase()}` : '');\n        classNames.push(even(current.date() - 1));\n        break;\n      case 'week':\n        classNames.push(`vis-week${current.format('w')}`);\n        classNames.push(currentWeek(current));\n        classNames.push(even(current.week()));\n        break;\n      case 'month':\n        classNames.push(`vis-${current.format('MMMM').toLowerCase()}`);\n        classNames.push(currentMonth(current));\n        classNames.push(even(current.month()));\n        break;\n      case 'year':\n        classNames.push(`vis-year${current.year()}`);\n        classNames.push(currentYear(current));\n        classNames.push(even(current.year()));\n        break;\n    }\n    return classNames.filter(String).join(\" \");\n  }\n}\n\n// Time formatting\nTimeStep.FORMAT = {\n  minorLabels: {\n    millisecond:'SSS',\n    second:     's',\n    minute:     'HH:mm',\n    hour:       'HH:mm',\n    weekday:    'ddd D',\n    day:        'D',\n    week:       'w',\n    month:      'MMM',\n    year:       'YYYY'\n  },\n  majorLabels: {\n    millisecond:'HH:mm:ss',\n    second:     'D MMMM HH:mm',\n    minute:     'ddd D MMMM',\n    hour:       'ddd D MMMM',\n    weekday:    'MMMM YYYY',\n    day:        'MMMM YYYY',\n    week:       'MMMM YYYY',\n    month:      'YYYY',\n    year:       ''\n  }\n};\n\nexport default TimeStep;\n","function styleInject(css, ref) {\n  if ( ref === void 0 ) ref = {};\n  var insertAt = ref.insertAt;\n\n  if (!css || typeof document === 'undefined') { return; }\n\n  var head = document.head || document.getElementsByTagName('head')[0];\n  var style = document.createElement('style');\n  style.type = 'text/css';\n\n  if (insertAt === 'top') {\n    if (head.firstChild) {\n      head.insertBefore(style, head.firstChild);\n    } else {\n      head.appendChild(style);\n    }\n  } else {\n    head.appendChild(style);\n  }\n\n  if (style.styleSheet) {\n    style.styleSheet.cssText = css;\n  } else {\n    style.appendChild(document.createTextNode(css));\n  }\n}\n\nexport default styleInject;\n","import util from '../../util';\nimport Component from'./Component';\nimport TimeStep from '../TimeStep';\nimport * as DateUtil  from '../DateUtil';\nimport moment from '../../module/moment';\n\nimport './css/timeaxis.css';\n\n/** A horizontal time axis */\nclass TimeAxis extends Component {\n/**\n * @param {{dom: Object, domProps: Object, emitter: Emitter, range: Range}} body\n * @param {Object} [options]        See TimeAxis.setOptions for the available\n *                                  options.\n * @constructor TimeAxis\n * @extends Component\n */\n  constructor(body, options) {\n    super();\n    this.dom = {\n      foreground: null,\n      lines: [],\n      majorTexts: [],\n      minorTexts: [],\n      redundant: {\n        lines: [],\n        majorTexts: [],\n        minorTexts: []\n      }\n    };\n    this.props = {\n      range: {\n        start: 0,\n        end: 0,\n        minimumStep: 0\n      },\n      lineTop: 0\n    };\n\n    this.defaultOptions = {\n      orientation: {\n        axis: 'bottom'\n      },  // axis orientation: 'top' or 'bottom'\n      showMinorLabels: true,\n      showMajorLabels: true,\n      showWeekScale: false,\n      maxMinorChars: 7,\n      format: TimeStep.FORMAT,\n      moment,\n      timeAxis: null\n    };\n    this.options = util.extend({}, this.defaultOptions);\n\n    this.body = body;\n\n    // create the HTML DOM\n    this._create();\n\n    this.setOptions(options);\n  }\n\n  /**\n   * Set options for the TimeAxis.\n   * Parameters will be merged in current options.\n   * @param {Object} options  Available options:\n   *                          {string} [orientation.axis]\n   *                          {boolean} [showMinorLabels]\n   *                          {boolean} [showMajorLabels]\n   *                          {boolean} [showWeekScale]\n   */\n  setOptions(options) {\n    if (options) {\n      // copy all options that we know\n      util.selectiveExtend([\n        'showMinorLabels',\n        'showMajorLabels',\n        'showWeekScale',\n        'maxMinorChars',\n        'hiddenDates',\n        'timeAxis',\n        'moment',\n        'rtl'\n      ], this.options, options);\n\n      // deep copy the format options\n      util.selectiveDeepExtend(['format'], this.options, options);\n\n      if ('orientation' in options) {\n        if (typeof options.orientation === 'string') {\n          this.options.orientation.axis = options.orientation;\n        }\n        else if (typeof options.orientation === 'object' && 'axis' in options.orientation) {\n          this.options.orientation.axis = options.orientation.axis;\n        }\n      }\n\n      // apply locale to moment.js\n      // TODO: not so nice, this is applied globally to moment.js\n      if ('locale' in options) {\n        if (typeof moment.locale === 'function') {\n          // moment.js 2.8.1+\n          moment.locale(options.locale);\n        }\n        else {\n          moment.lang(options.locale);\n        }\n      }\n    }\n  }\n\n  /**\n   * Create the HTML DOM for the TimeAxis\n   */\n  _create() {\n    this.dom.foreground = document.createElement('div');\n    this.dom.background = document.createElement('div');\n\n    this.dom.foreground.className = 'vis-time-axis vis-foreground';\n    this.dom.background.className = 'vis-time-axis vis-background';\n  }\n\n  /**\n   * Destroy the TimeAxis\n   */\n  destroy() {\n    // remove from DOM\n    if (this.dom.foreground.parentNode) {\n      this.dom.foreground.parentNode.removeChild(this.dom.foreground);\n    }\n    if (this.dom.background.parentNode) {\n      this.dom.background.parentNode.removeChild(this.dom.background);\n    }\n\n    this.body = null;\n  }\n\n  /**\n   * Repaint the component\n   * @return {boolean} Returns true if the component is resized\n   */\n  redraw() {\n    const props = this.props;\n    const foreground = this.dom.foreground;\n    const background = this.dom.background;\n\n    // determine the correct parent DOM element (depending on option orientation)\n    const parent = (this.options.orientation.axis == 'top') ? this.body.dom.top : this.body.dom.bottom;\n    const parentChanged = (foreground.parentNode !== parent);\n\n    // calculate character width and height\n    this._calculateCharSize();\n\n    // TODO: recalculate sizes only needed when parent is resized or options is changed\n    const showMinorLabels = this.options.showMinorLabels && this.options.orientation.axis !== 'none';\n    const showMajorLabels = this.options.showMajorLabels && this.options.orientation.axis !== 'none';\n\n    // determine the width and height of the elemens for the axis\n    props.minorLabelHeight = showMinorLabels ? props.minorCharHeight : 0;\n    props.majorLabelHeight = showMajorLabels ? props.majorCharHeight : 0;\n    props.height = props.minorLabelHeight + props.majorLabelHeight;\n    props.width = foreground.offsetWidth;\n\n    props.minorLineHeight = this.body.domProps.root.height - props.majorLabelHeight -\n        (this.options.orientation.axis == 'top' ? this.body.domProps.bottom.height : this.body.domProps.top.height);\n    props.minorLineWidth = 1; // TODO: really calculate width\n    props.majorLineHeight = props.minorLineHeight + props.majorLabelHeight;\n    props.majorLineWidth = 1; // TODO: really calculate width\n\n    //  take foreground and background offline while updating (is almost twice as fast)\n    const foregroundNextSibling = foreground.nextSibling;\n    const backgroundNextSibling = background.nextSibling;\n    foreground.parentNode && foreground.parentNode.removeChild(foreground);\n    background.parentNode && background.parentNode.removeChild(background);\n\n    foreground.style.height = `${this.props.height}px`;\n\n    this._repaintLabels();\n\n    // put DOM online again (at the same place)\n    if (foregroundNextSibling) {\n      parent.insertBefore(foreground, foregroundNextSibling);\n    }\n    else {\n      parent.appendChild(foreground)\n    }\n    if (backgroundNextSibling) {\n      this.body.dom.backgroundVertical.insertBefore(background, backgroundNextSibling);\n    }\n    else {\n      this.body.dom.backgroundVertical.appendChild(background)\n    }\n    return this._isResized() || parentChanged;\n  }\n\n  /**\n   * Repaint major and minor text labels and vertical grid lines\n   * @private\n   */\n  _repaintLabels() {\n    const orientation = this.options.orientation.axis;\n\n    // calculate range and step (step such that we have space for 7 characters per label)\n    const start = util.convert(this.body.range.start, 'Number');\n    const end = util.convert(this.body.range.end, 'Number');\n    const timeLabelsize = this.body.util.toTime((this.props.minorCharWidth || 10) * this.options.maxMinorChars).valueOf();\n    let minimumStep = timeLabelsize - DateUtil.getHiddenDurationBefore(this.options.moment, this.body.hiddenDates, this.body.range, timeLabelsize);\n    minimumStep -= this.body.util.toTime(0).valueOf();\n\n    const step = new TimeStep(new Date(start), new Date(end), minimumStep, this.body.hiddenDates, this.options);\n    step.setMoment(this.options.moment);\n    if (this.options.format) {\n      step.setFormat(this.options.format);\n    }\n    if (this.options.timeAxis) {\n      step.setScale(this.options.timeAxis);\n    }\n    this.step = step;\n\n    // Move all DOM elements to a \"redundant\" list, where they\n    // can be picked for re-use, and clear the lists with lines and texts.\n    // At the end of the function _repaintLabels, left over elements will be cleaned up\n    const dom = this.dom;\n    dom.redundant.lines = dom.lines;\n    dom.redundant.majorTexts = dom.majorTexts;\n    dom.redundant.minorTexts = dom.minorTexts;\n    dom.lines = [];\n    dom.majorTexts = [];\n    dom.minorTexts = [];\n\n    let current;\n    let next;\n    let x;\n    let xNext;\n    let isMajor;\n    let showMinorGrid;\n    let width = 0;\n    let prevWidth;\n    let line;\n    let xFirstMajorLabel = undefined;\n    let count = 0;\n    const MAX = 1000;\n    let className;\n\n    step.start();\n    next = step.getCurrent();\n    xNext = this.body.util.toScreen(next);\n    while (step.hasNext() && count < MAX) {\n      count++;\n\n      isMajor = step.isMajor();\n      className = step.getClassName();\n\n      current = next;\n      x = xNext;\n\n      step.next();\n      next = step.getCurrent();\n      xNext = this.body.util.toScreen(next);\n\n      prevWidth = width;\n      width = xNext - x;\n      switch (step.scale) {\n        case 'week':         showMinorGrid = true; break;\n        default:             showMinorGrid = (width >= prevWidth * 0.4); break; // prevent displaying of the 31th of the month on a scale of 5 days\n      }\n\n      if (this.options.showMinorLabels && showMinorGrid) {\n        var label = this._repaintMinorText(x, step.getLabelMinor(current), orientation, className);\n        label.style.width = `${width}px`; // set width to prevent overflow\n      }\n\n      if (isMajor && this.options.showMajorLabels) {\n        if (x > 0) {\n          if (xFirstMajorLabel == undefined) {\n            xFirstMajorLabel = x;\n          }\n          label = this._repaintMajorText(x, step.getLabelMajor(current), orientation, className);\n        }\n        line = this._repaintMajorLine(x, width, orientation, className);\n      }\n      else { // minor line\n        if (showMinorGrid) {\n          line = this._repaintMinorLine(x, width, orientation, className);\n        }\n        else {\n          if (line) {\n            // adjust the width of the previous grid\n            line.style.width = `${parseInt(line.style.width) + width}px`;\n          }\n        }\n      }\n    }\n\n    if (count === MAX && !warnedForOverflow) {\n        console.warn(`Something is wrong with the Timeline scale. Limited drawing of grid lines to ${MAX} lines.`);\n        warnedForOverflow = true;\n    }\n\n    // create a major label on the left when needed\n    if (this.options.showMajorLabels) {\n      const leftTime = this.body.util.toTime(0); // upper bound estimation\n      const leftText = step.getLabelMajor(leftTime);\n      const widthText = leftText.length * (this.props.majorCharWidth || 10) + 10;\n\n      if (xFirstMajorLabel == undefined || widthText < xFirstMajorLabel) {\n        this._repaintMajorText(0, leftText, orientation, className);\n      }\n    }\n\n    // Cleanup leftover DOM elements from the redundant list\n    util.forEach(this.dom.redundant, arr => {\n      while (arr.length) {\n        const elem = arr.pop();\n        if (elem && elem.parentNode) {\n          elem.parentNode.removeChild(elem);\n        }\n      }\n    });\n  }\n\n  /**\n   * Create a minor label for the axis at position x\n   * @param {number} x\n   * @param {string} text\n   * @param {string} orientation   \"top\" or \"bottom\" (default)\n   * @param {string} className\n   * @return {Element} Returns the HTML element of the created label\n   * @private\n   */\n  _repaintMinorText(x, text, orientation, className) {\n    // reuse redundant label\n    let label = this.dom.redundant.minorTexts.shift();\n\n    if (!label) {\n      // create new label\n      const content = document.createTextNode('');\n      label = document.createElement('div');\n      label.appendChild(content);\n      this.dom.foreground.appendChild(label);\n    }\n    this.dom.minorTexts.push(label);\n    label.innerHTML = text;\n\n\n    let y = (orientation == 'top') ? this.props.majorLabelHeight : 0;\n    this._setXY(label, x, y);\n\n    label.className = `vis-text vis-minor ${className}`;\n    //label.title = title;  // TODO: this is a heavy operation\n\n    return label;\n  }\n\n  /**\n   * Create a Major label for the axis at position x\n   * @param {number} x\n   * @param {string} text\n   * @param {string} orientation   \"top\" or \"bottom\" (default)\n   * @param {string} className\n   * @return {Element} Returns the HTML element of the created label\n   * @private\n   */\n  _repaintMajorText(x, text, orientation, className) {\n    // reuse redundant label\n    let label = this.dom.redundant.majorTexts.shift();\n\n    if (!label) {\n      // create label\n      const content = document.createElement('div');\n      label = document.createElement('div');\n      label.appendChild(content);\n      this.dom.foreground.appendChild(label);\n    }\n\n    label.childNodes[0].innerHTML = text;\n    label.className = `vis-text vis-major ${className}`;\n    //label.title = title; // TODO: this is a heavy operation\n\n    let y = (orientation == 'top') ? 0 : this.props.minorLabelHeight;\n    this._setXY(label, x, y);\n\n    this.dom.majorTexts.push(label);\n    return label;\n  }\n\n  /**\n   * sets xy\n   * @param {string} label\n   * @param {number} x\n   * @param {number} y\n   * @private\n   */\n  _setXY(label, x, y) {\n    // If rtl is true, inverse x.\n    const directionX = this.options.rtl ? (x * -1) : x;\n    label.style.transform = `translate(${directionX}px, ${y}px)`;\n  }\n\n  /**\n   * Create a minor line for the axis at position x\n   * @param {number} left\n   * @param {number} width\n   * @param {string} orientation   \"top\" or \"bottom\" (default)\n   * @param {string} className\n   * @return {Element} Returns the created line\n   * @private\n   */\n  _repaintMinorLine(left, width, orientation, className) {\n    // reuse redundant line\n    let line = this.dom.redundant.lines.shift();\n    if (!line) {\n      // create vertical line\n      line = document.createElement('div');\n      this.dom.background.appendChild(line);\n    }\n    this.dom.lines.push(line);\n\n    const props = this.props;\n    \n    line.style.width = `${width}px`;\n    line.style.height = `${props.minorLineHeight}px`;\n\n    let y = (orientation == 'top') ? props.majorLabelHeight : this.body.domProps.top.height;\n    let x = left - props.minorLineWidth / 2;\n\n    this._setXY(line, x, y);\n    line.className = `vis-grid ${this.options.rtl ?  'vis-vertical-rtl' : 'vis-vertical'} vis-minor ${className}`;\n\n    return line;\n  }\n\n  /**\n   * Create a Major line for the axis at position x\n   * @param {number} left\n   * @param {number} width\n   * @param {string} orientation   \"top\" or \"bottom\" (default)\n   * @param {string} className\n   * @return {Element} Returns the created line\n   * @private\n   */\n  _repaintMajorLine(left, width, orientation, className) {\n    // reuse redundant line\n    let line = this.dom.redundant.lines.shift();\n    if (!line) {\n      // create vertical line\n      line = document.createElement('div');\n      this.dom.background.appendChild(line);\n    }\n    this.dom.lines.push(line);\n\n    const props = this.props;\n    \n    line.style.width = `${width}px`;\n    line.style.height = `${props.majorLineHeight}px`;\n\n    let y = (orientation == 'top') ? 0 : this.body.domProps.top.height;\n    let x = left - props.majorLineWidth / 2;\n\n    this._setXY(line, x, y);\n    line.className = `vis-grid ${this.options.rtl ?  'vis-vertical-rtl' : 'vis-vertical'} vis-major ${className}`;\n\n    return line;\n  }\n\n  /**\n   * Determine the size of text on the axis (both major and minor axis).\n   * The size is calculated only once and then cached in this.props.\n   * @private\n   */\n  _calculateCharSize() {\n    // Note: We calculate char size with every redraw. Size may change, for\n    // example when any of the timelines parents had display:none for example.\n\n    // determine the char width and height on the minor axis\n    if (!this.dom.measureCharMinor) {\n      this.dom.measureCharMinor = document.createElement('DIV');\n      this.dom.measureCharMinor.className = 'vis-text vis-minor vis-measure';\n      this.dom.measureCharMinor.style.position = 'absolute';\n\n      this.dom.measureCharMinor.appendChild(document.createTextNode('0'));\n      this.dom.foreground.appendChild(this.dom.measureCharMinor);\n    }\n    this.props.minorCharHeight = this.dom.measureCharMinor.clientHeight;\n    this.props.minorCharWidth = this.dom.measureCharMinor.clientWidth;\n\n    // determine the char width and height on the major axis\n    if (!this.dom.measureCharMajor) {\n      this.dom.measureCharMajor = document.createElement('DIV');\n      this.dom.measureCharMajor.className = 'vis-text vis-major vis-measure';\n      this.dom.measureCharMajor.style.position = 'absolute';\n\n      this.dom.measureCharMajor.appendChild(document.createTextNode('0'));\n      this.dom.foreground.appendChild(this.dom.measureCharMajor);\n    }\n    this.props.majorCharHeight = this.dom.measureCharMajor.clientHeight;\n    this.props.majorCharWidth = this.dom.measureCharMajor.clientWidth;\n  }\n}\n\n\nvar warnedForOverflow = false;\n\nexport default TimeAxis;\n","\"use strict\";\r\n/**\r\n * Created by Alex on 11/6/2014.\r\n */\r\n\r\n// https://github.com/umdjs/umd/blob/master/returnExports.js#L40-L60\r\n// if the module has no dependencies, the above pattern can be simplified to\r\n(function (root, factory) {\r\n  if (typeof define === 'function' && define.amd) {\r\n    // AMD. Register as an anonymous module.\r\n    define([], factory);\r\n  } else if (typeof exports === 'object') {\r\n    // Node. Does not work with strict CommonJS, but\r\n    // only CommonJS-like environments that support module.exports,\r\n    // like Node.\r\n    module.exports = factory();\r\n  } else {\r\n    // Browser globals (root is window)\r\n    root.keycharm = factory();\r\n  }\r\n}(this, function () {\r\n\r\n  function keycharm(options) {\r\n    var preventDefault = options && options.preventDefault || false;\r\n\r\n    var container = options && options.container || window;\r\n\r\n    var _exportFunctions = {};\r\n    var _bound = {keydown:{}, keyup:{}};\r\n    var _keys = {};\r\n    var i;\r\n\r\n    // a - z\r\n    for (i = 97; i <= 122; i++) {_keys[String.fromCharCode(i)] = {code:65 + (i - 97), shift: false};}\r\n    // A - Z\r\n    for (i = 65; i <= 90; i++) {_keys[String.fromCharCode(i)] = {code:i, shift: true};}\r\n    // 0 - 9\r\n    for (i = 0;  i <= 9;   i++) {_keys['' + i] = {code:48 + i, shift: false};}\r\n    // F1 - F12\r\n    for (i = 1;  i <= 12;   i++) {_keys['F' + i] = {code:111 + i, shift: false};}\r\n    // num0 - num9\r\n    for (i = 0;  i <= 9;   i++) {_keys['num' + i] = {code:96 + i, shift: false};}\r\n\r\n    // numpad misc\r\n    _keys['num*'] = {code:106, shift: false};\r\n    _keys['num+'] = {code:107, shift: false};\r\n    _keys['num-'] = {code:109, shift: false};\r\n    _keys['num/'] = {code:111, shift: false};\r\n    _keys['num.'] = {code:110, shift: false};\r\n    // arrows\r\n    _keys['left']  = {code:37, shift: false};\r\n    _keys['up']    = {code:38, shift: false};\r\n    _keys['right'] = {code:39, shift: false};\r\n    _keys['down']  = {code:40, shift: false};\r\n    // extra keys\r\n    _keys['space'] = {code:32, shift: false};\r\n    _keys['enter'] = {code:13, shift: false};\r\n    _keys['shift'] = {code:16, shift: undefined};\r\n    _keys['esc']   = {code:27, shift: false};\r\n    _keys['backspace'] = {code:8, shift: false};\r\n    _keys['tab']       = {code:9, shift: false};\r\n    _keys['ctrl']      = {code:17, shift: false};\r\n    _keys['alt']       = {code:18, shift: false};\r\n    _keys['delete']    = {code:46, shift: false};\r\n    _keys['pageup']    = {code:33, shift: false};\r\n    _keys['pagedown']  = {code:34, shift: false};\r\n    // symbols\r\n    _keys['=']     = {code:187, shift: false};\r\n    _keys['-']     = {code:189, shift: false};\r\n    _keys[']']     = {code:221, shift: false};\r\n    _keys['[']     = {code:219, shift: false};\r\n\r\n\r\n\r\n    var down = function(event) {handleEvent(event,'keydown');};\r\n    var up = function(event) {handleEvent(event,'keyup');};\r\n\r\n    // handle the actualy bound key with the event\r\n    var handleEvent = function(event,type) {\r\n      if (_bound[type][event.keyCode] !== undefined) {\r\n        var bound = _bound[type][event.keyCode];\r\n        for (var i = 0; i < bound.length; i++) {\r\n          if (bound[i].shift === undefined) {\r\n            bound[i].fn(event);\r\n          }\r\n          else if (bound[i].shift == true && event.shiftKey == true) {\r\n            bound[i].fn(event);\r\n          }\r\n          else if (bound[i].shift == false && event.shiftKey == false) {\r\n            bound[i].fn(event);\r\n          }\r\n        }\r\n\r\n        if (preventDefault == true) {\r\n          event.preventDefault();\r\n        }\r\n      }\r\n    };\r\n\r\n    // bind a key to a callback\r\n    _exportFunctions.bind = function(key, callback, type) {\r\n      if (type === undefined) {\r\n        type = 'keydown';\r\n      }\r\n      if (_keys[key] === undefined) {\r\n        throw new Error(\"unsupported key: \" + key);\r\n      }\r\n      if (_bound[type][_keys[key].code] === undefined) {\r\n        _bound[type][_keys[key].code] = [];\r\n      }\r\n      _bound[type][_keys[key].code].push({fn:callback, shift:_keys[key].shift});\r\n    };\r\n\r\n\r\n    // bind all keys to a call back (demo purposes)\r\n    _exportFunctions.bindAll = function(callback, type) {\r\n      if (type === undefined) {\r\n        type = 'keydown';\r\n      }\r\n      for (var key in _keys) {\r\n        if (_keys.hasOwnProperty(key)) {\r\n          _exportFunctions.bind(key,callback,type);\r\n        }\r\n      }\r\n    };\r\n\r\n    // get the key label from an event\r\n    _exportFunctions.getKey = function(event) {\r\n      for (var key in _keys) {\r\n        if (_keys.hasOwnProperty(key)) {\r\n          if (event.shiftKey == true && _keys[key].shift == true && event.keyCode == _keys[key].code) {\r\n            return key;\r\n          }\r\n          else if (event.shiftKey == false && _keys[key].shift == false && event.keyCode == _keys[key].code) {\r\n            return key;\r\n          }\r\n          else if (event.keyCode == _keys[key].code && key == 'shift') {\r\n            return key;\r\n          }\r\n        }\r\n      }\r\n      return \"unknown key, currently not supported\";\r\n    };\r\n\r\n    // unbind either a specific callback from a key or all of them (by leaving callback undefined)\r\n    _exportFunctions.unbind = function(key, callback, type) {\r\n      if (type === undefined) {\r\n        type = 'keydown';\r\n      }\r\n      if (_keys[key] === undefined) {\r\n        throw new Error(\"unsupported key: \" + key);\r\n      }\r\n      if (callback !== undefined) {\r\n        var newBindings = [];\r\n        var bound = _bound[type][_keys[key].code];\r\n        if (bound !== undefined) {\r\n          for (var i = 0; i < bound.length; i++) {\r\n            if (!(bound[i].fn == callback && bound[i].shift == _keys[key].shift)) {\r\n              newBindings.push(_bound[type][_keys[key].code][i]);\r\n            }\r\n          }\r\n        }\r\n        _bound[type][_keys[key].code] = newBindings;\r\n      }\r\n      else {\r\n        _bound[type][_keys[key].code] = [];\r\n      }\r\n    };\r\n\r\n    // reset all bound variables.\r\n    _exportFunctions.reset = function() {\r\n      _bound = {keydown:{}, keyup:{}};\r\n    };\r\n\r\n    // unbind all listeners and reset all variables.\r\n    _exportFunctions.destroy = function() {\r\n      _bound = {keydown:{}, keyup:{}};\r\n      container.removeEventListener('keydown', down, true);\r\n      container.removeEventListener('keyup', up, true);\r\n    };\r\n\r\n    // create listeners.\r\n    container.addEventListener('keydown',down,true);\r\n    container.addEventListener('keyup',up,true);\r\n\r\n    // return the public functions.\r\n    return _exportFunctions;\r\n  }\r\n\r\n  return keycharm;\r\n}));\r\n\r\n\r\n","import keycharm from 'keycharm';\nimport Emitter from 'component-emitter';\nimport Hammer from '../module/hammer';\nimport util from '../util';\n\nimport './activator.css';\n\n/**\n * Turn an element into an clickToUse element.\n * When not active, the element has a transparent overlay. When the overlay is\n * clicked, the mode is changed to active.\n * When active, the element is displayed with a blue border around it, and\n * the interactive contents of the element can be used. When clicked outside\n * the element, the elements mode is changed to inactive.\n * @param {Element} container\n * @constructor Activator\n */\nfunction Activator(container) {\n  this.active = false;\n\n  this.dom = {\n    container: container\n  };\n\n  this.dom.overlay = document.createElement('div');\n  this.dom.overlay.className = 'vis-overlay';\n\n  this.dom.container.appendChild(this.dom.overlay);\n\n  this.hammer = Hammer(this.dom.overlay);\n  this.hammer.on('tap', this._onTapOverlay.bind(this));\n\n  // block all touch events (except tap)\n  var me = this;\n  var events = [\n    'tap', 'doubletap', 'press',\n    'pinch',\n    'pan', 'panstart', 'panmove', 'panend'\n  ];\n  events.forEach(function (event) {\n    me.hammer.on(event, function (event) {\n      event.stopPropagation();\n    });\n  });\n\n  // attach a click event to the window, in order to deactivate when clicking outside the timeline\n  if (document && document.body) {\n    this.onClick = function (event) {\n      if (!_hasParent(event.target, container)) {\n        me.deactivate();\n      }\n    };\n    document.body.addEventListener('click', this.onClick);\n  }\n\n  if (this.keycharm !== undefined) {\n    this.keycharm.destroy();\n  }\n  this.keycharm = keycharm();\n\n  // keycharm listener only bounded when active)\n  this.escListener = this.deactivate.bind(this);\n}\n\n// turn into an event emitter\nEmitter(Activator.prototype);\n\n// The currently active activator\nActivator.current = null;\n\n/**\n * Destroy the activator. Cleans up all created DOM and event listeners\n */\nActivator.prototype.destroy = function () {\n  this.deactivate();\n\n  // remove dom\n  this.dom.overlay.parentNode.removeChild(this.dom.overlay);\n\n  // remove global event listener\n  if (this.onClick) {\n    document.body.removeEventListener('click', this.onClick);\n  }\n  // remove keycharm\n  if (this.keycharm !== undefined) {\n    this.keycharm.destroy();\n  }\n  this.keycharm = null;\n  // cleanup hammer instances\n  this.hammer.destroy();\n  this.hammer = null;\n  // FIXME: cleaning up hammer instances doesn't work (Timeline not removed from memory)\n};\n\n/**\n * Activate the element\n * Overlay is hidden, element is decorated with a blue shadow border\n */\nActivator.prototype.activate = function () {\n  // we allow only one active activator at a time\n  if (Activator.current) {\n    Activator.current.deactivate();\n  }\n  Activator.current = this;\n\n  this.active = true;\n  this.dom.overlay.style.display = 'none';\n  util.addClassName(this.dom.container, 'vis-active');\n\n  this.emit('change');\n  this.emit('activate');\n\n  // ugly hack: bind ESC after emitting the events, as the Network rebinds all\n  // keyboard events on a 'change' event\n  this.keycharm.bind('esc', this.escListener);\n};\n\n/**\n * Deactivate the element\n * Overlay is displayed on top of the element\n */\nActivator.prototype.deactivate = function () {\n  if (Activator.current === this) {\n    Activator.current = null;\n  }\n\n  this.active = false;\n  this.dom.overlay.style.display = '';\n  util.removeClassName(this.dom.container, 'vis-active');\n  this.keycharm.unbind('esc', this.escListener);\n\n  this.emit('change');\n  this.emit('deactivate');\n};\n\n/**\n * Handle a tap event: activate the container\n * @param {Event}  event   The event\n * @private\n */\nActivator.prototype._onTapOverlay = function (event) {\n  // activate the container\n  this.activate();\n  event.stopPropagation();\n};\n\n/**\n * Test whether the element has the requested parent element somewhere in\n * its chain of parent nodes.\n * @param {HTMLElement} element\n * @param {HTMLElement} parent\n * @returns {boolean} Returns true when the parent is found somewhere in the\n *                    chain of parent nodes.\n * @private\n */\nfunction _hasParent(element, parent) {\n  while (element) {\n    if (element === parent) {\n      return true\n    }\n    element = element.parentNode;\n  }\n  return false;\n}\n\nexport default Activator;\n","/*\n * IMPORTANT: Locales for Moment has to be imported in the legacy and standalone\n * entry points. For the peer build it's users responsibility to do so.\n */\n\n// English\nconst en = {\n  current: 'current',\n  time: 'time',\n  deleteSelected: 'Delete selected',\n};\nconst en_EN = en;\nconst en_US = en;\n\n// Italiano\nconst it = {\n  current: 'attuale',\n  time: 'tempo',\n  deleteSelected: 'Cancella la selezione',\n};\nconst it_IT = it;\nconst it_CH = it;\n\n// Dutch\nconst nl = {\n  current: 'huidige',\n  time: 'tijd',\n  deleteSelected: 'Selectie verwijderen'\n};\nconst nl_NL = nl;\nconst nl_BE = nl;\n\n// German\nconst de = {\n  current: 'Aktuelle',\n  time: 'Zeit',\n  deleteSelected: 'L\\u00f6sche Auswahl',\n};\nconst de_DE = de;\n\n// French\nconst fr = {\n  current: 'actuel',\n  time: 'heure',\n  deleteSelected: 'Effacer la selection',\n};\nconst fr_FR = fr;\nconst fr_CA = fr;\nconst fr_BE = fr;\n\n// Espanol\nconst es = {\n  current: 'corriente',\n  time: 'hora',\n  deleteSelected: 'Eliminar selecci\\u00f3n',\n};\nconst es_ES = es;\n\n// Ukrainian\nconst uk = {\n  current: 'поточний',\n  time: 'час',\n  deleteSelected: 'Видалити обране',\n};\nconst uk_UA = uk;\n\n// Russian\nconst ru = {\n  current: 'текущее',\n  time: 'время',\n  deleteSelected: 'Удалить выбранное',\n};\nconst ru_RU = ru;\n\n// Polish\nconst pl = {\n  current: 'aktualny',\n  time: 'czas',\n  deleteSelected: 'Usuń wybrane',\n};\nconst pl_PL = pl;\n\n// Japanese\nconst ja = {\n  current: '現在',\n  time: '時刻',\n  deleteSelected: '選択されたものを削除',\n};\nconst ja_JP = ja;\n\nconst locales = {\n  en,\n  en_EN,\n  en_US,\n  it,\n  it_IT,\n  it_CH,\n  nl,\n  nl_NL,\n  nl_BE,\n  de,\n  de_DE,\n  fr,\n  fr_FR,\n  fr_CA,\n  fr_BE,\n  es,\n  es_ES,\n  uk,\n  uk_UA,\n  ru,\n  ru_RU,\n  pl,\n  pl_PL,\n  ja,\n  ja_JP,\n}\n\nexport default locales\n","import Hammer from '../../module/hammer';\nimport util from '../../util';\nimport Component from './Component';\nimport moment from '../../module/moment';\nimport locales from '../locales';\n\nimport './css/customtime.css';\n\n/** A custom time bar */\nclass CustomTime extends Component {\n /**\n * @param {{range: Range, dom: Object}} body\n * @param {Object} [options]        Available parameters:\n *                                  {number | string} id\n *                                  {string} locales\n *                                  {string} locale\n * @constructor CustomTime\n * @extends Component\n */\n  constructor(body, options) {\n    super()\n    this.body = body;\n\n    // default options\n    this.defaultOptions = {\n      moment,\n      locales,\n      locale: 'en',\n      id: undefined,\n      title: undefined\n    };\n    this.options = util.extend({}, this.defaultOptions);\n    this.setOptions(options);\n    this.options.locales = util.extend({}, locales, this.options.locales);\n    const defaultLocales = this.defaultOptions.locales[this.defaultOptions.locale];\n    Object.keys(this.options.locales).forEach(locale => {\n      this.options.locales[locale] = util.extend(\n        {},\n        defaultLocales,\n        this.options.locales[locale]\n      );\n    });\n\n    if (options && options.time != null) {\n      this.customTime = options.time;\n    } else {\n      this.customTime = new Date();\n    }\n\n    this.eventParams = {}; // stores state parameters while dragging the bar\n\n    // create the DOM\n    this._create();\n  }\n\n  /**\n   * Set options for the component. Options will be merged in current options.\n   * @param {Object} options  Available parameters:\n   *                                  {number | string} id\n   *                                  {string} locales\n   *                                  {string} locale\n   */\n  setOptions(options) {\n    if (options) {\n      // copy all options that we know\n      util.selectiveExtend(['moment', 'locale', 'locales', 'id', 'title', 'rtl', 'snap'], this.options, options);\n    }\n  }\n\n  /**\n   * Create the DOM for the custom time\n   * @private\n   */\n  _create() {\n    const bar = document.createElement('div');\n    bar['custom-time'] = this;\n    bar.className = `vis-custom-time ${this.options.id || ''}`;\n    bar.style.position = 'absolute';\n    bar.style.top = '0px';\n    bar.style.height = '100%';\n    this.bar = bar;\n\n    const drag = document.createElement('div');\n    drag.style.position = 'relative';\n    drag.style.top = '0px';\n    if(this.options.rtl) {\n      drag.style.right = '-10px';\n    } else  {\n       drag.style.left = '-10px';\n    }\n    drag.style.height = '100%';\n    drag.style.width = '20px';\n\n    /**\n     *\n     * @param {WheelEvent} e\n     */\n    function onMouseWheel (e) {\n      this.body.range._onMouseWheel(e);\n    }\n\n    if (drag.addEventListener) {\n      // IE9, Chrome, Safari, Opera\n      drag.addEventListener(\"mousewheel\", onMouseWheel.bind(this), false);\n      // Firefox\n      drag.addEventListener(\"DOMMouseScroll\", onMouseWheel.bind(this), false);\n    } else {\n      // IE 6/7/8\n      drag.attachEvent(\"onmousewheel\", onMouseWheel.bind(this));\n    }\n\n    bar.appendChild(drag);\n    // attach event listeners\n    this.hammer = new Hammer(drag);\n    this.hammer.on('panstart', this._onDragStart.bind(this));\n    this.hammer.on('panmove',  this._onDrag.bind(this));\n    this.hammer.on('panend',   this._onDragEnd.bind(this));\n    this.hammer.get('pan').set({threshold:5, direction: Hammer.DIRECTION_ALL});\n  }\n\n  /**\n   * Destroy the CustomTime bar\n   */\n  destroy() {\n    this.hide();\n\n    this.hammer.destroy();\n    this.hammer = null;\n\n    this.body = null;\n  }\n\n  /**\n   * Repaint the component\n   * @return {boolean} Returns true if the component is resized\n   */\n  redraw() {\n    const parent = this.body.dom.backgroundVertical;\n    if (this.bar.parentNode != parent) {\n      // attach to the dom\n      if (this.bar.parentNode) {\n        this.bar.parentNode.removeChild(this.bar);\n      }\n      parent.appendChild(this.bar);\n    }\n\n    const x = this.body.util.toScreen(this.customTime);\n\n    let locale = this.options.locales[this.options.locale];\n    if (!locale) {\n      if (!this.warned) {\n        console.warn(`WARNING: options.locales['${this.options.locale}'] not found. See https://visjs.github.io/vis-timeline/docs/timeline/#Localization`);\n        this.warned = true;\n      }\n      locale = this.options.locales['en']; // fall back on english when not available\n    }\n\n    let title = this.options.title;\n    // To hide the title completely use empty string ''.\n    if (title === undefined) {\n      title = `${locale.time}: ${this.options.moment(this.customTime).format('dddd, MMMM Do YYYY, H:mm:ss')}`;\n      title = title.charAt(0).toUpperCase() + title.substring(1);\n    } else if (typeof title === \"function\") {\n      title = title.call(this, this.customTime);\n    }\n\n    this.options.rtl ? this.bar.style.right = `${x}px` : this.bar.style.left = `${x}px`;\n    this.bar.title = title;\n\n    return false;\n  }\n\n  /**\n   * Remove the CustomTime from the DOM\n   */\n  hide() {\n    // remove the line from the DOM\n    if (this.bar.parentNode) {\n      this.bar.parentNode.removeChild(this.bar);\n    }\n  }\n\n  /**\n   * Set custom time.\n   * @param {Date | number | string} time\n   */\n  setCustomTime(time) {\n    this.customTime = util.convert(time, 'Date');\n    this.redraw();\n  }\n\n  /**\n   * Retrieve the current custom time.\n   * @return {Date} customTime\n   */\n  getCustomTime() {\n    return new Date(this.customTime.valueOf());\n  }\n\n  /**\n   * Set custom marker.\n   * @param {string} [title] Title of the custom marker\n   * @param {boolean} [editable] Make the custom marker editable.\n   */\n  setCustomMarker(title, editable) {\n    const marker = document.createElement('div');\n    marker.className = `vis-custom-time-marker`;\n    marker.innerHTML = title;\n    marker.style.position = 'absolute';\n\n    if (editable) {\n      marker.setAttribute('contenteditable', 'true');\n      marker.addEventListener('pointerdown', function () {\n        marker.focus();\n      });\n      marker.addEventListener('input', this._onMarkerChange.bind(this));\n      // The editable div element has no change event, so here emulates the change event.\n      marker.title = title;\n      marker.addEventListener('blur', function (event) {\n        if (this.title != event.target.innerHTML) {\n          this._onMarkerChanged(event);\n          this.title = event.target.innerHTML;\n        }\n      }.bind(this));\n    }\n\n    this.bar.appendChild(marker);\n  }\n\n  /**\n    * Set custom title.\n    * @param {Date | number | string} title\n    */\n  setCustomTitle(title) {\n    this.options.title = title;\n  }\n\n  /**\n   * Start moving horizontally\n   * @param {Event} event\n   * @private\n   */\n  _onDragStart(event) {\n    this.eventParams.dragging = true;\n    this.eventParams.customTime = this.customTime;\n\n    event.stopPropagation();\n  }\n\n  /**\n   * Perform moving operating.\n   * @param {Event} event\n   * @private\n   */\n  _onDrag(event) {\n    if (!this.eventParams.dragging) return;\n\n    let deltaX = this.options.rtl ? (-1) * event.deltaX : event.deltaX;\n\n    const x = this.body.util.toScreen(this.eventParams.customTime) + deltaX;\n    const time = this.body.util.toTime(x);\n\n    const scale = this.body.util.getScale();\n    const step = this.body.util.getStep();\n    const snap = this.options.snap;\n\n    const snappedTime = snap ? snap(time, scale, step) : time;\n\n    this.setCustomTime(snappedTime);\n\n    // fire a timechange event\n    this.body.emitter.emit('timechange', {\n      id: this.options.id,\n      time: new Date(this.customTime.valueOf()),\n      event\n    });\n\n    event.stopPropagation();\n  }\n\n  /**\n   * Stop moving operating.\n   * @param {Event} event\n   * @private\n   */\n  _onDragEnd(event) {\n    if (!this.eventParams.dragging) return;\n\n    // fire a timechanged event\n    this.body.emitter.emit('timechanged', {\n      id: this.options.id,\n      time: new Date(this.customTime.valueOf()),\n      event\n    });\n\n    event.stopPropagation();\n  }\n\n  /**\n   * Perform input operating.\n   * @param {Event} event\n   * @private\n   */\n  _onMarkerChange(event) {\n    this.body.emitter.emit('markerchange', {\n      id: this.options.id,\n      title: event.target.innerHTML,\n      event\n    });\n\n    event.stopPropagation();\n  }\n\n  /**\n   * Perform change operating.\n   * @param {Event} event\n   * @private\n   */\n  _onMarkerChanged(event) {\n    this.body.emitter.emit('markerchanged', {\n      id: this.options.id,\n      title: event.target.innerHTML,\n      event\n    });\n\n    event.stopPropagation();\n  }\n\n  /**\n   * Find a custom time from an event target:\n   * searches for the attribute 'custom-time' in the event target's element tree\n   * @param {Event} event\n   * @return {CustomTime | null} customTime\n   */\n  static customTimeFromTarget(event) {\n    let target = event.target;\n    while (target) {\n      if (target.hasOwnProperty('custom-time')) {\n        return target['custom-time'];\n      }\n      target = target.parentNode;\n    }\n\n    return null;\n  }\n}\n\nexport default CustomTime;\n","import Emitter from 'component-emitter';\nimport Hammer from '../module/hammer';\nimport * as hammerUtil from '../hammerUtil';\nimport util from '../util';\nimport TimeAxis from './component/TimeAxis';\nimport Activator from '../shared/Activator';\nimport * as DateUtil  from './DateUtil';\nimport CustomTime from './component/CustomTime';\n\nimport './component/css/animation.css';\nimport './component/css/currenttime.css';\nimport './component/css/panel.css';\nimport './component/css/pathStyles.css';\nimport './component/css/timeline.css';\nimport '../shared/bootstrap.css';\n\n/**\n * Create a timeline visualization\n * @constructor Core\n */\nclass Core {\n  /**\n   * Create the main DOM for the Core: a root panel containing left, right,\n   * top, bottom, content, and background panel.\n   * @param {Element} container  The container element where the Core will\n   *                             be attached.\n   * @protected\n   */\n  _create(container) {\n    this.dom = {};\n\n    this.dom.container = container;\n    this.dom.container.style.position = 'relative';\n\n    this.dom.root                 = document.createElement('div');\n    this.dom.background           = document.createElement('div');\n    this.dom.backgroundVertical   = document.createElement('div');\n    this.dom.backgroundHorizontal = document.createElement('div');\n    this.dom.centerContainer      = document.createElement('div');\n    this.dom.leftContainer        = document.createElement('div');\n    this.dom.rightContainer       = document.createElement('div');\n    this.dom.center               = document.createElement('div');\n    this.dom.left                 = document.createElement('div');\n    this.dom.right                = document.createElement('div');\n    this.dom.top                  = document.createElement('div');\n    this.dom.bottom               = document.createElement('div');\n    this.dom.shadowTop            = document.createElement('div');\n    this.dom.shadowBottom         = document.createElement('div');\n    this.dom.shadowTopLeft        = document.createElement('div');\n    this.dom.shadowBottomLeft     = document.createElement('div');\n    this.dom.shadowTopRight       = document.createElement('div');\n    this.dom.shadowBottomRight    = document.createElement('div');\n    this.dom.rollingModeBtn       = document.createElement('div');\n    this.dom.loadingScreen        = document.createElement('div');\n\n    this.dom.root.className                 = 'vis-timeline';\n    this.dom.background.className           = 'vis-panel vis-background';\n    this.dom.backgroundVertical.className   = 'vis-panel vis-background vis-vertical';\n    this.dom.backgroundHorizontal.className = 'vis-panel vis-background vis-horizontal';\n    this.dom.centerContainer.className      = 'vis-panel vis-center';\n    this.dom.leftContainer.className        = 'vis-panel vis-left';\n    this.dom.rightContainer.className       = 'vis-panel vis-right';\n    this.dom.top.className                  = 'vis-panel vis-top';\n    this.dom.bottom.className               = 'vis-panel vis-bottom';\n    this.dom.left.className                 = 'vis-content';\n    this.dom.center.className               = 'vis-content';\n    this.dom.right.className                = 'vis-content';\n    this.dom.shadowTop.className            = 'vis-shadow vis-top';\n    this.dom.shadowBottom.className         = 'vis-shadow vis-bottom';\n    this.dom.shadowTopLeft.className        = 'vis-shadow vis-top';\n    this.dom.shadowBottomLeft.className     = 'vis-shadow vis-bottom';\n    this.dom.shadowTopRight.className       = 'vis-shadow vis-top';\n    this.dom.shadowBottomRight.className    = 'vis-shadow vis-bottom';\n    this.dom.rollingModeBtn.className       = 'vis-rolling-mode-btn';\n    this.dom.loadingScreen.className        = 'vis-loading-screen';\n\n    this.dom.root.appendChild(this.dom.background);\n    this.dom.root.appendChild(this.dom.backgroundVertical);\n    this.dom.root.appendChild(this.dom.backgroundHorizontal);\n    this.dom.root.appendChild(this.dom.centerContainer);\n    this.dom.root.appendChild(this.dom.leftContainer);\n    this.dom.root.appendChild(this.dom.rightContainer);\n    this.dom.root.appendChild(this.dom.top);\n    this.dom.root.appendChild(this.dom.bottom);\n    this.dom.root.appendChild(this.dom.rollingModeBtn);\n\n    this.dom.centerContainer.appendChild(this.dom.center);\n    this.dom.leftContainer.appendChild(this.dom.left);\n    this.dom.rightContainer.appendChild(this.dom.right);\n    this.dom.centerContainer.appendChild(this.dom.shadowTop);\n    this.dom.centerContainer.appendChild(this.dom.shadowBottom);\n    this.dom.leftContainer.appendChild(this.dom.shadowTopLeft);\n    this.dom.leftContainer.appendChild(this.dom.shadowBottomLeft);\n    this.dom.rightContainer.appendChild(this.dom.shadowTopRight);\n    this.dom.rightContainer.appendChild(this.dom.shadowBottomRight);\n    \n    // size properties of each of the panels\n    this.props = {\n      root: {},\n      background: {},\n      centerContainer: {},\n      leftContainer: {},\n      rightContainer: {},\n      center: {},\n      left: {},\n      right: {},\n      top: {},\n      bottom: {},\n      border: {},\n      scrollTop: 0,\n      scrollTopMin: 0\n    };\n\n    this.on('rangechange', () => {\n      if (this.initialDrawDone === true) {\n        this._redraw();\n      }\n    });\n    this.on('rangechanged', () => {\n      if (!this.initialRangeChangeDone) {\n        this.initialRangeChangeDone = true;\n      }\n    });\n    this.on('touch',       this._onTouch.bind(this));\n    this.on('panmove',         this._onDrag.bind(this));\n\n    const me = this;\n    this._origRedraw = this._redraw.bind(this);\n    this._redraw = util.throttle(this._origRedraw);\n\n    this.on('_change', properties => {\n      if (me.itemSet && me.itemSet.initialItemSetDrawn && properties && properties.queue == true) {\n        me._redraw()\n      } else {\n        me._origRedraw();\n      }\n    });\n\n    // create event listeners for all interesting events, these events will be\n    // emitted via emitter\n    this.hammer = new Hammer(this.dom.root);\n    const pinchRecognizer = this.hammer.get('pinch').set({enable: true});\n    pinchRecognizer && hammerUtil.disablePreventDefaultVertically(pinchRecognizer);\n    this.hammer.get('pan').set({threshold:5, direction: Hammer.DIRECTION_ALL});\n    this.listeners = {};\n\n    const events = [\n      'tap', 'doubletap', 'press',\n      'pinch',\n      'pan', 'panstart', 'panmove', 'panend'\n        // TODO: cleanup\n      //'touch', 'pinch',\n      //'tap', 'doubletap', 'hold',\n      //'dragstart', 'drag', 'dragend',\n      //'mousewheel', 'DOMMouseScroll' // DOMMouseScroll is needed for Firefox\n    ];\n    events.forEach(type => {\n      const listener = event => {\n        if (me.isActive()) {\n          me.emit(type, event);\n        }\n      };\n      me.hammer.on(type, listener);\n      me.listeners[type] = listener;\n    });\n\n    // emulate a touch event (emitted before the start of a pan, pinch, tap, or press)\n    hammerUtil.onTouch(this.hammer, event => {\n      me.emit('touch', event);\n    });\n\n    // emulate a release event (emitted after a pan, pinch, tap, or press)\n    hammerUtil.onRelease(this.hammer, event => {\n      me.emit('release', event);\n    });\n\n    /**\n     *\n     * @param {WheelEvent} event\n     */\n    function onMouseWheel(event) {\n\n      // Reasonable default wheel deltas\n      const LINE_HEIGHT = 40;\n      const PAGE_HEIGHT = 800;\n\n      if (this.isActive()) {\n        this.emit('mousewheel', event);\n      }\n\n      // deltaX and deltaY normalization from jquery.mousewheel.js\n      let deltaX = 0;\n      let deltaY = 0;\n\n      // Old school scrollwheel delta\n      if ( 'detail'      in event ) { deltaY = event.detail * -1;      }\n      if ( 'wheelDelta'  in event ) { deltaY = event.wheelDelta;       }\n      if ( 'wheelDeltaY' in event ) { deltaY = event.wheelDeltaY;      }\n      if ( 'wheelDeltaX' in event ) { deltaX = event.wheelDeltaX * -1; }\n\n      // Firefox < 17 horizontal scrolling related to DOMMouseScroll event\n      if ( 'axis' in event && event.axis === event.HORIZONTAL_AXIS ) {\n        deltaX = deltaY * -1;\n        deltaY = 0;\n      }\n\n      // New school wheel delta (wheel event)\n      if ( 'deltaY' in event ) {\n        deltaY = event.deltaY * -1;\n      }\n      if ( 'deltaX' in event ) {\n        deltaX = event.deltaX;\n      }\n\n      // Normalize deltas\n      if (event.deltaMode) {\n          if (event.deltaMode === 1) {   // delta in LINE units\n            deltaX *= LINE_HEIGHT;\n            deltaY *= LINE_HEIGHT;\n          } else {                       // delta in PAGE units\n            deltaX *= LINE_HEIGHT;\n            deltaY *= PAGE_HEIGHT;\n          }\n        }\n        // Prevent scrolling when zooming (no zoom key, or pressing zoom key)\n      if (this.options.preferZoom) {\n        if (!this.options.zoomKey || event[this.options.zoomKey]) return;\n      } else {\n        if (this.options.zoomKey && event[this.options.zoomKey]) return\n      }\n      // Don't preventDefault if you can't scroll\n      if (!this.options.verticalScroll && !this.options.horizontalScroll) return;\n\n      if (this.options.verticalScroll && Math.abs(deltaY) >= Math.abs(deltaX)) {\n        const current = this.props.scrollTop;\n        const adjusted = current + deltaY;\n\n        if (this.isActive()) {\n          const newScrollTop = this._setScrollTop(adjusted);\n\n          if (newScrollTop !== current) {\n            this._redraw();\n            this.emit('scroll', event);\n\n            // Prevent default actions caused by mouse wheel\n            // (else the page and timeline both scroll)\n            event.preventDefault();\n          }\n        }\n      } else if (this.options.horizontalScroll) {\n        const delta = Math.abs(deltaX) >= Math.abs(deltaY) ? deltaX : deltaY;\n\n        // calculate a single scroll jump relative to the range scale\n        const diff = (delta / 120) * (this.range.end - this.range.start) / 20;\n        // calculate new start and end\n        const newStart = this.range.start + diff;\n        const newEnd = this.range.end + diff;\n\n        const options = {\n          animation: false,\n          byUser: true,\n          event\n        };\n        this.range.setRange(newStart, newEnd, options);\n\n        event.preventDefault();\n      }\n    }\n\n    // Add modern wheel event listener\n    const wheelType = \"onwheel\" in document.createElement(\"div\") ? \"wheel\" : // Modern browsers support \"wheel\"\n    document.onmousewheel !== undefined ? \"mousewheel\" : // Webkit and IE support at least \"mousewheel\"\n    \n    // DOMMouseScroll - Older Firefox versions use \"DOMMouseScroll\"\n    // onmousewheel - All the use \"onmousewheel\"\n    this.dom.centerContainer.addEventListener ? \"DOMMouseScroll\" :  \"onmousewheel\"\n    this.dom.top.addEventListener ? \"DOMMouseScroll\" : \"onmousewheel\"\n    this.dom.bottom.addEventListener ? \"DOMMouseScroll\" : \"onmousewheel\"\n    this.dom.centerContainer.addEventListener(wheelType, onMouseWheel.bind(this), false);\n    this.dom.top.addEventListener(wheelType, onMouseWheel.bind(this), false);\n    this.dom.bottom.addEventListener(wheelType, onMouseWheel.bind(this), false);\n\n\n    /**\n     *\n     * @param {scroll} event\n     */\n    function onMouseScrollSide(event) {\n      if (!me.options.verticalScroll) return;\n      \n      if (me._isProgramaticallyScrolled) {\n        me._isProgramaticallyScrolled = false;\n        return;\n      }\n      \n      event.preventDefault();\n      if (me.isActive()) {\n        const adjusted = -event.target.scrollTop;\n        me._setScrollTop(adjusted);\n        me._redraw();\n        me.emit('scrollSide', event);\n      }\n    }\n\n    this.dom.left.parentNode.addEventListener('scroll', onMouseScrollSide.bind(this));\n    this.dom.right.parentNode.addEventListener('scroll', onMouseScrollSide.bind(this));\n\n    let itemAddedToTimeline = false;\n\n    /**\n     *\n     * @param {dragover} event\n     * @returns {boolean}\n     */\n    function handleDragOver(event) {\n      if (event.preventDefault) {\n        me.emit('dragover', me.getEventProperties(event));\n        event.preventDefault(); // Necessary. Allows us to drop.\n      }\n\n      // make sure your target is a timeline element\n      if (!(event.target.className.indexOf(\"timeline\") > -1)) return;\n\n      // make sure only one item is added every time you're over the timeline\n      if (itemAddedToTimeline) return;\n\n      event.dataTransfer.dropEffect = 'move';  \n      itemAddedToTimeline = true;\n      return false;\n    }\n\n    /**\n     *\n     * @param {drop} event\n     * @returns {boolean}\n     */\n    function handleDrop(event) {\n      // prevent redirect to blank page - Firefox\n      if(event.preventDefault) { event.preventDefault(); }\n      if(event.stopPropagation) { event.stopPropagation(); }\n      // return when dropping non-timeline items\n      try {\n        var itemData = JSON.parse(event.dataTransfer.getData(\"text\"))\n        if (!itemData || !itemData.content) return\n      } catch (err) {\n        return false;\n      }\n\n      itemAddedToTimeline = false;\n      event.center = {\n        x: event.clientX,\n        y: event.clientY\n      };\n\n      if (itemData.target !== 'item') {\n        me.itemSet._onAddItem(event);\n      } else {\n        me.itemSet._onDropObjectOnItem(event);\n      }\n      me.emit('drop', me.getEventProperties(event))\n      return false;\n    }\n\n    this.dom.center.addEventListener('dragover', handleDragOver.bind(this), false);\n    this.dom.center.addEventListener('drop', handleDrop.bind(this), false);\n\n    this.customTimes = [];\n\n    // store state information needed for touch events\n    this.touch = {};\n\n    this.redrawCount = 0;\n    this.initialDrawDone = false;\n    this.initialRangeChangeDone = false;\n\n    // attach the root panel to the provided container\n    if (!container) throw new Error('No container provided');\n    container.appendChild(this.dom.root);\n    container.appendChild(this.dom.loadingScreen);\n  }\n\n  /**\n   * Set options. Options will be passed to all components loaded in the Timeline.\n   * @param {Object} [options]\n   *                           {String} orientation\n   *                              Vertical orientation for the Timeline,\n   *                              can be 'bottom' (default) or 'top'.\n   *                           {string | number} width\n   *                              Width for the timeline, a number in pixels or\n   *                              a css string like '1000px' or '75%'. '100%' by default.\n   *                           {string | number} height\n   *                              Fixed height for the Timeline, a number in pixels or\n   *                              a css string like '400px' or '75%'. If undefined,\n   *                              The Timeline will automatically size such that\n   *                              its contents fit.\n   *                           {string | number} minHeight\n   *                              Minimum height for the Timeline, a number in pixels or\n   *                              a css string like '400px' or '75%'.\n   *                           {string | number} maxHeight\n   *                              Maximum height for the Timeline, a number in pixels or\n   *                              a css string like '400px' or '75%'.\n   *                           {number | Date | string} start\n   *                              Start date for the visible window\n   *                           {number | Date | string} end\n   *                              End date for the visible window\n   */\n  setOptions(options) {\n    if (options) {\n      // copy the known options\n      const fields = [\n        'width', 'height', 'minHeight', 'maxHeight', 'autoResize',\n        'start', 'end', 'clickToUse', 'dataAttributes', 'hiddenDates',\n        'locale', 'locales', 'moment', 'preferZoom', 'rtl', 'zoomKey',\n        'horizontalScroll', 'verticalScroll', 'longSelectPressTime'\n      ];\n      util.selectiveExtend(fields, this.options, options);\n      this.dom.rollingModeBtn.style.visibility = 'hidden';\n\n      if (this.options.rtl) {\n        this.dom.container.style.direction = \"rtl\";\n        this.dom.backgroundVertical.className = 'vis-panel vis-background vis-vertical-rtl';    \n      }\n\n      if (this.options.verticalScroll) {\n        if (this.options.rtl) {\n          this.dom.rightContainer.className = 'vis-panel vis-right vis-vertical-scroll';\n        } else {\n          this.dom.leftContainer.className = 'vis-panel vis-left vis-vertical-scroll';\n        }\n      }\n\n      if (typeof this.options.orientation !== 'object') {\n        this.options.orientation = {item:undefined,axis:undefined};\n      }\n      if ('orientation' in options) {\n        if (typeof options.orientation === 'string') {\n          this.options.orientation = {\n            item: options.orientation,\n            axis: options.orientation\n          };\n        }\n        else if (typeof options.orientation === 'object') {\n          if ('item' in options.orientation) {\n            this.options.orientation.item = options.orientation.item;\n          }\n          if ('axis' in options.orientation) {\n            this.options.orientation.axis = options.orientation.axis;\n          }\n        }\n      }\n\n      if (this.options.orientation.axis === 'both') {\n        if (!this.timeAxis2) {\n          const timeAxis2 = this.timeAxis2 = new TimeAxis(this.body);\n          timeAxis2.setOptions = options => {\n            const _options = options ? util.extend({}, options) : {};\n            _options.orientation = 'top'; // override the orientation option, always top\n            TimeAxis.prototype.setOptions.call(timeAxis2, _options);\n          };\n          this.components.push(timeAxis2);\n        }\n      }\n      else {\n        if (this.timeAxis2) {\n          const index = this.components.indexOf(this.timeAxis2);\n          if (index !== -1) {\n            this.components.splice(index, 1);\n          }\n          this.timeAxis2.destroy();\n          this.timeAxis2 = null;\n        }\n      }\n\n      // if the graph2d's drawPoints is a function delegate the callback to the onRender property\n      if (typeof options.drawPoints == 'function') {\n        options.drawPoints = {\n            onRender: options.drawPoints\n        };\n      }\n\n      if ('hiddenDates' in this.options) {\n        DateUtil.convertHiddenOptions(this.options.moment, this.body, this.options.hiddenDates);\n      }\n\n      if ('clickToUse' in options) {\n        if (options.clickToUse) {\n          if (!this.activator) {\n            this.activator = new Activator(this.dom.root);\n          }\n        }\n        else {\n          if (this.activator) {\n            this.activator.destroy();\n            delete this.activator;\n          }\n        }\n      }\n\n      // enable/disable autoResize\n      this._initAutoResize();\n    }\n\n    // propagate options to all components\n    this.components.forEach(component => component.setOptions(options));\n\n    // enable/disable configure\n    if ('configure' in options) {\n      if (!this.configurator) {\n        this.configurator = this._createConfigurator();\n      }\n\n      this.configurator.setOptions(options.configure);\n\n      // collect the settings of all components, and pass them to the configuration system\n      const appliedOptions = util.deepExtend({}, this.options);\n      this.components.forEach(component => {\n        util.deepExtend(appliedOptions, component.options);\n      });\n      this.configurator.setModuleOptions({global: appliedOptions});\n    }\n\n    this._redraw();\n  }\n\n  /**\n   * Returns true when the Timeline is active.\n   * @returns {boolean}\n   */\n  isActive() {\n    return !this.activator || this.activator.active;\n  }\n\n  /**\n   * Destroy the Core, clean up all DOM elements and event listeners.\n   */\n  destroy() {\n    // unbind datasets\n    this.setItems(null);\n    this.setGroups(null);\n\n    // remove all event listeners\n    this.off();\n\n    // stop checking for changed size\n    this._stopAutoResize();\n\n    // remove from DOM\n    if (this.dom.root.parentNode) {\n      this.dom.root.parentNode.removeChild(this.dom.root);\n    }\n    this.dom = null;\n\n    // remove Activator\n    if (this.activator) {\n      this.activator.destroy();\n      delete this.activator;\n    }\n\n    // cleanup hammer touch events\n    for (const event in this.listeners) {\n      if (this.listeners.hasOwnProperty(event)) {\n        delete this.listeners[event];\n      }\n    }\n    this.listeners = null;\n    this.hammer && this.hammer.destroy();\n    this.hammer = null;\n\n    // give all components the opportunity to cleanup\n    this.components.forEach(component => component.destroy());\n\n    this.body = null;\n  }\n\n  /**\n   * Set a custom time bar\n   * @param {Date} time\n   * @param {number} [id=undefined] Optional id of the custom time bar to be adjusted.\n   */\n  setCustomTime(time, id) {\n    const customTimes = this.customTimes.filter(component => id === component.options.id);\n\n    if (customTimes.length === 0) {\n      throw new Error(`No custom time bar found with id ${JSON.stringify(id)}`)\n    }\n\n    if (customTimes.length > 0) {\n      customTimes[0].setCustomTime(time);\n    }\n  }\n\n  /**\n   * Retrieve the current custom time.\n   * @param {number} [id=undefined]    Id of the custom time bar.\n   * @return {Date | undefined} customTime\n   */\n  getCustomTime(id) {\n    const customTimes = this.customTimes.filter(component => component.options.id === id);\n\n    if (customTimes.length === 0) {\n      throw new Error(`No custom time bar found with id ${JSON.stringify(id)}`)\n    }\n    return customTimes[0].getCustomTime();\n  }\n\n  /**\n   * Set a custom marker for the custom time bar.\n   * @param {string} [title] Title of the custom marker.\n   * @param {number} [id=undefined] Id of the custom marker.\n   * @param {boolean} [editable=false] Make the custom marker editable.\n   */\n  setCustomTimeMarker(title, id, editable) {\n    const customTimes = this.customTimes.filter(component => component.options.id === id);\n\n    if (customTimes.length === 0) {\n      throw new Error(`No custom time bar found with id ${JSON.stringify(id)}`)\n    }\n    if (customTimes.length > 0) {\n      customTimes[0].setCustomMarker(title, editable);\n    }\n  }\n\n  /**\n   * Set a custom title for the custom time bar.\n   * @param {string} [title] Custom title\n   * @param {number} [id=undefined]    Id of the custom time bar.\n   * @returns {*}\n   */\n  setCustomTimeTitle(title, id) {\n    const customTimes = this.customTimes.filter(component => component.options.id === id);\n\n    if (customTimes.length === 0) {\n      throw new Error(`No custom time bar found with id ${JSON.stringify(id)}`)\n    }\n    if (customTimes.length > 0) {\n      return customTimes[0].setCustomTitle(title);\n    }\n  }\n\n  /**\n   * Retrieve meta information from an event.\n   * Should be overridden by classes extending Core\n   * @param {Event} event\n   * @return {Object} An object with related information.\n   */\n  getEventProperties(event) {\n    return { event };\n  }\n\n  /**\n   * Add custom vertical bar\n   * @param {Date | string | number} [time]  A Date, unix timestamp, or\n   *                                         ISO date string. Time point where\n   *                                         the new bar should be placed.\n   *                                         If not provided, `new Date()` will\n   *                                         be used.\n   * @param {number | string} [id=undefined] Id of the new bar. Optional\n   * @return {number | string}               Returns the id of the new bar\n   */\n  addCustomTime(time, id) {\n    const timestamp = time !== undefined\n        ? util.convert(time, 'Date')\n        : new Date();\n\n    const exists = this.customTimes.some(customTime => customTime.options.id === id);\n    if (exists) {\n      throw new Error(`A custom time with id ${JSON.stringify(id)} already exists`);\n    }\n\n    const customTime = new CustomTime(this.body, util.extend({}, this.options, {\n      time : timestamp,\n      id,\n      snap: this.itemSet.options.snap\n    }));\n\n    this.customTimes.push(customTime);\n    this.components.push(customTime);\n    this._redraw();\n\n    return id;\n  }\n\n  /**\n   * Remove previously added custom bar\n   * @param {int} id ID of the custom bar to be removed\n   * [at]returns {boolean} True if the bar exists and is removed, false otherwise\n   */\n  removeCustomTime(id) {\n    const customTimes = this.customTimes.filter(bar => bar.options.id === id);\n\n    if (customTimes.length === 0) {\n      throw new Error(`No custom time bar found with id ${JSON.stringify(id)}`)\n    }\n\n    customTimes.forEach(customTime => {\n      this.customTimes.splice(this.customTimes.indexOf(customTime), 1);\n      this.components.splice(this.components.indexOf(customTime), 1);\n      customTime.destroy();\n    })\n  }\n\n  /**\n   * Get the id's of the currently visible items.\n   * @returns {Array} The ids of the visible items\n   */\n  getVisibleItems() {\n    return this.itemSet && this.itemSet.getVisibleItems() || [];\n  }\n\n  /**\n   * Get the id's of the currently visible groups.\n   * @returns {Array} The ids of the visible groups\n   */\n  getVisibleGroups() {\n    return this.itemSet && this.itemSet.getVisibleGroups() || [];\n  }\n\n  /**\n   * Set Core window such that it fits all items\n   * @param {Object} [options]  Available options:\n   *                                `animation: boolean | {duration: number, easingFunction: string}`\n   *                                    If true (default), the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   * @param {function} [callback] a callback funtion to be executed at the end of this function\n   */\n  fit(options, callback) {\n    const range = this.getDataRange();\n\n    // skip range set if there is no min and max date\n    if (range.min === null && range.max === null) {\n      return;\n    }\n\n    // apply a margin of 1% left and right of the data\n    const interval = range.max - range.min;\n    const min = new Date(range.min.valueOf() - interval * 0.01);\n    const max = new Date(range.max.valueOf() + interval * 0.01);\n    const animation = (options && options.animation !== undefined) ? options.animation : true;\n    this.range.setRange(min, max, { animation }, callback);\n  }\n\n  /**\n   * Calculate the data range of the items start and end dates\n   * [at]returns {{min: [Date], max: [Date]}}\n   * @protected\n   */\n  getDataRange() {\n    // must be implemented by Timeline and Graph2d\n    throw new Error('Cannot invoke abstract method getDataRange');\n  }\n\n  /**\n   * Set the visible window. Both parameters are optional, you can change only\n   * start or only end. Syntax:\n   *\n   *     TimeLine.setWindow(start, end)\n   *     TimeLine.setWindow(start, end, options)\n   *     TimeLine.setWindow(range)\n   *\n   * Where start and end can be a Date, number, or string, and range is an\n   * object with properties start and end.\n   *\n   * @param {Date | number | string | Object} [start] Start date of visible window\n   * @param {Date | number | string} [end]            End date of visible window\n   * @param {Object} [options]  Available options:\n   *                                `animation: boolean | {duration: number, easingFunction: string}`\n   *                                    If true (default), the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   * @param {function} [callback] a callback funtion to be executed at the end of this function\n   */\n  setWindow(start, end, options, callback) {\n    if (typeof arguments[2] == \"function\") {\n      callback = arguments[2];\n      options = {};\n    }\n    let animation;\n    let range;\n    if (arguments.length == 1) {\n      range = arguments[0];\n      animation = (range.animation !== undefined) ? range.animation : true;\n      this.range.setRange(range.start, range.end, { animation });\n    }\n    else if (arguments.length == 2 && typeof arguments[1] == \"function\") {\n      range = arguments[0];\n      callback = arguments[1];\n      animation = (range.animation !== undefined) ? range.animation : true;\n      this.range.setRange(range.start, range.end, { animation }, callback);\n    }\n    else {\n      animation = (options && options.animation !== undefined) ? options.animation : true;\n      this.range.setRange(start, end, { animation }, callback);\n    }\n  }\n\n  /**\n   * Move the window such that given time is centered on screen.\n   * @param {Date | number | string} time\n   * @param {Object} [options]  Available options:\n   *                                `animation: boolean | {duration: number, easingFunction: string}`\n   *                                    If true (default), the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   * @param {function} [callback] a callback funtion to be executed at the end of this function\n   */\n  moveTo(time, options, callback) {\n    if (typeof arguments[1] == \"function\") {\n      callback = arguments[1];\n      options = {};\n    }\n    const interval = this.range.end - this.range.start;\n    const t = util.convert(time, 'Date').valueOf();\n\n    const start = t - interval / 2;\n    const end = t + interval / 2;\n    const animation = (options && options.animation !== undefined) ? options.animation : true;\n\n    this.range.setRange(start, end, { animation }, callback);\n  }\n\n  /**\n   * Get the visible window\n   * @return {{start: Date, end: Date}}   Visible range\n   */\n  getWindow() {\n    const range = this.range.getRange();\n    return {\n      start: new Date(range.start),\n      end: new Date(range.end)\n    };\n  }\n\n  /**\n   * Zoom in the window such that given time is centered on screen.\n   * @param {number} percentage - must be between [0..1]\n   * @param {Object} [options]  Available options:\n   *                                `animation: boolean | {duration: number, easingFunction: string}`\n   *                                    If true (default), the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   * @param {function} [callback] a callback funtion to be executed at the end of this function\n   */\n  zoomIn(percentage, options, callback) {\n    if (!percentage || percentage < 0 || percentage > 1) return;\n    if (typeof arguments[1] == \"function\") {\n      callback = arguments[1];\n      options = {};\n    }\n    const range = this.getWindow();\n    const start = range.start.valueOf();\n    const end = range.end.valueOf();\n    const interval = end - start;\n    const newInterval = interval / (1 + percentage);\n    const distance = (interval - newInterval) / 2;\n    const newStart = start + distance;\n    const newEnd = end - distance;\n\n    this.setWindow(newStart, newEnd, options, callback);\n  }\n\n  /**\n   * Zoom out the window such that given time is centered on screen.\n   * @param {number} percentage - must be between [0..1]\n   * @param {Object} [options]  Available options:\n   *                                `animation: boolean | {duration: number, easingFunction: string}`\n   *                                    If true (default), the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   * @param {function} [callback] a callback funtion to be executed at the end of this function\n   */\n  zoomOut(percentage, options, callback) {\n    if (!percentage || percentage < 0 || percentage > 1) return\n    if (typeof arguments[1] == \"function\") {\n      callback = arguments[1];\n      options = {};\n    }\n    const range = this.getWindow();\n    const start = range.start.valueOf();\n    const end = range.end.valueOf();\n    const interval = end - start;\n    const newStart = start - interval * percentage / 2;\n    const newEnd = end + interval * percentage / 2;\n\n    this.setWindow(newStart, newEnd, options, callback);\n  }\n\n  /**\n   * Force a redraw. Can be overridden by implementations of Core\n   *\n   * Note: this function will be overridden on construction with a trottled version\n   */\n  redraw() {\n    this._redraw();\n  }\n\n  /**\n   * Redraw for internal use. Redraws all components. See also the public\n   * method redraw.\n   * @protected\n   */\n  _redraw() {\n    this.redrawCount++;\n    const dom = this.dom;\n    \n    if (!dom || !dom.container || dom.root.offsetWidth == 0) return; // when destroyed, or invisible\n    \n    let resized = false;\n    const options = this.options;\n    const props = this.props;\n\n    DateUtil.updateHiddenDates(this.options.moment, this.body, this.options.hiddenDates);\n\n    // update class names\n    if (options.orientation == 'top') {\n      util.addClassName(dom.root, 'vis-top');\n      util.removeClassName(dom.root, 'vis-bottom');\n    }\n    else {\n      util.removeClassName(dom.root, 'vis-top');\n      util.addClassName(dom.root, 'vis-bottom');\n    }\n\n    if (options.rtl) {\n      util.addClassName(dom.root, 'vis-rtl');\n      util.removeClassName(dom.root, 'vis-ltr');\n    } else {\n      util.addClassName(dom.root, 'vis-ltr');\n      util.removeClassName(dom.root, 'vis-rtl');\n    }\n\n    // update root width and height options\n    dom.root.style.maxHeight = util.option.asSize(options.maxHeight, '');\n    dom.root.style.minHeight = util.option.asSize(options.minHeight, '');\n    dom.root.style.width = util.option.asSize(options.width, '');\n    const rootOffsetWidth = dom.root.offsetWidth;\n\n    // calculate border widths\n    props.border.left = 1\n    props.border.right = 1\n    props.border.top = 1\n    props.border.bottom = 1\n\n    // calculate the heights. If any of the side panels is empty, we set the height to\n    // minus the border width, such that the border will be invisible\n    props.center.height = dom.center.offsetHeight;\n    props.left.height   = dom.left.offsetHeight;\n    props.right.height  = dom.right.offsetHeight;\n    props.top.height    = dom.top.clientHeight    || -props.border.top;\n    props.bottom.height = dom.bottom.clientHeight || -props.border.bottom;\n\n    // TODO: compensate borders when any of the panels is empty.\n\n    // apply auto height\n    // TODO: only calculate autoHeight when needed (else we cause an extra reflow/repaint of the DOM)\n    const contentHeight = Math.max(props.left.height, props.center.height, props.right.height);\n    const autoHeight = props.top.height + contentHeight + props.bottom.height + props.border.top + props.border.bottom;\n    dom.root.style.height = util.option.asSize(options.height, `${autoHeight}px`);\n\n    // calculate heights of the content panels\n    props.root.height = dom.root.offsetHeight;\n    props.background.height = props.root.height;\n    const containerHeight = props.root.height - props.top.height - props.bottom.height;\n    props.centerContainer.height  = containerHeight;\n    props.leftContainer.height    = containerHeight;\n    props.rightContainer.height   = props.leftContainer.height;\n\n    // calculate the widths of the panels\n    props.root.width = rootOffsetWidth;\n    props.background.width = props.root.width;\n\n    if (!this.initialDrawDone) {\n      props.scrollbarWidth = util.getScrollBarWidth();\n    }\n\n    const leftContainerClientWidth = dom.leftContainer.clientWidth;\n    const rightContainerClientWidth = dom.rightContainer.clientWidth;\n\n    if (options.verticalScroll) {\n      if (options.rtl) {\n        props.left.width = leftContainerClientWidth || -props.border.left;\n        props.right.width = rightContainerClientWidth + props.scrollbarWidth || -props.border.right;\n      } else {\n        props.left.width = leftContainerClientWidth + props.scrollbarWidth || -props.border.left;\n        props.right.width = rightContainerClientWidth || -props.border.right;\n      }\n    } else {\n      props.left.width = leftContainerClientWidth  || -props.border.left;\n      props.right.width = rightContainerClientWidth || -props.border.right;\n    }\n\n    this._setDOM();\n\n    // update the scrollTop, feasible range for the offset can be changed\n    // when the height of the Core or of the contents of the center changed\n    let offset = this._updateScrollTop();\n\n    // reposition the scrollable contents\n    if (options.orientation.item != 'top') {\n      offset += Math.max(props.centerContainer.height - props.center.height -\n        props.border.top - props.border.bottom, 0);\n    }\n    dom.center.style.transform  = `translateY(${offset}px)`;\n\n    // show shadows when vertical scrolling is available\n    const visibilityTop = props.scrollTop == 0 ? 'hidden' : '';\n    const visibilityBottom = props.scrollTop == props.scrollTopMin ? 'hidden' : '';\n    dom.shadowTop.style.visibility          = visibilityTop;\n    dom.shadowBottom.style.visibility       = visibilityBottom;\n    dom.shadowTopLeft.style.visibility      = visibilityTop;\n    dom.shadowBottomLeft.style.visibility   = visibilityBottom;\n    dom.shadowTopRight.style.visibility     = visibilityTop;\n    dom.shadowBottomRight.style.visibility  = visibilityBottom;\n\n    if (options.verticalScroll) {\n      dom.rightContainer.className = 'vis-panel vis-right vis-vertical-scroll';\n      dom.leftContainer.className = 'vis-panel vis-left vis-vertical-scroll';\n\n      dom.shadowTopRight.style.visibility     = \"hidden\";\n      dom.shadowBottomRight.style.visibility  = \"hidden\";\n      dom.shadowTopLeft.style.visibility      = \"hidden\";\n      dom.shadowBottomLeft.style.visibility   = \"hidden\";\n\n      dom.left.style.top    = '0px';\n      dom.right.style.top   = '0px';\n    }\n\n    if (!options.verticalScroll ||  props.center.height < props.centerContainer.height) {\n      dom.left.style.top    = `${offset}px`;\n      dom.right.style.top   = `${offset}px`;\n      dom.rightContainer.className = dom.rightContainer.className.replace(new RegExp('(?:^|\\\\s)'+ 'vis-vertical-scroll' + '(?:\\\\s|$)'), ' ');\n      dom.leftContainer.className = dom.leftContainer.className.replace(new RegExp('(?:^|\\\\s)'+ 'vis-vertical-scroll' + '(?:\\\\s|$)'), ' ');\n      props.left.width = leftContainerClientWidth  || -props.border.left;\n      props.right.width = rightContainerClientWidth || -props.border.right;\n      this._setDOM();\n    }\n\n    // enable/disable vertical panning\n    const contentsOverflow = props.center.height > props.centerContainer.height;\n    this.hammer.get('pan').set({\n      direction: contentsOverflow ? Hammer.DIRECTION_ALL : Hammer.DIRECTION_HORIZONTAL\n    });\n\n    // set the long press time\n    this.hammer.get('press').set({\n      time: this.options.longSelectPressTime\n    });\n\n    // redraw all components\n    this.components.forEach(component => {\n      resized = component.redraw() || resized;\n    });\n    const MAX_REDRAW = 5;\n    if (resized) {\n      if (this.redrawCount < MAX_REDRAW) {\n        this.body.emitter.emit('_change');\n        return;\n      }\n      else {\n        console.log('WARNING: infinite loop in redraw?');\n      }\n    } else {\n      this.redrawCount = 0;\n    }\n\n    //Emit public 'changed' event for UI updates, see issue #1592\n    this.body.emitter.emit(\"changed\");\n  }\n\n  /**\n   * sets the basic DOM components needed for the timeline\\graph2d\n   */\n  _setDOM() {\n    const props = this.props;\n    const dom = this.dom;\n\n    props.leftContainer.width = props.left.width;\n    props.rightContainer.width = props.right.width;\n    const centerWidth = props.root.width - props.left.width - props.right.width;\n    props.center.width          = centerWidth;\n    props.centerContainer.width = centerWidth;\n    props.top.width             = centerWidth;\n    props.bottom.width          = centerWidth;\n\n    // resize the panels\n    dom.background.style.height           = `${props.background.height}px`;\n    dom.backgroundVertical.style.height   = `${props.background.height}px`;\n    dom.backgroundHorizontal.style.height = `${props.centerContainer.height}px`;\n    dom.centerContainer.style.height      = `${props.centerContainer.height}px`;\n    dom.leftContainer.style.height        = `${props.leftContainer.height}px`;\n    dom.rightContainer.style.height       = `${props.rightContainer.height}px`;\n\n    dom.background.style.width            = `${props.background.width}px`;\n    dom.backgroundVertical.style.width    = `${props.centerContainer.width}px`;\n    dom.backgroundHorizontal.style.width  = `${props.background.width}px`;\n    dom.centerContainer.style.width       = `${props.center.width}px`;\n    dom.top.style.width                   = `${props.top.width}px`;\n    dom.bottom.style.width                = `${props.bottom.width}px`;\n\n    // reposition the panels\n    dom.background.style.left           = '0';\n    dom.background.style.top            = '0';\n    dom.backgroundVertical.style.left   = `${props.left.width + props.border.left}px`;\n    dom.backgroundVertical.style.top    = '0';\n    dom.backgroundHorizontal.style.left = '0';\n    dom.backgroundHorizontal.style.top  = `${props.top.height}px`;\n    dom.centerContainer.style.left      = `${props.left.width}px`;\n    dom.centerContainer.style.top       = `${props.top.height}px`;\n    dom.leftContainer.style.left        = '0';\n    dom.leftContainer.style.top         = `${props.top.height}px`;\n    dom.rightContainer.style.left       = `${props.left.width + props.center.width}px`;\n    dom.rightContainer.style.top        = `${props.top.height}px`;\n    dom.top.style.left                  = `${props.left.width}px`;\n    dom.top.style.top                   = '0';\n    dom.bottom.style.left               = `${props.left.width}px`;\n    dom.bottom.style.top                = `${props.top.height + props.centerContainer.height}px`;\n    dom.center.style.left               = '0';\n    dom.left.style.left                 = '0';\n    dom.right.style.left                = '0';\n  }\n\n  /**\n   * Set a current time. This can be used for example to ensure that a client's\n   * time is synchronized with a shared server time.\n   * Only applicable when option `showCurrentTime` is true.\n   * @param {Date | string | number} time     A Date, unix timestamp, or\n   *                                          ISO date string.\n   */\n  setCurrentTime(time) {\n    if (!this.currentTime) {\n      throw new Error('Option showCurrentTime must be true');\n    }\n\n    this.currentTime.setCurrentTime(time);\n  }\n\n  /**\n   * Get the current time.\n   * Only applicable when option `showCurrentTime` is true.\n   * @return {Date} Returns the current time.\n   */\n  getCurrentTime() {\n    if (!this.currentTime) {\n      throw new Error('Option showCurrentTime must be true');\n    }\n\n    return this.currentTime.getCurrentTime();\n  }\n\n  /**\n   * Convert a position on screen (pixels) to a datetime\n   * @param {int}     x    Position on the screen in pixels\n   * @return {Date}   time The datetime the corresponds with given position x\n   * @protected\n   * TODO: move this function to Range\n   */\n  _toTime(x) {\n    return DateUtil.toTime(this, x, this.props.center.width);\n  }\n\n  /**\n   * Convert a position on the global screen (pixels) to a datetime\n   * @param {int}     x    Position on the screen in pixels\n   * @return {Date}   time The datetime the corresponds with given position x\n   * @protected\n   * TODO: move this function to Range\n   */\n  _toGlobalTime(x) {\n    return DateUtil.toTime(this, x, this.props.root.width);\n    //var conversion = this.range.conversion(this.props.root.width);\n    //return new Date(x / conversion.scale + conversion.offset);\n  }\n\n  /**\n   * Convert a datetime (Date object) into a position on the screen\n   * @param {Date}   time A date\n   * @return {int}   x    The position on the screen in pixels which corresponds\n   *                      with the given date.\n   * @protected\n   * TODO: move this function to Range\n   */\n  _toScreen(time) {\n    return DateUtil.toScreen(this, time, this.props.center.width);\n  }\n\n  /**\n   * Convert a datetime (Date object) into a position on the root\n   * This is used to get the pixel density estimate for the screen, not the center panel\n   * @param {Date}   time A date\n   * @return {int}   x    The position on root in pixels which corresponds\n   *                      with the given date.\n   * @protected\n   * TODO: move this function to Range\n   */\n  _toGlobalScreen(time) {\n    return DateUtil.toScreen(this, time, this.props.root.width);\n    //var conversion = this.range.conversion(this.props.root.width);\n    //return (time.valueOf() - conversion.offset) * conversion.scale;\n  }\n\n  /**\n   * Initialize watching when option autoResize is true\n   * @private\n   */\n  _initAutoResize() {\n    if (this.options.autoResize == true) {\n      this._startAutoResize();\n    }\n    else {\n      this._stopAutoResize();\n    }\n  }\n\n  /**\n   * Watch for changes in the size of the container. On resize, the Panel will\n   * automatically redraw itself.\n   * @private\n   */\n  _startAutoResize() {\n    const me = this;\n\n    this._stopAutoResize();\n\n    this._onResize = () => {\n      if (me.options.autoResize != true) {\n        // stop watching when the option autoResize is changed to false\n        me._stopAutoResize();\n        return;\n      }\n\n      if (me.dom.root) {\n        const rootOffsetHeight = me.dom.root.offsetHeight;\n        const rootOffsetWidth = me.dom.root.offsetWidth;\n        // check whether the frame is resized\n        // Note: we compare offsetWidth here, not clientWidth. For some reason,\n        // IE does not restore the clientWidth from 0 to the actual width after\n        // changing the timeline's container display style from none to visible\n        if ((rootOffsetWidth != me.props.lastWidth) ||\n          (rootOffsetHeight != me.props.lastHeight)) {\n          me.props.lastWidth = rootOffsetWidth;\n          me.props.lastHeight = rootOffsetHeight;\n          me.props.scrollbarWidth = util.getScrollBarWidth();\n\n          me.body.emitter.emit('_change');\n        }\n      }\n    };\n\n    // add event listener to window resize\n    util.addEventListener(window, 'resize', this._onResize);\n\n    //Prevent initial unnecessary redraw\n    if (me.dom.root) {\n        me.props.lastWidth = me.dom.root.offsetWidth;\n        me.props.lastHeight = me.dom.root.offsetHeight;\n    }\n\n    this.watchTimer = setInterval(this._onResize, 1000);\n  }\n\n  /**\n   * Stop watching for a resize of the frame.\n   * @private\n   */\n  _stopAutoResize() {\n    if (this.watchTimer) {\n      clearInterval(this.watchTimer);\n      this.watchTimer = undefined;\n    }\n\n    // remove event listener on window.resize\n    if (this._onResize) {\n      util.removeEventListener(window, 'resize', this._onResize);\n      this._onResize = null;\n    }\n  }\n\n  /**\n   * Start moving the timeline vertically\n   * @param {Event} event\n   * @private\n   */\n  _onTouch(event) {  // eslint-disable-line no-unused-vars\n    this.touch.allowDragging = true;\n    this.touch.initialScrollTop = this.props.scrollTop;\n  }\n\n  /**\n   * Start moving the timeline vertically\n   * @param {Event} event\n   * @private\n   */\n  _onPinch(event) {  // eslint-disable-line no-unused-vars\n    this.touch.allowDragging = false;\n  }\n\n  /**\n   * Move the timeline vertically\n   * @param {Event} event\n   * @private\n   */\n  _onDrag(event) {\n    if (!event) return\n    // refuse to drag when we where pinching to prevent the timeline make a jump\n    // when releasing the fingers in opposite order from the touch screen\n    if (!this.touch.allowDragging) return;\n\n    const delta = event.deltaY;\n\n    const oldScrollTop = this._getScrollTop();\n    const newScrollTop = this._setScrollTop(this.touch.initialScrollTop + delta);\n\n    if (this.options.verticalScroll) {\n      this.dom.left.parentNode.scrollTop = -this.props.scrollTop;\n      this.dom.right.parentNode.scrollTop = -this.props.scrollTop;\n    }\n\n    if (newScrollTop != oldScrollTop) {\n      this.emit(\"verticalDrag\");\n    }\n  }\n\n  /**\n   * Apply a scrollTop\n   * @param {number} scrollTop\n   * @returns {number} scrollTop  Returns the applied scrollTop\n   * @private\n   */\n  _setScrollTop(scrollTop) {\n    this.props.scrollTop = scrollTop;\n    this._updateScrollTop();\n    return this.props.scrollTop;\n  }\n\n  /**\n   * Update the current scrollTop when the height of  the containers has been changed\n   * @returns {number} scrollTop  Returns the applied scrollTop\n   * @private\n   */\n  _updateScrollTop() {\n    // recalculate the scrollTopMin\n    const scrollTopMin = Math.min(this.props.centerContainer.height - this.props.center.height, 0); // is negative or zero\n    if (scrollTopMin != this.props.scrollTopMin) {\n      // in case of bottom orientation, change the scrollTop such that the contents\n      // do not move relative to the time axis at the bottom\n      if (this.options.orientation.item != 'top') {\n        this.props.scrollTop += (scrollTopMin - this.props.scrollTopMin);\n      }\n      this.props.scrollTopMin = scrollTopMin;\n    }\n\n    // limit the scrollTop to the feasible scroll range\n    if (this.props.scrollTop > 0) this.props.scrollTop = 0;\n    if (this.props.scrollTop < scrollTopMin) this.props.scrollTop = scrollTopMin;\n\n    if (this.options.verticalScroll) {\n      this.dom.left.parentNode.scrollTop = -this.props.scrollTop;\n      this.dom.right.parentNode.scrollTop = -this.props.scrollTop;\n    }\n    this._isProgramaticallyScrolled = true;\n    return this.props.scrollTop;\n  }\n\n  /**\n   * Get the current scrollTop\n   * @returns {number} scrollTop\n   * @private\n   */\n  _getScrollTop() {\n    return this.props.scrollTop;\n  }\n\n  /**\n   * Load a configurator\n   * [at]returns {Object}\n   * @private\n   */\n  _createConfigurator() {\n    throw new Error('Cannot invoke abstract method _createConfigurator');\n  }\n}\n\n// turn Core into an event emitter\nEmitter(Core.prototype);\n\nexport default Core;\n","import util from '../../util';\nimport Component from './Component';\nimport moment from '../../module/moment';\nimport locales from '../locales';\n\n/**\n * A current time bar\n */\nclass CurrentTime extends Component {\n/**\n * @param {{range: Range, dom: Object, domProps: Object}} body\n * @param {Object} [options]        Available parameters:\n *                                  {Boolean} [showCurrentTime]\n *                                  {String}  [alignCurrentTime]\n * @constructor CurrentTime\n * @extends Component\n */\n  constructor(body, options) {\n    super()\n    this.body = body;\n\n    // default options\n    this.defaultOptions = {\n      rtl: false,\n      showCurrentTime: true,\n      alignCurrentTime: undefined,\n\n      moment,\n      locales,\n      locale: 'en'\n    };\n    this.options = util.extend({}, this.defaultOptions);\n    this.setOptions(options);\n    this.options.locales = util.extend({}, locales, this.options.locales);\n    const defaultLocales = this.defaultOptions.locales[this.defaultOptions.locale];\n    Object.keys(this.options.locales).forEach(locale => {\n      this.options.locales[locale] = util.extend(\n        {},\n        defaultLocales,\n        this.options.locales[locale]\n      );\n    });\n    this.offset = 0;\n\n    this._create();\n\n  }\n\n  /**\n   * Create the HTML DOM for the current time bar\n   * @private\n   */\n  _create() {\n    const bar = document.createElement('div');\n    bar.className = 'vis-current-time';\n    bar.style.position = 'absolute';\n    bar.style.top = '0px';\n    bar.style.height = '100%';\n\n    this.bar = bar;\n  }\n\n  /**\n   * Destroy the CurrentTime bar\n   */\n  destroy() {\n    this.options.showCurrentTime = false;\n    this.redraw(); // will remove the bar from the DOM and stop refreshing\n\n    this.body = null;\n  }\n\n  /**\n   * Set options for the component. Options will be merged in current options.\n   * @param {Object} options  Available parameters:\n   *                          {boolean} [showCurrentTime]\n   *                          {String}  [alignCurrentTime]\n   */\n  setOptions(options) {\n    if (options) {\n      // copy all options that we know\n      util.selectiveExtend(['rtl', 'showCurrentTime', 'alignCurrentTime', 'moment', 'locale', 'locales'], this.options, options);\n    }\n  }\n\n  /**\n   * Repaint the component\n   * @return {boolean} Returns true if the component is resized\n   */\n  redraw() {\n    if (this.options.showCurrentTime) {\n      const parent = this.body.dom.backgroundVertical;\n      if (this.bar.parentNode != parent) {\n        // attach to the dom\n        if (this.bar.parentNode) {\n          this.bar.parentNode.removeChild(this.bar);\n        }\n        parent.appendChild(this.bar);\n\n        this.start();\n      }\n\n      let now = this.options.moment(Date.now() + this.offset);\n\n      if (this.options.alignCurrentTime) {\n        now = now.startOf(this.options.alignCurrentTime);\n      }\n\n      const x = this.body.util.toScreen(now);\n\n      let locale = this.options.locales[this.options.locale];\n      if (!locale) {\n        if (!this.warned) {\n          console.warn(`WARNING: options.locales['${this.options.locale}'] not found. See https://visjs.github.io/vis-timeline/docs/timeline/#Localization`);\n          this.warned = true;\n        }\n        locale = this.options.locales['en']; // fall back on english when not available\n      }\n      let title = `${locale.current} ${locale.time}: ${now.format('dddd, MMMM Do YYYY, H:mm:ss')}`;\n      title = title.charAt(0).toUpperCase() + title.substring(1);\n\n      if (this.options.rtl) {\n        this.bar.style.transform = `translateX(${x * -1}px)`;\n      } else {\n        this.bar.style.transform = `translateX(${x}px)`;\n      }\n      this.bar.title = title;\n    }\n    else {\n      // remove the line from the DOM\n      if (this.bar.parentNode) {\n        this.bar.parentNode.removeChild(this.bar);\n      }\n      this.stop();\n    }\n\n    return false;\n  }\n\n  /**\n   * Start auto refreshing the current time bar\n   */\n  start() {\n    const me = this;\n\n      /**\n       *  Updates the current time.\n       */\n      function update () {\n      me.stop();\n\n      // determine interval to refresh\n      const scale = me.body.range.conversion(me.body.domProps.center.width).scale;\n      let interval = 1 / scale / 10;\n      if (interval < 30)   interval = 30;\n      if (interval > 1000) interval = 1000;\n\n      me.redraw();\n      me.body.emitter.emit('currentTimeTick');\n\n      // start a renderTimer to adjust for the new time\n      me.currentTimeTimer = setTimeout(update, interval);\n    }\n\n    update();\n  }\n\n  /**\n   * Stop auto refreshing the current time bar\n   */\n  stop() {\n    if (this.currentTimeTimer !== undefined) {\n      clearTimeout(this.currentTimeTimer);\n      delete this.currentTimeTimer;\n    }\n  }\n\n  /**\n   * Set a current time. This can be used for example to ensure that a client's\n   * time is synchronized with a shared server time.\n   * @param {Date | string | number} time     A Date, unix timestamp, or\n   *                                          ISO date string.\n   */\n  setCurrentTime(time) {\n    const t = util.convert(time, 'Date').valueOf();\n    const now = Date.now();\n    this.offset = t - now;\n    this.redraw();\n  }\n\n  /**\n   * Get the current time.\n   * @return {Date} Returns the current time.\n   */\n  getCurrentTime() {\n    return new Date(Date.now() + this.offset);\n  }\n}\n\nexport default CurrentTime;\n","'use strict';\nvar $ = require('../internals/export');\nvar $find = require('../internals/array-iteration').find;\nvar addToUnscopables = require('../internals/add-to-unscopables');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar FIND = 'find';\nvar SKIPS_HOLES = true;\n\nvar USES_TO_LENGTH = arrayMethodUsesToLength(FIND);\n\n// Shouldn't skip holes\nif (FIND in []) Array(1)[FIND](function () { SKIPS_HOLES = false; });\n\n// `Array.prototype.find` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.find\n$({ target: 'Array', proto: true, forced: SKIPS_HOLES || !USES_TO_LENGTH }, {\n  find: function find(callbackfn /* , that = undefined */) {\n    return $find(this, callbackfn, arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n\n// https://tc39.github.io/ecma262/#sec-array.prototype-@@unscopables\naddToUnscopables(FIND);\n","require('../../../modules/es.array.find');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').find;\n","var find = require('../array/virtual/find');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.find;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.find) ? find : own;\n};\n","// Unique ID creation requires a high quality random # generator. In the browser we therefore\n// require the crypto API and do not support built-in fallback to lower quality random number\n// generators (like Math.random()).\n// getRandomValues needs to be invoked in a context where \"this\" is a Crypto implementation. Also,\n// find the complete implementation of crypto (msCrypto) on IE11.\nvar getRandomValues = typeof crypto != 'undefined' && crypto.getRandomValues && crypto.getRandomValues.bind(crypto) || typeof msCrypto != 'undefined' && typeof msCrypto.getRandomValues == 'function' && msCrypto.getRandomValues.bind(msCrypto);\nvar rnds8 = new Uint8Array(16); // eslint-disable-line no-undef\n\nexport default function rng() {\n  if (!getRandomValues) {\n    throw new Error('crypto.getRandomValues() not supported. See https://github.com/uuidjs/uuid#getrandomvalues-not-supported');\n  }\n\n  return getRandomValues(rnds8);\n}","/**\n * Convert array of 16 byte values to UUID string format of the form:\n * XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX\n */\nvar byteToHex = [];\n\nfor (var i = 0; i < 256; ++i) {\n  byteToHex[i] = (i + 0x100).toString(16).substr(1);\n}\n\nfunction bytesToUuid(buf, offset) {\n  var i = offset || 0;\n  var bth = byteToHex; // join used to fix memory issue caused by concatenation: https://bugs.chromium.org/p/v8/issues/detail?id=3175#c4\n\n  return [bth[buf[i++]], bth[buf[i++]], bth[buf[i++]], bth[buf[i++]], '-', bth[buf[i++]], bth[buf[i++]], '-', bth[buf[i++]], bth[buf[i++]], '-', bth[buf[i++]], bth[buf[i++]], '-', bth[buf[i++]], bth[buf[i++]], bth[buf[i++]], bth[buf[i++]], bth[buf[i++]], bth[buf[i++]]].join('');\n}\n\nexport default bytesToUuid;","import rng from './rng.js';\nimport bytesToUuid from './bytesToUuid.js';\n\nfunction v4(options, buf, offset) {\n  var i = buf && offset || 0;\n\n  if (typeof options == 'string') {\n    buf = options === 'binary' ? new Array(16) : null;\n    options = null;\n  }\n\n  options = options || {};\n  var rnds = options.random || (options.rng || rng)(); // Per 4.4, set bits for version and `clock_seq_hi_and_reserved`\n\n  rnds[6] = rnds[6] & 0x0f | 0x40;\n  rnds[8] = rnds[8] & 0x3f | 0x80; // Copy bytes to buffer, if provided\n\n  if (buf) {\n    for (var ii = 0; ii < 16; ++ii) {\n      buf[i + ii] = rnds[ii];\n    }\n  }\n\n  return buf || bytesToUuid(rnds);\n}\n\nexport default v4;","'use strict';\nvar $ = require('../internals/export');\nvar $includes = require('../internals/array-includes').includes;\nvar addToUnscopables = require('../internals/add-to-unscopables');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar USES_TO_LENGTH = arrayMethodUsesToLength('indexOf', { ACCESSORS: true, 1: 0 });\n\n// `Array.prototype.includes` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.includes\n$({ target: 'Array', proto: true, forced: !USES_TO_LENGTH }, {\n  includes: function includes(el /* , fromIndex = 0 */) {\n    return $includes(this, el, arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n\n// https://tc39.github.io/ecma262/#sec-array.prototype-@@unscopables\naddToUnscopables('includes');\n","require('../../../modules/es.array.includes');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').includes;\n","var isObject = require('../internals/is-object');\nvar classof = require('../internals/classof-raw');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar MATCH = wellKnownSymbol('match');\n\n// `IsRegExp` abstract operation\n// https://tc39.github.io/ecma262/#sec-isregexp\nmodule.exports = function (it) {\n  var isRegExp;\n  return isObject(it) && ((isRegExp = it[MATCH]) !== undefined ? !!isRegExp : classof(it) == 'RegExp');\n};\n","var isRegExp = require('../internals/is-regexp');\n\nmodule.exports = function (it) {\n  if (isRegExp(it)) {\n    throw TypeError(\"The method doesn't accept regular expressions\");\n  } return it;\n};\n","var wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar MATCH = wellKnownSymbol('match');\n\nmodule.exports = function (METHOD_NAME) {\n  var regexp = /./;\n  try {\n    '/./'[METHOD_NAME](regexp);\n  } catch (e) {\n    try {\n      regexp[MATCH] = false;\n      return '/./'[METHOD_NAME](regexp);\n    } catch (f) { /* empty */ }\n  } return false;\n};\n","'use strict';\nvar $ = require('../internals/export');\nvar notARegExp = require('../internals/not-a-regexp');\nvar requireObjectCoercible = require('../internals/require-object-coercible');\nvar correctIsRegExpLogic = require('../internals/correct-is-regexp-logic');\n\n// `String.prototype.includes` method\n// https://tc39.github.io/ecma262/#sec-string.prototype.includes\n$({ target: 'String', proto: true, forced: !correctIsRegExpLogic('includes') }, {\n  includes: function includes(searchString /* , position = 0 */) {\n    return !!~String(requireObjectCoercible(this))\n      .indexOf(notARegExp(searchString), arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n","require('../../../modules/es.string.includes');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('String').includes;\n","var arrayIncludes = require('../array/virtual/includes');\nvar stringIncludes = require('../string/virtual/includes');\n\nvar ArrayPrototype = Array.prototype;\nvar StringPrototype = String.prototype;\n\nmodule.exports = function (it) {\n  var own = it.includes;\n  if (it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.includes)) return arrayIncludes;\n  if (typeof it === 'string' || it === StringPrototype || (it instanceof String && own === StringPrototype.includes)) {\n    return stringIncludes;\n  } return own;\n};\n","// Utility functions for ordering and stacking of items\nconst EPSILON = 0.001; // used when checking collisions, to prevent round-off errors\n\n/**\n * Order items by their start data\n * @param {Item[]} items\n */\nexport function orderByStart(items) {\n  items.sort((a, b) => a.data.start - b.data.start);\n}\n\n/**\n * Order items by their end date. If they have no end date, their start date\n * is used.\n * @param {Item[]} items\n */\nexport function orderByEnd(items) {\n  items.sort((a, b) => {\n    const aTime = ('end' in a.data) ? a.data.end : a.data.start;\n    const bTime = ('end' in b.data) ? b.data.end : b.data.start;\n\n    return aTime - bTime;\n  });\n}\n\n/**\n * Adjust vertical positions of the items such that they don't overlap each\n * other.\n * @param {Item[]} items\n *            All visible items\n * @param {{item: {horizontal: number, vertical: number}, axis: number}} margin\n *            Margins between items and between items and the axis.\n * @param {boolean} [force=false]\n *            If true, all items will be repositioned. If false (default), only\n *            items having a top===null will be re-stacked\n * @param {function} shouldBailItemsRedrawFunction\n *            bailing function\n * @return {boolean} shouldBail\n */\nexport function stack(items, margin, force, shouldBailItemsRedrawFunction) {\n  if (force) {\n    // reset top position of all items\n    for (var i = 0; i < items.length; i++) {\n      items[i].top = null;\n    }\n  }\n\n  // calculate new, non-overlapping positions\n  for (var i = 0; i < items.length; i++) {  // eslint-disable-line no-redeclare\n    const item = items[i];\n    if (item.stack && item.top === null) {\n      // initialize top position\n      item.top = margin.axis;\n      var shouldBail = false;\n\n      do {\n        // TODO: optimize checking for overlap. when there is a gap without items,\n        //       you only need to check for items from the next item on, not from zero\n        var collidingItem = null;\n        for (let j = 0, jj = items.length; j < jj; j++) {\n          const other = items[j];\n          shouldBail = shouldBailItemsRedrawFunction() || false;\n\n          if (shouldBail) { return true; }\n\n          if (other.top !== null && other !== item && other.stack && collision(item, other, margin.item, other.options.rtl)) {\n            collidingItem = other;\n            break;\n          }\n        }\n\n        if (collidingItem != null) {\n          // There is a collision. Reposition the items above the colliding element\n          item.top = collidingItem.top + collidingItem.height + margin.item.vertical;\n        }\n      } while (collidingItem);\n    }\n  }\n  return shouldBail;\n}\n\n/**\n * Adjust vertical positions of the items within a single subgroup such that they\n * don't overlap each other.\n * @param {Item[]} items\n *            All items withina subgroup\n * @param {{item: {horizontal: number, vertical: number}, axis: number}} margin\n *            Margins between items and between items and the axis.\n * @param {subgroup} subgroup\n *            The subgroup that is being stacked\n */\nexport function substack(items, margin, subgroup) {\n  for (var i = 0; i < items.length; i++) {\n    items[i].top = null;\n  }\n\n  // Set the initial height\n  let subgroupHeight = subgroup.height;\n\n  // calculate new, non-overlapping positions\n  for (i = 0; i < items.length; i++) {\n    const item = items[i];\n\n    if (item.stack && item.top === null) {\n      // initialize top position\n      item.top = item.baseTop;//margin.axis + item.baseTop;\n\n      do {\n        // TODO: optimize checking for overlap. when there is a gap without items,\n        //       you only need to check for items from the next item on, not from zero\n        var collidingItem = null;\n        for (let j = 0, jj = items.length; j < jj; j++) {\n          const other = items[j];\n          if (other.top !== null && other !== item /*&& other.stack*/ && collision(item, other, margin.item, other.options.rtl)) {\n            collidingItem = other;\n            break;\n          }\n        }\n\n        if (collidingItem != null) {\n          // There is a collision. Reposition the items above the colliding element\n          item.top = collidingItem.top + collidingItem.height + margin.item.vertical;// + item.baseTop;\n        }\n\n        if (item.top + item.height > subgroupHeight) {\n          subgroupHeight = item.top + item.height;\n        }\n      } while (collidingItem);\n    }\n  }\n\n  // Set the new height\n  subgroup.height = subgroupHeight - subgroup.top + 0.5 * margin.item.vertical;\n}\n\n/**\n * Adjust vertical positions of the items without stacking them\n * @param {Item[]} items\n *            All visible items\n * @param {{item: {horizontal: number, vertical: number}, axis: number}} margin\n *            Margins between items and between items and the axis.\n * @param {subgroups[]} subgroups\n *            All subgroups\n * @param {boolean} isStackSubgroups\n */\nexport function nostack(items, margin, subgroups, isStackSubgroups) {\n  for (let i = 0; i < items.length; i++) {\n    if (items[i].data.subgroup == undefined) {\n      items[i].top = margin.item.vertical;\n    } else if (items[i].data.subgroup !== undefined && isStackSubgroups) {\n      let newTop = 0;\n      for (const subgroup in subgroups) {\n        if (subgroups.hasOwnProperty(subgroup)) {\n          if (subgroups[subgroup].visible == true && subgroups[subgroup].index < subgroups[items[i].data.subgroup].index) {\n            newTop += subgroups[subgroup].height;\n            subgroups[items[i].data.subgroup].top = newTop;\n          }\n        }\n      }\n      items[i].top = newTop + 0.5 * margin.item.vertical;\n    }\n  }\n  if (!isStackSubgroups) {\n    stackSubgroups(items, margin, subgroups)\n  }\n}\n\n/**\n * Adjust vertical positions of the subgroups such that they don't overlap each\n * other.\n * @param {Array.<timeline.Item>} items\n * @param {{item: {horizontal: number, vertical: number}, axis: number}} margin Margins between items and between items and the axis.\n * @param {subgroups[]} subgroups\n *            All subgroups\n */\nexport function stackSubgroups(items, margin, subgroups) {\n  for (const subgroup in subgroups) {\n    if (subgroups.hasOwnProperty(subgroup)) {\n\n\n      subgroups[subgroup].top = 0;\n      do {\n        // TODO: optimize checking for overlap. when there is a gap without items,\n        //       you only need to check for items from the next item on, not from zero\n        var collidingItem = null;\n        for (const otherSubgroup in subgroups) {\n          if (subgroups[otherSubgroup].top !== null && otherSubgroup !== subgroup && subgroups[subgroup].index > subgroups[otherSubgroup].index && collisionByTimes(subgroups[subgroup], subgroups[otherSubgroup])) {\n            collidingItem = subgroups[otherSubgroup];\n            break;\n          }\n        }\n\n        if (collidingItem != null) {\n          // There is a collision. Reposition the subgroups above the colliding element\n          subgroups[subgroup].top = collidingItem.top + collidingItem.height;\n        }\n      } while (collidingItem);\n    }\n  }\n  for (let i = 0; i < items.length; i++) {\n    if (items[i].data.subgroup !== undefined) {\n      items[i].top = subgroups[items[i].data.subgroup].top + 0.5 * margin.item.vertical;\n    }\n  }\n}\n\n/**\n * Adjust vertical positions of the subgroups such that they don't overlap each\n * other, then stacks the contents of each subgroup individually.\n * @param {Item[]} subgroupItems\n *            All the items in a subgroup\n * @param {{item: {horizontal: number, vertical: number}, axis: number}} margin\n *            Margins between items and between items and the axis.\n * @param {subgroups[]} subgroups\n *            All subgroups\n */\nexport function stackSubgroupsWithInnerStack(subgroupItems, margin, subgroups) {\n  let doSubStack = false;\n\n  // Run subgroups in their order (if any)\n  const subgroupOrder = [];\n\n  for(var subgroup in subgroups) {\n    if (subgroups[subgroup].hasOwnProperty(\"index\")) {\n      subgroupOrder[subgroups[subgroup].index] = subgroup;\n    }\n    else {\n      subgroupOrder.push(subgroup);\n    }\n  }\n\n  for(let j = 0; j < subgroupOrder.length; j++) {\n    subgroup = subgroupOrder[j];\n    if (subgroups.hasOwnProperty(subgroup)) {\n\n      doSubStack = doSubStack || subgroups[subgroup].stack;\n      subgroups[subgroup].top = 0;\n\n      for (const otherSubgroup in subgroups) {\n        if (subgroups[otherSubgroup].visible && subgroups[subgroup].index > subgroups[otherSubgroup].index) {\n          subgroups[subgroup].top += subgroups[otherSubgroup].height;\n        }\n      }\n\n      const items = subgroupItems[subgroup];\n      for(let i = 0; i < items.length; i++) {\n        if (items[i].data.subgroup !== undefined) {\n          items[i].top = subgroups[items[i].data.subgroup].top + 0.5 * margin.item.vertical;\n\n          if (subgroups[subgroup].stack) {\n            items[i].baseTop = items[i].top;\n          }\n        }\n      }\n\n      if (doSubStack && subgroups[subgroup].stack) {\n        substack(subgroupItems[subgroup], margin, subgroups[subgroup]);\n      }\n    }\n  }\n}\n\n/**\n * Test if the two provided items collide\n * The items must have parameters left, width, top, and height.\n * @param {Item} a          The first item\n * @param {Item} b          The second item\n * @param {{horizontal: number, vertical: number}} margin\n *                          An object containing a horizontal and vertical\n *                          minimum required margin.\n * @param {boolean} rtl\n * @return {boolean}        true if a and b collide, else false\n */\nexport function collision(a, b, margin, rtl) {\n  if (rtl) {\n    return  ((a.right - margin.horizontal + EPSILON)  < (b.right + b.width) &&\n    (a.right + a.width + margin.horizontal - EPSILON) > b.right &&\n    (a.top - margin.vertical + EPSILON)              < (b.top + b.height) &&\n    (a.top + a.height + margin.vertical - EPSILON)   > b.top);\n  } else {\n    return ((a.left - margin.horizontal + EPSILON)   < (b.left + b.width) &&\n    (a.left + a.width + margin.horizontal - EPSILON) > b.left &&\n    (a.top - margin.vertical + EPSILON)              < (b.top + b.height) &&\n    (a.top + a.height + margin.vertical - EPSILON)   > b.top);\n  }\n}\n\n/**\n * Test if the two provided objects collide\n * The objects must have parameters start, end, top, and height.\n * @param {Object} a          The first Object\n * @param {Object} b          The second Object\n * @return {boolean}        true if a and b collide, else false\n */\nexport function collisionByTimes(a, b) {\n\n  // Check for overlap by time and height. Abutting is OK and\n  // not considered a collision while overlap is considered a collision.\n  const timeOverlap = a.start < b.end && a.end > b.start;\n  const heightOverlap = a.top < (b.top + b.height) && (a.top + a.height) > b.top;\n  return timeOverlap && heightOverlap;\n}\n","import util from '../../util';\nimport * as stack from '../Stack';\n\nconst UNGROUPED = '__ungrouped__';   // reserved group id for ungrouped items\nconst BACKGROUND = '__background__'; // reserved group id for background items without group\n\nexport const ReservedGroupIds = {\n  UNGROUPED,\n  BACKGROUND\n}\n\n\n/**\n * @constructor Group\n */\nclass Group {\n  /**\n * @param {number | string} groupId\n * @param {Object} data\n * @param {ItemSet} itemSet\n * @constructor Group\n */\n  constructor(groupId, data, itemSet) {\n    this.groupId = groupId;\n    this.subgroups = {};\n    this.subgroupStack = {};\n    this.subgroupStackAll = false;\n    this.subgroupVisibility = {};\n    this.doInnerStack = false;\n    this.shouldBailStackItems = false;\n    this.subgroupIndex = 0;\n    this.subgroupOrderer = data && data.subgroupOrder;\n    this.itemSet = itemSet;\n    this.isVisible = null;\n    this.stackDirty = true; // if true, items will be restacked on next redraw\n\n    // This is a stack of functions (`() => void`) that will be executed before\n    // the instance is disposed off (method `dispose`). Anything that needs to\n    // be manually disposed off before garbage collection happens (or so that\n    // garbage collection can happen) should be added to this stack.\n    this._disposeCallbacks = [];\n\n    if (data && data.nestedGroups) {\n      this.nestedGroups = data.nestedGroups;\n      if (data.showNested == false) {\n        this.showNested = false;\n      } else {\n        this.showNested = true;\n      }\n    }\n\n    if (data && data.subgroupStack) {\n      if (typeof data.subgroupStack === \"boolean\") {\n        this.doInnerStack = data.subgroupStack;\n        this.subgroupStackAll = data.subgroupStack;\n      }\n      else {\n        // We might be doing stacking on specific sub groups, but only\n        // if at least one is set to do stacking\n        for(const key in data.subgroupStack) {\n          this.subgroupStack[key] = data.subgroupStack[key];\n          this.doInnerStack = this.doInnerStack || data.subgroupStack[key];\n        }\n      }\n    }\n\n    if (data && data.heightMode) {\n      this.heightMode = data.heightMode;\n    } else {\n      this.heightMode = itemSet.options.groupHeightMode;\n    }\n\n    this.nestedInGroup = null;\n\n    this.dom = {};\n    this.props = {\n      label: {\n        width: 0,\n        height: 0\n      }\n    };\n    this.className = null;\n\n    this.items = {};        // items filtered by groupId of this group\n    this.visibleItems = []; // items currently visible in window\n    this.itemsInRange = []; // items currently in range\n    this.orderedItems = {\n      byStart: [],\n      byEnd: []\n    };\n    this.checkRangedItems = false; // needed to refresh the ranged items if the window is programatically changed with NO overlap.\n\n    const handleCheckRangedItems = () => {\n      this.checkRangedItems = true;\n    };\n    this.itemSet.body.emitter.on(\"checkRangedItems\", handleCheckRangedItems);\n    this._disposeCallbacks.push(() => {\n      this.itemSet.body.emitter.off(\"checkRangedItems\", handleCheckRangedItems);\n    });\n\n    this._create();\n\n    this.setData(data);\n  }\n\n  /**\n   * Create DOM elements for the group\n   * @private\n   */\n  _create() {\n    const label = document.createElement('div');\n    if (this.itemSet.options.groupEditable.order) {\n      label.className = 'vis-label draggable';\n    } else {\n      label.className = 'vis-label';\n    }\n    this.dom.label = label;\n\n    const inner = document.createElement('div');\n    inner.className = 'vis-inner';\n    label.appendChild(inner);\n    this.dom.inner = inner;\n\n    const foreground = document.createElement('div');\n    foreground.className = 'vis-group';\n    foreground['vis-group'] = this;\n    this.dom.foreground = foreground;\n\n    this.dom.background = document.createElement('div');\n    this.dom.background.className = 'vis-group';\n\n    this.dom.axis = document.createElement('div');\n    this.dom.axis.className = 'vis-group';\n\n    // create a hidden marker to detect when the Timelines container is attached\n    // to the DOM, or the style of a parent of the Timeline is changed from\n    // display:none is changed to visible.\n    this.dom.marker = document.createElement('div');\n    this.dom.marker.style.visibility = 'hidden';\n    this.dom.marker.style.position = 'absolute';\n    this.dom.marker.innerHTML = '';\n    this.dom.background.appendChild(this.dom.marker);\n  }\n\n  /**\n   * Set the group data for this group\n   * @param {Object} data   Group data, can contain properties content and className\n   */\n  setData(data) {\n    if (this.itemSet.groupTouchParams.isDragging) return;\n\n    // update contents\n    let content;\n    let templateFunction;\n\n    if (data && data.subgroupVisibility) {\n      for (const key in data.subgroupVisibility) {\n        this.subgroupVisibility[key] = data.subgroupVisibility[key];\n      }\n    }\n\n    if (this.itemSet.options && this.itemSet.options.groupTemplate) {\n      templateFunction = this.itemSet.options.groupTemplate.bind(this);\n      content = templateFunction(data, this.dom.inner);\n    } else {\n      content = data && data.content;\n    }\n\n    if (content instanceof Element) {\n      while (this.dom.inner.firstChild) {\n        this.dom.inner.removeChild(this.dom.inner.firstChild);\n      }\n      this.dom.inner.appendChild(content);\n    } else if (content instanceof Object && content.isReactComponent) {\n      // Do nothing. Component was rendered into the node be ReactDOM.render.\n      // That branch is necessary for evasion of a second call templateFunction.\n      // Supports only React < 16(due to the asynchronous nature of React 16).\n    } else if (content instanceof Object) {\n      templateFunction(data, this.dom.inner);\n    } else if (content !== undefined && content !== null) {\n      this.dom.inner.innerHTML = content;\n    } else {\n      this.dom.inner.innerHTML = this.groupId || ''; // groupId can be null\n    }\n\n    // update title\n    this.dom.label.title = data && data.title || '';\n    if (!this.dom.inner.firstChild) {\n      util.addClassName(this.dom.inner, 'vis-hidden');\n    }\n    else {\n      util.removeClassName(this.dom.inner, 'vis-hidden');\n    }\n\n    if (data && data.nestedGroups) {\n      if (!this.nestedGroups || this.nestedGroups != data.nestedGroups) {\n        this.nestedGroups = data.nestedGroups;\n      }\n\n      if (data.showNested !== undefined || this.showNested === undefined) {\n        if (data.showNested == false) {\n          this.showNested = false;\n        } else {\n          this.showNested = true;\n        }\n      }\n\n      util.addClassName(this.dom.label, 'vis-nesting-group');\n      if (this.showNested) {\n        util.removeClassName(this.dom.label, 'collapsed');\n        util.addClassName(this.dom.label, 'expanded');\n      } else {\n        util.removeClassName(this.dom.label, 'expanded');\n        util.addClassName(this.dom.label, 'collapsed');\n      }\n    } else if (this.nestedGroups) {\n      this.nestedGroups = null;\n      util.removeClassName(this.dom.label, 'collapsed');\n      util.removeClassName(this.dom.label, 'expanded');\n      util.removeClassName(this.dom.label, 'vis-nesting-group');\n    }\n\n    if (data && (data.treeLevel|| data.nestedInGroup)) {\n      util.addClassName(this.dom.label, 'vis-nested-group');\n      if (data.treeLevel) {\n        util.addClassName(this.dom.label, 'vis-group-level-' + data.treeLevel);\n      } else {\n        // Nesting level is unknown, but we're sure it's at least 1\n        util.addClassName(this.dom.label, 'vis-group-level-unknown-but-gte1');\n      }\n    } else {\n      util.addClassName(this.dom.label, 'vis-group-level-0');\n    }\n    \n    // update className\n    const className = data && data.className || null;\n    if (className != this.className) {\n      if (this.className) {\n        util.removeClassName(this.dom.label, this.className);\n        util.removeClassName(this.dom.foreground, this.className);\n        util.removeClassName(this.dom.background, this.className);\n        util.removeClassName(this.dom.axis, this.className);\n      }\n      util.addClassName(this.dom.label, className);\n      util.addClassName(this.dom.foreground, className);\n      util.addClassName(this.dom.background, className);\n      util.addClassName(this.dom.axis, className);\n      this.className = className;\n    }\n\n    // update style\n    if (this.style) {\n      util.removeCssText(this.dom.label, this.style);\n      this.style = null;\n    }\n    if (data && data.style) {\n      util.addCssText(this.dom.label, data.style);\n      this.style = data.style;\n    }\n  }\n\n  /**\n   * Get the width of the group label\n   * @return {number} width\n   */\n  getLabelWidth() {\n    return this.props.label.width;\n  }\n\n  /**\n   * check if group has had an initial height hange\n   * @returns {boolean} \n   */\n  _didMarkerHeightChange() {\n    const markerHeight = this.dom.marker.clientHeight;\n    if (markerHeight != this.lastMarkerHeight) {\n      this.lastMarkerHeight = markerHeight;\n      const redrawQueue = {};\n      let redrawQueueLength = 0;\n\n      util.forEach(this.items, (item, key) => {\n        item.dirty = true;\n        if (item.displayed) {\n          const returnQueue = true;\n          redrawQueue[key] = item.redraw(returnQueue);\n          redrawQueueLength = redrawQueue[key].length;\n        }\n      })\n\n      const needRedraw = redrawQueueLength > 0;\n      if (needRedraw) {\n        // redraw all regular items\n        for (let i = 0; i < redrawQueueLength; i++) {\n          util.forEach(redrawQueue, fns => {\n            fns[i]();\n          });\n        }\n      }\n      return true;\n    } else {\n      return false;\n    }\n  }\n\n  /**\n   * calculate group dimentions and position\n   * @param {number} pixels\n   */\n  _calculateGroupSizeAndPosition() {\n    const { offsetTop, offsetLeft, offsetWidth } = this.dom.foreground;\n    this.top = offsetTop;\n    this.right = offsetLeft;\n    this.width = offsetWidth;\n  }\n\n  /**\n   * checks if should bail redraw of items\n   * @returns {boolean} should bail \n   */\n  _shouldBailItemsRedraw() {\n    const me = this;\n    const timeoutOptions = this.itemSet.options.onTimeout;\n    const bailOptions = {\n      relativeBailingTime: this.itemSet.itemsSettingTime,\n      bailTimeMs: timeoutOptions && timeoutOptions.timeoutMs,\n      userBailFunction: timeoutOptions && timeoutOptions.callback,\n      shouldBailStackItems: this.shouldBailStackItems\n    };\n    let bail = null;\n    if (!this.itemSet.initialDrawDone) {\n      if (bailOptions.shouldBailStackItems) { return true; }\n      if (Math.abs(Date.now() - new Date(bailOptions.relativeBailingTime)) > bailOptions.bailTimeMs) {\n        if (bailOptions.userBailFunction && this.itemSet.userContinueNotBail == null) {\n          bailOptions.userBailFunction(didUserContinue => {\n            me.itemSet.userContinueNotBail = didUserContinue;\n            bail = !didUserContinue;\n          })\n        } else if (me.itemSet.userContinueNotBail == false) {\n          bail = true;\n        } else {\n          bail = false;\n        }\n      }\n    }\n\n    return bail;\n  }\n\n  /**\n   * redraws items\n   * @param {boolean} forceRestack\n   * @param {boolean} lastIsVisible\n   * @param {number} margin\n   * @param {object} range\n   * @private\n   */\n  _redrawItems(forceRestack, lastIsVisible, margin, range) {\n    const restack = forceRestack || this.stackDirty || this.isVisible && !lastIsVisible;\n\n    // if restacking, reposition visible items vertically\n    if (restack) {\n      const orderedItems = {\n        byEnd: this.orderedItems.byEnd.filter(item => !item.isCluster),\n        byStart: this.orderedItems.byStart.filter(item => !item.isCluster)\n      }\n\n      const orderedClusters = {\n        byEnd: [...new Set(this.orderedItems.byEnd.map(item => item.cluster).filter(item => !!item))],\n        byStart: [...new Set(this.orderedItems.byStart.map(item => item.cluster).filter(item => !!item))],\n      }\n\n     /**\n     * Get all visible items in range\n     * @return {array} items\n     */\n      const getVisibleItems = () => {\n        const visibleItems = this._updateItemsInRange(orderedItems, this.visibleItems.filter(item => !item.isCluster), range);\n        const visibleClusters = this._updateClustersInRange(orderedClusters, this.visibleItems.filter(item => item.isCluster), range);\n        return [...visibleItems, ...visibleClusters];\n      }\n\n      /**\n       * Get visible items grouped by subgroup\n       * @param {function} orderFn An optional function to order items inside the subgroups\n       * @return {Object}\n       */\n      const getVisibleItemsGroupedBySubgroup = orderFn => {\n        let visibleSubgroupsItems = {};\n        for (const subgroup in this.subgroups) {\n          const items = this.visibleItems.filter(item => item.data.subgroup === subgroup);\n          visibleSubgroupsItems[subgroup] = orderFn ? items.sort((a, b) => orderFn(a.data, b.data)) : items;\n        }\n        return visibleSubgroupsItems;\n      };\n\n      if (typeof this.itemSet.options.order === 'function') {\n        // a custom order function\n        //show all items\n        const me = this;\n        if (this.doInnerStack && this.itemSet.options.stackSubgroups) {\n          // Order the items within each subgroup\n          const visibleSubgroupsItems = getVisibleItemsGroupedBySubgroup(this.itemSet.options.order);\n          stack.stackSubgroupsWithInnerStack(visibleSubgroupsItems, margin, this.subgroups);\n          this.visibleItems = getVisibleItems();\n          this._updateSubGroupHeights(margin);\n        }\n        else {\n          this.visibleItems = getVisibleItems();\n          this._updateSubGroupHeights(margin);\n          // order all items and force a restacking\n           // order all items outside clusters and force a restacking\n          const customOrderedItems = this.visibleItems\n                                  .slice()\n                                  .filter(item => item.isCluster || (!item.isCluster && !item.cluster))\n                                  .sort((a, b) => {\n                                      return me.itemSet.options.order(a.data, b.data);\n                                  });\n          this.shouldBailStackItems = stack.stack(customOrderedItems, margin, true, this._shouldBailItemsRedraw.bind(this));\n        }\n      } else {\n        // no custom order function, lazy stacking\n        this.visibleItems = getVisibleItems();\n        this._updateSubGroupHeights(margin);\n\n        if (this.itemSet.options.stack) {\n          if (this.doInnerStack && this.itemSet.options.stackSubgroups) {\n            const visibleSubgroupsItems = getVisibleItemsGroupedBySubgroup();\n            stack.stackSubgroupsWithInnerStack(visibleSubgroupsItems, margin, this.subgroups);\n          }\n          else {\n            // TODO: ugly way to access options...\n            this.shouldBailStackItems = stack.stack(this.visibleItems, margin, true, this._shouldBailItemsRedraw.bind(this));\n          }\n        } else {\n          // no stacking\n          stack.nostack(this.visibleItems, margin, this.subgroups, this.itemSet.options.stackSubgroups);\n        }\n      }\n\n      for (let i = 0; i < this.visibleItems.length; i++) {\n        this.visibleItems[i].repositionX();\n        if (this.subgroupVisibility[this.visibleItems[i].data.subgroup] !== undefined) {\n          if (!this.subgroupVisibility[this.visibleItems[i].data.subgroup]) {\n            this.visibleItems[i].hide();\n          }\n        }\n      }\n\n      if (this.itemSet.options.cluster) {\n        util.forEach(this.items, item => {\n          if (item.cluster && item.displayed) {\n            item.hide();\n          }\n        });\n      }\n\n      if (this.shouldBailStackItems) {\n        this.itemSet.body.emitter.emit('destroyTimeline')\n      }\n      this.stackDirty = false;\n    }\n  }\n\n  /**\n   * check if group resized\n   * @param {boolean} resized\n   * @param {number} height\n   * @return {boolean} did resize\n   */\n  _didResize(resized, height) {\n    resized = util.updateProperty(this, 'height', height) || resized;\n    // recalculate size of label\n    const labelWidth = this.dom.inner.clientWidth;\n    const labelHeight = this.dom.inner.clientHeight;\n    resized = util.updateProperty(this.props.label, 'width', labelWidth) || resized;\n    resized = util.updateProperty(this.props.label, 'height', labelHeight) || resized;\n    return resized;\n  }\n\n  /**\n   * apply group height\n   * @param {number} height\n   */\n  _applyGroupHeight(height) {\n    this.dom.background.style.height = `${height}px`;\n    this.dom.foreground.style.height = `${height}px`;\n    this.dom.label.style.height = `${height}px`;\n  }\n\n  /**\n   * update vertical position of items after they are re-stacked and the height of the group is calculated\n   * @param {number} margin\n   */\n  _updateItemsVerticalPosition(margin) {\n    for (let i = 0, ii = this.visibleItems.length; i < ii; i++) {\n      const item = this.visibleItems[i];\n      item.repositionY(margin);\n      if (!this.isVisible && this.groupId != ReservedGroupIds.BACKGROUND) {\n        if (item.displayed) item.hide();\n      }\n    }\n  }\n\n  /**\n   * Repaint this group\n   * @param {{start: number, end: number}} range\n   * @param {{item: {horizontal: number, vertical: number}, axis: number}} margin\n   * @param {boolean} [forceRestack=false]  Force restacking of all items\n   * @param {boolean} [returnQueue=false]  return the queue or if the group resized\n   * @return {boolean} Returns true if the group is resized or the redraw queue if returnQueue=true\n   */\n  redraw(range, margin, forceRestack, returnQueue) {\n    let resized = false;\n    const lastIsVisible = this.isVisible;\n    let height;\n\n    const queue = [\n      () => {\n        forceRestack = this._didMarkerHeightChange.call(this) || forceRestack;\n      },\n      \n      // recalculate the height of the subgroups\n      this._updateSubGroupHeights.bind(this, margin),\n\n      // calculate actual size and position\n      this._calculateGroupSizeAndPosition.bind(this),\n\n      () => {\n        this.isVisible = this._isGroupVisible.bind(this)(range, margin);\n      },\n      \n      () => {\n        this._redrawItems.bind(this)(forceRestack, lastIsVisible, margin, range)\n      },\n\n      // update subgroups\n      this._updateSubgroupsSizes.bind(this),\n\n      () => {\n        height = this._calculateHeight.bind(this)(margin);\n      },\n\n      // calculate actual size and position again\n      this._calculateGroupSizeAndPosition.bind(this),\n\n      () => {\n        resized = this._didResize.bind(this)(resized, height)\n      },\n\n      () => {\n        this._applyGroupHeight.bind(this)(height)\n      },\n\n      () => {\n        this._updateItemsVerticalPosition.bind(this)(margin)\n      },\n\n      (() => {\n        if (!this.isVisible && this.height) {\n          resized = false;\n        }\n        return resized\n      }).bind(this)\n    ];\n\n    if (returnQueue) {\n      return queue;\n    } else {\n      let result;\n      queue.forEach(fn => {\n        result = fn();\n      });\n      return result;\n    }\n  }\n\n  /**\n   * recalculate the height of the subgroups\n   *\n   * @param {{item: timeline.Item}} margin\n   * @private\n   */\n  _updateSubGroupHeights(margin) {\n    if (Object.keys(this.subgroups).length > 0) {\n      const me = this;\n\n      this._resetSubgroups();\n\n      util.forEach(this.visibleItems, item => {\n        if (item.data.subgroup !== undefined) {\n          me.subgroups[item.data.subgroup].height = Math.max(me.subgroups[item.data.subgroup].height, item.height + margin.item.vertical);\n          me.subgroups[item.data.subgroup].visible = typeof this.subgroupVisibility[item.data.subgroup] === 'undefined' ? true : Boolean(this.subgroupVisibility[item.data.subgroup]);\n        }\n      });\n    }\n  }\n\n  /**\n   * check if group is visible\n   *\n   * @param {timeline.Range} range\n   * @param {{axis: timeline.DataAxis}} margin\n   * @returns {boolean} is visible\n   * @private\n   */\n  _isGroupVisible(range, margin) {\n    return (this.top <= range.body.domProps.centerContainer.height - range.body.domProps.scrollTop + margin.axis)\n    && (this.top + this.height + margin.axis >= - range.body.domProps.scrollTop);\n  }\n\n  /**\n   * recalculate the height of the group\n   * @param {{item: {horizontal: number, vertical: number}, axis: number}} margin\n   * @returns {number} Returns the height\n   * @private\n   */\n  _calculateHeight(margin) {\n    // recalculate the height of the group\n    let height;\n\n    let items;\n\n    if (this.heightMode === 'fixed') {\n      items = util.toArray(this.items);\n    } else {\n      // default or 'auto'\n      items = this.visibleItems;\n    }\n\n    if (items.length > 0) {\n      let min = items[0].top;\n      let max = items[0].top + items[0].height;\n      util.forEach(items, item => {\n        min = Math.min(min, item.top);\n        max = Math.max(max, (item.top + item.height));\n      });\n      if (min > margin.axis) {\n        // there is an empty gap between the lowest item and the axis\n        const offset = min - margin.axis;\n        max -= offset;\n        util.forEach(items, item => {\n          item.top -= offset;\n        });\n      }\n      height = Math.ceil(max + margin.item.vertical / 2);\n      if (this.heightMode !== \"fitItems\") {\n        height = Math.max(height, this.props.label.height);\n      }\n    }\n    else {\n      height = 0 || this.props.label.height;\n    }\n    return height;\n  }\n\n  /**\n   * Show this group: attach to the DOM\n   */\n  show() {\n    if (!this.dom.label.parentNode) {\n      this.itemSet.dom.labelSet.appendChild(this.dom.label);\n    }\n\n    if (!this.dom.foreground.parentNode) {\n      this.itemSet.dom.foreground.appendChild(this.dom.foreground);\n    }\n\n    if (!this.dom.background.parentNode) {\n      this.itemSet.dom.background.appendChild(this.dom.background);\n    }\n\n    if (!this.dom.axis.parentNode) {\n      this.itemSet.dom.axis.appendChild(this.dom.axis);\n    }\n  }\n\n  /**\n   * Hide this group: remove from the DOM\n   */\n  hide() {\n    const label = this.dom.label;\n    if (label.parentNode) {\n      label.parentNode.removeChild(label);\n    }\n\n    const foreground = this.dom.foreground;\n    if (foreground.parentNode) {\n      foreground.parentNode.removeChild(foreground);\n    }\n\n    const background = this.dom.background;\n    if (background.parentNode) {\n      background.parentNode.removeChild(background);\n    }\n\n    const axis = this.dom.axis;\n    if (axis.parentNode) {\n      axis.parentNode.removeChild(axis);\n    }\n  }\n\n  /**\n   * Add an item to the group\n   * @param {Item} item\n   */\n  add(item) {\n    this.items[item.id] = item;\n    item.setParent(this);\n    this.stackDirty = true;\n    // add to\n    if (item.data.subgroup !== undefined) {\n      this._addToSubgroup(item);\n      this.orderSubgroups();\n    }\n\n    if (!this.visibleItems.includes(item)) {\n      const range = this.itemSet.body.range; // TODO: not nice accessing the range like this\n      this._checkIfVisible(item, this.visibleItems, range);\n    }\n  }\n\n  /**\n   * add item to subgroup\n   * @param {object} item\n   * @param {string} subgroupId\n   */\n  _addToSubgroup(item, subgroupId=item.data.subgroup) {\n    if (subgroupId != undefined && this.subgroups[subgroupId] === undefined) {\n      this.subgroups[subgroupId] = {\n        height: 0,\n        top: 0,\n        start: item.data.start,\n        end: item.data.end || item.data.start,\n        visible: false,\n        index: this.subgroupIndex,\n        items: [],\n        stack: this.subgroupStackAll || this.subgroupStack[subgroupId] || false\n      };\n      this.subgroupIndex++;\n    }\n\n\n    if (new Date(item.data.start) < new Date(this.subgroups[subgroupId].start)) {\n      this.subgroups[subgroupId].start = item.data.start;\n    }\n\n    const itemEnd = item.data.end || item.data.start;\n    if (new Date(itemEnd) > new Date(this.subgroups[subgroupId].end)) {\n      this.subgroups[subgroupId].end = itemEnd;\n    }\n\n    this.subgroups[subgroupId].items.push(item);\n  }\n\n  /**\n   * update subgroup sizes\n   */\n  _updateSubgroupsSizes() {\n    const me = this;\n    if (me.subgroups) {\n      for (const subgroup in me.subgroups) {\n        const initialEnd = me.subgroups[subgroup].items[0].data.end || me.subgroups[subgroup].items[0].data.start;\n        let newStart = me.subgroups[subgroup].items[0].data.start;\n        let newEnd = initialEnd - 1;\n\n        me.subgroups[subgroup].items.forEach(item => {\n          if (new Date(item.data.start) < new Date(newStart)) {\n            newStart = item.data.start;\n          }\n\n          const itemEnd = item.data.end || item.data.start;\n          if (new Date(itemEnd) > new Date(newEnd)) {\n            newEnd = itemEnd;\n          }\n        })\n\n        me.subgroups[subgroup].start = newStart;\n        me.subgroups[subgroup].end = new Date(newEnd - 1) // -1 to compensate for colliding end to start subgroups;\n\n      }\n    }\n  }\n\n  /**\n   * order subgroups\n   */\n  orderSubgroups() {\n    if (this.subgroupOrderer !== undefined) {\n      const sortArray = [];\n      if (typeof this.subgroupOrderer == 'string') {\n        for (const subgroup in this.subgroups) {\n          sortArray.push({subgroup, sortField: this.subgroups[subgroup].items[0].data[this.subgroupOrderer]})\n        }\n        sortArray.sort((a, b) => a.sortField - b.sortField)\n      }\n      else if (typeof this.subgroupOrderer == 'function') {\n        for (const subgroup in this.subgroups) {\n          sortArray.push(this.subgroups[subgroup].items[0].data);\n        }\n        sortArray.sort(this.subgroupOrderer);\n      }\n\n      if (sortArray.length > 0) {\n        for (let i = 0; i < sortArray.length; i++) {\n          this.subgroups[sortArray[i].subgroup].index = i;\n        }\n      }\n    }\n  }\n\n  /**\n   * add item to subgroup\n   */\n  _resetSubgroups() {\n    for (const subgroup in this.subgroups) {\n      if (this.subgroups.hasOwnProperty(subgroup)) {\n        this.subgroups[subgroup].visible = false;\n        this.subgroups[subgroup].height = 0;\n      }\n    }\n  }\n\n  /**\n   * Remove an item from the group\n   * @param {Item} item\n   */\n  remove(item) {\n    delete this.items[item.id];\n    item.setParent(null);\n    this.stackDirty = true;\n\n    // remove from visible items\n    const index = this.visibleItems.indexOf(item);\n    if (index != -1) this.visibleItems.splice(index, 1);\n\n    if(item.data.subgroup !== undefined){\n      this._removeFromSubgroup(item);\n      this.orderSubgroups();\n    }\n  }\n\n  /**\n   * remove item from subgroup\n   * @param {object} item\n   * @param {string} subgroupId\n   */\n  _removeFromSubgroup(item, subgroupId=item.data.subgroup) {\n    if (subgroupId != undefined) {\n      const subgroup = this.subgroups[subgroupId];\n      if (subgroup){\n        const itemIndex = subgroup.items.indexOf(item);\n        //  Check the item is actually in this subgroup. How should items not in the group be handled?\n        if (itemIndex >= 0) {\n          subgroup.items.splice(itemIndex,1);\n          if (!subgroup.items.length){\n            delete this.subgroups[subgroupId];\n          } else {\n            this._updateSubgroupsSizes();\n          }\n        }\n      }\n    }\n  }\n\n  /**\n   * Remove an item from the corresponding DataSet\n   * @param {Item} item\n   */\n  removeFromDataSet(item) {\n    this.itemSet.removeItem(item.id);\n  }\n\n  /**\n   * Reorder the items\n   */\n  order() {\n    const array = util.toArray(this.items);\n    const startArray = [];\n    const endArray = [];\n\n    for (let i = 0; i < array.length; i++) {\n      if (array[i].data.end !== undefined) {\n        endArray.push(array[i]);\n      }\n      startArray.push(array[i]);\n    }\n    this.orderedItems = {\n      byStart: startArray,\n      byEnd: endArray\n    };\n\n    stack.orderByStart(this.orderedItems.byStart);\n    stack.orderByEnd(this.orderedItems.byEnd);\n  }\n\n  /**\n   * Update the visible items\n   * @param {{byStart: Item[], byEnd: Item[]}} orderedItems   All items ordered by start date and by end date\n   * @param {Item[]} oldVisibleItems                          The previously visible items.\n   * @param {{start: number, end: number}} range              Visible range\n   * @return {Item[]} visibleItems                            The new visible items.\n   * @private\n   */\n  _updateItemsInRange(orderedItems, oldVisibleItems, range) {\n    const visibleItems = [];\n    const visibleItemsLookup = {}; // we keep this to quickly look up if an item already exists in the list without using indexOf on visibleItems\n\n    if (!this.isVisible && this.groupId != ReservedGroupIds.BACKGROUND) {\n      for (let i = 0; i < oldVisibleItems.length; i++) {\n        var item = oldVisibleItems[i];\n        if (item.displayed) item.hide();\n      }\n      return visibleItems;\n    } \n\n    const interval = (range.end - range.start) / 4;\n    const lowerBound = range.start - interval;\n    const upperBound = range.end + interval;\n\n    // this function is used to do the binary search for items having start date only.\n    const startSearchFunction = value => {\n      if      (value < lowerBound)  {return -1;}\n      else if (value <= upperBound) {return  0;}\n      else                          {return  1;}\n    };\n\n    // this function is used to do the binary search for items having start and end dates (range).\n    const endSearchFunction = value => {\n      if      (value < lowerBound)  {return -1;}\n      else                          {return  0;}\n    }\n\n    // first check if the items that were in view previously are still in view.\n    // IMPORTANT: this handles the case for the items with startdate before the window and enddate after the window!\n    // also cleans up invisible items.\n    if (oldVisibleItems.length > 0) {\n      for (let i = 0; i < oldVisibleItems.length; i++) {\n        this._checkIfVisibleWithReference(oldVisibleItems[i], visibleItems, visibleItemsLookup, range);\n      }\n    }\n\n    // we do a binary search for the items that have only start values.\n    const initialPosByStart = util.binarySearchCustom(orderedItems.byStart, startSearchFunction, 'data','start');\n\n    // trace the visible items from the inital start pos both ways until an invisible item is found, we only look at the start values.\n    this._traceVisible(initialPosByStart, orderedItems.byStart, visibleItems, visibleItemsLookup, item => item.data.start < lowerBound || item.data.start > upperBound);\n\n    // if the window has changed programmatically without overlapping the old window, the ranged items with start < lowerBound and end > upperbound are not shown.\n    // We therefore have to brute force check all items in the byEnd list\n    if (this.checkRangedItems == true) {\n      this.checkRangedItems = false;\n      for (let i = 0; i < orderedItems.byEnd.length; i++) {\n        this._checkIfVisibleWithReference(orderedItems.byEnd[i], visibleItems, visibleItemsLookup, range);\n      }\n    }\n    else {\n      // we do a binary search for the items that have defined end times.\n      const initialPosByEnd = util.binarySearchCustom(orderedItems.byEnd, endSearchFunction, 'data','end');\n\n      // trace the visible items from the inital start pos both ways until an invisible item is found, we only look at the end values.\n      this._traceVisible(initialPosByEnd, orderedItems.byEnd, visibleItems, visibleItemsLookup, item => item.data.end < lowerBound || item.data.start > upperBound);\n    }\n\n    const redrawQueue = {};\n    let redrawQueueLength = 0;\n\n    for (let i = 0; i < visibleItems.length; i++) {\n      const item = visibleItems[i];\n      if (!item.displayed) {\n        const returnQueue = true;\n        redrawQueue[i] = item.redraw(returnQueue);\n        redrawQueueLength = redrawQueue[i].length;\n      }\n    }\n\n    const needRedraw = redrawQueueLength > 0;\n    if (needRedraw) {\n      // redraw all regular items\n      for (let j = 0; j < redrawQueueLength; j++) {\n        util.forEach(redrawQueue, fns => {\n          fns[j]();\n        });\n      }\n    }\n\n    for (let i = 0; i < visibleItems.length; i++) {\n      visibleItems[i].repositionX();\n    }\n\n    return visibleItems;\n  }\n\n  /**\n   * trace visible items in group\n   * @param {number} initialPos\n   * @param {array} items\n   * @param {aray} visibleItems\n   * @param {object} visibleItemsLookup\n   * @param {function} breakCondition\n   */\n  _traceVisible(initialPos, items, visibleItems, visibleItemsLookup, breakCondition) {\n    if (initialPos != -1) {\n      for (let i = initialPos; i >= 0; i--) {\n        let item = items[i];\n        if (breakCondition(item)) {\n          break;\n        }\n        else {\n          if (!(item.isCluster  && !item.hasItems()) && !item.cluster) {\n            if (visibleItemsLookup[item.id] === undefined) {\n              visibleItemsLookup[item.id] = true;\n              visibleItems.push(item);\n            }\n          }\n        }\n      }\n\n      for (let i = initialPos + 1; i < items.length; i++) {\n        let item = items[i];\n        if (breakCondition(item)) {\n          break;\n        }\n        else {\n          if (!(item.isCluster && !item.hasItems()) && !item.cluster) {\n            if (visibleItemsLookup[item.id] === undefined) {\n              visibleItemsLookup[item.id] = true;\n              visibleItems.push(item);\n            }\n          }\n        }\n      }\n    }\n  }\n\n  /**\n   * this function is very similar to the _checkIfInvisible() but it does not\n   * return booleans, hides the item if it should not be seen and always adds to\n   * the visibleItems.\n   * this one is for brute forcing and hiding.\n   *\n   * @param {Item} item\n   * @param {Array} visibleItems\n   * @param {{start:number, end:number}} range\n   * @private\n   */\n  _checkIfVisible(item, visibleItems, range) {\n      if (item.isVisible(range)) {\n        if (!item.displayed) item.show();\n        // reposition item horizontally\n        item.repositionX();\n        visibleItems.push(item);\n      }\n      else {\n        if (item.displayed) item.hide();\n      }\n  }\n\n  /**\n   * this function is very similar to the _checkIfInvisible() but it does not\n   * return booleans, hides the item if it should not be seen and always adds to\n   * the visibleItems.\n   * this one is for brute forcing and hiding.\n   *\n   * @param {Item} item\n   * @param {Array.<timeline.Item>} visibleItems\n   * @param {Object<number, boolean>} visibleItemsLookup\n   * @param {{start:number, end:number}} range\n   * @private\n   */\n  _checkIfVisibleWithReference(item, visibleItems, visibleItemsLookup, range) {\n    if (item.isVisible(range)) {\n      if (visibleItemsLookup[item.id] === undefined) {\n        visibleItemsLookup[item.id] = true;\n        visibleItems.push(item);\n      }\n    }\n    else {\n      if (item.displayed) item.hide();\n    }\n  }\n\n  /**\n   * Update the visible items\n   * @param {array} orderedClusters \n   * @param {array} oldVisibleClusters                         \n   * @param {{start: number, end: number}} range             \n   * @return {Item[]} visibleItems                            \n   * @private\n   */\n  _updateClustersInRange(orderedClusters, oldVisibleClusters, range) {\n    // Clusters can overlap each other so we cannot use binary search here\n    const visibleClusters = [];\n    const visibleClustersLookup = {}; // we keep this to quickly look up if an item already exists in the list without using indexOf on visibleItems\n  \n    if (oldVisibleClusters.length > 0) {\n      for (let i = 0; i < oldVisibleClusters.length; i++) {\n        this._checkIfVisibleWithReference(oldVisibleClusters[i], visibleClusters, visibleClustersLookup, range);\n      }\n    }\n  \n    for (let i = 0; i < orderedClusters.byStart.length; i++) {\n      this._checkIfVisibleWithReference(orderedClusters.byStart[i], visibleClusters, visibleClustersLookup, range);\n    }\n  \n    for (let i = 0; i < orderedClusters.byEnd.length; i++) {\n      this._checkIfVisibleWithReference(orderedClusters.byEnd[i], visibleClusters, visibleClustersLookup, range);\n    }\n  \n    const redrawQueue = {};\n    let redrawQueueLength = 0;\n  \n    for (let i = 0; i < visibleClusters.length; i++) {\n      const item = visibleClusters[i];\n      if (!item.displayed) {\n        const returnQueue = true;\n        redrawQueue[i] = item.redraw(returnQueue);\n        redrawQueueLength = redrawQueue[i].length;\n      }\n    }\n  \n    const needRedraw = redrawQueueLength > 0;\n    if (needRedraw) {\n      // redraw all regular items\n      for (var j = 0; j < redrawQueueLength; j++) {\n        util.forEach(redrawQueue, function (fns) {\n          fns[j]();\n        });\n      }\n    }\n  \n    for (let i = 0; i < visibleClusters.length; i++) {\n      visibleClusters[i].repositionX();\n    }\n    \n    return visibleClusters;\n  }\n\n  /**\n   * change item subgroup\n   * @param {object} item\n   * @param {string} oldSubgroup\n   * @param {string} newSubgroup\n   */\n  changeSubgroup(item, oldSubgroup, newSubgroup) {\n    this._removeFromSubgroup(item, oldSubgroup);\n    this._addToSubgroup(item, newSubgroup);\n    this.orderSubgroups();\n  }\n\n  /**\n   * Call this method before you lose the last reference to an instance of this.\n   * It will remove listeners etc.\n   */\n  dispose() {\n    this.hide();\n\n    let disposeCallback;\n    while ((disposeCallback = this._disposeCallbacks.pop())) {\n      disposeCallback();\n    }\n  }\n}\n\nexport default Group;\n","import Group from './Group'\n\n/**\n * @constructor BackgroundGroup\n * @extends Group\n */\nclass BackgroundGroup extends Group {\n  /**\n * @param {number | string} groupId\n * @param {Object} data\n * @param {ItemSet} itemSet\n */\n  constructor(groupId, data, itemSet) {\n    super(groupId, data, itemSet)\n    // Group.call(this, groupId, data, itemSet);\n\n    this.width = 0;\n    this.height = 0;\n    this.top = 0;\n    this.left = 0;\n  }\n\n  /**\n   * Repaint this group\n   * @param {{start: number, end: number}} range\n   * @param {{item: {horizontal: number, vertical: number}, axis: number}} margin\n   * @param {boolean} [forceRestack=false]  Force restacking of all items\n   * @return {boolean} Returns true if the group is resized\n   */\n  redraw(range, margin, forceRestack) {  // eslint-disable-line no-unused-vars\n    const resized = false;\n\n    this.visibleItems = this._updateItemsInRange(this.orderedItems, this.visibleItems, range);\n\n    // calculate actual size\n    this.width = this.dom.background.offsetWidth;\n\n    // apply new height (just always zero for BackgroundGroup\n    this.dom.background.style.height  = '0';\n\n    // update vertical position of items after they are re-stacked and the height of the group is calculated\n    for (let i = 0, ii = this.visibleItems.length; i < ii; i++) {\n      const item = this.visibleItems[i];\n      item.repositionY(margin);\n    }\n\n    return resized;\n  }\n\n  /**\n   * Show this group: attach to the DOM\n   */\n  show() {\n    if (!this.dom.background.parentNode) {\n      this.itemSet.dom.background.appendChild(this.dom.background);\n    }\n  }\n}\n\nexport default BackgroundGroup;","import Hammer from '../../../module/hammer';\nimport util from '../../../util';\nimport moment from '../../../module/moment';\nimport locales from '../../locales';\n\nimport '../css/item.css';\n\n/**\n * Item\n */\nclass Item {\n  /**\n * @constructor Item\n * @param {Object} data             Object containing (optional) parameters type,\n *                                  start, end, content, group, className.\n * @param {{toScreen: function, toTime: function}} conversion\n *                                  Conversion functions from time to screen and vice versa\n * @param {Object} options          Configuration options\n *                                  // TODO: describe available options\n */\n  constructor(data, conversion, options) {\n    this.id = null;\n    this.parent = null;\n    this.data = data;\n    this.dom = null;\n    this.conversion = conversion || {};\n    this.defaultOptions = {\n      locales,\n      locale: 'en'\n    };\n    this.options = util.extend({}, this.defaultOptions, options);\n    this.options.locales = util.extend({}, locales, this.options.locales);\n    const defaultLocales = this.defaultOptions.locales[this.defaultOptions.locale];\n    Object.keys(this.options.locales).forEach(locale => {\n      this.options.locales[locale] = util.extend(\n        {},\n        defaultLocales,\n        this.options.locales[locale]\n      );\n    });\n    this.selected = false;\n    this.displayed = false;\n    this.groupShowing = true;\n    this.selectable = (options && options.selectable) || false;\n    this.dirty = true;\n\n    this.top = null;\n    this.right = null;\n    this.left = null;\n    this.width = null;\n    this.height = null;\n\n    this.setSelectability(data);\n\n    this.editable = null;\n    this._updateEditStatus();\n  }\n\n  /**\n   * Select current item\n   */\n  select() {\n    if (this.selectable) {\n      this.selected = true;\n      this.dirty = true;\n      if (this.displayed) this.redraw();\n    }\n  }\n\n  /**\n   * Unselect current item\n   */\n  unselect() {\n    this.selected = false;\n    this.dirty = true;\n    if (this.displayed) this.redraw();\n  }\n\n  /**\n   * Set data for the item. Existing data will be updated. The id should not\n   * be changed. When the item is displayed, it will be redrawn immediately.\n   * @param {Object} data\n   */\n  setData(data) {\n    const groupChanged = data.group != undefined && this.data.group != data.group;\n    if (groupChanged && this.parent != null) {\n      this.parent.itemSet._moveToGroup(this, data.group);\n    }\n\n    this.setSelectability(data);\n\n    if (this.parent) {\n      this.parent.stackDirty = true;\n    }\n    \n    const subGroupChanged = data.subgroup != undefined && this.data.subgroup != data.subgroup;\n    if (subGroupChanged && this.parent != null) {\n      this.parent.changeSubgroup(this, this.data.subgroup, data.subgroup);\n    }\n\n    this.data = data;\n    this._updateEditStatus();\n    this.dirty = true;\n    if (this.displayed) this.redraw();\n  }\n\n  /**\n   * Set whether the item can be selected.\n   * Can only be set/unset if the timeline's `selectable` configuration option is `true`.\n   * @param {Object} data `data` from `constructor` and `setData`\n   */\n  setSelectability(data) {\n    if (data) {\n      this.selectable = typeof data.selectable === 'undefined' ? true : Boolean(data.selectable);\n    }\n  }\n\n  /**\n   * Set a parent for the item\n   * @param {Group} parent\n   */\n  setParent(parent) {\n    if (this.displayed) {\n      this.hide();\n      this.parent = parent;\n      if (this.parent) {\n        this.show();\n      }\n    }\n    else {\n      this.parent = parent;\n    }\n  }\n\n  /**\n   * Check whether this item is visible inside given range\n   * @param {timeline.Range} range with a timestamp for start and end\n   * @returns {boolean} True if visible\n   */\n  isVisible(range) {  // eslint-disable-line no-unused-vars\n    return false;\n  }\n\n  /**\n   * Show the Item in the DOM (when not already visible)\n   * @return {Boolean} changed\n   */\n  show() {\n    return false;\n  }\n\n  /**\n   * Hide the Item from the DOM (when visible)\n   * @return {Boolean} changed\n   */\n  hide() {\n    return false;\n  }\n\n  /**\n   * Repaint the item\n   */\n  redraw() {\n    // should be implemented by the item\n  }\n\n  /**\n   * Reposition the Item horizontally\n   */\n  repositionX() {\n    // should be implemented by the item\n  }\n\n  /**\n   * Reposition the Item vertically\n   */\n  repositionY() {\n    // should be implemented by the item\n  }\n\n  /**\n   * Repaint a drag area on the center of the item when the item is selected\n   * @protected\n   */\n  _repaintDragCenter() {\n    if (this.selected && this.editable.updateTime && !this.dom.dragCenter) {\n      const me = this;\n      // create and show drag area\n      const dragCenter = document.createElement('div');\n      dragCenter.className = 'vis-drag-center';\n      dragCenter.dragCenterItem = this;\n      this.hammerDragCenter = new Hammer(dragCenter);\n\n      this.hammerDragCenter.on('tap', event => {\n        me.parent.itemSet.body.emitter.emit('click',  {\n          event,\n          item: me.id\n        });\n      });\n      this.hammerDragCenter.on('doubletap', event => {\n        event.stopPropagation();\n        me.parent.itemSet._onUpdateItem(me);\n        me.parent.itemSet.body.emitter.emit('doubleClick', {\n          event,\n          item: me.id\n        });\n      });\n      this.hammerDragCenter.on('panstart', event => {\n        // do not allow this event to propagate to the Range\n        event.stopPropagation();\n        me.parent.itemSet._onDragStart(event);\n      })\n      this.hammerDragCenter.on('panmove',  me.parent.itemSet._onDrag.bind(me.parent.itemSet));\n      this.hammerDragCenter.on('panend',   me.parent.itemSet._onDragEnd.bind(me.parent.itemSet));\n\n      if (this.dom.box) {\n        if (this.dom.dragLeft) {\n          this.dom.box.insertBefore(dragCenter, this.dom.dragLeft);\n        }\n        else {\n          this.dom.box.appendChild(dragCenter);\n        }\n      }\n      else if (this.dom.point) {\n        this.dom.point.appendChild(dragCenter);\n      }\n      \n      this.dom.dragCenter = dragCenter;\n    }\n    else if (!this.selected && this.dom.dragCenter) {\n      // delete drag area\n      if (this.dom.dragCenter.parentNode) {\n        this.dom.dragCenter.parentNode.removeChild(this.dom.dragCenter);\n      }\n      this.dom.dragCenter = null;\n      \n      if (this.hammerDragCenter) {\n        this.hammerDragCenter.destroy();\n        this.hammerDragCenter = null;\n      }\n    }\n  }\n\n  /**\n   * Repaint a delete button on the top right of the item when the item is selected\n   * @param {HTMLElement} anchor\n   * @protected\n   */\n  _repaintDeleteButton(anchor) {\n    const editable = ((this.options.editable.overrideItems || this.editable == null) && this.options.editable.remove) ||\n                   (!this.options.editable.overrideItems && this.editable != null && this.editable.remove);\n\n    if (this.selected && editable && !this.dom.deleteButton) {\n      // create and show button\n      const me = this;\n\n      const deleteButton = document.createElement('div');\n\n      if (this.options.rtl) {\n        deleteButton.className = 'vis-delete-rtl';\n      } else {\n        deleteButton.className = 'vis-delete';\n      }\n      let optionsLocale = this.options.locales[this.options.locale];\n      if (!optionsLocale) {\n        if (!this.warned) {\n          console.warn(`WARNING: options.locales['${this.options.locale}'] not found. See https://visjs.github.io/vis-timeline/docs/timeline/#Localization`);\n          this.warned = true;\n        }\n        optionsLocale = this.options.locales['en']; // fall back on english when not available\n      }\n      deleteButton.title = optionsLocale.deleteSelected;\n\n      // TODO: be able to destroy the delete button\n      this.hammerDeleteButton = new Hammer(deleteButton).on('tap', event => {\n        event.stopPropagation();\n        me.parent.removeFromDataSet(me);\n      });\n\n      anchor.appendChild(deleteButton);\n      this.dom.deleteButton = deleteButton;\n    }\n    else if (!this.selected && this.dom.deleteButton) {\n      // remove button\n      if (this.dom.deleteButton.parentNode) {\n        this.dom.deleteButton.parentNode.removeChild(this.dom.deleteButton);\n      }\n      this.dom.deleteButton = null;\n\n      if (this.hammerDeleteButton) {\n        this.hammerDeleteButton.destroy();\n        this.hammerDeleteButton = null;\n      }\n    }\n  }\n\n  /**\n   * Repaint a onChange tooltip on the top right of the item when the item is selected\n   * @param {HTMLElement} anchor\n   * @protected\n   */\n  _repaintOnItemUpdateTimeTooltip(anchor) {\n    if (!this.options.tooltipOnItemUpdateTime) return;\n\n    const editable = (this.options.editable.updateTime || \n                    this.data.editable === true) &&\n                   this.data.editable !== false;\n\n    if (this.selected && editable && !this.dom.onItemUpdateTimeTooltip) {\n      const onItemUpdateTimeTooltip = document.createElement('div');\n\n      onItemUpdateTimeTooltip.className = 'vis-onUpdateTime-tooltip';\n      anchor.appendChild(onItemUpdateTimeTooltip);\n      this.dom.onItemUpdateTimeTooltip = onItemUpdateTimeTooltip;\n\n    } else if (!this.selected && this.dom.onItemUpdateTimeTooltip) {\n      // remove button\n      if (this.dom.onItemUpdateTimeTooltip.parentNode) {\n        this.dom.onItemUpdateTimeTooltip.parentNode.removeChild(this.dom.onItemUpdateTimeTooltip);\n      }\n      this.dom.onItemUpdateTimeTooltip = null;\n    }\n\n    // position onChange tooltip\n    if (this.dom.onItemUpdateTimeTooltip) {\n\n      // only show when editing\n      this.dom.onItemUpdateTimeTooltip.style.visibility = this.parent.itemSet.touchParams.itemIsDragging ? 'visible' : 'hidden';\n      \n      // position relative to item's content\n      this.dom.onItemUpdateTimeTooltip.style.transform = 'translateX(-50%)';\n      this.dom.onItemUpdateTimeTooltip.style.left = '50%';\n\n      // position above or below the item depending on the item's position in the window\n      const tooltipOffset = 50; // TODO: should be tooltip height (depends on template)\n      const scrollTop = this.parent.itemSet.body.domProps.scrollTop;\n\n        // TODO: this.top for orientation:true is actually the items distance from the bottom... \n        // (should be this.bottom)\n      let itemDistanceFromTop; \n      if (this.options.orientation.item == 'top') {\n        itemDistanceFromTop = this.top;\n      } else {\n        itemDistanceFromTop = (this.parent.height - this.top - this.height)\n      }\n      const isCloseToTop = itemDistanceFromTop + this.parent.top - tooltipOffset < -scrollTop;\n\n      if (isCloseToTop) {\n        this.dom.onItemUpdateTimeTooltip.style.bottom = \"\";\n        this.dom.onItemUpdateTimeTooltip.style.top = `${this.height + 2}px`;\n      } else {\n        this.dom.onItemUpdateTimeTooltip.style.top = \"\";\n        this.dom.onItemUpdateTimeTooltip.style.bottom = `${this.height + 2}px`;\n      }\n      \n      // handle tooltip content\n      let content;\n      let templateFunction;\n\n      if (this.options.tooltipOnItemUpdateTime && this.options.tooltipOnItemUpdateTime.template) {\n        templateFunction = this.options.tooltipOnItemUpdateTime.template.bind(this);\n        content = templateFunction(this.data);\n      } else {\n        content = `start: ${moment(this.data.start).format('MM/DD/YYYY hh:mm')}`;\n        if (this.data.end) { \n          content += `<br> end: ${moment(this.data.end).format('MM/DD/YYYY hh:mm')}`;\n        }\n      }\n      this.dom.onItemUpdateTimeTooltip.innerHTML = content;\n    }\n  }\n\n   /**\n   * get item data\n   * @return {object}\n   * @private\n   */\n  _getItemData() {\n    return this.parent.itemSet.itemsData.get(this.id);\n  }\n\n  /**\n   * Set HTML contents for the item\n   * @param {Element} element   HTML element to fill with the contents\n   * @private\n   */\n  _updateContents(element) {\n    let content;\n    let changed;\n    let templateFunction;\n    let itemVisibleFrameContent;\n    let visibleFrameTemplateFunction; \n    const itemData = this._getItemData(); // get a clone of the data from the dataset\n\n    const frameElement = this.dom.box || this.dom.point;\n    const itemVisibleFrameContentElement = frameElement.getElementsByClassName('vis-item-visible-frame')[0];\n\n    if (this.options.visibleFrameTemplate) {\n      visibleFrameTemplateFunction = this.options.visibleFrameTemplate.bind(this);\n      itemVisibleFrameContent = visibleFrameTemplateFunction(itemData, itemVisibleFrameContentElement);\n    } else {\n      itemVisibleFrameContent = '';\n    }\n    \n    if (itemVisibleFrameContentElement) {\n      if ((itemVisibleFrameContent instanceof Object) && !(itemVisibleFrameContent instanceof Element)) {\n        visibleFrameTemplateFunction(itemData, itemVisibleFrameContentElement)\n      } else {\n         changed = this._contentToString(this.itemVisibleFrameContent) !== this._contentToString(itemVisibleFrameContent);\n         if (changed) {\n          // only replace the content when changed\n          if (itemVisibleFrameContent instanceof Element) {\n            itemVisibleFrameContentElement.innerHTML = '';\n            itemVisibleFrameContentElement.appendChild(itemVisibleFrameContent);\n          }\n          else if (itemVisibleFrameContent != undefined) {\n            itemVisibleFrameContentElement.innerHTML = itemVisibleFrameContent;\n          }\n          else {\n            if (!(this.data.type == 'background' && this.data.content === undefined)) {\n              throw new Error(`Property \"content\" missing in item ${this.id}`);\n            }\n          }\n\n          this.itemVisibleFrameContent = itemVisibleFrameContent;\n         }\n      }\n    }\n\n    if (this.options.template) {\n      templateFunction = this.options.template.bind(this);\n      content = templateFunction(itemData, element, this.data);\n    } else {\n      content = this.data.content;\n    }\n\n    if ((content instanceof Object) && !(content instanceof Element)) {\n      templateFunction(itemData, element)\n    } else {\n      changed = this._contentToString(this.content) !== this._contentToString(content);\n      if (changed) {\n        // only replace the content when changed\n        if (content instanceof Element) {\n          element.innerHTML = '';\n          element.appendChild(content);\n        }\n        else if (content != undefined) {\n          element.innerHTML = content;\n        }\n        else {\n          if (!(this.data.type == 'background' && this.data.content === undefined)) {\n            throw new Error(`Property \"content\" missing in item ${this.id}`);\n          }\n        }\n        this.content = content;\n      }\n    }\n  }\n\n  /**\n   * Process dataAttributes timeline option and set as data- attributes on dom.content\n   * @param {Element} element   HTML element to which the attributes will be attached\n   * @private\n   */\n  _updateDataAttributes(element) {\n   if (this.options.dataAttributes && this.options.dataAttributes.length > 0) {\n     let attributes = [];\n\n     if (Array.isArray(this.options.dataAttributes)) {\n       attributes = this.options.dataAttributes;\n     }\n     else if (this.options.dataAttributes == 'all') {\n       attributes = Object.keys(this.data);\n     }\n     else {\n       return;\n     }\n\n     for (const name of attributes) {\n       const value = this.data[name];\n\n       if (value != null) {\n         element.setAttribute(`data-${name}`, value);\n       }\n       else {\n         element.removeAttribute(`data-${name}`);\n       }\n     }\n   }\n }\n\n  /**\n   * Update custom styles of the element\n   * @param {Element} element\n   * @private\n   */\n  _updateStyle(element) {\n    // remove old styles\n    if (this.style) {\n      util.removeCssText(element, this.style);\n      this.style = null;\n    }\n\n    // append new styles\n    if (this.data.style) {\n      util.addCssText(element, this.data.style);\n      this.style = this.data.style;\n    }\n  }\n\n  /**\n   * Stringify the items contents\n   * @param {string | Element | undefined} content\n   * @returns {string | undefined}\n   * @private\n   */\n  _contentToString(content) {\n    if (typeof content === 'string') return content;\n    if (content && 'outerHTML' in content) return content.outerHTML;\n    return content;\n  }\n\n  /**\n   * Update the editability of this item.\n   */\n  _updateEditStatus() {\n    if (this.options) {\n      if(typeof this.options.editable === 'boolean') {\n        this.editable = {\n          updateTime: this.options.editable,\n          updateGroup: this.options.editable,\n          remove: this.options.editable\n        };\n      } else if(typeof this.options.editable === 'object') {\n          this.editable = {};\n          util.selectiveExtend(['updateTime', 'updateGroup', 'remove'], this.editable, this.options.editable);\n      }\n    }\n    // Item data overrides, except if options.editable.overrideItems is set.\n    if (!this.options || !(this.options.editable) || (this.options.editable.overrideItems !== true)) {\n      if (this.data) {\n        if (typeof this.data.editable === 'boolean') {\n          this.editable = {\n            updateTime: this.data.editable,\n            updateGroup: this.data.editable,\n            remove: this.data.editable\n          }\n        } else if (typeof this.data.editable === 'object') {\n          // TODO: in timeline.js 5.0, we should change this to not reset options from the timeline configuration.\n          // Basically just remove the next line...\n          this.editable = {};\n          util.selectiveExtend(['updateTime', 'updateGroup', 'remove'], this.editable, this.data.editable);\n        }\n      }\n    }\n  }\n\n  /**\n   * Return the width of the item left from its start date\n   * @return {number}\n   */\n  getWidthLeft() {\n    return 0;\n  }\n\n  /**\n   * Return the width of the item right from the max of its start and end date\n   * @return {number}\n   */\n  getWidthRight() {\n    return 0;\n  }\n\n  /**\n   * Return the title of the item\n   * @return {string | undefined}\n   */\n  getTitle() {\n    if (this.options.tooltip && this.options.tooltip.template) {\n      const templateFunction = this.options.tooltip.template.bind(this);\n      return templateFunction(this._getItemData(), this.data);\n    }\n\n    return this.data.title;\n  }\n}\n\nItem.prototype.stack = true;\n\nexport default Item;\n","import Item from './Item';\n\n/**\n * @constructor BoxItem\n * @extends Item\n */\nclass BoxItem extends Item {\n  /**\n * @param {Object} data             Object containing parameters start\n *                                  content, className.\n * @param {{toScreen: function, toTime: function}} conversion\n *                                  Conversion functions from time to screen and vice versa\n * @param {Object} [options]        Configuration options\n *                                  // TODO: describe available options\n */\n  constructor(data, conversion, options) {\n    super(data, conversion, options)\n    this.props = {\n      dot: {\n        width: 0,\n        height: 0\n      },\n      line: {\n        width: 0,\n        height: 0\n      }\n    };\n    // validate data\n    if (data) {\n      if (data.start == undefined) {\n        throw new Error(`Property \"start\" missing in item ${data}`);\n      }\n    }\n  }\n\n  /**\n   * Check whether this item is visible inside given range\n   * @param {{start: number, end: number}} range with a timestamp for start and end\n   * @returns {boolean} True if visible\n   */\n  isVisible(range) {\n    if (this.cluster) {\n      return false;\n    }\n    // determine visibility\n    let isVisible;\n    const align = this.data.align || this.options.align;\n    const widthInMs = this.width * range.getMillisecondsPerPixel();\n\n    if (align == 'right') {\n      isVisible = (this.data.start.getTime() > range.start) && (this.data.start.getTime() - widthInMs < range.end);\n    }\n    else if (align == 'left') {\n      isVisible = (this.data.start.getTime() + widthInMs > range.start) && (this.data.start.getTime() < range.end);\n    }\n    else {\n      // default or 'center'\n      isVisible = (this.data.start.getTime() + widthInMs / 2 > range.start ) && (this.data.start.getTime() - widthInMs/2 < range.end);\n    }\n    return isVisible;\n  }\n\n   /**\n   * create DOM element\n   * @private\n   */\n  _createDomElement() {\n    if (!this.dom) {\n      // create DOM\n      this.dom = {};\n\n      // create main box\n      this.dom.box = document.createElement('DIV');\n\n      // contents box (inside the background box). used for making margins\n      this.dom.content = document.createElement('DIV');\n      this.dom.content.className = 'vis-item-content';\n      this.dom.box.appendChild(this.dom.content);\n\n      // line to axis\n      this.dom.line = document.createElement('DIV');\n      this.dom.line.className = 'vis-line';\n\n      // dot on axis\n      this.dom.dot = document.createElement('DIV');\n      this.dom.dot.className = 'vis-dot';\n\n      // attach this item as attribute\n      this.dom.box['vis-item'] = this;\n\n      this.dirty = true;\n    }\n  }\n\n  /**\n   * append DOM element\n   * @private\n   */\n  _appendDomElement() {\n    if (!this.parent) {\n      throw new Error('Cannot redraw item: no parent attached');\n    }\n    if (!this.dom.box.parentNode) {\n      const foreground = this.parent.dom.foreground;\n      if (!foreground) throw new Error('Cannot redraw item: parent has no foreground container element');\n      foreground.appendChild(this.dom.box);\n    }\n    if (!this.dom.line.parentNode) {\n      var background = this.parent.dom.background;\n      if (!background) throw new Error('Cannot redraw item: parent has no background container element');\n      background.appendChild(this.dom.line);\n    }\n    if (!this.dom.dot.parentNode) {\n      const axis = this.parent.dom.axis;\n      if (!background) throw new Error('Cannot redraw item: parent has no axis container element');\n      axis.appendChild(this.dom.dot);\n    }\n    this.displayed = true;\n  }\n\n  /**\n   * update dirty DOM element\n   * @private\n   */\n  _updateDirtyDomComponents() {\n    // An item is marked dirty when:\n    // - the item is not yet rendered\n    // - the item's data is changed\n    // - the item is selected/deselected\n    if (this.dirty) {\n      this._updateContents(this.dom.content);\n      this._updateDataAttributes(this.dom.box);\n      this._updateStyle(this.dom.box);\n\n      const editable = (this.editable.updateTime || this.editable.updateGroup);\n\n      // update class\n      const className = (this.data.className ? ' ' + this.data.className : '') +\n        (this.selected ? ' vis-selected' : '') +\n        (editable ? ' vis-editable' : ' vis-readonly');\n      this.dom.box.className = `vis-item vis-box${className}`;\n      this.dom.line.className = `vis-item vis-line${className}`;\n      this.dom.dot.className = `vis-item vis-dot${className}`;\n    }\n  }\n\n  /**\n   * get DOM components sizes\n   * @return {object}\n   * @private\n   */\n  _getDomComponentsSizes() {\n    return {\n      previous: {\n        right: this.dom.box.style.right,\n        left: this.dom.box.style.left\n      },\n      dot: {\n        height: this.dom.dot.offsetHeight,\n        width: this.dom.dot.offsetWidth\n      },\n      line: {\n        width: this.dom.line.offsetWidth\n      },\n      box: {\n        width: this.dom.box.offsetWidth,\n        height: this.dom.box.offsetHeight\n      }\n    }\n  }\n\n  /**\n   * update DOM components sizes\n   * @param {object} sizes\n   * @private\n   */\n  _updateDomComponentsSizes(sizes) {\n    if (this.options.rtl) {\n      this.dom.box.style.right = \"0px\";\n    } else {\n      this.dom.box.style.left = \"0px\";\n    }\n\n    // recalculate size\n    this.props.dot.height = sizes.dot.height;\n    this.props.dot.width = sizes.dot.width;\n    this.props.line.width = sizes.line.width;\n    this.width = sizes.box.width;\n    this.height = sizes.box.height;\n\n    // restore previous position\n    if (this.options.rtl) {\n      this.dom.box.style.right = sizes.previous.right;\n    } else {\n      this.dom.box.style.left = sizes.previous.left;\n    }\n\n    this.dirty = false;\n  }\n\n  /**\n   * repaint DOM additionals\n   * @private\n   */\n  _repaintDomAdditionals() {\n    this._repaintOnItemUpdateTimeTooltip(this.dom.box);\n    this._repaintDragCenter();\n    this._repaintDeleteButton(this.dom.box);\n  }\n\n  /**\n   * Repaint the item\n   * @param {boolean} [returnQueue=false]  return the queue\n   * @return {boolean} the redraw queue if returnQueue=true\n   */\n  redraw(returnQueue) {\n    let sizes;\n    const queue = [\n      // create item DOM\n      this._createDomElement.bind(this),\n\n      // append DOM to parent DOM\n      this._appendDomElement.bind(this),\n\n      // update dirty DOM\n      this._updateDirtyDomComponents.bind(this),\n\n      () => {\n        if (this.dirty) {\n          sizes = this._getDomComponentsSizes();\n        }\n      },\n\n      () => {\n        if (this.dirty) {\n          this._updateDomComponentsSizes.bind(this)(sizes);\n        }\n      },\n\n      // repaint DOM additionals\n      this._repaintDomAdditionals.bind(this)\n    ];\n\n    if (returnQueue) {\n      return queue;\n    } else {\n      let result;\n      queue.forEach(fn => {\n        result = fn();\n      });\n      return result;\n    }\n  }\n\n  /**\n   * Show the item in the DOM (when not already visible). The items DOM will\n   * be created when needed.\n   * @param {boolean} [returnQueue=false]  whether to return a queue of functions to execute instead of just executing them\n   * @return {boolean} the redraw queue if returnQueue=true\n   */\n  show(returnQueue) {\n    if (!this.displayed) {\n      return this.redraw(returnQueue);\n    }\n  }\n\n  /**\n   * Hide the item from the DOM (when visible)\n   */\n  hide() {\n    if (this.displayed) {\n      const dom = this.dom;\n\n      if (dom.box.remove) dom.box.remove();\n      else if (dom.box.parentNode) dom.box.parentNode.removeChild(dom.box); // IE11\n\n      if (dom.line.remove) dom.line.remove();\n      else if (dom.line.parentNode) dom.line.parentNode.removeChild(dom.line); // IE11\n      \n      if (dom.dot.remove) dom.dot.remove();\n      else if (dom.dot.parentNode) dom.dot.parentNode.removeChild(dom.dot); // IE11\n\n      this.displayed = false;\n    }\n  }\n\n  /**\n   * Reposition the item XY\n   */\n  repositionXY() {\n    const rtl = this.options.rtl;\n\n    const repositionXY = (element, x, y, rtl = false) => {\n      if (x === undefined && y === undefined) return;\n      // If rtl invert the number.\n      const directionX = rtl ? (x * -1) : x;\n\n      //no y. translate x\n      if (y === undefined) {\n        element.style.transform = `translateX(${directionX}px)`;\n        return;\n      }\n\n      //no x. translate y\n      if (x === undefined) {\n        element.style.transform = `translateY(${y}px)`;\n        return;\n      }\n\n      element.style.transform = `translate(${directionX}px, ${y}px)`;\n    }\n    repositionXY(this.dom.box, this.boxX, this.boxY, rtl);\n    repositionXY(this.dom.dot, this.dotX, this.dotY, rtl);\n    repositionXY(this.dom.line, this.lineX, this.lineY, rtl);\n  }\n\n  /**\n   * Reposition the item horizontally\n   * @Override\n   */\n  repositionX() {\n    const start = this.conversion.toScreen(this.data.start);\n    const align = this.data.align === undefined ? this.options.align : this.data.align\n    const lineWidth = this.props.line.width;\n    const dotWidth = this.props.dot.width;\n    \n    if (align == 'right') {\n      // calculate right position of the box\n      this.boxX = start - this.width;\n      this.lineX = start - lineWidth;\n      this.dotX = start - lineWidth / 2 - dotWidth / 2;\n    }\n    else if (align == 'left') {\n      // calculate left position of the box\n      this.boxX = start;\n      this.lineX = start;\n      this.dotX = start + lineWidth / 2 - dotWidth / 2;\n    }\n    else {\n      // default or 'center'\n      this.boxX = start - this.width / 2;\n      this.lineX = this.options.rtl ? start - lineWidth : start - lineWidth / 2;\n      this.dotX = start - dotWidth / 2;\n    }\n\n    if (this.options.rtl)\n      this.right = this.boxX;\n    else\n      this.left = this.boxX;\n\n    this.repositionXY();\n  }\n\n  /**\n   * Reposition the item vertically\n   * @Override\n   */\n  repositionY() {\n    const orientation = this.options.orientation.item;\n    const lineStyle = this.dom.line.style;\n\n    if (orientation == 'top') {\n      const lineHeight = this.parent.top + this.top + 1;\n\n      this.boxY = this.top || 0;\n      lineStyle.height = `${lineHeight}px`;\n      lineStyle.bottom = '';\n      lineStyle.top = '0';\n    }\n    else { // orientation 'bottom'\n      const itemSetHeight = this.parent.itemSet.props.height; // TODO: this is nasty\n      const lineHeight = itemSetHeight - this.parent.top - this.parent.height + this.top;\n\n      this.boxY = this.parent.height - this.top - (this.height || 0);\n      lineStyle.height = `${lineHeight}px`;\n      lineStyle.top = '';\n      lineStyle.bottom = '0';\n    }\n\n    this.dotY = -this.props.dot.height / 2;\n\n    this.repositionXY();\n  }\n\n  /**\n   * Return the width of the item left from its start date\n   * @return {number}\n   */\n  getWidthLeft() {\n    return this.width / 2;\n  }\n\n  /**\n   * Return the width of the item right from its start date\n   * @return {number}\n   */\n  getWidthRight() {\n    return this.width / 2;\n  }\n}\n\nexport default BoxItem;\n","import Item from './Item';\n\n/**\n * @constructor PointItem\n * @extends Item\n */\nclass PointItem extends Item {\n  /**\n * @param {Object} data             Object containing parameters start\n *                                  content, className.\n * @param {{toScreen: function, toTime: function}} conversion\n *                                  Conversion functions from time to screen and vice versa\n * @param {Object} [options]        Configuration options\n *                                  // TODO: describe available options\n */\n  constructor(data, conversion, options) {\n    super(data, conversion, options)\n    this.props = {\n      dot: {\n        top: 0,\n        width: 0,\n        height: 0\n      },\n      content: {\n        height: 0,\n        marginLeft: 0,\n        marginRight: 0\n      }\n    };\n    // validate data\n    if (data) {\n      if (data.start == undefined) {\n        throw new Error(`Property \"start\" missing in item ${data}`);\n      }\n    }\n  }\n\n  /**\n   * Check whether this item is visible inside given range\n   * @param {{start: number, end: number}} range with a timestamp for start and end\n   * @returns {boolean} True if visible\n   */\n  isVisible(range) {\n    if (this.cluster) {\n      return false;\n    }\n    // determine visibility\n    const widthInMs = this.width * range.getMillisecondsPerPixel();\n    \n    return (this.data.start.getTime() + widthInMs > range.start ) && (this.data.start < range.end);\n  }\n\n  /**\n   * create DOM element\n   * @private\n   */\n  _createDomElement() {\n    if (!this.dom) {\n      // create DOM\n      this.dom = {};\n\n      // background box\n      this.dom.point = document.createElement('div');\n      // className is updated in redraw()\n\n      // contents box, right from the dot\n      this.dom.content = document.createElement('div');\n      this.dom.content.className = 'vis-item-content';\n      this.dom.point.appendChild(this.dom.content);\n\n      // dot at start\n      this.dom.dot = document.createElement('div');\n      this.dom.point.appendChild(this.dom.dot);\n\n      // attach this item as attribute\n      this.dom.point['vis-item'] = this;\n\n      this.dirty = true;\n    }\n  }\n\n  /**\n   * append DOM element\n   * @private\n   */\n  _appendDomElement() {\n    if (!this.parent) {\n      throw new Error('Cannot redraw item: no parent attached');\n    }\n    if (!this.dom.point.parentNode) {\n      const foreground = this.parent.dom.foreground;\n      if (!foreground) {\n        throw new Error('Cannot redraw item: parent has no foreground container element');\n      }\n      foreground.appendChild(this.dom.point);\n    }\n    this.displayed = true;\n  }\n\n  /**\n   * update dirty DOM components\n   * @private\n   */\n  _updateDirtyDomComponents() {\n    // An item is marked dirty when:\n    // - the item is not yet rendered\n    // - the item's data is changed\n    // - the item is selected/deselected\n    if (this.dirty) {\n      this._updateContents(this.dom.content);\n      this._updateDataAttributes(this.dom.point);\n      this._updateStyle(this.dom.point);\n\n      const editable = (this.editable.updateTime || this.editable.updateGroup);\n      // update class\n      const className = (this.data.className ? ' ' + this.data.className : '') +\n          (this.selected ? ' vis-selected' : '') +\n          (editable ? ' vis-editable' : ' vis-readonly');\n      this.dom.point.className  = `vis-item vis-point${className}`;\n      this.dom.dot.className  = `vis-item vis-dot${className}`;\n    }\n  }\n\n  /**\n   * get DOM component sizes\n   * @return {object}\n   * @private\n   */\n  _getDomComponentsSizes() {\n    return {\n      dot:  {\n        width: this.dom.dot.offsetWidth,\n        height: this.dom.dot.offsetHeight\n      },\n      content: {\n        width: this.dom.content.offsetWidth,\n        height: this.dom.content.offsetHeight\n      },\n      point: {\n        width: this.dom.point.offsetWidth,\n        height: this.dom.point.offsetHeight\n      }\n    }\n  }\n\n  /**\n   * update DOM components sizes\n   * @param {array} sizes\n   * @private\n   */\n  _updateDomComponentsSizes(sizes) {\n    // recalculate size of dot and contents\n    this.props.dot.width = sizes.dot.width;\n    this.props.dot.height = sizes.dot.height;\n    this.props.content.height = sizes.content.height;\n\n    // resize contents\n    if (this.options.rtl) {\n      this.dom.content.style.marginRight = `${2 * this.props.dot.width}px`;\n    } else {\n      this.dom.content.style.marginLeft = `${2 * this.props.dot.width}px`;\n    }\n    //this.dom.content.style.marginRight = ... + 'px'; // TODO: margin right\n\n    // recalculate size\n    this.width = sizes.point.width;\n    this.height = sizes.point.height;\n\n    // reposition the dot\n    this.dom.dot.style.top = `${(this.height - this.props.dot.height) / 2}px`;\n    \n    const dotWidth = this.props.dot.width;\n    const translateX = this.options.rtl ? (dotWidth / 2) * -1 : dotWidth / 2;\n    this.dom.dot.style.transform = `translateX(${translateX}px`;\n    this.dirty = false;\n  }\n\n  /**\n   * Repain DOM additionals\n   * @private\n   */\n  _repaintDomAdditionals() {\n    this._repaintOnItemUpdateTimeTooltip(this.dom.point);\n    this._repaintDragCenter();\n    this._repaintDeleteButton(this.dom.point);\n  }\n\n  /**\n   * Repaint the item\n   * @param {boolean} [returnQueue=false]  return the queue\n   * @return {boolean} the redraw queue if returnQueue=true\n   */\n  redraw(returnQueue) {\n    let sizes;\n    const queue = [\n      // create item DOM\n      this._createDomElement.bind(this),\n\n      // append DOM to parent DOM\n      this._appendDomElement.bind(this),\n\n      // update dirty DOM\n      this._updateDirtyDomComponents.bind(this),\n\n      () => {\n        if (this.dirty) {\n          sizes = this._getDomComponentsSizes();\n        }\n      },\n\n      () => {\n        if (this.dirty) {\n          this._updateDomComponentsSizes.bind(this)(sizes);\n        }\n      },\n\n      // repaint DOM additionals\n      this._repaintDomAdditionals.bind(this)\n    ];\n\n    if (returnQueue) {\n      return queue;\n    } else {\n      let result;\n      queue.forEach(fn => {\n        result = fn();\n      });\n      return result;\n    }\n  }\n\n  \n  /**\n   * Reposition XY\n   */\n  repositionXY() {\n    const rtl = this.options.rtl;\n    \n    const repositionXY = (element, x, y, rtl = false) => {\n      if (x === undefined && y === undefined) return;\n      // If rtl invert the number.\n      const directionX = rtl ? (x * -1) : x;\n\n      //no y. translate x\n      if (y === undefined) {\n        element.style.transform = `translateX(${directionX}px)`;\n        return;\n      }\n\n      //no x. translate y\n      if (x === undefined) {\n        element.style.transform = `translateY(${y}px)`;\n        return;\n      }\n\n      element.style.transform = `translate(${directionX}px, ${y}px)`;\n    }\n    repositionXY(this.dom.point, this.pointX, this.pointY, rtl);\n  }\n\n  /**\n   * Show the item in the DOM (when not already visible). The items DOM will\n   * be created when needed.\n   * @param {boolean} [returnQueue=false]  whether to return a queue of functions to execute instead of just executing them\n   * @return {boolean} the redraw queue if returnQueue=true\n   */\n  show(returnQueue) {\n    if (!this.displayed) {\n      return this.redraw(returnQueue);\n    }\n  }\n\n  /**\n   * Hide the item from the DOM (when visible)\n   */\n  hide() {\n    if (this.displayed) {\n      if (this.dom.point.parentNode) {\n        this.dom.point.parentNode.removeChild(this.dom.point);\n      }\n\n      this.displayed = false;\n    }\n  }\n\n  /**\n   * Reposition the item horizontally\n   * @Override\n   */\n  repositionX() {\n    const start = this.conversion.toScreen(this.data.start);\n\n    this.pointX = start;\n    if (this.options.rtl) {\n      this.right = start - this.props.dot.width;\n    } else {\n      this.left = start - this.props.dot.width;\n    }\n\n    this.repositionXY();\n  }\n\n  /**\n   * Reposition the item vertically\n   * @Override\n   */\n  repositionY() {\n    const orientation = this.options.orientation.item;\n    if (orientation == 'top') {\n      this.pointY = this.top;\n    }\n    else {\n      this.pointY = this.parent.height - this.top - this.height\n    }\n\n    this.repositionXY();\n  }\n\n  /**\n   * Return the width of the item left from its start date\n   * @return {number}\n   */\n  getWidthLeft() {\n    return this.props.dot.width;\n  }\n\n  /**\n   * Return the width of the item right from  its start date\n   * @return {number}\n   */\n  getWidthRight() {\n    return this.props.dot.width;\n  }\n}\n\nexport default PointItem;\n","import Item from './Item';\n\n/**\n * @constructor RangeItem\n * @extends Item\n */\nclass RangeItem extends Item {\n  /**\n * @param {Object} data             Object containing parameters start, end\n *                                  content, className.\n * @param {{toScreen: function, toTime: function}} conversion\n *                                  Conversion functions from time to screen and vice versa\n * @param {Object} [options]        Configuration options\n *                                  // TODO: describe options\n */\n  constructor(data, conversion, options) {\n    super(data, conversion, options)\n    this.props = {\n      content: {\n        width: 0\n      }\n    };\n    this.overflow = false; // if contents can overflow (css styling), this flag is set to true\n    // validate data\n    if (data) {\n      if (data.start == undefined) {\n        throw new Error(`Property \"start\" missing in item ${data.id}`);\n      }\n      if (data.end == undefined) {\n        throw new Error(`Property \"end\" missing in item ${data.id}`);\n      }\n    }\n  }\n\n  /**\n   * Check whether this item is visible inside given range\n   *\n   * @param {timeline.Range} range with a timestamp for start and end\n   * @returns {boolean} True if visible\n   */\n  isVisible(range) {\n    if (this.cluster) {\n      return false;\n    }\n    // determine visibility\n    return (this.data.start < range.end) && (this.data.end > range.start);\n  }\n\n  /**\n   * create DOM elements\n   * @private\n   */\n  _createDomElement() {\n    if (!this.dom) {\n      // create DOM\n      this.dom = {};\n\n        // background box\n      this.dom.box = document.createElement('div');\n      // className is updated in redraw()\n\n      // frame box (to prevent the item contents from overflowing)\n      this.dom.frame = document.createElement('div');\n      this.dom.frame.className = 'vis-item-overflow';\n      this.dom.box.appendChild(this.dom.frame);\n    \n      // visible frame box (showing the frame that is always visible)\n      this.dom.visibleFrame = document.createElement('div');\n      this.dom.visibleFrame.className = 'vis-item-visible-frame';\n      this.dom.box.appendChild(this.dom.visibleFrame);\n\n      // contents box\n      this.dom.content = document.createElement('div');\n      this.dom.content.className = 'vis-item-content';\n      this.dom.frame.appendChild(this.dom.content);\n\n      // attach this item as attribute\n      this.dom.box['vis-item'] = this;\n\n      this.dirty = true;\n    }\n\n  }\n\n  /**\n   * append element to DOM\n   * @private\n   */\n  _appendDomElement() {\n    if (!this.parent) {\n      throw new Error('Cannot redraw item: no parent attached');\n    }\n    if (!this.dom.box.parentNode) {\n      const foreground = this.parent.dom.foreground;\n      if (!foreground) {\n        throw new Error('Cannot redraw item: parent has no foreground container element');\n      }\n      foreground.appendChild(this.dom.box);\n    }\n    this.displayed = true;\n  }\n\n  /**\n   * update dirty DOM components\n   * @private\n   */\n  _updateDirtyDomComponents() {\n    // update dirty DOM. An item is marked dirty when:\n    // - the item is not yet rendered\n    // - the item's data is changed\n    // - the item is selected/deselected\n    if (this.dirty) {\n      this._updateContents(this.dom.content);\n      this._updateDataAttributes(this.dom.box);\n      this._updateStyle(this.dom.box);\n\n      const editable = (this.editable.updateTime || this.editable.updateGroup);\n\n      // update class\n      const className = (this.data.className ? (' ' + this.data.className) : '') +\n          (this.selected ? ' vis-selected' : '') + \n          (editable ? ' vis-editable' : ' vis-readonly');\n      this.dom.box.className = this.baseClassName + className;\n\n      // turn off max-width to be able to calculate the real width\n      // this causes an extra browser repaint/reflow, but so be it\n      this.dom.content.style.maxWidth = 'none';\n    }\n  }\n\n  /**\n   * get DOM component sizes\n   * @return {object}\n   * @private\n   */\n  _getDomComponentsSizes() {\n    // determine from css whether this box has overflow\n    this.overflow = window.getComputedStyle(this.dom.frame).overflow !== 'hidden';\n    this.whiteSpace = window.getComputedStyle(this.dom.content).whiteSpace !== 'nowrap';\n    return {\n      content: {\n        width: this.dom.content.offsetWidth,\n      },\n      box: {\n        height: this.dom.box.offsetHeight\n      }\n    }\n  }\n\n  /**\n   * update DOM component sizes\n   * @param {array} sizes\n   * @private\n   */\n  _updateDomComponentsSizes(sizes) {\n    this.props.content.width = sizes.content.width;\n    this.height = sizes.box.height;\n    this.dom.content.style.maxWidth = '';\n    this.dirty = false;\n  }\n\n  /**\n   * repaint DOM additional components\n   * @private\n   */\n  _repaintDomAdditionals() {\n    this._repaintOnItemUpdateTimeTooltip(this.dom.box);\n    this._repaintDeleteButton(this.dom.box);\n    this._repaintDragCenter();\n    this._repaintDragLeft();\n    this._repaintDragRight();\n  }\n\n  /**\n   * Repaint the item\n   * @param {boolean} [returnQueue=false]  return the queue\n   * @return {boolean} the redraw queue if returnQueue=true\n   */\n  redraw(returnQueue) {\n    let sizes;\n    const queue = [\n      // create item DOM\n      this._createDomElement.bind(this),\n\n      // append DOM to parent DOM\n      this._appendDomElement.bind(this),\n\n      // update dirty DOM \n      this._updateDirtyDomComponents.bind(this),\n\n      () => {\n        if (this.dirty) {\n          sizes = this._getDomComponentsSizes.bind(this)();\n        }\n      },\n\n      () => {\n        if (this.dirty) {\n          this._updateDomComponentsSizes.bind(this)(sizes);\n        }\n      },\n\n      // repaint DOM additionals\n      this._repaintDomAdditionals.bind(this)\n    ];\n\n    if (returnQueue) {\n      return queue;\n    } else {\n      let result;\n      queue.forEach(fn => {\n        result = fn();\n      });\n      return result;\n    }\n  }\n\n  /**\n   * Show the item in the DOM (when not already visible). The items DOM will\n   * be created when needed.\n   * @param {boolean} [returnQueue=false]  whether to return a queue of functions to execute instead of just executing them\n   * @return {boolean} the redraw queue if returnQueue=true\n   */\n  show(returnQueue) {\n    if (!this.displayed) {\n      return this.redraw(returnQueue);\n    }\n  }\n\n  /**\n   * Hide the item from the DOM (when visible)\n   */\n  hide() {\n    if (this.displayed) {\n      const box = this.dom.box;\n\n      if (box.parentNode) {\n        box.parentNode.removeChild(box);\n      }\n\n      this.displayed = false;\n    }\n  }\n\n  /**\n   * Reposition the item horizontally\n   * @param {boolean} [limitSize=true] If true (default), the width of the range\n   *                                   item will be limited, as the browser cannot\n   *                                   display very wide divs. This means though\n   *                                   that the applied left and width may\n   *                                   not correspond to the ranges start and end\n   * @Override\n   */\n  repositionX(limitSize) {\n    const parentWidth = this.parent.width;\n    let start = this.conversion.toScreen(this.data.start);\n    let end = this.conversion.toScreen(this.data.end);\n    const align = this.data.align === undefined ? this.options.align : this.data.align;\n    let contentStartPosition;\n    let contentWidth;\n\n    // limit the width of the range, as browsers cannot draw very wide divs\n    // unless limitSize: false is explicitly set in item data\n    if (this.data.limitSize !== false && (limitSize === undefined || limitSize === true)) {\n      if (start < -parentWidth) {\n        start = -parentWidth;\n      }\n      if (end > 2 * parentWidth) {\n        end = 2 * parentWidth;\n      }\n    }\n\n    //round to 3 decimals to compensate floating-point values rounding\n    const boxWidth = Math.max(Math.round((end - start) * 1000) / 1000, 1);\n\n    if (this.overflow) {\n      if (this.options.rtl) {\n        this.right = start;\n      } else {\n        this.left = start;\n      }\n      this.width = boxWidth + this.props.content.width;\n      contentWidth = this.props.content.width;\n\n      // Note: The calculation of width is an optimistic calculation, giving\n      //       a width which will not change when moving the Timeline\n      //       So no re-stacking needed, which is nicer for the eye;\n    }\n    else {\n      if (this.options.rtl) {\n        this.right = start;\n      } else {\n        this.left = start;\n      }\n      this.width = boxWidth;\n      contentWidth = Math.min(end - start, this.props.content.width);\n    }\n\n    if (this.options.rtl) {\n      this.dom.box.style.transform = `translateX(${this.right * -1}px)`;\n    } else {\n      this.dom.box.style.transform = `translateX(${this.left}px)`;\n    }\n    this.dom.box.style.width = `${boxWidth}px`;\n    if (this.whiteSpace) {\n        this.height = this.dom.box.offsetHeight;\n    }\n\n    switch (align) {\n      case 'left':\n        this.dom.content.style.transform = 'translateX(0)';\n        break;\n\n      case 'right':\n        if (this.options.rtl) {\n          const translateX = Math.max((boxWidth - contentWidth), 0) * -1;\n          this.dom.content.style.transform = `translateX(${translateX}px)`;\n        } else {\n          this.dom.content.style.transform = `translateX(${Math.max((boxWidth - contentWidth), 0)}px)`;\n        }\n        break;\n\n      case 'center':\n        if (this.options.rtl) {\n          const translateX = Math.max((boxWidth - contentWidth) / 2, 0) * -1;\n          this.dom.content.style.transform = `translateX(${translateX}px)`;\n        } else {\n          this.dom.content.style.transform = `translateX(${Math.max((boxWidth - contentWidth) / 2, 0)}px)`;\n        }\n        \n        break;\n\n      default: // 'auto'\n        // when range exceeds left of the window, position the contents at the left of the visible area\n        if (this.overflow) {\n          if (end > 0) {\n            contentStartPosition = Math.max(-start, 0);\n          }\n          else {\n            contentStartPosition = -contentWidth; // ensure it's not visible anymore\n          }\n        }\n        else {\n          if (start < 0) {\n            contentStartPosition = -start;\n          }\n          else {\n            contentStartPosition = 0;\n          }\n        }\n        if (this.options.rtl) {\n          const translateX = contentStartPosition * -1;\n          this.dom.content.style.transform = `translateX(${translateX}px)`;\n        } else {\n          this.dom.content.style.transform = `translateX(${contentStartPosition}px)`;\n          // this.dom.content.style.width = `calc(100% - ${contentStartPosition}px)`;\n        }\n    }\n  }\n\n  /**\n   * Reposition the item vertically\n   * @Override\n   */\n  repositionY() {\n    const orientation = this.options.orientation.item;\n    const box = this.dom.box;\n\n    if (orientation == 'top') {\n      box.style.top = `${this.top}px`;\n    }\n    else {\n      box.style.top = `${this.parent.height - this.top - this.height}px`;\n    }\n  }\n\n  /**\n   * Repaint a drag area on the left side of the range when the range is selected\n   * @protected\n   */\n  _repaintDragLeft() {\n    if ((this.selected || this.options.itemsAlwaysDraggable.range) && this.editable.updateTime && !this.dom.dragLeft) {\n      // create and show drag area\n      const dragLeft = document.createElement('div');\n      dragLeft.className = 'vis-drag-left';\n      dragLeft.dragLeftItem = this;\n\n      this.dom.box.appendChild(dragLeft);\n      this.dom.dragLeft = dragLeft;\n    }\n    else if (!this.selected && !this.options.itemsAlwaysDraggable.range && this.dom.dragLeft) {\n      // delete drag area\n      if (this.dom.dragLeft.parentNode) {\n        this.dom.dragLeft.parentNode.removeChild(this.dom.dragLeft);\n      }\n      this.dom.dragLeft = null;\n    }\n  }\n\n  /**\n   * Repaint a drag area on the right side of the range when the range is selected\n   * @protected\n   */\n  _repaintDragRight() {\n    if ((this.selected || this.options.itemsAlwaysDraggable.range) && this.editable.updateTime && !this.dom.dragRight) {\n      // create and show drag area\n      const dragRight = document.createElement('div');\n      dragRight.className = 'vis-drag-right';\n      dragRight.dragRightItem = this;\n\n      this.dom.box.appendChild(dragRight);\n      this.dom.dragRight = dragRight;\n    }\n    else if (!this.selected && !this.options.itemsAlwaysDraggable.range && this.dom.dragRight) {\n      // delete drag area\n      if (this.dom.dragRight.parentNode) {\n        this.dom.dragRight.parentNode.removeChild(this.dom.dragRight);\n      }\n      this.dom.dragRight = null;\n    }\n  }\n}\n\nRangeItem.prototype.baseClassName = 'vis-item vis-range';\n\nexport default RangeItem;\n","import Item from './Item';\nimport BackgroundGroup from '../BackgroundGroup';\nimport RangeItem from './RangeItem';\n\n/**\n * @constructor BackgroundItem\n * @extends Item\n */\nclass BackgroundItem extends Item {\n  /**\n * @constructor BackgroundItem\n * @param {Object} data             Object containing parameters start, end\n *                                  content, className.\n * @param {{toScreen: function, toTime: function}} conversion\n *                                  Conversion functions from time to screen and vice versa\n * @param {Object} [options]        Configuration options\n *                                  // TODO: describe options\n * // TODO: implement support for the BackgroundItem just having a start, then being displayed as a sort of an annotation\n */\n  constructor(data, conversion, options) {\n    super(data, conversion, options)\n    this.props = {\n      content: {\n        width: 0\n      }\n    };\n    this.overflow = false; // if contents can overflow (css styling), this flag is set to true\n\n    // validate data\n    if (data) {\n      if (data.start == undefined) {\n        throw new Error(`Property \"start\" missing in item ${data.id}`);\n      }\n      if (data.end == undefined) {\n        throw new Error(`Property \"end\" missing in item ${data.id}`);\n      }\n    }\n  }\n\n  /**\n   * Check whether this item is visible inside given range\n   * @param {timeline.Range} range with a timestamp for start and end\n   * @returns {boolean} True if visible\n   */\n  isVisible(range) {\n    // determine visibility\n    return (this.data.start < range.end) && (this.data.end > range.start); \n  }\n\n  /**\n   * create DOM element\n   * @private\n   */\n  _createDomElement() {\n    if (!this.dom) {\n      // create DOM\n      this.dom = {};\n\n      // background box\n      this.dom.box = document.createElement('div');\n      // className is updated in redraw()\n\n      // frame box (to prevent the item contents from overflowing\n      this.dom.frame = document.createElement('div');\n      this.dom.frame.className = 'vis-item-overflow';\n      this.dom.box.appendChild(this.dom.frame);\n\n      // contents box\n      this.dom.content = document.createElement('div');\n      this.dom.content.className = 'vis-item-content';\n      this.dom.frame.appendChild(this.dom.content);\n\n      // Note: we do NOT attach this item as attribute to the DOM,\n      //       such that background items cannot be selected\n      //this.dom.box['vis-item'] = this;\n\n      this.dirty = true;\n    }\n  }\n\n  /**\n   * append DOM element\n   * @private\n   */\n  _appendDomElement() {\n    if (!this.parent) {\n      throw new Error('Cannot redraw item: no parent attached');\n    }\n    if (!this.dom.box.parentNode) {\n      const background = this.parent.dom.background;\n      if (!background) {\n        throw new Error('Cannot redraw item: parent has no background container element');\n      }\n      background.appendChild(this.dom.box);\n    }\n    this.displayed = true;\n  }\n\n  /**\n   * update DOM Dirty components\n   * @private\n   */\n  _updateDirtyDomComponents() {\n    // update dirty DOM. An item is marked dirty when:\n    // - the item is not yet rendered\n    // - the item's data is changed\n    // - the item is selected/deselected\n    if (this.dirty) {\n      this._updateContents(this.dom.content);\n      this._updateDataAttributes(this.dom.content);\n      this._updateStyle(this.dom.box);\n\n      // update class\n      const className = (this.data.className ? (' ' + this.data.className) : '') +\n          (this.selected ? ' vis-selected' : '');\n      this.dom.box.className = this.baseClassName + className;\n    }\n  }\n\n  /**\n   * get DOM components sizes\n   * @return {object}\n   * @private\n   */\n  _getDomComponentsSizes() {\n    // determine from css whether this box has overflow\n    this.overflow = window.getComputedStyle(this.dom.content).overflow !== 'hidden';\n    return {\n      content: {\n        width: this.dom.content.offsetWidth\n      }\n    }\n  }\n\n  /**\n   * update DOM components sizes\n   * @param {object} sizes\n   * @private\n   */\n  _updateDomComponentsSizes(sizes) {\n    // recalculate size\n    this.props.content.width = sizes.content.width;\n    this.height = 0; // set height zero, so this item will be ignored when stacking items\n\n    this.dirty = false;\n  }\n\n  /**\n   * repaint DOM additionals\n   * @private\n   */\n  _repaintDomAdditionals() {\n  }\n\n  /**\n   * Repaint the item\n   * @param {boolean} [returnQueue=false]  return the queue\n   * @return {boolean} the redraw result or the redraw queue if returnQueue=true\n   */\n  redraw(returnQueue) {\n    let sizes;\n    const queue = [\n      // create item DOM\n      this._createDomElement.bind(this),\n\n      // append DOM to parent DOM\n      this._appendDomElement.bind(this),\n\n      this._updateDirtyDomComponents.bind(this),\n\n      () => {\n        if (this.dirty) {\n          sizes = this._getDomComponentsSizes.bind(this)();\n        }\n      },\n\n      () => {\n        if (this.dirty) {\n          this._updateDomComponentsSizes.bind(this)(sizes);\n        }\n      },\n\n      // repaint DOM additionals\n      this._repaintDomAdditionals.bind(this)\n    ];\n\n    if (returnQueue) {\n      return queue;\n    } else {\n      let result;\n      queue.forEach(fn => {\n        result = fn();\n      });\n      return result;\n    }\n  }\n\n  /**\n   * Reposition the item vertically\n   * @Override\n   */\n  repositionY(margin) {  // eslint-disable-line no-unused-vars\n    let height;\n    const orientation = this.options.orientation.item;\n\n    // special positioning for subgroups\n    if (this.data.subgroup !== undefined) {\n      // TODO: instead of calculating the top position of the subgroups here for every BackgroundItem, calculate the top of the subgroup once in Itemset\n      const itemSubgroup = this.data.subgroup;\n\n      this.dom.box.style.height = `${this.parent.subgroups[itemSubgroup].height}px`;\n\n      if (orientation == 'top') { \n        this.dom.box.style.top = `${this.parent.top + this.parent.subgroups[itemSubgroup].top}px`;\n      } else {\n        this.dom.box.style.top = `${this.parent.top + this.parent.height - this.parent.subgroups[itemSubgroup].top - this.parent.subgroups[itemSubgroup].height}px`;\n      }\n      this.dom.box.style.bottom = '';\n    }\n    // and in the case of no subgroups:\n    else {\n      // we want backgrounds with groups to only show in groups.\n      if (this.parent instanceof BackgroundGroup) {\n        // if the item is not in a group:\n        height = Math.max(this.parent.height,\n            this.parent.itemSet.body.domProps.center.height,\n            this.parent.itemSet.body.domProps.centerContainer.height);\n        this.dom.box.style.bottom = orientation == 'bottom' ? '0' : '';\n        this.dom.box.style.top = orientation == 'top' ? '0' : '';\n      }\n      else {\n        height = this.parent.height;\n        // same alignment for items when orientation is top or bottom\n        this.dom.box.style.top = `${this.parent.top}px`;\n        this.dom.box.style.bottom = '';\n      }\n    }\n    this.dom.box.style.height = `${height}px`;\n  }\n}\n\nBackgroundItem.prototype.baseClassName = 'vis-item vis-background';\n\nBackgroundItem.prototype.stack = false;\n\n/**\n * Show the item in the DOM (when not already visible). The items DOM will\n * be created when needed.\n */\nBackgroundItem.prototype.show = RangeItem.prototype.show;\n\n/**\n * Hide the item from the DOM (when visible)\n * @return {Boolean} changed\n */\nBackgroundItem.prototype.hide = RangeItem.prototype.hide;\n\n/**\n * Reposition the item horizontally\n * @Override\n */\nBackgroundItem.prototype.repositionX = RangeItem.prototype.repositionX;\n\nexport default BackgroundItem;\n","import './tooltip.css';\n\n/**\n * Popup is a class to create a popup window with some text\n */\nclass Popup {\n  /**\n   * @param {Element} container       The container object.\n   * @param {string}  overflowMethod  How the popup should act to overflowing ('flip', 'cap' or 'none')\n   */\n  constructor(container, overflowMethod) {\n    this.container = container;\n    this.overflowMethod = overflowMethod || 'cap';\n\n    this.x = 0;\n    this.y = 0;\n    this.padding = 5;\n    this.hidden = false;\n\n    // create the frame\n    this.frame = document.createElement('div');\n    this.frame.className = 'vis-tooltip';\n    this.container.appendChild(this.frame);\n  }\n\n  /**\n   * @param {number} x   Horizontal position of the popup window\n   * @param {number} y   Vertical position of the popup window\n   */\n  setPosition(x, y) {\n    this.x = parseInt(x);\n    this.y = parseInt(y);\n  }\n\n  /**\n   * Set the content for the popup window. This can be HTML code or text.\n   * @param {string | Element} content\n   */\n  setText(content) {\n    if (content instanceof Element) {\n      this.frame.innerHTML = '';\n      this.frame.appendChild(content);\n    }\n    else {\n      this.frame.innerHTML = content; // string containing text or HTML\n    }\n  }\n\n  /**\n   * Show the popup window\n   * @param {boolean} [doShow]    Show or hide the window\n   */\n  show(doShow) {\n    if (doShow === undefined) {\n      doShow = true;\n    }\n\n    if (doShow === true) {\n      var height = this.frame.clientHeight;\n      var width = this.frame.clientWidth;\n      var maxHeight = this.frame.parentNode.clientHeight;\n      var maxWidth = this.frame.parentNode.clientWidth;\n\n      var left = 0, top = 0;\n\n      if (this.overflowMethod == 'flip' || this.overflowMethod == 'none') {\n        let isLeft = false, isTop = true; // Where around the position it's located\n\n        if (this.overflowMethod == 'flip') {\n          if (this.y - height < this.padding) {\n            isTop = false;\n          }\n\n          if (this.x + width > maxWidth - this.padding) {\n            isLeft = true;\n          }\n        }\n\n        if (isLeft) {\n          left = this.x - width;\n        } else {\n          left = this.x;\n        }\n\n        if (isTop) {\n          top = this.y - height;\n        } else {\n          top = this.y;\n        }\n      } else { // this.overflowMethod == 'cap'\n        top = (this.y - height);\n        if (top + height + this.padding > maxHeight) {\n          top = maxHeight - height - this.padding;\n        }\n        if (top < this.padding) {\n          top = this.padding;\n        }\n\n        left = this.x;\n        if (left + width + this.padding > maxWidth) {\n          left = maxWidth - width - this.padding;\n        }\n        if (left < this.padding) {\n          left = this.padding;\n        }\n      }\n\n      this.frame.style.left = left + \"px\";\n      this.frame.style.top = top + \"px\";\n      this.frame.style.visibility = \"visible\";\n      this.hidden = false;\n    }\n    else {\n      this.hide();\n    }\n  }\n\n  /**\n   * Hide the popup window\n   */\n  hide() {\n    this.hidden = true;\n    this.frame.style.left = \"0\";\n    this.frame.style.top = \"0\";\n    this.frame.style.visibility = \"hidden\";\n  }\n\n  /**\n   * Remove the popup window\n   */\n  destroy() {\n    this.frame.parentNode.removeChild(this.frame); // Remove element from DOM\n  }\n}\n\nexport default Popup;\n","'use strict';\nvar $ = require('../internals/export');\nvar $every = require('../internals/array-iteration').every;\nvar arrayMethodIsStrict = require('../internals/array-method-is-strict');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar STRICT_METHOD = arrayMethodIsStrict('every');\nvar USES_TO_LENGTH = arrayMethodUsesToLength('every');\n\n// `Array.prototype.every` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.every\n$({ target: 'Array', proto: true, forced: !STRICT_METHOD || !USES_TO_LENGTH }, {\n  every: function every(callbackfn /* , thisArg */) {\n    return $every(this, callbackfn, arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n","require('../../../modules/es.array.every');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').every;\n","var every = require('../array/virtual/every');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.every;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.every) ? every : own;\n};\n","import { v4 as randomUUID } from \"uuid\";\nimport Item from './Item';\n\n/**\n * ClusterItem\n */\nclass ClusterItem extends Item {\n  /**\n * @constructor Item\n * @param {Object} data             Object containing (optional) parameters type,\n *                                  start, end, content, group, className.\n * @param {{toScreen: function, toTime: function}} conversion\n *                                  Conversion functions from time to screen and vice versa\n * @param {Object} options          Configuration options\n *                                  // TODO: describe available options\n */\n  constructor(data, conversion, options) {\n    const modifiedOptions = Object.assign({}, {fitOnDoubleClick: true}, options, {editable: false});\n    super(data, conversion, modifiedOptions);\n\n    this.props = {\n      content: {\n        width: 0,\n        height: 0\n      },\n    };\n    \n    if (!data || data.uiItems == undefined) {\n      throw new Error('Property \"uiItems\" missing in item ' + data.id);\n    }\n  \n    this.id = randomUUID();\n    this.group = data.group;\n    this._setupRange();\n  \n    this.emitter = this.data.eventEmitter;\n    this.range = this.data.range;\n    this.attached = false;\n    this.isCluster = true;\n    this.data.isCluster = true;\n  }\n\n  /**\n   * check if there are items\n   * @return {boolean}\n   */\n  hasItems() {\n    return this.data.uiItems && this.data.uiItems.length && this.attached;\n  }\n  \n  /**\n   * set UI items\n   * @param {array} items\n   */\n  setUiItems(items) {\n    this.detach();\n  \n    this.data.uiItems = items;\n  \n    this._setupRange();\n  \n    this.attach();\n  }\n  \n  /**\n   * check is visible\n   * @param {object} range\n   * @return {boolean}\n   */\n  isVisible(range) {\n    const rangeWidth = this.data.end ? this.data.end - this.data.start : 0;\n    const widthInMs = this.width * range.getMillisecondsPerPixel();\n    const end = Math.max(rangeWidth, this.data.start.getTime() + widthInMs);\n    return (this.data.start < range.end) && (end > range.start) && this.hasItems();\n  }\n  \n  /**\n   * get cluster data\n   * @return {object}\n   */\n  getData() {\n    return {\n      isCluster: true,\n      id: this.id,\n      items: this.data.items || [],\n      data: this.data\n    }\n  }\n  \n  /**\n   * redraw cluster item\n   * @param {boolean} returnQueue\n   * @return {boolean}\n   */\n  redraw (returnQueue) {\n    var sizes\n    var queue = [\n      // create item DOM\n      this._createDomElement.bind(this),\n  \n      // append DOM to parent DOM\n      this._appendDomElement.bind(this),\n  \n      // update dirty DOM\n      this._updateDirtyDomComponents.bind(this),\n  \n      (function () {\n        if (this.dirty) {\n          sizes = this._getDomComponentsSizes();\n        }\n      }).bind(this),\n  \n      (function () {\n        if (this.dirty) {\n          this._updateDomComponentsSizes.bind(this)(sizes);\n        }\n      }).bind(this),\n  \n      // repaint DOM additionals\n      this._repaintDomAdditionals.bind(this)\n    ];\n  \n    if (returnQueue) {\n      return queue;\n    } else {\n      var result;\n      queue.forEach(function (fn) {\n        result = fn();\n      });\n      return result;\n    }\n  }\n  \n  /**\n   * show cluster item\n   */\n  show() {\n    if (!this.displayed) {\n      this.redraw();\n    }\n  }\n  \n  /**\n   * Hide the item from the DOM (when visible)\n   */\n  hide() {\n    if (this.displayed) {\n      var dom = this.dom;\n      if (dom.box.parentNode) {\n        dom.box.parentNode.removeChild(dom.box);\n      }\n\n      if (this.options.showStipes) {\n        if (dom.line.parentNode)  {\n          dom.line.parentNode.removeChild(dom.line);\n        }\n        if (dom.dot.parentNode)  {\n          dom.dot.parentNode.removeChild(dom.dot);\n        }\n      }\n      this.displayed = false;\n    }\n  }\n  \n  /**\n   * reposition item x axis\n   */\n  repositionX() {\n    let start = this.conversion.toScreen(this.data.start);\n    let end = this.data.end ? this.conversion.toScreen(this.data.end) : 0;\n    if (end) {\n      this.repositionXWithRanges(start, end);\n    } else {\n      let align = this.data.align === undefined ? this.options.align : this.data.align;\n      this.repositionXWithoutRanges(start, align);\n    }\n  \n    if (this.options.showStipes) {\n      this.dom.line.style.display = this._isStipeVisible() ? 'block' : 'none';\n      this.dom.dot.style.display = this._isStipeVisible() ? 'block' : 'none';\n\n      if (this._isStipeVisible()) {\n        this.repositionStype(start, end);\n      }\n    }\n  }\n\n  /**\n   * reposition item stype\n   * @param {date} start\n   * @param {date} end\n   */\n  repositionStype(start, end) {\n    this.dom.line.style.display = 'block';\n    this.dom.dot.style.display = 'block';\n    const lineOffsetWidth = this.dom.line.offsetWidth;\n    const dotOffsetWidth = this.dom.dot.offsetWidth;\n\n    if (end) {\n      const lineOffset = lineOffsetWidth + start + (end - start) / 2;\n      const dotOffset = lineOffset - dotOffsetWidth / 2;\n      const lineOffsetDirection = this.options.rtl ? lineOffset * -1 : lineOffset;\n      const dotOffsetDirection = this.options.rtl ? dotOffset * -1 : dotOffset;\n\n      this.dom.line.style.transform = `translateX(${lineOffsetDirection}px)`;\n      this.dom.dot.style.transform = `translateX(${dotOffsetDirection}px)`;\n    } else {\n      const lineOffsetDirection = this.options.rtl ? (start * -1) : start;\n      const dotOffsetDirection = this.options.rtl ? ((start - dotOffsetWidth / 2) * -1) : (start - dotOffsetWidth / 2);\n\n      this.dom.line.style.transform = `translateX(${lineOffsetDirection}px)`;\n      this.dom.dot.style.transform = `translateX(${dotOffsetDirection}px)`;\n    }\n  }\n  \n  /**\n   * reposition x without ranges\n   * @param {date} start\n   * @param {string} align\n   */\n  repositionXWithoutRanges(start, align) {\n    // calculate left position of the box\n    if (align == 'right') {\n      if (this.options.rtl) {\n        this.right = start - this.width;\n  \n        // reposition box, line, and dot\n        this.dom.box.style.right = this.right + 'px';\n      } else {\n        this.left = start - this.width;\n  \n        // reposition box, line, and dot\n        this.dom.box.style.left = this.left + 'px';\n      }\n    } else if (align == 'left') {\n      if (this.options.rtl) {\n        this.right = start;\n  \n        // reposition box, line, and dot\n        this.dom.box.style.right = this.right + 'px';\n      } else {\n        this.left = start;\n  \n        // reposition box, line, and dot\n        this.dom.box.style.left = this.left + 'px';\n      }\n    } else {\n      // default or 'center'\n      if (this.options.rtl) {\n        this.right = start - this.width / 2;\n  \n        // reposition box, line, and dot\n        this.dom.box.style.right = this.right + 'px';\n      } else {\n        this.left = start - this.width / 2;\n  \n        // reposition box, line, and dot\n        this.dom.box.style.left = this.left + 'px';\n      }\n    }\n  }\n  \n  /**\n   * reposition x with ranges\n   * @param {date} start\n   * @param {date} end\n   */\n  repositionXWithRanges(start, end) {\n    let boxWidth = Math.round(Math.max(end - start + 0.5, 1));\n  \n    if (this.options.rtl) {\n      this.right = start;\n    } else {\n      this.left = start;\n    }\n  \n    this.width = Math.max(boxWidth, this.minWidth || 0);\n  \n    if (this.options.rtl) {\n      this.dom.box.style.right = this.right + 'px';\n    } else {\n      this.dom.box.style.left = this.left + 'px';\n    }\n  \n    this.dom.box.style.width = boxWidth + 'px';\n  }\n  \n  /**\n   * reposition item y axis\n   */\n  repositionY() {\n    var orientation = this.options.orientation.item;\n    var box = this.dom.box;\n    if (orientation == 'top') {\n      box.style.top = (this.top || 0) + 'px';\n    } else { \n      // orientation 'bottom'\n      box.style.top = (this.parent.height - this.top - this.height || 0) + 'px';\n    }\n  \n    if (this.options.showStipes) {\n      if (orientation == 'top') {\n        this.dom.line.style.top    = '0';\n        this.dom.line.style.height = (this.parent.top + this.top + 1) + 'px';\n        this.dom.line.style.bottom = '';\n      }\n      else { \n        // orientation 'bottom'\n        var itemSetHeight = this.parent.itemSet.props.height;\n        var lineHeight = itemSetHeight - this.parent.top - this.parent.height + this.top;\n        this.dom.line.style.top    = (itemSetHeight - lineHeight) + 'px';\n        this.dom.line.style.bottom = '0';\n      }\n    \n      this.dom.dot.style.top = (-this.dom.dot.offsetHeight / 2) + 'px';\n    }\n  }\n\n  /**\n   * get width left\n   * @return {number}\n   */\n  getWidthLeft() {\n    return this.width / 2;\n  }\n  \n  /**\n   * get width right\n   * @return {number}\n   */\n  getWidthRight() {\n    return this.width / 2;\n  }\n  \n  /**\n   * move cluster item\n   */\n  move() {\n    this.repositionX();\n    this.repositionY();\n  }\n  \n  /**\n   * attach\n   */\n  attach() {\n    for (let item of this.data.uiItems) {\n      item.cluster = this;\n    }\n  \n    this.data.items = this.data.uiItems.map(item => item.data);\n\n    this.attached = true;\n    this.dirty = true;\n  }\n  \n  /**\n   * detach\n   * @param {boolean} detachFromParent\n   * @return {void}\n   */\n  detach(detachFromParent = false) {\n    if (!this.hasItems()) {\n      return;\n    }\n  \n    for (let item of this.data.uiItems) {\n      delete item.cluster;\n    }\n  \n    this.attached = false;\n  \n    if (detachFromParent && this.group) {\n      this.group.remove(this);\n      this.group = null;\n    }\n  \n    this.data.items = [];\n    this.dirty = true;\n  }\n  \n  /**\n   * handle on double click\n   */\n  _onDoubleClick() {\n   this._fit();\n  }\n  \n  /**\n   * set range\n   */\n  _setupRange() {\n    const stats = this.data.uiItems.map(item => ({\n      start: item.data.start.valueOf(),\n      end: item.data.end ? item.data.end.valueOf() : item.data.start.valueOf(),\n    }));\n  \n    this.data.min = Math.min(...stats.map(s => Math.min(s.start, s.end || s.start)));\n    this.data.max = Math.max(...stats.map(s => Math.max(s.start, s.end || s.start)));\n    const centers = this.data.uiItems.map(item => item.center);\n    const avg = centers.reduce((sum, value) => sum + value, 0) / this.data.uiItems.length;\n  \n    if (this.data.uiItems.some(item => item.data.end)) {\n      // contains ranges\n      this.data.start = new Date(this.data.min);\n      this.data.end = new Date(this.data.max);\n    } else {\n      this.data.start = new Date(avg);\n      this.data.end = null;\n    }\n  }\n  \n  /**\n   * get UI items\n   * @return {array}\n   */\n  _getUiItems() {\n    if (this.data.uiItems && this.data.uiItems.length) {\n      return this.data.uiItems.filter(item => item.cluster === this);\n    }\n  \n    return [];\n  }\n  \n  /**\n   * create DOM element\n   */\n  _createDomElement() {\n    if (!this.dom) {\n      // create DOM\n      this.dom = {};\n  \n      // create main box\n      this.dom.box = document.createElement('DIV');\n  \n      // contents box (inside the background box). used for making margins\n      this.dom.content = document.createElement('DIV');\n      this.dom.content.className = 'vis-item-content';\n      \n      this.dom.box.appendChild(this.dom.content);\n  \n      if (this.options.showStipes) {\n        // line to axis\n        this.dom.line = document.createElement('DIV');\n        this.dom.line.className = 'vis-cluster-line';\n        this.dom.line.style.display = 'none';\n  \n        // dot on axis\n        this.dom.dot = document.createElement('DIV');\n        this.dom.dot.className = 'vis-cluster-dot';\n        this.dom.dot.style.display = 'none';\n      }\n  \n      if (this.options.fitOnDoubleClick) {\n        this.dom.box.ondblclick = ClusterItem.prototype._onDoubleClick.bind(this);\n      }\n      \n      // attach this item as attribute\n      this.dom.box['vis-item'] = this;\n  \n      this.dirty = true;\n    }\n  }\n  \n  /**\n   * append element to DOM\n   */\n  _appendDomElement() {\n    if (!this.parent) {\n      throw new Error('Cannot redraw item: no parent attached');\n    }\n  \n    if (!this.dom.box.parentNode) {\n      const foreground = this.parent.dom.foreground;\n      if (!foreground) {\n        throw new Error('Cannot redraw item: parent has no foreground container element');\n      }\n  \n      foreground.appendChild(this.dom.box);\n    }\n  \n    const background = this.parent.dom.background;\n  \n    if (this.options.showStipes) {\n      if (!this.dom.line.parentNode) {\n        if (!background) throw new Error('Cannot redraw item: parent has no background container element');\n        background.appendChild(this.dom.line);\n      }\n    \n      if (!this.dom.dot.parentNode) {\n        var axis = this.parent.dom.axis;\n        if (!background) throw new Error('Cannot redraw item: parent has no axis container element');\n        axis.appendChild(this.dom.dot);\n      }\n    }\n  \n    this.displayed = true;\n  }\n  \n  /**\n   * update dirty DOM components\n   */\n  _updateDirtyDomComponents() {\n    // An item is marked dirty when:\n    // - the item is not yet rendered\n    // - the item's data is changed\n    // - the item is selected/deselected\n    if (this.dirty) {\n      this._updateContents(this.dom.content);\n      this._updateDataAttributes(this.dom.box);\n      this._updateStyle(this.dom.box);\n  \n      // update class\n      const className = this.baseClassName + ' ' + (this.data.className ? ' ' + this.data.className : '') +\n        (this.selected ? ' vis-selected' : '') + ' vis-readonly';\n      this.dom.box.className = 'vis-item ' + className;\n  \n      if (this.options.showStipes) {\n        this.dom.line.className = 'vis-item vis-cluster-line ' +  (this.selected ? ' vis-selected' : '');\n        this.dom.dot.className  = 'vis-item vis-cluster-dot ' +  (this.selected ? ' vis-selected' : '');\n      }\n  \n      if (this.data.end) {\n        // turn off max-width to be able to calculate the real width\n        // this causes an extra browser repaint/reflow, but so be it\n        this.dom.content.style.maxWidth = 'none';\n      }\n    }\n  }\n  \n  /**\n   * get DOM components sizes\n   * @return {object}\n   */\n  _getDomComponentsSizes() {\n    const sizes = {\n      previous: {\n        right: this.dom.box.style.right,\n        left: this.dom.box.style.left\n      },\n      box: {\n        width: this.dom.box.offsetWidth,\n        height: this.dom.box.offsetHeight\n      },\n    };\n\n    if (this.options.showStipes) {\n      sizes.dot = {\n        height: this.dom.dot.offsetHeight,\n        width: this.dom.dot.offsetWidth\n      };\n      sizes.line = {\n        width: this.dom.line.offsetWidth\n      };\n    }\n\n    return sizes;\n  }\n  \n  /**\n   * update DOM components sizes\n   * @param {object} sizes\n   */\n  _updateDomComponentsSizes(sizes) {\n    if (this.options.rtl) {\n      this.dom.box.style.right = \"0px\";\n    } else {\n      this.dom.box.style.left = \"0px\";\n    }\n  \n    // recalculate size\n    if (!this.data.end) {\n      this.width = sizes.box.width;\n    } else {\n      this.minWidth = sizes.box.width;\n    }\n  \n    this.height = sizes.box.height;\n  \n    // restore previous position\n    if (this.options.rtl) {\n      this.dom.box.style.right = sizes.previous.right;\n    } else {\n      this.dom.box.style.left = sizes.previous.left;\n    }\n  \n    this.dirty = false;\n  }\n  \n  /**\n   * repaint DOM additional components\n   */\n  _repaintDomAdditionals() {\n    this._repaintOnItemUpdateTimeTooltip(this.dom.box);\n  }\n  \n  /**\n   * check is stripe visible\n   * @return {number}\n   * @private\n   */\n  _isStipeVisible() {\n    return this.minWidth >= this.width || !this.data.end;\n  }\n  \n  /**\n   * get fit range\n   * @return {object}\n   * @private\n   */\n  _getFitRange() {\n    const offset = 0.05*(this.data.max - this.data.min) / 2;\n      return {\n        fitStart: this.data.min - offset,\n        fitEnd: this.data.max + offset\n      };\n  }\n  \n   /**\n   * fit\n   * @private\n   */\n  _fit() {\n    if (this.emitter) {\n      const {fitStart, fitEnd} = this._getFitRange();\n  \n      const fitArgs = {\n        start: new Date(fitStart),\n        end: new Date(fitEnd),\n        animation: true\n      };\n  \n      this.emitter.emit('fit', fitArgs);\n    }\n  }\n\n   /**\n   * get item data\n   * @return {object}\n   * @private\n   */\n  _getItemData() {\n    return this.data;\n  }\n}\n\nClusterItem.prototype.baseClassName = 'vis-item vis-range vis-cluster';\nexport default ClusterItem;\n","import ClusterItem from './item/ClusterItem';\n\nconst UNGROUPED = '__ungrouped__';   // reserved group id for ungrouped items\nconst BACKGROUND = '__background__'; // reserved group id for background items without group\n\nexport const ReservedGroupIds = {\n  UNGROUPED,\n  BACKGROUND\n}\n\n/**\n * An Cluster generator generates cluster items\n */\nexport default class ClusterGenerator {\n    /**\n     * @param {ItemSet} itemSet itemsSet instance\n     * @constructor ClusterGenerator\n    */\n    constructor(itemSet) {\n        this.itemSet = itemSet;\n        this.groups = {};\n        this.cache = {};\n        this.cache[-1] = [];\n    }\n\n    /**\n     * @param {Object} itemData             Object containing parameters start content, className.\n     * @param {{toScreen: function, toTime: function}} conversion\n     *                                  Conversion functions from time to screen and vice versa\n     * @param {Object} [options]        Configuration options\n     * @return {Object} newItem\n    */\n    createClusterItem(itemData, conversion, options) {\n        const newItem = new ClusterItem(itemData, conversion, options);\n        return newItem;\n    }\n\n    /**\n     * Set the items to be clustered.\n     * This will clear cached clusters.\n     * @param {Item[]} items\n     * @param {Object} [options]  Available options:\n     *                            {boolean} applyOnChangedLevel\n     *                                If true (default), the changed data is applied\n     *                                as soon the cluster level changes. If false,\n     *                                The changed data is applied immediately\n     */\n    setItems(items, options) {\n        this.items = items || [];\n        this.dataChanged = true;\n        this.applyOnChangedLevel = false;\n\n        if (options && options.applyOnChangedLevel) {\n            this.applyOnChangedLevel = options.applyOnChangedLevel;\n        }\n    }\n\n    /**\n     * Update the current data set: clear cache, and recalculate the clustering for\n     * the current level\n     */\n    updateData() {\n        this.dataChanged = true;\n        this.applyOnChangedLevel = false;\n    }\n\n    /**\n     * Cluster the items which are too close together\n     * @param {array} oldClusters \n     * @param {number} scale      The scale of the current window : (windowWidth / (endDate - startDate)) \n     * @param {{maxItems: number, clusterCriteria: function, titleTemplate: string}} options             \n     * @return {array} clusters\n    */\n    getClusters(oldClusters, scale, options) {\n        let { maxItems, clusterCriteria } = typeof options === \"boolean\" ? {} : options;\n    \n        if (!clusterCriteria) {\n            clusterCriteria = () => true;\n        }\n\n        maxItems = maxItems || 1;\n\n        let level = -1;\n        let granularity = 2;\n        let timeWindow = 0;\n\n        if (scale > 0) {\n            if (scale >= 1) {\n                return [];\n            }\n\n            level = Math.abs(Math.round(Math.log(100 / scale) / Math.log(granularity)));\n            timeWindow = Math.abs(Math.pow(granularity, level));\n        }\n\n        // clear the cache when and re-generate groups the data when needed.\n        if (this.dataChanged) {\n            const levelChanged = (level != this.cacheLevel);\n            const applyDataNow = this.applyOnChangedLevel ? levelChanged : true;\n            if (applyDataNow) {\n                this._dropLevelsCache();\n                this._filterData();\n            }\n        }\n\n        this.cacheLevel = level;\n        let clusters = this.cache[level];\n        if (!clusters) {\n            clusters = [];\n            for (let groupName in this.groups) {\n                if (this.groups.hasOwnProperty(groupName)) {\n                    const items = this.groups[groupName];\n                    const iMax = items.length;\n                    let i = 0;\n                    while (i < iMax) {\n                        // find all items around current item, within the timeWindow\n                        let item = items[i];\n                        let neighbors = 1; // start at 1, to include itself)\n\n                        // loop through items left from the current item\n                        let j = i - 1;\n                        while (j >= 0 && (item.center - items[j].center) < timeWindow / 2) {\n                            if (!items[j].cluster && clusterCriteria(item.data, items[j].data)) {\n                                neighbors++;\n                            }\n                            j--;\n                        }\n\n                        // loop through items right from the current item\n                        let k = i + 1;\n                        while (k < items.length && (items[k].center - item.center) < timeWindow / 2) {\n                            if (clusterCriteria(item.data, items[k].data)) {\n                                neighbors++;\n                            }\n                            k++;\n                        }\n\n                        // loop through the created clusters\n                        let l = clusters.length - 1;\n                        while (l >= 0 && (item.center - clusters[l].center) < timeWindow) {\n                            if (item.group == clusters[l].group && clusterCriteria(item.data, clusters[l].data)) {\n                                neighbors++;\n                            }\n                            l--;\n                        }\n\n                        // aggregate until the number of items is within maxItems\n                        if (neighbors > maxItems) {\n                            // too busy in this window.\n                            const num = neighbors - maxItems + 1;\n                            const clusterItems = [];\n\n                            // append the items to the cluster,\n                            // and calculate the average start for the cluster\n                            let m = i;\n                            while (clusterItems.length < num && m < items.length) {\n                                if (clusterCriteria(items[m].data, items[m].data)) {\n                                    clusterItems.push(items[m]);\n                                }\n                                m++;\n                            }\n\n                            const groupId = this.itemSet.getGroupId(item.data);\n                            const group = this.itemSet.groups[groupId] || this.itemSet.groups[ReservedGroupIds.UNGROUPED];\n                            let cluster = this._getClusterForItems(clusterItems, group, oldClusters, options);\n                            clusters.push(cluster);\n\n                            i += num;\n                        } else {\n                            delete item.cluster;\n                            i += 1;\n                        }\n                    }\n                }\n            }\n\n            this.cache[level] = clusters;\n        }\n\n        return clusters;\n    }\n\n    /**\n     * Filter the items per group.\n     * @private\n     */\n    _filterData() {\n        // filter per group\n        const groups = {};\n        this.groups = groups;\n\n        // split the items per group\n        for (const item of Object.values(this.items)) {\n            // put the item in the correct group\n            const groupName = item.parent ? item.parent.groupId : '';\n            let group = groups[groupName];\n            if (!group) {\n                group = [];\n                groups[groupName] = group;\n            }\n            group.push(item);\n\n            // calculate the center of the item\n            if (item.data.start) {\n                if (item.data.end) {\n                    // range\n                    item.center = (item.data.start.valueOf() + item.data.end.valueOf()) / 2;\n                } else {\n                    // box, dot\n                    item.center = item.data.start.valueOf();\n                }\n            }\n        }\n\n        // sort the items per group\n        for (let currentGroupName in groups) {\n            if (groups.hasOwnProperty(currentGroupName)) {\n                groups[currentGroupName].sort((a, b) => a.center - b.center);\n            }\n        }\n\n        this.dataChanged = false;\n    }\n\n    /**\n     * Create new cluster or return existing\n     * @private\n     * @param {array} clusterItems    \n     * @param {object} group \n     * @param {array} oldClusters \n     * @param {object} options \n     * @returns {object} cluster\n     */\n    _getClusterForItems(clusterItems, group, oldClusters, options) {\n        const oldClustersLookup = (oldClusters || []).map(cluster => ({\n            cluster,\n            itemsIds: new Set(cluster.data.uiItems.map(item => item.id))\n        }));\n        let cluster;\n        if (oldClustersLookup.length) {\n            for (let oldClusterData of oldClustersLookup) {\n                if (oldClusterData.itemsIds.size === clusterItems.length \n                    && clusterItems.every(clusterItem => oldClusterData.itemsIds.has(clusterItem.id))) {\n                    cluster = oldClusterData.cluster;\n                    break;\n                }\n            }\n        }\n\n        if (cluster) {\n            cluster.setUiItems(clusterItems);\n            if (cluster.group !== group) {\n                if (cluster.group) {\n                    cluster.group.remove(cluster);    \n                }\n\n                if (group) {\n                    group.add(cluster);    \n                    cluster.group = group;\n                }\n            }\n            return cluster;\n        }\n\n        let titleTemplate = options.titleTemplate || '';\n        const conversion = {\n            toScreen: this.itemSet.body.util.toScreen,\n            toTime: this.itemSet.body.util.toTime\n        };\n\n        const title = titleTemplate.replace(/{count}/, clusterItems.length);\n        const clusterContent = '<div title=\"' + title + '\">' + clusterItems.length + '</div>';\n        const clusterOptions = Object.assign({}, options, this.itemSet.options);\n        const data = {\n            'content': clusterContent,\n            'title': title,\n            'group': group,\n            'uiItems': clusterItems,\n            'eventEmitter': this.itemSet.body.emitter,\n            'range': this.itemSet.body.range\n        };\n        cluster = this.createClusterItem(data,\n        conversion,\n        clusterOptions);\n\n        if (group) {\n            group.add(cluster);\n            cluster.group = group;\n        }\n\n        cluster.attach();\n\n        return cluster;\n    }\n\n    /**\n     * Drop cache\n     * @private\n     */\n    _dropLevelsCache() {\n        this.cache = {};\n        this.cacheLevel = -1;\n        this.cache[this.cacheLevel] = [];\n    }\n}\n","import { v4 as randomUUID } from \"uuid\";\nimport Hammer from '../../module/hammer';\nimport util, { typeCoerceDataSet } from '../../util';\nimport { DataSet }  from 'vis-data/esnext';\nimport { DataView } from 'vis-data/esnext';\nimport TimeStep from '../TimeStep';\nimport Component from './Component';\nimport Group from './Group';\nimport BackgroundGroup from './BackgroundGroup';\nimport BoxItem from './item/BoxItem';\nimport PointItem from './item/PointItem';\nimport RangeItem from './item/RangeItem';\nimport BackgroundItem from './item/BackgroundItem';\nimport Popup from '../../shared/Popup';\nimport ClusterGenerator from './ClusterGenerator';\n\nimport './css/itemset.css';\nimport './css/labelset.css';\n\nconst UNGROUPED = '__ungrouped__';   // reserved group id for ungrouped items\nconst BACKGROUND = '__background__'; // reserved group id for background items without group\n\nexport const ReservedGroupIds = {\n  UNGROUPED,\n  BACKGROUND\n}\n\n/**\n * An ItemSet holds a set of items and ranges which can be displayed in a\n * range. The width is determined by the parent of the ItemSet, and the height\n * is determined by the size of the items.\n */\nclass ItemSet extends Component {\n  /**\n * @param {{dom: Object, domProps: Object, emitter: Emitter, range: Range}} body\n * @param {Object} [options]      See ItemSet.setOptions for the available options.\n * @constructor ItemSet\n * @extends Component\n */\n  constructor(body, options) {\n    super()\n    this.body = body;\n    this.defaultOptions = {\n      type: null,  // 'box', 'point', 'range', 'background'\n      orientation: {\n        item: 'bottom'   // item orientation: 'top' or 'bottom'\n      },\n      align: 'auto', // alignment of box items\n      stack: true,\n      stackSubgroups: true,\n      groupOrderSwap(fromGroup, toGroup, groups) {  // eslint-disable-line no-unused-vars\n        const targetOrder = toGroup.order;\n        toGroup.order = fromGroup.order;\n        fromGroup.order = targetOrder;\n      },\n      groupOrder: 'order',\n\n      selectable: true,\n      multiselect: false,\n      longSelectPressTime: 251,\n      itemsAlwaysDraggable: {\n        item: false,\n        range: false,\n      },\n\n      editable: {\n        updateTime: false,\n        updateGroup: false,\n        add: false,\n        remove: false,\n        overrideItems: false\n      },\n\n      groupEditable: {\n        order: false,\n        add: false,\n        remove: false\n      },\n\n      snap: TimeStep.snap,\n\n      // Only called when `objectData.target === 'item'.\n      onDropObjectOnItem(objectData, item, callback) {\n        callback(item)\n      },\n      onAdd(item, callback) {\n        callback(item);\n      },\n      onUpdate(item, callback) {\n        callback(item);\n      },\n      onMove(item, callback) {\n        callback(item);\n      },\n      onRemove(item, callback) {\n        callback(item);\n      },\n      onMoving(item, callback) {\n        callback(item);\n      },\n      onAddGroup(item, callback) {\n        callback(item);\n      },\n      onMoveGroup(item, callback) {\n        callback(item);\n      },\n      onRemoveGroup(item, callback) {\n        callback(item);\n      },\n\n      margin: {\n        item: {\n          horizontal: 10,\n          vertical: 10\n        },\n        axis: 20\n      },\n\n      showTooltips: true,\n\n      tooltip: {\n        followMouse: false,\n        overflowMethod: 'flip',\n        delay: 500\n      },\n\n      tooltipOnItemUpdateTime: false\n    };\n\n    // options is shared by this ItemSet and all its items\n    this.options = util.extend({}, this.defaultOptions);\n    this.options.rtl = options.rtl;\n    this.options.onTimeout = options.onTimeout;\n\n    this.conversion = {\n      toScreen: body.util.toScreen,\n      toTime: body.util.toTime\n    };\n    this.dom = {};\n    this.props = {};\n    this.hammer = null;\n    \n    const me = this;\n    this.itemsData = null;    // DataSet\n    this.groupsData = null;   // DataSet\n    this.itemsSettingTime = null;\n    this.initialItemSetDrawn = false;\n    this.userContinueNotBail = null;  \n\n    this.sequentialSelection = false;\n    \n    // listeners for the DataSet of the items\n    this.itemListeners = {\n      'add'(event, params, senderId) {  // eslint-disable-line no-unused-vars\n        me._onAdd(params.items);\n        if (me.options.cluster) {\n          me.clusterGenerator.setItems(me.items, { applyOnChangedLevel: false });\n        }\n        me.redraw();\n      },\n      'update'(event, params, senderId) {  // eslint-disable-line no-unused-vars\n        me._onUpdate(params.items);\n        if (me.options.cluster) {\n          me.clusterGenerator.setItems(me.items, { applyOnChangedLevel: false });\n        }\n        me.redraw();\n      },\n      'remove'(event, params, senderId) {  // eslint-disable-line no-unused-vars\n        me._onRemove(params.items);\n        if (me.options.cluster) {\n          me.clusterGenerator.setItems(me.items, { applyOnChangedLevel: false });\n        }\n        me.redraw();\n      }\n    };\n\n    // listeners for the DataSet of the groups\n    this.groupListeners = {\n      'add'(event, params, senderId) {  // eslint-disable-line no-unused-vars\n        me._onAddGroups(params.items);\n        \n        if (me.groupsData && me.groupsData.length > 0) {\n            const groupsData = me.groupsData.getDataSet();\n            groupsData.get().forEach(groupData => {\n            if (groupData.nestedGroups) {\n              if (groupData.showNested != false) {\n                groupData.showNested = true;\n              }\n              let updatedGroups = [];\n              groupData.nestedGroups.forEach(nestedGroupId => {\n                const updatedNestedGroup = groupsData.get(nestedGroupId);\n                if (!updatedNestedGroup) { return; }\n                updatedNestedGroup.nestedInGroup = groupData.id;\n                if (groupData.showNested == false) {\n                  updatedNestedGroup.visible = false;\n                }\n                updatedGroups = updatedGroups.concat(updatedNestedGroup);\n              });\n              groupsData.update(updatedGroups, senderId);\n            }\n          });\n        }\n      },\n      'update'(event, params, senderId) {  // eslint-disable-line no-unused-vars\n        me._onUpdateGroups(params.items);\n      },\n      'remove'(event, params, senderId) {  // eslint-disable-line no-unused-vars\n        me._onRemoveGroups(params.items);\n      }\n    };\n\n    this.items = {};      // object with an Item for every data item\n    this.groups = {};     // Group object for every group\n    this.groupIds = [];\n\n    this.selection = [];  // list with the ids of all selected nodes\n\n    this.popup = null;\n    this.popupTimer = null;\n\n    this.touchParams = {}; // stores properties while dragging\n    this.groupTouchParams = {\n      group: null,\n      isDragging: false\n    };\n  \n    // create the HTML DOM\n    this._create();\n\n    this.setOptions(options);\n    this.clusters = [];\n  }\n\n  /**\n   * Create the HTML DOM for the ItemSet\n   */\n  _create() {\n    const frame = document.createElement('div');\n    frame.className = 'vis-itemset';\n    frame['vis-itemset'] = this;\n    this.dom.frame = frame;\n\n    // create background panel\n    const background = document.createElement('div');\n    background.className = 'vis-background';\n    frame.appendChild(background);\n    this.dom.background = background;\n\n    // create foreground panel\n    const foreground = document.createElement('div');\n    foreground.className = 'vis-foreground';\n    frame.appendChild(foreground);\n    this.dom.foreground = foreground;\n\n    // create axis panel\n    const axis = document.createElement('div');\n    axis.className = 'vis-axis';\n    this.dom.axis = axis;\n\n    // create labelset\n    const labelSet = document.createElement('div');\n    labelSet.className = 'vis-labelset';\n    this.dom.labelSet = labelSet;\n\n    // create ungrouped Group\n    this._updateUngrouped();\n\n    // create background Group\n    const backgroundGroup = new BackgroundGroup(BACKGROUND, null, this);\n    backgroundGroup.show();\n    this.groups[BACKGROUND] = backgroundGroup;\n\n    // attach event listeners\n    // Note: we bind to the centerContainer for the case where the height\n    //       of the center container is larger than of the ItemSet, so we\n    //       can click in the empty area to create a new item or deselect an item.\n    this.hammer = new Hammer(this.body.dom.centerContainer);\n\n    // drag items when selected\n    this.hammer.on('hammer.input', event => {\n      if (event.isFirst) {\n        this._onTouch(event);\n      }\n    });\n    this.hammer.on('panstart', this._onDragStart.bind(this));\n    this.hammer.on('panmove',  this._onDrag.bind(this));\n    this.hammer.on('panend',   this._onDragEnd.bind(this));\n    this.hammer.get('pan').set({threshold:5, direction: Hammer.ALL});\n\n    // single select (or unselect) when tapping an item\n    this.hammer.on('tap',  this._onSelectItem.bind(this));\n\n    // multi select when holding mouse/touch, or on ctrl+click\n    this.hammer.on('press', this._onMultiSelectItem.bind(this));\n\n    // add item on doubletap\n    this.hammer.on('doubletap', this._onAddItem.bind(this));\n\n    if (this.options.rtl) {\n      this.groupHammer = new Hammer(this.body.dom.rightContainer);\n    } else {\n      this.groupHammer = new Hammer(this.body.dom.leftContainer);\n    }\n    \n    this.groupHammer.on('tap',      this._onGroupClick.bind(this));\n    this.groupHammer.on('panstart', this._onGroupDragStart.bind(this));\n    this.groupHammer.on('panmove',  this._onGroupDrag.bind(this));\n    this.groupHammer.on('panend',   this._onGroupDragEnd.bind(this));\n    this.groupHammer.get('pan').set({threshold:5, direction: Hammer.DIRECTION_VERTICAL});\n    \n    this.body.dom.centerContainer.addEventListener('mouseover', this._onMouseOver.bind(this));\n    this.body.dom.centerContainer.addEventListener('mouseout', this._onMouseOut.bind(this));\n    this.body.dom.centerContainer.addEventListener('mousemove', this._onMouseMove.bind(this));\n    // right-click on timeline \n    this.body.dom.centerContainer.addEventListener('contextmenu', this._onDragEnd.bind(this));\n\n    this.body.dom.centerContainer.addEventListener('mousewheel', this._onMouseWheel.bind(this));\n\n    // attach to the DOM\n    this.show();\n  }\n\n  /**\n   * Set options for the ItemSet. Existing options will be extended/overwritten.\n   * @param {Object} [options] The following options are available:\n   *                           {string} type\n   *                              Default type for the items. Choose from 'box'\n   *                              (default), 'point', 'range', or 'background'.\n   *                              The default style can be overwritten by\n   *                              individual items.\n   *                           {string} align\n   *                              Alignment for the items, only applicable for\n   *                              BoxItem. Choose 'center' (default), 'left', or\n   *                              'right'.\n   *                           {string} orientation.item\n   *                              Orientation of the item set. Choose 'top' or\n   *                              'bottom' (default).\n   *                           {Function} groupOrder\n   *                              A sorting function for ordering groups\n   *                           {boolean} stack\n   *                              If true (default), items will be stacked on\n   *                              top of each other.\n   *                           {number} margin.axis\n   *                              Margin between the axis and the items in pixels.\n   *                              Default is 20.\n   *                           {number} margin.item.horizontal\n   *                              Horizontal margin between items in pixels.\n   *                              Default is 10.\n   *                           {number} margin.item.vertical\n   *                              Vertical Margin between items in pixels.\n   *                              Default is 10.\n   *                           {number} margin.item\n   *                              Margin between items in pixels in both horizontal\n   *                              and vertical direction. Default is 10.\n   *                           {number} margin\n   *                              Set margin for both axis and items in pixels.\n   *                           {boolean} selectable\n   *                              If true (default), items can be selected.\n   *                           {boolean} multiselect\n   *                              If true, multiple items can be selected.\n   *                              False by default.\n   *                           {boolean} editable\n   *                              Set all editable options to true or false\n   *                           {boolean} editable.updateTime\n   *                              Allow dragging an item to an other moment in time\n   *                           {boolean} editable.updateGroup\n   *                              Allow dragging an item to an other group\n   *                           {boolean} editable.add\n   *                              Allow creating new items on double tap\n   *                           {boolean} editable.remove\n   *                              Allow removing items by clicking the delete button\n   *                              top right of a selected item.\n   *                           {Function(item: Item, callback: Function)} onAdd\n   *                              Callback function triggered when an item is about to be added:\n   *                              when the user double taps an empty space in the Timeline.\n   *                           {Function(item: Item, callback: Function)} onUpdate\n   *                              Callback function fired when an item is about to be updated.\n   *                              This function typically has to show a dialog where the user\n   *                              change the item. If not implemented, nothing happens.\n   *                           {Function(item: Item, callback: Function)} onMove\n   *                              Fired when an item has been moved. If not implemented,\n   *                              the move action will be accepted.\n   *                           {Function(item: Item, callback: Function)} onRemove\n   *                              Fired when an item is about to be deleted.\n   *                              If not implemented, the item will be always removed.\n   */\n  setOptions(options) {\n    if (options) {\n      // copy all options that we know\n      const fields = [\n        'type', 'rtl', 'align', 'order', 'stack', 'stackSubgroups', 'selectable', 'multiselect', 'sequentialSelection',\n        'multiselectPerGroup', 'longSelectPressTime', 'groupOrder', 'dataAttributes', 'template', 'groupTemplate', 'visibleFrameTemplate',\n        'hide', 'snap', 'groupOrderSwap', 'showTooltips', 'tooltip', 'tooltipOnItemUpdateTime', 'groupHeightMode', 'onTimeout'\n      ];\n      util.selectiveExtend(fields, this.options, options);\n\n      if ('itemsAlwaysDraggable' in options) {\n        if (typeof options.itemsAlwaysDraggable === 'boolean') {\n          this.options.itemsAlwaysDraggable.item = options.itemsAlwaysDraggable;\n          this.options.itemsAlwaysDraggable.range = false;\n        }\n        else if (typeof options.itemsAlwaysDraggable === 'object') {\n          util.selectiveExtend(['item', 'range'], this.options.itemsAlwaysDraggable, options.itemsAlwaysDraggable);\n          // only allow range always draggable when item is always draggable as well\n          if (! this.options.itemsAlwaysDraggable.item) {\n            this.options.itemsAlwaysDraggable.range = false;\n          }\n        }\n      }\n\n      if ('sequentialSelection' in options) {\n        if (typeof options.sequentialSelection === 'boolean') {\n          this.options.sequentialSelection = options.sequentialSelection;\n        }\n      }\n\n      if ('orientation' in options) {\n        if (typeof options.orientation === 'string') {\n          this.options.orientation.item = options.orientation === 'top' ? 'top' : 'bottom';\n        }\n        else if (typeof options.orientation === 'object' && 'item' in options.orientation) {\n          this.options.orientation.item = options.orientation.item;\n        }\n      }\n\n      if ('margin' in options) {\n        if (typeof options.margin === 'number') {\n          this.options.margin.axis = options.margin;\n          this.options.margin.item.horizontal = options.margin;\n          this.options.margin.item.vertical = options.margin;\n        }\n        else if (typeof options.margin === 'object') {\n          util.selectiveExtend(['axis'], this.options.margin, options.margin);\n          if ('item' in options.margin) {\n            if (typeof options.margin.item === 'number') {\n              this.options.margin.item.horizontal = options.margin.item;\n              this.options.margin.item.vertical = options.margin.item;\n            }\n            else if (typeof options.margin.item === 'object') {\n              util.selectiveExtend(['horizontal', 'vertical'], this.options.margin.item, options.margin.item);\n            }\n          }\n        }\n      }\n\n      ['locale', 'locales'].forEach(key => {\n        if (key in options) {\n          this.options[key] = options[key];\n        }\n      });\n\n      if ('editable' in options) {\n        if (typeof options.editable === 'boolean') {\n          this.options.editable.updateTime    = options.editable;\n          this.options.editable.updateGroup   = options.editable;\n          this.options.editable.add           = options.editable;\n          this.options.editable.remove        = options.editable;\n          this.options.editable.overrideItems = false;\n        }\n        else if (typeof options.editable === 'object') {\n          util.selectiveExtend(['updateTime', 'updateGroup', 'add', 'remove', 'overrideItems'], this.options.editable, options.editable);\n        }\n      }\n\n      if ('groupEditable' in options) {\n        if (typeof options.groupEditable === 'boolean') {\n          this.options.groupEditable.order  = options.groupEditable;\n          this.options.groupEditable.add    = options.groupEditable;\n          this.options.groupEditable.remove = options.groupEditable;\n        }\n        else if (typeof options.groupEditable === 'object') {\n          util.selectiveExtend(['order', 'add', 'remove'], this.options.groupEditable, options.groupEditable);\n        }\n      }\n\n      // callback functions\n      const addCallback = name => {\n        const fn = options[name];\n        if (fn) {\n          if (!(typeof fn === 'function')) {\n            throw new Error(`option ${name} must be a function ${name}(item, callback)`);\n          }\n          this.options[name] = fn;\n        }\n      };\n      ['onDropObjectOnItem', 'onAdd', 'onUpdate', 'onRemove', 'onMove', 'onMoving', 'onAddGroup', 'onMoveGroup', 'onRemoveGroup'].forEach(addCallback);\n\n      if (options.cluster) {\n        Object.assign(this.options, {\n          cluster: options.cluster\n        });\n        if (!this.clusterGenerator) {\n          this.clusterGenerator = new ClusterGenerator(this);\n        } \n        this.clusterGenerator.setItems(this.items, { applyOnChangedLevel: false });\n        this.markDirty({ refreshItems: true, restackGroups: true });\n\n        this.redraw();\n      } else if (this.clusterGenerator) {\n        this._detachAllClusters();\n        this.clusters = [];\n        this.clusterGenerator = null;\n        this.options.cluster = undefined;\n        this.markDirty({ refreshItems: true, restackGroups: true });\n\n        this.redraw();\n      } else {\n        // force the itemSet to refresh: options like orientation and margins may be changed\n        this.markDirty();\n      }\n    }\n  }\n\n  /**\n   * Mark the ItemSet dirty so it will refresh everything with next redraw.\n   * Optionally, all items can be marked as dirty and be refreshed.\n   * @param {{refreshItems: boolean}} [options]\n   */\n  markDirty(options) {\n    this.groupIds = [];\n\n    if (options) {\n      if (options.refreshItems) {\n        util.forEach(this.items, item => {\n          item.dirty = true;\n          if (item.displayed) item.redraw();\n        });\n      }\n      \n      if (options.restackGroups) {\n        util.forEach(this.groups, (group, key) => {\n          if (key === BACKGROUND) return;\n          group.stackDirty = true;\n        });\n      }\n    }\n  }\n\n  /**\n   * Destroy the ItemSet\n   */\n  destroy() {\n    this.clearPopupTimer();\n    this.hide();\n    this.setItems(null);\n    this.setGroups(null);\n\n    this.hammer && this.hammer.destroy();\n    this.groupHammer && this.groupHammer.destroy();\n    this.hammer = null;\n\n    this.body = null;\n    this.conversion = null;\n  }\n\n  /**\n   * Hide the component from the DOM\n   */\n  hide() {\n    // remove the frame containing the items\n    if (this.dom.frame.parentNode) {\n      this.dom.frame.parentNode.removeChild(this.dom.frame);\n    }\n\n    // remove the axis with dots\n    if (this.dom.axis.parentNode) {\n      this.dom.axis.parentNode.removeChild(this.dom.axis);\n    }\n\n    // remove the labelset containing all group labels\n    if (this.dom.labelSet.parentNode) {\n      this.dom.labelSet.parentNode.removeChild(this.dom.labelSet);\n    }\n  }\n\n  /**\n   * Show the component in the DOM (when not already visible).\n   */\n  show() {\n    // show frame containing the items\n    if (!this.dom.frame.parentNode) {\n      this.body.dom.center.appendChild(this.dom.frame);\n    }\n\n    // show axis with dots\n    if (!this.dom.axis.parentNode) {\n      this.body.dom.backgroundVertical.appendChild(this.dom.axis);\n    }\n\n    // show labelset containing labels\n    if (!this.dom.labelSet.parentNode) {\n      if (this.options.rtl) {\n        this.body.dom.right.appendChild(this.dom.labelSet);\n      } else {\n        this.body.dom.left.appendChild(this.dom.labelSet);\n      }\n    }\n  }\n\n  /**\n   * Activates the popup timer to show the given popup after a fixed time.\n   * @param {Popup} popup\n   */\n  setPopupTimer(popup) {\n    this.clearPopupTimer();\n    if (popup) {\n      const delay = this.options.tooltip.delay || typeof this.options.tooltip.delay === 'number' ?\n            this.options.tooltip.delay :\n            500;\n      this.popupTimer = setTimeout(\n        function () {\n          popup.show()\n        }, delay);\n    }\n  }\n\n  /**\n   * Clears the popup timer for the tooltip.\n   */\n  clearPopupTimer() {\n    if (this.popupTimer != null) {\n        clearTimeout(this.popupTimer);\n        this.popupTimer = null;\n    }\n  }\n  \n  /**\n   * Set selected items by their id. Replaces the current selection\n   * Unknown id's are silently ignored.\n   * @param {string[] | string} [ids] An array with zero or more id's of the items to be\n   *                                  selected, or a single item id. If ids is undefined\n   *                                  or an empty array, all items will be unselected.\n   */\n  setSelection(ids) {\n    if (ids == undefined) { \n      ids = [];\n    }\n    \n    if (!Array.isArray(ids)) {\n      ids = [ids];\n    }\n  \n\n    const idsToDeselect = this.selection.filter(id => ids.indexOf(id) === -1);\n\n    // unselect currently selected items\n    for (let selectedId of idsToDeselect) {\n      const item = this.getItemById(selectedId);\n      if (item) {\n        item.unselect();\n      }\n    }\n  \n    // select items\n    this.selection = [ ...ids ];\n    for (let id of ids) {\n      const item = this.getItemById(id);\n      if (item) {\n        item.select();\n      }\n    }\n  }\n\n  /**\n   * Get the selected items by their id\n   * @return {Array} ids  The ids of the selected items\n   */\n  getSelection() {\n    return this.selection.concat([]);\n  }\n\n  /**\n   * Get the id's of the currently visible items.\n   * @returns {Array} The ids of the visible items\n   */\n  getVisibleItems() {\n    const range = this.body.range.getRange();\n    let right;\n    let left;\n\n    if (this.options.rtl) { \n      right  = this.body.util.toScreen(range.start);\n      left = this.body.util.toScreen(range.end);\n    } else {\n      left  = this.body.util.toScreen(range.start);\n      right = this.body.util.toScreen(range.end);\n    }\n\n    const ids = [];\n    for (const groupId in this.groups) {\n      if (this.groups.hasOwnProperty(groupId)) {\n        const group = this.groups[groupId];\n        const rawVisibleItems = group.isVisible ? group.visibleItems : [];\n\n        // filter the \"raw\" set with visibleItems into a set which is really\n        // visible by pixels\n        for (const item of rawVisibleItems) {\n          // TODO: also check whether visible vertically\n          if (this.options.rtl) { \n            if ((item.right < left) && (item.right + item.width > right)) {\n              ids.push(item.id);\n            }\n          } else {\n            if ((item.left < right) && (item.left + item.width > left)) {\n              ids.push(item.id);\n            }\n          }\n        }\n      }\n    }\n\n    return ids;\n  }\n\n  /**\n   * Get the id's of the currently visible groups.\n   * @returns {Array} The ids of the visible groups\n   */\n  getVisibleGroups() {\n    const ids = [];\n\n    for (const groupId in this.groups) {\n      if (this.groups.hasOwnProperty(groupId)) {\n        const group = this.groups[groupId];\n        if (group.isVisible) {\n          ids.push(groupId)\n        }\n      }\n    }\n\n    return ids;\n  }\n  \n  /**\n   * get item by id\n   * @param {string} id\n   * @return {object} item\n   */\n  getItemById(id) {\n    return this.items[id] || this.clusters.find(cluster => cluster.id === id);\n  } \n\n  /**\n   * Deselect a selected item\n   * @param {string | number} id\n   * @private\n   */\n  _deselect(id) {\n    const selection = this.selection;\n    for (let i = 0, ii = selection.length; i < ii; i++) {\n      if (selection[i] == id) { // non-strict comparison!\n        selection.splice(i, 1);\n        break;\n      }\n    }\n  }\n\n  /**\n   * Repaint the component\n   * @return {boolean} Returns true if the component is resized\n   */\n  redraw() {\n    const margin = this.options.margin;\n    const range = this.body.range;\n    const asSize = util.option.asSize;\n    const options = this.options;\n    const orientation = options.orientation.item;\n    let resized = false;\n    const frame = this.dom.frame;\n\n    // recalculate absolute position (before redrawing groups)\n    this.props.top = this.body.domProps.top.height + this.body.domProps.border.top;\n\n    if (this.options.rtl) {\n      this.props.right = this.body.domProps.right.width + this.body.domProps.border.right;\n    } else {\n      this.props.left = this.body.domProps.left.width + this.body.domProps.border.left;\n    }\n\n    // update class name\n    frame.className = 'vis-itemset';\n\n    if (this.options.cluster) {\n      this._clusterItems();\n    }\n\n    // reorder the groups (if needed)\n    resized = this._orderGroups() || resized;\n\n    // check whether zoomed (in that case we need to re-stack everything)\n    // TODO: would be nicer to get this as a trigger from Range\n    const visibleInterval = range.end - range.start;\n    const zoomed = (visibleInterval != this.lastVisibleInterval) || (this.props.width != this.props.lastWidth);\n    const scrolled = range.start != this.lastRangeStart;\n    const changedStackOption = options.stack != this.lastStack;\n    const changedStackSubgroupsOption = options.stackSubgroups != this.lastStackSubgroups;\n    const forceRestack = (zoomed || scrolled || changedStackOption || changedStackSubgroupsOption);\n    this.lastVisibleInterval = visibleInterval;\n    this.lastRangeStart = range.start;\n    this.lastStack = options.stack;\n    this.lastStackSubgroups = options.stackSubgroups;\n\n    this.props.lastWidth = this.props.width;\n    const firstGroup = this._firstGroup();\n    const firstMargin = {\n      item: margin.item,\n      axis: margin.axis\n    };\n    const nonFirstMargin = {\n      item: margin.item,\n      axis: margin.item.vertical / 2\n    };\n    let height = 0;\n    const minHeight = margin.axis + margin.item.vertical;\n\n    // redraw the background group\n    this.groups[BACKGROUND].redraw(range, nonFirstMargin, forceRestack);\n\n    const redrawQueue = {};\n    let redrawQueueLength = 0;\n\n    // collect redraw functions\n    util.forEach(this.groups, (group, key) => {\n      if (key === BACKGROUND) return;\n      const groupMargin = group == firstGroup ? firstMargin : nonFirstMargin;\n      const returnQueue = true;\n      redrawQueue[key] = group.redraw(range, groupMargin, forceRestack, returnQueue);\n      redrawQueueLength = redrawQueue[key].length;\n    });\n\n    const needRedraw = redrawQueueLength > 0;\n    if (needRedraw) {\n      const redrawResults = {};\n\n      for (let i = 0; i < redrawQueueLength; i++) {\n        util.forEach(redrawQueue, (fns, key) => {\n          redrawResults[key] = fns[i]();\n        });\n      }\n\n      // redraw all regular groups\n      util.forEach(this.groups, (group, key) => {\n        if (key === BACKGROUND) return;\n        const groupResized = redrawResults[key];\n        resized = groupResized || resized;\n        height += group.height;\n      });\n      height = Math.max(height, minHeight);\n    }\n\n    height = Math.max(height, minHeight);\n\n    // update frame height\n    frame.style.height  = asSize(height);\n\n    // calculate actual size\n    this.props.width = frame.offsetWidth;\n    this.props.height = height;\n\n    // reposition axis\n    this.dom.axis.style.top = asSize((orientation == 'top') ?\n        (this.body.domProps.top.height + this.body.domProps.border.top) :\n        (this.body.domProps.top.height + this.body.domProps.centerContainer.height));\n    if (this.options.rtl) {\n      this.dom.axis.style.right = '0';\n    } else {\n      this.dom.axis.style.left = '0';\n    }\n\n    this.hammer.get('press').set({time: this.options.longSelectPressTime});\n\n    this.initialItemSetDrawn = true;\n    // check if this component is resized\n    resized = this._isResized() || resized;\n\n    return resized;\n  }\n\n  /**\n   * Get the first group, aligned with the axis\n   * @return {Group | null} firstGroup\n   * @private\n   */\n  _firstGroup() {\n    const firstGroupIndex = (this.options.orientation.item == 'top') ? 0 : (this.groupIds.length - 1);\n    const firstGroupId = this.groupIds[firstGroupIndex];\n    const firstGroup = this.groups[firstGroupId] || this.groups[UNGROUPED];\n\n    return firstGroup || null;\n  }\n\n  /**\n   * Create or delete the group holding all ungrouped items. This group is used when\n   * there are no groups specified.\n   * @protected\n   */\n  _updateUngrouped() {\n    let ungrouped = this.groups[UNGROUPED];\n    let item;\n    let itemId;\n\n    if (this.groupsData) {\n      // remove the group holding all ungrouped items\n      if (ungrouped) {\n        ungrouped.dispose();\n        delete this.groups[UNGROUPED];\n\n        for (itemId in this.items) {\n          if (this.items.hasOwnProperty(itemId)) {\n            item = this.items[itemId];\n            item.parent && item.parent.remove(item);\n            const groupId = this.getGroupId(item.data);\n            const group = this.groups[groupId];\n            group && group.add(item) || item.hide();\n          }\n        }\n      }\n    }\n    else {\n      // create a group holding all (unfiltered) items\n      if (!ungrouped) {\n        const id = null;\n        const data = null;\n        ungrouped = new Group(id, data, this);\n        this.groups[UNGROUPED] = ungrouped;\n\n        for (itemId in this.items) {\n          if (this.items.hasOwnProperty(itemId)) {\n            item = this.items[itemId];\n            ungrouped.add(item);\n          }\n        }\n\n        ungrouped.show();\n      }\n    }\n  }\n\n  /**\n   * Get the element for the labelset\n   * @return {HTMLElement} labelSet\n   */\n  getLabelSet() {\n    return this.dom.labelSet;\n  }\n\n  /**\n   * Set items\n   * @param {vis.DataSet | null} items\n   */\n  setItems(items) {\n    this.itemsSettingTime = new Date();\n    const me = this;\n    let ids;\n    const oldItemsData = this.itemsData;\n\n    // replace the dataset\n    if (!items) {\n      this.itemsData = null;\n    }\n    else if (items instanceof DataSet || items instanceof DataView) {\n      this.itemsData = typeCoerceDataSet(items);\n    }\n    else {\n      throw new TypeError('Data must be an instance of DataSet or DataView');\n    }\n\n    if (oldItemsData) {\n      // unsubscribe from old dataset\n      util.forEach(this.itemListeners, (callback, event) => {\n        oldItemsData.off(event, callback);\n      });\n\n      // stop maintaining a coerced version of the old data set\n      oldItemsData.dispose()\n\n      // remove all drawn items\n      ids = oldItemsData.getIds();\n      this._onRemove(ids);\n    }\n\n    if (this.itemsData) {\n      // subscribe to new dataset\n      const id = this.id;\n      util.forEach(this.itemListeners, (callback, event) => {\n        me.itemsData.on(event, callback, id);\n      });\n\n      // add all new items\n      ids = this.itemsData.getIds();\n      this._onAdd(ids);\n\n      // update the group holding all ungrouped items\n      this._updateUngrouped();\n    }\n\n    this.body.emitter.emit('_change', {queue: true});\n  }\n\n  /**\n   * Get the current items\n   * @returns {vis.DataSet | null}\n   */\n  getItems() {\n    return this.itemsData != null ? this.itemsData.rawDS : null;\n  }\n\n  /**\n   * Set groups\n   * @param {vis.DataSet} groups\n   */\n  setGroups(groups) {\n    const me = this;\n    let ids;\n\n    // unsubscribe from current dataset\n    if (this.groupsData) {\n      util.forEach(this.groupListeners, (callback, event) => {\n        me.groupsData.off(event, callback);\n      });\n\n      // remove all drawn groups\n      ids = this.groupsData.getIds();\n      this.groupsData = null;\n      this._onRemoveGroups(ids); // note: this will cause a redraw\n    }\n\n    // replace the dataset\n    if (!groups) {\n      this.groupsData = null;\n    }\n    else if (groups instanceof DataSet || groups instanceof DataView) {\n      this.groupsData = groups;\n    }\n    else {\n      throw new TypeError('Data must be an instance of DataSet or DataView');\n    }\n\n    if (this.groupsData) {\n      // go over all groups nesting\n      let groupsData = this.groupsData;\n      if (this.groupsData instanceof DataView) {\n        groupsData = this.groupsData.getDataSet()\n      }\n\n      groupsData.get().forEach(group => {\n        if (group.nestedGroups) {\n          group.nestedGroups.forEach(nestedGroupId => {\n            const updatedNestedGroup = groupsData.get(nestedGroupId);\n            updatedNestedGroup.nestedInGroup = group.id;\n            if (group.showNested == false) {\n              updatedNestedGroup.visible = false;\n            }\n            groupsData.update(updatedNestedGroup);\n          })\n        }\n      });\n\n      // subscribe to new dataset\n      const id = this.id;\n      util.forEach(this.groupListeners, (callback, event) => {\n        me.groupsData.on(event, callback, id);\n      });\n\n      // draw all ms\n      ids = this.groupsData.getIds();\n      this._onAddGroups(ids);\n    }\n\n    // update the group holding all ungrouped items\n    this._updateUngrouped();\n\n    // update the order of all items in each group\n    this._order();\n\n    if (this.options.cluster) {\n      this.clusterGenerator.updateData();\n      this._clusterItems();\n      this.markDirty({ refreshItems: true, restackGroups: true });\n    }\n\n    this.body.emitter.emit('_change', {queue: true});\n  }\n\n  /**\n   * Get the current groups\n   * @returns {vis.DataSet | null} groups\n   */\n  getGroups() {\n    return this.groupsData;\n  }\n\n  /**\n   * Remove an item by its id\n   * @param {string | number} id\n   */\n  removeItem(id) {\n    const item = this.itemsData.get(id);\n\n    if (item) {\n      // confirm deletion\n      this.options.onRemove(item, item => {\n        if (item) {\n          // remove by id here, it is possible that an item has no id defined\n          // itself, so better not delete by the item itself\n          this.itemsData.remove(id);\n        }\n      });\n    }\n  }\n\n  /**\n   * Get the time of an item based on it's data and options.type\n   * @param {Object} itemData\n   * @returns {string} Returns the type\n   * @private\n   */\n  _getType(itemData) {\n    return itemData.type || this.options.type || (itemData.end ? 'range' : 'box');\n  }\n\n  /**\n   * Get the group id for an item\n   * @param {Object} itemData\n   * @returns {string} Returns the groupId\n   * @private\n   */\n  getGroupId(itemData) {\n    const type = this._getType(itemData);\n    if (type == 'background' && itemData.group == undefined) {\n     return BACKGROUND;\n    }\n    else {\n      return this.groupsData ? itemData.group : UNGROUPED;\n    }\n  }\n\n  /**\n   * Handle updated items\n   * @param {number[]} ids\n   * @protected\n   */\n  _onUpdate(ids) {\n    const me = this;\n\n    ids.forEach(id => {\n      const itemData = me.itemsData.get(id);\n      let item = me.items[id];\n      const type = itemData ? me._getType(itemData) : null;\n\n      const constructor = ItemSet.types[type];\n      let selected;\n\n      if (item) {\n        // update item   \t\n        if (!constructor || !(item instanceof constructor)) {\n          // item type has changed, delete the item and recreate it\n          selected = item.selected; // preserve selection of this item\n          me._removeItem(item);\n          item = null;\n        }\n        else {\n          me._updateItem(item, itemData);\n        }\n      }\n\n      if (!item && itemData) {\n        // create item\n        if (constructor) {\n          item = new constructor(itemData, me.conversion, me.options);\n          item.id = id; // TODO: not so nice setting id afterwards\n\n          me._addItem(item);\n          if (selected) {\n            this.selection.push(id);\n            item.select();\n          }\n        }\n        else {\n          throw new TypeError(`Unknown item type \"${type}\"`);\n        }\n      }\n    });\n\n    this._order();\n    \n    if (this.options.cluster) {\n      this.clusterGenerator.setItems(this.items, { applyOnChangedLevel: false });\n      this._clusterItems();\n    }\n\n    this.body.emitter.emit('_change', {queue: true});\n  }\n\n  /**\n   * Handle removed items\n   * @param {number[]} ids\n   * @protected\n   */\n  _onRemove(ids) {\n    let count = 0;\n    const me = this;\n    ids.forEach(id => {\n      const item = me.items[id];\n      if (item) {\n        count++;\n        me._removeItem(item);\n      }\n    });\n\n    if (count) {\n      // update order\n      this._order();\n      this.body.emitter.emit('_change', {queue: true});\n    }\n  }\n\n  /**\n   * Update the order of item in all groups\n   * @private\n   */\n  _order() {\n    // reorder the items in all groups\n    // TODO: optimization: only reorder groups affected by the changed items\n    util.forEach(this.groups, group => {\n      group.order();\n    });\n  }\n\n  /**\n   * Handle updated groups\n   * @param {number[]} ids\n   * @private\n   */\n  _onUpdateGroups(ids) {\n    this._onAddGroups(ids);\n  }\n\n  /**\n   * Handle changed groups (added or updated)\n   * @param {number[]} ids\n   * @private\n   */\n  _onAddGroups(ids) {\n    const me = this;\n\n    ids.forEach(id => {\n      const groupData = me.groupsData.get(id);\n      let group = me.groups[id];\n\n      if (!group) {\n        // check for reserved ids\n        if (id == UNGROUPED || id == BACKGROUND) {\n          throw new Error(`Illegal group id. ${id} is a reserved id.`);\n        }\n\n        const groupOptions = Object.create(me.options);\n        util.extend(groupOptions, {\n          height: null\n        });\n\n        group = new Group(id, groupData, me);\n        me.groups[id] = group;\n\n        // add items with this groupId to the new group\n        for (const itemId in me.items) {\n          if (me.items.hasOwnProperty(itemId)) {\n            const item = me.items[itemId];\n            if (item.data.group == id) {\n              group.add(item);\n            }\n          }\n        }\n\n        group.order();\n        group.show();\n      }\n      else {\n        // update group\n        group.setData(groupData);\n      }\n    });\n\n    this.body.emitter.emit('_change', {queue: true});\n  }\n\n  /**\n   * Handle removed groups\n   * @param {number[]} ids\n   * @private\n   */\n  _onRemoveGroups(ids) {\n    ids.forEach(id => {\n      const group = this.groups[id];\n\n      if (group) {\n        group.dispose();\n        delete this.groups[id];\n      }\n    });\n\n    if (this.options.cluster) {\n      this.clusterGenerator.updateData();\n      this._clusterItems();\n    } \n\n    this.markDirty({ restackGroups: !!this.options.cluster });\n    this.body.emitter.emit('_change', {queue: true});\n  }\n\n  /**\n   * Reorder the groups if needed\n   * @return {boolean} changed\n   * @private\n   */\n  _orderGroups() {\n    if (this.groupsData) {\n      // reorder the groups\n      let groupIds = this.groupsData.getIds({\n        order: this.options.groupOrder\n      });\n\n      groupIds = this._orderNestedGroups(groupIds);\n\n      const changed = !util.equalArray(groupIds, this.groupIds);\n      if (changed) {\n        // hide all groups, removes them from the DOM\n        const groups = this.groups;\n        groupIds.forEach(groupId => {\n          groups[groupId].hide();\n        });\n\n        // show the groups again, attach them to the DOM in correct order\n        groupIds.forEach(groupId => {\n          groups[groupId].show();\n        });\n\n        this.groupIds = groupIds;\n      }\n\n      return changed;\n    }\n    else {\n      return false;\n    }\n  }\n\n  /**\n   * Reorder the nested groups\n   *\n   * @param {Array.<number>} groupIds\n   * @returns {Array.<number>}\n   * @private\n   */\n  _orderNestedGroups(groupIds) {\n    /**\n     * Recursively order nested groups\n     *\n     * @param {ItemSet} t\n     * @param {Array.<number>} groupIds\n     * @returns {Array.<number>}\n     * @private\n     */\n    function getOrderedNestedGroups(t, groupIds) {\n      let result = [];\n      groupIds.forEach(groupId => {\n        result.push(groupId);\n        const groupData = t.groupsData.get(groupId);\n        if (groupData.nestedGroups) {\n          const nestedGroupIds = t.groupsData.get({\n            filter(nestedGroup) {\n              return nestedGroup.nestedInGroup == groupId;\n            },\n            order: t.options.groupOrder\n          }).map(nestedGroup => nestedGroup.id);\n          result = result.concat(getOrderedNestedGroups(t, nestedGroupIds));\n        }\n      });\n\n      return result;\n    }\n\n    const topGroupIds = groupIds.filter(groupId => !this.groupsData.get(groupId).nestedInGroup);\n\n    return getOrderedNestedGroups(this, topGroupIds);\n  }\n\n  /**\n   * Add a new item\n   * @param {Item} item\n   * @private\n   */\n  _addItem(item) {\n    this.items[item.id] = item;\n\n    // add to group\n    const groupId = this.getGroupId(item.data);\n    const group = this.groups[groupId];\n\n    if (!group) {\n      item.groupShowing = false;\n    } else if (group && group.data && group.data.showNested) {\n      item.groupShowing = true;\n    }\n\n    if (group) group.add(item);\n  }\n\n  /**\n   * Update an existing item\n   * @param {Item} item\n   * @param {Object} itemData\n   * @private\n   */\n  _updateItem(item, itemData) {\n    // update the items data (will redraw the item when displayed)\n    item.setData(itemData);\n\n    const groupId = this.getGroupId(item.data);\n    const group = this.groups[groupId];\n    if (!group) {\n      item.groupShowing = false;\n    } else if (group && group.data && group.data.showNested) {\n      item.groupShowing = true;\n    }\n  }\n\n  /**\n   * Delete an item from the ItemSet: remove it from the DOM, from the map\n   * with items, and from the map with visible items, and from the selection\n   * @param {Item} item\n   * @private\n   */\n  _removeItem(item) {\n    // remove from DOM\n    item.hide();\n\n    // remove from items\n    delete this.items[item.id];\n\n    // remove from selection\n    const index = this.selection.indexOf(item.id);\n    if (index != -1) this.selection.splice(index, 1);\n\n    // remove from group\n    item.parent && item.parent.remove(item);\n\n    // remove Tooltip from DOM\n    if (this.popup != null) {\n      this.popup.hide();\n    }\n  }\n\n  /**\n   * Create an array containing all items being a range (having an end date)\n   * @param {Array.<Object>} array\n   * @returns {Array}\n   * @private\n   */\n  _constructByEndArray(array) {\n    const endArray = [];\n\n    for (let i = 0; i < array.length; i++) {\n      if (array[i] instanceof RangeItem) {\n        endArray.push(array[i]);\n      }\n    }\n    return endArray;\n  }\n\n  /**\n   * Register the clicked item on touch, before dragStart is initiated.\n   *\n   * dragStart is initiated from a mousemove event, AFTER the mouse/touch is\n   * already moving. Therefore, the mouse/touch can sometimes be above an other\n   * DOM element than the item itself.\n   *\n   * @param {Event} event\n   * @private\n   */\n  _onTouch(event) {\n    // store the touched item, used in _onDragStart\n    this.touchParams.item = this.itemFromTarget(event);\n    this.touchParams.dragLeftItem = event.target.dragLeftItem || false;\n    this.touchParams.dragRightItem = event.target.dragRightItem || false;\n    this.touchParams.itemProps = null;\n  }\n\n  /**\n   * Given an group id, returns the index it has.\n   *\n   * @param {number} groupId\n   * @returns {number} index / groupId\n   * @private\n   */\n  _getGroupIndex(groupId) {\n      for (let i = 0; i < this.groupIds.length; i++) {\n          if (groupId == this.groupIds[i])\n              return i;\n      }\n  }\n\n  /**\n   * Start dragging the selected events\n   * @param {Event} event\n   * @private\n   */\n  _onDragStart(event) {\n    if (this.touchParams.itemIsDragging) { return; }\n    const item = this.touchParams.item || null;\n    const me = this;\n    let props;\n\n    if (item && (item.selected || this.options.itemsAlwaysDraggable.item)) {\n\n      if (this.options.editable.overrideItems &&\n          !this.options.editable.updateTime &&\n          !this.options.editable.updateGroup) {\n        return;\n      }\n\n      // override options.editable\n      if ((item.editable != null && !item.editable.updateTime && !item.editable.updateGroup)\n          && !this.options.editable.overrideItems) {\n        return;\n      }\n\n      const dragLeftItem = this.touchParams.dragLeftItem;\n      const dragRightItem = this.touchParams.dragRightItem;\n      this.touchParams.itemIsDragging = true;\n      this.touchParams.selectedItem = item;\n\n      if (dragLeftItem) {\n        props = {\n          item: dragLeftItem,\n          initialX: event.center.x,\n          dragLeft:  true,\n          data: this._cloneItemData(item.data)\n        };\n\n        this.touchParams.itemProps = [props];\n      } else if (dragRightItem) {\n        props = {\n          item: dragRightItem,\n          initialX: event.center.x,\n          dragRight: true,\n          data: this._cloneItemData(item.data)\n        };\n\n        this.touchParams.itemProps = [props];\n      } else if (this.options.editable.add && (event.srcEvent.ctrlKey || event.srcEvent.metaKey)) {\n        // create a new range item when dragging with ctrl key down\n        this._onDragStartAddItem(event);\n      } else {\n        if(this.groupIds.length < 1) {\n          // Mitigates a race condition if _onDragStart() is\n          // called after markDirty() without redraw() being called between.\n          this.redraw();\n        }\n        \n        const baseGroupIndex = this._getGroupIndex(item.data.group);\n\n        const itemsToDrag = (this.options.itemsAlwaysDraggable.item && !item.selected) ? [item.id] : this.getSelection();\n\n        this.touchParams.itemProps = itemsToDrag.map(id => {\n          const item = me.items[id];\n          const groupIndex = me._getGroupIndex(item.data.group);\n          return {\n            item,\n            initialX: event.center.x,\n            groupOffset: baseGroupIndex-groupIndex,\n            data: this._cloneItemData(item.data)\n          };\n        });\n      }\n\n      event.stopPropagation();\n    } else if (this.options.editable.add && (event.srcEvent.ctrlKey || event.srcEvent.metaKey)) {\n      // create a new range item when dragging with ctrl key down\n      this._onDragStartAddItem(event);\n    }\n  }\n\n  /**\n   * Start creating a new range item by dragging.\n   * @param {Event} event\n   * @private\n   */\n  _onDragStartAddItem(event) {\n    const snap = this.options.snap || null;\n    const frameRect = this.dom.frame.getBoundingClientRect()\n\n    // plus (if rtl) 10 to compensate for the drag starting as soon as you've moved 10px\n    const x = this.options.rtl ? frameRect.right - event.center.x  + 10 : event.center.x - frameRect.left - 10;\n\n    const time = this.body.util.toTime(x);\n    const scale = this.body.util.getScale();\n    const step = this.body.util.getStep();\n    const start = snap ? snap(time, scale, step) : time;\n    const end = start;\n\n    const itemData = {\n      type: 'range',\n      start,\n      end,\n      content: 'new item'\n    };\n\n    const id = randomUUID();\n    itemData[this.itemsData.idProp] = id;\n\n    const group = this.groupFromTarget(event);\n    if (group) {\n      itemData.group = group.groupId;\n    }\n    const newItem = new RangeItem(itemData, this.conversion, this.options);\n    newItem.id = id; // TODO: not so nice setting id afterwards\n    newItem.data = this._cloneItemData(itemData);\n    this._addItem(newItem);\n    this.touchParams.selectedItem = newItem;\n    \n    const props = {\n      item: newItem,\n      initialX: event.center.x,\n      data: newItem.data\n    };\n\n    if (this.options.rtl) {\n      props.dragLeft = true;\n    } else {\n      props.dragRight = true;\n    }\n    this.touchParams.itemProps = [props];\n\n    event.stopPropagation();\n  }\n\n  /**\n   * Drag selected items\n   * @param {Event} event\n   * @private\n   */\n  _onDrag(event) {\n    if (this.popup != null && this.options.showTooltips && !this.popup.hidden) {\n      // this.popup.hide();\n      const container = this.body.dom.centerContainer;\n      const containerRect = container.getBoundingClientRect()\n      this.popup.setPosition(\n        event.center.x - containerRect.left + container.offsetLeft,\n        event.center.y - containerRect.top + container.offsetTop\n      );\n      this.popup.show(); // redraw\n    }\n    \n    if (this.touchParams.itemProps) {\n      event.stopPropagation();\n\n      const me = this;\n      const snap = this.options.snap || null;\n      const domRootOffsetLeft = this.body.dom.root.offsetLeft;\n      const xOffset = this.options.rtl ? domRootOffsetLeft + this.body.domProps.right.width : domRootOffsetLeft + this.body.domProps.left.width;\n      const scale = this.body.util.getScale();\n      const step = this.body.util.getStep();\n\n      //only calculate the new group for the item that's actually dragged\n      const selectedItem = this.touchParams.selectedItem;\n      const updateGroupAllowed = ((this.options.editable.overrideItems || selectedItem.editable == null) && this.options.editable.updateGroup) ||\n                               (!this.options.editable.overrideItems && selectedItem.editable != null && selectedItem.editable.updateGroup);\n      let newGroupBase = null;\n      if (updateGroupAllowed && selectedItem) {\n        if (selectedItem.data.group != undefined) {\n          // drag from one group to another\n          const group = me.groupFromTarget(event);\n          if (group) {\n            //we know the offset for all items, so the new group for all items\n            //will be relative to this one.\n            newGroupBase = this._getGroupIndex(group.groupId);\n          }\n        }\n      }\n\n      // move\n      this.touchParams.itemProps.forEach(props => {\n        const current = me.body.util.toTime(event.center.x - xOffset);\n        const initial = me.body.util.toTime(props.initialX - xOffset);\n        let offset;\n        let initialStart;\n        let initialEnd;\n        let start;\n        let end;\n\n        if (this.options.rtl) {\n          offset = -(current - initial); // ms\n        } else {\n          offset = (current - initial); // ms\n        }\n\n        let itemData = this._cloneItemData(props.item.data); // clone the data\n        if (props.item.editable != null\n          && !props.item.editable.updateTime\n          && !props.item.editable.updateGroup\n          && !me.options.editable.overrideItems) {\n          return;\n        }\n\n        const updateTimeAllowed = ((this.options.editable.overrideItems || selectedItem.editable == null) && this.options.editable.updateTime) ||\n                                 (!this.options.editable.overrideItems && selectedItem.editable != null && selectedItem.editable.updateTime);\n        if (updateTimeAllowed) {\n          if (props.dragLeft) {\n            // drag left side of a range item\n            if (this.options.rtl) {\n              if (itemData.end != undefined) {\n                initialEnd = util.convert(props.data.end, 'Date');\n                end = new Date(initialEnd.valueOf() + offset);\n                // TODO: pass a Moment instead of a Date to snap(). (Breaking change)\n                itemData.end = snap ? snap(end, scale, step) : end;\n              }\n            } else {\n              if (itemData.start != undefined) {\n                initialStart = util.convert(props.data.start, 'Date');\n                start = new Date(initialStart.valueOf() + offset);\n                // TODO: pass a Moment instead of a Date to snap(). (Breaking change)\n                itemData.start = snap ? snap(start, scale, step) : start;\n              }\n            }\n          }\n          else if (props.dragRight) {\n            // drag right side of a range item\n            if (this.options.rtl) {\n              if (itemData.start != undefined) {\n                initialStart = util.convert(props.data.start, 'Date');\n                start = new Date(initialStart.valueOf() + offset);\n                // TODO: pass a Moment instead of a Date to snap(). (Breaking change)\n                itemData.start = snap ? snap(start, scale, step) : start;\n              }\n            } else {\n              if (itemData.end != undefined) {\n                initialEnd = util.convert(props.data.end, 'Date');\n                end = new Date(initialEnd.valueOf() + offset);\n                // TODO: pass a Moment instead of a Date to snap(). (Breaking change)\n                itemData.end = snap ? snap(end, scale, step) : end;\n              }\n            }\n          }\n          else {\n            // drag both start and end\n            if (itemData.start != undefined) {\n\n              initialStart = util.convert(props.data.start, 'Date').valueOf();\n              start = new Date(initialStart + offset);\n\n              if (itemData.end != undefined) {\n                initialEnd = util.convert(props.data.end, 'Date');\n                const duration  = initialEnd.valueOf() - initialStart.valueOf();\n\n                // TODO: pass a Moment instead of a Date to snap(). (Breaking change)\n                itemData.start = snap ? snap(start, scale, step) : start;\n                itemData.end   = new Date(itemData.start.valueOf() + duration);\n              }\n              else {\n                // TODO: pass a Moment instead of a Date to snap(). (Breaking change)\n                itemData.start = snap ? snap(start, scale, step) : start;\n              }\n            }\n          }\n        }\n\n        if (updateGroupAllowed && (!props.dragLeft && !props.dragRight) && newGroupBase!=null) {\n          if (itemData.group != undefined) {\n            let newOffset = newGroupBase - props.groupOffset;\n\n            //make sure we stay in bounds\n            newOffset = Math.max(0, newOffset);\n            newOffset = Math.min(me.groupIds.length-1, newOffset);\n            itemData.group = me.groupIds[newOffset];\n          }\n        }\n\n        // confirm moving the item\n        itemData = this._cloneItemData(itemData);  // convert start and end to the correct type\n        me.options.onMoving(itemData, itemData => {\n          if (itemData) {\n            props.item.setData(this._cloneItemData(itemData, 'Date'));\n          }\n        });\n      });\n      \n      this.body.emitter.emit('_change');\n    }\n  }\n\n  /**\n   * Move an item to another group\n   * @param {Item} item\n   * @param {string | number} groupId\n   * @private\n   */\n  _moveToGroup(item, groupId) {\n    const group = this.groups[groupId];\n    if (group && group.groupId != item.data.group) {\n      const oldGroup = item.parent;\n      oldGroup.remove(item);\n      oldGroup.order();\n      \n      item.data.group = group.groupId;\n      \n      group.add(item);\n      group.order();\n    }\n  }\n\n  /**\n   * End of dragging selected items\n   * @param {Event} event\n   * @private\n   */\n  _onDragEnd(event) {\n    this.touchParams.itemIsDragging = false;\n    if (this.touchParams.itemProps) {\n      event.stopPropagation();\n\n      const me = this;\n      const itemProps = this.touchParams.itemProps;\n      this.touchParams.itemProps = null;\n\n      itemProps.forEach(props => {\n        const id = props.item.id;\n        const exists = me.itemsData.get(id) != null;\n\n        if (!exists) {\n          // add a new item\n          me.options.onAdd(props.item.data, itemData => {\n            me._removeItem(props.item); // remove temporary item\n            if (itemData) {\n              me.itemsData.add(itemData);\n            }\n\n            // force re-stacking of all items next redraw\n            me.body.emitter.emit('_change');\n          });\n        }\n        else {\n          // update existing item\n          const itemData = this._cloneItemData(props.item.data); // convert start and end to the correct type\n          me.options.onMove(itemData, itemData => {\n            if (itemData) {\n              // apply changes\n              itemData[this.itemsData.idProp] = id; // ensure the item contains its id (can be undefined)\n              this.itemsData.update(itemData);\n            }\n            else {\n              // restore original values\n              props.item.setData(props.data);\n\n              me.body.emitter.emit('_change');\n            }\n          });\n        }\n      });\n    }\n  }\n\n  /**\n   * On group click\n   * @param {Event} event\n   * @private\n   */\n  _onGroupClick(event) {\n    const group = this.groupFromTarget(event);\n    setTimeout(() => {\n      this.toggleGroupShowNested(group);\n    }, 1)\n  }\n  \n  /**\n   * Toggle show nested\n   * @param {object} group\n   * @param {boolean} force\n   */\n  toggleGroupShowNested(group, force = undefined) {\n\n    if (!group || !group.nestedGroups) return;\n\n    const groupsData = this.groupsData.getDataSet();\n\n    if (force != undefined) {\n      group.showNested = !!force;\n    } else {\n      group.showNested = !group.showNested;\n    }\n\n    let nestingGroup = groupsData.get(group.groupId);\n    nestingGroup.showNested = group.showNested;\n\n    let fullNestedGroups = group.nestedGroups;\n    let nextLevel = fullNestedGroups;\n    while (nextLevel.length > 0) {\n      let current = nextLevel;\n      nextLevel = [];\n      for (let i = 0; i < current.length; i++) {\n        let node = groupsData.get(current[i]);\n        if (node.nestedGroups) {\n          nextLevel = nextLevel.concat(node.nestedGroups);\n        }\n      }\n      if (nextLevel.length > 0) {\n        fullNestedGroups = fullNestedGroups.concat(nextLevel);\n      }\n    }\n    let nestedGroups = groupsData.get(fullNestedGroups).map(function (nestedGroup) {\n        if (nestedGroup.visible == undefined) {\n          nestedGroup.visible = true;\n        }\n        nestedGroup.visible = !!nestingGroup.showNested;\n        return nestedGroup;\n      });\n\n    groupsData.update(nestedGroups.concat(nestingGroup));\n\n    if (nestingGroup.showNested) {\n      util.removeClassName(group.dom.label, 'collapsed');\n      util.addClassName(group.dom.label, 'expanded');\n    } else {\n      util.removeClassName(group.dom.label, 'expanded');\n      util.addClassName(group.dom.label, 'collapsed');\n    }\n  }\n  \n  /**\n   * Toggle group drag classname\n   * @param {object} group\n   */\n  toggleGroupDragClassName(group) {\n    group.dom.label.classList.toggle('vis-group-is-dragging');\n    group.dom.foreground.classList.toggle('vis-group-is-dragging');\n  }\n  \n  /**\n   * on drag start\n   * @param {Event} event\n   * @return {void}   \n   * @private\n   */\n  _onGroupDragStart(event) {\n    if (this.groupTouchParams.isDragging) return;\n\n    if (this.options.groupEditable.order) {\n      this.groupTouchParams.group = this.groupFromTarget(event);\n      \n      if (this.groupTouchParams.group) {\n        event.stopPropagation();      \n        \n        this.groupTouchParams.isDragging = true;\n        this.toggleGroupDragClassName(this.groupTouchParams.group);\n        \n        this.groupTouchParams.originalOrder = this.groupsData.getIds({\n          order: this.options.groupOrder\n        });\n      }\n    }\n  }\n\n  /**\n   * on drag\n   * @param {Event} event\n   * @return {void}\n   * @private\n   */\n  _onGroupDrag(event) {\n      if (this.options.groupEditable.order && this.groupTouchParams.group) {\n          event.stopPropagation();\n          \n      let groupsData = this.groupsData;\n      if (this.groupsData instanceof DataView) {\n        groupsData = this.groupsData.getDataSet()\n      }\n          // drag from one group to another\n          const group = this.groupFromTarget(event);\n          \n          // try to avoid toggling when groups differ in height\n          if (group && group.height != this.groupTouchParams.group.height) {\n              const movingUp = (group.top < this.groupTouchParams.group.top);\n              const clientY = event.center ? event.center.y : event.clientY;\n              const targetGroup = group.dom.foreground.getBoundingClientRect()\n              const draggedGroupHeight = this.groupTouchParams.group.height;\n              if (movingUp) {\n                  // skip swapping the groups when the dragged group is not below clientY afterwards\n                  if (targetGroup.top + draggedGroupHeight < clientY) {\n                      return;\n                  }\n              } else {\n                  const targetGroupHeight = group.height;\n                  // skip swapping the groups when the dragged group is not below clientY afterwards\n                  if (targetGroup.top + targetGroupHeight - draggedGroupHeight > clientY) {\n                      return;\n                  }\n              }\n          }\n          \n          if (group && group != this.groupTouchParams.group) {\n              const targetGroup = groupsData.get(group.groupId);\n              const draggedGroup = groupsData.get(this.groupTouchParams.group.groupId);\n              \n              // switch groups\n              if (draggedGroup && targetGroup) {\n                  this.options.groupOrderSwap(draggedGroup, targetGroup, groupsData);\n                  groupsData.update(draggedGroup);\n                  groupsData.update(targetGroup);\n              }\n              \n              // fetch current order of groups\n              const newOrder = groupsData.getIds({\n                order: this.options.groupOrder\n              });\n\n              \n              // in case of changes since _onGroupDragStart\n              if (!util.equalArray(newOrder, this.groupTouchParams.originalOrder)) {\n                  const origOrder = this.groupTouchParams.originalOrder;\n                  const draggedId = this.groupTouchParams.group.groupId;\n                  const numGroups = Math.min(origOrder.length, newOrder.length);\n                  let curPos = 0;\n                  let newOffset = 0;\n                  let orgOffset = 0;\n                  while (curPos < numGroups) {\n                      // as long as the groups are where they should be step down along the groups order\n                      while ((curPos+newOffset) < numGroups \n                          && (curPos+orgOffset) < numGroups \n                          && newOrder[curPos+newOffset] == origOrder[curPos+orgOffset]) {\n                          curPos++;\n                      }\n                      \n                      // all ok\n                      if (curPos+newOffset >= numGroups) {\n                          break;\n                      }\n                      \n                      // not all ok\n                      // if dragged group was move upwards everything below should have an offset\n                      if (newOrder[curPos+newOffset] == draggedId) {\n                          newOffset = 1;\n\n                      }\n                      // if dragged group was move downwards everything above should have an offset\n                      else if (origOrder[curPos+orgOffset] == draggedId) {\n                          orgOffset = 1;\n\n                      } \n                      // found a group (apart from dragged group) that has the wrong position -> switch with the \n                      // group at the position where other one should be, fix index arrays and continue\n                      else {\n                          const slippedPosition = newOrder.indexOf(origOrder[curPos+orgOffset]);\n                          const switchGroup = groupsData.get(newOrder[curPos+newOffset]);\n                          const shouldBeGroup = groupsData.get(origOrder[curPos+orgOffset]);\n                          this.options.groupOrderSwap(switchGroup, shouldBeGroup, groupsData);\n                          groupsData.update(switchGroup);\n                          groupsData.update(shouldBeGroup);\n                          \n                          const switchGroupId = newOrder[curPos+newOffset];\n                          newOrder[curPos+newOffset] = origOrder[curPos+orgOffset];\n                          newOrder[slippedPosition] = switchGroupId;\n                          \n                          curPos++;\n                      }\n                  }\n              }\n              \n          }\n      }\n  }\n\n  /**\n   * on drag end\n   * @param {Event} event\n   * @return {void}\n   * @private\n   */\n  _onGroupDragEnd(event) {\n    this.groupTouchParams.isDragging = false;\n\n    if (this.options.groupEditable.order && this.groupTouchParams.group) {\n      event.stopPropagation();\n          \n      // update existing group\n      const me = this;\n      const id = me.groupTouchParams.group.groupId;\n      const dataset = me.groupsData.getDataSet();\n      const groupData = util.extend({}, dataset.get(id)); // clone the data\n      me.options.onMoveGroup(groupData, groupData => {\n        if (groupData) {\n          // apply changes\n          groupData[dataset._idProp] = id; // ensure the group contains its id (can be undefined)\n          dataset.update(groupData);\n        }\n        else {\n\n          // fetch current order of groups\n          const newOrder = dataset.getIds({\n              order: me.options.groupOrder\n          });\n\n          // restore original order\n          if (!util.equalArray(newOrder, me.groupTouchParams.originalOrder)) {\n            const origOrder = me.groupTouchParams.originalOrder;\n            const numGroups = Math.min(origOrder.length, newOrder.length);\n            let curPos = 0;\n            while (curPos < numGroups) {\n              // as long as the groups are where they should be step down along the groups order\n              while (curPos < numGroups && newOrder[curPos] == origOrder[curPos]) {\n                curPos++;\n              }\n\n              // all ok\n              if (curPos >= numGroups) {\n                break;\n              }\n\n              // found a group that has the wrong position -> switch with the\n              // group at the position where other one should be, fix index arrays and continue\n              const slippedPosition = newOrder.indexOf(origOrder[curPos]);\n              const switchGroup = dataset.get(newOrder[curPos]);\n              const shouldBeGroup = dataset.get(origOrder[curPos]);\n              me.options.groupOrderSwap(switchGroup, shouldBeGroup, dataset);\n              dataset.update(switchGroup);\n              dataset.update(shouldBeGroup);\n\n              const switchGroupId = newOrder[curPos];\n              newOrder[curPos] = origOrder[curPos];\n              newOrder[slippedPosition] = switchGroupId;\n\n              curPos++;\n            }\n          }\n        }\n      });\n\n      me.body.emitter.emit('groupDragged', { groupId: id });\n      this.toggleGroupDragClassName(this.groupTouchParams.group);\n      this.groupTouchParams.group = null;\n    }\n  }\n\n  /**\n   * Handle selecting/deselecting an item when tapping it\n   * @param {Event} event\n   * @private\n   */\n  _onSelectItem(event) {\n    if (!this.options.selectable) return;\n\n    const ctrlKey  = event.srcEvent && (event.srcEvent.ctrlKey || event.srcEvent.metaKey);\n    const shiftKey = event.srcEvent && event.srcEvent.shiftKey;\n    if (ctrlKey || shiftKey) {\n      this._onMultiSelectItem(event);\n      return;\n    }\n\n    const oldSelection = this.getSelection();\n\n    const item = this.itemFromTarget(event);\n    const selection = item && item.selectable ? [item.id] : [];\n    this.setSelection(selection);\n\n    const newSelection = this.getSelection();\n\n    // emit a select event,\n    // except when old selection is empty and new selection is still empty\n    if (newSelection.length > 0 || oldSelection.length > 0) {\n      this.body.emitter.emit('select', {\n        items: newSelection,\n        event\n      });\n    }\n  }\n\n  /**\n   * Handle hovering an item\n   * @param {Event} event\n   * @private\n   */\n  _onMouseOver(event) {\n    const item = this.itemFromTarget(event);\n    if (!item) return;\n\n    // Item we just left\n    const related = this.itemFromRelatedTarget(event);\n    if (item === related) {\n      // We haven't changed item, just element in the item\n      return;\n    }\n\n    const title = item.getTitle();\n    if (this.options.showTooltips && title) {\n      if (this.popup == null) {\n        this.popup = new Popup(this.body.dom.root,\n            this.options.tooltip.overflowMethod || 'flip');\n      }\n\n      this.popup.setText(title);\n      const container = this.body.dom.centerContainer;\n      const containerRect = container.getBoundingClientRect()\n      this.popup.setPosition(\n        event.clientX - containerRect.left + container.offsetLeft,\n        event.clientY - containerRect.top + container.offsetTop\n      );\n      this.setPopupTimer(this.popup);\n    } else {\n      // Hovering over item without a title, hide popup\n      // Needed instead of _just_ in _onMouseOut due to #2572\n      this.clearPopupTimer();\n      if (this.popup != null) {\n        this.popup.hide();\n      }\n    }\n\n    this.body.emitter.emit('itemover', {\n      item: item.id,\n      event\n    });\n  }\n\n  /**\n   * on mouse start\n   * @param {Event} event\n   * @return {void}   \n   * @private\n   */\n  _onMouseOut(event) {\n    const item = this.itemFromTarget(event);\n    if (!item) return;\n\n    // Item we are going to\n    const related = this.itemFromRelatedTarget(event);\n    if (item === related) {\n      // We aren't changing item, just element in the item\n      return;\n    }\n\n    this.clearPopupTimer();\n    if (this.popup != null) {\n      this.popup.hide();\n    }\n\n    this.body.emitter.emit('itemout', {\n      item: item.id,\n      event\n    });\n  }\n\n  /**\n   * on mouse move\n   * @param {Event} event\n   * @return {void}   \n   * @private\n   */\n  _onMouseMove(event) {\n    const item = this.itemFromTarget(event);\n    if (!item) return;\n\n    if (this.popupTimer != null) {\n      // restart timer\n      this.setPopupTimer(this.popup);\n    }\n    \n    if (this.options.showTooltips && this.options.tooltip.followMouse && this.popup && !this.popup.hidden) {\n      const container = this.body.dom.centerContainer;\n      const containerRect = container.getBoundingClientRect()\n      this.popup.setPosition(\n        event.clientX - containerRect.left + container.offsetLeft,\n        event.clientY - containerRect.top + container.offsetTop\n      );\n      this.popup.show(); // Redraw\n    }\n  }\n\n  /**\n   * Handle mousewheel\n   * @param {Event}  event   The event\n   * @private\n   */\n  _onMouseWheel(event) {\n    if (this.touchParams.itemIsDragging) {\n      this._onDragEnd(event);\n    }\n  }\n\n  /**\n   * Handle updates of an item on double tap\n   * @param {timeline.Item}  item   The item\n   * @private\n   */\n  _onUpdateItem(item) {\n    if (!this.options.selectable) return;\n    if (!this.options.editable.updateTime && !this.options.editable.updateGroup) return;\n\n    const me = this;\n   \n    if (item) {\n      // execute async handler to update the item (or cancel it)\n      const itemData = me.itemsData.get(item.id); // get a clone of the data from the dataset\n      this.options.onUpdate(itemData, itemData => {\n        if (itemData) {\n          me.itemsData.update(itemData);\n        }\n      });\n    }\n  }\n\n  /**\n   * Handle drop event of data on item\n   * Only called when `objectData.target === 'item'.\n   * @param {Event} event The event \n   * @private\n   */\n  _onDropObjectOnItem(event) {\n    const item = this.itemFromTarget(event);\n    const objectData = JSON.parse(event.dataTransfer.getData(\"text\"));\n    this.options.onDropObjectOnItem(objectData, item)\n  }\n\n  /**\n   * Handle creation of an item on double tap or drop of a drag event\n   * @param {Event} event   The event\n   * @private\n   */\n  _onAddItem(event) {\n    if (!this.options.selectable) return;\n    if (!this.options.editable.add) return;\n\n    const me = this;\n    const snap = this.options.snap || null;\n\n    // add item\n    const frameRect = this.dom.frame.getBoundingClientRect()\n    const x = this.options.rtl ? frameRect.right - event.center.x : event.center.x - frameRect.left;\n    const start = this.body.util.toTime(x);\n    const scale = this.body.util.getScale();\n    const step = this.body.util.getStep();\n    let end;\n\n    let newItemData;\n    if (event.type == 'drop') {\n      newItemData = JSON.parse(event.dataTransfer.getData(\"text\"));\n      newItemData.content = newItemData.content ? newItemData.content : 'new item';\n      newItemData.start = newItemData.start ? newItemData.start : (snap ? snap(start, scale, step) : start);\n      newItemData.type = newItemData.type || 'box';\n      newItemData[this.itemsData.idProp] = newItemData.id || randomUUID();\n\n      if (newItemData.type == 'range' && !newItemData.end) {\n        end = this.body.util.toTime(x + this.props.width / 5);\n        newItemData.end = snap ? snap(end, scale, step) : end;\n      }\n    } else {\n      newItemData = {\n        start: snap ? snap(start, scale, step) : start,\n        content: 'new item'\n      };\n      newItemData[this.itemsData.idProp] = randomUUID();\n\n      // when default type is a range, add a default end date to the new item\n      if (this.options.type === 'range') {\n        end = this.body.util.toTime(x + this.props.width / 5);\n        newItemData.end = snap ? snap(end, scale, step) : end;\n      }\n    }\n\n    const group = this.groupFromTarget(event);\n    if (group) {\n      newItemData.group = group.groupId;\n    }\n\n    // execute async handler to customize (or cancel) adding an item\n    newItemData = this._cloneItemData(newItemData);     // convert start and end to the correct type\n    this.options.onAdd(newItemData, item => {\n      if (item) {\n        me.itemsData.add(item);\n        if (event.type == 'drop') {\n          me.setSelection([item.id]);\n        }\n        // TODO: need to trigger a redraw?\n      }\n    });\n  }\n\n  /**\n   * Handle selecting/deselecting multiple items when holding an item\n   * @param {Event} event\n   * @private\n   */\n  _onMultiSelectItem(event) {\n    if (!this.options.selectable) return;\n\n    const item = this.itemFromTarget(event);\n\n    if (item) {\n      // multi select items (if allowed)\n\n      let selection = this.options.multiselect\n        ? this.getSelection() // take current selection\n        : [];                 // deselect current selection\n\n      const shiftKey = event.srcEvent && event.srcEvent.shiftKey || false;\n\n      if ((shiftKey || this.options.sequentialSelection) && this.options.multiselect) {\n        // select all items between the old selection and the tapped item\n        const itemGroup = this.itemsData.get(item.id).group;\n\n        // when filtering get the group of the last selected item\n        let lastSelectedGroup = undefined;\n        if (this.options.multiselectPerGroup) {\n          if (selection.length > 0) {\n            lastSelectedGroup = this.itemsData.get(selection[0]).group;\n          }\n        }\n\n        // determine the selection range\n        if (!this.options.multiselectPerGroup || lastSelectedGroup == undefined || lastSelectedGroup == itemGroup) {\n          selection.push(item.id);\n        }\n        const range = ItemSet._getItemRange(this.itemsData.get(selection));\n        \n        if (!this.options.multiselectPerGroup || lastSelectedGroup == itemGroup) {\n          // select all items within the selection range\n          selection = [];\n          for (const id in this.items) {\n            if (this.items.hasOwnProperty(id)) {\n              const _item = this.items[id];\n              const start = _item.data.start;\n              const end = (_item.data.end !== undefined) ? _item.data.end : start;\n\n              if (start >= range.min &&\n                  end <= range.max &&\n                  (!this.options.multiselectPerGroup || lastSelectedGroup == this.itemsData.get(_item.id).group) &&\n                  !(_item instanceof BackgroundItem)) {\n                selection.push(_item.id); // do not use id but item.id, id itself is stringified\n              }\n            }\n          }\n        }\n      }\n      else {\n        // add/remove this item from the current selection\n        const index = selection.indexOf(item.id);\n        if (index == -1) {\n          // item is not yet selected -> select it\n          selection.push(item.id);\n        }\n        else {\n          // item is already selected -> deselect it\n          selection.splice(index, 1);\n        }\n      }\n\n      const filteredSelection = selection.filter(item => this.getItemById(item).selectable);\n\n      this.setSelection(filteredSelection);\n\n      this.body.emitter.emit('select', {\n        items: this.getSelection(),\n        event\n      });\n    }\n  }\n\n  /**\n   * Calculate the time range of a list of items\n   * @param {Array.<Object>} itemsData\n   * @return {{min: Date, max: Date}} Returns the range of the provided items\n   * @private\n   */\n  static _getItemRange(itemsData) {\n    let max = null;\n    let min = null;\n\n    itemsData.forEach(data => {\n      if (min == null || data.start < min) {\n        min = data.start;\n      }\n\n      if (data.end != undefined) {\n        if (max == null || data.end > max) {\n          max = data.end;\n        }\n      }\n      else {\n        if (max == null || data.start > max) {\n          max = data.start;\n        }\n      }\n    });\n\n    return {\n      min,\n      max\n    }\n  }\n\n  /**\n   * Find an item from an element:\n   * searches for the attribute 'vis-item' in the element's tree\n   * @param {HTMLElement} element\n   * @return {Item | null} item\n   */\n  itemFromElement(element) {\n    let cur = element;\n    while (cur) {\n      if (cur.hasOwnProperty('vis-item')) {\n        return cur['vis-item'];\n      }\n      cur = cur.parentNode;\n    }\n\n    return null;\n  }\n\n  /**\n   * Find an item from an event target:\n   * searches for the attribute 'vis-item' in the event target's element tree\n   * @param {Event} event\n   * @return {Item | null} item\n   */\n  itemFromTarget(event) {\n    return this.itemFromElement(event.target);\n  }\n\n  /**\n   * Find an item from an event's related target:\n   * searches for the attribute 'vis-item' in the related target's element tree\n   * @param {Event} event\n   * @return {Item | null} item\n   */\n  itemFromRelatedTarget(event) {\n    return this.itemFromElement(event.relatedTarget);\n  }\n\n  /**\n   * Find the Group from an event target:\n   * searches for the attribute 'vis-group' in the event target's element tree\n   * @param {Event} event\n   * @return {Group | null} group\n   */\n  groupFromTarget(event) {\n    const clientY = event.center ? event.center.y : event.clientY;\n    let groupIds = this.groupIds;\n    \n    if (groupIds.length <= 0 && this.groupsData) {\n      groupIds = this.groupsData.getIds({\n        order: this.options.groupOrder\n      });\n    }\n    \n    for (let i = 0; i < groupIds.length; i++) {\n      const groupId = groupIds[i];\n      const group = this.groups[groupId];\n      const foreground = group.dom.foreground;\n      const foregroundRect = foreground.getBoundingClientRect()\n      if (clientY >= foregroundRect.top && clientY < foregroundRect.top + foreground.offsetHeight) {\n        return group;\n      }\n\n      if (this.options.orientation.item === 'top') {\n        if (i === this.groupIds.length - 1 && clientY > foregroundRect.top) {\n          return group;\n        }\n      }\n      else {\n        if (i === 0 && clientY < foregroundRect.top + foreground.offset) {\n          return group;\n        }\n      }\n    }\n\n    return null;\n  }\n\n  /**\n   * Find the ItemSet from an event target:\n   * searches for the attribute 'vis-itemset' in the event target's element tree\n   * @param {Event} event\n   * @return {ItemSet | null} item\n   */\n  static itemSetFromTarget(event) {\n    let target = event.target;\n    while (target) {\n      if (target.hasOwnProperty('vis-itemset')) {\n        return target['vis-itemset'];\n      }\n      target = target.parentNode;\n    }\n\n    return null;\n  }\n\n  /**\n   * Clone the data of an item, and \"normalize\" it: convert the start and end date\n   * to the type (Date, Moment, ...) configured in the DataSet. If not configured,\n   * start and end are converted to Date.\n   * @param {Object} itemData, typically `item.data`\n   * @param {string} [type]  Optional Date type. If not provided, the type from the DataSet is taken\n   * @return {Object} The cloned object\n   * @private\n   */\n  _cloneItemData(itemData, type) {\n    const clone = util.extend({}, itemData);\n\n    if (!type) {\n      // convert start and end date to the type (Date, Moment, ...) configured in the DataSet\n      type = this.itemsData.type;\n    }\n\n    if (clone.start != undefined) {\n      clone.start = util.convert(clone.start, type && type.start || 'Date');\n    }\n    if (clone.end != undefined) {\n      clone.end = util.convert(clone.end , type && type.end || 'Date');\n    }\n\n    return clone;\n  }\n\n  /**\n   * cluster items\n   * @return {void}   \n   * @private\n   */\n  _clusterItems() {\n    if (!this.options.cluster) {\n      return;\n    }\n\n    const { scale } = this.body.range.conversion(this.body.domProps.center.width);\n    const clusters = this.clusterGenerator.getClusters(this.clusters, scale, this.options.cluster);\n\n    if (this.clusters != clusters) {\n      this._detachAllClusters();\n\n      if (clusters) {\n        for (let cluster of clusters) {\n          cluster.attach();\n        }\n        this.clusters = clusters;\n      }\n\n      this._updateClusters(clusters);\n    }\n  }\n\n  /**\n   * detach all cluster items\n   * @private\n   */\n  _detachAllClusters() {\n    if (this.options.cluster) {\n      if (this.clusters && this.clusters.length) {\n        for (let cluster of this.clusters) {\n          cluster.detach();\n        }\n      }\n    }\n  }\n\n  /**\n   * update clusters\n   * @param {array} clusters\n   * @private\n   */\n  _updateClusters(clusters) {\n    if (this.clusters && this.clusters.length) {\n      const newClustersIds = new Set(clusters.map(cluster => cluster.id));\n      const clustersToUnselect = this.clusters.filter(cluster => !newClustersIds.has(cluster.id));\n      let selectionChanged = false;\n      for (let cluster of clustersToUnselect) {\n        const selectedIdx = this.selection.indexOf(cluster.id);\n        if (selectedIdx !== -1) {\n          cluster.unselect();\n          this.selection.splice(selectedIdx, 1);\n          selectionChanged = true;\n        }\n      }\n\n      if (selectionChanged) {\n        const newSelection = this.getSelection();\n        this.body.emitter.emit('select', {\n          items: newSelection,\n          event: event\n        });\n      }\n    }\n\n    this.clusters = clusters || [];\n  }\n}\n\n// available item types will be registered here\nItemSet.types = {\n  background: BackgroundItem,\n  box: BoxItem,\n  range: RangeItem,\n  point: PointItem\n};\n\n/**\n * Handle added items\n * @param {number[]} ids\n * @protected\n */\nItemSet.prototype._onAdd = ItemSet.prototype._onUpdate;\n\nexport default ItemSet;\n","import util from '../util';\n\nlet errorFound = false;\nlet allOptions;\nlet printStyle = 'background: #FFeeee; color: #dd0000';\n/**\n *  Used to validate options.\n */\nclass Validator {\n  /**\n   * @ignore\n   */\n  constructor() {\n  }\n\n  /**\n   * Main function to be called\n   * @param {Object} options\n   * @param {Object} referenceOptions\n   * @param {Object} subObject\n   * @returns {boolean}\n   * @static\n   */\n  static validate(options, referenceOptions, subObject) {\n    errorFound = false;\n    allOptions = referenceOptions;\n    let usedOptions = referenceOptions;\n    if (subObject !== undefined) {\n      usedOptions = referenceOptions[subObject];\n    }\n    Validator.parse(options, usedOptions, []);\n    return errorFound;\n  }\n\n\n  /**\n   * Will traverse an object recursively and check every value\n   * @param {Object} options\n   * @param {Object} referenceOptions\n   * @param {array} path    | where to look for the actual option\n   * @static\n   */\n  static parse(options, referenceOptions, path) {\n    for (let option in options) {\n      if (options.hasOwnProperty(option)) {\n        Validator.check(option, options, referenceOptions, path);\n      }\n    }\n  }\n\n\n  /**\n   * Check every value. If the value is an object, call the parse function on that object.\n   * @param {string} option\n   * @param {Object} options\n   * @param {Object} referenceOptions\n   * @param {array} path    | where to look for the actual option\n   * @static\n   */\n  static check(option, options, referenceOptions, path) {\n    if (referenceOptions[option] === undefined && referenceOptions.__any__ === undefined) {\n      Validator.getSuggestion(option, referenceOptions, path);\n      return;\n    }\n\n    let referenceOption = option;\n    let is_object = true;\n\n    if (referenceOptions[option] === undefined && referenceOptions.__any__ !== undefined) {\n      // NOTE: This only triggers if the __any__ is in the top level of the options object.\n      //       THAT'S A REALLY BAD PLACE TO ALLOW IT!!!!\n      // TODO: Examine if needed, remove if possible\n\n      // __any__ is a wildcard. Any value is accepted and will be further analysed by reference.\n      referenceOption = '__any__';\n\n      // if the any-subgroup is not a predefined object in the configurator,\n      // we do not look deeper into the object.\n      is_object = (Validator.getType(options[option]) === 'object');\n    }\n    else {\n      // Since all options in the reference are objects, we can check whether \n      // they are supposed to be the object to look for the __type__ field.\n      // if this is an object, we check if the correct type has been supplied to account for shorthand options.\n    }\n\n    let refOptionObj = referenceOptions[referenceOption];\n    if (is_object && refOptionObj.__type__ !== undefined) {\n      refOptionObj = refOptionObj.__type__;\n    }\n\n    Validator.checkFields(option, options, referenceOptions, referenceOption, refOptionObj, path);\n  }\n\n  /**\n   *\n   * @param {string}  option           | the option property\n   * @param {Object}  options          | The supplied options object\n   * @param {Object}  referenceOptions | The reference options containing all options and their allowed formats\n   * @param {string}  referenceOption  | Usually this is the same as option, except when handling an __any__ tag.\n   * @param {string}  refOptionObj     | This is the type object from the reference options\n   * @param {Array}   path             | where in the object is the option\n   * @static\n   */\n  static checkFields(option, options, referenceOptions, referenceOption, refOptionObj, path) {\n    let log = function(message) {\n      console.log('%c' + message + Validator.printLocation(path, option), printStyle);\n    };\n\n    let optionType = Validator.getType(options[option]);\n    let refOptionType = refOptionObj[optionType];\n\n    if (refOptionType !== undefined) {\n      // if the type is correct, we check if it is supposed to be one of a few select values\n      if (Validator.getType(refOptionType) === 'array' && refOptionType.indexOf(options[option]) === -1) {\n        log('Invalid option detected in \"' + option + '\".' +\n          ' Allowed values are:' + Validator.print(refOptionType) +\n          ' not \"' + options[option] + '\". ');\n        errorFound = true;\n      }\n      else if (optionType === 'object' && referenceOption !== \"__any__\") {\n        path = util.copyAndExtendArray(path, option);\n        Validator.parse(options[option], referenceOptions[referenceOption], path);\n      }\n    }\n    else if (refOptionObj['any'] === undefined) {\n      // type of the field is incorrect and the field cannot be any\n      log('Invalid type received for \"' + option +\n        '\". Expected: ' + Validator.print(Object.keys(refOptionObj)) +\n        '. Received ['  + optionType + '] \"' + options[option] + '\"');\n      errorFound = true;\n    }\n  }\n\n  /**\n   *\n   * @param {Object|boolean|number|string|Array.<number>|Date|Node|Moment|undefined|null} object\n   * @returns {string}\n   * @static\n   */\n  static getType(object) {\n    var type = typeof object;\n\n    if (type === 'object') {\n      if (object === null) {\n        return 'null';\n      }\n      if (object instanceof Boolean) {\n        return 'boolean';\n      }\n      if (object instanceof Number) {\n        return 'number';\n      }\n      if (object instanceof String) {\n        return 'string';\n      }\n      if (Array.isArray(object)) {\n        return 'array';\n      }\n      if (object instanceof Date) {\n        return 'date';\n      }\n      if (object.nodeType !== undefined) {\n        return 'dom';\n      }\n      if (object._isAMomentObject === true) {\n        return 'moment';\n      }\n      return 'object';\n    }\n    else if (type === 'number') {\n      return 'number';\n    }\n    else if (type === 'boolean') {\n      return 'boolean';\n    }\n    else if (type === 'string') {\n      return 'string';\n    }\n    else if (type === undefined) {\n      return 'undefined';\n    }\n    return type;\n  }\n\n  /**\n   * @param {string} option\n   * @param {Object} options\n   * @param {Array.<string>} path\n   * @static\n   */\n  static getSuggestion(option, options, path) {\n    let localSearch = Validator.findInOptions(option,options,path,false);\n    let globalSearch = Validator.findInOptions(option,allOptions,[],true);\n\n    let localSearchThreshold = 8;\n    let globalSearchThreshold = 4;\n\n    let msg;\n    if (localSearch.indexMatch !== undefined) {\n      msg = ' in ' + Validator.printLocation(localSearch.path, option,'') +\n        'Perhaps it was incomplete? Did you mean: \"' + localSearch.indexMatch + '\"?\\n\\n';\n    }\n    else if (globalSearch.distance <= globalSearchThreshold && localSearch.distance > globalSearch.distance) {\n      msg = ' in ' + Validator.printLocation(localSearch.path, option,'') +\n        'Perhaps it was misplaced? Matching option found at: ' +\n        Validator.printLocation(globalSearch.path, globalSearch.closestMatch,'');\n    }\n    else if (localSearch.distance <= localSearchThreshold) {\n      msg = '. Did you mean \"' + localSearch.closestMatch + '\"?' +\n        Validator.printLocation(localSearch.path, option);\n    }\n    else {\n      msg = '. Did you mean one of these: ' + Validator.print(Object.keys(options)) +\n      Validator.printLocation(path, option);\n    }\n\n    console.log('%cUnknown option detected: \"' + option + '\"' + msg, printStyle);\n    errorFound = true;\n  }\n\n  /**\n   * traverse the options in search for a match.\n   * @param {string} option\n   * @param {Object} options\n   * @param {Array} path    | where to look for the actual option\n   * @param {boolean} [recursive=false]\n   * @returns {{closestMatch: string, path: Array, distance: number}}\n   * @static\n   */\n  static findInOptions(option, options, path, recursive = false) {\n    let min = 1e9;\n    let closestMatch = '';\n    let closestMatchPath = [];\n    let lowerCaseOption = option.toLowerCase();\n    let indexMatch = undefined;\n    for (let op in options) {  // eslint-disable-line guard-for-in\n      let distance;\n      if (options[op].__type__ !== undefined && recursive === true) {\n        let result = Validator.findInOptions(option, options[op], util.copyAndExtendArray(path,op));\n        if (min > result.distance) {\n          closestMatch = result.closestMatch;\n          closestMatchPath = result.path;\n          min = result.distance;\n          indexMatch = result.indexMatch;\n        }\n      }\n      else {\n        if (op.toLowerCase().indexOf(lowerCaseOption) !== -1) {\n          indexMatch = op;\n        }\n        distance = Validator.levenshteinDistance(option, op);\n        if (min > distance) {\n          closestMatch = op;\n          closestMatchPath = util.copyArray(path);\n          min = distance;\n        }\n      }\n    }\n    return {closestMatch:closestMatch, path:closestMatchPath, distance:min, indexMatch: indexMatch};\n  }\n\n  /**\n   * @param {Array.<string>} path\n   * @param {Object} option\n   * @param {string} prefix\n   * @returns {String}\n   * @static\n   */\n  static printLocation(path, option, prefix = 'Problem value found at: \\n') {\n    let str = '\\n\\n' + prefix + 'options = {\\n';\n    for (let i = 0; i < path.length; i++) {\n      for (let j = 0; j < i + 1; j++) {\n        str += '  ';\n      }\n      str += path[i] + ': {\\n'\n    }\n    for (let j = 0; j < path.length + 1; j++) {\n      str += '  ';\n    }\n    str += option + '\\n';\n    for (let i = 0; i < path.length + 1; i++) {\n      for (let j = 0; j < path.length - i; j++) {\n        str += '  ';\n      }\n      str += '}\\n'\n    }\n    return str + '\\n\\n';\n  }\n\n  /**\n   * @param {Object} options\n   * @returns {String}\n   * @static\n   */\n  static print(options) {\n    return JSON.stringify(options).replace(/(\\\")|(\\[)|(\\])|(,\"__type__\")/g, \"\").replace(/(\\,)/g, ', ')\n  }\n\n\n  /**\n   *  Compute the edit distance between the two given strings\n   * http://en.wikibooks.org/wiki/Algorithm_Implementation/Strings/Levenshtein_distance#JavaScript\n   *\n   * Copyright (c) 2011 Andrei Mackenzie\n   *\n   * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n   *\n   * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n   *\n   * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n   *\n   * @param {string} a\n   * @param {string} b\n   * @returns {Array.<Array.<number>>}}\n   * @static\n   */\n  static levenshteinDistance(a, b) {\n    if (a.length === 0) return b.length;\n    if (b.length === 0) return a.length;\n\n    var matrix = [];\n\n    // increment along the first column of each row\n    var i;\n    for (i = 0; i <= b.length; i++) {\n      matrix[i] = [i];\n    }\n\n    // increment each column in the first row\n    var j;\n    for (j = 0; j <= a.length; j++) {\n      matrix[0][j] = j;\n    }\n\n    // Fill in the rest of the matrix\n    for (i = 1; i <= b.length; i++) {\n      for (j = 1; j <= a.length; j++) {\n        if (b.charAt(i - 1) == a.charAt(j - 1)) {\n          matrix[i][j] = matrix[i - 1][j - 1];\n        } else {\n          matrix[i][j] = Math.min(matrix[i - 1][j - 1] + 1, // substitution\n            Math.min(matrix[i][j - 1] + 1, // insertion\n              matrix[i - 1][j] + 1)); // deletion\n        }\n      }\n    }\n\n    return matrix[b.length][a.length];\n  }\n}\n\n\nexport {\n  Validator,\n  printStyle\n};\n","/**\n * This object contains all possible options. It will check if the types are correct, if required if the option is one\n * of the allowed values.\n *\n * __any__ means that the name of the property does not matter.\n * __type__ is a required field for all objects and contains the allowed types of all objects\n */\nlet string = 'string';\nlet bool = 'boolean';\nlet number = 'number';\nlet array = 'array';\nlet date = 'date';\nlet object = 'object'; // should only be in a __type__ property\nlet dom = 'dom';\nlet moment = 'moment';\nlet any = 'any';\n\nlet allOptions = {\n  configure: {\n    enabled: { 'boolean': bool},\n    filter: { 'boolean': bool,'function': 'function'},\n    container: {dom},\n    __type__: {object, 'boolean': bool,'function': 'function'}\n  },\n\n  //globals :\n  align: {string},\n  alignCurrentTime: {string, 'undefined': 'undefined'},\n  rtl: { 'boolean': bool, 'undefined': 'undefined'},\n  rollingMode: {\n    follow: { 'boolean': bool },\n    offset: {number,'undefined': 'undefined'},\n    __type__: {object}\n  },\n  onTimeout: {\n    timeoutMs: {number},\n    callback: {'function': 'function'},\n    __type__: {object}\n  },\n  verticalScroll: { 'boolean': bool, 'undefined': 'undefined'},\n  horizontalScroll: { 'boolean': bool, 'undefined': 'undefined'},\n  autoResize: { 'boolean': bool},\n  throttleRedraw: {number}, // TODO: DEPRICATED see https://github.com/almende/vis/issues/2511\n  clickToUse: { 'boolean': bool},\n  dataAttributes: {string, array},\n  editable: {\n    add: { 'boolean': bool, 'undefined': 'undefined'},\n    remove: { 'boolean': bool, 'undefined': 'undefined'},\n    updateGroup: { 'boolean': bool, 'undefined': 'undefined'},\n    updateTime: { 'boolean': bool, 'undefined': 'undefined'},\n    overrideItems: { 'boolean': bool, 'undefined': 'undefined'},\n    __type__: { 'boolean': bool, object}\n  },\n  end: {number, date, string, moment},\n  format: {\n    minorLabels: {\n      millisecond: {string,'undefined': 'undefined'},\n      second: {string,'undefined': 'undefined'},\n      minute: {string,'undefined': 'undefined'},\n      hour: {string,'undefined': 'undefined'},\n      weekday: {string,'undefined': 'undefined'},\n      day: {string,'undefined': 'undefined'},\n      week: {string,'undefined': 'undefined'},\n      month: {string,'undefined': 'undefined'},\n      year: {string,'undefined': 'undefined'},\n      __type__: {object, 'function': 'function'}\n    },\n    majorLabels: {\n      millisecond: {string,'undefined': 'undefined'},\n      second: {string,'undefined': 'undefined'},\n      minute: {string,'undefined': 'undefined'},\n      hour: {string,'undefined': 'undefined'},\n      weekday: {string,'undefined': 'undefined'},\n      day: {string,'undefined': 'undefined'},\n      week: {string,'undefined': 'undefined'},\n      month: {string,'undefined': 'undefined'},\n      year: {string,'undefined': 'undefined'},\n      __type__: {object, 'function': 'function'}\n    },\n    __type__: {object}\n  },\n  moment: {'function': 'function'},\n  groupHeightMode: {string},\n  groupOrder: {string, 'function': 'function'},\n  groupEditable: {\n    add: { 'boolean': bool, 'undefined': 'undefined'},\n    remove: { 'boolean': bool, 'undefined': 'undefined'},\n    order: { 'boolean': bool, 'undefined': 'undefined'},\n    __type__: { 'boolean': bool, object}\n  },\n  groupOrderSwap: {'function': 'function'},\n  height: {string, number},\n  hiddenDates: {\n    start: {date, number, string, moment},\n    end: {date, number, string, moment},\n    repeat: {string},\n    __type__: {object, array}\n  },\n  itemsAlwaysDraggable: {\n    item: { 'boolean': bool, 'undefined': 'undefined'},\n    range: { 'boolean': bool, 'undefined': 'undefined'},\n    __type__: { 'boolean': bool, object}\n  },\n  limitSize: {'boolean': bool},\n  locale:{string},\n  locales:{\n    __any__: {any},\n    __type__: {object}\n  },\n  longSelectPressTime: {number},\n  margin: {\n    axis: {number},\n    item: {\n      horizontal: {number,'undefined': 'undefined'},\n      vertical: {number,'undefined': 'undefined'},\n      __type__: {object,number}\n    },\n    __type__: {object,number}\n  },\n  max: {date, number, string, moment},\n  maxHeight: {number, string},\n  maxMinorChars: {number},\n  min: {date, number, string, moment},\n  minHeight: {number, string},\n  moveable: { 'boolean': bool},\n  multiselect: { 'boolean': bool},\n  multiselectPerGroup: { 'boolean': bool},\n  onAdd: {'function': 'function'},\n  onDropObjectOnItem: {'function': 'function'},\n  onUpdate: {'function': 'function'},\n  onMove: {'function': 'function'},\n  onMoving: {'function': 'function'},\n  onRemove: {'function': 'function'},\n  onAddGroup: {'function': 'function'},\n  onMoveGroup: {'function': 'function'},\n  onRemoveGroup: {'function': 'function'},\n  onInitialDrawComplete: {'function': 'function'},\n  order: {'function': 'function'},\n  orientation: {\n    axis: {string,'undefined': 'undefined'},\n    item: {string,'undefined': 'undefined'},\n    __type__: {string, object}\n  },\n  selectable: { 'boolean': bool},\n  sequentialSelection: { 'boolean': bool },\n  showCurrentTime: { 'boolean': bool},\n  showMajorLabels: { 'boolean': bool},\n  showMinorLabels: { 'boolean': bool},\n  showWeekScale: { 'boolean': bool},\n  stack: { 'boolean': bool},\n  stackSubgroups: { 'boolean': bool},\n  cluster: { \n    maxItems: {'number': number, 'undefined': 'undefined'},\n    titleTemplate: {'string': string, 'undefined': 'undefined'},\n    clusterCriteria: { 'function': 'function', 'undefined': 'undefined'},\n    showStipes: {'boolean': bool, 'undefined': 'undefined'},\n    fitOnDoubleClick: {'boolean': bool, 'undefined': 'undefined'},\n    __type__: {'boolean': bool, object}\n  },\n  snap: {'function': 'function', 'null': 'null'},\n  start: {date, number, string, moment},\n  template: {'function': 'function'},\n  loadingScreenTemplate: {'function': 'function'},\n  groupTemplate: {'function': 'function'},\n  visibleFrameTemplate: {string, 'function': 'function'},\n  showTooltips: { 'boolean': bool},\n  tooltip: {\n    followMouse: { 'boolean': bool },\n    overflowMethod: { 'string': ['cap', 'flip', 'none'] },\n    delay: {number},\n    template: {'function': 'function'},\n    __type__: {object}\n  },\n  tooltipOnItemUpdateTime: {\n    template: {'function': 'function'},\n    __type__: { 'boolean': bool, object}\n  },\n  timeAxis: {\n    scale: {string,'undefined': 'undefined'},\n    step: {number,'undefined': 'undefined'},\n    __type__: {object}\n  },\n  type: {string},\n  width: {string, number},\n  preferZoom: { 'boolean': bool},\n  zoomable: { 'boolean': bool},\n  zoomKey: {string: ['ctrlKey', 'altKey', 'shiftKey', 'metaKey', '']},\n  zoomFriction: {number},\n  zoomMax: {number},\n  zoomMin: {number},\n\n  __type__: {object}\n};\n\nlet configureOptions = {\n  global: {\n    align:  ['center', 'left', 'right'],\n    alignCurrentTime: ['none', 'year', 'month', 'quarter', 'week', 'isoWeek', 'day', 'date', 'hour', 'minute', 'second'],\n    direction:  false,\n    autoResize: true,\n    clickToUse: false,\n    // dataAttributes: ['all'], // FIXME: can be 'all' or string[]\n      editable: {\n      add: false,\n      remove: false,\n      updateGroup: false,\n      updateTime: false\n    },\n    end: '',\n    format: {\n      minorLabels: {\n        millisecond:'SSS',\n        second:     's',\n        minute:     'HH:mm',\n        hour:       'HH:mm',\n        weekday:    'ddd D',\n        day:        'D',\n        week:       'w',\n        month:      'MMM',\n        year:       'YYYY'\n      },\n      majorLabels: {\n        millisecond:'HH:mm:ss',\n        second:     'D MMMM HH:mm',\n        minute:     'ddd D MMMM',\n        hour:       'ddd D MMMM',\n        weekday:    'MMMM YYYY',\n        day:        'MMMM YYYY',\n        week:       'MMMM YYYY',\n        month:      'YYYY',\n        year:       ''\n      }\n    },\n    groupHeightMode: ['auto', 'fixed', 'fitItems'],\n    //groupOrder: {string, 'function': 'function'},\n    groupsDraggable: false,\n    height: '',\n    //hiddenDates: {object, array},\n    locale: '',\n    longSelectPressTime: 251,\n    margin: {\n      axis: [20, 0, 100, 1],\n      item: {\n        horizontal: [10, 0, 100, 1],\n        vertical: [10, 0, 100, 1]\n      }\n    },\n    max: '',\n    maxHeight: '',\n    maxMinorChars: [7, 0, 20, 1],\n    min: '',\n    minHeight: '',\n    moveable: false,\n    multiselect: false,\n    multiselectPerGroup: false,\n    //onAdd: {'function': 'function'},\n    //onUpdate: {'function': 'function'},\n    //onMove: {'function': 'function'},\n    //onMoving: {'function': 'function'},\n    //onRename: {'function': 'function'},\n    //order: {'function': 'function'},\n    orientation: {\n      axis: ['both', 'bottom', 'top'],\n      item: ['bottom', 'top']\n    },\n    preferZoom: false,\n    selectable: true,\n    showCurrentTime: false,\n    showMajorLabels: true,\n    showMinorLabels: true,\n    stack: true,\n    stackSubgroups: true,\n    cluster: false,\n    //snap: {'function': 'function', nada},\n    start: '',\n    //template: {'function': 'function'},\n    //timeAxis: {\n    //  scale: ['millisecond', 'second', 'minute', 'hour', 'weekday', 'day', 'week', 'month', 'year'],\n    //  step: [1, 1, 10, 1]\n    //},\n    showTooltips: true,\n    tooltip: {\n      followMouse: false,\n      overflowMethod: 'flip',\n      delay: [500, 0, 99999, 100],\n    },\n    tooltipOnItemUpdateTime: false,\n    type: ['box', 'point', 'range', 'background'],\n    width: '100%',\n    zoomable: true,\n    zoomKey: ['ctrlKey', 'altKey', 'shiftKey', 'metaKey', ''],\n    zoomMax: [315360000000000, 10, 315360000000000, 1],\n    zoomMin: [10, 10, 315360000000000, 1]\n  }\n};\n\nexport {allOptions, configureOptions};\n","var $ = require('../internals/export');\nvar fill = require('../internals/array-fill');\nvar addToUnscopables = require('../internals/add-to-unscopables');\n\n// `Array.prototype.fill` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.fill\n$({ target: 'Array', proto: true }, {\n  fill: fill\n});\n\n// https://tc39.github.io/ecma262/#sec-array.prototype-@@unscopables\naddToUnscopables('fill');\n","'use strict';\nvar toObject = require('../internals/to-object');\nvar toAbsoluteIndex = require('../internals/to-absolute-index');\nvar toLength = require('../internals/to-length');\n\n// `Array.prototype.fill` method implementation\n// https://tc39.github.io/ecma262/#sec-array.prototype.fill\nmodule.exports = function fill(value /* , start = 0, end = @length */) {\n  var O = toObject(this);\n  var length = toLength(O.length);\n  var argumentsLength = arguments.length;\n  var index = toAbsoluteIndex(argumentsLength > 1 ? arguments[1] : undefined, length);\n  var end = argumentsLength > 2 ? arguments[2] : undefined;\n  var endPos = end === undefined ? length : toAbsoluteIndex(end, length);\n  while (endPos > index) O[index++] = value;\n  return O;\n};\n","require('../../../modules/es.array.fill');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').fill;\n","var fill = require('../array/virtual/fill');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.fill;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.fill) ? fill : own;\n};\n","import Hammer from '../module/hammer';\nimport * as hammerUtil from '../hammerUtil';\nimport util from '../util';\n\n\nvar htmlColors = {black: '#000000', navy: '#000080', darkblue: '#00008B', mediumblue: '#0000CD', blue: '#0000FF', darkgreen: '#006400', green: '#008000', teal: '#008080', darkcyan: '#008B8B', deepskyblue: '#00BFFF', darkturquoise: '#00CED1', mediumspringgreen: '#00FA9A', lime: '#00FF00', springgreen: '#00FF7F', aqua: '#00FFFF', cyan: '#00FFFF', midnightblue: '#191970', dodgerblue: '#1E90FF', lightseagreen: '#20B2AA', forestgreen: '#228B22', seagreen: '#2E8B57', darkslategray: '#2F4F4F', limegreen: '#32CD32', mediumseagreen: '#3CB371', turquoise: '#40E0D0', royalblue: '#4169E1', steelblue: '#4682B4', darkslateblue: '#483D8B', mediumturquoise: '#48D1CC', indigo: '#4B0082', darkolivegreen: '#556B2F', cadetblue: '#5F9EA0', cornflowerblue: '#6495ED', mediumaquamarine: '#66CDAA', dimgray: '#696969', slateblue: '#6A5ACD', olivedrab: '#6B8E23', slategray: '#708090', lightslategray: '#778899', mediumslateblue: '#7B68EE', lawngreen: '#7CFC00', chartreuse: '#7FFF00', aquamarine: '#7FFFD4', maroon: '#800000', purple: '#800080', olive: '#808000', gray: '#808080', skyblue: '#87CEEB', lightskyblue: '#87CEFA', blueviolet: '#8A2BE2', darkred: '#8B0000', darkmagenta: '#8B008B', saddlebrown: '#8B4513', darkseagreen: '#8FBC8F', lightgreen: '#90EE90', mediumpurple: '#9370D8', darkviolet: '#9400D3', palegreen: '#98FB98', darkorchid: '#9932CC', yellowgreen: '#9ACD32', sienna: '#A0522D', brown: '#A52A2A', darkgray: '#A9A9A9', lightblue: '#ADD8E6', greenyellow: '#ADFF2F', paleturquoise: '#AFEEEE', lightsteelblue: '#B0C4DE', powderblue: '#B0E0E6', firebrick: '#B22222', darkgoldenrod: '#B8860B', mediumorchid: '#BA55D3', rosybrown: '#BC8F8F', darkkhaki: '#BDB76B', silver: '#C0C0C0', mediumvioletred: '#C71585', indianred: '#CD5C5C', peru: '#CD853F', chocolate: '#D2691E', tan: '#D2B48C', lightgrey: '#D3D3D3', palevioletred: '#D87093', thistle: '#D8BFD8', orchid: '#DA70D6', goldenrod: '#DAA520', crimson: '#DC143C', gainsboro: '#DCDCDC', plum: '#DDA0DD', burlywood: '#DEB887', lightcyan: '#E0FFFF', lavender: '#E6E6FA', darksalmon: '#E9967A', violet: '#EE82EE', palegoldenrod: '#EEE8AA', lightcoral: '#F08080', khaki: '#F0E68C', aliceblue: '#F0F8FF', honeydew: '#F0FFF0', azure: '#F0FFFF', sandybrown: '#F4A460', wheat: '#F5DEB3', beige: '#F5F5DC', whitesmoke: '#F5F5F5', mintcream: '#F5FFFA', ghostwhite: '#F8F8FF', salmon: '#FA8072', antiquewhite: '#FAEBD7', linen: '#FAF0E6', lightgoldenrodyellow: '#FAFAD2', oldlace: '#FDF5E6', red: '#FF0000', fuchsia: '#FF00FF', magenta: '#FF00FF', deeppink: '#FF1493', orangered: '#FF4500', tomato: '#FF6347', hotpink: '#FF69B4', coral: '#FF7F50', darkorange: '#FF8C00', lightsalmon: '#FFA07A', orange: '#FFA500', lightpink: '#FFB6C1', pink: '#FFC0CB', gold: '#FFD700', peachpuff: '#FFDAB9', navajowhite: '#FFDEAD', moccasin: '#FFE4B5', bisque: '#FFE4C4', mistyrose: '#FFE4E1', blanchedalmond: '#FFEBCD', papayawhip: '#FFEFD5', lavenderblush: '#FFF0F5', seashell: '#FFF5EE', cornsilk: '#FFF8DC', lemonchiffon: '#FFFACD', floralwhite: '#FFFAF0', snow: '#FFFAFA', yellow: '#FFFF00', lightyellow: '#FFFFE0', ivory: '#FFFFF0', white: '#FFFFFF'};\n\n/**\n * @param {number} [pixelRatio=1]\n */\nclass ColorPicker {\n  /**\n   * @param {number} [pixelRatio=1]\n   */\n  constructor(pixelRatio = 1) {\n    this.pixelRatio = pixelRatio;\n    this.generated = false;\n    this.centerCoordinates = {x:289/2, y:289/2};\n    this.r = 289 * 0.49;\n    this.color = {r:255,g:255,b:255,a:1.0};\n    this.hueCircle = undefined;\n    this.initialColor = {r:255,g:255,b:255,a:1.0};\n    this.previousColor= undefined;\n    this.applied = false;\n\n    // bound by\n    this.updateCallback = () => {};\n    this.closeCallback = () => {};\n\n    // create all DOM elements\n    this._create();\n  }\n\n\n  /**\n   * this inserts the colorPicker into a div from the DOM\n   * @param {Element} container\n   */\n  insertTo(container) {\n    if (this.hammer !== undefined) {\n      this.hammer.destroy();\n      this.hammer = undefined;\n    }\n    this.container = container;\n    this.container.appendChild(this.frame);\n    this._bindHammer();\n\n    this._setSize();\n  }\n\n  /**\n   * the callback is executed on apply and save. Bind it to the application\n   * @param {function} callback\n   */\n  setUpdateCallback(callback) {\n    if (typeof callback === 'function') {\n      this.updateCallback = callback;\n    }\n    else {\n      throw new Error(\"Function attempted to set as colorPicker update callback is not a function.\");\n    }\n  }\n\n  /**\n   * the callback is executed on apply and save. Bind it to the application\n   * @param {function} callback\n   */\n  setCloseCallback(callback) {\n    if (typeof callback === 'function') {\n      this.closeCallback = callback;\n    }\n    else {\n      throw new Error(\"Function attempted to set as colorPicker closing callback is not a function.\");\n    }\n  }\n\n  /**\n   *\n   * @param {string} color\n   * @returns {String}\n   * @private\n   */\n  _isColorString(color) {\n    if (typeof color === 'string') {\n      return htmlColors[color];\n    }\n  }\n\n\n  /**\n   * Set the color of the colorPicker\n   * Supported formats:\n   * 'red'                   --> HTML color string\n   * '#ffffff'               --> hex string\n   * 'rgb(255,255,255)'      --> rgb string\n   * 'rgba(255,255,255,1.0)' --> rgba string\n   * {r:255,g:255,b:255}     --> rgb object\n   * {r:255,g:255,b:255,a:1.0} --> rgba object\n   * @param {string|Object} color\n   * @param {boolean} [setInitial=true]\n   */\n  setColor(color, setInitial = true) {\n    if (color === 'none') {\n      return;\n    }\n\n    let rgba;\n\n    // if a html color shorthand is used, convert to hex\n    var htmlColor = this._isColorString(color);\n    if (htmlColor !== undefined) {\n      color = htmlColor;\n    }\n\n    // check format\n    if (util.isString(color) === true) {\n      if (util.isValidRGB(color) === true) {\n        let rgbaArray = color.substr(4).substr(0, color.length - 5).split(',');\n        rgba = {r:rgbaArray[0], g:rgbaArray[1], b:rgbaArray[2], a:1.0};\n      }\n      else if (util.isValidRGBA(color) === true) {\n        let rgbaArray = color.substr(5).substr(0, color.length - 6).split(',');\n        rgba = {r:rgbaArray[0], g:rgbaArray[1], b:rgbaArray[2], a:rgbaArray[3]};\n      }\n      else if (util.isValidHex(color) === true) {\n        let rgbObj = util.hexToRGB(color);\n        rgba = {r:rgbObj.r, g:rgbObj.g, b:rgbObj.b, a:1.0};\n      }\n    }\n    else {\n      if (color instanceof Object) {\n        if (color.r !== undefined && color.g !== undefined && color.b !== undefined) {\n          let alpha = color.a !== undefined ? color.a : '1.0';\n          rgba = {r:color.r, g:color.g, b:color.b, a:alpha};\n        }\n      }\n    }\n\n    // set color\n    if (rgba === undefined) {\n      throw new Error(\"Unknown color passed to the colorPicker. Supported are strings: rgb, hex, rgba. Object: rgb ({r:r,g:g,b:b,[a:a]}). Supplied: \" + JSON.stringify(color));\n    }\n    else {\n      this._setColor(rgba, setInitial);\n    }\n  }\n\n\n  /**\n   * this shows the color picker.\n   * The hue circle is constructed once and stored.\n   */\n  show() {\n    if (this.closeCallback !== undefined) {\n      this.closeCallback();\n      this.closeCallback = undefined;\n    }\n\n    this.applied = false;\n    this.frame.style.display = 'block';\n    this._generateHueCircle();\n  }\n\n  // ------------------------------------------ PRIVATE ----------------------------- //\n\n  /**\n   * Hide the picker. Is called by the cancel button.\n   * Optional boolean to store the previous color for easy access later on.\n   * @param {boolean} [storePrevious=true]\n   * @private\n   */\n  _hide(storePrevious = true) {\n    // store the previous color for next time;\n    if (storePrevious === true) {\n      this.previousColor = util.extend({}, this.color);\n    }\n\n    if (this.applied === true) {\n      this.updateCallback(this.initialColor);\n    }\n\n    this.frame.style.display = 'none';\n\n    // call the closing callback, restoring the onclick method.\n    // this is in a setTimeout because it will trigger the show again before the click is done.\n    setTimeout(() => {\n      if (this.closeCallback !== undefined) {\n        this.closeCallback();\n        this.closeCallback = undefined;\n      }\n    },0);\n  }\n\n\n  /**\n   * bound to the save button. Saves and hides.\n   * @private\n   */\n  _save() {\n    this.updateCallback(this.color);\n    this.applied = false;\n    this._hide();\n  }\n\n\n  /**\n   * Bound to apply button. Saves but does not close. Is undone by the cancel button.\n   * @private\n   */\n  _apply() {\n    this.applied = true;\n    this.updateCallback(this.color);\n    this._updatePicker(this.color);\n  }\n\n\n  /**\n   * load the color from the previous session.\n   * @private\n   */\n  _loadLast() {\n    if (this.previousColor !== undefined) {\n      this.setColor(this.previousColor, false);\n    }\n    else {\n      alert(\"There is no last color to load...\");\n    }\n  }\n\n\n  /**\n   * set the color, place the picker\n   * @param {Object} rgba\n   * @param {boolean} [setInitial=true]\n   * @private\n   */\n  _setColor(rgba, setInitial = true) {\n    // store the initial color\n    if (setInitial === true) {\n      this.initialColor = util.extend({}, rgba);\n    }\n\n    this.color = rgba;\n    let hsv = util.RGBToHSV(rgba.r, rgba.g, rgba.b);\n\n    let angleConvert = 2 * Math.PI;\n    let radius = this.r * hsv.s;\n    let x = this.centerCoordinates.x + radius * Math.sin(angleConvert * hsv.h);\n    let y = this.centerCoordinates.y + radius * Math.cos(angleConvert * hsv.h);\n\n    this.colorPickerSelector.style.left = x - 0.5 * this.colorPickerSelector.clientWidth + 'px';\n    this.colorPickerSelector.style.top = y - 0.5 * this.colorPickerSelector.clientHeight + 'px';\n\n    this._updatePicker(rgba);\n  }\n\n\n  /**\n   * bound to opacity control\n   * @param {number} value\n   * @private\n   */\n  _setOpacity(value) {\n    this.color.a = value / 100;\n    this._updatePicker(this.color);\n  }\n\n\n  /**\n   * bound to brightness control\n   * @param {number} value\n   * @private\n   */\n  _setBrightness(value) {\n    let hsv = util.RGBToHSV(this.color.r, this.color.g, this.color.b);\n    hsv.v = value / 100;\n    let rgba = util.HSVToRGB(hsv.h, hsv.s, hsv.v);\n    rgba['a'] = this.color.a;\n    this.color = rgba;\n    this._updatePicker();\n  }\n\n\n  /**\n   * update the color picker. A black circle overlays the hue circle to mimic the brightness decreasing.\n   * @param {Object} rgba\n   * @private\n   */\n  _updatePicker(rgba = this.color) {\n    let hsv = util.RGBToHSV(rgba.r, rgba.g, rgba.b);\n    let ctx = this.colorPickerCanvas.getContext('2d');\n    if (this.pixelRation === undefined) {\n      this.pixelRatio = (window.devicePixelRatio || 1) / (ctx.webkitBackingStorePixelRatio ||\n      ctx.mozBackingStorePixelRatio ||\n      ctx.msBackingStorePixelRatio ||\n      ctx.oBackingStorePixelRatio ||\n      ctx.backingStorePixelRatio || 1);\n    }\n    ctx.setTransform(this.pixelRatio, 0, 0, this.pixelRatio, 0, 0);\n\n    // clear the canvas\n    let w = this.colorPickerCanvas.clientWidth;\n    let h = this.colorPickerCanvas.clientHeight;\n    ctx.clearRect(0, 0, w, h);\n\n    ctx.putImageData(this.hueCircle, 0,0);\n    ctx.fillStyle = 'rgba(0,0,0,' + (1- hsv.v) + ')';\n    ctx.circle(this.centerCoordinates.x, this.centerCoordinates.y, this.r);\n    ctx.fill();\n\n    this.brightnessRange.value = 100 * hsv.v;\n    this.opacityRange.value    = 100 * rgba.a;\n\n    this.initialColorDiv.style.backgroundColor = 'rgba(' + this.initialColor.r + ',' + this.initialColor.g + ',' + this.initialColor.b + ',' + this.initialColor.a + ')';\n    this.newColorDiv.style.backgroundColor = 'rgba(' + this.color.r + ',' + this.color.g + ',' + this.color.b + ',' + this.color.a + ')';\n  }\n\n\n  /**\n   * used by create to set the size of the canvas.\n   * @private\n   */\n  _setSize() {\n    this.colorPickerCanvas.style.width = '100%';\n    this.colorPickerCanvas.style.height = '100%';\n\n    this.colorPickerCanvas.width = 289 * this.pixelRatio;\n    this.colorPickerCanvas.height = 289 * this.pixelRatio;\n  }\n\n\n  /**\n   * create all dom elements\n   * TODO: cleanup, lots of similar dom elements\n   * @private\n   */\n  _create() {\n    this.frame = document.createElement('div');\n    this.frame.className = 'vis-color-picker';\n\n    this.colorPickerDiv = document.createElement('div');\n    this.colorPickerSelector = document.createElement('div');\n    this.colorPickerSelector.className = 'vis-selector';\n    this.colorPickerDiv.appendChild(this.colorPickerSelector);\n\n    this.colorPickerCanvas = document.createElement('canvas');\n    this.colorPickerDiv.appendChild(this.colorPickerCanvas);\n\n    if (!this.colorPickerCanvas.getContext) {\n      let noCanvas = document.createElement( 'DIV' );\n      noCanvas.style.color = 'red';\n      noCanvas.style.fontWeight =  'bold' ;\n      noCanvas.style.padding =  '10px';\n      noCanvas.innerHTML =  'Error: your browser does not support HTML canvas';\n      this.colorPickerCanvas.appendChild(noCanvas);\n    }\n    else {\n      let ctx = this.colorPickerCanvas.getContext(\"2d\");\n      this.pixelRatio = (window.devicePixelRatio || 1) / (ctx.webkitBackingStorePixelRatio ||\n      ctx.mozBackingStorePixelRatio ||\n      ctx.msBackingStorePixelRatio ||\n      ctx.oBackingStorePixelRatio ||\n      ctx.backingStorePixelRatio || 1);\n      this.colorPickerCanvas.getContext(\"2d\").setTransform(this.pixelRatio, 0, 0, this.pixelRatio, 0, 0);\n    }\n\n    this.colorPickerDiv.className = 'vis-color';\n\n    this.opacityDiv = document.createElement('div');\n    this.opacityDiv.className = 'vis-opacity';\n\n    this.brightnessDiv = document.createElement('div');\n    this.brightnessDiv.className = 'vis-brightness';\n\n    this.arrowDiv = document.createElement('div');\n    this.arrowDiv.className = 'vis-arrow';\n\n    this.opacityRange = document.createElement('input');\n    try {\n      this.opacityRange.type = 'range'; // Not supported on IE9\n      this.opacityRange.min = '0';\n      this.opacityRange.max = '100';\n    }\n    // TODO: Add some error handling and remove this lint exception\n    catch (err) {}  // eslint-disable-line no-empty\n    this.opacityRange.value = '100';\n    this.opacityRange.className = 'vis-range';\n\n    this.brightnessRange = document.createElement('input');\n    try {\n      this.brightnessRange.type = 'range'; // Not supported on IE9\n      this.brightnessRange.min = '0';\n      this.brightnessRange.max = '100';\n    }\n    // TODO: Add some error handling and remove this lint exception\n    catch (err) {}  // eslint-disable-line no-empty\n    this.brightnessRange.value = '100';\n    this.brightnessRange.className = 'vis-range';\n\n    this.opacityDiv.appendChild(this.opacityRange);\n    this.brightnessDiv.appendChild(this.brightnessRange);\n\n    var me = this;\n    this.opacityRange.onchange = function () {me._setOpacity(this.value);};\n    this.opacityRange.oninput  = function () {me._setOpacity(this.value);};\n    this.brightnessRange.onchange = function () {me._setBrightness(this.value);};\n    this.brightnessRange.oninput  = function () {me._setBrightness(this.value);};\n\n    this.brightnessLabel = document.createElement(\"div\");\n    this.brightnessLabel.className = \"vis-label vis-brightness\";\n    this.brightnessLabel.innerHTML = 'brightness:';\n\n    this.opacityLabel = document.createElement(\"div\");\n    this.opacityLabel.className = \"vis-label vis-opacity\";\n    this.opacityLabel.innerHTML = 'opacity:';\n\n    this.newColorDiv = document.createElement(\"div\");\n    this.newColorDiv.className = \"vis-new-color\";\n    this.newColorDiv.innerHTML = 'new';\n\n    this.initialColorDiv = document.createElement(\"div\");\n    this.initialColorDiv.className = \"vis-initial-color\";\n    this.initialColorDiv.innerHTML = 'initial';\n\n    this.cancelButton = document.createElement(\"div\");\n    this.cancelButton.className = \"vis-button vis-cancel\";\n    this.cancelButton.innerHTML = 'cancel';\n    this.cancelButton.onclick = this._hide.bind(this, false);\n\n    this.applyButton = document.createElement(\"div\");\n    this.applyButton.className = \"vis-button vis-apply\";\n    this.applyButton.innerHTML = 'apply';\n    this.applyButton.onclick = this._apply.bind(this);\n\n    this.saveButton = document.createElement(\"div\");\n    this.saveButton.className = \"vis-button vis-save\";\n    this.saveButton.innerHTML = 'save';\n    this.saveButton.onclick = this._save.bind(this);\n\n    this.loadButton = document.createElement(\"div\");\n    this.loadButton.className = \"vis-button vis-load\";\n    this.loadButton.innerHTML = 'load last';\n    this.loadButton.onclick = this._loadLast.bind(this);\n\n    this.frame.appendChild(this.colorPickerDiv);\n    this.frame.appendChild(this.arrowDiv);\n    this.frame.appendChild(this.brightnessLabel);\n    this.frame.appendChild(this.brightnessDiv);\n    this.frame.appendChild(this.opacityLabel);\n    this.frame.appendChild(this.opacityDiv);\n    this.frame.appendChild(this.newColorDiv);\n    this.frame.appendChild(this.initialColorDiv);\n\n    this.frame.appendChild(this.cancelButton);\n    this.frame.appendChild(this.applyButton);\n    this.frame.appendChild(this.saveButton);\n    this.frame.appendChild(this.loadButton);\n  }\n\n\n  /**\n   * bind hammer to the color picker\n   * @private\n   */\n  _bindHammer() {\n    this.drag = {};\n    this.pinch = {};\n    this.hammer = new Hammer(this.colorPickerCanvas);\n    this.hammer.get('pinch').set({enable: true});\n\n    hammerUtil.onTouch(this.hammer, (event) => {this._moveSelector(event)});\n    this.hammer.on('tap',       (event) => {this._moveSelector(event)});\n    this.hammer.on('panstart',  (event) => {this._moveSelector(event)});\n    this.hammer.on('panmove',   (event) => {this._moveSelector(event)});\n    this.hammer.on('panend',    (event) => {this._moveSelector(event)});\n  }\n\n\n  /**\n   * generate the hue circle. This is relatively heavy (200ms) and is done only once on the first time it is shown.\n   * @private\n   */\n  _generateHueCircle() {\n    if (this.generated === false) {\n      let ctx = this.colorPickerCanvas.getContext('2d');\n      if (this.pixelRation === undefined) {\n        this.pixelRatio = (window.devicePixelRatio || 1) / (ctx.webkitBackingStorePixelRatio ||\n        ctx.mozBackingStorePixelRatio ||\n        ctx.msBackingStorePixelRatio ||\n        ctx.oBackingStorePixelRatio ||\n        ctx.backingStorePixelRatio || 1);\n      }\n      ctx.setTransform(this.pixelRatio, 0, 0, this.pixelRatio, 0, 0);\n\n      // clear the canvas\n      let w = this.colorPickerCanvas.clientWidth;\n      let h = this.colorPickerCanvas.clientHeight;\n      ctx.clearRect(0, 0, w, h);\n\n\n      // draw hue circle\n      let x, y, hue, sat;\n      this.centerCoordinates = {x: w * 0.5, y: h * 0.5};\n      this.r = 0.49 * w;\n      let angleConvert = (2 * Math.PI) / 360;\n      let hfac = 1 / 360;\n      let sfac = 1 / this.r;\n      let rgb;\n      for (hue = 0; hue < 360; hue++) {\n        for (sat = 0; sat < this.r; sat++) {\n          x = this.centerCoordinates.x + sat * Math.sin(angleConvert * hue);\n          y = this.centerCoordinates.y + sat * Math.cos(angleConvert * hue);\n          rgb = util.HSVToRGB(hue * hfac, sat * sfac, 1);\n          ctx.fillStyle = 'rgb(' + rgb.r + ',' + rgb.g + ',' + rgb.b + ')';\n          ctx.fillRect(x - 0.5, y - 0.5, 2, 2);\n        }\n      }\n      ctx.strokeStyle = 'rgba(0,0,0,1)';\n      ctx.circle(this.centerCoordinates.x, this.centerCoordinates.y, this.r);\n      ctx.stroke();\n\n      this.hueCircle = ctx.getImageData(0,0,w,h);\n    }\n    this.generated = true;\n  }\n\n\n  /**\n   * move the selector. This is called by hammer functions.\n   *\n   * @param {Event}  event   The event\n   * @private\n   */\n  _moveSelector(event) {\n    let rect = this.colorPickerDiv.getBoundingClientRect();\n    let left = event.center.x - rect.left;\n    let top = event.center.y - rect.top;\n\n    let centerY = 0.5 * this.colorPickerDiv.clientHeight;\n    let centerX = 0.5 * this.colorPickerDiv.clientWidth;\n\n    let x = left - centerX;\n    let y = top - centerY;\n\n    let angle = Math.atan2(x,y);\n    let radius = 0.98 * Math.min(Math.sqrt(x * x + y * y), centerX);\n\n    let newTop = Math.cos(angle) * radius + centerY;\n    let newLeft = Math.sin(angle) * radius + centerX;\n\n    this.colorPickerSelector.style.top = newTop - 0.5 * this.colorPickerSelector.clientHeight + 'px';\n    this.colorPickerSelector.style.left = newLeft - 0.5 * this.colorPickerSelector.clientWidth + 'px';\n\n    // set color\n    let h = angle / (2 * Math.PI);\n    h = h < 0 ? h + 1 : h;\n    let s = radius / this.r;\n    let hsv = util.RGBToHSV(this.color.r, this.color.g, this.color.b);\n    hsv.h = h;\n    hsv.s = s;\n    let rgba = util.HSVToRGB(hsv.h, hsv.s, hsv.v);\n    rgba['a'] = this.color.a;\n    this.color = rgba;\n\n    // update previews\n    this.initialColorDiv.style.backgroundColor = 'rgba(' + this.initialColor.r + ',' + this.initialColor.g + ',' + this.initialColor.b + ',' + this.initialColor.a + ')';\n    this.newColorDiv.style.backgroundColor = 'rgba(' + this.color.r + ',' + this.color.g + ',' + this.color.b + ',' + this.color.a + ')';\n  }\n}\n\nexport default ColorPicker;\n","import util from '../util';\nimport ColorPicker from './ColorPicker';\n\nimport './configuration.css';\n\n/**\n * The way this works is for all properties of this.possible options, you can supply the property name in any form to list the options.\n * Boolean options are recognised as Boolean\n * Number options should be written as array: [default value, min value, max value, stepsize]\n * Colors should be written as array: ['color', '#ffffff']\n * Strings with should be written as array: [option1, option2, option3, ..]\n *\n * The options are matched with their counterparts in each of the modules and the values used in the configuration are\n */\nclass Configurator {\n  /**\n   * @param {Object} parentModule        | the location where parentModule.setOptions() can be called\n   * @param {Object} defaultContainer    | the default container of the module\n   * @param {Object} configureOptions    | the fully configured and predefined options set found in allOptions.js\n   * @param {number} pixelRatio          | canvas pixel ratio\n   */\n  constructor(parentModule, defaultContainer, configureOptions, pixelRatio = 1) {\n    this.parent = parentModule;\n    this.changedOptions = [];\n    this.container = defaultContainer;\n    this.allowCreation = false;\n\n    this.options = {};\n    this.initialized = false;\n    this.popupCounter = 0;\n    this.defaultOptions = {\n      enabled: false,\n      filter: true,\n      container: undefined,\n      showButton: true\n    };\n    util.extend(this.options, this.defaultOptions);\n\n    this.configureOptions = configureOptions;\n    this.moduleOptions = {};\n    this.domElements = [];\n    this.popupDiv = {};\n    this.popupLimit = 5;\n    this.popupHistory = {};\n    this.colorPicker = new ColorPicker(pixelRatio);\n    this.wrapper = undefined;\n  }\n\n\n  /**\n   * refresh all options.\n   * Because all modules parse their options by themselves, we just use their options. We copy them here.\n   *\n   * @param {Object} options\n   */\n  setOptions(options) {\n    if (options !== undefined) {\n      // reset the popup history because the indices may have been changed.\n      this.popupHistory = {};\n      this._removePopup();\n\n      let enabled = true;\n      if (typeof options === 'string') {\n        this.options.filter = options;\n      }\n      else if (options instanceof Array) {\n        this.options.filter = options.join();\n      }\n      else if (typeof options === 'object') {\n        if (options == null) {\n          throw new TypeError('options cannot be null');\n        }\n        if (options.container !== undefined) {\n          this.options.container = options.container;\n        }\n        if (options.filter !== undefined) {\n          this.options.filter = options.filter;\n        }\n        if (options.showButton !== undefined) {\n          this.options.showButton = options.showButton;\n        }\n        if (options.enabled !== undefined) {\n          enabled = options.enabled;\n        }\n      }\n      else if (typeof options === 'boolean') {\n        this.options.filter = true;\n        enabled = options;\n      }\n      else if (typeof options === 'function') {\n        this.options.filter = options;\n        enabled = true;\n      }\n      if (this.options.filter === false) {\n        enabled = false;\n      }\n\n      this.options.enabled = enabled;\n    }\n    this._clean();\n  }\n\n  /**\n   *\n   * @param {Object} moduleOptions\n   */\n  setModuleOptions(moduleOptions) {\n    this.moduleOptions = moduleOptions;\n    if (this.options.enabled === true) {\n      this._clean();\n      if (this.options.container !== undefined) {\n        this.container = this.options.container;\n      }\n      this._create();\n    }\n  }\n\n  /**\n   * Create all DOM elements\n   * @private\n   */\n  _create() {\n    this._clean();\n    this.changedOptions = [];\n\n    let filter = this.options.filter;\n    let counter = 0;\n    let show = false;\n    for (let option in this.configureOptions) {\n      if (this.configureOptions.hasOwnProperty(option)) {\n        this.allowCreation = false;\n        show = false;\n        if (typeof filter === 'function') {\n          show = filter(option,[]);\n          show = show || this._handleObject(this.configureOptions[option], [option], true);\n        }\n        else if (filter === true || filter.indexOf(option) !== -1) {\n          show = true;\n        }\n\n        if (show !== false) {\n          this.allowCreation = true;\n\n          // linebreak between categories\n          if (counter > 0) {\n            this._makeItem([]);\n          }\n          // a header for the category\n          this._makeHeader(option);\n\n          // get the sub options\n          this._handleObject(this.configureOptions[option], [option]);\n        }\n        counter++;\n      }\n    }\n    this._makeButton();\n    this._push();\n    //~ this.colorPicker.insertTo(this.container);\n  }\n\n\n  /**\n   * draw all DOM elements on the screen\n   * @private\n   */\n  _push() {\n    this.wrapper = document.createElement('div');\n    this.wrapper.className = 'vis-configuration-wrapper';\n    this.container.appendChild(this.wrapper);\n    for (var i = 0; i < this.domElements.length; i++) {\n      this.wrapper.appendChild(this.domElements[i]);\n    }\n\n    this._showPopupIfNeeded()\n  }\n\n\n  /**\n   * delete all DOM elements\n   * @private\n   */\n  _clean() {\n    for (var i = 0; i < this.domElements.length; i++) {\n      this.wrapper.removeChild(this.domElements[i]);\n    }\n\n    if (this.wrapper !== undefined) {\n      this.container.removeChild(this.wrapper);\n      this.wrapper = undefined;\n    }\n    this.domElements = [];\n\n    this._removePopup();\n  }\n\n\n  /**\n   * get the value from the actualOptions if it exists\n   * @param {array} path    | where to look for the actual option\n   * @returns {*}\n   * @private\n   */\n  _getValue(path) {\n    let base = this.moduleOptions;\n    for (let i = 0; i < path.length; i++) {\n      if (base[path[i]] !== undefined) {\n        base = base[path[i]];\n      }\n      else {\n        base = undefined;\n        break;\n      }\n    }\n    return base;\n  }\n\n\n  /**\n   * all option elements are wrapped in an item\n   * @param {Array} path    | where to look for the actual option\n   * @param {Array.<Element>} domElements\n   * @returns {number}\n   * @private\n   */\n  _makeItem(path, ...domElements) {\n    if (this.allowCreation === true) {\n      let item = document.createElement('div');\n      item.className = 'vis-configuration vis-config-item vis-config-s' + path.length;\n      domElements.forEach((element) => {\n        item.appendChild(element);\n      });\n      this.domElements.push(item);\n      return this.domElements.length;\n    }\n    return 0;\n  }\n\n\n  /**\n   * header for major subjects\n   * @param {string} name\n   * @private\n   */\n  _makeHeader(name) {\n    let div = document.createElement('div');\n    div.className = 'vis-configuration vis-config-header';\n    div.innerHTML = name;\n    this._makeItem([],div);\n  }\n\n\n  /**\n   * make a label, if it is an object label, it gets different styling.\n   * @param {string} name\n   * @param {array} path    | where to look for the actual option\n   * @param {string} objectLabel\n   * @returns {HTMLElement}\n   * @private\n   */\n  _makeLabel(name, path, objectLabel = false) {\n    let div = document.createElement('div');\n    div.className = 'vis-configuration vis-config-label vis-config-s' + path.length;\n    if (objectLabel === true) {\n      div.innerHTML = '<i><b>' + name + ':</b></i>';\n    }\n    else {\n      div.innerHTML = name + ':';\n    }\n    return div;\n  }\n\n\n  /**\n   * make a dropdown list for multiple possible string optoins\n   * @param {Array.<number>} arr\n   * @param {number} value\n   * @param {array} path    | where to look for the actual option\n   * @private\n   */\n  _makeDropdown(arr, value, path) {\n    let select = document.createElement('select');\n    select.className = 'vis-configuration vis-config-select';\n    let selectedValue = 0;\n    if (value !== undefined) {\n      if (arr.indexOf(value) !== -1) {\n        selectedValue = arr.indexOf(value);\n      }\n    }\n\n    for (let i = 0; i < arr.length; i++) {\n      let option = document.createElement('option');\n      option.value = arr[i];\n      if (i === selectedValue) {\n        option.selected = 'selected';\n      }\n      option.innerHTML = arr[i];\n      select.appendChild(option);\n    }\n\n    let me = this;\n    select.onchange = function () {me._update(this.value, path);};\n\n    let label = this._makeLabel(path[path.length-1], path);\n    this._makeItem(path, label, select);\n  }\n\n\n  /**\n   * make a range object for numeric options\n   * @param {Array.<number>} arr\n   * @param {number} value\n   * @param {array} path    | where to look for the actual option\n   * @private\n   */\n  _makeRange(arr, value, path) {\n    let defaultValue = arr[0];\n    let min = arr[1];\n    let max = arr[2];\n    let step = arr[3];\n    let range = document.createElement('input');\n    range.className = 'vis-configuration vis-config-range';\n    try {\n      range.type = 'range'; // not supported on IE9\n      range.min = min;\n      range.max = max;\n    }\n    // TODO: Add some error handling and remove this lint exception\n    catch (err) {}  // eslint-disable-line no-empty\n    range.step = step;\n\n    // set up the popup settings in case they are needed.\n    let popupString = '';\n    let popupValue = 0;\n\n    if (value !== undefined) {\n      let factor = 1.20;\n      if (value < 0 && value * factor < min) {\n        range.min = Math.ceil(value * factor);\n        popupValue = range.min;\n        popupString = 'range increased';\n      }\n      else if (value / factor < min) {\n        range.min = Math.ceil(value / factor);\n        popupValue = range.min;\n        popupString = 'range increased';\n      }\n      if (value * factor > max && max !== 1) {\n        range.max = Math.ceil(value * factor);\n        popupValue = range.max;\n        popupString = 'range increased';\n      }\n      range.value = value;\n    }\n    else {\n      range.value = defaultValue;\n    }\n\n    let input = document.createElement('input');\n    input.className = 'vis-configuration vis-config-rangeinput';\n    input.value = Number(range.value);\n\n    var me = this;\n    range.onchange = function () {input.value = this.value; me._update(Number(this.value), path);};\n    range.oninput  = function () {input.value = this.value; };\n\n    let label = this._makeLabel(path[path.length-1], path);\n    let itemIndex = this._makeItem(path, label, range, input);\n\n    // if a popup is needed AND it has not been shown for this value, show it.\n    if (popupString !== '' && this.popupHistory[itemIndex] !== popupValue) {\n      this.popupHistory[itemIndex] = popupValue;\n      this._setupPopup(popupString, itemIndex);\n    }\n  }\n\n  /**\n   * make a button object\n   * @private\n   */\n  _makeButton() {\n    if (this.options.showButton === true) {\n      let generateButton = document.createElement('div');\n      generateButton.className = 'vis-configuration vis-config-button';\n      generateButton.innerHTML = 'generate options';\n      generateButton.onclick =     () => {this._printOptions();};\n      generateButton.onmouseover = () => {generateButton.className = 'vis-configuration vis-config-button hover';};\n      generateButton.onmouseout =  () => {generateButton.className = 'vis-configuration vis-config-button';};\n\n      this.optionsContainer = document.createElement('div');\n      this.optionsContainer.className = 'vis-configuration vis-config-option-container';\n\n      this.domElements.push(this.optionsContainer);\n      this.domElements.push(generateButton);\n    }\n  }\n\n\n  /**\n   * prepare the popup\n   * @param {string} string\n   * @param {number} index\n   * @private\n   */\n  _setupPopup(string, index) {\n    if (this.initialized === true && this.allowCreation === true && this.popupCounter < this.popupLimit) {\n      let div = document.createElement(\"div\");\n      div.id = \"vis-configuration-popup\";\n      div.className = \"vis-configuration-popup\";\n      div.innerHTML = string;\n      div.onclick = () => {this._removePopup()};\n      this.popupCounter += 1;\n      this.popupDiv = {html:div, index:index};\n    }\n  }\n\n\n  /**\n   * remove the popup from the dom\n   * @private\n   */\n  _removePopup() {\n    if (this.popupDiv.html !== undefined) {\n      this.popupDiv.html.parentNode.removeChild(this.popupDiv.html);\n      clearTimeout(this.popupDiv.hideTimeout);\n      clearTimeout(this.popupDiv.deleteTimeout);\n      this.popupDiv = {};\n    }\n  }\n\n\n  /**\n   * Show the popup if it is needed.\n   * @private\n   */\n  _showPopupIfNeeded() {\n    if (this.popupDiv.html !== undefined) {\n      let correspondingElement = this.domElements[this.popupDiv.index];\n      let rect = correspondingElement.getBoundingClientRect();\n      this.popupDiv.html.style.left = rect.left + \"px\";\n      this.popupDiv.html.style.top = rect.top - 30 + \"px\"; // 30 is the height;\n      document.body.appendChild(this.popupDiv.html)\n      this.popupDiv.hideTimeout = setTimeout(() => {\n        this.popupDiv.html.style.opacity = 0;\n      },1500);\n      this.popupDiv.deleteTimeout = setTimeout(() => {\n        this._removePopup();\n      },1800)\n    }\n  }\n\n  /**\n   * make a checkbox for boolean options.\n   * @param {number} defaultValue\n   * @param {number} value\n   * @param {array} path    | where to look for the actual option\n   * @private\n   */\n  _makeCheckbox(defaultValue, value, path) {\n    var checkbox = document.createElement('input');\n    checkbox.type = 'checkbox';\n    checkbox.className = 'vis-configuration vis-config-checkbox';\n    checkbox.checked = defaultValue;\n    if (value !== undefined) {\n      checkbox.checked = value;\n      if (value !== defaultValue) {\n        if (typeof defaultValue === 'object') {\n          if (value !== defaultValue.enabled) {\n            this.changedOptions.push({path:path, value:value});\n          }\n        }\n        else {\n          this.changedOptions.push({path:path, value:value});\n        }\n      }\n    }\n\n    let me = this;\n    checkbox.onchange = function() {me._update(this.checked, path)};\n\n    let label = this._makeLabel(path[path.length-1], path);\n    this._makeItem(path, label, checkbox);\n  }\n\n  /**\n   * make a text input field for string options.\n   * @param {number} defaultValue\n   * @param {number} value\n   * @param {array} path    | where to look for the actual option\n   * @private\n   */\n  _makeTextInput(defaultValue, value, path) {\n    var checkbox = document.createElement('input');\n    checkbox.type = 'text';\n    checkbox.className = 'vis-configuration vis-config-text';\n    checkbox.value = value;\n    if (value !== defaultValue) {\n      this.changedOptions.push({path:path, value:value});\n    }\n\n    let me = this;\n    checkbox.onchange = function() {me._update(this.value, path)};\n\n    let label = this._makeLabel(path[path.length-1], path);\n    this._makeItem(path, label, checkbox);\n  }\n\n\n  /**\n   * make a color field with a color picker for color fields\n   * @param {Array.<number>} arr\n   * @param {number} value\n   * @param {array} path    | where to look for the actual option\n   * @private\n   */\n  _makeColorField(arr, value, path) {\n    let defaultColor = arr[1];\n    let div = document.createElement('div');\n    value = value === undefined ? defaultColor : value;\n\n    if (value !== 'none') {\n      div.className = 'vis-configuration vis-config-colorBlock';\n      div.style.backgroundColor = value;\n    }\n    else {\n      div.className = 'vis-configuration vis-config-colorBlock none';\n    }\n\n    value = value === undefined ? defaultColor : value;\n    div.onclick = () => {\n      this._showColorPicker(value,div,path);\n    };\n\n    let label = this._makeLabel(path[path.length-1], path);\n    this._makeItem(path,label, div);\n  }\n\n\n  /**\n   * used by the color buttons to call the color picker.\n   * @param {number} value\n   * @param {HTMLElement} div\n   * @param {array} path    | where to look for the actual option\n   * @private\n   */\n  _showColorPicker(value, div, path) {\n    // clear the callback from this div\n    div.onclick = function() {};\n\n    this.colorPicker.insertTo(div);\n    this.colorPicker.show();\n\n    this.colorPicker.setColor(value);\n    this.colorPicker.setUpdateCallback((color) => {\n      let colorString = 'rgba(' + color.r + ',' + color.g + ',' + color.b + ',' + color.a + ')';\n      div.style.backgroundColor = colorString;\n      this._update(colorString,path);\n    });\n\n    // on close of the colorpicker, restore the callback.\n    this.colorPicker.setCloseCallback(() => {\n      div.onclick = () => {\n        this._showColorPicker(value,div,path);\n      };\n    });\n  }\n\n\n  /**\n   * parse an object and draw the correct items\n   * @param {Object} obj\n   * @param {array} [path=[]]    | where to look for the actual option\n   * @param {boolean} [checkOnly=false]\n   * @returns {boolean}\n   * @private\n   */\n  _handleObject(obj, path = [], checkOnly = false) {\n    let show = false;\n    let filter = this.options.filter;\n    let visibleInSet = false;\n    for (let subObj in obj) {\n      if (obj.hasOwnProperty(subObj)) {\n        show = true;\n        let item = obj[subObj];\n        let newPath = util.copyAndExtendArray(path, subObj);\n        if (typeof filter === 'function') {\n          show = filter(subObj,path);\n\n          // if needed we must go deeper into the object.\n          if (show === false) {\n            if (!(item instanceof Array) && typeof item !== 'string' && typeof item !== 'boolean' && item instanceof Object) {\n              this.allowCreation = false;\n              show = this._handleObject(item, newPath, true);\n              this.allowCreation = checkOnly === false;\n            }\n          }\n        }\n\n        if (show !== false) {\n          visibleInSet = true;\n          let value = this._getValue(newPath);\n\n          if (item instanceof Array) {\n            this._handleArray(item, value, newPath);\n          }\n          else if (typeof item === 'string') {\n            this._makeTextInput(item, value, newPath);\n          }\n          else if (typeof item === 'boolean') {\n            this._makeCheckbox(item, value, newPath);\n          }\n          else if (item instanceof Object) {\n            // collapse the physics options that are not enabled\n            let draw = true;\n            if (path.indexOf('physics') !== -1) {\n              if (this.moduleOptions.physics.solver !== subObj) {\n                draw = false;\n              }\n            }\n\n            if (draw === true) {\n              // initially collapse options with an disabled enabled option.\n              if (item.enabled !== undefined) {\n                let enabledPath = util.copyAndExtendArray(newPath, 'enabled');\n                let enabledValue = this._getValue(enabledPath);\n                if (enabledValue === true) {\n                  let label = this._makeLabel(subObj, newPath, true);\n                  this._makeItem(newPath, label);\n                  visibleInSet = this._handleObject(item, newPath) || visibleInSet;\n                }\n                else {\n                  this._makeCheckbox(item, enabledValue, newPath);\n                }\n              }\n              else {\n                let label = this._makeLabel(subObj, newPath, true);\n                this._makeItem(newPath, label);\n                visibleInSet = this._handleObject(item, newPath) || visibleInSet;\n              }\n            }\n          }\n          else {\n            console.error('dont know how to handle', item, subObj, newPath);\n          }\n        }\n      }\n    }\n    return visibleInSet;\n  }\n\n\n  /**\n   * handle the array type of option\n   * @param {Array.<number>} arr\n   * @param {number} value\n   * @param {array} path    | where to look for the actual option\n   * @private\n   */\n  _handleArray(arr, value, path) {\n    if (typeof arr[0] === 'string' && arr[0] === 'color') {\n      this._makeColorField(arr, value, path);\n      if (arr[1] !== value) {this.changedOptions.push({path:path, value:value});}\n    }\n    else if (typeof arr[0] === 'string') {\n      this._makeDropdown(arr, value, path);\n      if (arr[0] !== value) {this.changedOptions.push({path:path, value:value});}\n    }\n    else if (typeof arr[0] === 'number') {\n      this._makeRange(arr, value, path);\n      if (arr[0] !== value) {this.changedOptions.push({path:path, value:Number(value)});}\n    }\n  }\n\n\n\n  /**\n   * called to update the network with the new settings.\n   * @param {number} value\n   * @param {array} path    | where to look for the actual option\n   * @private\n   */\n  _update(value, path) {\n    let options = this._constructOptions(value,path);\n\n    if (this.parent.body && this.parent.body.emitter && this.parent.body.emitter.emit) {\n      this.parent.body.emitter.emit(\"configChange\", options);\n    }\n    this.initialized = true;\n    this.parent.setOptions(options);\n  }\n\n\n  /**\n   *\n   * @param {string|Boolean} value\n   * @param {Array.<string>} path\n   * @param {{}} optionsObj\n   * @returns {{}}\n   * @private\n   */\n  _constructOptions(value, path, optionsObj = {}) {\n    let pointer = optionsObj;\n\n    // when dropdown boxes can be string or boolean, we typecast it into correct types\n    value = value === 'true'  ? true  : value;\n    value = value === 'false' ? false : value;\n\n    for (let i = 0; i < path.length; i++) {\n      if (path[i] !== 'global') {\n        if (pointer[path[i]] === undefined) {\n          pointer[path[i]] = {};\n        }\n        if (i !== path.length - 1) {\n          pointer = pointer[path[i]];\n        }\n        else {\n          pointer[path[i]] = value;\n        }\n      }\n    }\n    return optionsObj;\n  }\n\n  /**\n   * @private\n   */\n  _printOptions() {\n    let options = this.getOptions();\n    this.optionsContainer.innerHTML = '<pre>var options = ' + JSON.stringify(options, null, 2) + '</pre>';\n  }\n\n  /**\n   *\n   * @returns {{}} options\n   */\n  getOptions() {\n    let options = {};\n    for (var i = 0; i < this.changedOptions.length; i++) {\n      this._constructOptions(this.changedOptions[i].value, this.changedOptions[i].path, options)\n    }\n    return options;\n  }\n}\n\n\nexport default Configurator;\n","import moment from '../module/moment';\nimport util, { typeCoerceDataSet } from '../util';\nimport { DataSet }  from 'vis-data/esnext';\nimport { DataView } from 'vis-data/esnext';\nimport Range from './Range';\nimport Core from './Core';\nimport TimeAxis from './component/TimeAxis';\nimport CurrentTime from './component/CurrentTime';\nimport CustomTime from './component/CustomTime';\nimport ItemSet from './component/ItemSet';\n\nimport { Validator } from '../shared/Validator';\nimport { printStyle } from '../shared/Validator';\nimport { allOptions } from './optionsTimeline';\nimport { configureOptions } from './optionsTimeline';\n\nimport Configurator from '../shared/Configurator';\n\n/**\n * Create a timeline visualization\n * @extends Core\n */\nexport default class Timeline extends Core {\n  /**\n * @param {HTMLElement} container\n * @param {vis.DataSet | vis.DataView | Array} [items]\n * @param {vis.DataSet | vis.DataView | Array} [groups]\n * @param {Object} [options]  See Timeline.setOptions for the available options.\n * @constructor Timeline\n */\n  constructor(container, items, groups, options) {\n    super()\n    this.initTime = new Date();\n    this.itemsDone = false;\n\n    if (!(this instanceof Timeline)) {\n      throw new SyntaxError('Constructor must be called with the new operator');\n    }\n\n    // if the third element is options, the forth is groups (optionally);\n    if (!(Array.isArray(groups) || groups instanceof DataSet || groups instanceof DataView) && groups instanceof Object) {\n      const forthArgument = options;\n      options = groups;\n      groups = forthArgument;\n    }\n\n    // TODO: REMOVE THIS in the next MAJOR release\n    // see https://github.com/almende/vis/issues/2511\n    if (options && options.throttleRedraw) {\n      console.warn(\"Timeline option \\\"throttleRedraw\\\" is DEPRICATED and no longer supported. It will be removed in the next MAJOR release.\");\n    }\n\n    const me = this;\n    this.defaultOptions = {\n      autoResize: true,\n      longSelectPressTime: 251,\n      orientation: {\n        axis: 'bottom',   // axis orientation: 'bottom', 'top', or 'both'\n        item: 'bottom'    // not relevant\n      },\n      moment,\n    };\n    this.options = util.deepExtend({}, this.defaultOptions);\n\n    // Create the DOM, props, and emitter\n    this._create(container);\n    if (!options || (options && typeof options.rtl == \"undefined\")) {\n      this.dom.root.style.visibility = 'hidden';\n      let directionFromDom;\n      let domNode = this.dom.root;\n      while (!directionFromDom && domNode) {\n        directionFromDom = window.getComputedStyle(domNode, null).direction;\n        domNode = domNode.parentElement;\n      }\n      this.options.rtl = (directionFromDom && (directionFromDom.toLowerCase() == \"rtl\"));\n    } else {\n      this.options.rtl = options.rtl;\n    }\n\n    if (options) {\n      if (options.rollingMode) { this.options.rollingMode = options.rollingMode; }\n      if (options.onInitialDrawComplete) { this.options.onInitialDrawComplete = options.onInitialDrawComplete; }\n      if (options.onTimeout) { this.options.onTimeout = options.onTimeout; }\n      if (options.loadingScreenTemplate) { this.options.loadingScreenTemplate = options.loadingScreenTemplate; }\n    }\n\n    // Prepare loading screen\n    const loadingScreenFragment = document.createElement('div');\n    if (this.options.loadingScreenTemplate) {\n      const templateFunction = this.options.loadingScreenTemplate.bind(this);\n      const loadingScreen = templateFunction(this.dom.loadingScreen);\n      if ((loadingScreen instanceof Object) && !(loadingScreen instanceof Element)) {\n        templateFunction(loadingScreenFragment)\n      } else {\n        if (loadingScreen instanceof Element) {\n          loadingScreenFragment.innerHTML = '';\n          loadingScreenFragment.appendChild(loadingScreen);\n        }\n        else if (loadingScreen != undefined) {\n          loadingScreenFragment.innerHTML = loadingScreen;\n        }\n      }\n    }\n    this.dom.loadingScreen.appendChild(loadingScreenFragment);\n\n    // all components listed here will be repainted automatically\n    this.components = [];\n\n    this.body = {\n      dom: this.dom,\n      domProps: this.props,\n      emitter: {\n        on: this.on.bind(this),\n        off: this.off.bind(this),\n        emit: this.emit.bind(this)\n      },\n      hiddenDates: [],\n      util: {\n        getScale() {\n          return me.timeAxis.step.scale;\n        },\n        getStep() {\n          return me.timeAxis.step.step;\n        },\n\n        toScreen: me._toScreen.bind(me),\n        toGlobalScreen: me._toGlobalScreen.bind(me), // this refers to the root.width\n        toTime: me._toTime.bind(me),\n        toGlobalTime : me._toGlobalTime.bind(me)\n      }\n    };\n\n    // range\n    this.range = new Range(this.body, this.options);\n    this.components.push(this.range);\n    this.body.range = this.range;\n\n    // time axis\n    this.timeAxis = new TimeAxis(this.body, this.options);\n    this.timeAxis2 = null; // used in case of orientation option 'both'\n    this.components.push(this.timeAxis);\n\n    // current time bar\n    this.currentTime = new CurrentTime(this.body, this.options);\n    this.components.push(this.currentTime);\n\n    // item set\n    this.itemSet = new ItemSet(this.body, this.options);\n    this.components.push(this.itemSet);\n\n    this.itemsData = null;      // DataSet\n    this.groupsData = null;     // DataSet\n\n    this.dom.root.onclick = event => {\n      me.emit('click', me.getEventProperties(event))\n    };\n    this.dom.root.ondblclick = event => {\n      me.emit('doubleClick', me.getEventProperties(event))\n    };\n    this.dom.root.oncontextmenu = event => {\n      me.emit('contextmenu', me.getEventProperties(event))\n    };\n    this.dom.root.onmouseover = event => {\n      me.emit('mouseOver', me.getEventProperties(event))\n    };\n    if(window.PointerEvent) {\n      this.dom.root.onpointerdown = event => {\n        me.emit('mouseDown', me.getEventProperties(event))\n      };\n      this.dom.root.onpointermove = event => {\n        me.emit('mouseMove', me.getEventProperties(event))\n      };\n      this.dom.root.onpointerup = event => {\n        me.emit('mouseUp', me.getEventProperties(event))\n      };\n    } else {\n      this.dom.root.onmousemove = event => {\n        me.emit('mouseMove', me.getEventProperties(event))\n      };\n      this.dom.root.onmousedown = event => {\n        me.emit('mouseDown', me.getEventProperties(event))\n      };\n      this.dom.root.onmouseup = event => {\n        me.emit('mouseUp', me.getEventProperties(event))\n      };\n    }\n\n    //Single time autoscale/fit\n    this.initialFitDone = false;\n    this.on('changed', () => {\n      if (me.itemsData == null) return;\n      if (!me.initialFitDone && !me.options.rollingMode) {\n        me.initialFitDone = true;\n        if (me.options.start != undefined || me.options.end != undefined) {\n          if (me.options.start == undefined || me.options.end == undefined) {\n            var range = me.getItemRange();\n          }\n\n          const start = me.options.start != undefined ? me.options.start : range.min;\n          const end   = me.options.end   != undefined ? me.options.end   : range.max;\n          me.setWindow(start, end, {animation: false});\n        } else {\n          me.fit({animation: false});\n        }\n      }\n\n      if (!me.initialDrawDone && (me.initialRangeChangeDone || (!me.options.start && !me.options.end) \n        || me.options.rollingMode)) {\n        me.initialDrawDone = true;\n        me.itemSet.initialDrawDone = true;\n        me.dom.root.style.visibility = 'visible';\n        me.dom.loadingScreen.parentNode.removeChild(me.dom.loadingScreen);\n        if (me.options.onInitialDrawComplete) {\n          setTimeout(() => {\n            return me.options.onInitialDrawComplete();\n          }, 0)\n        }\n      }\n    });\n\n    this.on('destroyTimeline', () => {\n      me.destroy()\n    });\n\n    // apply options\n    if (options) {\n      this.setOptions(options);\n    }\n\n    this.body.emitter.on('fit', (args) => {\n      this._onFit(args);\n      this.redraw();\n    });\n\n    // IMPORTANT: THIS HAPPENS BEFORE SET ITEMS!\n    if (groups) {\n      this.setGroups(groups);\n    }\n\n    // create itemset\n    if (items) {\n      this.setItems(items);\n    }\n\n    // draw for the first time\n    this._redraw();\n  }\n\n  /**\n   * Load a configurator\n   * @return {Object}\n   * @private\n   */\n  _createConfigurator() {\n    return new Configurator(this, this.dom.container, configureOptions);\n  }\n\n  /**\n   * Force a redraw. The size of all items will be recalculated.\n   * Can be useful to manually redraw when option autoResize=false and the window\n   * has been resized, or when the items CSS has been changed.\n   *\n   * Note: this function will be overridden on construction with a trottled version\n   */\n  redraw() {\n    this.itemSet && this.itemSet.markDirty({refreshItems: true});\n    this._redraw();\n  }\n\n  /**\n   * Remove an item from the group\n   * @param {object} options\n   */\n  setOptions(options) {\n    // validate options\n    let errorFound = Validator.validate(options, allOptions);\n\n    if (errorFound === true) {\n      console.log('%cErrors have been found in the supplied options object.', printStyle);\n    }\n\n    Core.prototype.setOptions.call(this, options);\n\n    if ('type' in options) {\n      if (options.type !== this.options.type) {\n        this.options.type = options.type;\n\n        // force recreation of all items\n        const itemsData = this.itemsData;\n        if (itemsData) {\n          const selection = this.getSelection();\n          this.setItems(null);            // remove all\n          this.setItems(itemsData.rawDS); // add all\n          this.setSelection(selection);   // restore selection\n        }\n      }\n    }\n  }\n\n  /**\n   * Set items\n   * @param {vis.DataSet | Array | null} items\n   */\n  setItems(items) {\n    this.itemsDone = false;\n    \n    // convert to type DataSet when needed\n    let newDataSet;\n    if (!items) {\n      newDataSet = null;\n    }\n    else if (items instanceof DataSet || items instanceof DataView) {\n      newDataSet = typeCoerceDataSet(items);\n    }\n    else {\n      // turn an array into a dataset\n      newDataSet = typeCoerceDataSet(new DataSet(items));\n    }\n\n    // set items\n    if (this.itemsData) {\n      // stop maintaining a coerced version of the old data set\n      this.itemsData.dispose();\n    }\n    this.itemsData = newDataSet;\n    this.itemSet && this.itemSet.setItems(newDataSet != null ? newDataSet.rawDS : null);\n  }\n\n  /**\n   * Set groups\n   * @param {vis.DataSet | Array} groups\n   */\n  setGroups(groups) {\n    // convert to type DataSet when needed\n    let newDataSet;\n    if (!groups) {\n      newDataSet = null;\n    }\n    else {\n      const filter = group => group.visible !== false;\n      if (groups instanceof DataSet || groups instanceof DataView) {\n        newDataSet = new DataView(groups,{filter});\n      }\n      else {\n        // turn an array into a dataset\n        newDataSet = new DataSet(groups.filter(filter));\n      }\n    }\n\n    // This looks weird but it's necessary to prevent memory leaks.\n    //\n    // The problem is that the DataView will exist as long as the DataSet it's\n    // connected to. This will force it to swap the groups DataSet for it's own\n    // DataSet. In this arrangement it will become unreferenced from the outside\n    // and garbage collected.\n    //\n    // IMPORTANT NOTE: If `this.groupsData` is a DataView was created in this\n    // method. Even if the original is a DataView already a new one has been\n    // created and assigned to `this.groupsData`. In case this changes in the\n    // future it will be necessary to rework this!!!!\n    if (this.groupsData instanceof DataView) {\n      this.groupsData.setData(null);\n    }\n    this.groupsData = newDataSet;\n    this.itemSet.setGroups(newDataSet);\n  }\n\n  /**\n   * Set both items and groups in one go\n   * @param {{items: (Array | vis.DataSet), groups: (Array | vis.DataSet)}} data\n   */\n  setData(data) {\n    if (data && data.groups) {\n      this.setGroups(data.groups);\n    }\n\n    if (data && data.items) {\n      this.setItems(data.items);\n    }\n  }\n\n  /**\n   * Set selected items by their id. Replaces the current selection\n   * Unknown id's are silently ignored.\n   * @param {string[] | string} [ids]  An array with zero or more id's of the items to be\n   *                                selected. If ids is an empty array, all items will be\n   *                                unselected.\n   * @param {Object} [options]      Available options:\n   *                                `focus: boolean`\n   *                                    If true, focus will be set to the selected item(s)\n   *                                `animation: boolean | {duration: number, easingFunction: string}`\n   *                                    If true (default), the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   *                                    Only applicable when option focus is true.\n   */\n  setSelection(ids, options) {\n    this.itemSet && this.itemSet.setSelection(ids);\n\n    if (options && options.focus) {\n      this.focus(ids, options);\n    }\n  }\n\n  /**\n   * Get the selected items by their id\n   * @return {Array} ids  The ids of the selected items\n   */\n  getSelection() {\n    return this.itemSet && this.itemSet.getSelection() || [];\n  }\n\n  /**\n   * Adjust the visible window such that the selected item (or multiple items)\n   * are centered on screen.\n   * @param {string | String[]} id     An item id or array with item ids\n   * @param {Object} [options]      Available options:\n   *                                `animation: boolean | {duration: number, easingFunction: string}`\n   *                                    If true (default), the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   *                                `zoom: boolean`\n   *                                    If true (default), the timeline will\n   *                                    zoom on the element after focus it.\n   */\n  focus(id, options) {\n    if (!this.itemsData || id == undefined) return;\n\n    const ids = Array.isArray(id) ? id : [id];\n\n    // get the specified item(s)\n    const itemsData = this.itemsData.get(ids);\n\n    // calculate minimum start and maximum end of specified items\n    let start = null;\n    let end = null;\n    itemsData.forEach(itemData => {\n      const s = itemData.start.valueOf();\n      const e = 'end' in itemData ? itemData.end.valueOf() : itemData.start.valueOf();\n\n      if (start === null || s < start) {\n        start = s;\n      }\n\n      if (end === null || e > end) {\n        end = e;\n      }\n    });\n\n\n    if (start !== null && end !== null) {\n      const me = this;\n      // Use the first item for the vertical focus\n      const item = this.itemSet.items[ids[0]];\n      let startPos = this._getScrollTop() * -1;\n      let initialVerticalScroll = null;\n\n      // Setup a handler for each frame of the vertical scroll\n      const verticalAnimationFrame = (ease, willDraw, done) => {\n        const verticalScroll = getItemVerticalScroll(me, item);\n\n        if (verticalScroll === false) {\n          return; // We don't need to scroll, so do nothing\n        }\n\n        if(!initialVerticalScroll) {\n          initialVerticalScroll = verticalScroll;\n        }\n\n        if(initialVerticalScroll.itemTop == verticalScroll.itemTop && !initialVerticalScroll.shouldScroll) {\n          return; // We don't need to scroll, so do nothing\n        }\n        else if(initialVerticalScroll.itemTop != verticalScroll.itemTop && verticalScroll.shouldScroll) {\n          // The redraw shifted elements, so reset the animation to correct\n          initialVerticalScroll = verticalScroll;\n          startPos = me._getScrollTop() * -1;\n        }      \n\n        const from = startPos;\n        const to = initialVerticalScroll.scrollOffset;\n        const scrollTop = done ? to : (from + (to - from) * ease);\n\n        me._setScrollTop(-scrollTop);\n\n        if(!willDraw) {\n          me._redraw();\n        }\n      };\n\n      // Enforces the final vertical scroll position\n      const setFinalVerticalPosition = () => {\n        const finalVerticalScroll = getItemVerticalScroll(me, item);\n\n        if (finalVerticalScroll.shouldScroll && finalVerticalScroll.itemTop != initialVerticalScroll.itemTop) {\n          me._setScrollTop(-finalVerticalScroll.scrollOffset);\n          me._redraw();\n        }\n      };\n\n      // Perform one last check at the end to make sure the final vertical\n      // position is correct\n      const finalVerticalCallback = () => {\n        // Double check we ended at the proper scroll position\n        setFinalVerticalPosition();\n\n        // Let the redraw settle and finalize the position.      \n        setTimeout(setFinalVerticalPosition, 100);\n      };\n\n      // calculate the new middle and interval for the window\n      const zoom = options && options.zoom !== undefined ? options.zoom : true;\n      const middle = (start + end) / 2;\n      const interval = zoom ? (end - start) * 1.1 : Math.max(this.range.end - this.range.start, (end - start) * 1.1);\n\n      const animation = options && options.animation !== undefined ? options.animation : true;\n\n      if (!animation) {\n        // We aren't animating so set a default so that the final callback forces the vertical location\n        initialVerticalScroll = { shouldScroll: false, scrollOffset: -1, itemTop: -1 };\n      }\n\n      this.range.setRange(middle - interval / 2, middle + interval / 2, { animation }, finalVerticalCallback, verticalAnimationFrame);  \n    }\n  }\n\n  /**\n   * Set Timeline window such that it fits all items\n   * @param {Object} [options]  Available options:\n   *                                `animation: boolean | {duration: number, easingFunction: string}`\n   *                                    If true (default), the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   * @param {function} [callback]\n   */\n  fit(options, callback) {\n    const animation = (options && options.animation !== undefined) ? options.animation : true;\n    let range;\n\n    if (this.itemsData.length === 1 && this.itemsData.get()[0].end === undefined) {\n      // a single item -> don't fit, just show a range around the item from -4 to +3 days\n      range = this.getDataRange();\n      this.moveTo(range.min.valueOf(), {animation}, callback);\n    }\n    else {\n      // exactly fit the items (plus a small margin)\n      range = this.getItemRange();\n      this.range.setRange(range.min, range.max, { animation }, callback);\n    }\n  }\n\n  /**\n   * Determine the range of the items, taking into account their actual width\n   * and a margin of 10 pixels on both sides.\n   *\n   * @returns {{min: Date, max: Date}}\n   */\n  getItemRange() {\n    // get a rough approximation for the range based on the items start and end dates\n    const range = this.getDataRange();\n    let min = range.min !== null ? range.min.valueOf() : null;\n    let max = range.max !== null ? range.max.valueOf() : null;\n    let minItem = null;\n    let maxItem = null;\n\n    if (min != null && max != null) {\n      let interval = (max - min); // ms\n      if (interval <= 0) {\n        interval = 10;\n      }\n      const factor = interval / this.props.center.width;\n\n      const redrawQueue = {};\n      let redrawQueueLength = 0;\n\n      // collect redraw functions\n      util.forEach(this.itemSet.items, (item, key) => {\n        if (item.groupShowing) {\n          const returnQueue = true;\n          redrawQueue[key] = item.redraw(returnQueue);\n          redrawQueueLength = redrawQueue[key].length;\n        }\n      })\n\n      const needRedraw = redrawQueueLength > 0;\n      if (needRedraw) {\n        // redraw all regular items\n        for (let i = 0; i < redrawQueueLength; i++) {\n          util.forEach(redrawQueue, fns => {\n            fns[i]();\n          });\n        }\n      }\n\n       // calculate the date of the left side and right side of the items given\n      util.forEach(this.itemSet.items, item => {\n        const start = getStart(item);\n        const end = getEnd(item);\n        let startSide;\n        let endSide;\n\n        if (this.options.rtl) {\n          startSide  = start - (item.getWidthRight()  + 10) * factor;\n          endSide = end   + (item.getWidthLeft() + 10) * factor;\n        } else {\n          startSide  = start - (item.getWidthLeft()  + 10) * factor;\n          endSide = end   + (item.getWidthRight() + 10) * factor;\n        }\n\n        if (startSide < min) {\n          min = startSide;\n          minItem = item;\n        }\n        if (endSide > max) {\n          max = endSide;\n          maxItem = item;\n        }\n      });\n\n      if (minItem && maxItem) {\n        const lhs = minItem.getWidthLeft() + 10;\n        const rhs = maxItem.getWidthRight() + 10;\n        const delta = this.props.center.width - lhs - rhs;  // px\n\n        if (delta > 0) {\n          if (this.options.rtl) {\n            min = getStart(minItem) - rhs * interval / delta; // ms\n            max = getEnd(maxItem)   + lhs * interval / delta; // ms\n          } else {\n            min = getStart(minItem) - lhs * interval / delta; // ms\n            max = getEnd(maxItem)   + rhs * interval / delta; // ms\n          }\n        }\n      }\n    }\n\n    return {\n      min: min != null ? new Date(min) : null,\n      max: max != null ? new Date(max) : null\n    }\n  }\n\n  /**\n   * Calculate the data range of the items start and end dates\n   * @returns {{min: Date, max: Date}}\n   */\n  getDataRange() {\n    let min = null;\n    let max = null;\n\n    if (this.itemsData) {\n      this.itemsData.forEach(item => {\n        const start = util.convert(item.start, 'Date').valueOf();\n        const end   = util.convert(item.end != undefined ? item.end : item.start, 'Date').valueOf();\n        if (min === null || start < min) {\n          min = start;\n        }\n        if (max === null || end > max) {\n          max = end;\n        }\n      });\n    }\n\n    return {\n      min: min != null ? new Date(min) : null,\n      max: max != null ? new Date(max) : null\n    }\n  }\n\n  /**\n   * Generate Timeline related information from an event\n   * @param {Event} event\n   * @return {Object} An object with related information, like on which area\n   *                  The event happened, whether clicked on an item, etc.\n   */\n  getEventProperties(event) {\n    const clientX = event.center ? event.center.x : event.clientX;\n    const clientY = event.center ? event.center.y : event.clientY;\n    const centerContainerRect = this.dom.centerContainer.getBoundingClientRect();\n    const x = this.options.rtl ? centerContainerRect.right - clientX : clientX - centerContainerRect.left;\n    const y = clientY - centerContainerRect.top;\n    \n    const item  = this.itemSet.itemFromTarget(event);\n    const group = this.itemSet.groupFromTarget(event);\n    const customTime = CustomTime.customTimeFromTarget(event);\n\n    const snap = this.itemSet.options.snap || null;\n    const scale = this.body.util.getScale();\n    const step = this.body.util.getStep();\n    const time = this._toTime(x);\n    const snappedTime = snap ? snap(time, scale, step) : time;\n\n    const element = util.getTarget(event);\n    let what = null;\n    if (item != null)                                                    {what = 'item';}\n    else if (customTime != null)                                         {what = 'custom-time';}\n    else if (util.hasParent(element, this.timeAxis.dom.foreground))      {what = 'axis';}\n    else if (this.timeAxis2 && util.hasParent(element, this.timeAxis2.dom.foreground)) {what = 'axis';}\n    else if (util.hasParent(element, this.itemSet.dom.labelSet))         {what = 'group-label';}\n    else if (util.hasParent(element, this.currentTime.bar))              {what = 'current-time';}\n    else if (util.hasParent(element, this.dom.center))                   {what = 'background';}\n\n    return {\n      event,\n      item: item ? item.id : null,\n      isCluster: item ? !!item.isCluster: false,\n      items: item ? item.items || []: null,\n      group: group ? group.groupId : null,\n      customTime: customTime ? customTime.options.id : null,\n      what,\n      pageX: event.srcEvent ? event.srcEvent.pageX : event.pageX,\n      pageY: event.srcEvent ? event.srcEvent.pageY : event.pageY,\n      x,\n      y,\n      time,\n      snappedTime\n    }\n  }\n\n  /**\n   * Toggle Timeline rolling mode\n   */\n  toggleRollingMode() {\n    if (this.range.rolling) {\n      this.range.stopRolling();\n    } else {\n      if (this.options.rollingMode == undefined) {\n        this.setOptions(this.options)\n      }\n      this.range.startRolling();\n    }\n  }\n\n  /**\n   * redraw\n   * @private\n   */\n  _redraw() {\n    Core.prototype._redraw.call(this);\n  }\n\n  /**\n   * on fit callback\n   * @param {object} args\n   * @private\n   */\n  _onFit(args) {\n    const { start, end, animation } = args;\n    if (!end) {\n      this.moveTo(start.valueOf(), {\n        animation\n      });\n    } else {\n      this.range.setRange(start, end, {\n        animation: animation\n      });\n    }\n  }\n}\n\n/**\n *\n * @param {timeline.Item} item\n * @returns {number}\n */\nfunction getStart(item) {\n  return util.convert(item.data.start, 'Date').valueOf()\n}\n\n/**\n *\n * @param {timeline.Item} item\n * @returns {number}\n */\nfunction getEnd(item) {\n  const end = item.data.end != undefined ? item.data.end : item.data.start;\n  return util.convert(end, 'Date').valueOf();\n}\n\n/**\n * @param {vis.Timeline} timeline\n * @param {timeline.Item} item\n * @return {{shouldScroll: bool, scrollOffset: number, itemTop: number}}\n */\nfunction getItemVerticalScroll(timeline, item) {\n  if (!item.parent) {\n    // The item no longer exists, so ignore this focus.\n    return false;\n  }\n\n  const itemsetHeight = timeline.options.rtl ? timeline.props.rightContainer.height : timeline.props.leftContainer.height;\n  const contentHeight = timeline.props.center.height;\n  \n  const group = item.parent;\n  let offset = group.top;\n  let shouldScroll = true;\n  const orientation = timeline.timeAxis.options.orientation.axis;\n  \n  const itemTop = () => {\n  if (orientation == \"bottom\") {\n      return group.height - item.top - item.height;\n    }\n    else {\n      return item.top;\n    }\n  };\n\n  const currentScrollHeight = timeline._getScrollTop() * -1;\n  const targetOffset = offset + itemTop();\n  const height = item.height;\n\n  if (targetOffset < currentScrollHeight) {\n    if (offset + itemsetHeight <= offset + itemTop() + height) {\n      offset += itemTop() - timeline.itemSet.options.margin.item.vertical;\n    }\n  }\n  else if (targetOffset + height > currentScrollHeight + itemsetHeight) {\n    offset += itemTop() + height - itemsetHeight + timeline.itemSet.options.margin.item.vertical;\n  }\n  else {\n    shouldScroll = false;\n  }\n\n  offset = Math.min(offset, contentHeight - itemsetHeight);\n\n  return { shouldScroll, scrollOffset: offset, itemTop: targetOffset };\n}\n","/** DataScale */\nclass DataScale {\n  /**\n   *\n   * @param {number} start\n   * @param {number} end\n   * @param {boolean} autoScaleStart\n   * @param {boolean} autoScaleEnd\n   * @param {number} containerHeight\n   * @param {number} majorCharHeight\n   * @param {boolean} zeroAlign\n   * @param {function} formattingFunction\n   * @constructor DataScale\n   */\n  constructor(\n    start,\n    end,\n    autoScaleStart,\n    autoScaleEnd,\n    containerHeight,\n    majorCharHeight,\n    zeroAlign = false,\n    formattingFunction=false) {\n    this.majorSteps = [1, 2, 5, 10];\n    this.minorSteps = [0.25, 0.5, 1, 2];\n    this.customLines = null;\n\n    this.containerHeight = containerHeight;\n    this.majorCharHeight = majorCharHeight;\n    this._start = start;\n    this._end = end;\n\n    this.scale = 1;\n    this.minorStepIdx = -1;\n    this.magnitudefactor = 1;\n    this.determineScale();\n\n    this.zeroAlign = zeroAlign;\n    this.autoScaleStart = autoScaleStart;\n    this.autoScaleEnd = autoScaleEnd;\n\n    this.formattingFunction = formattingFunction;\n\n    if (autoScaleStart || autoScaleEnd) {\n      const me = this;\n      const roundToMinor = value => {\n        const rounded = value - (value % (me.magnitudefactor * me.minorSteps[me.minorStepIdx]));\n        if (value % (me.magnitudefactor * me.minorSteps[me.minorStepIdx]) > 0.5 * (me.magnitudefactor * me.minorSteps[me.minorStepIdx])) {\n          return rounded + (me.magnitudefactor * me.minorSteps[me.minorStepIdx]);\n        }\n        else {\n          return rounded;\n        }\n      };\n      if (autoScaleStart) {\n        this._start -= this.magnitudefactor * 2 * this.minorSteps[this.minorStepIdx];\n        this._start = roundToMinor(this._start);\n      }\n\n      if (autoScaleEnd) {\n        this._end += this.magnitudefactor * this.minorSteps[this.minorStepIdx];\n        this._end = roundToMinor(this._end);\n      }\n      this.determineScale();\n    }\n  }\n\n  /**\n   * set chart height\n   * @param {number} majorCharHeight \n   */\n  setCharHeight(majorCharHeight) {\n    this.majorCharHeight = majorCharHeight;\n  }\n\n  /**\n   * set height\n   * @param {number} containerHeight \n   */\n  setHeight(containerHeight) {\n    this.containerHeight = containerHeight;\n  }\n\n  /**\n   * determine scale\n   */\n  determineScale() {\n    const range = this._end - this._start;\n    this.scale = this.containerHeight / range;\n    const minimumStepValue = this.majorCharHeight / this.scale;\n    const orderOfMagnitude = (range > 0)\n        ? Math.round(Math.log(range) / Math.LN10)\n        : 0;\n\n    this.minorStepIdx = -1;\n    this.magnitudefactor = Math.pow(10, orderOfMagnitude);\n\n    let start = 0;\n    if (orderOfMagnitude < 0) {\n      start = orderOfMagnitude;\n    }\n\n    let solutionFound = false;\n    for (let l = start; Math.abs(l) <= Math.abs(orderOfMagnitude); l++) {\n      this.magnitudefactor = Math.pow(10, l);\n      for (let j = 0; j < this.minorSteps.length; j++) {\n        const stepSize = this.magnitudefactor * this.minorSteps[j];\n        if (stepSize >= minimumStepValue) {\n          solutionFound = true;\n          this.minorStepIdx = j;\n          break;\n        }\n      }\n      if (solutionFound === true) {\n        break;\n      }\n    }\n  }\n\n  /**\n   * returns if value is major\n   * @param {number} value\n   * @returns {boolean} \n   */\n  is_major(value) {\n    return (value % (this.magnitudefactor * this.majorSteps[this.minorStepIdx]) === 0);\n  }\n\n  /**\n   * returns step size\n   * @returns {number} \n   */\n  getStep() {\n    return this.magnitudefactor * this.minorSteps[this.minorStepIdx];\n  }\n\n  /**\n   * returns first major\n   * @returns {number} \n   */\n  getFirstMajor() {\n    const majorStep = this.magnitudefactor * this.majorSteps[this.minorStepIdx];\n    return this.convertValue(this._start + ((majorStep - (this._start % majorStep)) % majorStep));\n  }\n\n  /**\n   * returns first major\n   * @param {date} current\n   * @returns {date} formatted date\n   */\n  formatValue(current) {\n    let returnValue = current.toPrecision(5);\n    if (typeof this.formattingFunction === 'function') {\n      returnValue = this.formattingFunction(current);\n    }\n\n    if (typeof returnValue === 'number') {\n      return `${returnValue}`;\n    }\n    else if (typeof returnValue === 'string') {\n      return returnValue;\n    }\n    else {\n      return current.toPrecision(5);\n    }\n\n  }\n\n  /**\n   * returns lines\n   * @returns {object} lines\n   */\n  getLines() {\n    const lines = [];\n    const step = this.getStep();\n    const bottomOffset = (step - (this._start % step)) % step;\n    for (let i = (this._start + bottomOffset); this._end-i > 0.00001; i += step) {\n      if (i != this._start) { //Skip the bottom line\n        lines.push({major: this.is_major(i), y: this.convertValue(i), val: this.formatValue(i)});\n      }\n    }\n    return lines;\n  }\n\n  /**\n   * follow scale\n   * @param {object} other\n   */\n  followScale(other) {\n    const oldStepIdx = this.minorStepIdx;\n    const oldStart = this._start;\n    const oldEnd = this._end;\n\n    const me = this;\n    const increaseMagnitude = () => {\n      me.magnitudefactor *= 2;\n    };\n    const decreaseMagnitude = () => {\n      me.magnitudefactor /= 2;\n    };\n\n    if ((other.minorStepIdx <= 1 && this.minorStepIdx <= 1) || (other.minorStepIdx > 1 && this.minorStepIdx > 1)) {\n      //easy, no need to change stepIdx nor multiplication factor\n    } else if (other.minorStepIdx < this.minorStepIdx) {\n      //I'm 5, they are 4 per major.\n      this.minorStepIdx = 1;\n      if (oldStepIdx == 2) {\n        increaseMagnitude();\n      } else {\n        increaseMagnitude();\n        increaseMagnitude();\n      }\n    } else {\n      //I'm 4, they are 5 per major\n      this.minorStepIdx = 2;\n      if (oldStepIdx == 1) {\n        decreaseMagnitude();\n      } else {\n        decreaseMagnitude();\n        decreaseMagnitude();\n      }\n    }\n\n    //Get masters stats:\n    const otherZero = other.convertValue(0);\n    const otherStep = other.getStep() * other.scale;\n\n    let done = false;\n    let count = 0;\n    //Loop until magnitude is correct for given constrains.\n    while (!done && count++ <5) {\n\n      //Get my stats:\n      this.scale = otherStep / (this.minorSteps[this.minorStepIdx] * this.magnitudefactor);\n      const newRange = this.containerHeight / this.scale;\n\n      //For the case the magnitudefactor has changed:\n      this._start = oldStart;\n      this._end = this._start + newRange;\n\n      const myOriginalZero = this._end * this.scale;\n      const majorStep = this.magnitudefactor * this.majorSteps[this.minorStepIdx];\n      const majorOffset = this.getFirstMajor() - other.getFirstMajor();\n\n      if (this.zeroAlign) {\n        const zeroOffset = otherZero - myOriginalZero;\n        this._end += (zeroOffset / this.scale);\n        this._start = this._end - newRange;\n      } else {\n        if (!this.autoScaleStart) {\n          this._start += majorStep - (majorOffset / this.scale);\n          this._end = this._start + newRange;\n        } else {\n          this._start -= majorOffset / this.scale;\n          this._end = this._start + newRange;\n        }\n      }\n      if (!this.autoScaleEnd && this._end > oldEnd+0.00001) {\n        //Need to decrease magnitude to prevent scale overshoot! (end)\n        decreaseMagnitude();\n        done = false;\n        continue;\n      }\n      if (!this.autoScaleStart && this._start < oldStart-0.00001) {\n        if (this.zeroAlign && oldStart >= 0) {\n          console.warn(\"Can't adhere to given 'min' range, due to zeroalign\");\n        } else {\n          //Need to decrease magnitude to prevent scale overshoot! (start)\n          decreaseMagnitude();\n          done = false;\n          continue;\n        }\n      }\n      if (this.autoScaleStart && this.autoScaleEnd && newRange < (oldEnd-oldStart)){\n        increaseMagnitude();\n        done = false;\n        continue;\n      }\n      done = true;\n    }\n  }\n\n  /**\n   * convert value\n   * @param {number} value\n   * @returns {number} \n   */\n  convertValue(value) {\n    return this.containerHeight - ((value - this._start) * this.scale);\n  }\n\n  /**\n   * returns screen to value\n   * @param {number} pixels\n   * @returns {number} \n   */\n  screenToValue(pixels) {\n    return ((this.containerHeight - pixels) / this.scale) + this._start;\n  }\n}\n\nexport default DataScale;","import { v4 as randomUUID } from \"uuid\";import util from '../../util';\nimport * as DOMutil from '../../DOMutil';\nimport Component from './Component';\nimport DataScale from './DataScale';\n\nimport './css/dataaxis.css';\n\n/** A horizontal time axis */\nclass DataAxis extends Component {\n  /**\n * @param {Object} body\n * @param {Object} [options]        See DataAxis.setOptions for the available\n *                                  options.\n * @param {SVGElement} svg\n * @param {timeline.LineGraph.options} linegraphOptions\n * @constructor DataAxis\n * @extends Component\n */\n  constructor(body, options, svg, linegraphOptions) {\n    super()\n    this.id = randomUUID();\n    this.body = body;\n\n    this.defaultOptions = {\n      orientation: 'left',  // supported: 'left', 'right'\n      showMinorLabels: true,\n      showMajorLabels: true,\n      showWeekScale: false,\n      icons: false,\n      majorLinesOffset: 7,\n      minorLinesOffset: 4,\n      labelOffsetX: 10,\n      labelOffsetY: 2,\n      iconWidth: 20,\n      width: '40px',\n      visible: true,\n      alignZeros: true,\n      left: {\n        range: {min: undefined, max: undefined},\n        format(value) {\n          return `${parseFloat(value.toPrecision(3))}`;\n        },\n        title: {text: undefined, style: undefined}\n      },\n      right: {\n        range: {min: undefined, max: undefined},\n        format(value) {\n          return `${parseFloat(value.toPrecision(3))}`;\n        },\n        title: {text: undefined, style: undefined}\n      }\n    };\n\n    this.linegraphOptions = linegraphOptions;\n    this.linegraphSVG = svg;\n    this.props = {};\n    this.DOMelements = { // dynamic elements\n      lines: {},\n      labels: {},\n      title: {}\n    };\n\n    this.dom = {};\n    this.scale = undefined;\n    this.range = {start: 0, end: 0};\n\n    this.options = util.extend({}, this.defaultOptions);\n    this.conversionFactor = 1;\n\n    this.setOptions(options);\n    this.width = Number((`${this.options.width}`).replace(\"px\", \"\"));\n    this.minWidth = this.width;\n    this.height = this.linegraphSVG.getBoundingClientRect().height;\n    this.hidden = false;\n\n    this.stepPixels = 25;\n    this.zeroCrossing = -1;\n    this.amountOfSteps = -1;\n\n    this.lineOffset = 0;\n    this.master = true;\n    this.masterAxis = null;\n    this.svgElements = {};\n    this.iconsRemoved = false;\n\n    this.groups = {};\n    this.amountOfGroups = 0;\n\n    // create the HTML DOM\n    this._create();\n    if (this.scale == undefined) {\n      this._redrawLabels();\n    }\n    this.framework = {svg: this.svg, svgElements: this.svgElements, options: this.options, groups: this.groups};\n\n    const me = this;\n    this.body.emitter.on(\"verticalDrag\", () => {\n      me.dom.lineContainer.style.top = `${me.body.domProps.scrollTop}px`;\n    });\n  }\n\n  /**\n   * Adds group to data axis\n   * @param {string} label \n   * @param {object} graphOptions\n   */\n  addGroup(label, graphOptions) {\n    if (!this.groups.hasOwnProperty(label)) {\n      this.groups[label] = graphOptions;\n    }\n    this.amountOfGroups += 1;\n  }\n\n  /**\n   * updates group of data axis\n   * @param {string} label \n   * @param {object} graphOptions\n   */\n  updateGroup(label, graphOptions) {\n    if (!this.groups.hasOwnProperty(label)) {\n      this.amountOfGroups += 1;\n    }\n    this.groups[label] = graphOptions;\n  }\n\n  /**\n   * removes group of data axis\n   * @param {string} label \n   */\n  removeGroup(label) {\n    if (this.groups.hasOwnProperty(label)) {\n      delete this.groups[label];\n      this.amountOfGroups -= 1;\n    }\n  }\n\n  /**\n   * sets options\n   * @param {object} options\n   */\n  setOptions(options) {\n    if (options) {\n      let redraw = false;\n      if (this.options.orientation != options.orientation && options.orientation !== undefined) {\n        redraw = true;\n      }\n      const fields = [\n        'orientation',\n        'showMinorLabels',\n        'showMajorLabels',\n        'icons',\n        'majorLinesOffset',\n        'minorLinesOffset',\n        'labelOffsetX',\n        'labelOffsetY',\n        'iconWidth',\n        'width',\n        'visible',\n        'left',\n        'right',\n        'alignZeros'\n      ];\n      util.selectiveDeepExtend(fields, this.options, options);\n\n      this.minWidth = Number((`${this.options.width}`).replace(\"px\", \"\"));\n      if (redraw === true && this.dom.frame) {\n        this.hide();\n        this.show();\n      }\n    }\n  }\n\n  /**\n   * Create the HTML DOM for the DataAxis\n   */\n  _create() {\n    this.dom.frame = document.createElement('div');\n    this.dom.frame.style.width = this.options.width;\n    this.dom.frame.style.height = this.height;\n\n    this.dom.lineContainer = document.createElement('div');\n    this.dom.lineContainer.style.width = '100%';\n    this.dom.lineContainer.style.height = this.height;\n    this.dom.lineContainer.style.position = 'relative';\n    this.dom.lineContainer.style.visibility = 'visible';\n    this.dom.lineContainer.style.display = 'block';\n\n    // create svg element for graph drawing.\n    this.svg = document.createElementNS('http://www.w3.org/2000/svg', \"svg\");\n    this.svg.style.position = \"absolute\";\n    this.svg.style.top = '0px';\n    this.svg.style.height = '100%';\n    this.svg.style.width = '100%';\n    this.svg.style.display = \"block\";\n    this.dom.frame.appendChild(this.svg);\n  }\n\n  /**\n   * redraws groups icons\n   */\n  _redrawGroupIcons() {\n    DOMutil.prepareElements(this.svgElements);\n\n    let x;\n    const iconWidth = this.options.iconWidth;\n    const iconHeight = 15;\n    const iconOffset = 4;\n    let y = iconOffset + 0.5 * iconHeight;\n\n    if (this.options.orientation === 'left') {\n      x = iconOffset;\n    }\n    else {\n      x = this.width - iconWidth - iconOffset;\n    }\n\n    const groupArray = Object.keys(this.groups);\n    groupArray.sort((a, b) => a < b ? -1 : 1)\n\n    for (const groupId of groupArray) {\n      if (this.groups[groupId].visible === true && (this.linegraphOptions.visibility[groupId] === undefined || this.linegraphOptions.visibility[groupId] === true)) {\n        this.groups[groupId].getLegend(iconWidth, iconHeight, this.framework, x, y);\n        y += iconHeight + iconOffset;\n      }\n    }\n\n    DOMutil.cleanupElements(this.svgElements);\n    this.iconsRemoved = false;\n  }\n\n  /**\n   * Cleans up icons\n   */\n  _cleanupIcons() {\n    if (this.iconsRemoved === false) {\n      DOMutil.prepareElements(this.svgElements);\n      DOMutil.cleanupElements(this.svgElements);\n      this.iconsRemoved = true;\n    }\n  }\n\n  /**\n   * Create the HTML DOM for the DataAxis\n   */\n  show() {\n    this.hidden = false;\n    if (!this.dom.frame.parentNode) {\n      if (this.options.orientation === 'left') {\n        this.body.dom.left.appendChild(this.dom.frame);\n      }\n      else {\n        this.body.dom.right.appendChild(this.dom.frame);\n      }\n    }\n\n    if (!this.dom.lineContainer.parentNode) {\n      this.body.dom.backgroundHorizontal.appendChild(this.dom.lineContainer);\n    }\n    this.dom.lineContainer.style.display = 'block';\n  }\n\n  /**\n   * Create the HTML DOM for the DataAxis\n   */\n  hide() {\n    this.hidden = true;\n    if (this.dom.frame.parentNode) {\n      this.dom.frame.parentNode.removeChild(this.dom.frame);\n    }\n\n    this.dom.lineContainer.style.display = 'none';\n  }\n\n  /**\n   * Set a range (start and end)\n   * @param {number} start\n   * @param {number} end\n   */\n  setRange(start, end) {\n    this.range.start = start;\n    this.range.end = end;\n  }\n\n  /**\n   * Repaint the component\n   * @return {boolean} Returns true if the component is resized\n   */\n  redraw() {\n    let resized = false;\n    let activeGroups = 0;\n\n    // Make sure the line container adheres to the vertical scrolling.\n    this.dom.lineContainer.style.top = `${this.body.domProps.scrollTop}px`;\n\n    for (const groupId in this.groups) {\n      if (this.groups.hasOwnProperty(groupId)) {\n        if (this.groups[groupId].visible === true && (this.linegraphOptions.visibility[groupId] === undefined || this.linegraphOptions.visibility[groupId] === true)) {\n          activeGroups++;\n        }\n      }\n    }\n    if (this.amountOfGroups === 0 || activeGroups === 0) {\n      this.hide();\n    }\n    else {\n      this.show();\n      this.height = Number(this.linegraphSVG.style.height.replace(\"px\", \"\"));\n\n      // svg offsetheight did not work in firefox and explorer...\n      this.dom.lineContainer.style.height = `${this.height}px`;\n      this.width = this.options.visible === true ? Number((`${this.options.width}`).replace(\"px\", \"\")) : 0;\n\n      const props = this.props;\n      const frame = this.dom.frame;\n\n      // update classname\n      frame.className = 'vis-data-axis';\n\n      // calculate character width and height\n      this._calculateCharSize();\n\n      const orientation = this.options.orientation;\n      const showMinorLabels = this.options.showMinorLabels;\n      const showMajorLabels = this.options.showMajorLabels;\n\n      const backgroundHorizontalOffsetWidth = this.body.dom.backgroundHorizontal.offsetWidth;\n\n      // determine the width and height of the elements for the axis\n      props.minorLabelHeight = showMinorLabels ? props.minorCharHeight : 0;\n      props.majorLabelHeight = showMajorLabels ? props.majorCharHeight : 0;\n\n      props.minorLineWidth = backgroundHorizontalOffsetWidth - this.lineOffset - this.width + 2 * this.options.minorLinesOffset;\n      props.minorLineHeight = 1;\n      props.majorLineWidth = backgroundHorizontalOffsetWidth - this.lineOffset - this.width + 2 * this.options.majorLinesOffset;\n      props.majorLineHeight = 1;\n\n      //  take frame offline while updating (is almost twice as fast)\n      if (orientation === 'left') {\n        frame.style.top = '0';\n        frame.style.left = '0';\n        frame.style.bottom = '';\n        frame.style.width = `${this.width}px`;\n        frame.style.height = `${this.height}px`;\n        this.props.width = this.body.domProps.left.width;\n        this.props.height = this.body.domProps.left.height;\n      }\n      else { // right\n        frame.style.top = '';\n        frame.style.bottom = '0';\n        frame.style.left = '0';\n        frame.style.width = `${this.width}px`;\n        frame.style.height = `${this.height}px`;\n        this.props.width = this.body.domProps.right.width;\n        this.props.height = this.body.domProps.right.height;\n      }\n\n      resized = this._redrawLabels();\n      resized = this._isResized() || resized;\n\n      if (this.options.icons === true) {\n        this._redrawGroupIcons();\n      }\n      else {\n        this._cleanupIcons();\n      }\n\n      this._redrawTitle(orientation);\n    }\n    return resized;\n  }\n\n  /**\n   * Repaint major and minor text labels and vertical grid lines\n   *\n   * @returns {boolean}\n   * @private\n   */\n  _redrawLabels() {\n    let resized = false;\n    DOMutil.prepareElements(this.DOMelements.lines);\n    DOMutil.prepareElements(this.DOMelements.labels);\n    const orientation = this.options['orientation'];\n    const customRange = this.options[orientation].range != undefined ? this.options[orientation].range : {};\n\n    //Override range with manual options:\n    let autoScaleEnd = true;\n    if (customRange.max != undefined) {\n      this.range.end = customRange.max;\n      autoScaleEnd = false;\n    }\n    let autoScaleStart = true;\n    if (customRange.min != undefined) {\n      this.range.start = customRange.min;\n      autoScaleStart = false;\n    }\n\n    this.scale = new DataScale(\n      this.range.start,\n      this.range.end,\n      autoScaleStart,\n      autoScaleEnd,\n      this.dom.frame.offsetHeight,\n      this.props.majorCharHeight,\n      this.options.alignZeros,\n      this.options[orientation].format\n    );\n\n    if (this.master === false && this.masterAxis != undefined) {\n      this.scale.followScale(this.masterAxis.scale);\n      this.dom.lineContainer.style.display = 'none';\n    } else {\n      this.dom.lineContainer.style.display = 'block';\n    }\n\n    //Is updated in side-effect of _redrawLabel():\n    this.maxLabelSize = 0;\n\n    const lines = this.scale.getLines();\n    lines.forEach(\n      line=> {\n        const y = line.y;\n        const isMajor = line.major;\n        if (this.options['showMinorLabels'] && isMajor === false) {\n          this._redrawLabel(y - 2, line.val, orientation, 'vis-y-axis vis-minor', this.props.minorCharHeight);\n        }\n        if (isMajor) {\n          if (y >= 0) {\n            this._redrawLabel(y - 2, line.val, orientation, 'vis-y-axis vis-major', this.props.majorCharHeight);\n          }\n        }\n        if (this.master === true) {\n          if (isMajor) {\n            this._redrawLine(y, orientation, 'vis-grid vis-horizontal vis-major', this.options.majorLinesOffset, this.props.majorLineWidth);\n          }\n          else {\n            this._redrawLine(y, orientation, 'vis-grid vis-horizontal vis-minor', this.options.minorLinesOffset, this.props.minorLineWidth);\n          }\n        }\n      });\n\n    // Note that title is rotated, so we're using the height, not width!\n    let titleWidth = 0;\n    if (this.options[orientation].title !== undefined && this.options[orientation].title.text !== undefined) {\n      titleWidth = this.props.titleCharHeight;\n    }\n    const offset = this.options.icons === true ? Math.max(this.options.iconWidth, titleWidth) + this.options.labelOffsetX + 15 : titleWidth + this.options.labelOffsetX + 15;\n\n    // this will resize the yAxis to accommodate the labels.\n    if (this.maxLabelSize > (this.width - offset) && this.options.visible === true) {\n      this.width = this.maxLabelSize + offset;\n      this.options.width = `${this.width}px`;\n      DOMutil.cleanupElements(this.DOMelements.lines);\n      DOMutil.cleanupElements(this.DOMelements.labels);\n      this.redraw();\n      resized = true;\n    }\n    // this will resize the yAxis if it is too big for the labels.\n    else if (this.maxLabelSize < (this.width - offset) && this.options.visible === true && this.width > this.minWidth) {\n      this.width = Math.max(this.minWidth, this.maxLabelSize + offset);\n      this.options.width = `${this.width}px`;\n      DOMutil.cleanupElements(this.DOMelements.lines);\n      DOMutil.cleanupElements(this.DOMelements.labels);\n      this.redraw();\n      resized = true;\n    }\n    else {\n      DOMutil.cleanupElements(this.DOMelements.lines);\n      DOMutil.cleanupElements(this.DOMelements.labels);\n      resized = false;\n    }\n\n    return resized;\n  }\n\n  /**\n   * converts value\n   * @param {number} value\n   * @returns {number} converted number\n   */\n  convertValue(value) {\n    return this.scale.convertValue(value);\n  }\n\n  /**\n   * converts value\n   * @param {number} x\n   * @returns {number} screen value\n   */\n  screenToValue(x) {\n    return this.scale.screenToValue(x);\n  }\n\n  /**\n   * Create a label for the axis at position x\n   *\n   * @param {number} y\n   * @param {string} text\n   * @param {'top'|'right'|'bottom'|'left'} orientation\n   * @param {string} className\n   * @param {number} characterHeight\n   * @private\n   */\n  _redrawLabel(y, text, orientation, className, characterHeight) {\n    // reuse redundant label\n    const label = DOMutil.getDOMElement('div', this.DOMelements.labels, this.dom.frame); //this.dom.redundant.labels.shift();\n    label.className = className;\n    label.innerHTML = text;\n    if (orientation === 'left') {\n      label.style.left = `-${this.options.labelOffsetX}px`;\n      label.style.textAlign = \"right\";\n    }\n    else {\n      label.style.right = `-${this.options.labelOffsetX}px`;\n      label.style.textAlign = \"left\";\n    }\n\n    label.style.top = `${y - 0.5 * characterHeight + this.options.labelOffsetY}px`;\n\n    text += '';\n\n    const largestWidth = Math.max(this.props.majorCharWidth, this.props.minorCharWidth);\n    if (this.maxLabelSize < text.length * largestWidth) {\n      this.maxLabelSize = text.length * largestWidth;\n    }\n  }\n\n  /**\n   * Create a minor line for the axis at position y\n   * @param {number} y\n   * @param {'top'|'right'|'bottom'|'left'} orientation\n   * @param {string} className\n   * @param {number} offset\n   * @param {number} width\n   */\n  _redrawLine(y, orientation, className, offset, width) {\n    if (this.master === true) {\n      const line = DOMutil.getDOMElement('div', this.DOMelements.lines, this.dom.lineContainer);  //this.dom.redundant.lines.shift();\n      line.className = className;\n      line.innerHTML = '';\n\n      if (orientation === 'left') {\n        line.style.left = `${this.width - offset}px`;\n      }\n      else {\n        line.style.right = `${this.width - offset}px`;\n      }\n\n      line.style.width = `${width}px`;\n      line.style.top = `${y}px`;\n    }\n  }\n\n  /**\n   * Create a title for the axis\n   * @private\n   * @param {'top'|'right'|'bottom'|'left'} orientation\n   */\n  _redrawTitle(orientation) {\n    DOMutil.prepareElements(this.DOMelements.title);\n\n    // Check if the title is defined for this axes\n    if (this.options[orientation].title !== undefined && this.options[orientation].title.text !== undefined) {\n      const title = DOMutil.getDOMElement('div', this.DOMelements.title, this.dom.frame);\n      title.className = `vis-y-axis vis-title vis-${orientation}`;\n      title.innerHTML = this.options[orientation].title.text;\n\n      // Add style - if provided\n      if (this.options[orientation].title.style !== undefined) {\n        util.addCssText(title, this.options[orientation].title.style);\n      }\n\n      if (orientation === 'left') {\n        title.style.left = `${this.props.titleCharHeight}px`;\n      }\n      else {\n        title.style.right = `${this.props.titleCharHeight}px`;\n      }\n\n      title.style.width = `${this.height}px`;\n    }\n\n    // we need to clean up in case we did not use all elements.\n    DOMutil.cleanupElements(this.DOMelements.title);\n  }\n\n  /**\n   * Determine the size of text on the axis (both major and minor axis).\n   * The size is calculated only once and then cached in this.props.\n   * @private\n   */\n  _calculateCharSize() {\n    // determine the char width and height on the minor axis\n    if (!('minorCharHeight' in this.props)) {\n      const textMinor = document.createTextNode('0');\n      const measureCharMinor = document.createElement('div');\n      measureCharMinor.className = 'vis-y-axis vis-minor vis-measure';\n      measureCharMinor.appendChild(textMinor);\n      this.dom.frame.appendChild(measureCharMinor);\n\n      this.props.minorCharHeight = measureCharMinor.clientHeight;\n      this.props.minorCharWidth = measureCharMinor.clientWidth;\n\n      this.dom.frame.removeChild(measureCharMinor);\n    }\n\n    if (!('majorCharHeight' in this.props)) {\n      const textMajor = document.createTextNode('0');\n      const measureCharMajor = document.createElement('div');\n      measureCharMajor.className = 'vis-y-axis vis-major vis-measure';\n      measureCharMajor.appendChild(textMajor);\n      this.dom.frame.appendChild(measureCharMajor);\n\n      this.props.majorCharHeight = measureCharMajor.clientHeight;\n      this.props.majorCharWidth = measureCharMajor.clientWidth;\n\n      this.dom.frame.removeChild(measureCharMajor);\n    }\n\n    if (!('titleCharHeight' in this.props)) {\n      const textTitle = document.createTextNode('0');\n      const measureCharTitle = document.createElement('div');\n      measureCharTitle.className = 'vis-y-axis vis-title vis-measure';\n      measureCharTitle.appendChild(textTitle);\n      this.dom.frame.appendChild(measureCharTitle);\n\n      this.props.titleCharHeight = measureCharTitle.clientHeight;\n      this.props.titleCharWidth = measureCharTitle.clientWidth;\n\n      this.dom.frame.removeChild(measureCharTitle);\n    }\n  }\n}\n\nexport default DataAxis;\n","import * as DOMutil from '../../../DOMutil';\n\n/**\n *\n * @param {number | string} groupId\n * @param {Object} options   // TODO: Describe options\n *\n * @constructor Points\n */\nfunction Points(groupId, options) {  // eslint-disable-line no-unused-vars\n}\n\n/**\n * draw the data points\n *\n * @param {Array} dataset\n * @param {GraphGroup} group\n * @param {Object} framework            | SVG DOM element\n * @param {number} [offset]\n */\nPoints.draw = function (dataset, group, framework, offset) {\n  offset = offset || 0;\n  var callback = getCallback(framework, group);\n\n  for (var i = 0; i < dataset.length; i++) {\n    if (!callback) {\n      // draw the point the simple way.\n      DOMutil.drawPoint(dataset[i].screen_x + offset, dataset[i].screen_y, getGroupTemplate(group), framework.svgElements, framework.svg, dataset[i].label);\n    }\n    else {\n      var callbackResult = callback(dataset[i], group); // result might be true, false or an object\n      if (callbackResult === true || typeof callbackResult === 'object') {\n        DOMutil.drawPoint(dataset[i].screen_x + offset, dataset[i].screen_y, getGroupTemplate(group, callbackResult), framework.svgElements, framework.svg, dataset[i].label);\n      }\n    }\n  }\n};\n\nPoints.drawIcon = function (group, x, y, iconWidth, iconHeight, framework) {\n  var fillHeight = iconHeight * 0.5;\n\n  var outline = DOMutil.getSVGElement(\"rect\", framework.svgElements, framework.svg);\n  outline.setAttributeNS(null, \"x\", x);\n  outline.setAttributeNS(null, \"y\", y - fillHeight);\n  outline.setAttributeNS(null, \"width\", iconWidth);\n  outline.setAttributeNS(null, \"height\", 2 * fillHeight);\n  outline.setAttributeNS(null, \"class\", \"vis-outline\");\n\n  //Don't call callback on icon\n  DOMutil.drawPoint(x + 0.5 * iconWidth, y, getGroupTemplate(group), framework.svgElements, framework.svg);\n};\n\n/**\n *\n * @param {vis.Group} group\n * @param {any} callbackResult\n * @returns {{style: *, styles: (*|string), size: *, className: *}}\n */\nfunction getGroupTemplate(group, callbackResult) {\n  callbackResult = (typeof callbackResult === 'undefined') ? {} : callbackResult;\n  return {\n    style: callbackResult.style || group.options.drawPoints.style,\n    styles: callbackResult.styles || group.options.drawPoints.styles,\n    size: callbackResult.size || group.options.drawPoints.size,\n    className: callbackResult.className || group.className\n  };\n}\n\n/**\n *\n * @param {Object} framework            | SVG DOM element\n * @param {vis.Group} group\n * @returns {function}\n */\nfunction getCallback(framework, group) {\n  var callback = undefined;\n  // check for the graph2d onRender\n  if (framework.options && framework.options.drawPoints && framework.options.drawPoints.onRender && typeof framework.options.drawPoints.onRender == 'function') {\n    callback = framework.options.drawPoints.onRender;\n  }\n\n  // override it with the group onRender if defined\n  if (group.group.options && group.group.options.drawPoints && group.group.options.drawPoints.onRender && typeof group.group.options.drawPoints.onRender == 'function') {\n    callback = group.group.options.drawPoints.onRender;\n  }\n  return callback;\n}\n\nexport default Points;\n","import * as DOMutil from '../../../DOMutil';\nimport Points from './points';\n\n/**\n *\n * @param {vis.GraphGroup.id} groupId\n * @param {Object} options   // TODO: Describe options\n * @constructor Bargraph\n */\nfunction Bargraph(groupId, options) {  // eslint-disable-line no-unused-vars\n}\n\nBargraph.drawIcon = function (group, x, y, iconWidth, iconHeight, framework) {\n  var fillHeight = iconHeight * 0.5;\n  var outline = DOMutil.getSVGElement(\"rect\", framework.svgElements, framework.svg);\n  outline.setAttributeNS(null, \"x\", x);\n  outline.setAttributeNS(null, \"y\", y - fillHeight);\n  outline.setAttributeNS(null, \"width\", iconWidth);\n  outline.setAttributeNS(null, \"height\", 2 * fillHeight);\n  outline.setAttributeNS(null, \"class\", \"vis-outline\");\n\n  var barWidth = Math.round(0.3 * iconWidth);\n  var originalWidth = group.options.barChart.width;\n  var scale = originalWidth / barWidth;\n  var bar1Height = Math.round(0.4 * iconHeight);\n  var bar2Height = Math.round(0.75 * iconHeight);\n\n  var offset = Math.round((iconWidth - (2 * barWidth)) / 3);\n\n  DOMutil.drawBar(x + 0.5 * barWidth + offset, y + fillHeight - bar1Height - 1, barWidth, bar1Height, group.className + ' vis-bar', framework.svgElements, framework.svg, group.style);\n  DOMutil.drawBar(x + 1.5 * barWidth + offset + 2, y + fillHeight - bar2Height - 1, barWidth, bar2Height, group.className + ' vis-bar', framework.svgElements, framework.svg, group.style);\n\n  if (group.options.drawPoints.enabled == true) {\n    var groupTemplate = {\n      style: group.options.drawPoints.style,\n      styles: group.options.drawPoints.styles,\n      size: (group.options.drawPoints.size / scale),\n      className: group.className\n    };\n    DOMutil.drawPoint(x + 0.5 * barWidth + offset, y + fillHeight - bar1Height - 1, groupTemplate, framework.svgElements, framework.svg);\n    DOMutil.drawPoint(x + 1.5 * barWidth + offset + 2, y + fillHeight - bar2Height - 1, groupTemplate, framework.svgElements, framework.svg);\n  }\n};\n\n/**\n * draw a bar graph\n *\n * @param {Array.<vis.GraphGroup.id>} groupIds\n * @param {Object} processedGroupData\n * @param {{svg: Object, svgElements: Array.<Object>, options: Object, groups: Array.<vis.Group>}} framework\n */\nBargraph.draw = function (groupIds, processedGroupData, framework) {\n  var combinedData = [];\n  var intersections = {};\n  var coreDistance;\n  var key, drawData;\n  var group;\n  var i, j;\n  var barPoints = 0;\n\n  // combine all barchart data\n  for (i = 0; i < groupIds.length; i++) {\n    group = framework.groups[groupIds[i]];\n    if (group.options.style === 'bar') {\n      if (group.visible === true && (framework.options.groups.visibility[groupIds[i]] === undefined || framework.options.groups.visibility[groupIds[i]] === true)) {\n        for (j = 0; j < processedGroupData[groupIds[i]].length; j++) {\n          combinedData.push({\n            screen_x: processedGroupData[groupIds[i]][j].screen_x,\n            screen_end: processedGroupData[groupIds[i]][j].screen_end,\n            screen_y: processedGroupData[groupIds[i]][j].screen_y,\n            x: processedGroupData[groupIds[i]][j].x,\n            end: processedGroupData[groupIds[i]][j].end,\n            y: processedGroupData[groupIds[i]][j].y,\n            groupId: groupIds[i],\n            label: processedGroupData[groupIds[i]][j].label\n          });\n          barPoints += 1;\n        }\n      }\n    }\n  }\n\n  if (barPoints === 0) {\n    return;\n  }\n\n  // sort by time and by group\n  combinedData.sort(function (a, b) {\n    if (a.screen_x === b.screen_x) {\n      return a.groupId < b.groupId ? -1 : 1;\n    }\n    else {\n      return a.screen_x - b.screen_x;\n    }\n  });\n\n  // get intersections\n  Bargraph._getDataIntersections(intersections, combinedData);\n\n  // plot barchart\n  for (i = 0; i < combinedData.length; i++) {\n    group = framework.groups[combinedData[i].groupId];\n    var minWidth = group.options.barChart.minWidth != undefined ? group.options.barChart.minWidth : 0.1 * group.options.barChart.width;\n\n    key = combinedData[i].screen_x;\n    var heightOffset = 0;\n    if (intersections[key] === undefined) {\n      if (i + 1 < combinedData.length) {\n        coreDistance = Math.abs(combinedData[i + 1].screen_x - key);\n      }\n      drawData = Bargraph._getSafeDrawData(coreDistance, group, minWidth);\n    }\n    else {\n      var nextKey = i + (intersections[key].amount - intersections[key].resolved);\n      if (nextKey < combinedData.length) {\n        coreDistance = Math.abs(combinedData[nextKey].screen_x - key);\n      }\n      drawData = Bargraph._getSafeDrawData(coreDistance, group, minWidth);\n      intersections[key].resolved += 1;\n\n      if (group.options.stack === true && group.options.excludeFromStacking !== true) {\n        if (combinedData[i].screen_y < group.zeroPosition) {\n          heightOffset = intersections[key].accumulatedNegative;\n          intersections[key].accumulatedNegative += group.zeroPosition - combinedData[i].screen_y;\n        }\n        else {\n          heightOffset = intersections[key].accumulatedPositive;\n          intersections[key].accumulatedPositive += group.zeroPosition - combinedData[i].screen_y;\n        }\n      }\n      else if (group.options.barChart.sideBySide === true) {\n        drawData.width = drawData.width / intersections[key].amount;\n        drawData.offset += (intersections[key].resolved) * drawData.width - (0.5 * drawData.width * (intersections[key].amount + 1));\n      }\n    }\n    \n    let dataWidth = drawData.width;\n    let start = combinedData[i].screen_x;\n\n    // are we drawing explicit boxes? (we supplied an end value)\n    if (combinedData[i].screen_end != undefined){\n      dataWidth = combinedData[i].screen_end - combinedData[i].screen_x;\n      start += (dataWidth * 0.5);\n    }\n    else {\n      start += drawData.offset\n    }\n\n    DOMutil.drawBar(start, combinedData[i].screen_y - heightOffset, dataWidth, group.zeroPosition - combinedData[i].screen_y, group.className + ' vis-bar', framework.svgElements, framework.svg, group.style);\n\n    // draw points\n    if (group.options.drawPoints.enabled === true) {\n      let pointData = {\n        screen_x: combinedData[i].screen_x,\n        screen_y: combinedData[i].screen_y - heightOffset,\n        x: combinedData[i].x,\n        y: combinedData[i].y,\n        groupId: combinedData[i].groupId,\n        label: combinedData[i].label\n      };\n      Points.draw([pointData], group, framework, drawData.offset);\n      //DOMutil.drawPoint(combinedData[i].x + drawData.offset, combinedData[i].y, group, framework.svgElements, framework.svg);\n    }\n  }\n};\n\n\n/**\n * Fill the intersections object with counters of how many datapoints share the same x coordinates\n * @param {Object} intersections\n * @param {Array.<Object>} combinedData\n * @private\n */\nBargraph._getDataIntersections = function (intersections, combinedData) {\n  // get intersections\n  var coreDistance;\n  for (var i = 0; i < combinedData.length; i++) {\n    if (i + 1 < combinedData.length) {\n      coreDistance = Math.abs(combinedData[i + 1].screen_x - combinedData[i].screen_x);\n    }\n    if (i > 0) {\n      coreDistance = Math.min(coreDistance, Math.abs(combinedData[i - 1].screen_x - combinedData[i].screen_x));\n    }\n    if (coreDistance === 0) {\n      if (intersections[combinedData[i].screen_x] === undefined) {\n        intersections[combinedData[i].screen_x] = {\n          amount: 0,\n          resolved: 0,\n          accumulatedPositive: 0,\n          accumulatedNegative: 0\n        };\n      }\n      intersections[combinedData[i].screen_x].amount += 1;\n    }\n  }\n};\n\n\n/**\n * Get the width and offset for bargraphs based on the coredistance between datapoints\n *\n * @param {number} coreDistance\n * @param {vis.Group} group\n * @param {number} minWidth\n * @returns {{width: number, offset: number}}\n * @private\n */\nBargraph._getSafeDrawData = function (coreDistance, group, minWidth) {\n  var width, offset;\n  if (coreDistance < group.options.barChart.width && coreDistance > 0) {\n    width = coreDistance < minWidth ? minWidth : coreDistance\n\n    offset = 0; // recalculate offset with the new width;\n    if (group.options.barChart.align === 'left') {\n      offset -= 0.5 * coreDistance;\n    }\n    else if (group.options.barChart.align === 'right') {\n      offset += 0.5 * coreDistance;\n    }\n  }\n  else {\n    // default settings\n    width = group.options.barChart.width;\n    offset = 0;\n    if (group.options.barChart.align === 'left') {\n      offset -= 0.5 * group.options.barChart.width;\n    }\n    else if (group.options.barChart.align === 'right') {\n      offset += 0.5 * group.options.barChart.width;\n    }\n  }\n\n  return {width: width, offset: offset};\n};\n\nBargraph.getStackedYRange = function (combinedData, groupRanges, groupIds, groupLabel, orientation) {\n  if (combinedData.length > 0) {\n    // sort by time and by group\n    combinedData.sort(function (a, b) {\n      if (a.screen_x === b.screen_x) {\n        return a.groupId < b.groupId ? -1 : 1;\n      }\n      else {\n        return a.screen_x - b.screen_x;\n      }\n    });\n    var intersections = {};\n\n    Bargraph._getDataIntersections(intersections, combinedData);\n    groupRanges[groupLabel] = Bargraph._getStackedYRange(intersections, combinedData);\n    groupRanges[groupLabel].yAxisOrientation = orientation;\n    groupIds.push(groupLabel);\n  }\n};\n\nBargraph._getStackedYRange = function (intersections, combinedData) {\n  var key;\n  var yMin = combinedData[0].screen_y;\n  var yMax = combinedData[0].screen_y;\n  for (var i = 0; i < combinedData.length; i++) {\n    key = combinedData[i].screen_x;\n    if (intersections[key] === undefined) {\n      yMin = yMin > combinedData[i].screen_y ? combinedData[i].screen_y : yMin;\n      yMax = yMax < combinedData[i].screen_y ? combinedData[i].screen_y : yMax;\n    }\n    else {\n      if (combinedData[i].screen_y < 0) {\n        intersections[key].accumulatedNegative += combinedData[i].screen_y;\n      }\n      else {\n        intersections[key].accumulatedPositive += combinedData[i].screen_y;\n      }\n    }\n  }\n  for (var xpos in intersections) {\n    if (intersections.hasOwnProperty(xpos)) {\n      yMin = yMin > intersections[xpos].accumulatedNegative ? intersections[xpos].accumulatedNegative : yMin;\n      yMin = yMin > intersections[xpos].accumulatedPositive ? intersections[xpos].accumulatedPositive : yMin;\n      yMax = yMax < intersections[xpos].accumulatedNegative ? intersections[xpos].accumulatedNegative : yMax;\n      yMax = yMax < intersections[xpos].accumulatedPositive ? intersections[xpos].accumulatedPositive : yMax;\n    }\n  }\n\n  return {min: yMin, max: yMax};\n};\n\nexport default Bargraph;","import * as DOMutil from '../../../DOMutil';\n\n/**\n *\n * @param {vis.GraphGroup.id} groupId\n * @param {Object} options   // TODO: Describe options\n * @constructor Line\n */\nfunction Line(groupId, options) {  // eslint-disable-line no-unused-vars\n}\n\nLine.calcPath = function (dataset, group) {\n    if (dataset != null) {\n        if (dataset.length > 0) {\n            var d = [];\n\n            // construct path from dataset\n            if (group.options.interpolation.enabled == true) {\n                d = Line._catmullRom(dataset, group);\n            }\n            else {\n                d = Line._linear(dataset);\n            }\n            return d;\n        }\n    }\n};\n\nLine.drawIcon = function (group, x, y, iconWidth, iconHeight, framework) {\n    var fillHeight = iconHeight * 0.5;\n    var path, fillPath;\n\n    var outline = DOMutil.getSVGElement(\"rect\", framework.svgElements, framework.svg);\n    outline.setAttributeNS(null, \"x\", x);\n    outline.setAttributeNS(null, \"y\", y - fillHeight);\n    outline.setAttributeNS(null, \"width\", iconWidth);\n    outline.setAttributeNS(null, \"height\", 2 * fillHeight);\n    outline.setAttributeNS(null, \"class\", \"vis-outline\");\n\n    path = DOMutil.getSVGElement(\"path\", framework.svgElements, framework.svg);\n    path.setAttributeNS(null, \"class\", group.className);\n    if (group.style !== undefined) {\n        path.setAttributeNS(null, \"style\", group.style);\n    }\n\n    path.setAttributeNS(null, \"d\", \"M\" + x + \",\" + y + \" L\" + (x + iconWidth) + \",\" + y + \"\");\n    if (group.options.shaded.enabled == true) {\n        fillPath = DOMutil.getSVGElement(\"path\", framework.svgElements, framework.svg);\n        if (group.options.shaded.orientation == 'top') {\n            fillPath.setAttributeNS(null, \"d\", \"M\" + x + \", \" + (y - fillHeight) +\n              \"L\" + x + \",\" + y + \" L\" + (x + iconWidth) + \",\" + y + \" L\" + (x + iconWidth) + \",\" + (y - fillHeight));\n        }\n        else {\n            fillPath.setAttributeNS(null, \"d\", \"M\" + x + \",\" + y + \" \" +\n              \"L\" + x + \",\" + (y + fillHeight) + \" \" +\n              \"L\" + (x + iconWidth) + \",\" + (y + fillHeight) +\n              \"L\" + (x + iconWidth) + \",\" + y);\n        }\n        fillPath.setAttributeNS(null, \"class\", group.className + \" vis-icon-fill\");\n        if (group.options.shaded.style !== undefined && group.options.shaded.style !== \"\") {\n            fillPath.setAttributeNS(null, \"style\", group.options.shaded.style);\n        }\n    }\n\n    if (group.options.drawPoints.enabled == true) {\n        var groupTemplate = {\n            style: group.options.drawPoints.style,\n            styles: group.options.drawPoints.styles,\n            size: group.options.drawPoints.size,\n            className: group.className\n        };\n        DOMutil.drawPoint(x + 0.5 * iconWidth, y, groupTemplate, framework.svgElements, framework.svg);\n    }\n};\n\nLine.drawShading = function (pathArray, group, subPathArray, framework) {\n    // append shading to the path\n    if (group.options.shaded.enabled == true) {\n        var svgHeight = Number(framework.svg.style.height.replace('px',''));\n        var fillPath = DOMutil.getSVGElement('path', framework.svgElements, framework.svg);\n        var type = \"L\";\n        if (group.options.interpolation.enabled == true){\n            type = \"C\";\n        }\n        var dFill;\n        var zero = 0;\n        if (group.options.shaded.orientation == 'top') {\n            zero = 0;\n        }\n        else if (group.options.shaded.orientation == 'bottom') {\n            zero = svgHeight;\n        }\n        else {\n            zero = Math.min(Math.max(0, group.zeroPosition), svgHeight);\n        }\n        if (group.options.shaded.orientation == 'group' && (subPathArray != null && subPathArray != undefined)) {\n            dFill = 'M' + pathArray[0][0]+ \",\"+pathArray[0][1] + \" \" +\n                    this.serializePath(pathArray,type,false) +\n                    ' L'+ subPathArray[subPathArray.length-1][0]+ \",\" + subPathArray[subPathArray.length-1][1] + \" \" +\n                    this.serializePath(subPathArray,type,true) +\n                    subPathArray[0][0]+ \",\"+subPathArray[0][1] + \" Z\";\n        }\n        else {\n            dFill = 'M' + pathArray[0][0]+ \",\"+pathArray[0][1] + \" \" +\n                    this.serializePath(pathArray,type,false) +\n                    ' V' + zero + ' H'+ pathArray[0][0] + \" Z\";\n        }\n\n        fillPath.setAttributeNS(null, 'class', group.className + ' vis-fill');\n        if (group.options.shaded.style !== undefined) {\n            fillPath.setAttributeNS(null, 'style', group.options.shaded.style);\n        }\n        fillPath.setAttributeNS(null, 'd', dFill);\n    }\n};\n\n/**\n * draw a line graph\n *\n * @param {Array.<Object>} pathArray\n * @param {vis.Group} group\n * @param {{svg: Object, svgElements: Array.<Object>, options: Object, groups: Array.<vis.Group>}} framework\n */\nLine.draw = function (pathArray, group, framework) {\n    if (pathArray != null && pathArray != undefined) {\n        var path = DOMutil.getSVGElement('path', framework.svgElements, framework.svg);\n        path.setAttributeNS(null, \"class\", group.className);\n        if (group.style !== undefined) {\n            path.setAttributeNS(null, \"style\", group.style);\n        }\n\n        var type = \"L\";\n        if (group.options.interpolation.enabled == true){\n            type = \"C\";\n        }\n        // copy properties to path for drawing.\n        path.setAttributeNS(null, 'd', 'M' + pathArray[0][0]+ \",\"+pathArray[0][1] + \" \" + this.serializePath(pathArray,type,false));\n    }\n};\n\nLine.serializePath = function(pathArray,type,inverse){\n    if (pathArray.length < 2){\n        //Too little data to create a path.\n        return \"\";\n    }\n    var d = type;\n    var i;\n    if (inverse){\n        for (i = pathArray.length-2; i > 0; i--){\n            d += pathArray[i][0] + \",\" + pathArray[i][1] + \" \";\n        }\n    }\n    else {\n        for (i = 1; i < pathArray.length; i++){\n            d += pathArray[i][0] + \",\" + pathArray[i][1] + \" \";\n        }\n    }\n    return d;\n};\n\n/**\n * This uses an uniform parametrization of the interpolation algorithm:\n * 'On the Parameterization of Catmull-Rom Curves' by Cem Yuksel et al.\n * @param {Array.<Object>} data\n * @returns {string}\n * @private\n */\nLine._catmullRomUniform = function (data) {\n    // catmull rom\n    var p0, p1, p2, p3, bp1, bp2;\n    var d = [];\n    d.push( [ Math.round(data[0].screen_x) , Math.round(data[0].screen_y) ]);\n    var normalization = 1 / 6;\n    var length = data.length;\n    for (var i = 0; i < length - 1; i++) {\n\n        p0 = (i == 0) ? data[0] : data[i - 1];\n        p1 = data[i];\n        p2 = data[i + 1];\n        p3 = (i + 2 < length) ? data[i + 2] : p2;\n\n\n        // Catmull-Rom to Cubic Bezier conversion matrix\n        //    0       1       0       0\n        //  -1/6      1      1/6      0\n        //    0      1/6      1     -1/6\n        //    0       0       1       0\n\n        //    bp0 = { x: p1.x,                               y: p1.y };\n        bp1 = {\n            screen_x: ((-p0.screen_x + 6 * p1.screen_x + p2.screen_x) * normalization),\n            screen_y: ((-p0.screen_y + 6 * p1.screen_y + p2.screen_y) * normalization)\n        };\n        bp2 = {\n            screen_x: (( p1.screen_x + 6 * p2.screen_x - p3.screen_x) * normalization),\n            screen_y: (( p1.screen_y + 6 * p2.screen_y - p3.screen_y) * normalization)\n        };\n        //    bp0 = { x: p2.x,                               y: p2.y };\n\n        d.push( [ bp1.screen_x , bp1.screen_y ]);\n        d.push( [ bp2.screen_x , bp2.screen_y ]);\n        d.push( [ p2.screen_x  , p2.screen_y  ]);\n    }\n\n    return d;\n};\n\n/**\n * This uses either the chordal or centripetal parameterization of the catmull-rom algorithm.\n * By default, the centripetal parameterization is used because this gives the nicest results.\n * These parameterizations are relatively heavy because the distance between 4 points have to be calculated.\n *\n * One optimization can be used to reuse distances since this is a sliding window approach.\n * @param {Array.<Object>} data\n * @param {vis.GraphGroup} group\n * @returns {string}\n * @private\n */\nLine._catmullRom = function (data, group) {\n    var alpha = group.options.interpolation.alpha;\n    if (alpha == 0 || alpha === undefined) {\n        return this._catmullRomUniform(data);\n    }\n    else {\n        var p0, p1, p2, p3, bp1, bp2, d1, d2, d3, A, B, N, M;\n        var d3powA, d2powA, d3pow2A, d2pow2A, d1pow2A, d1powA;\n        var d = [];\n        d.push( [ Math.round(data[0].screen_x) , Math.round(data[0].screen_y) ]);\n        var length = data.length;\n        for (var i = 0; i < length - 1; i++) {\n\n            p0 = (i == 0) ? data[0] : data[i - 1];\n            p1 = data[i];\n            p2 = data[i + 1];\n            p3 = (i + 2 < length) ? data[i + 2] : p2;\n\n            d1 = Math.sqrt(Math.pow(p0.screen_x - p1.screen_x, 2) + Math.pow(p0.screen_y - p1.screen_y, 2));\n            d2 = Math.sqrt(Math.pow(p1.screen_x - p2.screen_x, 2) + Math.pow(p1.screen_y - p2.screen_y, 2));\n            d3 = Math.sqrt(Math.pow(p2.screen_x - p3.screen_x, 2) + Math.pow(p2.screen_y - p3.screen_y, 2));\n\n            // Catmull-Rom to Cubic Bezier conversion matrix\n\n            // A = 2d1^2a + 3d1^a * d2^a + d3^2a\n            // B = 2d3^2a + 3d3^a * d2^a + d2^2a\n\n            // [   0             1            0          0          ]\n            // [   -d2^2a /N     A/N          d1^2a /N   0          ]\n            // [   0             d3^2a /M     B/M        -d2^2a /M  ]\n            // [   0             0            1          0          ]\n\n            d3powA = Math.pow(d3, alpha);\n            d3pow2A = Math.pow(d3, 2 * alpha);\n            d2powA = Math.pow(d2, alpha);\n            d2pow2A = Math.pow(d2, 2 * alpha);\n            d1powA = Math.pow(d1, alpha);\n            d1pow2A = Math.pow(d1, 2 * alpha);\n\n            A = 2 * d1pow2A + 3 * d1powA * d2powA + d2pow2A;\n            B = 2 * d3pow2A + 3 * d3powA * d2powA + d2pow2A;\n            N = 3 * d1powA * (d1powA + d2powA);\n            if (N > 0) {\n                N = 1 / N;\n            }\n            M = 3 * d3powA * (d3powA + d2powA);\n            if (M > 0) {\n                M = 1 / M;\n            }\n\n            bp1 = {\n                screen_x: ((-d2pow2A * p0.screen_x + A * p1.screen_x + d1pow2A * p2.screen_x) * N),\n                screen_y: ((-d2pow2A * p0.screen_y + A * p1.screen_y + d1pow2A * p2.screen_y) * N)\n            };\n\n            bp2 = {\n                screen_x: (( d3pow2A * p1.screen_x + B * p2.screen_x - d2pow2A * p3.screen_x) * M),\n                screen_y: (( d3pow2A * p1.screen_y + B * p2.screen_y - d2pow2A * p3.screen_y) * M)\n            };\n\n            if (bp1.screen_x == 0 && bp1.screen_y == 0) {\n                bp1 = p1;\n            }\n            if (bp2.screen_x == 0 && bp2.screen_y == 0) {\n                bp2 = p2;\n            }\n            d.push( [ bp1.screen_x , bp1.screen_y ]);\n            d.push( [ bp2.screen_x , bp2.screen_y ]);\n            d.push( [ p2.screen_x  , p2.screen_y  ]);\n        }\n\n        return d;\n    }\n};\n\n/**\n * this generates the SVG path for a linear drawing between datapoints.\n * @param {Array.<Object>} data\n * @returns {string}\n * @private\n */\nLine._linear = function (data) {\n    // linear\n    var d = [];\n    for (var i = 0; i < data.length; i++) {\n        d.push([ data[i].screen_x , data[i].screen_y ]);\n    }\n    return d;\n};\n\nexport default Line;\n","import util from '../../util';\nimport Bars from './graph2d_types/bar';\nimport Lines from './graph2d_types/line';\nimport Points from './graph2d_types/points';\n\n/**\n * /**\n * @param {object} group            | the object of the group from the dataset\n * @param {string} groupId          | ID of the group\n * @param {object} options          | the default options\n * @param {array} groupsUsingDefaultStyles  | this array has one entree.\n *                                            It is passed as an array so it is passed by reference.\n *                                            It enumerates through the default styles\n * @constructor GraphGroup\n */\nfunction GraphGroup(group, groupId, options, groupsUsingDefaultStyles) {\n  this.id = groupId;\n  var fields = ['sampling', 'style', 'sort', 'yAxisOrientation', 'barChart', 'drawPoints', 'shaded', 'interpolation', 'zIndex','excludeFromStacking', 'excludeFromLegend'];\n  this.options = util.selectiveBridgeObject(fields, options);\n  this.usingDefaultStyle = group.className === undefined;\n  this.groupsUsingDefaultStyles = groupsUsingDefaultStyles;\n  this.zeroPosition = 0;\n  this.update(group);\n  if (this.usingDefaultStyle == true) {\n    this.groupsUsingDefaultStyles[0] += 1;\n  }\n  this.itemsData = [];\n  this.visible = group.visible === undefined ? true : group.visible;\n}\n\n/**\n * this loads a reference to all items in this group into this group.\n * @param {array} items\n */\nGraphGroup.prototype.setItems = function (items) {\n  if (items != null) {\n    this.itemsData = items;\n    if (this.options.sort == true) {\n      util.insertSort(this.itemsData,function (a, b) {\n        return a.x > b.x ? 1 : -1;\n      });\n    }\n  }\n  else {\n    this.itemsData = [];\n  }\n};\n\nGraphGroup.prototype.getItems = function () {\n  return this.itemsData;\n};\n\n/**\n * this is used for barcharts and shading, this way, we only have to calculate it once.\n * @param {number} pos\n */\nGraphGroup.prototype.setZeroPosition = function (pos) {\n  this.zeroPosition = pos;\n};\n\n/**\n * set the options of the graph group over the default options.\n * @param {Object} options\n */\nGraphGroup.prototype.setOptions = function (options) {\n  if (options !== undefined) {\n    var fields = ['sampling', 'style', 'sort', 'yAxisOrientation', 'barChart', 'zIndex','excludeFromStacking', 'excludeFromLegend'];\n    util.selectiveDeepExtend(fields, this.options, options);\n\n    // if the group's drawPoints is a function delegate the callback to the onRender property\n    if (typeof options.drawPoints == 'function') {\n      options.drawPoints = {\n        onRender: options.drawPoints\n      }\n    }\n\n    util.mergeOptions(this.options, options, 'interpolation');\n    util.mergeOptions(this.options, options, 'drawPoints');\n    util.mergeOptions(this.options, options, 'shaded');\n\n    if (options.interpolation) {\n      if (typeof options.interpolation == 'object') {\n        if (options.interpolation.parametrization) {\n          if (options.interpolation.parametrization == 'uniform') {\n            this.options.interpolation.alpha = 0;\n          }\n          else if (options.interpolation.parametrization == 'chordal') {\n            this.options.interpolation.alpha = 1.0;\n          }\n          else {\n            this.options.interpolation.parametrization = 'centripetal';\n            this.options.interpolation.alpha = 0.5;\n          }\n        }\n      }\n    }\n  }\n};\n\n\n/**\n * this updates the current group class with the latest group dataset entree, used in _updateGroup in linegraph\n * @param {vis.Group} group\n */\nGraphGroup.prototype.update = function (group) {\n  this.group = group;\n  this.content = group.content || 'graph';\n  this.className = group.className || this.className || 'vis-graph-group' + this.groupsUsingDefaultStyles[0] % 10;\n  this.visible = group.visible === undefined ? true : group.visible;\n  this.style = group.style;\n  this.setOptions(group.options);\n};\n\n/**\n * return the legend entree for this group.\n *\n * @param {number} iconWidth\n * @param {number} iconHeight\n * @param {{svg: (*|Element), svgElements: Object, options: Object, groups: Array.<Object>}} framework\n * @param {number} x\n * @param {number} y\n * @returns {{icon: (*|Element), label: (*|string), orientation: *}}\n */\nGraphGroup.prototype.getLegend = function (iconWidth, iconHeight, framework, x, y) {\n  if (framework == undefined || framework == null) {\n    var svg = document.createElementNS('http://www.w3.org/2000/svg', \"svg\");\n    framework = {svg: svg, svgElements:{}, options: this.options, groups: [this]}\n  }\n  if (x == undefined || x == null){\n    x = 0;\n  }\n  if (y == undefined || y == null){\n    y = 0.5 * iconHeight;\n  }\n  switch (this.options.style){\n    case \"line\":\n      Lines.drawIcon(this, x, y, iconWidth, iconHeight, framework);\n      break;\n    case \"points\": //explicit no break\n    case \"point\":\n      Points.drawIcon(this, x, y, iconWidth, iconHeight, framework);\n      break;\n    case \"bar\":\n      Bars.drawIcon(this, x, y, iconWidth, iconHeight, framework);\n      break;\n  }\n  return {icon: framework.svg, label: this.content, orientation: this.options.yAxisOrientation};\n};\n\nGraphGroup.prototype.getYRange = function (groupData) {\n  var yMin = groupData[0].y;\n  var yMax = groupData[0].y;\n  for (var j = 0; j < groupData.length; j++) {\n    yMin = yMin > groupData[j].y ? groupData[j].y : yMin;\n    yMax = yMax < groupData[j].y ? groupData[j].y : yMax;\n  }\n  return {min: yMin, max: yMax, yAxisOrientation: this.options.yAxisOrientation};\n};\n\nexport default GraphGroup;\n","import util from '../../util';\nimport * as DOMutil from '../../DOMutil';\nimport Component from './Component';\n\n/**\n * Legend for Graph2d\n *\n * @param {vis.Graph2d.body} body\n * @param {vis.Graph2d.options} options\n * @param {number} side\n * @param {vis.LineGraph.options} linegraphOptions\n * @constructor Legend\n * @extends Component\n */\nfunction Legend(body, options, side, linegraphOptions) {\n  this.body = body;\n  this.defaultOptions = {\n    enabled: false,\n    icons: true,\n    iconSize: 20,\n    iconSpacing: 6,\n    left: {\n      visible: true,\n      position: 'top-left' // top/bottom - left,center,right\n    },\n    right: {\n      visible: true,\n      position: 'top-right' // top/bottom - left,center,right\n    }\n  };\n\n  this.side = side;\n  this.options = util.extend({}, this.defaultOptions);\n  this.linegraphOptions = linegraphOptions;\n\n  this.svgElements = {};\n  this.dom = {};\n  this.groups = {};\n  this.amountOfGroups = 0;\n  this._create();\n  this.framework = {svg: this.svg, svgElements: this.svgElements, options: this.options, groups: this.groups};\n\n  this.setOptions(options);\n}\n\nLegend.prototype = new Component();\n\nLegend.prototype.clear = function() {\n  this.groups = {};\n  this.amountOfGroups = 0;\n};\n\nLegend.prototype.addGroup = function(label, graphOptions) {\n\n  // Include a group only if the group option 'excludeFromLegend: false' is not set.\n  if (graphOptions.options.excludeFromLegend != true) {\n    if (!this.groups.hasOwnProperty(label)) {\n      this.groups[label] = graphOptions;\n    }\n    this.amountOfGroups += 1;\n  }\n};\n\nLegend.prototype.updateGroup = function(label, graphOptions) {\n  this.groups[label] = graphOptions;\n};\n\nLegend.prototype.removeGroup = function(label) {\n  if (this.groups.hasOwnProperty(label)) {\n    delete this.groups[label];\n    this.amountOfGroups -= 1;\n  }\n};\n\nLegend.prototype._create = function() {\n  this.dom.frame = document.createElement('div');\n  this.dom.frame.className = 'vis-legend';\n  this.dom.frame.style.position = \"absolute\";\n  this.dom.frame.style.top = \"10px\";\n  this.dom.frame.style.display = \"block\";\n\n  this.dom.textArea = document.createElement('div');\n  this.dom.textArea.className = 'vis-legend-text';\n  this.dom.textArea.style.position = \"relative\";\n  this.dom.textArea.style.top = \"0px\";\n\n  this.svg = document.createElementNS('http://www.w3.org/2000/svg',\"svg\");\n  this.svg.style.position = 'absolute';\n  this.svg.style.top = 0 +'px';\n  this.svg.style.width = this.options.iconSize + 5 + 'px';\n  this.svg.style.height = '100%';\n\n  this.dom.frame.appendChild(this.svg);\n  this.dom.frame.appendChild(this.dom.textArea);\n};\n\n/**\n * Hide the component from the DOM\n */\nLegend.prototype.hide = function() {\n  // remove the frame containing the items\n  if (this.dom.frame.parentNode) {\n    this.dom.frame.parentNode.removeChild(this.dom.frame);\n  }\n};\n\n/**\n * Show the component in the DOM (when not already visible).\n */\nLegend.prototype.show = function() {\n  // show frame containing the items\n  if (!this.dom.frame.parentNode) {\n    this.body.dom.center.appendChild(this.dom.frame);\n  }\n};\n\nLegend.prototype.setOptions = function(options) {\n  var fields = ['enabled','orientation','icons','left','right'];\n  util.selectiveDeepExtend(fields, this.options, options);\n};\n\nLegend.prototype.redraw = function() {\n  var activeGroups = 0;\n  var groupArray = Object.keys(this.groups);\n  groupArray.sort(function (a,b) {\n    return (a < b ? -1 : 1);\n  })\n\n  for (var i = 0; i < groupArray.length; i++) {\n    var groupId = groupArray[i];\n    if (this.groups[groupId].visible == true && (this.linegraphOptions.visibility[groupId] === undefined || this.linegraphOptions.visibility[groupId] == true)) {\n      activeGroups++;\n    }\n  }\n\n  if (this.options[this.side].visible == false || this.amountOfGroups == 0 || this.options.enabled == false || activeGroups == 0) {\n    this.hide();\n  }\n  else {\n    this.show();\n    if (this.options[this.side].position == 'top-left' || this.options[this.side].position == 'bottom-left') {\n      this.dom.frame.style.left = '4px';\n      this.dom.frame.style.textAlign = \"left\";\n      this.dom.textArea.style.textAlign = \"left\";\n      this.dom.textArea.style.left = (this.options.iconSize + 15) + 'px';\n      this.dom.textArea.style.right = '';\n      this.svg.style.left = 0 +'px';\n      this.svg.style.right = '';\n    }\n    else {\n      this.dom.frame.style.right = '4px';\n      this.dom.frame.style.textAlign = \"right\";\n      this.dom.textArea.style.textAlign = \"right\";\n      this.dom.textArea.style.right = (this.options.iconSize + 15) + 'px';\n      this.dom.textArea.style.left = '';\n      this.svg.style.right = 0 +'px';\n      this.svg.style.left = '';\n    }\n\n    if (this.options[this.side].position == 'top-left' || this.options[this.side].position == 'top-right') {\n      this.dom.frame.style.top = 4 - Number(this.body.dom.center.style.top.replace(\"px\",\"\")) + 'px';\n      this.dom.frame.style.bottom = '';\n    }\n    else {\n      var scrollableHeight = this.body.domProps.center.height - this.body.domProps.centerContainer.height;\n      this.dom.frame.style.bottom = 4 + scrollableHeight + Number(this.body.dom.center.style.top.replace(\"px\",\"\")) + 'px';\n      this.dom.frame.style.top = '';\n    }\n\n    if (this.options.icons == false) {\n      this.dom.frame.style.width = this.dom.textArea.offsetWidth + 10 + 'px';\n      this.dom.textArea.style.right = '';\n      this.dom.textArea.style.left = '';\n      this.svg.style.width = '0px';\n    }\n    else {\n      this.dom.frame.style.width = this.options.iconSize + 15 + this.dom.textArea.offsetWidth + 10 + 'px'\n      this.drawLegendIcons();\n    }\n\n    var content = '';\n    for (i = 0; i < groupArray.length; i++) {\n      groupId = groupArray[i];\n      if (this.groups[groupId].visible == true && (this.linegraphOptions.visibility[groupId] === undefined || this.linegraphOptions.visibility[groupId] == true)) {\n        content += this.groups[groupId].content + '<br />';\n      }\n    }\n    this.dom.textArea.innerHTML = content;\n    this.dom.textArea.style.lineHeight = ((0.75 * this.options.iconSize) + this.options.iconSpacing) + 'px';\n  }\n};\n\nLegend.prototype.drawLegendIcons = function() {\n  if (this.dom.frame.parentNode) {\n    var groupArray = Object.keys(this.groups);\n    groupArray.sort(function (a,b) {\n      return (a < b ? -1 : 1);\n    });\n\n    // this resets the elements so the order is maintained\n    DOMutil.resetElements(this.svgElements);\n\n    var padding = window.getComputedStyle(this.dom.frame).paddingTop;\n    var iconOffset = Number(padding.replace('px',''));\n    var x = iconOffset;\n    var iconWidth = this.options.iconSize;\n    var iconHeight = 0.75 * this.options.iconSize;\n    var y = iconOffset + 0.5 * iconHeight + 3;\n\n    this.svg.style.width = iconWidth + 5 + iconOffset + 'px';\n\n    for (var i = 0; i < groupArray.length; i++) {\n      var groupId = groupArray[i];\n      if (this.groups[groupId].visible == true && (this.linegraphOptions.visibility[groupId] === undefined || this.linegraphOptions.visibility[groupId] == true)) {\n        this.groups[groupId].getLegend(iconWidth, iconHeight, this.framework, x, y);\n        y += iconHeight + this.options.iconSpacing;\n      }\n    }\n  }\n};\n\nexport default Legend;\n","import { v4 as randomUUID } from \"uuid\";\nimport util, { typeCoerceDataSet } from '../../util';\nimport * as DOMutil from '../../DOMutil';\nimport { DataSet }  from 'vis-data/esnext';\nimport { DataView } from 'vis-data/esnext';\nimport Component from './Component';\nimport DataAxis from './DataAxis';\nimport GraphGroup from './GraphGroup';\nimport Legend from './Legend';\nimport Bars from './graph2d_types/bar';\nimport Lines from './graph2d_types/line';\nimport Points from './graph2d_types/points';\n\nvar UNGROUPED = '__ungrouped__'; // reserved group id for ungrouped items\n\n/**\n * This is the constructor of the LineGraph. It requires a Timeline body and options.\n *\n * @param {vis.Timeline.body} body\n * @param {Object} options\n * @constructor LineGraph\n * @extends Component\n */\nfunction LineGraph(body, options) {\n  this.id = randomUUID();\n  this.body = body;\n\n  this.defaultOptions = {\n    yAxisOrientation: 'left',\n    defaultGroup: 'default',\n    sort: true,\n    sampling: true,\n    stack: false,\n    graphHeight: '400px',\n    shaded: {\n      enabled: false,\n      orientation: 'bottom' // top, bottom, zero\n    },\n    style: 'line', // line, bar\n    barChart: {\n      width: 50,\n      sideBySide: false,\n      align: 'center' // left, center, right\n    },\n    interpolation: {\n      enabled: true,\n      parametrization: 'centripetal', // uniform (alpha = 0.0), chordal (alpha = 1.0), centripetal (alpha = 0.5)\n      alpha: 0.5\n    },\n    drawPoints: {\n      enabled: true,\n      size: 6,\n      style: 'square' // square, circle\n    },\n    dataAxis: {}, //Defaults are done on DataAxis level\n    legend: {}, //Defaults are done on Legend level\n    groups: {\n      visibility: {}\n    }\n  };\n\n  // options is shared by this lineGraph and all its items\n  this.options = util.extend({}, this.defaultOptions);\n  this.dom = {};\n  this.props = {};\n  this.hammer = null;\n  this.groups = {};\n  this.abortedGraphUpdate = false;\n  this.updateSVGheight = false;\n  this.updateSVGheightOnResize = false;\n  this.forceGraphUpdate = true;\n\n  var me = this;\n  this.itemsData = null;    // DataSet\n  this.groupsData = null;   // DataSet\n\n  // listeners for the DataSet of the items\n  this.itemListeners = {\n    'add': function (event, params, senderId) {  // eslint-disable-line no-unused-vars\n      me._onAdd(params.items);\n    },\n    'update': function (event, params, senderId) {  // eslint-disable-line no-unused-vars\n      me._onUpdate(params.items);\n    },\n    'remove': function (event, params, senderId) {  // eslint-disable-line no-unused-vars\n      me._onRemove(params.items);\n    }\n  };\n\n  // listeners for the DataSet of the groups\n  this.groupListeners = {\n    'add': function (event, params, senderId) {  // eslint-disable-line no-unused-vars\n      me._onAddGroups(params.items);\n    },\n    'update': function (event, params, senderId) {  // eslint-disable-line no-unused-vars\n      me._onUpdateGroups(params.items);\n    },\n    'remove': function (event, params, senderId) {  // eslint-disable-line no-unused-vars\n      me._onRemoveGroups(params.items);\n    }\n  };\n\n  this.items = {};      // object with an Item for every data item\n  this.selection = [];  // list with the ids of all selected nodes\n  this.lastStart = this.body.range.start;\n  this.touchParams = {}; // stores properties while dragging\n\n  this.svgElements = {};\n  this.setOptions(options);\n  this.groupsUsingDefaultStyles = [0];\n  this.body.emitter.on('rangechanged', function () {\n    me.lastStart = me.body.range.start;\n    me.svg.style.left = util.option.asSize(-me.props.width);\n\n    me.forceGraphUpdate = true;\n    //Is this local redraw necessary? (Core also does a change event!)\n    me.redraw.call(me);\n  });\n\n  // create the HTML DOM\n  this._create();\n  this.framework = {svg: this.svg, svgElements: this.svgElements, options: this.options, groups: this.groups};\n}\n\nLineGraph.prototype = new Component();\n\n/**\n * Create the HTML DOM for the ItemSet\n */\nLineGraph.prototype._create = function () {\n  var frame = document.createElement('div');\n  frame.className = 'vis-line-graph';\n  this.dom.frame = frame;\n\n  // create svg element for graph drawing.\n  this.svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg');\n  this.svg.style.position = 'relative';\n  this.svg.style.height = ('' + this.options.graphHeight).replace('px', '') + 'px';\n  this.svg.style.display = 'block';\n  frame.appendChild(this.svg);\n\n  // data axis\n  this.options.dataAxis.orientation = 'left';\n  this.yAxisLeft = new DataAxis(this.body, this.options.dataAxis, this.svg, this.options.groups);\n\n  this.options.dataAxis.orientation = 'right';\n  this.yAxisRight = new DataAxis(this.body, this.options.dataAxis, this.svg, this.options.groups);\n  delete this.options.dataAxis.orientation;\n\n  // legends\n  this.legendLeft = new Legend(this.body, this.options.legend, 'left', this.options.groups);\n  this.legendRight = new Legend(this.body, this.options.legend, 'right', this.options.groups);\n\n  this.show();\n};\n\n/**\n * set the options of the LineGraph. the mergeOptions is used for subObjects that have an enabled element.\n * @param {object} options\n */\nLineGraph.prototype.setOptions = function (options) {\n  if (options) {\n    var fields = ['sampling', 'defaultGroup', 'stack', 'height', 'graphHeight', 'yAxisOrientation', 'style', 'barChart', 'dataAxis', 'sort', 'groups'];\n    if (options.graphHeight === undefined && options.height !== undefined) {\n      this.updateSVGheight = true;\n      this.updateSVGheightOnResize = true;\n    }\n    else if (this.body.domProps.centerContainer.height !== undefined && options.graphHeight !== undefined) {\n      if (parseInt((options.graphHeight + '').replace(\"px\", '')) < this.body.domProps.centerContainer.height) {\n        this.updateSVGheight = true;\n      }\n    }\n    util.selectiveDeepExtend(fields, this.options, options);\n    util.mergeOptions(this.options, options, 'interpolation');\n    util.mergeOptions(this.options, options, 'drawPoints');\n    util.mergeOptions(this.options, options, 'shaded');\n    util.mergeOptions(this.options, options, 'legend');\n\n    if (options.interpolation) {\n      if (typeof options.interpolation == 'object') {\n        if (options.interpolation.parametrization) {\n          if (options.interpolation.parametrization == 'uniform') {\n            this.options.interpolation.alpha = 0;\n          }\n          else if (options.interpolation.parametrization == 'chordal') {\n            this.options.interpolation.alpha = 1.0;\n          }\n          else {\n            this.options.interpolation.parametrization = 'centripetal';\n            this.options.interpolation.alpha = 0.5;\n          }\n        }\n      }\n    }\n\n    if (this.yAxisLeft) {\n      if (options.dataAxis !== undefined) {\n        this.yAxisLeft.setOptions(this.options.dataAxis);\n        this.yAxisRight.setOptions(this.options.dataAxis);\n      }\n    }\n\n    if (this.legendLeft) {\n      if (options.legend !== undefined) {\n        this.legendLeft.setOptions(this.options.legend);\n        this.legendRight.setOptions(this.options.legend);\n      }\n    }\n\n    if (this.groups.hasOwnProperty(UNGROUPED)) {\n      this.groups[UNGROUPED].setOptions(options);\n    }\n  }\n\n  // this is used to redraw the graph if the visibility of the groups is changed.\n  if (this.dom.frame) { //not on initial run?\n    this.forceGraphUpdate=true;\n    this.body.emitter.emit(\"_change\",{queue: true});\n  }\n};\n\n/**\n * Hide the component from the DOM\n */\nLineGraph.prototype.hide = function () {\n  // remove the frame containing the items\n  if (this.dom.frame.parentNode) {\n    this.dom.frame.parentNode.removeChild(this.dom.frame);\n  }\n};\n\n\n/**\n * Show the component in the DOM (when not already visible).\n */\nLineGraph.prototype.show = function () {\n  // show frame containing the items\n  if (!this.dom.frame.parentNode) {\n    this.body.dom.center.appendChild(this.dom.frame);\n  }\n};\n\n\n/**\n * Set items\n * @param {vis.DataSet | null} items\n */\nLineGraph.prototype.setItems = function (items) {\n  var me = this,\n    ids,\n    oldItemsData = this.itemsData;\n\n  // replace the dataset\n  if (!items) {\n    this.itemsData = null;\n  }\n  else if (items instanceof DataSet || items instanceof DataView) {\n    this.itemsData = typeCoerceDataSet(items);\n  }\n  else {\n    throw new TypeError('Data must be an instance of DataSet or DataView');\n  }\n\n  if (oldItemsData) {\n    // unsubscribe from old dataset\n    util.forEach(this.itemListeners, function (callback, event) {\n      oldItemsData.off(event, callback);\n    });\n\n    // stop maintaining a coerced version of the old data set\n    oldItemsData.dispose()\n\n    // remove all drawn items\n    ids = oldItemsData.getIds();\n    this._onRemove(ids);\n  }\n\n  if (this.itemsData) {\n    // subscribe to new dataset\n    var id = this.id;\n    util.forEach(this.itemListeners, function (callback, event) {\n      me.itemsData.on(event, callback, id);\n    });\n\n    // add all new items\n    ids = this.itemsData.getIds();\n    this._onAdd(ids);\n  }\n};\n\n\n/**\n * Set groups\n * @param {vis.DataSet} groups\n */\nLineGraph.prototype.setGroups = function (groups) {\n  var me = this;\n  var ids;\n\n  // unsubscribe from current dataset\n  if (this.groupsData) {\n    util.forEach(this.groupListeners, function (callback, event) {\n      me.groupsData.off(event, callback);\n    });\n\n    // remove all drawn groups\n    ids = this.groupsData.getIds();\n    this.groupsData = null;\n    for (var i = 0; i < ids.length; i++) {\n      this._removeGroup(ids[i]);\n    }\n  }\n\n  // replace the dataset\n  if (!groups) {\n    this.groupsData = null;\n  }\n  else if (groups instanceof DataSet || groups instanceof DataView) {\n    this.groupsData = groups;\n  }\n  else {\n    throw new TypeError('Data must be an instance of DataSet or DataView');\n  }\n\n  if (this.groupsData) {\n    // subscribe to new dataset\n    var id = this.id;\n    util.forEach(this.groupListeners, function (callback, event) {\n      me.groupsData.on(event, callback, id);\n    });\n\n    // draw all ms\n    ids = this.groupsData.getIds();\n    this._onAddGroups(ids);\n  }\n};\n\nLineGraph.prototype._onUpdate = function (ids) {\n  this._updateAllGroupData(ids);\n};\nLineGraph.prototype._onAdd = function (ids) {\n  this._onUpdate(ids);\n};\nLineGraph.prototype._onRemove = function (ids) {\n  this._onUpdate(ids);\n};\nLineGraph.prototype._onUpdateGroups = function (groupIds) {\n  this._updateAllGroupData(null, groupIds);\n};\nLineGraph.prototype._onAddGroups = function (groupIds) {\n  this._onUpdateGroups(groupIds);\n};\n\n/**\n * this cleans the group out off the legends and the dataaxis, updates the ungrouped and updates the graph\n * @param {Array} groupIds\n * @private\n */\nLineGraph.prototype._onRemoveGroups = function (groupIds) {\n  for (var i = 0; i < groupIds.length; i++) {\n    this._removeGroup(groupIds[i]);\n  }\n  this.forceGraphUpdate = true;\n  this.body.emitter.emit(\"_change\",{queue: true});\n};\n\n/**\n * this cleans the group out off the legends and the dataaxis\n * @param {vis.GraphGroup.id} groupId\n * @private\n */\nLineGraph.prototype._removeGroup = function (groupId) {\n  if (this.groups.hasOwnProperty(groupId)) {\n    if (this.groups[groupId].options.yAxisOrientation == 'right') {\n      this.yAxisRight.removeGroup(groupId);\n      this.legendRight.removeGroup(groupId);\n      this.legendRight.redraw();\n    }\n    else {\n      this.yAxisLeft.removeGroup(groupId);\n      this.legendLeft.removeGroup(groupId);\n      this.legendLeft.redraw();\n    }\n    delete this.groups[groupId];\n  }\n};\n\n/**\n * update a group object with the group dataset entree\n *\n * @param {vis.GraphGroup} group\n * @param {vis.GraphGroup.id} groupId\n * @private\n */\nLineGraph.prototype._updateGroup = function (group, groupId) {\n  if (!this.groups.hasOwnProperty(groupId)) {\n    this.groups[groupId] = new GraphGroup(group, groupId, this.options, this.groupsUsingDefaultStyles);\n    if (this.groups[groupId].options.yAxisOrientation == 'right') {\n      this.yAxisRight.addGroup(groupId, this.groups[groupId]);\n      this.legendRight.addGroup(groupId, this.groups[groupId]);\n    }\n    else {\n      this.yAxisLeft.addGroup(groupId, this.groups[groupId]);\n      this.legendLeft.addGroup(groupId, this.groups[groupId]);\n    }\n  }\n  else {\n    this.groups[groupId].update(group);\n    if (this.groups[groupId].options.yAxisOrientation == 'right') {\n      this.yAxisRight.updateGroup(groupId, this.groups[groupId]);\n      this.legendRight.updateGroup(groupId, this.groups[groupId]);\n      //If yAxisOrientation changed, clean out the group from the other axis.\n      this.yAxisLeft.removeGroup(groupId);\n      this.legendLeft.removeGroup(groupId);\n    }\n    else {\n      this.yAxisLeft.updateGroup(groupId, this.groups[groupId]);\n      this.legendLeft.updateGroup(groupId, this.groups[groupId]);\n      //If yAxisOrientation changed, clean out the group from the other axis.\n      this.yAxisRight.removeGroup(groupId);\n      this.legendRight.removeGroup(groupId);\n    }\n  }\n  this.legendLeft.redraw();\n  this.legendRight.redraw();\n};\n\n\n/**\n * this updates all groups, it is used when there is an update the the itemset.\n *\n * @param  {Array} ids\n * @param  {Array} groupIds\n * @private\n */\nLineGraph.prototype._updateAllGroupData = function (ids, groupIds) {\n  if (this.itemsData != null) {\n    var groupsContent = {};\n    var items = this.itemsData.get();\n    var fieldId = this.itemsData.idProp;\n    var idMap = {};\n    if (ids){\n      ids.map(function (id) {\n        idMap[id] = id;\n      });\n    }\n\n    //pre-Determine array sizes, for more efficient memory claim\n    var groupCounts = {};\n    for (var i = 0; i < items.length; i++) {\n      var item = items[i];\n      var groupId = item.group;\n      if (groupId === null || groupId === undefined) {\n        groupId = UNGROUPED;\n      }\n      groupCounts.hasOwnProperty(groupId) ? groupCounts[groupId]++ : groupCounts[groupId] = 1;\n    }\n\n    //Pre-load arrays from existing groups if items are not changed (not in ids)\n    var existingItemsMap = {};\n    if (!groupIds && ids) {\n      for (groupId in this.groups) {\n        if (this.groups.hasOwnProperty(groupId)) {\n          group = this.groups[groupId];\n          var existing_items = group.getItems();\n\n          groupsContent[groupId] = existing_items.filter(function (item) {\n            existingItemsMap[item[fieldId]] = item[fieldId];\n            return (item[fieldId] !== idMap[item[fieldId]]);\n          });\n          var newLength = groupCounts[groupId];\n          groupCounts[groupId] -= groupsContent[groupId].length;\n          if (groupsContent[groupId].length < newLength) {\n            groupsContent[groupId][newLength - 1] = {};\n          }\n        }\n      }\n    }\n\n    //Now insert data into the arrays.\n    for (i = 0; i < items.length; i++) {\n      item = items[i];\n      groupId = item.group;\n      if (groupId === null || groupId === undefined) {\n        groupId = UNGROUPED;\n      }\n      if (!groupIds && ids && (item[fieldId] !== idMap[item[fieldId]]) && existingItemsMap.hasOwnProperty(item[fieldId])) {\n        continue;\n      }\n      if (!groupsContent.hasOwnProperty(groupId)) {\n        groupsContent[groupId] = new Array(groupCounts[groupId]);\n      }\n      //Copy data (because of unmodifiable DataView input.\n      var extended = util.bridgeObject(item);\n      extended.x = util.convert(item.x, 'Date');\n      extended.end = util.convert(item.end, 'Date');\n      extended.orginalY = item.y; //real Y\n      extended.y = Number(item.y);\n      extended[fieldId] = item[fieldId];\n\n      var index= groupsContent[groupId].length - groupCounts[groupId]--;\n      groupsContent[groupId][index] = extended;\n    }\n\n    //Make sure all groups are present, to allow removal of old groups\n    for (groupId in this.groups){\n      if (this.groups.hasOwnProperty(groupId)){\n        if (!groupsContent.hasOwnProperty(groupId)) {\n          groupsContent[groupId] = new Array(0);\n        }\n      }\n    }\n\n    //Update legendas, style and axis\n    for (groupId in groupsContent) {\n      if (groupsContent.hasOwnProperty(groupId)) {\n        if (groupsContent[groupId].length == 0) {\n          if (this.groups.hasOwnProperty(groupId)) {\n            this._removeGroup(groupId);\n          }\n        } else {\n          var group = undefined;\n          if (this.groupsData != undefined) {\n            group = this.groupsData.get(groupId);\n          }\n          if (group == undefined) {\n            group = {id: groupId, content: this.options.defaultGroup + groupId};\n          }\n          this._updateGroup(group, groupId);\n          this.groups[groupId].setItems(groupsContent[groupId]);\n        }\n      }\n    }\n    this.forceGraphUpdate = true;\n    this.body.emitter.emit(\"_change\",{queue: true});\n  }\n};\n\n/**\n * Redraw the component, mandatory function\n * @return {boolean} Returns true if the component is resized\n */\nLineGraph.prototype.redraw = function () {\n  var resized = false;\n\n  // calculate actual size and position\n  this.props.width = this.dom.frame.offsetWidth;\n  this.props.height = this.body.domProps.centerContainer.height\n    - this.body.domProps.border.top\n    - this.body.domProps.border.bottom;\n\n  // check if this component is resized\n  resized = this._isResized() || resized;\n\n  // check whether zoomed (in that case we need to re-stack everything)\n  var visibleInterval = this.body.range.end - this.body.range.start;\n  var zoomed = (visibleInterval != this.lastVisibleInterval);\n  this.lastVisibleInterval = visibleInterval;\n\n\n  // the svg element is three times as big as the width, this allows for fully dragging left and right\n  // without reloading the graph. the controls for this are bound to events in the constructor\n  if (resized == true) {\n    this.svg.style.width = util.option.asSize(3 * this.props.width);\n    this.svg.style.left = util.option.asSize(-this.props.width);\n\n    // if the height of the graph is set as proportional, change the height of the svg\n    if ((this.options.height + '').indexOf(\"%\") != -1 || this.updateSVGheightOnResize == true) {\n      this.updateSVGheight = true;\n    }\n  }\n\n  // update the height of the graph on each redraw of the graph.\n  if (this.updateSVGheight == true) {\n    if (this.options.graphHeight != this.props.height + 'px') {\n      this.options.graphHeight = this.props.height + 'px';\n      this.svg.style.height = this.props.height + 'px';\n    }\n    this.updateSVGheight = false;\n  }\n  else {\n    this.svg.style.height = ('' + this.options.graphHeight).replace('px', '') + 'px';\n  }\n\n  // zoomed is here to ensure that animations are shown correctly.\n  if (resized == true || zoomed == true || this.abortedGraphUpdate == true || this.forceGraphUpdate == true) {\n    resized = this._updateGraph() || resized;\n    this.forceGraphUpdate = false;\n  }\n  else {\n    // move the whole svg while dragging\n    if (this.lastStart != 0) {\n      var offset = this.body.range.start - this.lastStart;\n      var range = this.body.range.end - this.body.range.start;\n      if (this.props.width != 0) {\n        var rangePerPixelInv = this.props.width / range;\n        var xOffset = offset * rangePerPixelInv;\n        this.svg.style.left = (-this.props.width - xOffset) + 'px';\n      }\n    }\n  }\n  this.legendLeft.redraw();\n  this.legendRight.redraw();\n  return resized;\n};\n\n\nLineGraph.prototype._getSortedGroupIds = function(){\n  // getting group Ids\n  var grouplist = [];\n  for (var groupId in this.groups) {\n    if (this.groups.hasOwnProperty(groupId)) {\n      var group = this.groups[groupId];\n      if (group.visible == true && (this.options.groups.visibility[groupId] === undefined || this.options.groups.visibility[groupId] == true)) {\n        grouplist.push({id:groupId,zIndex:group.options.zIndex});\n      }\n    }\n  }\n  util.insertSort(grouplist,function(a,b){\n    var az = a.zIndex;\n    var bz = b.zIndex;\n    if (az === undefined) az=0;\n    if (bz === undefined) bz=0;\n    return az==bz? 0: (az<bz ? -1: 1);\n  });\n  var groupIds = new Array(grouplist.length);\n  for (var i=0; i< grouplist.length; i++){\n    groupIds[i] = grouplist[i].id;\n  }\n  return groupIds;\n};\n\n/**\n * Update and redraw the graph.\n *\n * @returns {boolean}\n * @private\n */\nLineGraph.prototype._updateGraph = function () {\n  // reset the svg elements\n  DOMutil.prepareElements(this.svgElements);\n  if (this.props.width != 0 && this.itemsData != null) {\n    var group, i;\n    var groupRanges = {};\n    var changeCalled = false;\n    // this is the range of the SVG canvas\n    var minDate = this.body.util.toGlobalTime(-this.body.domProps.root.width);\n    var maxDate = this.body.util.toGlobalTime(2 * this.body.domProps.root.width);\n\n    // getting group Ids\n    var groupIds = this._getSortedGroupIds();\n    if (groupIds.length > 0) {\n      var groupsData = {};\n\n      // fill groups data, this only loads the data we require based on the timewindow\n      this._getRelevantData(groupIds, groupsData, minDate, maxDate);\n\n      // apply sampling, if disabled, it will pass through this function.\n      this._applySampling(groupIds, groupsData);\n\n      // we transform the X coordinates to detect collisions\n      for (i = 0; i < groupIds.length; i++) {\n        this._convertXcoordinates(groupsData[groupIds[i]]);\n      }\n\n      // now all needed data has been collected we start the processing.\n      this._getYRanges(groupIds, groupsData, groupRanges);\n\n      // update the Y axis first, we use this data to draw at the correct Y points\n      changeCalled = this._updateYAxis(groupIds, groupRanges);\n\n      //  at changeCalled, abort this update cycle as the graph needs another update with new Width input from the Redraw container.\n      //  Cleanup SVG elements on abort.\n      if (changeCalled == true) {\n        DOMutil.cleanupElements(this.svgElements);\n        this.abortedGraphUpdate = true;\n        return true;\n      }\n      this.abortedGraphUpdate = false;\n\n      // With the yAxis scaled correctly, use this to get the Y values of the points.\n      var below = undefined;\n      for (i = 0; i < groupIds.length; i++) {\n        group = this.groups[groupIds[i]];\n        if (this.options.stack === true && this.options.style === 'line') {\n          if (group.options.excludeFromStacking == undefined || !group.options.excludeFromStacking) {\n            if (below != undefined) {\n              this._stack(groupsData[group.id], groupsData[below.id]);\n              if (group.options.shaded.enabled == true && group.options.shaded.orientation !== \"group\"){\n                if (group.options.shaded.orientation == \"top\" && below.options.shaded.orientation !== \"group\"){\n                  below.options.shaded.orientation=\"group\";\n                  below.options.shaded.groupId=group.id;\n                } else {\n                  group.options.shaded.orientation=\"group\";\n                  group.options.shaded.groupId=below.id;\n                }\n              }\n            }\n            below = group;\n          }\n        }\n        this._convertYcoordinates(groupsData[groupIds[i]], group);\n      }\n\n      //Precalculate paths and draw shading if appropriate. This will make sure the shading is always behind any lines.\n      var paths = {};\n      for (i = 0; i < groupIds.length; i++) {\n        group = this.groups[groupIds[i]];\n        if (group.options.style === 'line' && group.options.shaded.enabled == true) {\n          var dataset = groupsData[groupIds[i]];\n          if (dataset == null || dataset.length == 0) {\n            continue;\n          }\n          if (!paths.hasOwnProperty(groupIds[i])) {\n            paths[groupIds[i]] = Lines.calcPath(dataset, group);\n          }\n          if (group.options.shaded.orientation === \"group\") {\n            var subGroupId = group.options.shaded.groupId;\n            if (groupIds.indexOf(subGroupId) === -1) {\n              console.log(group.id + \": Unknown shading group target given:\" + subGroupId);\n              continue;\n            }\n            if (!paths.hasOwnProperty(subGroupId)) {\n              paths[subGroupId] = Lines.calcPath(groupsData[subGroupId], this.groups[subGroupId]);\n            }\n            Lines.drawShading(paths[groupIds[i]], group, paths[subGroupId], this.framework);\n          }\n          else {\n            Lines.drawShading(paths[groupIds[i]], group, undefined, this.framework);\n          }\n        }\n      }\n\n      // draw the groups, calculating paths if still necessary.\n      Bars.draw(groupIds, groupsData, this.framework);\n      for (i = 0; i < groupIds.length; i++) {\n        group = this.groups[groupIds[i]];\n        if (groupsData[groupIds[i]].length > 0) {\n          switch (group.options.style) {\n            case \"line\":\n              if (!paths.hasOwnProperty(groupIds[i])) {\n                paths[groupIds[i]] = Lines.calcPath(groupsData[groupIds[i]], group);\n              }\n              Lines.draw(paths[groupIds[i]], group, this.framework);\n            // eslint-disable-line no-fallthrough\n            case \"point\":\n            // eslint-disable-line no-fallthrough\n            case \"points\":\n              if (group.options.style == \"point\" || group.options.style == \"points\" || group.options.drawPoints.enabled == true) {\n                Points.draw(groupsData[groupIds[i]], group, this.framework);\n              }\n              break;\n            case \"bar\":\n            // bar needs to be drawn enmasse\n            // eslint-disable-line no-fallthrough\n            default:\n            //do nothing...\n          }\n        }\n\n      }\n    }\n  }\n\n  // cleanup unused svg elements\n  DOMutil.cleanupElements(this.svgElements);\n  return false;\n};\n\nLineGraph.prototype._stack = function (data, subData) {\n  var index, dx, dy, subPrevPoint, subNextPoint;\n  index = 0;\n  // for each data point we look for a matching on in the set below\n  for (var j = 0; j < data.length; j++) {\n    subPrevPoint = undefined;\n    subNextPoint = undefined;\n    // we look for time matches or a before-after point\n    for (var k = index; k < subData.length; k++) {\n      // if times match exactly\n      if (subData[k].x === data[j].x) {\n        subPrevPoint = subData[k];\n        subNextPoint = subData[k];\n        index = k;\n        break;\n      }\n      else if (subData[k].x > data[j].x) { // overshoot\n        subNextPoint = subData[k];\n        if (k == 0) {\n          subPrevPoint = subNextPoint;\n        }\n        else {\n          subPrevPoint = subData[k - 1];\n        }\n        index = k;\n        break;\n      }\n    }\n    // in case the last data point has been used, we assume it stays like this.\n    if (subNextPoint === undefined) {\n      subPrevPoint = subData[subData.length - 1];\n      subNextPoint = subData[subData.length - 1];\n    }\n    // linear interpolation\n    dx = subNextPoint.x - subPrevPoint.x;\n    dy = subNextPoint.y - subPrevPoint.y;\n    if (dx == 0) {\n      data[j].y = data[j].orginalY + subNextPoint.y;\n    }\n    else {\n      data[j].y = data[j].orginalY + (dy / dx) * (data[j].x - subPrevPoint.x) + subPrevPoint.y; // ax + b where b is data[j].y\n    }\n  }\n}\n\n\n/**\n * first select and preprocess the data from the datasets.\n * the groups have their preselection of data, we now loop over this data to see\n * what data we need to draw. Sorted data is much faster.\n * more optimization is possible by doing the sampling before and using the binary search\n * to find the end date to determine the increment.\n *\n * @param {array}  groupIds\n * @param {object} groupsData\n * @param {date}   minDate\n * @param {date}   maxDate\n * @private\n */\nLineGraph.prototype._getRelevantData = function (groupIds, groupsData, minDate, maxDate) {\n  var group, i, j, item;\n  if (groupIds.length > 0) {\n    for (i = 0; i < groupIds.length; i++) {\n      group = this.groups[groupIds[i]];\n      var itemsData = group.getItems();\n      // optimization for sorted data\n      if (group.options.sort == true) {\n        var dateComparator = function (a, b) {\n          return a.getTime() == b.getTime() ? 0 : a < b ? -1 : 1\n        };\n        var first = Math.max(0, util.binarySearchValue(itemsData, minDate, 'x', 'before', dateComparator));\n        var last = Math.min(itemsData.length, util.binarySearchValue(itemsData, maxDate, 'x', 'after', dateComparator) + 1);\n        if (last <= 0) {\n          last = itemsData.length;\n        }\n        var dataContainer = new Array(last-first);\n        for (j = first; j < last; j++) {\n          item = group.itemsData[j];\n          dataContainer[j-first] = item;\n        }\n        groupsData[groupIds[i]] = dataContainer;\n      }\n      else {\n        // If unsorted data, all data is relevant, just returning entire structure\n        groupsData[groupIds[i]] = group.itemsData;\n      }\n    }\n  }\n};\n\n\n/**\n *\n * @param {Array.<vis.GraphGroup.id>} groupIds\n * @param {vis.DataSet} groupsData\n * @private\n */\nLineGraph.prototype._applySampling = function (groupIds, groupsData) {\n  var group;\n  if (groupIds.length > 0) {\n    for (var i = 0; i < groupIds.length; i++) {\n      group = this.groups[groupIds[i]];\n      if (group.options.sampling == true) {\n        var dataContainer = groupsData[groupIds[i]];\n        if (dataContainer.length > 0) {\n          var increment = 1;\n          var amountOfPoints = dataContainer.length;\n\n          // the global screen is used because changing the width of the yAxis may affect the increment, resulting in an endless loop\n          // of width changing of the yAxis.\n          //TODO: This assumes sorted data, but that's not guaranteed!\n          var xDistance = this.body.util.toGlobalScreen(dataContainer[dataContainer.length - 1].x) - this.body.util.toGlobalScreen(dataContainer[0].x);\n          var pointsPerPixel = amountOfPoints / xDistance;\n          increment = Math.min(Math.ceil(0.2 * amountOfPoints), Math.max(1, Math.round(pointsPerPixel)));\n\n          var sampledData = new Array(amountOfPoints);\n          for (var j = 0; j < amountOfPoints; j += increment) {\n            var idx = Math.round(j/increment);\n            sampledData[idx]=dataContainer[j];\n          }\n          groupsData[groupIds[i]] = sampledData.splice(0,Math.round(amountOfPoints/increment));\n        }\n      }\n    }\n  }\n};\n\n\n/**\n *\n * @param {Array.<vis.GraphGroup.id>} groupIds\n * @param {vis.DataSet} groupsData\n * @param {object} groupRanges  | this is being filled here\n * @private\n */\nLineGraph.prototype._getYRanges = function (groupIds, groupsData, groupRanges) {\n  var groupData, group, i;\n  var combinedDataLeft = [];\n  var combinedDataRight = [];\n  var options;\n  if (groupIds.length > 0) {\n    for (i = 0; i < groupIds.length; i++) {\n      groupData = groupsData[groupIds[i]];\n      options = this.groups[groupIds[i]].options;\n      if (groupData.length > 0) {\n        group = this.groups[groupIds[i]];\n        // if bar graphs are stacked, their range need to be handled differently and accumulated over all groups.\n        if (options.stack === true && options.style === 'bar') {\n          if (options.yAxisOrientation === 'left') {\n            combinedDataLeft = combinedDataLeft.concat(groupData);\n          }\n          else {\n            combinedDataRight = combinedDataRight.concat(groupData);\n          }\n        }\n        else {\n          groupRanges[groupIds[i]] = group.getYRange(groupData, groupIds[i]);\n        }\n      }\n    }\n\n    // if bar graphs are stacked, their range need to be handled differently and accumulated over all groups.\n    Bars.getStackedYRange(combinedDataLeft, groupRanges, groupIds, '__barStackLeft', 'left');\n    Bars.getStackedYRange(combinedDataRight, groupRanges, groupIds, '__barStackRight', 'right');\n  }\n};\n\n\n/**\n * this sets the Y ranges for the Y axis. It also determines which of the axis should be shown or hidden.\n * @param {Array.<vis.GraphGroup.id>} groupIds\n * @param {Object} groupRanges\n * @returns {boolean} resized\n * @private\n */\nLineGraph.prototype._updateYAxis = function (groupIds, groupRanges) {\n  var resized = false;\n  var yAxisLeftUsed = false;\n  var yAxisRightUsed = false;\n  var minLeft = 1e9, minRight = 1e9, maxLeft = -1e9, maxRight = -1e9, minVal, maxVal;\n  // if groups are present\n  if (groupIds.length > 0) {\n    // this is here to make sure that if there are no items in the axis but there are groups, that there is no infinite draw/redraw loop.\n    for (var i = 0; i < groupIds.length; i++) {\n      var group = this.groups[groupIds[i]];\n      if (group && group.options.yAxisOrientation != 'right') {\n        yAxisLeftUsed = true;\n        minLeft = 1e9;\n        maxLeft = -1e9;\n      }\n      else if (group && group.options.yAxisOrientation) {\n        yAxisRightUsed = true;\n        minRight = 1e9;\n        maxRight = -1e9;\n      }\n    }\n\n    // if there are items:\n    for (i = 0; i < groupIds.length; i++) {\n      if (groupRanges.hasOwnProperty(groupIds[i])) {\n        if (groupRanges[groupIds[i]].ignore !== true) {\n          minVal = groupRanges[groupIds[i]].min;\n          maxVal = groupRanges[groupIds[i]].max;\n\n          if (groupRanges[groupIds[i]].yAxisOrientation != 'right') {\n            yAxisLeftUsed = true;\n            minLeft = minLeft > minVal ? minVal : minLeft;\n            maxLeft = maxLeft < maxVal ? maxVal : maxLeft;\n          }\n          else {\n            yAxisRightUsed = true;\n            minRight = minRight > minVal ? minVal : minRight;\n            maxRight = maxRight < maxVal ? maxVal : maxRight;\n          }\n        }\n      }\n    }\n\n    if (yAxisLeftUsed == true) {\n      this.yAxisLeft.setRange(minLeft, maxLeft);\n    }\n    if (yAxisRightUsed == true) {\n      this.yAxisRight.setRange(minRight, maxRight);\n    }\n  }\n  resized = this._toggleAxisVisiblity(yAxisLeftUsed, this.yAxisLeft) || resized;\n  resized = this._toggleAxisVisiblity(yAxisRightUsed, this.yAxisRight) || resized;\n\n  if (yAxisRightUsed == true && yAxisLeftUsed == true) {\n    this.yAxisLeft.drawIcons = true;\n    this.yAxisRight.drawIcons = true;\n  }\n  else {\n    this.yAxisLeft.drawIcons = false;\n    this.yAxisRight.drawIcons = false;\n  }\n  this.yAxisRight.master = !yAxisLeftUsed;\n  this.yAxisRight.masterAxis = this.yAxisLeft;\n\n  if (this.yAxisRight.master == false) {\n    if (yAxisRightUsed == true) {\n      this.yAxisLeft.lineOffset = this.yAxisRight.width;\n    }\n    else {\n      this.yAxisLeft.lineOffset = 0;\n    }\n\n    resized = this.yAxisLeft.redraw() || resized;\n    resized = this.yAxisRight.redraw() || resized;\n  }\n  else {\n    resized = this.yAxisRight.redraw() || resized;\n  }\n\n  // clean the accumulated lists\n  var tempGroups = ['__barStackLeft', '__barStackRight', '__lineStackLeft', '__lineStackRight'];\n  for (i = 0; i < tempGroups.length; i++) {\n    if (groupIds.indexOf(tempGroups[i]) != -1) {\n      groupIds.splice(groupIds.indexOf(tempGroups[i]), 1);\n    }\n  }\n\n  return resized;\n};\n\n\n/**\n * This shows or hides the Y axis if needed. If there is a change, the changed event is emitted by the updateYAxis function\n *\n * @param {boolean} axisUsed\n * @param {vis.DataAxis}  axis\n * @returns {boolean}\n * @private\n */\nLineGraph.prototype._toggleAxisVisiblity = function (axisUsed, axis) {\n  var changed = false;\n  if (axisUsed == false) {\n    if (axis.dom.frame.parentNode && axis.hidden == false) {\n      axis.hide();\n      changed = true;\n    }\n  }\n  else {\n    if (!axis.dom.frame.parentNode && axis.hidden == true) {\n      axis.show();\n      changed = true;\n    }\n  }\n  return changed;\n};\n\n\n/**\n * This uses the DataAxis object to generate the correct X coordinate on the SVG window. It uses the\n * util function toScreen to get the x coordinate from the timestamp. It also pre-filters the data and get the minMax ranges for\n * the yAxis.\n *\n * @param {Array.<Object>} datapoints\n * @private\n */\nLineGraph.prototype._convertXcoordinates = function (datapoints) {\n  var toScreen = this.body.util.toScreen;\n  for (var i = 0; i < datapoints.length; i++) {\n    datapoints[i].screen_x = toScreen(datapoints[i].x) + this.props.width;\n    datapoints[i].screen_y = datapoints[i].y; //starting point for range calculations\n    if (datapoints[i].end != undefined) {\n      datapoints[i].screen_end = toScreen(datapoints[i].end) + this.props.width;\n    }\n    else {\n      datapoints[i].screen_end = undefined;\n    }\n  }\n};\n\n\n/**\n * This uses the DataAxis object to generate the correct X coordinate on the SVG window. It uses the\n * util function toScreen to get the x coordinate from the timestamp. It also pre-filters the data and get the minMax ranges for\n * the yAxis.\n *\n * @param {Array.<Object>} datapoints\n * @param {vis.GraphGroup} group\n * @private\n */\nLineGraph.prototype._convertYcoordinates = function (datapoints, group) {\n  var axis = this.yAxisLeft;\n  var svgHeight = Number(this.svg.style.height.replace('px', ''));\n  if (group.options.yAxisOrientation == 'right') {\n    axis = this.yAxisRight;\n  }\n  for (var i = 0; i < datapoints.length; i++) {\n    datapoints[i].screen_y = Math.round(axis.convertValue(datapoints[i].y));\n  }\n  group.setZeroPosition(Math.min(svgHeight, axis.convertValue(0)));\n};\n\n\nexport default LineGraph;\n","/**\n * This object contains all possible options. It will check if the types are correct, if required if the option is one\n * of the allowed values.\n *\n * __any__ means that the name of the property does not matter.\n * __type__ is a required field for all objects and contains the allowed types of all objects\n */\nlet string = 'string';\nlet bool = 'boolean';\nlet number = 'number';\nlet array = 'array';\nlet date = 'date';\nlet object = 'object'; // should only be in a __type__ property\nlet dom = 'dom';\nlet moment = 'moment';\nlet any = 'any';\n\n\nlet allOptions = {\n  configure: {\n    enabled: {'boolean': bool},\n    filter: {'boolean': bool,'function': 'function'},\n    container: {dom},\n    __type__: {object,'boolean': bool,'function': 'function'}\n  },\n\n  //globals :\n  alignCurrentTime: {string, 'undefined': 'undefined'},  \n  yAxisOrientation: {string:['left','right']},\n  defaultGroup: {string},\n  sort: {'boolean': bool},\n  sampling: {'boolean': bool},\n  stack:{'boolean': bool},\n  graphHeight: {string, number},\n  shaded: {\n    enabled: {'boolean': bool},\n    orientation: {string:['bottom','top','zero','group']}, // top, bottom, zero, group\n    groupId: {object},\n    __type__: {'boolean': bool,object}\n  },\n  style: {string:['line','bar','points']}, // line, bar\n  barChart: {\n    width: {number},\n    minWidth: {number},\n    sideBySide: {'boolean': bool},\n    align: {string:['left','center','right']},\n    __type__: {object}\n  },\n  interpolation: {\n    enabled: {'boolean': bool},\n    parametrization: {string:['centripetal', 'chordal','uniform']}, // uniform (alpha = 0.0), chordal (alpha = 1.0), centripetal (alpha = 0.5)\n    alpha: {number},\n    __type__: {object,'boolean': bool}\n  },\n  drawPoints: {\n    enabled: {'boolean': bool},\n    onRender: { 'function': 'function' },\n    size: {number},\n    style: {string:['square','circle']}, // square, circle\n    __type__: {object,'boolean': bool,'function': 'function'}\n  },\n  dataAxis: {\n    showMinorLabels: {'boolean': bool},\n    showMajorLabels: {'boolean': bool},\n    showWeekScale: {'boolean': bool},\n    icons: {'boolean': bool},\n    width: {string, number},\n    visible: {'boolean': bool},\n    alignZeros: {'boolean': bool},\n    left:{\n      range: {min:{number,'undefined': 'undefined'},max:{number,'undefined': 'undefined'},__type__: {object}},\n      format: {'function': 'function'},\n      title: {text:{string,number,'undefined': 'undefined'},style:{string,'undefined': 'undefined'},__type__: {object}},\n      __type__: {object}\n    },\n    right:{\n      range: {min:{number,'undefined': 'undefined'},max:{number,'undefined': 'undefined'},__type__: {object}},\n      format: {'function': 'function'},\n      title: {text:{string,number,'undefined': 'undefined'},style:{string,'undefined': 'undefined'},__type__: {object}},\n      __type__: {object}\n    },\n    __type__: {object}\n  },\n  legend: {\n    enabled: {'boolean': bool},\n    icons: {'boolean': bool},\n    left: {\n      visible: {'boolean': bool},\n      position: {string:['top-right','bottom-right','top-left','bottom-left']},\n      __type__: {object}\n    },\n    right: {\n      visible: {'boolean': bool},\n      position: {string:['top-right','bottom-right','top-left','bottom-left']},\n      __type__: {object}\n    },\n    __type__: {object,'boolean': bool}\n  },\n  groups: {\n    visibility: {any},\n    __type__: {object}\n  },\n\n  autoResize: {'boolean': bool},\n  throttleRedraw: {number}, // TODO: DEPRICATED see https://github.com/almende/vis/issues/2511\n  clickToUse: {'boolean': bool},\n  end: {number, date, string, moment},\n  format: {\n    minorLabels: {\n      millisecond: {string,'undefined': 'undefined'},\n      second: {string,'undefined': 'undefined'},\n      minute: {string,'undefined': 'undefined'},\n      hour: {string,'undefined': 'undefined'},\n      weekday: {string,'undefined': 'undefined'},\n      day: {string,'undefined': 'undefined'},\n      week: {string,'undefined': 'undefined'},\n      month: {string,'undefined': 'undefined'},\n      quarter: {string,'undefined': 'undefined'},\n      year: {string,'undefined': 'undefined'},\n      __type__: {object}\n    },\n    majorLabels: {\n      millisecond: {string,'undefined': 'undefined'},\n      second: {string,'undefined': 'undefined'},\n      minute: {string,'undefined': 'undefined'},\n      hour: {string,'undefined': 'undefined'},\n      weekday: {string,'undefined': 'undefined'},\n      day: {string,'undefined': 'undefined'},\n      week: {string,'undefined': 'undefined'},\n      month: {string,'undefined': 'undefined'},\n      quarter: {string,'undefined': 'undefined'},\n      year: {string,'undefined': 'undefined'},\n      __type__: {object}\n    },\n    __type__: {object}\n  },\n  moment: {'function': 'function'},\n  height: {string, number},\n  hiddenDates: {\n    start: {date, number, string, moment},\n    end: {date, number, string, moment},\n    repeat: {string},\n    __type__: {object, array}\n  },\n  locale:{string},\n  locales:{\n    __any__: {any},\n    __type__: {object}\n  },\n  max: {date, number, string, moment},\n  maxHeight: {number, string},\n  maxMinorChars: {number},\n  min: {date, number, string, moment},\n  minHeight: {number, string},\n  moveable: {'boolean': bool},\n  multiselect: {'boolean': bool},\n  orientation: {string},\n  showCurrentTime: {'boolean': bool},\n  showMajorLabels: {'boolean': bool},\n  showMinorLabels: {'boolean': bool},\n  showWeekScale: {'boolean': bool},\n  start: {date, number, string, moment},\n  timeAxis: {\n    scale: {string,'undefined': 'undefined'},\n    step: {number,'undefined': 'undefined'},\n    __type__: {object}\n  },\n  width: {string, number},\n  zoomable: {'boolean': bool},\n  zoomKey: {string: ['ctrlKey', 'altKey', 'metaKey', '']},\n  zoomMax: {number},\n  zoomMin: {number},\n  zIndex: {number},\n  __type__: {object}\n};\n\nlet configureOptions = {\n  global: {\n    alignCurrentTime: ['none', 'year', 'month', 'quarter', 'week', 'isoWeek', 'day', 'date', 'hour', 'minute', 'second'],   \n    //yAxisOrientation: ['left','right'], // TDOO: enable as soon as Grahp2d doesn't crash when changing this on the fly\n    sort: true,\n    sampling: true,\n    stack:false,\n    shaded: {\n      enabled: false,\n      orientation: ['zero','top','bottom','group'] // zero, top, bottom\n    },\n    style: ['line','bar','points'], // line, bar\n    barChart: {\n      width: [50,5,100,5],\n      minWidth: [50,5,100,5],\n      sideBySide: false,\n      align: ['left','center','right'] // left, center, right\n    },\n    interpolation: {\n      enabled: true,\n      parametrization: ['centripetal','chordal','uniform'] // uniform (alpha = 0.0), chordal (alpha = 1.0), centripetal (alpha = 0.5)\n    },\n    drawPoints: {\n      enabled: true,\n      size: [6,2,30,1],\n      style: ['square', 'circle'] // square, circle\n    },\n    dataAxis: {\n      showMinorLabels: true,\n      showMajorLabels: true,\n      showWeekScale: false,\n      icons: false,\n      width: [40,0,200,1],\n      visible: true,\n      alignZeros: true,\n      left:{\n        //range: {min:'undefined': 'undefined'ined,max:'undefined': 'undefined'ined},\n        //format: function (value) {return value;},\n        title: {text:'',style:''}\n      },\n      right:{\n        //range: {min:'undefined': 'undefined'ined,max:'undefined': 'undefined'ined},\n        //format: function (value) {return value;},\n        title: {text:'',style:''}\n      }\n    },\n    legend: {\n      enabled: false,\n      icons: true,\n      left: {\n        visible: true,\n        position: ['top-right','bottom-right','top-left','bottom-left'] // top/bottom - left,right\n      },\n      right: {\n        visible: true,\n        position: ['top-right','bottom-right','top-left','bottom-left'] // top/bottom - left,right\n      }\n    },\n\n    autoResize: true,\n    clickToUse: false,\n    end: '',\n    format: {\n      minorLabels: {\n        millisecond:'SSS',\n        second:     's',\n        minute:     'HH:mm',\n        hour:       'HH:mm',\n        weekday:    'ddd D',\n        day:        'D',\n        week:       'w',\n        month:      'MMM',\n        quarter:    '[Q]Q',\n        year:       'YYYY'\n      },\n      majorLabels: {\n        millisecond:'HH:mm:ss',\n        second:     'D MMMM HH:mm',\n        minute:     'ddd D MMMM',\n        hour:       'ddd D MMMM',\n        weekday:    'MMMM YYYY',\n        day:        'MMMM YYYY',\n        week:       'MMMM YYYY',\n        month:      'YYYY',\n        quarter:    'YYYY',\n        year:       ''\n      }\n    },\n\n    height: '',\n    locale: '',\n    max: '',\n    maxHeight: '',\n    maxMinorChars: [7, 0, 20, 1],\n    min: '',\n    minHeight: '',\n    moveable:true,\n    orientation: ['both', 'bottom', 'top'],\n    showCurrentTime: false,\n    showMajorLabels: true,\n    showMinorLabels: true,\n    showWeekScale: false,\n    start: '',\n    width: '100%',\n    zoomable: true,\n    zoomKey: ['ctrlKey', 'altKey', 'metaKey', ''],\n    zoomMax: [315360000000000, 10, 315360000000000, 1],\n    zoomMin: [10, 10, 315360000000000, 1],\n    zIndex: 0\n  }\n};\n\nexport {allOptions, configureOptions};\n","import moment from '../module/moment';\nimport util, { typeCoerceDataSet } from '../util';\nimport { DataSet }  from 'vis-data/esnext';\nimport { DataView } from 'vis-data/esnext';\nimport Range from './Range';\nimport Core from './Core';\nimport TimeAxis from './component/TimeAxis';\nimport CurrentTime from './component/CurrentTime';\nimport CustomTime from './component/CustomTime';\nimport LineGraph from './component/LineGraph';\n\nimport { Validator } from '../shared/Validator';\nimport { printStyle } from '../shared/Validator';\nimport { allOptions } from './optionsGraph2d';\nimport { configureOptions } from './optionsGraph2d';\n\nimport Configurator from '../shared/Configurator';\n\n/**\n * Create a timeline visualization\n * @param {HTMLElement} container\n * @param {vis.DataSet | Array} [items]\n * @param {vis.DataSet | Array | vis.DataView | Object} [groups]\n * @param {Object} [options]  See Graph2d.setOptions for the available options.\n * @constructor Graph2d\n * @extends Core\n */\nfunction Graph2d (container, items, groups, options) {\n  // if the third element is options, the forth is groups (optionally);\n  if (!(Array.isArray(groups) || groups instanceof DataSet || groups instanceof DataView) && groups instanceof Object) {\n    var forthArgument = options;\n    options = groups;\n    groups = forthArgument;\n  }\n\n  // TODO: REMOVE THIS in the next MAJOR release\n  // see https://github.com/almende/vis/issues/2511\n  if (options && options.throttleRedraw) {\n    console.warn(\"Graph2d option \\\"throttleRedraw\\\" is DEPRICATED and no longer supported. It will be removed in the next MAJOR release.\");\n  }\n\n  var me = this;\n  this.defaultOptions = {\n    start: null,\n    end:   null,\n\n    autoResize: true,\n\n    orientation: {\n      axis: 'bottom',   // axis orientation: 'bottom', 'top', or 'both'\n      item: 'bottom'    // not relevant for Graph2d\n    },\n\n    moment: moment,\n\n    width: null,\n    height: null,\n    maxHeight: null,\n    minHeight: null\n  };\n  this.options = util.deepExtend({}, this.defaultOptions);\n\n  // Create the DOM, props, and emitter\n  this._create(container);\n\n  // all components listed here will be repainted automatically\n  this.components = [];\n\n  this.body = {\n    dom: this.dom,\n    domProps: this.props,\n    emitter: {\n      on: this.on.bind(this),\n      off: this.off.bind(this),\n      emit: this.emit.bind(this)\n    },\n    hiddenDates: [],\n    util: {\n      toScreen: me._toScreen.bind(me),\n      toGlobalScreen: me._toGlobalScreen.bind(me), // this refers to the root.width\n      toTime: me._toTime.bind(me),\n      toGlobalTime : me._toGlobalTime.bind(me)\n    }\n  };\n\n  // range\n  this.range = new Range(this.body);\n  this.components.push(this.range);\n  this.body.range = this.range;\n\n  // time axis\n  this.timeAxis = new TimeAxis(this.body);\n  this.components.push(this.timeAxis);\n  //this.body.util.snap = this.timeAxis.snap.bind(this.timeAxis);\n\n  // current time bar\n  this.currentTime = new CurrentTime(this.body);\n  this.components.push(this.currentTime);\n\n  // item set\n  this.linegraph = new LineGraph(this.body);\n\n  this.components.push(this.linegraph);\n\n  this.itemsData = null;      // DataSet\n  this.groupsData = null;     // DataSet\n\n\n  this.on('tap', function (event) {\n    me.emit('click', me.getEventProperties(event))\n  });\n  this.on('doubletap', function (event) {\n    me.emit('doubleClick', me.getEventProperties(event))\n  });\n  this.dom.root.oncontextmenu = function (event) {\n    me.emit('contextmenu', me.getEventProperties(event))\n  };\n  \n  //Single time autoscale/fit\n  this.initialFitDone = false;\n  this.on('changed', function (){\n    if (me.itemsData == null) return;\n    if (!me.initialFitDone && !me.options.rollingMode) {\n      me.initialFitDone = true;\n      if (me.options.start != undefined || me.options.end != undefined) {\n        if (me.options.start == undefined || me.options.end == undefined) {\n          var range = me.getItemRange();\n        }\n\n        var start = me.options.start != undefined ? me.options.start : range.min;\n        var end   = me.options.end   != undefined ? me.options.end   : range.max;\n        me.setWindow(start, end, {animation: false});\n      } else {\n        me.fit({animation: false});\n      }\n    }\n\n    if (!me.initialDrawDone && (me.initialRangeChangeDone || (!me.options.start && !me.options.end) \n      || me.options.rollingMode)) {\n      me.initialDrawDone = true;\n      me.dom.root.style.visibility = 'visible';\n      me.dom.loadingScreen.parentNode.removeChild(me.dom.loadingScreen);\n      if (me.options.onInitialDrawComplete) {\n        setTimeout(() => {\n          return me.options.onInitialDrawComplete();\n        }, 0)\n      }\n    }\n  });\n  \n  // apply options\n  if (options) {\n    this.setOptions(options);\n  }\n\n  // IMPORTANT: THIS HAPPENS BEFORE SET ITEMS!\n  if (groups) {\n    this.setGroups(groups);\n  }\n\n  // create itemset\n  if (items) {\n    this.setItems(items);\n  }\n\n  // draw for the first time\n  this._redraw();\n}\n\n// Extend the functionality from Core\nGraph2d.prototype = new Core();\n\nGraph2d.prototype.setOptions = function (options) {\n  // validate options\n  let errorFound = Validator.validate(options, allOptions);\n  if (errorFound === true) {\n    console.log('%cErrors have been found in the supplied options object.', printStyle);\n  }\n\n  Core.prototype.setOptions.call(this, options);\n};\n\n/**\n * Set items\n * @param {vis.DataSet | Array | null} items\n */\nGraph2d.prototype.setItems = function(items) {\n  var initialLoad = (this.itemsData == null);\n\n  // convert to type DataSet when needed\n  var newDataSet;\n  if (!items) {\n    newDataSet = null;\n  }\n  else if (items instanceof DataSet || items instanceof DataView) {\n    newDataSet = typeCoerceDataSet(items);\n  }\n  else {\n    // turn an array into a dataset\n    newDataSet = typeCoerceDataSet(new DataSet(items));\n  }\n\n  // set items\n  if (this.itemsData) {\n    // stop maintaining a coerced version of the old data set\n    this.itemsData.dispose()\n  }\n  this.itemsData = newDataSet;\n  this.linegraph && this.linegraph.setItems(newDataSet != null ? newDataSet.rawDS : null);\n\n  if (initialLoad) {\n    if (this.options.start != undefined || this.options.end != undefined) {\n      var start = this.options.start != undefined ? this.options.start : null;\n      var end   = this.options.end != undefined   ? this.options.end : null;\n      this.setWindow(start, end, {animation: false});\n    }\n    else {\n      this.fit({animation: false});\n    }\n  }\n};\n\n/**\n * Set groups\n * @param {vis.DataSet | Array} groups\n */\nGraph2d.prototype.setGroups = function(groups) {\n  // convert to type DataSet when needed\n  var newDataSet;\n  if (!groups) {\n    newDataSet = null;\n  }\n  else if (groups instanceof DataSet || groups instanceof DataView) {\n    newDataSet = groups;\n  }\n  else {\n    // turn an array into a dataset\n    newDataSet = new DataSet(groups);\n  }\n\n  this.groupsData = newDataSet;\n  this.linegraph.setGroups(newDataSet);\n};\n\n/**\n * Returns an object containing an SVG element with the icon of the group (size determined by iconWidth and iconHeight), the label of the group (content) and the yAxisOrientation of the group (left or right).\n * @param {vis.GraphGroup.id} groupId\n * @param {number} width\n * @param {number} height\n * @returns {{icon: SVGElement, label: string, orientation: string}|string}\n */\nGraph2d.prototype.getLegend = function(groupId, width, height) {\n  if (width  === undefined) {width  = 15;}\n  if (height === undefined) {height = 15;}\n  if (this.linegraph.groups[groupId] !== undefined) {\n    return this.linegraph.groups[groupId].getLegend(width,height);\n  }\n  else {\n    return \"cannot find group:'\" +  groupId + \"'\";\n  }\n};\n\n/**\n * This checks if the visible option of the supplied group (by ID) is true or false.\n * @param {vis.GraphGroup.id} groupId\n * @returns {boolean}\n */\nGraph2d.prototype.isGroupVisible = function(groupId) {\n  if (this.linegraph.groups[groupId] !== undefined) {\n    return (this.linegraph.groups[groupId].visible && (this.linegraph.options.groups.visibility[groupId] === undefined || this.linegraph.options.groups.visibility[groupId] == true));\n  }\n  else {\n    return false;\n  }\n};\n\n\n/**\n * Get the data range of the item set.\n * @returns {{min: Date, max: Date}} range  A range with a start and end Date.\n *                                          When no minimum is found, min==null\n *                                          When no maximum is found, max==null\n */\nGraph2d.prototype.getDataRange = function() {\n  var min = null;\n  var max = null;\n\n  // calculate min from start filed\n  for (var groupId in this.linegraph.groups) {\n    if (this.linegraph.groups.hasOwnProperty(groupId)) {\n      if (this.linegraph.groups[groupId].visible == true) {\n        for (var i = 0; i < this.linegraph.groups[groupId].itemsData.length; i++) {\n          var item = this.linegraph.groups[groupId].itemsData[i];\n          var value = util.convert(item.x, 'Date').valueOf();\n          min = min == null ? value : min > value ? value : min;\n          max = max == null ? value : max < value ? value : max;\n        }\n      }\n    }\n  }\n\n  return {\n    min: (min != null) ? new Date(min) : null,\n    max: (max != null) ? new Date(max) : null\n  };\n};\n\n\n/**\n * Generate Timeline related information from an event\n * @param {Event} event\n * @return {Object} An object with related information, like on which area\n *                  The event happened, whether clicked on an item, etc.\n */\nGraph2d.prototype.getEventProperties = function (event) {\n  var clientX = event.center ? event.center.x : event.clientX;\n  var clientY = event.center ? event.center.y : event.clientY;\n  var x = clientX - util.getAbsoluteLeft(this.dom.centerContainer);\n  var y = clientY - util.getAbsoluteTop(this.dom.centerContainer);\n  var time = this._toTime(x);\n\n  var customTime = CustomTime.customTimeFromTarget(event);\n\n  var element = util.getTarget(event);\n  var what = null;\n  if (util.hasParent(element, this.timeAxis.dom.foreground))              {what = 'axis';}\n  else if (this.timeAxis2 && util.hasParent(element, this.timeAxis2.dom.foreground)) {what = 'axis';}\n  else if (util.hasParent(element, this.linegraph.yAxisLeft.dom.frame))   {what = 'data-axis';}\n  else if (util.hasParent(element, this.linegraph.yAxisRight.dom.frame))  {what = 'data-axis';}\n  else if (util.hasParent(element, this.linegraph.legendLeft.dom.frame))  {what = 'legend';}\n  else if (util.hasParent(element, this.linegraph.legendRight.dom.frame)) {what = 'legend';}\n  else if (customTime != null)                {what = 'custom-time';}\n  else if (util.hasParent(element, this.currentTime.bar))                 {what = 'current-time';}\n  else if (util.hasParent(element, this.dom.center))                      {what = 'background';}\n\n  var value = [];\n  var yAxisLeft = this.linegraph.yAxisLeft;\n  var yAxisRight = this.linegraph.yAxisRight;\n  if (!yAxisLeft.hidden && this.itemsData.length > 0) {\n    value.push(yAxisLeft.screenToValue(y));\n  }\n  if (!yAxisRight.hidden && this.itemsData.length > 0) {\n    value.push(yAxisRight.screenToValue(y));\n  }\n\n  return {\n    event: event,\n    customTime: customTime ? customTime.options.id : null,\n    what: what,\n    pageX: event.srcEvent ? event.srcEvent.pageX : event.pageX,\n    pageY: event.srcEvent ? event.srcEvent.pageY : event.pageY,\n    x: x,\n    y: y,\n    time: time,\n    value: value\n  }\n};\n\n/**\n * Load a configurator\n * @return {Object}\n * @private\n */\nGraph2d.prototype._createConfigurator = function () {\n  return new Configurator(this, this.dom.container, configureOptions);\n};\n\n\nexport default Graph2d;\n","// locales\nimport \"moment/locale/de\";\nimport \"moment/locale/es\";\nimport \"moment/locale/fr\";\nimport \"moment/locale/it\";\nimport \"moment/locale/ja\";\nimport \"moment/locale/nl\";\nimport \"moment/locale/pl\";\nimport \"moment/locale/ru\";\nimport \"moment/locale/uk\";\n\n// utils\nimport * as util from \"vis-util/esnext\";\nimport * as DOMutil from \"./DOMutil\";\n\n// data\nimport { DataSet, DataView, Queue } from \"vis-data/esnext\";\n\n// Timeline\nimport Timeline from \"./timeline/Timeline\";\nimport Graph2d from \"./timeline/Graph2d\";\n\nimport Core from \"./timeline/Core\";\nimport * as DateUtil from \"./timeline/DateUtil\";\nimport Range from \"./timeline/Range\";\nimport * as stack from \"./timeline/Stack\";\nimport TimeStep from \"./timeline/TimeStep\";\n\nimport Item from \"./timeline/component/item/Item\";\nimport BackgroundItem from \"./timeline/component/item/BackgroundItem\";\nimport BoxItem from \"./timeline/component/item/BoxItem\";\nimport PointItem from \"./timeline/component/item/PointItem\";\nimport RangeItem from \"./timeline/component/item/RangeItem\";\n\nimport BackgroundGroup from \"./timeline/component/BackgroundGroup\";\nimport Component from \"./timeline/component/Component\";\nimport CurrentTime from \"./timeline/component/CurrentTime\";\nimport CustomTime from \"./timeline/component/CustomTime\";\nimport DataAxis from \"./timeline/component/DataAxis\";\nimport DataScale from \"./timeline/component/DataScale\";\nimport GraphGroup from \"./timeline/component/GraphGroup\";\nimport Group from \"./timeline/component/Group\";\nimport ItemSet from \"./timeline/component/ItemSet\";\nimport Legend from \"./timeline/component/Legend\";\nimport LineGraph from \"./timeline/component/LineGraph\";\nimport TimeAxis from \"./timeline/component/TimeAxis\";\n\n// bundled external libraries\nimport moment from \"./module/moment\";\nimport Hammer from \"./module/hammer\";\nimport keycharm from \"keycharm\";\n\n// TODO: This should probably be moved somewhere else to ensure that both builds\n// behave the same way.\nconst defaultLanguage = DOMutil.getNavigatorLanguage();\nmoment.locale(defaultLanguage);\n\nconst timeline = {\n  Core,\n  DateUtil,\n  Range,\n  stack,\n  TimeStep,\n\n  components: {\n    items: {\n      Item,\n      BackgroundItem,\n      BoxItem,\n      PointItem,\n      RangeItem\n    },\n\n    BackgroundGroup,\n    Component,\n    CurrentTime,\n    CustomTime,\n    DataAxis,\n    DataScale,\n    GraphGroup,\n    Group,\n    ItemSet,\n    Legend,\n    LineGraph,\n    TimeAxis\n  }\n};\n\nexport {\n  DOMutil,\n  DataSet,\n  DataView,\n  Graph2d,\n  Hammer,\n  Queue,\n  Timeline,\n  keycharm,\n  moment,\n  timeline,\n  util\n};\n"],"names":["module","hookCallback","some","hooks","apply","arguments","isArray","input","Array","Object","prototype","toString","call","isObject","isUndefined","isNumber","isDate","Date","map","arr","fn","i","res","length","push","hasOwnProp","a","b","hasOwnProperty","extend","valueOf","createUTC","format","locale","strict","createLocalOrUTC","utc","getParsingFlags","m","_pf","empty","unusedTokens","unusedInput","overflow","charsLeftOver","nullInput","invalidMonth","invalidFormat","userInvalidated","iso","parsedDateParts","meridiem","rfc2822","weekdayMismatch","isValid","_isValid","flags","parsedParts","isNowValid","isNaN","_d","getTime","invalidWeekday","_strict","undefined","bigHour","isFrozen","createInvalid","NaN","fun","t","this","len","momentProperties","copyConfig","to","from","prop","val","_isAMomentObject","_i","_f","_l","_tzm","_isUTC","_offset","_locale","updateInProgress","Moment","config","updateOffset","isMoment","obj","absFloor","number","Math","ceil","floor","toInt","argumentForCoercion","coercedNumber","value","isFinite","compareArrays","array1","array2","dontConvert","min","lengthDiff","abs","diffs","warn","msg","suppressDeprecationWarnings","console","deprecate","firstTime","deprecationHandler","arg","args","key","slice","join","Error","stack","keys","deprecations","deprecateSimple","name","isFunction","Function","mergeConfigs","parentConfig","childConfig","Locale","set","aliases","addUnitAlias","unit","shorthand","lowerCase","toLowerCase","normalizeUnits","units","normalizeObjectUnits","inputObject","normalizedProp","normalizedInput","priorities","addUnitPriority","priority","zeroFill","targetLength","forceSign","absNumber","zerosToFill","pow","max","substr","formattingTokens","localFormattingTokens","formatFunctions","formatTokenFunctions","addFormatToken","token","padded","ordinal","callback","func","localeData","formatMoment","expandFormat","array","match","replace","mom","output","makeFormatFunction","invalidDate","replaceLongDateFormatTokens","longDateFormat","lastIndex","test","match1","match2","match3","match4","match6","match1to2","match3to4","match5to6","match1to3","match1to4","match1to6","matchUnsigned","matchSigned","matchOffset","matchShortOffset","matchWord","regexes","addRegexToken","regex","strictRegex","isStrict","getParseRegexForToken","RegExp","regexEscape","matched","p1","p2","p3","p4","s","tokens","addParseToken","addWeekParseToken","_w","addTimeToArrayFromToken","_a","daysInYear","year","isLeapYear","y","parseTwoDigitYear","parseInt","indexOf","getSetYear","makeGetSet","keepTime","set$1","get","month","date","daysInMonth","x","modMonth","o","monthsShort","months","monthsShortRegex","monthsRegex","monthsParse","MONTHS_IN_FORMAT","defaultLocaleMonths","split","defaultLocaleMonthsShort","handleStrictParse","monthName","ii","llc","toLocaleLowerCase","_monthsParse","_longMonthsParse","_shortMonthsParse","setMonth","dayOfMonth","getSetMonth","defaultMonthsShortRegex","defaultMonthsRegex","computeMonthsParse","cmpLenRev","shortPieces","longPieces","mixedPieces","sort","_monthsRegex","_monthsShortRegex","_monthsStrictRegex","_monthsShortStrictRegex","createDate","d","h","M","ms","getFullYear","setFullYear","createUTCDate","UTC","getUTCFullYear","setUTCFullYear","firstWeekOffset","dow","doy","fwd","getUTCDay","dayOfYearFromWeeks","week","weekday","resYear","resDayOfYear","dayOfYear","weekOfYear","resWeek","weekOffset","weeksInYear","weekOffsetNext","shiftWeekdays","ws","n","concat","weekdaysMin","weekdaysShort","weekdays","weekdaysMinRegex","weekdaysShortRegex","weekdaysRegex","weekdaysParse","defaultLocaleWeekdays","defaultLocaleWeekdaysShort","defaultLocaleWeekdaysMin","handleStrictParse$1","weekdayName","_weekdaysParse","_shortWeekdaysParse","_minWeekdaysParse","day","defaultWeekdaysRegex","defaultWeekdaysShortRegex","defaultWeekdaysMinRegex","computeWeekdaysParse","minp","shortp","longp","minPieces","_weekdaysRegex","_weekdaysShortRegex","_weekdaysMinRegex","_weekdaysStrictRegex","_weekdaysShortStrictRegex","_weekdaysMinStrictRegex","hFormat","hours","lowercase","minutes","matchMeridiem","_meridiemParse","seconds","kInput","_isPm","isPM","_meridiem","pos","pos1","pos2","globalLocale","getSetHour","baseConfig","calendar","sameDay","nextDay","nextWeek","lastDay","lastWeek","sameElse","LTS","LT","L","LL","LLL","LLLL","dayOfMonthOrdinalParse","relativeTime","future","past","ss","mm","hh","dd","MM","yy","meridiemParse","locales","localeFamilies","normalizeLocale","loadLocale","oldLocale","exports","_abbr","require","getSetGlobalLocale","e","values","data","getLocale","defineLocale","abbr","_config","parentLocale","forEach","names","j","next","chooseLocale","checkOverflow","_overflowDayOfYear","_overflowWeeks","_overflowWeekday","defaults","c","configFromArray","currentDate","expectedWeekday","yearToUse","nowValue","now","_useUTC","getUTCMonth","getUTCDate","getMonth","getDate","currentDateArray","w","weekYear","temp","weekdayOverflow","GG","W","E","createLocal","_week","curWeek","gg","_dayOfYear","dayOfYearFromWeekInfo","_nextDay","getDay","setUTCMinutes","getUTCMinutes","extendedIsoRegex","basicIsoRegex","tzRegex","isoDates","isoTimes","aspNetJsonRegex","configFromISO","l","allowTime","dateFormat","timeFormat","tzFormat","string","exec","configFromStringAndFormat","untruncateYear","yearStr","obsOffsets","UT","GMT","EDT","EST","CDT","CST","MDT","MST","PDT","PST","configFromRFC2822","monthStr","dayStr","hourStr","minuteStr","secondStr","result","parsedArray","weekdayStr","parsedInput","checkWeekday","obsOffset","militaryOffset","numOffset","hm","calculateOffset","ISO_8601","RFC_2822","skipped","stringLength","totalParsedInputLength","hour","isPm","meridiemHour","meridiemFixWrap","prepareConfig","preparse","tempConfig","bestMoment","scoreToBeat","currentScore","score","configFromStringAndArray","createFromInputFallback","configFromString","minute","second","millisecond","configFromObject","configFromInput","isUTC","getOwnPropertyNames","k","isObjectEmpty","add","prototypeMin","other","prototypeMax","pickBy","moments","ordering","Duration","duration","years","quarters","quarter","weeks","isoWeek","days","milliseconds","unitHasDecimal","parseFloat","isDurationValid","_milliseconds","_days","_months","_data","_bubble","isDuration","absRound","round","offset","separator","utcOffset","sign","offsetFromString","chunkOffset","matcher","matches","parts","cloneWithOffset","model","diff","clone","setTime","local","getDateOffset","getTimezoneOffset","isUtc","aspNetRegex","isoRegex","createDuration","ret","diffRes","base","parseIso","isBefore","positiveMomentsDifference","inp","isAfter","createAdder","direction","period","tmp","addSubtract","isAdding","invalid","subtract","monthDiff","wholeMonthDiff","anchor","newLocaleData","defaultFormat","defaultFormatUtc","lang","mod$1","dividend","divisor","localStartOfDate","utcStartOfDate","addWeekYearFormatToken","getter","getSetWeekYearHelper","weeksTarget","setWeekAll","dayOfYearData","isoWeekYear","_dayOfMonthOrdinalParse","_ordinalParse","_dayOfMonthOrdinalParseLenient","getSetDayOfMonth","getSetMinute","getSetSecond","parseMs","getSetMillisecond","proto","preParsePostFormat","time","formats","sod","startOf","calendarFormat","asFloat","that","zoneDelta","endOf","startOfDate","isoWeekday","inputString","postformat","withoutSuffix","humanize","fromNow","toNow","invalidAt","localInput","isBetween","inclusivity","localFrom","localTo","isSame","inputMs","isSameOrAfter","isSameOrBefore","parsingFlags","prioritized","unitsObj","u","getPrioritizedUnits","toArray","toObject","toDate","toISOString","keepOffset","inspect","zone","isLocal","prefix","suffix","toJSON","unix","creationData","isoWeeks","weekInfo","isoWeeksInYear","parseWeekday","parseIsoWeekday","keepLocalTime","keepMinutes","localAdjust","_changeInProgress","parseZone","tZone","hasAlignedHourOffset","isDST","isUtcOffset","zoneAbbr","zoneName","dates","isDSTShifted","_isDSTShifted","proto$1","get$1","index","field","setter","listMonthsImpl","out","listWeekdaysImpl","localeSorted","shift","_calendar","_longDateFormat","formatUpper","toUpperCase","_invalidDate","_ordinal","isFuture","_relativeTime","pastFuture","source","isFormat","_monthsShort","_monthsParseExact","firstDayOfYear","firstDayOfWeek","_weekdays","_weekdaysMin","_weekdaysShort","_weekdaysParseExact","_fullWeekdaysParse","charAt","isLower","langData","mathAbs","addSubtract$1","absCeil","daysToMonths","monthsToDays","makeAs","alias","as","asMilliseconds","asSeconds","asMinutes","asHours","asDays","asWeeks","asMonths","asQuarters","asYears","makeGetter","thresholds","substituteTimeAgo","relativeTime$1","posNegDuration","abs$1","toISOString$1","Y","D","toFixed","total","totalSign","ymSign","daysSign","hmsSign","proto$2","monthsFromDays","withSuffix","toIsoString","version","updateLocale","tmpLocale","relativeTimeRounding","roundingFunction","relativeTimeThreshold","threshold","limit","myMoment","HTML5_FMT","DATETIME_LOCAL","DATETIME_LOCAL_SECONDS","DATETIME_LOCAL_MS","DATE","TIME","TIME_SECONDS","TIME_MS","WEEK","MONTH","factory","check","moment","processRelativeTime","monthsParseExact","weekdaysParseExact","require$$0","monthsShortDot","monthsStrictRegex","monthsShortStrictRegex","longMonthsParse","shortMonthsParse","ll","lll","llll","monthsShortWithDots","monthsShortWithoutDots","monthsNominative","monthsSubjective","plural","translate","momentToFormat","relativeTimeWithPlural","num","forms","standalone","processHoursFunction","str","it","globalThis","window","self","global","error","fails","defineProperty","nativePropertyIsEnumerable","propertyIsEnumerable","getOwnPropertyDescriptor","V","descriptor","enumerable","bitmap","configurable","writable","classof","TypeError","IndexedObject","requireObjectCoercible","PREFERRED_STRING","document","EXISTS","createElement","DESCRIPTORS","nativeGetOwnPropertyDescriptor","O","P","toIndexedObject","toPrimitive","IE8_DOM_DEFINE","has","createPropertyDescriptor","propertyIsEnumerableModule","f","replacement","isForced","feature","detection","normalize","POLYFILL","NATIVE","String","aFunction","nativeDefineProperty","Attributes","anObject","object","definePropertyModule","wrapConstructor","NativeConstructor","Wrapper","options","USE_NATIVE","VIRTUAL_PROTOTYPE","sourceProperty","targetProperty","nativeProperty","resultProperty","TARGET","target","GLOBAL","STATIC","stat","PROTO","nativeSource","path","targetPrototype","forced","noTargetGet","bind","wrap","sham","createNonEnumerableProperty","real","objectDefinePropertyModile","desc","parent","argument","toInteger","integer","createMethod","IS_INCLUDES","$this","el","fromIndex","toLength","toAbsoluteIndex","includes","hiddenKeys","internalObjectKeys","enumBugKeys","defineProperties","Properties","objectKeys","T","variable","namespace","method","getOwnPropertySymbols","getBuiltIn","getOwnPropertyNamesModule","getOwnPropertySymbolsModule","propertyKey","getOwnPropertyDescriptors","getOwnPropertyDescriptorModule","ownKeys","createProperty","FAILS_ON_PRIMITIVES","activeXDocument","Symbol","NATIVE_SYMBOL","iterator","setGlobal","store","mode","copyright","id","postfix","random","shared","uid","IE_PROTO","sharedKey","EmptyConstructor","scriptTag","content","NullProtoObject","domain","ActiveXObject","iframeDocument","iframe","write","close","parentWindow","NullProtoObjectViaActiveX","documentCreateElement","style","display","html","appendChild","src","contentWindow","open","F","create","nativeGetOwnPropertyNames","windowNames","getWindowNames","WellKnownSymbolsStore","createWellKnownSymbol","USE_SYMBOL_AS_UID","withoutSetter","wellKnownSymbol","NAME","wrappedWellKnownSymbolModule","TO_STRING_TAG","CORRECT_ARGUMENTS","classofRaw","TO_STRING_TAG_SUPPORT","tag","tryGet","callee","TAG","SET_METHOD","functionToString","inspectSource","WeakMap","NATIVE_WEAK_MAP","wmget","wmhas","wmset","metadata","STATE","objectHas","enforce","getterFor","TYPE","state","type","SPECIES","originalArray","C","constructor","IS_MAP","IS_FILTER","IS_SOME","IS_EVERY","IS_FIND_INDEX","NO_HOLES","callbackfn","specificCreate","boundFunction","arraySpeciesCreate","filter","every","find","findIndex","$forEach","HIDDEN","TO_PRIMITIVE","setInternalState","InternalStateModule","getInternalState","ObjectPrototype","$Symbol","$stringify","getOwnPropertyNamesExternal","AllSymbols","ObjectPrototypeSymbols","StringToSymbolRegistry","SymbolToStringRegistry","QObject","USE_SETTER","findChild","setSymbolDescriptor","nativeObjectCreate","ObjectPrototypeDescriptor","description","symbol","isSymbol","$defineProperty","$defineProperties","properties","$getOwnPropertySymbols","$propertyIsEnumerable","$getOwnPropertyDescriptor","$getOwnPropertyNames","IS_OBJECT_PROTOTYPE","redefine","defineWellKnownSymbol","keyFor","sym","useSetter","useSimple","FORCED_JSON_STRINGIFY","$","stringify","replacer","space","$replacer","setToStringTag","IteratorPrototype","PrototypeOfArrayIteratorPrototype","arrayIterator","getPrototypeOf","CORRECT_PROTOTYPE_GETTER","BUGGY_SAFARI_ITERATORS","returnThis","setPrototypeOf","CORRECT_SETTER","aPossiblePrototype","__proto__","IteratorsCore","ITERATOR","Iterable","IteratorConstructor","DEFAULT","IS_SET","FORCED","Iterators","createIteratorConstructor","CurrentIteratorPrototype","methods","KEY","getIterationMethod","KIND","defaultIterator","IterablePrototype","INCORRECT_VALUES_NAME","nativeIterator","anyNativeIterator","entries","defineIterator","iterated","kind","done","Arguments","COLLECTION_NAME","CSSRuleList","CSSStyleDeclaration","CSSValueList","ClientRectList","DOMRectList","DOMStringList","DOMTokenList","DataTransferItemList","FileList","HTMLAllCollection","HTMLCollection","HTMLFormElement","HTMLSelectElement","MediaList","MimeTypeArray","NamedNodeMap","NodeList","PaintRequestList","Plugin","PluginArray","SVGLengthList","SVGNumberList","SVGPathSegList","SVGPointList","SVGStringList","SVGTransformList","SourceBufferList","StyleSheetList","TextTrackCueList","TextTrackList","TouchList","Collection","CollectionPrototype","CONVERT_TO_STRING","first","S","position","size","charCodeAt","codeAt","point","iteratorMethod","getIteratorMethod","_Object$defineProperty","nativeKeys","METHOD_NAME","whitespace","whitespaces","ltrim","rtrim","start","end","trim","$trim","CONSTRUCTOR","entryVirtual","StringPrototype","own","cache","thrower","ACCESSORS","argument0","argument1","STRICT_METHOD","arrayMethodIsStrict","USES_TO_LENGTH","arrayMethodUsesToLength","ArrayPrototype","DOMIterables","process","versions","v8","userAgent","V8_VERSION","foo","Boolean","$map","HAS_SPECIES_SUPPORT","arrayMethodHasSpeciesSupport","$parseInt","hex","radix","parseIntImplementation","TO_ENTRIES","$values","$filter","IS_CONCAT_SPREADABLE","IS_CONCAT_SPREADABLE_SUPPORT","SPECIES_SUPPORT","isConcatSpreadable","spreadable","A","_Array$isArray","arr2","ENTRIES","returnMethod","SAFE_CLOSING","called","iteratorWithReturn","INCORRECT_ITERATION","SKIP_CLOSING","ITERATION_SUPPORT","checkCorrectnessOfIteration","iterable","arrayLike","step","argumentsLength","mapfn","mapping","isArrayIteratorMethod","callWithSafeIterationClosing","iter","_isIterable","_Array$from","arrayWithoutHoles","iterableToArray","nonIterableSpread","nativeSlice","Constructor","fin","nativeGetPrototypeOf","$indexOf","nativeIndexOf","NEGATIVE_ZERO","searchElement","nativeAssign","assign","B","chr","WrappedWellKnownSymbolModule","JSON","_typeof","_Symbol","_Symbol$iterator","_arr","_n","_e","_s","_getIterator","err","arrayWithHoles","iterableToArrayLimit","nonIterableRest","AleaImplementation","seed","mash","Mash","s0","s1","s2","mashSeed","uint32","fract53","algorithm","_Date$now","prepareElements","JSONcontainer","elementType","redundant","used","cleanupElements","parentNode","removeChild","resetElements","getSVGElement","svgContainer","element","createElementNS","getDOMElement","DOMContainer","insertBefore","drawPoint","groupTemplate","labelObj","setAttributeNS","styles","className","label","xOffset","yOffset","textContent","drawBar","width","height","rect","getNavigatorLanguage","navigator","languages","userLanguage","language","browserLanguage","ReferenceError","re","low","hi","fix","prev","core","$some","assertThisInitialized","_getPrototypeOf","_Object$setPrototypeOf","_Object$getPrototypeOf","_setPrototypeOf","p","subClass","superClass","_Object$create","nativeSort","FAILS_ON_UNDEFINED","FAILS_ON_NULL","comparefn","isExtensible","preventExtensions","METADATA","setMetadata","objectID","weakData","meta","REQUIRED","fastKey","getWeakData","onFreeze","FREEZING","Result","stopped","AS_ENTRIES","IS_ITERATOR","iterFn","stop","require$$1","internalStateGetterFor","CONSTRUCTOR_NAME","wrapper","common","IS_WEAK","ADDER","NativePrototype","exported","anInstance","collection","iterate","IS_ADDER","getConstructor","InternalMetadataModule","setStrong","unsafe","last","define","previous","entry","getEntry","removed","redefineAll","clear","ITERATOR_NAME","getInternalCollectionState","getInternalIteratorState","setSpecies","init","collectionStrong","Set","Map","runtime","Op","hasOwn","iteratorSymbol","asyncIteratorSymbol","asyncIterator","toStringTagSymbol","toStringTag","innerFn","outerFn","tryLocsList","protoGenerator","Generator","generator","context","Context","_invoke","doneResult","delegate","delegateResult","maybeInvokeDelegate","ContinueSentinel","sent","_sent","dispatchException","abrupt","record","tryCatch","makeInvokeMethod","GeneratorFunction","GeneratorFunctionPrototype","getProto","NativeIteratorPrototype","Gp","defineIteratorMethods","AsyncIterator","PromiseImpl","previousPromise","callInvokeWithMethodAndArg","resolve","reject","invoke","__await","then","unwrapped","info","resultName","nextLoc","pushTryEntry","locs","tryLoc","catchLoc","finallyLoc","afterLoc","tryEntries","resetTryEntry","completion","reset","displayName","isGeneratorFunction","genFun","ctor","mark","awrap","async","Promise","reverse","pop","skipTempReset","rootRecord","rval","exception","handle","loc","caught","hasCatch","hasFinally","finallyEntry","complete","finish","thrown","delegateYield","regeneratorRuntime","accidentalStrictMode","splice","deleteCount","insertCount","actualDeleteCount","actualStart","MSIE","scheduler","handler","timeout","boundArgs","setTimeout","setInterval","flattenIntoArray","original","sourceLen","depth","mapper","thisArg","targetIndex","sourceIndex","mapFn","flatMap","IS_RIGHT","memo","$reduce","left","right","reduce","factories","construct","argsLength","list","partArgs","FunctionPrototype","instance","_defineProperties","props","protoProps","staticProps","createNewDataPipeFrom","DataPipeUnderConstruction","SimpleDataPipe","_source","_transformers","_target","_listeners","_bindInstanceProperty","_add","remove","_remove","update","_update","_transformItems","on","off","items","_reduceInstanceProperty","transform","_name","payload","oldData","_filterInstanceProperty","_mapInstanceProperty","_flatMapInstanceProperty","ASPDateRegex","convert","isString","parse","Number","getType","typeCoerceDataSet","rawDS","idProp","_idProp","coercedDS","DataSet","fieldId","pipe","item","acc","all","getDataSet","updateOnly","getIds","dispose","util","$parseFloat","Infinity","trimmedString","parseFloatImplementation","Component","body","resized","_previousWidth","_previousHeight","repeat","count","RangeError","convertHiddenOptions","hiddenDates","_repeatInstanceProperty","dateItem","updateHiddenDates","domProps","centerContainer","range","pixelTime","startDate","endDate","runUntil","dayOffset","log","removeDuplicates","startHidden","getIsHidden","endHidden","rangeStart","rangeEnd","hidden","startToFront","endToFront","_applyRange","safeDates","stepOverHiddenDates","timeStep","previousTime","stepInHidden","currentValue","current","_end","prevValue","newValue","switchedYear","switchedMonth","switchedDay","toScreen","Core","conversion","scale","getHiddenDurationBetween","hiddenBeforeStart","getHiddenDurationBeforeStart","rangeAfterEnd","correctTimeForHidden","toTime","hiddenDuration","partialDuration","accumulatedHiddenDuration","getAccumulatedHiddenDuration","getHiddenDurationBefore","timeOffset","requiredDuration","previousPoint","snapAwayFromHidden","correctionEnabled","isHidden","Range","millisecondsPerPixelCache","rolling","deltaDifference","scaleOffset","defaultOptions","rtl","moveable","zoomable","zoomMin","zoomMax","rollingMode","follow","_this","touch","animationTimer","emitter","_onDragStart","_onDrag","_onDragEnd","_onMouseWheel","_onTouch","_onPinch","dom","rollingModeBtn","addEventListener","startRolling","setOptions","selectiveExtend","setRange","me","stopRolling","interval","rollingModeOffset","animation","center","visibility","currentTimeTimer","_setTimeout","clearTimeout","frameCallback","byUser","finalStart","finalEnd","_cancelAnimation","initStart","initEnd","easingName","easingFunction","easingFunctions","_JSON$stringify","_Object$keys","initTime","anyChanged","dragging","ease","changed","DateUtil","params","event","emit","timeoutID","clientWidth","newStart","newEnd","_parseFloat","totalHidden","previousDelta","_isInsideRange","allowDragging","root","cursor","validateDirection","delta","deltaX","deltaY","diffRange","safeStart","safeEnd","wheelDelta","detail","zoomKey","pointerDate","zoomFriction","pointer","getPointer","clientX","clientY","_pointerToDate","zoom","preventDefault","centerDate","hiddenDurationBefore","hiddenDurationAfter","centerContainerRect","getBoundingClientRect","elementRect","top","moveTo","Emitter","mixin","_callbacks","once","removeListener","removeAllListeners","removeEventListener","cb","callbacks","listeners","hasListeners","_firstTarget","propagating","hammer","_options","Manager","Hammer","PropagatingHammer","events","propagatedHandler","srcEvent","_handled","stopPropagation","srcStop","firstTarget","elem","elemHammer","_handlers","pointerType","isFirst","eventType","destroy","hammers","idx","assign$1","nextKey","modifiedHammer","noop","onTouch","inputHandler","TimeStep","minimumStep","_start","autoScale","FORMAT","deepExtend","setMinimumStep","roundToMinor","priorCurrent","showMajorLabels","enable","stepYear","stepMonth","stepDay","showWeekScale","stepHour","stepMinute","stepSecond","stepMillisecond","minorLabels","majorLabels","_moment","classNames","even","today","currentWeek","currentMonth","currentYear","_step","styleInject","css","ref","insertAt","head","getElementsByTagName","firstChild","styleSheet","cssText","createTextNode","TimeAxis","foreground","lines","majorTexts","minorTexts","lineTop","orientation","axis","showMinorLabels","maxMinorChars","timeAxis","_create","selectiveDeepExtend","background","bottom","parentChanged","_calculateCharSize","minorLabelHeight","minorCharHeight","majorLabelHeight","majorCharHeight","offsetWidth","minorLineHeight","minorLineWidth","majorLineHeight","majorLineWidth","foregroundNextSibling","nextSibling","backgroundNextSibling","_repaintLabels","backgroundVertical","_isResized","timeLabelsize","minorCharWidth","setMoment","setFormat","setScale","xNext","isMajor","showMinorGrid","prevWidth","line","xFirstMajorLabel","getCurrent","hasNext","getClassName","_repaintMinorText","getLabelMinor","_repaintMajorText","getLabelMajor","_repaintMajorLine","_repaintMinorLine","_parseInt","warnedForOverflow","leftTime","leftText","widthText","majorCharWidth","text","innerHTML","_setXY","childNodes","directionX","measureCharMinor","clientHeight","measureCharMajor","container","_exportFunctions","_bound","keydown","keyup","_keys","fromCharCode","code","down","handleEvent","up","keyCode","bound","shiftKey","bindAll","getKey","unbind","newBindings","Activator","active","overlay","_onTapOverlay","onClick","_hasParent","deactivate","keycharm","escListener","activate","addClassName","removeClassName","en","deleteSelected","nl","de","fr","es","uk","ru","pl","ja","en_EN","en_US","it_IT","it_CH","nl_NL","nl_BE","de_DE","fr_FR","fr_CA","fr_BE","es_ES","uk_UA","ru_RU","pl_PL","ja_JP","CustomTime","title","defaultLocales","customTime","eventParams","bar","drag","onMouseWheel","attachEvent","DIRECTION_ALL","hide","warned","substring","redraw","editable","marker","setAttribute","focus","_onMarkerChange","_onMarkerChanged","getScale","getStep","snap","snappedTime","setCustomTime","backgroundHorizontal","leftContainer","rightContainer","shadowTop","shadowBottom","shadowTopLeft","shadowBottomLeft","shadowTopRight","shadowBottomRight","loadingScreen","border","scrollTop","scrollTopMin","initialDrawDone","_redraw","initialRangeChangeDone","_origRedraw","throttle","itemSet","initialItemSetDrawn","queue","pinchRecognizer","getTouchAction","hammerUtil","isActive","wheelDeltaY","wheelDeltaX","HORIZONTAL_AXIS","deltaMode","preferZoom","verticalScroll","horizontalScroll","adjusted","_setScrollTop","listener","isFinal","wheelType","onmousewheel","onMouseScrollSide","_isProgramaticallyScrolled","itemAddedToTimeline","handleDragOver","getEventProperties","_indexOfInstanceProperty","dataTransfer","dropEffect","handleDrop","itemData","getData","_onAddItem","_onDropObjectOnItem","customTimes","redrawCount","timeAxis2","components","drawPoints","onRender","clickToUse","activator","_initAutoResize","component","configurator","_createConfigurator","configure","appliedOptions","setModuleOptions","setItems","setGroups","_stopAutoResize","getCustomTime","setCustomMarker","setCustomTitle","timestamp","_someInstanceProperty","_this2","getVisibleItems","getVisibleGroups","getDataRange","getRange","percentage","getWindow","newInterval","distance","setWindow","maxHeight","option","asSize","minHeight","rootOffsetWidth","offsetHeight","contentHeight","autoHeight","containerHeight","scrollbarWidth","getScrollBarWidth","leftContainerClientWidth","rightContainerClientWidth","_setDOM","_updateScrollTop","visibilityTop","visibilityBottom","contentsOverflow","DIRECTION_HORIZONTAL","longSelectPressTime","centerWidth","currentTime","setCurrentTime","getCurrentTime","autoResize","_startAutoResize","_onResize","rootOffsetHeight","lastWidth","lastHeight","watchTimer","_setInterval","clearInterval","initialScrollTop","oldScrollTop","_getScrollTop","newScrollTop","CurrentTime","showCurrentTime","alignCurrentTime","$find","SKIPS_HOLES","getRandomValues","crypto","msCrypto","rnds8","Uint8Array","rng","byteToHex","v4","buf","rnds","bth","bytesToUuid","$includes","MATCH","isRegExp","regexp","correctIsRegExpLogic","searchString","notARegExp","arrayIncludes","stringIncludes","orderByStart","orderByEnd","margin","force","shouldBailItemsRedrawFunction","shouldBail","collidingItem","jj","collision","vertical","substack","subgroup","subgroupHeight","baseTop","nostack","subgroups","isStackSubgroups","newTop","visible","stackSubgroups","otherSubgroup","collisionByTimes","stackSubgroupsWithInnerStack","subgroupItems","doSubStack","subgroupOrder","horizontal","timeOverlap","heightOverlap","ReservedGroupIds","Group","groupId","subgroupStack","subgroupStackAll","subgroupVisibility","doInnerStack","shouldBailStackItems","subgroupIndex","subgroupOrderer","isVisible","stackDirty","_disposeCallbacks","nestedGroups","showNested","heightMode","groupHeightMode","nestedInGroup","visibleItems","itemsInRange","orderedItems","byStart","byEnd","checkRangedItems","handleCheckRangedItems","setData","groupEditable","order","inner","groupTouchParams","isDragging","templateFunction","Element","isReactComponent","treeLevel","removeCssText","addCssText","markerHeight","lastMarkerHeight","redrawQueue","redrawQueueLength","dirty","displayed","fns","offsetTop","offsetLeft","timeoutOptions","onTimeout","bailOptions","relativeBailingTime","itemsSettingTime","bailTimeMs","timeoutMs","userBailFunction","bail","userContinueNotBail","didUserContinue","forceRestack","lastIsVisible","isCluster","orderedClusters","cluster","_updateItemsInRange","visibleClusters","_updateClustersInRange","getVisibleItemsGroupedBySubgroup","orderFn","visibleSubgroupsItems","_sortInstanceProperty","_updateSubGroupHeights","customOrderedItems","_shouldBailItemsRedraw","repositionX","updateProperty","labelWidth","labelHeight","repositionY","returnQueue","_this3","_didMarkerHeightChange","_calculateGroupSizeAndPosition","_isGroupVisible","_redrawItems","_updateSubgroupsSizes","_calculateHeight","_didResize","_applyGroupHeight","_updateItemsVerticalPosition","_resetSubgroups","_this4","labelSet","setParent","_addToSubgroup","orderSubgroups","_includesInstanceProperty","_checkIfVisible","subgroupId","itemEnd","initialEnd","sortArray","sortField","_spliceInstanceProperty","_removeFromSubgroup","itemIndex","removeItem","startArray","endArray","oldVisibleItems","visibleItemsLookup","lowerBound","upperBound","_checkIfVisibleWithReference","initialPosByStart","binarySearchCustom","_traceVisible","initialPosByEnd","initialPos","breakCondition","hasItems","show","oldVisibleClusters","visibleClustersLookup","oldSubgroup","newSubgroup","disposeCallback","BackgroundGroup","Item","selected","groupShowing","selectable","setSelectability","_updateEditStatus","group","_moveToGroup","changeSubgroup","updateTime","dragCenter","dragCenterItem","hammerDragCenter","_onUpdateItem","box","dragLeft","overrideItems","deleteButton","optionsLocale","hammerDeleteButton","removeFromDataSet","tooltipOnItemUpdateTime","onItemUpdateTimeTooltip","touchParams","itemIsDragging","template","itemsData","itemVisibleFrameContent","visibleFrameTemplateFunction","_getItemData","itemVisibleFrameContentElement","getElementsByClassName","visibleFrameTemplate","_contentToString","dataAttributes","attributes","removeAttribute","outerHTML","updateGroup","tooltip","BoxItem","dot","align","widthInMs","getMillisecondsPerPixel","_updateContents","_updateDataAttributes","_updateStyle","sizes","_repaintOnItemUpdateTimeTooltip","_repaintDragCenter","_repaintDeleteButton","_createDomElement","_appendDomElement","_updateDirtyDomComponents","_getDomComponentsSizes","_updateDomComponentsSizes","_repaintDomAdditionals","repositionXY","boxX","boxY","dotX","dotY","lineX","lineY","lineWidth","dotWidth","lineStyle","lineHeight","PointItem","marginLeft","marginRight","translateX","pointX","pointY","RangeItem","frame","visibleFrame","baseClassName","maxWidth","getComputedStyle","whiteSpace","_repaintDragLeft","_repaintDragRight","limitSize","contentStartPosition","contentWidth","parentWidth","boxWidth","itemsAlwaysDraggable","dragLeftItem","dragRight","dragRightItem","BackgroundItem","itemSubgroup","Popup","overflowMethod","padding","doShow","isLeft","isTop","$every","ClusterItem","modifiedOptions","_Object$assign","fitOnDoubleClick","uiItems","randomUUID","_setupRange","eventEmitter","attached","detach","attach","rangeWidth","showStipes","repositionXWithRanges","repositionXWithoutRanges","_isStipeVisible","repositionStype","lineOffsetWidth","dotOffsetWidth","lineOffset","dotOffset","lineOffsetDirection","dotOffsetDirection","minWidth","itemSetHeight","detachFromParent","_fit","stats","centers","avg","sum","ondblclick","_onDoubleClick","fitStart","fitEnd","_getFitRange","fitArgs","ClusterGenerator","groups","dataChanged","applyOnChangedLevel","oldClusters","maxItems","clusterCriteria","level","timeWindow","levelChanged","cacheLevel","_dropLevelsCache","_filterData","clusters","groupName","iMax","neighbors","clusterItems","getGroupId","_getClusterForItems","_Object$values2","currentGroupName","oldClustersLookup","itemsIds","oldClusterData","_everyInstanceProperty","clusterItem","setUiItems","titleTemplate","clusterContent","clusterOptions","createClusterItem","BACKGROUND","ItemSet","groupOrderSwap","fromGroup","toGroup","targetOrder","groupOrder","multiselect","onDropObjectOnItem","objectData","onAdd","onUpdate","onMove","onRemove","onMoving","onAddGroup","onMoveGroup","onRemoveGroup","showTooltips","followMouse","delay","groupsData","sequentialSelection","itemListeners","senderId","_onAdd","clusterGenerator","_onUpdate","_onRemove","groupListeners","_onAddGroups","groupData","updatedGroups","nestedGroupId","updatedNestedGroup","_concatInstanceProperty","_onUpdateGroups","_onRemoveGroups","groupIds","selection","popup","popupTimer","_updateUngrouped","backgroundGroup","ALL","_onSelectItem","_onMultiSelectItem","groupHammer","_onGroupClick","_onGroupDragStart","_onGroupDrag","_onGroupDragEnd","DIRECTION_VERTICAL","_onMouseOver","_onMouseOut","_onMouseMove","markDirty","refreshItems","restackGroups","_detachAllClusters","clearPopupTimer","ids","idsToDeselect","selectedId","getItemById","unselect","select","rawVisibleItems","_findInstanceProperty","_clusterItems","_orderGroups","visibleInterval","zoomed","lastVisibleInterval","scrolled","lastRangeStart","changedStackOption","lastStack","changedStackSubgroupsOption","lastStackSubgroups","firstGroup","_firstGroup","firstMargin","nonFirstMargin","groupMargin","redrawResults","groupResized","firstGroupIndex","firstGroupId","itemId","ungrouped","oldItemsData","DataView","_order","updateData","_this5","_getType","types","_updateItem","_removeItem","_addItem","_this6","groupOptions","_this7","_orderNestedGroups","equalArray","getOrderedNestedGroups","nestedGroupIds","nestedGroup","_this8","itemFromTarget","itemProps","selectedItem","initialX","_cloneItemData","ctrlKey","metaKey","_onDragStartAddItem","baseGroupIndex","_getGroupIndex","itemsToDrag","getSelection","groupIndex","groupOffset","_this9","frameRect","groupFromTarget","newItem","containerRect","setPosition","domRootOffsetLeft","updateGroupAllowed","newGroupBase","initialStart","initial","_this10","newOffset","oldGroup","_this11","_this12","toggleGroupShowNested","nestingGroup","fullNestedGroups","nextLevel","node","classList","toggle","toggleGroupDragClassName","originalOrder","movingUp","targetGroup","draggedGroupHeight","targetGroupHeight","draggedGroup","newOrder","origOrder","draggedId","numGroups","curPos","orgOffset","slippedPosition","switchGroup","shouldBeGroup","switchGroupId","dataset","oldSelection","setSelection","newSelection","itemFromRelatedTarget","getTitle","setText","setPopupTimer","newItemData","itemGroup","lastSelectedGroup","multiselectPerGroup","_getItemRange","_item","filteredSelection","_this13","cur","itemFromElement","relatedTarget","foregroundRect","getClusters","_updateClusters","newClustersIds","clustersToUnselect","selectionChanged","selectedIdx","allOptions","errorFound","printStyle","Validator","referenceOptions","subObject","usedOptions","__any__","referenceOption","is_object","refOptionObj","__type__","checkFields","getSuggestion","message","printLocation","optionType","refOptionType","print","copyAndExtendArray","nodeType","localSearch","findInOptions","globalSearch","indexMatch","closestMatch","recursive","closestMatchPath","lowerCaseOption","op","levenshteinDistance","copyArray","matrix","bool","enabled","throttleRedraw","any","onInitialDrawComplete","loadingScreenTemplate","configureOptions","groupsDraggable","fill","endPos","htmlColors","black","navy","darkblue","mediumblue","blue","darkgreen","green","teal","darkcyan","deepskyblue","darkturquoise","mediumspringgreen","lime","springgreen","aqua","cyan","midnightblue","dodgerblue","lightseagreen","forestgreen","seagreen","darkslategray","limegreen","mediumseagreen","turquoise","royalblue","steelblue","darkslateblue","mediumturquoise","indigo","darkolivegreen","cadetblue","cornflowerblue","mediumaquamarine","dimgray","slateblue","olivedrab","slategray","lightslategray","mediumslateblue","lawngreen","chartreuse","aquamarine","maroon","purple","olive","gray","skyblue","lightskyblue","blueviolet","darkred","darkmagenta","saddlebrown","darkseagreen","lightgreen","mediumpurple","darkviolet","palegreen","darkorchid","yellowgreen","sienna","brown","darkgray","lightblue","greenyellow","paleturquoise","lightsteelblue","powderblue","firebrick","darkgoldenrod","mediumorchid","rosybrown","darkkhaki","silver","mediumvioletred","indianred","peru","chocolate","tan","lightgrey","palevioletred","thistle","orchid","goldenrod","crimson","gainsboro","plum","burlywood","lightcyan","lavender","darksalmon","violet","palegoldenrod","lightcoral","khaki","aliceblue","honeydew","azure","sandybrown","wheat","beige","whitesmoke","mintcream","ghostwhite","salmon","antiquewhite","linen","lightgoldenrodyellow","oldlace","red","fuchsia","magenta","deeppink","orangered","tomato","hotpink","coral","darkorange","lightsalmon","orange","lightpink","pink","gold","peachpuff","navajowhite","moccasin","bisque","mistyrose","blanchedalmond","papayawhip","lavenderblush","seashell","cornsilk","lemonchiffon","floralwhite","snow","yellow","lightyellow","ivory","white","ColorPicker","pixelRatio","generated","centerCoordinates","r","color","g","hueCircle","initialColor","previousColor","applied","updateCallback","closeCallback","_bindHammer","_setSize","setInitial","rgba","htmlColor","_isColorString","isValidRGB","rgbaArray","isValidRGBA","isValidHex","rgbObj","hexToRGB","alpha","_setColor","_generateHueCircle","storePrevious","_hide","_updatePicker","setColor","alert","hsv","RGBToHSV","angleConvert","PI","radius","sin","cos","colorPickerSelector","v","HSVToRGB","ctx","colorPickerCanvas","getContext","pixelRation","devicePixelRatio","webkitBackingStorePixelRatio","mozBackingStorePixelRatio","msBackingStorePixelRatio","oBackingStorePixelRatio","backingStorePixelRatio","setTransform","clearRect","putImageData","fillStyle","circle","brightnessRange","opacityRange","initialColorDiv","backgroundColor","newColorDiv","colorPickerDiv","noCanvas","fontWeight","opacityDiv","brightnessDiv","arrowDiv","onchange","_setOpacity","oninput","_setBrightness","brightnessLabel","opacityLabel","cancelButton","onclick","applyButton","_apply","saveButton","_save","loadButton","_loadLast","pinch","_moveSelector","hue","sat","rgb","sfac","fillRect","strokeStyle","stroke","getImageData","centerY","centerX","angle","atan2","sqrt","newLeft","Configurator","parentModule","defaultContainer","changedOptions","allowCreation","initialized","popupCounter","showButton","moduleOptions","domElements","popupDiv","popupLimit","popupHistory","colorPicker","_removePopup","_clean","counter","_handleObject","_makeItem","_makeHeader","_makeButton","_push","_showPopupIfNeeded","div","objectLabel","selectedValue","_makeLabel","defaultValue","popupString","popupValue","_setupPopup","generateButton","_printOptions","onmouseover","onmouseout","optionsContainer","hideTimeout","deleteTimeout","opacity","checkbox","checked","defaultColor","_showColorPicker","insertTo","setUpdateCallback","colorString","setCloseCallback","checkOnly","visibleInSet","subObj","newPath","_getValue","_handleArray","_makeTextInput","_makeCheckbox","draw","physics","solver","enabledPath","enabledValue","_makeColorField","_makeDropdown","_makeRange","_constructOptions","optionsObj","getOptions","Timeline","itemsDone","_assertThisInitialized","SyntaxError","forthArgument","directionFromDom","domNode","parentElement","loadingScreenFragment","_toScreen","toGlobalScreen","_toGlobalScreen","_toTime","toGlobalTime","_toGlobalTime","oncontextmenu","PointerEvent","onpointerdown","onpointermove","onpointerup","onmousemove","onmousedown","onmouseup","initialFitDone","getItemRange","fit","_onFit","validate","newDataSet","startPos","initialVerticalScroll","setFinalVerticalPosition","finalVerticalScroll","getItemVerticalScroll","shouldScroll","itemTop","scrollOffset","middle","willDraw","minItem","maxItem","factor","startSide","endSide","getStart","getEnd","getWidthRight","getWidthLeft","lhs","rhs","customTimeFromTarget","getTarget","what","hasParent","pageX","pageY","timeline","itemsetHeight","currentScrollHeight","targetOffset","DataScale","autoScaleStart","autoScaleEnd","zeroAlign","formattingFunction","majorSteps","minorSteps","customLines","minorStepIdx","magnitudefactor","determineScale","rounded","minimumStepValue","orderOfMagnitude","LN10","solutionFound","majorStep","convertValue","returnValue","toPrecision","bottomOffset","major","is_major","formatValue","oldStepIdx","oldStart","oldEnd","increaseMagnitude","decreaseMagnitude","otherZero","otherStep","newRange","myOriginalZero","majorOffset","getFirstMajor","zeroOffset","pixels","DataAxis","svg","linegraphOptions","icons","majorLinesOffset","minorLinesOffset","labelOffsetX","labelOffsetY","iconWidth","alignZeros","linegraphSVG","DOMelements","labels","conversionFactor","stepPixels","zeroCrossing","amountOfSteps","master","masterAxis","svgElements","iconsRemoved","amountOfGroups","_redrawLabels","framework","lineContainer","graphOptions","DOMutil","iconOffset","groupArray","getLegend","iconHeight","activeGroups","backgroundHorizontalOffsetWidth","_redrawGroupIcons","_cleanupIcons","_redrawTitle","customRange","followScale","maxLabelSize","getLines","_redrawLabel","_redrawLine","titleWidth","titleCharHeight","screenToValue","characterHeight","textAlign","largestWidth","textMinor","textMajor","textTitle","measureCharTitle","titleCharWidth","Points","getGroupTemplate","callbackResult","Bargraph","Line","GraphGroup","groupsUsingDefaultStyles","selectiveBridgeObject","usingDefaultStyle","zeroPosition","Legend","side","iconSize","iconSpacing","getCallback","screen_x","screen_y","drawIcon","fillHeight","outline","barWidth","barChart","bar1Height","bar2Height","processedGroupData","coreDistance","drawData","combinedData","intersections","barPoints","screen_end","_getDataIntersections","heightOffset","_getSafeDrawData","amount","resolved","excludeFromStacking","accumulatedNegative","accumulatedPositive","sideBySide","dataWidth","pointData","getStackedYRange","groupRanges","groupLabel","_getStackedYRange","yAxisOrientation","yMin","yMax","xpos","calcPath","interpolation","_catmullRom","_linear","fillPath","shaded","drawShading","pathArray","subPathArray","dFill","svgHeight","zero","serializePath","inverse","_catmullRomUniform","p0","bp1","bp2","d1","d2","d3","N","d3powA","d2powA","d3pow2A","d2pow2A","d1pow2A","d1powA","insertSort","getItems","setZeroPosition","mergeOptions","parametrization","Lines","Bars","icon","getYRange","addGroup","excludeFromLegend","removeGroup","textArea","scrollableHeight","drawLegendIcons","paddingTop","LineGraph","defaultGroup","sampling","graphHeight","dataAxis","legend","abortedGraphUpdate","updateSVGheight","updateSVGheightOnResize","forceGraphUpdate","lastStart","yAxisLeft","yAxisRight","legendLeft","legendRight","_removeGroup","_updateAllGroupData","_updateGroup","groupsContent","idMap","groupCounts","existingItemsMap","existing_items","newLength","extended","bridgeObject","orginalY","_updateGraph","_getSortedGroupIds","grouplist","zIndex","az","bz","minDate","maxDate","_getRelevantData","_applySampling","_convertXcoordinates","_getYRanges","_updateYAxis","below","_stack","_convertYcoordinates","paths","subGroupId","subData","dx","dy","subPrevPoint","subNextPoint","dateComparator","binarySearchValue","dataContainer","increment","amountOfPoints","pointsPerPixel","sampledData","combinedDataLeft","combinedDataRight","minVal","maxVal","yAxisLeftUsed","yAxisRightUsed","minLeft","minRight","maxLeft","maxRight","ignore","_toggleAxisVisiblity","drawIcons","tempGroups","axisUsed","datapoints","Graph2d","linegraph","initialLoad","isGroupVisible","getAbsoluteLeft","getAbsoluteTop","defaultLanguage"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;8jBAGmEA,UAG1D,eAEDC,EA6GAC,WA3GKC,WACEF,EAAaG,MAAM,KAAMC,oBAS3BC,EAAQC,UACNA,aAAiBC,OAAmD,mBAA1CC,OAAOC,UAAUC,SAASC,KAAKL,YAG3DM,EAASN,UAGE,MAATA,GAA2D,oBAA1CE,OAAOC,UAAUC,SAASC,KAAKL,YAiBlDO,EAAYP,eACA,IAAVA,WAGFQ,EAASR,SACU,iBAAVA,GAAgE,oBAA1CE,OAAOC,UAAUC,SAASC,KAAKL,YAG9DS,EAAOT,UACLA,aAAiBU,MAAkD,kBAA1CR,OAAOC,UAAUC,SAASC,KAAKL,YAG1DW,EAAIC,EAAKC,OACAC,EAAVC,EAAM,OACLD,EAAI,EAAGA,EAAIF,EAAII,SAAUF,EAC1BC,EAAIE,KAAKJ,EAAGD,EAAIE,GAAIA,WAEjBC,WAGFG,EAAWC,EAAGC,UACZlB,OAAOC,UAAUkB,eAAehB,KAAKc,EAAGC,YAG1CE,EAAOH,EAAGC,OACV,IAAIN,KAAKM,EACNF,EAAWE,EAAGN,KACdK,EAAEL,GAAKM,EAAEN,WAIbI,EAAWE,EAAG,cACdD,EAAEf,SAAWgB,EAAEhB,UAGfc,EAAWE,EAAG,aACdD,EAAEI,QAAUH,EAAEG,SAGXJ,WAGFK,EAAWxB,EAAOyB,EAAQC,EAAQC,UAChCC,GAAiB5B,EAAOyB,EAAQC,EAAQC,GAAQ,GAAME,eAuBxDC,EAAgBC,UACR,MAATA,EAAEC,MACFD,EAAEC,IApBC,CACHC,OAAkB,EAClBC,aAAkB,GAClBC,YAAkB,GAClBC,UAAmB,EACnBC,cAAkB,EAClBC,WAAkB,EAClBC,aAAkB,KAClBC,eAAkB,EAClBC,iBAAkB,EAClBC,KAAkB,EAClBC,gBAAkB,GAClBC,SAAkB,KAClBC,SAAkB,EAClBC,iBAAkB,IAQff,EAAEC,aAqBJe,EAAQhB,MACK,MAAdA,EAAEiB,SAAkB,KAChBC,EAAQnB,EAAgBC,GACxBmB,EAAcvD,EAAKU,KAAK4C,EAAMN,iBAAiB,SAAU7B,UAC7C,MAALA,KAEPqC,GAAcC,MAAMrB,EAAEsB,GAAGC,YACzBL,EAAMb,SAAW,IAChBa,EAAMhB,QACNgB,EAAMV,eACNU,EAAMM,iBACNN,EAAMH,kBACNG,EAAMX,YACNW,EAAMT,gBACNS,EAAMR,mBACLQ,EAAML,UAAaK,EAAML,UAAYM,MAEvCnB,EAAEyB,UACFL,EAAaA,GACe,IAAxBF,EAAMZ,eACwB,IAA9BY,EAAMf,aAAalB,aACDyC,IAAlBR,EAAMS,SAGS,MAAnBxD,OAAOyD,UAAqBzD,OAAOyD,SAAS5B,UAIrCoB,EAHPpB,EAAEiB,SAAWG,SAMdpB,EAAEiB,kBAGJY,EAAeX,OAChBlB,EAAIP,EAAUqC,YACL,MAATZ,EACA3B,EAAOQ,EAAgBC,GAAIkB,GAG3BnB,EAAgBC,GAAGU,iBAAkB,EAGlCV,EA3DPpC,EADAM,MAAME,UAAUR,KACTM,MAAME,UAAUR,KAEhB,SAAUmE,WACTC,EAAI7D,OAAO8D,MACXC,EAAMF,EAAE/C,SAAW,EAEdF,EAAI,EAAGA,EAAImD,EAAKnD,OACjBA,KAAKiD,GAAKD,EAAIzD,KAAK2D,KAAMD,EAAEjD,GAAIA,EAAGiD,UAC3B,SAIR,OAoDXG,EAAmBtE,EAAMsE,iBAAmB,YAEvCC,EAAWC,EAAIC,OAChBvD,EAAGwD,EAAMC,KAERhE,EAAY8D,EAAKG,oBAClBJ,EAAGI,iBAAmBH,EAAKG,kBAE1BjE,EAAY8D,EAAKI,MAClBL,EAAGK,GAAKJ,EAAKI,IAEZlE,EAAY8D,EAAKK,MAClBN,EAAGM,GAAKL,EAAKK,IAEZnE,EAAY8D,EAAKM,MAClBP,EAAGO,GAAKN,EAAKM,IAEZpE,EAAY8D,EAAKb,WAClBY,EAAGZ,QAAUa,EAAKb,SAEjBjD,EAAY8D,EAAKO,QAClBR,EAAGQ,KAAOP,EAAKO,MAEdrE,EAAY8D,EAAKQ,UAClBT,EAAGS,OAASR,EAAKQ,QAEhBtE,EAAY8D,EAAKS,WAClBV,EAAGU,QAAUT,EAAKS,SAEjBvE,EAAY8D,EAAKrC,OAClBoC,EAAGpC,IAAMF,EAAgBuC,IAExB9D,EAAY8D,EAAKU,WAClBX,EAAGW,QAAUV,EAAKU,SAGlBb,EAAiBlD,OAAS,MACrBF,EAAI,EAAGA,EAAIoD,EAAiBlD,OAAQF,IAGhCP,EADLgE,EAAMF,EADNC,EAAOJ,EAAiBpD,OAGpBsD,EAAGE,GAAQC,UAKhBH,MAGPY,GAAmB,WAGdC,EAAOC,GACZf,EAAWH,KAAMkB,QACZ7B,GAAK,IAAI3C,KAAkB,MAAbwE,EAAO7B,GAAa6B,EAAO7B,GAAGC,UAAYO,KACxDG,KAAKjB,iBACDM,GAAK,IAAI3C,KAAKmD,OAIE,IAArBmB,IACAA,GAAmB,EACnBpF,EAAMuF,aAAanB,MACnBgB,GAAmB,YAIlBI,EAAUC,UACRA,aAAeJ,GAAkB,MAAPI,GAAuC,MAAxBA,EAAIb,0BAG/Cc,EAAUC,UACXA,EAAS,EAEFC,KAAKC,KAAKF,IAAW,EAErBC,KAAKE,MAAMH,YAIjBI,EAAMC,OACPC,GAAiBD,EACjBE,EAAQ,SAEU,IAAlBD,GAAuBE,SAASF,KAChCC,EAAQR,EAASO,IAGdC,WAIFE,EAAcC,EAAQC,EAAQC,OAI/BrF,EAHAmD,EAAMuB,KAAKY,IAAIH,EAAOjF,OAAQkF,EAAOlF,QACrCqF,EAAab,KAAKc,IAAIL,EAAOjF,OAASkF,EAAOlF,QAC7CuF,EAAQ,MAEPzF,EAAI,EAAGA,EAAImD,EAAKnD,KACZqF,GAAeF,EAAOnF,KAAOoF,EAAOpF,KACnCqF,GAAeR,EAAMM,EAAOnF,MAAQ6E,EAAMO,EAAOpF,MACnDyF,WAGDA,EAAQF,WAGVG,EAAKC,IACgC,IAAtC7G,EAAM8G,6BACmB,oBAAbC,SAA6BA,QAAQH,MACjDG,QAAQH,KAAK,wBAA0BC,YAItCG,EAAUH,EAAK5F,OAChBgG,GAAY,SAETvF,GAAO,cACsB,MAA5B1B,EAAMkH,oBACNlH,EAAMkH,mBAAmB,KAAML,GAE/BI,EAAW,SAEPE,EADAC,EAAO,GAEFlG,EAAI,EAAGA,EAAIhB,UAAUkB,OAAQF,IAAK,IACvCiG,EAAM,GACsB,iBAAjBjH,UAAUgB,GAAiB,KAE7B,IAAImG,KADTF,GAAO,MAAQjG,EAAI,KACHhB,UAAU,GACtBiH,GAAOE,EAAM,KAAOnH,UAAU,GAAGmH,GAAO,KAE5CF,EAAMA,EAAIG,MAAM,GAAI,QAEpBH,EAAMjH,UAAUgB,GAEpBkG,EAAK/F,KAAK8F,GAEdP,EAAKC,EAAM,gBAAkBxG,MAAME,UAAU+G,MAAM7G,KAAK2G,GAAMG,KAAK,IAAM,MAAQ,IAAIC,OAASC,OAC9FR,GAAY,SAEThG,EAAGhB,MAAMmE,KAAMlE,aACvBe,OAyEHyG,EAtEAC,EAAe,YAEVC,EAAgBC,EAAMhB,GACK,MAA5B7G,EAAMkH,oBACNlH,EAAMkH,mBAAmBW,EAAMhB,GAE9Bc,EAAaE,KACdjB,EAAKC,GACLc,EAAaE,IAAQ,YAOpBC,EAAW1H,UACTA,aAAiB2H,UAAsD,sBAA1CzH,OAAOC,UAAUC,SAASC,KAAKL,YAsB9D4H,EAAaC,EAAcC,OACIxD,EAAhCvD,EAAMO,EAAO,GAAIuG,OAChBvD,KAAQwD,EACL5G,EAAW4G,EAAaxD,KACpBhE,EAASuH,EAAavD,KAAUhE,EAASwH,EAAYxD,KACrDvD,EAAIuD,GAAQ,GACZhD,EAAOP,EAAIuD,GAAOuD,EAAavD,IAC/BhD,EAAOP,EAAIuD,GAAOwD,EAAYxD,KACF,MAArBwD,EAAYxD,GACnBvD,EAAIuD,GAAQwD,EAAYxD,UAEjBvD,EAAIuD,QAIlBA,KAAQuD,EACL3G,EAAW2G,EAAcvD,KACpBpD,EAAW4G,EAAaxD,IACzBhE,EAASuH,EAAavD,MAE1BvD,EAAIuD,GAAQhD,EAAO,GAAIP,EAAIuD,YAG5BvD,WAGFgH,EAAO7C,GACE,MAAVA,QACK8C,IAAI9C,GAtDjBtF,EAAM8G,6BAA8B,EACpC9G,EAAMkH,mBAAqB,KA4DvBQ,EADApH,OAAOoH,KACApH,OAAOoH,KAEP,SAAUjC,OACTvE,EAAGC,EAAM,OACRD,KAAKuE,EACFnE,EAAWmE,EAAKvE,IAChBC,EAAIE,KAAKH,UAGVC,OAoFXkH,EAAU,YAELC,EAAcC,EAAMC,OACrBC,EAAYF,EAAKG,cACrBL,EAAQI,GAAaJ,EAAQI,EAAY,KAAOJ,EAAQG,GAAaD,WAGhEI,EAAeC,SACI,iBAAVA,EAAqBP,EAAQO,IAAUP,EAAQO,EAAMF,oBAAiB7E,WAG/EgF,EAAqBC,OAEtBC,EACArE,EAFAsE,EAAkB,OAIjBtE,KAAQoE,EACLxH,EAAWwH,EAAapE,KACxBqE,EAAiBJ,EAAejE,MAE5BsE,EAAgBD,GAAkBD,EAAYpE,WAKnDsE,MAGPC,EAAa,YAERC,EAAgBX,EAAMY,GAC3BF,EAAWV,GAAQY,WAcdC,EAASzD,EAAQ0D,EAAcC,OAChCC,EAAY,GAAK3D,KAAKc,IAAIf,GAC1B6D,EAAcH,EAAeE,EAAUnI,cAChCuE,GAAU,EACL2D,EAAY,IAAM,GAAM,KACpC1D,KAAK6D,IAAI,GAAI7D,KAAK8D,IAAI,EAAGF,IAAchJ,WAAWmJ,OAAO,GAAKJ,MAGlEK,EAAmB,uLAEnBC,EAAwB,6CAExBC,EAAkB,GAElBC,EAAuB,YAMlBC,EAAgBC,EAAOC,EAAQC,EAASC,OACzCC,EAAOD,EACa,iBAAbA,IACPC,EAAO,kBACIjG,KAAKgG,OAGhBH,IACAF,EAAqBE,GAASI,GAE9BH,IACAH,EAAqBG,EAAO,IAAM,kBACvBd,EAASiB,EAAKpK,MAAMmE,KAAMlE,WAAYgK,EAAO,GAAIA,EAAO,MAGnEC,IACAJ,EAAqBI,GAAW,kBACrB/F,KAAKkG,aAAaH,QAAQE,EAAKpK,MAAMmE,KAAMlE,WAAY+J,cAiCjEM,EAAapI,EAAGN,UAChBM,EAAEgB,WAIPtB,EAAS2I,EAAa3I,EAAQM,EAAEmI,cAChCR,EAAgBjI,GAAUiI,EAAgBjI,aA3BlBA,OACoBX,EAAGE,EARnBhB,EAQxBqK,EAAQ5I,EAAO6I,MAAMd,OAEpB1I,EAAI,EAAGE,EAASqJ,EAAMrJ,OAAQF,EAAIE,EAAQF,IACvC6I,EAAqBU,EAAMvJ,IAC3BuJ,EAAMvJ,GAAK6I,EAAqBU,EAAMvJ,IAEtCuJ,EAAMvJ,IAdcd,EAccqK,EAAMvJ,IAbtCwJ,MAAM,YACLtK,EAAMuK,QAAQ,WAAY,IAE9BvK,EAAMuK,QAAQ,MAAO,WAcrB,SAAUC,OACI1J,EAAb2J,EAAS,OACR3J,EAAI,EAAGA,EAAIE,EAAQF,IACpB2J,GAAU/C,EAAW2C,EAAMvJ,IAAMuJ,EAAMvJ,GAAGT,KAAKmK,EAAK/I,GAAU4I,EAAMvJ,UAEjE2J,GAW0CC,CAAmBjJ,GAEjEiI,EAAgBjI,GAAQM,IANpBA,EAAEmI,aAAaS,uBASrBP,EAAa3I,EAAQC,OACtBZ,EAAI,WAEC8J,EAA4B5K,UAC1B0B,EAAOmJ,eAAe7K,IAAUA,MAG3CyJ,EAAsBqB,UAAY,EAC3BhK,GAAK,GAAK2I,EAAsBsB,KAAKtJ,IACxCA,EAASA,EAAO8I,QAAQd,EAAuBmB,GAC/CnB,EAAsBqB,UAAY,EAClChK,GAAK,SAGFW,MAGPuJ,EAAiB,KACjBC,EAAiB,OACjBC,EAAiB,QACjBC,EAAiB,QACjBC,EAAiB,aACjBC,EAAiB,QACjBC,EAAiB,YACjBC,GAAiB,gBACjBC,GAAiB,UACjBC,GAAiB,UACjBC,GAAiB,eAEjBC,GAAiB,MACjBC,GAAiB,WAEjBC,GAAiB,qBACjBC,GAAmB,0BAMnBC,GAAY,wJAEZC,GAAU,YAELC,GAAepC,EAAOqC,EAAOC,GAClCH,GAAQnC,GAASnC,EAAWwE,GAASA,EAAQ,SAAUE,EAAUlC,UACrDkC,GAAYD,EAAeA,EAAcD,YAIhDG,GAAuBxC,EAAO3E,UAC9BhE,EAAW8K,GAASnC,GAIlBmC,GAAQnC,GAAO3E,EAAO1B,QAAS0B,EAAOH,SAHlC,IAAIuH,OAQRC,GAR8B1C,EAQhBU,QAAQ,KAAM,IAAIA,QAAQ,uCAAuC,SAAUiC,EAASC,EAAIC,EAAIC,EAAIC,UAC1GH,GAAMC,GAAMC,GAAMC,gBAIxBL,GAAYM,UACVA,EAAEtC,QAAQ,yBAA0B,YAG3CuC,GAAS,YAEJC,GAAelD,EAAOG,OACvBlJ,EAAGmJ,EAAOD,MACO,iBAAVH,IACPA,EAAQ,CAACA,IAETrJ,EAASwJ,KACTC,EAAO,SAAUjK,EAAOqK,GACpBA,EAAML,GAAYrE,EAAM3F,KAG3Bc,EAAI,EAAGA,EAAI+I,EAAM7I,OAAQF,IAC1BgM,GAAOjD,EAAM/I,IAAMmJ,WAIlB+C,GAAmBnD,EAAOG,GAC/B+C,GAAclD,GAAO,SAAU7J,EAAOqK,EAAOnF,EAAQ2E,GACjD3E,EAAO+H,GAAK/H,EAAO+H,IAAM,GACzBjD,EAAShK,EAAOkF,EAAO+H,GAAI/H,EAAQ2E,eAIlCqD,GAAwBrD,EAAO7J,EAAOkF,GAC9B,MAATlF,GAAiBkB,EAAW4L,GAAQjD,IACpCiD,GAAOjD,GAAO7J,EAAOkF,EAAOiI,GAAIjI,EAAQ2E,YA0DvCuD,GAAWC,UACTC,GAAWD,GAAQ,IAAM,aAG3BC,GAAWD,UACRA,EAAO,GAAM,GAAKA,EAAO,KAAQ,GAAMA,EAAO,KAAQ,EA/ClEzD,EAAe,IAAK,EAAG,GAAG,eAClB2D,EAAIvJ,KAAKqJ,cACNE,GAAK,KAAO,GAAKA,EAAI,IAAMA,KAGtC3D,EAAe,EAAG,CAAC,KAAM,GAAI,GAAG,kBACrB5F,KAAKqJ,OAAS,OAGzBzD,EAAe,EAAG,CAAC,OAAU,GAAU,EAAG,QAC1CA,EAAe,EAAG,CAAC,QAAU,GAAU,EAAG,QAC1CA,EAAe,EAAG,CAAC,SAAU,GAAG,GAAO,EAAG,QAI1C1B,EAAa,OAAQ,KAIrBY,EAAgB,OAAQ,GAIxBmD,GAAc,IAAUL,IACxBK,GAAc,KAAUZ,EAAWJ,GACnCgB,GAAc,OAAUR,GAAWN,GACnCc,GAAc,QAAUP,GAAWN,GACnCa,GAAc,SAAUP,GAAWN,GAEnC2B,GAAc,CAAC,QAAS,UAzCb,GA0CXA,GAAc,QAAQ,SAAU/M,EAAOqK,GACnCA,EA3CO,GA2CwB,IAAjBrK,EAAMgB,OAAepB,EAAM4N,kBAAkBxN,GAAS2F,EAAM3F,MAE9E+M,GAAc,MAAM,SAAU/M,EAAOqK,GACjCA,EA9CO,GA8COzK,EAAM4N,kBAAkBxN,MAE1C+M,GAAc,KAAK,SAAU/M,EAAOqK,GAChCA,EAjDO,GAiDOoD,SAASzN,EAAO,OAelCJ,EAAM4N,kBAAoB,SAAUxN,UACzB2F,EAAM3F,IAAU2F,EAAM3F,GAAS,GAAK,KAAO,UAsElD0N,GAjEAC,GAAaC,GAAW,YAAY,YAM/BA,GAAYzF,EAAM0F,UAChB,SAAU/H,UACA,MAATA,GACAgI,GAAM9J,KAAMmE,EAAMrC,GAClBlG,EAAMuF,aAAanB,KAAM6J,GAClB7J,MAEA+J,GAAI/J,KAAMmE,aAKpB4F,GAAKvD,EAAKrC,UACRqC,EAAIzH,UACPyH,EAAInH,GAAG,OAASmH,EAAI3F,OAAS,MAAQ,IAAMsD,KAAUtE,aAGpDiK,GAAOtD,EAAKrC,EAAMrC,GACnB0E,EAAIzH,YAAcK,MAAM0C,KACX,aAATqC,GAAuBmF,GAAW9C,EAAI6C,SAA2B,IAAhB7C,EAAIwD,SAAgC,KAAfxD,EAAIyD,OAC1EzD,EAAInH,GAAG,OAASmH,EAAI3F,OAAS,MAAQ,IAAMsD,GAAMrC,EAAO0E,EAAIwD,QAASE,GAAYpI,EAAO0E,EAAIwD,UAG5FxD,EAAInH,GAAG,OAASmH,EAAI3F,OAAS,MAAQ,IAAMsD,GAAMrC,aAqDpDoI,GAAYb,EAAMW,MACnB5K,MAAMiK,IAASjK,MAAM4K,UACdnK,QAvBCsK,EAyBRC,GAAeJ,GAzBPG,EAyBc,IAxBRA,GAAKA,SAyBvBd,IAASW,EAAQI,GAAY,GACT,IAAbA,EAAkBd,GAAWD,GAAQ,GAAK,GAAO,GAAKe,EAAW,EAAI,EApB5EV,GADAzN,MAAME,UAAUuN,QACNzN,MAAME,UAAUuN,QAEhB,SAAUW,OAEZvN,MACCA,EAAI,EAAGA,EAAIkD,KAAKhD,SAAUF,KACvBkD,KAAKlD,KAAOuN,SACLvN,SAGP,GAehB8I,EAAe,IAAK,CAAC,KAAM,GAAI,MAAM,kBAC1B5F,KAAKgK,QAAU,KAG1BpE,EAAe,MAAO,EAAG,GAAG,SAAUnI,UAC3BuC,KAAKkG,aAAaoE,YAAYtK,KAAMvC,MAG/CmI,EAAe,OAAQ,EAAG,GAAG,SAAUnI,UAC5BuC,KAAKkG,aAAaqE,OAAOvK,KAAMvC,MAK1CyG,EAAa,QAAS,KAItBY,EAAgB,QAAS,GAIzBmD,GAAc,IAAQZ,GACtBY,GAAc,KAAQZ,EAAWJ,GACjCgB,GAAc,OAAQ,SAAUG,EAAU1K,UAC/BA,EAAO8M,iBAAiBpC,MAEnCH,GAAc,QAAQ,SAAUG,EAAU1K,UAC/BA,EAAO+M,YAAYrC,MAG9BW,GAAc,CAAC,IAAK,OAAO,SAAU/M,EAAOqK,GACxCA,EAlMQ,GAkMO1E,EAAM3F,GAAS,KAGlC+M,GAAc,CAAC,MAAO,SAAS,SAAU/M,EAAOqK,EAAOnF,EAAQ2E,OACvDmE,EAAQ9I,EAAOH,QAAQ2J,YAAY1O,EAAO6J,EAAO3E,EAAO1B,SAE/C,MAATwK,EACA3D,EAzMI,GAyMW2D,EAEflM,EAAgBoD,GAAQ3C,aAAevC,SAM3C2O,GAAmB,gCACnBC,GAAsB,wFAAwFC,MAAM,KAUpHC,GAA2B,kDAAkDD,MAAM,cAU9EE,GAAkBC,EAAWvN,EAAQE,OACtCb,EAAGmO,EAAIzE,EAAK0E,EAAMF,EAAUG,wBAC3BnL,KAAKoL,sBAEDA,aAAe,QACfC,iBAAmB,QACnBC,kBAAoB,GACpBxO,EAAI,EAAGA,EAAI,KAAMA,EAClB0J,EAAMhJ,EAAU,CAAC,IAAMV,SAClBwO,kBAAkBxO,GAAKkD,KAAKsK,YAAY9D,EAAK,IAAI2E,yBACjDE,iBAAiBvO,GAAKkD,KAAKuK,OAAO/D,EAAK,IAAI2E,2BAIpDxN,EACe,QAAXF,GAEe,KADfwN,EAAKvB,GAAQrN,KAAK2D,KAAKsL,kBAAmBJ,IACvBD,EAAK,MAGT,KADfA,EAAKvB,GAAQrN,KAAK2D,KAAKqL,iBAAkBH,IACtBD,EAAK,KAGb,QAAXxN,GAEY,KADZwN,EAAKvB,GAAQrN,KAAK2D,KAAKsL,kBAAmBJ,MAK3B,KADfD,EAAKvB,GAAQrN,KAAK2D,KAAKqL,iBAAkBH,IAF9BD,EAGa,MAGZ,KADZA,EAAKvB,GAAQrN,KAAK2D,KAAKqL,iBAAkBH,MAK1B,KADfD,EAAKvB,GAAQrN,KAAK2D,KAAKsL,kBAAmBJ,IAF/BD,EAGa,cA6C3BM,GAAU/E,EAAK1E,OAChB0J,MAEChF,EAAIzH,iBAEEyH,KAGU,iBAAV1E,KACH,QAAQiF,KAAKjF,GACbA,EAAQH,EAAMG,YAITtF,EAFLsF,EAAQ0E,EAAIN,aAAawE,YAAY5I,WAG1B0E,SAKnBgF,EAAahK,KAAKY,IAAIoE,EAAIyD,OAAQC,GAAY1D,EAAI6C,OAAQvH,IAC1D0E,EAAInH,GAAG,OAASmH,EAAI3F,OAAS,MAAQ,IAAM,SAASiB,EAAO0J,GACpDhF,WAGFiF,GAAa3J,UACL,MAATA,GACAyJ,GAASvL,KAAM8B,GACflG,EAAMuF,aAAanB,MAAM,GAClBA,MAEA+J,GAAI/J,KAAM,aAQrB0L,GAA0B3D,GAoB1B4D,GAAqB5D,YAoBhB6D,cACIC,EAAU1O,EAAGC,UACXA,EAAEJ,OAASG,EAAEH,WAIpBF,EAAG0J,EADHsF,EAAc,GAAIC,EAAa,GAAIC,EAAc,OAEhDlP,EAAI,EAAGA,EAAI,GAAIA,IAEhB0J,EAAMhJ,EAAU,CAAC,IAAMV,IACvBgP,EAAY7O,KAAK+C,KAAKsK,YAAY9D,EAAK,KACvCuF,EAAW9O,KAAK+C,KAAKuK,OAAO/D,EAAK,KACjCwF,EAAY/O,KAAK+C,KAAKuK,OAAO/D,EAAK,KAClCwF,EAAY/O,KAAK+C,KAAKsK,YAAY9D,EAAK,SAI3CsF,EAAYG,KAAKJ,GACjBE,EAAWE,KAAKJ,GAChBG,EAAYC,KAAKJ,GACZ/O,EAAI,EAAGA,EAAI,GAAIA,IAChBgP,EAAYhP,GAAKyL,GAAYuD,EAAYhP,IACzCiP,EAAWjP,GAAKyL,GAAYwD,EAAWjP,QAEtCA,EAAI,EAAGA,EAAI,GAAIA,IAChBkP,EAAYlP,GAAKyL,GAAYyD,EAAYlP,SAGxCoP,aAAe,IAAI5D,OAAO,KAAO0D,EAAY7I,KAAK,KAAO,IAAK,UAC9DgJ,kBAAoBnM,KAAKkM,kBACzBE,mBAAqB,IAAI9D,OAAO,KAAOyD,EAAW5I,KAAK,KAAO,IAAK,UACnEkJ,wBAA0B,IAAI/D,OAAO,KAAOwD,EAAY3I,KAAK,KAAO,IAAK,cAGzEmJ,GAAY/C,EAAGxL,EAAGwO,EAAGC,EAAGC,EAAG5D,EAAG6D,OAG/BzC,SAEAV,EAAI,KAAOA,GAAK,GAEhBU,EAAO,IAAIvN,KAAK6M,EAAI,IAAKxL,EAAGwO,EAAGC,EAAGC,EAAG5D,EAAG6D,GACpC3K,SAASkI,EAAK0C,gBACd1C,EAAK2C,YAAYrD,IAGrBU,EAAO,IAAIvN,KAAK6M,EAAGxL,EAAGwO,EAAGC,EAAGC,EAAG5D,EAAG6D,GAG/BzC,WAGF4C,GAAetD,OAChBU,KAEAV,EAAI,KAAOA,GAAK,EAAG,KACfvG,EAAO/G,MAAME,UAAU+G,MAAM7G,KAAKP,WAEtCkH,EAAK,GAAKuG,EAAI,IACdU,EAAO,IAAIvN,KAAKA,KAAKoQ,IAAIjR,MAAM,KAAMmH,IACjCjB,SAASkI,EAAK8C,mBACd9C,EAAK+C,eAAezD,QAGxBU,EAAO,IAAIvN,KAAKA,KAAKoQ,IAAIjR,MAAM,KAAMC,mBAGlCmO,WAIFgD,GAAgB5D,EAAM6D,EAAKC,OAE5BC,EAAM,EAAIF,EAAMC,UAEP,EAAIN,GAAcxD,EAAM,EAAG+D,GAAKC,YAAcH,GAAO,EAElDE,EAAM,WAIjBE,GAAmBjE,EAAMkE,EAAMC,EAASN,EAAKC,OAI9CM,EAASC,EADTC,EAAY,EAAI,GAAKJ,EAAO,IAFZ,EAAIC,EAAUN,GAAO,EACxBD,GAAgB5D,EAAM6D,EAAKC,UAIxCQ,GAAa,EAEbD,EAAetE,GADfqE,EAAUpE,EAAO,GACoBsE,EAC9BA,EAAYvE,GAAWC,IAC9BoE,EAAUpE,EAAO,EACjBqE,EAAeC,EAAYvE,GAAWC,KAEtCoE,EAAUpE,EACVqE,EAAeC,GAGZ,CACHtE,KAAMoE,EACNE,UAAWD,YAIVE,GAAWpH,EAAK0G,EAAKC,OAGtBU,EAASJ,EAFTK,EAAab,GAAgBzG,EAAI6C,OAAQ6D,EAAKC,GAC9CI,EAAO/L,KAAKE,OAAO8E,EAAImH,YAAcG,EAAa,GAAK,GAAK,SAG5DP,EAAO,EAEPM,EAAUN,EAAOQ,GADjBN,EAAUjH,EAAI6C,OAAS,EACe6D,EAAKC,GACpCI,EAAOQ,GAAYvH,EAAI6C,OAAQ6D,EAAKC,IAC3CU,EAAUN,EAAOQ,GAAYvH,EAAI6C,OAAQ6D,EAAKC,GAC9CM,EAAUjH,EAAI6C,OAAS,IAEvBoE,EAAUjH,EAAI6C,OACdwE,EAAUN,GAGP,CACHA,KAAMM,EACNxE,KAAMoE,YAILM,GAAY1E,EAAM6D,EAAKC,OACxBW,EAAab,GAAgB5D,EAAM6D,EAAKC,GACxCa,EAAiBf,GAAgB5D,EAAO,EAAG6D,EAAKC,UAC5C/D,GAAWC,GAAQyE,EAAaE,GAAkB,WAoJrDC,GAAeC,EAAIC,UACjBD,EAAGhL,MAAMiL,EAAG,GAAGC,OAAOF,EAAGhL,MAAM,EAAGiL,IAhJ7CvI,EAAe,IAAK,CAAC,KAAM,GAAI,KAAM,QACrCA,EAAe,IAAK,CAAC,KAAM,GAAI,KAAM,WAIrC1B,EAAa,OAAQ,KACrBA,EAAa,UAAW,KAIxBY,EAAgB,OAAQ,GACxBA,EAAgB,UAAW,GAI3BmD,GAAc,IAAMZ,GACpBY,GAAc,KAAMZ,EAAWJ,GAC/BgB,GAAc,IAAMZ,GACpBY,GAAc,KAAMZ,EAAWJ,GAE/B+B,GAAkB,CAAC,IAAK,KAAM,IAAK,OAAO,SAAUhN,EAAOuR,EAAMrM,EAAQ2E,GACrE0H,EAAK1H,EAAMN,OAAO,EAAG,IAAM5D,EAAM3F,MAsCrC4J,EAAe,IAAK,EAAG,KAAM,OAE7BA,EAAe,KAAM,EAAG,GAAG,SAAUnI,UAC1BuC,KAAKkG,aAAamI,YAAYrO,KAAMvC,MAG/CmI,EAAe,MAAO,EAAG,GAAG,SAAUnI,UAC3BuC,KAAKkG,aAAaoI,cAActO,KAAMvC,MAGjDmI,EAAe,OAAQ,EAAG,GAAG,SAAUnI,UAC5BuC,KAAKkG,aAAaqI,SAASvO,KAAMvC,MAG5CmI,EAAe,IAAK,EAAG,EAAG,WAC1BA,EAAe,IAAK,EAAG,EAAG,cAI1B1B,EAAa,MAAO,KACpBA,EAAa,UAAW,KACxBA,EAAa,aAAc,KAG3BY,EAAgB,MAAO,IACvBA,EAAgB,UAAW,IAC3BA,EAAgB,aAAc,IAI9BmD,GAAc,IAAQZ,GACtBY,GAAc,IAAQZ,GACtBY,GAAc,IAAQZ,GACtBY,GAAc,MAAQ,SAAUG,EAAU1K,UAC/BA,EAAO8Q,iBAAiBpG,MAEnCH,GAAc,OAAS,SAAUG,EAAU1K,UAChCA,EAAO+Q,mBAAmBrG,MAErCH,GAAc,QAAU,SAAUG,EAAU1K,UACjCA,EAAOgR,cAActG,MAGhCY,GAAkB,CAAC,KAAM,MAAO,SAAS,SAAUhN,EAAOuR,EAAMrM,EAAQ2E,OAChE2H,EAAUtM,EAAOH,QAAQ4N,cAAc3S,EAAO6J,EAAO3E,EAAO1B,SAEjD,MAAXgO,EACAD,EAAKhB,EAAIiB,EAET1P,EAAgBoD,GAAQ3B,eAAiBvD,KAIjDgN,GAAkB,CAAC,IAAK,IAAK,MAAM,SAAUhN,EAAOuR,EAAMrM,EAAQ2E,GAC9D0H,EAAK1H,GAASlE,EAAM3F,UAkCpB4S,GAAwB,2DAA2D/D,MAAM,KAQzFgE,GAA6B,8BAA8BhE,MAAM,KAMjEiE,GAA2B,uBAAuBjE,MAAM,cAMnDkE,GAAoBC,EAAavR,EAAQE,OAC1Cb,EAAGmO,EAAIzE,EAAK0E,EAAM8D,EAAY7D,wBAC7BnL,KAAKiP,wBACDA,eAAiB,QACjBC,oBAAsB,QACtBC,kBAAoB,GAEpBrS,EAAI,EAAGA,EAAI,IAAKA,EACjB0J,EAAMhJ,EAAU,CAAC,IAAM,IAAI4R,IAAItS,QAC1BqS,kBAAkBrS,GAAKkD,KAAKqO,YAAY7H,EAAK,IAAI2E,yBACjD+D,oBAAoBpS,GAAKkD,KAAKsO,cAAc9H,EAAK,IAAI2E,yBACrD8D,eAAenS,GAAKkD,KAAKuO,SAAS/H,EAAK,IAAI2E,2BAIpDxN,EACe,SAAXF,GAEe,KADfwN,EAAKvB,GAAQrN,KAAK2D,KAAKiP,eAAgB/D,IACpBD,EAAK,KACN,QAAXxN,GAEQ,KADfwN,EAAKvB,GAAQrN,KAAK2D,KAAKkP,oBAAqBhE,IACzBD,EAAK,MAGT,KADfA,EAAKvB,GAAQrN,KAAK2D,KAAKmP,kBAAmBjE,IACvBD,EAAK,KAGb,SAAXxN,GAEY,KADZwN,EAAKvB,GAAQrN,KAAK2D,KAAKiP,eAAgB/D,MAK3B,KADZD,EAAKvB,GAAQrN,KAAK2D,KAAKkP,oBAAqBhE,MAK7B,KADfD,EAAKvB,GAAQrN,KAAK2D,KAAKmP,kBAAmBjE,IAN/BD,EAOa,KACN,QAAXxN,GAEK,KADZwN,EAAKvB,GAAQrN,KAAK2D,KAAKkP,oBAAqBhE,MAKhC,KADZD,EAAKvB,GAAQrN,KAAK2D,KAAKiP,eAAgB/D,MAKxB,KADfD,EAAKvB,GAAQrN,KAAK2D,KAAKmP,kBAAmBjE,IAN/BD,EAOa,MAGZ,KADZA,EAAKvB,GAAQrN,KAAK2D,KAAKmP,kBAAmBjE,MAK9B,KADZD,EAAKvB,GAAQrN,KAAK2D,KAAKiP,eAAgB/D,MAKxB,KADfD,EAAKvB,GAAQrN,KAAK2D,KAAKkP,oBAAqBhE,IANjCD,EAOa,SAqFhCoE,GAAuBtH,GAoBvBuH,GAA4BvH,GAoB5BwH,GAA0BxH,YAqBrByH,cACI3D,EAAU1O,EAAGC,UACXA,EAAEJ,OAASG,EAAEH,WAIpBF,EAAG0J,EAAKiJ,EAAMC,EAAQC,EADtBC,EAAY,GAAI9D,EAAc,GAAIC,EAAa,GAAIC,EAAc,OAEhElP,EAAI,EAAGA,EAAI,EAAGA,IAEf0J,EAAMhJ,EAAU,CAAC,IAAM,IAAI4R,IAAItS,GAC/B2S,EAAOzP,KAAKqO,YAAY7H,EAAK,IAC7BkJ,EAAS1P,KAAKsO,cAAc9H,EAAK,IACjCmJ,EAAQ3P,KAAKuO,SAAS/H,EAAK,IAC3BoJ,EAAU3S,KAAKwS,GACf3D,EAAY7O,KAAKyS,GACjB3D,EAAW9O,KAAK0S,GAChB3D,EAAY/O,KAAKwS,GACjBzD,EAAY/O,KAAKyS,GACjB1D,EAAY/O,KAAK0S,OAIrBC,EAAU3D,KAAKJ,GACfC,EAAYG,KAAKJ,GACjBE,EAAWE,KAAKJ,GAChBG,EAAYC,KAAKJ,GACZ/O,EAAI,EAAGA,EAAI,EAAGA,IACfgP,EAAYhP,GAAKyL,GAAYuD,EAAYhP,IACzCiP,EAAWjP,GAAKyL,GAAYwD,EAAWjP,IACvCkP,EAAYlP,GAAKyL,GAAYyD,EAAYlP,SAGxC+S,eAAiB,IAAIvH,OAAO,KAAO0D,EAAY7I,KAAK,KAAO,IAAK,UAChE2M,oBAAsB9P,KAAK6P,oBAC3BE,kBAAoB/P,KAAK6P,oBAEzBG,qBAAuB,IAAI1H,OAAO,KAAOyD,EAAW5I,KAAK,KAAO,IAAK,UACrE8M,0BAA4B,IAAI3H,OAAO,KAAOwD,EAAY3I,KAAK,KAAO,IAAK,UAC3E+M,wBAA0B,IAAI5H,OAAO,KAAOsH,EAAUzM,KAAK,KAAO,IAAK,cAKvEgN,YACEnQ,KAAKoQ,QAAU,IAAM,YA6BvBxR,GAAUiH,EAAOwK,GACtBzK,EAAeC,EAAO,EAAG,GAAG,kBACjB7F,KAAKkG,aAAatH,SAASoB,KAAKoQ,QAASpQ,KAAKsQ,UAAWD,eAgB/DE,GAAenI,EAAU1K,UACvBA,EAAO8S,eAzClB5K,EAAe,IAAK,CAAC,KAAM,GAAI,EAAG,QAClCA,EAAe,IAAK,CAAC,KAAM,GAAI,EAAGuK,IAClCvK,EAAe,IAAK,CAAC,KAAM,GAAI,qBALpB5F,KAAKoQ,SAAW,MAO3BxK,EAAe,MAAO,EAAG,GAAG,iBACjB,GAAKuK,GAAQtU,MAAMmE,MAAQgF,EAAShF,KAAKsQ,UAAW,MAG/D1K,EAAe,QAAS,EAAG,GAAG,iBACnB,GAAKuK,GAAQtU,MAAMmE,MAAQgF,EAAShF,KAAKsQ,UAAW,GACvDtL,EAAShF,KAAKyQ,UAAW,MAGjC7K,EAAe,MAAO,EAAG,GAAG,iBACjB,GAAK5F,KAAKoQ,QAAUpL,EAAShF,KAAKsQ,UAAW,MAGxD1K,EAAe,QAAS,EAAG,GAAG,iBACnB,GAAK5F,KAAKoQ,QAAUpL,EAAShF,KAAKsQ,UAAW,GAChDtL,EAAShF,KAAKyQ,UAAW,MASjC7R,GAAS,KAAK,GACdA,GAAS,KAAK,GAIdsF,EAAa,OAAQ,KAGrBY,EAAgB,OAAQ,IAQxBmD,GAAc,IAAMsI,IACpBtI,GAAc,IAAMsI,IACpBtI,GAAc,IAAMZ,GACpBY,GAAc,IAAMZ,GACpBY,GAAc,IAAMZ,GACpBY,GAAc,KAAMZ,EAAWJ,GAC/BgB,GAAc,KAAMZ,EAAWJ,GAC/BgB,GAAc,KAAMZ,EAAWJ,GAE/BgB,GAAc,MAAOX,GACrBW,GAAc,QAASV,IACvBU,GAAc,MAAOX,GACrBW,GAAc,QAASV,IAEvBwB,GAAc,CAAC,IAAK,MA3+BT,GA4+BXA,GAAc,CAAC,IAAK,OAAO,SAAU/M,EAAOqK,EAAOnF,OAC3CwP,EAAS/O,EAAM3F,GACnBqK,EA9+BO,GA8+BkB,KAAXqK,EAAgB,EAAIA,KAEtC3H,GAAc,CAAC,IAAK,MAAM,SAAU/M,EAAOqK,EAAOnF,GAC9CA,EAAOyP,MAAQzP,EAAOH,QAAQ6P,KAAK5U,GACnCkF,EAAO2P,UAAY7U,KAEvB+M,GAAc,CAAC,IAAK,OAAO,SAAU/M,EAAOqK,EAAOnF,GAC/CmF,EAr/BO,GAq/BO1E,EAAM3F,GACpB8B,EAAgBoD,GAAQxB,SAAU,KAEtCqJ,GAAc,OAAO,SAAU/M,EAAOqK,EAAOnF,OACrC4P,EAAM9U,EAAMgB,OAAS,EACzBqJ,EA1/BO,GA0/BO1E,EAAM3F,EAAMuJ,OAAO,EAAGuL,IACpCzK,EA1/BS,GA0/BO1E,EAAM3F,EAAMuJ,OAAOuL,IACnChT,EAAgBoD,GAAQxB,SAAU,KAEtCqJ,GAAc,SAAS,SAAU/M,EAAOqK,EAAOnF,OACvC6P,EAAO/U,EAAMgB,OAAS,EACtBgU,EAAOhV,EAAMgB,OAAS,EAC1BqJ,EAjgCO,GAigCO1E,EAAM3F,EAAMuJ,OAAO,EAAGwL,IACpC1K,EAjgCS,GAigCO1E,EAAM3F,EAAMuJ,OAAOwL,EAAM,IACzC1K,EAjgCS,GAigCO1E,EAAM3F,EAAMuJ,OAAOyL,IACnClT,EAAgBoD,GAAQxB,SAAU,KAEtCqJ,GAAc,OAAO,SAAU/M,EAAOqK,EAAOnF,OACrC4P,EAAM9U,EAAMgB,OAAS,EACzBqJ,EAxgCO,GAwgCO1E,EAAM3F,EAAMuJ,OAAO,EAAGuL,IACpCzK,EAxgCS,GAwgCO1E,EAAM3F,EAAMuJ,OAAOuL,OAEvC/H,GAAc,SAAS,SAAU/M,EAAOqK,EAAOnF,OACvC6P,EAAO/U,EAAMgB,OAAS,EACtBgU,EAAOhV,EAAMgB,OAAS,EAC1BqJ,EA9gCO,GA8gCO1E,EAAM3F,EAAMuJ,OAAO,EAAGwL,IACpC1K,EA9gCS,GA8gCO1E,EAAM3F,EAAMuJ,OAAOwL,EAAM,IACzC1K,EA9gCS,GA8gCO1E,EAAM3F,EAAMuJ,OAAOyL,WAoDnCC,GAzBAC,GAAatH,GAAW,SAAS,GAEjCuH,GAAa,CACbC,SAj2CkB,CAClBC,QAAU,gBACVC,QAAU,mBACVC,SAAW,eACXC,QAAU,oBACVC,SAAW,sBACXC,SAAW,KA41CX7K,eAp1CwB,CACxB8K,IAAO,YACPC,GAAO,SACPC,EAAO,aACPC,GAAO,eACPC,IAAO,sBACPC,KAAO,6BA+0CPrL,YA7zCqB,eA8zCrBZ,QAxzCiB,KAyzCjBkM,uBAxzCgC,UAyzChCC,aAnzCsB,CACtBC,OAAS,QACTC,KAAS,SACTvJ,EAAK,gBACLwJ,GAAK,aACLtU,EAAK,WACLuU,GAAK,aACL9F,EAAK,UACL+F,GAAK,WACLhG,EAAK,QACLiG,GAAK,UACL/F,EAAK,UACLgG,GAAK,YACLlJ,EAAK,SACLmJ,GAAK,YAuyCLnI,OAAQK,GACRN,YAAaQ,GAEbyC,KA9gBoB,CACpBL,IAAM,EACNC,IAAM,GA8gBNoB,SAAUK,GACVP,YAAaS,GACbR,cAAeO,GAEf8D,cAnC6B,iBAuC7BC,GAAU,GACVC,GAAiB,YAGZC,GAAgB7P,UACdA,EAAMA,EAAIqB,cAAciC,QAAQ,IAAK,KAAOtD,WA8B9C8P,GAAWtP,OACZuP,EAAY,SAEXJ,GAAQnP,IACLhI,GAAUA,EAAOwX,YAEjBD,EAAY/B,GAAaiC,MACJC,IAErBC,GAAmBJ,GACrB,MAAOK,WAENT,GAAQnP,YAMV2P,GAAoBnQ,EAAKqQ,OAC1BC,SACAtQ,KAEIsQ,EADAhX,EAAY+W,GACLE,GAAUvQ,GAGVwQ,GAAaxQ,EAAKqQ,IAKzBrC,GAAesC,EAGU,oBAAb5Q,SAA6BA,QAAQH,MAE7CG,QAAQH,KAAK,UAAYS,EAAO,2CAKrCgO,GAAaiC,eAGfO,GAAchQ,EAAMvC,MACV,OAAXA,EAAiB,KACbxD,EAAQmG,EAAesN,MAC3BjQ,EAAOwS,KAAOjQ,EACO,MAAjBmP,GAAQnP,GACRD,EAAgB,uBACR,2OAIRK,EAAe+O,GAAQnP,GAAMkQ,aAC1B,GAA2B,MAAvBzS,EAAO0S,gBACsB,MAAhChB,GAAQ1R,EAAO0S,cACf/P,EAAe+O,GAAQ1R,EAAO0S,cAAcD,YACzC,IAEW,OADdjW,EAASqV,GAAW7R,EAAO0S,sBAIlBf,GAAe3R,EAAO0S,gBACvBf,GAAe3R,EAAO0S,cAAgB,IAE1Cf,GAAe3R,EAAO0S,cAAc3W,KAAK,CACrCwG,KAAMA,EACNvC,OAAQA,IAEL,KATP2C,EAAenG,EAAOiW,eAalCf,GAAQnP,GAAQ,IAAIM,EAAOH,EAAaC,EAAc3C,IAElD2R,GAAepP,IACfoP,GAAepP,GAAMoQ,SAAQ,SAAU1J,GACnCsJ,GAAatJ,EAAE1G,KAAM0G,EAAEjJ,WAO/BkS,GAAmB3P,GAGZmP,GAAQnP,iBAGRmP,GAAQnP,GACR,cAiCN+P,GAAWvQ,OACZvF,KAEAuF,GAAOA,EAAIlC,SAAWkC,EAAIlC,QAAQmS,QAClCjQ,EAAMA,EAAIlC,QAAQmS,QAGjBjQ,SACMgO,OAGNlV,EAAQkH,GAAM,IAEfvF,EAASqV,GAAW9P,UAETvF,EAEXuF,EAAM,CAACA,mBArKO6Q,WACPC,EAAGC,EAAMtW,EAAQmN,EAAxB/N,EAAI,EAEDA,EAAIgX,EAAM9W,QAAQ,KAErB+W,GADAlJ,EAAQiI,GAAgBgB,EAAMhX,IAAI+N,MAAM,MAC9B7N,OAEVgX,GADAA,EAAOlB,GAAgBgB,EAAMhX,EAAI,KACnBkX,EAAKnJ,MAAM,KAAO,KACzBkJ,EAAI,GAAG,IACVrW,EAASqV,GAAWlI,EAAM3H,MAAM,EAAG6Q,GAAG5Q,KAAK,aAEhCzF,KAEPsW,GAAQA,EAAKhX,QAAU+W,GAAK/R,EAAc6I,EAAOmJ,GAAM,IAASD,EAAI,QAIxEA,IAEJjX,WAEGmU,GAmJAgD,CAAahR,YAOfiR,GAAenW,OAChBK,EACAjB,EAAIY,EAAEoL,UAENhM,IAAsC,IAAjCW,EAAgBC,GAAGK,WACxBA,EACIjB,EApwCA,GAowCiB,GAAKA,EApwCtB,GAowCuC,GApwCvC,EAqwCAA,EApwCD,GAowCkB,GAAKA,EApwCvB,GAowCwC+M,GAAY/M,EAtwCpD,GAswC6DA,EArwC5D,IACD,EAqwCCA,EApwCD,GAowCkB,GAAKA,EApwCvB,GAowCwC,IAAmB,KAAZA,EApwC/C,KAowCgF,IAAdA,EAnwChE,IAmwCiG,IAAdA,EAlwCnF,IAkwCyH,IAAnBA,EAjwCjG,IAHP,EAqwCCA,EApwCC,GAowCgB,GAAKA,EApwCrB,GAowCsC,GApwCtC,EAqwCDA,EApwCC,GAowCgB,GAAKA,EApwCrB,GAowCsC,GApwCtC,EAqwCDA,EApwCM,GAowCW,GAAKA,EApwChB,GAowCiC,IApwCjC,GAqwCL,EAEDW,EAAgBC,GAAGoW,qBAAuB/V,EA7wC3C,GA6wC8DA,EA3wC9D,KA4wCCA,EA5wCD,GA8wCCN,EAAgBC,GAAGqW,iBAAgC,IAAdhW,IACrCA,EA1wCD,GA4wCCN,EAAgBC,GAAGsW,mBAAkC,IAAdjW,IACvCA,EA5wCE,GA+wCNN,EAAgBC,GAAGK,SAAWA,GAG3BL,WAIFuW,GAASnX,EAAGC,EAAGmX,UACX,MAALpX,EACOA,EAEF,MAALC,EACOA,EAEJmX,WAgBFC,GAAiBtT,OAClBpE,EAAGmN,EAAkBwK,EAAaC,EAAiBC,EAA1C3Y,EAAQ,OAEjBkF,EAAO7B,QAIXoV,WApBsBvT,OAElB0T,EAAW,IAAIlY,KAAKd,EAAMiZ,cAC1B3T,EAAO4T,QACA,CAACF,EAAS7H,iBAAkB6H,EAASG,cAAeH,EAASI,cAEjE,CAACJ,EAASjI,cAAeiI,EAASK,WAAYL,EAASM,WAchDC,CAAiBjU,GAG3BA,EAAO+H,IAAyB,MAAnB/H,EAAOiI,GA7zCjB,IA6zCyD,MAApBjI,EAAOiI,GA9zC3C,aAy3CmBjI,OACvBkU,EAAGC,EAAU9H,EAAMC,EAASN,EAAKC,EAAKmI,EAAMC,KAGpC,OADZH,EAAIlU,EAAO+H,IACLuM,IAAqB,MAAPJ,EAAEK,GAAoB,MAAPL,EAAEM,EACjCxI,EAAM,EACNC,EAAM,EAMNkI,EAAWf,GAASc,EAAEI,GAAItU,EAAOiI,GAt4C9B,GAs4CwCyE,GAAW+H,KAAe,EAAG,GAAGtM,MAC3EkE,EAAO+G,GAASc,EAAEK,EAAG,KACrBjI,EAAU8G,GAASc,EAAEM,EAAG,IACV,GAAKlI,EAAU,KACzB+H,GAAkB,OAEnB,CACHrI,EAAMhM,EAAOH,QAAQ6U,MAAM1I,IAC3BC,EAAMjM,EAAOH,QAAQ6U,MAAMzI,QAEvB0I,EAAUjI,GAAW+H,KAAezI,EAAKC,GAE7CkI,EAAWf,GAASc,EAAEU,GAAI5U,EAAOiI,GAl5C9B,GAk5CwC0M,EAAQxM,MAGnDkE,EAAO+G,GAASc,EAAEA,EAAGS,EAAQtI,MAElB,MAAP6H,EAAE7I,IAEFiB,EAAU4H,EAAE7I,GACE,GAAKiB,EAAU,KACzB+H,GAAkB,GAER,MAAPH,EAAE/B,GAET7F,EAAU4H,EAAE/B,EAAInG,GACZkI,EAAE/B,EAAI,GAAK+B,EAAE/B,EAAI,KACjBkC,GAAkB,IAItB/H,EAAUN,EAGdK,EAAO,GAAKA,EAAOQ,GAAYsH,EAAUnI,EAAKC,GAC9CrP,EAAgBoD,GAAQkT,gBAAiB,EACf,MAAnBmB,EACPzX,EAAgBoD,GAAQmT,kBAAmB,GAE3CiB,EAAOhI,GAAmB+H,EAAU9H,EAAMC,EAASN,EAAKC,GACxDjM,EAAOiI,GA96CJ,GA86CemM,EAAKjM,KACvBnI,EAAO6U,WAAaT,EAAK3H,WA/GzBqI,CAAsB9U,GAID,MAArBA,EAAO6U,aACPpB,EAAYL,GAASpT,EAAOiI,GAr0CzB,GAq0CmCsL,EAr0CnC,KAu0CCvT,EAAO6U,WAAa3M,GAAWuL,IAAoC,IAAtBzT,EAAO6U,cACpDjY,EAAgBoD,GAAQiT,oBAAqB,GAGjDlK,EAAO4C,GAAc8H,EAAW,EAAGzT,EAAO6U,YAC1C7U,EAAOiI,GA30CH,GA20Cec,EAAK8K,cACxB7T,EAAOiI,GA30CJ,GA20Cec,EAAK+K,cAQtBlY,EAAI,EAAGA,EAAI,GAAqB,MAAhBoE,EAAOiI,GAAGrM,KAAcA,EACzCoE,EAAOiI,GAAGrM,GAAKd,EAAMc,GAAK2X,EAAY3X,QAInCA,EAAI,EAAGA,IACVoE,EAAOiI,GAAGrM,GAAKd,EAAMc,GAAsB,MAAhBoE,EAAOiI,GAAGrM,GAAqB,IAANA,EAAU,EAAI,EAAKoE,EAAOiI,GAAGrM,GAI7D,KAApBoE,EAAOiI,GA51CJ,IA61CuB,IAAtBjI,EAAOiI,GA51CN,IA61CqB,IAAtBjI,EAAOiI,GA51CN,IA61C0B,IAA3BjI,EAAOiI,GA51CD,KA61CVjI,EAAO+U,UAAW,EAClB/U,EAAOiI,GAj2CJ,GAi2Ce,GAGtBjI,EAAO7B,IAAM6B,EAAO4T,QAAUjI,GAAgBP,IAAYzQ,MAAM,KAAMG,GACtE0Y,EAAkBxT,EAAO4T,QAAU5T,EAAO7B,GAAGgO,YAAcnM,EAAO7B,GAAG6W,SAIlD,MAAfhV,EAAON,MACPM,EAAO7B,GAAG8W,cAAcjV,EAAO7B,GAAG+W,gBAAkBlV,EAAON,MAG3DM,EAAO+U,WACP/U,EAAOiI,GA92CJ,GA82Ce,IAIlBjI,EAAO+H,SAA6B,IAAhB/H,EAAO+H,GAAGsD,GAAqBrL,EAAO+H,GAAGsD,IAAMmI,IACnE5W,EAAgBoD,GAAQpC,iBAAkB,QA+D9CuX,GAAmB,mJACnBC,GAAgB,8IAEhBC,GAAU,wBAEVC,GAAW,CACX,CAAC,eAAgB,uBACjB,CAAC,aAAc,mBACf,CAAC,eAAgB,kBACjB,CAAC,aAAc,eAAe,GAC9B,CAAC,WAAY,eACb,CAAC,UAAW,cAAc,GAC1B,CAAC,aAAc,cACf,CAAC,WAAY,UAEZ,aAAc,eACf,CAAC,YAAa,eAAe,GAC7B,CAAC,UAAW,UAIZC,GAAW,CACX,CAAC,gBAAiB,uBAClB,CAAC,gBAAiB,sBAClB,CAAC,WAAY,kBACb,CAAC,QAAS,aACV,CAAC,cAAe,qBAChB,CAAC,cAAe,oBAChB,CAAC,SAAU,gBACX,CAAC,OAAQ,YACT,CAAC,KAAM,SAGPC,GAAkB,+BAGbC,GAAczV,OACfpE,EAAG8Z,EAGHC,EAAWC,EAAYC,EAAYC,EAFnCC,EAAS/V,EAAOT,GAChB6F,EAAQ+P,GAAiBa,KAAKD,IAAWX,GAAcY,KAAKD,MAG5D3Q,EAAO,KACPxI,EAAgBoD,GAAQxC,KAAM,EAEzB5B,EAAI,EAAG8Z,EAAIJ,GAASxZ,OAAQF,EAAI8Z,EAAG9Z,OAChC0Z,GAAS1Z,GAAG,GAAGoa,KAAK5Q,EAAM,IAAK,CAC/BwQ,EAAaN,GAAS1Z,GAAG,GACzB+Z,GAA+B,IAAnBL,GAAS1Z,GAAG,YAId,MAAdga,cACA5V,EAAOlC,UAAW,MAGlBsH,EAAM,GAAI,KACLxJ,EAAI,EAAG8Z,EAAIH,GAASzZ,OAAQF,EAAI8Z,EAAG9Z,OAChC2Z,GAAS3Z,GAAG,GAAGoa,KAAK5Q,EAAM,IAAK,CAE/ByQ,GAAczQ,EAAM,IAAM,KAAOmQ,GAAS3Z,GAAG,YAInC,MAAdia,cACA7V,EAAOlC,UAAW,OAIrB6X,GAA2B,MAAdE,cACd7V,EAAOlC,UAAW,MAGlBsH,EAAM,GAAI,KACNiQ,GAAQW,KAAK5Q,EAAM,gBAGnBpF,EAAOlC,UAAW,GAFlBgY,EAAW,IAMnB9V,EAAOR,GAAKoW,GAAcC,GAAc,KAAOC,GAAY,IAC3DG,GAA0BjW,QAE1BA,EAAOlC,UAAW,MAKtBH,GAAU,mMAkBLuY,GAAeC,OAChBhO,EAAOI,SAAS4N,EAAS,WACzBhO,GAAQ,GACD,IAAOA,EACPA,GAAQ,IACR,KAAOA,EAEXA,MAsBPiO,GAAa,CACbC,GAAI,EACJC,IAAK,EACLC,KAAK,IACLC,KAAK,IACLC,KAAK,IACLC,KAAK,IACLC,KAAK,IACLC,KAAK,IACLC,KAAK,IACLC,KAAK,cAiBAC,GAAkB/W,OAxEQmW,EAASa,EAAUC,EAAQC,EAASC,EAAWC,EAC1EC,EAwEAjS,EAAQzH,GAAQqY,KAAuBhW,EAAOT,GA7CzC8F,QAAQ,oBAAqB,KAAKA,QAAQ,WAAY,KAAKA,QAAQ,SAAU,IAAIA,QAAQ,SAAU,QA8CxGD,EAAO,KACHkS,GA3EuBnB,EA2EiB/Q,EAAM,GA3Ed4R,EA2EkB5R,EAAM,GA3Ed6R,EA2EkB7R,EAAM,GA3EhB8R,EA2EoB9R,EAAM,GA3EjB+R,EA2EqB/R,EAAM,GA3EhBgS,EA2EoBhS,EAAM,GA1EpGiS,EAAS,CACTnB,GAAeC,GACfvM,GAAyBpB,QAAQwO,GACjCzO,SAAS0O,EAAQ,IACjB1O,SAAS2O,EAAS,IAClB3O,SAAS4O,EAAW,KAGpBC,GACAC,EAAOtb,KAAKwM,SAAS6O,EAAW,KAG7BC,gBAkBWE,EAAYC,EAAaxX,UACvCuX,GAEsB5J,GAA2BnF,QAAQ+O,KACrC,IAAI/b,KAAKgc,EAAY,GAAIA,EAAY,GAAIA,EAAY,IAAIxC,WAEzEpY,EAAgBoD,GAAQpC,iBAAkB,EAC1CoC,EAAOlC,UAAW,GACX,GAqCN2Z,CAAarS,EAAM,GAAIkS,EAAatX,UAIzCA,EAAOiI,GAAKqP,EACZtX,EAAON,cAvBUgY,EAAWC,EAAgBC,MAC5CF,SACOtB,GAAWsB,GACf,GAAIC,SAEA,MAEHE,EAAKtP,SAASqP,EAAW,IACzB/a,EAAIgb,EAAK,WAAUA,EAAKhb,GAAK,IACtB,GAAKA,EAcFib,CAAgB1S,EAAM,GAAIA,EAAM,GAAIA,EAAM,KAExDpF,EAAO7B,GAAKwN,GAAchR,MAAM,KAAMqF,EAAOiI,IAC7CjI,EAAO7B,GAAG8W,cAAcjV,EAAO7B,GAAG+W,gBAAkBlV,EAAON,MAE3D9C,EAAgBoD,GAAQrC,SAAU,OAElCqC,EAAOlC,UAAW,WAgDjBmY,GAA0BjW,MAE3BA,EAAOR,KAAO9E,EAAMqd,YAIpB/X,EAAOR,KAAO9E,EAAMsd,UAIxBhY,EAAOiI,GAAK,GACZrL,EAAgBoD,GAAQjD,OAAQ,MAI5BnB,EAAG4b,EAAa5P,EAAQjD,EAAOsT,EAD/BlC,EAAS,GAAK/V,EAAOT,GAErB2Y,EAAenC,EAAOja,OACtBqc,EAAyB,MAE7BvQ,EAAS1C,EAAalF,EAAOR,GAAIQ,EAAOH,SAASuF,MAAMd,IAAqB,GAEvE1I,EAAI,EAAGA,EAAIgM,EAAO9L,OAAQF,IAC3B+I,EAAQiD,EAAOhM,IACf4b,GAAezB,EAAO3Q,MAAM+B,GAAsBxC,EAAO3E,KAAY,IAAI,OAIrEiY,EAAUlC,EAAO1R,OAAO,EAAG0R,EAAOvN,QAAQgP,KAC9B1b,OAAS,GACjBc,EAAgBoD,GAAQ/C,YAAYlB,KAAKkc,GAE7ClC,EAASA,EAAO/T,MAAM+T,EAAOvN,QAAQgP,GAAeA,EAAY1b,QAChEqc,GAA0BX,EAAY1b,QAGtC2I,EAAqBE,IACjB6S,EACA5a,EAAgBoD,GAAQjD,OAAQ,EAGhCH,EAAgBoD,GAAQhD,aAAajB,KAAK4I,GAE9CqD,GAAwBrD,EAAO6S,EAAaxX,IAEvCA,EAAO1B,UAAYkZ,GACxB5a,EAAgBoD,GAAQhD,aAAajB,KAAK4I,GAKlD/H,EAAgBoD,GAAQ7C,cAAgB+a,EAAeC,EACnDpC,EAAOja,OAAS,GAChBc,EAAgBoD,GAAQ/C,YAAYlB,KAAKga,GAIzC/V,EAAOiI,GA7sDJ,IA6sDgB,KACiB,IAApCrL,EAAgBoD,GAAQxB,SACxBwB,EAAOiI,GA/sDJ,GA+sDe,IAClBrL,EAAgBoD,GAAQxB,aAAUD,GAGtC3B,EAAgBoD,GAAQvC,gBAAkBuC,EAAOiI,GAAGjG,MAAM,GAC1DpF,EAAgBoD,GAAQtC,SAAWsC,EAAO2P,UAE1C3P,EAAOiI,GAttDA,YA6tDezL,EAAQ4b,EAAM1a,OAChC2a,SAEY,MAAZ3a,EAEO0a,EAEgB,MAAvB5b,EAAO8b,aACA9b,EAAO8b,aAAaF,EAAM1a,GACX,MAAflB,EAAOkT,OAEd2I,EAAO7b,EAAOkT,KAAKhS,KACP0a,EAAO,KACfA,GAAQ,IAEPC,GAAiB,KAATD,IACTA,EAAO,GAEJA,GAGAA,EA5BOG,CAAgBvY,EAAOH,QAASG,EAAOiI,GAttDlD,GAstD4DjI,EAAO2P,WAE1E2D,GAAgBtT,GAChBgT,GAAchT,QA7DV+W,GAAkB/W,QAJlByV,GAAczV,YAmKbwY,GAAexY,OAChBlF,EAAQkF,EAAOT,GACfhD,EAASyD,EAAOR,UAEpBQ,EAAOH,QAAUG,EAAOH,SAAWyS,GAAUtS,EAAOP,IAEtC,OAAV3E,QAA8ByD,IAAXhC,GAAkC,KAAVzB,EACpC4D,EAAc,CAACtB,WAAW,KAGhB,iBAAVtC,IACPkF,EAAOT,GAAKzE,EAAQkF,EAAOH,QAAQ4Y,SAAS3d,IAG5CoF,EAASpF,GACF,IAAIiF,EAAOiT,GAAclY,KACzBS,EAAOT,GACdkF,EAAO7B,GAAKrD,EACLD,EAAQ0B,YAtFWyD,OAC1B0Y,EACAC,EAEAC,EACAhd,EACAid,KAEqB,IAArB7Y,EAAOR,GAAG1D,cACVc,EAAgBoD,GAAQ1C,eAAgB,OACxC0C,EAAO7B,GAAK,IAAI3C,KAAKmD,UAIpB/C,EAAI,EAAGA,EAAIoE,EAAOR,GAAG1D,OAAQF,IAC9Bid,EAAe,EACfH,EAAazZ,EAAW,GAAIe,GACN,MAAlBA,EAAO4T,UACP8E,EAAW9E,QAAU5T,EAAO4T,SAEhC8E,EAAWlZ,GAAKQ,EAAOR,GAAG5D,GAC1Bqa,GAA0ByC,GAErB7a,EAAQ6a,KAKbG,GAAgBjc,EAAgB8b,GAAYvb,cAG5C0b,GAAkE,GAAlDjc,EAAgB8b,GAAY1b,aAAalB,OAEzDc,EAAgB8b,GAAYI,MAAQD,GAEjB,MAAfD,GAAuBC,EAAeD,KACtCA,EAAcC,EACdF,EAAaD,IAIrBtc,EAAO4D,EAAQ2Y,GAAcD,GA8CzBK,CAAyB/Y,GAClBzD,EACP0Z,GAA0BjW,YAYTA,OACjBlF,EAAQkF,EAAOT,GACflE,EAAYP,GACZkF,EAAO7B,GAAK,IAAI3C,KAAKd,EAAMiZ,OACpBpY,EAAOT,GACdkF,EAAO7B,GAAK,IAAI3C,KAAKV,EAAMuB,WACH,iBAAVvB,WAxPIkF,OAClBsH,EAAUkO,GAAgBQ,KAAKhW,EAAOT,IAE1B,OAAZ+H,GAKJmO,GAAczV,IACU,IAApBA,EAAOlC,kBACAkC,EAAOlC,SAKlBiZ,GAAkB/W,IACM,IAApBA,EAAOlC,kBACAkC,EAAOlC,SAMlBpD,EAAMse,wBAAwBhZ,MAnB1BA,EAAO7B,GAAK,IAAI3C,MAAM8L,EAAQ,IAqP9B2R,CAAiBjZ,GACVnF,EAAQC,IACfkF,EAAOiI,GAAKxM,EAAIX,EAAMkH,MAAM,IAAI,SAAU7B,UAC/BoI,SAASpI,EAAK,OAEzBmT,GAAgBtT,IACT5E,EAASN,YAtEEkF,OAClBA,EAAO7B,QAIPvC,EAAI2H,EAAqBvD,EAAOT,IACpCS,EAAOiI,GAAKxM,EAAI,CAACG,EAAEuM,KAAMvM,EAAEkN,MAAOlN,EAAEsS,KAAOtS,EAAEmN,KAAMnN,EAAEwc,KAAMxc,EAAEsd,OAAQtd,EAAEud,OAAQvd,EAAEwd,cAAc,SAAUjZ,UAC9FA,GAAOoI,SAASpI,EAAK,OAGhCmT,GAAgBtT,IA6DZqZ,CAAiBrZ,GACV1E,EAASR,GAEhBkF,EAAO7B,GAAK,IAAI3C,KAAKV,GAErBJ,EAAMse,wBAAwBhZ,GA7B9BsZ,CAAgBtZ,GAGfnC,EAAQmC,KACTA,EAAO7B,GAAK,MAGT6B,aA0BFtD,GAAkB5B,EAAOyB,EAAQC,EAAQC,EAAQ8c,OAlElD1d,EAmEAwX,EAAI,UAEO,IAAX7W,IAA8B,IAAXA,IACnBC,EAASD,EACTA,OAAS+B,IAGRnD,EAASN,aA5iFKqF,MACfnF,OAAOwe,2BAC4C,IAA3Cxe,OAAOwe,oBAAoBrZ,GAAKrE,WAEpC2d,MACCA,KAAKtZ,KACFA,EAAIhE,eAAesd,UACZ,SAGR,EAkiFaC,CAAc5e,IAC7BD,EAAQC,IAA2B,IAAjBA,EAAMgB,UAC7BhB,OAAQyD,GAIZ8U,EAAE/T,kBAAmB,EACrB+T,EAAEO,QAAUP,EAAE1T,OAAS4Z,EACvBlG,EAAE5T,GAAKjD,EACP6W,EAAE9T,GAAKzE,EACPuY,EAAE7T,GAAKjD,EACP8W,EAAE/U,QAAU7B,GArFRZ,EAAM,IAAIkE,EAAOiT,GAAcwF,GAuFXnF,MAtFhB0B,WAEJlZ,EAAI8d,IAAI,EAAG,KACX9d,EAAIkZ,cAAWxW,GAGZ1C,WAmFF4Y,GAAa3Z,EAAOyB,EAAQC,EAAQC,UAClCC,GAAiB5B,EAAOyB,EAAQC,EAAQC,GAAQ,GAxQ3D/B,EAAMse,wBAA0BtX,EAC5B,kVAIA,SAAU1B,GACNA,EAAO7B,GAAK,IAAI3C,KAAKwE,EAAOT,IAAMS,EAAO4T,QAAU,OAAS,QAKpElZ,EAAMqd,SAAW,aAGjBrd,EAAMsd,SAAW,iBA6Pb4B,GAAelY,EACf,sGACA,eACQmY,EAAQpF,GAAY9Z,MAAM,KAAMC,kBAChCkE,KAAKjB,WAAagc,EAAMhc,UACjBgc,EAAQ/a,KAAOA,KAAO+a,EAEtBnb,OAKfob,GAAepY,EACf,sGACA,eACQmY,EAAQpF,GAAY9Z,MAAM,KAAMC,kBAChCkE,KAAKjB,WAAagc,EAAMhc,UACjBgc,EAAQ/a,KAAOA,KAAO+a,EAEtBnb,gBAUVqb,GAAOpe,EAAIqe,OACZne,EAAKD,KACc,IAAnBoe,EAAQle,QAAgBjB,EAAQmf,EAAQ,MACxCA,EAAUA,EAAQ,KAEjBA,EAAQle,cACF2Y,SAEX5Y,EAAMme,EAAQ,GACTpe,EAAI,EAAGA,EAAIoe,EAAQle,SAAUF,EACzBoe,EAAQpe,GAAGiC,YAAamc,EAAQpe,GAAGD,GAAIE,KACxCA,EAAMme,EAAQpe,WAGfC,MAoBPoe,GAAW,CAAC,OAAQ,UAAW,QAAS,OAAQ,MAAO,OAAQ,SAAU,SAAU,wBAgC9EC,GAAUC,OACXzW,EAAkBH,EAAqB4W,GACvCC,EAAQ1W,EAAgByE,MAAQ,EAChCkS,EAAW3W,EAAgB4W,SAAW,EACtCjR,EAAS3F,EAAgBoF,OAAS,EAClCyR,EAAQ7W,EAAgB2I,MAAQ3I,EAAgB8W,SAAW,EAC3DC,EAAO/W,EAAgBwK,KAAO,EAC9BgB,EAAQxL,EAAgB0U,MAAQ,EAChChJ,EAAU1L,EAAgBwV,QAAU,EACpC3J,EAAU7L,EAAgByV,QAAU,EACpCuB,EAAehX,EAAgB0V,aAAe,OAE7Ctb,kBA1CgBjB,OAChB,IAAIkF,KAAOlF,MAC2B,IAAjC2L,GAAQrN,KAAK8e,GAAUlY,IAA0B,MAAVlF,EAAEkF,IAAiB7D,MAAMrB,EAAEkF,WAC7D,UAIX4Y,GAAiB,EACZ/e,EAAI,EAAGA,EAAIqe,GAASne,SAAUF,KAC/BiB,EAAEod,GAASre,IAAK,IACZ+e,SACO,EAEPC,WAAW/d,EAAEod,GAASre,OAAS6E,EAAM5D,EAAEod,GAASre,OAChD+e,GAAiB,UAKtB,EAuBSE,CAAgBnX,QAG3BoX,eAAiBJ,EACR,IAAVnL,EACU,IAAVH,EACQ,IAARF,EAAe,GAAK,QAGnB6L,OAASN,EACF,EAARF,OAICS,SAAW3R,EACD,EAAXgR,EACQ,GAARD,OAECa,MAAQ,QAERpb,QAAUyS,UAEV4I,mBAGAC,GAAYhb,UACVA,aAAe+Z,YAGjBkB,GAAU/a,UACXA,EAAS,GACyB,EAA3BC,KAAK+a,OAAO,EAAIhb,GAEhBC,KAAK+a,MAAMhb,YAMjBib,GAAQ3W,EAAO4W,GACpB7W,EAAeC,EAAO,EAAG,GAAG,eACpB2W,EAASxc,KAAK0c,YACdC,EAAO,WACPH,EAAS,IACTA,GAAUA,EACVG,EAAO,KAEJA,EAAO3X,KAAYwX,EAAS,IAAK,GAAKC,EAAYzX,IAAYwX,EAAU,GAAI,MAI3FA,GAAO,IAAK,KACZA,GAAO,KAAM,IAIbvU,GAAc,IAAMH,IACpBG,GAAc,KAAMH,IACpBiB,GAAc,CAAC,IAAK,OAAO,SAAU/M,EAAOqK,EAAOnF,GAC/CA,EAAO4T,SAAU,EACjB5T,EAAON,KAAOgc,GAAiB9U,GAAkB9L,UAQjD6gB,GAAc,2BAETD,GAAiBE,EAAS7F,OAC3B8F,GAAW9F,GAAU,IAAI3Q,MAAMwW,MAEnB,OAAZC,SACO,SAIPC,IADUD,EAAQA,EAAQ/f,OAAS,IAAM,IACtB,IAAIsJ,MAAMuW,KAAgB,CAAC,IAAK,EAAG,GACtDvM,EAAuB,GAAX0M,EAAM,GAAWrb,EAAMqb,EAAM,WAE1B,IAAZ1M,EACL,EACa,MAAb0M,EAAM,GAAa1M,GAAWA,WAI3B2M,GAAgBjhB,EAAOkhB,OACxBngB,EAAKogB,SACLD,EAAMrc,QACN9D,EAAMmgB,EAAME,QACZD,GAAQ/b,EAASpF,IAAUS,EAAOT,GAASA,EAAMuB,UAAYoY,GAAY3Z,GAAOuB,WAAaR,EAAIQ,UAEjGR,EAAIsC,GAAGge,QAAQtgB,EAAIsC,GAAG9B,UAAY4f,GAClCvhB,EAAMuF,aAAapE,GAAK,GACjBA,GAEA4Y,GAAY3Z,GAAOshB,iBAIzBC,GAAexf,UAGgC,IAA5CyD,KAAK+a,MAAMxe,EAAEsB,GAAGme,oBAAsB,aAqJzCC,aACEzd,KAAKjB,WAAYiB,KAAKa,QAA2B,IAAjBb,KAAKc,QA/IhDlF,EAAMuF,aAAe,iBAmJjBuc,GAAc,2DAKdC,GAAW,+KAENC,GAAgB5hB,EAAOiH,OAIxB0Z,EACAkB,EACAC,EAiFmBC,EAAMhD,EACzBhe,EAvFAse,EAAWrf,EAEXsK,EAAQ,YAKR+V,GAAWrgB,GACXqf,EAAW,CACP3O,GAAK1Q,EAAMggB,cACXzP,EAAKvQ,EAAMigB,MACXxP,EAAKzQ,EAAMkgB,SAER1f,EAASR,IAChBqf,EAAW,GACPpY,EACAoY,EAASpY,GAAOjH,EAEhBqf,EAASO,aAAe5f,IAElBsK,EAAQoX,GAAYxG,KAAKlb,KACnC2gB,EAAqB,MAAbrW,EAAM,IAAe,EAAI,EACjC+U,EAAW,CACP9R,EAAK,EACLgD,EAAK5K,EAAM2E,EA7xEZ,IA6xEmDqW,EAClDnQ,EAAK7K,EAAM2E,EA7xEZ,IA6xEmDqW,EAClD5e,EAAK4D,EAAM2E,EA7xEV,IA6xEiDqW,EAClD9T,EAAKlH,EAAM2E,EA7xEV,IA6xEiDqW,EAClDjQ,GAAK/K,EAAM2a,GAA8B,IAArBhW,EA7xEd,KA6xE4CqW,KAE5CrW,EAAQqX,GAASzG,KAAKlb,KAChC2gB,EAAqB,MAAbrW,EAAM,IAAe,EAAI,EACjC+U,EAAW,CACP9R,EAAIyU,GAAS1X,EAAM,GAAIqW,GACvBlQ,EAAIuR,GAAS1X,EAAM,GAAIqW,GACvBvH,EAAI4I,GAAS1X,EAAM,GAAIqW,GACvBpQ,EAAIyR,GAAS1X,EAAM,GAAIqW,GACvBnQ,EAAIwR,GAAS1X,EAAM,GAAIqW,GACvB5e,EAAIigB,GAAS1X,EAAM,GAAIqW,GACvB9T,EAAImV,GAAS1X,EAAM,GAAIqW,KAER,MAAZtB,EACPA,EAAW,GACgB,iBAAbA,IAA0B,SAAUA,GAAY,OAAQA,KA2CnD0C,EA1CSpI,GAAY0F,EAAShb,MA0CxB0a,EA1C+BpF,GAAY0F,EAASjb,IAA7E0d,EA4CEC,EAAKhf,WAAagc,EAAMhc,WAI9Bgc,EAAQkC,GAAgBlC,EAAOgD,GAC3BA,EAAKE,SAASlD,GACdhe,EAAMmhB,GAA0BH,EAAMhD,KAEtChe,EAAMmhB,GAA0BnD,EAAOgD,IACnCnC,cAAgB7e,EAAI6e,aACxB7e,EAAIwN,QAAUxN,EAAIwN,QAGfxN,GAZI,CAAC6e,aAAc,EAAGrR,OAAQ,IA3CjC8Q,EAAW,IACF3O,GAAKoR,EAAQlC,aACtBP,EAAS5O,EAAIqR,EAAQvT,QAGzBsT,EAAM,IAAIzC,GAASC,GAEfgB,GAAWrgB,IAAUkB,EAAWlB,EAAO,aACvC6hB,EAAI9c,QAAU/E,EAAM+E,SAGjB8c,WAMFG,GAAUG,EAAKxB,OAIhB5f,EAAMohB,GAAOrC,WAAWqC,EAAI5X,QAAQ,IAAK,aAErCnH,MAAMrC,GAAO,EAAIA,GAAO4f,WAG3BuB,GAA0BH,EAAMhD,OACjChe,EAAM,UAEVA,EAAIwN,OAASwQ,EAAM/Q,QAAU+T,EAAK/T,QACC,IAA9B+Q,EAAM1R,OAAS0U,EAAK1U,QACrB0U,EAAKX,QAAQvC,IAAI9d,EAAIwN,OAAQ,KAAK6T,QAAQrD,MACxChe,EAAIwN,OAGVxN,EAAI6e,cAAgBb,GAAUgD,EAAKX,QAAQvC,IAAI9d,EAAIwN,OAAQ,KAEpDxN,WAsBFshB,GAAYC,EAAW7a,UACrB,SAAUlD,EAAKge,OACTC,SAEM,OAAXD,GAAoBnf,OAAOmf,KAC3B/a,EAAgBC,EAAM,YAAcA,EAAQ,uDAAyDA,EAAO,kGAE5G+a,EAAMje,EAAKA,EAAMge,EAAQA,EAASC,GAKtCC,GAAYze,KADN4d,GADNrd,EAAqB,iBAARA,GAAoBA,EAAMA,EACbge,GACHD,GAChBte,eAINye,GAAajY,EAAK6U,EAAUqD,EAAUvd,OACvCya,EAAeP,EAASW,cACxBL,EAAOW,GAASjB,EAASY,OACzB1R,EAAS+R,GAASjB,EAASa,SAE1B1V,EAAIzH,YAKToC,EAA+B,MAAhBA,GAA8BA,EAEzCoJ,GACAgB,GAAS/E,EAAKuD,GAAIvD,EAAK,SAAW+D,EAASmU,GAE3C/C,GACA7R,GAAMtD,EAAK,OAAQuD,GAAIvD,EAAK,QAAUmV,EAAO+C,GAE7C9C,GACApV,EAAInH,GAAGge,QAAQ7W,EAAInH,GAAG9B,UAAYqe,EAAe8C,GAEjDvd,GACAvF,EAAMuF,aAAaqF,EAAKmV,GAAQpR,IApFxCqT,GAAe/gB,GAAKue,GAASjf,UAC7ByhB,GAAee,0BAtVJf,GAAe/d,UA6atBgb,GAAWwD,GAAY,EAAG,OAC1BO,GAAWP,IAAa,EAAG,qBA0HtBQ,GAAW1hB,EAAGC,OAEf0hB,EAA0C,IAAvB1hB,EAAEiM,OAASlM,EAAEkM,SAAiBjM,EAAE4M,QAAU7M,EAAE6M,SAE/D+U,EAAS5hB,EAAEigB,QAAQvC,IAAIiE,EAAgB,kBAclCA,GAXL1hB,EAAI2hB,EAAS,GAGH3hB,EAAI2hB,IAAWA,EAFf5hB,EAAEigB,QAAQvC,IAAIiE,EAAiB,EAAG,YAMlC1hB,EAAI2hB,IAFJ5hB,EAAEigB,QAAQvC,IAAIiE,EAAiB,EAAG,UAETC,MAIF,WA6FhCrhB,GAAQuF,OACT+b,cAEQvf,IAARwD,EACOjD,KAAKe,QAAQmS,OAGC,OADrB8L,EAAgBxL,GAAUvQ,WAEjBlC,QAAUie,GAEZhf,MApGfpE,EAAMqjB,cAAgB,uBACtBrjB,EAAMsjB,iBAAmB,6BAuGrBC,GAAOvc,EACP,mJACA,SAAUK,eACMxD,IAARwD,EACOjD,KAAKkG,aAELlG,KAAKtC,OAAOuF,eAKtBiD,YACElG,KAAKe,iBASPqe,GAAMC,EAAUC,UACbD,EAAWC,EAAUA,GAAWA,WAGnCC,GAAiBhW,EAAGxL,EAAGwO,UAExBhD,EAAI,KAAOA,GAAK,EAET,IAAI7M,KAAK6M,EAAI,IAAKxL,EAAGwO,GAXb,YAaR,IAAI7P,KAAK6M,EAAGxL,EAAGwO,GAAGhP,mBAIxBiiB,GAAejW,EAAGxL,EAAGwO,UAEtBhD,EAAI,KAAOA,GAAK,EAET7M,KAAKoQ,IAAIvD,EAAI,IAAKxL,EAAGwO,GArBb,YAuBR7P,KAAKoQ,IAAIvD,EAAGxL,EAAGwO,YAuKrBkT,GAAwB5Z,EAAO6Z,GACpC9Z,EAAe,EAAG,CAACC,EAAOA,EAAM7I,QAAS,EAAG0iB,YA+DvCC,GAAqB3jB,EAAOuR,EAAMC,EAASN,EAAKC,OACjDyS,SACS,MAAT5jB,EACO4R,GAAW5N,KAAMkN,EAAKC,GAAK9D,MAG9BkE,GADJqS,EAAc7R,GAAY/R,EAAOkR,EAAKC,MAElCI,EAAOqS,GAEJC,GAAWxjB,KAAK2D,KAAMhE,EAAOuR,EAAMC,EAASN,EAAKC,aAIvD0S,GAAWxK,EAAU9H,EAAMC,EAASN,EAAKC,OAC1C2S,EAAgBxS,GAAmB+H,EAAU9H,EAAMC,EAASN,EAAKC,GACjElD,EAAO4C,GAAciT,EAAczW,KAAM,EAAGyW,EAAcnS,uBAEzDtE,KAAKY,EAAK8C,uBACV/C,MAAMC,EAAK8K,oBACX9K,KAAKA,EAAK+K,cACRhV,KA5FX4F,EAAe,EAAG,CAAC,KAAM,GAAI,GAAG,kBACrB5F,KAAKqV,WAAa,OAG7BzP,EAAe,EAAG,CAAC,KAAM,GAAI,GAAG,kBACrB5F,KAAK+f,cAAgB,OAOhCN,GAAuB,OAAY,YACnCA,GAAuB,QAAY,YACnCA,GAAuB,OAAS,eAChCA,GAAuB,QAAS,eAIhCvb,EAAa,WAAY,MACzBA,EAAa,cAAe,MAI5BY,EAAgB,WAAY,GAC5BA,EAAgB,cAAe,GAK/BmD,GAAc,IAAUL,IACxBK,GAAc,IAAUL,IACxBK,GAAc,KAAUZ,EAAWJ,GACnCgB,GAAc,KAAUZ,EAAWJ,GACnCgB,GAAc,OAAUR,GAAWN,GACnCc,GAAc,OAAUR,GAAWN,GACnCc,GAAc,QAAUP,GAAWN,GACnCa,GAAc,QAAUP,GAAWN,GAEnC4B,GAAkB,CAAC,OAAQ,QAAS,OAAQ,UAAU,SAAUhN,EAAOuR,EAAMrM,EAAQ2E,GACjF0H,EAAK1H,EAAMN,OAAO,EAAG,IAAM5D,EAAM3F,MAGrCgN,GAAkB,CAAC,KAAM,OAAO,SAAUhN,EAAOuR,EAAMrM,EAAQ2E,GAC3D0H,EAAK1H,GAASjK,EAAM4N,kBAAkBxN,MAqD1C4J,EAAe,IAAK,EAAG,KAAM,WAI7B1B,EAAa,UAAW,KAIxBY,EAAgB,UAAW,GAI3BmD,GAAc,IAAKjB,GACnB+B,GAAc,KAAK,SAAU/M,EAAOqK,GAChCA,EAz8FQ,GAy8F4B,GAApB1E,EAAM3F,GAAS,MAWnC4J,EAAe,IAAK,CAAC,KAAM,GAAI,KAAM,QAIrC1B,EAAa,OAAQ,KAGrBY,EAAgB,OAAQ,GAIxBmD,GAAc,IAAMZ,GACpBY,GAAc,KAAMZ,EAAWJ,GAC/BgB,GAAc,MAAM,SAAUG,EAAU1K,UAE7B0K,EACJ1K,EAAOsiB,yBAA2BtiB,EAAOuiB,cAC1CviB,EAAOwiB,kCAGbnX,GAAc,CAAC,IAAK,MAv+FT,GAw+FXA,GAAc,MAAM,SAAU/M,EAAOqK,GACjCA,EAz+FO,GAy+FO1E,EAAM3F,EAAMsK,MAAMe,GAAW,WAK3C8Y,GAAmBvW,GAAW,QAAQ,GAI1ChE,EAAe,MAAO,CAAC,OAAQ,GAAI,OAAQ,aAI3C1B,EAAa,YAAa,OAG1BY,EAAgB,YAAa,GAI7BmD,GAAc,MAAQT,IACtBS,GAAc,OAAQf,GACtB6B,GAAc,CAAC,MAAO,SAAS,SAAU/M,EAAOqK,EAAOnF,GACnDA,EAAO6U,WAAapU,EAAM3F,MAc9B4J,EAAe,IAAK,CAAC,KAAM,GAAI,EAAG,UAIlC1B,EAAa,SAAU,KAIvBY,EAAgB,SAAU,IAI1BmD,GAAc,IAAMZ,GACpBY,GAAc,KAAMZ,EAAWJ,GAC/B8B,GAAc,CAAC,IAAK,MA1hGP,OA8hGTqX,GAAexW,GAAW,WAAW,GAIzChE,EAAe,IAAK,CAAC,KAAM,GAAI,EAAG,UAIlC1B,EAAa,SAAU,KAIvBY,EAAgB,SAAU,IAI1BmD,GAAc,IAAMZ,GACpBY,GAAc,KAAMZ,EAAWJ,GAC/B8B,GAAc,CAAC,IAAK,MA/iGP,OAkmGTlD,GA/CAwa,GAAezW,GAAW,WAAW,OAIzChE,EAAe,IAAK,EAAG,GAAG,oBACZ5F,KAAKsa,cAAgB,QAGnC1U,EAAe,EAAG,CAAC,KAAM,GAAI,GAAG,oBAClB5F,KAAKsa,cAAgB,OAGnC1U,EAAe,EAAG,CAAC,MAAO,GAAI,EAAG,eACjCA,EAAe,EAAG,CAAC,OAAQ,GAAI,GAAG,kBACF,GAArB5F,KAAKsa,iBAEhB1U,EAAe,EAAG,CAAC,QAAS,GAAI,GAAG,kBACH,IAArB5F,KAAKsa,iBAEhB1U,EAAe,EAAG,CAAC,SAAU,GAAI,GAAG,kBACJ,IAArB5F,KAAKsa,iBAEhB1U,EAAe,EAAG,CAAC,UAAW,GAAI,GAAG,kBACL,IAArB5F,KAAKsa,iBAEhB1U,EAAe,EAAG,CAAC,WAAY,GAAI,GAAG,kBACN,IAArB5F,KAAKsa,iBAEhB1U,EAAe,EAAG,CAAC,YAAa,GAAI,GAAG,kBACP,IAArB5F,KAAKsa,iBAMhBpW,EAAa,cAAe,MAI5BY,EAAgB,cAAe,IAI/BmD,GAAc,IAAQT,GAAWR,GACjCiB,GAAc,KAAQT,GAAWP,GACjCgB,GAAc,MAAQT,GAAWN,GAG5BrB,GAAQ,OAAQA,GAAM7I,QAAU,EAAG6I,IAAS,IAC7CoC,GAAcpC,GAAO8B,aAGhB2Y,GAAQtkB,EAAOqK,GACpBA,EAvmGc,GAumGO1E,EAAuB,KAAhB,KAAO3F,QAGlC6J,GAAQ,IAAKA,GAAM7I,QAAU,EAAG6I,IAAS,IAC1CkD,GAAclD,GAAOya,QAIrBC,GAAoB3W,GAAW,gBAAgB,GAInDhE,EAAe,IAAM,EAAG,EAAG,YAC3BA,EAAe,KAAM,EAAG,EAAG,gBAYvB4a,GAAQvf,EAAO9E,mBAsFVskB,GAAoBxJ,UAClBA,EArFXuJ,GAAM3F,IAAoBA,GAC1B2F,GAAMpP,kBAjuBesP,EAAMC,OAGnB9L,EAAM6L,GAAQ/K,KACdiL,EAAM3D,GAAgBpI,EAAK7U,MAAM6gB,QAAQ,OACzCpjB,EAAS7B,EAAMklB,eAAe9gB,KAAM4gB,IAAQ,WAE5Cna,EAASka,IAAYjd,EAAWid,EAAQljB,IAAWkjB,EAAQljB,GAAQpB,KAAK2D,KAAM6U,GAAO8L,EAAQljB,WAE1FuC,KAAKvC,OAAOgJ,GAAUzG,KAAKkG,aAAakL,SAAS3T,EAAQuC,KAAM2V,GAAYd,MAytBtF2L,GAAMpD,wBArtBK,IAAInc,EAAOjB,OAstBtBwgB,GAAMrD,cAvpBSnhB,EAAOwI,EAAOuc,OACrBC,EACAC,EACAxa,MAECzG,KAAKjB,iBACCc,SAGXmhB,EAAO/D,GAAgBjhB,EAAOgE,OAEpBjB,iBACCc,WAGXohB,EAAoD,KAAvCD,EAAKtE,YAAc1c,KAAK0c,aAErClY,EAAQD,EAAeC,QAGd,OAAQiC,EAASoY,GAAU7e,KAAMghB,GAAQ,aACzC,QAASva,EAASoY,GAAU7e,KAAMghB,aAClC,UAAWva,EAASoY,GAAU7e,KAAMghB,GAAQ,YAC5C,SAAUva,GAAUzG,KAAOghB,GAAQ,cACnC,SAAUva,GAAUzG,KAAOghB,GAAQ,cACnC,OAAQva,GAAUzG,KAAOghB,GAAQ,eACjC,MAAOva,GAAUzG,KAAOghB,EAAOC,GAAa,gBAC5C,OAAQxa,GAAUzG,KAAOghB,EAAOC,GAAa,qBACzCxa,EAASzG,KAAOghB,SAGtBD,EAAUta,EAASnF,EAASmF,IAynBvC+Z,GAAMU,eA5ZU1c,OACRkc,UAEUjhB,KADd+E,EAAQD,EAAeC,KACc,gBAAVA,IAA4BxE,KAAKjB,iBACjDiB,SAGPmhB,EAAcnhB,KAAKa,OAAS2e,GAAiBD,UAEzC/a,OACC,OACDkc,EAAOS,EAAYnhB,KAAKqJ,OAAS,EAAG,EAAG,GAAK,YAE3C,UACDqX,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAUhK,KAAKgK,QAAU,EAAI,EAAG,GAAK,YAEzE,QACD0W,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAU,EAAG,GAAK,YAEtD,OACD0W,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAShK,KAAKiK,OAASjK,KAAKwN,UAAY,GAAK,YAEjF,UACDkT,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAShK,KAAKiK,QAAUjK,KAAKohB,aAAe,GAAK,GAAK,YAE1F,UACA,OACDV,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAShK,KAAKiK,OAAS,GAAK,YAEhE,OACDyW,EAAO1gB,KAAKX,GAAG9B,UACfmjB,GA3GM,KA2GgBtB,GAAMsB,GAAQ1gB,KAAKa,OAAS,EA5G1C,IA4G8Cb,KAAK0c,aA3GrD,MA2GkG,YAEvG,SACDgE,EAAO1gB,KAAKX,GAAG9B,UACfmjB,GAhHQ,IAgHgBtB,GAAMsB,EAhHtB,KAgH6C,YAEpD,SACDA,EAAO1gB,KAAKX,GAAG9B,UACfmjB,GArHQ,IAqHgBtB,GAAMsB,EArHtB,KAqH6C,cAIxDrhB,GAAGge,QAAQqD,GAChB9kB,EAAMuF,aAAanB,MAAM,GAClBA,MAgXXwgB,GAAM/iB,gBA/iBW4jB,GACRA,IACDA,EAAcrhB,KAAKyd,QAAU7hB,EAAMsjB,iBAAmBtjB,EAAMqjB,mBAE5DxY,EAASN,EAAanG,KAAMqhB,UACzBrhB,KAAKkG,aAAaob,WAAW7a,IA2iBxC+Z,GAAMngB,cAxiBSqgB,EAAMa,UACbvhB,KAAKjB,YACCqC,EAASsf,IAASA,EAAK3hB,WACxB4W,GAAY+K,GAAM3hB,WAChB6e,GAAe,CAACxd,GAAIJ,KAAMK,KAAMqgB,IAAOhjB,OAAOsC,KAAKtC,UAAU8jB,UAAUD,GAEvEvhB,KAAKkG,aAAaS,eAmiBjC6Z,GAAMiB,iBA/hBYF,UACPvhB,KAAKK,KAAKsV,KAAe4L,IA+hBpCf,GAAMpgB,YA5hBOsgB,EAAMa,UACXvhB,KAAKjB,YACCqC,EAASsf,IAASA,EAAK3hB,WACxB4W,GAAY+K,GAAM3hB,WAChB6e,GAAe,CAACvd,KAAML,KAAMI,GAAIsgB,IAAOhjB,OAAOsC,KAAKtC,UAAU8jB,UAAUD,GAEvEvhB,KAAKkG,aAAaS,eAuhBjC6Z,GAAMkB,eAnhBUH,UACLvhB,KAAKI,GAAGuV,KAAe4L,IAmhBlCf,GAAMzW,aAxiGcvF,UAEZd,EAAW1D,KADfwE,EAAQD,EAAeC,KAEZxE,KAAKwE,KAETxE,MAoiGXwgB,GAAMmB,4BAvUK7jB,EAAgBkC,MAAM5B,UAwUjCoiB,GAAMpC,iBA5tBYpiB,EAAOwI,OACjBod,EAAaxgB,EAASpF,GAASA,EAAQ2Z,GAAY3Z,YACjDgE,KAAKjB,YAAa6iB,EAAW7iB,aAIrB,iBADdyF,EAAQD,EAAeC,IAAU,eAEtBxE,KAAKzC,UAAYqkB,EAAWrkB,UAE5BqkB,EAAWrkB,UAAYyC,KAAKod,QAAQyD,QAAQrc,GAAOjH,YAotBlEijB,GAAMvC,kBAhtBajiB,EAAOwI,OAClBod,EAAaxgB,EAASpF,GAASA,EAAQ2Z,GAAY3Z,YACjDgE,KAAKjB,YAAa6iB,EAAW7iB,aAIrB,iBADdyF,EAAQD,EAAeC,IAAU,eAEtBxE,KAAKzC,UAAYqkB,EAAWrkB,UAE5ByC,KAAKod,QAAQ8D,MAAM1c,GAAOjH,UAAYqkB,EAAWrkB,YAwsBhEijB,GAAMqB,mBApsBcxhB,EAAMD,EAAIoE,EAAOsd,OAC7BC,EAAY3gB,EAASf,GAAQA,EAAOsV,GAAYtV,GAChD2hB,EAAU5gB,EAAShB,GAAMA,EAAKuV,GAAYvV,YACxCJ,KAAKjB,WAAagjB,EAAUhjB,WAAaijB,EAAQjjB,aAI5B,OAD3B+iB,EAAcA,GAAe,MACT,GAAa9hB,KAAKoe,QAAQ2D,EAAWvd,IAAUxE,KAAKie,SAAS8D,EAAWvd,MACpE,MAAnBsd,EAAY,GAAa9hB,KAAKie,SAAS+D,EAASxd,IAAUxE,KAAKoe,QAAQ4D,EAASxd,KA6rBzFgc,GAAMyB,gBA1rBWjmB,EAAOwI,OAEhB0d,EADAN,EAAaxgB,EAASpF,GAASA,EAAQ2Z,GAAY3Z,YAEjDgE,KAAKjB,YAAa6iB,EAAW7iB,aAIrB,iBADdyF,EAAQD,EAAeC,IAAU,eAEtBxE,KAAKzC,YAAcqkB,EAAWrkB,WAErC2kB,EAAUN,EAAWrkB,UACdyC,KAAKod,QAAQyD,QAAQrc,GAAOjH,WAAa2kB,GAAWA,GAAWliB,KAAKod,QAAQ8D,MAAM1c,GAAOjH,aAgrBxGijB,GAAM2B,uBA5qBkBnmB,EAAOwI,UACpBxE,KAAKiiB,OAAOjmB,EAAOwI,IAAUxE,KAAKoe,QAAQpiB,EAAOwI,IA4qB5Dgc,GAAM4B,wBAzqBmBpmB,EAAOwI,UACrBxE,KAAKiiB,OAAOjmB,EAAOwI,IAAUxE,KAAKie,SAASjiB,EAAOwI,IAyqB7Dgc,GAAMzhB,0BAtVKA,EAAQiB,OAuVnBwgB,GAAMrB,KAAoBA,GAC1BqB,GAAM9iB,OAAoBA,GAC1B8iB,GAAMta,WAAoBA,GAC1Bsa,GAAMlb,IAAoB0V,GAC1BwF,GAAMpe,IAAoB0Y,GAC1B0F,GAAM6B,+BAxVK/kB,EAAO,GAAIQ,EAAgBkC,QAyVtCwgB,GAAMxc,aA9iGcQ,EAAO1C,MACF,iBAAV0C,UAEH8d,WApTiBC,OACrB/d,EAAQ,OACP,IAAIge,KAAKD,EACV/d,EAAMvH,KAAK,CAACkH,KAAMqe,EAAGzd,SAAUF,EAAW2d,YAE9Che,EAAMyH,MAAK,SAAU9O,EAAGC,UACbD,EAAE4H,SAAW3H,EAAE2H,YAEnBP,EA4Seie,CADlBje,EAAQC,EAAqBD,IAEpB1H,EAAI,EAAGA,EAAIwlB,EAAYtlB,OAAQF,SAC/BwlB,EAAYxlB,GAAGqH,MAAMK,EAAM8d,EAAYxlB,GAAGqH,eAI/CT,EAAW1D,KADfwE,EAAQD,EAAeC,YAEZxE,KAAKwE,GAAO1C,UAGpB9B,MAkiGXwgB,GAAMK,iBAleYrc,OACVkc,UAEUjhB,KADd+E,EAAQD,EAAeC,KACc,gBAAVA,IAA4BxE,KAAKjB,iBACjDiB,SAGPmhB,EAAcnhB,KAAKa,OAAS2e,GAAiBD,UAEzC/a,OACC,OACDkc,EAAOS,EAAYnhB,KAAKqJ,OAAQ,EAAG,aAElC,UACDqX,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAUhK,KAAKgK,QAAU,EAAG,aAEhE,QACD0W,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAS,aAE7C,OACD0W,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAShK,KAAKiK,OAASjK,KAAKwN,qBAEhE,UACDkT,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAShK,KAAKiK,QAAUjK,KAAKohB,aAAe,cAEhF,UACA,OACDV,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAShK,KAAKiK,kBAElD,OACDyW,EAAO1gB,KAAKX,GAAG9B,UACfmjB,GAAQtB,GAAMsB,GAAQ1gB,KAAKa,OAAS,EA5D5B,IA4DgCb,KAAK0c,aA3DvC,gBA6DL,SACDgE,EAAO1gB,KAAKX,GAAG9B,UACfmjB,GAAQtB,GAAMsB,EAhEN,eAkEP,SACDA,EAAO1gB,KAAKX,GAAG9B,UACfmjB,GAAQtB,GAAMsB,EArEN,iBAyEXrhB,GAAGge,QAAQqD,GAChB9kB,EAAMuF,aAAanB,MAAM,GAClBA,MAsbXwgB,GAAM5B,SAAoBA,GAC1B4B,GAAMkC,uBAvXE3kB,EAAIiC,WACD,CAACjC,EAAEsL,OAAQtL,EAAEiM,QAASjM,EAAEkM,OAAQlM,EAAEub,OAAQvb,EAAEqc,SAAUrc,EAAEsc,SAAUtc,EAAEuc,gBAuX/EkG,GAAMmC,wBAnXE5kB,EAAIiC,WACD,CACHsb,MAAOvd,EAAEsL,OACTkB,OAAQxM,EAAEiM,QACVC,KAAMlM,EAAEkM,OACRmG,MAAOrS,EAAEqS,QACTE,QAASvS,EAAEuS,UACXG,QAAS1S,EAAE0S,UACXmL,aAAc7d,EAAE6d,iBA4WxB4E,GAAMoC,yBA7XK,IAAIlmB,KAAKsD,KAAKzC,YA8XzBijB,GAAMqC,qBArnBeC,OACZ9iB,KAAKjB,iBACC,SAEPlB,GAAqB,IAAfilB,EACN/kB,EAAIF,EAAMmC,KAAKod,QAAQvf,MAAQmC,YAC/BjC,EAAEsL,OAAS,GAAKtL,EAAEsL,OAAS,KACpBlD,EAAapI,EAAGF,EAAM,iCAAmC,gCAEhE6F,EAAWhH,KAAKP,UAAU0mB,aAEtBhlB,EACOmC,KAAK4iB,SAASC,cAEd,IAAInmB,KAAKsD,KAAKzC,UAA+B,GAAnByC,KAAK0c,YAAmB,KAAMmG,cAActc,QAAQ,IAAKJ,EAAapI,EAAG,MAG3GoI,EAAapI,EAAGF,EAAM,+BAAiC,+BAqmBlE2iB,GAAMuC,uBA3lBG/iB,KAAKjB,gBACC,qBAAuBiB,KAAKS,GAAK,WAExCwF,EAAO,SACP+c,EAAO,GACNhjB,KAAKijB,YACNhd,EAA4B,IAArBjG,KAAK0c,YAAoB,aAAe,mBAC/CsG,EAAO,SAEPE,EAAS,IAAMjd,EAAO,MACtBoD,EAAQ,GAAKrJ,KAAKqJ,QAAUrJ,KAAKqJ,QAAU,KAAQ,OAAS,SAE5D8Z,EAASH,EAAO,cAEbhjB,KAAKvC,OAAOylB,EAAS7Z,EAHb,wBAG+B8Z,IA8kBlD3C,GAAM4C,yBAzWKpjB,KAAKjB,UAAYiB,KAAK6iB,cAAgB,MA0WjDrC,GAAMpkB,2BA3nBK4D,KAAKod,QAAQ1f,OAAO,MAAMD,OAAO,qCA4nB5C+iB,GAAM6C,uBAtYK7hB,KAAKE,MAAM1B,KAAKzC,UAAY,MAuYvCijB,GAAMjjB,0BA3YKyC,KAAKX,GAAG9B,UAAmC,KAArByC,KAAKc,SAAW,IA4YjD0f,GAAM8C,8BA7VK,CACHtnB,MAAOgE,KAAKS,GACZhD,OAAQuC,KAAKU,GACbhD,OAAQsC,KAAKe,QACb0Z,MAAOza,KAAKa,OACZlD,OAAQqC,KAAKR,UAyVrBghB,GAAMnX,KAAaM,GACnB6W,GAAMlX,6BAtmGKA,GAAWtJ,KAAKqJ,SAumG3BmX,GAAMnL,kBApSmBrZ,UACd2jB,GAAqBtjB,KAAK2D,KACzBhE,EACAgE,KAAKuN,OACLvN,KAAKwN,UACLxN,KAAKkG,aAAa0P,MAAM1I,IACxBlN,KAAKkG,aAAa0P,MAAMzI,MA+RpCqT,GAAMT,qBA5RsB/jB,UACjB2jB,GAAqBtjB,KAAK2D,KACzBhE,EAAOgE,KAAK0b,UAAW1b,KAAKohB,aAAc,EAAG,IA2RzDZ,GAAMhF,QAAUgF,GAAMjF,kBAnOEvf,UACJ,MAATA,EAAgBwF,KAAKC,MAAMzB,KAAKgK,QAAU,GAAK,GAAKhK,KAAKgK,MAAoB,GAAbhO,EAAQ,GAASgE,KAAKgK,QAAU,IAmO3GwW,GAAMxW,MAAcyB,GACpB+U,GAAMtW,8BAx1FKA,GAAYlK,KAAKqJ,OAAQrJ,KAAKgK,UAy1FzCwW,GAAMjT,KAAiBiT,GAAM/E,eAznFRzf,OACbuR,EAAOvN,KAAKkG,aAAaqH,KAAKvN,aAClB,MAAThE,EAAgBuR,EAAOvN,KAAK6a,IAAqB,GAAhB7e,EAAQuR,GAAW,MAwnF/DiT,GAAM9E,QAAiB8E,GAAM+C,kBArnFLvnB,OAChBuR,EAAOK,GAAW5N,KAAM,EAAG,GAAGuN,YAClB,MAATvR,EAAgBuR,EAAOvN,KAAK6a,IAAqB,GAAhB7e,EAAQuR,GAAW,MAonF/DiT,GAAMzS,2BAxREyV,EAAWxjB,KAAKkG,aAAa0P,aAC1B7H,GAAY/N,KAAKqJ,OAAQma,EAAStW,IAAKsW,EAASrW,MAwR3DqT,GAAMiD,iCA7RK1V,GAAY/N,KAAKqJ,OAAQ,EAAG,IA8RvCmX,GAAMvW,KAAakW,GACnBK,GAAMpR,IAAaoR,GAAM7E,cA55EC3f,OACjBgE,KAAKjB,iBACU,MAAT/C,EAAgBgE,KAAOH,QAE9BuP,EAAMpP,KAAKa,OAASb,KAAKX,GAAGgO,YAAcrN,KAAKX,GAAG6W,gBACzC,MAATla,GACAA,WAjKcA,EAAO0B,SACJ,iBAAV1B,EACAA,EAGNoD,MAAMpD,GAKU,iBADrBA,EAAQ0B,EAAOiR,cAAc3S,IAElBA,EAGJ,KARIyN,SAASzN,EAAO,IA2Jf0nB,CAAa1nB,EAAOgE,KAAKkG,cAC1BlG,KAAK6a,IAAI7e,EAAQoT,EAAK,MAEtBA,GAo5EfoR,GAAMhT,iBAh5E0BxR,OACvBgE,KAAKjB,iBACU,MAAT/C,EAAgBgE,KAAOH,QAE9B2N,GAAWxN,KAAKoP,MAAQ,EAAIpP,KAAKkG,aAAa0P,MAAM1I,KAAO,SAC/C,MAATlR,EAAgBwR,EAAUxN,KAAK6a,IAAI7e,EAAQwR,EAAS,MA44E/DgT,GAAMY,oBAz4EuBplB,OACpBgE,KAAKjB,iBACU,MAAT/C,EAAgBgE,KAAOH,OAOrB,MAAT7D,EAAe,KACXwR,WAzKaxR,EAAO0B,SACP,iBAAV1B,EACA0B,EAAOiR,cAAc3S,GAAS,GAAK,EAEvCoD,MAAMpD,GAAS,KAAOA,EAqKX2nB,CAAgB3nB,EAAOgE,KAAKkG,qBACnClG,KAAKoP,IAAIpP,KAAKoP,MAAQ,EAAI5B,EAAUA,EAAU,UAE9CxN,KAAKoP,OAAS,GA63E7BoR,GAAM7S,mBApLoB3R,OAClB2R,EAAYnM,KAAK+a,OAAOvc,KAAKod,QAAQyD,QAAQ,OAAS7gB,KAAKod,QAAQyD,QAAQ,SAAW,OAAS,SACnF,MAAT7kB,EAAgB2R,EAAY3N,KAAK6a,IAAK7e,EAAQ2R,EAAY,MAmLrE6S,GAAMlH,KAAOkH,GAAMpQ,MAAQc,GAC3BsP,GAAMpG,OAASoG,GAAMlQ,QAAU8P,GAC/BI,GAAMnG,OAASmG,GAAM/P,QAAU4P,GAC/BG,GAAMlG,YAAckG,GAAM5E,aAAe2E,GACzCC,GAAM9D,mBAxkCiB1gB,EAAO4nB,EAAeC,OAErCC,EADAtH,EAASxc,KAAKc,SAAW,MAExBd,KAAKjB,iBACU,MAAT/C,EAAgBgE,KAAOH,OAErB,MAAT7D,EAAe,IACM,iBAAVA,MAEO,QADdA,EAAQ4gB,GAAiB9U,GAAkB9L,WAEhCgE,UAEJwB,KAAKc,IAAItG,GAAS,KAAO6nB,IAChC7nB,GAAgB,WAEfgE,KAAKa,QAAU+iB,IAChBE,EAAcvG,GAAcvd,YAE3Bc,QAAU9E,OACV6E,QAAS,EACK,MAAfijB,QACKjJ,IAAIiJ,EAAa,KAEtBtH,IAAWxgB,KACN4nB,GAAiB5jB,KAAK+jB,kBACvBtF,GAAYze,KAAM4d,GAAe5hB,EAAQwgB,EAAQ,KAAM,GAAG,GAClDxc,KAAK+jB,yBACRA,mBAAoB,EACzBnoB,EAAMuF,aAAanB,MAAM,QACpB+jB,kBAAoB,OAG1B/jB,YAEAA,KAAKa,OAAS2b,EAASe,GAAcvd,OAuiCpDwgB,GAAM3iB,aArhCmB+lB,UACd5jB,KAAK0c,UAAU,EAAGkH,IAqhC7BpD,GAAMlD,eAlhCqBsG,UACnB5jB,KAAKa,cACA6b,UAAU,EAAGkH,QACb/iB,QAAS,EAEV+iB,QACKhF,SAASrB,GAAcvd,MAAO,MAGpCA,MA0gCXwgB,GAAMwD,wBAtgCe,MAAbhkB,KAAKY,UACA8b,UAAU1c,KAAKY,MAAM,GAAO,QAC9B,GAAuB,iBAAZZ,KAAKS,GAAiB,KAChCwjB,EAAQrH,GAAiB/U,GAAa7H,KAAKS,IAClC,MAATwjB,OACKvH,UAAUuH,QAGVvH,UAAU,GAAG,UAGnB1c,MA4/BXwgB,GAAM0D,8BAz/ByBloB,WACtBgE,KAAKjB,YAGV/C,EAAQA,EAAQ2Z,GAAY3Z,GAAO0gB,YAAc,GAEzC1c,KAAK0c,YAAc1gB,GAAS,IAAO,IAo/B/CwkB,GAAM2D,wBA/+BEnkB,KAAK0c,YAAc1c,KAAKod,QAAQpT,MAAM,GAAG0S,aACzC1c,KAAK0c,YAAc1c,KAAKod,QAAQpT,MAAM,GAAG0S,aA++BjD8D,GAAMyC,2BAr9BKjjB,KAAKjB,YAAaiB,KAAKa,QAs9BlC2f,GAAM4D,+BAl9BKpkB,KAAKjB,WAAYiB,KAAKa,QAm9BjC2f,GAAM/C,MAAuBA,GAC7B+C,GAAM/F,MAAuBgD,GAC7B+C,GAAM6D,2BA7EKrkB,KAAKa,OAAS,MAAQ,IA8EjC2f,GAAM8D,2BA1EKtkB,KAAKa,OAAS,6BAA+B,IA2ExD2f,GAAM+D,MAAS3hB,EAAU,kDAAmDud,IAC5EK,GAAMjW,OAAS3H,EAAU,mDAAoD6I,IAC7E+U,GAAMlF,MAAS1Y,EAAU,iDAAkD+G,IAC3E6W,GAAMwC,KAASpgB,EAAU,qHAjjCJ5G,EAAO4nB,UACX,MAAT5nB,GACqB,iBAAVA,IACPA,GAASA,QAGR0gB,UAAU1gB,EAAO4nB,GAEf5jB,OAECA,KAAK0c,eAwiCrB8D,GAAMgE,aAAe5hB,EAAU,0HAp/BtBrG,EAAYyD,KAAKykB,sBACXzkB,KAAKykB,kBAGZlQ,EAAI,MAERpU,EAAWoU,EAAGvU,OACduU,EAAImF,GAAcnF,IAEZpL,GAAI,KACF4R,EAAQxG,EAAE1T,OAASrD,EAAU+W,EAAEpL,IAAMwM,GAAYpB,EAAEpL,SAClDsb,cAAgBzkB,KAAKjB,WACtBiD,EAAcuS,EAAEpL,GAAI4R,EAAM2H,WAAa,YAEtC+B,eAAgB,SAGlBzkB,KAAKykB,qBAi/BZC,GAAU3gB,EAAO5H,mBAiCZwoB,GAAOlnB,EAAQmnB,EAAOC,EAAOC,OAC9BpnB,EAAS8V,KACT3V,EAAML,IAAYwG,IAAI8gB,EAAQF,UAC3BlnB,EAAOmnB,GAAOhnB,EAAKJ,YAGrBsnB,GAAgBtnB,EAAQmnB,EAAOC,MAChCroB,EAASiB,KACTmnB,EAAQnnB,EACRA,OAASgC,GAGbhC,EAASA,GAAU,GAEN,MAATmnB,SACOD,GAAMlnB,EAAQmnB,EAAOC,EAAO,aAGnC/nB,EACAkoB,EAAM,OACLloB,EAAI,EAAGA,EAAI,GAAIA,IAChBkoB,EAAIloB,GAAK6nB,GAAMlnB,EAAQX,EAAG+nB,EAAO,gBAE9BG,WAWFC,GAAkBC,EAAcznB,EAAQmnB,EAAOC,GACxB,kBAAjBK,GACH1oB,EAASiB,KACTmnB,EAAQnnB,EACRA,OAASgC,GAGbhC,EAASA,GAAU,KAGnBmnB,EADAnnB,EAASynB,EAETA,GAAe,EAEX1oB,EAASiB,KACTmnB,EAAQnnB,EACRA,OAASgC,GAGbhC,EAASA,GAAU,QAUnBX,EAPAY,EAAS8V,KACT2R,EAAQD,EAAexnB,EAAOkY,MAAM1I,IAAM,KAEjC,MAAT0X,SACOD,GAAMlnB,GAASmnB,EAAQO,GAAS,EAAGN,EAAO,WAIjDG,EAAM,OACLloB,EAAI,EAAGA,EAAI,EAAGA,IACfkoB,EAAIloB,GAAK6nB,GAAMlnB,GAASX,EAAIqoB,GAAS,EAAGN,EAAO,cAE5CG,EAlGXN,GAAQtT,kBAzgHWnO,EAAKuD,EAAKqO,OACrBpO,EAASzG,KAAKolB,UAAUniB,IAAQjD,KAAKolB,UAAL,gBAC7B1hB,EAAW+C,GAAUA,EAAOpK,KAAKmK,EAAKqO,GAAOpO,GAwgHxDie,GAAQ7d,wBA5/GiB5D,OACjBxF,EAASuC,KAAKqlB,gBAAgBpiB,GAC9BqiB,EAActlB,KAAKqlB,gBAAgBpiB,EAAIsiB,sBAEvC9nB,IAAW6nB,EACJ7nB,QAGN4nB,gBAAgBpiB,GAAOqiB,EAAY/e,QAAQ,oBAAoB,SAAUhG,UACnEA,EAAI2C,MAAM,MAGdlD,KAAKqlB,gBAAgBpiB,KAi/GhCyhB,GAAQ/d,8BA3+GG3G,KAAKwlB,cA4+GhBd,GAAQ3e,iBAt+GUxE,UACPvB,KAAKylB,SAASlf,QAAQ,KAAMhF,IAs+GvCmjB,GAAQ/K,SAAkB8G,GAC1BiE,GAAQpD,WAAkBb,GAC1BiE,GAAQxS,sBAp9Ge3Q,EAAQggB,EAAetK,EAAQyO,OAC9Cjf,EAASzG,KAAK2lB,cAAc1O,UACxBvT,EAAW+C,GACfA,EAAOlF,EAAQggB,EAAetK,EAAQyO,GACtCjf,EAAOF,QAAQ,MAAOhF,IAi9G9BmjB,GAAQkB,oBA98GazI,EAAM1W,OACnBhJ,EAASuC,KAAK2lB,cAAcxI,EAAO,EAAI,SAAW,eAC/CzZ,EAAWjG,GAAUA,EAAOgJ,GAAUhJ,EAAO8I,QAAQ,MAAOE,IA68GvEie,GAAQ1gB,aA7lHM9C,OACNZ,EAAMxD,MACLA,KAAKoE,EAEFwC,EADJpD,EAAOY,EAAOpE,SAELA,GAAKwD,OAEL,IAAMxD,GAAKwD,OAGnBqT,QAAUzS,OAIVgf,+BAAiC,IAAI5X,QACrCtI,KAAKggB,wBAAwB6F,QAAU7lB,KAAKigB,cAAc4F,QACvD,IAAO,UAAWA,SA+kH9BnB,GAAQna,gBAxhGexM,EAAGN,UACjBM,EAIEhC,EAAQiE,KAAKkc,SAAWlc,KAAKkc,QAAQne,EAAEiM,SAC1ChK,KAAKkc,SAASlc,KAAKkc,QAAQ4J,UAAYnb,IAAkB5D,KAAKtJ,GAAU,SAAW,cAAcM,EAAEiM,SAJ5FjO,EAAQiE,KAAKkc,SAAWlc,KAAKkc,QAChClc,KAAKkc,QAAL,YAshGZwI,GAAQpa,qBA/gGoBvM,EAAGN,UACtBM,EAIEhC,EAAQiE,KAAK+lB,cAAgB/lB,KAAK+lB,aAAahoB,EAAEiM,SACpDhK,KAAK+lB,aAAapb,GAAiB5D,KAAKtJ,GAAU,SAAW,cAAcM,EAAEiM,SAJtEjO,EAAQiE,KAAK+lB,cAAgB/lB,KAAK+lB,aACrC/lB,KAAK+lB,aAAL,YA6gGZrB,GAAQha,qBA99FoBM,EAAWvN,EAAQE,OACvCb,EAAG0J,EAAK0B,KAERlI,KAAKgmB,yBACEjb,GAAkB1O,KAAK2D,KAAMgL,EAAWvN,EAAQE,OAGtDqC,KAAKoL,oBACDA,aAAe,QACfC,iBAAmB,QACnBC,kBAAoB,IAMxBxO,EAAI,EAAGA,EAAI,GAAIA,IAAK,IAErB0J,EAAMhJ,EAAU,CAAC,IAAMV,IACnBa,IAAWqC,KAAKqL,iBAAiBvO,UAC5BuO,iBAAiBvO,GAAK,IAAIwL,OAAO,IAAMtI,KAAKuK,OAAO/D,EAAK,IAAID,QAAQ,IAAK,IAAM,IAAK,UACpF+E,kBAAkBxO,GAAK,IAAIwL,OAAO,IAAMtI,KAAKsK,YAAY9D,EAAK,IAAID,QAAQ,IAAK,IAAM,IAAK,MAE9F5I,GAAWqC,KAAKoL,aAAatO,KAC9BoL,EAAQ,IAAMlI,KAAKuK,OAAO/D,EAAK,IAAM,KAAOxG,KAAKsK,YAAY9D,EAAK,SAC7D4E,aAAatO,GAAK,IAAIwL,OAAOJ,EAAM3B,QAAQ,IAAK,IAAK,MAG1D5I,GAAqB,SAAXF,GAAqBuC,KAAKqL,iBAAiBvO,GAAGiK,KAAKiE,UACtDlO,EACJ,GAAIa,GAAqB,QAAXF,GAAoBuC,KAAKsL,kBAAkBxO,GAAGiK,KAAKiE,UAC7DlO,EACJ,IAAKa,GAAUqC,KAAKoL,aAAatO,GAAGiK,KAAKiE,UACrClO,IA87FnB4nB,GAAQja,qBA33FcrC,UACdpI,KAAKgmB,mBACA9oB,EAAW8C,KAAM,iBAClB4L,GAAmBvP,KAAK2D,MAExBoI,EACOpI,KAAKoM,mBAELpM,KAAKkM,eAGXhP,EAAW8C,KAAM,uBACbkM,aAAeP,IAEjB3L,KAAKoM,oBAAsBhE,EAC9BpI,KAAKoM,mBAAqBpM,KAAKkM,eA62F3CwY,GAAQla,0BAh5FmBpC,UACnBpI,KAAKgmB,mBACA9oB,EAAW8C,KAAM,iBAClB4L,GAAmBvP,KAAK2D,MAExBoI,EACOpI,KAAKqM,wBAELrM,KAAKmM,oBAGXjP,EAAW8C,KAAM,4BACbmM,kBAAoBT,IAEtB1L,KAAKqM,yBAA2BjE,EACnCpI,KAAKqM,wBAA0BrM,KAAKmM,oBAk4FhDuY,GAAQnX,cAxsFa/G,UACVoH,GAAWpH,EAAKxG,KAAK4V,MAAM1I,IAAKlN,KAAK4V,MAAMzI,KAAKI,MAwsF3DmX,GAAQuB,iCA3rFGjmB,KAAK4V,MAAMzI,KA4rFtBuX,GAAQwB,iCAhsFGlmB,KAAK4V,MAAM1I,KAksFtBwX,GAAQnW,kBAplFiBxQ,EAAGN,OACpB8Q,EAAWxS,EAAQiE,KAAKmmB,WAAanmB,KAAKmmB,UAC1CnmB,KAAKmmB,UAAWpoB,IAAW,IAANA,GAAciC,KAAKmmB,UAAUL,SAAS/e,KAAKtJ,GAAW,SAAW,qBAC5E,IAANM,EAAckQ,GAAcM,EAAUvO,KAAK4V,MAAM1I,KAClDnP,EAAKwQ,EAASxQ,EAAEqR,OAASb,GAilFpCmW,GAAQrW,qBAvkFoBtQ,UACV,IAANA,EAAckQ,GAAcjO,KAAKomB,aAAcpmB,KAAK4V,MAAM1I,KAC3DnP,EAAKiC,KAAKomB,aAAaroB,EAAEqR,OAASpP,KAAKomB,cAskFlD1B,GAAQpW,uBA9kFsBvQ,UACZ,IAANA,EAAckQ,GAAcjO,KAAKqmB,eAAgBrmB,KAAK4V,MAAM1I,KAC7DnP,EAAKiC,KAAKqmB,eAAetoB,EAAEqR,OAASpP,KAAKqmB,gBA6kFpD3B,GAAQ/V,uBApgFsBK,EAAavR,EAAQE,OAC3Cb,EAAG0J,EAAK0B,KAERlI,KAAKsmB,2BACEvX,GAAoB1S,KAAK2D,KAAMgP,EAAavR,EAAQE,OAG1DqC,KAAKiP,sBACDA,eAAiB,QACjBE,kBAAoB,QACpBD,oBAAsB,QACtBqX,mBAAqB,IAGzBzpB,EAAI,EAAGA,EAAI,EAAGA,IAAK,IAGpB0J,EAAMhJ,EAAU,CAAC,IAAM,IAAI4R,IAAItS,GAC3Ba,IAAWqC,KAAKumB,mBAAmBzpB,UAC9BypB,mBAAmBzpB,GAAK,IAAIwL,OAAO,IAAMtI,KAAKuO,SAAS/H,EAAK,IAAID,QAAQ,IAAK,QAAU,IAAK,UAC5F2I,oBAAoBpS,GAAK,IAAIwL,OAAO,IAAMtI,KAAKsO,cAAc9H,EAAK,IAAID,QAAQ,IAAK,QAAU,IAAK,UAClG4I,kBAAkBrS,GAAK,IAAIwL,OAAO,IAAMtI,KAAKqO,YAAY7H,EAAK,IAAID,QAAQ,IAAK,QAAU,IAAK,MAElGvG,KAAKiP,eAAenS,KACrBoL,EAAQ,IAAMlI,KAAKuO,SAAS/H,EAAK,IAAM,KAAOxG,KAAKsO,cAAc9H,EAAK,IAAM,KAAOxG,KAAKqO,YAAY7H,EAAK,SACpGyI,eAAenS,GAAK,IAAIwL,OAAOJ,EAAM3B,QAAQ,IAAK,IAAK,MAG5D5I,GAAqB,SAAXF,GAAqBuC,KAAKumB,mBAAmBzpB,GAAGiK,KAAKiI,UACxDlS,EACJ,GAAIa,GAAqB,QAAXF,GAAoBuC,KAAKkP,oBAAoBpS,GAAGiK,KAAKiI,UAC/DlS,EACJ,GAAIa,GAAqB,OAAXF,GAAmBuC,KAAKmP,kBAAkBrS,GAAGiK,KAAKiI,UAC5DlS,EACJ,IAAKa,GAAUqC,KAAKiP,eAAenS,GAAGiK,KAAKiI,UACvClS,IAm+EnB4nB,GAAQhW,uBAr7EgBtG,UAChBpI,KAAKsmB,qBACAppB,EAAW8C,KAAM,mBAClBwP,GAAqBnT,KAAK2D,MAE1BoI,EACOpI,KAAKgQ,qBAELhQ,KAAK6P,iBAGX3S,EAAW8C,KAAM,yBACb6P,eAAiBR,IAEnBrP,KAAKgQ,sBAAwB5H,EAChCpI,KAAKgQ,qBAAuBhQ,KAAK6P,iBAu6E7C6U,GAAQjW,4BAl6EqBrG,UACrBpI,KAAKsmB,qBACAppB,EAAW8C,KAAM,mBAClBwP,GAAqBnT,KAAK2D,MAE1BoI,EACOpI,KAAKiQ,0BAELjQ,KAAK8P,sBAGX5S,EAAW8C,KAAM,8BACb8P,oBAAsBR,IAExBtP,KAAKiQ,2BAA6B7H,EACrCpI,KAAKiQ,0BAA4BjQ,KAAK8P,sBAo5ElD4U,GAAQlW,0BA/4EmBpG,UACnBpI,KAAKsmB,qBACAppB,EAAW8C,KAAM,mBAClBwP,GAAqBnT,KAAK2D,MAE1BoI,EACOpI,KAAKkQ,wBAELlQ,KAAK+P,oBAGX7S,EAAW8C,KAAM,4BACb+P,kBAAoBR,IAEtBvP,KAAKkQ,yBAA2B9H,EACnCpI,KAAKkQ,wBAA0BlQ,KAAK+P,oBAk4EhD2U,GAAQ9T,cAtuEa5U,SAGgC,OAAxCA,EAAQ,IAAIsI,cAAckiB,OAAO,IAouE9C9B,GAAQ9lB,kBAhuEiBwR,EAAOE,EAASmW,UACjCrW,EAAQ,GACDqW,EAAU,KAAO,KAEjBA,EAAU,KAAO,MAwzEhCrT,GAAmB,KAAM,CACrBnB,uBAAwB,uBACxBlM,QAAU,SAAUxE,OACZnE,EAAImE,EAAS,UAKVA,GAJoC,IAA7BI,EAAMJ,EAAS,IAAM,IAAa,KACrC,IAANnE,EAAW,KACL,IAANA,EAAW,KACL,IAANA,EAAW,KAAO,SAO/BxB,EAAMujB,KAAOvc,EAAU,wDAAyDwQ,IAChFxX,EAAM8qB,SAAW9jB,EAAU,gEAAiE4Q,QAExFmT,GAAUnlB,KAAKc,aAmBVskB,GAAevL,EAAUrf,EAAO8F,EAAOwc,OACxCvD,EAAQ6C,GAAe5hB,EAAO8F,UAElCuZ,EAASW,eAAiBsC,EAAYvD,EAAMiB,cAC5CX,EAASY,OAAiBqC,EAAYvD,EAAMkB,MAC5CZ,EAASa,SAAiBoC,EAAYvD,EAAMmB,QAErCb,EAASe,mBAaXyK,GAAStlB,UACVA,EAAS,EACFC,KAAKE,MAAMH,GAEXC,KAAKC,KAAKF,YAmDhBulB,GAAcnL,UAGL,KAAPA,EAAc,gBAGhBoL,GAAcxc,UAEH,OAATA,EAAkB,cAkDpByc,GAAQC,UACN,kBACIjnB,KAAKknB,GAAGD,QAInBE,GAAiBH,GAAO,MACxBI,GAAiBJ,GAAO,KACxBK,GAAiBL,GAAO,KACxBM,GAAiBN,GAAO,KACxBO,GAAiBP,GAAO,KACxBQ,GAAiBR,GAAO,KACxBS,GAAiBT,GAAO,KACxBU,GAAiBV,GAAO,KACxBW,GAAiBX,GAAO,cAWnBY,GAAWnkB,UACT,kBACIzD,KAAKjB,UAAYiB,KAAKmc,MAAM1Y,GAAQ5D,SAI/C+b,GAAegM,GAAW,gBAC1BnX,GAAemX,GAAW,WAC1BtX,GAAesX,GAAW,WAC1BxX,GAAewX,GAAW,SAC1BjM,GAAeiM,GAAW,QAC1Brd,GAAeqd,GAAW,UAC1BtM,GAAesM,GAAW,SAM1BrL,GAAQ/a,KAAK+a,MACbsL,GAAa,CACbxV,GAAI,GACJxJ,EAAI,GACJ9K,EAAI,GACJyO,EAAI,GACJD,EAAI,GACJE,EAAI,aAICqb,GAAkB7Q,EAAQ1V,EAAQggB,EAAemE,EAAUhoB,UACzDA,EAAOwU,aAAa3Q,GAAU,IAAKggB,EAAetK,EAAQyO,YAG5DqC,GAAgBC,EAAgBzG,EAAe7jB,OAChD2d,EAAWuC,GAAeoK,GAAgB1lB,MAC1CmO,EAAW8L,GAAMlB,EAAS6L,GAAG,MAC7B5W,EAAWiM,GAAMlB,EAAS6L,GAAG,MAC7B9W,EAAWmM,GAAMlB,EAAS6L,GAAG,MAC7BvL,EAAWY,GAAMlB,EAAS6L,GAAG,MAC7B3c,EAAWgS,GAAMlB,EAAS6L,GAAG,MAC7B5L,EAAWiB,GAAMlB,EAAS6L,GAAG,MAE7B/pB,EAAIsT,GAAWoX,GAAWxV,IAAM,CAAC,IAAK5B,IAClCA,EAAUoX,GAAWhf,GAAO,CAAC,KAAM4H,IACnCH,GAAW,GAAiB,CAAC,MAC7BA,EAAUuX,GAAW9pB,GAAO,CAAC,KAAMuS,IACnCF,GAAW,GAAiB,CAAC,MAC7BA,EAAUyX,GAAWrb,GAAO,CAAC,KAAM4D,IACnCuL,GAAW,GAAiB,CAAC,MAC7BA,EAAUkM,GAAWtb,GAAO,CAAC,KAAMoP,IACnCpR,GAAW,GAAiB,CAAC,MAC7BA,EAAUsd,GAAWpb,GAAO,CAAC,KAAMlC,IACnC+Q,GAAW,GAAiB,CAAC,MAAkB,CAAC,KAAMA,UAE9Dne,EAAE,GAAKokB,EACPpkB,EAAE,IAAM6qB,EAAiB,EACzB7qB,EAAE,GAAKO,EACAoqB,GAAkBjsB,MAAM,KAAMsB,OA6CrC8qB,GAAQzmB,KAAKc,aAERqa,GAAKxS,UACDA,EAAI,IAAMA,EAAI,KAAQA,WAG1B+d,SAQAloB,KAAKjB,iBACCiB,KAAKkG,aAAaS,kBAMzB2J,EAASF,EAHTK,EAAUwX,GAAMjoB,KAAKgc,eAAiB,IACtCL,EAAesM,GAAMjoB,KAAKic,OAC1B1R,EAAe0d,GAAMjoB,KAAKkc,SAI9B5L,EAAoBhP,EAASmP,EAAU,IACvCL,EAAoB9O,EAASgP,EAAU,IACvCG,GAAW,GACXH,GAAW,OAQP6X,EALK7mB,EAASiJ,EAAS,IAMvBkC,EALJlC,GAAU,GAMN6d,EAAIzM,EACJnP,EAAI4D,EACJrS,EAAIuS,EACJzH,EAAI4H,EAAUA,EAAQ4X,QAAQ,GAAG9hB,QAAQ,SAAU,IAAM,GACzD+hB,EAAQtoB,KAAKonB,gBAEZkB,QAGM,UAGPC,EAAYD,EAAQ,EAAI,IAAM,GAC9BE,EAAS7L,GAAK3c,KAAKkc,WAAaS,GAAK2L,GAAS,IAAM,GACpDG,EAAW9L,GAAK3c,KAAKic,SAAWU,GAAK2L,GAAS,IAAM,GACpDI,EAAU/L,GAAK3c,KAAKgc,iBAAmBW,GAAK2L,GAAS,IAAM,UAExDC,EAAY,KACdJ,EAAIK,EAASL,EAAI,IAAM,KACvB1b,EAAI+b,EAAS/b,EAAI,IAAM,KACvB2b,EAAIK,EAAWL,EAAI,IAAM,KACxB5b,GAAKzO,GAAK8K,EAAK,IAAM,KACtB2D,EAAIkc,EAAUlc,EAAI,IAAM,KACxBzO,EAAI2qB,EAAU3qB,EAAI,IAAM,KACxB8K,EAAI6f,EAAU7f,EAAI,IAAM,QAG7B8f,GAAUvN,GAASjf,iBAEvBwsB,GAAQ5pB,0BA3tDGiB,KAAKhB,UA4tDhB2pB,GAAQrmB,mBAvVAiR,EAAiBvT,KAAKmc,kBAErBH,cAAgB2K,GAAQ3mB,KAAKgc,oBAC7BC,MAAgB0K,GAAQ3mB,KAAKic,YAC7BC,QAAgByK,GAAQ3mB,KAAKkc,SAElC3I,EAAKqI,aAAgB+K,GAAQpT,EAAKqI,cAClCrI,EAAK9C,QAAgBkW,GAAQpT,EAAK9C,SAClC8C,EAAKjD,QAAgBqW,GAAQpT,EAAKjD,SAClCiD,EAAKnD,MAAgBuW,GAAQpT,EAAKnD,OAClCmD,EAAKhJ,OAAgBoc,GAAQpT,EAAKhJ,QAClCgJ,EAAK+H,MAAgBqL,GAAQpT,EAAK+H,OAE3Btb,MA2UX2oB,GAAQ9N,aA7TQ7e,EAAO8F,UACZ8kB,GAAc5mB,KAAMhE,EAAO8F,EAAO,IA6T7C6mB,GAAQ/J,kBAzTa5iB,EAAO8F,UACjB8kB,GAAc5mB,KAAMhE,EAAO8F,GAAQ,IAyT9C6mB,GAAQzB,YApPK1iB,OACJxE,KAAKjB,iBACCc,QAEP8b,EACApR,EACAqR,EAAe5b,KAAKgc,iBAIV,WAFdxX,EAAQD,EAAeC,KAEY,YAAVA,GAAiC,SAAVA,SAC5CmX,EAAO3b,KAAKic,MAAQL,EAAe,MACnCrR,EAASvK,KAAKkc,QAAU4K,GAAanL,GAC7BnX,OACC,eAAkB+F,MAClB,iBAAkBA,EAAS,MAC3B,cAAkBA,EAAS,eAIpCoR,EAAO3b,KAAKic,MAAQza,KAAK+a,MAAMwK,GAAa/mB,KAAKkc,UACzC1X,OACC,cAAkBmX,EAAO,EAAQC,EAAe,WAChD,aAAkBD,EAAeC,EAAe,UAChD,cAAyB,GAAPD,EAAeC,EAAe,SAChD,gBAAyB,KAAPD,EAAeC,EAAe,QAChD,gBAAyB,MAAPD,EAAeC,EAAe,QAEhD,qBAAsBpa,KAAKE,MAAa,MAAPia,GAAgBC,gBACvC,IAAIxY,MAAM,gBAAkBoB,KAwNvDmkB,GAAQxB,eAAiBA,GACzBwB,GAAQvB,UAAiBA,GACzBuB,GAAQtB,UAAiBA,GACzBsB,GAAQrB,QAAiBA,GACzBqB,GAAQpB,OAAiBA,GACzBoB,GAAQnB,QAAiBA,GACzBmB,GAAQlB,SAAiBA,GACzBkB,GAAQjB,WAAiBA,GACzBiB,GAAQhB,QAAiBA,GACzBgB,GAAQprB,0BA1NCyC,KAAKjB,UAINiB,KAAKgc,cACQ,MAAbhc,KAAKic,MACJjc,KAAKkc,QAAU,GAAM,OACK,QAA3Bva,EAAM3B,KAAKkc,QAAU,IANdrc,KA0Nf8oB,GAAQvM,uBApTA3L,EAASH,EAASF,EAAOkL,EAAOsN,EAJhChN,EAAe5b,KAAKgc,cACpBL,EAAe3b,KAAKic,MACpB1R,EAAevK,KAAKkc,QACpB3I,EAAevT,KAAKmc,aAKjBP,GAAgB,GAAKD,GAAQ,GAAKpR,GAAU,GAC1CqR,GAAgB,GAAKD,GAAQ,GAAKpR,GAAU,IACjDqR,GAAuD,MAAvCiL,GAAQE,GAAaxc,GAAUoR,GAC/CA,EAAO,EACPpR,EAAS,GAKbgJ,EAAKqI,aAAeA,EAAe,IAEnCnL,EAAoBnP,EAASsa,EAAe,KAC5CrI,EAAK9C,QAAeA,EAAU,GAE9BH,EAAoBhP,EAASmP,EAAU,IACvC8C,EAAKjD,QAAeA,EAAU,GAE9BF,EAAoB9O,EAASgP,EAAU,IACvCiD,EAAKnD,MAAeA,EAAQ,GAE5BuL,GAAQra,EAAS8O,EAAQ,IAGzBwY,EAAiBtnB,EAASwlB,GAAanL,IACvCpR,GAAUqe,EACVjN,GAAQkL,GAAQE,GAAa6B,IAG7BtN,EAAQha,EAASiJ,EAAS,IAC1BA,GAAU,GAEVgJ,EAAKoI,KAASA,EACdpI,EAAKhJ,OAASA,EACdgJ,EAAK+H,MAASA,EAEPtb,MA8QX2oB,GAAQvL,wBAhMGQ,GAAe5d,OAiM1B2oB,GAAQ5e,aA9LQvF,UACZA,EAAQD,EAAeC,GAChBxE,KAAKjB,UAAYiB,KAAKwE,EAAQ,OAAS3E,KA6LlD8oB,GAAQ/M,aAAiBA,GACzB+M,GAAQlY,QAAiBA,GACzBkY,GAAQrY,QAAiBA,GACzBqY,GAAQvY,MAAiBA,GACzBuY,GAAQhN,KAAiBA,GACzBgN,GAAQlN,wBAhLGna,EAAStB,KAAK2b,OAAS,IAiLlCgN,GAAQpe,OAAiBA,GACzBoe,GAAQrN,MAAiBA,GACzBqN,GAAQnH,kBA3GWqH,OACV7oB,KAAKjB,iBACCiB,KAAKkG,aAAaS,kBAGzBjJ,EAASsC,KAAKkG,aACdO,EAASshB,GAAe/nB,MAAO6oB,EAAYnrB,UAE3CmrB,IACApiB,EAAS/I,EAAOkoB,YAAY5lB,KAAMyG,IAG/B/I,EAAO4jB,WAAW7a,IAgG7BkiB,GAAQ9F,YAAiBqF,GACzBS,GAAQvsB,SAAiB8rB,GACzBS,GAAQvF,OAAiB8E,GACzBS,GAAQjrB,OAAiBA,GACzBirB,GAAQziB,WAAiBA,GAEzByiB,GAAQG,YAAclmB,EAAU,sFAAuFslB,IACvHS,GAAQxJ,KAAOA,GAMfvZ,EAAe,IAAK,EAAG,EAAG,QAC1BA,EAAe,IAAK,EAAG,EAAG,WAI1BqC,GAAc,IAAKL,IACnBK,GAAc,IAlzHO,wBAmzHrBc,GAAc,KAAK,SAAU/M,EAAOqK,EAAOnF,GACvCA,EAAO7B,GAAK,IAAI3C,KAA6B,IAAxBof,WAAW9f,EAAO,QAE3C+M,GAAc,KAAK,SAAU/M,EAAOqK,EAAOnF,GACvCA,EAAO7B,GAAK,IAAI3C,KAAKiF,EAAM3F,OAM/BJ,EAAMmtB,QAAU,SAz7IZrtB,EA27IYia,GAEhB/Z,EAAMiB,GAAwB2jB,GAC9B5kB,EAAMwG,mBAj0DEY,EAAO,GAAGE,MAAM7G,KAAKP,UAAW,UAE7Bmf,GAAO,WAAYjY,IAg0D9BpH,EAAM0J,mBA5zDEtC,EAAO,GAAGE,MAAM7G,KAAKP,UAAW,UAE7Bmf,GAAO,UAAWjY,IA2zD7BpH,EAAMiZ,IAxzDI,kBACCnY,KAAKmY,IAAMnY,KAAKmY,OAAU,IAAInY,MAwzDzCd,EAAMiC,IAAwBL,EAC9B5B,EAAMynB,cAnjBernB,UACV2Z,GAAoB,IAAR3Z,IAmjBvBJ,EAAM2O,gBAjce9M,EAAQmnB,UAClBG,GAAetnB,EAAQmnB,EAAO,WAiczChpB,EAAMa,OAAwBA,EAC9Bb,EAAM8B,OAAwB0V,GAC9BxX,EAAM+iB,QAAwB/e,EAC9BhE,EAAMyf,SAAwBuC,GAC9BhiB,EAAMwF,SAAwBA,EAC9BxF,EAAM2S,kBA/biB2W,EAAcznB,EAAQmnB,UAClCK,GAAiBC,EAAcznB,EAAQmnB,EAAO,aA+bzDhpB,EAAMooB,4BAtjBKrO,GAAY9Z,MAAM,KAAMC,WAAWkoB,aAujB9CpoB,EAAMsK,WAAwBsN,GAC9B5X,EAAMygB,WAAwBA,GAC9BzgB,EAAM0O,qBAvcoB7M,EAAQmnB,UACvBG,GAAetnB,EAAQmnB,EAAO,gBAuczChpB,EAAMyS,qBA5boB6W,EAAcznB,EAAQmnB,UACrCK,GAAiBC,EAAcznB,EAAQmnB,EAAO,gBA4bzDhpB,EAAM6X,aAAwBA,GAC9B7X,EAAMotB,sBA9kFgBvlB,EAAMvC,MACV,MAAVA,EAAgB,KACZxD,EAAQurB,EAAWplB,EAAesN,GAGrB,OADjB8X,EAAYlW,GAAWtP,MAEnBI,EAAeolB,EAAUtV,UAG7BjW,EAAS,IAAIqG,EADb7C,EAAS0C,EAAaC,EAAc3C,KAE7B0S,aAAehB,GAAQnP,GAC9BmP,GAAQnP,GAAQ/F,EAGhB0V,GAAmB3P,QAGE,MAAjBmP,GAAQnP,KAC0B,MAA9BmP,GAAQnP,GAAMmQ,aACdhB,GAAQnP,GAAQmP,GAAQnP,GAAMmQ,aACN,MAAjBhB,GAAQnP,WACRmP,GAAQnP,WAIpBmP,GAAQnP,IAsjFnB7H,EAAMgX,0BA1hFKtP,EAAKsP,KA2hFhBhX,EAAM0S,uBApcsB4W,EAAcznB,EAAQmnB,UACvCK,GAAiBC,EAAcznB,EAAQmnB,EAAO,kBAoczDhpB,EAAM2I,eAAwBA,EAC9B3I,EAAMstB,8BA/L+BC,eACR1pB,IAArB0pB,EACO5M,GAEsB,mBAAtB4M,IACP5M,GAAQ4M,GACD,IA0LfvtB,EAAMwtB,+BApLgCC,EAAWC,eACf7pB,IAA1BooB,GAAWwB,UAGD5pB,IAAV6pB,EACOzB,GAAWwB,IAEtBxB,GAAWwB,GAAaC,EACN,MAAdD,IACAxB,GAAWxV,GAAKiX,EAAQ,IAErB,KA0KX1tB,EAAMklB,wBA73CqByI,EAAU1U,OAC7BsI,EAAOoM,EAASpM,KAAKtI,EAAK,QAAQ,UAC/BsI,GAAQ,EAAI,WACXA,GAAQ,EAAI,WACZA,EAAO,EAAI,UACXA,EAAO,EAAI,UACXA,EAAO,EAAI,UACXA,EAAO,EAAI,WAAa,YAu3CpCvhB,EAAMO,UAAwBqkB,GAG9B5kB,EAAM4tB,UAAY,CACdC,eAAgB,mBAChBC,uBAAwB,sBACxBC,kBAAmB,0BACnBC,KAAM,aACNC,KAAM,QACNC,aAAc,WACdC,QAAS,eACTC,KAAM,aACNC,MAAO,WAGJruB,EAp/IyEsuB,MCHhFC,qBCOK,SAAUC,YAGNC,EAAoB9oB,EAAQggB,EAAete,EAAKyiB,OACjDjoB,EAAS,GACJ,CAAC,cAAe,kBAChB,CAAC,cAAe,kBAChB,CAAC,UAAW,gBACX,CAAC8D,EAAS,QAASA,EAAS,YAC7B,CAAC,YAAa,kBACb,CAACA,EAAS,UAAWA,EAAS,cAC/B,CAAC,WAAY,iBACZ,CAACA,EAAS,SAAUA,EAAS,mBAEhCggB,EAAgB9jB,EAAOwF,GAAK,GAAKxF,EAAOwF,GAAK,GAG/CmnB,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,qFAAqFM,MAAM,KACpGP,YAAc,6DAA6DO,MAAM,KACjFyf,kBAAmB,EACnB/b,SAAW,8DAA8D1D,MAAM,KAC/EyD,cAAgB,8BAA8BzD,MAAM,KACpDwD,YAAc,uBAAuBxD,MAAM,KAC3C0f,oBAAqB,EACrB1jB,eAAiB,CACb+K,GAAI,QACJD,IAAK,WACLE,EAAI,aACJC,GAAK,eACLC,IAAM,qBACNC,KAAO,4BAEXZ,SAAW,CACPC,QAAS,sBACTK,SAAU,IACVJ,QAAS,uBACTC,SAAU,qBACVC,QAAS,wBACTC,SAAU,gCAEdS,aAAe,CACXC,OAAS,QACTC,KAAO,SACPvJ,EAAI,oBACJwJ,GAAK,cACLtU,EAAIssB,EACJ/X,GAAK,aACL9F,EAAI6d,EACJ9X,GAAK,aACLhG,EAAI8d,EACJ7X,GAAK6X,EACL5d,EAAI4d,EACJ5X,GAAK4X,EACL9gB,EAAI8gB,EACJ3X,GAAK2X,GAETpY,uBAAwB,YACxBlM,QAAU,MACVwH,KAAO,CACHL,IAAM,EACNC,IAAM,MAhEwB+c,CAAQM,wBCGzC,SAAUJ,OAGXK,EAAiB,8DAA8D5f,MAAM,KACrFP,EAAc,kDAAkDO,MAAM,KAEtEH,EAAc,CAAC,QAAS,QAAS,QAAS,QAAS,QAAS,QAAS,QAAS,QAAS,QAAS,QAAS,QAAS,SAClHD,EAAc,mLAET2f,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,2FAA2FM,MAAM,KAC1GP,YAAc,SAAUvM,EAAGN,UAClBM,EAEM,QAAQgJ,KAAKtJ,GACb6M,EAAYvM,EAAEiM,SAEdygB,EAAe1sB,EAAEiM,SAJjBygB,GAOfhgB,YAAcA,EACdD,iBAAmBC,EACnBigB,kBAAoB,+FACpBC,uBAAyB,0FACzBjgB,YAAcA,EACdkgB,gBAAkBlgB,EAClBmgB,iBAAmBngB,EACnB6D,SAAW,uDAAuD1D,MAAM,KACxEyD,cAAgB,qCAAqCzD,MAAM,KAC3DwD,YAAc,uBAAuBxD,MAAM,KAC3C0f,oBAAqB,EACrB1jB,eAAiB,CACb+K,GAAK,OACLD,IAAM,UACNE,EAAI,aACJC,GAAK,wBACLC,IAAM,6BACNC,KAAO,oCAEXZ,SAAW,CACPC,QAAU,iBACC,aAAiC,IAAjBrR,KAAKoQ,QAAiB,IAAM,IAAM,QAE7DkB,QAAU,iBACC,gBAAoC,IAAjBtR,KAAKoQ,QAAiB,IAAM,IAAM,QAEhEmB,SAAW,iBACA,cAAkC,IAAjBvR,KAAKoQ,QAAiB,IAAM,IAAM,QAE9DoB,QAAU,iBACC,cAAkC,IAAjBxR,KAAKoQ,QAAiB,IAAM,IAAM,QAE9DqB,SAAW,iBACA,0BAA8C,IAAjBzR,KAAKoQ,QAAiB,IAAM,IAAM,QAE1EsB,SAAW,KAEfQ,aAAe,CACXC,OAAS,QACTC,KAAO,UACPvJ,EAAI,gBACJwJ,GAAK,cACLtU,EAAI,YACJuU,GAAK,aACL9F,EAAI,WACJ+F,GAAK,WACLhG,EAAI,SACJiG,GAAK,UACL/F,EAAI,SACJgG,GAAK,WACLlJ,EAAI,SACJmJ,GAAK,WAETT,uBAAyB,WACzBlM,QAAU,MACVwH,KAAO,CACHL,IAAM,EACNC,IAAM,MAhFwB+c,CAAQM,wBCGzC,SAAUJ,GAGNA,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,uFAAuFM,MAAM,KACtGP,YAAc,iEAAiEO,MAAM,KACrFyf,kBAAmB,EACnB/b,SAAW,sDAAsD1D,MAAM,KACvEyD,cAAgB,qCAAqCzD,MAAM,KAC3DwD,YAAc,uBAAuBxD,MAAM,KAC3C0f,oBAAqB,EACrB1jB,eAAiB,CACb+K,GAAK,QACLD,IAAM,WACNE,EAAI,aACJC,GAAK,cACLC,IAAM,oBACNC,KAAO,0BAEXZ,SAAW,CACPC,QAAU,qBACVC,QAAU,gBACVC,SAAW,cACXC,QAAU,cACVC,SAAW,sBACXC,SAAW,KAEfQ,aAAe,CACXC,OAAS,UACTC,KAAO,YACPvJ,EAAI,oBACJwJ,GAAK,cACLtU,EAAI,aACJuU,GAAK,aACL9F,EAAI,YACJ+F,GAAK,YACLhG,EAAI,UACJiG,GAAK,WACL/F,EAAI,UACJgG,GAAK,UACLlJ,EAAI,QACJmJ,GAAK,UAETT,uBAAwB,eACxBlM,QAAU,SAAUxE,EAAQgd,UAChBA,OAIC,WACMhd,GAAqB,IAAXA,EAAe,KAAO,gBAItC,QACA,QACA,UACA,WACMA,GAAqB,IAAXA,EAAe,KAAO,SAGtC,QACA,WACMA,GAAqB,IAAXA,EAAe,KAAO,OAGnDgM,KAAO,CACHL,IAAM,EACNC,IAAM,MAvEwB+c,CAAQM,wBCGzC,SAAUJ,GAGNA,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,gGAAgGM,MAAM,KAC/GP,YAAc,kDAAkDO,MAAM,KACtE0D,SAAW,2DAA2D1D,MAAM,KAC5EyD,cAAgB,8BAA8BzD,MAAM,KACpDwD,YAAc,uBAAuBxD,MAAM,KAC3ChE,eAAiB,CACb+K,GAAK,QACLD,IAAM,WACNE,EAAI,aACJC,GAAK,cACLC,IAAM,oBACNC,KAAO,0BAEXZ,SAAW,CACPC,QAAS,iBACTC,QAAS,mBACTC,SAAU,iBACVC,QAAS,iBACTC,SAAU,kBACEzR,KAAKoP,YACJ,QACM,2CAEA,+BAGnBsC,SAAU,KAEdQ,aAAe,CACXC,OAAS,SAAUtJ,UACN,YAAa9B,KAAK8B,GAAK,MAAQ,MAAQ,IAAMA,GAE1DuJ,KAAO,QACPvJ,EAAI,iBACJwJ,GAAK,aACLtU,EAAI,YACJuU,GAAK,YACL9F,EAAI,SACJ+F,GAAK,SACLhG,EAAI,YACJiG,GAAK,YACL/F,EAAI,UACJgG,GAAK,UACLlJ,EAAI,UACJmJ,GAAK,WAETT,uBAAyB,WACzBlM,QAAS,MACTwH,KAAO,CACHL,IAAM,EACNC,IAAM,MAzDwB+c,CAAQM,wBCGzC,SAAUJ,GAGNA,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,wCAAwCM,MAAM,KACvDP,YAAc,yCAAyCO,MAAM,KAC7D0D,SAAW,8BAA8B1D,MAAM,KAC/CyD,cAAgB,gBAAgBzD,MAAM,KACtCwD,YAAc,gBAAgBxD,MAAM,KACpChE,eAAiB,CACb+K,GAAK,QACLD,IAAM,WACNE,EAAI,aACJC,GAAK,YACLC,IAAM,kBACNC,KAAO,uBACP4E,EAAI,aACJkU,GAAK,YACLC,IAAM,kBACNC,KAAO,wBAEXrY,cAAe,SACf/B,KAAO,SAAU5U,SACI,OAAVA,GAEX4C,SAAW,SAAU0a,EAAMc,EAAQqM,UAC3BnN,EAAO,GACA,KAEA,MAGflI,SAAW,CACPC,QAAU,UACVC,QAAU,UACVC,SAAW,SAAUsD,UACbA,EAAItH,OAASvN,KAAKuN,OACX,cAEA,WAGfiE,QAAU,UACVC,SAAW,SAAUoD,UACb7U,KAAKuN,OAASsH,EAAItH,OACX,cAEA,WAGfmE,SAAW,KAEfO,uBAAyB,WACzBlM,QAAU,SAAUxE,EAAQgd,UAChBA,OACC,QACA,QACA,aACMhd,EAAS,mBAETA,IAGnB2Q,aAAe,CACXC,OAAS,MACTC,KAAO,MACPvJ,EAAI,KACJwJ,GAAK,MACLtU,EAAI,KACJuU,GAAK,MACL9F,EAAI,MACJ+F,GAAK,OACLhG,EAAI,KACJiG,GAAK,MACL/F,EAAI,MACJgG,GAAK,OACLlJ,EAAI,KACJmJ,GAAK,UAhFyBwX,CAAQM,wBCGzC,SAAUJ,OAGXa,EAAsB,6DAA6DpgB,MAAM,KACzFqgB,EAAyB,kDAAkDrgB,MAAM,KAEjFH,EAAc,CAAC,QAAS,QAAS,iBAAkB,QAAS,SAAU,cAAe,cAAe,QAAS,QAAS,QAAS,QAAS,SACxID,EAAc,qKAET2f,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,0FAA0FM,MAAM,KACzGP,YAAc,SAAUvM,EAAGN,UAClBM,EAEM,QAAQgJ,KAAKtJ,GACbytB,EAAuBntB,EAAEiM,SAEzBihB,EAAoBltB,EAAEiM,SAJtBihB,GAQfxgB,YAAaA,EACbD,iBAAkBC,EAClBigB,kBAAmB,4FACnBC,uBAAwB,mFAExBjgB,YAAcA,EACdkgB,gBAAkBlgB,EAClBmgB,iBAAmBngB,EAEnB6D,SAAW,6DAA6D1D,MAAM,KAC9EyD,cAAgB,8BAA8BzD,MAAM,KACpDwD,YAAc,uBAAuBxD,MAAM,KAC3C0f,oBAAqB,EACrB1jB,eAAiB,CACb+K,GAAK,QACLD,IAAM,WACNE,EAAI,aACJC,GAAK,cACLC,IAAM,oBACNC,KAAO,0BAEXZ,SAAW,CACPC,QAAS,kBACTC,QAAS,iBACTC,SAAU,eACVC,QAAS,mBACTC,SAAU,2BACVC,SAAU,KAEdQ,aAAe,CACXC,OAAS,UACTC,KAAO,aACPvJ,EAAI,oBACJwJ,GAAK,cACLtU,EAAI,aACJuU,GAAK,aACL9F,EAAI,UACJ+F,GAAK,SACLhG,EAAI,UACJiG,GAAK,WACL/F,EAAI,YACJgG,GAAK,aACLlJ,EAAI,WACJmJ,GAAK,WAETT,uBAAwB,kBACxBlM,QAAU,SAAUxE,UACTA,GAAsB,IAAXA,GAA2B,IAAXA,GAAgBA,GAAU,GAAM,MAAQ,OAE9EgM,KAAO,CACHL,IAAM,EACNC,IAAM,MA3EwB+c,CAAQM,wBCGzC,SAAUJ,OAGXe,EAAmB,mGAAmGtgB,MAAM,KAC5HugB,EAAmB,qGAAqGvgB,MAAM,cACzHwgB,EAAOld,UACJA,EAAI,GAAK,GAAOA,EAAI,GAAK,MAAWA,EAAI,IAAM,IAAQ,WAEzDmd,EAAU/pB,EAAQggB,EAAete,OAClCsV,EAAShX,EAAS,WACd0B,OACC,YACMsV,GAAU8S,EAAO9pB,GAAU,UAAY,cAC7C,WACMggB,EAAgB,SAAW,aACjC,YACMhJ,GAAU8S,EAAO9pB,GAAU,SAAW,aAC5C,WACMggB,EAAiB,UAAa,cACpC,YACMhJ,GAAU8S,EAAO9pB,GAAU,UAAY,cAC7C,YACMgX,GAAU8S,EAAO9pB,GAAU,WAAa,gBAC9C,YACMgX,GAAU8S,EAAO9pB,GAAU,OAAS,QAI9C6oB,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,SAAUghB,EAAgB9tB,UAC1B8tB,EAEiB,KAAX9tB,EAIA,IAAM2tB,EAAiBG,EAAevhB,SAAW,IAAMmhB,EAAiBI,EAAevhB,SAAW,IAClG,SAASjD,KAAKtJ,GACd2tB,EAAiBG,EAAevhB,SAEhCmhB,EAAiBI,EAAevhB,SAThCmhB,GAYf7gB,YAAc,kDAAkDO,MAAM,KACtE0D,SAAW,6DAA6D1D,MAAM,KAC9EyD,cAAgB,2BAA2BzD,MAAM,KACjDwD,YAAc,uBAAuBxD,MAAM,KAC3ChE,eAAiB,CACb+K,GAAK,QACLD,IAAM,WACNE,EAAI,aACJC,GAAK,cACLC,IAAM,oBACNC,KAAO,2BAEXZ,SAAW,CACPC,QAAS,cACTC,QAAS,eACTC,SAAU,kBACEvR,KAAKoP,YACJ,QACM,0BAEN,QACM,wBAEN,QACM,sBAEN,QACM,gCAGA,oBAGnBoC,QAAS,iBACTC,SAAU,kBACEzR,KAAKoP,YACJ,QACM,iCACN,QACM,6BACN,QACM,uCAEA,2BAGnBsC,SAAU,KAEdQ,aAAe,CACXC,OAAS,QACTC,KAAO,UACPvJ,EAAI,eACJwJ,GAAKiZ,EACLvtB,EAAIutB,EACJhZ,GAAKgZ,EACL9e,EAAI8e,EACJ/Y,GAAK+Y,EACL/e,EAAI,UACJiG,GAAK,SACL/F,EAAI,UACJgG,GAAK6Y,EACL/hB,EAAI,MACJmJ,GAAK4Y,GAETrZ,uBAAwB,YACxBlM,QAAU,MACVwH,KAAO,CACHL,IAAM,EACNC,IAAM,MAlHwB+c,CAAQM,wBCGzC,SAAUJ,YAONoB,EAAuBjqB,EAAQggB,EAAete,OAJjCwoB,EACdC,QAYQ,MAARzoB,EACOse,EAAgB,SAAW,SAG3BhgB,EAAS,KAjBFkqB,GAiB6BlqB,EAhB3CmqB,EAIS,IACHnK,EAAgB,yBAA2B,4BAC3CA,EAAgB,sBAAwB,yBACxC,oBACA,mBACA,0BACA,gBAM8Bte,GAhBvB4H,MAAM,KAChB4gB,EAAM,IAAO,GAAKA,EAAM,KAAQ,GAAKC,EAAM,GAAMD,EAAM,IAAM,GAAKA,EAAM,IAAM,IAAMA,EAAM,IAAM,IAAMA,EAAM,KAAO,IAAMC,EAAM,GAAKA,EAAM,QAkBjJhhB,EAAc,CAAC,QAAS,QAAS,QAAS,QAAS,WAAY,QAAS,QAAS,QAAS,QAAS,QAAS,QAAS,SAKhH0f,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,CACL9M,OAAQ,oFAAoFoN,MAAM,KAClG8gB,WAAY,kFAAkF9gB,MAAM,MAExGP,YAAc,CAEV7M,OAAQ,gEAAgEoN,MAAM,KAC9E8gB,WAAY,gEAAgE9gB,MAAM,MAEtF0D,SAAW,CACPod,WAAY,gEAAgE9gB,MAAM,KAClFpN,OAAQ,gEAAgEoN,MAAM,KAC9Eib,SAAU,kDAEdxX,cAAgB,uBAAuBzD,MAAM,KAC7CwD,YAAc,uBAAuBxD,MAAM,KAC3CH,YAAcA,EACdkgB,gBAAkBlgB,EAClBmgB,iBAAmBngB,EAGnBD,YAAa,2MAGbD,iBAAkB,2MAGlBkgB,kBAAmB,wHAGnBC,uBAAwB,6FACxB9jB,eAAiB,CACb+K,GAAK,OACLD,IAAM,UACNE,EAAI,aACJC,GAAK,iBACLC,IAAM,uBACNC,KAAO,8BAEXZ,SAAW,CACPC,QAAS,kBACTC,QAAS,iBACTE,QAAS,gBACTD,SAAU,SAAUsD,MACZA,EAAItH,SAAWvN,KAAKuN,cAcD,IAAfvN,KAAKoP,MACE,oBAEA,0BAhBHpP,KAAKoP,YACJ,QACM,kCACN,OACA,OACA,QACM,kCACN,OACA,OACA,QACM,+BAUvBqC,SAAU,SAAUoD,MACZA,EAAItH,SAAWvN,KAAKuN,cAcD,IAAfvN,KAAKoP,MACE,oBAEA,0BAhBHpP,KAAKoP,YACJ,QACM,gCACN,OACA,OACA,QACM,gCACN,OACA,OACA,QACM,6BAUvBsC,SAAU,KAEdQ,aAAe,CACXC,OAAS,WACTC,KAAO,WACPvJ,EAAI,mBACJwJ,GAAKmZ,EACLztB,EAAIytB,EACJlZ,GAAKkZ,EACLhf,EAAI,MACJ+F,GAAKiZ,EACLjf,EAAI,OACJiG,GAAKgZ,EACL/e,EAAI,QACJgG,GAAK+Y,EACLjiB,EAAI,MACJmJ,GAAK8Y,GAET7Y,cAAe,wBACf/B,KAAO,SAAU5U,SACN,iBAAiB+K,KAAK/K,IAEjC4C,SAAW,SAAU0a,EAAMc,EAAQqM,UAC3BnN,EAAO,EACA,OACAA,EAAO,GACP,OACAA,EAAO,GACP,MAEA,UAGfrH,uBAAwB,mBACxBlM,QAAS,SAAUxE,EAAQgd,UACfA,OACC,QACA,QACA,aACMhd,EAAS,SACf,WACMA,EAAS,UACf,QACA,WACMA,EAAS,oBAETA,IAGnBgM,KAAO,CACHL,IAAM,EACNC,IAAM,MA1KwB+c,CAAQM,wBCGzC,SAAUJ,YAONoB,EAAuBjqB,EAAQggB,EAAete,OAJjCwoB,EACdC,QAYQ,MAARzoB,EACOse,EAAgB,UAAY,UAEtB,MAARte,EACEse,EAAgB,SAAW,SAG3BhgB,EAAS,KApBFkqB,GAoB6BlqB,EAnB3CmqB,EAIS,IACHnK,EAAgB,yBAA2B,4BAC3CA,EAAgB,yBAA2B,4BAC3CA,EAAgB,sBAAwB,yBACxC,mBACA,2BACA,kBAS8Bte,GAnBvB4H,MAAM,KAChB4gB,EAAM,IAAO,GAAKA,EAAM,KAAQ,GAAKC,EAAM,GAAMD,EAAM,IAAM,GAAKA,EAAM,IAAM,IAAMA,EAAM,IAAM,IAAMA,EAAM,KAAO,IAAMC,EAAM,GAAKA,EAAM,aA0C5IE,EAAqBC,UACnB,kBACIA,EAAM,KAAwB,KAAjB7rB,KAAKoQ,QAAiB,IAAM,IAAM,QAIrDga,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,QACK,yFAAyFM,MAAM,gBAC3F,iGAAiGA,MAAM,MAEzHP,YAAc,yDAAyDO,MAAM,KAC7E0D,kBAjCyBxQ,EAAGN,OACxB8Q,EAAW,YACG,0DAA0D1D,MAAM,gBAChE,0DAA0DA,MAAM,cAClE,4DAA4DA,MAAM,aAGxE,IAAN9M,EACOwQ,EAAQ,WAAerL,MAAM,EAAG,GAAGkL,OAAOG,EAAQ,WAAerL,MAAM,EAAG,IAEhFnF,EASEwQ,EALS,qBAAsBxH,KAAKtJ,GACvC,aACE,sCAAuCsJ,KAAKtJ,GAC1C,WACA,cACkBM,EAAEqR,OARjBb,EAAQ,YAuBnBD,cAAgB,uBAAuBzD,MAAM,KAC7CwD,YAAc,uBAAuBxD,MAAM,KAC3ChE,eAAiB,CACb+K,GAAK,QACLD,IAAM,WACNE,EAAI,aACJC,GAAK,iBACLC,IAAM,wBACNC,KAAO,+BAEXZ,SAAW,CACPC,QAASua,EAAqB,cAC9Bta,QAASsa,EAAqB,YAC9Bpa,QAASoa,EAAqB,WAC9Bra,SAAUqa,EAAqB,cAC/Bna,SAAU,kBACEzR,KAAKoP,YACJ,OACA,OACA,OACA,SACMwc,EAAqB,oBAAoBvvB,KAAK2D,WACpD,OACA,OACA,SACM4rB,EAAqB,qBAAqBvvB,KAAK2D,QAGlE0R,SAAU,KAEdQ,aAAe,CACXC,OAAS,QACTC,KAAO,UACPvJ,EAAI,kBACJwJ,GAAKmZ,EACLztB,EAAIytB,EACJlZ,GAAKkZ,EACLhf,EAAI,SACJ+F,GAAKiZ,EACLjf,EAAI,OACJiG,GAAKgZ,EACL/e,EAAI,SACJgG,GAAK+Y,EACLjiB,EAAI,MACJmJ,GAAK8Y,GAGT7Y,cAAe,wBACf/B,KAAM,SAAU5U,SACL,iBAAiB+K,KAAK/K,IAEjC4C,SAAW,SAAU0a,EAAMc,EAAQqM,UAC3BnN,EAAO,EACA,OACAA,EAAO,GACP,QACAA,EAAO,GACP,MAEA,UAGfrH,uBAAwB,iBACxBlM,QAAS,SAAUxE,EAAQgd,UACfA,OACC,QACA,QACA,UACA,QACA,WACMhd,EAAS,SACf,WACMA,EAAS,qBAETA,IAGnBgM,KAAO,CACHL,IAAM,EACNC,IAAM,MA9IwB+c,CAAQM,MTJtC,SAAUsB,UACbA,GAAMA,EAAGtqB,MAAQA,MAAQsqB,MAMhC3B,EAA2B,iBAAd4B,YAA0BA,aACvC5B,EAAuB,iBAAV6B,QAAsBA,SACnC7B,EAAqB,iBAAR8B,MAAoBA,OACjC9B,EAAuB,iBAAV+B,GAAsBA,IAEnCvoB,SAAS,cAATA,KUZe,SAAUuT,eAEdA,IACT,MAAOiV,UACA,OCDOC,GAAM,kBACwD,GAAvElwB,OAAOmwB,eAAe,GAAI,EAAG,CAAEtiB,IAAK,kBAAqB,KAAQ,MCHtEuiB,EAA6B,GAAGC,qBAChCC,EAA2BtwB,OAAOswB,8BAGpBA,IAA6BF,EAA2BjwB,KAAK,GAAK,GAAK,GAI/D,SAA8BowB,OAClDC,EAAaF,EAAyBxsB,KAAMysB,WACvCC,GAAcA,EAAWC,YAChCL,KCZa,SAAUM,EAAQ9qB,SAC1B,CACL6qB,aAAuB,EAATC,GACdC,eAAyB,EAATD,GAChBE,WAAqB,EAATF,GACZ9qB,MAAOA,ICLP1F,EAAW,GAAGA,WAED,SAAU0vB,UAClB1vB,EAASC,KAAKyvB,GAAI5oB,MAAM,GAAI,ICAjC2H,EAAQ,GAAGA,QAGEuhB,GAAM,kBAGblwB,OAAO,KAAKqwB,qBAAqB,MACtC,SAAUT,SACS,UAAfiB,EAAQjB,GAAkBjhB,EAAMxO,KAAKyvB,EAAI,IAAM5vB,OAAO4vB,IAC3D5vB,SCVa,SAAU4vB,MACfrsB,MAANqsB,EAAiB,MAAMkB,UAAU,wBAA0BlB,UACxDA,KCAQ,SAAUA,UAClBmB,EAAcC,EAAuBpB,OCL7B,SAAUA,SACJ,iBAAPA,EAAyB,OAAPA,EAA4B,mBAAPA,KCKtC,SAAU9vB,EAAOmxB,OAC3B7wB,EAASN,GAAQ,OAAOA,MACzBa,EAAI0D,KACJ4sB,GAAoD,mBAAxBtwB,EAAKb,EAAMI,YAA4BE,EAASiE,EAAM1D,EAAGR,KAAKL,IAAS,OAAOuE,KAC3E,mBAAvB1D,EAAKb,EAAMuB,WAA2BjB,EAASiE,EAAM1D,EAAGR,KAAKL,IAAS,OAAOuE,MACpF4sB,GAAoD,mBAAxBtwB,EAAKb,EAAMI,YAA4BE,EAASiE,EAAM1D,EAAGR,KAAKL,IAAS,OAAOuE,QACzGysB,UAAU,4CCZd3vB,EAAiB,GAAGA,iBAEP,SAAUyuB,EAAI7oB,UACtB5F,EAAehB,KAAKyvB,EAAI7oB,ICA7BmqB,EAAWlB,EAAOkB,SAElBC,EAAS/wB,EAAS8wB,IAAa9wB,EAAS8wB,EAASE,iBAEpC,SAAUxB,UAClBuB,EAASD,EAASE,cAAcxB,GAAM,OCH7ByB,IAAgBnB,GAAM,kBAG9B,GAFDlwB,OAAOmwB,eAAeiB,EAAc,OAAQ,IAAK,CACtDvjB,IAAK,kBAAqB,KACzB5M,KCADqwB,EAAiCtxB,OAAOswB,8BAIhCe,EAAcC,EAAiC,SAAkCC,EAAGC,MAC9FD,EAAIE,EAAgBF,GACpBC,EAAIE,EAAYF,GAAG,GACfG,EAAgB,WACXL,EAA+BC,EAAGC,GACzC,MAAOvB,OACL2B,EAAIL,EAAGC,GAAI,OAAOK,GAA0BC,EAA2BC,EAAE5xB,KAAKoxB,EAAGC,GAAID,EAAEC,MChBzFQ,EAAc,kBAEdC,EAAW,SAAUC,EAASC,OAC5BvsB,EAAQyR,EAAK+a,EAAUF,WACpBtsB,GAASysB,GACZzsB,GAAS0sB,IACW,mBAAbH,EAA0BjC,EAAMiC,KACrCA,IAGJC,EAAYH,EAASG,UAAY,SAAUrX,UACtCwX,OAAOxX,GAAQ1Q,QAAQ2nB,EAAa,KAAK5pB,eAG9CiP,EAAO4a,EAAS5a,KAAO,GACvBib,EAASL,EAASK,OAAS,IAC3BD,EAAWJ,EAASI,SAAW,MAElBJ,ICpBA,KCAA,SAAUrC,MACR,mBAANA,QACHkB,UAAUyB,OAAO3C,GAAM,6BACtBA,KCAM,SAAUjvB,EAAImkB,EAAMhkB,MACnC0xB,EAAU7xB,QACG4C,IAATuhB,EAAoB,OAAOnkB,SACvBG,QACD,SAAU,kBACNH,EAAGR,KAAK2kB,SAEZ,SAAU,SAAU7jB,UAChBN,EAAGR,KAAK2kB,EAAM7jB,SAElB,SAAU,SAAUA,EAAGC,UACnBP,EAAGR,KAAK2kB,EAAM7jB,EAAGC,SAErB,SAAU,SAAUD,EAAGC,EAAGmX,UACtB1X,EAAGR,KAAK2kB,EAAM7jB,EAAGC,EAAGmX,WAGxB,kBACE1X,EAAGhB,MAAMmlB,EAAMllB,eCnBT,SAAUgwB,OACpBxvB,EAASwvB,SACNkB,UAAUyB,OAAO3C,GAAM,4BACtBA,GCAP6C,EAAuBzyB,OAAOmwB,oBAItBkB,EAAcoB,EAAuB,SAAwBlB,EAAGC,EAAGkB,MAC7EC,EAASpB,GACTC,EAAIE,EAAYF,GAAG,GACnBmB,EAASD,GACLf,EAAgB,WACXc,EAAqBlB,EAAGC,EAAGkB,GAClC,MAAOzC,OACL,QAASyC,GAAc,QAASA,EAAY,MAAM5B,UAAU,iCAC5D,UAAW4B,IAAYnB,EAAEC,GAAKkB,EAAW9sB,OACtC2rB,MCdQF,EAAc,SAAUuB,EAAQ7rB,EAAKnB,UAC7CitB,EAAqBd,EAAEa,EAAQ7rB,EAAK8qB,EAAyB,EAAGjsB,KACrE,SAAUgtB,EAAQ7rB,EAAKnB,UACzBgtB,EAAO7rB,GAAOnB,EACPgtB,GCNLtC,EAA2BhC,EAA2DyD,EAOtFe,EAAkB,SAAUC,OAC1BC,EAAU,SAAU/xB,EAAGC,EAAGmX,MACxBvU,gBAAgBivB,EAAmB,QAC7BnzB,UAAUkB,aACX,SAAU,IAAIiyB,OACd,SAAU,IAAIA,EAAkB9xB,QAChC,SAAU,IAAI8xB,EAAkB9xB,EAAGC,UACjC,IAAI6xB,EAAkB9xB,EAAGC,EAAGmX,UAC9B0a,EAAkBpzB,MAAMmE,KAAMlE,mBAEzCozB,EAAQ/yB,UAAY8yB,EAAkB9yB,UAC/B+yB,KAiBQ,SAAUC,EAAStJ,OAWtBuJ,EAAYC,EACpBpsB,EAAKqsB,EAAgBC,EAAgBC,EAAgBC,EAAgB/C,EAXrEgD,EAASP,EAAQQ,OACjBC,EAAST,EAAQjD,OACjB2D,EAASV,EAAQW,KACjBC,EAAQZ,EAAQ3O,MAEhBwP,EAAeJ,EAAS1D,EAAS2D,EAAS3D,EAAOwD,IAAWxD,EAAOwD,IAAW,IAAIvzB,UAElFwzB,EAASC,EAASK,EAAOA,EAAKP,KAAYO,EAAKP,GAAU,IACzDQ,EAAkBP,EAAOxzB,cAKxB8G,KAAO4iB,EAGVuJ,GAFSjB,EAASyB,EAAS3sB,EAAMysB,GAAUG,EAAS,IAAM,KAAO5sB,EAAKksB,EAAQgB,SAEtDH,GAAgBlC,EAAIkC,EAAc/sB,GAE1DssB,EAAiBI,EAAO1sB,GAEpBmsB,IAEFI,EAFkBL,EAAQiB,aAC1B1D,EAAaF,EAAyBwD,EAAc/sB,KACrBypB,EAAW5qB,MACpBkuB,EAAa/sB,IAGrCqsB,EAAkBF,GAAcI,EAAkBA,EAAiB3J,EAAO5iB,GAEtEmsB,UAAqBG,UAA0BD,IAGnBG,EAA5BN,EAAQkB,MAAQjB,EAA6BiB,EAAKf,EAAgBpD,GAE7DiD,EAAQmB,MAAQlB,EAA6BJ,EAAgBM,GAE7DS,GAAkC,mBAAlBT,EAA+Ce,EAAK1sB,SAAStH,KAAMizB,GAEtEA,GAGlBH,EAAQoB,MAASjB,GAAkBA,EAAeiB,MAAUhB,GAAkBA,EAAegB,OAC/FC,EAA4Bf,EAAgB,QAAQ,GAGtDE,EAAO1sB,GAAOwsB,EAEVM,IAEGjC,EAAImC,EADTZ,EAAoBK,EAAS,cAE3Bc,EAA4BP,EAAMZ,EAAmB,IAGvDY,EAAKZ,GAAmBpsB,GAAOqsB,EAE3BH,EAAQsB,MAAQP,IAAoBA,EAAgBjtB,IACtDutB,EAA4BN,EAAiBjtB,EAAKqsB,QCvFxD,CAAEK,OAAQ,SAAUG,MAAM,EAAMK,QAAS5C,EAAagD,MAAOhD,GAAe,CAC5ElB,eAAgBqE,EAA2BzC,6BCJzC/xB,EAAS+zB,EAAK/zB,OAEdmwB,EAAiB5wB,UAAiB,SAAwBqwB,EAAI7oB,EAAK0tB,UAC9Dz0B,EAAOmwB,eAAeP,EAAI7oB,EAAK0tB,IAGpCz0B,EAAOmwB,eAAekE,OAAMlE,EAAekE,MAAO,QCPrCK,ECFbnvB,EAAOD,KAAKC,KACZC,EAAQF,KAAKE,QAIA,SAAUmvB,UAClBzxB,MAAMyxB,GAAYA,GAAY,GAAKA,EAAW,EAAInvB,EAAQD,GAAMovB,ICJrEzuB,EAAMZ,KAAKY,MAIE,SAAUyuB,UAClBA,EAAW,EAAIzuB,EAAI0uB,EAAUD,GAAW,kBAAoB,GCLjEvrB,GAAM9D,KAAK8D,IACXlD,GAAMZ,KAAKY,OAKE,SAAUwiB,EAAO5nB,OAC5B+zB,EAAUD,EAAUlM,UACjBmM,EAAU,EAAIzrB,GAAIyrB,EAAU/zB,EAAQ,GAAKoF,GAAI2uB,EAAS/zB,ICL3Dg0B,GAAe,SAAUC,UACpB,SAAUC,EAAOC,EAAIC,OAItBtvB,EAHA2rB,EAAIE,EAAgBuD,GACpBl0B,EAASq0B,EAAS5D,EAAEzwB,QACpB4nB,EAAQ0M,GAAgBF,EAAWp0B,MAInCi0B,GAAeE,GAAMA,GAAI,KAAOn0B,EAAS4nB,OAC3C9iB,EAAQ2rB,EAAE7I,OAEG9iB,EAAO,OAAO,OAEtB,KAAM9E,EAAS4nB,EAAOA,QACtBqM,GAAerM,KAAS6I,IAAMA,EAAE7I,KAAWuM,EAAI,OAAOF,GAAerM,GAAS,SAC3EqM,IAAgB,OAIb,CAGfM,SAAUP,IAAa,GAGvBtnB,QAASsnB,IAAa,OC9BP,GCEbtnB,GAAU8gB,GAAuC9gB,WAGpC,SAAUolB,EAAQhb,OAI7B7Q,EAHAwqB,EAAIE,EAAgBmB,GACpBhyB,EAAI,EACJyb,EAAS,OAERtV,KAAOwqB,GAAIK,EAAI0D,GAAYvuB,IAAQ6qB,EAAIL,EAAGxqB,IAAQsV,EAAOtb,KAAKgG,QAE5D6Q,EAAM9W,OAASF,GAAOgxB,EAAIL,EAAGxqB,EAAM6Q,EAAMhX,SAC7C4M,GAAQ6O,EAAQtV,IAAQsV,EAAOtb,KAAKgG,WAEhCsV,MCdQ,CACf,cACA,iBACA,gBACA,uBACA,iBACA,WACA,cCHerc,OAAOoH,MAAQ,SAAcmqB,UACrCgE,GAAmBhE,EAAGiE,QCCdnE,EAAcrxB,OAAOy1B,iBAAmB,SAA0BlE,EAAGmE,GACpF/C,EAASpB,WAILxqB,EAHAK,EAAOuuB,GAAWD,GAClB50B,EAASsG,EAAKtG,OACd4nB,EAAQ,EAEL5nB,EAAS4nB,GAAOmK,EAAqBd,EAAER,EAAGxqB,EAAMK,EAAKshB,KAAUgN,EAAW3uB,WAC1EwqB,KCRP,CAAEkC,OAAQ,SAAUG,MAAM,EAAMK,QAAS5C,EAAagD,MAAOhD,GAAe,CAC5EoE,iBAAkBA,+BCJhBz1B,EAAS+zB,EAAK/zB,OAEdy1B,EAAmBl2B,UAAiB,SAA0Bq2B,EAAG1J,UAC5DlsB,EAAOy1B,iBAAiBG,EAAG1J,IAGhClsB,EAAOy1B,iBAAiBpB,OAAMoB,EAAiBpB,MAAO,MCNtD7B,GAAY,SAAUqD,SACE,mBAAZA,EAAyBA,OAAWtyB,MAGnC,SAAUuyB,EAAWC,UAC7Bn2B,UAAUkB,OAAS,EAAI0xB,GAAUuB,EAAK+B,KAAetD,GAAUxC,EAAO8F,IACzE/B,EAAK+B,IAAc/B,EAAK+B,GAAWC,IAAW/F,EAAO8F,IAAc9F,EAAO8F,GAAWC,ICNvFT,GAAaE,GAAYtjB,OAAO,SAAU,mBAIlClS,OAAOwe,qBAAuB,SAA6B+S,UAC9DgE,GAAmBhE,EAAG+D,YCRnBt1B,OAAOg2B,0BCMFC,GAAW,UAAW,YAAc,SAAiBrG,OAChExoB,EAAO8uB,GAA0BnE,EAAEY,EAAS/C,IAC5CoG,EAAwBG,GAA4BpE,SACjDiE,EAAwB5uB,EAAK8K,OAAO8jB,EAAsBpG,IAAOxoB,MCJzD,SAAUwrB,EAAQ7rB,EAAKnB,OAClCwwB,EAAc1E,EAAY3qB,GAC1BqvB,KAAexD,EAAQC,EAAqBd,EAAEa,EAAQwD,EAAavE,EAAyB,EAAGjsB,IAC9FgtB,EAAOwD,GAAexwB,KCC3B,CAAE6tB,OAAQ,SAAUG,MAAM,EAAMS,MAAOhD,GAAe,CACtDgF,0BAA2B,SAAmCzD,WAMxD7rB,EAAKypB,EALLe,EAAIE,EAAgBmB,GACpBtC,EAA2BgG,EAA+BvE,EAC1D3qB,EAAOmvB,GAAQhF,GACflV,EAAS,GACTqM,EAAQ,EAELthB,EAAKtG,OAAS4nB,QAEAnlB,KADnBitB,EAAaF,EAAyBiB,EAAGxqB,EAAMK,EAAKshB,QACtB8N,GAAena,EAAQtV,EAAKypB,UAErDnU,KClBX,OAAiB0X,EAAK/zB,OAAOq2B,0BCAzB/E,GAAiChD,EAA2DyD,EAG5F0E,GAAsBvG,GAAM,WAAcoB,GAA+B,QAK3E,CAAEmC,OAAQ,SAAUG,MAAM,EAAMK,QAJpB5C,GAAeoF,GAIqBpC,MAAOhD,GAAe,CACtEf,yBAA0B,SAAkCV,EAAI7oB,UACvDuqB,GAA+BG,EAAgB7B,GAAK7oB,UCsC3D2vB,yBChDA12B,EAAS+zB,EAAK/zB,OAEdswB,EAA2B/wB,UAAiB,SAAkCqwB,EAAI7oB,UAC7E/G,EAAOswB,yBAAyBV,EAAI7oB,IAGzC/G,EAAOswB,yBAAyB+D,OAAM/D,EAAyB+D,MAAO,WCPvDr0B,OAAOg2B,wBAA0B9F,GAAM,kBAGhDqC,OAAOoE,gBCHAC,KAEXD,OAAOtC,MAEkB,iBAAnBsC,OAAOE,YCFF92B,MAAMF,SAAW,SAAiBgH,SAC1B,SAAhBgqB,EAAQhqB,OCDA,SAAU8tB,UAClB30B,OAAOgxB,EAAuB2D,QCHtBsB,GAAW,WAAY,sBCE5BjG,EADC,uBCAI,SAAUjpB,EAAKnB,OAE5B0uB,EAA4BtE,EAAQjpB,EAAKnB,GACzC,MAAOqqB,GACPD,EAAOjpB,GAAOnB,SACPA,EDJmBkxB,CADjB,qBACmC,uBED/Cv3B,UAAiB,SAAUwH,EAAKnB,UACxBmxB,GAAMhwB,KAASgwB,GAAMhwB,QAAiBxD,IAAVqC,EAAsBA,EAAQ,MAChE,WAAY,IAAI7E,KAAK,CACtB8rB,QAAS,QACTmK,KAAgB,OAChBC,UAAW,4CCRTC,GAAK,EACLC,GAAU7xB,KAAK8xB,YAEF,SAAUrwB,SAClB,UAAYwrB,YAAehvB,IAARwD,EAAoB,GAAKA,GAAO,QAAUmwB,GAAKC,IAASj3B,SAAS,KCDzFkH,GAAOiwB,GAAO,WAED,SAAUtwB,UAClBK,GAAKL,KAASK,GAAKL,GAAOuwB,GAAIvwB,KXMnCwwB,GAAWC,GAAU,YAErBC,GAAmB,aAEnBC,GAAY,SAAUC,SACjBjiB,WAAmBiiB,EARnB,cA2CLC,GAAkB,eAGlBlB,GAAkBxF,SAAS2G,QAAU,IAAIC,cAAc,YACvD,MAAO7H,IA1BoB,IAIzB8H,EAFAC,EAyBJJ,GAAkBlB,GApCY,SAAUA,GACxCA,EAAgBuB,MAAMP,GAAU,KAChChB,EAAgBwB,YACZ9e,EAAOsd,EAAgByB,aAAan4B,cACxC02B,EAAkB,KACXtd,EA+B6Bgf,CAA0B1B,MAzB1DsB,EAASK,EAAsB,WAG5BC,MAAMC,QAAU,OACvBC,GAAKC,YAAYT,GAEjBA,EAAOU,IAAMnG,OALJ,gBAMTwF,EAAiBC,EAAOW,cAAczH,UACvB0H,OACfb,EAAeE,MAAMP,GAAU,sBAC/BK,EAAeG,QACRH,EAAec,WAelB/3B,EAAS00B,GAAY10B,OAClBA,YAAiB82B,GAAe,UAAYpC,GAAY10B,WACxD82B,MAGTtC,GAAWiC,KAAY,EAIvB,OAAiBv3B,OAAO84B,QAAU,SAAgBvH,EAAGmE,OAC/CrZ,SACM,OAANkV,GACFkG,GAAgB,UAAc9E,EAASpB,GACvClV,EAAS,IAAIob,GACbA,GAAgB,UAAc,KAE9Bpb,EAAOkb,IAAYhG,GACdlV,EAASub,UACMr0B,IAAfmyB,EAA2BrZ,EAASoZ,GAAiBpZ,EAAQqZ,IY3ElEqD,GAA4BzK,GAAsDyD,EAElF7xB,GAAW,GAAGA,SAEd84B,GAA+B,iBAAVlJ,QAAsBA,QAAU9vB,OAAOwe,oBAC5Dxe,OAAOwe,oBAAoBsR,QAAU,SAWtB,SAA6BF,UACvCoJ,IAAoC,mBAArB94B,GAASC,KAAKyvB,GAVjB,SAAUA,cAEpBmJ,GAA0BnJ,GACjC,MAAOK,UACA+I,GAAYhyB,SAOjBiyB,CAAerJ,GACfmJ,GAA0BtH,EAAgB7B,SClB/B,SAAU6D,EAAQ1sB,EAAKnB,EAAOqtB,GACzCA,GAAWA,EAAQxC,WAAYgD,EAAO1sB,GAAOnB,EAC5C0uB,EAA4Bb,EAAQ1sB,EAAKnB,ICG5CszB,GAAwB7B,GAAO,OAC/BV,GAAS3G,EAAO2G,OAChBwC,GAAwBC,GAAoBzC,GAASA,IAAUA,GAAO0C,eAAiB/B,MAE1E,SAAU/vB,UACpBqqB,EAAIsH,GAAuB3xB,KAC1BqvB,IAAiBhF,EAAI+E,GAAQpvB,GAAO2xB,GAAsB3xB,GAAQovB,GAAOpvB,GACxE2xB,GAAsB3xB,GAAQ4xB,GAAsB,UAAY5xB,IAC9D2xB,GAAsB3xB,UCbrB+xB,ICCRnJ,GAAiB7B,EAA+CyD,KAEnD,SAAUwH,OACrB5C,EAAS5C,EAAK4C,SAAW5C,EAAK4C,OAAS,IACtC/E,EAAI+E,EAAQ4C,IAAOpJ,GAAewG,EAAQ4C,EAAM,CACnD3zB,MAAO4zB,GAA6BzH,EAAEwH,MCLtC1uB,GAAO,GAEXA,GAHoByuB,GAAgB,gBAGd,IAEtB,OAAkC,eAAjB/G,OAAO1nB,ICHpB4uB,GAAgBH,GAAgB,eAEhCI,GAAuE,aAAnDC,EAAW,kBAAqB/5B,UAArB,OAUlBg6B,GAAwBD,EAAa,SAAU/J,OAC1D2B,EAAGsI,EAAKxd,cACE9Y,IAAPqsB,EAAmB,YAAqB,OAAPA,EAAc,OAEM,iBAAhDiK,EAXD,SAAUjK,EAAI7oB,cAEhB6oB,EAAG7oB,GACV,MAAOkpB,KAQS6J,CAAOvI,EAAIvxB,OAAO4vB,GAAK6J,KAA8BI,EAEnEH,GAAoBC,EAAWpI,GAEH,WAA3BlV,EAASsd,EAAWpI,KAAsC,mBAAZA,EAAEwI,OAAuB,YAAc1d,MClB3Eud,GAAwB,GAAG15B,SAAW,iBAC9C,WAAa2wB,GAAQ/sB,MAAQ,KCNlCqsB,GAAiB7B,EAA+CyD,EAMhE0H,GAAgBH,GAAgB,kBAEnB,SAAU1J,EAAIoK,EAAKrG,EAAQsG,MACtCrK,EAAI,KACF6D,EAASE,EAAS/D,EAAKA,EAAG3vB,UACzB2xB,EAAI6B,EAAQgG,KACftJ,GAAesD,EAAQgG,GAAe,CAAE9I,cAAc,EAAM/qB,MAAOo0B,IAEjEC,IAAeL,IACjBtF,EAA4Bb,EAAQ,WAAYvzB,MCdlDg6B,GAAmBzyB,SAASvH,SAGE,mBAAvB62B,GAAMoD,gBACfpD,GAAMoD,cAAgB,SAAUvK,UACvBsK,GAAiB/5B,KAAKyvB,KAIjC,ICFI9nB,GAAK+F,GAAK+jB,MDEGmF,GAAMoD,cERnBC,GAAUpK,EAAOoK,WAEe,mBAAZA,IAA0B,cAAcvvB,KAAKsvB,GAAcC,KDG/EA,GAAUpK,EAAOoK,QAgBrB,GAAIC,GAAiB,KACftD,GAAQ,IAAIqD,GACZE,GAAQvD,GAAMlpB,IACd0sB,GAAQxD,GAAMnF,IACd4I,GAAQzD,GAAMjvB,IAClBA,GAAM,SAAU8nB,EAAI6K,UAClBD,GAAMr6B,KAAK42B,GAAOnH,EAAI6K,GACfA,GAET5sB,GAAM,SAAU+hB,UACP0K,GAAMn6B,KAAK42B,GAAOnH,IAAO,IAElCgC,GAAM,SAAUhC,UACP2K,GAAMp6B,KAAK42B,GAAOnH,QAEtB,KACD8K,GAAQlD,GAAU,SACtBlC,GAAWoF,KAAS,EACpB5yB,GAAM,SAAU8nB,EAAI6K,UAClBnG,EAA4B1E,EAAI8K,GAAOD,GAChCA,GAET5sB,GAAM,SAAU+hB,UACP+K,EAAU/K,EAAI8K,IAAS9K,EAAG8K,IAAS,IAE5C9I,GAAM,SAAUhC,UACP+K,EAAU/K,EAAI8K,KAIzB,OAAiB,CACf5yB,IAAKA,GACL+F,IAAKA,GACL+jB,IAAKA,GACLgJ,QA/CY,SAAUhL,UACfgC,GAAIhC,GAAM/hB,GAAI+hB,GAAM9nB,GAAI8nB,EAAI,KA+CnCiL,UA5Cc,SAAUC,UACjB,SAAUlL,OACXmL,MACC36B,EAASwvB,KAAQmL,EAAQltB,GAAI+hB,IAAKoL,OAASF,QACxChK,UAAU,0BAA4BgK,EAAO,oBAC5CC,KEhBTE,GAAU3B,GAAgB,cAIb,SAAU4B,EAAep6B,OACpCq6B,SACAt7B,GAAQq7B,KAGM,mBAFhBC,EAAID,EAAcE,cAEaD,IAAMp7B,QAASF,GAAQs7B,EAAEl7B,WAC/CG,EAAS+6B,IAEN,QADVA,EAAIA,EAAEF,OACUE,OAAI53B,GAH+C43B,OAAI53B,GAKlE,SAAWA,IAAN43B,EAAkBp7B,MAAQo7B,GAAc,IAAXr6B,EAAe,EAAIA,ICZ5DC,GAAO,GAAGA,KAGV+zB,GAAe,SAAUgG,OACvBO,EAAiB,GAARP,EACTQ,EAAoB,GAARR,EACZS,EAAkB,GAART,EACVU,EAAmB,GAARV,EACXW,EAAwB,GAARX,EAChBY,EAAmB,GAARZ,GAAaW,SACrB,SAAUzG,EAAO2G,EAAY7W,EAAM8W,WAQpCh2B,EAAOyW,EAPPkV,EAAI9K,GAASuO,GACbjF,EAAOgB,EAAcQ,GACrBsK,EAAgB1H,EAAKwH,EAAY7W,EAAM,GACvChkB,EAASq0B,EAASpF,EAAKjvB,QACvB4nB,EAAQ,EACRoQ,EAAS8C,GAAkBE,GAC3BrI,EAAS4H,EAASvC,EAAO9D,EAAOl0B,GAAUw6B,EAAYxC,EAAO9D,EAAO,QAAKzxB,EAEvEzC,EAAS4nB,EAAOA,IAAS,IAAIgT,GAAYhT,KAASqH,KAEtD1T,EAASwf,EADTj2B,EAAQmqB,EAAKrH,GACiBA,EAAO6I,GACjCuJ,MACEO,EAAQ5H,EAAO/K,GAASrM,OACvB,GAAIA,EAAQ,OAAQye,QAClB,SAAU,OACV,SAAUl1B,OACV,SAAU8iB,OACV,EAAG3nB,GAAKZ,KAAKszB,EAAQ7tB,QACrB,GAAI41B,EAAU,OAAO,SAGzBC,GAAiB,EAAIF,GAAWC,EAAWA,EAAW/H,OAIhD,CAGf9b,QAASmd,GAAa,GAGtBr0B,IAAKq0B,GAAa,GAGlBiH,OAAQjH,GAAa,GAGrBr1B,KAAMq1B,GAAa,GAGnBkH,MAAOlH,GAAa,GAGpBmH,KAAMnH,GAAa,GAGnBoH,UAAWpH,GAAa,IC3BtBqH,GAAW7N,GAAwC3W,QAEnDykB,GAAS5E,GAAU,UAGnB6E,GAAe/C,GAAgB,eAC/BgD,GAAmBC,GAAoBz0B,IACvC00B,GAAmBD,GAAoB1B,UAJ9B,UAKT4B,GAAkBz8B,OAAM,UACxB08B,GAAU1M,EAAO2G,OACjBgG,GAAa1G,GAAW,OAAQ,aAChC3E,GAAiCgF,EAA+BvE,EAChEU,GAAuBI,EAAqBd,EAC5CgH,GAA4B6D,GAA4B7K,EACxD3B,GAA6B0B,EAA2BC,EACxD8K,GAAaxF,GAAO,WACpByF,GAAyBzF,GAAO,cAChC0F,GAAyB1F,GAAO,6BAChC2F,GAAyB3F,GAAO,6BAChC6B,GAAwB7B,GAAO,OAC/B4F,GAAUjN,EAAOiN,QAEjBC,IAAcD,KAAYA,GAAO,YAAgBA,GAAO,UAAYE,UAGpEC,GAAsB/L,GAAenB,GAAM,kBAGpC,GAFFmN,GAAmB5K,GAAqB,GAAI,IAAK,CACtD5kB,IAAK,kBAAqB4kB,GAAqB3uB,KAAM,IAAK,CAAE8B,MAAO,IAAK3E,MACtEA,KACD,SAAUswB,EAAGC,EAAGkB,OACf4K,EAA4BhM,GAA+BmL,GAAiBjL,GAC5E8L,UAAkCb,GAAgBjL,GACtDiB,GAAqBlB,EAAGC,EAAGkB,GACvB4K,GAA6B/L,IAAMkL,IACrChK,GAAqBgK,GAAiBjL,EAAG8L,IAEzC7K,GAEA2B,GAAO,SAAUyF,EAAK0D,OACpBC,EAASX,GAAWhD,GAAOwD,GAAmBX,GAAO,kBACzDJ,GAAiBkB,EAAQ,CACvBxC,KAtCS,SAuCTnB,IAAKA,EACL0D,YAAaA,IAEVlM,IAAamM,EAAOD,YAAcA,GAChCC,GAGLC,GAAWrE,GAAoB,SAAUxJ,SACvB,iBAANA,GACZ,SAAUA,UACL5vB,OAAO4vB,aAAe8M,IAG3BgB,GAAkB,SAAwBnM,EAAGC,EAAGkB,GAC9CnB,IAAMkL,IAAiBiB,GAAgBZ,GAAwBtL,EAAGkB,GACtEC,EAASpB,OACLxqB,EAAM2qB,EAAYF,GAAG,UACzBmB,EAASD,GACLd,EAAIiL,GAAY91B,IACb2rB,EAAWjC,YAIVmB,EAAIL,EAAG6K,KAAW7K,EAAE6K,IAAQr1B,KAAMwqB,EAAE6K,IAAQr1B,IAAO,GACvD2rB,EAAa2K,GAAmB3K,EAAY,CAAEjC,WAAYoB,EAAyB,GAAG,OAJjFD,EAAIL,EAAG6K,KAAS3J,GAAqBlB,EAAG6K,GAAQvK,EAAyB,EAAG,KACjFN,EAAE6K,IAAQr1B,IAAO,GAIVq2B,GAAoB7L,EAAGxqB,EAAK2rB,IAC9BD,GAAqBlB,EAAGxqB,EAAK2rB,IAGpCiL,GAAoB,SAA0BpM,EAAGmE,GACnD/C,EAASpB,OACLqM,EAAanM,EAAgBiE,GAC7BtuB,EAAOuuB,GAAWiI,GAAY1rB,OAAO2rB,GAAuBD,WAChEzB,GAAS/0B,GAAM,SAAUL,GAClBsqB,IAAeyM,GAAsB39B,KAAKy9B,EAAY72B,IAAM22B,GAAgBnM,EAAGxqB,EAAK62B,EAAW72B,OAE/FwqB,GAOLuM,GAAwB,SAA8BvN,OACpDiB,EAAIE,EAAYnB,GAAG,GACnBE,EAAaL,GAA2BjwB,KAAK2D,KAAM0tB,WACnD1tB,OAAS24B,IAAmB7K,EAAIiL,GAAYrL,KAAOI,EAAIkL,GAAwBtL,QAC5Ef,IAAemB,EAAI9tB,KAAM0tB,KAAOI,EAAIiL,GAAYrL,IAAMI,EAAI9tB,KAAMs4B,KAAWt4B,KAAKs4B,IAAQ5K,KAAKf,IAGlGsN,GAA4B,SAAkCxM,EAAGC,OAC/D5B,EAAK6B,EAAgBF,GACrBxqB,EAAM2qB,EAAYF,GAAG,MACrB5B,IAAO6M,KAAmB7K,EAAIiL,GAAY91B,IAAS6qB,EAAIkL,GAAwB/1B,QAC/EypB,EAAac,GAA+B1B,EAAI7oB,UAChDypB,IAAcoB,EAAIiL,GAAY91B,IAAU6qB,EAAIhC,EAAIwM,KAAWxM,EAAGwM,IAAQr1B,KACxEypB,EAAWC,YAAa,GAEnBD,IAGLwN,GAAuB,SAA6BzM,OAClD3Z,EAAQmhB,GAA0BtH,EAAgBF,IAClDlV,EAAS,UACb8f,GAASvkB,GAAO,SAAU7Q,GACnB6qB,EAAIiL,GAAY91B,IAAS6qB,EAAI0D,GAAYvuB,IAAMsV,EAAOtb,KAAKgG,MAE3DsV,GAGLwhB,GAAyB,SAA+BtM,OACtD0M,EAAsB1M,IAAMkL,GAC5B7kB,EAAQmhB,GAA0BkF,EAAsBnB,GAAyBrL,EAAgBF,IACjGlV,EAAS,UACb8f,GAASvkB,GAAO,SAAU7Q,IACpB6qB,EAAIiL,GAAY91B,IAAUk3B,IAAuBrM,EAAI6K,GAAiB11B,IACxEsV,EAAOtb,KAAK87B,GAAW91B,OAGpBsV,GAkHT,GA7GKua,KAcHsH,IAbAxB,GAAU,cACJ54B,gBAAgB44B,GAAS,MAAM5L,UAAU,mCACzCyM,EAAe39B,UAAUkB,aAA2ByC,IAAjB3D,UAAU,GAA+B2yB,OAAO3yB,UAAU,SAA7B2D,EAChEs2B,EAAMvC,GAAIiG,GACV3U,EAAS,SAAUhjB,GACjB9B,OAAS24B,IAAiB7T,EAAOzoB,KAAK28B,GAAwBl3B,GAC9DgsB,EAAI9tB,KAAMs4B,KAAWxK,EAAI9tB,KAAKs4B,IAASvC,KAAM/1B,KAAKs4B,IAAQvC,IAAO,GACrEuD,GAAoBt5B,KAAM+1B,EAAKhI,EAAyB,EAAGjsB,YAEzDyrB,GAAe6L,IAAYE,GAAoBX,GAAiB5C,EAAK,CAAElJ,cAAc,EAAM7oB,IAAK8gB,IAC7FwL,GAAKyF,EAAK0D,KAGH,UAAa,YAAY,kBAChCf,GAAiB14B,MAAM+1B,OAGhCqE,GAASxB,GAAS,iBAAiB,SAAUa,UACpCnJ,GAAKkD,GAAIiG,GAAcA,MAGhCzL,EAA2BC,EAAI+L,GAC/BjL,EAAqBd,EAAI2L,GACzBpH,EAA+BvE,EAAIgM,GACnC7H,GAA0BnE,EAAI6K,GAA4B7K,EAAIiM,GAC9D7H,GAA4BpE,EAAI8L,GAEhCrE,GAA6BzH,EAAI,SAAUxqB,UAClC6sB,GAAKkF,GAAgB/xB,GAAOA,IAGjC8pB,GAEFoB,GAAqBiK,GAAO,UAAa,cAAe,CACtD/L,cAAc,EACd9iB,IAAK,kBACI2uB,GAAiB14B,MAAMy5B,kBASpC,CAAEvN,QAAQ,EAAMoE,MAAM,EAAMH,QAAS2C,GAAevC,MAAOuC,IAAiB,CAC5ED,OAAQ+F,KAGVP,GAASxG,GAAWuD,KAAwB,SAAU3xB,GACpD42B,GAAsB52B,QAGtB,CAAEksB,OAjLS,SAiLOG,MAAM,EAAMK,QAAS2C,IAAiB,KAGjD,SAAU7vB,OACXgU,EAASwX,OAAOxrB,MAChB6qB,EAAImL,GAAwBhiB,GAAS,OAAOgiB,GAAuBhiB,OACnEyiB,EAASd,GAAQ3hB,UACrBgiB,GAAuBhiB,GAAUyiB,EACjCR,GAAuBQ,GAAUziB,EAC1ByiB,GAITY,OAAQ,SAAgBC,OACjBZ,GAASY,GAAM,MAAMvN,UAAUuN,EAAM,uBACtCzM,EAAIoL,GAAwBqB,GAAM,OAAOrB,GAAuBqB,IAEtEC,UAAW,WAAcpB,IAAa,GACtCqB,UAAW,WAAcrB,IAAa,OAGtC,CAAEzJ,OAAQ,SAAUG,MAAM,EAAMK,QAAS2C,GAAevC,MAAOhD,GAAe,CAG9EyH,OA3HY,SAAgBvH,EAAGmE,eACTnyB,IAAfmyB,EAA2B2H,GAAmB9L,GAAKoM,GAAkBN,GAAmB9L,GAAImE,IA6HnGvF,eAAgBuN,GAGhBjI,iBAAkBkI,GAGlBrN,yBAA0ByN,OAG1B,CAAEtK,OAAQ,SAAUG,MAAM,EAAMK,QAAS2C,IAAiB,CAG1DpY,oBAAqBwf,GAGrBhI,sBAAuB6H,OAKvB,CAAEpK,OAAQ,SAAUG,MAAM,EAAMK,OAAQ/D,GAAM,WAAciG,GAA4BpE,EAAE,OAAU,CACpGiE,sBAAuB,SAA+BpG,UAC7CuG,GAA4BpE,EAAEtL,GAASmJ,OAM9C+M,GAAY,KACV6B,IAAyB5H,IAAiB1G,GAAM,eAC9CsN,EAASd,WAEkB,UAAxBC,GAAW,CAACa,KAEe,MAA7Bb,GAAW,CAAE17B,EAAGu8B,KAEc,MAA9Bb,GAAW38B,OAAOw9B,OAGzBiB,EAAE,CAAEhL,OAAQ,OAAQG,MAAM,EAAMK,OAAQuK,IAAyB,CAE/DE,UAAW,SAAmB9O,EAAI+O,EAAUC,WAGtCC,EAFA/3B,EAAO,CAAC8oB,GACRlH,EAAQ,EAEL9oB,UAAUkB,OAAS4nB,GAAO5hB,EAAK/F,KAAKnB,UAAU8oB,SACrDmW,EAAYF,GACPv+B,EAASu+B,SAAoBp7B,IAAPqsB,KAAoB6N,GAAS7N,UACnD/vB,GAAQ8+B,KAAWA,EAAW,SAAU53B,EAAKnB,MACxB,mBAAbi5B,IAAyBj5B,EAAQi5B,EAAU1+B,KAAK2D,KAAMiD,EAAKnB,KACjE63B,GAAS73B,GAAQ,OAAOA,IAE/BkB,EAAK,GAAK63B,EACHhC,GAAWh9B,MAAM,KAAMmH,MAO/B41B,GAAO,UAAYL,KACtB/H,EAA4BoI,GAAO,UAAaL,GAAcK,GAAO,UAAYr7B,SAInFy9B,GAAepC,GA7QF,UA+QbpH,GAAW8G,KAAU,ECnTrB,ICWI2C,GAAmBC,GAAmCC,MDXzClL,EAAK/zB,OAAOg2B,yBEHZ,OCEC9F,GAAM,oBACb2I,YACTA,EAAE54B,UAAUm7B,YAAc,KACnBp7B,OAAOk/B,eAAe,IAAIrG,KAASA,EAAE54B,aCA1Cs3B,GAAWC,GAAU,YACrBiF,GAAkBz8B,OAAOC,aAIZk/B,GAA2Bn/B,OAAOk/B,eAAiB,SAAU3N,UAC5EA,EAAI9K,GAAS8K,GACTK,EAAIL,EAAGgG,IAAkBhG,EAAEgG,IACH,mBAAjBhG,EAAE6J,aAA6B7J,aAAaA,EAAE6J,YAChD7J,EAAE6J,YAAYn7B,UACdsxB,aAAavxB,OAASy8B,GAAkB,MHP/C2C,IADW9F,GAAgB,aACF,GAQzB,GAAGlyB,OAGC,SAFN63B,GAAgB,GAAG73B,SAIjB43B,GAAoCE,GAAeA,GAAeD,QACxBj/B,OAAOC,YAAW8+B,GAAoBC,IAHlDI,IAAyB,GAOlC77B,MAArBw7B,KAAgCA,GAAoB,IAOxD,OAAiB,CACfA,kBAAmBA,GACnBK,uBAAwBA,IIlCtBL,GAAoBzQ,GAAuCyQ,kBAM3DM,GAAa,kBAAqBv7B,SCArB9D,OAAOs/B,iBAAmB,aAAe,GAAK,eAGzD1W,EAFA2W,GAAiB,EACjB10B,EAAO,QAGT+d,EAAS5oB,OAAOswB,yBAAyBtwB,OAAOC,UAAW,aAAa6H,KACjE3H,KAAK0K,EAAM,IAClB00B,EAAiB10B,aAAgB9K,MACjC,MAAOkwB,WACF,SAAwBsB,EAAGjN,UAChCqO,EAASpB,GCfI,SAAU3B,OACpBxvB,EAASwvB,IAAc,OAAPA,QACbkB,UAAU,aAAeyB,OAAO3C,GAAM,mBDc5C4P,CAAmBlb,GACfib,EAAgB3W,EAAOzoB,KAAKoxB,EAAGjN,GAC9BiN,EAAEkO,UAAYnb,EACZiN,GAdoD,QAgBzDhuB,GEVFw7B,GAAoBW,GAAcX,kBAClCK,GAAyBM,GAAcN,uBACvCO,GAAWrG,GAAgB,YAK3B+F,GAAa,kBAAqBv7B,SAErB,SAAU87B,EAAUrG,EAAMsG,EAAqB/nB,EAAMgoB,EAASC,EAAQC,IHbtE,SAAUH,EAAqBtG,EAAMzhB,OAChD2hB,EAAgBF,EAAO,YAC3BsG,EAAoB5/B,UAAY64B,GAAOiG,GAAmB,CAAEjnB,KAAM+Z,EAAyB,EAAG/Z,KAC9FgnB,GAAee,EAAqBpG,GAAe,GAAO,GAC1DwG,GAAUxG,GAAiB4F,GGU3Ba,CAA0BL,EAAqBtG,EAAMzhB,OAoBjDqoB,EAA0BC,EAASC,EAlBnCC,EAAqB,SAAUC,MAC7BA,IAAST,GAAWU,EAAiB,OAAOA,MAC3CpB,IAA0BmB,KAAQE,EAAmB,OAAOA,EAAkBF,UAC3EA,OAZD,WACE,aACC,iBAaa,kBAA4B,IAAIV,EAAoB/7B,KAAMy8B,WACxE,kBAAqB,IAAIV,EAAoB/7B,QAGpD21B,EAAgBF,EAAO,YACvBmH,GAAwB,EACxBD,EAAoBb,EAAS3/B,UAC7B0gC,EAAiBF,EAAkBd,KAClCc,EAAkB,eAClBX,GAAWW,EAAkBX,GAC9BU,GAAmBpB,IAA0BuB,GAAkBL,EAAmBR,GAClFc,EAA4B,SAARrH,GAAkBkH,EAAkBI,SAA4BF,KAIpFC,IACFT,EAA2BjB,GAAe0B,EAAkBzgC,KAAK,IAAIy/B,IACjEb,KAAsB/+B,OAAOC,WAAakgC,EAAyBroB,OASrEgnB,GAAeqB,EAA0B1G,GAAe,GAAM,GACjDwG,GAAUxG,GAAiB4F,KAzCjC,UA8CPS,GAAqBa,GA9Cd,WA8CgCA,EAAep5B,OACxDm5B,GAAwB,EACxBF,EAAkB,kBAA2BG,EAAexgC,KAAK2D,QAI/D,GAAwB28B,EAAkBd,MAAca,GAC1DlM,EAA4BmM,EAAmBd,GAAUa,GAE3DP,GAAU1G,GAAQiH,EAGdV,KACFM,EAAU,CACRhpB,OAAQkpB,EA5DD,UA6DPl5B,KAAM24B,EAASS,EAAkBF,EA9D5B,QA+DLO,QAASP,EA7DD,YA+DNN,EAAQ,IAAKK,KAAOD,GAClBhB,KAA0BsB,GAA2BL,KAAOI,GAC9DvC,GAASuC,EAAmBJ,EAAKD,EAAQC,SAEtC5B,EAAE,CAAEhL,OAAQ8F,EAAMjV,OAAO,EAAM2P,OAAQmL,IAA0BsB,GAAyBN,UAG5FA,GChFL9D,GAAmBC,GAAoBz0B,IACvC00B,GAAmBD,GAAoB1B,UAFtB,kBAcJiG,GAAe/gC,MAAO,SAAS,SAAUghC,EAAUC,GAClE1E,GAAiBx4B,KAAM,CACrBk3B,KAhBiB,iBAiBjBvH,OAAQhC,EAAgBsP,GACxBrY,MAAO,EACPsY,KAAMA,OAIP,eACGjG,EAAQyB,GAAiB14B,MACzB2vB,EAASsH,EAAMtH,OACfuN,EAAOjG,EAAMiG,KACbtY,EAAQqS,EAAMrS,eACb+K,GAAU/K,GAAS+K,EAAO3yB,QAC7Bi6B,EAAMtH,YAASlwB,EACR,CAAEqC,WAAOrC,EAAW09B,MAAM,IAEvB,QAARD,EAAuB,CAAEp7B,MAAO8iB,EAAOuY,MAAM,GACrC,UAARD,EAAyB,CAAEp7B,MAAO6tB,EAAO/K,GAAQuY,MAAM,GACpD,CAAEr7B,MAAO,CAAC8iB,EAAO+K,EAAO/K,IAASuY,MAAM,KAC7C,aAKOC,UAAYjB,GAAUlgC,MC7ChC,ICMI05B,GAAgBH,GAAgB,eAEpC,IAAK,IAAI6H,KDRQ,CACfC,YAAa,EACbC,oBAAqB,EACrBC,aAAc,EACdC,eAAgB,EAChBC,YAAa,EACbC,cAAe,EACfC,aAAc,EACdC,qBAAsB,EACtBC,SAAU,EACVC,kBAAmB,EACnBC,eAAgB,EAChBC,gBAAiB,EACjBC,kBAAmB,EACnBC,UAAW,EACXC,cAAe,EACfC,aAAc,EACdC,SAAU,EACVC,iBAAkB,EAClBC,OAAQ,EACRC,YAAa,EACbC,cAAe,EACfC,cAAe,EACfC,eAAgB,EAChBC,aAAc,EACdC,cAAe,EACfC,iBAAkB,EAClBC,iBAAkB,EAClBC,eAAgB,EAChBC,iBAAkB,EAClBC,cAAe,EACfC,UAAW,GCvB6B,KACpCC,GAAanT,EAAOmR,IACpBiC,GAAsBD,IAAcA,GAAWljC,UAC/CmjC,IAAuBvS,GAAQuS,MAAyB3J,IAC1DnF,EAA4B8O,GAAqB3J,GAAe0H,IAElElB,GAAUkB,IAAmBlB,GAAUlgC,MCZzC,IAAI+0B,GAAe,SAAUuO,UACpB,SAAUrO,EAAOpgB,OAIlB0uB,EAAOnlB,EAHPolB,EAAIhR,OAAOvB,EAAuBgE,IAClCwO,EAAW5O,EAAUhgB,GACrB6uB,EAAOF,EAAEziC,cAET0iC,EAAW,GAAKA,GAAYC,EAAaJ,EAAoB,QAAK9/B,GACtE+/B,EAAQC,EAAEG,WAAWF,IACN,OAAUF,EAAQ,OAAUE,EAAW,IAAMC,IACtDtlB,EAASolB,EAAEG,WAAWF,EAAW,IAAM,OAAUrlB,EAAS,MAC1DklB,EAAoBE,EAAEjZ,OAAOkZ,GAAYF,EACzCD,EAAoBE,EAAEv8B,MAAMw8B,EAAUA,EAAW,GAA+BrlB,EAAS,OAAlCmlB,EAAQ,OAAU,IAA0B,QCdzGhZ,GDkBa,CAGfqZ,OAAQ7O,IAAa,GAGrBxK,OAAQwK,IAAa,ICxB+BxK,OAKlDgS,GAAmBC,GAAoBz0B,IACvC00B,GAAmBD,GAAoB1B,UAFrB,mBAMtBiG,GAAevO,OAAQ,UAAU,SAAUwO,GACzCzE,GAAiBx4B,KAAM,CACrBk3B,KARkB,kBASlBjgB,OAAQwX,OAAOwO,GACfrY,MAAO,OAIR,eAIGkb,EAHA7I,EAAQyB,GAAiB14B,MACzBiX,EAASggB,EAAMhgB,OACf2N,EAAQqS,EAAMrS,aAEdA,GAAS3N,EAAOja,OAAe,CAAE8E,WAAOrC,EAAW09B,MAAM,IAC7D2C,EAAQtZ,GAAOvP,EAAQ2N,GACvBqS,EAAMrS,OAASkb,EAAM9iC,OACd,CAAE8E,MAAOg+B,EAAO3C,MAAM,OCvB/B,IAAItB,GAAWrG,GAAgB,eAEd,SAAU1J,MACfrsB,MAANqsB,EAAiB,OAAOA,EAAG+P,KAC1B/P,EAAG,eACHqQ,GAAUpP,GAAQjB,QCNR,SAAUA,OACrBiU,EAAiBC,GAAkBlU,MACV,mBAAlBiU,QACH/S,UAAUyB,OAAO3C,GAAM,2BACtB+C,EAASkR,EAAe1jC,KAAKyvB,OCDtC,CAAE6D,OAAQ,SAAUG,MAAM,EAAMS,MAAOhD,GAAe,CACtDyH,OAAQA,KCJV,IAAI94B,GAAS+zB,EAAK/zB,UAED,SAAgBwxB,EAAGtF,UAC3BlsB,GAAO84B,OAAOtH,EAAGtF,OCJTwI,MCAAA,ECejB,OAfA,SAAyBvvB,EAAK4B,EAAKnB,UAC7BmB,KAAO5B,EACT4+B,GAAuB5+B,EAAK4B,EAAK,CAC/BnB,MAAOA,EACP6qB,YAAY,EACZE,cAAc,EACdC,UAAU,IAGZzrB,EAAI4B,GAAOnB,EAGNT,GCTLsxB,GAAsBvG,GAAM,WAAc8T,GAAW,QAIvD,CAAEvQ,OAAQ,SAAUG,MAAM,EAAMK,OAAQwC,IAAuB,CAC/DrvB,KAAM,SAAcwoB,UACXoU,GAAWvd,GAASmJ,OCR/B,ICI2BqU,MDJVlQ,EAAK/zB,OAAOoH,QEDZ,gDCCb88B,GAAa,IAAMC,GAAc,IACjCC,GAAQh4B,OAAO,IAAM83B,GAAaA,GAAa,KAC/CG,GAAQj4B,OAAO83B,GAAaA,GAAa,MAGzCpP,GAAe,SAAUgG,UACpB,SAAU9F,OACXja,EAASwX,OAAOvB,EAAuBgE,WAChC,EAAP8F,IAAU/f,EAASA,EAAO1Q,QAAQ+5B,GAAO,KAClC,EAAPtJ,IAAU/f,EAASA,EAAO1Q,QAAQg6B,GAAO,KACtCtpB,OAIM,CAGfupB,MAAOxP,GAAa,GAGpByP,IAAKzP,GAAa,GAGlB0P,KAAM1P,GAAa,ICxBjB2P,GAAQnW,GAAoCkW,OAK9C,CAAE/Q,OAAQ,SAAUnP,OAAO,EAAM2P,QHARgQ,GGAuC,OHCzD/T,GAAM,mBACFiU,GAAYF,OANf,OAAA,MAMqCA,OAAyBE,GAAYF,IAAa18B,OAAS08B,QGF7B,CAC3EO,KAAM,kBACGC,GAAM3gC,SCPjB,OAAiB,SAAU4gC,UAClB3Q,EAAK2Q,EAAc,iBCAXC,GAAa,UAAUH,KCDpCI,GAAkBrS,OAAOtyB,aAEZ,SAAU2vB,OACrBiV,EAAMjV,EAAG4U,WACQ,iBAAP5U,GAAmBA,IAAOgV,IAClChV,aAAc2C,QAAUsS,IAAQD,GAAgBJ,KAAQA,GAAOK,MCJtD,SAAUZ,EAAatP,OAClCoB,EAAS,GAAGkO,WACPlO,GAAU7F,GAAM,WAEvB6F,EAAO51B,KAAK,KAAMw0B,GAAY,iBAAoB,GAAM,OCHxDxE,GAAiBnwB,OAAOmwB,eACxB2U,GAAQ,GAERC,GAAU,SAAUnV,SAAYA,MAEnB,SAAUqU,EAAahR,MAClCrB,EAAIkT,GAAOb,GAAc,OAAOa,GAAMb,GACrChR,IAASA,EAAU,QACpB8C,EAAS,GAAGkO,GACZe,IAAYpT,EAAIqB,EAAS,cAAeA,EAAQ+R,UAChDC,EAAYrT,EAAIqB,EAAS,GAAKA,EAAQ,GAAK8R,GAC3CG,EAAYtT,EAAIqB,EAAS,GAAKA,EAAQ,QAAK1vB,SAExCuhC,GAAMb,KAAiBlO,IAAW7F,GAAM,cACzC8U,IAAc3T,EAAa,OAAO,MAClCE,EAAI,CAAEzwB,QAAS,GAEfkkC,EAAW7U,GAAeoB,EAAG,EAAG,CAAEd,YAAY,EAAM5iB,IAAKk3B,KACxDxT,EAAE,GAAK,EAEZwE,EAAO51B,KAAKoxB,EAAG0T,EAAWC,OCvB1B/I,GAAW7N,GAAwC3W,QAInDwtB,GAAgBC,GAAoB,WACpCC,GAAiBC,GAAwB,cAI1BH,IAAkBE,GAEjC,GAAG1tB,QAFgD,SAAiBgkB,UAC/DQ,GAASr4B,KAAM63B,EAAY/7B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCLxE,CAAEkwB,OAAQ,QAASnP,OAAO,EAAM2P,OAAQ,GAAGtc,SAAWA,IAAW,CACjEA,QAASA,KCJX,ICGIvN,GAAOyiB,MDHM8X,GAAa,SAAShtB,QEAnC4tB,GAAiBxlC,MAAME,UAEvBulC,GAAe,CACjB9D,cAAc,EACdU,UAAU,MAGK,SAAUxS,OACrBiV,EAAMjV,EAAGjY,eACNiY,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAe5tB,SAE1E6tB,GAAarkC,eAAe0vB,GAAQjB,IAAOjY,GAAUktB,MCZ3C5O,GAAW,YAAa,cAAgB,GFCrDwP,GAAUzV,EAAOyV,QACjBC,GAAWD,IAAWA,GAAQC,SAC9BC,GAAKD,IAAYA,GAASC,GAG1BA,GAEF9Y,IADAziB,GAAQu7B,GAAGh3B,MAAM,MACD,GAAKvE,GAAM,GAClBw7B,OACTx7B,GAAQw7B,GAAUx7B,MAAM,iBACVA,GAAM,IAAM,MACxBA,GAAQw7B,GAAUx7B,MAAM,oBACbyiB,GAAUziB,GAAM,IAI/B,OAAiByiB,KAAYA,GGfzBoO,GAAU3B,GAAgB,cAEb,SAAU2K,UAIlB4B,IAAc,KAAO3V,GAAM,eAC5B/lB,EAAQ,UACMA,EAAMixB,YAAc,IAC1BH,IAAW,iBACd,CAAE6K,IAAK,IAE2B,IAApC37B,EAAM85B,GAAa8B,SAASD,QCdnCE,GAAO1X,GAAwC7tB,IAI/CwlC,GAAsBC,GAA6B,OAEnDb,GAAiBC,GAAwB,SAK3C,CAAE7R,OAAQ,QAASnP,OAAO,EAAM2P,QAASgS,KAAwBZ,IAAkB,CACnF5kC,IAAK,SAAak7B,UACTqK,GAAKliC,KAAM63B,EAAY/7B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCZxE,OAAiBohC,GAAa,SAASlkC,ICDnC8kC,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGnvB,WACNmvB,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAe9kC,IAAOA,GAAMokC,GCL1FL,GAAOlW,GAAoCkW,KAG3C2B,GAAYnW,EAAOziB,SACnB64B,GAAM,iBACqC,IAAlCD,GAAUhC,GAAc,OAAmD,KAApCgC,GAAUhC,GAAc,QAIlD,SAAkBppB,EAAQsrB,OAC9C9C,EAAIiB,GAAKjS,OAAOxX,WACborB,GAAU5C,EAAI8C,IAAU,IAAOD,GAAIv7B,KAAK04B,GAAK,GAAK,MACvD4C,KCRF,CAAEnW,QAAQ,EAAMiE,OAAQ1mB,UAAY+4B,IAA0B,CAC9D/4B,SAAU+4B,KCHZ,OAAiBvS,EAAKxmB,SCAlB8iB,GAAuB/B,EAAsDyD,EAG7E+C,GAAe,SAAUyR,UACpB,SAAU3W,WAMX7oB,EALAwqB,EAAIE,EAAgB7B,GACpBxoB,EAAOuuB,GAAWpE,GAClBzwB,EAASsG,EAAKtG,OACdF,EAAI,EACJyb,EAAS,GAENvb,EAASF,GACdmG,EAAMK,EAAKxG,KACNywB,IAAehB,GAAqBlwB,KAAKoxB,EAAGxqB,IAC/CsV,EAAOtb,KAAKwlC,EAAa,CAACx/B,EAAKwqB,EAAExqB,IAAQwqB,EAAExqB,WAGxCsV,ICnBPmqB,GDuBa,CAGf3F,QAAS/L,IAAa,GAGtB1d,OAAQ0d,IAAa,IC7B+B1d,SAIpD,CAAEqc,OAAQ,SAAUG,MAAM,GAAQ,CAClCxc,OAAQ,SAAgBma,UACfiV,GAAQjV,MCJnB,OAAiBwC,EAAK/zB,OAAOoX,OCDzBqvB,GAAUnY,GAAwCyN,OAIlDkK,GAAsBC,GAA6B,UAEnDb,GAAiBC,GAAwB,YAK3C,CAAE7R,OAAQ,QAASnP,OAAO,EAAM2P,QAASgS,KAAwBZ,IAAkB,CACnFtJ,OAAQ,SAAgBJ,UACf8K,GAAQ3iC,KAAM63B,EAAY/7B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCZ3E,OAAiBohC,GAAa,SAAS5I,OCDnCwJ,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGmM,cACNnM,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAexJ,OAAUA,GAAS8I,GCOhG6B,GAAuBpN,GAAgB,sBAOvCqN,GAA+Bd,IAAc,KAAO3V,GAAM,eACxD/lB,EAAQ,UACZA,EAAMu8B,KAAwB,EACvBv8B,EAAM+H,SAAS,KAAO/H,KAG3By8B,GAAkBV,GAA6B,UAE/CW,GAAqB,SAAUtV,OAC5BnxB,EAASmxB,GAAI,OAAO,MACrBuV,EAAavV,EAAEmV,gBACGnjC,IAAfujC,IAA6BA,EAAajnC,GAAQ0xB,MAQzD,CAAEkC,OAAQ,QAASnP,OAAO,EAAM2P,QALpB0S,KAAiCC,IAKK,CAClD10B,OAAQ,SAAgBrL,OAIlBjG,EAAG6d,EAAG3d,EAAQiD,EAAKyV,EAHnB+X,EAAI9K,GAAS3iB,MACbijC,EAAIjL,GAAmBvK,EAAG,GAC1Btf,EAAI,MAEHrR,GAAK,EAAGE,EAASlB,UAAUkB,OAAQF,EAAIE,EAAQF,OAClD4Y,GAAW,IAAP5Y,EAAW2wB,EAAI3xB,UAAUgB,GACzBimC,GAAmBrtB,GAAI,IAErBvH,GADJlO,EAAMoxB,EAAS3b,EAAE1Y,SAlCF,iBAmCiB,MAAMgwB,UAlCT,sCAmCxBrS,EAAI,EAAGA,EAAI1a,EAAK0a,IAAKxM,IAASwM,KAAKjF,GAAGgd,GAAeuQ,EAAG90B,EAAGuH,EAAEiF,QAC7D,IACDxM,GAtCW,iBAsCY,MAAM6e,UArCJ,kCAsC7B0F,GAAeuQ,EAAG90B,IAAKuH,UAG3ButB,EAAEjmC,OAASmR,EACJ80B,KCtDX,OAAiBpC,GAAa,SAASzyB,OCDnCqzB,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAG1d,cACN0d,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAerzB,OAAUA,GAAS2yB,KCDlG,CAAEpR,OAAQ,QAASG,MAAM,GAAQ,CACjC/zB,QAASA,KCHX,OAAiBk0B,EAAKh0B,MAAMF,WCDX60B,GCUjB,OAVA,SAA4Bh0B,MACtBsmC,GAAetmC,GAAM,KAClB,IAAIE,EAAI,EAAGqmC,EAAO,IAAIlnC,MAAMW,EAAII,QAASF,EAAIF,EAAII,OAAQF,IAC5DqmC,EAAKrmC,GAAKF,EAAIE,UAGTqmC,OCLM,SAAUpQ,EAAUl2B,EAAIiF,EAAOshC,cAErCA,EAAUvmC,EAAGgyB,EAAS/sB,GAAO,GAAIA,EAAM,IAAMjF,EAAGiF,GAEvD,MAAOqqB,OACHkX,EAAetQ,EAAQ,kBACNtzB,IAAjB4jC,GAA4BxU,EAASwU,EAAahnC,KAAK02B,IACrD5G,ICPN0P,GAAWrG,GAAgB,YAC3BiM,GAAiBxlC,MAAME,aAGV,SAAU2vB,eACXrsB,IAAPqsB,IAAqBqQ,GAAUlgC,QAAU6vB,GAAM2V,GAAe5F,MAAc/P,ICNjF+P,GAAWrG,GAAgB,YAC3B8N,IAAe,EAEnB,QACMC,GAAS,EACTC,GAAqB,CACvBxvB,KAAM,iBACG,CAAEmpB,OAAQoG,cAET,WACRD,IAAe,IAGnBE,GAAmB3H,IAAY,kBACtB77B,MAGT/D,MAAMoE,KAAKmjC,IAAoB,iBAAoB,KACnD,MAAOrX,IAET,IClBIsX,IDkBa,SAAUvsB,EAAMwsB,OAC1BA,IAAiBJ,GAAc,OAAO,MACvCK,GAAoB,UAElB7U,EAAS,GACbA,EAAO+M,IAAY,iBACV,CACL7nB,KAAM,iBACG,CAAEmpB,KAAMwG,GAAoB,MAIzCzsB,EAAK4X,GACL,MAAO3C,WACFwX,EChCkBC,EAA4B,SAAUC,GAC/D5nC,MAAMoE,KAAKwjC,QAKX,CAAElU,OAAQ,QAASG,MAAM,EAAMK,OAAQsT,IAAuB,CAC9DpjC,KCAe,SAAcyjC,OAQzB9mC,EAAQub,EAAQwrB,EAAMhR,EAAU/e,EAAMlS,EAPtC2rB,EAAI9K,GAASmhB,GACbzM,EAAmB,mBAARr3B,KAAqBA,KAAO/D,MACvC+nC,EAAkBloC,UAAUkB,OAC5BinC,EAAQD,EAAkB,EAAIloC,UAAU,QAAK2D,EAC7CykC,OAAoBzkC,IAAVwkC,EACVlE,EAAiBC,GAAkBvS,GACnC7I,EAAQ,KAERsf,IAASD,EAAQ5T,EAAK4T,EAAOD,EAAkB,EAAIloC,UAAU,QAAK2D,EAAW,IAE3DA,MAAlBsgC,GAAiC1I,GAAKp7B,OAASkoC,GAAsBpE,OAUvExnB,EAAS,IAAI8e,EADbr6B,EAASq0B,EAAS5D,EAAEzwB,SAEdA,EAAS4nB,EAAOA,IACpB9iB,EAAQoiC,EAAUD,EAAMxW,EAAE7I,GAAQA,GAAS6I,EAAE7I,GAC7C8N,GAAena,EAAQqM,EAAO9iB,YAXhCkS,GADA+e,EAAWgN,EAAe1jC,KAAKoxB,IACfzZ,KAChBuE,EAAS,IAAI8e,IACL0M,EAAO/vB,EAAK3X,KAAK02B,IAAWoK,KAAMvY,IACxC9iB,EAAQoiC,EAAUE,GAA6BrR,EAAUkR,EAAO,CAACF,EAAKjiC,MAAO8iB,IAAQ,GAAQmf,EAAKjiC,MAClG4wB,GAAena,EAAQqM,EAAO9iB,UAUlCyW,EAAOvb,OAAS4nB,EACTrM,KCnCT,OAAiB0X,EAAKh0B,MAAMoE,KCAxBw7B,GAAWrG,GAAgB,eAEd,SAAU1J,OACrB2B,EAAIvxB,OAAO4vB,eACQrsB,IAAhBguB,EAAEoO,KACJ,eAAgBpO,GAEhB0O,GAAU9+B,eAAe0vB,GAAQU,KCHxC,OAJA,SAA0B4W,MACpBC,GAAYpoC,OAAOmoC,KAAmD,uBAAzCnoC,OAAOC,UAAUC,SAASC,KAAKgoC,GAAgC,OAAOE,GAAYF,ICDrH,OAJA,iBACQ,IAAIrX,UAAU,oDCStB,OAJA,SAA4BpwB,UACnB4nC,GAAkB5nC,IAAQ6nC,GAAgB7nC,IAAQ8nC,MCKvDvC,GAAsBC,GAA6B,SACnDb,GAAiBC,GAAwB,QAAS,CAAEN,WAAW,IAAS,IAAM,IAE9E/J,GAAU3B,GAAgB,WAC1BmP,GAAc,GAAGzhC,MACjBoC,GAAM9D,KAAK8D,MAKb,CAAEqqB,OAAQ,QAASnP,OAAO,EAAM2P,QAASgS,KAAwBZ,IAAkB,CACnFr+B,MAAO,SAAes9B,EAAOC,OAMvBmE,EAAarsB,EAAQpK,EALrBsf,EAAIE,EAAgB3tB,MACpBhD,EAASq0B,EAAS5D,EAAEzwB,QACpB2d,EAAI2W,GAAgBkP,EAAOxjC,GAC3B6nC,EAAMvT,QAAwB7xB,IAARghC,EAAoBzjC,EAASyjC,EAAKzjC,MAGxDjB,GAAQ0xB,KAGgB,mBAF1BmX,EAAcnX,EAAE6J,cAEyBsN,IAAgB3oC,QAASF,GAAQ6oC,EAAYzoC,WAE3EG,EAASsoC,IAEE,QADpBA,EAAcA,EAAYzN,OACAyN,OAAcnlC,GAHxCmlC,OAAcnlC,EAKZmlC,IAAgB3oC,YAAyBwD,IAAhBmlC,UACpBD,GAAYtoC,KAAKoxB,EAAG9S,EAAGkqB,OAGlCtsB,EAAS,SAAqB9Y,IAAhBmlC,EAA4B3oC,MAAQ2oC,GAAat/B,GAAIu/B,EAAMlqB,EAAG,IACvExM,EAAI,EAAGwM,EAAIkqB,EAAKlqB,IAAKxM,IAASwM,KAAK8S,GAAGiF,GAAena,EAAQpK,EAAGsf,EAAE9S,WACvEpC,EAAOvb,OAASmR,EACToK,KC3CX,OAAiBsoB,GAAa,SAAS39B,MCDnCu+B,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAG5oB,aACN4oB,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAev+B,MAASA,GAAQ69B,GCA9FpO,GAAsBvG,GAAM,WAAc0Y,GAAqB,QAIjE,CAAEnV,OAAQ,SAAUG,MAAM,EAAMK,OAAQwC,GAAqBpC,MAAO8K,IAA4B,CAChGD,eAAgB,SAAwBtP,UAC/BgZ,GAAqBniB,GAASmJ,OCTzC,OAAiBmE,EAAK/zB,OAAOk/B,kBCDZxK,GCAbmU,GAAWva,GAAuC9gB,QAIlDs7B,GAAgB,GAAGt7B,QAEnBu7B,KAAkBD,IAAiB,EAAI,CAAC,GAAGt7B,QAAQ,GAAI,GAAK,EAC5D23B,GAAgBC,GAAoB,WACpCC,GAAiBC,GAAwB,UAAW,CAAEN,WAAW,IAAS,MAI5E,CAAEvR,OAAQ,QAASnP,OAAO,EAAM2P,OAAQ8U,KAAkB5D,KAAkBE,IAAkB,CAC9F73B,QAAS,SAAiBw7B,UACjBD,GAEHD,GAAcnpC,MAAMmE,KAAMlE,YAAc,EACxCipC,GAAS/kC,KAAMklC,EAAeppC,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MChB5E,OAAiBohC,GAAa,SAASn3B,QCDnC+3B,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGpiB,eACNoiB,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAe/3B,QAAWA,GAAUq3B,MCJrFnQ,GCObuU,GAAejpC,OAAOkpC,OACtB/Y,GAAiBnwB,OAAOmwB,mBAIV8Y,IAAgB/Y,GAAM,cAElCmB,GAQiB,IARF4X,GAAa,CAAE/nC,EAAG,GAAK+nC,GAAa9Y,GAAe,GAAI,IAAK,CAC7EM,YAAY,EACZ5iB,IAAK,WACHsiB,GAAersB,KAAM,IAAK,CACxB8B,MAAO,EACP6qB,YAAY,OAGd,CAAEvvB,EAAG,KAAMA,EAAS,OAAO,MAE3B6lC,EAAI,GACJoC,EAAI,GAEJ3L,EAAS7G,gBAEboQ,EAAEvJ,GAAU,EADG,uBAEN7uB,MAAM,IAAIgJ,SAAQ,SAAUyxB,GAAOD,EAAEC,GAAOA,KACf,GAA/BH,GAAa,GAAIlC,GAAGvJ,IAHZ,wBAG4B7H,GAAWsT,GAAa,GAAIE,IAAIliC,KAAK,OAC7E,SAAgBwsB,EAAQ9J,WACvBiM,EAAInP,GAASgN,GACbqU,EAAkBloC,UAAUkB,OAC5B4nB,EAAQ,EACRsN,EAAwBG,GAA4BpE,EACpD1B,EAAuByB,EAA2BC,EAC/C+V,EAAkBpf,WAKnB3hB,EAJAw8B,EAAIxS,EAAcnxB,UAAU8oB,MAC5BthB,EAAO4uB,EAAwBL,GAAW4N,GAAGrxB,OAAO8jB,EAAsBuN,IAAM5N,GAAW4N,GAC3FziC,EAASsG,EAAKtG,OACd+W,EAAI,EAED/W,EAAS+W,GACd9Q,EAAMK,EAAKyQ,KACNwZ,IAAehB,EAAqBlwB,KAAKojC,EAAGx8B,KAAM6uB,EAAE7uB,GAAOw8B,EAAEx8B,WAE7D6uB,GACPqT,KC9CF,CAAExV,OAAQ,SAAUG,MAAM,EAAMK,OAAQj0B,OAAOkpC,SAAWA,IAAU,CACpEA,OAAQA,KCHV,OAAiBnV,EAAK/zB,OAAOkpC,OCC7B/K,GAAsB,YCCtB,OAAiBkL,GAA6BtX,EAAE,eCH/B2C,sCCEjByJ,GAAsB,wDCAtBA,GAAsB,eCAtBA,GAAsB,sBCAtBA,GAAsB,SCDtBA,GAAsB,YCCtBA,GAAsB,WCAtBA,GAAsB,UCAtBA,GAAsB,WCAtBA,GAAsB,SCAtBA,GAAsB,eCAtBA,GAAsB,eCAtBA,GAAsB,eCAtBW,GAAex5B,KAAM,QAAQ,GCC7Bw5B,GAAe9O,EAAOsZ,KAAM,QAAQ,eCgBpC,OAAiBvV,EAAK4C,OCjBtBwH,GAAsB,gBCAtBA,GAAsB,WCAtBA,GAAsB,cCAtBA,GAAsB,gBCDtBA,GAAsB,cCKtB,OAAiBzJ,8BCJR6U,EAAQpkC,UAIb5F,UAAiBgqC,EADI,mBAAZC,IAAsD,iBAArBC,GACf,SAAiBtkC,iBAC5BA,GAGW,SAAiBA,UACnCA,GAA0B,mBAAZqkC,IAA0BrkC,EAAIi2B,cAAgBoO,IAAWrkC,IAAQqkC,GAAQvpC,UAAY,gBAAkBkF,GAIzHokC,EAAQpkC,GAGjB5F,UAAiBgqC,KCdjB,OAJA,SAAyB7oC,MACnBsmC,GAAetmC,GAAM,OAAOA,GC+BlC,OA9BA,SAA+BA,EAAKE,MAC5BwnC,GAAYpoC,OAAOU,KAAiD,uBAAxCV,OAAOC,UAAUC,SAASC,KAAKO,QAI7DgpC,EAAO,GACPC,GAAK,EACLxmC,GAAK,EACLymC,OAAKrmC,UAGF,IAA4BsmC,EAAxBtlC,EAAKulC,GAAappC,KAAYipC,GAAME,EAAKtlC,EAAGuT,QAAQmpB,QAC3DyI,EAAK3oC,KAAK8oC,EAAGjkC,QAEThF,GAAK8oC,EAAK5oC,SAAWF,GAHyC+oC,GAAK,IAKzE,MAAOI,GACP5mC,GAAK,EACLymC,EAAKG,cAGEJ,GAAsB,MAAhBplC,EAAE,QAAoBA,EAAE,oBAE/BpB,EAAI,MAAMymC,UAIXF,IC3BT,OAJA,iBACQ,IAAI5Y,UAAU,yDCStB,OAJA,SAAwBpwB,EAAKE,UACpBopC,GAAetpC,IAAQupC,GAAqBvpC,EAAKE,IAAMspC,QCH9D,CAAEzW,OAAQ,OAAQG,MAAM,GAAQ,CAChCjb,IAAK,kBACI,IAAInY,MAAO4C,aCHtB,OAAiB2wB,EAAKvzB,KAAKmY,6IC4C3B,SAASwxB,GAAmBC,SA+B5B,mBACQC,EAAOC,KAETC,EAAKF,EAAK,KACVG,EAAKH,EAAK,KACVI,EAAKJ,EAAK,KAELzpC,EAAI,EAAGA,EAAIhB,UAAKkB,OAAQF,KAC/B2pC,GAAMF,EAAUzpC,uBAAAA,mBAAAA,KACP,IACP2pC,GAAM,IAERC,GAAMH,EAAUzpC,uBAAAA,mBAAAA,KACP,IACP4pC,GAAM,IAERC,GAAMJ,EAAUzpC,uBAAAA,mBAAAA,KACP,IACP6pC,GAAM,SAIH,CAACF,EAAIC,EAAIC,GApDGC,CAASN,aAAvBG,OAAIC,OAAIC,OACTpyB,EAAI,EAEF+e,EAAc,eACZvzB,EAAI,QAAU0mC,EAAS,uBAAJlyB,SACzBkyB,EAAKC,EACLA,EAAKC,EACGA,EAAK5mC,GAAKwU,EAAQ,EAAJxU,WAGxBuzB,EAAOuT,OAAS,kBAAyB,WAAXvT,KAE9BA,EAAOwT,QAAU,kBACfxT,IAAyC,uBAAjB,QAAXA,IAAuB,IAEtCA,EAAOyT,UAAY,OACnBzT,EAAOgT,KAAOA,EACdhT,EAAOvK,QAAU,MAEVuK,EA+CT,SAASkT,SACHr4B,EAAI,kBAED,SAASoF,WACR0D,EAAS1D,EAAKnX,WACXU,EAAI,EAAGA,EAAIma,EAAOja,OAAQF,IAAK,KAElC0P,EAAI,oBADR2B,GAAK8I,EAAO2oB,WAAW9iC,IAGvB0P,GADA2B,EAAI3B,IAAM,EAGV2B,GADA3B,GAAK2B,KACK,EAEVA,GAAS,YADT3B,GAAK2B,UAGY,wBAAXA,IAAM;;;;;;;;;;;;;;;;;;;;;;;;;;sCA7FMm4B,2BAAAA,yBACfD,GAAmBC,EAAKtpC,OAASspC,EAAO,CAACU,ytSC9B3C,SAASC,GAAgBC,OAEzB,IAAIC,KAAeD,EAClBA,EAAc7pC,eAAe8pC,KAC/BD,EAAcC,GAAaC,UAAYF,EAAcC,GAAaE,KAClEH,EAAcC,GAAaE,KAAO,IAYjC,SAASC,GAAgBJ,OAEzB,IAAIC,KAAeD,KAClBA,EAAc7pC,eAAe8pC,IAC3BD,EAAcC,GAAaC,UAAW,KACnC,IAAItqC,EAAI,EAAGA,EAAIoqC,EAAcC,GAAaC,UAAUpqC,OAAQF,IAC/DoqC,EAAcC,GAAaC,UAAUtqC,GAAGyqC,WAAWC,YAAYN,EAAcC,GAAaC,UAAUtqC,IAEtGoqC,EAAcC,GAAaC,UAAY,IAUxC,SAASK,GAAcP,GAC5BD,GAAgBC,GAChBI,GAAgBJ,GAChBD,GAAgBC,GAaX,SAASQ,GAAcP,EAAaD,EAAeS,OACpDC,SAEAV,EAAc7pC,eAAe8pC,GAE3BD,EAAcC,GAAaC,UAAUpqC,OAAS,GAChD4qC,EAAUV,EAAcC,GAAaC,UAAU,GAC/CF,EAAcC,GAAaC,UAAUjiB,UAIrCyiB,EAAUxa,SAASya,gBAAgB,6BAA8BV,GACjEQ,EAAahT,YAAYiT,KAK3BA,EAAUxa,SAASya,gBAAgB,6BAA8BV,GACjED,EAAcC,GAAe,CAACE,KAAM,GAAID,UAAW,IACnDO,EAAahT,YAAYiT,IAE3BV,EAAcC,GAAaE,KAAKpqC,KAAK2qC,GAC9BA,EAcF,SAASE,GAAcX,EAAaD,EAAea,EAAcC,OAClEJ,SAEAV,EAAc7pC,eAAe8pC,GAE3BD,EAAcC,GAAaC,UAAUpqC,OAAS,GAChD4qC,EAAUV,EAAcC,GAAaC,UAAU,GAC/CF,EAAcC,GAAaC,UAAUjiB,UAIrCyiB,EAAUxa,SAASE,cAAc6Z,QACZ1nC,IAAjBuoC,EACFD,EAAaC,aAAaJ,EAASI,GAGnCD,EAAapT,YAAYiT,KAM7BA,EAAUxa,SAASE,cAAc6Z,GACjCD,EAAcC,GAAe,CAACE,KAAM,GAAID,UAAW,SAC9B3nC,IAAjBuoC,EACFD,EAAaC,aAAaJ,EAASI,GAGnCD,EAAapT,YAAYiT,IAG7BV,EAAcC,GAAaE,KAAKpqC,KAAK2qC,GAC9BA,EAmBF,SAASK,GAAU99B,EAAGZ,EAAG2+B,EAAehB,EAAeS,EAAcQ,OACtErI,KACuB,UAAvBoI,EAAc1T,QAChBsL,EAAQ4H,GAAc,SAAUR,EAAeS,IACzCS,eAAe,KAAM,KAAMj+B,GACjC21B,EAAMsI,eAAe,KAAM,KAAM7+B,GACjCu2B,EAAMsI,eAAe,KAAM,IAAK,GAAMF,EAAcvI,SAGpDG,EAAQ4H,GAAc,OAAQR,EAAeS,IACvCS,eAAe,KAAM,IAAKj+B,EAAI,GAAM+9B,EAAcvI,MACxDG,EAAMsI,eAAe,KAAM,IAAK7+B,EAAI,GAAM2+B,EAAcvI,MACxDG,EAAMsI,eAAe,KAAM,QAASF,EAAcvI,MAClDG,EAAMsI,eAAe,KAAM,SAAUF,EAAcvI,YAGxBlgC,IAAzByoC,EAAcG,QAChBvI,EAAMsI,eAAe,KAAM,QAASF,EAAcG,QAEpDvI,EAAMsI,eAAe,KAAM,QAASF,EAAcI,UAAY,cAI1DH,EAAU,KACRI,EAAQb,GAAc,OAAQR,EAAeS,GAC7CQ,EAASK,UACXr+B,GAAQg+B,EAASK,SAGfL,EAASM,UACXl/B,GAAQ4+B,EAASM,SAEfN,EAAStU,UACX0U,EAAMG,YAAcP,EAAStU,SAG3BsU,EAASG,WACXC,EAAMH,eAAe,KAAM,QAASD,EAASG,UAAa,cAE5DC,EAAMH,eAAe,KAAM,IAAKj+B,GAChCo+B,EAAMH,eAAe,KAAM,IAAK7+B,UAG3Bu2B,EAeF,SAAS6I,GAASx+B,EAAGZ,EAAGq/B,EAAOC,EAAQP,EAAWpB,EAAeS,EAAcnT,MACtE,GAAVqU,EAAa,CACXA,EAAS,IAEXt/B,GADAs/B,IAAW,OAGTC,EAAOpB,GAAc,OAAOR,EAAeS,GAC/CmB,EAAKV,eAAe,KAAM,IAAKj+B,EAAI,GAAMy+B,GACzCE,EAAKV,eAAe,KAAM,IAAK7+B,GAC/Bu/B,EAAKV,eAAe,KAAM,QAASQ,GACnCE,EAAKV,eAAe,KAAM,SAAUS,GACpCC,EAAKV,eAAe,KAAM,QAASE,GAC/B9T,GACFsU,EAAKV,eAAe,KAAM,QAAS5T,IASlC,SAASuU,gBAEPC,UACDA,UAAUC,WAAaD,UAAUC,UAAUjsC,OACtCgsC,UAAUC,UAEVD,UAAUE,cAAgBF,UAAUG,UAAYH,UAAUI,iBAAmB,KAJ/D,KAOzB,MAAMjd,SACG,sLCnOX,OARA,SAAgCF,WACjB,IAATA,QACI,IAAIod,eAAe,oEAGpBpd,MCFQ4U,GAAa,SAAS9D,QCAnC0E,GAAiBxlC,MAAME,UAEvBulC,GAAe,CACjB9D,cAAc,EACdU,UAAU,MAGK,SAAUxS,OACrBiV,EAAMjV,EAAGiR,eACNjR,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAe1E,SAE1E2E,GAAarkC,eAAe0vB,GAAQjB,IAAOiR,GAAUgE,GCVxDlI,GAAa1G,GAAW,OAAQ,aAChCmX,GAAK,mBACLC,GAAM,oBACNC,GAAK,oBAELC,GAAM,SAAUnjC,EAAOkW,EAAQvF,OAC7ByyB,EAAOzyB,EAAOuP,OAAOhK,EAAS,GAC9BxI,EAAOiD,EAAOuP,OAAOhK,EAAS,UAC7B+sB,GAAIxiC,KAAKT,KAAWkjC,GAAGziC,KAAKiN,IAAWw1B,GAAGziC,KAAKT,KAAWijC,GAAIxiC,KAAK2iC,GAC/D,MAAQpjC,EAAMs5B,WAAW,GAAGxjC,SAAS,IACrCkK,GAGP41B,GAAS9P,GAAM,iBACqB,qBAA/ByM,GAAW,iBACY,cAAzBA,GAAW,aAGdA,IAEF8B,EAAE,CAAEhL,OAAQ,OAAQG,MAAM,EAAMK,OAAQ+L,IAAU,CAEhDtB,UAAW,SAAmB9O,EAAI+O,EAAUC,OACtCviB,EAASsgB,GAAWh9B,MAAM,KAAMC,iBACZ,iBAAVyc,EAAqBA,EAAOhS,QAAQ+iC,GAAIG,IAAOlxB,KCzB9DoxB,EAAKnE,OAAMmE,EAAKnE,KAAO,CAAE5K,UAAW4K,KAAK5K,YAG9C,OAAiB,SAAmB9O,EAAI+O,EAAUC,UACzC6O,EAAKnE,KAAK5K,UAAU/+B,MAAM,KAAMC,eCJxB+kC,GAAa,SAASvtB,OCAnCmuB,GAAiBxlC,MAAME,UAEvBulC,GAAe,CACjB9D,cAAc,EACdU,UAAU,MAGK,SAAUxS,OACrBiV,EAAMjV,EAAGxY,cACNwY,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAenuB,QAE1EouB,GAAarkC,eAAe0vB,GAAQjB,IAAOxY,GAASytB,MCX1CF,GAAa,SAASv9B,KCAnCm+B,GAAiBxlC,MAAME,UAEvBulC,GAAe,CACjB9D,cAAc,EACdU,UAAU,MAGK,SAAUxS,OACrBiV,EAAMjV,EAAGxoB,YACNwoB,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAen+B,MAE1Eo+B,GAAarkC,eAAe0vB,GAAQjB,IAAOxoB,GAAOy9B,GCZrD6I,GAAQpf,GAAwC7uB,KAIhD0lC,GAAgBC,GAAoB,QACpCC,GAAiBC,GAAwB,UAI3C,CAAE7R,OAAQ,QAASnP,OAAO,EAAM2P,QAASkR,KAAkBE,IAAkB,CAC7E5lC,KAAM,SAAck8B,UACX+R,GAAM5pC,KAAM63B,EAAY/7B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCVzE,OAAiBohC,GAAa,SAASllC,KCDnC8lC,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGnwB,YACNmwB,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAe9lC,KAAQA,GAAOolC,GCMhG,OARA,SAAoC9U,EAAM5vB,UACpCA,GAA2B,WAAlBopC,GAAQppC,IAAsC,mBAATA,EAI3CwtC,GAAsB5d,GAHpB5vB,MCJMu0B,KCGf,CAAEjB,OAAQ,SAAUG,MAAM,GAAQ,CAClC0L,eAAgBA,KCHlB,OAAiBvL,EAAK/zB,OAAOs/B,0CCCpBsO,EAAgBz/B,UACvB5O,UAAiBquC,EAAkBC,GAAyBC,GAAyB,SAAyB3/B,UACrGA,EAAEsxB,WAAaqO,GAAuB3/B,IAExCy/B,EAAgBz/B,GAGzB5O,UAAiBquC,QCTAlZ,8BCARqZ,EAAgB5/B,EAAG6/B,UAC1BzuC,UAAiBwuC,EAAkBF,IAA0B,SAAyB1/B,EAAG6/B,UACvF7/B,EAAEsxB,UAAYuO,EACP7/B,GAGF4/B,EAAgB5/B,EAAG6/B,GAG5BzuC,UAAiBwuC,KCQjB,OAfA,SAAmBE,EAAUC,MACD,mBAAfA,GAA4C,OAAfA,QAChC,IAAIpd,UAAU,sDAGtBmd,EAAShuC,UAAYkuC,GAAeD,GAAcA,EAAWjuC,UAAW,CACtEm7B,YAAa,CACXx1B,MAAOqoC,EACPrd,UAAU,EACVD,cAAc,KAGdud,GAAY5O,GAAe2O,EAAUC,ICTvCrjC,GAAO,GACPujC,GAAavjC,GAAKkF,KAGlBs+B,GAAqBne,GAAM,WAC7BrlB,GAAKkF,UAAKxM,MAGR+qC,GAAgBpe,GAAM,WACxBrlB,GAAKkF,KAAK,SAGRo1B,GAAgBC,GAAoB,UAMtC,CAAE3R,OAAQ,QAASnP,OAAO,EAAM2P,OAJrBoa,KAAuBC,KAAkBnJ,IAIF,CAClDp1B,KAAM,SAAcw+B,eACGhrC,IAAdgrC,EACHH,GAAWjuC,KAAKsmB,GAAS3iB,OACzBsqC,GAAWjuC,KAAKsmB,GAAS3iB,MAAO0uB,EAAU+b,OC1BlD,OAAiB5J,GAAa,SAAS50B,KCDnCw1B,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAG7f,YACN6f,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAex1B,KAAQA,GAAO80B,OCJ9E3U,GAAM,kBACflwB,OAAOwuC,aAAaxuC,OAAOyuC,kBAAkB,8BCAlDte,EAAiB7B,EAA+CyD,EAIhE2c,EAAWpX,GAAI,QACfJ,EAAK,EAELsX,EAAexuC,OAAOwuC,cAAgB,kBACjC,GAGLG,EAAc,SAAU/e,GAC1BO,EAAeP,EAAI8e,EAAU,CAAE9oC,MAAO,CACpCgpC,SAAU,OAAQ1X,EAClB2X,SAAU,OAoCVC,EAAOvvC,UAAiB,CAC1BwvC,UAAU,EACVC,QAlCY,SAAUpf,EAAIkJ,OAErB14B,EAASwvB,GAAK,MAAoB,iBAANA,EAAiBA,GAAmB,iBAANA,EAAiB,IAAM,KAAOA,MACxFgC,EAAIhC,EAAI8e,GAAW,KAEjBF,EAAa5e,GAAK,MAAO,QAEzBkJ,EAAQ,MAAO,IAEpB6V,EAAY/e,UAELA,EAAG8e,GAAUE,UAwBtBK,YArBgB,SAAUrf,EAAIkJ,OACzBlH,EAAIhC,EAAI8e,GAAW,KAEjBF,EAAa5e,GAAK,OAAO,MAEzBkJ,EAAQ,OAAO,EAEpB6V,EAAY/e,UAELA,EAAG8e,GAAUG,UAatBK,SATa,SAAUtf,UACnBuf,IAAYL,EAAKC,UAAYP,EAAa5e,KAAQgC,EAAIhC,EAAI8e,IAAWC,EAAY/e,GAC9EA,IAUT0F,GAAWoZ,IAAY,8ECrDnBU,EAAS,SAAUC,EAAShzB,QACzBgzB,QAAUA,OACVhzB,OAASA,IAGF9c,UAAiB,SAAUooC,EAAUhnC,EAAImkB,EAAMwqB,EAAYC,OAEnE1Y,EAAU2Y,EAAQ9mB,EAAO5nB,EAAQub,EAAQvE,EAAM+vB,EAD/ChM,EAAgB1H,EAAKxzB,EAAImkB,EAAMwqB,EAAa,EAAI,MAGhDC,EACF1Y,EAAW8Q,MACN,IAEgB,mBADrB6H,EAAS1L,GAAkB6D,IACM,MAAM7W,UAAU,6BAE7CmX,GAAsBuH,GAAS,KAC5B9mB,EAAQ,EAAG5nB,EAASq0B,EAASwS,EAAS7mC,QAASA,EAAS4nB,EAAOA,QAClErM,EAASizB,EACLzT,EAAclJ,EAASkV,EAAOF,EAASjf,IAAQ,GAAImf,EAAK,IACxDhM,EAAc8L,EAASjf,MACbrM,aAAkB+yB,EAAQ,OAAO/yB,SACxC,IAAI+yB,GAAO,GAEtBvY,EAAW2Y,EAAOrvC,KAAKwnC,OAGzB7vB,EAAO+e,EAAS/e,OACP+vB,EAAO/vB,EAAK3X,KAAK02B,IAAWoK,SAEd,iBADrB5kB,EAAS6rB,GAA6BrR,EAAUgF,EAAegM,EAAKjiC,MAAO0pC,KAC1CjzB,GAAUA,aAAkB+yB,EAAQ,OAAO/yB,SACrE,IAAI+yB,GAAO,KAGdK,KAAO,SAAUpzB,UAChB,IAAI+yB,GAAO,EAAM/yB,WCzCT,SAAUuT,EAAI8Y,EAAanhC,QACpCqoB,aAAc8Y,SACZ5X,UAAU,cAAgBvpB,EAAOA,EAAO,IAAM,IAAM,qBACnDqoB,GCOPO,GAAiB7B,EAA+CyD,EAChEpa,GAAU+3B,GAAwC/3B,QAIlD2kB,GAAmBC,GAAoBz0B,IACvC6nC,GAAyBpT,GAAoB1B,aAEhC,SAAU+U,EAAkBC,EAASC,OAOhDpH,EANArN,GAA8C,IAArCuU,EAAiBpiC,QAAQ,OAClCuiC,GAAgD,IAAtCH,EAAiBpiC,QAAQ,QACnCwiC,EAAQ3U,EAAS,MAAQ,MACzBtI,EAAoB/C,EAAO4f,GAC3BK,EAAkBld,GAAqBA,EAAkB9yB,UACzDiwC,EAAW,MAGV7e,GAA2C,mBAArB0B,IACpBgd,GAAWE,EAAgBt4B,UAAYuY,GAAM,gBAAkB6C,GAAoB8N,UAAU/oB,WAK7F,CACL4wB,EAAcmH,GAAQ,SAAUpc,EAAQkU,GACtCrL,GAAiB6T,GAAW1c,EAAQiV,EAAakH,GAAmB,CAClE5U,KAAM4U,EACNQ,WAAY,IAAIrd,IAEFxvB,MAAZokC,GAAuB0I,GAAQ1I,EAAUlU,EAAOuc,GAAQvc,EAAQ4H,UAGlEmB,EAAmBmT,GAAuBC,GAE9Cj4B,GAAQ,CAAC,MAAO,QAAS,SAAU,UAAW,MAAO,MAAO,MAAO,OAAQ,SAAU,YAAY,SAAU0oB,OACrGiQ,EAAkB,OAAPjQ,GAAuB,OAAPA,EAC3BA,KAAO4P,KAAqBF,GAAkB,SAAP1P,IACzC/L,EAA4BoU,EAAYzoC,UAAWogC,GAAK,SAAUp/B,EAAGC,OAC/DkvC,EAAa5T,EAAiB14B,MAAMssC,eACnCE,GAAYP,IAAY3vC,EAASa,GAAI,MAAc,OAAPo/B,QAAe98B,MAC5D8Y,EAAS+zB,EAAW/P,GAAW,IAANp/B,EAAU,EAAIA,EAAGC,UACvCovC,EAAWxsC,KAAOuY,QAK/B0zB,GAAW5f,GAAeuY,EAAYzoC,UAAW,OAAQ,CACvD0wB,cAAc,EACd9iB,IAAK,kBACI2uB,EAAiB14B,MAAMssC,WAAW3M,aA5B7CiF,EAAcoH,EAAOS,eAAeV,EAASD,EAAkBvU,EAAQ2U,GACvEQ,GAAuBzB,UAAW,SAgCpCjQ,GAAe4J,EAAakH,GAAkB,GAAO,GAErDM,EAASN,GAAoBlH,EAC7BjK,EAAE,CAAEzO,QAAQ,EAAMiE,QAAQ,GAAQic,GAE7BH,GAASD,EAAOW,UAAU/H,EAAakH,EAAkBvU,GAEvDqN,MCrEQ,SAAUjV,EAAQiF,EAAKzF,OACjC,IAAIlsB,KAAO2xB,EACVzF,GAAWA,EAAQyd,QAAUjd,EAAO1sB,GAAM0sB,EAAO1sB,GAAO2xB,EAAI3xB,GAC3Dm3B,GAASzK,EAAQ1sB,EAAK2xB,EAAI3xB,GAAMksB,UAC9BQ,GCAPwH,GAAU3B,GAAgB,WCL1BnJ,GAAiB7B,EAA+CyD,EAShEid,GAAUU,GAA0CV,QAGpD1S,GAAmBC,GAAoBz0B,IACvC6nC,GAAyBpT,GAAoB1B,aAEhC,CACf0V,eAAgB,SAAUV,EAASD,EAAkBvU,EAAQ2U,OACvD7U,EAAI0U,GAAQ,SAAU/qB,EAAM6iB,GAC9BwI,GAAWrrB,EAAMqW,EAAGyU,GACpBtT,GAAiBxX,EAAM,CACrBkW,KAAM4U,EACNlnB,MAAOoQ,GAAO,MACdwK,WAAO//B,EACPotC,UAAMptC,EACNkgC,KAAM,IAEHpS,IAAavM,EAAK2e,KAAO,GACdlgC,MAAZokC,GAAuB0I,GAAQ1I,EAAU7iB,EAAKkrB,GAAQlrB,EAAMuW,MAG9DmB,EAAmBmT,GAAuBC,GAE1CgB,EAAS,SAAU9rB,EAAM/d,EAAKnB,OAG5BirC,EAAUnoB,EAFVqS,EAAQyB,EAAiB1X,GACzBgsB,EAAQC,EAASjsB,EAAM/d,UAGvB+pC,EACFA,EAAMlrC,MAAQA,GAGdm1B,EAAM4V,KAAOG,EAAQ,CACnBpoB,MAAOA,EAAQsmB,GAAQjoC,GAAK,GAC5BA,IAAKA,EACLnB,MAAOA,EACPirC,SAAUA,EAAW9V,EAAM4V,KAC3B74B,UAAMvU,EACNytC,SAAS,GAENjW,EAAMuI,QAAOvI,EAAMuI,MAAQwN,GAC5BD,IAAUA,EAAS/4B,KAAOg5B,GAC1Bzf,EAAa0J,EAAM0I,OAClB3e,EAAK2e,OAEI,MAAV/a,IAAeqS,EAAMrS,MAAMA,GAASooB,IACjChsB,GAGPisB,EAAW,SAAUjsB,EAAM/d,OAIzB+pC,EAHA/V,EAAQyB,EAAiB1X,GAEzB4D,EAAQsmB,GAAQjoC,MAEN,MAAV2hB,EAAe,OAAOqS,EAAMrS,MAAMA,OAEjCooB,EAAQ/V,EAAMuI,MAAOwN,EAAOA,EAAQA,EAAMh5B,QACzCg5B,EAAM/pC,KAAOA,EAAK,OAAO+pC,UAIjCG,GAAY9V,EAAEl7B,UAAW,CAGvBixC,MAAO,mBAEDnW,EAAQyB,EADD14B,MAEPuT,EAAO0jB,EAAMrS,MACbooB,EAAQ/V,EAAMuI,MACXwN,GACLA,EAAME,SAAU,EACZF,EAAMD,WAAUC,EAAMD,SAAWC,EAAMD,SAAS/4B,UAAOvU,UACpD8T,EAAKy5B,EAAMpoB,OAClBooB,EAAQA,EAAMh5B,KAEhBijB,EAAMuI,MAAQvI,EAAM4V,UAAOptC,EACvB8tB,EAAa0J,EAAM0I,KAAO,EAXnB3/B,KAYD2/B,KAAO,UAIT,SAAU18B,OAEdg0B,EAAQyB,EADD14B,MAEPgtC,EAAQC,EAFDjtC,KAEgBiD,MACvB+pC,EAAO,KACLh5B,EAAOg5B,EAAMh5B,KACb01B,EAAOsD,EAAMD,gBACV9V,EAAMrS,MAAMooB,EAAMpoB,OACzBooB,EAAME,SAAU,EACZxD,IAAMA,EAAK11B,KAAOA,GAClBA,IAAMA,EAAK+4B,SAAWrD,GACtBzS,EAAMuI,OAASwN,IAAO/V,EAAMuI,MAAQxrB,GACpCijB,EAAM4V,MAAQG,IAAO/V,EAAM4V,KAAOnD,GAClCnc,EAAa0J,EAAM0I,OAZd3/B,KAaC2/B,eACDqN,GAIbn5B,QAAS,SAAiBgkB,WAGpBmV,EAFA/V,EAAQyB,EAAiB14B,MACzB+3B,EAAgB1H,EAAKwH,EAAY/7B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,EAAW,GAE/EutC,EAAQA,EAAQA,EAAMh5B,KAAOijB,EAAMuI,WACxCzH,EAAciV,EAAMlrC,MAAOkrC,EAAM/pC,IAAKjD,MAE/BgtC,GAASA,EAAME,SAASF,EAAQA,EAAMD,UAKjDjf,IAAK,SAAa7qB,WACPgqC,EAASjtC,KAAMiD,MAI5BkqC,GAAY9V,EAAEl7B,UAAWo7B,EAAS,CAEhCxtB,IAAK,SAAa9G,OACZ+pC,EAAQC,EAASjtC,KAAMiD,UACpB+pC,GAASA,EAAMlrC,OAGxBkC,IAAK,SAAaf,EAAKnB,UACdgrC,EAAO9sC,KAAc,IAARiD,EAAY,EAAIA,EAAKnB,KAEzC,CAEF+Y,IAAK,SAAa/Y,UACTgrC,EAAO9sC,KAAM8B,EAAkB,IAAVA,EAAc,EAAIA,EAAOA,MAGrDyrB,GAAalB,GAAegL,EAAEl7B,UAAW,OAAQ,CACnD4N,IAAK,kBACI2uB,EAAiB14B,MAAM2/B,QAG3BtI,GAETsV,UAAW,SAAUtV,EAAGyU,EAAkBvU,OACpC8V,EAAgBvB,EAAmB,YACnCwB,EAA6BzB,GAAuBC,GACpDyB,EAA2B1B,GAAuBwB,GAGtDrQ,GAAe3F,EAAGyU,GAAkB,SAAU7O,EAAUC,GACtD1E,GAAiBx4B,KAAM,CACrBk3B,KAAMmW,EACN1d,OAAQsN,EACRhG,MAAOqW,EAA2BrQ,GAClCC,KAAMA,EACN2P,UAAMptC,OAEP,mBACGw3B,EAAQsW,EAAyBvtC,MACjCk9B,EAAOjG,EAAMiG,KACb8P,EAAQ/V,EAAM4V,KAEXG,GAASA,EAAME,SAASF,EAAQA,EAAMD,gBAExC9V,EAAMtH,SAAYsH,EAAM4V,KAAOG,EAAQA,EAAQA,EAAMh5B,KAAOijB,EAAMA,MAAMuI,OAMjE,QAARtC,EAAuB,CAAEp7B,MAAOkrC,EAAM/pC,IAAKk6B,MAAM,GACzC,UAARD,EAAyB,CAAEp7B,MAAOkrC,EAAMlrC,MAAOq7B,MAAM,GAClD,CAAEr7B,MAAO,CAACkrC,EAAM/pC,IAAK+pC,EAAMlrC,OAAQq7B,MAAM,IAN9ClG,EAAMtH,YAASlwB,EACR,CAAEqC,WAAOrC,EAAW09B,MAAM,MAMlC5F,EAAS,UAAY,UAAWA,GAAQ,GD5K9B,SAAUuU,OACrBlH,EAAczS,GAAW2Z,GACzBzf,EAAiB0C,EAAqBd,EAEtCV,GAAeqX,IAAgBA,EAAYzN,KAC7C9K,EAAeuY,EAAazN,GAAS,CACnCtK,cAAc,EACd9iB,IAAK,kBAAqB/J,QCwK5BwtC,CAAW1B,SCjLEQ,GAAW,OAAO,SAAUmB,UACpC,kBAAwBA,EAAKztC,KAAMlE,UAAUkB,OAASlB,UAAU,QAAK2D,MAC3EiuC,ICFczd,EAAK0d,SCALrB,GAAW,OAAO,SAAUmB,UACpC,kBAAwBA,EAAKztC,KAAMlE,UAAUkB,OAASlB,UAAU,QAAK2D,MAC3EiuC,ICFczd,EAAK2d,2BCClBC,EAAW,SAAU56B,OAGnB66B,EAAK5xC,OAAOC,UACZ4xC,EAASD,EAAGzwC,eAEZu7B,EAA4B,mBAAX/F,OAAwBA,OAAS,GAClDmb,EAAiBpV,EAAQ7F,UAAY,aACrCkb,EAAsBrV,EAAQsV,eAAiB,kBAC/CC,EAAoBvV,EAAQwV,aAAe,yBAEtC9d,EAAK+d,EAASC,EAASriB,EAAMsiB,OAEhCC,EAAiBF,GAAWA,EAAQnyC,qBAAqBsyC,EAAYH,EAAUG,EAC/EC,EAAYxyC,OAAO84B,OAAOwZ,EAAeryC,WACzCwyC,EAAU,IAAIC,EAAQL,GAAe,WAIzCG,EAAUG,iBAqMcR,EAASpiB,EAAM0iB,OACnC1X,EA9KuB,wBAgLpB,SAAgBhF,EAAQlvB,MA9KT,cA+KhBk0B,QACI,IAAI7zB,MAAM,mCA/KE,cAkLhB6zB,EAA6B,IAChB,UAAXhF,QACIlvB,SAKD+rC,QAGTH,EAAQ1c,OAASA,EACjB0c,EAAQ5rC,IAAMA,IAED,KACPgsC,EAAWJ,EAAQI,YACnBA,EAAU,KACRC,EAAiBC,EAAoBF,EAAUJ,MAC/CK,EAAgB,IACdA,IAAmBE,EAAkB,gBAClCF,MAIY,SAAnBL,EAAQ1c,OAGV0c,EAAQQ,KAAOR,EAAQS,MAAQT,EAAQ5rC,SAElC,GAAuB,UAAnB4rC,EAAQ1c,OAAoB,IAjNhB,mBAkNjBgF,QACFA,EAhNc,YAiNR0X,EAAQ5rC,IAGhB4rC,EAAQU,kBAAkBV,EAAQ5rC,SAEN,WAAnB4rC,EAAQ1c,QACjB0c,EAAQW,OAAO,SAAUX,EAAQ5rC,KAGnCk0B,EA3NkB,gBA6NdsY,EAASC,EAASnB,EAASpiB,EAAM0iB,MACjB,WAAhBY,EAAOrY,KAAmB,IAG5BD,EAAQ0X,EAAQxR,KAhOA,YAFK,iBAsOjBoS,EAAOxsC,MAAQmsC,iBAIZ,CACLptC,MAAOytC,EAAOxsC,IACdo6B,KAAMwR,EAAQxR,MAGS,UAAhBoS,EAAOrY,OAChBD,EA9OgB,YAiPhB0X,EAAQ1c,OAAS,QACjB0c,EAAQ5rC,IAAMwsC,EAAOxsC,OA7QP0sC,CAAiBpB,EAASpiB,EAAM0iB,GAE7CD,WAcAc,EAAS3yC,EAAIwE,EAAK0B,aAEhB,CAAEm0B,KAAM,SAAUn0B,IAAKlG,EAAGR,KAAKgF,EAAK0B,IAC3C,MAAOkjC,SACA,CAAE/O,KAAM,QAASn0B,IAAKkjC,IAhBjChzB,EAAQqd,KAAOA,MA2BX4e,EAAmB,YAMdT,cACAiB,cACAC,SAIL1U,EAAoB,GACxBA,EAAkB+S,GAAkB,kBAC3BhuC,UAGL4vC,EAAW1zC,OAAOk/B,eAClByU,EAA0BD,GAAYA,EAASA,EAASt8B,EAAO,MAC/Du8B,GACAA,IAA4B/B,GAC5BC,EAAO1xC,KAAKwzC,EAAyB7B,KAGvC/S,EAAoB4U,OAGlBC,EAAKH,EAA2BxzC,UAClCsyC,EAAUtyC,UAAYD,OAAO84B,OAAOiG,YAQ7B8U,EAAsB5zC,IAC5B,OAAQ,QAAS,UAAU0X,SAAQ,SAASoe,GAC3C91B,EAAU81B,GAAU,SAASlvB,UACpB/C,KAAK6uC,QAAQ5c,EAAQlvB,gBAoCzBitC,EAActB,EAAWuB,OAgC5BC,OAgCCrB,iBA9BY5c,EAAQlvB,YACdotC,WACA,IAAIF,GAAY,SAASG,EAASC,aAnCpCC,EAAOre,EAAQlvB,EAAKqtC,EAASC,OAChCd,EAASC,EAASd,EAAUzc,GAASyc,EAAW3rC,MAChC,UAAhBwsC,EAAOrY,KAEJ,KACD3e,EAASg3B,EAAOxsC,IAChBjB,EAAQyW,EAAOzW,aACfA,GACiB,iBAAVA,GACPisC,EAAO1xC,KAAKyF,EAAO,WACdmuC,EAAYG,QAAQtuC,EAAMyuC,SAASC,MAAK,SAAS1uC,GACtDwuC,EAAO,OAAQxuC,EAAOsuC,EAASC,MAC9B,SAASpK,GACVqK,EAAO,QAASrK,EAAKmK,EAASC,MAI3BJ,EAAYG,QAAQtuC,GAAO0uC,MAAK,SAASC,GAI9Cl4B,EAAOzW,MAAQ2uC,EACfL,EAAQ73B,MACP,SAAS4T,UAGHmkB,EAAO,QAASnkB,EAAOikB,EAASC,MAvBzCA,EAAOd,EAAOxsC,KAiCZutC,CAAOre,EAAQlvB,EAAKqtC,EAASC,aAI1BH,EAaLA,EAAkBA,EAAgBM,KAChCL,EAGAA,GACEA,cAkHDlB,EAAoBF,EAAUJ,OACjC1c,EAAS8c,EAAShc,SAAS4b,EAAQ1c,gBAtSrCxyB,IAuSEwyB,EAAsB,IAGxB0c,EAAQI,SAAW,KAEI,UAAnBJ,EAAQ1c,OAAoB,IAE1B8c,EAAShc,SAAT,SAGF4b,EAAQ1c,OAAS,SACjB0c,EAAQ5rC,SAlTZtD,EAmTIwvC,EAAoBF,EAAUJ,GAEP,UAAnBA,EAAQ1c,eAGHid,EAIXP,EAAQ1c,OAAS,QACjB0c,EAAQ5rC,IAAM,IAAIiqB,UAChB,yDAGGkiB,MAGLK,EAASC,EAASvd,EAAQ8c,EAAShc,SAAU4b,EAAQ5rC,QAErC,UAAhBwsC,EAAOrY,YACTyX,EAAQ1c,OAAS,QACjB0c,EAAQ5rC,IAAMwsC,EAAOxsC,IACrB4rC,EAAQI,SAAW,KACZG,MAGLwB,EAAOnB,EAAOxsC,WAEZ2tC,EAOFA,EAAKvT,MAGPwR,EAAQI,EAAS4B,YAAcD,EAAK5uC,MAGpC6sC,EAAQ36B,KAAO+6B,EAAS6B,QAQD,WAAnBjC,EAAQ1c,SACV0c,EAAQ1c,OAAS,OACjB0c,EAAQ5rC,SAtWVtD,GAgXFkvC,EAAQI,SAAW,KACZG,GANEwB,GA3BP/B,EAAQ1c,OAAS,QACjB0c,EAAQ5rC,IAAM,IAAIiqB,UAAU,oCAC5B2hB,EAAQI,SAAW,KACZG,YAoDF2B,EAAaC,OAChB9D,EAAQ,CAAE+D,OAAQD,EAAK,IAEvB,KAAKA,IACP9D,EAAMgE,SAAWF,EAAK,IAGpB,KAAKA,IACP9D,EAAMiE,WAAaH,EAAK,GACxB9D,EAAMkE,SAAWJ,EAAK,SAGnBK,WAAWl0C,KAAK+vC,YAGdoE,EAAcpE,OACjBuC,EAASvC,EAAMqE,YAAc,GACjC9B,EAAOrY,KAAO,gBACPqY,EAAOxsC,IACdiqC,EAAMqE,WAAa9B,WAGZX,EAAQL,QAIV4C,WAAa,CAAC,CAAEJ,OAAQ,SAC7BxC,EAAY16B,QAAQg9B,EAAc7wC,WAC7BsxC,OAAM,YA8BJh+B,EAAOuwB,MACVA,EAAU,KACR9D,EAAiB8D,EAASmK,MAC1BjO,SACKA,EAAe1jC,KAAKwnC,MAGA,mBAAlBA,EAAS7vB,YACX6vB,MAGJzkC,MAAMykC,EAAS7mC,QAAS,KACvBF,GAAK,EAAGkX,EAAO,SAASA,WACjBlX,EAAI+mC,EAAS7mC,WAChB+wC,EAAO1xC,KAAKwnC,EAAU/mC,UACxBkX,EAAKlS,MAAQ+hC,EAAS/mC,GACtBkX,EAAKmpB,MAAO,EACLnpB,SAIXA,EAAKlS,WAtdTrC,EAudIuU,EAAKmpB,MAAO,EAELnpB,UAGFA,EAAKA,KAAOA,SAKhB,CAAEA,KAAM86B,YAIRA,UACA,CAAEhtC,WAtePrC,EAseyB09B,MAAM,UA5ZnCuS,EAAkBvzC,UAAY2zC,EAAGxY,YAAcqY,EAC/CA,EAA2BrY,YAAcoY,EACzCC,EAA2BxB,GACzBuB,EAAkB6B,YAAc,oBAYlCt+B,EAAQu+B,oBAAsB,SAASC,OACjCC,EAAyB,mBAAXD,GAAyBA,EAAOna,oBAC3Coa,IACHA,IAAShC,GAG2B,uBAAnCgC,EAAKH,aAAeG,EAAKjuC,QAIhCwP,EAAQ0+B,KAAO,SAASF,UAClBv1C,OAAOs/B,eACTt/B,OAAOs/B,eAAeiW,EAAQ9B,IAE9B8B,EAAO9V,UAAYgU,EACbxB,KAAqBsD,IACzBA,EAAOtD,GAAqB,sBAGhCsD,EAAOt1C,UAAYD,OAAO84B,OAAO8a,GAC1B2B,GAOTx+B,EAAQ2+B,MAAQ,SAAS7uC,SAChB,CAAEwtC,QAASxtC,IAsEpBgtC,EAAsBC,EAAc7zC,WACpC6zC,EAAc7zC,UAAU8xC,GAAuB,kBACtCjuC,MAETiT,EAAQ+8B,cAAgBA,EAKxB/8B,EAAQ4+B,MAAQ,SAASxD,EAASC,EAASriB,EAAMsiB,EAAa0B,QACxC,IAAhBA,IAAwBA,EAAc6B,aAEtCzN,EAAO,IAAI2L,EACb1f,EAAK+d,EAASC,EAASriB,EAAMsiB,GAC7B0B,UAGKh9B,EAAQu+B,oBAAoBlD,GAC/BjK,EACAA,EAAKrwB,OAAOw8B,MAAK,SAASj4B,UACjBA,EAAO4kB,KAAO5kB,EAAOzW,MAAQuiC,EAAKrwB,WAuKjD+7B,EAAsBD,GAEtBA,EAAG3B,GAAqB,YAOxB2B,EAAG9B,GAAkB,kBACZhuC,MAGT8vC,EAAG1zC,SAAW,iBACL,sBAkCT6W,EAAQ3P,KAAO,SAASwrB,OAClBxrB,EAAO,OACN,IAAIL,KAAO6rB,EACdxrB,EAAKrG,KAAKgG,UAEZK,EAAKyuC,UAIE,SAAS/9B,SACP1Q,EAAKtG,QAAQ,KACdiG,EAAMK,EAAK0uC,SACX/uC,KAAO6rB,SACT9a,EAAKlS,MAAQmB,EACb+Q,EAAKmpB,MAAO,EACLnpB,SAOXA,EAAKmpB,MAAO,EACLnpB,IAsCXf,EAAQK,OAASA,EAMjBs7B,EAAQzyC,UAAY,CAClBm7B,YAAasX,EAEb0C,MAAO,SAASW,WACTvI,KAAO,OACP11B,KAAO,OAGPm7B,KAAOnvC,KAAKovC,WAjfjB3vC,OAkfK09B,MAAO,OACP4R,SAAW,UAEX9c,OAAS,YACTlvB,SAtfLtD,OAwfK0xC,WAAWt9B,QAAQu9B,IAEnBa,MACE,IAAIxuC,KAAQzD,KAEQ,MAAnByD,EAAK+iB,OAAO,IACZunB,EAAO1xC,KAAK2D,KAAMyD,KACjBrE,OAAOqE,EAAKP,MAAM,WAChBO,QAhgBXhE,IAsgBFksC,KAAM,gBACCxO,MAAO,MAGR+U,EADYlyC,KAAKmxC,WAAW,GACLE,cACH,UAApBa,EAAWhb,WACPgb,EAAWnvC,WAGZ/C,KAAKmyC,MAGd9C,kBAAmB,SAAS+C,MACtBpyC,KAAKm9B,WACDiV,MAGJzD,EAAU3uC,cACLqyC,EAAOC,EAAKC,UACnBhD,EAAOrY,KAAO,QACdqY,EAAOxsC,IAAMqvC,EACbzD,EAAQ36B,KAAOs+B,EAEXC,IAGF5D,EAAQ1c,OAAS,OACjB0c,EAAQ5rC,SAjiBZtD,KAoiBY8yC,MAGP,IAAIz1C,EAAIkD,KAAKmxC,WAAWn0C,OAAS,EAAGF,GAAK,IAAKA,EAAG,KAChDkwC,EAAQhtC,KAAKmxC,WAAWr0C,GACxByyC,EAASvC,EAAMqE,cAEE,SAAjBrE,EAAM+D,cAIDsB,EAAO,UAGZrF,EAAM+D,QAAU/wC,KAAK0pC,KAAM,KACzB8I,EAAWzE,EAAO1xC,KAAK2wC,EAAO,YAC9ByF,EAAa1E,EAAO1xC,KAAK2wC,EAAO,iBAEhCwF,GAAYC,EAAY,IACtBzyC,KAAK0pC,KAAOsD,EAAMgE,gBACbqB,EAAOrF,EAAMgE,UAAU,GACzB,GAAIhxC,KAAK0pC,KAAOsD,EAAMiE,kBACpBoB,EAAOrF,EAAMiE,iBAGjB,GAAIuB,MACLxyC,KAAK0pC,KAAOsD,EAAMgE,gBACbqB,EAAOrF,EAAMgE,UAAU,OAG3B,CAAA,IAAIyB,QAMH,IAAIrvC,MAAM,6CALZpD,KAAK0pC,KAAOsD,EAAMiE,kBACboB,EAAOrF,EAAMiE,gBAU9B3B,OAAQ,SAASpY,EAAMn0B,OAChB,IAAIjG,EAAIkD,KAAKmxC,WAAWn0C,OAAS,EAAGF,GAAK,IAAKA,EAAG,KAChDkwC,EAAQhtC,KAAKmxC,WAAWr0C,MACxBkwC,EAAM+D,QAAU/wC,KAAK0pC,MACrBqE,EAAO1xC,KAAK2wC,EAAO,eACnBhtC,KAAK0pC,KAAOsD,EAAMiE,WAAY,KAC5ByB,EAAe1F,SAKnB0F,IACU,UAATxb,GACS,aAATA,IACDwb,EAAa3B,QAAUhuC,GACvBA,GAAO2vC,EAAazB,aAGtByB,EAAe,UAGbnD,EAASmD,EAAeA,EAAarB,WAAa,UACtD9B,EAAOrY,KAAOA,EACdqY,EAAOxsC,IAAMA,EAET2vC,QACGzgB,OAAS,YACTje,KAAO0+B,EAAazB,WAClB/B,GAGFlvC,KAAK2yC,SAASpD,IAGvBoD,SAAU,SAASpD,EAAQ2B,MACL,UAAhB3B,EAAOrY,WACHqY,EAAOxsC,UAGK,UAAhBwsC,EAAOrY,MACS,aAAhBqY,EAAOrY,UACJljB,KAAOu7B,EAAOxsC,IACM,WAAhBwsC,EAAOrY,WACXib,KAAOnyC,KAAK+C,IAAMwsC,EAAOxsC,SACzBkvB,OAAS,cACTje,KAAO,OACa,WAAhBu7B,EAAOrY,MAAqBga,SAChCl9B,KAAOk9B,GAGPhC,GAGT0D,OAAQ,SAAS3B,OACV,IAAIn0C,EAAIkD,KAAKmxC,WAAWn0C,OAAS,EAAGF,GAAK,IAAKA,EAAG,KAChDkwC,EAAQhtC,KAAKmxC,WAAWr0C,MACxBkwC,EAAMiE,aAAeA,cAClB0B,SAAS3F,EAAMqE,WAAYrE,EAAMkE,UACtCE,EAAcpE,GACPkC,UAKJ,SAAS6B,OACX,IAAIj0C,EAAIkD,KAAKmxC,WAAWn0C,OAAS,EAAGF,GAAK,IAAKA,EAAG,KAChDkwC,EAAQhtC,KAAKmxC,WAAWr0C,MACxBkwC,EAAM+D,SAAWA,EAAQ,KACvBxB,EAASvC,EAAMqE,cACC,UAAhB9B,EAAOrY,KAAkB,KACvB2b,EAAStD,EAAOxsC,IACpBquC,EAAcpE,UAET6F,SAML,IAAIzvC,MAAM,0BAGlB0vC,cAAe,SAASjP,EAAU8M,EAAYC,eACvC7B,SAAW,CACdhc,SAAUzf,EAAOuwB,GACjB8M,WAAYA,EACZC,QAASA,GAGS,SAAhB5wC,KAAKiyB,cAGFlvB,SA1qBPtD,GA6qBOyvC,IAQJj8B,EA1rBM,CAisBgBxX,EAAOwX,aAIpC8/B,mBAAqBlF,EACrB,MAAOmF,GAUPrvC,SAAS,IAAK,yBAAdA,CAAwCkqC,UCrtBzBjd,GCSbuR,GAAsBC,GAA6B,UACnDb,GAAiBC,GAAwB,SAAU,CAAEN,WAAW,IAAS,IAAM,IAE/E57B,GAAM9D,KAAK8D,IACXlD,GAAMZ,KAAKY,MAOb,CAAEutB,OAAQ,QAASnP,OAAO,EAAM2P,QAASgS,KAAwBZ,IAAkB,CACnF0R,OAAQ,SAAgBzS,EAAO0S,OAKzBC,EAAaC,EAAmBnQ,EAAGtoB,EAAGta,EAAMD,EAJ5CqtB,EAAI9K,GAAS3iB,MACbC,EAAMoxB,EAAS5D,EAAEzwB,QACjBq2C,EAAc/hB,GAAgBkP,EAAOvgC,GACrC+jC,EAAkBloC,UAAUkB,UAER,IAApBgnC,EACFmP,EAAcC,EAAoB,EACL,IAApBpP,GACTmP,EAAc,EACdC,EAAoBnzC,EAAMozC,IAE1BF,EAAcnP,EAAkB,EAChCoP,EAAoBhxC,GAAIkD,GAAIwrB,EAAUoiB,GAAc,GAAIjzC,EAAMozC,IAE5DpzC,EAAMkzC,EAAcC,EAtBL,uBAuBXpmB,UAtB0B,uCAwBlCiW,EAAIjL,GAAmBvK,EAAG2lB,GACrBz4B,EAAI,EAAGA,EAAIy4B,EAAmBz4B,KACjCta,EAAOgzC,EAAc14B,KACT8S,GAAGiF,GAAeuQ,EAAGtoB,EAAG8S,EAAEptB,OAExC4iC,EAAEjmC,OAASo2C,EACPD,EAAcC,EAAmB,KAC9Bz4B,EAAI04B,EAAa14B,EAAI1a,EAAMmzC,EAAmBz4B,IAEjDva,EAAKua,EAAIw4B,GADT9yC,EAAOsa,EAAIy4B,KAEC3lB,EAAGA,EAAErtB,GAAMqtB,EAAEptB,UACbotB,EAAErtB,OAEXua,EAAI1a,EAAK0a,EAAI1a,EAAMmzC,EAAoBD,EAAax4B,WAAY8S,EAAE9S,EAAI,QACtE,GAAIw4B,EAAcC,MAClBz4B,EAAI1a,EAAMmzC,EAAmBz4B,EAAI04B,EAAa14B,IAEjDva,EAAKua,EAAIw4B,EAAc,GADvB9yC,EAAOsa,EAAIy4B,EAAoB,KAEnB3lB,EAAGA,EAAErtB,GAAMqtB,EAAEptB,UACbotB,EAAErtB,OAGbua,EAAI,EAAGA,EAAIw4B,EAAax4B,IAC3B8S,EAAE9S,EAAI04B,GAAev3C,UAAU6e,EAAI,UAErC8S,EAAEzwB,OAASiD,EAAMmzC,EAAoBD,EAC9BlQ,KChEX,OAAiBpC,GAAa,SAASoS,OCDnCxR,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGmnB,cACNnnB,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAewR,OAAUA,GAASlS,GCFhG79B,GAAQ,GAAGA,MACXowC,GAAO,WAAWvsC,KAAK+6B,IAEvBxR,GAAO,SAAUijB,UACZ,SAAUC,EAASC,OACpBC,EAAY53C,UAAUkB,OAAS,EAC/BgG,EAAO0wC,EAAYxwC,GAAM7G,KAAKP,UAAW,QAAK2D,SAC3C8zC,EAAUG,EAAY,YAER,mBAAXF,EAAwBA,EAAU7vC,SAAS6vC,IAAU33C,MAAMmE,KAAMgD,IACvEwwC,EAASC,OAMf,CAAEvnB,QAAQ,EAAMmE,MAAM,EAAMF,OAAQmjB,IAAQ,CAG5CK,WAAYrjB,GAAKpE,EAAOynB,YAGxBC,YAAatjB,GAAKpE,EAAO0nB,eCvB3B,OAAiB3jB,EAAK0jB,WCIlBE,GAAmB,SAAUlkB,EAAQmkB,EAAUjuB,EAAQkuB,EAAWvT,EAAOwT,EAAOC,EAAQC,WAItFtM,EAHAuM,EAAc3T,EACd4T,EAAc,EACdC,IAAQJ,GAAS5jB,EAAK4jB,EAAQC,EAAS,GAGpCE,EAAcL,GAAW,IAC1BK,KAAevuB,EAAQ,IACzB+hB,EAAUyM,EAAQA,EAAMxuB,EAAOuuB,GAAcA,EAAaN,GAAYjuB,EAAOuuB,GAEzEJ,EAAQ,GAAKj4C,GAAQ6rC,GACvBuM,EAAcN,GAAiBlkB,EAAQmkB,EAAUlM,EAASvW,EAASuW,EAAQ5qC,QAASm3C,EAAaH,EAAQ,GAAK,MACzG,IACDG,GAAe,iBAAkB,MAAMnnB,UAAU,sCACrD2C,EAAOwkB,GAAevM,EAGxBuM,IAEFC,WAEKD,MAGQN,KCrBf,CAAElkB,OAAQ,QAASnP,OAAO,GAAQ,CAClC8zB,QAAS,SAAiBzc,OAGpBoL,EAFAxV,EAAI9K,GAAS3iB,MACb+zC,EAAY1iB,EAAS5D,EAAEzwB,eAE3B0xB,EAAUmJ,IACVoL,EAAIjL,GAAmBvK,EAAG,IACxBzwB,OAAS62C,GAAiB5Q,EAAGxV,EAAGA,EAAGsmB,EAAW,EAAG,EAAGlc,EAAY/7B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,GACjGwjC,KCdX,OAAiBpC,GAAa,SAASyT,QCFnC7S,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGwoB,eACNxoB,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAe6S,QAAWA,GAAUvT,GCAlG/P,GAAe,SAAUujB,UACpB,SAAUvzB,EAAM6W,EAAYmM,EAAiBwQ,GAClD9lB,EAAUmJ,OACNpK,EAAI9K,GAAS3B,GACbiL,EAAOgB,EAAcQ,GACrBzwB,EAASq0B,EAAS5D,EAAEzwB,QACpB4nB,EAAQ2vB,EAAWv3C,EAAS,EAAI,EAChCF,EAAIy3C,GAAY,EAAI,KACpBvQ,EAAkB,EAAG,OAAa,IAChCpf,KAASqH,EAAM,CACjBuoB,EAAOvoB,EAAKrH,GACZA,GAAS9nB,WAGX8nB,GAAS9nB,EACLy3C,EAAW3vB,EAAQ,EAAI5nB,GAAU4nB,QAC7BoI,UAAU,oDAGdunB,EAAW3vB,GAAS,EAAI5nB,EAAS4nB,EAAOA,GAAS9nB,EAAO8nB,KAASqH,IACrEuoB,EAAO3c,EAAW2c,EAAMvoB,EAAKrH,GAAQA,EAAO6I,WAEvC+mB,IC1BPC,GD8Ba,CAGfC,KAAM1jB,IAAa,GAGnB2jB,MAAO3jB,IAAa,ICpC6B0jB,KAI/CrT,GAAgBC,GAAoB,UACpCC,GAAiBC,GAAwB,SAAU,GAAK,MAI1D,CAAE7R,OAAQ,QAASnP,OAAO,EAAM2P,QAASkR,KAAkBE,IAAkB,CAC7EqT,OAAQ,SAAgB/c,UACf4c,GAAQz0C,KAAM63B,EAAY/7B,UAAUkB,OAAQlB,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCV7F,OAAiBohC,GAAa,SAAS+T,OCDnCnT,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAG8oB,cACN9oB,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAemT,OAAUA,GAAS7T,GCFhG79B,GAAQ,GAAGA,MACX2xC,GAAY,GAEZC,GAAY,SAAUzd,EAAG0d,EAAY/xC,QACjC+xC,KAAcF,IAAY,KACzB,IAAIG,EAAO,GAAIl4C,EAAI,EAAGA,EAAIi4C,EAAYj4C,IAAKk4C,EAAKl4C,GAAK,KAAOA,EAAI,IAErE+3C,GAAUE,GAAcpxC,SAAS,MAAO,gBAAkBqxC,EAAK7xC,KAAK,KAAO,YACpE0xC,GAAUE,GAAY1d,EAAGr0B,OAKnBW,SAAS0sB,MAAQ,SAAcrP,OAC1CnkB,EAAK6xB,EAAU1uB,MACfi1C,EAAW/xC,GAAM7G,KAAKP,UAAW,GACjCi8B,EAAgB,eACd/0B,EAAOiyC,EAAS7mC,OAAOlL,GAAM7G,KAAKP,mBAC/BkE,gBAAgB+3B,EAAgB+c,GAAUj4C,EAAImG,EAAKhG,OAAQgG,GAAQnG,EAAGhB,MAAMmlB,EAAMhe,WAEvF1G,EAASO,EAAGV,aAAY47B,EAAc57B,UAAYU,EAAGV,WAClD47B,KCpBP,CAAEpI,OAAQ,WAAYnP,OAAO,GAAQ,CACrC6P,KAAMA,KCHR,OAAiBwQ,GAAa,YAAYxQ,KCDtC6kB,GAAoBvxC,SAASxH,aAEhB,SAAU2vB,OACrBiV,EAAMjV,EAAGuE,YACNvE,IAAOopB,IAAsBppB,aAAcnoB,UAAYo9B,IAAQmU,GAAkB7kB,KAAQA,GAAO0Q,GCAzG,OANA,SAAyBoU,EAAUvQ,QAC3BuQ,aAAoBvQ,SAClB,IAAI5X,UAAU,sCCAxB,SAASooB,GAAkBzlB,EAAQ0lB,OAC5B,IAAIv4C,EAAI,EAAGA,EAAIu4C,EAAMr4C,OAAQF,IAAK,KACjC4vB,EAAa2oB,EAAMv4C,GACvB4vB,EAAWC,WAAaD,EAAWC,aAAc,EACjDD,EAAWG,cAAe,EACtB,UAAWH,IAAYA,EAAWI,UAAW,GAEjDmT,GAAuBtQ,EAAQjD,EAAWzpB,IAAKypB,IAUnD,OANA,SAAsBkY,EAAa0Q,EAAYC,UACzCD,GAAYF,GAAkBxQ,EAAYzoC,UAAWm5C,GACrDC,GAAaH,GAAkBxQ,EAAa2Q,GACzC3Q,qJCwDO4Q,GAGdn1C,UACO,IAAIo1C,GAA0Bp1C,OAcjCq1C,yBAwBeC,EACAC,EACAC,6BAFAF,QAAAA,OACAC,cAAAA,OACAC,QAAAA,OAjBFC,WAAqC,CACpDj7B,IAAKk7B,UAAKC,aAAUh2C,MACpBi2C,OAAQF,UAAKG,gBAAal2C,MAC1Bm2C,OAAQJ,UAAKK,gBAAap2C,4DAmBrB61C,QAAQM,OAAOn2C,KAAKq2C,gBAAgBr2C,KAAK21C,QAAQ5rC,QAC/C/J,iDAKF21C,QAAQW,GAAG,MAAOt2C,KAAK81C,WAAWj7B,UAClC86B,QAAQW,GAAG,SAAUt2C,KAAK81C,WAAWG,aACrCN,QAAQW,GAAG,SAAUt2C,KAAK81C,WAAWK,QAEnCn2C,gDAKF21C,QAAQY,IAAI,MAAOv2C,KAAK81C,WAAWj7B,UACnC86B,QAAQY,IAAI,SAAUv2C,KAAK81C,WAAWG,aACtCN,QAAQY,IAAI,SAAUv2C,KAAK81C,WAAWK,QAEpCn2C,6CAUew2C,gBACfC,UAAKb,uBAAqB,SAACY,EAAOE,UAChCA,EAAUF,KAChBA,gCAUHG,EACAC,GAEe,MAAXA,QAICf,QAAQh7B,IAAI7a,KAAKq2C,gBAAgBr2C,KAAK21C,QAAQ5rC,IAAI6sC,EAAQJ,yCAU/DG,EACAC,GAEe,MAAXA,QAICf,QAAQM,OAAOn2C,KAAKq2C,gBAAgBr2C,KAAK21C,QAAQ5rC,IAAI6sC,EAAQJ,yCAUlEG,EACAC,GAEe,MAAXA,QAICf,QAAQI,OAAOj2C,KAAKq2C,gBAAgBO,EAAQC,mBAW/CpB,yBAgBgCE,mBAAAA,QAAAA,OARnBC,cAAoC,6CAmBnD5vC,eAEK4vC,cAAc34C,MAAK,SAACjB,UAAqB86C,GAAA96C,QAAAA,EAAagK,MACpDhG,iCAePgG,eAEK4vC,cAAc34C,MAAK,SAACjB,UAAqB+6C,GAAA/6C,QAAAA,EAAUgK,MAChDhG,qCAeRgG,eAEK4vC,cAAc34C,MAAK,SAACjB,UAAqBg7C,GAAAh7C,QAAAA,EAAcgK,MACpDhG,gCAWA2vB,UACD,IAAI+lB,GAAe11C,KAAK21C,QAAS31C,KAAK41C,cAAejmB,8vsBCjS5B,oBAAX3D,QAA2BA,OAAM,QAAcxB,2ICQxE,IAAMysB,GAAe,qBAWd,SAASC,GAAQpoB,EAAQoI,OAC1B5wB,UAEW7G,IAAXqvB,MAGW,OAAXA,SACK,SAGJoI,SACIpI,KAEa,iBAAToI,KAAwBA,aAAgBzI,cAC7C,IAAIrrB,MAAM,gCAIV8zB,OACD,cACA,iBACI+K,QAAQnT,OAEZ,aACA,gBACCqoB,GAASroB,KAAY1vB,MAAM1C,KAAK06C,MAAMtoB,IACjC1E,EAAO0E,GAAQvxB,UAKf85C,OAAOvoB,EAAOvxB,eAEpB,aACA,gBACIkxB,OAAOK,OAEX,UACCtyB,GAASsyB,UACJ,IAAIpyB,KAAKoyB,MAEdA,aAAkBpyB,YACb,IAAIA,KAAKoyB,EAAOvxB,WAClB,GAAI6sB,EAAOhpB,SAAS0tB,UAClB,IAAIpyB,KAAKoyB,EAAOvxB,cAErB45C,GAASroB,UACXxoB,EAAQ2wC,GAAa//B,KAAK4X,IAGjB,IAAIpyB,KAAK26C,OAAO/wC,EAAM,KAEtB8jB,EAAO,IAAI1tB,KAAKoyB,IAASlM,eAG5B,IAAIxf,MACR,iCAAmCk0C,GAAQxoB,GAAU,qBAItD,YACCtyB,GAASsyB,UACJ1E,EAAO0E,MAEZA,aAAkBpyB,YACb0tB,EAAO0E,EAAOvxB,WAChB,GAAI6sB,EAAOhpB,SAAS0tB,UAClB1E,EAAO0E,MAEZqoB,GAASroB,UACXxoB,EAAQ2wC,GAAa//B,KAAK4X,GAGjB1E,EAFL9jB,EAEY+wC,OAAO/wC,EAAM,IAEbwoB,SAGV,IAAI1rB,MACR,iCAAmCk0C,GAAQxoB,GAAU,qBAItD,aACCtyB,GAASsyB,UACJ,IAAIpyB,KAAKoyB,GACX,GAAIA,aAAkBpyB,YACpBoyB,EAAOjM,cACT,GAAIuH,EAAOhpB,SAAS0tB,UAClBA,EAAOlM,SAASC,cAClB,GAAIs0B,GAASroB,UAClBxoB,EAAQ2wC,GAAa//B,KAAK4X,IAGjB,IAAIpyB,KAAK26C,OAAO/wC,EAAM,KAAKuc,cAE3BuH,EAAO0E,GAAQrxB,eAGlB,IAAI2F,MACR,iCACEk0C,GAAQxoB,GACR,wBAIH,aACCtyB,GAASsyB,SACJ,SAAWA,EAAS,KACtB,GAAIA,aAAkBpyB,MAAQ0tB,EAAOhpB,SAAS0tB,SAC5C,SAAWA,EAAOvxB,UAAY,KAChC,GAAI45C,GAASroB,SASX,WARPxoB,EAAQ2wC,GAAa//B,KAAK4X,IAIhB,IAAIpyB,KAAK26C,OAAO/wC,EAAM,KAAK/I,UAE3B,IAAIb,KAAKoyB,GAAQvxB,WAED,WAEpB,IAAI6F,MACR,iCACEk0C,GAAQxoB,GACR,kCAKA,IAAI1rB,6BAAsB8zB,MA8B/B,SAASqgB,GACdC,mBACAtgB,yDAAO,CAAEsJ,MAAO,OAAQC,IAAK,QAEvBgX,EAASD,EAAME,QACfC,EAAY,IAAIC,GAAQ,CAAEC,QAASJ,IAEnCK,EAAOf,KAAAvB,GAAsBgC,YAC5B,SAAAO,gBACHtB,QAAYsB,YAAa,SAACC,EAAK/0C,UAC7B+0C,EAAI/0C,GAAOi0C,GAAQa,EAAK90C,GAAMi0B,EAAKj0B,IAC5B+0C,IACN,OAEJ53C,GAAGu3C,UAENG,EAAKG,MAAMzX,QAEJ,CAEL3lB,IAAK,0BAAa28B,EAAMU,cAAar9B,wBACrCo7B,OAAQ,0BAAauB,EAAMU,cAAajC,2BACxCE,OAAQ,0BAAaqB,EAAMU,cAAa/B,2BACxCgC,WAAY,0BAAaX,EAAMU,cAAaC,+BAG5CtkC,QAASkiC,QAAA4B,WAAuBA,GAChC5tC,IAAKgsC,KAAA4B,EAAU5tC,YAAS4tC,GACxBS,OAAQrC,KAAA4B,EAAUS,eAAYT,GAC9BpB,IAAKR,KAAA4B,EAAUpB,YAASoB,GACxBrB,GAAIP,KAAA4B,EAAUrB,WAAQqB,uBAGbA,EAAU36C,QAInBy6C,OAAAA,EACAvgB,KAAAA,EAEAsgB,MAAAA,EACAG,UAAAA,EACAU,QAAS,kBAAMP,EAAKnM,uRAKnB2M,IACHpB,QAAAA,KCtOExW,GAAOlW,GAAoCkW,KAG3C6X,GAAcrsB,EAAOpQ,cACZ,EAAIy8B,GAAYlY,GAAc,QAAWmY,EAAAA,EAI5B,SAAoBvhC,OACxCwhC,EAAgB/X,GAAKjS,OAAOxX,IAC5BsB,EAASggC,GAAYE,UACP,IAAXlgC,GAA2C,KAA3BkgC,EAAcjyB,OAAO,IAAa,EAAIjO,GAC3DggC,KCRF,CAAErsB,QAAQ,EAAMiE,OAAQrU,YAAc48B,IAA4B,CAClE58B,WAAY48B,KCHd,OAAiBzoB,EAAKnU,WCAD68B,yBAKPC,EAAMzpB,mBACXA,QAAU,UACVkmB,MAAQ,mDAQJlmB,GACLA,GACFmpB,GAAKh7C,OAAO0C,KAAKmvB,QAASA,2CAUrB,8EAiBD0pB,EACJ74C,KAAKq1C,MAAMyD,iBAAmB94C,KAAKq1C,MAAMzM,OACzC5oC,KAAKq1C,MAAM0D,kBAAoB/4C,KAAKq1C,MAAMxM,mBAGvCwM,MAAMyD,eAAiB94C,KAAKq1C,MAAMzM,WAClCyM,MAAM0D,gBAAkB/4C,KAAKq1C,MAAMxM,OAEjCgQ,aClDT,CAAElpB,OAAQ,SAAUnP,OAAO,GAAQ,CACnCw4B,OCAe,GAAGA,QAAU,SAAgBC,OACxCptB,EAAM4C,OAAOvB,EAAuBltB,OACpCuY,EAAS,GACTpK,EAAI2iB,EAAUmoB,MACd9qC,EAAI,GAAKA,GAAKqqC,EAAAA,EAAU,MAAMU,WAAW,oCACvC/qC,EAAI,GAAIA,KAAO,KAAO0d,GAAOA,GAAc,EAAJ1d,IAAOoK,GAAUsT,UACvDtT,KCTT,OAAiBsoB,GAAa,UAAUmY,OCDpClY,GAAkBrS,OAAOtyB,aAEZ,SAAU2vB,OACrBiV,EAAMjV,EAAGktB,aACQ,iBAAPltB,GAAmBA,IAAOgV,IAClChV,aAAc2C,QAAUsS,IAAQD,GAAgBkY,OAAUA,GAASjY,GCEpE,SAASoY,GAAqB/uB,EAAQwuB,EAAMQ,MAC7CA,IAAgBlW,GAAckW,UACzBD,GAAqB/uB,EAAQwuB,EAAM,CAACQ,OAG7CR,EAAKQ,YAAc,GACfA,GACgC,GAA9BlW,GAAckW,GAAsB,WAC7Bt8C,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,YACR2C,IAA1B45C,GAAAD,EAAYt8C,IAAyB,KACjCw8C,EAAW,GACjBA,EAAS9Y,MAAQpW,EAAOgvB,EAAYt8C,GAAG0jC,OAAO5d,SAASrlB,UACvD+7C,EAAS7Y,IAAMrW,EAAOgvB,EAAYt8C,GAAG2jC,KAAK7d,SAASrlB,UACnDq7C,EAAKQ,YAAYn8C,KAAKq8C,QAG1BV,EAAKQ,qBAAiB,SAACj8C,EAAGC,UAAMD,EAAEqjC,MAAQpjC,EAAEojC,UAa3C,SAAS+Y,GAAkBnvB,EAAQwuB,EAAMQ,MAC1CA,IAAgBlW,GAAckW,UACzBG,GAAkBnvB,EAAQwuB,EAAM,CAACQ,OAGtCA,QAAuD35C,IAAxCm5C,EAAKY,SAASC,gBAAgB7Q,MAAqB,CACpEuQ,GAAqB/uB,EAAQwuB,EAAMQ,WAE7B5Y,EAAQpW,EAAOwuB,EAAKc,MAAMlZ,OAC1BC,EAAMrW,EAAOwuB,EAAKc,MAAMjZ,KAGxBkZ,GADcf,EAAKc,MAAMjZ,IAAMmY,EAAKc,MAAMlZ,OACjBoY,EAAKY,SAASC,gBAAgB7Q,MAEpD9rC,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,YACR2C,IAA1B45C,GAAAD,EAAYt8C,IAAyB,KACjC88C,EAAYxvB,EAAOgvB,EAAYt8C,GAAG0jC,OACpCqZ,EAAUzvB,EAAOgvB,EAAYt8C,GAAG2jC,QAEhB,gBAAhBmZ,EAAUv6C,SACN,IAAI+D,kDAA2Cg2C,EAAYt8C,GAAG0jC,WAEpD,gBAAdqZ,EAAQx6C,SACJ,IAAI+D,gDAAyCg2C,EAAYt8C,GAAG2jC,SAGnDoZ,EAAUD,GACX,EAAID,EAAW,KAEzBn9B,EAAS,EACPs9B,EAAWrZ,EAAIrjB,kBACbg8B,EAAYt8C,SACb,QACC88C,EAAUxqC,OAASyqC,EAAQzqC,QAC7BoN,EAAS,GAEXo9B,EAAUjsC,UAAU6yB,EAAM7yB,aAC1BisC,EAAUvwC,KAAKm3B,EAAMn3B,QACrBuwC,EAAUh7B,SAAS,EAAE,QAErBi7B,EAAQlsC,UAAU6yB,EAAM7yB,aACxBksC,EAAQxwC,KAAKm3B,EAAMn3B,QACnBwwC,EAAQj7B,SAAS,EAAIpC,EAAO,QAE5Bs9B,EAASj/B,IAAI,EAAG,mBAEb,aACGk/B,EAAYF,EAAQ18B,KAAKy8B,EAAU,QACnCxqC,EAAMwqC,EAAUxqC,MAGtBwqC,EAAU3vC,KAAKu2B,EAAMv2B,QACrB2vC,EAAU5vC,MAAMw2B,EAAMx2B,SACtB4vC,EAAUvwC,KAAKm3B,EAAMn3B,QACrBwwC,EAAUD,EAAUx8B,QAGpBw8B,EAAUxqC,IAAIA,GACdyqC,EAAQzqC,IAAIA,GACZyqC,EAAQh/B,IAAIk/B,EAAU,QAEtBH,EAAUh7B,SAAS,EAAE,SACrBi7B,EAAQj7B,SAAS,EAAE,SAEnBk7B,EAASj/B,IAAI,EAAG,mBAGb,UACC++B,EAAU5vC,SAAW6vC,EAAQ7vC,UAC/BwS,EAAS,GAEXo9B,EAAU5vC,MAAMw2B,EAAMx2B,SACtB4vC,EAAUvwC,KAAKm3B,EAAMn3B,QACrBuwC,EAAUh7B,SAAS,EAAE,UAErBi7B,EAAQ7vC,MAAMw2B,EAAMx2B,SACpB6vC,EAAQxwC,KAAKm3B,EAAMn3B,QACnBwwC,EAAQj7B,SAAS,EAAE,UACnBi7B,EAAQh/B,IAAI2B,EAAO,UAEnBs9B,EAASj/B,IAAI,EAAG,oBAEb,SACC++B,EAAUvwC,QAAUwwC,EAAQxwC,SAC9BmT,EAAS,GAEXo9B,EAAUvwC,KAAKm3B,EAAMn3B,QACrBuwC,EAAUh7B,SAAS,EAAE,SACrBi7B,EAAQxwC,KAAKm3B,EAAMn3B,QACnBwwC,EAAQj7B,SAAS,EAAE,SACnBi7B,EAAQh/B,IAAI2B,EAAO,SAEnBs9B,EAASj/B,IAAI,EAAG,mCAGhBlY,QAAQq3C,IAAI,8EAA4EZ,EAAYt8C,UAGjG88C,EAAYE,UACjBlB,EAAKQ,YAAYn8C,KAAK,CAACujC,MAAOoZ,EAAUr8C,UAAWkjC,IAAKoZ,EAAQt8C,eACxD67C,EAAYt8C,SACb,QACH88C,EAAU/+B,IAAI,EAAG,QACjBg/B,EAAQh/B,IAAI,EAAG,kBAEZ,SACH++B,EAAU/+B,IAAI,EAAG,SACjBg/B,EAAQh/B,IAAI,EAAG,mBAEZ,UACH++B,EAAU/+B,IAAI,EAAG,UACjBg/B,EAAQh/B,IAAI,EAAG,oBAEZ,SACH++B,EAAU/+B,IAAI,EAAG,KACjBg/B,EAAQh/B,IAAI,EAAG,+BAGflY,QAAQq3C,IAAI,8EAA4EZ,EAAYt8C,KAI1G87C,EAAKQ,YAAYn8C,KAAK,CAACujC,MAAOoZ,EAAUr8C,UAAWkjC,IAAKoZ,EAAQt8C,aAKtE08C,GAAiBrB,OAEXsB,EAAcC,GAAYvB,EAAKc,MAAMlZ,MAAOoY,EAAKQ,aACjDgB,EAAYD,GAAYvB,EAAKc,MAAMjZ,IAAImY,EAAKQ,aAC9CiB,EAAazB,EAAKc,MAAMlZ,MACxB8Z,EAAW1B,EAAKc,MAAMjZ,IACA,GAAtByZ,EAAYK,SAAiBF,EAAwC,GAA3BzB,EAAKc,MAAMc,aAAuBN,EAAYN,UAAY,EAAIM,EAAYL,QAAU,GAC1G,GAApBO,EAAUG,SAAmBD,EAAsC,GAAzB1B,EAAKc,MAAMe,WAAuBL,EAAUR,UAAY,EAAMQ,EAAUP,QAAU,GACtG,GAAtBK,EAAYK,QAAsC,GAApBH,EAAUG,QAC1C3B,EAAKc,MAAMgB,YAAYL,EAAYC,IAYlC,SAASL,GAAiBrB,aACzBQ,EAAcR,EAAKQ,YACnBuB,EAAY,GACT79C,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,QACjC,IAAIiX,EAAI,EAAGA,EAAIqlC,EAAYp8C,OAAQ+W,IAClCjX,GAAKiX,GAA8B,GAAzBqlC,EAAYrlC,GAAGkiC,QAA2C,GAAzBmD,EAAYt8C,GAAGm5C,SAExDmD,EAAYrlC,GAAGysB,OAAS4Y,EAAYt8C,GAAG0jC,OAAS4Y,EAAYrlC,GAAG0sB,KAAO2Y,EAAYt8C,GAAG2jC,IACvF2Y,EAAYrlC,GAAGkiC,QAAS,EAGjBmD,EAAYrlC,GAAGysB,OAAS4Y,EAAYt8C,GAAG0jC,OAAS4Y,EAAYrlC,GAAGysB,OAAS4Y,EAAYt8C,GAAG2jC,KAC9F2Y,EAAYt8C,GAAG2jC,IAAM2Y,EAAYrlC,GAAG0sB,IACpC2Y,EAAYrlC,GAAGkiC,QAAS,GAGjBmD,EAAYrlC,GAAG0sB,KAAO2Y,EAAYt8C,GAAG0jC,OAAS4Y,EAAYrlC,GAAG0sB,KAAO2Y,EAAYt8C,GAAG2jC,MAC1F2Y,EAAYt8C,GAAG0jC,MAAQ4Y,EAAYrlC,GAAGysB,MACtC4Y,EAAYrlC,GAAGkiC,QAAS,QAM3Bn5C,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,KACJ,IAA1Bs8C,EAAYt8C,GAAGm5C,QACjB0E,EAAU19C,KAAKm8C,EAAYt8C,IAI/B87C,EAAKQ,YAAcuB,OACnB/B,EAAKQ,qBAAiB,SAACj8C,EAAGC,UAAMD,EAAEqjC,MAAQpjC,EAAEojC,SAmBvC,SAASoa,GAAoBxwB,EAAQywB,EAAUC,WAChDC,GAAe,EACbC,EAAeH,EAASI,QAAQ19C,UAC7BT,EAAI,EAAGA,EAAI+9C,EAASzB,YAAYp8C,OAAQF,IAAK,KAC9C88C,EAAYiB,EAASzB,YAAYt8C,GAAG0jC,MACtCqZ,EAAUgB,EAASzB,YAAYt8C,GAAG2jC,OAClCua,GAAgBpB,GAAaoB,EAAenB,EAAS,CACvDkB,GAAe,YAKC,GAAhBA,GAAwBC,EAAeH,EAASK,KAAK39C,WAAay9C,GAAgBF,EAAc,KAC5FK,EAAY/wB,EAAO0wB,GACnBM,EAAWhxB,EAAOyvB,GAEpBsB,EAAU9xC,QAAU+xC,EAAS/xC,OAASwxC,EAASQ,cAAe,EACzDF,EAAUnxC,SAAWoxC,EAASpxC,QAAU6wC,EAASS,eAAgB,EACjEH,EAAUxtC,aAAeytC,EAASztC,cAAcktC,EAASU,aAAc,GAEhFV,EAASI,QAAUG,GAmChB,SAASI,GAASC,EAAM/6B,EAAMkoB,OAC/B8S,KACgC,GAAhCD,EAAK7C,KAAKQ,YAAYp8C,cACtB0+C,EAAaD,EAAK/B,MAAMgC,WAAW9S,IAC3BloB,EAAKnjB,UAAYm+C,EAAWl/B,QAAUk/B,EAAWC,UAEnDpB,EAASJ,GAAYz5B,EAAM+6B,EAAK7C,KAAKQ,aACtB,GAAjBmB,EAAOA,SACT75B,EAAO65B,EAAOX,eAGVv+B,EAAWugC,GAAyBH,EAAK7C,KAAKQ,YAAaqC,EAAK/B,MAAMlZ,MAAOib,EAAK/B,MAAMjZ,QAC1F/f,EAAO+6B,EAAK/B,MAAMlZ,MAAO,CAC3Bkb,EAAaD,EAAK/B,MAAMgC,WAAW9S,EAAOvtB,OACpCwgC,EAAoBC,GAA6BL,EAAK7C,KAAKQ,YAAa14B,EAAMg7B,EAAWl/B,eAC/FkE,EAAO+6B,EAAKtsB,QAAQ/E,OAAO1J,GAAMkC,SAASrlB,UAC1CmjB,GAAcm7B,IACLH,EAAWl/B,OAASkE,EAAKnjB,WAAam+C,EAAWC,MAErD,GAAIj7B,EAAO+6B,EAAK/B,MAAMjZ,IAAK,KAC1Bsb,EAAgB,CAACvb,MAAOib,EAAK/B,MAAMlZ,MAAOC,IAAK/f,UACrDA,EAAOs7B,GAAqBP,EAAKtsB,QAAQ/E,OAAQqxB,EAAK7C,KAAKQ,YAAa2C,EAAer7B,GACvFg7B,EAAaD,EAAK/B,MAAMgC,WAAW9S,EAAOvtB,IAClCqF,EAAKnjB,UAAYm+C,EAAWl/B,QAAUk/B,EAAWC,aAGzDj7B,EAAOs7B,GAAqBP,EAAKtsB,QAAQ/E,OAAQqxB,EAAK7C,KAAKQ,YAAaqC,EAAK/B,MAAOh5B,GACpFg7B,EAAaD,EAAK/B,MAAMgC,WAAW9S,EAAOvtB,IAClCqF,EAAKnjB,UAAYm+C,EAAWl/B,QAAUk/B,EAAWC,MAa1D,SAASM,GAAOR,EAAMtxC,EAAGy+B,MACM,GAAhC6S,EAAK7C,KAAKQ,YAAYp8C,OAAa,KAC/B0+C,EAAaD,EAAK/B,MAAMgC,WAAW9S,UAClC,IAAIlsC,KAAKyN,EAAIuxC,EAAWC,MAAQD,EAAWl/B,YAG5C0/B,EAAiBN,GAAyBH,EAAK7C,KAAKQ,YAAaqC,EAAK/B,MAAMlZ,MAAOib,EAAK/B,MAAMjZ,KAE9F0b,GADgBV,EAAK/B,MAAMjZ,IAAMgb,EAAK/B,MAAMlZ,MAAQ0b,GAClB/xC,EAAIy+B,EACtCwT,EAA4BC,GAA6BZ,EAAK7C,KAAKQ,YAAaqC,EAAK/B,MAAOyC,UAE3F,IAAIz/C,KAAK0/C,EAA4BD,EAAkBV,EAAK/B,MAAMlZ,OAYtE,SAASob,GAAyBxC,EAAa5Y,EAAOC,WACvDplB,EAAW,EACNve,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,IAAK,KACrC88C,EAAYR,EAAYt8C,GAAG0jC,MAC3BqZ,EAAUT,EAAYt8C,GAAG2jC,IAE3BmZ,GAAapZ,GAASqZ,EAAUpZ,IAClCplB,GAAYw+B,EAAUD,UAGnBv+B,EAWF,SAASygC,GAA6B1C,EAAa5Y,EAAOC,WAC3DplB,EAAW,EACNve,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,IAAK,KACrC88C,EAAYR,EAAYt8C,GAAG0jC,MAC3BqZ,EAAUT,EAAYt8C,GAAG2jC,IAE3BmZ,GAAapZ,GAASqZ,GAAWpZ,IACnCplB,GAAYw+B,EAAUD,UAGnBv+B,EAWF,SAAS2gC,GAAqB5xB,EAAQgvB,EAAaM,EAAOh5B,UAC/DA,EAAO0J,EAAO1J,GAAMkC,SAASrlB,UAC7BmjB,GAAQ47B,GAAwBlyB,EAAQgvB,EAAYM,EAAMh5B,GAYrD,SAAS47B,GAAwBlyB,EAAQgvB,EAAaM,EAAOh5B,OAC9D67B,EAAa,EACjB77B,EAAO0J,EAAO1J,GAAMkC,SAASrlB,cAExB,IAAIT,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,IAAK,KACrC88C,EAAYR,EAAYt8C,GAAG0jC,MAC3BqZ,EAAUT,EAAYt8C,GAAG2jC,IAE3BmZ,GAAaF,EAAMlZ,OAASqZ,EAAUH,EAAMjZ,KAC1C/f,GAAQm5B,IACV0C,GAAe1C,EAAUD,UAIxB2C,EAWF,SAASF,GAA6BjD,EAAaM,EAAO8C,WAC3DN,EAAiB,EACjB7gC,EAAW,EACXohC,EAAgB/C,EAAMlZ,MAEjB1jC,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,IAAK,KACrC88C,EAAYR,EAAYt8C,GAAG0jC,MAC3BqZ,EAAUT,EAAYt8C,GAAG2jC,OAE3BmZ,GAAaF,EAAMlZ,OAASqZ,EAAUH,EAAMjZ,IAAK,IACnDplB,GAAYu+B,EAAY6C,EACxBA,EAAgB5C,EACZx+B,GAAYmhC,QAIdN,GAAkBrC,EAAUD,UAK3BsC,EAWF,SAASQ,GAAmBtD,EAAa14B,EAAMpC,EAAWq+B,OACzDC,EAAWzC,GAAYz5B,EAAM04B,UACZ,GAAnBwD,EAASrC,OACPj8B,EAAY,EACW,GAArBq+B,EACKC,EAAShD,WAAagD,EAAS/C,QAAUn5B,GAAQ,EAGjDk8B,EAAShD,UAAY,EAIL,GAArB+C,EACKC,EAAS/C,SAAWn5B,EAAOk8B,EAAShD,WAAa,EAGjDgD,EAAS/C,QAAU,EAKvBn5B,EAYJ,SAASy5B,GAAYz5B,EAAM04B,OAC3B,IAAIt8C,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,IAAK,KACvC88C,EAAYR,EAAYt8C,GAAG0jC,MAC3BqZ,EAAUT,EAAYt8C,GAAG2jC,OAEzB/f,GAAQk5B,GAAal5B,EAAOm5B,QACvB,CAACU,QAAQ,EAAMX,UAAAA,EAAWC,QAAAA,SAG9B,CAACU,QAAQ,EAAOX,UAAAA,EAAWC,QAAAA,oHA9R7B,SAAoBt1B,OACpB,IAAIznB,EAAG,EAAGA,EAAIynB,EAAMvnB,OAAQF,IAC/B6F,QAAQq3C,IAAIl9C,EAAG,IAAIJ,KAAK6nB,EAAMznB,GAAG0jC,OAAO,IAAI9jC,KAAK6nB,EAAMznB,GAAG2jC,KAAMlc,EAAMznB,GAAG0jC,MAAOjc,EAAMznB,GAAG2jC,IAAKlc,EAAMznB,GAAGm5C,4OCxNtF4G,0BAOPjE,EAAMzpB,kEAEVta,EAAMuV,KAASha,MAAM,GAAGE,QAAQ,GAAGG,QAAQ,GAAGmL,aAAa,GAC3D4kB,EAAQ3rB,EAAIuI,QAAQvC,KAAK,EAAG,QAAQtd,UACpCkjC,EAAM5rB,EAAIuI,QAAQvC,IAAI,EAAG,QAAQtd,mBAClCu/C,+BAA4Br9C,OAElBA,IAAZ0vB,KACIqR,MAAQA,IACRC,IAAMA,MAEND,MAAQrR,EAAQqR,OAASA,IACzBC,IAAMtR,EAAQsR,KAAOA,KAGvBsc,SAAU,IAEVnE,KAAOA,IACPoE,gBAAkB,IAClBC,YAAc,IACdzC,cAAe,IACfC,YAAa,IAGbyC,eAAiB,CACpBC,KAAK,EACL3c,MAAO,KACPC,IAAK,KACLrW,OAAAA,GACA9L,UAAW,aACX8+B,UAAU,EACVC,UAAU,EACVj7C,IAAK,KACLkD,IAAK,KACLg4C,QAAS,GACTC,QAAS,SACTC,YAAa,CACXC,QAAQ,EACRjhC,OAAQ,OAGP2S,QAAUmpB,GAAKh7C,OAAO,GAAIogD,EAAKR,kBAC/B7H,MAAQ,CACXsI,MAAO,MAEJC,eAAiB,OAGjBhF,KAAKiF,QAAQvH,GAAG,WAAYP,OAAK+H,+BACjClF,KAAKiF,QAAQvH,GAAG,UAAYP,OAAKgI,0BACjCnF,KAAKiF,QAAQvH,GAAG,SAAYP,OAAKiI,6BAGjCpF,KAAKiF,QAAQvH,GAAG,aAAcP,OAAKkI,gCAGnCrF,KAAKiF,QAAQvH,GAAG,QAASP,OAAKmI,2BAC9BtF,KAAKiF,QAAQvH,GAAG,QAASP,OAAKoI,2BAG9BvF,KAAKwF,IAAIC,eAAeC,iBAAiB,QAASvI,OAAKwI,+BAEvDC,WAAWrvB,2DAmBPA,MACLA,EAAS,CAMXmpB,GAAKmG,gBAJU,CACb,YAAa,YAAa,MAAO,MAAO,UAAW,UAAW,WAAY,WAC1E,SAAU,WAAY,cAAe,UAAW,eAAgB,MAAO,kBAAmB,cAAe,oBAE9Ez+C,KAAKmvB,QAASA,GAEvCA,EAAQquB,aAAeruB,EAAQquB,YAAYC,aACxCc,gBAEH,UAAWpvB,GAAW,QAASA,SAE5BuvB,SAASvvB,EAAQqR,MAAOrR,EAAQsR,iDASnCke,EAAK3+C,eAKFm2C,IACPwI,EAAGC,cACHD,EAAG5B,SAAU,MAGT8B,EAAWF,EAAGle,IAAMke,EAAGne,MACrBzgC,EAAIu4C,GAAKpB,QAAQ,IAAIx6C,KAAQ,QAAQa,UACrCuhD,EAAoBH,EAAGxvB,QAAQquB,aAAemB,EAAGxvB,QAAQquB,YAAYhhC,QAAU,GAE/EgkB,EAAQzgC,EAAI8+C,EAAYC,EACxBre,EAAM1gC,EAAI8+C,GAAY,EAAIC,GAKhCH,EAAGD,SAASle,EAAOC,EAHH,CACdse,WAAW,KAMbF,EAAW,EADGF,EAAGjD,WAAWiD,EAAG/F,KAAKY,SAASwF,OAAOpW,OAAO+S,MACpC,IACR,KAAMkD,EAAW,IAC5BA,EAAW,MAAMA,EAAW,KAEhCF,EAAG/F,KAAKwF,IAAIC,eAAe7pB,MAAMyqB,WAAa,SAE9CN,EAAGO,iBAAmBC,GAAWhJ,EAAQ0I,GAG3C1I,8CAO8B12C,IAA1BO,KAAKk/C,mBACPE,aAAap/C,KAAKk/C,uBACbnC,SAAU,OACVnE,KAAKwF,IAAIC,eAAe7pB,MAAMyqB,WAAa,4CAyB3Cze,EAAOC,EAAKtR,EAASnpB,EAAUq5C,GACjClwB,IACHA,EAAU,KAEW,IAAnBA,EAAQmwB,SACVnwB,EAAQmwB,QAAS,OAEbX,EAAK3+C,KACLu/C,EAAsB9/C,MAAT+gC,EAAqB8X,GAAKpB,QAAQ1W,EAAO,QAAQjjC,UAAY,KAC1EiiD,EAAoB//C,MAAPghC,EAAqB6X,GAAKpB,QAAQzW,EAAK,QAAQljC,UAAc,aAC3EkiD,wBACA3C,+BAA4Br9C,EAE7B0vB,EAAQ4vB,UAAW,OACfW,EAAY1/C,KAAKwgC,MACjBmf,EAAU3/C,KAAKygC,IACfplB,EAAyC,WAA7BoqB,GAAOtW,EAAQ4vB,YAA0B,aAAc5vB,EAAQ4vB,UAAa5vB,EAAQ4vB,UAAU1jC,SAAW,IACrHukC,EAA2C,WAA7Bna,GAAOtW,EAAQ4vB,YAA0B,mBAAoB5vB,EAAQ4vB,UAAa5vB,EAAQ4vB,UAAUc,eAAiB,gBACnIA,EAAiBvH,GAAKwH,gBAAgBF,OACvCC,QACG,IAAIz8C,6CAAiC28C,GAAeH,8BAA6BI,GAAY1H,GAAKwH,iBAAiB38C,KAAK,YAG1H88C,EAAWjZ,KACbkZ,GAAa,SAEJ,SAAPlsC,QACC2qC,EAAGtJ,MAAMsI,MAAMwC,SAAU,KAEtBz/B,EADMsmB,KACOiZ,EACbG,EAAOP,EAAen/B,EAAOrF,GAC7B8hB,EAAOzc,EAAOrF,EACdxS,EAAKs0B,GAAuB,OAAfoiB,EAAuBA,EAAaG,GAAaH,EAAaG,GAAaU,EACxF/sC,EAAK8pB,GAAuB,OAAfqiB,EAAuBA,EAAaG,GAAaH,EAAaG,GAAaS,EAE9FC,EAAU1B,EAAGjE,YAAY7xC,EAAGwK,GAC5BitC,GAA2B3B,EAAGxvB,QAAQ/E,OAAQu0B,EAAG/F,KAAM+F,EAAGxvB,QAAQiqB,aAClE8G,EAAaA,GAAcG,MAErBE,EAAS,CACb/f,MAAO,IAAI9jC,KAAKiiD,EAAGne,OACnBC,IAAK,IAAI/jC,KAAKiiD,EAAGle,KACjB6e,OAAQnwB,EAAQmwB,OAChBkB,MAAOrxB,EAAQqxB,UAGbnB,GAAiBA,EAAce,EAAMC,EAASljB,GAE9CkjB,GACF1B,EAAG/F,KAAKiF,QAAQ4C,KAAK,cAAeF,GAGlCpjB,MACE+iB,IACFvB,EAAG/F,KAAKiF,QAAQ4C,KAAK,eAAgBF,GACjCv6C,UAAmBA,SAMzB24C,EAAGf,eAAiBuB,GAAWnrC,EAAM,KAKpCA,OAGHqsC,EAAUrgD,KAAK06C,YAAY6E,EAAYC,MAC3Cc,GAA2BtgD,KAAKmvB,QAAQ/E,OAAQpqB,KAAK44C,KAAM54C,KAAKmvB,QAAQiqB,aACpEiH,EAAS,KACLE,EAAS,CACb/f,MAAO,IAAI9jC,KAAKsD,KAAKwgC,OACrBC,IAAK,IAAI/jC,KAAKsD,KAAKygC,KACnB6e,OAAQnwB,EAAQmwB,OAChBkB,MAAOrxB,EAAQqxB,eAGZ5H,KAAKiF,QAAQ4C,KAAK,cAAeF,GACtCnB,aAAcT,EAAG+B,WACjB/B,EAAG+B,UAAYvB,IAAY,WACzBR,EAAG/F,KAAKiF,QAAQ4C,KAAK,eAAgBF,KACpC,KACCv6C,SAAmBA,mEAWYvG,IAAnCO,KAAK88C,iCACFA,2BAA6B98C,KAAKygC,IAAMzgC,KAAKwgC,OAASxgC,KAAK44C,KAAKwF,IAAIY,OAAO2B,aAE3E3gD,KAAK88C,qEAQR98C,KAAK49C,iBACPwB,aAAap/C,KAAK49C,qBACbA,eAAiB,0CAadpd,EAAOC,OAKbtjB,EAJAyjC,EAAqB,MAATpgB,EAAiB8X,GAAKpB,QAAQ1W,EAAO,QAAQjjC,UAAYyC,KAAKwgC,MAC1EqgB,EAAmB,MAAPpgB,EAAiB6X,GAAKpB,QAAQzW,EAAK,QAAQljC,UAAcyC,KAAKygC,IACxEn7B,EAA2B,MAApBtF,KAAKmvB,QAAQ7pB,IAAegzC,GAAKpB,QAAQl3C,KAAKmvB,QAAQ7pB,IAAK,QAAQ/H,UAAY,KACtF6E,EAA2B,MAApBpC,KAAKmvB,QAAQ/sB,IAAek2C,GAAKpB,QAAQl3C,KAAKmvB,QAAQ/sB,IAAK,QAAQ7E,UAAY,QAIxF6B,MAAMwhD,IAA0B,OAAbA,QACf,IAAIx9C,+BAAwBo9B,WAEhCphC,MAAMyhD,IAAsB,OAAXA,QACb,IAAIz9C,6BAAsBq9B,WAI9BogB,EAASD,IACXC,EAASD,GAIC,OAARx+C,GACEw+C,EAAWx+C,IAEbw+C,GADAzjC,EAAQ/a,EAAMw+C,EAEdC,GAAU1jC,EAGC,MAAP7X,GACEu7C,EAASv7C,IACXu7C,EAASv7C,IAOL,OAARA,GACEu7C,EAASv7C,IAEXs7C,GADAzjC,EAAQ0jC,EAASv7C,EAEjBu7C,GAAU1jC,EAGC,MAAP/a,GACEw+C,EAAWx+C,IACbw+C,EAAWx+C,IAOU,OAAzBpC,KAAKmvB,QAAQmuB,QAAkB,KAC7BA,EAAUwD,GAAW9gD,KAAKmvB,QAAQmuB,YAClCA,EAAU,IACZA,EAAU,GAEPuD,EAASD,EAAYtD,EAAS,CAG5Bt9C,KAAKygC,IAAMzgC,KAAKwgC,QAAW8c,GAAWsD,GAAY5gD,KAAKwgC,MADvC,IAC+DqgB,GAAU7gD,KAAKygC,KAEjGmgB,EAAW5gD,KAAKwgC,MAChBqgB,EAAS7gD,KAAKygC,MAKdmgB,IADAzjC,EAAQmgC,GAAWuD,EAASD,IACT,EACnBC,GAAU1jC,EAAO,OAMM,OAAzBnd,KAAKmvB,QAAQouB,QAAkB,KAC7BA,EAAUuD,GAAW9gD,KAAKmvB,QAAQouB,SAClCA,EAAU,IACZA,EAAU,GAGPsD,EAASD,EAAYrD,IACnBv9C,KAAKygC,IAAMzgC,KAAKwgC,QAAW+c,GAAWqD,EAAW5gD,KAAKwgC,OAASqgB,EAAS7gD,KAAKygC,KAEhFmgB,EAAW5gD,KAAKwgC,MAChBqgB,EAAS7gD,KAAKygC,MAKdmgB,IADAzjC,EAAS0jC,EAASD,EAAYrD,GACX,EACnBsD,GAAU1jC,EAAO,QAKjBkjC,EAAWrgD,KAAKwgC,OAASogB,GAAY5gD,KAAKygC,KAAOogB,SAGhDD,GAAY5gD,KAAKwgC,OAASogB,GAAc5gD,KAAKygC,KAASogB,GAAY7gD,KAAKwgC,OAASqgB,GAAY7gD,KAAKygC,KACjGzgC,KAAKwgC,OAASogB,GAAY5gD,KAAKwgC,OAASqgB,GAAc7gD,KAAKygC,KAAOmgB,GAAc5gD,KAAKygC,KAAOogB,QAC5FjI,KAAKiF,QAAQ4C,KAAK,yBAGpBjgB,MAAQogB,OACRngB,IAAMogB,EACJR,2CAQA,CACL7f,MAAOxgC,KAAKwgC,MACZC,IAAKzgC,KAAKygC,wCAWHmI,EAAOmY,UACTlE,EAAMnB,WAAW17C,KAAKwgC,MAAOxgC,KAAKygC,IAAKmI,EAAOmY,wCAmC1CP,QACNxD,gBAAkB,OAClBgE,cAAgB,EAGhBhhD,KAAKmvB,QAAQiuB,UAGbp9C,KAAKihD,eAAeT,IAIpBxgD,KAAKq1C,MAAMsI,MAAMuD,qBAEjBtC,mBAEAvJ,MAAMsI,MAAMnd,MAAQxgC,KAAKwgC,WACzB6U,MAAMsI,MAAMld,IAAMzgC,KAAKygC,SACvB4U,MAAMsI,MAAMwC,UAAW,EAExBngD,KAAK44C,KAAKwF,IAAI+C,YACXvI,KAAKwF,IAAI+C,KAAK3sB,MAAM4sB,OAAS,yCAS9BZ,MACDA,GAEAxgD,KAAKq1C,MAAMsI,MAAMwC,UAGjBngD,KAAKmvB,QAAQiuB,UAKbp9C,KAAKq1C,MAAMsI,MAAMuD,mBAEhB5iC,EAAYte,KAAKmvB,QAAQ7Q,UAC/B+iC,GAAkB/iC,OACdgjC,EAAsB,cAAbhjC,EAA6BkiC,EAAMe,OAASf,EAAMgB,OAC/DF,GAASthD,KAAKg9C,oBACV6B,EAAY7+C,KAAKq1C,MAAMsI,MAAMld,IAAMzgC,KAAKq1C,MAAMsI,MAAMnd,MAIxDqe,GADiByB,GAAkCtgD,KAAK44C,KAAKQ,YAAap5C,KAAKwgC,MAAOxgC,KAAKygC,SAIvFghB,EADE7Y,EAAsB,cAAbtqB,EAA6Bte,KAAK44C,KAAKY,SAASwF,OAAOpW,MAAQ5oC,KAAK44C,KAAKY,SAASwF,OAAOnW,OAGtG4Y,EADEzhD,KAAKmvB,QAAQguB,IACHmE,EAAQ1Y,EAAQiW,GAEfyC,EAAQ1Y,EAAQiW,MAGzB+B,EAAW5gD,KAAKq1C,MAAMsI,MAAMnd,MAAQihB,EACpCZ,EAAS7gD,KAAKq1C,MAAMsI,MAAMld,IAAMghB,EAGhCC,EAAYpB,GAA4BtgD,KAAK44C,KAAKQ,YAAawH,EAAU5gD,KAAKghD,cAAcM,GAAO,GACnGK,EAAUrB,GAA4BtgD,KAAK44C,KAAKQ,YAAayH,EAAQ7gD,KAAKghD,cAAcM,GAAO,MACjGI,GAAad,GAAYe,GAAWd,cACjC7D,iBAAmBsE,OACnBjM,MAAMsI,MAAMnd,MAAQkhB,OACpBrM,MAAMsI,MAAMld,IAAMkhB,YAClB5D,QAAQyC,QAIVQ,cAAgBM,OAChB5G,YAAYkG,EAAUC,OAGrBjH,EAAY,IAAIl9C,KAAKsD,KAAKwgC,OAC1BqZ,EAAU,IAAIn9C,KAAKsD,KAAKygC,UAGzBmY,KAAKiF,QAAQ4C,KAAK,cAAe,CACpCjgB,MAAOoZ,EACPnZ,IAAOoZ,EACPyF,QAAQ,EACRkB,MAAAA,SAIG5H,KAAKiF,QAAQ4C,KAAK,+CAQdD,GACJxgD,KAAKq1C,MAAMsI,MAAMwC,UAGjBngD,KAAKmvB,QAAQiuB,UAKbp9C,KAAKq1C,MAAMsI,MAAMuD,qBAEjB7L,MAAMsI,MAAMwC,UAAW,EACxBngD,KAAK44C,KAAKwF,IAAI+C,YACXvI,KAAKwF,IAAI+C,KAAK3sB,MAAM4sB,OAAS,aAI/BxI,KAAKiF,QAAQ4C,KAAK,eAAgB,CACrCjgB,MAAO,IAAI9jC,KAAKsD,KAAKwgC,OACrBC,IAAO,IAAI/jC,KAAKsD,KAAKygC,KACrB6e,QAAQ,EACRkB,MAAAA,2CAUUA,OAERc,EAAQ,KACRd,EAAMoB,WACRN,EAAQd,EAAMoB,WAAa,IAClBpB,EAAMqB,OAGfP,GAASd,EAAMqB,OAAS,EACfrB,EAAMgB,SACfF,GAASd,EAAMgB,OAAS,KAIrBxhD,KAAKmvB,QAAQ2yB,UAAYtB,EAAMxgD,KAAKmvB,QAAQ2yB,UAAY9hD,KAAKmvB,QAAQkuB,WACnEr9C,KAAKmvB,QAAQkuB,UAAYr9C,KAAKmvB,QAAQiuB,WAKvCp9C,KAAKmvB,QAAQkuB,UAAYr9C,KAAKmvB,QAAQiuB,UAGvCp9C,KAAKihD,eAAeT,IAKrBc,EAAO,KAOL3F,EASAoG,EAVEC,EAAehiD,KAAKmvB,QAAQ6yB,cAAgB,KAGhDrG,EADE2F,EAAQ,EACF,EAAKA,EAAQU,EAGb,GAAK,EAAKV,EAAQU,GAKxBhiD,KAAK+8C,QAAS,KACV+B,EAAoB9+C,KAAKmvB,QAAQquB,aAAex9C,KAAKmvB,QAAQquB,YAAYhhC,QAAU,GACzFulC,EAAc/hD,KAAKwgC,OAAUxgC,KAAKygC,IAAMzgC,KAAKwgC,OAASse,MACjD,KACCmD,EAAUjiD,KAAKkiD,WAAW,CAAC/3C,EAAGq2C,EAAM2B,QAAS54C,EAAGi3C,EAAM4B,SAAUpiD,KAAK44C,KAAKwF,IAAIY,QACpF+C,EAAc/hD,KAAKqiD,eAAeJ,QAE/BK,KAAK3G,EAAOoG,EAAaT,EAAOd,GAIrCA,EAAM+B,mDASD/B,QACFnL,MAAMsI,MAAMnd,MAAQxgC,KAAKwgC,WACzB6U,MAAMsI,MAAMld,IAAMzgC,KAAKygC,SACvB4U,MAAMsI,MAAMuD,eAAgB,OAC5B7L,MAAMsI,MAAMqB,OAAS,UACrB3J,MAAMsI,MAAM6E,WAAa,UACzBvF,YAAc,OACdD,gBAAkB,EAEvB1E,GAAKiK,eAAe/B,oCAQbA,MAEDxgD,KAAKmvB,QAAQkuB,UAAYr9C,KAAKmvB,QAAQiuB,UAG5C9E,GAAKiK,eAAe/B,QAEfnL,MAAMsI,MAAMuD,eAAgB,EAE5BlhD,KAAKq1C,MAAMsI,MAAMqB,cACf3J,MAAMsI,MAAMqB,OAASh/C,KAAKkiD,WAAW1B,EAAMxB,OAAQh/C,KAAK44C,KAAKwF,IAAIY,aACjE3J,MAAMsI,MAAM6E,WAAaxiD,KAAKqiD,eAAeriD,KAAKq1C,MAAMsI,MAAMqB,cAGhEJ,kBACCjD,EAAQ,GAAK6E,EAAM7E,MAAQ37C,KAAKi9C,aAChCuF,EAAaxiD,KAAKq1C,MAAMsI,MAAM6E,WAE9BtG,EAAiBoE,GAAkCtgD,KAAK44C,KAAKQ,YAAap5C,KAAKwgC,MAAOxgC,KAAKygC,KAC3FgiB,EAAuBnC,GAAiCtgD,KAAKmvB,QAAQ/E,OAAQpqB,KAAK44C,KAAKQ,YAAap5C,KAAMwiD,GAC1GE,EAAsBxG,EAAiBuG,EAGzC7B,EAAY4B,EAAaC,GAAyBziD,KAAKq1C,MAAMsI,MAAMnd,OAASgiB,EAAaC,IAAyB9G,EAClHkF,EAAU2B,EAAaE,GAAwB1iD,KAAKq1C,MAAMsI,MAAMld,KAAO+hB,EAAaE,IAAwB/G,OAG3GnB,aAAe,EAAImB,GAAS,OAC5BlB,WAAakB,EAAQ,GAAK,MAEzB+F,EAAYpB,GAA4BtgD,KAAK44C,KAAKQ,YAAawH,EAAU,EAAIjF,GAAO,GACpFgG,EAAUrB,GAA4BtgD,KAAK44C,KAAKQ,YAAayH,EAAQlF,EAAQ,GAAG,GAClF+F,GAAad,GAAYe,GAAWd,SACjCxL,MAAMsI,MAAMnd,MAAQkhB,OACpBrM,MAAMsI,MAAMld,IAAMkhB,OAClB1E,YAAc,EAAIuD,EAAM7E,MAC7BiF,EAAWc,EACXb,EAASc,OAGLxyB,EAAU,CACd4vB,WAAW,EACXO,QAAQ,EACRkB,MAAAA,QAEG9B,SAASkC,EAAUC,EAAQ1xB,QAE3BqrB,cAAe,OACfC,YAAa,0CAUL+F,OAGP2B,EAAU3B,EAAMxB,OAASwB,EAAMxB,OAAO70C,EAAIq2C,EAAM2B,QAChDQ,EAAsB3iD,KAAK44C,KAAKwF,IAAI3E,gBAAgBmJ,wBACpDz4C,EAAInK,KAAKmvB,QAAQguB,IAAMgF,EAAUQ,EAAoBjO,KAAOiO,EAAoBhO,MAAQwN,EACxFzhC,EAAO1gB,KAAK44C,KAAKN,KAAK2D,OAAO9xC,UAE5BuW,GAAQ1gB,KAAKwgC,OAAS9f,GAAQ1gB,KAAKygC,2CAS7BwhB,OACTvG,EACEp9B,EAAYte,KAAKmvB,QAAQ7Q,aAE/B+iC,GAAkB/iC,GAED,cAAbA,SACKte,KAAK44C,KAAKN,KAAK2D,OAAOgG,EAAQ93C,GAAG5M,cAGlCsrC,EAAS7oC,KAAK44C,KAAKY,SAASwF,OAAOnW,cACzC6S,EAAa17C,KAAK07C,WAAW7S,GACtBoZ,EAAQ14C,EAAImyC,EAAWC,MAAQD,EAAWl/B,0CAW1CmhC,EAAO/V,OACVib,EAAcjb,EAAQgb,+BACxB5iD,KAAKmvB,QAAQguB,IACR,CACLhzC,EAAG04C,EAAYlO,MAAQgJ,EAAMxzC,EAC7BZ,EAAGo0C,EAAMp0C,EAAIs5C,EAAYC,KAGpB,CACL34C,EAAGwzC,EAAMxzC,EAAI04C,EAAYnO,KACzBnrC,EAAGo0C,EAAMp0C,EAAIs5C,EAAYC,kCAiB1BnH,EAAOqD,EAAQsC,EAAOd,GAEX,MAAVxB,IACFA,GAAUh/C,KAAKwgC,MAAQxgC,KAAKygC,KAAO,OAG/Byb,EAAiBoE,GAAkCtgD,KAAK44C,KAAKQ,YAAap5C,KAAKwgC,MAAOxgC,KAAKygC,KAC3FgiB,EAAuBnC,GAAiCtgD,KAAKmvB,QAAQ/E,OAAQpqB,KAAK44C,KAAKQ,YAAap5C,KAAMg/C,GAC1G0D,EAAsBxG,EAAiBuG,EAGzC7B,EAAY5B,EAAOyD,GAAyBziD,KAAKwgC,OAASwe,EAAOyD,IAAyB9G,EAC1FkF,EAAY7B,EAAO0D,GAAwB1iD,KAAKygC,KAAOue,EAAO0D,IAAwB/G,OAGrFnB,eAAe8G,EAAQ,QACvB7G,cAAc6G,EAAS,OACtBI,EAAYpB,GAA4BtgD,KAAK44C,KAAKQ,YAAawH,EAAUU,GAAO,GAChFK,EAAUrB,GAA4BtgD,KAAK44C,KAAKQ,YAAayH,GAASS,GAAO,GAC/EI,GAAad,GAAYe,GAAWd,IACtCD,EAAWc,EACXb,EAASc,OAGLxyB,EAAU,CACd4vB,WAAW,EACXO,QAAQ,EACRkB,MAAAA,QAEG9B,SAASkC,EAAUC,EAAQ1xB,QAE3BqrB,cAAe,OACfC,YAAa,+BASf6G,OAEGnkC,EAAQnd,KAAKygC,IAAMzgC,KAAKwgC,MAGxBogB,EAAW5gD,KAAKwgC,MAAQrjB,EAAOmkC,EAC/BT,EAAS7gD,KAAKygC,IAAMtjB,EAAOmkC,OAI5B9gB,MAAQogB,OACRngB,IAAMogB,iCAONkC,OAGC5lC,GAFUnd,KAAKwgC,MAAQxgC,KAAKygC,KAAO,EAEnBsiB,EAGhBnC,EAAW5gD,KAAKwgC,MAAQrjB,EACxB0jC,EAAS7gD,KAAKygC,IAAMtjB,OAOrBuhC,SAASkC,EAAUC,EALR,CACd9B,WAAW,EACXO,QAAQ,EACRkB,MAAO,4CA/aOhgB,EAAOC,EAAKmI,EAAOmY,eACfthD,IAAhBshD,IACFA,EAAc,GAEH,GAATnY,GAAenI,EAAMD,GAAS,EACzB,CACLhkB,OAAQgkB,EACRmb,MAAO/S,GAASnI,EAAMD,EAAQugB,IAIzB,CACLvkC,OAAQ,EACRm/B,MAAO,UAtcoBhD,IAk3BnC,SAAS0I,GAAmB/iC,MACT,cAAbA,GAA0C,YAAbA,QACzB,IAAI0O,uCAAgC1O,4CC33B9C,OAAiB2R,EAAK2jB,uCCYboP,EAAQ3hD,MACXA,EAAK,gBAWIA,OACR,IAAI4B,KAAO+/C,EAAQ7mD,UACtBkF,EAAI4B,GAAO+/C,EAAQ7mD,UAAU8G,UAExB5B,EAfS4hD,CAAM5hD,GAVtB5F,UAAiBunD,EAqCnBA,EAAQ7mD,UAAUm6C,GAClB0M,EAAQ7mD,UAAUmiD,iBAAmB,SAASkC,EAAO3jD,eAC9CqmD,WAAaljD,KAAKkjD,YAAc,IACpCljD,KAAKkjD,WAAW,IAAM1C,GAASxgD,KAAKkjD,WAAW,IAAM1C,IAAU,IAC7DvjD,KAAKJ,GACDmD,MAaTgjD,EAAQ7mD,UAAUgnD,KAAO,SAAS3C,EAAO3jD,YAC9By5C,SACFC,IAAIiK,EAAOlK,GAChBz5C,EAAGhB,MAAMmE,KAAMlE,kBAGjBw6C,EAAGz5C,GAAKA,OACHy5C,GAAGkK,EAAOlK,GACRt2C,MAaTgjD,EAAQ7mD,UAAUo6C,IAClByM,EAAQ7mD,UAAUinD,eAClBJ,EAAQ7mD,UAAUknD,mBAClBL,EAAQ7mD,UAAUmnD,oBAAsB,SAAS9C,EAAO3jD,WACjDqmD,WAAaljD,KAAKkjD,YAAc,GAGjC,GAAKpnD,UAAUkB,mBACZkmD,WAAa,GACXljD,SAcLujD,EAVAC,EAAYxjD,KAAKkjD,WAAW,IAAM1C,OACjCgD,EAAW,OAAOxjD,QAGnB,GAAKlE,UAAUkB,qBACVgD,KAAKkjD,WAAW,IAAM1C,GACtBxgD,SAKJ,IAAIlD,EAAI,EAAGA,EAAI0mD,EAAUxmD,OAAQF,QACpCymD,EAAKC,EAAU1mD,MACJD,GAAM0mD,EAAG1mD,KAAOA,EAAI,CAC7B2mD,EAAUvQ,OAAOn2C,EAAG,gBAOC,IAArB0mD,EAAUxmD,eACLgD,KAAKkjD,WAAW,IAAM1C,GAGxBxgD,MAWTgjD,EAAQ7mD,UAAUskD,KAAO,SAASD,QAC3B0C,WAAaljD,KAAKkjD,YAAc,WAEjClgD,EAAO,IAAI/G,MAAMH,UAAUkB,OAAS,GACpCwmD,EAAYxjD,KAAKkjD,WAAW,IAAM1C,GAE7B1jD,EAAI,EAAGA,EAAIhB,UAAUkB,OAAQF,IACpCkG,EAAKlG,EAAI,GAAKhB,UAAUgB,MAGtB0mD,EAEG,CAAI1mD,EAAI,MAAR,IAAWmD,GADhBujD,EAAYA,EAAUtgD,MAAM,IACIlG,OAAQF,EAAImD,IAAOnD,EACjD0mD,EAAU1mD,GAAGjB,MAAMmE,KAAMgD,UAItBhD,MAWTgjD,EAAQ7mD,UAAUsnD,UAAY,SAASjD,eAChC0C,WAAaljD,KAAKkjD,YAAc,GAC9BljD,KAAKkjD,WAAW,IAAM1C,IAAU,IAWzCwC,EAAQ7mD,UAAUunD,aAAe,SAASlD,WAC9BxgD,KAAKyjD,UAAUjD,GAAOxjD,mCC7J5B2mD,EANFloD,WAMEkoD,EAAe,KAwBZ,SAASC,EAAYC,EAAQ10B,OAC9B20B,EAAW30B,GAAW,CACxBozB,gBAAgB,MAGdsB,EAAOE,QAAS,KAGdC,EAASH,EAETI,EAAoB,SAASrc,EAASzY,OACpC9kB,EAAInO,OAAO84B,OAAO8uB,UAClB30B,GAAS60B,EAAO5e,OAAO/6B,EAAG8kB,GACvBy0B,EAAY,IAAII,EAAOpc,EAASv9B,GAAIA,WAE7C25C,EAAO5e,OAAO6e,EAAmBD,GAEjCC,EAAkBF,QAAU,SAAUnc,EAASzY,OACzC9kB,EAAInO,OAAO84B,OAAO8uB,UAClB30B,GAAS60B,EAAO5e,OAAO/6B,EAAG8kB,GACvBy0B,EAAY,IAAII,EAAOD,QAAQnc,EAASv9B,GAAIA,IAG9C45C,MAKLlY,EAAU7vC,OAAO84B,OAAO6uB,GAGxBjc,EAAUic,EAAOjc,iBAiGZ/8B,EAAMq5C,UACNA,EAAO59C,MAAM,mBAOb69C,EAAkB3D,MAEN,iBAAfA,EAAMtpB,KAAyB,IAG5BspB,EAAM4D,SAASC,WAClB7D,EAAM4D,SAASC,SAAW,IAGxB7D,EAAM4D,SAASC,SAAS7D,EAAMtpB,aAIhCspB,EAAM4D,SAASC,SAAS7D,EAAMtpB,OAAQ,MAKtCqU,GAAU,EACdiV,EAAM8D,gBAAkB,WACtB/Y,GAAU,OAIRgZ,EAAU/D,EAAM4D,SAASE,gBAAgBj0B,KAAKmwB,EAAM4D,UACnC,mBAAXG,IACR/D,EAAM4D,SAASE,gBAAkB,WAC/BC,IACA/D,EAAM8D,oBAKV9D,EAAMgE,YAAcb,UAGhBc,EAAOd,EACJc,IAASlZ,GAAS,KACnBmZ,EAAaD,EAAKZ,UACnBa,UACGC,EACIhqC,EAAI,EAAGA,EAAI+pC,EAAW1nD,OAAQ2d,OACpCgqC,EAAYD,EAAW/pC,GAAGgqC,UAAUnE,EAAMtpB,MAC5B,IAAK,IAAIp6B,EAAI,EAAGA,EAAI6nD,EAAU3nD,SAAWuuC,EAASzuC,IAC9D6nD,EAAU7nD,GAAG0jD,GAInBiE,EAAOA,EAAKld,mBAvJZK,EAAQic,SAAQjc,EAAQic,OAAS,IACrCjc,EAAQic,OAAO5mD,KAAK8uC,GAIpB8X,EAAOvN,GAAG,gBAAgB,SAAUkK,IACF,IAA5BsD,EAASvB,gBAA4BuB,EAASvB,iBAAmB/B,EAAMoE,aACzEpE,EAAM+B,iBAEJ/B,EAAMqE,UACRlB,EAAenD,EAAM7wB,WAKzBoc,EAAQ4Y,UAAY,GAQpB5Y,EAAQuK,GAAK,SAAU4N,EAAQ1Q,UAE7B3oC,EAAMq5C,GAAQrwC,SAAQ,SAAU2sC,OAC1BmE,EAAY5Y,EAAQ4Y,UAAUnE,GAC7BmE,IACH5Y,EAAQ4Y,UAAUnE,GAASmE,EAAY,GAGvCd,EAAOvN,GAAGkK,EAAO2D,IAEnBQ,EAAU1nD,KAAKu2C,MAGVzH,GAWTA,EAAQwK,IAAM,SAAU2N,EAAQ1Q,UAE9B3oC,EAAMq5C,GAAQrwC,SAAQ,SAAU2sC,OAC1BmE,EAAY5Y,EAAQ4Y,UAAUnE,GAC9BmE,KACFA,EAAYnR,EAAUmR,EAAU1sB,QAAO,SAAUzrB,UACxCA,IAAMgnC,KACV,IAESx2C,OAAS,EACrB+uC,EAAQ4Y,UAAUnE,GAASmE,GAI3Bd,EAAOtN,IAAIiK,EAAO2D,UACXpY,EAAQ4Y,UAAUnE,QAKxBzU,GAQTA,EAAQ0U,KAAO,SAASqE,EAAWtE,GACjCmD,EAAenD,EAAM7wB,OACrBk0B,EAAOpD,KAAKqE,EAAWtE,IAGzBzU,EAAQgZ,QAAU,eAEZC,EAAUnB,EAAOjc,QAAQic,OACzBoB,EAAMD,EAAQt7C,QAAQqiC,IACd,IAATkZ,GAAYD,EAAQ/R,OAAOgS,EAAI,GAC9BD,EAAQhoD,eAAe6mD,EAAOjc,QAAQic,OAG1C9X,EAAQ4Y,UAAY,GAGpBd,EAAOkB,WAgEFhZ;;;;;;;6aCpMX,OAAAmZ,GAvBA,mBAAWhpD,OAAPkpC,OACO,SAAAzV,MACHA,MAAAA,QACI,IAAA3C,UAAN,sDAGEvmB,EAASvK,OAAbyzB,GACS/K,EAAT,EAAoBA,EAAQ9oB,UAA5BkB,OAA8C4nB,IAAS,KAC/CiB,EAAS/pB,UAAf8oB,MACIiB,MAAAA,MACG,IAALs/B,KAAAt/B,EACMA,EAAAA,eAAJs/B,KACE1+C,EAAAA,GAAkBof,EAAlBpf,WAKRA,GAGOvK,OAATkpC,utnBCFEggB,ygCAEkB,oBAAXp5B,OAETo5B,GAAiBnB,GADCj4B,OAAM,QAAcg4B,GACQ,CAC5CzB,eAAgB,UAGlB6C,GAAiB,iBAtBV,CACL9O,GAHI+O,EAAO,aAIX9O,IAAK8O,EACLN,QAASM,EACT5E,KAAM4E,EAENt7C,aAAIhM,SACK,CACLiG,IAAKqhD,KAXb,IACQA,UA4BOD,GClCR,SAASE,GAASzB,EAAQ79C,GAC/BA,EAASu/C,aAAe,SAAU/E,GAC5BA,EAAMqE,SACR7+C,EAASw6C,IAIbqD,EAAOvN,GAAG,eAAgBtwC,EAASu/C,kBCa/BC,yBAUQhlB,EAAOC,EAAKglB,EAAarM,EAAajqB,mBAC3C/E,OAAU+E,GAAWA,EAAQ/E,QAAWA,QACxC+E,QAAUA,GAAoB,QAG9B8rB,QAAUj7C,KAAKoqB,cACfs7B,OAAS1lD,KAAKoqB,cACd8wB,KAAOl7C,KAAKoqB,cAEZu7B,WAAa,OACbhK,MAAQ,WACR5X,KAAO,OAGP2a,SAASle,EAAOC,EAAKglB,QAGrBlK,aAAc,OACdD,eAAgB,OAChBD,cAAe,EAChBnY,GAAckW,QACXA,YAAcA,OAGdA,YADiB35C,MAAf25C,EACY,CAACA,GAGD,QAGhB37C,OAAS+nD,EAASI,oDAQfx7B,QACHA,OAASA,OAGT6wB,QAAUj7C,KAAKoqB,OAAOpqB,KAAKi7C,QAAQ19C,gBACnCmoD,OAAS1lD,KAAKoqB,OAAOpqB,KAAK0lD,OAAOnoD,gBACjC29C,KAAOl7C,KAAKoqB,OAAOpqB,KAAKk7C,KAAK39C,6CAS1BE,OACFwhB,EAAgBq5B,GAAKuN,WAAW,GAAIL,EAASI,aAC9CnoD,OAAS66C,GAAKuN,WAAW5mC,EAAexhB,oCAatC+iC,EAAOC,EAAKglB,QACbjlB,aAAiB9jC,MAAW+jC,aAAe/jC,WACxC,qDAGJgpD,OAAmBjmD,MAAT+gC,EAAsBxgC,KAAKoqB,OAAOoW,EAAMjjC,WAAaypC,UAC/DkU,KAAez7C,MAAPghC,EAAoBzgC,KAAKoqB,OAAOqW,EAAIljC,WAAaypC,KAE1DhnC,KAAK2lD,gBACFG,eAAeL,wCAQjBxK,QAAUj7C,KAAK0lD,OAAOtoC,aACtB2oC,6DAUa,QAAd/lD,KAAK27C,YACFV,QAAQztC,QAAQ,GAIfxN,KAAK27C,WACN,YACEV,QAAQ5xC,KAAKrJ,KAAK+jC,KAAOviC,KAAKE,MAAM1B,KAAKi7C,QAAQ5xC,OAASrJ,KAAK+jC,YAC/DkX,QAAQjxC,MAAM,OAChB,aAAqBixC,QAAQhxC,KAAK,OAClC,WACA,UACA,eAAqBgxC,QAAQ7qC,MAAM,OACnC,YAAqB6qC,QAAQ3qC,QAAQ,OACrC,cAAqB2qC,QAAQxqC,QAAQ,OACrC,cAAqBwqC,QAAQr/B,aAAa,MAIhC,GAAb5b,KAAK+jC,KAAW,KAEbiiB,EAAehmD,KAAKi7C,QAAQ79B,eACzBpd,KAAK27C,WACN,mBAAqBV,QAAQr8B,SAAS5e,KAAKi7C,QAAQr/B,eAAiB5b,KAAK+jC,KAAM,0BAC/E,cAAqBkX,QAAQr8B,SAAS5e,KAAKi7C,QAAQxqC,UAAYzQ,KAAK+jC,KAAM,qBAC1E,cAAqBkX,QAAQr8B,SAAS5e,KAAKi7C,QAAQ3qC,UAAYtQ,KAAK+jC,KAAM,qBAC1E,YAAqBkX,QAAQr8B,SAAS5e,KAAKi7C,QAAQ7qC,QAAUpQ,KAAK+jC,KAAM,mBACxE,cACA,WAAqBkX,QAAQr8B,UAAU5e,KAAKi7C,QAAQhxC,OAAS,GAAKjK,KAAK+jC,KAAM,iBAC7E,YAAqBkX,QAAQr8B,SAAS5e,KAAKi7C,QAAQ1tC,OAASvN,KAAK+jC,KAAM,kBACvE,aAAqBkX,QAAQr8B,SAAS5e,KAAKi7C,QAAQjxC,QAAUhK,KAAK+jC,KAAM,mBACxE,YAAqBkX,QAAQr8B,SAAS5e,KAAKi7C,QAAQ5xC,OAASrJ,KAAK+jC,KAAM,QAGzEiiB,EAAa/jC,OAAOjiB,KAAKi7C,gBACrBA,QAAUj7C,KAAKoqB,OAAOk2B,GAA4BtgD,KAAKo5C,YAAap5C,KAAKi7C,QAAQ19C,WAAY,GAAG,+CAUnGyC,KAAKi7C,QAAQ19C,WAAayC,KAAKk7C,KAAK39C,6CAOtCmsC,EAAO1pC,KAAKi7C,QAAQ19C,iBAIlByC,KAAK27C,WACN,mBAAqBV,QAAQpgC,IAAI7a,KAAK+jC,KAAM,yBAC5C,cAAqBkX,QAAQpgC,IAAI7a,KAAK+jC,KAAM,oBAC5C,cAAqBkX,QAAQpgC,IAAI7a,KAAK+jC,KAAM,oBAC5C,YACEkX,QAAQpgC,IAAI7a,KAAK+jC,KAAM,QAExB/jC,KAAKi7C,QAAQjxC,QAAU,OACpBixC,QAAQr8B,SAAS5e,KAAKi7C,QAAQ7qC,QAAUpQ,KAAK+jC,KAAM,QAEpD/jC,KAAKi7C,QAAQ7qC,QAAUpQ,KAAK+jC,MAAS,QAClCkX,QAAQpgC,IAAI7a,KAAK+jC,KAAO/jC,KAAKi7C,QAAQ7qC,QAAUpQ,KAAK+jC,KAAM,kBAIhE,cACA,WAAqBkX,QAAQpgC,IAAI7a,KAAK+jC,KAAM,iBAC5C,UAC4B,IAA3B/jC,KAAKi7C,QAAQztC,eACVytC,QAAQztC,QAAQ,QAChBytC,QAAQpgC,IAAI7a,KAAK+jC,KAAM,aACvB,IAAoC,IAAjC/jC,KAAKmvB,QAAQ82B,qBAChBhL,QAAQpgC,IAAI7a,KAAK+jC,KAAM,YACvB,KACCxyB,EAAWvR,KAAKi7C,QAAQ79B,QAC9B7L,EAASsJ,IAAI,EAAG,QACbtJ,EAAS0Q,OAAOjiB,KAAKi7C,QAAS,cAC1BA,QAAQpgC,IAAI7a,KAAK+jC,KAAM,cAEvBkX,QAAQpgC,IAAI7a,KAAK+jC,KAAM,aACvBkX,QAAQhxC,KAAK,cAInB,aAAqBgxC,QAAQpgC,IAAI7a,KAAK+jC,KAAM,mBAC5C,YAAqBkX,QAAQpgC,IAAI7a,KAAK+jC,KAAM,WAIlC,GAAb/jC,KAAK+jC,YAEC/jC,KAAK27C,WACN,cAAmB37C,KAAKi7C,QAAQr/B,eAAiB,GAAK5b,KAAKi7C,QAAQr/B,eAAiB5b,KAAK+jC,MAAM/jC,KAAKi7C,QAAQr/B,aAAa,aACzH,SAAmB5b,KAAKi7C,QAAQxqC,UAAY,GAAKzQ,KAAKi7C,QAAQxqC,UAAYzQ,KAAK+jC,MAAM/jC,KAAKi7C,QAAQxqC,QAAQ,aAC1G,SAAmBzQ,KAAKi7C,QAAQ3qC,UAAY,GAAKtQ,KAAKi7C,QAAQ3qC,UAAYtQ,KAAK+jC,MAAM/jC,KAAKi7C,QAAQ3qC,QAAQ,aAC1G,OAAmBtQ,KAAKi7C,QAAQ7qC,QAAU,GAAKpQ,KAAKi7C,QAAQ7qC,QAAUpQ,KAAK+jC,MAAM/jC,KAAKi7C,QAAQ7qC,MAAM,aACpG,cACA,MAAmBpQ,KAAKi7C,QAAQhxC,OAASjK,KAAK+jC,KAAK,GAAG/jC,KAAKi7C,QAAQhxC,KAAK,aACxE,OAAmBjK,KAAKi7C,QAAQ1tC,OAASvN,KAAK+jC,MAAM/jC,KAAKi7C,QAAQ1tC,KAAK,aACtE,QAAmBvN,KAAKi7C,QAAQjxC,QAAUhK,KAAK+jC,MAAM/jC,KAAKi7C,QAAQjxC,MAAM,GAO7EhK,KAAKi7C,QAAQ19C,WAAamsC,SACvBuR,QAAUj7C,KAAKk7C,KAAK99B,cAItBm+B,aAAc,OACdD,eAAgB,OAChBD,cAAe,EAEpBiF,GAA6BtgD,KAAKoqB,OAAQpqB,KAAM0pC,+CAQzC1pC,KAAKi7C,QAAQ79B,yCAebmjC,GACHA,GAAiC,iBAAhBA,EAAO5E,aACrBA,MAAQ4E,EAAO5E,WACf5X,KAAOwc,EAAOxc,KAAO,EAAIwc,EAAOxc,KAAO,OACvC4hB,WAAY,wCAQRO,QACNP,UAAYO,yCAOJT,MACMhmD,MAAfgmD,GAeAU,QAAgBV,SAA0B9J,MAAQ,YAAoB5X,KAAO,KAC7EoiB,QAAeV,SAA2B9J,MAAQ,YAAoB5X,KAAO,KAC7EoiB,QAAeV,SAA2B9J,MAAQ,YAAoB5X,KAAO,KAC7EoiB,QAAcV,SAA4B9J,MAAQ,YAAoB5X,KAAO,IAC7EoiB,QAAcV,SAA4B9J,MAAQ,YAAoB5X,KAAO,IAC7EoiB,QAAaV,SAA6B9J,MAAQ,YAAoB5X,KAAO,GAdzD,QAeT0hB,SAA+B9J,MAAQ,YAAoB5X,KAAO,GAC7EqiB,OAAcX,SAA4B9J,MAAQ,aAAoB5X,KAAO,GAfzD,OAgBR0hB,SAA8B9J,MAAQ,aAAoB5X,KAAO,GAC7EsiB,OAAYZ,GAAezlD,KAAKmvB,QAAQm3B,qBAAgC3K,MAAQ,YAAoB5X,KAAO,GAC3GsiB,OAAYZ,SAA8B9J,MAAQ,WAAoB5X,KAAO,GAjBzD,MAkBV0hB,SAAgC9J,MAAQ,WAAoB5X,KAAO,GAC7EsiB,MAAYZ,SAA8B9J,MAAQ,eAAoB5X,KAAO,GAC7EwiB,MAAad,SAA6B9J,MAAQ,YAAoB5X,KAAO,GAnBzD,KAoBT0hB,SAA+B9J,MAAQ,YAAoB5X,KAAO,GAC7EyiB,IAAgBf,SAA0B9J,MAAQ,cAAoB5X,KAAO,IAC7EyiB,IAAgBf,SAA0B9J,MAAQ,cAAoB5X,KAAO,IAC7EyiB,IAAef,SAA2B9J,MAAQ,cAAoB5X,KAAO,GAtBzD,IAuBP0hB,SAA6B9J,MAAQ,cAAoB5X,KAAO,GAC7E0iB,KAAgBhB,SAA0B9J,MAAQ,cAAoB5X,KAAO,IAC7E0iB,IAAgBhB,SAA0B9J,MAAQ,cAAoB5X,KAAO,IAC7E0iB,IAAehB,SAA2B9J,MAAQ,cAAoB5X,KAAO,GAzBzD,IA0BP0hB,SAA6B9J,MAAQ,cAAoB5X,KAAO,GAC7E2iB,IAAsBjB,SAAoB9J,MAAQ,mBAAoB5X,KAAO,KAC7E2iB,IAAsBjB,SAAoB9J,MAAQ,mBAAoB5X,KAAO,KAC7E2iB,GAAqBjB,SAAqB9J,MAAQ,mBAAoB5X,KAAO,IAC7E2iB,GAAqBjB,SAAqB9J,MAAQ,mBAAoB5X,KAAO,IAC7E2iB,EAAoBjB,SAAsB9J,MAAQ,mBAAoB5X,KAAO,GA9BzD,EA+BF0hB,SAAwB9J,MAAQ,mBAAoB5X,KAAO,yCAqIxD,GAArB/jC,KAAKq7C,oBACCr7C,KAAK27C,WACN,WACA,YACA,WACA,cACA,UACA,WACA,aACA,aACA,qBACI,iBAEA,OAGR,GAA0B,GAAtB37C,KAAKs7C,qBACJt7C,KAAK27C,WACN,WACA,cACA,UACA,WACA,aACA,aACA,qBACI,iBAEA,OAGR,GAAwB,GAApB37C,KAAKu7C,mBACJv7C,KAAK27C,WACN,kBACA,aACA,aACA,cACI,iBAEA,MAIP1xC,EAAOjK,KAAKoqB,OAAOpqB,KAAKi7C,gBACtBj7C,KAAK27C,WACN,qBAC4B,GAAvB1xC,EAAK2R,mBACV,gBACuB,GAAlB3R,EAAKwG,cACV,gBACqB,GAAhBxG,EAAKmG,SAAoC,GAAlBnG,EAAKqG,cACjC,cACqB,GAAhBrG,EAAKmG,YACV,cACA,UAEA,cACoB,GAAfnG,EAAKA,WACV,eACqB,GAAhBA,EAAKD,YACV,sBAGI,yCAWCC,MACAxK,MAARwK,IACFA,EAAOjK,KAAKi7C,SAEVhxC,aAAgBvN,OAClBuN,EAAOjK,KAAKoqB,OAAOngB,IAGmB,mBAA7BjK,KAAKvC,OAAOkpD,mBACd3mD,KAAKvC,OAAOkpD,YAAY18C,EAAMjK,KAAK27C,MAAO37C,KAAK+jC,UAGlDtmC,EAASuC,KAAKvC,OAAOkpD,YAAY3mD,KAAK27C,cAEpC37C,KAAK27C,WACN,UAGgB,IAAhB1xC,EAAKA,QAAmC,IAAnBA,EAAKuD,gBAClB,kBAGH/P,GAAUA,EAAOT,OAAS,EAAKgD,KAAKoqB,OAAOngB,GAAMxM,OAAOA,GAAU,0CAWlEwM,MACAxK,MAARwK,IACFA,EAAOjK,KAAKi7C,SAEVhxC,aAAgBvN,OAClBuN,EAAOjK,KAAKoqB,OAAOngB,IAGmB,mBAA7BjK,KAAKvC,OAAOmpD,mBACd5mD,KAAKvC,OAAOmpD,YAAY38C,EAAMjK,KAAK27C,MAAO37C,KAAK+jC,UAGlDtmC,EAASuC,KAAKvC,OAAOmpD,YAAY5mD,KAAK27C,cACpCl+C,GAAUA,EAAOT,OAAS,EAAKgD,KAAKoqB,OAAOngB,GAAMxM,OAAOA,GAAU,gDAQpEopD,EAAU7mD,KAAKoqB,OACfrsB,EAAIiC,KAAKoqB,OAAOpqB,KAAKi7C,SACrBA,EAAUl9C,EAAEL,OAASK,EAAEL,OAAO,MAAQK,EAAEohB,KAAK,MAC7C4kB,EAAO/jC,KAAK+jC,KACZ+iB,EAAa,YAOVC,EAAKjlD,UACJA,EAAQiiC,EAAO,GAAK,EAAK,YAAc,oBAQxCijB,EAAM/8C,UACTA,EAAKgY,OAAO+kB,KAAY,OACnB,aAEL/8B,EAAKgY,OAAO4kC,IAAUhsC,IAAI,EAAG,OAAQ,OAChC,gBAEL5Q,EAAKgY,OAAO4kC,IAAUhsC,KAAK,EAAG,OAAQ,OACjC,iBAEF,YAQAosC,EAAYh9C,UACZA,EAAKgY,OAAO+kB,KAAY,QAAU,oBAAsB,YAQxDkgB,EAAaj9C,UACbA,EAAKgY,OAAO+kB,KAAY,SAAW,qBAAuB,UAY3DhnC,KAAK27C,WACN,cACHmL,EAAW7pD,KAAK+pD,EAAM/L,IACtB6L,EAAW7pD,KAAK8pD,EAAK9L,EAAQr/B,2BAE1B,SACHkrC,EAAW7pD,KAAK+pD,EAAM/L,IACtB6L,EAAW7pD,KAAK8pD,EAAK9L,EAAQxqC,sBAE1B,SACHq2C,EAAW7pD,KAAK+pD,EAAM/L,IACtB6L,EAAW7pD,KAAK8pD,EAAK9L,EAAQ3qC,sBAE1B,OACHw2C,EAAW7pD,yBAAag+C,EAAQ7qC,iBAAuB,GAAbpQ,KAAK+jC,KAAY,MAAQkX,EAAQ7qC,QAAU,GAAK,KAC1F02C,EAAW7pD,KAAK+pD,EAAM/L,IACtB6L,EAAW7pD,KAAK8pD,EAAK9L,EAAQ7qC,oBAE1B,UACH02C,EAAW7pD,mBAAYg+C,EAAQx9C,OAAO,QAAQ6G,gBAC9CwiD,EAAW7pD,KAAK+pD,EAAM/L,IACtB6L,EAAW7pD,KAAKgqD,EAAYhM,IAC5B6L,EAAW7pD,KAAK8pD,EAAK9L,EAAQhxC,mBAE1B,MACH68C,EAAW7pD,sBAAeg+C,EAAQhxC,SAClC68C,EAAW7pD,mBAAYg+C,EAAQx9C,OAAO,QAAQ6G,gBAC9CwiD,EAAW7pD,KAAK+pD,EAAM/L,IACtB6L,EAAW7pD,KAAKiqD,EAAajM,IAC7B6L,EAAW7pD,KAAK+C,KAAK+jC,MAAQ,EAAIijB,EAAM/L,GAAW,IAClD6L,EAAW7pD,KAAK+C,KAAK+jC,MAAQ,gBAAWkX,EAAQx9C,OAAO,QAAQ6G,eAAkB,IACjFwiD,EAAW7pD,KAAK8pD,EAAK9L,EAAQhxC,OAAS,cAEnC,OACH68C,EAAW7pD,uBAAgBg+C,EAAQx9C,OAAO,OAC1CqpD,EAAW7pD,KAAKgqD,EAAYhM,IAC5B6L,EAAW7pD,KAAK8pD,EAAK9L,EAAQ1tC,mBAE1B,QACHu5C,EAAW7pD,mBAAYg+C,EAAQx9C,OAAO,QAAQ6G,gBAC9CwiD,EAAW7pD,KAAKiqD,EAAajM,IAC7B6L,EAAW7pD,KAAK8pD,EAAK9L,EAAQjxC,oBAE1B,OACH88C,EAAW7pD,uBAAgBg+C,EAAQ5xC,SACnCy9C,EAAW7pD,cAjDMgN,UACZA,EAAKgY,OAAO+kB,KAAY,QAAU,oBAAsB,GAgD7CmgB,CAAYlM,IAC5B6L,EAAW7pD,KAAK8pD,EAAK9L,EAAQ5xC,gBAG1BytC,GAAAgQ,QAAAA,EAAkBr4B,QAAQtrB,KAAK,oCApW5B8G,EAAM0xC,EAAO5X,OACjB3mB,EAAQgN,GAAOngB,MAER,QAAT0xC,EAAiB,KACbtyC,EAAO+T,EAAM/T,OAAS7H,KAAK+a,MAAMa,EAAMpT,QAAU,IACvDoT,EAAM/T,KAAK7H,KAAK+a,MAAMlT,EAAO06B,GAAQA,GACrC3mB,EAAMpT,MAAM,GACZoT,EAAMnT,KAAK,GACXmT,EAAMhN,MAAM,GACZgN,EAAM9M,QAAQ,GACd8M,EAAM3M,QAAQ,GACd2M,EAAMxB,aAAa,QAEhB,GAAa,SAAT+/B,EACHv+B,EAAMnT,OAAS,IACjBmT,EAAMnT,KAAK,GACXmT,EAAMvC,IAAI,EAAG,UAIbuC,EAAMnT,KAAK,GAGbmT,EAAMhN,MAAM,GACZgN,EAAM9M,QAAQ,GACd8M,EAAM3M,QAAQ,GACd2M,EAAMxB,aAAa,QAEhB,GAAa,QAAT+/B,EACDv+B,EAAM5P,UAAY,GAClB4P,EAAM5P,QAAQ,GACd4P,EAAMvC,IAAI,EAAG,SAGbuC,EAAM5P,QAAQ,GAGlB4P,EAAMhN,MAAM,GACZgN,EAAM9M,QAAQ,GACd8M,EAAM3M,QAAQ,GACd2M,EAAMxB,aAAa,QAElB,GAAa,OAAT+/B,EAAgB,QAEf5X,QACD,OACA,EACH3mB,EAAMhN,MAAuC,GAAjC5O,KAAK+a,MAAMa,EAAMhN,QAAU,mBAEvCgN,EAAMhN,MAAuC,GAAjC5O,KAAK+a,MAAMa,EAAMhN,QAAU,KAE3CgN,EAAM9M,QAAQ,GACd8M,EAAM3M,QAAQ,GACd2M,EAAMxB,aAAa,QAEhB,GAAa,WAAT+/B,EAAoB,QAEnB5X,QACD,OACA,EACH3mB,EAAMhN,MAAuC,GAAjC5O,KAAK+a,MAAMa,EAAMhN,QAAU,mBAEvCgN,EAAMhN,MAAsC,EAAhC5O,KAAK+a,MAAMa,EAAMhN,QAAU,IAE3CgN,EAAM9M,QAAQ,GACd8M,EAAM3M,QAAQ,GACd2M,EAAMxB,aAAa,QAEhB,GAAa,QAAT+/B,EAAiB,QAChB5X,QACD,EACH3mB,EAAM9M,QAA2C,GAAnC9O,KAAK+a,MAAMa,EAAM9M,UAAY,mBAE3C8M,EAAM9M,QAA2C,GAAnC9O,KAAK+a,MAAMa,EAAM9M,UAAY,KAE/C8M,EAAM3M,QAAQ,GACd2M,EAAMxB,aAAa,QACd,GAAa,UAAT+/B,EAAmB,QAEpB5X,QACD,QACA,GACH3mB,EAAM9M,QAA0C,EAAlC9O,KAAK+a,MAAMa,EAAM9M,UAAY,IAC3C8M,EAAM3M,QAAQ,cAEX,EACH2M,EAAM3M,QAA2C,GAAnCjP,KAAK+a,MAAMa,EAAM3M,UAAY,mBAE3C2M,EAAM3M,QAA2C,GAAnCjP,KAAK+a,MAAMa,EAAM3M,UAAY,KAE/C2M,EAAMxB,aAAa,QAEhB,GAAa,UAAT+/B,SAEC5X,QACD,QACA,GACH3mB,EAAM3M,QAA0C,EAAlCjP,KAAK+a,MAAMa,EAAM3M,UAAY,IAC3C2M,EAAMxB,aAAa,cAEhB,EACHwB,EAAMxB,aAAuD,IAA1Cpa,KAAK+a,MAAMa,EAAMxB,eAAiB,oBAErDwB,EAAMxB,aAAsD,IAAzCpa,KAAK+a,MAAMa,EAAMxB,eAAiB,WAGtD,GAAa,eAAT+/B,EAAwB,KACzByL,EAAQrjB,EAAO,EAAIA,EAAO,EAAI,EACpC3mB,EAAMxB,aAAapa,KAAK+a,MAAMa,EAAMxB,eAAiBwrC,GAASA,UAGzDhqC,WC/cX,SAASiqC,GAAYC,EAAKC,QACX,IAARA,IAAiBA,EAAM,QACxBC,EAAWD,EAAIC,YAEdF,GAA2B,oBAAbl6B,cAEfq6B,EAAOr6B,SAASq6B,MAAQr6B,SAASs6B,qBAAqB,QAAQ,GAC9DlzB,EAAQpH,SAASE,cAAc,SACnCkH,EAAM0C,KAAO,WAEI,QAAbswB,GACEC,EAAKE,WACPF,EAAKzf,aAAaxT,EAAOizB,EAAKE,YAKhCF,EAAK9yB,YAAYH,GAGfA,EAAMozB,WACRpzB,EAAMozB,WAAWC,QAAUP,EAE3B9yB,EAAMG,YAAYvH,SAAS06B,eAAeR,KDkrB9C9B,GAASI,OAAS,CAChBe,YAAa,CACXrsC,YAAY,MACZD,OAAY,IACZD,OAAY,QACZd,KAAY,QACZ9L,QAAY,QACZ4B,IAAY,IACZ7B,KAAY,IACZvD,MAAY,MACZX,KAAY,QAEdu9C,YAAa,CACXtsC,YAAY,WACZD,OAAY,eACZD,OAAY,aACZd,KAAY,aACZ9L,QAAY,YACZ4B,IAAY,YACZ7B,KAAY,YACZvD,MAAY,OACZX,KAAY,smFErtBV0+C,0BAQQnP,EAAMzpB,yDAEXivB,IAAM,CACT4J,WAAY,KACZC,MAAO,GACPC,WAAY,GACZC,WAAY,GACZ/gB,UAAW,CACT6gB,MAAO,GACPC,WAAY,GACZC,WAAY,OAGX9S,MAAQ,CACXqE,MAAO,CACLlZ,MAAO,EACPC,IAAK,EACLglB,YAAa,GAEf2C,QAAS,KAGNlL,eAAiB,CACpBmL,YAAa,CACXC,KAAM,UAERC,iBAAiB,EACjBtC,iBAAiB,EACjBK,eAAe,EACfkC,cAAe,EACf/qD,OAAQ+nD,GAASI,OACjBx7B,OAAAA,GACAq+B,SAAU,QAEPt5B,QAAUmpB,GAAKh7C,OAAO,GAAIogD,EAAKR,kBAE/BtE,KAAOA,IAGP8P,YAEAlK,WAAWrvB,2DAYPA,GACLA,IAEFmpB,GAAKmG,gBAAgB,CACnB,kBACA,kBACA,gBACA,gBACA,cACA,WACA,SACA,OACCz+C,KAAKmvB,QAASA,GAGjBmpB,GAAKqQ,oBAAoB,CAAC,UAAW3oD,KAAKmvB,QAASA,GAE/C,gBAAiBA,IACgB,iBAAxBA,EAAQk5B,iBACZl5B,QAAQk5B,YAAYC,KAAOn5B,EAAQk5B,YAEF,WAA/B5iB,GAAOtW,EAAQk5B,cAA4B,SAAUl5B,EAAQk5B,mBAC/Dl5B,QAAQk5B,YAAYC,KAAOn5B,EAAQk5B,YAAYC,OAMpD,WAAYn5B,IACe,mBAAlB/E,GAAO1sB,OAEhB0sB,GAAO1sB,OAAOyxB,EAAQzxB,QAGtB0sB,GAAOjL,KAAKgQ,EAAQzxB,iDAUrB0gD,IAAI4J,WAAa56B,SAASE,cAAc,YACxC8wB,IAAIwK,WAAax7B,SAASE,cAAc,YAExC8wB,IAAI4J,WAAW1f,UAAY,oCAC3B8V,IAAIwK,WAAWtgB,UAAY,iEAQ5BtoC,KAAKo+C,IAAI4J,WAAWzgB,iBACjB6W,IAAI4J,WAAWzgB,WAAWC,YAAYxnC,KAAKo+C,IAAI4J,YAElDhoD,KAAKo+C,IAAIwK,WAAWrhB,iBACjB6W,IAAIwK,WAAWrhB,WAAWC,YAAYxnC,KAAKo+C,IAAIwK,iBAGjDhQ,KAAO,0CAQNvD,EAAQr1C,KAAKq1C,MACb2S,EAAahoD,KAAKo+C,IAAI4J,WACtBY,EAAa5oD,KAAKo+C,IAAIwK,WAGtBh4B,EAA2C,OAAjC5wB,KAAKmvB,QAAQk5B,YAAYC,KAAiBtoD,KAAK44C,KAAKwF,IAAI0E,IAAM9iD,KAAK44C,KAAKwF,IAAIyK,OACtFC,EAAiBd,EAAWzgB,aAAe3W,OAG5Cm4B,yBAGCR,EAAkBvoD,KAAKmvB,QAAQo5B,iBAAqD,SAAlCvoD,KAAKmvB,QAAQk5B,YAAYC,KAC3ErC,EAAkBjmD,KAAKmvB,QAAQ82B,iBAAqD,SAAlCjmD,KAAKmvB,QAAQk5B,YAAYC,KAGjFjT,EAAM2T,iBAAmBT,EAAkBlT,EAAM4T,gBAAkB,EACnE5T,EAAM6T,iBAAmBjD,EAAkB5Q,EAAM8T,gBAAkB,EACnE9T,EAAMxM,OAASwM,EAAM2T,iBAAmB3T,EAAM6T,iBAC9C7T,EAAMzM,MAAQof,EAAWoB,YAEzB/T,EAAMgU,gBAAkBrpD,KAAK44C,KAAKY,SAAS2H,KAAKtY,OAASwM,EAAM6T,kBACzB,OAAjClpD,KAAKmvB,QAAQk5B,YAAYC,KAAgBtoD,KAAK44C,KAAKY,SAASqP,OAAOhgB,OAAS7oC,KAAK44C,KAAKY,SAASsJ,IAAIja,QACxGwM,EAAMiU,eAAiB,EACvBjU,EAAMkU,gBAAkBlU,EAAMgU,gBAAkBhU,EAAM6T,iBACtD7T,EAAMmU,eAAiB,MAGjBC,EAAwBzB,EAAW0B,YACnCC,EAAwBf,EAAWc,mBACzC1B,EAAWzgB,YAAcygB,EAAWzgB,WAAWC,YAAYwgB,GAC3DY,EAAWrhB,YAAcqhB,EAAWrhB,WAAWC,YAAYohB,GAE3DZ,EAAWxzB,MAAMqU,iBAAY7oC,KAAKq1C,MAAMxM,kBAEnC+gB,iBAGDH,EACF74B,EAAOoX,aAAaggB,EAAYyB,GAGhC74B,EAAO+D,YAAYqzB,GAEjB2B,OACG/Q,KAAKwF,IAAIyL,mBAAmB7hB,aAAa4gB,EAAYe,QAGrD/Q,KAAKwF,IAAIyL,mBAAmBl1B,YAAYi0B,GAExC5oD,KAAK8pD,cAAgBhB,+CAQtBT,EAAcroD,KAAKmvB,QAAQk5B,YAAYC,KAGvC9nB,EAAQ8X,GAAKpB,QAAQl3C,KAAK44C,KAAKc,MAAMlZ,MAAO,UAC5CC,EAAM6X,GAAKpB,QAAQl3C,KAAK44C,KAAKc,MAAMjZ,IAAK,UACxCspB,EAAgB/pD,KAAK44C,KAAKN,KAAK2D,QAAQj8C,KAAKq1C,MAAM2U,gBAAkB,IAAMhqD,KAAKmvB,QAAQq5B,eAAejrD,UACxGkoD,EAAcsE,EAAgBzJ,GAAiCtgD,KAAKmvB,QAAQ/E,OAAQpqB,KAAK44C,KAAKQ,YAAap5C,KAAK44C,KAAKc,MAAOqQ,GAChItE,GAAezlD,KAAK44C,KAAKN,KAAK2D,OAAO,GAAG1+C,cAElCwmC,EAAO,IAAIyhB,GAAS,IAAI9oD,KAAK8jC,GAAQ,IAAI9jC,KAAK+jC,GAAMglB,EAAazlD,KAAK44C,KAAKQ,YAAap5C,KAAKmvB,SACnG4U,EAAKkmB,UAAUjqD,KAAKmvB,QAAQ/E,QACxBpqB,KAAKmvB,QAAQ1xB,QACfsmC,EAAKmmB,UAAUlqD,KAAKmvB,QAAQ1xB,QAE1BuC,KAAKmvB,QAAQs5B,UACf1kB,EAAKomB,SAASnqD,KAAKmvB,QAAQs5B,eAExB1kB,KAAOA,MAaRkX,EACAjnC,EACA7J,EACAigD,EACAC,EACAC,EAbElM,EAAMp+C,KAAKo+C,IACjBA,EAAIhX,UAAU6gB,MAAQ7J,EAAI6J,MAC1B7J,EAAIhX,UAAU8gB,WAAa9J,EAAI8J,WAC/B9J,EAAIhX,UAAU+gB,WAAa/J,EAAI+J,WAC/B/J,EAAI6J,MAAQ,GACZ7J,EAAI8J,WAAa,GACjB9J,EAAI+J,WAAa,OASboC,EACAC,EAIAliB,EANAM,EAAQ,EAGR6hB,OAAmBhrD,EACnBw5C,EAAQ,MAIZlV,EAAKvD,QACLxsB,EAAO+vB,EAAK2mB,aACZN,EAAQpqD,KAAK44C,KAAKN,KAAKkD,SAASxnC,GACzB+vB,EAAK4mB,WAAa1R,EANb,KAM0B,QACpCA,IAEAoR,EAAUtmB,EAAKsmB,UACf/hB,EAAYvE,EAAK6mB,eAEjB3P,EAAUjnC,EACV7J,EAAIigD,EAEJrmB,EAAK/vB,OACLA,EAAO+vB,EAAK2mB,aAGZH,EAAY3hB,EACZA,GAHAwhB,EAAQpqD,KAAK44C,KAAKN,KAAKkD,SAASxnC,IAGhB7J,EACR45B,EAAK4X,WACN,OAAgB2O,GAAgB,gBAChBA,EAAiB1hB,GAAqB,GAAZ2hB,KAG7CvqD,KAAKmvB,QAAQo5B,iBAAmB+B,EAAe,KAC7C/hB,EAAQvoC,KAAK6qD,kBAAkB1gD,EAAG45B,EAAK+mB,cAAc7P,GAAUoN,EAAa/f,GAChFC,EAAM/T,MAAMoU,gBAAWA,QAGrByhB,GAAWrqD,KAAKmvB,QAAQ82B,iBACtB97C,EAAI,IACkB1K,MAApBgrD,IACFA,EAAmBtgD,GAErBo+B,EAAQvoC,KAAK+qD,kBAAkB5gD,EAAG45B,EAAKinB,cAAc/P,GAAUoN,EAAa/f,IAE9EkiB,EAAOxqD,KAAKirD,kBAAkB9gD,EAAGy+B,EAAOyf,EAAa/f,IAGjDgiB,EACFE,EAAOxqD,KAAKkrD,kBAAkB/gD,EAAGy+B,EAAOyf,EAAa/f,GAGjDkiB,IAEFA,EAAKh2B,MAAMoU,gBAAWuiB,GAASX,EAAKh2B,MAAMoU,OAASA,YA/C/C,MAqDRqQ,GAAkBmS,KAClBzoD,QAAQH,4FAtDA,gBAuDR4oD,IAAoB,GAIpBprD,KAAKmvB,QAAQ82B,gBAAiB,KAC1BoF,EAAWrrD,KAAK44C,KAAKN,KAAK2D,OAAO,GACjCqP,EAAWvnB,EAAKinB,cAAcK,GAC9BE,EAAYD,EAAStuD,QAAUgD,KAAKq1C,MAAMmW,gBAAkB,IAAM,IAEhD/rD,MAApBgrD,GAAiCc,EAAYd,SAC1CM,kBAAkB,EAAGO,EAAUjD,EAAa/f,MAKrDgQ,SAAAA,GAAat4C,KAAKo+C,IAAIhX,WAAW,SAAAxqC,QACxBA,EAAII,QAAQ,KACXynD,EAAO7nD,EAAIo1C,MACbyS,GAAQA,EAAKld,YACfkd,EAAKld,WAAWC,YAAYid,iDAelBt6C,EAAGshD,EAAMpD,EAAa/f,OAElCC,EAAQvoC,KAAKo+C,IAAIhX,UAAU+gB,WAAWhjC,YAErCojB,EAAO,KAEJ1U,EAAUzG,SAAS06B,eAAe,KACxCvf,EAAQnb,SAASE,cAAc,QACzBqH,YAAYd,QACbuqB,IAAI4J,WAAWrzB,YAAY4T,QAE7B6V,IAAI+J,WAAWlrD,KAAKsrC,GACzBA,EAAMmjB,UAAYD,MAGdliD,EAAoB,OAAf8+C,EAAwBroD,KAAKq1C,MAAM6T,iBAAmB,cAC1DyC,OAAOpjB,EAAOp+B,EAAGZ,GAEtBg/B,EAAMD,uCAAkCA,GAGjCC,4CAYSp+B,EAAGshD,EAAMpD,EAAa/f,OAElCC,EAAQvoC,KAAKo+C,IAAIhX,UAAU8gB,WAAW/iC,YAErCojB,EAAO,KAEJ1U,EAAUzG,SAASE,cAAc,QACvCib,EAAQnb,SAASE,cAAc,QACzBqH,YAAYd,QACbuqB,IAAI4J,WAAWrzB,YAAY4T,GAGlCA,EAAMqjB,WAAW,GAAGF,UAAYD,EAChCljB,EAAMD,uCAAkCA,OAGpC/+B,EAAoB,OAAf8+C,EAAwB,EAAIroD,KAAKq1C,MAAM2T,6BAC3C2C,OAAOpjB,EAAOp+B,EAAGZ,QAEjB60C,IAAI8J,WAAWjrD,KAAKsrC,GAClBA,iCAUFA,EAAOp+B,EAAGZ,SAETsiD,EAAa7rD,KAAKmvB,QAAQguB,KAAY,EAALhzC,EAAUA,EACjDo+B,EAAM/T,MAAMkiB,mCAAyBmV,kBAAiBtiD,mDAYtCmrC,EAAM9L,EAAOyf,EAAa/f,SAEtCkiB,EAAOxqD,KAAKo+C,IAAIhX,UAAU6gB,MAAM9iC,QAC/BqlC,IAEHA,EAAOp9B,SAASE,cAAc,YACzB8wB,IAAIwK,WAAWj0B,YAAY61B,SAE7BpM,IAAI6J,MAAMhrD,KAAKutD,OAEdnV,EAAQr1C,KAAKq1C,MAEnBmV,EAAKh2B,MAAMoU,gBAAWA,QACtB4hB,EAAKh2B,MAAMqU,iBAAYwM,EAAMgU,0BAEzB9/C,EAAoB,OAAf8+C,EAAwBhT,EAAM6T,iBAAmBlpD,KAAK44C,KAAKY,SAASsJ,IAAIja,OAC7E1+B,EAAIuqC,EAAOW,EAAMiU,eAAiB,cAEjCqC,OAAOnB,EAAMrgD,EAAGZ,GACrBihD,EAAKliB,kCAAwBtoC,KAAKmvB,QAAQguB,IAAO,mBAAqB,sCAA4B7U,GAE3FkiB,4CAYS9V,EAAM9L,EAAOyf,EAAa/f,SAEtCkiB,EAAOxqD,KAAKo+C,IAAIhX,UAAU6gB,MAAM9iC,QAC/BqlC,IAEHA,EAAOp9B,SAASE,cAAc,YACzB8wB,IAAIwK,WAAWj0B,YAAY61B,SAE7BpM,IAAI6J,MAAMhrD,KAAKutD,OAEdnV,EAAQr1C,KAAKq1C,MAEnBmV,EAAKh2B,MAAMoU,gBAAWA,QACtB4hB,EAAKh2B,MAAMqU,iBAAYwM,EAAMkU,0BAEzBhgD,EAAoB,OAAf8+C,EAAwB,EAAIroD,KAAK44C,KAAKY,SAASsJ,IAAIja,OACxD1+B,EAAIuqC,EAAOW,EAAMmU,eAAiB,cAEjCmC,OAAOnB,EAAMrgD,EAAGZ,GACrBihD,EAAKliB,kCAAwBtoC,KAAKmvB,QAAQguB,IAAO,mBAAqB,sCAA4B7U,GAE3FkiB,+CAaFxqD,KAAKo+C,IAAI0N,wBACP1N,IAAI0N,iBAAmB1+B,SAASE,cAAc,YAC9C8wB,IAAI0N,iBAAiBxjB,UAAY,sCACjC8V,IAAI0N,iBAAiBt3B,MAAMkL,SAAW,gBAEtC0e,IAAI0N,iBAAiBn3B,YAAYvH,SAAS06B,eAAe,WACzD1J,IAAI4J,WAAWrzB,YAAY30B,KAAKo+C,IAAI0N,wBAEtCzW,MAAM4T,gBAAkBjpD,KAAKo+C,IAAI0N,iBAAiBC,kBAClD1W,MAAM2U,eAAiBhqD,KAAKo+C,IAAI0N,iBAAiBnL,YAGjD3gD,KAAKo+C,IAAI4N,wBACP5N,IAAI4N,iBAAmB5+B,SAASE,cAAc,YAC9C8wB,IAAI4N,iBAAiB1jB,UAAY,sCACjC8V,IAAI4N,iBAAiBx3B,MAAMkL,SAAW,gBAEtC0e,IAAI4N,iBAAiBr3B,YAAYvH,SAAS06B,eAAe,WACzD1J,IAAI4J,WAAWrzB,YAAY30B,KAAKo+C,IAAI4N,wBAEtC3W,MAAM8T,gBAAkBnpD,KAAKo+C,IAAI4N,iBAAiBD,kBAClD1W,MAAMmW,eAAiBxrD,KAAKo+C,IAAI4N,iBAAiBrL,mBAtenChI,IA2enByS,IAAoB,sBCrepB3vD,mBAOgB0zB,OAQZryB,EAPAylD,EAAiBpzB,GAAWA,EAAQozB,iBAAkB,EAEtD0J,EAAY98B,GAAWA,EAAQ88B,WAAajgC,OAE5CkgC,EAAmB,GACnBC,EAAS,CAACC,QAAQ,GAAIC,MAAM,IAC5BC,EAAQ,OAIPxvD,EAAI,GAAIA,GAAK,IAAKA,IAAMwvD,EAAM79B,OAAO89B,aAAazvD,IAAM,CAAC0vD,KAAW1vD,EAAI,GAAV,GAAeqoB,OAAO,OAEpFroB,EAAI,GAAIA,GAAK,GAAIA,IAAMwvD,EAAM79B,OAAO89B,aAAazvD,IAAM,CAAC0vD,KAAK1vD,EAAGqoB,OAAO,OAEvEroB,EAAI,EAAIA,GAAK,EAAKA,IAAMwvD,EAAM,GAAKxvD,GAAK,CAAC0vD,KAAK,GAAK1vD,EAAGqoB,OAAO,OAE7DroB,EAAI,EAAIA,GAAK,GAAMA,IAAMwvD,EAAM,IAAMxvD,GAAK,CAAC0vD,KAAK,IAAM1vD,EAAGqoB,OAAO,OAEhEroB,EAAI,EAAIA,GAAK,EAAKA,IAAMwvD,EAAM,MAAQxvD,GAAK,CAAC0vD,KAAK,GAAK1vD,EAAGqoB,OAAO,GAGrEmnC,EAAM,QAAU,CAACE,KAAK,IAAKrnC,OAAO,GAClCmnC,EAAM,QAAU,CAACE,KAAK,IAAKrnC,OAAO,GAClCmnC,EAAM,QAAU,CAACE,KAAK,IAAKrnC,OAAO,GAClCmnC,EAAM,QAAU,CAACE,KAAK,IAAKrnC,OAAO,GAClCmnC,EAAM,QAAU,CAACE,KAAK,IAAKrnC,OAAO,GAElCmnC,EAAK,KAAY,CAACE,KAAK,GAAIrnC,OAAO,GAClCmnC,EAAK,GAAY,CAACE,KAAK,GAAIrnC,OAAO,GAClCmnC,EAAK,MAAY,CAACE,KAAK,GAAIrnC,OAAO,GAClCmnC,EAAK,KAAY,CAACE,KAAK,GAAIrnC,OAAO,GAElCmnC,EAAK,MAAY,CAACE,KAAK,GAAIrnC,OAAO,GAClCmnC,EAAK,MAAY,CAACE,KAAK,GAAIrnC,OAAO,GAClCmnC,EAAK,MAAY,CAACE,KAAK,GAAIrnC,WAAO1lB,GAClC6sD,EAAK,IAAY,CAACE,KAAK,GAAIrnC,OAAO,GAClCmnC,EAAK,UAAgB,CAACE,KAAK,EAAGrnC,OAAO,GACrCmnC,EAAK,IAAgB,CAACE,KAAK,EAAGrnC,OAAO,GACrCmnC,EAAK,KAAgB,CAACE,KAAK,GAAIrnC,OAAO,GACtCmnC,EAAK,IAAgB,CAACE,KAAK,GAAIrnC,OAAO,GACtCmnC,EAAK,OAAgB,CAACE,KAAK,GAAIrnC,OAAO,GACtCmnC,EAAK,OAAgB,CAACE,KAAK,GAAIrnC,OAAO,GACtCmnC,EAAK,SAAgB,CAACE,KAAK,GAAIrnC,OAAO,GAEtCmnC,EAAM,KAAW,CAACE,KAAK,IAAKrnC,OAAO,GACnCmnC,EAAM,KAAW,CAACE,KAAK,IAAKrnC,OAAO,GACnCmnC,EAAM,KAAW,CAACE,KAAK,IAAKrnC,OAAO,GACnCmnC,EAAM,KAAW,CAACE,KAAK,IAAKrnC,OAAO,OAI/BsnC,EAAO,SAASjM,GAAQkM,EAAYlM,EAAM,YAC1CmM,EAAK,SAASnM,GAAQkM,EAAYlM,EAAM,UAGxCkM,EAAc,SAASlM,EAAMtpB,WACKz3B,IAAhC0sD,EAAOj1B,GAAMspB,EAAMoM,SAAwB,SACzCC,EAAQV,EAAOj1B,GAAMspB,EAAMoM,SACtB9vD,EAAI,EAAGA,EAAI+vD,EAAM7vD,OAAQF,UACT2C,IAAnBotD,EAAM/vD,GAAGqoB,OAGc,GAAlB0nC,EAAM/vD,GAAGqoB,OAAmC,GAAlBq7B,EAAMsM,UAGd,GAAlBD,EAAM/vD,GAAGqoB,OAAoC,GAAlBq7B,EAAMsM,WALxCD,EAAM/vD,GAAGD,GAAG2jD,GAUM,GAAlB+B,GACF/B,EAAM+B,0BAMZ2J,EAAiB77B,KAAO,SAASptB,EAAK+C,EAAUkxB,WACjCz3B,IAATy3B,IACFA,EAAO,gBAEUz3B,IAAf6sD,EAAMrpD,SACF,IAAIG,MAAM,oBAAsBH,QAEFxD,IAAlC0sD,EAAOj1B,GAAMo1B,EAAMrpD,GAAKupD,QAC1BL,EAAOj1B,GAAMo1B,EAAMrpD,GAAKupD,MAAQ,IAElCL,EAAOj1B,GAAMo1B,EAAMrpD,GAAKupD,MAAMvvD,KAAK,CAACJ,GAAGmJ,EAAUmf,MAAMmnC,EAAMrpD,GAAKkiB,SAKpE+mC,EAAiBa,QAAU,SAAS/mD,EAAUkxB,OAIvC,IAAIj0B,UAHIxD,IAATy3B,IACFA,EAAO,WAEOo1B,EACVA,EAAMjvD,eAAe4F,IACvBipD,EAAiB77B,KAAKptB,EAAI+C,EAASkxB,IAMzCg1B,EAAiBc,OAAS,SAASxM,OAC5B,IAAIv9C,KAAOqpD,KACVA,EAAMjvD,eAAe4F,GAAM,IACP,GAAlBu9C,EAAMsM,UAAwC,GAApBR,EAAMrpD,GAAKkiB,OAAiBq7B,EAAMoM,SAAWN,EAAMrpD,GAAKupD,YAC7EvpD,EAEJ,GAAsB,GAAlBu9C,EAAMsM,UAAyC,GAApBR,EAAMrpD,GAAKkiB,OAAkBq7B,EAAMoM,SAAWN,EAAMrpD,GAAKupD,YACpFvpD,EAEJ,GAAIu9C,EAAMoM,SAAWN,EAAMrpD,GAAKupD,MAAe,SAAPvpD,SACpCA,QAIN,wCAITipD,EAAiBe,OAAS,SAAShqD,EAAK+C,EAAUkxB,WACnCz3B,IAATy3B,IACFA,EAAO,gBAEUz3B,IAAf6sD,EAAMrpD,SACF,IAAIG,MAAM,oBAAsBH,WAEvBxD,IAAbuG,EAAwB,KACtBknD,EAAc,GACdL,EAAQV,EAAOj1B,GAAMo1B,EAAMrpD,GAAKupD,cACtB/sD,IAAVotD,MACG,IAAI/vD,EAAI,EAAGA,EAAI+vD,EAAM7vD,OAAQF,IAC1B+vD,EAAM/vD,GAAGD,IAAMmJ,GAAY6mD,EAAM/vD,GAAGqoB,OAASmnC,EAAMrpD,GAAKkiB,OAC5D+nC,EAAYjwD,KAAKkvD,EAAOj1B,GAAMo1B,EAAMrpD,GAAKupD,MAAM1vD,IAIrDqvD,EAAOj1B,GAAMo1B,EAAMrpD,GAAKupD,MAAQU,OAGhCf,EAAOj1B,GAAMo1B,EAAMrpD,GAAKupD,MAAQ,IAKpCN,EAAiB5a,MAAQ,WACvB6a,EAAS,CAACC,QAAQ,GAAIC,MAAM,KAI9BH,EAAiBnH,QAAU,WACzBoH,EAAS,CAACC,QAAQ,GAAIC,MAAM,IAC5BJ,EAAU3I,oBAAoB,UAAWmJ,GAAM,GAC/CR,EAAU3I,oBAAoB,QAASqJ,GAAI,IAI7CV,EAAU3N,iBAAiB,UAAUmO,GAAK,GAC1CR,EAAU3N,iBAAiB,QAAQqO,GAAG,GAG/BT,MCzKX,SAASiB,GAAUlB,gBACZmB,QAAS,OAEThP,IAAM,CACT6N,UAAWA,QAGR7N,IAAIiP,QAAUjgC,SAASE,cAAc,YACrC8wB,IAAIiP,QAAQ/kB,UAAY,mBAExB8V,IAAI6N,UAAUt3B,YAAY30B,KAAKo+C,IAAIiP,cAEnCxJ,OAASG,GAAOhkD,KAAKo+C,IAAIiP,cACzBxJ,OAAOvN,GAAG,MAAOP,UAAKuX,sBAAmBttD,WAG1C2+C,EAAK3+C,KACLkkD,EAAS,CACX,MAAO,YAAa,QACpB,QACA,MAAO,WAAY,UAAW,aAEhCA,QAAAA,GAAe,SAAU1D,GACvB7B,EAAGkF,OAAOvN,GAAGkK,GAAO,SAAUA,GAC5BA,EAAM8D,wBAKNl3B,UAAYA,SAASwrB,YAClB2U,QAAU,SAAU/M,IA4G7B,SAAoB5Y,EAAShX,QACpBgX,GAAS,IACVA,IAAYhX,SACP,EAETgX,EAAUA,EAAQL,kBAEb,GAlHEimB,CAAWhN,EAAM7wB,OAAQs8B,IAC5BtN,EAAG8O,cAGPrgC,SAASwrB,KAAK0F,iBAAiB,QAASt+C,KAAKutD,eAGzB9tD,IAAlBO,KAAK0tD,eACFA,SAAS3I,eAEX2I,SAAWA,UAGXC,YAAc5X,UAAK0X,mBAAgBztD,kwBAIlCmtD,GAAUhxD,WAGlBgxD,GAAUlS,QAAU,KAKpBkS,GAAUhxD,UAAU4oD,QAAU,gBACvB0I,kBAGArP,IAAIiP,QAAQ9lB,WAAWC,YAAYxnC,KAAKo+C,IAAIiP,SAG7CrtD,KAAKutD,SACPngC,SAASwrB,KAAK0K,oBAAoB,QAAStjD,KAAKutD,cAG5B9tD,IAAlBO,KAAK0tD,eACFA,SAAS3I,eAEX2I,SAAW,UAEX7J,OAAOkB,eACPlB,OAAS,MAQhBsJ,GAAUhxD,UAAUyxD,SAAW,iBAEzBT,GAAUlS,SACZkS,GAAUlS,QAAQwS,aAEpBN,GAAUlS,QAAUj7C,UAEfotD,QAAS,OACThP,IAAIiP,QAAQ74B,MAAMC,QAAU,OACjC6jB,GAAKuV,aAAa7tD,KAAKo+C,IAAI6N,UAAW,mBAEjCxL,KAAK,eACLA,KAAK,sBAILiN,iBAAc,MAAO1tD,KAAK2tD,cAOjCR,GAAUhxD,UAAUsxD,WAAa,WAC3BN,GAAUlS,UAAYj7C,OACxBmtD,GAAUlS,QAAU,WAGjBmS,QAAS,OACThP,IAAIiP,QAAQ74B,MAAMC,QAAU,GACjC6jB,GAAKwV,gBAAgB9tD,KAAKo+C,IAAI6N,UAAW,mBACpCyB,SAAST,OAAO,MAAOjtD,KAAK2tD,kBAE5BlN,KAAK,eACLA,KAAK,eAQZ0M,GAAUhxD,UAAUmxD,cAAgB,SAAU9M,QAEvCoN,WACLpN,EAAM8D,mBCzIR,IAAMyJ,GAAK,CACT9S,QAAS,UACTv6B,KAAM,OACNstC,eAAgB,mBAMZliC,GAAK,CACTmvB,QAAS,UACTv6B,KAAM,QACNstC,eAAgB,yBAMZC,GAAK,CACThT,QAAS,UACTv6B,KAAM,OACNstC,eAAgB,wBAMZE,GAAK,CACTjT,QAAS,WACTv6B,KAAM,OACNstC,eAAgB,kBAKZG,GAAK,CACTlT,QAAS,SACTv6B,KAAM,QACNstC,eAAgB,wBAOZI,GAAK,CACTnT,QAAS,YACTv6B,KAAM,OACNstC,eAAgB,sBAKZK,GAAK,CACTpT,QAAS,WACTv6B,KAAM,MACNstC,eAAgB,mBAKZM,GAAK,CACTrT,QAAS,UACTv6B,KAAM,QACNstC,eAAgB,qBAKZO,GAAK,CACTtT,QAAS,WACTv6B,KAAM,OACNstC,eAAgB,gBAKZQ,GAAK,CACTvT,QAAS,KACTv6B,KAAM,KACNstC,eAAgB,cAIZp7C,GAAU,CACdm7C,GAAAA,GACAU,MAjFYV,GAkFZW,MAjFYX,GAkFZjiC,GAAAA,GACA6iC,MA3EY7iC,GA4EZ8iC,MA3EY9iC,GA4EZmiC,GAAAA,GACAY,MArEYZ,GAsEZa,MArEYb,GAsEZC,GAAAA,GACAa,MA/DYb,GAgEZC,GAAAA,GACAa,MAzDYb,GA0DZc,MAzDYd,GA0DZe,MAzDYf,GA0DZC,GAAAA,GACAe,MAnDYf,GAoDZC,GAAAA,GACAe,MA7CYf,GA8CZC,GAAAA,GACAe,MAvCYf,GAwCZC,GAAAA,GACAe,MAjCYf,GAkCZC,GAAAA,GACAe,MA3BYf,yiCC/ERgB,0BAUQ5W,EAAMzpB,oDAEXypB,KAAOA,IAGPsE,eAAiB,CACpB9yB,OAAAA,GACAxX,QAAAA,GACAlV,OAAQ,KACR01B,QAAI3zB,EACJgwD,WAAOhwD,KAEJ0vB,QAAUmpB,GAAKh7C,OAAO,GAAIogD,EAAKR,kBAC/BsB,WAAWrvB,KACXA,QAAQvc,QAAU0lC,GAAKh7C,OAAO,GAAIsV,GAAS8qC,EAAKvuB,QAAQvc,aACvD88C,EAAiBhS,EAAKR,eAAetqC,QAAQ8qC,EAAKR,eAAex/C,uBAC3DggD,EAAKvuB,QAAQvc,kBAAiB,SAAAlV,KACnCyxB,QAAQvc,QAAQlV,GAAU46C,GAAKh7C,OAClC,GACAoyD,EACAhS,EAAKvuB,QAAQvc,QAAQlV,OAIrByxB,GAA2B,MAAhBA,EAAQzO,OAChBivC,WAAaxgC,EAAQzO,OAErBivC,WAAa,IAAIjzD,OAGnBkzD,YAAc,KAGdlH,kEAUIv5B,GACLA,GAEFmpB,GAAKmG,gBAAgB,CAAC,SAAU,SAAU,UAAW,KAAM,QAAS,MAAO,QAASz+C,KAAKmvB,QAASA,+CAS9F0gC,EAAMziC,SAASE,cAAc,OACnCuiC,EAAI,eAAiB7vD,KACrB6vD,EAAIvnB,oCAA+BtoC,KAAKmvB,QAAQiE,IAAM,IACtDy8B,EAAIr7B,MAAMkL,SAAW,WACrBmwB,EAAIr7B,MAAMsuB,IAAM,MAChB+M,EAAIr7B,MAAMqU,OAAS,YACdgnB,IAAMA,MAELC,EAAO1iC,SAASE,cAAc,gBAe3ByiC,EAAc18C,QAChBulC,KAAKc,MAAMuE,cAAc5qC,GAfhCy8C,EAAKt7B,MAAMkL,SAAW,WACtBowB,EAAKt7B,MAAMsuB,IAAM,MACd9iD,KAAKmvB,QAAQguB,IACd2S,EAAKt7B,MAAMmgB,MAAQ,QAElBmb,EAAKt7B,MAAMkgB,KAAO,QAErBob,EAAKt7B,MAAMqU,OAAS,OACpBinB,EAAKt7B,MAAMoU,MAAQ,OAUfknB,EAAKxR,kBAEPwR,EAAKxR,iBAAiB,aAAcvI,GAAAga,QAAAA,EAAkB/vD,OAAO,GAE7D8vD,EAAKxR,iBAAiB,iBAAkBvI,GAAAga,QAAAA,EAAkB/vD,OAAO,IAGjE8vD,EAAKE,YAAY,eAAgBja,GAAAga,QAAAA,EAAkB/vD,OAGrD6vD,EAAIl7B,YAAYm7B,QAEXjM,OAAS,IAAIG,GAAO8L,QACpBjM,OAAOvN,GAAG,WAAYP,UAAK+H,qBAAkB99C,YAC7C6jD,OAAOvN,GAAG,UAAYP,UAAKgI,gBAAa/9C,YACxC6jD,OAAOvN,GAAG,SAAYP,UAAKiI,mBAAgBh+C,YAC3C6jD,OAAO95C,IAAI,OAAO/F,IAAI,CAACqlB,UAAU,EAAG/K,UAAW0lC,GAAOiM,uDAOtDC,YAEArM,OAAOkB,eACPlB,OAAS,UAETjL,KAAO,0CAQNhoB,EAAS5wB,KAAK44C,KAAKwF,IAAIyL,mBACzB7pD,KAAK6vD,IAAItoB,YAAc3W,IAErB5wB,KAAK6vD,IAAItoB,iBACNsoB,IAAItoB,WAAWC,YAAYxnC,KAAK6vD,KAEvCj/B,EAAO+D,YAAY30B,KAAK6vD,UAGpB1lD,EAAInK,KAAK44C,KAAKN,KAAKkD,SAASx7C,KAAK2vD,YAEnCjyD,EAASsC,KAAKmvB,QAAQvc,QAAQ5S,KAAKmvB,QAAQzxB,QAC1CA,IACEsC,KAAKmwD,SACRxtD,QAAQH,yCAAkCxC,KAAKmvB,QAAQzxB,mGAClDyyD,QAAS,GAEhBzyD,EAASsC,KAAKmvB,QAAQvc,QAAb,UAGP68C,EAAQzvD,KAAKmvB,QAAQsgC,WAEXhwD,IAAVgwD,EAEFA,GADAA,iBAAW/xD,EAAOgjB,mBAAS1gB,KAAKmvB,QAAQ/E,OAAOpqB,KAAK2vD,YAAYlyD,OAAO,iCACzD+oB,OAAO,GAAGjB,cAAgBkqC,EAAMW,UAAU,GAC9B,mBAAVX,IAChBA,EAAQA,EAAMpzD,KAAK2D,KAAMA,KAAK2vD,yBAG3BxgC,QAAQguB,IAAMn9C,KAAK6vD,IAAIr7B,MAAMmgB,gBAAWxqC,QAAQnK,KAAK6vD,IAAIr7B,MAAMkgB,eAAUvqC,aACzE0lD,IAAIJ,MAAQA,GAEV,iCAQHzvD,KAAK6vD,IAAItoB,iBACNsoB,IAAItoB,WAAWC,YAAYxnC,KAAK6vD,2CAQ3BnvC,QACPivC,WAAarX,GAAKpB,QAAQx2B,EAAM,aAChC2vC,wDAQE,IAAI3zD,KAAKsD,KAAK2vD,WAAWpyD,mDAQlBkyD,EAAOa,WACfC,EAASnjC,SAASE,cAAc,QACtCijC,EAAOjoB,mCACPioB,EAAO7E,UAAY+D,EACnBc,EAAO/7B,MAAMkL,SAAW,WAEpB4wB,KACFC,EAAOC,aAAa,kBAAmB,QACvCD,EAAOjS,iBAAiB,eAAe,WACrCiS,EAAOE,WAETF,EAAOjS,iBAAiB,QAASvI,UAAK2a,wBAAqB1wD,OAE3DuwD,EAAOd,MAAQA,EACfc,EAAOjS,iBAAiB,OAAQvI,cAAUyK,GACpCxgD,KAAKyvD,OAASjP,EAAM7wB,OAAO+7B,iBACxBiF,iBAAiBnQ,QACjBiP,MAAQjP,EAAM7wB,OAAO+7B,oBAEvB1rD,aAGJ6vD,IAAIl7B,YAAY47B,0CAORd,QACRtgC,QAAQsgC,MAAQA,uCAQVjP,QACNoP,YAAYzP,UAAW,OACvByP,YAAYD,WAAa3vD,KAAK2vD,WAEnCnP,EAAM8D,kDAQA9D,MACDxgD,KAAK4vD,YAAYzP,cAElBoB,EAASvhD,KAAKmvB,QAAQguB,KAAQ,EAAKqD,EAAMe,OAASf,EAAMe,OAEtDp3C,EAAInK,KAAK44C,KAAKN,KAAKkD,SAASx7C,KAAK4vD,YAAYD,YAAcpO,EAC3D7gC,EAAO1gB,KAAK44C,KAAKN,KAAK2D,OAAO9xC,GAE7BwxC,EAAQ37C,KAAK44C,KAAKN,KAAKsY,WACvB7sB,EAAO/jC,KAAK44C,KAAKN,KAAKuY,UACtBC,EAAO9wD,KAAKmvB,QAAQ2hC,KAEpBC,EAAcD,EAAOA,EAAKpwC,EAAMi7B,EAAO5X,GAAQrjB,OAEhDswC,cAAcD,QAGdnY,KAAKiF,QAAQ4C,KAAK,aAAc,CACnCrtB,GAAIpzB,KAAKmvB,QAAQiE,GACjB1S,KAAM,IAAIhkB,KAAKsD,KAAK2vD,WAAWpyD,WAC/BijD,MAAAA,IAGFA,EAAM8D,sDAQG9D,GACJxgD,KAAK4vD,YAAYzP,gBAGjBvH,KAAKiF,QAAQ4C,KAAK,cAAe,CACpCrtB,GAAIpzB,KAAKmvB,QAAQiE,GACjB1S,KAAM,IAAIhkB,KAAKsD,KAAK2vD,WAAWpyD,WAC/BijD,MAAAA,IAGFA,EAAM8D,2DAQQ9D,QACT5H,KAAKiF,QAAQ4C,KAAK,eAAgB,CACrCrtB,GAAIpzB,KAAKmvB,QAAQiE,GACjBq8B,MAAOjP,EAAM7wB,OAAO+7B,UACpBlL,MAAAA,IAGFA,EAAM8D,2DAQS9D,QACV5H,KAAKiF,QAAQ4C,KAAK,gBAAiB,CACtCrtB,GAAIpzB,KAAKmvB,QAAQiE,GACjBq8B,MAAOjP,EAAM7wB,OAAO+7B,UACpBlL,MAAAA,IAGFA,EAAM8D,iEASoB9D,WACtB7wB,EAAS6wB,EAAM7wB,OACZA,GAAQ,IACTA,EAAOtyB,eAAe,sBACjBsyB,EAAO,eAEhBA,EAASA,EAAO4X,kBAGX,YA9UcoR,klWCWnB8C,iFAQIwQ,yBACD7N,IAAM,QAENA,IAAI6N,UAAYA,OAChB7N,IAAI6N,UAAUz3B,MAAMkL,SAAW,gBAE/B0e,IAAI+C,KAAuB/zB,SAASE,cAAc,YAClD8wB,IAAIwK,WAAuBx7B,SAASE,cAAc,YAClD8wB,IAAIyL,mBAAuBz8B,SAASE,cAAc,YAClD8wB,IAAI6S,qBAAuB7jC,SAASE,cAAc,YAClD8wB,IAAI3E,gBAAuBrsB,SAASE,cAAc,YAClD8wB,IAAI8S,cAAuB9jC,SAASE,cAAc,YAClD8wB,IAAI+S,eAAuB/jC,SAASE,cAAc,YAClD8wB,IAAIY,OAAuB5xB,SAASE,cAAc,YAClD8wB,IAAI1J,KAAuBtnB,SAASE,cAAc,YAClD8wB,IAAIzJ,MAAuBvnB,SAASE,cAAc,YAClD8wB,IAAI0E,IAAuB11B,SAASE,cAAc,YAClD8wB,IAAIyK,OAAuBz7B,SAASE,cAAc,YAClD8wB,IAAIgT,UAAuBhkC,SAASE,cAAc,YAClD8wB,IAAIiT,aAAuBjkC,SAASE,cAAc,YAClD8wB,IAAIkT,cAAuBlkC,SAASE,cAAc,YAClD8wB,IAAImT,iBAAuBnkC,SAASE,cAAc,YAClD8wB,IAAIoT,eAAuBpkC,SAASE,cAAc,YAClD8wB,IAAIqT,kBAAuBrkC,SAASE,cAAc,YAClD8wB,IAAIC,eAAuBjxB,SAASE,cAAc,YAClD8wB,IAAIsT,cAAuBtkC,SAASE,cAAc,YAElD8wB,IAAI+C,KAAK7Y,UAA4B,oBACrC8V,IAAIwK,WAAWtgB,UAAsB,gCACrC8V,IAAIyL,mBAAmBvhB,UAAc,6CACrC8V,IAAI6S,qBAAqB3oB,UAAY,+CACrC8V,IAAI3E,gBAAgBnR,UAAiB,4BACrC8V,IAAI8S,cAAc5oB,UAAmB,0BACrC8V,IAAI+S,eAAe7oB,UAAkB,2BACrC8V,IAAI0E,IAAIxa,UAA6B,yBACrC8V,IAAIyK,OAAOvgB,UAA0B,4BACrC8V,IAAI1J,KAAKpM,UAA4B,mBACrC8V,IAAIY,OAAO1W,UAA0B,mBACrC8V,IAAIzJ,MAAMrM,UAA2B,mBACrC8V,IAAIgT,UAAU9oB,UAAuB,0BACrC8V,IAAIiT,aAAa/oB,UAAoB,6BACrC8V,IAAIkT,cAAchpB,UAAmB,0BACrC8V,IAAImT,iBAAiBjpB,UAAgB,6BACrC8V,IAAIoT,eAAelpB,UAAkB,0BACrC8V,IAAIqT,kBAAkBnpB,UAAe,6BACrC8V,IAAIC,eAAe/V,UAAkB,4BACrC8V,IAAIsT,cAAcppB,UAAmB,0BAErC8V,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAIwK,iBAC9BxK,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAIyL,yBAC9BzL,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAI6S,2BAC9B7S,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAI3E,sBAC9B2E,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAI8S,oBAC9B9S,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAI+S,qBAC9B/S,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAI0E,UAC9B1E,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAIyK,aAC9BzK,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAIC,qBAE9BD,IAAI3E,gBAAgB9kB,YAAY30B,KAAKo+C,IAAIY,aACzCZ,IAAI8S,cAAcv8B,YAAY30B,KAAKo+C,IAAI1J,WACvC0J,IAAI+S,eAAex8B,YAAY30B,KAAKo+C,IAAIzJ,YACxCyJ,IAAI3E,gBAAgB9kB,YAAY30B,KAAKo+C,IAAIgT,gBACzChT,IAAI3E,gBAAgB9kB,YAAY30B,KAAKo+C,IAAIiT,mBACzCjT,IAAI8S,cAAcv8B,YAAY30B,KAAKo+C,IAAIkT,oBACvClT,IAAI8S,cAAcv8B,YAAY30B,KAAKo+C,IAAImT,uBACvCnT,IAAI+S,eAAex8B,YAAY30B,KAAKo+C,IAAIoT,qBACxCpT,IAAI+S,eAAex8B,YAAY30B,KAAKo+C,IAAIqT,wBAGxCpc,MAAQ,CACX8L,KAAM,GACNyH,WAAY,GACZnP,gBAAiB,GACjByX,cAAe,GACfC,eAAgB,GAChBnS,OAAQ,GACRtK,KAAM,GACNC,MAAO,GACPmO,IAAK,GACL+F,OAAQ,GACR8I,OAAQ,GACRC,UAAW,EACXC,aAAc,QAGXvb,GAAG,eAAe,YACQ,IAAzBoH,EAAKoU,iBACPpU,EAAKqU,kBAGJzb,GAAG,gBAAgB,WACjBoH,EAAKsU,yBACRtU,EAAKsU,wBAAyB,WAG7B1b,GAAG,QAAeP,UAAKmI,iBAAcl+C,YACrCs2C,GAAG,UAAmBP,UAAKgI,gBAAa/9C,WAEvC2+C,EAAK3+C,UACNiyD,YAAclc,UAAKgc,gBAAa/xD,WAChC+xD,QAAUzZ,GAAK4Z,SAASlyD,KAAKiyD,kBAE7B3b,GAAG,WAAW,SAAAxc,GACb6kB,EAAGwT,SAAWxT,EAAGwT,QAAQC,qBAAuBt4B,GAAkC,GAApBA,EAAWu4B,MAC3E1T,EAAGoT,UAEHpT,EAAGsT,sBAMFpO,OAAS,IAAIG,GAAOhkD,KAAKo+C,IAAI+C,UAC5BmR,EAAkBtyD,KAAK6jD,OAAO95C,IAAI,SAAS/F,IAAI,CAACkiD,QAAQ,IAC9DoM,GRpFG,SAA0CA,GAG/CA,EAAgBC,eAAiB,iBAExB,CAJkB,UQmFNC,CAA2CF,QACzDzO,OAAO95C,IAAI,OAAO/F,IAAI,CAACqlB,UAAU,EAAG/K,UAAW0lC,GAAOiM,qBACtDxM,UAAY,OR3HMI,EAAQ79C,EQ6HzBk+C,EAAS,CACb,MAAO,YAAa,QACpB,QACA,MAAO,WAAY,UAAW,mBA+BvB6L,EAAavP,GAMhBxgD,KAAKyyD,iBACFhS,KAAK,aAAcD,OAItBe,EAAS,EACTC,EAAS,KAGR,WAAiBhB,IAAUgB,GAAyB,EAAhBhB,EAAMqB,QAC1C,eAAiBrB,IAAUgB,EAAShB,EAAMoB,YAC1C,gBAAiBpB,IAAUgB,EAAShB,EAAMkS,aAC1C,gBAAiBlS,IAAUe,GAA8B,EAArBf,EAAMmS,aAG1C,SAAUnS,GAASA,EAAM8H,OAAS9H,EAAMoS,kBAC3CrR,GAAmB,EAAVC,EACTA,EAAS,GAIN,WAAYhB,IACfgB,GAAyB,EAAhBhB,EAAMgB,QAEZ,WAAYhB,IACfe,EAASf,EAAMe,QAIbf,EAAMqS,YACkB,IAApBrS,EAAMqS,WACRtR,GAlCc,GAmCdC,GAnCc,KAqCdD,GArCc,GAsCdC,GArCc,MAyChBxhD,KAAKmvB,QAAQ2jC,gBACV9yD,KAAKmvB,QAAQ2yB,SAAWtB,EAAMxgD,KAAKmvB,QAAQ2yB,SAAU,eAEtD9hD,KAAKmvB,QAAQ2yB,SAAWtB,EAAMxgD,KAAKmvB,QAAQ2yB,SAAU,UAGtD9hD,KAAKmvB,QAAQ4jC,gBAAmB/yD,KAAKmvB,QAAQ6jC,oBAE9ChzD,KAAKmvB,QAAQ4jC,gBAAkBvxD,KAAKc,IAAIk/C,IAAWhgD,KAAKc,IAAIi/C,GAAS,KACjEtG,EAAUj7C,KAAKq1C,MAAMuc,UACrBqB,EAAWhY,EAAUuG,KAEvBxhD,KAAKyyD,WACczyD,KAAKkzD,cAAcD,KAEnBhY,SACd8W,eACAtR,KAAK,SAAUD,GAIpBA,EAAM+B,uBAGL,GAAIviD,KAAKmvB,QAAQ6jC,iBAAkB,KAIlC71C,GAHQ3b,KAAKc,IAAIi/C,IAAW//C,KAAKc,IAAIk/C,GAAUD,EAASC,GAGxC,KAAQxhD,KAAK05C,MAAMjZ,IAAMzgC,KAAK05C,MAAMlZ,OAAS,GAE7DogB,EAAW5gD,KAAK05C,MAAMlZ,MAAQrjB,EAC9B0jC,EAAS7gD,KAAK05C,MAAMjZ,IAAMtjB,EAE1BgS,EAAU,CACd4vB,WAAW,EACXO,QAAQ,EACRkB,MAAAA,QAEG9G,MAAMgF,SAASkC,EAAUC,EAAQ1xB,GAEtCqxB,EAAM+B,qBA7GV2B,QAAAA,GAAe,SAAAhtB,OACPi8B,EAAW,SAAA3S,GACX7B,EAAG8T,YACL9T,EAAG8B,KAAKvpB,EAAMspB,IAGlB7B,EAAGkF,OAAOvN,GAAGpf,EAAMi8B,GACnBxU,EAAG8E,UAAUvsB,GAAQi8B,KAIvBX,GAAmBxyD,KAAK6jD,QAAQ,SAAArD,GAC9B7B,EAAG8B,KAAK,QAASD,MRnJIqD,EQuJF7jD,KAAK6jD,QRvJK79C,EQuJG,SAAAw6C,GAChC7B,EAAG8B,KAAK,UAAWD,KRvJd+E,aAAe,SAAU/E,GAC5BA,EAAM4S,SACRptD,EAASw6C,IAINqD,EAAOvN,GAAG,eAAgBtwC,EAASu/C,kBQkPlC8N,EAAY,YAAajmC,SAASE,cAAc,OAAS,aACrC7tB,IAA1B2tB,SAASkmC,aAA6B,kBAIjClV,IAAI3E,gBAAgB6E,iBAAmB,iBAAoB,wBAYvDiV,EAAkB/S,MACpB7B,EAAGxvB,QAAQ4jC,kBAEZpU,EAAG6U,2BACL7U,EAAG6U,4BAA6B,UAIlChT,EAAM+B,iBACF5D,EAAG8T,WAAY,KACXQ,GAAYzS,EAAM7wB,OAAOiiC,UAC/BjT,EAAGuU,cAAcD,GACjBtU,EAAGoT,UACHpT,EAAG8B,KAAK,aAAcD,SAxBrBpC,IAAI0E,IAAIxE,sBACRF,IAAIyK,OAAOvK,sBACXF,IAAI3E,gBAAgB6E,iBAAiB+U,EAAWtd,GAAAga,QAAAA,EAAkB/vD,OAAO,QACzEo+C,IAAI0E,IAAIxE,iBAAiB+U,EAAWtd,GAAAga,QAAAA,EAAkB/vD,OAAO,QAC7Do+C,IAAIyK,OAAOvK,iBAAiB+U,EAAWtd,GAAAga,QAAAA,EAAkB/vD,OAAO,QAwBhEo+C,IAAI1J,KAAKnN,WAAW+W,iBAAiB,SAAUvI,GAAAwd,QAAAA,EAAuBvzD,YACtEo+C,IAAIzJ,MAAMpN,WAAW+W,iBAAiB,SAAUvI,GAAAwd,QAAAA,EAAuBvzD,WAExEyzD,GAAsB,WAOjBC,EAAelT,YAClBA,EAAM+B,iBACR5D,EAAG8B,KAAK,WAAY9B,EAAGgV,mBAAmBnT,IAC1CA,EAAM+B,kBAIFqR,KAAApT,EAAM7wB,OAAO2Y,kBAAkB,aAAe,IAGhDmrB,SAEJjT,EAAMqT,aAAaC,WAAa,OAChCL,GAAsB,GACf,WAQAM,EAAWvT,GAEfA,EAAM+B,gBAAkB/B,EAAM+B,iBAC9B/B,EAAM8D,iBAAmB9D,EAAM8D,0BAG5B0P,EAAWxuB,KAAK4R,MAAMoJ,EAAMqT,aAAaI,QAAQ,aAChDD,IAAaA,EAASngC,QAAS,OACpC,MAAOoS,UACA,SAGTwtB,GAAsB,EACtBjT,EAAMxB,OAAS,CACb70C,EAAGq2C,EAAM2B,QACT54C,EAAGi3C,EAAM4B,SAGa,SAApB4R,EAASrkC,OACXgvB,EAAGwT,QAAQ+B,WAAW1T,GAEtB7B,EAAGwT,QAAQgC,oBAAoB3T,GAEjC7B,EAAG8B,KAAK,OAAQ9B,EAAGgV,mBAAmBnT,KAC/B,UAGJpC,IAAIY,OAAOV,iBAAiB,WAAYvI,GAAA2d,QAAAA,EAAoB1zD,OAAO,QACnEo+C,IAAIY,OAAOV,iBAAiB,OAAQvI,GAAAge,QAAAA,EAAgB/zD,OAAO,QAE3Do0D,YAAc,QAGdzW,MAAQ,QAER0W,YAAc,OACdvC,iBAAkB,OAClBE,wBAAyB,GAGzB/F,EAAW,MAAM,IAAI7oD,MAAM,yBAChC6oD,EAAUt3B,YAAY30B,KAAKo+C,IAAI+C,MAC/B8K,EAAUt3B,YAAY30B,KAAKo+C,IAAIsT,kDA4BtBviC,YACLA,EAAS,IAQXmpB,GAAKmG,gBANU,CACb,QAAS,SAAU,YAAa,YAAa,aAC7C,QAAS,MAAO,aAAc,iBAAkB,cAChD,SAAU,UAAW,SAAU,aAAc,MAAO,UACpD,mBAAoB,iBAAkB,uBAEXz+C,KAAKmvB,QAASA,QACtCivB,IAAIC,eAAe7pB,MAAMyqB,WAAa,SAEvCj/C,KAAKmvB,QAAQguB,WACViB,IAAI6N,UAAUz3B,MAAMlW,UAAY,WAChC8/B,IAAIyL,mBAAmBvhB,UAAY,6CAGtCtoC,KAAKmvB,QAAQ4jC,iBACX/yD,KAAKmvB,QAAQguB,SACViB,IAAI+S,eAAe7oB,UAAY,+CAE/B8V,IAAI8S,cAAc5oB,UAAY,0CAIC,WAApC7C,GAAOzlC,KAAKmvB,QAAQk5B,oBACjBl5B,QAAQk5B,YAAc,CAACtQ,UAAKt4C,EAAU6oD,UAAK7oD,IAE9C,gBAAiB0vB,IACgB,iBAAxBA,EAAQk5B,iBACZl5B,QAAQk5B,YAAc,CACzBtQ,KAAM5oB,EAAQk5B,YACdC,KAAMn5B,EAAQk5B,aAGsB,WAA/B5iB,GAAOtW,EAAQk5B,eAClB,SAAUl5B,EAAQk5B,mBACfl5B,QAAQk5B,YAAYtQ,KAAO5oB,EAAQk5B,YAAYtQ,MAElD,SAAU5oB,EAAQk5B,mBACfl5B,QAAQk5B,YAAYC,KAAOn5B,EAAQk5B,YAAYC,QAKpB,SAAlCtoD,KAAKmvB,QAAQk5B,YAAYC,UACtBtoD,KAAKs0D,UAAW,KACbA,EAAYt0D,KAAKs0D,UAAY,IAAIvM,GAAS/nD,KAAK44C,MACrD0b,EAAU9V,WAAa,SAAArvB,OACf20B,EAAW30B,EAAUmpB,GAAKh7C,OAAO,GAAI6xB,GAAW,GACtD20B,EAASuE,YAAc,MACvBN,GAAS5rD,UAAUqiD,WAAWniD,KAAKi4D,EAAWxQ,SAE3CyQ,WAAWt3D,KAAKq3D,YAInBt0D,KAAKs0D,UAAW,SACZ1vC,EAAQgvC,UAAKW,mBAAmBv0D,KAAKs0D,eAC5B,IAAX1vC,YACG2vC,mBAAkB3vC,EAAO,QAE3B0vC,UAAUvP,eACVuP,UAAY,KAKY,mBAAtBnlC,EAAQqlC,aACjBrlC,EAAQqlC,WAAa,CACjBC,SAAUtlC,EAAQqlC,aAIpB,gBAAiBx0D,KAAKmvB,SACxBmxB,GAA8BtgD,KAAKmvB,QAAQ/E,OAAQpqB,KAAK44C,KAAM54C,KAAKmvB,QAAQiqB,aAGzE,eAAgBjqB,IACdA,EAAQulC,WACL10D,KAAK20D,iBACHA,UAAY,IAAIxH,GAAUntD,KAAKo+C,IAAI+C,OAItCnhD,KAAK20D,iBACFA,UAAU5P,iBACR/kD,KAAK20D,iBAMbC,+BAIFL,oBAAmB,SAAAM,UAAaA,EAAUrW,WAAWrvB,MAGtD,cAAeA,EAAS,OACrBnvB,KAAK80D,oBACHA,aAAe90D,KAAK+0D,4BAGtBD,aAAatW,WAAWrvB,EAAQ6lC,eAG/BC,EAAiB3c,GAAKuN,WAAW,GAAI7lD,KAAKmvB,mBAC3ColC,oBAAmB,SAAAM,GACtBvc,GAAKuN,WAAWoP,EAAgBJ,EAAU1lC,iBAEvC2lC,aAAaI,iBAAiB,CAAChpC,OAAQ+oC,SAGzClD,oDAQG/xD,KAAK20D,WAAa30D,KAAK20D,UAAUvH,mDA8BpC,IAAM5M,UAtBN2U,SAAS,WACTC,UAAU,WAGV7e,WAGA8e,kBAGDr1D,KAAKo+C,IAAI+C,KAAK5Z,iBACX6W,IAAI+C,KAAK5Z,WAAWC,YAAYxnC,KAAKo+C,IAAI+C,WAE3C/C,IAAM,KAGPp+C,KAAK20D,iBACFA,UAAU5P,iBACR/kD,KAAK20D,WAIM30D,KAAKyjD,UACnBzjD,KAAKyjD,UAAUpmD,eAAemjD,WACzBxgD,KAAKyjD,UAAUjD,QAGrBiD,UAAY,UACZI,QAAU7jD,KAAK6jD,OAAOkB,eACtBlB,OAAS,eAGT0Q,oBAAmB,SAAAM,UAAaA,EAAU9P,kBAE1CnM,KAAO,2CAQAl4B,EAAM0S,SACZghC,EAActd,UAAKsd,qBAAmB,SAAAS,UAAazhC,IAAOyhC,EAAU1lC,QAAQiE,SAEvD,IAAvBghC,EAAYp3D,aACR,IAAIoG,iDAA0C28C,GAAe3sB,KAGjEghC,EAAYp3D,OAAS,GACvBo3D,EAAY,GAAGpD,cAActwC,yCASnB0S,SACNghC,EAActd,UAAKsd,qBAAmB,SAAAS,UAAaA,EAAU1lC,QAAQiE,KAAOA,QAEvD,IAAvBghC,EAAYp3D,aACR,IAAIoG,iDAA0C28C,GAAe3sB,YAE9DghC,EAAY,GAAGkB,4DASJ7F,EAAOr8B,EAAIk9B,SACvB8D,EAActd,UAAKsd,qBAAmB,SAAAS,UAAaA,EAAU1lC,QAAQiE,KAAOA,QAEvD,IAAvBghC,EAAYp3D,aACR,IAAIoG,iDAA0C28C,GAAe3sB,KAEjEghC,EAAYp3D,OAAS,GACvBo3D,EAAY,GAAGmB,gBAAgB9F,EAAOa,8CAUvBb,EAAOr8B,SAClBghC,EAActd,UAAKsd,qBAAmB,SAAAS,UAAaA,EAAU1lC,QAAQiE,KAAOA,QAEvD,IAAvBghC,EAAYp3D,aACR,IAAIoG,iDAA0C28C,GAAe3sB,QAEjEghC,EAAYp3D,OAAS,SAChBo3D,EAAY,GAAGoB,eAAe/F,8CAUtBjP,SACV,CAAEA,MAAAA,yCAaG9/B,EAAM0S,SACZqiC,OAAqBh2D,IAATihB,EACZ43B,GAAKpB,QAAQx2B,EAAM,QACnB,IAAIhkB,QAEKg5D,UAAKtB,qBAAiB,SAAAzE,UAAcA,EAAWxgC,QAAQiE,KAAOA,WAErE,IAAIhwB,sCAA+B28C,GAAe3sB,2BAGpDu8B,EAAa,IAAIH,GAAWxvD,KAAK44C,KAAMN,GAAKh7C,OAAO,GAAI0C,KAAKmvB,QAAS,CACzEzO,KAAO+0C,EACPriC,GAAAA,EACA09B,KAAM9wD,KAAKmyD,QAAQhjC,QAAQ2hC,oBAGxBsD,YAAYn3D,KAAK0yD,QACjB4E,WAAWt3D,KAAK0yD,QAChBoC,UAEE3+B,2CAQQA,gBACTghC,EAActd,UAAKsd,qBAAmB,SAAAvE,UAAOA,EAAI1gC,QAAQiE,KAAOA,QAE3C,IAAvBghC,EAAYp3D,aACR,IAAIoG,iDAA0C28C,GAAe3sB,QAGrEghC,QAAAA,GAAoB,SAAAzE,oBAClBgG,EAAKvB,oBAAmBR,KAAA+B,EAAKvB,oBAAoBzE,GAAa,QAC9DgG,EAAKpB,mBAAkBX,KAAA+B,EAAKpB,mBAAmB5E,GAAa,GAC5DA,EAAW5K,8DASN/kD,KAAKmyD,SAAWnyD,KAAKmyD,QAAQyD,mBAAqB,qDAQlD51D,KAAKmyD,SAAWnyD,KAAKmyD,QAAQ0D,oBAAsB,+BAcxD1mC,EAASnpB,OACL0zC,EAAQ15C,KAAK81D,kBAGD,OAAdpc,EAAMt3C,KAA8B,OAAds3C,EAAMp0C,SAK1Bu5C,EAAWnF,EAAMp0C,IAAMo0C,EAAMt3C,IAC7BA,EAAM,IAAI1F,KAAKg9C,EAAMt3C,IAAI7E,UAAuB,IAAXshD,GACrCv5C,EAAM,IAAI5I,KAAKg9C,EAAMp0C,IAAI/H,UAAuB,IAAXshD,GACrCE,GAAa5vB,QAAiC1vB,IAAtB0vB,EAAQ4vB,WAA2B5vB,EAAQ4vB,eACpErF,MAAMgF,SAASt8C,EAAKkD,EAAK,CAAEy5C,UAAAA,GAAa/4C,iDAUvC,IAAI5C,MAAM,gFAyBRo9B,EAAOC,EAAKtR,EAASnpB,OAKzB+4C,EACArF,EALuB,mBAAhB59C,UAAU,KACnBkK,EAAWlK,UAAU,GACrBqzB,EAAU,IAIY,GAApBrzB,UAAUkB,QAEZ+hD,OAAiCt/C,KADjCi6C,EAAQ59C,UAAU,IACCijD,WAA2BrF,EAAMqF,eAC/CrF,MAAMgF,SAAShF,EAAMlZ,MAAOkZ,EAAMjZ,IAAK,CAAEse,UAAAA,KAEnB,GAApBjjD,UAAUkB,QAAsC,mBAAhBlB,UAAU,IACjD49C,EAAQ59C,UAAU,GAClBkK,EAAWlK,UAAU,GACrBijD,OAAiCt/C,IAApBi6C,EAAMqF,WAA2BrF,EAAMqF,eAC/CrF,MAAMgF,SAAShF,EAAMlZ,MAAOkZ,EAAMjZ,IAAK,CAAEse,UAAAA,GAAa/4C,KAG3D+4C,GAAa5vB,QAAiC1vB,IAAtB0vB,EAAQ4vB,WAA2B5vB,EAAQ4vB,eAC9DrF,MAAMgF,SAASle,EAAOC,EAAK,CAAEse,UAAAA,GAAa/4C,mCAgB5C0a,EAAMyO,EAASnpB,GACO,mBAAhBlK,UAAU,KACnBkK,EAAWlK,UAAU,GACrBqzB,EAAU,QAEN0vB,EAAW7+C,KAAK05C,MAAMjZ,IAAMzgC,KAAK05C,MAAMlZ,MACvCzgC,EAAIu4C,GAAKpB,QAAQx2B,EAAM,QAAQnjB,UAE/BijC,EAAQzgC,EAAI8+C,EAAW,EACvBpe,EAAM1gC,EAAI8+C,EAAW,EACrBE,GAAa5vB,QAAiC1vB,IAAtB0vB,EAAQ4vB,WAA2B5vB,EAAQ4vB,eAEpErF,MAAMgF,SAASle,EAAOC,EAAK,CAAEse,UAAAA,GAAa/4C,2CAQzC0zC,EAAQ15C,KAAK05C,MAAMqc,iBAClB,CACLv1B,MAAO,IAAI9jC,KAAKg9C,EAAMlZ,OACtBC,IAAK,IAAI/jC,KAAKg9C,EAAMjZ,qCAgBjBu1B,EAAY7mC,EAASnpB,SACrBgwD,GAAcA,EAAa,GAAKA,EAAa,IACvB,mBAAhBl6D,UAAU,KACnBkK,EAAWlK,UAAU,GACrBqzB,EAAU,QAENuqB,EAAQ15C,KAAKi2D,YACbz1B,EAAQkZ,EAAMlZ,MAAMjjC,UACpBkjC,EAAMiZ,EAAMjZ,IAAIljC,UAChBshD,EAAWpe,EAAMD,EACjB01B,EAAcrX,GAAY,EAAImX,GAC9BG,GAAYtX,EAAWqX,GAAe,EACtCtV,EAAWpgB,EAAQ21B,EACnBtV,EAASpgB,EAAM01B,OAEhBC,UAAUxV,EAAUC,EAAQ1xB,EAASnpB,oCAepCgwD,EAAY7mC,EAASnpB,SACtBgwD,GAAcA,EAAa,GAAKA,EAAa,IACvB,mBAAhBl6D,UAAU,KACnBkK,EAAWlK,UAAU,GACrBqzB,EAAU,QAENuqB,EAAQ15C,KAAKi2D,YACbz1B,EAAQkZ,EAAMlZ,MAAMjjC,UACpBkjC,EAAMiZ,EAAMjZ,IAAIljC,UAChBshD,EAAWpe,EAAMD,EACjBogB,EAAWpgB,EAAQqe,EAAWmX,EAAa,EAC3CnV,EAASpgB,EAAMoe,EAAWmX,EAAa,OAExCI,UAAUxV,EAAUC,EAAQ1xB,EAASnpB,0CASrC+rD,uDASAsC,kBACCjW,EAAMp+C,KAAKo+C,OAEZA,GAAQA,EAAI6N,WAAqC,GAAxB7N,EAAI+C,KAAKiI,iBAEnCvQ,GAAU,EACR1pB,EAAUnvB,KAAKmvB,QACfkmB,EAAQr1C,KAAKq1C,MAEnBiL,GAA2BtgD,KAAKmvB,QAAQ/E,OAAQpqB,KAAK44C,KAAM54C,KAAKmvB,QAAQiqB,aAG7C,OAAvBjqB,EAAQk5B,aACV/P,GAAKuV,aAAazP,EAAI+C,KAAM,WAC5B7I,GAAKwV,gBAAgB1P,EAAI+C,KAAM,gBAG/B7I,GAAKwV,gBAAgB1P,EAAI+C,KAAM,WAC/B7I,GAAKuV,aAAazP,EAAI+C,KAAM,eAG1BhyB,EAAQguB,KACV7E,GAAKuV,aAAazP,EAAI+C,KAAM,WAC5B7I,GAAKwV,gBAAgB1P,EAAI+C,KAAM,aAE/B7I,GAAKuV,aAAazP,EAAI+C,KAAM,WAC5B7I,GAAKwV,gBAAgB1P,EAAI+C,KAAM,YAIjC/C,EAAI+C,KAAK3sB,MAAM6hC,UAAY/d,GAAKge,OAAOC,OAAOpnC,EAAQknC,UAAW,IACjEjY,EAAI+C,KAAK3sB,MAAMgiC,UAAYle,GAAKge,OAAOC,OAAOpnC,EAAQqnC,UAAW,IACjEpY,EAAI+C,KAAK3sB,MAAMoU,MAAQ0P,GAAKge,OAAOC,OAAOpnC,EAAQyZ,MAAO,QACnD6tB,EAAkBrY,EAAI+C,KAAKiI,YAGjC/T,EAAMsc,OAAOjd,KAAO,EACpBW,EAAMsc,OAAOhd,MAAQ,EACrBU,EAAMsc,OAAO7O,IAAM,EACnBzN,EAAMsc,OAAO9I,OAAS,EAItBxT,EAAM2J,OAAOnW,OAASuV,EAAIY,OAAO0X,aACjCrhB,EAAMX,KAAK7L,OAAWuV,EAAI1J,KAAKgiB,aAC/BrhB,EAAMV,MAAM9L,OAAUuV,EAAIzJ,MAAM+hB,aAChCrhB,EAAMyN,IAAIja,OAAYuV,EAAI0E,IAAIiJ,eAAoB1W,EAAMsc,OAAO7O,IAC/DzN,EAAMwT,OAAOhgB,OAASuV,EAAIyK,OAAOkD,eAAiB1W,EAAMsc,OAAO9I,WAMzD8N,EAAgBn1D,KAAK8D,IAAI+vC,EAAMX,KAAK7L,OAAQwM,EAAM2J,OAAOnW,OAAQwM,EAAMV,MAAM9L,QAC7E+tB,EAAavhB,EAAMyN,IAAIja,OAAS8tB,EAAgBthB,EAAMwT,OAAOhgB,OAASwM,EAAMsc,OAAO7O,IAAMzN,EAAMsc,OAAO9I,OAC5GzK,EAAI+C,KAAK3sB,MAAMqU,OAASyP,GAAKge,OAAOC,OAAOpnC,EAAQ0Z,iBAAW+tB,SAG9DvhB,EAAM8L,KAAKtY,OAASuV,EAAI+C,KAAKuV,aAC7BrhB,EAAMuT,WAAW/f,OAASwM,EAAM8L,KAAKtY,WAC/BguB,EAAkBxhB,EAAM8L,KAAKtY,OAASwM,EAAMyN,IAAIja,OAASwM,EAAMwT,OAAOhgB,OAC5EwM,EAAMoE,gBAAgB5Q,OAAUguB,EAChCxhB,EAAM6b,cAAcroB,OAAYguB,EAChCxhB,EAAM8b,eAAetoB,OAAWwM,EAAM6b,cAAcroB,OAGpDwM,EAAM8L,KAAKvY,MAAQ6tB,EACnBphB,EAAMuT,WAAWhgB,MAAQyM,EAAM8L,KAAKvY,MAE/B5oC,KAAK8xD,kBACRzc,EAAMyhB,eAAiBxe,GAAKye,yBAGxBC,EAA2B5Y,EAAI8S,cAAcvQ,YAC7CsW,EAA4B7Y,EAAI+S,eAAexQ,YAEjDxxB,EAAQ4jC,eACN5jC,EAAQguB,KACV9H,EAAMX,KAAK9L,MAAQouB,IAA6B3hB,EAAMsc,OAAOjd,KAC7DW,EAAMV,MAAM/L,MAAQquB,EAA4B5hB,EAAMyhB,iBAAmBzhB,EAAMsc,OAAOhd,QAEtFU,EAAMX,KAAK9L,MAAQouB,EAA2B3hB,EAAMyhB,iBAAmBzhB,EAAMsc,OAAOjd,KACpFW,EAAMV,MAAM/L,MAAQquB,IAA8B5hB,EAAMsc,OAAOhd,QAGjEU,EAAMX,KAAK9L,MAAQouB,IAA8B3hB,EAAMsc,OAAOjd,KAC9DW,EAAMV,MAAM/L,MAAQquB,IAA8B5hB,EAAMsc,OAAOhd,YAG5DuiB,cAID16C,EAASxc,KAAKm3D,mBAGc,OAA5BhoC,EAAQk5B,YAAYtQ,OACtBv7B,GAAUhb,KAAK8D,IAAI+vC,EAAMoE,gBAAgB5Q,OAASwM,EAAM2J,OAAOnW,OAC7DwM,EAAMsc,OAAO7O,IAAMzN,EAAMsc,OAAO9I,OAAQ,IAE5CzK,EAAIY,OAAOxqB,MAAMkiB,+BAA2Bl6B,aAGtC46C,EAAmC,GAAnB/hB,EAAMuc,UAAiB,SAAW,GAClDyF,EAAmBhiB,EAAMuc,WAAavc,EAAMwc,aAAe,SAAW,GAC5EzT,EAAIgT,UAAU58B,MAAMyqB,WAAsBmY,EAC1ChZ,EAAIiT,aAAa78B,MAAMyqB,WAAmBoY,EAC1CjZ,EAAIkT,cAAc98B,MAAMyqB,WAAkBmY,EAC1ChZ,EAAImT,iBAAiB/8B,MAAMyqB,WAAeoY,EAC1CjZ,EAAIoT,eAAeh9B,MAAMyqB,WAAiBmY,EAC1ChZ,EAAIqT,kBAAkBj9B,MAAMyqB,WAAcoY,EAEtCloC,EAAQ4jC,iBACV3U,EAAI+S,eAAe7oB,UAAY,0CAC/B8V,EAAI8S,cAAc5oB,UAAY,yCAE9B8V,EAAIoT,eAAeh9B,MAAMyqB,WAAiB,SAC1Cb,EAAIqT,kBAAkBj9B,MAAMyqB,WAAc,SAC1Cb,EAAIkT,cAAc98B,MAAMyqB,WAAkB,SAC1Cb,EAAImT,iBAAiB/8B,MAAMyqB,WAAe,SAE1Cb,EAAI1J,KAAKlgB,MAAMsuB,IAAS,MACxB1E,EAAIzJ,MAAMngB,MAAMsuB,IAAQ,SAGrB3zB,EAAQ4jC,gBAAmB1d,EAAM2J,OAAOnW,OAASwM,EAAMoE,gBAAgB5Q,UAC1EuV,EAAI1J,KAAKlgB,MAAMsuB,cAAYtmC,QAC3B4hC,EAAIzJ,MAAMngB,MAAMsuB,cAAWtmC,QAC3B4hC,EAAI+S,eAAe7oB,UAAY8V,EAAI+S,eAAe7oB,UAAU/hC,QAAQ,IAAI+B,OAAO,yCAAmD,KAClI81C,EAAI8S,cAAc5oB,UAAY8V,EAAI8S,cAAc5oB,UAAU/hC,QAAQ,IAAI+B,OAAO,yCAAmD,KAChI+sC,EAAMX,KAAK9L,MAAQouB,IAA8B3hB,EAAMsc,OAAOjd,KAC9DW,EAAMV,MAAM/L,MAAQquB,IAA8B5hB,EAAMsc,OAAOhd,WAC1DuiB,eAIDI,EAAmBjiB,EAAM2J,OAAOnW,OAASwM,EAAMoE,gBAAgB5Q,YAChEgb,OAAO95C,IAAI,OAAO/F,IAAI,CACzBsa,UAAWg5C,EAAmBtT,GAAOiM,cAAgBjM,GAAOuT,4BAIzD1T,OAAO95C,IAAI,SAAS/F,IAAI,CAC3B0c,KAAM1gB,KAAKmvB,QAAQqoC,gCAIhBjD,oBAAmB,SAAAM,GACtBhc,EAAUgc,EAAUxE,UAAYxX,QAG9BA,EAAS,IACP74C,KAAKq0D,YAFQ,mBAGVzb,KAAKiF,QAAQ4C,KAAK,WAIvB99C,QAAQq3C,IAAI,+CAGTqa,YAAc,OAIhBzb,KAAKiF,QAAQ4C,KAAK,kDAOjBpL,EAAQr1C,KAAKq1C,MACb+I,EAAMp+C,KAAKo+C,IAEjB/I,EAAM6b,cAActoB,MAAQyM,EAAMX,KAAK9L,MACvCyM,EAAM8b,eAAevoB,MAAQyM,EAAMV,MAAM/L,UACnC6uB,EAAcpiB,EAAM8L,KAAKvY,MAAQyM,EAAMX,KAAK9L,MAAQyM,EAAMV,MAAM/L,MACtEyM,EAAM2J,OAAOpW,MAAiB6uB,EAC9BpiB,EAAMoE,gBAAgB7Q,MAAQ6uB,EAC9BpiB,EAAMyN,IAAIla,MAAoB6uB,EAC9BpiB,EAAMwT,OAAOjgB,MAAiB6uB,EAG9BrZ,EAAIwK,WAAWp0B,MAAMqU,iBAAsBwM,EAAMuT,WAAW/f,aAC5DuV,EAAIyL,mBAAmBr1B,MAAMqU,iBAAcwM,EAAMuT,WAAW/f,aAC5DuV,EAAI6S,qBAAqBz8B,MAAMqU,iBAAYwM,EAAMoE,gBAAgB5Q,aACjEuV,EAAI3E,gBAAgBjlB,MAAMqU,iBAAiBwM,EAAMoE,gBAAgB5Q,aACjEuV,EAAI8S,cAAc18B,MAAMqU,iBAAmBwM,EAAM6b,cAAcroB,aAC/DuV,EAAI+S,eAAe38B,MAAMqU,iBAAkBwM,EAAM8b,eAAetoB,aAEhEuV,EAAIwK,WAAWp0B,MAAMoU,gBAAsByM,EAAMuT,WAAWhgB,YAC5DwV,EAAIyL,mBAAmBr1B,MAAMoU,gBAAcyM,EAAMoE,gBAAgB7Q,YACjEwV,EAAI6S,qBAAqBz8B,MAAMoU,gBAAYyM,EAAMuT,WAAWhgB,YAC5DwV,EAAI3E,gBAAgBjlB,MAAMoU,gBAAiByM,EAAM2J,OAAOpW,YACxDwV,EAAI0E,IAAItuB,MAAMoU,gBAA6ByM,EAAMyN,IAAIla,YACrDwV,EAAIyK,OAAOr0B,MAAMoU,gBAA0ByM,EAAMwT,OAAOjgB,YAGxDwV,EAAIwK,WAAWp0B,MAAMkgB,KAAiB,IACtC0J,EAAIwK,WAAWp0B,MAAMsuB,IAAiB,IACtC1E,EAAIyL,mBAAmBr1B,MAAMkgB,eAAYW,EAAMX,KAAK9L,MAAQyM,EAAMsc,OAAOjd,WACzE0J,EAAIyL,mBAAmBr1B,MAAMsuB,IAAS,IACtC1E,EAAI6S,qBAAqBz8B,MAAMkgB,KAAO,IACtC0J,EAAI6S,qBAAqBz8B,MAAMsuB,cAAUzN,EAAMyN,IAAIja,aACnDuV,EAAI3E,gBAAgBjlB,MAAMkgB,eAAeW,EAAMX,KAAK9L,YACpDwV,EAAI3E,gBAAgBjlB,MAAMsuB,cAAezN,EAAMyN,IAAIja,aACnDuV,EAAI8S,cAAc18B,MAAMkgB,KAAc,IACtC0J,EAAI8S,cAAc18B,MAAMsuB,cAAiBzN,EAAMyN,IAAIja,aACnDuV,EAAI+S,eAAe38B,MAAMkgB,eAAgBW,EAAMX,KAAK9L,MAAQyM,EAAM2J,OAAOpW,YACzEwV,EAAI+S,eAAe38B,MAAMsuB,cAAgBzN,EAAMyN,IAAIja,aACnDuV,EAAI0E,IAAItuB,MAAMkgB,eAA2BW,EAAMX,KAAK9L,YACpDwV,EAAI0E,IAAItuB,MAAMsuB,IAAwB,IACtC1E,EAAIyK,OAAOr0B,MAAMkgB,eAAwBW,EAAMX,KAAK9L,YACpDwV,EAAIyK,OAAOr0B,MAAMsuB,cAAwBzN,EAAMyN,IAAIja,OAASwM,EAAMoE,gBAAgB5Q,aAClFuV,EAAIY,OAAOxqB,MAAMkgB,KAAqB,IACtC0J,EAAI1J,KAAKlgB,MAAMkgB,KAAuB,IACtC0J,EAAIzJ,MAAMngB,MAAMkgB,KAAsB,2CAUzBh0B,OACR1gB,KAAK03D,kBACF,IAAIt0D,MAAM,4CAGbs0D,YAAYC,eAAej3C,gDAS3B1gB,KAAK03D,kBACF,IAAIt0D,MAAM,8CAGXpD,KAAK03D,YAAYE,iDAUlBztD,UACCm2C,GAAgBtgD,KAAMmK,EAAGnK,KAAKq1C,MAAM2J,OAAOpW,6CAUtCz+B,UACLm2C,GAAgBtgD,KAAMmK,EAAGnK,KAAKq1C,MAAM8L,KAAKvY,yCAaxCloB,UACD4/B,GAAkBtgD,KAAM0gB,EAAM1gB,KAAKq1C,MAAM2J,OAAOpW,+CAYzCloB,UACP4/B,GAAkBtgD,KAAM0gB,EAAM1gB,KAAKq1C,MAAM8L,KAAKvY,iDAUtB,GAA3B5oC,KAAKmvB,QAAQ0oC,gBACVC,wBAGAzC,iEAUD1W,EAAK3+C,UAENq1D,uBAEA0C,UAAY,cACc,GAAzBpZ,EAAGxvB,QAAQ0oC,eAMXlZ,EAAGP,IAAI+C,KAAM,KACT6W,EAAmBrZ,EAAGP,IAAI+C,KAAKuV,aAC/BD,EAAkB9X,EAAGP,IAAI+C,KAAKiI,YAK/BqN,GAAmB9X,EAAGtJ,MAAM4iB,WAC9BD,GAAoBrZ,EAAGtJ,MAAM6iB,aAC9BvZ,EAAGtJ,MAAM4iB,UAAYxB,EACrB9X,EAAGtJ,MAAM6iB,WAAaF,EACtBrZ,EAAGtJ,MAAMyhB,eAAiBxe,GAAKye,oBAE/BpY,EAAG/F,KAAKiF,QAAQ4C,KAAK,kBAjBvB9B,EAAG0W,mBAuBP/c,GAAKgG,iBAAiBtyB,OAAQ,SAAUhsB,KAAK+3D,WAGzCpZ,EAAGP,IAAI+C,OACPxC,EAAGtJ,MAAM4iB,UAAYtZ,EAAGP,IAAI+C,KAAKiI,YACjCzK,EAAGtJ,MAAM6iB,WAAavZ,EAAGP,IAAI+C,KAAKuV,mBAGjCyB,WAAaC,GAAYp4D,KAAK+3D,UAAW,+CAQ1C/3D,KAAKm4D,aACPE,cAAcr4D,KAAKm4D,iBACdA,gBAAa14D,GAIhBO,KAAK+3D,YACPzf,GAAKgL,oBAAoBt3B,OAAQ,SAAUhsB,KAAK+3D,gBAC3CA,UAAY,uCASZvX,QACF7C,MAAMuD,eAAgB,OACtBvD,MAAM2a,iBAAmBt4D,KAAKq1C,MAAMuc,2CAQlCpR,QACF7C,MAAMuD,eAAgB,kCAQrBV,MACDA,GAGAxgD,KAAK29C,MAAMuD,mBAEVI,EAAQd,EAAMgB,OAEd+W,EAAev4D,KAAKw4D,gBACpBC,EAAez4D,KAAKkzD,cAAclzD,KAAK29C,MAAM2a,iBAAmBhX,GAElEthD,KAAKmvB,QAAQ4jC,sBACV3U,IAAI1J,KAAKnN,WAAWqqB,WAAa5xD,KAAKq1C,MAAMuc,eAC5CxT,IAAIzJ,MAAMpN,WAAWqqB,WAAa5xD,KAAKq1C,MAAMuc,WAGhD6G,GAAgBF,QACb9X,KAAK,uDAUAmR,eACPvc,MAAMuc,UAAYA,OAClBuF,mBACEn3D,KAAKq1C,MAAMuc,yDAUZC,EAAerwD,KAAKY,IAAIpC,KAAKq1C,MAAMoE,gBAAgB5Q,OAAS7oC,KAAKq1C,MAAM2J,OAAOnW,OAAQ,UACxFgpB,GAAgB7xD,KAAKq1C,MAAMwc,eAGQ,OAAjC7xD,KAAKmvB,QAAQk5B,YAAYtQ,YACtB1C,MAAMuc,WAAcC,EAAe7xD,KAAKq1C,MAAMwc,mBAEhDxc,MAAMwc,aAAeA,GAIxB7xD,KAAKq1C,MAAMuc,UAAY,IAAG5xD,KAAKq1C,MAAMuc,UAAY,GACjD5xD,KAAKq1C,MAAMuc,UAAYC,IAAc7xD,KAAKq1C,MAAMuc,UAAYC,GAE5D7xD,KAAKmvB,QAAQ4jC,sBACV3U,IAAI1J,KAAKnN,WAAWqqB,WAAa5xD,KAAKq1C,MAAMuc,eAC5CxT,IAAIzJ,MAAMpN,WAAWqqB,WAAa5xD,KAAKq1C,MAAMuc,gBAE/C4B,4BAA6B,EAC3BxzD,KAAKq1C,MAAMuc,yDASX5xD,KAAKq1C,MAAMuc,8DASZ,IAAIxuD,MAAM,iEAKZq4C,GAAKt/C,eCz2CPu8D,0BASQ9f,EAAMzpB,oDAEXypB,KAAOA,IAGPsE,eAAiB,CACpBC,KAAK,EACLwb,iBAAiB,EACjBC,sBAAkBn5D,EAElB2qB,OAAAA,GACAxX,QAAAA,GACAlV,OAAQ,QAELyxB,QAAUmpB,GAAKh7C,OAAO,GAAIogD,EAAKR,kBAC/BsB,WAAWrvB,KACXA,QAAQvc,QAAU0lC,GAAKh7C,OAAO,GAAIsV,GAAS8qC,EAAKvuB,QAAQvc,aACvD88C,EAAiBhS,EAAKR,eAAetqC,QAAQ8qC,EAAKR,eAAex/C,uBAC3DggD,EAAKvuB,QAAQvc,kBAAiB,SAAAlV,KACnCyxB,QAAQvc,QAAQlV,GAAU46C,GAAKh7C,OAClC,GACAoyD,EACAhS,EAAKvuB,QAAQvc,QAAQlV,SAGpB8e,OAAS,IAETksC,qEASCmH,EAAMziC,SAASE,cAAc,OACnCuiC,EAAIvnB,UAAY,mBAChBunB,EAAIr7B,MAAMkL,SAAW,WACrBmwB,EAAIr7B,MAAMsuB,IAAM,MAChB+M,EAAIr7B,MAAMqU,OAAS,YAEdgnB,IAAMA,yCAON1gC,QAAQwpC,iBAAkB,OAC1BtI,cAEAzX,KAAO,wCASHzpB,GACLA,GAEFmpB,GAAKmG,gBAAgB,CAAC,MAAO,kBAAmB,mBAAoB,SAAU,SAAU,WAAYz+C,KAAKmvB,QAASA,uCAShHnvB,KAAKmvB,QAAQwpC,gBAAiB,SAC1B/nC,EAAS5wB,KAAK44C,KAAKwF,IAAIyL,mBACzB7pD,KAAK6vD,IAAItoB,YAAc3W,IAErB5wB,KAAK6vD,IAAItoB,iBACNsoB,IAAItoB,WAAWC,YAAYxnC,KAAK6vD,KAEvCj/B,EAAO+D,YAAY30B,KAAK6vD,UAEnBrvB,aAGH3rB,EAAM7U,KAAKmvB,QAAQ/E,OAAO4c,KAAahnC,KAAKwc,QAE5Cxc,KAAKmvB,QAAQypC,mBACf/jD,EAAMA,EAAIgM,QAAQ7gB,KAAKmvB,QAAQypC,uBAG3BzuD,EAAInK,KAAK44C,KAAKN,KAAKkD,SAAS3mC,GAE9BnX,EAASsC,KAAKmvB,QAAQvc,QAAQ5S,KAAKmvB,QAAQzxB,QAC1CA,IACEsC,KAAKmwD,SACRxtD,QAAQH,yCAAkCxC,KAAKmvB,QAAQzxB,mGAClDyyD,QAAS,GAEhBzyD,EAASsC,KAAKmvB,QAAQvc,QAAb,QAEP68C,sBAAW/xD,EAAOu9C,qBAAWv9C,EAAOgjB,mBAAS7L,EAAIpX,OAAO,gCAC5DgyD,EAAQA,EAAMjpC,OAAO,GAAGjB,cAAgBkqC,EAAMW,UAAU,GAEpDpwD,KAAKmvB,QAAQguB,SACV0S,IAAIr7B,MAAMkiB,gCAA+B,EAALvsC,cAEpC0lD,IAAIr7B,MAAMkiB,+BAA0BvsC,cAEtC0lD,IAAIJ,MAAQA,OAIbzvD,KAAK6vD,IAAItoB,iBACNsoB,IAAItoB,WAAWC,YAAYxnC,KAAK6vD,UAElClkB,cAGA,sCAODgT,EAAK3+C,eAKAm2C,IACTwI,EAAGhT,WAICkT,EAAW,EADDF,EAAG/F,KAAKc,MAAMgC,WAAWiD,EAAG/F,KAAKY,SAASwF,OAAOpW,OAAO+S,MAC3C,GACvBkD,EAAW,KAAMA,EAAW,IAC5BA,EAAW,MAAMA,EAAW,KAEhCF,EAAG0R,SACH1R,EAAG/F,KAAKiF,QAAQ4C,KAAK,mBAGrB9B,EAAGO,iBAAmBC,GAAWhJ,EAAQ0I,GAG3C1I,uCAO8B12C,IAA1BO,KAAKk/C,mBACPE,aAAap/C,KAAKk/C,yBACXl/C,KAAKk/C,yDAUDx+B,OACP3gB,EAAIu4C,GAAKpB,QAAQx2B,EAAM,QAAQnjB,UAC/BsX,EAAMmyB,UACPxqB,OAASzc,EAAI8U,OACbw7C,yDAQE,IAAI3zD,KAAKsqC,KAAahnC,KAAKwc,eA3LZm8B,ICNtBkgB,GAAQruC,GAAwC2N,KAKhD2gC,IAAc,EAEdv3B,GAAiBC,GAHV,QAAA,QAMC,IAAIvlC,MAAM,GAAN,MAAe,WAAc68D,IAAc,OAIzD,CAAEnpC,OAAQ,QAASnP,OAAO,EAAM2P,OAAQ2oC,KAAgBv3B,IAAkB,CAC1EpJ,KAAM,SAAcN,UACXghC,GAAM74D,KAAM63B,EAAY/7B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCfzE,OAAiBohC,GAAa,SAAS1I,KCDnCsJ,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGqM,YACNrM,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAetJ,KAAQA,GAAO4I,GCD5Fg4B,GAAmC,oBAAVC,QAAyBA,OAAOD,iBAAmBC,OAAOD,gBAAgB1oC,KAAK2oC,SAA8B,oBAAZC,UAA8D,mBAA5BA,SAASF,iBAAiCE,SAASF,gBAAgB1oC,KAAK4oC,UACpOC,GAAQ,IAAIC,WAAW,IAEZ,SAASC,SACjBL,SACG,IAAI31D,MAAM,mHAGX21D,GAAgBG,ICPzB,IAFA,IAAIG,GAAY,GAEPv8D,GAAI,EAAGA,GAAI,MAAOA,GACzBu8D,GAAUv8D,KAAMA,GAAI,KAAOV,SAAS,IAAImJ,OAAO,GCJjD,SAAS+zD,GAAGnqC,EAASoqC,EAAK/8C,OACpB1f,EAAIy8D,GAAO/8C,GAAU,EAEH,iBAAX2S,IACToqC,EAAkB,WAAZpqC,EAAuB,IAAIlzB,MAAM,IAAM,KAC7CkzB,EAAU,UAIRqqC,GADJrqC,EAAUA,GAAW,IACFmE,SAAWnE,EAAQiqC,KAAOA,SAE7CI,EAAK,GAAe,GAAVA,EAAK,GAAY,GAC3BA,EAAK,GAAe,GAAVA,EAAK,GAAY,IAEvBD,MACG,IAAItuD,EAAK,EAAGA,EAAK,KAAMA,EAC1BsuD,EAAIz8D,EAAImO,GAAMuuD,EAAKvuD,UAIhBsuD,GDbT,SAAqBA,EAAK/8C,OACpB1f,EAAI0f,GAAU,EACdi9C,EAAMJ,SAEH,CAACI,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO,IAAK28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO,IAAK28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO,IAAK28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO,IAAK28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,OAAOqG,KAAK,ICSnQu2D,CAAYF,GCrB5B,IAAIG,GAAYnvC,GAAuC+G,SAInDgQ,GAAiBC,GAAwB,UAAW,CAAEN,WAAW,IAAS,MAI5E,CAAEvR,OAAQ,QAASnP,OAAO,EAAM2P,QAASoR,IAAkB,CAC3DhQ,SAAU,SAAkBJ,UACnBwoC,GAAU35D,KAAMmxB,EAAIr1B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCTrE,OAAiBohC,GAAa,SAAStP,SCCnCqoC,GAAQpkC,GAAgB,YCFX,SAAU1J,MDMV,SAAUA,OACrB+tC,SACGv9D,EAASwvB,UAAmCrsB,KAA1Bo6D,EAAW/tC,EAAG8tC,OAA0BC,EAA0B,UAAf9sC,EAAQjB,ICPhF+tC,CAAS/tC,SACLkB,UAAU,wDACTlB,GCHP8tC,GAAQpkC,GAAgB,WCM1B,CAAE7F,OAAQ,SAAUnP,OAAO,EAAM2P,QDJlB,SAAUgQ,OACrB25B,EAAS,cAEL35B,GAAa25B,GACnB,MAAOzmD,cAELymD,EAAOF,KAAS,EACT,MAAMz5B,GAAa25B,GAC1B,MAAO7rC,YACF,ECLiC8rC,CAAqB,aAAe,CAC9ExoC,SAAU,SAAkByoC,YAChBvrC,OAAOvB,EAAuBltB,OACrC0J,QAAQuwD,GAAWD,GAAel+D,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCR/E,OAAiBohC,GAAa,UAAUtP,SCApCkQ,GAAiBxlC,MAAME,UACvB2kC,GAAkBrS,OAAOtyB,aAEZ,SAAU2vB,OACrBiV,EAAMjV,EAAGyF,gBACTzF,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAelQ,SAAkB2oC,GAC5E,iBAAPpuC,GAAmBA,IAAOgV,IAAoBhV,aAAc2C,QAAUsS,IAAQD,GAAgBvP,SAChG4oC,GACAp5B,GCJJ,SAASq5B,GAAa5jB,MAC3BA,QAAAA,GAAW,SAACr5C,EAAGC,UAAMD,EAAEoW,KAAKitB,MAAQpjC,EAAEmW,KAAKitB,SAQtC,SAAS65B,GAAW7jB,MACzBA,QAAAA,GAAW,SAACr5C,EAAGC,UACE,QAASD,EAAEoW,KAAQpW,EAAEoW,KAAKktB,IAAMtjC,EAAEoW,KAAKitB,QACvC,QAASpjC,EAAEmW,KAAQnW,EAAEmW,KAAKktB,IAAMrjC,EAAEmW,KAAKitB,UAoBnD,SAASn9B,GAAMmzC,EAAO8jB,EAAQC,EAAOC,MACtCD,MAEG,IAAIz9D,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,IAChC05C,EAAM15C,GAAGgmD,IAAM,SAKVhmD,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,IAAK,KAC/Bi7C,EAAOvB,EAAM15C,MACfi7C,EAAK10C,OAAsB,OAAb00C,EAAK+K,IAAc,CAEnC/K,EAAK+K,IAAMwX,EAAOhS,SACdmS,GAAa,IAEd,SAGGC,EAAgB,KACX3mD,EAAI,EAAG4mD,EAAKnkB,EAAMx5C,OAAQ+W,EAAI4mD,EAAI5mD,IAAK,KACxCgH,EAAQy7B,EAAMziC,MACpB0mD,EAAaD,MAAmC,SAEvB,KAEP,OAAdz/C,EAAM+nC,KAAgB/nC,IAAUg9B,GAAQh9B,EAAM1X,OAASu3D,GAAU7iB,EAAMh9B,EAAOu/C,EAAOviB,KAAMh9B,EAAMoU,QAAQguB,KAAM,CACjHud,EAAgB3/C,SAKC,MAAjB2/C,IAEF3iB,EAAK+K,IAAM4X,EAAc5X,IAAM4X,EAAc7xB,OAASyxB,EAAOviB,KAAK8iB,gBAE7DH,WAGND,EAaF,SAASK,GAAStkB,EAAO8jB,EAAQS,OACjC,IAAIj+D,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,IAChC05C,EAAM15C,GAAGgmD,IAAM,SAIbkY,EAAiBD,EAASlyB,WAGzB/rC,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,IAAK,KAC3Bi7C,EAAOvB,EAAM15C,MAEfi7C,EAAK10C,OAAsB,OAAb00C,EAAK+K,IAAc,CAEnC/K,EAAK+K,IAAM/K,EAAKkjB,UAEb,SAGGP,EAAgB,KACX3mD,EAAI,EAAG4mD,EAAKnkB,EAAMx5C,OAAQ+W,EAAI4mD,EAAI5mD,IAAK,KACxCgH,EAAQy7B,EAAMziC,MACF,OAAdgH,EAAM+nC,KAAgB/nC,IAAUg9B,GAA2B6iB,GAAU7iB,EAAMh9B,EAAOu/C,EAAOviB,KAAMh9B,EAAMoU,QAAQguB,KAAM,CACrHud,EAAgB3/C,SAKC,MAAjB2/C,IAEF3iB,EAAK+K,IAAM4X,EAAc5X,IAAM4X,EAAc7xB,OAASyxB,EAAOviB,KAAK8iB,UAGhE9iB,EAAK+K,IAAM/K,EAAKlP,OAASmyB,IAC3BA,EAAiBjjB,EAAK+K,IAAM/K,EAAKlP,cAE5B6xB,IAKbK,EAASlyB,OAASmyB,EAAiBD,EAASjY,IAAM,GAAMwX,EAAOviB,KAAK8iB,SAa/D,SAASK,GAAQ1kB,EAAO8jB,EAAQa,EAAWC,OAC3C,IAAIt+D,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,OACF2C,MAA1B+2C,EAAM15C,GAAGyW,KAAKwnD,SAChBvkB,EAAM15C,GAAGgmD,IAAMwX,EAAOviB,KAAK8iB,cACtB,QAA+Bp7D,IAA3B+2C,EAAM15C,GAAGyW,KAAKwnD,UAA0BK,EAAkB,KAC/DC,EAAS,MACR,IAAMN,KAAYI,EACjBA,EAAU99D,eAAe09D,IACQ,GAA/BI,EAAUJ,GAAUO,SAAmBH,EAAUJ,GAAUn2C,MAAQu2C,EAAU3kB,EAAM15C,GAAGyW,KAAKwnD,UAAUn2C,QACvGy2C,GAAUF,EAAUJ,GAAUlyB,OAC9BsyB,EAAU3kB,EAAM15C,GAAGyW,KAAKwnD,UAAUjY,IAAMuY,GAI9C7kB,EAAM15C,GAAGgmD,IAAMuY,EAAS,GAAMf,EAAOviB,KAAK8iB,SAGzCO,GACHG,GAAe/kB,EAAO8jB,EAAQa,GAY3B,SAASI,GAAe/kB,EAAO8jB,EAAQa,OACvC,IAAMJ,KAAYI,KACjBA,EAAU99D,eAAe09D,GAAW,CAGtCI,EAAUJ,GAAUjY,IAAM,IACvB,KAGG4X,EAAgB,SACf,IAAMc,KAAiBL,KACW,OAAjCA,EAAUK,GAAe1Y,KAAgB0Y,IAAkBT,GAAYI,EAAUJ,GAAUn2C,MAAQu2C,EAAUK,GAAe52C,OAAS62C,GAAiBN,EAAUJ,GAAWI,EAAUK,IAAiB,CACxMd,EAAgBS,EAAUK,SAKT,MAAjBd,IAEFS,EAAUJ,GAAUjY,IAAM4X,EAAc5X,IAAM4X,EAAc7xB,cAEvD6xB,OAGR,IAAI59D,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,SACD2C,IAA3B+2C,EAAM15C,GAAGyW,KAAKwnD,WAChBvkB,EAAM15C,GAAGgmD,IAAMqY,EAAU3kB,EAAM15C,GAAGyW,KAAKwnD,UAAUjY,IAAM,GAAMwX,EAAOviB,KAAK8iB,UAexE,SAASa,GAA6BC,EAAerB,EAAQa,OAC9DS,GAAa,EAGXC,EAAgB,OAElB,IAAId,KAAYI,EACdA,EAAUJ,GAAU19D,eAAe,SACrCw+D,EAAcV,EAAUJ,GAAUn2C,OAASm2C,EAG3Cc,EAAc5+D,KAAK89D,OAInB,IAAIhnD,EAAI,EAAGA,EAAI8nD,EAAc7+D,OAAQ+W,OACvCgnD,EAAWc,EAAc9nD,GACrBonD,EAAU99D,eAAe09D,GAAW,KAKjC,IAAMS,KAHXI,EAAaA,GAAcT,EAAUJ,GAAU13D,MAC/C83D,EAAUJ,GAAUjY,IAAM,EAEEqY,EACtBA,EAAUK,GAAeF,SAAWH,EAAUJ,GAAUn2C,MAAQu2C,EAAUK,GAAe52C,QAC3Fu2C,EAAUJ,GAAUjY,KAAOqY,EAAUK,GAAe3yB,gBAIlD2N,EAAQmlB,EAAcZ,GACpBj+D,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,SACA2C,IAA3B+2C,EAAM15C,GAAGyW,KAAKwnD,WAChBvkB,EAAM15C,GAAGgmD,IAAMqY,EAAU3kB,EAAM15C,GAAGyW,KAAKwnD,UAAUjY,IAAM,GAAMwX,EAAOviB,KAAK8iB,SAErEM,EAAUJ,GAAU13D,QACtBmzC,EAAM15C,GAAGm+D,QAAUzkB,EAAM15C,GAAGgmD,MAK9B8Y,GAAcT,EAAUJ,GAAU13D,OACpCy3D,GAASa,EAAcZ,GAAWT,EAAQa,EAAUJ,KAiBrD,SAASH,GAAUz9D,EAAGC,EAAGk9D,EAAQnd,UAClCA,EACQhgD,EAAEw3C,MAAQ2lB,EAAOwB,WAlRf,KAkRyC1+D,EAAEu3C,MAAQv3C,EAAEwrC,OAChEzrC,EAAEw3C,MAAQx3C,EAAEyrC,MAAQ0xB,EAAOwB,WAnRhB,KAmRwC1+D,EAAEu3C,OACrDx3C,EAAE2lD,IAAMwX,EAAOO,SApRJ,KAoRwCz9D,EAAE0lD,IAAM1lD,EAAEyrC,QAC7D1rC,EAAE2lD,IAAM3lD,EAAE0rC,OAASyxB,EAAOO,SArRf,KAqRuCz9D,EAAE0lD,IAE5C3lD,EAAEu3C,KAAO4lB,EAAOwB,WAvRb,KAuRwC1+D,EAAEs3C,KAAOt3C,EAAEwrC,OAC9DzrC,EAAEu3C,KAAOv3C,EAAEyrC,MAAQ0xB,EAAOwB,WAxRf,KAwRuC1+D,EAAEs3C,MACpDv3C,EAAE2lD,IAAMwX,EAAOO,SAzRJ,KAyRwCz9D,EAAE0lD,IAAM1lD,EAAEyrC,QAC7D1rC,EAAE2lD,IAAM3lD,EAAE0rC,OAASyxB,EAAOO,SA1Rf,KA0RuCz9D,EAAE0lD,IAWlD,SAAS2Y,GAAiBt+D,EAAGC,OAI5B2+D,EAAc5+D,EAAEqjC,MAAQpjC,EAAEqjC,KAAOtjC,EAAEsjC,IAAMrjC,EAAEojC,MAC3Cw7B,EAAgB7+D,EAAE2lD,IAAO1lD,EAAE0lD,IAAM1lD,EAAEyrC,QAAY1rC,EAAE2lD,IAAM3lD,EAAE0rC,OAAUzrC,EAAE0lD,WACpEiZ,GAAeC,0LCtSXC,GAFM,iBAWbC,yBAOQC,EAAS5oD,EAAM4+C,iCACpBgK,QAAUA,OACVhB,UAAY,QACZiB,cAAgB,QAChBC,kBAAmB,OACnBC,mBAAqB,QACrBC,cAAe,OACfC,sBAAuB,OACvBC,cAAgB,OAChBC,gBAAkBnpD,GAAQA,EAAKsoD,mBAC/B1J,QAAUA,OACVwK,UAAY,UACZC,YAAa,OAMbC,kBAAoB,GAErBtpD,GAAQA,EAAKupD,oBACVA,aAAevpD,EAAKupD,aACF,GAAnBvpD,EAAKwpD,gBACFA,YAAa,OAEbA,YAAa,GAIlBxpD,GAAQA,EAAK6oD,iBACmB,kBAAvB7oD,EAAK6oD,mBACTG,aAAehpD,EAAK6oD,mBACpBC,iBAAmB9oD,EAAK6oD,uBAKzB,IAAMn5D,KAAOsQ,EAAK6oD,mBACfA,cAAcn5D,GAAOsQ,EAAK6oD,cAAcn5D,QACxCs5D,aAAev8D,KAAKu8D,cAAgBhpD,EAAK6oD,cAAcn5D,GAK9DsQ,GAAQA,EAAKypD,gBACVA,WAAazpD,EAAKypD,gBAElBA,WAAa7K,EAAQhjC,QAAQ8tC,qBAG/BC,cAAgB,UAEhB9e,IAAM,QACN/I,MAAQ,CACX9M,MAAO,CACLK,MAAO,EACPC,OAAQ,SAGPP,UAAY,UAEZkO,MAAQ,QACR2mB,aAAe,QACfC,aAAe,QACfC,aAAe,CAClBC,QAAS,GACTC,MAAO,SAEJC,kBAAmB,MAElBC,EAAyB,WAC7B/f,EAAK8f,kBAAmB,QAErBrL,QAAQvZ,KAAKiF,QAAQvH,GAAG,mBAAoBmnB,QAC5CZ,kBAAkB5/D,MAAK,WAC1BygD,EAAKyU,QAAQvZ,KAAKiF,QAAQtH,IAAI,mBAAoBknB,WAG/C/U,eAEAgV,QAAQnqD,oDAQPg1B,EAAQnb,SAASE,cAAc,OACjCttB,KAAKmyD,QAAQhjC,QAAQwuC,cAAcC,MACrCr1B,EAAMD,UAAY,sBAElBC,EAAMD,UAAY,iBAEf8V,IAAI7V,MAAQA,MAEXs1B,EAAQzwC,SAASE,cAAc,OACrCuwC,EAAMv1B,UAAY,YAClBC,EAAM5T,YAAYkpC,QACbzf,IAAIyf,MAAQA,MAEX7V,EAAa56B,SAASE,cAAc,OAC1C06B,EAAW1f,UAAY,YACvB0f,EAAW,aAAehoD,UACrBo+C,IAAI4J,WAAaA,OAEjB5J,IAAIwK,WAAax7B,SAASE,cAAc,YACxC8wB,IAAIwK,WAAWtgB,UAAY,iBAE3B8V,IAAIkK,KAAOl7B,SAASE,cAAc,YAClC8wB,IAAIkK,KAAKhgB,UAAY,iBAKrB8V,IAAImS,OAASnjC,SAASE,cAAc,YACpC8wB,IAAImS,OAAO/7B,MAAMyqB,WAAa,cAC9Bb,IAAImS,OAAO/7B,MAAMkL,SAAW,gBAC5B0e,IAAImS,OAAO7E,UAAY,QACvBtN,IAAIwK,WAAWj0B,YAAY30B,KAAKo+C,IAAImS,wCAOnCh9C,OACFvT,KAAKmyD,QAAQ2L,iBAAiBC,gBAG9BlqC,EACAmqC,OAEAzqD,GAAQA,EAAK+oD,uBACV,IAAMr5D,KAAOsQ,EAAK+oD,wBAChBA,mBAAmBr5D,GAAOsQ,EAAK+oD,mBAAmBr5D,MAIvDjD,KAAKmyD,QAAQhjC,SAAWnvB,KAAKmyD,QAAQhjC,QAAQ+Y,cAE/CrU,GADAmqC,EAAmBjoB,UAAKoc,QAAQhjC,QAAQ+Y,sBAAmBloC,OAChCuT,EAAMvT,KAAKo+C,IAAIyf,YAE1ChqC,EAAUtgB,GAAQA,EAAKsgB,WAGrBA,aAAmBoqC,QAAS,MACvBj+D,KAAKo+C,IAAIyf,MAAMlW,iBACfvJ,IAAIyf,MAAMr2B,YAAYxnC,KAAKo+C,IAAIyf,MAAMlW,iBAEvCvJ,IAAIyf,MAAMlpC,YAAYd,QAClBA,aAAmB33B,QAAU23B,EAAQqqC,mBAIrCrqC,aAAmB33B,OAC5B8hE,EAAiBzqD,EAAMvT,KAAKo+C,IAAIyf,YAE3Bzf,IAAIyf,MAAMnS,UADN73B,MAAAA,EACkBA,EAEA7zB,KAAKm8D,SAAW,SAIxC/d,IAAI7V,MAAMknB,MAAQl8C,GAAQA,EAAKk8C,OAAS,GACxCzvD,KAAKo+C,IAAIyf,MAAMlW,WAIlBrP,GAAKwV,gBAAgB9tD,KAAKo+C,IAAIyf,MAAO,cAHrCvlB,GAAKuV,aAAa7tD,KAAKo+C,IAAIyf,MAAO,cAMhCtqD,GAAQA,EAAKupD,cACV98D,KAAK88D,cAAgB98D,KAAK88D,cAAgBvpD,EAAKupD,oBAC7CA,aAAevpD,EAAKupD,mBAGHr9D,IAApB8T,EAAKwpD,iBAAgDt9D,IAApBO,KAAK+8D,aACjB,GAAnBxpD,EAAKwpD,gBACFA,YAAa,OAEbA,YAAa,GAItBzkB,GAAKuV,aAAa7tD,KAAKo+C,IAAI7V,MAAO,qBAC9BvoC,KAAK+8D,YACPzkB,GAAKwV,gBAAgB9tD,KAAKo+C,IAAI7V,MAAO,aACrC+P,GAAKuV,aAAa7tD,KAAKo+C,IAAI7V,MAAO,cAElC+P,GAAKwV,gBAAgB9tD,KAAKo+C,IAAI7V,MAAO,YACrC+P,GAAKuV,aAAa7tD,KAAKo+C,IAAI7V,MAAO,eAE3BvoC,KAAK88D,oBACTA,aAAe,KACpBxkB,GAAKwV,gBAAgB9tD,KAAKo+C,IAAI7V,MAAO,aACrC+P,GAAKwV,gBAAgB9tD,KAAKo+C,IAAI7V,MAAO,YACrC+P,GAAKwV,gBAAgB9tD,KAAKo+C,IAAI7V,MAAO,sBAGnCh1B,IAASA,EAAK4qD,WAAY5qD,EAAK2pD,gBACjC5kB,GAAKuV,aAAa7tD,KAAKo+C,IAAI7V,MAAO,oBAC9Bh1B,EAAK4qD,UACP7lB,GAAKuV,aAAa7tD,KAAKo+C,IAAI7V,MAAO,mBAAqBh1B,EAAK4qD,WAG5D7lB,GAAKuV,aAAa7tD,KAAKo+C,IAAI7V,MAAO,qCAGpC+P,GAAKuV,aAAa7tD,KAAKo+C,IAAI7V,MAAO,yBAI9BD,EAAY/0B,GAAQA,EAAK+0B,WAAa,KACxCA,GAAatoC,KAAKsoC,YAChBtoC,KAAKsoC,YACPgQ,GAAKwV,gBAAgB9tD,KAAKo+C,IAAI7V,MAAOvoC,KAAKsoC,WAC1CgQ,GAAKwV,gBAAgB9tD,KAAKo+C,IAAI4J,WAAYhoD,KAAKsoC,WAC/CgQ,GAAKwV,gBAAgB9tD,KAAKo+C,IAAIwK,WAAY5oD,KAAKsoC,WAC/CgQ,GAAKwV,gBAAgB9tD,KAAKo+C,IAAIkK,KAAMtoD,KAAKsoC,YAE3CgQ,GAAKuV,aAAa7tD,KAAKo+C,IAAI7V,MAAOD,GAClCgQ,GAAKuV,aAAa7tD,KAAKo+C,IAAI4J,WAAY1f,GACvCgQ,GAAKuV,aAAa7tD,KAAKo+C,IAAIwK,WAAYtgB,GACvCgQ,GAAKuV,aAAa7tD,KAAKo+C,IAAIkK,KAAMhgB,QAC5BA,UAAYA,GAIftoC,KAAKw0B,QACP8jB,GAAK8lB,cAAcp+D,KAAKo+C,IAAI7V,MAAOvoC,KAAKw0B,YACnCA,MAAQ,MAEXjhB,GAAQA,EAAKihB,QACf8jB,GAAK+lB,WAAWr+D,KAAKo+C,IAAI7V,MAAOh1B,EAAKihB,YAChCA,MAAQjhB,EAAKihB,uDASbx0B,KAAKq1C,MAAM9M,MAAMK,2DAQlB01B,EAAet+D,KAAKo+C,IAAImS,OAAOxE,gBACjCuS,GAAgBt+D,KAAKu+D,iBAAkB,MACpCA,iBAAmBD,MAClBE,EAAc,GAChBC,EAAoB,QAExBnmB,SAAAA,GAAat4C,KAAKw2C,OAAO,SAACuB,EAAM90C,MAC9B80C,EAAK2mB,OAAQ,EACT3mB,EAAK4mB,UAAW,CAElBH,EAAYv7D,GAAO80C,EAAKsY,QADJ,GAEpBoO,EAAoBD,EAAYv7D,GAAKjG,WAItByhE,EAAoB,qBAG5B3hE,MACPw7C,SAAAA,GAAakmB,GAAa,SAAAI,GACxBA,EAAI9hE,SAFCA,EAAI,EAAGA,EAAI2hE,EAAmB3hE,MAA9BA,UAMJ,SAEA,iEASsCkD,KAAKo+C,IAAI4J,WAAhD6W,IAAAA,UAAWC,IAAAA,WAAY1V,IAAAA,iBAC1BtG,IAAM+b,OACNlqB,MAAQmqB,OACRl2B,MAAQwgB,uDAQPzK,EAAK3+C,KACL++D,EAAiB/+D,KAAKmyD,QAAQhjC,QAAQ6vC,UACtCC,EAAc,CAClBC,oBAAqBl/D,KAAKmyD,QAAQgN,iBAClCC,WAAYL,GAAkBA,EAAeM,UAC7CC,iBAAkBP,GAAkBA,EAAe/4D,SACnDw2D,qBAAsBx8D,KAAKw8D,sBAEzB+C,EAAO,SACNv/D,KAAKmyD,QAAQL,gBAAiB,IAC7BmN,EAAYzC,4BAA+B,EAC3Ch7D,KAAKc,IAAI0kC,KAAa,IAAItqC,KAAKuiE,EAAYC,sBAAwBD,EAAYG,aAC7EH,EAAYK,kBAAwD,MAApCt/D,KAAKmyD,QAAQqN,oBAC/CP,EAAYK,kBAAiB,SAAAG,GAC3B9gB,EAAGwT,QAAQqN,oBAAsBC,EACjCF,GAAQE,KAGVF,EAD2C,GAAlC5gB,EAAGwT,QAAQqN,4BAQnBD,uCAWIG,EAAcC,EAAerF,EAAQ5gB,iBAChCgmB,GAAgB1/D,KAAK48D,YAAc58D,KAAK28D,YAAcgD,EAGzD,iBACLtC,EAAe,CACnBE,MAAOzmB,UAAKumB,aAAaE,eAAa,SAAAxlB,UAASA,EAAK6nB,aACpDtC,QAASxmB,UAAKumB,aAAaC,iBAAe,SAAAvlB,UAASA,EAAK6nB,cAGpDC,EAAkB,CACtBtC,SAAW,OAAQzmB,eAAKumB,aAAaE,eAAU,SAAAxlB,UAAQA,EAAK+nB,oBAAgB,SAAA/nB,WAAUA,OACtFulB,WAAa,OAAQxmB,eAAKumB,aAAaC,iBAAY,SAAAvlB,UAAQA,EAAK+nB,oBAAgB,SAAA/nB,WAAUA,QAOtF6d,EAAkB,qBAChBuH,EAAexH,EAAKoK,oBAAoB1C,EAAcvmB,KAAA6e,EAAKwH,sBAAoB,SAAAplB,UAASA,EAAK6nB,aAAYlmB,GACzGsmB,EAAkBrK,EAAKsK,uBAAuBJ,EAAiB/oB,KAAA6e,EAAKwH,sBAAoB,SAAAplB,UAAQA,EAAK6nB,aAAYlmB,6BAC5GyjB,MAAiB6C,KAQxBE,EAAmC,SAAAC,OACnCC,EAAwB,cACjBrF,SACHvkB,EAAQM,KAAA6e,EAAKwH,sBAAoB,SAAAplB,UAAQA,EAAKxkC,KAAKwnD,WAAaA,KACtEqF,EAAsBrF,GAAYoF,EAAUE,GAAA7pB,QAAAA,GAAW,SAACr5C,EAAGC,UAAM+iE,EAAQhjE,EAAEoW,KAAMnW,EAAEmW,SAASijC,OAFzF,IAAMukB,KAAYpF,EAAKwF,YAAjBJ,UAIJqF,MAGiC,mBAA/BpgE,KAAKmyD,QAAQhjC,QAAQyuC,MAAsB,KAG9Cjf,EAAK3+C,QACPA,KAAKu8D,cAAgBv8D,KAAKmyD,QAAQhjC,QAAQosC,eAAgB,CAG5Dl4D,GAD8B68D,EAAiClgE,KAAKmyD,QAAQhjC,QAAQyuC,OAC1BtD,EAAQt6D,KAAKm7D,gBAClEgC,aAAevH,SACf0K,uBAAuBhG,OAEzB,kBACE6C,aAAevH,SACf0K,uBAAuBhG,OAGtBiG,EAAqBF,oBAAKlD,+BAEA,SAAAplB,UAAQA,EAAK6nB,YAAe7nB,EAAK6nB,YAAc7nB,EAAK+nB,oBACtD,SAAC3iE,EAAGC,UACCuhD,EAAGwT,QAAQhjC,QAAQyuC,MAAMzgE,EAAEoW,KAAMnW,EAAEmW,cAEjEipD,qBAAuBn5D,GAAYk9D,EAAoBjG,GAAQ,EAAMvkB,UAAKyqB,+BAA4BxgE,YAExG,eAEAm9D,aAAevH,SACf0K,uBAAuBhG,GAExBt6D,KAAKmyD,QAAQhjC,QAAQ9rB,SACnBrD,KAAKu8D,cAAgBv8D,KAAKmyD,QAAQhjC,QAAQosC,eAE5Cl4D,GAD8B68D,IAC4B5F,EAAQt6D,KAAKm7D,qBAIlEqB,qBAAuBn5D,GAAYrD,KAAKm9D,aAAc7C,GAAQ,EAAMvkB,UAAKyqB,+BAA4BxgE,YAI5GqD,GAAcrD,KAAKm9D,aAAc7C,EAAQt6D,KAAKm7D,UAAWn7D,KAAKmyD,QAAQhjC,QAAQosC,oBAI7E,IAAIz+D,EAAI,EAAGA,EAAIkD,KAAKm9D,aAAangE,OAAQF,SACvCqgE,aAAargE,GAAG2jE,mBAC+ChhE,IAAhEO,KAAKs8D,mBAAmBt8D,KAAKm9D,aAAargE,GAAGyW,KAAKwnD,YAC/C/6D,KAAKs8D,mBAAmBt8D,KAAKm9D,aAAargE,GAAGyW,KAAKwnD,gBAChDoC,aAAargE,GAAGozD,QAKvBlwD,KAAKmyD,QAAQhjC,QAAQ2wC,YACvBxnB,SAAAA,GAAat4C,KAAKw2C,OAAO,SAAAuB,GACnBA,EAAK+nB,SAAW/nB,EAAK4mB,WACvB5mB,EAAKmY,UAKPlwD,KAAKw8D,2BACFrK,QAAQvZ,KAAKiF,QAAQ4C,KAAK,wBAE5Bmc,YAAa,sCAUX/jB,EAAShQ,GAClBgQ,EAAUP,GAAKooB,eAAe1gE,KAAM,SAAU6oC,IAAWgQ,MAEnD8nB,EAAa3gE,KAAKo+C,IAAIyf,MAAMld,YAC5BigB,EAAc5gE,KAAKo+C,IAAIyf,MAAM9R,oBACnClT,EAAUP,GAAKooB,eAAe1gE,KAAKq1C,MAAM9M,MAAO,QAASo4B,IAAe9nB,EACxEA,EAAUP,GAAKooB,eAAe1gE,KAAKq1C,MAAM9M,MAAO,SAAUq4B,IAAgB/nB,4CAQ1DhQ,QACXuV,IAAIwK,WAAWp0B,MAAMqU,iBAAYA,aACjCuV,IAAI4J,WAAWxzB,MAAMqU,iBAAYA,aACjCuV,IAAI7V,MAAM/T,MAAMqU,iBAAYA,6DAONyxB,OACtB,IAAIx9D,EAAI,EAAGmO,EAAKjL,KAAKm9D,aAAangE,OAAQF,EAAImO,EAAInO,IAAK,KACpDi7C,EAAO/3C,KAAKm9D,aAAargE,GAC/Bi7C,EAAK8oB,YAAYvG,GACZt6D,KAAK28D,WAAa38D,KAAKm8D,SAAWF,IACjClkB,EAAK4mB,WAAW5mB,EAAKmY,uCAaxBxW,EAAO4gB,EAAQoF,EAAcoB,iBAG9Bj4B,EAsDEtwB,SAxDFsgC,GAAU,EACR8mB,EAAgB3/D,KAAK28D,UAGrBtK,EAAQ,CACZ,WACEqN,EAAeqB,EAAKC,uBAAuB3kE,KAAK0kE,IAASrB,aAItDY,+BAA4BtgE,KAAMs6D,aAGlC2G,uCAAoCjhE,MAEzC,iBACE+gE,EAAKpE,UAAY5mB,KAAAgrB,EAAKG,wBAAqBH,EAA1BhrB,CAAgC2D,EAAO4gB,IAG1D,sBACEyG,EAAKI,qBAAkBJ,GAAMrB,EAAcC,EAAerF,EAAQ5gB,cAI/D0nB,8BAA2BphE,MAEhC,iBACE6oC,EAASkN,KAAAgrB,EAAKM,yBAAsBN,EAA3BhrB,CAAiCukB,cAIvC2G,uCAAoCjhE,MAEzC,iBACE64C,EAAU9C,KAAAgrB,EAAKO,mBAAgBP,EAArBhrB,CAA2B8C,EAAShQ,IAGhD,sBACEk4B,EAAKQ,0BAAuBR,GAAMl4B,IAGpC,sBACEk4B,EAAKS,qCAAkCT,GAAMzG,IAG/CvkB,KAAC,kBACMgrB,EAAKpE,WAAaoE,EAAKl4B,SAC1BgQ,GAAU,GAELA,WACD74C,cAGN8gE,EACKzO,MAGPA,QAAAA,GAAc,SAAAx1D,GACZ0b,EAAS1b,OAEJ0b,kDAUY+hD,iBACjBta,GAAYhgD,KAAKm7D,WAAWn+D,OAAS,EAAG,KACpC2hD,EAAK3+C,UAENyhE,qBAELnpB,SAAAA,GAAat4C,KAAKm9D,cAAc,SAAAplB,QACHt4C,IAAvBs4C,EAAKxkC,KAAKwnD,WACZpc,EAAGwc,UAAUpjB,EAAKxkC,KAAKwnD,UAAUlyB,OAASrnC,KAAK8D,IAAIq5C,EAAGwc,UAAUpjB,EAAKxkC,KAAKwnD,UAAUlyB,OAAQkP,EAAKlP,OAASyxB,EAAOviB,KAAK8iB,UACtHlc,EAAGwc,UAAUpjB,EAAKxkC,KAAKwnD,UAAUO,aAAiE,IAAhDoG,EAAKpF,mBAAmBvkB,EAAKxkC,KAAKwnD,WAAmC94B,QAAQy/B,EAAKpF,mBAAmBvkB,EAAKxkC,KAAKwnD,wDAczJrhB,EAAO4gB,UACbt6D,KAAK8iD,KAAOpJ,EAAMd,KAAKY,SAASC,gBAAgB5Q,OAAS6Q,EAAMd,KAAKY,SAASoY,UAAY0I,EAAOhS,MACpGtoD,KAAK8iD,IAAM9iD,KAAK6oC,OAASyxB,EAAOhS,OAAU5O,EAAMd,KAAKY,SAASoY,mDASnD0I,OAEXzxB,EAEA2N,MAGFA,EADsB,UAApBx2C,KAAKg9D,WACC1kB,GAAK51B,QAAQ1iB,KAAKw2C,OAGlBx2C,KAAKm9D,cAGLngE,OAAS,EAAG,KAChBoF,EAAMo0C,EAAM,GAAGsM,IACfx9C,EAAMkxC,EAAM,GAAGsM,IAAMtM,EAAM,GAAG3N,aAClCyP,SAAAA,GAAa9B,GAAO,SAAAuB,GAClB31C,EAAMZ,KAAKY,IAAIA,EAAK21C,EAAK+K,KACzBx9C,EAAM9D,KAAK8D,IAAIA,EAAMyyC,EAAK+K,IAAM/K,EAAKlP,WAEnCzmC,EAAMk4D,EAAOhS,KAAM,KAEf9rC,EAASpa,EAAMk4D,EAAOhS,KAC5BhjD,GAAOkX,KACP87B,SAAAA,GAAa9B,GAAO,SAAAuB,GAClBA,EAAK+K,KAAOtmC,KAGhBqsB,EAASrnC,KAAKC,KAAK6D,EAAMg1D,EAAOviB,KAAK8iB,SAAW,GACxB,aAApB76D,KAAKg9D,aACPn0B,EAASrnC,KAAK8D,IAAIujC,EAAQ7oC,KAAKq1C,MAAM9M,MAAMM,cAI7CA,EAAc7oC,KAAKq1C,MAAM9M,MAAMM,cAE1BA,iCAOF7oC,KAAKo+C,IAAI7V,MAAMhB,iBACb4qB,QAAQ/T,IAAIujB,SAAShtC,YAAY30B,KAAKo+C,IAAI7V,OAG5CvoC,KAAKo+C,IAAI4J,WAAWzgB,iBAClB4qB,QAAQ/T,IAAI4J,WAAWrzB,YAAY30B,KAAKo+C,IAAI4J,YAG9ChoD,KAAKo+C,IAAIwK,WAAWrhB,iBAClB4qB,QAAQ/T,IAAIwK,WAAWj0B,YAAY30B,KAAKo+C,IAAIwK,YAG9C5oD,KAAKo+C,IAAIkK,KAAK/gB,iBACZ4qB,QAAQ/T,IAAIkK,KAAK3zB,YAAY30B,KAAKo+C,IAAIkK,yCAQvC/f,EAAQvoC,KAAKo+C,IAAI7V,MACnBA,EAAMhB,YACRgB,EAAMhB,WAAWC,YAAYe,OAGzByf,EAAahoD,KAAKo+C,IAAI4J,WACxBA,EAAWzgB,YACbygB,EAAWzgB,WAAWC,YAAYwgB,OAG9BY,EAAa5oD,KAAKo+C,IAAIwK,WACxBA,EAAWrhB,YACbqhB,EAAWrhB,WAAWC,YAAYohB,OAG9BN,EAAOtoD,KAAKo+C,IAAIkK,KAClBA,EAAK/gB,YACP+gB,EAAK/gB,WAAWC,YAAY8gB,+BAQ5BvQ,iBACGvB,MAAMuB,EAAK3kB,IAAM2kB,EACtBA,EAAK6pB,UAAU5hE,WACV48D,YAAa,OAESn9D,IAAvBs4C,EAAKxkC,KAAKwnD,gBACP8G,eAAe9pB,QACf+pB,mBAGFC,UAAK5E,qBAAsBplB,GAAO,KAC/B2B,EAAQ15C,KAAKmyD,QAAQvZ,KAAKc,WAC3BsoB,gBAAgBjqB,EAAM/3C,KAAKm9D,aAAczjB,2CASnC3B,OAAMkqB,yDAAWlqB,EAAKxkC,KAAKwnD,SACtBt7D,MAAdwiE,QAA0DxiE,IAA/BO,KAAKm7D,UAAU8G,UACvC9G,UAAU8G,GAAc,CAC3Bp5B,OAAQ,EACRia,IAAK,EACLtiB,MAAOuX,EAAKxkC,KAAKitB,MACjBC,IAAKsX,EAAKxkC,KAAKktB,KAAOsX,EAAKxkC,KAAKitB,MAChC86B,SAAS,EACT12C,MAAO5kB,KAAKy8D,cACZjmB,MAAO,GACPnzC,MAAOrD,KAAKq8D,kBAAoBr8D,KAAKo8D,cAAc6F,KAAe,QAE/DxF,iBAIH,IAAI//D,KAAKq7C,EAAKxkC,KAAKitB,OAAS,IAAI9jC,KAAKsD,KAAKm7D,UAAU8G,GAAYzhC,cAC7D26B,UAAU8G,GAAYzhC,MAAQuX,EAAKxkC,KAAKitB,WAGzC0hC,EAAUnqB,EAAKxkC,KAAKktB,KAAOsX,EAAKxkC,KAAKitB,MACvC,IAAI9jC,KAAKwlE,GAAW,IAAIxlE,KAAKsD,KAAKm7D,UAAU8G,GAAYxhC,YACrD06B,UAAU8G,GAAYxhC,IAAMyhC,QAG9B/G,UAAU8G,GAAYzrB,MAAMv5C,KAAK86C,uDAOhC4G,EAAK3+C,QACP2+C,EAAGwc,UAAW,gBACLJ,SACHoH,EAAaxjB,EAAGwc,UAAUJ,GAAUvkB,MAAM,GAAGjjC,KAAKktB,KAAOke,EAAGwc,UAAUJ,GAAUvkB,MAAM,GAAGjjC,KAAKitB,MAChGogB,EAAWjC,EAAGwc,UAAUJ,GAAUvkB,MAAM,GAAGjjC,KAAKitB,MAChDqgB,EAASshB,EAAa,OAE1BxjB,EAAGwc,UAAUJ,GAAUvkB,eAAc,SAAAuB,GAC/B,IAAIr7C,KAAKq7C,EAAKxkC,KAAKitB,OAAS,IAAI9jC,KAAKkkD,KACvCA,EAAW7I,EAAKxkC,KAAKitB,WAGjB0hC,EAAUnqB,EAAKxkC,KAAKktB,KAAOsX,EAAKxkC,KAAKitB,MACvC,IAAI9jC,KAAKwlE,GAAW,IAAIxlE,KAAKmkD,KAC/BA,EAASqhB,MAIbvjB,EAAGwc,UAAUJ,GAAUv6B,MAAQogB,EAC/BjC,EAAGwc,UAAUJ,GAAUt6B,IAAM,IAAI/jC,KAAKmkD,EAAS,QAjB5C,IAAMka,KAAYpc,EAAGwc,YAAfJ,qDA2BgBt7D,IAAzBO,KAAK08D,gBAA+B,KAChC0F,EAAY,MACiB,iBAAxBpiE,KAAK08D,gBAA6B,KACtC,IAAM3B,KAAY/6D,KAAKm7D,UAC1BiH,EAAUnlE,KAAK,CAAC89D,SAAAA,EAAUsH,UAAWriE,KAAKm7D,UAAUJ,GAAUvkB,MAAM,GAAGjjC,KAAKvT,KAAK08D,sBAEnF0F,QAAAA,GAAe,SAACjlE,EAAGC,UAAMD,EAAEklE,UAAYjlE,EAAEilE,kBAEtC,GAAmC,mBAAxBriE,KAAK08D,gBAA+B,KAC7C,IAAM3B,KAAY/6D,KAAKm7D,UAC1BiH,EAAUnlE,KAAK+C,KAAKm7D,UAAUJ,GAAUvkB,MAAM,GAAGjjC,SAEnD6uD,QAAAA,EAAepiE,KAAK08D,oBAGlB0F,EAAUplE,OAAS,MAChB,IAAIF,EAAI,EAAGA,EAAIslE,EAAUplE,OAAQF,SAC/Bq+D,UAAUiH,EAAUtlE,GAAGi+D,UAAUn2C,MAAQ9nB,iDAU/C,IAAMi+D,KAAY/6D,KAAKm7D,UACtBn7D,KAAKm7D,UAAU99D,eAAe09D,UAC3BI,UAAUJ,GAAUO,SAAU,OAC9BH,UAAUJ,GAAUlyB,OAAS,kCASjCkP,kBACE/3C,KAAKw2C,MAAMuB,EAAK3kB,IACvB2kB,EAAK6pB,UAAU,WACVhF,YAAa,MAGZh4C,EAAQgvC,UAAKuJ,qBAAqBplB,IAC1B,GAAVnzB,GAAa09C,UAAKnF,qBAAoBv4C,EAAO,QAEvBnlB,IAAvBs4C,EAAKxkC,KAAKwnD,gBACNwH,oBAAoBxqB,QACpB+pB,8DASW/pB,OAAMkqB,yDAAWlqB,EAAKxkC,KAAKwnD,YAC3Bt7D,MAAdwiE,EAAyB,KACrBlH,EAAW/6D,KAAKm7D,UAAU8G,MAC5BlH,EAAS,SACLyH,EAAY5O,KAAAmH,EAASvkB,cAAcuB,MAErCyqB,GAAa,OACfzH,EAASvkB,cAAagsB,EAAU,GAC3BzH,EAASvkB,MAAMx5C,YAGbokE,+BAFEphE,KAAKm7D,UAAU8G,+CAadlqB,QACXoa,QAAQsQ,WAAW1qB,EAAK3kB,4CAOvB/sB,EAAQiyC,GAAK51B,QAAQ1iB,KAAKw2C,OAC1BksB,EAAa,GACbC,EAAW,GAER7lE,EAAI,EAAGA,EAAIuJ,EAAMrJ,OAAQF,SACN2C,IAAtB4G,EAAMvJ,GAAGyW,KAAKktB,KAChBkiC,EAAS1lE,KAAKoJ,EAAMvJ,IAEtB4lE,EAAWzlE,KAAKoJ,EAAMvJ,SAEnBugE,aAAe,CAClBC,QAASoF,EACTnF,MAAOoF,GAGTt/D,GAAmBrD,KAAKq9D,aAAaC,SACrCj6D,GAAiBrD,KAAKq9D,aAAaE,mDAWjBF,EAAcuF,EAAiBlpB,OAC3CyjB,EAAe,GACf0F,EAAqB,OAEtB7iE,KAAK28D,WAAa38D,KAAKm8D,SAAWF,GAA6B,KAC7D,IAAIn/D,EAAI,EAAGA,EAAI8lE,EAAgB5lE,OAAQF,IAAK,KAC3Ci7C,EAAO6qB,EAAgB9lE,GACvBi7C,EAAK4mB,WAAW5mB,EAAKmY,cAEpBiN,MAGHte,GAAYnF,EAAMjZ,IAAMiZ,EAAMlZ,OAAS,EACvCsiC,EAAappB,EAAMlZ,MAAQqe,EAC3BkkB,EAAarpB,EAAMjZ,IAAMoe,KAkB3B+jB,EAAgB5lE,OAAS,MACtB,IAAIF,EAAI,EAAGA,EAAI8lE,EAAgB5lE,OAAQF,SACrCkmE,6BAA6BJ,EAAgB9lE,GAAIqgE,EAAc0F,EAAoBnpB,OAKtFupB,EAAoB3qB,GAAK4qB,mBAAmB7F,EAAaC,SAtBnC,SAAAx7D,UACjBA,EAAQghE,GAAsB,EAC9BhhE,GAASihE,EAAqB,EACA,IAmBoD,OAAO,iBAG/FI,cAAcF,EAAmB5F,EAAaC,QAASH,EAAc0F,GAAoB,SAAA9qB,UAAQA,EAAKxkC,KAAKitB,MAAQsiC,GAAc/qB,EAAKxkC,KAAKitB,MAAQuiC,KAI3H,GAAzB/iE,KAAKw9D,iBAA0B,MAC5BA,kBAAmB,MACnB,IAAI1gE,EAAI,EAAGA,EAAIugE,EAAaE,MAAMvgE,OAAQF,SACxCkmE,6BAA6B3F,EAAaE,MAAMzgE,GAAIqgE,EAAc0F,EAAoBnpB,OAG1F,KAEG0pB,EAAkB9qB,GAAK4qB,mBAAmB7F,EAAaE,OA9BrC,SAAAz7D,UACfA,EAAQghE,GAAsB,EACA,IA4BgD,OAAO,YAGzFK,cAAcC,EAAiB/F,EAAaE,MAAOJ,EAAc0F,GAAoB,SAAA9qB,UAAQA,EAAKxkC,KAAKktB,IAAMqiC,GAAc/qB,EAAKxkC,KAAKitB,MAAQuiC,aAG9IvE,EAAc,GAChBC,EAAoB,EAEf3hE,EAAI,EAAGA,EAAIqgE,EAAangE,OAAQF,IAAK,KACtCi7C,EAAOolB,EAAargE,OACrBi7C,EAAK4mB,UAAW,CAEnBH,EAAY1hE,GAAKi7C,EAAKsY,QADF,GAEpBoO,EAAoBD,EAAY1hE,GAAGE,WAIpByhE,EAAoB,qBAG5B1qD,MACPukC,SAAAA,GAAakmB,GAAa,SAAAI,GACxBA,EAAI7qD,SAFCA,EAAI,EAAGA,EAAI0qD,EAAmB1qD,MAA9BA,OAON,IAAIjX,EAAI,EAAGA,EAAIqgE,EAAangE,OAAQF,IACvCqgE,EAAargE,GAAG2jE,qBAGXtD,wCAWKkG,EAAY7sB,EAAO2mB,EAAc0F,EAAoBS,OAC9C,GAAfD,EAAkB,KACf,IAAIvmE,EAAIumE,EAAYvmE,GAAK,EAAGA,IAAK,KAChCi7C,EAAOvB,EAAM15C,MACbwmE,EAAevrB,SAIXA,EAAK6nB,YAAe7nB,EAAKwrB,YAAgBxrB,EAAK+nB,cACdrgE,IAAhCojE,EAAmB9qB,EAAK3kB,MAC1ByvC,EAAmB9qB,EAAK3kB,KAAM,EAC9B+pC,EAAalgE,KAAK86C,QAMrB,IAAIj7C,EAAIumE,EAAa,EAAGvmE,EAAI05C,EAAMx5C,OAAQF,IAAK,KAC9Ci7C,EAAOvB,EAAM15C,MACbwmE,EAAevrB,SAIXA,EAAK6nB,YAAc7nB,EAAKwrB,YAAgBxrB,EAAK+nB,cACbrgE,IAAhCojE,EAAmB9qB,EAAK3kB,MAC1ByvC,EAAmB9qB,EAAK3kB,KAAM,EAC9B+pC,EAAalgE,KAAK86C,8CAmBdA,EAAMolB,EAAczjB,GAC5B3B,EAAK4kB,UAAUjjB,IACZ3B,EAAK4mB,WAAW5mB,EAAKyrB,OAE1BzrB,EAAK0oB,cACLtD,EAAalgE,KAAK86C,IAGdA,EAAK4mB,WAAW5mB,EAAKmY,4DAgBFnY,EAAMolB,EAAc0F,EAAoBnpB,GAC/D3B,EAAK4kB,UAAUjjB,QACmBj6C,IAAhCojE,EAAmB9qB,EAAK3kB,MAC1ByvC,EAAmB9qB,EAAK3kB,KAAM,EAC9B+pC,EAAalgE,KAAK86C,IAIhBA,EAAK4mB,WAAW5mB,EAAKmY,sDAYN2P,EAAiB4D,EAAoB/pB,OAEpDsmB,EAAkB,GAClB0D,EAAwB,MAE1BD,EAAmBzmE,OAAS,MACzB,IAAIF,EAAI,EAAGA,EAAI2mE,EAAmBzmE,OAAQF,SACxCkmE,6BAA6BS,EAAmB3mE,GAAIkjE,EAAiB0D,EAAuBhqB,OAIhG,IAAI58C,EAAI,EAAGA,EAAI+iE,EAAgBvC,QAAQtgE,OAAQF,SAC7CkmE,6BAA6BnD,EAAgBvC,QAAQxgE,GAAIkjE,EAAiB0D,EAAuBhqB,OAGnG,IAAI58C,EAAI,EAAGA,EAAI+iE,EAAgBtC,MAAMvgE,OAAQF,SAC3CkmE,6BAA6BnD,EAAgBtC,MAAMzgE,GAAIkjE,EAAiB0D,EAAuBhqB,WAGhG8kB,EAAc,GAChBC,EAAoB,EAEf3hE,EAAI,EAAGA,EAAIkjE,EAAgBhjE,OAAQF,IAAK,KACzCi7C,EAAOioB,EAAgBljE,OACxBi7C,EAAK4mB,UAAW,CAEnBH,EAAY1hE,GAAKi7C,EAAKsY,QADF,GAEpBoO,EAAoBD,EAAY1hE,GAAGE,WAIpByhE,EAAoB,MAGhC,IAAI1qD,EAAI,EAAGA,EAAI0qD,EAAmB1qD,OACrCukC,SAAAA,GAAakmB,GAAa,SAAUI,GAClCA,EAAI7qD,YAKL,IAAIjX,EAAI,EAAGA,EAAIkjE,EAAgBhjE,OAAQF,IAC1CkjE,EAAgBljE,GAAG2jE,qBAGdT,yCASMjoB,EAAM4rB,EAAaC,QAC3BrB,oBAAoBxqB,EAAM4rB,QAC1B9B,eAAe9pB,EAAM6rB,QACrB9B,uDAUD+B,WAFC3T,OAGG2T,EAAkB7jE,KAAK68D,kBAAkB7qB,OAC/C6xB,aChoCAC,0BAMQ3H,EAAS5oD,EAAM4+C,sDACnBgK,EAAS5oD,EAAM4+C,KAGhBvpB,MAAQ,IACRC,OAAS,IACTia,IAAM,IACNpO,KAAO,sDAUPgF,EAAO4gB,EAAQoF,QAGfvC,aAAen9D,KAAK+/D,oBAAoB//D,KAAKq9D,aAAcr9D,KAAKm9D,aAAczjB,QAG9E9Q,MAAQ5oC,KAAKo+C,IAAIwK,WAAWQ,iBAG5BhL,IAAIwK,WAAWp0B,MAAMqU,OAAU,QAG/B,IAAI/rC,EAAI,EAAGmO,EAAKjL,KAAKm9D,aAAangE,OAAQF,EAAImO,EAAInO,IAAK,CAC7CkD,KAAKm9D,aAAargE,GAC1B+jE,YAAYvG,UAbH,iCAuBXt6D,KAAKo+C,IAAIwK,WAAWrhB,iBAClB4qB,QAAQ/T,IAAIwK,WAAWj0B,YAAY30B,KAAKo+C,IAAIwK,mBAhDzBsT,imVCIxB6H,yBAUQxwD,EAAMmoC,EAAYvsB,gCACvBiE,GAAK,UACLxC,OAAS,UACTrd,KAAOA,OACP6qC,IAAM,UACN1C,WAAaA,GAAc,QAC3BwB,eAAiB,CACpBtqC,QAAAA,GACAlV,OAAQ,WAELyxB,QAAUmpB,GAAKh7C,OAAO,GAAI0C,KAAKk9C,eAAgB/tB,QAC/CA,QAAQvc,QAAU0lC,GAAKh7C,OAAO,GAAIsV,GAAS5S,KAAKmvB,QAAQvc,aACvD88C,EAAiB1vD,KAAKk9C,eAAetqC,QAAQ5S,KAAKk9C,eAAex/C,gBAC3DsC,KAAKmvB,QAAQvc,kBAAiB,SAAAlV,GACxCggD,EAAKvuB,QAAQvc,QAAQlV,GAAU46C,GAAKh7C,OAClC,GACAoyD,EACAhS,EAAKvuB,QAAQvc,QAAQlV,YAGpBsmE,UAAW,OACXrF,WAAY,OACZsF,cAAe,OACfC,WAAc/0C,GAAWA,EAAQ+0C,aAAe,OAChDxF,OAAQ,OAER5b,IAAM,UACNnO,MAAQ,UACRD,KAAO,UACP9L,MAAQ,UACRC,OAAS,UAETs7B,iBAAiB5wD,QAEjB+8C,SAAW,UACX8T,gEAODpkE,KAAKkkE,kBACFF,UAAW,OACXtF,OAAQ,EACT1+D,KAAK2+D,WAAW3+D,KAAKqwD,kDAQtB2T,UAAW,OACXtF,OAAQ,EACT1+D,KAAK2+D,WAAW3+D,KAAKqwD,yCAQnB98C,GAC6B9T,MAAd8T,EAAK8wD,OAAsBrkE,KAAKuT,KAAK8wD,OAAS9wD,EAAK8wD,OACrC,MAAfrkE,KAAK4wB,aAClBA,OAAOuhC,QAAQmS,aAAatkE,KAAMuT,EAAK8wD,YAGzCF,iBAAiB5wD,GAElBvT,KAAK4wB,cACFA,OAAOgsC,YAAa,GAGcn9D,MAAjB8T,EAAKwnD,UAAyB/6D,KAAKuT,KAAKwnD,UAAYxnD,EAAKwnD,UAC3C,MAAf/6D,KAAK4wB,aACrBA,OAAO2zC,eAAevkE,KAAMA,KAAKuT,KAAKwnD,SAAUxnD,EAAKwnD,eAGvDxnD,KAAOA,OACP6wD,yBACA1F,OAAQ,EACT1+D,KAAK2+D,WAAW3+D,KAAKqwD,kDAQV98C,GACXA,SACG2wD,gBAAwC,IAApB3wD,EAAK2wD,YAAoCjiC,QAAQ1uB,EAAK2wD,+CAQzEtzC,GACJ5wB,KAAK2+D,gBACFzO,YACAt/B,OAASA,EACV5wB,KAAK4wB,aACF4yC,aAIF5yC,OAASA,oCASR8oB,UACD,wCAQA,wCAQA,kKA6BH15C,KAAKgkE,UAAYhkE,KAAKswD,SAASkU,aAAexkE,KAAKo+C,IAAIqmB,WAAY,SAC/D9lB,EAAK3+C,KAELykE,EAAar3C,SAASE,cAAc,OAC1Cm3C,EAAWn8B,UAAY,kBACvBm8B,EAAWC,eAAiB1kE,UACvB2kE,iBAAmB,IAAI3gB,GAAOygB,QAE9BE,iBAAiBruB,GAAG,OAAO,SAAAkK,GAC9B7B,EAAG/tB,OAAOuhC,QAAQvZ,KAAKiF,QAAQ4C,KAAK,QAAU,CAC5CD,MAAAA,EACAzI,KAAM4G,EAAGvrB,aAGRuxC,iBAAiBruB,GAAG,aAAa,SAAAkK,GACpCA,EAAM8D,kBACN3F,EAAG/tB,OAAOuhC,QAAQyS,cAAcjmB,GAChCA,EAAG/tB,OAAOuhC,QAAQvZ,KAAKiF,QAAQ4C,KAAK,cAAe,CACjDD,MAAAA,EACAzI,KAAM4G,EAAGvrB,aAGRuxC,iBAAiBruB,GAAG,YAAY,SAAAkK,GAEnCA,EAAM8D,kBACN3F,EAAG/tB,OAAOuhC,QAAQrU,aAAa0C,WAE5BmkB,iBAAiBruB,GAAG,UAAYP,KAAA4I,EAAG/tB,OAAOuhC,QAAQpU,gBAAaY,EAAG/tB,OAAOuhC,eACzEwS,iBAAiBruB,GAAG,SAAYP,KAAA4I,EAAG/tB,OAAOuhC,QAAQnU,mBAAgBW,EAAG/tB,OAAOuhC,UAE7EnyD,KAAKo+C,IAAIymB,IACP7kE,KAAKo+C,IAAI0mB,cACN1mB,IAAIymB,IAAI78B,aAAay8B,EAAYzkE,KAAKo+C,IAAI0mB,eAG1C1mB,IAAIymB,IAAIlwC,YAAY8vC,GAGpBzkE,KAAKo+C,IAAIte,YACXse,IAAIte,MAAMnL,YAAY8vC,QAGxBrmB,IAAIqmB,WAAaA,OAEdzkE,KAAKgkE,UAAYhkE,KAAKo+C,IAAIqmB,aAE9BzkE,KAAKo+C,IAAIqmB,WAAWl9B,iBACjB6W,IAAIqmB,WAAWl9B,WAAWC,YAAYxnC,KAAKo+C,IAAIqmB,iBAEjDrmB,IAAIqmB,WAAa,KAElBzkE,KAAK2kE,wBACFA,iBAAiB5f,eACjB4f,iBAAmB,oDAUT5lD,OACbuxC,GAAatwD,KAAKmvB,QAAQmhC,SAASyU,eAAkC,MAAjB/kE,KAAKswD,WAAqBtwD,KAAKmvB,QAAQmhC,SAASra,SACzFj2C,KAAKmvB,QAAQmhC,SAASyU,eAAkC,MAAjB/kE,KAAKswD,UAAoBtwD,KAAKswD,SAASra,UAE3Fj2C,KAAKgkE,UAAY1T,IAAatwD,KAAKo+C,IAAI4mB,aAAc,KAEjDrmB,EAAK3+C,KAELglE,EAAe53C,SAASE,cAAc,OAExCttB,KAAKmvB,QAAQguB,IACf6nB,EAAa18B,UAAY,iBAEzB08B,EAAa18B,UAAY,iBAEvB28B,EAAgBjlE,KAAKmvB,QAAQvc,QAAQ5S,KAAKmvB,QAAQzxB,QACjDunE,IACEjlE,KAAKmwD,SACRxtD,QAAQH,yCAAkCxC,KAAKmvB,QAAQzxB,mGAClDyyD,QAAS,GAEhB8U,EAAgBjlE,KAAKmvB,QAAQvc,QAAb,IAElBoyD,EAAavV,MAAQwV,EAAcjX,oBAG9BkX,mBAAqB,IAAIlhB,GAAOghB,GAAc1uB,GAAG,OAAO,SAAAkK,GAC3DA,EAAM8D,kBACN3F,EAAG/tB,OAAOu0C,kBAAkBxmB,MAG9B5/B,EAAO4V,YAAYqwC,QACd5mB,IAAI4mB,aAAeA,OAEhBhlE,KAAKgkE,UAAYhkE,KAAKo+C,IAAI4mB,eAE9BhlE,KAAKo+C,IAAI4mB,aAAaz9B,iBACnB6W,IAAI4mB,aAAaz9B,WAAWC,YAAYxnC,KAAKo+C,IAAI4mB,mBAEnD5mB,IAAI4mB,aAAe,KAEpBhlE,KAAKklE,0BACFA,mBAAmBngB,eACnBmgB,mBAAqB,+DAUAnmD,MACzB/e,KAAKmvB,QAAQi2C,6BAEZ9U,GAAYtwD,KAAKmvB,QAAQmhC,SAASkU,aACD,IAAvBxkE,KAAKuT,KAAK+8C,YACY,IAAvBtwD,KAAKuT,KAAK+8C,YAErBtwD,KAAKgkE,UAAY1T,IAAatwD,KAAKo+C,IAAIinB,wBAAyB,KAC5DA,EAA0Bj4C,SAASE,cAAc,OAEvD+3C,EAAwB/8B,UAAY,2BACpCvpB,EAAO4V,YAAY0wC,QACdjnB,IAAIinB,wBAA0BA,OAEzBrlE,KAAKgkE,UAAYhkE,KAAKo+C,IAAIinB,0BAEhCrlE,KAAKo+C,IAAIinB,wBAAwB99B,iBAC9B6W,IAAIinB,wBAAwB99B,WAAWC,YAAYxnC,KAAKo+C,IAAIinB,8BAE9DjnB,IAAIinB,wBAA0B,SAIjCrlE,KAAKo+C,IAAIinB,wBAAyB,MAG/BjnB,IAAIinB,wBAAwB7wC,MAAMyqB,WAAaj/C,KAAK4wB,OAAOuhC,QAAQmT,YAAYC,eAAiB,UAAY,cAG5GnnB,IAAIinB,wBAAwB7wC,MAAMkiB,UAAY,wBAC9C0H,IAAIinB,wBAAwB7wC,MAAMkgB,KAAO,UAyB1C7gB,IArBE+9B,EAAY5xD,KAAK4wB,OAAOuhC,QAAQvZ,KAAKY,SAASoY,cAKf,OAAjC5xD,KAAKmvB,QAAQk5B,YAAYtQ,KACL/3C,KAAK8iD,IAEJ9iD,KAAK4wB,OAAOiY,OAAS7oC,KAAK8iD,IAAM9iD,KAAK6oC,QAEnB7oC,KAAK4wB,OAAOkyB,IAXjC,IAWwD8O,QAGvExT,IAAIinB,wBAAwB7wC,MAAMq0B,OAAS,QAC3CzK,IAAIinB,wBAAwB7wC,MAAMsuB,cAAS9iD,KAAK6oC,OAAS,eAEzDuV,IAAIinB,wBAAwB7wC,MAAMsuB,IAAM,QACxC1E,IAAIinB,wBAAwB7wC,MAAMq0B,iBAAY7oD,KAAK6oC,OAAS,SAO/D7oC,KAAKmvB,QAAQi2C,yBAA2BplE,KAAKmvB,QAAQi2C,wBAAwBI,SAE/E3xC,EADmBkiB,UAAK5mB,QAAQi2C,wBAAwBI,iBAAcxlE,KAC5Dg+D,CAAiBh+D,KAAKuT,WAEhCsgB,mBAAoBzJ,GAAOpqB,KAAKuT,KAAKitB,OAAO/iC,OAAO,qBAC/CuC,KAAKuT,KAAKktB,MACZ5M,uBAAwBzJ,GAAOpqB,KAAKuT,KAAKktB,KAAKhjC,OAAO,2BAGpD2gD,IAAIinB,wBAAwB3Z,UAAY73B,kDAUxC7zB,KAAK4wB,OAAOuhC,QAAQsT,UAAU17D,IAAI/J,KAAKozB,4CAQhCwU,OACV/T,EAEAmqC,EACA0H,EACAC,MACE3R,EAAWh0D,KAAK4lE,eAGhBC,GADe7lE,KAAKo+C,IAAIymB,KAAO7kE,KAAKo+C,IAAIte,OACMgmC,uBAAuB,0BAA0B,GAEjG9lE,KAAKmvB,QAAQ42C,qBAEfL,GADAC,EAA+B5vB,UAAK5mB,QAAQ42C,6BAA0B/lE,OACfg0D,EAAU6R,GAEjEH,EAA0B,MAGxBG,KACGH,aAAmCxpE,UAAawpE,aAAmCzH,SACtF0H,EAA6B3R,EAAU6R,WAE5B7lE,KAAKgmE,iBAAiBhmE,KAAK0lE,2BAA6B1lE,KAAKgmE,iBAAiBN,GAC3E,IAERA,aAAmCzH,QACrC4H,EAA+Bna,UAAY,GAC3Cma,EAA+BlxC,YAAY+wC,QAExC,GAA+BjmE,MAA3BimE,EACPG,EAA+Bna,UAAYga,UAGnB,cAAlB1lE,KAAKuT,KAAK2jB,WAA8Cz3B,IAAtBO,KAAKuT,KAAKsgB,cAC1C,IAAIzwB,mDAA4CpD,KAAKozB,UAI1DsyC,wBAA0BA,EAKjC1lE,KAAKmvB,QAAQq2C,SAEf3xC,GADAmqC,EAAmBjoB,UAAK5mB,QAAQq2C,iBAAcxlE,OACnBg0D,EAAUpsB,EAAS5nC,KAAKuT,MAEnDsgB,EAAU7zB,KAAKuT,KAAKsgB,WAGjBA,aAAmB33B,UAAa23B,aAAmBoqC,SACtDD,EAAiBhK,EAAUpsB,WAEjB5nC,KAAKgmE,iBAAiBhmE,KAAK6zB,WAAa7zB,KAAKgmE,iBAAiBnyC,GAC3D,IAEPA,aAAmBoqC,QACrBr2B,EAAQ8jB,UAAY,GACpB9jB,EAAQjT,YAAYd,QAEjB,GAAep0B,MAAXo0B,EACP+T,EAAQ8jB,UAAY73B,UAGI,cAAlB7zB,KAAKuT,KAAK2jB,WAA8Cz3B,IAAtBO,KAAKuT,KAAKsgB,cAC1C,IAAIzwB,mDAA4CpD,KAAKozB,UAG1DS,QAAUA,iDAUC+T,MACjB5nC,KAAKmvB,QAAQ82C,gBAAkBjmE,KAAKmvB,QAAQ82C,eAAejpE,OAAS,EAAG,KACrEkpE,EAAa,MAEbhjC,GAAcljC,KAAKmvB,QAAQ82C,gBAC7BC,EAAalmE,KAAKmvB,QAAQ82C,mBAEvB,CAAA,GAAmC,OAA/BjmE,KAAKmvB,QAAQ82C,sBACpBC,EAAalmB,GAAYhgD,KAAKuT,gDAMb2yD,+BAAY,KAApBziE,UACH3B,EAAQ9B,KAAKuT,KAAK9P,GAEX,MAAT3B,EACF8lC,EAAQ4oB,4BAAqB/sD,GAAQ3B,GAGrC8lC,EAAQu+B,+BAAwB1iE,8HAW1BmkC,GAEP5nC,KAAKw0B,QACP8jB,GAAK8lB,cAAcx2B,EAAS5nC,KAAKw0B,YAC5BA,MAAQ,MAIXx0B,KAAKuT,KAAKihB,QACZ8jB,GAAK+lB,WAAWz2B,EAAS5nC,KAAKuT,KAAKihB,YAC9BA,MAAQx0B,KAAKuT,KAAKihB,gDAUVX,SACQ,iBAAZA,EAA6BA,EACpCA,GAAW,cAAeA,EAAgBA,EAAQuyC,UAC/CvyC,8CAOH7zB,KAAKmvB,UAC6B,kBAA1BnvB,KAAKmvB,QAAQmhC,cAChBA,SAAW,CACdkU,WAAYxkE,KAAKmvB,QAAQmhC,SACzB+V,YAAarmE,KAAKmvB,QAAQmhC,SAC1Bra,OAAQj2C,KAAKmvB,QAAQmhC,UAEkB,WAAjC7qB,GAAOzlC,KAAKmvB,QAAQmhC,iBACrBA,SAAW,GAChBhY,GAAKmG,gBAAgB,CAAC,aAAc,cAAe,UAAWz+C,KAAKswD,SAAUtwD,KAAKmvB,QAAQmhC,YAI3FtwD,KAAKmvB,SAAanvB,KAAKmvB,QAAQmhC,WAAsD,IAAxCtwD,KAAKmvB,QAAQmhC,SAASyU,eAClE/kE,KAAKuT,OAC2B,kBAAvBvT,KAAKuT,KAAK+8C,cACdA,SAAW,CACdkU,WAAYxkE,KAAKuT,KAAK+8C,SACtB+V,YAAarmE,KAAKuT,KAAK+8C,SACvBra,OAAQj2C,KAAKuT,KAAK+8C,UAEmB,WAA9B7qB,GAAOzlC,KAAKuT,KAAK+8C,iBAGrBA,SAAW,GAChBhY,GAAKmG,gBAAgB,CAAC,aAAc,cAAe,UAAWz+C,KAAKswD,SAAUtwD,KAAKuT,KAAK+8C,0DAWtF,iDAQA,kDAQHtwD,KAAKmvB,QAAQm3C,SAAWtmE,KAAKmvB,QAAQm3C,QAAQd,SACtBzvB,UAAK5mB,QAAQm3C,QAAQd,iBAAcxlE,KACrDg+D,CAAiBh+D,KAAK4lE,eAAgB5lE,KAAKuT,MAG7CvT,KAAKuT,KAAKk8C,eAIrBsU,GAAK5nE,UAAUkH,OAAQ,MCrkBjBkjE,0BASQhzD,EAAMmoC,EAAYvsB,kDACtB5b,EAAMmoC,EAAYvsB,KACnBkmB,MAAQ,CACXmxB,IAAK,CACH59B,MAAO,EACPC,OAAQ,GAEV2hB,KAAM,CACJ5hB,MAAO,EACPC,OAAQ,IAIRt1B,GACgB9T,MAAd8T,EAAKitB,YACD,IAAIp9B,iDAA0CmQ,kEAUhDmmC,MACJ15C,KAAK8/D,eACA,MAGLnD,EACE8J,EAAQzmE,KAAKuT,KAAKkzD,OAASzmE,KAAKmvB,QAAQs3C,MACxCC,EAAY1mE,KAAK4oC,MAAQ8Q,EAAMitB,iCAGnChK,EADW,SAAT8J,EACWzmE,KAAKuT,KAAKitB,MAAMlhC,UAAYo6C,EAAMlZ,OAAWxgC,KAAKuT,KAAKitB,MAAMlhC,UAAYonE,EAAYhtB,EAAMjZ,IAExF,QAATgmC,EACMzmE,KAAKuT,KAAKitB,MAAMlhC,UAAYonE,EAAYhtB,EAAMlZ,OAAWxgC,KAAKuT,KAAKitB,MAAMlhC,UAAYo6C,EAAMjZ,IAI3FzgC,KAAKuT,KAAKitB,MAAMlhC,UAAYonE,EAAY,EAAIhtB,EAAMlZ,OAAYxgC,KAAKuT,KAAKitB,MAAMlhC,UAAYonE,EAAU,EAAIhtB,EAAMjZ,IAEtHk8B,8CAQF38D,KAAKo+C,WAEHA,IAAM,QAGNA,IAAIymB,IAAMz3C,SAASE,cAAc,YAGjC8wB,IAAIvqB,QAAUzG,SAASE,cAAc,YACrC8wB,IAAIvqB,QAAQyU,UAAY,wBACxB8V,IAAIymB,IAAIlwC,YAAY30B,KAAKo+C,IAAIvqB,cAG7BuqB,IAAIoM,KAAOp9B,SAASE,cAAc,YAClC8wB,IAAIoM,KAAKliB,UAAY,gBAGrB8V,IAAIooB,IAAMp5C,SAASE,cAAc,YACjC8wB,IAAIooB,IAAIl+B,UAAY,eAGpB8V,IAAIymB,IAAI,YAAc7kE,UAEtB0+D,OAAQ,mDASV1+D,KAAK4wB,aACF,IAAIxtB,MAAM,8CAEbpD,KAAKo+C,IAAIymB,IAAIt9B,WAAY,KACtBygB,EAAahoD,KAAK4wB,OAAOwtB,IAAI4J,eAC9BA,EAAY,MAAM,IAAI5kD,MAAM,kEACjC4kD,EAAWrzB,YAAY30B,KAAKo+C,IAAIymB,SAE7B7kE,KAAKo+C,IAAIoM,KAAKjjB,WAAY,KACzBqhB,EAAa5oD,KAAK4wB,OAAOwtB,IAAIwK,eAC5BA,EAAY,MAAM,IAAIxlD,MAAM,kEACjCwlD,EAAWj0B,YAAY30B,KAAKo+C,IAAIoM,UAE7BxqD,KAAKo+C,IAAIooB,IAAIj/B,WAAY,KACtB+gB,EAAOtoD,KAAK4wB,OAAOwtB,IAAIkK,SACxBM,EAAY,MAAM,IAAIxlD,MAAM,4DACjCklD,EAAK3zB,YAAY30B,KAAKo+C,IAAIooB,UAEvB7H,WAAY,yDAYb3+D,KAAK0+D,MAAO,MACTkI,gBAAgB5mE,KAAKo+C,IAAIvqB,cACzBgzC,sBAAsB7mE,KAAKo+C,IAAIymB,UAC/BiC,aAAa9mE,KAAKo+C,IAAIymB,SAErBvU,EAAYtwD,KAAKswD,SAASkU,YAAcxkE,KAAKswD,SAAS+V,YAGtD/9B,GAAatoC,KAAKuT,KAAK+0B,UAAY,IAAMtoC,KAAKuT,KAAK+0B,UAAY,KAClEtoC,KAAKgkE,SAAW,gBAAkB,KAClC1T,EAAW,gBAAkB,sBAC3BlS,IAAIymB,IAAIv8B,oCAA+BA,QACvC8V,IAAIoM,KAAKliB,qCAAgCA,QACzC8V,IAAIooB,IAAIl+B,oCAA+BA,2DAUvC,CACLyE,SAAU,CACR4H,MAAO30C,KAAKo+C,IAAIymB,IAAIrwC,MAAMmgB,MAC1BD,KAAM10C,KAAKo+C,IAAIymB,IAAIrwC,MAAMkgB,MAE3B8xB,IAAK,CACH39B,OAAQ7oC,KAAKo+C,IAAIooB,IAAI9P,aACrB9tB,MAAO5oC,KAAKo+C,IAAIooB,IAAIpd,aAEtBoB,KAAM,CACJ5hB,MAAO5oC,KAAKo+C,IAAIoM,KAAKpB,aAEvByb,IAAK,CACHj8B,MAAO5oC,KAAKo+C,IAAIymB,IAAIzb,YACpBvgB,OAAQ7oC,KAAKo+C,IAAIymB,IAAInO,iEAUDqQ,GACpB/mE,KAAKmvB,QAAQguB,SACViB,IAAIymB,IAAIrwC,MAAMmgB,MAAQ,WAEtByJ,IAAIymB,IAAIrwC,MAAMkgB,KAAO,WAIvBW,MAAMmxB,IAAI39B,OAASk+B,EAAMP,IAAI39B,YAC7BwM,MAAMmxB,IAAI59B,MAAQm+B,EAAMP,IAAI59B,WAC5ByM,MAAMmV,KAAK5hB,MAAQm+B,EAAMvc,KAAK5hB,WAC9BA,MAAQm+B,EAAMlC,IAAIj8B,WAClBC,OAASk+B,EAAMlC,IAAIh8B,OAGpB7oC,KAAKmvB,QAAQguB,SACViB,IAAIymB,IAAIrwC,MAAMmgB,MAAQoyB,EAAMh6B,SAAS4H,WAErCyJ,IAAIymB,IAAIrwC,MAAMkgB,KAAOqyB,EAAMh6B,SAAS2H,UAGtCgqB,OAAQ,wDAQRsI,gCAAgChnE,KAAKo+C,IAAIymB,UACzCoC,0BACAC,qBAAqBlnE,KAAKo+C,IAAIymB,oCAQ9B/D,eACDiG,EA8BExuD,SA7BA85C,EAAQ,WAEP8U,0BAAuBnnE,gBAGvBonE,0BAAuBpnE,gBAGvBqnE,kCAA+BrnE,MAEpC,WACM21D,EAAK+I,QACPqI,EAAQpR,EAAK2R,2BAIjB,iBACM3R,EAAK+I,YACP/I,EAAK4R,kCAA+B5R,GAAMoR,cAKzCS,+BAA4BxnE,cAG/B8gE,EACKzO,MAGPA,QAAAA,GAAc,SAAAx1D,GACZ0b,EAAS1b,OAEJ0b,gCAUNuoD,OACE9gE,KAAK2+D,iBACD3+D,KAAKqwD,OAAOyQ,qCAQjB9gE,KAAK2+D,UAAW,KACZvgB,EAAMp+C,KAAKo+C,IAEbA,EAAIymB,IAAI5uB,OAAQmI,EAAIymB,IAAI5uB,SACnBmI,EAAIymB,IAAIt9B,YAAY6W,EAAIymB,IAAIt9B,WAAWC,YAAY4W,EAAIymB,KAE5DzmB,EAAIoM,KAAKvU,OAAQmI,EAAIoM,KAAKvU,SACrBmI,EAAIoM,KAAKjjB,YAAY6W,EAAIoM,KAAKjjB,WAAWC,YAAY4W,EAAIoM,MAE9DpM,EAAIooB,IAAIvwB,OAAQmI,EAAIooB,IAAIvwB,SACnBmI,EAAIooB,IAAIj/B,YAAY6W,EAAIooB,IAAIj/B,WAAWC,YAAY4W,EAAIooB,UAE3D7H,WAAY,8CAQbxhB,EAAMn9C,KAAKmvB,QAAQguB,IAEnBsqB,EAAe,SAAC7/B,EAASz9B,EAAGZ,SAAG4zC,kEACzB19C,IAAN0K,QAAyB1K,IAAN8J,OAEjBsiD,EAAa1O,GAAY,EAALhzC,EAAUA,EAcpCy9B,EAAQpT,MAAMkiB,eAXJj3C,IAAN8J,OAMM9J,IAAN0K,2BAKmC0hD,kBAAiBtiD,8BAJdA,8BANAsiD,WAY5C4b,EAAaznE,KAAKo+C,IAAIymB,IAAK7kE,KAAK0nE,KAAM1nE,KAAK2nE,KAAMxqB,GACjDsqB,EAAaznE,KAAKo+C,IAAIooB,IAAKxmE,KAAK4nE,KAAM5nE,KAAK6nE,KAAM1qB,GACjDsqB,EAAaznE,KAAKo+C,IAAIoM,KAAMxqD,KAAK8nE,MAAO9nE,KAAK+nE,MAAO5qB,6CAQ9C3c,EAAQxgC,KAAK07C,WAAWF,SAASx7C,KAAKuT,KAAKitB,OAC3CimC,OAA4BhnE,IAApBO,KAAKuT,KAAKkzD,MAAsBzmE,KAAKmvB,QAAQs3C,MAAQzmE,KAAKuT,KAAKkzD,MACvEuB,EAAYhoE,KAAKq1C,MAAMmV,KAAK5hB,MAC5Bq/B,EAAWjoE,KAAKq1C,MAAMmxB,IAAI59B,MAEnB,SAAT69B,QAEGiB,KAAOlnC,EAAQxgC,KAAK4oC,WACpBk/B,MAAQtnC,EAAQwnC,OAChBJ,KAAOpnC,EAAQwnC,EAAY,EAAIC,EAAW,GAE/B,QAATxB,QAEFiB,KAAOlnC,OACPsnC,MAAQtnC,OACRonC,KAAOpnC,EAAQwnC,EAAY,EAAIC,EAAW,SAI1CP,KAAOlnC,EAAQxgC,KAAK4oC,MAAQ,OAC5Bk/B,MAAQ9nE,KAAKmvB,QAAQguB,IAAM3c,EAAQwnC,EAAYxnC,EAAQwnC,EAAY,OACnEJ,KAAOpnC,EAAQynC,EAAW,GAG7BjoE,KAAKmvB,QAAQguB,IACfn9C,KAAK20C,MAAQ30C,KAAK0nE,KAElB1nE,KAAK00C,KAAO10C,KAAK0nE,UAEdD,yDAQCpf,EAAcroD,KAAKmvB,QAAQk5B,YAAYtQ,KACvCmwB,EAAYloE,KAAKo+C,IAAIoM,KAAKh2B,SAEb,OAAf6zB,EAAsB,KAClB8f,EAAanoE,KAAK4wB,OAAOkyB,IAAM9iD,KAAK8iD,IAAM,OAE3C6kB,KAAO3nE,KAAK8iD,KAAO,EACxBolB,EAAUr/B,iBAAYs/B,QACtBD,EAAUrf,OAAS,GACnBqf,EAAUplB,IAAM,QAEb,KAEGqlB,EADgBnoE,KAAK4wB,OAAOuhC,QAAQ9c,MAAMxM,OACb7oC,KAAK4wB,OAAOkyB,IAAM9iD,KAAK4wB,OAAOiY,OAAS7oC,KAAK8iD,SAE1E6kB,KAAO3nE,KAAK4wB,OAAOiY,OAAS7oC,KAAK8iD,KAAO9iD,KAAK6oC,QAAU,GAC5Dq/B,EAAUr/B,iBAAYs/B,QACtBD,EAAUplB,IAAM,GAChBolB,EAAUrf,OAAS,SAGhBgf,MAAQ7nE,KAAKq1C,MAAMmxB,IAAI39B,OAAS,OAEhC4+B,6DAQEznE,KAAK4oC,MAAQ,iDAQb5oC,KAAK4oC,MAAQ,SAvYFm7B,ICAhBqE,0BASQ70D,EAAMmoC,EAAYvsB,kDACtB5b,EAAMmoC,EAAYvsB,KACnBkmB,MAAQ,CACXmxB,IAAK,CACH1jB,IAAK,EACLla,MAAO,EACPC,OAAQ,GAEVhV,QAAS,CACPgV,OAAQ,EACRw/B,WAAY,EACZC,YAAa,IAIb/0D,GACgB9T,MAAd8T,EAAKitB,YACD,IAAIp9B,iDAA0CmQ,kEAUhDmmC,MACJ15C,KAAK8/D,eACA,MAGH4G,EAAY1mE,KAAK4oC,MAAQ8Q,EAAMitB,iCAE7B3mE,KAAKuT,KAAKitB,MAAMlhC,UAAYonE,EAAYhtB,EAAMlZ,OAAYxgC,KAAKuT,KAAKitB,MAAQkZ,EAAMjZ,gDAQrFzgC,KAAKo+C,WAEHA,IAAM,QAGNA,IAAIte,MAAQ1S,SAASE,cAAc,YAInC8wB,IAAIvqB,QAAUzG,SAASE,cAAc,YACrC8wB,IAAIvqB,QAAQyU,UAAY,wBACxB8V,IAAIte,MAAMnL,YAAY30B,KAAKo+C,IAAIvqB,cAG/BuqB,IAAIooB,IAAMp5C,SAASE,cAAc,YACjC8wB,IAAIte,MAAMnL,YAAY30B,KAAKo+C,IAAIooB,UAG/BpoB,IAAIte,MAAM,YAAc9/B,UAExB0+D,OAAQ,mDASV1+D,KAAK4wB,aACF,IAAIxtB,MAAM,8CAEbpD,KAAKo+C,IAAIte,MAAMyH,WAAY,KACxBygB,EAAahoD,KAAK4wB,OAAOwtB,IAAI4J,eAC9BA,QACG,IAAI5kD,MAAM,kEAElB4kD,EAAWrzB,YAAY30B,KAAKo+C,IAAIte,YAE7B6+B,WAAY,yDAYb3+D,KAAK0+D,MAAO,MACTkI,gBAAgB5mE,KAAKo+C,IAAIvqB,cACzBgzC,sBAAsB7mE,KAAKo+C,IAAIte,YAC/BgnC,aAAa9mE,KAAKo+C,IAAIte,WAErBwwB,EAAYtwD,KAAKswD,SAASkU,YAAcxkE,KAAKswD,SAAS+V,YAEtD/9B,GAAatoC,KAAKuT,KAAK+0B,UAAY,IAAMtoC,KAAKuT,KAAK+0B,UAAY,KAChEtoC,KAAKgkE,SAAW,gBAAkB,KAClC1T,EAAW,gBAAkB,sBAC7BlS,IAAIte,MAAMwI,sCAAkCA,QAC5C8V,IAAIooB,IAAIl+B,oCAAgCA,2DAUxC,CACLk+B,IAAM,CACJ59B,MAAO5oC,KAAKo+C,IAAIooB,IAAIpd,YACpBvgB,OAAQ7oC,KAAKo+C,IAAIooB,IAAI9P,cAEvB7iC,QAAS,CACP+U,MAAO5oC,KAAKo+C,IAAIvqB,QAAQu1B,YACxBvgB,OAAQ7oC,KAAKo+C,IAAIvqB,QAAQ6iC,cAE3B52B,MAAO,CACL8I,MAAO5oC,KAAKo+C,IAAIte,MAAMspB,YACtBvgB,OAAQ7oC,KAAKo+C,IAAIte,MAAM42B,iEAUHqQ,QAEnB1xB,MAAMmxB,IAAI59B,MAAQm+B,EAAMP,IAAI59B,WAC5ByM,MAAMmxB,IAAI39B,OAASk+B,EAAMP,IAAI39B,YAC7BwM,MAAMxhB,QAAQgV,OAASk+B,EAAMlzC,QAAQgV,OAGtC7oC,KAAKmvB,QAAQguB,SACViB,IAAIvqB,QAAQW,MAAM8zC,sBAAiB,EAAItoE,KAAKq1C,MAAMmxB,IAAI59B,iBAEtDwV,IAAIvqB,QAAQW,MAAM6zC,qBAAgB,EAAIroE,KAAKq1C,MAAMmxB,IAAI59B,iBAKvDA,MAAQm+B,EAAMjnC,MAAM8I,WACpBC,OAASk+B,EAAMjnC,MAAM+I,YAGrBuV,IAAIooB,IAAIhyC,MAAMsuB,eAAU9iD,KAAK6oC,OAAS7oC,KAAKq1C,MAAMmxB,IAAI39B,QAAU,YAE9Do/B,EAAWjoE,KAAKq1C,MAAMmxB,IAAI59B,MAC1B2/B,EAAavoE,KAAKmvB,QAAQguB,IAAO8qB,EAAW,GAAM,EAAIA,EAAW,OAClE7pB,IAAIooB,IAAIhyC,MAAMkiB,+BAA0B6xB,aACxC7J,OAAQ,wDAQRsI,gCAAgChnE,KAAKo+C,IAAIte,YACzCmnC,0BACAC,qBAAqBlnE,KAAKo+C,IAAIte,sCAQ9BghC,eACDiG,EA8BExuD,SA7BA85C,EAAQ,WAEP8U,0BAAuBnnE,gBAGvBonE,0BAAuBpnE,gBAGvBqnE,kCAA+BrnE,MAEpC,WACM21D,EAAK+I,QACPqI,EAAQpR,EAAK2R,2BAIjB,iBACM3R,EAAK+I,YACP/I,EAAK4R,kCAA+B5R,GAAMoR,cAKzCS,+BAA4BxnE,cAG/B8gE,EACKzO,MAGPA,QAAAA,GAAc,SAAAx1D,GACZ0b,EAAS1b,OAEJ0b,8CASH4kC,EAAMn9C,KAAKmvB,QAAQguB,IAEnBsqB,EAAe,SAAC7/B,EAASz9B,EAAGZ,SAAG4zC,kEACzB19C,IAAN0K,QAAyB1K,IAAN8J,OAEjBsiD,EAAa1O,GAAY,EAALhzC,EAAUA,EAcpCy9B,EAAQpT,MAAMkiB,eAXJj3C,IAAN8J,OAMM9J,IAAN0K,2BAKmC0hD,kBAAiBtiD,8BAJdA,8BANAsiD,WAY5C4b,EAAaznE,KAAKo+C,IAAIte,MAAO9/B,KAAKwoE,OAAQxoE,KAAKyoE,OAAQtrB,gCASpD2jB,OACE9gE,KAAK2+D,iBACD3+D,KAAKqwD,OAAOyQ,kCAQjB9gE,KAAK2+D,YACH3+D,KAAKo+C,IAAIte,MAAMyH,iBACZ6W,IAAIte,MAAMyH,WAAWC,YAAYxnC,KAAKo+C,IAAIte,YAG5C6+B,WAAY,6CASbn+B,EAAQxgC,KAAK07C,WAAWF,SAASx7C,KAAKuT,KAAKitB,YAE5CgoC,OAAShoC,EACVxgC,KAAKmvB,QAAQguB,SACVxI,MAAQnU,EAAQxgC,KAAKq1C,MAAMmxB,IAAI59B,WAE/B8L,KAAOlU,EAAQxgC,KAAKq1C,MAAMmxB,IAAI59B,WAGhC6+B,yDAQCpf,EAAcroD,KAAKmvB,QAAQk5B,YAAYtQ,UAEtC0wB,OADY,OAAfpgB,EACYroD,KAAK8iD,IAGL9iD,KAAK4wB,OAAOiY,OAAS7oC,KAAK8iD,IAAM9iD,KAAK6oC,YAGhD4+B,6DAQEznE,KAAKq1C,MAAMmxB,IAAI59B,qDAQf5oC,KAAKq1C,MAAMmxB,IAAI59B,aArUFm7B,ICAlB2E,0BASQn1D,EAAMmoC,EAAYvsB,kDACtB5b,EAAMmoC,EAAYvsB,KACnBkmB,MAAQ,CACXxhB,QAAS,CACP+U,MAAO,MAGNxqC,UAAW,EAEZmV,EAAM,IACU9T,MAAd8T,EAAKitB,YACD,IAAIp9B,iDAA0CmQ,EAAK6f,QAE3C3zB,MAAZ8T,EAAKktB,UACD,IAAIr9B,+CAAwCmQ,EAAK6f,mEAWnDsmB,UACJ15C,KAAK8/D,UAID9/D,KAAKuT,KAAKitB,MAAQkZ,EAAMjZ,KAASzgC,KAAKuT,KAAKktB,IAAMiZ,EAAMlZ,mDAQ1DxgC,KAAKo+C,WAEHA,IAAM,QAGNA,IAAIymB,IAAMz3C,SAASE,cAAc,YAIjC8wB,IAAIuqB,MAAQv7C,SAASE,cAAc,YACnC8wB,IAAIuqB,MAAMrgC,UAAY,yBACtB8V,IAAIymB,IAAIlwC,YAAY30B,KAAKo+C,IAAIuqB,YAG7BvqB,IAAIwqB,aAAex7C,SAASE,cAAc,YAC1C8wB,IAAIwqB,aAAatgC,UAAY,8BAC7B8V,IAAIymB,IAAIlwC,YAAY30B,KAAKo+C,IAAIwqB,mBAG7BxqB,IAAIvqB,QAAUzG,SAASE,cAAc,YACrC8wB,IAAIvqB,QAAQyU,UAAY,wBACxB8V,IAAIuqB,MAAMh0C,YAAY30B,KAAKo+C,IAAIvqB,cAG/BuqB,IAAIymB,IAAI,YAAc7kE,UAEtB0+D,OAAQ,mDAUV1+D,KAAK4wB,aACF,IAAIxtB,MAAM,8CAEbpD,KAAKo+C,IAAIymB,IAAIt9B,WAAY,KACtBygB,EAAahoD,KAAK4wB,OAAOwtB,IAAI4J,eAC9BA,QACG,IAAI5kD,MAAM,kEAElB4kD,EAAWrzB,YAAY30B,KAAKo+C,IAAIymB,UAE7BlG,WAAY,yDAYb3+D,KAAK0+D,MAAO,MACTkI,gBAAgB5mE,KAAKo+C,IAAIvqB,cACzBgzC,sBAAsB7mE,KAAKo+C,IAAIymB,UAC/BiC,aAAa9mE,KAAKo+C,IAAIymB,SAErBvU,EAAYtwD,KAAKswD,SAASkU,YAAcxkE,KAAKswD,SAAS+V,YAGtD/9B,GAAatoC,KAAKuT,KAAK+0B,UAAa,IAAMtoC,KAAKuT,KAAK+0B,UAAa,KAClEtoC,KAAKgkE,SAAW,gBAAkB,KAClC1T,EAAW,gBAAkB,sBAC7BlS,IAAIymB,IAAIv8B,UAAYtoC,KAAK6oE,cAAgBvgC,OAIzC8V,IAAIvqB,QAAQW,MAAMs0C,SAAW,qEAW/B1qE,SAAgE,WAArD4tB,OAAO+8C,iBAAiB/oE,KAAKo+C,IAAIuqB,OAAOvqE,cACnD4qE,WAAsE,WAAzDh9C,OAAO+8C,iBAAiB/oE,KAAKo+C,IAAIvqB,SAASm1C,WACrD,CACLn1C,QAAS,CACP+U,MAAO5oC,KAAKo+C,IAAIvqB,QAAQu1B,aAE1Byb,IAAK,CACHh8B,OAAQ7oC,KAAKo+C,IAAIymB,IAAInO,iEAUDqQ,QACnB1xB,MAAMxhB,QAAQ+U,MAAQm+B,EAAMlzC,QAAQ+U,WACpCC,OAASk+B,EAAMlC,IAAIh8B,YACnBuV,IAAIvqB,QAAQW,MAAMs0C,SAAW,QAC7BpK,OAAQ,wDAQRsI,gCAAgChnE,KAAKo+C,IAAIymB,UACzCqC,qBAAqBlnE,KAAKo+C,IAAIymB,UAC9BoC,0BACAgC,wBACAC,mDAQApI,eACDiG,EA8BExuD,SA7BA85C,EAAQ,WAEP8U,0BAAuBnnE,gBAGvBonE,0BAAuBpnE,gBAGvBqnE,kCAA+BrnE,MAEpC,iBACM21D,EAAK+I,QACPqI,EAAQhxB,KAAA4f,EAAK2R,+BAA4B3R,EAAjC5f,KAIZ,iBACM4f,EAAK+I,YACP/I,EAAK4R,kCAA+B5R,GAAMoR,cAKzCS,+BAA4BxnE,cAG/B8gE,EACKzO,MAGPA,QAAAA,GAAc,SAAAx1D,GACZ0b,EAAS1b,OAEJ0b,gCAUNuoD,OACE9gE,KAAK2+D,iBACD3+D,KAAKqwD,OAAOyQ,qCAQjB9gE,KAAK2+D,UAAW,KACZkG,EAAM7kE,KAAKo+C,IAAIymB,IAEjBA,EAAIt9B,YACNs9B,EAAIt9B,WAAWC,YAAYq9B,QAGxBlG,WAAY,uCAaTwK,OAKNC,EACAC,EALEC,EAActpE,KAAK4wB,OAAOgY,MAC5BpI,EAAQxgC,KAAK07C,WAAWF,SAASx7C,KAAKuT,KAAKitB,OAC3CC,EAAMzgC,KAAK07C,WAAWF,SAASx7C,KAAKuT,KAAKktB,KACvCgmC,OAA4BhnE,IAApBO,KAAKuT,KAAKkzD,MAAsBzmE,KAAKmvB,QAAQs3C,MAAQzmE,KAAKuT,KAAKkzD,OAMjD,IAAxBzmE,KAAKuT,KAAK41D,gBAAsC1pE,IAAd0pE,IAAyC,IAAdA,IAC3D3oC,GAAS8oC,IACX9oC,GAAS8oC,GAEP7oC,EAAM,EAAI6oC,IACZ7oC,EAAM,EAAI6oC,QAKRC,EAAW/nE,KAAK8D,IAAI9D,KAAK+a,MAAsB,KAAfkkB,EAAMD,IAAiB,IAAM,UAE/DxgC,KAAK5B,UACH4B,KAAKmvB,QAAQguB,SACVxI,MAAQnU,OAERkU,KAAOlU,OAEToI,MAAQ2gC,EAAWvpE,KAAKq1C,MAAMxhB,QAAQ+U,MAC3CygC,EAAerpE,KAAKq1C,MAAMxhB,QAAQ+U,QAO9B5oC,KAAKmvB,QAAQguB,SACVxI,MAAQnU,OAERkU,KAAOlU,OAEToI,MAAQ2gC,EACbF,EAAe7nE,KAAKY,IAAIq+B,EAAMD,EAAOxgC,KAAKq1C,MAAMxhB,QAAQ+U,QAGtD5oC,KAAKmvB,QAAQguB,SACViB,IAAIymB,IAAIrwC,MAAMkiB,gCAAwC,EAAd12C,KAAK20C,kBAE7CyJ,IAAIymB,IAAIrwC,MAAMkiB,+BAA0B12C,KAAK00C,iBAE/C0J,IAAIymB,IAAIrwC,MAAMoU,gBAAW2gC,QAC1BvpE,KAAKgpE,kBACAngC,OAAS7oC,KAAKo+C,IAAIymB,IAAInO,cAGvB+P,OACD,YACEroB,IAAIvqB,QAAQW,MAAMkiB,UAAY,0BAGhC,WACC12C,KAAKmvB,QAAQguB,IAAK,KACdorB,GAAuD,EAA1C/mE,KAAK8D,IAAKikE,EAAWF,EAAe,QAClDjrB,IAAIvqB,QAAQW,MAAMkiB,+BAA0B6xB,mBAE5CnqB,IAAIvqB,QAAQW,MAAMkiB,+BAA0Bl1C,KAAK8D,IAAKikE,EAAWF,EAAe,oBAIpF,YACCrpE,KAAKmvB,QAAQguB,IAAK,KACdorB,GAA2D,EAA9C/mE,KAAK8D,KAAKikE,EAAWF,GAAgB,EAAG,QACtDjrB,IAAIvqB,QAAQW,MAAMkiB,+BAA0B6xB,mBAE5CnqB,IAAIvqB,QAAQW,MAAMkiB,+BAA0Bl1C,KAAK8D,KAAKikE,EAAWF,GAAgB,EAAG,2BASvFD,EAFAppE,KAAK5B,SACHqiC,EAAM,EACej/B,KAAK8D,KAAKk7B,EAAO,IAGhB6oC,EAItB7oC,EAAQ,GACcA,EAGD,EAGvBxgC,KAAKmvB,QAAQguB,IAAK,KACdorB,GAAqC,EAAxBa,OACdhrB,IAAIvqB,QAAQW,MAAMkiB,+BAA0B6xB,mBAE5CnqB,IAAIvqB,QAAQW,MAAMkiB,+BAA0B0yB,oDAWjD/gB,EAAcroD,KAAKmvB,QAAQk5B,YAAYtQ,KACvC8sB,EAAM7kE,KAAKo+C,IAAIymB,IAGnBA,EAAIrwC,MAAMsuB,cADO,OAAfuF,EACiBroD,KAAK8iD,IAGL9iD,KAAK4wB,OAAOiY,OAAS7oC,KAAK8iD,IAAM9iD,KAAK6oC,4DASrD7oC,KAAKgkE,UAAYhkE,KAAKmvB,QAAQq6C,qBAAqB9vB,QAAU15C,KAAKswD,SAASkU,aAAexkE,KAAKo+C,IAAI0mB,SAAU,KAE1GA,EAAW13C,SAASE,cAAc,OACxCw3C,EAASx8B,UAAY,gBACrBw8B,EAAS2E,aAAezpE,UAEnBo+C,IAAIymB,IAAIlwC,YAAYmwC,QACpB1mB,IAAI0mB,SAAWA,OAEZ9kE,KAAKgkE,UAAahkE,KAAKmvB,QAAQq6C,qBAAqB9vB,QAAS15C,KAAKo+C,IAAI0mB,WAE1E9kE,KAAKo+C,IAAI0mB,SAASv9B,iBACf6W,IAAI0mB,SAASv9B,WAAWC,YAAYxnC,KAAKo+C,IAAI0mB,eAE/C1mB,IAAI0mB,SAAW,sDASjB9kE,KAAKgkE,UAAYhkE,KAAKmvB,QAAQq6C,qBAAqB9vB,QAAU15C,KAAKswD,SAASkU,aAAexkE,KAAKo+C,IAAIsrB,UAAW,KAE3GA,EAAYt8C,SAASE,cAAc,OACzCo8C,EAAUphC,UAAY,iBACtBohC,EAAUC,cAAgB3pE,UAErBo+C,IAAIymB,IAAIlwC,YAAY+0C,QACpBtrB,IAAIsrB,UAAYA,OAEb1pE,KAAKgkE,UAAahkE,KAAKmvB,QAAQq6C,qBAAqB9vB,QAAS15C,KAAKo+C,IAAIsrB,YAE1E1pE,KAAKo+C,IAAIsrB,UAAUniC,iBAChB6W,IAAIsrB,UAAUniC,WAAWC,YAAYxnC,KAAKo+C,IAAIsrB,gBAEhDtrB,IAAIsrB,UAAY,aA5ZH3F,IAiaxB2E,GAAUvsE,UAAU0sE,cAAgB,yBC/Z9Be,0BAWQr2D,EAAMmoC,EAAYvsB,kDACtB5b,EAAMmoC,EAAYvsB,KACnBkmB,MAAQ,CACXxhB,QAAS,CACP+U,MAAO,MAGNxqC,UAAW,EAGZmV,EAAM,IACU9T,MAAd8T,EAAKitB,YACD,IAAIp9B,iDAA0CmQ,EAAK6f,QAE3C3zB,MAAZ8T,EAAKktB,UACD,IAAIr9B,+CAAwCmQ,EAAK6f,mEAUnDsmB,UAEA15C,KAAKuT,KAAKitB,MAAQkZ,EAAMjZ,KAASzgC,KAAKuT,KAAKktB,IAAMiZ,EAAMlZ,kDAQ1DxgC,KAAKo+C,WAEHA,IAAM,QAGNA,IAAIymB,IAAMz3C,SAASE,cAAc,YAIjC8wB,IAAIuqB,MAAQv7C,SAASE,cAAc,YACnC8wB,IAAIuqB,MAAMrgC,UAAY,yBACtB8V,IAAIymB,IAAIlwC,YAAY30B,KAAKo+C,IAAIuqB,YAG7BvqB,IAAIvqB,QAAUzG,SAASE,cAAc,YACrC8wB,IAAIvqB,QAAQyU,UAAY,wBACxB8V,IAAIuqB,MAAMh0C,YAAY30B,KAAKo+C,IAAIvqB,cAM/B6qC,OAAQ,mDASV1+D,KAAK4wB,aACF,IAAIxtB,MAAM,8CAEbpD,KAAKo+C,IAAIymB,IAAIt9B,WAAY,KACtBqhB,EAAa5oD,KAAK4wB,OAAOwtB,IAAIwK,eAC9BA,QACG,IAAIxlD,MAAM,kEAElBwlD,EAAWj0B,YAAY30B,KAAKo+C,IAAIymB,UAE7BlG,WAAY,yDAYb3+D,KAAK0+D,MAAO,MACTkI,gBAAgB5mE,KAAKo+C,IAAIvqB,cACzBgzC,sBAAsB7mE,KAAKo+C,IAAIvqB,cAC/BizC,aAAa9mE,KAAKo+C,IAAIymB,SAGrBv8B,GAAatoC,KAAKuT,KAAK+0B,UAAa,IAAMtoC,KAAKuT,KAAK+0B,UAAa,KAClEtoC,KAAKgkE,SAAW,gBAAkB,SAClC5lB,IAAIymB,IAAIv8B,UAAYtoC,KAAK6oE,cAAgBvgC,gEAW3ClqC,SAAkE,WAAvD4tB,OAAO+8C,iBAAiB/oE,KAAKo+C,IAAIvqB,SAASz1B,SACnD,CACLy1B,QAAS,CACP+U,MAAO5oC,KAAKo+C,IAAIvqB,QAAQu1B,gEAUJ2d,QAEnB1xB,MAAMxhB,QAAQ+U,MAAQm+B,EAAMlzC,QAAQ+U,WACpCC,OAAS,OAET61B,OAAQ,mFAeRoC,eACDiG,EA6BExuD,SA5BA85C,EAAQ,WAEP8U,0BAAuBnnE,gBAGvBonE,0BAAuBpnE,MAE5B+1C,UAAKsxB,kCAA+BrnE,MAEpC,iBACM21D,EAAK+I,QACPqI,EAAQhxB,KAAA4f,EAAK2R,+BAA4B3R,EAAjC5f,KAIZ,iBACM4f,EAAK+I,YACP/I,EAAK4R,kCAA+B5R,GAAMoR,cAKzCS,+BAA4BxnE,cAG/B8gE,EACKzO,MAGPA,QAAAA,GAAc,SAAAx1D,GACZ0b,EAAS1b,OAEJ0b,uCAQC+hD,OACNzxB,EACEwf,EAAcroD,KAAKmvB,QAAQk5B,YAAYtQ,aAGlBt4C,IAAvBO,KAAKuT,KAAKwnD,SAAwB,KAE9B8O,EAAe7pE,KAAKuT,KAAKwnD,cAE1B3c,IAAIymB,IAAIrwC,MAAMqU,iBAAY7oC,KAAK4wB,OAAOuqC,UAAU0O,GAAchhC,kBAG5DuV,IAAIymB,IAAIrwC,MAAMsuB,cADF,OAAfuF,EAC0BroD,KAAK4wB,OAAOkyB,IAAM9iD,KAAK4wB,OAAOuqC,UAAU0O,GAAc/mB,IAEtD9iD,KAAK4wB,OAAOkyB,IAAM9iD,KAAK4wB,OAAOiY,OAAS7oC,KAAK4wB,OAAOuqC,UAAU0O,GAAc/mB,IAAM9iD,KAAK4wB,OAAOuqC,UAAU0O,GAAchhC,kBAE9IuV,IAAIymB,IAAIrwC,MAAMq0B,OAAS,QAKxB7oD,KAAK4wB,kBAAkBkzC,IAEzBj7B,EAASrnC,KAAK8D,IAAItF,KAAK4wB,OAAOiY,OAC1B7oC,KAAK4wB,OAAOuhC,QAAQvZ,KAAKY,SAASwF,OAAOnW,OACzC7oC,KAAK4wB,OAAOuhC,QAAQvZ,KAAKY,SAASC,gBAAgB5Q,aACjDuV,IAAIymB,IAAIrwC,MAAMq0B,OAAwB,UAAfR,EAA0B,IAAM,QACvDjK,IAAIymB,IAAIrwC,MAAMsuB,IAAqB,OAAfuF,EAAuB,IAAM,KAGtDxf,EAAS7oC,KAAK4wB,OAAOiY,YAEhBuV,IAAIymB,IAAIrwC,MAAMsuB,cAAS9iD,KAAK4wB,OAAOkyB,eACnC1E,IAAIymB,IAAIrwC,MAAMq0B,OAAS,SAG3BzK,IAAIymB,IAAIrwC,MAAMqU,iBAAYA,eArONk7B,IAyO7B6F,GAAeztE,UAAU0sE,cAAgB,0BAEzCe,GAAeztE,UAAUkH,OAAQ,EAMjCumE,GAAeztE,UAAUqnE,KAAOkF,GAAUvsE,UAAUqnE,KAMpDoG,GAAeztE,UAAU+zD,KAAOwY,GAAUvsE,UAAU+zD,KAMpD0Z,GAAeztE,UAAUskE,YAAciI,GAAUvsE,UAAUskE,qzCChQrDqJ,yBAKQ7d,EAAW8d,mBAChB9d,UAAYA,OACZ8d,eAAiBA,GAAkB,WAEnC5/D,EAAI,OACJZ,EAAI,OACJygE,QAAU,OACVzvB,QAAS,OAGTouB,MAAQv7C,SAASE,cAAc,YAC/Bq7C,MAAMrgC,UAAY,mBAClB2jB,UAAUt3B,YAAY30B,KAAK2oE,sDAOtBx+D,EAAGZ,QACRY,EAAIghD,GAAShhD,QACbZ,EAAI4hD,GAAS5hD,mCAOZsqB,GACFA,aAAmBoqC,cAChB0K,MAAMjd,UAAY,QAClBid,MAAMh0C,YAAYd,SAGlB80C,MAAMjd,UAAY73B,+BAQtBo2C,WACYxqE,IAAXwqE,IACFA,GAAS,IAGI,IAAXA,EAAiB,KACfphC,EAAS7oC,KAAK2oE,MAAM5c,aACpBnjB,EAAQ5oC,KAAK2oE,MAAMhoB,YACnB0V,EAAYr2D,KAAK2oE,MAAMphC,WAAWwkB,aAClC+c,EAAW9oE,KAAK2oE,MAAMphC,WAAWoZ,YAEjCjM,EAAO,EAAGoO,EAAM,KAEO,QAAvB9iD,KAAK+pE,gBAAmD,QAAvB/pE,KAAK+pE,eAA0B,KAC9DG,GAAS,EAAOC,GAAQ,EAED,QAAvBnqE,KAAK+pE,iBACH/pE,KAAKuJ,EAAIs/B,EAAS7oC,KAAKgqE,UACzBG,GAAQ,GAGNnqE,KAAKmK,EAAIy+B,EAAQkgC,EAAW9oE,KAAKgqE,UACnCE,GAAS,IAKXx1B,EADEw1B,EACKlqE,KAAKmK,EAAIy+B,EAET5oC,KAAKmK,EAIZ24C,EADEqnB,EACInqE,KAAKuJ,EAAIs/B,EAET7oC,KAAKuJ,OAGbu5C,EAAO9iD,KAAKuJ,EAAIs/B,GACNA,EAAS7oC,KAAKgqE,QAAU3T,IAChCvT,EAAMuT,EAAYxtB,EAAS7oC,KAAKgqE,SAE9BlnB,EAAM9iD,KAAKgqE,UACblnB,EAAM9iD,KAAKgqE,UAGbt1B,EAAO10C,KAAKmK,GACDy+B,EAAQ5oC,KAAKgqE,QAAUlB,IAChCp0B,EAAOo0B,EAAWlgC,EAAQ5oC,KAAKgqE,SAE7Bt1B,EAAO10C,KAAKgqE,UACdt1B,EAAO10C,KAAKgqE,cAIXrB,MAAMn0C,MAAMkgB,KAAOA,EAAO,UAC1Bi0B,MAAMn0C,MAAMsuB,IAAMA,EAAM,UACxB6lB,MAAMn0C,MAAMyqB,WAAa,eACzB1E,QAAS,YAGT2V,2CAQF3V,QAAS,OACTouB,MAAMn0C,MAAMkgB,KAAO,SACnBi0B,MAAMn0C,MAAMsuB,IAAM,SAClB6lB,MAAMn0C,MAAMyqB,WAAa,gDAOzB0pB,MAAMphC,WAAWC,YAAYxnC,KAAK2oE,gBCjIvCyB,GAAS5/C,GAAwC0N,MAIjDmJ,GAAgBC,GAAoB,SACpCC,GAAiBC,GAAwB,WAI3C,CAAE7R,OAAQ,QAASnP,OAAO,EAAM2P,QAASkR,KAAkBE,IAAkB,CAC7ErJ,MAAO,SAAeL,UACbuyC,GAAOpqE,KAAM63B,EAAY/7B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCV1E,OAAiBohC,GAAa,SAAS3I,MCDnCuJ,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGoM,aACNpM,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAevJ,MAASA,GAAQ6I,GCA5FspC,0BAUQ92D,EAAMmoC,EAAYvsB,wBACtBm7C,EAAkBC,GAAc,GAAI,CAACC,kBAAkB,GAAOr7C,EAAS,CAACmhC,UAAU,kCAClF/8C,EAAMmoC,EAAY4uB,KAEnBj1B,MAAQ,CACXxhB,QAAS,CACP+U,MAAO,EACPC,OAAQ,KAIPt1B,GAAwB9T,MAAhB8T,EAAKk3D,cACV,IAAIrnE,MAAM,sCAAwCmQ,EAAK6f,aAG1DA,GAAKs3C,OACLrG,MAAQ9wD,EAAK8wD,QACbsG,gBAEA9sB,QAAUH,EAAKnqC,KAAKq3D,eACpBlxB,MAAQgE,EAAKnqC,KAAKmmC,QAClBmxB,UAAW,IACXjL,WAAY,IACZrsD,KAAKqsD,WAAY,iEAQf5/D,KAAKuT,KAAKk3D,SAAWzqE,KAAKuT,KAAKk3D,QAAQztE,QAAUgD,KAAK6qE,4CAOpDr0B,QACJs0B,cAEAv3D,KAAKk3D,QAAUj0B,OAEfm0B,mBAEAI,2CAQGrxB,OACFsxB,EAAahrE,KAAKuT,KAAKktB,IAAMzgC,KAAKuT,KAAKktB,IAAMzgC,KAAKuT,KAAKitB,MAAQ,EAC/DkmC,EAAY1mE,KAAK4oC,MAAQ8Q,EAAMitB,0BAC/BlmC,EAAMj/B,KAAK8D,IAAI0lE,EAAYhrE,KAAKuT,KAAKitB,MAAMlhC,UAAYonE,UACrD1mE,KAAKuT,KAAKitB,MAAQkZ,EAAMjZ,KAASA,EAAMiZ,EAAMlZ,OAAUxgC,KAAKujE,mDAQ7D,CACL3D,WAAW,EACXxsC,GAAIpzB,KAAKozB,GACTojB,MAAOx2C,KAAKuT,KAAKijC,OAAS,GAC1BjjC,KAAMvT,KAAKuT,qCASPutD,mBACFiG,EA8BExuD,EA7BF85C,EAAQ,WAEL8U,0BAAuBnnE,gBAGvBonE,0BAAuBpnE,gBAGvBqnE,kCAA+BrnE,MAEpC+1C,KAAC,WACK/1C,KAAK0+D,QACPqI,EAAQ/mE,KAAKsnE,mCAETtnE,MAER+1C,KAAC,iBACK/1C,KAAK0+D,iBACF6I,kCAA+BvnE,MAAM+mE,YAEtC/mE,gBAGHwnE,+BAA4BxnE,cAG/B8gE,EACKzO,MAGPA,QAAAA,GAAc,SAAUx1D,GACtB0b,EAAS1b,OAEJ0b,kCAQJvY,KAAK2+D,gBACHtO,2CAQHrwD,KAAK2+D,UAAW,KACdvgB,EAAMp+C,KAAKo+C,IACXA,EAAIymB,IAAIt9B,YACV6W,EAAIymB,IAAIt9B,WAAWC,YAAY4W,EAAIymB,KAGjC7kE,KAAKmvB,QAAQ87C,aACX7sB,EAAIoM,KAAKjjB,YACX6W,EAAIoM,KAAKjjB,WAAWC,YAAY4W,EAAIoM,MAElCpM,EAAIooB,IAAIj/B,YACV6W,EAAIooB,IAAIj/B,WAAWC,YAAY4W,EAAIooB,WAGlC7H,WAAY,6CAQfn+B,EAAQxgC,KAAK07C,WAAWF,SAASx7C,KAAKuT,KAAKitB,OAC3CC,EAAMzgC,KAAKuT,KAAKktB,IAAMzgC,KAAK07C,WAAWF,SAASx7C,KAAKuT,KAAKktB,KAAO,KAChEA,OACGyqC,sBAAsB1qC,EAAOC,OAC7B,KACDgmC,OAA4BhnE,IAApBO,KAAKuT,KAAKkzD,MAAsBzmE,KAAKmvB,QAAQs3C,MAAQzmE,KAAKuT,KAAKkzD,WACtE0E,yBAAyB3qC,EAAOimC,GAGnCzmE,KAAKmvB,QAAQ87C,kBACV7sB,IAAIoM,KAAKh2B,MAAMC,QAAUz0B,KAAKorE,kBAAoB,QAAU,YAC5DhtB,IAAIooB,IAAIhyC,MAAMC,QAAUz0B,KAAKorE,kBAAoB,QAAU,OAE5DprE,KAAKorE,wBACFC,gBAAgB7qC,EAAOC,4CAUlBD,EAAOC,QAChB2d,IAAIoM,KAAKh2B,MAAMC,QAAU,aACzB2pB,IAAIooB,IAAIhyC,MAAMC,QAAU,YACvB62C,EAAkBtrE,KAAKo+C,IAAIoM,KAAKpB,YAChCmiB,EAAiBvrE,KAAKo+C,IAAIooB,IAAIpd,eAEhC3oB,EAAK,KACD+qC,EAAaF,EAAkB9qC,GAASC,EAAMD,GAAS,EACvDirC,EAAYD,EAAaD,EAAiB,EAC1CG,EAAsB1rE,KAAKmvB,QAAQguB,KAAoB,EAAdquB,EAAkBA,EAC3DG,EAAqB3rE,KAAKmvB,QAAQguB,KAAmB,EAAbsuB,EAAiBA,OAE1DrtB,IAAIoM,KAAKh2B,MAAMkiB,+BAA0Bg1B,cACzCttB,IAAIooB,IAAIhyC,MAAMkiB,+BAA0Bi1B,aACxC,KACCD,EAAsB1rE,KAAKmvB,QAAQguB,KAAgB,EAAT3c,EAAcA,EACxDmrC,EAAqB3rE,KAAKmvB,QAAQguB,KAAuC,GAA/B3c,EAAQ+qC,EAAiB,GAAY/qC,EAAQ+qC,EAAiB,OAEzGntB,IAAIoM,KAAKh2B,MAAMkiB,+BAA0Bg1B,cACzCttB,IAAIooB,IAAIhyC,MAAMkiB,+BAA0Bi1B,2DASxBnrC,EAAOimC,GAEjB,SAATA,EACEzmE,KAAKmvB,QAAQguB,UACVxI,MAAQnU,EAAQxgC,KAAK4oC,WAGrBwV,IAAIymB,IAAIrwC,MAAMmgB,MAAQ30C,KAAK20C,MAAQ,YAEnCD,KAAOlU,EAAQxgC,KAAK4oC,WAGpBwV,IAAIymB,IAAIrwC,MAAMkgB,KAAO10C,KAAK00C,KAAO,MAEtB,QAAT+xB,EACLzmE,KAAKmvB,QAAQguB,UACVxI,MAAQnU,OAGR4d,IAAIymB,IAAIrwC,MAAMmgB,MAAQ30C,KAAK20C,MAAQ,YAEnCD,KAAOlU,OAGP4d,IAAIymB,IAAIrwC,MAAMkgB,KAAO10C,KAAK00C,KAAO,MAIpC10C,KAAKmvB,QAAQguB,UACVxI,MAAQnU,EAAQxgC,KAAK4oC,MAAQ,OAG7BwV,IAAIymB,IAAIrwC,MAAMmgB,MAAQ30C,KAAK20C,MAAQ,YAEnCD,KAAOlU,EAAQxgC,KAAK4oC,MAAQ,OAG5BwV,IAAIymB,IAAIrwC,MAAMkgB,KAAO10C,KAAK00C,KAAO,oDAUtBlU,EAAOC,OACvB8oC,EAAW/nE,KAAK+a,MAAM/a,KAAK8D,IAAIm7B,EAAMD,EAAQ,GAAK,IAElDxgC,KAAKmvB,QAAQguB,SACVxI,MAAQnU,OAERkU,KAAOlU,OAGToI,MAAQpnC,KAAK8D,IAAIikE,EAAUvpE,KAAK4rE,UAAY,GAE7C5rE,KAAKmvB,QAAQguB,SACViB,IAAIymB,IAAIrwC,MAAMmgB,MAAQ30C,KAAK20C,MAAQ,UAEnCyJ,IAAIymB,IAAIrwC,MAAMkgB,KAAO10C,KAAK00C,KAAO,UAGnC0J,IAAIymB,IAAIrwC,MAAMoU,MAAQ2gC,EAAW,+CAOlClhB,EAAcroD,KAAKmvB,QAAQk5B,YAAYtQ,KACvC8sB,EAAM7kE,KAAKo+C,IAAIymB,OAEjBA,EAAIrwC,MAAMsuB,IADO,OAAfuF,GACeroD,KAAK8iD,KAAO,GAAK,MAGjB9iD,KAAK4wB,OAAOiY,OAAS7oC,KAAK8iD,IAAM9iD,KAAK6oC,QAAU,GAAK,KAGnE7oC,KAAKmvB,QAAQ87C,WAAY,IACR,OAAf5iB,OACGjK,IAAIoM,KAAKh2B,MAAMsuB,IAAS,SACxB1E,IAAIoM,KAAKh2B,MAAMqU,OAAU7oC,KAAK4wB,OAAOkyB,IAAM9iD,KAAK8iD,IAAM,EAAK,UAC3D1E,IAAIoM,KAAKh2B,MAAMq0B,OAAS,OAE1B,KAECgjB,EAAgB7rE,KAAK4wB,OAAOuhC,QAAQ9c,MAAMxM,OAC1Cs/B,EAAa0D,EAAgB7rE,KAAK4wB,OAAOkyB,IAAM9iD,KAAK4wB,OAAOiY,OAAS7oC,KAAK8iD,SACxE1E,IAAIoM,KAAKh2B,MAAMsuB,IAAU+oB,EAAgB1D,EAAc,UACvD/pB,IAAIoM,KAAKh2B,MAAMq0B,OAAS,SAG1BzK,IAAIooB,IAAIhyC,MAAMsuB,KAAQ9iD,KAAKo+C,IAAIooB,IAAI9P,aAAe,EAAK,oDASvD12D,KAAK4oC,MAAQ,iDAQb5oC,KAAK4oC,MAAQ,sCAOf63B,mBACAI,2FAOY7gE,KAAKuT,KAAKk3D,qCAAS,SAC7B3K,QAAU9/D,6FAGZuT,KAAKijC,MAAQO,UAAKxjC,KAAKk3D,iBAAY,SAAA1yB,UAAQA,EAAKxkC,aAEhDs3D,UAAW,OACXnM,OAAQ,uCAQRoN,6DACA9rE,KAAKujE,sDAIOvjE,KAAKuT,KAAKk3D,qCAAS,KAA3B1yB,iBACAA,EAAK+nB,gGAGT+K,UAAW,EAEZiB,GAAoB9rE,KAAKqkE,aACtBA,MAAMpuB,OAAOj2C,WACbqkE,MAAQ,WAGV9wD,KAAKijC,MAAQ,QACbkoB,OAAQ,iDAOTqN,uDAOEC,EAAQj1B,UAAKxjC,KAAKk3D,iBAAY,SAAA1yB,SAAS,CAC3CvX,MAAOuX,EAAKxkC,KAAKitB,MAAMjjC,UACvBkjC,IAAKsX,EAAKxkC,KAAKktB,IAAMsX,EAAKxkC,KAAKktB,IAAIljC,UAAYw6C,EAAKxkC,KAAKitB,MAAMjjC,mBAG5DgW,KAAKnR,IAAMZ,KAAKY,UAALZ,QAAYu1C,GAAAi1B,QAAAA,GAAU,SAAAnjE,UAAKrH,KAAKY,IAAIyG,EAAE23B,MAAO33B,EAAE43B,KAAO53B,EAAE23B,iBACnEjtB,KAAKjO,IAAM9D,KAAK8D,UAAL9D,QAAYu1C,GAAAi1B,QAAAA,GAAU,SAAAnjE,UAAKrH,KAAK8D,IAAIuD,EAAE23B,MAAO33B,EAAE43B,KAAO53B,EAAE23B,gBAClEyrC,EAAUl1B,UAAKxjC,KAAKk3D,iBAAY,SAAA1yB,UAAQA,EAAKiH,UAC7CktB,EAAMz1B,GAAAw1B,QAAAA,GAAe,SAACE,EAAKrqE,UAAUqqE,EAAMrqE,IAAO,GAAK9B,KAAKuT,KAAKk3D,QAAQztE,OAE3E04D,UAAKniD,KAAKk3D,iBAAa,SAAA1yB,UAAQA,EAAKxkC,KAAKktB,aAEtCltB,KAAKitB,MAAQ,IAAI9jC,KAAKsD,KAAKuT,KAAKnR,UAChCmR,KAAKktB,IAAM,IAAI/jC,KAAKsD,KAAKuT,KAAKjO,YAE9BiO,KAAKitB,MAAQ,IAAI9jC,KAAKwvE,QACtB34D,KAAKktB,IAAM,gEASdzgC,KAAKuT,KAAKk3D,SAAWzqE,KAAKuT,KAAKk3D,QAAQztE,OAClC85C,UAAKvjC,KAAKk3D,iBAAe,SAAA1yB,UAAQA,EAAK+nB,UAAYnK,KAGpD,mDAOF31D,KAAKo+C,IAAK,eAERA,IAAM,QAGNA,IAAIymB,IAAMz3C,SAASE,cAAc,YAGjC8wB,IAAIvqB,QAAUzG,SAASE,cAAc,YACrC8wB,IAAIvqB,QAAQyU,UAAY,wBAExB8V,IAAIymB,IAAIlwC,YAAY30B,KAAKo+C,IAAIvqB,SAE9B7zB,KAAKmvB,QAAQ87C,kBAEV7sB,IAAIoM,KAAOp9B,SAASE,cAAc,YAClC8wB,IAAIoM,KAAKliB,UAAY,wBACrB8V,IAAIoM,KAAKh2B,MAAMC,QAAU,YAGzB2pB,IAAIooB,IAAMp5C,SAASE,cAAc,YACjC8wB,IAAIooB,IAAIl+B,UAAY,uBACpB8V,IAAIooB,IAAIhyC,MAAMC,QAAU,QAG3Bz0B,KAAKmvB,QAAQq7C,sBACVpsB,IAAIymB,IAAIuH,WAAar2B,KAAAs0B,EAAYluE,UAAUkwE,uBAAoBrsE,WAIjEo+C,IAAIymB,IAAI,YAAc7kE,UAEtB0+D,OAAQ,mDAQV1+D,KAAK4wB,aACF,IAAIxtB,MAAM,8CAGbpD,KAAKo+C,IAAIymB,IAAIt9B,WAAY,KACtBygB,EAAahoD,KAAK4wB,OAAOwtB,IAAI4J,eAC9BA,QACG,IAAI5kD,MAAM,kEAGlB4kD,EAAWrzB,YAAY30B,KAAKo+C,IAAIymB,SAG5Bjc,EAAa5oD,KAAK4wB,OAAOwtB,IAAIwK,cAE/B5oD,KAAKmvB,QAAQ87C,WAAY,KACtBjrE,KAAKo+C,IAAIoM,KAAKjjB,WAAY,KACxBqhB,EAAY,MAAM,IAAIxlD,MAAM,kEACjCwlD,EAAWj0B,YAAY30B,KAAKo+C,IAAIoM,UAG7BxqD,KAAKo+C,IAAIooB,IAAIj/B,WAAY,KACxB+gB,EAAOtoD,KAAK4wB,OAAOwtB,IAAIkK,SACtBM,EAAY,MAAM,IAAIxlD,MAAM,4DACjCklD,EAAK3zB,YAAY30B,KAAKo+C,IAAIooB,WAIzB7H,WAAY,yDAWb3+D,KAAK0+D,MAAO,MACTkI,gBAAgB5mE,KAAKo+C,IAAIvqB,cACzBgzC,sBAAsB7mE,KAAKo+C,IAAIymB,UAC/BiC,aAAa9mE,KAAKo+C,IAAIymB,SAGrBv8B,EAAYtoC,KAAK6oE,cAAgB,KAAO7oE,KAAKuT,KAAK+0B,UAAY,IAAMtoC,KAAKuT,KAAK+0B,UAAY,KAC7FtoC,KAAKgkE,SAAW,gBAAkB,IAAM,qBACtC5lB,IAAIymB,IAAIv8B,UAAY,YAAcA,EAEnCtoC,KAAKmvB,QAAQ87C,kBACV7sB,IAAIoM,KAAKliB,UAAY,8BAAiCtoC,KAAKgkE,SAAW,gBAAkB,SACxF5lB,IAAIooB,IAAIl+B,UAAa,6BAAgCtoC,KAAKgkE,SAAW,gBAAkB,KAG1FhkE,KAAKuT,KAAKktB,WAGP2d,IAAIvqB,QAAQW,MAAMs0C,SAAW,8DAUhC/B,EAAQ,CACZh6B,SAAU,CACR4H,MAAO30C,KAAKo+C,IAAIymB,IAAIrwC,MAAMmgB,MAC1BD,KAAM10C,KAAKo+C,IAAIymB,IAAIrwC,MAAMkgB,MAE3BmwB,IAAK,CACHj8B,MAAO5oC,KAAKo+C,IAAIymB,IAAIzb,YACpBvgB,OAAQ7oC,KAAKo+C,IAAIymB,IAAInO,sBAIrB12D,KAAKmvB,QAAQ87C,aACflE,EAAMP,IAAM,CACV39B,OAAQ7oC,KAAKo+C,IAAIooB,IAAI9P,aACrB9tB,MAAO5oC,KAAKo+C,IAAIooB,IAAIpd,aAEtB2d,EAAMvc,KAAO,CACX5hB,MAAO5oC,KAAKo+C,IAAIoM,KAAKpB,cAIlB2d,oDAOiBA,GACpB/mE,KAAKmvB,QAAQguB,SACViB,IAAIymB,IAAIrwC,MAAMmgB,MAAQ,WAEtByJ,IAAIymB,IAAIrwC,MAAMkgB,KAAO,MAIvB10C,KAAKuT,KAAKktB,SAGRmrC,SAAW7E,EAAMlC,IAAIj8B,WAFrBA,MAAQm+B,EAAMlC,IAAIj8B,WAKpBC,OAASk+B,EAAMlC,IAAIh8B,OAGpB7oC,KAAKmvB,QAAQguB,SACViB,IAAIymB,IAAIrwC,MAAMmgB,MAAQoyB,EAAMh6B,SAAS4H,WAErCyJ,IAAIymB,IAAIrwC,MAAMkgB,KAAOqyB,EAAMh6B,SAAS2H,UAGtCgqB,OAAQ,wDAORsI,gCAAgChnE,KAAKo+C,IAAIymB,sDASvC7kE,KAAK4rE,UAAY5rE,KAAK4oC,QAAU5oC,KAAKuT,KAAKktB,+CAS3CjkB,EAAS,KAAMxc,KAAKuT,KAAKjO,IAAMtF,KAAKuT,KAAKnR,KAAO,QAC7C,CACLkqE,SAAUtsE,KAAKuT,KAAKnR,IAAMoa,EAC1B+vD,OAAQvsE,KAAKuT,KAAKjO,IAAMkX,qCASxBxc,KAAK69C,QAAS,OACW79C,KAAKwsE,eAAzBF,IAAAA,SAAUC,IAAAA,OAEXE,EAAU,CACdjsC,MAAO,IAAI9jC,KAAK4vE,GAChB7rC,IAAK,IAAI/jC,KAAK6vE,GACdxtB,WAAW,QAGRlB,QAAQ4C,KAAK,MAAOgsB,kDAUpBzsE,KAAKuT,YA5nBUwwD,IAgoB1BsG,GAAYluE,UAAU0sE,cAAgB,iCCpoBtC,IAGa5M,GAHK,gBAWGyQ,yBAKLva,mBACHA,QAAUA,OACVwa,OAAS,QACT3rC,MAAQ,QACRA,OAAO,GAAK,wDAUHgzB,EAAUtY,EAAYvsB,UACpB,IAAIk7C,GAAYrW,EAAUtY,EAAYvsB,oCAcjDqnB,EAAOrnB,QACPqnB,MAAQA,GAAS,QACjBo2B,aAAc,OACdC,qBAAsB,EAEvB19C,GAAWA,EAAQ09C,2BACdA,oBAAsB19C,EAAQ09C,+DASlCD,aAAc,OACdC,qBAAsB,sCAUnBC,EAAanxB,EAAOxsB,SAC2B,kBAAZA,EAAwB,GAAKA,EAAlE49C,IAAAA,SAAUC,IAAAA,gBAEXA,IACDA,EAAkB,kBAAM,IAG5BD,EAAWA,GAAY,MAEnBE,GAAS,EAETC,EAAa,KAEbvxB,EAAQ,EAAG,IACPA,GAAS,QACF,GAGXsxB,EAAQzrE,KAAKc,IAAId,KAAK+a,MAAM/a,KAAKw4C,IAAI,IAAM2B,GAASn6C,KAAKw4C,IAR3C,KASdkzB,EAAa1rE,KAAKc,IAAId,KAAK6D,IATb,EAS8B4nE,OAI5CjtE,KAAK4sE,YAAa,KACZO,EAAgBF,GAASjtE,KAAKotE,aACfptE,KAAK6sE,qBAAsBM,UAEvCE,wBACAC,oBAIRF,WAAaH,MACdM,EAAWvtE,KAAKghC,MAAMisC,OACrBM,EAAU,KAEN,IAAIC,KADTD,EAAW,GACWvtE,KAAK2sE,UACnB3sE,KAAK2sE,OAAOtvE,eAAemwE,WACrBh3B,EAAQx2C,KAAK2sE,OAAOa,GACpBC,EAAOj3B,EAAMx5C,OACfF,EAAI,EACDA,EAAI2wE,GAAM,SAET11B,EAAOvB,EAAM15C,GACb4wE,EAAY,EAGZ35D,EAAIjX,EAAI,EACLiX,GAAK,GAAMgkC,EAAKiH,OAASxI,EAAMziC,GAAGirC,OAAUkuB,EAAa,IACvD12B,EAAMziC,GAAG+rD,SAAWkN,EAAgBj1B,EAAKxkC,KAAMijC,EAAMziC,GAAGR,OACzDm6D,IAEJ35D,YAIA4G,EAAI7d,EAAI,EACL6d,EAAI67B,EAAMx5C,QAAWw5C,EAAM77B,GAAGqkC,OAASjH,EAAKiH,OAAUkuB,EAAa,GAClEF,EAAgBj1B,EAAKxkC,KAAMijC,EAAM77B,GAAGpH,OACpCm6D,IAEJ/yD,YAIA/D,EAAI22D,EAASvwE,OAAS,EACnB4Z,GAAK,GAAMmhC,EAAKiH,OAASuuB,EAAS32D,GAAGooC,OAAUkuB,GAC9Cn1B,EAAKssB,OAASkJ,EAAS32D,GAAGytD,OAAS2I,EAAgBj1B,EAAKxkC,KAAMg6D,EAAS32D,GAAGrD,OAC1Em6D,IAEJ92D,OAIA82D,EAAYX,EAAU,SAEhBthD,EAAMiiD,EAAYX,EAAW,EAC7BY,EAAe,GAIjB5vE,EAAIjB,EACD6wE,EAAa3wE,OAASyuB,GAAO1tB,EAAIy4C,EAAMx5C,QACtCgwE,EAAgBx2B,EAAMz4C,GAAGwV,KAAMijC,EAAMz4C,GAAGwV,OACxCo6D,EAAa1wE,KAAKu5C,EAAMz4C,IAE5BA,QAGEo+D,EAAUn8D,KAAKmyD,QAAQyb,WAAW71B,EAAKxkC,MACvC8wD,EAAQrkE,KAAKmyD,QAAQwa,OAAOxQ,IAAYn8D,KAAKmyD,QAAQwa,OAAO1Q,IAC9D6D,EAAU9/D,KAAK6tE,oBAAoBF,EAActJ,EAAOyI,EAAa39C,GACzEo+C,EAAStwE,KAAK6iE,GAEdhjE,GAAK2uB,cAEEssB,EAAK+nB,QACZhjE,GAAK,OAMhBkkC,MAAMisC,GAASM,SAGjBA,4CASDZ,EAAS,QACVA,OAASA,gBAGKmB,GAAc9tE,KAAKw2C,sBAAQ,KAAnCuB,OAEDy1B,EAAYz1B,EAAKnnB,OAASmnB,EAAKnnB,OAAOurC,QAAU,GAClDkI,EAAQsI,EAAOa,GACdnJ,IACDA,EAAQ,GACRsI,EAAOa,GAAanJ,GAExBA,EAAMpnE,KAAK86C,GAGPA,EAAKxkC,KAAKitB,QACNuX,EAAKxkC,KAAKktB,IAEVsX,EAAKiH,QAAUjH,EAAKxkC,KAAKitB,MAAMjjC,UAAYw6C,EAAKxkC,KAAKktB,IAAIljC,WAAa,EAGtEw6C,EAAKiH,OAASjH,EAAKxkC,KAAKitB,MAAMjjC,eAMrC,IAAIwwE,KAAoBpB,EAAQ,UAC7BA,EAAOtvE,eAAe0wE,QACtBpB,EAAOoB,YAAuB,SAAC5wE,EAAGC,UAAMD,EAAE6hD,OAAS5hD,EAAE4hD,eAIxD4tB,aAAc,8CAYHe,EAActJ,EAAOyI,EAAa39C,SAK9C2wC,EAJEkO,EAAoBj3B,KAAC+1B,GAAe,YAAQ,SAAAhN,eAAY,CAC1DA,QAAAA,EACAmO,SAAU,OAAQl3B,KAAA+oB,EAAQvsD,KAAKk3D,iBAAY,SAAA1yB,UAAQA,EAAK3kB,cAGxD46C,EAAkBhxE,OAAQ,uDACjBkxE,aACDA,EAAeD,SAAStuC,OAASguC,EAAa3wE,QAC3CmxE,GAAAR,QAAAA,GAAmB,SAAAS,UAAeF,EAAeD,SAASngD,IAAIsgD,EAAYh7C,cAC7E0sC,EAAUoO,EAAepO,sBAHNkO,+BAAmB,kBAItC,6FAKRlO,SACAA,EAAQuO,WAAWV,GACf7N,EAAQuE,QAAUA,IACdvE,EAAQuE,OACRvE,EAAQuE,MAAMpuB,OAAO6pB,GAGrBuE,IACAA,EAAMxpD,IAAIilD,GACVA,EAAQuE,MAAQA,IAGjBvE,MAGPwO,EAAgBn/C,EAAQm/C,eAAiB,GACvC5yB,EAAa,CACfF,SAAUx7C,KAAKmyD,QAAQvZ,KAAKN,KAAKkD,SACjCS,OAAQj8C,KAAKmyD,QAAQvZ,KAAKN,KAAK2D,QAG7BwT,EAAQ6e,EAAc/nE,QAAQ,UAAWonE,EAAa3wE,QACtDuxE,EAAiB,eAAiB9e,EAAQ,KAAOke,EAAa3wE,OAAS,SACvEwxE,EAAiBjE,GAAc,GAAIp7C,EAASnvB,KAAKmyD,QAAQhjC,SACzD5b,EAAO,SACEg7D,QACF9e,QACA4U,UACEsJ,eACK3tE,KAAKmyD,QAAQvZ,KAAKiF,cACzB79C,KAAKmyD,QAAQvZ,KAAKc,cAE/BomB,EAAU9/D,KAAKyuE,kBAAkBl7D,EACjCmoC,EACA8yB,GAEInK,IACAA,EAAMxpD,IAAIilD,GACVA,EAAQuE,MAAQA,GAGpBvE,EAAQiL,SAEDjL,kDAQF9+B,MAAQ,QACRosC,YAAc,OACdpsC,MAAMhhC,KAAKotE,YAAc,ssaC3RtC,IACMsB,GAAa,iBAYbC,0BAOQ/1B,EAAMzpB,kDAEXypB,KAAOA,IACPsE,eAAiB,CACpBhmB,KAAM,KACNmxB,YAAa,CACXtQ,KAAM,UAER0uB,MAAO,OACPpjE,OAAO,EACPk4D,gBAAgB,EAChBqT,wBAAeC,EAAWC,EAASnC,OAC3BoC,EAAcD,EAAQlR,MAC5BkR,EAAQlR,MAAQiR,EAAUjR,MAC1BiR,EAAUjR,MAAQmR,GAEpBC,WAAY,QAEZ9K,YAAY,EACZ+K,aAAa,EACbzX,oBAAqB,IACrBgS,qBAAsB,CACpBzxB,MAAM,EACN2B,OAAO,GAGT4W,SAAU,CACRkU,YAAY,EACZ6B,aAAa,EACbxrD,KAAK,EACLo7B,QAAQ,EACR8uB,eAAe,GAGjBpH,cAAe,CACbC,OAAO,EACP/iD,KAAK,EACLo7B,QAAQ,GAGV6a,KAAMtL,GAASsL,KAGfoe,4BAAmBC,EAAYp3B,EAAM/xC,GACnCA,EAAS+xC,IAEXq3B,eAAMr3B,EAAM/xC,GACVA,EAAS+xC,IAEXs3B,kBAASt3B,EAAM/xC,GACbA,EAAS+xC,IAEXu3B,gBAAOv3B,EAAM/xC,GACXA,EAAS+xC,IAEXw3B,kBAASx3B,EAAM/xC,GACbA,EAAS+xC,IAEXy3B,kBAASz3B,EAAM/xC,GACbA,EAAS+xC,IAEX03B,oBAAW13B,EAAM/xC,GACfA,EAAS+xC,IAEX23B,qBAAY33B,EAAM/xC,GAChBA,EAAS+xC,IAEX43B,uBAAc53B,EAAM/xC,GAClBA,EAAS+xC,IAGXuiB,OAAQ,CACNviB,KAAM,CACJ+jB,WAAY,GACZjB,SAAU,IAEZvS,KAAM,IAGRsnB,cAAc,EAEdtJ,QAAS,CACPuJ,aAAa,EACb9F,eAAgB,OAChB+F,MAAO,KAGT1K,yBAAyB,KAItBj2C,QAAUmpB,GAAKh7C,OAAO,GAAIogD,EAAKR,kBAC/B/tB,QAAQguB,IAAMhuB,EAAQguB,MACtBhuB,QAAQ6vC,UAAY7vC,EAAQ6vC,YAE5BtjB,WAAa,CAChBF,SAAU5C,EAAKN,KAAKkD,SACpBS,OAAQrD,EAAKN,KAAK2D,UAEfmC,IAAM,KACN/I,MAAQ,KACRwO,OAAS,SAERlF,iBACD8mB,UAAY,OACZsK,WAAa,OACb5Q,iBAAmB,OACnB/M,qBAAsB,IACtBoN,oBAAsB,OAEtBwQ,qBAAsB,IAGtBC,cAAgB,cACbzvB,EAAOD,EAAQ2vB,GACnBvxB,EAAGwxB,OAAO5vB,EAAO/J,OACbmI,EAAGxvB,QAAQ2wC,SACbnhB,EAAGyxB,iBAAiBjb,SAASxW,EAAGnI,MAAO,CAAEq2B,qBAAqB,IAEhEluB,EAAG0R,0BAEI7P,EAAOD,EAAQ2vB,GACtBvxB,EAAG0xB,UAAU9vB,EAAO/J,OAChBmI,EAAGxvB,QAAQ2wC,SACbnhB,EAAGyxB,iBAAiBjb,SAASxW,EAAGnI,MAAO,CAAEq2B,qBAAqB,IAEhEluB,EAAG0R,0BAEI7P,EAAOD,EAAQ2vB,GACtBvxB,EAAG2xB,UAAU/vB,EAAO/J,OAChBmI,EAAGxvB,QAAQ2wC,SACbnhB,EAAGyxB,iBAAiBjb,SAASxW,EAAGnI,MAAO,CAAEq2B,qBAAqB,IAEhEluB,EAAG0R,aAKFkgB,eAAiB,cACd/vB,EAAOD,EAAQ2vB,MACnBvxB,EAAG6xB,aAAajwB,EAAO/J,OAEnBmI,EAAGoxB,YAAcpxB,EAAGoxB,WAAW/yE,OAAS,EAAG,OACrC+yE,EAAapxB,EAAGoxB,WAAW73B,kBACjC63B,EAAWhmE,eAAc,SAAA0mE,MACrBA,EAAU3T,aAAc,OACE,GAAxB2T,EAAU1T,aACZ0T,EAAU1T,YAAa,OAErB2T,EAAgB,QACpBD,EAAU3T,sBAAqB,SAAA6T,OACvBC,EAAqBb,EAAWhmE,IAAI4mE,GACrCC,IACLA,EAAmB1T,cAAgBuT,EAAUr9C,GACjB,GAAxBq9C,EAAU1T,aACZ6T,EAAmBtV,SAAU,GAE/BoV,EAAgBG,GAAAH,QAAAA,EAAqBE,OAEvCb,EAAW55B,OAAOu6B,EAAeR,yBAKhC1vB,EAAOD,EAAQ2vB,GACtBvxB,EAAGmyB,gBAAgBvwB,EAAO/J,wBAEnBgK,EAAOD,EAAQ2vB,GACtBvxB,EAAGoyB,gBAAgBxwB,EAAO/J,WAIzBA,MAAQ,KACRm2B,OAAS,KACTqE,SAAW,KAEXC,UAAY,KAEZC,MAAQ,OACRC,WAAa,OAEb7L,YAAc,KACdxH,iBAAmB,CACtBuG,MAAO,KACPtG,YAAY,KAITrV,YAEAlK,WAAWrvB,KACXo+C,SAAW,mGAOV5E,EAAQv7C,SAASE,cAAc,OACrCq7C,EAAMrgC,UAAY,cAClBqgC,EAAM,eAAiB3oE,UAClBo+C,IAAIuqB,MAAQA,MAGX/f,EAAax7B,SAASE,cAAc,OAC1Cs7B,EAAWtgB,UAAY,iBACvBqgC,EAAMh0C,YAAYi0B,QACbxK,IAAIwK,WAAaA,MAGhBZ,EAAa56B,SAASE,cAAc,OAC1C06B,EAAW1f,UAAY,iBACvBqgC,EAAMh0C,YAAYqzB,QACb5J,IAAI4J,WAAaA,MAGhBM,EAAOl7B,SAASE,cAAc,OACpCg7B,EAAKhgB,UAAY,gBACZ8V,IAAIkK,KAAOA,MAGVqZ,EAAWv0C,SAASE,cAAc,OACxCq0C,EAASr5B,UAAY,oBAChB8V,IAAIujB,SAAWA,OAGfyP,uBAGCC,EAAkB,IAAIvN,GAAgB4K,GAAY,KAAM1uE,MAC9DqxE,EAAgB7N,YACXmJ,OAAO+B,IAAc2C,OAMrBxtB,OAAS,IAAIG,GAAOhkD,KAAK44C,KAAKwF,IAAI3E,sBAGlCoK,OAAOvN,GAAG,gBAAgB,SAAAkK,GACzBA,EAAMqE,SACR8Q,EAAKzX,SAASsC,WAGbqD,OAAOvN,GAAG,WAAYP,UAAK+H,qBAAkB99C,YAC7C6jD,OAAOvN,GAAG,UAAYP,UAAKgI,gBAAa/9C,YACxC6jD,OAAOvN,GAAG,SAAYP,UAAKiI,mBAAgBh+C,YAC3C6jD,OAAO95C,IAAI,OAAO/F,IAAI,CAACqlB,UAAU,EAAG/K,UAAW0lC,GAAOstB,WAGtDztB,OAAOvN,GAAG,MAAQP,UAAKw7B,sBAAmBvxE,YAG1C6jD,OAAOvN,GAAG,QAASP,UAAKy7B,2BAAwBxxE,YAGhD6jD,OAAOvN,GAAG,YAAaP,UAAKme,mBAAgBl0D,OAE7CA,KAAKmvB,QAAQguB,SACVs0B,YAAc,IAAIztB,GAAOhkD,KAAK44C,KAAKwF,IAAI+S,qBAEvCsgB,YAAc,IAAIztB,GAAOhkD,KAAK44C,KAAKwF,IAAI8S,oBAGzCugB,YAAYn7B,GAAG,MAAYP,UAAK27B,sBAAmB1xE,YACnDyxE,YAAYn7B,GAAG,WAAYP,UAAK47B,0BAAuB3xE,YACvDyxE,YAAYn7B,GAAG,UAAYP,UAAK67B,qBAAkB5xE,YAClDyxE,YAAYn7B,GAAG,SAAYP,UAAK87B,wBAAqB7xE,YACrDyxE,YAAY1nE,IAAI,OAAO/F,IAAI,CAACqlB,UAAU,EAAG/K,UAAW0lC,GAAO8tB,0BAE3Dl5B,KAAKwF,IAAI3E,gBAAgB6E,iBAAiB,YAAavI,UAAKg8B,qBAAkB/xE,YAC9E44C,KAAKwF,IAAI3E,gBAAgB6E,iBAAiB,WAAYvI,UAAKi8B,oBAAiBhyE,YAC5E44C,KAAKwF,IAAI3E,gBAAgB6E,iBAAiB,YAAavI,UAAKk8B,qBAAkBjyE,YAE9E44C,KAAKwF,IAAI3E,gBAAgB6E,iBAAiB,cAAevI,UAAKiI,mBAAgBh+C,YAE9E44C,KAAKwF,IAAI3E,gBAAgB6E,iBAAiB,aAAcvI,UAAKkI,sBAAmBj+C,YAGhFwjE,0CAmEIr0C,iBACLA,EAAS,SAOXmpB,GAAKmG,gBALU,CACb,OAAQ,MAAO,QAAS,QAAS,QAAS,iBAAkB,aAAc,cAAe,sBACzF,sBAAuB,sBAAuB,aAAc,iBAAkB,WAAY,gBAAiB,uBAC3G,OAAQ,OAAQ,iBAAkB,eAAgB,UAAW,0BAA2B,kBAAmB,aAEhFz+C,KAAKmvB,QAASA,GAEvC,yBAA0BA,IACgB,kBAAjCA,EAAQq6C,2BACZr6C,QAAQq6C,qBAAqBzxB,KAAO5oB,EAAQq6C,0BAC5Cr6C,QAAQq6C,qBAAqB9vB,OAAQ,GAEK,WAAxCjU,GAAOtW,EAAQq6C,wBACtBlxB,GAAKmG,gBAAgB,CAAC,OAAQ,SAAUz+C,KAAKmvB,QAAQq6C,qBAAsBr6C,EAAQq6C,sBAE7ExpE,KAAKmvB,QAAQq6C,qBAAqBzxB,YACjC5oB,QAAQq6C,qBAAqB9vB,OAAQ,KAK5C,wBAAyBvqB,GACgB,kBAAhCA,EAAQ6gD,2BACZ7gD,QAAQ6gD,oBAAsB7gD,EAAQ6gD,qBAI3C,gBAAiB7gD,IACgB,iBAAxBA,EAAQk5B,iBACZl5B,QAAQk5B,YAAYtQ,KAA+B,QAAxB5oB,EAAQk5B,YAAwB,MAAQ,SAElC,WAA/B5iB,GAAOtW,EAAQk5B,cAA4B,SAAUl5B,EAAQk5B,mBAC/Dl5B,QAAQk5B,YAAYtQ,KAAO5oB,EAAQk5B,YAAYtQ,OAIpD,WAAY5oB,IACgB,iBAAnBA,EAAQmrC,aACZnrC,QAAQmrC,OAAOhS,KAAOn5B,EAAQmrC,YAC9BnrC,QAAQmrC,OAAOviB,KAAK+jB,WAAa3sC,EAAQmrC,YACzCnrC,QAAQmrC,OAAOviB,KAAK8iB,SAAW1rC,EAAQmrC,QAEX,WAA1B70B,GAAOtW,EAAQmrC,UACtBhiB,GAAKmG,gBAAgB,CAAC,QAASz+C,KAAKmvB,QAAQmrC,OAAQnrC,EAAQmrC,QACxD,SAAUnrC,EAAQmrC,SACe,iBAAxBnrC,EAAQmrC,OAAOviB,WACnB5oB,QAAQmrC,OAAOviB,KAAK+jB,WAAa3sC,EAAQmrC,OAAOviB,UAChD5oB,QAAQmrC,OAAOviB,KAAK8iB,SAAW1rC,EAAQmrC,OAAOviB,MAEb,WAA/BtS,GAAOtW,EAAQmrC,OAAOviB,OAC7BO,GAAKmG,gBAAgB,CAAC,aAAc,YAAaz+C,KAAKmvB,QAAQmrC,OAAOviB,KAAM5oB,EAAQmrC,OAAOviB,eAMjG,SAAU,oBAAmB,SAAA90C,GACxBA,KAAOksB,IACT4xC,EAAK5xC,QAAQlsB,GAAOksB,EAAQlsB,OAI5B,aAAcksB,IACgB,kBAArBA,EAAQmhC,eACZnhC,QAAQmhC,SAASkU,WAAgBr1C,EAAQmhC,cACzCnhC,QAAQmhC,SAAS+V,YAAgBl3C,EAAQmhC,cACzCnhC,QAAQmhC,SAASz1C,IAAgBsU,EAAQmhC,cACzCnhC,QAAQmhC,SAASra,OAAgB9mB,EAAQmhC,cACzCnhC,QAAQmhC,SAASyU,eAAgB,GAEH,WAA5Bt/B,GAAOtW,EAAQmhC,WACtBhY,GAAKmG,gBAAgB,CAAC,aAAc,cAAe,MAAO,SAAU,iBAAkBz+C,KAAKmvB,QAAQmhC,SAAUnhC,EAAQmhC,WAIrH,kBAAmBnhC,IACgB,kBAA1BA,EAAQwuC,oBACZxuC,QAAQwuC,cAAcC,MAASzuC,EAAQwuC,mBACvCxuC,QAAQwuC,cAAc9iD,IAASsU,EAAQwuC,mBACvCxuC,QAAQwuC,cAAc1nB,OAAS9mB,EAAQwuC,eAEJ,WAAjCl4B,GAAOtW,EAAQwuC,gBACtBrlB,GAAKmG,gBAAgB,CAAC,QAAS,MAAO,UAAWz+C,KAAKmvB,QAAQwuC,cAAexuC,EAAQwuC,sBAcxF,qBAAsB,QAAS,WAAY,WAAY,SAAU,WAAY,aAAc,cAAe,0BATvF,SAAAl6D,OACZ5G,EAAKsyB,EAAQ1rB,MACf5G,EAAI,UACc,mBAAPA,QACL,IAAIuG,4BAAgBK,kCAA2BA,uBAEvDs9D,EAAK5xC,QAAQ1rB,GAAQ5G,MAKrBsyB,EAAQ2wC,YACI9/D,KAAKmvB,QAAS,CAC1B2wC,QAAS3wC,EAAQ2wC,UAEd9/D,KAAKowE,wBACHA,iBAAmB,IAAI1D,GAAiB1sE,YAE1CowE,iBAAiBjb,SAASn1D,KAAKw2C,MAAO,CAAEq2B,qBAAqB,SAC7DqF,UAAU,CAAEC,cAAc,EAAMC,eAAe,SAE/C/hB,UACIrwD,KAAKowE,uBACTiC,0BACA9E,SAAW,QACX6C,iBAAmB,UACnBjhD,QAAQ2wC,aAAUrgE,OAClByyE,UAAU,CAAEC,cAAc,EAAMC,eAAe,SAE/C/hB,eAGA6hB,+CAUD/iD,QACH6hD,SAAW,GAEZ7hD,IACEA,EAAQgjD,iBACV75B,SAAAA,GAAat4C,KAAKw2C,OAAO,SAAAuB,GACvBA,EAAK2mB,OAAQ,EACT3mB,EAAK4mB,WAAW5mB,EAAKsY,YAIzBlhC,EAAQijD,kBACV95B,SAAAA,GAAat4C,KAAK2sE,QAAQ,SAACtI,EAAOphE,GAC5BA,IAAQyrE,KACZrK,EAAMzH,YAAa,8CAUpB0V,uBACApiB,YACAiF,SAAS,WACTC,UAAU,WAEVvR,QAAU7jD,KAAK6jD,OAAOkB,eACtB0sB,aAAezxE,KAAKyxE,YAAY1sB,eAChClB,OAAS,UAETjL,KAAO,UACP8C,WAAa,oCAQd17C,KAAKo+C,IAAIuqB,MAAMphC,iBACZ6W,IAAIuqB,MAAMphC,WAAWC,YAAYxnC,KAAKo+C,IAAIuqB,OAI7C3oE,KAAKo+C,IAAIkK,KAAK/gB,iBACX6W,IAAIkK,KAAK/gB,WAAWC,YAAYxnC,KAAKo+C,IAAIkK,MAI5CtoD,KAAKo+C,IAAIujB,SAASp6B,iBACf6W,IAAIujB,SAASp6B,WAAWC,YAAYxnC,KAAKo+C,IAAIujB,yCAS/C3hE,KAAKo+C,IAAIuqB,MAAMphC,iBACbqR,KAAKwF,IAAIY,OAAOrqB,YAAY30B,KAAKo+C,IAAIuqB,OAIvC3oE,KAAKo+C,IAAIkK,KAAK/gB,iBACZqR,KAAKwF,IAAIyL,mBAAmBl1B,YAAY30B,KAAKo+C,IAAIkK,MAInDtoD,KAAKo+C,IAAIujB,SAASp6B,aACjBvnC,KAAKmvB,QAAQguB,SACVvE,KAAKwF,IAAIzJ,MAAMhgB,YAAY30B,KAAKo+C,IAAIujB,eAEpC/oB,KAAKwF,IAAI1J,KAAK/f,YAAY30B,KAAKo+C,IAAIujB,iDAShCuP,WACPoB,kBACDpB,EAAO,KACHpB,EAAQ9vE,KAAKmvB,QAAQm3C,QAAQwJ,OAA+C,iBAA/B9vE,KAAKmvB,QAAQm3C,QAAQwJ,MAClE9vE,KAAKmvB,QAAQm3C,QAAQwJ,MACrB,SACDqB,WAAahyB,IAChB,WACE+xB,EAAM1N,SACLsM,8CAQgB,MAAnB9vE,KAAKmxE,aACL/xB,aAAap/C,KAAKmxE,iBACbA,WAAa,2CAWXoB,SACA9yE,MAAP8yE,IACFA,EAAM,IAGHrvC,GAAcqvC,KACjBA,EAAM,CAACA,QAIHC,EAAgB17B,UAAKm6B,mBAAiB,SAAA79C,UAA2B,IAArBwgC,GAAA2e,QAAAA,EAAYn/C,4CAGvCo/C,+BAAe,KAA7BC,UACD16B,EAAO/3C,KAAK0yE,YAAYD,GAC1B16B,GACFA,EAAK46B,mGAKJ1B,aAAiBsB,6CACPA,+BAAK,KAAXn/C,UACD2kB,EAAO/3C,KAAK0yE,YAAYt/C,GAC1B2kB,GACFA,EAAK66B,iJAUF/B,UAAKI,kBAAiB,kDASzBt8B,EACAD,EAFEgF,EAAQ15C,KAAK44C,KAAKc,MAAMqc,WAI1B/1D,KAAKmvB,QAAQguB,KACfxI,EAAS30C,KAAK44C,KAAKN,KAAKkD,SAAS9B,EAAMlZ,OACvCkU,EAAO10C,KAAK44C,KAAKN,KAAKkD,SAAS9B,EAAMjZ,OAErCiU,EAAQ10C,KAAK44C,KAAKN,KAAKkD,SAAS9B,EAAMlZ,OACtCmU,EAAQ30C,KAAK44C,KAAKN,KAAKkD,SAAS9B,EAAMjZ,UAGlC8xC,EAAM,OACP,IAAMpW,KAAWn8D,KAAK2sE,UACrB3sE,KAAK2sE,OAAOtvE,eAAe8+D,GAAU,KACjCkI,EAAQrkE,KAAK2sE,OAAOxQ,GACpB0W,EAAkBxO,EAAM1H,UAAY0H,EAAMlH,aAAe,yCAI5C0V,+BAAiB,KAAzB96B,UAEL/3C,KAAKmvB,QAAQguB,IACVpF,EAAKpD,MAAQD,GAAUqD,EAAKpD,MAAQoD,EAAKnP,MAAQ+L,GACpD49B,EAAIt1E,KAAK86C,EAAK3kB,IAGX2kB,EAAKrD,KAAOC,GAAWoD,EAAKrD,KAAOqD,EAAKnP,MAAQ8L,GACnD69B,EAAIt1E,KAAK86C,EAAK3kB,+FAOjBm/C,iDAQDA,EAAM,OAEP,IAAMpW,KAAWn8D,KAAK2sE,OAAQ,IAC7B3sE,KAAK2sE,OAAOtvE,eAAe8+D,GACfn8D,KAAK2sE,OAAOxQ,GAChBQ,WACR4V,EAAIt1E,KAAKk/D,UAKRoW,sCAQGn/C,gBACHpzB,KAAKw2C,MAAMpjB,IAAO0/C,UAAKvF,kBAAc,SAAAzN,UAAWA,EAAQ1sC,KAAOA,uCAQ9DA,WACF69C,EAAYjxE,KAAKixE,UACdn0E,EAAI,EAAGmO,EAAKgmE,EAAUj0E,OAAQF,EAAImO,EAAInO,OACzCm0E,EAAUn0E,IAAMs2B,EAAI,IACtB69C,QAAAA,EAAiBn0E,EAAG,sDAWlBw9D,EAASt6D,KAAKmvB,QAAQmrC,OACtB5gB,EAAQ15C,KAAK44C,KAAKc,MAClB6c,EAASje,GAAKge,OAAOC,OACrBpnC,EAAUnvB,KAAKmvB,QACfk5B,EAAcl5B,EAAQk5B,YAAYtQ,KACpCc,GAAU,EACR8vB,EAAQ3oE,KAAKo+C,IAAIuqB,WAGlBtzB,MAAMyN,IAAM9iD,KAAK44C,KAAKY,SAASsJ,IAAIja,OAAS7oC,KAAK44C,KAAKY,SAASmY,OAAO7O,IAEvE9iD,KAAKmvB,QAAQguB,SACV9H,MAAMV,MAAQ30C,KAAK44C,KAAKY,SAAS7E,MAAM/L,MAAQ5oC,KAAK44C,KAAKY,SAASmY,OAAOhd,WAEzEU,MAAMX,KAAO10C,KAAK44C,KAAKY,SAAS9E,KAAK9L,MAAQ5oC,KAAK44C,KAAKY,SAASmY,OAAOjd,KAI9Ei0B,EAAMrgC,UAAY,cAEdtoC,KAAKmvB,QAAQ2wC,cACViT,gBAIPl6B,EAAU74C,KAAKgzE,gBAAkBn6B,MAI3Bo6B,EAAkBv5B,EAAMjZ,IAAMiZ,EAAMlZ,MACpC0yC,EAAUD,GAAmBjzE,KAAKmzE,qBAAyBnzE,KAAKq1C,MAAMzM,OAAS5oC,KAAKq1C,MAAM4iB,UAC1Fmb,EAAW15B,EAAMlZ,OAASxgC,KAAKqzE,eAC/BC,EAAqBnkD,EAAQ9rB,OAASrD,KAAKuzE,UAC3CC,EAA8BrkD,EAAQosC,gBAAkBv7D,KAAKyzE,mBAC7D/T,EAAgBwT,GAAUE,GAAYE,GAAsBE,OAC7DL,oBAAsBF,OACtBI,eAAiB35B,EAAMlZ,WACvB+yC,UAAYpkD,EAAQ9rB,WACpBowE,mBAAqBtkD,EAAQosC,oBAE7BlmB,MAAM4iB,UAAYj4D,KAAKq1C,MAAMzM,UAC5B8qC,EAAa1zE,KAAK2zE,cAClBC,EAAc,CAClB77B,KAAMuiB,EAAOviB,KACbuQ,KAAMgS,EAAOhS,MAETurB,EAAiB,CACrB97B,KAAMuiB,EAAOviB,KACbuQ,KAAMgS,EAAOviB,KAAK8iB,SAAW,GAE3BhyB,EAAS,EACP2tB,EAAY8D,EAAOhS,KAAOgS,EAAOviB,KAAK8iB,cAGvC8R,OAAO+B,IAAYre,OAAO3W,EAAOm6B,EAAgBnU,OAEhDlB,EAAc,GAChBC,EAAoB,YAGxBnmB,SAAAA,GAAat4C,KAAK2sE,QAAQ,SAACtI,EAAOphE,MAC5BA,IAAQyrE,QACNoF,EAAczP,GAASqP,EAAaE,EAAcC,EAExDrV,EAAYv7D,GAAOohE,EAAMhU,OAAO3W,EAAOo6B,EAAapU,GADhC,GAEpBjB,EAAoBD,EAAYv7D,GAAKjG,WAGpByhE,EAAoB,sBAE/BsV,EAAgB,cAEbj3E,MACPw7C,SAAAA,GAAakmB,GAAa,SAACI,EAAK37D,GAC9B8wE,EAAc9wE,GAAO27D,EAAI9hE,SAFpBA,EAAI,EAAGA,EAAI2hE,EAAmB3hE,MAA9BA,MAOTw7C,SAAAA,GAAaopB,EAAKiL,QAAQ,SAACtI,EAAOphE,MAC5BA,IAAQyrE,QACNsF,EAAeD,EAAc9wE,GACnC41C,EAAUm7B,GAAgBn7B,EAC1BhQ,GAAUw7B,EAAMx7B,WAElBA,EAASrnC,KAAK8D,IAAIujC,EAAQ2tB,MAG5B3tB,EAASrnC,KAAK8D,IAAIujC,EAAQ2tB,GAG1BmS,EAAMn0C,MAAMqU,OAAU0tB,EAAO1tB,QAGxBwM,MAAMzM,MAAQ+/B,EAAMvf,iBACpB/T,MAAMxM,OAASA,OAGfuV,IAAIkK,KAAK9zB,MAAMsuB,IAAMyT,EAAuB,OAAflO,EAC7BroD,KAAK44C,KAAKY,SAASsJ,IAAIja,OAAS7oC,KAAK44C,KAAKY,SAASmY,OAAO7O,IAC1D9iD,KAAK44C,KAAKY,SAASsJ,IAAIja,OAAS7oC,KAAK44C,KAAKY,SAASC,gBAAgB5Q,QACpE7oC,KAAKmvB,QAAQguB,SACViB,IAAIkK,KAAK9zB,MAAMmgB,MAAQ,SAEvByJ,IAAIkK,KAAK9zB,MAAMkgB,KAAO,SAGxBmP,OAAO95C,IAAI,SAAS/F,IAAI,CAAC0c,KAAM1gB,KAAKmvB,QAAQqoC,2BAE5CpF,qBAAsB,EAE3BvZ,EAAU74C,KAAK8pD,cAAgBjR,4CAWzBo7B,EAAoD,OAAjCj0E,KAAKmvB,QAAQk5B,YAAYtQ,KAAiB,EAAK/3C,KAAKgxE,SAASh0E,OAAS,EACzFk3E,EAAel0E,KAAKgxE,SAASiD,UAChBj0E,KAAK2sE,OAAOuH,IAAiBl0E,KAAK2sE,OAAL,eAE3B,oDAUjB50B,EACAo8B,EAFAC,EAAYp0E,KAAK2sE,OAAL,iBAIZ3sE,KAAK+vE,eAEHqE,MAIGD,KAHLC,EAAU/7B,iBACHr4C,KAAK2sE,OAAL,cAEQ3sE,KAAKw2C,SACdx2C,KAAKw2C,MAAMn5C,eAAe82E,GAAS,EACrCp8B,EAAO/3C,KAAKw2C,MAAM29B,IACbvjD,QAAUmnB,EAAKnnB,OAAOqlB,OAAO8B,OAC5BokB,EAAUn8D,KAAK4tE,WAAW71B,EAAKxkC,MAC/B8wD,EAAQrkE,KAAK2sE,OAAOxQ,GAC1BkI,GAASA,EAAMxpD,IAAIk9B,IAASA,EAAKmY,iBAOlCkkB,EAAW,KAMTD,KAHLC,EAAY,IAAIlY,GAFL,KACE,KACmBl8D,WAC3B2sE,OAAL,cAAyByH,EAEVp0E,KAAKw2C,MACdx2C,KAAKw2C,MAAMn5C,eAAe82E,KAC5Bp8B,EAAO/3C,KAAKw2C,MAAM29B,GAClBC,EAAUv5D,IAAIk9B,IAIlBq8B,EAAU5Q,qDAUPxjE,KAAKo+C,IAAIujB,0CAOTnrB,QACF2oB,iBAAmB,IAAIziE,SAExB61E,EADE5zB,EAAK3+C,KAELq0E,EAAer0E,KAAKylE,aAGrBjvB,EAGA,CAAA,KAAIA,aAAiBoB,IAAWpB,aAAiB89B,UAI9C,IAAItnD,UAAU,wDAHfy4C,UAAYluB,GAAkBf,aAH9BivB,UAAY,QASf4O,OAEF/7B,SAAAA,GAAat4C,KAAKiwE,eAAe,SAACjqE,EAAUw6C,GAC1C6zB,EAAa99B,IAAIiK,EAAOx6C,MAI1BquE,EAAah8B,UAGbk6B,EAAM8B,EAAaj8B,cACdk4B,UAAUiC,IAGbvyE,KAAKylE,UAAW,KAEZryC,EAAKpzB,KAAKozB,MAChBklB,SAAAA,GAAat4C,KAAKiwE,eAAe,SAACjqE,EAAUw6C,GAC1C7B,EAAG8mB,UAAUnvB,GAAGkK,EAAOx6C,EAAUotB,MAInCm/C,EAAMvyE,KAAKylE,UAAUrtB,cAChB+3B,OAAOoC,QAGPnB,wBAGFx4B,KAAKiF,QAAQ4C,KAAK,UAAW,CAAC4R,OAAO,8CAQjB,MAAlBryD,KAAKylE,UAAoBzlE,KAAKylE,UAAUjuB,MAAQ,uCAO/Cm1B,OAEJ4F,EADE5zB,EAAK3+C,QAIPA,KAAK+vE,gBACPz3B,SAAAA,GAAat4C,KAAKuwE,gBAAgB,SAACvqE,EAAUw6C,GAC3C7B,EAAGoxB,WAAWx5B,IAAIiK,EAAOx6C,MAI3BusE,EAAMvyE,KAAK+vE,WAAW33B,cACjB23B,WAAa,UACbgB,gBAAgBwB,IAIlB5F,EAGA,CAAA,KAAIA,aAAkB/0B,IAAW+0B,aAAkB2H,UAIhD,IAAItnD,UAAU,wDAHf+iD,WAAapD,YAHboD,WAAa,QAShB/vE,KAAK+vE,WAAY,OAEfA,EAAa/vE,KAAK+vE,WAClB/vE,KAAK+vE,sBAAsBuE,KAC7BvE,EAAa/vE,KAAK+vE,WAAW73B,mBAG/B63B,EAAWhmE,eAAc,SAAAs6D,SACnBA,EAAMvH,mBACRuH,EAAMvH,sBAAqB,SAAA6T,OACnBC,EAAqBb,EAAWhmE,IAAI4mE,GAC1CC,EAAmB1T,cAAgBmH,EAAMjxC,GACjB,GAApBixC,EAAMtH,aACR6T,EAAmBtV,SAAU,GAE/ByU,EAAW55B,OAAOy6B,aAMlBx9C,EAAKpzB,KAAKozB,MAChBklB,SAAAA,GAAat4C,KAAKuwE,gBAAgB,SAACvqE,EAAUw6C,GAC3C7B,EAAGoxB,WAAWz5B,GAAGkK,EAAOx6C,EAAUotB,MAIpCm/C,EAAMvyE,KAAK+vE,WAAW33B,cACjBo4B,aAAa+B,QAIfnB,wBAGAmD,SAEDv0E,KAAKmvB,QAAQ2wC,eACVsQ,iBAAiBoE,kBACjBzB,qBACAb,UAAU,CAAEC,cAAc,EAAMC,eAAe,UAGjDx5B,KAAKiF,QAAQ4C,KAAK,UAAW,CAAC4R,OAAO,+CAQnCryD,KAAK+vE,8CAOH38C,cACH2kB,EAAO/3C,KAAKylE,UAAU17D,IAAIqpB,GAE5B2kB,QAEG5oB,QAAQogD,SAASx3B,GAAM,SAAAA,GACtBA,GAGF08B,EAAKhP,UAAUxvB,OAAO7iB,uCAYrB4gC,UACAA,EAAS98B,MAAQl3B,KAAKmvB,QAAQ+H,OAAS88B,EAASvzB,IAAM,QAAU,0CAS9DuzB,SAEG,cADCh0D,KAAK00E,SAAS1gB,IACmBv0D,MAAlBu0D,EAASqQ,MAC7BqK,GAGC1uE,KAAK+vE,WAAa/b,EAASqQ,MA1lCtB,kDAmmCNkO,cACF5zB,EAAK3+C,QAEXuyE,QAAAA,GAAY,SAAAn/C,OAMN4wC,EALEhQ,EAAWrV,EAAG8mB,UAAU17D,IAAIqpB,GAC9B2kB,EAAO4G,EAAGnI,MAAMpjB,GACd8D,EAAO88B,EAAWrV,EAAG+1B,SAAS1gB,GAAY,KAE1C18B,EAAcq3C,EAAQgG,MAAMz9C,MAG9B6gB,IAEGzgB,GAAiBygB,aAAgBzgB,EAOpCqnB,EAAGi2B,YAAY78B,EAAMic,IALrBgQ,EAAWjsB,EAAKisB,SAChBrlB,EAAGk2B,YAAY98B,GACfA,EAAO,QAONA,GAAQic,EAAU,KAEjB18B,QAWI,IAAItK,uCAAgCkK,SAV1C6gB,EAAO,IAAIzgB,EAAY08B,EAAUrV,EAAGjD,WAAYiD,EAAGxvB,UAC9CiE,GAAKA,EAEVurB,EAAGm2B,SAAS/8B,GACRisB,IACF+Q,EAAK9D,UAAUh0E,KAAKm2B,GACpB2kB,EAAK66B,mBASR2B,SAEDv0E,KAAKmvB,QAAQ2wC,eACVsQ,iBAAiBjb,SAASn1D,KAAKw2C,MAAO,CAAEq2B,qBAAqB,SAC7DkG,sBAGFn6B,KAAKiF,QAAQ4C,KAAK,UAAW,CAAC4R,OAAO,sCAQlCkgB,OACJt5B,EAAQ,EACN0F,EAAK3+C,QACXuyE,QAAAA,GAAY,SAAAn/C,OACJ2kB,EAAO4G,EAAGnI,MAAMpjB,GAClB2kB,IACFkB,IACA0F,EAAGk2B,YAAY98B,OAIfkB,SAEGs7B,cACA37B,KAAKiF,QAAQ4C,KAAK,UAAW,CAAC4R,OAAO,yCAW5C/Z,SAAAA,GAAat4C,KAAK2sE,QAAQ,SAAAtI,GACxBA,EAAMzG,mDASM2U,QACT/B,aAAa+B,wCAQPA,OACL5zB,EAAK3+C,QAEXuyE,QAAAA,GAAY,SAAAn/C,OACJq9C,EAAY9xB,EAAGoxB,WAAWhmE,IAAIqpB,GAChCixC,EAAQ1lB,EAAGguB,OAAOv5C,MAEjBixC,EA6BHA,EAAM3G,QAAQ+S,OA7BJ,IA/sCA,iBAitCNr9C,GAAmBA,GAAMs7C,SACrB,IAAItrE,kCAA2BgwB,6BAGjC4hD,EAAe3qC,GAAcsU,EAAGxvB,aASjC,IAAMglD,KARX77B,GAAKh7C,OAAO03E,EAAc,CACxBnsC,OAAQ,OAGVw7B,EAAQ,IAAInI,GAAM9oC,EAAIq9C,EAAW9xB,GACjCA,EAAGguB,OAAOv5C,GAAMixC,EAGK1lB,EAAGnI,SAClBmI,EAAGnI,MAAMn5C,eAAe82E,GAAS,KAC7Bp8B,EAAO4G,EAAGnI,MAAM29B,GAClBp8B,EAAKxkC,KAAK8wD,OAASjxC,GACrBixC,EAAMxpD,IAAIk9B,GAKhBssB,EAAMzG,QACNyG,EAAMb,gBAQL5qB,KAAKiF,QAAQ4C,KAAK,UAAW,CAAC4R,OAAO,4CAQ5BkgB,iBACdA,QAAAA,GAAY,SAAAn/C,OACJixC,EAAQ4Q,EAAKtI,OAAOv5C,GAEtBixC,IACFA,EAAMhsB,iBACC48B,EAAKtI,OAAOv5C,OAInBpzB,KAAKmvB,QAAQ2wC,eACVsQ,iBAAiBoE,kBACjBzB,sBAGFb,UAAU,CAAEE,gBAAiBpyE,KAAKmvB,QAAQ2wC,eAC1ClnB,KAAKiF,QAAQ4C,KAAK,UAAW,CAAC4R,OAAO,8CAStCryD,KAAK+vE,WAAY,KAEfiB,EAAWhxE,KAAK+vE,WAAW33B,OAAO,CACpCwlB,MAAO59D,KAAKmvB,QAAQ6/C,aAGtBgC,EAAWhxE,KAAKk1E,mBAAmBlE,OAE7B3wB,GAAW/H,GAAK68B,WAAWnE,EAAUhxE,KAAKgxE,aAC5C3wB,EAAS,KAELssB,EAAS3sE,KAAK2sE,UACpBqE,QAAAA,GAAiB,SAAA7U,GACfwQ,EAAOxQ,GAASjM,aAIlB8gB,QAAAA,GAAiB,SAAA7U,GACfwQ,EAAOxQ,GAASqH,eAGbwN,SAAWA,SAGX3wB,SAGA,6CAWQ2wB,8BASRoE,EAAuBr1E,EAAGixE,OAC7Bz4D,EAAS,aACby4D,QAAAA,GAAiB,SAAA7U,MACf5jD,EAAOtb,KAAKk/D,GACMp8D,EAAEgwE,WAAWhmE,IAAIoyD,GACrBW,aAAc,OACpBuY,EAAiBt+B,KAAAh3C,EAAEgwE,WAAWhmE,IAAI,CACtCkuB,gBAAOq9C,UACEA,EAAYpY,eAAiBf,GAEtCyB,MAAO79D,EAAEovB,QAAQ6/C,sBACZ,SAAAsG,UAAeA,EAAYliD,MAClC7a,EAASs4D,GAAAt4D,QAAAA,EAAc68D,EAAuBr1E,EAAGs1E,QAI9C98D,EAKF68D,CAAuBp1E,KAFV82C,GAAAk6B,QAAAA,GAAgB,SAAA7U,UAAYoZ,EAAKxF,WAAWhmE,IAAIoyD,GAASe,mDAUtEnlB,QACFvB,MAAMuB,EAAK3kB,IAAM2kB,MAGhBokB,EAAUn8D,KAAK4tE,WAAW71B,EAAKxkC,MAC/B8wD,EAAQrkE,KAAK2sE,OAAOxQ,GAErBkI,EAEMA,GAASA,EAAM9wD,MAAQ8wD,EAAM9wD,KAAKwpD,aAC3ChlB,EAAKksB,cAAe,GAFpBlsB,EAAKksB,cAAe,EAKlBI,GAAOA,EAAMxpD,IAAIk9B,uCASXA,EAAMic,GAEhBjc,EAAK2lB,QAAQ1J,OAEPmI,EAAUn8D,KAAK4tE,WAAW71B,EAAKxkC,MAC/B8wD,EAAQrkE,KAAK2sE,OAAOxQ,GACrBkI,EAEMA,GAASA,EAAM9wD,MAAQ8wD,EAAM9wD,KAAKwpD,aAC3ChlB,EAAKksB,cAAe,GAFpBlsB,EAAKksB,cAAe,sCAYZlsB,WAEVA,EAAKmY,cAGElwD,KAAKw2C,MAAMuB,EAAK3kB,QAGjBxO,EAAQgvC,UAAKqd,kBAAkBl5B,EAAK3kB,KAC5B,GAAVxO,GAAa09C,UAAK2O,kBAAiBrsD,EAAO,GAG9CmzB,EAAKnnB,QAAUmnB,EAAKnnB,OAAOqlB,OAAO8B,GAGhB,MAAd/3C,KAAKkxE,YACFA,MAAMhhB,oDAUM7pD,WACbs8D,EAAW,GAER7lE,EAAI,EAAGA,EAAIuJ,EAAMrJ,OAAQF,IAC5BuJ,EAAMvJ,aAAc4rE,IACtB/F,EAAS1lE,KAAKoJ,EAAMvJ,WAGjB6lE,mCAaAniB,QAEF8kB,YAAYvtB,KAAO/3C,KAAKw1E,eAAeh1B,QACvC8kB,YAAYmE,aAAejpB,EAAM7wB,OAAO85C,eAAgB,OACxDnE,YAAYqE,cAAgBnpB,EAAM7wB,OAAOg6C,gBAAiB,OAC1DrE,YAAYmQ,UAAY,4CAUhBtZ,OACN,IAAIr/D,EAAI,EAAGA,EAAIkD,KAAKgxE,SAASh0E,OAAQF,OAClCq/D,GAAWn8D,KAAKgxE,SAASl0E,GACzB,OAAOA,uCASN0jD,kBACPxgD,KAAKslE,YAAYC,oBAGjBlwB,EAFE0C,EAAO/3C,KAAKslE,YAAYvtB,MAAQ,KAChC4G,EAAK3+C,QAGP+3C,IAASA,EAAKisB,UAAYhkE,KAAKmvB,QAAQq6C,qBAAqBzxB,MAAO,IAEjE/3C,KAAKmvB,QAAQmhC,SAASyU,gBACrB/kE,KAAKmvB,QAAQmhC,SAASkU,aACtBxkE,KAAKmvB,QAAQmhC,SAAS+V,sBAKL,MAAjBtuB,EAAKuY,WAAqBvY,EAAKuY,SAASkU,aAAezsB,EAAKuY,SAAS+V,cAClErmE,KAAKmvB,QAAQmhC,SAASyU,yBAIxB0E,EAAezpE,KAAKslE,YAAYmE,aAChCE,EAAgB3pE,KAAKslE,YAAYqE,sBAClCrE,YAAYC,gBAAiB,OAC7BD,YAAYoQ,aAAe39B,EAE5B0xB,EACFp0B,EAAQ,CACN0C,KAAM0xB,EACNkM,SAAUn1B,EAAMxB,OAAO70C,EACvB26D,UAAW,EACXvxD,KAAMvT,KAAK41E,eAAe79B,EAAKxkC,YAG5B+xD,YAAYmQ,UAAY,CAACpgC,QACzB,GAAIs0B,EACTt0B,EAAQ,CACN0C,KAAM4xB,EACNgM,SAAUn1B,EAAMxB,OAAO70C,EACvBu/D,WAAW,EACXn2D,KAAMvT,KAAK41E,eAAe79B,EAAKxkC,YAG5B+xD,YAAYmQ,UAAY,CAACpgC,QACzB,GAAIr1C,KAAKmvB,QAAQmhC,SAASz1C,MAAQ2lC,EAAM4D,SAASyxB,SAAWr1B,EAAM4D,SAAS0xB,cAE3EC,oBAAoBv1B,OACpB,CACFxgD,KAAKgxE,SAASh0E,OAAS,QAGnBqzD,aAGD2lB,EAAiBh2E,KAAKi2E,eAAel+B,EAAKxkC,KAAK8wD,OAE/C6R,EAAel2E,KAAKmvB,QAAQq6C,qBAAqBzxB,OAASA,EAAKisB,SAAY,CAACjsB,EAAK3kB,IAAMpzB,KAAKm2E,oBAE7F7Q,YAAYmQ,UAAY1+B,GAAAm/B,QAAAA,GAAgB,SAAA9iD,OACrC2kB,EAAO4G,EAAGnI,MAAMpjB,GAChBgjD,EAAaz3B,EAAGs3B,eAAel+B,EAAKxkC,KAAK8wD,aACxC,CACLtsB,KAAAA,EACA49B,SAAUn1B,EAAMxB,OAAO70C,EACvBksE,YAAaL,EAAeI,EAC5B7iE,KAAM+iE,EAAKV,eAAe79B,EAAKxkC,UAKrCitC,EAAM8D,uBACGtkD,KAAKmvB,QAAQmhC,SAASz1C,MAAQ2lC,EAAM4D,SAASyxB,SAAWr1B,EAAM4D,SAAS0xB,eAE3EC,oBAAoBv1B,gDASTA,OACZsQ,EAAO9wD,KAAKmvB,QAAQ2hC,MAAQ,KAC5BylB,EAAYv2E,KAAKo+C,IAAIuqB,MAAM/lB,wBAG3Bz4C,EAAInK,KAAKmvB,QAAQguB,IAAMo5B,EAAU5hC,MAAQ6L,EAAMxB,OAAO70C,EAAK,GAAKq2C,EAAMxB,OAAO70C,EAAIosE,EAAU7hC,KAAO,GAElGh0B,EAAO1gB,KAAK44C,KAAKN,KAAK2D,OAAO9xC,GAC7BwxC,EAAQ37C,KAAK44C,KAAKN,KAAKsY,WACvB7sB,EAAO/jC,KAAK44C,KAAKN,KAAKuY,UACtBrwB,EAAQswB,EAAOA,EAAKpwC,EAAMi7B,EAAO5X,GAAQrjB,EAGzCszC,EAAW,CACf98B,KAAM,QACNsJ,MAAAA,EACAC,IALUD,EAMV3M,QAAS,YAGLT,EAAKs3C,KACX1W,EAASh0D,KAAKylE,UAAUhuB,QAAUrkB,MAE5BixC,EAAQrkE,KAAKw2E,gBAAgBh2B,GAC/B6jB,IACFrQ,EAASqQ,MAAQA,EAAMlI,aAEnBsa,EAAU,IAAI/N,GAAU1U,EAAUh0D,KAAK07C,WAAY17C,KAAKmvB,SAC9DsnD,EAAQrjD,GAAKA,EACbqjD,EAAQljE,KAAOvT,KAAK41E,eAAe5hB,QAC9B8gB,SAAS2B,QACTnR,YAAYoQ,aAAee,MAE1BphC,EAAQ,CACZ0C,KAAM0+B,EACNd,SAAUn1B,EAAMxB,OAAO70C,EACvBoJ,KAAMkjE,EAAQljE,MAGZvT,KAAKmvB,QAAQguB,IACf9H,EAAMyvB,UAAW,EAEjBzvB,EAAMq0B,WAAY,OAEfpE,YAAYmQ,UAAY,CAACpgC,GAE9BmL,EAAM8D,kDAQA9D,iBACY,MAAdxgD,KAAKkxE,OAAiBlxE,KAAKmvB,QAAQygD,eAAiB5vE,KAAKkxE,MAAM32B,OAAQ,KAEnE0R,EAAYjsD,KAAK44C,KAAKwF,IAAI3E,gBAC1Bi9B,EAAgBzqB,EAAUrJ,6BAC3BsuB,MAAMyF,YACTn2B,EAAMxB,OAAO70C,EAAIusE,EAAchiC,KAAOuX,EAAU6S,WAChDte,EAAMxB,OAAOz1C,EAAImtE,EAAc5zB,IAAMmJ,EAAU4S,gBAE5CqS,MAAM1N,UAGTxjE,KAAKslE,YAAYmQ,UAAW,OAC9Bj1B,EAAM8D,sBAEA3F,EAAK3+C,KACL8wD,EAAO9wD,KAAKmvB,QAAQ2hC,MAAQ,KAC5B8lB,EAAoB52E,KAAK44C,KAAKwF,IAAI+C,KAAK2d,WACvCt2B,EAAUxoC,KAAKmvB,QAAQguB,IAAMy5B,EAAoB52E,KAAK44C,KAAKY,SAAS7E,MAAM/L,MAAQguC,EAAoB52E,KAAK44C,KAAKY,SAAS9E,KAAK9L,MAC9H+S,EAAQ37C,KAAK44C,KAAKN,KAAKsY,WACvB7sB,EAAO/jC,KAAK44C,KAAKN,KAAKuY,UAGtB6kB,EAAe11E,KAAKslE,YAAYoQ,aAChCmB,GAAuB72E,KAAKmvB,QAAQmhC,SAASyU,eAA0C,MAAzB2Q,EAAaplB,WAAqBtwD,KAAKmvB,QAAQmhC,SAAS+V,cACjGrmE,KAAKmvB,QAAQmhC,SAASyU,eAA0C,MAAzB2Q,EAAaplB,UAAoBolB,EAAaplB,SAAS+V,YACrHyQ,EAAe,QACfD,GAAsBnB,GACOj2E,MAA3Bi2E,EAAaniE,KAAK8wD,MAAoB,KAElCA,EAAQ1lB,EAAG63B,gBAAgBh2B,GAC7B6jB,IAGFyS,EAAe92E,KAAKi2E,eAAe5R,EAAMlI,oBAM1CmJ,YAAYmQ,mBAAkB,SAAApgC,OAG7B74B,EACAu6D,EACA5U,EACA3hC,EACAC,EANEwa,EAAU0D,EAAG/F,KAAKN,KAAK2D,OAAOuE,EAAMxB,OAAO70C,EAAIq+B,GAC/CwuC,EAAUr4B,EAAG/F,KAAKN,KAAK2D,OAAO5G,EAAMsgC,SAAWntC,GAQnDhsB,EADEy6D,EAAK9nD,QAAQguB,MACJlC,EAAU+7B,GAEX/7B,EAAU+7B,MAGlBhjB,EAAWijB,EAAKrB,eAAevgC,EAAM0C,KAAKxkC,SACnB,MAAvB8hC,EAAM0C,KAAKuY,UACTjb,EAAM0C,KAAKuY,SAASkU,YACpBnvB,EAAM0C,KAAKuY,SAAS+V,aACpB1nB,EAAGxvB,QAAQmhC,SAASyU,mBAIEkS,EAAK9nD,QAAQmhC,SAASyU,eAA0C,MAAzB2Q,EAAaplB,WAAqB2mB,EAAK9nD,QAAQmhC,SAASkU,aAChGyS,EAAK9nD,QAAQmhC,SAASyU,eAA0C,MAAzB2Q,EAAaplB,UAAoBolB,EAAaplB,SAASkU,cAEnHnvB,EAAMyvB,SAEJmS,EAAK9nD,QAAQguB,IACK19C,MAAhBu0D,EAASvzB,MACX0hC,EAAa7pB,GAAKpB,QAAQ7B,EAAM9hC,KAAKktB,IAAK,QAC1CA,EAAM,IAAI/jC,KAAKylE,EAAW5kE,UAAYif,GAEtCw3C,EAASvzB,IAAMqwB,EAAOA,EAAKrwB,EAAKkb,EAAO5X,GAAQtD,GAG3BhhC,MAAlBu0D,EAASxzB,QACXu2C,EAAez+B,GAAKpB,QAAQ7B,EAAM9hC,KAAKitB,MAAO,QAC9CA,EAAQ,IAAI9jC,KAAKq6E,EAAax5E,UAAYif,GAE1Cw3C,EAASxzB,MAAQswB,EAAOA,EAAKtwB,EAAOmb,EAAO5X,GAAQvD,QAIpD,GAAI6U,EAAMq0B,UAETuN,EAAK9nD,QAAQguB,IACO19C,MAAlBu0D,EAASxzB,QACXu2C,EAAez+B,GAAKpB,QAAQ7B,EAAM9hC,KAAKitB,MAAO,QAC9CA,EAAQ,IAAI9jC,KAAKq6E,EAAax5E,UAAYif,GAE1Cw3C,EAASxzB,MAAQswB,EAAOA,EAAKtwB,EAAOmb,EAAO5X,GAAQvD,GAGjC/gC,MAAhBu0D,EAASvzB,MACX0hC,EAAa7pB,GAAKpB,QAAQ7B,EAAM9hC,KAAKktB,IAAK,QAC1CA,EAAM,IAAI/jC,KAAKylE,EAAW5kE,UAAYif,GAEtCw3C,EAASvzB,IAAMqwB,EAAOA,EAAKrwB,EAAKkb,EAAO5X,GAAQtD,WAM7BhhC,MAAlBu0D,EAASxzB,SAEXu2C,EAAez+B,GAAKpB,QAAQ7B,EAAM9hC,KAAKitB,MAAO,QAAQjjC,UACtDijC,EAAQ,IAAI9jC,KAAKq6E,EAAev6D,GAEZ/c,MAAhBu0D,EAASvzB,IAAkB,KAEvBplB,GADN8mD,EAAa7pB,GAAKpB,QAAQ7B,EAAM9hC,KAAKktB,IAAK,SACbljC,UAAYw5E,EAAax5E,UAGtDy2D,EAASxzB,MAAQswB,EAAOA,EAAKtwB,EAAOmb,EAAO5X,GAAQvD,EACnDwzB,EAASvzB,IAAQ,IAAI/jC,KAAKs3D,EAASxzB,MAAMjjC,UAAY8d,QAIrD24C,EAASxzB,MAAQswB,EAAOA,EAAKtwB,EAAOmb,EAAO5X,GAAQvD,KAMvDq2C,IAAwBxhC,EAAMyvB,WAAazvB,EAAMq0B,WAA4B,MAAdoN,GAC3Cr3E,MAAlBu0D,EAASqQ,MAAoB,KAC3B6S,EAAYJ,EAAezhC,EAAMghC,YAGrCa,EAAY11E,KAAK8D,IAAI,EAAG4xE,GACxBA,EAAY11E,KAAKY,IAAIu8C,EAAGqyB,SAASh0E,OAAO,EAAGk6E,GAC3CljB,EAASqQ,MAAQ1lB,EAAGqyB,SAASkG,GAKjCljB,EAAWijB,EAAKrB,eAAe5hB,GAC/BrV,EAAGxvB,QAAQqgD,SAASxb,GAAU,SAAAA,GACxBA,GACF3e,EAAM0C,KAAK2lB,QAAQuZ,EAAKrB,eAAe5hB,EAAU,qBAKlDpb,KAAKiF,QAAQ4C,KAAK,iDAUd1I,EAAMokB,OACXkI,EAAQrkE,KAAK2sE,OAAOxQ,MACtBkI,GAASA,EAAMlI,SAAWpkB,EAAKxkC,KAAK8wD,MAAO,KACvC8S,EAAWp/B,EAAKnnB,OACtBumD,EAASlhC,OAAO8B,GAChBo/B,EAASvZ,QAET7lB,EAAKxkC,KAAK8wD,MAAQA,EAAMlI,QAExBkI,EAAMxpD,IAAIk9B,GACVssB,EAAMzG,4CASCpd,sBACJ8kB,YAAYC,gBAAiB,EAC9BvlE,KAAKslE,YAAYmQ,UAAW,CAC9Bj1B,EAAM8D,sBAEA3F,EAAK3+C,KACLy1E,EAAYz1E,KAAKslE,YAAYmQ,eAC9BnQ,YAAYmQ,UAAY,QAE7BA,QAAAA,GAAkB,SAAApgC,OACVjiB,EAAKiiB,EAAM0C,KAAK3kB,MACiB,MAAxBurB,EAAG8mB,UAAU17D,IAAIqpB,GAc3B,KAEG4gC,EAAWojB,EAAKxB,eAAevgC,EAAM0C,KAAKxkC,MAChDorC,EAAGxvB,QAAQmgD,OAAOtb,GAAU,SAAAA,GACtBA,GAEFA,EAASojB,EAAK3R,UAAUhuB,QAAUrkB,EAClCgkD,EAAK3R,UAAUtvB,OAAO6d,KAItB3e,EAAM0C,KAAK2lB,QAAQroB,EAAM9hC,MAEzBorC,EAAG/F,KAAKiF,QAAQ4C,KAAK,oBAvBzB9B,EAAGxvB,QAAQigD,MAAM/5B,EAAM0C,KAAKxkC,MAAM,SAAAygD,GAChCrV,EAAGk2B,YAAYx/B,EAAM0C,MACjBic,GACFrV,EAAG8mB,UAAU5qD,IAAIm5C,GAInBrV,EAAG/F,KAAKiF,QAAQ4C,KAAK,wDA6BjBD,cACN6jB,EAAQrkE,KAAKw2E,gBAAgBh2B,OACxB,WACT62B,EAAKC,sBAAsBjT,KAC1B,iDAQiBA,SAAO9J,8DAAQ96D,KAE9B4kE,GAAUA,EAAMvH,kBAEfiT,EAAa/vE,KAAK+vE,WAAW73B,aAGjCmsB,EAAMtH,WADKt9D,MAAT86D,IACmBA,GAED8J,EAAMtH,eAGxBwa,EAAexH,EAAWhmE,IAAIs6D,EAAMlI,SACxCob,EAAaxa,WAAasH,EAAMtH,mBAE5Bya,EAAmBnT,EAAMvH,aACzB2a,EAAYD,EACTC,EAAUz6E,OAAS,GAAG,KACvBi+C,EAAUw8B,EACdA,EAAY,OACP,IAAI36E,EAAI,EAAGA,EAAIm+C,EAAQj+C,OAAQF,IAAK,KACnC46E,EAAO3H,EAAWhmE,IAAIkxC,EAAQn+C,IAC9B46E,EAAK5a,eACP2a,EAAY5G,GAAA4G,QAAAA,EAAiBC,EAAK5a,eAGlC2a,EAAUz6E,OAAS,IACrBw6E,EAAmB3G,GAAA2G,QAAAA,EAAwBC,QAG3C3a,EAAe/lB,KAAAg5B,EAAWhmE,IAAIytE,YAAsB,SAAUlC,UACnC71E,MAAvB61E,EAAYha,UACdga,EAAYha,SAAU,GAExBga,EAAYha,UAAYic,EAAaxa,WAC9BuY,KAGXvF,EAAW55B,OAAO06B,GAAA/T,QAAAA,EAAoBya,IAElCA,EAAaxa,YACfzkB,GAAKwV,gBAAgBuW,EAAMjmB,IAAI7V,MAAO,aACtC+P,GAAKuV,aAAawW,EAAMjmB,IAAI7V,MAAO,cAEnC+P,GAAKwV,gBAAgBuW,EAAMjmB,IAAI7V,MAAO,YACtC+P,GAAKuV,aAAawW,EAAMjmB,IAAI7V,MAAO,gEAQd87B,GACvBA,EAAMjmB,IAAI7V,MAAMovC,UAAUC,OAAO,yBACjCvT,EAAMjmB,IAAI4J,WAAW2vB,UAAUC,OAAO,mEAStBp3B,GACZxgD,KAAK89D,iBAAiBC,YAEtB/9D,KAAKmvB,QAAQwuC,cAAcC,aACxBE,iBAAiBuG,MAAQrkE,KAAKw2E,gBAAgBh2B,GAE/CxgD,KAAK89D,iBAAiBuG,QACxB7jB,EAAM8D,uBAEDwZ,iBAAiBC,YAAa,OAC9B8Z,yBAAyB73E,KAAK89D,iBAAiBuG,YAE/CvG,iBAAiBga,cAAgB93E,KAAK+vE,WAAW33B,OAAO,CAC3DwlB,MAAO59D,KAAKmvB,QAAQ6/C,oDAYfxuB,MACLxgD,KAAKmvB,QAAQwuC,cAAcC,OAAS59D,KAAK89D,iBAAiBuG,MAAO,CACjE7jB,EAAM8D,sBAENyrB,EAAa/vE,KAAK+vE,WAClB/vE,KAAK+vE,sBAAsBuE,KAC7BvE,EAAa/vE,KAAK+vE,WAAW73B,kBAGrBmsB,EAAQrkE,KAAKw2E,gBAAgBh2B,MAG/B6jB,GAASA,EAAMx7B,QAAU7oC,KAAK89D,iBAAiBuG,MAAMx7B,OAAQ,KACvDkvC,EAAY1T,EAAMvhB,IAAM9iD,KAAK89D,iBAAiBuG,MAAMvhB,IACpDV,EAAU5B,EAAMxB,OAASwB,EAAMxB,OAAOz1C,EAAIi3C,EAAM4B,QAChD41B,EAAc3T,EAAMjmB,IAAI4J,WAAWpF,wBACnCq1B,EAAqBj4E,KAAK89D,iBAAiBuG,MAAMx7B,UACnDkvC,MAEIC,EAAYl1B,IAAMm1B,EAAqB71B,aAGxC,KACG81B,EAAoB7T,EAAMx7B,UAE5BmvC,EAAYl1B,IAAMo1B,EAAoBD,EAAqB71B,aAMnEiiB,GAASA,GAASrkE,KAAK89D,iBAAiBuG,MAAO,KACzC2T,EAAcjI,EAAWhmE,IAAIs6D,EAAMlI,SACnCgc,EAAepI,EAAWhmE,IAAI/J,KAAK89D,iBAAiBuG,MAAMlI,SAG5Dgc,GAAgBH,SACX7oD,QAAQy/C,eAAeuJ,EAAcH,EAAajI,GACvDA,EAAW55B,OAAOgiC,GAClBpI,EAAW55B,OAAO6hC,QAIhBI,EAAWrI,EAAW33B,OAAO,CACjCwlB,MAAO59D,KAAKmvB,QAAQ6/C,iBAKjB12B,GAAK68B,WAAWiD,EAAUp4E,KAAK89D,iBAAiBga,uBAC3CO,EAAYr4E,KAAK89D,iBAAiBga,cAClCQ,EAAYt4E,KAAK89D,iBAAiBuG,MAAMlI,QACxCoc,EAAY/2E,KAAKY,IAAIi2E,EAAUr7E,OAAQo7E,EAASp7E,QAClDw7E,EAAS,EACTtB,EAAY,EACZuB,EAAY,EACTD,EAASD,GAAW,MAEfC,EAAOtB,EAAaqB,GACpBC,EAAOC,EAAaF,GACrBH,EAASI,EAAOtB,IAAcmB,EAAUG,EAAOC,IAClDD,OAIAA,EAAOtB,GAAaqB,WAMpBH,EAASI,EAAOtB,IAAcoB,EAC9BpB,EAAY,OAIX,GAAImB,EAAUG,EAAOC,IAAcH,EACpCG,EAAY,MAKX,KACKC,EAAkB9kB,GAAAwkB,QAAAA,EAAiBC,EAAUG,EAAOC,IACpDE,EAAc5I,EAAWhmE,IAAIquE,EAASI,EAAOtB,IAC7C0B,EAAgB7I,EAAWhmE,IAAIsuE,EAAUG,EAAOC,SACjDtpD,QAAQy/C,eAAe+J,EAAaC,EAAe7I,GACxDA,EAAW55B,OAAOwiC,GAClB5I,EAAW55B,OAAOyiC,OAEZC,EAAgBT,EAASI,EAAOtB,GACtCkB,EAASI,EAAOtB,GAAamB,EAAUG,EAAOC,GAC9CL,EAASM,GAAmBG,EAE5BL,gDAeRh4B,WACTsd,iBAAiBC,YAAa,EAE/B/9D,KAAKmvB,QAAQwuC,cAAcC,OAAS59D,KAAK89D,iBAAiBuG,MAAO,CACnE7jB,EAAM8D,sBAGA3F,EAAK3+C,KACLozB,EAAKurB,EAAGmf,iBAAiBuG,MAAMlI,QAC/B2c,EAAUn6B,EAAGoxB,WAAW73B,aACxBu4B,EAAYn4B,GAAKh7C,OAAO,GAAIw7E,EAAQ/uE,IAAIqpB,IAC9CurB,EAAGxvB,QAAQugD,YAAYe,GAAW,SAAAA,MAC5BA,EAEFA,EAAUqI,EAAQphC,SAAWtkB,EAC7B0lD,EAAQ3iC,OAAOs6B,OAEZ,KAGG2H,EAAWU,EAAQ1gC,OAAO,CAC5BwlB,MAAOjf,EAAGxvB,QAAQ6/C,iBAIjB12B,GAAK68B,WAAWiD,EAAUz5B,EAAGmf,iBAAiBga,uBAC3CO,EAAY15B,EAAGmf,iBAAiBga,cAChCS,EAAY/2E,KAAKY,IAAIi2E,EAAUr7E,OAAQo7E,EAASp7E,QAClDw7E,EAAS,EACNA,EAASD,GAAW,MAElBC,EAASD,GAAaH,EAASI,IAAWH,EAAUG,IACzDA,OAIEA,GAAUD,YAMRG,EAAkB9kB,GAAAwkB,QAAAA,EAAiBC,EAAUG,IAC7CG,EAAcG,EAAQ/uE,IAAIquE,EAASI,IACnCI,EAAgBE,EAAQ/uE,IAAIsuE,EAAUG,IAC5C75B,EAAGxvB,QAAQy/C,eAAe+J,EAAaC,EAAeE,GACtDA,EAAQ3iC,OAAOwiC,GACfG,EAAQ3iC,OAAOyiC,OAETC,EAAgBT,EAASI,GAC/BJ,EAASI,GAAUH,EAAUG,GAC7BJ,EAASM,GAAmBG,EAE5BL,SAMR75B,EAAG/F,KAAKiF,QAAQ4C,KAAK,eAAgB,CAAE0b,QAAS/oC,SAC3CykD,yBAAyB73E,KAAK89D,iBAAiBuG,YAC/CvG,iBAAiBuG,MAAQ,4CASpB7jB,MACPxgD,KAAKmvB,QAAQ+0C,gBAEZ2R,EAAWr1B,EAAM4D,WAAa5D,EAAM4D,SAASyxB,SAAWr1B,EAAM4D,SAAS0xB,SACvEhpB,EAAWtM,EAAM4D,UAAY5D,EAAM4D,SAAS0I,YAC9C+oB,GAAW/oB,OACR0kB,mBAAmBhxB,YAIpBu4B,EAAe/4E,KAAKm2E,eAEpBp+B,EAAO/3C,KAAKw1E,eAAeh1B,GAC3BywB,EAAYl5B,GAAQA,EAAKmsB,WAAa,CAACnsB,EAAK3kB,IAAM,QACnD4lD,aAAa/H,OAEZgI,EAAej5E,KAAKm2E,gBAItB8C,EAAaj8E,OAAS,GAAK+7E,EAAa/7E,OAAS,SAC9C47C,KAAKiF,QAAQ4C,KAAK,SAAU,CAC/BjK,MAAOyiC,EACPz4B,MAAAA,2CAUOA,OACLzI,EAAO/3C,KAAKw1E,eAAeh1B,MAC5BzI,GAIDA,IADY/3C,KAAKk5E,sBAAsB14B,QAMrCiP,EAAQ1X,EAAKohC,cACfn5E,KAAKmvB,QAAQygD,cAAgBngB,EAAO,CACpB,MAAdzvD,KAAKkxE,aACFA,MAAQ,IAAIpH,GAAM9pE,KAAK44C,KAAKwF,IAAI+C,KACjCnhD,KAAKmvB,QAAQm3C,QAAQyD,gBAAkB,cAGxCmH,MAAMkI,QAAQ3pB,OACbxD,EAAYjsD,KAAK44C,KAAKwF,IAAI3E,gBAC1Bi9B,EAAgBzqB,EAAUrJ,6BAC3BsuB,MAAMyF,YACTn2B,EAAM2B,QAAUu0B,EAAchiC,KAAOuX,EAAU6S,WAC/Cte,EAAM4B,QAAUs0B,EAAc5zB,IAAMmJ,EAAU4S,gBAE3Cwa,cAAcr5E,KAAKkxE,iBAInBoB,kBACa,MAAdtyE,KAAKkxE,YACFA,MAAMhhB,YAIVtX,KAAKiF,QAAQ4C,KAAK,WAAY,CACjC1I,KAAMA,EAAK3kB,GACXotB,MAAAA,yCAUQA,OACJzI,EAAO/3C,KAAKw1E,eAAeh1B,GAC5BzI,IAIDA,IADY/3C,KAAKk5E,sBAAsB14B,UAMtC8xB,kBACa,MAAdtyE,KAAKkxE,YACFA,MAAMhhB,YAGRtX,KAAKiF,QAAQ4C,KAAK,UAAW,CAChC1I,KAAMA,EAAK3kB,GACXotB,MAAAA,2CAUSA,MACExgD,KAAKw1E,eAAeh1B,KAGV,MAAnBxgD,KAAKmxE,iBAEFkI,cAAcr5E,KAAKkxE,OAGtBlxE,KAAKmvB,QAAQygD,cAAgB5vE,KAAKmvB,QAAQm3C,QAAQuJ,aAAe7vE,KAAKkxE,QAAUlxE,KAAKkxE,MAAM32B,QAAQ,KAC/F0R,EAAYjsD,KAAK44C,KAAKwF,IAAI3E,gBAC1Bi9B,EAAgBzqB,EAAUrJ,6BAC3BsuB,MAAMyF,YACTn2B,EAAM2B,QAAUu0B,EAAchiC,KAAOuX,EAAU6S,WAC/Cte,EAAM4B,QAAUs0B,EAAc5zB,IAAMmJ,EAAU4S,gBAE3CqS,MAAM1N,8CASDhjB,GACRxgD,KAAKslE,YAAYC,qBACdvnB,WAAWwC,yCASNzI,MACP/3C,KAAKmvB,QAAQ+0C,aACblkE,KAAKmvB,QAAQmhC,SAASkU,YAAexkE,KAAKmvB,QAAQmhC,SAAS+V,kBAE1D1nB,EAAK3+C,QAEP+3C,EAAM,KAEFic,EAAWrV,EAAG8mB,UAAU17D,IAAIguC,EAAK3kB,SAClCjE,QAAQkgD,SAASrb,GAAU,SAAAA,GAC1BA,GACFrV,EAAG8mB,UAAUtvB,OAAO6d,oDAYRxT,OACZzI,EAAO/3C,KAAKw1E,eAAeh1B,GAC3B2uB,EAAa3pC,KAAK4R,MAAMoJ,EAAMqT,aAAaI,QAAQ,cACpD9kC,QAAQ+/C,mBAAmBC,EAAYp3B,sCAQnCyI,MACJxgD,KAAKmvB,QAAQ+0C,YACblkE,KAAKmvB,QAAQmhC,SAASz1C,SAWvB4lB,EAEA64C,EAXE36B,EAAK3+C,KACL8wD,EAAO9wD,KAAKmvB,QAAQ2hC,MAAQ,KAG5BylB,EAAYv2E,KAAKo+C,IAAIuqB,MAAM/lB,wBAC3Bz4C,EAAInK,KAAKmvB,QAAQguB,IAAMo5B,EAAU5hC,MAAQ6L,EAAMxB,OAAO70C,EAAIq2C,EAAMxB,OAAO70C,EAAIosE,EAAU7hC,KACrFlU,EAAQxgC,KAAK44C,KAAKN,KAAK2D,OAAO9xC,GAC9BwxC,EAAQ37C,KAAK44C,KAAKN,KAAKsY,WACvB7sB,EAAO/jC,KAAK44C,KAAKN,KAAKuY,UAIV,QAAdrQ,EAAMtpB,OACRoiD,EAAc9zC,KAAK4R,MAAMoJ,EAAMqT,aAAaI,QAAQ,UACxCpgC,QAAUylD,EAAYzlD,QAAUylD,EAAYzlD,QAAU,WAClEylD,EAAY94C,MAAQ84C,EAAY94C,MAAQ84C,EAAY94C,MAASswB,EAAOA,EAAKtwB,EAAOmb,EAAO5X,GAAQvD,EAC/F84C,EAAYpiD,KAAOoiD,EAAYpiD,MAAQ,MACvCoiD,EAAYt5E,KAAKylE,UAAUhuB,QAAU6hC,EAAYlmD,IAAMs3C,KAE/B,SAApB4O,EAAYpiD,MAAoBoiD,EAAY74C,MAC9CA,EAAMzgC,KAAK44C,KAAKN,KAAK2D,OAAO9xC,EAAInK,KAAKq1C,MAAMzM,MAAQ,GACnD0wC,EAAY74C,IAAMqwB,EAAOA,EAAKrwB,EAAKkb,EAAO5X,GAAQtD,MAGpD64C,EAAc,CACZ94C,MAAOswB,EAAOA,EAAKtwB,EAAOmb,EAAO5X,GAAQvD,EACzC3M,QAAS,aAEC7zB,KAAKylE,UAAUhuB,QAAUizB,KAGX,UAAtB1qE,KAAKmvB,QAAQ+H,OACfuJ,EAAMzgC,KAAK44C,KAAKN,KAAK2D,OAAO9xC,EAAInK,KAAKq1C,MAAMzM,MAAQ,GACnD0wC,EAAY74C,IAAMqwB,EAAOA,EAAKrwB,EAAKkb,EAAO5X,GAAQtD,QAIhD4jC,EAAQrkE,KAAKw2E,gBAAgBh2B,GAC/B6jB,IACFiV,EAAYjV,MAAQA,EAAMlI,SAI5Bmd,EAAct5E,KAAK41E,eAAe0D,QAC7BnqD,QAAQigD,MAAMkK,GAAa,SAAAvhC,GAC1BA,IACF4G,EAAG8mB,UAAU5qD,IAAIk9B,GACC,QAAdyI,EAAMtpB,MACRynB,EAAGq6B,aAAa,CAACjhC,EAAK3kB,qDAYXotB,iBACZxgD,KAAKmvB,QAAQ+0C,gBAEZnsB,EAAO/3C,KAAKw1E,eAAeh1B,MAE7BzI,EAAM,KAGJk5B,EAAYjxE,KAAKmvB,QAAQ8/C,YACzBjvE,KAAKm2E,eACL,OAEa31B,EAAM4D,UAAY5D,EAAM4D,SAAS0I,WAAY,GAE7C9sD,KAAKmvB,QAAQ6gD,sBAAwBhwE,KAAKmvB,QAAQ8/C,YAAa,KAExEsK,EAAYv5E,KAAKylE,UAAU17D,IAAIguC,EAAK3kB,IAAIixC,MAG1CmV,OAAoB/5E,EACpBO,KAAKmvB,QAAQsqD,qBACXxI,EAAUj0E,OAAS,IACrBw8E,EAAoBx5E,KAAKylE,UAAU17D,IAAIknE,EAAU,IAAI5M,OAKpDrkE,KAAKmvB,QAAQsqD,qBAA4Ch6E,MAArB+5E,GAAkCA,GAAqBD,GAC9FtI,EAAUh0E,KAAK86C,EAAK3kB,QAEhBsmB,EAAQi1B,EAAQ+K,cAAc15E,KAAKylE,UAAU17D,IAAIknE,QAElDjxE,KAAKmvB,QAAQsqD,qBAAuBD,GAAqBD,MAGvD,IAAMnmD,KADX69C,EAAY,GACKjxE,KAAKw2C,SAChBx2C,KAAKw2C,MAAMn5C,eAAe+1B,GAAK,KAC3BumD,EAAQ35E,KAAKw2C,MAAMpjB,GACnBoN,EAAQm5C,EAAMpmE,KAAKitB,MACnBC,OAA0BhhC,IAAnBk6E,EAAMpmE,KAAKktB,IAAqBk5C,EAAMpmE,KAAKktB,IAAMD,IAE1DA,GAASkZ,EAAMt3C,KACfq+B,GAAOiZ,EAAMp0C,MACXtF,KAAKmvB,QAAQsqD,qBAAuBD,GAAqBx5E,KAAKylE,UAAU17D,IAAI4vE,EAAMvmD,IAAIixC,OACtFsV,aAAiB/P,IACrBqH,EAAUh0E,KAAK08E,EAAMvmD,SAM1B,KAEGxO,EAAQgvC,GAAAqd,QAAAA,EAAkBl5B,EAAK3kB,KACvB,GAAVxO,EAEFqsD,EAAUh0E,KAAK86C,EAAK3kB,OAIpB69C,QAAAA,EAAiBrsD,EAAO,OAItBg1D,EAAoB9iC,GAAAm6B,QAAAA,GAAiB,SAAAl5B,UAAQ8hC,EAAKnH,YAAY36B,GAAMmsB,mBAErE8U,aAAaY,QAEbhhC,KAAKiF,QAAQ4C,KAAK,SAAU,CAC/BjK,MAAOx2C,KAAKm2E,eACZ31B,MAAAA,8CA4CU5Y,WACVkyC,EAAMlyC,EACHkyC,GAAK,IACNA,EAAIz8E,eAAe,mBACdy8E,EAAI,YAEbA,EAAMA,EAAIvyC,kBAGL,4CASMiZ,UACNxgD,KAAK+5E,gBAAgBv5B,EAAM7wB,sDASd6wB,UACbxgD,KAAK+5E,gBAAgBv5B,EAAMw5B,uDASpBx5B,OACR4B,EAAU5B,EAAMxB,OAASwB,EAAMxB,OAAOz1C,EAAIi3C,EAAM4B,QAClD4uB,EAAWhxE,KAAKgxE,SAEhBA,EAASh0E,QAAU,GAAKgD,KAAK+vE,aAC/BiB,EAAWhxE,KAAK+vE,WAAW33B,OAAO,CAChCwlB,MAAO59D,KAAKmvB,QAAQ6/C,kBAInB,IAAIlyE,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,IAAK,KAClCq/D,EAAU6U,EAASl0E,GACnBunE,EAAQrkE,KAAK2sE,OAAOxQ,GACpBnU,EAAaqc,EAAMjmB,IAAI4J,WACvBiyB,EAAiBjyB,EAAWpF,2BAC9BR,GAAW63B,EAAen3B,KAAOV,EAAU63B,EAAen3B,IAAMkF,EAAW0O,oBACtE2N,KAG6B,QAAlCrkE,KAAKmvB,QAAQk5B,YAAYtQ,SACvBj7C,IAAMkD,KAAKgxE,SAASh0E,OAAS,GAAKolD,EAAU63B,EAAen3B,WACtDuhB,UAIC,IAANvnE,GAAWslD,EAAU63B,EAAen3B,IAAMkF,EAAWxrC,cAChD6nD,SAKN,4CA8BMrQ,EAAU98B,OACjB9Z,EAAQk7B,GAAKh7C,OAAO,GAAI02D,UAEzB98B,IAEHA,EAAOl3B,KAAKylE,UAAUvuC,MAGLz3B,MAAf2d,EAAMojB,QACRpjB,EAAMojB,MAAQ8X,GAAKpB,QAAQ95B,EAAMojB,MAAOtJ,GAAQA,EAAKsJ,OAAS,SAE/C/gC,MAAb2d,EAAMqjB,MACRrjB,EAAMqjB,IAAM6X,GAAKpB,QAAQ95B,EAAMqjB,IAAMvJ,GAAQA,EAAKuJ,KAAO,SAGpDrjB,6CASFpd,KAAKmvB,QAAQ2wC,aAIVnkB,EAAU37C,KAAK44C,KAAKc,MAAMgC,WAAW17C,KAAK44C,KAAKY,SAASwF,OAAOpW,OAA/D+S,MACF4xB,EAAWvtE,KAAKowE,iBAAiB8J,YAAYl6E,KAAKutE,SAAU5xB,EAAO37C,KAAKmvB,QAAQ2wC,YAElF9/D,KAAKutE,UAAYA,EAAU,SACxB8E,qBAED9E,EAAU,2CACQA,+BAAU,SACpBxC,iGAELwC,SAAWA,OAGb4M,gBAAgB5M,qDASnBvtE,KAAKmvB,QAAQ2wC,SACX9/D,KAAKutE,UAAYvtE,KAAKutE,SAASvwE,OAAQ,2CACrBgD,KAAKutE,sCAAU,SACzBzC,sIAWAyC,MACVvtE,KAAKutE,UAAYvtE,KAAKutE,SAASvwE,OAAQ,OACnCo9E,EAAiB,OAAQrjC,GAAAw2B,QAAAA,GAAa,SAAAzN,UAAWA,EAAQ1sC,OACzDinD,EAAqBvjC,UAAKy2B,kBAAgB,SAAAzN,UAAYsa,EAAetsD,IAAIgyC,EAAQ1sC,OACnFknD,GAAmB,wCACHD,+BAAoB,SAA/Bva,UACDya,EAAc3mB,UAAKqd,kBAAkBnR,EAAQ1sC,QAC9B,IAAjBmnD,EACFza,EAAQ6S,qBACH1B,kBAAiBsJ,EAAa,GACnCD,GAAmB,wFAInBA,EAAkB,KACdrB,EAAej5E,KAAKm2E,oBACrBv9B,KAAKiF,QAAQ4C,KAAK,SAAU,CAC/BjK,MAAOyiC,EACPz4B,MAAOA,cAKR+sB,SAAWA,GAAY,2CA3NT9H,OACfngE,EAAM,KACNlD,EAAM,eAEVqjE,QAAAA,GAAkB,SAAAlyD,IACL,MAAPnR,GAAemR,EAAKitB,MAAQp+B,KAC9BA,EAAMmR,EAAKitB,OAGG/gC,MAAZ8T,EAAKktB,KACI,MAAPn7B,GAAeiO,EAAKktB,IAAMn7B,KAC5BA,EAAMiO,EAAKktB,MAIF,MAAPn7B,GAAeiO,EAAKitB,MAAQl7B,KAC9BA,EAAMiO,EAAKitB,UAKV,CACLp+B,IAAAA,EACAkD,IAAAA,6CAwFqBk7C,WACnB7wB,EAAS6wB,EAAM7wB,OACZA,GAAQ,IACTA,EAAOtyB,eAAe,sBACjBsyB,EAAO,eAEhBA,EAASA,EAAO4X,kBAGX,YA9/EWoR,IAsmFtBg2B,GAAQgG,MAAQ,CACd/rB,WAAYghB,GACZ/E,IAAK0B,GACL7sB,MAAOgvB,GACP5oC,MAAOsoC,IAQTuG,GAAQxyE,UAAUg0E,OAASxB,GAAQxyE,UAAUk0E,UChpF7C,IACImK,GADAC,IAAa,EAEbC,GAAa,sCAIXC,uFAeYxrD,EAASyrD,EAAkBC,GACzCJ,IAAa,EACbD,GAAaI,MACTE,EAAcF,cACAn7E,IAAdo7E,IACFC,EAAcF,EAAiBC,IAEjCF,EAAUvjC,MAAMjoB,EAAS2rD,EAAa,IAC/BL,iCAWItrD,EAASyrD,EAAkB3qD,OACjC,IAAIqmC,KAAUnnC,EACbA,EAAQ9xB,eAAei5D,IACzBqkB,EAAUxwD,MAAMmsC,EAAQnnC,EAASyrD,EAAkB3qD,iCAc5CqmC,EAAQnnC,EAASyrD,EAAkB3qD,WACbxwB,IAA7Bm7E,EAAiBtkB,SAAsD72D,IAA7Bm7E,EAAiBG,aAK3DC,EAAkB1kB,EAClB2kB,GAAY,OAEiBx7E,IAA7Bm7E,EAAiBtkB,SAAsD72D,IAA7Bm7E,EAAiBG,UAM7DC,EAAkB,UAIlBC,EAAoD,WAAvCN,EAAUrjC,QAAQnoB,EAAQmnC,SAQrC4kB,EAAeN,EAAiBI,GAChCC,QAAuCx7E,IAA1By7E,EAAaC,WAC5BD,EAAeA,EAAaC,UAG9BR,EAAUS,YAAY9kB,EAAQnnC,EAASyrD,EAAkBI,EAAiBE,EAAcjrD,QA9BtF0qD,EAAUU,cAAc/kB,EAAQskB,EAAkB3qD,uCA2CnCqmC,EAAQnnC,EAASyrD,EAAkBI,EAAiBE,EAAcjrD,OAC/E+pB,EAAM,SAASshC,GACjB34E,QAAQq3C,IAAI,KAAOshC,EAAUX,EAAUY,cAActrD,EAAMqmC,GAASokB,KAGlEc,EAAab,EAAUrjC,QAAQnoB,EAAQmnC,IACvCmlB,EAAgBP,EAAaM,QAEX/7E,IAAlBg8E,EAEuC,UAArCd,EAAUrjC,QAAQmkC,KAA0E,IAA5C7nB,GAAA6nB,QAAAA,EAAsBtsD,EAAQmnC,KAChFtc,EAAI,+BAAiCsc,EAAS,yBACnBqkB,EAAUe,MAAMD,GACzC,SAAWtsD,EAAQmnC,GAAU,OAC/BmkB,IAAa,GAES,WAAfe,GAA+C,YAApBR,IAClC/qD,EAAOqoB,GAAKqjC,mBAAmB1rD,EAAMqmC,GACrCqkB,EAAUvjC,MAAMjoB,EAAQmnC,GAASskB,EAAiBI,GAAkB/qD,SAGvCxwB,IAAxBy7E,EAAY,MAEnBlhC,EAAI,8BAAgCsc,EAClC,gBAAkBqkB,EAAUe,MAAM17B,GAAYk7B,IAC9C,eAAkBM,EAAa,MAAQrsD,EAAQmnC,GAAU,KAC3DmkB,IAAa,mCAUF3rD,OACToI,KAAcpI,SAEL,WAAToI,EACa,OAAXpI,EACK,OAELA,aAAkBmT,QACb,UAELnT,aAAkBuoB,OACb,SAELvoB,aAAkBL,OACb,SAELyU,GAAcpU,GACT,QAELA,aAAkBpyB,KACb,YAEe+C,IAApBqvB,EAAO8sD,SACF,OAEuB,IAA5B9sD,EAAOtuB,iBACF,SAEF,SAES,WAAT02B,EACA,SAES,YAATA,EACA,UAES,WAATA,EACA,cAESz3B,IAATy3B,EACA,YAEFA,wCASYo/B,EAAQnnC,EAASc,OAOhCxtB,EANAo5E,EAAclB,EAAUmB,cAAcxlB,EAAOnnC,EAAQc,GAAK,GAC1D8rD,EAAepB,EAAUmB,cAAcxlB,EAAOkkB,GAAW,IAAG,GAO9D/3E,OAD6BhD,IAA3Bo8E,EAAYG,WACR,OAASrB,EAAUY,cAAcM,EAAY5rD,KAAMqmC,EAAO,IAC9D,6CAA+CulB,EAAYG,WAAa,SAEnED,EAAa5lB,UAPM,GAO+B0lB,EAAY1lB,SAAW4lB,EAAa5lB,SACvF,OAASwkB,EAAUY,cAAcM,EAAY5rD,KAAMqmC,EAAO,IAC9D,uDACAqkB,EAAUY,cAAcQ,EAAa9rD,KAAM8rD,EAAaE,aAAa,IAEhEJ,EAAY1lB,UAbM,EAcnB,mBAAqB0lB,EAAYI,aAAe,KACpDtB,EAAUY,cAAcM,EAAY5rD,KAAMqmC,GAGtC,gCAAkCqkB,EAAUe,MAAM17B,GAAY7wB,IACpEwrD,EAAUY,cAActrD,EAAMqmC,GAGhC3zD,QAAQq3C,IAAI,+BAAiCsc,EAAS,IAAM7zD,EAAKi4E,IACjED,IAAa,wCAYMnkB,EAAQnnC,EAASc,OAAMisD,0DACtC95E,EAAM,IACN65E,EAAe,GACfE,EAAmB,GACnBC,EAAkB9lB,EAAOhyD,cACzB03E,OAAav8E,MACZ,IAAI48E,KAAMltD,EAAS,KAClBgnC,iBACyB12D,IAAzB0vB,EAAQktD,GAAIlB,WAAwC,IAAde,EAAoB,KACxD3jE,EAASoiE,EAAUmB,cAAcxlB,EAAQnnC,EAAQktD,GAAK/jC,GAAKqjC,mBAAmB1rD,EAAKosD,IACnFj6E,EAAMmW,EAAO49C,WACf8lB,EAAe1jE,EAAO0jE,aACtBE,EAAmB5jE,EAAO0X,KAC1B7tB,EAAMmW,EAAO49C,SACb6lB,EAAazjE,EAAOyjE,gBAGnB,QACgD,IAA/CpoB,KAAAyoB,EAAG/3E,sBAAsB83E,KAC3BJ,EAAaK,GAGXj6E,GADJ+zD,EAAWwkB,EAAU2B,oBAAoBhmB,EAAQ+lB,MAE/CJ,EAAeI,EACfF,EAAmB7jC,GAAKikC,UAAUtsD,GAClC7tB,EAAM+zD,UAIL,CAAC8lB,aAAaA,EAAchsD,KAAKksD,EAAkBhmB,SAAS/zD,EAAK45E,WAAYA,yCAUjE/rD,EAAMqmC,WAAQpzC,yDAAS,6BACtC2I,EAAM,OAAS3I,EAAS,gBACnBpmB,EAAI,EAAGA,EAAImzB,EAAKjzB,OAAQF,IAAK,KAC/B,IAAIiX,EAAI,EAAGA,EAAIjX,EAAI,EAAGiX,IACzB8X,GAAO,KAETA,GAAOoE,EAAKnzB,GAAK,YAEd,IAAIiX,EAAI,EAAGA,EAAIkc,EAAKjzB,OAAS,EAAG+W,IACnC8X,GAAO,KAETA,GAAOyqC,EAAS,SACX,IAAIx5D,EAAI,EAAGA,EAAImzB,EAAKjzB,OAAS,EAAGF,IAAK,KACnC,IAAIiX,EAAI,EAAGA,EAAIkc,EAAKjzB,OAASF,EAAGiX,IACnC8X,GAAO,KAETA,GAAO,aAEFA,EAAM,qCAQFsD,UACJ4wB,GAAe5wB,GAAS5oB,QAAQ,gCAAiC,IAAIA,QAAQ,QAAS,kDAqBpEpJ,EAAGC,MACX,IAAbD,EAAEH,OAAc,OAAOI,EAAEJ,UACZ,IAAbI,EAAEJ,OAAc,OAAOG,EAAEH,WAKzBF,EAMAiX,EATAyoE,EAAS,OAIR1/E,EAAI,EAAGA,GAAKM,EAAEJ,OAAQF,IACzB0/E,EAAO1/E,GAAK,CAACA,OAKViX,EAAI,EAAGA,GAAK5W,EAAEH,OAAQ+W,IACzByoE,EAAO,GAAGzoE,GAAKA,MAIZjX,EAAI,EAAGA,GAAKM,EAAEJ,OAAQF,QACpBiX,EAAI,EAAGA,GAAK5W,EAAEH,OAAQ+W,IACrB3W,EAAEopB,OAAO1pB,EAAI,IAAMK,EAAEqpB,OAAOzS,EAAI,GAClCyoE,EAAO1/E,GAAGiX,GAAKyoE,EAAO1/E,EAAI,GAAGiX,EAAI,GAEjCyoE,EAAO1/E,GAAGiX,GAAKvS,KAAKY,IAAIo6E,EAAO1/E,EAAI,GAAGiX,EAAI,GAAK,EAC7CvS,KAAKY,IAAIo6E,EAAO1/E,GAAGiX,EAAI,GAAK,EAC1ByoE,EAAO1/E,EAAI,GAAGiX,GAAK,WAKtByoE,EAAOp/E,EAAEJ,QAAQG,EAAEH,iBCrV1Bia,GAAS,SACTwlE,GAAO,UACPl7E,GAAS,SAGTutB,GAAS,SAKT0rD,GAAa,CACfxlB,UAAW,CACT0nB,QAAS,SAAaD,IACtBxkD,OAAQ,SAAawkD,YAAiB,YACtCxwB,UAAW,CAAC7N,IARN,OASN+8B,SAAU,CAACrsD,OAAAA,WAAmB2tD,YAAiB,aAIjDhW,MAAO,CAACxvD,OAAAA,IACR2hD,iBAAkB,CAAC3hD,OAAAA,aAAqB,aACxCkmC,IAAK,SAAas/B,aAAmB,aACrCj/B,YAAa,CACXC,OAAQ,SAAag/B,IACrBjgE,OAAQ,CAACjb,OAAAA,aAAoB,aAC7B45E,SAAU,CAACrsD,OAAAA,KAEbkwC,UAAW,CACTK,UAAW,CAAC99D,OAAAA,IACZyE,SAAU,UAAa,YACvBm1E,SAAU,CAACrsD,OAAAA,KAEbikC,eAAgB,SAAa0pB,aAAmB,aAChDzpB,iBAAkB,SAAaypB,aAAmB,aAClD5kB,WAAY,SAAa4kB,IACzBE,eAAgB,CAACp7E,OAAAA,IACjBmzD,WAAY,SAAa+nB,IACzBxW,eAAgB,CAAChvD,OAAAA,GAAQ5Q,MAlCf,SAmCViqD,SAAU,CACRz1C,IAAK,SAAa4hE,aAAmB,aACrCxmC,OAAQ,SAAawmC,aAAmB,aACxCpW,YAAa,SAAaoW,aAAmB,aAC7CjY,WAAY,SAAaiY,aAAmB,aAC5C1X,cAAe,SAAa0X,aAAmB,aAC/CtB,SAAU,SAAasB,GAAM3tD,OAAAA,KAE/B2R,IAAK,CAACl/B,OAAAA,GAAQ0I,KA1CL,OA0CWgN,OAAAA,GAAQmT,OAvCjB,UAwCX3sB,OAAQ,CACNkpD,YAAa,CACXrsC,YAAa,CAACrD,OAAAA,aAAoB,aAClCoD,OAAQ,CAACpD,OAAAA,aAAoB,aAC7BmD,OAAQ,CAACnD,OAAAA,aAAoB,aAC7BqC,KAAM,CAACrC,OAAAA,aAAoB,aAC3BzJ,QAAS,CAACyJ,OAAAA,aAAoB,aAC9B7H,IAAK,CAAC6H,OAAAA,aAAoB,aAC1B1J,KAAM,CAAC0J,OAAAA,aAAoB,aAC3BjN,MAAO,CAACiN,OAAAA,aAAoB,aAC5B5N,KAAM,CAAC4N,OAAAA,aAAoB,aAC3BkkE,SAAU,CAACrsD,OAAAA,YAAoB,aAEjC83B,YAAa,CACXtsC,YAAa,CAACrD,OAAAA,aAAoB,aAClCoD,OAAQ,CAACpD,OAAAA,aAAoB,aAC7BmD,OAAQ,CAACnD,OAAAA,aAAoB,aAC7BqC,KAAM,CAACrC,OAAAA,aAAoB,aAC3BzJ,QAAS,CAACyJ,OAAAA,aAAoB,aAC9B7H,IAAK,CAAC6H,OAAAA,aAAoB,aAC1B1J,KAAM,CAAC0J,OAAAA,aAAoB,aAC3BjN,MAAO,CAACiN,OAAAA,aAAoB,aAC5B5N,KAAM,CAAC4N,OAAAA,aAAoB,aAC3BkkE,SAAU,CAACrsD,OAAAA,YAAoB,aAEjCqsD,SAAU,CAACrsD,OAAAA,KAEb1E,OAAQ,UAAa,YACrB6yC,gBAAiB,CAAChmD,OAAAA,IAClB+3D,WAAY,CAAC/3D,OAAAA,YAAoB,YACjC0mD,cAAe,CACb9iD,IAAK,SAAa4hE,aAAmB,aACrCxmC,OAAQ,SAAawmC,aAAmB,aACxC7e,MAAO,SAAa6e,aAAmB,aACvCtB,SAAU,SAAasB,GAAM3tD,OAAAA,KAE/B8/C,eAAgB,UAAa,YAC7B/lC,OAAQ,CAAC5xB,OAAAA,GAAQ1V,OAAAA,IACjB63C,YAAa,CACX5Y,MAAO,CAACv2B,KAlFD,OAkFO1I,OAAAA,GAAQ0V,OAAAA,GAAQmT,OA/ErB,UAgFTqW,IAAK,CAACx2B,KAnFC,OAmFK1I,OAAAA,GAAQ0V,OAAAA,GAAQmT,OAhFnB,UAiFT4uB,OAAQ,CAAC/hC,OAAAA,IACTkkE,SAAU,CAACrsD,OAAAA,GAAQzoB,MAtFX,UAwFVmjE,qBAAsB,CACpBzxB,KAAM,SAAa0kC,aAAmB,aACtC/iC,MAAO,SAAa+iC,aAAmB,aACvCtB,SAAU,SAAasB,GAAM3tD,OAAAA,KAE/Bq6C,UAAW,SAAYsT,IACvB/+E,OAAO,CAACuZ,OAAAA,IACRrE,QAAQ,CACNmoE,QAAS,CAAC6B,IA3FJ,OA4FNzB,SAAU,CAACrsD,OAAAA,KAEb0oC,oBAAqB,CAACj2D,OAAAA,IACtB+4D,OAAQ,CACNhS,KAAM,CAAC/mD,OAAAA,IACPw2C,KAAM,CACJ+jB,WAAY,CAACv6D,OAAAA,aAAoB,aACjCs5D,SAAU,CAACt5D,OAAAA,aAAoB,aAC/B45E,SAAU,CAACrsD,OAAAA,GAAOvtB,OAAAA,KAEpB45E,SAAU,CAACrsD,OAAAA,GAAOvtB,OAAAA,KAEpB+D,IAAK,CAAC2E,KA5GG,OA4GG1I,OAAAA,GAAQ0V,OAAAA,GAAQmT,OAzGjB,UA0GXisC,UAAW,CAAC90D,OAAAA,GAAQ0V,OAAAA,IACpBuxC,cAAe,CAACjnD,OAAAA,IAChBa,IAAK,CAAC6H,KA/GG,OA+GG1I,OAAAA,GAAQ0V,OAAAA,GAAQmT,OA5GjB,UA6GXosC,UAAW,CAACj1D,OAAAA,GAAQ0V,OAAAA,IACpBmmC,SAAU,SAAaq/B,IACvBxN,YAAa,SAAawN,IAC1BhD,oBAAqB,SAAagD,IAClCrN,MAAO,UAAa,YACpBF,mBAAoB,UAAa,YACjCG,SAAU,UAAa,YACvBC,OAAQ,UAAa,YACrBE,SAAU,UAAa,YACvBD,SAAU,UAAa,YACvBE,WAAY,UAAa,YACzBC,YAAa,UAAa,YAC1BC,cAAe,UAAa,YAC5BkN,sBAAuB,UAAa,YACpCjf,MAAO,UAAa,YACpBvV,YAAa,CACXC,KAAM,CAACrxC,OAAAA,aAAoB,aAC3B8gC,KAAM,CAAC9gC,OAAAA,aAAoB,aAC3BkkE,SAAU,CAAClkE,OAAAA,GAAQ6X,OAAAA,KAErBo1C,WAAY,SAAauY,IACzBzM,oBAAqB,SAAayM,IAClC9jB,gBAAiB,SAAa8jB,IAC9Bx2B,gBAAiB,SAAaw2B,IAC9Bl0B,gBAAiB,SAAak0B,IAC9Bn2B,cAAe,SAAam2B,IAC5Bp5E,MAAO,SAAao5E,IACpBlhB,eAAgB,SAAakhB,IAC7B3c,QAAS,CACPiN,SAAU,QAAWxrE,aAAqB,aAC1C+sE,cAAe,QAAWr3D,aAAqB,aAC/C+1D,gBAAiB,UAAc,qBAAyB,aACxD/B,WAAY,SAAYwR,aAAmB,aAC3CjS,iBAAkB,SAAYiS,aAAmB,aACjDtB,SAAU,SAAYsB,GAAM3tD,OAAAA,KAE9BgiC,KAAM,UAAa,gBAAoB,QACvCtwB,MAAO,CAACv2B,KArJC,OAqJK1I,OAAAA,GAAQ0V,OAAAA,GAAQmT,OAlJnB,UAmJXo7C,SAAU,UAAa,YACvBsX,sBAAuB,UAAa,YACpC50C,cAAe,UAAa,YAC5B69B,qBAAsB,CAAC9uD,OAAAA,YAAoB,YAC3C24D,aAAc,SAAa6M,IAC3BnW,QAAS,CACPuJ,YAAa,SAAa4M,IAC1B1S,eAAgB,QAAY,CAAC,MAAO,OAAQ,SAC5C+F,MAAO,CAACvuE,OAAAA,IACRikE,SAAU,UAAa,YACvB2V,SAAU,CAACrsD,OAAAA,KAEbs2C,wBAAyB,CACvBI,SAAU,UAAa,YACvB2V,SAAU,SAAasB,GAAM3tD,OAAAA,KAE/B25B,SAAU,CACR9M,MAAO,CAAC1kC,OAAAA,aAAoB,aAC5B8sB,KAAM,CAACxiC,OAAAA,aAAoB,aAC3B45E,SAAU,CAACrsD,OAAAA,KAEboI,KAAM,CAACjgB,OAAAA,IACP2xB,MAAO,CAAC3xB,OAAAA,GAAQ1V,OAAAA,IAChBuxD,WAAY,SAAa2pB,IACzBp/B,SAAU,SAAao/B,IACvB36B,QAAS,CAAC7qC,OAAQ,CAAC,UAAW,SAAU,WAAY,UAAW,KAC/D+qC,aAAc,CAACzgD,OAAAA,IACfg8C,QAAS,CAACh8C,OAAAA,IACV+7C,QAAS,CAAC/7C,OAAAA,IAEV45E,SAAU,CAACrsD,OAAAA,KAGTiuD,GAAmB,CACrB7wD,OAAQ,CACNu6C,MAAQ,CAAC,SAAU,OAAQ,SAC3B7N,iBAAkB,CAAC,OAAQ,OAAQ,QAAS,UAAW,OAAQ,UAAW,MAAO,OAAQ,OAAQ,SAAU,UAC3Gt6C,WAAY,EACZu5C,YAAY,EACZnD,YAAY,EAEVpE,SAAU,CACVz1C,KAAK,EACLo7B,QAAQ,EACRowB,aAAa,EACb7B,YAAY,GAEd/jC,IAAK,GACLhjC,OAAQ,CACNkpD,YAAa,CACXrsC,YAAY,MACZD,OAAY,IACZD,OAAY,QACZd,KAAY,QACZ9L,QAAY,QACZ4B,IAAY,IACZ7B,KAAY,IACZvD,MAAY,MACZX,KAAY,QAEdu9C,YAAa,CACXtsC,YAAY,WACZD,OAAY,eACZD,OAAY,aACZd,KAAY,aACZ9L,QAAY,YACZ4B,IAAY,YACZ7B,KAAY,YACZvD,MAAY,OACZX,KAAY,KAGhB4zD,gBAAiB,CAAC,OAAQ,QAAS,YAEnC+f,iBAAiB,EACjBn0C,OAAQ,GAERnrC,OAAQ,GACR85D,oBAAqB,IACrB8C,OAAQ,CACNhS,KAAM,CAAC,GAAI,EAAG,IAAK,GACnBvQ,KAAM,CACJ+jB,WAAY,CAAC,GAAI,EAAG,IAAK,GACzBjB,SAAU,CAAC,GAAI,EAAG,IAAK,KAG3Bv1D,IAAK,GACL+wD,UAAW,GACX7N,cAAe,CAAC,EAAG,EAAG,GAAI,GAC1BpmD,IAAK,GACLo0D,UAAW,GACXpZ,UAAU,EACV6xB,aAAa,EACbwK,qBAAqB,EAOrBpxB,YAAa,CACXC,KAAM,CAAC,OAAQ,SAAU,OACzBvQ,KAAM,CAAC,SAAU,QAEnB+a,YAAY,EACZoR,YAAY,EACZvL,iBAAiB,EACjB1S,iBAAiB,EACjBsC,iBAAiB,EACjBllD,OAAO,EACPk4D,gBAAgB,EAChBuE,SAAS,EAETt/B,MAAO,GAMPovC,cAAc,EACdtJ,QAAS,CACPuJ,aAAa,EACb9F,eAAgB,OAChB+F,MAAO,CAAC,IAAK,EAAG,MAAO,MAEzB1K,yBAAyB,EACzBluC,KAAM,CAAC,MAAO,QAAS,QAAS,cAChC0R,MAAO,OACPyU,UAAU,EACVyE,QAAS,CAAC,UAAW,SAAU,WAAY,UAAW,IACtDvE,QAAS,CAAC,SAAiB,GAAI,SAAiB,GAChDD,QAAS,CAAC,GAAI,GAAI,SAAiB,OC9RrC,CAAE3tB,OAAQ,QAASnP,OAAO,GAAQ,CAClCy8D,KCAe,SAAcn7E,WACzB2rB,EAAI9K,GAAS3iB,MACbhD,EAASq0B,EAAS5D,EAAEzwB,QACpBgnC,EAAkBloC,UAAUkB,OAC5B4nB,EAAQ0M,GAAgB0S,EAAkB,EAAIloC,UAAU,QAAK2D,EAAWzC,GACxEyjC,EAAMuD,EAAkB,EAAIloC,UAAU,QAAK2D,EAC3Cy9E,OAAiBz9E,IAARghC,EAAoBzjC,EAASs0B,GAAgBmP,EAAKzjC,GACxDkgF,EAASt4D,GAAO6I,EAAE7I,KAAW9iB,SAC7B2rB,KCZT,OAAiBoT,GAAa,SAASo8C,KCDnCx7C,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGmxD,YACNnxD,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAew7C,KAAQA,GAAOl8C,GCD5Fo8C,GAAa,CAACC,MAAO,UAAWC,KAAM,UAAWC,SAAU,UAAWC,WAAY,UAAWC,KAAM,UAAWC,UAAW,UAAWC,MAAO,UAAWC,KAAM,UAAWC,SAAU,UAAWC,YAAa,UAAWC,cAAe,UAAWC,kBAAmB,UAAWC,KAAM,UAAWC,YAAa,UAAWC,KAAM,UAAWC,KAAM,UAAWC,aAAc,UAAWC,WAAY,UAAWC,cAAe,UAAWC,YAAa,UAAWC,SAAU,UAAWC,cAAe,UAAWC,UAAW,UAAWC,eAAgB,UAAWC,UAAW,UAAWC,UAAW,UAAWC,UAAW,UAAWC,cAAe,UAAWC,gBAAiB,UAAWC,OAAQ,UAAWC,eAAgB,UAAWC,UAAW,UAAWC,eAAgB,UAAWC,iBAAkB,UAAWC,QAAS,UAAWC,UAAW,UAAWC,UAAW,UAAWC,UAAW,UAAWC,eAAgB,UAAWC,gBAAiB,UAAWC,UAAW,UAAWC,WAAY,UAAWC,WAAY,UAAWC,OAAQ,UAAWC,OAAQ,UAAWC,MAAO,UAAWC,KAAM,UAAWC,QAAS,UAAWC,aAAc,UAAWC,WAAY,UAAWC,QAAS,UAAWC,YAAa,UAAWC,YAAa,UAAWC,aAAc,UAAWC,WAAY,UAAWC,aAAc,UAAWC,WAAY,UAAWC,UAAW,UAAWC,WAAY,UAAWC,YAAa,UAAWC,OAAQ,UAAWC,MAAO,UAAWC,SAAU,UAAWC,UAAW,UAAWC,YAAa,UAAWC,cAAe,UAAWC,eAAgB,UAAWC,WAAY,UAAWC,UAAW,UAAWC,cAAe,UAAWC,aAAc,UAAWC,UAAW,UAAWC,UAAW,UAAWC,OAAQ,UAAWC,gBAAiB,UAAWC,UAAW,UAAWC,KAAM,UAAWC,UAAW,UAAWC,IAAK,UAAWC,UAAW,UAAWC,cAAe,UAAWC,QAAS,UAAWC,OAAQ,UAAWC,UAAW,UAAWC,QAAS,UAAWC,UAAW,UAAWC,KAAM,UAAWC,UAAW,UAAWC,UAAW,UAAWC,SAAU,UAAWC,WAAY,UAAWC,OAAQ,UAAWC,cAAe,UAAWC,WAAY,UAAWC,MAAO,UAAWC,UAAW,UAAWC,SAAU,UAAWC,MAAO,UAAWC,WAAY,UAAWC,MAAO,UAAWC,MAAO,UAAWC,WAAY,UAAWC,UAAW,UAAWC,WAAY,UAAWC,OAAQ,UAAWC,aAAc,UAAWC,MAAO,UAAWC,qBAAsB,UAAWC,QAAS,UAAWC,IAAK,UAAWC,QAAS,UAAWC,QAAS,UAAWC,SAAU,UAAWC,UAAW,UAAWC,OAAQ,UAAWC,QAAS,UAAWC,MAAO,UAAWC,WAAY,UAAWC,YAAa,UAAWC,OAAQ,UAAWC,UAAW,UAAWC,KAAM,UAAWC,KAAM,UAAWC,UAAW,UAAWC,YAAa,UAAWC,SAAU,UAAWC,OAAQ,UAAWC,UAAW,UAAWC,eAAgB,UAAWC,WAAY,UAAWC,cAAe,UAAWC,SAAU,UAAWC,SAAU,UAAWC,aAAc,UAAWC,YAAa,UAAWC,KAAM,UAAWC,OAAQ,UAAWC,YAAa,UAAWC,MAAO,UAAWC,MAAO,WAK1/FC,+BAIQC,yDAAa,kBAClBA,WAAaA,OACbC,WAAY,OACZC,kBAAoB,CAACh8E,EAAE,MAAOZ,EAAE,YAChC68E,EAAI,IAAM,SACVC,MAAQ,CAACD,EAAE,IAAIE,EAAE,IAAIlpF,EAAE,IAAID,EAAE,QAC7BopF,eAAY9mF,OACZ+mF,aAAe,CAACJ,EAAE,IAAIE,EAAE,IAAIlpF,EAAE,IAAID,EAAE,QACpCspF,mBAAehnF,OACfinF,SAAU,OAGVC,eAAiB,kBACjBC,cAAgB,kBAGhBl+B,sDAQEuD,QACaxsD,IAAhBO,KAAK6jD,cACFA,OAAOkB,eACPlB,YAASpkD,QAEXwsD,UAAYA,OACZA,UAAUt3B,YAAY30B,KAAK2oE,YAC3Bke,mBAEAC,qDAOW9gF,MACQ,mBAAbA,QAIH,IAAI5C,MAAM,oFAHXujF,eAAiB3gF,2CAWTA,MACS,mBAAbA,QAIH,IAAI5C,MAAM,qFAHXwjF,cAAgB5gF,yCAaVqgF,MACQ,iBAAVA,SACFlJ,GAAWkJ,oCAiBbA,OAAOU,gEACA,SAAVV,OAIAW,EAGAC,EAAYjnF,KAAKknF,eAAeb,WAClB5mF,IAAdwnF,IACFZ,EAAQY,IAImB,IAAzB3uC,GAAKnB,SAASkvC,QACe,IAA3B/tC,GAAK6uC,WAAWd,GAAiB,KAC/Be,EAAYf,EAAM9gF,OAAO,GAAGA,OAAO,EAAG8gF,EAAMrpF,OAAS,GAAG6N,MAAM,KAClEm8E,EAAO,CAACZ,EAAEgB,EAAU,GAAId,EAAEc,EAAU,GAAIhqF,EAAEgqF,EAAU,GAAIjqF,EAAE,QAEvD,IAAgC,IAA5Bm7C,GAAK+uC,YAAYhB,GAAiB,KACrCe,EAAYf,EAAM9gF,OAAO,GAAGA,OAAO,EAAG8gF,EAAMrpF,OAAS,GAAG6N,MAAM,KAClEm8E,EAAO,CAACZ,EAAEgB,EAAU,GAAId,EAAEc,EAAU,GAAIhqF,EAAEgqF,EAAU,GAAIjqF,EAAEiqF,EAAU,SAEjE,IAA+B,IAA3B9uC,GAAKgvC,WAAWjB,GAAiB,KACpCkB,EAASjvC,GAAKkvC,SAASnB,GAC3BW,EAAO,CAACZ,EAAEmB,EAAOnB,EAAGE,EAAEiB,EAAOjB,EAAGlpF,EAAEmqF,EAAOnqF,EAAGD,EAAE,YAI5CkpF,aAAiBnqF,aACHuD,IAAZ4mF,EAAMD,QAA+B3mF,IAAZ4mF,EAAMC,QAA+B7mF,IAAZ4mF,EAAMjpF,EAAiB,KACvEqqF,OAAoBhoF,IAAZ4mF,EAAMlpF,EAAkBkpF,EAAMlpF,EAAI,MAC9C6pF,EAAO,CAACZ,EAAEC,EAAMD,EAAGE,EAAED,EAAMC,EAAGlpF,EAAEipF,EAAMjpF,EAAGD,EAAEsqF,WAMpChoF,IAATunF,QACI,IAAI5jF,MAAM,gIAAkI28C,GAAesmC,SAG5JqB,UAAUV,EAAMD,wCAUItnF,IAAvBO,KAAK4mF,qBACFA,qBACAA,mBAAgBnnF,QAGlBinF,SAAU,OACV/d,MAAMn0C,MAAMC,QAAU,aACtBkzD,gEAWDC,8DAEkB,IAAlBA,SACGnB,cAAgBnuC,GAAKh7C,OAAO,GAAI0C,KAAKqmF,SAGvB,IAAjBrmF,KAAK0mF,cACFC,eAAe3mF,KAAKwmF,mBAGtB7d,MAAMn0C,MAAMC,QAAU,WAIhB,gBACkBh1B,IAAvBi+C,EAAKkpC,gBACPlpC,EAAKkpC,gBACLlpC,EAAKkpC,mBAAgBnnF,KAEvB,wCASGknF,eAAe3mF,KAAKqmF,YACpBK,SAAU,OACVmB,8CASAnB,SAAU,OACVC,eAAe3mF,KAAKqmF,YACpByB,cAAc9nF,KAAKqmF,gDASG5mF,IAAvBO,KAAKymF,mBACFsB,SAAS/nF,KAAKymF,eAAe,GAGlCuB,MAAM,uEAWAhB,OAAMD,8DAEK,IAAfA,SACGP,aAAeluC,GAAKh7C,OAAO,GAAI0pF,SAGjCX,MAAQW,MACTiB,EAAM3vC,GAAK4vC,SAASlB,EAAKZ,EAAGY,EAAKV,EAAGU,EAAK5pF,GAEzC+qF,EAAe,EAAI3mF,KAAK4mF,GACxBC,EAASroF,KAAKomF,EAAI6B,EAAIp/E,EACtBsB,EAAInK,KAAKmmF,kBAAkBh8E,EAAIk+E,EAAS7mF,KAAK8mF,IAAIH,EAAeF,EAAIz7E,GACpEjD,EAAIvJ,KAAKmmF,kBAAkB58E,EAAI8+E,EAAS7mF,KAAK+mF,IAAIJ,EAAeF,EAAIz7E,QAEnEg8E,oBAAoBh0D,MAAMkgB,KAAOvqC,EAAI,GAAMnK,KAAKwoF,oBAAoB7nC,YAAc,UAClF6nC,oBAAoBh0D,MAAMsuB,IAAMv5C,EAAI,GAAMvJ,KAAKwoF,oBAAoBz8B,aAAe,UAElF+7B,cAAcd,uCASTllF,QACLukF,MAAMlpF,EAAI2E,EAAQ,SAClBgmF,cAAc9nF,KAAKqmF,8CASXvkF,OACTmmF,EAAM3vC,GAAK4vC,SAASloF,KAAKqmF,MAAMD,EAAGpmF,KAAKqmF,MAAMC,EAAGtmF,KAAKqmF,MAAMjpF,GAC/D6qF,EAAIQ,EAAI3mF,EAAQ,QACZklF,EAAO1uC,GAAKowC,SAAST,EAAIz7E,EAAGy7E,EAAIp/E,EAAGo/E,EAAIQ,GAC3CzB,EAAI,EAAQhnF,KAAKqmF,MAAMlpF,OAClBkpF,MAAQW,OACRc,4DASOd,yDAAOhnF,KAAKqmF,MACpB4B,EAAM3vC,GAAK4vC,SAASlB,EAAKZ,EAAGY,EAAKV,EAAGU,EAAK5pF,GACzCurF,EAAM3oF,KAAK4oF,kBAAkBC,WAAW,WACnBppF,IAArBO,KAAK8oF,mBACF7C,YAAcj6D,OAAO+8D,kBAAoB,IAAMJ,EAAIK,8BACxDL,EAAIM,2BACJN,EAAIO,0BACJP,EAAIQ,yBACJR,EAAIS,wBAA0B,IAEhCT,EAAIU,aAAarpF,KAAKimF,WAAY,EAAG,EAAGjmF,KAAKimF,WAAY,EAAG,OAGxD7wE,EAAIpV,KAAK4oF,kBAAkBjoC,YAC3Bn0C,EAAIxM,KAAK4oF,kBAAkB78B,aAC/B48B,EAAIW,UAAU,EAAG,EAAGl0E,EAAG5I,GAEvBm8E,EAAIY,aAAavpF,KAAKumF,UAAW,EAAE,GACnCoC,EAAIa,UAAY,eAAiB,EAAGvB,EAAIQ,GAAK,IAC7CE,EAAIc,OAAOzpF,KAAKmmF,kBAAkBh8E,EAAGnK,KAAKmmF,kBAAkB58E,EAAGvJ,KAAKomF,MACpEuC,QAAAA,QAEKe,gBAAgB5nF,MAAQ,IAAMmmF,EAAIQ,OAClCkB,aAAa7nF,MAAW,IAAMklF,EAAK7pF,OAEnCysF,gBAAgBp1D,MAAMq1D,gBAAkB,QAAU7pF,KAAKwmF,aAAaJ,EAAI,IAAMpmF,KAAKwmF,aAAaF,EAAI,IAAMtmF,KAAKwmF,aAAappF,EAAI,IAAM4C,KAAKwmF,aAAarpF,EAAI,SAC5J2sF,YAAYt1D,MAAMq1D,gBAAkB,QAAU7pF,KAAKqmF,MAAMD,EAAI,IAAMpmF,KAAKqmF,MAAMC,EAAI,IAAMtmF,KAAKqmF,MAAMjpF,EAAI,IAAM4C,KAAKqmF,MAAMlpF,EAAI,4CAS5HyrF,kBAAkBp0D,MAAMoU,MAAQ,YAChCggD,kBAAkBp0D,MAAMqU,OAAS,YAEjC+/C,kBAAkBhgD,MAAQ,IAAM5oC,KAAKimF,gBACrC2C,kBAAkB//C,OAAS,IAAM7oC,KAAKimF,iEAUtCtd,MAAQv7C,SAASE,cAAc,YAC/Bq7C,MAAMrgC,UAAY,wBAElByhD,eAAiB38D,SAASE,cAAc,YACxCk7D,oBAAsBp7D,SAASE,cAAc,YAC7Ck7D,oBAAoBlgD,UAAY,oBAChCyhD,eAAep1D,YAAY30B,KAAKwoF,0BAEhCI,kBAAoBx7D,SAASE,cAAc,eAC3Cy8D,eAAep1D,YAAY30B,KAAK4oF,mBAEhC5oF,KAAK4oF,kBAAkBC,WAQvB,KACCF,EAAM3oF,KAAK4oF,kBAAkBC,WAAW,WACvC5C,YAAcj6D,OAAO+8D,kBAAoB,IAAMJ,EAAIK,8BACxDL,EAAIM,2BACJN,EAAIO,0BACJP,EAAIQ,yBACJR,EAAIS,wBAA0B,QACzBR,kBAAkBC,WAAW,MAAMQ,aAAarpF,KAAKimF,WAAY,EAAG,EAAGjmF,KAAKimF,WAAY,EAAG,OAf1D,KAClC+D,EAAW58D,SAASE,cAAe,OACvC08D,EAASx1D,MAAM6xD,MAAQ,MACvB2D,EAASx1D,MAAMy1D,WAAc,OAC7BD,EAASx1D,MAAMw1C,QAAW,OAC1BggB,EAASt+B,UAAa,wDACjBk9B,kBAAkBj0D,YAAYq1D,QAYhCD,eAAezhD,UAAY,iBAE3B4hD,WAAa98D,SAASE,cAAc,YACpC48D,WAAW5hD,UAAY,mBAEvB6hD,cAAgB/8D,SAASE,cAAc,YACvC68D,cAAc7hD,UAAY,sBAE1B8hD,SAAWh9D,SAASE,cAAc,YAClC88D,SAAS9hD,UAAY,iBAErBqhD,aAAev8D,SAASE,cAAc,kBAEpCq8D,aAAazyD,KAAO,aACpByyD,aAAavnF,IAAM,SACnBunF,aAAarkF,IAAM,YAGnB2gC,SACF0jD,aAAa7nF,MAAQ,WACrB6nF,aAAarhD,UAAY,iBAEzBohD,gBAAkBt8D,SAASE,cAAc,kBAEvCo8D,gBAAgBxyD,KAAO,aACvBwyD,gBAAgBtnF,IAAM,SACtBsnF,gBAAgBpkF,IAAM,YAGtB2gC,SACFyjD,gBAAgB5nF,MAAQ,WACxB4nF,gBAAgBphD,UAAY,iBAE5B4hD,WAAWv1D,YAAY30B,KAAK2pF,mBAC5BQ,cAAcx1D,YAAY30B,KAAK0pF,qBAEhC/qC,EAAK3+C,UACJ2pF,aAAaU,SAAW,WAAa1rC,EAAG2rC,YAAYtqF,KAAK8B,aACzD6nF,aAAaY,QAAW,WAAa5rC,EAAG2rC,YAAYtqF,KAAK8B,aACzD4nF,gBAAgBW,SAAW,WAAa1rC,EAAG6rC,eAAexqF,KAAK8B,aAC/D4nF,gBAAgBa,QAAW,WAAa5rC,EAAG6rC,eAAexqF,KAAK8B,aAE/D2oF,gBAAkBr9D,SAASE,cAAc,YACzCm9D,gBAAgBniD,UAAY,gCAC5BmiD,gBAAgB/+B,UAAY,mBAE5Bg/B,aAAet9D,SAASE,cAAc,YACtCo9D,aAAapiD,UAAY,6BACzBoiD,aAAah/B,UAAY,gBAEzBo+B,YAAc18D,SAASE,cAAc,YACrCw8D,YAAYxhD,UAAY,qBACxBwhD,YAAYp+B,UAAY,WAExBk+B,gBAAkBx8D,SAASE,cAAc,YACzCs8D,gBAAgBthD,UAAY,yBAC5BshD,gBAAgBl+B,UAAY,eAE5Bi/B,aAAev9D,SAASE,cAAc,YACtCq9D,aAAariD,UAAY,6BACzBqiD,aAAaj/B,UAAY,cACzBi/B,aAAaC,QAAU70C,UAAK8xC,cAAW7nF,MAAM,QAE7C6qF,YAAcz9D,SAASE,cAAc,YACrCu9D,YAAYviD,UAAY,4BACxBuiD,YAAYn/B,UAAY,aACxBm/B,YAAYD,QAAU70C,UAAK+0C,eAAY9qF,WAEvC+qF,WAAa39D,SAASE,cAAc,YACpCy9D,WAAWziD,UAAY,2BACvByiD,WAAWr/B,UAAY,YACvBq/B,WAAWH,QAAU70C,UAAKi1C,cAAWhrF,WAErCirF,WAAa79D,SAASE,cAAc,YACpC29D,WAAW3iD,UAAY,2BACvB2iD,WAAWv/B,UAAY,iBACvBu/B,WAAWL,QAAU70C,UAAKm1C,kBAAelrF,WAEzC2oE,MAAMh0C,YAAY30B,KAAK+pF,qBACvBphB,MAAMh0C,YAAY30B,KAAKoqF,eACvBzhB,MAAMh0C,YAAY30B,KAAKyqF,sBACvB9hB,MAAMh0C,YAAY30B,KAAKmqF,oBACvBxhB,MAAMh0C,YAAY30B,KAAK0qF,mBACvB/hB,MAAMh0C,YAAY30B,KAAKkqF,iBACvBvhB,MAAMh0C,YAAY30B,KAAK8pF,kBACvBnhB,MAAMh0C,YAAY30B,KAAK4pF,sBAEvBjhB,MAAMh0C,YAAY30B,KAAK2qF,mBACvBhiB,MAAMh0C,YAAY30B,KAAK6qF,kBACvBliB,MAAMh0C,YAAY30B,KAAK+qF,iBACvBpiB,MAAMh0C,YAAY30B,KAAKirF,kEASvBn7B,KAAO,QACPq7B,MAAQ,QACRtnC,OAAS,IAAIG,GAAOhkD,KAAK4oF,wBACzB/kC,OAAO95C,IAAI,SAAS/F,IAAI,CAACkiD,QAAQ,IAEtCsM,GAAmBxyD,KAAK6jD,QAAQ,SAACrD,GAAWmV,EAAKy1B,cAAc5qC,WAC1DqD,OAAOvN,GAAG,OAAa,SAACkK,GAAWmV,EAAKy1B,cAAc5qC,WACtDqD,OAAOvN,GAAG,YAAa,SAACkK,GAAWmV,EAAKy1B,cAAc5qC,WACtDqD,OAAOvN,GAAG,WAAa,SAACkK,GAAWmV,EAAKy1B,cAAc5qC,WACtDqD,OAAOvN,GAAG,UAAa,SAACkK,GAAWmV,EAAKy1B,cAAc5qC,uDASpC,IAAnBxgD,KAAKkmF,UAAqB,KACxByC,EAAM3oF,KAAK4oF,kBAAkBC,WAAW,WACnBppF,IAArBO,KAAK8oF,mBACF7C,YAAcj6D,OAAO+8D,kBAAoB,IAAMJ,EAAIK,8BACxDL,EAAIM,2BACJN,EAAIO,0BACJP,EAAIQ,yBACJR,EAAIS,wBAA0B,IAEhCT,EAAIU,aAAarpF,KAAKimF,WAAY,EAAG,EAAGjmF,KAAKimF,WAAY,EAAG,OASxD97E,EAAGZ,EAAG8hF,EAAKC,EANXl2E,EAAIpV,KAAK4oF,kBAAkBjoC,YAC3Bn0C,EAAIxM,KAAK4oF,kBAAkB78B,aAC/B48B,EAAIW,UAAU,EAAG,EAAGl0E,EAAG5I,QAKlB25E,kBAAoB,CAACh8E,EAAO,GAAJiL,EAAS7L,EAAO,GAAJiD,QACpC45E,EAAI,IAAOhxE,MAIZm2E,EAHApD,EAAgB,EAAI3mF,KAAK4mF,GAAM,IAE/BoD,EAAO,EAAIxrF,KAAKomF,MAEfiF,EAAM,EAAGA,EAAM,IAAKA,QAClBC,EAAM,EAAGA,EAAMtrF,KAAKomF,EAAGkF,IAC1BnhF,EAAInK,KAAKmmF,kBAAkBh8E,EAAImhF,EAAM9pF,KAAK8mF,IAAIH,EAAekD,GAC7D9hF,EAAIvJ,KAAKmmF,kBAAkB58E,EAAI+hF,EAAM9pF,KAAK+mF,IAAIJ,EAAekD,GAC7DE,EAAMjzC,GAAKowC,SAAS2C,GAPb,EAAI,KAOqBC,EAAME,EAAM,GAC5C7C,EAAIa,UAAY,OAAS+B,EAAInF,EAAI,IAAMmF,EAAIjF,EAAI,IAAMiF,EAAInuF,EAAI,IAC7DurF,EAAI8C,SAASthF,EAAI,GAAKZ,EAAI,GAAK,EAAG,GAGtCo/E,EAAI+C,YAAc,gBAClB/C,EAAIc,OAAOzpF,KAAKmmF,kBAAkBh8E,EAAGnK,KAAKmmF,kBAAkB58E,EAAGvJ,KAAKomF,GACpEuC,EAAIgD,cAECpF,UAAYoC,EAAIiD,aAAa,EAAE,EAAEx2E,EAAE5I,QAErC05E,WAAY,wCAUL1lC,OACR1X,EAAO9oC,KAAK+pF,eAAennC,wBAC3BlO,EAAO8L,EAAMxB,OAAO70C,EAAI2+B,EAAK4L,KAC7BoO,EAAMtC,EAAMxB,OAAOz1C,EAAIu/B,EAAKga,IAE5B+oC,EAAU,GAAM7rF,KAAK+pF,eAAeh+B,aACpC+/B,EAAU,GAAM9rF,KAAK+pF,eAAeppC,YAEpCx2C,EAAIuqC,EAAOo3C,EACXviF,EAAIu5C,EAAM+oC,EAEVE,EAAQvqF,KAAKwqF,MAAM7hF,EAAEZ,GACrB8+E,EAAS,IAAO7mF,KAAKY,IAAIZ,KAAKyqF,KAAK9hF,EAAIA,EAAIZ,EAAIA,GAAIuiF,GAEnDzwB,EAAS75D,KAAK+mF,IAAIwD,GAAS1D,EAASwD,EACpCK,EAAU1qF,KAAK8mF,IAAIyD,GAAS1D,EAASyD,OAEpCtD,oBAAoBh0D,MAAMsuB,IAAMuY,EAAS,GAAMr7D,KAAKwoF,oBAAoBz8B,aAAe,UACvFy8B,oBAAoBh0D,MAAMkgB,KAAOw3C,EAAU,GAAMlsF,KAAKwoF,oBAAoB7nC,YAAc,SAGzFn0C,EAAIu/E,GAAS,EAAIvqF,KAAK4mF,IAC1B57E,EAAIA,EAAI,EAAIA,EAAI,EAAIA,MAChB3D,EAAIw/E,EAASroF,KAAKomF,EAClB6B,EAAM3vC,GAAK4vC,SAASloF,KAAKqmF,MAAMD,EAAGpmF,KAAKqmF,MAAMC,EAAGtmF,KAAKqmF,MAAMjpF,GAC/D6qF,EAAIz7E,EAAIA,EACRy7E,EAAIp/E,EAAIA,MACJm+E,EAAO1uC,GAAKowC,SAAST,EAAIz7E,EAAGy7E,EAAIp/E,EAAGo/E,EAAIQ,GAC3CzB,EAAI,EAAQhnF,KAAKqmF,MAAMlpF,OAClBkpF,MAAQW,OAGR4C,gBAAgBp1D,MAAMq1D,gBAAkB,QAAU7pF,KAAKwmF,aAAaJ,EAAI,IAAMpmF,KAAKwmF,aAAaF,EAAI,IAAMtmF,KAAKwmF,aAAappF,EAAI,IAAM4C,KAAKwmF,aAAarpF,EAAI,SAC5J2sF,YAAYt1D,MAAMq1D,gBAAkB,QAAU7pF,KAAKqmF,MAAMD,EAAI,IAAMpmF,KAAKqmF,MAAMC,EAAI,IAAMtmF,KAAKqmF,MAAMjpF,EAAI,IAAM4C,KAAKqmF,MAAMlpF,EAAI,ylqBCxiB/HgvF,yBAOQC,EAAcC,EAAkBtP,OAAkBkJ,yDAAa,kBACpEr1D,OAASw7D,OACTE,eAAiB,QACjBrgC,UAAYogC,OACZE,eAAgB,OAEhBp9D,QAAU,QACVq9D,aAAc,OACdC,aAAe,OACfvvC,eAAiB,CACpBw/B,SAAS,EACTzkD,QAAQ,EACRg0B,eAAWxsD,EACXitF,YAAY,GAEdp0C,GAAKh7C,OAAO0C,KAAKmvB,QAASnvB,KAAKk9C,qBAE1B6/B,iBAAmBA,OACnB4P,cAAgB,QAChBC,YAAc,QACdC,SAAW,QACXC,WAAa,OACbC,aAAe,QACfC,YAAc,IAAIhH,GAAYC,QAC9Bl6C,aAAUtsC,gDAUN0vB,WACO1vB,IAAZ0vB,EAAuB,MAEpB49D,aAAe,QACfE,mBAEDvQ,GAAU,KACS,iBAAZvtD,OACJA,QAAQ8I,OAAS9I,OAEnB,GAAIA,aAAmBlzB,WACrBkzB,QAAQ8I,OAAS9I,EAAQhsB,YAE3B,GAAuB,WAAnBsiC,GAAOtW,GAAsB,IACrB,MAAXA,QACI,IAAInC,UAAU,+BAEIvtB,IAAtB0vB,EAAQ88B,iBACL98B,QAAQ88B,UAAY98B,EAAQ88B,gBAEZxsD,IAAnBq3C,GAAA3nB,UACGA,QAAQ8I,UAAS9I,SAEG1vB,IAAvB0vB,EAAQu9D,kBACLv9D,QAAQu9D,WAAav9D,EAAQu9D,iBAEZjtF,IAApB0vB,EAAQutD,UACVA,EAAUvtD,EAAQutD,aAGM,kBAAZvtD,QACTA,QAAQ8I,QAAS,EACtBykD,EAAUvtD,GAEgB,mBAAZA,SACTA,QAAQ8I,OAAS9I,EACtButD,GAAU,IAEgB,IAAxB5lC,QAAK3nB,WACPutD,GAAU,QAGPvtD,QAAQutD,QAAUA,OAEpBwQ,kDAOUP,QACVA,cAAgBA,GACQ,IAAzB3sF,KAAKmvB,QAAQutD,eACVwQ,cAC0BztF,IAA3BO,KAAKmvB,QAAQ88B,iBACVA,UAAYjsD,KAAKmvB,QAAQ88B,gBAE3BvD,kDASFwkC,cACAZ,eAAiB,OAElBr0D,KAASj4B,KAAKmvB,SACdg+D,EAAU,EACV3pB,GAAO,MACN,IAAIlN,KAAUt2D,KAAK+8E,iBAClB/8E,KAAK+8E,iBAAiB1/E,eAAei5D,UAClCi2B,eAAgB,EACrB/oB,GAAO,EACe,mBAAXvrC,EAETurC,GADAA,EAAOvrC,EAAOq+B,EAAO,MACNt2D,KAAKotF,cAAcptF,KAAK+8E,iBAAiBzmB,GAAS,CAACA,IAAS,IAEzD,IAAXr+B,IAA+C,IAA5B27B,GAAA37B,QAAAA,EAAeq+B,KACzCkN,GAAO,IAGI,IAATA,SACG+oB,eAAgB,EAGjBY,EAAU,QACPE,UAAU,SAGZC,YAAYh3B,QAGZ82B,cAAcptF,KAAK+8E,iBAAiBzmB,GAAS,CAACA,KAErD62B,UAGCI,mBACAC,6CAUAzhD,QAAU3e,SAASE,cAAc,YACjCye,QAAQzD,UAAY,iCACpB2jB,UAAUt3B,YAAY30B,KAAK+rC,aAC3B,IAAIjvC,EAAI,EAAGA,EAAIkD,KAAK4sF,YAAY5vF,OAAQF,SACtCivC,QAAQpX,YAAY30B,KAAK4sF,YAAY9vF,SAGvC2wF,0DASA,IAAI3wF,EAAI,EAAGA,EAAIkD,KAAK4sF,YAAY5vF,OAAQF,SACtCivC,QAAQvE,YAAYxnC,KAAK4sF,YAAY9vF,SAGvB2C,IAAjBO,KAAK+rC,eACFkgB,UAAUzkB,YAAYxnC,KAAK+rC,cAC3BA,aAAUtsC,QAEZmtF,YAAc,QAEdK,iDAUGh9D,WACJlS,EAAO/d,KAAK2sF,cACP7vF,EAAI,EAAGA,EAAImzB,EAAKjzB,OAAQF,IAAK,SACd2C,IAAlBse,EAAKkS,EAAKnzB,IAGT,CACHihB,OAAOte,QAHPse,EAAOA,EAAKkS,EAAKnzB,WAOdihB,oCAWCkS,OACmB,IAAvBjwB,KAAKusF,cAAwB,KAC3Bx0C,EAAO3qB,SAASE,cAAc,OAClCyqB,EAAKzP,UAAY,iDAAmDrY,EAAKjzB,kCAH1D4vF,mCAAAA,8BAIfA,QAAAA,GAAoB,SAAChlD,GACnBmQ,EAAKpjB,YAAYiT,WAEdglD,YAAY3vF,KAAK86C,GACf/3C,KAAK4sF,YAAY5vF,cAEnB,sCASGyG,OACNiqF,EAAMtgE,SAASE,cAAc,OACjCogE,EAAIplD,UAAY,sCAChBolD,EAAIhiC,UAAYjoD,OACX4pF,UAAU,GAAGK,sCAYTjqF,EAAMwsB,OAAM09D,0DACjBD,EAAMtgE,SAASE,cAAc,cACjCogE,EAAIplD,UAAY,kDAAoDrY,EAAKjzB,OAEvE0wF,EAAIhiC,WADc,IAAhBiiC,EACc,SAAWlqF,EAAO,YAGlBA,EAAO,IAElBiqF,wCAWK9wF,EAAKkF,EAAOmuB,OACpB2iD,EAASxlD,SAASE,cAAc,UACpCslD,EAAOtqC,UAAY,0CACfslD,EAAgB,OACNnuF,IAAVqC,IAC0B,IAAxB8xD,GAAAh3D,QAAAA,EAAYkF,KACd8rF,EAAgBh6B,GAAAh3D,QAAAA,EAAYkF,QAI3B,IAAIhF,EAAI,EAAGA,EAAIF,EAAII,OAAQF,IAAK,KAC/Bw5D,EAASlpC,SAASE,cAAc,UACpCgpC,EAAOx0D,MAAQlF,EAAIE,GACfA,IAAM8wF,IACRt3B,EAAO0N,SAAW,YAEpB1N,EAAO5K,UAAY9uD,EAAIE,GACvB81E,EAAOj+C,YAAY2hC,OAGjB3X,EAAK3+C,KACT4yE,EAAOyX,SAAW,WAAa1rC,EAAGvI,QAAQp2C,KAAK8B,MAAOmuB,QAElDsY,EAAQvoC,KAAK6tF,WAAW59D,EAAKA,EAAKjzB,OAAO,GAAIizB,QAC5Co9D,UAAUp9D,EAAMsY,EAAOqqC,sCAWnBh2E,EAAKkF,EAAOmuB,OACjB69D,EAAelxF,EAAI,GACnBwF,EAAMxF,EAAI,GACV0I,EAAM1I,EAAI,GACVmnC,EAAOnnC,EAAI,GACX88C,EAAQtsB,SAASE,cAAc,SACnCosB,EAAMpR,UAAY,yCAEhBoR,EAAMxiB,KAAO,QACbwiB,EAAMt3C,IAAMA,EACZs3C,EAAMp0C,IAAMA,QAGP2gC,IACPyT,EAAM3V,KAAOA,MAGTgqD,EAAc,GACdC,EAAa,UAEHvuF,IAAVqC,EAAqB,CAEnBA,EAAQ,GADC,IACIA,EAAiBM,GAChCs3C,EAAMt3C,IAAMZ,KAAKC,KAFN,IAEWK,GACtBksF,EAAat0C,EAAMt3C,IACnB2rF,EAAc,mBAEPjsF,EANI,IAMaM,IACxBs3C,EAAMt3C,IAAMZ,KAAKC,KAAKK,EAPX,KAQXksF,EAAat0C,EAAMt3C,IACnB2rF,EAAc,mBATH,IAWTjsF,EAAiBwD,GAAe,IAARA,IAC1Bo0C,EAAMp0C,IAAM9D,KAAKC,KAZN,IAYWK,GACtBksF,EAAat0C,EAAMp0C,IACnByoF,EAAc,mBAEhBr0C,EAAM53C,MAAQA,OAGd43C,EAAM53C,MAAQgsF,MAGZ9xF,EAAQoxB,SAASE,cAAc,SACnCtxB,EAAMssC,UAAY,0CAClBtsC,EAAM8F,MAAQu1C,OAAOqC,EAAM53C,WAEvB68C,EAAK3+C,KACT05C,EAAM2wC,SAAW,WAAaruF,EAAM8F,MAAQ9B,KAAK8B,MAAO68C,EAAGvI,QAAQiB,OAAOr3C,KAAK8B,OAAQmuB,IACvFypB,EAAM6wC,QAAW,WAAavuF,EAAM8F,MAAQ9B,KAAK8B,WAE7CymC,EAAQvoC,KAAK6tF,WAAW59D,EAAKA,EAAKjzB,OAAO,GAAIizB,GAC7CuyC,EAAYxiE,KAAKqtF,UAAUp9D,EAAMsY,EAAOmR,EAAO19C,GAG/B,KAAhB+xF,GAAsB/tF,KAAK+sF,aAAavqB,KAAewrB,SACpDjB,aAAavqB,GAAawrB,OAC1BC,YAAYF,EAAavrB,yDASA,IAA5BxiE,KAAKmvB,QAAQu9D,WAAqB,KAChCwB,EAAiB9gE,SAASE,cAAc,OAC5C4gE,EAAe5lD,UAAY,sCAC3B4lD,EAAexiC,UAAY,mBAC3BwiC,EAAetD,QAAc,WAAOltC,EAAKywC,iBACzCD,EAAeE,YAAc,WAAOF,EAAe5lD,UAAY,6CAC/D4lD,EAAeG,WAAc,WAAOH,EAAe5lD,UAAY,4CAE1DgmD,iBAAmBlhE,SAASE,cAAc,YAC1CghE,iBAAiBhmD,UAAY,qDAE7BskD,YAAY3vF,KAAK+C,KAAKsuF,uBACtB1B,YAAY3vF,KAAKixF,wCAWdj3E,EAAQ2N,kBACO,IAArB5kB,KAAKwsF,cAA+C,IAAvBxsF,KAAKusF,eAA0BvsF,KAAKysF,aAAezsF,KAAK8sF,WAAY,KAC/FY,EAAMtgE,SAASE,cAAc,OACjCogE,EAAIt6D,GAAK,0BACTs6D,EAAIplD,UAAY,0BAChBolD,EAAIhiC,UAAYz0C,EAChBy2E,EAAI9C,QAAU,WAAOj1B,EAAKs3B,qBACrBR,cAAgB,OAChBI,SAAW,CAACn4D,KAAKg5D,EAAK9oE,MAAMA,gDAURnlB,IAAvBO,KAAK6sF,SAASn4D,YACXm4D,SAASn4D,KAAK6S,WAAWC,YAAYxnC,KAAK6sF,SAASn4D,MACxD0qB,aAAap/C,KAAK6sF,SAAS0B,aAC3BnvC,aAAap/C,KAAK6sF,SAAS2B,oBACtB3B,SAAW,oEAUSptF,IAAvBO,KAAK6sF,SAASn4D,KAAoB,KAEhCoU,EADuB9oC,KAAK4sF,YAAY5sF,KAAK6sF,SAASjoE,OAC1Bg+B,6BAC3BiqC,SAASn4D,KAAKF,MAAMkgB,KAAO5L,EAAK4L,KAAO,UACvCm4C,SAASn4D,KAAKF,MAAMsuB,IAAMha,EAAKga,IAAM,GAAK,KAC/C11B,SAASwrB,KAAKjkB,YAAY30B,KAAK6sF,SAASn4D,WACnCm4D,SAAS0B,YAAcpvC,IAAW,WACrC4hB,EAAK8rB,SAASn4D,KAAKF,MAAMi6D,QAAU,IACnC,WACG5B,SAAS2B,cAAgBrvC,IAAW,WACvC4hB,EAAKksB,iBACL,6CAWQa,EAAchsF,EAAOmuB,OAC7By+D,EAAWthE,SAASE,cAAc,SACtCohE,EAASx3D,KAAO,WAChBw3D,EAASpmD,UAAY,wCACrBomD,EAASC,QAAUb,OACLruF,IAAVqC,IACF4sF,EAASC,QAAU7sF,EACfA,IAAUgsF,IACgB,WAAxBroD,GAAOqoD,GACLhsF,IAAUgsF,EAAapR,cACpB4P,eAAervF,KAAK,CAACgzB,KAAKA,EAAMnuB,MAAMA,SAIxCwqF,eAAervF,KAAK,CAACgzB,KAAKA,EAAMnuB,MAAMA,UAK7C68C,EAAK3+C,KACT0uF,EAASrE,SAAW,WAAY1rC,EAAGvI,QAAQp2C,KAAK2uF,QAAS1+D,QAErDsY,EAAQvoC,KAAK6tF,WAAW59D,EAAKA,EAAKjzB,OAAO,GAAIizB,QAC5Co9D,UAAUp9D,EAAMsY,EAAOmmD,0CAUfZ,EAAchsF,EAAOmuB,OAC9By+D,EAAWthE,SAASE,cAAc,SACtCohE,EAASx3D,KAAO,OAChBw3D,EAASpmD,UAAY,oCACrBomD,EAAS5sF,MAAQA,EACbA,IAAUgsF,QACPxB,eAAervF,KAAK,CAACgzB,KAAKA,EAAMnuB,MAAMA,QAGzC68C,EAAK3+C,KACT0uF,EAASrE,SAAW,WAAY1rC,EAAGvI,QAAQp2C,KAAK8B,MAAOmuB,QAEnDsY,EAAQvoC,KAAK6tF,WAAW59D,EAAKA,EAAKjzB,OAAO,GAAIizB,QAC5Co9D,UAAUp9D,EAAMsY,EAAOmmD,2CAWd9xF,EAAKkF,EAAOmuB,cACtB2+D,EAAehyF,EAAI,GACnB8wF,EAAMtgE,SAASE,cAAc,OAGnB,UAFdxrB,OAAkBrC,IAAVqC,EAAsB8sF,EAAe9sF,IAG3C4rF,EAAIplD,UAAY,0CAChBolD,EAAIl5D,MAAMq1D,gBAAkB/nF,GAG5B4rF,EAAIplD,UAAY,+CAGlBxmC,OAAkBrC,IAAVqC,EAAsB8sF,EAAe9sF,EAC7C4rF,EAAI9C,QAAU,WACZlpB,EAAKmtB,iBAAiB/sF,EAAM4rF,EAAIz9D,QAG9BsY,EAAQvoC,KAAK6tF,WAAW59D,EAAKA,EAAKjzB,OAAO,GAAIizB,QAC5Co9D,UAAUp9D,EAAKsY,EAAOmlD,4CAWZ5rF,EAAO4rF,EAAKz9D,cAE3By9D,EAAI9C,QAAU,kBAEToC,YAAY8B,SAASpB,QACrBV,YAAYxpB,YAEZwpB,YAAYjF,SAASjmF,QACrBkrF,YAAY+B,mBAAkB,SAAC1I,OAC9B2I,EAAc,QAAU3I,EAAMD,EAAI,IAAMC,EAAMC,EAAI,IAAMD,EAAMjpF,EAAI,IAAMipF,EAAMlpF,EAAI,IACtFuwF,EAAIl5D,MAAMq1D,gBAAkBmF,EAC5Bva,EAAKr+B,QAAQ44C,EAAY/+D,WAItB+8D,YAAYiC,kBAAiB,WAChCvB,EAAI9C,QAAU,WACZnW,EAAKoa,iBAAiB/sF,EAAM4rF,EAAIz9D,6CAcxB5uB,OAAK4uB,yDAAO,GAAIi/D,0DACxB1rB,GAAO,EACPvrC,KAASj4B,KAAKmvB,SACdggE,GAAe,MACd,IAAIC,KAAU/tF,KACbA,EAAIhE,eAAe+xF,GAAS,CAC9B5rB,GAAO,MACHzrB,EAAO12C,EAAI+tF,GACXC,EAAU/2C,GAAKqjC,mBAAmB1rD,EAAMm/D,MACtB,mBAAXn3D,IAII,KAHburC,EAAOvrC,EAAOm3D,EAAOn/D,OAIb8nB,aAAgB97C,QAA0B,iBAAT87C,GAAqC,kBAATA,GAAsBA,aAAgB77C,cAClGqwF,eAAgB,EACrB/oB,EAAOxjE,KAAKotF,cAAcr1C,EAAMs3C,GAAS,QACpC9C,eAA8B,IAAd2C,IAKd,IAAT1rB,EAAgB,CAClB2rB,GAAe,MACXrtF,EAAQ9B,KAAKsvF,UAAUD,MAEvBt3C,aAAgB97C,WACbszF,aAAax3C,EAAMj2C,EAAOutF,QAE5B,GAAoB,iBAATt3C,OACTy3C,eAAez3C,EAAMj2C,EAAOutF,QAE9B,GAAoB,kBAATt3C,OACT03C,cAAc13C,EAAMj2C,EAAOutF,QAE7B,GAAIt3C,aAAgB77C,OAAQ,KAE3BwzF,GAAO,MACsB,IAA7B97B,GAAA3jC,QAAAA,EAAa,YACXjwB,KAAK2sF,cAAcgD,QAAQC,SAAWR,IACxCM,GAAO,IAIE,IAATA,UAEmBjwF,IAAjBs4C,EAAK2kC,QAAuB,KAC1BmT,EAAcv3C,GAAKqjC,mBAAmB0T,EAAS,WAC/CS,EAAe9vF,KAAKsvF,UAAUO,OACb,IAAjBC,EAAuB,KACrBvnD,EAAQvoC,KAAK6tF,WAAWuB,EAAQC,GAAS,QACxChC,UAAUgC,EAAS9mD,GACxB4mD,EAAenvF,KAAKotF,cAAcr1C,EAAMs3C,IAAYF,YAG/CM,cAAc13C,EAAM+3C,EAAcT,OAGtC,KACC9mD,EAAQvoC,KAAK6tF,WAAWuB,EAAQC,GAAS,QACxChC,UAAUgC,EAAS9mD,GACxB4mD,EAAenvF,KAAKotF,cAAcr1C,EAAMs3C,IAAYF,QAKxDxsF,QAAQwpB,MAAM,0BAA2B4rB,EAAMq3C,EAAQC,WAKxDF,uCAWIvyF,EAAKkF,EAAOmuB,GACD,iBAAXrzB,EAAI,IAA8B,UAAXA,EAAI,SAC/BmzF,gBAAgBnzF,EAAKkF,EAAOmuB,GAC7BrzB,EAAI,KAAOkF,QAAawqF,eAAervF,KAAK,CAACgzB,KAAKA,EAAMnuB,MAAMA,KAEzC,iBAAXlF,EAAI,SACbozF,cAAcpzF,EAAKkF,EAAOmuB,GAC3BrzB,EAAI,KAAOkF,QAAawqF,eAAervF,KAAK,CAACgzB,KAAKA,EAAMnuB,MAAMA,KAEzC,iBAAXlF,EAAI,UACbqzF,WAAWrzF,EAAKkF,EAAOmuB,GACxBrzB,EAAI,KAAOkF,QAAawqF,eAAervF,KAAK,CAACgzB,KAAKA,EAAMnuB,MAAMu1C,OAAOv1C,sCAYrEA,EAAOmuB,OACTd,EAAUnvB,KAAKkwF,kBAAkBpuF,EAAMmuB,GAEvCjwB,KAAK4wB,OAAOgoB,MAAQ54C,KAAK4wB,OAAOgoB,KAAKiF,SAAW79C,KAAK4wB,OAAOgoB,KAAKiF,QAAQ4C,WACtE7vB,OAAOgoB,KAAKiF,QAAQ4C,KAAK,eAAgBtxB,QAE3Cq9D,aAAc,OACd57D,OAAO4tB,WAAWrvB,6CAYPrtB,EAAOmuB,OAAMkgE,yDAAa,GACtCluC,EAAUkuC,EAIdruF,EAAkB,WADlBA,EAAkB,SAAVA,GAA4BA,IACAA,MAE/B,IAAIhF,EAAI,EAAGA,EAAImzB,EAAKjzB,OAAQF,IACf,WAAZmzB,EAAKnzB,UACkB2C,IAArBwiD,EAAQhyB,EAAKnzB,MACfmlD,EAAQhyB,EAAKnzB,IAAM,IAEjBA,IAAMmzB,EAAKjzB,OAAS,EACtBilD,EAAUA,EAAQhyB,EAAKnzB,IAGvBmlD,EAAQhyB,EAAKnzB,IAAMgF,UAIlBquF,8CAOHhhE,EAAUnvB,KAAKowF,kBACd9B,iBAAiB5iC,UAAY,sBAAwB3L,GAAe5wB,EAAS,KAAM,GAAK,sDAQzFA,EAAU,GACLryB,EAAI,EAAGA,EAAIkD,KAAKssF,eAAetvF,OAAQF,SACzCozF,kBAAkBlwF,KAAKssF,eAAexvF,GAAGgF,MAAO9B,KAAKssF,eAAexvF,GAAGmzB,KAAMd,UAE7EA,WC9sBUkhE,0BAQPpkC,EAAWzV,EAAOm2B,EAAQx9C,mEAE/B8wB,SAAW,IAAIvjD,OACf4zF,WAAY,IAEXC,gBAAgBF,SACd,IAAIG,YAAY,yDAIlBttD,GAAcypC,IAAWA,aAAkB/0B,IAAW+0B,aAAkB2H,KAAa3H,aAAkBzwE,OAAQ,KAC7Gu0F,EAAgBthE,EACtBA,EAAUw9C,EACVA,EAAS8jB,EAKPthE,GAAWA,EAAQwtD,gBACrBh6E,QAAQH,KAAK,6HAGTm8C,aACDzB,eAAiB,CACpB2a,YAAY,EACZL,oBAAqB,IACrBnP,YAAa,CACXC,KAAM,SACNvQ,KAAM,UAER3tB,OAAAA,MAEG+E,QAAUmpB,GAAKuN,WAAW,GAAInI,EAAKR,kBAGnCwL,QAAQuD,IACR98B,GAAYA,QAAiC,IAAfA,EAAQguB,IAAqB,KAE1DuzC,IADCtyC,IAAI+C,KAAK3sB,MAAMyqB,WAAa,iBAE7B0xC,EAAUjzC,EAAKU,IAAI+C,MACfuvC,GAAoBC,GAC1BD,EAAmB1kE,OAAO+8C,iBAAiB4nB,EAAS,MAAMryE,UAC1DqyE,EAAUA,EAAQC,gBAEfzhE,QAAQguB,IAAOuzC,GAAuD,OAAlCA,EAAiBpsF,qBAErD6qB,QAAQguB,IAAMhuB,EAAQguB,IAGzBhuB,IACEA,EAAQquB,gBAAoBruB,QAAQquB,YAAcruB,EAAQquB,aAC1DruB,EAAQ0tD,0BAA8B1tD,QAAQ0tD,sBAAwB1tD,EAAQ0tD,uBAC9E1tD,EAAQ6vC,cAAkB7vC,QAAQ6vC,UAAY7vC,EAAQ6vC,WACtD7vC,EAAQ2tD,0BAA8B3tD,QAAQ2tD,sBAAwB3tD,EAAQ2tD,4BAI9E+T,EAAwBzjE,SAASE,cAAc,UACjDowB,EAAKvuB,QAAQ2tD,sBAAuB,OAChC9e,EAAmBjoB,OAAK5mB,QAAQ2tD,qCAChCprB,EAAgBsM,EAAiBtgB,EAAKU,IAAIsT,eAC3CA,aAAyBx1D,UAAaw1D,aAAyBuM,SAClED,EAAiB6yB,GAEbn/B,aAAyBuM,SAC3B4yB,EAAsBnlC,UAAY,GAClCmlC,EAAsBl8D,YAAY+8B,IAEVjyD,MAAjBiyD,IACPm/B,EAAsBnlC,UAAYgG,YAInCtT,IAAIsT,cAAc/8B,YAAYk8D,KAG9Bt8B,WAAa,KAEb3b,KAAO,CACVwF,IAAKV,EAAKU,IACV5E,SAAUkE,EAAKrI,MACfwI,QAAS,CACPvH,GAAIP,OAAKO,kBACTC,IAAKR,OAAKQ,mBACVkK,KAAM1K,OAAK0K,qBAEbrH,YAAa,GACbd,KAAM,CACJsY,2BACSjS,EAAG8J,SAAS1kB,KAAK4X,OAE1BkV,0BACSlS,EAAG8J,SAAS1kB,KAAKA,MAG1ByX,SAAUzF,KAAA4I,EAAGmyC,kBAAenyC,GAC5BoyC,eAAgBh7C,KAAA4I,EAAGqyC,wBAAqBryC,GACxC1C,OAAQlG,KAAA4I,EAAGsyC,gBAAatyC,GACxBuyC,aAAen7C,KAAA4I,EAAGwyC,sBAAmBxyC,OAKpCjF,MAAQ,IAAImD,GAAMa,EAAK9E,KAAM8E,EAAKvuB,WAClColC,WAAWt3D,KAAKygD,EAAKhE,SACrBd,KAAKc,MAAQgE,EAAKhE,QAGlB+O,SAAW,IAAIV,GAASrK,EAAK9E,KAAM8E,EAAKvuB,WACxCmlC,UAAY,OACZC,WAAWt3D,KAAKygD,EAAK+K,YAGrBiP,YAAc,IAAIgB,GAAYhb,EAAK9E,KAAM8E,EAAKvuB,WAC9ColC,WAAWt3D,KAAKygD,EAAKga,eAGrBvF,QAAU,IAAIwc,GAAQjxB,EAAK9E,KAAM8E,EAAKvuB,WACtColC,WAAWt3D,KAAKygD,EAAKyU,WAErBsT,UAAY,OACZsK,WAAa,OAEb3xB,IAAI+C,KAAKypC,QAAU,SAAApqC,GACtB7B,EAAG8B,KAAK,QAAS9B,EAAGgV,mBAAmBnT,OAEpCpC,IAAI+C,KAAKirB,WAAa,SAAA5rB,GACzB7B,EAAG8B,KAAK,cAAe9B,EAAGgV,mBAAmBnT,OAE1CpC,IAAI+C,KAAKiwC,cAAgB,SAAA5wC,GAC5B7B,EAAG8B,KAAK,cAAe9B,EAAGgV,mBAAmBnT,OAE1CpC,IAAI+C,KAAKitC,YAAc,SAAA5tC,GAC1B7B,EAAG8B,KAAK,YAAa9B,EAAGgV,mBAAmBnT,KAE1Cx0B,OAAOqlE,gBACHjzC,IAAI+C,KAAKmwC,cAAgB,SAAA9wC,GAC5B7B,EAAG8B,KAAK,YAAa9B,EAAGgV,mBAAmBnT,OAExCpC,IAAI+C,KAAKowC,cAAgB,SAAA/wC,GAC5B7B,EAAG8B,KAAK,YAAa9B,EAAGgV,mBAAmBnT,OAExCpC,IAAI+C,KAAKqwC,YAAc,SAAAhxC,GAC1B7B,EAAG8B,KAAK,UAAW9B,EAAGgV,mBAAmBnT,SAGtCpC,IAAI+C,KAAKswC,YAAc,SAAAjxC,GAC1B7B,EAAG8B,KAAK,YAAa9B,EAAGgV,mBAAmBnT,OAExCpC,IAAI+C,KAAKuwC,YAAc,SAAAlxC,GAC1B7B,EAAG8B,KAAK,YAAa9B,EAAGgV,mBAAmBnT,OAExCpC,IAAI+C,KAAKwwC,UAAY,SAAAnxC,GACxB7B,EAAG8B,KAAK,UAAW9B,EAAGgV,mBAAmBnT,QAKxCoxC,gBAAiB,IACjBt7C,GAAG,WAAW,cACG,MAAhBqI,EAAG8mB,eACF9mB,EAAGizC,iBAAmBjzC,EAAGxvB,QAAQquB,eACpCmB,EAAGizC,gBAAiB,EACInyF,MAApBk/C,EAAGxvB,QAAQqR,OAAwC/gC,MAAlBk/C,EAAGxvB,QAAQsR,IAAkB,IACxChhC,MAApBk/C,EAAGxvB,QAAQqR,OAAwC/gC,MAAlBk/C,EAAGxvB,QAAQsR,QAC1CiZ,EAAQiF,EAAGkzC,mBAGXrxD,EAA4B/gC,MAApBk/C,EAAGxvB,QAAQqR,MAAqBme,EAAGxvB,QAAQqR,MAAQkZ,EAAMt3C,IACjEq+B,EAA4BhhC,MAApBk/C,EAAGxvB,QAAQsR,IAAqBke,EAAGxvB,QAAQsR,IAAQiZ,EAAMp0C,IACvEq5C,EAAGyX,UAAU51B,EAAOC,EAAK,CAACse,WAAW,SAErCJ,EAAGmzC,IAAI,CAAC/yC,WAAW,IAIlBJ,EAAGmT,kBAAoBnT,EAAGqT,yBAA4BrT,EAAGxvB,QAAQqR,OAAUme,EAAGxvB,QAAQsR,OACtFke,EAAGxvB,QAAQquB,cACdmB,EAAGmT,iBAAkB,EACrBnT,EAAGwT,QAAQL,iBAAkB,EAC7BnT,EAAGP,IAAI+C,KAAK3sB,MAAMyqB,WAAa,UAC/BN,EAAGP,IAAIsT,cAAcnqB,WAAWC,YAAYmX,EAAGP,IAAIsT,eAC/C/S,EAAGxvB,QAAQ0tD,2BACF,kBACFl+B,EAAGxvB,QAAQ0tD,0BACjB,UAKJvmC,GAAG,mBAAmB,WACzBqI,EAAGoG,aAID51B,KACGqvB,WAAWrvB,KAGbypB,KAAKiF,QAAQvH,GAAG,OAAO,SAACtzC,KACtB+uF,OAAO/uF,KACPqtD,YAIHsc,KACGvX,UAAUuX,GAIbn2B,KACG2e,SAAS3e,KAIXub,oFASE,IAAIo6B,GAAansF,KAAMA,KAAKo+C,IAAI6N,UAAW8wB,0CAW7C5qB,SAAWnyD,KAAKmyD,QAAQ+f,UAAU,CAACC,cAAc,SACjDpgB,6CAOI5iC,OAIU,IAFFwrD,GAAUqX,SAAS7iE,EAASqrD,KAG3C73E,QAAQq3C,IAAI,2DAA4D0gC,IAG1Ej/B,GAAKt/C,UAAUqiD,WAAWniD,KAAK2D,KAAMmvB,GAEjC,SAAUA,GACRA,EAAQ+H,OAASl3B,KAAKmvB,QAAQ+H,KAAM,MACjC/H,QAAQ+H,KAAO/H,EAAQ+H,SAGtBuuC,EAAYzlE,KAAKylE,aACnBA,EAAW,KACPwL,EAAYjxE,KAAKm2E,oBAClBhhB,SAAS,WACTA,SAASsQ,EAAUjuB,YACnBwhC,aAAa/H,sCAUjBz6B,OAIHy7C,OAHC3B,WAAY,EAQf2B,EAJGz7C,EAIUe,GADNf,aAAiBoB,IAAWpB,aAAiB89B,GACrB99B,EAIA,IAAIoB,GAAQpB,IAP9B,KAWXx2C,KAAKylE,gBAEFA,UAAUptB,eAEZotB,UAAYwsB,OACZ9/B,SAAWnyD,KAAKmyD,QAAQgD,SAAuB,MAAd88B,EAAqBA,EAAWz6C,MAAQ,wCAOtEm1B,OAEJslB,KACCtlB,EAGA,KACG10C,EAAS,SAAAosC,UAA2B,IAAlBA,EAAM/I,SAE5B22B,EADEtlB,aAAkB/0B,IAAW+0B,aAAkB2H,GACpC,IAAIA,GAAS3H,EAAO,CAAC10C,OAAAA,IAIrB,IAAI2f,GAAQd,GAAA61B,QAAAA,EAAc10C,SATzCg6D,EAAa,KAwBXjyF,KAAK+vE,sBAAsBuE,SACxBvE,WAAWrS,QAAQ,WAErBqS,WAAakiB,OACb9/B,QAAQiD,UAAU68B,mCAOjB1+E,GACFA,GAAQA,EAAKo5D,aACVvX,UAAU7hD,EAAKo5D,QAGlBp5D,GAAQA,EAAKijC,YACV2e,SAAS5hD,EAAKijC,4CAqBV+7B,EAAKpjD,QACXgjC,SAAWnyD,KAAKmyD,QAAQ6mB,aAAazG,GAEtCpjD,GAAWA,EAAQshC,YAChBA,MAAM8hB,EAAKpjD,iDASXnvB,KAAKmyD,SAAWnyD,KAAKmyD,QAAQgkB,gBAAkB,iCAkBlD/iD,EAAIjE,MACHnvB,KAAKylE,WAAmBhmE,MAAN2zB,OAEjBm/C,EAAMrvC,GAAc9P,GAAMA,EAAK,CAACA,GAGhCqyC,EAAYzlE,KAAKylE,UAAU17D,IAAIwoE,GAGjC/xC,EAAQ,KACRC,EAAM,WACVglC,QAAAA,GAAkB,SAAAzR,OACVnrD,EAAImrD,EAASxzB,MAAMjjC,UACnB8V,EAAI,QAAS2gD,EAAWA,EAASvzB,IAAIljC,UAAYy2D,EAASxzB,MAAMjjC,WAExD,OAAVijC,GAAkB33B,EAAI23B,KACxBA,EAAQ33B,IAGE,OAAR43B,GAAgBptB,EAAIotB,KACtBA,EAAMptB,MAKI,OAAVmtB,GAA0B,OAARC,EAAc,KAC5Bke,EAAK3+C,KAEL+3C,EAAO/3C,KAAKmyD,QAAQ3b,MAAM+7B,EAAI,IAChC2f,GAAmC,EAAxBlyF,KAAKw4D,gBAChB25B,EAAwB,KAmCtBC,EAA2B,eACzBC,EAAsBC,GAAsB3zC,EAAI5G,GAElDs6C,EAAoBE,cAAgBF,EAAoBG,SAAWL,EAAsBK,UAC3F7zC,EAAGuU,eAAem/B,EAAoBI,cACtC9zC,EAAGoT,YAeDzP,GAAOnzB,QAA4B1vB,IAAjB0vB,EAAQmzB,MAAqBnzB,EAAQmzB,KACvDowC,GAAUlyD,EAAQC,GAAO,EACzBoe,EAAWyD,EAAuB,KAAf7hB,EAAMD,GAAeh/B,KAAK8D,IAAItF,KAAK05C,MAAMjZ,IAAMzgC,KAAK05C,MAAMlZ,MAAuB,KAAfC,EAAMD,IAE3Fue,GAAY5vB,QAAiC1vB,IAAtB0vB,EAAQ4vB,WAA0B5vB,EAAQ4vB,UAElEA,IAEHozC,EAAwB,CAAEI,cAAc,EAAOE,cAAe,EAAGD,SAAU,SAGxE94C,MAAMgF,SAASg0C,EAAS7zC,EAAW,EAAG6zC,EAAS7zC,EAAW,EAAG,CAAEE,UAAAA,IApBtC,WAE5BqzC,OAGWA,EAA0B,QAhDR,SAAChyC,EAAMuyC,EAAUx1D,OACxC41B,EAAiBu/B,GAAsB3zC,EAAI5G,OAE1B,IAAnBgb,IAIAo/B,IACFA,EAAwBp/B,GAGvBo/B,EAAsBK,SAAWz/B,EAAey/B,SAAYL,EAAsBI,eAG7EJ,EAAsBK,SAAWz/B,EAAey/B,SAAWz/B,EAAew/B,eAEhFJ,EAAwBp/B,EACxBm/B,GAAiC,EAAtBvzC,EAAG6Z,qBAGVn4D,EAAO6xF,EACP9xF,EAAK+xF,EAAsBM,aAC3B7gC,EAAYz0B,EAAO/8B,EAAMC,GAAQD,EAAKC,GAAQ+/C,EAEpDzB,EAAGuU,eAAetB,GAEd+gC,GACFh0C,EAAGoT,4CAmDP5iC,EAASnpB,OAEP0zC,EADEqF,GAAa5vB,QAAiC1vB,IAAtB0vB,EAAQ4vB,WAA2B5vB,EAAQ4vB,UAG3C,IAA1B/+C,KAAKylE,UAAUzoE,aAAgDyC,IAAhCO,KAAKylE,UAAU17D,MAAM,GAAG02B,KAEzDiZ,EAAQ15C,KAAK81D,oBACR/S,OAAOrJ,EAAMt3C,IAAI7E,UAAW,CAACwhD,UAAAA,GAAY/4C,KAI9C0zC,EAAQ15C,KAAK6xF,oBACRn4C,MAAMgF,SAAShF,EAAMt3C,IAAKs3C,EAAMp0C,IAAK,CAAEy5C,UAAAA,GAAa/4C,sDAYrD0zC,EAAQ15C,KAAK81D,eACf1zD,EAAoB,OAAds3C,EAAMt3C,IAAes3C,EAAMt3C,IAAI7E,UAAY,KACjD+H,EAAoB,OAAdo0C,EAAMp0C,IAAeo0C,EAAMp0C,IAAI/H,UAAY,KACjDq1F,EAAU,KACVC,EAAU,QAEH,MAAPzwF,GAAsB,MAAPkD,EAAa,KAC1Bu5C,EAAYv5C,EAAMlD,EAClBy8C,GAAY,IACdA,EAAW,QAEPi0C,EAASj0C,EAAW7+C,KAAKq1C,MAAM2J,OAAOpW,MAEtC41B,EAAc,GAChBC,EAAoB,QAGxBnmB,SAAAA,GAAat4C,KAAKmyD,QAAQ3b,OAAO,SAACuB,EAAM90C,MAClC80C,EAAKksB,aAAc,CAErBzF,EAAYv7D,GAAO80C,EAAKsY,QADJ,GAEpBoO,EAAoBD,EAAYv7D,GAAKjG,WAItByhE,EAAoB,qBAG5B3hE,MACPw7C,SAAAA,GAAakmB,GAAa,SAAAI,GACxBA,EAAI9hE,SAFCA,EAAI,EAAGA,EAAI2hE,EAAmB3hE,MAA9BA,SAQXw7C,SAAAA,GAAat4C,KAAKmyD,QAAQ3b,OAAO,SAAAuB,OAG3Bg7C,EACAC,EAHExyD,EAAQyyD,GAASl7C,GACjBtX,EAAMyyD,GAAOn7C,GAIf4d,EAAKxmC,QAAQguB,KACf41C,EAAavyD,GAASuX,EAAKo7C,gBAAmB,IAAML,EACpDE,EAAUvyD,GAASsX,EAAKq7C,eAAiB,IAAMN,IAE/CC,EAAavyD,GAASuX,EAAKq7C,eAAkB,IAAMN,EACnDE,EAAUvyD,GAASsX,EAAKo7C,gBAAkB,IAAML,GAG9CC,EAAY3wF,IACdA,EAAM2wF,EACNH,EAAU76C,GAERi7C,EAAU1tF,IACZA,EAAM0tF,EACNH,EAAU96C,MAIV66C,GAAWC,EAAS,KAChBQ,EAAMT,EAAQQ,eAAiB,GAC/BE,EAAMT,EAAQM,gBAAkB,GAChC7xC,EAAQthD,KAAKq1C,MAAM2J,OAAOpW,MAAQyqD,EAAMC,EAE1ChyC,EAAQ,IACNthD,KAAKmvB,QAAQguB,KACf/6C,EAAM6wF,GAASL,GAAWU,EAAMz0C,EAAWyC,EAC3Ch8C,EAAM4tF,GAAOL,GAAaQ,EAAMx0C,EAAWyC,IAE3Cl/C,EAAM6wF,GAASL,GAAWS,EAAMx0C,EAAWyC,EAC3Ch8C,EAAM4tF,GAAOL,GAAaS,EAAMz0C,EAAWyC,WAM5C,CACLl/C,IAAY,MAAPA,EAAc,IAAI1F,KAAK0F,GAAO,KACnCkD,IAAY,MAAPA,EAAc,IAAI5I,KAAK4I,GAAO,mDASjClD,EAAM,KACNkD,EAAM,KAENtF,KAAKylE,qBACFA,mBAAkB,SAAA1tB,OACfvX,EAAQ8X,GAAKpB,QAAQa,EAAKvX,MAAO,QAAQjjC,UACzCkjC,EAAQ6X,GAAKpB,QAAoBz3C,MAAZs4C,EAAKtX,IAAmBsX,EAAKtX,IAAMsX,EAAKvX,MAAO,QAAQjjC,WACtE,OAAR6E,GAAgBo+B,EAAQp+B,KAC1BA,EAAMo+B,IAEI,OAARl7B,GAAgBm7B,EAAMn7B,KACxBA,EAAMm7B,YAKL,CACLr+B,IAAY,MAAPA,EAAc,IAAI1F,KAAK0F,GAAO,KACnCkD,IAAY,MAAPA,EAAc,IAAI5I,KAAK4I,GAAO,iDAUpBk7C,OACX2B,EAAU3B,EAAMxB,OAASwB,EAAMxB,OAAO70C,EAAIq2C,EAAM2B,QAChDC,EAAU5B,EAAMxB,OAASwB,EAAMxB,OAAOz1C,EAAIi3C,EAAM4B,QAChDO,EAAsB3iD,KAAKo+C,IAAI3E,gBAAgBmJ,wBAC/Cz4C,EAAInK,KAAKmvB,QAAQguB,IAAMwF,EAAoBhO,MAAQwN,EAAUA,EAAUQ,EAAoBjO,KAC3FnrC,EAAI64C,EAAUO,EAAoBG,IAElC/K,EAAQ/3C,KAAKmyD,QAAQqjB,eAAeh1B,GACpC6jB,EAAQrkE,KAAKmyD,QAAQqkB,gBAAgBh2B,GACrCmP,EAAaH,GAAW+jC,qBAAqB/yC,GAE7CsQ,EAAO9wD,KAAKmyD,QAAQhjC,QAAQ2hC,MAAQ,KACpCnV,EAAQ37C,KAAK44C,KAAKN,KAAKsY,WACvB7sB,EAAO/jC,KAAK44C,KAAKN,KAAKuY,UACtBnwC,EAAO1gB,KAAKixF,QAAQ9mF,GACpB4mD,EAAcD,EAAOA,EAAKpwC,EAAMi7B,EAAO5X,GAAQrjB,EAE/CknB,EAAU0Q,GAAKk7C,UAAUhzC,GAC3BizC,EAAO,YACC,MAAR17C,EAAkE07C,EAAO,OACtD,MAAd9jC,EAA6D8jC,EAAO,cACpEn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAKyoD,SAASrK,IAAI4J,aAC1ChoD,KAAKs0D,WAAahc,GAAKo7C,UAAU9rD,EAAS5nC,KAAKs0D,UAAUlW,IAAI4J,YADAyrC,EAAO,OAEpEn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAKmyD,QAAQ/T,IAAIujB,UAAoB8xB,EAAO,cACpEn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAK03D,YAAY7H,KAAoB4jC,EAAO,eACpEn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAKo+C,IAAIY,UAA4By0C,EAAO,cAEtE,CACLjzC,MAAAA,EACAzI,KAAMA,EAAOA,EAAK3kB,GAAK,KACvBwsC,YAAW7nB,KAASA,EAAK6nB,UACzBppB,MAAOuB,EAAOA,EAAKvB,OAAS,GAAI,KAChC6tB,MAAOA,EAAQA,EAAMlI,QAAU,KAC/BxM,WAAYA,EAAaA,EAAWxgC,QAAQiE,GAAK,KACjDqgE,KAAAA,EACAE,MAAOnzC,EAAM4D,SAAW5D,EAAM4D,SAASuvC,MAAQnzC,EAAMmzC,MACrDC,MAAOpzC,EAAM4D,SAAW5D,EAAM4D,SAASwvC,MAAQpzC,EAAMozC,MACrDzpF,EAAAA,EACAZ,EAAAA,EACAmX,KAAAA,EACAqwC,YAAAA,+CAQE/wD,KAAK05C,MAAMqD,aACRrD,MAAMkF,eAEqBn/C,MAA5BO,KAAKmvB,QAAQquB,kBACVgB,WAAWx+C,KAAKmvB,cAElBuqB,MAAM6E,kDASb9C,GAAKt/C,UAAU41D,QAAQ11D,KAAK2D,qCAQvBgD,OACGw9B,EAA0Bx9B,EAA1Bw9B,MAAOC,EAAmBz9B,EAAnBy9B,IAAKse,EAAc/7C,EAAd+7C,UACfte,OAKEiZ,MAAMgF,SAASle,EAAOC,EAAK,CAC9Bse,UAAWA,SALRgE,OAAOviB,EAAMjjC,UAAW,CAC3BwhD,UAAAA,WA7tB8BtD,IA4uBtC,SAASw3C,GAASl7C,UACTO,GAAKpB,QAAQa,EAAKxkC,KAAKitB,MAAO,QAAQjjC,UAQ/C,SAAS21F,GAAOn7C,OACRtX,EAAuBhhC,MAAjBs4C,EAAKxkC,KAAKktB,IAAmBsX,EAAKxkC,KAAKktB,IAAMsX,EAAKxkC,KAAKitB,aAC5D8X,GAAKpB,QAAQzW,EAAK,QAAQljC,UAQnC,SAAS+0F,GAAsBuB,EAAU97C,OAClCA,EAAKnnB,cAED,MAGHkjE,EAAgBD,EAAS1kE,QAAQguB,IAAM02C,EAASx+C,MAAM8b,eAAetoB,OAASgrD,EAASx+C,MAAM6b,cAAcroB,OAC3G8tB,EAAgBk9B,EAASx+C,MAAM2J,OAAOnW,OAEtCw7B,EAAQtsB,EAAKnnB,OACfpU,EAAS6nD,EAAMvhB,IACfyvC,GAAe,EACblqC,EAAcwrC,EAASprC,SAASt5B,QAAQk5B,YAAYC,KAEpDkqC,EAAU,iBACG,UAAfnqC,EACOgc,EAAMx7B,OAASkP,EAAK+K,IAAM/K,EAAKlP,OAG/BkP,EAAK+K,KAIVixC,GAAkD,EAA5BF,EAASr7B,gBAC/Bw7B,EAAex3E,EAASg2E,IACxB3pD,EAASkP,EAAKlP,cAEhBmrD,EAAeD,EACbv3E,EAASs3E,GAAiBt3E,EAASg2E,IAAY3pD,IACjDrsB,GAAUg2E,IAAYqB,EAAS1hC,QAAQhjC,QAAQmrC,OAAOviB,KAAK8iB,UAGtDm5B,EAAenrD,EAASkrD,EAAsBD,EACrDt3E,GAAUg2E,IAAY3pD,EAASirD,EAAgBD,EAAS1hC,QAAQhjC,QAAQmrC,OAAOviB,KAAK8iB,SAGpF03B,GAAe,EAKV,CAAEA,aAAAA,EAAcE,aAFvBj2E,EAAShb,KAAKY,IAAIoa,EAAQm6C,EAAgBm9B,GAEGtB,QAASwB,OC7zBlDC,yBAcFzzD,EACAC,EACAyzD,EACAC,EACAt9B,EACA1N,OACAirC,0DACAC,6EACKC,WAAa,CAAC,EAAG,EAAG,EAAG,SACvBC,WAAa,CAAC,IAAM,GAAK,EAAG,QAC5BC,YAAc,UAEd39B,gBAAkBA,OAClB1N,gBAAkBA,OAClBzD,OAASllB,OACT0a,KAAOza,OAEPkb,MAAQ,OACR84C,cAAgB,OAChBC,gBAAkB,OAClBC,sBAEAP,UAAYA,OACZF,eAAiBA,OACjBC,aAAeA,OAEfE,mBAAqBA,EAEtBH,GAAkBC,EAAc,KAC5Bx1C,EAAK3+C,KACL+lD,EAAe,SAAAjkD,OACb8yF,EAAU9yF,EAASA,GAAS68C,EAAG+1C,gBAAkB/1C,EAAG41C,WAAW51C,EAAG81C,sBACpE3yF,GAAS68C,EAAG+1C,gBAAkB/1C,EAAG41C,WAAW51C,EAAG81C,eAAwB91C,EAAG+1C,gBAAkB/1C,EAAG41C,WAAW51C,EAAG81C,cAA7C,GAC3DG,EAAWj2C,EAAG+1C,gBAAkB/1C,EAAG41C,WAAW51C,EAAG81C,cAGjDG,GAGPV,SACGxuC,QAAiC,EAAvB1lD,KAAK00F,gBAAsB10F,KAAKu0F,WAAWv0F,KAAKy0F,mBAC1D/uC,OAASK,EAAa/lD,KAAK0lD,SAG9ByuC,SACGj5C,MAAQl7C,KAAK00F,gBAAkB10F,KAAKu0F,WAAWv0F,KAAKy0F,mBACpDv5C,KAAO6K,EAAa/lD,KAAKk7C,YAE3By5C,mEAQKxrC,QACPA,gBAAkBA,oCAOf0N,QACHA,gBAAkBA,+CAOjBnd,EAAQ15C,KAAKk7C,KAAOl7C,KAAK0lD,YAC1B/J,MAAQ37C,KAAK62D,gBAAkBnd,MAC9Bm7C,EAAmB70F,KAAKmpD,gBAAkBnpD,KAAK27C,MAC/Cm5C,EAAoBp7C,EAAQ,EAC5Bl4C,KAAK+a,MAAM/a,KAAKw4C,IAAIN,GAASl4C,KAAKuzF,MAClC,OAEDN,cAAgB,OAChBC,gBAAkBlzF,KAAK6D,IAAI,GAAIyvF,OAEhCt0D,EAAQ,EACRs0D,EAAmB,IACrBt0D,EAAQs0D,WAGNE,GAAgB,EACXp+E,EAAI4pB,EAAOh/B,KAAKc,IAAIsU,IAAMpV,KAAKc,IAAIwyF,GAAmBl+E,IAAK,MAC7D89E,gBAAkBlzF,KAAK6D,IAAI,GAAIuR,OAC/B,IAAI7C,EAAI,EAAGA,EAAI/T,KAAKu0F,WAAWv3F,OAAQ+W,IAAK,IAC9B/T,KAAK00F,gBAAkB10F,KAAKu0F,WAAWxgF,IACxC8gF,EAAkB,CAChCG,GAAgB,OACXP,aAAe1gF,aAIF,IAAlBihF,0CAWClzF,UACCA,GAAS9B,KAAK00F,gBAAkB10F,KAAKs0F,WAAWt0F,KAAKy0F,gBAAmB,2CAQzEz0F,KAAK00F,gBAAkB10F,KAAKu0F,WAAWv0F,KAAKy0F,0DAQ7CQ,EAAYj1F,KAAK00F,gBAAkB10F,KAAKs0F,WAAWt0F,KAAKy0F,qBACvDz0F,KAAKk1F,aAAal1F,KAAK0lD,QAAWuvC,EAAaj1F,KAAK0lD,OAASuvC,GAAcA,uCAQxEh6C,OACNk6C,EAAcl6C,EAAQm6C,YAAY,SACC,mBAA5Bp1F,KAAKq0F,qBACdc,EAAcn1F,KAAKq0F,mBAAmBp5C,IAGb,iBAAhBk6C,YACCA,GAEoB,iBAAhBA,EACPA,EAGAl6C,EAAQm6C,YAAY,8CAUvBntC,EAAQ,GACRlkB,EAAO/jC,KAAK6wD,UACZwkC,GAAgBtxD,EAAQ/jC,KAAK0lD,OAAS3hB,GAASA,EAC5CjnC,EAAKkD,KAAK0lD,OAAS2vC,EAAer1F,KAAKk7C,KAAKp+C,EAAI,KAASA,GAAKinC,EACjEjnC,GAAKkD,KAAK0lD,QACZuC,EAAMhrD,KAAK,CAACq4F,MAAOt1F,KAAKu1F,SAASz4F,GAAIyM,EAAGvJ,KAAKk1F,aAAap4F,GAAIyD,IAAKP,KAAKw1F,YAAY14F,YAGjFmrD,sCAOGltC,OACJ06E,EAAaz1F,KAAKy0F,aAClBiB,EAAW11F,KAAK0lD,OAChBiwC,EAAS31F,KAAKk7C,KAEdyD,EAAK3+C,KACL41F,EAAoB,WACxBj3C,EAAG+1C,iBAAmB,GAElBmB,EAAoB,WACxBl3C,EAAG+1C,iBAAmB,GAGnB35E,EAAM05E,cAAgB,GAAKz0F,KAAKy0F,cAAgB,GAAO15E,EAAM05E,aAAe,GAAKz0F,KAAKy0F,aAAe,IAE/F15E,EAAM05E,aAAez0F,KAAKy0F,mBAE9BA,aAAe,EACF,GAAdgB,GAGFG,IAFAA,WAOGnB,aAAe,EACF,GAAdgB,GAGFI,IAFAA,cAQEC,EAAY/6E,EAAMm6E,aAAa,GAC/Ba,EAAYh7E,EAAM81C,UAAY91C,EAAM4gC,MAEtCxe,GAAO,EACP8b,EAAQ,GAEJ9b,GAAQ8b,IAAS,GAAG,MAGrB0C,MAAQo6C,GAAa/1F,KAAKu0F,WAAWv0F,KAAKy0F,cAAgBz0F,KAAK00F,qBAC9DsB,EAAWh2F,KAAK62D,gBAAkB72D,KAAK27C,WAGxC+J,OAASgwC,OACTx6C,KAAOl7C,KAAK0lD,OAASswC,MAEpBC,EAAiBj2F,KAAKk7C,KAAOl7C,KAAK27C,MAClCs5C,EAAYj1F,KAAK00F,gBAAkB10F,KAAKs0F,WAAWt0F,KAAKy0F,cACxDyB,EAAcl2F,KAAKm2F,gBAAkBp7E,EAAMo7E,mBAE7Cn2F,KAAKo0F,UAAW,KACZgC,EAAaN,EAAYG,OAC1B/6C,MAASk7C,EAAap2F,KAAK27C,WAC3B+J,OAAS1lD,KAAKk7C,KAAO86C,OAErBh2F,KAAKk0F,qBAIHxuC,QAAUwwC,EAAcl2F,KAAK27C,WAC7BT,KAAOl7C,KAAK0lD,OAASswC,SAJrBtwC,QAAUuvC,EAAaiB,EAAcl2F,KAAK27C,WAC1CT,KAAOl7C,KAAK0lD,OAASswC,OAMzBh2F,KAAKm0F,cAAgBn0F,KAAKk7C,KAAOy6C,EAAO,KAE3CE,IACA14D,GAAO,WAGJn9B,KAAKk0F,gBAAkBl0F,KAAK0lD,OAASgwC,EAAS,KAAS,MACtD11F,KAAKo0F,WAAasB,GAAY,GAE3B,CAELG,IACA14D,GAAO,WAJPx6B,QAAQH,KAAK,uDAQbxC,KAAKk0F,gBAAkBl0F,KAAKm0F,cAAgB6B,EAAYL,EAAOD,GACjEE,IACAz4D,GAAO,GAGTA,GAAO,yCASEr7B,UACJ9B,KAAK62D,iBAAoB/0D,EAAQ9B,KAAK0lD,QAAU1lD,KAAK27C,4CAQhD06C,UACHr2F,KAAK62D,gBAAkBw/B,GAAUr2F,KAAK27C,MAAS37C,KAAK0lD,4+LCjS3D4wC,0BAUQ19C,EAAMzpB,EAASonE,EAAKC,kDAEzBpjE,GAAKs3C,OACL9xB,KAAOA,IAEPsE,eAAiB,CACpBmL,YAAa,OACbE,iBAAiB,EACjBtC,iBAAiB,EACjBK,eAAe,EACfmwC,OAAO,EACPC,iBAAkB,EAClBC,iBAAkB,EAClBC,aAAc,GACdC,aAAc,EACdC,UAAW,GACXluD,MAAO,OACP0yB,SAAS,EACTy7B,YAAY,EACZriD,KAAM,CACJgF,MAAO,CAACt3C,SAAK3C,EAAW6F,SAAK7F,GAC7BhC,gBAAOqE,mBACKg/C,GAAWh/C,EAAMszF,YAAY,MAEzC3lC,MAAO,CAAChE,UAAMhsD,EAAW+0B,WAAO/0B,IAElCk1C,MAAO,CACL+E,MAAO,CAACt3C,SAAK3C,EAAW6F,SAAK7F,GAC7BhC,gBAAOqE,mBACKg/C,GAAWh/C,EAAMszF,YAAY,MAEzC3lC,MAAO,CAAChE,UAAMhsD,EAAW+0B,WAAO/0B,OAI/B+2F,iBAAmBA,IACnBQ,aAAeT,IACflhD,MAAQ,KACR4hD,YAAc,CACjBhvC,MAAO,GACPivC,OAAQ,GACRznC,MAAO,MAGJrR,IAAM,KACNzC,WAAQl8C,IACRi6C,MAAQ,CAAClZ,MAAO,EAAGC,IAAK,KAExBtR,QAAUmpB,GAAKh7C,OAAO,GAAIogD,EAAKR,kBAC/Bi6C,iBAAmB,IAEnB34C,WAAWrvB,KACXyZ,MAAQyO,OAAO,UAAIqG,EAAKvuB,QAAQyZ,OAASriC,QAAQ,KAAM,OACvDqlE,SAAWluB,EAAK9U,QAChBC,OAAS6U,EAAKs5C,aAAap0C,wBAAwB/Z,SACnD0R,QAAS,IAET68C,WAAa,KACbC,cAAgB,IAChBC,eAAiB,IAEjB9rB,WAAa,IACb+rB,QAAS,IACTC,WAAa,OACbC,YAAc,KACdC,cAAe,IAEf/qB,OAAS,KACTgrB,eAAiB,IAGjBjvC,UACajpD,MAAdi+C,EAAK/B,SACFi8C,kBAEFC,UAAY,CAACtB,IAAK74C,EAAK64C,IAAKkB,YAAa/5C,EAAK+5C,YAAatoE,QAASuuB,EAAKvuB,QAASw9C,OAAQjvB,EAAKivB,YAE9FhuB,iBACD/F,KAAKiF,QAAQvH,GAAG,gBAAgB,WACnCqI,EAAGP,IAAI05C,cAActjE,MAAMsuB,cAASnE,EAAG/F,KAAKY,SAASoY,yEAShDrpB,EAAOwvD,GACT/3F,KAAK2sE,OAAOtvE,eAAekrC,UACzBokC,OAAOpkC,GAASwvD,QAElBJ,gBAAkB,sCAQbpvD,EAAOwvD,GACZ/3F,KAAK2sE,OAAOtvE,eAAekrC,UACzBovD,gBAAkB,QAEpBhrB,OAAOpkC,GAASwvD,sCAOXxvD,GACNvoC,KAAK2sE,OAAOtvE,eAAekrC,YACtBvoC,KAAK2sE,OAAOpkC,QACdovD,gBAAkB,sCAQhBxoE,MACLA,EAAS,KACPkhC,GAAS,EACTrwD,KAAKmvB,QAAQk5B,aAAel5B,EAAQk5B,kBAAuC5oD,IAAxB0vB,EAAQk5B,cAC7DgI,GAAS,GAkBX/X,GAAKqQ,oBAhBU,CACb,cACA,kBACA,kBACA,QACA,mBACA,mBACA,eACA,eACA,YACA,QACA,UACA,OACA,QACA,cAE+B3oD,KAAKmvB,QAASA,QAE1Cy8C,SAAWv0B,OAAO,UAAIr3C,KAAKmvB,QAAQyZ,OAASriC,QAAQ,KAAM,MAChD,IAAX8pD,GAAmBrwD,KAAKo+C,IAAIuqB,aACzBzY,YACAsT,gDASJplB,IAAIuqB,MAAQv7C,SAASE,cAAc,YACnC8wB,IAAIuqB,MAAMn0C,MAAMoU,MAAQ5oC,KAAKmvB,QAAQyZ,WACrCwV,IAAIuqB,MAAMn0C,MAAMqU,OAAS7oC,KAAK6oC,YAE9BuV,IAAI05C,cAAgB1qE,SAASE,cAAc,YAC3C8wB,IAAI05C,cAActjE,MAAMoU,MAAQ,YAChCwV,IAAI05C,cAActjE,MAAMqU,OAAS7oC,KAAK6oC,YACtCuV,IAAI05C,cAActjE,MAAMkL,SAAW,gBACnC0e,IAAI05C,cAActjE,MAAMyqB,WAAa,eACrCb,IAAI05C,cAActjE,MAAMC,QAAU,aAGlC8hE,IAAMnpE,SAASya,gBAAgB,6BAA8B,YAC7D0uD,IAAI/hE,MAAMkL,SAAW,gBACrB62D,IAAI/hE,MAAMsuB,IAAM,WAChByzC,IAAI/hE,MAAMqU,OAAS,YACnB0tD,IAAI/hE,MAAMoU,MAAQ,YAClB2tD,IAAI/hE,MAAMC,QAAU,aACpB2pB,IAAIuqB,MAAMh0C,YAAY30B,KAAKu2F,qDAS5BpsF,EAFJ6tF,GAAwBh4F,KAAKy3F,iBAGvBX,EAAY92F,KAAKmvB,QAAQ2nE,UAG3BvtF,EAAI0uF,KAGN9tF,EAD+B,SAA7BnK,KAAKmvB,QAAQk5B,YAHE,EAObroD,KAAK4oC,MAAQkuD,EAPA,MAUboB,EAAal4C,GAAYhgD,KAAK2sE,WACpCurB,QAAAA,GAAgB,SAAC/6F,EAAGC,UAAMD,EAAIC,GAAK,EAAI,+CAEjB86F,+BAAY,KAAvB/7B,WAC4B,IAAjCn8D,KAAK2sE,OAAOxQ,GAASb,cAAmE77D,IAA9CO,KAAKw2F,iBAAiBv3C,WAAWkd,KAAwE,IAA9Cn8D,KAAKw2F,iBAAiBv3C,WAAWkd,UACnIwQ,OAAOxQ,GAASg8B,UAAUrB,EAhBhB,GAgBuC92F,KAAK63F,UAAW1tF,EAAGZ,GACzEA,GAAK6uF,uFAITJ,GAAwBh4F,KAAKy3F,kBACxBC,cAAe,2CAOM,IAAtB13F,KAAK03F,eACPM,GAAwBh4F,KAAKy3F,aAC7BO,GAAwBh4F,KAAKy3F,kBACxBC,cAAe,uCAQjBn9C,QAAS,EACTv6C,KAAKo+C,IAAIuqB,MAAMphC,aACe,SAA7BvnC,KAAKmvB,QAAQk5B,iBACVzP,KAAKwF,IAAI1J,KAAK/f,YAAY30B,KAAKo+C,IAAIuqB,YAGnC/vB,KAAKwF,IAAIzJ,MAAMhgB,YAAY30B,KAAKo+C,IAAIuqB,QAIxC3oE,KAAKo+C,IAAI05C,cAAcvwD,iBACrBqR,KAAKwF,IAAI6S,qBAAqBt8B,YAAY30B,KAAKo+C,IAAI05C,oBAErD15C,IAAI05C,cAActjE,MAAMC,QAAU,4CAOlC8lB,QAAS,EACVv6C,KAAKo+C,IAAIuqB,MAAMphC,iBACZ6W,IAAIuqB,MAAMphC,WAAWC,YAAYxnC,KAAKo+C,IAAIuqB,YAG5CvqB,IAAI05C,cAActjE,MAAMC,QAAU,wCAQhC+L,EAAOC,QACTiZ,MAAMlZ,MAAQA,OACdkZ,MAAMjZ,IAAMA,uCAQboY,GAAU,EACVw/C,EAAe,MAKd,IAAMl8B,UAFN/d,IAAI05C,cAActjE,MAAMsuB,cAAS9iD,KAAK44C,KAAKY,SAASoY,gBAEnC5xD,KAAK2sE,OACrB3sE,KAAK2sE,OAAOtvE,eAAe8+D,MACQ,IAAjCn8D,KAAK2sE,OAAOxQ,GAASb,cAAmE77D,IAA9CO,KAAKw2F,iBAAiBv3C,WAAWkd,KAAwE,IAA9Cn8D,KAAKw2F,iBAAiBv3C,WAAWkd,IACxIk8B,QAIsB,IAAxBr4F,KAAK23F,gBAAyC,IAAjBU,OAC1BnoC,WAEF,MACEsT,YACA36B,OAASwO,OAAOr3C,KAAKg3F,aAAaxiE,MAAMqU,OAAOtiC,QAAQ,KAAM,UAG7D63C,IAAI05C,cAActjE,MAAMqU,iBAAY7oC,KAAK6oC,kBACzCD,OAAiC,IAAzB5oC,KAAKmvB,QAAQmsC,QAAmBjkB,OAAO,UAAIr3C,KAAKmvB,QAAQyZ,OAASriC,QAAQ,KAAM,KAAO,MAE7F8uC,EAAQr1C,KAAKq1C,MACbszB,EAAQ3oE,KAAKo+C,IAAIuqB,MAGvBA,EAAMrgC,UAAY,qBAGbygB,yBAECV,EAAcroD,KAAKmvB,QAAQk5B,YAC3BE,EAAkBvoD,KAAKmvB,QAAQo5B,gBAC/BtC,EAAkBjmD,KAAKmvB,QAAQ82B,gBAE/BqyC,EAAkCt4F,KAAK44C,KAAKwF,IAAI6S,qBAAqB7H,YAG3E/T,EAAM2T,iBAAmBT,EAAkBlT,EAAM4T,gBAAkB,EACnE5T,EAAM6T,iBAAmBjD,EAAkB5Q,EAAM8T,gBAAkB,EAEnE9T,EAAMiU,eAAiBgvC,EAAkCt4F,KAAKwrE,WAAaxrE,KAAK4oC,MAAQ,EAAI5oC,KAAKmvB,QAAQwnE,iBACzGthD,EAAMgU,gBAAkB,EACxBhU,EAAMmU,eAAiB8uC,EAAkCt4F,KAAKwrE,WAAaxrE,KAAK4oC,MAAQ,EAAI5oC,KAAKmvB,QAAQunE,iBACzGrhD,EAAMkU,gBAAkB,EAGJ,SAAhBlB,GACFsgB,EAAMn0C,MAAMsuB,IAAM,IAClB6lB,EAAMn0C,MAAMkgB,KAAO,IACnBi0B,EAAMn0C,MAAMq0B,OAAS,GACrB8f,EAAMn0C,MAAMoU,gBAAW5oC,KAAK4oC,YAC5B+/B,EAAMn0C,MAAMqU,iBAAY7oC,KAAK6oC,kBACxBwM,MAAMzM,MAAQ5oC,KAAK44C,KAAKY,SAAS9E,KAAK9L,WACtCyM,MAAMxM,OAAS7oC,KAAK44C,KAAKY,SAAS9E,KAAK7L,SAG5C8/B,EAAMn0C,MAAMsuB,IAAM,GAClB6lB,EAAMn0C,MAAMq0B,OAAS,IACrB8f,EAAMn0C,MAAMkgB,KAAO,IACnBi0B,EAAMn0C,MAAMoU,gBAAW5oC,KAAK4oC,YAC5B+/B,EAAMn0C,MAAMqU,iBAAY7oC,KAAK6oC,kBACxBwM,MAAMzM,MAAQ5oC,KAAK44C,KAAKY,SAAS7E,MAAM/L,WACvCyM,MAAMxM,OAAS7oC,KAAK44C,KAAKY,SAAS7E,MAAM9L,QAG/CgQ,EAAU74C,KAAK43F,gBACf/+C,EAAU74C,KAAK8pD,cAAgBjR,GAEJ,IAAvB74C,KAAKmvB,QAAQsnE,WACV8B,yBAGAC,qBAGFC,aAAapwC,UAEbxP,qDAUHA,GAAU,EACdm/C,GAAwBh4F,KAAKi3F,YAAYhvC,OACzC+vC,GAAwBh4F,KAAKi3F,YAAYC,YACnC7uC,EAAcroD,KAAKmvB,QAAL,YACdupE,EAAiDj5F,MAAnCO,KAAKmvB,QAAQk5B,GAAa3O,MAAqB15C,KAAKmvB,QAAQk5B,GAAa3O,MAAQ,GAGjGy6C,GAAe,EACI10F,MAAnBi5F,EAAYpzF,WACTo0C,MAAMjZ,IAAMi4D,EAAYpzF,IAC7B6uF,GAAe,OAEbD,GAAiB,EACEz0F,MAAnBi5F,EAAYt2F,WACTs3C,MAAMlZ,MAAQk4D,EAAYt2F,IAC/B8xF,GAAiB,QAGdv4C,MAAQ,IAAIs4C,GACfj0F,KAAK05C,MAAMlZ,MACXxgC,KAAK05C,MAAMjZ,IACXyzD,EACAC,EACAn0F,KAAKo+C,IAAIuqB,MAAMjS,aACf12D,KAAKq1C,MAAM8T,gBACXnpD,KAAKmvB,QAAQ4nE,WACb/2F,KAAKmvB,QAAQk5B,GAAa5qD,SAGR,IAAhBuC,KAAKu3F,QAAuC93F,MAAnBO,KAAKw3F,iBAC3B77C,MAAMg9C,YAAY34F,KAAKw3F,WAAW77C,YAClCyC,IAAI05C,cAActjE,MAAMC,QAAU,aAElC2pB,IAAI05C,cAActjE,MAAMC,QAAU,aAIpCmkE,aAAe,MAEd3wC,EAAQjoD,KAAK27C,MAAMk9C,cACzB5wC,QAAAA,GACE,SAAAuC,OACQjhD,EAAIihD,EAAKjhD,EACT8gD,EAAUG,EAAK8qC,MACjB3/B,EAAKxmC,QAAL,kBAA+C,IAAZk7B,GACrCsL,EAAKmjC,aAAavvF,EAAI,EAAGihD,EAAKjqD,IAAK8nD,EAAa,uBAAwBsN,EAAKtgB,MAAM4T,iBAEjFoB,GACE9gD,GAAK,GACPosD,EAAKmjC,aAAavvF,EAAI,EAAGihD,EAAKjqD,IAAK8nD,EAAa,uBAAwBsN,EAAKtgB,MAAM8T,kBAGnE,IAAhBwM,EAAK4hC,SACHltC,EACFsL,EAAKojC,YAAYxvF,EAAG8+C,EAAa,oCAAqCsN,EAAKxmC,QAAQunE,iBAAkB/gC,EAAKtgB,MAAMmU,gBAGhHmM,EAAKojC,YAAYxvF,EAAG8+C,EAAa,oCAAqCsN,EAAKxmC,QAAQwnE,iBAAkBhhC,EAAKtgB,MAAMiU,wBAMpH0vC,EAAa,OACuBv5F,IAApCO,KAAKmvB,QAAQk5B,GAAaoH,YAAgEhwD,IAAzCO,KAAKmvB,QAAQk5B,GAAaoH,MAAMhE,OACnFutC,EAAah5F,KAAKq1C,MAAM4jD,qBAEpBz8E,GAAgC,IAAvBxc,KAAKmvB,QAAQsnE,MAAiBj1F,KAAK8D,IAAItF,KAAKmvB,QAAQ2nE,UAAWkC,GAAch5F,KAAKmvB,QAAQynE,aAAe,GAAKoC,EAAah5F,KAAKmvB,QAAQynE,aAAe,UAGlK52F,KAAK44F,aAAgB54F,KAAK4oC,MAAQpsB,IAAoC,IAAzBxc,KAAKmvB,QAAQmsC,cACvD1yB,MAAQ5oC,KAAK44F,aAAep8E,OAC5B2S,QAAQyZ,gBAAW5oC,KAAK4oC,YAC7BovD,GAAwBh4F,KAAKi3F,YAAYhvC,OACzC+vC,GAAwBh4F,KAAKi3F,YAAYC,aACpC7mC,SACLxX,GAAU,GAGH74C,KAAK44F,aAAgB54F,KAAK4oC,MAAQpsB,IAAoC,IAAzBxc,KAAKmvB,QAAQmsC,SAAoBt7D,KAAK4oC,MAAQ5oC,KAAK4rE,eAClGhjC,MAAQpnC,KAAK8D,IAAItF,KAAK4rE,SAAU5rE,KAAK44F,aAAep8E,QACpD2S,QAAQyZ,gBAAW5oC,KAAK4oC,YAC7BovD,GAAwBh4F,KAAKi3F,YAAYhvC,OACzC+vC,GAAwBh4F,KAAKi3F,YAAYC,aACpC7mC,SACLxX,GAAU,IAGVm/C,GAAwBh4F,KAAKi3F,YAAYhvC,OACzC+vC,GAAwBh4F,KAAKi3F,YAAYC,QACzCr+C,GAAU,GAGLA,uCAQI/2C,UACJ9B,KAAK27C,MAAMu5C,aAAapzF,yCAQnBqI,UACLnK,KAAK27C,MAAMu9C,cAAc/uF,wCAarBZ,EAAGkiD,EAAMpD,EAAa/f,EAAW6wD,OAEtC5wD,EAAQyvD,GAAsB,MAAOh4F,KAAKi3F,YAAYC,OAAQl3F,KAAKo+C,IAAIuqB,OAC7EpgC,EAAMD,UAAYA,EAClBC,EAAMmjB,UAAYD,EACE,SAAhBpD,GACF9f,EAAM/T,MAAMkgB,gBAAW10C,KAAKmvB,QAAQynE,mBACpCruD,EAAM/T,MAAM4kE,UAAY,UAGxB7wD,EAAM/T,MAAMmgB,iBAAY30C,KAAKmvB,QAAQynE,mBACrCruD,EAAM/T,MAAM4kE,UAAY,QAG1B7wD,EAAM/T,MAAMsuB,cAASv5C,EAAI,GAAM4vF,EAAkBn5F,KAAKmvB,QAAQ0nE,mBAE9DprC,GAAQ,OAEF4tC,EAAe73F,KAAK8D,IAAItF,KAAKq1C,MAAMmW,eAAgBxrD,KAAKq1C,MAAM2U,gBAChEhqD,KAAK44F,aAAentC,EAAKzuD,OAASq8F,SAC/BT,aAAentC,EAAKzuD,OAASq8F,uCAY1B9vF,EAAG8+C,EAAa/f,EAAW9rB,EAAQosB,OACzB,IAAhB5oC,KAAKu3F,OAAiB,KAClB/sC,EAAOwtC,GAAsB,MAAOh4F,KAAKi3F,YAAYhvC,MAAOjoD,KAAKo+C,IAAI05C,eAC3EttC,EAAKliB,UAAYA,EACjBkiB,EAAKkB,UAAY,GAEG,SAAhBrD,EACFmC,EAAKh2B,MAAMkgB,eAAU10C,KAAK4oC,MAAQpsB,QAGlCguC,EAAKh2B,MAAMmgB,gBAAW30C,KAAK4oC,MAAQpsB,QAGrCguC,EAAKh2B,MAAMoU,gBAAWA,QACtB4hB,EAAKh2B,MAAMsuB,cAASv5C,8CASX8+C,MACX2vC,GAAwBh4F,KAAKi3F,YAAYxnC,YAGDhwD,IAApCO,KAAKmvB,QAAQk5B,GAAaoH,YAAgEhwD,IAAzCO,KAAKmvB,QAAQk5B,GAAaoH,MAAMhE,KAAoB,KACjGgE,EAAQuoC,GAAsB,MAAOh4F,KAAKi3F,YAAYxnC,MAAOzvD,KAAKo+C,IAAIuqB,OAC5ElZ,EAAMnnB,6CAAwC+f,GAC9CoH,EAAM/D,UAAY1rD,KAAKmvB,QAAQk5B,GAAaoH,MAAMhE,UAGJhsD,IAA1CO,KAAKmvB,QAAQk5B,GAAaoH,MAAMj7B,OAClC8jB,GAAK+lB,WAAW5O,EAAOzvD,KAAKmvB,QAAQk5B,GAAaoH,MAAMj7B,OAGrC,SAAhB6zB,EACFoH,EAAMj7B,MAAMkgB,eAAU10C,KAAKq1C,MAAM4jD,sBAGjCxpC,EAAMj7B,MAAMmgB,gBAAW30C,KAAKq1C,MAAM4jD,sBAGpCxpC,EAAMj7B,MAAMoU,gBAAW5oC,KAAK6oC,aAI9BmvD,GAAwBh4F,KAAKi3F,YAAYxnC,yDAUnC,oBAAqBzvD,KAAKq1C,OAAQ,KAChCikD,EAAYlsE,SAAS06B,eAAe,KACpCgE,EAAmB1+B,SAASE,cAAc,OAChDw+B,EAAiBxjB,UAAY,mCAC7BwjB,EAAiBn3B,YAAY2kE,QACxBl7C,IAAIuqB,MAAMh0C,YAAYm3B,QAEtBzW,MAAM4T,gBAAkB6C,EAAiBC,kBACzC1W,MAAM2U,eAAiB8B,EAAiBnL,iBAExCvC,IAAIuqB,MAAMnhC,YAAYskB,QAGvB,oBAAqB9rD,KAAKq1C,OAAQ,KAChCkkD,EAAYnsE,SAAS06B,eAAe,KACpCkE,EAAmB5+B,SAASE,cAAc,OAChD0+B,EAAiB1jB,UAAY,mCAC7B0jB,EAAiBr3B,YAAY4kE,QACxBn7C,IAAIuqB,MAAMh0C,YAAYq3B,QAEtB3W,MAAM8T,gBAAkB6C,EAAiBD,kBACzC1W,MAAMmW,eAAiBQ,EAAiBrL,iBAExCvC,IAAIuqB,MAAMnhC,YAAYwkB,QAGvB,oBAAqBhsD,KAAKq1C,OAAQ,KAChCmkD,EAAYpsE,SAAS06B,eAAe,KACpC2xC,EAAmBrsE,SAASE,cAAc,OAChDmsE,EAAiBnxD,UAAY,mCAC7BmxD,EAAiB9kE,YAAY6kE,QACxBp7C,IAAIuqB,MAAMh0C,YAAY8kE,QAEtBpkD,MAAM4jD,gBAAkBQ,EAAiB1tC,kBACzC1W,MAAMqkD,eAAiBD,EAAiB94C,iBAExCvC,IAAIuqB,MAAMnhC,YAAYiyD,WA5mBV9gD,ICCvB,SAASghD,GAAOx9B,EAAShtC,IAiDzB,SAASyqE,GAAiBv1B,EAAOw1B,SAExB,CACLrlE,OAFFqlE,OAA4C,IAAnBA,EAAkC,GAAKA,GAExCrlE,OAAS6vC,EAAMl1C,QAAQqlC,WAAWhgC,MACxD6T,OAAQwxD,EAAexxD,QAAUg8B,EAAMl1C,QAAQqlC,WAAWnsB,OAC1D1I,KAAMk6D,EAAel6D,MAAQ0kC,EAAMl1C,QAAQqlC,WAAW70B,KACtD2I,UAAWuxD,EAAevxD,WAAa+7B,EAAM/7B,WCvDjD,SAASwxD,GAAS39B,EAAShtC,ICD3B,SAAS4qE,GAAK59B,EAAShtC,ICOvB,SAAS6qE,GAAW31B,EAAOlI,EAAShtC,EAAS8qE,QACtC7mE,GAAK+oC,OAELhtC,QAAUmpB,GAAK4hD,sBADP,CAAC,WAAY,QAAS,OAAQ,mBAAoB,WAAY,aAAc,SAAU,gBAAiB,SAAS,sBAAuB,qBAClG/qE,QAC7CgrE,uBAAwC16F,IAApB4kE,EAAM/7B,eAC1B2xD,yBAA2BA,OAC3BG,aAAe,OACfjkD,OAAOkuB,GACkB,GAA1BrkE,KAAKm6F,yBACFF,yBAAyB,IAAM,QAEjCx0B,UAAY,QACZnK,aAA4B77D,IAAlB4kE,EAAM/I,SAA+B+I,EAAM/I,QCb5D,SAAS++B,GAAOzhD,EAAMzpB,EAASmrE,EAAM9D,QAC9B59C,KAAOA,OACPsE,eAAiB,CACpBw/B,SAAS,EACT+Z,OAAO,EACP8D,SAAU,GACVC,YAAa,EACb9lD,KAAM,CACJ4mB,SAAS,EACT57B,SAAU,YAEZiV,MAAO,CACL2mB,SAAS,EACT57B,SAAU,mBAIT46D,KAAOA,OACPnrE,QAAUmpB,GAAKh7C,OAAO,GAAI0C,KAAKk9C,qBAC/Bs5C,iBAAmBA,OAEnBiB,YAAc,QACdr5C,IAAM,QACNuuB,OAAS,QACTgrB,eAAiB,OACjBjvC,eACAmvC,UAAY,CAACtB,IAAKv2F,KAAKu2F,IAAKkB,YAAaz3F,KAAKy3F,YAAatoE,QAASnvB,KAAKmvB,QAASw9C,OAAQ3sE,KAAK2sE,aAE/FnuB,WAAWrvB,GJtBlBwqE,GAAOjK,KAAO,SAAU5W,EAASzU,EAAOwzB,EAAWr7E,GACjDA,EAASA,GAAU,UACfxW,EAoDN,SAAqB6xF,EAAWxzB,OAC1Br+D,OAAWvG,EAEXo4F,EAAU1oE,SAAW0oE,EAAU1oE,QAAQqlC,YAAcqjC,EAAU1oE,QAAQqlC,WAAWC,UAA4D,mBAAzCojC,EAAU1oE,QAAQqlC,WAAWC,WACpIzuD,EAAW6xF,EAAU1oE,QAAQqlC,WAAWC,UAItC4P,EAAMA,MAAMl1C,SAAWk1C,EAAMA,MAAMl1C,QAAQqlC,YAAc6P,EAAMA,MAAMl1C,QAAQqlC,WAAWC,UAA8D,mBAA3C4P,EAAMA,MAAMl1C,QAAQqlC,WAAWC,WAC5IzuD,EAAWq+D,EAAMA,MAAMl1C,QAAQqlC,WAAWC,iBAErCzuD,EA/DQy0F,CAAY5C,EAAWxzB,GAE7BvnE,EAAI,EAAGA,EAAIg8E,EAAQ97E,OAAQF,OAC7BkJ,EAIA,KACC6zF,EAAiB7zF,EAAS8yE,EAAQh8E,GAAIunE,IACnB,IAAnBw1B,GAAqD,WAA1Bp0D,GAAOo0D,IACpC7B,GAAkBlf,EAAQh8E,GAAG49F,SAAWl+E,EAAQs8D,EAAQh8E,GAAG69F,SAAUf,GAAiBv1B,EAAOw1B,GAAiBhC,EAAUJ,YAAaI,EAAUtB,IAAKzd,EAAQh8E,GAAGyrC,YALjKyvD,GAAkBlf,EAAQh8E,GAAG49F,SAAWl+E,EAAQs8D,EAAQh8E,GAAG69F,SAAUf,GAAiBv1B,GAAQwzB,EAAUJ,YAAaI,EAAUtB,IAAKzd,EAAQh8E,GAAGyrC,QAWrJoxD,GAAOiB,SAAW,SAAUv2B,EAAOl6D,EAAGZ,EAAGutF,EAAWsB,EAAYP,OAC1DgD,EAA0B,GAAbzC,EAEb0C,EAAU9C,GAAsB,OAAQH,EAAUJ,YAAaI,EAAUtB,KAC7EuE,EAAQ1yD,eAAe,KAAM,IAAKj+B,GAClC2wF,EAAQ1yD,eAAe,KAAM,IAAK7+B,EAAIsxF,GACtCC,EAAQ1yD,eAAe,KAAM,QAAS0uD,GACtCgE,EAAQ1yD,eAAe,KAAM,SAAU,EAAIyyD,GAC3CC,EAAQ1yD,eAAe,KAAM,QAAS,eAGtC4vD,GAAkB7tF,EAAI,GAAM2sF,EAAWvtF,EAAGqwF,GAAiBv1B,GAAQwzB,EAAUJ,YAAaI,EAAUtB,MCrCtGuD,GAASc,SAAW,SAAUv2B,EAAOl6D,EAAGZ,EAAGutF,EAAWsB,EAAYP,OAC5DgD,EAA0B,GAAbzC,EACb0C,EAAU9C,GAAsB,OAAQH,EAAUJ,YAAaI,EAAUtB,KAC7EuE,EAAQ1yD,eAAe,KAAM,IAAKj+B,GAClC2wF,EAAQ1yD,eAAe,KAAM,IAAK7+B,EAAIsxF,GACtCC,EAAQ1yD,eAAe,KAAM,QAAS0uD,GACtCgE,EAAQ1yD,eAAe,KAAM,SAAU,EAAIyyD,GAC3CC,EAAQ1yD,eAAe,KAAM,QAAS,mBAElC2yD,EAAWv5F,KAAK+a,MAAM,GAAMu6E,GAE5Bn7C,EADgB0oB,EAAMl1C,QAAQ6rE,SAASpyD,MACfmyD,EACxBE,EAAaz5F,KAAK+a,MAAM,GAAM67E,GAC9B8C,EAAa15F,KAAK+a,MAAM,IAAO67E,GAE/B57E,EAAShb,KAAK+a,OAAOu6E,EAAa,EAAIiE,GAAa,MAEvD/C,GAAgB7tF,EAAI,GAAM4wF,EAAWv+E,EAAQjT,EAAIsxF,EAAaI,EAAa,EAAGF,EAAUE,EAAY52B,EAAM/7B,UAAY,WAAYuvD,EAAUJ,YAAaI,EAAUtB,IAAKlyB,EAAM7vC,OAC9KwjE,GAAgB7tF,EAAI,IAAM4wF,EAAWv+E,EAAS,EAAGjT,EAAIsxF,EAAaK,EAAa,EAAGH,EAAUG,EAAY72B,EAAM/7B,UAAY,WAAYuvD,EAAUJ,YAAaI,EAAUtB,IAAKlyB,EAAM7vC,OAE1I,GAApC6vC,EAAMl1C,QAAQqlC,WAAWkoB,QAAiB,KACxCx0C,EAAgB,CAClB1T,MAAO6vC,EAAMl1C,QAAQqlC,WAAWhgC,MAChC6T,OAAQg8B,EAAMl1C,QAAQqlC,WAAWnsB,OACjC1I,KAAO0kC,EAAMl1C,QAAQqlC,WAAW70B,KAAOgc,EACvCrT,UAAW+7B,EAAM/7B,WAEnB0vD,GAAkB7tF,EAAI,GAAM4wF,EAAWv+E,EAAQjT,EAAIsxF,EAAaI,EAAa,EAAG/yD,EAAe2vD,EAAUJ,YAAaI,EAAUtB,KAChIyB,GAAkB7tF,EAAI,IAAM4wF,EAAWv+E,EAAS,EAAGjT,EAAIsxF,EAAaK,EAAa,EAAGhzD,EAAe2vD,EAAUJ,YAAaI,EAAUtB,OAWxIuD,GAASpK,KAAO,SAAU1e,EAAUmqB,EAAoBtD,OAGlDuD,EACAn4F,EAAKo4F,EACLh3B,EACAvnE,EAAGiX,EALHunF,EAAe,GACfC,EAAgB,GAKhBC,EAAY,MAGX1+F,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,OAEH,SAD5BunE,EAAQwzB,EAAUlrB,OAAOqE,EAASl0E,KACxBqyB,QAAQqF,QACM,IAAlB6vC,EAAM/I,eAA0E77D,IAArDo4F,EAAU1oE,QAAQw9C,OAAO1tB,WAAW+xB,EAASl0E,MAA0E,IAArD+6F,EAAU1oE,QAAQw9C,OAAO1tB,WAAW+xB,EAASl0E,SACvIiX,EAAI,EAAGA,EAAIonF,EAAmBnqB,EAASl0E,IAAIE,OAAQ+W,IACtDunF,EAAar+F,KAAK,CAChBy9F,SAAUS,EAAmBnqB,EAASl0E,IAAIiX,GAAG2mF,SAC7Ce,WAAYN,EAAmBnqB,EAASl0E,IAAIiX,GAAG0nF,WAC/Cd,SAAUQ,EAAmBnqB,EAASl0E,IAAIiX,GAAG4mF,SAC7CxwF,EAAGgxF,EAAmBnqB,EAASl0E,IAAIiX,GAAG5J,EACtCs2B,IAAK06D,EAAmBnqB,EAASl0E,IAAIiX,GAAG0sB,IACxCl3B,EAAG4xF,EAAmBnqB,EAASl0E,IAAIiX,GAAGxK,EACtC4yD,QAAS6U,EAASl0E,GAClByrC,MAAO4yD,EAAmBnqB,EAASl0E,IAAIiX,GAAGw0B,QAE5CizD,GAAa,KAMH,IAAdA,SAKJF,QAAAA,GAAkB,SAAUn+F,EAAGC,UACzBD,EAAEu9F,WAAat9F,EAAEs9F,SACZv9F,EAAEg/D,QAAU/+D,EAAE++D,SAAW,EAAI,EAG7Bh/D,EAAEu9F,SAAWt9F,EAAEs9F,YAK1BZ,GAAS4B,sBAAsBH,EAAeD,GAGzCx+F,EAAI,EAAGA,EAAIw+F,EAAat+F,OAAQF,IAAK,KAEpC8uE,EAA8CnsE,OADlD4kE,EAAQwzB,EAAUlrB,OAAO2uB,EAAax+F,GAAGq/D,UACpBhtC,QAAQ6rE,SAASpvB,SAAwBvH,EAAMl1C,QAAQ6rE,SAASpvB,SAAW,GAAMvH,EAAMl1C,QAAQ6rE,SAASpyD,MAGzH+yD,EAAe,UACQl8F,IAAvB87F,EAFJt4F,EAAMq4F,EAAax+F,GAAG49F,UAGhB59F,EAAI,EAAIw+F,EAAat+F,SACvBo+F,EAAe55F,KAAKc,IAAIg5F,EAAax+F,EAAI,GAAG49F,SAAWz3F,IAEzDo4F,EAAWvB,GAAS8B,iBAAiBR,EAAc/2B,EAAOuH,OAEvD,KACCzmB,EAAUroD,GAAKy+F,EAAct4F,GAAK44F,OAASN,EAAct4F,GAAK64F,UAC9D32C,EAAUm2C,EAAat+F,SACzBo+F,EAAe55F,KAAKc,IAAIg5F,EAAan2C,GAASu1C,SAAWz3F,IAE3Do4F,EAAWvB,GAAS8B,iBAAiBR,EAAc/2B,EAAOuH,GAC1D2vB,EAAct4F,GAAK64F,UAAY,GAEH,IAAxBz3B,EAAMl1C,QAAQ9rB,QAAwD,IAAtCghE,EAAMl1C,QAAQ4sE,oBAC5CT,EAAax+F,GAAG69F,SAAWt2B,EAAM+1B,cACnCuB,EAAeJ,EAAct4F,GAAK+4F,oBAClCT,EAAct4F,GAAK+4F,qBAAuB33B,EAAM+1B,aAAekB,EAAax+F,GAAG69F,WAG/EgB,EAAeJ,EAAct4F,GAAKg5F,oBAClCV,EAAct4F,GAAKg5F,qBAAuB53B,EAAM+1B,aAAekB,EAAax+F,GAAG69F,WAGpC,IAAtCt2B,EAAMl1C,QAAQ6rE,SAASkB,aAC9Bb,EAASzyD,MAAQyyD,EAASzyD,MAAQ2yD,EAAct4F,GAAK44F,OACrDR,EAAS7+E,QAAW++E,EAAct4F,GAAK64F,SAAYT,EAASzyD,MAAS,GAAMyyD,EAASzyD,OAAS2yD,EAAct4F,GAAK44F,OAAS,QAIzHM,EAAYd,EAASzyD,MACrBpI,EAAQ86D,EAAax+F,GAAG49F,YAGMj7F,MAA9B67F,EAAax+F,GAAG2+F,WAElBj7D,GAAsB,IADtB27D,EAAYb,EAAax+F,GAAG2+F,WAAaH,EAAax+F,GAAG49F,UAIzDl6D,GAAS66D,EAAS7+E,OAGpBw7E,GAAgBx3D,EAAO86D,EAAax+F,GAAG69F,SAAWgB,EAAcQ,EAAW93B,EAAM+1B,aAAekB,EAAax+F,GAAG69F,SAAUt2B,EAAM/7B,UAAY,WAAYuvD,EAAUJ,YAAaI,EAAUtB,IAAKlyB,EAAM7vC,QAG3J,IAArC6vC,EAAMl1C,QAAQqlC,WAAWkoB,QAAkB,KACzC0f,EAAY,CACd1B,SAAUY,EAAax+F,GAAG49F,SAC1BC,SAAUW,EAAax+F,GAAG69F,SAAWgB,EACrCxxF,EAAGmxF,EAAax+F,GAAGqN,EACnBZ,EAAG+xF,EAAax+F,GAAGyM,EACnB4yD,QAASm/B,EAAax+F,GAAGq/D,QACzB5zB,MAAO+yD,EAAax+F,GAAGyrC,OAEzBoxD,GAAOjK,KAAK,CAAC0M,GAAY/3B,EAAOwzB,EAAWwD,EAAS7+E,WAa1Ds9E,GAAS4B,sBAAwB,SAAUH,EAAeD,WAEpDF,EACKt+F,EAAI,EAAGA,EAAIw+F,EAAat+F,OAAQF,IACnCA,EAAI,EAAIw+F,EAAat+F,SACvBo+F,EAAe55F,KAAKc,IAAIg5F,EAAax+F,EAAI,GAAG49F,SAAWY,EAAax+F,GAAG49F,WAErE59F,EAAI,IACNs+F,EAAe55F,KAAKY,IAAIg5F,EAAc55F,KAAKc,IAAIg5F,EAAax+F,EAAI,GAAG49F,SAAWY,EAAax+F,GAAG49F,YAE3E,IAAjBU,SAC8C37F,IAA5C87F,EAAcD,EAAax+F,GAAG49F,YAChCa,EAAcD,EAAax+F,GAAG49F,UAAY,CACxCmB,OAAQ,EACRC,SAAU,EACVG,oBAAqB,EACrBD,oBAAqB,IAGzBT,EAAcD,EAAax+F,GAAG49F,UAAUmB,QAAU,IAexD/B,GAAS8B,iBAAmB,SAAUR,EAAc/2B,EAAOuH,OACrDhjC,EAAOpsB,SACP4+E,EAAe/2B,EAAMl1C,QAAQ6rE,SAASpyD,OAASwyD,EAAe,GAChExyD,EAAQwyD,EAAexvB,EAAWA,EAAWwvB,EAE7C5+E,EAAS,EAC4B,SAAjC6nD,EAAMl1C,QAAQ6rE,SAASv0B,MACzBjqD,GAAU,GAAM4+E,EAEwB,UAAjC/2B,EAAMl1C,QAAQ6rE,SAASv0B,QAC9BjqD,GAAU,GAAM4+E,KAKlBxyD,EAAQy7B,EAAMl1C,QAAQ6rE,SAASpyD,MAC/BpsB,EAAS,EAC4B,SAAjC6nD,EAAMl1C,QAAQ6rE,SAASv0B,MACzBjqD,GAAU,GAAM6nD,EAAMl1C,QAAQ6rE,SAASpyD,MAEC,UAAjCy7B,EAAMl1C,QAAQ6rE,SAASv0B,QAC9BjqD,GAAU,GAAM6nD,EAAMl1C,QAAQ6rE,SAASpyD,QAIpC,CAACA,MAAOA,EAAOpsB,OAAQA,IAGhCs9E,GAASuC,iBAAmB,SAAUf,EAAcgB,EAAatrB,EAAUurB,EAAYl0C,MACjFizC,EAAat+F,OAAS,EAAG,IAE3Bs+F,QAAAA,GAAkB,SAAUn+F,EAAGC,UACzBD,EAAEu9F,WAAat9F,EAAEs9F,SACZv9F,EAAEg/D,QAAU/+D,EAAE++D,SAAW,EAAI,EAG7Bh/D,EAAEu9F,SAAWt9F,EAAEs9F,gBAGtBa,EAAgB,GAEpBzB,GAAS4B,sBAAsBH,EAAeD,GAC9CgB,EAAYC,GAAczC,GAAS0C,kBAAkBjB,EAAeD,GACpEgB,EAAYC,GAAYE,iBAAmBp0C,EAC3C2oB,EAAS/zE,KAAKs/F,KAIlBzC,GAAS0C,kBAAoB,SAAUjB,EAAeD,WAChDr4F,EACAy5F,EAAOpB,EAAa,GAAGX,SACvBgC,EAAOrB,EAAa,GAAGX,SAClB79F,EAAI,EAAGA,EAAIw+F,EAAat+F,OAAQF,SAEZ2C,IAAvB87F,EADJt4F,EAAMq4F,EAAax+F,GAAG49F,WAEpBgC,EAAOA,EAAOpB,EAAax+F,GAAG69F,SAAWW,EAAax+F,GAAG69F,SAAW+B,EACpEC,EAAOA,EAAOrB,EAAax+F,GAAG69F,SAAWW,EAAax+F,GAAG69F,SAAWgC,GAGhErB,EAAax+F,GAAG69F,SAAW,EAC7BY,EAAct4F,GAAK+4F,qBAAuBV,EAAax+F,GAAG69F,SAG1DY,EAAct4F,GAAKg5F,qBAAuBX,EAAax+F,GAAG69F,aAI3D,IAAIiC,KAAQrB,EACXA,EAAcl+F,eAAeu/F,KAE/BF,GADAA,EAAOA,EAAOnB,EAAcqB,GAAMZ,oBAAsBT,EAAcqB,GAAMZ,oBAAsBU,GACpFnB,EAAcqB,GAAMX,oBAAsBV,EAAcqB,GAAMX,oBAAsBS,EAElGC,GADAA,EAAOA,EAAOpB,EAAcqB,GAAMZ,oBAAsBT,EAAcqB,GAAMZ,oBAAsBW,GACpFpB,EAAcqB,GAAMX,oBAAsBV,EAAcqB,GAAMX,oBAAsBU,SAI/F,CAACv6F,IAAKs6F,EAAMp3F,IAAKq3F,IChR1B5C,GAAK8C,SAAW,SAAU/jB,EAASzU,MAChB,MAAXyU,GACIA,EAAQ97E,OAAS,EAAG,QAIuB,GAAvCqnE,EAAMl1C,QAAQ2tE,cAAcpgB,QACxBqd,GAAKgD,YAAYjkB,EAASzU,GAG1B01B,GAAKiD,QAAQlkB,KAOjCihB,GAAKa,SAAW,SAAUv2B,EAAOl6D,EAAGZ,EAAGutF,EAAWsB,EAAYP,OAEtD5nE,EAAMgtE,EADNpC,EAA0B,GAAbzC,EAGb0C,EAAU9C,GAAsB,OAAQH,EAAUJ,YAAaI,EAAUtB,MAC7EuE,EAAQ1yD,eAAe,KAAM,IAAKj+B,GAClC2wF,EAAQ1yD,eAAe,KAAM,IAAK7+B,EAAIsxF,GACtCC,EAAQ1yD,eAAe,KAAM,QAAS0uD,GACtCgE,EAAQ1yD,eAAe,KAAM,SAAU,EAAIyyD,GAC3CC,EAAQ1yD,eAAe,KAAM,QAAS,gBAEtCnY,EAAO+nE,GAAsB,OAAQH,EAAUJ,YAAaI,EAAUtB,MACjEnuD,eAAe,KAAM,QAASi8B,EAAM/7B,gBACrB7oC,IAAhB4kE,EAAM7vC,OACNvE,EAAKmY,eAAe,KAAM,QAASi8B,EAAM7vC,OAG7CvE,EAAKmY,eAAe,KAAM,IAAK,IAAMj+B,EAAI,IAAMZ,EAAI,MAAQY,EAAI2sF,GAAa,IAAMvtF,GAC9C,GAAhC86D,EAAMl1C,QAAQ+tE,OAAOxgB,UACrBugB,EAAWjF,GAAsB,OAAQH,EAAUJ,YAAaI,EAAUtB,KAClC,OAApClyB,EAAMl1C,QAAQ+tE,OAAO70C,YACrB40C,EAAS70D,eAAe,KAAM,IAAK,IAAMj+B,EAAI,MAAQZ,EAAIsxF,GACvD,IAAM1wF,EAAI,IAAMZ,EAAI,MAAQY,EAAI2sF,GAAa,IAAMvtF,EAAI,MAAQY,EAAI2sF,GAAa,KAAOvtF,EAAIsxF,IAG7FoC,EAAS70D,eAAe,KAAM,IAAK,IAAMj+B,EAAI,IAAMZ,EAAI,KAC/CY,EAAI,KAAOZ,EAAIsxF,GAAc,MAC5B1wF,EAAI2sF,GAAa,KAAOvtF,EAAIsxF,GACnC,KAAO1wF,EAAI2sF,GAAa,IAAMvtF,GAEpC0zF,EAAS70D,eAAe,KAAM,QAASi8B,EAAM/7B,UAAY,uBACtB7oC,IAA/B4kE,EAAMl1C,QAAQ+tE,OAAO1oE,OAAsD,KAA/B6vC,EAAMl1C,QAAQ+tE,OAAO1oE,OACjEyoE,EAAS70D,eAAe,KAAM,QAASi8B,EAAMl1C,QAAQ+tE,OAAO1oE,QAI5B,GAApC6vC,EAAMl1C,QAAQqlC,WAAWkoB,UAOzBsb,GAAkB7tF,EAAI,GAAM2sF,EAAWvtF,EANnB,CAChBirB,MAAO6vC,EAAMl1C,QAAQqlC,WAAWhgC,MAChC6T,OAAQg8B,EAAMl1C,QAAQqlC,WAAWnsB,OACjC1I,KAAM0kC,EAAMl1C,QAAQqlC,WAAW70B,KAC/B2I,UAAW+7B,EAAM/7B,WAEoCuvD,EAAUJ,YAAaI,EAAUtB,MAIlGwD,GAAKoD,YAAc,SAAUC,EAAW/4B,EAAOg5B,EAAcxF,MAErB,GAAhCxzB,EAAMl1C,QAAQ+tE,OAAOxgB,QAAiB,KAOlC4gB,EANAC,EAAYlmD,OAAOwgD,EAAUtB,IAAI/hE,MAAMqU,OAAOtiC,QAAQ,KAAK,KAC3D02F,EAAWjF,GAAsB,OAAQH,EAAUJ,YAAaI,EAAUtB,KAC1Er/D,EAAO,IACgC,GAAvCmtC,EAAMl1C,QAAQ2tE,cAAcpgB,UAC5BxlD,EAAO,SAGPsmE,EAAO,EAEPA,EADoC,OAApCn5B,EAAMl1C,QAAQ+tE,OAAO70C,YACd,EAEkC,UAApCgc,EAAMl1C,QAAQ+tE,OAAO70C,YACnBk1C,EAGA/7F,KAAKY,IAAIZ,KAAK8D,IAAI,EAAG++D,EAAM+1B,cAAemD,GAGjDD,EADoC,SAApCj5B,EAAMl1C,QAAQ+tE,OAAO70C,aAA2C,MAAhBg1C,GAAwC59F,MAAhB49F,EAChE,IAAMD,EAAU,GAAG,GAAI,IAAIA,EAAU,GAAG,GAAK,IAC7Cp9F,KAAKy9F,cAAcL,EAAUlmE,GAAK,GAClC,KAAMmmE,EAAaA,EAAargG,OAAO,GAAG,GAAI,IAAMqgG,EAAaA,EAAargG,OAAO,GAAG,GAAK,IAC7FgD,KAAKy9F,cAAcJ,EAAanmE,GAAK,GACrCmmE,EAAa,GAAG,GAAI,IAAIA,EAAa,GAAG,GAAK,KAG7C,IAAMD,EAAU,GAAG,GAAI,IAAIA,EAAU,GAAG,GAAK,IAC7Cp9F,KAAKy9F,cAAcL,EAAUlmE,GAAK,GAClC,KAAOsmE,EAAO,KAAMJ,EAAU,GAAG,GAAK,KAGlDH,EAAS70D,eAAe,KAAM,QAASi8B,EAAM/7B,UAAY,kBACtB7oC,IAA/B4kE,EAAMl1C,QAAQ+tE,OAAO1oE,OACrByoE,EAAS70D,eAAe,KAAM,QAASi8B,EAAMl1C,QAAQ+tE,OAAO1oE,OAEhEyoE,EAAS70D,eAAe,KAAM,IAAKk1D,KAW3CvD,GAAKrK,KAAO,SAAU0N,EAAW/4B,EAAOwzB,MACnB,MAAbuF,GAAkC39F,MAAb29F,EAAwB,KACzCntE,EAAO+nE,GAAsB,OAAQH,EAAUJ,YAAaI,EAAUtB,KAC1EtmE,EAAKmY,eAAe,KAAM,QAASi8B,EAAM/7B,gBACrB7oC,IAAhB4kE,EAAM7vC,OACNvE,EAAKmY,eAAe,KAAM,QAASi8B,EAAM7vC,WAGzC0C,EAAO,IACgC,GAAvCmtC,EAAMl1C,QAAQ2tE,cAAcpgB,UAC5BxlD,EAAO,KAGXjH,EAAKmY,eAAe,KAAM,IAAK,IAAMg1D,EAAU,GAAG,GAAI,IAAIA,EAAU,GAAG,GAAK,IAAMp9F,KAAKy9F,cAAcL,EAAUlmE,GAAK,MAI5H6iE,GAAK0D,cAAgB,SAASL,EAAUlmE,EAAKwmE,MACrCN,EAAUpgG,OAAS,QAEZ,OAGPF,EADAyP,EAAI2qB,KAEJwmE,MACK5gG,EAAIsgG,EAAUpgG,OAAO,EAAGF,EAAI,EAAGA,IAChCyP,GAAK6wF,EAAUtgG,GAAG,GAAK,IAAMsgG,EAAUtgG,GAAG,GAAK,aAI9CA,EAAI,EAAGA,EAAIsgG,EAAUpgG,OAAQF,IAC9ByP,GAAK6wF,EAAUtgG,GAAG,GAAK,IAAMsgG,EAAUtgG,GAAG,GAAK,WAGhDyP,GAUXwtF,GAAK4D,mBAAqB,SAAUpqF,OAE5BqqF,EAAIn1F,EAAIC,EAAIC,EAAIk1F,EAAKC,EACrBvxF,EAAI,GACRA,EAAEtP,KAAM,CAAEuE,KAAK+a,MAAMhJ,EAAK,GAAGmnF,UAAYl5F,KAAK+a,MAAMhJ,EAAK,GAAGonF,oBAExD39F,EAASuW,EAAKvW,OACTF,EAAI,EAAGA,EAAIE,EAAS,EAAGF,IAE5B8gG,EAAW,GAAL9gG,EAAUyW,EAAK,GAAKA,EAAKzW,EAAI,GACnC2L,EAAK8K,EAAKzW,GACV4L,EAAK6K,EAAKzW,EAAI,GACd6L,EAAM7L,EAAI,EAAIE,EAAUuW,EAAKzW,EAAI,GAAK4L,EAUtCm1F,EAAM,CACFnD,WAAakD,EAAGlD,SAAW,EAAIjyF,EAAGiyF,SAAWhyF,EAAGgyF,WAlBpC,EAAI,GAmBhBC,WAAaiD,EAAGjD,SAAW,EAAIlyF,EAAGkyF,SAAWjyF,EAAGiyF,WAnBpC,EAAI,IAqBpBmD,EAAM,CACFpD,UAAajyF,EAAGiyF,SAAW,EAAIhyF,EAAGgyF,SAAW/xF,EAAG+xF,WAtBpC,EAAI,GAuBhBC,UAAalyF,EAAGkyF,SAAW,EAAIjyF,EAAGiyF,SAAWhyF,EAAGgyF,WAvBpC,EAAI,IA2BpBpuF,EAAEtP,KAAM,CAAE4gG,EAAInD,SAAWmD,EAAIlD,WAC7BpuF,EAAEtP,KAAM,CAAE6gG,EAAIpD,SAAWoD,EAAInD,WAC7BpuF,EAAEtP,KAAM,CAAEyL,EAAGgyF,SAAYhyF,EAAGiyF,kBAGzBpuF,GAcXwtF,GAAKgD,YAAc,SAAUxpF,EAAM8wD,OAC3BojB,EAAQpjB,EAAMl1C,QAAQ2tE,cAAcrV,SAC3B,GAATA,QAAwBhoF,IAAVgoF,SACPznF,KAAK29F,mBAAmBpqF,OAG3BqqF,EAAIn1F,EAAIC,EAAIC,EAAIk1F,EAAKC,EAAKC,EAAIC,EAAIC,EAAIh7D,EAAGoC,EAAG64D,EAAGzxF,EAC/C0xF,EAAQC,EAAQC,EAASC,EAASC,EAASC,EAC3CjyF,EAAI,GACRA,EAAEtP,KAAM,CAAEuE,KAAK+a,MAAMhJ,EAAK,GAAGmnF,UAAYl5F,KAAK+a,MAAMhJ,EAAK,GAAGonF,oBACxD39F,EAASuW,EAAKvW,OACTF,EAAI,EAAGA,EAAIE,EAAS,EAAGF,IAE5B8gG,EAAW,GAAL9gG,EAAUyW,EAAK,GAAKA,EAAKzW,EAAI,GACnC2L,EAAK8K,EAAKzW,GACV4L,EAAK6K,EAAKzW,EAAI,GACd6L,EAAM7L,EAAI,EAAIE,EAAUuW,EAAKzW,EAAI,GAAK4L,EAEtCq1F,EAAKv8F,KAAKyqF,KAAKzqF,KAAK6D,IAAIu4F,EAAGlD,SAAWjyF,EAAGiyF,SAAU,GAAKl5F,KAAK6D,IAAIu4F,EAAGjD,SAAWlyF,EAAGkyF,SAAU,IAC5FqD,EAAKx8F,KAAKyqF,KAAKzqF,KAAK6D,IAAIoD,EAAGiyF,SAAWhyF,EAAGgyF,SAAU,GAAKl5F,KAAK6D,IAAIoD,EAAGkyF,SAAWjyF,EAAGiyF,SAAU,IAC5FsD,EAAKz8F,KAAKyqF,KAAKzqF,KAAK6D,IAAIqD,EAAGgyF,SAAW/xF,EAAG+xF,SAAU,GAAKl5F,KAAK6D,IAAIqD,EAAGiyF,SAAWhyF,EAAGgyF,SAAU,IAY5FwD,EAAS38F,KAAK6D,IAAI44F,EAAIxW,GACtB4W,EAAU78F,KAAK6D,IAAI44F,EAAI,EAAIxW,GAC3B2W,EAAS58F,KAAK6D,IAAI24F,EAAIvW,GACtB6W,EAAU98F,KAAK6D,IAAI24F,EAAI,EAAIvW,GAC3B+W,EAASh9F,KAAK6D,IAAI04F,EAAItW,GAGtBxkD,EAAI,GAFJs7D,EAAU/8F,KAAK6D,IAAI04F,EAAI,EAAItW,IAET,EAAI+W,EAASJ,EAASE,EACxCj5D,EAAI,EAAIg5D,EAAU,EAAIF,EAASC,EAASE,GACxCJ,EAAI,EAAIM,GAAUA,EAASJ,IACnB,IACJF,EAAI,EAAIA,IAEZzxF,EAAI,EAAI0xF,GAAUA,EAASC,IACnB,IACJ3xF,EAAI,EAAIA,GAGZoxF,EAAM,CACFnD,WAAa4D,EAAUV,EAAGlD,SAAWz3D,EAAIx6B,EAAGiyF,SAAW6D,EAAU71F,EAAGgyF,UAAYwD,EAChFvD,WAAa2D,EAAUV,EAAGjD,SAAW13D,EAAIx6B,EAAGkyF,SAAW4D,EAAU71F,EAAGiyF,UAAYuD,GAGpFJ,EAAM,CACFpD,UAAa2D,EAAU51F,EAAGiyF,SAAWr1D,EAAI38B,EAAGgyF,SAAW4D,EAAU31F,EAAG+xF,UAAYjuF,EAChFkuF,UAAa0D,EAAU51F,EAAGkyF,SAAWt1D,EAAI38B,EAAGiyF,SAAW2D,EAAU31F,EAAGgyF,UAAYluF,GAGhE,GAAhBoxF,EAAInD,UAAiC,GAAhBmD,EAAIlD,WACzBkD,EAAMp1F,GAEU,GAAhBq1F,EAAIpD,UAAiC,GAAhBoD,EAAInD,WACzBmD,EAAMp1F,GAEV6D,EAAEtP,KAAM,CAAE4gG,EAAInD,SAAWmD,EAAIlD,WAC7BpuF,EAAEtP,KAAM,CAAE6gG,EAAIpD,SAAWoD,EAAInD,WAC7BpuF,EAAEtP,KAAM,CAAEyL,EAAGgyF,SAAYhyF,EAAGiyF,kBAGzBpuF,GAUfwtF,GAAKiD,QAAU,SAAUzpF,WAEjBhH,EAAI,GACCzP,EAAI,EAAGA,EAAIyW,EAAKvW,OAAQF,IAC7ByP,EAAEtP,KAAK,CAAEsW,EAAKzW,GAAG49F,SAAWnnF,EAAKzW,GAAG69F,kBAEjCpuF,GC/QXytF,GAAW79F,UAAUg5D,SAAW,SAAU3e,GAC3B,MAATA,QACGivB,UAAYjvB,EACQ,GAArB6pB,QAAKlxC,UACPmpB,GAAKmmD,WAAWz+F,KAAKylE,WAAU,SAAUtoE,EAAGC,UACnCD,EAAEgN,EAAI/M,EAAE+M,EAAI,GAAK,WAKvBs7D,UAAY,IAIrBu0B,GAAW79F,UAAUuiG,SAAW,kBACvB1+F,KAAKylE,WAOdu0B,GAAW79F,UAAUwiG,gBAAkB,SAAU7tF,QAC1CspF,aAAetpF,GAOtBkpF,GAAW79F,UAAUqiD,WAAa,SAAUrvB,WAC1B1vB,IAAZ0vB,EAAuB,CAEzBmpB,GAAKqQ,oBADQ,CAAC,WAAY,QAAS,OAAQ,mBAAoB,WAAY,SAAS,sBAAuB,qBAC1E3oD,KAAKmvB,QAASA,GAGd,mBAAtBA,EAAQqlC,aACjBrlC,EAAQqlC,WAAa,CACnBC,SAAUtlC,EAAQqlC,aAItBlc,GAAKsmD,aAAa5+F,KAAKmvB,QAASA,EAAS,iBACzCmpB,GAAKsmD,aAAa5+F,KAAKmvB,QAASA,EAAS,cACzCmpB,GAAKsmD,aAAa5+F,KAAKmvB,QAASA,EAAS,UAErCA,EAAQ2tE,eAC0B,UAAhCr3D,GAAOtW,EAAQ2tE,gBACb3tE,EAAQ2tE,cAAc+B,kBACqB,WAAzC1vE,EAAQ2tE,cAAc+B,qBACnB1vE,QAAQ2tE,cAAcrV,MAAQ,EAEa,WAAzCt4D,EAAQ2tE,cAAc+B,qBACxB1vE,QAAQ2tE,cAAcrV,MAAQ,QAG9Bt4D,QAAQ2tE,cAAc+B,gBAAkB,mBACxC1vE,QAAQ2tE,cAAcrV,MAAQ,OAa/CuS,GAAW79F,UAAUg6C,OAAS,SAAUkuB,QACjCA,MAAQA,OACRxwC,QAAUwwC,EAAMxwC,SAAW,aAC3ByU,UAAY+7B,EAAM/7B,WAAatoC,KAAKsoC,WAAa,kBAAoBtoC,KAAKi6F,yBAAyB,GAAK,QACxG3+B,aAA4B77D,IAAlB4kE,EAAM/I,SAA+B+I,EAAM/I,aACrD9mC,MAAQ6vC,EAAM7vC,WACdgqB,WAAW6lB,EAAMl1C,UAaxB6qE,GAAW79F,UAAUg8F,UAAY,SAAUrB,EAAWsB,EAAYP,EAAW1tF,EAAGZ,GAC7D9J,MAAbo4F,GAAuC,MAAbA,IAE5BA,EAAY,CAACtB,IADHnpE,SAASya,gBAAgB,6BAA8B,OAC1C4vD,YAAY,GAAItoE,QAASnvB,KAAKmvB,QAASw9C,OAAQ,CAAC3sE,eAEhEP,MAAL0K,GAAuB,MAALA,IACpBA,EAAI,GAEG1K,MAAL8J,GAAuB,MAALA,IACpBA,EAAI,GAAM6uF,GAEJp4F,KAAKmvB,QAAQqF,WACd,OACHsqE,GAAMlE,SAAS56F,KAAMmK,EAAGZ,EAAGutF,EAAWsB,EAAYP,aAE/C,aACA,QACH8B,GAAOiB,SAAS56F,KAAMmK,EAAGZ,EAAGutF,EAAWsB,EAAYP,aAEhD,MACHkH,GAAKnE,SAAS56F,KAAMmK,EAAGZ,EAAGutF,EAAWsB,EAAYP,SAG9C,CAACmH,KAAMnH,EAAUtB,IAAKhuD,MAAOvoC,KAAK6zB,QAASw0B,YAAaroD,KAAKmvB,QAAQstE,mBAG9EzC,GAAW79F,UAAU8iG,UAAY,SAAUxuB,WACrCisB,EAAOjsB,EAAU,GAAGlnE,EACpBozF,EAAOlsB,EAAU,GAAGlnE,EACfwK,EAAI,EAAGA,EAAI08D,EAAUzzE,OAAQ+W,IACpC2oF,EAAOA,EAAOjsB,EAAU18D,GAAGxK,EAAIknE,EAAU18D,GAAGxK,EAAImzF,EAChDC,EAAOA,EAAOlsB,EAAU18D,GAAGxK,EAAIknE,EAAU18D,GAAGxK,EAAIozF,QAE3C,CAACv6F,IAAKs6F,EAAMp3F,IAAKq3F,EAAMF,iBAAkBz8F,KAAKmvB,QAAQstE,mBC/G/DpC,GAAOl+F,UAAY,IAAIw8C,GAEvB0hD,GAAOl+F,UAAUixC,MAAQ,gBAClBu/B,OAAS,QACTgrB,eAAiB,GAGxB0C,GAAOl+F,UAAU+iG,SAAW,SAAS32D,EAAOwvD,GAGI,GAA1CA,EAAa5oE,QAAQgwE,oBAClBn/F,KAAK2sE,OAAOtvE,eAAekrC,UACzBokC,OAAOpkC,GAASwvD,QAElBJ,gBAAkB,IAI3B0C,GAAOl+F,UAAUkqE,YAAc,SAAS99B,EAAOwvD,QACxCprB,OAAOpkC,GAASwvD,GAGvBsC,GAAOl+F,UAAUijG,YAAc,SAAS72D,GAClCvoC,KAAK2sE,OAAOtvE,eAAekrC,YACtBvoC,KAAK2sE,OAAOpkC,QACdovD,gBAAkB,IAI3B0C,GAAOl+F,UAAUusD,QAAU,gBACpBtK,IAAIuqB,MAAQv7C,SAASE,cAAc,YACnC8wB,IAAIuqB,MAAMrgC,UAAY,kBACtB8V,IAAIuqB,MAAMn0C,MAAMkL,SAAW,gBAC3B0e,IAAIuqB,MAAMn0C,MAAMsuB,IAAM,YACtB1E,IAAIuqB,MAAMn0C,MAAMC,QAAU,aAE1B2pB,IAAIihD,SAAWjyE,SAASE,cAAc,YACtC8wB,IAAIihD,SAAS/2D,UAAY,uBACzB8V,IAAIihD,SAAS7qE,MAAMkL,SAAW,gBAC9B0e,IAAIihD,SAAS7qE,MAAMsuB,IAAM,WAEzByzC,IAAMnpE,SAASya,gBAAgB,6BAA6B,YAC5D0uD,IAAI/hE,MAAMkL,SAAW,gBACrB62D,IAAI/hE,MAAMsuB,IAAM,WAChByzC,IAAI/hE,MAAMoU,MAAQ5oC,KAAKmvB,QAAQorE,SAAW,EAAI,UAC9ChE,IAAI/hE,MAAMqU,OAAS,YAEnBuV,IAAIuqB,MAAMh0C,YAAY30B,KAAKu2F,UAC3Bn4C,IAAIuqB,MAAMh0C,YAAY30B,KAAKo+C,IAAIihD,WAMtChF,GAAOl+F,UAAU+zD,KAAO,WAElBlwD,KAAKo+C,IAAIuqB,MAAMphC,iBACZ6W,IAAIuqB,MAAMphC,WAAWC,YAAYxnC,KAAKo+C,IAAIuqB,QAOnD0xB,GAAOl+F,UAAUqnE,KAAO,WAEjBxjE,KAAKo+C,IAAIuqB,MAAMphC,iBACbqR,KAAKwF,IAAIY,OAAOrqB,YAAY30B,KAAKo+C,IAAIuqB,QAI9C0xB,GAAOl+F,UAAUqiD,WAAa,SAASrvB,GAErCmpB,GAAKqQ,oBADQ,CAAC,UAAU,cAAc,QAAQ,OAAO,SACpB3oD,KAAKmvB,QAASA,IAGjDkrE,GAAOl+F,UAAUk0D,OAAS,eACpBgoC,EAAe,EACfH,EAAal4C,GAAYhgD,KAAK2sE,WAClCurB,QAAAA,GAAgB,SAAU/6F,EAAEC,UAClBD,EAAIC,GAAK,EAAI,SAGlB,IAAIN,EAAI,EAAGA,EAAIo7F,EAAWl7F,OAAQF,IAAK,KACtCq/D,EAAU+7B,EAAWp7F,GACW,GAAhCkD,KAAK2sE,OAAOxQ,GAASb,cAAkE77D,IAA9CO,KAAKw2F,iBAAiBv3C,WAAWkd,IAAuE,GAA7Cn8D,KAAKw2F,iBAAiBv3C,WAAWkd,IACvIk8B,OAImC,GAAnCr4F,KAAKmvB,QAAQnvB,KAAKs6F,MAAMh/B,SAA2C,GAAvBt7D,KAAK23F,gBAA+C,GAAxB33F,KAAKmvB,QAAQutD,SAAoC,GAAhB2b,OACtGnoC,WAEF,SACEsT,OACmC,YAApCxjE,KAAKmvB,QAAQnvB,KAAKs6F,MAAM56D,UAA8D,eAApC1/B,KAAKmvB,QAAQnvB,KAAKs6F,MAAM56D,eACvE0e,IAAIuqB,MAAMn0C,MAAMkgB,KAAO,WACvB0J,IAAIuqB,MAAMn0C,MAAM4kE,UAAY,YAC5Bh7C,IAAIihD,SAAS7qE,MAAM4kE,UAAY,YAC/Bh7C,IAAIihD,SAAS7qE,MAAMkgB,KAAQ10C,KAAKmvB,QAAQorE,SAAW,GAAM,UACzDn8C,IAAIihD,SAAS7qE,MAAMmgB,MAAQ,QAC3B4hD,IAAI/hE,MAAMkgB,KAAO,WACjB6hD,IAAI/hE,MAAMmgB,MAAQ,UAGlByJ,IAAIuqB,MAAMn0C,MAAMmgB,MAAQ,WACxByJ,IAAIuqB,MAAMn0C,MAAM4kE,UAAY,aAC5Bh7C,IAAIihD,SAAS7qE,MAAM4kE,UAAY,aAC/Bh7C,IAAIihD,SAAS7qE,MAAMmgB,MAAS30C,KAAKmvB,QAAQorE,SAAW,GAAM,UAC1Dn8C,IAAIihD,SAAS7qE,MAAMkgB,KAAO,QAC1B6hD,IAAI/hE,MAAMmgB,MAAQ,WAClB4hD,IAAI/hE,MAAMkgB,KAAO,IAGgB,YAApC10C,KAAKmvB,QAAQnvB,KAAKs6F,MAAM56D,UAA8D,aAApC1/B,KAAKmvB,QAAQnvB,KAAKs6F,MAAM56D,cACvE0e,IAAIuqB,MAAMn0C,MAAMsuB,IAAM,EAAIzL,OAAOr3C,KAAK44C,KAAKwF,IAAIY,OAAOxqB,MAAMsuB,IAAIv8C,QAAQ,KAAK,KAAO,UACpF63C,IAAIuqB,MAAMn0C,MAAMq0B,OAAS,OAE3B,KACCy2C,EAAmBt/F,KAAK44C,KAAKY,SAASwF,OAAOnW,OAAS7oC,KAAK44C,KAAKY,SAASC,gBAAgB5Q,YACxFuV,IAAIuqB,MAAMn0C,MAAMq0B,OAAS,EAAIy2C,EAAmBjoD,OAAOr3C,KAAK44C,KAAKwF,IAAIY,OAAOxqB,MAAMsuB,IAAIv8C,QAAQ,KAAK,KAAO,UAC1G63C,IAAIuqB,MAAMn0C,MAAMsuB,IAAM,GAGH,GAAtB9iD,KAAKmvB,QAAQsnE,YACVr4C,IAAIuqB,MAAMn0C,MAAMoU,MAAQ5oC,KAAKo+C,IAAIihD,SAASj2C,YAAc,GAAK,UAC7DhL,IAAIihD,SAAS7qE,MAAMmgB,MAAQ,QAC3ByJ,IAAIihD,SAAS7qE,MAAMkgB,KAAO,QAC1B6hD,IAAI/hE,MAAMoU,MAAQ,aAGlBwV,IAAIuqB,MAAMn0C,MAAMoU,MAAQ5oC,KAAKmvB,QAAQorE,SAAW,GAAKv6F,KAAKo+C,IAAIihD,SAASj2C,YAAc,GAAK,UAC1Fm2C,uBAGH1rE,EAAU,OACT/2B,EAAI,EAAGA,EAAIo7F,EAAWl7F,OAAQF,IACjCq/D,EAAU+7B,EAAWp7F,GACe,GAAhCkD,KAAK2sE,OAAOxQ,GAASb,cAAkE77D,IAA9CO,KAAKw2F,iBAAiBv3C,WAAWkd,IAAuE,GAA7Cn8D,KAAKw2F,iBAAiBv3C,WAAWkd,KACvItoC,GAAW7zB,KAAK2sE,OAAOxQ,GAAStoC,QAAU,eAGzCuqB,IAAIihD,SAAS3zC,UAAY73B,OACzBuqB,IAAIihD,SAAS7qE,MAAM2zC,WAAe,IAAOnoE,KAAKmvB,QAAQorE,SAAYv6F,KAAKmvB,QAAQqrE,YAAe,OAIvGH,GAAOl+F,UAAUojG,gBAAkB,cAC7Bv/F,KAAKo+C,IAAIuqB,MAAMphC,WAAY,KACzB2wD,EAAal4C,GAAYhgD,KAAK2sE,WAClCurB,QAAAA,GAAgB,SAAU/6F,EAAEC,UAClBD,EAAIC,GAAK,EAAI,KAIvB46F,GAAsBh4F,KAAKy3F,iBAEvBztB,EAAUh+C,OAAO+8C,iBAAiB/oE,KAAKo+C,IAAIuqB,OAAO62B,WAClDvH,EAAa5gD,OAAO2yB,EAAQzjE,QAAQ,KAAK,KACzC4D,EAAI8tF,EACJnB,EAAY92F,KAAKmvB,QAAQorE,SACzBnC,EAAa,IAAOp4F,KAAKmvB,QAAQorE,SACjChxF,EAAI0uF,EAAa,GAAMG,EAAa,OAEnC7B,IAAI/hE,MAAMoU,MAAQkuD,EAAY,EAAImB,EAAa,SAE/C,IAAIn7F,EAAI,EAAGA,EAAIo7F,EAAWl7F,OAAQF,IAAK,KACtCq/D,EAAU+7B,EAAWp7F,GACW,GAAhCkD,KAAK2sE,OAAOxQ,GAASb,cAAkE77D,IAA9CO,KAAKw2F,iBAAiBv3C,WAAWkd,IAAuE,GAA7Cn8D,KAAKw2F,iBAAiBv3C,WAAWkd,UAClIwQ,OAAOxQ,GAASg8B,UAAUrB,EAAWsB,EAAYp4F,KAAK63F,UAAW1tF,EAAGZ,GACzEA,GAAK6uF,EAAap4F,KAAKmvB,QAAQqrE,gBChMvC,SAASiF,GAAU7mD,EAAMzpB,QAClBiE,GAAKs3C,UACL9xB,KAAOA,OAEPsE,eAAiB,CACpBu/C,iBAAkB,OAClBiD,aAAc,UACdzzF,MAAM,EACN0zF,UAAU,EACVt8F,OAAO,EACPu8F,YAAa,QACb1C,OAAQ,CACNxgB,SAAS,EACTr0B,YAAa,UAEf7zB,MAAO,OACPwmE,SAAU,CACRpyD,MAAO,GACPszD,YAAY,EACZz1B,MAAO,UAETq2B,cAAe,CACbpgB,SAAS,EACTmiB,gBAAiB,cACjBpX,MAAO,IAETjzB,WAAY,CACVkoB,SAAS,EACT/8C,KAAM,EACNnL,MAAO,UAETqrE,SAAU,GACVC,OAAQ,GACRnzB,OAAQ,CACN1tB,WAAY,UAKX9vB,QAAUmpB,GAAKh7C,OAAO,GAAI0C,KAAKk9C,qBAC/BkB,IAAM,QACN/I,MAAQ,QACRwO,OAAS,UACT8oB,OAAS,QACTozB,oBAAqB,OACrBC,iBAAkB,OAClBC,yBAA0B,OAC1BC,kBAAmB,MAEpBvhD,EAAK3+C,UACJylE,UAAY,UACZsK,WAAa,UAGbE,cAAgB,KACZ,SAAUzvB,EAAOD,EAAQ2vB,GAC9BvxB,EAAGwxB,OAAO5vB,EAAO/J,eAET,SAAUgK,EAAOD,EAAQ2vB,GACjCvxB,EAAG0xB,UAAU9vB,EAAO/J,eAEZ,SAAUgK,EAAOD,EAAQ2vB,GACjCvxB,EAAG2xB,UAAU/vB,EAAO/J,cAKnB+5B,eAAiB,KACb,SAAU/vB,EAAOD,EAAQ2vB,GAC9BvxB,EAAG6xB,aAAajwB,EAAO/J,eAEf,SAAUgK,EAAOD,EAAQ2vB,GACjCvxB,EAAGmyB,gBAAgBvwB,EAAO/J,eAElB,SAAUgK,EAAOD,EAAQ2vB,GACjCvxB,EAAGoyB,gBAAgBxwB,EAAO/J,cAIzBA,MAAQ,QACRy6B,UAAY,QACZkvB,UAAYngG,KAAK44C,KAAKc,MAAMlZ,WAC5B8kC,YAAc,QAEdmyB,YAAc,QACdj5C,WAAWrvB,QACX8qE,yBAA2B,CAAC,QAC5BrhD,KAAKiF,QAAQvH,GAAG,gBAAgB,WACnCqI,EAAGwhD,UAAYxhD,EAAG/F,KAAKc,MAAMlZ,MAC7Bme,EAAG43C,IAAI/hE,MAAMkgB,KAAO4D,GAAKge,OAAOC,QAAQ5X,EAAGtJ,MAAMzM,OAEjD+V,EAAGuhD,kBAAmB,EAEtBvhD,EAAG0R,OAAOh0D,KAAKsiD,WAIZ+J,eACAmvC,UAAY,CAACtB,IAAKv2F,KAAKu2F,IAAKkB,YAAaz3F,KAAKy3F,YAAatoE,QAASnvB,KAAKmvB,QAASw9C,OAAQ3sE,KAAK2sE,QAGtG8yB,GAAUtjG,UAAY,IAAIw8C,GAK1B8mD,GAAUtjG,UAAUusD,QAAU,eACxBigB,EAAQv7C,SAASE,cAAc,OACnCq7C,EAAMrgC,UAAY,sBACb8V,IAAIuqB,MAAQA,OAGZ4tB,IAAMnpE,SAASya,gBAAgB,6BAA8B,YAC7D0uD,IAAI/hE,MAAMkL,SAAW,gBACrB62D,IAAI/hE,MAAMqU,QAAU,GAAK7oC,KAAKmvB,QAAQywE,aAAar5F,QAAQ,KAAM,IAAM,UACvEgwF,IAAI/hE,MAAMC,QAAU,QACzBk0C,EAAMh0C,YAAY30B,KAAKu2F,UAGlBpnE,QAAQ0wE,SAASx3C,YAAc,YAC/B+3C,UAAY,IAAI9J,GAASt2F,KAAK44C,KAAM54C,KAAKmvB,QAAQ0wE,SAAU7/F,KAAKu2F,IAAKv2F,KAAKmvB,QAAQw9C,aAElFx9C,QAAQ0wE,SAASx3C,YAAc,aAC/Bg4C,WAAa,IAAI/J,GAASt2F,KAAK44C,KAAM54C,KAAKmvB,QAAQ0wE,SAAU7/F,KAAKu2F,IAAKv2F,KAAKmvB,QAAQw9C,eACjF3sE,KAAKmvB,QAAQ0wE,SAASx3C,iBAGxBi4C,WAAa,IAAIjG,GAAOr6F,KAAK44C,KAAM54C,KAAKmvB,QAAQ2wE,OAAQ,OAAQ9/F,KAAKmvB,QAAQw9C,aAC7E4zB,YAAc,IAAIlG,GAAOr6F,KAAK44C,KAAM54C,KAAKmvB,QAAQ2wE,OAAQ,QAAS9/F,KAAKmvB,QAAQw9C,aAE/EnJ,QAOPi8B,GAAUtjG,UAAUqiD,WAAa,SAAUrvB,MACrCA,EAAS,MAEiB1vB,IAAxB0vB,EAAQywE,kBAAgDngG,IAAnB0vB,EAAQ0Z,aAC1Cm3D,iBAAkB,OAClBC,yBAA0B,QAEsBxgG,IAA9CO,KAAK44C,KAAKY,SAASC,gBAAgB5Q,aAAgDppC,IAAxB0vB,EAAQywE,aACtEz0C,IAAUh8B,EAAQywE,YAAc,IAAIr5F,QAAQ,KAAM,KAAOvG,KAAK44C,KAAKY,SAASC,gBAAgB5Q,cACzFm3D,iBAAkB,GAG3B1nD,GAAKqQ,oBAVQ,CAAC,WAAY,eAAgB,QAAS,SAAU,cAAe,mBAAoB,QAAS,WAAY,WAAY,OAAQ,UAUxG3oD,KAAKmvB,QAASA,GAC/CmpB,GAAKsmD,aAAa5+F,KAAKmvB,QAASA,EAAS,iBACzCmpB,GAAKsmD,aAAa5+F,KAAKmvB,QAASA,EAAS,cACzCmpB,GAAKsmD,aAAa5+F,KAAKmvB,QAASA,EAAS,UACzCmpB,GAAKsmD,aAAa5+F,KAAKmvB,QAASA,EAAS,UAErCA,EAAQ2tE,eAC0B,UAAhCr3D,GAAOtW,EAAQ2tE,gBACb3tE,EAAQ2tE,cAAc+B,kBACqB,WAAzC1vE,EAAQ2tE,cAAc+B,qBACnB1vE,QAAQ2tE,cAAcrV,MAAQ,EAEa,WAAzCt4D,EAAQ2tE,cAAc+B,qBACxB1vE,QAAQ2tE,cAAcrV,MAAQ,QAG9Bt4D,QAAQ2tE,cAAc+B,gBAAkB,mBACxC1vE,QAAQ2tE,cAAcrV,MAAQ,KAMvCznF,KAAKogG,gBACkB3gG,IAArB0vB,EAAQ0wE,gBACLO,UAAU5hD,WAAWx+C,KAAKmvB,QAAQ0wE,eAClCQ,WAAW7hD,WAAWx+C,KAAKmvB,QAAQ0wE,WAIxC7/F,KAAKsgG,iBACgB7gG,IAAnB0vB,EAAQ2wE,cACLQ,WAAW9hD,WAAWx+C,KAAKmvB,QAAQ2wE,aACnCS,YAAY/hD,WAAWx+C,KAAKmvB,QAAQ2wE,SAIzC9/F,KAAK2sE,OAAOtvE,eApMJ,uBAqMLsvE,OAAL,cAAuBnuB,WAAWrvB,GAKlCnvB,KAAKo+C,IAAIuqB,aACNu3B,kBAAiB,OACjBtnD,KAAKiF,QAAQ4C,KAAK,UAAU,CAAC4R,OAAO,MAO7CotC,GAAUtjG,UAAU+zD,KAAO,WAErBlwD,KAAKo+C,IAAIuqB,MAAMphC,iBACZ6W,IAAIuqB,MAAMphC,WAAWC,YAAYxnC,KAAKo+C,IAAIuqB,QAQnD82B,GAAUtjG,UAAUqnE,KAAO,WAEpBxjE,KAAKo+C,IAAIuqB,MAAMphC,iBACbqR,KAAKwF,IAAIY,OAAOrqB,YAAY30B,KAAKo+C,IAAIuqB,QAS9C82B,GAAUtjG,UAAUg5D,SAAW,SAAU3e,OAErC+7B,EADE5zB,EAAK3+C,KAEPq0E,EAAer0E,KAAKylE,aAGjBjvB,EAGA,CAAA,KAAIA,aAAiBoB,IAAWpB,aAAiB89B,UAI9C,IAAItnD,UAAU,wDAHfy4C,UAAYluB,GAAkBf,aAH9BivB,UAAY,QASf4O,OAEF/7B,SAAAA,GAAat4C,KAAKiwE,eAAe,SAAUjqE,EAAUw6C,GACnD6zB,EAAa99B,IAAIiK,EAAOx6C,MAI1BquE,EAAah8B,UAGbk6B,EAAM8B,EAAaj8B,cACdk4B,UAAUiC,IAGbvyE,KAAKylE,UAAW,KAEdryC,EAAKpzB,KAAKozB,MACdklB,SAAAA,GAAat4C,KAAKiwE,eAAe,SAAUjqE,EAAUw6C,GACnD7B,EAAG8mB,UAAUnvB,GAAGkK,EAAOx6C,EAAUotB,MAInCm/C,EAAMvyE,KAAKylE,UAAUrtB,cAChB+3B,OAAOoC,KAShBktB,GAAUtjG,UAAUi5D,UAAY,SAAUuX,OAEpC4F,EADA5zB,EAAK3+C,QAILA,KAAK+vE,WAAY,IACnBz3B,SAAAA,GAAat4C,KAAKuwE,gBAAgB,SAAUvqE,EAAUw6C,GACpD7B,EAAGoxB,WAAWx5B,IAAIiK,EAAOx6C,MAI3BusE,EAAMvyE,KAAK+vE,WAAW33B,cACjB23B,WAAa,SACb,IAAIjzE,EAAI,EAAGA,EAAIy1E,EAAIv1E,OAAQF,SACzB0jG,aAAajuB,EAAIz1E,OAKrB6vE,EAGA,CAAA,KAAIA,aAAkB/0B,IAAW+0B,aAAkB2H,UAIhD,IAAItnD,UAAU,wDAHf+iD,WAAapD,YAHboD,WAAa,QAShB/vE,KAAK+vE,WAAY,KAEf38C,EAAKpzB,KAAKozB,MACdklB,SAAAA,GAAat4C,KAAKuwE,gBAAgB,SAAUvqE,EAAUw6C,GACpD7B,EAAGoxB,WAAWz5B,GAAGkK,EAAOx6C,EAAUotB,MAIpCm/C,EAAMvyE,KAAK+vE,WAAW33B,cACjBo4B,aAAa+B,KAItBktB,GAAUtjG,UAAUk0E,UAAY,SAAUkC,QACnCkuB,oBAAoBluB,IAE3BktB,GAAUtjG,UAAUg0E,OAAS,SAAUoC,QAChClC,UAAUkC,IAEjBktB,GAAUtjG,UAAUm0E,UAAY,SAAUiC,QACnClC,UAAUkC,IAEjBktB,GAAUtjG,UAAU20E,gBAAkB,SAAUE,QACzCyvB,oBAAoB,KAAMzvB,IAEjCyuB,GAAUtjG,UAAUq0E,aAAe,SAAUQ,QACtCF,gBAAgBE,IAQvByuB,GAAUtjG,UAAU40E,gBAAkB,SAAUC,OACzC,IAAIl0E,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,SAC9B0jG,aAAaxvB,EAASl0E,SAExBojG,kBAAmB,OACnBtnD,KAAKiF,QAAQ4C,KAAK,UAAU,CAAC4R,OAAO,KAQ3CotC,GAAUtjG,UAAUqkG,aAAe,SAAUrkC,GACvCn8D,KAAK2sE,OAAOtvE,eAAe8+D,KACwB,SAAjDn8D,KAAK2sE,OAAOxQ,GAAShtC,QAAQstE,uBAC1B4D,WAAWjB,YAAYjjC,QACvBokC,YAAYnB,YAAYjjC,QACxBokC,YAAYlwC,gBAGZ+vC,UAAUhB,YAAYjjC,QACtBmkC,WAAWlB,YAAYjjC,QACvBmkC,WAAWjwC,iBAEXrwD,KAAK2sE,OAAOxQ,KAWvBsjC,GAAUtjG,UAAUukG,aAAe,SAAUr8B,EAAOlI,GAC7Cn8D,KAAK2sE,OAAOtvE,eAAe8+D,SAYzBwQ,OAAOxQ,GAAShmB,OAAOkuB,GACyB,SAAjDrkE,KAAK2sE,OAAOxQ,GAAShtC,QAAQstE,uBAC1B4D,WAAWh6B,YAAYlK,EAASn8D,KAAK2sE,OAAOxQ,SAC5CokC,YAAYl6B,YAAYlK,EAASn8D,KAAK2sE,OAAOxQ,SAE7CikC,UAAUhB,YAAYjjC,QACtBmkC,WAAWlB,YAAYjjC,UAGvBikC,UAAU/5B,YAAYlK,EAASn8D,KAAK2sE,OAAOxQ,SAC3CmkC,WAAWj6B,YAAYlK,EAASn8D,KAAK2sE,OAAOxQ,SAE5CkkC,WAAWjB,YAAYjjC,QACvBokC,YAAYnB,YAAYjjC,WAxB1BwQ,OAAOxQ,GAAW,IAAI69B,GAAW31B,EAAOlI,EAASn8D,KAAKmvB,QAASnvB,KAAKi6F,0BACpB,SAAjDj6F,KAAK2sE,OAAOxQ,GAAShtC,QAAQstE,uBAC1B4D,WAAWnB,SAAS/iC,EAASn8D,KAAK2sE,OAAOxQ,SACzCokC,YAAYrB,SAAS/iC,EAASn8D,KAAK2sE,OAAOxQ,WAG1CikC,UAAUlB,SAAS/iC,EAASn8D,KAAK2sE,OAAOxQ,SACxCmkC,WAAWpB,SAAS/iC,EAASn8D,KAAK2sE,OAAOxQ,WAoB7CmkC,WAAWjwC,cACXkwC,YAAYlwC,UAWnBovC,GAAUtjG,UAAUskG,oBAAsB,SAAUluB,EAAKvB,MACjC,MAAlBhxE,KAAKylE,UAAmB,KACtBk7B,EAAgB,GAChBnqD,EAAQx2C,KAAKylE,UAAU17D,MACvB8tC,EAAU73C,KAAKylE,UAAUhuB,OACzBmpD,EAAQ,GACRruB,MACFA,QAAAA,GAAQ,SAAUn/C,GAChBwtE,EAAMxtE,GAAMA,aAKZytE,EAAc,GACT/jG,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,IAAK,KACjCi7C,EAAOvB,EAAM15C,GACbq/D,EAAUpkB,EAAKssB,MACflI,MAAAA,IACFA,EAxbQ,iBA0bV0kC,EAAYxjG,eAAe8+D,GAAW0kC,EAAY1kC,KAAa0kC,EAAY1kC,GAAW,MAIpF2kC,EAAmB,OAClB9vB,GAAYuB,MACVpW,KAAWn8D,KAAK2sE,UACf3sE,KAAK2sE,OAAOtvE,eAAe8+D,GAAU,KAEnC4kC,GADJ18B,EAAQrkE,KAAK2sE,OAAOxQ,IACOuiC,WAE3BiC,EAAcxkC,GAAWrlB,GAAAiqD,QAAAA,GAAsB,SAAUhpD,UACvD+oD,EAAiB/oD,EAAKF,IAAYE,EAAKF,GAC/BE,EAAKF,KAAa+oD,EAAM7oD,EAAKF,WAEnCmpD,EAAYH,EAAY1kC,GAC5B0kC,EAAY1kC,IAAYwkC,EAAcxkC,GAASn/D,OAC3C2jG,EAAcxkC,GAASn/D,OAASgkG,IAClCL,EAAcxkC,GAAS6kC,EAAY,GAAK,QAO3ClkG,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,OAGxBq/D,OADJA,GADApkB,EAAOvB,EAAM15C,IACEunE,SAEblI,EAvdQ,iBAydL6U,IAAYuB,GAAQx6B,EAAKF,KAAa+oD,EAAM7oD,EAAKF,MAAcipD,EAAiBzjG,eAAe06C,EAAKF,KAGpG8oD,EAActjG,eAAe8+D,KAChCwkC,EAAcxkC,GAAW,IAAIlgE,MAAM4kG,EAAY1kC,SAG7C8kC,EAAW3oD,GAAK4oD,aAAanpD,GACjCkpD,EAAS92F,EAAImuC,GAAKpB,QAAQa,EAAK5tC,EAAG,QAClC82F,EAASxgE,IAAM6X,GAAKpB,QAAQa,EAAKtX,IAAK,QACtCwgE,EAASE,SAAWppD,EAAKxuC,EACzB03F,EAAS13F,EAAI8tC,OAAOU,EAAKxuC,GACzB03F,EAASppD,GAAWE,EAAKF,OAErBjzB,EAAO+7E,EAAcxkC,GAASn/D,OAAS6jG,EAAY1kC,KACvDwkC,EAAcxkC,GAASv3C,GAASq8E,MAI7B9kC,KAAWn8D,KAAK2sE,OACf3sE,KAAK2sE,OAAOtvE,eAAe8+D,KACxBwkC,EAActjG,eAAe8+D,KAChCwkC,EAAcxkC,GAAW,IAAIlgE,MAAM,SAMpCkgE,KAAWwkC,KACVA,EAActjG,eAAe8+D,MACM,GAAjCwkC,EAAcxkC,GAASn/D,OACrBgD,KAAK2sE,OAAOtvE,eAAe8+D,SACxBqkC,aAAarkC,OAEf,KACDkI,OAAQ5kE,EACWA,MAAnBO,KAAK+vE,aACP1L,EAAQrkE,KAAK+vE,WAAWhmE,IAAIoyD,IAEjB18D,MAAT4kE,IACFA,EAAQ,CAACjxC,GAAI+oC,EAAStoC,QAAS7zB,KAAKmvB,QAAQuwE,aAAevjC,SAExDukC,aAAar8B,EAAOlI,QACpBwQ,OAAOxQ,GAAShH,SAASwrC,EAAcxkC,SAI7C+jC,kBAAmB,OACnBtnD,KAAKiF,QAAQ4C,KAAK,UAAU,CAAC4R,OAAO,MAQ7CotC,GAAUtjG,UAAUk0D,OAAS,eACvBxX,GAAU,OAGTxD,MAAMzM,MAAQ5oC,KAAKo+C,IAAIuqB,MAAMvf,iBAC7B/T,MAAMxM,OAAS7oC,KAAK44C,KAAKY,SAASC,gBAAgB5Q,OACnD7oC,KAAK44C,KAAKY,SAASmY,OAAO7O,IAC1B9iD,KAAK44C,KAAKY,SAASmY,OAAO9I,OAG9BhQ,EAAU74C,KAAK8pD,cAAgBjR,QAG3Bo6B,EAAkBjzE,KAAK44C,KAAKc,MAAMjZ,IAAMzgC,KAAK44C,KAAKc,MAAMlZ,MACxD0yC,EAAUD,GAAmBjzE,KAAKmzE,0BACjCA,oBAAsBF,EAKZ,GAAXp6B,UACG09C,IAAI/hE,MAAMoU,MAAQ0P,GAAKge,OAAOC,OAAO,EAAIv2D,KAAKq1C,MAAMzM,YACpD2tD,IAAI/hE,MAAMkgB,KAAO4D,GAAKge,OAAOC,QAAQv2D,KAAKq1C,MAAMzM,QAGL,GAA5CgrB,KAAC5zD,KAAKmvB,QAAQ0Z,OAAS,WAAY,MAA8C,GAAhC7oC,KAAKigG,+BACnDD,iBAAkB,OAKC,GAAxBhgG,KAAKggG,iBACHhgG,KAAKmvB,QAAQywE,aAAe5/F,KAAKq1C,MAAMxM,OAAS,YAC7C1Z,QAAQywE,YAAc5/F,KAAKq1C,MAAMxM,OAAS,UAC1C0tD,IAAI/hE,MAAMqU,OAAS7oC,KAAKq1C,MAAMxM,OAAS,WAEzCm3D,iBAAkB,QAGlBzJ,IAAI/hE,MAAMqU,QAAU,GAAK7oC,KAAKmvB,QAAQywE,aAAar5F,QAAQ,KAAM,IAAM,KAI/D,GAAXsyC,GAA6B,GAAVq6B,GAA6C,GAA3BlzE,KAAK+/F,oBAAuD,GAAzB//F,KAAKkgG,iBAC/ErnD,EAAU74C,KAAKohG,gBAAkBvoD,OAC5BqnD,kBAAmB,UAIF,GAAlBlgG,KAAKmgG,UAAgB,KACnB3jF,EAASxc,KAAK44C,KAAKc,MAAMlZ,MAAQxgC,KAAKmgG,UACtCzmD,EAAQ15C,KAAK44C,KAAKc,MAAMjZ,IAAMzgC,KAAK44C,KAAKc,MAAMlZ,SAC1B,GAApBxgC,KAAKq1C,MAAMzM,MAAY,KAErBJ,EAAUhsB,GADSxc,KAAKq1C,MAAMzM,MAAQ8Q,QAErC68C,IAAI/hE,MAAMkgB,MAAS10C,KAAKq1C,MAAMzM,MAAQJ,EAAW,kBAIvD83D,WAAWjwC,cACXkwC,YAAYlwC,SACVxX,GAIT4mD,GAAUtjG,UAAUklG,mBAAqB,eAEnCC,EAAY,OACX,IAAInlC,KAAWn8D,KAAK2sE,UACnB3sE,KAAK2sE,OAAOtvE,eAAe8+D,GAAU,KACnCkI,EAAQrkE,KAAK2sE,OAAOxQ,GACH,GAAjBkI,EAAM/I,cAAgE77D,IAA5CO,KAAKmvB,QAAQw9C,OAAO1tB,WAAWkd,IAAqE,GAA3Cn8D,KAAKmvB,QAAQw9C,OAAO1tB,WAAWkd,IACpHmlC,EAAUrkG,KAAK,CAACm2B,GAAG+oC,EAAQolC,OAAOl9B,EAAMl1C,QAAQoyE,SAItDjpD,GAAKmmD,WAAW6C,GAAU,SAASnkG,EAAEC,OAC/BokG,EAAKrkG,EAAEokG,OACPE,EAAKrkG,EAAEmkG,mBACA9hG,IAAP+hG,IAAkBA,EAAG,QACd/hG,IAAPgiG,IAAkBA,EAAG,GAClBD,GAAIC,EAAI,EAAID,EAAGC,GAAM,EAAG,aAE7BzwB,EAAW,IAAI/0E,MAAMqlG,EAAUtkG,QAC1BF,EAAE,EAAGA,EAAGwkG,EAAUtkG,OAAQF,IACjCk0E,EAASl0E,GAAKwkG,EAAUxkG,GAAGs2B,UAEtB49C,GASTyuB,GAAUtjG,UAAUilG,aAAe,cAEjCpJ,GAAwBh4F,KAAKy3F,aACL,GAApBz3F,KAAKq1C,MAAMzM,OAAgC,MAAlB5oC,KAAKylE,UAAmB,KAC/CpB,EAAOvnE,EACPw/F,EAAc,GAGdoF,EAAU1hG,KAAK44C,KAAKN,KAAK44C,cAAclxF,KAAK44C,KAAKY,SAAS2H,KAAKvY,OAC/D+4D,EAAU3hG,KAAK44C,KAAKN,KAAK44C,aAAa,EAAIlxF,KAAK44C,KAAKY,SAAS2H,KAAKvY,OAGlEooC,EAAWhxE,KAAKqhG,wBAChBrwB,EAASh0E,OAAS,EAAG,KACnB+yE,EAAa,YAGZ6xB,iBAAiB5wB,EAAUjB,EAAY2xB,EAASC,QAGhDE,eAAe7wB,EAAUjB,GAGzBjzE,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,SAC1BglG,qBAAqB/xB,EAAWiB,EAASl0E,aAI3CilG,YAAY/wB,EAAUjB,EAAYusB,GAOnB,GAJLt8F,KAAKgiG,aAAahxB,EAAUsrB,UAKzCtE,GAAwBh4F,KAAKy3F,kBACxBsI,oBAAqB,GACnB,OAEJA,oBAAqB,MAGtBkC,OAAQxiG,MACP3C,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,IAC/BunE,EAAQrkE,KAAK2sE,OAAOqE,EAASl0E,KACF,IAAvBkD,KAAKmvB,QAAQ9rB,OAAyC,SAAvBrD,KAAKmvB,QAAQqF,QACL/0B,MAArC4kE,EAAMl1C,QAAQ4sE,qBAAqC13B,EAAMl1C,QAAQ4sE,sBACtDt8F,MAATwiG,SACGC,OAAOnyB,EAAW1L,EAAMjxC,IAAK28C,EAAWkyB,EAAM7uE,KACf,GAAhCixC,EAAMl1C,QAAQ+tE,OAAOxgB,SAAwD,UAArCrY,EAAMl1C,QAAQ+tE,OAAO70C,cACvB,OAApCgc,EAAMl1C,QAAQ+tE,OAAO70C,aAA6D,UAArC45C,EAAM9yE,QAAQ+tE,OAAO70C,aACpE45C,EAAM9yE,QAAQ+tE,OAAO70C,YAAY,QACjC45C,EAAM9yE,QAAQ+tE,OAAO/gC,QAAQkI,EAAMjxC,KAEnCixC,EAAMl1C,QAAQ+tE,OAAO70C,YAAY,QACjCgc,EAAMl1C,QAAQ+tE,OAAO/gC,QAAQ8lC,EAAM7uE,MAIzC6uE,EAAQ59B,SAGP89B,qBAAqBpyB,EAAWiB,EAASl0E,IAAKunE,OAIjD+9B,EAAQ,OACPtlG,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,OAEH,UAD5BunE,EAAQrkE,KAAK2sE,OAAOqE,EAASl0E,KACnBqyB,QAAQqF,OAAoD,GAAhC6vC,EAAMl1C,QAAQ+tE,OAAOxgB,QAAiB,KACtE5D,EAAU/I,EAAWiB,EAASl0E,OACnB,MAAXg8E,GAAqC,GAAlBA,EAAQ97E,mBAG1BolG,EAAM/kG,eAAe2zE,EAASl0E,MACjCslG,EAAMpxB,EAASl0E,IAAMgiG,GAAMjC,SAAS/jB,EAASzU,IAEN,UAArCA,EAAMl1C,QAAQ+tE,OAAO70C,YAAyB,KAC5Cg6C,EAAah+B,EAAMl1C,QAAQ+tE,OAAO/gC,YACA,IAAlCvI,GAAAod,QAAAA,EAAiBqxB,GAAoB,CACvC1/F,QAAQq3C,IAAIqqB,EAAMjxC,GAAK,wCAA0CivE,YAG9DD,EAAM/kG,eAAeglG,KACxBD,EAAMC,GAAcvD,GAAMjC,SAAS9sB,EAAWsyB,GAAariG,KAAK2sE,OAAO01B,KAEzEvD,GAAM3B,YAAYiF,EAAMpxB,EAASl0E,IAAKunE,EAAO+9B,EAAMC,GAAariG,KAAK63F,gBAGrEiH,GAAM3B,YAAYiF,EAAMpxB,EAASl0E,IAAKunE,OAAO5kE,EAAWO,KAAK63F,eAMnEkH,GAAKrP,KAAK1e,EAAUjB,EAAY/vE,KAAK63F,WAChC/6F,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,OAC/BunE,EAAQrkE,KAAK2sE,OAAOqE,EAASl0E,IACzBizE,EAAWiB,EAASl0E,IAAIE,OAAS,SAC3BqnE,EAAMl1C,QAAQqF,WACf,OACE4tE,EAAM/kG,eAAe2zE,EAASl0E,MACjCslG,EAAMpxB,EAASl0E,IAAMgiG,GAAMjC,SAAS9sB,EAAWiB,EAASl0E,IAAKunE,IAE/Dy6B,GAAMpP,KAAK0S,EAAMpxB,EAASl0E,IAAKunE,EAAOrkE,KAAK63F,eAExC,YAEA,SACwB,SAAvBxzB,EAAMl1C,QAAQqF,OAA2C,UAAvB6vC,EAAMl1C,QAAQqF,OAAyD,GAApC6vC,EAAMl1C,QAAQqlC,WAAWkoB,SAChGid,GAAOjK,KAAK3f,EAAWiB,EAASl0E,IAAKunE,EAAOrkE,KAAK63F,oBAgB/DG,GAAwBh4F,KAAKy3F,cACtB,GAGTgI,GAAUtjG,UAAU+lG,OAAS,SAAU3uF,EAAM+uF,OACvC19E,EAAO29E,EAAIC,EAAIC,EAAcC,EACjC99E,EAAQ,MAEH,IAAI7Q,EAAI,EAAGA,EAAIR,EAAKvW,OAAQ+W,IAAK,CACpC0uF,OAAehjG,EACfijG,OAAejjG,MAEV,IAAIkb,EAAIiK,EAAOjK,EAAI2nF,EAAQtlG,OAAQ2d,IAAK,IAEvC2nF,EAAQ3nF,GAAGxQ,IAAMoJ,EAAKQ,GAAG5J,EAAG,CAC9Bs4F,EAAeH,EAAQ3nF,GACvB+nF,EAAeJ,EAAQ3nF,GACvBiK,EAAQjK,QAGL,GAAI2nF,EAAQ3nF,GAAGxQ,EAAIoJ,EAAKQ,GAAG5J,EAAG,CACjCu4F,EAAeJ,EAAQ3nF,GAErB8nF,EADO,GAAL9nF,EACa+nF,EAGAJ,EAAQ3nF,EAAI,GAE7BiK,EAAQjK,cAKSlb,IAAjBijG,IACFD,EAAeH,EAAQA,EAAQtlG,OAAS,GACxC0lG,EAAeJ,EAAQA,EAAQtlG,OAAS,IAG1CulG,EAAKG,EAAav4F,EAAIs4F,EAAat4F,EACnCq4F,EAAKE,EAAan5F,EAAIk5F,EAAal5F,EAEjCgK,EAAKQ,GAAGxK,EADA,GAANg5F,EACUhvF,EAAKQ,GAAGotF,SAAWuB,EAAan5F,EAGhCgK,EAAKQ,GAAGotF,SAAYqB,EAAKD,GAAOhvF,EAAKQ,GAAG5J,EAAIs4F,EAAat4F,GAAKs4F,EAAal5F,IAmB7Fk2F,GAAUtjG,UAAUylG,iBAAmB,SAAU5wB,EAAUjB,EAAY2xB,EAASC,OAC1Et9B,EAAOvnE,EAAGiX,EAAGgkC,KACbi5B,EAASh0E,OAAS,MACfF,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,IAAK,KAEhC2oE,GADJpB,EAAQrkE,KAAK2sE,OAAOqE,EAASl0E,KACP4hG,cAEI,GAAtBr+B,GAAAgE,EAAMl1C,SAAsB,KAC1BwzE,EAAiB,SAAUxlG,EAAGC,UACzBD,EAAEmC,WAAalC,EAAEkC,UAAY,EAAInC,EAAIC,GAAK,EAAI,GAEnDoiC,EAAQh+B,KAAK8D,IAAI,EAAGgzC,GAAKsqD,kBAAkBn9B,EAAWi8B,EAAS,IAAK,SAAUiB,IAC9E91D,EAAOrrC,KAAKY,IAAIqjE,EAAUzoE,OAAQs7C,GAAKsqD,kBAAkBn9B,EAAWk8B,EAAS,IAAK,QAASgB,GAAkB,GAC7G91D,GAAQ,IACVA,EAAO44B,EAAUzoE,YAEf6lG,EAAgB,IAAI5mG,MAAM4wC,EAAKrN,OAC9BzrB,EAAIyrB,EAAOzrB,EAAI84B,EAAM94B,IACxBgkC,EAAOssB,EAAMoB,UAAU1xD,GACvB8uF,EAAc9uF,EAAEyrB,GAASuY,EAE3Bg4B,EAAWiB,EAASl0E,IAAM+lG,OAI1B9yB,EAAWiB,EAASl0E,IAAMunE,EAAMoB,YAaxCg6B,GAAUtjG,UAAU0lG,eAAiB,SAAU7wB,EAAUjB,MAEnDiB,EAASh0E,OAAS,MACf,IAAIF,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,OAEL,GADtBkD,KAAK2sE,OAAOqE,EAASl0E,IACnBqyB,QAAQwwE,SAAkB,KAC9BkD,EAAgB9yB,EAAWiB,EAASl0E,OACpC+lG,EAAc7lG,OAAS,EAAG,KACxB8lG,EACAC,EAAiBF,EAAc7lG,OAM/BgmG,EAAiBD,GADL/iG,KAAK44C,KAAKN,KAAKy4C,eAAe8R,EAAcA,EAAc7lG,OAAS,GAAGmN,GAAKnK,KAAK44C,KAAKN,KAAKy4C,eAAe8R,EAAc,GAAG14F,IAE1I24F,EAAYthG,KAAKY,IAAIZ,KAAKC,KAAK,GAAMshG,GAAiBvhG,KAAK8D,IAAI,EAAG9D,KAAK+a,MAAMymF,aAEzEC,EAAc,IAAIhnG,MAAM8mG,GACnBhvF,EAAI,EAAGA,EAAIgvF,EAAgBhvF,GAAK+uF,EAAW,CAElDG,EADUzhG,KAAK+a,MAAMxI,EAAE+uF,IACND,EAAc9uF,GAEjCg8D,EAAWiB,EAASl0E,IAAMwlE,GAAA2gC,QAAAA,EAAmB,EAAEzhG,KAAK+a,MAAMwmF,EAAeD,OAenFrD,GAAUtjG,UAAU4lG,YAAc,SAAU/wB,EAAUjB,EAAYusB,OAC5D7rB,EAAWpM,EAAOvnE,EAGlBqyB,EAFA+zE,EAAmB,GACnBC,EAAoB,MAEpBnyB,EAASh0E,OAAS,EAAG,KAClBF,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,IAC/B2zE,EAAYV,EAAWiB,EAASl0E,IAChCqyB,EAAUnvB,KAAK2sE,OAAOqE,EAASl0E,IAAIqyB,QAC/BshD,EAAUzzE,OAAS,IACrBqnE,EAAQrkE,KAAK2sE,OAAOqE,EAASl0E,KAEP,IAAlBqyB,EAAQ9rB,OAAoC,QAAlB8rB,EAAQqF,MACH,SAA7BrF,EAAQstE,iBACVyG,EAAmBryB,GAAAqyB,QAAAA,EAAwBzyB,GAG3C0yB,EAAoBtyB,GAAAsyB,QAAAA,EAAyB1yB,GAI/C6rB,EAAYtrB,EAASl0E,IAAMunE,EAAM46B,UAAUxuB,EAAWO,EAASl0E,KAMrEiiG,GAAK1C,iBAAiB6G,EAAkB5G,EAAatrB,EAAU,iBAAkB,QACjF+tB,GAAK1C,iBAAiB8G,EAAmB7G,EAAatrB,EAAU,kBAAmB,WAYvFyuB,GAAUtjG,UAAU6lG,aAAe,SAAUhxB,EAAUsrB,OAIe8G,EAAQC,EAHxExqD,GAAU,EACVyqD,GAAgB,EAChBC,GAAiB,EACjBC,EAAU,IAAKC,EAAW,IAAKC,GAAW,IAAKC,GAAY,OAE3D3yB,EAASh0E,OAAS,EAAG,KAElB,IAAIF,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,IAAK,KACpCunE,EAAQrkE,KAAK2sE,OAAOqE,EAASl0E,IAC7BunE,GAA2C,SAAlCA,EAAMl1C,QAAQstE,kBACzB6G,GAAgB,EAChBE,EAAU,IACVE,GAAW,KAEJr/B,GAASA,EAAMl1C,QAAQstE,mBAC9B8G,GAAiB,EACjBE,EAAW,IACXE,GAAY,SAKX7mG,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,IAC3Bw/F,EAAYj/F,eAAe2zE,EAASl0E,MACE,IAApCw/F,EAAYtrB,EAASl0E,IAAI8mG,SAC3BR,EAAS9G,EAAYtrB,EAASl0E,IAAIsF,IAClCihG,EAAS/G,EAAYtrB,EAASl0E,IAAIwI,IAEe,SAA7Cg3F,EAAYtrB,EAASl0E,IAAI2/F,kBAC3B6G,GAAgB,EAChBE,EAAUA,EAAUJ,EAASA,EAASI,EACtCE,EAAUA,EAAUL,EAASA,EAASK,IAGtCH,GAAiB,EACjBE,EAAWA,EAAWL,EAASA,EAASK,EACxCE,EAAWA,EAAWN,EAASA,EAASM,IAM3B,GAAjBL,QACGlD,UAAU1hD,SAAS8kD,EAASE,GAEb,GAAlBH,QACGlD,WAAW3hD,SAAS+kD,EAAUE,GAGvC9qD,EAAU74C,KAAK6jG,qBAAqBP,EAAetjG,KAAKogG,YAAcvnD,EACtEA,EAAU74C,KAAK6jG,qBAAqBN,EAAgBvjG,KAAKqgG,aAAexnD,EAElD,GAAlB0qD,GAA2C,GAAjBD,QACvBlD,UAAU0D,WAAY,OACtBzD,WAAWyD,WAAY,SAGvB1D,UAAU0D,WAAY,OACtBzD,WAAWyD,WAAY,QAEzBzD,WAAW9I,QAAU+L,OACrBjD,WAAW7I,WAAax3F,KAAKogG,UAEJ,GAA1BpgG,KAAKqgG,WAAW9I,aAEX6I,UAAU50B,WADK,GAAlB+3B,EAC0BvjG,KAAKqgG,WAAWz3D,MAGhB,EAG9BiQ,EAAU74C,KAAKogG,UAAU/vC,UAAYxX,EACrCA,EAAU74C,KAAKqgG,WAAWhwC,UAAYxX,GAGtCA,EAAU74C,KAAKqgG,WAAWhwC,UAAYxX,MAIpCkrD,EAAa,CAAC,iBAAkB,kBAAmB,kBAAmB,wBACrEjnG,EAAI,EAAGA,EAAIinG,EAAW/mG,OAAQF,KACO,GAApC82D,GAAAod,QAAAA,EAAiB+yB,EAAWjnG,QAC9Bk0E,QAAAA,EAAgBpd,GAAAod,QAAAA,EAAiB+yB,EAAWjnG,IAAK,UAI9C+7C,GAYT4mD,GAAUtjG,UAAU0nG,qBAAuB,SAAUG,EAAU17C,OACzDjI,GAAU,SACE,GAAZ2jD,EACE17C,EAAKlK,IAAIuqB,MAAMphC,YAA6B,GAAf+gB,EAAK/N,SACpC+N,EAAK4H,OACL7P,GAAU,GAIPiI,EAAKlK,IAAIuqB,MAAMphC,YAA6B,GAAf+gB,EAAK/N,SACrC+N,EAAKkb,OACLnjB,GAAU,GAGPA,GAYTo/C,GAAUtjG,UAAU2lG,qBAAuB,SAAUmC,WAC/CzoD,EAAWx7C,KAAK44C,KAAKN,KAAKkD,SACrB1+C,EAAI,EAAGA,EAAImnG,EAAWjnG,OAAQF,IACrCmnG,EAAWnnG,GAAG49F,SAAWl/C,EAASyoD,EAAWnnG,GAAGqN,GAAKnK,KAAKq1C,MAAMzM,MAChEq7D,EAAWnnG,GAAG69F,SAAWsJ,EAAWnnG,GAAGyM,EACd9J,MAArBwkG,EAAWnnG,GAAG2jC,IAChBwjE,EAAWnnG,GAAG2+F,WAAajgD,EAASyoD,EAAWnnG,GAAG2jC,KAAOzgC,KAAKq1C,MAAMzM,MAGpEq7D,EAAWnnG,GAAG2+F,gBAAah8F,GAejCggG,GAAUtjG,UAAUgmG,qBAAuB,SAAU8B,EAAY5/B,OAC3D/b,EAAOtoD,KAAKogG,UACZ7C,EAAYlmD,OAAOr3C,KAAKu2F,IAAI/hE,MAAMqU,OAAOtiC,QAAQ,KAAM,KACrB,SAAlC89D,EAAMl1C,QAAQstE,mBAChBn0C,EAAOtoD,KAAKqgG,gBAET,IAAIvjG,EAAI,EAAGA,EAAImnG,EAAWjnG,OAAQF,IACrCmnG,EAAWnnG,GAAG69F,SAAWn5F,KAAK+a,MAAM+rC,EAAK4sC,aAAa+O,EAAWnnG,GAAGyM,IAEtE86D,EAAMs6B,gBAAgBn9F,KAAKY,IAAIm7F,EAAWj1C,EAAK4sC,aAAa,MCvkC9D,IACIzY,GAAO,UAUPjC,GAAa,CACfxlB,UAAW,CACT0nB,QAAS,SAAYD,IACrBxkD,OAAQ,SAAYwkD,YAAiB,YACrCxwB,UAAW,CAAC7N,IATN,OAUN+8B,SAAU,CAACrsD,OAXF,iBAWoB2tD,YAAiB,aAIhD7jB,iBAAkB,CAAC3hD,OApBR,mBAoB6B,aACxCwlF,iBAAkB,CAACxlF,OAAO,CAAC,OAAO,UAClCyoF,aAAc,CAACzoF,OAtBJ,UAuBXhL,KAAM,SAAYwwE,IAClBkjB,SAAU,SAAYljB,IACtBp5E,MAAM,SAAYo5E,IAClBmjB,YAAa,CAAC3oF,OA1BH,SA0BW1V,OAxBX,UAyBX27F,OAAQ,CACNxgB,QAAS,SAAYD,IACrBp0B,YAAa,CAACpxC,OAAO,CAAC,SAAS,MAAM,OAAO,UAC5CklD,QAAS,CAACrtC,OAzBD,UA0BTqsD,SAAU,SAAYsB,GAAK3tD,OA1BlB,WA4BX0F,MAAO,CAACvd,OAAO,CAAC,OAAO,MAAM,WAC7B+jF,SAAU,CACRpyD,MAAO,CAACrnC,OAjCC,UAkCTqqE,SAAU,CAACrqE,OAlCF,UAmCT26F,WAAY,SAAYzf,IACxBhW,MAAO,CAACxvD,OAAO,CAAC,OAAO,SAAS,UAChCkkE,SAAU,CAACrsD,OAlCF,WAoCXguE,cAAe,CACbpgB,QAAS,SAAYD,IACrBoiB,gBAAiB,CAAC5nF,OAAO,CAAC,cAAe,UAAU,YACnDwwE,MAAO,CAAClmF,OA1CC,UA2CT45E,SAAU,CAACrsD,OAxCF,iBAwCoB2tD,KAE/BjoB,WAAY,CACVkoB,QAAS,SAAYD,IACrBhoB,SAAU,UAAc,YACxB90B,KAAM,CAACp+B,OAhDE,UAiDTizB,MAAO,CAACvd,OAAO,CAAC,SAAS,WACzBkkE,SAAU,CAACrsD,OA/CF,iBA+CoB2tD,YAAiB,aAEhDojB,SAAU,CACRt3C,gBAAiB,SAAYk0B,IAC7Bx2B,gBAAiB,SAAYw2B,IAC7Bn2B,cAAe,SAAYm2B,IAC3Bga,MAAO,SAAYha,IACnB7zC,MAAO,CAAC3xB,OA3DC,SA2DO1V,OAzDP,UA0DT+5D,QAAS,SAAYmhB,IACrBsa,WAAY,SAAYta,IACxB/nC,KAAK,CACHgF,MAAO,CAACt3C,IAAI,CAACb,OA7DN,mBA6D0B,aAAa+D,IAAI,CAAC/D,OA7D5C,mBA6DgE,aAAa45E,SAAU,CAACrsD,OA1DxF,WA2DPrxB,OAAQ,UAAa,YACrBgyD,MAAO,CAAChE,KAAK,CAACx0C,OAjEP,SAiEc1V,OA/Dd,mBA+DkC,aAAaizB,MAAM,CAACvd,OAjEtD,mBAiE0E,aAAakkE,SAAU,CAACrsD,OA5DlG,WA6DPqsD,SAAU,CAACrsD,OA7DJ,WA+DT6lB,MAAM,CACJ+E,MAAO,CAACt3C,IAAI,CAACb,OAnEN,mBAmE0B,aAAa+D,IAAI,CAAC/D,OAnE5C,mBAmEgE,aAAa45E,SAAU,CAACrsD,OAhExF,WAiEPrxB,OAAQ,UAAa,YACrBgyD,MAAO,CAAChE,KAAK,CAACx0C,OAvEP,SAuEc1V,OArEd,mBAqEkC,aAAaizB,MAAM,CAACvd,OAvEtD,mBAuE0E,aAAakkE,SAAU,CAACrsD,OAlElG,WAmEPqsD,SAAU,CAACrsD,OAnEJ,WAqETqsD,SAAU,CAACrsD,OArEF,WAuEXgxE,OAAQ,CACNpjB,QAAS,SAAYD,IACrBga,MAAO,SAAYha,IACnB/nC,KAAM,CACJ4mB,QAAS,SAAYmhB,IACrB/8C,SAAU,CAACzoB,OAAO,CAAC,YAAY,eAAe,WAAW,gBACzDkkE,SAAU,CAACrsD,OA7EJ,WA+ET6lB,MAAO,CACL2mB,QAAS,SAAYmhB,IACrB/8C,SAAU,CAACzoB,OAAO,CAAC,YAAY,eAAe,WAAW,gBACzDkkE,SAAU,CAACrsD,OAlFJ,WAoFTqsD,SAAU,CAACrsD,OApFF,iBAoFoB2tD,KAE/B9P,OAAQ,CACN1tB,WAAY,CAAC29B,IApFP,OAqFNzB,SAAU,CAACrsD,OAxFF,WA2FX+oC,WAAY,SAAY4kB,IACxBE,eAAgB,CAACp7E,OA/FN,UAgGXmzD,WAAY,SAAY+nB,IACxBh8C,IAAK,CAACl/B,OAjGK,SAiGG0I,KA/FL,OA+FWgN,OAnGT,SAmGiBmT,OA5FjB,UA6FX3sB,OAAQ,CACNkpD,YAAa,CACXrsC,YAAa,CAACrD,OAtGP,mBAsG2B,aAClCoD,OAAQ,CAACpD,OAvGF,mBAuGsB,aAC7BmD,OAAQ,CAACnD,OAxGF,mBAwGsB,aAC7BqC,KAAM,CAACrC,OAzGA,mBAyGoB,aAC3BzJ,QAAS,CAACyJ,OA1GH,mBA0GuB,aAC9B7H,IAAK,CAAC6H,OA3GC,mBA2GmB,aAC1B1J,KAAM,CAAC0J,OA5GA,mBA4GoB,aAC3BjN,MAAO,CAACiN,OA7GD,mBA6GqB,aAC5BuE,QAAS,CAACvE,OA9GH,mBA8GuB,aAC9B5N,KAAM,CAAC4N,OA/GA,mBA+GoB,aAC3BkkE,SAAU,CAACrsD,OA3GJ,WA6GT83B,YAAa,CACXtsC,YAAa,CAACrD,OAnHP,mBAmH2B,aAClCoD,OAAQ,CAACpD,OApHF,mBAoHsB,aAC7BmD,OAAQ,CAACnD,OArHF,mBAqHsB,aAC7BqC,KAAM,CAACrC,OAtHA,mBAsHoB,aAC3BzJ,QAAS,CAACyJ,OAvHH,mBAuHuB,aAC9B7H,IAAK,CAAC6H,OAxHC,mBAwHmB,aAC1B1J,KAAM,CAAC0J,OAzHA,mBAyHoB,aAC3BjN,MAAO,CAACiN,OA1HD,mBA0HqB,aAC5BuE,QAAS,CAACvE,OA3HH,mBA2HuB,aAC9B5N,KAAM,CAAC4N,OA5HA,mBA4HoB,aAC3BkkE,SAAU,CAACrsD,OAxHJ,WA0HTqsD,SAAU,CAACrsD,OA1HF,WA4HX1E,OAAQ,UAAa,YACrBye,OAAQ,CAAC5xB,OAlIE,SAkIM1V,OAhIN,UAiIX63C,YAAa,CACX5Y,MAAO,CAACv2B,KAhID,OAgIO1I,OAlIL,SAkIa0V,OApIb,SAoIqBmT,OA7HrB,UA8HTqW,IAAK,CAACx2B,KAjIC,OAiIK1I,OAnIH,SAmIW0V,OArIX,SAqImBmT,OA9HnB,UA+HT4uB,OAAQ,CAAC/hC,OAtIA,UAuITkkE,SAAU,CAACrsD,OAlIF,SAkIUzoB,MApIX,UAsIV3I,OAAO,CAACuZ,OAzIG,UA0IXrE,QAAQ,CACNmoE,QAAS,CAAC6B,IAnIJ,OAoINzB,SAAU,CAACrsD,OAvIF,WAyIXxpB,IAAK,CAAC2E,KA1IG,OA0IG1I,OA5ID,SA4IS0V,OA9IT,SA8IiBmT,OAvIjB,UAwIXisC,UAAW,CAAC90D,OA7ID,SA6IS0V,OA/IT,UAgJXuxC,cAAe,CAACjnD,OA9IL,UA+IXa,IAAK,CAAC6H,KA7IG,OA6IG1I,OA/ID,SA+IS0V,OAjJT,SAiJiBmT,OA1IjB,UA2IXosC,UAAW,CAACj1D,OAhJD,SAgJS0V,OAlJT,UAmJXmmC,SAAU,SAAYq/B,IACtBxN,YAAa,SAAYwN,IACzBp0B,YAAa,CAACpxC,OArJH,UAsJX0hD,gBAAiB,SAAY8jB,IAC7Bx2B,gBAAiB,SAAYw2B,IAC7Bl0B,gBAAiB,SAAYk0B,IAC7Bn2B,cAAe,SAAYm2B,IAC3Bj8C,MAAO,CAACv2B,KAtJC,OAsJK1I,OAxJH,SAwJW0V,OA1JX,SA0JmBmT,OAnJnB,UAoJXq+B,SAAU,CACR9M,MAAO,CAAC1kC,OA5JC,mBA4JmB,aAC5B8sB,KAAM,CAACxiC,OA3JE,mBA2JkB,aAC3B45E,SAAU,CAACrsD,OAzJF,WA2JX8Z,MAAO,CAAC3xB,OAhKG,SAgKK1V,OA9JL,UA+JX87C,SAAU,SAAYo/B,IACtB36B,QAAS,CAAC7qC,OAAQ,CAAC,UAAW,SAAU,UAAW,KACnDsmC,QAAS,CAACh8C,OAjKC,UAkKX+7C,QAAS,CAAC/7C,OAlKC,UAmKXggG,OAAQ,CAAChgG,OAnKE,UAoKX45E,SAAU,CAACrsD,OAjKA,WAoKTiuD,GAAmB,CACrB7wD,OAAQ,CACN0sC,iBAAkB,CAAC,OAAQ,OAAQ,QAAS,UAAW,OAAQ,UAAW,MAAO,OAAQ,OAAQ,SAAU,UAE3G3sD,MAAM,EACN0zF,UAAU,EACVt8F,OAAM,EACN65F,OAAQ,CACNxgB,SAAS,EACTr0B,YAAa,CAAC,OAAO,MAAM,SAAS,UAEtC7zB,MAAO,CAAC,OAAO,MAAM,UACrBwmE,SAAU,CACRpyD,MAAO,CAAC,GAAG,EAAE,IAAI,GACjBgjC,SAAU,CAAC,GAAG,EAAE,IAAI,GACpBswB,YAAY,EACZz1B,MAAO,CAAC,OAAO,SAAS,UAE1Bq2B,cAAe,CACbpgB,SAAS,EACTmiB,gBAAiB,CAAC,cAAc,UAAU,YAE5CrqC,WAAY,CACVkoB,SAAS,EACT/8C,KAAM,CAAC,EAAE,EAAE,GAAG,GACdnL,MAAO,CAAC,SAAU,WAEpBqrE,SAAU,CACRt3C,iBAAiB,EACjBtC,iBAAiB,EACjBK,eAAe,EACfmwC,OAAO,EACP7tD,MAAO,CAAC,GAAG,EAAE,IAAI,GACjB0yB,SAAS,EACTy7B,YAAY,EACZriD,KAAK,CAGH+a,MAAO,CAAChE,KAAK,GAAGj3B,MAAM,KAExBmgB,MAAM,CAGJ8a,MAAO,CAAChE,KAAK,GAAGj3B,MAAM,MAG1BsrE,OAAQ,CACNpjB,SAAS,EACT+Z,OAAO,EACP/hD,KAAM,CACJ4mB,SAAS,EACT57B,SAAU,CAAC,YAAY,eAAe,WAAW,gBAEnDiV,MAAO,CACL2mB,SAAS,EACT57B,SAAU,CAAC,YAAY,eAAe,WAAW,iBAIrDm4B,YAAY,EACZnD,YAAY,EACZj0B,IAAK,GACLhjC,OAAQ,CACNkpD,YAAa,CACXrsC,YAAY,MACZD,OAAY,IACZD,OAAY,QACZd,KAAY,QACZ9L,QAAY,QACZ4B,IAAY,IACZ7B,KAAY,IACZvD,MAAY,MACZwR,QAAY,OACZnS,KAAY,QAEdu9C,YAAa,CACXtsC,YAAY,WACZD,OAAY,eACZD,OAAY,aACZd,KAAY,aACZ9L,QAAY,YACZ4B,IAAY,YACZ7B,KAAY,YACZvD,MAAY,OACZwR,QAAY,OACZnS,KAAY,KAIhBw/B,OAAQ,GACRnrC,OAAQ,GACR4H,IAAK,GACL+wD,UAAW,GACX7N,cAAe,CAAC,EAAG,EAAG,GAAI,GAC1BpmD,IAAK,GACLo0D,UAAW,GACXpZ,UAAS,EACTiL,YAAa,CAAC,OAAQ,SAAU,OAChCsQ,iBAAiB,EACjB1S,iBAAiB,EACjBsC,iBAAiB,EACjBjC,eAAe,EACf9lB,MAAO,GACPoI,MAAO,OACPyU,UAAU,EACVyE,QAAS,CAAC,UAAW,SAAU,UAAW,IAC1CvE,QAAS,CAAC,SAAiB,GAAI,SAAiB,GAChDD,QAAS,CAAC,GAAI,GAAI,SAAiB,GACnCikD,OAAQ,ICjQZ,SAAS2C,GAASj4C,EAAWzV,EAAOm2B,EAAQx9C,0BAEpC+T,GAAcypC,IAAWA,aAAkB/0B,IAAW+0B,aAAkB2H,KAAa3H,aAAkBzwE,OAAQ,KAC/Gu0F,EAAgBthE,EACpBA,EAAUw9C,EACVA,EAAS8jB,EAKPthE,GAAWA,EAAQwtD,gBACrBh6E,QAAQH,KAAK,4HAGXm8C,EAAK3+C,UACJk9C,eAAiB,CACpB1c,MAAO,KACPC,IAAO,KAEPo3B,YAAY,EAEZxP,YAAa,CACXC,KAAM,SACNvQ,KAAM,UAGR3tB,OAAQA,GAERwe,MAAO,KACPC,OAAQ,KACRwtB,UAAW,KACXG,UAAW,WAERrnC,QAAUmpB,GAAKuN,WAAW,GAAI7lD,KAAKk9C,qBAGnCwL,QAAQuD,QAGRsI,WAAa,QAEb3b,KAAO,CACVwF,IAAKp+C,KAAKo+C,IACV5E,SAAUx5C,KAAKq1C,MACfwI,QAAS,CACPvH,GAAIP,UAAKO,WAAQt2C,MACjBu2C,IAAKR,UAAKQ,YAASv2C,MACnBygD,KAAM1K,UAAK0K,aAAUzgD,OAEvBo5C,YAAa,GACbd,KAAM,CACJkD,SAAUzF,KAAA4I,EAAGmyC,kBAAenyC,GAC5BoyC,eAAgBh7C,KAAA4I,EAAGqyC,wBAAqBryC,GACxC1C,OAAQlG,KAAA4I,EAAGsyC,gBAAatyC,GACxBuyC,aAAen7C,KAAA4I,EAAGwyC,sBAAmBxyC,UAKpCjF,MAAQ,IAAImD,GAAM78C,KAAK44C,WACvB2b,WAAWt3D,KAAK+C,KAAK05C,YACrBd,KAAKc,MAAQ15C,KAAK05C,WAGlB+O,SAAW,IAAIV,GAAS/nD,KAAK44C,WAC7B2b,WAAWt3D,KAAK+C,KAAKyoD,eAIrBiP,YAAc,IAAIgB,GAAY14D,KAAK44C,WACnC2b,WAAWt3D,KAAK+C,KAAK03D,kBAGrBysC,UAAY,IAAI1E,GAAUz/F,KAAK44C,WAE/B2b,WAAWt3D,KAAK+C,KAAKmkG,gBAErB1+B,UAAY,UACZsK,WAAa,UAGbz5B,GAAG,OAAO,SAAUkK,GACvB7B,EAAG8B,KAAK,QAAS9B,EAAGgV,mBAAmBnT,YAEpClK,GAAG,aAAa,SAAUkK,GAC7B7B,EAAG8B,KAAK,cAAe9B,EAAGgV,mBAAmBnT,YAE1CpC,IAAI+C,KAAKiwC,cAAgB,SAAU5wC,GACtC7B,EAAG8B,KAAK,cAAe9B,EAAGgV,mBAAmBnT,UAI1CoxC,gBAAiB,OACjBt7C,GAAG,WAAW,cACG,MAAhBqI,EAAG8mB,eACF9mB,EAAGizC,iBAAmBjzC,EAAGxvB,QAAQquB,eACpCmB,EAAGizC,gBAAiB,EACInyF,MAApBk/C,EAAGxvB,QAAQqR,OAAwC/gC,MAAlBk/C,EAAGxvB,QAAQsR,IAAkB,IACxChhC,MAApBk/C,EAAGxvB,QAAQqR,OAAwC/gC,MAAlBk/C,EAAGxvB,QAAQsR,QAC1CiZ,EAAQiF,EAAGkzC,mBAGbrxD,EAA4B/gC,MAApBk/C,EAAGxvB,QAAQqR,MAAqBme,EAAGxvB,QAAQqR,MAAQkZ,EAAMt3C,IACjEq+B,EAA4BhhC,MAApBk/C,EAAGxvB,QAAQsR,IAAqBke,EAAGxvB,QAAQsR,IAAQiZ,EAAMp0C,IACrEq5C,EAAGyX,UAAU51B,EAAOC,EAAK,CAACse,WAAW,SAErCJ,EAAGmzC,IAAI,CAAC/yC,WAAW,IAIlBJ,EAAGmT,kBAAoBnT,EAAGqT,yBAA4BrT,EAAGxvB,QAAQqR,OAAUme,EAAGxvB,QAAQsR,OACtFke,EAAGxvB,QAAQquB,cACdmB,EAAGmT,iBAAkB,EACrBnT,EAAGP,IAAI+C,KAAK3sB,MAAMyqB,WAAa,UAC/BN,EAAGP,IAAIsT,cAAcnqB,WAAWC,YAAYmX,EAAGP,IAAIsT,eAC/C/S,EAAGxvB,QAAQ0tD,2BACF,kBACFl+B,EAAGxvB,QAAQ0tD,0BACjB,QAML1tD,QACGqvB,WAAWrvB,GAIdw9C,QACGvX,UAAUuX,GAIbn2B,QACG2e,SAAS3e,QAIXub,UAIPmyC,GAAQ/nG,UAAY,IAAIs/C,GAExByoD,GAAQ/nG,UAAUqiD,WAAa,SAAUrvB,IAGpB,IADFwrD,GAAUqX,SAAS7iE,EAASqrD,KAE3C73E,QAAQq3C,IAAI,2DAA4D0gC,IAG1Ej/B,GAAKt/C,UAAUqiD,WAAWniD,KAAK2D,KAAMmvB,IAOvC+0E,GAAQ/nG,UAAUg5D,SAAW,SAAS3e,OAIhCy7C,EAHAmS,EAAiC,MAAlBpkG,KAAKylE,aAQtBwsB,EAJGz7C,EAIUe,GADNf,aAAiBoB,IAAWpB,aAAiB89B,GACrB99B,EAIA,IAAIoB,GAAQpB,IAP9B,KAWXx2C,KAAKylE,gBAEFA,UAAUptB,eAEZotB,UAAYwsB,OACZkS,WAAankG,KAAKmkG,UAAUhvC,SAAuB,MAAd88B,EAAqBA,EAAWz6C,MAAQ,MAE9E4sD,KACwB3kG,MAAtBO,KAAKmvB,QAAQqR,OAA0C/gC,MAApBO,KAAKmvB,QAAQsR,IAAkB,KAChED,EAA8B/gC,MAAtBO,KAAKmvB,QAAQqR,MAAqBxgC,KAAKmvB,QAAQqR,MAAQ,KAC/DC,EAA4BhhC,MAApBO,KAAKmvB,QAAQsR,IAAqBzgC,KAAKmvB,QAAQsR,IAAM,UAC5D21B,UAAU51B,EAAOC,EAAK,CAACse,WAAW,cAGlC+yC,IAAI,CAAC/yC,WAAW,KAS3BmlD,GAAQ/nG,UAAUi5D,UAAY,SAASuX,OAEjCslB,EAKFA,EAJGtlB,EAGIA,aAAkB/0B,IAAW+0B,aAAkB2H,GACzC3H,EAIA,IAAI/0B,GAAQ+0B,GAPZ,UAUVoD,WAAakiB,OACbkS,UAAU/uC,UAAU68B,IAU3BiS,GAAQ/nG,UAAUg8F,UAAY,SAASh8B,EAASvzB,EAAOC,eACtCppC,IAAXmpC,IAAuBA,EAAS,SACrBnpC,IAAXopC,IAAuBA,EAAS,SACGppC,IAAnCO,KAAKmkG,UAAUx3B,OAAOxQ,GACjBn8D,KAAKmkG,UAAUx3B,OAAOxQ,GAASg8B,UAAUvvD,EAAMC,GAG/C,sBAAyBszB,EAAU,KAS9C+nC,GAAQ/nG,UAAUkoG,eAAiB,SAASloC,eACH18D,IAAnCO,KAAKmkG,UAAUx3B,OAAOxQ,KAChBn8D,KAAKmkG,UAAUx3B,OAAOxQ,GAASb,eAAkE77D,IAAtDO,KAAKmkG,UAAUh1E,QAAQw9C,OAAO1tB,WAAWkd,IAA+E,GAArDn8D,KAAKmkG,UAAUh1E,QAAQw9C,OAAO1tB,WAAWkd,MAcnK+nC,GAAQ/nG,UAAU25D,aAAe,eAC3B1zD,EAAM,KACNkD,EAAM,SAGL,IAAI62D,KAAWn8D,KAAKmkG,UAAUx3B,UAC7B3sE,KAAKmkG,UAAUx3B,OAAOtvE,eAAe8+D,IACO,GAA1Cn8D,KAAKmkG,UAAUx3B,OAAOxQ,GAASb,YAC5B,IAAIx+D,EAAI,EAAGA,EAAIkD,KAAKmkG,UAAUx3B,OAAOxQ,GAASsJ,UAAUzoE,OAAQF,IAAK,KACpEi7C,EAAO/3C,KAAKmkG,UAAUx3B,OAAOxQ,GAASsJ,UAAU3oE,GAChDgF,EAAQw2C,GAAKpB,QAAQa,EAAK5tC,EAAG,QAAQ5M,UACzC6E,EAAa,MAAPA,GAAsBA,EAAMN,EAAdA,EAA8BM,EAClDkD,EAAa,MAAPA,GAAsBA,EAAMxD,EAAdA,EAA8BwD,QAMnD,CACLlD,IAAa,MAAPA,EAAe,IAAI1F,KAAK0F,GAAO,KACrCkD,IAAa,MAAPA,EAAe,IAAI5I,KAAK4I,GAAO,OAWzC4+F,GAAQ/nG,UAAUw3D,mBAAqB,SAAUnT,OAC3C2B,EAAU3B,EAAMxB,OAASwB,EAAMxB,OAAO70C,EAAIq2C,EAAM2B,QAChDC,EAAU5B,EAAMxB,OAASwB,EAAMxB,OAAOz1C,EAAIi3C,EAAM4B,QAChDj4C,EAAIg4C,EAAU7J,GAAKgsD,gBAAgBtkG,KAAKo+C,IAAI3E,iBAC5ClwC,EAAI64C,EAAU9J,GAAKisD,eAAevkG,KAAKo+C,IAAI3E,iBAC3C/4B,EAAO1gB,KAAKixF,QAAQ9mF,GAEpBwlD,EAAaH,GAAW+jC,qBAAqB/yC,GAE7C5Y,EAAU0Q,GAAKk7C,UAAUhzC,GACzBizC,EAAO,KACPn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAKyoD,SAASrK,IAAI4J,aACrChoD,KAAKs0D,WAAahc,GAAKo7C,UAAU9rD,EAAS5nC,KAAKs0D,UAAUlW,IAAI4J,YADGyrC,EAAO,OAEvEn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAKmkG,UAAU/D,UAAUhiD,IAAIuqB,QACrDrwB,GAAKo7C,UAAU9rD,EAAS5nC,KAAKmkG,UAAU9D,WAAWjiD,IAAIuqB,OADU8qB,EAAO,YAEvEn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAKmkG,UAAU7D,WAAWliD,IAAIuqB,QACtDrwB,GAAKo7C,UAAU9rD,EAAS5nC,KAAKmkG,UAAU5D,YAAYniD,IAAIuqB,OADS8qB,EAAO,SAEzD,MAAd9jC,EAAoC8jC,EAAO,cAC3Cn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAK03D,YAAY7H,KAAuB4jC,EAAO,eACvEn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAKo+C,IAAIY,UAA+By0C,EAAO,kBAE5E3xF,EAAQ,GACRs+F,EAAYpgG,KAAKmkG,UAAU/D,UAC3BC,EAAargG,KAAKmkG,UAAU9D,kBAC3BD,EAAU7lD,QAAUv6C,KAAKylE,UAAUzoE,OAAS,GAC/C8E,EAAM7E,KAAKmjG,EAAUlH,cAAc3vF,KAEhC82F,EAAW9lD,QAAUv6C,KAAKylE,UAAUzoE,OAAS,GAChD8E,EAAM7E,KAAKojG,EAAWnH,cAAc3vF,IAG/B,CACLi3C,MAAOA,EACPmP,WAAYA,EAAaA,EAAWxgC,QAAQiE,GAAK,KACjDqgE,KAAMA,EACNE,MAAOnzC,EAAM4D,SAAW5D,EAAM4D,SAASuvC,MAAQnzC,EAAMmzC,MACrDC,MAAOpzC,EAAM4D,SAAW5D,EAAM4D,SAASwvC,MAAQpzC,EAAMozC,MACrDzpF,EAAGA,EACHZ,EAAGA,EACHmX,KAAMA,EACN5e,MAAOA,IASXoiG,GAAQ/nG,UAAU44D,oBAAsB,kBAC/B,IAAIo3B,GAAansF,KAAMA,KAAKo+C,IAAI6N,UAAW8wB,KCtTpD,IAAMynB,GAAkBxM,QACjBt6F,OAAO8mG,QAER3Q,GAAW,CACfp4C,KAAAA,GACA6E,SAAAA,GACAzD,MAAAA,GACAx5C,MAAAA,GACAmiD,SAAAA,GAEA+O,WAAY,CACV/d,MAAO,CACLutB,KAAAA,GACA6F,eAAAA,GACArD,QAAAA,GACA6B,UAAAA,GACAM,UAAAA,IAGF5E,gBAAAA,GACAnrB,UAAAA,GACA+f,YAAAA,GACAlJ,WAAAA,GACA8mC,SAAAA,GACArC,UAAAA,GACA+F,WAAAA,GACA99B,MAAAA,GACAyS,QAAAA,GACA0rB,OAAAA,GACAoF,UAAAA,GACA13C,SAAAA"}
\ No newline at end of file
diff --git a/core/src/main/resources/org/apache/spark/ui/static/webui.css b/core/src/main/resources/org/apache/spark/ui/static/webui.css
index f7f8a0e0e9061..fad28ce333675 100755
--- a/core/src/main/resources/org/apache/spark/ui/static/webui.css
+++ b/core/src/main/resources/org/apache/spark/ui/static/webui.css
@@ -80,6 +80,10 @@ a:not([href]):hover {
   padding: 0;
 }
 
+.navbar-brand a:hover {
+  text-decoration: none;
+}
+
 .navbar .navbar-nav .nav-link {
   height: 50px;
   padding: 10px 15px 10px;
@@ -109,7 +113,7 @@ table.sortable td {
   box-shadow: inset 1px 0 0 rgba(0,0,0,.15), inset 0 -1px 0 rgba(0,0,0,.15);
 }
 
-.progress.progress-started {
+.progress .progress-bar.progress-started {
   background-color: #A0DFFF;
   background-image: -moz-linear-gradient(top, #A4EDFF, #94DDFF);
   background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#A4EDFF), to(#94DDFF));
@@ -120,7 +124,7 @@ table.sortable td {
   filter: progid:dximagetransform.microsoft.gradient(startColorstr='#FFA4EDFF', endColorstr='#FF94DDFF', GradientType=0);
 }
 
-.progress .progress-bar {
+.progress .progress-bar.progress-completed {
   background-color: #3EC0FF;
   background-image: -moz-linear-gradient(top, #44CBFF, #34B0EE);
   background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#44CBFF), to(#34B0EE));
@@ -317,10 +321,6 @@ a.expandbutton {
   width: 100%;
 }
 
-.container-fluid-div {
-  width: 200px;
-}
-
 .select-all-div-checkbox-div {
   width: 90px;
 }
diff --git a/core/src/main/resources/org/apache/spark/ui/static/webui.js b/core/src/main/resources/org/apache/spark/ui/static/webui.js
index 0ba461f02317f..c149f2d84337e 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/webui.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/webui.js
@@ -15,12 +15,20 @@
  * limitations under the License.
  */
 
+/* global $ */
+/* eslint-disable no-unused-vars */
 var uiRoot = "";
+var appBasePath = "";
 
 function setUIRoot(val) {
-    uiRoot = val;
+  uiRoot = val;
 }
 
+function setAppBasePath(path) {
+  appBasePath = path;
+}
+/* eslint-enable no-unused-vars */
+
 function collapseTablePageLoad(name, table){
   if (window.localStorage.getItem(name) == "true") {
     // Set it to false so that the click function can revert it
@@ -30,20 +38,20 @@ function collapseTablePageLoad(name, table){
 }
 
 function collapseTable(thisName, table){
-    var status = window.localStorage.getItem(thisName) == "true";
-    status = !status;
+  var status = window.localStorage.getItem(thisName) == "true";
+  status = !status;
 
-    var thisClass = '.' + thisName
+  var thisClass = '.' + thisName;
 
-    // Expand the list of additional metrics.
-    var tableDiv = $(thisClass).parent().find('.' + table);
-    $(tableDiv).toggleClass('collapsed');
+  // Expand the list of additional metrics.
+  var tableDiv = $(thisClass).parent().find('.' + table);
+  $(tableDiv).toggleClass('collapsed');
 
-    // Switch the class of the arrow from open to closed.
-    $(thisClass).find('.collapse-table-arrow').toggleClass('arrow-open');
-    $(thisClass).find('.collapse-table-arrow').toggleClass('arrow-closed');
+  // Switch the class of the arrow from open to closed.
+  $(thisClass).find('.collapse-table-arrow').toggleClass('arrow-open');
+  $(thisClass).find('.collapse-table-arrow').toggleClass('arrow-closed');
 
-    window.localStorage.setItem(thisName, "" + status);
+  window.localStorage.setItem(thisName, "" + status);
 }
 
 // Add a call to collapseTablePageLoad() on each collapsible table
@@ -82,7 +90,8 @@ $(function() {
   collapseTablePageLoad('collapse-aggregated-poolActiveStages','aggregated-poolActiveStages');
   collapseTablePageLoad('collapse-aggregated-tasks','aggregated-tasks');
   collapseTablePageLoad('collapse-aggregated-rdds','aggregated-rdds');
-  collapseTablePageLoad('collapse-aggregated-activeBatches','aggregated-activeBatches');
+  collapseTablePageLoad('collapse-aggregated-waitingBatches','aggregated-waitingBatches');
+  collapseTablePageLoad('collapse-aggregated-runningBatches','aggregated-runningBatches');
   collapseTablePageLoad('collapse-aggregated-completedBatches','aggregated-completedBatches');
   collapseTablePageLoad('collapse-aggregated-runningExecutions','aggregated-runningExecutions');
   collapseTablePageLoad('collapse-aggregated-completedExecutions','aggregated-completedExecutions');
@@ -95,8 +104,8 @@ $(function() {
 });
 
 $(function() {
-    // Show/hide full job description on click event.
-    $(".description-input").click(function() {
-        $(this).toggleClass("description-input-full");
-    });
+  // Show/hide full job description on click event.
+  $(".description-input").click(function() {
+    $(this).toggleClass("description-input-full");
+  });
 });
diff --git a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
index 4d765481eb836..09fa91655fba5 100644
--- a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
+++ b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
@@ -21,7 +21,6 @@ import java.util.{Properties, Timer, TimerTask}
 
 import scala.collection.JavaConverters._
 import scala.concurrent.duration._
-import scala.language.postfixOps
 import scala.util.{Failure, Success => ScalaSuccess, Try}
 
 import org.apache.spark.annotation.{Experimental, Since}
diff --git a/core/src/main/scala/org/apache/spark/ContextAwareIterator.scala b/core/src/main/scala/org/apache/spark/ContextAwareIterator.scala
new file mode 100644
index 0000000000000..84ae93f178823
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/ContextAwareIterator.scala
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark
+
+import org.apache.spark.annotation.DeveloperApi
+
+/**
+ * :: DeveloperApi ::
+ * A TaskContext aware iterator.
+ *
+ * As the Python evaluation consumes the parent iterator in a separate thread,
+ * it could consume more data from the parent even after the task ends and the parent is closed.
+ * If an off-heap access exists in the parent iterator, it could cause segmentation fault
+ * which crashes the executor.
+ * Thus, we should use [[ContextAwareIterator]] to stop consuming after the task ends.
+ *
+ * @since 3.1.0
+ */
+@DeveloperApi
+class ContextAwareIterator[+T](val context: TaskContext, val delegate: Iterator[T])
+  extends Iterator[T] {
+
+  override def hasNext: Boolean =
+    !context.isCompleted() && !context.isInterrupted() && delegate.hasNext
+
+  override def next(): T = delegate.next()
+}
diff --git a/core/src/main/scala/org/apache/spark/ContextCleaner.scala b/core/src/main/scala/org/apache/spark/ContextCleaner.scala
index cfa1139140025..091b5e1600d9e 100644
--- a/core/src/main/scala/org/apache/spark/ContextCleaner.scala
+++ b/core/src/main/scala/org/apache/spark/ContextCleaner.scala
@@ -27,6 +27,7 @@ import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config._
 import org.apache.spark.rdd.{RDD, ReliableRDDCheckpointData}
+import org.apache.spark.scheduler.SparkListener
 import org.apache.spark.shuffle.api.ShuffleDriverComponents
 import org.apache.spark.util.{AccumulatorContext, AccumulatorV2, ThreadUtils, Utils}
 
@@ -39,6 +40,7 @@ private case class CleanShuffle(shuffleId: Int) extends CleanupTask
 private case class CleanBroadcast(broadcastId: Long) extends CleanupTask
 private case class CleanAccum(accId: Long) extends CleanupTask
 private case class CleanCheckpoint(rddId: Int) extends CleanupTask
+private case class CleanSparkListener(listener: SparkListener) extends CleanupTask
 
 /**
  * A WeakReference associated with a CleanupTask.
@@ -170,6 +172,13 @@ private[spark] class ContextCleaner(
     registerForCleanup(rdd, CleanCheckpoint(parentId))
   }
 
+  /** Register a SparkListener to be cleaned up when its owner is garbage collected. */
+  def registerSparkListenerForCleanup(
+      listenerOwner: AnyRef,
+      listener: SparkListener): Unit = {
+    registerForCleanup(listenerOwner, CleanSparkListener(listener))
+  }
+
   /** Register an object for cleanup. */
   private def registerForCleanup(objectForCleanup: AnyRef, task: CleanupTask): Unit = {
     referenceBuffer.add(new CleanupTaskWeakReference(task, objectForCleanup, referenceQueue))
@@ -197,6 +206,8 @@ private[spark] class ContextCleaner(
                 doCleanupAccum(accId, blocking = blockOnCleanupTasks)
               case CleanCheckpoint(rddId) =>
                 doCleanCheckpoint(rddId)
+              case CleanSparkListener(listener) =>
+                doCleanSparkListener(listener)
             }
           }
         }
@@ -276,6 +287,16 @@ private[spark] class ContextCleaner(
     }
   }
 
+  def doCleanSparkListener(listener: SparkListener): Unit = {
+    try {
+      logDebug(s"Cleaning Spark listener $listener")
+      sc.listenerBus.removeListener(listener)
+      logDebug(s"Cleaned Spark listener $listener")
+    } catch {
+      case e: Exception => logError(s"Error cleaning Spark listener $listener", e)
+    }
+  }
+
   private def broadcastManager = sc.env.broadcastManager
   private def mapOutputTrackerMaster = sc.env.mapOutputTracker.asInstanceOf[MapOutputTrackerMaster]
 }
diff --git a/core/src/main/scala/org/apache/spark/Dependency.scala b/core/src/main/scala/org/apache/spark/Dependency.scala
index ba8e4d69ba755..d21b9d9833e9e 100644
--- a/core/src/main/scala/org/apache/spark/Dependency.scala
+++ b/core/src/main/scala/org/apache/spark/Dependency.scala
@@ -23,6 +23,7 @@ import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.rdd.RDD
 import org.apache.spark.serializer.Serializer
 import org.apache.spark.shuffle.{ShuffleHandle, ShuffleWriteProcessor}
+import org.apache.spark.storage.BlockManagerId
 
 /**
  * :: DeveloperApi ::
@@ -95,6 +96,20 @@ class ShuffleDependency[K: ClassTag, V: ClassTag, C: ClassTag](
   val shuffleHandle: ShuffleHandle = _rdd.context.env.shuffleManager.registerShuffle(
     shuffleId, this)
 
+  /**
+   * Stores the location of the list of chosen external shuffle services for handling the
+   * shuffle merge requests from mappers in this shuffle map stage.
+   */
+  private[spark] var mergerLocs: Seq[BlockManagerId] = Nil
+
+  def setMergerLocs(mergerLocs: Seq[BlockManagerId]): Unit = {
+    if (mergerLocs != null) {
+      this.mergerLocs = mergerLocs
+    }
+  }
+
+  def getMergerLocs: Seq[BlockManagerId] = mergerLocs
+
   _rdd.sparkContext.cleaner.foreach(_.registerShuffleForCleanup(this))
   _rdd.sparkContext.shuffleDriverComponents.registerShuffle(shuffleId)
 }
diff --git a/core/src/main/scala/org/apache/spark/ExecutorAllocationClient.scala b/core/src/main/scala/org/apache/spark/ExecutorAllocationClient.scala
index 00bd0063c9e3a..5b587d7fbadbb 100644
--- a/core/src/main/scala/org/apache/spark/ExecutorAllocationClient.scala
+++ b/core/src/main/scala/org/apache/spark/ExecutorAllocationClient.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark
 
+import org.apache.spark.scheduler.ExecutorDecommissionInfo
+
 /**
  * A client that communicates with the cluster manager to request or kill executors.
  * This is currently supported only in YARN mode.
@@ -81,6 +83,59 @@ private[spark] trait ExecutorAllocationClient {
     countFailures: Boolean,
     force: Boolean = false): Seq[String]
 
+  /**
+   * Request that the cluster manager decommission the specified executors.
+   * Default implementation delegates to kill, scheduler must override
+   * if it supports graceful decommissioning.
+   *
+   * @param executorsAndDecomInfo identifiers of executors & decom info.
+   * @param adjustTargetNumExecutors whether the target number of executors will be adjusted down
+   *                                 after these executors have been decommissioned.
+   * @param triggeredByExecutor whether the decommission is triggered at executor.
+   * @return the ids of the executors acknowledged by the cluster manager to be removed.
+   */
+  def decommissionExecutors(
+      executorsAndDecomInfo: Array[(String, ExecutorDecommissionInfo)],
+      adjustTargetNumExecutors: Boolean,
+      triggeredByExecutor: Boolean): Seq[String] = {
+    killExecutors(executorsAndDecomInfo.map(_._1),
+      adjustTargetNumExecutors,
+      countFailures = false)
+  }
+
+
+  /**
+   * Request that the cluster manager decommission the specified executor.
+   * Delegates to decommissionExecutors.
+   *
+   * @param executorId identifiers of executor to decommission
+   * @param decommissionInfo information about the decommission (reason, host loss)
+   * @param adjustTargetNumExecutors if we should adjust the target number of executors.
+   * @param triggeredByExecutor whether the decommission is triggered at executor.
+   *                            (TODO: add a new type like `ExecutorDecommissionInfo` for the
+   *                            case where executor is decommissioned at executor first, so we
+   *                            don't need this extra parameter.)
+   * @return whether the request is acknowledged by the cluster manager.
+   */
+  final def decommissionExecutor(
+      executorId: String,
+      decommissionInfo: ExecutorDecommissionInfo,
+      adjustTargetNumExecutors: Boolean,
+      triggeredByExecutor: Boolean = false): Boolean = {
+    val decommissionedExecutors = decommissionExecutors(
+      Array((executorId, decommissionInfo)),
+      adjustTargetNumExecutors = adjustTargetNumExecutors,
+      triggeredByExecutor = triggeredByExecutor)
+    decommissionedExecutors.nonEmpty && decommissionedExecutors(0).equals(executorId)
+  }
+
+  /**
+   * Request that the cluster manager decommission every executor on the specified host.
+   *
+   * @return whether the request is acknowledged by the cluster manager.
+   */
+  def decommissionExecutorsOnHost(host: String): Boolean
+
   /**
    * Request that the cluster manager kill every executor on the specified host.
    *
diff --git a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
index 4df58354c99ca..779559b1169f0 100644
--- a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
+++ b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
@@ -23,11 +23,12 @@ import scala.collection.mutable
 import scala.collection.mutable.ArrayBuffer
 import scala.util.control.{ControlThrowable, NonFatal}
 
-import com.codahale.metrics.{Gauge, MetricRegistry}
+import com.codahale.metrics.{Counter, Gauge, MetricRegistry}
 
 import org.apache.spark.internal.{config, Logging}
 import org.apache.spark.internal.config._
-import org.apache.spark.internal.config.Tests.TEST_SCHEDULE_INTERVAL
+import org.apache.spark.internal.config.DECOMMISSION_ENABLED
+import org.apache.spark.internal.config.Tests.TEST_DYNAMIC_ALLOCATION_SCHEDULE_ENABLED
 import org.apache.spark.metrics.source.Source
 import org.apache.spark.resource.ResourceProfile.UNKNOWN_RESOURCE_PROFILE_ID
 import org.apache.spark.resource.ResourceProfileManager
@@ -47,8 +48,8 @@ import org.apache.spark.util.{Clock, SystemClock, ThreadUtils, Utils}
  * executors that could run all current running and pending tasks at once.
  *
  * Increasing the target number of executors happens in response to backlogged tasks waiting to be
- * scheduled. If the scheduler queue is not drained in N seconds, then new executors are added. If
- * the queue persists for another M seconds, then more executors are added and so on. The number
+ * scheduled. If the scheduler queue is not drained in M seconds, then new executors are added. If
+ * the queue persists for another N seconds, then more executors are added and so on. The number
  * added in each round increases exponentially from the previous round until an upper bound has been
  * reached. The upper bound is based both on a configured property and on the current number of
  * running and pending tasks, as described above.
@@ -127,19 +128,21 @@ private[spark] class ExecutorAllocationManager(
   private val executorAllocationRatio =
     conf.get(DYN_ALLOCATION_EXECUTOR_ALLOCATION_RATIO)
 
+  private val decommissionEnabled = conf.get(DECOMMISSION_ENABLED)
+
   private val defaultProfileId = resourceProfileManager.defaultResourceProfile.id
 
   validateSettings()
 
   // Number of executors to add for each ResourceProfile in the next round
-  private val numExecutorsToAddPerResourceProfileId = new mutable.HashMap[Int, Int]
+  private[spark] val numExecutorsToAddPerResourceProfileId = new mutable.HashMap[Int, Int]
   numExecutorsToAddPerResourceProfileId(defaultProfileId) = 1
 
   // The desired number of executors at this moment in time. If all our executors were to die, this
   // is the number of executors we would immediately want from the cluster manager.
   // Note every profile will be allowed to have initial number,
   // we may want to make this configurable per Profile in the future
-  private val numExecutorsTargetPerResourceProfileId = new mutable.HashMap[Int, Int]
+  private[spark] val numExecutorsTargetPerResourceProfileId = new mutable.HashMap[Int, Int]
   numExecutorsTargetPerResourceProfileId(defaultProfileId) = initialNumExecutors
 
   // A timestamp of when an addition should be triggered, or NOT_SET if it is not set
@@ -147,23 +150,20 @@ private[spark] class ExecutorAllocationManager(
   private var addTime: Long = NOT_SET
 
   // Polling loop interval (ms)
-  private val intervalMillis: Long = if (Utils.isTesting) {
-      conf.get(TEST_SCHEDULE_INTERVAL)
-    } else {
-      100
-    }
+  private val intervalMillis: Long = 100
 
   // Listener for Spark events that impact the allocation policy
   val listener = new ExecutorAllocationListener
 
-  val executorMonitor = new ExecutorMonitor(conf, client, listenerBus, clock)
-
   // Executor that handles the scheduling task.
   private val executor =
     ThreadUtils.newDaemonSingleThreadScheduledExecutor("spark-dynamic-executor-allocation")
 
   // Metric source for ExecutorAllocationManager to expose internal status to MetricsSystem.
-  val executorAllocationManagerSource = new ExecutorAllocationManagerSource
+  val executorAllocationManagerSource = new ExecutorAllocationManagerSource(this)
+
+  val executorMonitor =
+    new ExecutorMonitor(conf, client, listenerBus, clock, executorAllocationManagerSource)
 
   // Whether we are still waiting for the initial set of executors to be allocated.
   // While this is true, we will not cancel outstanding executor requests. This is
@@ -204,7 +204,12 @@ private[spark] class ExecutorAllocationManager(
         s"s${DYN_ALLOCATION_SUSTAINED_SCHEDULER_BACKLOG_TIMEOUT.key} must be > 0!")
     }
     if (!conf.get(config.SHUFFLE_SERVICE_ENABLED)) {
-      if (conf.get(config.DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED)) {
+      // If dynamic allocation shuffle tracking or worker decommissioning along with
+      // storage shuffle decommissioning is enabled we have *experimental* support for
+      // decommissioning without a shuffle service.
+      if (conf.get(config.DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED) ||
+          (decommissionEnabled &&
+            conf.get(config.STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED))) {
         logWarning("Dynamic allocation without a shuffle service is an experimental feature.")
       } else if (!testing) {
         throw new SparkException("Dynamic allocation of executors requires the external " +
@@ -239,9 +244,12 @@ private[spark] class ExecutorAllocationManager(
         }
       }
     }
-    executor.scheduleWithFixedDelay(scheduleTask, 0, intervalMillis, TimeUnit.MILLISECONDS)
 
-    // copy the maps inside synchonize to ensure not being modified
+    if (!testing || conf.get(TEST_DYNAMIC_ALLOCATION_SCHEDULE_ENABLED)) {
+      executor.scheduleWithFixedDelay(scheduleTask, 0, intervalMillis, TimeUnit.MILLISECONDS)
+    }
+
+    // copy the maps inside synchronize to ensure not being modified
     val (numExecutorsTarget, numLocalityAware) = synchronized {
       val numTarget = numExecutorsTargetPerResourceProfileId.toMap
       val numLocality = numLocalityAwareTasksPerResourceProfileId.toMap
@@ -271,6 +279,9 @@ private[spark] class ExecutorAllocationManager(
     numExecutorsTargetPerResourceProfileId.keys.foreach { rpId =>
       numExecutorsTargetPerResourceProfileId(rpId) = initialNumExecutors
     }
+    numExecutorsToAddPerResourceProfileId.keys.foreach { rpId =>
+      numExecutorsToAddPerResourceProfileId(rpId) = 1
+    }
     executorMonitor.reset()
   }
 
@@ -278,24 +289,39 @@ private[spark] class ExecutorAllocationManager(
    * The maximum number of executors, for the ResourceProfile id passed in, that we would need
    * under the current load to satisfy all running and pending tasks, rounded up.
    */
-  private def maxNumExecutorsNeededPerResourceProfile(rpId: Int): Int = {
-    val pending = listener.totalPendingTasksPerResourceProfile(rpId)
+  private[spark] def maxNumExecutorsNeededPerResourceProfile(rpId: Int): Int = {
+    val pendingTask = listener.pendingTasksPerResourceProfile(rpId)
     val pendingSpeculative = listener.pendingSpeculativeTasksPerResourceProfile(rpId)
+    val unschedulableTaskSets = listener.pendingUnschedulableTaskSetsPerResourceProfile(rpId)
     val running = listener.totalRunningTasksPerResourceProfile(rpId)
-    val numRunningOrPendingTasks = pending + running
+    val numRunningOrPendingTasks = pendingTask + pendingSpeculative + running
     val rp = resourceProfileManager.resourceProfileFromId(rpId)
     val tasksPerExecutor = rp.maxTasksPerExecutor(conf)
     logDebug(s"max needed for rpId: $rpId numpending: $numRunningOrPendingTasks," +
       s" tasksperexecutor: $tasksPerExecutor")
     val maxNeeded = math.ceil(numRunningOrPendingTasks * executorAllocationRatio /
       tasksPerExecutor).toInt
-    if (tasksPerExecutor > 1 && maxNeeded == 1 && pendingSpeculative > 0) {
+
+    val maxNeededWithSpeculationLocalityOffset =
+      if (tasksPerExecutor > 1 && maxNeeded == 1 && pendingSpeculative > 0) {
       // If we have pending speculative tasks and only need a single executor, allocate one more
       // to satisfy the locality requirements of speculation
       maxNeeded + 1
     } else {
       maxNeeded
     }
+
+    if (unschedulableTaskSets > 0) {
+      // Request additional executors to account for task sets having tasks that are unschedulable
+      // due to executors excluded for failures when the active executor count has already reached
+      // the max needed which we would normally get.
+      val maxNeededForUnschedulables = math.ceil(unschedulableTaskSets * executorAllocationRatio /
+        tasksPerExecutor).toInt
+      math.max(maxNeededWithSpeculationLocalityOffset,
+        executorMonitor.executorCountWithResourceProfile(rpId) + maxNeededForUnschedulables)
+    } else {
+      maxNeededWithSpeculationLocalityOffset
+    }
   }
 
   private def totalRunningTasksPerResourceProfile(id: Int): Int = synchronized {
@@ -354,7 +380,7 @@ private[spark] class ExecutorAllocationManager(
 
           // We lower the target number of executors but don't actively kill any yet.  Killing is
           // controlled separately by an idle timeout.  It's still helpful to reduce
-          // the target number in case an executor just happens to get lost (eg., bad hardware,
+          // the target number in case an executor just happens to get lost (e.g., bad hardware,
           // or the cluster manager preempts it) -- in that case, there is no point in trying
           // to immediately  get a new executor, since we wouldn't even use it yet.
           decrementExecutorsFromTarget(maxNeeded, rpId, updatesNeeded)
@@ -518,13 +544,15 @@ private[spark] class ExecutorAllocationManager(
         if (testing) {
           throw new SparkException("ResourceProfile Id was UNKNOWN, this is not expected")
         }
-        logWarning(s"Not removing executor $executorIdsToBeRemoved because the " +
+        logWarning(s"Not removing executor $executorIdToBeRemoved because the " +
           "ResourceProfile was UNKNOWN!")
       } else {
         // get the running total as we remove or initialize it to the count - pendingRemoval
         val newExecutorTotal = numExecutorsTotalPerRpId.getOrElseUpdate(rpId,
           (executorMonitor.executorCountWithResourceProfile(rpId) -
-            executorMonitor.pendingRemovalCountPerResourceProfileId(rpId)))
+            executorMonitor.pendingRemovalCountPerResourceProfileId(rpId) -
+            executorMonitor.decommissioningPerResourceProfileId(rpId)
+          ))
         if (newExecutorTotal - 1 < minNumExecutors) {
           logDebug(s"Not removing idle executor $executorIdToBeRemoved because there " +
             s"are only $newExecutorTotal executor(s) left (minimum number of executor limit " +
@@ -550,8 +578,17 @@ private[spark] class ExecutorAllocationManager(
     } else {
       // We don't want to change our target number of executors, because we already did that
       // when the task backlog decreased.
-      client.killExecutors(executorIdsToBeRemoved, adjustTargetNumExecutors = false,
-        countFailures = false, force = false)
+      if (decommissionEnabled) {
+        val executorIdsWithoutHostLoss = executorIdsToBeRemoved.toSeq.map(
+          id => (id, ExecutorDecommissionInfo("spark scale down"))).toArray
+        client.decommissionExecutors(
+          executorIdsWithoutHostLoss,
+          adjustTargetNumExecutors = false,
+          triggeredByExecutor = false)
+      } else {
+        client.killExecutors(executorIdsToBeRemoved.toSeq, adjustTargetNumExecutors = false,
+          countFailures = false, force = false)
+      }
     }
 
     // [SPARK-21834] killExecutors api reduces the target number of executors.
@@ -563,9 +600,13 @@ private[spark] class ExecutorAllocationManager(
 
     // reset the newExecutorTotal to the existing number of executors
     if (testing || executorsRemoved.nonEmpty) {
-      executorMonitor.executorsKilled(executorsRemoved)
+      if (decommissionEnabled) {
+        executorMonitor.executorsDecommissioned(executorsRemoved.toSeq)
+      } else {
+        executorMonitor.executorsKilled(executorsRemoved.toSeq)
+      }
       logInfo(s"Executors ${executorsRemoved.mkString(",")} removed due to idle timeout.")
-      executorsRemoved
+      executorsRemoved.toSeq
     } else {
       logWarning(s"Unable to reach the cluster manager to kill executor/s " +
         s"${executorIdsToBeRemoved.mkString(",")} or no executor eligible to kill!")
@@ -622,6 +663,12 @@ private[spark] class ExecutorAllocationManager(
     private val resourceProfileIdToStageAttempt =
       new mutable.HashMap[Int, mutable.Set[StageAttempt]]
 
+    // Keep track of unschedulable task sets because of executor/node exclusions from too many task
+    // failures. This is a Set of StageAttempt's because we'll only take the last unschedulable task
+    // in a taskset although there can be more. This is done in order to avoid costly loops in the
+    // scheduling. Check TaskSetManager#getCompletelyExcludedTaskIfAny for more details.
+    private val unschedulableTaskSets = new mutable.HashSet[StageAttempt]
+
     // stageAttempt to tuple (the number of task with locality preferences, a map where each pair
     // is a node and the number of tasks that would like to be scheduled on that node, and
     // the resource profile id) map,
@@ -752,7 +799,11 @@ private[spark] class ExecutorAllocationManager(
         }
         if (taskEnd.taskInfo.speculative) {
           stageAttemptToSpeculativeTaskIndices.get(stageAttempt).foreach {_.remove{taskIndex}}
-          stageAttemptToNumSpeculativeTasks(stageAttempt) -= 1
+          // If the previous task attempt succeeded first and it was the last task in a stage,
+          // the stage may have been removed before handing this speculative TaskEnd event.
+          if (stageAttemptToNumSpeculativeTasks.contains(stageAttempt)) {
+            stageAttemptToNumSpeculativeTasks(stageAttempt) -= 1
+          }
         }
 
         taskEnd.reason match {
@@ -789,6 +840,28 @@ private[spark] class ExecutorAllocationManager(
       }
     }
 
+    override def onUnschedulableTaskSetAdded(
+        unschedulableTaskSetAdded: SparkListenerUnschedulableTaskSetAdded): Unit = {
+      val stageId = unschedulableTaskSetAdded.stageId
+      val stageAttemptId = unschedulableTaskSetAdded.stageAttemptId
+      val stageAttempt = StageAttempt(stageId, stageAttemptId)
+      allocationManager.synchronized {
+        unschedulableTaskSets.add(stageAttempt)
+        allocationManager.onSchedulerBacklogged()
+      }
+    }
+
+    override def onUnschedulableTaskSetRemoved(
+        unschedulableTaskSetRemoved: SparkListenerUnschedulableTaskSetRemoved): Unit = {
+      val stageId = unschedulableTaskSetRemoved.stageId
+      val stageAttemptId = unschedulableTaskSetRemoved.stageAttemptId
+      val stageAttempt = StageAttempt(stageId, stageAttemptId)
+      allocationManager.synchronized {
+        // Clear unschedulableTaskSets since atleast one task becomes schedulable now
+        unschedulableTaskSets.remove(stageAttempt)
+      }
+    }
+
     /**
      * An estimate of the total number of pending tasks remaining for currently running stages. Does
      * not account for tasks which may have failed and been resubmitted.
@@ -829,20 +902,18 @@ private[spark] class ExecutorAllocationManager(
       numTotalTasks - numRunning
     }
 
-    def hasPendingTasks: Boolean = {
-      hasPendingSpeculativeTasks || hasPendingRegularTasks
-    }
-
-    def totalPendingTasksPerResourceProfile(rp: Int): Int = {
-      pendingTasksPerResourceProfile(rp) + pendingSpeculativeTasksPerResourceProfile(rp)
-    }
-
     /**
-     * The number of tasks currently running across all stages.
-     * Include running-but-zombie stage attempts
+     * Currently we only know when a task set has an unschedulable task, we don't know
+     * the exact number and since the allocation manager isn't tied closely with the scheduler,
+     * we use the number of tasks sets that are unschedulable as a heuristic to add more executors.
      */
-    def totalRunningTasks(): Int = {
-      stageAttemptToNumRunningTask.values.sum
+    def pendingUnschedulableTaskSetsPerResourceProfile(rp: Int): Int = {
+      val attempts = resourceProfileIdToStageAttempt.getOrElse(rp, Set.empty).toSeq
+      attempts.count(attempt => unschedulableTaskSets.contains(attempt))
+    }
+
+    def hasPendingTasks: Boolean = {
+      hasPendingSpeculativeTasks || hasPendingRegularTasks
     }
 
     def totalRunningTasksPerResourceProfile(rp: Int): Int = {
@@ -885,34 +956,47 @@ private[spark] class ExecutorAllocationManager(
         rplocalityToCount.map { case (k, v) => (k, v.toMap)}.toMap
     }
   }
+}
 
-  /**
-   * Metric source for ExecutorAllocationManager to expose its internal executor allocation
-   * status to MetricsSystem.
-   * Note: These metrics heavily rely on the internal implementation of
-   * ExecutorAllocationManager, metrics or value of metrics will be changed when internal
-   * implementation is changed, so these metrics are not stable across Spark version.
-   */
-  private[spark] class ExecutorAllocationManagerSource extends Source {
-    val sourceName = "ExecutorAllocationManager"
-    val metricRegistry = new MetricRegistry()
-
-    private def registerGauge[T](name: String, value: => T, defaultValue: T): Unit = {
-      metricRegistry.register(MetricRegistry.name("executors", name), new Gauge[T] {
-        override def getValue: T = synchronized { Option(value).getOrElse(defaultValue) }
-      })
-    }
-
-    // The metrics are going to return the sum for all the different ResourceProfiles.
-    registerGauge("numberExecutorsToAdd",
-      numExecutorsToAddPerResourceProfileId.values.sum, 0)
-    registerGauge("numberExecutorsPendingToRemove", executorMonitor.pendingRemovalCount, 0)
-    registerGauge("numberAllExecutors", executorMonitor.executorCount, 0)
-    registerGauge("numberTargetExecutors",
-      numExecutorsTargetPerResourceProfileId.values.sum, 0)
-    registerGauge("numberMaxNeededExecutors", numExecutorsTargetPerResourceProfileId.keys
-        .map(maxNumExecutorsNeededPerResourceProfile(_)).sum, 0)
+/**
+ * Metric source for ExecutorAllocationManager to expose its internal executor allocation
+ * status to MetricsSystem.
+ * Note: These metrics heavily rely on the internal implementation of
+ * ExecutorAllocationManager, metrics or value of metrics will be changed when internal
+ * implementation is changed, so these metrics are not stable across Spark version.
+ */
+private[spark] class ExecutorAllocationManagerSource(
+    executorAllocationManager: ExecutorAllocationManager) extends Source {
+  val sourceName = "ExecutorAllocationManager"
+  val metricRegistry = new MetricRegistry()
+
+  private def registerGauge[T](name: String, value: => T, defaultValue: T): Unit = {
+    metricRegistry.register(MetricRegistry.name("executors", name), new Gauge[T] {
+      override def getValue: T = synchronized { Option(value).getOrElse(defaultValue) }
+    })
   }
+
+  private def getCounter(name: String): Counter = {
+    metricRegistry.counter(MetricRegistry.name("executors", name))
+  }
+
+  val gracefullyDecommissioned: Counter = getCounter("numberExecutorsGracefullyDecommissioned")
+  val decommissionUnfinished: Counter = getCounter("numberExecutorsDecommissionUnfinished")
+  val driverKilled: Counter = getCounter("numberExecutorsKilledByDriver")
+  val exitedUnexpectedly: Counter = getCounter("numberExecutorsExitedUnexpectedly")
+
+  // The metrics are going to return the sum for all the different ResourceProfiles.
+  registerGauge("numberExecutorsToAdd",
+    executorAllocationManager.numExecutorsToAddPerResourceProfileId.values.sum, 0)
+  registerGauge("numberExecutorsPendingToRemove",
+    executorAllocationManager.executorMonitor.pendingRemovalCount, 0)
+  registerGauge("numberAllExecutors",
+    executorAllocationManager.executorMonitor.executorCount, 0)
+  registerGauge("numberTargetExecutors",
+    executorAllocationManager.numExecutorsTargetPerResourceProfileId.values.sum, 0)
+  registerGauge("numberMaxNeededExecutors",
+    executorAllocationManager.numExecutorsTargetPerResourceProfileId.keys
+      .map(executorAllocationManager.maxNumExecutorsNeededPerResourceProfile(_)).sum, 0)
 }
 
 private object ExecutorAllocationManager {
diff --git a/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala b/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala
index 2ac72e66d6f32..71419d5aea0b4 100644
--- a/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala
+++ b/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala
@@ -67,7 +67,7 @@ private[spark] case class HeartbeatResponse(reregisterBlockManager: Boolean)
 private[spark] class HeartbeatReceiver(sc: SparkContext, clock: Clock)
   extends SparkListener with ThreadSafeRpcEndpoint with Logging {
 
-  def this(sc: SparkContext) {
+  def this(sc: SparkContext) = {
     this(sc, new SystemClock)
   }
 
@@ -80,7 +80,9 @@ private[spark] class HeartbeatReceiver(sc: SparkContext, clock: Clock)
   // executor ID -> timestamp of when the last heartbeat from this executor was received
   private val executorLastSeen = new HashMap[String, Long]
 
-  private val executorTimeoutMs = sc.conf.get(config.STORAGE_BLOCKMANAGER_SLAVE_TIMEOUT)
+  private val executorTimeoutMs = sc.conf.get(
+    config.STORAGE_BLOCKMANAGER_HEARTBEAT_TIMEOUT
+  ).getOrElse(Utils.timeStringAsMs(s"${sc.conf.get(Network.NETWORK_TIMEOUT)}s"))
 
   private val checkTimeoutIntervalMs = sc.conf.get(Network.NETWORK_TIMEOUT_INTERVAL)
 
@@ -88,10 +90,10 @@ private[spark] class HeartbeatReceiver(sc: SparkContext, clock: Clock)
 
   require(checkTimeoutIntervalMs <= executorTimeoutMs,
     s"${Network.NETWORK_TIMEOUT_INTERVAL.key} should be less than or " +
-      s"equal to ${config.STORAGE_BLOCKMANAGER_SLAVE_TIMEOUT.key}.")
+      s"equal to ${config.STORAGE_BLOCKMANAGER_HEARTBEAT_TIMEOUT.key}.")
   require(executorHeartbeatIntervalMs <= executorTimeoutMs,
     s"${config.EXECUTOR_HEARTBEAT_INTERVAL.key} should be less than or " +
-      s"equal to ${config.STORAGE_BLOCKMANAGER_SLAVE_TIMEOUT.key}")
+      s"equal to ${config.STORAGE_BLOCKMANAGER_HEARTBEAT_TIMEOUT.key}")
 
   private var timeoutCheckingTask: ScheduledFuture[_] = null
 
@@ -126,6 +128,7 @@ private[spark] class HeartbeatReceiver(sc: SparkContext, clock: Clock)
 
     // Messages received from executors
     case heartbeat @ Heartbeat(executorId, accumUpdates, blockManagerId, executorUpdates) =>
+      var reregisterBlockManager = !sc.isStopped
       if (scheduler != null) {
         if (executorLastSeen.contains(executorId)) {
           executorLastSeen(executorId) = clock.getTimeMillis()
@@ -133,7 +136,8 @@ private[spark] class HeartbeatReceiver(sc: SparkContext, clock: Clock)
             override def run(): Unit = Utils.tryLogNonFatalError {
               val unknownExecutor = !scheduler.executorHeartbeatReceived(
                 executorId, accumUpdates, blockManagerId, executorUpdates)
-              val response = HeartbeatResponse(reregisterBlockManager = unknownExecutor)
+              reregisterBlockManager &= unknownExecutor
+              val response = HeartbeatResponse(reregisterBlockManager)
               context.reply(response)
             }
           })
@@ -143,14 +147,14 @@ private[spark] class HeartbeatReceiver(sc: SparkContext, clock: Clock)
           // not log warning here. Otherwise there may be a lot of noise especially if
           // we explicitly remove executors (SPARK-4134).
           logDebug(s"Received heartbeat from unknown executor $executorId")
-          context.reply(HeartbeatResponse(reregisterBlockManager = true))
+          context.reply(HeartbeatResponse(reregisterBlockManager))
         }
       } else {
         // Because Executor will sleep several seconds before sending the first "Heartbeat", this
         // case rarely happens. However, if it really happens, log it and ask the executor to
         // register itself again.
         logWarning(s"Dropping $heartbeat because TaskScheduler is not ready yet")
-        context.reply(HeartbeatResponse(reregisterBlockManager = true))
+        context.reply(HeartbeatResponse(reregisterBlockManager))
       }
   }
 
@@ -218,7 +222,8 @@ private[spark] class HeartbeatReceiver(sc: SparkContext, clock: Clock)
             sc.schedulerBackend match {
               case backend: CoarseGrainedSchedulerBackend =>
                 backend.driverEndpoint.send(RemoveExecutor(executorId,
-                  SlaveLost(s"Executor heartbeat timed out after ${now - lastSeenMs} ms")))
+                  ExecutorProcessLost(
+                    s"Executor heartbeat timed out after ${now - lastSeenMs} ms")))
 
               // LocalSchedulerBackend is used locally and only has one single executor
               case _: LocalSchedulerBackend =>
diff --git a/core/src/main/scala/org/apache/spark/MapOutputTracker.scala b/core/src/main/scala/org/apache/spark/MapOutputTracker.scala
index ec8621bc55cf3..b749d7e8626ba 100644
--- a/core/src/main/scala/org/apache/spark/MapOutputTracker.scala
+++ b/core/src/main/scala/org/apache/spark/MapOutputTracker.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark
 
-import java.io.{ByteArrayInputStream, ObjectInputStream, ObjectOutputStream}
+import java.io.{ByteArrayInputStream, IOException, ObjectInputStream, ObjectOutputStream}
 import java.util.concurrent.{ConcurrentHashMap, LinkedBlockingQueue, ThreadPoolExecutor, TimeUnit}
 import java.util.concurrent.locks.ReentrantReadWriteLock
 
@@ -29,13 +29,14 @@ import scala.reflect.ClassTag
 import scala.util.control.NonFatal
 
 import org.apache.commons.io.output.{ByteArrayOutputStream => ApacheByteArrayOutputStream}
+import org.roaringbitmap.RoaringBitmap
 
 import org.apache.spark.broadcast.{Broadcast, BroadcastManager}
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config._
 import org.apache.spark.io.CompressionCodec
 import org.apache.spark.rpc.{RpcCallContext, RpcEndpoint, RpcEndpointRef, RpcEnv}
-import org.apache.spark.scheduler.{ExecutorCacheTaskLocation, MapStatus}
+import org.apache.spark.scheduler.{MapStatus, MergeStatus, ShuffleOutputStatus}
 import org.apache.spark.shuffle.MetadataFetchFailedException
 import org.apache.spark.storage.{BlockId, BlockManagerId, ShuffleBlockId}
 import org.apache.spark.util._
@@ -49,7 +50,9 @@ import org.apache.spark.util._
  *
  * All public methods of this class are thread-safe.
  */
-private class ShuffleStatus(numPartitions: Int) {
+private class ShuffleStatus(
+    numPartitions: Int,
+    numReducers: Int = -1) extends Logging {
 
   private val (readLock, writeLock) = {
     val lock = new ReentrantReadWriteLock()
@@ -86,6 +89,19 @@ private class ShuffleStatus(numPartitions: Int) {
   // Exposed for testing
   val mapStatuses = new Array[MapStatus](numPartitions)
 
+  /**
+   * MergeStatus for each shuffle partition when push-based shuffle is enabled. The index of the
+   * array is the shuffle partition id (reduce id). Each value in the array is the MergeStatus for
+   * a shuffle partition, or null if not available. When push-based shuffle is enabled, this array
+   * provides a reducer oriented view of the shuffle status specifically for the results of
+   * merging shuffle partition blocks into per-partition merged shuffle files.
+   */
+  val mergeStatuses = if (numReducers > 0) {
+    new Array[MergeStatus](numReducers)
+  } else {
+    Array.empty[MergeStatus]
+  }
+
   /**
    * The cached result of serializing the map statuses array. This cache is lazily populated when
    * [[serializedMapStatus]] is called. The cache is invalidated when map outputs are removed.
@@ -100,14 +116,26 @@ private class ShuffleStatus(numPartitions: Int) {
    * broadcast variable in order to keep it from being garbage collected and to allow for it to be
    * explicitly destroyed later on when the ShuffleMapStage is garbage-collected.
    */
-  private[this] var cachedSerializedBroadcast: Broadcast[Array[Byte]] = _
+  private[spark] var cachedSerializedBroadcast: Broadcast[Array[Byte]] = _
+
+  /**
+   * Similar to cachedSerializedMapStatus and cachedSerializedBroadcast, but for MergeStatus.
+   */
+  private[this] var cachedSerializedMergeStatus: Array[Byte] = _
+
+  private[this] var cachedSerializedBroadcastMergeStatus: Broadcast[Array[Byte]] = _
 
   /**
    * Counter tracking the number of partitions that have output. This is a performance optimization
    * to avoid having to count the number of non-null entries in the `mapStatuses` array and should
    * be equivalent to`mapStatuses.count(_ ne null)`.
    */
-  private[this] var _numAvailableOutputs: Int = 0
+  private[this] var _numAvailableMapOutputs: Int = 0
+
+  /**
+   * Counter tracking the number of MergeStatus results received so far from the shuffle services.
+   */
+  private[this] var _numAvailableMergeResults: Int = 0
 
   /**
    * Register a map output. If there is already a registered location for the map output then it
@@ -115,30 +143,76 @@ private class ShuffleStatus(numPartitions: Int) {
    */
   def addMapOutput(mapIndex: Int, status: MapStatus): Unit = withWriteLock {
     if (mapStatuses(mapIndex) == null) {
-      _numAvailableOutputs += 1
+      _numAvailableMapOutputs += 1
       invalidateSerializedMapOutputStatusCache()
     }
     mapStatuses(mapIndex) = status
   }
 
+  /**
+   * Update the map output location (e.g. during migration).
+   */
+  def updateMapOutput(mapId: Long, bmAddress: BlockManagerId): Unit = withWriteLock {
+    try {
+      val mapStatusOpt = mapStatuses.find(_.mapId == mapId)
+      mapStatusOpt match {
+        case Some(mapStatus) =>
+          logInfo(s"Updating map output for ${mapId} to ${bmAddress}")
+          mapStatus.updateLocation(bmAddress)
+          invalidateSerializedMapOutputStatusCache()
+        case None =>
+          logWarning(s"Asked to update map output ${mapId} for untracked map status.")
+      }
+    } catch {
+      case e: java.lang.NullPointerException =>
+        logWarning(s"Unable to update map output for ${mapId}, status removed in-flight")
+    }
+  }
+
   /**
    * Remove the map output which was served by the specified block manager.
    * This is a no-op if there is no registered map output or if the registered output is from a
    * different block manager.
    */
   def removeMapOutput(mapIndex: Int, bmAddress: BlockManagerId): Unit = withWriteLock {
+    logDebug(s"Removing existing map output ${mapIndex} ${bmAddress}")
     if (mapStatuses(mapIndex) != null && mapStatuses(mapIndex).location == bmAddress) {
-      _numAvailableOutputs -= 1
+      _numAvailableMapOutputs -= 1
       mapStatuses(mapIndex) = null
       invalidateSerializedMapOutputStatusCache()
     }
   }
 
+  /**
+   * Register a merge result.
+   */
+  def addMergeResult(reduceId: Int, status: MergeStatus): Unit = withWriteLock {
+    if (mergeStatuses(reduceId) != status) {
+      _numAvailableMergeResults += 1
+      invalidateSerializedMergeOutputStatusCache()
+    }
+    mergeStatuses(reduceId) = status
+  }
+
+  // TODO support updateMergeResult for similar use cases as updateMapOutput
+
+  /**
+   * Remove the merge result which was served by the specified block manager.
+   */
+  def removeMergeResult(reduceId: Int, bmAddress: BlockManagerId): Unit = withWriteLock {
+    if (mergeStatuses(reduceId) != null && mergeStatuses(reduceId).location == bmAddress) {
+      _numAvailableMergeResults -= 1
+      mergeStatuses(reduceId) = null
+      invalidateSerializedMergeOutputStatusCache()
+    }
+  }
+
   /**
    * Removes all shuffle outputs associated with this host. Note that this will also remove
    * outputs which are served by an external shuffle server (if one exists).
    */
   def removeOutputsOnHost(host: String): Unit = withWriteLock {
+    logDebug(s"Removing outputs for host ${host}")
     removeOutputsByFilter(x => x.host == host)
   }
 
@@ -148,6 +222,7 @@ private class ShuffleStatus(numPartitions: Int) {
    * still registered with that execId.
    */
   def removeOutputsOnExecutor(execId: String): Unit = withWriteLock {
+    logDebug(s"Removing outputs for execId ${execId}")
     removeOutputsByFilter(x => x.executorId == execId)
   }
 
@@ -158,18 +233,33 @@ private class ShuffleStatus(numPartitions: Int) {
   def removeOutputsByFilter(f: BlockManagerId => Boolean): Unit = withWriteLock {
     for (mapIndex <- mapStatuses.indices) {
       if (mapStatuses(mapIndex) != null && f(mapStatuses(mapIndex).location)) {
-        _numAvailableOutputs -= 1
+        _numAvailableMapOutputs -= 1
         mapStatuses(mapIndex) = null
         invalidateSerializedMapOutputStatusCache()
       }
     }
+    for (reduceId <- mergeStatuses.indices) {
+      if (mergeStatuses(reduceId) != null && f(mergeStatuses(reduceId).location)) {
+        _numAvailableMergeResults -= 1
+        mergeStatuses(reduceId) = null
+        invalidateSerializedMergeOutputStatusCache()
+      }
+    }
   }
 
   /**
-   * Number of partitions that have shuffle outputs.
+   * Number of partitions that have shuffle map outputs.
    */
-  def numAvailableOutputs: Int = withReadLock {
-    _numAvailableOutputs
+  def numAvailableMapOutputs: Int = withReadLock {
+    _numAvailableMapOutputs
+  }
+
+  /**
+   * Number of shuffle partitions that have already been merge finalized when push-based
+   * is enabled.
+   */
+  def numAvailableMergeResults: Int = withReadLock {
+    _numAvailableMergeResults
   }
 
   /**
@@ -177,19 +267,19 @@ private class ShuffleStatus(numPartitions: Int) {
    */
   def findMissingPartitions(): Seq[Int] = withReadLock {
     val missing = (0 until numPartitions).filter(id => mapStatuses(id) == null)
-    assert(missing.size == numPartitions - _numAvailableOutputs,
-      s"${missing.size} missing, expected ${numPartitions - _numAvailableOutputs}")
+    assert(missing.size == numPartitions - _numAvailableMapOutputs,
+      s"${missing.size} missing, expected ${numPartitions - _numAvailableMapOutputs}")
     missing
   }
 
   /**
    * Serializes the mapStatuses array into an efficient compressed format. See the comments on
-   * `MapOutputTracker.serializeMapStatuses()` for more details on the serialization format.
+   * `MapOutputTracker.serializeOutputStatuses()` for more details on the serialization format.
    *
    * This method is designed to be called multiple times and implements caching in order to speed
    * up subsequent requests. If the cache is empty and multiple threads concurrently attempt to
-   * serialize the map statuses then serialization will only be performed in a single thread and all
-   * other threads will block until the cache is populated.
+   * serialize the map statuses then serialization will only be performed in a single thread and
+   * all other threads will block until the cache is populated.
    */
   def serializedMapStatus(
       broadcastManager: BroadcastManager,
@@ -197,7 +287,6 @@ private class ShuffleStatus(numPartitions: Int) {
       minBroadcastSize: Int,
       conf: SparkConf): Array[Byte] = {
     var result: Array[Byte] = null
-
     withReadLock {
       if (cachedSerializedMapStatus != null) {
         result = cachedSerializedMapStatus
@@ -206,7 +295,7 @@ private class ShuffleStatus(numPartitions: Int) {
 
     if (result == null) withWriteLock {
       if (cachedSerializedMapStatus == null) {
-        val serResult = MapOutputTracker.serializeMapStatuses(
+        val serResult = MapOutputTracker.serializeOutputStatuses[MapStatus](
           mapStatuses, broadcastManager, isLocal, minBroadcastSize, conf)
         cachedSerializedMapStatus = serResult._1
         cachedSerializedBroadcast = serResult._2
@@ -218,6 +307,47 @@ private class ShuffleStatus(numPartitions: Int) {
     result
   }
 
+  /**
+   * Serializes the mapStatuses and mergeStatuses array into an efficient compressed format.
+   * See the comments on `MapOutputTracker.serializeOutputStatuses()` for more details
+   * on the serialization format.
+   *
+   * This method is designed to be called multiple times and implements caching in order to speed
+   * up subsequent requests. If the cache is empty and multiple threads concurrently attempt to
+   * serialize the statuses array then serialization will only be performed in a single thread and
+   * all other threads will block until the cache is populated.
+   */
+  def serializedMapAndMergeStatus(
+      broadcastManager: BroadcastManager,
+      isLocal: Boolean,
+      minBroadcastSize: Int,
+      conf: SparkConf): (Array[Byte], Array[Byte]) = {
+    val mapStatusesBytes: Array[Byte] =
+      serializedMapStatus(broadcastManager, isLocal, minBroadcastSize, conf)
+    var mergeStatusesBytes: Array[Byte] = null
+
+    withReadLock {
+      if (cachedSerializedMergeStatus != null) {
+        mergeStatusesBytes = cachedSerializedMergeStatus
+      }
+    }
+
+    if (mergeStatusesBytes == null) withWriteLock {
+      if (cachedSerializedMergeStatus == null) {
+        val serResult = MapOutputTracker.serializeOutputStatuses[MergeStatus](
+          mergeStatuses, broadcastManager, isLocal, minBroadcastSize, conf)
+        cachedSerializedMergeStatus = serResult._1
+        cachedSerializedBroadcastMergeStatus = serResult._2
+      }
+
+      // The following line has to be outside if statement since it's possible that another
+      // thread initializes cachedSerializedMergeStatus in-between `withReadLock` and
+      // `withWriteLock`.
+      mergeStatusesBytes = cachedSerializedMergeStatus
+    }
+    (mapStatusesBytes, mergeStatusesBytes)
+  }
+
   // Used in testing.
   def hasCachedSerializedBroadcast: Boolean = withReadLock {
     cachedSerializedBroadcast != null
@@ -231,6 +361,10 @@ private class ShuffleStatus(numPartitions: Int) {
     f(mapStatuses)
   }
 
+  def withMergeStatuses[T](f: Array[MergeStatus] => T): T = withReadLock {
+    f(mergeStatuses)
+  }
+
   /**
    * Clears the cached serialized map output statuses.
    */
@@ -246,14 +380,35 @@ private class ShuffleStatus(numPartitions: Int) {
     }
     cachedSerializedMapStatus = null
   }
+
+  /**
+   * Clears the cached serialized merge result statuses.
+   */
+  def invalidateSerializedMergeOutputStatusCache(): Unit = withWriteLock {
+    if (cachedSerializedBroadcastMergeStatus != null) {
+      Utils.tryLogNonFatalError {
+        // Use `blocking = false` so that this operation doesn't hang while trying to send cleanup
+        // RPCs to dead executors.
+        cachedSerializedBroadcastMergeStatus.destroy()
+      }
+      cachedSerializedBroadcastMergeStatus = null
+    }
+    cachedSerializedMergeStatus = null
+  }
 }
 
 private[spark] sealed trait MapOutputTrackerMessage
 private[spark] case class GetMapOutputStatuses(shuffleId: Int)
   extends MapOutputTrackerMessage
+private[spark] case class GetMapAndMergeResultStatuses(shuffleId: Int)
+  extends MapOutputTrackerMessage
 private[spark] case object StopMapOutputTracker extends MapOutputTrackerMessage
 
-private[spark] case class GetMapOutputMessage(shuffleId: Int, context: RpcCallContext)
+private[spark] sealed trait MapOutputTrackerMasterMessage
+private[spark] case class GetMapOutputMessage(shuffleId: Int,
+  context: RpcCallContext) extends MapOutputTrackerMasterMessage
+private[spark] case class GetMapAndMergeOutputMessage(shuffleId: Int,
+  context: RpcCallContext) extends MapOutputTrackerMasterMessage
 
 /** RpcEndpoint class for MapOutputTrackerMaster */
 private[spark] class MapOutputTrackerMasterEndpoint(
@@ -265,8 +420,13 @@ private[spark] class MapOutputTrackerMasterEndpoint(
   override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
     case GetMapOutputStatuses(shuffleId: Int) =>
       val hostPort = context.senderAddress.hostPort
-      logInfo("Asked to send map output locations for shuffle " + shuffleId + " to " + hostPort)
-      tracker.post(new GetMapOutputMessage(shuffleId, context))
+      logInfo(s"Asked to send map output locations for shuffle $shuffleId to $hostPort")
+      tracker.post(GetMapOutputMessage(shuffleId, context))
+
+    case GetMapAndMergeResultStatuses(shuffleId: Int) =>
+      val hostPort = context.senderAddress.hostPort
+      logInfo(s"Asked to send map/merge result locations for shuffle $shuffleId to $hostPort")
+      tracker.post(GetMapAndMergeOutputMessage(shuffleId, context))
 
     case StopMapOutputTracker =>
       logInfo("MapOutputTrackerMasterEndpoint stopped!")
@@ -322,41 +482,61 @@ private[spark] abstract class MapOutputTracker(conf: SparkConf) extends Logging
   // For testing
   def getMapSizesByExecutorId(shuffleId: Int, reduceId: Int)
       : Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])] = {
-    getMapSizesByExecutorId(shuffleId, reduceId, reduceId + 1)
+    getMapSizesByExecutorId(shuffleId, 0, Int.MaxValue, reduceId, reduceId + 1)
   }
 
   /**
    * Called from executors to get the server URIs and output sizes for each shuffle block that
    * needs to be read from a given range of map output partitions (startPartition is included but
-   * endPartition is excluded from the range).
+   * endPartition is excluded from the range) within a range of mappers (startMapIndex is included
+   * but endMapIndex is excluded). If endMapIndex=Int.MaxValue, the actual endMapIndex will be
+   * changed to the length of total map outputs.
    *
    * @return A sequence of 2-item tuples, where the first item in the tuple is a BlockManagerId,
    *         and the second item is a sequence of (shuffle block id, shuffle block size, map index)
    *         tuples describing the shuffle blocks that are stored at that block manager.
+   *         Note that zero-sized blocks are excluded in the result.
    */
   def getMapSizesByExecutorId(
       shuffleId: Int,
+      startMapIndex: Int,
+      endMapIndex: Int,
       startPartition: Int,
-      endPartition: Int)
-  : Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])]
+      endPartition: Int): Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])]
 
   /**
-   * Called from executors to get the server URIs and output sizes for each shuffle block that
-   * needs to be read from a given range of map output partitions (startPartition is included but
-   * endPartition is excluded from the range) and is produced by
-   * a range of mappers (startMapIndex, endMapIndex, startMapIndex is included and
-   * the endMapIndex is excluded).
+   * Called from executors upon fetch failure on an entire merged shuffle reduce partition.
+   * Such failures can happen if the shuffle client fails to fetch the metadata for the given
+   * merged shuffle partition. This method is to get the server URIs and output sizes for each
+   * shuffle block that is merged in the specified merged shuffle block so fetch failure on a
+   * merged shuffle block can fall back to fetching the unmerged blocks.
    *
    * @return A sequence of 2-item tuples, where the first item in the tuple is a BlockManagerId,
-   *         and the second item is a sequence of (shuffle block id, shuffle block size, map index)
+   *         and the second item is a sequence of (shuffle block ID, shuffle block size, map index)
    *         tuples describing the shuffle blocks that are stored at that block manager.
    */
-  def getMapSizesByRange(
+  def getMapSizesForMergeResult(
       shuffleId: Int,
-      startMapIndex: Int,
-      endMapIndex: Int,
-      startPartition: Int,
-      endPartition: Int): Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])]
+      partitionId: Int): Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])]
+
+  /**
+   * Called from executors upon fetch failure on a merged shuffle reduce partition chunk. This is
+   * to get the server URIs and output sizes for each shuffle block that is merged in the specified
+   * merged shuffle partition chunk so fetch failure on a merged shuffle block chunk can fall back
+   * to fetching the unmerged blocks.
+   *
+   * chunkBitMap tracks the mapIds which are part of the current merged chunk, this way if there is
+   * a fetch failure on the merged chunk, it can fallback to fetching the corresponding original
+   * blocks part of this merged chunk.
+   *
+   * @return A sequence of 2-item tuples, where the first item in the tuple is a BlockManagerId,
+   *         and the second item is a sequence of (shuffle block ID, shuffle block size, map index)
+   *         tuples describing the shuffle blocks that are stored at that block manager.
+   */
+  def getMapSizesForMergeResult(
+      shuffleId: Int,
+      partitionId: Int,
+      chunkBitmap: RoaringBitmap): Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])]
 
   /**
    * Deletes map output status information for the specified shuffle stage.
@@ -406,8 +586,11 @@ private[spark] class MapOutputTrackerMaster(
 
   private val maxRpcMessageSize = RpcUtils.maxMessageSizeBytes(conf)
 
-  // requests for map output statuses
-  private val mapOutputRequests = new LinkedBlockingQueue[GetMapOutputMessage]
+  // requests for MapOutputTrackerMasterMessages
+  private val mapOutputTrackerMasterMessages =
+    new LinkedBlockingQueue[MapOutputTrackerMasterMessage]
+
+  private val pushBasedShuffleEnabled = Utils.isPushBasedShuffleEnabled(conf)
 
   // Thread pool used for handling map output status requests. This is a separate thread pool
   // to ensure we don't block the normal dispatcher threads.
@@ -430,31 +613,47 @@ private[spark] class MapOutputTrackerMaster(
     throw new IllegalArgumentException(msg)
   }
 
-  def post(message: GetMapOutputMessage): Unit = {
-    mapOutputRequests.offer(message)
+  def post(message: MapOutputTrackerMasterMessage): Unit = {
+    mapOutputTrackerMasterMessages.offer(message)
   }
 
   /** Message loop used for dispatching messages. */
   private class MessageLoop extends Runnable {
+    private def handleStatusMessage(
+        shuffleId: Int,
+        context: RpcCallContext,
+        needMergeOutput: Boolean): Unit = {
+      val hostPort = context.senderAddress.hostPort
+      val shuffleStatus = shuffleStatuses.get(shuffleId).head
+      logDebug(s"Handling request to send ${if (needMergeOutput) "map" else "map/merge"}" +
+        s" output locations for shuffle $shuffleId to $hostPort")
+      if (needMergeOutput) {
+        context.reply(
+          shuffleStatus.
+            serializedMapAndMergeStatus(broadcastManager, isLocal, minSizeForBroadcast, conf))
+      } else {
+        context.reply(
+          shuffleStatus.serializedMapStatus(broadcastManager, isLocal, minSizeForBroadcast, conf))
+      }
+    }
+
     override def run(): Unit = {
       try {
         while (true) {
           try {
-            val data = mapOutputRequests.take()
-             if (data == PoisonPill) {
+            val data = mapOutputTrackerMasterMessages.take()
+            if (data == PoisonPill) {
               // Put PoisonPill back so that other MessageLoops can see it.
-              mapOutputRequests.offer(PoisonPill)
+              mapOutputTrackerMasterMessages.offer(PoisonPill)
               return
             }
-            val context = data.context
-            val shuffleId = data.shuffleId
-            val hostPort = context.senderAddress.hostPort
-            logDebug("Handling request to send map output locations for shuffle " + shuffleId +
-              " to " + hostPort)
-            val shuffleStatus = shuffleStatuses.get(shuffleId).head
-            context.reply(
-              shuffleStatus.serializedMapStatus(broadcastManager, isLocal, minSizeForBroadcast,
-                conf))
+
+            data match {
+              case GetMapOutputMessage(shuffleId, context) =>
+                handleStatusMessage(shuffleId, context, false)
+              case GetMapAndMergeOutputMessage(shuffleId, context) =>
+                handleStatusMessage(shuffleId, context, true)
+            }
           } catch {
             case NonFatal(e) => logError(e.getMessage, e)
           }
@@ -466,16 +665,31 @@ private[spark] class MapOutputTrackerMaster(
   }
 
   /** A poison endpoint that indicates MessageLoop should exit its message loop. */
-  private val PoisonPill = new GetMapOutputMessage(-99, null)
+  private val PoisonPill = GetMapOutputMessage(-99, null)
 
   // Used only in unit tests.
   private[spark] def getNumCachedSerializedBroadcast: Int = {
     shuffleStatuses.valuesIterator.count(_.hasCachedSerializedBroadcast)
   }
 
-  def registerShuffle(shuffleId: Int, numMaps: Int): Unit = {
-    if (shuffleStatuses.put(shuffleId, new ShuffleStatus(numMaps)).isDefined) {
-      throw new IllegalArgumentException("Shuffle ID " + shuffleId + " registered twice")
+  def registerShuffle(shuffleId: Int, numMaps: Int, numReduces: Int): Unit = {
+    if (pushBasedShuffleEnabled) {
+      if (shuffleStatuses.put(shuffleId, new ShuffleStatus(numMaps, numReduces)).isDefined) {
+        throw new IllegalArgumentException("Shuffle ID " + shuffleId + " registered twice")
+      }
+    } else {
+      if (shuffleStatuses.put(shuffleId, new ShuffleStatus(numMaps)).isDefined) {
+        throw new IllegalArgumentException("Shuffle ID " + shuffleId + " registered twice")
+      }
+    }
+  }
+
+  def updateMapOutput(shuffleId: Int, mapId: Long, bmAddress: BlockManagerId): Unit = {
+    shuffleStatuses.get(shuffleId) match {
+      case Some(shuffleStatus) =>
+        shuffleStatus.updateMapOutput(mapId, bmAddress)
+      case None =>
+        logError(s"Asked to update map output for unknown shuffle ${shuffleId}")
     }
   }
 
@@ -506,10 +720,49 @@ private[spark] class MapOutputTrackerMaster(
     }
   }
 
+  def registerMergeResult(shuffleId: Int, reduceId: Int, status: MergeStatus) {
+    shuffleStatuses(shuffleId).addMergeResult(reduceId, status)
+  }
+
+  def registerMergeResults(shuffleId: Int, statuses: Seq[(Int, MergeStatus)]): Unit = {
+    statuses.foreach {
+      case (reduceId, status) => registerMergeResult(shuffleId, reduceId, status)
+    }
+  }
+
+  /**
+   * Unregisters a merge result corresponding to the reduceId if present. If the optional mapId
+   * is specified, it will only unregister the merge result if the mapId is part of that merge
+   * result.
+   *
+   * @param shuffleId the shuffleId.
+   * @param reduceId  the reduceId.
+   * @param bmAddress block manager address.
+   * @param mapId     the optional mapId which should be checked to see it was part of the merge
+   *                  result.
+   */
+  def unregisterMergeResult(
+    shuffleId: Int,
+    reduceId: Int,
+    bmAddress: BlockManagerId,
+    mapId: Option[Int] = None) {
+    shuffleStatuses.get(shuffleId) match {
+      case Some(shuffleStatus) =>
+        val mergeStatus = shuffleStatus.mergeStatuses(reduceId)
+        if (mergeStatus != null && (mapId.isEmpty || mergeStatus.tracker.contains(mapId.get))) {
+          shuffleStatus.removeMergeResult(reduceId, bmAddress)
+          incrementEpoch()
+        }
+      case None =>
+        throw new SparkException("unregisterMergeResult called for nonexistent shuffle ID")
+    }
+  }
+
   /** Unregister shuffle data */
   def unregisterShuffle(shuffleId: Int): Unit = {
     shuffleStatuses.remove(shuffleId).foreach { shuffleStatus =>
       shuffleStatus.invalidateSerializedMapOutputStatusCache()
+      shuffleStatus.invalidateSerializedMergeOutputStatusCache()
     }
   }
 
@@ -536,7 +789,12 @@ private[spark] class MapOutputTrackerMaster(
   def containsShuffle(shuffleId: Int): Boolean = shuffleStatuses.contains(shuffleId)
 
   def getNumAvailableOutputs(shuffleId: Int): Int = {
-    shuffleStatuses.get(shuffleId).map(_.numAvailableOutputs).getOrElse(0)
+    shuffleStatuses.get(shuffleId).map(_.numAvailableMapOutputs).getOrElse(0)
+  }
+
+  /** VisibleForTest. Invoked in test only. */
+  private[spark] def getNumAvailableMergeResults(shuffleId: Int): Int = {
+    shuffleStatuses.get(shuffleId).map(_.numAvailableMergeResults).getOrElse(0)
   }
 
   /**
@@ -615,7 +873,9 @@ private[spark] class MapOutputTrackerMaster(
 
   /**
    * Return the preferred hosts on which to run the given map output partition in a given shuffle,
-   * i.e. the nodes that the most outputs for that partition are on.
+   * i.e. the nodes that the most outputs for that partition are on. If the map output is
+   * pre-merged, then return the node where the merged block is located if the merge ratio is
+   * above the threshold.
    *
    * @param dep shuffle dependency object
    * @param partitionId map output partition that we want to read
@@ -623,15 +883,40 @@ private[spark] class MapOutputTrackerMaster(
    */
   def getPreferredLocationsForShuffle(dep: ShuffleDependency[_, _, _], partitionId: Int)
       : Seq[String] = {
-    if (shuffleLocalityEnabled && dep.rdd.partitions.length < SHUFFLE_PREF_MAP_THRESHOLD &&
-        dep.partitioner.numPartitions < SHUFFLE_PREF_REDUCE_THRESHOLD) {
-      val blockManagerIds = getLocationsWithLargestOutputs(dep.shuffleId, partitionId,
-        dep.partitioner.numPartitions, REDUCER_PREF_LOCS_FRACTION)
-      if (blockManagerIds.nonEmpty) {
-        blockManagerIds.get.map(_.host)
+    val shuffleStatus = shuffleStatuses.get(dep.shuffleId).orNull
+    if (shuffleStatus != null) {
+      // Check if the map output is pre-merged and if the merge ratio is above the threshold.
+      // If so, the location of the merged block is the preferred location.
+      val preferredLoc = if (pushBasedShuffleEnabled) {
+        shuffleStatus.withMergeStatuses { statuses =>
+          val status = statuses(partitionId)
+          val numMaps = dep.rdd.partitions.length
+          if (status != null && status.getNumMissingMapOutputs(numMaps).toDouble / numMaps
+            <= (1 - REDUCER_PREF_LOCS_FRACTION)) {
+            Seq(status.location.host)
+          } else {
+            Nil
+          }
+        }
       } else {
         Nil
       }
+      if (preferredLoc.nonEmpty) {
+        preferredLoc
+      } else {
+        if (shuffleLocalityEnabled && dep.rdd.partitions.length < SHUFFLE_PREF_MAP_THRESHOLD &&
+          dep.partitioner.numPartitions < SHUFFLE_PREF_REDUCE_THRESHOLD) {
+          val blockManagerIds = getLocationsWithLargestOutputs(dep.shuffleId, partitionId,
+            dep.partitioner.numPartitions, REDUCER_PREF_LOCS_FRACTION)
+          if (blockManagerIds.nonEmpty) {
+            blockManagerIds.get.map(_.host)
+          } else {
+            Nil
+          }
+        } else {
+          Nil
+        }
+      }
     } else {
       Nil
     }
@@ -734,48 +1019,54 @@ private[spark] class MapOutputTrackerMaster(
     }
   }
 
-  // Get blocks sizes by executor Id. Note that zero-sized blocks are excluded in the result.
   // This method is only called in local-mode.
   def getMapSizesByExecutorId(
-      shuffleId: Int,
-      startPartition: Int,
-      endPartition: Int)
-  : Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])] = {
-    logDebug(s"Fetching outputs for shuffle $shuffleId, partitions $startPartition-$endPartition")
-    shuffleStatuses.get(shuffleId) match {
-      case Some (shuffleStatus) =>
-        shuffleStatus.withMapStatuses { statuses =>
-          MapOutputTracker.convertMapStatuses(
-            shuffleId, startPartition, endPartition, statuses, 0, shuffleStatus.mapStatuses.length)
-        }
-      case None =>
-        Iterator.empty
-    }
-  }
-
-  override def getMapSizesByRange(
       shuffleId: Int,
       startMapIndex: Int,
       endMapIndex: Int,
       startPartition: Int,
       endPartition: Int): Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])] = {
-    logDebug(s"Fetching outputs for shuffle $shuffleId, mappers $startMapIndex-$endMapIndex" +
-      s"partitions $startPartition-$endPartition")
+    logDebug(s"Fetching outputs for shuffle $shuffleId")
     shuffleStatuses.get(shuffleId) match {
       case Some(shuffleStatus) =>
         shuffleStatus.withMapStatuses { statuses =>
+          val actualEndMapIndex = if (endMapIndex == Int.MaxValue) statuses.length else endMapIndex
+          logDebug(s"Convert map statuses for shuffle $shuffleId, " +
+            s"mappers $startMapIndex-$actualEndMapIndex, partitions $startPartition-$endPartition")
           MapOutputTracker.convertMapStatuses(
-            shuffleId, startPartition, endPartition, statuses, startMapIndex, endMapIndex)
+            shuffleId, startPartition, endPartition, statuses, startMapIndex, actualEndMapIndex)
         }
       case None =>
         Iterator.empty
     }
   }
 
+  // This method is only called in local-mode. Since push based shuffle won't be
+  // enabled in local-mode, this method returns empty list.
+  override def getMapSizesForMergeResult(
+      shuffleId: Int,
+      partitionId: Int): Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])] = {
+    Seq.empty.toIterator
+  }
+
+  // This method is only called in local-mode. Since push based shuffle won't be
+  // enabled in local-mode, this method returns empty list.
+  override def getMapSizesForMergeResult(
+      shuffleId: Int,
+      partitionId: Int,
+      chunkTracker: RoaringBitmap): Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])] = {
+    Seq.empty.toIterator
+  }
+
   override def stop(): Unit = {
-    mapOutputRequests.offer(PoisonPill)
+    mapOutputTrackerMasterMessages.offer(PoisonPill)
     threadpool.shutdown()
-    sendTracker(StopMapOutputTracker)
+    try {
+      sendTracker(StopMapOutputTracker)
+    } catch {
+      case e: SparkException =>
+        logError("Could not tell tracker we are stopping.", e)
+    }
     trackerEndpoint = null
     shuffleStatuses.clear()
   }
@@ -792,84 +1083,168 @@ private[spark] class MapOutputTrackerWorker(conf: SparkConf) extends MapOutputTr
   val mapStatuses: Map[Int, Array[MapStatus]] =
     new ConcurrentHashMap[Int, Array[MapStatus]]().asScala
 
+  val mergeStatuses: Map[Int, Array[MergeStatus]] =
+    new ConcurrentHashMap[Int, Array[MergeStatus]]().asScala
+
+  private val fetchMergeResult = Utils.isPushBasedShuffleEnabled(conf)
+
   /**
    * A [[KeyLock]] whose key is a shuffle id to ensure there is only one thread fetching
    * the same shuffle block.
    */
   private val fetchingLock = new KeyLock[Int]
 
-  // Get blocks sizes by executor Id. Note that zero-sized blocks are excluded in the result.
   override def getMapSizesByExecutorId(
       shuffleId: Int,
+      startMapIndex: Int,
+      endMapIndex: Int,
       startPartition: Int,
-      endPartition: Int)
-    : Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])] = {
-    logDebug(s"Fetching outputs for shuffle $shuffleId, partitions $startPartition-$endPartition")
-    val statuses = getStatuses(shuffleId, conf)
+      endPartition: Int): Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])] = {
+    logDebug(s"Fetching outputs for shuffle $shuffleId")
+    val (mapOutputStatuses, mergedOutputStatuses) = getStatuses(shuffleId, conf)
     try {
+      val actualEndMapIndex =
+        if (endMapIndex == Int.MaxValue) mapOutputStatuses.length else endMapIndex
+      logDebug(s"Convert map statuses for shuffle $shuffleId, " +
+        s"mappers $startMapIndex-$actualEndMapIndex, partitions $startPartition-$endPartition")
       MapOutputTracker.convertMapStatuses(
-        shuffleId, startPartition, endPartition, statuses, 0, statuses.length)
+        shuffleId, startPartition, endPartition, mapOutputStatuses, startMapIndex,
+          actualEndMapIndex, Option(mergedOutputStatuses))
     } catch {
       case e: MetadataFetchFailedException =>
         // We experienced a fetch failure so our mapStatuses cache is outdated; clear it:
         mapStatuses.clear()
+        mergeStatuses.clear()
         throw e
     }
   }
 
-  override def getMapSizesByRange(
+  override def getMapSizesForMergeResult(
       shuffleId: Int,
-      startMapIndex: Int,
-      endMapIndex: Int,
-      startPartition: Int,
-      endPartition: Int): Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])] = {
-    logDebug(s"Fetching outputs for shuffle $shuffleId, mappers $startMapIndex-$endMapIndex" +
-      s"partitions $startPartition-$endPartition")
-    val statuses = getStatuses(shuffleId, conf)
+      partitionId: Int): Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])] = {
+    logDebug(s"Fetching backup outputs for shuffle $shuffleId, partition $partitionId")
+    // Fetch the map statuses and merge statuses again since they might have already been
+    // cleared by another task running in the same executor.
+    val (mapOutputStatuses, mergeResultStatuses) = getStatuses(shuffleId, conf)
     try {
-      MapOutputTracker.convertMapStatuses(
-        shuffleId, startPartition, endPartition, statuses, startMapIndex, endMapIndex)
+      val mergeStatus = mergeResultStatuses(partitionId)
+      // If the original MergeStatus is no longer available, we cannot identify the list of
+      // unmerged blocks to fetch in this case. Throw MetadataFetchFailedException in this case.
+      MapOutputTracker.validateStatus(mergeStatus, shuffleId, partitionId)
+      // Use the MergeStatus's partition level bitmap since we are doing partition level fallback
+      MapOutputTracker.getMapStatusesForMergeStatus(shuffleId, partitionId,
+        mapOutputStatuses, mergeStatus.tracker)
     } catch {
+      // We experienced a fetch failure so our mapStatuses cache is outdated; clear it
+      case e: MetadataFetchFailedException =>
+        mapStatuses.clear()
+        mergeStatuses.clear()
+        throw e
+    }
+  }
+
+  override def getMapSizesForMergeResult(
+      shuffleId: Int,
+      partitionId: Int,
+      chunkTracker: RoaringBitmap): Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])] = {
+    logDebug(s"Fetching backup outputs for shuffle $shuffleId, partition $partitionId")
+    // Fetch the map statuses and merge statuses again since they might have already been
+    // cleared by another task running in the same executor.
+    val (mapOutputStatuses, _) = getStatuses(shuffleId, conf)
+    try {
+      MapOutputTracker.getMapStatusesForMergeStatus(shuffleId, partitionId, mapOutputStatuses,
+        chunkTracker)
+    } catch {
+      // We experienced a fetch failure so our mapStatuses cache is outdated; clear it:
       case e: MetadataFetchFailedException =>
-        // We experienced a fetch failure so our mapStatuses cache is outdated; clear it:
         mapStatuses.clear()
+        mergeStatuses.clear()
         throw e
     }
   }
 
   /**
-   * Get or fetch the array of MapStatuses for a given shuffle ID. NOTE: clients MUST synchronize
+   * Get or fetch the array of MapStatuses and MergeStatuses if push based shuffle enabled
+   * for a given shuffle ID. NOTE: clients MUST synchronize
    * on this array when reading it, because on the driver, we may be changing it in place.
    *
    * (It would be nice to remove this restriction in the future.)
    */
-  private def getStatuses(shuffleId: Int, conf: SparkConf): Array[MapStatus] = {
-    val statuses = mapStatuses.get(shuffleId).orNull
-    if (statuses == null) {
-      logInfo("Don't have map outputs for shuffle " + shuffleId + ", fetching them")
-      val startTimeNs = System.nanoTime()
-      fetchingLock.withLock(shuffleId) {
-        var fetchedStatuses = mapStatuses.get(shuffleId).orNull
-        if (fetchedStatuses == null) {
-          logInfo("Doing the fetch; tracker endpoint = " + trackerEndpoint)
-          val fetchedBytes = askTracker[Array[Byte]](GetMapOutputStatuses(shuffleId))
-          fetchedStatuses = MapOutputTracker.deserializeMapStatuses(fetchedBytes, conf)
-          logInfo("Got the output locations")
-          mapStatuses.put(shuffleId, fetchedStatuses)
+  private def getStatuses(
+      shuffleId: Int,
+      conf: SparkConf): (Array[MapStatus], Array[MergeStatus]) = {
+    if (fetchMergeResult) {
+      val mapOutputStatuses = mapStatuses.get(shuffleId).orNull
+      val mergeOutputStatuses = mergeStatuses.get(shuffleId).orNull
+
+      if (mapOutputStatuses == null || mergeOutputStatuses == null) {
+        logInfo("Don't have map/merge outputs for shuffle " + shuffleId + ", fetching them")
+        val startTimeNs = System.nanoTime()
+        fetchingLock.withLock(shuffleId) {
+          var fetchedMapStatuses = mapStatuses.get(shuffleId).orNull
+          var fetchedMergeStatuses = mergeStatuses.get(shuffleId).orNull
+          if (fetchedMapStatuses == null || fetchedMergeStatuses == null) {
+            logInfo("Doing the fetch; tracker endpoint = " + trackerEndpoint)
+            val fetchedBytes =
+              askTracker[(Array[Byte], Array[Byte])](GetMapAndMergeResultStatuses(shuffleId))
+            try {
+              fetchedMapStatuses =
+                MapOutputTracker.deserializeOutputStatuses[MapStatus](fetchedBytes._1, conf)
+              fetchedMergeStatuses =
+                MapOutputTracker.deserializeOutputStatuses[MergeStatus](fetchedBytes._2, conf)
+            } catch {
+              case e: SparkException =>
+                throw new MetadataFetchFailedException(shuffleId, -1,
+                  s"Unable to deserialize broadcasted map/merge statuses" +
+                    s" for shuffle $shuffleId: " + e.getCause)
+            }
+            logInfo("Got the map/merge output locations")
+            mapStatuses.put(shuffleId, fetchedMapStatuses)
+            mergeStatuses.put(shuffleId, fetchedMergeStatuses)
+          }
+          logDebug(s"Fetching map/merge output statuses for shuffle $shuffleId took " +
+            s"${TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNs)} ms")
+          (fetchedMapStatuses, fetchedMergeStatuses)
         }
-        logDebug(s"Fetching map output statuses for shuffle $shuffleId took " +
-          s"${TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNs)} ms")
-        fetchedStatuses
+      } else {
+        (mapOutputStatuses, mergeOutputStatuses)
       }
     } else {
-      statuses
+      val statuses = mapStatuses.get(shuffleId).orNull
+      if (statuses == null) {
+        logInfo("Don't have map outputs for shuffle " + shuffleId + ", fetching them")
+        val startTimeNs = System.nanoTime()
+        fetchingLock.withLock(shuffleId) {
+          var fetchedStatuses = mapStatuses.get(shuffleId).orNull
+          if (fetchedStatuses == null) {
+            logInfo("Doing the fetch; tracker endpoint = " + trackerEndpoint)
+            val fetchedBytes = askTracker[Array[Byte]](GetMapOutputStatuses(shuffleId))
+            try {
+              fetchedStatuses =
+                MapOutputTracker.deserializeOutputStatuses[MapStatus](fetchedBytes, conf)
+            } catch {
+              case e: SparkException =>
+                throw new MetadataFetchFailedException(shuffleId, -1,
+                  s"Unable to deserialize broadcasted map statuses for shuffle $shuffleId: " +
+                    e.getCause)
+            }
+            logInfo("Got the map output locations")
+            mapStatuses.put(shuffleId, fetchedStatuses)
+          }
+          logDebug(s"Fetching map output statuses for shuffle $shuffleId took " +
+            s"${TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNs)} ms")
+          (fetchedStatuses, null)
+        }
+      } else {
+        (statuses, null)
+      }
     }
   }
 
-
   /** Unregister shuffle data. */
   def unregisterShuffle(shuffleId: Int): Unit = {
     mapStatuses.remove(shuffleId)
+    mergeStatuses.remove(shuffleId)
   }
 
   /**
@@ -883,6 +1258,7 @@ private[spark] class MapOutputTrackerWorker(conf: SparkConf) extends MapOutputTr
         logInfo("Updating epoch to " + newEpoch + " and clearing cache")
         epoch = newEpoch
         mapStatuses.clear()
+        mergeStatuses.clear()
       }
     }
   }
@@ -894,11 +1270,13 @@ private[spark] object MapOutputTracker extends Logging {
   private val DIRECT = 0
   private val BROADCAST = 1
 
-  // Serialize an array of map output locations into an efficient byte format so that we can send
-  // it to reduce tasks. We do this by compressing the serialized bytes using Zstd. They will
-  // generally be pretty compressible because many map outputs will be on the same hostname.
-  def serializeMapStatuses(
-      statuses: Array[MapStatus],
+  private val SHUFFLE_PUSH_MAP_ID = -1
+
+  // Serialize an array of map/merge output locations into an efficient byte format so that we can
+  // send it to reduce tasks. We do this by compressing the serialized bytes using Zstd. They will
+  // generally be pretty compressible because many outputs will be on the same hostname.
+  def serializeOutputStatuses[T <: ShuffleOutputStatus](
+      statuses: Array[T],
       broadcastManager: BroadcastManager,
       isLocal: Boolean,
       minBroadcastSize: Int,
@@ -934,15 +1312,16 @@ private[spark] object MapOutputTracker extends Logging {
         oos.close()
       }
       val outArr = out.toByteArray
-      logInfo("Broadcast mapstatuses size = " + outArr.length + ", actual size = " + arr.length)
+      logInfo("Broadcast outputstatuses size = " + outArr.length + ", actual size = " + arr.length)
       (outArr, bcast)
     } else {
       (arr, null)
     }
   }
 
-  // Opposite of serializeMapStatuses.
-  def deserializeMapStatuses(bytes: Array[Byte], conf: SparkConf): Array[MapStatus] = {
+  // Opposite of serializeOutputStatuses.
+  def deserializeOutputStatuses[T <: ShuffleOutputStatus](
+      bytes: Array[Byte], conf: SparkConf): Array[T] = {
     assert (bytes.length > 0)
 
     def deserializeObject(arr: Array[Byte], off: Int, len: Int): AnyRef = {
@@ -961,15 +1340,23 @@ private[spark] object MapOutputTracker extends Logging {
 
     bytes(0) match {
       case DIRECT =>
-        deserializeObject(bytes, 1, bytes.length - 1).asInstanceOf[Array[MapStatus]]
+        deserializeObject(bytes, 1, bytes.length - 1).asInstanceOf[Array[T]]
       case BROADCAST =>
-        // deserialize the Broadcast, pull .value array out of it, and then deserialize that
-        val bcast = deserializeObject(bytes, 1, bytes.length - 1).
-          asInstanceOf[Broadcast[Array[Byte]]]
-        logInfo("Broadcast mapstatuses size = " + bytes.length +
-          ", actual size = " + bcast.value.length)
-        // Important - ignore the DIRECT tag ! Start from offset 1
-        deserializeObject(bcast.value, 1, bcast.value.length - 1).asInstanceOf[Array[MapStatus]]
+        try {
+          // deserialize the Broadcast, pull .value array out of it, and then deserialize that
+          val bcast = deserializeObject(bytes, 1, bytes.length - 1).
+            asInstanceOf[Broadcast[Array[Byte]]]
+          logInfo("Broadcast outputstatuses size = " + bytes.length +
+            ", actual size = " + bcast.value.length)
+          // Important - ignore the DIRECT tag ! Start from offset 1
+          deserializeObject(bcast.value, 1, bcast.value.length - 1).asInstanceOf[Array[T]]
+        } catch {
+          case e: IOException =>
+            logWarning("Exception encountered during deserializing broadcasted" +
+              " output statuses: ", e)
+            throw new SparkException("Unable to deserialize broadcasted" +
+              " output statuses", e)
+        }
       case _ => throw new IllegalArgumentException("Unexpected byte tag = " + bytes(0))
     }
   }
@@ -980,15 +1367,19 @@ private[spark] object MapOutputTracker extends Logging {
    * stored at that block manager.
    * Note that empty blocks are filtered in the result.
    *
+   * If push-based shuffle is enabled and an array of merge statuses is available, prioritize
+   * the locations of the merged shuffle partitions over unmerged shuffle blocks.
+   *
    * If any of the statuses is null (indicating a missing location due to a failed mapper),
    * throws a FetchFailedException.
    *
    * @param shuffleId Identifier for the shuffle
    * @param startPartition Start of map output partition ID range (included in range)
    * @param endPartition End of map output partition ID range (excluded from range)
-   * @param statuses List of map statuses, indexed by map partition index.
+   * @param mapStatuses List of map statuses, indexed by map partition index.
    * @param startMapIndex Start Map index.
    * @param endMapIndex End Map index.
+   * @param mergeStatuses List of merge statuses, index by reduce ID.
    * @return A sequence of 2-item tuples, where the first item in the tuple is a BlockManagerId,
    *         and the second item is a sequence of (shuffle block id, shuffle block size, map index)
    *         tuples describing the shuffle blocks that are stored at that block manager.
@@ -997,18 +1388,57 @@ private[spark] object MapOutputTracker extends Logging {
       shuffleId: Int,
       startPartition: Int,
       endPartition: Int,
-      statuses: Array[MapStatus],
+      mapStatuses: Array[MapStatus],
       startMapIndex : Int,
-      endMapIndex: Int): Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])] = {
-    assert (statuses != null)
+      endMapIndex: Int,
+      mergeStatuses: Option[Array[MergeStatus]] = None):
+      Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])] = {
+    assert (mapStatuses != null)
     val splitsByAddress = new HashMap[BlockManagerId, ListBuffer[(BlockId, Long, Int)]]
-    val iter = statuses.iterator.zipWithIndex
-    for ((status, mapIndex) <- iter.slice(startMapIndex, endMapIndex)) {
-      if (status == null) {
-        val errorMessage = s"Missing an output location for shuffle $shuffleId"
-        logError(errorMessage)
-        throw new MetadataFetchFailedException(shuffleId, startPartition, errorMessage)
-      } else {
+    // Only use MergeStatus for reduce tasks that fetch all map outputs. Since a merged shuffle
+    // partition consists of blocks merged in random order, we are unable to serve map index
+    // subrange requests. However, when a reduce task needs to fetch blocks from a subrange of
+    // map outputs, it usually indicates skewed partitions which push-based shuffle delegates
+    // to AQE to handle.
+    // TODO: SPARK-35036: Instead of reading map blocks in case of AQE with Push based shuffle,
+    // TODO: improve push based shuffle to read partial merged blocks satisfying the start/end
+    // TODO: map indexes
+    if (mergeStatuses.exists(_.nonEmpty) && startMapIndex == 0
+      && endMapIndex == mapStatuses.length) {
+      // We have MergeStatus and full range of mapIds are requested so return a merged block.
+      val numMaps = mapStatuses.length
+      mergeStatuses.get.zipWithIndex.slice(startPartition, endPartition).foreach {
+        case (mergeStatus, partId) =>
+          val remainingMapStatuses = if (mergeStatus != null && mergeStatus.totalSize > 0) {
+            // If MergeStatus is available for the given partition, add location of the
+            // pre-merged shuffle partition for this partition ID. Here we create a
+            // ShuffleBlockId with mapId being SHUFFLE_PUSH_MAP_ID to indicate this is
+            // a merged shuffle block.
+            splitsByAddress.getOrElseUpdate(mergeStatus.location, ListBuffer()) +=
+              ((ShuffleBlockId(shuffleId, SHUFFLE_PUSH_MAP_ID, partId), mergeStatus.totalSize, -1))
+            // For the "holes" in this pre-merged shuffle partition, i.e., unmerged mapper
+            // shuffle partition blocks, fetch the original map produced shuffle partition blocks
+            val mapStatusesWithIndex = mapStatuses.zipWithIndex
+            mergeStatus.getMissingMaps(numMaps).map(mapStatusesWithIndex)
+          } else {
+            // If MergeStatus is not available for the given partition, fall back to
+            // fetching all the original mapper shuffle partition blocks
+            mapStatuses.zipWithIndex.toSeq
+          }
+          // Add location for the mapper shuffle partition blocks
+          for ((mapStatus, mapIndex) <- remainingMapStatuses) {
+            validateStatus(mapStatus, shuffleId, partId)
+            val size = mapStatus.getSizeForBlock(partId)
+            if (size != 0) {
+              splitsByAddress.getOrElseUpdate(mapStatus.location, ListBuffer()) +=
+                ((ShuffleBlockId(shuffleId, mapStatus.mapId, partId), size, mapIndex))
+            }
+          }
+      }
+    } else {
+      val iter = mapStatuses.iterator.zipWithIndex
+      for ((status, mapIndex) <- iter.slice(startMapIndex, endMapIndex)) {
+        validateStatus(status, shuffleId, startPartition)
         for (part <- startPartition until endPartition) {
           val size = status.getSizeForBlock(part)
           if (size != 0) {
@@ -1019,6 +1449,49 @@ private[spark] object MapOutputTracker extends Logging {
       }
     }
 
-    splitsByAddress.iterator
+    splitsByAddress.mapValues(_.toSeq).iterator
+  }
+
+  /**
+   * Given a shuffle ID, a partition ID, an array of map statuses, and bitmap corresponding
+   * to either a merged shuffle partition or a merged shuffle partition chunk, identify
+   * the metadata about the shuffle partition blocks that are merged into the merged shuffle
+   * partition or partition chunk represented by the bitmap.
+   *
+   * @param shuffleId Identifier for the shuffle
+   * @param partitionId The partition ID of the MergeStatus for which we look for the metadata
+   *                    of the merged shuffle partition blocks
+   * @param mapStatuses List of map statuses, indexed by map ID
+   * @param tracker     bitmap containing mapIndexes that belong to the merged block or merged
+   *                    block chunk.
+   * @return A sequence of 2-item tuples, where the first item in the tuple is a BlockManagerId,
+   *         and the second item is a sequence of (shuffle block ID, shuffle block size) tuples
+   *         describing the shuffle blocks that are stored at that block manager.
+   */
+  def getMapStatusesForMergeStatus(
+      shuffleId: Int,
+      partitionId: Int,
+      mapStatuses: Array[MapStatus],
+      tracker: RoaringBitmap): Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])] = {
+    assert (mapStatuses != null && tracker != null)
+    val splitsByAddress = new HashMap[BlockManagerId, ListBuffer[(BlockId, Long, Int)]]
+    for ((status, mapIndex) <- mapStatuses.zipWithIndex) {
+      // Only add blocks that are merged
+      if (tracker.contains(mapIndex)) {
+        MapOutputTracker.validateStatus(status, shuffleId, partitionId)
+        splitsByAddress.getOrElseUpdate(status.location, ListBuffer()) +=
+          ((ShuffleBlockId(shuffleId, status.mapId, partitionId),
+            status.getSizeForBlock(partitionId), mapIndex))
+      }
+    }
+    splitsByAddress.mapValues(_.toSeq).iterator
+  }
+
+  def validateStatus(status: ShuffleOutputStatus, shuffleId: Int, partition: Int) : Unit = {
+    if (status == null) {
+      val errorMessage = s"Missing an output location for shuffle $shuffleId partition $partition"
+      logError(errorMessage)
+      throw new MetadataFetchFailedException(shuffleId, partition, errorMessage)
+    }
   }
 }
diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala b/core/src/main/scala/org/apache/spark/SparkConf.scala
index 40915e3904f7e..5f37a1abb1909 100644
--- a/core/src/main/scala/org/apache/spark/SparkConf.scala
+++ b/core/src/main/scala/org/apache/spark/SparkConf.scala
@@ -173,15 +173,6 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging with Seria
     this
   }
 
-  /**
-   * Set multiple parameters together
-   */
-  @deprecated("Use setAll(Iterable) instead", "3.0.0")
-  def setAll(settings: Traversable[(String, String)]): SparkConf = {
-    settings.foreach { case (k, v) => set(k, v) }
-    this
-  }
-
   /** Set a parameter if it isn't already configured */
   def setIfMissing(key: String, value: String): SparkConf = {
     if (settings.putIfAbsent(key, value) == null) {
@@ -577,7 +568,7 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging with Seria
     // If spark.executor.heartbeatInterval bigger than spark.network.timeout,
     // it will almost always cause ExecutorLostFailure. See SPARK-22754.
     require(executorTimeoutThresholdMs > executorHeartbeatIntervalMs, "The value of " +
-      s"${networkTimeout}=${executorTimeoutThresholdMs}ms must be no less than the value of " +
+      s"${networkTimeout}=${executorTimeoutThresholdMs}ms must be greater than the value of " +
       s"${EXECUTOR_HEARTBEAT_INTERVAL.key}=${executorHeartbeatIntervalMs}ms.")
   }
 
@@ -612,7 +603,7 @@ private[spark] object SparkConf extends Logging {
           "are no longer accepted. To specify the equivalent now, one may use '64k'."),
       DeprecatedConfig("spark.rpc", "2.0", "Not used anymore."),
       DeprecatedConfig("spark.scheduler.executorTaskBlacklistTime", "2.1.0",
-        "Please use the new blacklisting options, spark.blacklist.*"),
+        "Please use the new excludedOnFailure options, spark.excludeOnFailure.*"),
       DeprecatedConfig("spark.yarn.am.port", "2.0.0", "Not used anymore"),
       DeprecatedConfig("spark.executor.port", "2.0.0", "Not used anymore"),
       DeprecatedConfig("spark.shuffle.service.index.cache.entries", "2.3.0",
@@ -621,7 +612,31 @@ private[spark] object SparkConf extends Logging {
       DeprecatedConfig("spark.yarn.credentials.file.retention.days", "2.4.0", "Not used anymore."),
       DeprecatedConfig("spark.yarn.services", "3.0.0", "Feature no longer available."),
       DeprecatedConfig("spark.executor.plugins", "3.0.0",
-        "Feature replaced with new plugin API. See Monitoring documentation.")
+        "Feature replaced with new plugin API. See Monitoring documentation."),
+      DeprecatedConfig("spark.blacklist.enabled", "3.1.0",
+        "Please use spark.excludeOnFailure.enabled"),
+      DeprecatedConfig("spark.blacklist.task.maxTaskAttemptsPerExecutor", "3.1.0",
+        "Please use spark.excludeOnFailure.task.maxTaskAttemptsPerExecutor"),
+      DeprecatedConfig("spark.blacklist.task.maxTaskAttemptsPerNode", "3.1.0",
+        "Please use spark.excludeOnFailure.task.maxTaskAttemptsPerNode"),
+      DeprecatedConfig("spark.blacklist.application.maxFailedTasksPerExecutor", "3.1.0",
+        "Please use spark.excludeOnFailure.application.maxFailedTasksPerExecutor"),
+      DeprecatedConfig("spark.blacklist.stage.maxFailedTasksPerExecutor", "3.1.0",
+        "Please use spark.excludeOnFailure.stage.maxFailedTasksPerExecutor"),
+      DeprecatedConfig("spark.blacklist.application.maxFailedExecutorsPerNode", "3.1.0",
+        "Please use spark.excludeOnFailure.application.maxFailedExecutorsPerNode"),
+      DeprecatedConfig("spark.blacklist.stage.maxFailedExecutorsPerNode", "3.1.0",
+        "Please use spark.excludeOnFailure.stage.maxFailedExecutorsPerNode"),
+      DeprecatedConfig("spark.blacklist.timeout", "3.1.0",
+        "Please use spark.excludeOnFailure.timeout"),
+      DeprecatedConfig("spark.blacklist.application.fetchFailure.enabled", "3.1.0",
+        "Please use spark.excludeOnFailure.application.fetchFailure.enabled"),
+      DeprecatedConfig("spark.scheduler.blacklist.unschedulableTaskSetTimeout", "3.1.0",
+        "Please use spark.scheduler.excludeOnFailure.unschedulableTaskSetTimeout"),
+      DeprecatedConfig("spark.blacklist.killBlacklistedExecutors", "3.1.0",
+        "Please use spark.excludeOnFailure.killExcludedExecutors"),
+      DeprecatedConfig("spark.yarn.blacklist.executor.launch.blacklisting.enabled", "3.1.0",
+        "Please use spark.yarn.executor.launch.excludeOnFailure.enabled")
     )
 
     Map(configs.map { cfg => (cfg.key -> cfg) } : _*)
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index 5c92527b7b80e..ef47252189bbb 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -22,6 +22,7 @@ import java.net.URI
 import java.util.{Arrays, Locale, Properties, ServiceLoader, UUID}
 import java.util.concurrent.{ConcurrentHashMap, ConcurrentMap}
 import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger, AtomicReference}
+import javax.ws.rs.core.UriBuilder
 
 import scala.collection.JavaConverters._
 import scala.collection.Map
@@ -39,10 +40,10 @@ import org.apache.hadoop.mapred.{FileInputFormat, InputFormat, JobConf, Sequence
 import org.apache.hadoop.mapreduce.{InputFormat => NewInputFormat, Job => NewHadoopJob}
 import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat => NewFileInputFormat}
 
-import org.apache.spark.annotation.DeveloperApi
+import org.apache.spark.annotation.{DeveloperApi, Experimental}
 import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.deploy.{LocalSparkCluster, SparkHadoopUtil}
-import org.apache.spark.executor.{ExecutorMetrics, ExecutorMetricsSource}
+import org.apache.spark.executor.{Executor, ExecutorMetrics, ExecutorMetricsSource}
 import org.apache.spark.input.{FixedLengthBinaryInputFormat, PortableDataStream, StreamInputFormat, WholeTextFileInputFormat}
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config._
@@ -83,6 +84,11 @@ class SparkContext(config: SparkConf) extends Logging {
   // The call site where this SparkContext was constructed.
   private val creationSite: CallSite = Utils.getCallSite()
 
+  if (!config.get(EXECUTOR_ALLOW_SPARK_CONTEXT)) {
+    // In order to prevent SparkContext from being created in executors.
+    SparkContext.assertOnDriver()
+  }
+
   // In order to prevent multiple SparkContexts from being active at the same time, mark this
   // context as having started construction.
   // NOTE: this must be placed at the beginning of the SparkContext constructor.
@@ -216,6 +222,7 @@ class SparkContext(config: SparkConf) extends Logging {
   private var _listenerBusStarted: Boolean = false
   private var _jars: Seq[String] = _
   private var _files: Seq[String] = _
+  private var _archives: Seq[String] = _
   private var _shutdownHookRef: AnyRef = _
   private var _statusStore: AppStatusStore = _
   private var _heartbeater: Heartbeater = _
@@ -241,6 +248,7 @@ class SparkContext(config: SparkConf) extends Logging {
 
   def jars: Seq[String] = _jars
   def files: Seq[String] = _files
+  def archives: Seq[String] = _archives
   def master: String = _conf.get("spark.master")
   def deployMode: String = _conf.get(SUBMIT_DEPLOY_MODE)
   def appName: String = _conf.get("spark.app.name")
@@ -273,6 +281,7 @@ class SparkContext(config: SparkConf) extends Logging {
 
   // Used to store a URL for each static file/jar together with the file's local timestamp
   private[spark] val addedFiles = new ConcurrentHashMap[String, Long]().asScala
+  private[spark] val addedArchives = new ConcurrentHashMap[String, Long]().asScala
   private[spark] val addedJars = new ConcurrentHashMap[String, Long]().asScala
 
   // Keeps track of all persisted RDDs
@@ -379,6 +388,7 @@ class SparkContext(config: SparkConf) extends Logging {
   try {
     _conf = config.clone()
     _conf.validateSettings()
+    _conf.set("spark.app.startTime", startTime.toString)
 
     if (!_conf.contains("spark.master")) {
       throw new SparkException("A master URL must be set in your configuration")
@@ -386,6 +396,8 @@ class SparkContext(config: SparkConf) extends Logging {
     if (!_conf.contains("spark.app.name")) {
       throw new SparkException("An application name must be set in your configuration")
     }
+    // This should be set as early as possible.
+    SparkContext.fillMissingMagicCommitterConfsIfNeeded(_conf)
 
     _driverLogger = DriverLogger(_conf)
 
@@ -416,6 +428,7 @@ class SparkContext(config: SparkConf) extends Logging {
     _jars = Utils.getUserJars(_conf)
     _files = _conf.getOption(FILES.key).map(_.split(",")).map(_.filter(_.nonEmpty))
       .toSeq.flatten
+    _archives = _conf.getOption(ARCHIVES.key).map(Utils.stringToSeq).toSeq.flatten
 
     _eventLogDir =
       if (isEventLogEnabled) {
@@ -435,7 +448,7 @@ class SparkContext(config: SparkConf) extends Logging {
     }
 
     _listenerBus = new LiveListenerBus(_conf)
-    _resourceProfileManager = new ResourceProfileManager(_conf)
+    _resourceProfileManager = new ResourceProfileManager(_conf, _listenerBus)
 
     // Initialize the app status store and listener before SparkEnv is created so that it gets
     // all events.
@@ -487,11 +500,24 @@ class SparkContext(config: SparkConf) extends Logging {
 
     // Add each JAR given through the constructor
     if (jars != null) {
-      jars.foreach(addJar)
+      jars.foreach(jar => addJar(jar, true))
+      if (addedJars.nonEmpty) {
+        _conf.set("spark.app.initial.jar.urls", addedJars.keys.toSeq.mkString(","))
+      }
     }
 
     if (files != null) {
-      files.foreach(addFile)
+      files.foreach(file => addFile(file, false, true))
+      if (addedFiles.nonEmpty) {
+        _conf.set("spark.app.initial.file.urls", addedFiles.keys.toSeq.mkString(","))
+      }
+    }
+
+    if (archives != null) {
+      archives.foreach(file => addFile(file, false, true, isArchive = true))
+      if (addedArchives.nonEmpty) {
+        _conf.set("spark.app.initial.archive.urls", addedArchives.keys.toSeq.mkString(","))
+      }
     }
 
     _executorMemory = _conf.getOption(EXECUTOR_MEMORY.key)
@@ -558,10 +584,13 @@ class SparkContext(config: SparkConf) extends Logging {
     _applicationAttemptId = _taskScheduler.applicationAttemptId()
     _conf.set("spark.app.id", _applicationId)
     if (_conf.get(UI_REVERSE_PROXY)) {
-      System.setProperty("spark.ui.proxyBase", "/proxy/" + _applicationId)
+      val proxyUrl = _conf.get(UI_REVERSE_PROXY_URL.key, "").stripSuffix("/") +
+        "/proxy/" + _applicationId
+      System.setProperty("spark.ui.proxyBase", proxyUrl)
     }
     _ui.foreach(_.setAppId(_applicationId))
     _env.blockManager.initialize(_applicationId)
+    FallbackStorage.registerBlockManagerIfNeeded(_env.blockManager.master, _conf)
 
     // The metrics system for Driver need to be set spark.app.id to app ID.
     // So it should start after we get app ID from the task scheduler and set spark.app.id.
@@ -611,6 +640,9 @@ class SparkContext(config: SparkConf) extends Logging {
 
     // Post init
     _taskScheduler.postStartHook()
+    if (isLocal) {
+      _env.metricsSystem.registerSource(Executor.executorSourceLocalModeOnly)
+    }
     _env.metricsSystem.registerSource(_dagScheduler.metricsSource)
     _env.metricsSystem.registerSource(new BlockManagerSource(_env.blockManager))
     _env.metricsSystem.registerSource(new JVMCPUSource())
@@ -1495,7 +1527,7 @@ class SparkContext(config: SparkConf) extends Logging {
    * @note A path can be added only once. Subsequent additions of the same path are ignored.
    */
   def addFile(path: String): Unit = {
-    addFile(path, false)
+    addFile(path, false, false)
   }
 
   /**
@@ -1503,6 +1535,36 @@ class SparkContext(config: SparkConf) extends Logging {
    */
   def listFiles(): Seq[String] = addedFiles.keySet.toSeq
 
+  /**
+   * :: Experimental ::
+   * Add an archive to be downloaded and unpacked with this Spark job on every node.
+   *
+   * If an archive is added during execution, it will not be available until the next TaskSet
+   * starts.
+   *
+   * @param path can be either a local file, a file in HDFS (or other Hadoop-supported
+   * filesystems), or an HTTP, HTTPS or FTP URI. To access the file in Spark jobs,
+   * use `SparkFiles.get(paths-to-files)` to find its download/unpacked location.
+   * The given path should be one of .zip, .tar, .tar.gz, .tgz and .jar.
+   *
+   * @note A path can be added only once. Subsequent additions of the same path are ignored.
+   *
+   * @since 3.1.0
+   */
+  @Experimental
+  def addArchive(path: String): Unit = {
+    addFile(path, false, false, isArchive = true)
+  }
+
+  /**
+   * :: Experimental ::
+   * Returns a list of archive paths that are added to resources.
+   *
+   * @since 3.1.0
+   */
+  @Experimental
+  def listArchives(): Seq[String] = addedArchives.keySet.toSeq
+
   /**
    * Add a file to be downloaded with this Spark job on every node.
    *
@@ -1517,19 +1579,25 @@ class SparkContext(config: SparkConf) extends Logging {
    * @note A path can be added only once. Subsequent additions of the same path are ignored.
    */
   def addFile(path: String, recursive: Boolean): Unit = {
-    val uri = new Path(path).toUri
+    addFile(path, recursive, false)
+  }
+
+  private def addFile(
+      path: String, recursive: Boolean, addedOnSubmit: Boolean, isArchive: Boolean = false
+    ): Unit = {
+    val uri = Utils.resolveURI(path)
     val schemeCorrectedURI = uri.getScheme match {
       case null => new File(path).getCanonicalFile.toURI
       case "local" =>
-        logWarning("File with 'local' scheme is not supported to add to file server, since " +
-          "it is already available on every node.")
+        logWarning(s"File with 'local' scheme $path is not supported to add to file server, " +
+          s"since it is already available on every node.")
         return
       case _ => uri
     }
 
     val hadoopPath = new Path(schemeCorrectedURI)
     val scheme = schemeCorrectedURI.getScheme
-    if (!Array("http", "https", "ftp").contains(scheme)) {
+    if (!Array("http", "https", "ftp").contains(scheme) && !isArchive) {
       val fs = hadoopPath.getFileSystem(hadoopConfiguration)
       val isDir = fs.getFileStatus(hadoopPath).isDirectory
       if (!isLocal && scheme == "file" && isDir) {
@@ -1547,24 +1615,42 @@ class SparkContext(config: SparkConf) extends Logging {
 
     val key = if (!isLocal && scheme == "file") {
       env.rpcEnv.fileServer.addFile(new File(uri.getPath))
+    } else if (uri.getScheme == null) {
+      schemeCorrectedURI.toString
     } else {
-        if (uri.getScheme == null) {
-          schemeCorrectedURI.toString
-        } else {
-          path
-        }
+      uri.toString
     }
-    val timestamp = System.currentTimeMillis
-    if (addedFiles.putIfAbsent(key, timestamp).isEmpty) {
+
+    val timestamp = if (addedOnSubmit) startTime else System.currentTimeMillis
+    if (!isArchive && addedFiles.putIfAbsent(key, timestamp).isEmpty) {
       logInfo(s"Added file $path at $key with timestamp $timestamp")
       // Fetch the file locally so that closures which are run on the driver can still use the
       // SparkFiles API to access files.
       Utils.fetchFile(uri.toString, new File(SparkFiles.getRootDirectory()), conf,
-        env.securityManager, hadoopConfiguration, timestamp, useCache = false)
+        hadoopConfiguration, timestamp, useCache = false)
+      postEnvironmentUpdate()
+    } else if (
+      isArchive &&
+        addedArchives.putIfAbsent(
+          UriBuilder.fromUri(new URI(key)).fragment(uri.getFragment).build().toString,
+          timestamp).isEmpty) {
+      logInfo(s"Added archive $path at $key with timestamp $timestamp")
+      // If the scheme is file, use URI to simply copy instead of downloading.
+      val uriToUse = if (!isLocal && scheme == "file") uri else new URI(key)
+      val uriToDownload = UriBuilder.fromUri(uriToUse).fragment(null).build()
+      val source = Utils.fetchFile(uriToDownload.toString, Utils.createTempDir(), conf,
+        hadoopConfiguration, timestamp, useCache = false, shouldUntar = false)
+      val dest = new File(
+        SparkFiles.getRootDirectory(),
+        if (uri.getFragment != null) uri.getFragment else source.getName)
+      logInfo(
+        s"Unpacking an archive $path from ${source.getAbsolutePath} to ${dest.getAbsolutePath}")
+      Utils.deleteRecursively(dest)
+      Utils.unpack(source, dest)
       postEnvironmentUpdate()
     } else {
       logWarning(s"The path $path has been added already. Overwriting of added paths " +
-       "is not supported in the current version.")
+        "is not supported in the current version.")
     }
   }
 
@@ -1598,7 +1684,7 @@ class SparkContext(config: SparkConf) extends Logging {
 
   /**
    * Get the max number of tasks that can be concurrent launched based on the ResourceProfile
-   * being used.
+   * could be used, even if some of them are being used at the moment.
    * Note that please don't cache the value returned by this method, because the number can change
    * due to add/remove executors.
    *
@@ -1729,7 +1815,7 @@ class SparkContext(config: SparkConf) extends Logging {
   def version: String = SPARK_VERSION
 
   /**
-   * Return a map from the slave to the max memory available for caching and the remaining
+   * Return a map from the block manager to the max memory available for caching and the remaining
    * memory available for caching.
    */
   def getExecutorMemoryStatus: Map[String, (Long, Long)] = {
@@ -1835,7 +1921,11 @@ class SparkContext(config: SparkConf) extends Logging {
    * @note A path can be added only once. Subsequent additions of the same path are ignored.
    */
   def addJar(path: String): Unit = {
-    def addLocalJarFile(file: File): String = {
+    addJar(path, false)
+  }
+
+  private def addJar(path: String, addedOnSubmit: Boolean): Unit = {
+    def addLocalJarFile(file: File): Seq[String] = {
       try {
         if (!file.exists()) {
           throw new FileNotFoundException(s"Jar ${file.getAbsolutePath} not found")
@@ -1844,15 +1934,15 @@ class SparkContext(config: SparkConf) extends Logging {
           throw new IllegalArgumentException(
             s"Directory ${file.getAbsoluteFile} is not allowed for addJar")
         }
-        env.rpcEnv.fileServer.addJar(file)
+        Seq(env.rpcEnv.fileServer.addJar(file))
       } catch {
         case NonFatal(e) =>
           logError(s"Failed to add $path to Spark environment", e)
-          null
+          Nil
       }
     }
 
-    def checkRemoteJarFile(path: String): String = {
+    def checkRemoteJarFile(path: String): Seq[String] = {
       val hadoopPath = new Path(path)
       val scheme = hadoopPath.toUri.getScheme
       if (!Array("http", "https", "ftp").contains(scheme)) {
@@ -1861,32 +1951,33 @@ class SparkContext(config: SparkConf) extends Logging {
           if (!fs.exists(hadoopPath)) {
             throw new FileNotFoundException(s"Jar ${path} not found")
           }
-          if (fs.isDirectory(hadoopPath)) {
+          if (fs.getFileStatus(hadoopPath).isDirectory) {
             throw new IllegalArgumentException(
               s"Directory ${path} is not allowed for addJar")
           }
-          path
+          Seq(path)
         } catch {
           case NonFatal(e) =>
             logError(s"Failed to add $path to Spark environment", e)
-            null
+            Nil
         }
       } else {
-        path
+        Seq(path)
       }
     }
 
     if (path == null || path.isEmpty) {
       logWarning("null or empty path specified as parameter to addJar")
     } else {
-      val key = if (path.contains("\\")) {
+      val (keys, scheme) = if (path.contains("\\") && Utils.isWindows) {
         // For local paths with backslashes on Windows, URI throws an exception
-        addLocalJarFile(new File(path))
+        (addLocalJarFile(new File(path)), "local")
       } else {
-        val uri = new Path(path).toUri
+        val uri = Utils.resolveURI(path)
         // SPARK-17650: Make sure this is a valid URL before adding it to the list of dependencies
         Utils.validateURL(uri)
-        uri.getScheme match {
+        val uriScheme = uri.getScheme
+        val jarPaths = uriScheme match {
           // A JAR file which exists only on the driver node
           case null =>
             // SPARK-22585 path without schema is not url encoded
@@ -1894,18 +1985,28 @@ class SparkContext(config: SparkConf) extends Logging {
           // A JAR file which exists only on the driver node
           case "file" => addLocalJarFile(new File(uri.getPath))
           // A JAR file which exists locally on every worker node
-          case "local" => "file:" + uri.getPath
+          case "local" => Seq("file:" + uri.getPath)
+          case "ivy" =>
+            // Since `new Path(path).toUri` will lose query information,
+            // so here we use `URI.create(path)`
+            DependencyUtils.resolveMavenDependencies(URI.create(path))
+              .flatMap(jar => addLocalJarFile(new File(jar)))
           case _ => checkRemoteJarFile(path)
         }
+        (jarPaths, uriScheme)
       }
-      if (key != null) {
-        val timestamp = System.currentTimeMillis
-        if (addedJars.putIfAbsent(key, timestamp).isEmpty) {
-          logInfo(s"Added JAR $path at $key with timestamp $timestamp")
+      if (keys.nonEmpty) {
+        val timestamp = if (addedOnSubmit) startTime else System.currentTimeMillis
+        val (added, existed) = keys.partition(addedJars.putIfAbsent(_, timestamp).isEmpty)
+        if (added.nonEmpty) {
+          val jarMessage = if (scheme != "ivy") "JAR" else "dependency jars of Ivy URI"
+          logInfo(s"Added $jarMessage $path at ${added.mkString(",")} with timestamp $timestamp")
           postEnvironmentUpdate()
-        } else {
-          logWarning(s"The jar $path has been added already. Overwriting of added jars " +
-            "is not supported in the current version.")
+        }
+        if (existed.nonEmpty) {
+          val jarMessage = if (scheme != "ivy") "JAR" else "dependency jars of Ivy URI"
+          logInfo(s"The $jarMessage $path at ${existed.mkString(",")} has been added already." +
+            " Overwriting of added jar is not supported in the current version.")
         }
       }
     }
@@ -1991,6 +2092,7 @@ class SparkContext(config: SparkConf) extends Logging {
     Utils.tryLogNonFatalError {
       _plugins.foreach(_.shutdown())
     }
+    FallbackStorage.cleanUp(_conf, _hadoopConfiguration)
     Utils.tryLogNonFatalError {
       _eventLogger.foreach(_.stop())
     }
@@ -2469,8 +2571,9 @@ class SparkContext(config: SparkConf) extends Logging {
       val schedulingMode = getSchedulingMode.toString
       val addedJarPaths = addedJars.keys.toSeq
       val addedFilePaths = addedFiles.keys.toSeq
+      val addedArchivePaths = addedArchives.keys.toSeq
       val environmentDetails = SparkEnv.environmentDetails(conf, hadoopConfiguration,
-        schedulingMode, addedJarPaths, addedFilePaths)
+        schedulingMode, addedJarPaths, addedFilePaths, addedArchivePaths)
       val environmentUpdate = SparkListenerEnvironmentUpdate(environmentDetails)
       listenerBus.post(environmentUpdate)
     }
@@ -2554,6 +2657,19 @@ object SparkContext extends Logging {
     }
   }
 
+  /**
+   * Called to ensure that SparkContext is created or accessed only on the Driver.
+   *
+   * Throws an exception if a SparkContext is about to be created in executors.
+   */
+  private def assertOnDriver(): Unit = {
+    if (TaskContext.get != null) {
+      // we're accessing it during task execution, fail.
+      throw new IllegalStateException(
+        "SparkContext should only be created and accessed on the driver.")
+    }
+  }
+
   /**
    * This function may be used to get or instantiate a SparkContext and register it as a
    * singleton object. Because we can only have one active SparkContext per JVM,
@@ -2814,14 +2930,14 @@ object SparkContext extends Logging {
         scheduler.initialize(backend)
         (backend, scheduler)
 
-      case LOCAL_CLUSTER_REGEX(numSlaves, coresPerSlave, memoryPerSlave) =>
-        checkResourcesPerTask(coresPerSlave.toInt)
-        // Check to make sure memory requested <= memoryPerSlave. Otherwise Spark will just hang.
-        val memoryPerSlaveInt = memoryPerSlave.toInt
-        if (sc.executorMemory > memoryPerSlaveInt) {
+      case LOCAL_CLUSTER_REGEX(numWorkers, coresPerWorker, memoryPerWorker) =>
+        checkResourcesPerTask(coresPerWorker.toInt)
+        // Check to make sure memory requested <= memoryPerWorker. Otherwise Spark will just hang.
+        val memoryPerWorkerInt = memoryPerWorker.toInt
+        if (sc.executorMemory > memoryPerWorkerInt) {
           throw new SparkException(
             "Asked to launch cluster with %d MiB RAM / worker but requested %d MiB/worker".format(
-              memoryPerSlaveInt, sc.executorMemory))
+              memoryPerWorkerInt, sc.executorMemory))
         }
 
         // For host local mode setting the default of SHUFFLE_HOST_LOCAL_DISK_READING_ENABLED
@@ -2834,7 +2950,7 @@ object SparkContext extends Logging {
 
         val scheduler = new TaskSchedulerImpl(sc)
         val localCluster = new LocalSparkCluster(
-          numSlaves.toInt, coresPerSlave.toInt, memoryPerSlaveInt, sc.conf)
+          numWorkers.toInt, coresPerWorker.toInt, memoryPerWorkerInt, sc.conf)
         val masterUrls = localCluster.start()
         val backend = new StandaloneSchedulerBackend(scheduler, sc, masterUrls)
         scheduler.initialize(backend)
@@ -2871,6 +2987,30 @@ object SparkContext extends Logging {
     }
     serviceLoaders.headOption
   }
+
+  /**
+   * This is a helper function to complete the missing S3A magic committer configurations
+   * based on a single conf: `spark.hadoop.fs.s3a.bucket.<bucket>.committer.magic.enabled`
+   */
+  private def fillMissingMagicCommitterConfsIfNeeded(conf: SparkConf): Unit = {
+    val magicCommitterConfs = conf
+      .getAllWithPrefix("spark.hadoop.fs.s3a.bucket.")
+      .filter(_._1.endsWith(".committer.magic.enabled"))
+      .filter(_._2.equalsIgnoreCase("true"))
+    if (magicCommitterConfs.nonEmpty) {
+      // Try to enable S3 magic committer if missing
+      conf.setIfMissing("spark.hadoop.fs.s3a.committer.magic.enabled", "true")
+      if (conf.get("spark.hadoop.fs.s3a.committer.magic.enabled").equals("true")) {
+        conf.setIfMissing("spark.hadoop.fs.s3a.committer.name", "magic")
+        conf.setIfMissing("spark.hadoop.mapreduce.outputcommitter.factory.scheme.s3a",
+          "org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory")
+        conf.setIfMissing("spark.sql.parquet.output.committer.class",
+          "org.apache.spark.internal.io.cloud.BindingParquetOutputCommitter")
+        conf.setIfMissing("spark.sql.sources.commitProtocolClass",
+          "org.apache.spark.internal.io.cloud.PathOutputCommitProtocol")
+      }
+    }
+  }
 }
 
 /**
diff --git a/core/src/main/scala/org/apache/spark/SparkEnv.scala b/core/src/main/scala/org/apache/spark/SparkEnv.scala
index 8ba1739831803..ed8dc43b16c96 100644
--- a/core/src/main/scala/org/apache/spark/SparkEnv.scala
+++ b/core/src/main/scala/org/apache/spark/SparkEnv.scala
@@ -314,7 +314,7 @@ object SparkEnv extends Logging {
       }
     }
 
-    val broadcastManager = new BroadcastManager(isDriver, conf, securityManager)
+    val broadcastManager = new BroadcastManager(isDriver, conf)
 
     val mapOutputTracker = if (isDriver) {
       new MapOutputTrackerMaster(conf, broadcastManager, isLocal)
@@ -367,7 +367,8 @@ object SparkEnv extends Logging {
             externalShuffleClient
           } else {
             None
-          }, blockManagerInfo)),
+          }, blockManagerInfo,
+          mapOutputTracker.asInstanceOf[MapOutputTrackerMaster])),
       registerOrLookupEndpoint(
         BlockManagerMaster.DRIVER_HEARTBEAT_ENDPOINT_NAME,
         new BlockManagerMasterHeartbeatEndpoint(rpcEnv, isLocal, blockManagerInfo)),
@@ -396,14 +397,13 @@ object SparkEnv extends Logging {
       // Don't start metrics system right now for Driver.
       // We need to wait for the task scheduler to give us an app ID.
       // Then we can start the metrics system.
-      MetricsSystem.createMetricsSystem(MetricsSystemInstances.DRIVER, conf, securityManager)
+      MetricsSystem.createMetricsSystem(MetricsSystemInstances.DRIVER, conf)
     } else {
       // We need to set the executor ID before the MetricsSystem is created because sources and
       // sinks specified in the metrics configuration file will want to incorporate this executor's
       // ID into the metrics they report.
       conf.set(EXECUTOR_ID, executorId)
-      val ms = MetricsSystem.createMetricsSystem(MetricsSystemInstances.EXECUTOR, conf,
-        securityManager)
+      val ms = MetricsSystem.createMetricsSystem(MetricsSystemInstances.EXECUTOR, conf)
       ms.start(conf.get(METRICS_STATIC_SOURCES_ENABLED))
       ms
     }
@@ -453,7 +453,8 @@ object SparkEnv extends Logging {
       hadoopConf: Configuration,
       schedulingMode: String,
       addedJars: Seq[String],
-      addedFiles: Seq[String]): Map[String, Seq[(String, String)]] = {
+      addedFiles: Seq[String],
+      addedArchives: Seq[String]): Map[String, Seq[(String, String)]] = {
 
     import Properties._
     val jvmInformation = Seq(
@@ -483,7 +484,7 @@ object SparkEnv extends Logging {
       .split(File.pathSeparator)
       .filterNot(_.isEmpty)
       .map((_, "System Classpath"))
-    val addedJarsAndFiles = (addedJars ++ addedFiles).map((_, "Added By User"))
+    val addedJarsAndFiles = (addedJars ++ addedFiles ++ addedArchives).map((_, "Added By User"))
     val classPaths = (addedJarsAndFiles ++ classPathEntries).sorted
 
     // Add Hadoop properties, it will not ignore configs including in Spark. Some spark
diff --git a/core/src/main/scala/org/apache/spark/SparkStatusTracker.scala b/core/src/main/scala/org/apache/spark/SparkStatusTracker.scala
index 555c085d85a1e..37e673cd8c7e1 100644
--- a/core/src/main/scala/org/apache/spark/SparkStatusTracker.scala
+++ b/core/src/main/scala/org/apache/spark/SparkStatusTracker.scala
@@ -21,6 +21,7 @@ import java.util.Arrays
 
 import org.apache.spark.status.AppStatusStore
 import org.apache.spark.status.api.v1.StageStatus
+import org.apache.spark.util.Utils
 
 /**
  * Low-level status reporting APIs for monitoring job and stage progress.
@@ -103,10 +104,7 @@ class SparkStatusTracker private[spark] (sc: SparkContext, store: AppStatusStore
    */
   def getExecutorInfos: Array[SparkExecutorInfo] = {
     store.executorList(true).map { exec =>
-      val (host, port) = exec.hostPort.split(":", 2) match {
-        case Array(h, p) => (h, p.toInt)
-        case Array(h) => (h, -1)
-      }
+      val (host, port) = Utils.parseHostPort(exec.hostPort)
       val cachedMem = exec.memoryMetrics.map { mem =>
         mem.usedOnHeapStorageMemory + mem.usedOffHeapStorageMemory
       }.getOrElse(0L)
diff --git a/core/src/main/scala/org/apache/spark/TaskContextImpl.scala b/core/src/main/scala/org/apache/spark/TaskContextImpl.scala
index 08a58a029528b..db4b74bb89f0c 100644
--- a/core/src/main/scala/org/apache/spark/TaskContextImpl.scala
+++ b/core/src/main/scala/org/apache/spark/TaskContextImpl.scala
@@ -111,7 +111,7 @@ private[spark] class TaskContextImpl(
     if (failed) return
     failed = true
     failure = error
-    invokeListeners(onFailureCallbacks, "TaskFailureListener", Option(error)) {
+    invokeListeners(onFailureCallbacks.toSeq, "TaskFailureListener", Option(error)) {
       _.onTaskFailure(this, error)
     }
   }
@@ -120,7 +120,7 @@ private[spark] class TaskContextImpl(
   private[spark] override def markTaskCompleted(error: Option[Throwable]): Unit = synchronized {
     if (completed) return
     completed = true
-    invokeListeners(onCompleteCallbacks, "TaskCompletionListener", error) {
+    invokeListeners(onCompleteCallbacks.toSeq, "TaskCompletionListener", error) {
       _.onTaskCompletion(this)
     }
   }
@@ -142,7 +142,7 @@ private[spark] class TaskContextImpl(
       }
     }
     if (errorMsgs.nonEmpty) {
-      throw new TaskCompletionListenerException(errorMsgs, error)
+      throw new TaskCompletionListenerException(errorMsgs.toSeq, error)
     }
   }
 
diff --git a/core/src/main/scala/org/apache/spark/TaskEndReason.scala b/core/src/main/scala/org/apache/spark/TaskEndReason.scala
index b13028f868072..5dc70e9834b0b 100644
--- a/core/src/main/scala/org/apache/spark/TaskEndReason.scala
+++ b/core/src/main/scala/org/apache/spark/TaskEndReason.scala
@@ -90,17 +90,19 @@ case class FetchFailed(
   extends TaskFailedReason {
   override def toErrorString: String = {
     val bmAddressString = if (bmAddress == null) "null" else bmAddress.toString
-    s"FetchFailed($bmAddressString, shuffleId=$shuffleId, mapIndex=$mapIndex, " +
+    val mapIndexString = if (mapIndex == Int.MinValue) "Unknown" else mapIndex.toString
+    s"FetchFailed($bmAddressString, shuffleId=$shuffleId, mapIndex=$mapIndexString, " +
       s"mapId=$mapId, reduceId=$reduceId, message=\n$message\n)"
   }
 
   /**
    * Fetch failures lead to a different failure handling path: (1) we don't abort the stage after
    * 4 task failures, instead we immediately go back to the stage which generated the map output,
-   * and regenerate the missing data.  (2) we don't count fetch failures for blacklisting, since
-   * presumably its not the fault of the executor where the task ran, but the executor which
-   * stored the data. This is especially important because we might rack up a bunch of
-   * fetch-failures in rapid succession, on all nodes of the cluster, due to one bad node.
+   * and regenerate the missing data. (2) we don't count fetch failures from executors excluded
+   * due to too many task failures, since presumably its not the fault of the executor where
+   * the task ran, but the executor which stored the data. This is especially important because
+   * we might rack up a bunch of fetch-failures in rapid succession, on all nodes of the cluster,
+   * due to one bad node.
    */
   override def countTowardsTaskFailures: Boolean = false
 }
@@ -141,12 +143,12 @@ case class ExceptionFailure(
   private[spark] def this(
       e: Throwable,
       accumUpdates: Seq[AccumulableInfo],
-      preserveCause: Boolean) {
+      preserveCause: Boolean) = {
     this(e.getClass.getName, e.getMessage, e.getStackTrace, Utils.exceptionString(e),
       if (preserveCause) Some(new ThrowableSerializationWrapper(e)) else None, accumUpdates)
   }
 
-  private[spark] def this(e: Throwable, accumUpdates: Seq[AccumulableInfo]) {
+  private[spark] def this(e: Throwable, accumUpdates: Seq[AccumulableInfo]) = {
     this(e, accumUpdates, preserveCause = true)
   }
 
diff --git a/core/src/main/scala/org/apache/spark/TestUtils.scala b/core/src/main/scala/org/apache/spark/TestUtils.scala
index d459627930f4c..dcbb9baa20928 100644
--- a/core/src/main/scala/org/apache/spark/TestUtils.scala
+++ b/core/src/main/scala/org/apache/spark/TestUtils.scala
@@ -18,26 +18,34 @@
 package org.apache.spark
 
 import java.io.{ByteArrayInputStream, File, FileInputStream, FileOutputStream}
-import java.net.{HttpURLConnection, URI, URL}
+import java.net.{HttpURLConnection, InetSocketAddress, URI, URL}
 import java.nio.charset.StandardCharsets
-import java.nio.file.{Files => JavaFiles}
+import java.nio.file.{Files => JavaFiles, Paths}
 import java.nio.file.attribute.PosixFilePermission.{OWNER_EXECUTE, OWNER_READ, OWNER_WRITE}
 import java.security.SecureRandom
 import java.security.cert.X509Certificate
 import java.util.{Arrays, EnumSet, Locale, Properties}
 import java.util.concurrent.{TimeoutException, TimeUnit}
 import java.util.jar.{JarEntry, JarOutputStream, Manifest}
+import java.util.regex.Pattern
 import javax.net.ssl._
 import javax.tools.{JavaFileObject, SimpleJavaFileObject, ToolProvider}
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable
 import scala.collection.mutable.ArrayBuffer
+import scala.reflect.{classTag, ClassTag}
 import scala.sys.process.{Process, ProcessLogger}
 import scala.util.Try
 
 import com.google.common.io.{ByteStreams, Files}
+import org.apache.commons.lang3.StringUtils
 import org.apache.log4j.PropertyConfigurator
+import org.eclipse.jetty.server.Handler
+import org.eclipse.jetty.server.Server
+import org.eclipse.jetty.server.handler.DefaultHandler
+import org.eclipse.jetty.server.handler.HandlerList
+import org.eclipse.jetty.server.handler.ResourceHandler
 import org.json4s.JsonAST.JValue
 import org.json4s.jackson.JsonMethods.{compact, render}
 
@@ -179,11 +187,20 @@ private[spark] object TestUtils {
       destDir: File,
       toStringValue: String = "",
       baseClass: String = null,
-      classpathUrls: Seq[URL] = Seq.empty): File = {
+      classpathUrls: Seq[URL] = Seq.empty,
+      implementsClasses: Seq[String] = Seq.empty,
+      extraCodeBody: String = ""): File = {
     val extendsText = Option(baseClass).map { c => s" extends ${c}" }.getOrElse("")
+    val implementsText =
+      "implements " + (implementsClasses :+ "java.io.Serializable").mkString(", ")
     val sourceFile = new JavaSourceFromString(className,
-      "public class " + className + extendsText + " implements java.io.Serializable {" +
-      "  @Override public String toString() { return \"" + toStringValue + "\"; }}")
+      s"""
+         |public class $className $extendsText $implementsText {
+         |  @Override public String toString() { return "$toStringValue"; }
+         |
+         |  $extraCodeBody
+         |}
+        """.stripMargin)
     createCompiledClass(className, destDir, sourceFile, classpathUrls)
   }
 
@@ -212,34 +229,84 @@ private[spark] object TestUtils {
 
   /**
    * Asserts that exception message contains the message. Please note this checks all
-   * exceptions in the tree.
+   * exceptions in the tree. If a type parameter `E` is supplied, this will additionally confirm
+   * that the exception is a subtype of the exception provided in the type parameter.
    */
-  def assertExceptionMsg(exception: Throwable, msg: String, ignoreCase: Boolean = false): Unit = {
-    def contain(msg1: String, msg2: String): Boolean = {
+  def assertExceptionMsg[E <: Throwable : ClassTag](
+      exception: Throwable,
+      msg: String,
+      ignoreCase: Boolean = false): Unit = {
+
+    val (typeMsg, typeCheck) = if (classTag[E] == classTag[Nothing]) {
+      ("", (_: Throwable) => true)
+    } else {
+      val clazz = classTag[E].runtimeClass
+      (s"of type ${clazz.getName} ", (e: Throwable) => clazz.isAssignableFrom(e.getClass))
+    }
+
+    def contain(e: Throwable, msg: String): Boolean = {
       if (ignoreCase) {
-        msg1.toLowerCase(Locale.ROOT).contains(msg2.toLowerCase(Locale.ROOT))
+        e.getMessage.toLowerCase(Locale.ROOT).contains(msg.toLowerCase(Locale.ROOT))
       } else {
-        msg1.contains(msg2)
-      }
+        e.getMessage.contains(msg)
+      } && typeCheck(e)
     }
 
     var e = exception
-    var contains = contain(e.getMessage, msg)
+    var contains = contain(e, msg)
     while (e.getCause != null && !contains) {
       e = e.getCause
-      contains = contain(e.getMessage, msg)
+      contains = contain(e, msg)
     }
-    assert(contains, s"Exception tree doesn't contain the expected message: $msg")
+    assert(contains,
+      s"Exception tree doesn't contain the expected exception ${typeMsg}with message: $msg")
   }
 
   /**
    * Test if a command is available.
    */
   def testCommandAvailable(command: String): Boolean = {
-    val attempt = Try(Process(command).run(ProcessLogger(_ => ())).exitValue())
+    val attempt = if (Utils.isWindows) {
+      Try(Process(Seq(
+        "cmd.exe", "/C", s"where $command")).run(ProcessLogger(_ => ())).exitValue())
+    } else {
+      Try(Process(Seq(
+        "sh", "-c", s"command -v $command")).run(ProcessLogger(_ => ())).exitValue())
+    }
+    attempt.isSuccess && attempt.get == 0
+  }
+
+  def isPythonVersionAtLeast38(): Boolean = {
+    val attempt = if (Utils.isWindows) {
+      Try(Process(Seq("cmd.exe", "/C", "python3 --version"))
+        .run(ProcessLogger(s => s.startsWith("Python 3.8") || s.startsWith("Python 3.9")))
+        .exitValue())
+    } else {
+      Try(Process(Seq("sh", "-c", "python3 --version"))
+        .run(ProcessLogger(s => s.startsWith("Python 3.8") || s.startsWith("Python 3.9")))
+        .exitValue())
+    }
     attempt.isSuccess && attempt.get == 0
   }
 
+  /**
+   * Get the absolute path from the executable. This implementation was borrowed from
+   * `spark/dev/sparktestsupport/shellutils.py`.
+   */
+  def getAbsolutePathFromExecutable(executable: String): Option[String] = {
+    val command = if (Utils.isWindows) s"$executable.exe" else executable
+    if (command.split(File.separator, 2).length == 1 &&
+        JavaFiles.isRegularFile(Paths.get(command)) &&
+        JavaFiles.isExecutable(Paths.get(command))) {
+      Some(Paths.get(command).toAbsolutePath.toString)
+    } else {
+      sys.env("PATH").split(Pattern.quote(File.pathSeparator))
+        .map(path => Paths.get(s"${StringUtils.strip(path, "\"")}${File.separator}$command"))
+        .find(p => JavaFiles.isRegularFile(p) && JavaFiles.isExecutable(p))
+        .map(_.toString)
+    }
+  }
+
   /**
    * Returns the response code from an HTTP(S) URL.
    */
@@ -302,6 +369,22 @@ private[spark] object TestUtils {
     }
   }
 
+  def withHttpServer(resBaseDir: String = ".")(body: URL => Unit): Unit = {
+    // 0 as port means choosing randomly from the available ports
+    val server = new Server(new InetSocketAddress(Utils.localCanonicalHostName, 0))
+    val resHandler = new ResourceHandler()
+    resHandler.setResourceBase(resBaseDir)
+    val handlers = new HandlerList()
+    handlers.setHandlers(Array[Handler](resHandler, new DefaultHandler()))
+    server.setHandler(handlers)
+    server.start()
+    try {
+      body(server.getURI.toURL)
+    } finally {
+      server.stop()
+    }
+  }
+
   /**
    * Wait until at least `numExecutors` executors are up, or throw `TimeoutException` if the waiting
    * time elapsed before `numExecutors` executors up. Exposed for testing.
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala b/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala
index 1bcd203f2e435..6dd36309378cc 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala
@@ -941,7 +941,7 @@ class JavaPairRDD[K, V](val rdd: RDD[(K, V)])
   /**
    * Return a RDD containing only the elements in the inclusive range `lower` to `upper`.
    * If the RDD has been partitioned using a `RangePartitioner`, then this operation can be
-   * performed efficiently by only scanning the partitions that might containt matching elements.
+   * performed efficiently by only scanning the partitions that might contain matching elements.
    * Otherwise, a standard `filter` is applied to all partitions.
    *
    * @since 3.1.0
@@ -955,7 +955,7 @@ class JavaPairRDD[K, V](val rdd: RDD[(K, V)])
   /**
    * Return a RDD containing only the elements in the inclusive range `lower` to `upper`.
    * If the RDD has been partitioned using a `RangePartitioner`, then this operation can be
-   * performed efficiently by only scanning the partitions that might containt matching elements.
+   * performed efficiently by only scanning the partitions that might contain matching elements.
    * Otherwise, a standard `filter` is applied to all partitions.
    *
    * @since 3.1.0
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala b/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala
index e4140f659d979..15cb01a173287 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala
@@ -256,7 +256,7 @@ object JavaRDD {
       } catch {
         case eof: EOFException => // No-op
       }
-      JavaRDD.fromRDD(sc.parallelize(objs, parallelism))
+      JavaRDD.fromRDD(sc.parallelize(objs.toSeq, parallelism))
     } finally {
       din.close()
     }
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala b/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
index 1ca5262742665..306af24ada584 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
@@ -78,7 +78,7 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
 
   /**
    * Internal method to this RDD; will read from cache if applicable, or otherwise compute it.
-   * This should ''not'' be called by users directly, but is available for implementors of custom
+   * This should ''not'' be called by users directly, but is available for implementers of custom
    * subclasses of RDD.
    */
   def iterator(split: Partition, taskContext: TaskContext): JIterator[T] =
@@ -265,14 +265,14 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
    * Return an RDD created by piping elements to a forked external process.
    */
   def pipe(command: JList[String]): JavaRDD[String] = {
-    rdd.pipe(command.asScala)
+    rdd.pipe(command.asScala.toSeq)
   }
 
   /**
    * Return an RDD created by piping elements to a forked external process.
    */
   def pipe(command: JList[String], env: JMap[String, String]): JavaRDD[String] = {
-    rdd.pipe(command.asScala, env.asScala)
+    rdd.pipe(command.asScala.toSeq, env.asScala)
   }
 
   /**
@@ -282,7 +282,7 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
            env: JMap[String, String],
            separateWorkingDir: Boolean,
            bufferSize: Int): JavaRDD[String] = {
-    rdd.pipe(command.asScala, env.asScala, null, null, separateWorkingDir, bufferSize)
+    rdd.pipe(command.asScala.toSeq, env.asScala, null, null, separateWorkingDir, bufferSize)
   }
 
   /**
@@ -293,7 +293,8 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
            separateWorkingDir: Boolean,
            bufferSize: Int,
            encoding: String): JavaRDD[String] = {
-    rdd.pipe(command.asScala, env.asScala, null, null, separateWorkingDir, bufferSize, encoding)
+    rdd.pipe(command.asScala.toSeq, env.asScala, null, null, separateWorkingDir, bufferSize,
+      encoding)
   }
 
   /**
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
index 149def29b8fbd..608158caacd37 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
@@ -74,7 +74,7 @@ class JavaSparkContext(val sc: SparkContext) extends Closeable {
   /**
    * @param master Cluster URL to connect to (e.g. mesos://host:port, spark://host:port, local[4]).
    * @param appName A name for your application, to display on the cluster web UI
-   * @param sparkHome The SPARK_HOME directory on the slave nodes
+   * @param sparkHome The SPARK_HOME directory on the worker nodes
    * @param jarFile JAR file to send to the cluster. This can be a path on the local file system
    *                or an HDFS, HTTP, HTTPS, or FTP URL.
    */
@@ -84,7 +84,7 @@ class JavaSparkContext(val sc: SparkContext) extends Closeable {
   /**
    * @param master Cluster URL to connect to (e.g. mesos://host:port, spark://host:port, local[4]).
    * @param appName A name for your application, to display on the cluster web UI
-   * @param sparkHome The SPARK_HOME directory on the slave nodes
+   * @param sparkHome The SPARK_HOME directory on the worker nodes
    * @param jars Collection of JARs to send to the cluster. These can be paths on the local file
    *             system or HDFS, HTTP, HTTPS, or FTP URLs.
    */
@@ -94,7 +94,7 @@ class JavaSparkContext(val sc: SparkContext) extends Closeable {
   /**
    * @param master Cluster URL to connect to (e.g. mesos://host:port, spark://host:port, local[4]).
    * @param appName A name for your application, to display on the cluster web UI
-   * @param sparkHome The SPARK_HOME directory on the slave nodes
+   * @param sparkHome The SPARK_HOME directory on the worker nodes
    * @param jars Collection of JARs to send to the cluster. These can be paths on the local file
    *             system or HDFS, HTTP, HTTPS, or FTP URLs.
    * @param environment Environment variables to set on worker nodes
@@ -133,7 +133,7 @@ class JavaSparkContext(val sc: SparkContext) extends Closeable {
   /** Distribute a local Scala collection to form an RDD. */
   def parallelize[T](list: java.util.List[T], numSlices: Int): JavaRDD[T] = {
     implicit val ctag: ClassTag[T] = fakeClassTag
-    sc.parallelize(list.asScala, numSlices)
+    sc.parallelize(list.asScala.toSeq, numSlices)
   }
 
   /** Get an RDD that has no partitions or elements. */
@@ -152,7 +152,7 @@ class JavaSparkContext(val sc: SparkContext) extends Closeable {
   : JavaPairRDD[K, V] = {
     implicit val ctagK: ClassTag[K] = fakeClassTag
     implicit val ctagV: ClassTag[V] = fakeClassTag
-    JavaPairRDD.fromRDD(sc.parallelize(list.asScala, numSlices))
+    JavaPairRDD.fromRDD(sc.parallelize(list.asScala.toSeq, numSlices))
   }
 
   /** Distribute a local Scala collection to form an RDD. */
@@ -161,7 +161,7 @@ class JavaSparkContext(val sc: SparkContext) extends Closeable {
 
   /** Distribute a local Scala collection to form an RDD. */
   def parallelizeDoubles(list: java.util.List[java.lang.Double], numSlices: Int): JavaDoubleRDD =
-    JavaDoubleRDD.fromRDD(sc.parallelize(list.asScala.map(_.doubleValue()), numSlices))
+    JavaDoubleRDD.fromRDD(sc.parallelize(list.asScala.map(_.doubleValue()).toSeq, numSlices))
 
   /** Distribute a local Scala collection to form an RDD. */
   def parallelizeDoubles(list: java.util.List[java.lang.Double]): JavaDoubleRDD =
@@ -727,7 +727,7 @@ class JavaSparkContext(val sc: SparkContext) extends Closeable {
    * @note This does not necessarily mean the caching or computation was successful.
    */
   def getPersistentRDDs: JMap[java.lang.Integer, JavaRDD[_]] = {
-    sc.getPersistentRDDs.mapValues(s => JavaRDD.fromRDD(s))
+    sc.getPersistentRDDs.mapValues(s => JavaRDD.fromRDD(s)).toMap
       .asJava.asInstanceOf[JMap[java.lang.Integer, JavaRDD[_]]]
   }
 
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala b/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
index a577194a48006..6d4dc3d3dfe92 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
@@ -48,14 +48,14 @@ import org.apache.spark.util._
 private[spark] class PythonRDD(
     parent: RDD[_],
     func: PythonFunction,
-    preservePartitoning: Boolean,
+    preservePartitioning: Boolean,
     isFromBarrier: Boolean = false)
   extends RDD[Array[Byte]](parent) {
 
   override def getPartitions: Array[Partition] = firstParent.partitions
 
   override val partitioner: Option[Partitioner] = {
-    if (preservePartitoning) firstParent.partitioner else None
+    if (preservePartitioning) firstParent.partitioner else None
   }
 
   val asJavaRDD: JavaRDD[Array[Byte]] = JavaRDD.fromRDD(this)
@@ -74,13 +74,25 @@ private[spark] class PythonRDD(
  * runner.
  */
 private[spark] case class PythonFunction(
-    command: Array[Byte],
+    command: Seq[Byte],
     envVars: JMap[String, String],
     pythonIncludes: JList[String],
     pythonExec: String,
     pythonVer: String,
     broadcastVars: JList[Broadcast[PythonBroadcast]],
-    accumulator: PythonAccumulatorV2)
+    accumulator: PythonAccumulatorV2) {
+
+  def this(
+      command: Array[Byte],
+      envVars: JMap[String, String],
+      pythonIncludes: JList[String],
+      pythonExec: String,
+      pythonVer: String,
+      broadcastVars: JList[Broadcast[PythonBroadcast]],
+      accumulator: PythonAccumulatorV2) = {
+    this(command.toSeq, envVars, pythonIncludes, pythonExec, pythonVer, broadcastVars, accumulator)
+  }
+}
 
 /**
  * A wrapper for chained Python functions (from bottom to top).
@@ -151,7 +163,7 @@ private[spark] object PythonRDD extends Logging {
     type ByteArray = Array[Byte]
     type UnrolledPartition = Array[ByteArray]
     val allPartitions: Array[UnrolledPartition] =
-      sc.runJob(rdd, (x: Iterator[ByteArray]) => x.toArray, partitions.asScala)
+      sc.runJob(rdd, (x: Iterator[ByteArray]) => x.toArray, partitions.asScala.toSeq)
     val flattenedPartition: UnrolledPartition = Array.concat(allPartitions: _*)
     serveIterator(flattenedPartition.iterator,
       s"serve RDD ${rdd.id} with partitions ${partitions.asScala.mkString(",")}")
@@ -825,7 +837,7 @@ private[spark] class PythonBroadcast(@transient var path: String) extends Serial
  * We might be serializing a really large object from python -- we don't want
  * python to buffer the whole thing in memory, nor can it write to a file,
  * so we don't know the length in advance.  So python writes it in chunks, each chunk
- * preceeded by a length, till we get a "length" of -1 which serves as EOF.
+ * preceded by a length, till we get a "length" of -1 which serves as EOF.
  *
  * Tested from python tests.
  */
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
index f34316424c4ca..6e2b6add96fd4 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
@@ -21,16 +21,12 @@ import java.io._
 import java.net._
 import java.nio.charset.StandardCharsets
 import java.nio.charset.StandardCharsets.UTF_8
+import java.util.concurrent.ConcurrentHashMap
 import java.util.concurrent.atomic.AtomicBoolean
 
 import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
 import scala.util.control.NonFatal
 
-import org.json4s.JsonAST._
-import org.json4s.JsonDSL._
-import org.json4s.jackson.JsonMethods.{compact, render}
-
 import org.apache.spark._
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.{BUFFER_SIZE, EXECUTOR_CORES}
@@ -85,7 +81,9 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
 
   private val conf = SparkEnv.get.conf
   protected val bufferSize: Int = conf.get(BUFFER_SIZE)
+  protected val authSocketTimeout = conf.get(PYTHON_AUTH_SOCKET_TIMEOUT)
   private val reuseWorker = conf.get(PYTHON_WORKER_REUSE)
+  protected val simplifiedTraceback: Boolean = false
 
   // All the Python functions should have the same exec, version and envvars.
   protected val envVars: java.util.Map[String, String] = funcs.head.funcs.head.envVars
@@ -133,6 +131,9 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
     if (reuseWorker) {
       envVars.put("SPARK_REUSE_WORKER", "1")
     }
+    if (simplifiedTraceback) {
+      envVars.put("SPARK_SIMPLIFIED_TRACEBACK", "1")
+    }
     // SPARK-30299 this could be wrong with standalone mode when executor
     // cores might not be correct because it defaults to all cores on the box.
     val execCores = execCoresProp.map(_.toInt).getOrElse(conf.get(EXECUTOR_CORES))
@@ -140,6 +141,7 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
     if (workerMemoryMb.isDefined) {
       envVars.put("PYSPARK_EXECUTOR_MEMORY_MB", workerMemoryMb.get.toString)
     }
+    envVars.put("SPARK_AUTH_SOCKET_TIMEOUT", authSocketTimeout.toString)
     envVars.put("SPARK_BUFFER_SIZE", bufferSize.toString)
     val worker: Socket = env.createPythonWorker(pythonExec, envVars.asScala.toMap)
     // Whether is the worker released into idle pool or closed. When any codes try to release or
@@ -163,7 +165,16 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
     }
 
     writerThread.start()
-    new MonitorThread(env, worker, context).start()
+    if (reuseWorker) {
+      val key = (worker, context.taskAttemptId)
+      // SPARK-35009: avoid creating multiple monitor threads for the same python worker
+      // and task context
+      if (PythonRunner.runningMonitorThreads.add(key)) {
+        new MonitorThread(SparkEnv.get, worker, context).start()
+      }
+    } else {
+      new MonitorThread(SparkEnv.get, worker, context).start()
+    }
 
     // Return an iterator that read lines from the process's stdout
     val stream = new DataInputStream(new BufferedInputStream(worker.getInputStream, bufferSize))
@@ -243,7 +254,7 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
             /* backlog */ 1,
             InetAddress.getByName("localhost")))
           // A call to accept() for ServerSocket shall block infinitely.
-          serverSocket.map(_.setSoTimeout(0))
+          serverSocket.foreach(_.setSoTimeout(0))
           new Thread("accept-connections") {
             setDaemon(true)
 
@@ -563,7 +574,7 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
 
     setDaemon(true)
 
-    override def run(): Unit = {
+    private def monitorWorker(): Unit = {
       // Kill the worker if it is interrupted, checking until task completion.
       // TODO: This has a race condition if interruption occurs, as completed may still become true.
       while (!context.isInterrupted && !context.isCompleted) {
@@ -585,11 +596,25 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
         }
       }
     }
+
+    override def run(): Unit = {
+      try {
+        monitorWorker()
+      } finally {
+        if (reuseWorker) {
+          val key = (worker, context.taskAttemptId)
+          PythonRunner.runningMonitorThreads.remove(key)
+        }
+      }
+    }
   }
 }
 
 private[spark] object PythonRunner {
 
+  // already running worker monitor threads for worker and task attempts ID pairs
+  val runningMonitorThreads = ConcurrentHashMap.newKeySet[(Socket, Long)]()
+
   def apply(func: PythonFunction): PythonRunner = {
     new PythonRunner(Seq(ChainedPythonFunctions(Seq(func))))
   }
@@ -613,7 +638,7 @@ private[spark] class PythonRunner(funcs: Seq[ChainedPythonFunctions])
       protected override def writeCommand(dataOut: DataOutputStream): Unit = {
         val command = funcs.head.funcs.head.command
         dataOut.writeInt(command.length)
-        dataOut.write(command)
+        dataOut.write(command.toArray)
       }
 
       protected override def writeIteratorToStream(dataOut: DataOutputStream): Unit = {
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala b/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala
index 490b48719b6be..549edc446415b 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala
@@ -27,7 +27,7 @@ import org.apache.spark.SparkContext
 import org.apache.spark.api.java.{JavaRDD, JavaSparkContext}
 
 private[spark] object PythonUtils {
-  val PY4J_ZIP_NAME = "py4j-0.10.9-src.zip"
+  val PY4J_ZIP_NAME = "py4j-0.10.9.2-src.zip"
 
   /** Get the PYTHONPATH for PySpark, either from SPARK_HOME, if it is set, or from our JAR */
   def sparkPythonPath: String = {
@@ -54,7 +54,7 @@ private[spark] object PythonUtils {
    * Convert list of T into seq of T (for calling API with varargs)
    */
   def toSeq[T](vs: JList[T]): Seq[T] = {
-    vs.asScala
+    vs.asScala.toSeq
   }
 
   /**
@@ -85,4 +85,12 @@ private[spark] object PythonUtils {
   def getBroadcastThreshold(sc: JavaSparkContext): Long = {
     sc.conf.get(org.apache.spark.internal.config.BROADCAST_FOR_UDF_COMPRESSION_THRESHOLD)
   }
+
+  def getPythonAuthSocketTimeout(sc: JavaSparkContext): Long = {
+    sc.conf.get(org.apache.spark.internal.config.Python.PYTHON_AUTH_SOCKET_TIMEOUT)
+  }
+
+  def getSparkBufferSize(sc: JavaSparkContext): Int = {
+    sc.conf.get(org.apache.spark.internal.config.BUFFER_SIZE)
+  }
 }
diff --git a/core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala b/core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala
index 01e64b6972ae2..dd962ca11ecc2 100644
--- a/core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.api.python
 
-import java.nio.ByteOrder
-import java.nio.charset.StandardCharsets
 import java.util.{ArrayList => JArrayList}
 
 import scala.collection.JavaConverters._
@@ -45,71 +43,6 @@ private[spark] object SerDeUtil extends Logging {
       }
     }
   }
-  // Unpickle array.array generated by Python 2.6
-  class ArrayConstructor extends net.razorvine.pickle.objects.ArrayConstructor {
-    //  /* Description of types */
-    //  static struct arraydescr descriptors[] = {
-    //    {'c', sizeof(char), c_getitem, c_setitem},
-    //    {'b', sizeof(char), b_getitem, b_setitem},
-    //    {'B', sizeof(char), BB_getitem, BB_setitem},
-    //    #ifdef Py_USING_UNICODE
-    //      {'u', sizeof(Py_UNICODE), u_getitem, u_setitem},
-    //    #endif
-    //    {'h', sizeof(short), h_getitem, h_setitem},
-    //    {'H', sizeof(short), HH_getitem, HH_setitem},
-    //    {'i', sizeof(int), i_getitem, i_setitem},
-    //    {'I', sizeof(int), II_getitem, II_setitem},
-    //    {'l', sizeof(long), l_getitem, l_setitem},
-    //    {'L', sizeof(long), LL_getitem, LL_setitem},
-    //    {'f', sizeof(float), f_getitem, f_setitem},
-    //    {'d', sizeof(double), d_getitem, d_setitem},
-    //    {'\0', 0, 0, 0} /* Sentinel */
-    //  };
-    val machineCodes: Map[Char, Int] = if (ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) {
-      Map('B' -> 0, 'b' -> 1, 'H' -> 3, 'h' -> 5, 'I' -> 7, 'i' -> 9,
-        'L' -> 11, 'l' -> 13, 'f' -> 15, 'd' -> 17, 'u' -> 21
-      )
-    } else {
-      Map('B' -> 0, 'b' -> 1, 'H' -> 2, 'h' -> 4, 'I' -> 6, 'i' -> 8,
-        'L' -> 10, 'l' -> 12, 'f' -> 14, 'd' -> 16, 'u' -> 20
-      )
-    }
-    override def construct(args: Array[Object]): Object = {
-      if (args.length == 1) {
-        construct(args ++ Array(""))
-      } else if (args.length == 2 && args(1).isInstanceOf[String]) {
-        val typecode = args(0).asInstanceOf[String].charAt(0)
-        // This must be ISO 8859-1 / Latin 1, not UTF-8, to interoperate correctly
-        val data = args(1).asInstanceOf[String].getBytes(StandardCharsets.ISO_8859_1)
-        if (typecode == 'c') {
-          // It seems like the pickle of pypy uses the similar protocol to Python 2.6, which uses
-          // a string for array data instead of list as Python 2.7, and handles an array of
-          // typecode 'c' as 1-byte character.
-          val result = new Array[Char](data.length)
-          var i = 0
-          while (i < data.length) {
-            result(i) = data(i).toChar
-            i += 1
-          }
-          result
-        } else {
-          construct(typecode, machineCodes(typecode), data)
-        }
-      } else if (args.length == 2 && args(0) == "l") {
-        // On Python 2, an array of typecode 'l' should be handled as long rather than int.
-        val values = args(1).asInstanceOf[JArrayList[_]]
-        val result = new Array[Long](values.size)
-        var i = 0
-        while (i < values.size) {
-          result(i) = values.get(i).asInstanceOf[Number].longValue()
-          i += 1
-        }
-        result
-      } else {
-        super.construct(args)
-      }
-    }
-  }
 
   private var initialized = false
   // This should be called before trying to unpickle array.array from Python
@@ -117,7 +50,6 @@ private[spark] object SerDeUtil extends Logging {
   def initialize(): Unit = {
     synchronized{
       if (!initialized) {
-        Unpickler.registerConstructor("array", "array", new ArrayConstructor())
         Unpickler.registerConstructor("__builtin__", "bytearray", new ByteArrayConstructor())
         Unpickler.registerConstructor("builtins", "bytearray", new ByteArrayConstructor())
         Unpickler.registerConstructor("__builtin__", "bytes", new ByteArrayConstructor())
@@ -146,7 +78,8 @@ private[spark] object SerDeUtil extends Logging {
    * Choose batch size based on size of objects
    */
   private[spark] class AutoBatchedPickler(iter: Iterator[Any]) extends Iterator[Array[Byte]] {
-    private val pickle = new Pickler()
+    private val pickle = new Pickler(/* useMemo = */ true,
+      /* valueCompare = */ false)
     private var batch = 1
     private val buffer = new mutable.ArrayBuffer[Any]
 
@@ -199,7 +132,8 @@ private[spark] object SerDeUtil extends Logging {
   }
 
   private def checkPickle(t: (Any, Any)): (Boolean, Boolean) = {
-    val pickle = new Pickler
+    val pickle = new Pickler(/* useMemo = */ true,
+      /* valueCompare = */ false)
     val kt = Try {
       pickle.dumps(t._1)
     }
@@ -250,7 +184,8 @@ private[spark] object SerDeUtil extends Logging {
       if (batchSize == 0) {
         new AutoBatchedPickler(cleaned)
       } else {
-        val pickle = new Pickler
+        val pickle = new Pickler(/* useMemo = */ true,
+          /* valueCompare = */ false)
         cleaned.grouped(batchSize).map(batched => pickle.dumps(batched.asJava))
       }
     }
diff --git a/core/src/main/scala/org/apache/spark/api/r/RRunner.scala b/core/src/main/scala/org/apache/spark/api/r/RRunner.scala
index 20ab6fc2f348d..41c66024272b9 100644
--- a/core/src/main/scala/org/apache/spark/api/r/RRunner.scala
+++ b/core/src/main/scala/org/apache/spark/api/r/RRunner.scala
@@ -19,7 +19,6 @@ package org.apache.spark.api.r
 
 import java.io._
 
-import org.apache.spark._
 import org.apache.spark.broadcast.Broadcast
 
 /**
diff --git a/core/src/main/scala/org/apache/spark/api/r/RUtils.scala b/core/src/main/scala/org/apache/spark/api/r/RUtils.scala
index 311fade127839..784a57e7b98a8 100644
--- a/core/src/main/scala/org/apache/spark/api/r/RUtils.scala
+++ b/core/src/main/scala/org/apache/spark/api/r/RUtils.scala
@@ -43,9 +43,9 @@ private[spark] object RUtils {
    * Check if SparkR is installed before running tests that use SparkR.
    */
   def isSparkRInstalled: Boolean = {
-    localSparkRPackagePath.filter { pkgDir =>
+    localSparkRPackagePath.exists { pkgDir =>
       new File(Seq(pkgDir, "SparkR").mkString(File.separator)).exists
-    }.isDefined
+    }
   }
 
   /**
diff --git a/core/src/main/scala/org/apache/spark/broadcast/BroadcastFactory.scala b/core/src/main/scala/org/apache/spark/broadcast/BroadcastFactory.scala
index ece4ae6ab0310..9891582501b8b 100644
--- a/core/src/main/scala/org/apache/spark/broadcast/BroadcastFactory.scala
+++ b/core/src/main/scala/org/apache/spark/broadcast/BroadcastFactory.scala
@@ -19,7 +19,6 @@ package org.apache.spark.broadcast
 
 import scala.reflect.ClassTag
 
-import org.apache.spark.SecurityManager
 import org.apache.spark.SparkConf
 
 /**
@@ -29,7 +28,7 @@ import org.apache.spark.SparkConf
  */
 private[spark] trait BroadcastFactory {
 
-  def initialize(isDriver: Boolean, conf: SparkConf, securityMgr: SecurityManager): Unit
+  def initialize(isDriver: Boolean, conf: SparkConf): Unit
 
   /**
    * Creates a new broadcast variable.
diff --git a/core/src/main/scala/org/apache/spark/broadcast/BroadcastManager.scala b/core/src/main/scala/org/apache/spark/broadcast/BroadcastManager.scala
index c93cadf1ab3e8..989a1941d1791 100644
--- a/core/src/main/scala/org/apache/spark/broadcast/BroadcastManager.scala
+++ b/core/src/main/scala/org/apache/spark/broadcast/BroadcastManager.scala
@@ -24,15 +24,12 @@ import scala.reflect.ClassTag
 
 import org.apache.commons.collections.map.{AbstractReferenceMap, ReferenceMap}
 
-import org.apache.spark.{SecurityManager, SparkConf}
+import org.apache.spark.SparkConf
 import org.apache.spark.api.python.PythonBroadcast
 import org.apache.spark.internal.Logging
 
 private[spark] class BroadcastManager(
-    val isDriver: Boolean,
-    conf: SparkConf,
-    securityManager: SecurityManager)
-  extends Logging {
+    val isDriver: Boolean, conf: SparkConf) extends Logging {
 
   private var initialized = false
   private var broadcastFactory: BroadcastFactory = null
@@ -44,7 +41,7 @@ private[spark] class BroadcastManager(
     synchronized {
       if (!initialized) {
         broadcastFactory = new TorrentBroadcastFactory
-        broadcastFactory.initialize(isDriver, conf, securityManager)
+        broadcastFactory.initialize(isDriver, conf)
         initialized = true
       }
     }
diff --git a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala
index 77fbbc08c2103..1024d9b5060bc 100644
--- a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala
+++ b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala
@@ -133,22 +133,30 @@ private[spark] class TorrentBroadcast[T: ClassTag](obj: T, id: Long)
     if (!blockManager.putSingle(broadcastId, value, MEMORY_AND_DISK, tellMaster = false)) {
       throw new SparkException(s"Failed to store $broadcastId in BlockManager")
     }
-    val blocks =
-      TorrentBroadcast.blockifyObject(value, blockSize, SparkEnv.get.serializer, compressionCodec)
-    if (checksumEnabled) {
-      checksums = new Array[Int](blocks.length)
-    }
-    blocks.zipWithIndex.foreach { case (block, i) =>
+    try {
+      val blocks =
+        TorrentBroadcast.blockifyObject(value, blockSize, SparkEnv.get.serializer, compressionCodec)
       if (checksumEnabled) {
-        checksums(i) = calcChecksum(block)
+        checksums = new Array[Int](blocks.length)
       }
-      val pieceId = BroadcastBlockId(id, "piece" + i)
-      val bytes = new ChunkedByteBuffer(block.duplicate())
-      if (!blockManager.putBytes(pieceId, bytes, MEMORY_AND_DISK_SER, tellMaster = true)) {
-        throw new SparkException(s"Failed to store $pieceId of $broadcastId in local BlockManager")
+      blocks.zipWithIndex.foreach { case (block, i) =>
+        if (checksumEnabled) {
+          checksums(i) = calcChecksum(block)
+        }
+        val pieceId = BroadcastBlockId(id, "piece" + i)
+        val bytes = new ChunkedByteBuffer(block.duplicate())
+        if (!blockManager.putBytes(pieceId, bytes, MEMORY_AND_DISK_SER, tellMaster = true)) {
+          throw new SparkException(s"Failed to store $pieceId of $broadcastId " +
+            s"in local BlockManager")
+        }
       }
+      blocks.length
+    } catch {
+      case t: Throwable =>
+        logError(s"Store broadcast $broadcastId fail, remove all pieces of the broadcast")
+        blockManager.removeBroadcast(id, tellMaster = true)
+        throw t
     }
-    blocks.length
   }
 
   /** Fetch torrent blocks from the driver and/or other executors. */
diff --git a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcastFactory.scala b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcastFactory.scala
index 65fb5186afae1..6846e1967c4d6 100644
--- a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcastFactory.scala
+++ b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcastFactory.scala
@@ -19,7 +19,7 @@ package org.apache.spark.broadcast
 
 import scala.reflect.ClassTag
 
-import org.apache.spark.{SecurityManager, SparkConf}
+import org.apache.spark.SparkConf
 
 /**
  * A [[org.apache.spark.broadcast.Broadcast]] implementation that uses a BitTorrent-like
@@ -28,8 +28,7 @@ import org.apache.spark.{SecurityManager, SparkConf}
  */
 private[spark] class TorrentBroadcastFactory extends BroadcastFactory {
 
-  override def initialize(isDriver: Boolean, conf: SparkConf,
-      securityMgr: SecurityManager): Unit = { }
+  override def initialize(isDriver: Boolean, conf: SparkConf): Unit = { }
 
   override def newBroadcast[T: ClassTag](value_ : T, isLocal: Boolean, id: Long): Broadcast[T] = {
     new TorrentBroadcast[T](value_, id)
diff --git a/core/src/main/scala/org/apache/spark/deploy/Client.scala b/core/src/main/scala/org/apache/spark/deploy/Client.scala
index 7022b986ea025..7c5ab43a9e1b3 100644
--- a/core/src/main/scala/org/apache/spark/deploy/Client.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/Client.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.deploy
 
+import java.util.concurrent.TimeUnit
+
 import scala.collection.mutable.HashSet
 import scala.concurrent.ExecutionContext
 import scala.reflect.ClassTag
@@ -27,6 +29,7 @@ import org.apache.log4j.Logger
 import org.apache.spark.{SecurityManager, SparkConf}
 import org.apache.spark.deploy.DeployMessages._
 import org.apache.spark.deploy.master.{DriverState, Master}
+import org.apache.spark.deploy.master.DriverState.DriverState
 import org.apache.spark.internal.{config, Logging}
 import org.apache.spark.internal.config.Network.RPC_ASK_TIMEOUT
 import org.apache.spark.resource.ResourceUtils
@@ -61,6 +64,11 @@ private class ClientEndpoint(
 
    private val lostMasters = new HashSet[RpcAddress]
    private var activeMasterEndpoint: RpcEndpointRef = null
+   private val waitAppCompletion = conf.get(config.STANDALONE_SUBMIT_WAIT_APP_COMPLETION)
+   private val REPORT_DRIVER_STATUS_INTERVAL = 10000
+   private var submittedDriverID = ""
+   private var driverStatusReported = false
+
 
   private def getProperty(key: String, conf: SparkConf): Option[String] = {
     sys.props.get(key).orElse(conf.getOption(key))
@@ -107,8 +115,13 @@ private class ClientEndpoint(
 
       case "kill" =>
         val driverId = driverArgs.driverId
+        submittedDriverID = driverId
         asyncSendToMasterAndForwardReply[KillDriverResponse](RequestKillDriver(driverId))
     }
+    logInfo("... waiting before polling master for driver state")
+    forwardMessageThread.scheduleAtFixedRate(() => Utils.tryLogNonFatalError {
+      monitorDriverStatus()
+    }, 5000, REPORT_DRIVER_STATUS_INTERVAL, TimeUnit.MILLISECONDS)
   }
 
   /**
@@ -124,58 +137,87 @@ private class ClientEndpoint(
     }
   }
 
-  /* Find out driver status then exit the JVM */
-  def pollAndReportStatus(driverId: String): Unit = {
-    // Since ClientEndpoint is the only RpcEndpoint in the process, blocking the event loop thread
-    // is fine.
-    logInfo("... waiting before polling master for driver state")
-    Thread.sleep(5000)
-    logInfo("... polling master for driver state")
-    val statusResponse =
-      activeMasterEndpoint.askSync[DriverStatusResponse](RequestDriverStatus(driverId))
-    if (statusResponse.found) {
-      logInfo(s"State of $driverId is ${statusResponse.state.get}")
-      // Worker node, if present
-      (statusResponse.workerId, statusResponse.workerHostPort, statusResponse.state) match {
-        case (Some(id), Some(hostPort), Some(DriverState.RUNNING)) =>
-          logInfo(s"Driver running on $hostPort ($id)")
-        case _ =>
+  private def monitorDriverStatus(): Unit = {
+    if (submittedDriverID != "") {
+      asyncSendToMasterAndForwardReply[DriverStatusResponse](RequestDriverStatus(submittedDriverID))
+    }
+  }
+
+  /**
+   * Processes and reports the driver status then exit the JVM if the
+   * waitAppCompletion is set to false, else reports the driver status
+   * if debug logs are enabled.
+   */
+
+  def reportDriverStatus(
+      found: Boolean,
+      state: Option[DriverState],
+      workerId: Option[String],
+      workerHostPort: Option[String],
+      exception: Option[Exception]): Unit = {
+    if (found) {
+      // Using driverStatusReported to avoid writing following
+      // logs again when waitAppCompletion is set to true
+      if (!driverStatusReported) {
+        driverStatusReported = true
+        logInfo(s"State of $submittedDriverID is ${state.get}")
+        // Worker node, if present
+        (workerId, workerHostPort, state) match {
+          case (Some(id), Some(hostPort), Some(DriverState.RUNNING)) =>
+            logInfo(s"Driver running on $hostPort ($id)")
+          case _ =>
+        }
       }
       // Exception, if present
-      statusResponse.exception match {
+      exception match {
         case Some(e) =>
           logError(s"Exception from cluster was: $e")
           e.printStackTrace()
           System.exit(-1)
         case _ =>
-          System.exit(0)
+          state.get match {
+            case DriverState.FINISHED | DriverState.FAILED |
+                 DriverState.ERROR | DriverState.KILLED =>
+              logInfo(s"State of driver $submittedDriverID is ${state.get}, " +
+                s"exiting spark-submit JVM.")
+              System.exit(0)
+            case _ =>
+              if (!waitAppCompletion) {
+                logInfo(s"spark-submit not configured to wait for completion, " +
+                  s"exiting spark-submit JVM.")
+                System.exit(0)
+              } else {
+                logDebug(s"State of driver $submittedDriverID is ${state.get}, " +
+                  s"continue monitoring driver status.")
+              }
+            }
+        }
+      } else {
+        logError(s"ERROR: Cluster master did not recognize $submittedDriverID")
+        System.exit(-1)
       }
-    } else {
-      logError(s"ERROR: Cluster master did not recognize $driverId")
-      System.exit(-1)
     }
-  }
-
   override def receive: PartialFunction[Any, Unit] = {
 
     case SubmitDriverResponse(master, success, driverId, message) =>
       logInfo(message)
       if (success) {
         activeMasterEndpoint = master
-        pollAndReportStatus(driverId.get)
+        submittedDriverID = driverId.get
       } else if (!Utils.responseFromBackup(message)) {
         System.exit(-1)
       }
 
-
     case KillDriverResponse(master, driverId, success, message) =>
       logInfo(message)
       if (success) {
         activeMasterEndpoint = master
-        pollAndReportStatus(driverId)
       } else if (!Utils.responseFromBackup(message)) {
         System.exit(-1)
       }
+
+    case DriverStatusResponse(found, state, workerId, workerHostPort, exception) =>
+      reportDriverStatus(found, state, workerId, workerHostPort, exception)
   }
 
   override def onDisconnected(remoteAddress: RpcAddress): Unit = {
diff --git a/core/src/main/scala/org/apache/spark/deploy/DependencyUtils.scala b/core/src/main/scala/org/apache/spark/deploy/DependencyUtils.scala
deleted file mode 100644
index 5a17a6b6e169c..0000000000000
--- a/core/src/main/scala/org/apache/spark/deploy/DependencyUtils.scala
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.deploy
-
-import java.io.File
-import java.net.URI
-
-import org.apache.commons.lang3.StringUtils
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.{FileSystem, Path}
-
-import org.apache.spark.{SecurityManager, SparkConf, SparkException}
-import org.apache.spark.internal.Logging
-import org.apache.spark.util.{MutableURLClassLoader, Utils}
-
-private[deploy] object DependencyUtils extends Logging {
-
-  def resolveMavenDependencies(
-      packagesExclusions: String,
-      packages: String,
-      repositories: String,
-      ivyRepoPath: String,
-      ivySettingsPath: Option[String]): String = {
-    val exclusions: Seq[String] =
-      if (!StringUtils.isBlank(packagesExclusions)) {
-        packagesExclusions.split(",")
-      } else {
-        Nil
-      }
-    // Create the IvySettings, either load from file or build defaults
-    val ivySettings = ivySettingsPath match {
-      case Some(path) =>
-        SparkSubmitUtils.loadIvySettings(path, Option(repositories), Option(ivyRepoPath))
-
-      case None =>
-        SparkSubmitUtils.buildIvySettings(Option(repositories), Option(ivyRepoPath))
-    }
-
-    SparkSubmitUtils.resolveMavenCoordinates(packages, ivySettings, exclusions = exclusions)
-  }
-
-  def resolveAndDownloadJars(
-      jars: String,
-      userJar: String,
-      sparkConf: SparkConf,
-      hadoopConf: Configuration,
-      secMgr: SecurityManager): String = {
-    val targetDir = Utils.createTempDir()
-    val userJarName = userJar.split(File.separatorChar).last
-    Option(jars)
-      .map {
-        resolveGlobPaths(_, hadoopConf)
-          .split(",")
-          .filterNot(_.contains(userJarName))
-          .mkString(",")
-      }
-      .filterNot(_ == "")
-      .map(downloadFileList(_, targetDir, sparkConf, hadoopConf, secMgr))
-      .orNull
-  }
-
-  def addJarsToClassPath(jars: String, loader: MutableURLClassLoader): Unit = {
-    if (jars != null) {
-      for (jar <- jars.split(",")) {
-        addJarToClasspath(jar, loader)
-      }
-    }
-  }
-
-  /**
-   * Download a list of remote files to temp local files. If the file is local, the original file
-   * will be returned.
-   *
-   * @param fileList A comma separated file list.
-   * @param targetDir A temporary directory for which downloaded files.
-   * @param sparkConf Spark configuration.
-   * @param hadoopConf Hadoop configuration.
-   * @param secMgr Spark security manager.
-   * @return A comma separated local files list.
-   */
-  def downloadFileList(
-      fileList: String,
-      targetDir: File,
-      sparkConf: SparkConf,
-      hadoopConf: Configuration,
-      secMgr: SecurityManager): String = {
-    require(fileList != null, "fileList cannot be null.")
-    Utils.stringToSeq(fileList)
-      .map(downloadFile(_, targetDir, sparkConf, hadoopConf, secMgr))
-      .mkString(",")
-  }
-
-  /**
-   * Download a file from the remote to a local temporary directory. If the input path points to
-   * a local path, returns it with no operation.
-   *
-   * @param path A file path from where the files will be downloaded.
-   * @param targetDir A temporary directory for which downloaded files.
-   * @param sparkConf Spark configuration.
-   * @param hadoopConf Hadoop configuration.
-   * @param secMgr Spark security manager.
-   * @return Path to the local file.
-   */
-  def downloadFile(
-      path: String,
-      targetDir: File,
-      sparkConf: SparkConf,
-      hadoopConf: Configuration,
-      secMgr: SecurityManager): String = {
-    require(path != null, "path cannot be null.")
-    val uri = Utils.resolveURI(path)
-
-    uri.getScheme match {
-      case "file" | "local" => path
-      case "http" | "https" | "ftp" if Utils.isTesting =>
-        // This is only used for SparkSubmitSuite unit test. Instead of downloading file remotely,
-        // return a dummy local path instead.
-        val file = new File(uri.getPath)
-        new File(targetDir, file.getName).toURI.toString
-      case _ =>
-        val fname = new Path(uri).getName()
-        val localFile = Utils.doFetchFile(uri.toString(), targetDir, fname, sparkConf, secMgr,
-          hadoopConf)
-        localFile.toURI().toString()
-    }
-  }
-
-  def resolveGlobPaths(paths: String, hadoopConf: Configuration): String = {
-    require(paths != null, "paths cannot be null.")
-    Utils.stringToSeq(paths).flatMap { path =>
-      val (base, fragment) = splitOnFragment(path)
-      (resolveGlobPath(base, hadoopConf), fragment) match {
-        case (resolved, Some(_)) if resolved.length > 1 => throw new SparkException(
-            s"${base.toString} resolves ambiguously to multiple files: ${resolved.mkString(",")}")
-        case (resolved, Some(namedAs)) => resolved.map(_ + "#" + namedAs)
-        case (resolved, _) => resolved
-      }
-    }.mkString(",")
-  }
-
-  def addJarToClasspath(localJar: String, loader: MutableURLClassLoader): Unit = {
-    val uri = Utils.resolveURI(localJar)
-    uri.getScheme match {
-      case "file" | "local" =>
-        val file = new File(uri.getPath)
-        if (file.exists()) {
-          loader.addURL(file.toURI.toURL)
-        } else {
-          logWarning(s"Local jar $file does not exist, skipping.")
-        }
-      case _ =>
-        logWarning(s"Skip remote jar $uri.")
-    }
-  }
-
-  /**
-   * Merge a sequence of comma-separated file lists, some of which may be null to indicate
-   * no files, into a single comma-separated string.
-   */
-  def mergeFileLists(lists: String*): String = {
-    val merged = lists.filterNot(StringUtils.isBlank)
-      .flatMap(Utils.stringToSeq)
-    if (merged.nonEmpty) merged.mkString(",") else null
-  }
-
-  private def splitOnFragment(path: String): (URI, Option[String]) = {
-    val uri = Utils.resolveURI(path)
-    val withoutFragment = new URI(uri.getScheme, uri.getSchemeSpecificPart, null)
-    (withoutFragment, Option(uri.getFragment))
-  }
-
-  private def resolveGlobPath(uri: URI, hadoopConf: Configuration): Array[String] = {
-    uri.getScheme match {
-      case "local" | "http" | "https" | "ftp" => Array(uri.toString)
-      case _ =>
-        val fs = FileSystem.get(uri, hadoopConf)
-        Option(fs.globStatus(new Path(uri))).map { status =>
-          status.filter(_.isFile).map(_.getPath.toUri.toString)
-        }.getOrElse(Array(uri.toString))
-    }
-  }
-
-}
diff --git a/core/src/main/scala/org/apache/spark/deploy/DeployMessage.scala b/core/src/main/scala/org/apache/spark/deploy/DeployMessage.scala
index 18305ad3746a6..727cdbc4ef2d1 100644
--- a/core/src/main/scala/org/apache/spark/deploy/DeployMessage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/DeployMessage.scala
@@ -61,13 +61,35 @@ private[deploy] object DeployMessages {
   }
 
   /**
+   * An internal message that used by Master itself, in order to handle the
+   * `DecommissionWorkersOnHosts` request from `MasterWebUI` asynchronously.
+   * @param ids A collection of Worker ids, which should be decommissioned.
+   */
+  case class DecommissionWorkers(ids: Seq[String]) extends DeployMessage
+
+  /**
+   * A message that sent from Master to Worker to decommission the Worker.
+   * It's used for the case where decommission is triggered at MasterWebUI.
+   *
+   * Note that decommission a Worker will cause all the executors on that Worker
+   * to be decommissioned as well.
+   */
+  object DecommissionWorker extends DeployMessage
+
+  /**
+   * A message that sent by the Worker to itself when it receives a signal,
+   * indicating the Worker starts to decommission.
+   */
+  object WorkerDecommissionSigReceived extends DeployMessage
+
+  /**
+   * A message sent from Worker to Master to tell Master that the Worker has started
+   * decommissioning. It's used for the case where decommission is triggered at Worker.
+   *
    * @param id the worker id
-   * @param worker the worker endpoint ref
+   * @param workerRef the worker endpoint ref
    */
-  case class WorkerDecommission(
-      id: String,
-      worker: RpcEndpointRef)
-    extends DeployMessage
+  case class WorkerDecommissioning(id: String, workerRef: RpcEndpointRef) extends DeployMessage
 
   case class ExecutorStateChanged(
       appId: String,
@@ -108,6 +130,13 @@ private[deploy] object DeployMessages {
 
   case class Heartbeat(workerId: String, worker: RpcEndpointRef) extends DeployMessage
 
+  /**
+   * Used by the MasterWebUI to request the master to decommission all workers that are active on
+   * any of the given hostnames.
+   * @param hostnames: A list of hostnames without the ports. Like "localhost", "foo.bar.com" etc
+   */
+  case class DecommissionWorkersOnHosts(hostnames: Seq[String])
+
   // Master to Worker
 
   sealed trait RegisterWorkerResponse
@@ -158,8 +187,6 @@ private[deploy] object DeployMessages {
 
   case object ReregisterWithMaster // used when a worker attempts to reconnect to a master
 
-  case object DecommissionSelf // Mark as decommissioned. May be Master to Worker in the future.
-
   // AppClient to Master
 
   case class RegisterApplication(appDescription: ApplicationDescription, driver: RpcEndpointRef)
@@ -182,8 +209,10 @@ private[deploy] object DeployMessages {
     Utils.checkHostPort(hostPort)
   }
 
+  // When the host of Worker is lost or decommissioned, the `workerHost` is the host address
+  // of that Worker. Otherwise, it's None.
   case class ExecutorUpdated(id: Int, state: ExecutorState, message: Option[String],
-    exitStatus: Option[Int], workerLost: Boolean)
+    exitStatus: Option[Int], workerHost: Option[String])
 
   case class ApplicationRemoved(message: String)
 
diff --git a/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala b/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala
index ebfff89308886..eff1e15659fc4 100644
--- a/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala
@@ -44,8 +44,7 @@ private[deploy]
 class ExternalShuffleService(sparkConf: SparkConf, securityManager: SecurityManager)
   extends Logging {
   protected val masterMetricsSystem =
-    MetricsSystem.createMetricsSystem(MetricsSystemInstances.SHUFFLE_SERVICE,
-      sparkConf, securityManager)
+    MetricsSystem.createMetricsSystem(MetricsSystemInstances.SHUFFLE_SERVICE, sparkConf)
 
   private val enabled = sparkConf.get(config.SHUFFLE_SERVICE_ENABLED)
   private val port = sparkConf.get(config.SHUFFLE_SERVICE_PORT)
diff --git a/core/src/main/scala/org/apache/spark/deploy/FaultToleranceTest.scala b/core/src/main/scala/org/apache/spark/deploy/FaultToleranceTest.scala
index 6ff68b694f8f3..ab389f99b11a7 100644
--- a/core/src/main/scala/org/apache/spark/deploy/FaultToleranceTest.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/FaultToleranceTest.scala
@@ -205,7 +205,7 @@ private object FaultToleranceTest extends App with Logging {
 
   private def addWorkers(num: Int): Unit = {
     logInfo(s">>>>> ADD WORKERS $num <<<<<")
-    val masterUrls = getMasterUrls(masters)
+    val masterUrls = getMasterUrls(masters.toSeq)
     (1 to num).foreach { _ => workers += SparkDocker.startWorker(dockerMountDir, masterUrls) }
   }
 
@@ -216,7 +216,7 @@ private object FaultToleranceTest extends App with Logging {
     // Counter-hack: Because of a hack in SparkEnv#create() that changes this
     // property, we need to reset it.
     System.setProperty(config.DRIVER_PORT.key, "0")
-    sc = new SparkContext(getMasterUrls(masters), "fault-tolerance", containerSparkHome)
+    sc = new SparkContext(getMasterUrls(masters.toSeq), "fault-tolerance", containerSparkHome)
   }
 
   private def getMasterUrls(masters: Seq[TestMasterInfo]): String = {
@@ -279,7 +279,7 @@ private object FaultToleranceTest extends App with Logging {
     var liveWorkerIPs: Seq[String] = List()
 
     def stateValid(): Boolean = {
-      (workers.map(_.ip) -- liveWorkerIPs).isEmpty &&
+      workers.map(_.ip).forall(liveWorkerIPs.contains) &&
         numAlive == 1 && numStandby == masters.size - 1 && numLiveApps >= 1
     }
 
diff --git a/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala
index 6c3276c5c790a..f697892aacc83 100644
--- a/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala
@@ -22,7 +22,6 @@ import org.json4s.JsonDSL._
 
 import org.apache.spark.deploy.DeployMessages.{MasterStateResponse, WorkerStateResponse}
 import org.apache.spark.deploy.master._
-import org.apache.spark.deploy.master.RecoveryState.MasterState
 import org.apache.spark.deploy.worker.ExecutorRunner
 import org.apache.spark.resource.{ResourceInformation, ResourceRequirement}
 
@@ -81,7 +80,7 @@ private[deploy] object JsonProtocol {
   }
 
   /**
-   * Export the [[ApplicationInfo]] to a Json objec. An [[ApplicationInfo]] consists of the
+   * Export the [[ApplicationInfo]] to a Json object. An [[ApplicationInfo]] consists of the
    * information of an application.
    *
    * @return a Json object containing the following fields:
@@ -90,11 +89,12 @@ private[deploy] object JsonProtocol {
    *         `name` the description of the application
    *         `cores` total cores granted to the application
    *         `user` name of the user who submitted the application
-   *         `memoryperslave` minimal memory in MB required to each executor
-   *         `resourcesperslave` minimal resources required to each executor
+   *         `memoryperexecutor` minimal memory in MB required to each executor
+   *         `resourcesperexecutor` minimal resources required to each executor
    *         `submitdate` time in Date that the application is submitted
    *         `state` state of the application, see [[ApplicationState]]
    *         `duration` time in milliseconds that the application has been running
+   * For compatibility also returns the deprecated `memoryperslave` & `resourcesperslave` fields.
    */
   def writeApplicationInfo(obj: ApplicationInfo): JObject = {
     ("id" -> obj.id) ~
@@ -102,7 +102,10 @@ private[deploy] object JsonProtocol {
     ("name" -> obj.desc.name) ~
     ("cores" -> obj.coresGranted) ~
     ("user" -> obj.desc.user) ~
+    ("memoryperexecutor" -> obj.desc.memoryPerExecutorMB) ~
     ("memoryperslave" -> obj.desc.memoryPerExecutorMB) ~
+    ("resourcesperexecutor" -> obj.desc.resourceReqsPerExecutor
+      .toList.map(writeResourceRequirement)) ~
     ("resourcesperslave" -> obj.desc.resourceReqsPerExecutor
       .toList.map(writeResourceRequirement)) ~
     ("submitdate" -> obj.submitDate.toString) ~
@@ -117,14 +120,17 @@ private[deploy] object JsonProtocol {
    * @return a Json object containing the following fields:
    *         `name` the description of the application
    *         `cores` max cores that can be allocated to the application, 0 means unlimited
-   *         `memoryperslave` minimal memory in MB required to each executor
-   *         `resourcesperslave` minimal resources required to each executor
+   *         `memoryperexecutor` minimal memory in MB required to each executor
+   *         `resourcesperexecutor` minimal resources required to each executor
    *         `user` name of the user who submitted the application
    *         `command` the command string used to submit the application
+   * For compatibility also returns the deprecated `memoryperslave` & `resourcesperslave` fields.
    */
   def writeApplicationDescription(obj: ApplicationDescription): JObject = {
     ("name" -> obj.name) ~
     ("cores" -> obj.maxCores.getOrElse(0)) ~
+    ("memoryperexecutor" -> obj.memoryPerExecutorMB) ~
+    ("resourcesperexecutor" -> obj.resourceReqsPerExecutor.toList.map(writeResourceRequirement)) ~
     ("memoryperslave" -> obj.memoryPerExecutorMB) ~
     ("resourcesperslave" -> obj.resourceReqsPerExecutor.toList.map(writeResourceRequirement)) ~
     ("user" -> obj.user) ~
@@ -201,7 +207,8 @@ private[deploy] object JsonProtocol {
    *         master
    *         `completeddrivers` a list of Json objects of [[DriverInfo]] of the completed drivers
    *         of the master
-   *         `status` status of the master, see [[MasterState]]
+   *         `status` status of the master,
+   *         see [[org.apache.spark.deploy.master.RecoveryState.MasterState]]
    */
   def writeMasterState(obj: MasterStateResponse): JObject = {
     val aliveWorkers = obj.workers.filter(_.isAlive())
diff --git a/core/src/main/scala/org/apache/spark/deploy/LocalSparkCluster.scala b/core/src/main/scala/org/apache/spark/deploy/LocalSparkCluster.scala
index fc849d7f4372f..33851d9145d0a 100644
--- a/core/src/main/scala/org/apache/spark/deploy/LocalSparkCluster.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/LocalSparkCluster.scala
@@ -76,8 +76,8 @@ class LocalSparkCluster(
     logInfo("Shutting down local Spark cluster.")
     // Stop the workers before the master so they don't get upset that it disconnected
     workerRpcEnvs.foreach(_.shutdown())
-    masterRpcEnvs.foreach(_.shutdown())
     workerRpcEnvs.foreach(_.awaitTermination())
+    masterRpcEnvs.foreach(_.shutdown())
     masterRpcEnvs.foreach(_.awaitTermination())
     masterRpcEnvs.clear()
     workerRpcEnvs.clear()
diff --git a/core/src/main/scala/org/apache/spark/deploy/PythonRunner.scala b/core/src/main/scala/org/apache/spark/deploy/PythonRunner.scala
index 574ce60b19b4e..c3f73ed745da4 100644
--- a/core/src/main/scala/org/apache/spark/deploy/PythonRunner.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/PythonRunner.scala
@@ -44,7 +44,7 @@ object PythonRunner {
       .orElse(sparkConf.get(PYSPARK_PYTHON))
       .orElse(sys.env.get("PYSPARK_DRIVER_PYTHON"))
       .orElse(sys.env.get("PYSPARK_PYTHON"))
-      .getOrElse("python")
+      .getOrElse("python3")
 
     // Format python file paths before adding them to the PYTHONPATH
     val formattedPythonFile = formatPath(pythonFile)
@@ -69,7 +69,7 @@ object PythonRunner {
     pathElements ++= formattedPyFiles
     pathElements += PythonUtils.sparkPythonPath
     pathElements += sys.env.getOrElse("PYTHONPATH", "")
-    val pythonPath = PythonUtils.mergePythonPaths(pathElements: _*)
+    val pythonPath = PythonUtils.mergePythonPaths(pathElements.toSeq: _*)
 
     // Launch Python process
     val builder = new ProcessBuilder((Seq(pythonExec, formattedPythonFile) ++ otherArgs).asJava)
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
index 1180501e8c738..c719aef6f4934 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
@@ -392,7 +392,7 @@ private[spark] class SparkHadoopUtil extends Logging {
 
 }
 
-private[spark] object SparkHadoopUtil {
+private[spark] object SparkHadoopUtil extends Logging {
 
   private lazy val instance = new SparkHadoopUtil
 
@@ -450,6 +450,7 @@ private[spark] object SparkHadoopUtil {
           hadoopConf.set("fs.s3a.session.token", sessionToken)
         }
       }
+      appendHiveConfigs(hadoopConf)
       appendSparkHadoopConfigs(conf, hadoopConf)
       appendSparkHiveConfigs(conf, hadoopConf)
       val bufferSize = conf.get(BUFFER_SIZE).toString
@@ -457,11 +458,31 @@ private[spark] object SparkHadoopUtil {
     }
   }
 
+  private lazy val hiveConfKeys = {
+    val configFile = Utils.getContextOrSparkClassLoader.getResource("hive-site.xml")
+    if (configFile != null) {
+      val conf = new Configuration(false)
+      conf.addResource(configFile)
+      conf.iterator().asScala.toSeq
+    } else {
+      Nil
+    }
+  }
+
+  private def appendHiveConfigs(hadoopConf: Configuration): Unit = {
+    hiveConfKeys.foreach { kv =>
+      hadoopConf.set(kv.getKey, kv.getValue)
+    }
+  }
+
   private def appendSparkHadoopConfigs(conf: SparkConf, hadoopConf: Configuration): Unit = {
     // Copy any "spark.hadoop.foo=bar" spark properties into conf as "foo=bar"
     for ((key, value) <- conf.getAll if key.startsWith("spark.hadoop.")) {
       hadoopConf.set(key.substring("spark.hadoop.".length), value)
     }
+    if (conf.getOption("spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version").isEmpty) {
+      hadoopConf.set("mapreduce.fileoutputcommitter.algorithm.version", "1")
+    }
   }
 
   private def appendSparkHiveConfigs(conf: SparkConf, hadoopConf: Configuration): Unit = {
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
index 1271a3dbfc3f6..568bcf9b848a8 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -24,13 +24,13 @@ import java.security.PrivilegedExceptionAction
 import java.text.ParseException
 import java.util.{ServiceLoader, UUID}
 import java.util.jar.JarInputStream
+import javax.ws.rs.core.UriBuilder
 
 import scala.annotation.tailrec
 import scala.collection.JavaConverters._
 import scala.collection.mutable.ArrayBuffer
-import scala.util.{Failure, Properties, Success, Try}
+import scala.util.{Properties, Try}
 
-import org.apache.commons.io.FilenameUtils
 import org.apache.commons.lang3.StringUtils
 import org.apache.hadoop.conf.{Configuration => HadoopConfiguration}
 import org.apache.hadoop.fs.{FileSystem, Path}
@@ -240,7 +240,7 @@ private[spark] class SparkSubmit extends Logging {
     }
 
     // Set the deploy mode; default is client mode
-    var deployMode: Int = args.deployMode match {
+    val deployMode: Int = args.deployMode match {
       case "client" | null => CLIENT
       case "cluster" => CLUSTER
       case _ =>
@@ -304,28 +304,29 @@ private[spark] class SparkSubmit extends Logging {
       // Resolve maven dependencies if there are any and add classpath to jars. Add them to py-files
       // too for packages that include Python code
       val resolvedMavenCoordinates = DependencyUtils.resolveMavenDependencies(
-        args.packagesExclusions, args.packages, args.repositories, args.ivyRepoPath,
-        args.ivySettingsPath)
+        packagesTransitive = true, args.packagesExclusions, args.packages,
+        args.repositories, args.ivyRepoPath, args.ivySettingsPath)
 
-      if (!StringUtils.isBlank(resolvedMavenCoordinates)) {
+      if (resolvedMavenCoordinates.nonEmpty) {
         // In K8s client mode, when in the driver, add resolved jars early as we might need
         // them at the submit time for artifact downloading.
         // For example we might use the dependencies for downloading
-        // files from a Hadoop Compatible fs eg. S3. In this case the user might pass:
+        // files from a Hadoop Compatible fs e.g. S3. In this case the user might pass:
         // --packages com.amazonaws:aws-java-sdk:1.7.4:org.apache.hadoop:hadoop-aws:2.7.6
         if (isKubernetesClusterModeDriver) {
           val loader = getSubmitClassLoader(sparkConf)
-          for (jar <- resolvedMavenCoordinates.split(",")) {
+          for (jar <- resolvedMavenCoordinates) {
             addJarToClasspath(jar, loader)
           }
         } else if (isKubernetesCluster) {
           // We need this in K8s cluster mode so that we can upload local deps
           // via the k8s application, like in cluster mode driver
-          childClasspath ++= resolvedMavenCoordinates.split(",")
+          childClasspath ++= resolvedMavenCoordinates
         } else {
-          args.jars = mergeFileLists(args.jars, resolvedMavenCoordinates)
+          args.jars = mergeFileLists(args.jars, mergeFileLists(resolvedMavenCoordinates: _*))
           if (args.isPython || isInternal(args.primaryResource)) {
-            args.pyFiles = mergeFileLists(args.pyFiles, resolvedMavenCoordinates)
+            args.pyFiles = mergeFileLists(args.pyFiles,
+              mergeFileLists(resolvedMavenCoordinates: _*))
           }
         }
       }
@@ -365,7 +366,6 @@ private[spark] class SparkSubmit extends Logging {
     args.pyFiles = Option(args.pyFiles).map(resolveGlobPaths(_, hadoopConf)).orNull
     args.archives = Option(args.archives).map(resolveGlobPaths(_, hadoopConf)).orNull
 
-    lazy val secMgr = new SecurityManager(sparkConf)
 
     // In client mode, download remote files.
     var localPrimaryResource: String = null
@@ -373,24 +373,53 @@ private[spark] class SparkSubmit extends Logging {
     var localPyFiles: String = null
     if (deployMode == CLIENT) {
       localPrimaryResource = Option(args.primaryResource).map {
-        downloadFile(_, targetDir, sparkConf, hadoopConf, secMgr)
+        downloadFile(_, targetDir, sparkConf, hadoopConf)
       }.orNull
       localJars = Option(args.jars).map {
-        downloadFileList(_, targetDir, sparkConf, hadoopConf, secMgr)
+        downloadFileList(_, targetDir, sparkConf, hadoopConf)
       }.orNull
       localPyFiles = Option(args.pyFiles).map {
-        downloadFileList(_, targetDir, sparkConf, hadoopConf, secMgr)
+        downloadFileList(_, targetDir, sparkConf, hadoopConf)
       }.orNull
 
       if (isKubernetesClusterModeDriver) {
         // Replace with the downloaded local jar path to avoid propagating hadoop compatible uris.
         // Executors will get the jars from the Spark file server.
         // Explicitly download the related files here
-        args.jars = renameResourcesToLocalFS(args.jars, localJars)
-        val localFiles = Option(args.files).map {
-          downloadFileList(_, targetDir, sparkConf, hadoopConf, secMgr)
+        args.jars = localJars
+        val filesLocalFiles = Option(args.files).map {
+          downloadFileList(_, targetDir, sparkConf, hadoopConf)
         }.orNull
-        args.files = renameResourcesToLocalFS(args.files, localFiles)
+        val archiveLocalFiles = Option(args.archives).map { uris =>
+          val resolvedUris = Utils.stringToSeq(uris).map(Utils.resolveURI)
+          val localArchives = downloadFileList(
+            resolvedUris.map(
+              UriBuilder.fromUri(_).fragment(null).build().toString).mkString(","),
+            targetDir, sparkConf, hadoopConf)
+
+          // SPARK-33748: this mimics the behaviour of Yarn cluster mode. If the driver is running
+          // in cluster mode, the archives should be available in the driver's current working
+          // directory too.
+          Utils.stringToSeq(localArchives).map(Utils.resolveURI).zip(resolvedUris).map {
+            case (localArchive, resolvedUri) =>
+              val source = new File(localArchive.getPath)
+              val dest = new File(
+                ".",
+                if (resolvedUri.getFragment != null) resolvedUri.getFragment else source.getName)
+              logInfo(
+                s"Unpacking an archive $resolvedUri " +
+                  s"from ${source.getAbsolutePath} to ${dest.getAbsolutePath}")
+              Utils.deleteRecursively(dest)
+              Utils.unpack(source, dest)
+
+              // Keep the URIs of local files with the given fragments.
+              UriBuilder.fromUri(
+                localArchive).fragment(resolvedUri.getFragment).build().toString
+          }.mkString(",")
+        }.orNull
+        args.files = filesLocalFiles
+        args.archives = archiveLocalFiles
+        args.pyFiles = localPyFiles
       }
     }
 
@@ -417,7 +446,7 @@ private[spark] class SparkSubmit extends Logging {
             if (file.exists()) {
               file.toURI.toString
             } else {
-              downloadFile(resource, targetDir, sparkConf, hadoopConf, secMgr)
+              downloadFile(resource, targetDir, sparkConf, hadoopConf)
             }
           case _ => uri.toString
         }
@@ -606,6 +635,8 @@ private[spark] class SparkSubmit extends Logging {
         confKey = CORES_MAX.key),
       OptionAssigner(args.files, LOCAL | STANDALONE | MESOS | KUBERNETES, ALL_DEPLOY_MODES,
         confKey = FILES.key),
+      OptionAssigner(args.archives, LOCAL | STANDALONE | MESOS | KUBERNETES, ALL_DEPLOY_MODES,
+        confKey = ARCHIVES.key),
       OptionAssigner(args.jars, LOCAL, CLIENT, confKey = JARS.key),
       OptionAssigner(args.jars, STANDALONE | MESOS | KUBERNETES, ALL_DEPLOY_MODES,
         confKey = JARS.key),
@@ -795,6 +826,7 @@ private[spark] class SparkSubmit extends Logging {
     val pathConfigs = Seq(
       JARS.key,
       FILES.key,
+      ARCHIVES.key,
       "spark.yarn.dist.files",
       "spark.yarn.dist.archives",
       "spark.yarn.dist.jars")
@@ -820,22 +852,10 @@ private[spark] class SparkSubmit extends Logging {
     }
     sparkConf.set(SUBMIT_PYTHON_FILES, formattedPyFiles.split(",").toSeq)
 
-    (childArgs, childClasspath, sparkConf, childMainClass)
-  }
-
-  private def renameResourcesToLocalFS(resources: String, localResources: String): String = {
-    if (resources != null && localResources != null) {
-      val localResourcesSeq = Utils.stringToSeq(localResources)
-      Utils.stringToSeq(resources).map { resource =>
-        val filenameRemote = FilenameUtils.getName(new URI(resource).getPath)
-        localResourcesSeq.find { localUri =>
-          val filenameLocal = FilenameUtils.getName(new URI(localUri).getPath)
-          filenameRemote == filenameLocal
-        }.getOrElse(resource)
-      }.mkString(",")
-    } else {
-      resources
+    if (args.verbose) {
+      childArgs ++= Seq("--verbose")
     }
+    (childArgs.toSeq, childClasspath.toSeq, sparkConf, childMainClass)
   }
 
   // [SPARK-20328]. HadoopRDD calls into a Hadoop library that fetches delegation tokens with
@@ -935,6 +955,15 @@ private[spark] class SparkSubmit extends Logging {
     } catch {
       case t: Throwable =>
         throw findCause(t)
+    } finally {
+      if (!isShell(args.primaryResource) && !isSqlShell(args.mainClass) &&
+        !isThriftServer(args.mainClass)) {
+        try {
+          SparkContext.getActive.foreach(_.stop())
+        } catch {
+          case e: Throwable => logError(s"Failed to close SparkContext: $e")
+        }
+      }
     }
   }
 
@@ -1071,7 +1100,7 @@ object SparkSubmit extends CommandLineUtils with Logging {
 }
 
 /** Provides utility functions to be used inside SparkSubmit. */
-private[spark] object SparkSubmitUtils {
+private[spark] object SparkSubmitUtils extends Logging {
 
   // Exposed for testing
   var printStream = SparkSubmit.printStream
@@ -1160,33 +1189,49 @@ private[spark] object SparkSubmitUtils {
     val br: IBiblioResolver = new IBiblioResolver
     br.setM2compatible(true)
     br.setUsepoms(true)
+    val defaultInternalRepo : Option[String] = sys.env.get("DEFAULT_ARTIFACT_REPOSITORY")
+    br.setRoot(defaultInternalRepo.getOrElse("https://repo1.maven.org/maven2/"))
     br.setName("central")
     cr.add(br)
 
     val sp: IBiblioResolver = new IBiblioResolver
     sp.setM2compatible(true)
     sp.setUsepoms(true)
-    sp.setRoot("https://dl.bintray.com/spark-packages/maven")
+    sp.setRoot(sys.env.getOrElse(
+      "DEFAULT_ARTIFACT_REPOSITORY", "https://repos.spark-packages.org/"))
     sp.setName("spark-packages")
     cr.add(sp)
     cr
   }
 
   /**
-   * Output a comma-delimited list of paths for the downloaded jars to be added to the classpath
+   * Output a list of paths for the downloaded jars to be added to the classpath
    * (will append to jars in SparkSubmit).
    * @param artifacts Sequence of dependencies that were resolved and retrieved
-   * @param cacheDirectory directory where jars are cached
-   * @return a comma-delimited list of paths for the dependencies
+   * @param cacheDirectory Directory where jars are cached
+   * @return List of paths for the dependencies
    */
   def resolveDependencyPaths(
       artifacts: Array[AnyRef],
-      cacheDirectory: File): String = {
-    artifacts.map { artifactInfo =>
-      val artifact = artifactInfo.asInstanceOf[Artifact].getModuleRevisionId
+      cacheDirectory: File): Seq[String] = {
+    artifacts.map(_.asInstanceOf[Artifact]).filter { artifactInfo =>
+      if (artifactInfo.getExt == "jar") {
+        true
+      } else {
+        logInfo(s"Skipping non-jar dependency ${artifactInfo.getId}")
+        false
+      }
+    }.map { artifactInfo =>
+      val artifact = artifactInfo.getModuleRevisionId
+      val extraAttrs = artifactInfo.getExtraAttributes
+      val classifier = if (extraAttrs.containsKey("classifier")) {
+        "-" + extraAttrs.get("classifier")
+      } else {
+        ""
+      }
       cacheDirectory.getAbsolutePath + File.separator +
-        s"${artifact.getOrganisation}_${artifact.getName}-${artifact.getRevision}.jar"
-    }.mkString(",")
+        s"${artifact.getOrganisation}_${artifact.getName}-${artifact.getRevision}$classifier.jar"
+    }
   }
 
   /** Adds the given maven coordinates to Ivy's module descriptor. */
@@ -1250,7 +1295,12 @@ private[spark] object SparkSubmitUtils {
       settingsFile: String,
       remoteRepos: Option[String],
       ivyPath: Option[String]): IvySettings = {
-    val file = new File(settingsFile)
+    val uri = new URI(settingsFile)
+    val file = Option(uri.getScheme).getOrElse("file") match {
+      case "file" => new File(uri.getPath)
+      case scheme => throw new IllegalArgumentException(s"Scheme $scheme not supported in " +
+        "spark.jars.ivySettings")
+    }
     require(file.exists(), s"Ivy settings file $file does not exist")
     require(file.isFile(), s"Ivy settings file $file is not a normal file")
     val ivySettings: IvySettings = new IvySettings
@@ -1334,19 +1384,28 @@ private[spark] object SparkSubmitUtils {
    * Resolves any dependencies that were supplied through maven coordinates
    * @param coordinates Comma-delimited string of maven coordinates
    * @param ivySettings An IvySettings containing resolvers to use
+   * @param transitive Whether resolving transitive dependencies, default is true
    * @param exclusions Exclusions to apply when resolving transitive dependencies
-   * @return The comma-delimited path to the jars of the given maven artifacts including their
+   * @return Seq of path to the jars of the given maven artifacts including their
    *         transitive dependencies
    */
   def resolveMavenCoordinates(
       coordinates: String,
       ivySettings: IvySettings,
+      transitive: Boolean,
       exclusions: Seq[String] = Nil,
-      isTest: Boolean = false): String = {
+      isTest: Boolean = false): Seq[String] = {
     if (coordinates == null || coordinates.trim.isEmpty) {
-      ""
+      Nil
     } else {
       val sysOut = System.out
+      // Default configuration name for ivy
+      val ivyConfName = "default"
+
+      // A Module descriptor must be specified. Entries are dummy strings
+      val md = getModuleDescriptor
+
+      md.setDefaultConf(ivyConfName)
       try {
         // To prevent ivy from logging to system out
         System.setOut(printStream)
@@ -1363,7 +1422,7 @@ private[spark] object SparkSubmitUtils {
         val ivy = Ivy.newInstance(ivySettings)
         // Set resolve options to download transitive dependencies as well
         val resolveOptions = new ResolveOptions
-        resolveOptions.setTransitive(true)
+        resolveOptions.setTransitive(transitive)
         val retrieveOptions = new RetrieveOptions
         // Turn downloading and logging off for testing
         if (isTest) {
@@ -1374,14 +1433,6 @@ private[spark] object SparkSubmitUtils {
           resolveOptions.setDownload(true)
         }
 
-        // Default configuration name for ivy
-        val ivyConfName = "default"
-
-        // A Module descriptor must be specified. Entries are dummy strings
-        val md = getModuleDescriptor
-
-        md.setDefaultConf(ivyConfName)
-
         // Add exclusion rules for Spark and Scala Library
         addExclusionRules(ivySettings, ivyConfName, md)
         // add all supplied maven artifacts as dependencies
@@ -1399,12 +1450,10 @@ private[spark] object SparkSubmitUtils {
           packagesDirectory.getAbsolutePath + File.separator +
             "[organization]_[artifact]-[revision](-[classifier]).[ext]",
           retrieveOptions.setConfs(Array(ivyConfName)))
-        val paths = resolveDependencyPaths(rr.getArtifacts.toArray, packagesDirectory)
-        val mdId = md.getModuleRevisionId
-        clearIvyResolutionFiles(mdId, ivySettings, ivyConfName)
-        paths
+        resolveDependencyPaths(rr.getArtifacts.toArray, packagesDirectory)
       } finally {
         System.setOut(sysOut)
+        clearIvyResolutionFiles(md.getModuleRevisionId, ivySettings, ivyConfName)
       }
     }
   }
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
index 3090a3b10a97c..692e7eab2a8cb 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
@@ -183,6 +183,7 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
     name = Option(name).orElse(sparkProperties.get("spark.app.name")).orNull
     jars = Option(jars).orElse(sparkProperties.get(config.JARS.key)).orNull
     files = Option(files).orElse(sparkProperties.get(config.FILES.key)).orNull
+    archives = Option(archives).orElse(sparkProperties.get(config.ARCHIVES.key)).orNull
     pyFiles = Option(pyFiles).orElse(sparkProperties.get(config.SUBMIT_PYTHON_FILES.key)).orNull
     ivyRepoPath = sparkProperties.get("spark.jars.ivy").orNull
     ivySettingsPath = sparkProperties.get("spark.jars.ivySettings")
@@ -252,6 +253,9 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
         && Try(JavaUtils.byteStringAsBytes(executorMemory)).getOrElse(-1L) <= 0) {
       error("Executor memory must be a positive number")
     }
+    if (driverCores != null && Try(driverCores.toInt).getOrElse(-1) <= 0) {
+      error("Driver cores must be a positive number")
+    }
     if (executorCores != null && Try(executorCores.toInt).getOrElse(-1) <= 0) {
       error("Executor cores must be a positive number")
     }
@@ -512,6 +516,8 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
         |  --files FILES               Comma-separated list of files to be placed in the working
         |                              directory of each executor. File paths of these files
         |                              in executors can be accessed via SparkFiles.get(fileName).
+        |  --archives ARCHIVES         Comma-separated list of archives to be extracted into the
+        |                              working directory of each executor.
         |
         |  --conf, -c PROP=VALUE       Arbitrary Spark configuration property.
         |  --properties-file FILE      Path to a file from which to load extra properties. If not
@@ -562,8 +568,6 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
         |
         | Spark on YARN only:
         |  --queue QUEUE_NAME          The YARN queue to submit to (Default: "default").
-        |  --archives ARCHIVES         Comma separated list of archives to be extracted into the
-        |                              working directory of each executor.
       """.stripMargin
     )
 
diff --git a/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala b/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala
index e08709ec36ab0..c7c31a85b0636 100644
--- a/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala
@@ -149,11 +149,11 @@ private[spark] object StandaloneResourceUtils extends Logging {
 
   // used for UI
   def formatResourcesUsed(
-      resourcesTotal: Map[String, ResourceInformation],
-      resourcesUsed: Map[String, ResourceInformation]): String = {
-    resourcesTotal.map { case (rName, rInfo) =>
-      val used = resourcesUsed(rName).addresses.length
-      val total = rInfo.addresses.length
+      resourcesTotal: Map[String, Int],
+      resourcesUsed: Map[String, Int]): String = {
+    resourcesTotal.map { case (rName, totalSize) =>
+      val used = resourcesUsed(rName)
+      val total = totalSize
       s"$used / $total $rName"
     }.mkString(", ")
   }
diff --git a/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClient.scala b/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClient.scala
index eedf5e969e291..e5efb15f6bc51 100644
--- a/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClient.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClient.scala
@@ -31,6 +31,7 @@ import org.apache.spark.deploy.DeployMessages._
 import org.apache.spark.deploy.master.Master
 import org.apache.spark.internal.Logging
 import org.apache.spark.rpc._
+import org.apache.spark.scheduler.ExecutorDecommissionInfo
 import org.apache.spark.util.{RpcUtils, ThreadUtils}
 
 /**
@@ -174,14 +175,15 @@ private[spark] class StandaloneAppClient(
           cores))
         listener.executorAdded(fullId, workerId, hostPort, cores, memory)
 
-      case ExecutorUpdated(id, state, message, exitStatus, workerLost) =>
+      case ExecutorUpdated(id, state, message, exitStatus, workerHost) =>
         val fullId = appId + "/" + id
         val messageText = message.map(s => " (" + s + ")").getOrElse("")
         logInfo("Executor updated: %s is now %s%s".format(fullId, state, messageText))
         if (ExecutorState.isFinished(state)) {
-          listener.executorRemoved(fullId, message.getOrElse(""), exitStatus, workerLost)
+          listener.executorRemoved(fullId, message.getOrElse(""), exitStatus, workerHost)
         } else if (state == ExecutorState.DECOMMISSIONED) {
-          listener.executorDecommissioned(fullId, message.getOrElse(""))
+          listener.executorDecommissioned(fullId,
+            ExecutorDecommissionInfo(message.getOrElse(""), workerHost))
         }
 
       case WorkerRemoved(id, host, message) =>
diff --git a/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClientListener.scala b/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClientListener.scala
index 2e38a6847891d..76970ac9829c9 100644
--- a/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClientListener.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClientListener.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.deploy.client
 
+import org.apache.spark.scheduler.ExecutorDecommissionInfo
+
 /**
  * Callbacks invoked by deploy client when various events happen. There are currently five events:
  * connecting to the cluster, disconnecting, being given an executor, having an executor removed
@@ -37,9 +39,9 @@ private[spark] trait StandaloneAppClientListener {
       fullId: String, workerId: String, hostPort: String, cores: Int, memory: Int): Unit
 
   def executorRemoved(
-      fullId: String, message: String, exitStatus: Option[Int], workerLost: Boolean): Unit
+      fullId: String, message: String, exitStatus: Option[Int], workerHost: Option[String]): Unit
 
-  def executorDecommissioned(fullId: String, message: String): Unit
+  def executorDecommissioned(fullId: String, decommissionInfo: ExecutorDecommissionInfo): Unit
 
   def workerRemoved(workerId: String, host: String, message: String): Unit
 }
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationCache.scala b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationCache.scala
index fb2a67c2ab103..89b30a35ebebc 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationCache.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationCache.scala
@@ -156,18 +156,19 @@ private[history] class ApplicationCache(
    */
   @throws[NoSuchElementException]
   private def loadApplicationEntry(appId: String, attemptId: Option[String]): CacheEntry = {
-    logDebug(s"Loading application Entry $appId/$attemptId")
+    lazy val application = s"$appId/${attemptId.mkString}"
+    logDebug(s"Loading application Entry $application")
     metrics.loadCount.inc()
     val loadedUI = time(metrics.loadTimer) {
       metrics.lookupCount.inc()
       operations.getAppUI(appId, attemptId) match {
         case Some(loadedUI) =>
-          logDebug(s"Loaded application $appId/$attemptId")
+          logDebug(s"Loaded application $application")
           loadedUI
         case None =>
           metrics.lookupFailureCount.inc()
           // guava's cache logs via java.util log, so is of limited use. Hence: our own message
-          logInfo(s"Failed to load application attempt $appId/$attemptId")
+          logInfo(s"Failed to load application attempt $application")
           throw new NoSuchElementException(s"no application with application Id '$appId'" +
               attemptId.map { id => s" attemptId '$id'" }.getOrElse(" and no attempt Id"))
       }
@@ -182,7 +183,7 @@ private[history] class ApplicationCache(
       new CacheEntry(loadedUI, completed)
     } catch {
       case e: Exception =>
-        logWarning(s"Failed to initialize application UI for $appId/$attemptId", e)
+        logWarning(s"Failed to initialize application UI for $application", e)
         operations.detachSparkUI(appId, attemptId, loadedUI.ui)
         throw e
     }
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala
index 472b52957ed7f..f3f7db6bb0aba 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala
@@ -150,4 +150,11 @@ private[history] abstract class ApplicationHistoryProvider {
    */
   def onUIDetached(appId: String, attemptId: Option[String], ui: SparkUI): Unit = { }
 
+  /**
+   * Returns true if the given user has permission to view the UI of the given attempt.
+   *
+   * @throws NoSuchElementException if the given attempt doesn't exist
+   */
+  def checkUIViewPermissions(appId: String, attemptId: Option[String], user: String): Boolean
+
 }
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/BasicEventFilterBuilder.scala b/core/src/main/scala/org/apache/spark/deploy/history/BasicEventFilterBuilder.scala
index b18bf2665d6ce..57b05ff245258 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/BasicEventFilterBuilder.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/BasicEventFilterBuilder.scala
@@ -19,7 +19,6 @@ package org.apache.spark.deploy.history
 
 import scala.collection.mutable
 
-import org.apache.spark.SparkContext
 import org.apache.spark.deploy.history.EventFilter.FilterStatistics
 import org.apache.spark.internal.Logging
 import org.apache.spark.scheduler._
@@ -160,6 +159,8 @@ private[spark] class BasicEventFilter(
     case e: SparkListenerExecutorRemoved => liveExecutors.contains(e.executorId)
     case e: SparkListenerExecutorBlacklisted => liveExecutors.contains(e.executorId)
     case e: SparkListenerExecutorUnblacklisted => liveExecutors.contains(e.executorId)
+    case e: SparkListenerExecutorExcluded => liveExecutors.contains(e.executorId)
+    case e: SparkListenerExecutorUnexcluded => liveExecutors.contains(e.executorId)
     case e: SparkListenerStageExecutorMetrics => liveExecutors.contains(e.execId)
     case e: SparkListenerBlockManagerAdded => acceptBlockManagerEvent(e.blockManagerId)
     case e: SparkListenerBlockManagerRemoved => acceptBlockManagerEvent(e.blockManagerId)
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala
index 9f63a6441a838..b4771c80a175f 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala
@@ -116,7 +116,7 @@ object EventLogFileReader {
 
   def apply(fs: FileSystem, status: FileStatus): Option[EventLogFileReader] = {
     if (isSingleEventLog(status)) {
-      Some(new SingleFileEventLogFileReader(fs, status.getPath))
+      Some(new SingleFileEventLogFileReader(fs, status.getPath, Option(status)))
     } else if (isRollingEventLogs(status)) {
       Some(new RollingEventLogFilesFileReader(fs, status.getPath))
     } else {
@@ -164,10 +164,13 @@ object EventLogFileReader {
  * FileNotFoundException could occur if the log file is renamed before getting the
  * status of log file.
  */
-class SingleFileEventLogFileReader(
+private[history] class SingleFileEventLogFileReader(
     fs: FileSystem,
-    path: Path) extends EventLogFileReader(fs, path) {
-  private lazy val status = fileSystem.getFileStatus(rootPath)
+    path: Path,
+    maybeStatus: Option[FileStatus]) extends EventLogFileReader(fs, path) {
+  private lazy val status = maybeStatus.getOrElse(fileSystem.getFileStatus(rootPath))
+
+  def this(fs: FileSystem, path: Path) = this(fs, path, None)
 
   override def lastIndex: Option[Long] = None
 
@@ -203,7 +206,7 @@ class SingleFileEventLogFileReader(
  * This reader lists the files only once; if caller would like to play with updated list,
  * it needs to create another reader instance.
  */
-class RollingEventLogFilesFileReader(
+private[history] class RollingEventLogFilesFileReader(
     fs: FileSystem,
     path: Path) extends EventLogFileReader(fs, path) {
   import RollingEventLogFilesWriter._
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
index 25ea75acc37d3..d35d8606eb4b4 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -21,12 +21,13 @@ import java.io.{File, FileNotFoundException, IOException}
 import java.lang.{Long => JLong}
 import java.nio.file.Files
 import java.util.{Date, NoSuchElementException, ServiceLoader}
-import java.util.concurrent.{ConcurrentHashMap, ExecutorService, Future, TimeUnit}
+import java.util.concurrent.{ConcurrentHashMap, ExecutorService, TimeUnit}
 import java.util.zip.ZipOutputStream
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable
 import scala.io.Source
+import scala.util.control.NonFatal
 import scala.xml.Node
 
 import com.fasterxml.jackson.annotation.JsonIgnore
@@ -128,6 +129,8 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
   private val storePath = conf.get(LOCAL_STORE_DIR).map(new File(_))
   private val fastInProgressParsing = conf.get(FAST_IN_PROGRESS_PARSING)
 
+  private val hybridStoreEnabled = conf.get(History.HYBRID_STORE_ENABLED)
+
   // Visible for testing.
   private[history] val listing: KVStore = storePath.map { path =>
     val dbPath = Files.createDirectories(new File(path, "listing.ldb").toPath()).toFile()
@@ -158,6 +161,11 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
     new HistoryServerDiskManager(conf, path, listing, clock)
   }
 
+  private var memoryManager: HistoryServerMemoryManager = null
+  if (hybridStoreEnabled) {
+    memoryManager = new HistoryServerMemoryManager(conf)
+  }
+
   private val fileCompactor = new EventLogFileCompactor(conf, hadoopConf, fs,
     conf.get(EVENT_LOG_ROLLING_MAX_FILES_TO_RETAIN), conf.get(EVENT_LOG_COMPACTION_SCORE_THRESHOLD))
 
@@ -181,23 +189,24 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
     processing.remove(path.getName)
   }
 
-  private val blacklist = new ConcurrentHashMap[String, Long]
+  private val inaccessibleList = new ConcurrentHashMap[String, Long]
 
   // Visible for testing
-  private[history] def isBlacklisted(path: Path): Boolean = {
-    blacklist.containsKey(path.getName)
+  private[history] def isAccessible(path: Path): Boolean = {
+    !inaccessibleList.containsKey(path.getName)
   }
 
-  private def blacklist(path: Path): Unit = {
-    blacklist.put(path.getName, clock.getTimeMillis())
+  private def markInaccessible(path: Path): Unit = {
+    inaccessibleList.put(path.getName, clock.getTimeMillis())
   }
 
   /**
-   * Removes expired entries in the blacklist, according to the provided `expireTimeInSeconds`.
+   * Removes expired entries in the inaccessibleList, according to the provided
+   * `expireTimeInSeconds`.
    */
-  private def clearBlacklist(expireTimeInSeconds: Long): Unit = {
+  private def clearInaccessibleList(expireTimeInSeconds: Long): Unit = {
     val expiredThreshold = clock.getTimeMillis() - expireTimeInSeconds * 1000
-    blacklist.asScala.retain((_, creationTime) => creationTime >= expiredThreshold)
+    inaccessibleList.asScala.retain((_, creationTime) => creationTime >= expiredThreshold)
   }
 
   private val activeUIs = new mutable.HashMap[(String, Option[String]), LoadedAppUI]()
@@ -262,6 +271,9 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
 
   private def startPolling(): Unit = {
     diskManager.foreach(_.initialize())
+    if (memoryManager != null) {
+      memoryManager.initialize()
+    }
 
     // Validate the log directory.
     val path = new Path(logDir)
@@ -347,15 +359,7 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
     }
 
     val conf = this.conf.clone()
-    val secManager = new SecurityManager(conf)
-
-    secManager.setAcls(historyUiAclsEnable)
-    // make sure to set admin acls before view acls so they are properly picked up
-    secManager.setAdminAcls(historyUiAdminAcls ++ stringToSeq(attempt.adminAcls.getOrElse("")))
-    secManager.setViewAcls(attempt.info.sparkUser, stringToSeq(attempt.viewAcls.getOrElse("")))
-    secManager.setAdminAclsGroups(historyUiAdminAclsGroups ++
-      stringToSeq(attempt.adminAclsGroups.getOrElse("")))
-    secManager.setViewAclsGroups(stringToSeq(attempt.viewAclsGroups.getOrElse("")))
+    val secManager = createSecurityManager(conf, attempt)
 
     val kvstore = try {
       diskManager match {
@@ -449,6 +453,17 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
     }
   }
 
+  override def checkUIViewPermissions(appId: String, attemptId: Option[String],
+      user: String): Boolean = {
+    val app = load(appId)
+    val attempt = app.attempts.find(_.info.attemptId == attemptId).orNull
+    if (attempt == null) {
+      throw new NoSuchElementException()
+    }
+    val secManager = createSecurityManager(this.conf.clone(), attempt)
+    secManager.checkUIViewPermissions(user)
+  }
+
   /**
    * Builds the application list based on the current contents of the log directory.
    * Tries to reuse as much of the data already in memory as possible, by not reading
@@ -459,9 +474,21 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
       val newLastScanTime = clock.getTimeMillis()
       logDebug(s"Scanning $logDir with lastScanTime==$lastScanTime")
 
+      // Mark entries that are processing as not stale. Such entries do not have a chance to be
+      // updated with the new 'lastProcessed' time and thus any entity that completes processing
+      // right after this check and before the check for stale entities will be identified as stale
+      // and will be deleted from the UI until the next 'checkForLogs' run.
+      val notStale = mutable.HashSet[String]()
       val updated = Option(fs.listStatus(new Path(logDir))).map(_.toSeq).getOrElse(Nil)
-        .filter { entry => !isBlacklisted(entry.getPath) }
-        .filter { entry => !isProcessing(entry.getPath) }
+        .filter { entry => isAccessible(entry.getPath) }
+        .filter { entry =>
+          if (isProcessing(entry.getPath)) {
+            notStale.add(entry.getPath.toString())
+            false
+          } else {
+            true
+          }
+        }
         .flatMap { entry => EventLogFileReader(fs, entry) }
         .filter { reader =>
           try {
@@ -519,10 +546,21 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
               // If the file is currently not being tracked by the SHS, add an entry for it and try
               // to parse it. This will allow the cleaner code to detect the file as stale later on
               // if it was not possible to parse it.
-              listing.write(LogInfo(reader.rootPath.toString(), newLastScanTime, LogType.EventLogs,
-                None, None, reader.fileSizeForLastIndex, reader.lastIndex, None,
-                reader.completed))
-              reader.fileSizeForLastIndex > 0
+              try {
+                listing.write(LogInfo(reader.rootPath.toString(), newLastScanTime,
+                  LogType.EventLogs, None, None, reader.fileSizeForLastIndex, reader.lastIndex,
+                  None, reader.completed))
+                reader.fileSizeForLastIndex > 0
+              } catch {
+                case _: FileNotFoundException => false
+                case NonFatal(e) =>
+                  logWarning(s"Error while reading new log ${reader.rootPath}", e)
+                  false
+              }
+
+            case NonFatal(e) =>
+              logWarning(s"Error while filtering log ${reader.rootPath}", e)
+              false
           }
         }
         .sortWith { case (entry1, entry2) =>
@@ -550,12 +588,14 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
         .last(newLastScanTime - 1)
         .asScala
         .toList
-      stale.filterNot(isProcessing).foreach { log =>
-        log.appId.foreach { appId =>
-          cleanAppData(appId, log.attemptId, log.logPath)
-          listing.delete(classOf[LogInfo], log.logPath)
+      stale.filterNot(isProcessing)
+        .filterNot(info => notStale.contains(info.logPath))
+        .foreach { log =>
+          log.appId.foreach { appId =>
+            cleanAppData(appId, log.attemptId, log.logPath)
+            listing.delete(classOf[LogInfo], log.logPath)
+          }
         }
-      }
 
       lastScanTime.set(newLastScanTime)
     } catch {
@@ -677,8 +717,8 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
       case e: AccessControlException =>
         // We don't have read permissions on the log file
         logWarning(s"Unable to read log $rootPath", e)
-        blacklist(rootPath)
-        // SPARK-28157 We should remove this blacklisted entry from the KVStore
+        markInaccessible(rootPath)
+        // SPARK-28157 We should remove this inaccessible entry from the KVStore
         // to handle permission-only changes with the same file sizes later.
         listing.delete(classOf[LogInfo], rootPath.toString)
       case e: Exception =>
@@ -696,7 +736,7 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
 
   /**
    * Replay the given log file, saving the application in the listing db.
-   * Visable for testing
+   * Visible for testing
    */
   private[history] def doMergeApplicationListing(
       reader: EventLogFileReader,
@@ -946,8 +986,8 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
       }
     }
 
-    // Clean the blacklist from the expired entries.
-    clearBlacklist(CLEAN_INTERVAL_S)
+    // Clean the inaccessibleList from the expired entries.
+    clearInaccessibleList(CLEAN_INTERVAL_S)
   }
 
   private def deleteAttemptLogs(
@@ -1167,6 +1207,95 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
     // At this point the disk data either does not exist or was deleted because it failed to
     // load, so the event log needs to be replayed.
 
+    // If the hybrid store is enabled, try it first and fail back to leveldb store.
+    if (hybridStoreEnabled) {
+      try {
+        return createHybridStore(dm, appId, attempt, metadata)
+      } catch {
+        case e: Exception =>
+          logInfo(s"Failed to create HybridStore for $appId/${attempt.info.attemptId}." +
+            " Using LevelDB.", e)
+      }
+    }
+
+    createLevelDBStore(dm, appId, attempt, metadata)
+  }
+
+  private def createHybridStore(
+      dm: HistoryServerDiskManager,
+      appId: String,
+      attempt: AttemptInfoWrapper,
+      metadata: AppStatusStoreMetadata): KVStore = {
+    var retried = false
+    var hybridStore: HybridStore = null
+    val reader = EventLogFileReader(fs, new Path(logDir, attempt.logPath),
+      attempt.lastIndex)
+
+    // Use InMemoryStore to rebuild app store
+    while (hybridStore == null) {
+      // A RuntimeException will be thrown if the heap memory is not sufficient
+      memoryManager.lease(appId, attempt.info.attemptId, reader.totalSize,
+        reader.compressionCodec)
+      var store: HybridStore = null
+      try {
+        store = new HybridStore()
+        rebuildAppStore(store, reader, attempt.info.lastUpdated.getTime())
+        hybridStore = store
+      } catch {
+        case _: IOException if !retried =>
+          // compaction may touch the file(s) which app rebuild wants to read
+          // compaction wouldn't run in short interval, so try again...
+          logWarning(s"Exception occurred while rebuilding log path ${attempt.logPath} - " +
+            "trying again...")
+          store.close()
+          memoryManager.release(appId, attempt.info.attemptId)
+          retried = true
+        case e: Exception =>
+          store.close()
+          memoryManager.release(appId, attempt.info.attemptId)
+          throw e
+      }
+    }
+
+    // Create a LevelDB and start a background thread to dump data to LevelDB
+    var lease: dm.Lease = null
+    try {
+      logInfo(s"Leasing disk manager space for app $appId / ${attempt.info.attemptId}...")
+      lease = dm.lease(reader.totalSize, reader.compressionCodec.isDefined)
+      val levelDB = KVUtils.open(lease.tmpPath, metadata)
+      hybridStore.setLevelDB(levelDB)
+      hybridStore.switchToLevelDB(new HybridStore.SwitchToLevelDBListener {
+        override def onSwitchToLevelDBSuccess: Unit = {
+          logInfo(s"Completely switched to LevelDB for app $appId / ${attempt.info.attemptId}.")
+          levelDB.close()
+          val newStorePath = lease.commit(appId, attempt.info.attemptId)
+          hybridStore.setLevelDB(KVUtils.open(newStorePath, metadata))
+          memoryManager.release(appId, attempt.info.attemptId)
+        }
+        override def onSwitchToLevelDBFail(e: Exception): Unit = {
+          logWarning(s"Failed to switch to LevelDB for app $appId / ${attempt.info.attemptId}", e)
+          levelDB.close()
+          lease.rollback()
+        }
+      }, appId, attempt.info.attemptId)
+    } catch {
+      case e: Exception =>
+        hybridStore.close()
+        memoryManager.release(appId, attempt.info.attemptId)
+        if (lease != null) {
+          lease.rollback()
+        }
+        throw e
+    }
+
+    hybridStore
+  }
+
+  private def createLevelDBStore(
+      dm: HistoryServerDiskManager,
+      appId: String,
+      attempt: AttemptInfoWrapper,
+      metadata: AppStatusStoreMetadata): KVStore = {
     var retried = false
     var newStorePath: File = null
     while (newStorePath == null) {
@@ -1235,7 +1364,7 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
 
   private def deleteLog(fs: FileSystem, log: Path): Boolean = {
     var deleted = false
-    if (isBlacklisted(log)) {
+    if (!isAccessible(log)) {
       logDebug(s"Skipping deleting $log as we don't have permissions on it.")
     } else {
       try {
@@ -1264,6 +1393,19 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
         endProcessing(rootPath)
     }
   }
+
+  private def createSecurityManager(conf: SparkConf,
+      attempt: AttemptInfoWrapper): SecurityManager = {
+    val secManager = new SecurityManager(conf)
+    secManager.setAcls(historyUiAclsEnable)
+    // make sure to set admin acls before view acls so they are properly picked up
+    secManager.setAdminAcls(historyUiAdminAcls ++ stringToSeq(attempt.adminAcls.getOrElse("")))
+    secManager.setViewAcls(attempt.info.sparkUser, stringToSeq(attempt.viewAcls.getOrElse("")))
+    secManager.setAdminAclsGroups(historyUiAdminAclsGroups ++
+      stringToSeq(attempt.adminAclsGroups.getOrElse("")))
+    secManager.setViewAclsGroups(stringToSeq(attempt.viewAclsGroups.getOrElse("")))
+    secManager
+  }
 }
 
 private[history] object FsHistoryProvider {
@@ -1418,14 +1560,9 @@ private[history] class AppListingListener(
   private class MutableApplicationInfo {
     var id: String = null
     var name: String = null
-    var coresGranted: Option[Int] = None
-    var maxCores: Option[Int] = None
-    var coresPerExecutor: Option[Int] = None
-    var memoryPerExecutorMB: Option[Int] = None
 
     def toView(): ApplicationInfoWrapper = {
-      val apiInfo = ApplicationInfo(id, name, coresGranted, maxCores, coresPerExecutor,
-        memoryPerExecutorMB, Nil)
+      val apiInfo = ApplicationInfo(id, name, None, None, None, None, Nil)
       new ApplicationInfoWrapper(apiInfo, List(attempt.toView()))
     }
 
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryAppStatusStore.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryAppStatusStore.scala
index 741050027fc6b..ac0f102d81a6a 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryAppStatusStore.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryAppStatusStore.scala
@@ -72,7 +72,8 @@ private[spark] class HistoryAppStatusStore(
       source.totalGCTime, source.totalInputBytes, source.totalShuffleRead,
       source.totalShuffleWrite, source.isBlacklisted, source.maxMemory, source.addTime,
       source.removeTime, source.removeReason, newExecutorLogs, source.memoryMetrics,
-      source.blacklistedInStages, source.peakMemoryMetrics, source.attributes, source.resources)
+      source.blacklistedInStages, source.peakMemoryMetrics, source.attributes, source.resources,
+      source.resourceProfileId, source.isExcluded, source.excludedInStages)
   }
 
 }
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
index aa9e9a6dd4887..bb13f34818a62 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
@@ -76,9 +76,7 @@ class HistoryServer(
       // attempt ID (separated by a slash).
       val parts = Option(req.getPathInfo()).getOrElse("").split("/")
       if (parts.length < 2) {
-        res.sendError(HttpServletResponse.SC_BAD_REQUEST,
-          s"Unexpected path info in request (URI = ${req.getRequestURI()}")
-        return
+        res.sendRedirect("/")
       }
 
       val appId = parts(1)
@@ -130,6 +128,11 @@ class HistoryServer(
     appCache.withSparkUI(appId, attemptId)(fn)
   }
 
+  override def checkUIViewPermissions(appId: String, attemptId: Option[String],
+      user: String): Boolean = {
+    provider.checkUIViewPermissions(appId, attemptId, user)
+  }
+
   initialize()
 
   /**
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala
index 0a1f33395ad62..31f9d185174dc 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala
@@ -75,14 +75,29 @@ private class HistoryServerDiskManager(
 
     // Go through the recorded store directories and remove any that may have been removed by
     // external code.
-    val orphans = listing.view(classOf[ApplicationStoreInfo]).asScala.filter { info =>
-      !new File(info.path).exists()
-    }.toSeq
+    val (existences, orphans) = listing
+      .view(classOf[ApplicationStoreInfo])
+      .asScala
+      .toSeq
+      .partition { info =>
+        new File(info.path).exists()
+      }
 
     orphans.foreach { info =>
       listing.delete(info.getClass(), info.path)
     }
 
+    // Reading level db would trigger table file compaction, then it may cause size of level db
+    // directory changed. When service restarts, "currentUsage" is calculated from real directory
+    // size. Update "ApplicationStoreInfo.size" to ensure "currentUsage" equals
+    // sum of "ApplicationStoreInfo.size".
+    existences.foreach { info =>
+      val fileSize = sizeOf(new File(info.path))
+      if (fileSize != info.size) {
+        listing.write(info.copy(size = fileSize))
+      }
+    }
+
     logInfo("Initialized disk manager: " +
       s"current usage = ${Utils.bytesToString(currentUsage.get())}, " +
       s"max usage = ${Utils.bytesToString(maxUsage)}")
@@ -122,10 +137,12 @@ private class HistoryServerDiskManager(
    * being used so that it's not evicted when running out of designated space.
    */
   def openStore(appId: String, attemptId: Option[String]): Option[File] = {
+    var newSize: Long = 0
     val storePath = active.synchronized {
       val path = appStorePath(appId, attemptId)
       if (path.isDirectory()) {
-        active(appId -> attemptId) = sizeOf(path)
+        newSize = sizeOf(path)
+        active(appId -> attemptId) = newSize
         Some(path)
       } else {
         None
@@ -133,7 +150,7 @@ private class HistoryServerDiskManager(
     }
 
     storePath.foreach { path =>
-      updateAccessTime(appId, attemptId)
+      updateApplicationStoreInfo(appId, attemptId, newSize)
     }
 
     storePath
@@ -233,15 +250,16 @@ private class HistoryServerDiskManager(
     }
   }
 
-  private def appStorePath(appId: String, attemptId: Option[String]): File = {
+  private[history] def appStorePath(appId: String, attemptId: Option[String]): File = {
     val fileName = appId + attemptId.map("_" + _).getOrElse("") + ".ldb"
     new File(appStoreDir, fileName)
   }
 
-  private def updateAccessTime(appId: String, attemptId: Option[String]): Unit = {
+  private def updateApplicationStoreInfo(
+      appId: String, attemptId: Option[String], newSize: Long): Unit = {
     val path = appStorePath(appId, attemptId)
-    val info = ApplicationStoreInfo(path.getAbsolutePath(), clock.getTimeMillis(), appId, attemptId,
-      sizeOf(path))
+    val info = ApplicationStoreInfo(path.getAbsolutePath(), clock.getTimeMillis(), appId,
+      attemptId, newSize)
     listing.write(info)
   }
 
@@ -297,7 +315,7 @@ private class HistoryServerDiskManager(
           s"exceeded ($current > $max)")
       }
 
-      updateAccessTime(appId, attemptId)
+      updateApplicationStoreInfo(appId, attemptId, newSize)
 
       active.synchronized {
         active(appId -> attemptId) = newSize
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerMemoryManager.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerMemoryManager.scala
new file mode 100644
index 0000000000000..00e58cbdc57b9
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerMemoryManager.scala
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.deploy.history
+
+import java.util.concurrent.atomic.AtomicLong
+
+import scala.collection.mutable.HashMap
+
+import org.apache.spark.SparkConf
+import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config.History._
+import org.apache.spark.util.Utils
+
+/**
+ * A class used to keep track of in-memory store usage by the SHS.
+ */
+private class HistoryServerMemoryManager(
+    conf: SparkConf) extends Logging {
+
+  private val maxUsage = conf.get(MAX_IN_MEMORY_STORE_USAGE)
+  // Visible for testing.
+  private[history] val currentUsage = new AtomicLong(0L)
+  private[history] val active = new HashMap[(String, Option[String]), Long]()
+
+  def initialize(): Unit = {
+    logInfo("Initialized memory manager: " +
+      s"current usage = ${Utils.bytesToString(currentUsage.get())}, " +
+      s"max usage = ${Utils.bytesToString(maxUsage)}")
+  }
+
+  def lease(
+      appId: String,
+      attemptId: Option[String],
+      eventLogSize: Long,
+      codec: Option[String]): Unit = {
+    val memoryUsage = approximateMemoryUsage(eventLogSize, codec)
+    if (memoryUsage + currentUsage.get > maxUsage) {
+      throw new RuntimeException("Not enough memory to create hybrid store " +
+        s"for app $appId / $attemptId.")
+    }
+    active.synchronized {
+      active(appId -> attemptId) = memoryUsage
+    }
+    currentUsage.addAndGet(memoryUsage)
+    logInfo(s"Leasing ${Utils.bytesToString(memoryUsage)} memory usage for " +
+      s"app $appId / $attemptId")
+  }
+
+  def release(appId: String, attemptId: Option[String]): Unit = {
+    val memoryUsage = active.synchronized { active.remove(appId -> attemptId) }
+
+    memoryUsage match {
+      case Some(m) =>
+        currentUsage.addAndGet(-m)
+        logInfo(s"Released ${Utils.bytesToString(m)} memory usage for " +
+          s"app $appId / $attemptId")
+      case None =>
+    }
+  }
+
+  private def approximateMemoryUsage(eventLogSize: Long, codec: Option[String]): Long = {
+    codec match {
+      case Some("zstd") =>
+        eventLogSize * 10
+      case Some(_) =>
+        eventLogSize * 4
+      case None =>
+        eventLogSize / 2
+    }
+  }
+}
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HybridStore.scala b/core/src/main/scala/org/apache/spark/deploy/history/HybridStore.scala
new file mode 100644
index 0000000000000..4eb5c15d4ed18
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HybridStore.scala
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.deploy.history
+
+import java.util.Collection
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.atomic.AtomicBoolean
+
+import scala.collection.JavaConverters._
+
+import com.google.common.collect.Lists;
+
+import org.apache.spark.util.kvstore._
+
+/**
+ * An implementation of KVStore that accelerates event logs loading.
+ *
+ * When rebuilding the application state from event logs, HybridStore will
+ * write data to InMemoryStore at first and use a background thread to dump
+ * data to LevelDB once the app store is restored. We don't expect write
+ * operations (except the case for caching) after calling switch to level DB.
+ */
+
+private[history] class HybridStore extends KVStore {
+
+  private val inMemoryStore = new InMemoryStore()
+
+  private var levelDB: LevelDB = null
+
+  // Flag to indicate whether we should use inMemoryStore or levelDB
+  private val shouldUseInMemoryStore = new AtomicBoolean(true)
+
+  // Flag to indicate whether this hybrid store is closed, use this flag
+  // to avoid starting background thread after the store is closed
+  private val closed = new AtomicBoolean(false)
+
+  // A background thread that dumps data from inMemoryStore to levelDB
+  private var backgroundThread: Thread = null
+
+  // A hash map that stores all classes that had been written to inMemoryStore
+  // Visible for testing
+  private[history] val klassMap = new ConcurrentHashMap[Class[_], Boolean]
+
+  override def getMetadata[T](klass: Class[T]): T = {
+    getStore().getMetadata(klass)
+  }
+
+  override def setMetadata(value: Object): Unit = {
+    getStore().setMetadata(value)
+  }
+
+  override def read[T](klass: Class[T], naturalKey: Object): T = {
+    getStore().read(klass, naturalKey)
+  }
+
+  override def write(value: Object): Unit = {
+    getStore().write(value)
+
+    if (backgroundThread == null) {
+      // New classes won't be dumped once the background thread is started
+      klassMap.putIfAbsent(value.getClass(), true)
+    }
+  }
+
+  override def delete(klass: Class[_], naturalKey: Object): Unit = {
+    if (backgroundThread != null) {
+      throw new IllegalStateException("delete() shouldn't be called after " +
+        "the hybrid store begins switching to levelDB")
+    }
+
+    getStore().delete(klass, naturalKey)
+  }
+
+  override def view[T](klass: Class[T]): KVStoreView[T] = {
+    getStore().view(klass)
+  }
+
+  override def count(klass: Class[_]): Long = {
+    getStore().count(klass)
+  }
+
+  override def count(klass: Class[_], index: String, indexedValue: Object): Long = {
+    getStore().count(klass, index, indexedValue)
+  }
+
+  override def close(): Unit = {
+    try {
+      closed.set(true)
+      if (backgroundThread != null && backgroundThread.isAlive()) {
+        // The background thread is still running, wait for it to finish
+        backgroundThread.join()
+      }
+    } finally {
+      inMemoryStore.close()
+      if (levelDB != null) {
+        levelDB.close()
+      }
+    }
+  }
+
+  override def removeAllByIndexValues[T](
+      klass: Class[T],
+      index: String,
+      indexValues: Collection[_]): Boolean = {
+    if (backgroundThread != null) {
+      throw new IllegalStateException("removeAllByIndexValues() shouldn't be " +
+        "called after the hybrid store begins switching to levelDB")
+    }
+
+    getStore().removeAllByIndexValues(klass, index, indexValues)
+  }
+
+  def setLevelDB(levelDB: LevelDB): Unit = {
+    this.levelDB = levelDB
+  }
+
+  /**
+   * This method is called when the writing is done for inMemoryStore. A
+   * background thread will be created and be started to dump data in inMemoryStore
+   * to levelDB. Once the dumping is completed, the underlying kvstore will be
+   * switched to levelDB.
+   */
+  def switchToLevelDB(
+      listener: HybridStore.SwitchToLevelDBListener,
+      appId: String,
+      attemptId: Option[String]): Unit = {
+    if (closed.get) {
+      return
+    }
+
+    backgroundThread = new Thread(() => {
+      try {
+        for (klass <- klassMap.keys().asScala) {
+          val values = Lists.newArrayList(
+              inMemoryStore.view(klass).closeableIterator())
+          levelDB.writeAll(values)
+        }
+        listener.onSwitchToLevelDBSuccess()
+        shouldUseInMemoryStore.set(false)
+        inMemoryStore.close()
+      } catch {
+        case e: Exception =>
+          listener.onSwitchToLevelDBFail(e)
+      }
+    })
+    backgroundThread.setDaemon(true)
+    backgroundThread.setName(s"hybridstore-$appId-$attemptId")
+    backgroundThread.start()
+  }
+
+  /**
+   * This method return the store that we should use.
+   * Visible for testing.
+   */
+  private[history] def getStore(): KVStore = {
+    if (shouldUseInMemoryStore.get) {
+      inMemoryStore
+    } else {
+      levelDB
+    }
+  }
+}
+
+private[history] object HybridStore {
+
+  trait SwitchToLevelDBListener {
+
+    def onSwitchToLevelDBSuccess(): Unit
+
+    def onSwitchToLevelDBFail(e: Exception): Unit
+  }
+}
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
index 71df5dfa423a9..c964e343ca6c9 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
@@ -87,9 +87,9 @@ private[deploy] class Master(
   Utils.checkHost(address.host)
 
   private val masterMetricsSystem =
-    MetricsSystem.createMetricsSystem(MetricsSystemInstances.MASTER, conf, securityMgr)
+    MetricsSystem.createMetricsSystem(MetricsSystemInstances.MASTER, conf)
   private val applicationMetricsSystem =
-    MetricsSystem.createMetricsSystem(MetricsSystemInstances.APPLICATIONS, conf, securityMgr)
+    MetricsSystem.createMetricsSystem(MetricsSystemInstances.APPLICATIONS, conf)
   private val masterSource = new MasterSource(this)
 
   // After onStart, webUi will be set
@@ -143,9 +143,15 @@ private[deploy] class Master(
     logInfo(s"Running Spark version ${org.apache.spark.SPARK_VERSION}")
     webUi = new MasterWebUI(this, webUiPort)
     webUi.bind()
-    masterWebUiUrl = s"${webUi.scheme}$masterPublicAddress:${webUi.boundPort}"
+    masterWebUiUrl = webUi.webUrl
     if (reverseProxy) {
-      masterWebUiUrl = conf.get(UI_REVERSE_PROXY_URL).orElse(Some(masterWebUiUrl)).get
+      val uiReverseProxyUrl = conf.get(UI_REVERSE_PROXY_URL).map(_.stripSuffix("/"))
+      if (uiReverseProxyUrl.nonEmpty) {
+        System.setProperty("spark.ui.proxyBase", uiReverseProxyUrl.get)
+        // If the master URL has a path component, it must end with a slash.
+        // Otherwise the browser generates incorrect relative links
+        masterWebUiUrl = uiReverseProxyUrl.get + "/"
+      }
       webUi.addProxy()
       logInfo(s"Spark Master is acting as a reverse proxy. Master, Workers and " +
        s"Applications UIs are available at $masterWebUiUrl")
@@ -243,8 +249,7 @@ private[deploy] class Master(
       logError("Leadership has been revoked -- master shutting down.")
       System.exit(0)
 
-    case WorkerDecommission(id, workerRef) =>
-      logInfo("Recording worker %s decommissioning".format(id))
+    case WorkerDecommissioning(id, workerRef) =>
       if (state == RecoveryState.STANDBY) {
         workerRef.send(MasterInStandby)
       } else {
@@ -252,6 +257,19 @@ private[deploy] class Master(
         idToWorker.get(id).foreach(decommissionWorker)
       }
 
+    case DecommissionWorkers(ids) =>
+      // The caller has already checked the state when handling DecommissionWorkersOnHosts,
+      // so it should not be the STANDBY
+      assert(state != RecoveryState.STANDBY)
+      ids.foreach ( id =>
+        // We use foreach since get gives us an option and we can skip the failures.
+        idToWorker.get(id).foreach { w =>
+          decommissionWorker(w)
+          // Also send a message to the worker node to notify.
+          w.endpoint.send(DecommissionWorker)
+        }
+      )
+
     case RegisterWorker(
       id, workerHost, workerPort, workerRef, cores, memory, workerWebUiUrl,
       masterAddress, resources) =>
@@ -292,54 +310,6 @@ private[deploy] class Master(
         schedule()
       }
 
-    case ExecutorStateChanged(appId, execId, state, message, exitStatus) =>
-      val execOption = idToApp.get(appId).flatMap(app => app.executors.get(execId))
-      execOption match {
-        case Some(exec) =>
-          val appInfo = idToApp(appId)
-          val oldState = exec.state
-          exec.state = state
-
-          if (state == ExecutorState.RUNNING) {
-            assert(oldState == ExecutorState.LAUNCHING,
-              s"executor $execId state transfer from $oldState to RUNNING is illegal")
-            appInfo.resetRetryCount()
-          }
-
-          exec.application.driver.send(ExecutorUpdated(execId, state, message, exitStatus, false))
-
-          if (ExecutorState.isFinished(state)) {
-            // Remove this executor from the worker and app
-            logInfo(s"Removing executor ${exec.fullId} because it is $state")
-            // If an application has already finished, preserve its
-            // state to display its information properly on the UI
-            if (!appInfo.isFinished) {
-              appInfo.removeExecutor(exec)
-            }
-            exec.worker.removeExecutor(exec)
-
-            val normalExit = exitStatus == Some(0)
-            // Only retry certain number of times so we don't go into an infinite loop.
-            // Important note: this code path is not exercised by tests, so be very careful when
-            // changing this `if` condition.
-            // We also don't count failures from decommissioned workers since they are "expected."
-            if (!normalExit
-                && oldState != ExecutorState.DECOMMISSIONED
-                && appInfo.incrementRetryCount() >= maxExecutorRetries
-                && maxExecutorRetries >= 0) { // < 0 disables this application-killing path
-              val execs = appInfo.executors.values
-              if (!execs.exists(_.state == ExecutorState.RUNNING)) {
-                logError(s"Application ${appInfo.desc.name} with ID ${appInfo.id} failed " +
-                  s"${appInfo.retryCount} times; removing it")
-                removeApplication(appInfo, ApplicationState.FAILED)
-              }
-            }
-          }
-          schedule()
-        case None =>
-          logWarning(s"Got status update for unknown executor $appId/$execId")
-      }
-
     case DriverStateChanged(driverId, state, exception) =>
       state match {
         case DriverState.ERROR | DriverState.FINISHED | DriverState.KILLED | DriverState.FAILED =>
@@ -525,6 +495,62 @@ private[deploy] class Master(
     case KillExecutors(appId, executorIds) =>
       val formattedExecutorIds = formatExecutorIds(executorIds)
       context.reply(handleKillExecutors(appId, formattedExecutorIds))
+
+    case DecommissionWorkersOnHosts(hostnames) =>
+      if (state != RecoveryState.STANDBY) {
+        context.reply(decommissionWorkersOnHosts(hostnames))
+      } else {
+        context.reply(0)
+      }
+
+    case ExecutorStateChanged(appId, execId, state, message, exitStatus) =>
+      val execOption = idToApp.get(appId).flatMap(app => app.executors.get(execId))
+      execOption match {
+        case Some(exec) =>
+          val appInfo = idToApp(appId)
+          val oldState = exec.state
+          exec.state = state
+
+          if (state == ExecutorState.RUNNING) {
+            assert(oldState == ExecutorState.LAUNCHING,
+              s"executor $execId state transfer from $oldState to RUNNING is illegal")
+            appInfo.resetRetryCount()
+          }
+
+          exec.application.driver.send(ExecutorUpdated(execId, state, message, exitStatus, None))
+
+          if (ExecutorState.isFinished(state)) {
+            // Remove this executor from the worker and app
+            logInfo(s"Removing executor ${exec.fullId} because it is $state")
+            // If an application has already finished, preserve its
+            // state to display its information properly on the UI
+            if (!appInfo.isFinished) {
+              appInfo.removeExecutor(exec)
+            }
+            exec.worker.removeExecutor(exec)
+
+            val normalExit = exitStatus == Some(0)
+            // Only retry certain number of times so we don't go into an infinite loop.
+            // Important note: this code path is not exercised by tests, so be very careful when
+            // changing this `if` condition.
+            // We also don't count failures from decommissioned workers since they are "expected."
+            if (!normalExit
+              && oldState != ExecutorState.DECOMMISSIONED
+              && appInfo.incrementRetryCount() >= maxExecutorRetries
+              && maxExecutorRetries >= 0) { // < 0 disables this application-killing path
+              val execs = appInfo.executors.values
+              if (!execs.exists(_.state == ExecutorState.RUNNING)) {
+                logError(s"Application ${appInfo.desc.name} with ID ${appInfo.id} failed " +
+                  s"${appInfo.retryCount} times; removing it")
+                removeApplication(appInfo, ApplicationState.FAILED)
+              }
+            }
+          }
+          schedule()
+        case None =>
+          logWarning(s"Got status update for unknown executor $appId/$execId")
+      }
+      context.reply(true)
   }
 
   override def onDisconnected(address: RpcAddress): Unit = {
@@ -715,7 +741,9 @@ private[deploy] class Master(
         val usableWorkers = workers.toArray.filter(_.state == WorkerState.ALIVE)
           .filter(canLaunchExecutor(_, app.desc))
           .sortBy(_.coresFree).reverse
-        if (waitingApps.length == 1 && usableWorkers.isEmpty) {
+        val appMayHang = waitingApps.length == 1 &&
+          waitingApps.head.executors.isEmpty && usableWorkers.isEmpty
+        if (appMayHang) {
           logWarning(s"App ${app.id} requires more resource than any of Workers could have.")
         }
         val assignedCores = scheduleExecutorsOnWorkers(app, usableWorkers, spreadOutApps)
@@ -861,6 +889,31 @@ private[deploy] class Master(
     true
   }
 
+  /**
+   * Decommission all workers that are active on any of the given hostnames. The decommissioning is
+   * asynchronously done by enqueueing WorkerDecommission messages to self. No checks are done about
+   * the prior state of the worker. So an already decommissioned worker will match as well.
+   *
+   * @param hostnames: A list of hostnames without the ports. Like "localhost", "foo.bar.com" etc
+   *
+   * Returns the number of workers that matched the hostnames.
+   */
+  private def decommissionWorkersOnHosts(hostnames: Seq[String]): Integer = {
+    val hostnamesSet = hostnames.map(_.toLowerCase(Locale.ROOT)).toSet
+    val workersToRemove = addressToWorker
+      .filterKeys(addr => hostnamesSet.contains(addr.host.toLowerCase(Locale.ROOT)))
+      .values
+
+    val workersToRemoveHostPorts = workersToRemove.map(_.hostPort)
+    logInfo(s"Decommissioning the workers with host:ports ${workersToRemoveHostPorts}")
+
+    // The workers are removed async to avoid blocking the receive loop for the entire batch
+    self.send(DecommissionWorkers(workersToRemove.map(_.id).toSeq))
+
+    // Return the count of workers actually removed
+    workersToRemove.size
+  }
+
   private def decommissionWorker(worker: WorkerInfo): Unit = {
     if (worker.state != WorkerState.DECOMMISSIONED) {
       logInfo("Decommissioning worker %s on %s:%d".format(worker.id, worker.host, worker.port))
@@ -869,7 +922,11 @@ private[deploy] class Master(
         logInfo("Telling app of decommission executors")
         exec.application.driver.send(ExecutorUpdated(
           exec.id, ExecutorState.DECOMMISSIONED,
-          Some("worker decommissioned"), None, workerLost = false))
+          Some("worker decommissioned"), None,
+          // worker host is being set here to let the driver know that the host (aka. worker)
+          // is also being decommissioned. So the driver can unregister all the shuffle map
+          // statues located at this host when it receives the executor lost event.
+          Some(worker.host)))
         exec.state = ExecutorState.DECOMMISSIONED
         exec.application.removeExecutor(exec)
       }
@@ -890,7 +947,7 @@ private[deploy] class Master(
     for (exec <- worker.executors.values) {
       logInfo("Telling app of lost executor: " + exec.id)
       exec.application.driver.send(ExecutorUpdated(
-        exec.id, ExecutorState.LOST, Some("worker lost"), None, workerLost = true))
+        exec.id, ExecutorState.LOST, Some("worker lost"), None, Some(worker.host)))
       exec.state = ExecutorState.LOST
       exec.application.removeExecutor(exec)
     }
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ZooKeeperPersistenceEngine.scala b/core/src/main/scala/org/apache/spark/deploy/master/ZooKeeperPersistenceEngine.scala
index 8eae445b439d9..ded816b992db8 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/ZooKeeperPersistenceEngine.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/ZooKeeperPersistenceEngine.scala
@@ -52,7 +52,7 @@ private[master] class ZooKeeperPersistenceEngine(conf: SparkConf, val serializer
 
   override def read[T: ClassTag](prefix: String): Seq[T] = {
     zk.getChildren.forPath(workingDir).asScala
-      .filter(_.startsWith(prefix)).flatMap(deserializeFromFile[T])
+      .filter(_.startsWith(prefix)).flatMap(deserializeFromFile[T]).toSeq
   }
 
   override def close(): Unit = {
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala b/core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala
index 97d6d01fb1d91..10d3acaa4e1bd 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala
@@ -148,9 +148,9 @@ private[ui] class ApplicationPage(parent: MasterWebUI) extends WebUIPage("app")
       <td>{formatResourcesAddresses(executor.resources)}</td>
       <td>{executor.state}</td>
       <td>
-        <a href={s"$workerUrlRef/logPage?appId=${executor.application.id}&executorId=${executor.
+        <a href={s"$workerUrlRef/logPage/?appId=${executor.application.id}&executorId=${executor.
           id}&logType=stdout"}>stdout</a>
-        <a href={s"$workerUrlRef/logPage?appId=${executor.application.id}&executorId=${executor.
+        <a href={s"$workerUrlRef/logPage/?appId=${executor.application.id}&executorId=${executor.
           id}&logType=stderr"}>stderr</a>
       </td>
     </tr>
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
index 47bc34e1b7930..6143321427d4c 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
@@ -76,19 +76,17 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {
 
   private def formatMasterResourcesInUse(aliveWorkers: Array[WorkerInfo]): String = {
     val totalInfo = aliveWorkers.map(_.resourcesInfo)
-      .map(resources => toMutable(resources))
       .flatMap(_.toIterator)
       .groupBy(_._1) // group by resource name
       .map { case (rName, rInfoArr) =>
-        rName -> rInfoArr.map(_._2).reduce(_ + _)
-      }.map { case (k, v) => (k, v.toResourceInformation) }
+      rName -> rInfoArr.map(_._2.addresses.size).sum
+    }
     val usedInfo = aliveWorkers.map(_.resourcesInfoUsed)
-      .map (resources => toMutable(resources))
       .flatMap(_.toIterator)
       .groupBy(_._1) // group by resource name
       .map { case (rName, rInfoArr) =>
-      rName -> rInfoArr.map(_._2).reduce(_ + _)
-    }.map { case (k, v) => (k, v.toResourceInformation) }
+      rName -> rInfoArr.map(_._2.addresses.size).sum
+    }
     formatResourcesUsed(totalInfo, usedInfo)
   }
 
@@ -286,7 +284,7 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {
     }
     <tr>
       <td>
-        <a href={"app?appId=" + app.id}>{app.id}</a>
+        <a href={"app/?appId=" + app.id}>{app.id}</a>
         {killLink}
       </td>
       <td>
@@ -311,7 +309,9 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {
       <td>{UIUtils.formatDate(app.submitDate)}</td>
       <td>{app.desc.user}</td>
       <td>{app.state.toString}</td>
-      <td>{UIUtils.formatDuration(app.duration)}</td>
+      <td sorttable_customkey={app.duration.toString}>
+        {UIUtils.formatDuration(app.duration)}
+      </td>
     </tr>
   }
 
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala
index 86554ec4ec1c9..af94bd6d9e0f2 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala
@@ -17,9 +17,13 @@
 
 package org.apache.spark.deploy.master.ui
 
-import org.apache.spark.deploy.DeployMessages.{MasterStateResponse, RequestMasterState}
+import java.net.{InetAddress, NetworkInterface, SocketException}
+import javax.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse}
+
+import org.apache.spark.deploy.DeployMessages.{DecommissionWorkersOnHosts, MasterStateResponse, RequestMasterState}
 import org.apache.spark.deploy.master.Master
 import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config.UI.MASTER_UI_DECOMMISSION_ALLOW_MODE
 import org.apache.spark.internal.config.UI.UI_KILL_ENABLED
 import org.apache.spark.ui.{SparkUI, WebUI}
 import org.apache.spark.ui.JettyUtils._
@@ -36,6 +40,7 @@ class MasterWebUI(
 
   val masterEndpointRef = master.self
   val killEnabled = master.conf.get(UI_KILL_ENABLED)
+  val decommissionAllowMode = master.conf.get(MASTER_UI_DECOMMISSION_ALLOW_MODE)
 
   initialize()
 
@@ -49,6 +54,27 @@ class MasterWebUI(
       "/app/kill", "/", masterPage.handleAppKillRequest, httpMethods = Set("POST")))
     attachHandler(createRedirectHandler(
       "/driver/kill", "/", masterPage.handleDriverKillRequest, httpMethods = Set("POST")))
+    attachHandler(createServletHandler("/workers/kill", new HttpServlet {
+      override def doPost(req: HttpServletRequest, resp: HttpServletResponse): Unit = {
+        val hostnames: Seq[String] = Option(req.getParameterValues("host"))
+          .getOrElse(Array[String]()).toSeq
+        if (!isDecommissioningRequestAllowed(req)) {
+          resp.sendError(HttpServletResponse.SC_METHOD_NOT_ALLOWED)
+        } else {
+          val removedWorkers = masterEndpointRef.askSync[Integer](
+            DecommissionWorkersOnHosts(hostnames))
+          logInfo(s"Decommissioning of hosts $hostnames decommissioned $removedWorkers workers")
+          if (removedWorkers > 0) {
+            resp.setStatus(HttpServletResponse.SC_OK)
+          } else if (removedWorkers == 0) {
+            resp.sendError(HttpServletResponse.SC_NOT_FOUND)
+          } else {
+            // We shouldn't even see this case.
+            resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR)
+          }
+        }
+      }
+    }, ""))
   }
 
   def addProxy(): Unit = {
@@ -64,6 +90,25 @@ class MasterWebUI(
     maybeWorkerUiAddress.orElse(maybeAppUiAddress)
   }
 
+  private def isLocal(address: InetAddress): Boolean = {
+    if (address.isAnyLocalAddress || address.isLoopbackAddress) {
+      return true
+    }
+    try {
+      NetworkInterface.getByInetAddress(address) != null
+    } catch {
+      case _: SocketException => false
+    }
+  }
+
+  private def isDecommissioningRequestAllowed(req: HttpServletRequest): Boolean = {
+    decommissionAllowMode match {
+      case "ALLOW" => true
+      case "LOCAL" => isLocal(InetAddress.getByName(req.getRemoteAddr))
+      case _ => false
+    }
+  }
+
 }
 
 private[master] object MasterWebUI {
diff --git a/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionClient.scala b/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionClient.scala
index 1648ba516d9b6..cc1d60a097b2e 100644
--- a/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionClient.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionClient.scala
@@ -411,7 +411,7 @@ private[spark] object RestSubmissionClient {
 
   // SPARK_HOME and SPARK_CONF_DIR are filtered out because they are usually wrong
   // on the remote machine (SPARK-12345) (SPARK-25934)
-  private val BLACKLISTED_SPARK_ENV_VARS = Set("SPARK_ENV_LOADED", "SPARK_HOME", "SPARK_CONF_DIR")
+  private val EXCLUDED_SPARK_ENV_VARS = Set("SPARK_ENV_LOADED", "SPARK_HOME", "SPARK_CONF_DIR")
   private val REPORT_DRIVER_STATUS_INTERVAL = 1000
   private val REPORT_DRIVER_STATUS_MAX_TRIES = 10
   val PROTOCOL_VERSION = "v1"
@@ -421,8 +421,8 @@ private[spark] object RestSubmissionClient {
    */
   private[rest] def filterSystemEnvironment(env: Map[String, String]): Map[String, String] = {
     env.filterKeys { k =>
-      (k.startsWith("SPARK_") && !BLACKLISTED_SPARK_ENV_VARS.contains(k)) || k.startsWith("MESOS_")
-    }
+      (k.startsWith("SPARK_") && !EXCLUDED_SPARK_ENV_VARS.contains(k)) || k.startsWith("MESOS_")
+    }.toMap
   }
 
   private[spark] def supportsRestClient(master: String): Boolean = {
diff --git a/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala b/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala
index 3168c763df4df..6ce195b6c7a34 100644
--- a/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala
@@ -178,7 +178,7 @@ private[spark] class HadoopDelegationTokenManager(
 
   private def scheduleRenewal(delay: Long): Unit = {
     val _delay = math.max(0, delay)
-    logInfo(s"Scheduling renewal in ${UIUtils.formatDuration(delay)}.")
+    logInfo(s"Scheduling renewal in ${UIUtils.formatDuration(_delay)}.")
 
     val renewalTask = new Runnable() {
       override def run(): Unit = {
@@ -230,6 +230,8 @@ private[spark] class HadoopDelegationTokenManager(
         val now = System.currentTimeMillis
         val ratio = sparkConf.get(CREDENTIALS_RENEWAL_INTERVAL_RATIO)
         val delay = (ratio * (nextRenewal - now)).toLong
+        logInfo(s"Calculated delay on renewal is $delay, based on next renewal $nextRenewal " +
+          s"and the ratio $ratio, and current time $now")
         scheduleRenewal(delay)
         creds
       }
diff --git a/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala b/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala
index 4e91e72361488..5c98762d4181d 100644
--- a/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala
@@ -48,7 +48,12 @@ private[deploy] class HadoopFSDelegationTokenProvider
       creds: Credentials): Option[Long] = {
     try {
       val fileSystems = HadoopFSDelegationTokenProvider.hadoopFSsToAccess(sparkConf, hadoopConf)
-      val fetchCreds = fetchDelegationTokens(getTokenRenewer(hadoopConf), fileSystems, creds)
+      // The hosts on which the file systems to be excluded from token renewal
+      val fsToExclude = sparkConf.get(YARN_KERBEROS_FILESYSTEM_RENEWAL_EXCLUDE)
+        .map(new Path(_).getFileSystem(hadoopConf).getUri.getHost)
+        .toSet
+      val fetchCreds = fetchDelegationTokens(getTokenRenewer(hadoopConf), fileSystems, creds,
+        fsToExclude)
 
       // Get the token renewal interval if it is not set. It will only be called once.
       if (tokenRenewalInterval == null) {
@@ -63,7 +68,8 @@ private[deploy] class HadoopFSDelegationTokenProvider
             val identifier = token
               .decodeIdentifier()
               .asInstanceOf[AbstractDelegationTokenIdentifier]
-            identifier.getIssueDate + interval
+            val tokenKind = token.getKind.toString
+            getIssueDate(tokenKind, identifier) + interval
           }
         if (nextRenewalDates.isEmpty) None else Some(nextRenewalDates.min)
       }
@@ -98,11 +104,18 @@ private[deploy] class HadoopFSDelegationTokenProvider
   private def fetchDelegationTokens(
       renewer: String,
       filesystems: Set[FileSystem],
-      creds: Credentials): Credentials = {
+      creds: Credentials,
+      fsToExclude: Set[String]): Credentials = {
 
     filesystems.foreach { fs =>
-      logInfo(s"getting token for: $fs with renewer $renewer")
-      fs.addDelegationTokens(renewer, creds)
+      if (fsToExclude.contains(fs.getUri.getHost)) {
+        // YARN RM skips renewing token with empty renewer
+        logInfo(s"getting token for: $fs with empty renewer to skip renewal")
+        fs.addDelegationTokens("", creds)
+      } else {
+        logInfo(s"getting token for: $fs with renewer $renewer")
+        fs.addDelegationTokens(renewer, creds)
+      }
     }
 
     creds
@@ -118,7 +131,7 @@ private[deploy] class HadoopFSDelegationTokenProvider
     val renewer = UserGroupInformation.getCurrentUser().getUserName()
 
     val creds = new Credentials()
-    fetchDelegationTokens(renewer, filesystems, creds)
+    fetchDelegationTokens(renewer, filesystems, creds, Set.empty)
 
     val renewIntervals = creds.getAllTokens.asScala.filter {
       _.decodeIdentifier().isInstanceOf[AbstractDelegationTokenIdentifier]
@@ -126,20 +139,42 @@ private[deploy] class HadoopFSDelegationTokenProvider
       Try {
         val newExpiration = token.renew(hadoopConf)
         val identifier = token.decodeIdentifier().asInstanceOf[AbstractDelegationTokenIdentifier]
-        val interval = newExpiration - identifier.getIssueDate
-        logInfo(s"Renewal interval is $interval for token ${token.getKind.toString}")
+        val tokenKind = token.getKind.toString
+        val interval = newExpiration - getIssueDate(tokenKind, identifier)
+        logInfo(s"Renewal interval is $interval for token $tokenKind")
         interval
       }.toOption
     }
     if (renewIntervals.isEmpty) None else Some(renewIntervals.min)
   }
+
+  private def getIssueDate(kind: String, identifier: AbstractDelegationTokenIdentifier): Long = {
+    val now = System.currentTimeMillis()
+    val issueDate = identifier.getIssueDate
+    if (issueDate > now) {
+      logWarning(s"Token $kind has set up issue date later than current time. (provided: " +
+        s"$issueDate / current timestamp: $now) Please make sure clocks are in sync between " +
+        "machines. If the issue is not a clock mismatch, consult token implementor to check " +
+        "whether issue date is valid.")
+      issueDate
+    } else if (issueDate > 0L) {
+      issueDate
+    } else {
+      logWarning(s"Token $kind has not set up issue date properly. (provided: $issueDate) " +
+        s"Using current timestamp ($now) as issue date instead. Consult token implementor to fix " +
+        "the behavior.")
+      now
+    }
+  }
 }
 
 private[deploy] object HadoopFSDelegationTokenProvider {
   def hadoopFSsToAccess(
       sparkConf: SparkConf,
       hadoopConf: Configuration): Set[FileSystem] = {
+    // scalastyle:off FileSystemGet
     val defaultFS = FileSystem.get(hadoopConf)
+    // scalastyle:on FileSystemGet
 
     val filesystemsToAccess = sparkConf.get(KERBEROS_FILESYSTEMS_TO_ACCESS)
       .map(new Path(_).getFileSystem(hadoopConf))
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala
index f7423f1fc3f1c..8240bd6d2f438 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala
@@ -61,7 +61,7 @@ object CommandUtils extends Logging {
     // SPARK-698: do not call the run.cmd script, as process.destroy()
     // fails to kill a process tree on Windows
     val cmd = new WorkerCommandBuilder(sparkHome, memory, command).buildCommand()
-    cmd.asScala ++ Seq(command.mainClass) ++ command.arguments
+    (cmd.asScala ++ Seq(command.mainClass) ++ command.arguments).toSeq
   }
 
   /**
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala b/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala
index 53ec7b3a88f35..910c27fc71044 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala
@@ -33,9 +33,11 @@ import org.apache.spark.deploy.master.DriverState
 import org.apache.spark.deploy.master.DriverState.DriverState
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.{DRIVER_RESOURCES_FILE, SPARK_DRIVER_PREFIX}
+import org.apache.spark.internal.config.UI.UI_REVERSE_PROXY
 import org.apache.spark.internal.config.Worker.WORKER_DRIVER_TERMINATE_TIMEOUT
 import org.apache.spark.resource.ResourceInformation
 import org.apache.spark.rpc.RpcEndpointRef
+import org.apache.spark.ui.UIUtils
 import org.apache.spark.util.{Clock, ShutdownHookManager, SystemClock, Utils}
 
 /**
@@ -50,6 +52,7 @@ private[deploy] class DriverRunner(
     val driverDesc: DriverDescription,
     val worker: RpcEndpointRef,
     val workerUrl: String,
+    val workerWebUiUrl: String,
     val securityManager: SecurityManager,
     val resources: Map[String, ResourceInformation] = Map.empty)
   extends Logging {
@@ -160,7 +163,6 @@ private[deploy] class DriverRunner(
         driverDesc.jarUrl,
         driverDir,
         conf,
-        securityManager,
         SparkHadoopUtil.get.newConfiguration(conf),
         System.currentTimeMillis(),
         useCache = false)
@@ -190,6 +192,14 @@ private[deploy] class DriverRunner(
     val builder = CommandUtils.buildProcessBuilder(driverDesc.command.copy(javaOpts = javaOpts),
       securityManager, driverDesc.mem, sparkHome.getAbsolutePath, substituteVariables)
 
+    // add WebUI driver log url to environment
+    val reverseProxy = conf.get(UI_REVERSE_PROXY)
+    val workerUrlRef = UIUtils.makeHref(reverseProxy, driverId, workerWebUiUrl)
+    builder.environment.put("SPARK_DRIVER_LOG_URL_STDOUT",
+      s"$workerUrlRef/logPage/?driverId=$driverId&logType=stdout")
+    builder.environment.put("SPARK_DRIVER_LOG_URL_STDERR",
+      s"$workerUrlRef/logPage/?driverId=$driverId&logType=stderr")
+
     runDriver(builder, driverDir, driverDesc.supervise)
   }
 
@@ -201,7 +211,7 @@ private[deploy] class DriverRunner(
       CommandUtils.redirectStream(process.getInputStream, stdout)
 
       val stderr = new File(baseDir, "stderr")
-      val redactedCommand = Utils.redactCommandLineArgs(conf, builder.command.asScala)
+      val redactedCommand = Utils.redactCommandLineArgs(conf, builder.command.asScala.toSeq)
         .mkString("\"", "\" \"", "\"")
       val header = "Launch Command: %s\n%s\n\n".format(redactedCommand, "=" * 40)
       Files.append(header, stderr, StandardCharsets.UTF_8)
@@ -262,6 +272,6 @@ private[deploy] trait ProcessBuilderLike {
 private[deploy] object ProcessBuilderLike {
   def apply(processBuilder: ProcessBuilder): ProcessBuilderLike = new ProcessBuilderLike {
     override def start(): Process = processBuilder.start()
-    override def command: Seq[String] = processBuilder.command().asScala
+    override def command: Seq[String] = processBuilder.command().asScala.toSeq
   }
 }
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/DriverWrapper.scala b/core/src/main/scala/org/apache/spark/deploy/worker/DriverWrapper.scala
index 45ffdde58d6c3..9176897163846 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/DriverWrapper.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/DriverWrapper.scala
@@ -19,10 +19,8 @@ package org.apache.spark.deploy.worker
 
 import java.io.File
 
-import org.apache.commons.lang3.StringUtils
-
 import org.apache.spark.{SecurityManager, SparkConf}
-import org.apache.spark.deploy.{DependencyUtils, SparkHadoopUtil}
+import org.apache.spark.deploy.SparkHadoopUtil
 import org.apache.spark.internal.{config, Logging}
 import org.apache.spark.rpc.RpcEnv
 import org.apache.spark.util._
@@ -76,30 +74,23 @@ object DriverWrapper extends Logging {
 
   private def setupDependencies(loader: MutableURLClassLoader, userJar: String): Unit = {
     val sparkConf = new SparkConf()
-    val secMgr = new SecurityManager(sparkConf)
     val hadoopConf = SparkHadoopUtil.newConfiguration(sparkConf)
 
-    val Seq(packagesExclusions, packages, repositories, ivyRepoPath, ivySettingsPath) =
-      Seq(
-        "spark.jars.excludes",
-        "spark.jars.packages",
-        "spark.jars.repositories",
-        "spark.jars.ivy",
-        "spark.jars.ivySettings"
-      ).map(sys.props.get(_).orNull)
+    val ivyProperties = DependencyUtils.getIvyProperties()
 
-    val resolvedMavenCoordinates = DependencyUtils.resolveMavenDependencies(packagesExclusions,
-      packages, repositories, ivyRepoPath, Option(ivySettingsPath))
+    val resolvedMavenCoordinates = DependencyUtils.resolveMavenDependencies(true,
+      ivyProperties.packagesExclusions, ivyProperties.packages, ivyProperties.repositories,
+      ivyProperties.ivyRepoPath, Option(ivyProperties.ivySettingsPath))
     val jars = {
       val jarsProp = sys.props.get(config.JARS.key).orNull
-      if (!StringUtils.isBlank(resolvedMavenCoordinates)) {
-        DependencyUtils.mergeFileLists(jarsProp, resolvedMavenCoordinates)
+      if (resolvedMavenCoordinates.nonEmpty) {
+        DependencyUtils.mergeFileLists(jarsProp,
+          DependencyUtils.mergeFileLists(resolvedMavenCoordinates: _*))
       } else {
         jarsProp
       }
     }
-    val localJars = DependencyUtils.resolveAndDownloadJars(jars, userJar, sparkConf, hadoopConf,
-      secMgr)
+    val localJars = DependencyUtils.resolveAndDownloadJars(jars, userJar, sparkConf, hadoopConf)
     DependencyUtils.addJarsToClassPath(localJars, loader)
   }
 }
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala b/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
index 2a5528bbe89cb..2e26ccf671d88 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
@@ -158,7 +158,7 @@ private[deploy] class ExecutorRunner(
       val builder = CommandUtils.buildProcessBuilder(subsCommand, new SecurityManager(conf),
         memory, sparkHome.getAbsolutePath, substituteVariables)
       val command = builder.command()
-      val redactedCommand = Utils.redactCommandLineArgs(conf, command.asScala)
+      val redactedCommand = Utils.redactCommandLineArgs(conf, command.asScala.toSeq)
         .mkString("\"", "\" \"", "\"")
       logInfo(s"Launch command: $redactedCommand")
 
@@ -171,7 +171,8 @@ private[deploy] class ExecutorRunner(
       // Add webUI log urls
       val baseUrl =
         if (conf.get(UI_REVERSE_PROXY)) {
-          s"/proxy/$workerId/logPage/?appId=$appId&executorId=$execId&logType="
+          conf.get(UI_REVERSE_PROXY_URL.key, "").stripSuffix("/") +
+            s"/proxy/$workerId/logPage/?appId=$appId&executorId=$execId&logType="
         } else {
           s"$webUiScheme$publicAddress:$webUiPort/logPage/?appId=$appId&executorId=$execId&logType="
         }
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
index aa8c46fc68315..05e8e5a6b6766 100755
--- a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
@@ -26,7 +26,7 @@ import java.util.function.Supplier
 
 import scala.collection.mutable.{HashMap, HashSet, LinkedHashMap}
 import scala.concurrent.ExecutionContext
-import scala.util.Random
+import scala.util.{Failure, Random, Success}
 import scala.util.control.NonFatal
 
 import org.apache.spark.{SecurityManager, SparkConf}
@@ -66,13 +66,17 @@ private[deploy] class Worker(
   Utils.checkHost(host)
   assert (port > 0)
 
-  // If worker decommissioning is enabled register a handler on PWR to shutdown.
-  if (conf.get(WORKER_DECOMMISSION_ENABLED)) {
-    logInfo("Registering SIGPWR handler to trigger decommissioning.")
-    SignalUtils.register("PWR", "Failed to register SIGPWR handler - " +
-      "disabling worker decommission feature.")(decommissionSelf)
+  // If worker decommissioning is enabled register a handler on the configured signal to shutdown.
+  if (conf.get(config.DECOMMISSION_ENABLED)) {
+    val signal = conf.get(config.Worker.WORKER_DECOMMISSION_SIGNAL)
+    logInfo(s"Registering SIG$signal handler to trigger decommissioning.")
+    SignalUtils.register(signal, s"Failed to register SIG$signal handler - " +
+      "disabling worker decommission feature.") {
+       self.send(WorkerDecommissionSigReceived)
+       true
+    }
   } else {
-    logInfo("Worker decommissioning not enabled, SIGPWR will result in exiting.")
+    logInfo("Worker decommissioning not enabled.")
   }
 
   // A scheduled executor used to send messages at the specified time.
@@ -137,7 +141,8 @@ private[deploy] class Worker(
   private var registered = false
   private var connected = false
   private var decommissioned = false
-  private val workerId = generateWorkerId()
+  // expose for test
+  private[spark] val workerId = generateWorkerId()
   private val sparkHome =
     if (sys.props.contains(IS_TESTING.key)) {
       assert(sys.props.contains("spark.test.home"), "spark.test.home is not set!")
@@ -154,6 +159,18 @@ private[deploy] class Worker(
   val appDirectories = new HashMap[String, Seq[String]]
   val finishedApps = new HashSet[String]
 
+  // Record the consecutive failure attempts of executor state change syncing with Master,
+  // so we don't try it endless. We will exit the Worker process at the end if the failure
+  // attempts reach the max attempts(5). In that case, it's highly possible the Worker
+  // suffers a severe network issue, and the Worker would exit finally either reaches max
+  // re-register attempts or max state syncing attempts.
+  // Map from executor fullId to its consecutive failure attempts number. It's supposed
+  // to be very small since it's only used for the temporary network drop, which doesn't
+  // happen frequently and recover soon.
+  private val executorStateSyncFailureAttempts = new HashMap[String, Int]()
+  lazy private val executorStateSyncFailureHandler = ExecutionContext.fromExecutor(
+    ThreadUtils.newDaemonSingleThreadExecutor("executor-state-sync-failure-handler"))
+
   val retainedExecutors = conf.get(WORKER_UI_RETAINED_EXECUTORS)
   val retainedDrivers = conf.get(WORKER_UI_RETAINED_DRIVERS)
 
@@ -173,7 +190,7 @@ private[deploy] class Worker(
   private var connectionAttemptCount = 0
 
   private val metricsSystem =
-    MetricsSystem.createMetricsSystem(MetricsSystemInstances.WORKER, conf, securityMgr)
+    MetricsSystem.createMetricsSystem(MetricsSystemInstances.WORKER, conf)
   private val workerSource = new WorkerSource(this)
 
   val reverseProxy = conf.get(UI_REVERSE_PROXY)
@@ -272,7 +289,14 @@ private[deploy] class Worker(
     master = Some(masterRef)
     connected = true
     if (reverseProxy) {
-      logInfo(s"WorkerWebUI is available at $activeMasterWebUiUrl/proxy/$workerId")
+      logInfo("WorkerWebUI is available at %s/proxy/%s".format(
+        activeMasterWebUiUrl.stripSuffix("/"), workerId))
+      // if reverseProxyUrl is not set, then we continue to generate relative URLs
+      // starting with "/" throughout the UI and do not use activeMasterWebUiUrl
+      val proxyUrl = conf.get(UI_REVERSE_PROXY_URL.key, "").stripSuffix("/")
+      // In the method `UIUtils.makeHref`, the URL segment "/proxy/$worker_id" will be appended
+      // after `proxyUrl`, so no need to set the worker ID in the `spark.ui.proxyBase` here.
+      System.setProperty("spark.ui.proxyBase", proxyUrl)
     }
     // Cancel any outstanding re-registration attempts because we found a new master
     cancelLastRegistrationRetry()
@@ -608,7 +632,7 @@ private[deploy] class Worker(
               executors(appId + "/" + execId).kill()
               executors -= appId + "/" + execId
             }
-            sendToMaster(ExecutorStateChanged(appId, execId, ExecutorState.FAILED,
+            syncExecutorStateWithMaster(ExecutorStateChanged(appId, execId, ExecutorState.FAILED,
               Some(e.toString), None))
         }
       }
@@ -640,6 +664,7 @@ private[deploy] class Worker(
         driverDesc.copy(command = Worker.maybeUpdateSSLSettings(driverDesc.command, conf)),
         self,
         workerUri,
+        workerWebUiUrl,
         securityMgr,
         resources_)
       drivers(driverId) = driver
@@ -668,8 +693,14 @@ private[deploy] class Worker(
       finishedApps += id
       maybeCleanupApplication(id)
 
-    case DecommissionSelf =>
+    case DecommissionWorker =>
+      decommissionSelf()
+
+    case WorkerDecommissionSigReceived =>
       decommissionSelf()
+      // Tell the Master that we are starting decommissioning
+      // so it stops trying to launch executor/driver on us
+      sendToMaster(WorkerDecommissioning(workerId, self))
   }
 
   override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
@@ -731,6 +762,43 @@ private[deploy] class Worker(
     }
   }
 
+  /**
+   * Send `ExecutorStateChanged` to the current master. Unlike `sendToMaster`, we use `askSync`
+   * to send the message in order to ensure Master can receive the message.
+   */
+  private def syncExecutorStateWithMaster(newState: ExecutorStateChanged): Unit = {
+    master match {
+      case Some(masterRef) =>
+        val fullId = s"${newState.appId}/${newState.execId}"
+        // SPARK-34245: We used async `send` to send the state previously. In that case, the
+        // finished executor can be leaked if Worker fails to send `ExecutorStateChanged`
+        // message to Master due to some unexpected errors, e.g., temporary network error.
+        // In the worst case, the application can get hang if the leaked executor is the only
+        // or last executor for the application. Therefore, we switch to `ask` to ensure
+        // the state is handled by Master.
+        masterRef.ask[Boolean](newState).onComplete {
+          case Success(_) =>
+            executorStateSyncFailureAttempts.remove(fullId)
+
+          case Failure(t) =>
+            val failures = executorStateSyncFailureAttempts.getOrElse(fullId, 0) + 1
+            if (failures < 5) {
+              logError(s"Failed to send $newState to Master $masterRef, " +
+                s"will retry ($failures/5).", t)
+              executorStateSyncFailureAttempts(fullId) = failures
+              self.send(newState)
+            } else {
+              logError(s"Failed to send $newState to Master $masterRef for 5 times. Giving up.")
+              System.exit(1)
+            }
+        }(executorStateSyncFailureHandler)
+
+      case None =>
+        logWarning(
+          s"Dropping $newState because the connection to master has not yet been established")
+    }
+  }
+
   private def generateWorkerId(): String = {
     "worker-%s-%s-%d".format(createDateFormat.format(new Date), host, port)
   }
@@ -768,16 +836,15 @@ private[deploy] class Worker(
     }
   }
 
-  private[deploy] def decommissionSelf(): Boolean = {
-    if (conf.get(WORKER_DECOMMISSION_ENABLED)) {
-      logDebug("Decommissioning self")
+  private[deploy] def decommissionSelf(): Unit = {
+    if (conf.get(config.DECOMMISSION_ENABLED) && !decommissioned) {
       decommissioned = true
-      sendToMaster(WorkerDecommission(workerId, self))
+      logInfo(s"Decommission worker $workerId.")
+    } else if (decommissioned) {
+      logWarning(s"Worker $workerId already started decommissioning.")
     } else {
-      logWarning("Asked to decommission self, but decommissioning not enabled")
+      logWarning(s"Receive decommission request, but decommission feature is disabled.")
     }
-    // Return true since can be called as a signal handler
-    true
   }
 
   private[worker] def handleDriverStateChanged(driverStateChanged: DriverStateChanged): Unit = {
@@ -807,7 +874,7 @@ private[deploy] class Worker(
 
   private[worker] def handleExecutorStateChanged(executorStateChanged: ExecutorStateChanged):
     Unit = {
-    sendToMaster(executorStateChanged)
+    syncExecutorStateWithMaster(executorStateChanged)
     val state = executorStateChanged.state
     if (ExecutorState.isFinished(state)) {
       val appId = executorStateChanged.appId
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/ui/WorkerPage.scala b/core/src/main/scala/org/apache/spark/deploy/worker/ui/WorkerPage.scala
index ce0863dd293bd..3171d3f16e8a0 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/ui/WorkerPage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/ui/WorkerPage.scala
@@ -189,9 +189,9 @@ private[ui] class WorkerPage(parent: WorkerWebUI) extends WebUIPage("") {
         </ul>
       </td>
       <td>
-        <a href={s"$workerUrlRef/logPage?appId=${executor
+        <a href={s"$workerUrlRef/logPage/?appId=${executor
           .appId}&executorId=${executor.execId}&logType=stdout"}>stdout</a>
-        <a href={s"$workerUrlRef/logPage?appId=${executor
+        <a href={s"$workerUrlRef/logPage/?appId=${executor
           .appId}&executorId=${executor.execId}&logType=stderr"}>stderr</a>
       </td>
     </tr>
diff --git a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
index 6625457749f6a..43c122a94fff7 100644
--- a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
+++ b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
@@ -17,7 +17,6 @@
 
 package org.apache.spark.executor
 
-import java.io.File
 import java.net.URL
 import java.nio.ByteBuffer
 import java.util.Locale
@@ -40,7 +39,7 @@ import org.apache.spark.resource.ResourceProfile
 import org.apache.spark.resource.ResourceProfile._
 import org.apache.spark.resource.ResourceUtils._
 import org.apache.spark.rpc._
-import org.apache.spark.scheduler.{ExecutorLossReason, TaskDescription}
+import org.apache.spark.scheduler.{ExecutorLossMessage, ExecutorLossReason, TaskDescription}
 import org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages._
 import org.apache.spark.serializer.SerializerInstance
 import org.apache.spark.util.{ChildFirstURLClassLoader, MutableURLClassLoader, SignalUtils, ThreadUtils, Utils}
@@ -64,7 +63,6 @@ private[spark] class CoarseGrainedExecutorBackend(
 
   private[this] val stopping = new AtomicBoolean(false)
   var executor: Executor = null
-  @volatile private var decommissioned = false
   @volatile var driver: Option[RpcEndpointRef] = None
 
   // If this CoarseGrainedExecutorBackend is changed to support multiple threads, then this may need
@@ -80,10 +78,15 @@ private[spark] class CoarseGrainedExecutorBackend(
    */
   private[executor] val taskResources = new mutable.HashMap[Long, Map[String, ResourceInformation]]
 
+  private var decommissioned = false
+
   override def onStart(): Unit = {
-    logInfo("Registering PWR handler.")
-    SignalUtils.register("PWR", "Failed to register SIGPWR handler - " +
-      "disabling decommission feature.")(decommissionSelf)
+    if (env.conf.get(DECOMMISSION_ENABLED)) {
+      val signal = env.conf.get(EXECUTOR_DECOMMISSION_SIGNAL)
+      logInfo(s"Registering SIG$signal handler to trigger decommissioning.")
+      SignalUtils.register(signal, s"Failed to register SIG$signal handler - disabling" +
+        s" executor decommission feature.") (self.askSync[Boolean](ExecutorDecommissionSigReceived))
+    }
 
     logInfo("Connecting to driver: " + driverUrl)
     try {
@@ -140,13 +143,13 @@ private[spark] class CoarseGrainedExecutorBackend(
   def extractLogUrls: Map[String, String] = {
     val prefix = "SPARK_LOG_URL_"
     sys.env.filterKeys(_.startsWith(prefix))
-      .map(e => (e._1.substring(prefix.length).toLowerCase(Locale.ROOT), e._2))
+      .map(e => (e._1.substring(prefix.length).toLowerCase(Locale.ROOT), e._2)).toMap
   }
 
   def extractAttributes: Map[String, String] = {
     val prefix = "SPARK_EXECUTOR_ATTRIBUTE_"
     sys.env.filterKeys(_.startsWith(prefix))
-      .map(e => (e._1.substring(prefix.length).toUpperCase(Locale.ROOT), e._2))
+      .map(e => (e._1.substring(prefix.length).toUpperCase(Locale.ROOT), e._2)).toMap
   }
 
   override def receive: PartialFunction[Any, Unit] = {
@@ -165,16 +168,6 @@ private[spark] class CoarseGrainedExecutorBackend(
       if (executor == null) {
         exitExecutor(1, "Received LaunchTask command but executor was null")
       } else {
-        if (decommissioned) {
-          logError("Asked to launch a task while decommissioned.")
-          driver match {
-            case Some(endpoint) =>
-              logInfo("Sending DecommissionExecutor to driver.")
-              endpoint.send(DecommissionExecutor(executorId))
-            case _ =>
-              logError("No registered driver to send Decommission to.")
-          }
-        }
         val taskDesc = TaskDescription.decode(data.value)
         logInfo("Got assigned task " + taskDesc.taskId)
         taskResources(taskDesc.taskId) = taskDesc.resources
@@ -210,6 +203,30 @@ private[spark] class CoarseGrainedExecutorBackend(
     case UpdateDelegationTokens(tokenBytes) =>
       logInfo(s"Received tokens of ${tokenBytes.length} bytes")
       SparkHadoopUtil.get.addDelegationTokens(tokenBytes, env.conf)
+
+    case DecommissionExecutor =>
+      decommissionSelf()
+  }
+
+  override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
+    case ExecutorDecommissionSigReceived =>
+      var driverNotified = false
+      try {
+        driver.foreach { driverRef =>
+          // Tell driver that we are starting decommissioning so it stops trying to schedule us
+          driverNotified = driverRef.askSync[Boolean](ExecutorDecommissioning(executorId))
+          if (driverNotified) decommissionSelf()
+        }
+      } catch {
+        case e: Exception =>
+          if (driverNotified) {
+            logError("Fail to decommission self (but driver has been notified).", e)
+          } else {
+            logError("Fail to tell driver that we are starting decommissioning", e)
+          }
+          decommissioned = false
+      }
+      context.reply(decommissioned)
   }
 
   override def onDisconnected(remoteAddress: RpcAddress): Unit = {
@@ -258,26 +275,82 @@ private[spark] class CoarseGrainedExecutorBackend(
     System.exit(code)
   }
 
-  private def decommissionSelf(): Boolean = {
-    logInfo("Decommissioning self w/sync")
+  private def decommissionSelf(): Unit = {
+    if (!env.conf.get(DECOMMISSION_ENABLED)) {
+      logWarning(s"Receive decommission request, but decommission feature is disabled.")
+      return
+    } else if (decommissioned) {
+      logWarning(s"Executor $executorId already started decommissioning.")
+      return
+    }
+    val msg = s"Decommission executor $executorId."
+    logInfo(msg)
     try {
       decommissioned = true
-      // Tell master we are are decommissioned so it stops trying to schedule us
-      if (driver.nonEmpty) {
-        driver.get.askSync[Boolean](DecommissionExecutor(executorId))
-      } else {
-        logError("No driver to message decommissioning.")
+      if (env.conf.get(STORAGE_DECOMMISSION_ENABLED)) {
+        env.blockManager.decommissionBlockManager()
       }
       if (executor != null) {
         executor.decommission()
       }
-      logInfo("Done decommissioning self.")
-      // Return true since we are handling a signal
-      true
+      // Shutdown the executor once all tasks are gone & any configured migrations completed.
+      // Detecting migrations completion doesn't need to be perfect and we want to minimize the
+      // overhead for executors that are not in decommissioning state as overall that will be
+      // more of the executors. For example, this will not catch a block which is already in
+      // the process of being put from a remote executor before migration starts. This trade-off
+      // is viewed as acceptable to minimize introduction of any new locking structures in critical
+      // code paths.
+
+      val shutdownThread = new Thread("wait-for-blocks-to-migrate") {
+        override def run(): Unit = {
+          var lastTaskRunningTime = System.nanoTime()
+          val sleep_time = 1000 // 1s
+          // This config is internal and only used by unit tests to force an executor
+          // to hang around for longer when decommissioned.
+          val initialSleepMillis = env.conf.getInt(
+            "spark.test.executor.decommission.initial.sleep.millis", sleep_time)
+          if (initialSleepMillis > 0) {
+            Thread.sleep(initialSleepMillis)
+          }
+          while (true) {
+            logInfo("Checking to see if we can shutdown.")
+            if (executor == null || executor.numRunningTasks == 0) {
+              if (env.conf.get(STORAGE_DECOMMISSION_ENABLED)) {
+                logInfo("No running tasks, checking migrations")
+                val (migrationTime, allBlocksMigrated) = env.blockManager.lastMigrationInfo()
+                // We can only trust allBlocksMigrated boolean value if there were no tasks running
+                // since the start of computing it.
+                if (allBlocksMigrated && (migrationTime > lastTaskRunningTime)) {
+                  logInfo("No running tasks, all blocks migrated, stopping.")
+                  exitExecutor(0, ExecutorLossMessage.decommissionFinished, notifyDriver = true)
+                } else {
+                  logInfo("All blocks not yet migrated.")
+                }
+              } else {
+                logInfo("No running tasks, no block migration configured, stopping.")
+                exitExecutor(0, ExecutorLossMessage.decommissionFinished, notifyDriver = true)
+              }
+            } else {
+              logInfo("Blocked from shutdown by running ${executor.numRunningtasks} tasks")
+              // If there is a running task it could store blocks, so make sure we wait for a
+              // migration loop to complete after the last task is done.
+              // Note: this is only advanced if there is a running task, if there
+              // is no running task but the blocks are not done migrating this does not
+              // move forward.
+              lastTaskRunningTime = System.nanoTime()
+            }
+            Thread.sleep(sleep_time)
+          }
+        }
+      }
+      shutdownThread.setDaemon(true)
+      shutdownThread.start()
+
+      logInfo("Will exit when finished decommissioning")
     } catch {
       case e: Exception =>
-        logError(s"Error ${e} during attempt to decommission self")
-        false
+        decommissioned = false
+        logError("Unexpected error while decommissioning self", e)
     }
   }
 }
@@ -304,8 +377,8 @@ private[spark] object CoarseGrainedExecutorBackend extends Logging {
     val createFn: (RpcEnv, Arguments, SparkEnv, ResourceProfile) =>
       CoarseGrainedExecutorBackend = { case (rpcEnv, arguments, env, resourceProfile) =>
       new CoarseGrainedExecutorBackend(rpcEnv, arguments.driverUrl, arguments.executorId,
-        arguments.bindAddress, arguments.hostname, arguments.cores, arguments.userClassPath, env,
-        arguments.resourcesFileOpt, resourceProfile)
+        arguments.bindAddress, arguments.hostname, arguments.cores, arguments.userClassPath.toSeq,
+        env, arguments.resourcesFileOpt, resourceProfile)
     }
     run(parseArguments(args, this.getClass.getCanonicalName.stripSuffix("$")), createFn)
     System.exit(0)
diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala
index 2bfa1cea4b26f..acb83f33b2e3c 100644
--- a/core/src/main/scala/org/apache/spark/executor/Executor.scala
+++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala
@@ -22,10 +22,11 @@ import java.lang.Thread.UncaughtExceptionHandler
 import java.lang.management.ManagementFactory
 import java.net.{URI, URL}
 import java.nio.ByteBuffer
-import java.util.Properties
+import java.util.{Locale, Properties}
 import java.util.concurrent._
 import java.util.concurrent.atomic.AtomicBoolean
 import javax.annotation.concurrent.GuardedBy
+import javax.ws.rs.core.UriBuilder
 
 import scala.collection.JavaConverters._
 import scala.collection.immutable
@@ -34,6 +35,7 @@ import scala.concurrent.duration._
 import scala.util.control.NonFatal
 
 import com.google.common.util.concurrent.ThreadFactoryBuilder
+import org.slf4j.MDC
 
 import org.apache.spark._
 import org.apache.spark.deploy.SparkHadoopUtil
@@ -45,7 +47,7 @@ import org.apache.spark.metrics.source.JVMCPUSource
 import org.apache.spark.resource.ResourceInformation
 import org.apache.spark.rpc.RpcTimeout
 import org.apache.spark.scheduler._
-import org.apache.spark.shuffle.FetchFailedException
+import org.apache.spark.shuffle.{FetchFailedException, ShuffleBlockPusher}
 import org.apache.spark.storage.{StorageLevel, TaskResultBlockId}
 import org.apache.spark.util._
 import org.apache.spark.util.io.ChunkedByteBuffer
@@ -53,7 +55,7 @@ import org.apache.spark.util.io.ChunkedByteBuffer
 /**
  * Spark executor, backed by a threadpool to run tasks.
  *
- * This can be used with Mesos, YARN, and the standalone scheduler.
+ * This can be used with Mesos, YARN, kubernetes and the standalone scheduler.
  * An internal RPC interface is used for communication with the driver,
  * except in the case of Mesos fine-grained mode.
  */
@@ -77,6 +79,7 @@ private[spark] class Executor(
   // Each map holds the master's timestamp for the version of that file or JAR we got.
   private val currentFiles: HashMap[String, Long] = new HashMap[String, Long]()
   private val currentJars: HashMap[String, Long] = new HashMap[String, Long]()
+  private val currentArchives: HashMap[String, Long] = new HashMap[String, Long]()
 
   private val EMPTY_BYTE_BUFFER = ByteBuffer.wrap(new Array[Byte](0))
 
@@ -109,7 +112,9 @@ private[spark] class Executor(
       .build()
     Executors.newCachedThreadPool(threadFactory).asInstanceOf[ThreadPoolExecutor]
   }
-  private val executorSource = new ExecutorSource(threadPool, executorId)
+  private val schemes = conf.get(EXECUTOR_METRICS_FILESYSTEM_SCHEMES)
+    .toLowerCase(Locale.ROOT).split(",").map(_.trim).filter(_.nonEmpty)
+  private val executorSource = new ExecutorSource(threadPool, executorId, schemes)
   // Pool used for threads that supervise task killing / cancellation
   private val taskReaperPool = ThreadUtils.newDaemonCachedThreadPool("Task reaper")
   // For tasks which are in the process of being killed, this map holds the most recently created
@@ -134,6 +139,11 @@ private[spark] class Executor(
     env.metricsSystem.registerSource(new JVMCPUSource())
     executorMetricsSource.foreach(_.register(env.metricsSystem))
     env.metricsSystem.registerSource(env.blockManager.shuffleMetricsSource)
+  } else {
+    // This enable the registration of the executor source in local mode.
+    // The actual registration happens in SparkContext,
+    // it cannot be done here as the appId is not available yet
+    Executor.executorSourceLocalModeOnly = executorSource
   }
 
   // Whether to load classes in user jars before those in Spark jars
@@ -142,6 +152,8 @@ private[spark] class Executor(
   // Whether to monitor killed / interrupted tasks
   private val taskReaperEnabled = conf.get(TASK_REAPER_ENABLED)
 
+  private val killOnFatalErrorDepth = conf.get(EXECUTOR_KILL_ON_FATAL_ERROR_DEPTH)
+
   // Create our ClassLoader
   // do this after SparkEnv creation so can access the SecurityManager
   private val urlClassLoader = createClassLoader()
@@ -153,11 +165,6 @@ private[spark] class Executor(
   // for fetching remote cached RDD blocks, so need to make sure it uses the right classloader too.
   env.serializerManager.setDefaultClassLoader(replClassLoader)
 
-  // Plugins need to load using a class loader that includes the executor's user classpath
-  private val plugins: Option[PluginContainer] = Utils.withContextClassLoader(replClassLoader) {
-    PluginContainer(env, resources.asJava)
-  }
-
   // Max size of direct result. If task result is bigger than this, we use the block manager
   // to send the result back.
   private val maxDirectResultSize = Math.min(
@@ -224,6 +231,28 @@ private[spark] class Executor(
 
   heartbeater.start()
 
+  private val appStartTime = conf.getLong("spark.app.startTime", 0)
+
+  // To allow users to distribute plugins and their required files
+  // specified by --jars, --files and --archives on application submission, those
+  // jars/files/archives should be downloaded and added to the class loader via
+  // updateDependencies. This should be done before plugin initialization below
+  // because executors search plugins from the class loader and initialize them.
+  private val Seq(initialUserJars, initialUserFiles, initialUserArchives) =
+    Seq("jar", "file", "archive").map { key =>
+      conf.getOption(s"spark.app.initial.$key.urls").map { urls =>
+        Map(urls.split(",").map(url => (url, appStartTime)): _*)
+      }.getOrElse(Map.empty)
+    }
+  updateDependencies(initialUserFiles, initialUserJars, initialUserArchives)
+
+  // Plugins need to load using a class loader that includes the executor's user classpath.
+  // Plugins also needs to be initialized after the heartbeater started
+  // to avoid blocking to send heartbeat (see SPARK-32175).
+  private val plugins: Option[PluginContainer] = Utils.withContextClassLoader(replClassLoader) {
+    PluginContainer(env, resources.asJava)
+  }
+
   metricsPoller.start()
 
   private[executor] def numRunningTasks: Int = runningTasks.size()
@@ -236,7 +265,7 @@ private[spark] class Executor(
   }
 
   def launchTask(context: ExecutorBackend, taskDescription: TaskDescription): Unit = {
-    val tr = new TaskRunner(context, taskDescription)
+    val tr = new TaskRunner(context, taskDescription, plugins)
     runningTasks.put(taskDescription.taskId, tr)
     threadPool.execute(tr)
     if (decommissioned) {
@@ -296,6 +325,7 @@ private[spark] class Executor(
         case NonFatal(e) =>
           logWarning("Unable to stop heartbeater", e)
       }
+      ShuffleBlockPusher.stop()
       threadPool.shutdown()
 
       // Notify plugins that executor is shutting down so they can terminate cleanly
@@ -315,12 +345,15 @@ private[spark] class Executor(
 
   class TaskRunner(
       execBackend: ExecutorBackend,
-      private val taskDescription: TaskDescription)
+      private val taskDescription: TaskDescription,
+      private val plugins: Option[PluginContainer])
     extends Runnable {
 
     val taskId = taskDescription.taskId
-    val threadName = s"Executor task launch worker for task $taskId"
-    private val taskName = taskDescription.name
+    val taskName = taskDescription.name
+    val threadName = s"Executor task launch worker for $taskName"
+    val mdcProperties = taskDescription.properties.asScala
+      .filter(_._1.startsWith("mdc.")).toSeq
 
     /** If specified, this task has been killed and this option contains the reason. */
     @volatile private var reasonIfKilled: Option[String] = None
@@ -345,7 +378,7 @@ private[spark] class Executor(
     @volatile var task: Task[Any] = _
 
     def kill(interruptThread: Boolean, reason: String): Unit = {
-      logInfo(s"Executor is trying to kill $taskName (TID $taskId), reason: $reason")
+      logInfo(s"Executor is trying to kill $taskName, reason: $reason")
       reasonIfKilled = Some(reason)
       if (task != null) {
         synchronized {
@@ -381,7 +414,9 @@ private[spark] class Executor(
       // Report executor runtime and JVM gc time
       Option(task).foreach(t => {
         t.metrics.setExecutorRunTime(TimeUnit.NANOSECONDS.toMillis(
-          System.nanoTime() - taskStartTimeNs))
+          // SPARK-32898: it's possible that a task is killed when taskStartTimeNs has the initial
+          // value(=0) still. In this case, the executorRunTime should be considered as 0.
+          if (taskStartTimeNs > 0) System.nanoTime() - taskStartTimeNs else 0))
         t.metrics.setJvmGCTime(computeTotalGcTime() - startGCTime)
       })
 
@@ -395,6 +430,7 @@ private[spark] class Executor(
     }
 
     override def run(): Unit = {
+      setMDCForTask(taskName, mdcProperties)
       threadId = Thread.currentThread.getId
       Thread.currentThread.setName(threadName)
       val threadMXBean = ManagementFactory.getThreadMXBean
@@ -405,7 +441,7 @@ private[spark] class Executor(
       } else 0L
       Thread.currentThread.setContextClassLoader(replClassLoader)
       val ser = env.closureSerializer.newInstance()
-      logInfo(s"Running $taskName (TID $taskId)")
+      logInfo(s"Running $taskName")
       execBackend.statusUpdate(taskId, TaskState.RUNNING, EMPTY_BYTE_BUFFER)
       var taskStartTimeNs: Long = 0
       var taskStartCpu: Long = 0
@@ -417,7 +453,8 @@ private[spark] class Executor(
         // requires access to properties contained within (e.g. for access control).
         Executor.taskDeserializationProps.set(taskDescription.properties)
 
-        updateDependencies(taskDescription.addedFiles, taskDescription.addedJars)
+        updateDependencies(
+          taskDescription.addedFiles, taskDescription.addedJars, taskDescription.addedArchives)
         task = ser.deserialize[Task[Any]](
           taskDescription.serializedTask, Thread.currentThread.getContextClassLoader)
         task.localProperties = taskDescription.properties
@@ -439,7 +476,7 @@ private[spark] class Executor(
         // MapOutputTrackerMaster and its cache invalidation is not based on epoch numbers so
         // we don't need to make any special calls here.
         if (!isLocal) {
-          logDebug("Task " + taskId + "'s epoch is " + task.epoch)
+          logDebug(s"$taskName's epoch is ${task.epoch}")
           env.mapOutputTracker.asInstanceOf[MapOutputTrackerWorker].updateEpoch(task.epoch)
         }
 
@@ -457,7 +494,8 @@ private[spark] class Executor(
             taskAttemptId = taskId,
             attemptNumber = taskDescription.attemptNumber,
             metricsSystem = env.metricsSystem,
-            resources = taskDescription.resources)
+            resources = taskDescription.resources,
+            plugins = plugins)
           threwException = false
           res
         } {
@@ -465,7 +503,7 @@ private[spark] class Executor(
           val freedMemory = taskMemoryManager.cleanUpAllAllocatedMemory()
 
           if (freedMemory > 0 && !threwException) {
-            val errMsg = s"Managed memory leak detected; size = $freedMemory bytes, TID = $taskId"
+            val errMsg = s"Managed memory leak detected; size = $freedMemory bytes, $taskName"
             if (conf.get(UNSAFE_EXCEPTION_ON_MEMORY_LEAK)) {
               throw new SparkException(errMsg)
             } else {
@@ -475,7 +513,7 @@ private[spark] class Executor(
 
           if (releasedLocks.nonEmpty && !threwException) {
             val errMsg =
-              s"${releasedLocks.size} block locks were not released by TID = $taskId:\n" +
+              s"${releasedLocks.size} block locks were not released by $taskName\n" +
                 releasedLocks.mkString("[", ", ", "]")
             if (conf.get(STORAGE_EXCEPTION_PIN_LEAK)) {
               throw new SparkException(errMsg)
@@ -488,7 +526,7 @@ private[spark] class Executor(
           // uh-oh.  it appears the user code has caught the fetch-failure without throwing any
           // other exceptions.  Its *possible* this is what the user meant to do (though highly
           // unlikely).  So we will log an error and keep going.
-          logError(s"TID ${taskId} completed successfully though internally it encountered " +
+          logError(s"$taskName completed successfully though internally it encountered " +
             s"unrecoverable fetch failures!  Most likely this means user code is incorrectly " +
             s"swallowing Spark's internal ${classOf[FetchFailedException]}", fetchFailure)
         }
@@ -572,7 +610,7 @@ private[spark] class Executor(
         // directSend = sending directly back to the driver
         val serializedResult: ByteBuffer = {
           if (maxResultSize > 0 && resultSize > maxResultSize) {
-            logWarning(s"Finished $taskName (TID $taskId). Result is larger than maxResultSize " +
+            logWarning(s"Finished $taskName. Result is larger than maxResultSize " +
               s"(${Utils.bytesToString(resultSize)} > ${Utils.bytesToString(maxResultSize)}), " +
               s"dropping it.")
             ser.serialize(new IndirectTaskResult[Any](TaskResultBlockId(taskId), resultSize))
@@ -582,100 +620,105 @@ private[spark] class Executor(
               blockId,
               new ChunkedByteBuffer(serializedDirectResult.duplicate()),
               StorageLevel.MEMORY_AND_DISK_SER)
-            logInfo(
-              s"Finished $taskName (TID $taskId). $resultSize bytes result sent via BlockManager)")
+            logInfo(s"Finished $taskName. $resultSize bytes result sent via BlockManager)")
             ser.serialize(new IndirectTaskResult[Any](blockId, resultSize))
           } else {
-            logInfo(s"Finished $taskName (TID $taskId). $resultSize bytes result sent to driver")
+            logInfo(s"Finished $taskName. $resultSize bytes result sent to driver")
             serializedDirectResult
           }
         }
 
         executorSource.SUCCEEDED_TASKS.inc(1L)
         setTaskFinishedAndClearInterruptStatus()
+        plugins.foreach(_.onTaskSucceeded())
         execBackend.statusUpdate(taskId, TaskState.FINISHED, serializedResult)
       } catch {
         case t: TaskKilledException =>
-          logInfo(s"Executor killed $taskName (TID $taskId), reason: ${t.reason}")
+          logInfo(s"Executor killed $taskName, reason: ${t.reason}")
 
           val (accums, accUpdates) = collectAccumulatorsAndResetStatusOnFailure(taskStartTimeNs)
           // Here and below, put task metric peaks in a WrappedArray to expose them as a Seq
           // without requiring a copy.
           val metricPeaks = WrappedArray.make(metricsPoller.getTaskMetricPeaks(taskId))
-          val serializedTK = ser.serialize(TaskKilled(t.reason, accUpdates, accums, metricPeaks))
-          execBackend.statusUpdate(taskId, TaskState.KILLED, serializedTK)
+          val reason = TaskKilled(t.reason, accUpdates, accums, metricPeaks.toSeq)
+          plugins.foreach(_.onTaskFailed(reason))
+          execBackend.statusUpdate(taskId, TaskState.KILLED, ser.serialize(reason))
 
         case _: InterruptedException | NonFatal(_) if
             task != null && task.reasonIfKilled.isDefined =>
           val killReason = task.reasonIfKilled.getOrElse("unknown reason")
-          logInfo(s"Executor interrupted and killed $taskName (TID $taskId), reason: $killReason")
+          logInfo(s"Executor interrupted and killed $taskName, reason: $killReason")
 
           val (accums, accUpdates) = collectAccumulatorsAndResetStatusOnFailure(taskStartTimeNs)
           val metricPeaks = WrappedArray.make(metricsPoller.getTaskMetricPeaks(taskId))
-          val serializedTK = ser.serialize(TaskKilled(killReason, accUpdates, accums, metricPeaks))
-          execBackend.statusUpdate(taskId, TaskState.KILLED, serializedTK)
+          val reason = TaskKilled(killReason, accUpdates, accums, metricPeaks.toSeq)
+          plugins.foreach(_.onTaskFailed(reason))
+          execBackend.statusUpdate(taskId, TaskState.KILLED, ser.serialize(reason))
 
-        case t: Throwable if hasFetchFailure && !Utils.isFatalError(t) =>
+        case t: Throwable if hasFetchFailure && !Executor.isFatalError(t, killOnFatalErrorDepth) =>
           val reason = task.context.fetchFailed.get.toTaskFailedReason
           if (!t.isInstanceOf[FetchFailedException]) {
             // there was a fetch failure in the task, but some user code wrapped that exception
             // and threw something else.  Regardless, we treat it as a fetch failure.
             val fetchFailedCls = classOf[FetchFailedException].getName
-            logWarning(s"TID ${taskId} encountered a ${fetchFailedCls} and " +
+            logWarning(s"$taskName encountered a ${fetchFailedCls} and " +
               s"failed, but the ${fetchFailedCls} was hidden by another " +
               s"exception.  Spark is handling this like a fetch failure and ignoring the " +
               s"other exception: $t")
           }
           setTaskFinishedAndClearInterruptStatus()
+          plugins.foreach(_.onTaskFailed(reason))
           execBackend.statusUpdate(taskId, TaskState.FAILED, ser.serialize(reason))
 
         case CausedBy(cDE: CommitDeniedException) =>
           val reason = cDE.toTaskCommitDeniedReason
           setTaskFinishedAndClearInterruptStatus()
+          plugins.foreach(_.onTaskFailed(reason))
           execBackend.statusUpdate(taskId, TaskState.KILLED, ser.serialize(reason))
 
         case t: Throwable if env.isStopped =>
           // Log the expected exception after executor.stop without stack traces
           // see: SPARK-19147
-          logError(s"Exception in $taskName (TID $taskId): ${t.getMessage}")
+          logError(s"Exception in $taskName: ${t.getMessage}")
 
         case t: Throwable =>
           // Attempt to exit cleanly by informing the driver of our failure.
           // If anything goes wrong (or this was a fatal exception), we will delegate to
           // the default uncaught exception handler, which will terminate the Executor.
-          logError(s"Exception in $taskName (TID $taskId)", t)
+          logError(s"Exception in $taskName", t)
 
           // SPARK-20904: Do not report failure to driver if if happened during shut down. Because
           // libraries may set up shutdown hooks that race with running tasks during shutdown,
           // spurious failures may occur and can result in improper accounting in the driver (e.g.
-          // the task failure would not be ignored if the shutdown happened because of premption,
+          // the task failure would not be ignored if the shutdown happened because of preemption,
           // instead of an app issue).
           if (!ShutdownHookManager.inShutdown()) {
             val (accums, accUpdates) = collectAccumulatorsAndResetStatusOnFailure(taskStartTimeNs)
             val metricPeaks = WrappedArray.make(metricsPoller.getTaskMetricPeaks(taskId))
 
-            val serializedTaskEndReason = {
+            val (taskFailureReason, serializedTaskFailureReason) = {
               try {
                 val ef = new ExceptionFailure(t, accUpdates).withAccums(accums)
-                  .withMetricPeaks(metricPeaks)
-                ser.serialize(ef)
+                  .withMetricPeaks(metricPeaks.toSeq)
+                (ef, ser.serialize(ef))
               } catch {
                 case _: NotSerializableException =>
                   // t is not serializable so just send the stacktrace
                   val ef = new ExceptionFailure(t, accUpdates, false).withAccums(accums)
-                    .withMetricPeaks(metricPeaks)
-                  ser.serialize(ef)
+                    .withMetricPeaks(metricPeaks.toSeq)
+                  (ef, ser.serialize(ef))
               }
             }
             setTaskFinishedAndClearInterruptStatus()
-            execBackend.statusUpdate(taskId, TaskState.FAILED, serializedTaskEndReason)
+            plugins.foreach(_.onTaskFailed(taskFailureReason))
+            execBackend.statusUpdate(taskId, TaskState.FAILED, serializedTaskFailureReason)
           } else {
             logInfo("Not reporting error to driver during JVM shutdown.")
           }
 
           // Don't forcibly exit unless the exception was inherently fatal, to avoid
           // stopping other tasks unnecessarily.
-          if (!t.isInstanceOf[SparkOutOfMemoryError] && Utils.isFatalError(t)) {
+          if (Executor.isFatalError(t, killOnFatalErrorDepth)) {
             uncaughtExceptionHandler.uncaughtException(Thread.currentThread(), t)
           }
       } finally {
@@ -693,12 +736,20 @@ private[spark] class Executor(
     }
   }
 
+  private def setMDCForTask(taskName: String, mdc: Seq[(String, String)]): Unit = {
+    // make sure we run the task with the user-specified mdc properties only
+    MDC.clear()
+    mdc.foreach { case (key, value) => MDC.put(key, value) }
+    // avoid overriding the takName by the user
+    MDC.put("mdc.taskName", taskName)
+  }
+
   /**
    * Supervises the killing / cancellation of a task by sending the interrupted flag, optionally
    * sending a Thread.interrupt(), and monitoring the task until it finishes.
    *
    * Spark's current task cancellation / task killing mechanism is "best effort" because some tasks
-   * may not be interruptable or may not respond to their "killed" flags being set. If a significant
+   * may not be interruptible or may not respond to their "killed" flags being set. If a significant
    * fraction of a cluster's task slots are occupied by tasks that have been marked as killed but
    * remain running then this can lead to a situation where new jobs and tasks are starved of
    * resources that are being used by these zombie tasks.
@@ -733,6 +784,7 @@ private[spark] class Executor(
     private[this] val takeThreadDump: Boolean = conf.get(TASK_REAPER_THREAD_DUMP)
 
     override def run(): Unit = {
+      setMDCForTask(taskRunner.taskName, taskRunner.mdcProperties)
       val startTimeNs = System.nanoTime()
       def elapsedTimeNs = System.nanoTime() - startTimeNs
       def timeoutExceeded(): Boolean = killTimeoutNs > 0 && elapsedTimeNs > killTimeoutNs
@@ -862,32 +914,50 @@ private[spark] class Executor(
    * Download any missing dependencies if we receive a new set of files and JARs from the
    * SparkContext. Also adds any new JARs we fetched to the class loader.
    */
-  private def updateDependencies(newFiles: Map[String, Long], newJars: Map[String, Long]): Unit = {
+  private def updateDependencies(
+      newFiles: Map[String, Long],
+      newJars: Map[String, Long],
+      newArchives: Map[String, Long]): Unit = {
     lazy val hadoopConf = SparkHadoopUtil.get.newConfiguration(conf)
     synchronized {
       // Fetch missing dependencies
       for ((name, timestamp) <- newFiles if currentFiles.getOrElse(name, -1L) < timestamp) {
-        logInfo("Fetching " + name + " with timestamp " + timestamp)
+        logInfo(s"Fetching $name with timestamp $timestamp")
         // Fetch file with useCache mode, close cache for local mode.
         Utils.fetchFile(name, new File(SparkFiles.getRootDirectory()), conf,
-          env.securityManager, hadoopConf, timestamp, useCache = !isLocal)
+          hadoopConf, timestamp, useCache = !isLocal)
         currentFiles(name) = timestamp
       }
+      for ((name, timestamp) <- newArchives if currentArchives.getOrElse(name, -1L) < timestamp) {
+        logInfo(s"Fetching $name with timestamp $timestamp")
+        val sourceURI = new URI(name)
+        val uriToDownload = UriBuilder.fromUri(sourceURI).fragment(null).build()
+        val source = Utils.fetchFile(uriToDownload.toString, Utils.createTempDir(), conf,
+          hadoopConf, timestamp, useCache = !isLocal, shouldUntar = false)
+        val dest = new File(
+          SparkFiles.getRootDirectory(),
+          if (sourceURI.getFragment != null) sourceURI.getFragment else source.getName)
+        logInfo(
+          s"Unpacking an archive $name from ${source.getAbsolutePath} to ${dest.getAbsolutePath}")
+        Utils.deleteRecursively(dest)
+        Utils.unpack(source, dest)
+        currentArchives(name) = timestamp
+      }
       for ((name, timestamp) <- newJars) {
         val localName = new URI(name).getPath.split("/").last
         val currentTimeStamp = currentJars.get(name)
           .orElse(currentJars.get(localName))
           .getOrElse(-1L)
         if (currentTimeStamp < timestamp) {
-          logInfo("Fetching " + name + " with timestamp " + timestamp)
+          logInfo(s"Fetching $name with timestamp $timestamp")
           // Fetch file with useCache mode, close cache for local mode.
           Utils.fetchFile(name, new File(SparkFiles.getRootDirectory()), conf,
-            env.securityManager, hadoopConf, timestamp, useCache = !isLocal)
+            hadoopConf, timestamp, useCache = !isLocal)
           currentJars(name) = timestamp
           // Add it to our class loader
           val url = new File(SparkFiles.getRootDirectory(), localName).toURI.toURL
           if (!urlClassLoader.getURLs().contains(url)) {
-            logInfo("Adding " + url + " to class loader")
+            logInfo(s"Adding $url to class loader")
             urlClassLoader.addURL(url)
           }
         }
@@ -926,7 +996,7 @@ private[spark] class Executor(
     try {
       val response = heartbeatReceiverRef.askSync[HeartbeatResponse](
         message, new RpcTimeout(HEARTBEAT_INTERVAL_MS.millis, EXECUTOR_HEARTBEAT_INTERVAL.key))
-      if (response.reregisterBlockManager) {
+      if (!executorShutdown.get && response.reregisterBlockManager) {
         logInfo("Told to re-register on heartbeat")
         env.blockManager.reregister()
       }
@@ -949,4 +1019,29 @@ private[spark] object Executor {
   // task is fully deserialized. When possible, the TaskContext.getLocalProperty call should be
   // used instead.
   val taskDeserializationProps: ThreadLocal[Properties] = new ThreadLocal[Properties]
+
+  // Used to store executorSource, for local mode only
+  var executorSourceLocalModeOnly: ExecutorSource = null
+
+  /**
+   * Whether a `Throwable` thrown from a task is a fatal error. We will use this to decide whether
+   * to kill the executor.
+   *
+   * @param depthToCheck The max depth of the exception chain we should search for a fatal error. 0
+   *                     means not checking any fatal error (in other words, return false), 1 means
+   *                     checking only the exception but not the cause, and so on. This is to avoid
+   *                     `StackOverflowError` when hitting a cycle in the exception chain.
+   */
+  def isFatalError(t: Throwable, depthToCheck: Int): Boolean = {
+    if (depthToCheck <= 0) {
+      false
+    } else {
+      t match {
+        case _: SparkOutOfMemoryError => false
+        case e if Utils.isFatalError(e) => true
+        case e if e.getCause != null => isFatalError(e.getCause, depthToCheck - 1)
+        case _ => false
+      }
+    }
+  }
 }
diff --git a/core/src/main/scala/org/apache/spark/executor/ExecutorMetrics.scala b/core/src/main/scala/org/apache/spark/executor/ExecutorMetrics.scala
index d9aa3ef60fc9e..486e59652218b 100644
--- a/core/src/main/scala/org/apache/spark/executor/ExecutorMetrics.scala
+++ b/core/src/main/scala/org/apache/spark/executor/ExecutorMetrics.scala
@@ -44,12 +44,12 @@ class ExecutorMetrics private[spark] extends Serializable {
   /** Returns true if the values for the metrics have been set, false otherwise. */
   def isSet(): Boolean = metrics(0) > -1
 
-  private[spark] def this(metrics: Array[Long]) {
+  private[spark] def this(metrics: Array[Long]) = {
     this()
     Array.copy(metrics, 0, this.metrics, 0, Math.min(metrics.size, this.metrics.size))
   }
 
-  private[spark] def this(metrics: AtomicLongArray) {
+  private[spark] def this(metrics: AtomicLongArray) = {
     this()
     ExecutorMetricType.metricToOffset.foreach { case (_, i) =>
       this.metrics(i) = metrics.get(i)
@@ -61,7 +61,7 @@ class ExecutorMetrics private[spark] extends Serializable {
    *
    * @param executorMetrics map of executor metric name to value
    */
-  private[spark] def this(executorMetrics: Map[String, Long]) {
+  private[spark] def this(executorMetrics: Map[String, Long]) = {
     this()
     ExecutorMetricType.metricToOffset.foreach { case (name, idx) =>
       metrics(idx) = executorMetrics.getOrElse(name, 0L)
diff --git a/core/src/main/scala/org/apache/spark/executor/ExecutorMetricsPoller.scala b/core/src/main/scala/org/apache/spark/executor/ExecutorMetricsPoller.scala
index 1c1a1ca8035d0..0cdb306af74a3 100644
--- a/core/src/main/scala/org/apache/spark/executor/ExecutorMetricsPoller.scala
+++ b/core/src/main/scala/org/apache/spark/executor/ExecutorMetricsPoller.scala
@@ -53,10 +53,10 @@ private[spark] class ExecutorMetricsPoller(
 
   type StageKey = (Int, Int)
   // Task Count and Metric Peaks
-  private case class TCMP(count: AtomicLong, peaks: AtomicLongArray)
+  private[executor] case class TCMP(count: AtomicLong, peaks: AtomicLongArray)
 
   // Map of (stageId, stageAttemptId) to (count of running tasks, executor metric peaks)
-  private val stageTCMP = new ConcurrentHashMap[StageKey, TCMP]
+  private[executor] val stageTCMP = new ConcurrentHashMap[StageKey, TCMP]
 
   // Map of taskId to executor metric peaks
   private val taskMetricPeaks = new ConcurrentHashMap[Long, AtomicLongArray]
@@ -124,17 +124,12 @@ private[spark] class ExecutorMetricsPoller(
    */
   def onTaskCompletion(taskId: Long, stageId: Int, stageAttemptId: Int): Unit = {
     // Decrement the task count.
-    // Remove the entry from stageTCMP if the task count reaches zero.
 
     def decrementCount(stage: StageKey, countAndPeaks: TCMP): TCMP = {
       val countValue = countAndPeaks.count.decrementAndGet()
-      if (countValue == 0L) {
-        logDebug(s"removing (${stage._1}, ${stage._2}) from stageTCMP")
-        null
-      } else {
-        logDebug(s"stageTCMP: (${stage._1}, ${stage._2}) -> " + countValue)
-        countAndPeaks
-      }
+      assert(countValue >= 0, "task count shouldn't below 0")
+      logDebug(s"stageTCMP: (${stage._1}, ${stage._2}) -> " + countValue)
+      countAndPeaks
     }
 
     stageTCMP.computeIfPresent((stageId, stageAttemptId), decrementCount)
@@ -176,6 +171,20 @@ private[spark] class ExecutorMetricsPoller(
 
     stageTCMP.replaceAll(getUpdateAndResetPeaks)
 
+    def removeIfInactive(k: StageKey, v: TCMP): TCMP = {
+      if (v.count.get == 0) {
+        logDebug(s"removing (${k._1}, ${k._2}) from stageTCMP")
+        null
+      } else {
+        v
+      }
+    }
+
+    // Remove the entry from stageTCMP if the task count reaches zero.
+    executorUpdates.foreach { case (k, _) =>
+      stageTCMP.computeIfPresent(k, removeIfInactive)
+    }
+
     executorUpdates
   }
 
diff --git a/core/src/main/scala/org/apache/spark/executor/ExecutorMetricsSource.scala b/core/src/main/scala/org/apache/spark/executor/ExecutorMetricsSource.scala
index 14645f73ef278..12e2c9c98abc3 100644
--- a/core/src/main/scala/org/apache/spark/executor/ExecutorMetricsSource.scala
+++ b/core/src/main/scala/org/apache/spark/executor/ExecutorMetricsSource.scala
@@ -52,7 +52,7 @@ private[spark] class ExecutorMetricsSource extends Source {
   def register(metricsSystem: MetricsSystem): Unit = {
     val gauges: IndexedSeq[ExecutorMetricGauge] = (0 until ExecutorMetricType.numMetrics).map {
       idx => new ExecutorMetricGauge(idx)
-    }.toIndexedSeq
+    }
 
     ExecutorMetricType.metricToOffset.foreach {
       case (name, idx) =>
diff --git a/core/src/main/scala/org/apache/spark/executor/ExecutorSource.scala b/core/src/main/scala/org/apache/spark/executor/ExecutorSource.scala
index 50207aeb3ef6b..d2765d061d662 100644
--- a/core/src/main/scala/org/apache/spark/executor/ExecutorSource.scala
+++ b/core/src/main/scala/org/apache/spark/executor/ExecutorSource.scala
@@ -27,7 +27,10 @@ import org.apache.hadoop.fs.FileSystem
 import org.apache.spark.metrics.source.Source
 
 private[spark]
-class ExecutorSource(threadPool: ThreadPoolExecutor, executorId: String) extends Source {
+class ExecutorSource(
+    threadPool: ThreadPoolExecutor,
+    executorId: String,
+    fileSystemSchemes: Array[String]) extends Source {
 
   private def fileStats(scheme: String) : Option[FileSystem.Statistics] =
     FileSystem.getAllStatistics.asScala.find(s => s.getScheme.equals(scheme))
@@ -70,7 +73,7 @@ class ExecutorSource(threadPool: ThreadPoolExecutor, executorId: String) extends
   })
 
   // Gauge for file system stats of this executor
-  for (scheme <- Array("hdfs", "file")) {
+  for (scheme <- fileSystemSchemes) {
     registerFileSystemStat(scheme, "read_bytes", _.getBytesRead(), 0L)
     registerFileSystemStat(scheme, "write_bytes", _.getBytesWritten(), 0L)
     registerFileSystemStat(scheme, "read_ops", _.getReadOps(), 0)
diff --git a/core/src/main/scala/org/apache/spark/executor/ProcfsMetricsGetter.scala b/core/src/main/scala/org/apache/spark/executor/ProcfsMetricsGetter.scala
index 80ef757332e43..5682a21e9560d 100644
--- a/core/src/main/scala/org/apache/spark/executor/ProcfsMetricsGetter.scala
+++ b/core/src/main/scala/org/apache/spark/executor/ProcfsMetricsGetter.scala
@@ -101,7 +101,8 @@ private[spark] class ProcfsMetricsGetter(procfsDir: String = "/proc/") extends L
     }
   }
 
-  private def computeProcessTree(): Set[Int] = {
+  // Exposed for testing
+  private[executor] def computeProcessTree(): Set[Int] = {
     if (!isAvailable || testing) {
       return Set()
     }
@@ -159,7 +160,8 @@ private[spark] class ProcfsMetricsGetter(procfsDir: String = "/proc/") extends L
     }
   }
 
-  def addProcfsMetricsFromOneProcess(
+  // Exposed for testing
+  private[executor] def addProcfsMetricsFromOneProcess(
       allMetrics: ProcfsMetrics,
       pid: Int): ProcfsMetrics = {
 
@@ -199,7 +201,7 @@ private[spark] class ProcfsMetricsGetter(procfsDir: String = "/proc/") extends L
       case f: IOException =>
         logWarning("There was a problem with reading" +
           " the stat file of the process. ", f)
-        ProcfsMetrics(0, 0, 0, 0, 0, 0)
+        throw f
     }
   }
 
@@ -210,11 +212,16 @@ private[spark] class ProcfsMetricsGetter(procfsDir: String = "/proc/") extends L
     val pids = computeProcessTree
     var allMetrics = ProcfsMetrics(0, 0, 0, 0, 0, 0)
     for (p <- pids) {
-      allMetrics = addProcfsMetricsFromOneProcess(allMetrics, p)
-      // if we had an error getting any of the metrics, we don't want to report partial metrics, as
-      // that would be misleading.
-      if (!isAvailable) {
-        return ProcfsMetrics(0, 0, 0, 0, 0, 0)
+      try {
+        allMetrics = addProcfsMetricsFromOneProcess(allMetrics, p)
+        // if we had an error getting any of the metrics, we don't want to
+        // report partial metrics, as that would be misleading.
+        if (!isAvailable) {
+          return ProcfsMetrics(0, 0, 0, 0, 0, 0)
+        }
+      } catch {
+        case _: IOException =>
+          return ProcfsMetrics(0, 0, 0, 0, 0, 0)
       }
     }
     allMetrics
diff --git a/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala b/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala
index 1470a23884bb0..43742a4d46cbb 100644
--- a/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala
+++ b/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala
@@ -123,7 +123,7 @@ class TaskMetrics private[spark] () extends Serializable {
   def updatedBlockStatuses: Seq[(BlockId, BlockStatus)] = {
     // This is called on driver. All accumulator updates have a fixed value. So it's safe to use
     // `asScala` which accesses the internal values using `java.util.Iterator`.
-    _updatedBlockStatuses.value.asScala
+    _updatedBlockStatuses.value.asScala.toSeq
   }
 
   // Setters and increment-ers
@@ -199,7 +199,7 @@ class TaskMetrics private[spark] () extends Serializable {
    */
   private[spark] def mergeShuffleReadMetrics(): Unit = synchronized {
     if (tempShuffleReadMetrics.nonEmpty) {
-      shuffleReadMetrics.setMergeValues(tempShuffleReadMetrics)
+      shuffleReadMetrics.setMergeValues(tempShuffleReadMetrics.toSeq)
     }
   }
 
diff --git a/core/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala b/core/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala
index b98c7436f9906..a295ef06a6376 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala
@@ -275,7 +275,8 @@ private[spark] object ConfigEntry {
 
   val UNDEFINED = "<undefined>"
 
-  private val knownConfigs = new java.util.concurrent.ConcurrentHashMap[String, ConfigEntry[_]]()
+  private[spark] val knownConfigs =
+    new java.util.concurrent.ConcurrentHashMap[String, ConfigEntry[_]]()
 
   def registerEntry(entry: ConfigEntry[_]): Unit = {
     val existing = knownConfigs.putIfAbsent(entry.key, entry)
diff --git a/core/src/main/scala/org/apache/spark/internal/config/History.scala b/core/src/main/scala/org/apache/spark/internal/config/History.scala
index 581777de366ef..a6d1c044130f5 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/History.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/History.scala
@@ -195,4 +195,20 @@ private[spark] object History {
       .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
+
+  val HYBRID_STORE_ENABLED = ConfigBuilder("spark.history.store.hybridStore.enabled")
+    .doc("Whether to use HybridStore as the store when parsing event logs. " +
+      "HybridStore will first write data to an in-memory store and having a background thread " +
+      "that dumps data to a disk store after the writing to in-memory store is completed.")
+    .version("3.1.0")
+    .booleanConf
+    .createWithDefault(false)
+
+  val MAX_IN_MEMORY_STORE_USAGE = ConfigBuilder("spark.history.store.hybridStore.maxMemoryUsage")
+    .doc("Maximum memory space that can be used to create HybridStore. The HybridStore co-uses " +
+      "the heap memory, so the heap memory should be increased through the memory option for SHS " +
+      "if the HybridStore is enabled.")
+    .version("3.1.0")
+    .bytesConf(ByteUnit.BYTE)
+    .createWithDefaultString("2g")
 }
diff --git a/core/src/main/scala/org/apache/spark/internal/config/Kryo.scala b/core/src/main/scala/org/apache/spark/internal/config/Kryo.scala
index 646d8556538c2..90c59b079461c 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Kryo.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Kryo.scala
@@ -29,7 +29,8 @@ private[spark] object Kryo {
   val KRYO_USER_REGISTRATORS = ConfigBuilder("spark.kryo.registrator")
     .version("0.5.0")
     .stringConf
-    .createOptional
+    .toSequence
+    .createWithDefault(Nil)
 
   val KRYO_CLASSES_TO_REGISTER = ConfigBuilder("spark.kryo.classesToRegister")
     .version("1.2.0")
diff --git a/core/src/main/scala/org/apache/spark/internal/config/Python.scala b/core/src/main/scala/org/apache/spark/internal/config/Python.scala
index 188d884319644..348a33e129d65 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Python.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Python.scala
@@ -50,4 +50,10 @@ private[spark] object Python {
     .version("2.4.0")
     .bytesConf(ByteUnit.MiB)
     .createOptional
+
+  val PYTHON_AUTH_SOCKET_TIMEOUT = ConfigBuilder("spark.python.authenticate.socketTimeout")
+    .internal()
+    .version("3.1.0")
+    .timeConf(TimeUnit.SECONDS)
+    .createWithDefaultString("15s")
 }
diff --git a/core/src/main/scala/org/apache/spark/internal/config/Tests.scala b/core/src/main/scala/org/apache/spark/internal/config/Tests.scala
index a1ebe5ce0ca32..7b8b204bab640 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Tests.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Tests.scala
@@ -26,11 +26,11 @@ private[spark] object Tests {
     .longConf
     .createWithDefault(Runtime.getRuntime.maxMemory)
 
-  val TEST_SCHEDULE_INTERVAL =
-    ConfigBuilder("spark.testing.dynamicAllocation.scheduleInterval")
-      .version("2.3.0")
-      .longConf
-      .createWithDefault(100)
+  val TEST_DYNAMIC_ALLOCATION_SCHEDULE_ENABLED =
+    ConfigBuilder("spark.testing.dynamicAllocation.schedule.enabled")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(true)
 
   val IS_TESTING = ConfigBuilder("spark.testing")
     .version("1.0.1")
diff --git a/core/src/main/scala/org/apache/spark/internal/config/UI.scala b/core/src/main/scala/org/apache/spark/internal/config/UI.scala
index 231eecf086bbe..1790e97b35ae2 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/UI.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/UI.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.internal.config
 
+import java.util.Locale
 import java.util.concurrent.TimeUnit
 
 import org.apache.spark.network.util.ByteUnit
@@ -128,6 +129,21 @@ private[spark] object UI {
     .intConf
     .createWithDefault(1000)
 
+  val UI_TIMELINE_JOBS_MAXIMUM = ConfigBuilder("spark.ui.timeline.jobs.maximum")
+    .version("3.2.0")
+    .intConf
+    .createWithDefault(500)
+
+  val UI_TIMELINE_STAGES_MAXIMUM = ConfigBuilder("spark.ui.timeline.stages.maximum")
+    .version("3.2.0")
+    .intConf
+    .createWithDefault(500)
+
+  val UI_TIMELINE_EXECUTORS_MAXIMUM = ConfigBuilder("spark.ui.timeline.executors.maximum")
+    .version("3.2.0")
+    .intConf
+    .createWithDefault(250)
+
   val ACLS_ENABLE = ConfigBuilder("spark.acls.enable")
     .version("1.1.0")
     .booleanConf
@@ -191,4 +207,15 @@ private[spark] object UI {
     .version("3.0.0")
     .stringConf
     .createOptional
+
+  val MASTER_UI_DECOMMISSION_ALLOW_MODE = ConfigBuilder("spark.master.ui.decommission.allow.mode")
+    .doc("Specifies the behavior of the Master Web UI's /workers/kill endpoint. Possible choices" +
+      " are: `LOCAL` means allow this endpoint from IP's that are local to the machine running" +
+      " the Master, `DENY` means to completely disable this endpoint, `ALLOW` means to allow" +
+      " calling this endpoint from any IP.")
+    .internal()
+    .version("3.1.0")
+    .stringConf
+    .transform(_.toUpperCase(Locale.ROOT))
+    .createWithDefault("LOCAL")
 }
diff --git a/core/src/main/scala/org/apache/spark/internal/config/Worker.scala b/core/src/main/scala/org/apache/spark/internal/config/Worker.scala
index 52fcafad8b49a..fda3a57546b67 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Worker.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Worker.scala
@@ -83,9 +83,10 @@ private[spark] object Worker {
       .intConf
       .createWithDefault(100)
 
-  private[spark] val WORKER_DECOMMISSION_ENABLED =
-    ConfigBuilder("spark.worker.decommission.enabled")
-      .version("3.1.0")
-      .booleanConf
-      .createWithDefault(false)
+  val WORKER_DECOMMISSION_SIGNAL =
+    ConfigBuilder("spark.worker.decommission.signal")
+      .doc("The signal that used to trigger the worker to start decommission.")
+      .version("3.2.0")
+      .stringConf
+      .createWithDefaultString("PWR")
 }
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index 8ef0c37198568..1a18856e4156c 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -271,6 +271,13 @@ package object config {
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("0")
 
+  private[spark] val EXECUTOR_METRICS_FILESYSTEM_SCHEMES =
+    ConfigBuilder("spark.executor.metrics.fileSystemSchemes")
+      .doc("The file system schemes to report in executor metrics.")
+      .version("3.1.0")
+      .stringConf
+      .createWithDefaultString("file,hdfs")
+
   private[spark] val EXECUTOR_JAVA_OPTIONS =
     ConfigBuilder(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS)
       .withPrepended(SparkLauncher.EXECUTOR_DEFAULT_JAVA_OPTIONS)
@@ -302,8 +309,8 @@ package object config {
     .createWithDefaultString("1g")
 
   private[spark] val EXECUTOR_MEMORY_OVERHEAD = ConfigBuilder("spark.executor.memoryOverhead")
-    .doc("The amount of non-heap memory to be allocated per executor in cluster mode, " +
-      "in MiB unless otherwise specified.")
+    .doc("The amount of non-heap memory to be allocated per executor, in MiB unless otherwise" +
+      " specified.")
     .version("2.3.0")
     .bytesConf(ByteUnit.MiB)
     .createOptional
@@ -377,7 +384,7 @@ package object config {
         "get the replication level of the block to the initial number")
       .version("2.2.0")
       .booleanConf
-      .createWithDefault(false)
+      .createWithDefault(true)
 
   private[spark] val STORAGE_MEMORY_MAP_THRESHOLD =
     ConfigBuilder("spark.storage.memoryMapThreshold")
@@ -420,6 +427,29 @@ package object config {
       .booleanConf
       .createWithDefault(false)
 
+  private[spark] val STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED =
+    ConfigBuilder("spark.storage.decommission.shuffleBlocks.enabled")
+      .doc("Whether to transfer shuffle blocks during block manager decommissioning. Requires " +
+        "a migratable shuffle resolver (like sort based shuffle)")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(false)
+
+  private[spark] val STORAGE_DECOMMISSION_SHUFFLE_MAX_THREADS =
+    ConfigBuilder("spark.storage.decommission.shuffleBlocks.maxThreads")
+      .doc("Maximum number of threads to use in migrating shuffle files.")
+      .version("3.1.0")
+      .intConf
+      .checkValue(_ > 0, "The maximum number of threads should be positive")
+      .createWithDefault(8)
+
+  private[spark] val STORAGE_DECOMMISSION_RDD_BLOCKS_ENABLED =
+    ConfigBuilder("spark.storage.decommission.rddBlocks.enabled")
+      .doc("Whether to transfer RDD blocks during block manager decommissioning.")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(false)
+
   private[spark] val STORAGE_DECOMMISSION_MAX_REPLICATION_FAILURE_PER_BLOCK =
     ConfigBuilder("spark.storage.decommission.maxReplicationFailuresPerBlock")
       .internal()
@@ -441,6 +471,34 @@ package object config {
         "cache block replication should be positive.")
       .createWithDefaultString("30s")
 
+  private[spark] val STORAGE_DECOMMISSION_FALLBACK_STORAGE_PATH =
+    ConfigBuilder("spark.storage.decommission.fallbackStorage.path")
+      .doc("The location for fallback storage during block manager decommissioning. " +
+        "For example, `s3a://spark-storage/`. In case of empty, fallback storage is disabled. " +
+        "The storage should be managed by TTL because Spark will not clean it up.")
+      .version("3.1.0")
+      .stringConf
+      .checkValue(_.endsWith(java.io.File.separator), "Path should end with separator.")
+      .createOptional
+
+  private[spark] val STORAGE_DECOMMISSION_FALLBACK_STORAGE_CLEANUP =
+    ConfigBuilder("spark.storage.decommission.fallbackStorage.cleanUp")
+      .doc("If true, Spark cleans up its fallback storage data during shutting down.")
+      .version("3.2.0")
+      .booleanConf
+      .createWithDefault(false)
+
+  private[spark] val STORAGE_DECOMMISSION_SHUFFLE_MAX_DISK_SIZE =
+    ConfigBuilder("spark.storage.decommission.shuffleBlocks.maxDiskSize")
+      .doc("Maximum disk space to use to store shuffle blocks before rejecting remote " +
+        "shuffle blocks. Rejecting remote shuffle blocks means that an executor will not receive " +
+        "any shuffle migrations, and if there are no other executors available for migration " +
+        "then shuffle blocks will be lost unless " +
+        s"${STORAGE_DECOMMISSION_FALLBACK_STORAGE_PATH.key} is configured.")
+      .version("3.2.0")
+      .bytesConf(ByteUnit.BYTE)
+      .createOptional
+
   private[spark] val STORAGE_REPLICATION_TOPOLOGY_FILE =
     ConfigBuilder("spark.storage.replication.topologyFile")
       .version("2.1.0")
@@ -459,11 +517,19 @@ package object config {
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("60s")
 
-  private[spark] val STORAGE_BLOCKMANAGER_SLAVE_TIMEOUT =
-    ConfigBuilder("spark.storage.blockManagerSlaveTimeoutMs")
+  private[spark] val STORAGE_BLOCKMANAGER_MASTER_DRIVER_HEARTBEAT_TIMEOUT =
+    ConfigBuilder("spark.storage.blockManagerMasterDriverHeartbeatTimeoutMs")
+      .doc("A timeout used for block manager master's driver heartbeat endpoint.")
+      .version("3.2.0")
+      .timeConf(TimeUnit.MILLISECONDS)
+      .createWithDefaultString("10m")
+
+  private[spark] val STORAGE_BLOCKMANAGER_HEARTBEAT_TIMEOUT =
+    ConfigBuilder("spark.storage.blockManagerHeartbeatTimeoutMs")
       .version("0.7.0")
+      .withAlternative("spark.storage.blockManagerSlaveTimeoutMs")
       .timeConf(TimeUnit.MILLISECONDS)
-      .createWithDefaultString(Network.NETWORK_TIMEOUT.defaultValueString)
+      .createOptional
 
   private[spark] val STORAGE_CLEANUP_FILES_AFTER_EXECUTOR_EXIT =
     ConfigBuilder("spark.storage.cleanupFilesAfterExecutorExit")
@@ -614,6 +680,16 @@ package object config {
   private[spark] val SHUFFLE_SERVICE_PORT =
     ConfigBuilder("spark.shuffle.service.port").version("1.2.0").intConf.createWithDefault(7337)
 
+  private[spark] val SHUFFLE_SERVICE_NAME =
+    ConfigBuilder("spark.shuffle.service.name")
+      .doc("The configured name of the Spark shuffle service the client should communicate with. " +
+        "This must match the name used to configure the Shuffle within the YARN NodeManager " +
+        "configuration (`yarn.nodemanager.aux-services`). Only takes effect when " +
+        s"$SHUFFLE_SERVICE_ENABLED is set to true.")
+      .version("3.2.0")
+      .stringConf
+      .createWithDefault("spark_shuffle")
+
   private[spark] val KEYTAB = ConfigBuilder("spark.kerberos.keytab")
     .doc("Location of user's keytab.")
     .version("3.0.0")
@@ -650,6 +726,18 @@ package object config {
     .toSequence
     .createWithDefault(Nil)
 
+  private[spark] val YARN_KERBEROS_FILESYSTEM_RENEWAL_EXCLUDE =
+    ConfigBuilder("spark.yarn.kerberos.renewal.excludeHadoopFileSystems")
+      .doc("The list of Hadoop filesystem URLs whose hosts will be excluded from " +
+        "delegation token renewal at resource scheduler. Currently this is known to " +
+        "work under YARN, so YARN Resource Manager won't renew tokens for the application. " +
+        "Note that as resource scheduler does not renew token, so any application running " +
+        "longer than the original token expiration that tries to use that token will likely fail.")
+      .version("3.2.0")
+      .stringConf
+      .toSequence
+      .createWithDefault(Nil)
+
   private[spark] val EXECUTOR_INSTANCES = ConfigBuilder("spark.executor.instances")
     .version("1.0.0")
     .intConf
@@ -698,74 +786,90 @@ package object config {
       .booleanConf
       .createWithDefault(true)
 
-  // Blacklist confs
-  private[spark] val BLACKLIST_ENABLED =
-    ConfigBuilder("spark.blacklist.enabled")
-      .version("2.1.0")
+  private[spark] val EXCLUDE_ON_FAILURE_ENABLED =
+    ConfigBuilder("spark.excludeOnFailure.enabled")
+      .version("3.1.0")
+      .withAlternative("spark.blacklist.enabled")
       .booleanConf
       .createOptional
 
   private[spark] val MAX_TASK_ATTEMPTS_PER_EXECUTOR =
-    ConfigBuilder("spark.blacklist.task.maxTaskAttemptsPerExecutor")
-      .version("2.1.0")
+    ConfigBuilder("spark.excludeOnFailure.task.maxTaskAttemptsPerExecutor")
+      .version("3.1.0")
+      .withAlternative("spark.blacklist.task.maxTaskAttemptsPerExecutor")
       .intConf
       .createWithDefault(1)
 
   private[spark] val MAX_TASK_ATTEMPTS_PER_NODE =
-    ConfigBuilder("spark.blacklist.task.maxTaskAttemptsPerNode")
-      .version("2.1.0")
+    ConfigBuilder("spark.excludeOnFailure.task.maxTaskAttemptsPerNode")
+      .version("3.1.0")
+      .withAlternative("spark.blacklist.task.maxTaskAttemptsPerNode")
       .intConf
       .createWithDefault(2)
 
   private[spark] val MAX_FAILURES_PER_EXEC =
-    ConfigBuilder("spark.blacklist.application.maxFailedTasksPerExecutor")
-      .version("2.2.0")
+    ConfigBuilder("spark.excludeOnFailure.application.maxFailedTasksPerExecutor")
+      .version("3.1.0")
+      .withAlternative("spark.blacklist.application.maxFailedTasksPerExecutor")
       .intConf
       .createWithDefault(2)
 
   private[spark] val MAX_FAILURES_PER_EXEC_STAGE =
-    ConfigBuilder("spark.blacklist.stage.maxFailedTasksPerExecutor")
-      .version("2.1.0")
+    ConfigBuilder("spark.excludeOnFailure.stage.maxFailedTasksPerExecutor")
+      .version("3.1.0")
+      .withAlternative("spark.blacklist.stage.maxFailedTasksPerExecutor")
       .intConf
       .createWithDefault(2)
 
   private[spark] val MAX_FAILED_EXEC_PER_NODE =
-    ConfigBuilder("spark.blacklist.application.maxFailedExecutorsPerNode")
-      .version("2.2.0")
+    ConfigBuilder("spark.excludeOnFailure.application.maxFailedExecutorsPerNode")
+      .version("3.1.0")
+      .withAlternative("spark.blacklist.application.maxFailedExecutorsPerNode")
       .intConf
       .createWithDefault(2)
 
   private[spark] val MAX_FAILED_EXEC_PER_NODE_STAGE =
-    ConfigBuilder("spark.blacklist.stage.maxFailedExecutorsPerNode")
-      .version("2.1.0")
+    ConfigBuilder("spark.excludeOnFailure.stage.maxFailedExecutorsPerNode")
+      .version("3.1.0")
+      .withAlternative("spark.blacklist.stage.maxFailedExecutorsPerNode")
       .intConf
       .createWithDefault(2)
 
-  private[spark] val BLACKLIST_TIMEOUT_CONF =
-    ConfigBuilder("spark.blacklist.timeout")
-      .version("2.1.0")
+  private[spark] val EXCLUDE_ON_FAILURE_TIMEOUT_CONF =
+    ConfigBuilder("spark.excludeOnFailure.timeout")
+      .version("3.1.0")
+      .withAlternative("spark.blacklist.timeout")
       .timeConf(TimeUnit.MILLISECONDS)
       .createOptional
 
-  private[spark] val BLACKLIST_KILL_ENABLED =
-    ConfigBuilder("spark.blacklist.killBlacklistedExecutors")
-      .version("2.2.0")
+  private[spark] val EXCLUDE_ON_FAILURE_KILL_ENABLED =
+    ConfigBuilder("spark.excludeOnFailure.killExcludedExecutors")
+      .version("3.1.0")
+      .withAlternative("spark.blacklist.killBlacklistedExecutors")
+      .booleanConf
+      .createWithDefault(false)
+
+  private[spark] val EXCLUDE_ON_FAILURE_DECOMMISSION_ENABLED =
+    ConfigBuilder("spark.excludeOnFailure.killExcludedExecutors.decommission")
+      .doc("Attempt decommission of excluded nodes instead of going directly to kill")
+      .version("3.2.0")
       .booleanConf
       .createWithDefault(false)
 
-  private[spark] val BLACKLIST_LEGACY_TIMEOUT_CONF =
-    ConfigBuilder("spark.scheduler.executorTaskBlacklistTime")
+  private[spark] val EXCLUDE_ON_FAILURE_LEGACY_TIMEOUT_CONF =
+    ConfigBuilder("spark.scheduler.executorTaskExcludeOnFailureTime")
       .internal()
-      .version("1.0.0")
+      .version("3.1.0")
+      .withAlternative("spark.scheduler.executorTaskBlacklistTime")
       .timeConf(TimeUnit.MILLISECONDS)
       .createOptional
 
-  private[spark] val BLACKLIST_FETCH_FAILURE_ENABLED =
-    ConfigBuilder("spark.blacklist.application.fetchFailure.enabled")
-      .version("2.3.0")
+  private[spark] val EXCLUDE_ON_FAILURE_FETCH_FAILURE_ENABLED =
+    ConfigBuilder("spark.excludeOnFailure.application.fetchFailure.enabled")
+      .version("3.1.0")
+      .withAlternative("spark.blacklist.application.fetchFailure.enabled")
       .booleanConf
       .createWithDefault(false)
-  // End blacklist confs
 
   private[spark] val UNREGISTER_OUTPUT_ON_HOST_ON_FETCH_FAILURE =
     ConfigBuilder("spark.files.fetchFailure.unRegisterOutputOnHost")
@@ -955,7 +1059,7 @@ package object config {
       .doc("When true, HadoopRDD/NewHadoopRDD will not create partitions for empty input splits.")
       .version("2.3.0")
       .booleanConf
-      .createWithDefault(false)
+      .createWithDefault(true)
 
   private[spark] val SECRET_REDACTION_PATTERN =
     ConfigBuilder("spark.redaction.regex")
@@ -965,7 +1069,7 @@ package object config {
         "like YARN and event logs.")
       .version("2.1.2")
       .regexConf
-      .createWithDefault("(?i)secret|password|token".r)
+      .createWithDefault("(?i)secret|password|token|access[.]key".r)
 
   private[spark] val STRING_REDACTION_PATTERN =
     ConfigBuilder("spark.redaction.string.regex")
@@ -1391,10 +1495,9 @@ package object config {
 
   private[spark] val SHUFFLE_HOST_LOCAL_DISK_READING_ENABLED =
     ConfigBuilder("spark.shuffle.readHostLocalDisk")
-      .doc(s"If enabled (and `${SHUFFLE_USE_OLD_FETCH_PROTOCOL.key}` is disabled and external " +
-        s"shuffle `${SHUFFLE_SERVICE_ENABLED.key}` is enabled), shuffle " +
-        "blocks requested from those block managers which are running on the same host are read " +
-        "from the disk directly instead of being fetched as remote blocks over the network.")
+      .doc(s"If enabled (and `${SHUFFLE_USE_OLD_FETCH_PROTOCOL.key}` is disabled, shuffle " +
+        "blocks requested from those block managers which are running on the same host are " +
+        "read from the disk directly instead of being fetched as remote blocks over the network.")
       .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
@@ -1430,10 +1533,12 @@ package object config {
       .createWithDefaultString("365d")
 
   private[spark] val UNSCHEDULABLE_TASKSET_TIMEOUT =
-    ConfigBuilder("spark.scheduler.blacklist.unschedulableTaskSetTimeout")
+    ConfigBuilder("spark.scheduler.excludeOnFailure.unschedulableTaskSetTimeout")
       .doc("The timeout in seconds to wait to acquire a new executor and schedule a task " +
-        "before aborting a TaskSet which is unschedulable because of being completely blacklisted.")
-      .version("2.4.1")
+        "before aborting a TaskSet which is unschedulable because all executors are " +
+        "excluded due to failures.")
+      .version("3.1.0")
+      .withAlternative("spark.scheduler.blacklist.unschedulableTaskSetTimeout")
       .timeConf(TimeUnit.SECONDS)
       .checkValue(v => v >= 0, "The value should be a non negative time value.")
       .createWithDefault(120)
@@ -1615,6 +1720,13 @@ package object config {
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("32k")
 
+  private[spark] val IO_COMPRESSION_ZSTD_BUFFERPOOL_ENABLED =
+    ConfigBuilder("spark.io.compression.zstd.bufferPool.enabled")
+      .doc("If true, enable buffer pool of ZSTD JNI library.")
+      .version("3.2.0")
+      .booleanConf
+      .createWithDefault(true)
+
   private[spark] val IO_COMPRESSION_ZSTD_LEVEL =
     ConfigBuilder("spark.io.compression.zstd.level")
       .doc("Compression level for Zstd compression codec. Increasing the compression " +
@@ -1636,9 +1748,10 @@ package object config {
     ConfigBuilder("spark.eventLog.compression.codec")
       .doc("The codec used to compress event log. By default, Spark provides four codecs: " +
         "lz4, lzf, snappy, and zstd. You can also use fully qualified class names to specify " +
-        "the codec. If this is not given, spark.io.compression.codec will be used.")
+        "the codec.")
       .version("3.0.0")
-      .fallbackConf(IO_COMPRESSION_CODEC)
+      .stringConf
+      .createWithDefault("zstd")
 
   private[spark] val BUFFER_SIZE =
     ConfigBuilder("spark.buffer.size")
@@ -1762,6 +1875,16 @@ package object config {
     .toSequence
     .createWithDefault(Nil)
 
+  private[spark] val ARCHIVES = ConfigBuilder("spark.archives")
+    .version("3.1.0")
+    .doc("Comma-separated list of archives to be extracted into the working directory of each " +
+      "executor. .jar, .tar.gz, .tgz and .zip are supported. You can specify the directory " +
+      "name to unpack via adding '#' after the file name to unpack, for example, " +
+      "'file.zip#directory'. This configuration is experimental.")
+    .stringConf
+    .toSequence
+    .createWithDefault(Nil)
+
   private[spark] val SUBMIT_DEPLOY_MODE = ConfigBuilder("spark.submit.deployMode")
     .version("1.5.0")
     .stringConf
@@ -1828,6 +1951,14 @@ package object config {
       .doubleConf
       .createWithDefault(0.75)
 
+  private[spark] val SPECULATION_MIN_THRESHOLD =
+    ConfigBuilder("spark.speculation.min.threshold")
+      .doc("Minimum amount of time a task runs before being considered for speculation. " +
+        "This can be used to avoid launching speculative copies of tasks that are very short.")
+      .version("3.2.0")
+      .timeConf(TimeUnit.MILLISECONDS)
+      .createWithDefault(100)
+
   private[spark] val SPECULATION_TASK_DURATION_THRESHOLD =
     ConfigBuilder("spark.speculation.task.duration.threshold")
       .doc("Task duration after which scheduler would try to speculative run the task. If " +
@@ -1842,6 +1973,47 @@ package object config {
       .timeConf(TimeUnit.MILLISECONDS)
       .createOptional
 
+  private[spark] val DECOMMISSION_ENABLED =
+    ConfigBuilder("spark.decommission.enabled")
+      .doc("When decommission enabled, Spark will try its best to shutdown the executor " +
+        s"gracefully. Spark will try to migrate all the RDD blocks (controlled by " +
+        s"${STORAGE_DECOMMISSION_RDD_BLOCKS_ENABLED.key}) and shuffle blocks (controlled by " +
+        s"${STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED.key}) from the decommissioning " +
+        s"executor to a remote executor when ${STORAGE_DECOMMISSION_ENABLED.key} is enabled. " +
+        s"With decommission enabled, Spark will also decommission an executor instead of " +
+        s"killing when ${DYN_ALLOCATION_ENABLED.key} enabled.")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(false)
+
+  private[spark] val EXECUTOR_DECOMMISSION_KILL_INTERVAL =
+    ConfigBuilder("spark.executor.decommission.killInterval")
+      .doc("Duration after which a decommissioned executor will be killed forcefully " +
+        "*by an outside* (e.g. non-spark) service. " +
+        "This config is useful for cloud environments where we know in advance when " +
+        "an executor is going to go down after decommissioning signal i.e. around 2 mins " +
+        "in aws spot nodes, 1/2 hrs in spot block nodes etc. This config is currently " +
+        "used to decide what tasks running on decommission executors to speculate.")
+      .version("3.1.0")
+      .timeConf(TimeUnit.SECONDS)
+      .createOptional
+
+  private[spark] val EXECUTOR_DECOMMISSION_FORCE_KILL_TIMEOUT =
+    ConfigBuilder("spark.executor.decommission.forceKillTimeout")
+      .doc("Duration after which a Spark will force a decommissioning executor to exit." +
+        " this should be set to a high value in most situations as low values will prevent " +
+        " block migrations from having enough time to complete.")
+      .version("3.2.0")
+      .timeConf(TimeUnit.SECONDS)
+      .createOptional
+
+  private[spark] val EXECUTOR_DECOMMISSION_SIGNAL =
+    ConfigBuilder("spark.executor.decommission.signal")
+      .doc("The signal that used to trigger the executor to start decommission.")
+      .version("3.2.0")
+      .stringConf
+      .createWithDefaultString("PWR")
+
   private[spark] val STAGING_DIR = ConfigBuilder("spark.yarn.stagingDir")
     .doc("Staging directory used while submitting applications.")
     .version("2.0.0")
@@ -1864,4 +2036,107 @@ package object config {
       .version("3.1.0")
       .booleanConf
       .createWithDefault(false)
+
+  private[spark] val STANDALONE_SUBMIT_WAIT_APP_COMPLETION =
+    ConfigBuilder("spark.standalone.submit.waitAppCompletion")
+      .doc("In standalone cluster mode, controls whether the client waits to exit until the " +
+        "application completes. If set to true, the client process will stay alive polling " +
+        "the driver's status. Otherwise, the client process will exit after submission.")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(false)
+
+  private[spark] val EXECUTOR_ALLOW_SPARK_CONTEXT =
+    ConfigBuilder("spark.executor.allowSparkContext")
+      .doc("If set to true, SparkContext can be created in executors.")
+      .version("3.0.1")
+      .booleanConf
+      .createWithDefault(false)
+
+  private[spark] val EXECUTOR_KILL_ON_FATAL_ERROR_DEPTH =
+    ConfigBuilder("spark.executor.killOnFatalError.depth")
+      .doc("The max depth of the exception chain in a failed task Spark will search for a fatal " +
+        "error to check whether it should kill an executor. 0 means not checking any fatal " +
+        "error, 1 means checking only the exception but not the cause, and so on.")
+      .internal()
+      .version("3.1.0")
+      .intConf
+      .checkValue(_ >= 0, "needs to be a non-negative value")
+      .createWithDefault(5)
+
+  private[spark] val PUSH_BASED_SHUFFLE_ENABLED =
+    ConfigBuilder("spark.shuffle.push.enabled")
+      .doc("Set to 'true' to enable push-based shuffle on the client side and this works in " +
+        "conjunction with the server side flag spark.shuffle.server.mergedShuffleFileManagerImpl " +
+        "which needs to be set with the appropriate " +
+        "org.apache.spark.network.shuffle.MergedShuffleFileManager implementation for push-based " +
+        "shuffle to be enabled")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(false)
+
+  private[spark] val SHUFFLE_MERGER_MAX_RETAINED_LOCATIONS =
+    ConfigBuilder("spark.shuffle.push.maxRetainedMergerLocations")
+      .doc("Maximum number of shuffle push merger locations cached for push based shuffle. " +
+        "Currently, shuffle push merger locations are nothing but external shuffle services " +
+        "which are responsible for handling pushed blocks and merging them and serving " +
+        "merged blocks for later shuffle fetch.")
+      .version("3.1.0")
+      .intConf
+      .createWithDefault(500)
+
+  private[spark] val SHUFFLE_MERGER_LOCATIONS_MIN_THRESHOLD_RATIO =
+    ConfigBuilder("spark.shuffle.push.mergersMinThresholdRatio")
+      .doc("The minimum number of shuffle merger locations required to enable push based " +
+        "shuffle for a stage. This is specified as a ratio of the number of partitions in " +
+        "the child stage. For example, a reduce stage which has 100 partitions and uses the " +
+        "default value 0.05 requires at least 5 unique merger locations to enable push based " +
+        "shuffle. Merger locations are currently defined as external shuffle services.")
+      .version("3.1.0")
+      .doubleConf
+      .createWithDefault(0.05)
+
+  private[spark] val SHUFFLE_MERGER_LOCATIONS_MIN_STATIC_THRESHOLD =
+    ConfigBuilder("spark.shuffle.push.mergersMinStaticThreshold")
+      .doc(s"The static threshold for number of shuffle push merger locations should be " +
+        "available in order to enable push based shuffle for a stage. Note this config " +
+        s"works in conjunction with ${SHUFFLE_MERGER_LOCATIONS_MIN_THRESHOLD_RATIO.key}. " +
+        "Maximum of spark.shuffle.push.mergersMinStaticThreshold and " +
+        s"${SHUFFLE_MERGER_LOCATIONS_MIN_THRESHOLD_RATIO.key} ratio number of mergers needed to " +
+        "enable push based shuffle for a stage. For eg: with 1000 partitions for the child " +
+        "stage with spark.shuffle.push.mergersMinStaticThreshold as 5 and " +
+        s"${SHUFFLE_MERGER_LOCATIONS_MIN_THRESHOLD_RATIO.key} set to 0.05, we would need " +
+        "at least 50 mergers to enable push based shuffle for that stage.")
+      .version("3.1.0")
+      .doubleConf
+      .createWithDefault(5)
+
+  private[spark] val SHUFFLE_NUM_PUSH_THREADS =
+    ConfigBuilder("spark.shuffle.push.numPushThreads")
+      .doc("Specify the number of threads in the block pusher pool. These threads assist " +
+        "in creating connections and pushing blocks to remote shuffle services. By default, the " +
+        "threadpool size is equal to the number of spark executor cores.")
+      .version("3.2.0")
+      .intConf
+      .createOptional
+
+  private[spark] val SHUFFLE_MAX_BLOCK_SIZE_TO_PUSH =
+    ConfigBuilder("spark.shuffle.push.maxBlockSizeToPush")
+      .doc("The max size of an individual block to push to the remote shuffle services. Blocks " +
+       "larger than this threshold are not pushed to be merged remotely. These shuffle blocks " +
+       "will be fetched by the executors in the original manner.")
+      .version("3.2.0")
+      .bytesConf(ByteUnit.BYTE)
+      .createWithDefaultString("1m")
+
+  private[spark] val SHUFFLE_MAX_BLOCK_BATCH_SIZE_FOR_PUSH =
+    ConfigBuilder("spark.shuffle.push.maxBlockBatchSize")
+      .doc("The max size of a batch of shuffle blocks to be grouped into a single push request.")
+      .version("3.2.0")
+      .bytesConf(ByteUnit.BYTE)
+      // Default is 3m because it is greater than 2m which is the default value for
+      // TransportConf#memoryMapBytes. If this defaults to 2m as well it is very likely that each
+      // batch of block will be loaded in memory with memory mapping, which has higher overhead
+      // with small MB sized chunk of data.
+      .createWithDefaultString("3m")
 }
diff --git a/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala b/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala
index 0746e43babf9a..d9d7b06cdb8ce 100644
--- a/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala
@@ -169,4 +169,8 @@ object FileCommitProtocol extends Logging {
         ctor.newInstance(jobId, outputPath)
     }
   }
+
+  def getStagingDir(path: String, jobId: String): Path = {
+    new Path(path, ".spark-staging-" + jobId)
+  }
 }
diff --git a/core/src/main/scala/org/apache/spark/internal/io/HadoopMapReduceCommitProtocol.scala b/core/src/main/scala/org/apache/spark/internal/io/HadoopMapReduceCommitProtocol.scala
index 11ce608f52ee2..30f9a650a69c9 100644
--- a/core/src/main/scala/org/apache/spark/internal/io/HadoopMapReduceCommitProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/internal/io/HadoopMapReduceCommitProtocol.scala
@@ -41,13 +41,28 @@ import org.apache.spark.mapred.SparkHadoopMapRedUtil
  * @param jobId the job's or stage's id
  * @param path the job's output path, or null if committer acts as a noop
  * @param dynamicPartitionOverwrite If true, Spark will overwrite partition directories at runtime
- *                                  dynamically, i.e., we first write files under a staging
- *                                  directory with partition path, e.g.
- *                                  /path/to/staging/a=1/b=1/xxx.parquet. When committing the job,
- *                                  we first clean up the corresponding partition directories at
- *                                  destination path, e.g. /path/to/destination/a=1/b=1, and move
- *                                  files from staging directory to the corresponding partition
- *                                  directories under destination path.
+ *                                  dynamically. Suppose final path is /path/to/outputPath, output
+ *                                  path of [[FileOutputCommitter]] is an intermediate path, e.g.
+ *                                  /path/to/outputPath/.spark-staging-{jobId}, which is a staging
+ *                                  directory. Task attempts firstly write files under the
+ *                                  intermediate path, e.g.
+ *                                  /path/to/outputPath/.spark-staging-{jobId}/_temporary/
+ *                                  {appAttemptId}/_temporary/{taskAttemptId}/a=1/b=1/xxx.parquet.
+ *
+ *                                  1. When [[FileOutputCommitter]] algorithm version set to 1,
+ *                                  we firstly move task attempt output files to
+ *                                  /path/to/outputPath/.spark-staging-{jobId}/_temporary/
+ *                                  {appAttemptId}/{taskId}/a=1/b=1,
+ *                                  then move them to
+ *                                  /path/to/outputPath/.spark-staging-{jobId}/a=1/b=1.
+ *                                  2. When [[FileOutputCommitter]] algorithm version set to 2,
+ *                                  committing tasks directly move task attempt output files to
+ *                                  /path/to/outputPath/.spark-staging-{jobId}/a=1/b=1.
+ *
+ *                                  At the end of committing job, we move output files from
+ *                                  intermediate path to final path, e.g., move files from
+ *                                  /path/to/outputPath/.spark-staging-{jobId}/a=1/b=1
+ *                                  to /path/to/outputPath/a=1/b=1
  */
 class HadoopMapReduceCommitProtocol(
     jobId: String,
@@ -89,7 +104,7 @@ class HadoopMapReduceCommitProtocol(
    * The staging directory of this write job. Spark uses it to deal with files with absolute output
    * path, or writing data into partitioned directory with dynamicPartitionOverwrite=true.
    */
-  private def stagingDir = new Path(path, ".spark-staging-" + jobId)
+  protected def stagingDir = getStagingDir(path, jobId)
 
   protected def setupCommitter(context: TaskAttemptContext): OutputCommitter = {
     val format = context.getOutputFormatClass.getConstructor().newInstance()
@@ -106,13 +121,13 @@ class HadoopMapReduceCommitProtocol(
     val filename = getFilename(taskContext, ext)
 
     val stagingDir: Path = committer match {
-      case _ if dynamicPartitionOverwrite =>
-        assert(dir.isDefined,
-          "The dataset to be written must be partitioned when dynamicPartitionOverwrite is true.")
-        partitionPaths += dir.get
-        this.stagingDir
       // For FileOutputCommitter it has its own staging path called "work path".
       case f: FileOutputCommitter =>
+        if (dynamicPartitionOverwrite) {
+          assert(dir.isDefined,
+            "The dataset to be written must be partitioned when dynamicPartitionOverwrite is true.")
+          partitionPaths += dir.get
+        }
         new Path(Option(f.getWorkPath).map(_.toString).getOrElse(path))
       case _ => new Path(path)
     }
diff --git a/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriter.scala b/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriter.scala
index a619f10bbf064..4eeec6386c0b0 100644
--- a/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriter.scala
+++ b/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriter.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.internal.io
 
 import java.text.NumberFormat
-import java.util.{Date, Locale}
+import java.util.{Date, Locale, UUID}
 
 import scala.reflect.ClassTag
 
@@ -70,6 +70,11 @@ object SparkHadoopWriter extends Logging {
     // Assert the output format/key/value class is set in JobConf.
     config.assertConf(jobContext, rdd.conf)
 
+    // propagate the description UUID into the jobs, so that committers
+    // get an ID guaranteed to be unique.
+    jobContext.getConfiguration.set("spark.sql.sources.writeJobUUID",
+      UUID.randomUUID.toString)
+
     val committer = config.createCommitter(commitJobId)
     committer.setupJob(jobContext)
 
@@ -91,8 +96,9 @@ object SparkHadoopWriter extends Logging {
           iterator = iter)
       })
 
-      committer.commitJob(jobContext, ret)
-      logInfo(s"Job ${jobContext.getJobID} committed.")
+      logInfo(s"Start to commit write Job ${jobContext.getJobID}.")
+      val (_, duration) = Utils.timeTakenMs { committer.commitJob(jobContext, ret) }
+      logInfo(s"Write Job ${jobContext.getJobID} committed. Elapsed time: $duration ms.")
     } catch {
       case cause: Throwable =>
         logError(s"Aborting job ${jobContext.getJobID}.", cause)
@@ -222,7 +228,9 @@ class HadoopMapRedWriteConfigUtil[K, V: ClassTag](conf: SerializableJobConf)
       if (path != null) {
         path.getFileSystem(getConf)
       } else {
+        // scalastyle:off FileSystemGet
         FileSystem.get(getConf)
+        // scalastyle:on FileSystemGet
       }
     }
 
@@ -285,7 +293,9 @@ class HadoopMapRedWriteConfigUtil[K, V: ClassTag](conf: SerializableJobConf)
 
     if (SparkHadoopWriterUtils.isOutputSpecValidationEnabled(conf)) {
       // FileOutputFormat ignores the filesystem parameter
+      // scalastyle:off FileSystemGet
       val ignoredFs = FileSystem.get(getConf)
+      // scalastyle:on FileSystemGet
       getOutputFormat().checkOutputSpecs(ignoredFs, getConf)
     }
   }
diff --git a/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriterUtils.scala b/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriterUtils.scala
index de828a6d6156e..657842c620f30 100644
--- a/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriterUtils.scala
+++ b/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriterUtils.scala
@@ -20,7 +20,7 @@ package org.apache.spark.internal.io
 import java.text.SimpleDateFormat
 import java.util.{Date, Locale}
 
-import scala.util.DynamicVariable
+import scala.util.{DynamicVariable, Random}
 
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.mapred.{JobConf, JobID}
@@ -37,14 +37,35 @@ private[spark]
 object SparkHadoopWriterUtils {
 
   private val RECORDS_BETWEEN_BYTES_WRITTEN_METRIC_UPDATES = 256
+  private val RAND = new Random()
 
+  /**
+   * Create a job ID.
+   *
+   * @param time (current) time
+   * @param id job number
+   * @return a job ID
+   */
   def createJobID(time: Date, id: Int): JobID = {
+    if (id < 0) {
+      throw new IllegalArgumentException("Job number is negative")
+    }
     val jobtrackerID = createJobTrackerID(time)
     new JobID(jobtrackerID, id)
   }
 
+  /**
+   * Generate an ID for a job tracker.
+   * @param time (current) time
+   * @return a string for a job ID
+   */
   def createJobTrackerID(time: Date): String = {
-    new SimpleDateFormat("yyyyMMddHHmmss", Locale.US).format(time)
+    val base = new SimpleDateFormat("yyyyMMddHHmmss", Locale.US).format(time)
+    var l1 = RAND.nextLong()
+    if (l1 < 0) {
+      l1 = -l1
+    }
+    base + l1
   }
 
   def createPathFromString(path: String, conf: JobConf): Path = {
diff --git a/core/src/main/scala/org/apache/spark/internal/plugin/PluginContainer.scala b/core/src/main/scala/org/apache/spark/internal/plugin/PluginContainer.scala
index 4eda4767094ad..f78ec250f7173 100644
--- a/core/src/main/scala/org/apache/spark/internal/plugin/PluginContainer.scala
+++ b/core/src/main/scala/org/apache/spark/internal/plugin/PluginContainer.scala
@@ -20,7 +20,7 @@ package org.apache.spark.internal.plugin
 import scala.collection.JavaConverters._
 import scala.util.{Either, Left, Right}
 
-import org.apache.spark.{SparkContext, SparkEnv}
+import org.apache.spark.{SparkContext, SparkEnv, TaskFailedReason}
 import org.apache.spark.api.plugin._
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config._
@@ -31,6 +31,9 @@ sealed abstract class PluginContainer {
 
   def shutdown(): Unit
   def registerMetrics(appId: String): Unit
+  def onTaskStart(): Unit
+  def onTaskSucceeded(): Unit
+  def onTaskFailed(failureReason: TaskFailedReason): Unit
 
 }
 
@@ -85,6 +88,17 @@ private class DriverPluginContainer(
     }
   }
 
+  override def onTaskStart(): Unit = {
+    throw new IllegalStateException("Should not be called for the driver container.")
+  }
+
+  override def onTaskSucceeded(): Unit = {
+    throw new IllegalStateException("Should not be called for the driver container.")
+  }
+
+  override def onTaskFailed(failureReason: TaskFailedReason): Unit = {
+    throw new IllegalStateException("Should not be called for the driver container.")
+  }
 }
 
 private class ExecutorPluginContainer(
@@ -134,6 +148,39 @@ private class ExecutorPluginContainer(
       }
     }
   }
+
+  override def onTaskStart(): Unit = {
+    executorPlugins.foreach { case (name, plugin) =>
+      try {
+        plugin.onTaskStart()
+      } catch {
+        case t: Throwable =>
+          logInfo(s"Exception while calling onTaskStart on plugin $name.", t)
+      }
+    }
+  }
+
+  override def onTaskSucceeded(): Unit = {
+    executorPlugins.foreach { case (name, plugin) =>
+      try {
+        plugin.onTaskSucceeded()
+      } catch {
+        case t: Throwable =>
+          logInfo(s"Exception while calling onTaskSucceeded on plugin $name.", t)
+      }
+    }
+  }
+
+  override def onTaskFailed(failureReason: TaskFailedReason): Unit = {
+    executorPlugins.foreach { case (name, plugin) =>
+      try {
+        plugin.onTaskFailed(failureReason)
+      } catch {
+        case t: Throwable =>
+          logInfo(s"Exception while calling onTaskFailed on plugin $name.", t)
+      }
+    }
+  }
 }
 
 object PluginContainer {
diff --git a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
index 5205a2d568ac3..0b2d2677571e0 100644
--- a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
+++ b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
@@ -20,7 +20,7 @@ package org.apache.spark.io
 import java.io._
 import java.util.Locale
 
-import com.github.luben.zstd.{ZstdInputStream, ZstdOutputStream}
+import com.github.luben.zstd.{NoPool, RecyclingBufferPool, ZstdInputStreamNoFinalizer, ZstdOutputStreamNoFinalizer}
 import com.ning.compress.lzf.{LZFInputStream, LZFOutputStream}
 import net.jpountz.lz4.{LZ4BlockInputStream, LZ4BlockOutputStream, LZ4Factory}
 import net.jpountz.xxhash.XXHashFactory
@@ -107,7 +107,6 @@ private[spark] object CompressionCodec {
   }
 
   val FALLBACK_COMPRESSION_CODEC = "snappy"
-  val DEFAULT_COMPRESSION_CODEC = "lz4"
   val ALL_COMPRESSION_CODECS = shortCompressionCodecNames.values.toSeq
 }
 
@@ -218,22 +217,31 @@ class ZStdCompressionCodec(conf: SparkConf) extends CompressionCodec {
   // fastest of all with reasonably high compression ratio.
   private val level = conf.get(IO_COMPRESSION_ZSTD_LEVEL)
 
+  private val bufferPool = if (conf.get(IO_COMPRESSION_ZSTD_BUFFERPOOL_ENABLED)) {
+    RecyclingBufferPool.INSTANCE
+  } else {
+    NoPool.INSTANCE
+  }
+
   override def compressedOutputStream(s: OutputStream): OutputStream = {
     // Wrap the zstd output stream in a buffered output stream, so that we can
     // avoid overhead excessive of JNI call while trying to compress small amount of data.
-    new BufferedOutputStream(new ZstdOutputStream(s, level), bufferSize)
+    val os = new ZstdOutputStreamNoFinalizer(s, bufferPool).setLevel(level)
+    new BufferedOutputStream(os, bufferSize)
   }
 
   override private[spark] def compressedContinuousOutputStream(s: OutputStream) = {
     // SPARK-29322: Set "closeFrameOnFlush" to 'true' to let continuous input stream not being
     // stuck on reading open frame.
-    new BufferedOutputStream(new ZstdOutputStream(s, level).setCloseFrameOnFlush(true), bufferSize)
+    val os = new ZstdOutputStreamNoFinalizer(s, bufferPool)
+      .setLevel(level).setCloseFrameOnFlush(true)
+    new BufferedOutputStream(os, bufferSize)
   }
 
   override def compressedInputStream(s: InputStream): InputStream = {
     // Wrap the zstd input stream in a buffered input stream so that we can
     // avoid overhead excessive of JNI call while trying to uncompress small amount of data.
-    new BufferedInputStream(new ZstdInputStream(s), bufferSize)
+    new BufferedInputStream(new ZstdInputStreamNoFinalizer(s, bufferPool), bufferSize)
   }
 
   override def compressedContinuousInputStream(s: InputStream): InputStream = {
@@ -241,6 +249,7 @@ class ZStdCompressionCodec(conf: SparkConf) extends CompressionCodec {
     // Reading). By default `isContinuous` is false, and when we try to read from open frames,
     // `compressedInputStream` method above throws truncated error exception. This method set
     // `isContinuous` true to allow reading from open frames.
-    new BufferedInputStream(new ZstdInputStream(s).setContinuous(true), bufferSize)
+    new BufferedInputStream(
+      new ZstdInputStreamNoFinalizer(s, bufferPool).setContinuous(true), bufferSize)
   }
 }
diff --git a/core/src/main/scala/org/apache/spark/metrics/MetricsConfig.scala b/core/src/main/scala/org/apache/spark/metrics/MetricsConfig.scala
index d98d5e3b81aa0..bddd18adc683e 100644
--- a/core/src/main/scala/org/apache/spark/metrics/MetricsConfig.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/MetricsConfig.scala
@@ -102,7 +102,7 @@ private[spark] class MetricsConfig(conf: SparkConf) extends Logging {
    *
    * @param prop the flat list of properties to "unflatten" based on prefixes
    * @param regex the regex that the prefix has to comply with
-   * @return an unflatted map, mapping prefix with sub-properties under that prefix
+   * @return an unflattened map, mapping prefix with sub-properties under that prefix
    */
   def subProperties(prop: Properties, regex: Regex): mutable.HashMap[String, Properties] = {
     val subProperties = new mutable.HashMap[String, Properties]
diff --git a/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala b/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala
index 57dcbe501c6dd..abece1ec0955f 100644
--- a/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala
@@ -25,7 +25,7 @@ import scala.collection.mutable
 import com.codahale.metrics.{Metric, MetricRegistry}
 import org.eclipse.jetty.servlet.ServletContextHandler
 
-import org.apache.spark.{SecurityManager, SparkConf}
+import org.apache.spark.SparkConf
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config._
 import org.apache.spark.metrics.sink.{MetricsServlet, PrometheusServlet, Sink}
@@ -68,10 +68,7 @@ import org.apache.spark.util.Utils
  * [options] represent the specific property of this source or sink.
  */
 private[spark] class MetricsSystem private (
-    val instance: String,
-    conf: SparkConf,
-    securityMgr: SecurityManager)
-  extends Logging {
+    val instance: String, conf: SparkConf) extends Logging {
 
   private[this] val metricsConfig = new MetricsConfig(conf)
 
@@ -155,11 +152,14 @@ private[spark] class MetricsSystem private (
     } else { defaultName }
   }
 
-  def getSourcesByName(sourceName: String): Seq[Source] =
-    sources.filter(_.sourceName == sourceName)
+  def getSourcesByName(sourceName: String): Seq[Source] = sources.synchronized {
+    sources.filter(_.sourceName == sourceName).toSeq
+  }
 
   def registerSource(source: Source): Unit = {
-    sources += source
+    sources.synchronized {
+      sources += source
+    }
     try {
       val regName = buildRegistryName(source)
       registry.register(regName, source.metricRegistry)
@@ -169,7 +169,9 @@ private[spark] class MetricsSystem private (
   }
 
   def removeSource(source: Source): Unit = {
-    sources -= source
+    sources.synchronized {
+      sources -= source
+    }
     val regName = buildRegistryName(source)
     registry.removeMatching((name: String, _: Metric) => name.startsWith(regName))
   }
@@ -200,21 +202,18 @@ private[spark] class MetricsSystem private (
         try {
           if (kv._1 == "servlet") {
             val servlet = Utils.classForName[MetricsServlet](classPath)
-              .getConstructor(
-                classOf[Properties], classOf[MetricRegistry], classOf[SecurityManager])
-              .newInstance(kv._2, registry, securityMgr)
+              .getConstructor(classOf[Properties], classOf[MetricRegistry])
+              .newInstance(kv._2, registry)
             metricsServlet = Some(servlet)
           } else if (kv._1 == "prometheusServlet") {
             val servlet = Utils.classForName[PrometheusServlet](classPath)
-              .getConstructor(
-                classOf[Properties], classOf[MetricRegistry], classOf[SecurityManager])
-              .newInstance(kv._2, registry, securityMgr)
+              .getConstructor(classOf[Properties], classOf[MetricRegistry])
+              .newInstance(kv._2, registry)
             prometheusServlet = Some(servlet)
           } else {
             val sink = Utils.classForName[Sink](classPath)
-              .getConstructor(
-                classOf[Properties], classOf[MetricRegistry], classOf[SecurityManager])
-              .newInstance(kv._2, registry, securityMgr)
+              .getConstructor(classOf[Properties], classOf[MetricRegistry])
+              .newInstance(kv._2, registry)
             sinks += sink
           }
         } catch {
@@ -242,9 +241,8 @@ private[spark] object MetricsSystem {
     }
   }
 
-  def createMetricsSystem(
-      instance: String, conf: SparkConf, securityMgr: SecurityManager): MetricsSystem = {
-    new MetricsSystem(instance, conf, securityMgr)
+  def createMetricsSystem(instance: String, conf: SparkConf): MetricsSystem = {
+    new MetricsSystem(instance, conf)
   }
 }
 
diff --git a/core/src/main/scala/org/apache/spark/metrics/sink/ConsoleSink.scala b/core/src/main/scala/org/apache/spark/metrics/sink/ConsoleSink.scala
index bfd23168e4003..c8a3e4488a019 100644
--- a/core/src/main/scala/org/apache/spark/metrics/sink/ConsoleSink.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/sink/ConsoleSink.scala
@@ -22,11 +22,10 @@ import java.util.concurrent.TimeUnit
 
 import com.codahale.metrics.{ConsoleReporter, MetricRegistry}
 
-import org.apache.spark.SecurityManager
 import org.apache.spark.metrics.MetricsSystem
 
-private[spark] class ConsoleSink(val property: Properties, val registry: MetricRegistry,
-    securityMgr: SecurityManager) extends Sink {
+private[spark] class ConsoleSink(
+    val property: Properties, val registry: MetricRegistry) extends Sink {
   val CONSOLE_DEFAULT_PERIOD = 10
   val CONSOLE_DEFAULT_UNIT = "SECONDS"
 
diff --git a/core/src/main/scala/org/apache/spark/metrics/sink/CsvSink.scala b/core/src/main/scala/org/apache/spark/metrics/sink/CsvSink.scala
index 579b8e0c0e984..101691f640029 100644
--- a/core/src/main/scala/org/apache/spark/metrics/sink/CsvSink.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/sink/CsvSink.scala
@@ -23,11 +23,10 @@ import java.util.concurrent.TimeUnit
 
 import com.codahale.metrics.{CsvReporter, MetricRegistry}
 
-import org.apache.spark.SecurityManager
 import org.apache.spark.metrics.MetricsSystem
 
-private[spark] class CsvSink(val property: Properties, val registry: MetricRegistry,
-    securityMgr: SecurityManager) extends Sink {
+private[spark] class CsvSink(
+    val property: Properties, val registry: MetricRegistry) extends Sink {
   val CSV_KEY_PERIOD = "period"
   val CSV_KEY_UNIT = "unit"
   val CSV_KEY_DIR = "directory"
diff --git a/core/src/main/scala/org/apache/spark/metrics/sink/GraphiteSink.scala b/core/src/main/scala/org/apache/spark/metrics/sink/GraphiteSink.scala
index 6ce64cd3543fe..1c59e191db531 100644
--- a/core/src/main/scala/org/apache/spark/metrics/sink/GraphiteSink.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/sink/GraphiteSink.scala
@@ -23,11 +23,10 @@ import java.util.concurrent.TimeUnit
 import com.codahale.metrics.{Metric, MetricFilter, MetricRegistry}
 import com.codahale.metrics.graphite.{Graphite, GraphiteReporter, GraphiteUDP}
 
-import org.apache.spark.SecurityManager
 import org.apache.spark.metrics.MetricsSystem
 
-private[spark] class GraphiteSink(val property: Properties, val registry: MetricRegistry,
-    securityMgr: SecurityManager) extends Sink {
+private[spark] class GraphiteSink(
+    val property: Properties, val registry: MetricRegistry) extends Sink {
   val GRAPHITE_DEFAULT_PERIOD = 10
   val GRAPHITE_DEFAULT_UNIT = "SECONDS"
   val GRAPHITE_DEFAULT_PREFIX = ""
diff --git a/core/src/main/scala/org/apache/spark/metrics/sink/JmxSink.scala b/core/src/main/scala/org/apache/spark/metrics/sink/JmxSink.scala
index a7b7b5573cfe8..7ca581aee6ba6 100644
--- a/core/src/main/scala/org/apache/spark/metrics/sink/JmxSink.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/sink/JmxSink.scala
@@ -22,10 +22,9 @@ import java.util.Properties
 import com.codahale.metrics.MetricRegistry
 import com.codahale.metrics.jmx.JmxReporter
 
-import org.apache.spark.SecurityManager
 
-private[spark] class JmxSink(val property: Properties, val registry: MetricRegistry,
-    securityMgr: SecurityManager) extends Sink {
+private[spark] class JmxSink(
+    val property: Properties, val registry: MetricRegistry) extends Sink {
 
   val reporter: JmxReporter = JmxReporter.forRegistry(registry).build()
 
diff --git a/core/src/main/scala/org/apache/spark/metrics/sink/MetricsServlet.scala b/core/src/main/scala/org/apache/spark/metrics/sink/MetricsServlet.scala
index 7dd27d4fb9bf3..46d2c6821fea1 100644
--- a/core/src/main/scala/org/apache/spark/metrics/sink/MetricsServlet.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/sink/MetricsServlet.scala
@@ -26,14 +26,11 @@ import com.codahale.metrics.json.MetricsModule
 import com.fasterxml.jackson.databind.ObjectMapper
 import org.eclipse.jetty.servlet.ServletContextHandler
 
-import org.apache.spark.{SecurityManager, SparkConf}
+import org.apache.spark.SparkConf
 import org.apache.spark.ui.JettyUtils._
 
 private[spark] class MetricsServlet(
-    val property: Properties,
-    val registry: MetricRegistry,
-    securityMgr: SecurityManager)
-  extends Sink {
+    val property: Properties, val registry: MetricRegistry) extends Sink {
 
   val SERVLET_KEY_PATH = "path"
   val SERVLET_KEY_SAMPLE = "sample"
diff --git a/core/src/main/scala/org/apache/spark/metrics/sink/PrometheusServlet.scala b/core/src/main/scala/org/apache/spark/metrics/sink/PrometheusServlet.scala
index 59b863b89f75a..c087ee7c000c3 100644
--- a/core/src/main/scala/org/apache/spark/metrics/sink/PrometheusServlet.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/sink/PrometheusServlet.scala
@@ -23,24 +23,18 @@ import javax.servlet.http.HttpServletRequest
 import com.codahale.metrics.MetricRegistry
 import org.eclipse.jetty.servlet.ServletContextHandler
 
-import org.apache.spark.{SecurityManager, SparkConf}
-import org.apache.spark.annotation.Experimental
+import org.apache.spark.SparkConf
 import org.apache.spark.ui.JettyUtils._
 
 /**
- * :: Experimental ::
  * This exposes the metrics of the given registry with Prometheus format.
  *
  * The output is consistent with /metrics/json result in terms of item ordering
  * and with the previous result of Spark JMX Sink + Prometheus JMX Converter combination
  * in terms of key string format.
  */
-@Experimental
 private[spark] class PrometheusServlet(
-    val property: Properties,
-    val registry: MetricRegistry,
-    securityMgr: SecurityManager)
-  extends Sink {
+    val property: Properties, val registry: MetricRegistry) extends Sink {
 
   val SERVLET_KEY_PATH = "path"
 
@@ -56,7 +50,7 @@ private[spark] class PrometheusServlet(
   def getMetricsSnapshot(request: HttpServletRequest): String = {
     import scala.collection.JavaConverters._
 
-    val guagesLabel = """{type="gauges"}"""
+    val gaugesLabel = """{type="gauges"}"""
     val countersLabel = """{type="counters"}"""
     val metersLabel = countersLabel
     val histogramslabels = """{type="histograms"}"""
@@ -65,8 +59,8 @@ private[spark] class PrometheusServlet(
     val sb = new StringBuilder()
     registry.getGauges.asScala.foreach { case (k, v) =>
       if (!v.getValue.isInstanceOf[String]) {
-        sb.append(s"${normalizeKey(k)}Number$guagesLabel ${v.getValue}\n")
-        sb.append(s"${normalizeKey(k)}Value$guagesLabel ${v.getValue}\n")
+        sb.append(s"${normalizeKey(k)}Number$gaugesLabel ${v.getValue}\n")
+        sb.append(s"${normalizeKey(k)}Value$gaugesLabel ${v.getValue}\n")
       }
     }
     registry.getCounters.asScala.foreach { case (k, v) =>
@@ -102,7 +96,7 @@ private[spark] class PrometheusServlet(
       val snapshot = timer.getSnapshot
       sb.append(s"${prefix}Count$timersLabels ${timer.getCount}\n")
       sb.append(s"${prefix}Max$timersLabels ${snapshot.getMax}\n")
-      sb.append(s"${prefix}Mean$timersLabels ${snapshot.getMax}\n")
+      sb.append(s"${prefix}Mean$timersLabels ${snapshot.getMean}\n")
       sb.append(s"${prefix}Min$timersLabels ${snapshot.getMin}\n")
       sb.append(s"${prefix}50thPercentile$timersLabels ${snapshot.getMedian}\n")
       sb.append(s"${prefix}75thPercentile$timersLabels ${snapshot.get75thPercentile}\n")
diff --git a/core/src/main/scala/org/apache/spark/metrics/sink/Slf4jSink.scala b/core/src/main/scala/org/apache/spark/metrics/sink/Slf4jSink.scala
index 968d5ca809e72..728687f8f78ba 100644
--- a/core/src/main/scala/org/apache/spark/metrics/sink/Slf4jSink.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/sink/Slf4jSink.scala
@@ -22,14 +22,10 @@ import java.util.concurrent.TimeUnit
 
 import com.codahale.metrics.{MetricRegistry, Slf4jReporter}
 
-import org.apache.spark.SecurityManager
 import org.apache.spark.metrics.MetricsSystem
 
 private[spark] class Slf4jSink(
-    val property: Properties,
-    val registry: MetricRegistry,
-    securityMgr: SecurityManager)
-  extends Sink {
+    val property: Properties, val registry: MetricRegistry) extends Sink {
   val SLF4J_DEFAULT_PERIOD = 10
   val SLF4J_DEFAULT_UNIT = "SECONDS"
 
diff --git a/core/src/main/scala/org/apache/spark/metrics/sink/StatsdSink.scala b/core/src/main/scala/org/apache/spark/metrics/sink/StatsdSink.scala
index 61e74e05169cc..c6e7bcccd4ce9 100644
--- a/core/src/main/scala/org/apache/spark/metrics/sink/StatsdSink.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/sink/StatsdSink.scala
@@ -22,7 +22,6 @@ import java.util.concurrent.TimeUnit
 
 import com.codahale.metrics.MetricRegistry
 
-import org.apache.spark.SecurityManager
 import org.apache.spark.internal.Logging
 import org.apache.spark.metrics.MetricsSystem
 
@@ -41,10 +40,7 @@ private[spark] object StatsdSink {
 }
 
 private[spark] class StatsdSink(
-    val property: Properties,
-    val registry: MetricRegistry,
-    securityMgr: SecurityManager)
-  extends Sink with Logging {
+    val property: Properties, val registry: MetricRegistry) extends Sink with Logging {
   import StatsdSink._
 
   val host = property.getProperty(STATSD_KEY_HOST, STATSD_DEFAULT_HOST)
diff --git a/core/src/main/scala/org/apache/spark/network/BlockDataManager.scala b/core/src/main/scala/org/apache/spark/network/BlockDataManager.scala
index 0bd5774b632bf..cafb39ea82ad9 100644
--- a/core/src/main/scala/org/apache/spark/network/BlockDataManager.scala
+++ b/core/src/main/scala/org/apache/spark/network/BlockDataManager.scala
@@ -22,11 +22,16 @@ import scala.reflect.ClassTag
 import org.apache.spark.TaskContext
 import org.apache.spark.network.buffer.ManagedBuffer
 import org.apache.spark.network.client.StreamCallbackWithID
-import org.apache.spark.storage.{BlockId, ShuffleBlockId, StorageLevel}
+import org.apache.spark.storage.{BlockId, StorageLevel}
 
 private[spark]
 trait BlockDataManager {
 
+  /**
+   * Get the local directories that used by BlockManager to save the blocks to disk
+   */
+  def getLocalDiskDirs: Array[String]
+
   /**
    * Interface to get host-local shuffle block data. Throws an exception if the block cannot be
    * found or cannot be read successfully.
diff --git a/core/src/main/scala/org/apache/spark/network/BlockTransferService.scala b/core/src/main/scala/org/apache/spark/network/BlockTransferService.scala
index 70a159f3eeecf..635efc3e22628 100644
--- a/core/src/main/scala/org/apache/spark/network/BlockTransferService.scala
+++ b/core/src/main/scala/org/apache/spark/network/BlockTransferService.scala
@@ -23,7 +23,6 @@ import scala.concurrent.{Future, Promise}
 import scala.concurrent.duration.Duration
 import scala.reflect.ClassTag
 
-import org.apache.spark.internal.Logging
 import org.apache.spark.network.buffer.{FileSegmentManagedBuffer, ManagedBuffer, NioManagedBuffer}
 import org.apache.spark.network.shuffle.{BlockFetchingListener, BlockStoreClient, DownloadFileManager}
 import org.apache.spark.storage.{BlockId, EncryptedManagedBuffer, StorageLevel}
@@ -34,7 +33,7 @@ import org.apache.spark.util.ThreadUtils
  * BlockTransferService contains both client and server inside.
  */
 private[spark]
-abstract class BlockTransferService extends BlockStoreClient with Logging {
+abstract class BlockTransferService extends BlockStoreClient {
 
   /**
    * Initialize the transfer service by giving it the BlockDataManager that can be used to fetch
@@ -110,6 +109,7 @@ abstract class BlockTransferService extends BlockStoreClient with Logging {
    * This method is similar to [[uploadBlock]], except this one blocks the thread
    * until the upload finishes.
    */
+  @throws[java.io.IOException]
   def uploadBlockSync(
       hostname: String,
       port: Int,
diff --git a/core/src/main/scala/org/apache/spark/network/netty/NettyBlockRpcServer.scala b/core/src/main/scala/org/apache/spark/network/netty/NettyBlockRpcServer.scala
index 62726f7e147c5..5f831dc666ca5 100644
--- a/core/src/main/scala/org/apache/spark/network/netty/NettyBlockRpcServer.scala
+++ b/core/src/main/scala/org/apache/spark/network/netty/NettyBlockRpcServer.scala
@@ -29,7 +29,7 @@ import org.apache.spark.network.client.{RpcResponseCallback, StreamCallbackWithI
 import org.apache.spark.network.server.{OneForOneStreamManager, RpcHandler, StreamManager}
 import org.apache.spark.network.shuffle.protocol._
 import org.apache.spark.serializer.Serializer
-import org.apache.spark.storage.{BlockId, ShuffleBlockBatchId, ShuffleBlockId, StorageLevel}
+import org.apache.spark.storage.{BlockId, BlockManager, ShuffleBlockBatchId, ShuffleBlockId, StorageLevel}
 
 /**
  * Serves requests to open blocks by simply registering one chunk per block requested.
@@ -113,6 +113,26 @@ class NettyBlockRpcServer(
             s"when there is not sufficient space available to store the block.")
           responseContext.onFailure(exception)
         }
+
+      case getLocalDirs: GetLocalDirsForExecutors =>
+        val isIncorrectAppId = getLocalDirs.appId != appId
+        val execNum = getLocalDirs.execIds.length
+        if (isIncorrectAppId || execNum != 1) {
+          val errorMsg = "Invalid GetLocalDirsForExecutors request: " +
+            s"${if (isIncorrectAppId) s"incorrect application id: ${getLocalDirs.appId};"}" +
+            s"${if (execNum != 1) s"incorrect executor number: $execNum (expected 1);"}"
+          responseContext.onFailure(new IllegalStateException(errorMsg))
+        } else {
+          val expectedExecId = blockManager.asInstanceOf[BlockManager].executorId
+          val actualExecId = getLocalDirs.execIds.head
+          if (actualExecId != expectedExecId) {
+            responseContext.onFailure(new IllegalStateException(
+              s"Invalid executor id: $actualExecId, expected $expectedExecId."))
+          } else {
+            responseContext.onSuccess(new LocalDirsForExecutors(
+              Map(actualExecId -> blockManager.getLocalDiskDirs).asJava).toByteBuffer)
+          }
+        }
     }
   }
 
diff --git a/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala b/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala
index 3de7377f99202..828849812bbd1 100644
--- a/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala
+++ b/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala
@@ -33,7 +33,7 @@ import org.apache.spark.ExecutorDeadException
 import org.apache.spark.internal.config
 import org.apache.spark.network._
 import org.apache.spark.network.buffer.{ManagedBuffer, NioManagedBuffer}
-import org.apache.spark.network.client.{RpcResponseCallback, TransportClientBootstrap, TransportClientFactory}
+import org.apache.spark.network.client.{RpcResponseCallback, TransportClientBootstrap}
 import org.apache.spark.network.crypto.{AuthClientBootstrap, AuthServerBootstrap}
 import org.apache.spark.network.server._
 import org.apache.spark.network.shuffle.{BlockFetchingListener, DownloadFileManager, OneForOneBlockFetcher, RetryingBlockFetcher}
@@ -65,8 +65,6 @@ private[spark] class NettyBlockTransferService(
 
   private[this] var transportContext: TransportContext = _
   private[this] var server: TransportServer = _
-  private[this] var clientFactory: TransportClientFactory = _
-  private[this] var appId: String = _
 
   override def init(blockDataManager: BlockDataManager): Unit = {
     val rpcHandler = new NettyBlockRpcServer(conf.getAppId, serializer, blockDataManager)
@@ -80,7 +78,7 @@ private[spark] class NettyBlockTransferService(
     clientFactory = transportContext.createClientFactory(clientBootstrap.toSeq.asJava)
     server = createServer(serverBootstrap.toList)
     appId = conf.getAppId
-    logInfo(s"Server created on ${hostName}:${server.getPort}")
+    logger.info(s"Server created on $hostName:${server.getPort}")
   }
 
   /** Creates and binds the TransportServer, possibly trying multiple ports. */
@@ -113,7 +111,9 @@ private[spark] class NettyBlockTransferService(
       blockIds: Array[String],
       listener: BlockFetchingListener,
       tempFileManager: DownloadFileManager): Unit = {
-    logTrace(s"Fetch blocks from $host:$port (executor id $execId)")
+    if (logger.isTraceEnabled) {
+      logger.trace(s"Fetch blocks from $host:$port (executor id $execId)")
+    }
     try {
       val maxRetries = transportConf.maxIORetries()
       val blockFetchStarter = new RetryingBlockFetcher.BlockFetchStarter {
@@ -146,7 +146,7 @@ private[spark] class NettyBlockTransferService(
       }
     } catch {
       case e: Exception =>
-        logError("Exception while beginning fetchBlocks", e)
+        logger.error("Exception while beginning fetchBlocks", e)
         blockIds.foreach(listener.onBlockFetchFailure(_, e))
     }
   }
@@ -168,15 +168,20 @@ private[spark] class NettyBlockTransferService(
     // Everything else is encoded using our binary protocol.
     val metadata = JavaUtils.bufferToArray(serializer.newInstance().serialize((level, classTag)))
 
-    val asStream = blockData.size() > conf.get(config.MAX_REMOTE_BLOCK_SIZE_FETCH_TO_MEM)
+    // We always transfer shuffle blocks as a stream for simplicity with the receiving code since
+    // they are always written to disk. Otherwise we check the block size.
+    val asStream = (blockData.size() > conf.get(config.MAX_REMOTE_BLOCK_SIZE_FETCH_TO_MEM) ||
+      blockId.isShuffle)
     val callback = new RpcResponseCallback {
       override def onSuccess(response: ByteBuffer): Unit = {
-        logTrace(s"Successfully uploaded block $blockId${if (asStream) " as stream" else ""}")
+        if (logger.isTraceEnabled) {
+          logger.trace(s"Successfully uploaded block $blockId${if (asStream) " as stream" else ""}")
+        }
         result.success((): Unit)
       }
 
       override def onFailure(e: Throwable): Unit = {
-        logError(s"Error while uploading $blockId${if (asStream) " as stream" else ""}", e)
+        logger.error(s"Error while uploading $blockId${if (asStream) " as stream" else ""}", e)
         result.failure(e)
       }
     }
diff --git a/core/src/main/scala/org/apache/spark/rdd/DoubleRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/DoubleRDDFunctions.scala
index 943abae17a911..39f69567981ea 100644
--- a/core/src/main/scala/org/apache/spark/rdd/DoubleRDDFunctions.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/DoubleRDDFunctions.scala
@@ -173,7 +173,7 @@ class DoubleRDDFunctions(self: RDD[Double]) extends Logging with Serializable {
     if (buckets.length < 2) {
       throw new IllegalArgumentException("buckets array must have at least two elements")
     }
-    // The histogramPartition function computes the partail histogram for a given
+    // The histogramPartition function computes the partial histogram for a given
     // partition. The provided bucketFunction determines which bucket in the array
     // to increment or returns None if there is no bucket. This is done so we can
     // specialize for uniformly distributed buckets and save the O(log n) binary
diff --git a/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala b/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
index 9742d12cfe01e..5fc0b4f736d55 100644
--- a/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
@@ -81,7 +81,7 @@ private[spark] class HadoopPartition(rddId: Int, override val index: Int, s: Inp
  * @param sc The SparkContext to associate the RDD with.
  * @param broadcastedConf A general Hadoop Configuration, or a subclass of it. If the enclosed
  *   variable references an instance of JobConf, then that JobConf will be used for the Hadoop job.
- *   Otherwise, a new JobConf will be created on each slave using the enclosed Configuration.
+ *   Otherwise, a new JobConf will be created on each executor using the enclosed Configuration.
  * @param initLocalJobConfFuncOpt Optional closure used to initialize any JobConf that HadoopRDD
  *     creates.
  * @param inputFormatClass Storage format of the data to be read.
@@ -140,7 +140,7 @@ class HadoopRDD[K, V](
 
   private val ignoreEmptySplits = sparkContext.conf.get(HADOOP_RDD_IGNORE_EMPTY_SPLITS)
 
-  // Returns a JobConf that will be used on slaves to obtain input splits for Hadoop reads.
+  // Returns a JobConf that will be used on executors to obtain input splits for Hadoop reads.
   protected def getJobConf(): JobConf = {
     val conf: Configuration = broadcastedConf.value.value
     if (shouldCloneJobConf) {
@@ -232,6 +232,10 @@ class HadoopRDD[K, V](
         logWarning(s"${jobConf.get(FileInputFormat.INPUT_DIR)} doesn't exist and no" +
             s" partitions returned from this path.", e)
         Array.empty[Partition]
+      case e: IOException if e.getMessage.startsWith("Not a file:") =>
+        val path = e.getMessage.split(":").map(_.trim).apply(2)
+        throw new IOException(s"Path: ${path} is a directory, which is not supported by the " +
+          s"record reader when `mapreduce.input.fileinputformat.input.dir.recursive` is false.")
     }
   }
 
diff --git a/core/src/main/scala/org/apache/spark/rdd/InputFileBlockHolder.scala b/core/src/main/scala/org/apache/spark/rdd/InputFileBlockHolder.scala
index 1beb085db27d9..8230144025feb 100644
--- a/core/src/main/scala/org/apache/spark/rdd/InputFileBlockHolder.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/InputFileBlockHolder.scala
@@ -34,7 +34,7 @@ private[spark] object InputFileBlockHolder {
    * @param length size of the block, in bytes, or -1 if not available.
    */
   private class FileBlock(val filePath: UTF8String, val startOffset: Long, val length: Long) {
-    def this() {
+    def this() = {
       this(UTF8String.fromString(""), -1, -1)
     }
   }
diff --git a/core/src/main/scala/org/apache/spark/rdd/LocalCheckpointRDD.scala b/core/src/main/scala/org/apache/spark/rdd/LocalCheckpointRDD.scala
index 503aa0dffc9f3..113ed2db7f546 100644
--- a/core/src/main/scala/org/apache/spark/rdd/LocalCheckpointRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/LocalCheckpointRDD.scala
@@ -40,7 +40,7 @@ private[spark] class LocalCheckpointRDD[T: ClassTag](
     numPartitions: Int)
   extends CheckpointRDD[T](sc) {
 
-  def this(rdd: RDD[T]) {
+  def this(rdd: RDD[T]) = {
     this(rdd.context, rdd.id, rdd.partitions.length)
   }
 
diff --git a/core/src/main/scala/org/apache/spark/rdd/OrderedRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/OrderedRDDFunctions.scala
index 5b1c024257529..2701f457ee8af 100644
--- a/core/src/main/scala/org/apache/spark/rdd/OrderedRDDFunctions.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/OrderedRDDFunctions.scala
@@ -19,9 +19,10 @@ package org.apache.spark.rdd
 
 import scala.reflect.ClassTag
 
-import org.apache.spark.{Partitioner, RangePartitioner}
+import org.apache.spark.{InterruptibleIterator, Partitioner, RangePartitioner, TaskContext}
 import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.internal.Logging
+import org.apache.spark.util.collection.ExternalSorter
 
 /**
  * Extra functions available on RDDs of (key, value) pairs where the key is sortable through
@@ -73,7 +74,16 @@ class OrderedRDDFunctions[K : Ordering : ClassTag,
    * because it can push the sorting down into the shuffle machinery.
    */
   def repartitionAndSortWithinPartitions(partitioner: Partitioner): RDD[(K, V)] = self.withScope {
-    new ShuffledRDD[K, V, V](self, partitioner).setKeyOrdering(ordering)
+    if (self.partitioner == Some(partitioner)) {
+      self.mapPartitions(iter => {
+        val context = TaskContext.get()
+        val sorter = new ExternalSorter[K, V, V](context, None, None, Some(ordering))
+        new InterruptibleIterator(context,
+          sorter.insertAllAndUpdateMetrics(iter).asInstanceOf[Iterator[(K, V)]])
+      }, preservesPartitioning = true)
+    } else {
+      new ShuffledRDD[K, V, V](self, partitioner).setKeyOrdering(ordering)
+    }
   }
 
   /**
@@ -88,10 +98,10 @@ class OrderedRDDFunctions[K : Ordering : ClassTag,
 
     val rddToFilter: RDD[P] = self.partitioner match {
       case Some(rp: RangePartitioner[K, V]) =>
-        val partitionIndicies = (rp.getPartition(lower), rp.getPartition(upper)) match {
+        val partitionIndices = (rp.getPartition(lower), rp.getPartition(upper)) match {
           case (l, u) => Math.min(l, u) to Math.max(l, u)
         }
-        PartitionPruningRDD.create(self, partitionIndicies.contains)
+        PartitionPruningRDD.create(self, partitionIndices.contains)
       case _ =>
         self
     }
diff --git a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
index 1e39e10856877..f280c220a2c8d 100644
--- a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
@@ -934,7 +934,7 @@ class PairRDDFunctions[K, V](self: RDD[(K, V)])
           for (pair <- it if pair._1 == key) {
             buf += pair._2
           }
-          buf
+          buf.toSeq
         } : Seq[V]
         val res = self.context.runJob(self, process, Array(index))
         res(0)
diff --git a/core/src/main/scala/org/apache/spark/rdd/ParallelCollectionRDD.scala b/core/src/main/scala/org/apache/spark/rdd/ParallelCollectionRDD.scala
index 9f8019b80a4dd..f0239cdd9136d 100644
--- a/core/src/main/scala/org/apache/spark/rdd/ParallelCollectionRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/ParallelCollectionRDD.scala
@@ -19,7 +19,6 @@ package org.apache.spark.rdd
 
 import java.io._
 
-import scala.Serializable
 import scala.collection.Map
 import scala.collection.immutable.NumericRange
 import scala.collection.mutable.ArrayBuffer
@@ -133,12 +132,11 @@ private object ParallelCollectionRDD {
           // If the range is inclusive, use inclusive range for the last slice
           if (r.isInclusive && index == numSlices - 1) {
             new Range.Inclusive(r.start + start * r.step, r.end, r.step)
-          }
-          else {
-            new Range(r.start + start * r.step, r.start + end * r.step, r.step)
+          } else {
+            new Range.Inclusive(r.start + start * r.step, r.start + (end - 1) * r.step, r.step)
           }
         }.toSeq.asInstanceOf[Seq[Seq[T]]]
-      case nr: NumericRange[_] =>
+      case nr: NumericRange[T] =>
         // For ranges of Long, Double, BigInteger, etc
         val slices = new ArrayBuffer[Seq[T]](numSlices)
         var r = nr
@@ -147,7 +145,7 @@ private object ParallelCollectionRDD {
           slices += r.take(sliceSize).asInstanceOf[Seq[T]]
           r = r.drop(sliceSize)
         }
-        slices
+        slices.toSeq
       case _ =>
         val array = seq.toArray // To prevent O(n^2) operations for List etc
         positions(array.length, numSlices).map { case (start, end) =>
diff --git a/core/src/main/scala/org/apache/spark/rdd/PipedRDD.scala b/core/src/main/scala/org/apache/spark/rdd/PipedRDD.scala
index 3b11e82dab196..5dd8cb8440be6 100644
--- a/core/src/main/scala/org/apache/spark/rdd/PipedRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/PipedRDD.scala
@@ -238,7 +238,7 @@ private object PipedRDD {
     while(tok.hasMoreElements) {
       buf += tok.nextToken()
     }
-    buf
+    buf.toSeq
   }
 
   val STDIN_WRITER_THREAD_PREFIX = "stdin writer for"
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
index 39068a5ab046d..eb7bf4dd51056 100644
--- a/core/src/main/scala/org/apache/spark/rdd/RDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
@@ -327,7 +327,7 @@ abstract class RDD[T: ClassTag](
 
   /**
    * Internal method to this RDD; will read from cache if applicable, or otherwise compute it.
-   * This should ''not'' be called by users directly, but is available for implementors of custom
+   * This should ''not'' be called by users directly, but is available for implementers of custom
    * subclasses of RDD.
    */
   final def iterator(split: Partition, context: TaskContext): Iterator[T] = {
@@ -1780,10 +1780,9 @@ abstract class RDD[T: ClassTag](
    * It will result in new executors with the resources specified being acquired to
    * calculate the RDD.
    */
-  // PRIVATE for now, added for testing purposes, will be made public with SPARK-29150
   @Experimental
-  @Since("3.0.0")
-  private[spark] def withResources(rp: ResourceProfile): this.type = {
+  @Since("3.1.0")
+  def withResources(rp: ResourceProfile): this.type = {
     resourceProfile = Option(rp)
     sc.resourceProfileManager.addResourceProfile(resourceProfile.get)
     this
@@ -1794,10 +1793,9 @@ abstract class RDD[T: ClassTag](
    * @return the user specified ResourceProfile or null (for Java compatibility) if
    *         none was specified
    */
-  // PRIVATE for now, added for testing purposes, will be made public with SPARK-29150
   @Experimental
-  @Since("3.0.0")
-  private[spark] def getResourceProfile(): ResourceProfile = resourceProfile.getOrElse(null)
+  @Since("3.1.0")
+  def getResourceProfile(): ResourceProfile = resourceProfile.getOrElse(null)
 
   // =======================================================================
   // Other internal methods and fields
@@ -1921,9 +1919,8 @@ abstract class RDD[T: ClassTag](
 
       val persistence = if (storageLevel != StorageLevel.NONE) storageLevel.description else ""
       val storageInfo = rdd.context.getRDDStorageInfo(_.id == rdd.id).map(info =>
-        "    CachedPartitions: %d; MemorySize: %s; ExternalBlockStoreSize: %s; DiskSize: %s".format(
-          info.numCachedPartitions, bytesToString(info.memSize),
-          bytesToString(info.externalBlockStoreSize), bytesToString(info.diskSize)))
+        "    CachedPartitions: %d; MemorySize: %s; DiskSize: %s".format(
+          info.numCachedPartitions, bytesToString(info.memSize), bytesToString(info.diskSize)))
 
       s"$rdd [$persistence]" +: storageInfo
     }
@@ -2012,6 +2009,9 @@ abstract class RDD[T: ClassTag](
   @transient protected lazy val isBarrier_ : Boolean =
     dependencies.filter(!_.isInstanceOf[ShuffleDependency[_, _, _]]).exists(_.rdd.isBarrier())
 
+  private final lazy val _outputDeterministicLevel: DeterministicLevel.Value =
+    getOutputDeterministicLevel
+
   /**
    * Returns the deterministic level of this RDD's output. Please refer to [[DeterministicLevel]]
    * for the definition.
@@ -2022,14 +2022,14 @@ abstract class RDD[T: ClassTag](
    * candidate that is deterministic least. Please override [[getOutputDeterministicLevel]] to
    * provide custom logic of calculating output deterministic level.
    */
-  // TODO: make it public so users can set deterministic level to their custom RDDs.
+  // TODO(SPARK-34612): make it public so users can set deterministic level to their custom RDDs.
   // TODO: this can be per-partition. e.g. UnionRDD can have different deterministic level for
   // different partitions.
-  private[spark] final lazy val outputDeterministicLevel: DeterministicLevel.Value = {
+  private[spark] final def outputDeterministicLevel: DeterministicLevel.Value = {
     if (isReliablyCheckpointed) {
       DeterministicLevel.DETERMINATE
     } else {
-      getOutputDeterministicLevel
+      _outputDeterministicLevel
     }
   }
 
diff --git a/core/src/main/scala/org/apache/spark/rdd/ReliableCheckpointRDD.scala b/core/src/main/scala/org/apache/spark/rdd/ReliableCheckpointRDD.scala
index 576a83f6ab4d9..5093a12777ad3 100644
--- a/core/src/main/scala/org/apache/spark/rdd/ReliableCheckpointRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/ReliableCheckpointRDD.scala
@@ -20,7 +20,6 @@ package org.apache.spark.rdd
 import java.io.{FileNotFoundException, IOException}
 import java.util.concurrent.TimeUnit
 
-import scala.collection.mutable
 import scala.reflect.ClassTag
 import scala.util.control.NonFatal
 
diff --git a/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala
index 02def89dd8c2b..2f6ff0acdf024 100644
--- a/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala
@@ -32,16 +32,13 @@ import org.apache.spark.internal.Logging
  * @note This can't be part of PairRDDFunctions because we need more implicit parameters to
  * convert our keys and values to Writable.
  */
-class SequenceFileRDDFunctions[K <% Writable: ClassTag, V <% Writable : ClassTag](
+class SequenceFileRDDFunctions[K: IsWritable: ClassTag, V: IsWritable: ClassTag](
     self: RDD[(K, V)],
     _keyWritableClass: Class[_ <: Writable],
     _valueWritableClass: Class[_ <: Writable])
   extends Logging
   with Serializable {
 
-  // TODO the context bound (<%) above should be replaced with simple type bound and implicit
-  // conversion but is a breaking change. This should be fixed in Spark 3.x.
-
   /**
    * Output the RDD as a Hadoop SequenceFile using the Writable types we infer from the RDD's key
    * and value types. If the key or value are Writable, then we use their classes directly;
@@ -52,7 +49,7 @@ class SequenceFileRDDFunctions[K <% Writable: ClassTag, V <% Writable : ClassTag
   def saveAsSequenceFile(
       path: String,
       codec: Option[Class[_ <: CompressionCodec]] = None): Unit = self.withScope {
-    def anyToWritable[U <% Writable](u: U): Writable = u
+    def anyToWritable[U: IsWritable](u: U): Writable = u
 
     // TODO We cannot force the return type of `anyToWritable` be same as keyWritableClass and
     // valueWritableClass at the compile time. To implement that, we need to add type parameters to
diff --git a/core/src/main/scala/org/apache/spark/rdd/SubtractedRDD.scala b/core/src/main/scala/org/apache/spark/rdd/SubtractedRDD.scala
index d5a811d4dc3fd..1dec977f47a1b 100644
--- a/core/src/main/scala/org/apache/spark/rdd/SubtractedRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/SubtractedRDD.scala
@@ -124,7 +124,7 @@ private[spark] class SubtractedRDD[K: ClassTag, V: ClassTag, W: ClassTag](
     integrate(0, t => getSeq(t._1) += t._2)
     // the second dep is rdd2; remove all of its keys
     integrate(1, t => map.remove(t._1))
-    map.asScala.iterator.map(t => t._2.iterator.map((t._1, _))).flatten
+    map.asScala.iterator.flatMap(t => t._2.iterator.map((t._1, _)))
   }
 
   override def clearDependencies(): Unit = {
diff --git a/core/src/main/scala/org/apache/spark/rdd/UnionRDD.scala b/core/src/main/scala/org/apache/spark/rdd/UnionRDD.scala
index 63fa3c2487c33..0a93023443704 100644
--- a/core/src/main/scala/org/apache/spark/rdd/UnionRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/UnionRDD.scala
@@ -98,7 +98,7 @@ class UnionRDD[T: ClassTag](
       deps += new RangeDependency(rdd, 0, pos, rdd.partitions.length)
       pos += rdd.partitions.length
     }
-    deps
+    deps.toSeq
   }
 
   override def compute(s: Partition, context: TaskContext): Iterator[T] = {
diff --git a/core/src/main/scala/org/apache/spark/rdd/package.scala b/core/src/main/scala/org/apache/spark/rdd/package.scala
index 55fc6e4d2b4df..43ca6d7643b17 100644
--- a/core/src/main/scala/org/apache/spark/rdd/package.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/package.scala
@@ -17,7 +17,11 @@
 
 package org.apache.spark
 
+import org.apache.hadoop.io.Writable
+
 /**
  * Provides several RDD implementations. See [[org.apache.spark.rdd.RDD]].
  */
-package object rdd
+package object rdd {
+  type IsWritable[A] = A => Writable
+}
diff --git a/core/src/main/scala/org/apache/spark/resource/ExecutorResourceRequest.scala b/core/src/main/scala/org/apache/spark/resource/ExecutorResourceRequest.scala
index 9a920914ed674..76af41ad5b5ea 100644
--- a/core/src/main/scala/org/apache/spark/resource/ExecutorResourceRequest.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ExecutorResourceRequest.scala
@@ -17,8 +17,10 @@
 
 package org.apache.spark.resource
 
+import org.apache.spark.annotation.{Evolving, Since}
+
 /**
- * An Executor resource request. This is used in conjunction with the ResourceProfile to
+ * An Executor resource request. This is used in conjunction with the [[ResourceProfile]] to
  * programmatically specify the resources needed for an RDD that will be applied at the
  * stage level.
  *
@@ -37,7 +39,7 @@ package org.apache.spark.resource
  *
  * See the configuration and cluster specific docs for more details.
  *
- * Use ExecutorResourceRequests class as a convenience API.
+ * Use [[ExecutorResourceRequests]] class as a convenience API.
  *
  * @param resourceName Name of the resource
  * @param amount Amount requesting
@@ -46,11 +48,10 @@ package org.apache.spark.resource
  *                        allocated. The script runs on Executors startup to discover the addresses
  *                        of the resources available.
  * @param vendor Optional vendor, required for some cluster managers
- *
- * This api is currently private until the rest of the pieces are in place and then it
- * will become public.
  */
-private[spark] class ExecutorResourceRequest(
+@Evolving
+@Since("3.1.0")
+class ExecutorResourceRequest(
     val resourceName: String,
     val amount: Long,
     val discoveryScript: String = "",
diff --git a/core/src/main/scala/org/apache/spark/resource/ExecutorResourceRequests.scala b/core/src/main/scala/org/apache/spark/resource/ExecutorResourceRequests.scala
index 4ee1a07564042..b6992f4f88380 100644
--- a/core/src/main/scala/org/apache/spark/resource/ExecutorResourceRequests.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ExecutorResourceRequests.scala
@@ -22,6 +22,7 @@ import java.util.concurrent.ConcurrentHashMap
 
 import scala.collection.JavaConverters._
 
+import org.apache.spark.annotation.{Evolving, Since}
 import org.apache.spark.network.util.JavaUtils
 import org.apache.spark.resource.ResourceProfile._
 
@@ -29,20 +30,26 @@ import org.apache.spark.resource.ResourceProfile._
  * A set of Executor resource requests. This is used in conjunction with the ResourceProfile to
  * programmatically specify the resources needed for an RDD that will be applied at the
  * stage level.
- *
- * This api is currently private until the rest of the pieces are in place and then it
- * will become public.
  */
-private[spark] class ExecutorResourceRequests() extends Serializable {
+@Evolving
+@Since("3.1.0")
+class ExecutorResourceRequests() extends Serializable {
 
   private val _executorResources = new ConcurrentHashMap[String, ExecutorResourceRequest]()
 
+  /**
+   * Returns all the resource requests for the task.
+   */
   def requests: Map[String, ExecutorResourceRequest] = _executorResources.asScala.toMap
 
+  /**
+   * (Java-specific) Returns all the resource requests for the executor.
+   */
   def requestsJMap: JMap[String, ExecutorResourceRequest] = requests.asJava
 
   /**
    * Specify heap memory. The value specified will be converted to MiB.
+   * This is a convenient API to add [[ExecutorResourceRequest]] for "memory" resource.
    *
    * @param amount Amount of memory. In the same format as JVM memory strings (e.g. 512m, 2g).
    *               Default unit is MiB if not specified.
@@ -54,8 +61,24 @@ private[spark] class ExecutorResourceRequests() extends Serializable {
     this
   }
 
+  /**
+   * Specify off heap memory. The value specified will be converted to MiB.
+   * This value only take effect when MEMORY_OFFHEAP_ENABLED is true.
+   * This is a convenient API to add [[ExecutorResourceRequest]] for "offHeap" resource.
+   *
+   * @param amount Amount of memory. In the same format as JVM memory strings (e.g. 512m, 2g).
+   *               Default unit is MiB if not specified.
+   */
+  def offHeapMemory(amount: String): this.type = {
+    val amountMiB = JavaUtils.byteStringAsMb(amount)
+    val req = new ExecutorResourceRequest(OFFHEAP_MEM, amountMiB)
+    _executorResources.put(OFFHEAP_MEM, req)
+    this
+  }
+
   /**
    * Specify overhead memory. The value specified will be converted to MiB.
+   * This is a convenient API to add [[ExecutorResourceRequest]] for "memoryOverhead" resource.
    *
    * @param amount Amount of memory. In the same format as JVM memory strings (e.g. 512m, 2g).
    *               Default unit is MiB if not specified.
@@ -69,6 +92,7 @@ private[spark] class ExecutorResourceRequests() extends Serializable {
 
   /**
    * Specify pyspark memory. The value specified will be converted to MiB.
+   * This is a convenient API to add [[ExecutorResourceRequest]] for "pyspark.memory" resource.
    *
    * @param amount Amount of memory. In the same format as JVM memory strings (e.g. 512m, 2g).
    *               Default unit is MiB if not specified.
@@ -82,6 +106,7 @@ private[spark] class ExecutorResourceRequests() extends Serializable {
 
   /**
    * Specify number of cores per Executor.
+   * This is a convenient API to add [[ExecutorResourceRequest]] for "cores" resource.
    *
    * @param amount Number of cores to allocate per Executor.
    */
@@ -97,6 +122,7 @@ private[spark] class ExecutorResourceRequests() extends Serializable {
    *  like GPUs are gpu (spark configs spark.executor.resource.gpu.*). If you pass in a resource
    *  that the cluster manager doesn't support the result is undefined, it may error or may just
    *  be ignored.
+   *  This is a convenient API to add [[ExecutorResourceRequest]] for custom resources.
    *
    * @param resourceName Name of the resource.
    * @param amount amount of that resource per executor to use.
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceAllocator.scala b/core/src/main/scala/org/apache/spark/resource/ResourceAllocator.scala
index 22272a0f98a6c..7605e8c44b931 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceAllocator.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceAllocator.scala
@@ -20,13 +20,12 @@ package org.apache.spark.resource
 import scala.collection.mutable
 
 import org.apache.spark.SparkException
-import org.apache.spark.util.collection.OpenHashMap
 
 /**
  * Trait used to help executor/worker allocate resources.
  * Please note that this is intended to be used in a single thread.
  */
-trait ResourceAllocator {
+private[spark] trait ResourceAllocator {
 
   protected def resourceName: String
   protected def resourceAddresses: Seq[String]
@@ -40,7 +39,7 @@ trait ResourceAllocator {
    * can be a multiple, such that each address can be allocated up to [[slotsPerAddress]]
    * times.
    *
-   * TODO Use [[OpenHashMap]] instead to gain better performance.
+   * TODO Use [[org.apache.spark.util.collection.OpenHashMap]] instead to gain better performance.
    */
   private lazy val addressAvailabilityMap = {
     mutable.HashMap(resourceAddresses.map(_ -> slotsPerAddress): _*)
@@ -56,7 +55,7 @@ trait ResourceAllocator {
   def availableAddrs: Seq[String] = addressAvailabilityMap
     .flatMap { case (addr, available) =>
       (0 until available).map(_ => addr)
-    }.toSeq
+    }.toSeq.sorted
 
   /**
    * Sequence of currently assigned resource addresses.
@@ -68,7 +67,7 @@ trait ResourceAllocator {
   private[spark] def assignedAddrs: Seq[String] = addressAvailabilityMap
     .flatMap { case (addr, available) =>
       (0 until slotsPerAddress - available).map(_ => addr)
-    }.toSeq
+    }.toSeq.sorted
 
   /**
    * Acquire a sequence of resource addresses (to a launched task), these addresses must be
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala b/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala
index 7027d1e3511b5..d861e91771673 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala
@@ -29,9 +29,11 @@ import org.apache.spark.util.Utils.executeAndGetOutput
 /**
  * The default plugin that is loaded into a Spark application to control how custom
  * resources are discovered. This executes the discovery script specified by the user
- * and gets the json output back and contructs ResourceInformation objects from that.
+ * and gets the json output back and constructs ResourceInformation objects from that.
  * If the user specifies custom plugins, this is the last one to be executed and
  * throws if the resource isn't discovered.
+ *
+ * @since 3.0.0
  */
 @DeveloperApi
 class ResourceDiscoveryScriptPlugin extends ResourceDiscoveryPlugin with Logging {
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceInformation.scala b/core/src/main/scala/org/apache/spark/resource/ResourceInformation.scala
index d5ac41b995559..be056e15b6d03 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceInformation.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceInformation.scala
@@ -33,6 +33,8 @@ import org.apache.spark.annotation.Evolving
  *
  * @param name the name of the resource
  * @param addresses an array of strings describing the addresses of the resource
+ *
+ * @since 3.0.0
  */
 @Evolving
 class ResourceInformation(
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala b/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala
index 761c4450ca5f1..1ebd8bd89fd42 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala
@@ -25,18 +25,26 @@ import scala.collection.JavaConverters._
 import scala.collection.mutable
 
 import org.apache.spark.{SparkConf, SparkException}
-import org.apache.spark.annotation.Evolving
+import org.apache.spark.annotation.{Evolving, Since}
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config._
 import org.apache.spark.internal.config.Python.PYSPARK_EXECUTOR_MEMORY
+import org.apache.spark.util.Utils
 
 /**
  * Resource profile to associate with an RDD. A ResourceProfile allows the user to
  * specify executor and task requirements for an RDD that will get applied during a
  * stage. This allows the user to change the resource requirements between stages.
- * This is meant to be immutable so user can't change it after building.
+ * This is meant to be immutable so user can't change it after building. Users
+ * should use [[ResourceProfileBuilder]] to build it.
+ *
+ * @param executorResources Resource requests for executors. Mapped from the resource
+ *                          name (e.g., cores, memory, CPU) to its specific request.
+ * @param taskResources Resource requests for tasks. Mapped from the resource
+ *                      name (e.g., cores, memory, CPU) to its specific request.
  */
 @Evolving
+@Since("3.1.0")
 class ResourceProfile(
     val executorResources: Map[String, ExecutorResourceRequest],
     val taskResources: Map[String, TaskResourceRequest]) extends Serializable with Logging {
@@ -51,6 +59,9 @@ class ResourceProfile(
   private var _maxTasksPerExecutor: Option[Int] = None
   private var _coresLimitKnown: Boolean = false
 
+  /**
+   * A unique id of this ResourceProfile
+   */
   def id: Int = _id
 
   /**
@@ -240,19 +251,45 @@ class ResourceProfile(
 
 object ResourceProfile extends Logging {
   // task resources
+  /**
+   * built-in task resource: cpus
+   */
   val CPUS = "cpus"
   // Executor resources
+  // Make sure add new executor resource in below allSupportedExecutorResources
+  /**
+   * built-in executor resource: cores
+   */
   val CORES = "cores"
+  /**
+   * built-in executor resource: cores
+   */
   val MEMORY = "memory"
+  /**
+   * built-in executor resource: offHeap
+   */
+  val OFFHEAP_MEM = "offHeap"
+  /**
+   * built-in executor resource: memoryOverhead
+   */
   val OVERHEAD_MEM = "memoryOverhead"
+  /**
+   * built-in executor resource: pyspark.memory
+   */
   val PYSPARK_MEM = "pyspark.memory"
 
-  // all supported spark executor resources (minus the custom resources like GPUs/FPGAs)
-  val allSupportedExecutorResources = Seq(CORES, MEMORY, OVERHEAD_MEM, PYSPARK_MEM)
+  /**
+   * Return all supported Spark built-in executor resources, custom resources like GPUs/FPGAs
+   * are excluded.
+   */
+  def allSupportedExecutorResources: Array[String] =
+    Array(CORES, MEMORY, OVERHEAD_MEM, PYSPARK_MEM, OFFHEAP_MEM)
 
   val UNKNOWN_RESOURCE_PROFILE_ID = -1
   val DEFAULT_RESOURCE_PROFILE_ID = 0
 
+  private[spark] val MEMORY_OVERHEAD_MIN_MIB = 384L
+
   private lazy val nextProfileId = new AtomicInteger(0)
   private val DEFAULT_PROFILE_LOCK = new Object()
 
@@ -260,6 +297,7 @@ object ResourceProfile extends Logging {
   // var so that it can be reset for testing purposes.
   @GuardedBy("DEFAULT_PROFILE_LOCK")
   private var defaultProfile: Option[ResourceProfile] = None
+  private var defaultProfileExecutorResources: Option[DefaultProfileExecutorResources] = None
 
   private[spark] def getNextProfileId: Int = nextProfileId.getAndIncrement()
 
@@ -281,6 +319,14 @@ object ResourceProfile extends Logging {
     }
   }
 
+  private[spark] def getDefaultProfileExecutorResources(
+      conf: SparkConf): DefaultProfileExecutorResources = {
+    defaultProfileExecutorResources.getOrElse {
+      getOrCreateDefaultProfile(conf)
+      defaultProfileExecutorResources.get
+    }
+  }
+
   private def getDefaultTaskResources(conf: SparkConf): Map[String, TaskResourceRequest] = {
     val cpusPerTask = conf.get(CPUS_PER_TASK)
     val treqs = new TaskResourceRequests().cpus(cpusPerTask)
@@ -290,16 +336,26 @@ object ResourceProfile extends Logging {
 
   private def getDefaultExecutorResources(conf: SparkConf): Map[String, ExecutorResourceRequest] = {
     val ereqs = new ExecutorResourceRequests()
-    ereqs.cores(conf.get(EXECUTOR_CORES))
-    ereqs.memory(conf.get(EXECUTOR_MEMORY).toString)
-    conf.get(EXECUTOR_MEMORY_OVERHEAD).map(mem => ereqs.memoryOverhead(mem.toString))
-    conf.get(PYSPARK_EXECUTOR_MEMORY).map(mem => ereqs.pysparkMemory(mem.toString))
+    val cores = conf.get(EXECUTOR_CORES)
+    ereqs.cores(cores)
+    val memory = conf.get(EXECUTOR_MEMORY)
+    ereqs.memory(memory.toString)
+    val overheadMem = conf.get(EXECUTOR_MEMORY_OVERHEAD)
+    overheadMem.map(mem => ereqs.memoryOverhead(mem.toString))
+    val pysparkMem = conf.get(PYSPARK_EXECUTOR_MEMORY)
+    pysparkMem.map(mem => ereqs.pysparkMemory(mem.toString))
+    val offheapMem = Utils.executorOffHeapMemorySizeAsMb(conf)
+    ereqs.offHeapMemory(offheapMem.toString)
     val execReq = ResourceUtils.parseAllResourceRequests(conf, SPARK_EXECUTOR_PREFIX)
     execReq.foreach { req =>
-      val name = req.id.resourceName
-      ereqs.resource(name, req.amount, req.discoveryScript.orElse(""),
+      ereqs.resource(req.id.resourceName, req.amount, req.discoveryScript.orElse(""),
         req.vendor.orElse(""))
     }
+    val customResourceNames = execReq.map(_.id.resourceName).toSet
+    val customResources = ereqs.requests.filter(v => customResourceNames.contains(v._1))
+    defaultProfileExecutorResources =
+      Some(DefaultProfileExecutorResources(cores, memory, offheapMem, pysparkMem,
+        overheadMem, customResources))
     ereqs.requests
   }
 
@@ -313,17 +369,19 @@ object ResourceProfile extends Logging {
   private[spark] def clearDefaultProfile(): Unit = {
     DEFAULT_PROFILE_LOCK.synchronized {
       defaultProfile = None
+      defaultProfileExecutorResources = None
     }
   }
 
   private[spark] def getCustomTaskResources(
       rp: ResourceProfile): Map[String, TaskResourceRequest] = {
-    rp.taskResources.filterKeys(k => !k.equals(ResourceProfile.CPUS))
+    rp.taskResources.filterKeys(k => !k.equals(ResourceProfile.CPUS)).toMap
   }
 
   private[spark] def getCustomExecutorResources(
       rp: ResourceProfile): Map[String, ExecutorResourceRequest] = {
-    rp.executorResources.filterKeys(k => !ResourceProfile.allSupportedExecutorResources.contains(k))
+    rp.executorResources.
+      filterKeys(k => !ResourceProfile.allSupportedExecutorResources.contains(k)).toMap
   }
 
   /*
@@ -334,6 +392,100 @@ object ResourceProfile extends Logging {
     rp.getTaskCpus.getOrElse(conf.get(CPUS_PER_TASK))
   }
 
+  /**
+   * Get offHeap memory size from [[ExecutorResourceRequest]]
+   * return 0 if MEMORY_OFFHEAP_ENABLED is false.
+   */
+  private[spark] def executorOffHeapMemorySizeAsMb(sparkConf: SparkConf,
+      execRequest: ExecutorResourceRequest): Long = {
+    Utils.checkOffHeapEnabled(sparkConf, execRequest.amount)
+  }
+
+  private[spark] case class ExecutorResourcesOrDefaults(
+      cores: Int,
+      executorMemoryMiB: Long,
+      memoryOffHeapMiB: Long,
+      pysparkMemoryMiB: Long,
+      memoryOverheadMiB: Long,
+      totalMemMiB: Long,
+      customResources: Map[String, ExecutorResourceRequest])
+
+  private[spark] case class DefaultProfileExecutorResources(
+      cores: Int,
+      executorMemoryMiB: Long,
+      memoryOffHeapMiB: Long,
+      pysparkMemoryMiB: Option[Long],
+      memoryOverheadMiB: Option[Long],
+      customResources: Map[String, ExecutorResourceRequest])
+
+  private[spark] def calculateOverHeadMemory(
+      overHeadMemFromConf: Option[Long],
+      executorMemoryMiB: Long,
+      overheadFactor: Double): Long = {
+    overHeadMemFromConf.getOrElse(math.max((overheadFactor * executorMemoryMiB).toInt,
+        ResourceProfile.MEMORY_OVERHEAD_MIN_MIB))
+  }
+
+  /**
+   * Gets the full list of resources to allow a cluster manager to request the appropriate
+   * container. If the resource profile is not the default one we either get the resources
+   * specified in the profile or fall back to the default profile resource size for everything
+   * except for custom resources.
+   */
+  private[spark] def getResourcesForClusterManager(
+      rpId: Int,
+      execResources: Map[String, ExecutorResourceRequest],
+      overheadFactor: Double,
+      conf: SparkConf,
+      isPythonApp: Boolean,
+      resourceMappings: Map[String, String]): ExecutorResourcesOrDefaults = {
+    val defaultResources = getDefaultProfileExecutorResources(conf)
+    // set all the default values, which may change for custom ResourceProfiles
+    var cores = defaultResources.cores
+    var executorMemoryMiB = defaultResources.executorMemoryMiB
+    var memoryOffHeapMiB = defaultResources.memoryOffHeapMiB
+    var pysparkMemoryMiB = defaultResources.pysparkMemoryMiB.getOrElse(0L)
+    var memoryOverheadMiB = calculateOverHeadMemory(defaultResources.memoryOverheadMiB,
+      executorMemoryMiB, overheadFactor)
+
+    val finalCustomResources = if (rpId != DEFAULT_RESOURCE_PROFILE_ID) {
+      val customResources = new mutable.HashMap[String, ExecutorResourceRequest]
+      execResources.foreach { case (r, execReq) =>
+        r match {
+          case ResourceProfile.MEMORY =>
+            executorMemoryMiB = execReq.amount
+          case ResourceProfile.OVERHEAD_MEM =>
+            memoryOverheadMiB = execReq.amount
+          case ResourceProfile.PYSPARK_MEM =>
+            pysparkMemoryMiB = execReq.amount
+          case ResourceProfile.OFFHEAP_MEM =>
+            memoryOffHeapMiB = executorOffHeapMemorySizeAsMb(conf, execReq)
+          case ResourceProfile.CORES =>
+            cores = execReq.amount.toInt
+          case rName =>
+            val nameToUse = resourceMappings.get(rName).getOrElse(rName)
+            customResources(nameToUse) = execReq
+        }
+      }
+      customResources.toMap
+    } else {
+      defaultResources.customResources.map { case (rName, execReq) =>
+        val nameToUse = resourceMappings.get(rName).getOrElse(rName)
+        (nameToUse, execReq)
+      }
+    }
+    // only add in pyspark memory if actually a python application
+    val pysparkMemToUseMiB = if (isPythonApp) {
+      pysparkMemoryMiB
+    } else {
+      0L
+    }
+    val totalMemMiB =
+      (executorMemoryMiB + memoryOverheadMiB + memoryOffHeapMiB + pysparkMemToUseMiB)
+    ExecutorResourcesOrDefaults(cores, executorMemoryMiB, memoryOffHeapMiB,
+      pysparkMemToUseMiB, memoryOverheadMiB, totalMemMiB, finalCustomResources)
+  }
+
   private[spark] val PYSPARK_MEMORY_LOCAL_PROPERTY = "resource.pyspark.memory"
   private[spark] val EXECUTOR_CORES_LOCAL_PROPERTY = "resource.executor.cores"
 }
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceProfileBuilder.scala b/core/src/main/scala/org/apache/spark/resource/ResourceProfileBuilder.scala
index 26f23f4bf0476..f6b30d327375e 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceProfileBuilder.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceProfileBuilder.scala
@@ -22,18 +22,23 @@ import java.util.concurrent.ConcurrentHashMap
 
 import scala.collection.JavaConverters._
 
-import org.apache.spark.annotation.Evolving
+import org.apache.spark.annotation.{Evolving, Since}
+
 
 /**
- * Resource profile builder to build a Resource profile to associate with an RDD.
- * A ResourceProfile allows the user to specify executor and task requirements for an RDD
- * that will get applied during a stage. This allows the user to change the resource
+ * Resource profile builder to build a [[ResourceProfile]] to associate with an RDD.
+ * A [[ResourceProfile]] allows the user to specify executor and task resource requirements
+ * for an RDD that will get applied during a stage. This allows the user to change the resource
  * requirements between stages.
+ *
  */
 @Evolving
-private[spark] class ResourceProfileBuilder() {
+@Since("3.1.0")
+class ResourceProfileBuilder() {
 
+  // Task resource requests specified by users, mapped from resource name to the request.
   private val _taskResources = new ConcurrentHashMap[String, TaskResourceRequest]()
+  // Executor resource requests specified by users, mapped from resource name to the request.
   private val _executorResources = new ConcurrentHashMap[String, ExecutorResourceRequest]()
 
   def taskResources: Map[String, TaskResourceRequest] = _taskResources.asScala.toMap
@@ -51,11 +56,21 @@ private[spark] class ResourceProfileBuilder() {
     _executorResources.asScala.asJava
   }
 
+  /**
+   * Add executor resource requests
+   * @param requests The detailed executor resource requests, see [[ExecutorResourceRequests]]
+   * @return This ResourceProfileBuilder
+   */
   def require(requests: ExecutorResourceRequests): this.type = {
     _executorResources.putAll(requests.requests.asJava)
     this
   }
 
+  /**
+   * Add task resource requests
+   * @param requests The detailed task resource requests, see [[TaskResourceRequest]]
+   * @return This ResourceProfileBuilder
+   */
   def require(requests: TaskResourceRequests): this.type = {
     _taskResources.putAll(requests.requests.asJava)
     this
@@ -77,7 +92,7 @@ private[spark] class ResourceProfileBuilder() {
       s"task resources: ${_taskResources.asScala.map(pair => s"${pair._1}=${pair._2.toString()}")}"
   }
 
-  def build: ResourceProfile = {
+  def build(): ResourceProfile = {
     new ResourceProfile(executorResources, taskResources)
   }
 }
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceProfileManager.scala b/core/src/main/scala/org/apache/spark/resource/ResourceProfileManager.scala
index c3e244474a692..d538f0bcc423e 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceProfileManager.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceProfileManager.scala
@@ -25,17 +25,19 @@ import org.apache.spark.{SparkConf, SparkException}
 import org.apache.spark.annotation.Evolving
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.Tests._
+import org.apache.spark.scheduler.{LiveListenerBus, SparkListenerResourceProfileAdded}
 import org.apache.spark.util.Utils
 import org.apache.spark.util.Utils.isTesting
 
 /**
  * Manager of resource profiles. The manager allows one place to keep the actual ResourceProfiles
  * and everywhere else we can use the ResourceProfile Id to save on space.
- * Note we never remove a resource profile at this point. Its expected this number if small
+ * Note we never remove a resource profile at this point. Its expected this number is small
  * so this shouldn't be much overhead.
  */
 @Evolving
-private[spark] class ResourceProfileManager(sparkConf: SparkConf) extends Logging {
+private[spark] class ResourceProfileManager(sparkConf: SparkConf,
+    listenerBus: LiveListenerBus) extends Logging {
   private val resourceProfileIdToResourceProfile = new HashMap[Int, ResourceProfile]()
 
   private val (readLock, writeLock) = {
@@ -50,18 +52,25 @@ private[spark] class ResourceProfileManager(sparkConf: SparkConf) extends Loggin
 
   private val dynamicEnabled = Utils.isDynamicAllocationEnabled(sparkConf)
   private val master = sparkConf.getOption("spark.master")
-  private val isNotYarn = master.isDefined && !master.get.equals("yarn")
-  private val errorForTesting = !isTesting || sparkConf.get(RESOURCE_PROFILE_MANAGER_TESTING)
+  private val isYarn = master.isDefined && master.get.equals("yarn")
+  private val isK8s = master.isDefined && master.get.startsWith("k8s://")
+  private val notRunningUnitTests = !isTesting
+  private val testExceptionThrown = sparkConf.get(RESOURCE_PROFILE_MANAGER_TESTING)
 
   // If we use anything except the default profile, its only supported on YARN right now.
   // Throw an exception if not supported.
   private[spark] def isSupported(rp: ResourceProfile): Boolean = {
     val isNotDefaultProfile = rp.id != ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID
-    val notYarnAndNotDefaultProfile = isNotDefaultProfile && isNotYarn
-    val YarnNotDynAllocAndNotDefaultProfile = isNotDefaultProfile && !isNotYarn && !dynamicEnabled
-    if (errorForTesting && (notYarnAndNotDefaultProfile || YarnNotDynAllocAndNotDefaultProfile)) {
-      throw new SparkException("ResourceProfiles are only supported on YARN with dynamic " +
-        "allocation enabled.")
+    val notYarnOrK8sAndNotDefaultProfile = isNotDefaultProfile && !(isYarn || isK8s)
+    val YarnOrK8sNotDynAllocAndNotDefaultProfile =
+      isNotDefaultProfile && (isYarn || isK8s) && !dynamicEnabled
+    // We want the exception to be thrown only when we are specifically testing for the
+    // exception or in a real application. Otherwise in all other testing scenarios we want
+    // to skip throwing the exception so that we can test in other modes to make testing easier.
+    if ((notRunningUnitTests || testExceptionThrown) &&
+        (notYarnOrK8sAndNotDefaultProfile || YarnOrK8sNotDynAllocAndNotDefaultProfile)) {
+      throw new SparkException("ResourceProfiles are only supported on YARN and Kubernetes " +
+        "with dynamic allocation enabled.")
     }
     true
   }
@@ -83,6 +92,7 @@ private[spark] class ResourceProfileManager(sparkConf: SparkConf) extends Loggin
       // force the computation of maxTasks and limitingResource now so we don't have cost later
       rp.limitingResource(sparkConf)
       logInfo(s"Added ResourceProfile id: ${rp.id}")
+      listenerBus.post(SparkListenerResourceProfileAdded(rp))
     }
   }
 
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala b/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala
index 162f090f011c6..837b2d80aace6 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala
@@ -29,8 +29,8 @@ import org.apache.spark.{SparkConf, SparkException}
 import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.api.resource.ResourceDiscoveryPlugin
 import org.apache.spark.internal.Logging
-import org.apache.spark.internal.config.{CPUS_PER_TASK, EXECUTOR_CORES, RESOURCES_DISCOVERY_PLUGIN, SPARK_TASK_PREFIX}
-import org.apache.spark.internal.config.Tests.{RESOURCES_WARNING_TESTING, SKIP_VALIDATE_CORES_TESTING}
+import org.apache.spark.internal.config.{EXECUTOR_CORES, RESOURCES_DISCOVERY_PLUGIN, SPARK_TASK_PREFIX}
+import org.apache.spark.internal.config.Tests.{RESOURCES_WARNING_TESTING}
 import org.apache.spark.util.Utils
 
 /**
@@ -149,7 +149,12 @@ private[spark] object ResourceUtils extends Logging {
 
   def listResourceIds(sparkConf: SparkConf, componentName: String): Seq[ResourceID] = {
     sparkConf.getAllWithPrefix(s"$componentName.$RESOURCE_PREFIX.").map { case (key, _) =>
-      key.substring(0, key.indexOf('.'))
+      val index = key.indexOf('.')
+      if (index < 0) {
+        throw new SparkException(s"You must specify an amount config for resource: $key " +
+          s"config: $componentName.$RESOURCE_PREFIX.$key")
+      }
+      key.substring(0, index)
     }.distinct.map(name => new ResourceID(componentName, name))
   }
 
diff --git a/core/src/main/scala/org/apache/spark/resource/TaskResourceRequest.scala b/core/src/main/scala/org/apache/spark/resource/TaskResourceRequest.scala
index bffb0a2f523b1..cbd57808213aa 100644
--- a/core/src/main/scala/org/apache/spark/resource/TaskResourceRequest.scala
+++ b/core/src/main/scala/org/apache/spark/resource/TaskResourceRequest.scala
@@ -17,17 +17,24 @@
 
 package org.apache.spark.resource
 
+import org.apache.spark.annotation.{Evolving, Since}
+
 /**
- * A task resource request. This is used in conjuntion with the ResourceProfile to
+ * A task resource request. This is used in conjunction with the [[ResourceProfile]] to
  * programmatically specify the resources needed for an RDD that will be applied at the
  * stage level.
  *
- * Use TaskResourceRequests class as a convenience API.
+ * Use [[TaskResourceRequests]] class as a convenience API.
  *
- * This api is currently private until the rest of the pieces are in place and then it
- * will become public.
+ * @param resourceName Resource name
+ * @param amount Amount requesting as a Double to support fractional resource requests.
+ *               Valid values are less than or equal to 0.5 or whole numbers. This essentially
+ *               lets you configure X number of tasks to run on a single resource,
+ *               ie amount equals 0.5 translates into 2 tasks per resource address.
  */
-private[spark] class TaskResourceRequest(val resourceName: String, val amount: Double)
+@Evolving
+@Since("3.1.0")
+class TaskResourceRequest(val resourceName: String, val amount: Double)
   extends Serializable {
 
   assert(amount <= 0.5 || amount % 1 == 0,
diff --git a/core/src/main/scala/org/apache/spark/resource/TaskResourceRequests.scala b/core/src/main/scala/org/apache/spark/resource/TaskResourceRequests.scala
index 09f4e02eee9e0..1d5fc73a152f6 100644
--- a/core/src/main/scala/org/apache/spark/resource/TaskResourceRequests.scala
+++ b/core/src/main/scala/org/apache/spark/resource/TaskResourceRequests.scala
@@ -22,26 +22,33 @@ import java.util.concurrent.ConcurrentHashMap
 
 import scala.collection.JavaConverters._
 
+import org.apache.spark.annotation.{Evolving, Since}
 import org.apache.spark.resource.ResourceProfile._
 
 /**
  * A set of task resource requests. This is used in conjunction with the ResourceProfile to
  * programmatically specify the resources needed for an RDD that will be applied at the
  * stage level.
- *
- * This api is currently private until the rest of the pieces are in place and then it
- * will become public.
  */
-private[spark] class TaskResourceRequests() extends Serializable {
+@Evolving
+@Since("3.1.0")
+class TaskResourceRequests() extends Serializable {
 
   private val _taskResources = new ConcurrentHashMap[String, TaskResourceRequest]()
 
+  /**
+   * Returns all the resource requests for the task.
+   */
   def requests: Map[String, TaskResourceRequest] = _taskResources.asScala.toMap
 
+  /**
+   * (Java-specific) Returns all the resource requests for the task.
+   */
   def requestsJMap: JMap[String, TaskResourceRequest] = requests.asJava
 
   /**
    * Specify number of cpus per Task.
+   * This is a convenient API to add [[TaskResourceRequest]] for cpus.
    *
    * @param amount Number of cpus to allocate per Task.
    */
@@ -52,7 +59,8 @@ private[spark] class TaskResourceRequests() extends Serializable {
   }
 
   /**
-   *  Amount of a particular custom resource(GPU, FPGA, etc) to use.
+   * Amount of a particular custom resource(GPU, FPGA, etc) to use.
+   * This is a convenient API to add [[TaskResourceRequest]] for custom resources.
    *
    * @param resourceName Name of the resource.
    * @param amount Amount requesting as a Double to support fractional resource requests.
@@ -66,6 +74,9 @@ private[spark] class TaskResourceRequests() extends Serializable {
     this
   }
 
+  /**
+   * Add a certain [[TaskResourceRequest]] to the request set.
+   */
   def addRequest(treq: TaskResourceRequest): this.type = {
     _taskResources.put(treq.resourceName, treq)
     this
diff --git a/core/src/main/scala/org/apache/spark/rpc/netty/Dispatcher.scala b/core/src/main/scala/org/apache/spark/rpc/netty/Dispatcher.scala
index 41d6d146a86d7..14198743c4801 100644
--- a/core/src/main/scala/org/apache/spark/rpc/netty/Dispatcher.scala
+++ b/core/src/main/scala/org/apache/spark/rpc/netty/Dispatcher.scala
@@ -24,7 +24,7 @@ import scala.collection.JavaConverters._
 import scala.concurrent.Promise
 import scala.util.control.NonFatal
 
-import org.apache.spark.SparkException
+import org.apache.spark.{SparkEnv, SparkException}
 import org.apache.spark.internal.Logging
 import org.apache.spark.network.client.RpcResponseCallback
 import org.apache.spark.rpc._
@@ -122,7 +122,7 @@ private[netty] class Dispatcher(nettyEnv: NettyRpcEnv, numUsableCores: Int) exte
     while (iter.hasNext) {
       val name = iter.next
         postMessage(name, message, (e) => { e match {
-          case e: RpcEnvStoppedException => logDebug (s"Message $message dropped. ${e.getMessage}")
+          case e: RpcEnvStoppedException => logDebug(s"Message $message dropped. ${e.getMessage}")
           case e: Throwable => logWarning(s"Message $message dropped. ${e.getMessage}")
         }}
       )}
@@ -147,7 +147,16 @@ private[netty] class Dispatcher(nettyEnv: NettyRpcEnv, numUsableCores: Int) exte
   /** Posts a one-way message. */
   def postOneWayMessage(message: RequestMessage): Unit = {
     postMessage(message.receiver.name, OneWayMessage(message.senderAddress, message.content),
-      (e) => throw e)
+      {
+        // SPARK-31922: in local cluster mode, there's always a RpcEnvStoppedException when
+        // stop is called due to some asynchronous message handling. We catch the exception
+        // and log it at debug level to avoid verbose error message when user stop a local
+        // cluster in spark shell.
+        case re: RpcEnvStoppedException => logDebug(s"Message $message dropped. ${re.getMessage}")
+        case e if SparkEnv.get.isStopped =>
+          logWarning(s"Message $message dropped due to sparkEnv is stopped. ${e.getMessage}")
+        case e => throw e
+      })
   }
 
   /**
diff --git a/core/src/main/scala/org/apache/spark/rpc/netty/Inbox.scala b/core/src/main/scala/org/apache/spark/rpc/netty/Inbox.scala
index 2ed03f7430c32..472401b23fe8e 100644
--- a/core/src/main/scala/org/apache/spark/rpc/netty/Inbox.scala
+++ b/core/src/main/scala/org/apache/spark/rpc/netty/Inbox.scala
@@ -200,6 +200,16 @@ private[netty] class Inbox(val endpointName: String, val endpoint: RpcEndpoint)
    * Calls action closure, and calls the endpoint's onError function in the case of exceptions.
    */
   private def safelyCall(endpoint: RpcEndpoint)(action: => Unit): Unit = {
+    def dealWithFatalError(fatal: Throwable): Unit = {
+      inbox.synchronized {
+        assert(numActiveThreads > 0, "The number of active threads should be positive.")
+        // Should reduce the number of active threads before throw the error.
+        numActiveThreads -= 1
+      }
+      logError(s"An error happened while processing message in the inbox for $endpointName", fatal)
+      throw fatal
+    }
+
     try action catch {
       case NonFatal(e) =>
         try endpoint.onError(e) catch {
@@ -209,8 +219,18 @@ private[netty] class Inbox(val endpointName: String, val endpoint: RpcEndpoint)
             } else {
               logError("Ignoring error", ee)
             }
+          case fatal: Throwable =>
+            dealWithFatalError(fatal)
         }
+      case fatal: Throwable =>
+        dealWithFatalError(fatal)
     }
   }
 
+  // exposed only for testing
+  def getNumActiveThreads: Int = {
+    inbox.synchronized {
+      inbox.numActiveThreads
+    }
+  }
 }
diff --git a/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala b/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala
index 3886cc5baa48e..5864e9e2ceac0 100644
--- a/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala
+++ b/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala
@@ -213,7 +213,7 @@ private[netty] class NettyRpcEnv(
     def onFailure(e: Throwable): Unit = {
       if (!promise.tryFailure(e)) {
         e match {
-          case e : RpcEnvStoppedException => logDebug (s"Ignored failure: $e")
+          case e : RpcEnvStoppedException => logDebug(s"Ignored failure: $e")
           case _ => logWarning(s"Ignored failure: $e")
         }
       }
@@ -254,14 +254,14 @@ private[netty] class NettyRpcEnv(
 
       val timeoutCancelable = timeoutScheduler.schedule(new Runnable {
         override def run(): Unit = {
-          val remoteReceAddr = if (remoteAddr == null) {
+          val remoteRecAddr = if (remoteAddr == null) {
             Try {
               message.receiver.client.getChannel.remoteAddress()
             }.toOption.orNull
           } else {
             remoteAddr
           }
-          onFailure(new TimeoutException(s"Cannot receive any reply from ${remoteReceAddr} " +
+          onFailure(new TimeoutException(s"Cannot receive any reply from ${remoteRecAddr} " +
             s"in ${timeout.duration}"))
         }
       }, timeout.duration.toNanos, TimeUnit.NANOSECONDS)
diff --git a/core/src/main/scala/org/apache/spark/scheduler/BarrierJobAllocationFailed.scala b/core/src/main/scala/org/apache/spark/scheduler/BarrierJobAllocationFailed.scala
index 2274e6898adf6..8f0764ed1a61e 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/BarrierJobAllocationFailed.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/BarrierJobAllocationFailed.scala
@@ -45,10 +45,10 @@ private[spark] object BarrierJobAllocationFailed {
   val ERROR_MESSAGE_RUN_BARRIER_WITH_UNSUPPORTED_RDD_CHAIN_PATTERN =
     "[SPARK-24820][SPARK-24821]: Barrier execution mode does not allow the following pattern of " +
       "RDD chain within a barrier stage:\n1. Ancestor RDDs that have different number of " +
-      "partitions from the resulting RDD (eg. union()/coalesce()/first()/take()/" +
+      "partitions from the resulting RDD (e.g. union()/coalesce()/first()/take()/" +
       "PartitionPruningRDD). A workaround for first()/take() can be barrierRdd.collect().head " +
       "(scala) or barrierRdd.collect()[0] (python).\n" +
-      "2. An RDD that depends on multiple barrier RDDs (eg. barrierRdd1.zip(barrierRdd2))."
+      "2. An RDD that depends on multiple barrier RDDs (e.g. barrierRdd1.zip(barrierRdd2))."
 
   // Error message when running a barrier stage with dynamic resource allocation enabled.
   val ERROR_MESSAGE_RUN_BARRIER_WITH_DYN_ALLOCATION =
@@ -60,6 +60,6 @@ private[spark] object BarrierJobAllocationFailed {
   val ERROR_MESSAGE_BARRIER_REQUIRE_MORE_SLOTS_THAN_CURRENT_TOTAL_NUMBER =
     "[SPARK-24819]: Barrier execution mode does not allow run a barrier stage that requires " +
       "more slots than the total number of slots in the cluster currently. Please init a new " +
-      "cluster with more CPU cores or repartition the input RDD(s) to reduce the number of " +
-      "slots required to run this barrier stage."
+      "cluster with more resources(e.g. CPU, GPU) or repartition the input RDD(s) to reduce " +
+      "the number of slots required to run this barrier stage."
 }
diff --git a/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala b/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala
deleted file mode 100644
index 9e524c52267be..0000000000000
--- a/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala
+++ /dev/null
@@ -1,477 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.scheduler
-
-import java.util.concurrent.atomic.AtomicReference
-
-import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet}
-
-import org.apache.spark.{ExecutorAllocationClient, SparkConf, SparkContext}
-import org.apache.spark.internal.Logging
-import org.apache.spark.internal.config
-import org.apache.spark.util.{Clock, SystemClock, Utils}
-
-/**
- * BlacklistTracker is designed to track problematic executors and nodes.  It supports blacklisting
- * executors and nodes across an entire application (with a periodic expiry).  TaskSetManagers add
- * additional blacklisting of executors and nodes for individual tasks and stages which works in
- * concert with the blacklisting here.
- *
- * The tracker needs to deal with a variety of workloads, eg.:
- *
- *  * bad user code --  this may lead to many task failures, but that should not count against
- *      individual executors
- *  * many small stages -- this may prevent a bad executor for having many failures within one
- *      stage, but still many failures over the entire application
- *  * "flaky" executors -- they don't fail every task, but are still faulty enough to merit
- *      blacklisting
- *
- * See the design doc on SPARK-8425 for a more in-depth discussion.
- *
- * THREADING: As with most helpers of TaskSchedulerImpl, this is not thread-safe.  Though it is
- * called by multiple threads, callers must already have a lock on the TaskSchedulerImpl.  The
- * one exception is [[nodeBlacklist()]], which can be called without holding a lock.
- */
-private[scheduler] class BlacklistTracker (
-    private val listenerBus: LiveListenerBus,
-    conf: SparkConf,
-    allocationClient: Option[ExecutorAllocationClient],
-    clock: Clock = new SystemClock()) extends Logging {
-
-  def this(sc: SparkContext, allocationClient: Option[ExecutorAllocationClient]) = {
-    this(sc.listenerBus, sc.conf, allocationClient)
-  }
-
-  BlacklistTracker.validateBlacklistConfs(conf)
-  private val MAX_FAILURES_PER_EXEC = conf.get(config.MAX_FAILURES_PER_EXEC)
-  private val MAX_FAILED_EXEC_PER_NODE = conf.get(config.MAX_FAILED_EXEC_PER_NODE)
-  val BLACKLIST_TIMEOUT_MILLIS = BlacklistTracker.getBlacklistTimeout(conf)
-  private val BLACKLIST_FETCH_FAILURE_ENABLED = conf.get(config.BLACKLIST_FETCH_FAILURE_ENABLED)
-
-  /**
-   * A map from executorId to information on task failures.  Tracks the time of each task failure,
-   * so that we can avoid blacklisting executors due to failures that are very far apart.  We do not
-   * actively remove from this as soon as tasks hit their timeouts, to avoid the time it would take
-   * to do so.  But it will not grow too large, because as soon as an executor gets too many
-   * failures, we blacklist the executor and remove its entry here.
-   */
-  private val executorIdToFailureList = new HashMap[String, ExecutorFailureList]()
-  val executorIdToBlacklistStatus = new HashMap[String, BlacklistedExecutor]()
-  val nodeIdToBlacklistExpiryTime = new HashMap[String, Long]()
-  /**
-   * An immutable copy of the set of nodes that are currently blacklisted.  Kept in an
-   * AtomicReference to make [[nodeBlacklist()]] thread-safe.
-   */
-  private val _nodeBlacklist = new AtomicReference[Set[String]](Set())
-  /**
-   * Time when the next blacklist will expire.  Used as a
-   * shortcut to avoid iterating over all entries in the blacklist when none will have expired.
-   */
-  var nextExpiryTime: Long = Long.MaxValue
-  /**
-   * Mapping from nodes to all of the executors that have been blacklisted on that node. We do *not*
-   * remove from this when executors are removed from spark, so we can track when we get multiple
-   * successive blacklisted executors on one node.  Nonetheless, it will not grow too large because
-   * there cannot be many blacklisted executors on one node, before we stop requesting more
-   * executors on that node, and we clean up the list of blacklisted executors once an executor has
-   * been blacklisted for BLACKLIST_TIMEOUT_MILLIS.
-   */
-  val nodeToBlacklistedExecs = new HashMap[String, HashSet[String]]()
-
-  /**
-   * Un-blacklists executors and nodes that have been blacklisted for at least
-   * BLACKLIST_TIMEOUT_MILLIS
-   */
-  def applyBlacklistTimeout(): Unit = {
-    val now = clock.getTimeMillis()
-    // quickly check if we've got anything to expire from blacklist -- if not, avoid doing any work
-    if (now > nextExpiryTime) {
-      // Apply the timeout to blacklisted nodes and executors
-      val execsToUnblacklist = executorIdToBlacklistStatus.filter(_._2.expiryTime < now).keys
-      if (execsToUnblacklist.nonEmpty) {
-        // Un-blacklist any executors that have been blacklisted longer than the blacklist timeout.
-        logInfo(s"Removing executors $execsToUnblacklist from blacklist because the blacklist " +
-          s"for those executors has timed out")
-        execsToUnblacklist.foreach { exec =>
-          val status = executorIdToBlacklistStatus.remove(exec).get
-          val failedExecsOnNode = nodeToBlacklistedExecs(status.node)
-          listenerBus.post(SparkListenerExecutorUnblacklisted(now, exec))
-          failedExecsOnNode.remove(exec)
-          if (failedExecsOnNode.isEmpty) {
-            nodeToBlacklistedExecs.remove(status.node)
-          }
-        }
-      }
-      val nodesToUnblacklist = nodeIdToBlacklistExpiryTime.filter(_._2 < now).keys
-      if (nodesToUnblacklist.nonEmpty) {
-        // Un-blacklist any nodes that have been blacklisted longer than the blacklist timeout.
-        logInfo(s"Removing nodes $nodesToUnblacklist from blacklist because the blacklist " +
-          s"has timed out")
-        nodesToUnblacklist.foreach { node =>
-          nodeIdToBlacklistExpiryTime.remove(node)
-          listenerBus.post(SparkListenerNodeUnblacklisted(now, node))
-        }
-        _nodeBlacklist.set(nodeIdToBlacklistExpiryTime.keySet.toSet)
-      }
-      updateNextExpiryTime()
-    }
-  }
-
-  private def updateNextExpiryTime(): Unit = {
-    val execMinExpiry = if (executorIdToBlacklistStatus.nonEmpty) {
-      executorIdToBlacklistStatus.map{_._2.expiryTime}.min
-    } else {
-      Long.MaxValue
-    }
-    val nodeMinExpiry = if (nodeIdToBlacklistExpiryTime.nonEmpty) {
-      nodeIdToBlacklistExpiryTime.values.min
-    } else {
-      Long.MaxValue
-    }
-    nextExpiryTime = math.min(execMinExpiry, nodeMinExpiry)
-  }
-
-  private def killExecutor(exec: String, msg: String): Unit = {
-    allocationClient match {
-      case Some(a) =>
-        logInfo(msg)
-        a.killExecutors(Seq(exec), adjustTargetNumExecutors = false, countFailures = false,
-          force = true)
-      case None =>
-        logInfo(s"Not attempting to kill blacklisted executor id $exec " +
-          s"since allocation client is not defined.")
-    }
-  }
-
-  private def killBlacklistedExecutor(exec: String): Unit = {
-    if (conf.get(config.BLACKLIST_KILL_ENABLED)) {
-      killExecutor(exec,
-        s"Killing blacklisted executor id $exec since ${config.BLACKLIST_KILL_ENABLED.key} is set.")
-    }
-  }
-
-  private[scheduler] def killBlacklistedIdleExecutor(exec: String): Unit = {
-    killExecutor(exec,
-      s"Killing blacklisted idle executor id $exec because of task unschedulability and trying " +
-        "to acquire a new executor.")
-  }
-
-  private def killExecutorsOnBlacklistedNode(node: String): Unit = {
-    if (conf.get(config.BLACKLIST_KILL_ENABLED)) {
-      allocationClient match {
-        case Some(a) =>
-          logInfo(s"Killing all executors on blacklisted host $node " +
-            s"since ${config.BLACKLIST_KILL_ENABLED.key} is set.")
-          if (a.killExecutorsOnHost(node) == false) {
-            logError(s"Killing executors on node $node failed.")
-          }
-        case None =>
-          logWarning(s"Not attempting to kill executors on blacklisted host $node " +
-            s"since allocation client is not defined.")
-      }
-    }
-  }
-
-  def updateBlacklistForFetchFailure(host: String, exec: String): Unit = {
-    if (BLACKLIST_FETCH_FAILURE_ENABLED) {
-      // If we blacklist on fetch failures, we are implicitly saying that we believe the failure is
-      // non-transient, and can't be recovered from (even if this is the first fetch failure,
-      // stage is retried after just one failure, so we don't always get a chance to collect
-      // multiple fetch failures).
-      // If the external shuffle-service is on, then every other executor on this node would
-      // be suffering from the same issue, so we should blacklist (and potentially kill) all
-      // of them immediately.
-
-      val now = clock.getTimeMillis()
-      val expiryTimeForNewBlacklists = now + BLACKLIST_TIMEOUT_MILLIS
-
-      if (conf.get(config.SHUFFLE_SERVICE_ENABLED)) {
-        if (!nodeIdToBlacklistExpiryTime.contains(host)) {
-          logInfo(s"blacklisting node $host due to fetch failure of external shuffle service")
-
-          nodeIdToBlacklistExpiryTime.put(host, expiryTimeForNewBlacklists)
-          listenerBus.post(SparkListenerNodeBlacklisted(now, host, 1))
-          _nodeBlacklist.set(nodeIdToBlacklistExpiryTime.keySet.toSet)
-          killExecutorsOnBlacklistedNode(host)
-          updateNextExpiryTime()
-        }
-      } else if (!executorIdToBlacklistStatus.contains(exec)) {
-        logInfo(s"Blacklisting executor $exec due to fetch failure")
-
-        executorIdToBlacklistStatus.put(exec, BlacklistedExecutor(host, expiryTimeForNewBlacklists))
-        // We hardcoded number of failure tasks to 1 for fetch failure, because there's no
-        // reattempt for such failure.
-        listenerBus.post(SparkListenerExecutorBlacklisted(now, exec, 1))
-        updateNextExpiryTime()
-        killBlacklistedExecutor(exec)
-
-        val blacklistedExecsOnNode = nodeToBlacklistedExecs.getOrElseUpdate(host, HashSet[String]())
-        blacklistedExecsOnNode += exec
-      }
-    }
-  }
-
-  def updateBlacklistForSuccessfulTaskSet(
-      stageId: Int,
-      stageAttemptId: Int,
-      failuresByExec: HashMap[String, ExecutorFailuresInTaskSet]): Unit = {
-    // if any tasks failed, we count them towards the overall failure count for the executor at
-    // this point.
-    val now = clock.getTimeMillis()
-    failuresByExec.foreach { case (exec, failuresInTaskSet) =>
-      val appFailuresOnExecutor =
-        executorIdToFailureList.getOrElseUpdate(exec, new ExecutorFailureList)
-      appFailuresOnExecutor.addFailures(stageId, stageAttemptId, failuresInTaskSet)
-      appFailuresOnExecutor.dropFailuresWithTimeoutBefore(now)
-      val newTotal = appFailuresOnExecutor.numUniqueTaskFailures
-
-      val expiryTimeForNewBlacklists = now + BLACKLIST_TIMEOUT_MILLIS
-      // If this pushes the total number of failures over the threshold, blacklist the executor.
-      // If its already blacklisted, we avoid "re-blacklisting" (which can happen if there were
-      // other tasks already running in another taskset when it got blacklisted), because it makes
-      // some of the logic around expiry times a little more confusing.  But it also wouldn't be a
-      // problem to re-blacklist, with a later expiry time.
-      if (newTotal >= MAX_FAILURES_PER_EXEC && !executorIdToBlacklistStatus.contains(exec)) {
-        logInfo(s"Blacklisting executor id: $exec because it has $newTotal" +
-          s" task failures in successful task sets")
-        val node = failuresInTaskSet.node
-        executorIdToBlacklistStatus.put(exec, BlacklistedExecutor(node, expiryTimeForNewBlacklists))
-        listenerBus.post(SparkListenerExecutorBlacklisted(now, exec, newTotal))
-        executorIdToFailureList.remove(exec)
-        updateNextExpiryTime()
-        killBlacklistedExecutor(exec)
-
-        // In addition to blacklisting the executor, we also update the data for failures on the
-        // node, and potentially put the entire node into a blacklist as well.
-        val blacklistedExecsOnNode = nodeToBlacklistedExecs.getOrElseUpdate(node, HashSet[String]())
-        blacklistedExecsOnNode += exec
-        // If the node is already in the blacklist, we avoid adding it again with a later expiry
-        // time.
-        if (blacklistedExecsOnNode.size >= MAX_FAILED_EXEC_PER_NODE &&
-            !nodeIdToBlacklistExpiryTime.contains(node)) {
-          logInfo(s"Blacklisting node $node because it has ${blacklistedExecsOnNode.size} " +
-            s"executors blacklisted: ${blacklistedExecsOnNode}")
-          nodeIdToBlacklistExpiryTime.put(node, expiryTimeForNewBlacklists)
-          listenerBus.post(SparkListenerNodeBlacklisted(now, node, blacklistedExecsOnNode.size))
-          _nodeBlacklist.set(nodeIdToBlacklistExpiryTime.keySet.toSet)
-          killExecutorsOnBlacklistedNode(node)
-        }
-      }
-    }
-  }
-
-  def isExecutorBlacklisted(executorId: String): Boolean = {
-    executorIdToBlacklistStatus.contains(executorId)
-  }
-
-  /**
-   * Get the full set of nodes that are blacklisted.  Unlike other methods in this class, this *IS*
-   * thread-safe -- no lock required on a taskScheduler.
-   */
-  def nodeBlacklist(): Set[String] = {
-    _nodeBlacklist.get()
-  }
-
-  def isNodeBlacklisted(node: String): Boolean = {
-    nodeIdToBlacklistExpiryTime.contains(node)
-  }
-
-  def handleRemovedExecutor(executorId: String): Unit = {
-    // We intentionally do not clean up executors that are already blacklisted in
-    // nodeToBlacklistedExecs, so that if another executor on the same node gets blacklisted, we can
-    // blacklist the entire node.  We also can't clean up executorIdToBlacklistStatus, so we can
-    // eventually remove the executor after the timeout.  Despite not clearing those structures
-    // here, we don't expect they will grow too big since you won't get too many executors on one
-    // node, and the timeout will clear it up periodically in any case.
-    executorIdToFailureList -= executorId
-  }
-
-
-  /**
-   * Tracks all failures for one executor (that have not passed the timeout).
-   *
-   * In general we actually expect this to be extremely small, since it won't contain more than the
-   * maximum number of task failures before an executor is failed (default 2).
-   */
-  private[scheduler] final class ExecutorFailureList extends Logging {
-
-    private case class TaskId(stage: Int, stageAttempt: Int, taskIndex: Int)
-
-    /**
-     * All failures on this executor in successful task sets.
-     */
-    private var failuresAndExpiryTimes = ArrayBuffer[(TaskId, Long)]()
-    /**
-     * As an optimization, we track the min expiry time over all entries in failuresAndExpiryTimes
-     * so its quick to tell if there are any failures with expiry before the current time.
-     */
-    private var minExpiryTime = Long.MaxValue
-
-    def addFailures(
-        stage: Int,
-        stageAttempt: Int,
-        failuresInTaskSet: ExecutorFailuresInTaskSet): Unit = {
-      failuresInTaskSet.taskToFailureCountAndFailureTime.foreach {
-        case (taskIdx, (_, failureTime)) =>
-          val expiryTime = failureTime + BLACKLIST_TIMEOUT_MILLIS
-          failuresAndExpiryTimes += ((TaskId(stage, stageAttempt, taskIdx), expiryTime))
-          if (expiryTime < minExpiryTime) {
-            minExpiryTime = expiryTime
-          }
-      }
-    }
-
-    /**
-     * The number of unique tasks that failed on this executor.  Only counts failures within the
-     * timeout, and in successful tasksets.
-     */
-    def numUniqueTaskFailures: Int = failuresAndExpiryTimes.size
-
-    def isEmpty: Boolean = failuresAndExpiryTimes.isEmpty
-
-    /**
-     * Apply the timeout to individual tasks.  This is to prevent one-off failures that are very
-     * spread out in time (and likely have nothing to do with problems on the executor) from
-     * triggering blacklisting.  However, note that we do *not* remove executors and nodes from
-     * the blacklist as we expire individual task failures -- each have their own timeout.  Eg.,
-     * suppose:
-     *  * timeout = 10, maxFailuresPerExec = 2
-     *  * Task 1 fails on exec 1 at time 0
-     *  * Task 2 fails on exec 1 at time 5
-     * -->  exec 1 is blacklisted from time 5 - 15.
-     * This is to simplify the implementation, as well as keep the behavior easier to understand
-     * for the end user.
-     */
-    def dropFailuresWithTimeoutBefore(dropBefore: Long): Unit = {
-      if (minExpiryTime < dropBefore) {
-        var newMinExpiry = Long.MaxValue
-        val newFailures = new ArrayBuffer[(TaskId, Long)]
-        failuresAndExpiryTimes.foreach { case (task, expiryTime) =>
-          if (expiryTime >= dropBefore) {
-            newFailures += ((task, expiryTime))
-            if (expiryTime < newMinExpiry) {
-              newMinExpiry = expiryTime
-            }
-          }
-        }
-        failuresAndExpiryTimes = newFailures
-        minExpiryTime = newMinExpiry
-      }
-    }
-
-    override def toString(): String = {
-      s"failures = $failuresAndExpiryTimes"
-    }
-  }
-
-}
-
-private[spark] object BlacklistTracker extends Logging {
-
-  private val DEFAULT_TIMEOUT = "1h"
-
-  /**
-   * Returns true if the blacklist is enabled, based on checking the configuration in the following
-   * order:
-   * 1. Is it specifically enabled or disabled?
-   * 2. Is it enabled via the legacy timeout conf?
-   * 3. Default is off
-   */
-  def isBlacklistEnabled(conf: SparkConf): Boolean = {
-    conf.get(config.BLACKLIST_ENABLED) match {
-      case Some(enabled) =>
-        enabled
-      case None =>
-        // if they've got a non-zero setting for the legacy conf, always enable the blacklist,
-        // otherwise, use the default.
-        val legacyKey = config.BLACKLIST_LEGACY_TIMEOUT_CONF.key
-        conf.get(config.BLACKLIST_LEGACY_TIMEOUT_CONF).exists { legacyTimeout =>
-          if (legacyTimeout == 0) {
-            logWarning(s"Turning off blacklisting due to legacy configuration: $legacyKey == 0")
-            false
-          } else {
-            logWarning(s"Turning on blacklisting due to legacy configuration: $legacyKey > 0")
-            true
-          }
-        }
-    }
-  }
-
-  def getBlacklistTimeout(conf: SparkConf): Long = {
-    conf.get(config.BLACKLIST_TIMEOUT_CONF).getOrElse {
-      conf.get(config.BLACKLIST_LEGACY_TIMEOUT_CONF).getOrElse {
-        Utils.timeStringAsMs(DEFAULT_TIMEOUT)
-      }
-    }
-  }
-
-  /**
-   * Verify that blacklist configurations are consistent; if not, throw an exception.  Should only
-   * be called if blacklisting is enabled.
-   *
-   * The configuration for the blacklist is expected to adhere to a few invariants.  Default
-   * values follow these rules of course, but users may unwittingly change one configuration
-   * without making the corresponding adjustment elsewhere.  This ensures we fail-fast when
-   * there are such misconfigurations.
-   */
-  def validateBlacklistConfs(conf: SparkConf): Unit = {
-
-    def mustBePos(k: String, v: String): Unit = {
-      throw new IllegalArgumentException(s"$k was $v, but must be > 0.")
-    }
-
-    Seq(
-      config.MAX_TASK_ATTEMPTS_PER_EXECUTOR,
-      config.MAX_TASK_ATTEMPTS_PER_NODE,
-      config.MAX_FAILURES_PER_EXEC_STAGE,
-      config.MAX_FAILED_EXEC_PER_NODE_STAGE,
-      config.MAX_FAILURES_PER_EXEC,
-      config.MAX_FAILED_EXEC_PER_NODE
-    ).foreach { config =>
-      val v = conf.get(config)
-      if (v <= 0) {
-        mustBePos(config.key, v.toString)
-      }
-    }
-
-    val timeout = getBlacklistTimeout(conf)
-    if (timeout <= 0) {
-      // first, figure out where the timeout came from, to include the right conf in the message.
-      conf.get(config.BLACKLIST_TIMEOUT_CONF) match {
-        case Some(t) =>
-          mustBePos(config.BLACKLIST_TIMEOUT_CONF.key, timeout.toString)
-        case None =>
-          mustBePos(config.BLACKLIST_LEGACY_TIMEOUT_CONF.key, timeout.toString)
-      }
-    }
-
-    val maxTaskFailures = conf.get(config.TASK_MAX_FAILURES)
-    val maxNodeAttempts = conf.get(config.MAX_TASK_ATTEMPTS_PER_NODE)
-
-    if (maxNodeAttempts >= maxTaskFailures) {
-      throw new IllegalArgumentException(s"${config.MAX_TASK_ATTEMPTS_PER_NODE.key} " +
-        s"( = ${maxNodeAttempts}) was >= ${config.TASK_MAX_FAILURES.key} " +
-        s"( = ${maxTaskFailures} ).  Though blacklisting is enabled, with this configuration, " +
-        s"Spark will not be robust to one bad node.  Decrease " +
-        s"${config.MAX_TASK_ATTEMPTS_PER_NODE.key}, increase ${config.TASK_MAX_FAILURES.key}, " +
-        s"or disable blacklisting with ${config.BLACKLIST_ENABLED.key}")
-    }
-  }
-}
-
-private final case class BlacklistedExecutor(node: String, expiryTime: Long)
diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
index 37f9e0bb483c2..b359501793dbd 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
@@ -170,13 +170,34 @@ private[spark] class DAGScheduler(
    */
   private val cacheLocs = new HashMap[Int, IndexedSeq[Seq[TaskLocation]]]
 
-  // For tracking failed nodes, we use the MapOutputTracker's epoch number, which is sent with
-  // every task. When we detect a node failing, we note the current epoch number and failed
-  // executor, increment it for new tasks, and use this to ignore stray ShuffleMapTask results.
-  //
-  // TODO: Garbage collect information about failure epochs when we know there are no more
-  //       stray messages to detect.
-  private val failedEpoch = new HashMap[String, Long]
+  /**
+   * Tracks the latest epoch of a fully processed error related to the given executor. (We use
+   * the MapOutputTracker's epoch number, which is sent with every task.)
+   *
+   * When an executor fails, it can affect the results of many tasks, and we have to deal with
+   * all of them consistently. We don't simply ignore all future results from that executor,
+   * as the failures may have been transient; but we also don't want to "overreact" to follow-
+   * on errors we receive. Furthermore, we might receive notification of a task success, after
+   * we find out the executor has actually failed; we'll assume those successes are, in fact,
+   * simply delayed notifications and the results have been lost, if the tasks started in the
+   * same or an earlier epoch. In particular, we use this to control when we tell the
+   * BlockManagerMaster that the BlockManager has been lost.
+   */
+  private val executorFailureEpoch = new HashMap[String, Long]
+
+  /**
+   * Tracks the latest epoch of a fully processed error where shuffle files have been lost from
+   * the given executor.
+   *
+   * This is closely related to executorFailureEpoch. They only differ for the executor when
+   * there is an external shuffle service serving shuffle files and we haven't been notified that
+   * the entire worker has been lost. In that case, when an executor is lost, we do not update
+   * the shuffleFileLostEpoch; we wait for a fetch failure. This way, if only the executor
+   * fails, we do not unregister the shuffle data as it can still be served; but if there is
+   * a failure in the shuffle service (resulting in fetch failure), we unregister the shuffle
+   * data only once, even if we get many fetch failures.
+   */
+  private val shuffleFileLostEpoch = new HashMap[String, Long]
 
   private [scheduler] val outputCommitCoordinator = env.outputCommitCoordinator
 
@@ -228,6 +249,11 @@ private[spark] class DAGScheduler(
   private[spark] val eventProcessLoop = new DAGSchedulerEventProcessLoop(this)
   taskScheduler.setDAGScheduler(this)
 
+  private val pushBasedShuffleEnabled = Utils.isPushBasedShuffleEnabled(sc.getConf)
+
+  private val blockManagerMasterDriverHeartbeatTimeout =
+    sc.getConf.get(config.STORAGE_BLOCKMANAGER_MASTER_DRIVER_HEARTBEAT_TIMEOUT).millis
+
   /**
    * Called by the TaskSetManager to report task's starting.
    */
@@ -272,7 +298,8 @@ private[spark] class DAGScheduler(
     listenerBus.post(SparkListenerExecutorMetricsUpdate(execId, accumUpdates,
       executorUpdates))
     blockManagerMaster.driverHeartbeatEndPoint.askSync[Boolean](
-      BlockManagerHeartbeat(blockManagerId), new RpcTimeout(10.minutes, "BlockManagerHeartbeat"))
+      BlockManagerHeartbeat(blockManagerId),
+      new RpcTimeout(blockManagerMasterDriverHeartbeatTimeout, "BlockManagerHeartbeat"))
   }
 
   /**
@@ -311,6 +338,26 @@ private[spark] class DAGScheduler(
     eventProcessLoop.post(SpeculativeTaskSubmitted(task))
   }
 
+  /**
+   * Called by the TaskSetManager when a taskset becomes unschedulable due to executors being
+   * excluded because of too many task failures and dynamic allocation is enabled.
+   */
+  def unschedulableTaskSetAdded(
+      stageId: Int,
+      stageAttemptId: Int): Unit = {
+    eventProcessLoop.post(UnschedulableTaskSetAdded(stageId, stageAttemptId))
+  }
+
+  /**
+   * Called by the TaskSetManager when an unschedulable taskset becomes schedulable and dynamic
+   * allocation is enabled.
+   */
+  def unschedulableTaskSetRemoved(
+      stageId: Int,
+      stageAttemptId: Int): Unit = {
+    eventProcessLoop.post(UnschedulableTaskSetRemoved(stageId, stageAttemptId))
+  }
+
   private[scheduler]
   def getCacheLocs(rdd: RDD[_]): IndexedSeq[Seq[TaskLocation]] = cacheLocs.synchronized {
     // Note: this doesn't use `getOrElse()` because this method is called O(num tasks) times
@@ -366,9 +413,9 @@ private[spark] class DAGScheduler(
   /**
    * Check to make sure we don't launch a barrier stage with unsupported RDD chain pattern. The
    * following patterns are not supported:
-   * 1. Ancestor RDDs that have different number of partitions from the resulting RDD (eg.
+   * 1. Ancestor RDDs that have different number of partitions from the resulting RDD (e.g.
    * union()/coalesce()/first()/take()/PartitionPruningRDD);
-   * 2. An RDD that depends on multiple barrier RDDs (eg. barrierRdd1.zip(barrierRdd2)).
+   * 2. An RDD that depends on multiple barrier RDDs (e.g. barrierRdd1.zip(barrierRdd2)).
    */
   private def checkBarrierStageWithRDDChainPattern(rdd: RDD[_], numTasksInStage: Int): Unit = {
     if (rdd.isBarrier() &&
@@ -409,14 +456,15 @@ private[spark] class DAGScheduler(
       // since we can't do it in the RDD constructor because # of partitions is unknown
       logInfo(s"Registering RDD ${rdd.id} (${rdd.getCreationSite}) as input to " +
         s"shuffle ${shuffleDep.shuffleId}")
-      mapOutputTracker.registerShuffle(shuffleDep.shuffleId, rdd.partitions.length)
+      mapOutputTracker.registerShuffle(shuffleDep.shuffleId, rdd.partitions.length,
+        shuffleDep.partitioner.numPartitions)
     }
     stage
   }
 
   /**
    * We don't support run a barrier stage with dynamic resource allocation enabled, it shall lead
-   * to some confusing behaviors (eg. with dynamic resource allocation enabled, it may happen that
+   * to some confusing behaviors (e.g. with dynamic resource allocation enabled, it may happen that
    * we acquire some executors (but not enough to launch all the tasks in a barrier stage) and
    * later release them due to executor idle time expire, and then acquire again).
    *
@@ -439,10 +487,12 @@ private[spark] class DAGScheduler(
    * submission.
    */
   private def checkBarrierStageWithNumSlots(rdd: RDD[_], rp: ResourceProfile): Unit = {
-    val numPartitions = rdd.getNumPartitions
-    val maxNumConcurrentTasks = sc.maxNumConcurrentTasks(rp)
-    if (rdd.isBarrier() && numPartitions > maxNumConcurrentTasks) {
-      throw new BarrierJobSlotsNumberCheckFailed(numPartitions, maxNumConcurrentTasks)
+    if (rdd.isBarrier()) {
+      val numPartitions = rdd.getNumPartitions
+      val maxNumConcurrentTasks = sc.maxNumConcurrentTasks(rp)
+      if (numPartitions > maxNumConcurrentTasks) {
+        throw new BarrierJobSlotsNumberCheckFailed(numPartitions, maxNumConcurrentTasks)
+      }
     }
   }
 
@@ -845,10 +895,11 @@ private[spark] class DAGScheduler(
       timeout: Long,
       properties: Properties): PartialResult[R] = {
     val jobId = nextJobId.getAndIncrement()
+    val clonedProperties = Utils.cloneProperties(properties)
     if (rdd.partitions.isEmpty) {
       // Return immediately if the job is running 0 tasks
       val time = clock.getTimeMillis()
-      listenerBus.post(SparkListenerJobStart(jobId, time, Seq[StageInfo](), properties))
+      listenerBus.post(SparkListenerJobStart(jobId, time, Seq[StageInfo](), clonedProperties))
       listenerBus.post(SparkListenerJobEnd(jobId, time, JobSucceeded))
       return new PartialResult(evaluator.currentResult(), true)
     }
@@ -856,7 +907,7 @@ private[spark] class DAGScheduler(
     val func2 = func.asInstanceOf[(TaskContext, Iterator[_]) => _]
     eventProcessLoop.post(JobSubmitted(
       jobId, rdd, func2, rdd.partitions.indices.toArray, callSite, listener,
-      Utils.cloneProperties(properties)))
+      clonedProperties))
     listener.awaitResult()    // Will throw an exception if the job fails
   }
 
@@ -1014,6 +1065,18 @@ private[spark] class DAGScheduler(
     listenerBus.post(SparkListenerSpeculativeTaskSubmitted(task.stageId, task.stageAttemptId))
   }
 
+  private[scheduler] def handleUnschedulableTaskSetAdded(
+      stageId: Int,
+      stageAttemptId: Int): Unit = {
+    listenerBus.post(SparkListenerUnschedulableTaskSetAdded(stageId, stageAttemptId))
+  }
+
+  private[scheduler] def handleUnschedulableTaskSetRemoved(
+      stageId: Int,
+      stageAttemptId: Int): Unit = {
+    listenerBus.post(SparkListenerUnschedulableTaskSetRemoved(stageId, stageAttemptId))
+  }
+
   private[scheduler] def handleTaskSetFailed(
       taskSet: TaskSet,
       reason: String,
@@ -1105,7 +1168,8 @@ private[spark] class DAGScheduler(
     val stageIds = jobIdToStageIds(jobId).toArray
     val stageInfos = stageIds.flatMap(id => stageIdToStage.get(id).map(_.latestInfo))
     listenerBus.post(
-      SparkListenerJobStart(job.jobId, jobSubmissionTime, stageInfos, properties))
+      SparkListenerJobStart(job.jobId, jobSubmissionTime, stageInfos,
+        Utils.cloneProperties(properties)))
     submitStage(finalStage)
   }
 
@@ -1143,7 +1207,8 @@ private[spark] class DAGScheduler(
     val stageIds = jobIdToStageIds(jobId).toArray
     val stageInfos = stageIds.flatMap(id => stageIdToStage.get(id).map(_.latestInfo))
     listenerBus.post(
-      SparkListenerJobStart(job.jobId, jobSubmissionTime, stageInfos, properties))
+      SparkListenerJobStart(job.jobId, jobSubmissionTime, stageInfos,
+        Utils.cloneProperties(properties)))
     submitStage(finalStage)
 
     // If the whole stage has already finished, tell the listener and remove it
@@ -1197,6 +1262,33 @@ private[spark] class DAGScheduler(
     execCores.map(cores => properties.setProperty(EXECUTOR_CORES_LOCAL_PROPERTY, cores))
   }
 
+  /**
+   * If push based shuffle is enabled, set the shuffle services to be used for the given
+   * shuffle map stage for block push/merge.
+   *
+   * Even with dynamic resource allocation kicking in and significantly reducing the number
+   * of available active executors, we would still be able to get sufficient shuffle service
+   * locations for block push/merge by getting the historical locations of past executors.
+   */
+  private def prepareShuffleServicesForShuffleMapStage(stage: ShuffleMapStage): Unit = {
+    // TODO(SPARK-32920) Handle stage reuse/retry cases separately as without finalize
+    // TODO changes we cannot disable shuffle merge for the retry/reuse cases
+    val mergerLocs = sc.schedulerBackend.getShufflePushMergerLocations(
+      stage.shuffleDep.partitioner.numPartitions, stage.resourceProfileId)
+
+    if (mergerLocs.nonEmpty) {
+      stage.shuffleDep.setMergerLocs(mergerLocs)
+      logInfo(s"Push-based shuffle enabled for $stage (${stage.name}) with" +
+        s" ${stage.shuffleDep.getMergerLocs.size} merger locations")
+
+      logDebug("List of shuffle push merger locations " +
+        s"${stage.shuffleDep.getMergerLocs.map(_.host).mkString(", ")}")
+    } else {
+      logInfo("No available merger locations." +
+        s" Push-based shuffle disabled for $stage (${stage.name})")
+    }
+  }
+
   /** Called when stage's parents are available and we can now do its task. */
   private def submitMissingTasks(stage: Stage, jobId: Int): Unit = {
     logDebug("submitMissingTasks(" + stage + ")")
@@ -1226,6 +1318,12 @@ private[spark] class DAGScheduler(
     stage match {
       case s: ShuffleMapStage =>
         outputCommitCoordinator.stageStart(stage = s.id, maxPartitionId = s.numPartitions - 1)
+        // Only generate merger location for a given shuffle dependency once. This way, even if
+        // this stage gets retried, it would still be merging blocks using the same set of
+        // shuffle services.
+        if (pushBasedShuffleEnabled) {
+          prepareShuffleServicesForShuffleMapStage(s)
+        }
       case s: ResultStage =>
         outputCommitCoordinator.stageStart(
           stage = s.id, maxPartitionId = s.rdd.partitions.length - 1)
@@ -1243,7 +1341,8 @@ private[spark] class DAGScheduler(
     } catch {
       case NonFatal(e) =>
         stage.makeNewStageAttempt(partitionsToCompute.size)
-        listenerBus.post(SparkListenerStageSubmitted(stage.latestInfo, properties))
+        listenerBus.post(SparkListenerStageSubmitted(stage.latestInfo,
+          Utils.cloneProperties(properties)))
         abortStage(stage, s"Task creation failed: $e\n${Utils.exceptionString(e)}", Some(e))
         runningStages -= stage
         return
@@ -1257,7 +1356,8 @@ private[spark] class DAGScheduler(
     if (partitionsToCompute.nonEmpty) {
       stage.latestInfo.submissionTime = Some(clock.getTimeMillis())
     }
-    listenerBus.post(SparkListenerStageSubmitted(stage.latestInfo, properties))
+    listenerBus.post(SparkListenerStageSubmitted(stage.latestInfo,
+      Utils.cloneProperties(properties)))
 
     // TODO: Maybe we can keep the taskBinary in Stage to avoid serializing it multiple times.
     // Broadcasted binary for the task, used to dispatch tasks to executors. Note that we broadcast
@@ -1465,7 +1565,7 @@ private[spark] class DAGScheduler(
       event.reason)
 
     if (!stageIdToStage.contains(task.stageId)) {
-      // The stage may have already finished when we get this event -- eg. maybe it was a
+      // The stage may have already finished when we get this event -- e.g. maybe it was a
       // speculative task. It is important that we send the TaskEnd event in any case, so listeners
       // are properly notified and can chose to handle it. For instance, some listeners are
       // doing their own accounting and if they don't get the task end event they think
@@ -1566,7 +1666,8 @@ private[spark] class DAGScheduler(
             val status = event.result.asInstanceOf[MapStatus]
             val execId = status.location.executorId
             logDebug("ShuffleMapTask finished on " + execId)
-            if (failedEpoch.contains(execId) && smt.epoch <= failedEpoch(execId)) {
+            if (executorFailureEpoch.contains(execId) &&
+                smt.epoch <= executorFailureEpoch(execId)) {
               logInfo(s"Ignoring possibly bogus $smt completion from executor $execId")
             } else {
               // The epoch of the task is acceptable (i.e., the task was launched after the most
@@ -1664,10 +1765,9 @@ private[spark] class DAGScheduler(
             val abortMessage = if (disallowStageRetryForTest) {
               "Fetch failure will not retry stage due to testing config"
             } else {
-              s"""$failedStage (${failedStage.name})
-                 |has failed the maximum allowable number of
-                 |times: $maxConsecutiveStageAttempts.
-                 |Most recent failure reason: $failureMessage""".stripMargin.replaceAll("\n", " ")
+              s"$failedStage (${failedStage.name}) has failed the maximum allowable number of " +
+                s"times: $maxConsecutiveStageAttempts. Most recent failure reason:\n" +
+                failureMessage
             }
             abortStage(failedStage, abortMessage, None)
           } else { // update failedStages and make sure a ResubmitFailedStages event is enqueued
@@ -1767,10 +1867,19 @@ private[spark] class DAGScheduler(
 
           // TODO: mark the executor as failed only if there were lots of fetch failures on it
           if (bmAddress != null) {
-            val hostToUnregisterOutputs = if (env.blockManager.externalShuffleServiceEnabled &&
-              unRegisterOutputOnHostOnFetchFailure) {
-              // We had a fetch failure with the external shuffle service, so we
-              // assume all shuffle data on the node is bad.
+            val externalShuffleServiceEnabled = env.blockManager.externalShuffleServiceEnabled
+            val isHostDecommissioned = taskScheduler
+              .getExecutorDecommissionState(bmAddress.executorId)
+              .exists(_.workerHost.isDefined)
+
+            // Shuffle output of all executors on host `bmAddress.host` may be lost if:
+            // - External shuffle service is enabled, so we assume that all shuffle data on node is
+            //   bad.
+            // - Host is decommissioned, thus all executors on that host will die.
+            val shuffleOutputOfEntireHostLost = externalShuffleServiceEnabled ||
+              isHostDecommissioned
+            val hostToUnregisterOutputs = if (shuffleOutputOfEntireHostLost
+              && unRegisterOutputOnHostOnFetchFailure) {
               Some(bmAddress.host)
             } else {
               // Unregister shuffle data just for one executor (we don't have any
@@ -1781,7 +1890,14 @@ private[spark] class DAGScheduler(
               execId = bmAddress.executorId,
               fileLost = true,
               hostToUnregisterOutputs = hostToUnregisterOutputs,
-              maybeEpoch = Some(task.epoch))
+              maybeEpoch = Some(task.epoch),
+              // shuffleFileLostEpoch is ignored when a host is decommissioned because some
+              // decommissioned executors on that host might have been removed before this fetch
+              // failure and might have bumped up the shuffleFileLostEpoch. We ignore that, and
+              // proceed with unconditional removal of shuffle outputs from all executors on that
+              // host, including from those that we still haven't confirmed as lost due to heartbeat
+              // delays.
+              ignoreShuffleFileLostEpoch = isHostDecommissioned)
           }
         }
 
@@ -1809,7 +1925,9 @@ private[spark] class DAGScheduler(
             // killAllTaskAttempts will fail if a SchedulerBackend does not implement killTask.
             val reason = s"Task $task from barrier stage $failedStage (${failedStage.name}) " +
               "failed."
-            taskScheduler.killAllTaskAttempts(stageId, interruptThread = false, reason)
+            val job = jobIdToActiveJob.get(failedStage.firstJobId)
+            val shouldInterrupt = job.exists(j => shouldInterruptTaskThread(j))
+            taskScheduler.killAllTaskAttempts(stageId, shouldInterrupt, reason)
           } catch {
             case e: UnsupportedOperationException =>
               // Cannot continue with barrier stage if failed to cancel zombie barrier tasks.
@@ -1832,11 +1950,8 @@ private[spark] class DAGScheduler(
               "Barrier stage will not retry stage due to testing config. Most recent failure " +
                 s"reason: $message"
             } else {
-              s"""$failedStage (${failedStage.name})
-                 |has failed the maximum allowable number of
-                 |times: $maxConsecutiveStageAttempts.
-                 |Most recent failure reason: $message
-               """.stripMargin.replaceAll("\n", " ")
+              s"$failedStage (${failedStage.name}) has failed the maximum allowable number of " +
+                s"times: $maxConsecutiveStageAttempts. Most recent failure reason: $message"
             }
             abortStage(failedStage, abortMessage, None)
           } else {
@@ -1912,50 +2027,76 @@ private[spark] class DAGScheduler(
    * modify the scheduler's internal state. Use executorLost() to post a loss event from outside.
    *
    * We will also assume that we've lost all shuffle blocks associated with the executor if the
-   * executor serves its own blocks (i.e., we're not using external shuffle), the entire slave
-   * is lost (likely including the shuffle service), or a FetchFailed occurred, in which case we
-   * presume all shuffle data related to this executor to be lost.
-   *
-   * Optionally the epoch during which the failure was caught can be passed to avoid allowing
-   * stray fetch failures from possibly retriggering the detection of a node as lost.
+   * executor serves its own blocks (i.e., we're not using an external shuffle service), or the
+   * entire Standalone worker is lost.
    */
   private[scheduler] def handleExecutorLost(
       execId: String,
-      workerLost: Boolean): Unit = {
+      workerHost: Option[String]): Unit = {
     // if the cluster manager explicitly tells us that the entire worker was lost, then
     // we know to unregister shuffle output.  (Note that "worker" specifically refers to the process
     // from a Standalone cluster, where the shuffle service lives in the Worker.)
-    val fileLost = workerLost || !env.blockManager.externalShuffleServiceEnabled
+    val fileLost = workerHost.isDefined || !env.blockManager.externalShuffleServiceEnabled
     removeExecutorAndUnregisterOutputs(
       execId = execId,
       fileLost = fileLost,
-      hostToUnregisterOutputs = None,
+      hostToUnregisterOutputs = workerHost,
       maybeEpoch = None)
   }
 
+  /**
+   * Handles removing an executor from the BlockManagerMaster as well as unregistering shuffle
+   * outputs for the executor or optionally its host.
+   *
+   * @param execId executor to be removed
+   * @param fileLost If true, indicates that we assume we've lost all shuffle blocks associated
+   *   with the executor; this happens if the executor serves its own blocks (i.e., we're not
+   *   using an external shuffle service), the entire Standalone worker is lost, or a FetchFailed
+   *   occurred (in which case we presume all shuffle data related to this executor to be lost).
+   * @param hostToUnregisterOutputs (optional) executor host if we're unregistering all the
+   *   outputs on the host
+   * @param maybeEpoch (optional) the epoch during which the failure was caught (this prevents
+   *   reprocessing for follow-on fetch failures)
+   */
   private def removeExecutorAndUnregisterOutputs(
       execId: String,
       fileLost: Boolean,
       hostToUnregisterOutputs: Option[String],
-      maybeEpoch: Option[Long] = None): Unit = {
+      maybeEpoch: Option[Long] = None,
+      ignoreShuffleFileLostEpoch: Boolean = false): Unit = {
     val currentEpoch = maybeEpoch.getOrElse(mapOutputTracker.getEpoch)
-    if (!failedEpoch.contains(execId) || failedEpoch(execId) < currentEpoch) {
-      failedEpoch(execId) = currentEpoch
-      logInfo("Executor lost: %s (epoch %d)".format(execId, currentEpoch))
+    logDebug(s"Considering removal of executor $execId; " +
+      s"fileLost: $fileLost, currentEpoch: $currentEpoch")
+    if (!executorFailureEpoch.contains(execId) || executorFailureEpoch(execId) < currentEpoch) {
+      executorFailureEpoch(execId) = currentEpoch
+      logInfo(s"Executor lost: $execId (epoch $currentEpoch)")
+      if (pushBasedShuffleEnabled) {
+        // Remove fetchFailed host in the shuffle push merger list for push based shuffle
+        hostToUnregisterOutputs.foreach(
+          host => blockManagerMaster.removeShufflePushMergerLocation(host))
+      }
       blockManagerMaster.removeExecutor(execId)
-      if (fileLost) {
+      clearCacheLocs()
+    }
+    if (fileLost) {
+      val remove = if (ignoreShuffleFileLostEpoch) {
+        true
+      } else if (!shuffleFileLostEpoch.contains(execId) ||
+        shuffleFileLostEpoch(execId) < currentEpoch) {
+        shuffleFileLostEpoch(execId) = currentEpoch
+        true
+      } else {
+        false
+      }
+      if (remove) {
         hostToUnregisterOutputs match {
           case Some(host) =>
-            logInfo("Shuffle files lost for host: %s (epoch %d)".format(host, currentEpoch))
+            logInfo(s"Shuffle files lost for host: $host (epoch $currentEpoch)")
             mapOutputTracker.removeOutputsOnHost(host)
           case None =>
-            logInfo("Shuffle files lost for executor: %s (epoch %d)".format(execId, currentEpoch))
+            logInfo(s"Shuffle files lost for executor: $execId (epoch $currentEpoch)")
             mapOutputTracker.removeOutputsOnExecutor(execId)
         }
-        clearCacheLocs()
-
-      } else {
-        logDebug("Additional executor lost message for %s (epoch %d)".format(execId, currentEpoch))
       }
     }
   }
@@ -1981,11 +2122,12 @@ private[spark] class DAGScheduler(
   }
 
   private[scheduler] def handleExecutorAdded(execId: String, host: String): Unit = {
-    // remove from failedEpoch(execId) ?
-    if (failedEpoch.contains(execId)) {
+    // remove from executorFailureEpoch(execId) ?
+    if (executorFailureEpoch.contains(execId)) {
       logInfo("Host added was in lost list earlier: " + host)
-      failedEpoch -= execId
+      executorFailureEpoch -= execId
     }
+    shuffleFileLostEpoch -= execId
   }
 
   private[scheduler] def handleStageCancellation(stageId: Int, reason: Option[String]): Unit = {
@@ -2272,11 +2414,12 @@ private[scheduler] class DAGSchedulerEventProcessLoop(dagScheduler: DAGScheduler
       dagScheduler.handleExecutorAdded(execId, host)
 
     case ExecutorLost(execId, reason) =>
-      val workerLost = reason match {
-        case SlaveLost(_, true) => true
-        case _ => false
+      val workerHost = reason match {
+        case ExecutorProcessLost(_, workerHost, _) => workerHost
+        case ExecutorDecommission(workerHost) => workerHost
+        case _ => None
       }
-      dagScheduler.handleExecutorLost(execId, workerLost)
+      dagScheduler.handleExecutorLost(execId, workerHost)
 
     case WorkerRemoved(workerId, host, message) =>
       dagScheduler.handleWorkerRemoved(workerId, host, message)
@@ -2287,6 +2430,12 @@ private[scheduler] class DAGSchedulerEventProcessLoop(dagScheduler: DAGScheduler
     case SpeculativeTaskSubmitted(task) =>
       dagScheduler.handleSpeculativeTaskSubmitted(task)
 
+    case UnschedulableTaskSetAdded(stageId, stageAttemptId) =>
+      dagScheduler.handleUnschedulableTaskSetAdded(stageId, stageAttemptId)
+
+    case UnschedulableTaskSetRemoved(stageId, stageAttemptId) =>
+      dagScheduler.handleUnschedulableTaskSetRemoved(stageId, stageAttemptId)
+
     case GettingResultEvent(taskInfo) =>
       dagScheduler.handleGetTaskResult(taskInfo)
 
diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGSchedulerEvent.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGSchedulerEvent.scala
index 78d458338e8fb..d226fe88614d2 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/DAGSchedulerEvent.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/DAGSchedulerEvent.scala
@@ -97,3 +97,11 @@ private[scheduler] case object ResubmitFailedStages extends DAGSchedulerEvent
 private[scheduler]
 case class SpeculativeTaskSubmitted(task: Task[_]) extends DAGSchedulerEvent
 
+private[scheduler]
+case class UnschedulableTaskSetAdded(stageId: Int, stageAttemptId: Int)
+  extends DAGSchedulerEvent
+
+private[scheduler]
+case class UnschedulableTaskSetRemoved(stageId: Int, stageAttemptId: Int)
+  extends DAGSchedulerEvent
+
diff --git a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
index 24e2a5e4d4a62..c57894b9f4f8f 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
@@ -18,7 +18,9 @@
 package org.apache.spark.scheduler
 
 import java.net.URI
+import java.util.Properties
 
+import scala.collection.JavaConverters._
 import scala.collection.mutable
 
 import org.apache.hadoop.conf.Configuration
@@ -103,7 +105,7 @@ private[spark] class EventLoggingListener(
 
   // Events that do not trigger a flush
   override def onStageSubmitted(event: SparkListenerStageSubmitted): Unit = {
-    logEvent(event)
+    logEvent(event.copy(properties = redactProperties(event.properties)))
     if (shouldLogStageExecutorMetrics) {
       // record the peak metrics for the new stage
       liveStageExecutorMetrics.put((event.stageInfo.stageId, event.stageInfo.attemptNumber()),
@@ -128,7 +130,7 @@ private[spark] class EventLoggingListener(
   }
 
   override def onEnvironmentUpdate(event: SparkListenerEnvironmentUpdate): Unit = {
-    logEvent(redactEvent(event))
+    logEvent(redactEvent(sparkConf, event))
   }
 
   // Events that trigger a flush
@@ -156,7 +158,9 @@ private[spark] class EventLoggingListener(
     logEvent(event, flushLogger = true)
   }
 
-  override def onJobStart(event: SparkListenerJobStart): Unit = logEvent(event, flushLogger = true)
+  override def onJobStart(event: SparkListenerJobStart): Unit = {
+    logEvent(event.copy(properties = redactProperties(event.properties)), flushLogger = true)
+  }
 
   override def onJobEnd(event: SparkListenerJobEnd): Unit = logEvent(event, flushLogger = true)
 
@@ -191,27 +195,53 @@ private[spark] class EventLoggingListener(
     logEvent(event, flushLogger = true)
   }
 
+  override def onExecutorExcluded(event: SparkListenerExecutorExcluded): Unit = {
+    logEvent(event, flushLogger = true)
+  }
+
   override def onExecutorBlacklistedForStage(
       event: SparkListenerExecutorBlacklistedForStage): Unit = {
     logEvent(event, flushLogger = true)
   }
 
+  override def onExecutorExcludedForStage(
+      event: SparkListenerExecutorExcludedForStage): Unit = {
+    logEvent(event, flushLogger = true)
+  }
+
   override def onNodeBlacklistedForStage(event: SparkListenerNodeBlacklistedForStage): Unit = {
     logEvent(event, flushLogger = true)
   }
 
+  override def onNodeExcludedForStage(event: SparkListenerNodeExcludedForStage): Unit = {
+    logEvent(event, flushLogger = true)
+  }
+
   override def onExecutorUnblacklisted(event: SparkListenerExecutorUnblacklisted): Unit = {
     logEvent(event, flushLogger = true)
   }
 
+  override def onExecutorUnexcluded(event: SparkListenerExecutorUnexcluded): Unit = {
+    logEvent(event, flushLogger = true)
+  }
+
+
   override def onNodeBlacklisted(event: SparkListenerNodeBlacklisted): Unit = {
     logEvent(event, flushLogger = true)
   }
 
+  override def onNodeExcluded(event: SparkListenerNodeExcluded): Unit = {
+    logEvent(event, flushLogger = true)
+  }
+
   override def onNodeUnblacklisted(event: SparkListenerNodeUnblacklisted): Unit = {
     logEvent(event, flushLogger = true)
   }
 
+  override def onNodeUnexcluded(event: SparkListenerNodeUnexcluded): Unit = {
+    logEvent(event, flushLogger = true)
+  }
+
   override def onBlockUpdated(event: SparkListenerBlockUpdated): Unit = {
     if (shouldLogBlockUpdates) {
       logEvent(event, flushLogger = true)
@@ -235,6 +265,10 @@ private[spark] class EventLoggingListener(
     }
   }
 
+  override def onResourceProfileAdded(event: SparkListenerResourceProfileAdded): Unit = {
+    logEvent(event, flushLogger = true)
+  }
+
   override def onOtherEvent(event: SparkListenerEvent): Unit = {
     if (event.logEvent) {
       logEvent(event, flushLogger = true)
@@ -246,7 +280,30 @@ private[spark] class EventLoggingListener(
     logWriter.stop()
   }
 
+  private def redactProperties(properties: Properties): Properties = {
+    if (properties == null) {
+      return properties
+    }
+    val redactedProperties = new Properties
+    // properties may contain some custom local properties such as stage/job description
+    // only properties in sparkConf need to be redacted.
+    val (globalProperties, localProperties) = properties.asScala.toSeq.partition {
+      case (key, _) => sparkConf.contains(key)
+    }
+    (Utils.redact(sparkConf, globalProperties) ++ localProperties).foreach {
+      case (key, value) => redactedProperties.setProperty(key, value)
+    }
+    redactedProperties
+  }
+}
+
+private[spark] object EventLoggingListener extends Logging {
+  val DEFAULT_LOG_DIR = "/tmp/spark-events"
+  // Dummy stage key used by driver in executor metrics updates
+  val DRIVER_STAGE_KEY = (-1, -1)
+
   private[spark] def redactEvent(
+      sparkConf: SparkConf,
       event: SparkListenerEnvironmentUpdate): SparkListenerEnvironmentUpdate = {
     // environmentDetails maps a string descriptor to a set of properties
     // Similar to:
@@ -262,11 +319,4 @@ private[spark] class EventLoggingListener(
     }
     SparkListenerEnvironmentUpdate(redactedProps)
   }
-
-}
-
-private[spark] object EventLoggingListener extends Logging {
-  val DEFAULT_LOG_DIR = "/tmp/spark-events"
-  // Dummy stage key used by driver in executor metrics updates
-  val DRIVER_STAGE_KEY = (-1, -1)
 }
diff --git a/core/src/main/scala/org/apache/spark/scheduler/ExecutorDecommissionInfo.scala b/core/src/main/scala/org/apache/spark/scheduler/ExecutorDecommissionInfo.scala
new file mode 100644
index 0000000000000..da61f438502e9
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/scheduler/ExecutorDecommissionInfo.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.scheduler
+
+/**
+ * Message providing more detail when an executor is being decommissioned.
+ * @param message Human readable reason for why the decommissioning is happening.
+ * @param workerHost When workerHost is defined, it means the host (aka the `node` or `worker`
+ *                in other places) has been decommissioned too. Used to infer if the
+ *                shuffle data might be lost even if the external shuffle service is enabled.
+ */
+private[spark]
+case class ExecutorDecommissionInfo(message: String, workerHost: Option[String] = None)
+
+/**
+ * State related to decommissioning that is kept by the TaskSchedulerImpl. This state is derived
+ * from the info message above but it is kept distinct to allow the state to evolve independently
+ * from the message.
+ */
+private[scheduler] case class ExecutorDecommissionState(
+    // Timestamp the decommissioning commenced as per the Driver's clock,
+    // to estimate when the executor might eventually be lost if EXECUTOR_DECOMMISSION_KILL_INTERVAL
+    // is configured.
+    startTime: Long,
+    workerHost: Option[String] = None)
diff --git a/core/src/main/scala/org/apache/spark/scheduler/ExecutorFailuresInTaskSet.scala b/core/src/main/scala/org/apache/spark/scheduler/ExecutorFailuresInTaskSet.scala
index 70553d8be28b5..f27c1560f8272 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/ExecutorFailuresInTaskSet.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/ExecutorFailuresInTaskSet.scala
@@ -19,7 +19,7 @@ package org.apache.spark.scheduler
 import scala.collection.mutable.HashMap
 
 /**
- * Small helper for tracking failed tasks for blacklisting purposes.  Info on all failures on one
+ * Small helper for tracking failed tasks for exclusion purposes.  Info on all failures on one
  * executor, within one task set.
  */
 private[scheduler] class ExecutorFailuresInTaskSet(val node: String) {
diff --git a/core/src/main/scala/org/apache/spark/scheduler/ExecutorLossReason.scala b/core/src/main/scala/org/apache/spark/scheduler/ExecutorLossReason.scala
index ee31093ec0652..f333c01bb890d 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/ExecutorLossReason.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/ExecutorLossReason.scala
@@ -20,7 +20,7 @@ package org.apache.spark.scheduler
 import org.apache.spark.executor.ExecutorExitCode
 
 /**
- * Represents an explanation for an executor or whole slave failing or exiting.
+ * Represents an explanation for an executor or whole process failing or exiting.
  */
 private[spark]
 class ExecutorLossReason(val message: String) extends Serializable {
@@ -40,6 +40,10 @@ private[spark] object ExecutorExited {
   }
 }
 
+private[spark] object ExecutorLossMessage {
+  val decommissionFinished = "Finished decommissioning"
+}
+
 private[spark] object ExecutorKilled extends ExecutorLossReason("Executor killed by driver.")
 
 /**
@@ -53,10 +57,16 @@ private [spark] object LossReasonPending extends ExecutorLossReason("Pending los
 
 /**
  * @param _message human readable loss reason
- * @param workerLost whether the worker is confirmed lost too (i.e. including shuffle service)
+ * @param workerHost it's defined when the host is confirmed lost too (i.e. including
+ *                   shuffle service)
+ * @param causedByApp whether the loss of the executor is the fault of the running app.
+ *                    (assumed true by default unless known explicitly otherwise)
  */
 private[spark]
-case class SlaveLost(_message: String = "Slave lost", workerLost: Boolean = false)
+case class ExecutorProcessLost(
+    _message: String = "Executor Process Lost",
+    workerHost: Option[String] = None,
+    causedByApp: Boolean = true)
   extends ExecutorLossReason(_message)
 
 /**
@@ -64,5 +74,9 @@ case class SlaveLost(_message: String = "Slave lost", workerLost: Boolean = fals
  *
  * This is used by the task scheduler to remove state associated with the executor, but
  * not yet fail any tasks that were running in the executor before the executor is "fully" lost.
+ * If you update this code make sure to re-run the K8s integration tests.
+ *
+ * @param workerHost it is defined when the worker is decommissioned too
  */
-private [spark] object ExecutorDecommission extends ExecutorLossReason("Executor decommission.")
+private [spark] case class ExecutorDecommission(workerHost: Option[String] = None)
+ extends ExecutorLossReason("Executor decommission.")
diff --git a/core/src/main/scala/org/apache/spark/scheduler/ExecutorResourceInfo.scala b/core/src/main/scala/org/apache/spark/scheduler/ExecutorResourceInfo.scala
index fd04db8c09d76..508c6cebd9fe3 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/ExecutorResourceInfo.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/ExecutorResourceInfo.scala
@@ -36,4 +36,5 @@ private[spark] class ExecutorResourceInfo(
   override protected def resourceName = this.name
   override protected def resourceAddresses = this.addresses
   override protected def slotsPerAddress: Int = numParts
+  def totalAddressAmount: Int = resourceAddresses.length * slotsPerAddress
 }
diff --git a/core/src/main/scala/org/apache/spark/scheduler/HealthTracker.scala b/core/src/main/scala/org/apache/spark/scheduler/HealthTracker.scala
new file mode 100644
index 0000000000000..6bd5668651aad
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/scheduler/HealthTracker.scala
@@ -0,0 +1,512 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.scheduler
+
+import java.util.concurrent.atomic.AtomicReference
+
+import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet}
+
+import org.apache.spark.{ExecutorAllocationClient, SparkConf, SparkContext}
+import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config
+import org.apache.spark.util.{Clock, SystemClock, Utils}
+
+/**
+ * HealthTracker is designed to track problematic executors and nodes.  It supports excluding
+ * executors and nodes across an entire application (with a periodic expiry). TaskSetManagers add
+ * additional logic for exclusion of executors and nodes for individual tasks and stages which
+ * works in concert with the logic here.
+ *
+ * The tracker needs to deal with a variety of workloads, e.g.:
+ *
+ *  * bad user code -- this may lead to many task failures, but that should not count against
+ *      individual executors
+ *  * many small stages -- this may prevent a bad executor for having many failures within one
+ *      stage, but still many failures over the entire application
+ *  * "flaky" executors -- they don't fail every task, but are still faulty enough to merit
+ *      excluding
+ *  * missing shuffle files -- may trigger fetch failures on healthy executors.
+ *
+ * See the design doc on SPARK-8425 for a more in-depth discussion. Note SPARK-32037 renamed
+ * the feature.
+ *
+ * THREADING: As with most helpers of TaskSchedulerImpl, this is not thread-safe.  Though it is
+ * called by multiple threads, callers must already have a lock on the TaskSchedulerImpl.  The
+ * one exception is [[excludedNodeList()]], which can be called without holding a lock.
+ */
+private[scheduler] class HealthTracker (
+    private val listenerBus: LiveListenerBus,
+    conf: SparkConf,
+    allocationClient: Option[ExecutorAllocationClient],
+    clock: Clock = new SystemClock()) extends Logging {
+
+  def this(sc: SparkContext, allocationClient: Option[ExecutorAllocationClient]) = {
+    this(sc.listenerBus, sc.conf, allocationClient)
+  }
+
+  HealthTracker.validateExcludeOnFailureConfs(conf)
+  private val MAX_FAILURES_PER_EXEC = conf.get(config.MAX_FAILURES_PER_EXEC)
+  private val MAX_FAILED_EXEC_PER_NODE = conf.get(config.MAX_FAILED_EXEC_PER_NODE)
+  val EXCLUDE_ON_FAILURE_TIMEOUT_MILLIS = HealthTracker.getExludeOnFailureTimeout(conf)
+  private val EXCLUDE_FETCH_FAILURE_ENABLED =
+    conf.get(config.EXCLUDE_ON_FAILURE_FETCH_FAILURE_ENABLED)
+  private val EXCLUDE_ON_FAILURE_DECOMMISSION_ENABLED =
+    conf.get(config.EXCLUDE_ON_FAILURE_DECOMMISSION_ENABLED)
+
+  /**
+   * A map from executorId to information on task failures. Tracks the time of each task failure,
+   * so that we can avoid excluding executors due to failures that are very far apart. We do not
+   * actively remove from this as soon as tasks hit their timeouts, to avoid the time it would take
+   * to do so. But it will not grow too large, because as soon as an executor gets too many
+   * failures, we exclude the executor and remove its entry here.
+   */
+  private val executorIdToFailureList = new HashMap[String, ExecutorFailureList]()
+  val executorIdToExcludedStatus = new HashMap[String, ExcludedExecutor]()
+  val nodeIdToExcludedExpiryTime = new HashMap[String, Long]()
+  /**
+   * An immutable copy of the set of nodes that are currently excluded.  Kept in an
+   * AtomicReference to make [[excludedNodeList()]] thread-safe.
+   */
+  private val _excludedNodeList = new AtomicReference[Set[String]](Set())
+  /**
+   * Time when the next excluded node will expire.  Used as a shortcut to
+   * avoid iterating over all entries in the excludedNodeList when none will have expired.
+   */
+  var nextExpiryTime: Long = Long.MaxValue
+  /**
+   * Mapping from nodes to all of the executors that have been excluded on that node. We do *not*
+   * remove from this when executors are removed from spark, so we can track when we get multiple
+   * successive excluded executors on one node.  Nonetheless, it will not grow too large because
+   * there cannot be many excluded executors on one node, before we stop requesting more
+   * executors on that node, and we clean up the list of exluded executors once an executor has
+   * been excluded for EXCLUDE_ON_FAILURE_TIMEOUT_MILLIS.
+   */
+  val nodeToExcludedExecs = new HashMap[String, HashSet[String]]()
+
+  /**
+   * Include executors and nodes that have been excluded for at least
+   * EXCLUDE_ON_FAILURE_TIMEOUT_MILLIS
+   */
+  def applyExcludeOnFailureTimeout(): Unit = {
+    val now = clock.getTimeMillis()
+    // quickly check if we've got anything to expire that is excluded -- if not,
+    // avoid doing any work
+    if (now > nextExpiryTime) {
+      // Apply the timeout to excluded nodes and executors
+      val execsToInclude = executorIdToExcludedStatus.filter(_._2.expiryTime < now).keys
+      if (execsToInclude.nonEmpty) {
+        // Include any executors that have been exluded longer than the excludeOnFailure timeout.
+        logInfo(s"Removing executors $execsToInclude from exclude list because the " +
+          s"the executors have reached the timed out")
+        execsToInclude.foreach { exec =>
+          val status = executorIdToExcludedStatus.remove(exec).get
+          val failedExecsOnNode = nodeToExcludedExecs(status.node)
+          // post both to keep backwards compatibility
+          listenerBus.post(SparkListenerExecutorUnblacklisted(now, exec))
+          listenerBus.post(SparkListenerExecutorUnexcluded(now, exec))
+          failedExecsOnNode.remove(exec)
+          if (failedExecsOnNode.isEmpty) {
+            nodeToExcludedExecs.remove(status.node)
+          }
+        }
+      }
+      val nodesToInclude = nodeIdToExcludedExpiryTime.filter(_._2 < now).keys
+      if (nodesToInclude.nonEmpty) {
+        // Include any nodes that have been excluded longer than the excludeOnFailure timeout.
+        logInfo(s"Removing nodes $nodesToInclude from exclude list because the " +
+          s"nodes have reached has timed out")
+        nodesToInclude.foreach { node =>
+          nodeIdToExcludedExpiryTime.remove(node)
+          // post both to keep backwards compatibility
+          listenerBus.post(SparkListenerNodeUnblacklisted(now, node))
+          listenerBus.post(SparkListenerNodeUnexcluded(now, node))
+        }
+        _excludedNodeList.set(nodeIdToExcludedExpiryTime.keySet.toSet)
+      }
+      updateNextExpiryTime()
+    }
+  }
+
+  private def updateNextExpiryTime(): Unit = {
+    val execMinExpiry = if (executorIdToExcludedStatus.nonEmpty) {
+      executorIdToExcludedStatus.map{_._2.expiryTime}.min
+    } else {
+      Long.MaxValue
+    }
+    val nodeMinExpiry = if (nodeIdToExcludedExpiryTime.nonEmpty) {
+      nodeIdToExcludedExpiryTime.values.min
+    } else {
+      Long.MaxValue
+    }
+    nextExpiryTime = math.min(execMinExpiry, nodeMinExpiry)
+  }
+
+  private def killExecutor(exec: String, msg: String): Unit = {
+    val fullMsg = if (EXCLUDE_ON_FAILURE_DECOMMISSION_ENABLED) {
+      s"${msg} (actually decommissioning)"
+    } else {
+      msg
+    }
+    allocationClient match {
+      case Some(a) =>
+        logInfo(fullMsg)
+        if (EXCLUDE_ON_FAILURE_DECOMMISSION_ENABLED) {
+          a.decommissionExecutor(exec, ExecutorDecommissionInfo(fullMsg),
+            adjustTargetNumExecutors = false)
+        } else {
+          a.killExecutors(Seq(exec), adjustTargetNumExecutors = false, countFailures = false,
+            force = true)
+        }
+      case None =>
+        logInfo(s"Not attempting to kill excluded executor id $exec " +
+          s"since allocation client is not defined.")
+    }
+  }
+
+  private def killExcludedExecutor(exec: String): Unit = {
+    if (conf.get(config.EXCLUDE_ON_FAILURE_KILL_ENABLED)) {
+      killExecutor(exec, s"Killing excluded executor id $exec since " +
+        s"${config.EXCLUDE_ON_FAILURE_KILL_ENABLED.key} is set.")
+    }
+  }
+
+  private[scheduler] def killExcludedIdleExecutor(exec: String): Unit = {
+    killExecutor(exec,
+      s"Killing excluded idle executor id $exec because of task unschedulability and trying " +
+        "to acquire a new executor.")
+  }
+
+  private def killExecutorsOnExcludedNode(node: String): Unit = {
+    if (conf.get(config.EXCLUDE_ON_FAILURE_KILL_ENABLED)) {
+      allocationClient match {
+        case Some(a) =>
+          if (EXCLUDE_ON_FAILURE_DECOMMISSION_ENABLED) {
+            logInfo(s"Decommissioning all executors on excluded host $node " +
+              s"since ${config.EXCLUDE_ON_FAILURE_KILL_ENABLED.key} is set.")
+            if (!a.decommissionExecutorsOnHost(node)) {
+              logError(s"Decommissioning executors on $node failed.")
+            }
+          } else {
+            logInfo(s"Killing all executors on excluded host $node " +
+              s"since ${config.EXCLUDE_ON_FAILURE_KILL_ENABLED.key} is set.")
+            if (!a.killExecutorsOnHost(node)) {
+              logError(s"Killing executors on node $node failed.")
+            }
+          }
+        case None =>
+          logWarning(s"Not attempting to kill executors on excluded host $node " +
+            s"since allocation client is not defined.")
+      }
+    }
+  }
+
+  def updateExcludedForFetchFailure(host: String, exec: String): Unit = {
+    if (EXCLUDE_FETCH_FAILURE_ENABLED) {
+      // If we exclude on fetch failures, we are implicitly saying that we believe the failure is
+      // non-transient, and can't be recovered from (even if this is the first fetch failure,
+      // stage is retried after just one failure, so we don't always get a chance to collect
+      // multiple fetch failures).
+      // If the external shuffle-service is on, then every other executor on this node would
+      // be suffering from the same issue, so we should exclude (and potentially kill) all
+      // of them immediately.
+
+      val now = clock.getTimeMillis()
+      val expiryTimeForNewExcludes = now + EXCLUDE_ON_FAILURE_TIMEOUT_MILLIS
+
+      if (conf.get(config.SHUFFLE_SERVICE_ENABLED)) {
+        if (!nodeIdToExcludedExpiryTime.contains(host)) {
+          logInfo(s"excluding node $host due to fetch failure of external shuffle service")
+
+          nodeIdToExcludedExpiryTime.put(host, expiryTimeForNewExcludes)
+          // post both to keep backwards compatibility
+          listenerBus.post(SparkListenerNodeBlacklisted(now, host, 1))
+          listenerBus.post(SparkListenerNodeExcluded(now, host, 1))
+          _excludedNodeList.set(nodeIdToExcludedExpiryTime.keySet.toSet)
+          killExecutorsOnExcludedNode(host)
+          updateNextExpiryTime()
+        }
+      } else if (!executorIdToExcludedStatus.contains(exec)) {
+        logInfo(s"Excluding executor $exec due to fetch failure")
+
+        executorIdToExcludedStatus.put(exec, ExcludedExecutor(host, expiryTimeForNewExcludes))
+        // We hardcoded number of failure tasks to 1 for fetch failure, because there's no
+        // reattempt for such failure.
+        // post both to keep backwards compatibility
+        listenerBus.post(SparkListenerExecutorBlacklisted(now, exec, 1))
+        listenerBus.post(SparkListenerExecutorExcluded(now, exec, 1))
+        updateNextExpiryTime()
+        killExcludedExecutor(exec)
+
+        val excludedExecsOnNode = nodeToExcludedExecs.getOrElseUpdate(host, HashSet[String]())
+        excludedExecsOnNode += exec
+      }
+    }
+  }
+
+  def updateExcludedForSuccessfulTaskSet(
+      stageId: Int,
+      stageAttemptId: Int,
+      failuresByExec: HashMap[String, ExecutorFailuresInTaskSet]): Unit = {
+    // if any tasks failed, we count them towards the overall failure count for the executor at
+    // this point.
+    val now = clock.getTimeMillis()
+    failuresByExec.foreach { case (exec, failuresInTaskSet) =>
+      val appFailuresOnExecutor =
+        executorIdToFailureList.getOrElseUpdate(exec, new ExecutorFailureList)
+      appFailuresOnExecutor.addFailures(stageId, stageAttemptId, failuresInTaskSet)
+      appFailuresOnExecutor.dropFailuresWithTimeoutBefore(now)
+      val newTotal = appFailuresOnExecutor.numUniqueTaskFailures
+
+      val expiryTimeForNewExcludes = now + EXCLUDE_ON_FAILURE_TIMEOUT_MILLIS
+      // If this pushes the total number of failures over the threshold, exclude the executor.
+      // If its already excluded, we avoid "re-excluding" (which can happen if there were
+      // other tasks already running in another taskset when it got excluded), because it makes
+      // some of the logic around expiry times a little more confusing.  But it also wouldn't be a
+      // problem to re-exclude, with a later expiry time.
+      if (newTotal >= MAX_FAILURES_PER_EXEC && !executorIdToExcludedStatus.contains(exec)) {
+        logInfo(s"Excluding executor id: $exec because it has $newTotal" +
+          s" task failures in successful task sets")
+        val node = failuresInTaskSet.node
+        executorIdToExcludedStatus.put(exec, ExcludedExecutor(node, expiryTimeForNewExcludes))
+        // post both to keep backwards compatibility
+        listenerBus.post(SparkListenerExecutorBlacklisted(now, exec, newTotal))
+        listenerBus.post(SparkListenerExecutorExcluded(now, exec, newTotal))
+        executorIdToFailureList.remove(exec)
+        updateNextExpiryTime()
+        killExcludedExecutor(exec)
+
+        // In addition to excluding the executor, we also update the data for failures on the
+        // node, and potentially exclude the entire node as well.
+        val excludedExecsOnNode = nodeToExcludedExecs.getOrElseUpdate(node, HashSet[String]())
+        excludedExecsOnNode += exec
+        // If the node is already excluded, we avoid adding it again with a later expiry
+        // time.
+        if (excludedExecsOnNode.size >= MAX_FAILED_EXEC_PER_NODE &&
+            !nodeIdToExcludedExpiryTime.contains(node)) {
+          logInfo(s"Excluding node $node because it has ${excludedExecsOnNode.size} " +
+            s"executors excluded: ${excludedExecsOnNode}")
+          nodeIdToExcludedExpiryTime.put(node, expiryTimeForNewExcludes)
+          // post both to keep backwards compatibility
+          listenerBus.post(SparkListenerNodeBlacklisted(now, node, excludedExecsOnNode.size))
+          listenerBus.post(SparkListenerNodeExcluded(now, node, excludedExecsOnNode.size))
+          _excludedNodeList.set(nodeIdToExcludedExpiryTime.keySet.toSet)
+          killExecutorsOnExcludedNode(node)
+        }
+      }
+    }
+  }
+
+  def isExecutorExcluded(executorId: String): Boolean = {
+    executorIdToExcludedStatus.contains(executorId)
+  }
+
+  /**
+   * Get the full set of nodes that are excluded.  Unlike other methods in this class, this *IS*
+   * thread-safe -- no lock required on a taskScheduler.
+   */
+  def excludedNodeList(): Set[String] = {
+    _excludedNodeList.get()
+  }
+
+  def isNodeExcluded(node: String): Boolean = {
+    nodeIdToExcludedExpiryTime.contains(node)
+  }
+
+  def handleRemovedExecutor(executorId: String): Unit = {
+    // We intentionally do not clean up executors that are already excluded in
+    // nodeToExcludedExecs, so that if another executor on the same node gets excluded, we can
+    // exclude the entire node. We also can't clean up executorIdToExcludedStatus, so we can
+    // eventually remove the executor after the timeout. Despite not clearing those structures
+    // here, we don't expect they will grow too big since you won't get too many executors on one
+    // node, and the timeout will clear it up periodically in any case.
+    executorIdToFailureList -= executorId
+  }
+
+  /**
+   * Tracks all failures for one executor (that have not passed the timeout).
+   *
+   * In general we actually expect this to be extremely small, since it won't contain more than the
+   * maximum number of task failures before an executor is failed (default 2).
+   */
+  private[scheduler] final class ExecutorFailureList extends Logging {
+
+    private case class TaskId(stage: Int, stageAttempt: Int, taskIndex: Int)
+
+    /**
+     * All failures on this executor in successful task sets.
+     */
+    private var failuresAndExpiryTimes = ArrayBuffer[(TaskId, Long)]()
+    /**
+     * As an optimization, we track the min expiry time over all entries in failuresAndExpiryTimes
+     * so its quick to tell if there are any failures with expiry before the current time.
+     */
+    private var minExpiryTime = Long.MaxValue
+
+    def addFailures(
+        stage: Int,
+        stageAttempt: Int,
+        failuresInTaskSet: ExecutorFailuresInTaskSet): Unit = {
+      failuresInTaskSet.taskToFailureCountAndFailureTime.foreach {
+        case (taskIdx, (_, failureTime)) =>
+          val expiryTime = failureTime + EXCLUDE_ON_FAILURE_TIMEOUT_MILLIS
+          failuresAndExpiryTimes += ((TaskId(stage, stageAttempt, taskIdx), expiryTime))
+          if (expiryTime < minExpiryTime) {
+            minExpiryTime = expiryTime
+          }
+      }
+    }
+
+    /**
+     * The number of unique tasks that failed on this executor.  Only counts failures within the
+     * timeout, and in successful tasksets.
+     */
+    def numUniqueTaskFailures: Int = failuresAndExpiryTimes.size
+
+    def isEmpty: Boolean = failuresAndExpiryTimes.isEmpty
+
+    /**
+     * Apply the timeout to individual tasks.  This is to prevent one-off failures that are very
+     * spread out in time (and likely have nothing to do with problems on the executor) from
+     * triggering exlusion.  However, note that we do *not* remove executors and nodes from
+     * being excluded as we expire individual task failures -- each have their own timeout.  E.g.,
+     * suppose:
+     *  * timeout = 10, maxFailuresPerExec = 2
+     *  * Task 1 fails on exec 1 at time 0
+     *  * Task 2 fails on exec 1 at time 5
+     * -->  exec 1 is excluded from time 5 - 15.
+     * This is to simplify the implementation, as well as keep the behavior easier to understand
+     * for the end user.
+     */
+    def dropFailuresWithTimeoutBefore(dropBefore: Long): Unit = {
+      if (minExpiryTime < dropBefore) {
+        var newMinExpiry = Long.MaxValue
+        val newFailures = new ArrayBuffer[(TaskId, Long)]
+        failuresAndExpiryTimes.foreach { case (task, expiryTime) =>
+          if (expiryTime >= dropBefore) {
+            newFailures += ((task, expiryTime))
+            if (expiryTime < newMinExpiry) {
+              newMinExpiry = expiryTime
+            }
+          }
+        }
+        failuresAndExpiryTimes = newFailures
+        minExpiryTime = newMinExpiry
+      }
+    }
+
+    override def toString(): String = {
+      s"failures = $failuresAndExpiryTimes"
+    }
+  }
+
+}
+
+private[spark] object HealthTracker extends Logging {
+
+  private val DEFAULT_TIMEOUT = "1h"
+
+  /**
+   * Returns true if the excludeOnFailure is enabled, based on checking the configuration
+   * in the following order:
+   * 1. Is it specifically enabled or disabled?
+   * 2. Is it enabled via the legacy timeout conf?
+   * 3. Default is off
+   */
+  def isExcludeOnFailureEnabled(conf: SparkConf): Boolean = {
+    conf.get(config.EXCLUDE_ON_FAILURE_ENABLED) match {
+      case Some(enabled) =>
+        enabled
+      case None =>
+        // if they've got a non-zero setting for the legacy conf, always enable it,
+        // otherwise, use the default.
+        val legacyKey = config.EXCLUDE_ON_FAILURE_LEGACY_TIMEOUT_CONF.key
+        conf.get(config.EXCLUDE_ON_FAILURE_LEGACY_TIMEOUT_CONF).exists { legacyTimeout =>
+          if (legacyTimeout == 0) {
+            logWarning(s"Turning off excludeOnFailure due to legacy configuration: $legacyKey == 0")
+            false
+          } else {
+            logWarning(s"Turning on excludeOnFailure due to legacy configuration: $legacyKey > 0")
+            true
+          }
+        }
+    }
+  }
+
+  def getExludeOnFailureTimeout(conf: SparkConf): Long = {
+    conf.get(config.EXCLUDE_ON_FAILURE_TIMEOUT_CONF).getOrElse {
+      conf.get(config.EXCLUDE_ON_FAILURE_LEGACY_TIMEOUT_CONF).getOrElse {
+        Utils.timeStringAsMs(DEFAULT_TIMEOUT)
+      }
+    }
+  }
+
+  /**
+   * Verify that exclude on failure configurations are consistent; if not, throw an exception.
+   * Should only be called if excludeOnFailure is enabled.
+   *
+   * The configuration is expected to adhere to a few invariants.  Default values
+   * follow these rules of course, but users may unwittingly change one configuration
+   * without making the corresponding adjustment elsewhere. This ensures we fail-fast when
+   * there are such misconfigurations.
+   */
+  def validateExcludeOnFailureConfs(conf: SparkConf): Unit = {
+
+    def mustBePos(k: String, v: String): Unit = {
+      throw new IllegalArgumentException(s"$k was $v, but must be > 0.")
+    }
+
+    Seq(
+      config.MAX_TASK_ATTEMPTS_PER_EXECUTOR,
+      config.MAX_TASK_ATTEMPTS_PER_NODE,
+      config.MAX_FAILURES_PER_EXEC_STAGE,
+      config.MAX_FAILED_EXEC_PER_NODE_STAGE,
+      config.MAX_FAILURES_PER_EXEC,
+      config.MAX_FAILED_EXEC_PER_NODE
+    ).foreach { config =>
+      val v = conf.get(config)
+      if (v <= 0) {
+        mustBePos(config.key, v.toString)
+      }
+    }
+
+    val timeout = getExludeOnFailureTimeout(conf)
+    if (timeout <= 0) {
+      // first, figure out where the timeout came from, to include the right conf in the message.
+      conf.get(config.EXCLUDE_ON_FAILURE_TIMEOUT_CONF) match {
+        case Some(t) =>
+          mustBePos(config.EXCLUDE_ON_FAILURE_TIMEOUT_CONF.key, timeout.toString)
+        case None =>
+          mustBePos(config.EXCLUDE_ON_FAILURE_LEGACY_TIMEOUT_CONF.key, timeout.toString)
+      }
+    }
+
+    val maxTaskFailures = conf.get(config.TASK_MAX_FAILURES)
+    val maxNodeAttempts = conf.get(config.MAX_TASK_ATTEMPTS_PER_NODE)
+
+    if (maxNodeAttempts >= maxTaskFailures) {
+      throw new IllegalArgumentException(s"${config.MAX_TASK_ATTEMPTS_PER_NODE.key} " +
+        s"( = ${maxNodeAttempts}) was >= ${config.TASK_MAX_FAILURES.key} " +
+        s"( = ${maxTaskFailures} ). Though excludeOnFailure is enabled, with this configuration, " +
+        s"Spark will not be robust to one bad node. Decrease " +
+        s"${config.MAX_TASK_ATTEMPTS_PER_NODE.key}, increase ${config.TASK_MAX_FAILURES.key}, " +
+        s"or disable excludeOnFailure with ${config.EXCLUDE_ON_FAILURE_ENABLED.key}")
+    }
+  }
+}
+
+private final case class ExcludedExecutor(node: String, expiryTime: Long)
diff --git a/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala b/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala
index 95b0096cade38..f13f1eaeeaa43 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala
@@ -232,7 +232,7 @@ private[spark] class LiveListenerBus(conf: SparkConf) {
 
   // For testing only.
   private[spark] def findListenersByClass[T <: SparkListenerInterface : ClassTag](): Seq[T] = {
-    queues.asScala.flatMap { queue => queue.findListenersByClass[T]() }
+    queues.asScala.flatMap { queue => queue.findListenersByClass[T]() }.toSeq
   }
 
   // For testing only.
diff --git a/core/src/main/scala/org/apache/spark/scheduler/MapStatus.scala b/core/src/main/scala/org/apache/spark/scheduler/MapStatus.scala
index 7f8893ff3b9d8..07eed76805dd2 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/MapStatus.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/MapStatus.scala
@@ -28,14 +28,23 @@ import org.apache.spark.internal.config
 import org.apache.spark.storage.BlockManagerId
 import org.apache.spark.util.Utils
 
+/**
+ * A common trait between [[MapStatus]] and [[MergeStatus]]. This allows us to reuse existing
+ * code to handle MergeStatus inside MapOutputTracker.
+ */
+private[spark] trait ShuffleOutputStatus
+
 /**
  * Result returned by a ShuffleMapTask to a scheduler. Includes the block manager address that the
- * task ran on as well as the sizes of outputs for each reducer, for passing on to the reduce tasks.
+ * task has shuffle files stored on as well as the sizes of outputs for each reducer, for passing
+ * on to the reduce tasks.
  */
-private[spark] sealed trait MapStatus {
-  /** Location where this task was run. */
+private[spark] sealed trait MapStatus extends ShuffleOutputStatus {
+  /** Location where this task output is. */
   def location: BlockManagerId
 
+  def updateLocation(newLoc: BlockManagerId): Unit
+
   /**
    * Estimated size for the reduce block, in bytes.
    *
@@ -120,12 +129,16 @@ private[spark] class CompressedMapStatus(
   // For deserialization only
   protected def this() = this(null, null.asInstanceOf[Array[Byte]], -1)
 
-  def this(loc: BlockManagerId, uncompressedSizes: Array[Long], mapTaskId: Long) {
+  def this(loc: BlockManagerId, uncompressedSizes: Array[Long], mapTaskId: Long) = {
     this(loc, uncompressedSizes.map(MapStatus.compressSize), mapTaskId)
   }
 
   override def location: BlockManagerId = loc
 
+  override def updateLocation(newLoc: BlockManagerId): Unit = {
+    loc = newLoc
+  }
+
   override def getSizeForBlock(reduceId: Int): Long = {
     MapStatus.decompressSize(compressedSizes(reduceId))
   }
@@ -178,6 +191,10 @@ private[spark] class HighlyCompressedMapStatus private (
 
   override def location: BlockManagerId = loc
 
+  override def updateLocation(newLoc: BlockManagerId): Unit = {
+    loc = newLoc
+  }
+
   override def getSizeForBlock(reduceId: Int): Long = {
     assert(hugeBlockSizes != null)
     if (emptyBlocks.contains(reduceId)) {
@@ -194,7 +211,7 @@ private[spark] class HighlyCompressedMapStatus private (
 
   override def writeExternal(out: ObjectOutput): Unit = Utils.tryOrIOException {
     loc.writeExternal(out)
-    emptyBlocks.writeExternal(out)
+    emptyBlocks.serialize(out)
     out.writeLong(avgSize)
     out.writeInt(hugeBlockSizes.size)
     hugeBlockSizes.foreach { kv =>
@@ -206,8 +223,9 @@ private[spark] class HighlyCompressedMapStatus private (
 
   override def readExternal(in: ObjectInput): Unit = Utils.tryOrIOException {
     loc = BlockManagerId(in)
+    numNonEmptyBlocks = -1 // SPARK-32436 Scala 2.13 doesn't initialize this during deserialization
     emptyBlocks = new RoaringBitmap()
-    emptyBlocks.readExternal(in)
+    emptyBlocks.deserialize(in)
     avgSize = in.readLong()
     val count = in.readInt()
     val hugeBlockSizesImpl = mutable.Map.empty[Int, Byte]
diff --git a/core/src/main/scala/org/apache/spark/scheduler/MergeStatus.scala b/core/src/main/scala/org/apache/spark/scheduler/MergeStatus.scala
new file mode 100644
index 0000000000000..77d8f8e040da1
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/scheduler/MergeStatus.scala
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.scheduler
+
+import java.io.{Externalizable, ObjectInput, ObjectOutput}
+
+import org.roaringbitmap.RoaringBitmap
+
+import org.apache.spark.network.shuffle.protocol.MergeStatuses
+import org.apache.spark.storage.BlockManagerId
+import org.apache.spark.util.Utils
+
+/**
+ * The status for the result of merging shuffle partition blocks per individual shuffle partition
+ * maintained by the scheduler. The scheduler would separate the
+ * [[org.apache.spark.network.shuffle.protocol.MergeStatuses]] received from
+ * ExternalShuffleService into individual [[MergeStatus]] which is maintained inside
+ * MapOutputTracker to be served to the reducers when they start fetching shuffle partition
+ * blocks. Note that, the reducers are ultimately fetching individual chunks inside a merged
+ * shuffle file, as explained in [[org.apache.spark.network.shuffle.RemoteBlockPushResolver]].
+ * Between the scheduler maintained MergeStatus and the shuffle service maintained per shuffle
+ * partition meta file, we are effectively dividing the metadata for a push-based shuffle into
+ * 2 layers. The scheduler would track the top-level metadata at the shuffle partition level
+ * with MergeStatus, and the shuffle service would maintain the partition level metadata about
+ * how to further divide a merged shuffle partition into multiple chunks with the per-partition
+ * meta file. This helps to reduce the amount of data the scheduler needs to maintain for
+ * push-based shuffle.
+ */
+private[spark] class MergeStatus(
+    private[this] var loc: BlockManagerId,
+    private[this] var mapTracker: RoaringBitmap,
+    private[this] var size: Long)
+  extends Externalizable with ShuffleOutputStatus {
+
+  protected def this() = this(null, null, -1) // For deserialization only
+
+  def location: BlockManagerId = loc
+
+  def totalSize: Long = size
+
+  def tracker: RoaringBitmap = mapTracker
+
+  /**
+   * Get the list of mapper IDs for missing mapper partition blocks that are not merged.
+   * The reducer will use this information to decide which shuffle partition blocks to
+   * fetch in the original way.
+   */
+  def getMissingMaps(numMaps: Int): Seq[Int] = {
+    (0 until numMaps).filter(i => !mapTracker.contains(i))
+  }
+
+  /**
+   * Get the number of missing map outputs for missing mapper partition blocks that are not merged.
+   */
+  def getNumMissingMapOutputs(numMaps: Int): Int = {
+    (0 until numMaps).count(i => !mapTracker.contains(i))
+  }
+
+  override def writeExternal(out: ObjectOutput): Unit = Utils.tryOrIOException {
+    loc.writeExternal(out)
+    mapTracker.writeExternal(out)
+    out.writeLong(size)
+  }
+
+  override def readExternal(in: ObjectInput): Unit = Utils.tryOrIOException {
+    loc = BlockManagerId(in)
+    mapTracker = new RoaringBitmap()
+    mapTracker.readExternal(in)
+    size = in.readLong()
+  }
+}
+
+private[spark] object MergeStatus {
+  // Dummy number of reduces for the tests where push based shuffle is not enabled
+  val SHUFFLE_PUSH_DUMMY_NUM_REDUCES = 1
+
+  /**
+   * Separate a MergeStatuses received from an ExternalShuffleService into individual
+   * MergeStatus. The scheduler is responsible for providing the location information
+   * for the given ExternalShuffleService.
+   */
+  def convertMergeStatusesToMergeStatusArr(
+      mergeStatuses: MergeStatuses,
+      loc: BlockManagerId): Seq[(Int, MergeStatus)] = {
+    assert(mergeStatuses.bitmaps.length == mergeStatuses.reduceIds.length &&
+      mergeStatuses.bitmaps.length == mergeStatuses.sizes.length)
+    val mergerLoc = BlockManagerId(BlockManagerId.SHUFFLE_MERGER_IDENTIFIER, loc.host, loc.port)
+    mergeStatuses.bitmaps.zipWithIndex.map {
+      case (bitmap, index) =>
+        val mergeStatus = new MergeStatus(mergerLoc, bitmap, mergeStatuses.sizes(index))
+        (mergeStatuses.reduceIds(index), mergeStatus)
+    }
+  }
+
+  def apply(loc: BlockManagerId, bitmap: RoaringBitmap, size: Long): MergeStatus = {
+    new MergeStatus(loc, bitmap, size)
+  }
+}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/MiscellaneousProcessDetails.scala b/core/src/main/scala/org/apache/spark/scheduler/MiscellaneousProcessDetails.scala
new file mode 100644
index 0000000000000..0e5fe0ad4517a
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/scheduler/MiscellaneousProcessDetails.scala
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.scheduler
+
+import org.apache.spark.annotation.DeveloperApi
+
+/**
+ * :: DeveloperApi ::
+ * Stores information about an Miscellaneous Process to pass from the scheduler to SparkListeners.
+ */
+
+@DeveloperApi
+class MiscellaneousProcessDetails(
+    val hostPort: String,
+    val cores: Int,
+    val logUrlInfo: Map[String, String]) extends Serializable
diff --git a/core/src/main/scala/org/apache/spark/scheduler/OutputCommitCoordinator.scala b/core/src/main/scala/org/apache/spark/scheduler/OutputCommitCoordinator.scala
index b382d623806e2..a5858ebf9cdcc 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/OutputCommitCoordinator.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/OutputCommitCoordinator.scala
@@ -151,7 +151,7 @@ private[spark] class OutputCommitCoordinator(conf: SparkConf, isDriver: Boolean)
         logInfo(s"Task was denied committing, stage: $stage.$stageAttempt, " +
           s"partition: $partition, attempt: $attemptNumber")
       case _ =>
-        // Mark the attempt as failed to blacklist from future commit protocol
+        // Mark the attempt as failed to exclude from future commit protocol
         val taskId = TaskIdentifier(stageAttempt, attemptNumber)
         stageState.failures.getOrElseUpdate(partition, mutable.Set()) += taskId
         if (stageState.authorizedCommitters(partition) == taskId) {
diff --git a/core/src/main/scala/org/apache/spark/scheduler/Pool.scala b/core/src/main/scala/org/apache/spark/scheduler/Pool.scala
index 2e2851eb9070b..de4c9d39ddb51 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/Pool.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/Pool.scala
@@ -59,6 +59,8 @@ private[spark] class Pool(
     }
   }
 
+  override def isSchedulable: Boolean = true
+
   override def addSchedulable(schedulable: Schedulable): Unit = {
     require(schedulable != null)
     schedulableQueue.add(schedulable)
@@ -92,7 +94,7 @@ private[spark] class Pool(
     schedulableQueue.asScala.foreach(_.executorDecommission(executorId))
   }
 
-  override def checkSpeculatableTasks(minTimeToSpeculation: Int): Boolean = {
+  override def checkSpeculatableTasks(minTimeToSpeculation: Long): Boolean = {
     var shouldRevive = false
     for (schedulable <- schedulableQueue.asScala) {
       shouldRevive |= schedulable.checkSpeculatableTasks(minTimeToSpeculation)
@@ -105,7 +107,7 @@ private[spark] class Pool(
     val sortedSchedulableQueue =
       schedulableQueue.asScala.toSeq.sortWith(taskSetSchedulingAlgorithm.comparator)
     for (schedulable <- sortedSchedulableQueue) {
-      sortedTaskSetQueue ++= schedulable.getSortedTaskSetQueue
+      sortedTaskSetQueue ++= schedulable.getSortedTaskSetQueue.filter(_.isSchedulable)
     }
     sortedTaskSetQueue
   }
diff --git a/core/src/main/scala/org/apache/spark/scheduler/Schedulable.scala b/core/src/main/scala/org/apache/spark/scheduler/Schedulable.scala
index 8cc239c81d11a..e549ce62e74f1 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/Schedulable.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/Schedulable.scala
@@ -39,11 +39,12 @@ private[spark] trait Schedulable {
   def stageId: Int
   def name: String
 
+  def isSchedulable: Boolean
   def addSchedulable(schedulable: Schedulable): Unit
   def removeSchedulable(schedulable: Schedulable): Unit
   def getSchedulableByName(name: String): Schedulable
   def executorLost(executorId: String, host: String, reason: ExecutorLossReason): Unit
   def executorDecommission(executorId: String): Unit
-  def checkSpeculatableTasks(minTimeToSpeculation: Int): Boolean
+  def checkSpeculatableTasks(minTimeToSpeculation: Long): Boolean
   def getSortedTaskSetQueue: ArrayBuffer[TaskSetManager]
 }
diff --git a/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala b/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala
index 8f6a22177a5b8..e7c45a9faa1d0 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala
@@ -17,13 +17,15 @@
 
 package org.apache.spark.scheduler
 
-import java.io.{FileInputStream, InputStream}
+import java.io.InputStream
 import java.util.{Locale, NoSuchElementException, Properties}
 
 import scala.util.control.NonFatal
 import scala.xml.{Node, XML}
 
-import org.apache.spark.{SparkConf, SparkContext}
+import org.apache.hadoop.fs.Path
+
+import org.apache.spark.SparkContext
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.SCHEDULER_ALLOCATION_FILE
 import org.apache.spark.scheduler.SchedulingMode.SchedulingMode
@@ -54,10 +56,10 @@ private[spark] class FIFOSchedulableBuilder(val rootPool: Pool)
   }
 }
 
-private[spark] class FairSchedulableBuilder(val rootPool: Pool, conf: SparkConf)
+private[spark] class FairSchedulableBuilder(val rootPool: Pool, sc: SparkContext)
   extends SchedulableBuilder with Logging {
 
-  val schedulerAllocFile = conf.get(SCHEDULER_ALLOCATION_FILE)
+  val schedulerAllocFile = sc.conf.get(SCHEDULER_ALLOCATION_FILE)
   val DEFAULT_SCHEDULER_FILE = "fairscheduler.xml"
   val FAIR_SCHEDULER_PROPERTIES = SparkContext.SPARK_SCHEDULER_POOL
   val DEFAULT_POOL_NAME = "default"
@@ -74,7 +76,8 @@ private[spark] class FairSchedulableBuilder(val rootPool: Pool, conf: SparkConf)
     var fileData: Option[(InputStream, String)] = None
     try {
       fileData = schedulerAllocFile.map { f =>
-        val fis = new FileInputStream(f)
+        val filePath = new Path(f)
+        val fis = filePath.getFileSystem(sc.hadoopConfiguration).open(filePath)
         logInfo(s"Creating Fair Scheduler pools from $f")
         Some((fis, f))
       }.getOrElse {
diff --git a/core/src/main/scala/org/apache/spark/scheduler/SchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/SchedulerBackend.scala
index a5bba645be14c..b2acdb3e12a6d 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/SchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/SchedulerBackend.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.scheduler
 
 import org.apache.spark.resource.ResourceProfile
+import org.apache.spark.storage.BlockManagerId
 
 /**
  * A backend interface for scheduling systems that allows plugging in different ones under
@@ -83,7 +84,7 @@ private[spark] trait SchedulerBackend {
 
   /**
    * Get the max number of tasks that can be concurrent launched based on the ResourceProfile
-   * being used.
+   * could be used, even if some of them are being used at the moment.
    * Note that please don't cache the value returned by this method, because the number can change
    * due to add/remove executors.
    *
@@ -92,4 +93,16 @@ private[spark] trait SchedulerBackend {
    */
   def maxNumConcurrentTasks(rp: ResourceProfile): Int
 
+  /**
+   * Get the list of host locations for push based shuffle
+   *
+   * Currently push based shuffle is disabled for both stage retry and stage reuse cases
+   * (for eg: in the case where few partitions are lost due to failure). Hence this method
+   * should be invoked only once for a ShuffleDependency.
+   * @return List of external shuffle services locations
+   */
+  def getShufflePushMergerLocations(
+      numPartitions: Int,
+      resourceProfileId: Int): Seq[BlockManagerId] = Nil
+
 }
diff --git a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapStage.scala b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapStage.scala
index be1984de9837f..db09d19d0acff 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapStage.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapStage.scala
@@ -52,7 +52,7 @@ private[spark] class ShuffleMapStage(
    * Partitions that either haven't yet been computed, or that were computed on an executor
    * that has since been lost, so should be re-computed.  This variable is used by the
    * DAGScheduler to determine when a stage has completed. Task successes in both the active
-   * attempt for the stage or in earlier attempts for this stage can cause paritition ids to get
+   * attempt for the stage or in earlier attempts for this stage can cause partition ids to get
    * removed from pendingPartitions. As a result, this variable may be inconsistent with the pending
    * tasks in the TaskSetManager for the active attempt for the stage (the partitions stored here
    * will always be a subset of the partitions that the TaskSetManager thinks are pending).
diff --git a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala
index a0ba9208ea647..89db3a86f4ce8 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala
@@ -66,7 +66,7 @@ private[spark] class ShuffleMapTask(
   with Logging {
 
   /** A constructor used only in test suites. This does not require passing in an RDD. */
-  def this(partitionId: Int) {
+  def this(partitionId: Int) = {
     this(0, 0, null, new Partition { override def index: Int = 0 }, null, new Properties, null)
   }
 
diff --git a/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala b/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala
index c150b0341500c..d00866edf7503 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala
@@ -25,8 +25,9 @@ import scala.collection.Map
 import com.fasterxml.jackson.annotation.JsonTypeInfo
 
 import org.apache.spark.TaskEndReason
-import org.apache.spark.annotation.DeveloperApi
+import org.apache.spark.annotation.{DeveloperApi, Since}
 import org.apache.spark.executor.{ExecutorMetrics, TaskMetrics}
+import org.apache.spark.resource.ResourceProfile
 import org.apache.spark.scheduler.cluster.ExecutorInfo
 import org.apache.spark.storage.{BlockManagerId, BlockUpdatedInfo}
 
@@ -117,12 +118,22 @@ case class SparkListenerExecutorRemoved(time: Long, executorId: String, reason:
   extends SparkListenerEvent
 
 @DeveloperApi
+@deprecated("use SparkListenerExecutorExcluded instead", "3.1.0")
 case class SparkListenerExecutorBlacklisted(
     time: Long,
     executorId: String,
     taskFailures: Int)
   extends SparkListenerEvent
 
+@DeveloperApi
+@Since("3.1.0")
+case class SparkListenerExecutorExcluded(
+    time: Long,
+    executorId: String,
+    taskFailures: Int)
+  extends SparkListenerEvent
+
+@deprecated("use SparkListenerExecutorExcludedForStage instead", "3.1.0")
 @DeveloperApi
 case class SparkListenerExecutorBlacklistedForStage(
     time: Long,
@@ -132,6 +143,18 @@ case class SparkListenerExecutorBlacklistedForStage(
     stageAttemptId: Int)
   extends SparkListenerEvent
 
+
+@DeveloperApi
+@Since("3.1.0")
+case class SparkListenerExecutorExcludedForStage(
+    time: Long,
+    executorId: String,
+    taskFailures: Int,
+    stageId: Int,
+    stageAttemptId: Int)
+  extends SparkListenerEvent
+
+@deprecated("use SparkListenerNodeExcludedForStage instead", "3.1.0")
 @DeveloperApi
 case class SparkListenerNodeBlacklistedForStage(
     time: Long,
@@ -141,10 +164,28 @@ case class SparkListenerNodeBlacklistedForStage(
     stageAttemptId: Int)
   extends SparkListenerEvent
 
+
+@DeveloperApi
+@Since("3.1.0")
+case class SparkListenerNodeExcludedForStage(
+    time: Long,
+    hostId: String,
+    executorFailures: Int,
+    stageId: Int,
+    stageAttemptId: Int)
+  extends SparkListenerEvent
+
+@deprecated("use SparkListenerExecutorUnexcluded instead", "3.1.0")
 @DeveloperApi
 case class SparkListenerExecutorUnblacklisted(time: Long, executorId: String)
   extends SparkListenerEvent
 
+
+@DeveloperApi
+case class SparkListenerExecutorUnexcluded(time: Long, executorId: String)
+  extends SparkListenerEvent
+
+@deprecated("use SparkListenerNodeExcluded instead", "3.1.0")
 @DeveloperApi
 case class SparkListenerNodeBlacklisted(
     time: Long,
@@ -152,18 +193,51 @@ case class SparkListenerNodeBlacklisted(
     executorFailures: Int)
   extends SparkListenerEvent
 
+
+@DeveloperApi
+@Since("3.1.0")
+case class SparkListenerNodeExcluded(
+    time: Long,
+    hostId: String,
+    executorFailures: Int)
+  extends SparkListenerEvent
+
+@deprecated("use SparkListenerNodeUnexcluded instead", "3.1.0")
 @DeveloperApi
 case class SparkListenerNodeUnblacklisted(time: Long, hostId: String)
   extends SparkListenerEvent
 
+@DeveloperApi
+@Since("3.1.0")
+case class SparkListenerNodeUnexcluded(time: Long, hostId: String)
+  extends SparkListenerEvent
+
+@DeveloperApi
+@Since("3.1.0")
+case class SparkListenerUnschedulableTaskSetAdded(
+  stageId: Int,
+  stageAttemptId: Int) extends SparkListenerEvent
+
+@DeveloperApi
+@Since("3.1.0")
+case class SparkListenerUnschedulableTaskSetRemoved(
+  stageId: Int,
+  stageAttemptId: Int) extends SparkListenerEvent
+
 @DeveloperApi
 case class SparkListenerBlockUpdated(blockUpdatedInfo: BlockUpdatedInfo) extends SparkListenerEvent
 
+@DeveloperApi
+case class SparkListenerMiscellaneousProcessAdded(time: Long, processId: String,
+    info: MiscellaneousProcessDetails) extends SparkListenerEvent
+
 /**
  * Periodic updates from executors.
  * @param execId executor id
  * @param accumUpdates sequence of (taskId, stageId, stageAttemptId, accumUpdates)
  * @param executorUpdates executor level per-stage metrics updates
+ *
+ * @since 3.1.0
  */
 @DeveloperApi
 case class SparkListenerExecutorMetricsUpdate(
@@ -207,6 +281,11 @@ case class SparkListenerApplicationEnd(time: Long) extends SparkListenerEvent
 @DeveloperApi
 case class SparkListenerLogStart(sparkVersion: String) extends SparkListenerEvent
 
+@DeveloperApi
+@Since("3.1.0")
+case class SparkListenerResourceProfileAdded(resourceProfile: ResourceProfile)
+  extends SparkListenerEvent
+
 /**
  * Interface for listening to events from the Spark scheduler. Most applications should probably
  * extend SparkListener or SparkFirehoseListener directly, rather than implementing this class.
@@ -304,36 +383,87 @@ private[spark] trait SparkListenerInterface {
   def onExecutorRemoved(executorRemoved: SparkListenerExecutorRemoved): Unit
 
   /**
-   * Called when the driver blacklists an executor for a Spark application.
+   * Called when the driver excludes an executor for a Spark application.
    */
+  @deprecated("use onExecutorExcluded instead", "3.1.0")
   def onExecutorBlacklisted(executorBlacklisted: SparkListenerExecutorBlacklisted): Unit
 
   /**
-   * Called when the driver blacklists an executor for a stage.
+   * Called when the driver excludes an executor for a Spark application.
+   */
+  def onExecutorExcluded(executorExcluded: SparkListenerExecutorExcluded): Unit
+
+  /**
+   * Called when the driver excludes an executor for a stage.
    */
+  @deprecated("use onExecutorExcludedForStage instead", "3.1.0")
   def onExecutorBlacklistedForStage(
       executorBlacklistedForStage: SparkListenerExecutorBlacklistedForStage): Unit
 
   /**
-   * Called when the driver blacklists a node for a stage.
+   * Called when the driver excludes an executor for a stage.
    */
+  def onExecutorExcludedForStage(
+      executorExcludedForStage: SparkListenerExecutorExcludedForStage): Unit
+
+  /**
+   * Called when the driver excludes a node for a stage.
+   */
+  @deprecated("use onNodeExcludedForStage instead", "3.1.0")
   def onNodeBlacklistedForStage(nodeBlacklistedForStage: SparkListenerNodeBlacklistedForStage): Unit
 
   /**
-   * Called when the driver re-enables a previously blacklisted executor.
+   * Called when the driver excludes a node for a stage.
+   */
+  def onNodeExcludedForStage(nodeExcludedForStage: SparkListenerNodeExcludedForStage): Unit
+
+  /**
+   * Called when the driver re-enables a previously excluded executor.
    */
+  @deprecated("use onExecutorUnexcluded instead", "3.1.0")
   def onExecutorUnblacklisted(executorUnblacklisted: SparkListenerExecutorUnblacklisted): Unit
 
   /**
-   * Called when the driver blacklists a node for a Spark application.
+   * Called when the driver re-enables a previously excluded executor.
+   */
+  def onExecutorUnexcluded(executorUnexcluded: SparkListenerExecutorUnexcluded): Unit
+
+  /**
+   * Called when the driver excludes a node for a Spark application.
    */
+  @deprecated("use onNodeExcluded instead", "3.1.0")
   def onNodeBlacklisted(nodeBlacklisted: SparkListenerNodeBlacklisted): Unit
 
   /**
-   * Called when the driver re-enables a previously blacklisted node.
+   * Called when the driver excludes a node for a Spark application.
    */
+  def onNodeExcluded(nodeExcluded: SparkListenerNodeExcluded): Unit
+
+  /**
+   * Called when the driver re-enables a previously excluded node.
+   */
+  @deprecated("use onNodeUnexcluded instead", "3.1.0")
   def onNodeUnblacklisted(nodeUnblacklisted: SparkListenerNodeUnblacklisted): Unit
 
+  /**
+   * Called when the driver re-enables a previously excluded node.
+   */
+  def onNodeUnexcluded(nodeUnexcluded: SparkListenerNodeUnexcluded): Unit
+
+  /**
+   * Called when a taskset becomes unschedulable due to exludeOnFailure and dynamic allocation
+   * is enabled.
+   */
+  def onUnschedulableTaskSetAdded(
+      unschedulableTaskSetAdded: SparkListenerUnschedulableTaskSetAdded): Unit
+
+  /**
+   * Called when an unschedulable taskset becomes schedulable and dynamic allocation
+   * is enabled.
+   */
+  def onUnschedulableTaskSetRemoved(
+      unschedulableTaskSetRemoved: SparkListenerUnschedulableTaskSetRemoved): Unit
+
   /**
    * Called when the driver receives a block update info.
    */
@@ -348,6 +478,11 @@ private[spark] trait SparkListenerInterface {
    * Called when other events like SQL-specific events are posted.
    */
   def onOtherEvent(event: SparkListenerEvent): Unit
+
+  /**
+   * Called when a Resource Profile is added to the manager.
+   */
+  def onResourceProfileAdded(event: SparkListenerResourceProfileAdded): Unit
 }
 
 
@@ -399,21 +534,39 @@ abstract class SparkListener extends SparkListenerInterface {
 
   override def onExecutorBlacklisted(
       executorBlacklisted: SparkListenerExecutorBlacklisted): Unit = { }
+  override def onExecutorExcluded(
+      executorExcluded: SparkListenerExecutorExcluded): Unit = { }
 
-  def onExecutorBlacklistedForStage(
+  override def onExecutorBlacklistedForStage(
       executorBlacklistedForStage: SparkListenerExecutorBlacklistedForStage): Unit = { }
+  override def onExecutorExcludedForStage(
+      executorExcludedForStage: SparkListenerExecutorExcludedForStage): Unit = { }
 
-  def onNodeBlacklistedForStage(
+  override def onNodeBlacklistedForStage(
       nodeBlacklistedForStage: SparkListenerNodeBlacklistedForStage): Unit = { }
+  override def onNodeExcludedForStage(
+      nodeExcludedForStage: SparkListenerNodeExcludedForStage): Unit = { }
 
   override def onExecutorUnblacklisted(
       executorUnblacklisted: SparkListenerExecutorUnblacklisted): Unit = { }
+  override def onExecutorUnexcluded(
+      executorUnexcluded: SparkListenerExecutorUnexcluded): Unit = { }
 
   override def onNodeBlacklisted(
       nodeBlacklisted: SparkListenerNodeBlacklisted): Unit = { }
+  override def onNodeExcluded(
+      nodeExcluded: SparkListenerNodeExcluded): Unit = { }
 
   override def onNodeUnblacklisted(
       nodeUnblacklisted: SparkListenerNodeUnblacklisted): Unit = { }
+  override def onNodeUnexcluded(
+      nodeUnexcluded: SparkListenerNodeUnexcluded): Unit = { }
+
+  override def onUnschedulableTaskSetAdded(
+      unschedulableTaskSetAdded: SparkListenerUnschedulableTaskSetAdded): Unit = { }
+
+  override def onUnschedulableTaskSetRemoved(
+      unschedulableTaskSetRemoved: SparkListenerUnschedulableTaskSetRemoved): Unit = { }
 
   override def onBlockUpdated(blockUpdated: SparkListenerBlockUpdated): Unit = { }
 
@@ -421,4 +574,6 @@ abstract class SparkListener extends SparkListenerInterface {
       speculativeTask: SparkListenerSpeculativeTaskSubmitted): Unit = { }
 
   override def onOtherEvent(event: SparkListenerEvent): Unit = { }
+
+  override def onResourceProfileAdded(event: SparkListenerResourceProfileAdded): Unit = { }
 }
diff --git a/core/src/main/scala/org/apache/spark/scheduler/SparkListenerBus.scala b/core/src/main/scala/org/apache/spark/scheduler/SparkListenerBus.scala
index 8f6b7ad309602..ec0c0cf3cf82b 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/SparkListenerBus.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/SparkListenerBus.scala
@@ -75,10 +75,28 @@ private[spark] trait SparkListenerBus
         listener.onNodeBlacklisted(nodeBlacklisted)
       case nodeUnblacklisted: SparkListenerNodeUnblacklisted =>
         listener.onNodeUnblacklisted(nodeUnblacklisted)
+      case executorExcludedForStage: SparkListenerExecutorExcludedForStage =>
+        listener.onExecutorExcludedForStage(executorExcludedForStage)
+      case nodeExcludedForStage: SparkListenerNodeExcludedForStage =>
+        listener.onNodeExcludedForStage(nodeExcludedForStage)
+      case executorExcluded: SparkListenerExecutorExcluded =>
+        listener.onExecutorExcluded(executorExcluded)
+      case executorUnexcluded: SparkListenerExecutorUnexcluded =>
+        listener.onExecutorUnexcluded(executorUnexcluded)
+      case nodeExcluded: SparkListenerNodeExcluded =>
+        listener.onNodeExcluded(nodeExcluded)
+      case nodeUnexcluded: SparkListenerNodeUnexcluded =>
+        listener.onNodeUnexcluded(nodeUnexcluded)
       case blockUpdated: SparkListenerBlockUpdated =>
         listener.onBlockUpdated(blockUpdated)
       case speculativeTaskSubmitted: SparkListenerSpeculativeTaskSubmitted =>
         listener.onSpeculativeTaskSubmitted(speculativeTaskSubmitted)
+      case unschedulableTaskSetAdded: SparkListenerUnschedulableTaskSetAdded =>
+        listener.onUnschedulableTaskSetAdded(unschedulableTaskSetAdded)
+      case unschedulableTaskSetRemoved: SparkListenerUnschedulableTaskSetRemoved =>
+        listener.onUnschedulableTaskSetRemoved(unschedulableTaskSetRemoved)
+      case resourceProfileAdded: SparkListenerResourceProfileAdded =>
+        listener.onResourceProfileAdded(resourceProfileAdded)
       case _ => listener.onOtherEvent(event)
     }
   }
diff --git a/core/src/main/scala/org/apache/spark/scheduler/SplitInfo.scala b/core/src/main/scala/org/apache/spark/scheduler/SplitInfo.scala
index bc1431835e258..6112d8ef051e0 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/SplitInfo.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/SplitInfo.scala
@@ -69,7 +69,7 @@ object SplitInfo {
     for (host <- mapredSplit.getLocations) {
       retval += new SplitInfo(inputFormatClazz, host, path, length, mapredSplit)
     }
-    retval
+    retval.toSeq
   }
 
   def toSplitInfo(inputFormatClazz: Class[_], path: String,
@@ -79,6 +79,6 @@ object SplitInfo {
     for (host <- mapreduceSplit.getLocations) {
       retval += new SplitInfo(inputFormatClazz, host, path, length, mapreduceSplit)
     }
-    retval
+    retval.toSeq
   }
 }
diff --git a/core/src/main/scala/org/apache/spark/scheduler/StatsReportListener.scala b/core/src/main/scala/org/apache/spark/scheduler/StatsReportListener.scala
index ca48775e77f27..be881481bf4ff 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/StatsReportListener.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/StatsReportListener.scala
@@ -47,19 +47,19 @@ class StatsReportListener extends SparkListener with Logging {
   override def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit = {
     implicit val sc = stageCompleted
     this.logInfo(s"Finished stage: ${getStatusDetail(stageCompleted.stageInfo)}")
-    showMillisDistribution("task runtime:", (info, _) => info.duration, taskInfoMetrics)
+    showMillisDistribution("task runtime:", (info, _) => info.duration, taskInfoMetrics.toSeq)
 
     // Shuffle write
     showBytesDistribution("shuffle bytes written:",
-      (_, metric) => metric.shuffleWriteMetrics.bytesWritten, taskInfoMetrics)
+      (_, metric) => metric.shuffleWriteMetrics.bytesWritten, taskInfoMetrics.toSeq)
 
     // Fetch & I/O
     showMillisDistribution("fetch wait time:",
-      (_, metric) => metric.shuffleReadMetrics.fetchWaitTime, taskInfoMetrics)
+      (_, metric) => metric.shuffleReadMetrics.fetchWaitTime, taskInfoMetrics.toSeq)
     showBytesDistribution("remote bytes read:",
-      (_, metric) => metric.shuffleReadMetrics.remoteBytesRead, taskInfoMetrics)
+      (_, metric) => metric.shuffleReadMetrics.remoteBytesRead, taskInfoMetrics.toSeq)
     showBytesDistribution("task result size:",
-      (_, metric) => metric.resultSize, taskInfoMetrics)
+      (_, metric) => metric.resultSize, taskInfoMetrics.toSeq)
 
     // Runtime breakdown
     val runtimePcts = taskInfoMetrics.map { case (info, metrics) =>
diff --git a/core/src/main/scala/org/apache/spark/scheduler/Task.scala b/core/src/main/scala/org/apache/spark/scheduler/Task.scala
index ebc1c05435fee..81f984bb2b511 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/Task.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/Task.scala
@@ -23,6 +23,7 @@ import java.util.Properties
 import org.apache.spark._
 import org.apache.spark.executor.TaskMetrics
 import org.apache.spark.internal.config.APP_CALLER_CONTEXT
+import org.apache.spark.internal.plugin.PluginContainer
 import org.apache.spark.memory.{MemoryMode, TaskMemoryManager}
 import org.apache.spark.metrics.MetricsSystem
 import org.apache.spark.rdd.InputFileBlockHolder
@@ -82,7 +83,8 @@ private[spark] abstract class Task[T](
       taskAttemptId: Long,
       attemptNumber: Int,
       metricsSystem: MetricsSystem,
-      resources: Map[String, ResourceInformation]): T = {
+      resources: Map[String, ResourceInformation],
+      plugins: Option[PluginContainer]): T = {
     SparkEnv.get.blockManager.registerTask(taskAttemptId)
     // TODO SPARK-24874 Allow create BarrierTaskContext based on partitions, instead of whether
     // the stage is barrier.
@@ -123,6 +125,8 @@ private[spark] abstract class Task[T](
       Option(taskAttemptId),
       Option(attemptNumber)).setCurrentContext()
 
+    plugins.foreach(_.onTaskStart())
+
     try {
       runTask(context)
     } catch {
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskDescription.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskDescription.scala
index 247cfe721b553..12b911d06153b 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskDescription.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskDescription.scala
@@ -55,11 +55,12 @@ private[spark] class TaskDescription(
     val partitionId: Int,
     val addedFiles: Map[String, Long],
     val addedJars: Map[String, Long],
+    val addedArchives: Map[String, Long],
     val properties: Properties,
     val resources: immutable.Map[String, ResourceInformation],
     val serializedTask: ByteBuffer) {
 
-  override def toString: String = "TaskDescription(TID=%d, index=%d)".format(taskId, index)
+  override def toString: String = s"TaskDescription($name)"
 }
 
 private[spark] object TaskDescription {
@@ -99,6 +100,9 @@ private[spark] object TaskDescription {
     // Write jars.
     serializeStringLongMap(taskDescription.addedJars, dataOut)
 
+    // Write archives.
+    serializeStringLongMap(taskDescription.addedArchives, dataOut)
+
     // Write properties.
     dataOut.writeInt(taskDescription.properties.size())
     taskDescription.properties.asScala.foreach { case (key, value) =>
@@ -167,6 +171,9 @@ private[spark] object TaskDescription {
     // Read jars.
     val taskJars = deserializeStringLongMap(dataIn)
 
+    // Read archives.
+    val taskArchives = deserializeStringLongMap(dataIn)
+
     // Read properties.
     val properties = new Properties()
     val numProperties = dataIn.readInt()
@@ -185,6 +192,6 @@ private[spark] object TaskDescription {
     val serializedTask = byteBuffer.slice()
 
     new TaskDescription(taskId, attemptNumber, executorId, name, index, partitionId, taskFiles,
-      taskJars, properties, resources, serializedTask)
+      taskJars, taskArchives, properties, resources, serializedTask)
   }
 }
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskResult.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskResult.scala
index b6df216d537e4..11d969e1aba90 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskResult.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskResult.scala
@@ -71,7 +71,7 @@ private[spark] class DirectTaskResult[T](
       for (i <- 0 until numUpdates) {
         _accumUpdates += in.readObject.asInstanceOf[AccumulatorV2[_, _]]
       }
-      accumUpdates = _accumUpdates
+      accumUpdates = _accumUpdates.toSeq
     }
 
     val numMetrics = in.readInt
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala
index 6c3d2a4ee3125..2dabee391310a 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala
@@ -83,7 +83,7 @@ private[spark] class TaskResultGetter(sparkEnv: SparkEnv, scheduler: TaskSchedul
                   "Tasks result size has exceeded maxResultSize"))
                 return
               }
-              logDebug("Fetching indirect task result for TID %s".format(tid))
+              logDebug(s"Fetching indirect task result for ${taskSetManager.taskName(tid)}")
               scheduler.handleTaskGettingResult(taskSetManager, tid)
               val serializedTaskResult = sparkEnv.blockManager.getRemoteBytes(blockId)
               if (serializedTaskResult.isEmpty) {
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskScheduler.scala
index e9e638a3645ac..0fa80bbafdedd 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskScheduler.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskScheduler.scala
@@ -45,7 +45,7 @@ private[spark] trait TaskScheduler {
 
   // Invoked after system has successfully initialized (typically in spark context).
   // Yarn uses this to bootstrap allocation of resources based on preferred locations,
-  // wait for slave registrations, etc.
+  // wait for executor registrations, etc.
   def postStartHook(): Unit = { }
 
   // Disconnect from the cluster.
@@ -101,7 +101,12 @@ private[spark] trait TaskScheduler {
   /**
    * Process a decommissioning executor.
    */
-  def executorDecommission(executorId: String): Unit
+  def executorDecommission(executorId: String, decommissionInfo: ExecutorDecommissionInfo): Unit
+
+  /**
+   * If an executor is decommissioned, return its corresponding decommission info
+   */
+  def getExecutorDecommissionState(executorId: String): Option[ExecutorDecommissionState]
 
   /**
    * Process a lost executor
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
index 2c37fec271766..b48ba127ce4b5 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
@@ -88,9 +88,9 @@ private[spark] class TaskSchedulerImpl(
     this(sc, sc.conf.get(config.TASK_MAX_FAILURES))
   }
 
-  // Lazily initializing blacklistTrackerOpt to avoid getting empty ExecutorAllocationClient,
+  // Lazily initializing healthTrackerOpt to avoid getting empty ExecutorAllocationClient,
   // because ExecutorAllocationClient is created after this TaskSchedulerImpl.
-  private[scheduler] lazy val blacklistTrackerOpt = maybeCreateBlacklistTracker(sc)
+  private[scheduler] lazy val healthTrackerOpt = maybeCreateHealthTracker(sc)
 
   val conf = sc.conf
 
@@ -100,7 +100,7 @@ private[spark] class TaskSchedulerImpl(
   // Duplicate copies of a task will only be launched if the original copy has been running for
   // at least this amount of time. This is to avoid the overhead of launching speculative copies
   // of tasks that are very short.
-  val MIN_TIME_TO_SPECULATION = 100
+  val MIN_TIME_TO_SPECULATION = conf.get(SPECULATION_MIN_THRESHOLD)
 
   private val speculationScheduler =
     ThreadUtils.newDaemonSingleThreadScheduledExecutor("task-scheduler-speculation")
@@ -128,7 +128,7 @@ private[spark] class TaskSchedulerImpl(
 
   @volatile private var hasReceivedTask = false
   @volatile private var hasLaunchedTask = false
-  private val starvationTimer = new Timer(true)
+  private val starvationTimer = new Timer("task-starvation-timer", true)
 
   // Incrementing task IDs
   val nextTaskId = new AtomicLong(0)
@@ -136,8 +136,12 @@ private[spark] class TaskSchedulerImpl(
   // IDs of the tasks running on each executor
   private val executorIdToRunningTaskIds = new HashMap[String, HashSet[Long]]
 
+  // We add executors here when we first get decommission notification for them. Executors can
+  // continue to run even after being asked to decommission, but they will eventually exit.
+  val executorsPendingDecommission = new HashMap[String, ExecutorDecommissionState]
+
   def runningTasksByExecutors: Map[String, Int] = synchronized {
-    executorIdToRunningTaskIds.toMap.mapValues(_.size)
+    executorIdToRunningTaskIds.toMap.mapValues(_.size).toMap
   }
 
   // The set of executors we have on each host; this is used to compute hostsAlive, which
@@ -148,7 +152,7 @@ private[spark] class TaskSchedulerImpl(
 
   protected val executorIdToHost = new HashMap[String, String]
 
-  private val abortTimer = new Timer(true)
+  private val abortTimer = new Timer("task-abort-timer", true)
   // Exposed for testing
   val unschedulableTaskSetToExpiryTime = new HashMap[TaskSetManager, Long]
 
@@ -201,7 +205,7 @@ private[spark] class TaskSchedulerImpl(
         case SchedulingMode.FIFO =>
           new FIFOSchedulableBuilder(rootPool)
         case SchedulingMode.FAIR =>
-          new FairSchedulableBuilder(rootPool, conf)
+          new FairSchedulableBuilder(rootPool, sc)
         case _ =>
           throw new IllegalArgumentException(s"Unsupported $SCHEDULER_MODE_PROPERTY: " +
           s"$schedulingMode")
@@ -274,7 +278,7 @@ private[spark] class TaskSchedulerImpl(
   private[scheduler] def createTaskSetManager(
       taskSet: TaskSet,
       maxTaskFailures: Int): TaskSetManager = {
-    new TaskSetManager(this, taskSet, maxTaskFailures, blacklistTrackerOpt)
+    new TaskSetManager(this, taskSet, maxTaskFailures, healthTrackerOpt, clock)
   }
 
   override def cancelTasks(stageId: Int, interruptThread: Boolean): Unit = synchronized {
@@ -360,7 +364,6 @@ private[spark] class TaskSchedulerImpl(
    * @param availableResources remaining resources per offer,
    *                           value at index 'i' corresponds to shuffledOffers[i]
    * @param tasks tasks scheduled per offer, value at index 'i' corresponds to shuffledOffers[i]
-   * @param addressesWithDescs tasks scheduler per host:port, used for barrier tasks
    * @return tuple of (no delay schedule rejects?, option of min locality of launched task)
    */
   private def resourceOfferSingleTaskSet(
@@ -369,12 +372,11 @@ private[spark] class TaskSchedulerImpl(
       shuffledOffers: Seq[WorkerOffer],
       availableCpus: Array[Int],
       availableResources: Array[Map[String, Buffer[String]]],
-      tasks: IndexedSeq[ArrayBuffer[TaskDescription]],
-      addressesWithDescs: ArrayBuffer[(String, TaskDescription)])
+      tasks: IndexedSeq[ArrayBuffer[TaskDescription]])
     : (Boolean, Option[TaskLocality]) = {
     var noDelayScheduleRejects = true
     var minLaunchedLocality: Option[TaskLocality] = None
-    // nodes and executors that are blacklisted for the entire application have already been
+    // nodes and executors that are excluded for the entire application have already been
     // filtered out by this point
     for (i <- 0 until shuffledOffers.size) {
       val execId = shuffledOffers(i).executorId
@@ -389,31 +391,32 @@ private[spark] class TaskSchedulerImpl(
           try {
             val prof = sc.resourceProfileManager.resourceProfileFromId(taskSetRpID)
             val taskCpus = ResourceProfile.getTaskCpusOrDefaultForProfile(prof, conf)
-            val (taskDescOption, didReject) =
+            val (taskDescOption, didReject, index) =
               taskSet.resourceOffer(execId, host, maxLocality, taskResAssignments)
             noDelayScheduleRejects &= !didReject
             for (task <- taskDescOption) {
-              tasks(i) += task
-              val tid = task.taskId
-              val locality = taskSet.taskInfos(task.taskId).taskLocality
+              val (locality, resources) = if (task != null) {
+                tasks(i) += task
+                addRunningTask(task.taskId, execId, taskSet)
+                (taskSet.taskInfos(task.taskId).taskLocality, task.resources)
+              } else {
+                assert(taskSet.isBarrier, "TaskDescription can only be null for barrier task")
+                val barrierTask = taskSet.barrierPendingLaunchTasks(index)
+                barrierTask.assignedOfferIndex = i
+                barrierTask.assignedCores = taskCpus
+                (barrierTask.taskLocality, barrierTask.assignedResources)
+              }
+
               minLaunchedLocality = minTaskLocality(minLaunchedLocality, Some(locality))
-              taskIdToTaskSetManager.put(tid, taskSet)
-              taskIdToExecutorId(tid) = execId
-              executorIdToRunningTaskIds(execId).add(tid)
               availableCpus(i) -= taskCpus
               assert(availableCpus(i) >= 0)
-              task.resources.foreach { case (rName, rInfo) =>
+              resources.foreach { case (rName, rInfo) =>
                 // Remove the first n elements from availableResources addresses, these removed
                 // addresses are the same as that we allocated in taskResourceAssignments since it's
                 // synchronized. We don't remove the exact addresses allocated because the current
                 // approach produces the identical result with less time complexity.
                 availableResources(i)(rName).remove(0, rInfo.addresses.size)
               }
-              // Only update hosts for a barrier task.
-              if (taskSet.isBarrier) {
-                // The executor address is expected to be non empty.
-                addressesWithDescs += (shuffledOffers(i).address.get -> task)
-              }
             }
           } catch {
             case e: TaskNotSerializableException =>
@@ -428,6 +431,15 @@ private[spark] class TaskSchedulerImpl(
     (noDelayScheduleRejects, minLaunchedLocality)
   }
 
+  /**
+   * Add the running task to TaskScheduler's related structures
+   */
+  private def addRunningTask(tid: Long, execId: String, taskSet: TaskSetManager): Unit = {
+    taskIdToTaskSetManager.put(tid, taskSet)
+    taskIdToExecutorId(tid) = execId
+    executorIdToRunningTaskIds(execId).add(tid)
+  }
+
   /**
    * Check whether the resources from the WorkerOffer are enough to run at least one task.
    * Returns None if the resources don't meet the task requirements, otherwise returns
@@ -466,51 +478,6 @@ private[spark] class TaskSchedulerImpl(
     Some(localTaskReqAssign.toMap)
   }
 
-  // Use the resource that the resourceProfile has as the limiting resource to calculate the
-  // total number of slots available based on the current offers.
-  private def calculateAvailableSlots(
-      resourceProfileIds: Array[Int],
-      availableCpus: Array[Int],
-      availableResources: Array[Map[String, Buffer[String]]],
-      taskSet: TaskSetManager): Int = {
-    val resourceProfile = sc.resourceProfileManager.resourceProfileFromId(
-      taskSet.taskSet.resourceProfileId)
-    val offersForResourceProfile = resourceProfileIds.zipWithIndex.filter { case (id, _) =>
-      (id == resourceProfile.id)
-    }
-    val coresKnown = resourceProfile.isCoresLimitKnown
-    var limitingResource = resourceProfile.limitingResource(conf)
-    val taskCpus = ResourceProfile.getTaskCpusOrDefaultForProfile(resourceProfile, conf)
-
-    offersForResourceProfile.map { case (o, index) =>
-      val numTasksPerExecCores = availableCpus(index) / taskCpus
-      // if limiting resource is empty then we have no other resources, so it has to be CPU
-      if (limitingResource == ResourceProfile.CPUS || limitingResource.isEmpty) {
-        numTasksPerExecCores
-      } else {
-        val taskLimit = resourceProfile.taskResources.get(limitingResource).map(_.amount)
-          .getOrElse {
-            val errorMsg = "limitingResource returns from ResourceProfile " +
-              s"$resourceProfile doesn't actually contain that task resource!"
-            taskSet.abort(errorMsg)
-            throw new SparkException(errorMsg)
-          }
-        // available addresses already takes into account if there are fractional
-        // task resource requests
-        val availAddrs = availableResources(index).get(limitingResource).map(_.size).getOrElse(0)
-        val resourceLimit = (availAddrs / taskLimit).toInt
-        if (!coresKnown) {
-          // when executor cores config isn't set, we can't calculate the real limiting resource
-          // and number of tasks per executor ahead of time, so calculate it now based on what
-          // is available.
-          if (numTasksPerExecCores <= resourceLimit) numTasksPerExecCores else resourceLimit
-        } else {
-          resourceLimit
-        }
-      }
-    }.sum
-  }
-
   private def minTaskLocality(
       l1: Option[TaskLocality],
       l2: Option[TaskLocality]) : Option[TaskLocality] = {
@@ -526,14 +493,14 @@ private[spark] class TaskSchedulerImpl(
   }
 
   /**
-   * Called by cluster manager to offer resources on slaves. We respond by asking our active task
+   * Called by cluster manager to offer resources on workers. We respond by asking our active task
    * sets for tasks in order of priority. We fill each node with tasks in a round-robin manner so
    * that tasks are balanced across the cluster.
    */
   def resourceOffers(
       offers: IndexedSeq[WorkerOffer],
       isAllFreeResources: Boolean = true): Seq[Seq[TaskDescription]] = synchronized {
-    // Mark each slave as alive and remember its hostname
+    // Mark each worker as alive and remember its hostname
     // Also track if new executor is added
     var newExecAvail = false
     for (o <- offers) {
@@ -553,15 +520,15 @@ private[spark] class TaskSchedulerImpl(
       hostsByRack.getOrElseUpdate(rack, new HashSet[String]()) += host
     }
 
-    // Before making any offers, remove any nodes from the blacklist whose blacklist has expired. Do
+    // Before making any offers, include any nodes whose expireOnFailure timeout has expired. Do
     // this here to avoid a separate thread and added synchronization overhead, and also because
-    // updating the blacklist is only relevant when task offers are being made.
-    blacklistTrackerOpt.foreach(_.applyBlacklistTimeout())
+    // updating the excluded executors and nodes is only relevant when task offers are being made.
+    healthTrackerOpt.foreach(_.applyExcludeOnFailureTimeout())
 
-    val filteredOffers = blacklistTrackerOpt.map { blacklistTracker =>
+    val filteredOffers = healthTrackerOpt.map { healthTracker =>
       offers.filter { offer =>
-        !blacklistTracker.isNodeBlacklisted(offer.host) &&
-          !blacklistTracker.isExecutorBlacklisted(offer.executorId)
+        !healthTracker.isNodeExcluded(offer.host) &&
+          !healthTracker.isExecutorExcluded(offer.executorId)
       }
     }.getOrElse(offers)
 
@@ -573,7 +540,7 @@ private[spark] class TaskSchedulerImpl(
     val availableResources = shuffledOffers.map(_.resources).toArray
     val availableCpus = shuffledOffers.map(o => o.cores).toArray
     val resourceProfileIds = shuffledOffers.map(o => o.resourceProfileId).toArray
-    val sortedTaskSets = rootPool.getSortedTaskSetQueue.filterNot(_.isZombie)
+    val sortedTaskSets = rootPool.getSortedTaskSetQueue
     for (taskSet <- sortedTaskSets) {
       logDebug("parentName: %s, name: %s, runningTasks: %s".format(
         taskSet.parent.name, taskSet.name, taskSet.runningTasks))
@@ -589,9 +556,14 @@ private[spark] class TaskSchedulerImpl(
       // we only need to calculate available slots if using barrier scheduling, otherwise the
       // value is -1
       val numBarrierSlotsAvailable = if (taskSet.isBarrier) {
-        val slots = calculateAvailableSlots(resourceProfileIds, availableCpus, availableResources,
-          taskSet)
-        slots
+        val rpId = taskSet.taskSet.resourceProfileId
+        val availableResourcesAmount = availableResources.map { resourceMap =>
+          // available addresses already takes into account if there are fractional
+          // task resource requests
+          resourceMap.map { case (name, addresses) => (name, addresses.length) }
+        }
+        calculateAvailableSlots(this, conf, rpId, resourceProfileIds, availableCpus,
+          availableResourcesAmount)
       } else {
         -1
       }
@@ -607,14 +579,12 @@ private[spark] class TaskSchedulerImpl(
         var launchedAnyTask = false
         var noDelaySchedulingRejects = true
         var globalMinLocality: Option[TaskLocality] = None
-        // Record all the executor IDs assigned barrier tasks on.
-        val addressesWithDescs = ArrayBuffer[(String, TaskDescription)]()
         for (currentMaxLocality <- taskSet.myLocalityLevels) {
           var launchedTaskAtCurrentMaxLocality = false
           do {
             val (noDelayScheduleReject, minLocality) = resourceOfferSingleTaskSet(
               taskSet, currentMaxLocality, shuffledOffers, availableCpus,
-              availableResources, tasks, addressesWithDescs)
+              availableResources, tasks)
             launchedTaskAtCurrentMaxLocality = minLocality.isDefined
             launchedAnyTask |= launchedTaskAtCurrentMaxLocality
             noDelaySchedulingRejects &= noDelayScheduleReject
@@ -635,81 +605,137 @@ private[spark] class TaskSchedulerImpl(
         }
 
         if (!launchedAnyTask) {
-          taskSet.getCompletelyBlacklistedTaskIfAny(hostToExecutors).foreach { taskIndex =>
-              // If the taskSet is unschedulable we try to find an existing idle blacklisted
-              // executor. If we cannot find one, we abort immediately. Else we kill the idle
-              // executor and kick off an abortTimer which if it doesn't schedule a task within the
-              // the timeout will abort the taskSet if we were unable to schedule any task from the
-              // taskSet.
+          taskSet.getCompletelyExcludedTaskIfAny(hostToExecutors).foreach { taskIndex =>
+              // If the taskSet is unschedulable we try to find an existing idle excluded
+              // executor and kill the idle executor and kick off an abortTimer which if it doesn't
+              // schedule a task within the the timeout will abort the taskSet if we were unable to
+              // schedule any task from the taskSet.
               // Note 1: We keep track of schedulability on a per taskSet basis rather than on a per
               // task basis.
               // Note 2: The taskSet can still be aborted when there are more than one idle
-              // blacklisted executors and dynamic allocation is on. This can happen when a killed
+              // excluded executors and dynamic allocation is on. This can happen when a killed
               // idle executor isn't replaced in time by ExecutorAllocationManager as it relies on
               // pending tasks and doesn't kill executors on idle timeouts, resulting in the abort
               // timer to expire and abort the taskSet.
+              //
+              // If there are no idle executors and dynamic allocation is enabled, then we would
+              // notify ExecutorAllocationManager to allocate more executors to schedule the
+              // unschedulable tasks else we will abort immediately.
               executorIdToRunningTaskIds.find(x => !isExecutorBusy(x._1)) match {
                 case Some ((executorId, _)) =>
                   if (!unschedulableTaskSetToExpiryTime.contains(taskSet)) {
-                    blacklistTrackerOpt.foreach(blt => blt.killBlacklistedIdleExecutor(executorId))
-
-                    val timeout = conf.get(config.UNSCHEDULABLE_TASKSET_TIMEOUT) * 1000
-                    unschedulableTaskSetToExpiryTime(taskSet) = clock.getTimeMillis() + timeout
-                    logInfo(s"Waiting for $timeout ms for completely "
-                      + s"blacklisted task to be schedulable again before aborting $taskSet.")
-                    abortTimer.schedule(
-                      createUnschedulableTaskSetAbortTimer(taskSet, taskIndex), timeout)
+                    healthTrackerOpt.foreach(blt => blt.killExcludedIdleExecutor(executorId))
+                    updateUnschedulableTaskSetTimeoutAndStartAbortTimer(taskSet, taskIndex)
+                  }
+                case None =>
+                  //  Notify ExecutorAllocationManager about the unschedulable task set,
+                  // in order to provision more executors to make them schedulable
+                  if (Utils.isDynamicAllocationEnabled(conf)) {
+                    if (!unschedulableTaskSetToExpiryTime.contains(taskSet)) {
+                      logInfo("Notifying ExecutorAllocationManager to allocate more executors to" +
+                        " schedule the unschedulable task before aborting" +
+                        " stage ${taskSet.stageId}.")
+                      dagScheduler.unschedulableTaskSetAdded(taskSet.taskSet.stageId,
+                        taskSet.taskSet.stageAttemptId)
+                      updateUnschedulableTaskSetTimeoutAndStartAbortTimer(taskSet, taskIndex)
+                    }
+                  } else {
+                    // Abort Immediately
+                    logInfo("Cannot schedule any task because all executors excluded from " +
+                      "failures. No idle executors can be found to kill. Aborting stage " +
+                      s"${taskSet.stageId}.")
+                    taskSet.abortSinceCompletelyExcludedOnFailure(taskIndex)
                   }
-                case None => // Abort Immediately
-                  logInfo("Cannot schedule any task because of complete blacklisting. No idle" +
-                    s" executors can be found to kill. Aborting $taskSet." )
-                  taskSet.abortSinceCompletelyBlacklisted(taskIndex)
               }
           }
         } else {
-          // We want to defer killing any taskSets as long as we have a non blacklisted executor
+          // We want to defer killing any taskSets as long as we have a non excluded executor
           // which can be used to schedule a task from any active taskSets. This ensures that the
           // job can make progress.
           // Note: It is theoretically possible that a taskSet never gets scheduled on a
-          // non-blacklisted executor and the abort timer doesn't kick in because of a constant
+          // non-excluded executor and the abort timer doesn't kick in because of a constant
           // submission of new TaskSets. See the PR for more details.
           if (unschedulableTaskSetToExpiryTime.nonEmpty) {
             logInfo("Clearing the expiry times for all unschedulable taskSets as a task was " +
               "recently scheduled.")
+            // Notify ExecutorAllocationManager as well as other subscribers that a task now
+            // recently becomes schedulable
+            dagScheduler.unschedulableTaskSetRemoved(taskSet.taskSet.stageId,
+              taskSet.taskSet.stageAttemptId)
             unschedulableTaskSetToExpiryTime.clear()
           }
         }
 
         if (launchedAnyTask && taskSet.isBarrier) {
+          val barrierPendingLaunchTasks = taskSet.barrierPendingLaunchTasks.values.toArray
           // Check whether the barrier tasks are partially launched.
-          // TODO SPARK-24818 handle the assert failure case (that can happen when some locality
-          // requirements are not fulfilled, and we should revert the launched tasks).
-          if (addressesWithDescs.size != taskSet.numTasks) {
-            val errorMsg =
-              s"Fail resource offers for barrier stage ${taskSet.stageId} because only " +
-                s"${addressesWithDescs.size} out of a total number of ${taskSet.numTasks}" +
-                s" tasks got resource offers. This happens because barrier execution currently " +
-                s"does not work gracefully with delay scheduling. We highly recommend you to " +
-                s"disable delay scheduling by setting spark.locality.wait=0 as a workaround if " +
-                s"you see this error frequently."
-            logWarning(errorMsg)
-            taskSet.abort(errorMsg)
-            throw new SparkException(errorMsg)
-          }
+          if (barrierPendingLaunchTasks.length != taskSet.numTasks) {
+            if (legacyLocalityWaitReset) {
+              // Legacy delay scheduling always reset the timer when there's a task that is able
+              // to be scheduled. Thus, whenever there's a timer reset could happen during a single
+              // round resourceOffer, tasks that don't get or have the preferred locations would
+              // always reject the offered resources. As a result, the barrier taskset can't get
+              // launched. And if we retry the resourceOffer, we'd go through the same path again
+              // and get into the endless loop in the end.
+              val errorMsg = s"Fail resource offers for barrier stage ${taskSet.stageId} " +
+                s"because only ${barrierPendingLaunchTasks.length} out of a total number " +
+                s"of ${taskSet.numTasks} tasks got resource offers. We highly recommend " +
+                "you to use the non-legacy delay scheduling by setting " +
+                s"${LEGACY_LOCALITY_WAIT_RESET.key} to false to get rid of this error."
+              logWarning(errorMsg)
+              taskSet.abort(errorMsg)
+              throw new SparkException(errorMsg)
+            } else {
+              val curTime = clock.getTimeMillis()
+              if (curTime - taskSet.lastResourceOfferFailLogTime >
+                TaskSetManager.BARRIER_LOGGING_INTERVAL) {
+                logInfo("Releasing the assigned resource offers since only partial tasks can " +
+                  "be launched. Waiting for later round resource offers.")
+                taskSet.lastResourceOfferFailLogTime = curTime
+              }
+              barrierPendingLaunchTasks.foreach { task =>
+                // revert all assigned resources
+                availableCpus(task.assignedOfferIndex) += task.assignedCores
+                task.assignedResources.foreach { case (rName, rInfo) =>
+                  availableResources(task.assignedOfferIndex)(rName).appendAll(rInfo.addresses)
+                }
+                // re-add the task to the schedule pending list
+                taskSet.addPendingTask(task.index)
+              }
+            }
+          } else {
+            // All tasks are able to launch in this barrier task set. Let's do
+            // some preparation work before launching them.
+            val launchTime = clock.getTimeMillis()
+            val addressesWithDescs = barrierPendingLaunchTasks.map { task =>
+              val taskDesc = taskSet.prepareLaunchingTask(
+                task.execId,
+                task.host,
+                task.index,
+                task.taskLocality,
+                false,
+                task.assignedResources,
+                launchTime)
+              addRunningTask(taskDesc.taskId, taskDesc.executorId, taskSet)
+              tasks(task.assignedOfferIndex) += taskDesc
+              shuffledOffers(task.assignedOfferIndex).address.get -> taskDesc
+            }
 
-          // materialize the barrier coordinator.
-          maybeInitBarrierCoordinator()
+            // materialize the barrier coordinator.
+            maybeInitBarrierCoordinator()
 
-          // Update the taskInfos into all the barrier task properties.
-          val addressesStr = addressesWithDescs
-            // Addresses ordered by partitionId
-            .sortBy(_._2.partitionId)
-            .map(_._1)
-            .mkString(",")
-          addressesWithDescs.foreach(_._2.properties.setProperty("addresses", addressesStr))
+            // Update the taskInfos into all the barrier task properties.
+            val addressesStr = addressesWithDescs
+              // Addresses ordered by partitionId
+              .sortBy(_._2.partitionId)
+              .map(_._1)
+              .mkString(",")
+            addressesWithDescs.foreach(_._2.properties.setProperty("addresses", addressesStr))
 
-          logInfo(s"Successfully scheduled all the ${addressesWithDescs.size} tasks for barrier " +
-            s"stage ${taskSet.stageId}.")
+            logInfo(s"Successfully scheduled all the ${addressesWithDescs.size} tasks for " +
+              s"barrier stage ${taskSet.stageId}.")
+          }
+          taskSet.barrierPendingLaunchTasks.clear()
         }
       }
     }
@@ -719,7 +745,18 @@ private[spark] class TaskSchedulerImpl(
     if (tasks.nonEmpty) {
       hasLaunchedTask = true
     }
-    return tasks
+    return tasks.map(_.toSeq)
+  }
+
+  private def updateUnschedulableTaskSetTimeoutAndStartAbortTimer(
+      taskSet: TaskSetManager,
+      taskIndex: Int): Unit = {
+    val timeout = conf.get(config.UNSCHEDULABLE_TASKSET_TIMEOUT) * 1000
+    unschedulableTaskSetToExpiryTime(taskSet) = clock.getTimeMillis() + timeout
+    logInfo(s"Waiting for $timeout ms for completely " +
+      s"excluded task to be schedulable again before aborting stage ${taskSet.stageId}.")
+    abortTimer.schedule(
+      createUnschedulableTaskSetAbortTimer(taskSet, taskIndex), timeout)
   }
 
   private def createUnschedulableTaskSetAbortTimer(
@@ -729,9 +766,9 @@ private[spark] class TaskSchedulerImpl(
       override def run(): Unit = TaskSchedulerImpl.this.synchronized {
         if (unschedulableTaskSetToExpiryTime.contains(taskSet) &&
             unschedulableTaskSetToExpiryTime(taskSet) <= clock.getTimeMillis()) {
-          logInfo("Cannot schedule any task because of complete blacklisting. " +
-            s"Wait time for scheduling expired. Aborting $taskSet.")
-          taskSet.abortSinceCompletelyBlacklisted(taskIndex)
+          logInfo("Cannot schedule any task because all executors excluded due to failures. " +
+            s"Wait time for scheduling expired. Aborting stage ${taskSet.stageId}.")
+          taskSet.abortSinceCompletelyExcludedOnFailure(taskIndex)
         } else {
           this.cancel()
         }
@@ -765,7 +802,8 @@ private[spark] class TaskSchedulerImpl(
               })
               if (executorIdToRunningTaskIds.contains(execId)) {
                 reason = Some(
-                  SlaveLost(s"Task $tid was lost, so marking the executor as lost as well."))
+                  ExecutorProcessLost(
+                    s"Task $tid was lost, so marking the executor as lost as well."))
                 removeExecutor(execId, reason.get)
                 failedExecutor = Some(execId)
               }
@@ -911,11 +949,24 @@ private[spark] class TaskSchedulerImpl(
     }
   }
 
-  override def executorDecommission(executorId: String): Unit = {
+  override def executorDecommission(
+      executorId: String, decommissionInfo: ExecutorDecommissionInfo): Unit = {
+    synchronized {
+      // Don't bother noting decommissioning for executors that we don't know about
+      if (executorIdToHost.contains(executorId)) {
+        executorsPendingDecommission(executorId) =
+          ExecutorDecommissionState(clock.getTimeMillis(), decommissionInfo.workerHost)
+      }
+    }
     rootPool.executorDecommission(executorId)
     backend.reviveOffers()
   }
 
+  override def getExecutorDecommissionState(executorId: String)
+    : Option[ExecutorDecommissionState] = synchronized {
+    executorsPendingDecommission.get(executorId)
+  }
+
   override def executorLost(executorId: String, reason: ExecutorLossReason): Unit = {
     var failedExecutor: Option[String] = None
 
@@ -936,7 +987,7 @@ private[spark] class TaskSchedulerImpl(
 
           case None =>
             // We may get multiple executorLost() calls with different loss reasons. For example,
-            // one may be triggered by a dropped connection from the slave while another may be a
+            // one may be triggered by a dropped connection from the worker while another may be a
             // report of executor termination from Mesos. We produce log messages for both so we
             // eventually report the termination reason.
             logError(s"Lost an executor $executorId (already removed): $reason")
@@ -1006,11 +1057,13 @@ private[spark] class TaskSchedulerImpl(
       }
     }
 
+    executorsPendingDecommission.remove(executorId)
+
     if (reason != LossReasonPending) {
       executorIdToHost -= executorId
       rootPool.executorLost(executorId, host, reason)
     }
-    blacklistTrackerOpt.foreach(_.handleRemovedExecutor(executorId))
+    healthTrackerOpt.foreach(_.handleRemovedExecutor(executorId))
   }
 
   def executorAdded(execId: String, host: String): Unit = {
@@ -1018,31 +1071,44 @@ private[spark] class TaskSchedulerImpl(
   }
 
   def getExecutorsAliveOnHost(host: String): Option[Set[String]] = synchronized {
-    hostToExecutors.get(host).map(_.toSet)
+    hostToExecutors.get(host).map(_.filterNot(isExecutorDecommissioned)).map(_.toSet)
   }
 
   def hasExecutorsAliveOnHost(host: String): Boolean = synchronized {
-    hostToExecutors.contains(host)
+    hostToExecutors.get(host)
+      .exists(executors => executors.exists(e => !isExecutorDecommissioned(e)))
   }
 
   def hasHostAliveOnRack(rack: String): Boolean = synchronized {
-    hostsByRack.contains(rack)
+    hostsByRack.get(rack)
+      .exists(hosts => hosts.exists(h => !isHostDecommissioned(h)))
   }
 
   def isExecutorAlive(execId: String): Boolean = synchronized {
-    executorIdToRunningTaskIds.contains(execId)
+    executorIdToRunningTaskIds.contains(execId) && !isExecutorDecommissioned(execId)
   }
 
   def isExecutorBusy(execId: String): Boolean = synchronized {
     executorIdToRunningTaskIds.get(execId).exists(_.nonEmpty)
   }
 
+  // exposed for test
+  protected final def isExecutorDecommissioned(execId: String): Boolean =
+    getExecutorDecommissionState(execId).isDefined
+
+  // exposed for test
+  protected final def isHostDecommissioned(host: String): Boolean = {
+    hostToExecutors.get(host).exists { executors =>
+      executors.exists(e => getExecutorDecommissionState(e).exists(_.workerHost.isDefined))
+    }
+  }
+
   /**
-   * Get a snapshot of the currently blacklisted nodes for the entire application.  This is
+   * Get a snapshot of the currently excluded nodes for the entire application. This is
    * thread-safe -- it can be called without a lock on the TaskScheduler.
    */
-  def nodeBlacklist(): Set[String] = {
-    blacklistTrackerOpt.map(_.nodeBlacklist()).getOrElse(Set.empty)
+  def excludedNodes(): Set[String] = {
+    healthTrackerOpt.map(_.excludedNodeList()).getOrElse(Set.empty)
   }
 
   /**
@@ -1103,6 +1169,63 @@ private[spark] object TaskSchedulerImpl {
 
   val SCHEDULER_MODE_PROPERTY = SCHEDULER_MODE.key
 
+  /**
+   * Calculate the max available task slots given the `availableCpus` and `availableResources`
+   * from a collection of ResourceProfiles. And only those ResourceProfiles who has the
+   * same id with the `rpId` can be used to calculate the task slots.
+   *
+   * @param scheduler the TaskSchedulerImpl instance
+   * @param conf SparkConf used to calculate the limiting resource and get the cpu amount per task
+   * @param rpId the target ResourceProfile id. Only those ResourceProfiles who has the same id
+   *             with it can be used to calculate the task slots.
+   * @param availableRPIds an Array of ids of the available ResourceProfiles from the executors.
+   * @param availableCpus an Array of the amount of available cpus from the executors.
+   * @param availableResources an Array of the resources map from the executors. In the resource
+   *                           map, it maps from the resource name to its amount.
+   * @return the number of max task slots
+   */
+  def calculateAvailableSlots(
+      scheduler: TaskSchedulerImpl,
+      conf: SparkConf,
+      rpId: Int,
+      availableRPIds: Array[Int],
+      availableCpus: Array[Int],
+      availableResources: Array[Map[String, Int]]): Int = {
+    val resourceProfile = scheduler.sc.resourceProfileManager.resourceProfileFromId(rpId)
+    val coresKnown = resourceProfile.isCoresLimitKnown
+    val (limitingResource, limitedByCpu) = {
+      val limiting = resourceProfile.limitingResource(conf)
+      // if limiting resource is empty then we have no other resources, so it has to be CPU
+      if (limiting == ResourceProfile.CPUS || limiting.isEmpty) {
+        (ResourceProfile.CPUS, true)
+      } else {
+        (limiting, false)
+      }
+    }
+    val cpusPerTask = ResourceProfile.getTaskCpusOrDefaultForProfile(resourceProfile, conf)
+    val taskLimit = resourceProfile.taskResources.get(limitingResource).map(_.amount).get
+
+    availableCpus.zip(availableResources).zip(availableRPIds)
+      .filter { case (_, id) => id == rpId }
+      .map { case ((cpu, resources), _) =>
+        val numTasksPerExecCores = cpu / cpusPerTask
+        if (limitedByCpu) {
+          numTasksPerExecCores
+        } else {
+          val availAddrs = resources.getOrElse(limitingResource, 0)
+          val resourceLimit = (availAddrs / taskLimit).toInt
+          // when executor cores config isn't set, we can't calculate the real limiting resource
+          // and number of tasks per executor ahead of time, so calculate it now based on what
+          // is available.
+          if (!coresKnown && numTasksPerExecCores <= resourceLimit) {
+            numTasksPerExecCores
+          } else {
+            resourceLimit
+          }
+        }
+      }.sum
+  }
+
   /**
    * Used to balance containers across hosts.
    *
@@ -1144,13 +1267,13 @@ private[spark] object TaskSchedulerImpl {
     retval.toList
   }
 
-  private def maybeCreateBlacklistTracker(sc: SparkContext): Option[BlacklistTracker] = {
-    if (BlacklistTracker.isBlacklistEnabled(sc.conf)) {
+  private def maybeCreateHealthTracker(sc: SparkContext): Option[HealthTracker] = {
+    if (HealthTracker.isExcludeOnFailureEnabled(sc.conf)) {
       val executorAllocClient: Option[ExecutorAllocationClient] = sc.schedulerBackend match {
         case b: ExecutorAllocationClient => Some(b)
         case _ => None
       }
-      Some(new BlacklistTracker(sc, executorAllocClient))
+      Some(new HealthTracker(sc, executorAllocClient))
     } else {
       None
     }
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetBlacklist.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetBlacklist.scala
deleted file mode 100644
index 4df2889089ee9..0000000000000
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetBlacklist.scala
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.scheduler
-
-import scala.collection.mutable.{HashMap, HashSet}
-
-import org.apache.spark.SparkConf
-import org.apache.spark.internal.Logging
-import org.apache.spark.internal.config
-import org.apache.spark.util.Clock
-
-/**
- * Handles blacklisting executors and nodes within a taskset.  This includes blacklisting specific
- * (task, executor) / (task, nodes) pairs, and also completely blacklisting executors and nodes
- * for the entire taskset.
- *
- * It also must store sufficient information in task failures for application level blacklisting,
- * which is handled by [[BlacklistTracker]].  Note that BlacklistTracker does not know anything
- * about task failures until a taskset completes successfully.
- *
- * THREADING:  This class is a helper to [[TaskSetManager]]; as with the methods in
- * [[TaskSetManager]] this class is designed only to be called from code with a lock on the
- * TaskScheduler (e.g. its event handlers). It should not be called from other threads.
- */
-private[scheduler] class TaskSetBlacklist(
-    private val listenerBus: LiveListenerBus,
-    val conf: SparkConf,
-    val stageId: Int,
-    val stageAttemptId: Int,
-    val clock: Clock) extends Logging {
-
-  private val MAX_TASK_ATTEMPTS_PER_EXECUTOR = conf.get(config.MAX_TASK_ATTEMPTS_PER_EXECUTOR)
-  private val MAX_TASK_ATTEMPTS_PER_NODE = conf.get(config.MAX_TASK_ATTEMPTS_PER_NODE)
-  private val MAX_FAILURES_PER_EXEC_STAGE = conf.get(config.MAX_FAILURES_PER_EXEC_STAGE)
-  private val MAX_FAILED_EXEC_PER_NODE_STAGE = conf.get(config.MAX_FAILED_EXEC_PER_NODE_STAGE)
-
-  /**
-   * A map from each executor to the task failures on that executor.  This is used for blacklisting
-   * within this taskset, and it is also relayed onto [[BlacklistTracker]] for app-level
-   * blacklisting if this taskset completes successfully.
-   */
-  val execToFailures = new HashMap[String, ExecutorFailuresInTaskSet]()
-
-  /**
-   * Map from node to all executors on it with failures.  Needed because we want to know about
-   * executors on a node even after they have died. (We don't want to bother tracking the
-   * node -> execs mapping in the usual case when there aren't any failures).
-   */
-  private val nodeToExecsWithFailures = new HashMap[String, HashSet[String]]()
-  private val nodeToBlacklistedTaskIndexes = new HashMap[String, HashSet[Int]]()
-  private val blacklistedExecs = new HashSet[String]()
-  private val blacklistedNodes = new HashSet[String]()
-
-  private var latestFailureReason: String = null
-
-  /**
-   * Get the most recent failure reason of this TaskSet.
-   */
-  def getLatestFailureReason: String = {
-    latestFailureReason
-  }
-
-  /**
-   * Return true if this executor is blacklisted for the given task.  This does *not*
-   * need to return true if the executor is blacklisted for the entire stage, or blacklisted
-   * for the entire application.  That is to keep this method as fast as possible in the inner-loop
-   * of the scheduler, where those filters will have already been applied.
-   */
-  def isExecutorBlacklistedForTask(executorId: String, index: Int): Boolean = {
-    execToFailures.get(executorId).exists { execFailures =>
-      execFailures.getNumTaskFailures(index) >= MAX_TASK_ATTEMPTS_PER_EXECUTOR
-    }
-  }
-
-  def isNodeBlacklistedForTask(node: String, index: Int): Boolean = {
-    nodeToBlacklistedTaskIndexes.get(node).exists(_.contains(index))
-  }
-
-  /**
-   * Return true if this executor is blacklisted for the given stage.  Completely ignores whether
-   * the executor is blacklisted for the entire application (or anything to do with the node the
-   * executor is on).  That is to keep this method as fast as possible in the inner-loop of the
-   * scheduler, where those filters will already have been applied.
-   */
-  def isExecutorBlacklistedForTaskSet(executorId: String): Boolean = {
-    blacklistedExecs.contains(executorId)
-  }
-
-  def isNodeBlacklistedForTaskSet(node: String): Boolean = {
-    blacklistedNodes.contains(node)
-  }
-
-  private[scheduler] def updateBlacklistForFailedTask(
-      host: String,
-      exec: String,
-      index: Int,
-      failureReason: String): Unit = {
-    latestFailureReason = failureReason
-    val execFailures = execToFailures.getOrElseUpdate(exec, new ExecutorFailuresInTaskSet(host))
-    execFailures.updateWithFailure(index, clock.getTimeMillis())
-
-    // check if this task has also failed on other executors on the same host -- if its gone
-    // over the limit, blacklist this task from the entire host.
-    val execsWithFailuresOnNode = nodeToExecsWithFailures.getOrElseUpdate(host, new HashSet())
-    execsWithFailuresOnNode += exec
-    val failuresOnHost = execsWithFailuresOnNode.toIterator.flatMap { exec =>
-      execToFailures.get(exec).map { failures =>
-        // We count task attempts here, not the number of unique executors with failures.  This is
-        // because jobs are aborted based on the number task attempts; if we counted unique
-        // executors, it would be hard to config to ensure that you try another
-        // node before hitting the max number of task failures.
-        failures.getNumTaskFailures(index)
-      }
-    }.sum
-    if (failuresOnHost >= MAX_TASK_ATTEMPTS_PER_NODE) {
-      nodeToBlacklistedTaskIndexes.getOrElseUpdate(host, new HashSet()) += index
-    }
-
-    // Check if enough tasks have failed on the executor to blacklist it for the entire stage.
-    val numFailures = execFailures.numUniqueTasksWithFailures
-    if (numFailures >= MAX_FAILURES_PER_EXEC_STAGE) {
-      if (blacklistedExecs.add(exec)) {
-        logInfo(s"Blacklisting executor ${exec} for stage $stageId")
-        // This executor has been pushed into the blacklist for this stage.  Let's check if it
-        // pushes the whole node into the blacklist.
-        val blacklistedExecutorsOnNode =
-          execsWithFailuresOnNode.filter(blacklistedExecs.contains(_))
-        val now = clock.getTimeMillis()
-        listenerBus.post(
-          SparkListenerExecutorBlacklistedForStage(now, exec, numFailures, stageId, stageAttemptId))
-        val numFailExec = blacklistedExecutorsOnNode.size
-        if (numFailExec >= MAX_FAILED_EXEC_PER_NODE_STAGE) {
-          if (blacklistedNodes.add(host)) {
-            logInfo(s"Blacklisting ${host} for stage $stageId")
-            listenerBus.post(
-              SparkListenerNodeBlacklistedForStage(now, host, numFailExec, stageId, stageAttemptId))
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetExcludeList.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetExcludeList.scala
new file mode 100644
index 0000000000000..d8c46db166fc5
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetExcludeList.scala
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.scheduler
+
+import scala.collection.mutable.{HashMap, HashSet}
+
+import org.apache.spark.SparkConf
+import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config
+import org.apache.spark.util.Clock
+
+/**
+ * Handles excluding executors and nodes within a taskset.  This includes excluding specific
+ * (task, executor) / (task, nodes) pairs, and also completely excluding executors and nodes
+ * for the entire taskset.
+ *
+ * It also must store sufficient information in task failures for application level exclusion,
+ * which is handled by [[HealthTracker]].  Note that HealthTracker does not know anything
+ * about task failures until a taskset completes successfully.
+ *
+ * THREADING:  This class is a helper to [[TaskSetManager]]; as with the methods in
+ * [[TaskSetManager]] this class is designed only to be called from code with a lock on the
+ * TaskScheduler (e.g. its event handlers). It should not be called from other threads.
+ */
+private[scheduler] class TaskSetExcludelist(
+    private val listenerBus: LiveListenerBus,
+    val conf: SparkConf,
+    val stageId: Int,
+    val stageAttemptId: Int,
+    val clock: Clock) extends Logging {
+
+  private val MAX_TASK_ATTEMPTS_PER_EXECUTOR = conf.get(config.MAX_TASK_ATTEMPTS_PER_EXECUTOR)
+  private val MAX_TASK_ATTEMPTS_PER_NODE = conf.get(config.MAX_TASK_ATTEMPTS_PER_NODE)
+  private val MAX_FAILURES_PER_EXEC_STAGE = conf.get(config.MAX_FAILURES_PER_EXEC_STAGE)
+  private val MAX_FAILED_EXEC_PER_NODE_STAGE = conf.get(config.MAX_FAILED_EXEC_PER_NODE_STAGE)
+
+  /**
+   * A map from each executor to the task failures on that executor.  This is used for exclusion
+   * within this taskset, and it is also relayed onto [[HealthTracker]] for app-level
+   * exlucsion if this taskset completes successfully.
+   */
+  val execToFailures = new HashMap[String, ExecutorFailuresInTaskSet]()
+
+  /**
+   * Map from node to all executors on it with failures.  Needed because we want to know about
+   * executors on a node even after they have died. (We don't want to bother tracking the
+   * node -> execs mapping in the usual case when there aren't any failures).
+   */
+  private val nodeToExecsWithFailures = new HashMap[String, HashSet[String]]()
+  private val nodeToExcludedTaskIndexes = new HashMap[String, HashSet[Int]]()
+  private val excludedExecs = new HashSet[String]()
+  private val excludedNodes = new HashSet[String]()
+
+  private var latestFailureReason: String = null
+
+  /**
+   * Get the most recent failure reason of this TaskSet.
+   */
+  def getLatestFailureReason: String = {
+    latestFailureReason
+  }
+
+  /**
+   * Return true if this executor is excluded for the given task.  This does *not*
+   * need to return true if the executor is excluded for the entire stage, or excluded
+   * for the entire application.  That is to keep this method as fast as possible in the inner-loop
+   * of the scheduler, where those filters will have already been applied.
+   */
+  def isExecutorExcludedForTask(executorId: String, index: Int): Boolean = {
+    execToFailures.get(executorId).exists { execFailures =>
+      execFailures.getNumTaskFailures(index) >= MAX_TASK_ATTEMPTS_PER_EXECUTOR
+    }
+  }
+
+  def isNodeExcludedForTask(node: String, index: Int): Boolean = {
+    nodeToExcludedTaskIndexes.get(node).exists(_.contains(index))
+  }
+
+  /**
+   * Return true if this executor is excluded for the given stage.  Completely ignores whether
+   * the executor is excluded for the entire application (or anything to do with the node the
+   * executor is on).  That is to keep this method as fast as possible in the inner-loop of the
+   * scheduler, where those filters will already have been applied.
+   */
+  def isExecutorExcludedForTaskSet(executorId: String): Boolean = {
+    excludedExecs.contains(executorId)
+  }
+
+  def isNodeExcludedForTaskSet(node: String): Boolean = {
+    excludedNodes.contains(node)
+  }
+
+  private[scheduler] def updateExcludedForFailedTask(
+      host: String,
+      exec: String,
+      index: Int,
+      failureReason: String): Unit = {
+    latestFailureReason = failureReason
+    val execFailures = execToFailures.getOrElseUpdate(exec, new ExecutorFailuresInTaskSet(host))
+    execFailures.updateWithFailure(index, clock.getTimeMillis())
+
+    // check if this task has also failed on other executors on the same host -- if its gone
+    // over the limit, exclude this task from the entire host.
+    val execsWithFailuresOnNode = nodeToExecsWithFailures.getOrElseUpdate(host, new HashSet())
+    execsWithFailuresOnNode += exec
+    val failuresOnHost = execsWithFailuresOnNode.toIterator.flatMap { exec =>
+      execToFailures.get(exec).map { failures =>
+        // We count task attempts here, not the number of unique executors with failures.  This is
+        // because jobs are aborted based on the number task attempts; if we counted unique
+        // executors, it would be hard to config to ensure that you try another
+        // node before hitting the max number of task failures.
+        failures.getNumTaskFailures(index)
+      }
+    }.sum
+    if (failuresOnHost >= MAX_TASK_ATTEMPTS_PER_NODE) {
+      nodeToExcludedTaskIndexes.getOrElseUpdate(host, new HashSet()) += index
+    }
+
+    // Check if enough tasks have failed on the executor to exclude it for the entire stage.
+    val numFailures = execFailures.numUniqueTasksWithFailures
+    if (numFailures >= MAX_FAILURES_PER_EXEC_STAGE) {
+      if (excludedExecs.add(exec)) {
+        logInfo(s"Excluding executor ${exec} for stage $stageId")
+        // This executor has been excluded for this stage.  Let's check if it
+        // the whole node should be excluded.
+        val excludedExecutorsOnNode =
+          execsWithFailuresOnNode.filter(excludedExecs.contains(_))
+        val now = clock.getTimeMillis()
+        // SparkListenerExecutorBlacklistedForStage is deprecated but post both events
+        // to keep backward compatibility
+        listenerBus.post(
+          SparkListenerExecutorBlacklistedForStage(now, exec, numFailures, stageId, stageAttemptId))
+        listenerBus.post(
+          SparkListenerExecutorExcludedForStage(now, exec, numFailures, stageId, stageAttemptId))
+        val numFailExec = excludedExecutorsOnNode.size
+        if (numFailExec >= MAX_FAILED_EXEC_PER_NODE_STAGE) {
+          if (excludedNodes.add(host)) {
+            logInfo(s"Excluding ${host} for stage $stageId")
+            // SparkListenerNodeBlacklistedForStage is deprecated but post both events
+            // to keep backward compatibility
+            listenerBus.post(
+              SparkListenerNodeBlacklistedForStage(now, host, numFailExec, stageId, stageAttemptId))
+            listenerBus.post(
+              SparkListenerNodeExcludedForStage(now, host, numFailExec, stageId, stageAttemptId))
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
index a0e84b94735ec..3b72103f9930d 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
@@ -19,7 +19,7 @@ package org.apache.spark.scheduler
 
 import java.io.NotSerializableException
 import java.nio.ByteBuffer
-import java.util.concurrent.ConcurrentLinkedQueue
+import java.util.concurrent.{ConcurrentLinkedQueue, TimeUnit}
 
 import scala.collection.immutable.Map
 import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet}
@@ -55,7 +55,7 @@ private[spark] class TaskSetManager(
     sched: TaskSchedulerImpl,
     val taskSet: TaskSet,
     val maxTaskFailures: Int,
-    blacklistTracker: Option[BlacklistTracker] = None,
+    healthTracker: Option[HealthTracker] = None,
     clock: Clock = new SystemClock()) extends Schedulable with Logging {
 
   private val conf = sched.sc.conf
@@ -63,6 +63,7 @@ private[spark] class TaskSetManager(
   // SPARK-21563 make a copy of the jars/files so they are consistent across the TaskSet
   private val addedJars = HashMap[String, Long](sched.sc.addedJars.toSeq: _*)
   private val addedFiles = HashMap[String, Long](sched.sc.addedFiles.toSeq: _*)
+  private val addedArchives = HashMap[String, Long](sched.sc.addedArchives.toSeq: _*)
 
   val maxResultSize = conf.get(config.MAX_RESULT_SIZE)
 
@@ -71,6 +72,7 @@ private[spark] class TaskSetManager(
   val ser = env.closureSerializer.newInstance()
 
   val tasks = taskSet.tasks
+  private val isShuffleMapTasks = tasks(0).isInstanceOf[ShuffleMapTask]
   private[scheduler] val partitionToIndex = tasks.zipWithIndex
     .map { case (t, idx) => t.partitionId -> idx }.toMap
   val numTasks = tasks.length
@@ -103,6 +105,9 @@ private[spark] class TaskSetManager(
     numTasks <= slots
   }
 
+  private val executorDecommissionKillInterval =
+    conf.get(EXECUTOR_DECOMMISSION_KILL_INTERVAL).map(TimeUnit.SECONDS.toMillis)
+
   // For each task, tracks whether a copy of the task has succeeded. A task will also be
   // marked as "succeeded" if it failed with a fetch failure, in which case it should not
   // be re-run because the missing map data needs to be regenerated first.
@@ -120,15 +125,15 @@ private[spark] class TaskSetManager(
   val weight = 1
   val minShare = 0
   var priority = taskSet.priority
-  var stageId = taskSet.stageId
+  val stageId = taskSet.stageId
   val name = "TaskSet_" + taskSet.id
   var parent: Pool = null
   private var totalResultSize = 0L
   private var calculatedTasks = 0
 
-  private[scheduler] val taskSetBlacklistHelperOpt: Option[TaskSetBlacklist] = {
-    blacklistTracker.map { _ =>
-      new TaskSetBlacklist(sched.sc.listenerBus, conf, stageId, taskSet.stageAttemptId, clock)
+  private[scheduler] val taskSetExcludelistHelperOpt: Option[TaskSetExcludelist] = {
+    healthTracker.map { _ =>
+      new TaskSetExcludelist(sched.sc.listenerBus, conf, stageId, taskSet.stageAttemptId, clock)
     }
   }
 
@@ -152,6 +157,14 @@ private[spark] class TaskSetManager(
   // same time for a barrier stage.
   private[scheduler] def isBarrier = taskSet.tasks.nonEmpty && taskSet.tasks(0).isBarrier
 
+  // Barrier tasks that are pending to launch in a single resourceOffers round. Tasks will only get
+  // launched when all tasks are added to this pending list in a single round. Otherwise, we'll
+  // revert everything we did during task scheduling.
+  private[scheduler] val barrierPendingLaunchTasks = new HashMap[Int, BarrierPendingLaunchTask]()
+
+  // Record the last log time of the barrier TaskSetManager that failed to get all tasks launched.
+  private[scheduler] var lastResourceOfferFailLogTime = clock.getTimeMillis()
+
   // Store tasks waiting to be scheduled by locality preferences
   private[scheduler] val pendingTasks = new PendingTasksByLocality()
 
@@ -212,14 +225,11 @@ private[spark] class TaskSetManager(
   /**
    * Track the set of locality levels which are valid given the tasks locality preferences and
    * the set of currently available executors.  This is updated as executors are added and removed.
-   * This allows a performance optimization, of skipping levels that aren't relevant (eg., skip
+   * This allows a performance optimization, of skipping levels that aren't relevant (e.g., skip
    * PROCESS_LOCAL if no tasks could be run PROCESS_LOCAL for the current set of executors).
    */
   private[scheduler] var myLocalityLevels = computeValidLocalityLevels()
 
-  // Time to wait at each level
-  private[scheduler] var localityWaits = myLocalityLevels.map(getLocalityWait)
-
   // Delay scheduling variables: we keep track of our current locality level and the time we
   // last reset the locality wait timer, and move up a level when localityWaits[curLevel] expires.
   // We then move down if we manage to launch a "more local" task when resetting the timer
@@ -227,6 +237,9 @@ private[spark] class TaskSetManager(
   private var currentLocalityIndex = 0 // Index of our current locality level in validLocalityLevels
   private var lastLocalityWaitResetTime = clock.getTimeMillis()  // Time we last reset locality wait
 
+  // Time to wait at each level
+  private[scheduler] var localityWaits = myLocalityLevels.map(getLocalityWait)
+
   override def schedulableQueue: ConcurrentLinkedQueue[Schedulable] = null
 
   override def schedulingMode: SchedulingMode = SchedulingMode.NONE
@@ -290,14 +303,14 @@ private[spark] class TaskSetManager(
     while (indexOffset > 0) {
       indexOffset -= 1
       val index = list(indexOffset)
-      if (!isTaskBlacklistedOnExecOrNode(index, execId, host) &&
+      if (!isTaskExcludededOnExecOrNode(index, execId, host) &&
           !(speculative && hasAttemptOnHost(index, host))) {
         // This should almost always be list.trimEnd(1) to remove tail
         list.remove(indexOffset)
         // Speculatable task should only be launched when at most one copy of the
         // original task is running
         if (!successful(index)) {
-          if (copiesRunning(index) == 0) {
+          if (copiesRunning(index) == 0 && !barrierPendingLaunchTasks.contains(index)) {
             return Some(index)
           } else if (speculative && copiesRunning(index) == 1) {
             return Some(index)
@@ -313,10 +326,10 @@ private[spark] class TaskSetManager(
     taskAttempts(taskIndex).exists(_.host == host)
   }
 
-  private def isTaskBlacklistedOnExecOrNode(index: Int, execId: String, host: String): Boolean = {
-    taskSetBlacklistHelperOpt.exists { blacklist =>
-      blacklist.isNodeBlacklistedForTask(host, index) ||
-        blacklist.isExecutorBlacklistedForTask(execId, index)
+  private def isTaskExcludededOnExecOrNode(index: Int, execId: String, host: String): Boolean = {
+    taskSetExcludelistHelperOpt.exists { excludeList =>
+      excludeList.isNodeExcludedForTask(host, index) ||
+        excludeList.isExecutorExcludedForTask(execId, index)
     }
   }
 
@@ -406,8 +419,10 @@ private[spark] class TaskSetManager(
    * @param host  the host Id of the offered resource
    * @param maxLocality the maximum locality we want to schedule the tasks at
    *
-   * @return Tuple containing:
-   *         (TaskDescription of launched task if any, rejected resource due to delay scheduling?)
+   * @return Triple containing:
+   *         (TaskDescription of launched task if any,
+   *         rejected resource due to delay scheduling?,
+   *         dequeued task index)
    */
   @throws[TaskNotSerializableException]
   def resourceOffer(
@@ -415,13 +430,13 @@ private[spark] class TaskSetManager(
       host: String,
       maxLocality: TaskLocality.TaskLocality,
       taskResourceAssignments: Map[String, ResourceInformation] = Map.empty)
-    : (Option[TaskDescription], Boolean) =
+    : (Option[TaskDescription], Boolean, Int) =
   {
-    val offerBlacklisted = taskSetBlacklistHelperOpt.exists { blacklist =>
-      blacklist.isNodeBlacklistedForTaskSet(host) ||
-        blacklist.isExecutorBlacklistedForTaskSet(execId)
+    val offerExcluded = taskSetExcludelistHelperOpt.exists { excludeList =>
+      excludeList.isNodeExcludedForTaskSet(host) ||
+        excludeList.isExecutorExcludedForTaskSet(execId)
     }
-    if (!isZombie && !offerBlacklisted) {
+    if (!isZombie && !offerExcluded) {
       val curTime = clock.getTimeMillis()
 
       var allowedLocality = maxLocality
@@ -434,84 +449,123 @@ private[spark] class TaskSetManager(
         }
       }
 
+      var dequeuedTaskIndex: Option[Int] = None
       val taskDescription =
         dequeueTask(execId, host, allowedLocality)
           .map { case (index, taskLocality, speculative) =>
-        // Found a task; do some bookkeeping and return a task description
-        val task = tasks(index)
-        val taskId = sched.newTaskId()
-        // Do various bookkeeping
-        copiesRunning(index) += 1
-        val attemptNum = taskAttempts(index).size
-        val info = new TaskInfo(taskId, index, attemptNum, curTime,
-          execId, host, taskLocality, speculative)
-        taskInfos(taskId) = info
-        taskAttempts(index) = info :: taskAttempts(index)
-        if (legacyLocalityWaitReset && maxLocality != TaskLocality.NO_PREF) {
-          resetDelayScheduleTimer(Some(taskLocality))
-        }
-        // Serialize and return the task
-        val serializedTask: ByteBuffer = try {
-          ser.serialize(task)
-        } catch {
-          // If the task cannot be serialized, then there's no point to re-attempt the task,
-          // as it will always fail. So just abort the whole task-set.
-          case NonFatal(e) =>
-            val msg = s"Failed to serialize task $taskId, not attempting to retry it."
-            logError(msg, e)
-            abort(s"$msg Exception during serialization: $e")
-            throw new TaskNotSerializableException(e)
-        }
-        if (serializedTask.limit() > TaskSetManager.TASK_SIZE_TO_WARN_KIB * 1024 &&
-          !emittedTaskSizeWarning) {
-          emittedTaskSizeWarning = true
-          logWarning(s"Stage ${task.stageId} contains a task of very large size " +
-            s"(${serializedTask.limit() / 1024} KiB). The maximum recommended task size is " +
-            s"${TaskSetManager.TASK_SIZE_TO_WARN_KIB} KiB.")
-        }
-        addRunningTask(taskId)
-
-        // We used to log the time it takes to serialize the task, but task size is already
-        // a good proxy to task serialization time.
-        // val timeTaken = clock.getTime() - startTime
-        val taskName = s"task ${info.id} in stage ${taskSet.id}"
-        logInfo(s"Starting $taskName (TID $taskId, $host, executor ${info.executorId}, " +
-          s"partition ${task.partitionId}, $taskLocality, ${serializedTask.limit()} bytes) " +
-          s"taskResourceAssignments ${taskResourceAssignments}")
-
-        sched.dagScheduler.taskStarted(task, info)
-        new TaskDescription(
-          taskId,
-          attemptNum,
-          execId,
-          taskName,
-          index,
-          task.partitionId,
-          addedFiles,
-          addedJars,
-          task.localProperties,
-          taskResourceAssignments,
-          serializedTask)
-      }
+            dequeuedTaskIndex = Some(index)
+            if (legacyLocalityWaitReset && maxLocality != TaskLocality.NO_PREF) {
+              resetDelayScheduleTimer(Some(taskLocality))
+            }
+            if (isBarrier) {
+              barrierPendingLaunchTasks(index) =
+                BarrierPendingLaunchTask(
+                  execId,
+                  host,
+                  index,
+                  taskLocality,
+                  taskResourceAssignments)
+              // return null since the TaskDescription for the barrier task is not ready yet
+              null
+            } else {
+              prepareLaunchingTask(
+                execId,
+                host,
+                index,
+                taskLocality,
+                speculative,
+                taskResourceAssignments,
+                curTime)
+            }
+          }
       val hasPendingTasks = pendingTasks.all.nonEmpty || pendingSpeculatableTasks.all.nonEmpty
       val hasScheduleDelayReject =
         taskDescription.isEmpty &&
           maxLocality == TaskLocality.ANY &&
           hasPendingTasks
-      (taskDescription, hasScheduleDelayReject)
+      (taskDescription, hasScheduleDelayReject, dequeuedTaskIndex.getOrElse(-1))
     } else {
-      (None, false)
+      (None, false, -1)
+    }
+  }
+
+  def prepareLaunchingTask(
+      execId: String,
+      host: String,
+      index: Int,
+      taskLocality: TaskLocality.Value,
+      speculative: Boolean,
+      taskResourceAssignments: Map[String, ResourceInformation],
+      launchTime: Long): TaskDescription = {
+    // Found a task; do some bookkeeping and return a task description
+    val task = tasks(index)
+    val taskId = sched.newTaskId()
+    // Do various bookkeeping
+    copiesRunning(index) += 1
+    val attemptNum = taskAttempts(index).size
+    val info = new TaskInfo(taskId, index, attemptNum, launchTime,
+      execId, host, taskLocality, speculative)
+    taskInfos(taskId) = info
+    taskAttempts(index) = info :: taskAttempts(index)
+    // Serialize and return the task
+    val serializedTask: ByteBuffer = try {
+      ser.serialize(task)
+    } catch {
+      // If the task cannot be serialized, then there's no point to re-attempt the task,
+      // as it will always fail. So just abort the whole task-set.
+      case NonFatal(e) =>
+        val msg = s"Failed to serialize task $taskId, not attempting to retry it."
+        logError(msg, e)
+        abort(s"$msg Exception during serialization: $e")
+        throw new TaskNotSerializableException(e)
+    }
+    if (serializedTask.limit() > TaskSetManager.TASK_SIZE_TO_WARN_KIB * 1024 &&
+      !emittedTaskSizeWarning) {
+      emittedTaskSizeWarning = true
+      logWarning(s"Stage ${task.stageId} contains a task of very large size " +
+        s"(${serializedTask.limit() / 1024} KiB). The maximum recommended task size is " +
+        s"${TaskSetManager.TASK_SIZE_TO_WARN_KIB} KiB.")
     }
+    addRunningTask(taskId)
+
+    // We used to log the time it takes to serialize the task, but task size is already
+    // a good proxy to task serialization time.
+    // val timeTaken = clock.getTime() - startTime
+    val tName = taskName(taskId)
+    logInfo(s"Starting $tName ($host, executor ${info.executorId}, " +
+      s"partition ${task.partitionId}, $taskLocality, ${serializedTask.limit()} bytes) " +
+      s"taskResourceAssignments ${taskResourceAssignments}")
+
+    sched.dagScheduler.taskStarted(task, info)
+    new TaskDescription(
+      taskId,
+      attemptNum,
+      execId,
+      tName,
+      index,
+      task.partitionId,
+      addedFiles,
+      addedJars,
+      addedArchives,
+      task.localProperties,
+      taskResourceAssignments,
+      serializedTask)
+  }
+
+  def taskName(tid: Long): String = {
+    val info = taskInfos.get(tid)
+    assert(info.isDefined, s"Can not find TaskInfo for task (TID $tid)")
+    s"task ${info.get.id} in stage ${taskSet.id} (TID $tid)"
   }
 
   private def maybeFinishTaskSet(): Unit = {
     if (isZombie && runningTasks == 0) {
       sched.taskSetFinished(this)
       if (tasksSuccessful == numTasks) {
-        blacklistTracker.foreach(_.updateBlacklistForSuccessfulTaskSet(
+        healthTracker.foreach(_.updateExcludedForSuccessfulTaskSet(
           taskSet.stageId,
           taskSet.stageAttemptId,
-          taskSetBlacklistHelperOpt.get.execToFailures))
+          taskSetExcludelistHelperOpt.get.execToFailures))
       }
     }
   }
@@ -596,12 +650,13 @@ private[spark] class TaskSetManager(
   }
 
   /**
-   * Check whether the given task set has been blacklisted to the point that it can't run anywhere.
+   * Check whether the given task set has been excluded to the point that it can't run anywhere.
    *
    * It is possible that this taskset has become impossible to schedule *anywhere* due to the
-   * blacklist.  The most common scenario would be if there are fewer executors than
-   * spark.task.maxFailures. We need to detect this so we can avoid the job from being hung.
-   * We try to acquire new executor/s by killing an existing idle blacklisted executor.
+   * failures that lead executors being excluded from the ones we can run on. The most common
+   * scenario would be if there are fewer executors than spark.task.maxFailures.
+   * We need to detect this so we can avoid the job from being hung. We try to acquire new
+   * executor/s by killing an existing idle excluded executor.
    *
    * There's a tradeoff here: we could make sure all tasks in the task set are schedulable, but that
    * would add extra time to each iteration of the scheduling loop. Here, we take the approach of
@@ -610,12 +665,12 @@ private[spark] class TaskSetManager(
    * method is faster in the typical case. In the worst case, this method can take
    * O(maxTaskFailures + numTasks) time, but it will be faster when there haven't been any task
    * failures (this is because the method picks one unscheduled task, and then iterates through each
-   * executor until it finds one that the task isn't blacklisted on).
+   * executor until it finds one that the task isn't excluded on).
    */
-  private[scheduler] def getCompletelyBlacklistedTaskIfAny(
+  private[scheduler] def getCompletelyExcludedTaskIfAny(
       hostToExecutors: HashMap[String, HashSet[String]]): Option[Int] = {
-    taskSetBlacklistHelperOpt.flatMap { taskSetBlacklist =>
-      val appBlacklist = blacklistTracker.get
+    taskSetExcludelistHelperOpt.flatMap { taskSetExcludelist =>
+      val appHealthTracker = healthTracker.get
       // Only look for unschedulable tasks when at least one executor has registered. Otherwise,
       // task sets will be (unnecessarily) aborted in cases when no executors have registered yet.
       if (hostToExecutors.nonEmpty) {
@@ -641,18 +696,18 @@ private[spark] class TaskSetManager(
           // when that unschedulable task is the last task remaining.
           hostToExecutors.forall { case (host, execsOnHost) =>
             // Check if the task can run on the node
-            val nodeBlacklisted =
-              appBlacklist.isNodeBlacklisted(host) ||
-                taskSetBlacklist.isNodeBlacklistedForTaskSet(host) ||
-                taskSetBlacklist.isNodeBlacklistedForTask(host, indexInTaskSet)
-            if (nodeBlacklisted) {
+            val nodeExcluded =
+              appHealthTracker.isNodeExcluded(host) ||
+                taskSetExcludelist.isNodeExcludedForTaskSet(host) ||
+                taskSetExcludelist.isNodeExcludedForTask(host, indexInTaskSet)
+            if (nodeExcluded) {
               true
             } else {
               // Check if the task can run on any of the executors
               execsOnHost.forall { exec =>
-                appBlacklist.isExecutorBlacklisted(exec) ||
-                  taskSetBlacklist.isExecutorBlacklistedForTaskSet(exec) ||
-                  taskSetBlacklist.isExecutorBlacklistedForTask(exec, indexInTaskSet)
+                appHealthTracker.isExecutorExcluded(exec) ||
+                  taskSetExcludelist.isExecutorExcludedForTaskSet(exec) ||
+                  taskSetExcludelist.isExecutorExcludedForTask(exec, indexInTaskSet)
               }
             }
           }
@@ -663,16 +718,16 @@ private[spark] class TaskSetManager(
     }
   }
 
-  private[scheduler] def abortSinceCompletelyBlacklisted(indexInTaskSet: Int): Unit = {
-    taskSetBlacklistHelperOpt.foreach { taskSetBlacklist =>
+  private[scheduler] def abortSinceCompletelyExcludedOnFailure(indexInTaskSet: Int): Unit = {
+    taskSetExcludelistHelperOpt.foreach { taskSetExcludelist =>
       val partition = tasks(indexInTaskSet).partitionId
       abort(s"""
          |Aborting $taskSet because task $indexInTaskSet (partition $partition)
-         |cannot run anywhere due to node and executor blacklist.
+         |cannot run anywhere due to node and executor excludeOnFailure.
          |Most recent failure:
-         |${taskSetBlacklist.getLatestFailureReason}
+         |${taskSetExcludelist.getLatestFailureReason}
          |
-         |Blacklisting behavior can be configured via spark.blacklist.*.
+         |ExcludeOnFailure behavior can be configured via spark.excludeOnFailure.*.
          |""".stripMargin)
     }
   }
@@ -687,12 +742,14 @@ private[spark] class TaskSetManager(
   }
 
   /**
-   * Check whether has enough quota to fetch the result with `size` bytes
+   * Check whether has enough quota to fetch the result with `size` bytes.
+   * This check does not apply to shuffle map tasks as they return map status and metrics updates,
+   * which will be discarded by the driver after being processed.
    */
   def canFetchMoreResults(size: Long): Boolean = sched.synchronized {
     totalResultSize += size
     calculatedTasks += 1
-    if (maxResultSize > 0 && totalResultSize > maxResultSize) {
+    if (!isShuffleMapTasks && maxResultSize > 0 && totalResultSize > maxResultSize) {
       val msg = s"Total size of serialized results of ${calculatedTasks} tasks " +
         s"(${Utils.bytesToString(totalResultSize)}) is bigger than ${config.MAX_RESULT_SIZE.key} " +
         s"(${Utils.bytesToString(maxResultSize)})"
@@ -736,9 +793,8 @@ private[spark] class TaskSetManager(
     // Kill any other attempts for the same task (since those are unnecessary now that one
     // attempt completed successfully).
     for (attemptInfo <- taskAttempts(index) if attemptInfo.running) {
-      logInfo(s"Killing attempt ${attemptInfo.attemptNumber} for task ${attemptInfo.id} " +
-        s"in stage ${taskSet.id} (TID ${attemptInfo.taskId}) on ${attemptInfo.host} " +
-        s"as the attempt ${info.attemptNumber} succeeded on ${info.host}")
+      logInfo(s"Killing attempt ${attemptInfo.attemptNumber} for ${taskName(attemptInfo.taskId)}" +
+        s" on ${attemptInfo.host} as the attempt ${info.attemptNumber} succeeded on ${info.host}")
       killedByOtherAttempt += attemptInfo.taskId
       sched.backend.killTask(
         attemptInfo.taskId,
@@ -748,17 +804,16 @@ private[spark] class TaskSetManager(
     }
     if (!successful(index)) {
       tasksSuccessful += 1
-      logInfo(s"Finished task ${info.id} in stage ${taskSet.id} (TID ${info.taskId}) in" +
-        s" ${info.duration} ms on ${info.host} (executor ${info.executorId})" +
-        s" ($tasksSuccessful/$numTasks)")
+      logInfo(s"Finished ${taskName(info.taskId)} in ${info.duration} ms " +
+        s"on ${info.host} (executor ${info.executorId}) ($tasksSuccessful/$numTasks)")
       // Mark successful and stop if all the tasks have succeeded.
       successful(index) = true
       if (tasksSuccessful == numTasks) {
         isZombie = true
       }
     } else {
-      logInfo("Ignoring task-finished event for " + info.id + " in stage " + taskSet.id +
-        " because task " + index + " has already completed successfully")
+      logInfo(s"Ignoring task-finished event for ${taskName(info.taskId)} " +
+        s"because it has already completed successfully")
     }
     // This method is called by "TaskSchedulerImpl.handleSuccessfulTask" which holds the
     // "TaskSchedulerImpl" lock until exiting. To avoid the SPARK-7655 issue, we should not
@@ -799,8 +854,8 @@ private[spark] class TaskSetManager(
     copiesRunning(index) -= 1
     var accumUpdates: Seq[AccumulatorV2[_, _]] = Seq.empty
     var metricPeaks: Array[Long] = Array.empty
-    val failureReason = s"Lost task ${info.id} in stage ${taskSet.id} (TID $tid, ${info.host}," +
-      s" executor ${info.executorId}): ${reason.toErrorString}"
+    val failureReason = s"Lost ${taskName(tid)} (${info.host} " +
+      s"executor ${info.executorId}): ${reason.toErrorString}"
     val failureException: Option[Throwable] = reason match {
       case fetchFailed: FetchFailed =>
         logWarning(failureReason)
@@ -811,7 +866,7 @@ private[spark] class TaskSetManager(
         isZombie = true
 
         if (fetchFailed.bmAddress != null) {
-          blacklistTracker.foreach(_.updateBlacklistForFetchFailure(
+          healthTracker.foreach(_.updateExcludedForFetchFailure(
             fetchFailed.bmAddress.host, fetchFailed.bmAddress.executorId))
         }
 
@@ -821,12 +876,11 @@ private[spark] class TaskSetManager(
         // ExceptionFailure's might have accumulator updates
         accumUpdates = ef.accums
         metricPeaks = ef.metricPeaks.toArray
+        val task = taskName(tid)
         if (ef.className == classOf[NotSerializableException].getName) {
           // If the task result wasn't serializable, there's no point in trying to re-execute it.
-          logError("Task %s in stage %s (TID %d) had a not serializable result: %s; not retrying"
-            .format(info.id, taskSet.id, tid, ef.description))
-          abort("Task %s in stage %s (TID %d) had a not serializable result: %s".format(
-            info.id, taskSet.id, tid, ef.description))
+          logError(s"$task had a not serializable result: ${ef.description}; not retrying")
+          abort(s"$task had a not serializable result: ${ef.description}")
           return
         }
         if (ef.className == classOf[TaskOutputFileAlreadyExistException].getName) {
@@ -859,8 +913,8 @@ private[spark] class TaskSetManager(
           logWarning(failureReason)
         } else {
           logInfo(
-            s"Lost task ${info.id} in stage ${taskSet.id} (TID $tid) on ${info.host}, executor" +
-              s" ${info.executorId}: ${ef.className} (${ef.description}) [duplicate $dupCount]")
+            s"Lost $task on ${info.host}, executor ${info.executorId}: " +
+              s"${ef.className} (${ef.description}) [duplicate $dupCount]")
         }
         ef.exception
 
@@ -872,7 +926,7 @@ private[spark] class TaskSetManager(
         None
 
       case e: ExecutorLostFailure if !e.exitCausedByApp =>
-        logInfo(s"Task $tid failed because while it was being computed, its executor " +
+        logInfo(s"${taskName(tid)} failed because while it was being computed, its executor " +
           "exited for a reason unrelated to the task. Not counting this failure towards the " +
           "maximum number of failures for the task.")
         None
@@ -890,7 +944,7 @@ private[spark] class TaskSetManager(
 
     if (!isZombie && reason.countTowardsTaskFailures) {
       assert (null != failureReason)
-      taskSetBlacklistHelperOpt.foreach(_.updateBlacklistForFailedTask(
+      taskSetExcludelistHelperOpt.foreach(_.updateExcludedForFailedTask(
         info.host, info.executorId, index, failureReason))
       numFailures(index) += 1
       if (numFailures(index) >= maxTaskFailures) {
@@ -903,10 +957,10 @@ private[spark] class TaskSetManager(
     }
 
     if (successful(index)) {
-      logInfo(s"Task ${info.id} in stage ${taskSet.id} (TID $tid) failed, but the task will not" +
-        s" be re-executed (either because the task failed with a shuffle data fetch failure," +
-        s" so the previous stage needs to be re-run, or because a different copy of the task" +
-        s" has already succeeded).")
+      logInfo(s"${taskName(info.taskId)} failed, but the task will not" +
+        " be re-executed (either because the task failed with a shuffle data fetch failure," +
+        " so the previous stage needs to be re-run, or because a different copy of the task" +
+        " has already succeeded).")
     } else {
       addPendingTask(index)
     }
@@ -942,6 +996,9 @@ private[spark] class TaskSetManager(
     null
   }
 
+  override def isSchedulable: Boolean = !isZombie &&
+    (pendingTasks.all.nonEmpty || pendingSpeculatableTasks.all.nonEmpty)
+
   override def addSchedulable(schedulable: Schedulable): Unit = {}
 
   override def removeSchedulable(schedulable: Schedulable): Unit = {}
@@ -958,8 +1015,7 @@ private[spark] class TaskSetManager(
     // and we are not using an external shuffle server which could serve the shuffle outputs.
     // The reason is the next stage wouldn't be able to fetch the data from this dead executor
     // so we would need to rerun these tasks on other executors.
-    if (tasks(0).isInstanceOf[ShuffleMapTask] && !env.blockManager.externalShuffleServiceEnabled
-        && !isZombie) {
+    if (isShuffleMapTasks && !env.blockManager.externalShuffleServiceEnabled && !isZombie) {
       for ((tid, info) <- taskInfos if info.executorId == execId) {
         val index = taskInfos(tid).index
         // We may have a running task whose partition has been marked as successful,
@@ -980,7 +1036,8 @@ private[spark] class TaskSetManager(
     for ((tid, info) <- taskInfos if info.running && info.executorId == execId) {
       val exitCausedByApp: Boolean = reason match {
         case exited: ExecutorExited => exited.exitCausedByApp
-        case ExecutorKilled => false
+        case ExecutorKilled | ExecutorDecommission(_) => false
+        case ExecutorProcessLost(_, _, false) => false
         case _ => true
       }
       handleFailedTask(tid, TaskState.FAILED, ExecutorLostFailure(info.executorId, exitCausedByApp,
@@ -1020,7 +1077,7 @@ private[spark] class TaskSetManager(
    * by the TaskScheduler.
    *
    */
-  override def checkSpeculatableTasks(minTimeToSpeculation: Int): Boolean = {
+  override def checkSpeculatableTasks(minTimeToSpeculation: Long): Boolean = {
     // No need to speculate if the task set is zombie or is from a barrier stage. If there is only
     // one task we don't speculate since we don't have metrics to decide whether it's taking too
     // long or not, unless a task duration threshold is explicitly provided.
@@ -1042,7 +1099,25 @@ private[spark] class TaskSetManager(
       // bound based on that.
       logDebug("Task length threshold for speculation: " + threshold)
       for (tid <- runningTasksSet) {
-        foundTasks |= checkAndSubmitSpeculatableTask(tid, time, threshold)
+        var speculated = checkAndSubmitSpeculatableTask(tid, time, threshold)
+        if (!speculated && executorDecommissionKillInterval.isDefined) {
+          val taskInfo = taskInfos(tid)
+          val decomState = sched.getExecutorDecommissionState(taskInfo.executorId)
+          if (decomState.isDefined) {
+            // Check if this task might finish after this executor is decommissioned.
+            // We estimate the task's finish time by using the median task duration.
+            // Whereas the time when the executor might be decommissioned is estimated using the
+            // config executorDecommissionKillInterval. If the task is going to finish after
+            // decommissioning, then we will eagerly speculate the task.
+            val taskEndTimeBasedOnMedianDuration = taskInfos(tid).launchTime + medianDuration
+            val executorDecomTime = decomState.get.startTime + executorDecommissionKillInterval.get
+            val canExceedDeadline = executorDecomTime < taskEndTimeBasedOnMedianDuration
+            if (canExceedDeadline) {
+              speculated = checkAndSubmitSpeculatableTask(tid, time, 0)
+            }
+          }
+        }
+        foundTasks |= speculated
       }
     } else if (speculationTaskDurationThresOpt.isDefined && speculationTasksLessEqToSlots) {
       val time = clock.getTimeMillis()
@@ -1056,6 +1131,8 @@ private[spark] class TaskSetManager(
   }
 
   private def getLocalityWait(level: TaskLocality.TaskLocality): Long = {
+    if (legacyLocalityWaitReset && isBarrier) return 0
+
     val localityWait = level match {
       case TaskLocality.PROCESS_LOCAL => config.LOCALITY_WAIT_PROCESS
       case TaskLocality.NODE_LOCAL => config.LOCALITY_WAIT_NODE
@@ -1100,17 +1177,24 @@ private[spark] class TaskSetManager(
 
   def executorDecommission(execId: String): Unit = {
     recomputeLocality()
-    // Future consideration: if an executor is decommissioned it may make sense to add the current
-    // tasks to the spec exec queue.
   }
 
   def recomputeLocality(): Unit = {
     // A zombie TaskSetManager may reach here while executorLost happens
     if (isZombie) return
+    val previousLocalityIndex = currentLocalityIndex
     val previousLocalityLevel = myLocalityLevels(currentLocalityIndex)
+    val previousMyLocalityLevels = myLocalityLevels
     myLocalityLevels = computeValidLocalityLevels()
     localityWaits = myLocalityLevels.map(getLocalityWait)
     currentLocalityIndex = getLocalityIndex(previousLocalityLevel)
+    if (currentLocalityIndex > previousLocalityIndex) {
+      // SPARK-31837: If the new level is more local, shift to the new most local locality
+      // level in terms of better data locality. For example, say the previous locality
+      // levels are [PROCESS, NODE, ANY] and current level is ANY. After recompute, the
+      // locality levels are [PROCESS, NODE, RACK, ANY]. Then, we'll shift to RACK level.
+      currentLocalityIndex = getLocalityIndex(myLocalityLevels.diff(previousMyLocalityLevels).head)
+    }
   }
 
   def executorAdded(): Unit = {
@@ -1122,6 +1206,9 @@ private[spark] object TaskSetManager {
   // The user will be warned if any stages contain a task that has a serialized size greater than
   // this.
   val TASK_SIZE_TO_WARN_KIB = 1000
+
+  // 1 minute
+  val BARRIER_LOGGING_INTERVAL = 60000
 }
 
 /**
@@ -1152,3 +1239,16 @@ private[scheduler] class PendingTasksByLocality {
   // Set containing all pending tasks (also used as a stack, as above).
   val all = new ArrayBuffer[Int]
 }
+
+private[scheduler] case class BarrierPendingLaunchTask(
+    execId: String,
+    host: String,
+    index: Int,
+    taskLocality: TaskLocality.TaskLocality,
+    assignedResources: Map[String, ResourceInformation]) {
+  // Stored the corresponding index of the WorkerOffer which is responsible to launch the task.
+  // Used to revert the assigned resources (e.g., cores, custome resources) when the barrier
+  // task set doesn't launch successfully in a single resourceOffers round.
+  var assignedOfferIndex: Int = _
+  var assignedCores: Int = 0
+}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala
index 465c0d20de481..66ac40f5c74c9 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala
@@ -22,7 +22,7 @@ import java.nio.ByteBuffer
 import org.apache.spark.TaskState.TaskState
 import org.apache.spark.resource.{ResourceInformation, ResourceProfile}
 import org.apache.spark.rpc.RpcEndpointRef
-import org.apache.spark.scheduler.ExecutorLossReason
+import org.apache.spark.scheduler.{ExecutorLossReason, MiscellaneousProcessDetails}
 import org.apache.spark.util.SerializableBuffer
 
 private[spark] sealed trait CoarseGrainedClusterMessage extends Serializable
@@ -49,6 +49,9 @@ private[spark] object CoarseGrainedClusterMessages {
   case class KillExecutorsOnHost(host: String)
     extends CoarseGrainedClusterMessage
 
+  case class DecommissionExecutorsOnHost(host: String)
+    extends CoarseGrainedClusterMessage
+
   case class UpdateDelegationTokens(tokens: Array[Byte])
     extends CoarseGrainedClusterMessage
 
@@ -94,7 +97,17 @@ private[spark] object CoarseGrainedClusterMessages {
   case class RemoveExecutor(executorId: String, reason: ExecutorLossReason)
     extends CoarseGrainedClusterMessage
 
-  case class DecommissionExecutor(executorId: String)  extends CoarseGrainedClusterMessage
+  // A message that sent from executor to driver to tell driver that the executor has started
+  // decommissioning. It's used for the case where decommission is triggered at executor (e.g., K8S)
+  case class ExecutorDecommissioning(executorId: String) extends CoarseGrainedClusterMessage
+
+  // A message that sent from driver to executor to decommission that executor.
+  // It's used for Standalone's cases, where decommission is triggered at MasterWebUI or Worker.
+  object DecommissionExecutor extends CoarseGrainedClusterMessage
+
+  // A message that sent to the executor itself when it receives a signal,
+  // indicating the executor starts to decommission.
+  object ExecutorDecommissionSigReceived extends CoarseGrainedClusterMessage
 
   case class RemoveWorker(workerId: String, host: String, message: String)
     extends CoarseGrainedClusterMessage
@@ -111,6 +124,11 @@ private[spark] object CoarseGrainedClusterMessages {
 
   case class RegisterClusterManager(am: RpcEndpointRef) extends CoarseGrainedClusterMessage
 
+  // Send Miscellaneous Process information to the driver
+  case class MiscellaneousProcessAdded(
+      time: Long, processId: String, info: MiscellaneousProcessDetails)
+    extends CoarseGrainedClusterMessage
+
   // Used by YARN's client mode AM to retrieve the current set of delegation tokens.
   object RetrieveDelegationTokens extends CoarseGrainedClusterMessage
 
@@ -120,7 +138,7 @@ private[spark] object CoarseGrainedClusterMessages {
       resourceProfileToTotalExecs: Map[ResourceProfile, Int],
       numLocalityAwareTasksPerResourceProfileId: Map[Int, Int],
       hostToLocalTaskCount: Map[Int, Map[String, Int]],
-      nodeBlacklist: Set[String])
+      excludedNodes: Set[String])
     extends CoarseGrainedClusterMessage
 
   // Check if an executor was force-killed but for a reason unrelated to the running tasks.
@@ -132,4 +150,6 @@ private[spark] object CoarseGrainedClusterMessages {
   // Used internally by executors to shut themselves down.
   case object Shutdown extends CoarseGrainedClusterMessage
 
+  // The message to check if `CoarseGrainedSchedulerBackend` thinks the executor is alive or not.
+  case class IsExecutorAlive(executorId: String) extends CoarseGrainedClusterMessage
 }
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
index 67638a5f9593c..3c121c5e0b19b 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.scheduler.cluster
 
-import java.util.concurrent.TimeUnit
+import java.util.concurrent.{ScheduledExecutorService, TimeUnit}
 import java.util.concurrent.atomic.{AtomicInteger, AtomicReference}
 import javax.annotation.concurrent.GuardedBy
 
@@ -92,8 +92,8 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
   // Executors that have been lost, but for which we don't yet know the real exit reason.
   private val executorsPendingLossReason = new HashSet[String]
 
-  // Executors which are being decommissioned
-  protected val executorsPendingDecommission = new HashSet[String]
+  // Executors which are being decommissioned. Maps from executorId to workerHost.
+  protected val executorsPendingDecommission = new HashMap[String, Option[String]]
 
   // A map of ResourceProfile id to map of hostname with its possible task number running on it
   @GuardedBy("CoarseGrainedSchedulerBackend.this")
@@ -115,6 +115,11 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
   private val reviveThread =
     ThreadUtils.newDaemonSingleThreadScheduledExecutor("driver-revive-thread")
 
+  private val cleanupService: Option[ScheduledExecutorService] =
+    conf.get(EXECUTOR_DECOMMISSION_FORCE_KILL_TIMEOUT).map { _ =>
+      ThreadUtils.newDaemonSingleThreadScheduledExecutor("cleanup-decommission-execs")
+    }
+
   class DriverEndpoint extends IsolatedRpcEndpoint with Logging {
 
     override val rpcEnv: RpcEnv = CoarseGrainedSchedulerBackend.this.rpcEnv
@@ -176,11 +181,20 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
         }
 
       case KillExecutorsOnHost(host) =>
-        scheduler.getExecutorsAliveOnHost(host).foreach { exec =>
-          killExecutors(exec.toSeq, adjustTargetNumExecutors = false, countFailures = false,
+        scheduler.getExecutorsAliveOnHost(host).foreach { execs =>
+          killExecutors(execs.toSeq, adjustTargetNumExecutors = false, countFailures = false,
             force = true)
         }
 
+      case DecommissionExecutorsOnHost(host) =>
+        val reason = ExecutorDecommissionInfo(s"Decommissioning all executors on $host.")
+        scheduler.getExecutorsAliveOnHost(host).foreach { execs =>
+          val execsWithReasons = execs.map(exec => (exec, reason)).toArray
+
+          decommissionExecutors(execsWithReasons, adjustTargetNumExecutors = false,
+            triggeredByExecutor = false)
+        }
+
       case UpdateDelegationTokens(newDelegationTokens) =>
         updateDelegationTokens(newDelegationTokens)
 
@@ -191,10 +205,6 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
         executorDataMap.get(executorId).foreach(_.executorEndpoint.send(StopExecutor))
         removeExecutor(executorId, reason)
 
-      case DecommissionExecutor(executorId) =>
-        logError(s"Received decommission executor message ${executorId}.")
-        decommissionExecutor(executorId)
-
       case RemoveWorker(workerId, host, message) =>
         removeWorker(workerId, host, message)
 
@@ -203,6 +213,11 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
           data.freeCores = data.totalCores
         }
         makeOffers(executorId)
+
+      case MiscellaneousProcessAdded(time: Long,
+          processId: String, info: MiscellaneousProcessDetails) =>
+        listenerBus.post(SparkListenerMiscellaneousProcessAdded(time, processId, info))
+
       case e =>
         logError(s"Received unexpected message. ${e}")
     }
@@ -213,13 +228,14 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
           attributes, resources, resourceProfileId) =>
         if (executorDataMap.contains(executorId)) {
           context.sendFailure(new IllegalStateException(s"Duplicate executor ID: $executorId"))
-        } else if (scheduler.nodeBlacklist.contains(hostname) ||
-            isBlacklisted(executorId, hostname)) {
-          // If the cluster manager gives us an executor on a blacklisted node (because it
-          // already started allocating those resources before we informed it of our blacklist,
-          // or if it ignored our blacklist), then we reject that executor immediately.
-          logInfo(s"Rejecting $executorId as it has been blacklisted.")
-          context.sendFailure(new IllegalStateException(s"Executor is blacklisted: $executorId"))
+        } else if (scheduler.excludedNodes.contains(hostname) ||
+            isExecutorExcluded(executorId, hostname)) {
+          // If the cluster manager gives us an executor on an excluded node (because it
+          // already started allocating those resources before we informed it of our exclusion,
+          // or if it ignored our exclusion), then we reject that executor immediately.
+          logInfo(s"Rejecting $executorId as it has been excluded.")
+          context.sendFailure(
+            new IllegalStateException(s"Executor is excluded due to failures: $executorId"))
         } else {
           // If the executor's rpc env is not listening for incoming connections, `hostPort`
           // will be null, and the client connection should be used to contact the executor.
@@ -242,7 +258,7 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
           }
           val data = new ExecutorData(executorRef, executorAddress, hostname,
             0, cores, logUrlHandler.applyPattern(logUrls, attributes), attributes,
-            resourcesInfo, resourceProfileId)
+            resourcesInfo, resourceProfileId, registrationTs = System.currentTimeMillis())
           // This must be synchronized because variables mutated
           // in this block are read when requesting executors
           CoarseGrainedSchedulerBackend.this.synchronized {
@@ -272,10 +288,15 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
         removeWorker(workerId, host, message)
         context.reply(true)
 
-      case DecommissionExecutor(executorId) =>
-        logError(s"Received decommission executor message ${executorId}.")
-        decommissionExecutor(executorId)
-        context.reply(true)
+      // Do not change this code without running the K8s integration suites
+      case ExecutorDecommissioning(executorId) =>
+        logWarning(s"Received executor $executorId decommissioned message")
+        context.reply(
+          decommissionExecutor(
+            executorId,
+            ExecutorDecommissionInfo(s"Executor $executorId is decommissioned."),
+            adjustTargetNumExecutors = false,
+            triggeredByExecutor = true))
 
       case RetrieveSparkAppConfig(resourceProfileId) =>
         val rp = scheduler.sc.resourceProfileManager.resourceProfileFromId(resourceProfileId)
@@ -285,6 +306,9 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
           Option(delegationTokens.get()),
           rp)
         context.reply(reply)
+
+      case IsExecutorAlive(executorId) => context.reply(isExecutorActive(executorId))
+
       case e =>
         logError(s"Received unexpected ask ${e}")
     }
@@ -313,9 +337,10 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
     override def onDisconnected(remoteAddress: RpcAddress): Unit = {
       addressToExecutorId
         .get(remoteAddress)
-        .foreach(removeExecutor(_, SlaveLost("Remote RPC client disassociated. Likely due to " +
-          "containers exceeding thresholds, or network issues. Check driver logs for WARN " +
-          "messages.")))
+        .foreach(removeExecutor(_,
+          ExecutorProcessLost("Remote RPC client disassociated. Likely due to " +
+            "containers exceeding thresholds, or network issues. Check driver logs for WARN " +
+            "messages.")))
     }
 
     // Make fake resource offers on just one executor
@@ -379,23 +404,30 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
       }
     }
 
-    // Remove a disconnected slave from the cluster
+    // Remove a disconnected executor from the cluster
     private def removeExecutor(executorId: String, reason: ExecutorLossReason): Unit = {
       logDebug(s"Asked to remove executor $executorId with reason $reason")
       executorDataMap.get(executorId) match {
         case Some(executorInfo) =>
           // This must be synchronized because variables mutated
           // in this block are read when requesting executors
-          val killed = CoarseGrainedSchedulerBackend.this.synchronized {
+          val lossReason = CoarseGrainedSchedulerBackend.this.synchronized {
             addressToExecutorId -= executorInfo.executorAddress
             executorDataMap -= executorId
             executorsPendingLossReason -= executorId
-            executorsPendingDecommission -= executorId
-            executorsPendingToRemove.remove(executorId).getOrElse(false)
+            val killedByDriver = executorsPendingToRemove.remove(executorId).getOrElse(false)
+            val workerHostOpt = executorsPendingDecommission.remove(executorId)
+            if (killedByDriver) {
+              ExecutorKilled
+            } else if (workerHostOpt.isDefined) {
+              ExecutorDecommission(workerHostOpt.get)
+            } else {
+              reason
+            }
           }
           totalCoreCount.addAndGet(-executorInfo.totalCores)
           totalRegisteredExecutors.addAndGet(-1)
-          scheduler.executorLost(executorId, if (killed) ExecutorKilled else reason)
+          scheduler.executorLost(executorId, lossReason)
           listenerBus.post(
             SparkListenerExecutorRemoved(System.currentTimeMillis(), executorId, reason.toString))
         case None =>
@@ -415,48 +447,6 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
       scheduler.workerRemoved(workerId, host, message)
     }
 
-    /**
-     * Mark a given executor as decommissioned and stop making resource offers for it.
-     */
-    private def decommissionExecutor(executorId: String): Boolean = {
-      val shouldDisable = CoarseGrainedSchedulerBackend.this.synchronized {
-        // Only bother decommissioning executors which are alive.
-        if (isExecutorActive(executorId)) {
-          executorsPendingDecommission += executorId
-          true
-        } else {
-          false
-        }
-      }
-
-      if (shouldDisable) {
-        logInfo(s"Starting decommissioning executor $executorId.")
-        try {
-          scheduler.executorDecommission(executorId)
-        } catch {
-          case e: Exception =>
-            logError(s"Unexpected error during decommissioning ${e.toString}", e)
-        }
-        logInfo(s"Finished decommissioning executor $executorId.")
-
-        if (conf.get(STORAGE_DECOMMISSION_ENABLED)) {
-          try {
-            logInfo("Starting decommissioning block manager corresponding to " +
-              s"executor $executorId.")
-            scheduler.sc.env.blockManager.master.decommissionBlockManagers(Seq(executorId))
-          } catch {
-            case e: Exception =>
-              logError("Unexpected error during block manager " +
-                s"decommissioning for executor $executorId: ${e.toString}", e)
-          }
-          logInfo(s"Acknowledged decommissioning block manager corresponding to $executorId.")
-        }
-      } else {
-        logInfo(s"Skipping decommissioning of executor $executorId.")
-      }
-      shouldDisable
-    }
-
     /**
      * Stop making resource offers for the given executor. The executor is marked as lost with
      * the loss reason still pending.
@@ -488,6 +478,71 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
 
   protected def minRegisteredRatio: Double = _minRegisteredRatio
 
+  /**
+   * Request that the cluster manager decommission the specified executors.
+   *
+   * @param executorsAndDecomInfo Identifiers of executors & decommission info.
+   * @param adjustTargetNumExecutors whether the target number of executors will be adjusted down
+   *                                 after these executors have been decommissioned.
+   * @param triggeredByExecutor whether the decommission is triggered at executor.
+   * @return the ids of the executors acknowledged by the cluster manager to be removed.
+   */
+  override def decommissionExecutors(
+      executorsAndDecomInfo: Array[(String, ExecutorDecommissionInfo)],
+      adjustTargetNumExecutors: Boolean,
+      triggeredByExecutor: Boolean): Seq[String] = withLock {
+    // Do not change this code without running the K8s integration suites
+    val executorsToDecommission = executorsAndDecomInfo.flatMap { case (executorId, decomInfo) =>
+      // Only bother decommissioning executors which are alive.
+      if (isExecutorActive(executorId)) {
+        scheduler.executorDecommission(executorId, decomInfo)
+        executorsPendingDecommission(executorId) = decomInfo.workerHost
+        Some(executorId)
+      } else {
+        None
+      }
+    }
+    logInfo(s"Decommission executors: ${executorsToDecommission.mkString(", ")}")
+
+    // If we don't want to replace the executors we are decommissioning
+    if (adjustTargetNumExecutors) {
+      adjustExecutors(executorsToDecommission)
+    }
+
+    // Mark those corresponding BlockManagers as decommissioned first before we sending
+    // decommission notification to executors. So, it's less likely to lead to the race
+    // condition where `getPeer` request from the decommissioned executor comes first
+    // before the BlockManagers are marked as decommissioned.
+    // Note that marking BlockManager as decommissioned doesn't need depend on
+    // `spark.storage.decommission.enabled`. Because it's meaningless to save more blocks
+    // for the BlockManager since the executor will be shutdown soon.
+    scheduler.sc.env.blockManager.master.decommissionBlockManagers(executorsToDecommission)
+
+    if (!triggeredByExecutor) {
+      executorsToDecommission.foreach { executorId =>
+        logInfo(s"Notify executor $executorId to decommissioning.")
+        executorDataMap(executorId).executorEndpoint.send(DecommissionExecutor)
+      }
+    }
+
+    conf.get(EXECUTOR_DECOMMISSION_FORCE_KILL_TIMEOUT).map { cleanupInterval =>
+      val cleanupTask = new Runnable() {
+        override def run(): Unit = Utils.tryLogNonFatalError {
+          val stragglers = CoarseGrainedSchedulerBackend.this.synchronized {
+            executorsToDecommission.filter(executorsPendingDecommission.contains)
+          }
+          if (stragglers.nonEmpty) {
+            logInfo(s"${stragglers.toList} failed to decommission in ${cleanupInterval}, killing.")
+            killExecutors(stragglers, false, false, true)
+          }
+        }
+      }
+      cleanupService.map(_.schedule(cleanupTask, cleanupInterval, TimeUnit.SECONDS))
+    }
+
+    executorsToDecommission
+  }
+
   override def start(): Unit = {
     if (UserGroupInformation.isSecurityEnabled()) {
       delegationTokenManager = createTokenManager()
@@ -527,6 +582,7 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
 
   override def stop(): Unit = {
     reviveThread.shutdownNow()
+    cleanupService.foreach(_.shutdownNow())
     stopExecutors()
     delegationTokenManager.foreach(_.stop())
     try {
@@ -553,11 +609,12 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
     // Remove all the lingering executors that should be removed but not yet. The reason might be
     // because (1) disconnected event is not yet received; (2) executors die silently.
     executors.foreach { eid =>
-      removeExecutor(eid, SlaveLost("Stale executor after cluster manager re-registered."))
+      removeExecutor(eid,
+        ExecutorProcessLost("Stale executor after cluster manager re-registered."))
     }
   }
 
-  override def reviveOffers(): Unit = {
+  override def reviveOffers(): Unit = Utils.tryLogNonFatalError {
     driverEndpoint.send(ReviveOffers)
   }
 
@@ -582,16 +639,6 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
     driverEndpoint.send(RemoveWorker(workerId, host, message))
   }
 
-  /**
-   * Called by subclasses when notified of a decommissioning executor.
-   */
-  private[spark] def decommissionExecutor(executorId: String): Unit = {
-    if (driverEndpoint != null) {
-      logInfo("Propagating executor decommission to driver.")
-      driverEndpoint.send(DecommissionExecutor(executorId))
-    }
-  }
-
   def sufficientResourcesRegistered(): Boolean = true
 
   override def isReady(): Boolean = {
@@ -617,18 +664,39 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
     executorDataMap.keySet.toSeq
   }
 
+  def getExecutorsWithRegistrationTs(): Map[String, Long] = synchronized {
+    executorDataMap.mapValues(v => v.registrationTs).toMap
+  }
+
   override def isExecutorActive(id: String): Boolean = synchronized {
     executorDataMap.contains(id) &&
       !executorsPendingToRemove.contains(id) &&
       !executorsPendingLossReason.contains(id) &&
       !executorsPendingDecommission.contains(id)
-
   }
 
+  /**
+   * Get the max number of tasks that can be concurrent launched based on the ResourceProfile
+   * could be used, even if some of them are being used at the moment.
+   * Note that please don't cache the value returned by this method, because the number can change
+   * due to add/remove executors.
+   *
+   * @param rp ResourceProfile which to use to calculate max concurrent tasks.
+   * @return The max number of tasks that can be concurrent launched currently.
+   */
   override def maxNumConcurrentTasks(rp: ResourceProfile): Int = synchronized {
-    val cpusPerTask = ResourceProfile.getTaskCpusOrDefaultForProfile(rp, conf)
-    val executorsWithResourceProfile = executorDataMap.values.filter(_.resourceProfileId == rp.id)
-    executorsWithResourceProfile.map(_.totalCores / cpusPerTask).sum
+    val (rpIds, cpus, resources) = {
+      executorDataMap
+        .filter { case (id, _) => isExecutorActive(id) }
+        .values.toArray.map { executor =>
+          (
+            executor.resourceProfileId,
+            executor.totalCores,
+            executor.resourcesInfo.map { case (name, rInfo) => (name, rInfo.totalAddressAmount) }
+          )
+        }.unzip3
+    }
+    TaskSchedulerImpl.calculateAvailableSlots(scheduler, conf, rp.id, rpIds, cpus, resources)
   }
 
   // this function is for testing only
@@ -725,6 +793,31 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
       resourceProfileToTotalExecs: Map[ResourceProfile, Int]): Future[Boolean] =
     Future.successful(false)
 
+  /**
+   * Adjust the number of executors being requested to no longer include the provided executors.
+   */
+  private def adjustExecutors(executorIds: Seq[String]) = {
+    if (executorIds.nonEmpty) {
+      executorIds.foreach { exec =>
+        withLock {
+          val rpId = executorDataMap(exec).resourceProfileId
+          val rp = scheduler.sc.resourceProfileManager.resourceProfileFromId(rpId)
+          if (requestedTotalExecutorsPerResourceProfile.isEmpty) {
+            // Assume that we are killing an executor that was started by default and
+            // not through the request api
+            requestedTotalExecutorsPerResourceProfile(rp) = 0
+          } else {
+            val requestedTotalForRp = requestedTotalExecutorsPerResourceProfile(rp)
+            requestedTotalExecutorsPerResourceProfile(rp) = math.max(requestedTotalForRp - 1, 0)
+          }
+        }
+      }
+      doRequestTotalExecutors(requestedTotalExecutorsPerResourceProfile.toMap)
+    } else {
+      Future.successful(true)
+    }
+  }
+
   /**
    * Request that the cluster manager kill the specified executors.
    *
@@ -763,19 +856,7 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
       // take into account executors that are pending to be added or removed.
       val adjustTotalExecutors =
         if (adjustTargetNumExecutors) {
-          executorsToKill.foreach { exec =>
-            val rpId = executorDataMap(exec).resourceProfileId
-            val rp = scheduler.sc.resourceProfileManager.resourceProfileFromId(rpId)
-            if (requestedTotalExecutorsPerResourceProfile.isEmpty) {
-              // Assume that we are killing an executor that was started by default and
-              // not through the request api
-              requestedTotalExecutorsPerResourceProfile(rp) = 0
-            } else {
-              val requestedTotalForRp = requestedTotalExecutorsPerResourceProfile(rp)
-              requestedTotalExecutorsPerResourceProfile(rp) = math.max(requestedTotalForRp - 1, 0)
-            }
-          }
-          doRequestTotalExecutors(requestedTotalExecutorsPerResourceProfile.toMap)
+          adjustExecutors(executorsToKill)
         } else {
           Future.successful(true)
         }
@@ -804,14 +885,30 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
   protected def doKillExecutors(executorIds: Seq[String]): Future[Boolean] =
     Future.successful(false)
 
+  /**
+   * Request that the cluster manager decommissions all executors on a given host.
+   * @return whether the decommission request is acknowledged.
+   */
+  final override def decommissionExecutorsOnHost(host: String): Boolean = {
+    logInfo(s"Requesting to kill any and all executors on host $host")
+    // A potential race exists if a new executor attempts to register on a host
+    // that is on the exclude list and is no longer valid. To avoid this race,
+    // all executor registration and decommissioning happens in the event loop. This way, either
+    // an executor will fail to register, or will be decommed when all executors on a host
+    // are decommed.
+    // Decommission all the executors on this host in an event loop to ensure serialization.
+    driverEndpoint.send(DecommissionExecutorsOnHost(host))
+    true
+  }
+
   /**
    * Request that the cluster manager kill all executors on a given host.
    * @return whether the kill request is acknowledged.
    */
   final override def killExecutorsOnHost(host: String): Boolean = {
-    logInfo(s"Requesting to kill any and all executors on host ${host}")
+    logInfo(s"Requesting to kill any and all executors on host $host")
     // A potential race exists if a new executor attempts to register on a host
-    // that is on the blacklist and is no no longer valid. To avoid this race,
+    // that is on the exclude list and is no longer valid. To avoid this race,
     // all executor registration and killing happens in the event loop. This way, either
     // an executor will fail to register, or will be killed when all executors on a host
     // are killed.
@@ -843,13 +940,13 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
   protected def currentDelegationTokens: Array[Byte] = delegationTokens.get()
 
   /**
-   * Checks whether the executor is blacklisted. This is called when the executor tries to
-   * register with the scheduler, and will deny registration if this method returns true.
+   * Checks whether the executor is excluded due to failure(s). This is called when the executor
+   * tries to register with the scheduler, and will deny registration if this method returns true.
    *
-   * This is in addition to the blacklist kept by the task scheduler, so custom implementations
+   * This is in addition to the exclude list kept by the task scheduler, so custom implementations
    * don't need to check there.
    */
-  protected def isBlacklisted(executorId: String, hostname: String): Boolean = false
+  protected def isExecutorExcluded(executorId: String, hostname: String): Boolean = false
 
   // SPARK-27112: We need to ensure that there is ordering of lock acquisition
   // between TaskSchedulerImpl and CoarseGrainedSchedulerBackend objects in order to fix
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/ExecutorData.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/ExecutorData.scala
index 062146174f6a8..86b44e835368c 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/ExecutorData.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/ExecutorData.scala
@@ -30,6 +30,7 @@ import org.apache.spark.scheduler.ExecutorResourceInfo
  * @param totalCores The total number of cores available to the executor
  * @param resourcesInfo The information of the currently available resources on the executor
  * @param resourceProfileId The id of the ResourceProfile being used by this executor
+ * @param registrationTs The registration timestamp of this executor
  */
 private[cluster] class ExecutorData(
     val executorEndpoint: RpcEndpointRef,
@@ -40,6 +41,7 @@ private[cluster] class ExecutorData(
     override val logUrlMap: Map[String, String],
     override val attributes: Map[String, String],
     override val resourcesInfo: Map[String, ExecutorResourceInfo],
-    override val resourceProfileId: Int
+    override val resourceProfileId: Int,
+    val registrationTs: Long
 ) extends ExecutorInfo(executorHost, totalCores, logUrlMap, attributes,
   resourcesInfo, resourceProfileId)
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala
index 42c46464d79e1..7a05569601a03 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.scheduler.cluster
 
+import java.util.Locale
 import java.util.concurrent.Semaphore
 import java.util.concurrent.atomic.AtomicBoolean
 
@@ -44,7 +45,7 @@ private[spark] class StandaloneSchedulerBackend(
   with StandaloneAppClientListener
   with Logging {
 
-  private var client: StandaloneAppClient = null
+  private[spark] var client: StandaloneAppClient = null
   private val stopping = new AtomicBoolean(false)
   private val launcherBackend = new LauncherBackend() {
     override protected def conf: SparkConf = sc.conf
@@ -165,19 +166,27 @@ private[spark] class StandaloneSchedulerBackend(
   }
 
   override def executorRemoved(
-      fullId: String, message: String, exitStatus: Option[Int], workerLost: Boolean): Unit = {
+      fullId: String,
+      message: String,
+      exitStatus: Option[Int],
+      workerHost: Option[String]): Unit = {
     val reason: ExecutorLossReason = exitStatus match {
       case Some(code) => ExecutorExited(code, exitCausedByApp = true, message)
-      case None => SlaveLost(message, workerLost = workerLost)
+      case None => ExecutorProcessLost(message, workerHost)
     }
     logInfo("Executor %s removed: %s".format(fullId, message))
     removeExecutor(fullId.split("/")(1), reason)
   }
 
-  override def executorDecommissioned(fullId: String, message: String) {
-    logInfo("Asked to decommission executor")
-    decommissionExecutor(fullId.split("/")(1))
-    logInfo("Executor %s decommissioned: %s".format(fullId, message))
+  override def executorDecommissioned(fullId: String,
+      decommissionInfo: ExecutorDecommissionInfo): Unit = {
+    logInfo(s"Asked to decommission executor $fullId")
+    val execId = fullId.split("/")(1)
+    decommissionExecutors(
+      Array((execId, decommissionInfo)),
+      adjustTargetNumExecutors = false,
+      triggeredByExecutor = false)
+    logInfo("Executor %s decommissioned: %s".format(fullId, decommissionInfo))
   }
 
   override def workerRemoved(workerId: String, host: String, message: String): Unit = {
@@ -227,6 +236,13 @@ private[spark] class StandaloneSchedulerBackend(
     }
   }
 
+  override def getDriverLogUrls: Option[Map[String, String]] = {
+    val prefix = "SPARK_DRIVER_LOG_URL_"
+    val driverLogUrls = sys.env.filterKeys(_.startsWith(prefix))
+      .map(e => (e._1.substring(prefix.length).toLowerCase(Locale.ROOT), e._2)).toMap
+    if (driverLogUrls.nonEmpty) Some(driverLogUrls) else None
+  }
+
   private def waitForRegistration() = {
     registrationBarrier.acquire()
   }
diff --git a/core/src/main/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitor.scala b/core/src/main/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitor.scala
index 4d7190726f067..cecd4b0ed987d 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitor.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitor.scala
@@ -28,7 +28,7 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config._
 import org.apache.spark.resource.ResourceProfile.UNKNOWN_RESOURCE_PROFILE_ID
 import org.apache.spark.scheduler._
-import org.apache.spark.storage.RDDBlockId
+import org.apache.spark.storage.{RDDBlockId, ShuffleDataBlockId}
 import org.apache.spark.util.Clock
 
 /**
@@ -38,7 +38,9 @@ private[spark] class ExecutorMonitor(
     conf: SparkConf,
     client: ExecutorAllocationClient,
     listenerBus: LiveListenerBus,
-    clock: Clock) extends SparkListener with CleanerListener with Logging {
+    clock: Clock,
+    metrics: ExecutorAllocationManagerSource = null)
+  extends SparkListener with CleanerListener with Logging {
 
   private val idleTimeoutNs = TimeUnit.SECONDS.toNanos(
     conf.get(DYN_ALLOCATION_EXECUTOR_IDLE_TIMEOUT))
@@ -114,7 +116,8 @@ private[spark] class ExecutorMonitor(
 
       var newNextTimeout = Long.MaxValue
       timedOutExecs = executors.asScala
-        .filter { case (_, exec) => !exec.pendingRemoval && !exec.hasActiveShuffle }
+        .filter { case (_, exec) =>
+          !exec.pendingRemoval && !exec.hasActiveShuffle && !exec.decommissioning}
         .filter { case (_, exec) =>
           val deadline = exec.timeoutAt
           if (deadline > now) {
@@ -135,6 +138,7 @@ private[spark] class ExecutorMonitor(
 
   /**
    * Mark the given executors as pending to be removed. Should only be called in the EAM thread.
+   * This covers both kills and decommissions.
    */
   def executorsKilled(ids: Seq[String]): Unit = {
     ids.foreach { id =>
@@ -149,6 +153,19 @@ private[spark] class ExecutorMonitor(
     nextTimeout.set(Long.MinValue)
   }
 
+  private[spark] def executorsDecommissioned(ids: Seq[String]): Unit = {
+    ids.foreach { id =>
+      val tracker = executors.get(id)
+      if (tracker != null) {
+        tracker.decommissioning = true
+      }
+    }
+
+    // Recompute timed out executors in the next EAM callback, since this call invalidates
+    // the current list.
+    nextTimeout.set(Long.MinValue)
+  }
+
   def executorCount: Int = executors.size()
 
   def executorCountWithResourceProfile(id: Int): Int = {
@@ -168,7 +185,17 @@ private[spark] class ExecutorMonitor(
   def pendingRemovalCount: Int = executors.asScala.count { case (_, exec) => exec.pendingRemoval }
 
   def pendingRemovalCountPerResourceProfileId(id: Int): Int = {
-    executors.asScala.filter { case (k, v) => v.resourceProfileId == id && v.pendingRemoval }.size
+    executors.asScala.count { case (k, v) => v.resourceProfileId == id && v.pendingRemoval }
+  }
+
+  def decommissioningCount: Int = executors.asScala.count { case (_, exec) =>
+    exec.decommissioning
+  }
+
+  def decommissioningPerResourceProfileId(id: Int): Int = {
+    executors.asScala.count { case (k, v) =>
+      v.resourceProfileId == id && v.decommissioning
+    }
   }
 
   override def onJobStart(event: SparkListenerJobStart): Unit = {
@@ -202,7 +229,7 @@ private[spark] class ExecutorMonitor(
     }
 
     if (updateExecutors) {
-      val activeShuffleIds = shuffleStages.map(_._2).toSeq
+      val activeShuffleIds = shuffleStages.map(_._2)
       var needTimeoutUpdate = false
       val activatedExecs = new ExecutorIdCollector()
       executors.asScala.foreach { case (id, exec) =>
@@ -226,7 +253,7 @@ private[spark] class ExecutorMonitor(
     }
 
     stageToShuffleID ++= shuffleStages
-    jobToStageIDs(event.jobId) = shuffleStages.map(_._1).toSeq
+    jobToStageIDs(event.jobId) = shuffleStages.map(_._1)
   }
 
   override def onJobEnd(event: SparkListenerJobEnd): Unit = {
@@ -298,6 +325,7 @@ private[spark] class ExecutorMonitor(
       //
       // This means that an executor may be marked as having shuffle data, and thus prevented
       // from being removed, even though the data may not be used.
+      // TODO: Only track used files (SPARK-31974)
       if (shuffleTrackingEnabled && event.reason == Success) {
         stageToShuffleID.get(event.stageId).foreach { shuffleId =>
           exec.addShuffle(shuffleId)
@@ -326,18 +354,51 @@ private[spark] class ExecutorMonitor(
     val removed = executors.remove(event.executorId)
     if (removed != null) {
       decrementExecResourceProfileCount(removed.resourceProfileId)
-      if (!removed.pendingRemoval) {
+      if (removed.decommissioning) {
+        if (event.reason == ExecutorLossMessage.decommissionFinished) {
+          metrics.gracefullyDecommissioned.inc()
+        } else {
+          metrics.decommissionUnfinished.inc()
+        }
+      } else if (removed.pendingRemoval) {
+        metrics.driverKilled.inc()
+      } else {
+        metrics.exitedUnexpectedly.inc()
+      }
+      logInfo(s"Executor ${event.executorId} is removed. Remove reason statistics: (" +
+        s"gracefully decommissioned: ${metrics.gracefullyDecommissioned.getCount()}, " +
+        s"decommision unfinished: ${metrics.decommissionUnfinished.getCount()}, " +
+        s"driver killed: ${metrics.driverKilled.getCount()}, " +
+        s"unexpectedly exited: ${metrics.exitedUnexpectedly.getCount()}).")
+      if (!removed.pendingRemoval || !removed.decommissioning) {
         nextTimeout.set(Long.MinValue)
       }
     }
   }
 
   override def onBlockUpdated(event: SparkListenerBlockUpdated): Unit = {
+    val exec = ensureExecutorIsTracked(event.blockUpdatedInfo.blockManagerId.executorId,
+      UNKNOWN_RESOURCE_PROFILE_ID)
+
+    // Check if it is a shuffle file, or RDD to pick the correct codepath for update
     if (!event.blockUpdatedInfo.blockId.isInstanceOf[RDDBlockId]) {
+      if (event.blockUpdatedInfo.blockId.isInstanceOf[ShuffleDataBlockId] &&
+        shuffleTrackingEnabled) {
+        /**
+         * The executor monitor keeps track of locations of cache and shuffle blocks and this can
+         * be used to decide which executor(s) Spark should shutdown first. Since we move shuffle
+         * blocks around now this wires it up so that it keeps track of it. We only do this for
+         * data blocks as index and other blocks blocks do not necessarily mean the entire block
+         * has been committed.
+         */
+        event.blockUpdatedInfo.blockId match {
+          case ShuffleDataBlockId(shuffleId, _, _) => exec.addShuffle(shuffleId)
+          case _ => // For now we only update on data blocks
+        }
+      }
       return
     }
-    val exec = ensureExecutorIsTracked(event.blockUpdatedInfo.blockManagerId.executorId,
-      UNKNOWN_RESOURCE_PROFILE_ID)
+
     val storageLevel = event.blockUpdatedInfo.storageLevel
     val blockId = event.blockUpdatedInfo.blockId.asInstanceOf[RDDBlockId]
 
@@ -410,10 +471,15 @@ private[spark] class ExecutorMonitor(
   }
 
   // Visible for testing
-  def executorsPendingToRemove(): Set[String] = {
+  private[spark] def executorsPendingToRemove(): Set[String] = {
     executors.asScala.filter { case (_, exec) => exec.pendingRemoval }.keys.toSet
   }
 
+  // Visible for testing
+  private[spark] def executorsDecommissioning(): Set[String] = {
+    executors.asScala.filter { case (_, exec) => exec.decommissioning }.keys.toSet
+  }
+
   /**
    * This method should be used when updating executor state. It guards against a race condition in
    * which the `SparkListenerTaskStart` event is posted before the `SparkListenerBlockManagerAdded`
@@ -466,6 +532,7 @@ private[spark] class ExecutorMonitor(
     @volatile var timedOut: Boolean = false
 
     var pendingRemoval: Boolean = false
+    var decommissioning: Boolean = false
     var hasActiveShuffle: Boolean = false
 
     private var idleStart: Long = -1
diff --git a/core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala b/core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala
index a4df0d543ecbe..4ebb7b0defd7f 100644
--- a/core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala
+++ b/core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala
@@ -167,7 +167,7 @@ private[spark] object CryptoStreamUtils extends Logging {
   }
 
   /**
-   * SPARK-25535. The commons-cryto library will throw InternalError if something goes
+   * SPARK-25535. The commons-crypto library will throw InternalError if something goes
    * wrong, and leave bad state behind in the Java wrappers, so it's not safe to use them
    * afterwards. This wrapper detects that situation and avoids further calls into the
    * commons-crypto code, while still allowing the underlying streams to be closed.
diff --git a/core/src/main/scala/org/apache/spark/security/SecurityConfigurationLock.scala b/core/src/main/scala/org/apache/spark/security/SecurityConfigurationLock.scala
new file mode 100644
index 0000000000000..7c80e59319048
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/security/SecurityConfigurationLock.scala
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.security
+
+/**
+ * There are cases when global JVM security configuration must be modified.
+ * In order to avoid race the modification must be synchronized with this.
+ */
+private[spark] object SecurityConfigurationLock
diff --git a/core/src/main/scala/org/apache/spark/security/SocketAuthHelper.scala b/core/src/main/scala/org/apache/spark/security/SocketAuthHelper.scala
index dbcb376905338..f800553c5388b 100644
--- a/core/src/main/scala/org/apache/spark/security/SocketAuthHelper.scala
+++ b/core/src/main/scala/org/apache/spark/security/SocketAuthHelper.scala
@@ -34,7 +34,7 @@ import org.apache.spark.util.Utils
  *
  * There's no secrecy, so this relies on the sockets being either local or somehow encrypted.
  */
-private[spark] class SocketAuthHelper(conf: SparkConf) {
+private[spark] class SocketAuthHelper(val conf: SparkConf) {
 
   val secret = Utils.createSecret(conf)
 
diff --git a/core/src/main/scala/org/apache/spark/security/SocketAuthServer.scala b/core/src/main/scala/org/apache/spark/security/SocketAuthServer.scala
index 548fd1b07ddc5..35990b5a59281 100644
--- a/core/src/main/scala/org/apache/spark/security/SocketAuthServer.scala
+++ b/core/src/main/scala/org/apache/spark/security/SocketAuthServer.scala
@@ -25,6 +25,8 @@ import scala.concurrent.duration.Duration
 import scala.util.Try
 
 import org.apache.spark.SparkEnv
+import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config.Python.PYTHON_AUTH_SOCKET_TIMEOUT
 import org.apache.spark.network.util.JavaUtils
 import org.apache.spark.util.{ThreadUtils, Utils}
 
@@ -34,11 +36,11 @@ import org.apache.spark.util.{ThreadUtils, Utils}
  * handling one batch of data, with authentication and error handling.
  *
  * The socket server can only accept one connection, or close if no connection
- * in 15 seconds.
+ * in configurable amount of seconds (default 15).
  */
 private[spark] abstract class SocketAuthServer[T](
     authHelper: SocketAuthHelper,
-    threadName: String) {
+    threadName: String) extends Logging {
 
   def this(env: SparkEnv, threadName: String) = this(new SocketAuthHelper(env.conf), threadName)
   def this(threadName: String) = this(SparkEnv.get, threadName)
@@ -46,19 +48,26 @@ private[spark] abstract class SocketAuthServer[T](
   private val promise = Promise[T]()
 
   private def startServer(): (Int, String) = {
+    logTrace("Creating listening socket")
     val serverSocket = new ServerSocket(0, 1, InetAddress.getByAddress(Array(127, 0, 0, 1)))
-    // Close the socket if no connection in 15 seconds
-    serverSocket.setSoTimeout(15000)
+    // Close the socket if no connection in the configured seconds
+    val timeout = authHelper.conf.get(PYTHON_AUTH_SOCKET_TIMEOUT).toInt
+    logTrace(s"Setting timeout to $timeout sec")
+    serverSocket.setSoTimeout(timeout * 1000)
 
     new Thread(threadName) {
       setDaemon(true)
       override def run(): Unit = {
         var sock: Socket = null
         try {
+          logTrace(s"Waiting for connection on port ${serverSocket.getLocalPort}")
           sock = serverSocket.accept()
+          logTrace(s"Connection accepted from address ${sock.getRemoteSocketAddress}")
           authHelper.authClient(sock)
+          logTrace("Client authenticated")
           promise.complete(Try(handleConnection(sock)))
         } finally {
+          logTrace("Closing server")
           JavaUtils.closeQuietly(serverSocket)
           JavaUtils.closeQuietly(sock)
         }
diff --git a/core/src/main/scala/org/apache/spark/serializer/GenericAvroSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/GenericAvroSerializer.scala
index f0ed41f6903f4..c1ef3ee769af2 100644
--- a/core/src/main/scala/org/apache/spark/serializer/GenericAvroSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/serializer/GenericAvroSerializer.scala
@@ -26,7 +26,7 @@ import scala.collection.mutable
 import com.esotericsoftware.kryo.{Kryo, Serializer => KSerializer}
 import com.esotericsoftware.kryo.io.{Input => KryoInput, Output => KryoOutput}
 import org.apache.avro.{Schema, SchemaNormalization}
-import org.apache.avro.generic.{GenericData, GenericRecord}
+import org.apache.avro.generic.{GenericContainer, GenericData}
 import org.apache.avro.io._
 import org.apache.commons.io.IOUtils
 
@@ -35,7 +35,7 @@ import org.apache.spark.io.CompressionCodec
 import org.apache.spark.util.Utils
 
 /**
- * Custom serializer used for generic Avro records. If the user registers the schemas
+ * Custom serializer used for generic Avro containers. If the user registers the schemas
  * ahead of time, then the schema's fingerprint will be sent with each message instead of the actual
  * schema, as to reduce network IO.
  * Actions like parsing or compressing schemas are computationally expensive so the serializer
@@ -43,9 +43,10 @@ import org.apache.spark.util.Utils
  * @param schemas a map where the keys are unique IDs for Avro schemas and the values are the
  *                string representation of the Avro schema, used to decrease the amount of data
  *                that needs to be serialized.
+ * @tparam D the subtype of [[GenericContainer]] handled by this serializer
  */
-private[serializer] class GenericAvroSerializer(schemas: Map[Long, String])
-  extends KSerializer[GenericRecord] {
+private[serializer] class GenericAvroSerializer[D <: GenericContainer]
+  (schemas: Map[Long, String]) extends KSerializer[D] {
 
   /** Used to reduce the amount of effort to compress the schema */
   private val compressCache = new mutable.HashMap[Schema, Array[Byte]]()
@@ -100,10 +101,10 @@ private[serializer] class GenericAvroSerializer(schemas: Map[Long, String])
   })
 
   /**
-   * Serializes a record to the given output stream. It caches a lot of the internal data as
-   * to not redo work
+   * Serializes a generic container to the given output stream. It caches a lot of the internal
+   * data as to not redo work
    */
-  def serializeDatum[R <: GenericRecord](datum: R, output: KryoOutput): Unit = {
+  def serializeDatum(datum: D, output: KryoOutput): Unit = {
     val encoder = EncoderFactory.get.binaryEncoder(output, null)
     val schema = datum.getSchema
     val fingerprint = fingerprintCache.getOrElseUpdate(schema, {
@@ -121,16 +122,16 @@ private[serializer] class GenericAvroSerializer(schemas: Map[Long, String])
     }
 
     writerCache.getOrElseUpdate(schema, GenericData.get.createDatumWriter(schema))
-      .asInstanceOf[DatumWriter[R]]
+      .asInstanceOf[DatumWriter[D]]
       .write(datum, encoder)
     encoder.flush()
   }
 
   /**
-   * Deserializes generic records into their in-memory form. There is internal
+   * Deserializes generic containers into their in-memory form. There is internal
    * state to keep a cache of already seen schemas and datum readers.
    */
-  def deserializeDatum(input: KryoInput): GenericRecord = {
+  def deserializeDatum(input: KryoInput): D = {
     val schema = {
       if (input.readBoolean()) {
         val fingerprint = input.readLong()
@@ -151,13 +152,13 @@ private[serializer] class GenericAvroSerializer(schemas: Map[Long, String])
     }
     val decoder = DecoderFactory.get.directBinaryDecoder(input, null)
     readerCache.getOrElseUpdate(schema, GenericData.get.createDatumReader(schema))
-      .asInstanceOf[DatumReader[GenericRecord]]
-      .read(null, decoder)
+      .asInstanceOf[DatumReader[D]]
+      .read(null.asInstanceOf[D], decoder)
   }
 
-  override def write(kryo: Kryo, output: KryoOutput, datum: GenericRecord): Unit =
+  override def write(kryo: Kryo, output: KryoOutput, datum: D): Unit =
     serializeDatum(datum, output)
 
-  override def read(kryo: Kryo, input: KryoInput, datumClass: Class[GenericRecord]): GenericRecord =
+  override def read(kryo: Kryo, input: KryoInput, datumClass: Class[D]): D =
     deserializeDatum(input)
 }
diff --git a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
index 55ac2c410953b..ce45762d20eab 100644
--- a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
@@ -33,7 +33,7 @@ import com.esotericsoftware.kryo.io.{UnsafeInput => KryoUnsafeInput, UnsafeOutpu
 import com.esotericsoftware.kryo.pool.{KryoCallback, KryoFactory, KryoPool}
 import com.esotericsoftware.kryo.serializers.{JavaSerializer => KryoJavaSerializer}
 import com.twitter.chill.{AllScalaRegistrar, EmptyScalaKryoInstantiator}
-import org.apache.avro.generic.{GenericData, GenericRecord}
+import org.apache.avro.generic.{GenericContainer, GenericData, GenericRecord}
 import org.roaringbitmap.RoaringBitmap
 
 import org.apache.spark._
@@ -153,8 +153,18 @@ class KryoSerializer(conf: SparkConf)
     kryo.register(classOf[SerializableJobConf], new KryoJavaSerializer())
     kryo.register(classOf[PythonBroadcast], new KryoJavaSerializer())
 
-    kryo.register(classOf[GenericRecord], new GenericAvroSerializer(avroSchemas))
-    kryo.register(classOf[GenericData.Record], new GenericAvroSerializer(avroSchemas))
+    // Register serializers for Avro GenericContainer classes
+    // We do not handle SpecificRecordBase and SpecificFixed here. They are abstract classes and
+    // we will need to register serializers for their concrete implementations individually.
+    // Also, their serialization requires the use of SpecificDatum(Reader|Writer) instead of
+    // GenericDatum(Reader|Writer).
+    def registerAvro[T <: GenericContainer]()(implicit ct: ClassTag[T]): Unit =
+      kryo.register(ct.runtimeClass, new GenericAvroSerializer[T](avroSchemas))
+    registerAvro[GenericRecord]
+    registerAvro[GenericData.Record]
+    registerAvro[GenericData.Array[_]]
+    registerAvro[GenericData.EnumSymbol]
+    registerAvro[GenericData.Fixed]
 
     // Use the default classloader when calling the user registrator.
     Utils.withContextClassLoader(classLoader) {
diff --git a/core/src/main/scala/org/apache/spark/shuffle/BlockStoreShuffleReader.scala b/core/src/main/scala/org/apache/spark/shuffle/BlockStoreShuffleReader.scala
index bc2a0fbc36d5b..ea90ec41fb2ac 100644
--- a/core/src/main/scala/org/apache/spark/shuffle/BlockStoreShuffleReader.scala
+++ b/core/src/main/scala/org/apache/spark/shuffle/BlockStoreShuffleReader.scala
@@ -35,7 +35,6 @@ private[spark] class BlockStoreShuffleReader[K, C](
     readMetrics: ShuffleReadMetricsReporter,
     serializerManager: SerializerManager = SparkEnv.get.serializerManager,
     blockManager: BlockManager = SparkEnv.get.blockManager,
-    mapOutputTracker: MapOutputTracker = SparkEnv.get.mapOutputTracker,
     shouldBatchFetch: Boolean = false)
   extends ShuffleReader[K, C] with Logging {
 
@@ -51,15 +50,17 @@ private[spark] class BlockStoreShuffleReader[K, C](
       true
     }
     val useOldFetchProtocol = conf.get(config.SHUFFLE_USE_OLD_FETCH_PROTOCOL)
+    // SPARK-34790: Fetching continuous blocks in batch is incompatible with io encryption.
+    val ioEncryption = conf.get(config.IO_ENCRYPTION_ENABLED)
 
     val doBatchFetch = shouldBatchFetch && serializerRelocatable &&
-      (!compressed || codecConcatenation) && !useOldFetchProtocol
+      (!compressed || codecConcatenation) && !useOldFetchProtocol && !ioEncryption
     if (shouldBatchFetch && !doBatchFetch) {
       logDebug("The feature tag of continuous shuffle block fetching is set to true, but " +
         "we can not enable the feature because other conditions are not satisfied. " +
         s"Shuffle compress: $compressed, serializer relocatable: $serializerRelocatable, " +
         s"codec concatenation: $codecConcatenation, use old shuffle fetch protocol: " +
-        s"$useOldFetchProtocol.")
+        s"$useOldFetchProtocol, io encryption: $ioEncryption.")
     }
     doBatchFetch
   }
@@ -120,20 +121,12 @@ private[spark] class BlockStoreShuffleReader[K, C](
     }
 
     // Sort the output if there is a sort ordering defined.
-    val resultIter = dep.keyOrdering match {
+    val resultIter: Iterator[Product2[K, C]] = dep.keyOrdering match {
       case Some(keyOrd: Ordering[K]) =>
         // Create an ExternalSorter to sort the data.
         val sorter =
           new ExternalSorter[K, C, C](context, ordering = Some(keyOrd), serializer = dep.serializer)
-        sorter.insertAll(aggregatedIter)
-        context.taskMetrics().incMemoryBytesSpilled(sorter.memoryBytesSpilled)
-        context.taskMetrics().incDiskBytesSpilled(sorter.diskBytesSpilled)
-        context.taskMetrics().incPeakExecutionMemory(sorter.peakMemoryUsedBytes)
-        // Use completion callback to stop sorter if task was finished/cancelled.
-        context.addTaskCompletionListener[Unit](_ => {
-          sorter.stop()
-        })
-        CompletionIterator[Product2[K, C], Iterator[Product2[K, C]]](sorter.iterator, sorter.stop())
+        sorter.insertAllAndUpdateMetrics(aggregatedIter)
       case None =>
         aggregatedIter
     }
diff --git a/core/src/main/scala/org/apache/spark/shuffle/FetchFailedException.scala b/core/src/main/scala/org/apache/spark/shuffle/FetchFailedException.scala
index 6509a04dc4893..208c676a1c352 100644
--- a/core/src/main/scala/org/apache/spark/shuffle/FetchFailedException.scala
+++ b/core/src/main/scala/org/apache/spark/shuffle/FetchFailedException.scala
@@ -48,7 +48,7 @@ private[spark] class FetchFailedException(
       mapTaskId: Long,
       mapIndex: Int,
       reduceId: Int,
-      cause: Throwable) {
+      cause: Throwable) = {
     this(bmAddress, shuffleId, mapTaskId, mapIndex, reduceId, cause.getMessage, cause)
   }
 
diff --git a/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala b/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala
index af2c82e771970..d30b73a74e12e 100644
--- a/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala
+++ b/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala
@@ -18,15 +18,18 @@
 package org.apache.spark.shuffle
 
 import java.io._
+import java.nio.ByteBuffer
 import java.nio.channels.Channels
 import java.nio.file.Files
 
-import org.apache.spark.{SparkConf, SparkEnv}
-import org.apache.spark.internal.Logging
+import org.apache.spark.{SparkConf, SparkEnv, SparkException}
+import org.apache.spark.internal.{config, Logging}
 import org.apache.spark.io.NioBufferedFileInputStream
 import org.apache.spark.network.buffer.{FileSegmentManagedBuffer, ManagedBuffer}
+import org.apache.spark.network.client.StreamCallbackWithID
 import org.apache.spark.network.netty.SparkTransportConf
 import org.apache.spark.network.shuffle.ExecutorDiskUtils
+import org.apache.spark.serializer.SerializerManager
 import org.apache.spark.shuffle.IndexShuffleBlockResolver.NOOP_REDUCE_ID
 import org.apache.spark.storage._
 import org.apache.spark.util.Utils
@@ -44,17 +47,40 @@ import org.apache.spark.util.Utils
 // org.apache.spark.network.shuffle.ExternalShuffleBlockResolver#getSortBasedShuffleBlockData().
 private[spark] class IndexShuffleBlockResolver(
     conf: SparkConf,
-    _blockManager: BlockManager = null)
+    // var for testing
+    var _blockManager: BlockManager = null)
   extends ShuffleBlockResolver
-  with Logging {
+  with Logging with MigratableResolver {
 
   private lazy val blockManager = Option(_blockManager).getOrElse(SparkEnv.get.blockManager)
 
   private val transportConf = SparkTransportConf.fromSparkConf(conf, "shuffle")
 
+  private val remoteShuffleMaxDisk: Option[Long] =
+    conf.get(config.STORAGE_DECOMMISSION_SHUFFLE_MAX_DISK_SIZE)
 
   def getDataFile(shuffleId: Int, mapId: Long): File = getDataFile(shuffleId, mapId, None)
 
+  /**
+   * Get the shuffle files that are stored locally. Used for block migrations.
+   */
+  override def getStoredShuffles(): Seq[ShuffleBlockInfo] = {
+    val allBlocks = blockManager.diskBlockManager.getAllBlocks()
+    allBlocks.flatMap {
+      case ShuffleIndexBlockId(shuffleId, mapId, _) =>
+        Some(ShuffleBlockInfo(shuffleId, mapId))
+      case _ =>
+        None
+    }
+  }
+
+  private def getShuffleBytesStored(): Long = {
+    val shuffleFiles: Seq[File] = getStoredShuffles().map {
+      si => getDataFile(si.shuffleId, si.mapId)
+    }
+    shuffleFiles.map(_.length()).sum
+  }
+
   /**
    * Get the shuffle data file.
    *
@@ -74,7 +100,7 @@ private[spark] class IndexShuffleBlockResolver(
    * When the dirs parameter is None then use the disk manager's local directories. Otherwise,
    * read from the specified directories.
    */
-  private def getIndexFile(
+  def getIndexFile(
       shuffleId: Int,
       mapId: Long,
       dirs: Option[Array[String]] = None): File = {
@@ -148,6 +174,107 @@ private[spark] class IndexShuffleBlockResolver(
     }
   }
 
+  /**
+   * Write a provided shuffle block as a stream. Used for block migrations.
+   * ShuffleBlockBatchIds must contain the full range represented in the ShuffleIndexBlock.
+   * Requires the caller to delete any shuffle index blocks where the shuffle block fails to
+   * put.
+   */
+  override def putShuffleBlockAsStream(blockId: BlockId, serializerManager: SerializerManager):
+      StreamCallbackWithID = {
+    // Throw an exception if we have exceeded maximum shuffle files stored
+    remoteShuffleMaxDisk.foreach { maxBytes =>
+      val bytesUsed = getShuffleBytesStored()
+      if (maxBytes < bytesUsed) {
+        throw new SparkException(s"Not storing remote shuffles $bytesUsed exceeds $maxBytes")
+      }
+    }
+    val file = blockId match {
+      case ShuffleIndexBlockId(shuffleId, mapId, _) =>
+        getIndexFile(shuffleId, mapId)
+      case ShuffleDataBlockId(shuffleId, mapId, _) =>
+        getDataFile(shuffleId, mapId)
+      case _ =>
+        throw new IllegalStateException(s"Unexpected shuffle block transfer ${blockId} as " +
+          s"${blockId.getClass().getSimpleName()}")
+    }
+    val fileTmp = Utils.tempFileWith(file)
+    val channel = Channels.newChannel(
+      serializerManager.wrapStream(blockId,
+        new FileOutputStream(fileTmp)))
+
+    new StreamCallbackWithID {
+
+      override def getID: String = blockId.name
+
+      override def onData(streamId: String, buf: ByteBuffer): Unit = {
+        while (buf.hasRemaining) {
+          channel.write(buf)
+        }
+      }
+
+      override def onComplete(streamId: String): Unit = {
+        logTrace(s"Done receiving shuffle block $blockId, now storing on local disk.")
+        channel.close()
+        val diskSize = fileTmp.length()
+        this.synchronized {
+          if (file.exists()) {
+            file.delete()
+          }
+          if (!fileTmp.renameTo(file)) {
+            throw new IOException(s"fail to rename file ${fileTmp} to ${file}")
+          }
+        }
+        blockManager.reportBlockStatus(blockId, BlockStatus(StorageLevel.DISK_ONLY, 0, diskSize))
+      }
+
+      override def onFailure(streamId: String, cause: Throwable): Unit = {
+        // the framework handles the connection itself, we just need to do local cleanup
+        logWarning(s"Error while uploading $blockId", cause)
+        channel.close()
+        fileTmp.delete()
+      }
+    }
+  }
+
+  /**
+   * Get the index & data block for migration.
+   */
+  def getMigrationBlocks(shuffleBlockInfo: ShuffleBlockInfo): List[(BlockId, ManagedBuffer)] = {
+    try {
+      val shuffleId = shuffleBlockInfo.shuffleId
+      val mapId = shuffleBlockInfo.mapId
+      // Load the index block
+      val indexFile = getIndexFile(shuffleId, mapId)
+      val indexBlockId = ShuffleIndexBlockId(shuffleId, mapId, NOOP_REDUCE_ID)
+      val indexFileSize = indexFile.length()
+      val indexBlockData = new FileSegmentManagedBuffer(
+        transportConf, indexFile, 0, indexFileSize)
+
+      // Load the data block
+      val dataFile = getDataFile(shuffleId, mapId)
+      val dataBlockId = ShuffleDataBlockId(shuffleId, mapId, NOOP_REDUCE_ID)
+      val dataBlockData = new FileSegmentManagedBuffer(
+        transportConf, dataFile, 0, dataFile.length())
+
+      // Make sure the index exist.
+      if (!indexFile.exists()) {
+        throw new FileNotFoundException("Index file is deleted already.")
+      }
+      if (dataFile.exists()) {
+        List((indexBlockId, indexBlockData), (dataBlockId, dataBlockData))
+      } else {
+        List((indexBlockId, indexBlockData))
+      }
+    } catch {
+      case _: Exception => // If we can't load the blocks ignore them.
+        logWarning(s"Failed to resolve shuffle block ${shuffleBlockInfo}. " +
+          "This is expected to occur if a block is removed after decommissioning has started.")
+        List.empty[(BlockId, ManagedBuffer)]
+    }
+  }
+
+
   /**
    * Write an index file with the offsets of each block, plus a final offset at the end for the
    * end of the output file. This will be used by getBlockData to figure out where each block
@@ -169,7 +296,7 @@ private[spark] class IndexShuffleBlockResolver(
       val dataFile = getDataFile(shuffleId, mapId)
       // There is only one IndexShuffleBlockResolver per executor, this synchronization make sure
       // the following check and rename are atomic.
-      synchronized {
+      this.synchronized {
         val existingLengths = checkIndexAndDataFile(indexFile, dataFile, lengths.length)
         if (existingLengths != null) {
           // Another attempt for the same task has already written our map outputs successfully,
@@ -209,6 +336,7 @@ private[spark] class IndexShuffleBlockResolver(
         }
       }
     } finally {
+      logDebug(s"Shuffle index for mapId $mapId: ${lengths.mkString("[", ",", "]")}")
       if (indexTmp.exists() && !indexTmp.delete()) {
         logError(s"Failed to delete temporary index file at ${indexTmp.getAbsolutePath}")
       }
diff --git a/core/src/main/scala/org/apache/spark/shuffle/MigratableResolver.scala b/core/src/main/scala/org/apache/spark/shuffle/MigratableResolver.scala
new file mode 100644
index 0000000000000..9908281deed84
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/shuffle/MigratableResolver.scala
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.shuffle
+
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.network.buffer.ManagedBuffer
+import org.apache.spark.network.client.StreamCallbackWithID
+import org.apache.spark.serializer.SerializerManager
+import org.apache.spark.storage.BlockId
+
+/**
+ * :: Experimental ::
+ * An experimental trait to allow Spark to migrate shuffle blocks.
+ */
+@Experimental
+@Since("3.1.0")
+trait MigratableResolver {
+  /**
+   * Get the shuffle ids that are stored locally. Used for block migrations.
+   */
+  def getStoredShuffles(): Seq[ShuffleBlockInfo]
+
+  /**
+   * Write a provided shuffle block as a stream. Used for block migrations.
+   * Up to the implementation to support STORAGE_REMOTE_SHUFFLE_MAX_DISK
+   */
+  def putShuffleBlockAsStream(blockId: BlockId, serializerManager: SerializerManager):
+      StreamCallbackWithID
+
+  /**
+   * Get the blocks for migration for a particular shuffle and map.
+   */
+  def getMigrationBlocks(shuffleBlockInfo: ShuffleBlockInfo): List[(BlockId, ManagedBuffer)]
+}
diff --git a/core/src/main/scala/org/apache/spark/shuffle/ShuffleBlockInfo.scala b/core/src/main/scala/org/apache/spark/shuffle/ShuffleBlockInfo.scala
new file mode 100644
index 0000000000000..99ceee81d349d
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/shuffle/ShuffleBlockInfo.scala
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.shuffle
+
+import org.apache.spark.annotation.Experimental
+
+/**
+ * :: Experimental ::
+ * An experimental case class used by MigratableResolver to return the shuffleId and mapId in a
+ * type safe way.
+ */
+@Experimental
+case class ShuffleBlockInfo(shuffleId: Int, mapId: Long)
diff --git a/core/src/main/scala/org/apache/spark/shuffle/ShuffleBlockPusher.scala b/core/src/main/scala/org/apache/spark/shuffle/ShuffleBlockPusher.scala
new file mode 100644
index 0000000000000..88d084ce1b2f4
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/shuffle/ShuffleBlockPusher.scala
@@ -0,0 +1,450 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.shuffle
+
+import java.io.File
+import java.net.ConnectException
+import java.nio.ByteBuffer
+import java.util.concurrent.ExecutorService
+
+import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet, Queue}
+
+import com.google.common.base.Throwables
+
+import org.apache.spark.{ShuffleDependency, SparkConf, SparkEnv}
+import org.apache.spark.annotation.Since
+import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config._
+import org.apache.spark.launcher.SparkLauncher
+import org.apache.spark.network.buffer.{FileSegmentManagedBuffer, ManagedBuffer, NioManagedBuffer}
+import org.apache.spark.network.netty.SparkTransportConf
+import org.apache.spark.network.shuffle.BlockFetchingListener
+import org.apache.spark.network.shuffle.ErrorHandler.BlockPushErrorHandler
+import org.apache.spark.network.util.TransportConf
+import org.apache.spark.shuffle.ShuffleBlockPusher._
+import org.apache.spark.storage.{BlockId, BlockManagerId, ShufflePushBlockId}
+import org.apache.spark.util.{ThreadUtils, Utils}
+
+/**
+ * Used for pushing shuffle blocks to remote shuffle services when push shuffle is enabled.
+ * When push shuffle is enabled, it is created after the shuffle writer finishes writing the shuffle
+ * file and initiates the block push process.
+ *
+ * @param conf spark configuration
+ */
+@Since("3.2.0")
+private[spark] class ShuffleBlockPusher(conf: SparkConf) extends Logging {
+  private[this] val maxBlockSizeToPush = conf.get(SHUFFLE_MAX_BLOCK_SIZE_TO_PUSH)
+  private[this] val maxBlockBatchSize = conf.get(SHUFFLE_MAX_BLOCK_BATCH_SIZE_FOR_PUSH)
+  private[this] val maxBytesInFlight =
+    conf.getSizeAsMb("spark.reducer.maxSizeInFlight", "48m") * 1024 * 1024
+  private[this] val maxReqsInFlight = conf.getInt("spark.reducer.maxReqsInFlight", Int.MaxValue)
+  private[this] val maxBlocksInFlightPerAddress = conf.get(REDUCER_MAX_BLOCKS_IN_FLIGHT_PER_ADDRESS)
+  private[this] var bytesInFlight = 0L
+  private[this] var reqsInFlight = 0
+  private[this] val numBlocksInFlightPerAddress = new HashMap[BlockManagerId, Int]()
+  private[this] val deferredPushRequests = new HashMap[BlockManagerId, Queue[PushRequest]]()
+  private[this] val pushRequests = new Queue[PushRequest]
+  private[this] val errorHandler = createErrorHandler()
+  // VisibleForTesting
+  private[shuffle] val unreachableBlockMgrs = new HashSet[BlockManagerId]()
+
+  // VisibleForTesting
+  private[shuffle] def createErrorHandler(): BlockPushErrorHandler = {
+    new BlockPushErrorHandler() {
+      // For a connection exception against a particular host, we will stop pushing any
+      // blocks to just that host and continue push blocks to other hosts. So, here push of
+      // all blocks will only stop when it is "Too Late". Also see updateStateAndCheckIfPushMore.
+      override def shouldRetryError(t: Throwable): Boolean = {
+        // If the block is too late, there is no need to retry it
+        !Throwables.getStackTraceAsString(t).contains(BlockPushErrorHandler.TOO_LATE_MESSAGE_SUFFIX)
+      }
+    }
+  }
+
+  /**
+   * Initiates the block push.
+   *
+   * @param dataFile         mapper generated shuffle data file
+   * @param partitionLengths array of shuffle block size so we can tell shuffle block
+   * @param dep              shuffle dependency to get shuffle ID and the location of remote shuffle
+   *                         services to push local shuffle blocks
+   * @param mapIndex      map index of the shuffle map task
+   */
+  private[shuffle] def initiateBlockPush(
+      dataFile: File,
+      partitionLengths: Array[Long],
+      dep: ShuffleDependency[_, _, _],
+      mapIndex: Int): Unit = {
+    val numPartitions = dep.partitioner.numPartitions
+    val transportConf = SparkTransportConf.fromSparkConf(conf, "shuffle")
+    val requests = prepareBlockPushRequests(numPartitions, mapIndex, dep.shuffleId, dataFile,
+      partitionLengths, dep.getMergerLocs, transportConf)
+    // Randomize the orders of the PushRequest, so different mappers pushing blocks at the same
+    // time won't be pushing the same ranges of shuffle partitions.
+    pushRequests ++= Utils.randomize(requests)
+
+    submitTask(() => {
+      pushUpToMax()
+    })
+  }
+
+  /**
+   * Triggers the push. It's a separate method for testing.
+   * VisibleForTesting
+   */
+  protected def submitTask(task: Runnable): Unit = {
+    if (BLOCK_PUSHER_POOL != null) {
+      BLOCK_PUSHER_POOL.execute(task)
+    }
+  }
+
+  /**
+   * Since multiple block push threads could potentially be calling pushUpToMax for the same
+   * mapper, we synchronize access to this method so that only one thread can push blocks for
+   * a given mapper. This helps to simplify access to the shared states. The down side of this
+   * is that we could unnecessarily block other mappers' block pushes if all the threads
+   * are occupied by block pushes from the same mapper.
+   *
+   * This code is similar to ShuffleBlockFetcherIterator#fetchUpToMaxBytes in how it throttles
+   * the data transfer between shuffle client/server.
+   */
+  private def pushUpToMax(): Unit = synchronized {
+    // Process any outstanding deferred push requests if possible.
+    if (deferredPushRequests.nonEmpty) {
+      for ((remoteAddress, defReqQueue) <- deferredPushRequests) {
+        while (isRemoteBlockPushable(defReqQueue) &&
+          !isRemoteAddressMaxedOut(remoteAddress, defReqQueue.front)) {
+          val request = defReqQueue.dequeue()
+          logDebug(s"Processing deferred push request for $remoteAddress with "
+            + s"${request.blocks.length} blocks")
+          sendRequest(request)
+          if (defReqQueue.isEmpty) {
+            deferredPushRequests -= remoteAddress
+          }
+        }
+      }
+    }
+
+    // Process any regular push requests if possible.
+    while (isRemoteBlockPushable(pushRequests)) {
+      val request = pushRequests.dequeue()
+      val remoteAddress = request.address
+      if (isRemoteAddressMaxedOut(remoteAddress, request)) {
+        logDebug(s"Deferring push request for $remoteAddress with ${request.blocks.size} blocks")
+        deferredPushRequests.getOrElseUpdate(remoteAddress, new Queue[PushRequest]())
+          .enqueue(request)
+      } else {
+        sendRequest(request)
+      }
+    }
+
+    def isRemoteBlockPushable(pushReqQueue: Queue[PushRequest]): Boolean = {
+      pushReqQueue.nonEmpty &&
+        (bytesInFlight == 0 ||
+          (reqsInFlight + 1 <= maxReqsInFlight &&
+            bytesInFlight + pushReqQueue.front.size <= maxBytesInFlight))
+    }
+
+    // Checks if sending a new push request will exceed the max no. of blocks being pushed to a
+    // given remote address.
+    def isRemoteAddressMaxedOut(remoteAddress: BlockManagerId, request: PushRequest): Boolean = {
+      (numBlocksInFlightPerAddress.getOrElse(remoteAddress, 0)
+        + request.blocks.size) > maxBlocksInFlightPerAddress
+    }
+  }
+
+  /**
+   * Push blocks to remote shuffle server. The callback listener will invoke #pushUpToMax again
+   * to trigger pushing the next batch of blocks once some block transfer is done in the current
+   * batch. This way, we decouple the map task from the block push process, since it is netty
+   * client thread instead of task execution thread which takes care of majority of the block
+   * pushes.
+   */
+  private def sendRequest(request: PushRequest): Unit = {
+    bytesInFlight +=  request.size
+    reqsInFlight += 1
+    numBlocksInFlightPerAddress(request.address) = numBlocksInFlightPerAddress.getOrElseUpdate(
+      request.address, 0) + request.blocks.length
+
+    val sizeMap = request.blocks.map { case (blockId, size) => (blockId.toString, size) }.toMap
+    val address = request.address
+    val blockIds = request.blocks.map(_._1.toString)
+    val remainingBlocks = new HashSet[String]() ++= blockIds
+
+    val blockPushListener = new BlockFetchingListener {
+      // Initiating a connection and pushing blocks to a remote shuffle service is always handled by
+      // the block-push-threads. We should not initiate the connection creation in the
+      // blockPushListener callbacks which are invoked by the netty eventloop because:
+      // 1. TrasportClient.createConnection(...) blocks for connection to be established and it's
+      // recommended to avoid any blocking operations in the eventloop;
+      // 2. The actual connection creation is a task that gets added to the task queue of another
+      // eventloop which could have eventloops eventually blocking each other.
+      // Once the blockPushListener is notified of the block push success or failure, we
+      // just delegate it to block-push-threads.
+      def handleResult(result: PushResult): Unit = {
+        submitTask(() => {
+          if (updateStateAndCheckIfPushMore(
+            sizeMap(result.blockId), address, remainingBlocks, result)) {
+            pushUpToMax()
+          }
+        })
+      }
+
+      override def onBlockFetchSuccess(blockId: String, data: ManagedBuffer): Unit = {
+        logTrace(s"Push for block $blockId to $address successful.")
+        handleResult(PushResult(blockId, null))
+      }
+
+      override def onBlockFetchFailure(blockId: String, exception: Throwable): Unit = {
+        // check the message or it's cause to see it needs to be logged.
+        if (!errorHandler.shouldLogError(exception)) {
+          logTrace(s"Pushing block $blockId to $address failed.", exception)
+        } else {
+          logWarning(s"Pushing block $blockId to $address failed.", exception)
+        }
+        handleResult(PushResult(blockId, exception))
+      }
+    }
+    SparkEnv.get.blockManager.blockStoreClient.pushBlocks(
+      address.host, address.port, blockIds.toArray,
+      sliceReqBufferIntoBlockBuffers(request.reqBuffer, request.blocks.map(_._2)),
+      blockPushListener)
+  }
+
+  /**
+   * Given the ManagedBuffer representing all the continuous blocks inside the shuffle data file
+   * for a PushRequest and an array of individual block sizes, load the buffer from disk into
+   * memory and slice it into multiple smaller buffers representing each block.
+   *
+   * With nio ByteBuffer, the individual block buffers share data with the initial in memory
+   * buffer loaded from disk. Thus only one copy of the block data is kept in memory.
+   * @param reqBuffer A {{FileSegmentManagedBuffer}} representing all the continuous blocks in
+   *                  the shuffle data file for a PushRequest
+   * @param blockSizes Array of block sizes
+   * @return Array of in memory buffer for each individual block
+   */
+  private def sliceReqBufferIntoBlockBuffers(
+      reqBuffer: ManagedBuffer,
+      blockSizes: Seq[Int]): Array[ManagedBuffer] = {
+    if (blockSizes.size == 1) {
+      Array(reqBuffer)
+    } else {
+      val inMemoryBuffer = reqBuffer.nioByteBuffer()
+      val blockOffsets = new Array[Int](blockSizes.size)
+      var offset = 0
+      for (index <- blockSizes.indices) {
+        blockOffsets(index) = offset
+        offset += blockSizes(index)
+      }
+      blockOffsets.zip(blockSizes).map {
+        case (offset, size) =>
+          new NioManagedBuffer(inMemoryBuffer.duplicate()
+            .position(offset)
+            .limit(offset + size).asInstanceOf[ByteBuffer].slice())
+      }.toArray
+    }
+  }
+
+  /**
+   * Updates the stats and based on the previous push result decides whether to push more blocks
+   * or stop.
+   *
+   * @param bytesPushed     number of bytes pushed.
+   * @param address         address of the remote service
+   * @param remainingBlocks remaining blocks
+   * @param pushResult      result of the last push
+   * @return true if more blocks should be pushed; false otherwise.
+   */
+  private def updateStateAndCheckIfPushMore(
+      bytesPushed: Long,
+      address: BlockManagerId,
+      remainingBlocks: HashSet[String],
+      pushResult: PushResult): Boolean = synchronized {
+    remainingBlocks -= pushResult.blockId
+    bytesInFlight -= bytesPushed
+    numBlocksInFlightPerAddress(address) = numBlocksInFlightPerAddress(address) - 1
+    if (remainingBlocks.isEmpty) {
+      reqsInFlight -= 1
+    }
+    if (pushResult.failure != null && pushResult.failure.getCause.isInstanceOf[ConnectException]) {
+      // Remove all the blocks for this address just once because removing from pushRequests
+      // is expensive. If there is a ConnectException for the first block, all the subsequent
+      // blocks to that address will fail, so should avoid removing multiple times.
+      if (!unreachableBlockMgrs.contains(address)) {
+        var removed = 0
+        unreachableBlockMgrs.add(address)
+        removed += pushRequests.dequeueAll(req => req.address == address).length
+        removed += deferredPushRequests.remove(address).map(_.length).getOrElse(0)
+        logWarning(s"Received a ConnectException from $address. " +
+          s"Dropping $removed push-requests and " +
+          s"not pushing any more blocks to this address.")
+      }
+    }
+    if (pushResult.failure != null && !errorHandler.shouldRetryError(pushResult.failure)) {
+      logDebug(s"Received after merge is finalized from $address. Not pushing any more blocks.")
+      return false
+    } else {
+      remainingBlocks.isEmpty && (pushRequests.nonEmpty || deferredPushRequests.nonEmpty)
+    }
+  }
+
+  /**
+   * Convert the shuffle data file of the current mapper into a list of PushRequest. Basically,
+   * continuous blocks in the shuffle file are grouped into a single request to allow more
+   * efficient read of the block data. Each mapper for a given shuffle will receive the same
+   * list of BlockManagerIds as the target location to push the blocks to. All mappers in the
+   * same shuffle will map shuffle partition ranges to individual target locations in a consistent
+   * manner to make sure each target location receives shuffle blocks belonging to the same set
+   * of partition ranges. 0-length blocks and blocks that are large enough will be skipped.
+   *
+   * @param numPartitions sumber of shuffle partitions in the shuffle file
+   * @param partitionId map index of the current mapper
+   * @param shuffleId shuffleId of current shuffle
+   * @param dataFile shuffle data file
+   * @param partitionLengths array of sizes of blocks in the shuffle data file
+   * @param mergerLocs target locations to push blocks to
+   * @param transportConf transportConf used to create FileSegmentManagedBuffer
+   * @return List of the PushRequest, randomly shuffled.
+   *
+   * VisibleForTesting
+   */
+  private[shuffle] def prepareBlockPushRequests(
+      numPartitions: Int,
+      partitionId: Int,
+      shuffleId: Int,
+      dataFile: File,
+      partitionLengths: Array[Long],
+      mergerLocs: Seq[BlockManagerId],
+      transportConf: TransportConf): Seq[PushRequest] = {
+    var offset = 0L
+    var currentReqSize = 0
+    var currentReqOffset = 0L
+    var currentMergerId = 0
+    val numMergers = mergerLocs.length
+    val requests = new ArrayBuffer[PushRequest]
+    var blocks = new ArrayBuffer[(BlockId, Int)]
+    for (reduceId <- 0 until numPartitions) {
+      val blockSize = partitionLengths(reduceId)
+      logDebug(
+        s"Block ${ShufflePushBlockId(shuffleId, partitionId, reduceId)} is of size $blockSize")
+      // Skip 0-length blocks and blocks that are large enough
+      if (blockSize > 0) {
+        val mergerId = math.min(math.floor(reduceId * 1.0 / numPartitions * numMergers),
+          numMergers - 1).asInstanceOf[Int]
+        // Start a new PushRequest if the current request goes beyond the max batch size,
+        // or the number of blocks in the current request goes beyond the limit per destination,
+        // or the next block push location is for a different shuffle service, or the next block
+        // exceeds the max block size to push limit. This guarantees that each PushRequest
+        // represents continuous blocks in the shuffle file to be pushed to the same shuffle
+        // service, and does not go beyond existing limitations.
+        if (currentReqSize + blockSize <= maxBlockBatchSize
+          && blocks.size < maxBlocksInFlightPerAddress
+          && mergerId == currentMergerId && blockSize <= maxBlockSizeToPush) {
+          // Add current block to current batch
+          currentReqSize += blockSize.toInt
+        } else {
+          if (blocks.nonEmpty) {
+            // Convert the previous batch into a PushRequest
+            requests += PushRequest(mergerLocs(currentMergerId), blocks.toSeq,
+              createRequestBuffer(transportConf, dataFile, currentReqOffset, currentReqSize))
+            blocks = new ArrayBuffer[(BlockId, Int)]
+          }
+          // Start a new batch
+          currentReqSize = 0
+          // Set currentReqOffset to -1 so we are able to distinguish between the initial value
+          // of currentReqOffset and when we are about to start a new batch
+          currentReqOffset = -1
+          currentMergerId = mergerId
+        }
+        // Only push blocks under the size limit
+        if (blockSize <= maxBlockSizeToPush) {
+          val blockSizeInt = blockSize.toInt
+          blocks += ((ShufflePushBlockId(shuffleId, partitionId, reduceId), blockSizeInt))
+          // Only update currentReqOffset if the current block is the first in the request
+          if (currentReqOffset == -1) {
+            currentReqOffset = offset
+          }
+          if (currentReqSize == 0) {
+            currentReqSize += blockSizeInt
+          }
+        }
+      }
+      offset += blockSize
+    }
+    // Add in the final request
+    if (blocks.nonEmpty) {
+      requests += PushRequest(mergerLocs(currentMergerId), blocks.toSeq,
+        createRequestBuffer(transportConf, dataFile, currentReqOffset, currentReqSize))
+    }
+    requests.toSeq
+  }
+
+  // Visible for testing
+  protected def createRequestBuffer(
+      conf: TransportConf,
+      dataFile: File,
+      offset: Long,
+      length: Long): ManagedBuffer = {
+    new FileSegmentManagedBuffer(conf, dataFile, offset, length)
+  }
+}
+
+private[spark] object ShuffleBlockPusher {
+
+  /**
+   * A request to push blocks to a remote shuffle service
+   * @param address remote shuffle service location to push blocks to
+   * @param blocks list of block IDs and their sizes
+   * @param reqBuffer a chunk of data in the shuffle data file corresponding to the continuous
+   *                  blocks represented in this request
+   */
+  private[spark] case class PushRequest(
+    address: BlockManagerId,
+    blocks: Seq[(BlockId, Int)],
+    reqBuffer: ManagedBuffer) {
+    val size = blocks.map(_._2).sum
+  }
+
+  /**
+   * Result of the block push.
+   * @param blockId blockId
+   * @param failure exception if the push was unsuccessful; null otherwise;
+   */
+  private case class PushResult(blockId: String, failure: Throwable)
+
+  private val BLOCK_PUSHER_POOL: ExecutorService = {
+    val conf = SparkEnv.get.conf
+    if (Utils.isPushBasedShuffleEnabled(conf)) {
+      val numThreads = conf.get(SHUFFLE_NUM_PUSH_THREADS)
+        .getOrElse(conf.getInt(SparkLauncher.EXECUTOR_CORES, 1))
+      ThreadUtils.newDaemonFixedThreadPool(numThreads, "shuffle-block-push-thread")
+    } else {
+      null
+    }
+  }
+
+  /**
+   * Stop the shuffle pusher pool if it isn't null.
+   */
+  private[spark] def stop(): Unit = {
+    if (BLOCK_PUSHER_POOL != null) {
+      BLOCK_PUSHER_POOL.shutdown()
+    }
+  }
+}
diff --git a/core/src/main/scala/org/apache/spark/shuffle/ShuffleManager.scala b/core/src/main/scala/org/apache/spark/shuffle/ShuffleManager.scala
index 057b0d6e0b0a7..4e2183451c258 100644
--- a/core/src/main/scala/org/apache/spark/shuffle/ShuffleManager.scala
+++ b/core/src/main/scala/org/apache/spark/shuffle/ShuffleManager.scala
@@ -24,8 +24,12 @@ import org.apache.spark.{ShuffleDependency, TaskContext}
  * and on each executor, based on the spark.shuffle.manager setting. The driver registers shuffles
  * with it, and executors (or tasks running locally in the driver) can ask to read and write data.
  *
- * NOTE: this will be instantiated by SparkEnv so its constructor can take a SparkConf and
+ * NOTE:
+ * 1. This will be instantiated by SparkEnv so its constructor can take a SparkConf and
  * boolean isDriver as parameters.
+ * 2. This contains a method ShuffleBlockResolver which interacts with External Shuffle Service
+ * when it is enabled. Need to pay attention to that, if implementing a custom ShuffleManager, to
+ * make sure the custom ShuffleManager could co-exist with External Shuffle Service.
  */
 private[spark] trait ShuffleManager {
 
@@ -43,23 +47,31 @@ private[spark] trait ShuffleManager {
       context: TaskContext,
       metrics: ShuffleWriteMetricsReporter): ShuffleWriter[K, V]
 
+
   /**
-   * Get a reader for a range of reduce partitions (startPartition to endPartition-1, inclusive).
+   * Get a reader for a range of reduce partitions (startPartition to endPartition-1, inclusive) to
+   * read from all map outputs of the shuffle.
+   *
    * Called on executors by reduce tasks.
    */
-  def getReader[K, C](
+  final def getReader[K, C](
       handle: ShuffleHandle,
       startPartition: Int,
       endPartition: Int,
       context: TaskContext,
-      metrics: ShuffleReadMetricsReporter): ShuffleReader[K, C]
+      metrics: ShuffleReadMetricsReporter): ShuffleReader[K, C] = {
+    getReader(handle, 0, Int.MaxValue, startPartition, endPartition, context, metrics)
+  }
 
   /**
    * Get a reader for a range of reduce partitions (startPartition to endPartition-1, inclusive) to
-   * read from map output (startMapIndex to endMapIndex - 1, inclusive).
+   * read from a range of map outputs(startMapIndex to endMapIndex-1, inclusive).
+   * If endMapIndex=Int.MaxValue, the actual endMapIndex will be changed to the length of total map
+   * outputs of the shuffle in `getMapSizesByExecutorId`.
+   *
    * Called on executors by reduce tasks.
    */
-  def getReaderForRange[K, C](
+  def getReader[K, C](
       handle: ShuffleHandle,
       startMapIndex: Int,
       endMapIndex: Int,
diff --git a/core/src/main/scala/org/apache/spark/shuffle/ShuffleWriteProcessor.scala b/core/src/main/scala/org/apache/spark/shuffle/ShuffleWriteProcessor.scala
index 1429144c6f6e2..abff650b0611b 100644
--- a/core/src/main/scala/org/apache/spark/shuffle/ShuffleWriteProcessor.scala
+++ b/core/src/main/scala/org/apache/spark/shuffle/ShuffleWriteProcessor.scala
@@ -21,6 +21,7 @@ import org.apache.spark.{Partition, ShuffleDependency, SparkEnv, TaskContext}
 import org.apache.spark.internal.Logging
 import org.apache.spark.rdd.RDD
 import org.apache.spark.scheduler.MapStatus
+import org.apache.spark.util.Utils
 
 /**
  * The interface for customizing shuffle write process. The driver create a ShuffleWriteProcessor
@@ -57,7 +58,23 @@ private[spark] class ShuffleWriteProcessor extends Serializable with Logging {
         createMetricsReporter(context))
       writer.write(
         rdd.iterator(partition, context).asInstanceOf[Iterator[_ <: Product2[Any, Any]]])
-      writer.stop(success = true).get
+      val mapStatus = writer.stop(success = true)
+      if (mapStatus.isDefined) {
+        // Initiate shuffle push process if push based shuffle is enabled
+        // The map task only takes care of converting the shuffle data file into multiple
+        // block push requests. It delegates pushing the blocks to a different thread-pool -
+        // ShuffleBlockPusher.BLOCK_PUSHER_POOL.
+        if (Utils.isPushBasedShuffleEnabled(SparkEnv.get.conf) && dep.getMergerLocs.nonEmpty) {
+          manager.shuffleBlockResolver match {
+            case resolver: IndexShuffleBlockResolver =>
+              val dataFile = resolver.getDataFile(dep.shuffleId, mapId)
+              new ShuffleBlockPusher(SparkEnv.get.conf)
+                .initiateBlockPush(dataFile, writer.getPartitionLengths(), dep, partition.index)
+            case _ =>
+          }
+        }
+      }
+      mapStatus.get
     } catch {
       case e: Exception =>
         try {
diff --git a/core/src/main/scala/org/apache/spark/shuffle/ShuffleWriter.scala b/core/src/main/scala/org/apache/spark/shuffle/ShuffleWriter.scala
index 4cc4ef5f1886e..a279b4c8f42f4 100644
--- a/core/src/main/scala/org/apache/spark/shuffle/ShuffleWriter.scala
+++ b/core/src/main/scala/org/apache/spark/shuffle/ShuffleWriter.scala
@@ -31,4 +31,7 @@ private[spark] abstract class ShuffleWriter[K, V] {
 
   /** Close this writer, passing along whether the map completed */
   def stop(success: Boolean): Option[MapStatus]
+
+  /** Get the lengths of each partition */
+  def getPartitionLengths(): Array[Long]
 }
diff --git a/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala b/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala
index aefcb59b8bb87..d3cc5ed10799e 100644
--- a/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala
+++ b/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala
@@ -22,10 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
 import scala.collection.JavaConverters._
 
 import org.apache.spark._
-import org.apache.spark.internal.{config, Logging}
+import org.apache.spark.internal.Logging
 import org.apache.spark.shuffle._
-import org.apache.spark.shuffle.api.{ShuffleDataIO, ShuffleExecutorComponents}
-import org.apache.spark.util.Utils
+import org.apache.spark.shuffle.api.ShuffleExecutorComponents
 import org.apache.spark.util.collection.OpenHashSet
 
 /**
@@ -115,23 +114,14 @@ private[spark] class SortShuffleManager(conf: SparkConf) extends ShuffleManager
   }
 
   /**
-   * Get a reader for a range of reduce partitions (startPartition to endPartition-1, inclusive).
+   * Get a reader for a range of reduce partitions (startPartition to endPartition-1, inclusive) to
+   * read from a range of map outputs(startMapIndex to endMapIndex-1, inclusive).
+   * If endMapIndex=Int.MaxValue, the actual endMapIndex will be changed to the length of total map
+   * outputs of the shuffle in `getMapSizesByExecutorId`.
+   *
    * Called on executors by reduce tasks.
    */
   override def getReader[K, C](
-      handle: ShuffleHandle,
-      startPartition: Int,
-      endPartition: Int,
-      context: TaskContext,
-      metrics: ShuffleReadMetricsReporter): ShuffleReader[K, C] = {
-    val blocksByAddress = SparkEnv.get.mapOutputTracker.getMapSizesByExecutorId(
-      handle.shuffleId, startPartition, endPartition)
-    new BlockStoreShuffleReader(
-      handle.asInstanceOf[BaseShuffleHandle[K, _, C]], blocksByAddress, context, metrics,
-      shouldBatchFetch = canUseBatchFetch(startPartition, endPartition, context))
-  }
-
-  override def getReaderForRange[K, C](
       handle: ShuffleHandle,
       startMapIndex: Int,
       endMapIndex: Int,
@@ -139,7 +129,7 @@ private[spark] class SortShuffleManager(conf: SparkConf) extends ShuffleManager
       endPartition: Int,
       context: TaskContext,
       metrics: ShuffleReadMetricsReporter): ShuffleReader[K, C] = {
-    val blocksByAddress = SparkEnv.get.mapOutputTracker.getMapSizesByRange(
+    val blocksByAddress = SparkEnv.get.mapOutputTracker.getMapSizesByExecutorId(
       handle.shuffleId, startMapIndex, endMapIndex, startPartition, endPartition)
     new BlockStoreShuffleReader(
       handle.asInstanceOf[BaseShuffleHandle[K, _, C]], blocksByAddress, context, metrics,
@@ -154,7 +144,7 @@ private[spark] class SortShuffleManager(conf: SparkConf) extends ShuffleManager
       metrics: ShuffleWriteMetricsReporter): ShuffleWriter[K, V] = {
     val mapTaskIds = taskIdMapsForShuffle.computeIfAbsent(
       handle.shuffleId, _ => new OpenHashSet[Long](16))
-    mapTaskIds.synchronized { mapTaskIds.add(context.taskAttemptId()) }
+    mapTaskIds.synchronized { mapTaskIds.add(mapId) }
     val env = SparkEnv.get
     handle match {
       case unsafeShuffleHandle: SerializedShuffleHandle[K @unchecked, V @unchecked] =>
@@ -176,8 +166,7 @@ private[spark] class SortShuffleManager(conf: SparkConf) extends ShuffleManager
           metrics,
           shuffleExecutorComponents)
       case other: BaseShuffleHandle[K @unchecked, V @unchecked, _] =>
-        new SortShuffleWriter(
-          shuffleBlockResolver, other, mapId, context, shuffleExecutorComponents)
+        new SortShuffleWriter(other, mapId, context, shuffleExecutorComponents)
     }
   }
 
diff --git a/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleWriter.scala b/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleWriter.scala
index a391bdf2db44e..adbe6eca5800b 100644
--- a/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleWriter.scala
+++ b/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleWriter.scala
@@ -20,12 +20,11 @@ package org.apache.spark.shuffle.sort
 import org.apache.spark._
 import org.apache.spark.internal.{config, Logging}
 import org.apache.spark.scheduler.MapStatus
-import org.apache.spark.shuffle.{BaseShuffleHandle, IndexShuffleBlockResolver, ShuffleWriter}
+import org.apache.spark.shuffle.{BaseShuffleHandle, ShuffleWriter}
 import org.apache.spark.shuffle.api.ShuffleExecutorComponents
 import org.apache.spark.util.collection.ExternalSorter
 
 private[spark] class SortShuffleWriter[K, V, C](
-    shuffleBlockResolver: IndexShuffleBlockResolver,
     handle: BaseShuffleHandle[K, V, C],
     mapId: Long,
     context: TaskContext,
@@ -45,6 +44,8 @@ private[spark] class SortShuffleWriter[K, V, C](
 
   private var mapStatus: MapStatus = null
 
+  private var partitionLengths: Array[Long] = _
+
   private val writeMetrics = context.taskMetrics().shuffleWriteMetrics
 
   /** Write a bunch of records to this task's output */
@@ -67,7 +68,7 @@ private[spark] class SortShuffleWriter[K, V, C](
     val mapOutputWriter = shuffleExecutorComponents.createMapOutputWriter(
       dep.shuffleId, mapId, dep.partitioner.numPartitions)
     sorter.writePartitionedMapOutput(dep.shuffleId, mapId, mapOutputWriter)
-    val partitionLengths = mapOutputWriter.commitAllPartitions()
+    partitionLengths = mapOutputWriter.commitAllPartitions().getPartitionLengths
     mapStatus = MapStatus(blockManager.shuffleServerId, partitionLengths, mapId)
   }
 
@@ -93,6 +94,8 @@ private[spark] class SortShuffleWriter[K, V, C](
       }
     }
   }
+
+  override def getPartitionLengths(): Array[Long] = partitionLengths
 }
 
 private[spark] object SortShuffleWriter {
diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
index c3f22f32993a8..7c403c4ec9ede 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
@@ -24,10 +24,11 @@ import scala.collection.JavaConverters._
 import scala.collection.mutable.HashMap
 
 import org.apache.spark._
-import org.apache.spark.executor.TaskMetrics
+import org.apache.spark.executor.{ExecutorMetrics, TaskMetrics}
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.CPUS_PER_TASK
 import org.apache.spark.internal.config.Status._
+import org.apache.spark.resource.ResourceProfile.CPUS
 import org.apache.spark.scheduler._
 import org.apache.spark.status.api.v1
 import org.apache.spark.storage._
@@ -51,7 +52,7 @@ private[spark] class AppStatusListener(
   private var sparkVersion = SPARK_VERSION
   private var appInfo: v1.ApplicationInfo = null
   private var appSummary = new AppSummary(0, 0)
-  private var coresPerTask: Int = 1
+  private var defaultCpusPerTask: Int = 1
 
   // How often to update live entities. -1 means "never update" when replaying applications,
   // meaning only the last write will happen. For live applications, this avoids a few
@@ -76,6 +77,8 @@ private[spark] class AppStatusListener(
   private val liveTasks = new HashMap[Long, LiveTask]()
   private val liveRDDs = new HashMap[Int, LiveRDD]()
   private val pools = new HashMap[String, SchedulerPool]()
+  private val liveResourceProfiles = new HashMap[Int, LiveResourceProfile]()
+  private[spark] val liveMiscellaneousProcess = new HashMap[String, LiveMiscellaneousProcess]()
 
   private val SQL_EXECUTION_ID_KEY = "spark.sql.execution.id"
   // Keep the active executor count as a separate variable to avoid having to do synchronization
@@ -105,6 +108,8 @@ private[spark] class AppStatusListener(
 
   override def onOtherEvent(event: SparkListenerEvent): Unit = event match {
     case SparkListenerLogStart(version) => sparkVersion = version
+    case processInfoEvent: SparkListenerMiscellaneousProcessAdded =>
+      onMiscellaneousProcessAdded(processInfoEvent)
     case _ =>
   }
 
@@ -145,6 +150,20 @@ private[spark] class AppStatusListener(
     }
   }
 
+  override def onResourceProfileAdded(event: SparkListenerResourceProfileAdded): Unit = {
+    val maxTasks = if (event.resourceProfile.isCoresLimitKnown) {
+      Some(event.resourceProfile.maxTasksPerExecutor(conf))
+    } else {
+      None
+    }
+    val liveRP = new LiveResourceProfile(event.resourceProfile.id,
+      event.resourceProfile.executorResources, event.resourceProfile.taskResources, maxTasks)
+    liveResourceProfiles(event.resourceProfile.id) = liveRP
+    val rpInfo = new v1.ResourceProfileInfo(liveRP.resourceProfileId,
+      liveRP.executorResources, liveRP.taskResources)
+    kvstore.write(new ResourceProfileWrapper(rpInfo))
+  }
+
   override def onEnvironmentUpdate(event: SparkListenerEnvironmentUpdate): Unit = {
     val details = event.environmentDetails
 
@@ -159,10 +178,11 @@ private[spark] class AppStatusListener(
       details.getOrElse("Spark Properties", Nil),
       details.getOrElse("Hadoop Properties", Nil),
       details.getOrElse("System Properties", Nil),
-      details.getOrElse("Classpath Entries", Nil))
+      details.getOrElse("Classpath Entries", Nil),
+      Nil)
 
-    coresPerTask = envInfo.sparkProperties.toMap.get(CPUS_PER_TASK.key).map(_.toInt)
-      .getOrElse(coresPerTask)
+    defaultCpusPerTask = envInfo.sparkProperties.toMap.get(CPUS_PER_TASK.key).map(_.toInt)
+      .getOrElse(defaultCpusPerTask)
 
     kvstore.write(new ApplicationEnvironmentInfoWrapper(envInfo))
   }
@@ -197,10 +217,16 @@ private[spark] class AppStatusListener(
     exec.host = event.executorInfo.executorHost
     exec.isActive = true
     exec.totalCores = event.executorInfo.totalCores
-    exec.maxTasks = event.executorInfo.totalCores / coresPerTask
+    val rpId = event.executorInfo.resourceProfileId
+    val liveRP = liveResourceProfiles.get(rpId)
+    val cpusPerTask = liveRP.flatMap(_.taskResources.get(CPUS))
+      .map(_.amount.toInt).getOrElse(defaultCpusPerTask)
+    val maxTasksPerExec = liveRP.flatMap(_.maxTasksPerExecutor)
+    exec.maxTasks = maxTasksPerExec.getOrElse(event.executorInfo.totalCores / cpusPerTask)
     exec.executorLogs = event.executorInfo.logUrlMap
     exec.resources = event.executorInfo.resourcesInfo
     exec.attributes = event.executorInfo.attributes
+    exec.resourceProfileId = rpId
     liveUpdate(exec, System.nanoTime())
   }
 
@@ -260,82 +286,143 @@ private[spark] class AppStatusListener(
     }
   }
 
+  // Note, the blacklisted functions are left here for backwards compatibility to allow
+  // new history server to properly read and display older event logs.
   override def onExecutorBlacklisted(event: SparkListenerExecutorBlacklisted): Unit = {
-    updateBlackListStatus(event.executorId, true)
+    updateExecExclusionStatus(event.executorId, true)
+  }
+
+  override def onExecutorExcluded(event: SparkListenerExecutorExcluded): Unit = {
+    updateExecExclusionStatus(event.executorId, true)
   }
 
   override def onExecutorBlacklistedForStage(
       event: SparkListenerExecutorBlacklistedForStage): Unit = {
-    val now = System.nanoTime()
+    updateExclusionStatusForStage(event.stageId, event.stageAttemptId, event.executorId)
+  }
 
-    Option(liveStages.get((event.stageId, event.stageAttemptId))).foreach { stage =>
-      setStageBlackListStatus(stage, now, event.executorId)
-    }
-    liveExecutors.get(event.executorId).foreach { exec =>
-      addBlackListedStageTo(exec, event.stageId, now)
-    }
+  override def onExecutorExcludedForStage(
+      event: SparkListenerExecutorExcludedForStage): Unit = {
+    updateExclusionStatusForStage(event.stageId, event.stageAttemptId, event.executorId)
   }
 
   override def onNodeBlacklistedForStage(event: SparkListenerNodeBlacklistedForStage): Unit = {
-    val now = System.nanoTime()
+    updateNodeExclusionStatusForStage(event.stageId, event.stageAttemptId, event.hostId)
+  }
 
-    // Implicitly blacklist every available executor for the stage associated with this node
-    Option(liveStages.get((event.stageId, event.stageAttemptId))).foreach { stage =>
-      val executorIds = liveExecutors.values.filter(_.host == event.hostId).map(_.executorId).toSeq
-      setStageBlackListStatus(stage, now, executorIds: _*)
-    }
-    liveExecutors.values.filter(_.hostname == event.hostId).foreach { exec =>
-      addBlackListedStageTo(exec, event.stageId, now)
-    }
+  override def onNodeExcludedForStage(event: SparkListenerNodeExcludedForStage): Unit = {
+    updateNodeExclusionStatusForStage(event.stageId, event.stageAttemptId, event.hostId)
   }
 
-  private def addBlackListedStageTo(exec: LiveExecutor, stageId: Int, now: Long): Unit = {
-    exec.blacklistedInStages += stageId
+  private def addExcludedStageTo(exec: LiveExecutor, stageId: Int, now: Long): Unit = {
+    exec.excludedInStages += stageId
     liveUpdate(exec, now)
   }
 
   private def setStageBlackListStatus(stage: LiveStage, now: Long, executorIds: String*): Unit = {
     executorIds.foreach { executorId =>
       val executorStageSummary = stage.executorSummary(executorId)
-      executorStageSummary.isBlacklisted = true
+      executorStageSummary.isExcluded = true
       maybeUpdate(executorStageSummary, now)
     }
-    stage.blackListedExecutors ++= executorIds
+    stage.excludedExecutors ++= executorIds
+    maybeUpdate(stage, now)
+  }
+
+  private def setStageExcludedStatus(stage: LiveStage, now: Long, executorIds: String*): Unit = {
+    executorIds.foreach { executorId =>
+      val executorStageSummary = stage.executorSummary(executorId)
+      executorStageSummary.isExcluded = true
+      maybeUpdate(executorStageSummary, now)
+    }
+    stage.excludedExecutors ++= executorIds
     maybeUpdate(stage, now)
   }
 
   override def onExecutorUnblacklisted(event: SparkListenerExecutorUnblacklisted): Unit = {
-    updateBlackListStatus(event.executorId, false)
+    updateExecExclusionStatus(event.executorId, false)
+  }
+
+  override def onExecutorUnexcluded(event: SparkListenerExecutorUnexcluded): Unit = {
+    updateExecExclusionStatus(event.executorId, false)
   }
 
   override def onNodeBlacklisted(event: SparkListenerNodeBlacklisted): Unit = {
-    updateNodeBlackList(event.hostId, true)
+    updateNodeExcluded(event.hostId, true)
+  }
+
+  override def onNodeExcluded(event: SparkListenerNodeExcluded): Unit = {
+    updateNodeExcluded(event.hostId, true)
   }
 
   override def onNodeUnblacklisted(event: SparkListenerNodeUnblacklisted): Unit = {
-    updateNodeBlackList(event.hostId, false)
+    updateNodeExcluded(event.hostId, false)
+  }
+
+  override def onNodeUnexcluded(event: SparkListenerNodeUnexcluded): Unit = {
+    updateNodeExcluded(event.hostId, false)
+  }
+
+  private def updateNodeExclusionStatusForStage(stageId: Int, stageAttemptId: Int,
+      hostId: String): Unit = {
+    val now = System.nanoTime()
+
+    // Implicitly exclude every available executor for the stage associated with this node
+    Option(liveStages.get((stageId, stageAttemptId))).foreach { stage =>
+      val executorIds = liveExecutors.values.filter(exec => exec.host == hostId
+        && exec.executorId != SparkContext.DRIVER_IDENTIFIER).map(_.executorId).toSeq
+      setStageExcludedStatus(stage, now, executorIds: _*)
+    }
+    liveExecutors.values.filter(exec => exec.hostname == hostId
+      && exec.executorId != SparkContext.DRIVER_IDENTIFIER).foreach { exec =>
+      addExcludedStageTo(exec, stageId, now)
+    }
+  }
+
+  private def updateExclusionStatusForStage(stageId: Int, stageAttemptId: Int,
+      execId: String): Unit = {
+    val now = System.nanoTime()
+
+    Option(liveStages.get((stageId, stageAttemptId))).foreach { stage =>
+      setStageExcludedStatus(stage, now, execId)
+    }
+    liveExecutors.get(execId).foreach { exec =>
+      addExcludedStageTo(exec, stageId, now)
+    }
   }
 
-  private def updateBlackListStatus(execId: String, blacklisted: Boolean): Unit = {
+  private def updateExecExclusionStatus(execId: String, excluded: Boolean): Unit = {
     liveExecutors.get(execId).foreach { exec =>
-      exec.isBlacklisted = blacklisted
-      if (blacklisted) {
+      updateExecExclusionStatus(exec, excluded, System.nanoTime())
+    }
+  }
+
+  private def updateExecExclusionStatus(exec: LiveExecutor, excluded: Boolean, now: Long): Unit = {
+    // Since we are sending both blacklisted and excluded events for backwards compatibility
+    // we need to protect against double counting so don't increment if already in
+    // that state. Also protects against executor being excluded and then node being
+    // separately excluded which could result in this being called twice for same
+    // executor.
+    if (exec.isExcluded != excluded) {
+      if (excluded) {
         appStatusSource.foreach(_.BLACKLISTED_EXECUTORS.inc())
+        appStatusSource.foreach(_.EXCLUDED_EXECUTORS.inc())
       } else {
         appStatusSource.foreach(_.UNBLACKLISTED_EXECUTORS.inc())
+        appStatusSource.foreach(_.UNEXCLUDED_EXECUTORS.inc())
       }
-      liveUpdate(exec, System.nanoTime())
+      exec.isExcluded = excluded
+      liveUpdate(exec, now)
     }
   }
 
-  private def updateNodeBlackList(host: String, blacklisted: Boolean): Unit = {
+  private def updateNodeExcluded(host: String, excluded: Boolean): Unit = {
     val now = System.nanoTime()
 
-    // Implicitly (un)blacklist every executor associated with the node.
+    // Implicitly (un)exclude every executor associated with the node.
     liveExecutors.values.foreach { exec =>
-      if (exec.hostname == host) {
-        exec.isBlacklisted = blacklisted
-        liveUpdate(exec, now)
+      if (exec.hostname == host && exec.executorId != SparkContext.DRIVER_IDENTIFIER) {
+        updateExecExclusionStatus(exec, excluded, now)
       }
     }
   }
@@ -605,6 +692,10 @@ private[spark] class AppStatusListener(
         stage.killedSummary = killedTasksSummary(event.reason, stage.killedSummary)
       }
       stage.activeTasksPerExecutor(event.taskInfo.executorId) -= 1
+
+      stage.peakExecutorMetrics.compareAndUpdatePeakValues(event.taskExecutorMetrics)
+      stage.executorSummary(event.taskInfo.executorId).peakExecutorMetrics
+        .compareAndUpdatePeakValues(event.taskExecutorMetrics)
       // [SPARK-24415] Wait for all tasks to finish before removing stage from live list
       val removeStage =
         stage.activeTasks == 0 &&
@@ -671,6 +762,7 @@ private[spark] class AppStatusListener(
       exec.completedTasks += completedDelta
       exec.failedTasks += failedDelta
       exec.totalDuration += event.taskInfo.duration
+      exec.peakExecutorMetrics.compareAndUpdatePeakValues(event.taskExecutorMetrics)
 
       // Note: For resubmitted tasks, we continue to use the metrics that belong to the
       // first attempt of this task. This may not be 100% accurate because the first attempt
@@ -736,10 +828,10 @@ private[spark] class AppStatusListener(
         update(pool, now)
       }
 
-      val executorIdsForStage = stage.blackListedExecutors
+      val executorIdsForStage = stage.excludedExecutors
       executorIdsForStage.foreach { executorId =>
         liveExecutors.get(executorId).foreach { exec =>
-          removeBlackListedStageFrom(exec, event.stageInfo.stageId, now)
+          removeExcludedStageFrom(exec, event.stageInfo.stageId, now)
         }
       }
 
@@ -759,8 +851,8 @@ private[spark] class AppStatusListener(
     deadExecutors.retain((execId, exec) => isExecutorActiveForLiveStages(exec))
   }
 
-  private def removeBlackListedStageFrom(exec: LiveExecutor, stageId: Int, now: Long) = {
-    exec.blacklistedInStages -= stageId
+  private def removeExcludedStageFrom(exec: LiveExecutor, stageId: Int, now: Long) = {
+    exec.excludedInStages -= stageId
     liveUpdate(exec, now)
   }
 
@@ -845,13 +937,17 @@ private[spark] class AppStatusListener(
     // check if there is a new peak value for any of the executor level memory metrics
     // for the live UI. SparkListenerExecutorMetricsUpdate events are only processed
     // for the live UI.
-    event.executorUpdates.foreach { case (_, peakUpdates) =>
+    event.executorUpdates.foreach { case (key, peakUpdates) =>
       liveExecutors.get(event.execId).foreach { exec =>
         if (exec.peakExecutorMetrics.compareAndUpdatePeakValues(peakUpdates)) {
-          maybeUpdate(exec, now)
+          update(exec, now)
         }
       }
+
+      // Update stage level peak executor metrics.
+      updateStageLevelPeakExecutorMetrics(key._1, key._2, event.execId, peakUpdates, now)
     }
+
     // Flush updates if necessary. Executor heartbeat is an event that happens periodically. Flush
     // here to ensure the staleness of Spark UI doesn't last more than
     // `max(heartbeat interval, liveUpdateMinFlushPeriod)`.
@@ -862,17 +958,38 @@ private[spark] class AppStatusListener(
     }
   }
 
-  override def onStageExecutorMetrics(executorMetrics: SparkListenerStageExecutorMetrics): Unit = {
+  override def onStageExecutorMetrics(event: SparkListenerStageExecutorMetrics): Unit = {
     val now = System.nanoTime()
 
     // check if there is a new peak value for any of the executor level memory metrics,
     // while reading from the log. SparkListenerStageExecutorMetrics are only processed
     // when reading logs.
-    liveExecutors.get(executorMetrics.execId).orElse(
-      deadExecutors.get(executorMetrics.execId)).foreach { exec =>
-        if (exec.peakExecutorMetrics.compareAndUpdatePeakValues(executorMetrics.executorMetrics)) {
-          update(exec, now)
-        }
+    liveExecutors.get(event.execId).orElse(
+      deadExecutors.get(event.execId)).foreach { exec =>
+      if (exec.peakExecutorMetrics.compareAndUpdatePeakValues(event.executorMetrics)) {
+        update(exec, now)
+      }
+    }
+
+    // Update stage level peak executor metrics.
+    updateStageLevelPeakExecutorMetrics(
+      event.stageId, event.stageAttemptId, event.execId, event.executorMetrics, now)
+  }
+
+  private def updateStageLevelPeakExecutorMetrics(
+      stageId: Int,
+      stageAttemptId: Int,
+      executorId: String,
+      executorMetrics: ExecutorMetrics,
+      now: Long): Unit = {
+    Option(liveStages.get((stageId, stageAttemptId))).foreach { stage =>
+      if (stage.peakExecutorMetrics.compareAndUpdatePeakValues(executorMetrics)) {
+        update(stage, now)
+      }
+      val esummary = stage.executorSummary(executorId)
+      if (esummary.peakExecutorMetrics.compareAndUpdatePeakValues(executorMetrics)) {
+        update(esummary, now)
+      }
     }
   }
 
@@ -904,7 +1021,7 @@ private[spark] class AppStatusListener(
    */
   def activeStages(): Seq[v1.StageData] = {
     liveStages.values.asScala
-      .filter(_.info.submissionTime.isDefined)
+      .filter(s => Option(s.info).exists(_.submissionTime.isDefined))
       .map(_.toApi())
       .toList
       .sortBy(_.stageId)
@@ -1010,6 +1127,13 @@ private[spark] class AppStatusListener(
     })
   }
 
+  private def getOrCreateOtherProcess(processId: String,
+      addTime: Long): LiveMiscellaneousProcess = {
+    liveMiscellaneousProcess.getOrElseUpdate(processId, {
+      new LiveMiscellaneousProcess(processId, addTime)
+    })
+  }
+
   private def updateStreamBlock(event: SparkListenerBlockUpdated, stream: StreamBlockId): Unit = {
     val storageLevel = event.blockUpdatedInfo.storageLevel
     if (storageLevel.isValid) {
@@ -1065,7 +1189,7 @@ private[spark] class AppStatusListener(
 
   private def getOrCreateStage(info: StageInfo): LiveStage = {
     val stage = liveStages.computeIfAbsent((info.stageId, info.attemptNumber),
-      (_: (Int, Int)) => new LiveStage())
+      (_: (Int, Int)) => new LiveStage(info))
     stage.info = info
     stage
   }
@@ -1239,4 +1363,16 @@ private[spark] class AppStatusListener(
     }
   }
 
+  private def onMiscellaneousProcessAdded(
+      processInfoEvent: SparkListenerMiscellaneousProcessAdded): Unit = {
+    val processInfo = processInfoEvent.info
+    val miscellaneousProcess =
+      getOrCreateOtherProcess(processInfoEvent.processId, processInfoEvent.time)
+    miscellaneousProcess.processLogs = processInfo.logUrlInfo
+    miscellaneousProcess.hostPort = processInfo.hostPort
+    miscellaneousProcess.isActive = true
+    miscellaneousProcess.totalCores = processInfo.cores
+    update(miscellaneousProcess, System.nanoTime())
+  }
+
 }
diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusSource.scala b/core/src/main/scala/org/apache/spark/status/AppStatusSource.scala
index 20f171bd3c375..d19744db089ba 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusSource.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusSource.scala
@@ -59,9 +59,25 @@ private[spark] class AppStatusSource extends Source {
 
   val SKIPPED_TASKS = getCounter("tasks", "skippedTasks")
 
+  // This is the count of how many executors have been blacklisted at the application level,
+  // does not include stage level blacklisting.
+  // this is private but user visible from metrics so just deprecate
+  @deprecated("use excludedExecutors instead", "3.1.0")
   val BLACKLISTED_EXECUTORS = getCounter("tasks", "blackListedExecutors")
 
+  // This is the count of how many executors have been unblacklisted at the application level,
+  // does not include stage level unblacklisting.
+  @deprecated("use unexcludedExecutors instead", "3.1.0")
   val UNBLACKLISTED_EXECUTORS = getCounter("tasks", "unblackListedExecutors")
+
+  // This is the count of how many executors have been excluded at the application level,
+  // does not include stage level exclusion.
+  val EXCLUDED_EXECUTORS = getCounter("tasks", "excludedExecutors")
+
+  // This is the count of how many executors have been unexcluded at the application level,
+  // does not include stage level unexclusion.
+  val UNEXCLUDED_EXECUTORS = getCounter("tasks", "unexcludedExecutors")
+
 }
 
 private[spark] object AppStatusSource {
diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
index 42d4071d61c8b..a8a16cda96c87 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
@@ -22,8 +22,9 @@ import java.util.{List => JList}
 import scala.collection.JavaConverters._
 import scala.collection.mutable.HashMap
 
-import org.apache.spark.{JobExecutionStatus, SparkConf, SparkException}
+import org.apache.spark.{JobExecutionStatus, SparkConf}
 import org.apache.spark.status.api.v1
+import org.apache.spark.storage.FallbackStorage.FALLBACK_BLOCK_MANAGER_ID
 import org.apache.spark.ui.scope._
 import org.apache.spark.util.Utils
 import org.apache.spark.util.kvstore.{InMemoryStore, KVStore}
@@ -38,10 +39,16 @@ private[spark] class AppStatusStore(
   def applicationInfo(): v1.ApplicationInfo = {
     try {
       // The ApplicationInfo may not be available when Spark is starting up.
-      store.view(classOf[ApplicationInfoWrapper]).max(1).iterator().next().info
+      Utils.tryWithResource(
+        store.view(classOf[ApplicationInfoWrapper])
+          .max(1)
+          .closeableIterator()
+      ) { it =>
+        it.next().info
+      }
     } catch {
       case _: NoSuchElementException =>
-        throw new SparkException("Failed to get the application information. " +
+        throw new NoSuchElementException("Failed to get the application information. " +
           "If you are starting up Spark, please wait a while until it's ready.")
     }
   }
@@ -51,6 +58,10 @@ private[spark] class AppStatusStore(
     store.read(klass, klass.getName()).info
   }
 
+  def resourceProfileInfo(): Seq[v1.ResourceProfileInfo] = {
+    store.view(classOf[ResourceProfileWrapper]).asScala.map(_.rpInfo).toSeq
+  }
+
   def jobsList(statuses: JList[JobExecutionStatus]): Seq[v1.JobData] = {
     val it = store.view(classOf[JobDataWrapper]).reverse().asScala.map(_.info)
     if (statuses != null && !statuses.isEmpty()) {
@@ -78,6 +89,16 @@ private[spark] class AppStatusStore(
     } else {
       base
     }
+    filtered.asScala.map(_.info).filter(_.id != FALLBACK_BLOCK_MANAGER_ID.executorId).toSeq
+  }
+
+  def miscellaneousProcessList(activeOnly: Boolean): Seq[v1.ProcessSummary] = {
+    val base = store.view(classOf[ProcessSummaryWrapper])
+    val filtered = if (activeOnly) {
+      base.index("active").reverse().first(true).last(true)
+    } else {
+      base
+    }
     filtered.asScala.map(_.info).toSeq
   }
 
@@ -93,19 +114,35 @@ private[spark] class AppStatusStore(
     listener.map(_.activeStages()).getOrElse(Nil)
   }
 
-  def stageList(statuses: JList[v1.StageStatus]): Seq[v1.StageData] = {
+  def stageList(
+    statuses: JList[v1.StageStatus],
+    details: Boolean = false,
+    withSummaries: Boolean = false,
+    unsortedQuantiles: Array[Double] = Array.empty,
+    taskStatus: JList[v1.TaskStatus] = List().asJava): Seq[v1.StageData] = {
+    val quantiles = unsortedQuantiles.sorted
     val it = store.view(classOf[StageDataWrapper]).reverse().asScala.map(_.info)
-    if (statuses != null && !statuses.isEmpty()) {
+    val ret = if (statuses != null && !statuses.isEmpty()) {
       it.filter { s => statuses.contains(s.status) }.toSeq
     } else {
       it.toSeq
     }
+    ret.map { s =>
+      newStageData(s, withDetail = details, taskStatus = taskStatus,
+        withSummaries = withSummaries, unsortedQuantiles = quantiles)
+    }
   }
 
-  def stageData(stageId: Int, details: Boolean = false): Seq[v1.StageData] = {
+  def stageData(
+    stageId: Int,
+    details: Boolean = false,
+    taskStatus: JList[v1.TaskStatus] = List().asJava,
+    withSummaries: Boolean = false,
+    unsortedQuantiles: Array[Double] = Array.empty[Double]): Seq[v1.StageData] = {
     store.view(classOf[StageDataWrapper]).index("stageId").first(stageId).last(stageId)
       .asScala.map { s =>
-        if (details) stageWithDetails(s.info) else s.info
+        newStageData(s.info, withDetail = details, taskStatus = taskStatus,
+          withSummaries = withSummaries, unsortedQuantiles = unsortedQuantiles)
       }.toSeq
   }
 
@@ -127,11 +164,16 @@ private[spark] class AppStatusStore(
     }
   }
 
-  def stageAttempt(stageId: Int, stageAttemptId: Int,
-      details: Boolean = false): (v1.StageData, Seq[Int]) = {
+  def stageAttempt(
+      stageId: Int, stageAttemptId: Int,
+      details: Boolean = false,
+      taskStatus: JList[v1.TaskStatus] = List().asJava,
+      withSummaries: Boolean = false,
+      unsortedQuantiles: Array[Double] = Array.empty[Double]): (v1.StageData, Seq[Int]) = {
     val stageKey = Array(stageId, stageAttemptId)
     val stageDataWrapper = store.read(classOf[StageDataWrapper], stageKey)
-    val stage = if (details) stageWithDetails(stageDataWrapper.info) else stageDataWrapper.info
+    val stage = newStageData(stageDataWrapper.info, withDetail = details, taskStatus = taskStatus,
+      withSummaries = withSummaries, unsortedQuantiles = unsortedQuantiles)
     (stage, stageDataWrapper.jobIds.toSeq)
   }
 
@@ -196,6 +238,7 @@ private[spark] class AppStatusStore(
 
       val distributions = new v1.TaskMetricDistributions(
         quantiles = quantiles,
+        duration = toValues(_.duration),
         executorDeserializeTime = toValues(_.executorDeserializeTime),
         executorDeserializeCpuTime = toValues(_.executorDeserializeCpuTime),
         executorRunTime = toValues(_.executorRunTime),
@@ -265,6 +308,9 @@ private[spark] class AppStatusStore(
 
     val computedQuantiles = new v1.TaskMetricDistributions(
       quantiles = quantiles,
+      duration = scanTasks(TaskIndexNames.DURATION) { t =>
+        t.duration
+      },
       executorDeserializeTime = scanTasks(TaskIndexNames.DESER_TIME) { t =>
         t.executorDeserializeTime
       },
@@ -316,6 +362,7 @@ private[spark] class AppStatusStore(
       .filter { case (q, _) => quantiles.contains(q) && shouldCacheQuantile(q) }
       .foreach { case (q, idx) =>
         val cached = new CachedQuantile(stageId, stageAttemptId, quantileToString(q), count,
+          duration = computedQuantiles.duration(idx),
           executorDeserializeTime = computedQuantiles.executorDeserializeTime(idx),
           executorDeserializeCpuTime = computedQuantiles.executorDeserializeCpuTime(idx),
           executorRunTime = computedQuantiles.executorRunTime(idx),
@@ -375,7 +422,8 @@ private[spark] class AppStatusStore(
       stageAttemptId: Int,
       offset: Int,
       length: Int,
-      sortBy: v1.TaskSorting): Seq[v1.TaskData] = {
+      sortBy: v1.TaskSorting,
+      statuses: JList[v1.TaskStatus]): Seq[v1.TaskData] = {
     val (indexName, ascending) = sortBy match {
       case v1.TaskSorting.ID =>
         (None, true)
@@ -384,7 +432,7 @@ private[spark] class AppStatusStore(
       case v1.TaskSorting.DECREASING_RUNTIME =>
         (Some(TaskIndexNames.EXEC_RUN_TIME), false)
     }
-    taskList(stageId, stageAttemptId, offset, length, indexName, ascending)
+    taskList(stageId, stageAttemptId, offset, length, indexName, ascending, statuses)
   }
 
   def taskList(
@@ -393,7 +441,8 @@ private[spark] class AppStatusStore(
       offset: Int,
       length: Int,
       sortBy: Option[String],
-      ascending: Boolean): Seq[v1.TaskData] = {
+      ascending: Boolean,
+      statuses: JList[v1.TaskStatus] = List().asJava): Seq[v1.TaskData] = {
     val stageKey = Array(stageId, stageAttemptId)
     val base = store.view(classOf[TaskDataWrapper])
     val indexed = sortBy match {
@@ -406,7 +455,13 @@ private[spark] class AppStatusStore(
     }
 
     val ordered = if (ascending) indexed else indexed.reverse()
-    val taskDataWrapperIter = ordered.skip(offset).max(length).asScala
+    val taskDataWrapperIter = if (statuses != null && !statuses.isEmpty) {
+      val statusesStr = statuses.asScala.map(_.toString).toSet
+      ordered.asScala.filter(s => statusesStr.contains(s.status)).slice(offset, offset + length)
+    } else {
+      ordered.skip(offset).max(length).asScala
+    }
+
     constructTaskDataList(taskDataWrapperIter)
   }
 
@@ -434,59 +489,140 @@ private[spark] class AppStatusStore(
     }
   }
 
-  private def stageWithDetails(stage: v1.StageData): v1.StageData = {
-    val tasks = taskList(stage.stageId, stage.attemptId, Int.MaxValue)
-      .map { t => (t.taskId, t) }
-      .toMap
-
-    new v1.StageData(
-      status = stage.status,
-      stageId = stage.stageId,
-      attemptId = stage.attemptId,
-      numTasks = stage.numTasks,
-      numActiveTasks = stage.numActiveTasks,
-      numCompleteTasks = stage.numCompleteTasks,
-      numFailedTasks = stage.numFailedTasks,
-      numKilledTasks = stage.numKilledTasks,
-      numCompletedIndices = stage.numCompletedIndices,
-      submissionTime = stage.submissionTime,
-      firstTaskLaunchedTime = stage.firstTaskLaunchedTime,
-      completionTime = stage.completionTime,
-      failureReason = stage.failureReason,
-      executorDeserializeTime = stage.executorDeserializeTime,
-      executorDeserializeCpuTime = stage.executorDeserializeCpuTime,
-      executorRunTime = stage.executorRunTime,
-      executorCpuTime = stage.executorCpuTime,
-      resultSize = stage.resultSize,
-      jvmGcTime = stage.jvmGcTime,
-      resultSerializationTime = stage.resultSerializationTime,
-      memoryBytesSpilled = stage.memoryBytesSpilled,
-      diskBytesSpilled = stage.diskBytesSpilled,
-      peakExecutionMemory = stage.peakExecutionMemory,
-      inputBytes = stage.inputBytes,
-      inputRecords = stage.inputRecords,
-      outputBytes = stage.outputBytes,
-      outputRecords = stage.outputRecords,
-      shuffleRemoteBlocksFetched = stage.shuffleRemoteBlocksFetched,
-      shuffleLocalBlocksFetched = stage.shuffleLocalBlocksFetched,
-      shuffleFetchWaitTime = stage.shuffleFetchWaitTime,
-      shuffleRemoteBytesRead = stage.shuffleRemoteBytesRead,
-      shuffleRemoteBytesReadToDisk = stage.shuffleRemoteBytesReadToDisk,
-      shuffleLocalBytesRead = stage.shuffleLocalBytesRead,
-      shuffleReadBytes = stage.shuffleReadBytes,
-      shuffleReadRecords = stage.shuffleReadRecords,
-      shuffleWriteBytes = stage.shuffleWriteBytes,
-      shuffleWriteTime = stage.shuffleWriteTime,
-      shuffleWriteRecords = stage.shuffleWriteRecords,
-      name = stage.name,
-      description = stage.description,
-      details = stage.details,
-      schedulingPool = stage.schedulingPool,
-      rddIds = stage.rddIds,
-      accumulatorUpdates = stage.accumulatorUpdates,
-      tasks = Some(tasks),
-      executorSummary = Some(executorSummary(stage.stageId, stage.attemptId)),
-      killedTasksSummary = stage.killedTasksSummary)
+  def newStageData(
+    stage: v1.StageData,
+    withDetail: Boolean = false,
+    taskStatus: JList[v1.TaskStatus] = List().asJava,
+    withSummaries: Boolean = false,
+    unsortedQuantiles: Array[Double] = Array.empty[Double]): v1.StageData = {
+    if (!withDetail && !withSummaries) {
+      stage
+    } else {
+      val quantiles = unsortedQuantiles.sorted
+      val tasks: Option[Map[Long, v1.TaskData]] = if (withDetail) {
+        val tasks =
+          taskList(stage.stageId, stage.attemptId, 0, Int.MaxValue, None, false, taskStatus)
+            .map { t => (t.taskId, t) }
+            .toMap
+        Some(tasks)
+      } else {
+        None
+      }
+      val executorSummaries: Option[Map[String, v1.ExecutorStageSummary]] = if (withDetail) {
+        Some(executorSummary(stage.stageId, stage.attemptId))
+      } else {
+        None
+      }
+      val taskMetricsDistribution: Option[v1.TaskMetricDistributions] = if (withSummaries) {
+        taskSummary(stage.stageId, stage.attemptId, quantiles)
+      } else {
+        None
+      }
+      val executorMetricsDistributions: Option[v1.ExecutorMetricsDistributions] =
+        if (withSummaries) {
+          stageExecutorSummary(stage.stageId, stage.attemptId, quantiles)
+        } else {
+          None
+        }
+
+      new v1.StageData(
+        status = stage.status,
+        stageId = stage.stageId,
+        attemptId = stage.attemptId,
+        numTasks = stage.numTasks,
+        numActiveTasks = stage.numActiveTasks,
+        numCompleteTasks = stage.numCompleteTasks,
+        numFailedTasks = stage.numFailedTasks,
+        numKilledTasks = stage.numKilledTasks,
+        numCompletedIndices = stage.numCompletedIndices,
+        submissionTime = stage.submissionTime,
+        firstTaskLaunchedTime = stage.firstTaskLaunchedTime,
+        completionTime = stage.completionTime,
+        failureReason = stage.failureReason,
+        executorDeserializeTime = stage.executorDeserializeTime,
+        executorDeserializeCpuTime = stage.executorDeserializeCpuTime,
+        executorRunTime = stage.executorRunTime,
+        executorCpuTime = stage.executorCpuTime,
+        resultSize = stage.resultSize,
+        jvmGcTime = stage.jvmGcTime,
+        resultSerializationTime = stage.resultSerializationTime,
+        memoryBytesSpilled = stage.memoryBytesSpilled,
+        diskBytesSpilled = stage.diskBytesSpilled,
+        peakExecutionMemory = stage.peakExecutionMemory,
+        inputBytes = stage.inputBytes,
+        inputRecords = stage.inputRecords,
+        outputBytes = stage.outputBytes,
+        outputRecords = stage.outputRecords,
+        shuffleRemoteBlocksFetched = stage.shuffleRemoteBlocksFetched,
+        shuffleLocalBlocksFetched = stage.shuffleLocalBlocksFetched,
+        shuffleFetchWaitTime = stage.shuffleFetchWaitTime,
+        shuffleRemoteBytesRead = stage.shuffleRemoteBytesRead,
+        shuffleRemoteBytesReadToDisk = stage.shuffleRemoteBytesReadToDisk,
+        shuffleLocalBytesRead = stage.shuffleLocalBytesRead,
+        shuffleReadBytes = stage.shuffleReadBytes,
+        shuffleReadRecords = stage.shuffleReadRecords,
+        shuffleWriteBytes = stage.shuffleWriteBytes,
+        shuffleWriteTime = stage.shuffleWriteTime,
+        shuffleWriteRecords = stage.shuffleWriteRecords,
+        name = stage.name,
+        description = stage.description,
+        details = stage.details,
+        schedulingPool = stage.schedulingPool,
+        rddIds = stage.rddIds,
+        accumulatorUpdates = stage.accumulatorUpdates,
+        tasks = tasks,
+        executorSummary = executorSummaries,
+        killedTasksSummary = stage.killedTasksSummary,
+        resourceProfileId = stage.resourceProfileId,
+        peakExecutorMetrics = stage.peakExecutorMetrics,
+        taskMetricsDistributions = taskMetricsDistribution,
+        executorMetricsDistributions = executorMetricsDistributions)
+    }
+  }
+
+  def stageExecutorSummary(
+    stageId: Int,
+    stageAttemptId: Int,
+    unsortedQuantiles: Array[Double]): Option[v1.ExecutorMetricsDistributions] = {
+    val quantiles = unsortedQuantiles.sorted
+    val summary = executorSummary(stageId, stageAttemptId)
+    if (summary.isEmpty) {
+      None
+    } else {
+      val values = summary.values.toIndexedSeq
+      Some(new v1.ExecutorMetricsDistributions(
+        quantiles = quantiles,
+        taskTime = getQuantilesValue(values.map(_.taskTime.toDouble).sorted, quantiles),
+        failedTasks = getQuantilesValue(values.map(_.failedTasks.toDouble).sorted, quantiles),
+        succeededTasks = getQuantilesValue(values.map(_.succeededTasks.toDouble).sorted, quantiles),
+        killedTasks = getQuantilesValue(values.map(_.killedTasks.toDouble).sorted, quantiles),
+        inputBytes = getQuantilesValue(values.map(_.inputBytes.toDouble).sorted, quantiles),
+        inputRecords = getQuantilesValue(values.map(_.inputRecords.toDouble).sorted, quantiles),
+        outputBytes = getQuantilesValue(values.map(_.outputBytes.toDouble).sorted, quantiles),
+        outputRecords = getQuantilesValue(values.map(_.outputRecords.toDouble).sorted, quantiles),
+        shuffleRead = getQuantilesValue(values.map(_.shuffleRead.toDouble).sorted, quantiles),
+        shuffleReadRecords =
+          getQuantilesValue(values.map(_.shuffleReadRecords.toDouble).sorted, quantiles),
+        shuffleWrite = getQuantilesValue(values.map(_.shuffleWrite.toDouble).sorted, quantiles),
+        shuffleWriteRecords =
+          getQuantilesValue(values.map(_.shuffleWriteRecords.toDouble).sorted, quantiles),
+        memoryBytesSpilled =
+          getQuantilesValue(values.map(_.memoryBytesSpilled.toDouble).sorted, quantiles),
+        diskBytesSpilled =
+          getQuantilesValue(values.map(_.diskBytesSpilled.toDouble).sorted, quantiles),
+        peakMemoryMetrics =
+          new v1.ExecutorPeakMetricsDistributions(quantiles,
+            values.flatMap(_.peakMemoryMetrics))
+      ))
+    }
+  }
+
+  def getQuantilesValue(
+    values: IndexedSeq[Double],
+    quantiles: Array[Double]): IndexedSeq[Double] = {
+    val count = values.size
+    val indices = quantiles.map { q => math.min((q * count).toLong, count - 1) }
+    indices.map(i => values(i.toInt)).toIndexedSeq
   }
 
   def rdd(rddId: Int): v1.RDDStorageInfo = {
@@ -519,7 +655,13 @@ private[spark] class AppStatusStore(
   }
 
   def appSummary(): AppSummary = {
-    store.read(classOf[AppSummary], classOf[AppSummary].getName())
+    try {
+      store.read(classOf[AppSummary], classOf[AppSummary].getName())
+    } catch {
+      case _: NoSuchElementException =>
+        throw new NoSuchElementException("Failed to get the application summary. " +
+          "If you are starting up Spark, please wait a while until it's ready.")
+    }
   }
 
   def close(): Unit = {
diff --git a/core/src/main/scala/org/apache/spark/status/KVUtils.scala b/core/src/main/scala/org/apache/spark/status/KVUtils.scala
index 45348be5c98b9..c79f2dcd86533 100644
--- a/core/src/main/scala/org/apache/spark/status/KVUtils.scala
+++ b/core/src/main/scala/org/apache/spark/status/KVUtils.scala
@@ -21,7 +21,6 @@ import java.io.File
 
 import scala.annotation.meta.getter
 import scala.collection.JavaConverters._
-import scala.language.implicitConversions
 import scala.reflect.{classTag, ClassTag}
 
 import com.fasterxml.jackson.annotation.JsonInclude
diff --git a/core/src/main/scala/org/apache/spark/status/LiveEntity.scala b/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
index 2714f30de14f0..5af76e9624c46 100644
--- a/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
+++ b/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
@@ -28,12 +28,12 @@ import com.google.common.collect.Interners
 
 import org.apache.spark.JobExecutionStatus
 import org.apache.spark.executor.{ExecutorMetrics, TaskMetrics}
-import org.apache.spark.resource.ResourceInformation
+import org.apache.spark.resource.{ExecutorResourceRequest, ResourceInformation, ResourceProfile, TaskResourceRequest}
 import org.apache.spark.scheduler.{AccumulableInfo, StageInfo, TaskInfo}
 import org.apache.spark.status.api.v1
 import org.apache.spark.storage.{RDDInfo, StorageLevel}
 import org.apache.spark.ui.SparkUI
-import org.apache.spark.util.AccumulatorContext
+import org.apache.spark.util.{AccumulatorContext, Utils}
 import org.apache.spark.util.collection.OpenHashSet
 
 /**
@@ -245,6 +245,21 @@ private class LiveTask(
 
 }
 
+private class LiveResourceProfile(
+    val resourceProfileId: Int,
+    val executorResources: Map[String, ExecutorResourceRequest],
+    val taskResources: Map[String, TaskResourceRequest],
+    val maxTasksPerExecutor: Option[Int]) extends LiveEntity {
+
+  def toApi(): v1.ResourceProfileInfo = {
+    new v1.ResourceProfileInfo(resourceProfileId, executorResources, taskResources)
+  }
+
+  override protected def doUpdate(): Any = {
+    new ResourceProfileWrapper(toApi())
+  }
+}
+
 private[spark] class LiveExecutor(val executorId: String, _addTime: Long) extends LiveEntity {
 
   var hostPort: String = null
@@ -271,8 +286,8 @@ private[spark] class LiveExecutor(val executorId: String, _addTime: Long) extend
   var totalInputBytes = 0L
   var totalShuffleRead = 0L
   var totalShuffleWrite = 0L
-  var isBlacklisted = false
-  var blacklistedInStages: Set[Int] = TreeSet()
+  var isExcluded = false
+  var excludedInStages: Set[Int] = TreeSet()
 
   var executorLogs = Map[String, String]()
   var attributes = Map[String, String]()
@@ -285,12 +300,14 @@ private[spark] class LiveExecutor(val executorId: String, _addTime: Long) extend
   var usedOnHeap = 0L
   var usedOffHeap = 0L
 
+  var resourceProfileId = ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID
+
   def hasMemoryInfo: Boolean = totalOnHeap >= 0L
 
   // peak values for executor level metrics
   val peakExecutorMetrics = new ExecutorMetrics()
 
-  def hostname: String = if (host != null) host else hostPort.split(":")(0)
+  def hostname: String = if (host != null) host else Utils.parseHostPort(hostPort)._1
 
   override protected def doUpdate(): Any = {
     val memoryMetrics = if (totalOnHeap >= 0) {
@@ -317,17 +334,20 @@ private[spark] class LiveExecutor(val executorId: String, _addTime: Long) extend
       totalInputBytes,
       totalShuffleRead,
       totalShuffleWrite,
-      isBlacklisted,
+      isExcluded,
       maxMemory,
       addTime,
       Option(removeTime),
       Option(removeReason),
       executorLogs,
       memoryMetrics,
-      blacklistedInStages,
+      excludedInStages,
       Some(peakExecutorMetrics).filter(_.isSet),
       attributes,
-      resources)
+      resources,
+      resourceProfileId,
+      isExcluded,
+      excludedInStages)
     new ExecutorSummaryWrapper(info)
   }
 }
@@ -343,10 +363,12 @@ private class LiveExecutorStageSummary(
   var succeededTasks = 0
   var failedTasks = 0
   var killedTasks = 0
-  var isBlacklisted = false
+  var isExcluded = false
 
   var metrics = createMetrics(default = 0L)
 
+  val peakExecutorMetrics = new ExecutorMetrics()
+
   override protected def doUpdate(): Any = {
     val info = new v1.ExecutorStageSummary(
       taskTime,
@@ -363,20 +385,21 @@ private class LiveExecutorStageSummary(
       metrics.shuffleWriteMetrics.recordsWritten,
       metrics.memoryBytesSpilled,
       metrics.diskBytesSpilled,
-      isBlacklisted)
+      isExcluded,
+      Some(peakExecutorMetrics).filter(_.isSet),
+      isExcluded)
     new ExecutorStageSummaryWrapper(stageId, attemptId, executorId, info)
   }
 
 }
 
-private class LiveStage extends LiveEntity {
+private class LiveStage(var info: StageInfo) extends LiveEntity {
 
   import LiveEntityHelpers._
 
   var jobs = Seq[LiveJob]()
   var jobIds = Set[Int]()
 
-  var info: StageInfo = null
   var status = v1.StageStatus.PENDING
 
   var description: Option[String] = None
@@ -400,11 +423,13 @@ private class LiveStage extends LiveEntity {
 
   val activeTasksPerExecutor = new HashMap[String, Int]().withDefaultValue(0)
 
-  var blackListedExecutors = new HashSet[String]()
+  var excludedExecutors = new HashSet[String]()
+
+  val peakExecutorMetrics = new ExecutorMetrics()
 
   // Used for cleanup of tasks after they reach the configured limit. Not written to the store.
   @volatile var cleaning = false
-  var savedTasks = new AtomicInteger(0)
+  val savedTasks = new AtomicInteger(0)
 
   def executorSummary(executorId: String): LiveExecutorStageSummary = {
     executorSummaries.getOrElseUpdate(executorId,
@@ -465,7 +490,11 @@ private class LiveStage extends LiveEntity {
       accumulatorUpdates = newAccumulatorInfos(info.accumulables.values),
       tasks = None,
       executorSummary = None,
-      killedTasksSummary = killedSummary)
+      killedTasksSummary = killedSummary,
+      resourceProfileId = info.resourceProfileId,
+      peakExecutorMetrics = Some(peakExecutorMetrics).filter(_.isSet),
+      taskMetricsDistributions = None,
+      executorMetricsDistributions = None)
   }
 
   override protected def doUpdate(): Any = {
@@ -885,3 +914,27 @@ private class RDDPartitionSeq extends Seq[v1.RDDPartitionInfo] {
   }
 
 }
+
+private[spark] class LiveMiscellaneousProcess(val processId: String,
+    creationTime: Long) extends LiveEntity {
+
+  var hostPort: String = null
+  var isActive = true
+  var totalCores = 0
+  val addTime = new Date(creationTime)
+  var removeTime: Date = null
+  var processLogs = Map[String, String]()
+
+  override protected def doUpdate(): Any = {
+
+    val info = new v1.ProcessSummary(
+      processId,
+      hostPort,
+      isActive,
+      totalCores,
+      addTime,
+      Option(removeTime),
+      processLogs)
+    new ProcessSummaryWrapper(info)
+  }
+}
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala
index 83f76db7e89da..cc21c1488f67c 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala
@@ -95,6 +95,8 @@ private[spark] trait UIRoot {
       .build()
   }
   def securityManager: SecurityManager
+
+  def checkUIViewPermissions(appId: String, attemptId: Option[String], user: String): Boolean
 }
 
 private[v1] object UIRootFromServletContext {
@@ -145,6 +147,19 @@ private[v1] trait BaseAppResource extends ApiRequestContext {
         throw new NotFoundException(s"no such app: $appKey")
     }
   }
+
+  protected def checkUIViewPermissions(): Unit = {
+    try {
+      val user = httpRequest.getRemoteUser()
+      if (!uiRoot.checkUIViewPermissions(appId, Option(attemptId), user)) {
+        throw new ForbiddenException(raw"""user "$user" is not authorized""")
+      }
+    } catch {
+      case _: NoSuchElementException =>
+        val appKey = Option(attemptId).map(appId + "/" + _).getOrElse(appId)
+        throw new NotFoundException(s"no such app: $appKey")
+    }
+  }
 }
 
 private[v1] class ForbiddenException(msg: String) extends WebApplicationException(
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/OneApplicationResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/OneApplicationResource.scala
index cf5c759bebdbb..ef17168ebce62 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/OneApplicationResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/OneApplicationResource.scala
@@ -19,7 +19,7 @@ package org.apache.spark.status.api.v1
 import java.io.OutputStream
 import java.util.{List => JList}
 import java.util.zip.ZipOutputStream
-import javax.ws.rs._
+import javax.ws.rs.{NotFoundException => _, _}
 import javax.ws.rs.core.{MediaType, Response, StreamingOutput}
 
 import scala.util.control.NonFatal
@@ -79,6 +79,10 @@ private[v1] class AbstractApplicationResource extends BaseAppResource {
   @Path("allexecutors")
   def allExecutorList(): Seq[ExecutorSummary] = withUI(_.store.executorList(false))
 
+  @GET
+  @Path("allmiscellaneousprocess")
+  def allProcessList(): Seq[ProcessSummary] = withUI(_.store.miscellaneousProcessList(false))
+
   @Path("stages")
   def stages(): Class[StagesResource] = classOf[StagesResource]
 
@@ -101,27 +105,28 @@ private[v1] class AbstractApplicationResource extends BaseAppResource {
   @Path("environment")
   def environmentInfo(): ApplicationEnvironmentInfo = withUI { ui =>
     val envInfo = ui.store.environmentInfo()
+    val resourceProfileInfo = ui.store.resourceProfileInfo()
     new v1.ApplicationEnvironmentInfo(
       envInfo.runtime,
-      Utils.redact(ui.conf, envInfo.sparkProperties),
-      Utils.redact(ui.conf, envInfo.hadoopProperties),
-      Utils.redact(ui.conf, envInfo.systemProperties),
-      envInfo.classpathEntries)
+      Utils.redact(ui.conf, envInfo.sparkProperties).sortBy(_._1),
+      Utils.redact(ui.conf, envInfo.hadoopProperties).sortBy(_._1),
+      Utils.redact(ui.conf, envInfo.systemProperties).sortBy(_._1),
+      envInfo.classpathEntries.sortBy(_._1),
+      resourceProfileInfo)
   }
 
   @GET
   @Path("logs")
   @Produces(Array(MediaType.APPLICATION_OCTET_STREAM))
   def getEventLogs(): Response = {
-    // Retrieve the UI for the application just to do access permission checks. For backwards
-    // compatibility, this code also tries with attemptId "1" if the UI without an attempt ID does
-    // not exist.
+    // For backwards compatibility, this code also tries with attemptId "1" if the UI
+    // without an attempt ID does not exist.
     try {
-      withUI { _ => }
+      checkUIViewPermissions()
     } catch {
       case _: NotFoundException if attemptId == null =>
         attemptId = "1"
-        withUI { _ => }
+        checkUIViewPermissions()
         attemptId = null
     }
 
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala
index 44ee322a22a10..26dfa5af101e3 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala
@@ -17,9 +17,12 @@
 package org.apache.spark.status.api.v1
 
 import java.util.{HashMap, List => JList, Locale}
-import javax.ws.rs._
+import javax.ws.rs.{NotFoundException => _, _}
 import javax.ws.rs.core.{Context, MediaType, MultivaluedMap, UriInfo}
 
+import scala.collection.JavaConverters._
+
+import org.apache.spark.status.api.v1.TaskStatus._
 import org.apache.spark.ui.UIUtils
 import org.apache.spark.ui.jobs.ApiHelper._
 import org.apache.spark.util.Utils
@@ -28,17 +31,47 @@ import org.apache.spark.util.Utils
 private[v1] class StagesResource extends BaseAppResource {
 
   @GET
-  def stageList(@QueryParam("status") statuses: JList[StageStatus]): Seq[StageData] = {
-    withUI(_.store.stageList(statuses))
+  def stageList(
+      @QueryParam("status") statuses: JList[StageStatus],
+      @QueryParam("details") @DefaultValue("false") details: Boolean,
+      @QueryParam("withSummaries") @DefaultValue("false") withSummaries: Boolean,
+      @QueryParam("quantiles") @DefaultValue("0.0,0.25,0.5,0.75,1.0") quantileString: String,
+      @QueryParam("taskStatus") taskStatus: JList[TaskStatus]): Seq[StageData] = {
+    withUI {
+      val quantiles = parseQuantileString(quantileString)
+      ui => {
+        ui.store.stageList(statuses, details, withSummaries, quantiles, taskStatus)
+          .filter { stage =>
+            if (details && taskStatus.asScala.nonEmpty) {
+              taskStatus.asScala.exists {
+                case FAILED => stage.numFailedTasks > 0
+                case KILLED => stage.numKilledTasks > 0
+                case RUNNING => stage.numActiveTasks > 0
+                case SUCCESS => stage.numCompleteTasks > 0
+                case UNKNOWN => stage.numTasks - stage.numFailedTasks - stage.numKilledTasks -
+                  stage.numActiveTasks - stage.numCompleteTasks > 0
+              }
+            } else {
+              true
+            }
+          }
+      }
+    }
   }
 
   @GET
   @Path("{stageId: \\d+}")
   def stageData(
       @PathParam("stageId") stageId: Int,
-      @QueryParam("details") @DefaultValue("true") details: Boolean): Seq[StageData] = {
+      @QueryParam("details") @DefaultValue("true") details: Boolean,
+      @QueryParam("taskStatus") taskStatus: JList[TaskStatus],
+      @QueryParam("withSummaries") @DefaultValue("false") withSummaries: Boolean,
+      @QueryParam("quantiles") @DefaultValue("0.0,0.25,0.5,0.75,1.0") quantileString: String):
+  Seq[StageData] = {
     withUI { ui =>
-      val ret = ui.store.stageData(stageId, details = details)
+      val quantiles = parseQuantileString(quantileString)
+      val ret = ui.store.stageData(stageId, details = details, taskStatus = taskStatus,
+        withSummaries = withSummaries, unsortedQuantiles = quantiles)
       if (ret.nonEmpty) {
         ret
       } else {
@@ -52,13 +85,19 @@ private[v1] class StagesResource extends BaseAppResource {
   def oneAttemptData(
       @PathParam("stageId") stageId: Int,
       @PathParam("stageAttemptId") stageAttemptId: Int,
-      @QueryParam("details") @DefaultValue("true") details: Boolean): StageData = withUI { ui =>
+      @QueryParam("details") @DefaultValue("true") details: Boolean,
+      @QueryParam("taskStatus") taskStatus: JList[TaskStatus],
+      @QueryParam("withSummaries") @DefaultValue("false") withSummaries: Boolean,
+      @QueryParam("quantiles") @DefaultValue("0.0,0.25,0.5,0.75,1.0") quantileString: String):
+  StageData = withUI { ui =>
     try {
-      ui.store.stageAttempt(stageId, stageAttemptId, details = details)._1
+      val quantiles = parseQuantileString(quantileString)
+      ui.store.stageAttempt(stageId, stageAttemptId, details = details, taskStatus = taskStatus,
+        withSummaries = withSummaries, unsortedQuantiles = quantiles)._1
     } catch {
       case _: NoSuchElementException =>
         // Change the message depending on whether there are any attempts for the requested stage.
-        val all = ui.store.stageData(stageId)
+        val all = ui.store.stageData(stageId, false, taskStatus)
         val msg = if (all.nonEmpty) {
           val ids = all.map(_.attemptId)
           s"unknown attempt for stage $stageId.  Found attempts: [${ids.mkString(",")}]"
@@ -76,15 +115,7 @@ private[v1] class StagesResource extends BaseAppResource {
       @PathParam("stageAttemptId") stageAttemptId: Int,
       @DefaultValue("0.05,0.25,0.5,0.75,0.95") @QueryParam("quantiles") quantileString: String)
   : TaskMetricDistributions = withUI { ui =>
-    val quantiles = quantileString.split(",").map { s =>
-      try {
-        s.toDouble
-      } catch {
-        case nfe: NumberFormatException =>
-          throw new BadParameterException("quantiles", "double", s)
-      }
-    }
-
+    val quantiles = parseQuantileString(quantileString)
     ui.store.taskSummary(stageId, stageAttemptId, quantiles).getOrElse(
       throw new NotFoundException(s"No tasks reported metrics for $stageId / $stageAttemptId yet."))
   }
@@ -96,8 +127,9 @@ private[v1] class StagesResource extends BaseAppResource {
       @PathParam("stageAttemptId") stageAttemptId: Int,
       @DefaultValue("0") @QueryParam("offset") offset: Int,
       @DefaultValue("20") @QueryParam("length") length: Int,
-      @DefaultValue("ID") @QueryParam("sortBy") sortBy: TaskSorting): Seq[TaskData] = {
-    withUI(_.store.taskList(stageId, stageAttemptId, offset, length, sortBy))
+      @DefaultValue("ID") @QueryParam("sortBy") sortBy: TaskSorting,
+      @QueryParam("status") statuses: JList[TaskStatus]): Seq[TaskData] = {
+    withUI(_.store.taskList(stageId, stageAttemptId, offset, length, sortBy, statuses))
   }
 
   // This api needs to stay formatted exactly as it is below, since, it is being used by the
@@ -225,4 +257,14 @@ private[v1] class StagesResource extends BaseAppResource {
     filteredTaskDataSequence
   }
 
+  def parseQuantileString(quantileString: String): Array[Double] = {
+    quantileString.split(",").map { s =>
+      try {
+        s.toDouble
+      } catch {
+        case nfe: NumberFormatException =>
+          throw new BadParameterException("quantiles", "double", s)
+      }
+    }
+  }
 }
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
index 5ec9b36393764..8c08232ec0510 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
@@ -30,7 +30,7 @@ import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize
 import org.apache.spark.JobExecutionStatus
 import org.apache.spark.executor.ExecutorMetrics
 import org.apache.spark.metrics.ExecutorMetricType
-import org.apache.spark.resource.ResourceInformation
+import org.apache.spark.resource.{ExecutorResourceRequest, ResourceInformation, TaskResourceRequest}
 
 case class ApplicationInfo private[spark](
     id: String,
@@ -62,6 +62,11 @@ case class ApplicationAttemptInfo private[spark](
 
 }
 
+class ResourceProfileInfo private[spark](
+    val id: Int,
+    val executorResources: Map[String, ExecutorResourceRequest],
+    val taskResources: Map[String, TaskResourceRequest])
+
 class ExecutorStageSummary private[spark](
     val taskTime : Long,
     val failedTasks : Int,
@@ -77,7 +82,12 @@ class ExecutorStageSummary private[spark](
     val shuffleWriteRecords : Long,
     val memoryBytesSpilled : Long,
     val diskBytesSpilled : Long,
-    val isBlacklistedForStage: Boolean)
+    @deprecated("use isExcludedForStage instead", "3.1.0")
+    val isBlacklistedForStage: Boolean,
+    @JsonSerialize(using = classOf[ExecutorMetricsJsonSerializer])
+    @JsonDeserialize(using = classOf[ExecutorMetricsJsonDeserializer])
+    val peakMemoryMetrics: Option[ExecutorMetrics],
+    val isExcludedForStage: Boolean)
 
 class ExecutorSummary private[spark](
     val id: String,
@@ -97,6 +107,7 @@ class ExecutorSummary private[spark](
     val totalInputBytes: Long,
     val totalShuffleRead: Long,
     val totalShuffleWrite: Long,
+    @deprecated("use isExcluded instead", "3.1.0")
     val isBlacklisted: Boolean,
     val maxMemory: Long,
     val addTime: Date,
@@ -104,12 +115,16 @@ class ExecutorSummary private[spark](
     val removeReason: Option[String],
     val executorLogs: Map[String, String],
     val memoryMetrics: Option[MemoryMetrics],
+    @deprecated("use excludedInStages instead", "3.1.0")
     val blacklistedInStages: Set[Int],
     @JsonSerialize(using = classOf[ExecutorMetricsJsonSerializer])
     @JsonDeserialize(using = classOf[ExecutorMetricsJsonDeserializer])
     val peakMemoryMetrics: Option[ExecutorMetrics],
     val attributes: Map[String, String],
-    val resources: Map[String, ResourceInformation])
+    val resources: Map[String, ResourceInformation],
+    val resourceProfileId: Int,
+    val isExcluded: Boolean,
+    val excludedInStages: Set[Int])
 
 class MemoryMetrics private[spark](
     val usedOnHeapStorageMemory: Long,
@@ -127,6 +142,10 @@ private[spark] class ExecutorMetricsJsonDeserializer
       new TypeReference[Option[Map[String, java.lang.Long]]] {})
     metricsMap.map(metrics => new ExecutorMetrics(metrics))
   }
+
+  override def getNullValue(ctxt: DeserializationContext): Option[ExecutorMetrics] = {
+    None
+  }
 }
 /** serializer for peakMemoryMetrics: convert ExecutorMetrics to map with metric name as key */
 private[spark] class ExecutorMetricsJsonSerializer
@@ -135,11 +154,15 @@ private[spark] class ExecutorMetricsJsonSerializer
       metrics: Option[ExecutorMetrics],
       jsonGenerator: JsonGenerator,
       serializerProvider: SerializerProvider): Unit = {
-    metrics.foreach { m: ExecutorMetrics =>
-      val metricsMap = ExecutorMetricType.metricToOffset.map { case (metric, _) =>
-        metric -> m.getMetricValue(metric)
+    if (metrics.isEmpty) {
+      jsonGenerator.writeNull()
+    } else {
+      metrics.foreach { m: ExecutorMetrics =>
+        val metricsMap = ExecutorMetricType.metricToOffset.map { case (metric, _) =>
+          metric -> m.getMetricValue(metric)
+        }
+        jsonGenerator.writeObject(metricsMap)
       }
-      jsonGenerator.writeObject(metricsMap)
     }
   }
 
@@ -147,6 +170,19 @@ private[spark] class ExecutorMetricsJsonSerializer
     value.isEmpty
 }
 
+private[spark] class ExecutorPeakMetricsDistributionsJsonSerializer
+  extends JsonSerializer[ExecutorPeakMetricsDistributions] {
+  override def serialize(
+    metrics: ExecutorPeakMetricsDistributions,
+    jsonGenerator: JsonGenerator,
+    serializerProvider: SerializerProvider): Unit = {
+    val metricsMap = ExecutorMetricType.metricToOffset.map { case (metric, _) =>
+      metric -> metrics.getMetricDistribution(metric)
+    }
+    jsonGenerator.writeObject(metricsMap)
+  }
+}
+
 class JobData private[spark](
     val jobId: Int,
     val name: String,
@@ -252,7 +288,13 @@ class StageData private[spark](
     val accumulatorUpdates: Seq[AccumulableInfo],
     val tasks: Option[Map[Long, TaskData]],
     val executorSummary: Option[Map[String, ExecutorStageSummary]],
-    val killedTasksSummary: Map[String, Int])
+    val killedTasksSummary: Map[String, Int],
+    val resourceProfileId: Int,
+    @JsonSerialize(using = classOf[ExecutorMetricsJsonSerializer])
+    @JsonDeserialize(using = classOf[ExecutorMetricsJsonDeserializer])
+    val peakExecutorMetrics: Option[ExecutorMetrics],
+    val taskMetricsDistributions: Option[TaskMetricDistributions],
+    val executorMetricsDistributions: Option[ExecutorMetricsDistributions])
 
 class TaskData private[spark](
     val taskId: Long,
@@ -315,6 +357,7 @@ class ShuffleWriteMetrics private[spark](
 class TaskMetricDistributions private[spark](
     val quantiles: IndexedSeq[Double],
 
+    val duration: IndexedSeq[Double],
     val executorDeserializeTime: IndexedSeq[Double],
     val executorDeserializeCpuTime: IndexedSeq[Double],
     val executorRunTime: IndexedSeq[Double],
@@ -341,6 +384,41 @@ class OutputMetricDistributions private[spark](
     val bytesWritten: IndexedSeq[Double],
     val recordsWritten: IndexedSeq[Double])
 
+class ExecutorMetricsDistributions private[spark](
+  val quantiles: IndexedSeq[Double],
+
+  val taskTime: IndexedSeq[Double],
+  val failedTasks: IndexedSeq[Double],
+  val succeededTasks: IndexedSeq[Double],
+  val killedTasks: IndexedSeq[Double],
+  val inputBytes: IndexedSeq[Double],
+  val inputRecords: IndexedSeq[Double],
+  val outputBytes: IndexedSeq[Double],
+  val outputRecords: IndexedSeq[Double],
+  val shuffleRead: IndexedSeq[Double],
+  val shuffleReadRecords: IndexedSeq[Double],
+  val shuffleWrite: IndexedSeq[Double],
+  val shuffleWriteRecords: IndexedSeq[Double],
+  val memoryBytesSpilled: IndexedSeq[Double],
+  val diskBytesSpilled: IndexedSeq[Double],
+  @JsonSerialize(using = classOf[ExecutorPeakMetricsDistributionsJsonSerializer])
+  val peakMemoryMetrics: ExecutorPeakMetricsDistributions
+)
+
+@JsonSerialize(using = classOf[ExecutorPeakMetricsDistributionsJsonSerializer])
+class ExecutorPeakMetricsDistributions private[spark](
+  val quantiles: IndexedSeq[Double],
+  val executorMetrics: IndexedSeq[ExecutorMetrics]) {
+  private lazy val count = executorMetrics.length
+  private lazy val indices = quantiles.map { q => math.min((q * count).toLong, count - 1) }
+
+  /** Returns the distributions for the specified metric. */
+  def getMetricDistribution(metricName: String): IndexedSeq[Double] = {
+    val sorted = executorMetrics.map(_.getMetricValue(metricName)).sorted
+    indices.map(i => sorted(i.toInt).toDouble).toIndexedSeq
+  }
+}
+
 class ShuffleReadMetricDistributions private[spark](
     val readBytes: IndexedSeq[Double],
     val readRecords: IndexedSeq[Double],
@@ -365,12 +443,15 @@ class AccumulableInfo private[spark](
 class VersionInfo private[spark](
   val spark: String)
 
+// Note the resourceProfiles information are only added here on return from the
+// REST call, they are not stored with it.
 class ApplicationEnvironmentInfo private[spark] (
     val runtime: RuntimeInfo,
     val sparkProperties: Seq[(String, String)],
     val hadoopProperties: Seq[(String, String)],
     val systemProperties: Seq[(String, String)],
-    val classpathEntries: Seq[(String, String)])
+    val classpathEntries: Seq[(String, String)],
+    val resourceProfiles: Seq[ResourceProfileInfo])
 
 class RuntimeInfo private[spark](
     val javaVersion: String,
@@ -405,3 +486,12 @@ case class ThreadStackTrace(
     val blockedByThreadId: Option[Long],
     val blockedByLock: String,
     val holdingLocks: Seq[String])
+
+class ProcessSummary private[spark](
+     val id: String,
+     val hostPort: String,
+     val isActive: Boolean,
+     val totalCores: Int,
+     val addTime: Date,
+     val removeTime: Option[Date],
+     val processLogs: Map[String, String])
diff --git a/core/src/main/scala/org/apache/spark/status/storeTypes.scala b/core/src/main/scala/org/apache/spark/status/storeTypes.scala
index c957ff75a501f..a34b1a55647b5 100644
--- a/core/src/main/scala/org/apache/spark/status/storeTypes.scala
+++ b/core/src/main/scala/org/apache/spark/status/storeTypes.scala
@@ -26,6 +26,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize
 import org.apache.spark.status.KVUtils._
 import org.apache.spark.status.api.v1._
 import org.apache.spark.ui.scope._
+import org.apache.spark.util.Utils
 import org.apache.spark.util.kvstore.KVIndex
 
 private[spark] case class AppStatusStoreMetadata(version: Long)
@@ -57,7 +58,7 @@ private[spark] class ExecutorSummaryWrapper(val info: ExecutorSummary) {
   private def active: Boolean = info.isActive
 
   @JsonIgnore @KVIndex("host")
-  val host: String = info.hostPort.split(":")(0)
+  val host: String = Utils.parseHostPort(info.hostPort)._1
 
 }
 
@@ -374,6 +375,13 @@ private[spark] class RDDStorageInfoWrapper(val info: RDDStorageInfo) {
 
 }
 
+private[spark] class ResourceProfileWrapper(val rpInfo: ResourceProfileInfo) {
+
+  @JsonIgnore @KVIndex
+  def id: Int = rpInfo.id
+
+}
+
 private[spark] class ExecutorStageSummaryWrapper(
     val stageId: Int,
     val stageAttemptId: Int,
@@ -466,6 +474,7 @@ private[spark] class CachedQuantile(
     val taskCount: Long,
 
     // The following fields are an exploded view of a single entry for TaskMetricDistributions.
+    val duration: Double,
     val executorDeserializeTime: Double,
     val executorDeserializeCpuTime: Double,
     val executorRunTime: Double,
@@ -505,3 +514,16 @@ private[spark] class CachedQuantile(
   def stage: Array[Int] = Array(stageId, stageAttemptId)
 
 }
+
+private[spark] class ProcessSummaryWrapper(val info: ProcessSummary) {
+
+  @JsonIgnore @KVIndex
+  private def id: String = info.id
+
+  @JsonIgnore @KVIndex("active")
+  private def active: Boolean = info.isActive
+
+  @JsonIgnore @KVIndex("host")
+  val host: String = Utils.parseHostPort(info.hostPort)._1
+
+}
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockId.scala b/core/src/main/scala/org/apache/spark/storage/BlockId.scala
index 68ed3aa5b062f..73bf809a08a68 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockId.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockId.scala
@@ -20,7 +20,7 @@ package org.apache.spark.storage
 import java.util.UUID
 
 import org.apache.spark.SparkException
-import org.apache.spark.annotation.DeveloperApi
+import org.apache.spark.annotation.{DeveloperApi, Since}
 
 /**
  * :: DeveloperApi ::
@@ -38,7 +38,10 @@ sealed abstract class BlockId {
   // convenience methods
   def asRDDId: Option[RDDBlockId] = if (isRDD) Some(asInstanceOf[RDDBlockId]) else None
   def isRDD: Boolean = isInstanceOf[RDDBlockId]
-  def isShuffle: Boolean = isInstanceOf[ShuffleBlockId] || isInstanceOf[ShuffleBlockBatchId]
+  def isShuffle: Boolean = {
+    (isInstanceOf[ShuffleBlockId] || isInstanceOf[ShuffleBlockBatchId] ||
+     isInstanceOf[ShuffleDataBlockId] || isInstanceOf[ShuffleIndexBlockId])
+  }
   def isBroadcast: Boolean = isInstanceOf[BroadcastBlockId]
 
   override def toString: String = name
@@ -78,6 +81,12 @@ case class ShuffleIndexBlockId(shuffleId: Int, mapId: Long, reduceId: Int) exten
   override def name: String = "shuffle_" + shuffleId + "_" + mapId + "_" + reduceId + ".index"
 }
 
+@Since("3.2.0")
+@DeveloperApi
+case class ShufflePushBlockId(shuffleId: Int, mapIndex: Int, reduceId: Int) extends BlockId {
+  override def name: String = "shufflePush_" + shuffleId + "_" + mapIndex + "_" + reduceId
+}
+
 @DeveloperApi
 case class BroadcastBlockId(broadcastId: Long, field: String = "") extends BlockId {
   override def name: String = "broadcast_" + broadcastId + (if (field == "") "" else "_" + field)
@@ -119,6 +128,7 @@ object BlockId {
   val SHUFFLE_BATCH = "shuffle_([0-9]+)_([0-9]+)_([0-9]+)_([0-9]+)".r
   val SHUFFLE_DATA = "shuffle_([0-9]+)_([0-9]+)_([0-9]+).data".r
   val SHUFFLE_INDEX = "shuffle_([0-9]+)_([0-9]+)_([0-9]+).index".r
+  val SHUFFLE_PUSH = "shufflePush_([0-9]+)_([0-9]+)_([0-9]+)".r
   val BROADCAST = "broadcast_([0-9]+)([_A-Za-z0-9]*)".r
   val TASKRESULT = "taskresult_([0-9]+)".r
   val STREAM = "input-([0-9]+)-([0-9]+)".r
@@ -137,6 +147,8 @@ object BlockId {
       ShuffleDataBlockId(shuffleId.toInt, mapId.toLong, reduceId.toInt)
     case SHUFFLE_INDEX(shuffleId, mapId, reduceId) =>
       ShuffleIndexBlockId(shuffleId.toInt, mapId.toLong, reduceId.toInt)
+    case SHUFFLE_PUSH(shuffleId, mapIndex, reduceId) =>
+      ShufflePushBlockId(shuffleId.toInt, mapIndex.toInt, reduceId.toInt)
     case BROADCAST(broadcastId, field) =>
       BroadcastBlockId(broadcastId.toLong, field.stripPrefix("_"))
     case TASKRESULT(taskId) =>
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockInfoManager.scala b/core/src/main/scala/org/apache/spark/storage/BlockInfoManager.scala
index 219a0e799cc73..95d901f292971 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockInfoManager.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockInfoManager.scala
@@ -367,7 +367,7 @@ private[storage] class BlockInfoManager extends Logging {
 
     notifyAll()
 
-    blocksWithReleasedLocks
+    blocksWithReleasedLocks.toSeq
   }
 
   /** Returns the number of locks held by the given task.  Used only for testing. */
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala
index e0478ad09601d..4c09e1615affb 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala
@@ -24,6 +24,7 @@ import java.nio.channels.Channels
 import java.util.Collections
 import java.util.concurrent.{CompletableFuture, ConcurrentHashMap, TimeUnit}
 
+import scala.collection.JavaConverters._
 import scala.collection.mutable
 import scala.collection.mutable.HashMap
 import scala.concurrent.{ExecutionContext, Future}
@@ -53,8 +54,8 @@ import org.apache.spark.network.util.TransportConf
 import org.apache.spark.rpc.RpcEnv
 import org.apache.spark.scheduler.ExecutorCacheTaskLocation
 import org.apache.spark.serializer.{SerializerInstance, SerializerManager}
-import org.apache.spark.shuffle.{ShuffleManager, ShuffleWriteMetricsReporter}
-import org.apache.spark.storage.BlockManagerMessages.ReplicateBlock
+import org.apache.spark.shuffle.{MigratableResolver, ShuffleManager, ShuffleWriteMetricsReporter}
+import org.apache.spark.storage.BlockManagerMessages.{DecommissionBlockManager, ReplicateBlock}
 import org.apache.spark.storage.memory._
 import org.apache.spark.unsafe.Platform
 import org.apache.spark.util._
@@ -118,9 +119,7 @@ private[spark] class ByteBufferBlockData(
 private[spark] class HostLocalDirManager(
     futureExecutionContext: ExecutionContext,
     cacheSize: Int,
-    externalBlockStoreClient: ExternalBlockStoreClient,
-    host: String,
-    externalShuffleServicePort: Int) extends Logging {
+    blockStoreClient: BlockStoreClient) extends Logging {
 
   private val executorIdToLocalDirsCache =
     CacheBuilder
@@ -128,24 +127,25 @@ private[spark] class HostLocalDirManager(
       .maximumSize(cacheSize)
       .build[String, Array[String]]()
 
-  private[spark] def getCachedHostLocalDirs()
-      : scala.collection.Map[String, Array[String]] = executorIdToLocalDirsCache.synchronized {
-    import scala.collection.JavaConverters._
-    return executorIdToLocalDirsCache.asMap().asScala
-  }
+  private[spark] def getCachedHostLocalDirs: Map[String, Array[String]] =
+    executorIdToLocalDirsCache.synchronized {
+      executorIdToLocalDirsCache.asMap().asScala.toMap
+    }
 
   private[spark] def getHostLocalDirs(
+      host: String,
+      port: Int,
       executorIds: Array[String])(
-      callback: Try[java.util.Map[String, Array[String]]] => Unit): Unit = {
+      callback: Try[Map[String, Array[String]]] => Unit): Unit = {
     val hostLocalDirsCompletable = new CompletableFuture[java.util.Map[String, Array[String]]]
-    externalBlockStoreClient.getHostLocalDirs(
+    blockStoreClient.getHostLocalDirs(
       host,
-      externalShuffleServicePort,
+      port,
       executorIds,
       hostLocalDirsCompletable)
     hostLocalDirsCompletable.whenComplete { (hostLocalDirs, throwable) =>
       if (hostLocalDirs != null) {
-        callback(Success(hostLocalDirs))
+        callback(Success(hostLocalDirs.asScala.toMap))
         executorIdToLocalDirsCache.synchronized {
           executorIdToLocalDirsCache.putAll(hostLocalDirs)
         }
@@ -163,7 +163,7 @@ private[spark] class HostLocalDirManager(
  * Note that [[initialize()]] must be called before the BlockManager is usable.
  */
 private[spark] class BlockManager(
-    executorId: String,
+    val executorId: String,
     rpcEnv: RpcEnv,
     val master: BlockManagerMaster,
     val serializerManager: SerializerManager,
@@ -210,7 +210,7 @@ private[spark] class BlockManager(
   private val maxOnHeapMemory = memoryManager.maxOnHeapStorageMemory
   private val maxOffHeapMemory = memoryManager.maxOffHeapStorageMemory
 
-  private val externalShuffleServicePort = StorageUtils.externalShuffleServicePort(conf)
+  private[spark] val externalShuffleServicePort = StorageUtils.externalShuffleServicePort(conf)
 
   var blockManagerId: BlockManagerId = _
 
@@ -226,9 +226,9 @@ private[spark] class BlockManager(
   private val maxFailuresBeforeLocationRefresh =
     conf.get(config.BLOCK_FAILURES_BEFORE_LOCATION_REFRESH)
 
-  private val slaveEndpoint = rpcEnv.setupEndpoint(
+  private val storageEndpoint = rpcEnv.setupEndpoint(
     "BlockManagerEndpoint" + BlockManager.ID_GENERATOR.next,
-    new BlockManagerSlaveEndpoint(rpcEnv, this, mapOutputTracker))
+    new BlockManagerStorageEndpoint(rpcEnv, this, mapOutputTracker))
 
   // Pending re-registration action being executed asynchronously or null if none is pending.
   // Accesses should synchronize on asyncReregisterLock.
@@ -242,8 +242,9 @@ private[spark] class BlockManager(
 
   private var blockReplicationPolicy: BlockReplicationPolicy = _
 
-  private var blockManagerDecommissioning: Boolean = false
-  private var decommissionManager: Option[BlockManagerDecommissionManager] = None
+  // visible for test
+  // This is volatile since if it's defined we should not accept remote blocks.
+  @volatile private[spark] var decommissioner: Option[BlockManagerDecommissioner] = None
 
   // A DownloadFileManager used to track all the files of remote blocks which are above the
   // specified memory threshold. Files will be deleted automatically based on weak reference.
@@ -254,6 +255,26 @@ private[spark] class BlockManager(
 
   var hostLocalDirManager: Option[HostLocalDirManager] = None
 
+  @inline final private def isDecommissioning() = {
+    decommissioner.isDefined
+  }
+
+  @inline final private def checkShouldStore(blockId: BlockId) = {
+    // Don't reject broadcast blocks since they may be stored during task exec and
+    // don't need to be migrated.
+    if (isDecommissioning() && !blockId.isBroadcast) {
+        throw new BlockSavedOnDecommissionedBlockManagerException(blockId)
+    }
+  }
+
+  // This is a lazy val so someone can migrating RDDs even if they don't have a MigratableResolver
+  // for shuffles. Used in BlockManagerDecommissioner & block puts.
+  private[storage] lazy val migratableResolver: MigratableResolver = {
+    shuffleManager.shuffleBlockResolver.asInstanceOf[MigratableResolver]
+  }
+
+  override def getLocalDiskDirs: Array[String] = diskBlockManager.localDirsString
+
   /**
    * Abstraction for storing blocks from bytes, whether they start in memory or on disk.
    *
@@ -364,7 +385,7 @@ private[spark] class BlockManager(
             ThreadUtils.awaitReady(replicationFuture, Duration.Inf)
           } catch {
             case NonFatal(t) =>
-              throw new Exception("Error occurred while waiting for replication to finish", t)
+              throw new SparkException("Error occurred while waiting for replication to finish", t)
           }
         }
         if (blockWasSuccessfullyStored) {
@@ -465,7 +486,7 @@ private[spark] class BlockManager(
       diskBlockManager.localDirsString,
       maxOnHeapMemory,
       maxOffHeapMemory,
-      slaveEndpoint)
+      storageEndpoint)
 
     blockManagerId = if (idFromMaster != null) idFromMaster else id
 
@@ -481,20 +502,17 @@ private[spark] class BlockManager(
       registerWithExternalShuffleServer()
     }
 
-    hostLocalDirManager =
+    hostLocalDirManager = {
       if (conf.get(config.SHUFFLE_HOST_LOCAL_DISK_READING_ENABLED) &&
           !conf.get(config.SHUFFLE_USE_OLD_FETCH_PROTOCOL)) {
-        externalBlockStoreClient.map { blockStoreClient =>
-          new HostLocalDirManager(
-            futureExecutionContext,
-            conf.get(config.STORAGE_LOCAL_DISK_BY_EXECUTORS_CACHE_SIZE),
-            blockStoreClient,
-            blockManagerId.host,
-            externalShuffleServicePort)
-        }
+        Some(new HostLocalDirManager(
+          futureExecutionContext,
+          conf.get(config.STORAGE_LOCAL_DISK_BY_EXECUTORS_CACHE_SIZE),
+          blockStoreClient))
       } else {
         None
       }
+    }
 
     logInfo(s"Initialized BlockManager: $blockManagerId")
   }
@@ -543,8 +561,8 @@ private[spark] class BlockManager(
    * an executor crash.
    *
    * This function deliberately fails silently if the master returns false (indicating that
-   * the slave needs to re-register). The error condition will be detected again by the next
-   * heart beat attempt or new block registration and another try to re-register all blocks
+   * the storage endpoint needs to re-register). The error condition will be detected again by the
+   * next heart beat attempt or new block registration and another try to re-register all blocks
    * will be made then.
    */
   private def reportAllBlocks(): Unit = {
@@ -568,7 +586,7 @@ private[spark] class BlockManager(
     // TODO: We might need to rate limit re-registering.
     logInfo(s"BlockManager $blockManagerId re-registering with master")
     master.registerBlockManager(blockManagerId, diskBlockManager.localDirsString, maxOnHeapMemory,
-      maxOffHeapMemory, slaveEndpoint)
+      maxOffHeapMemory, storageEndpoint)
     reportAllBlocks()
   }
 
@@ -617,7 +635,17 @@ private[spark] class BlockManager(
    */
   override def getLocalBlockData(blockId: BlockId): ManagedBuffer = {
     if (blockId.isShuffle) {
-      shuffleManager.shuffleBlockResolver.getBlockData(blockId)
+      logDebug(s"Getting local shuffle block ${blockId}")
+      try {
+        shuffleManager.shuffleBlockResolver.getBlockData(blockId)
+      } catch {
+        case e: IOException =>
+          if (conf.get(config.STORAGE_DECOMMISSION_FALLBACK_STORAGE_PATH).isDefined) {
+            FallbackStorage.read(conf, blockId)
+          } else {
+            throw e
+          }
+      }
     } else {
       getLocalBytes(blockId) match {
         case Some(blockData) =>
@@ -650,6 +678,21 @@ private[spark] class BlockManager(
       blockId: BlockId,
       level: StorageLevel,
       classTag: ClassTag[_]): StreamCallbackWithID = {
+
+    checkShouldStore(blockId)
+
+    if (blockId.isShuffle) {
+      logDebug(s"Putting shuffle block ${blockId}")
+      try {
+        return migratableResolver.putShuffleBlockAsStream(blockId, serializerManager)
+      } catch {
+        case e: ClassCastException => throw new SparkException(
+          s"Unexpected shuffle block ${blockId} with unsupported shuffle " +
+          s"resolver ${shuffleManager.shuffleBlockResolver}")
+      }
+    }
+    logDebug(s"Putting regular block ${blockId}")
+    // All other blocks
     val (_, tmpFile) = diskBlockManager.createTempLocalBlock()
     val channel = new CountingWritableChannel(
       Channels.newChannel(serializerManager.wrapForEncryption(new FileOutputStream(tmpFile))))
@@ -718,9 +761,9 @@ private[spark] class BlockManager(
    *
    * droppedMemorySize exists to account for when the block is dropped from memory to disk (so
    * it is still valid). This ensures that update in master will compensate for the increase in
-   * memory on slave.
+   * memory on the storage endpoint.
    */
-  private def reportBlockStatus(
+  private[spark] def reportBlockStatus(
       blockId: BlockId,
       status: BlockStatus,
       droppedMemorySize: Long = 0L): Unit = {
@@ -736,7 +779,7 @@ private[spark] class BlockManager(
   /**
    * Actually send a UpdateBlockInfo message. Returns the master's response,
    * which will be true if the block was successfully recorded and false if
-   * the slave needs to re-register.
+   * the storage endpoint needs to re-register.
    */
   private def tryToReportBlockStatus(
       blockId: BlockId,
@@ -934,7 +977,7 @@ private[spark] class BlockManager(
     require(blockId != null, "BlockId is null")
 
     // Because all the remote blocks are registered in driver, it is not necessary to ask
-    // all the slave executors to get block status.
+    // all the storage endpoints to get block status.
     val locationsAndStatusOption = master.getLocationsAndStatus(blockId, blockManagerId.host)
     if (locationsAndStatusOption.isEmpty) {
       logDebug(s"Block $blockId is unknown by block manager master")
@@ -1076,7 +1119,7 @@ private[spark] class BlockManager(
       blockSize: Long): Option[ManagedBuffer] = {
     val file = ExecutorDiskUtils.getFile(localDirs, subDirsPerLocalDir, blockId.name)
     if (file.exists()) {
-      val mangedBuffer = securityManager.getIOEncryptionKey() match {
+      val managedBuffer = securityManager.getIOEncryptionKey() match {
         case Some(key) =>
           // Encrypted blocks cannot be memory mapped; return a special object that does decryption
           // and provides InputStream / FileRegion implementations for reading the data.
@@ -1087,7 +1130,7 @@ private[spark] class BlockManager(
           val transportConf = SparkTransportConf.fromSparkConf(conf, "shuffle")
           new FileSegmentManagedBuffer(transportConf, file, 0, file.length)
       }
-      Some(mangedBuffer)
+      Some(managedBuffer)
     } else {
       None
     }
@@ -1285,6 +1328,7 @@ private[spark] class BlockManager(
 
     require(blockId != null, "BlockId is null")
     require(level != null && level.isValid, "StorageLevel is null or invalid")
+    checkShouldStore(blockId)
 
     val putBlockInfo = {
       val newInfo = new BlockInfo(level, classTag, tellMaster)
@@ -1540,7 +1584,7 @@ private[spark] class BlockManager(
   /**
    * Get peer block managers in the system.
    */
-  private def getPeers(forceFetch: Boolean): Seq[BlockManagerId] = {
+  private[storage] def getPeers(forceFetch: Boolean): Seq[BlockManagerId] = {
     peerFetchLock.synchronized {
       val cachedPeersTtl = conf.get(config.STORAGE_CACHED_PEERS_TTL) // milliseconds
       val diff = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - lastPeerFetchTimeNs)
@@ -1550,7 +1594,12 @@ private[spark] class BlockManager(
         lastPeerFetchTimeNs = System.nanoTime()
         logDebug("Fetched peers from master: " + cachedPeers.mkString("[", ",", "]"))
       }
-      cachedPeers
+      if (cachedPeers.isEmpty &&
+          conf.get(config.STORAGE_DECOMMISSION_FALLBACK_STORAGE_PATH).isDefined) {
+        Seq(FallbackStorage.FALLBACK_BLOCK_MANAGER_ID)
+      } else {
+        cachedPeers
+      }
     }
   }
 
@@ -1779,60 +1828,30 @@ private[spark] class BlockManager(
     blocksToRemove.size
   }
 
-  def decommissionBlockManager(): Unit = {
-    if (!blockManagerDecommissioning) {
-      logInfo("Starting block manager decommissioning process")
-      blockManagerDecommissioning = true
-      decommissionManager = Some(new BlockManagerDecommissionManager(conf))
-      decommissionManager.foreach(_.start())
-    } else {
-      logDebug("Block manager already in decommissioning state")
+  def decommissionBlockManager(): Unit = storageEndpoint.ask(DecommissionBlockManager)
+
+  private[spark] def decommissionSelf(): Unit = synchronized {
+    decommissioner match {
+      case None =>
+        logInfo("Starting block manager decommissioning process...")
+        decommissioner = Some(new BlockManagerDecommissioner(conf, this))
+        decommissioner.foreach(_.start())
+      case Some(_) =>
+        logDebug("Block manager already in decommissioning state")
     }
   }
 
   /**
-   * Tries to offload all cached RDD blocks from this BlockManager to peer BlockManagers
-   * Visible for testing
+   *  Returns the last migration time and a boolean denoting if all the blocks have been migrated.
+   *  If there are any tasks running since that time the boolean may be incorrect.
    */
-  def decommissionRddCacheBlocks(): Unit = {
-    val replicateBlocksInfo = master.getReplicateInfoForRDDBlocks(blockManagerId)
-
-    if (replicateBlocksInfo.nonEmpty) {
-      logInfo(s"Need to replicate ${replicateBlocksInfo.size} blocks " +
-        "for block manager decommissioning")
-    } else {
-      logWarning(s"Asked to decommission RDD cache blocks, but no blocks to migrate")
-      return
-    }
-
-    // Maximum number of storage replication failure which replicateBlock can handle
-    val maxReplicationFailures = conf.get(
-      config.STORAGE_DECOMMISSION_MAX_REPLICATION_FAILURE_PER_BLOCK)
-
-    // TODO: We can sort these blocks based on some policy (LRU/blockSize etc)
-    //   so that we end up prioritize them over each other
-    val blocksFailedReplication = replicateBlocksInfo.map {
-      case ReplicateBlock(blockId, existingReplicas, maxReplicas) =>
-        val replicatedSuccessfully = replicateBlock(
-          blockId,
-          existingReplicas.toSet,
-          maxReplicas,
-          maxReplicationFailures = Some(maxReplicationFailures))
-        if (replicatedSuccessfully) {
-          logInfo(s"Block $blockId offloaded successfully, Removing block now")
-          removeBlock(blockId)
-          logInfo(s"Block $blockId removed")
-        } else {
-          logWarning(s"Failed to offload block $blockId")
-        }
-        (blockId, replicatedSuccessfully)
-    }.filterNot(_._2).map(_._1)
-    if (blocksFailedReplication.nonEmpty) {
-      logWarning("Blocks failed replication in cache decommissioning " +
-        s"process: ${blocksFailedReplication.mkString(",")}")
-    }
+  private[spark] def lastMigrationInfo(): (Long, Boolean) = {
+    decommissioner.map(_.lastMigrationInfo()).getOrElse((0, false))
   }
 
+  private[storage] def getMigratableRDDBlocks(): Seq[ReplicateBlock] =
+    master.getReplicateInfoForRDDBlocks(blockManagerId)
+
   /**
    * Remove all blocks belonging to the given broadcast.
    */
@@ -1901,58 +1920,8 @@ private[spark] class BlockManager(
     data.dispose()
   }
 
-  /**
-   * Class to handle block manager decommissioning retries
-   * It creates a Thread to retry offloading all RDD cache blocks
-   */
-  private class BlockManagerDecommissionManager(conf: SparkConf) {
-    @volatile private var stopped = false
-    private val sleepInterval = conf.get(
-      config.STORAGE_DECOMMISSION_REPLICATION_REATTEMPT_INTERVAL)
-
-    private val blockReplicationThread = new Thread {
-      override def run(): Unit = {
-        var failures = 0
-        while (blockManagerDecommissioning
-          && !stopped
-          && !Thread.interrupted()
-          && failures < 20) {
-          try {
-            logDebug("Attempting to replicate all cached RDD blocks")
-            decommissionRddCacheBlocks()
-            logInfo("Attempt to replicate all cached blocks done")
-            Thread.sleep(sleepInterval)
-          } catch {
-            case _: InterruptedException =>
-              logInfo("Interrupted during migration, will not refresh migrations.")
-              stopped = true
-            case NonFatal(e) =>
-              failures += 1
-              logError("Error occurred while trying to replicate cached RDD blocks" +
-                s" for block manager decommissioning (failure count: $failures)", e)
-          }
-        }
-      }
-    }
-    blockReplicationThread.setDaemon(true)
-    blockReplicationThread.setName("block-replication-thread")
-
-    def start(): Unit = {
-      logInfo("Starting block replication thread")
-      blockReplicationThread.start()
-    }
-
-    def stop(): Unit = {
-      if (!stopped) {
-        stopped = true
-        logInfo("Stopping block replication thread")
-        blockReplicationThread.interrupt()
-      }
-    }
-  }
-
   def stop(): Unit = {
-    decommissionManager.foreach(_.stop())
+    decommissioner.foreach(_.stop())
     blockTransferService.close()
     if (blockStoreClient ne blockTransferService) {
       // Closing should be idempotent, but maybe not for the NioBlockTransferService.
@@ -1960,7 +1929,7 @@ private[spark] class BlockManager(
     }
     remoteBlockTempFileManager.stop()
     diskBlockManager.stop()
-    rpcEnv.stop(slaveEndpoint)
+    rpcEnv.stop(storageEndpoint)
     blockInfoManager.clear()
     memoryStore.clear()
     futureExecutionContext.shutdownNow()
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerDecommissioner.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerDecommissioner.scala
new file mode 100644
index 0000000000000..faccd92322c97
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerDecommissioner.scala
@@ -0,0 +1,425 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.storage
+
+import java.io.IOException
+import java.util.concurrent.atomic.AtomicInteger
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+import scala.util.control.NonFatal
+
+import org.apache.spark._
+import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config
+import org.apache.spark.shuffle.ShuffleBlockInfo
+import org.apache.spark.storage.BlockManagerMessages.ReplicateBlock
+import org.apache.spark.util.ThreadUtils
+
+/**
+ * Class to handle block manager decommissioning retries.
+ * It creates a Thread to retry offloading all RDD cache and Shuffle blocks
+ */
+private[storage] class BlockManagerDecommissioner(
+    conf: SparkConf,
+    bm: BlockManager) extends Logging {
+
+  private val fallbackStorage = FallbackStorage.getFallbackStorage(conf)
+  private val maxReplicationFailuresForDecommission =
+    conf.get(config.STORAGE_DECOMMISSION_MAX_REPLICATION_FAILURE_PER_BLOCK)
+
+  // Used for tracking if our migrations are complete. Readable for testing
+  @volatile private[storage] var lastRDDMigrationTime: Long = 0
+  @volatile private[storage] var lastShuffleMigrationTime: Long = 0
+  @volatile private[storage] var rddBlocksLeft: Boolean = true
+  @volatile private[storage] var shuffleBlocksLeft: Boolean = true
+
+  /**
+   * This runnable consumes any shuffle blocks in the queue for migration. This part of a
+   * producer/consumer where the main migration loop updates the queue of blocks to be migrated
+   * periodically. On migration failure, the current thread will reinsert the block for another
+   * thread to consume. Each thread migrates blocks to a different particular executor to avoid
+   * distribute the blocks as quickly as possible without overwhelming any particular executor.
+   *
+   * There is no preference for which peer a given block is migrated to.
+   * This is notable different than the RDD cache block migration (further down in this file)
+   * which uses the existing priority mechanism for determining where to replicate blocks to.
+   * Generally speaking cache blocks are less impactful as they normally represent narrow
+   * transformations and we normally have less cache present than shuffle data.
+   *
+   * The producer/consumer model is chosen for shuffle block migration to maximize
+   * the chance of migrating all shuffle blocks before the executor is forced to exit.
+   */
+  private class ShuffleMigrationRunnable(peer: BlockManagerId) extends Runnable {
+    @volatile var running = true
+    override def run(): Unit = {
+      var migrating: Option[(ShuffleBlockInfo, Int)] = None
+      logInfo(s"Starting migration thread for ${peer}")
+      // Once a block fails to transfer to an executor stop trying to transfer more blocks
+      try {
+        while (running && !Thread.interrupted()) {
+          migrating = Option(shufflesToMigrate.poll())
+          migrating match {
+            case None =>
+              logDebug("Nothing to migrate")
+              // Nothing to do right now, but maybe a transfer will fail or a new block
+              // will finish being committed.
+              val SLEEP_TIME_SECS = 1
+              Thread.sleep(SLEEP_TIME_SECS * 1000L)
+            case Some((shuffleBlockInfo, retryCount)) =>
+              if (retryCount < maxReplicationFailuresForDecommission) {
+                val blocks = bm.migratableResolver.getMigrationBlocks(shuffleBlockInfo)
+                if (blocks.isEmpty) {
+                  logInfo(s"Ignore empty shuffle block $shuffleBlockInfo")
+                } else {
+                  logInfo(s"Got migration sub-blocks ${blocks}")
+                  logInfo(s"Trying to migrate shuffle ${shuffleBlockInfo} to ${peer} " +
+                    s"($retryCount / $maxReplicationFailuresForDecommission)")
+
+                  // Migrate the components of the blocks.
+                  try {
+                    blocks.foreach { case (blockId, buffer) =>
+                      logDebug(s"Migrating sub-block ${blockId}")
+                      bm.blockTransferService.uploadBlockSync(
+                        peer.host,
+                        peer.port,
+                        peer.executorId,
+                        blockId,
+                        buffer,
+                        StorageLevel.DISK_ONLY,
+                        null) // class tag, we don't need for shuffle
+                      logDebug(s"Migrated sub block ${blockId}")
+                    }
+                    logInfo(s"Migrated ${shuffleBlockInfo} to ${peer}")
+                  } catch {
+                    case e: IOException =>
+                      // If a block got deleted before netty opened the file handle, then trying to
+                      // load the blocks now will fail. This is most likely to occur if we start
+                      // migrating blocks and then the shuffle TTL cleaner kicks in. However this
+                      // could also happen with manually managed shuffles or a GC event on the
+                      // driver a no longer referenced RDD with shuffle files.
+                      if (bm.migratableResolver.getMigrationBlocks(shuffleBlockInfo).isEmpty) {
+                        logWarning(s"Skipping block ${shuffleBlockInfo}, block deleted.")
+                      } else if (fallbackStorage.isDefined) {
+                        fallbackStorage.foreach(_.copy(shuffleBlockInfo, bm))
+                      } else {
+                        throw e
+                      }
+                  }
+                }
+              } else {
+                logError(s"Skipping block ${shuffleBlockInfo} because it has failed ${retryCount}")
+              }
+              numMigratedShuffles.incrementAndGet()
+          }
+        }
+        // This catch is intentionally outside of the while running block.
+        // if we encounter errors migrating to an executor we want to stop.
+      } catch {
+        case e: Exception =>
+          migrating match {
+            case Some((shuffleMap, retryCount)) =>
+              logError(s"Error during migration, adding ${shuffleMap} back to migration queue", e)
+              shufflesToMigrate.add((shuffleMap, retryCount + 1))
+              running = false
+            case None =>
+              logError(s"Error while waiting for block to migrate", e)
+          }
+      }
+    }
+  }
+
+  // Shuffles which are either in queue for migrations or migrated
+  protected[storage] val migratingShuffles = mutable.HashSet[ShuffleBlockInfo]()
+
+  // Shuffles which have migrated. This used to know when we are "done", being done can change
+  // if a new shuffle file is created by a running task.
+  private[storage] val numMigratedShuffles = new AtomicInteger(0)
+
+  // Shuffles which are queued for migration & number of retries so far.
+  // Visible in storage for testing.
+  private[storage] val shufflesToMigrate =
+    new java.util.concurrent.ConcurrentLinkedQueue[(ShuffleBlockInfo, Int)]()
+
+  // Set if we encounter an error attempting to migrate and stop.
+  @volatile private var stopped = false
+  @volatile private var stoppedRDD =
+    !conf.get(config.STORAGE_DECOMMISSION_RDD_BLOCKS_ENABLED)
+  @volatile private var stoppedShuffle =
+    !conf.get(config.STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED)
+
+  private val migrationPeers =
+    mutable.HashMap[BlockManagerId, ShuffleMigrationRunnable]()
+
+  private lazy val rddBlockMigrationExecutor =
+    ThreadUtils.newDaemonSingleThreadExecutor("block-manager-decommission-rdd")
+
+  private val rddBlockMigrationRunnable = new Runnable {
+    val sleepInterval = conf.get(config.STORAGE_DECOMMISSION_REPLICATION_REATTEMPT_INTERVAL)
+
+    override def run(): Unit = {
+      assert(conf.get(config.STORAGE_DECOMMISSION_RDD_BLOCKS_ENABLED))
+      while (!stopped && !stoppedRDD && !Thread.interrupted()) {
+        logInfo("Iterating on migrating from the block manager.")
+        // Validate we have peers to migrate to.
+        val peers = bm.getPeers(false)
+        // If we have no peers give up.
+        if (peers.isEmpty) {
+          stopped = true
+          stoppedRDD = true
+        }
+        try {
+          val startTime = System.nanoTime()
+          logDebug("Attempting to replicate all cached RDD blocks")
+          rddBlocksLeft = decommissionRddCacheBlocks()
+          lastRDDMigrationTime = startTime
+          logInfo("Attempt to replicate all cached blocks done")
+          logInfo(s"Waiting for ${sleepInterval} before refreshing migrations.")
+          Thread.sleep(sleepInterval)
+        } catch {
+          case e: InterruptedException =>
+            logInfo("Interrupted during RDD migration, stopping")
+            stoppedRDD = true
+          case NonFatal(e) =>
+            logError("Error occurred replicating RDD for block manager decommissioning.",
+              e)
+            stoppedRDD = true
+        }
+      }
+    }
+  }
+
+  private lazy val shuffleBlockMigrationRefreshExecutor =
+    ThreadUtils.newDaemonSingleThreadExecutor("block-manager-decommission-shuffle")
+
+  private val shuffleBlockMigrationRefreshRunnable = new Runnable {
+    val sleepInterval = conf.get(config.STORAGE_DECOMMISSION_REPLICATION_REATTEMPT_INTERVAL)
+
+    override def run(): Unit = {
+      assert(conf.get(config.STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED))
+      while (!stopped && !stoppedShuffle && !Thread.interrupted()) {
+        try {
+          logDebug("Attempting to replicate all shuffle blocks")
+          val startTime = System.nanoTime()
+          shuffleBlocksLeft = refreshOffloadingShuffleBlocks()
+          lastShuffleMigrationTime = startTime
+          logInfo("Done starting workers to migrate shuffle blocks")
+          Thread.sleep(sleepInterval)
+        } catch {
+          case e: InterruptedException =>
+            logInfo("Interrupted during migration, will not refresh migrations.")
+            stoppedShuffle = true
+          case NonFatal(e) =>
+            logError("Error occurred while trying to replicate for block manager decommissioning.",
+              e)
+            stoppedShuffle = true
+        }
+      }
+    }
+  }
+
+  lazy val shuffleMigrationPool = ThreadUtils.newDaemonCachedThreadPool(
+    "migrate-shuffles",
+    conf.get(config.STORAGE_DECOMMISSION_SHUFFLE_MAX_THREADS))
+
+  /**
+   * Tries to offload all shuffle blocks that are registered with the shuffle service locally.
+   * Note: this does not delete the shuffle files in-case there is an in-progress fetch
+   * but rather shadows them.
+   * Requires an Indexed based shuffle resolver.
+   * Note: if called in testing please call stopOffloadingShuffleBlocks to avoid thread leakage.
+   * Returns true if we are not done migrating shuffle blocks.
+   */
+  private[storage] def refreshOffloadingShuffleBlocks(): Boolean = {
+    // Update the queue of shuffles to be migrated
+    logInfo("Offloading shuffle blocks")
+    val localShuffles = bm.migratableResolver.getStoredShuffles().toSet
+    val newShufflesToMigrate = (localShuffles.diff(migratingShuffles)).toSeq
+      .sortBy(b => (b.shuffleId, b.mapId))
+    shufflesToMigrate.addAll(newShufflesToMigrate.map(x => (x, 0)).asJava)
+    migratingShuffles ++= newShufflesToMigrate
+    logInfo(s"${newShufflesToMigrate.size} of ${localShuffles.size} local shuffles " +
+      s"are added. In total, ${migratingShuffles.size} shuffles are remained.")
+
+    // Update the threads doing migrations
+    val livePeerSet = bm.getPeers(false).toSet
+    val currentPeerSet = migrationPeers.keys.toSet
+    val deadPeers = currentPeerSet.diff(livePeerSet)
+    val newPeers = livePeerSet.diff(currentPeerSet)
+    migrationPeers ++= newPeers.map { peer =>
+      logDebug(s"Starting thread to migrate shuffle blocks to ${peer}")
+      val runnable = new ShuffleMigrationRunnable(peer)
+      shuffleMigrationPool.submit(runnable)
+      (peer, runnable)
+    }
+    // A peer may have entered a decommissioning state, don't transfer any new blocks
+    deadPeers.foreach { peer =>
+        migrationPeers.get(peer).foreach(_.running = false)
+    }
+    // If we don't have anyone to migrate to give up
+    if (!migrationPeers.values.exists(_.running)) {
+      stoppedShuffle = true
+    }
+    // If we found any new shuffles to migrate or otherwise have not migrated everything.
+    newShufflesToMigrate.nonEmpty || migratingShuffles.size > numMigratedShuffles.get()
+  }
+
+  /**
+   * Stop migrating shuffle blocks.
+   */
+  private[storage] def stopOffloadingShuffleBlocks(): Unit = {
+    logInfo("Stopping offloading shuffle blocks.")
+    // Stop as gracefully as possible.
+    migrationPeers.values.foreach{ _.running = false }
+    shuffleMigrationPool.shutdown()
+    shuffleMigrationPool.shutdownNow()
+  }
+
+  /**
+   * Tries to offload all cached RDD blocks from this BlockManager to peer BlockManagers
+   * Visible for testing
+   * Returns true if we have not migrated all of our RDD blocks.
+   */
+  private[storage] def decommissionRddCacheBlocks(): Boolean = {
+    val replicateBlocksInfo = bm.getMigratableRDDBlocks()
+    // Refresh peers and validate we have somewhere to move blocks.
+
+    if (replicateBlocksInfo.nonEmpty) {
+      logInfo(s"Need to replicate ${replicateBlocksInfo.size} RDD blocks " +
+        "for block manager decommissioning")
+    } else {
+      logWarning(s"Asked to decommission RDD cache blocks, but no blocks to migrate")
+      return false
+    }
+
+    // TODO: We can sort these blocks based on some policy (LRU/blockSize etc)
+    //   so that we end up prioritize them over each other
+    val blocksFailedReplication = replicateBlocksInfo.map { replicateBlock =>
+        val replicatedSuccessfully = migrateBlock(replicateBlock)
+        (replicateBlock.blockId, replicatedSuccessfully)
+    }.filterNot(_._2).map(_._1)
+    if (blocksFailedReplication.nonEmpty) {
+      logWarning("Blocks failed replication in cache decommissioning " +
+        s"process: ${blocksFailedReplication.mkString(",")}")
+      return true
+    }
+    return false
+  }
+
+  private def migrateBlock(blockToReplicate: ReplicateBlock): Boolean = {
+    val replicatedSuccessfully = bm.replicateBlock(
+      blockToReplicate.blockId,
+      blockToReplicate.replicas.toSet,
+      blockToReplicate.maxReplicas,
+      maxReplicationFailures = Some(maxReplicationFailuresForDecommission))
+    if (replicatedSuccessfully) {
+      logInfo(s"Block ${blockToReplicate.blockId} offloaded successfully, Removing block now")
+      bm.removeBlock(blockToReplicate.blockId)
+      logInfo(s"Block ${blockToReplicate.blockId} removed")
+    } else {
+      logWarning(s"Failed to offload block ${blockToReplicate.blockId}")
+    }
+    replicatedSuccessfully
+  }
+
+  def start(): Unit = {
+    logInfo("Starting block migration thread")
+    if (conf.get(config.STORAGE_DECOMMISSION_RDD_BLOCKS_ENABLED)) {
+      rddBlockMigrationExecutor.submit(rddBlockMigrationRunnable)
+    }
+    if (conf.get(config.STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED)) {
+      shuffleBlockMigrationRefreshExecutor.submit(shuffleBlockMigrationRefreshRunnable)
+    }
+    if (!conf.get(config.STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED) &&
+      !conf.get(config.STORAGE_DECOMMISSION_RDD_BLOCKS_ENABLED)) {
+      logError(s"Storage decommissioning attempted but neither " +
+        s"${config.STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED.key} or " +
+        s"${config.STORAGE_DECOMMISSION_RDD_BLOCKS_ENABLED.key} is enabled ")
+      stopped = true
+    }
+  }
+
+  def stop(): Unit = {
+    if (stopped) {
+      return
+    } else {
+      stopped = true
+    }
+    try {
+      rddBlockMigrationExecutor.shutdown()
+    } catch {
+      case e: Exception =>
+        logError(s"Error during shutdown", e)
+    }
+    try {
+      shuffleBlockMigrationRefreshExecutor.shutdown()
+    } catch {
+      case e: Exception =>
+        logError(s"Error during shutdown", e)
+    }
+    try {
+      stopOffloadingShuffleBlocks()
+    } catch {
+      case e: Exception =>
+        logError(s"Error during shutdown", e)
+    }
+    logInfo("Forcing block migrations threads to stop")
+    try {
+      rddBlockMigrationExecutor.shutdownNow()
+    } catch {
+      case e: Exception =>
+        logError(s"Error during shutdown", e)
+    }
+    try {
+      shuffleBlockMigrationRefreshExecutor.shutdownNow()
+    } catch {
+      case e: Exception =>
+        logError(s"Error during shutdown", e)
+    }
+    logInfo("Stopped storage decommissioner")
+  }
+
+  /*
+   *  Returns the last migration time and a boolean for if all blocks have been migrated.
+   *  The last migration time is calculated to be the minimum of the last migration of any
+   *  running migration (and if there are now current running migrations it is set to current).
+   *  This provides a timeStamp which, if there have been no tasks running since that time
+   *  we can know that all potential blocks that can be have been migrated off.
+   */
+  private[storage] def lastMigrationInfo(): (Long, Boolean) = {
+    if (stopped || (stoppedRDD && stoppedShuffle)) {
+      // Since we don't have anything left to migrate ever (since we don't restart once
+      // stopped), return that we're done with a validity timestamp that doesn't expire.
+      (Long.MaxValue, true)
+    } else {
+      // Chose the min of the active times. See the function description for more information.
+      val lastMigrationTime = if (!stoppedRDD && !stoppedShuffle) {
+        Math.min(lastRDDMigrationTime, lastShuffleMigrationTime)
+      } else if (!stoppedShuffle) {
+        lastShuffleMigrationTime
+      } else {
+        lastRDDMigrationTime
+      }
+
+      // Technically we could have blocks left if we encountered an error, but those blocks will
+      // never be migrated, so we don't care about them.
+      val blocksMigrated = (!shuffleBlocksLeft || stoppedShuffle) && (!rddBlocksLeft || stoppedRDD)
+      (lastMigrationTime, blocksMigrated)
+    }
+  }
+}
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerId.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerId.scala
index 49e32d04d450a..c6a4457d8f910 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManagerId.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerId.scala
@@ -145,4 +145,6 @@ private[spark] object BlockManagerId {
   def getCachedBlockManagerId(id: BlockManagerId): BlockManagerId = {
     blockManagerIdCache.get(id)
   }
+
+  private[spark] val SHUFFLE_MERGER_IDENTIFIER = "shuffle-push-merger"
 }
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala
index 3cfa5d2a25818..fe1a5aef9499c 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala
@@ -43,9 +43,11 @@ class BlockManagerMaster(
     logInfo("Removed " + execId + " successfully in removeExecutor")
   }
 
-  /** Decommission block managers corresponding to given set of executors */
+  /** Decommission block managers corresponding to given set of executors
+   * Non-blocking.
+   */
   def decommissionBlockManagers(executorIds: Seq[String]): Unit = {
-    driverEndpoint.ask[Unit](DecommissionBlockManagers(executorIds))
+    driverEndpoint.ask[Boolean](DecommissionBlockManagers(executorIds))
   }
 
   /** Get Replication Info for all the RDD blocks stored in given blockManagerId */
@@ -71,10 +73,10 @@ class BlockManagerMaster(
       localDirs: Array[String],
       maxOnHeapMemSize: Long,
       maxOffHeapMemSize: Long,
-      slaveEndpoint: RpcEndpointRef): BlockManagerId = {
+      storageEndpoint: RpcEndpointRef): BlockManagerId = {
     logInfo(s"Registering BlockManager $id")
     val updatedId = driverEndpoint.askSync[BlockManagerId](
-      RegisterBlockManager(id, localDirs, maxOnHeapMemSize, maxOffHeapMemSize, slaveEndpoint))
+      RegisterBlockManager(id, localDirs, maxOnHeapMemSize, maxOffHeapMemSize, storageEndpoint))
     logInfo(s"Registered BlockManager $updatedId")
     updatedId
   }
@@ -123,12 +125,32 @@ class BlockManagerMaster(
     driverEndpoint.askSync[Seq[BlockManagerId]](GetPeers(blockManagerId))
   }
 
+  /**
+   * Get a list of unique shuffle service locations where an executor is successfully
+   * registered in the past for block push/merge with push based shuffle.
+   */
+  def getShufflePushMergerLocations(
+      numMergersNeeded: Int,
+      hostsToFilter: Set[String]): Seq[BlockManagerId] = {
+    driverEndpoint.askSync[Seq[BlockManagerId]](
+      GetShufflePushMergerLocations(numMergersNeeded, hostsToFilter))
+  }
+
+  /**
+   * Remove the host from the candidate list of shuffle push mergers. This can be
+   * triggered if there is a FetchFailedException on the host
+   * @param host
+   */
+  def removeShufflePushMergerLocation(host: String): Unit = {
+    driverEndpoint.askSync[Seq[BlockManagerId]](RemoveShufflePushMergerLocation(host))
+  }
+
   def getExecutorEndpointRef(executorId: String): Option[RpcEndpointRef] = {
     driverEndpoint.askSync[Option[RpcEndpointRef]](GetExecutorEndpointRef(executorId))
   }
 
   /**
-   * Remove a block from the slaves that have it. This can only be used to remove
+   * Remove a block from the storage endpoints that have it. This can only be used to remove
    * blocks that the driver knows about.
    */
   def removeBlock(blockId: BlockId): Unit = {
@@ -142,7 +164,8 @@ class BlockManagerMaster(
       logWarning(s"Failed to remove RDD $rddId - ${e.getMessage}", e)
     )(ThreadUtils.sameThread)
     if (blocking) {
-      timeout.awaitResult(future)
+      // the underlying Futures will timeout anyway, so it's safe to use infinite timeout here
+      RpcUtils.INFINITE_TIMEOUT.awaitResult(future)
     }
   }
 
@@ -153,7 +176,8 @@ class BlockManagerMaster(
       logWarning(s"Failed to remove shuffle $shuffleId - ${e.getMessage}", e)
     )(ThreadUtils.sameThread)
     if (blocking) {
-      timeout.awaitResult(future)
+      // the underlying Futures will timeout anyway, so it's safe to use infinite timeout here
+      RpcUtils.INFINITE_TIMEOUT.awaitResult(future)
     }
   }
 
@@ -166,7 +190,8 @@ class BlockManagerMaster(
         s" with removeFromMaster = $removeFromMaster - ${e.getMessage}", e)
     )(ThreadUtils.sameThread)
     if (blocking) {
-      timeout.awaitResult(future)
+      // the underlying Futures will timeout anyway, so it's safe to use infinite timeout here
+      RpcUtils.INFINITE_TIMEOUT.awaitResult(future)
     }
   }
 
@@ -190,14 +215,14 @@ class BlockManagerMaster(
    * Return the block's status on all block managers, if any. NOTE: This is a
    * potentially expensive operation and should only be used for testing.
    *
-   * If askSlaves is true, this invokes the master to query each block manager for the most
-   * updated block statuses. This is useful when the master is not informed of the given block
+   * If askStorageEndpoints is true, this invokes the master to query each block manager for the
+   * most updated block statuses. This is useful when the master is not informed of the given block
    * by all block managers.
    */
   def getBlockStatus(
       blockId: BlockId,
-      askSlaves: Boolean = true): Map[BlockManagerId, BlockStatus] = {
-    val msg = GetBlockStatus(blockId, askSlaves)
+      askStorageEndpoints: Boolean = true): Map[BlockManagerId, BlockStatus] = {
+    val msg = GetBlockStatus(blockId, askStorageEndpoints)
     /*
      * To avoid potential deadlocks, the use of Futures is necessary, because the master endpoint
      * should not block on waiting for a block manager, which can in turn be waiting for the
@@ -226,14 +251,14 @@ class BlockManagerMaster(
    * Return a list of ids of existing blocks such that the ids match the given filter. NOTE: This
    * is a potentially expensive operation and should only be used for testing.
    *
-   * If askSlaves is true, this invokes the master to query each block manager for the most
-   * updated block statuses. This is useful when the master is not informed of the given block
+   * If askStorageEndpoints is true, this invokes the master to query each block manager for the
+   * most updated block statuses. This is useful when the master is not informed of the given block
    * by all block managers.
    */
   def getMatchingBlockIds(
       filter: BlockId => Boolean,
-      askSlaves: Boolean): Seq[BlockId] = {
-    val msg = GetMatchingBlockIds(filter, askSlaves)
+      askStorageEndpoints: Boolean): Seq[BlockId] = {
+    val msg = GetMatchingBlockIds(filter, askStorageEndpoints)
     val future = driverEndpoint.askSync[Future[Seq[BlockId]]](msg)
     timeout.awaitResult(future)
   }
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
index d936420a99276..eada4b3ee2e38 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
@@ -23,23 +23,25 @@ import java.util.concurrent.TimeUnit
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable
-import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.{ExecutionContext, Future, TimeoutException}
 import scala.util.Random
+import scala.util.control.NonFatal
 
 import com.google.common.cache.CacheBuilder
 
-import org.apache.spark.SparkConf
+import org.apache.spark.{MapOutputTrackerMaster, SparkConf}
 import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.internal.{config, Logging}
 import org.apache.spark.network.shuffle.ExternalBlockStoreClient
 import org.apache.spark.rpc.{IsolatedRpcEndpoint, RpcCallContext, RpcEndpointRef, RpcEnv}
 import org.apache.spark.scheduler._
+import org.apache.spark.scheduler.cluster.{CoarseGrainedClusterMessages, CoarseGrainedSchedulerBackend}
 import org.apache.spark.storage.BlockManagerMessages._
 import org.apache.spark.util.{RpcUtils, ThreadUtils, Utils}
 
 /**
  * BlockManagerMasterEndpoint is an [[IsolatedRpcEndpoint]] on the master node to track statuses
- * of all slaves' block managers.
+ * of all the storage endpoints' block managers.
  */
 private[spark]
 class BlockManagerMasterEndpoint(
@@ -48,7 +50,8 @@ class BlockManagerMasterEndpoint(
     conf: SparkConf,
     listenerBus: LiveListenerBus,
     externalBlockStoreClient: Option[ExternalBlockStoreClient],
-    blockManagerInfo: mutable.Map[BlockManagerId, BlockManagerInfo])
+    blockManagerInfo: mutable.Map[BlockManagerId, BlockManagerInfo],
+    mapOutputTracker: MapOutputTrackerMaster)
   extends IsolatedRpcEndpoint with Logging {
 
   // Mapping from executor id to the block manager's local disk directories.
@@ -71,6 +74,14 @@ class BlockManagerMasterEndpoint(
   // Mapping from block id to the set of block managers that have the block.
   private val blockLocations = new JHashMap[BlockId, mutable.HashSet[BlockManagerId]]
 
+  // Mapping from host name to shuffle (mergers) services where the current app
+  // registered an executor in the past. Older hosts are removed when the
+  // maxRetainedMergerLocations size is reached in favor of newer locations.
+  private val shuffleMergerLocations = new mutable.LinkedHashMap[String, BlockManagerId]()
+
+  // Maximum number of merger locations to cache
+  private val maxRetainedMergerLocations = conf.get(config.SHUFFLE_MERGER_MAX_RETAINED_LOCATIONS)
+
   private val askThreadPool =
     ThreadUtils.newDaemonCachedThreadPool("block-manager-ask-thread-pool", 100)
   private implicit val askExecutionContext = ExecutionContext.fromExecutorService(askThreadPool)
@@ -89,15 +100,20 @@ class BlockManagerMasterEndpoint(
 
   val defaultRpcTimeout = RpcUtils.askRpcTimeout(conf)
 
+  private val pushBasedShuffleEnabled = Utils.isPushBasedShuffleEnabled(conf)
+
   logInfo("BlockManagerMasterEndpoint up")
   // same as `conf.get(config.SHUFFLE_SERVICE_ENABLED)
   //   && conf.get(config.SHUFFLE_SERVICE_FETCH_RDD_ENABLED)`
   private val externalShuffleServiceRddFetchEnabled: Boolean = externalBlockStoreClient.isDefined
   private val externalShuffleServicePort: Int = StorageUtils.externalShuffleServicePort(conf)
 
+  private lazy val driverEndpoint =
+    RpcUtils.makeDriverRef(CoarseGrainedSchedulerBackend.ENDPOINT_NAME, conf, rpcEnv)
+
   override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
-    case RegisterBlockManager(id, localDirs, maxOnHeapMemSize, maxOffHeapMemSize, slaveEndpoint) =>
-      context.reply(register(id, localDirs, maxOnHeapMemSize, maxOffHeapMemSize, slaveEndpoint))
+    case RegisterBlockManager(id, localDirs, maxOnHeapMemSize, maxOffHeapMemSize, endpoint) =>
+      context.reply(register(id, localDirs, maxOnHeapMemSize, maxOffHeapMemSize, endpoint))
 
     case _updateBlockInfo @
         UpdateBlockInfo(blockManagerId, blockId, storageLevel, deserializedSize, size) =>
@@ -130,14 +146,20 @@ class BlockManagerMasterEndpoint(
     case GetStorageStatus =>
       context.reply(storageStatus)
 
-    case GetBlockStatus(blockId, askSlaves) =>
-      context.reply(blockStatus(blockId, askSlaves))
+    case GetBlockStatus(blockId, askStorageEndpoints) =>
+      context.reply(blockStatus(blockId, askStorageEndpoints))
+
+    case GetShufflePushMergerLocations(numMergersNeeded, hostsToFilter) =>
+      context.reply(getShufflePushMergerLocations(numMergersNeeded, hostsToFilter))
+
+    case RemoveShufflePushMergerLocation(host) =>
+      context.reply(removeShufflePushMergerLocation(host))
 
     case IsExecutorAlive(executorId) =>
       context.reply(blockManagerIdByExecutor.contains(executorId))
 
-    case GetMatchingBlockIds(filter, askSlaves) =>
-      context.reply(getMatchingBlockIds(filter, askSlaves))
+    case GetMatchingBlockIds(filter, askStorageEndpoints) =>
+      context.reply(getMatchingBlockIds(filter, askStorageEndpoints))
 
     case RemoveRdd(rddId) =>
       context.reply(removeRdd(rddId))
@@ -157,7 +179,14 @@ class BlockManagerMasterEndpoint(
       context.reply(true)
 
     case DecommissionBlockManagers(executorIds) =>
-      decommissionBlockManagers(executorIds.flatMap(blockManagerIdByExecutor.get))
+      // Mark corresponding BlockManagers as being decommissioning by adding them to
+      // decommissioningBlockManagerSet, so they won't be used to replicate or migrate blocks.
+      // Note that BlockManagerStorageEndpoint will be notified about decommissioning when the
+      // executor is notified(see BlockManager.decommissionSelf), so we don't need to send the
+      // notification here.
+      val bms = executorIds.flatMap(blockManagerIdByExecutor.get)
+      logInfo(s"Mark BlockManagers (${bms.mkString(", ")}) as being decommissioning.")
+      decommissioningBlockManagerSet ++= bms
       context.reply(true)
 
     case GetReplicateInfoForRDDBlocks(blockManagerId) =>
@@ -168,16 +197,60 @@ class BlockManagerMasterEndpoint(
       stop()
   }
 
+  /**
+   * A function that used to handle the failures when removing blocks. In general, the failure
+   * should be considered as non-fatal since it won't cause any correctness issue. Therefore,
+   * this function would prefer to log the exception and return the default value. We only throw
+   * the exception when there's a TimeoutException from an active executor, which implies the
+   * unhealthy status of the executor while the driver still not be aware of it.
+   * @param blockType should be one of "RDD", "shuffle", "broadcast", "block", used for log
+   * @param blockId the string value of a certain block id, used for log
+   * @param bmId the BlockManagerId of the BlockManager, where we're trying to remove the block
+   * @param defaultValue the return value of a failure removal. e.g., 0 means no blocks are removed
+   * @tparam T the generic type for defaultValue, Int or Boolean.
+   * @return the defaultValue or throw exception if the executor is active but reply late.
+   */
+  private def handleBlockRemovalFailure[T](
+      blockType: String,
+      blockId: String,
+      bmId: BlockManagerId,
+      defaultValue: T): PartialFunction[Throwable, T] = {
+    case e: IOException =>
+      logWarning(s"Error trying to remove $blockType $blockId" +
+        s" from block manager $bmId", e)
+      defaultValue
+
+    case t: TimeoutException =>
+      val executorId = bmId.executorId
+      val isAlive = try {
+        driverEndpoint.askSync[Boolean](CoarseGrainedClusterMessages.IsExecutorAlive(executorId))
+      } catch {
+        // ignore the non-fatal error from driverEndpoint since the caller doesn't really
+        // care about the return result of removing blocks. And so we could avoid breaking
+        // down the whole application.
+        case NonFatal(e) =>
+          logError(s"Fail to know the executor $executorId is alive or not.", e)
+          false
+      }
+      if (!isAlive) {
+        logWarning(s"Error trying to remove $blockType $blockId. " +
+          s"The executor $executorId may have been lost.", t)
+        defaultValue
+      } else {
+        throw t
+      }
+  }
+
   private def removeRdd(rddId: Int): Future[Seq[Int]] = {
     // First remove the metadata for the given RDD, and then asynchronously remove the blocks
-    // from the slaves.
+    // from the storage endpoints.
 
-    // The message sent to the slaves to remove the RDD
+    // The message sent to the storage endpoints to remove the RDD
     val removeMsg = RemoveRdd(rddId)
 
     // Find all blocks for the given RDD, remove the block from both blockLocations and
     // the blockManagerInfo that is tracking the blocks and create the futures which asynchronously
-    // remove the blocks from slaves and gives back the number of removed blocks
+    // remove the blocks from storage endpoints and gives back the number of removed blocks
     val blocks = blockLocations.asScala.keys.flatMap(_.asRDDId).filter(_.rddId == rddId)
     val blocksToDeleteByShuffleService =
       new mutable.HashMap[BlockManagerId, mutable.HashSet[RDDBlockId]]
@@ -206,11 +279,9 @@ class BlockManagerMasterEndpoint(
       }
     }
     val removeRddFromExecutorsFutures = blockManagerInfo.values.map { bmInfo =>
-      bmInfo.slaveEndpoint.ask[Int](removeMsg).recover {
-        case e: IOException =>
-          logWarning(s"Error trying to remove RDD ${removeMsg.rddId} " +
-            s"from block manager ${bmInfo.blockManagerId}", e)
-          0 // zero blocks were removed
+      bmInfo.storageEndpoint.ask[Int](removeMsg).recover {
+        // use 0 as default value means no blocks were removed
+        handleBlockRemovalFailure("RDD", rddId.toString, bmInfo.blockManagerId, 0)
       }
     }.toSeq
 
@@ -229,13 +300,15 @@ class BlockManagerMasterEndpoint(
 
     Future.sequence(removeRddFromExecutorsFutures ++ removeRddBlockViaExtShuffleServiceFutures)
   }
-
   private def removeShuffle(shuffleId: Int): Future[Seq[Boolean]] = {
     // Nothing to do in the BlockManagerMasterEndpoint data structures
     val removeMsg = RemoveShuffle(shuffleId)
     Future.sequence(
       blockManagerInfo.values.map { bm =>
-        bm.slaveEndpoint.ask[Boolean](removeMsg)
+        bm.storageEndpoint.ask[Boolean](removeMsg).recover {
+          // use false as default value means no shuffle data were removed
+          handleBlockRemovalFailure("shuffle", shuffleId.toString, bm.blockManagerId, false)
+        }
       }.toSeq
     )
   }
@@ -251,11 +324,9 @@ class BlockManagerMasterEndpoint(
       removeFromDriver || !info.blockManagerId.isDriver
     }
     val futures = requiredBlockManagers.map { bm =>
-      bm.slaveEndpoint.ask[Int](removeMsg).recover {
-        case e: IOException =>
-          logWarning(s"Error trying to remove broadcast $broadcastId from block manager " +
-            s"${bm.blockManagerId}", e)
-          0 // zero blocks were removed
+      bm.storageEndpoint.ask[Int](removeMsg).recover {
+        // use 0 as default value means no blocks were removed
+        handleBlockRemovalFailure("broadcast", broadcastId.toString, bm.blockManagerId, 0)
       }
     }.toSeq
 
@@ -286,7 +357,7 @@ class BlockManagerMasterEndpoint(
         blockLocations.remove(blockId)
         logWarning(s"No more replicas available for $blockId !")
       } else if (proactivelyReplicate && (blockId.isRDD || blockId.isInstanceOf[TestBlockId])) {
-        // As a heursitic, assume single executor failure to find out the number of replicas that
+        // As a heuristic, assume single executor failure to find out the number of replicas that
         // existed before failure
         val maxReplicas = locations.size + 1
         val i = (new Random(blockId.hashCode)).nextInt(locations.size)
@@ -295,7 +366,7 @@ class BlockManagerMasterEndpoint(
         blockManagerInfo.get(candidateBMId).foreach { bm =>
           val remainingLocations = locations.toSeq.filter(bm => bm != candidateBMId)
           val replicateMsg = ReplicateBlock(blockId, remainingLocations, maxReplicas)
-          bm.slaveEndpoint.ask[Boolean](replicateMsg)
+          bm.storageEndpoint.ask[Boolean](replicateMsg)
         }
       }
     }
@@ -305,56 +376,61 @@ class BlockManagerMasterEndpoint(
 
   }
 
+  private def addMergerLocation(blockManagerId: BlockManagerId): Unit = {
+    if (!blockManagerId.isDriver && !shuffleMergerLocations.contains(blockManagerId.host)) {
+      val shuffleServerId = BlockManagerId(BlockManagerId.SHUFFLE_MERGER_IDENTIFIER,
+        blockManagerId.host, externalShuffleServicePort)
+      if (shuffleMergerLocations.size >= maxRetainedMergerLocations) {
+        shuffleMergerLocations -= shuffleMergerLocations.head._1
+      }
+      shuffleMergerLocations(shuffleServerId.host) = shuffleServerId
+    }
+  }
+
   private def removeExecutor(execId: String): Unit = {
     logInfo("Trying to remove executor " + execId + " from BlockManagerMaster.")
     blockManagerIdByExecutor.get(execId).foreach(removeBlockManager)
   }
 
-  /**
-   * Decommission the given Seq of blockmanagers
-   *    - Adds these block managers to decommissioningBlockManagerSet Set
-   *    - Sends the DecommissionBlockManager message to each of the [[BlockManagerSlaveEndpoint]]
-   */
-  def decommissionBlockManagers(blockManagerIds: Seq[BlockManagerId]): Future[Seq[Unit]] = {
-    val newBlockManagersToDecommission = blockManagerIds.toSet.diff(decommissioningBlockManagerSet)
-    val futures = newBlockManagersToDecommission.map { blockManagerId =>
-      decommissioningBlockManagerSet.add(blockManagerId)
-      val info = blockManagerInfo(blockManagerId)
-      info.slaveEndpoint.ask[Unit](DecommissionBlockManager)
-    }
-    Future.sequence{ futures.toSeq }
-  }
-
   /**
    * Returns a Seq of ReplicateBlock for each RDD block stored by given blockManagerId
    * @param blockManagerId - block manager id for which ReplicateBlock info is needed
    * @return Seq of ReplicateBlock
    */
   private def getReplicateInfoForRDDBlocks(blockManagerId: BlockManagerId): Seq[ReplicateBlock] = {
-    val info = blockManagerInfo(blockManagerId)
+    try {
+      val info = blockManagerInfo(blockManagerId)
 
-    val rddBlocks = info.blocks.keySet().asScala.filter(_.isRDD)
-    rddBlocks.map { blockId =>
-      val currentBlockLocations = blockLocations.get(blockId)
-      val maxReplicas = currentBlockLocations.size + 1
-      val remainingLocations = currentBlockLocations.toSeq.filter(bm => bm != blockManagerId)
-      val replicateMsg = ReplicateBlock(blockId, remainingLocations, maxReplicas)
-      replicateMsg
-    }.toSeq
+      val rddBlocks = info.blocks.keySet().asScala.filter(_.isRDD)
+      rddBlocks.map { blockId =>
+        val currentBlockLocations = blockLocations.get(blockId)
+        val maxReplicas = currentBlockLocations.size + 1
+        val remainingLocations = currentBlockLocations.toSeq.filter(bm => bm != blockManagerId)
+        val replicateMsg = ReplicateBlock(blockId, remainingLocations, maxReplicas)
+        replicateMsg
+      }.toSeq
+    } catch {
+      // If the block manager has already exited, nothing to replicate.
+      case e: java.util.NoSuchElementException =>
+        Seq.empty[ReplicateBlock]
+    }
   }
 
-  // Remove a block from the slaves that have it. This can only be used to remove
+  // Remove a block from the workers that have it. This can only be used to remove
   // blocks that the master knows about.
   private def removeBlockFromWorkers(blockId: BlockId): Unit = {
     val locations = blockLocations.get(blockId)
     if (locations != null) {
       locations.foreach { blockManagerId: BlockManagerId =>
         val blockManager = blockManagerInfo.get(blockManagerId)
-        if (blockManager.isDefined) {
-          // Remove the block from the slave's BlockManager.
+        blockManager.foreach { bm =>
+          // Remove the block from the BlockManager.
           // Doesn't actually wait for a confirmation and the message might get lost.
           // If message loss becomes frequent, we should add retry logic here.
-          blockManager.get.slaveEndpoint.ask[Boolean](RemoveBlock(blockId))
+          bm.storageEndpoint.ask[Boolean](RemoveBlock(blockId)).recover {
+            // use false as default value means no blocks were removed
+            handleBlockRemovalFailure("block", blockId.toString, bm.blockManagerId, false)
+          }
         }
       }
     }
@@ -378,13 +454,13 @@ class BlockManagerMasterEndpoint(
    * Return the block's status for all block managers, if any. NOTE: This is a
    * potentially expensive operation and should only be used for testing.
    *
-   * If askSlaves is true, the master queries each block manager for the most updated block
-   * statuses. This is useful when the master is not informed of the given block by all block
+   * If askStorageEndpoints is true, the master queries each block manager for the most updated
+   * block statuses. This is useful when the master is not informed of the given block by all block
    * managers.
    */
   private def blockStatus(
       blockId: BlockId,
-      askSlaves: Boolean): Map[BlockManagerId, Future[Option[BlockStatus]]] = {
+      askStorageEndpoints: Boolean): Map[BlockManagerId, Future[Option[BlockStatus]]] = {
     val getBlockStatus = GetBlockStatus(blockId)
     /*
      * Rather than blocking on the block status query, master endpoint should simply return
@@ -393,8 +469,8 @@ class BlockManagerMasterEndpoint(
      */
     blockManagerInfo.values.map { info =>
       val blockStatusFuture =
-        if (askSlaves) {
-          info.slaveEndpoint.ask[Option[BlockStatus]](getBlockStatus)
+        if (askStorageEndpoints) {
+          info.storageEndpoint.ask[Option[BlockStatus]](getBlockStatus)
         } else {
           Future { info.getStatus(blockId) }
         }
@@ -406,19 +482,19 @@ class BlockManagerMasterEndpoint(
    * Return the ids of blocks present in all the block managers that match the given filter.
    * NOTE: This is a potentially expensive operation and should only be used for testing.
    *
-   * If askSlaves is true, the master queries each block manager for the most updated block
-   * statuses. This is useful when the master is not informed of the given block by all block
+   * If askStorageEndpoints is true, the master queries each block manager for the most updated
+   * block statuses. This is useful when the master is not informed of the given block by all block
    * managers.
    */
   private def getMatchingBlockIds(
       filter: BlockId => Boolean,
-      askSlaves: Boolean): Future[Seq[BlockId]] = {
+      askStorageEndpoints: Boolean): Future[Seq[BlockId]] = {
     val getMatchingBlockIds = GetMatchingBlockIds(filter)
     Future.sequence(
       blockManagerInfo.values.map { info =>
         val future =
-          if (askSlaves) {
-            info.slaveEndpoint.ask[Seq[BlockId]](getMatchingBlockIds)
+          if (askStorageEndpoints) {
+            info.storageEndpoint.ask[Seq[BlockId]](getMatchingBlockIds)
           } else {
             Future { info.blocks.asScala.keys.filter(filter).toSeq }
           }
@@ -441,7 +517,7 @@ class BlockManagerMasterEndpoint(
       localDirs: Array[String],
       maxOnHeapMemSize: Long,
       maxOffHeapMemSize: Long,
-      slaveEndpoint: RpcEndpointRef): BlockManagerId = {
+      storageEndpoint: RpcEndpointRef): BlockManagerId = {
     // the dummy id is not expected to contain the topology information.
     // we get that info here and respond back with a more fleshed out block manager id
     val id = BlockManagerId(
@@ -476,7 +552,11 @@ class BlockManagerMasterEndpoint(
         }
 
       blockManagerInfo(id) = new BlockManagerInfo(id, System.currentTimeMillis(),
-        maxOnHeapMemSize, maxOffHeapMemSize, slaveEndpoint, externalShuffleServiceBlockStatus)
+        maxOnHeapMemSize, maxOffHeapMemSize, storageEndpoint, externalShuffleServiceBlockStatus)
+
+      if (pushBasedShuffleEnabled) {
+        addMergerLocation(id)
+      }
     }
     listenerBus.post(SparkListenerBlockManagerAdded(time, id, maxOnHeapMemSize + maxOffHeapMemSize,
         Some(maxOnHeapMemSize), Some(maxOffHeapMemSize)))
@@ -489,6 +569,24 @@ class BlockManagerMasterEndpoint(
       storageLevel: StorageLevel,
       memSize: Long,
       diskSize: Long): Boolean = {
+    logDebug(s"Updating block info on master ${blockId} for ${blockManagerId}")
+
+    if (blockId.isShuffle) {
+      blockId match {
+        case ShuffleIndexBlockId(shuffleId, mapId, _) =>
+          // Don't update the map output on just the index block
+          logDebug(s"Received shuffle index block update for ${shuffleId} ${mapId}, ignoring.")
+          return true
+        case ShuffleDataBlockId(shuffleId: Int, mapId: Long, reduceId: Int) =>
+          logDebug(s"Received shuffle data block update for ${shuffleId} ${mapId}, updating.")
+          mapOutputTracker.updateMapOutput(shuffleId, mapId, blockManagerId)
+          return true
+        case _ =>
+          logDebug(s"Unexpected shuffle block type ${blockId}" +
+            s"as ${blockId.getClass().getSimpleName()}")
+          return false
+      }
+    }
 
     if (!blockManagerInfo.contains(blockManagerId)) {
       if (blockManagerId.isDriver && !isLocal) {
@@ -530,7 +628,7 @@ class BlockManagerMasterEndpoint(
       }
     }
 
-    // Remove the block from master tracking if it has been removed on all slaves.
+    // Remove the block from master tracking if it has been removed on all endpoints.
     if (locations.size == 0) {
       blockLocations.remove(blockId)
     }
@@ -590,15 +688,49 @@ class BlockManagerMasterEndpoint(
     }
   }
 
+  private def getShufflePushMergerLocations(
+      numMergersNeeded: Int,
+      hostsToFilter: Set[String]): Seq[BlockManagerId] = {
+    val blockManagerHosts = blockManagerIdByExecutor.values.map(_.host).toSet
+    val filteredBlockManagerHosts = blockManagerHosts.filterNot(hostsToFilter.contains(_))
+    val filteredMergersWithExecutors = filteredBlockManagerHosts.map(
+      BlockManagerId(BlockManagerId.SHUFFLE_MERGER_IDENTIFIER, _, externalShuffleServicePort))
+    // Enough mergers are available as part of active executors list
+    if (filteredMergersWithExecutors.size >= numMergersNeeded) {
+      filteredMergersWithExecutors.toSeq
+    } else {
+      // Delta mergers added from inactive mergers list to the active mergers list
+      val filteredMergersWithExecutorsHosts = filteredMergersWithExecutors.map(_.host)
+      val filteredMergersWithoutExecutors = shuffleMergerLocations.values
+        .filterNot(x => hostsToFilter.contains(x.host))
+        .filterNot(x => filteredMergersWithExecutorsHosts.contains(x.host))
+      val randomFilteredMergersLocations =
+        if (filteredMergersWithoutExecutors.size >
+          numMergersNeeded - filteredMergersWithExecutors.size) {
+          Utils.randomize(filteredMergersWithoutExecutors)
+            .take(numMergersNeeded - filteredMergersWithExecutors.size)
+        } else {
+          filteredMergersWithoutExecutors
+        }
+      filteredMergersWithExecutors.toSeq ++ randomFilteredMergersLocations
+    }
+  }
+
+  private def removeShufflePushMergerLocation(host: String): Unit = {
+    if (shuffleMergerLocations.contains(host)) {
+      shuffleMergerLocations.remove(host)
+    }
+  }
+
   /**
-   * Returns an [[RpcEndpointRef]] of the [[BlockManagerSlaveEndpoint]] for sending RPC messages.
+   * Returns an [[RpcEndpointRef]] of the [[BlockManagerReplicaEndpoint]] for sending RPC messages.
    */
   private def getExecutorEndpointRef(executorId: String): Option[RpcEndpointRef] = {
     for (
       blockManagerId <- blockManagerIdByExecutor.get(executorId);
       info <- blockManagerInfo.get(blockManagerId)
     ) yield {
-      info.slaveEndpoint
+      info.storageEndpoint
     }
   }
 
@@ -622,7 +754,7 @@ private[spark] class BlockManagerInfo(
     timeMs: Long,
     val maxOnHeapMem: Long,
     val maxOffHeapMem: Long,
-    val slaveEndpoint: RpcEndpointRef,
+    val storageEndpoint: RpcEndpointRef,
     val externalShuffleServiceBlockStatus: Option[JHashMap[BlockId, BlockStatus]])
   extends Logging {
 
@@ -656,7 +788,7 @@ private[spark] class BlockManagerInfo(
     var originalLevel: StorageLevel = StorageLevel.NONE
 
     if (blockExists) {
-      // The block exists on the slave already.
+      // The block exists on the storage endpoint already.
       val blockStatus: BlockStatus = _blocks.get(blockId)
       originalLevel = blockStatus.storageLevel
       originalMemSize = blockStatus.memSize
@@ -670,7 +802,6 @@ private[spark] class BlockManagerInfo(
     if (storageLevel.isValid) {
       /* isValid means it is either stored in-memory or on-disk.
        * The memSize here indicates the data size in or dropped from memory,
-       * externalBlockStoreSize here indicates the data size in or dropped from externalBlockStore,
        * and the diskSize here indicates the data size in or dropped to disk.
        * They can be both larger than 0, when a block is dropped from memory to disk.
        * Therefore, a safe way to set BlockStatus is to set its info in accurate modes. */
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerMessages.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerMessages.scala
index 7d4f2fff5c34c..afe416a55ed0d 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManagerMessages.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerMessages.scala
@@ -24,37 +24,37 @@ import org.apache.spark.util.Utils
 
 private[spark] object BlockManagerMessages {
   //////////////////////////////////////////////////////////////////////////////////
-  // Messages from the master to slaves.
+  // Messages from the master to storage endpoints.
   //////////////////////////////////////////////////////////////////////////////////
-  sealed trait ToBlockManagerSlave
+  sealed trait ToBlockManagerMasterStorageEndpoint
 
-  // Remove a block from the slaves that have it. This can only be used to remove
+  // Remove a block from the storage endpoints that have it. This can only be used to remove
   // blocks that the master knows about.
-  case class RemoveBlock(blockId: BlockId) extends ToBlockManagerSlave
+  case class RemoveBlock(blockId: BlockId) extends ToBlockManagerMasterStorageEndpoint
 
   // Replicate blocks that were lost due to executor failure
   case class ReplicateBlock(blockId: BlockId, replicas: Seq[BlockManagerId], maxReplicas: Int)
-    extends ToBlockManagerSlave
+    extends ToBlockManagerMasterStorageEndpoint
 
-  case object DecommissionBlockManager extends ToBlockManagerSlave
+  case object DecommissionBlockManager extends ToBlockManagerMasterStorageEndpoint
 
   // Remove all blocks belonging to a specific RDD.
-  case class RemoveRdd(rddId: Int) extends ToBlockManagerSlave
+  case class RemoveRdd(rddId: Int) extends ToBlockManagerMasterStorageEndpoint
 
   // Remove all blocks belonging to a specific shuffle.
-  case class RemoveShuffle(shuffleId: Int) extends ToBlockManagerSlave
+  case class RemoveShuffle(shuffleId: Int) extends ToBlockManagerMasterStorageEndpoint
 
   // Remove all blocks belonging to a specific broadcast.
   case class RemoveBroadcast(broadcastId: Long, removeFromDriver: Boolean = true)
-    extends ToBlockManagerSlave
+    extends ToBlockManagerMasterStorageEndpoint
 
   /**
    * Driver to Executor message to trigger a thread dump.
    */
-  case object TriggerThreadDump extends ToBlockManagerSlave
+  case object TriggerThreadDump extends ToBlockManagerMasterStorageEndpoint
 
   //////////////////////////////////////////////////////////////////////////////////
-  // Messages from slaves to the master.
+  // Messages from storage endpoints to the master.
   //////////////////////////////////////////////////////////////////////////////////
   sealed trait ToBlockManagerMaster
 
@@ -132,13 +132,19 @@ private[spark] object BlockManagerMessages {
   case class GetReplicateInfoForRDDBlocks(blockManagerId: BlockManagerId)
     extends ToBlockManagerMaster
 
-  case class GetBlockStatus(blockId: BlockId, askSlaves: Boolean = true)
+  case class GetBlockStatus(blockId: BlockId, askStorageEndpoints: Boolean = true)
     extends ToBlockManagerMaster
 
-  case class GetMatchingBlockIds(filter: BlockId => Boolean, askSlaves: Boolean = true)
+  case class GetMatchingBlockIds(filter: BlockId => Boolean, askStorageEndpoints: Boolean = true)
     extends ToBlockManagerMaster
 
   case class BlockManagerHeartbeat(blockManagerId: BlockManagerId) extends ToBlockManagerMaster
 
   case class IsExecutorAlive(executorId: String) extends ToBlockManagerMaster
+
+  case class GetShufflePushMergerLocations(numMergersNeeded: Int, hostsToFilter: Set[String])
+    extends ToBlockManagerMaster
+
+  case class RemoveShufflePushMergerLocation(host: String) extends ToBlockManagerMaster
+
 }
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerSlaveEndpoint.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerSlaveEndpoint.scala
deleted file mode 100644
index a3a7149103491..0000000000000
--- a/core/src/main/scala/org/apache/spark/storage/BlockManagerSlaveEndpoint.scala
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.storage
-
-import scala.concurrent.{ExecutionContext, Future}
-
-import org.apache.spark.{MapOutputTracker, SparkEnv}
-import org.apache.spark.internal.Logging
-import org.apache.spark.rpc.{IsolatedRpcEndpoint, RpcCallContext, RpcEnv}
-import org.apache.spark.storage.BlockManagerMessages._
-import org.apache.spark.util.{ThreadUtils, Utils}
-
-/**
- * An RpcEndpoint to take commands from the master to execute options. For example,
- * this is used to remove blocks from the slave's BlockManager.
- */
-private[storage]
-class BlockManagerSlaveEndpoint(
-    override val rpcEnv: RpcEnv,
-    blockManager: BlockManager,
-    mapOutputTracker: MapOutputTracker)
-  extends IsolatedRpcEndpoint with Logging {
-
-  private val asyncThreadPool =
-    ThreadUtils.newDaemonCachedThreadPool("block-manager-slave-async-thread-pool", 100)
-  private implicit val asyncExecutionContext = ExecutionContext.fromExecutorService(asyncThreadPool)
-
-  // Operations that involve removing blocks may be slow and should be done asynchronously
-  override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
-    case RemoveBlock(blockId) =>
-      doAsync[Boolean]("removing block " + blockId, context) {
-        blockManager.removeBlock(blockId)
-        true
-      }
-
-    case RemoveRdd(rddId) =>
-      doAsync[Int]("removing RDD " + rddId, context) {
-        blockManager.removeRdd(rddId)
-      }
-
-    case RemoveShuffle(shuffleId) =>
-      doAsync[Boolean]("removing shuffle " + shuffleId, context) {
-        if (mapOutputTracker != null) {
-          mapOutputTracker.unregisterShuffle(shuffleId)
-        }
-        SparkEnv.get.shuffleManager.unregisterShuffle(shuffleId)
-      }
-
-    case DecommissionBlockManager =>
-      context.reply(blockManager.decommissionBlockManager())
-
-    case RemoveBroadcast(broadcastId, _) =>
-      doAsync[Int]("removing broadcast " + broadcastId, context) {
-        blockManager.removeBroadcast(broadcastId, tellMaster = true)
-      }
-
-    case GetBlockStatus(blockId, _) =>
-      context.reply(blockManager.getStatus(blockId))
-
-    case GetMatchingBlockIds(filter, _) =>
-      context.reply(blockManager.getMatchingBlockIds(filter))
-
-    case TriggerThreadDump =>
-      context.reply(Utils.getThreadDump())
-
-    case ReplicateBlock(blockId, replicas, maxReplicas) =>
-      context.reply(blockManager.replicateBlock(blockId, replicas.toSet, maxReplicas))
-
-  }
-
-  private def doAsync[T](actionMessage: String, context: RpcCallContext)(body: => T): Unit = {
-    val future = Future {
-      logDebug(actionMessage)
-      body
-    }
-    future.foreach { response =>
-      logDebug(s"Done $actionMessage, response is $response")
-      context.reply(response)
-      logDebug(s"Sent response: $response to ${context.senderAddress}")
-    }
-    future.failed.foreach { t =>
-      logError(s"Error in $actionMessage", t)
-      context.sendFailure(t)
-    }
-  }
-
-  override def onStop(): Unit = {
-    asyncThreadPool.shutdownNow()
-  }
-}
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerStorageEndpoint.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerStorageEndpoint.scala
new file mode 100644
index 0000000000000..54a72568b18fa
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerStorageEndpoint.scala
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.storage
+
+import scala.concurrent.{ExecutionContext, Future}
+
+import org.apache.spark.{MapOutputTracker, SparkEnv}
+import org.apache.spark.internal.Logging
+import org.apache.spark.rpc.{IsolatedRpcEndpoint, RpcCallContext, RpcEnv}
+import org.apache.spark.storage.BlockManagerMessages._
+import org.apache.spark.util.{ThreadUtils, Utils}
+
+/**
+ * An RpcEndpoint to take commands from the master to execute options. For example,
+ * this is used to remove blocks from the storage endpoint's BlockManager.
+ */
+private[storage]
+class BlockManagerStorageEndpoint(
+    override val rpcEnv: RpcEnv,
+    blockManager: BlockManager,
+    mapOutputTracker: MapOutputTracker)
+  extends IsolatedRpcEndpoint with Logging {
+
+  private val asyncThreadPool =
+    ThreadUtils.newDaemonCachedThreadPool("block-manager-storage-async-thread-pool", 100)
+  private implicit val asyncExecutionContext = ExecutionContext.fromExecutorService(asyncThreadPool)
+
+  // Operations that involve removing blocks may be slow and should be done asynchronously
+  override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
+    case RemoveBlock(blockId) =>
+      doAsync[Boolean]("removing block " + blockId, context) {
+        blockManager.removeBlock(blockId)
+        true
+      }
+
+    case RemoveRdd(rddId) =>
+      doAsync[Int]("removing RDD " + rddId, context) {
+        blockManager.removeRdd(rddId)
+      }
+
+    case RemoveShuffle(shuffleId) =>
+      doAsync[Boolean]("removing shuffle " + shuffleId, context) {
+        if (mapOutputTracker != null) {
+          mapOutputTracker.unregisterShuffle(shuffleId)
+        }
+        SparkEnv.get.shuffleManager.unregisterShuffle(shuffleId)
+      }
+
+    case DecommissionBlockManager =>
+      context.reply(blockManager.decommissionSelf())
+
+    case RemoveBroadcast(broadcastId, _) =>
+      doAsync[Int]("removing broadcast " + broadcastId, context) {
+        blockManager.removeBroadcast(broadcastId, tellMaster = true)
+      }
+
+    case GetBlockStatus(blockId, _) =>
+      context.reply(blockManager.getStatus(blockId))
+
+    case GetMatchingBlockIds(filter, _) =>
+      context.reply(blockManager.getMatchingBlockIds(filter))
+
+    case TriggerThreadDump =>
+      context.reply(Utils.getThreadDump())
+
+    case ReplicateBlock(blockId, replicas, maxReplicas) =>
+      context.reply(blockManager.replicateBlock(blockId, replicas.toSet, maxReplicas))
+
+  }
+
+  private def doAsync[T](actionMessage: String, context: RpcCallContext)(body: => T): Unit = {
+    val future = Future {
+      logDebug(actionMessage)
+      body
+    }
+    future.foreach { response =>
+      logDebug(s"Done $actionMessage, response is $response")
+      context.reply(response)
+      logDebug(s"Sent response: $response to ${context.senderAddress}")
+    }
+    future.failed.foreach { t =>
+      logError(s"Error in $actionMessage", t)
+      context.sendFailure(t)
+    }
+  }
+
+  override def onStop(): Unit = {
+    asyncThreadPool.shutdownNow()
+  }
+}
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockSavedOnDecommissionedBlockManagerException.scala b/core/src/main/scala/org/apache/spark/storage/BlockSavedOnDecommissionedBlockManagerException.scala
new file mode 100644
index 0000000000000..24d1a83bebcf2
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/storage/BlockSavedOnDecommissionedBlockManagerException.scala
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.storage
+
+private[storage] class BlockSavedOnDecommissionedBlockManagerException(blockId: BlockId)
+  extends Exception(s"Block $blockId cannot be saved on decommissioned executor")
diff --git a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
index f2113947f6bf5..5db4965b67347 100644
--- a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
+++ b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.storage
 
 import java.io.{File, IOException}
+import java.nio.file.Files
 import java.util.UUID
 
 import org.apache.spark.SparkConf
@@ -69,8 +70,8 @@ private[spark] class DiskBlockManager(conf: SparkConf, deleteFilesOnStop: Boolea
         old
       } else {
         val newDir = new File(localDirs(dirId), "%02x".format(subDirId))
-        if (!newDir.exists() && !newDir.mkdir()) {
-          throw new IOException(s"Failed to create local dir in $newDir.")
+        if (!newDir.exists()) {
+          Files.createDirectory(newDir.toPath)
         }
         subDirs(dirId)(subDirId) = newDir
         newDir
@@ -97,7 +98,7 @@ private[spark] class DiskBlockManager(conf: SparkConf, deleteFilesOnStop: Boolea
       }
     }.filter(_ != null).flatMap { dir =>
       val files = dir.listFiles()
-      if (files != null) files else Seq.empty
+      if (files != null) files.toSeq else Seq.empty
     }
   }
 
diff --git a/core/src/main/scala/org/apache/spark/storage/FallbackStorage.scala b/core/src/main/scala/org/apache/spark/storage/FallbackStorage.scala
new file mode 100644
index 0000000000000..76137133227f0
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/storage/FallbackStorage.scala
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.storage
+
+import java.io.DataInputStream
+import java.nio.ByteBuffer
+
+import scala.concurrent.Future
+import scala.reflect.ClassTag
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.{FileSystem, Path}
+
+import org.apache.spark.SparkConf
+import org.apache.spark.deploy.SparkHadoopUtil
+import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config.{STORAGE_DECOMMISSION_FALLBACK_STORAGE_CLEANUP, STORAGE_DECOMMISSION_FALLBACK_STORAGE_PATH}
+import org.apache.spark.network.buffer.{ManagedBuffer, NioManagedBuffer}
+import org.apache.spark.rpc.{RpcAddress, RpcEndpointRef, RpcTimeout}
+import org.apache.spark.shuffle.{IndexShuffleBlockResolver, ShuffleBlockInfo}
+import org.apache.spark.shuffle.IndexShuffleBlockResolver.NOOP_REDUCE_ID
+import org.apache.spark.util.Utils
+
+/**
+ * A fallback storage used by storage decommissioners.
+ */
+private[storage] class FallbackStorage(conf: SparkConf) extends Logging {
+  require(conf.contains("spark.app.id"))
+  require(conf.get(STORAGE_DECOMMISSION_FALLBACK_STORAGE_PATH).isDefined)
+
+  private val fallbackPath = new Path(conf.get(STORAGE_DECOMMISSION_FALLBACK_STORAGE_PATH).get)
+  private val hadoopConf = SparkHadoopUtil.get.newConfiguration(conf)
+  private val fallbackFileSystem = FileSystem.get(fallbackPath.toUri, hadoopConf)
+  private val appId = conf.getAppId
+
+  // Visible for testing
+  def copy(
+      shuffleBlockInfo: ShuffleBlockInfo,
+      bm: BlockManager): Unit = {
+    val shuffleId = shuffleBlockInfo.shuffleId
+    val mapId = shuffleBlockInfo.mapId
+
+    bm.migratableResolver match {
+      case r: IndexShuffleBlockResolver =>
+        val indexFile = r.getIndexFile(shuffleId, mapId)
+
+        if (indexFile.exists()) {
+          fallbackFileSystem.copyFromLocalFile(
+            new Path(indexFile.getAbsolutePath),
+            new Path(fallbackPath, s"$appId/$shuffleId/${indexFile.getName}"))
+
+          val dataFile = r.getDataFile(shuffleId, mapId)
+          if (dataFile.exists()) {
+            fallbackFileSystem.copyFromLocalFile(
+              new Path(dataFile.getAbsolutePath),
+              new Path(fallbackPath, s"$appId/$shuffleId/${dataFile.getName}"))
+          }
+
+          // Report block statuses
+          val reduceId = NOOP_REDUCE_ID
+          val indexBlockId = ShuffleIndexBlockId(shuffleId, mapId, reduceId)
+          FallbackStorage.reportBlockStatus(bm, indexBlockId, indexFile.length)
+          if (dataFile.exists) {
+            val dataBlockId = ShuffleDataBlockId(shuffleId, mapId, reduceId)
+            FallbackStorage.reportBlockStatus(bm, dataBlockId, dataFile.length)
+          }
+        }
+      case r =>
+        logWarning(s"Unsupported Resolver: ${r.getClass.getName}")
+    }
+  }
+
+  def exists(shuffleId: Int, filename: String): Boolean = {
+    fallbackFileSystem.exists(new Path(fallbackPath, s"$appId/$shuffleId/$filename"))
+  }
+}
+
+private[storage] class NoopRpcEndpointRef(conf: SparkConf) extends RpcEndpointRef(conf) {
+  import scala.concurrent.ExecutionContext.Implicits.global
+  override def address: RpcAddress = null
+  override def name: String = "fallback"
+  override def send(message: Any): Unit = {}
+  override def ask[T: ClassTag](message: Any, timeout: RpcTimeout): Future[T] = {
+    Future{true.asInstanceOf[T]}
+  }
+}
+
+private[spark] object FallbackStorage extends Logging {
+  /** We use one block manager id as a place holder. */
+  val FALLBACK_BLOCK_MANAGER_ID: BlockManagerId = BlockManagerId("fallback", "remote", 7337)
+
+  def getFallbackStorage(conf: SparkConf): Option[FallbackStorage] = {
+    if (conf.get(STORAGE_DECOMMISSION_FALLBACK_STORAGE_PATH).isDefined) {
+      Some(new FallbackStorage(conf))
+    } else {
+      None
+    }
+  }
+
+  /** Register the fallback block manager and its RPC endpoint. */
+  def registerBlockManagerIfNeeded(master: BlockManagerMaster, conf: SparkConf): Unit = {
+    if (conf.get(STORAGE_DECOMMISSION_FALLBACK_STORAGE_PATH).isDefined) {
+      master.registerBlockManager(
+        FALLBACK_BLOCK_MANAGER_ID, Array.empty[String], 0, 0, new NoopRpcEndpointRef(conf))
+    }
+  }
+
+  /** Clean up the generated fallback location for this app. */
+  def cleanUp(conf: SparkConf, hadoopConf: Configuration): Unit = {
+    if (conf.get(STORAGE_DECOMMISSION_FALLBACK_STORAGE_PATH).isDefined &&
+        conf.get(STORAGE_DECOMMISSION_FALLBACK_STORAGE_CLEANUP) &&
+        conf.contains("spark.app.id")) {
+      val fallbackPath =
+        new Path(conf.get(STORAGE_DECOMMISSION_FALLBACK_STORAGE_PATH).get, conf.getAppId)
+      val fallbackUri = fallbackPath.toUri
+      val fallbackFileSystem = FileSystem.get(fallbackUri, hadoopConf)
+      // The fallback directory for this app may not be created yet.
+      if (fallbackFileSystem.exists(fallbackPath)) {
+        if (fallbackFileSystem.delete(fallbackPath, true)) {
+          logInfo(s"Succeed to clean up: $fallbackUri")
+        } else {
+          // Clean-up can fail due to the permission issues.
+          logWarning(s"Failed to clean up: $fallbackUri")
+        }
+      }
+    }
+  }
+
+  /** Report block status to block manager master and map output tracker master. */
+  private def reportBlockStatus(blockManager: BlockManager, blockId: BlockId, dataLength: Long) = {
+    assert(blockManager.master != null)
+    blockManager.master.updateBlockInfo(
+      FALLBACK_BLOCK_MANAGER_ID, blockId, StorageLevel.DISK_ONLY, memSize = 0, dataLength)
+  }
+
+  /**
+   * Read a ManagedBuffer.
+   */
+  def read(conf: SparkConf, blockId: BlockId): ManagedBuffer = {
+    logInfo(s"Read $blockId")
+    val fallbackPath = new Path(conf.get(STORAGE_DECOMMISSION_FALLBACK_STORAGE_PATH).get)
+    val hadoopConf = SparkHadoopUtil.get.newConfiguration(conf)
+    val fallbackFileSystem = FileSystem.get(fallbackPath.toUri, hadoopConf)
+    val appId = conf.getAppId
+
+    val (shuffleId, mapId, startReduceId, endReduceId) = blockId match {
+      case id: ShuffleBlockId =>
+        (id.shuffleId, id.mapId, id.reduceId, id.reduceId + 1)
+      case batchId: ShuffleBlockBatchId =>
+        (batchId.shuffleId, batchId.mapId, batchId.startReduceId, batchId.endReduceId)
+      case _ =>
+        throw new IllegalArgumentException("unexpected shuffle block id format: " + blockId)
+    }
+
+    val name = ShuffleIndexBlockId(shuffleId, mapId, NOOP_REDUCE_ID).name
+    val indexFile = new Path(fallbackPath, s"$appId/$shuffleId/$name")
+    val start = startReduceId * 8L
+    val end = endReduceId * 8L
+    Utils.tryWithResource(fallbackFileSystem.open(indexFile)) { inputStream =>
+      Utils.tryWithResource(new DataInputStream(inputStream)) { index =>
+        index.skip(start)
+        val offset = index.readLong()
+        index.skip(end - (start + 8L))
+        val nextOffset = index.readLong()
+        val name = ShuffleDataBlockId(shuffleId, mapId, NOOP_REDUCE_ID).name
+        val dataFile = new Path(fallbackPath, s"$appId/$shuffleId/$name")
+        val f = fallbackFileSystem.open(dataFile)
+        val size = nextOffset - offset
+        logDebug(s"To byte array $size")
+        val array = new Array[Byte](size.toInt)
+        val startTimeNs = System.nanoTime()
+        f.seek(offset)
+        f.read(array)
+        logDebug(s"Took ${(System.nanoTime() - startTimeNs) / (1000 * 1000)}ms")
+        f.close()
+        new NioManagedBuffer(ByteBuffer.wrap(array))
+      }
+    }
+  }
+}
diff --git a/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala b/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala
index 27a4d4b64175e..3a9559797964c 100644
--- a/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala
+++ b/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala
@@ -20,7 +20,7 @@ package org.apache.spark.storage
 import org.apache.spark.SparkEnv
 import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.internal.config._
-import org.apache.spark.rdd.{RDD, RDDOperationScope}
+import org.apache.spark.rdd.{DeterministicLevel, RDD, RDDOperationScope}
 import org.apache.spark.util.Utils
 
 @DeveloperApi
@@ -32,13 +32,13 @@ class RDDInfo(
     val isBarrier: Boolean,
     val parentIds: Seq[Int],
     val callSite: String = "",
-    val scope: Option[RDDOperationScope] = None)
+    val scope: Option[RDDOperationScope] = None,
+    val outputDeterministicLevel: DeterministicLevel.Value = DeterministicLevel.DETERMINATE)
   extends Ordered[RDDInfo] {
 
   var numCachedPartitions = 0
   var memSize = 0L
   var diskSize = 0L
-  var externalBlockStoreSize = 0L
 
   def isCached: Boolean = (memSize + diskSize > 0) && numCachedPartitions > 0
 
@@ -69,6 +69,7 @@ private[spark] object RDDInfo {
       rdd.creationSite.shortForm
     }
     new RDDInfo(rdd.id, rddName, rdd.partitions.length,
-      rdd.getStorageLevel, rdd.isBarrier(), parentIds, callSite, rdd.scope)
+      rdd.getStorageLevel, rdd.isBarrier(), parentIds, callSite, rdd.scope,
+      rdd.outputDeterministicLevel)
   }
 }
diff --git a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
index 5efbc0703f729..f30437f404455 100644
--- a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
+++ b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
@@ -210,13 +210,18 @@ final class ShuffleBlockFetcherIterator(
     while (iter.hasNext) {
       val result = iter.next()
       result match {
-        case SuccessFetchResult(_, _, address, _, buf, _) =>
+        case SuccessFetchResult(blockId, mapIndex, address, _, buf, _) =>
           if (address != blockManager.blockManagerId) {
-            shuffleMetrics.incRemoteBytesRead(buf.size)
-            if (buf.isInstanceOf[FileSegmentManagedBuffer]) {
-              shuffleMetrics.incRemoteBytesReadToDisk(buf.size)
+            if (hostLocalBlocks.contains(blockId -> mapIndex)) {
+              shuffleMetrics.incLocalBlocksFetched(1)
+              shuffleMetrics.incLocalBytesRead(buf.size)
+            } else {
+              shuffleMetrics.incRemoteBytesRead(buf.size)
+              if (buf.isInstanceOf[FileSegmentManagedBuffer]) {
+                shuffleMetrics.incRemoteBytesReadToDisk(buf.size)
+              }
+              shuffleMetrics.incRemoteBlocksFetched(1)
             }
-            shuffleMetrics.incRemoteBlocksFetched(1)
           }
           buf.release()
         case _ =>
@@ -290,18 +295,17 @@ final class ShuffleBlockFetcherIterator(
     var hostLocalBlockBytes = 0L
     var remoteBlockBytes = 0L
 
-    val hostLocalDirReadingEnabled =
-      blockManager.hostLocalDirManager != null && blockManager.hostLocalDirManager.isDefined
-
+    val fallback = FallbackStorage.FALLBACK_BLOCK_MANAGER_ID.executorId
     for ((address, blockInfos) <- blocksByAddress) {
-      if (address.executorId == blockManager.blockManagerId.executorId) {
+      if (Seq(blockManager.blockManagerId.executorId, fallback).contains(address.executorId)) {
         checkBlockSizes(blockInfos)
         val mergedBlockInfos = mergeContinuousShuffleBlockIdsIfNeeded(
           blockInfos.map(info => FetchBlockInfo(info._1, info._2, info._3)), doBatchFetch)
         numBlocksToFetch += mergedBlockInfos.size
         localBlocks ++= mergedBlockInfos.map(info => (info.blockId, info.mapIndex))
         localBlockBytes += mergedBlockInfos.map(_.size).sum
-      } else if (hostLocalDirReadingEnabled && address.host == blockManager.blockManagerId.host) {
+      } else if (blockManager.hostLocalDirManager.isDefined &&
+        address.host == blockManager.blockManagerId.host) {
         checkBlockSizes(blockInfos)
         val mergedBlockInfos = mergeContinuousShuffleBlockIdsIfNeeded(
           blockInfos.map(info => FetchBlockInfo(info._1, info._2, info._3)), doBatchFetch)
@@ -368,25 +372,25 @@ final class ShuffleBlockFetcherIterator(
       collectedRemoteRequests: ArrayBuffer[FetchRequest]): Unit = {
     val iterator = blockInfos.iterator
     var curRequestSize = 0L
-    var curBlocks = new ArrayBuffer[FetchBlockInfo]
+    var curBlocks = Seq.empty[FetchBlockInfo]
 
     while (iterator.hasNext) {
       val (blockId, size, mapIndex) = iterator.next()
       assertPositiveBlockSize(blockId, size)
-      curBlocks += FetchBlockInfo(blockId, size, mapIndex)
+      curBlocks = curBlocks ++ Seq(FetchBlockInfo(blockId, size, mapIndex))
       curRequestSize += size
       // For batch fetch, the actual block in flight should count for merged block.
       val mayExceedsMaxBlocks = !doBatchFetch && curBlocks.size >= maxBlocksInFlightPerAddress
       if (curRequestSize >= targetRemoteRequestSize || mayExceedsMaxBlocks) {
         curBlocks = createFetchRequests(curBlocks, address, isLast = false,
-          collectedRemoteRequests).to[ArrayBuffer]
+          collectedRemoteRequests)
         curRequestSize = curBlocks.map(_.size).sum
       }
     }
     // Add in the final request
     if (curBlocks.nonEmpty) {
       curBlocks = createFetchRequests(curBlocks, address, isLast = true,
-        collectedRemoteRequests).to[ArrayBuffer]
+        collectedRemoteRequests)
       curRequestSize = curBlocks.map(_.size).sum
     }
   }
@@ -463,54 +467,72 @@ final class ShuffleBlockFetcherIterator(
    * track in-memory are the ManagedBuffer references themselves.
    */
   private[this] def fetchHostLocalBlocks(hostLocalDirManager: HostLocalDirManager): Unit = {
-    val cachedDirsByExec = hostLocalDirManager.getCachedHostLocalDirs()
-    val (hostLocalBlocksWithCachedDirs, hostLocalBlocksWithMissingDirs) =
-      hostLocalBlocksByExecutor
-        .map { case (hostLocalBmId, bmInfos) =>
-          (hostLocalBmId, bmInfos, cachedDirsByExec.get(hostLocalBmId.executorId))
-        }.partition(_._3.isDefined)
-    val bmId = blockManager.blockManagerId
-    val immutableHostLocalBlocksWithoutDirs =
-      hostLocalBlocksWithMissingDirs.map { case (hostLocalBmId, bmInfos, _) =>
-        hostLocalBmId -> bmInfos
-      }.toMap
-    if (immutableHostLocalBlocksWithoutDirs.nonEmpty) {
+    val cachedDirsByExec = hostLocalDirManager.getCachedHostLocalDirs
+    val (hostLocalBlocksWithCachedDirs, hostLocalBlocksWithMissingDirs) = {
+      val (hasCache, noCache) = hostLocalBlocksByExecutor.partition { case (hostLocalBmId, _) =>
+        cachedDirsByExec.contains(hostLocalBmId.executorId)
+      }
+      (hasCache.toMap, noCache.toMap)
+    }
+
+    if (hostLocalBlocksWithMissingDirs.nonEmpty) {
       logDebug(s"Asynchronous fetching host-local blocks without cached executors' dir: " +
-        s"${immutableHostLocalBlocksWithoutDirs.mkString(", ")}")
-      val execIdsWithoutDirs = immutableHostLocalBlocksWithoutDirs.keys.map(_.executorId).toArray
-      hostLocalDirManager.getHostLocalDirs(execIdsWithoutDirs) {
-        case Success(dirs) =>
-          immutableHostLocalBlocksWithoutDirs.foreach { case (hostLocalBmId, blockInfos) =>
-            blockInfos.takeWhile { case (blockId, _, mapIndex) =>
-              fetchHostLocalBlock(
-                blockId,
-                mapIndex,
-                dirs.get(hostLocalBmId.executorId),
-                hostLocalBmId)
-            }
-          }
-          logDebug(s"Got host-local blocks (without cached executors' dir) in " +
-            s"${Utils.getUsedTimeNs(startTimeNs)}")
-
-        case Failure(throwable) =>
-          logError(s"Error occurred while fetching host local blocks", throwable)
-          val (hostLocalBmId, blockInfoSeq) = immutableHostLocalBlocksWithoutDirs.head
-          val (blockId, _, mapIndex) = blockInfoSeq.head
-          results.put(FailureFetchResult(blockId, mapIndex, hostLocalBmId, throwable))
+        s"${hostLocalBlocksWithMissingDirs.mkString(", ")}")
+
+      // If the external shuffle service is enabled, we'll fetch the local directories for
+      // multiple executors from the external shuffle service, which located at the same host
+      // with the executors, in once. Otherwise, we'll fetch the local directories from those
+      // executors directly one by one. The fetch requests won't be too much since one host is
+      // almost impossible to have many executors at the same time practically.
+      val dirFetchRequests = if (blockManager.externalShuffleServiceEnabled) {
+        val host = blockManager.blockManagerId.host
+        val port = blockManager.externalShuffleServicePort
+        Seq((host, port, hostLocalBlocksWithMissingDirs.keys.toArray))
+      } else {
+        hostLocalBlocksWithMissingDirs.keys.map(bmId => (bmId.host, bmId.port, Array(bmId))).toSeq
+      }
+
+      dirFetchRequests.foreach { case (host, port, bmIds) =>
+        hostLocalDirManager.getHostLocalDirs(host, port, bmIds.map(_.executorId)) {
+          case Success(dirsByExecId) =>
+            fetchMultipleHostLocalBlocks(
+              hostLocalBlocksWithMissingDirs.filterKeys(bmIds.contains).toMap,
+              dirsByExecId,
+              cached = false)
+
+          case Failure(throwable) =>
+            logError("Error occurred while fetching host local blocks", throwable)
+            val bmId = bmIds.head
+            val blockInfoSeq = hostLocalBlocksWithMissingDirs(bmId)
+            val (blockId, _, mapIndex) = blockInfoSeq.head
+            results.put(FailureFetchResult(blockId, mapIndex, bmId, throwable))
+        }
       }
     }
+
     if (hostLocalBlocksWithCachedDirs.nonEmpty) {
       logDebug(s"Synchronous fetching host-local blocks with cached executors' dir: " +
           s"${hostLocalBlocksWithCachedDirs.mkString(", ")}")
-      hostLocalBlocksWithCachedDirs.foreach { case (_, blockInfos, localDirs) =>
-        blockInfos.foreach { case (blockId, _, mapIndex) =>
-          if (!fetchHostLocalBlock(blockId, mapIndex, localDirs.get, bmId)) {
-            return
-          }
-        }
+      fetchMultipleHostLocalBlocks(hostLocalBlocksWithCachedDirs, cachedDirsByExec, cached = true)
+    }
+  }
+
+  private def fetchMultipleHostLocalBlocks(
+      bmIdToBlocks: Map[BlockManagerId, Seq[(BlockId, Long, Int)]],
+      localDirsByExecId: Map[String, Array[String]],
+      cached: Boolean): Unit = {
+    // We use `forall` because once there's a failed block fetch, `fetchHostLocalBlock` will put
+    // a `FailureFetchResult` immediately to the `results`. So there's no reason to fetch the
+    // remaining blocks.
+    val allFetchSucceeded = bmIdToBlocks.forall { case (bmId, blockInfos) =>
+      blockInfos.forall { case (blockId, _, mapIndex) =>
+        fetchHostLocalBlock(blockId, mapIndex, localDirsByExecId(bmId.executorId), bmId)
       }
-      logDebug(s"Got host-local blocks (with cached executors' dir) in " +
-        s"${Utils.getUsedTimeNs(startTimeNs)}")
+    }
+    if (allFetchSucceeded) {
+      logDebug(s"Got host-local blocks from ${bmIdToBlocks.keys.mkString(", ")} " +
+        s"(${if (cached) "with" else "without"} cached executors' dir) " +
+        s"in ${Utils.getUsedTimeNs(startTimeNs)}")
     }
   }
 
@@ -529,8 +551,10 @@ final class ShuffleBlockFetcherIterator(
     // Send out initial requests for blocks, up to our maxBytesInFlight
     fetchUpToMaxBytes()
 
-    val numFetches = remoteRequests.size - fetchRequests.size
-    logInfo(s"Started $numFetches remote fetches in ${Utils.getUsedTimeNs(startTimeNs)}")
+    val numDeferredRequest = deferredFetchRequests.values.map(_.size).sum
+    val numFetches = remoteRequests.size - fetchRequests.size - numDeferredRequest
+    logInfo(s"Started $numFetches remote fetches in ${Utils.getUsedTimeNs(startTimeNs)}" +
+      (if (numDeferredRequest > 0 ) s", deferred $numDeferredRequest requests" else ""))
 
     // Get Local Blocks
     fetchLocalBlocks()
@@ -770,15 +794,8 @@ private class BufferReleasingInputStream(
   extends InputStream {
   private[this] var closed = false
 
-  override def read(): Int = {
-    try {
-      delegate.read()
-    } catch {
-      case e: IOException if detectCorruption =>
-        IOUtils.closeQuietly(this)
-        iterator.throwFetchFailedException(blockId, mapIndex, address, e)
-    }
-  }
+  override def read(): Int =
+    tryOrFetchFailedException(delegate.read())
 
   override def close(): Unit = {
     if (!closed) {
@@ -792,39 +809,33 @@ private class BufferReleasingInputStream(
 
   override def mark(readlimit: Int): Unit = delegate.mark(readlimit)
 
-  override def skip(n: Long): Long = {
-    try {
-      delegate.skip(n)
-    } catch {
-      case e: IOException if detectCorruption =>
-        IOUtils.closeQuietly(this)
-        iterator.throwFetchFailedException(blockId, mapIndex, address, e)
-    }
-  }
+  override def skip(n: Long): Long =
+    tryOrFetchFailedException(delegate.skip(n))
 
   override def markSupported(): Boolean = delegate.markSupported()
 
-  override def read(b: Array[Byte]): Int = {
-    try {
-      delegate.read(b)
-    } catch {
-      case e: IOException if detectCorruption =>
-        IOUtils.closeQuietly(this)
-        iterator.throwFetchFailedException(blockId, mapIndex, address, e)
-    }
-  }
+  override def read(b: Array[Byte]): Int =
+    tryOrFetchFailedException(delegate.read(b))
+
+  override def read(b: Array[Byte], off: Int, len: Int): Int =
+    tryOrFetchFailedException(delegate.read(b, off, len))
 
-  override def read(b: Array[Byte], off: Int, len: Int): Int = {
+  override def reset(): Unit = delegate.reset()
+
+  /**
+   * Execute a block of code that returns a value, close this stream quietly and re-throwing
+   * IOException as FetchFailedException when detectCorruption is true. This method is only
+   * used by the `read` and `skip` methods inside `BufferReleasingInputStream` currently.
+   */
+  private def tryOrFetchFailedException[T](block: => T): T = {
     try {
-      delegate.read(b, off, len)
+      block
     } catch {
       case e: IOException if detectCorruption =>
         IOUtils.closeQuietly(this)
         iterator.throwFetchFailedException(blockId, mapIndex, address, e)
     }
   }
-
-  override def reset(): Unit = delegate.reset()
 }
 
 /**
@@ -928,7 +939,7 @@ object ShuffleBlockFetcherIterator {
     } else {
       blocks
     }
-    result
+    result.toSeq
   }
 
   /**
diff --git a/core/src/main/scala/org/apache/spark/storage/StorageLevel.scala b/core/src/main/scala/org/apache/spark/storage/StorageLevel.scala
index 4c6998d7a8e20..ce89c2ae90b49 100644
--- a/core/src/main/scala/org/apache/spark/storage/StorageLevel.scala
+++ b/core/src/main/scala/org/apache/spark/storage/StorageLevel.scala
@@ -45,7 +45,7 @@ class StorageLevel private(
   extends Externalizable {
 
   // TODO: Also add fields for caching priority, dataset ID, and flushing.
-  private def this(flags: Int, replication: Int) {
+  private def this(flags: Int, replication: Int) = {
     this((flags & 8) != 0, (flags & 4) != 0, (flags & 2) != 0, (flags & 1) != 0, replication)
   }
 
@@ -153,6 +153,7 @@ object StorageLevel {
   val NONE = new StorageLevel(false, false, false, false)
   val DISK_ONLY = new StorageLevel(true, false, false, false)
   val DISK_ONLY_2 = new StorageLevel(true, false, false, false, 2)
+  val DISK_ONLY_3 = new StorageLevel(true, false, false, false, 3)
   val MEMORY_ONLY = new StorageLevel(false, true, false, true)
   val MEMORY_ONLY_2 = new StorageLevel(false, true, false, true, 2)
   val MEMORY_ONLY_SER = new StorageLevel(false, true, false, false)
@@ -172,6 +173,7 @@ object StorageLevel {
     case "NONE" => NONE
     case "DISK_ONLY" => DISK_ONLY
     case "DISK_ONLY_2" => DISK_ONLY_2
+    case "DISK_ONLY_3" => DISK_ONLY_3
     case "MEMORY_ONLY" => MEMORY_ONLY
     case "MEMORY_ONLY_2" => MEMORY_ONLY_2
     case "MEMORY_ONLY_SER" => MEMORY_ONLY_SER
diff --git a/core/src/main/scala/org/apache/spark/storage/StorageUtils.scala b/core/src/main/scala/org/apache/spark/storage/StorageUtils.scala
index fc426eee608c0..c607fb28b2f56 100644
--- a/core/src/main/scala/org/apache/spark/storage/StorageUtils.scala
+++ b/core/src/main/scala/org/apache/spark/storage/StorageUtils.scala
@@ -61,7 +61,7 @@ private[spark] class StorageStatus(
       maxMemory: Long,
       maxOnHeapMem: Option[Long],
       maxOffHeapMem: Option[Long],
-      initialBlocks: Map[BlockId, BlockStatus]) {
+      initialBlocks: Map[BlockId, BlockStatus]) = {
     this(bmid, maxMemory, maxOnHeapMem, maxOffHeapMem)
     initialBlocks.foreach { case (bid, bstatus) => addBlock(bid, bstatus) }
   }
@@ -169,7 +169,7 @@ private[spark] class StorageStatus(
           .getOrElse((0L, 0L))
       case _ if !level.useOffHeap =>
         (_nonRddStorageInfo.onHeapUsage, _nonRddStorageInfo.diskUsage)
-      case _ if level.useOffHeap =>
+      case _ =>
         (_nonRddStorageInfo.offHeapUsage, _nonRddStorageInfo.diskUsage)
     }
     val newMem = math.max(oldMem + changeInMem, 0L)
diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
index 4b4788f453243..663da0d33e20b 100644
--- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
@@ -17,12 +17,13 @@
 
 package org.apache.spark.ui
 
-import java.net.{URI, URL}
+import java.net.{URI, URL, URLDecoder}
 import java.util.EnumSet
 import javax.servlet.DispatcherType
 import javax.servlet.http._
 
 import scala.language.implicitConversions
+import scala.util.Try
 import scala.xml.Node
 
 import org.eclipse.jetty.client.HttpClient
@@ -376,8 +377,7 @@ private[spark] object JettyUtils extends Logging {
         if (baseRequest.isSecure) {
           return
         }
-        val httpsURI = createRedirectURI(scheme, baseRequest.getServerName, securePort,
-          baseRequest.getRequestURI, baseRequest.getQueryString)
+        val httpsURI = createRedirectURI(scheme, securePort, baseRequest)
         response.setContentLength(0)
         response.sendRedirect(response.encodeRedirectURL(httpsURI))
         baseRequest.setHandled(true)
@@ -401,17 +401,13 @@ private[spark] object JettyUtils extends Logging {
       uri.append(rest)
     }
 
-    val rewrittenURI = URI.create(uri.toString())
-    if (query != null) {
-      return new URI(
-          rewrittenURI.getScheme(),
-          rewrittenURI.getAuthority(),
-          rewrittenURI.getPath(),
-          query,
-          rewrittenURI.getFragment()
-        ).normalize()
+    val queryString = if (query == null) {
+      ""
+    } else {
+      s"?$query"
     }
-    rewrittenURI.normalize()
+    // SPARK-33611: use method `URI.create` to avoid percent-encoding twice on the query string.
+    URI.create(uri.toString() + queryString).normalize()
   }
 
   def createProxyLocationHeader(
@@ -439,16 +435,34 @@ private[spark] object JettyUtils extends Logging {
     handler.addFilter(holder, "/*", EnumSet.allOf(classOf[DispatcherType]))
   }
 
+  private def decodeURL(url: String, encoding: String): String = {
+    if (url == null) {
+      null
+    } else {
+      URLDecoder.decode(url, encoding)
+    }
+  }
+
   // Create a new URI from the arguments, handling IPv6 host encoding and default ports.
-  private def createRedirectURI(
-      scheme: String, server: String, port: Int, path: String, query: String) = {
+  private def createRedirectURI(scheme: String, port: Int, request: Request): String = {
+    val server = request.getServerName
     val redirectServer = if (server.contains(":") && !server.startsWith("[")) {
       s"[${server}]"
     } else {
       server
     }
     val authority = s"$redirectServer:$port"
-    new URI(scheme, authority, path, query, null).toString
+    val queryEncoding = if (request.getQueryEncoding != null) {
+      request.getQueryEncoding
+    } else {
+      // By default decoding the URI as "UTF-8" should be enough for SparkUI
+      "UTF-8"
+    }
+    // The request URL can be raw or encoded here. To avoid the request URL being
+    // encoded twice, let's decode it here.
+    val requestURI = decodeURL(request.getRequestURI, queryEncoding)
+    val queryString = decodeURL(request.getQueryString, queryEncoding)
+    new URI(scheme, authority, requestURI, queryString, null).toString
   }
 
   def toVirtualHosts(connectors: String*): Array[String] = connectors.map("@" + _).toArray
@@ -500,7 +514,11 @@ private[spark] case class ServerInfo(
     threadPool match {
       case pool: QueuedThreadPool =>
         // Workaround for SPARK-30385 to avoid Jetty's acceptor thread shrink.
-        pool.setIdleTimeout(0)
+        // As of Jetty 9.4.21, the implementation of
+        // QueuedThreadPool#setIdleTimeout is changed and IllegalStateException
+        // will be thrown if we try to set idle timeout after the server has started.
+        // But this workaround works for Jetty 9.4.28 by ignoring the exception.
+        Try(pool.setIdleTimeout(0))
       case _ =>
     }
     server.stop()
diff --git a/core/src/main/scala/org/apache/spark/ui/PagedTable.scala b/core/src/main/scala/org/apache/spark/ui/PagedTable.scala
index 008dcc6200d37..a002af70a919d 100644
--- a/core/src/main/scala/org/apache/spark/ui/PagedTable.scala
+++ b/core/src/main/scala/org/apache/spark/ui/PagedTable.scala
@@ -17,8 +17,9 @@
 
 package org.apache.spark.ui
 
-import java.net.URLDecoder
+import java.net.{URLDecoder, URLEncoder}
 import java.nio.charset.StandardCharsets.UTF_8
+import javax.servlet.http.HttpServletRequest
 
 import scala.collection.JavaConverters._
 import scala.xml.{Node, Unparsed}
@@ -297,4 +298,102 @@ private[spark] trait PagedTable[T] {
    * Returns the submission path for the "go to page #" form.
    */
   def goButtonFormPath: String
+
+  /**
+   * Returns parameters of other tables in the page.
+   */
+  def getParameterOtherTable(request: HttpServletRequest, tableTag: String): String = {
+    request.getParameterMap.asScala
+      .filterNot(_._1.startsWith(tableTag))
+      .map(parameter => parameter._1 + "=" + parameter._2(0))
+      .mkString("&")
+  }
+
+  /**
+   * Returns parameter of this table.
+   */
+  def getTableParameters(
+      request: HttpServletRequest,
+      tableTag: String,
+      defaultSortColumn: String): (String, Boolean, Int) = {
+    val parameterSortColumn = request.getParameter(s"$tableTag.sort")
+    val parameterSortDesc = request.getParameter(s"$tableTag.desc")
+    val parameterPageSize = request.getParameter(s"$tableTag.pageSize")
+    val sortColumn = Option(parameterSortColumn).map { sortColumn =>
+      UIUtils.decodeURLParameter(sortColumn)
+    }.getOrElse(defaultSortColumn)
+    val desc = Option(parameterSortDesc).map(_.toBoolean).getOrElse(
+      sortColumn == defaultSortColumn
+    )
+    val pageSize = Option(parameterPageSize).map(_.toInt).getOrElse(100)
+
+    (sortColumn, desc, pageSize)
+  }
+
+  /**
+   * Check if given sort column is valid or not. If invalid then an exception is thrown.
+   */
+  def isSortColumnValid(
+      headerInfo: Seq[(String, Boolean, Option[String])],
+      sortColumn: String): Unit = {
+    if (!headerInfo.filter(_._2).map(_._1).contains(sortColumn)) {
+      throw new IllegalArgumentException(s"Unknown column: $sortColumn")
+    }
+  }
+
+  def headerRow(
+      headerInfo: Seq[(String, Boolean, Option[String])],
+      desc: Boolean,
+      pageSize: Int,
+      sortColumn: String,
+      parameterPath: String,
+      tableTag: String,
+      headerId: String): Seq[Node] = {
+    val row: Seq[Node] = {
+      headerInfo.map { case (header, sortable, tooltip) =>
+        if (header == sortColumn) {
+          val headerLink = Unparsed(
+            parameterPath +
+              s"&$tableTag.sort=${URLEncoder.encode(header, UTF_8.name())}" +
+              s"&$tableTag.desc=${!desc}" +
+              s"&$tableTag.pageSize=$pageSize" +
+              s"#$headerId")
+          val arrow = if (desc) "&#x25BE;" else "&#x25B4;" // UP or DOWN
+
+          <th>
+            <a href={headerLink}>
+              <span data-toggle="tooltip" data-placement="top" title={tooltip.getOrElse("")}>
+                {header}&nbsp;{Unparsed(arrow)}
+              </span>
+            </a>
+          </th>
+        } else {
+          if (sortable) {
+            val headerLink = Unparsed(
+              parameterPath +
+                s"&$tableTag.sort=${URLEncoder.encode(header, UTF_8.name())}" +
+                s"&$tableTag.pageSize=$pageSize" +
+                s"#$headerId")
+
+            <th>
+              <a href={headerLink}>
+                <span data-toggle="tooltip" data-placement="top" title={tooltip.getOrElse("")}>
+                  {header}
+                </span>
+              </a>
+            </th>
+          } else {
+            <th>
+              <span data-toggle="tooltip" data-placement="top" title={tooltip.getOrElse("")}>
+                {header}
+              </span>
+            </th>
+          }
+        }
+      }
+    }
+    <thead>
+      <tr>{row}</tr>
+    </thead>
+  }
 }
diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
index 8ae9828c3fee1..b1769a8a9c9ee 100644
--- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
+++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
@@ -110,6 +110,11 @@ private[spark] class SparkUI private (
     }
   }
 
+  override def checkUIViewPermissions(appId: String, attemptId: Option[String],
+      user: String): Boolean = {
+    securityManager.checkUIViewPermissions(user)
+  }
+
   def getApplicationInfoList: Iterator[ApplicationInfo] = {
     Iterator(new ApplicationInfo(
       id = appId,
diff --git a/core/src/main/scala/org/apache/spark/ui/ToolTips.scala b/core/src/main/scala/org/apache/spark/ui/ToolTips.scala
index aefd001e573f9..a7c42b86468b2 100644
--- a/core/src/main/scala/org/apache/spark/ui/ToolTips.scala
+++ b/core/src/main/scala/org/apache/spark/ui/ToolTips.scala
@@ -91,9 +91,6 @@ private[spark] object ToolTips {
   val TASK_TIME =
   "Shaded red when garbage collection (GC) time is over 10% of task time"
 
-  val BLACKLISTED =
-  "Shows if this executor has been blacklisted by the scheduler due to task failures."
-
   val APPLICATION_EXECUTOR_LIMIT =
     """Maximum number of executors that this application will use. This limit is finite only when
        dynamic allocation is enabled. The number of granted executors may exceed the limit
diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
index 90167858df663..60db87c18a977 100644
--- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
@@ -231,7 +231,7 @@ private[spark] object UIUtils extends Logging {
     <link rel="stylesheet"
           href={prependBaseUri(request, "/static/timeline-view.css")} type="text/css"/>
     <script src={prependBaseUri(request, "/static/sorttable.js")} ></script>
-    <script src={prependBaseUri(request, "/static/jquery-3.4.1.min.js")}></script>
+    <script src={prependBaseUri(request, "/static/jquery-3.5.1.min.js")}></script>
     <script src={prependBaseUri(request, "/static/vis-timeline-graph2d.min.js")}></script>
     <script src={prependBaseUri(request, "/static/bootstrap.bundle.min.js")}></script>
     <script src={prependBaseUri(request, "/static/initialize-tooltips.js")}></script>
@@ -292,6 +292,7 @@ private[spark] object UIUtils extends Logging {
     <html>
       <head>
         {commonHeaderNodes(request)}
+        <script>setAppBasePath('{activeTab.basePath}')</script>
         {if (showVisualization) vizHeaderNodes(request) else Seq.empty}
         {if (useDataTables) dataTablesHeaderNodes(request) else Seq.empty}
         <link rel="shortcut icon"
@@ -324,7 +325,8 @@ private[spark] object UIUtils extends Logging {
         <div class="container-fluid">
           <div class="row">
             <div class="col-12">
-              <h3 style="vertical-align: bottom; display: inline-block;">
+              <h3 style="vertical-align: bottom; white-space: nowrap; overflow: hidden;
+                text-overflow: ellipsis;">
                 {title}
                 {helpButton}
               </h3>
@@ -442,7 +444,7 @@ private[spark] object UIUtils extends Logging {
             </th>
           case None => <th width={colWidthAttr} class={getClass(x._2)}>{getHeaderContent(x._1)}</th>
         }
-      }
+      }.toSeq
     }
     <table class={listingTableClass} id={id.map(Text.apply)}>
       <thead>{headerRow}</thead>
@@ -459,13 +461,14 @@ private[spark] object UIUtils extends Logging {
       skipped: Int,
       reasonToNumKilled: Map[String, Int],
       total: Int): Seq[Node] = {
-    val ratio = if (total == 0) 100.0 else (completed.toDouble/total)*100
+    val ratio = if (total == 0) 100.0 else (completed.toDouble / total) * 100
     val completeWidth = "width: %s%%".format(ratio)
     // started + completed can be > total when there are speculative tasks
     val boundedStarted = math.min(started, total - completed)
-    val startWidth = "width: %s%%".format((boundedStarted.toDouble/total)*100)
+    val startRatio = if (total == 0) 0.0 else (boundedStarted.toDouble / total) * 100
+    val startWidth = "width: %s%%".format(startRatio)
 
-    <div class={ if (started > 0) s"progress progress-started" else s"progress" }>
+    <div class="progress">
       <span style="text-align:center; position:absolute; width:100%;">
         {completed}/{total}
         { if (failed == 0 && skipped == 0 && started > 0) s"($started running)" }
@@ -476,7 +479,8 @@ private[spark] object UIUtils extends Logging {
           }
         }
       </span>
-      <div class="progress-bar" style={completeWidth}></div>
+      <div class="progress-bar progress-completed" style={completeWidth}></div>
+      <div class="progress-bar progress-started" style={startWidth}></div>
     </div>
   }
 
@@ -523,6 +527,9 @@ private[spark] object UIUtils extends Logging {
                 } ++
                 g.rootCluster.getBarrierClusters.map { c =>
                   <div class="barrier-rdd">{c.id}</div>
+                } ++
+                g.rootCluster.getIndeterminateNodes.map { n =>
+                  <div class="indeterminate-rdd">{n.id}</div>
                 }
               }
             </div>
@@ -638,7 +645,8 @@ private[spark] object UIUtils extends Logging {
   */
   def makeHref(proxy: Boolean, id: String, origHref: String): String = {
     if (proxy) {
-      s"/proxy/$id"
+      val proxyPrefix = sys.props.getOrElse("spark.ui.proxyBase", "")
+      proxyPrefix + "/proxy/" + id
     } else {
       origHref
     }
diff --git a/core/src/main/scala/org/apache/spark/ui/WebUI.scala b/core/src/main/scala/org/apache/spark/ui/WebUI.scala
index 9faa3dcf2cdf2..a4e87704927c6 100644
--- a/core/src/main/scala/org/apache/spark/ui/WebUI.scala
+++ b/core/src/main/scala/org/apache/spark/ui/WebUI.scala
@@ -58,11 +58,11 @@ private[spark] abstract class WebUI(
   private val className = Utils.getFormattedClassName(this)
 
   def getBasePath: String = basePath
-  def getTabs: Seq[WebUITab] = tabs
-  def getHandlers: Seq[ServletContextHandler] = handlers
+  def getTabs: Seq[WebUITab] = tabs.toSeq
+  def getHandlers: Seq[ServletContextHandler] = handlers.toSeq
 
   def getDelegatingHandlers: Seq[DelegatingServletContextHandler] = {
-    handlers.map(new DelegatingServletContextHandler(_))
+    handlers.map(new DelegatingServletContextHandler(_)).toSeq
   }
 
   /** Attaches a tab to this UI, along with all of its attached pages. */
diff --git a/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala b/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala
index c6eb461ad601c..2f5b73118927b 100644
--- a/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala
@@ -19,9 +19,11 @@ package org.apache.spark.ui.env
 
 import javax.servlet.http.HttpServletRequest
 
+import scala.collection.mutable.StringBuilder
 import scala.xml.Node
 
 import org.apache.spark.SparkConf
+import org.apache.spark.resource.{ExecutorResourceRequest, TaskResourceRequest}
 import org.apache.spark.status.AppStatusStore
 import org.apache.spark.ui._
 import org.apache.spark.util.Utils
@@ -38,6 +40,37 @@ private[ui] class EnvironmentPage(
       "Java Home" -> appEnv.runtime.javaHome,
       "Scala Version" -> appEnv.runtime.scalaVersion)
 
+    def constructExecutorRequestString(execReqs: Map[String, ExecutorResourceRequest]): String = {
+      execReqs.map {
+        case (_, execReq) =>
+          val execStr = new StringBuilder(s"\t${execReq.resourceName}: [amount: ${execReq.amount}")
+          if (execReq.discoveryScript.nonEmpty) {
+            execStr ++= s", discovery: ${execReq.discoveryScript}"
+          }
+          if (execReq.vendor.nonEmpty) {
+            execStr ++= s", vendor: ${execReq.vendor}"
+          }
+          execStr ++= "]"
+          execStr.toString()
+      }.mkString("\n")
+    }
+
+    def constructTaskRequestString(taskReqs: Map[String, TaskResourceRequest]): String = {
+      taskReqs.map {
+        case (_, taskReq) => s"\t${taskReq.resourceName}: [amount: ${taskReq.amount}]"
+      }.mkString("\n")
+    }
+
+    val resourceProfileInfo = store.resourceProfileInfo().map { rinfo =>
+      val einfo = constructExecutorRequestString(rinfo.executorResources)
+      val tinfo = constructTaskRequestString(rinfo.taskResources)
+      val res = s"Executor Reqs:\n$einfo\nTask Reqs:\n$tinfo"
+      (rinfo.id.toString, res)
+    }.toMap
+
+    val resourceProfileInformationTable = UIUtils.listingTable(resourceProfileHeader,
+      jvmRowDataPre, resourceProfileInfo.toSeq.sortWith(_._1.toInt < _._1.toInt),
+      fixedWidth = true, headerClasses = headerClassesNoSortValues)
     val runtimeInformationTable = UIUtils.listingTable(
       propertyHeader, jvmRow, jvmInformation.toSeq.sorted, fixedWidth = true,
       headerClasses = headerClasses)
@@ -77,6 +110,17 @@ private[ui] class EnvironmentPage(
         <div class="aggregated-sparkProperties collapsible-table">
           {sparkPropertiesTable}
         </div>
+        <span class="collapse-aggregated-execResourceProfileInformation collapse-table"
+              onClick="collapseTable('collapse-aggregated-execResourceProfileInformation',
+            'aggregated-execResourceProfileInformation')">
+          <h4>
+            <span class="collapse-table-arrow arrow-open"></span>
+            <a>Resource Profiles</a>
+          </h4>
+        </span>
+        <div class="aggregated-execResourceProfileInformation collapsible-table">
+          {resourceProfileInformationTable}
+        </div>
         <span class="collapse-aggregated-hadoopProperties collapse-table"
               onClick="collapseTable('collapse-aggregated-hadoopProperties',
             'aggregated-hadoopProperties')">
@@ -115,10 +159,14 @@ private[ui] class EnvironmentPage(
     UIUtils.headerSparkPage(request, "Environment", content, parent)
   }
 
+  private def resourceProfileHeader = Seq("Resource Profile Id", "Resource Profile Contents")
   private def propertyHeader = Seq("Name", "Value")
   private def classPathHeader = Seq("Resource", "Source")
   private def headerClasses = Seq("sorttable_alpha", "sorttable_alpha")
+  private def headerClassesNoSortValues = Seq("sorttable_numeric", "sorttable_nosort")
 
+  private def jvmRowDataPre(kv: (String, String)) =
+    <tr><td>{kv._1}</td><td><pre>{kv._2}</pre></td></tr>
   private def jvmRow(kv: (String, String)) = <tr><td>{kv._1}</td><td>{kv._2}</td></tr>
   private def propertyRow(kv: (String, String)) = <tr><td>{kv._1}</td><td>{kv._2}</td></tr>
   private def classPathRow(data: (String, String)) = <tr><td>{data._1}</td><td>{data._2}</td></tr>
diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala
index 2c7aeeabb3601..c3246dc90976c 100644
--- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala
@@ -41,10 +41,10 @@ private[ui] class ExecutorThreadDumpPage(
       val dumpRows = threadDump.map { thread =>
         val threadId = thread.threadId
         val blockedBy = thread.blockedByThreadId match {
-          case Some(_) =>
+          case Some(blockingThreadId) =>
             <div>
-              Blocked by <a href={s"#${thread.blockedByThreadId}_td_id"}>
-              Thread {thread.blockedByThreadId} {thread.blockedByLock}</a>
+              Blocked by <a href={s"#${blockingThreadId}_td_id"}>
+              Thread {blockingThreadId} {thread.blockedByLock}</a>
             </div>
           case None => Text("")
         }
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala
index 0b362201a7846..ae0e4728a9edf 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala
@@ -19,10 +19,9 @@ package org.apache.spark.ui.jobs
 
 import java.net.URLEncoder
 import java.nio.charset.StandardCharsets.UTF_8
-import java.util.Date
+import java.util.{Date, Locale}
 import javax.servlet.http.HttpServletRequest
 
-import scala.collection.JavaConverters._
 import scala.collection.mutable.ListBuffer
 import scala.xml._
 
@@ -30,6 +29,7 @@ import org.apache.commons.text.StringEscapeUtils
 
 import org.apache.spark.JobExecutionStatus
 import org.apache.spark.internal.config.SCHEDULER_MODE
+import org.apache.spark.internal.config.UI._
 import org.apache.spark.scheduler._
 import org.apache.spark.status.AppStatusStore
 import org.apache.spark.status.api.v1
@@ -41,6 +41,9 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We
 
   import ApiHelper._
 
+  private val MAX_TIMELINE_JOBS = parent.conf.get(UI_TIMELINE_JOBS_MAXIMUM)
+  private val MAX_TIMELINE_EXECUTORS = parent.conf.get(UI_TIMELINE_EXECUTORS_MAXIMUM)
+
   private val JOBS_LEGEND =
     <div class="legend-area"><svg width="150px" height="85px">
       <rect class="succeeded-job-legend"
@@ -65,9 +68,12 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We
     </svg></div>.toString.filter(_ != '\n')
 
   private def makeJobEvent(jobs: Seq[v1.JobData]): Seq[String] = {
+    val now = System.currentTimeMillis()
     jobs.filter { job =>
       job.status != JobExecutionStatus.UNKNOWN && job.submissionTime.isDefined
-    }.map { job =>
+    }.sortBy { j =>
+      (j.completionTime.map(_.getTime).getOrElse(now), j.submissionTime.get.getTime)
+    }.takeRight(MAX_TIMELINE_JOBS).map { job =>
       val jobId = job.jobId
       val status = job.status
       val (_, lastStageDescription) = lastStageNameAndDescription(store, job)
@@ -77,7 +83,7 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We
         plainText = true).text
 
       val submissionTime = job.submissionTime.get.getTime()
-      val completionTime = job.completionTime.map(_.getTime()).getOrElse(System.currentTimeMillis())
+      val completionTime = job.completionTime.map(_.getTime()).getOrElse(now)
       val classNameByStatus = status match {
         case JobExecutionStatus.SUCCEEDED => "succeeded"
         case JobExecutionStatus.FAILED => "failed"
@@ -86,7 +92,7 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We
       }
 
       // The timeline library treats contents as HTML, so we have to escape them. We need to add
-      // extra layers of escaping in order to embed this in a Javascript string literal.
+      // extra layers of escaping in order to embed this in a JavaScript string literal.
       val escapedDesc = Utility.escape(jobDescription)
       val jsEscapedDescForTooltip = StringEscapeUtils.escapeEcmaScript(Utility.escape(escapedDesc))
       val jsEscapedDescForLabel = StringEscapeUtils.escapeEcmaScript(escapedDesc)
@@ -119,7 +125,9 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We
   private def makeExecutorEvent(executors: Seq[v1.ExecutorSummary]):
       Seq[String] = {
     val events = ListBuffer[String]()
-    executors.foreach { e =>
+    executors.sortBy { e =>
+      e.removeTime.map(_.getTime).getOrElse(e.addTime.getTime)
+    }.takeRight(MAX_TIMELINE_EXECUTORS).foreach { e =>
       val addedEvent =
         s"""
            |{
@@ -148,7 +156,8 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We
              |    'Removed at ${UIUtils.formatDate(removeTime)}' +
              |    '${
                       e.removeReason.map { reason =>
-                        s"""<br>Reason: ${reason.replace("\n", " ")}"""
+                        s"""<br>Reason: ${StringEscapeUtils.escapeEcmaScript(
+                          reason.replace("\n", " "))}"""
                       }.getOrElse("")
                    }"' +
              |    'data-html="true">Executor ${e.id} removed</div>'
@@ -192,6 +201,30 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We
       </a>
     </span> ++
     <div id="application-timeline" class="collapsed">
+      {
+        if (MAX_TIMELINE_JOBS < jobs.size) {
+          <div>
+            <strong>
+              Only the most recent {MAX_TIMELINE_JOBS} submitted/completed jobs
+              (of {jobs.size} total) are shown.
+            </strong>
+          </div>
+        } else {
+          Seq.empty
+        }
+      }
+      {
+        if (MAX_TIMELINE_EXECUTORS < executors.size) {
+          <div>
+            <strong>
+              Only the most recent {MAX_TIMELINE_EXECUTORS} added/removed executors
+              (of {executors.size} total) are shown.
+            </strong>
+          </div>
+        } else {
+          Seq.empty
+        }
+      }
       <div class="control-panel">
         <div id="application-timeline-zoom-lock">
           <input type="checkbox"></input>
@@ -211,45 +244,22 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We
       jobTag: String,
       jobs: Seq[v1.JobData],
       killEnabled: Boolean): Seq[Node] = {
-    val parameterOtherTable = request.getParameterMap().asScala
-      .filterNot(_._1.startsWith(jobTag))
-      .map(para => para._1 + "=" + para._2(0))
 
     val someJobHasJobGroup = jobs.exists(_.jobGroup.isDefined)
     val jobIdTitle = if (someJobHasJobGroup) "Job Id (Job Group)" else "Job Id"
-
-    val parameterJobPage = request.getParameter(jobTag + ".page")
-    val parameterJobSortColumn = request.getParameter(jobTag + ".sort")
-    val parameterJobSortDesc = request.getParameter(jobTag + ".desc")
-    val parameterJobPageSize = request.getParameter(jobTag + ".pageSize")
-
-    val jobPage = Option(parameterJobPage).map(_.toInt).getOrElse(1)
-    val jobSortColumn = Option(parameterJobSortColumn).map { sortColumn =>
-      UIUtils.decodeURLParameter(sortColumn)
-    }.getOrElse(jobIdTitle)
-    val jobSortDesc = Option(parameterJobSortDesc).map(_.toBoolean).getOrElse(
-      // New jobs should be shown above old jobs by default.
-      jobSortColumn == jobIdTitle
-    )
-    val jobPageSize = Option(parameterJobPageSize).map(_.toInt).getOrElse(100)
-
-    val currentTime = System.currentTimeMillis()
+    val jobPage = Option(request.getParameter(jobTag + ".page")).map(_.toInt).getOrElse(1)
 
     try {
       new JobPagedTable(
+        request,
         store,
         jobs,
         tableHeaderId,
         jobTag,
         UIUtils.prependBaseUri(request, parent.basePath),
         "jobs", // subPath
-        parameterOtherTable,
         killEnabled,
-        currentTime,
-        jobIdTitle,
-        pageSize = jobPageSize,
-        sortColumn = jobSortColumn,
-        desc = jobSortDesc
+        jobIdTitle
       ).table(jobPage)
     } catch {
       case e @ (_ : IllegalArgumentException | _ : IndexOutOfBoundsException) =>
@@ -283,11 +293,11 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We
     }
 
     val activeJobsTable =
-      jobsTable(request, "active", "activeJob", activeJobs, killEnabled = parent.killEnabled)
+      jobsTable(request, "active", "activeJob", activeJobs.toSeq, killEnabled = parent.killEnabled)
     val completedJobsTable =
-      jobsTable(request, "completed", "completedJob", completedJobs, killEnabled = false)
+      jobsTable(request, "completed", "completedJob", completedJobs.toSeq, killEnabled = false)
     val failedJobsTable =
-      jobsTable(request, "failed", "failedJob", failedJobs, killEnabled = false)
+      jobsTable(request, "failed", "failedJob", failedJobs.toSeq, killEnabled = false)
 
     val shouldShowActiveJobs = activeJobs.nonEmpty
     val shouldShowCompletedJobs = completedJobs.nonEmpty
@@ -300,15 +310,17 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We
       s"${appSummary.numCompletedJobs}, only showing ${completedJobs.size}"
     }
 
+    // SPARK-33991 Avoid enumeration conversion error.
     val schedulingMode = store.environmentInfo().sparkProperties.toMap
       .get(SCHEDULER_MODE.key)
-      .map { mode => SchedulingMode.withName(mode).toString }
+      .map { mode => SchedulingMode.withName(mode.toUpperCase(Locale.ROOT)).toString }
       .getOrElse("Unknown")
 
     val summary: NodeSeq =
       <div>
         <ul class="list-unstyled">
           <li>
+
             <strong>User:</strong>
             {parent.getSparkUser}
           </li>
@@ -354,7 +366,7 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We
       </div>
 
     var content = summary
-    content ++= makeTimeline(activeJobs ++ completedJobs ++ failedJobs,
+    content ++= makeTimeline((activeJobs ++ completedJobs ++ failedJobs).toSeq,
       store.executorList(false), startTime)
 
     if (shouldShowActiveJobs) {
@@ -421,7 +433,6 @@ private[ui] class JobDataSource(
     store: AppStatusStore,
     jobs: Seq[v1.JobData],
     basePath: String,
-    currentTime: Long,
     pageSize: Int,
     sortColumn: String,
     desc: Boolean) extends PagedDataSource[JobTableRowData](pageSize) {
@@ -432,15 +443,9 @@ private[ui] class JobDataSource(
   // so that we can avoid creating duplicate contents during sorting the data
   private val data = jobs.map(jobRow).sorted(ordering(sortColumn, desc))
 
-  private var _slicedJobIds: Set[Int] = null
-
   override def dataSize: Int = data.size
 
-  override def sliceData(from: Int, to: Int): Seq[JobTableRowData] = {
-    val r = data.slice(from, to)
-    _slicedJobIds = r.map(_.jobData.jobId).toSet
-    r
-  }
+  override def sliceData(from: Int, to: Int): Seq[JobTableRowData] = data.slice(from, to)
 
   private def jobRow(jobData: v1.JobData): JobTableRowData = {
     val duration: Option[Long] = JobDataUtil.getDuration(jobData)
@@ -493,27 +498,25 @@ private[ui] class JobDataSource(
 }
 
 private[ui] class JobPagedTable(
+    request: HttpServletRequest,
     store: AppStatusStore,
     data: Seq[v1.JobData],
     tableHeaderId: String,
     jobTag: String,
     basePath: String,
     subPath: String,
-    parameterOtherTable: Iterable[String],
     killEnabled: Boolean,
-    currentTime: Long,
-    jobIdTitle: String,
-    pageSize: Int,
-    sortColumn: String,
-    desc: Boolean
+    jobIdTitle: String
   ) extends PagedTable[JobTableRowData] {
-  val parameterPath = basePath + s"/$subPath/?" + parameterOtherTable.mkString("&")
+
+  private val (sortColumn, desc, pageSize) = getTableParameters(request, jobTag, jobIdTitle)
+  private val parameterPath = basePath + s"/$subPath/?" + getParameterOtherTable(request, jobTag)
+  private val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
 
   override def tableId: String = jobTag + "-table"
 
   override def tableCssClass: String =
-    "table table-bordered table-sm table-striped " +
-      "table-head-clickable table-cell-width-limited"
+    "table table-bordered table-sm table-striped table-head-clickable table-cell-width-limited"
 
   override def pageSizeFormField: String = jobTag + ".pageSize"
 
@@ -523,13 +526,11 @@ private[ui] class JobPagedTable(
     store,
     data,
     basePath,
-    currentTime,
     pageSize,
     sortColumn,
     desc)
 
   override def pageLink(page: Int): String = {
-    val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
     parameterPath +
       s"&$pageNumberFormField=$page" +
       s"&$jobTag.sort=$encodedSortColumn" +
@@ -538,96 +539,26 @@ private[ui] class JobPagedTable(
       s"#$tableHeaderId"
   }
 
-  override def goButtonFormPath: String = {
-    val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
+  override def goButtonFormPath: String =
     s"$parameterPath&$jobTag.sort=$encodedSortColumn&$jobTag.desc=$desc#$tableHeaderId"
-  }
 
   override def headers: Seq[Node] = {
-    // Information for each header: title, cssClass, and sortable
-    val jobHeadersAndCssClasses: Seq[(String, String, Boolean, Option[String])] =
+    // Information for each header: title, sortable, tooltip
+    val jobHeadersAndCssClasses: Seq[(String, Boolean, Option[String])] =
       Seq(
-        (jobIdTitle, "", true, None),
-        ("Description", "", true, None),
-        ("Submitted", "", true, None),
-        ("Duration", "", true, Some("Elapsed time since the job was submitted " +
+        (jobIdTitle, true, None),
+        ("Description", true, None),
+        ("Submitted", true, None),
+        ("Duration", true, Some("Elapsed time since the job was submitted " +
           "until execution completion of all its stages.")),
-        ("Stages: Succeeded/Total", "", false, None),
-        ("Tasks (for all stages): Succeeded/Total", "", false, None)
+        ("Stages: Succeeded/Total", false, None),
+        ("Tasks (for all stages): Succeeded/Total", false, None)
       )
 
-    if (!jobHeadersAndCssClasses.filter(_._3).map(_._1).contains(sortColumn)) {
-      throw new IllegalArgumentException(s"Unknown column: $sortColumn")
-    }
+    isSortColumnValid(jobHeadersAndCssClasses, sortColumn)
 
-    val headerRow: Seq[Node] = {
-      jobHeadersAndCssClasses.map { case (header, cssClass, sortable, tooltip) =>
-        if (header == sortColumn) {
-          val headerLink = Unparsed(
-            parameterPath +
-              s"&$jobTag.sort=${URLEncoder.encode(header, UTF_8.name())}" +
-              s"&$jobTag.desc=${!desc}" +
-              s"&$jobTag.pageSize=$pageSize" +
-              s"#$tableHeaderId")
-          val arrow = if (desc) "&#x25BE;" else "&#x25B4;" // UP or DOWN
-
-          <th class={cssClass}>
-            <a href={headerLink}>
-              {
-                if (tooltip.nonEmpty) {
-                  <span data-toggle="tooltip" data-placement="top" title={tooltip.get}>
-                    {header}&nbsp;{Unparsed(arrow)}
-                  </span>
-                } else {
-                  <span>
-                    {header}&nbsp;{Unparsed(arrow)}
-                  </span>
-                }
-              }
-            </a>
-          </th>
-        } else {
-          if (sortable) {
-            val headerLink = Unparsed(
-              parameterPath +
-                s"&$jobTag.sort=${URLEncoder.encode(header, UTF_8.name())}" +
-                s"&$jobTag.pageSize=$pageSize" +
-                s"#$tableHeaderId")
-
-            <th class={cssClass}>
-              <a href={headerLink}>
-                {
-                  if (tooltip.nonEmpty) {
-                    <span data-toggle="tooltip" data-placement="top" title={tooltip.get}>
-                      {header}
-                    </span>
-                  } else {
-                    <span>
-                      {header}
-                    </span>
-                  }
-                }
-               </a>
-            </th>
-          } else {
-            <th class={cssClass}>
-              {
-                if (tooltip.nonEmpty) {
-                  <span data-toggle="tooltip" data-placement="top" title={tooltip.get}>
-                    {header}
-                  </span>
-                } else {
-                  <span>
-                    {header}
-                  </span>
-                }
-              }
-            </th>
-          }
-        }
-      }
-    }
-    <thead><tr>{headerRow}</tr></thead>
+    headerRow(jobHeadersAndCssClasses, desc, pageSize, sortColumn, parameterPath,
+      jobTag, tableHeaderId)
   }
 
   override def row(jobTableRow: JobTableRowData): Seq[Node] = {
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala
index 9be7124adcf7b..08e86524b0821 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala
@@ -26,6 +26,8 @@ import scala.xml.{Node, NodeSeq, Unparsed, Utility}
 import org.apache.commons.text.StringEscapeUtils
 
 import org.apache.spark.JobExecutionStatus
+import org.apache.spark.internal.config.UI._
+import org.apache.spark.resource.ResourceProfile
 import org.apache.spark.status.AppStatusStore
 import org.apache.spark.status.api.v1
 import org.apache.spark.ui._
@@ -33,6 +35,9 @@ import org.apache.spark.ui._
 /** Page showing statistics and stage list for a given job */
 private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIPage("job") {
 
+  private val MAX_TIMELINE_STAGES = parent.conf.get(UI_TIMELINE_STAGES_MAXIMUM)
+  private val MAX_TIMELINE_EXECUTORS = parent.conf.get(UI_TIMELINE_EXECUTORS_MAXIMUM)
+
   private val STAGES_LEGEND =
     <div class="legend-area"><svg width="150px" height="85px">
       <rect class="completed-stage-legend"
@@ -57,17 +62,20 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP
     </svg></div>.toString.filter(_ != '\n')
 
   private def makeStageEvent(stageInfos: Seq[v1.StageData]): Seq[String] = {
-    stageInfos.map { stage =>
+    val now = System.currentTimeMillis()
+    stageInfos.sortBy { s =>
+      (s.completionTime.map(_.getTime).getOrElse(now), s.submissionTime.get.getTime)
+    }.takeRight(MAX_TIMELINE_STAGES).map { stage =>
       val stageId = stage.stageId
       val attemptId = stage.attemptId
       val name = stage.name
       val status = stage.status.toString.toLowerCase(Locale.ROOT)
       val submissionTime = stage.submissionTime.get.getTime()
       val completionTime = stage.completionTime.map(_.getTime())
-        .getOrElse(System.currentTimeMillis())
+        .getOrElse(now)
 
       // The timeline library treats contents as HTML, so we have to escape them. We need to add
-      // extra layers of escaping in order to embed this in a Javascript string literal.
+      // extra layers of escaping in order to embed this in a JavaScript string literal.
       val escapedName = Utility.escape(name)
       val jsEscapedNameForTooltip = StringEscapeUtils.escapeEcmaScript(Utility.escape(escapedName))
       val jsEscapedNameForLabel = StringEscapeUtils.escapeEcmaScript(escapedName)
@@ -97,7 +105,9 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP
 
   def makeExecutorEvent(executors: Seq[v1.ExecutorSummary]): Seq[String] = {
     val events = ListBuffer[String]()
-    executors.foreach { e =>
+    executors.sortBy { e =>
+      e.removeTime.map(_.getTime).getOrElse(e.addTime.getTime)
+    }.takeRight(MAX_TIMELINE_EXECUTORS).foreach { e =>
       val addedEvent =
         s"""
            |{
@@ -126,7 +136,8 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP
              |    'Removed at ${UIUtils.formatDate(removeTime)}' +
              |    '${
                       e.removeReason.map { reason =>
-                        s"""<br>Reason: ${reason.replace("\n", " ")}"""
+                        s"""<br>Reason: ${StringEscapeUtils.escapeEcmaScript(
+                          reason.replace("\n", " "))}"""
                       }.getOrElse("")
                    }"' +
              |    'data-html="true">Executor ${e.id} removed</div>'
@@ -170,6 +181,30 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP
       </a>
     </span> ++
     <div id="job-timeline" class="collapsed">
+      {
+      if (MAX_TIMELINE_STAGES < stages.size) {
+        <div>
+          <strong>
+            Only the most recent {MAX_TIMELINE_STAGES} submitted/completed stages
+            (of {stages.size} total) are shown.
+          </strong>
+        </div>
+      } else {
+        Seq.empty
+      }
+      }
+      {
+      if (MAX_TIMELINE_EXECUTORS < executors.size) {
+        <div>
+          <strong>
+            Only the most recent {MAX_TIMELINE_EXECUTORS} added/removed executors
+            (of {executors.size} total) are shown.
+          </strong>
+        </div>
+      } else {
+        Seq.empty
+      }
+      }
       <div class="control-panel">
         <div id="job-timeline-zoom-lock">
           <input type="checkbox"></input>
@@ -253,7 +288,11 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP
           accumulatorUpdates = Nil,
           tasks = None,
           executorSummary = None,
-          killedTasksSummary = Map())
+          killedTasksSummary = Map(),
+          ResourceProfile.UNKNOWN_RESOURCE_PROFILE_ID,
+          peakExecutorMetrics = None,
+          taskMetricsDistributions = None,
+          executorMetricsDistributions = None)
       }
     }
 
@@ -280,26 +319,26 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP
 
     val pendingOrSkippedTableId =
       if (isComplete) {
-        "pending"
-      } else {
         "skipped"
+      } else {
+        "pending"
       }
 
     val activeStagesTable =
-      new StageTableBase(store, request, activeStages, "active", "activeStage", parent.basePath,
-        basePath, parent.isFairScheduler,
+      new StageTableBase(store, request, activeStages.toSeq, "active", "activeStage",
+        parent.basePath, basePath, parent.isFairScheduler,
         killEnabled = parent.killEnabled, isFailedStage = false)
     val pendingOrSkippedStagesTable =
-      new StageTableBase(store, request, pendingOrSkippedStages, pendingOrSkippedTableId,
+      new StageTableBase(store, request, pendingOrSkippedStages.toSeq, pendingOrSkippedTableId,
         "pendingStage", parent.basePath, basePath, parent.isFairScheduler,
         killEnabled = false, isFailedStage = false)
     val completedStagesTable =
-      new StageTableBase(store, request, completedStages, "completed", "completedStage",
+      new StageTableBase(store, request, completedStages.toSeq, "completed", "completedStage",
         parent.basePath, basePath, parent.isFairScheduler,
         killEnabled = false, isFailedStage = false)
     val failedStagesTable =
-      new StageTableBase(store, request, failedStages, "failed", "failedStage", parent.basePath,
-        basePath, parent.isFairScheduler,
+      new StageTableBase(store, request, failedStages.toSeq, "failed", "failedStage",
+        parent.basePath, basePath, parent.isFairScheduler,
         killEnabled = false, isFailedStage = true)
 
     val shouldShowActiveStages = activeStages.nonEmpty
@@ -389,7 +428,7 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP
     var content = summary
     val appStartTime = store.applicationInfo().attempts.head.startTime.getTime()
 
-    content ++= makeTimeline(activeStages ++ completedStages ++ failedStages,
+    content ++= makeTimeline((activeStages ++ completedStages ++ failedStages).toSeq,
       store.executorList(false), appStartTime)
 
     val operationGraphContent = store.asOption(store.operationGraphForJob(jobId)) match {
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobsTab.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobsTab.scala
index dc3106400dd2b..f5a2721ca4f51 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/JobsTab.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobsTab.scala
@@ -30,6 +30,7 @@ private[ui] class JobsTab(parent: SparkUI, store: AppStatusStore)
   extends SparkUITab(parent, "jobs") {
 
   val sc = parent.sc
+  val conf = parent.conf
   val killEnabled = parent.killEnabled
 
   // Show pool information for only live UI.
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
index 7973d30493a5a..777a6b014ed29 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
@@ -142,6 +142,10 @@ private[ui] class StagePage(parent: StagesTab, store: AppStatusStore) extends We
     val summary =
       <div>
         <ul class="list-unstyled">
+          <li>
+            <strong>Resource Profile Id: </strong>
+            {stageData.resourceProfileId}
+          </li>
           <li>
             <strong>Total Time Across All Tasks: </strong>
             {UIUtils.formatDuration(stageData.executorRunTime)}
@@ -208,7 +212,6 @@ private[ui] class StagePage(parent: StagesTab, store: AppStatusStore) extends We
         stageData,
         UIUtils.prependBaseUri(request, parent.basePath) +
           s"/stages/stage/?id=${stageId}&attempt=${stageAttemptId}",
-        currentTime,
         pageSize = taskPageSize,
         sortColumn = taskSortColumn,
         desc = taskSortDesc,
@@ -395,8 +398,7 @@ private[ui] class StagePage(parent: StagesTab, store: AppStatusStore) extends We
       {
         if (MAX_TIMELINE_TASKS < tasks.size) {
           <strong>
-            This page has more than the maximum number of tasks that can be shown in the
-            visualization! Only the most recent {MAX_TIMELINE_TASKS} tasks
+            Only the most recent {MAX_TIMELINE_TASKS} tasks
             (of {tasks.size} total) are shown.
           </strong>
         } else {
@@ -448,7 +450,6 @@ private[ui] class StagePage(parent: StagesTab, store: AppStatusStore) extends We
 
 private[ui] class TaskDataSource(
     stage: StageData,
-    currentTime: Long,
     pageSize: Int,
     sortColumn: String,
     desc: Boolean,
@@ -470,8 +471,6 @@ private[ui] class TaskDataSource(
     _tasksToShow
   }
 
-  def tasks: Seq[TaskData] = _tasksToShow
-
   def executorLogs(id: String): Map[String, String] = {
     executorIdToLogs.getOrElseUpdate(id,
       store.asOption(store.executorSummary(id)).map(_.executorLogs).getOrElse(Map.empty))
@@ -482,7 +481,6 @@ private[ui] class TaskDataSource(
 private[ui] class TaskPagedTable(
     stage: StageData,
     basePath: String,
-    currentTime: Long,
     pageSize: Int,
     sortColumn: String,
     desc: Boolean,
@@ -490,6 +488,8 @@ private[ui] class TaskPagedTable(
 
   import ApiHelper._
 
+  private val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
+
   override def tableId: String = "task-table"
 
   override def tableCssClass: String =
@@ -501,14 +501,12 @@ private[ui] class TaskPagedTable(
 
   override val dataSource: TaskDataSource = new TaskDataSource(
     stage,
-    currentTime,
     pageSize,
     sortColumn,
     desc,
     store)
 
   override def pageLink(page: Int): String = {
-    val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
     basePath +
       s"&$pageNumberFormField=$page" +
       s"&task.sort=$encodedSortColumn" +
@@ -516,10 +514,7 @@ private[ui] class TaskPagedTable(
       s"&$pageSizeFormField=$pageSize"
   }
 
-  override def goButtonFormPath: String = {
-    val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
-    s"$basePath&task.sort=$encodedSortColumn&task.desc=$desc"
-  }
+  override def goButtonFormPath: String = s"$basePath&task.sort=$encodedSortColumn&task.desc=$desc"
 
   def headers: Seq[Node] = {
     import ApiHelper._
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
index a29483b5d5a5e..9e6eb418fe134 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
@@ -22,7 +22,6 @@ import java.nio.charset.StandardCharsets.UTF_8
 import java.util.Date
 import javax.servlet.http.HttpServletRequest
 
-import scala.collection.JavaConverters._
 import scala.xml._
 
 import org.apache.commons.text.StringEscapeUtils
@@ -43,24 +42,8 @@ private[ui] class StageTableBase(
     isFairScheduler: Boolean,
     killEnabled: Boolean,
     isFailedStage: Boolean) {
-  val parameterOtherTable = request.getParameterMap().asScala
-    .filterNot(_._1.startsWith(stageTag))
-    .map(para => para._1 + "=" + para._2(0))
-
-  val parameterStagePage = request.getParameter(stageTag + ".page")
-  val parameterStageSortColumn = request.getParameter(stageTag + ".sort")
-  val parameterStageSortDesc = request.getParameter(stageTag + ".desc")
-  val parameterStagePageSize = request.getParameter(stageTag + ".pageSize")
-
-  val stagePage = Option(parameterStagePage).map(_.toInt).getOrElse(1)
-  val stageSortColumn = Option(parameterStageSortColumn).map { sortColumn =>
-    UIUtils.decodeURLParameter(sortColumn)
-  }.getOrElse("Stage Id")
-  val stageSortDesc = Option(parameterStageSortDesc).map(_.toBoolean).getOrElse(
-    // New stages should be shown above old jobs by default.
-    stageSortColumn == "Stage Id"
-  )
-  val stagePageSize = Option(parameterStagePageSize).map(_.toInt).getOrElse(100)
+
+  val stagePage = Option(request.getParameter(stageTag + ".page")).map(_.toInt).getOrElse(1)
 
   val currentTime = System.currentTimeMillis()
 
@@ -75,11 +58,7 @@ private[ui] class StageTableBase(
       isFairScheduler,
       killEnabled,
       currentTime,
-      stagePageSize,
-      stageSortColumn,
-      stageSortDesc,
       isFailedStage,
-      parameterOtherTable,
       request
     ).table(stagePage)
   } catch {
@@ -131,25 +110,24 @@ private[ui] class StagePagedTable(
     isFairScheduler: Boolean,
     killEnabled: Boolean,
     currentTime: Long,
-    pageSize: Int,
-    sortColumn: String,
-    desc: Boolean,
     isFailedStage: Boolean,
-    parameterOtherTable: Iterable[String],
     request: HttpServletRequest) extends PagedTable[StageTableRowData] {
 
   override def tableId: String = stageTag + "-table"
 
   override def tableCssClass: String =
-    "table table-bordered table-sm table-striped " +
-      "table-head-clickable table-cell-width-limited"
+    "table table-bordered table-sm table-striped table-head-clickable table-cell-width-limited"
 
   override def pageSizeFormField: String = stageTag + ".pageSize"
 
   override def pageNumberFormField: String = stageTag + ".page"
 
-  val parameterPath = UIUtils.prependBaseUri(request, basePath) + s"/$subPath/?" +
-    parameterOtherTable.mkString("&")
+  private val (sortColumn, desc, pageSize) = getTableParameters(request, stageTag, "Stage Id")
+
+  private val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
+
+  private val parameterPath = UIUtils.prependBaseUri(request, basePath) + s"/$subPath/?" +
+    getParameterOtherTable(request, stageTag)
 
   override val dataSource = new StageDataSource(
     store,
@@ -161,7 +139,6 @@ private[ui] class StagePagedTable(
   )
 
   override def pageLink(page: Int): String = {
-    val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
     parameterPath +
       s"&$pageNumberFormField=$page" +
       s"&$stageTag.sort=$encodedSortColumn" +
@@ -170,82 +147,31 @@ private[ui] class StagePagedTable(
       s"#$tableHeaderId"
   }
 
-  override def goButtonFormPath: String = {
-    val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
+  override def goButtonFormPath: String =
     s"$parameterPath&$stageTag.sort=$encodedSortColumn&$stageTag.desc=$desc#$tableHeaderId"
-  }
 
   override def headers: Seq[Node] = {
-    // stageHeadersAndCssClasses has three parts: header title, tooltip information, and sortable.
+    // stageHeadersAndCssClasses has three parts: header title, sortable and tooltip information.
     // The tooltip information could be None, which indicates it does not have a tooltip.
-    // Otherwise, it has two parts: tooltip text, and position (true for left, false for default).
-    val stageHeadersAndCssClasses: Seq[(String, String, Boolean)] =
-      Seq(("Stage Id", null, true)) ++
-      {if (isFairScheduler) {Seq(("Pool Name", null, true))} else Seq.empty} ++
+    val stageHeadersAndCssClasses: Seq[(String, Boolean, Option[String])] =
+      Seq(("Stage Id", true, None)) ++
+      {if (isFairScheduler) {Seq(("Pool Name", true, None))} else Seq.empty} ++
       Seq(
-        ("Description", null, true),
-        ("Submitted", null, true),
-        ("Duration", ToolTips.DURATION, true),
-        ("Tasks: Succeeded/Total", null, false),
-        ("Input", ToolTips.INPUT, true),
-        ("Output", ToolTips.OUTPUT, true),
-        ("Shuffle Read", ToolTips.SHUFFLE_READ, true),
-        ("Shuffle Write", ToolTips.SHUFFLE_WRITE, true)
+        ("Description", true, None),
+        ("Submitted", true, None),
+        ("Duration", true, Some(ToolTips.DURATION)),
+        ("Tasks: Succeeded/Total", false, None),
+        ("Input", true, Some(ToolTips.INPUT)),
+        ("Output", true, Some(ToolTips.OUTPUT)),
+        ("Shuffle Read", true, Some(ToolTips.SHUFFLE_READ)),
+        ("Shuffle Write", true, Some(ToolTips.SHUFFLE_WRITE))
       ) ++
-      {if (isFailedStage) {Seq(("Failure Reason", null, false))} else Seq.empty}
-
-    if (!stageHeadersAndCssClasses.filter(_._3).map(_._1).contains(sortColumn)) {
-      throw new IllegalArgumentException(s"Unknown column: $sortColumn")
-    }
+      {if (isFailedStage) {Seq(("Failure Reason", false, None))} else Seq.empty}
 
-    val headerRow: Seq[Node] = {
-      stageHeadersAndCssClasses.map { case (header, tooltip, sortable) =>
-        val headerSpan = if (null != tooltip && !tooltip.isEmpty) {
-            <span data-toggle="tooltip" data-placement="top" title={tooltip}>
-              {header}
-            </span>
-        } else {
-          {header}
-        }
+    isSortColumnValid(stageHeadersAndCssClasses, sortColumn)
 
-        if (header == sortColumn) {
-          val headerLink = Unparsed(
-            parameterPath +
-              s"&$stageTag.sort=${URLEncoder.encode(header, UTF_8.name())}" +
-              s"&$stageTag.desc=${!desc}" +
-              s"&$stageTag.pageSize=$pageSize") +
-              s"#$tableHeaderId"
-          val arrow = if (desc) "&#x25BE;" else "&#x25B4;" // UP or DOWN
-
-          <th>
-            <a href={headerLink}>
-              {headerSpan}<span>
-              &nbsp;{Unparsed(arrow)}
-            </span>
-            </a>
-          </th>
-        } else {
-          if (sortable) {
-            val headerLink = Unparsed(
-              parameterPath +
-                s"&$stageTag.sort=${URLEncoder.encode(header, UTF_8.name())}" +
-                s"&$stageTag.pageSize=$pageSize") +
-                s"#$tableHeaderId"
-
-            <th>
-              <a href={headerLink}>
-                {headerSpan}
-              </a>
-            </th>
-          } else {
-            <th>
-              {headerSpan}
-            </th>
-          }
-        }
-      }
-    }
-    <thead>{headerRow}</thead>
+    headerRow(stageHeadersAndCssClasses, desc, pageSize, sortColumn, parameterPath,
+      stageTag, tableHeaderId)
   }
 
   override def row(data: StageTableRowData): Seq[Node] = {
@@ -383,15 +309,9 @@ private[ui] class StageDataSource(
   // table so that we can avoid creating duplicate contents during sorting the data
   private val data = stages.map(stageRow).sorted(ordering(sortColumn, desc))
 
-  private var _slicedStageIds: Set[Int] = _
-
   override def dataSize: Int = data.size
 
-  override def sliceData(from: Int, to: Int): Seq[StageTableRowData] = {
-    val r = data.slice(from, to)
-    _slicedStageIds = r.map(_.stageId).toSet
-    r
-  }
+  override def sliceData(from: Int, to: Int): Seq[StageTableRowData] = data.slice(from, to)
 
   private def stageRow(stageData: v1.StageData): StageTableRowData = {
     val formattedSubmissionTime = stageData.submissionTime match {
@@ -422,7 +342,6 @@ private[ui] class StageDataSource(
     val shuffleWrite = stageData.shuffleWriteBytes
     val shuffleWriteWithUnit = if (shuffleWrite > 0) Utils.bytesToString(shuffleWrite) else ""
 
-
     new StageTableRowData(
       stageData,
       Some(stageData),
diff --git a/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala b/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala
index 842ee7aaf49bf..e1f7609dfc7f8 100644
--- a/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala
+++ b/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala
@@ -26,6 +26,7 @@ import scala.xml.Utility
 import org.apache.commons.text.StringEscapeUtils
 
 import org.apache.spark.internal.Logging
+import org.apache.spark.rdd.DeterministicLevel
 import org.apache.spark.scheduler.StageInfo
 import org.apache.spark.storage.StorageLevel
 
@@ -48,7 +49,8 @@ private[spark] case class RDDOperationNode(
     name: String,
     cached: Boolean,
     barrier: Boolean,
-    callsite: String)
+    callsite: String,
+    outputDeterministicLevel: DeterministicLevel.Value)
 
 /**
  * A directed edge connecting two nodes in an RDDOperationGraph.
@@ -81,11 +83,16 @@ private[spark] class RDDOperationCluster(
 
   /** Return all the nodes which are cached. */
   def getCachedNodes: Seq[RDDOperationNode] = {
-    _childNodes.filter(_.cached) ++ _childClusters.flatMap(_.getCachedNodes)
+    (_childNodes.filter(_.cached) ++ _childClusters.flatMap(_.getCachedNodes)).toSeq
   }
 
   def getBarrierClusters: Seq[RDDOperationCluster] = {
-    _childClusters.filter(_.barrier) ++ _childClusters.flatMap(_.getBarrierClusters)
+    (_childClusters.filter(_.barrier) ++ _childClusters.flatMap(_.getBarrierClusters)).toSeq
+  }
+
+  def getIndeterminateNodes: Seq[RDDOperationNode] = {
+    (_childNodes.filter(_.outputDeterministicLevel == DeterministicLevel.INDETERMINATE) ++
+      _childClusters.flatMap(_.getIndeterminateNodes)).toSeq
   }
 
   def canEqual(other: Any): Boolean = other.isInstanceOf[RDDOperationCluster]
@@ -156,7 +163,8 @@ private[spark] object RDDOperationGraph extends Logging {
 
       // TODO: differentiate between the intention to cache an RDD and whether it's actually cached
       val node = nodes.getOrElseUpdate(rdd.id, RDDOperationNode(
-        rdd.id, rdd.name, rdd.storageLevel != StorageLevel.NONE, rdd.isBarrier, rdd.callSite))
+        rdd.id, rdd.name, rdd.storageLevel != StorageLevel.NONE, rdd.isBarrier, rdd.callSite,
+        rdd.outputDeterministicLevel))
       if (rdd.scope.isEmpty) {
         // This RDD has no encompassing scope, so we put it directly in the root cluster
         // This should happen only if an RDD is instantiated outside of a public RDD API
@@ -210,7 +218,7 @@ private[spark] object RDDOperationGraph extends Logging {
       }
     }
 
-    RDDOperationGraph(internalEdges, outgoingEdges, incomingEdges, rootCluster)
+    RDDOperationGraph(internalEdges.toSeq, outgoingEdges.toSeq, incomingEdges.toSeq, rootCluster)
   }
 
   /**
@@ -246,8 +254,14 @@ private[spark] object RDDOperationGraph extends Logging {
     } else {
       ""
     }
+    val outputDeterministicLevel = node.outputDeterministicLevel match {
+      case DeterministicLevel.DETERMINATE => ""
+      case DeterministicLevel.INDETERMINATE => " [Indeterminate]"
+      case DeterministicLevel.UNORDERED => " [Unordered]"
+    }
     val escapedCallsite = Utility.escape(node.callsite)
-    val label = s"${node.name} [${node.id}]$isCached$isBarrier<br>${escapedCallsite}"
+    val label = s"${node.name} [${node.id}]$isCached$isBarrier$outputDeterministicLevel" +
+      s"<br>${escapedCallsite}"
     s"""${node.id} [labelType="html" label="${StringEscapeUtils.escapeJava(label)}"]"""
   }
 
diff --git a/core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala b/core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala
index 04f8d8edd4d50..97f3cf534fb2c 100644
--- a/core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala
@@ -35,15 +35,7 @@ private[ui] class RDDPage(parent: SparkUITab, store: AppStatusStore) extends Web
     val parameterId = request.getParameter("id")
     require(parameterId != null && parameterId.nonEmpty, "Missing id parameter")
 
-    val parameterBlockPage = request.getParameter("block.page")
-    val parameterBlockSortColumn = request.getParameter("block.sort")
-    val parameterBlockSortDesc = request.getParameter("block.desc")
-    val parameterBlockPageSize = request.getParameter("block.pageSize")
-
-    val blockPage = Option(parameterBlockPage).map(_.toInt).getOrElse(1)
-    val blockSortColumn = Option(parameterBlockSortColumn).getOrElse("Block Name")
-    val blockSortDesc = Option(parameterBlockSortDesc).map(_.toBoolean).getOrElse(false)
-    val blockPageSize = Option(parameterBlockPageSize).map(_.toInt).getOrElse(100)
+    val blockPage = Option(request.getParameter("block.page")).map(_.toInt).getOrElse(1)
 
     val rddId = parameterId.toInt
     val rddStorageInfo = try {
@@ -60,11 +52,10 @@ private[ui] class RDDPage(parent: SparkUITab, store: AppStatusStore) extends Web
 
     val blockTableHTML = try {
       val _blockTable = new BlockPagedTable(
+        request,
+        "block",
         UIUtils.prependBaseUri(request, parent.basePath) + s"/storage/rdd/?id=${rddId}",
         rddStorageInfo.partitions.get,
-        blockPageSize,
-        blockSortColumn,
-        blockSortDesc,
         store.executorList(true))
       _blockTable.table(blockPage)
     } catch {
@@ -216,21 +207,22 @@ private[ui] class BlockDataSource(
 }
 
 private[ui] class BlockPagedTable(
+    request: HttpServletRequest,
+    rddTag: String,
     basePath: String,
     rddPartitions: Seq[RDDPartitionInfo],
-    pageSize: Int,
-    sortColumn: String,
-    desc: Boolean,
     executorSummaries: Seq[ExecutorSummary]) extends PagedTable[BlockTableRowData] {
 
+  private val (sortColumn, desc, pageSize) = getTableParameters(request, rddTag, "Block Name")
+
   override def tableId: String = "rdd-storage-by-block-table"
 
   override def tableCssClass: String =
     "table table-bordered table-sm table-striped table-head-clickable"
 
-  override def pageSizeFormField: String = "block.pageSize"
+  override def pageSizeFormField: String = s"$rddTag.pageSize"
 
-  override def pageNumberFormField: String = "block.page"
+  override def pageNumberFormField: String = s"$rddTag.page"
 
   override val dataSource: BlockDataSource = new BlockDataSource(
     rddPartitions,
@@ -254,46 +246,16 @@ private[ui] class BlockPagedTable(
   }
 
   override def headers: Seq[Node] = {
-    val blockHeaders = Seq(
+    val blockHeaders: Seq[(String, Boolean, Option[String])] = Seq(
       "Block Name",
       "Storage Level",
       "Size in Memory",
       "Size on Disk",
-      "Executors")
+      "Executors").map(x => (x, true, None))
 
-    if (!blockHeaders.contains(sortColumn)) {
-      throw new IllegalArgumentException(s"Unknown column: $sortColumn")
-    }
+    isSortColumnValid(blockHeaders, sortColumn)
 
-    val headerRow: Seq[Node] = {
-      blockHeaders.map { header =>
-        if (header == sortColumn) {
-          val headerLink = Unparsed(
-            basePath +
-              s"&block.sort=${URLEncoder.encode(header, UTF_8.name())}" +
-              s"&block.desc=${!desc}" +
-              s"&block.pageSize=$pageSize")
-          val arrow = if (desc) "&#x25BE;" else "&#x25B4;" // UP or DOWN
-          <th>
-            <a href={headerLink}>
-              {header}
-              <span>&nbsp;{Unparsed(arrow)}</span>
-            </a>
-          </th>
-        } else {
-          val headerLink = Unparsed(
-            basePath +
-              s"&block.sort=${URLEncoder.encode(header, UTF_8.name())}" +
-              s"&block.pageSize=$pageSize")
-          <th>
-            <a href={headerLink}>
-              {header}
-            </a>
-          </th>
-        }
-      }
-    }
-    <thead>{headerRow}</thead>
+    headerRow(blockHeaders, desc, pageSize, sortColumn, basePath, rddTag, "block")
   }
 
   override def row(block: BlockTableRowData): Seq[Node] = {
diff --git a/core/src/main/scala/org/apache/spark/util/AccumulatorV2.scala b/core/src/main/scala/org/apache/spark/util/AccumulatorV2.scala
index d5b3ce36e742a..1453840b834f2 100644
--- a/core/src/main/scala/org/apache/spark/util/AccumulatorV2.scala
+++ b/core/src/main/scala/org/apache/spark/util/AccumulatorV2.scala
@@ -19,7 +19,7 @@ package org.apache.spark.util
 
 import java.{lang => jl}
 import java.io.ObjectInputStream
-import java.util.{ArrayList, Collections}
+import java.util.ArrayList
 import java.util.concurrent.ConcurrentHashMap
 import java.util.concurrent.atomic.AtomicLong
 
@@ -449,39 +449,46 @@ class DoubleAccumulator extends AccumulatorV2[jl.Double, jl.Double] {
  * @since 2.0.0
  */
 class CollectionAccumulator[T] extends AccumulatorV2[T, java.util.List[T]] {
-  private val _list: java.util.List[T] = Collections.synchronizedList(new ArrayList[T]())
+  private var _list: java.util.List[T] = _
+
+  private def getOrCreate = {
+    _list = Option(_list).getOrElse(new java.util.ArrayList[T]())
+    _list
+  }
 
   /**
    * Returns false if this accumulator instance has any values in it.
    */
-  override def isZero: Boolean = _list.isEmpty
+  override def isZero: Boolean = this.synchronized(getOrCreate.isEmpty)
 
   override def copyAndReset(): CollectionAccumulator[T] = new CollectionAccumulator
 
   override def copy(): CollectionAccumulator[T] = {
     val newAcc = new CollectionAccumulator[T]
-    _list.synchronized {
-      newAcc._list.addAll(_list)
+    this.synchronized {
+      newAcc.getOrCreate.addAll(getOrCreate)
     }
     newAcc
   }
 
-  override def reset(): Unit = _list.clear()
+  override def reset(): Unit = this.synchronized {
+    _list = null
+  }
 
-  override def add(v: T): Unit = _list.add(v)
+  override def add(v: T): Unit = this.synchronized(getOrCreate.add(v))
 
   override def merge(other: AccumulatorV2[T, java.util.List[T]]): Unit = other match {
-    case o: CollectionAccumulator[T] => _list.addAll(o.value)
+    case o: CollectionAccumulator[T] => this.synchronized(getOrCreate.addAll(o.value))
     case _ => throw new UnsupportedOperationException(
       s"Cannot merge ${this.getClass.getName} with ${other.getClass.getName}")
   }
 
-  override def value: java.util.List[T] = _list.synchronized {
-    java.util.Collections.unmodifiableList(new ArrayList[T](_list))
+  override def value: java.util.List[T] = this.synchronized {
+    java.util.Collections.unmodifiableList(new ArrayList[T](getOrCreate))
   }
 
-  private[spark] def setValue(newValue: java.util.List[T]): Unit = {
-    _list.clear()
-    _list.addAll(newValue)
+  private[spark] def setValue(newValue: java.util.List[T]): Unit = this.synchronized {
+    _list = null
+    getOrCreate.addAll(newValue)
   }
 }
diff --git a/core/src/main/scala/org/apache/spark/util/ClosureCleaner.scala b/core/src/main/scala/org/apache/spark/util/ClosureCleaner.scala
index 6ffd6605f75b8..7e2b9c72ad91b 100644
--- a/core/src/main/scala/org/apache/spark/util/ClosureCleaner.scala
+++ b/core/src/main/scala/org/apache/spark/util/ClosureCleaner.scala
@@ -285,7 +285,7 @@ private[spark] object ClosureCleaner extends Logging {
           logDebug(s" + outermost object is a closure, so we clone it: ${outermostClass}")
         } else if (outermostClass.getName.startsWith("$line")) {
           // SPARK-14558: if the outermost object is a REPL line object, we should clone
-          // and clean it as it may carray a lot of unnecessary information,
+          // and clean it as it may carry a lot of unnecessary information,
           // e.g. hadoop conf, spark conf, etc.
           logDebug(s" + outermost object is a REPL line object, so we clone it:" +
             s" ${outermostClass}")
diff --git a/core/src/main/scala/org/apache/spark/util/DependencyUtils.scala b/core/src/main/scala/org/apache/spark/util/DependencyUtils.scala
new file mode 100644
index 0000000000000..f7135edd2129d
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/util/DependencyUtils.scala
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util
+
+import java.io.File
+import java.net.URI
+
+import org.apache.commons.lang3.StringUtils
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.{FileSystem, Path}
+
+import org.apache.spark.{SparkConf, SparkException}
+import org.apache.spark.deploy.SparkSubmitUtils
+import org.apache.spark.internal.Logging
+
+case class IvyProperties(
+    packagesExclusions: String,
+    packages: String,
+    repositories: String,
+    ivyRepoPath: String,
+    ivySettingsPath: String)
+
+private[spark] object DependencyUtils extends Logging {
+
+  def getIvyProperties(): IvyProperties = {
+    val Seq(packagesExclusions, packages, repositories, ivyRepoPath, ivySettingsPath) = Seq(
+      "spark.jars.excludes",
+      "spark.jars.packages",
+      "spark.jars.repositories",
+      "spark.jars.ivy",
+      "spark.jars.ivySettings"
+    ).map(sys.props.get(_).orNull)
+    IvyProperties(packagesExclusions, packages, repositories, ivyRepoPath, ivySettingsPath)
+  }
+
+  private def isInvalidQueryString(tokens: Array[String]): Boolean = {
+    tokens.length != 2 || StringUtils.isBlank(tokens(0)) || StringUtils.isBlank(tokens(1))
+  }
+
+  /**
+   * Parse URI query string's parameter value of `transitive` and `exclude`.
+   * Other invalid parameters will be ignored.
+   *
+   * @param uri Ivy URI need to be downloaded.
+   * @return Tuple value of parameter `transitive` and `exclude` value.
+   *
+   *         1. transitive: whether to download dependency jar of Ivy URI, default value is true
+   *            and this parameter value is case-insensitive. This mimics Hive's behaviour for
+   *            parsing the transitive parameter. Invalid value will be treat as false.
+   *            Example: Input:  exclude=org.mortbay.jetty:jetty&transitive=true
+   *            Output:  true
+   *
+   *         2. exclude: comma separated exclusions to apply when resolving transitive dependencies,
+   *            consists of `group:module` pairs separated by commas.
+   *            Example: Input:  excludeorg.mortbay.jetty:jetty,org.eclipse.jetty:jetty-http
+   *            Output:  [org.mortbay.jetty:jetty,org.eclipse.jetty:jetty-http]
+   */
+  private def parseQueryParams(uri: URI): (Boolean, String) = {
+    val uriQuery = uri.getQuery
+    if (uriQuery == null) {
+      (true, "")
+    } else {
+      val mapTokens = uriQuery.split("&").map(_.split("="))
+      if (mapTokens.exists(isInvalidQueryString)) {
+        throw new IllegalArgumentException(
+          s"Invalid query string in Ivy URI ${uri.toString}: $uriQuery")
+      }
+      val groupedParams = mapTokens.map(kv => (kv(0), kv(1))).groupBy(_._1)
+
+      // Parse transitive parameters (e.g., transitive=true) in an Ivy URI, default value is true
+      val transitiveParams = groupedParams.get("transitive")
+      if (transitiveParams.map(_.size).getOrElse(0) > 1) {
+        logWarning("It's best to specify `transitive` parameter in ivy URI query only once." +
+          " If there are multiple `transitive` parameter, we will select the last one")
+      }
+      val transitive =
+        transitiveParams.flatMap(_.takeRight(1).map(_._2.equalsIgnoreCase("true")).headOption)
+          .getOrElse(true)
+
+      // Parse an excluded list (e.g., exclude=org.mortbay.jetty:jetty,org.eclipse.jetty:jetty-http)
+      // in an Ivy URI. When download Ivy URI jar, Spark won't download transitive jar
+      // in a excluded list.
+      val exclusionList = groupedParams.get("exclude").map { params =>
+        params.map(_._2).flatMap { excludeString =>
+          val excludes = excludeString.split(",")
+          if (excludes.map(_.split(":")).exists(isInvalidQueryString)) {
+            throw new IllegalArgumentException(
+              s"Invalid exclude string in Ivy URI ${uri.toString}:" +
+                " expected 'org:module,org:module,..', found " + excludeString)
+          }
+          excludes
+        }.mkString(",")
+      }.getOrElse("")
+
+      val validParams = Set("transitive", "exclude")
+      val invalidParams = groupedParams.keys.filterNot(validParams.contains).toSeq
+      if (invalidParams.nonEmpty) {
+        logWarning(s"Invalid parameters `${invalidParams.sorted.mkString(",")}` found " +
+          s"in Ivy URI query `$uriQuery`.")
+      }
+
+      (transitive, exclusionList)
+    }
+  }
+
+  /**
+   * Download Ivy URI's dependency jars.
+   *
+   * @param uri Ivy URI need to be downloaded. The URI format should be:
+   *              `ivy://group:module:version[?query]`
+   *            Ivy URI query part format should be:
+   *              `parameter=value&parameter=value...`
+   *            Note that currently Ivy URI query part support two parameters:
+   *             1. transitive: whether to download dependent jars related to your Ivy URI.
+   *                transitive=false or `transitive=true`, if not set, the default value is true.
+   *             2. exclude: exclusion list when download Ivy URI jar and dependency jars.
+   *                The `exclude` parameter content is a ',' separated `group:module` pair string :
+   *                `exclude=group:module,group:module...`
+   * @return List of jars downloaded.
+   */
+  def resolveMavenDependencies(uri: URI): Seq[String] = {
+    val ivyProperties = DependencyUtils.getIvyProperties()
+    val authority = uri.getAuthority
+    if (authority == null) {
+      throw new IllegalArgumentException(
+        s"Invalid Ivy URI authority in uri ${uri.toString}:" +
+          " Expected 'org:module:version', found null.")
+    }
+    if (authority.split(":").length != 3) {
+      throw new IllegalArgumentException(
+        s"Invalid Ivy URI authority in uri ${uri.toString}:" +
+          s" Expected 'org:module:version', found $authority.")
+    }
+
+    val (transitive, exclusionList) = parseQueryParams(uri)
+
+    resolveMavenDependencies(
+      transitive,
+      exclusionList,
+      authority,
+      ivyProperties.repositories,
+      ivyProperties.ivyRepoPath,
+      Option(ivyProperties.ivySettingsPath)
+    )
+  }
+
+  def resolveMavenDependencies(
+      packagesTransitive: Boolean,
+      packagesExclusions: String,
+      packages: String,
+      repositories: String,
+      ivyRepoPath: String,
+      ivySettingsPath: Option[String]): Seq[String] = {
+    val exclusions: Seq[String] =
+      if (!StringUtils.isBlank(packagesExclusions)) {
+        packagesExclusions.split(",")
+      } else {
+        Nil
+      }
+    // Create the IvySettings, either load from file or build defaults
+    val ivySettings = ivySettingsPath match {
+      case Some(path) =>
+        SparkSubmitUtils.loadIvySettings(path, Option(repositories), Option(ivyRepoPath))
+
+      case None =>
+        SparkSubmitUtils.buildIvySettings(Option(repositories), Option(ivyRepoPath))
+    }
+
+    SparkSubmitUtils.resolveMavenCoordinates(packages, ivySettings,
+      transitive = packagesTransitive, exclusions = exclusions)
+  }
+
+  def resolveAndDownloadJars(
+      jars: String,
+      userJar: String,
+      sparkConf: SparkConf,
+      hadoopConf: Configuration): String = {
+    val targetDir = Utils.createTempDir()
+    val userJarName = userJar.split(File.separatorChar).last
+    Option(jars)
+      .map {
+        resolveGlobPaths(_, hadoopConf)
+          .split(",")
+          .filterNot(_.contains(userJarName))
+          .mkString(",")
+      }
+      .filterNot(_ == "")
+      .map(downloadFileList(_, targetDir, sparkConf, hadoopConf))
+      .orNull
+  }
+
+  def addJarsToClassPath(jars: String, loader: MutableURLClassLoader): Unit = {
+    if (jars != null) {
+      for (jar <- jars.split(",")) {
+        addJarToClasspath(jar, loader)
+      }
+    }
+  }
+
+  /**
+   * Download a list of remote files to temp local files. If the file is local, the original file
+   * will be returned.
+   *
+   * @param fileList A comma separated file list.
+   * @param targetDir A temporary directory for which downloaded files.
+   * @param sparkConf Spark configuration.
+   * @param hadoopConf Hadoop configuration.
+   * @return A comma separated local files list.
+   */
+  def downloadFileList(
+      fileList: String,
+      targetDir: File,
+      sparkConf: SparkConf,
+      hadoopConf: Configuration): String = {
+    require(fileList != null, "fileList cannot be null.")
+    Utils.stringToSeq(fileList)
+      .map(downloadFile(_, targetDir, sparkConf, hadoopConf))
+      .mkString(",")
+  }
+
+  /**
+   * Download a file from the remote to a local temporary directory. If the input path points to
+   * a local path, returns it with no operation.
+   *
+   * @param path A file path from where the files will be downloaded.
+   * @param targetDir A temporary directory for which downloaded files.
+   * @param sparkConf Spark configuration.
+   * @param hadoopConf Hadoop configuration.
+   * @return Path to the local file.
+   */
+  def downloadFile(
+      path: String,
+      targetDir: File,
+      sparkConf: SparkConf,
+      hadoopConf: Configuration): String = {
+    require(path != null, "path cannot be null.")
+    val uri = Utils.resolveURI(path)
+
+    uri.getScheme match {
+      case "file" | "local" => path
+      case "http" | "https" | "ftp" if Utils.isTesting =>
+        // This is only used for SparkSubmitSuite unit test. Instead of downloading file remotely,
+        // return a dummy local path instead.
+        val file = new File(uri.getPath)
+        new File(targetDir, file.getName).toURI.toString
+      case _ =>
+        val fname = new Path(uri).getName()
+        val localFile = Utils.doFetchFile(uri.toString(), targetDir, fname, sparkConf, hadoopConf)
+        localFile.toURI().toString()
+    }
+  }
+
+  def resolveGlobPaths(paths: String, hadoopConf: Configuration): String = {
+    require(paths != null, "paths cannot be null.")
+    Utils.stringToSeq(paths).flatMap { path =>
+      val (base, fragment) = splitOnFragment(path)
+      (resolveGlobPath(base, hadoopConf), fragment) match {
+        case (resolved, Some(_)) if resolved.length > 1 => throw new SparkException(
+            s"${base.toString} resolves ambiguously to multiple files: ${resolved.mkString(",")}")
+        case (resolved, Some(namedAs)) => resolved.map(_ + "#" + namedAs)
+        case (resolved, _) => resolved
+      }
+    }.mkString(",")
+  }
+
+  def addJarToClasspath(localJar: String, loader: MutableURLClassLoader): Unit = {
+    val uri = Utils.resolveURI(localJar)
+    uri.getScheme match {
+      case "file" | "local" =>
+        val file = new File(uri.getPath)
+        if (file.exists()) {
+          loader.addURL(file.toURI.toURL)
+        } else {
+          logWarning(s"Local jar $file does not exist, skipping.")
+        }
+      case _ =>
+        logWarning(s"Skip remote jar $uri.")
+    }
+  }
+
+  /**
+   * Merge a sequence of comma-separated file lists, some of which may be null to indicate
+   * no files, into a single comma-separated string.
+   */
+  def mergeFileLists(lists: String*): String = {
+    val merged = lists.filterNot(StringUtils.isBlank)
+      .flatMap(Utils.stringToSeq)
+    if (merged.nonEmpty) merged.mkString(",") else null
+  }
+
+  private def splitOnFragment(path: String): (URI, Option[String]) = {
+    val uri = Utils.resolveURI(path)
+    val withoutFragment = new URI(uri.getScheme, uri.getSchemeSpecificPart, null)
+    (withoutFragment, Option(uri.getFragment))
+  }
+
+  private def resolveGlobPath(uri: URI, hadoopConf: Configuration): Array[String] = {
+    uri.getScheme match {
+      case "local" | "http" | "https" | "ftp" => Array(uri.toString)
+      case _ =>
+        val fs = FileSystem.get(uri, hadoopConf)
+        Option(fs.globStatus(new Path(uri))).map { status =>
+          status.filter(_.isFile).map(_.getPath.toUri.toString)
+        }.getOrElse(Array(uri.toString))
+    }
+  }
+
+}
diff --git a/core/src/main/scala/org/apache/spark/util/HadoopFSUtils.scala b/core/src/main/scala/org/apache/spark/util/HadoopFSUtils.scala
new file mode 100644
index 0000000000000..60a73adc8582e
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/util/HadoopFSUtils.scala
@@ -0,0 +1,370 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util
+
+import java.io.FileNotFoundException
+
+import scala.collection.mutable
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs._
+import org.apache.hadoop.fs.viewfs.ViewFileSystem
+import org.apache.hadoop.hdfs.DistributedFileSystem
+
+import org.apache.spark._
+import org.apache.spark.internal.Logging
+import org.apache.spark.metrics.source.HiveCatalogMetrics
+
+/**
+ * Utility functions to simplify and speed-up file listing.
+ */
+private[spark] object HadoopFSUtils extends Logging {
+  /**
+   * Lists a collection of paths recursively. Picks the listing strategy adaptively depending
+   * on the number of paths to list.
+   *
+   * This may only be called on the driver.
+   *
+   * @param sc Spark context used to run parallel listing.
+   * @param paths Input paths to list
+   * @param hadoopConf Hadoop configuration
+   * @param filter Path filter used to exclude leaf files from result
+   * @param ignoreMissingFiles Ignore missing files that happen during recursive listing
+   *                           (e.g., due to race conditions)
+   * @param ignoreLocality Whether to fetch data locality info when listing leaf files. If false,
+   *                       this will return `FileStatus` without `BlockLocation` info.
+   * @param parallelismThreshold The threshold to enable parallelism. If the number of input paths
+   *                             is smaller than this value, this will fallback to use
+   *                             sequential listing.
+   * @param parallelismMax The maximum parallelism for listing. If the number of input paths is
+   *                       larger than this value, parallelism will be throttled to this value
+   *                       to avoid generating too many tasks.
+   * @return for each input path, the set of discovered files for the path
+   */
+  def parallelListLeafFiles(
+    sc: SparkContext,
+    paths: Seq[Path],
+    hadoopConf: Configuration,
+    filter: PathFilter,
+    ignoreMissingFiles: Boolean,
+    ignoreLocality: Boolean,
+    parallelismThreshold: Int,
+    parallelismMax: Int): Seq[(Path, Seq[FileStatus])] = {
+    parallelListLeafFilesInternal(sc, paths, hadoopConf, filter, isRootLevel = true,
+      ignoreMissingFiles, ignoreLocality, parallelismThreshold, parallelismMax)
+  }
+
+  private def parallelListLeafFilesInternal(
+      sc: SparkContext,
+      paths: Seq[Path],
+      hadoopConf: Configuration,
+      filter: PathFilter,
+      isRootLevel: Boolean,
+      ignoreMissingFiles: Boolean,
+      ignoreLocality: Boolean,
+      parallelismThreshold: Int,
+      parallelismMax: Int): Seq[(Path, Seq[FileStatus])] = {
+
+    // Short-circuits parallel listing when serial listing is likely to be faster.
+    if (paths.size <= parallelismThreshold) {
+      return paths.map { path =>
+        val leafFiles = listLeafFiles(
+          path,
+          hadoopConf,
+          filter,
+          Some(sc),
+          ignoreMissingFiles = ignoreMissingFiles,
+          ignoreLocality = ignoreLocality,
+          isRootPath = isRootLevel,
+          parallelismThreshold = parallelismThreshold,
+          parallelismMax = parallelismMax)
+        (path, leafFiles)
+      }
+    }
+
+    logInfo(s"Listing leaf files and directories in parallel under ${paths.length} paths." +
+      s" The first several paths are: ${paths.take(10).mkString(", ")}.")
+    HiveCatalogMetrics.incrementParallelListingJobCount(1)
+
+    val serializableConfiguration = new SerializableConfiguration(hadoopConf)
+    val serializedPaths = paths.map(_.toString)
+
+    // Set the number of parallelism to prevent following file listing from generating many tasks
+    // in case of large #defaultParallelism.
+    val numParallelism = Math.min(paths.size, parallelismMax)
+
+    val previousJobDescription = sc.getLocalProperty(SparkContext.SPARK_JOB_DESCRIPTION)
+    val statusMap = try {
+      val description = paths.size match {
+        case 0 =>
+          "Listing leaf files and directories 0 paths"
+        case 1 =>
+          s"Listing leaf files and directories for 1 path:<br/>${paths(0)}"
+        case s =>
+          s"Listing leaf files and directories for $s paths:<br/>${paths(0)}, ..."
+      }
+      sc.setJobDescription(description)
+      sc
+        .parallelize(serializedPaths, numParallelism)
+        .mapPartitions { pathStrings =>
+          val hadoopConf = serializableConfiguration.value
+          pathStrings.map(new Path(_)).toSeq.map { path =>
+            val leafFiles = listLeafFiles(
+              path = path,
+              hadoopConf = hadoopConf,
+              filter = filter,
+              contextOpt = None, // Can't execute parallel scans on workers
+              ignoreMissingFiles = ignoreMissingFiles,
+              ignoreLocality = ignoreLocality,
+              isRootPath = isRootLevel,
+              parallelismThreshold = Int.MaxValue,
+              parallelismMax = 0)
+            (path, leafFiles)
+          }.iterator
+        }.map { case (path, statuses) =>
+            val serializableStatuses = statuses.map { status =>
+              // Turn FileStatus into SerializableFileStatus so we can send it back to the driver
+              val blockLocations = status match {
+                case f: LocatedFileStatus =>
+                  f.getBlockLocations.map { loc =>
+                    SerializableBlockLocation(
+                      loc.getNames,
+                      loc.getHosts,
+                      loc.getOffset,
+                      loc.getLength)
+                  }
+
+                case _ =>
+                  Array.empty[SerializableBlockLocation]
+              }
+
+              SerializableFileStatus(
+                status.getPath.toString,
+                status.getLen,
+                status.isDirectory,
+                status.getReplication,
+                status.getBlockSize,
+                status.getModificationTime,
+                status.getAccessTime,
+                blockLocations)
+            }
+            (path.toString, serializableStatuses)
+        }.collect()
+    } finally {
+      sc.setJobDescription(previousJobDescription)
+    }
+
+    // turn SerializableFileStatus back to Status
+    statusMap.map { case (path, serializableStatuses) =>
+      val statuses = serializableStatuses.map { f =>
+        val blockLocations = f.blockLocations.map { loc =>
+          new BlockLocation(loc.names, loc.hosts, loc.offset, loc.length)
+        }
+        new LocatedFileStatus(
+          new FileStatus(
+            f.length, f.isDir, f.blockReplication, f.blockSize, f.modificationTime,
+            new Path(f.path)),
+          blockLocations)
+      }
+      (new Path(path), statuses)
+    }
+  }
+
+  // scalastyle:off argcount
+  /**
+   * Lists a single filesystem path recursively. If a `SparkContext` object is specified, this
+   * function may launch Spark jobs to parallelize listing based on `parallelismThreshold`.
+   *
+   * If sessionOpt is None, this may be called on executors.
+   *
+   * @return all children of path that match the specified filter.
+   */
+  private def listLeafFiles(
+      path: Path,
+      hadoopConf: Configuration,
+      filter: PathFilter,
+      contextOpt: Option[SparkContext],
+      ignoreMissingFiles: Boolean,
+      ignoreLocality: Boolean,
+      isRootPath: Boolean,
+      parallelismThreshold: Int,
+      parallelismMax: Int): Seq[FileStatus] = {
+
+    logTrace(s"Listing $path")
+    val fs = path.getFileSystem(hadoopConf)
+
+    // Note that statuses only include FileStatus for the files and dirs directly under path,
+    // and does not include anything else recursively.
+    val statuses: Array[FileStatus] = try {
+      fs match {
+        // DistributedFileSystem overrides listLocatedStatus to make 1 single call to namenode
+        // to retrieve the file status with the file block location. The reason to still fallback
+        // to listStatus is because the default implementation would potentially throw a
+        // FileNotFoundException which is better handled by doing the lookups manually below.
+        case (_: DistributedFileSystem | _: ViewFileSystem) if !ignoreLocality =>
+          val remoteIter = fs.listLocatedStatus(path)
+          new Iterator[LocatedFileStatus]() {
+            def next(): LocatedFileStatus = remoteIter.next
+            def hasNext(): Boolean = remoteIter.hasNext
+          }.toArray
+        case _ => fs.listStatus(path)
+      }
+    } catch {
+      // If we are listing a root path for SQL (e.g. a top level directory of a table), we need to
+      // ignore FileNotFoundExceptions during this root level of the listing because
+      //
+      //  (a) certain code paths might construct an InMemoryFileIndex with root paths that
+      //      might not exist (i.e. not all callers are guaranteed to have checked
+      //      path existence prior to constructing InMemoryFileIndex) and,
+      //  (b) we need to ignore deleted root paths during REFRESH TABLE, otherwise we break
+      //      existing behavior and break the ability drop SessionCatalog tables when tables'
+      //      root directories have been deleted (which breaks a number of Spark's own tests).
+      //
+      // If we are NOT listing a root path then a FileNotFoundException here means that the
+      // directory was present in a previous level of file listing but is absent in this
+      // listing, likely indicating a race condition (e.g. concurrent table overwrite or S3
+      // list inconsistency).
+      //
+      // The trade-off in supporting existing behaviors / use-cases is that we won't be
+      // able to detect race conditions involving root paths being deleted during
+      // InMemoryFileIndex construction. However, it's still a net improvement to detect and
+      // fail-fast on the non-root cases. For more info see the SPARK-27676 review discussion.
+      case _: FileNotFoundException if isRootPath || ignoreMissingFiles =>
+        logWarning(s"The directory $path was not found. Was it deleted very recently?")
+        Array.empty[FileStatus]
+    }
+
+    val filteredStatuses =
+      statuses.filterNot(status => shouldFilterOutPathName(status.getPath.getName))
+
+    val allLeafStatuses = {
+      val (dirs, topLevelFiles) = filteredStatuses.partition(_.isDirectory)
+      val nestedFiles: Seq[FileStatus] = contextOpt match {
+        case Some(context) if dirs.size > parallelismThreshold =>
+          parallelListLeafFilesInternal(
+            context,
+            dirs.map(_.getPath),
+            hadoopConf = hadoopConf,
+            filter = filter,
+            isRootLevel = false,
+            ignoreMissingFiles = ignoreMissingFiles,
+            ignoreLocality = ignoreLocality,
+            parallelismThreshold = parallelismThreshold,
+            parallelismMax = parallelismMax
+          ).flatMap(_._2)
+        case _ =>
+          dirs.flatMap { dir =>
+            listLeafFiles(
+              path = dir.getPath,
+              hadoopConf = hadoopConf,
+              filter = filter,
+              contextOpt = contextOpt,
+              ignoreMissingFiles = ignoreMissingFiles,
+              ignoreLocality = ignoreLocality,
+              isRootPath = false,
+              parallelismThreshold = parallelismThreshold,
+              parallelismMax = parallelismMax)
+          }
+      }
+      val allFiles = topLevelFiles ++ nestedFiles
+      if (filter != null) allFiles.filter(f => filter.accept(f.getPath)) else allFiles
+    }
+
+    val missingFiles = mutable.ArrayBuffer.empty[String]
+    val resolvedLeafStatuses = allLeafStatuses.flatMap {
+      case f: LocatedFileStatus =>
+        Some(f)
+
+      // NOTE:
+      //
+      // - Although S3/S3A/S3N file system can be quite slow for remote file metadata
+      //   operations, calling `getFileBlockLocations` does no harm here since these file system
+      //   implementations don't actually issue RPC for this method.
+      //
+      // - Here we are calling `getFileBlockLocations` in a sequential manner, but it should not
+      //   be a big deal since we always use to `parallelListLeafFiles` when the number of
+      //   paths exceeds threshold.
+      case f if !ignoreLocality =>
+        // The other constructor of LocatedFileStatus will call FileStatus.getPermission(),
+        // which is very slow on some file system (RawLocalFileSystem, which is launch a
+        // subprocess and parse the stdout).
+        try {
+          val locations = fs.getFileBlockLocations(f, 0, f.getLen).map { loc =>
+            // Store BlockLocation objects to consume less memory
+            if (loc.getClass == classOf[BlockLocation]) {
+              loc
+            } else {
+              new BlockLocation(loc.getNames, loc.getHosts, loc.getOffset, loc.getLength)
+            }
+          }
+          val lfs = new LocatedFileStatus(f.getLen, f.isDirectory, f.getReplication, f.getBlockSize,
+            f.getModificationTime, 0, null, null, null, null, f.getPath, locations)
+          if (f.isSymlink) {
+            lfs.setSymlink(f.getSymlink)
+          }
+          Some(lfs)
+        } catch {
+          case _: FileNotFoundException if ignoreMissingFiles =>
+            missingFiles += f.getPath.toString
+            None
+        }
+
+      case f => Some(f)
+    }
+
+    if (missingFiles.nonEmpty) {
+      logWarning(
+        s"the following files were missing during file scan:\n  ${missingFiles.mkString("\n  ")}")
+    }
+
+    resolvedLeafStatuses
+  }
+  // scalastyle:on argcount
+
+  /** A serializable variant of HDFS's BlockLocation. This is required by Hadoop 2.7. */
+  private case class SerializableBlockLocation(
+    names: Array[String],
+    hosts: Array[String],
+    offset: Long,
+    length: Long)
+
+  /** A serializable variant of HDFS's FileStatus. This is required by Hadoop 2.7. */
+  private case class SerializableFileStatus(
+    path: String,
+    length: Long,
+    isDir: Boolean,
+    blockReplication: Short,
+    blockSize: Long,
+    modificationTime: Long,
+    accessTime: Long,
+    blockLocations: Array[SerializableBlockLocation])
+
+  /** Checks if we should filter out this path name. */
+  def shouldFilterOutPathName(pathName: String): Boolean = {
+    // We filter follow paths:
+    // 1. everything that starts with _ and ., except _common_metadata and _metadata
+    // because Parquet needs to find those metadata files from leaf files returned by this method.
+    // We should refactor this logic to not mix metadata files with data files.
+    // 2. everything that ends with `._COPYING_`, because this is a intermediate state of file. we
+    // should skip this file in case of double reading.
+    val exclude = (pathName.startsWith("_") && !pathName.contains("=")) ||
+      pathName.startsWith(".") || pathName.endsWith("._COPYING_")
+    val include = pathName.startsWith("_common_metadata") || pathName.startsWith("_metadata")
+    exclude && !include
+  }
+}
diff --git a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
index 9254ac94005f1..c74cca9b93b89 100644
--- a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
@@ -32,8 +32,8 @@ import org.json4s.jackson.JsonMethods._
 import org.apache.spark._
 import org.apache.spark.executor._
 import org.apache.spark.metrics.ExecutorMetricType
-import org.apache.spark.rdd.RDDOperationScope
-import org.apache.spark.resource.{ResourceInformation, ResourceProfile}
+import org.apache.spark.rdd.{DeterministicLevel, RDDOperationScope}
+import org.apache.spark.resource.{ExecutorResourceRequest, ResourceInformation, ResourceProfile, TaskResourceRequest}
 import org.apache.spark.scheduler._
 import org.apache.spark.scheduler.cluster.ExecutorInfo
 import org.apache.spark.storage._
@@ -105,6 +105,8 @@ private[spark] object JsonProtocol {
         stageExecutorMetricsToJson(stageExecutorMetrics)
       case blockUpdate: SparkListenerBlockUpdated =>
         blockUpdateToJson(blockUpdate)
+      case resourceProfileAdded: SparkListenerResourceProfileAdded =>
+        resourceProfileAddedToJson(resourceProfileAdded)
       case _ => parse(mapper.writeValueAsString(event))
     }
   }
@@ -224,6 +226,15 @@ private[spark] object JsonProtocol {
     ("Timestamp" -> applicationEnd.time)
   }
 
+  def resourceProfileAddedToJson(profileAdded: SparkListenerResourceProfileAdded): JValue = {
+    ("Event" -> SPARK_LISTENER_EVENT_FORMATTED_CLASS_NAMES.resourceProfileAdded) ~
+      ("Resource Profile Id" -> profileAdded.resourceProfile.id) ~
+      ("Executor Resource Requests" ->
+        executorResourceRequestMapToJson(profileAdded.resourceProfile.executorResources)) ~
+      ("Task Resource Requests" ->
+        taskResourceRequestMapToJson(profileAdded.resourceProfile.taskResources))
+  }
+
   def executorAddedToJson(executorAdded: SparkListenerExecutorAdded): JValue = {
     ("Event" -> SPARK_LISTENER_EVENT_FORMATTED_CLASS_NAMES.executorAdded) ~
     ("Timestamp" -> executorAdded.time) ~
@@ -297,7 +308,8 @@ private[spark] object JsonProtocol {
     ("Submission Time" -> submissionTime) ~
     ("Completion Time" -> completionTime) ~
     ("Failure Reason" -> failureReason) ~
-    ("Accumulables" -> accumulablesToJson(stageInfo.accumulables.values))
+    ("Accumulables" -> accumulablesToJson(stageInfo.accumulables.values)) ~
+    ("Resource Profile Id" -> stageInfo.resourceProfileId)
   }
 
   def taskInfoToJson(taskInfo: TaskInfo): JValue = {
@@ -316,12 +328,12 @@ private[spark] object JsonProtocol {
     ("Accumulables" -> accumulablesToJson(taskInfo.accumulables))
   }
 
-  private lazy val accumulableBlacklist = Set("internal.metrics.updatedBlockStatuses")
+  private lazy val accumulableExcludeList = Set("internal.metrics.updatedBlockStatuses")
 
   def accumulablesToJson(accumulables: Iterable[AccumulableInfo]): JArray = {
     JArray(accumulables
-        .filterNot(_.name.exists(accumulableBlacklist.contains))
-        .toList.map(accumulableInfoToJson))
+        .filterNot(_.name.exists(accumulableExcludeList.contains))
+        .toList.sortBy(_.id).map(accumulableInfoToJson))
   }
 
   def accumulableInfoToJson(accumulableInfo: AccumulableInfo): JValue = {
@@ -351,12 +363,22 @@ private[spark] object JsonProtocol {
         case v: Long => JInt(v)
         // We only have 3 kind of internal accumulator types, so if it's not int or long, it must be
         // the blocks accumulator, whose type is `java.util.List[(BlockId, BlockStatus)]`
-        case v =>
-          JArray(v.asInstanceOf[java.util.List[(BlockId, BlockStatus)]].asScala.toList.map {
-            case (id, status) =>
-              ("Block ID" -> id.toString) ~
-              ("Status" -> blockStatusToJson(status))
+        case v: java.util.List[_] =>
+          JArray(v.asScala.toList.flatMap {
+            case (id: BlockId, status: BlockStatus) =>
+              Some(
+                ("Block ID" -> id.toString) ~
+                ("Status" -> blockStatusToJson(status))
+              )
+            case _ =>
+              // Ignore unsupported types. A user may put `METRICS_PREFIX` in the name. We should
+              // not crash.
+              None
           })
+        case _ =>
+          // Ignore unsupported types. A user may put `METRICS_PREFIX` in the name. We should not
+          // crash.
+          JNothing
       }
     } else {
       // For all external accumulators, just use strings
@@ -475,6 +497,8 @@ private[spark] object JsonProtocol {
     ("Callsite" -> rddInfo.callSite) ~
     ("Parent IDs" -> parentIds) ~
     ("Storage Level" -> storageLevel) ~
+    ("Barrier" -> rddInfo.isBarrier) ~
+    ("DeterministicLevel" -> rddInfo.outputDeterministicLevel.toString) ~
     ("Number of Partitions" -> rddInfo.numPartitions) ~
     ("Number of Cached Partitions" -> rddInfo.numCachedPartitions) ~
     ("Memory Size" -> rddInfo.memSize) ~
@@ -500,7 +524,8 @@ private[spark] object JsonProtocol {
     ("Total Cores" -> executorInfo.totalCores) ~
     ("Log Urls" -> mapToJson(executorInfo.logUrlMap)) ~
     ("Attributes" -> mapToJson(executorInfo.attributes)) ~
-    ("Resources" -> resourcesMapToJson(executorInfo.resourcesInfo))
+    ("Resources" -> resourcesMapToJson(executorInfo.resourcesInfo)) ~
+    ("Resource Profile Id" -> executorInfo.resourceProfileId)
   }
 
   def resourcesMapToJson(m: Map[String, ResourceInformation]): JValue = {
@@ -518,6 +543,34 @@ private[spark] object JsonProtocol {
     ("Disk Size" -> blockUpdatedInfo.diskSize)
   }
 
+  def executorResourceRequestToJson(execReq: ExecutorResourceRequest): JValue = {
+    ("Resource Name" -> execReq.resourceName) ~
+    ("Amount" -> execReq.amount) ~
+    ("Discovery Script" -> execReq.discoveryScript) ~
+    ("Vendor" -> execReq.vendor)
+  }
+
+  def executorResourceRequestMapToJson(m: Map[String, ExecutorResourceRequest]): JValue = {
+    val jsonFields = m.map {
+      case (k, execReq) =>
+        JField(k, executorResourceRequestToJson(execReq))
+    }
+    JObject(jsonFields.toList)
+  }
+
+  def taskResourceRequestToJson(taskReq: TaskResourceRequest): JValue = {
+    ("Resource Name" -> taskReq.resourceName) ~
+    ("Amount" -> taskReq.amount)
+  }
+
+  def taskResourceRequestMapToJson(m: Map[String, TaskResourceRequest]): JValue = {
+    val jsonFields = m.map {
+      case (k, taskReq) =>
+        JField(k, taskResourceRequestToJson(taskReq))
+    }
+    JObject(jsonFields.toList)
+  }
+
   /** ------------------------------ *
    * Util JSON serialization methods |
    * ------------------------------- */
@@ -577,6 +630,7 @@ private[spark] object JsonProtocol {
     val metricsUpdate = Utils.getFormattedClassName(SparkListenerExecutorMetricsUpdate)
     val stageExecutorMetrics = Utils.getFormattedClassName(SparkListenerStageExecutorMetrics)
     val blockUpdate = Utils.getFormattedClassName(SparkListenerBlockUpdated)
+    val resourceProfileAdded = Utils.getFormattedClassName(SparkListenerResourceProfileAdded)
   }
 
   def sparkEventFromJson(json: JValue): SparkListenerEvent = {
@@ -602,6 +656,7 @@ private[spark] object JsonProtocol {
       case `metricsUpdate` => executorMetricsUpdateFromJson(json)
       case `stageExecutorMetrics` => stageExecutorMetricsFromJson(json)
       case `blockUpdate` => blockUpdateFromJson(json)
+      case `resourceProfileAdded` => resourceProfileAddedFromJson(json)
       case other => mapper.readValue(compact(render(json)), Utils.classForName(other))
         .asInstanceOf[SparkListenerEvent]
     }
@@ -678,6 +733,45 @@ private[spark] object JsonProtocol {
     SparkListenerJobEnd(jobId, completionTime, jobResult)
   }
 
+  def resourceProfileAddedFromJson(json: JValue): SparkListenerResourceProfileAdded = {
+    val profId = (json \ "Resource Profile Id").extract[Int]
+    val executorReqs = executorResourceRequestMapFromJson(json \ "Executor Resource Requests")
+    val taskReqs = taskResourceRequestMapFromJson(json \ "Task Resource Requests")
+    val rp = new ResourceProfile(executorReqs.toMap, taskReqs.toMap)
+    rp.setResourceProfileId(profId)
+    SparkListenerResourceProfileAdded(rp)
+  }
+
+  def executorResourceRequestFromJson(json: JValue): ExecutorResourceRequest = {
+    val rName = (json \ "Resource Name").extract[String]
+    val amount = (json \ "Amount").extract[Int]
+    val discoveryScript = (json \ "Discovery Script").extract[String]
+    val vendor = (json \ "Vendor").extract[String]
+    new ExecutorResourceRequest(rName, amount, discoveryScript, vendor)
+  }
+
+  def taskResourceRequestFromJson(json: JValue): TaskResourceRequest = {
+    val rName = (json \ "Resource Name").extract[String]
+    val amount = (json \ "Amount").extract[Int]
+    new TaskResourceRequest(rName, amount)
+  }
+
+  def taskResourceRequestMapFromJson(json: JValue): Map[String, TaskResourceRequest] = {
+    val jsonFields = json.asInstanceOf[JObject].obj
+    jsonFields.collect { case JField(k, v) =>
+      val req = taskResourceRequestFromJson(v)
+      (k, req)
+    }.toMap
+  }
+
+  def executorResourceRequestMapFromJson(json: JValue): Map[String, ExecutorResourceRequest] = {
+    val jsonFields = json.asInstanceOf[JObject].obj
+    jsonFields.collect { case JField(k, v) =>
+      val req = executorResourceRequestFromJson(v)
+      (k, req)
+    }.toMap
+  }
+
   def environmentUpdateFromJson(json: JValue): SparkListenerEnvironmentUpdate = {
     // For compatible with previous event logs
     val hadoopProperties = jsonOption(json \ "Hadoop Properties").map(mapFromJson(_).toSeq)
@@ -804,9 +898,10 @@ private[spark] object JsonProtocol {
       }
     }
 
-    val stageInfo = new StageInfo(
-      stageId, attemptId, stageName, numTasks, rddInfos, parentIds, details,
-      resourceProfileId = ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID)
+    val rpId = jsonOption(json \ "Resource Profile Id").map(_.extract[Int])
+    val stageProf = rpId.getOrElse(ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID)
+    val stageInfo = new StageInfo(stageId, attemptId, stageName, numTasks, rddInfos,
+      parentIds, details, resourceProfileId = stageProf)
     stageInfo.submissionTime = submissionTime
     stageInfo.completionTime = completionTime
     stageInfo.failureReason = failureReason
@@ -984,7 +1079,14 @@ private[spark] object JsonProtocol {
         val blockManagerAddress = blockManagerIdFromJson(json \ "Block Manager Address")
         val shuffleId = (json \ "Shuffle ID").extract[Int]
         val mapId = (json \ "Map ID").extract[Long]
-        val mapIndex = (json \ "Map Index").extract[Int]
+        val mapIndex = json \ "Map Index" match {
+          case JNothing =>
+            // Note, we use the invalid value Int.MinValue here to fill the map index for backward
+            // compatibility. Otherwise, the fetch failed event will be dropped when the history
+            // server loads the event log written by the Spark version before 3.0.
+            Int.MinValue
+          case x => x.extract[Int]
+        }
         val reduceId = (json \ "Reduce ID").extract[Int]
         val message = jsonOption(json \ "Message").map(_.extract[String])
         new FetchFailed(blockManagerAddress, shuffleId, mapId, mapIndex, reduceId,
@@ -1074,8 +1176,12 @@ private[spark] object JsonProtocol {
     val memSize = (json \ "Memory Size").extract[Long]
     val diskSize = (json \ "Disk Size").extract[Long]
 
+    val outputDeterministicLevel = DeterministicLevel.withName(
+      jsonOption(json \ "DeterministicLevel").map(_.extract[String]).getOrElse("DETERMINATE"))
+
     val rddInfo =
-      new RDDInfo(rddId, name, numPartitions, storageLevel, isBarrier, parentIds, callsite, scope)
+      new RDDInfo(rddId, name, numPartitions, storageLevel, isBarrier, parentIds, callsite, scope,
+        outputDeterministicLevel)
     rddInfo.numCachedPartitions = numCachedPartitions
     rddInfo.memSize = memSize
     rddInfo.diskSize = diskSize
@@ -1109,7 +1215,12 @@ private[spark] object JsonProtocol {
       case Some(resources) => resourcesMapFromJson(resources).toMap
       case None => Map.empty[String, ResourceInformation]
     }
-    new ExecutorInfo(executorHost, totalCores, logUrls, attributes, resources)
+    val resourceProfileId = jsonOption(json \ "Resource Profile Id") match {
+      case Some(id) => id.extract[Int]
+      case None => ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID
+    }
+    new ExecutorInfo(executorHost, totalCores, logUrls, attributes.toMap, resources.toMap,
+      resourceProfileId)
   }
 
   def blockUpdatedInfoFromJson(json: JValue): BlockUpdatedInfo = {
@@ -1123,7 +1234,7 @@ private[spark] object JsonProtocol {
 
   def resourcesMapFromJson(json: JValue): Map[String, ResourceInformation] = {
     val jsonFields = json.asInstanceOf[JObject].obj
-    jsonFields.map { case JField(k, v) =>
+    jsonFields.collect { case JField(k, v) =>
       val resourceInfo = ResourceInformation.parseJson(v)
       (k, resourceInfo)
     }.toMap
@@ -1135,7 +1246,7 @@ private[spark] object JsonProtocol {
 
   def mapFromJson(json: JValue): Map[String, String] = {
     val jsonFields = json.asInstanceOf[JObject].obj
-    jsonFields.map { case JField(k, JString(v)) => (k, v) }.toMap
+    jsonFields.collect { case JField(k, JString(v)) => (k, v) }.toMap
   }
 
   def propertiesFromJson(json: JValue): Properties = {
diff --git a/core/src/main/scala/org/apache/spark/util/ListenerBus.scala b/core/src/main/scala/org/apache/spark/util/ListenerBus.scala
index 51cd7d1284ff3..3520fa870c91b 100644
--- a/core/src/main/scala/org/apache/spark/util/ListenerBus.scala
+++ b/core/src/main/scala/org/apache/spark/util/ListenerBus.scala
@@ -27,6 +27,8 @@ import com.codahale.metrics.Timer
 
 import org.apache.spark.SparkEnv
 import org.apache.spark.internal.{config, Logging}
+import org.apache.spark.scheduler.EventLoggingListener
+import org.apache.spark.scheduler.SparkListenerEnvironmentUpdate
 
 /**
  * An event bus which posts events to its listeners.
@@ -128,7 +130,7 @@ private[spark] trait ListenerBus[L <: AnyRef, E] extends Logging {
         if (maybeTimerContext != null) {
           val elapsed = maybeTimerContext.stop()
           if (logSlowEventEnabled && elapsed > logSlowEventThreshold) {
-            logInfo(s"Process of event ${event} by listener ${listenerName} took " +
+            logInfo(s"Process of event ${redactEvent(event)} by listener ${listenerName} took " +
               s"${elapsed / 1000000000d}s.")
           }
         }
@@ -150,4 +152,12 @@ private[spark] trait ListenerBus[L <: AnyRef, E] extends Logging {
     listeners.asScala.filter(_.getClass == c).map(_.asInstanceOf[T]).toSeq
   }
 
+  private def redactEvent(e: E): E = {
+    e match {
+      case event: SparkListenerEnvironmentUpdate =>
+        EventLoggingListener.redactEvent(env.conf, event).asInstanceOf[E]
+      case _ => e
+    }
+  }
+
 }
diff --git a/core/src/main/scala/org/apache/spark/util/RpcUtils.scala b/core/src/main/scala/org/apache/spark/util/RpcUtils.scala
index 7272b375e5388..0e4debc595345 100644
--- a/core/src/main/scala/org/apache/spark/util/RpcUtils.scala
+++ b/core/src/main/scala/org/apache/spark/util/RpcUtils.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.util
 
+import scala.concurrent.duration._
+
 import org.apache.spark.SparkConf
 import org.apache.spark.internal.config
 import org.apache.spark.internal.config.Network._
@@ -54,6 +56,14 @@ private[spark] object RpcUtils {
     RpcTimeout(conf, Seq(RPC_LOOKUP_TIMEOUT.key, NETWORK_TIMEOUT.key), "120s")
   }
 
+  /**
+   * Infinite timeout is used internally, so there's no timeout configuration property that
+   * controls it. Therefore, we use "infinite" without any specific reason as its timeout
+   * configuration property. And its timeout property should never be accessed since infinite
+   * means we never timeout.
+   */
+  val INFINITE_TIMEOUT = new RpcTimeout(Long.MaxValue.nanos, "infinite")
+
   private val MAX_MESSAGE_SIZE_IN_MB = Int.MaxValue / 1024 / 1024
 
   /** Returns the configured max message size for messages in bytes. */
diff --git a/core/src/main/scala/org/apache/spark/util/ThreadUtils.scala b/core/src/main/scala/org/apache/spark/util/ThreadUtils.scala
index 78206c51c1028..d45dc937910d9 100644
--- a/core/src/main/scala/org/apache/spark/util/ThreadUtils.scala
+++ b/core/src/main/scala/org/apache/spark/util/ThreadUtils.scala
@@ -23,7 +23,6 @@ import java.util.concurrent.locks.ReentrantLock
 
 import scala.concurrent.{Awaitable, ExecutionContext, ExecutionContextExecutor, Future}
 import scala.concurrent.duration.{Duration, FiniteDuration}
-import scala.language.higherKinds
 import scala.util.control.NonFatal
 
 import com.google.common.util.concurrent.ThreadFactoryBuilder
diff --git a/core/src/main/scala/org/apache/spark/util/UninterruptibleThread.scala b/core/src/main/scala/org/apache/spark/util/UninterruptibleThread.scala
index 6a58ec142dd7f..24788d69121b2 100644
--- a/core/src/main/scala/org/apache/spark/util/UninterruptibleThread.scala
+++ b/core/src/main/scala/org/apache/spark/util/UninterruptibleThread.scala
@@ -31,7 +31,7 @@ private[spark] class UninterruptibleThread(
     target: Runnable,
     name: String) extends Thread(target, name) {
 
-  def this(name: String) {
+  def this(name: String) = {
     this(null, name)
   }
 
diff --git a/core/src/main/scala/org/apache/spark/util/UninterruptibleThreadRunner.scala b/core/src/main/scala/org/apache/spark/util/UninterruptibleThreadRunner.scala
new file mode 100644
index 0000000000000..18108aa819db9
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/util/UninterruptibleThreadRunner.scala
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util
+
+import java.util.concurrent.Executors
+
+import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.duration.Duration
+
+/**
+ * [[UninterruptibleThreadRunner]] ensures that all tasks are running in an
+ * [[UninterruptibleThread]]. A good example is Kafka consumer usage.
+ */
+private[spark] class UninterruptibleThreadRunner(threadName: String) {
+  private val thread = Executors.newSingleThreadExecutor((r: Runnable) => {
+    val t = new UninterruptibleThread(threadName) {
+      override def run(): Unit = {
+        r.run()
+      }
+    }
+    t.setDaemon(true)
+    t
+  })
+  private val execContext = ExecutionContext.fromExecutorService(thread)
+
+  def runUninterruptibly[T](body: => T): T = {
+    if (!Thread.currentThread.isInstanceOf[UninterruptibleThread]) {
+      val future = Future {
+        body
+      }(execContext)
+      ThreadUtils.awaitResult(future, Duration.Inf)
+    } else {
+      body
+    }
+  }
+
+  def shutdown(): Unit = {
+    thread.shutdown()
+  }
+}
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index c7db2127a6f04..e27666b284b53 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -28,7 +28,7 @@ import java.nio.channels.{Channels, FileChannel, WritableByteChannel}
 import java.nio.charset.StandardCharsets
 import java.nio.file.Files
 import java.security.SecureRandom
-import java.util.{Arrays, Locale, Properties, Random, UUID}
+import java.util.{Locale, Properties, Random, UUID}
 import java.util.concurrent._
 import java.util.concurrent.TimeUnit.NANOSECONDS
 import java.util.zip.GZIPInputStream
@@ -53,6 +53,7 @@ import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.{FileSystem, FileUtil, Path}
 import org.apache.hadoop.io.compress.{CompressionCodecFactory, SplittableCompressionCodec}
 import org.apache.hadoop.security.UserGroupInformation
+import org.apache.hadoop.util.{RunJar, StringUtils}
 import org.apache.hadoop.yarn.conf.YarnConfiguration
 import org.eclipse.jetty.util.MultiException
 import org.slf4j.Logger
@@ -376,7 +377,7 @@ private[spark] object Utils extends Logging {
    * This returns a new InputStream which contains the same data as the original input stream.
    * It may be entirely on in-memory buffer, or it may be a combination of in-memory data, and then
    * continue to read from the original stream. The only real use of this is if the original input
-   * stream will potentially detect corruption while the data is being read (eg. from compression).
+   * stream will potentially detect corruption while the data is being read (e.g. from compression).
    * This allows for an eager check of corruption in the first maxSize bytes of data.
    *
    * @return An InputStream which includes all data from the original stream (combining buffered
@@ -486,15 +487,19 @@ private[spark] object Utils extends Logging {
    *
    * Throws SparkException if the target file already exists and has different contents than
    * the requested file.
+   *
+   * If `shouldUntar` is true, it untars the given url if it is a tar.gz or tgz into `targetDir`.
+   * This is a legacy behavior, and users should better use `spark.archives` configuration or
+   * `SparkContext.addArchive`
    */
   def fetchFile(
       url: String,
       targetDir: File,
       conf: SparkConf,
-      securityMgr: SecurityManager,
       hadoopConf: Configuration,
       timestamp: Long,
-      useCache: Boolean): File = {
+      useCache: Boolean,
+      shouldUntar: Boolean = true): File = {
     val fileName = decodeFileNameInURI(new URI(url))
     val targetFile = new File(targetDir, fileName)
     val fetchCacheEnabled = conf.getBoolean("spark.files.useFetchCache", defaultValue = true)
@@ -519,7 +524,7 @@ private[spark] object Utils extends Logging {
       val cachedFile = new File(localDir, cachedFileName)
       try {
         if (!cachedFile.exists()) {
-          doFetchFile(url, localDir, cachedFileName, conf, securityMgr, hadoopConf)
+          doFetchFile(url, localDir, cachedFileName, conf, hadoopConf)
         }
       } finally {
         lock.release()
@@ -532,16 +537,26 @@ private[spark] object Utils extends Logging {
         conf.getBoolean("spark.files.overwrite", false)
       )
     } else {
-      doFetchFile(url, targetDir, fileName, conf, securityMgr, hadoopConf)
-    }
-
-    // Decompress the file if it's a .tar or .tar.gz
-    if (fileName.endsWith(".tar.gz") || fileName.endsWith(".tgz")) {
-      logInfo("Untarring " + fileName)
-      executeAndGetOutput(Seq("tar", "-xzf", fileName), targetDir)
-    } else if (fileName.endsWith(".tar")) {
-      logInfo("Untarring " + fileName)
-      executeAndGetOutput(Seq("tar", "-xf", fileName), targetDir)
+      doFetchFile(url, targetDir, fileName, conf, hadoopConf)
+    }
+
+    if (shouldUntar) {
+      // Decompress the file if it's a .tar or .tar.gz
+      if (fileName.endsWith(".tar.gz") || fileName.endsWith(".tgz")) {
+        logWarning(
+          "Untarring behavior will be deprecated at spark.files and " +
+            "SparkContext.addFile. Consider using spark.archives or SparkContext.addArchive " +
+            "instead.")
+        logInfo("Untarring " + fileName)
+        executeAndGetOutput(Seq("tar", "-xzf", fileName), targetDir)
+      } else if (fileName.endsWith(".tar")) {
+        logWarning(
+          "Untarring behavior will be deprecated at spark.files and " +
+            "SparkContext.addFile. Consider using spark.archives or SparkContext.addArchive " +
+            "instead.")
+        logInfo("Untarring " + fileName)
+        executeAndGetOutput(Seq("tar", "-xf", fileName), targetDir)
+      }
     }
     // Make the file executable - That's necessary for scripts
     FileUtil.chmod(targetFile.getAbsolutePath, "a+x")
@@ -555,6 +570,26 @@ private[spark] object Utils extends Logging {
     targetFile
   }
 
+  /**
+   * Unpacks an archive file into the specified directory. It expects .jar, .zip, .tar.gz, .tgz
+   * and .tar files. This behaves same as Hadoop's archive in distributed cache. This method is
+   * basically copied from `org.apache.hadoop.yarn.util.FSDownload.unpack`.
+   */
+  def unpack(source: File, dest: File): Unit = {
+    val lowerSrc = StringUtils.toLowerCase(source.getName)
+    if (lowerSrc.endsWith(".jar")) {
+      RunJar.unJar(source, dest, RunJar.MATCH_ANY)
+    } else if (lowerSrc.endsWith(".zip")) {
+      FileUtil.unZip(source, dest)
+    } else if (
+      lowerSrc.endsWith(".tar.gz") || lowerSrc.endsWith(".tgz") || lowerSrc.endsWith(".tar")) {
+      FileUtil.unTar(source, dest)
+    } else {
+      logWarning(s"Cannot unpack $source, just copying it to $dest.")
+      copyRecursive(source, dest)
+    }
+  }
+
   /** Records the duration of running `body`. */
   def timeTakenMs[T](body: => T): (T, Long) = {
     val startTime = System.nanoTime()
@@ -705,7 +740,6 @@ private[spark] object Utils extends Logging {
       targetDir: File,
       filename: String,
       conf: SparkConf,
-      securityMgr: SecurityManager,
       hadoopConf: Configuration): File = {
     val targetFile = new File(targetDir, filename)
     val uri = new URI(url)
@@ -1026,13 +1060,27 @@ private[spark] object Utils extends Logging {
     customHostname.getOrElse(InetAddresses.toUriString(localIpAddress))
   }
 
+  /**
+   * Checks if the host contains only valid hostname/ip without port
+   * NOTE: Incase of IPV6 ip it should be enclosed inside []
+   */
   def checkHost(host: String): Unit = {
-    assert(host != null && host.indexOf(':') == -1, s"Expected hostname (not IP) but got $host")
+    if (host != null && host.split(":").length > 2) {
+      assert(host.startsWith("[") && host.endsWith("]"),
+        s"Expected hostname or IPv6 IP enclosed in [] but got $host")
+    } else {
+      assert(host != null && host.indexOf(':') == -1, s"Expected hostname or IP but got $host")
+    }
   }
 
   def checkHostPort(hostPort: String): Unit = {
-    assert(hostPort != null && hostPort.indexOf(':') != -1,
-      s"Expected host and port but got $hostPort")
+    if (hostPort != null && hostPort.split(":").length > 2) {
+      assert(hostPort != null && hostPort.indexOf("]:") != -1,
+        s"Expected host and port but got $hostPort")
+    } else {
+      assert(hostPort != null && hostPort.indexOf(':') != -1,
+        s"Expected host and port but got $hostPort")
+    }
   }
 
   // Typically, this will be of order of number of nodes in cluster
@@ -1046,18 +1094,30 @@ private[spark] object Utils extends Logging {
       return cached
     }
 
-    val indx: Int = hostPort.lastIndexOf(':')
-    // This is potentially broken - when dealing with ipv6 addresses for example, sigh ...
-    // but then hadoop does not support ipv6 right now.
-    // For now, we assume that if port exists, then it is valid - not check if it is an int > 0
-    if (-1 == indx) {
+    def setDefaultPortValue: (String, Int) = {
       val retval = (hostPort, 0)
       hostPortParseResults.put(hostPort, retval)
-      return retval
+      retval
+    }
+    // checks if the hostport contains IPV6 ip and parses the host, port
+    if (hostPort != null && hostPort.split(":").length > 2) {
+      val index: Int = hostPort.lastIndexOf("]:")
+      if (-1 == index) {
+        return setDefaultPortValue
+      }
+      val port = hostPort.substring(index + 2).trim()
+      val retval = (hostPort.substring(0, index + 1).trim(), if (port.isEmpty) 0 else port.toInt)
+      hostPortParseResults.putIfAbsent(hostPort, retval)
+    } else {
+      val index: Int = hostPort.lastIndexOf(':')
+      if (-1 == index) {
+        return setDefaultPortValue
+      }
+      val port = hostPort.substring(index + 1).trim()
+      val retval = (hostPort.substring(0, index).trim(), if (port.isEmpty) 0 else port.toInt)
+      hostPortParseResults.putIfAbsent(hostPort, retval)
     }
 
-    val retval = (hostPort.substring(0, indx).trim(), hostPort.substring(indx + 1).trim().toInt)
-    hostPortParseResults.putIfAbsent(hostPort, retval)
     hostPortParseResults.get(hostPort)
   }
 
@@ -1716,7 +1776,7 @@ private[spark] object Utils extends Logging {
     if (inWord || inDoubleQuote || inSingleQuote) {
       endWord()
     }
-    buf
+    buf.toSeq
   }
 
  /* Calculates 'x' modulo 'mod', takes to consideration sign of x,
@@ -1879,7 +1939,9 @@ private[spark] object Utils extends Logging {
    * Indicates whether Spark is currently running unit tests.
    */
   def isTesting: Boolean = {
-    sys.env.contains("SPARK_TESTING") || sys.props.contains(IS_TESTING.key)
+    // Scala's `sys.env` creates a ton of garbage by constructing Scala immutable maps, so
+    // we directly use the Java APIs instead.
+    System.getenv("SPARK_TESTING") != null || System.getProperty(IS_TESTING.key) != null
   }
 
   /**
@@ -2001,6 +2063,17 @@ private[spark] object Utils extends Logging {
     }
   }
 
+  /** Check whether a path is an absolute URI. */
+  def isAbsoluteURI(path: String): Boolean = {
+    try {
+      val uri = new URI(path: String)
+      uri.isAbsolute
+    } catch {
+      case _: URISyntaxException =>
+        false
+    }
+  }
+
   /** Return all non-local paths from a comma-separated list of paths. */
   def nonLocalPaths(paths: String, testWindows: Boolean = false): Array[String] = {
     val windows = isWindows || testWindows
@@ -2492,6 +2565,14 @@ private[spark] object Utils extends Logging {
     master == "local" || master.startsWith("local[")
   }
 
+  /**
+   * Push based shuffle can only be enabled when external shuffle service is enabled.
+   */
+  def isPushBasedShuffleEnabled(conf: SparkConf): Boolean = {
+    conf.get(PUSH_BASED_SHUFFLE_ENABLED) &&
+      (conf.get(IS_TESTING).getOrElse(false) || conf.get(SHUFFLE_SERVICE_ENABLED))
+  }
+
   /**
    * Return whether dynamic allocation is enabled in the given conf.
    */
@@ -2788,6 +2869,34 @@ private[spark] object Utils extends Logging {
     Hex.encodeHexString(secretBytes)
   }
 
+  /**
+   * Returns true if and only if the underlying class is a member class.
+   *
+   * Note: jdk8u throws a "Malformed class name" error if a given class is a deeply-nested
+   * inner class (See SPARK-34607 for details). This issue has already been fixed in jdk9+, so
+   * we can remove this helper method safely if we drop the support of jdk8u.
+   */
+  def isMemberClass(cls: Class[_]): Boolean = {
+    try {
+      cls.isMemberClass
+    } catch {
+      case _: InternalError =>
+        // We emulate jdk8u `Class.isMemberClass` below:
+        //   public boolean isMemberClass() {
+        //     return getSimpleBinaryName() != null && !isLocalOrAnonymousClass();
+        //   }
+        // `getSimpleBinaryName()` returns null if a given class is a top-level class,
+        // so we replace it with `cls.getEnclosingClass != null`. The second condition checks
+        // if a given class is not a local or an anonymous class, so we replace it with
+        // `cls.getEnclosingMethod == null` because `cls.getEnclosingMethod()` return a value
+        // only in either case (JVM Spec 4.8.6).
+        //
+        // Note: The newer jdk evaluates `!isLocalOrAnonymousClass()` first,
+        // we reorder the conditions to follow it.
+        cls.getEnclosingMethod == null && cls.getEnclosingClass != null
+    }
+  }
+
   /**
    * Safer than Class obj's getSimpleName which may throw Malformed class name error in scala.
    * This method mimics scalatest's getSimpleNameOfAnObjectsClass.
@@ -2820,11 +2929,11 @@ private[spark] object Utils extends Logging {
     if (lastDollarIndex < s.length - 1) {
       // The last char is not a dollar sign
       if (lastDollarIndex == -1 || !s.contains("$iw")) {
-        // The name does not have dollar sign or is not an intepreter
+        // The name does not have dollar sign or is not an interpreter
         // generated class, so we should return the full string
         s
       } else {
-        // The class name is intepreter generated,
+        // The class name is interpreter generated,
         // return the part after the last dollar sign
         // This is the same behavior as getClass.getSimpleName
         s.substring(lastDollarIndex + 1)
@@ -2857,14 +2966,14 @@ private[spark] object Utils extends Logging {
    */
   private val fullWidthRegex = ("""[""" +
     // scalastyle:off nonascii
-    """\u1100-\u115F""" +
-    """\u2E80-\uA4CF""" +
-    """\uAC00-\uD7A3""" +
-    """\uF900-\uFAFF""" +
-    """\uFE10-\uFE19""" +
-    """\uFE30-\uFE6F""" +
-    """\uFF00-\uFF60""" +
-    """\uFFE0-\uFFE6""" +
+    "\u1100-\u115F" +
+    "\u2E80-\uA4CF" +
+    "\uAC00-\uD7A3" +
+    "\uF900-\uFAFF" +
+    "\uFE10-\uFE19" +
+    "\uFE30-\uFE6F" +
+    "\uFF00-\uFF60" +
+    "\uFFE0-\uFFE6" +
     // scalastyle:on nonascii
     """]""").r
 
@@ -2900,10 +3009,58 @@ private[spark] object Utils extends Logging {
 
   /** Create a new properties object with the same values as `props` */
   def cloneProperties(props: Properties): Properties = {
+    if (props == null) {
+      return props
+    }
     val resultProps = new Properties()
     props.forEach((k, v) => resultProps.put(k, v))
     resultProps
   }
+
+  /**
+   * Convert a sequence of `Path`s to a metadata string. When the length of metadata string
+   * exceeds `stopAppendingThreshold`, stop appending paths for saving memory.
+   */
+  def buildLocationMetadata(paths: Seq[Path], stopAppendingThreshold: Int): String = {
+    val metadata = new StringBuilder(s"(${paths.length} paths)[")
+    var index: Int = 0
+    while (index < paths.length && metadata.length < stopAppendingThreshold) {
+      if (index > 0) {
+        metadata.append(", ")
+      }
+      metadata.append(paths(index).toString)
+      index += 1
+    }
+    if (paths.length > index) {
+      if (index > 0) {
+        metadata.append(", ")
+      }
+      metadata.append("...")
+    }
+    metadata.append("]")
+    metadata.toString
+  }
+
+  /**
+   * Convert MEMORY_OFFHEAP_SIZE to MB Unit, return 0 if MEMORY_OFFHEAP_ENABLED is false.
+   */
+  def executorOffHeapMemorySizeAsMb(sparkConf: SparkConf): Int = {
+    val sizeInMB = Utils.memoryStringToMb(sparkConf.get(MEMORY_OFFHEAP_SIZE).toString)
+    checkOffHeapEnabled(sparkConf, sizeInMB).toInt
+  }
+
+  /**
+   * return 0 if MEMORY_OFFHEAP_ENABLED is false.
+   */
+  def checkOffHeapEnabled(sparkConf: SparkConf, offHeapSize: Long): Long = {
+    if (sparkConf.get(MEMORY_OFFHEAP_ENABLED)) {
+      require(offHeapSize > 0,
+        s"${MEMORY_OFFHEAP_SIZE.key} must be > 0 when ${MEMORY_OFFHEAP_ENABLED.key} == true")
+      offHeapSize
+    } else {
+      0
+    }
+  }
 }
 
 private[util] object CallerContext extends Logging {
diff --git a/core/src/main/scala/org/apache/spark/util/VersionUtils.scala b/core/src/main/scala/org/apache/spark/util/VersionUtils.scala
index c0f8866dd58dc..e97d1c9393701 100644
--- a/core/src/main/scala/org/apache/spark/util/VersionUtils.scala
+++ b/core/src/main/scala/org/apache/spark/util/VersionUtils.scala
@@ -24,6 +24,7 @@ private[spark] object VersionUtils {
 
   private val majorMinorRegex = """^(\d+)\.(\d+)(\..*)?$""".r
   private val shortVersionRegex = """^(\d+\.\d+\.\d+)(.*)?$""".r
+  private val majorMinorPatchRegex = """^(\d+)(?:\.(\d+)(?:\.(\d+)(?:[.-].*)?)?)?$""".r
 
   /**
    * Given a Spark version string, return the major version number.
@@ -63,4 +64,36 @@ private[spark] object VersionUtils {
           s" version string, but it could not find the major and minor version numbers.")
     }
   }
+
+  /**
+   * Extracts the major, minor and patch parts from the input `version`. Note that if minor or patch
+   * version is missing from the input, this will return 0 for these parts. Returns `None` if the
+   * input is not of a valid format.
+   *
+   * Examples of valid version:
+   *  - 1   (extracts to (1, 0, 0))
+   *  - 2.4   (extracts to (2, 4, 0))
+   *  - 3.2.2   (extracts to (3, 2, 2))
+   *  - 3.2.2.4   (extracts to 3, 2, 2))
+   *  - 3.3.1-SNAPSHOT   (extracts to (3, 3, 1))
+   *  - 3.2.2.4SNAPSHOT   (extracts to (3, 2, 2), only the first 3 components)
+   *
+   * Examples of invalid version:
+   *  - ABC
+   *  - 1X
+   *  - 2.4XYZ
+   *  - 2.4-SNAPSHOT
+   *  - 3.4.5ABC
+   *
+   *  @return A non-empty option containing a 3-value tuple (major, minor, patch) iff the
+   *          input is a valid version. `None` otherwise.
+   */
+  def majorMinorPatchVersion(version: String): Option[(Int, Int, Int)] = {
+    majorMinorPatchRegex.findFirstMatchIn(version).map { m =>
+      val major = m.group(1).toInt
+      val minor = Option(m.group(2)).map(_.toInt).getOrElse(0)
+      val patch = Option(m.group(3)).map(_.toInt).getOrElse(0)
+      (major, minor, patch)
+    }
+  }
 }
diff --git a/core/src/main/scala/org/apache/spark/util/collection/BitSet.scala b/core/src/main/scala/org/apache/spark/util/collection/BitSet.scala
index 098f389829ec5..61386114997f6 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/BitSet.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/BitSet.scala
@@ -236,6 +236,18 @@ class BitSet(numBits: Int) extends Serializable {
     -1
   }
 
+  /**
+   * Compute bit-wise union with another BitSet and overwrite bits in this BitSet with the result.
+   */
+  def union(other: BitSet): Unit = {
+    require(this.numWords <= other.numWords)
+    var ind = 0
+    while( ind < this.numWords ) {
+      this.words(ind) = this.words(ind) | other.words(ind)
+      ind += 1
+    }
+  }
+
   /** Return the number of longs it would take to hold numBits. */
   private def bit2words(numBits: Int) = ((numBits - 1) >> 6) + 1
 }
diff --git a/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala b/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
index 7f40b469a95e9..731131b688ca7 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
@@ -76,7 +76,7 @@ class ExternalAppendOnlyMap[K, V, C](
       mergeValue: (C, V) => C,
       mergeCombiners: (C, C) => C,
       serializer: Serializer,
-      blockManager: BlockManager) {
+      blockManager: BlockManager) = {
     this(createCombiner, mergeValue, mergeCombiners, serializer, blockManager, TaskContext.get())
   }
 
diff --git a/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala b/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala
index cc97bbfa7201f..1913637371e31 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala
@@ -32,7 +32,7 @@ import org.apache.spark.serializer._
 import org.apache.spark.shuffle.ShufflePartitionPairsWriter
 import org.apache.spark.shuffle.api.{ShuffleMapOutputWriter, ShufflePartitionWriter}
 import org.apache.spark.storage.{BlockId, DiskBlockObjectWriter, ShuffleBlockId}
-import org.apache.spark.util.{Utils => TryUtils}
+import org.apache.spark.util.{CompletionIterator, Utils => TryUtils}
 
 /**
  * Sorts and potentially merges a number of key-value pairs of type (K, V) to produce key-combiner
@@ -263,7 +263,7 @@ private[spark] class ExternalSorter[K, V, C](
   /**
    * Spill contents of in-memory iterator to a temporary file on disk.
    */
-  private[this] def spillMemoryIteratorToDisk(inMemoryIterator: WritablePartitionedIterator)
+  private[this] def spillMemoryIteratorToDisk(inMemoryIterator: WritablePartitionedIterator[K, C])
       : SpilledFile = {
     // Because these files may be read during shuffle, their compression must be controlled by
     // spark.shuffle.compress instead of spark.shuffle.spill.compress, so we need to use
@@ -659,7 +659,7 @@ private[spark] class ExternalSorter[K, V, C](
       }
     } else {
       // Merge spilled and in-memory data
-      merge(spills, destructiveIterator(
+      merge(spills.toSeq, destructiveIterator(
         collection.partitionedDestructiveSortedIterator(comparator)))
     }
   }
@@ -672,6 +672,22 @@ private[spark] class ExternalSorter[K, V, C](
     partitionedIterator.flatMap(pair => pair._2)
   }
 
+  /**
+   * Insert all records, updates related task metrics, and return a completion iterator
+   * over all the data written to this object, aggregated by our aggregator.
+   * On task completion (success, failure, or cancellation), it releases resources by
+   * calling `stop()`.
+   */
+  def insertAllAndUpdateMetrics(records: Iterator[Product2[K, V]]): Iterator[Product2[K, C]] = {
+    insertAll(records)
+    context.taskMetrics().incMemoryBytesSpilled(memoryBytesSpilled)
+    context.taskMetrics().incDiskBytesSpilled(diskBytesSpilled)
+    context.taskMetrics().incPeakExecutionMemory(peakMemoryUsedBytes)
+    // Use completion callback to stop sorter if task was finished/cancelled.
+    context.addTaskCompletionListener[Unit](_ => stop())
+    CompletionIterator[Product2[K, C], Iterator[Product2[K, C]]](iterator, stop())
+  }
+
   /**
    * TODO(SPARK-28764): remove this, as this is only used by UnsafeRowSerializerSuite in the SQL
    * project. We should figure out an alternative way to test that so that we can remove this
@@ -734,7 +750,7 @@ private[spark] class ExternalSorter[K, V, C](
       // Case where we only have in-memory data
       val collection = if (aggregator.isDefined) map else buffer
       val it = collection.destructiveSortedWritablePartitionedIterator(comparator)
-      while (it.hasNext()) {
+      while (it.hasNext) {
         val partitionId = it.nextPartition()
         var partitionWriter: ShufflePartitionWriter = null
         var partitionPairsWriter: ShufflePartitionPairsWriter = null
@@ -850,18 +866,7 @@ private[spark] class ExternalSorter[K, V, C](
       if (hasSpilled) {
         false
       } else {
-        val inMemoryIterator = new WritablePartitionedIterator {
-          private[this] var cur = if (upstream.hasNext) upstream.next() else null
-
-          def writeNext(writer: PairsWriter): Unit = {
-            writer.write(cur._1._2, cur._2)
-            cur = if (upstream.hasNext) upstream.next() else null
-          }
-
-          def hasNext(): Boolean = cur != null
-
-          def nextPartition(): Int = cur._1._1
-        }
+        val inMemoryIterator = new WritablePartitionedIterator[K, C](upstream)
         logInfo(s"Task ${TaskContext.get().taskAttemptId} force spilling in-memory map to disk " +
           s"and it will release ${org.apache.spark.util.Utils.bytesToString(getUsed())} memory")
         val spillFile = spillMemoryIteratorToDisk(inMemoryIterator)
diff --git a/core/src/main/scala/org/apache/spark/util/collection/ImmutableBitSet.scala b/core/src/main/scala/org/apache/spark/util/collection/ImmutableBitSet.scala
new file mode 100644
index 0000000000000..82413f4317d62
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/util/collection/ImmutableBitSet.scala
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util.collection
+
+private object ErrorMessage {
+  final val msg: String = "mutable operation is not supported"
+}
+
+// An immutable BitSet that initializes set bits in its constructor.
+class ImmutableBitSet(val numBits: Int, val bitsToSet: Int*) extends BitSet(numBits) {
+
+  // Initialize the set bits.
+  {
+    val bitsIterator = bitsToSet.iterator
+    while (bitsIterator.hasNext) {
+      super.set(bitsIterator.next)
+    }
+  }
+
+  override def clear(): Unit = {
+    throw new UnsupportedOperationException(ErrorMessage.msg)
+  }
+
+  override def clearUntil(bitIndex: Int): Unit = {
+    throw new UnsupportedOperationException(ErrorMessage.msg)
+  }
+
+  override def set(index: Int): Unit = {
+    throw new UnsupportedOperationException(ErrorMessage.msg)
+  }
+
+  override def setUntil(bitIndex: Int): Unit = {
+    throw new UnsupportedOperationException(ErrorMessage.msg)
+  }
+
+  override def unset(index: Int): Unit = {
+    throw new UnsupportedOperationException(ErrorMessage.msg)
+  }
+
+  override def union(other: BitSet): Unit = {
+    throw new UnsupportedOperationException(ErrorMessage.msg)
+  }
+}
diff --git a/core/src/main/scala/org/apache/spark/util/collection/MedianHeap.scala b/core/src/main/scala/org/apache/spark/util/collection/MedianHeap.scala
index 6e57c3c5bee8c..f1a3932bb0e25 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/MedianHeap.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/MedianHeap.scala
@@ -37,13 +37,13 @@ private[spark] class MedianHeap(implicit val ord: Ordering[Double]) {
    * Stores all the numbers less than the current median in a smallerHalf,
    * i.e median is the maximum, at the root.
    */
-  private[this] var smallerHalf = PriorityQueue.empty[Double](ord)
+  private[this] val smallerHalf = PriorityQueue.empty[Double](ord)
 
   /**
    * Stores all the numbers greater than the current median in a largerHalf,
    * i.e median is the minimum, at the root.
    */
-  private[this] var largerHalf = PriorityQueue.empty[Double](ord.reverse)
+  private[this] val largerHalf = PriorityQueue.empty[Double](ord.reverse)
 
   def isEmpty(): Boolean = {
     smallerHalf.isEmpty && largerHalf.isEmpty
diff --git a/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala b/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala
index 1983b0002853d..fe488f9cf0daf 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala
@@ -27,7 +27,7 @@ import org.apache.spark.memory.{MemoryConsumer, MemoryMode, TaskMemoryManager}
  * has been exceeded.
  */
 private[spark] abstract class Spillable[C](taskMemoryManager: TaskMemoryManager)
-  extends MemoryConsumer(taskMemoryManager) with Logging {
+  extends MemoryConsumer(taskMemoryManager, MemoryMode.ON_HEAP) with Logging {
   /**
    * Spills the current in-memory collection to disk, and releases the memory.
    *
diff --git a/core/src/main/scala/org/apache/spark/util/collection/WritablePartitionedPairCollection.scala b/core/src/main/scala/org/apache/spark/util/collection/WritablePartitionedPairCollection.scala
index 9624b02cb407c..3472a08cc329c 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/WritablePartitionedPairCollection.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/WritablePartitionedPairCollection.scala
@@ -46,20 +46,9 @@ private[spark] trait WritablePartitionedPairCollection[K, V] {
    * This may destroy the underlying collection.
    */
   def destructiveSortedWritablePartitionedIterator(keyComparator: Option[Comparator[K]])
-    : WritablePartitionedIterator = {
+    : WritablePartitionedIterator[K, V] = {
     val it = partitionedDestructiveSortedIterator(keyComparator)
-    new WritablePartitionedIterator {
-      private[this] var cur = if (it.hasNext) it.next() else null
-
-      def writeNext(writer: PairsWriter): Unit = {
-        writer.write(cur._1._2, cur._2)
-        cur = if (it.hasNext) it.next() else null
-      }
-
-      def hasNext(): Boolean = cur != null
-
-      def nextPartition(): Int = cur._1._1
-    }
+    new WritablePartitionedIterator[K, V](it)
   }
 }
 
@@ -87,10 +76,15 @@ private[spark] object WritablePartitionedPairCollection {
  * Iterator that writes elements to a DiskBlockObjectWriter instead of returning them. Each element
  * has an associated partition.
  */
-private[spark] trait WritablePartitionedIterator {
-  def writeNext(writer: PairsWriter): Unit
+private[spark] class WritablePartitionedIterator[K, V](it: Iterator[((Int, K), V)]) {
+  private[this] var cur = if (it.hasNext) it.next() else null
+
+  def writeNext(writer: PairsWriter): Unit = {
+    writer.write(cur._1._2, cur._2)
+    cur = if (it.hasNext) it.next() else null
+  }
 
-  def hasNext(): Boolean
+  def hasNext: Boolean = cur != null
 
-  def nextPartition(): Int
+  def nextPartition(): Int = cur._1._1
 }
diff --git a/core/src/main/scala/org/apache/spark/util/io/ChunkedByteBuffer.scala b/core/src/main/scala/org/apache/spark/util/io/ChunkedByteBuffer.scala
index 2c3730de08b5b..8635f1a3d702e 100644
--- a/core/src/main/scala/org/apache/spark/util/io/ChunkedByteBuffer.scala
+++ b/core/src/main/scala/org/apache/spark/util/io/ChunkedByteBuffer.scala
@@ -193,7 +193,7 @@ private[spark] object ChunkedByteBuffer {
       length: Long): ChunkedByteBuffer = {
     // We do *not* memory map the file, because we may end up putting this into the memory store,
     // and spark currently is not expecting memory-mapped buffers in the memory store, it conflicts
-    // with other parts that manage the lifecyle of buffers and dispose them.  See SPARK-25422.
+    // with other parts that manage the lifecycle of buffers and dispose them.  See SPARK-25422.
     val is = new FileInputStream(file)
     ByteStreams.skipFully(is, offset)
     val in = new LimitedInputStream(is, length)
diff --git a/core/src/test/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriterSuite.java b/core/src/test/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriterSuite.java
index ee8e38c24b47f..5666bb3e5f140 100644
--- a/core/src/test/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriterSuite.java
+++ b/core/src/test/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriterSuite.java
@@ -68,10 +68,10 @@
 public class UnsafeShuffleWriterSuite {
 
   static final int DEFAULT_INITIAL_SORT_BUFFER_SIZE = 4096;
-  static final int NUM_PARTITITONS = 4;
+  static final int NUM_PARTITIONS = 4;
   TestMemoryManager memoryManager;
   TaskMemoryManager taskMemoryManager;
-  final HashPartitioner hashPartitioner = new HashPartitioner(NUM_PARTITITONS);
+  final HashPartitioner hashPartitioner = new HashPartitioner(NUM_PARTITIONS);
   File mergedOutputFile;
   File tempDir;
   long[] partitionSizesInMergedFile;
@@ -97,8 +97,8 @@ public void tearDown() {
 
   @Before
   @SuppressWarnings("unchecked")
-  public void setUp() throws IOException {
-    MockitoAnnotations.initMocks(this);
+  public void setUp() throws Exception {
+    MockitoAnnotations.openMocks(this).close();
     tempDir = Utils.createTempDir(null, "test");
     mergedOutputFile = File.createTempFile("mergedoutput", "", tempDir);
     partitionSizesInMergedFile = null;
@@ -194,7 +194,7 @@ private void assertSpillFilesWereCleanedUp() {
   private List<Tuple2<Object, Object>> readRecordsFromFile() throws IOException {
     final ArrayList<Tuple2<Object, Object>> recordsList = new ArrayList<>();
     long startOffset = 0;
-    for (int i = 0; i < NUM_PARTITITONS; i++) {
+    for (int i = 0; i < NUM_PARTITIONS; i++) {
       final long partitionSize = partitionSizesInMergedFile[i];
       if (partitionSize > 0) {
         FileInputStream fin = new FileInputStream(mergedOutputFile);
@@ -253,7 +253,7 @@ public void writeEmptyIterator() throws Exception {
     assertTrue(mapStatus.isDefined());
     assertTrue(mergedOutputFile.exists());
     assertEquals(0, spillFilesCreated.size());
-    assertArrayEquals(new long[NUM_PARTITITONS], partitionSizesInMergedFile);
+    assertArrayEquals(new long[NUM_PARTITIONS], partitionSizesInMergedFile);
     assertEquals(0, taskMetrics.shuffleWriteMetrics().recordsWritten());
     assertEquals(0, taskMetrics.shuffleWriteMetrics().bytesWritten());
     assertEquals(0, taskMetrics.diskBytesSpilled());
@@ -264,7 +264,7 @@ public void writeEmptyIterator() throws Exception {
   public void writeWithoutSpilling() throws Exception {
     // In this example, each partition should have exactly one record:
     final ArrayList<Product2<Object, Object>> dataToWrite = new ArrayList<>();
-    for (int i = 0; i < NUM_PARTITITONS; i++) {
+    for (int i = 0; i < NUM_PARTITIONS; i++) {
       dataToWrite.add(new Tuple2<>(i, i));
     }
     final UnsafeShuffleWriter<Object, Object> writer = createWriter(true);
diff --git a/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java b/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java
index 6e995a3929a75..fec205924938a 100644
--- a/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java
+++ b/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java
@@ -24,6 +24,7 @@
 
 import scala.Tuple2$;
 
+import org.hamcrest.MatcherAssert;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -75,7 +76,7 @@ public abstract class AbstractBytesToBytesMapSuite {
   @Mock(answer = RETURNS_SMART_NULLS) DiskBlockManager diskBlockManager;
 
   @Before
-  public void setup() {
+  public void setup() throws Exception {
     memoryManager =
       new TestMemoryManager(
         new SparkConf()
@@ -87,7 +88,7 @@ public void setup() {
 
     tempDir = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "unsafe-test");
     spillFilesCreated.clear();
-    MockitoAnnotations.initMocks(this);
+    MockitoAnnotations.openMocks(this).close();
     when(blockManager.diskBlockManager()).thenReturn(diskBlockManager);
     when(diskBlockManager.createTempLocalBlock()).thenAnswer(invocationOnMock -> {
       TempLocalBlockId blockId = new TempLocalBlockId(UUID.randomUUID());
@@ -172,6 +173,7 @@ public void emptyMap() {
       final byte[] key = getRandomByteArray(keyLengthInWords);
       Assert.assertFalse(map.lookup(key, Platform.BYTE_ARRAY_OFFSET, keyLengthInBytes).isDefined());
       Assert.assertFalse(map.iterator().hasNext());
+      Assert.assertFalse(map.iteratorWithKeyIndex().hasNext());
     } finally {
       map.free();
     }
@@ -233,9 +235,10 @@ public void setAndRetrieveAKey() {
     }
   }
 
-  private void iteratorTestBase(boolean destructive) throws Exception {
+  private void iteratorTestBase(boolean destructive, boolean isWithKeyIndex) throws Exception {
     final int size = 4096;
     BytesToBytesMap map = new BytesToBytesMap(taskMemoryManager, size / 2, PAGE_SIZE_BYTES);
+    Assert.assertEquals(size / 2, map.maxNumKeysIndex());
     try {
       for (long i = 0; i < size; i++) {
         final long[] value = new long[] { i };
@@ -267,6 +270,8 @@ private void iteratorTestBase(boolean destructive) throws Exception {
       final Iterator<BytesToBytesMap.Location> iter;
       if (destructive) {
         iter = map.destructiveIterator();
+      } else if (isWithKeyIndex) {
+        iter = map.iteratorWithKeyIndex();
       } else {
         iter = map.iterator();
       }
@@ -291,6 +296,12 @@ private void iteratorTestBase(boolean destructive) throws Exception {
             countFreedPages++;
           }
         }
+        if (keyLength != 0 && isWithKeyIndex) {
+          final BytesToBytesMap.Location expectedLoc = map.lookup(
+            loc.getKeyBase(), loc.getKeyOffset(), loc.getKeyLength());
+          Assert.assertTrue(expectedLoc.isDefined() &&
+            expectedLoc.getKeyIndex() == loc.getKeyIndex());
+        }
       }
       if (destructive) {
         // Latest page is not freed by iterator but by map itself
@@ -304,12 +315,17 @@ private void iteratorTestBase(boolean destructive) throws Exception {
 
   @Test
   public void iteratorTest() throws Exception {
-    iteratorTestBase(false);
+    iteratorTestBase(false, false);
   }
 
   @Test
   public void destructiveIteratorTest() throws Exception {
-    iteratorTestBase(true);
+    iteratorTestBase(true, false);
+  }
+
+  @Test
+  public void iteratorWithKeyIndexTest() throws Exception {
+    iteratorTestBase(false, true);
   }
 
   @Test
@@ -524,7 +540,7 @@ public void failureToGrow() {
           break;
         }
       }
-      Assert.assertThat(i, greaterThan(0));
+      MatcherAssert.assertThat(i, greaterThan(0));
       Assert.assertFalse(success);
     } finally {
       map.free();
@@ -561,6 +577,8 @@ public void spillInIterator() throws IOException {
         iter2.next();
       }
       assertFalse(iter2.hasNext());
+      // calls hasNext twice deliberately, make sure it's idempotent
+      assertFalse(iter2.hasNext());
     } finally {
       map.free();
       for (File spillFile : spillFilesCreated) {
@@ -603,6 +621,12 @@ public void multipleValuesForSameKey() {
         final BytesToBytesMap.Location loc = iter.next();
         assert loc.isDefined();
       }
+      BytesToBytesMap.MapIteratorWithKeyIndex iterWithKeyIndex = map.iteratorWithKeyIndex();
+      for (i = 0; i < 2048; i++) {
+        assert iterWithKeyIndex.hasNext();
+        final BytesToBytesMap.Location loc = iterWithKeyIndex.next();
+        assert loc.isDefined() && loc.getKeyIndex() >= 0;
+      }
     } finally {
       map.free();
     }
diff --git a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java
index 43977717f6c97..b96f2cc0d7159 100644
--- a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java
+++ b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java
@@ -23,9 +23,9 @@
 import java.util.LinkedList;
 import java.util.UUID;
 
-import org.hamcrest.Matchers;
 import scala.Tuple2$;
 
+import org.hamcrest.MatcherAssert;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -38,7 +38,6 @@
 import org.apache.spark.executor.TaskMetrics;
 import org.apache.spark.internal.config.package$;
 import org.apache.spark.memory.TestMemoryManager;
-import org.apache.spark.memory.SparkOutOfMemoryError;
 import org.apache.spark.memory.TaskMemoryManager;
 import org.apache.spark.serializer.JavaSerializer;
 import org.apache.spark.serializer.SerializerInstance;
@@ -95,8 +94,8 @@ public int compare(
     (int) conf.get(package$.MODULE$.SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD());
 
   @Before
-  public void setUp() {
-    MockitoAnnotations.initMocks(this);
+  public void setUp() throws Exception {
+    MockitoAnnotations.openMocks(this).close();
     tempDir = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "unsafe-test");
     spillFilesCreated.clear();
     taskContext = mock(TaskContext.class);
@@ -226,7 +225,7 @@ public void testSortTimeMetric() throws Exception {
 
     sorter.insertRecord(null, 0, 0, 0, false);
     sorter.spill();
-    assertThat(sorter.getSortTimeNanos(), greaterThan(prevSortTime));
+    MatcherAssert.assertThat(sorter.getSortTimeNanos(), greaterThan(prevSortTime));
     prevSortTime = sorter.getSortTimeNanos();
 
     sorter.spill();  // no sort needed
@@ -234,7 +233,7 @@ public void testSortTimeMetric() throws Exception {
 
     sorter.insertRecord(null, 0, 0, 0, false);
     UnsafeSorterIterator iter = sorter.getSortedIterator();
-    assertThat(sorter.getSortTimeNanos(), greaterThan(prevSortTime));
+    MatcherAssert.assertThat(sorter.getSortTimeNanos(), greaterThan(prevSortTime));
 
     sorter.cleanupResources();
     assertSpillFilesWereCleanedUp();
@@ -253,7 +252,7 @@ public void spillingOccursInResponseToMemoryPressure() throws Exception {
     // The insertion of this record should trigger a spill:
     insertNumber(sorter, 0);
     // Ensure that spill files were created
-    assertThat(tempDir.listFiles().length, greaterThanOrEqualTo(1));
+    MatcherAssert.assertThat(tempDir.listFiles().length, greaterThanOrEqualTo(1));
     // Read back the sorted data:
     UnsafeSorterIterator iter = sorter.getSortedIterator();
 
@@ -359,6 +358,69 @@ public void forcedSpillingWithReadIterator() throws Exception {
     assertSpillFilesWereCleanedUp();
   }
 
+  @Test
+  public void forcedSpillingNullsWithReadIterator() throws Exception {
+    final UnsafeExternalSorter sorter = newSorter();
+    long[] record = new long[100];
+    final int recordSize = record.length * 8;
+    final int n = (int) pageSizeBytes / recordSize * 3;
+    for (int i = 0; i < n; i++) {
+      boolean isNull = i % 2 == 0;
+      sorter.insertRecord(record, Platform.LONG_ARRAY_OFFSET, recordSize, 0, isNull);
+    }
+    assertTrue(sorter.getNumberOfAllocatedPages() >= 2);
+
+    UnsafeExternalSorter.SpillableIterator iter =
+            (UnsafeExternalSorter.SpillableIterator) sorter.getSortedIterator();
+    final int numRecordsToReadBeforeSpilling = n / 3;
+    for (int i = 0; i < numRecordsToReadBeforeSpilling; i++) {
+      assertTrue(iter.hasNext());
+      iter.loadNext();
+    }
+
+    assertTrue(iter.spill() > 0);
+    assertEquals(0, iter.spill());
+
+    for (int i = numRecordsToReadBeforeSpilling; i < n; i++) {
+      assertTrue(iter.hasNext());
+      iter.loadNext();
+    }
+    assertFalse(iter.hasNext());
+
+    sorter.cleanupResources();
+    assertSpillFilesWereCleanedUp();
+  }
+
+  @Test
+  public void forcedSpillingWithFullyReadIterator() throws Exception {
+    final UnsafeExternalSorter sorter = newSorter();
+    long[] record = new long[100];
+    final int recordSize = record.length * 8;
+    final int n = (int) pageSizeBytes / recordSize * 3;
+    for (int i = 0; i < n; i++) {
+      record[0] = i;
+      sorter.insertRecord(record, Platform.LONG_ARRAY_OFFSET, recordSize, 0, false);
+    }
+    assertTrue(sorter.getNumberOfAllocatedPages() >= 2);
+
+    UnsafeExternalSorter.SpillableIterator iter =
+            (UnsafeExternalSorter.SpillableIterator) sorter.getSortedIterator();
+    for (int i = 0; i < n; i++) {
+      assertTrue(iter.hasNext());
+      iter.loadNext();
+      assertEquals(i, Platform.getLong(iter.getBaseObject(), iter.getBaseOffset()));
+    }
+    assertFalse(iter.hasNext());
+
+    assertTrue(iter.spill() > 0);
+    assertEquals(0, iter.spill());
+    assertEquals(n - 1, Platform.getLong(iter.getBaseObject(), iter.getBaseOffset()));
+    assertFalse(iter.hasNext());
+
+    sorter.cleanupResources();
+    assertSpillFilesWereCleanedUp();
+  }
+
   @Test
   public void forcedSpillingWithNotReadIterator() throws Exception {
     final UnsafeExternalSorter sorter = newSorter();
@@ -518,40 +580,28 @@ public void testGetIterator() throws Exception {
   }
 
   @Test
-  public void testOOMDuringSpill() throws Exception {
+  public void testNoOOMDuringSpill() throws Exception {
     final UnsafeExternalSorter sorter = newSorter();
-    // we assume that given default configuration,
-    // the size of the data we insert to the sorter (ints)
-    // and assuming we shouldn't spill before pointers array is exhausted
-    // (memory manager is not configured to throw at this point)
-    // - so this loop runs a reasonable number of iterations (<2000).
-    // test indeed completed within <30ms (on a quad i7 laptop).
-    for (int i = 0; sorter.hasSpaceForAnotherRecord(); ++i) {
+    for (int i = 0; i < 100; i++) {
       insertNumber(sorter, i);
     }
-    // we expect the next insert to attempt growing the pointerssArray first
-    // allocation is expected to fail, then a spill is triggered which
-    // attempts another allocation which also fails and we expect to see this
-    // OOM here.  the original code messed with a released array within the
-    // spill code and ended up with a failed assertion.  we also expect the
-    // location of the OOM to be
-    // org.apache.spark.util.collection.unsafe.sort.UnsafeInMemorySorter.reset
-    memoryManager.markconsequentOOM(2);
-    try {
-      insertNumber(sorter, 1024);
-      fail("expected OutOfMmoryError but it seems operation surprisingly succeeded");
-    }
-    // we expect an SparkOutOfMemoryError here, anything else (i.e the original NPE is a failure)
-    catch (SparkOutOfMemoryError oom){
-      String oomStackTrace = Utils.exceptionString(oom);
-      assertThat("expected SparkOutOfMemoryError in " +
-        "org.apache.spark.util.collection.unsafe.sort.UnsafeInMemorySorter.reset",
-        oomStackTrace,
-        Matchers.containsString(
-          "org.apache.spark.util.collection.unsafe.sort.UnsafeInMemorySorter.reset"));
+
+    // Check that spilling still succeeds when the task is starved for memory.
+    memoryManager.markconsequentOOM(Integer.MAX_VALUE);
+    sorter.spill();
+    memoryManager.resetConsequentOOM();
+
+    // Ensure that records can be appended after spilling, i.e. check that the sorter will allocate
+    // the new pointer array that it could not allocate while spilling.
+    for (int i = 0; i < 100; ++i) {
+      insertNumber(sorter, i);
     }
+
+    sorter.cleanupResources();
+    assertSpillFilesWereCleanedUp();
   }
 
+
   private void verifyIntIterator(UnsafeSorterIterator iter, int start, int end)
       throws IOException {
     for (int i = start; i < end; i++) {
diff --git a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java
index 2b8a0602730e1..9d4909ddce792 100644
--- a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java
+++ b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java
@@ -20,6 +20,7 @@
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 
+import org.apache.spark.unsafe.array.LongArray;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -27,7 +28,6 @@
 import org.apache.spark.SparkConf;
 import org.apache.spark.memory.TestMemoryConsumer;
 import org.apache.spark.memory.TestMemoryManager;
-import org.apache.spark.memory.SparkOutOfMemoryError;
 import org.apache.spark.memory.TaskMemoryManager;
 import org.apache.spark.unsafe.Platform;
 import org.apache.spark.unsafe.memory.MemoryBlock;
@@ -37,7 +37,6 @@
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.isIn;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
 import static org.mockito.Mockito.mock;
 
 public class UnsafeInMemorySorterSuite {
@@ -147,7 +146,7 @@ public int compare(
   }
 
   @Test
-  public void freeAfterOOM() {
+  public void testNoOOMDuringReset() {
     final SparkConf sparkConf = new SparkConf();
     sparkConf.set(package$.MODULE$.MEMORY_OFFHEAP_ENABLED(), false);
 
@@ -156,12 +155,7 @@ public void freeAfterOOM() {
     final TaskMemoryManager memoryManager = new TaskMemoryManager(
             testMemoryManager, 0);
     final TestMemoryConsumer consumer = new TestMemoryConsumer(memoryManager);
-    final MemoryBlock dataPage = memoryManager.allocatePage(2048, consumer);
-    final Object baseObject = dataPage.getBaseObject();
-    // Write the records into the data page:
-    long position = dataPage.getBaseOffset();
 
-    final HashPartitioner hashPartitioner = new HashPartitioner(4);
     // Use integer comparison for comparing prefixes (which are partition ids, in this case)
     final PrefixComparator prefixComparator = PrefixComparators.LONG;
     final RecordComparator recordComparator = new RecordComparator() {
@@ -179,18 +173,24 @@ public int compare(
     UnsafeInMemorySorter sorter = new UnsafeInMemorySorter(consumer, memoryManager,
             recordComparator, prefixComparator, 100, shouldUseRadixSort());
 
-    testMemoryManager.markExecutionAsOutOfMemoryOnce();
-    try {
-      sorter.reset();
-      fail("expected SparkOutOfMemoryError but it seems operation surprisingly succeeded");
-    } catch (SparkOutOfMemoryError oom) {
-      // as expected
-    }
-    // [SPARK-21907] this failed on NPE at
-    // org.apache.spark.memory.MemoryConsumer.freeArray(MemoryConsumer.java:108)
-    sorter.free();
-    // simulate a 'back to back' free.
-    sorter.free();
+    // Ensure that the sorter does not OOM while freeing its memory.
+    testMemoryManager.markconsequentOOM(Integer.MAX_VALUE);
+    sorter.freeMemory();
+    testMemoryManager.resetConsequentOOM();
+    Assert.assertFalse(sorter.hasSpaceForAnotherRecord());
+
+    // Get the sorter in an usable state again by allocating a new pointer array.
+    LongArray array = consumer.allocateArray(1000);
+    sorter.expandPointerArray(array);
+
+    // Ensure that it is safe to call freeMemory() multiple times.
+    testMemoryManager.markconsequentOOM(Integer.MAX_VALUE);
+    sorter.freeMemory();
+    sorter.freeMemory();
+    testMemoryManager.resetConsequentOOM();
+    Assert.assertFalse(sorter.hasSpaceForAnotherRecord());
+
+    assertEquals(0L, memoryManager.cleanUpAllAllocatedMemory());
   }
 
 }
diff --git a/core/src/test/java/test/org/apache/spark/JavaAPISuite.java b/core/src/test/java/test/org/apache/spark/JavaAPISuite.java
index dbaca71c5fdc3..e73ac0e9fb7a6 100644
--- a/core/src/test/java/test/org/apache/spark/JavaAPISuite.java
+++ b/core/src/test/java/test/org/apache/spark/JavaAPISuite.java
@@ -1518,7 +1518,7 @@ public void testAsyncActionErrorWrapping() throws Exception {
     JavaFutureAction<Long> future = rdd.map(new BuggyMapFunction<>()).countAsync();
     try {
       future.get(2, TimeUnit.SECONDS);
-      fail("Expected future.get() for failed job to throw ExcecutionException");
+      fail("Expected future.get() for failed job to throw ExecutionException");
     } catch (ExecutionException ee) {
       assertTrue(Throwables.getStackTraceAsString(ee).contains("Custom exception!"));
     }
diff --git a/core/src/test/java/test/org/apache/spark/JavaSparkContextSuite.java b/core/src/test/java/test/org/apache/spark/JavaSparkContextSuite.java
index 0f489fb219010..b188ee16b97d0 100644
--- a/core/src/test/java/test/org/apache/spark/JavaSparkContextSuite.java
+++ b/core/src/test/java/test/org/apache/spark/JavaSparkContextSuite.java
@@ -28,6 +28,7 @@
 
 import org.apache.spark.api.java.*;
 import org.apache.spark.*;
+import org.apache.spark.util.Utils;
 
 /**
  * Java apps can use both Java-friendly JavaSparkContext and Scala SparkContext.
@@ -35,14 +36,16 @@
 public class JavaSparkContextSuite implements Serializable {
 
   @Test
-  public void javaSparkContext() {
+  public void javaSparkContext() throws IOException {
+    File tempDir = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "spark");
+    String dummyJarFile = File.createTempFile(tempDir.toString(), "jarFile").toString();
     String[] jars = new String[] {};
     java.util.Map<String, String> environment = new java.util.HashMap<>();
 
     new JavaSparkContext(new SparkConf().setMaster("local").setAppName("name")).stop();
     new JavaSparkContext("local", "name", new SparkConf()).stop();
     new JavaSparkContext("local", "name").stop();
-    new JavaSparkContext("local", "name", "sparkHome", "jarFile").stop();
+    new JavaSparkContext("local", "name", "sparkHome", dummyJarFile).stop();
     new JavaSparkContext("local", "name", "sparkHome", jars).stop();
     new JavaSparkContext("local", "name", "sparkHome", jars, environment).stop();
   }
diff --git a/core/src/test/resources/HistoryServerExpectations/app_environment_expectation.json b/core/src/test/resources/HistoryServerExpectations/app_environment_expectation.json
index a64617256d63a..c2616129de954 100644
--- a/core/src/test/resources/HistoryServerExpectations/app_environment_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/app_environment_expectation.json
@@ -5,282 +5,283 @@
     "scalaVersion" : "version 2.11.8"
   },
   "sparkProperties" : [
-    [ "spark.blacklist.task.maxTaskAttemptsPerExecutor", "3" ],
+    [ "spark.app.id", "app-20161116163331-0000" ],
+    [ "spark.app.name", "Spark shell" ],
+    [ "spark.blacklist.application.maxFailedExecutorsPerNode", "2" ],
+    [ "spark.blacklist.application.maxFailedTasksPerExecutor", "1" ],
     [ "spark.blacklist.enabled", "TRUE" ],
-    [ "spark.driver.host", "172.22.0.167" ],
+    [ "spark.blacklist.stage.maxFailedExecutorsPerNode", "3" ],
+    [ "spark.blacklist.stage.maxFailedTasksPerExecutor", "3" ],
+    [ "spark.blacklist.task.maxTaskAttemptsPerExecutor", "3" ],
     [ "spark.blacklist.task.maxTaskAttemptsPerNode", "3" ],
-    [ "spark.eventLog.enabled", "TRUE" ],
+    [ "spark.blacklist.timeout", "1000000" ],
+    [ "spark.driver.host", "172.22.0.167" ],
     [ "spark.driver.port", "51459" ],
-    [ "spark.repl.class.uri", "spark://172.22.0.167:51459/classes" ],
-    [ "spark.jars", "" ],
-    [ "spark.repl.class.outputDir", "/private/var/folders/l4/d46wlzj16593f3d812vk49tw0000gp/T/spark-1cbc97d0-7fe6-4c9f-8c2c-f6fe51ee3cf2/repl-39929169-ac4c-4c6d-b116-f648e4dd62ed" ],
-    [ "spark.app.name", "Spark shell" ],
-    [ "spark.blacklist.stage.maxFailedExecutorsPerNode", "3" ],
-    [ "spark.scheduler.mode", "FIFO" ],
+    [ "spark.eventLog.compress", "FALSE" ],
+    [ "spark.eventLog.dir", "/Users/jose/logs" ],
+    [ "spark.eventLog.enabled", "TRUE" ],
     [ "spark.eventLog.overwrite", "TRUE" ],
-    [ "spark.blacklist.stage.maxFailedTasksPerExecutor", "3" ],
     [ "spark.executor.id", "driver" ],
-    [ "spark.blacklist.application.maxFailedExecutorsPerNode", "2" ],
-    [ "spark.submit.deployMode", "client" ],
-    [ "spark.master", "local-cluster[4,4,1024]" ],
     [ "spark.home", "/Users/Jose/IdeaProjects/spark" ],
-    [ "spark.eventLog.dir", "/Users/jose/logs" ],
+    [ "spark.jars", "" ],
+    [ "spark.master", "local-cluster[4,4,1024]" ],
+    [ "spark.repl.class.outputDir", "/private/var/folders/l4/d46wlzj16593f3d812vk49tw0000gp/T/spark-1cbc97d0-7fe6-4c9f-8c2c-f6fe51ee3cf2/repl-39929169-ac4c-4c6d-b116-f648e4dd62ed" ],
+    [ "spark.repl.class.uri", "spark://172.22.0.167:51459/classes" ],
+    [ "spark.scheduler.mode", "FIFO" ],
     [ "spark.sql.catalogImplementation", "in-memory" ],
-    [ "spark.eventLog.compress", "FALSE" ],
-    [ "spark.blacklist.application.maxFailedTasksPerExecutor", "1" ],
-    [ "spark.blacklist.timeout", "1000000" ],
-    [ "spark.app.id", "app-20161116163331-0000" ],
+    [ "spark.submit.deployMode", "client" ],
     [ "spark.task.maxFailures", "4" ]
   ],
   "hadoopProperties" : [
+    [ "mapreduce.jobhistory.client.thread-count", "10" ],
     [ "mapreduce.jobtracker.address", "local" ],
-    [ "yarn.resourcemanager.scheduler.monitor.policies", "org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy" ],
-    [ "mapreduce.jobhistory.client.thread-count", "10" ]
+    [ "yarn.resourcemanager.scheduler.monitor.policies", "org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy" ]
   ],
   "systemProperties" : [
-    [ "java.io.tmpdir", "/var/folders/l4/d46wlzj16593f3d812vk49tw0000gp/T/" ],
-    [ "line.separator", "\n" ],
-    [ "path.separator", ":" ],
-    [ "sun.management.compiler", "HotSpot 64-Bit Tiered Compilers" ],
     [ "SPARK_SUBMIT", "true" ],
-    [ "sun.cpu.endian", "little" ],
-    [ "java.specification.version", "1.8" ],
-    [ "java.vm.specification.name", "Java Virtual Machine Specification" ],
-    [ "java.vendor", "Oracle Corporation" ],
-    [ "java.vm.specification.version", "1.8" ],
-    [ "user.home", "/Users/Jose" ],
+    [ "awt.toolkit", "sun.lwawt.macosx.LWCToolkit" ],
+    [ "file.encoding", "UTF-8" ],
     [ "file.encoding.pkg", "sun.io" ],
-    [ "sun.nio.ch.bugLevel", "" ],
+    [ "file.separator", "/" ],
     [ "ftp.nonProxyHosts", "local|*.local|169.254/16|*.169.254/16" ],
-    [ "sun.arch.data.model", "64" ],
-    [ "sun.boot.library.path", "/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib" ],
-    [ "user.dir", "/Users/Jose/IdeaProjects/spark" ],
-    [ "java.library.path", "/Users/Jose/Library/Java/Extensions:/Library/Java/Extensions:/Network/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java:." ],
-    [ "sun.cpu.isalist", "" ],
-    [ "os.arch", "x86_64" ],
-    [ "java.vm.version", "25.92-b14" ],
+    [ "gopherProxySet", "false" ],
+    [ "http.nonProxyHosts", "local|*.local|169.254/16|*.169.254/16" ],
+    [ "io.netty.maxDirectMemory", "0" ],
+    [ "java.awt.graphicsenv", "sun.awt.CGraphicsEnvironment" ],
+    [ "java.awt.printerjob", "sun.lwawt.macosx.CPrinterJob" ],
+    [ "java.class.version", "52.0" ],
     [ "java.endorsed.dirs", "/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/endorsed" ],
-    [ "java.runtime.version", "1.8.0_92-b14" ],
-    [ "java.vm.info", "mixed mode" ],
     [ "java.ext.dirs", "/Users/Jose/Library/Java/Extensions:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/ext:/Library/Java/Extensions:/Network/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java" ],
+    [ "java.home", "/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre" ],
+    [ "java.io.tmpdir", "/var/folders/l4/d46wlzj16593f3d812vk49tw0000gp/T/" ],
+    [ "java.library.path", "/Users/Jose/Library/Java/Extensions:/Library/Java/Extensions:/Network/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java:." ],
     [ "java.runtime.name", "Java(TM) SE Runtime Environment" ],
-    [ "file.separator", "/" ],
-    [ "io.netty.maxDirectMemory", "0" ],
-    [ "java.class.version", "52.0" ],
-    [ "scala.usejavacp", "true" ],
+    [ "java.runtime.version", "1.8.0_92-b14" ],
     [ "java.specification.name", "Java Platform API Specification" ],
-    [ "sun.boot.class.path", "/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/resources.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/rt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/sunrsasign.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/jsse.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/jce.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/charsets.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/jfr.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/classes" ],
-    [ "file.encoding", "UTF-8" ],
-    [ "user.timezone", "America/Chicago" ],
     [ "java.specification.vendor", "Oracle Corporation" ],
-    [ "sun.java.launcher", "SUN_STANDARD" ],
+    [ "java.specification.version", "1.8" ],
+    [ "java.vendor", "Oracle Corporation" ],
+    [ "java.vendor.url", "http://java.oracle.com/" ],
+    [ "java.vendor.url.bug", "http://bugreport.sun.com/bugreport/" ],
+    [ "java.version", "1.8.0_92" ],
+    [ "java.vm.info", "mixed mode" ],
+    [ "java.vm.name", "Java HotSpot(TM) 64-Bit Server VM" ],
+    [ "java.vm.specification.name", "Java Virtual Machine Specification" ],
+    [ "java.vm.specification.vendor", "Oracle Corporation" ],
+    [ "java.vm.specification.version", "1.8" ],
+    [ "java.vm.vendor", "Oracle Corporation" ],
+    [ "java.vm.version", "25.92-b14" ],
+    [ "line.separator", "\n" ],
+    [ "os.arch", "x86_64" ],
+    [ "os.name", "Mac OS X" ],
     [ "os.version", "10.11.6" ],
+    [ "path.separator", ":" ],
+    [ "scala.usejavacp", "true" ],
+    [ "socksNonProxyHosts", "local|*.local|169.254/16|*.169.254/16" ],
+    [ "sun.arch.data.model", "64" ],
+    [ "sun.boot.class.path", "/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/resources.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/rt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/sunrsasign.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/jsse.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/jce.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/charsets.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/jfr.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/classes" ],
+    [ "sun.boot.library.path", "/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib" ],
+    [ "sun.cpu.endian", "little" ],
+    [ "sun.cpu.isalist", "" ],
+    [ "sun.io.unicode.encoding", "UnicodeBig" ],
+    [ "sun.java.command", "org.apache.spark.deploy.SparkSubmit --master local-cluster[4,4,1024] --conf spark.blacklist.enabled=TRUE --conf spark.blacklist.timeout=1000000 --conf spark.blacklist.application.maxFailedTasksPerExecutor=1 --conf spark.eventLog.overwrite=TRUE --conf spark.blacklist.task.maxTaskAttemptsPerNode=3 --conf spark.blacklist.stage.maxFailedTasksPerExecutor=3 --conf spark.blacklist.task.maxTaskAttemptsPerExecutor=3 --conf spark.eventLog.compress=FALSE --conf spark.blacklist.stage.maxFailedExecutorsPerNode=3 --conf spark.eventLog.enabled=TRUE --conf spark.eventLog.dir=/Users/jose/logs --conf spark.blacklist.application.maxFailedExecutorsPerNode=2 --conf spark.task.maxFailures=4 --class org.apache.spark.repl.Main --name Spark shell spark-shell -i /Users/Jose/dev/jose-utils/blacklist/test-blacklist.scala" ],
+    [ "sun.java.launcher", "SUN_STANDARD" ],
+    [ "sun.jnu.encoding", "UTF-8" ],
+    [ "sun.management.compiler", "HotSpot 64-Bit Tiered Compilers" ],
+    [ "sun.nio.ch.bugLevel", "" ],
     [ "sun.os.patch.level", "unknown" ],
-    [ "gopherProxySet", "false" ],
-    [ "java.vm.specification.vendor", "Oracle Corporation" ],
     [ "user.country", "US" ],
-    [ "sun.jnu.encoding", "UTF-8" ],
-    [ "http.nonProxyHosts", "local|*.local|169.254/16|*.169.254/16" ],
+    [ "user.dir", "/Users/Jose/IdeaProjects/spark" ],
+    [ "user.home", "/Users/Jose" ],
     [ "user.language", "en" ],
-    [ "socksNonProxyHosts", "local|*.local|169.254/16|*.169.254/16" ],
-    [ "java.vendor.url", "http://java.oracle.com/" ],
-    [ "java.awt.printerjob", "sun.lwawt.macosx.CPrinterJob" ],
-    [ "java.awt.graphicsenv", "sun.awt.CGraphicsEnvironment" ],
-    [ "awt.toolkit", "sun.lwawt.macosx.LWCToolkit" ],
-    [ "os.name", "Mac OS X" ],
-    [ "java.vm.vendor", "Oracle Corporation" ],
-    [ "java.vendor.url.bug", "http://bugreport.sun.com/bugreport/" ],
     [ "user.name", "jose" ],
-    [ "java.vm.name", "Java HotSpot(TM) 64-Bit Server VM" ],
-    [ "sun.java.command", "org.apache.spark.deploy.SparkSubmit --master local-cluster[4,4,1024] --conf spark.blacklist.enabled=TRUE --conf spark.blacklist.timeout=1000000 --conf spark.blacklist.application.maxFailedTasksPerExecutor=1 --conf spark.eventLog.overwrite=TRUE --conf spark.blacklist.task.maxTaskAttemptsPerNode=3 --conf spark.blacklist.stage.maxFailedTasksPerExecutor=3 --conf spark.blacklist.task.maxTaskAttemptsPerExecutor=3 --conf spark.eventLog.compress=FALSE --conf spark.blacklist.stage.maxFailedExecutorsPerNode=3 --conf spark.eventLog.enabled=TRUE --conf spark.eventLog.dir=/Users/jose/logs --conf spark.blacklist.application.maxFailedExecutorsPerNode=2 --conf spark.task.maxFailures=4 --class org.apache.spark.repl.Main --name Spark shell spark-shell -i /Users/Jose/dev/jose-utils/blacklist/test-blacklist.scala" ],
-    [ "java.home", "/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre" ],
-    [ "java.version", "1.8.0_92" ],
-    [ "sun.io.unicode.encoding", "UnicodeBig" ]
+    [ "user.timezone", "America/Chicago" ]
   ],
   "classpathEntries" : [
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/avro-mapred-1.7.7-hadoop2.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-mapreduce-client-core-2.2.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-servlet-9.2.16.v20160414.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/parquet-column-1.8.1.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/snappy-java-1.1.2.6.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/oro-2.0.8.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/RoaringBitmap-0.5.11.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/antlr4-runtime-4.5.3.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/aopalliance-1.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/aopalliance-repackaged-2.4.0-b34.jar", "System Classpath" ],
     [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/arpack_combined_all-0.1.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/pmml-schema-1.2.15.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-assembly_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/javassist-3.18.1-GA.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-tags_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-launcher_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/avro-1.7.7.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/avro-ipc-1.7.7-tests.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/avro-ipc-1.7.7.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/avro-mapred-1.7.7-hadoop2.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/breeze-macros_2.11-0.12.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/breeze_2.11-0.12.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/cglib-2.2.1-v20090111.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/chill-java-0.8.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/chill_2.11-0.8.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-beanutils-1.7.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-beanutils-core-1.8.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-cli-1.2.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-codec-1.10.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-collections-3.2.1.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-compiler-3.0.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-compress-1.4.1.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-configuration-1.6.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-crypto-1.0.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-digester-1.8.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-httpclient-3.1.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-io-2.1.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-lang-2.5.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-lang3-3.5.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-math-2.1.jar", "System Classpath" ],
     [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-math3-3.4.1.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-net-3.1.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/compress-lzf-1.0.3.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/core-1.1.2.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/curator-client-2.4.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/curator-framework-2.4.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/curator-recipes-2.4.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/guava-14.0.1.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/guice-3.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-annotations-2.2.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-auth-2.2.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-client-2.2.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-common-2.2.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-hdfs-2.2.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-mapreduce-client-app-2.2.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-mapreduce-client-common-2.2.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-mapreduce-client-core-2.2.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-mapreduce-client-jobclient-2.2.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-mapreduce-client-shuffle-2.2.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-yarn-api-2.2.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-yarn-client-2.2.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-yarn-common-2.2.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-yarn-server-common-2.2.0.jar", "System Classpath" ],
     [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hk2-api-2.4.0-b34.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/scala-xml_2.11-1.0.4.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/objenesis-2.1.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spire-macros_2.11-0.7.4.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/scala-reflect-2.11.8.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-mllib-local_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-mllib_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jersey-server-2.22.2.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/core/target/scala-2.11/classes/", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hk2-locator-2.4.0-b34.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hk2-utils-2.4.0-b34.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/ivy-2.4.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jackson-annotations-2.6.5.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jackson-core-2.6.5.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jackson-core-asl-1.9.13.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jackson-databind-2.6.5.jar", "System Classpath" ],
     [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jackson-mapper-asl-1.9.13.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jackson-module-paranamer-2.6.5.jar", "System Classpath" ],
     [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jackson-module-scala_2.11-2.6.5.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/curator-framework-2.4.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/janino-3.0.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/javassist-3.18.1-GA.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/javax.annotation-api-1.2.jar", "System Classpath" ],
     [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/javax.inject-1.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/curator-client-2.4.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jackson-core-asl-1.9.13.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/common/network-common/target/scala-2.11/classes/", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/zookeeper-3.4.5.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-auth-2.2.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/repl/target/scala-2.11/classes/", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jul-to-slf4j-1.7.16.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jersey-media-jaxb-2.22.2.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-io-9.2.16.v20160414.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/RoaringBitmap-0.5.11.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/javax.inject-2.4.0-b34.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/javax.servlet-api-3.1.0.jar", "System Classpath" ],
     [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/javax.ws.rs-api-2.0.1.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/sql/catalyst/target/scala-2.11/classes/", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-unsafe_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-repl_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-continuation-9.2.16.v20160414.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-yarn-client-2.2.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/sql/hive-thriftserver/target/scala-2.11/classes", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-annotations-2.2.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/metrics-graphite-3.1.2.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-yarn-api-2.2.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jcl-over-slf4j-1.7.16.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jersey-client-2.22.2.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jersey-common-2.22.2.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jersey-container-servlet-2.22.2.jar", "System Classpath" ],
     [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jersey-container-servlet-core-2.22.2.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/streaming/target/scala-2.11/classes/", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-net-3.1.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jersey-guava-2.22.2.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jersey-media-jaxb-2.22.2.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jersey-server-2.22.2.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jets3t-0.7.1.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-client-9.2.16.v20160414.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-continuation-9.2.16.v20160414.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-http-9.2.16.v20160414.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-io-9.2.16.v20160414.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-jndi-9.2.16.v20160414.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-plus-9.2.16.v20160414.jar", "System Classpath" ],
     [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-proxy-9.2.16.v20160414.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-catalyst_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/lz4-1.3.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-crypto-1.0.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/common/network-yarn/target/scala-2.11/classes", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/javax.annotation-api-1.2.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-sql_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/guava-14.0.1.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/javax.servlet-api-3.1.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-collections-3.2.1.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/conf/", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/unused-1.0.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/aopalliance-1.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/parquet-encoding-1.8.1.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/common/tags/target/scala-2.11/classes/", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/json4s-jackson_2.11-3.2.11.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-cli-1.2.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-yarn-server-common-2.2.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/cglib-2.2.1-v20090111.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/pyrolite-4.13.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/scala-library-2.11.8.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/scala-parser-combinators_2.11-1.0.4.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-security-9.2.16.v20160414.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-server-9.2.16.v20160414.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-servlet-9.2.16.v20160414.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-servlets-9.2.16.v20160414.jar", "System Classpath" ],
     [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-util-6.1.26.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/py4j-0.10.4.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-configuration-1.6.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/core-1.1.2.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/core/target/jars/*", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/common/network-shuffle/target/scala-2.11/classes/", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/parquet-format-2.3.0-incubating.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-util-9.2.16.v20160414.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-webapp-9.2.16.v20160414.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-xml-9.2.16.v20160414.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jline-2.12.1.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/json4s-ast_2.11-3.2.11.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/json4s-core_2.11-3.2.11.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/json4s-jackson_2.11-3.2.11.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jsr305-1.3.9.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jtransforms-2.4.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jul-to-slf4j-1.7.16.jar", "System Classpath" ],
     [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/kryo-shaded-3.0.3.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/sql/core/target/scala-2.11/classes/", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/chill-java-0.8.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jackson-annotations-2.6.5.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/parquet-hadoop-1.8.1.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/sql/hive/target/scala-2.11/classes/", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/avro-ipc-1.7.7.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/xz-1.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/parquet-jackson-1.8.1.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/aopalliance-repackaged-2.4.0-b34.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jersey-common-2.22.2.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/leveldbjni-all-1.8.jar", "System Classpath" ],
     [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/log4j-1.2.17.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/lz4-1.3.0.jar", "System Classpath" ],
     [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/metrics-core-3.1.2.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-util-9.2.16.v20160414.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/scalap-2.11.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/osgi-resource-locator-1.0.1.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-beanutils-1.7.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-compress-1.4.1.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jcl-over-slf4j-1.7.16.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/yarn/target/scala-2.11/classes", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-plus-9.2.16.v20160414.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/protobuf-java-2.5.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/common/unsafe/target/scala-2.11/classes/", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jackson-module-paranamer-2.6.5.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/leveldbjni-all-1.8.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jackson-core-2.6.5.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/slf4j-api-1.7.16.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/compress-lzf-1.0.3.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/stream-2.7.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-mapreduce-client-shuffle-2.2.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-codec-1.10.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-yarn-common-2.2.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/common/sketch/target/scala-2.11/classes/", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/breeze_2.11-0.12.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-mapreduce-client-common-2.2.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-core_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jersey-container-servlet-2.22.2.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-network-shuffle_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-lang-2.5.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/ivy-2.4.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-common-2.2.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-math-2.1.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-hdfs-2.2.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/scala-compiler-2.11.8.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/metrics-graphite-3.1.2.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/metrics-json-3.1.2.jar", "System Classpath" ],
     [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/metrics-jvm-3.1.2.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-lang3-3.5.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jsr305-1.3.9.jar", "System Classpath" ],
     [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/minlog-1.3.0.jar", "System Classpath" ],
     [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/netty-3.8.0.Final.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-webapp-9.2.16.v20160414.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/json4s-ast_2.11-3.2.11.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/xbean-asm5-shaded-4.4.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-io-2.1.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/slf4j-log4j12-1.7.16.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hk2-locator-2.4.0-b34.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/netty-all-4.0.41.Final.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/objenesis-2.1.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/opencsv-2.3.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/oro-2.0.8.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/osgi-resource-locator-1.0.1.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/paranamer-2.6.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/parquet-column-1.8.1.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/parquet-common-1.8.1.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/parquet-encoding-1.8.1.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/parquet-format-2.3.0-incubating.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/parquet-hadoop-1.8.1.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/parquet-jackson-1.8.1.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/pmml-model-1.2.15.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/pmml-schema-1.2.15.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/protobuf-java-2.5.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/py4j-0.10.4.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/pyrolite-4.13.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/scala-compiler-2.11.8.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/scala-library-2.11.8.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/scala-parser-combinators_2.11-1.0.4.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/scala-reflect-2.11.8.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/scala-xml_2.11-1.0.4.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/scalap-2.11.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/scalatest_2.11-2.2.6.jar", "System Classpath" ],
     [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/shapeless_2.11-2.0.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/slf4j-api-1.7.16.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/slf4j-log4j12-1.7.16.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/snappy-java-1.1.2.6.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-assembly_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-catalyst_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-core_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-graphx_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-launcher_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-mllib-local_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-mllib_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
     [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-network-common_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-xml-9.2.16.v20160414.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-httpclient-3.1.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/javax.inject-2.4.0-b34.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/mllib/target/scala-2.11/classes/", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/scalatest_2.11-2.2.6.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hk2-utils-2.4.0-b34.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-client-9.2.16.v20160414.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jersey-guava-2.22.2.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-jndi-9.2.16.v20160414.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/graphx/target/scala-2.11/classes/", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-mapreduce-client-app-2.2.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/examples/target/scala-2.11/classes/", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/xmlenc-0.52.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jets3t-0.7.1.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/curator-recipes-2.4.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/opencsv-2.3.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jtransforms-2.4.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/antlr4-runtime-4.5.3.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/chill_2.11-0.8.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-digester-1.8.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/univocity-parsers-2.2.1.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jline-2.12.1.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-network-shuffle_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-repl_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-sketch_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-sql_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
     [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-streaming_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/launcher/target/scala-2.11/classes/", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/breeze-macros_2.11-0.12.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jersey-client-2.22.2.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jackson-databind-2.6.5.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-servlets-9.2.16.v20160414.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/paranamer-2.6.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-security-9.2.16.v20160414.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/avro-ipc-1.7.7-tests.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/avro-1.7.7.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-tags_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-unsafe_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spire-macros_2.11-0.7.4.jar", "System Classpath" ],
     [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spire_2.11-0.7.4.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-client-2.2.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/metrics-json-3.1.2.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-beanutils-core-1.8.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/stream-2.7.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/univocity-parsers-2.2.1.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/unused-1.0.0.jar", "System Classpath" ],
     [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/validation-api-1.1.0.Final.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-graphx_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/netty-all-4.0.41.Final.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/janino-3.0.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/json4s-core_2.11-3.2.11.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-compiler-3.0.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/guice-3.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-server-9.2.16.v20160414.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-http-9.2.16.v20160414.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/parquet-common-1.8.1.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-mapreduce-client-jobclient-2.2.0.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-sketch_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
-    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/pmml-model-1.2.15.jar", "System Classpath" ]
-  ]
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/xbean-asm5-shaded-4.4.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/xmlenc-0.52.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/xz-1.0.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/zookeeper-3.4.5.jar", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/common/network-common/target/scala-2.11/classes/", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/common/network-shuffle/target/scala-2.11/classes/", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/common/network-yarn/target/scala-2.11/classes", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/common/sketch/target/scala-2.11/classes/", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/common/tags/target/scala-2.11/classes/", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/common/unsafe/target/scala-2.11/classes/", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/conf/", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/core/target/jars/*", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/core/target/scala-2.11/classes/", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/examples/target/scala-2.11/classes/", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/graphx/target/scala-2.11/classes/", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/launcher/target/scala-2.11/classes/", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/mllib/target/scala-2.11/classes/", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/repl/target/scala-2.11/classes/", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/sql/catalyst/target/scala-2.11/classes/", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/sql/core/target/scala-2.11/classes/", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/sql/hive-thriftserver/target/scala-2.11/classes", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/sql/hive/target/scala-2.11/classes/", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/streaming/target/scala-2.11/classes/", "System Classpath" ],
+    [ "/Users/Jose/IdeaProjects/spark/yarn/target/scala-2.11/classes", "System Classpath" ]
+  ],
+  "resourceProfiles" : [ ]
 }
diff --git a/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json
index 6e6d28b6a57ec..06015ec46e44d 100644
--- a/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json
@@ -1,4 +1,34 @@
 [ {
+  "id" : "app-20200706201101-0003",
+  "name" : "Spark shell",
+  "attempts" : [ {
+    "startTime" : "2020-07-07T03:11:00.235GMT",
+    "endTime" : "2020-07-07T03:17:04.231GMT",
+    "lastUpdated" : "",
+    "duration" : 363996,
+    "sparkUser" : "terryk",
+    "completed" : true,
+    "appSparkVersion" : "3.1.0-SNAPSHOT",
+    "endTimeEpoch" : 1594091824231,
+    "startTimeEpoch" : 1594091460235,
+    "lastUpdatedEpoch" : 0
+  } ]
+}, {
+  "id" : "application_1578436911597_0052",
+  "name" : "Spark shell",
+  "attempts" : [ {
+    "startTime" : "2020-01-11T17:44:22.851GMT",
+    "endTime" : "2020-01-11T17:46:42.615GMT",
+    "lastUpdated" : "",
+    "duration" : 139764,
+    "sparkUser" : "tgraves",
+    "completed" : true,
+    "appSparkVersion" : "3.0.0-SNAPSHOT",
+    "endTimeEpoch" : 1578764802615,
+    "startTimeEpoch" : 1578764662851,
+    "lastUpdatedEpoch" : 0
+  } ]
+}, {
   "id" : "application_1555004656427_0144",
   "name" : "Spark shell",
   "attempts" : [ {
diff --git a/core/src/test/resources/HistoryServerExpectations/blacklisting_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/blacklisting_for_stage_expectation.json
deleted file mode 100644
index b18b19f7eeffb..0000000000000
--- a/core/src/test/resources/HistoryServerExpectations/blacklisting_for_stage_expectation.json
+++ /dev/null
@@ -1,721 +0,0 @@
-{
-  "status" : "COMPLETE",
-  "stageId" : 0,
-  "attemptId" : 0,
-  "numTasks" : 10,
-  "numActiveTasks" : 0,
-  "numCompleteTasks" : 10,
-  "numFailedTasks" : 2,
-  "numKilledTasks" : 0,
-  "numCompletedIndices" : 10,
-  "submissionTime" : "2018-01-09T10:21:18.152GMT",
-  "firstTaskLaunchedTime" : "2018-01-09T10:21:18.347GMT",
-  "completionTime" : "2018-01-09T10:21:19.062GMT",
-  "executorDeserializeTime" : 327,
-  "executorDeserializeCpuTime" : 225900000,
-  "executorRunTime" : 761,
-  "executorCpuTime" : 269916000,
-  "resultSize" : 10376,
-  "jvmGcTime" : 27,
-  "resultSerializationTime" : 1,
-  "memoryBytesSpilled" : 0,
-  "diskBytesSpilled" : 0,
-  "peakExecutionMemory" : 0,
-  "inputBytes" : 0,
-  "inputRecords" : 0,
-  "outputBytes" : 0,
-  "outputRecords" : 0,
-  "shuffleRemoteBlocksFetched" : 0,
-  "shuffleLocalBlocksFetched" : 0,
-  "shuffleFetchWaitTime" : 0,
-  "shuffleRemoteBytesRead" : 0,
-  "shuffleRemoteBytesReadToDisk" : 0,
-  "shuffleLocalBytesRead" : 0,
-  "shuffleReadBytes" : 0,
-  "shuffleReadRecords" : 0,
-  "shuffleWriteBytes" : 460,
-  "shuffleWriteTime" : 8711515,
-  "shuffleWriteRecords" : 10,
-  "name" : "map at <console>:26",
-  "details" : "org.apache.spark.rdd.RDD.map(RDD.scala:370)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:26)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:34)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:36)\n$line17.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:38)\n$line17.$read$$iw$$iw$$iw$$iw.<init>(<console>:40)\n$line17.$read$$iw$$iw$$iw.<init>(<console>:42)\n$line17.$read$$iw$$iw.<init>(<console>:44)\n$line17.$read$$iw.<init>(<console>:46)\n$line17.$read.<init>(<console>:48)\n$line17.$read$.<init>(<console>:52)\n$line17.$read$.<clinit>(<console>)\n$line17.$eval$.$print$lzycompute(<console>:7)\n$line17.$eval$.$print(<console>:6)\n$line17.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786)",
-  "schedulingPool" : "default",
-  "rddIds" : [ 1, 0 ],
-  "accumulatorUpdates" : [ ],
-  "tasks" : {
-    "0" : {
-      "taskId" : 0,
-      "index" : 0,
-      "attempt" : 0,
-      "launchTime" : "2018-01-09T10:21:18.347GMT",
-      "duration" : 562,
-      "executorId" : "0",
-      "host" : "172.30.65.138",
-      "status" : "FAILED",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "errorMessage" : "java.lang.RuntimeException: Bad executor\n\tat $line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(<console>:27)\n\tat $line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(<console>:26)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:193)\n\tat org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:63)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:109)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\n",
-      "taskMetrics" : {
-        "executorDeserializeTime" : 0,
-        "executorDeserializeCpuTime" : 0,
-        "executorRunTime" : 460,
-        "executorCpuTime" : 0,
-        "resultSize" : 0,
-        "jvmGcTime" : 14,
-        "resultSerializationTime" : 0,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 0,
-          "writeTime" : 3873006,
-          "recordsWritten" : 0
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://172.30.65.138:64279/logPage/?appId=app-20180109111548-0000&executorId=0&logType=stdout",
-        "stderr" : "http://172.30.65.138:64279/logPage/?appId=app-20180109111548-0000&executorId=0&logType=stderr"
-      },
-      "schedulerDelay" : 102,
-      "gettingResultTime" : 0
-    },
-    "5" : {
-      "taskId" : 5,
-      "index" : 3,
-      "attempt" : 0,
-      "launchTime" : "2018-01-09T10:21:18.958GMT",
-      "duration" : 22,
-      "executorId" : "1",
-      "host" : "172.30.65.138",
-      "status" : "SUCCESS",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "taskMetrics" : {
-        "executorDeserializeTime" : 3,
-        "executorDeserializeCpuTime" : 2586000,
-        "executorRunTime" : 9,
-        "executorCpuTime" : 9635000,
-        "resultSize" : 1029,
-        "jvmGcTime" : 0,
-        "resultSerializationTime" : 0,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 46,
-          "writeTime" : 262919,
-          "recordsWritten" : 1
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
-        "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
-      },
-      "schedulerDelay" : 10,
-      "gettingResultTime" : 0
-    },
-    "10" : {
-      "taskId" : 10,
-      "index" : 8,
-      "attempt" : 0,
-      "launchTime" : "2018-01-09T10:21:19.034GMT",
-      "duration" : 12,
-      "executorId" : "1",
-      "host" : "172.30.65.138",
-      "status" : "SUCCESS",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "taskMetrics" : {
-        "executorDeserializeTime" : 2,
-        "executorDeserializeCpuTime" : 1803000,
-        "executorRunTime" : 6,
-        "executorCpuTime" : 6157000,
-        "resultSize" : 1029,
-        "jvmGcTime" : 0,
-        "resultSerializationTime" : 0,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 46,
-          "writeTime" : 243647,
-          "recordsWritten" : 1
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
-        "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
-      },
-      "schedulerDelay" : 4,
-      "gettingResultTime" : 0
-    },
-    "1" : {
-      "taskId" : 1,
-      "index" : 1,
-      "attempt" : 0,
-      "launchTime" : "2018-01-09T10:21:18.364GMT",
-      "duration" : 565,
-      "executorId" : "1",
-      "host" : "172.30.65.138",
-      "status" : "SUCCESS",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "taskMetrics" : {
-        "executorDeserializeTime" : 301,
-        "executorDeserializeCpuTime" : 200029000,
-        "executorRunTime" : 212,
-        "executorCpuTime" : 198479000,
-        "resultSize" : 1115,
-        "jvmGcTime" : 13,
-        "resultSerializationTime" : 1,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 46,
-          "writeTime" : 2409488,
-          "recordsWritten" : 1
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
-        "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
-      },
-      "schedulerDelay" : 51,
-      "gettingResultTime" : 0
-    },
-    "6" : {
-      "taskId" : 6,
-      "index" : 4,
-      "attempt" : 0,
-      "launchTime" : "2018-01-09T10:21:18.980GMT",
-      "duration" : 16,
-      "executorId" : "1",
-      "host" : "172.30.65.138",
-      "status" : "SUCCESS",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "taskMetrics" : {
-        "executorDeserializeTime" : 3,
-        "executorDeserializeCpuTime" : 2610000,
-        "executorRunTime" : 10,
-        "executorCpuTime" : 9622000,
-        "resultSize" : 1029,
-        "jvmGcTime" : 0,
-        "resultSerializationTime" : 0,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 46,
-          "writeTime" : 385110,
-          "recordsWritten" : 1
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
-        "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
-      },
-      "schedulerDelay" : 3,
-      "gettingResultTime" : 0
-    },
-    "9" : {
-      "taskId" : 9,
-      "index" : 7,
-      "attempt" : 0,
-      "launchTime" : "2018-01-09T10:21:19.022GMT",
-      "duration" : 12,
-      "executorId" : "1",
-      "host" : "172.30.65.138",
-      "status" : "SUCCESS",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "taskMetrics" : {
-        "executorDeserializeTime" : 2,
-        "executorDeserializeCpuTime" : 1981000,
-        "executorRunTime" : 7,
-        "executorCpuTime" : 6335000,
-        "resultSize" : 1029,
-        "jvmGcTime" : 0,
-        "resultSerializationTime" : 0,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 46,
-          "writeTime" : 259354,
-          "recordsWritten" : 1
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
-        "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
-      },
-      "schedulerDelay" : 3,
-      "gettingResultTime" : 0
-    },
-    "2" : {
-      "taskId" : 2,
-      "index" : 2,
-      "attempt" : 0,
-      "launchTime" : "2018-01-09T10:21:18.899GMT",
-      "duration" : 27,
-      "executorId" : "0",
-      "host" : "172.30.65.138",
-      "status" : "FAILED",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "errorMessage" : "java.lang.RuntimeException: Bad executor\n\tat $line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(<console>:27)\n\tat $line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(<console>:26)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:193)\n\tat org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:63)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:109)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\n",
-      "taskMetrics" : {
-        "executorDeserializeTime" : 0,
-        "executorDeserializeCpuTime" : 0,
-        "executorRunTime" : 16,
-        "executorCpuTime" : 0,
-        "resultSize" : 0,
-        "jvmGcTime" : 0,
-        "resultSerializationTime" : 0,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 0,
-          "writeTime" : 126128,
-          "recordsWritten" : 0
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://172.30.65.138:64279/logPage/?appId=app-20180109111548-0000&executorId=0&logType=stdout",
-        "stderr" : "http://172.30.65.138:64279/logPage/?appId=app-20180109111548-0000&executorId=0&logType=stderr"
-      },
-      "schedulerDelay" : 11,
-      "gettingResultTime" : 0
-    },
-    "7" : {
-      "taskId" : 7,
-      "index" : 5,
-      "attempt" : 0,
-      "launchTime" : "2018-01-09T10:21:18.996GMT",
-      "duration" : 15,
-      "executorId" : "1",
-      "host" : "172.30.65.138",
-      "status" : "SUCCESS",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "taskMetrics" : {
-        "executorDeserializeTime" : 2,
-        "executorDeserializeCpuTime" : 2231000,
-        "executorRunTime" : 9,
-        "executorCpuTime" : 8407000,
-        "resultSize" : 1029,
-        "jvmGcTime" : 0,
-        "resultSerializationTime" : 0,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 46,
-          "writeTime" : 205520,
-          "recordsWritten" : 1
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
-        "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
-      },
-      "schedulerDelay" : 4,
-      "gettingResultTime" : 0
-    },
-    "3" : {
-      "taskId" : 3,
-      "index" : 0,
-      "attempt" : 1,
-      "launchTime" : "2018-01-09T10:21:18.919GMT",
-      "duration" : 24,
-      "executorId" : "1",
-      "host" : "172.30.65.138",
-      "status" : "SUCCESS",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "taskMetrics" : {
-        "executorDeserializeTime" : 8,
-        "executorDeserializeCpuTime" : 8878000,
-        "executorRunTime" : 10,
-        "executorCpuTime" : 9364000,
-        "resultSize" : 1029,
-        "jvmGcTime" : 0,
-        "resultSerializationTime" : 0,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 46,
-          "writeTime" : 207014,
-          "recordsWritten" : 1
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
-        "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
-      },
-      "schedulerDelay" : 6,
-      "gettingResultTime" : 0
-    },
-    "11" : {
-      "taskId" : 11,
-      "index" : 9,
-      "attempt" : 0,
-      "launchTime" : "2018-01-09T10:21:19.045GMT",
-      "duration" : 15,
-      "executorId" : "1",
-      "host" : "172.30.65.138",
-      "status" : "SUCCESS",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "taskMetrics" : {
-        "executorDeserializeTime" : 3,
-        "executorDeserializeCpuTime" : 2017000,
-        "executorRunTime" : 6,
-        "executorCpuTime" : 6676000,
-        "resultSize" : 1029,
-        "jvmGcTime" : 0,
-        "resultSerializationTime" : 0,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 46,
-          "writeTime" : 233652,
-          "recordsWritten" : 1
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
-        "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
-      },
-      "schedulerDelay" : 6,
-      "gettingResultTime" : 0
-    },
-    "8" : {
-      "taskId" : 8,
-      "index" : 6,
-      "attempt" : 0,
-      "launchTime" : "2018-01-09T10:21:19.011GMT",
-      "duration" : 11,
-      "executorId" : "1",
-      "host" : "172.30.65.138",
-      "status" : "SUCCESS",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "taskMetrics" : {
-        "executorDeserializeTime" : 1,
-        "executorDeserializeCpuTime" : 1554000,
-        "executorRunTime" : 7,
-        "executorCpuTime" : 6034000,
-        "resultSize" : 1029,
-        "jvmGcTime" : 0,
-        "resultSerializationTime" : 0,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 46,
-          "writeTime" : 213296,
-          "recordsWritten" : 1
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
-        "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
-      },
-      "schedulerDelay" : 3,
-      "gettingResultTime" : 0
-    },
-    "4" : {
-      "taskId" : 4,
-      "index" : 2,
-      "attempt" : 1,
-      "launchTime" : "2018-01-09T10:21:18.943GMT",
-      "duration" : 16,
-      "executorId" : "1",
-      "host" : "172.30.65.138",
-      "status" : "SUCCESS",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "taskMetrics" : {
-        "executorDeserializeTime" : 2,
-        "executorDeserializeCpuTime" : 2211000,
-        "executorRunTime" : 9,
-        "executorCpuTime" : 9207000,
-        "resultSize" : 1029,
-        "jvmGcTime" : 0,
-        "resultSerializationTime" : 0,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 46,
-          "writeTime" : 292381,
-          "recordsWritten" : 1
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
-        "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
-      },
-      "schedulerDelay" : 5,
-      "gettingResultTime" : 0
-    }
-  },
-  "executorSummary" : {
-    "0" : {
-      "taskTime" : 589,
-      "failedTasks" : 2,
-      "succeededTasks" : 0,
-      "killedTasks" : 0,
-      "inputBytes" : 0,
-      "inputRecords" : 0,
-      "outputBytes" : 0,
-      "outputRecords" : 0,
-      "shuffleRead" : 0,
-      "shuffleReadRecords" : 0,
-      "shuffleWrite" : 0,
-      "shuffleWriteRecords" : 0,
-      "memoryBytesSpilled" : 0,
-      "diskBytesSpilled" : 0,
-      "isBlacklistedForStage" : true
-    },
-    "1" : {
-      "taskTime" : 708,
-      "failedTasks" : 0,
-      "succeededTasks" : 10,
-      "killedTasks" : 0,
-      "inputBytes" : 0,
-      "inputRecords" : 0,
-      "outputBytes" : 0,
-      "outputRecords" : 0,
-      "shuffleRead" : 0,
-      "shuffleReadRecords" : 0,
-      "shuffleWrite" : 460,
-      "shuffleWriteRecords" : 10,
-      "memoryBytesSpilled" : 0,
-      "diskBytesSpilled" : 0,
-      "isBlacklistedForStage" : false
-    }
-  },
-  "killedTasksSummary" : { }
-}
diff --git a/core/src/test/resources/HistoryServerExpectations/blacklisting_node_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/blacklisting_node_for_stage_expectation.json
deleted file mode 100644
index 8d11081247913..0000000000000
--- a/core/src/test/resources/HistoryServerExpectations/blacklisting_node_for_stage_expectation.json
+++ /dev/null
@@ -1,880 +0,0 @@
-{
-  "status" : "COMPLETE",
-  "stageId" : 0,
-  "attemptId" : 0,
-  "numTasks" : 10,
-  "numActiveTasks" : 0,
-  "numCompleteTasks" : 10,
-  "numFailedTasks" : 4,
-  "numKilledTasks" : 0,
-  "numCompletedIndices" : 10,
-  "submissionTime" : "2018-01-18T18:33:12.658GMT",
-  "firstTaskLaunchedTime" : "2018-01-18T18:33:12.816GMT",
-  "completionTime" : "2018-01-18T18:33:15.279GMT",
-  "executorDeserializeTime" : 3679,
-  "executorDeserializeCpuTime" : 1029819716,
-  "executorRunTime" : 5080,
-  "executorCpuTime" : 1163210819,
-  "resultSize" : 10824,
-  "jvmGcTime" : 370,
-  "resultSerializationTime" : 5,
-  "memoryBytesSpilled" : 0,
-  "diskBytesSpilled" : 0,
-  "peakExecutionMemory" : 0,
-  "inputBytes" : 0,
-  "inputRecords" : 0,
-  "outputBytes" : 0,
-  "outputRecords" : 0,
-  "shuffleRemoteBlocksFetched" : 0,
-  "shuffleLocalBlocksFetched" : 0,
-  "shuffleFetchWaitTime" : 0,
-  "shuffleRemoteBytesRead" : 0,
-  "shuffleRemoteBytesReadToDisk" : 0,
-  "shuffleLocalBytesRead" : 0,
-  "shuffleReadBytes" : 0,
-  "shuffleReadRecords" : 0,
-  "shuffleWriteBytes" : 1461,
-  "shuffleWriteTime" : 33251697,
-  "shuffleWriteRecords" : 30,
-  "name" : "map at <console>:27",
-  "details" : "org.apache.spark.rdd.RDD.map(RDD.scala:370)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:27)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:35)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:37)\n$line15.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:39)\n$line15.$read$$iw$$iw$$iw$$iw.<init>(<console>:41)\n$line15.$read$$iw$$iw$$iw.<init>(<console>:43)\n$line15.$read$$iw$$iw.<init>(<console>:45)\n$line15.$read$$iw.<init>(<console>:47)\n$line15.$read.<init>(<console>:49)\n$line15.$read$.<init>(<console>:53)\n$line15.$read$.<clinit>(<console>)\n$line15.$eval$.$print$lzycompute(<console>:7)\n$line15.$eval$.$print(<console>:6)\n$line15.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786)",
-  "schedulingPool" : "default",
-  "rddIds" : [ 1, 0 ],
-  "accumulatorUpdates" : [ ],
-  "tasks" : {
-    "0" : {
-      "taskId" : 0,
-      "index" : 0,
-      "attempt" : 0,
-      "launchTime" : "2018-01-18T18:33:12.816GMT",
-      "duration" : 2064,
-      "executorId" : "1",
-      "host" : "apiros-3.gce.test.com",
-      "status" : "SUCCESS",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "taskMetrics" : {
-        "executorDeserializeTime" : 1081,
-        "executorDeserializeCpuTime" : 353981050,
-        "executorRunTime" : 914,
-        "executorCpuTime" : 368865439,
-        "resultSize" : 1134,
-        "jvmGcTime" : 75,
-        "resultSerializationTime" : 1,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 144,
-          "writeTime" : 3662221,
-          "recordsWritten" : 3
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stdout?start=-4096",
-        "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stderr?start=-4096"
-      },
-      "schedulerDelay" : 68,
-      "gettingResultTime" : 0
-    },
-    "5" : {
-      "taskId" : 5,
-      "index" : 5,
-      "attempt" : 0,
-      "launchTime" : "2018-01-18T18:33:14.320GMT",
-      "duration" : 73,
-      "executorId" : "5",
-      "host" : "apiros-2.gce.test.com",
-      "status" : "FAILED",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "errorMessage" : "java.lang.RuntimeException: Bad executor\n\tat $line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(<console>:28)\n\tat $line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(<console>:27)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:193)\n\tat org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:63)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:109)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\n",
-      "taskMetrics" : {
-        "executorDeserializeTime" : 0,
-        "executorDeserializeCpuTime" : 0,
-        "executorRunTime" : 27,
-        "executorCpuTime" : 0,
-        "resultSize" : 0,
-        "jvmGcTime" : 0,
-        "resultSerializationTime" : 0,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 0,
-          "writeTime" : 191901,
-          "recordsWritten" : 0
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000007/attilapiros/stdout?start=-4096",
-        "stderr" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000007/attilapiros/stderr?start=-4096"
-      },
-      "schedulerDelay" : 46,
-      "gettingResultTime" : 0
-    },
-    "10" : {
-      "taskId" : 10,
-      "index" : 1,
-      "attempt" : 1,
-      "launchTime" : "2018-01-18T18:33:15.069GMT",
-      "duration" : 132,
-      "executorId" : "2",
-      "host" : "apiros-3.gce.test.com",
-      "status" : "SUCCESS",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "taskMetrics" : {
-        "executorDeserializeTime" : 5,
-        "executorDeserializeCpuTime" : 4598966,
-        "executorRunTime" : 76,
-        "executorCpuTime" : 20826337,
-        "resultSize" : 1091,
-        "jvmGcTime" : 0,
-        "resultSerializationTime" : 1,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 144,
-          "writeTime" : 301705,
-          "recordsWritten" : 3
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stdout?start=-4096",
-        "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stderr?start=-4096"
-      },
-      "schedulerDelay" : 50,
-      "gettingResultTime" : 0
-    },
-    "1" : {
-      "taskId" : 1,
-      "index" : 1,
-      "attempt" : 0,
-      "launchTime" : "2018-01-18T18:33:12.832GMT",
-      "duration" : 1506,
-      "executorId" : "5",
-      "host" : "apiros-2.gce.test.com",
-      "status" : "FAILED",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "errorMessage" : "java.lang.RuntimeException: Bad executor\n\tat $line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(<console>:28)\n\tat $line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(<console>:27)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:193)\n\tat org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:63)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:109)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\n",
-      "taskMetrics" : {
-        "executorDeserializeTime" : 0,
-        "executorDeserializeCpuTime" : 0,
-        "executorRunTime" : 1332,
-        "executorCpuTime" : 0,
-        "resultSize" : 0,
-        "jvmGcTime" : 33,
-        "resultSerializationTime" : 0,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 0,
-          "writeTime" : 3075188,
-          "recordsWritten" : 0
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000007/attilapiros/stdout?start=-4096",
-        "stderr" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000007/attilapiros/stderr?start=-4096"
-      },
-      "schedulerDelay" : 174,
-      "gettingResultTime" : 0
-    },
-    "6" : {
-      "taskId" : 6,
-      "index" : 6,
-      "attempt" : 0,
-      "launchTime" : "2018-01-18T18:33:14.323GMT",
-      "duration" : 67,
-      "executorId" : "4",
-      "host" : "apiros-2.gce.test.com",
-      "status" : "FAILED",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "errorMessage" : "java.lang.RuntimeException: Bad executor\n\tat $line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(<console>:28)\n\tat $line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(<console>:27)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:193)\n\tat org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:63)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:109)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\n",
-      "taskMetrics" : {
-        "executorDeserializeTime" : 0,
-        "executorDeserializeCpuTime" : 0,
-        "executorRunTime" : 51,
-        "executorCpuTime" : 0,
-        "resultSize" : 0,
-        "jvmGcTime" : 0,
-        "resultSerializationTime" : 0,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 0,
-          "writeTime" : 183718,
-          "recordsWritten" : 0
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000005/attilapiros/stdout?start=-4096",
-        "stderr" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000005/attilapiros/stderr?start=-4096"
-      },
-      "schedulerDelay" : 16,
-      "gettingResultTime" : 0
-    },
-    "9" : {
-      "taskId" : 9,
-      "index" : 4,
-      "attempt" : 1,
-      "launchTime" : "2018-01-18T18:33:14.973GMT",
-      "duration" : 96,
-      "executorId" : "2",
-      "host" : "apiros-3.gce.test.com",
-      "status" : "SUCCESS",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "taskMetrics" : {
-        "executorDeserializeTime" : 5,
-        "executorDeserializeCpuTime" : 4793905,
-        "executorRunTime" : 48,
-        "executorCpuTime" : 25678331,
-        "resultSize" : 1091,
-        "jvmGcTime" : 0,
-        "resultSerializationTime" : 1,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 147,
-          "writeTime" : 366050,
-          "recordsWritten" : 3
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stdout?start=-4096",
-        "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stderr?start=-4096"
-      },
-      "schedulerDelay" : 42,
-      "gettingResultTime" : 0
-    },
-    "13" : {
-      "taskId" : 13,
-      "index" : 9,
-      "attempt" : 0,
-      "launchTime" : "2018-01-18T18:33:15.200GMT",
-      "duration" : 76,
-      "executorId" : "2",
-      "host" : "apiros-3.gce.test.com",
-      "status" : "SUCCESS",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "taskMetrics" : {
-        "executorDeserializeTime" : 25,
-        "executorDeserializeCpuTime" : 5860574,
-        "executorRunTime" : 25,
-        "executorCpuTime" : 20585619,
-        "resultSize" : 1048,
-        "jvmGcTime" : 0,
-        "resultSerializationTime" : 0,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 147,
-          "writeTime" : 369513,
-          "recordsWritten" : 3
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stdout?start=-4096",
-        "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stderr?start=-4096"
-      },
-      "schedulerDelay" : 26,
-      "gettingResultTime" : 0
-    },
-    "2" : {
-      "taskId" : 2,
-      "index" : 2,
-      "attempt" : 0,
-      "launchTime" : "2018-01-18T18:33:12.832GMT",
-      "duration" : 1774,
-      "executorId" : "3",
-      "host" : "apiros-2.gce.test.com",
-      "status" : "SUCCESS",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "taskMetrics" : {
-        "executorDeserializeTime" : 1206,
-        "executorDeserializeCpuTime" : 263386625,
-        "executorRunTime" : 493,
-        "executorCpuTime" : 278399617,
-        "resultSize" : 1134,
-        "jvmGcTime" : 78,
-        "resultSerializationTime" : 1,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 144,
-          "writeTime" : 3322956,
-          "recordsWritten" : 3
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000004/attilapiros/stdout?start=-4096",
-        "stderr" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000004/attilapiros/stderr?start=-4096"
-      },
-      "schedulerDelay" : 74,
-      "gettingResultTime" : 0
-    },
-    "12" : {
-      "taskId" : 12,
-      "index" : 8,
-      "attempt" : 0,
-      "launchTime" : "2018-01-18T18:33:15.165GMT",
-      "duration" : 60,
-      "executorId" : "1",
-      "host" : "apiros-3.gce.test.com",
-      "status" : "SUCCESS",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "taskMetrics" : {
-        "executorDeserializeTime" : 4,
-        "executorDeserializeCpuTime" : 4010338,
-        "executorRunTime" : 34,
-        "executorCpuTime" : 21657558,
-        "resultSize" : 1048,
-        "jvmGcTime" : 0,
-        "resultSerializationTime" : 0,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 147,
-          "writeTime" : 319101,
-          "recordsWritten" : 3
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stdout?start=-4096",
-        "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stderr?start=-4096"
-      },
-      "schedulerDelay" : 22,
-      "gettingResultTime" : 0
-    },
-    "7" : {
-      "taskId" : 7,
-      "index" : 5,
-      "attempt" : 1,
-      "launchTime" : "2018-01-18T18:33:14.859GMT",
-      "duration" : 115,
-      "executorId" : "2",
-      "host" : "apiros-3.gce.test.com",
-      "status" : "SUCCESS",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "taskMetrics" : {
-        "executorDeserializeTime" : 11,
-        "executorDeserializeCpuTime" : 10894331,
-        "executorRunTime" : 84,
-        "executorCpuTime" : 28283110,
-        "resultSize" : 1048,
-        "jvmGcTime" : 0,
-        "resultSerializationTime" : 0,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 147,
-          "writeTime" : 377601,
-          "recordsWritten" : 3
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stdout?start=-4096",
-        "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stderr?start=-4096"
-      },
-      "schedulerDelay" : 20,
-      "gettingResultTime" : 0
-    },
-    "3" : {
-      "taskId" : 3,
-      "index" : 3,
-      "attempt" : 0,
-      "launchTime" : "2018-01-18T18:33:12.833GMT",
-      "duration" : 2027,
-      "executorId" : "2",
-      "host" : "apiros-3.gce.test.com",
-      "status" : "SUCCESS",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "taskMetrics" : {
-        "executorDeserializeTime" : 1282,
-        "executorDeserializeCpuTime" : 365807898,
-        "executorRunTime" : 681,
-        "executorCpuTime" : 349920830,
-        "resultSize" : 1134,
-        "jvmGcTime" : 102,
-        "resultSerializationTime" : 1,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 147,
-          "writeTime" : 3587839,
-          "recordsWritten" : 3
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stdout?start=-4096",
-        "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stderr?start=-4096"
-      },
-      "schedulerDelay" : 63,
-      "gettingResultTime" : 0
-    },
-    "11" : {
-      "taskId" : 11,
-      "index" : 7,
-      "attempt" : 0,
-      "launchTime" : "2018-01-18T18:33:15.072GMT",
-      "duration" : 93,
-      "executorId" : "1",
-      "host" : "apiros-3.gce.test.com",
-      "status" : "SUCCESS",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "taskMetrics" : {
-        "executorDeserializeTime" : 4,
-        "executorDeserializeCpuTime" : 4239884,
-        "executorRunTime" : 77,
-        "executorCpuTime" : 21689428,
-        "resultSize" : 1048,
-        "jvmGcTime" : 0,
-        "resultSerializationTime" : 0,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 147,
-          "writeTime" : 323898,
-          "recordsWritten" : 3
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stdout?start=-4096",
-        "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stderr?start=-4096"
-      },
-      "schedulerDelay" : 12,
-      "gettingResultTime" : 0
-    },
-    "8" : {
-      "taskId" : 8,
-      "index" : 6,
-      "attempt" : 1,
-      "launchTime" : "2018-01-18T18:33:14.879GMT",
-      "duration" : 194,
-      "executorId" : "1",
-      "host" : "apiros-3.gce.test.com",
-      "status" : "SUCCESS",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "taskMetrics" : {
-        "executorDeserializeTime" : 56,
-        "executorDeserializeCpuTime" : 12246145,
-        "executorRunTime" : 54,
-        "executorCpuTime" : 27304550,
-        "resultSize" : 1048,
-        "jvmGcTime" : 0,
-        "resultSerializationTime" : 0,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 147,
-          "writeTime" : 311940,
-          "recordsWritten" : 3
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stdout?start=-4096",
-        "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stderr?start=-4096"
-      },
-      "schedulerDelay" : 84,
-      "gettingResultTime" : 0
-    },
-    "4" : {
-      "taskId" : 4,
-      "index" : 4,
-      "attempt" : 0,
-      "launchTime" : "2018-01-18T18:33:12.833GMT",
-      "duration" : 1522,
-      "executorId" : "4",
-      "host" : "apiros-2.gce.test.com",
-      "status" : "FAILED",
-      "taskLocality" : "PROCESS_LOCAL",
-      "speculative" : false,
-      "accumulatorUpdates" : [ ],
-      "errorMessage" : "java.lang.RuntimeException: Bad executor\n\tat $line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(<console>:28)\n\tat $line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(<console>:27)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:193)\n\tat org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:63)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:109)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\n",
-      "taskMetrics" : {
-        "executorDeserializeTime" : 0,
-        "executorDeserializeCpuTime" : 0,
-        "executorRunTime" : 1184,
-        "executorCpuTime" : 0,
-        "resultSize" : 0,
-        "jvmGcTime" : 82,
-        "resultSerializationTime" : 0,
-        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0,
-        "peakExecutionMemory" : 0,
-        "inputMetrics" : {
-          "bytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "outputMetrics" : {
-          "bytesWritten" : 0,
-          "recordsWritten" : 0
-        },
-        "shuffleReadMetrics" : {
-          "remoteBlocksFetched" : 0,
-          "localBlocksFetched" : 0,
-          "fetchWaitTime" : 0,
-          "remoteBytesRead" : 0,
-          "remoteBytesReadToDisk" : 0,
-          "localBytesRead" : 0,
-          "recordsRead" : 0
-        },
-        "shuffleWriteMetrics" : {
-          "bytesWritten" : 0,
-          "writeTime" : 16858066,
-          "recordsWritten" : 0
-        }
-      },
-      "executorLogs" : {
-        "stdout" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000005/attilapiros/stdout?start=-4096",
-        "stderr" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000005/attilapiros/stderr?start=-4096"
-      },
-      "schedulerDelay" : 338,
-      "gettingResultTime" : 0
-    }
-  },
-  "executorSummary" : {
-    "4" : {
-      "taskTime" : 1589,
-      "failedTasks" : 2,
-      "succeededTasks" : 0,
-      "killedTasks" : 0,
-      "inputBytes" : 0,
-      "inputRecords" : 0,
-      "outputBytes" : 0,
-      "outputRecords" : 0,
-      "shuffleRead" : 0,
-      "shuffleReadRecords" : 0,
-      "shuffleWrite" : 0,
-      "shuffleWriteRecords" : 0,
-      "memoryBytesSpilled" : 0,
-      "diskBytesSpilled" : 0,
-      "isBlacklistedForStage" : true
-    },
-    "5" : {
-      "taskTime" : 1579,
-      "failedTasks" : 2,
-      "succeededTasks" : 0,
-      "killedTasks" : 0,
-      "inputBytes" : 0,
-      "inputRecords" : 0,
-      "outputBytes" : 0,
-      "outputRecords" : 0,
-      "shuffleRead" : 0,
-      "shuffleReadRecords" : 0,
-      "shuffleWrite" : 0,
-      "shuffleWriteRecords" : 0,
-      "memoryBytesSpilled" : 0,
-      "diskBytesSpilled" : 0,
-      "isBlacklistedForStage" : true
-    },
-    "1" : {
-      "taskTime" : 2411,
-      "failedTasks" : 0,
-      "succeededTasks" : 4,
-      "killedTasks" : 0,
-      "inputBytes" : 0,
-      "inputRecords" : 0,
-      "outputBytes" : 0,
-      "outputRecords" : 0,
-      "shuffleRead" : 0,
-      "shuffleReadRecords" : 0,
-      "shuffleWrite" : 585,
-      "shuffleWriteRecords" : 12,
-      "memoryBytesSpilled" : 0,
-      "diskBytesSpilled" : 0,
-      "isBlacklistedForStage" : false
-    },
-    "2" : {
-      "taskTime" : 2446,
-      "failedTasks" : 0,
-      "succeededTasks" : 5,
-      "killedTasks" : 0,
-      "inputBytes" : 0,
-      "inputRecords" : 0,
-      "outputBytes" : 0,
-      "outputRecords" : 0,
-      "shuffleRead" : 0,
-      "shuffleReadRecords" : 0,
-      "shuffleWrite" : 732,
-      "shuffleWriteRecords" : 15,
-      "memoryBytesSpilled" : 0,
-      "diskBytesSpilled" : 0,
-      "isBlacklistedForStage" : false
-    },
-    "3" : {
-      "taskTime" : 1774,
-      "failedTasks" : 0,
-      "succeededTasks" : 1,
-      "killedTasks" : 0,
-      "inputBytes" : 0,
-      "inputRecords" : 0,
-      "outputBytes" : 0,
-      "outputRecords" : 0,
-      "shuffleRead" : 0,
-      "shuffleReadRecords" : 0,
-      "shuffleWrite" : 144,
-      "shuffleWriteRecords" : 3,
-      "memoryBytesSpilled" : 0,
-      "diskBytesSpilled" : 0,
-      "isBlacklistedForStage" : true
-    }
-  },
-  "killedTasksSummary" : { }
-}
diff --git a/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json
index a47cd26ed102b..f04543e037c48 100644
--- a/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json
@@ -41,7 +41,30 @@
   "schedulingPool" : "default",
   "rddIds" : [ 6, 5 ],
   "accumulatorUpdates" : [ ],
-  "killedTasksSummary" : { }
+  "killedTasksSummary" : { },
+  "resourceProfileId" : 0,
+  "peakExecutorMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  }
 }, {
   "status" : "COMPLETE",
   "stageId" : 1,
@@ -85,7 +108,30 @@
   "schedulingPool" : "default",
   "rddIds" : [ 1, 0 ],
   "accumulatorUpdates" : [ ],
-  "killedTasksSummary" : { }
+  "killedTasksSummary" : { },
+  "resourceProfileId" : 0,
+  "peakExecutorMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  }
 }, {
   "status" : "COMPLETE",
   "stageId" : 0,
@@ -129,5 +175,28 @@
   "schedulingPool" : "default",
   "rddIds" : [ 0 ],
   "accumulatorUpdates" : [ ],
-  "killedTasksSummary" : { }
+  "killedTasksSummary" : { },
+  "resourceProfileId" : 0,
+  "peakExecutorMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  }
 } ]
diff --git a/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json
index 6e6d28b6a57ec..06015ec46e44d 100644
--- a/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json
@@ -1,4 +1,34 @@
 [ {
+  "id" : "app-20200706201101-0003",
+  "name" : "Spark shell",
+  "attempts" : [ {
+    "startTime" : "2020-07-07T03:11:00.235GMT",
+    "endTime" : "2020-07-07T03:17:04.231GMT",
+    "lastUpdated" : "",
+    "duration" : 363996,
+    "sparkUser" : "terryk",
+    "completed" : true,
+    "appSparkVersion" : "3.1.0-SNAPSHOT",
+    "endTimeEpoch" : 1594091824231,
+    "startTimeEpoch" : 1594091460235,
+    "lastUpdatedEpoch" : 0
+  } ]
+}, {
+  "id" : "application_1578436911597_0052",
+  "name" : "Spark shell",
+  "attempts" : [ {
+    "startTime" : "2020-01-11T17:44:22.851GMT",
+    "endTime" : "2020-01-11T17:46:42.615GMT",
+    "lastUpdated" : "",
+    "duration" : 139764,
+    "sparkUser" : "tgraves",
+    "completed" : true,
+    "appSparkVersion" : "3.0.0-SNAPSHOT",
+    "endTimeEpoch" : 1578764802615,
+    "startTimeEpoch" : 1578764662851,
+    "lastUpdatedEpoch" : 0
+  } ]
+}, {
   "id" : "application_1555004656427_0144",
   "name" : "Spark shell",
   "attempts" : [ {
diff --git a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json
new file mode 100644
index 0000000000000..dcad8a6895ed8
--- /dev/null
+++ b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json
@@ -0,0 +1,790 @@
+{
+  "status" : "COMPLETE",
+  "stageId" : 0,
+  "attemptId" : 0,
+  "numTasks" : 10,
+  "numActiveTasks" : 0,
+  "numCompleteTasks" : 10,
+  "numFailedTasks" : 2,
+  "numKilledTasks" : 0,
+  "numCompletedIndices" : 10,
+  "submissionTime" : "2018-01-09T10:21:18.152GMT",
+  "firstTaskLaunchedTime" : "2018-01-09T10:21:18.347GMT",
+  "completionTime" : "2018-01-09T10:21:19.062GMT",
+  "executorDeserializeTime" : 327,
+  "executorDeserializeCpuTime" : 225900000,
+  "executorRunTime" : 761,
+  "executorCpuTime" : 269916000,
+  "resultSize" : 10376,
+  "jvmGcTime" : 27,
+  "resultSerializationTime" : 1,
+  "memoryBytesSpilled" : 0,
+  "diskBytesSpilled" : 0,
+  "peakExecutionMemory" : 0,
+  "inputBytes" : 0,
+  "inputRecords" : 0,
+  "outputBytes" : 0,
+  "outputRecords" : 0,
+  "shuffleRemoteBlocksFetched" : 0,
+  "shuffleLocalBlocksFetched" : 0,
+  "shuffleFetchWaitTime" : 0,
+  "shuffleRemoteBytesRead" : 0,
+  "shuffleRemoteBytesReadToDisk" : 0,
+  "shuffleLocalBytesRead" : 0,
+  "shuffleReadBytes" : 0,
+  "shuffleReadRecords" : 0,
+  "shuffleWriteBytes" : 460,
+  "shuffleWriteTime" : 8711515,
+  "shuffleWriteRecords" : 10,
+  "name" : "map at <console>:26",
+  "details" : "org.apache.spark.rdd.RDD.map(RDD.scala:370)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:26)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:34)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:36)\n$line17.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:38)\n$line17.$read$$iw$$iw$$iw$$iw.<init>(<console>:40)\n$line17.$read$$iw$$iw$$iw.<init>(<console>:42)\n$line17.$read$$iw$$iw.<init>(<console>:44)\n$line17.$read$$iw.<init>(<console>:46)\n$line17.$read.<init>(<console>:48)\n$line17.$read$.<init>(<console>:52)\n$line17.$read$.<clinit>(<console>)\n$line17.$eval$.$print$lzycompute(<console>:7)\n$line17.$eval$.$print(<console>:6)\n$line17.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786)",
+  "schedulingPool" : "default",
+  "rddIds" : [ 1, 0 ],
+  "accumulatorUpdates" : [ ],
+  "tasks" : {
+    "0" : {
+      "taskId" : 0,
+      "index" : 0,
+      "attempt" : 0,
+      "launchTime" : "2018-01-09T10:21:18.347GMT",
+      "duration" : 562,
+      "executorId" : "0",
+      "host" : "172.30.65.138",
+      "status" : "FAILED",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "errorMessage" : "java.lang.RuntimeException: Bad executor\n\tat $line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(<console>:27)\n\tat $line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(<console>:26)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:193)\n\tat org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:63)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:109)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\n",
+      "taskMetrics" : {
+        "executorDeserializeTime" : 0,
+        "executorDeserializeCpuTime" : 0,
+        "executorRunTime" : 460,
+        "executorCpuTime" : 0,
+        "resultSize" : 0,
+        "jvmGcTime" : 14,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 3873006,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://172.30.65.138:64279/logPage/?appId=app-20180109111548-0000&executorId=0&logType=stdout",
+        "stderr" : "http://172.30.65.138:64279/logPage/?appId=app-20180109111548-0000&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 102,
+      "gettingResultTime" : 0
+    },
+    "5" : {
+      "taskId" : 5,
+      "index" : 3,
+      "attempt" : 0,
+      "launchTime" : "2018-01-09T10:21:18.958GMT",
+      "duration" : 22,
+      "executorId" : "1",
+      "host" : "172.30.65.138",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 3,
+        "executorDeserializeCpuTime" : 2586000,
+        "executorRunTime" : 9,
+        "executorCpuTime" : 9635000,
+        "resultSize" : 1029,
+        "jvmGcTime" : 0,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 46,
+          "writeTime" : 262919,
+          "recordsWritten" : 1
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
+        "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
+      },
+      "schedulerDelay" : 10,
+      "gettingResultTime" : 0
+    },
+    "10" : {
+      "taskId" : 10,
+      "index" : 8,
+      "attempt" : 0,
+      "launchTime" : "2018-01-09T10:21:19.034GMT",
+      "duration" : 12,
+      "executorId" : "1",
+      "host" : "172.30.65.138",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 2,
+        "executorDeserializeCpuTime" : 1803000,
+        "executorRunTime" : 6,
+        "executorCpuTime" : 6157000,
+        "resultSize" : 1029,
+        "jvmGcTime" : 0,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 46,
+          "writeTime" : 243647,
+          "recordsWritten" : 1
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
+        "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
+      },
+      "schedulerDelay" : 4,
+      "gettingResultTime" : 0
+    },
+    "1" : {
+      "taskId" : 1,
+      "index" : 1,
+      "attempt" : 0,
+      "launchTime" : "2018-01-09T10:21:18.364GMT",
+      "duration" : 565,
+      "executorId" : "1",
+      "host" : "172.30.65.138",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 301,
+        "executorDeserializeCpuTime" : 200029000,
+        "executorRunTime" : 212,
+        "executorCpuTime" : 198479000,
+        "resultSize" : 1115,
+        "jvmGcTime" : 13,
+        "resultSerializationTime" : 1,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 46,
+          "writeTime" : 2409488,
+          "recordsWritten" : 1
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
+        "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
+      },
+      "schedulerDelay" : 51,
+      "gettingResultTime" : 0
+    },
+    "6" : {
+      "taskId" : 6,
+      "index" : 4,
+      "attempt" : 0,
+      "launchTime" : "2018-01-09T10:21:18.980GMT",
+      "duration" : 16,
+      "executorId" : "1",
+      "host" : "172.30.65.138",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 3,
+        "executorDeserializeCpuTime" : 2610000,
+        "executorRunTime" : 10,
+        "executorCpuTime" : 9622000,
+        "resultSize" : 1029,
+        "jvmGcTime" : 0,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 46,
+          "writeTime" : 385110,
+          "recordsWritten" : 1
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
+        "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
+      },
+      "schedulerDelay" : 3,
+      "gettingResultTime" : 0
+    },
+    "9" : {
+      "taskId" : 9,
+      "index" : 7,
+      "attempt" : 0,
+      "launchTime" : "2018-01-09T10:21:19.022GMT",
+      "duration" : 12,
+      "executorId" : "1",
+      "host" : "172.30.65.138",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 2,
+        "executorDeserializeCpuTime" : 1981000,
+        "executorRunTime" : 7,
+        "executorCpuTime" : 6335000,
+        "resultSize" : 1029,
+        "jvmGcTime" : 0,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 46,
+          "writeTime" : 259354,
+          "recordsWritten" : 1
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
+        "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
+      },
+      "schedulerDelay" : 3,
+      "gettingResultTime" : 0
+    },
+    "2" : {
+      "taskId" : 2,
+      "index" : 2,
+      "attempt" : 0,
+      "launchTime" : "2018-01-09T10:21:18.899GMT",
+      "duration" : 27,
+      "executorId" : "0",
+      "host" : "172.30.65.138",
+      "status" : "FAILED",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "errorMessage" : "java.lang.RuntimeException: Bad executor\n\tat $line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(<console>:27)\n\tat $line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(<console>:26)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:193)\n\tat org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:63)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:109)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\n",
+      "taskMetrics" : {
+        "executorDeserializeTime" : 0,
+        "executorDeserializeCpuTime" : 0,
+        "executorRunTime" : 16,
+        "executorCpuTime" : 0,
+        "resultSize" : 0,
+        "jvmGcTime" : 0,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 126128,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://172.30.65.138:64279/logPage/?appId=app-20180109111548-0000&executorId=0&logType=stdout",
+        "stderr" : "http://172.30.65.138:64279/logPage/?appId=app-20180109111548-0000&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 11,
+      "gettingResultTime" : 0
+    },
+    "7" : {
+      "taskId" : 7,
+      "index" : 5,
+      "attempt" : 0,
+      "launchTime" : "2018-01-09T10:21:18.996GMT",
+      "duration" : 15,
+      "executorId" : "1",
+      "host" : "172.30.65.138",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 2,
+        "executorDeserializeCpuTime" : 2231000,
+        "executorRunTime" : 9,
+        "executorCpuTime" : 8407000,
+        "resultSize" : 1029,
+        "jvmGcTime" : 0,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 46,
+          "writeTime" : 205520,
+          "recordsWritten" : 1
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
+        "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
+      },
+      "schedulerDelay" : 4,
+      "gettingResultTime" : 0
+    },
+    "3" : {
+      "taskId" : 3,
+      "index" : 0,
+      "attempt" : 1,
+      "launchTime" : "2018-01-09T10:21:18.919GMT",
+      "duration" : 24,
+      "executorId" : "1",
+      "host" : "172.30.65.138",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 8,
+        "executorDeserializeCpuTime" : 8878000,
+        "executorRunTime" : 10,
+        "executorCpuTime" : 9364000,
+        "resultSize" : 1029,
+        "jvmGcTime" : 0,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 46,
+          "writeTime" : 207014,
+          "recordsWritten" : 1
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
+        "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
+      },
+      "schedulerDelay" : 6,
+      "gettingResultTime" : 0
+    },
+    "11" : {
+      "taskId" : 11,
+      "index" : 9,
+      "attempt" : 0,
+      "launchTime" : "2018-01-09T10:21:19.045GMT",
+      "duration" : 15,
+      "executorId" : "1",
+      "host" : "172.30.65.138",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 3,
+        "executorDeserializeCpuTime" : 2017000,
+        "executorRunTime" : 6,
+        "executorCpuTime" : 6676000,
+        "resultSize" : 1029,
+        "jvmGcTime" : 0,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 46,
+          "writeTime" : 233652,
+          "recordsWritten" : 1
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
+        "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
+      },
+      "schedulerDelay" : 6,
+      "gettingResultTime" : 0
+    },
+    "8" : {
+      "taskId" : 8,
+      "index" : 6,
+      "attempt" : 0,
+      "launchTime" : "2018-01-09T10:21:19.011GMT",
+      "duration" : 11,
+      "executorId" : "1",
+      "host" : "172.30.65.138",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 1,
+        "executorDeserializeCpuTime" : 1554000,
+        "executorRunTime" : 7,
+        "executorCpuTime" : 6034000,
+        "resultSize" : 1029,
+        "jvmGcTime" : 0,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 46,
+          "writeTime" : 213296,
+          "recordsWritten" : 1
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
+        "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
+      },
+      "schedulerDelay" : 3,
+      "gettingResultTime" : 0
+    },
+    "4" : {
+      "taskId" : 4,
+      "index" : 2,
+      "attempt" : 1,
+      "launchTime" : "2018-01-09T10:21:18.943GMT",
+      "duration" : 16,
+      "executorId" : "1",
+      "host" : "172.30.65.138",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 2,
+        "executorDeserializeCpuTime" : 2211000,
+        "executorRunTime" : 9,
+        "executorCpuTime" : 9207000,
+        "resultSize" : 1029,
+        "jvmGcTime" : 0,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 46,
+          "writeTime" : 292381,
+          "recordsWritten" : 1
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
+        "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
+      },
+      "schedulerDelay" : 5,
+      "gettingResultTime" : 0
+    }
+  },
+  "executorSummary" : {
+    "0" : {
+      "taskTime" : 589,
+      "failedTasks" : 2,
+      "succeededTasks" : 0,
+      "killedTasks" : 0,
+      "inputBytes" : 0,
+      "inputRecords" : 0,
+      "outputBytes" : 0,
+      "outputRecords" : 0,
+      "shuffleRead" : 0,
+      "shuffleReadRecords" : 0,
+      "shuffleWrite" : 0,
+      "shuffleWriteRecords" : 0,
+      "memoryBytesSpilled" : 0,
+      "diskBytesSpilled" : 0,
+      "isBlacklistedForStage" : true,
+      "peakMemoryMetrics" : {
+        "JVMHeapMemory" : 0,
+        "JVMOffHeapMemory" : 0,
+        "OnHeapExecutionMemory" : 0,
+        "OffHeapExecutionMemory" : 0,
+        "OnHeapStorageMemory" : 0,
+        "OffHeapStorageMemory" : 0,
+        "OnHeapUnifiedMemory" : 0,
+        "OffHeapUnifiedMemory" : 0,
+        "DirectPoolMemory" : 0,
+        "MappedPoolMemory" : 0,
+        "ProcessTreeJVMVMemory" : 0,
+        "ProcessTreeJVMRSSMemory" : 0,
+        "ProcessTreePythonVMemory" : 0,
+        "ProcessTreePythonRSSMemory" : 0,
+        "ProcessTreeOtherVMemory" : 0,
+        "ProcessTreeOtherRSSMemory" : 0,
+        "MinorGCCount" : 0,
+        "MinorGCTime" : 0,
+        "MajorGCCount" : 0,
+        "MajorGCTime" : 0
+      },
+      "isExcludedForStage" : true
+    },
+    "1" : {
+      "taskTime" : 708,
+      "failedTasks" : 0,
+      "succeededTasks" : 10,
+      "killedTasks" : 0,
+      "inputBytes" : 0,
+      "inputRecords" : 0,
+      "outputBytes" : 0,
+      "outputRecords" : 0,
+      "shuffleRead" : 0,
+      "shuffleReadRecords" : 0,
+      "shuffleWrite" : 460,
+      "shuffleWriteRecords" : 10,
+      "memoryBytesSpilled" : 0,
+      "diskBytesSpilled" : 0,
+      "isBlacklistedForStage" : false,
+      "peakMemoryMetrics" : {
+        "JVMHeapMemory" : 0,
+        "JVMOffHeapMemory" : 0,
+        "OnHeapExecutionMemory" : 0,
+        "OffHeapExecutionMemory" : 0,
+        "OnHeapStorageMemory" : 0,
+        "OffHeapStorageMemory" : 0,
+        "OnHeapUnifiedMemory" : 0,
+        "OffHeapUnifiedMemory" : 0,
+        "DirectPoolMemory" : 0,
+        "MappedPoolMemory" : 0,
+        "ProcessTreeJVMVMemory" : 0,
+        "ProcessTreeJVMRSSMemory" : 0,
+        "ProcessTreePythonVMemory" : 0,
+        "ProcessTreePythonRSSMemory" : 0,
+        "ProcessTreeOtherVMemory" : 0,
+        "ProcessTreeOtherRSSMemory" : 0,
+        "MinorGCCount" : 0,
+        "MinorGCTime" : 0,
+        "MajorGCCount" : 0,
+        "MajorGCTime" : 0
+      },
+      "isExcludedForStage" : false
+    }
+  },
+  "killedTasksSummary" : { },
+  "resourceProfileId" : 0,
+  "peakExecutorMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  }
+}
diff --git a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json
new file mode 100644
index 0000000000000..2ab1546bd4a86
--- /dev/null
+++ b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json
@@ -0,0 +1,1018 @@
+{
+  "status" : "COMPLETE",
+  "stageId" : 0,
+  "attemptId" : 0,
+  "numTasks" : 10,
+  "numActiveTasks" : 0,
+  "numCompleteTasks" : 10,
+  "numFailedTasks" : 4,
+  "numKilledTasks" : 0,
+  "numCompletedIndices" : 10,
+  "submissionTime" : "2018-01-18T18:33:12.658GMT",
+  "firstTaskLaunchedTime" : "2018-01-18T18:33:12.816GMT",
+  "completionTime" : "2018-01-18T18:33:15.279GMT",
+  "executorDeserializeTime" : 3679,
+  "executorDeserializeCpuTime" : 1029819716,
+  "executorRunTime" : 5080,
+  "executorCpuTime" : 1163210819,
+  "resultSize" : 10824,
+  "jvmGcTime" : 370,
+  "resultSerializationTime" : 5,
+  "memoryBytesSpilled" : 0,
+  "diskBytesSpilled" : 0,
+  "peakExecutionMemory" : 0,
+  "inputBytes" : 0,
+  "inputRecords" : 0,
+  "outputBytes" : 0,
+  "outputRecords" : 0,
+  "shuffleRemoteBlocksFetched" : 0,
+  "shuffleLocalBlocksFetched" : 0,
+  "shuffleFetchWaitTime" : 0,
+  "shuffleRemoteBytesRead" : 0,
+  "shuffleRemoteBytesReadToDisk" : 0,
+  "shuffleLocalBytesRead" : 0,
+  "shuffleReadBytes" : 0,
+  "shuffleReadRecords" : 0,
+  "shuffleWriteBytes" : 1461,
+  "shuffleWriteTime" : 33251697,
+  "shuffleWriteRecords" : 30,
+  "name" : "map at <console>:27",
+  "details" : "org.apache.spark.rdd.RDD.map(RDD.scala:370)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:27)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:35)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:37)\n$line15.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:39)\n$line15.$read$$iw$$iw$$iw$$iw.<init>(<console>:41)\n$line15.$read$$iw$$iw$$iw.<init>(<console>:43)\n$line15.$read$$iw$$iw.<init>(<console>:45)\n$line15.$read$$iw.<init>(<console>:47)\n$line15.$read.<init>(<console>:49)\n$line15.$read$.<init>(<console>:53)\n$line15.$read$.<clinit>(<console>)\n$line15.$eval$.$print$lzycompute(<console>:7)\n$line15.$eval$.$print(<console>:6)\n$line15.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786)",
+  "schedulingPool" : "default",
+  "rddIds" : [ 1, 0 ],
+  "accumulatorUpdates" : [ ],
+  "tasks" : {
+    "0" : {
+      "taskId" : 0,
+      "index" : 0,
+      "attempt" : 0,
+      "launchTime" : "2018-01-18T18:33:12.816GMT",
+      "duration" : 2064,
+      "executorId" : "1",
+      "host" : "apiros-3.gce.test.com",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 1081,
+        "executorDeserializeCpuTime" : 353981050,
+        "executorRunTime" : 914,
+        "executorCpuTime" : 368865439,
+        "resultSize" : 1134,
+        "jvmGcTime" : 75,
+        "resultSerializationTime" : 1,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 144,
+          "writeTime" : 3662221,
+          "recordsWritten" : 3
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stdout?start=-4096",
+        "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stderr?start=-4096"
+      },
+      "schedulerDelay" : 68,
+      "gettingResultTime" : 0
+    },
+    "5" : {
+      "taskId" : 5,
+      "index" : 5,
+      "attempt" : 0,
+      "launchTime" : "2018-01-18T18:33:14.320GMT",
+      "duration" : 73,
+      "executorId" : "5",
+      "host" : "apiros-2.gce.test.com",
+      "status" : "FAILED",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "errorMessage" : "java.lang.RuntimeException: Bad executor\n\tat $line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(<console>:28)\n\tat $line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(<console>:27)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:193)\n\tat org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:63)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:109)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\n",
+      "taskMetrics" : {
+        "executorDeserializeTime" : 0,
+        "executorDeserializeCpuTime" : 0,
+        "executorRunTime" : 27,
+        "executorCpuTime" : 0,
+        "resultSize" : 0,
+        "jvmGcTime" : 0,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 191901,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000007/attilapiros/stdout?start=-4096",
+        "stderr" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000007/attilapiros/stderr?start=-4096"
+      },
+      "schedulerDelay" : 46,
+      "gettingResultTime" : 0
+    },
+    "10" : {
+      "taskId" : 10,
+      "index" : 1,
+      "attempt" : 1,
+      "launchTime" : "2018-01-18T18:33:15.069GMT",
+      "duration" : 132,
+      "executorId" : "2",
+      "host" : "apiros-3.gce.test.com",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 5,
+        "executorDeserializeCpuTime" : 4598966,
+        "executorRunTime" : 76,
+        "executorCpuTime" : 20826337,
+        "resultSize" : 1091,
+        "jvmGcTime" : 0,
+        "resultSerializationTime" : 1,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 144,
+          "writeTime" : 301705,
+          "recordsWritten" : 3
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stdout?start=-4096",
+        "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stderr?start=-4096"
+      },
+      "schedulerDelay" : 50,
+      "gettingResultTime" : 0
+    },
+    "1" : {
+      "taskId" : 1,
+      "index" : 1,
+      "attempt" : 0,
+      "launchTime" : "2018-01-18T18:33:12.832GMT",
+      "duration" : 1506,
+      "executorId" : "5",
+      "host" : "apiros-2.gce.test.com",
+      "status" : "FAILED",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "errorMessage" : "java.lang.RuntimeException: Bad executor\n\tat $line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(<console>:28)\n\tat $line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(<console>:27)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:193)\n\tat org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:63)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:109)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\n",
+      "taskMetrics" : {
+        "executorDeserializeTime" : 0,
+        "executorDeserializeCpuTime" : 0,
+        "executorRunTime" : 1332,
+        "executorCpuTime" : 0,
+        "resultSize" : 0,
+        "jvmGcTime" : 33,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 3075188,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000007/attilapiros/stdout?start=-4096",
+        "stderr" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000007/attilapiros/stderr?start=-4096"
+      },
+      "schedulerDelay" : 174,
+      "gettingResultTime" : 0
+    },
+    "6" : {
+      "taskId" : 6,
+      "index" : 6,
+      "attempt" : 0,
+      "launchTime" : "2018-01-18T18:33:14.323GMT",
+      "duration" : 67,
+      "executorId" : "4",
+      "host" : "apiros-2.gce.test.com",
+      "status" : "FAILED",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "errorMessage" : "java.lang.RuntimeException: Bad executor\n\tat $line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(<console>:28)\n\tat $line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(<console>:27)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:193)\n\tat org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:63)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:109)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\n",
+      "taskMetrics" : {
+        "executorDeserializeTime" : 0,
+        "executorDeserializeCpuTime" : 0,
+        "executorRunTime" : 51,
+        "executorCpuTime" : 0,
+        "resultSize" : 0,
+        "jvmGcTime" : 0,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 183718,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000005/attilapiros/stdout?start=-4096",
+        "stderr" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000005/attilapiros/stderr?start=-4096"
+      },
+      "schedulerDelay" : 16,
+      "gettingResultTime" : 0
+    },
+    "9" : {
+      "taskId" : 9,
+      "index" : 4,
+      "attempt" : 1,
+      "launchTime" : "2018-01-18T18:33:14.973GMT",
+      "duration" : 96,
+      "executorId" : "2",
+      "host" : "apiros-3.gce.test.com",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 5,
+        "executorDeserializeCpuTime" : 4793905,
+        "executorRunTime" : 48,
+        "executorCpuTime" : 25678331,
+        "resultSize" : 1091,
+        "jvmGcTime" : 0,
+        "resultSerializationTime" : 1,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 147,
+          "writeTime" : 366050,
+          "recordsWritten" : 3
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stdout?start=-4096",
+        "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stderr?start=-4096"
+      },
+      "schedulerDelay" : 42,
+      "gettingResultTime" : 0
+    },
+    "13" : {
+      "taskId" : 13,
+      "index" : 9,
+      "attempt" : 0,
+      "launchTime" : "2018-01-18T18:33:15.200GMT",
+      "duration" : 76,
+      "executorId" : "2",
+      "host" : "apiros-3.gce.test.com",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 25,
+        "executorDeserializeCpuTime" : 5860574,
+        "executorRunTime" : 25,
+        "executorCpuTime" : 20585619,
+        "resultSize" : 1048,
+        "jvmGcTime" : 0,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 147,
+          "writeTime" : 369513,
+          "recordsWritten" : 3
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stdout?start=-4096",
+        "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stderr?start=-4096"
+      },
+      "schedulerDelay" : 26,
+      "gettingResultTime" : 0
+    },
+    "2" : {
+      "taskId" : 2,
+      "index" : 2,
+      "attempt" : 0,
+      "launchTime" : "2018-01-18T18:33:12.832GMT",
+      "duration" : 1774,
+      "executorId" : "3",
+      "host" : "apiros-2.gce.test.com",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 1206,
+        "executorDeserializeCpuTime" : 263386625,
+        "executorRunTime" : 493,
+        "executorCpuTime" : 278399617,
+        "resultSize" : 1134,
+        "jvmGcTime" : 78,
+        "resultSerializationTime" : 1,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 144,
+          "writeTime" : 3322956,
+          "recordsWritten" : 3
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000004/attilapiros/stdout?start=-4096",
+        "stderr" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000004/attilapiros/stderr?start=-4096"
+      },
+      "schedulerDelay" : 74,
+      "gettingResultTime" : 0
+    },
+    "12" : {
+      "taskId" : 12,
+      "index" : 8,
+      "attempt" : 0,
+      "launchTime" : "2018-01-18T18:33:15.165GMT",
+      "duration" : 60,
+      "executorId" : "1",
+      "host" : "apiros-3.gce.test.com",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 4,
+        "executorDeserializeCpuTime" : 4010338,
+        "executorRunTime" : 34,
+        "executorCpuTime" : 21657558,
+        "resultSize" : 1048,
+        "jvmGcTime" : 0,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 147,
+          "writeTime" : 319101,
+          "recordsWritten" : 3
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stdout?start=-4096",
+        "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stderr?start=-4096"
+      },
+      "schedulerDelay" : 22,
+      "gettingResultTime" : 0
+    },
+    "7" : {
+      "taskId" : 7,
+      "index" : 5,
+      "attempt" : 1,
+      "launchTime" : "2018-01-18T18:33:14.859GMT",
+      "duration" : 115,
+      "executorId" : "2",
+      "host" : "apiros-3.gce.test.com",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 11,
+        "executorDeserializeCpuTime" : 10894331,
+        "executorRunTime" : 84,
+        "executorCpuTime" : 28283110,
+        "resultSize" : 1048,
+        "jvmGcTime" : 0,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 147,
+          "writeTime" : 377601,
+          "recordsWritten" : 3
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stdout?start=-4096",
+        "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stderr?start=-4096"
+      },
+      "schedulerDelay" : 20,
+      "gettingResultTime" : 0
+    },
+    "3" : {
+      "taskId" : 3,
+      "index" : 3,
+      "attempt" : 0,
+      "launchTime" : "2018-01-18T18:33:12.833GMT",
+      "duration" : 2027,
+      "executorId" : "2",
+      "host" : "apiros-3.gce.test.com",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 1282,
+        "executorDeserializeCpuTime" : 365807898,
+        "executorRunTime" : 681,
+        "executorCpuTime" : 349920830,
+        "resultSize" : 1134,
+        "jvmGcTime" : 102,
+        "resultSerializationTime" : 1,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 147,
+          "writeTime" : 3587839,
+          "recordsWritten" : 3
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stdout?start=-4096",
+        "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stderr?start=-4096"
+      },
+      "schedulerDelay" : 63,
+      "gettingResultTime" : 0
+    },
+    "11" : {
+      "taskId" : 11,
+      "index" : 7,
+      "attempt" : 0,
+      "launchTime" : "2018-01-18T18:33:15.072GMT",
+      "duration" : 93,
+      "executorId" : "1",
+      "host" : "apiros-3.gce.test.com",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 4,
+        "executorDeserializeCpuTime" : 4239884,
+        "executorRunTime" : 77,
+        "executorCpuTime" : 21689428,
+        "resultSize" : 1048,
+        "jvmGcTime" : 0,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 147,
+          "writeTime" : 323898,
+          "recordsWritten" : 3
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stdout?start=-4096",
+        "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stderr?start=-4096"
+      },
+      "schedulerDelay" : 12,
+      "gettingResultTime" : 0
+    },
+    "8" : {
+      "taskId" : 8,
+      "index" : 6,
+      "attempt" : 1,
+      "launchTime" : "2018-01-18T18:33:14.879GMT",
+      "duration" : 194,
+      "executorId" : "1",
+      "host" : "apiros-3.gce.test.com",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 56,
+        "executorDeserializeCpuTime" : 12246145,
+        "executorRunTime" : 54,
+        "executorCpuTime" : 27304550,
+        "resultSize" : 1048,
+        "jvmGcTime" : 0,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 147,
+          "writeTime" : 311940,
+          "recordsWritten" : 3
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stdout?start=-4096",
+        "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stderr?start=-4096"
+      },
+      "schedulerDelay" : 84,
+      "gettingResultTime" : 0
+    },
+    "4" : {
+      "taskId" : 4,
+      "index" : 4,
+      "attempt" : 0,
+      "launchTime" : "2018-01-18T18:33:12.833GMT",
+      "duration" : 1522,
+      "executorId" : "4",
+      "host" : "apiros-2.gce.test.com",
+      "status" : "FAILED",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "errorMessage" : "java.lang.RuntimeException: Bad executor\n\tat $line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(<console>:28)\n\tat $line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(<console>:27)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:193)\n\tat org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:63)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:109)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\n",
+      "taskMetrics" : {
+        "executorDeserializeTime" : 0,
+        "executorDeserializeCpuTime" : 0,
+        "executorRunTime" : 1184,
+        "executorCpuTime" : 0,
+        "resultSize" : 0,
+        "jvmGcTime" : 82,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 16858066,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000005/attilapiros/stdout?start=-4096",
+        "stderr" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000005/attilapiros/stderr?start=-4096"
+      },
+      "schedulerDelay" : 338,
+      "gettingResultTime" : 0
+    }
+  },
+  "executorSummary" : {
+    "4" : {
+      "taskTime" : 1589,
+      "failedTasks" : 2,
+      "succeededTasks" : 0,
+      "killedTasks" : 0,
+      "inputBytes" : 0,
+      "inputRecords" : 0,
+      "outputBytes" : 0,
+      "outputRecords" : 0,
+      "shuffleRead" : 0,
+      "shuffleReadRecords" : 0,
+      "shuffleWrite" : 0,
+      "shuffleWriteRecords" : 0,
+      "memoryBytesSpilled" : 0,
+      "diskBytesSpilled" : 0,
+      "isBlacklistedForStage" : true,
+      "peakMemoryMetrics" : {
+        "JVMHeapMemory" : 0,
+        "JVMOffHeapMemory" : 0,
+        "OnHeapExecutionMemory" : 0,
+        "OffHeapExecutionMemory" : 0,
+        "OnHeapStorageMemory" : 0,
+        "OffHeapStorageMemory" : 0,
+        "OnHeapUnifiedMemory" : 0,
+        "OffHeapUnifiedMemory" : 0,
+        "DirectPoolMemory" : 0,
+        "MappedPoolMemory" : 0,
+        "ProcessTreeJVMVMemory" : 0,
+        "ProcessTreeJVMRSSMemory" : 0,
+        "ProcessTreePythonVMemory" : 0,
+        "ProcessTreePythonRSSMemory" : 0,
+        "ProcessTreeOtherVMemory" : 0,
+        "ProcessTreeOtherRSSMemory" : 0,
+        "MinorGCCount" : 0,
+        "MinorGCTime" : 0,
+        "MajorGCCount" : 0,
+        "MajorGCTime" : 0
+      },
+      "isExcludedForStage" : true
+    },
+    "5" : {
+      "taskTime" : 1579,
+      "failedTasks" : 2,
+      "succeededTasks" : 0,
+      "killedTasks" : 0,
+      "inputBytes" : 0,
+      "inputRecords" : 0,
+      "outputBytes" : 0,
+      "outputRecords" : 0,
+      "shuffleRead" : 0,
+      "shuffleReadRecords" : 0,
+      "shuffleWrite" : 0,
+      "shuffleWriteRecords" : 0,
+      "memoryBytesSpilled" : 0,
+      "diskBytesSpilled" : 0,
+      "isBlacklistedForStage" : true,
+      "peakMemoryMetrics" : {
+        "JVMHeapMemory" : 0,
+        "JVMOffHeapMemory" : 0,
+        "OnHeapExecutionMemory" : 0,
+        "OffHeapExecutionMemory" : 0,
+        "OnHeapStorageMemory" : 0,
+        "OffHeapStorageMemory" : 0,
+        "OnHeapUnifiedMemory" : 0,
+        "OffHeapUnifiedMemory" : 0,
+        "DirectPoolMemory" : 0,
+        "MappedPoolMemory" : 0,
+        "ProcessTreeJVMVMemory" : 0,
+        "ProcessTreeJVMRSSMemory" : 0,
+        "ProcessTreePythonVMemory" : 0,
+        "ProcessTreePythonRSSMemory" : 0,
+        "ProcessTreeOtherVMemory" : 0,
+        "ProcessTreeOtherRSSMemory" : 0,
+        "MinorGCCount" : 0,
+        "MinorGCTime" : 0,
+        "MajorGCCount" : 0,
+        "MajorGCTime" : 0
+      },
+      "isExcludedForStage" : true
+    },
+    "1" : {
+      "taskTime" : 2411,
+      "failedTasks" : 0,
+      "succeededTasks" : 4,
+      "killedTasks" : 0,
+      "inputBytes" : 0,
+      "inputRecords" : 0,
+      "outputBytes" : 0,
+      "outputRecords" : 0,
+      "shuffleRead" : 0,
+      "shuffleReadRecords" : 0,
+      "shuffleWrite" : 585,
+      "shuffleWriteRecords" : 12,
+      "memoryBytesSpilled" : 0,
+      "diskBytesSpilled" : 0,
+      "isBlacklistedForStage" : false,
+      "peakMemoryMetrics" : {
+        "JVMHeapMemory" : 0,
+        "JVMOffHeapMemory" : 0,
+        "OnHeapExecutionMemory" : 0,
+        "OffHeapExecutionMemory" : 0,
+        "OnHeapStorageMemory" : 0,
+        "OffHeapStorageMemory" : 0,
+        "OnHeapUnifiedMemory" : 0,
+        "OffHeapUnifiedMemory" : 0,
+        "DirectPoolMemory" : 0,
+        "MappedPoolMemory" : 0,
+        "ProcessTreeJVMVMemory" : 0,
+        "ProcessTreeJVMRSSMemory" : 0,
+        "ProcessTreePythonVMemory" : 0,
+        "ProcessTreePythonRSSMemory" : 0,
+        "ProcessTreeOtherVMemory" : 0,
+        "ProcessTreeOtherRSSMemory" : 0,
+        "MinorGCCount" : 0,
+        "MinorGCTime" : 0,
+        "MajorGCCount" : 0,
+        "MajorGCTime" : 0
+      },
+      "isExcludedForStage" : false
+    },
+    "2" : {
+      "taskTime" : 2446,
+      "failedTasks" : 0,
+      "succeededTasks" : 5,
+      "killedTasks" : 0,
+      "inputBytes" : 0,
+      "inputRecords" : 0,
+      "outputBytes" : 0,
+      "outputRecords" : 0,
+      "shuffleRead" : 0,
+      "shuffleReadRecords" : 0,
+      "shuffleWrite" : 732,
+      "shuffleWriteRecords" : 15,
+      "memoryBytesSpilled" : 0,
+      "diskBytesSpilled" : 0,
+      "isBlacklistedForStage" : false,
+      "peakMemoryMetrics" : {
+        "JVMHeapMemory" : 0,
+        "JVMOffHeapMemory" : 0,
+        "OnHeapExecutionMemory" : 0,
+        "OffHeapExecutionMemory" : 0,
+        "OnHeapStorageMemory" : 0,
+        "OffHeapStorageMemory" : 0,
+        "OnHeapUnifiedMemory" : 0,
+        "OffHeapUnifiedMemory" : 0,
+        "DirectPoolMemory" : 0,
+        "MappedPoolMemory" : 0,
+        "ProcessTreeJVMVMemory" : 0,
+        "ProcessTreeJVMRSSMemory" : 0,
+        "ProcessTreePythonVMemory" : 0,
+        "ProcessTreePythonRSSMemory" : 0,
+        "ProcessTreeOtherVMemory" : 0,
+        "ProcessTreeOtherRSSMemory" : 0,
+        "MinorGCCount" : 0,
+        "MinorGCTime" : 0,
+        "MajorGCCount" : 0,
+        "MajorGCTime" : 0
+      },
+      "isExcludedForStage" : false
+    },
+    "3" : {
+      "taskTime" : 1774,
+      "failedTasks" : 0,
+      "succeededTasks" : 1,
+      "killedTasks" : 0,
+      "inputBytes" : 0,
+      "inputRecords" : 0,
+      "outputBytes" : 0,
+      "outputRecords" : 0,
+      "shuffleRead" : 0,
+      "shuffleReadRecords" : 0,
+      "shuffleWrite" : 144,
+      "shuffleWriteRecords" : 3,
+      "memoryBytesSpilled" : 0,
+      "diskBytesSpilled" : 0,
+      "isBlacklistedForStage" : true,
+      "peakMemoryMetrics" : {
+        "JVMHeapMemory" : 0,
+        "JVMOffHeapMemory" : 0,
+        "OnHeapExecutionMemory" : 0,
+        "OffHeapExecutionMemory" : 0,
+        "OnHeapStorageMemory" : 0,
+        "OffHeapStorageMemory" : 0,
+        "OnHeapUnifiedMemory" : 0,
+        "OffHeapUnifiedMemory" : 0,
+        "DirectPoolMemory" : 0,
+        "MappedPoolMemory" : 0,
+        "ProcessTreeJVMVMemory" : 0,
+        "ProcessTreeJVMRSSMemory" : 0,
+        "ProcessTreePythonVMemory" : 0,
+        "ProcessTreePythonRSSMemory" : 0,
+        "ProcessTreeOtherVMemory" : 0,
+        "ProcessTreeOtherRSSMemory" : 0,
+        "MinorGCCount" : 0,
+        "MinorGCTime" : 0,
+        "MajorGCCount" : 0,
+        "MajorGCTime" : 0
+      },
+      "isExcludedForStage" : true
+    }
+  },
+  "killedTasksSummary" : { },
+  "resourceProfileId" : 0,
+  "peakExecutorMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  }
+}
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
index eadf27164c814..be125075874a2 100644
--- a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
@@ -21,6 +21,31 @@
   "addTime" : "2015-02-03T16:43:00.906GMT",
   "executorLogs" : { },
   "blacklistedInStages" : [ ],
+  "peakMemoryMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  },
   "attributes" : { },
-  "resources" : { }
+  "resources" : { },
+  "resourceProfileId" : 0,
+  "isExcluded" : false,
+  "excludedInStages" : [ ]
 } ]
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json
index d322485baa8de..bf3e93f3d3783 100644
--- a/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json
@@ -50,7 +50,10 @@
     "MajorGCTime" : 144
   },
   "attributes" : { },
-  "resources" : { }
+  "resources" : { },
+  "resourceProfileId" : 0,
+  "isExcluded" : false,
+  "excludedInStages" : [ ]
 }, {
   "id" : "3",
   "hostPort" : "test-3.vpc.company.com:37641",
@@ -116,7 +119,10 @@
     "NM_HOST" : "test-3.vpc.company.com",
     "CONTAINER_ID" : "container_1553914137147_0018_01_000004"
   },
-  "resources" : { }
+  "resources" : { },
+  "resourceProfileId" : 0,
+  "isExcluded" : false,
+  "excludedInStages" : [ ]
 }, {
   "id" : "2",
   "hostPort" : "test-4.vpc.company.com:33179",
@@ -182,7 +188,10 @@
     "NM_HOST" : "test-4.vpc.company.com",
     "CONTAINER_ID" : "container_1553914137147_0018_01_000003"
   },
-  "resources" : { }
+  "resources" : { },
+  "resourceProfileId" : 0,
+  "isExcluded" : false,
+  "excludedInStages" : [ ]
 }, {
   "id" : "1",
   "hostPort" : "test-2.vpc.company.com:43764",
@@ -248,5 +257,8 @@
     "NM_HOST" : "test-2.vpc.company.com",
     "CONTAINER_ID" : "container_1553914137147_0018_01_000002"
   },
-  "resources" : { }
+  "resources" : { },
+  "resourceProfileId" : 0,
+  "isExcluded" : false,
+  "excludedInStages" : [ ]
 } ]
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
index 7c3f77d8c10cf..0a3eb81140cdb 100644
--- a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
@@ -16,7 +16,7 @@
   "totalInputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 0,
-  "isBlacklisted" : true,
+  "isBlacklisted" : false,
   "maxMemory" : 908381388,
   "addTime" : "2016-11-16T22:33:31.477GMT",
   "executorLogs" : { },
@@ -28,7 +28,10 @@
   },
   "blacklistedInStages" : [ ],
   "attributes" : { },
-  "resources" : { }
+  "resources" : { },
+  "resourceProfileId" : 0,
+  "isExcluded" : false,
+  "excludedInStages" : [ ]
 }, {
   "id" : "3",
   "hostPort" : "172.22.0.167:51485",
@@ -61,8 +64,33 @@
     "totalOffHeapStorageMemory" : 524288000
   },
   "blacklistedInStages" : [ ],
+  "peakMemoryMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  },
   "attributes" : { },
-  "resources" : { }
+  "resources" : { },
+  "resourceProfileId" : 0,
+  "isExcluded" : true,
+  "excludedInStages" : [ ]
 } ,{
   "id" : "2",
   "hostPort" : "172.22.0.167:51487",
@@ -95,8 +123,33 @@
     "totalOffHeapStorageMemory" : 524288000
   },
   "blacklistedInStages" : [ ],
+  "peakMemoryMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  },
   "attributes" : { },
-  "resources" : { }
+  "resources" : { },
+  "resourceProfileId" : 0,
+  "isExcluded" : true,
+  "excludedInStages" : [ ]
 }, {
   "id" : "1",
   "hostPort" : "172.22.0.167:51490",
@@ -129,8 +182,33 @@
     "totalOffHeapStorageMemory": 524288000
   },
   "blacklistedInStages" : [ ],
+  "peakMemoryMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  },
   "attributes" : { },
-  "resources" : { }
+  "resources" : { },
+  "resourceProfileId" : 0,
+  "isExcluded" : true,
+  "excludedInStages" : [ ]
 }, {
   "id" : "0",
   "hostPort" : "172.22.0.167:51491",
@@ -163,6 +241,31 @@
     "totalOffHeapStorageMemory" : 524288000
   },
   "blacklistedInStages" : [ ],
+  "peakMemoryMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  },
   "attributes" : { },
-  "resources" : { }
+  "resources" : { },
+  "resourceProfileId" : 0,
+  "isExcluded" : true,
+  "excludedInStages" : [ ]
 } ]
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
deleted file mode 100644
index 0986e85f16b3e..0000000000000
--- a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
+++ /dev/null
@@ -1,168 +0,0 @@
-[ {
-  "id" : "driver",
-  "hostPort" : "172.22.0.167:51475",
-  "isActive" : true,
-  "rddBlocks" : 0,
-  "memoryUsed" : 0,
-  "diskUsed" : 0,
-  "totalCores" : 0,
-  "maxTasks" : 0,
-  "activeTasks" : 0,
-  "failedTasks" : 0,
-  "completedTasks" : 0,
-  "totalTasks" : 0,
-  "totalDuration" : 0,
-  "totalGCTime" : 0,
-  "totalInputBytes" : 0,
-  "totalShuffleRead" : 0,
-  "totalShuffleWrite" : 0,
-  "isBlacklisted" : true,
-  "maxMemory" : 908381388,
-  "addTime" : "2016-11-16T22:33:31.477GMT",
-  "executorLogs" : { },
-  "memoryMetrics": {
-    "usedOnHeapStorageMemory" : 0,
-    "usedOffHeapStorageMemory" : 0,
-    "totalOnHeapStorageMemory" : 384093388,
-    "totalOffHeapStorageMemory" : 524288000
-  },
-  "blacklistedInStages" : [ ],
-  "attributes" : { },
-  "resources" : { }
-}, {
-  "id" : "3",
-  "hostPort" : "172.22.0.167:51485",
-  "isActive" : true,
-  "rddBlocks" : 0,
-  "memoryUsed" : 0,
-  "diskUsed" : 0,
-  "totalCores" : 4,
-  "maxTasks" : 4,
-  "activeTasks" : 0,
-  "failedTasks" : 0,
-  "completedTasks" : 12,
-  "totalTasks" : 12,
-  "totalDuration" : 2453,
-  "totalGCTime" : 72,
-  "totalInputBytes" : 0,
-  "totalShuffleRead" : 0,
-  "totalShuffleWrite" : 0,
-  "isBlacklisted" : true,
-  "maxMemory" : 908381388,
-  "addTime" : "2016-11-16T22:33:35.320GMT",
-  "executorLogs" : {
-    "stdout" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stdout",
-    "stderr" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stderr"
-  },
-  "memoryMetrics": {
-    "usedOnHeapStorageMemory" : 0,
-    "usedOffHeapStorageMemory" : 0,
-    "totalOnHeapStorageMemory" : 384093388,
-    "totalOffHeapStorageMemory" : 524288000
-  },
-  "blacklistedInStages" : [ ],
-  "attributes" : { },
-  "resources" : { }
-}, {
-  "id" : "2",
-  "hostPort" : "172.22.0.167:51487",
-  "isActive" : true,
-  "rddBlocks" : 0,
-  "memoryUsed" : 0,
-  "diskUsed" : 0,
-  "totalCores" : 4,
-  "maxTasks" : 4,
-  "activeTasks" : 0,
-  "failedTasks" : 4,
-  "completedTasks" : 0,
-  "totalTasks" : 4,
-  "totalDuration" : 2537,
-  "totalGCTime" : 88,
-  "totalInputBytes" : 0,
-  "totalShuffleRead" : 0,
-  "totalShuffleWrite" : 0,
-  "isBlacklisted" : true,
-  "maxMemory" : 908381388,
-  "addTime" : "2016-11-16T22:33:35.393GMT",
-  "executorLogs" : {
-    "stdout" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stdout",
-    "stderr" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stderr"
-  },
-  "memoryMetrics": {
-    "usedOnHeapStorageMemory" : 0,
-    "usedOffHeapStorageMemory" : 0,
-    "totalOnHeapStorageMemory" : 384093388,
-    "totalOffHeapStorageMemory" : 524288000
-  },
-  "blacklistedInStages" : [ ],
-  "attributes" : { },
-  "resources" : { }
-}, {
-  "id" : "1",
-  "hostPort" : "172.22.0.167:51490",
-  "isActive" : true,
-  "rddBlocks" : 0,
-  "memoryUsed" : 0,
-  "diskUsed" : 0,
-  "totalCores" : 4,
-  "maxTasks" : 4,
-  "activeTasks" : 0,
-  "failedTasks" : 0,
-  "completedTasks" : 4,
-  "totalTasks" : 4,
-  "totalDuration" : 3152,
-  "totalGCTime" : 68,
-  "totalInputBytes" : 0,
-  "totalShuffleRead" : 0,
-  "totalShuffleWrite" : 0,
-  "isBlacklisted" : true,
-  "maxMemory" : 908381388,
-  "addTime" : "2016-11-16T22:33:35.443GMT",
-  "executorLogs" : {
-    "stdout" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stdout",
-    "stderr" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stderr"
-  },
-  "memoryMetrics": {
-    "usedOnHeapStorageMemory": 0,
-    "usedOffHeapStorageMemory": 0,
-    "totalOnHeapStorageMemory": 384093388,
-    "totalOffHeapStorageMemory": 524288000
-  },
-  "blacklistedInStages" : [ ],
-  "attributes" : { },
-  "resources" : { }
-}, {
-  "id" : "0",
-  "hostPort" : "172.22.0.167:51491",
-  "isActive" : true,
-  "rddBlocks" : 0,
-  "memoryUsed" : 0,
-  "diskUsed" : 0,
-  "totalCores" : 4,
-  "maxTasks" : 4,
-  "activeTasks" : 0,
-  "failedTasks" : 4,
-  "completedTasks" : 0,
-  "totalTasks" : 4,
-  "totalDuration" : 2551,
-  "totalGCTime" : 116,
-  "totalInputBytes" : 0,
-  "totalShuffleRead" : 0,
-  "totalShuffleWrite" : 0,
-  "isBlacklisted" : true,
-  "maxMemory" : 908381388,
-  "addTime" : "2016-11-16T22:33:35.462GMT",
-  "executorLogs" : {
-    "stdout" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stdout",
-    "stderr" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stderr"
-  },
-  "memoryMetrics": {
-    "usedOnHeapStorageMemory": 0,
-    "usedOffHeapStorageMemory": 0,
-    "totalOnHeapStorageMemory": 384093388,
-    "totalOffHeapStorageMemory": 524288000
-  },
-  "blacklistedInStages" : [ ],
-  "attributes" : { },
-  "resources" : { }
-} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_unblacklisting_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_unblacklisting_expectation.json
deleted file mode 100644
index 26d665151a52d..0000000000000
--- a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_unblacklisting_expectation.json
+++ /dev/null
@@ -1,138 +0,0 @@
-[ {
-  "id" : "driver",
-  "hostPort" : "172.22.0.111:64527",
-  "isActive" : true,
-  "rddBlocks" : 0,
-  "memoryUsed" : 0,
-  "diskUsed" : 0,
-  "totalCores" : 0,
-  "maxTasks" : 0,
-  "activeTasks" : 0,
-  "failedTasks" : 0,
-  "completedTasks" : 0,
-  "totalTasks" : 0,
-  "totalDuration" : 0,
-  "totalGCTime" : 0,
-  "totalInputBytes" : 0,
-  "totalShuffleRead" : 0,
-  "totalShuffleWrite" : 0,
-  "isBlacklisted" : false,
-  "maxMemory" : 384093388,
-  "addTime" : "2016-11-15T23:20:38.836GMT",
-  "executorLogs" : { },
-  "blacklistedInStages" : [ ],
-  "attributes" : { },
-  "resources" : { }
-}, {
-  "id" : "3",
-  "hostPort" : "172.22.0.111:64543",
-  "isActive" : true,
-  "rddBlocks" : 0,
-  "memoryUsed" : 0,
-  "diskUsed" : 0,
-  "totalCores" : 4,
-  "maxTasks" : 4,
-  "activeTasks" : 0,
-  "failedTasks" : 0,
-  "completedTasks" : 4,
-  "totalTasks" : 4,
-  "totalDuration" : 3457,
-  "totalGCTime" : 72,
-  "totalInputBytes" : 0,
-  "totalShuffleRead" : 0,
-  "totalShuffleWrite" : 0,
-  "isBlacklisted" : false,
-  "maxMemory" : 384093388,
-  "addTime" : "2016-11-15T23:20:42.711GMT",
-  "executorLogs" : {
-    "stdout" : "http://172.22.0.111:64521/logPage/?appId=app-20161115172038-0000&executorId=3&logType=stdout",
-    "stderr" : "http://172.22.0.111:64521/logPage/?appId=app-20161115172038-0000&executorId=3&logType=stderr"
-  },
-  "blacklistedInStages" : [ ],
-  "attributes" : { },
-  "resources" : { }
-}, {
-  "id" : "2",
-  "hostPort" : "172.22.0.111:64539",
-  "isActive" : true,
-  "rddBlocks" : 0,
-  "memoryUsed" : 0,
-  "diskUsed" : 0,
-  "totalCores" : 4,
-  "maxTasks" : 4,
-  "activeTasks" : 0,
-  "failedTasks" : 6,
-  "completedTasks" : 0,
-  "totalTasks" : 6,
-  "totalDuration" : 2792,
-  "totalGCTime" : 128,
-  "totalInputBytes" : 0,
-  "totalShuffleRead" : 0,
-  "totalShuffleWrite" : 0,
-  "isBlacklisted" : false,
-  "maxMemory" : 384093388,
-  "addTime" : "2016-11-15T23:20:42.589GMT",
-  "executorLogs" : {
-    "stdout" : "http://172.22.0.111:64519/logPage/?appId=app-20161115172038-0000&executorId=2&logType=stdout",
-    "stderr" : "http://172.22.0.111:64519/logPage/?appId=app-20161115172038-0000&executorId=2&logType=stderr"
-  },
-  "blacklistedInStages" : [ ],
-  "attributes" : { },
-  "resources" : { }
-}, {
-  "id" : "1",
-  "hostPort" : "172.22.0.111:64541",
-  "isActive" : true,
-  "rddBlocks" : 0,
-  "memoryUsed" : 0,
-  "diskUsed" : 0,
-  "totalCores" : 4,
-  "maxTasks" : 4,
-  "activeTasks" : 0,
-  "failedTasks" : 0,
-  "completedTasks" : 12,
-  "totalTasks" : 12,
-  "totalDuration" : 2613,
-  "totalGCTime" : 84,
-  "totalInputBytes" : 0,
-  "totalShuffleRead" : 0,
-  "totalShuffleWrite" : 0,
-  "isBlacklisted" : false,
-  "maxMemory" : 384093388,
-  "addTime" : "2016-11-15T23:20:42.629GMT",
-  "executorLogs" : {
-    "stdout" : "http://172.22.0.111:64518/logPage/?appId=app-20161115172038-0000&executorId=1&logType=stdout",
-    "stderr" : "http://172.22.0.111:64518/logPage/?appId=app-20161115172038-0000&executorId=1&logType=stderr"
-  },
-  "blacklistedInStages" : [ ],
-  "attributes" : { },
-  "resources" : { }
-}, {
-  "id" : "0",
-  "hostPort" : "172.22.0.111:64540",
-  "isActive" : true,
-  "rddBlocks" : 0,
-  "memoryUsed" : 0,
-  "diskUsed" : 0,
-  "totalCores" : 4,
-  "maxTasks" : 4,
-  "activeTasks" : 0,
-  "failedTasks" : 4,
-  "completedTasks" : 0,
-  "totalTasks" : 4,
-  "totalDuration" : 2741,
-  "totalGCTime" : 120,
-  "totalInputBytes" : 0,
-  "totalShuffleRead" : 0,
-  "totalShuffleWrite" : 0,
-  "isBlacklisted" : false,
-  "maxMemory" : 384093388,
-  "addTime" : "2016-11-15T23:20:42.593GMT",
-  "executorLogs" : {
-    "stdout" : "http://172.22.0.111:64517/logPage/?appId=app-20161115172038-0000&executorId=0&logType=stdout",
-    "stderr" : "http://172.22.0.111:64517/logPage/?appId=app-20161115172038-0000&executorId=0&logType=stderr"
-  },
-  "blacklistedInStages" : [ ],
-  "attributes" : { },
-  "resources" : { }
-} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json
new file mode 100644
index 0000000000000..8869fb4e296e6
--- /dev/null
+++ b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json
@@ -0,0 +1,271 @@
+[ {
+  "id" : "driver",
+  "hostPort" : "172.22.0.167:51475",
+  "isActive" : true,
+  "rddBlocks" : 0,
+  "memoryUsed" : 0,
+  "diskUsed" : 0,
+  "totalCores" : 0,
+  "maxTasks" : 0,
+  "activeTasks" : 0,
+  "failedTasks" : 0,
+  "completedTasks" : 0,
+  "totalTasks" : 0,
+  "totalDuration" : 0,
+  "totalGCTime" : 0,
+  "totalInputBytes" : 0,
+  "totalShuffleRead" : 0,
+  "totalShuffleWrite" : 0,
+  "isBlacklisted" : false,
+  "maxMemory" : 908381388,
+  "addTime" : "2016-11-16T22:33:31.477GMT",
+  "executorLogs" : { },
+  "memoryMetrics": {
+    "usedOnHeapStorageMemory" : 0,
+    "usedOffHeapStorageMemory" : 0,
+    "totalOnHeapStorageMemory" : 384093388,
+    "totalOffHeapStorageMemory" : 524288000
+  },
+  "blacklistedInStages" : [ ],
+  "attributes" : { },
+  "resources" : { },
+  "resourceProfileId" : 0,
+  "isExcluded" : false,
+  "excludedInStages" : [ ]
+}, {
+  "id" : "3",
+  "hostPort" : "172.22.0.167:51485",
+  "isActive" : true,
+  "rddBlocks" : 0,
+  "memoryUsed" : 0,
+  "diskUsed" : 0,
+  "totalCores" : 4,
+  "maxTasks" : 4,
+  "activeTasks" : 0,
+  "failedTasks" : 0,
+  "completedTasks" : 12,
+  "totalTasks" : 12,
+  "totalDuration" : 2453,
+  "totalGCTime" : 72,
+  "totalInputBytes" : 0,
+  "totalShuffleRead" : 0,
+  "totalShuffleWrite" : 0,
+  "isBlacklisted" : true,
+  "maxMemory" : 908381388,
+  "addTime" : "2016-11-16T22:33:35.320GMT",
+  "executorLogs" : {
+    "stdout" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stdout",
+    "stderr" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stderr"
+  },
+  "memoryMetrics": {
+    "usedOnHeapStorageMemory" : 0,
+    "usedOffHeapStorageMemory" : 0,
+    "totalOnHeapStorageMemory" : 384093388,
+    "totalOffHeapStorageMemory" : 524288000
+  },
+  "blacklistedInStages" : [ ],
+  "peakMemoryMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  },
+  "attributes" : { },
+  "resources" : { },
+  "resourceProfileId" : 0,
+  "isExcluded" : true,
+  "excludedInStages" : [ ]
+}, {
+  "id" : "2",
+  "hostPort" : "172.22.0.167:51487",
+  "isActive" : true,
+  "rddBlocks" : 0,
+  "memoryUsed" : 0,
+  "diskUsed" : 0,
+  "totalCores" : 4,
+  "maxTasks" : 4,
+  "activeTasks" : 0,
+  "failedTasks" : 4,
+  "completedTasks" : 0,
+  "totalTasks" : 4,
+  "totalDuration" : 2537,
+  "totalGCTime" : 88,
+  "totalInputBytes" : 0,
+  "totalShuffleRead" : 0,
+  "totalShuffleWrite" : 0,
+  "isBlacklisted" : true,
+  "maxMemory" : 908381388,
+  "addTime" : "2016-11-16T22:33:35.393GMT",
+  "executorLogs" : {
+    "stdout" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stdout",
+    "stderr" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stderr"
+  },
+  "memoryMetrics": {
+    "usedOnHeapStorageMemory" : 0,
+    "usedOffHeapStorageMemory" : 0,
+    "totalOnHeapStorageMemory" : 384093388,
+    "totalOffHeapStorageMemory" : 524288000
+  },
+  "blacklistedInStages" : [ ],
+  "peakMemoryMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  },
+  "attributes" : { },
+  "resources" : { },
+  "resourceProfileId" : 0,
+  "isExcluded" : true,
+  "excludedInStages" : [ ]
+}, {
+  "id" : "1",
+  "hostPort" : "172.22.0.167:51490",
+  "isActive" : true,
+  "rddBlocks" : 0,
+  "memoryUsed" : 0,
+  "diskUsed" : 0,
+  "totalCores" : 4,
+  "maxTasks" : 4,
+  "activeTasks" : 0,
+  "failedTasks" : 0,
+  "completedTasks" : 4,
+  "totalTasks" : 4,
+  "totalDuration" : 3152,
+  "totalGCTime" : 68,
+  "totalInputBytes" : 0,
+  "totalShuffleRead" : 0,
+  "totalShuffleWrite" : 0,
+  "isBlacklisted" : true,
+  "maxMemory" : 908381388,
+  "addTime" : "2016-11-16T22:33:35.443GMT",
+  "executorLogs" : {
+    "stdout" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stdout",
+    "stderr" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stderr"
+  },
+  "memoryMetrics": {
+    "usedOnHeapStorageMemory": 0,
+    "usedOffHeapStorageMemory": 0,
+    "totalOnHeapStorageMemory": 384093388,
+    "totalOffHeapStorageMemory": 524288000
+  },
+  "blacklistedInStages" : [ ],
+  "peakMemoryMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  },
+  "attributes" : { },
+  "resources" : { },
+  "resourceProfileId" : 0,
+  "isExcluded" : true,
+  "excludedInStages" : [ ]
+}, {
+  "id" : "0",
+  "hostPort" : "172.22.0.167:51491",
+  "isActive" : true,
+  "rddBlocks" : 0,
+  "memoryUsed" : 0,
+  "diskUsed" : 0,
+  "totalCores" : 4,
+  "maxTasks" : 4,
+  "activeTasks" : 0,
+  "failedTasks" : 4,
+  "completedTasks" : 0,
+  "totalTasks" : 4,
+  "totalDuration" : 2551,
+  "totalGCTime" : 116,
+  "totalInputBytes" : 0,
+  "totalShuffleRead" : 0,
+  "totalShuffleWrite" : 0,
+  "isBlacklisted" : true,
+  "maxMemory" : 908381388,
+  "addTime" : "2016-11-16T22:33:35.462GMT",
+  "executorLogs" : {
+    "stdout" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stdout",
+    "stderr" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stderr"
+  },
+  "memoryMetrics": {
+    "usedOnHeapStorageMemory": 0,
+    "usedOffHeapStorageMemory": 0,
+    "totalOnHeapStorageMemory": 384093388,
+    "totalOffHeapStorageMemory": 524288000
+  },
+  "blacklistedInStages" : [ ],
+  "peakMemoryMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  },
+  "attributes" : { },
+  "resources" : { },
+  "resourceProfileId" : 0,
+  "isExcluded" : true,
+  "excludedInStages" : [ ]
+} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json
new file mode 100644
index 0000000000000..21cc9d0812990
--- /dev/null
+++ b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json
@@ -0,0 +1,241 @@
+[ {
+  "id" : "driver",
+  "hostPort" : "172.22.0.111:64527",
+  "isActive" : true,
+  "rddBlocks" : 0,
+  "memoryUsed" : 0,
+  "diskUsed" : 0,
+  "totalCores" : 0,
+  "maxTasks" : 0,
+  "activeTasks" : 0,
+  "failedTasks" : 0,
+  "completedTasks" : 0,
+  "totalTasks" : 0,
+  "totalDuration" : 0,
+  "totalGCTime" : 0,
+  "totalInputBytes" : 0,
+  "totalShuffleRead" : 0,
+  "totalShuffleWrite" : 0,
+  "isBlacklisted" : false,
+  "maxMemory" : 384093388,
+  "addTime" : "2016-11-15T23:20:38.836GMT",
+  "executorLogs" : { },
+  "blacklistedInStages" : [ ],
+  "attributes" : { },
+  "resources" : { },
+  "resourceProfileId" : 0,
+  "isExcluded" : false,
+  "excludedInStages" : [ ]
+}, {
+  "id" : "3",
+  "hostPort" : "172.22.0.111:64543",
+  "isActive" : true,
+  "rddBlocks" : 0,
+  "memoryUsed" : 0,
+  "diskUsed" : 0,
+  "totalCores" : 4,
+  "maxTasks" : 4,
+  "activeTasks" : 0,
+  "failedTasks" : 0,
+  "completedTasks" : 4,
+  "totalTasks" : 4,
+  "totalDuration" : 3457,
+  "totalGCTime" : 72,
+  "totalInputBytes" : 0,
+  "totalShuffleRead" : 0,
+  "totalShuffleWrite" : 0,
+  "isBlacklisted" : false,
+  "maxMemory" : 384093388,
+  "addTime" : "2016-11-15T23:20:42.711GMT",
+  "executorLogs" : {
+    "stdout" : "http://172.22.0.111:64521/logPage/?appId=app-20161115172038-0000&executorId=3&logType=stdout",
+    "stderr" : "http://172.22.0.111:64521/logPage/?appId=app-20161115172038-0000&executorId=3&logType=stderr"
+  },
+  "blacklistedInStages" : [ ],
+  "peakMemoryMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  },
+  "attributes" : { },
+  "resources" : { },
+  "resourceProfileId" : 0,
+  "isExcluded" : false,
+  "excludedInStages" : [ ]
+}, {
+  "id" : "2",
+  "hostPort" : "172.22.0.111:64539",
+  "isActive" : true,
+  "rddBlocks" : 0,
+  "memoryUsed" : 0,
+  "diskUsed" : 0,
+  "totalCores" : 4,
+  "maxTasks" : 4,
+  "activeTasks" : 0,
+  "failedTasks" : 6,
+  "completedTasks" : 0,
+  "totalTasks" : 6,
+  "totalDuration" : 2792,
+  "totalGCTime" : 128,
+  "totalInputBytes" : 0,
+  "totalShuffleRead" : 0,
+  "totalShuffleWrite" : 0,
+  "isBlacklisted" : false,
+  "maxMemory" : 384093388,
+  "addTime" : "2016-11-15T23:20:42.589GMT",
+  "executorLogs" : {
+    "stdout" : "http://172.22.0.111:64519/logPage/?appId=app-20161115172038-0000&executorId=2&logType=stdout",
+    "stderr" : "http://172.22.0.111:64519/logPage/?appId=app-20161115172038-0000&executorId=2&logType=stderr"
+  },
+  "blacklistedInStages" : [ ],
+  "peakMemoryMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  },
+  "attributes" : { },
+  "resources" : { },
+  "resourceProfileId" : 0,
+  "isExcluded" : false,
+  "excludedInStages" : [ ]
+}, {
+  "id" : "1",
+  "hostPort" : "172.22.0.111:64541",
+  "isActive" : true,
+  "rddBlocks" : 0,
+  "memoryUsed" : 0,
+  "diskUsed" : 0,
+  "totalCores" : 4,
+  "maxTasks" : 4,
+  "activeTasks" : 0,
+  "failedTasks" : 0,
+  "completedTasks" : 12,
+  "totalTasks" : 12,
+  "totalDuration" : 2613,
+  "totalGCTime" : 84,
+  "totalInputBytes" : 0,
+  "totalShuffleRead" : 0,
+  "totalShuffleWrite" : 0,
+  "isBlacklisted" : false,
+  "maxMemory" : 384093388,
+  "addTime" : "2016-11-15T23:20:42.629GMT",
+  "executorLogs" : {
+    "stdout" : "http://172.22.0.111:64518/logPage/?appId=app-20161115172038-0000&executorId=1&logType=stdout",
+    "stderr" : "http://172.22.0.111:64518/logPage/?appId=app-20161115172038-0000&executorId=1&logType=stderr"
+  },
+  "blacklistedInStages" : [ ],
+  "peakMemoryMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  },
+  "attributes" : { },
+  "resources" : { },
+  "resourceProfileId" : 0,
+  "isExcluded" : false,
+  "excludedInStages" : [ ]
+}, {
+  "id" : "0",
+  "hostPort" : "172.22.0.111:64540",
+  "isActive" : true,
+  "rddBlocks" : 0,
+  "memoryUsed" : 0,
+  "diskUsed" : 0,
+  "totalCores" : 4,
+  "maxTasks" : 4,
+  "activeTasks" : 0,
+  "failedTasks" : 4,
+  "completedTasks" : 0,
+  "totalTasks" : 4,
+  "totalDuration" : 2741,
+  "totalGCTime" : 120,
+  "totalInputBytes" : 0,
+  "totalShuffleRead" : 0,
+  "totalShuffleWrite" : 0,
+  "isBlacklisted" : false,
+  "maxMemory" : 384093388,
+  "addTime" : "2016-11-15T23:20:42.593GMT",
+  "executorLogs" : {
+    "stdout" : "http://172.22.0.111:64517/logPage/?appId=app-20161115172038-0000&executorId=0&logType=stdout",
+    "stderr" : "http://172.22.0.111:64517/logPage/?appId=app-20161115172038-0000&executorId=0&logType=stderr"
+  },
+  "blacklistedInStages" : [ ],
+  "peakMemoryMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  },
+  "attributes" : { },
+  "resources" : { },
+  "resourceProfileId" : 0,
+  "isExcluded" : false,
+  "excludedInStages" : [ ]
+} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_resource_information_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_resource_information_expectation.json
index e69ab3b49d455..53ae9a0c7909e 100644
--- a/core/src/test/resources/HistoryServerExpectations/executor_resource_information_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/executor_resource_information_expectation.json
@@ -28,7 +28,10 @@
   },
   "blacklistedInStages" : [ ],
   "attributes" : { },
-  "resources" : { }
+  "resources" : { },
+  "resourceProfileId" : 0,
+  "isExcluded" : false,
+  "excludedInStages" : [ ]
 }, {
   "id" : "2",
   "hostPort" : "tomg-test:46005",
@@ -77,7 +80,10 @@
       "name" : "gpu",
       "addresses" : [ "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12" ]
     }
-  }
+  },
+  "resourceProfileId" : 0,
+  "isExcluded" : false,
+  "excludedInStages" : [ ]
 }, {
   "id" : "1",
   "hostPort" : "tomg-test:44873",
@@ -126,5 +132,8 @@
       "name" : "gpu",
       "addresses" : [ "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12" ]
     }
-  }
+  },
+  "resourceProfileId" : 0,
+  "isExcluded" : false,
+  "excludedInStages" : [ ]
 } ]
diff --git a/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json
index da26271e66bc4..5573cf98db26a 100644
--- a/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json
@@ -42,5 +42,28 @@
   "schedulingPool" : "default",
   "rddIds" : [ 3, 2 ],
   "accumulatorUpdates" : [ ],
-  "killedTasksSummary" : { }
+  "killedTasksSummary" : { },
+  "resourceProfileId" : 0,
+  "peakExecutorMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  }
 } ]
diff --git a/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json
index 3102909f81116..8e6be68b4670c 100644
--- a/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json
@@ -1,46 +1,46 @@
 [ {
-  "id" : "application_1555004656427_0144",
+  "id" : "app-20200706201101-0003",
   "name" : "Spark shell",
   "attempts" : [ {
-    "startTime" : "2019-07-02T21:02:17.180GMT",
-    "endTime" : "2019-07-02T21:02:35.974GMT",
+    "startTime" : "2020-07-07T03:11:00.235GMT",
+    "endTime" : "2020-07-07T03:17:04.231GMT",
     "lastUpdated" : "",
-    "duration" : 18794,
-    "sparkUser" : "tgraves",
+    "duration" : 363996,
+    "sparkUser" : "terryk",
     "completed" : true,
-    "appSparkVersion" : "3.0.0-SNAPSHOT",
-    "startTimeEpoch" : 1562101337180,
+    "appSparkVersion" : "3.1.0-SNAPSHOT",
     "lastUpdatedEpoch" : 0,
-    "endTimeEpoch" : 1562101355974
+    "endTimeEpoch" : 1594091824231,
+    "startTimeEpoch" : 1594091460235
   } ]
 }, {
-  "id" : "application_1553914137147_0018",
-  "name" : "LargeBlocks",
+  "id" : "application_1578436911597_0052",
+  "name" : "Spark shell",
   "attempts" : [ {
-    "startTime" : "2019-04-08T20:39:44.286GMT",
-    "endTime" : "2019-04-08T20:40:46.454GMT",
+    "startTime" : "2020-01-11T17:44:22.851GMT",
+    "endTime" : "2020-01-11T17:46:42.615GMT",
     "lastUpdated" : "",
-    "duration" : 62168,
-    "sparkUser" : "systest",
+    "duration" : 139764,
+    "sparkUser" : "tgraves",
     "completed" : true,
     "appSparkVersion" : "3.0.0-SNAPSHOT",
-    "startTimeEpoch" : 1554755984286,
-    "endTimeEpoch" : 1554756046454,
+    "endTimeEpoch" : 1578764802615,
+    "startTimeEpoch" : 1578764662851,
     "lastUpdatedEpoch" : 0
   } ]
 }, {
-  "id" : "application_1516285256255_0012",
+  "id" : "application_1555004656427_0144",
   "name" : "Spark shell",
   "attempts" : [ {
-    "startTime" : "2018-01-18T18:30:35.119GMT",
-    "endTime" : "2018-01-18T18:38:27.938GMT",
+    "startTime" : "2019-07-02T21:02:17.180GMT",
+    "endTime" : "2019-07-02T21:02:35.974GMT",
     "lastUpdated" : "",
-    "duration" : 472819,
-    "sparkUser" : "attilapiros",
+    "duration" : 18794,
+    "sparkUser" : "tgraves",
     "completed" : true,
-    "appSparkVersion" : "2.3.0-SNAPSHOT",
+    "appSparkVersion" : "3.0.0-SNAPSHOT",
+    "startTimeEpoch" : 1562101337180,
     "lastUpdatedEpoch" : 0,
-    "startTimeEpoch" : 1516300235119,
-    "endTimeEpoch" : 1516300707938
+    "endTimeEpoch" : 1562101355974
   } ]
 } ]
diff --git a/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json
index 794f1514a6708..35d71f9d4409b 100644
--- a/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json
@@ -1,21 +1,48 @@
-[
-  {
-    "id": "application_1555004656427_0144",
-    "name": "Spark shell",
-    "attempts": [
-      {
-        "startTime": "2019-07-02T21:02:17.180GMT",
-        "endTime": "2019-07-02T21:02:35.974GMT",
-        "lastUpdated": "",
-        "duration": 18794,
-        "sparkUser": "tgraves",
-        "completed": true,
-        "appSparkVersion": "3.0.0-SNAPSHOT",
-        "startTimeEpoch": 1562101337180,
-        "lastUpdatedEpoch": 0,
-        "endTimeEpoch": 1562101355974
-      }
-    ]
+[ {
+  "id" : "app-20200706201101-0003",
+  "name" : "Spark shell",
+  "attempts" : [ {
+    "startTime" : "2020-07-07T03:11:00.235GMT",
+    "endTime" : "2020-07-07T03:17:04.231GMT",
+    "lastUpdated" : "",
+    "duration" : 363996,
+    "sparkUser" : "terryk",
+    "completed" : true,
+    "appSparkVersion" : "3.1.0-SNAPSHOT",
+    "endTimeEpoch" : 1594091824231,
+    "startTimeEpoch" : 1594091460235,
+    "lastUpdatedEpoch" : 0
+  } ]
+}, {
+  "id" : "application_1578436911597_0052",
+  "name" : "Spark shell",
+  "attempts" : [ {
+    "startTime" : "2020-01-11T17:44:22.851GMT",
+    "endTime" : "2020-01-11T17:46:42.615GMT",
+    "lastUpdated" : "",
+    "duration" : 139764,
+    "sparkUser" : "tgraves",
+    "completed" : true,
+    "appSparkVersion" : "3.0.0-SNAPSHOT",
+    "endTimeEpoch" : 1578764802615,
+    "startTimeEpoch" : 1578764662851,
+    "lastUpdatedEpoch" : 0
+  } ]
+}, {
+  "id" : "application_1555004656427_0144",
+  "name" : "Spark shell",
+  "attempts" : [ {
+    "startTime" : "2019-07-02T21:02:17.180GMT",
+    "endTime" : "2019-07-02T21:02:35.974GMT",
+    "lastUpdated" : "",
+    "duration" : 18794,
+    "sparkUser" : "tgraves",
+    "completed" : true,
+    "appSparkVersion" : "3.0.0-SNAPSHOT",
+    "endTimeEpoch" : 1562101355974,
+    "startTimeEpoch" : 1562101337180,
+    "lastUpdatedEpoch" : 0
+  } ]
 }, {
   "id" : "application_1553914137147_0018",
   "name" : "LargeBlocks",
diff --git a/core/src/test/resources/HistoryServerExpectations/minEndDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/minEndDate_app_list_json_expectation.json
index adcdccef48450..c6530b14bf271 100644
--- a/core/src/test/resources/HistoryServerExpectations/minEndDate_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/minEndDate_app_list_json_expectation.json
@@ -1,4 +1,34 @@
 [ {
+  "id" : "app-20200706201101-0003",
+  "name" : "Spark shell",
+  "attempts" : [ {
+    "startTime" : "2020-07-07T03:11:00.235GMT",
+    "endTime" : "2020-07-07T03:17:04.231GMT",
+    "lastUpdated" : "",
+    "duration" : 363996,
+    "sparkUser" : "terryk",
+    "completed" : true,
+    "appSparkVersion" : "3.1.0-SNAPSHOT",
+    "endTimeEpoch" : 1594091824231,
+    "startTimeEpoch" : 1594091460235,
+    "lastUpdatedEpoch" : 0
+  } ]
+}, {
+  "id" : "application_1578436911597_0052",
+  "name" : "Spark shell",
+  "attempts" : [ {
+    "startTime" : "2020-01-11T17:44:22.851GMT",
+    "endTime" : "2020-01-11T17:46:42.615GMT",
+    "lastUpdated" : "",
+    "duration" : 139764,
+    "sparkUser" : "tgraves",
+    "completed" : true,
+    "appSparkVersion" : "3.0.0-SNAPSHOT",
+    "endTimeEpoch" : 1578764802615,
+    "startTimeEpoch" : 1578764662851,
+    "lastUpdatedEpoch" : 0
+  } ]
+}, {
   "id" : "application_1555004656427_0144",
   "name" : "Spark shell",
   "attempts" : [ {
diff --git a/core/src/test/resources/HistoryServerExpectations/miscellaneous_process_expectation.json b/core/src/test/resources/HistoryServerExpectations/miscellaneous_process_expectation.json
new file mode 100644
index 0000000000000..93d674a1f9210
--- /dev/null
+++ b/core/src/test/resources/HistoryServerExpectations/miscellaneous_process_expectation.json
@@ -0,0 +1,11 @@
+[ {
+  "id" : "yarn-am",
+  "hostPort" : "192.168.1.19:8042",
+  "isActive" : true,
+  "totalCores" : 1,
+  "addTime" : "2021-04-19T15:35:50.218GMT",
+  "processLogs" : {
+    "stdout" : "http://192.168.1.19:8042/node/containerlogs/container_1555004656427_0144_01_000001/test/stdout?start=-4096",
+    "stderr" : "http://192.168.1.19:8042/node/containerlogs/container_1555004656427_0144_01_000001/test/stderr?start=-4096"
+  }
+} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/multiple_resource_profiles_expectation.json b/core/src/test/resources/HistoryServerExpectations/multiple_resource_profiles_expectation.json
new file mode 100644
index 0000000000000..5c1e4cc2337be
--- /dev/null
+++ b/core/src/test/resources/HistoryServerExpectations/multiple_resource_profiles_expectation.json
@@ -0,0 +1,112 @@
+{
+  "runtime" : {
+    "javaVersion" : "1.8.0_232 (Private Build)",
+    "javaHome" : "/usr/lib/jvm/java-8-openjdk-amd64/jre",
+    "scalaVersion" : "version 2.12.10"
+  },
+  "sparkProperties" : [ ],
+  "hadoopProperties" : [ ],
+  "systemProperties" : [ ],
+  "classpathEntries" : [ ],
+  "resourceProfiles" : [ {
+    "id" : 0,
+    "executorResources" : {
+      "cores" : {
+        "resourceName" : "cores",
+        "amount" : 1,
+        "discoveryScript" : "",
+        "vendor" : ""
+      },
+      "memory" : {
+        "resourceName" : "memory",
+        "amount" : 1024,
+        "discoveryScript" : "",
+        "vendor" : ""
+      },
+      "gpu" : {
+        "resourceName" : "gpu",
+        "amount" : 1,
+        "discoveryScript" : "/home/tgraves/getGpus",
+        "vendor" : ""
+      }
+    },
+    "taskResources" : {
+      "cpus" : {
+        "resourceName" : "cpus",
+        "amount" : 1.0
+      },
+      "gpu" : {
+        "resourceName" : "gpu",
+        "amount" : 1.0
+      }
+    }
+  }, {
+    "id" : 1,
+    "executorResources" : {
+      "cores" : {
+        "resourceName" : "cores",
+        "amount" : 4,
+        "discoveryScript" : "",
+        "vendor" : ""
+      },
+      "gpu" : {
+        "resourceName" : "gpu",
+        "amount" : 1,
+        "discoveryScript" : "./getGpus",
+        "vendor" : ""
+      }
+    },
+    "taskResources" : {
+      "cpus" : {
+        "resourceName" : "cpus",
+        "amount" : 1.0
+      },
+      "gpu" : {
+        "resourceName" : "gpu",
+        "amount" : 1.0
+      }
+    }
+  }, {
+    "id" : 2,
+    "executorResources" : {
+      "cores" : {
+        "resourceName" : "cores",
+        "amount" : 2,
+        "discoveryScript" : "",
+        "vendor" : ""
+      }
+    },
+    "taskResources" : {
+      "cpus" : {
+        "resourceName" : "cpus",
+        "amount" : 2.0
+      }
+    }
+  }, {
+    "id" : 3,
+    "executorResources" : {
+      "cores" : {
+        "resourceName" : "cores",
+        "amount" : 4,
+        "discoveryScript" : "",
+        "vendor" : ""
+      },
+      "gpu" : {
+        "resourceName" : "gpu",
+        "amount" : 1,
+        "discoveryScript" : "./getGpus",
+        "vendor" : ""
+      }
+    },
+    "taskResources" : {
+      "cpus" : {
+        "resourceName" : "cpus",
+        "amount" : 2.0
+      },
+      "gpu" : {
+        "resourceName" : "gpu",
+        "amount" : 1.0
+      }
+    }
+  } ]
+}
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json
new file mode 100644
index 0000000000000..4579d3b9a84d4
--- /dev/null
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json
@@ -0,0 +1,111 @@
+{
+  "status" : "COMPLETE",
+  "stageId" : 1,
+  "attemptId" : 0,
+  "numTasks" : 8,
+  "numActiveTasks" : 0,
+  "numCompleteTasks" : 8,
+  "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
+  "numCompletedIndices" : 8,
+  "submissionTime" : "2015-02-03T16:43:05.829GMT",
+  "firstTaskLaunchedTime" : "2015-02-03T16:43:05.829GMT",
+  "completionTime" : "2015-02-03T16:43:06.286GMT",
+  "executorDeserializeTime" : 13,
+  "executorDeserializeCpuTime" : 0,
+  "executorRunTime" : 3476,
+  "executorCpuTime" : 0,
+  "resultSize" : 15216,
+  "jvmGcTime" : 152,
+  "resultSerializationTime" : 9,
+  "memoryBytesSpilled" : 0,
+  "diskBytesSpilled" : 0,
+  "peakExecutionMemory" : 0,
+  "inputBytes" : 28000128,
+  "inputRecords" : 0,
+  "outputBytes" : 0,
+  "outputRecords" : 0,
+  "shuffleRemoteBlocksFetched" : 0,
+  "shuffleLocalBlocksFetched" : 0,
+  "shuffleFetchWaitTime" : 0,
+  "shuffleRemoteBytesRead" : 0,
+  "shuffleRemoteBytesReadToDisk" : 0,
+  "shuffleLocalBytesRead" : 0,
+  "shuffleReadBytes" : 0,
+  "shuffleReadRecords" : 0,
+  "shuffleWriteBytes" : 13180,
+  "shuffleWriteTime" : 692000,
+  "shuffleWriteRecords" : 0,
+  "name" : "map at <console>:14",
+  "details" : "org.apache.spark.rdd.RDD.map(RDD.scala:271)\n$line10.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:14)\n$line10.$read$$iwC$$iwC$$iwC.<init>(<console>:19)\n$line10.$read$$iwC$$iwC.<init>(<console>:21)\n$line10.$read$$iwC.<init>(<console>:23)\n$line10.$read.<init>(<console>:25)\n$line10.$read$.<init>(<console>:29)\n$line10.$read$.<clinit>(<console>)\n$line10.$eval$.<init>(<console>:7)\n$line10.$eval$.<clinit>(<console>)\n$line10.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)",
+  "schedulingPool" : "default",
+  "rddIds" : [ 1, 0 ],
+  "accumulatorUpdates" : [ ],
+  "tasks" : { },
+  "executorSummary" : {
+    "driver" : {
+      "taskTime" : 3624,
+      "failedTasks" : 0,
+      "succeededTasks" : 8,
+      "killedTasks" : 0,
+      "inputBytes" : 28000128,
+      "inputRecords" : 0,
+      "outputBytes" : 0,
+      "outputRecords" : 0,
+      "shuffleRead" : 0,
+      "shuffleReadRecords" : 0,
+      "shuffleWrite" : 13180,
+      "shuffleWriteRecords" : 0,
+      "memoryBytesSpilled" : 0,
+      "diskBytesSpilled" : 0,
+      "isBlacklistedForStage" : false,
+      "peakMemoryMetrics" : {
+        "JVMHeapMemory" : 0,
+        "JVMOffHeapMemory" : 0,
+        "OnHeapExecutionMemory" : 0,
+        "OffHeapExecutionMemory" : 0,
+        "OnHeapStorageMemory" : 0,
+        "OffHeapStorageMemory" : 0,
+        "OnHeapUnifiedMemory" : 0,
+        "OffHeapUnifiedMemory" : 0,
+        "DirectPoolMemory" : 0,
+        "MappedPoolMemory" : 0,
+        "ProcessTreeJVMVMemory" : 0,
+        "ProcessTreeJVMRSSMemory" : 0,
+        "ProcessTreePythonVMemory" : 0,
+        "ProcessTreePythonRSSMemory" : 0,
+        "ProcessTreeOtherVMemory" : 0,
+        "ProcessTreeOtherRSSMemory" : 0,
+        "MinorGCCount" : 0,
+        "MinorGCTime" : 0,
+        "MajorGCCount" : 0,
+        "MajorGCTime" : 0
+      },
+      "isExcludedForStage" : false
+    }
+  },
+  "killedTasksSummary" : { },
+  "resourceProfileId" : 0,
+  "peakExecutorMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  }
+}
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
index 791907045e500..9edb518132e87 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
@@ -459,8 +459,54 @@
       "shuffleWriteRecords" : 0,
       "memoryBytesSpilled" : 0,
       "diskBytesSpilled" : 0,
-      "isBlacklistedForStage" : false
+      "isBlacklistedForStage" : false,
+      "peakMemoryMetrics" : {
+        "JVMHeapMemory" : 0,
+        "JVMOffHeapMemory" : 0,
+        "OnHeapExecutionMemory" : 0,
+        "OffHeapExecutionMemory" : 0,
+        "OnHeapStorageMemory" : 0,
+        "OffHeapStorageMemory" : 0,
+        "OnHeapUnifiedMemory" : 0,
+        "OffHeapUnifiedMemory" : 0,
+        "DirectPoolMemory" : 0,
+        "MappedPoolMemory" : 0,
+        "ProcessTreeJVMVMemory" : 0,
+        "ProcessTreeJVMRSSMemory" : 0,
+        "ProcessTreePythonVMemory" : 0,
+        "ProcessTreePythonRSSMemory" : 0,
+        "ProcessTreeOtherVMemory" : 0,
+        "ProcessTreeOtherRSSMemory" : 0,
+        "MinorGCCount" : 0,
+        "MinorGCTime" : 0,
+        "MajorGCCount" : 0,
+        "MajorGCTime" : 0
+      },
+      "isExcludedForStage" : false
     }
   },
-  "killedTasksSummary" : { }
+  "killedTasksSummary" : { },
+  "resourceProfileId" : 0,
+  "peakExecutorMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  }
 }
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
index 50d3f74ae775f..9e661bdf8a034 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
@@ -459,8 +459,54 @@
       "shuffleWriteRecords" : 0,
       "memoryBytesSpilled" : 0,
       "diskBytesSpilled" : 0,
-      "isBlacklistedForStage" : false
+      "isBlacklistedForStage" : false,
+      "peakMemoryMetrics" : {
+        "JVMHeapMemory" : 0,
+        "JVMOffHeapMemory" : 0,
+        "OnHeapExecutionMemory" : 0,
+        "OffHeapExecutionMemory" : 0,
+        "OnHeapStorageMemory" : 0,
+        "OffHeapStorageMemory" : 0,
+        "OnHeapUnifiedMemory" : 0,
+        "OffHeapUnifiedMemory" : 0,
+        "DirectPoolMemory" : 0,
+        "MappedPoolMemory" : 0,
+        "ProcessTreeJVMVMemory" : 0,
+        "ProcessTreeJVMRSSMemory" : 0,
+        "ProcessTreePythonVMemory" : 0,
+        "ProcessTreePythonRSSMemory" : 0,
+        "ProcessTreeOtherVMemory" : 0,
+        "ProcessTreeOtherRSSMemory" : 0,
+        "MinorGCCount" : 0,
+        "MinorGCTime" : 0,
+        "MajorGCCount" : 0,
+        "MajorGCTime" : 0
+      },
+      "isExcludedForStage" : false
     }
   },
-  "killedTasksSummary" : { }
+  "killedTasksSummary" : { },
+  "resourceProfileId" : 0,
+  "peakExecutorMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  }
 } ]
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json
new file mode 100644
index 0000000000000..9e661bdf8a034
--- /dev/null
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json
@@ -0,0 +1,512 @@
+[ {
+  "status" : "COMPLETE",
+  "stageId" : 1,
+  "attemptId" : 0,
+  "numTasks" : 8,
+  "numActiveTasks" : 0,
+  "numCompleteTasks" : 8,
+  "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
+  "numCompletedIndices" : 8,
+  "submissionTime" : "2015-02-03T16:43:05.829GMT",
+  "firstTaskLaunchedTime" : "2015-02-03T16:43:05.829GMT",
+  "completionTime" : "2015-02-03T16:43:06.286GMT",
+  "executorDeserializeTime" : 13,
+  "executorDeserializeCpuTime" : 0,
+  "executorRunTime" : 3476,
+  "executorCpuTime" : 0,
+  "resultSize" : 15216,
+  "jvmGcTime" : 152,
+  "resultSerializationTime" : 9,
+  "memoryBytesSpilled" : 0,
+  "diskBytesSpilled" : 0,
+  "peakExecutionMemory" : 0,
+  "inputBytes" : 28000128,
+  "inputRecords" : 0,
+  "outputBytes" : 0,
+  "outputRecords" : 0,
+  "shuffleRemoteBlocksFetched" : 0,
+  "shuffleLocalBlocksFetched" : 0,
+  "shuffleFetchWaitTime" : 0,
+  "shuffleRemoteBytesRead" : 0,
+  "shuffleRemoteBytesReadToDisk" : 0,
+  "shuffleLocalBytesRead" : 0,
+  "shuffleReadBytes" : 0,
+  "shuffleReadRecords" : 0,
+  "shuffleWriteBytes" : 13180,
+  "shuffleWriteTime" : 692000,
+  "shuffleWriteRecords" : 0,
+  "name" : "map at <console>:14",
+  "details" : "org.apache.spark.rdd.RDD.map(RDD.scala:271)\n$line10.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:14)\n$line10.$read$$iwC$$iwC$$iwC.<init>(<console>:19)\n$line10.$read$$iwC$$iwC.<init>(<console>:21)\n$line10.$read$$iwC.<init>(<console>:23)\n$line10.$read.<init>(<console>:25)\n$line10.$read$.<init>(<console>:29)\n$line10.$read$.<clinit>(<console>)\n$line10.$eval$.<init>(<console>:7)\n$line10.$eval$.<clinit>(<console>)\n$line10.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)",
+  "schedulingPool" : "default",
+  "rddIds" : [ 1, 0 ],
+  "accumulatorUpdates" : [ ],
+  "tasks" : {
+    "10" : {
+      "taskId" : 10,
+      "index" : 2,
+      "attempt" : 0,
+      "launchTime" : "2015-02-03T16:43:05.830GMT",
+      "duration" : 456,
+      "executorId" : "driver",
+      "host" : "localhost",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 2,
+        "executorDeserializeCpuTime" : 0,
+        "executorRunTime" : 434,
+        "executorCpuTime" : 0,
+        "resultSize" : 1902,
+        "jvmGcTime" : 19,
+        "resultSerializationTime" : 1,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 3500016,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 1648,
+          "writeTime" : 76000,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : { },
+      "schedulerDelay" : 19,
+      "gettingResultTime" : 0
+    },
+    "14" : {
+      "taskId" : 14,
+      "index" : 6,
+      "attempt" : 0,
+      "launchTime" : "2015-02-03T16:43:05.832GMT",
+      "duration" : 450,
+      "executorId" : "driver",
+      "host" : "localhost",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 2,
+        "executorDeserializeCpuTime" : 0,
+        "executorRunTime" : 434,
+        "executorCpuTime" : 0,
+        "resultSize" : 1902,
+        "jvmGcTime" : 19,
+        "resultSerializationTime" : 1,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 3500016,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 1648,
+          "writeTime" : 88000,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : { },
+      "schedulerDelay" : 13,
+      "gettingResultTime" : 0
+    },
+    "9" : {
+      "taskId" : 9,
+      "index" : 1,
+      "attempt" : 0,
+      "launchTime" : "2015-02-03T16:43:05.830GMT",
+      "duration" : 454,
+      "executorId" : "driver",
+      "host" : "localhost",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 1,
+        "executorDeserializeCpuTime" : 0,
+        "executorRunTime" : 436,
+        "executorCpuTime" : 0,
+        "resultSize" : 1902,
+        "jvmGcTime" : 19,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 3500016,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 1648,
+          "writeTime" : 98000,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : { },
+      "schedulerDelay" : 17,
+      "gettingResultTime" : 0
+    },
+    "13" : {
+      "taskId" : 13,
+      "index" : 5,
+      "attempt" : 0,
+      "launchTime" : "2015-02-03T16:43:05.831GMT",
+      "duration" : 452,
+      "executorId" : "driver",
+      "host" : "localhost",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 2,
+        "executorDeserializeCpuTime" : 0,
+        "executorRunTime" : 434,
+        "executorCpuTime" : 0,
+        "resultSize" : 1902,
+        "jvmGcTime" : 19,
+        "resultSerializationTime" : 2,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 3500016,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 1648,
+          "writeTime" : 73000,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : { },
+      "schedulerDelay" : 14,
+      "gettingResultTime" : 0
+    },
+    "12" : {
+      "taskId" : 12,
+      "index" : 4,
+      "attempt" : 0,
+      "launchTime" : "2015-02-03T16:43:05.831GMT",
+      "duration" : 454,
+      "executorId" : "driver",
+      "host" : "localhost",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 2,
+        "executorDeserializeCpuTime" : 0,
+        "executorRunTime" : 434,
+        "executorCpuTime" : 0,
+        "resultSize" : 1902,
+        "jvmGcTime" : 19,
+        "resultSerializationTime" : 1,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 3500016,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 1645,
+          "writeTime" : 101000,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : { },
+      "schedulerDelay" : 17,
+      "gettingResultTime" : 0
+    },
+    "11" : {
+      "taskId" : 11,
+      "index" : 3,
+      "attempt" : 0,
+      "launchTime" : "2015-02-03T16:43:05.830GMT",
+      "duration" : 454,
+      "executorId" : "driver",
+      "host" : "localhost",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 2,
+        "executorDeserializeCpuTime" : 0,
+        "executorRunTime" : 434,
+        "executorCpuTime" : 0,
+        "resultSize" : 1902,
+        "jvmGcTime" : 19,
+        "resultSerializationTime" : 1,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 3500016,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 1647,
+          "writeTime" : 83000,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : { },
+      "schedulerDelay" : 17,
+      "gettingResultTime" : 0
+    },
+    "8" : {
+      "taskId" : 8,
+      "index" : 0,
+      "attempt" : 0,
+      "launchTime" : "2015-02-03T16:43:05.829GMT",
+      "duration" : 454,
+      "executorId" : "driver",
+      "host" : "localhost",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 1,
+        "executorDeserializeCpuTime" : 0,
+        "executorRunTime" : 435,
+        "executorCpuTime" : 0,
+        "resultSize" : 1902,
+        "jvmGcTime" : 19,
+        "resultSerializationTime" : 2,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 3500016,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 1648,
+          "writeTime" : 94000,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : { },
+      "schedulerDelay" : 16,
+      "gettingResultTime" : 0
+    },
+    "15" : {
+      "taskId" : 15,
+      "index" : 7,
+      "attempt" : 0,
+      "launchTime" : "2015-02-03T16:43:05.833GMT",
+      "duration" : 450,
+      "executorId" : "driver",
+      "host" : "localhost",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 1,
+        "executorDeserializeCpuTime" : 0,
+        "executorRunTime" : 435,
+        "executorCpuTime" : 0,
+        "resultSize" : 1902,
+        "jvmGcTime" : 19,
+        "resultSerializationTime" : 1,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 3500016,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 1648,
+          "writeTime" : 79000,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : { },
+      "schedulerDelay" : 13,
+      "gettingResultTime" : 0
+    }
+  },
+  "executorSummary" : {
+    "driver" : {
+      "taskTime" : 3624,
+      "failedTasks" : 0,
+      "succeededTasks" : 8,
+      "killedTasks" : 0,
+      "inputBytes" : 28000128,
+      "inputRecords" : 0,
+      "outputBytes" : 0,
+      "outputRecords" : 0,
+      "shuffleRead" : 0,
+      "shuffleReadRecords" : 0,
+      "shuffleWrite" : 13180,
+      "shuffleWriteRecords" : 0,
+      "memoryBytesSpilled" : 0,
+      "diskBytesSpilled" : 0,
+      "isBlacklistedForStage" : false,
+      "peakMemoryMetrics" : {
+        "JVMHeapMemory" : 0,
+        "JVMOffHeapMemory" : 0,
+        "OnHeapExecutionMemory" : 0,
+        "OffHeapExecutionMemory" : 0,
+        "OnHeapStorageMemory" : 0,
+        "OffHeapStorageMemory" : 0,
+        "OnHeapUnifiedMemory" : 0,
+        "OffHeapUnifiedMemory" : 0,
+        "DirectPoolMemory" : 0,
+        "MappedPoolMemory" : 0,
+        "ProcessTreeJVMVMemory" : 0,
+        "ProcessTreeJVMRSSMemory" : 0,
+        "ProcessTreePythonVMemory" : 0,
+        "ProcessTreePythonRSSMemory" : 0,
+        "ProcessTreeOtherVMemory" : 0,
+        "ProcessTreeOtherRSSMemory" : 0,
+        "MinorGCCount" : 0,
+        "MinorGCTime" : 0,
+        "MajorGCCount" : 0,
+        "MajorGCTime" : 0
+      },
+      "isExcludedForStage" : false
+    }
+  },
+  "killedTasksSummary" : { },
+  "resourceProfileId" : 0,
+  "peakExecutorMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  }
+} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json
index edbac7127039d..d109c73b46133 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json
@@ -41,7 +41,30 @@
   "schedulingPool" : "default",
   "rddIds" : [ 6, 5 ],
   "accumulatorUpdates" : [ ],
-  "killedTasksSummary" : { }
+  "killedTasksSummary" : { },
+  "resourceProfileId" : 0,
+  "peakExecutorMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  }
 }, {
   "status" : "FAILED",
   "stageId" : 2,
@@ -86,7 +109,30 @@
   "schedulingPool" : "default",
   "rddIds" : [ 3, 2 ],
   "accumulatorUpdates" : [ ],
-  "killedTasksSummary" : { }
+  "killedTasksSummary" : { },
+  "resourceProfileId" : 0,
+  "peakExecutorMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  }
 }, {
   "status" : "COMPLETE",
   "stageId" : 1,
@@ -130,7 +176,30 @@
   "schedulingPool" : "default",
   "rddIds" : [ 1, 0 ],
   "accumulatorUpdates" : [ ],
-  "killedTasksSummary" : { }
+  "killedTasksSummary" : { },
+  "resourceProfileId" : 0,
+  "peakExecutorMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  }
 }, {
   "status" : "COMPLETE",
   "stageId" : 0,
@@ -174,5 +243,28 @@
   "schedulingPool" : "default",
   "rddIds" : [ 0 ],
   "accumulatorUpdates" : [ ],
-  "killedTasksSummary" : { }
+  "killedTasksSummary" : { },
+  "resourceProfileId" : 0,
+  "peakExecutorMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  }
 } ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json
index 836f2cb095097..7901c4f93367b 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json
@@ -45,5 +45,28 @@
     "name" : "my counter",
     "value" : "5050"
   } ],
-  "killedTasksSummary" : { }
+  "killedTasksSummary" : { },
+  "resourceProfileId" : 0,
+  "peakExecutorMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  }
 } ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json
new file mode 100644
index 0000000000000..d455b97fdafa4
--- /dev/null
+++ b/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json
@@ -0,0 +1,204 @@
+[ {
+  "status" : "COMPLETE",
+  "stageId" : 2,
+  "attemptId" : 0,
+  "numTasks" : 16,
+  "numActiveTasks" : 0,
+  "numCompleteTasks" : 16,
+  "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
+  "numCompletedIndices" : 16,
+  "submissionTime" : "2020-07-07T03:11:21.040GMT",
+  "firstTaskLaunchedTime" : "2020-07-07T03:11:21.077GMT",
+  "completionTime" : "2020-07-07T03:11:23.044GMT",
+  "executorDeserializeTime" : 3905,
+  "executorDeserializeCpuTime" : 979900000,
+  "executorRunTime" : 25579,
+  "executorCpuTime" : 8810338000,
+  "resultSize" : 33883,
+  "jvmGcTime" : 1010,
+  "resultSerializationTime" : 11,
+  "memoryBytesSpilled" : 0,
+  "diskBytesSpilled" : 0,
+  "peakExecutionMemory" : 384640,
+  "inputBytes" : 0,
+  "inputRecords" : 0,
+  "outputBytes" : 0,
+  "outputRecords" : 0,
+  "shuffleRemoteBlocksFetched" : 0,
+  "shuffleLocalBlocksFetched" : 0,
+  "shuffleFetchWaitTime" : 0,
+  "shuffleRemoteBytesRead" : 0,
+  "shuffleRemoteBytesReadToDisk" : 0,
+  "shuffleLocalBytesRead" : 0,
+  "shuffleReadBytes" : 0,
+  "shuffleReadRecords" : 0,
+  "shuffleWriteBytes" : 0,
+  "shuffleWriteTime" : 0,
+  "shuffleWriteRecords" : 0,
+  "name" : "foreach at <console>:26",
+  "details" : "org.apache.spark.sql.Dataset.foreach(Dataset.scala:2862)\n$line19.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:26)\n$line19.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:30)\n$line19.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:32)\n$line19.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:34)\n$line19.$read$$iw$$iw$$iw$$iw.<init>(<console>:36)\n$line19.$read$$iw$$iw$$iw.<init>(<console>:38)\n$line19.$read$$iw$$iw.<init>(<console>:40)\n$line19.$read$$iw.<init>(<console>:42)\n$line19.$read.<init>(<console>:44)\n$line19.$read$.<init>(<console>:48)\n$line19.$read$.<clinit>(<console>)\n$line19.$eval$.$print$lzycompute(<console>:7)\n$line19.$eval$.$print(<console>:6)\n$line19.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)",
+  "schedulingPool" : "default",
+  "rddIds" : [ 10, 8, 6, 7, 9 ],
+  "accumulatorUpdates" : [ ],
+  "killedTasksSummary" : { },
+  "resourceProfileId" : 0,
+  "peakExecutorMetrics" : {
+    "JVMHeapMemory" : 213367864,
+    "JVMOffHeapMemory" : 189011656,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 2133349,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 2133349,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 282024,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 13,
+    "MinorGCTime" : 115,
+    "MajorGCCount" : 4,
+    "MajorGCTime" : 339
+  }
+}, {
+  "status" : "COMPLETE",
+  "stageId" : 1,
+  "attemptId" : 0,
+  "numTasks" : 16,
+  "numActiveTasks" : 0,
+  "numCompleteTasks" : 16,
+  "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
+  "numCompletedIndices" : 16,
+  "submissionTime" : "2020-07-07T03:11:20.499GMT",
+  "firstTaskLaunchedTime" : "2020-07-07T03:11:20.502GMT",
+  "completionTime" : "2020-07-07T03:11:20.930GMT",
+  "executorDeserializeTime" : 424,
+  "executorDeserializeCpuTime" : 63666000,
+  "executorRunTime" : 6105,
+  "executorCpuTime" : 426449000,
+  "resultSize" : 31546,
+  "jvmGcTime" : 0,
+  "resultSerializationTime" : 0,
+  "memoryBytesSpilled" : 0,
+  "diskBytesSpilled" : 0,
+  "peakExecutionMemory" : 16256,
+  "inputBytes" : 0,
+  "inputRecords" : 0,
+  "outputBytes" : 0,
+  "outputRecords" : 0,
+  "shuffleRemoteBlocksFetched" : 0,
+  "shuffleLocalBlocksFetched" : 0,
+  "shuffleFetchWaitTime" : 0,
+  "shuffleRemoteBytesRead" : 0,
+  "shuffleRemoteBytesReadToDisk" : 0,
+  "shuffleLocalBytesRead" : 0,
+  "shuffleReadBytes" : 0,
+  "shuffleReadRecords" : 0,
+  "shuffleWriteBytes" : 0,
+  "shuffleWriteTime" : 0,
+  "shuffleWriteRecords" : 0,
+  "name" : "$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266",
+  "description" : "broadcast exchange (runId bac7289a-c1d8-4966-a6a3-d9f347e13a5d)",
+  "details" : "org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:185)\njava.util.concurrent.FutureTask.run(FutureTask.java:266)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)",
+  "schedulingPool" : "default",
+  "rddIds" : [ 5, 3, 4 ],
+  "accumulatorUpdates" : [ ],
+  "killedTasksSummary" : { },
+  "resourceProfileId" : 0,
+  "peakExecutorMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  }
+}, {
+  "status" : "COMPLETE",
+  "stageId" : 0,
+  "attemptId" : 0,
+  "numTasks" : 16,
+  "numActiveTasks" : 0,
+  "numCompleteTasks" : 16,
+  "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
+  "numCompletedIndices" : 16,
+  "submissionTime" : "2020-07-07T03:11:18.860GMT",
+  "firstTaskLaunchedTime" : "2020-07-07T03:11:19.253GMT",
+  "completionTime" : "2020-07-07T03:11:20.381GMT",
+  "executorDeserializeTime" : 15399,
+  "executorDeserializeCpuTime" : 3668889000,
+  "executorRunTime" : 1292,
+  "executorCpuTime" : 172863000,
+  "resultSize" : 22375,
+  "jvmGcTime" : 1744,
+  "resultSerializationTime" : 3,
+  "memoryBytesSpilled" : 0,
+  "diskBytesSpilled" : 0,
+  "peakExecutionMemory" : 0,
+  "inputBytes" : 0,
+  "inputRecords" : 0,
+  "outputBytes" : 0,
+  "outputRecords" : 0,
+  "shuffleRemoteBlocksFetched" : 0,
+  "shuffleLocalBlocksFetched" : 0,
+  "shuffleFetchWaitTime" : 0,
+  "shuffleRemoteBytesRead" : 0,
+  "shuffleRemoteBytesReadToDisk" : 0,
+  "shuffleLocalBytesRead" : 0,
+  "shuffleReadBytes" : 0,
+  "shuffleReadRecords" : 0,
+  "shuffleWriteBytes" : 0,
+  "shuffleWriteTime" : 0,
+  "shuffleWriteRecords" : 0,
+  "name" : "$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266",
+  "description" : "broadcast exchange (runId bb0234e5-4157-49a4-b40c-6d538d9f2ec8)",
+  "details" : "org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:185)\njava.util.concurrent.FutureTask.run(FutureTask.java:266)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)",
+  "schedulingPool" : "default",
+  "rddIds" : [ 2, 0, 1 ],
+  "accumulatorUpdates" : [ ],
+  "killedTasksSummary" : { },
+  "resourceProfileId" : 0,
+  "peakExecutorMetrics" : {
+    "JVMHeapMemory" : 155100856,
+    "JVMOffHeapMemory" : 64239224,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 6964,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 6964,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 1852,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 7,
+    "MinorGCTime" : 33,
+    "MajorGCCount" : 3,
+    "MajorGCTime" : 110
+  }
+} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___offset___length_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___offset___length_expectation.json
new file mode 100644
index 0000000000000..28509e33c5dcc
--- /dev/null
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___offset___length_expectation.json
@@ -0,0 +1,99 @@
+[ {
+  "taskId" : 1,
+  "index" : 1,
+  "attempt" : 0,
+  "launchTime" : "2015-05-06T13:03:06.502GMT",
+  "duration" : 421,
+  "executorId" : "driver",
+  "host" : "localhost",
+  "status" : "SUCCESS",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "taskMetrics" : {
+    "executorDeserializeTime" : 31,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 350,
+    "executorCpuTime" : 0,
+    "resultSize" : 2010,
+    "jvmGcTime" : 7,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 60488,
+      "recordsRead" : 10000
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 1710,
+      "writeTime" : 3934399,
+      "recordsWritten" : 10
+    }
+  },
+  "executorLogs" : { },
+  "schedulerDelay" : 40,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 2,
+  "index" : 2,
+  "attempt" : 0,
+  "launchTime" : "2015-05-06T13:03:06.503GMT",
+  "duration" : 419,
+  "executorId" : "driver",
+  "host" : "localhost",
+  "status" : "SUCCESS",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "taskMetrics" : {
+    "executorDeserializeTime" : 32,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 348,
+    "executorCpuTime" : 0,
+    "resultSize" : 2010,
+    "jvmGcTime" : 7,
+    "resultSerializationTime" : 2,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 60488,
+      "recordsRead" : 10000
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 1710,
+      "writeTime" : 89885,
+      "recordsWritten" : 10
+    }
+  },
+  "executorLogs" : { },
+  "schedulerDelay" : 37,
+  "gettingResultTime" : 0
+} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___sortBy_short_names__runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___sortBy_short_names__runtime_expectation.json
new file mode 100644
index 0000000000000..01eef1b565bf6
--- /dev/null
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___sortBy_short_names__runtime_expectation.json
@@ -0,0 +1,981 @@
+[ {
+  "taskId" : 40,
+  "index" : 40,
+  "attempt" : 0,
+  "launchTime" : "2015-05-06T13:03:07.197GMT",
+  "duration" : 24,
+  "executorId" : "driver",
+  "host" : "localhost",
+  "status" : "SUCCESS",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "taskMetrics" : {
+    "executorDeserializeTime" : 4,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 14,
+    "executorCpuTime" : 0,
+    "resultSize" : 2065,
+    "jvmGcTime" : 0,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 70564,
+      "recordsRead" : 10000
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 1710,
+      "writeTime" : 94792,
+      "recordsWritten" : 10
+    }
+  },
+  "executorLogs" : { },
+  "schedulerDelay" : 6,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 41,
+  "index" : 41,
+  "attempt" : 0,
+  "launchTime" : "2015-05-06T13:03:07.200GMT",
+  "duration" : 24,
+  "executorId" : "driver",
+  "host" : "localhost",
+  "status" : "SUCCESS",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "taskMetrics" : {
+    "executorDeserializeTime" : 2,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 16,
+    "executorCpuTime" : 0,
+    "resultSize" : 2065,
+    "jvmGcTime" : 0,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 70564,
+      "recordsRead" : 10000
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 1710,
+      "writeTime" : 90765,
+      "recordsWritten" : 10
+    }
+  },
+  "executorLogs" : { },
+  "schedulerDelay" : 6,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 43,
+  "index" : 43,
+  "attempt" : 0,
+  "launchTime" : "2015-05-06T13:03:07.204GMT",
+  "duration" : 39,
+  "executorId" : "driver",
+  "host" : "localhost",
+  "status" : "SUCCESS",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "taskMetrics" : {
+    "executorDeserializeTime" : 2,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 16,
+    "executorCpuTime" : 0,
+    "resultSize" : 2065,
+    "jvmGcTime" : 0,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 70564,
+      "recordsRead" : 10000
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 1710,
+      "writeTime" : 171516,
+      "recordsWritten" : 10
+    }
+  },
+  "executorLogs" : { },
+  "schedulerDelay" : 21,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 57,
+  "index" : 57,
+  "attempt" : 0,
+  "launchTime" : "2015-05-06T13:03:07.257GMT",
+  "duration" : 21,
+  "executorId" : "driver",
+  "host" : "localhost",
+  "status" : "SUCCESS",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "taskMetrics" : {
+    "executorDeserializeTime" : 3,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 16,
+    "executorCpuTime" : 0,
+    "resultSize" : 2065,
+    "jvmGcTime" : 0,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 70564,
+      "recordsRead" : 10000
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 1710,
+      "writeTime" : 96849,
+      "recordsWritten" : 10
+    }
+  },
+  "executorLogs" : { },
+  "schedulerDelay" : 2,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 58,
+  "index" : 58,
+  "attempt" : 0,
+  "launchTime" : "2015-05-06T13:03:07.263GMT",
+  "duration" : 23,
+  "executorId" : "driver",
+  "host" : "localhost",
+  "status" : "SUCCESS",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "taskMetrics" : {
+    "executorDeserializeTime" : 3,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 16,
+    "executorCpuTime" : 0,
+    "resultSize" : 2065,
+    "jvmGcTime" : 0,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 70564,
+      "recordsRead" : 10000
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 1710,
+      "writeTime" : 97521,
+      "recordsWritten" : 10
+    }
+  },
+  "executorLogs" : { },
+  "schedulerDelay" : 4,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 68,
+  "index" : 68,
+  "attempt" : 0,
+  "launchTime" : "2015-05-06T13:03:07.306GMT",
+  "duration" : 22,
+  "executorId" : "driver",
+  "host" : "localhost",
+  "status" : "SUCCESS",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "taskMetrics" : {
+    "executorDeserializeTime" : 2,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 16,
+    "executorCpuTime" : 0,
+    "resultSize" : 2065,
+    "jvmGcTime" : 0,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 70564,
+      "recordsRead" : 10000
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 1710,
+      "writeTime" : 101750,
+      "recordsWritten" : 10
+    }
+  },
+  "executorLogs" : { },
+  "schedulerDelay" : 4,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 86,
+  "index" : 86,
+  "attempt" : 0,
+  "launchTime" : "2015-05-06T13:03:07.374GMT",
+  "duration" : 28,
+  "executorId" : "driver",
+  "host" : "localhost",
+  "status" : "SUCCESS",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "taskMetrics" : {
+    "executorDeserializeTime" : 3,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 16,
+    "executorCpuTime" : 0,
+    "resultSize" : 2065,
+    "jvmGcTime" : 0,
+    "resultSerializationTime" : 1,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 70564,
+      "recordsRead" : 10000
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 1710,
+      "writeTime" : 95848,
+      "recordsWritten" : 10
+    }
+  },
+  "executorLogs" : { },
+  "schedulerDelay" : 8,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 32,
+  "index" : 32,
+  "attempt" : 0,
+  "launchTime" : "2015-05-06T13:03:07.148GMT",
+  "duration" : 33,
+  "executorId" : "driver",
+  "host" : "localhost",
+  "status" : "SUCCESS",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "taskMetrics" : {
+    "executorDeserializeTime" : 3,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 17,
+    "executorCpuTime" : 0,
+    "resultSize" : 2065,
+    "jvmGcTime" : 0,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 70564,
+      "recordsRead" : 10000
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 1710,
+      "writeTime" : 89603,
+      "recordsWritten" : 10
+    }
+  },
+  "executorLogs" : { },
+  "schedulerDelay" : 13,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 39,
+  "index" : 39,
+  "attempt" : 0,
+  "launchTime" : "2015-05-06T13:03:07.180GMT",
+  "duration" : 32,
+  "executorId" : "driver",
+  "host" : "localhost",
+  "status" : "SUCCESS",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "taskMetrics" : {
+    "executorDeserializeTime" : 2,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 17,
+    "executorCpuTime" : 0,
+    "resultSize" : 2065,
+    "jvmGcTime" : 0,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 70564,
+      "recordsRead" : 10000
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 1710,
+      "writeTime" : 98748,
+      "recordsWritten" : 10
+    }
+  },
+  "executorLogs" : { },
+  "schedulerDelay" : 13,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 42,
+  "index" : 42,
+  "attempt" : 0,
+  "launchTime" : "2015-05-06T13:03:07.203GMT",
+  "duration" : 42,
+  "executorId" : "driver",
+  "host" : "localhost",
+  "status" : "SUCCESS",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "taskMetrics" : {
+    "executorDeserializeTime" : 10,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 17,
+    "executorCpuTime" : 0,
+    "resultSize" : 2065,
+    "jvmGcTime" : 0,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 70564,
+      "recordsRead" : 10000
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 1710,
+      "writeTime" : 103713,
+      "recordsWritten" : 10
+    }
+  },
+  "executorLogs" : { },
+  "schedulerDelay" : 15,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 51,
+  "index" : 51,
+  "attempt" : 0,
+  "launchTime" : "2015-05-06T13:03:07.242GMT",
+  "duration" : 21,
+  "executorId" : "driver",
+  "host" : "localhost",
+  "status" : "SUCCESS",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "taskMetrics" : {
+    "executorDeserializeTime" : 2,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 17,
+    "executorCpuTime" : 0,
+    "resultSize" : 2065,
+    "jvmGcTime" : 0,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 70564,
+      "recordsRead" : 10000
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 1710,
+      "writeTime" : 96013,
+      "recordsWritten" : 10
+    }
+  },
+  "executorLogs" : { },
+  "schedulerDelay" : 2,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 59,
+  "index" : 59,
+  "attempt" : 0,
+  "launchTime" : "2015-05-06T13:03:07.265GMT",
+  "duration" : 23,
+  "executorId" : "driver",
+  "host" : "localhost",
+  "status" : "SUCCESS",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "taskMetrics" : {
+    "executorDeserializeTime" : 3,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 17,
+    "executorCpuTime" : 0,
+    "resultSize" : 2065,
+    "jvmGcTime" : 0,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 70564,
+      "recordsRead" : 10000
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 1710,
+      "writeTime" : 100753,
+      "recordsWritten" : 10
+    }
+  },
+  "executorLogs" : { },
+  "schedulerDelay" : 3,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 63,
+  "index" : 63,
+  "attempt" : 0,
+  "launchTime" : "2015-05-06T13:03:07.276GMT",
+  "duration" : 40,
+  "executorId" : "driver",
+  "host" : "localhost",
+  "status" : "SUCCESS",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "taskMetrics" : {
+    "executorDeserializeTime" : 20,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 17,
+    "executorCpuTime" : 0,
+    "resultSize" : 2065,
+    "jvmGcTime" : 5,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 70564,
+      "recordsRead" : 10000
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 1710,
+      "writeTime" : 102779,
+      "recordsWritten" : 10
+    }
+  },
+  "executorLogs" : { },
+  "schedulerDelay" : 3,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 87,
+  "index" : 87,
+  "attempt" : 0,
+  "launchTime" : "2015-05-06T13:03:07.374GMT",
+  "duration" : 36,
+  "executorId" : "driver",
+  "host" : "localhost",
+  "status" : "SUCCESS",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "taskMetrics" : {
+    "executorDeserializeTime" : 12,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 17,
+    "executorCpuTime" : 0,
+    "resultSize" : 2065,
+    "jvmGcTime" : 0,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 70564,
+      "recordsRead" : 10000
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 1710,
+      "writeTime" : 102159,
+      "recordsWritten" : 10
+    }
+  },
+  "executorLogs" : { },
+  "schedulerDelay" : 7,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 90,
+  "index" : 90,
+  "attempt" : 0,
+  "launchTime" : "2015-05-06T13:03:07.385GMT",
+  "duration" : 23,
+  "executorId" : "driver",
+  "host" : "localhost",
+  "status" : "SUCCESS",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "taskMetrics" : {
+    "executorDeserializeTime" : 2,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 17,
+    "executorCpuTime" : 0,
+    "resultSize" : 2065,
+    "jvmGcTime" : 0,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 70564,
+      "recordsRead" : 10000
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 1710,
+      "writeTime" : 98472,
+      "recordsWritten" : 10
+    }
+  },
+  "executorLogs" : { },
+  "schedulerDelay" : 4,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 99,
+  "index" : 99,
+  "attempt" : 0,
+  "launchTime" : "2015-05-06T13:03:07.426GMT",
+  "duration" : 22,
+  "executorId" : "driver",
+  "host" : "localhost",
+  "status" : "SUCCESS",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "taskMetrics" : {
+    "executorDeserializeTime" : 2,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 17,
+    "executorCpuTime" : 0,
+    "resultSize" : 2065,
+    "jvmGcTime" : 0,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 70565,
+      "recordsRead" : 10000
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 1710,
+      "writeTime" : 133964,
+      "recordsWritten" : 10
+    }
+  },
+  "executorLogs" : { },
+  "schedulerDelay" : 3,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 44,
+  "index" : 44,
+  "attempt" : 0,
+  "launchTime" : "2015-05-06T13:03:07.205GMT",
+  "duration" : 37,
+  "executorId" : "driver",
+  "host" : "localhost",
+  "status" : "SUCCESS",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "taskMetrics" : {
+    "executorDeserializeTime" : 3,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 18,
+    "executorCpuTime" : 0,
+    "resultSize" : 2065,
+    "jvmGcTime" : 0,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 70564,
+      "recordsRead" : 10000
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 1710,
+      "writeTime" : 98293,
+      "recordsWritten" : 10
+    }
+  },
+  "executorLogs" : { },
+  "schedulerDelay" : 16,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 47,
+  "index" : 47,
+  "attempt" : 0,
+  "launchTime" : "2015-05-06T13:03:07.212GMT",
+  "duration" : 33,
+  "executorId" : "driver",
+  "host" : "localhost",
+  "status" : "SUCCESS",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "taskMetrics" : {
+    "executorDeserializeTime" : 2,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 18,
+    "executorCpuTime" : 0,
+    "resultSize" : 2065,
+    "jvmGcTime" : 0,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 70564,
+      "recordsRead" : 10000
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 1710,
+      "writeTime" : 103015,
+      "recordsWritten" : 10
+    }
+  },
+  "executorLogs" : { },
+  "schedulerDelay" : 13,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 50,
+  "index" : 50,
+  "attempt" : 0,
+  "launchTime" : "2015-05-06T13:03:07.240GMT",
+  "duration" : 26,
+  "executorId" : "driver",
+  "host" : "localhost",
+  "status" : "SUCCESS",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "taskMetrics" : {
+    "executorDeserializeTime" : 4,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 18,
+    "executorCpuTime" : 0,
+    "resultSize" : 2065,
+    "jvmGcTime" : 0,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 70564,
+      "recordsRead" : 10000
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 1710,
+      "writeTime" : 90836,
+      "recordsWritten" : 10
+    }
+  },
+  "executorLogs" : { },
+  "schedulerDelay" : 4,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 52,
+  "index" : 52,
+  "attempt" : 0,
+  "launchTime" : "2015-05-06T13:03:07.243GMT",
+  "duration" : 28,
+  "executorId" : "driver",
+  "host" : "localhost",
+  "status" : "SUCCESS",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "taskMetrics" : {
+    "executorDeserializeTime" : 5,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 18,
+    "executorCpuTime" : 0,
+    "resultSize" : 2065,
+    "jvmGcTime" : 0,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 70564,
+      "recordsRead" : 10000
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 1710,
+      "writeTime" : 89664,
+      "recordsWritten" : 10
+    }
+  },
+  "executorLogs" : { },
+  "schedulerDelay" : 5,
+  "gettingResultTime" : 0
+} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status_expectation.json
new file mode 100644
index 0000000000000..9896aceb275de
--- /dev/null
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status_expectation.json
@@ -0,0 +1,531 @@
+[ {
+  "taskId" : 1,
+  "index" : 1,
+  "attempt" : 0,
+  "launchTime" : "2016-11-15T23:20:44.052GMT",
+  "duration" : 675,
+  "executorId" : "0",
+  "host" : "172.22.0.111",
+  "status" : "FAILED",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "errorMessage" : "java.lang.RuntimeException: bad exec\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply$mcII$sp(<console>:26)\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:26)\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:26)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1757)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1135)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1135)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1927)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1927)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:99)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:282)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)\n\tat java.lang.Thread.run(Thread.java:745)\n",
+  "taskMetrics" : {
+    "executorDeserializeTime" : 0,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 494,
+    "executorCpuTime" : 0,
+    "resultSize" : 0,
+    "jvmGcTime" : 30,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
+  },
+  "executorLogs" : {
+    "stdout" : "http://172.22.0.111:64517/logPage/?appId=app-20161115172038-0000&executorId=0&logType=stdout",
+    "stderr" : "http://172.22.0.111:64517/logPage/?appId=app-20161115172038-0000&executorId=0&logType=stderr"
+  },
+  "schedulerDelay" : 181,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 3,
+  "index" : 3,
+  "attempt" : 0,
+  "launchTime" : "2016-11-15T23:20:44.053GMT",
+  "duration" : 725,
+  "executorId" : "2",
+  "host" : "172.22.0.111",
+  "status" : "FAILED",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "errorMessage" : "java.lang.RuntimeException: bad exec\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply$mcII$sp(<console>:26)\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:26)\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:26)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1757)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1135)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1135)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1927)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1927)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:99)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:282)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)\n\tat java.lang.Thread.run(Thread.java:745)\n",
+  "taskMetrics" : {
+    "executorDeserializeTime" : 0,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 456,
+    "executorCpuTime" : 0,
+    "resultSize" : 0,
+    "jvmGcTime" : 32,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
+  },
+  "executorLogs" : {
+    "stdout" : "http://172.22.0.111:64519/logPage/?appId=app-20161115172038-0000&executorId=2&logType=stdout",
+    "stderr" : "http://172.22.0.111:64519/logPage/?appId=app-20161115172038-0000&executorId=2&logType=stderr"
+  },
+  "schedulerDelay" : 269,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 5,
+  "index" : 5,
+  "attempt" : 0,
+  "launchTime" : "2016-11-15T23:20:44.055GMT",
+  "duration" : 665,
+  "executorId" : "0",
+  "host" : "172.22.0.111",
+  "status" : "FAILED",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "errorMessage" : "java.lang.RuntimeException: bad exec\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply$mcII$sp(<console>:26)\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:26)\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:26)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1757)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1135)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1135)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1927)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1927)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:99)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:282)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)\n\tat java.lang.Thread.run(Thread.java:745)\n",
+  "taskMetrics" : {
+    "executorDeserializeTime" : 0,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 495,
+    "executorCpuTime" : 0,
+    "resultSize" : 0,
+    "jvmGcTime" : 30,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
+  },
+  "executorLogs" : {
+    "stdout" : "http://172.22.0.111:64517/logPage/?appId=app-20161115172038-0000&executorId=0&logType=stdout",
+    "stderr" : "http://172.22.0.111:64517/logPage/?appId=app-20161115172038-0000&executorId=0&logType=stderr"
+  },
+  "schedulerDelay" : 170,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 7,
+  "index" : 7,
+  "attempt" : 0,
+  "launchTime" : "2016-11-15T23:20:44.056GMT",
+  "duration" : 685,
+  "executorId" : "2",
+  "host" : "172.22.0.111",
+  "status" : "FAILED",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "errorMessage" : "java.lang.RuntimeException: bad exec\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply$mcII$sp(<console>:26)\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:26)\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:26)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1757)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1135)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1135)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1927)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1927)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:99)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:282)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)\n\tat java.lang.Thread.run(Thread.java:745)\n",
+  "taskMetrics" : {
+    "executorDeserializeTime" : 0,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 448,
+    "executorCpuTime" : 0,
+    "resultSize" : 0,
+    "jvmGcTime" : 32,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
+  },
+  "executorLogs" : {
+    "stdout" : "http://172.22.0.111:64519/logPage/?appId=app-20161115172038-0000&executorId=2&logType=stdout",
+    "stderr" : "http://172.22.0.111:64519/logPage/?appId=app-20161115172038-0000&executorId=2&logType=stderr"
+  },
+  "schedulerDelay" : 237,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 9,
+  "index" : 9,
+  "attempt" : 0,
+  "launchTime" : "2016-11-15T23:20:44.057GMT",
+  "duration" : 732,
+  "executorId" : "0",
+  "host" : "172.22.0.111",
+  "status" : "FAILED",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "errorMessage" : "java.lang.RuntimeException: bad exec\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply$mcII$sp(<console>:26)\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:26)\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:26)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1757)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1135)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1135)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1927)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1927)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:99)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:282)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)\n\tat java.lang.Thread.run(Thread.java:745)\n",
+  "taskMetrics" : {
+    "executorDeserializeTime" : 0,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 503,
+    "executorCpuTime" : 0,
+    "resultSize" : 0,
+    "jvmGcTime" : 30,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
+  },
+  "executorLogs" : {
+    "stdout" : "http://172.22.0.111:64517/logPage/?appId=app-20161115172038-0000&executorId=0&logType=stdout",
+    "stderr" : "http://172.22.0.111:64517/logPage/?appId=app-20161115172038-0000&executorId=0&logType=stderr"
+  },
+  "schedulerDelay" : 229,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 11,
+  "index" : 11,
+  "attempt" : 0,
+  "launchTime" : "2016-11-15T23:20:44.058GMT",
+  "duration" : 678,
+  "executorId" : "2",
+  "host" : "172.22.0.111",
+  "status" : "FAILED",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "errorMessage" : "java.lang.RuntimeException: bad exec\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply$mcII$sp(<console>:26)\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:26)\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:26)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1757)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1135)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1135)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1927)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1927)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:99)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:282)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)\n\tat java.lang.Thread.run(Thread.java:745)\n",
+  "taskMetrics" : {
+    "executorDeserializeTime" : 0,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 451,
+    "executorCpuTime" : 0,
+    "resultSize" : 0,
+    "jvmGcTime" : 32,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
+  },
+  "executorLogs" : {
+    "stdout" : "http://172.22.0.111:64519/logPage/?appId=app-20161115172038-0000&executorId=2&logType=stdout",
+    "stderr" : "http://172.22.0.111:64519/logPage/?appId=app-20161115172038-0000&executorId=2&logType=stderr"
+  },
+  "schedulerDelay" : 227,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 13,
+  "index" : 13,
+  "attempt" : 0,
+  "launchTime" : "2016-11-15T23:20:44.060GMT",
+  "duration" : 669,
+  "executorId" : "0",
+  "host" : "172.22.0.111",
+  "status" : "FAILED",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "errorMessage" : "java.lang.RuntimeException: bad exec\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply$mcII$sp(<console>:26)\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:26)\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:26)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1757)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1135)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1135)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1927)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1927)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:99)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:282)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)\n\tat java.lang.Thread.run(Thread.java:745)\n",
+  "taskMetrics" : {
+    "executorDeserializeTime" : 0,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 494,
+    "executorCpuTime" : 0,
+    "resultSize" : 0,
+    "jvmGcTime" : 30,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
+  },
+  "executorLogs" : {
+    "stdout" : "http://172.22.0.111:64517/logPage/?appId=app-20161115172038-0000&executorId=0&logType=stdout",
+    "stderr" : "http://172.22.0.111:64517/logPage/?appId=app-20161115172038-0000&executorId=0&logType=stderr"
+  },
+  "schedulerDelay" : 175,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 15,
+  "index" : 15,
+  "attempt" : 0,
+  "launchTime" : "2016-11-15T23:20:44.065GMT",
+  "duration" : 672,
+  "executorId" : "2",
+  "host" : "172.22.0.111",
+  "status" : "FAILED",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "errorMessage" : "java.lang.RuntimeException: bad exec\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply$mcII$sp(<console>:26)\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:26)\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:26)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1757)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1135)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1135)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1927)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1927)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:99)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:282)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)\n\tat java.lang.Thread.run(Thread.java:745)\n",
+  "taskMetrics" : {
+    "executorDeserializeTime" : 0,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 446,
+    "executorCpuTime" : 0,
+    "resultSize" : 0,
+    "jvmGcTime" : 32,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
+  },
+  "executorLogs" : {
+    "stdout" : "http://172.22.0.111:64519/logPage/?appId=app-20161115172038-0000&executorId=2&logType=stdout",
+    "stderr" : "http://172.22.0.111:64519/logPage/?appId=app-20161115172038-0000&executorId=2&logType=stderr"
+  },
+  "schedulerDelay" : 226,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 19,
+  "index" : 11,
+  "attempt" : 1,
+  "launchTime" : "2016-11-15T23:20:44.736GMT",
+  "duration" : 13,
+  "executorId" : "2",
+  "host" : "172.22.0.111",
+  "status" : "FAILED",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "errorMessage" : "java.lang.RuntimeException: bad exec\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply$mcII$sp(<console>:26)\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:26)\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:26)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1757)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1135)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1135)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1927)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1927)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:99)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:282)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)\n\tat java.lang.Thread.run(Thread.java:745)\n",
+  "taskMetrics" : {
+    "executorDeserializeTime" : 0,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 2,
+    "executorCpuTime" : 0,
+    "resultSize" : 0,
+    "jvmGcTime" : 0,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
+  },
+  "executorLogs" : {
+    "stdout" : "http://172.22.0.111:64519/logPage/?appId=app-20161115172038-0000&executorId=2&logType=stdout",
+    "stderr" : "http://172.22.0.111:64519/logPage/?appId=app-20161115172038-0000&executorId=2&logType=stderr"
+  },
+  "schedulerDelay" : 11,
+  "gettingResultTime" : 0
+}, {
+  "taskId" : 20,
+  "index" : 15,
+  "attempt" : 1,
+  "launchTime" : "2016-11-15T23:20:44.737GMT",
+  "duration" : 19,
+  "executorId" : "2",
+  "host" : "172.22.0.111",
+  "status" : "FAILED",
+  "taskLocality" : "PROCESS_LOCAL",
+  "speculative" : false,
+  "accumulatorUpdates" : [ ],
+  "errorMessage" : "java.lang.RuntimeException: bad exec\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply$mcII$sp(<console>:26)\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:26)\n\tat $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:26)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1757)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1135)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1135)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1927)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1927)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:99)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:282)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)\n\tat java.lang.Thread.run(Thread.java:745)\n",
+  "taskMetrics" : {
+    "executorDeserializeTime" : 0,
+    "executorDeserializeCpuTime" : 0,
+    "executorRunTime" : 10,
+    "executorCpuTime" : 0,
+    "resultSize" : 0,
+    "jvmGcTime" : 0,
+    "resultSerializationTime" : 0,
+    "memoryBytesSpilled" : 0,
+    "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "remoteBytesReadToDisk" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
+  },
+  "executorLogs" : {
+    "stdout" : "http://172.22.0.111:64519/logPage/?appId=app-20161115172038-0000&executorId=2&logType=stdout",
+    "stderr" : "http://172.22.0.111:64519/logPage/?appId=app-20161115172038-0000&executorId=2&logType=stderr"
+  },
+  "schedulerDelay" : 9,
+  "gettingResultTime" : 0
+} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json
index 5c42ac1d87f4c..51af1f3b82c7a 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json
@@ -1,5 +1,6 @@
 {
   "quantiles" : [ 0.01, 0.5, 0.99 ],
+  "duration" : [ 21.0, 40.0, 435.0 ],
   "executorDeserializeTime" : [ 1.0, 3.0, 36.0 ],
   "executorDeserializeCpuTime" : [ 0.0, 0.0, 0.0 ],
   "executorRunTime" : [ 16.0, 28.0, 351.0 ],
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json
index e6b705989cc97..a2ac950753258 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json
@@ -1,5 +1,6 @@
 {
   "quantiles" : [ 0.05, 0.25, 0.5, 0.75, 0.95 ],
+  "duration" : [ 37.0, 81.0, 83.0, 84.0, 86.0 ],
   "executorDeserializeTime" : [ 1.0, 2.0, 2.0, 2.0, 3.0 ],
   "executorDeserializeCpuTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
   "executorRunTime" : [ 30.0, 74.0, 75.0, 76.0, 79.0 ],
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json
index 788f28cf7b365..78aa1b1fa809d 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json
@@ -1,5 +1,6 @@
 {
   "quantiles" : [ 0.05, 0.25, 0.5, 0.75, 0.95 ],
+  "duration" : [ 23.0, 33.0, 40.0, 74.0, 419.0 ],
   "executorDeserializeTime" : [ 2.0, 2.0, 3.0, 7.0, 31.0 ],
   "executorDeserializeCpuTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
   "executorRunTime" : [ 16.0, 18.0, 28.0, 49.0, 349.0 ],
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
index 735a8257fc343..a5958e0a093f1 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
@@ -503,8 +503,54 @@
       "shuffleWriteRecords" : 0,
       "memoryBytesSpilled" : 0,
       "diskBytesSpilled" : 0,
-      "isBlacklistedForStage" : false
+      "isBlacklistedForStage" : false,
+      "peakMemoryMetrics" : {
+        "JVMHeapMemory" : 0,
+        "JVMOffHeapMemory" : 0,
+        "OnHeapExecutionMemory" : 0,
+        "OffHeapExecutionMemory" : 0,
+        "OnHeapStorageMemory" : 0,
+        "OffHeapStorageMemory" : 0,
+        "OnHeapUnifiedMemory" : 0,
+        "OffHeapUnifiedMemory" : 0,
+        "DirectPoolMemory" : 0,
+        "MappedPoolMemory" : 0,
+        "ProcessTreeJVMVMemory" : 0,
+        "ProcessTreeJVMRSSMemory" : 0,
+        "ProcessTreePythonVMemory" : 0,
+        "ProcessTreePythonRSSMemory" : 0,
+        "ProcessTreeOtherVMemory" : 0,
+        "ProcessTreeOtherRSSMemory" : 0,
+        "MinorGCCount" : 0,
+        "MinorGCTime" : 0,
+        "MajorGCCount" : 0,
+        "MajorGCTime" : 0
+      },
+      "isExcludedForStage" : false
     }
   },
-  "killedTasksSummary" : { }
+  "killedTasksSummary" : { },
+  "resourceProfileId" : 0,
+  "peakExecutorMetrics" : {
+    "JVMHeapMemory" : 0,
+    "JVMOffHeapMemory" : 0,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 0,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 0,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 0,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 0,
+    "MinorGCTime" : 0,
+    "MajorGCCount" : 0,
+    "MajorGCTime" : 0
+  }
 }
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json
new file mode 100644
index 0000000000000..20a958073245a
--- /dev/null
+++ b/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json
@@ -0,0 +1,1000 @@
+{
+  "status" : "COMPLETE",
+  "stageId" : 2,
+  "attemptId" : 0,
+  "numTasks" : 16,
+  "numActiveTasks" : 0,
+  "numCompleteTasks" : 16,
+  "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
+  "numCompletedIndices" : 16,
+  "submissionTime" : "2020-07-07T03:11:21.040GMT",
+  "firstTaskLaunchedTime" : "2020-07-07T03:11:21.077GMT",
+  "completionTime" : "2020-07-07T03:11:23.044GMT",
+  "executorDeserializeTime" : 3905,
+  "executorDeserializeCpuTime" : 979900000,
+  "executorRunTime" : 25579,
+  "executorCpuTime" : 8810338000,
+  "resultSize" : 33883,
+  "jvmGcTime" : 1010,
+  "resultSerializationTime" : 11,
+  "memoryBytesSpilled" : 0,
+  "diskBytesSpilled" : 0,
+  "peakExecutionMemory" : 384640,
+  "inputBytes" : 0,
+  "inputRecords" : 0,
+  "outputBytes" : 0,
+  "outputRecords" : 0,
+  "shuffleRemoteBlocksFetched" : 0,
+  "shuffleLocalBlocksFetched" : 0,
+  "shuffleFetchWaitTime" : 0,
+  "shuffleRemoteBytesRead" : 0,
+  "shuffleRemoteBytesReadToDisk" : 0,
+  "shuffleLocalBytesRead" : 0,
+  "shuffleReadBytes" : 0,
+  "shuffleReadRecords" : 0,
+  "shuffleWriteBytes" : 0,
+  "shuffleWriteTime" : 0,
+  "shuffleWriteRecords" : 0,
+  "name" : "foreach at <console>:26",
+  "details" : "org.apache.spark.sql.Dataset.foreach(Dataset.scala:2862)\n$line19.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:26)\n$line19.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:30)\n$line19.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:32)\n$line19.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:34)\n$line19.$read$$iw$$iw$$iw$$iw.<init>(<console>:36)\n$line19.$read$$iw$$iw$$iw.<init>(<console>:38)\n$line19.$read$$iw$$iw.<init>(<console>:40)\n$line19.$read$$iw.<init>(<console>:42)\n$line19.$read.<init>(<console>:44)\n$line19.$read$.<init>(<console>:48)\n$line19.$read$.<clinit>(<console>)\n$line19.$eval$.$print$lzycompute(<console>:7)\n$line19.$eval$.$print(<console>:6)\n$line19.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)",
+  "schedulingPool" : "default",
+  "rddIds" : [ 10, 8, 6, 7, 9 ],
+  "accumulatorUpdates" : [ ],
+  "tasks" : {
+    "42" : {
+      "taskId" : 42,
+      "index" : 10,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.120GMT",
+      "duration" : 1923,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 229,
+        "executorDeserializeCpuTime" : 58152000,
+        "executorRunTime" : 1624,
+        "executorCpuTime" : 508230000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 66,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 70,
+      "gettingResultTime" : 0
+    },
+    "37" : {
+      "taskId" : 37,
+      "index" : 5,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.100GMT",
+      "duration" : 1915,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 256,
+        "executorDeserializeCpuTime" : 60890000,
+        "executorRunTime" : 1596,
+        "executorCpuTime" : 507192000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 62,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 63,
+      "gettingResultTime" : 0
+    },
+    "46" : {
+      "taskId" : 46,
+      "index" : 14,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.132GMT",
+      "duration" : 1905,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 218,
+        "executorDeserializeCpuTime" : 51464000,
+        "executorRunTime" : 1618,
+        "executorCpuTime" : 490927000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 66,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 69,
+      "gettingResultTime" : 0
+    },
+    "38" : {
+      "taskId" : 38,
+      "index" : 6,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.104GMT",
+      "duration" : 1835,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 255,
+        "executorDeserializeCpuTime" : 60358000,
+        "executorRunTime" : 1498,
+        "executorCpuTime" : 414110000,
+        "resultSize" : 2158,
+        "jvmGcTime" : 62,
+        "resultSerializationTime" : 11,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 71,
+      "gettingResultTime" : 0
+    },
+    "33" : {
+      "taskId" : 33,
+      "index" : 1,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.082GMT",
+      "duration" : 1943,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 267,
+        "executorDeserializeCpuTime" : 54442000,
+        "executorRunTime" : 1597,
+        "executorCpuTime" : 519178000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 62,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 79,
+      "gettingResultTime" : 0
+    },
+    "41" : {
+      "taskId" : 41,
+      "index" : 9,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.116GMT",
+      "duration" : 1916,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 240,
+        "executorDeserializeCpuTime" : 55787000,
+        "executorRunTime" : 1614,
+        "executorCpuTime" : 489923000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 66,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 62,
+      "gettingResultTime" : 0
+    },
+    "32" : {
+      "taskId" : 32,
+      "index" : 0,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.077GMT",
+      "duration" : 1960,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 271,
+        "executorDeserializeCpuTime" : 56827000,
+        "executorRunTime" : 1619,
+        "executorCpuTime" : 496683000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 66,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 70,
+      "gettingResultTime" : 0
+    },
+    "34" : {
+      "taskId" : 34,
+      "index" : 2,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.087GMT",
+      "duration" : 1939,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 265,
+        "executorDeserializeCpuTime" : 69492000,
+        "executorRunTime" : 1606,
+        "executorCpuTime" : 508433000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 66,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 68,
+      "gettingResultTime" : 0
+    },
+    "45" : {
+      "taskId" : 45,
+      "index" : 13,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.129GMT",
+      "duration" : 1895,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 221,
+        "executorDeserializeCpuTime" : 54222000,
+        "executorRunTime" : 1595,
+        "executorCpuTime" : 495138000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 62,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 79,
+      "gettingResultTime" : 0
+    },
+    "44" : {
+      "taskId" : 44,
+      "index" : 12,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.126GMT",
+      "duration" : 1917,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 222,
+        "executorDeserializeCpuTime" : 51988000,
+        "executorRunTime" : 1624,
+        "executorCpuTime" : 498187000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 66,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 71,
+      "gettingResultTime" : 0
+    },
+    "39" : {
+      "taskId" : 39,
+      "index" : 7,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.109GMT",
+      "duration" : 1915,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 254,
+        "executorDeserializeCpuTime" : 64380000,
+        "executorRunTime" : 1596,
+        "executorCpuTime" : 539451000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 62,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 65,
+      "gettingResultTime" : 0
+    },
+    "35" : {
+      "taskId" : 35,
+      "index" : 3,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.091GMT",
+      "duration" : 1925,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 263,
+        "executorDeserializeCpuTime" : 62944000,
+        "executorRunTime" : 1598,
+        "executorCpuTime" : 502908000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 62,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 64,
+      "gettingResultTime" : 0
+    },
+    "43" : {
+      "taskId" : 43,
+      "index" : 11,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.123GMT",
+      "duration" : 1906,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 225,
+        "executorDeserializeCpuTime" : 48849000,
+        "executorRunTime" : 1609,
+        "executorCpuTime" : 502120000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 66,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 72,
+      "gettingResultTime" : 0
+    },
+    "40" : {
+      "taskId" : 40,
+      "index" : 8,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.112GMT",
+      "duration" : 1904,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 246,
+        "executorDeserializeCpuTime" : 69760000,
+        "executorRunTime" : 1595,
+        "executorCpuTime" : 510597000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 62,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 63,
+      "gettingResultTime" : 0
+    },
+    "36" : {
+      "taskId" : 36,
+      "index" : 4,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.095GMT",
+      "duration" : 1920,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 260,
+        "executorDeserializeCpuTime" : 112849000,
+        "executorRunTime" : 1596,
+        "executorCpuTime" : 503010000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 62,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 64,
+      "gettingResultTime" : 0
+    },
+    "47" : {
+      "taskId" : 47,
+      "index" : 15,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.136GMT",
+      "duration" : 1878,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 213,
+        "executorDeserializeCpuTime" : 47496000,
+        "executorRunTime" : 1594,
+        "executorCpuTime" : 1324251000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 52,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 71,
+      "gettingResultTime" : 0
+    }
+  },
+  "executorSummary" : {
+    "0" : {
+      "taskTime" : 30596,
+      "failedTasks" : 0,
+      "succeededTasks" : 16,
+      "killedTasks" : 0,
+      "inputBytes" : 0,
+      "inputRecords" : 0,
+      "outputBytes" : 0,
+      "outputRecords" : 0,
+      "shuffleRead" : 0,
+      "shuffleReadRecords" : 0,
+      "shuffleWrite" : 0,
+      "shuffleWriteRecords" : 0,
+      "memoryBytesSpilled" : 0,
+      "diskBytesSpilled" : 0,
+      "isBlacklistedForStage" : false,
+      "peakMemoryMetrics" : {
+        "JVMHeapMemory" : 0,
+        "JVMOffHeapMemory" : 0,
+        "OnHeapExecutionMemory" : 0,
+        "OffHeapExecutionMemory" : 0,
+        "OnHeapStorageMemory" : 0,
+        "OffHeapStorageMemory" : 0,
+        "OnHeapUnifiedMemory" : 0,
+        "OffHeapUnifiedMemory" : 0,
+        "DirectPoolMemory" : 0,
+        "MappedPoolMemory" : 0,
+        "ProcessTreeJVMVMemory" : 0,
+        "ProcessTreeJVMRSSMemory" : 0,
+        "ProcessTreePythonVMemory" : 0,
+        "ProcessTreePythonRSSMemory" : 0,
+        "ProcessTreeOtherVMemory" : 0,
+        "ProcessTreeOtherRSSMemory" : 0,
+        "MinorGCCount" : 0,
+        "MinorGCTime" : 0,
+        "MajorGCCount" : 0,
+        "MajorGCTime" : 0
+      },
+      "isExcludedForStage" : false
+    },
+    "driver" : {
+      "taskTime" : 0,
+      "failedTasks" : 0,
+      "succeededTasks" : 0,
+      "killedTasks" : 0,
+      "inputBytes" : 0,
+      "inputRecords" : 0,
+      "outputBytes" : 0,
+      "outputRecords" : 0,
+      "shuffleRead" : 0,
+      "shuffleReadRecords" : 0,
+      "shuffleWrite" : 0,
+      "shuffleWriteRecords" : 0,
+      "memoryBytesSpilled" : 0,
+      "diskBytesSpilled" : 0,
+      "isBlacklistedForStage" : false,
+      "peakMemoryMetrics" : {
+        "JVMHeapMemory" : 213367864,
+        "JVMOffHeapMemory" : 189011656,
+        "OnHeapExecutionMemory" : 0,
+        "OffHeapExecutionMemory" : 0,
+        "OnHeapStorageMemory" : 2133349,
+        "OffHeapStorageMemory" : 0,
+        "OnHeapUnifiedMemory" : 2133349,
+        "OffHeapUnifiedMemory" : 0,
+        "DirectPoolMemory" : 282024,
+        "MappedPoolMemory" : 0,
+        "ProcessTreeJVMVMemory" : 0,
+        "ProcessTreeJVMRSSMemory" : 0,
+        "ProcessTreePythonVMemory" : 0,
+        "ProcessTreePythonRSSMemory" : 0,
+        "ProcessTreeOtherVMemory" : 0,
+        "ProcessTreeOtherRSSMemory" : 0,
+        "MinorGCCount" : 13,
+        "MinorGCTime" : 115,
+        "MajorGCCount" : 4,
+        "MajorGCTime" : 339
+      },
+      "isExcludedForStage" : false
+    }
+  },
+  "killedTasksSummary" : { },
+  "resourceProfileId" : 0,
+  "peakExecutorMetrics" : {
+    "JVMHeapMemory" : 213367864,
+    "JVMOffHeapMemory" : 189011656,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 2133349,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 2133349,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 282024,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 13,
+    "MinorGCTime" : 115,
+    "MajorGCCount" : 4,
+    "MajorGCTime" : 339
+  }
+}
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json
new file mode 100644
index 0000000000000..f200da37e860e
--- /dev/null
+++ b/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json
@@ -0,0 +1,1078 @@
+{
+  "status" : "COMPLETE",
+  "stageId" : 2,
+  "attemptId" : 0,
+  "numTasks" : 16,
+  "numActiveTasks" : 0,
+  "numCompleteTasks" : 16,
+  "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
+  "numCompletedIndices" : 16,
+  "submissionTime" : "2020-07-07T03:11:21.040GMT",
+  "firstTaskLaunchedTime" : "2020-07-07T03:11:21.077GMT",
+  "completionTime" : "2020-07-07T03:11:23.044GMT",
+  "executorDeserializeTime" : 3905,
+  "executorDeserializeCpuTime" : 979900000,
+  "executorRunTime" : 25579,
+  "executorCpuTime" : 8810338000,
+  "resultSize" : 33883,
+  "jvmGcTime" : 1010,
+  "resultSerializationTime" : 11,
+  "memoryBytesSpilled" : 0,
+  "diskBytesSpilled" : 0,
+  "peakExecutionMemory" : 384640,
+  "inputBytes" : 0,
+  "inputRecords" : 0,
+  "outputBytes" : 0,
+  "outputRecords" : 0,
+  "shuffleRemoteBlocksFetched" : 0,
+  "shuffleLocalBlocksFetched" : 0,
+  "shuffleFetchWaitTime" : 0,
+  "shuffleRemoteBytesRead" : 0,
+  "shuffleRemoteBytesReadToDisk" : 0,
+  "shuffleLocalBytesRead" : 0,
+  "shuffleReadBytes" : 0,
+  "shuffleReadRecords" : 0,
+  "shuffleWriteBytes" : 0,
+  "shuffleWriteTime" : 0,
+  "shuffleWriteRecords" : 0,
+  "name" : "foreach at <console>:26",
+  "details" : "org.apache.spark.sql.Dataset.foreach(Dataset.scala:2862)\n$line19.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:26)\n$line19.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:30)\n$line19.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:32)\n$line19.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:34)\n$line19.$read$$iw$$iw$$iw$$iw.<init>(<console>:36)\n$line19.$read$$iw$$iw$$iw.<init>(<console>:38)\n$line19.$read$$iw$$iw.<init>(<console>:40)\n$line19.$read$$iw.<init>(<console>:42)\n$line19.$read.<init>(<console>:44)\n$line19.$read$.<init>(<console>:48)\n$line19.$read$.<clinit>(<console>)\n$line19.$eval$.$print$lzycompute(<console>:7)\n$line19.$eval$.$print(<console>:6)\n$line19.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)",
+  "schedulingPool" : "default",
+  "rddIds" : [ 10, 8, 6, 7, 9 ],
+  "accumulatorUpdates" : [ ],
+  "tasks" : {
+    "42" : {
+      "taskId" : 42,
+      "index" : 10,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.120GMT",
+      "duration" : 1923,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 229,
+        "executorDeserializeCpuTime" : 58152000,
+        "executorRunTime" : 1624,
+        "executorCpuTime" : 508230000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 66,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 70,
+      "gettingResultTime" : 0
+    },
+    "37" : {
+      "taskId" : 37,
+      "index" : 5,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.100GMT",
+      "duration" : 1915,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 256,
+        "executorDeserializeCpuTime" : 60890000,
+        "executorRunTime" : 1596,
+        "executorCpuTime" : 507192000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 62,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 63,
+      "gettingResultTime" : 0
+    },
+    "46" : {
+      "taskId" : 46,
+      "index" : 14,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.132GMT",
+      "duration" : 1905,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 218,
+        "executorDeserializeCpuTime" : 51464000,
+        "executorRunTime" : 1618,
+        "executorCpuTime" : 490927000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 66,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 69,
+      "gettingResultTime" : 0
+    },
+    "38" : {
+      "taskId" : 38,
+      "index" : 6,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.104GMT",
+      "duration" : 1835,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 255,
+        "executorDeserializeCpuTime" : 60358000,
+        "executorRunTime" : 1498,
+        "executorCpuTime" : 414110000,
+        "resultSize" : 2158,
+        "jvmGcTime" : 62,
+        "resultSerializationTime" : 11,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 71,
+      "gettingResultTime" : 0
+    },
+    "33" : {
+      "taskId" : 33,
+      "index" : 1,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.082GMT",
+      "duration" : 1943,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 267,
+        "executorDeserializeCpuTime" : 54442000,
+        "executorRunTime" : 1597,
+        "executorCpuTime" : 519178000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 62,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 79,
+      "gettingResultTime" : 0
+    },
+    "41" : {
+      "taskId" : 41,
+      "index" : 9,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.116GMT",
+      "duration" : 1916,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 240,
+        "executorDeserializeCpuTime" : 55787000,
+        "executorRunTime" : 1614,
+        "executorCpuTime" : 489923000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 66,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 62,
+      "gettingResultTime" : 0
+    },
+    "32" : {
+      "taskId" : 32,
+      "index" : 0,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.077GMT",
+      "duration" : 1960,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 271,
+        "executorDeserializeCpuTime" : 56827000,
+        "executorRunTime" : 1619,
+        "executorCpuTime" : 496683000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 66,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 70,
+      "gettingResultTime" : 0
+    },
+    "34" : {
+      "taskId" : 34,
+      "index" : 2,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.087GMT",
+      "duration" : 1939,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 265,
+        "executorDeserializeCpuTime" : 69492000,
+        "executorRunTime" : 1606,
+        "executorCpuTime" : 508433000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 66,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 68,
+      "gettingResultTime" : 0
+    },
+    "45" : {
+      "taskId" : 45,
+      "index" : 13,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.129GMT",
+      "duration" : 1895,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 221,
+        "executorDeserializeCpuTime" : 54222000,
+        "executorRunTime" : 1595,
+        "executorCpuTime" : 495138000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 62,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 79,
+      "gettingResultTime" : 0
+    },
+    "44" : {
+      "taskId" : 44,
+      "index" : 12,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.126GMT",
+      "duration" : 1917,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 222,
+        "executorDeserializeCpuTime" : 51988000,
+        "executorRunTime" : 1624,
+        "executorCpuTime" : 498187000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 66,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 71,
+      "gettingResultTime" : 0
+    },
+    "39" : {
+      "taskId" : 39,
+      "index" : 7,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.109GMT",
+      "duration" : 1915,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 254,
+        "executorDeserializeCpuTime" : 64380000,
+        "executorRunTime" : 1596,
+        "executorCpuTime" : 539451000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 62,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 65,
+      "gettingResultTime" : 0
+    },
+    "35" : {
+      "taskId" : 35,
+      "index" : 3,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.091GMT",
+      "duration" : 1925,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 263,
+        "executorDeserializeCpuTime" : 62944000,
+        "executorRunTime" : 1598,
+        "executorCpuTime" : 502908000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 62,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 64,
+      "gettingResultTime" : 0
+    },
+    "43" : {
+      "taskId" : 43,
+      "index" : 11,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.123GMT",
+      "duration" : 1906,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 225,
+        "executorDeserializeCpuTime" : 48849000,
+        "executorRunTime" : 1609,
+        "executorCpuTime" : 502120000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 66,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 72,
+      "gettingResultTime" : 0
+    },
+    "40" : {
+      "taskId" : 40,
+      "index" : 8,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.112GMT",
+      "duration" : 1904,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 246,
+        "executorDeserializeCpuTime" : 69760000,
+        "executorRunTime" : 1595,
+        "executorCpuTime" : 510597000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 62,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 63,
+      "gettingResultTime" : 0
+    },
+    "36" : {
+      "taskId" : 36,
+      "index" : 4,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.095GMT",
+      "duration" : 1920,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 260,
+        "executorDeserializeCpuTime" : 112849000,
+        "executorRunTime" : 1596,
+        "executorCpuTime" : 503010000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 62,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 64,
+      "gettingResultTime" : 0
+    },
+    "47" : {
+      "taskId" : 47,
+      "index" : 15,
+      "attempt" : 0,
+      "launchTime" : "2020-07-07T03:11:21.136GMT",
+      "duration" : 1878,
+      "executorId" : "0",
+      "host" : "127.0.0.1",
+      "status" : "SUCCESS",
+      "taskLocality" : "PROCESS_LOCAL",
+      "speculative" : false,
+      "accumulatorUpdates" : [ ],
+      "taskMetrics" : {
+        "executorDeserializeTime" : 213,
+        "executorDeserializeCpuTime" : 47496000,
+        "executorRunTime" : 1594,
+        "executorCpuTime" : 1324251000,
+        "resultSize" : 2115,
+        "jvmGcTime" : 52,
+        "resultSerializationTime" : 0,
+        "memoryBytesSpilled" : 0,
+        "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 24040,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "remoteBytesReadToDisk" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
+      },
+      "executorLogs" : {
+        "stdout" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout",
+        "stderr" : "http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"
+      },
+      "schedulerDelay" : 71,
+      "gettingResultTime" : 0
+    }
+  },
+  "executorSummary" : {
+    "0" : {
+      "taskTime" : 30596,
+      "failedTasks" : 0,
+      "succeededTasks" : 16,
+      "killedTasks" : 0,
+      "inputBytes" : 0,
+      "inputRecords" : 0,
+      "outputBytes" : 0,
+      "outputRecords" : 0,
+      "shuffleRead" : 0,
+      "shuffleReadRecords" : 0,
+      "shuffleWrite" : 0,
+      "shuffleWriteRecords" : 0,
+      "memoryBytesSpilled" : 0,
+      "diskBytesSpilled" : 0,
+      "isBlacklistedForStage" : false,
+      "peakMemoryMetrics" : {
+        "JVMHeapMemory" : 0,
+        "JVMOffHeapMemory" : 0,
+        "OnHeapExecutionMemory" : 0,
+        "OffHeapExecutionMemory" : 0,
+        "OnHeapStorageMemory" : 0,
+        "OffHeapStorageMemory" : 0,
+        "OnHeapUnifiedMemory" : 0,
+        "OffHeapUnifiedMemory" : 0,
+        "DirectPoolMemory" : 0,
+        "MappedPoolMemory" : 0,
+        "ProcessTreeJVMVMemory" : 0,
+        "ProcessTreeJVMRSSMemory" : 0,
+        "ProcessTreePythonVMemory" : 0,
+        "ProcessTreePythonRSSMemory" : 0,
+        "ProcessTreeOtherVMemory" : 0,
+        "ProcessTreeOtherRSSMemory" : 0,
+        "MinorGCCount" : 0,
+        "MinorGCTime" : 0,
+        "MajorGCCount" : 0,
+        "MajorGCTime" : 0
+      },
+      "isExcludedForStage" : false
+    },
+    "driver" : {
+      "taskTime" : 0,
+      "failedTasks" : 0,
+      "succeededTasks" : 0,
+      "killedTasks" : 0,
+      "inputBytes" : 0,
+      "inputRecords" : 0,
+      "outputBytes" : 0,
+      "outputRecords" : 0,
+      "shuffleRead" : 0,
+      "shuffleReadRecords" : 0,
+      "shuffleWrite" : 0,
+      "shuffleWriteRecords" : 0,
+      "memoryBytesSpilled" : 0,
+      "diskBytesSpilled" : 0,
+      "isBlacklistedForStage" : false,
+      "peakMemoryMetrics" : {
+        "JVMHeapMemory" : 213367864,
+        "JVMOffHeapMemory" : 189011656,
+        "OnHeapExecutionMemory" : 0,
+        "OffHeapExecutionMemory" : 0,
+        "OnHeapStorageMemory" : 2133349,
+        "OffHeapStorageMemory" : 0,
+        "OnHeapUnifiedMemory" : 2133349,
+        "OffHeapUnifiedMemory" : 0,
+        "DirectPoolMemory" : 282024,
+        "MappedPoolMemory" : 0,
+        "ProcessTreeJVMVMemory" : 0,
+        "ProcessTreeJVMRSSMemory" : 0,
+        "ProcessTreePythonVMemory" : 0,
+        "ProcessTreePythonRSSMemory" : 0,
+        "ProcessTreeOtherVMemory" : 0,
+        "ProcessTreeOtherRSSMemory" : 0,
+        "MinorGCCount" : 13,
+        "MinorGCTime" : 115,
+        "MajorGCCount" : 4,
+        "MajorGCTime" : 339
+      },
+      "isExcludedForStage" : false
+    }
+  },
+  "killedTasksSummary" : { },
+  "resourceProfileId" : 0,
+  "peakExecutorMetrics" : {
+    "JVMHeapMemory" : 213367864,
+    "JVMOffHeapMemory" : 189011656,
+    "OnHeapExecutionMemory" : 0,
+    "OffHeapExecutionMemory" : 0,
+    "OnHeapStorageMemory" : 2133349,
+    "OffHeapStorageMemory" : 0,
+    "OnHeapUnifiedMemory" : 2133349,
+    "OffHeapUnifiedMemory" : 0,
+    "DirectPoolMemory" : 282024,
+    "MappedPoolMemory" : 0,
+    "ProcessTreeJVMVMemory" : 0,
+    "ProcessTreeJVMRSSMemory" : 0,
+    "ProcessTreePythonVMemory" : 0,
+    "ProcessTreePythonRSSMemory" : 0,
+    "ProcessTreeOtherVMemory" : 0,
+    "ProcessTreeOtherRSSMemory" : 0,
+    "MinorGCCount" : 13,
+    "MinorGCTime" : 115,
+    "MajorGCCount" : 4,
+    "MajorGCTime" : 339
+  },
+  "taskMetricsDistributions" : {
+    "quantiles" : [ 0.0, 0.25, 0.5, 0.75, 1.0 ],
+    "duration" : [ 1835.0, 1905.0, 1916.0, 1925.0, 1960.0 ],
+    "executorDeserializeTime" : [ 213.0, 225.0, 254.0, 263.0, 271.0 ],
+    "executorDeserializeCpuTime" : [ 4.7496E7, 5.4222E7, 5.8152E7, 6.438E7, 1.12849E8 ],
+    "executorRunTime" : [ 1498.0, 1596.0, 1598.0, 1618.0, 1624.0 ],
+    "executorCpuTime" : [ 4.1411E8, 4.96683E8, 5.0301E8, 5.10597E8, 1.324251E9 ],
+    "resultSize" : [ 2115.0, 2115.0, 2115.0, 2115.0, 2158.0 ],
+    "jvmGcTime" : [ 52.0, 62.0, 62.0, 66.0, 66.0 ],
+    "resultSerializationTime" : [ 0.0, 0.0, 0.0, 0.0, 11.0 ],
+    "gettingResultTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "schedulerDelay" : [ 62.0, 64.0, 70.0, 71.0, 79.0 ],
+    "peakExecutionMemory" : [ 24040.0, 24040.0, 24040.0, 24040.0, 24040.0 ],
+    "memoryBytesSpilled" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "diskBytesSpilled" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "inputMetrics" : {
+      "bytesRead" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+      "recordsRead" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ]
+    },
+    "outputMetrics" : {
+      "bytesWritten" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+      "recordsWritten" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ]
+    },
+    "shuffleReadMetrics" : {
+      "readBytes" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+      "readRecords" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+      "remoteBlocksFetched" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+      "localBlocksFetched" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+      "fetchWaitTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+      "remoteBytesRead" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+      "remoteBytesReadToDisk" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+      "totalBlocksFetched" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ]
+    },
+    "shuffleWriteMetrics" : {
+      "writeBytes" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+      "writeRecords" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+      "writeTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ]
+    }
+  },
+  "executorMetricsDistributions" : {
+    "quantiles" : [ 0.0, 0.25, 0.5, 0.75, 1.0 ],
+    "taskTime" : [ 0.0, 0.0, 30596.0, 30596.0, 30596.0 ],
+    "failedTasks" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "succeededTasks" : [ 0.0, 0.0, 16.0, 16.0, 16.0 ],
+    "killedTasks" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "inputBytes" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "inputRecords" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "outputBytes" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "outputRecords" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "shuffleRead" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "shuffleReadRecords" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "shuffleWrite" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "shuffleWriteRecords" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "memoryBytesSpilled" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "diskBytesSpilled" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "peakMemoryMetrics" : {
+      "JVMHeapMemory" : [ 0.0, 0.0, 2.13367864E8, 2.13367864E8, 2.13367864E8 ],
+      "JVMOffHeapMemory" : [ 0.0, 0.0, 1.89011656E8, 1.89011656E8, 1.89011656E8 ],
+      "OnHeapExecutionMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+      "OffHeapExecutionMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+      "OnHeapStorageMemory" : [ 0.0, 0.0, 2133349.0, 2133349.0, 2133349.0 ],
+      "OffHeapStorageMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+      "OnHeapUnifiedMemory" : [ 0.0, 0.0, 2133349.0, 2133349.0, 2133349.0 ],
+      "OffHeapUnifiedMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+      "DirectPoolMemory" : [ 0.0, 0.0, 282024.0, 282024.0, 282024.0 ],
+      "MappedPoolMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+      "ProcessTreeJVMVMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+      "ProcessTreeJVMRSSMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+      "ProcessTreePythonVMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+      "ProcessTreePythonRSSMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+      "ProcessTreeOtherVMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+      "ProcessTreeOtherRSSMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+      "MinorGCCount" : [ 0.0, 0.0, 13.0, 13.0, 13.0 ],
+      "MinorGCTime" : [ 0.0, 0.0, 115.0, 115.0, 115.0 ],
+      "MajorGCCount" : [ 0.0, 0.0, 4.0, 4.0, 4.0 ],
+      "MajorGCTime" : [ 0.0, 0.0, 339.0, 339.0, 339.0 ]
+    }
+  }
+}
diff --git a/core/src/test/resources/hive-site.xml b/core/src/test/resources/hive-site.xml
new file mode 100644
index 0000000000000..d7117c3f20248
--- /dev/null
+++ b/core/src/test/resources/hive-site.xml
@@ -0,0 +1,34 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<configuration>
+    <property>
+        <name>hive.in.test</name>
+        <value>true</value>
+        <description>Internal marker for test.</description>
+    </property>
+    <property>
+        <name>hadoop.tmp.dir</name>
+        <value>/tmp/hive_one</value>
+        <description>default is /tmp/hadoop-${user.name} and will be overridden</description>
+    </property>
+
+    <property>
+        <name>io.file.buffer.size</name>
+        <value>201811</value>
+    </property>
+</configuration>
diff --git a/core/src/test/resources/spark-events/app-20200706201101-0003 b/core/src/test/resources/spark-events/app-20200706201101-0003
new file mode 100644
index 0000000000000..b2923ca0f001f
--- /dev/null
+++ b/core/src/test/resources/spark-events/app-20200706201101-0003
@@ -0,0 +1,124 @@
+{"Event":"SparkListenerLogStart","Spark Version":"3.1.0-SNAPSHOT"}
+{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"cores":{"Resource Name":"cores","Amount":1,"Discovery Script":"","Vendor":""},"memory":{"Resource Name":"memory","Amount":1024,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}}
+{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"localhost","Port":64415},"Maximum Memory":384093388,"Timestamp":1594091461118,"Maximum Onheap Memory":384093388,"Maximum Offheap Memory":0}
+{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/Library/Java/JavaVirtualMachines/jdk1.8.0_231.jdk/Contents/Home/jre","Java Version":"1.8.0_231 (Oracle Corporation)","Scala Version":"version 2.12.10"},"Spark Properties":{"spark.driver.host":"localhost","spark.eventLog.enabled":"true","spark.driver.port":"64413","spark.repl.class.uri":"spark://localhost:64413/classes","spark.jars":"","spark.repl.class.outputDir":"/private/var/folders/p_/5j1mtw1x0579b34vy4ztcbs40000gn/T/spark-0c713088-21d7-4b35-9b52-7a4dd4d15ae1/repl-361cef81-42cd-4875-a8a6-ce9dfe55682a","spark.app.name":"Spark shell","spark.scheduler.mode":"FIFO","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.executor.id":"driver","spark.eventLog.logStageExecutorMetrics":"true","spark.submit.deployMode":"client","spark.master":"*********(redacted)","spark.home":"*********(redacted)","spark.sql.catalogImplementation":"in-memory","spark.app.id":"app-20200706201101-0003"},"Hadoop Properties":{"hadoop.service.shutdown.timeout":"30s","yarn.resourcemanager.amlauncher.thread-count":"50","yarn.sharedcache.enabled":"false","fs.s3a.connection.maximum":"15","yarn.nodemanager.numa-awareness.numactl.cmd":"/usr/bin/numactl","fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","yarn.timeline-service.timeline-client.number-of-async-entities-to-merge":"10","hadoop.security.kms.client.timeout":"60","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.jobhistory.loadedjob.tasks.max":"-1","mapreduce.framework.name":"local","yarn.sharedcache.uploader.server.thread-count":"50","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"30","hadoop.security.groups.cache.background.reload.threads":"3","yarn.resourcemanager.webapp.cross-origin.enabled":"false","fs.AbstractFileSystem.ftp.impl":"org.apache.hadoop.fs.ftp.FtpFs","hadoop.registry.secure":"false","hadoop.shell.safely.delete.limit.num.files":"100","dfs.bytes-per-checksum":"512","mapreduce.job.acl-view-job":" ","fs.s3a.s3guard.ddb.background.sleep":"25ms","fs.s3a.retry.limit":"${fs.s3a.attempts.maximum}","mapreduce.jobhistory.loadedjobs.cache.size":"5","fs.s3a.s3guard.ddb.table.create":"false","yarn.nodemanager.amrmproxy.enabled":"false","yarn.timeline-service.entity-group-fs-store.with-user-dir":"false","mapreduce.input.fileinputformat.split.minsize":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","yarn.sharedcache.checksum.algo.impl":"org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl","yarn.nodemanager.amrmproxy.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor","yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size":"10485760","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","mapreduce.task.profile.maps":"0-2","yarn.scheduler.include-port-in-node-name":"false","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","yarn.resourcemanager.node-removal-untracked.timeout-ms":"60000","mapreduce.am.max-attempts":"2","hadoop.security.kms.client.failover.sleep.base.millis":"100","mapreduce.jobhistory.webapp.https.address":"0.0.0.0:19890","yarn.node-labels.fs-store.impl.class":"org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore","yarn.nodemanager.collector-service.address":"${yarn.nodemanager.hostname}:8048","fs.trash.checkpoint.interval":"0","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","yarn.resourcemanager.node-ip-cache.expiry-interval-secs":"-1","hadoop.http.authentication.signature.secret.file":"*********(redacted)","hadoop.jetty.logs.serve.aliases":"true","yarn.resourcemanager.placement-constraints.handler":"disabled","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"1000","yarn.resourcemanager.system-metrics-publisher.enabled":"false","yarn.resourcemanager.placement-constraints.algorithm.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm","yarn.sharedcache.webapp.address":"0.0.0.0:8788","yarn.resourcemanager.delegation.token.renew-interval":"*********(redacted)","yarn.sharedcache.nm.uploader.replication.factor":"10","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","mapreduce.reduce.skip.proc-count.auto-incr":"true","hadoop.security.group.mapping.ldap.posix.attr.gid.name":"gidNumber","ipc.client.fallback-to-simple-auth-allowed":"false","yarn.nodemanager.resource.memory.enforced":"true","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","yarn.acl.reservation-enable":"false","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","yarn.app.mapreduce.am.hard-kill-timeout-ms":"10000","fs.s3a.etag.checksum.enabled":"false","yarn.nodemanager.container-metrics.enable":"true","yarn.timeline-service.client.fd-clean-interval-secs":"60","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","hadoop.common.configuration.version":"3.0.0","fs.s3a.s3guard.ddb.table.capacity.read":"500","yarn.nodemanager.remote-app-log-dir-suffix":"logs","yarn.nodemanager.windows-container.cpu-limit.enabled":"false","yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed":"false","file.blocksize":"67108864","hadoop.registry.zk.retry.ceiling.ms":"60000","yarn.scheduler.configuration.leveldb-store.path":"${hadoop.tmp.dir}/yarn/system/confstore","yarn.sharedcache.store.in-memory.initial-delay-mins":"10","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","mapreduce.map.skip.proc-count.auto-incr":"true","fs.s3a.committer.name":"file","mapreduce.task.profile.reduces":"0-2","hadoop.zk.num-retries":"1000","yarn.webapp.xfs-filter.enabled":"true","seq.io.sort.mb":"100","yarn.scheduler.configuration.max.version":"100","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"${yarn.resourcemanager.hostname}:8030","yarn.node-labels.enabled":"false","yarn.resourcemanager.webapp.ui-actions.enabled":"true","mapreduce.task.timeout":"600000","yarn.sharedcache.client-server.thread-count":"50","hadoop.security.groups.shell.command.timeout":"0s","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.nodemanager.elastic-memory-control.oom-handler":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.DefaultOOMHandler","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"file:///","yarn.minicluster.use-rpc":"false","fs.har.impl.disable.cache":"true","yarn.webapp.ui2.enable":"false","io.compression.codec.bzip2.library":"system-native","yarn.nodemanager.distributed-scheduling.enabled":"false","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"${yarn.resourcemanager.hostname}:8090","mapreduce.jobhistory.address":"0.0.0.0:10020","yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.is.minicluster":"false","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","fs.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A","mapreduce.task.combine.progress.records":"10000","yarn.resourcemanager.epoch.range":"0","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","fs.AbstractFileSystem.wasbs.impl":"org.apache.hadoop.fs.azure.Wasbs","yarn.timeline-service.entity-group-fs-store.cache-store-class":"org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore","fs.ftp.transfer.mode":"BLOCK_TRANSFER_MODE","ipc.server.log.slow.rpc":"false","yarn.resourcemanager.node-labels.provider.fetch-interval-ms":"1800000","yarn.router.webapp.https.address":"0.0.0.0:8091","yarn.nodemanager.webapp.cross-origin.enabled":"false","fs.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","yarn.resourcemanager.auto-update.containers":"false","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","yarn.scheduler.configuration.zk-store.parent-path":"/confstore","yarn.nodemanager.default-container-executor.log-dirs.permissions":"710","yarn.app.attempt.diagnostics.limit.kc":"64","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"-1","fs.AbstractFileSystem.abfs.impl":"org.apache.hadoop.fs.azurebfs.Abfs","yarn.timeline-service.writer.flush-interval-seconds":"60","fs.s3a.fast.upload.active.blocks":"4","hadoop.security.credential.clear-text-fallback":"true","yarn.nodemanager.collector-service.thread-count":"5","fs.azure.secure.mode":"false","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","yarn.resourcemanager.fs.state-store.num-retries":"0","yarn.resourcemanager.nodemanager-connect-retries":"10","yarn.nodemanager.log-aggregation.num-log-files-per-app":"30","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","fs.s3a.committer.magic.enabled":"false","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","yarn.system-metrics-publisher.enabled":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","mapreduce.task.io.sort.factor":"10","yarn.nodemanager.amrmproxy.client.thread-count":"25","ha.failover-controller.new-active.rpc-timeout.ms":"60000","yarn.nodemanager.container-localizer.java.opts":"-Xmx256m","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","yarn.nodemanager.windows-container.memory-limit.enabled":"false","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.minicluster.fixed.ports":"false","hadoop.tags.system":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n      ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.cluster.max-application-priority":"0","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","hadoop.caller.context.signature.max.size":"40","yarn.client.load.resource-types.from-server":"false","ha.zookeeper.session-timeout.ms":"10000","tfile.io.chunk.size":"1048576","fs.s3a.s3guard.ddb.table.capacity.write":"100","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.kms.client.failover.sleep.max.millis":"2000","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","yarn.scheduler.configuration.store.max-logs":"1000","yarn.nodemanager.node-attributes.provider.fetch-interval-ms":"600000","fs.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","yarn.nodemanager.local-cache.max-files-per-directory":"8192","hadoop.http.cross-origin.enabled":"false","hadoop.zk.acl":"world:anyone:rwcda","mapreduce.map.sort.spill.percent":"0.80","yarn.timeline-service.entity-group-fs-store.scan-interval-seconds":"60","yarn.node-attribute.fs-store.impl.class":"org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore","fs.s3a.retry.interval":"500ms","yarn.timeline-service.client.best-effort":"false","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","hadoop.security.group.mapping.ldap.posix.attr.uid.name":"uidNumber","fs.AbstractFileSystem.swebhdfs.impl":"org.apache.hadoop.fs.SWebHdfs","yarn.nodemanager.elastic-memory-control.timeout-sec":"5","mapreduce.ifile.readahead":"true","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","yarn.timeline-service.reader.webapp.address":"${yarn.timeline-service.webapp.address}","yarn.resourcemanager.placement-constraints.algorithm.pool-size":"1","yarn.timeline-service.hbase.coprocessor.jar.hdfs.location":"/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler","yarn.app.mapreduce.am.command-opts":"-Xmx1024m","mapreduce.cluster.local.dir":"${hadoop.tmp.dir}/mapred/local","io.mapfile.bloom.error.rate":"0.005","fs.client.resolve.topology.enabled":"false","yarn.nodemanager.runtime.linux.allowed-runtimes":"default","yarn.sharedcache.store.class":"org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","fs.s3a.metadatastore.impl":"org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore","io.skip.checksum.errors":"false","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","yarn.timeline-service.webapp.xfs-filter.xframe-options":"SAMEORIGIN","fs.s3a.connection.timeout":"200000","mapreduce.job.max.split.locations":"15","yarn.resourcemanager.nm-container-queuing.max-queue-length":"15","hadoop.registry.zk.session.timeout.ms":"60000","yarn.federation.cache-ttl.secs":"300","mapreduce.jvm.system-properties-to-log":"os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name","yarn.resourcemanager.opportunistic-container-allocation.nodes-used":"10","yarn.timeline-service.entity-group-fs-store.active-dir":"/tmp/entity-file-history/active","mapreduce.shuffle.transfer.buffer.size":"131072","yarn.timeline-service.client.retry-interval-ms":"1000","yarn.http.policy":"HTTP_ONLY","fs.s3a.socket.send.buffer":"8192","fs.AbstractFileSystem.abfss.impl":"org.apache.hadoop.fs.azurebfs.Abfss","yarn.sharedcache.uploader.server.address":"0.0.0.0:8046","yarn.resourcemanager.delegation-token.max-conf-size-bytes":"*********(redacted)","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","yarn.minicluster.yarn.nodemanager.resource.memory-mb":"4096","mapreduce.job.emit-timeline-data":"false","yarn.nodemanager.resource.system-reserved-memory-mb":"-1","hadoop.kerberos.min.seconds.before.relogin":"60","mapreduce.jobhistory.move.thread-count":"3","yarn.resourcemanager.admin.client.thread-count":"1","yarn.dispatcher.drain-events.timeout":"300000","fs.s3a.buffer.dir":"${hadoop.tmp.dir}/s3a","hadoop.ssl.enabled.protocols":"TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2","mapreduce.jobhistory.admin.address":"0.0.0.0:10033","yarn.log-aggregation-status.time-out.ms":"600000","fs.s3a.assumed.role.sts.endpoint.region":"us-west-1","mapreduce.shuffle.port":"13562","yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory":"10","yarn.nodemanager.health-checker.interval-ms":"600000","yarn.router.clientrm.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor","yarn.resourcemanager.zk-appid-node.split-index":"0","ftp.blocksize":"67108864","yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions":"read","yarn.router.rmadmin.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor","yarn.nodemanager.log-container-debug-info.enabled":"true","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms":"20","yarn.nodemanager.delete.debug-delay-sec":"0","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","yarn.nodemanager.container-retry-minimum-interval-ms":"1000","hadoop.security.groups.cache.secs":"300","yarn.federation.enabled":"false","fs.azure.local.sas.key.mode":"false","ipc.maximum.data.length":"67108864","mapreduce.shuffle.max.threads":"0","yarn.router.pipeline.cache-max-size":"25","yarn.resourcemanager.nm-container-queuing.load-comparator":"QUEUE_LENGTH","hadoop.security.authorization":"false","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","yarn.nodemanager.amrmproxy.ha.enable":"false","mapreduce.jobhistory.http.policy":"HTTP_ONLY","yarn.sharedcache.store.in-memory.check-period-mins":"720","hadoop.security.group.mapping.ldap.ssl":"false","yarn.client.application-client-protocol.poll-interval-ms":"200","yarn.scheduler.configuration.leveldb-store.compaction-interval-secs":"86400","yarn.timeline-service.writer.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl","ha.zookeeper.parent-znode":"/hadoop-ha","yarn.nodemanager.log-aggregation.policy.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","yarn.resourcemanager.placement-constraints.scheduler.pool-size":"1","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","yarn.nodemanager.recovery.supervised":"false","yarn.sharedcache.admin.thread-count":"1","yarn.resourcemanager.ha.automatic-failover.enabled":"true","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.resourcemanager.address":"${yarn.resourcemanager.hostname}:8032","ipc.client.ping":"true","mapreduce.task.local-fs.write-limit.bytes":"-1","fs.adl.oauth2.access.token.provider.type":"*********(redacted)","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","yarn.nodemanager.resource-plugins.gpu.docker-plugin":"nvidia-docker-v1","hadoop.ssl.enabled":"false","fs.s3a.multipart.purge":"false","yarn.scheduler.configuration.store.class":"file","yarn.resourcemanager.nm-container-queuing.queue-limit-stdev":"1.0f","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","ipc.client.bind.wildcard.addr":"false","yarn.resourcemanager.webapp.rest-csrf.enabled":"false","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"10","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","yarn.nodemanager.runtime.linux.docker.allowed-container-networks":"host,none,bridge","yarn.nodemanager.node-labels.resync-interval-ms":"120000","hadoop.tmp.dir":"/tmp/hadoop-${user.name}","mapreduce.job.maps":"2","mapreduce.jobhistory.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","yarn.rm.system-metrics-publisher.emit-container-events":"false","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","fs.s3a.s3guard.ddb.throttle.retry.interval":"100ms","fs.AbstractFileSystem.wasb.impl":"org.apache.hadoop.fs.azure.Wasb","mapreduce.client.submit.file.replication":"10","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"2147483647","yarn.resourcemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold":"1","yarn.nodemanager.webapp.rest-csrf.enabled":"false","fs.ftp.host.port":"21","ipc.ping.interval":"60000","yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size":"10","yarn.resourcemanager.admin.address":"${yarn.resourcemanager.hostname}:8033","file.client-write-packet-size":"65536","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"true","hadoop.security.key.default.bitlength":"128","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","yarn.nodemanager.log.deletion-threads-count":"4","yarn.webapp.filter-entity-list-by-user":"false","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"100","yarn.nodemanager.localizer.client.thread-count":"5","io.erasurecode.codec.rs.rawcoders":"rs_native,rs_java","io.erasurecode.codec.rs-legacy.rawcoders":"rs-legacy_java","yarn.sharedcache.admin.address":"0.0.0.0:8047","yarn.resourcemanager.placement-constraints.algorithm.iterator":"SERIAL","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec","mapreduce.job.cache.limit.max-resources-mb":"0","fs.s3a.connection.ssl.enabled":"true","yarn.nodemanager.process-kill-wait.ms":"5000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","hadoop.workaround.non.threadsafe.getpwuid":"true","fs.df.interval":"60000","fs.s3a.multiobjectdelete.enable":"true","yarn.sharedcache.cleaner.resource-sleep-ms":"0","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","hadoop.shell.missing.defaultFs.warning":"false","io.file.buffer.size":"65536","hadoop.security.group.mapping.ldap.search.attr.member":"member","hadoop.security.random.device.file.path":"/dev/urandom","hadoop.security.sensitive-config-keys":"*********(redacted)","fs.s3a.s3guard.ddb.max.retries":"9","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","yarn.intermediate-data-encryption.enable":"false","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.nodemanager.container.stderr.pattern":"{*stderr*,*STDERR*}","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/tmp/hadoop-yarn/staging","mapreduce.reduce.shuffle.read.timeout":"180000","hadoop.http.cross-origin.max-age":"1800","io.erasurecode.codec.xor.rawcoders":"xor_native,xor_java","fs.s3a.connection.establish.timeout":"5000","mapreduce.job.running.map.limit":"0","yarn.minicluster.control-resource-monitoring":"false","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","yarn.federation.state-store.class":"org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore","mapreduce.reduce.log.level":"INFO","hadoop.security.dns.log-slow-lookups.threshold.ms":"1000","mapreduce.job.ubertask.enable":"false","adl.http.timeout":"-1","yarn.resourcemanager.placement-constraints.retry-attempts":"3","hadoop.caller.context.enabled":"false","yarn.nodemanager.vmem-pmem-ratio":"2.1","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","hadoop.zk.timeout-ms":"10000","fs.s3a.s3guard.cli.prune.age":"86400000","yarn.nodemanager.resource.pcores-vcores-multiplier":"1.0","yarn.nodemanager.runtime.linux.sandbox-mode":"disabled","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.s3a.committer.threads":"8","hadoop.zk.retry-interval-ms":"1000","hadoop.security.crypto.buffer.size":"8192","yarn.nodemanager.node-labels.provider.fetch-interval-ms":"600000","mapreduce.jobhistory.recovery.store.leveldb.path":"${hadoop.tmp.dir}/mapred/history/recoverystore","yarn.client.failover-retries-on-socket-timeouts":"0","yarn.nodemanager.resource.memory.enabled":"false","fs.azure.authorization.caching.enable":"true","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","mapreduce.job.finish-when-all-reducers-done":"true","hadoop.registry.jaas.context":"Client","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","io.map.index.interval":"128","yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms":"100","fs.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","mapreduce.job.counters.max":"120","mapreduce.jobhistory.webapp.rest-csrf.enabled":"false","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","yarn.timeline-service.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/timeline","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","hadoop.security.auth_to_local.mechanism":"hadoop","yarn.timeline-service.app-collector.linger-period.ms":"60000","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.reservation-system.planfollower.time-step":"1000","yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed":"true","yarn.webapp.api-service.enable":"false","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","fs.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","yarn.nodemanager.container.stderr.tail.bytes":"4096","hadoop.security.group.mapping.ldap.read.timeout.ms":"60000","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","mapreduce.outputcommitter.factory.scheme.s3a":"org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory","hadoop.security.groups.cache.background.reload":"false","yarn.nodemanager.container-monitor.enabled":"true","yarn.nodemanager.elastic-memory-control.enabled":"false","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","yarn.resourcemanager.rm.container-allocation.expiry-interval-ms":"600000","mapreduce.fileoutputcommitter.algorithm.version":"2","yarn.resourcemanager.work-preserving-recovery.enabled":"true","mapreduce.map.skip.maxrecords":"0","yarn.sharedcache.root-dir":"/sharedcache","fs.s3a.retry.throttle.limit":"${fs.s3a.attempts.maximum}","hadoop.http.authentication.type":"simple","mapreduce.job.cache.limit.max-resources":"0","mapreduce.task.userlog.limit.kb":"0","yarn.resourcemanager.scheduler.monitor.enable":"false","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","yarn.nodemanager.resource-monitor.interval-ms":"3000","yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices":"auto","mapreduce.job.sharedcache.mode":"disabled","yarn.nodemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.shuffle.listen.queue.size":"128","yarn.scheduler.configuration.mutation.acl-policy.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy","mapreduce.map.cpu.vcores":"1","yarn.log-aggregation.file-formats":"TFile","yarn.timeline-service.client.fd-retain-secs":"300","hadoop.user.group.static.mapping.overrides":"dr.who=;","fs.azure.sas.expiry.period":"90d","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"false","yarn.router.webapp.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST","yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage":"90.0","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","yarn.nodemanager.disk-validator":"basic","yarn.app.mapreduce.client.job.max-retries":"3","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","hadoop.security.group.mapping.ldap.connection.timeout.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","yarn.nodemanager.container-diagnostics-maximum-size":"10000","hadoop.registry.zk.retry.interval.ms":"1000","yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms":"1000","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","fs.s3a.assumed.role.session.duration":"30m","hadoop.security.group.mapping.ldap.conversion.rule":"none","hadoop.ssl.server.conf":"ssl-server.xml","fs.s3a.retry.throttle.interval":"1000ms","seq.io.sort.factor":"100","yarn.sharedcache.cleaner.initial-delay-mins":"10","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.nodemanager.runtime.linux.docker.capabilities":"CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE","yarn.acl.enable":"false","yarn.timeline-service.entity-group-fs-store.done-dir":"/tmp/entity-file-history/done/","mapreduce.task.profile":"false","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","mapreduce.jobhistory.always-scan-user-dir":"false","yarn.nodemanager.opportunistic-containers-use-pause-for-preemption":"false","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"nobody","yarn.timeline-service.reader.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold":"1","yarn.resourcemanager.configuration.file-system-based-store":"/yarn/conf","mapreduce.job.cache.limit.max-single-resource-mb":"0","yarn.nodemanager.runtime.linux.docker.stop.grace-period":"10","yarn.resourcemanager.resource-profiles.source-file":"resource-profiles.json","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"${yarn.log.dir}/userlogs","fs.automatic.close":"true","yarn.nodemanager.hostname":"0.0.0.0","yarn.nodemanager.resource.memory.cgroups.swappiness":"0","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","yarn.timeline-service.app-aggregation-interval-secs":"15","hadoop.security.group.mapping.ldap.search.filter.user":"(&(objectClass=user)(sAMAccountName={0}))","yarn.nodemanager.container-localizer.log.level":"INFO","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","mapreduce.job.ubertask.maxmaps":"9","fs.s3a.threads.keepalivetime":"60","mapreduce.jobhistory.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","yarn.resourcemanager.delegation.token.max-lifetime":"*********(redacted)","yarn.timeline-service.client.drain-entities.timeout.ms":"2000","yarn.nodemanager.resource-plugins.fpga.vendor-plugin.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin","yarn.timeline-service.entity-group-fs-store.summary-store":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.reduce.cpu.vcores":"1","mapreduce.job.encrypted-intermediate-data.buffer.kb":"128","fs.client.resolve.remote.symlinks":"true","yarn.nodemanager.webapp.https.address":"0.0.0.0:8044","hadoop.http.cross-origin.allowed-origins":"*","mapreduce.job.encrypted-intermediate-data":"false","yarn.timeline-service.entity-group-fs-store.retain-seconds":"604800","yarn.resourcemanager.metrics.runtime.buckets":"60,300,1440","yarn.timeline-service.generic-application-history.max-applications":"10000","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","yarn.node-labels.configuration-type":"centralized","fs.s3a.path.style.access":"false","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","yarn.sharedcache.store.in-memory.staleness-period-mins":"10080","fs.adl.impl":"org.apache.hadoop.fs.adl.AdlFileSystem","yarn.resourcemanager.nodemanager.minimum.version":"NONE","mapreduce.jobhistory.webapp.xfs-filter.xframe-options":"SAMEORIGIN","yarn.app.mapreduce.am.staging-dir.erasurecoding.enabled":"false","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","yarn.timeline-service.reader.webapp.https.address":"${yarn.timeline-service.webapp.https.address}","fs.ftp.data.connection.mode":"ACTIVE_LOCAL_DATA_CONNECTION_MODE","mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed":"true","yarn.scheduler.maximum-allocation-vcores":"4","hadoop.http.cross-origin.allowed-headers":"X-Requested-With,Content-Type,Accept,Origin","yarn.nodemanager.log-aggregation.compression-type":"none","yarn.timeline-service.version":"1.0f","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","mapreduce.reduce.maxattempts":"4","hadoop.security.dns.log-slow-lookups.enabled":"false","mapreduce.job.committer.setup.cleanup.needed":"true","mapreduce.job.running.reduce.limit":"0","ipc.maximum.response.length":"134217728","yarn.resourcemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","hadoop.caller.context.max.size":"128","yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed":"false","yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed":"false","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","fs.s3a.fast.upload.buffer":"disk","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","fs.s3a.max.total.tasks":"5","fs.s3a.readahead.range":"64K","hadoop.http.authentication.simple.anonymous.allowed":"true","fs.s3a.attempts.maximum":"20","hadoop.registry.zk.connection.timeout.ms":"15000","yarn.resourcemanager.delegation-token-renewer.thread-count":"*********(redacted)","yarn.nodemanager.health-checker.script.timeout-ms":"1200000","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","yarn.resourcemanager.resource-profiles.enabled":"false","yarn.timeline-service.hbase-schema.prefix":"prod.","fs.azure.authorization":"false","mapreduce.map.log.level":"INFO","yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs":"20","mapreduce.output.fileoutputformat.compress.type":"RECORD","yarn.resourcemanager.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.timeline-service.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.ifile.readahead.bytes":"4194304","yarn.sharedcache.app-checker.class":"org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"true","yarn.nodemanager.resource.detect-hardware-capabilities":"false","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","hadoop.security.group.mapping.ldap.search.group.hierarchy.levels":"0","yarn.resourcemanager.fs.state-store.retry-interval-ms":"1000","file.stream-buffer-size":"4096","yarn.resourcemanager.application-timeouts.monitor.interval-ms":"3000","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","mapreduce.map.speculative":"true","mapreduce.job.speculative.retry-after-speculate":"15000","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.app.mapreduce.am.container.log.backups":"0","yarn.app.mapreduce.am.log.level":"INFO","mapreduce.job.reduce.slowstart.completedmaps":"0.05","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices":"0,1","yarn.timeline-service.client.internal-timers-ttl-secs":"420","hadoop.http.logs.enabled":"true","fs.s3a.block.size":"32M","yarn.sharedcache.client-server.address":"0.0.0.0:8045","yarn.nodemanager.logaggregation.threadpool-size-max":"100","yarn.resourcemanager.hostname":"0.0.0.0","yarn.resourcemanager.delegation.key.update-interval":"86400000","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"-1","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","yarn.nodemanager.disk-health-checker.enable":"true","ipc.client.tcpnodelay":"true","ipc.client.rpc-timeout.ms":"0","yarn.nodemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","ipc.client.low-latency":"false","mapreduce.input.lineinputformat.linespermap":"1","yarn.router.interceptor.user.threadpool-size":"5","ipc.client.connect.max.retries.on.timeouts":"45","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.timeline-service.entity-group-fs-store.app-cache-size":"10","fs.s3a.socket.recv.buffer":"8192","yarn.resourcemanager.resource-tracker.address":"${yarn.resourcemanager.hostname}:8031","yarn.nodemanager.node-labels.provider.fetch-timeout-ms":"1200000","mapreduce.job.heap.memory-mb.ratio":"0.8","yarn.resourcemanager.leveldb-state-store.compaction-interval-secs":"3600","yarn.resourcemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","yarn.scheduler.configuration.fs.path":"file://${hadoop.tmp.dir}/yarn/system/schedconf","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"org.apache.hadoop.http.lib.StaticUserWebFilter","mapreduce.reduce.memory.mb":"-1","yarn.timeline-service.hostname":"0.0.0.0","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","yarn.nodemanager.container-metrics.period-ms":"-1","mapreduce.fileoutputcommitter.task.cleanup.enabled":"false","yarn.nodemanager.log.retain-seconds":"10800","yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds":"3600","yarn.resourcemanager.keytab":"/etc/krb5.keytab","hadoop.security.group.mapping.providers.combined":"true","mapreduce.reduce.merge.inmem.threshold":"1000","yarn.timeline-service.recovery.enabled":"false","fs.azure.saskey.usecontainersaskeyforallaccess":"true","yarn.sharedcache.nm.uploader.thread-count":"20","yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs":"3600","mapreduce.shuffle.ssl.enabled":"false","yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds":"259200000","fs.s3a.committer.staging.abort.pending.uploads":"true","yarn.nodemanager.opportunistic-containers-max-queue-length":"0","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","mapreduce.job.speculative.minimum-allowed-tasks":"10","yarn.log-aggregation.retain-seconds":"-1","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.http.cross-origin.allowed-methods":"GET,POST,HEAD","yarn.resourcemanager.opportunistic-container-allocation.enabled":"false","mapreduce.jobhistory.webapp.address":"0.0.0.0:19888","hadoop.system.tags":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n      ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.log-aggregation.file-controller.TFile.class":"org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"${yarn.resourcemanager.hostname}:8088","mapreduce.jobhistory.recovery.enable":"false","mapreduce.reduce.shuffle.parallelcopies":"5","fs.AbstractFileSystem.webhdfs.impl":"org.apache.hadoop.fs.WebHdfs","fs.trash.interval":"0","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","yarn.app.mapreduce.am.resource.mb":"1536","mapreduce.input.fileinputformat.list-status.num-threads":"1","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.resourcemanager.nm-container-queuing.min-queue-length":"5","yarn.nodemanager.resource.cpu-vcores":"-1","mapreduce.job.reduces":"1","fs.s3a.multipart.size":"100M","yarn.scheduler.minimum-allocation-vcores":"1","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","mapreduce.job.encrypted-intermediate-data-key-size-bits":"128","fs.s3a.metadatastore.authoritative":"false","yarn.nodemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","ha.health-monitor.sleep-after-disconnect.ms":"1000","yarn.app.mapreduce.shuffle.log.limit.kb":"0","hadoop.security.group.mapping":"org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback","yarn.client.application-client-protocol.poll-timeout-ms":"-1","mapreduce.jobhistory.jhist.format":"binary","yarn.resourcemanager.ha.enabled":"false","hadoop.http.staticuser.user":"dr.who","mapreduce.task.exit.timeout.check-interval-ms":"20000","mapreduce.jobhistory.intermediate-user-done-dir.permissions":"770","mapreduce.task.exit.timeout":"60000","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","yarn.resourcemanager.reservation-system.enable":"false","mapreduce.map.output.compress":"false","ha.zookeeper.acl":"world:anyone:rwcda","ipc.server.max.connections":"0","yarn.nodemanager.runtime.linux.docker.default-container-network":"host","yarn.router.webapp.address":"0.0.0.0:8089","yarn.scheduler.maximum-allocation-mb":"8192","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.sharedcache.cleaner.period-mins":"1440","yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint":"http://localhost:3476/v1.0/docker/cli","yarn.app.mapreduce.am.container.log.limit.kb":"0","ipc.client.connect.retry.interval":"1000","yarn.timeline-service.http-cross-origin.enabled":"false","fs.wasbs.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure","yarn.federation.subcluster-resolver.class":"org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","mapreduce.jobhistory.cleaner.enable":"true","yarn.timeline-service.client.fd-flush-interval-secs":"10","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms":"1000","fs.s3a.committer.staging.tmp.path":"tmp/staging","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms":"10","mapreduce.job.end-notification.retry.attempts":"0","yarn.nodemanager.resource.count-logical-processors-as-cores":"false","hadoop.registry.zk.root":"/registry","adl.feature.ownerandgroup.enableupn":"false","yarn.resourcemanager.zk-max-znode-size.bytes":"1048576","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","yarn.nodemanager.localizer.cache.target-size-mb":"10240","fs.s3a.committer.staging.conflict-mode":"fail","mapreduce.client.libjars.wildcard":"true","fs.s3a.committer.staging.unique-filenames":"true","yarn.nodemanager.node-attributes.provider.fetch-timeout-ms":"1200000","fs.s3a.list.version":"2","ftp.client-write-packet-size":"65536","fs.AbstractFileSystem.adl.impl":"org.apache.hadoop.fs.adl.Adl","hadoop.security.key.default.cipher":"AES/CTR/NoPadding","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"86400","mapreduce.job.local-fs.single-disk-limit.check.interval-ms":"5000","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","yarn.nodemanager.amrmproxy.address":"0.0.0.0:8049","ipc.server.listen.queue.size":"128","map.sort.class":"org.apache.hadoop.util.QuickSort","fs.viewfs.rename.strategy":"SAME_MOUNTPOINT","hadoop.security.kms.client.authentication.retry-count":"1","fs.permissions.umask-mode":"022","fs.s3a.assumed.role.credentials.provider":"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider","yarn.nodemanager.vmem-check-enabled":"true","yarn.nodemanager.numa-awareness.enabled":"false","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","yarn.federation.registry.base-dir":"yarnfederation/","mapreduce.job.max.map":"-1","mapreduce.job.local-fs.single-disk-limit.bytes":"-1","mapreduce.job.ubertask.maxreduces":"1","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","mapreduce.jobhistory.jobname.limit":"50","yarn.client.nodemanager-connect.retry-interval-ms":"10000","yarn.timeline-service.state-store-class":"org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ","yarn.sharedcache.nested-level":"3","yarn.timeline-service.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","fs.azure.user.agent.prefix":"unknown","yarn.resourcemanager.zk-delegation-token-node.split-index":"*********(redacted)","yarn.nodemanager.numa-awareness.read-topology":"false","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","yarn.registry.class":"org.apache.hadoop.registry.client.impl.FSRegistryOperationsService","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","yarn.scheduler.queue-placement-rules":"user-group","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","yarn.resourcemanager.recovery.enabled":"false","yarn.timeline-service.webapp.rest-csrf.enabled":"false"},"System Properties":{"java.io.tmpdir":"/var/folders/p_/5j1mtw1x0579b34vy4ztcbs40000gn/T/","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.version":"1.8","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Oracle Corporation","java.vm.specification.version":"1.8","user.home":"*********(redacted)","file.encoding.pkg":"sun.io","sun.nio.ch.bugLevel":"","sun.arch.data.model":"64","sun.boot.library.path":"/Library/Java/JavaVirtualMachines/jdk1.8.0_231.jdk/Contents/Home/jre/lib","user.dir":"*********(redacted)","java.library.path":"*********(redacted)/Library/Java/Extensions:/Library/Java/Extensions:/Network/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java:.","sun.cpu.isalist":"","os.arch":"x86_64","java.vm.version":"25.231-b11","jetty.git.hash":"ab228fde9e55e9164c738d7fa121f8ac5acd51c9","java.endorsed.dirs":"/Library/Java/JavaVirtualMachines/jdk1.8.0_231.jdk/Contents/Home/jre/lib/endorsed","java.runtime.version":"1.8.0_231-b11","java.vm.info":"mixed mode","java.ext.dirs":"*********(redacted)/Library/Java/Extensions:/Library/Java/JavaVirtualMachines/jdk1.8.0_231.jdk/Contents/Home/jre/lib/ext:/Library/Java/Extensions:/Network/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java","java.runtime.name":"Java(TM) SE Runtime Environment","file.separator":"/","java.class.version":"52.0","scala.usejavacp":"true","java.specification.name":"Java Platform API Specification","sun.boot.class.path":"/Library/Java/JavaVirtualMachines/jdk1.8.0_231.jdk/Contents/Home/jre/lib/resources.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_231.jdk/Contents/Home/jre/lib/rt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_231.jdk/Contents/Home/jre/lib/sunrsasign.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_231.jdk/Contents/Home/jre/lib/jsse.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_231.jdk/Contents/Home/jre/lib/jce.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_231.jdk/Contents/Home/jre/lib/charsets.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_231.jdk/Contents/Home/jre/lib/jfr.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_231.jdk/Contents/Home/jre/classes","file.encoding":"UTF-8","user.timezone":"America/Los_Angeles","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","os.version":"10.15.5","sun.os.patch.level":"unknown","gopherProxySet":"false","java.vm.specification.vendor":"Oracle Corporation","jdk.lang.Process.launchMechanism":"POSIX_SPAWN","user.country":"US","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.url":"http://java.oracle.com/","java.awt.printerjob":"sun.lwawt.macosx.CPrinterJob","java.awt.graphicsenv":"sun.awt.CGraphicsEnvironment","awt.toolkit":"sun.lwawt.macosx.LWCToolkit","os.name":"Mac OS X","java.vm.vendor":"Oracle Corporation","java.vendor.url.bug":"http://bugreport.sun.com/bugreport/","user.name":"terryk","java.vm.name":"Java HotSpot(TM) 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.SparkSubmit --master spark://*********(redacted) --conf spark.eventLog.logStageExecutorMetrics=true --conf spark.eventLog.enabled=true --class org.apache.spark.repl.Main --name Spark shell spark-shell","java.home":"/Library/Java/JavaVirtualMachines/jdk1.8.0_231.jdk/Contents/Home/jre","java.version":"1.8.0_231","sun.io.unicode.encoding":"UnicodeBig"},"Classpath Entries":{"*********(redacted)/audience-annotations-0.5.0.jar":"System Classpath","*********(redacted)/zstd-jni-1.4.5-2.jar":"System Classpath","*********(redacted)/hadoop-yarn-common-3.2.0.jar":"System Classpath","*********(redacted)/apache-spark/common/unsafe/target/scala-2.12/classes/":"System Classpath","*********(redacted)/kerb-identity-1.0.1.jar":"System Classpath","*********(redacted)/jersey-hk2-2.30.jar":"System Classpath","*********(redacted)/apache-spark/mllib/target/jars/*":"System Classpath","*********(redacted)/slf4j-log4j12-1.7.30.jar":"System Classpath","*********(redacted)/kerby-pkix-1.0.1.jar":"System Classpath","*********(redacted)/metrics-jmx-4.1.1.jar":"System Classpath","*********(redacted)/xz-1.5.jar":"System Classpath","*********(redacted)/jakarta.validation-api-2.0.2.jar":"System Classpath","*********(redacted)/jetty-webapp-9.4.28.v20200408.jar":"System Classpath","*********(redacted)/metrics-core-4.1.1.jar":"System Classpath","*********(redacted)/apache-spark/graphx/target/scala-2.12/classes/":"System Classpath","*********(redacted)/metrics-graphite-4.1.1.jar":"System Classpath","*********(redacted)/jersey-server-2.30.jar":"System Classpath","*********(redacted)/jetty-jndi-9.4.28.v20200408.jar":"System Classpath","*********(redacted)/jetty-proxy-9.4.28.v20200408.jar":"System Classpath","*********(redacted)/parquet-jackson-1.10.1.jar":"System Classpath","*********(redacted)/commons-crypto-1.0.0.jar":"System Classpath","*********(redacted)/xbean-asm7-shaded-4.15.jar":"System Classpath","*********(redacted)/spark-kvstore_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","*********(redacted)/netty-3.10.6.Final.jar":"System Classpath","*********(redacted)/jsr305-3.0.2.jar":"System Classpath","*********(redacted)/osgi-resource-locator-1.0.3.jar":"System Classpath","*********(redacted)/kerb-server-1.0.1.jar":"System Classpath","*********(redacted)/avro-1.8.2.jar":"System Classpath","*********(redacted)/jersey-media-jaxb-2.30.jar":"System Classpath","*********(redacted)/accessors-smart-1.2.jar":"System Classpath","*********(redacted)/spark-network-shuffle_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","*********(redacted)/protobuf-java-2.5.0.jar":"System Classpath","*********(redacted)/minlog-1.3.0.jar":"System Classpath","*********(redacted)/scala-parser-combinators_2.12-1.1.2.jar":"System Classpath","*********(redacted)/jetty-continuation-9.4.28.v20200408.jar":"System Classpath","*********(redacted)/apache-spark/sql/hive/target/scala-2.12/classes/":"System Classpath","*********(redacted)/commons-net-3.6.jar":"System Classpath","*********(redacted)/apache-spark/core/target/jars/*":"System Classpath","*********(redacted)/istack-commons-runtime-3.0.8.jar":"System Classpath","*********(redacted)/jsp-api-2.1.jar":"System Classpath","*********(redacted)/hadoop-annotations-3.2.0.jar":"System Classpath","*********(redacted)/spark-launcher_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","*********(redacted)/jetty-servlets-9.4.28.v20200408.jar":"System Classpath","*********(redacted)/hadoop-hdfs-client-3.2.0.jar":"System Classpath","*********(redacted)/hive-storage-api-2.7.1.jar":"System Classpath","*********(redacted)/aopalliance-repackaged-2.6.1.jar":"System Classpath","*********(redacted)/kerby-xdr-1.0.1.jar":"System Classpath","*********(redacted)/orc-mapreduce-1.5.10.jar":"System Classpath","*********(redacted)/jackson-databind-2.10.0.jar":"System Classpath","*********(redacted)/spark-mllib_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","*********(redacted)/hadoop-mapreduce-client-jobclient-3.2.0.jar":"System Classpath","*********(redacted)/hk2-utils-2.6.1.jar":"System Classpath","*********(redacted)/jetty-security-9.4.28.v20200408.jar":"System Classpath","*********(redacted)/hadoop-client-3.2.0.jar":"System Classpath","*********(redacted)/hadoop-auth-3.2.0.jar":"System Classpath","*********(redacted)/gson-2.2.4.jar":"System Classpath","*********(redacted)/htrace-core4-4.1.0-incubating.jar":"System Classpath","*********(redacted)/curator-client-2.12.0.jar":"System Classpath","*********(redacted)/kerby-util-1.0.1.jar":"System Classpath","*********(redacted)/jackson-jaxrs-base-2.9.5.jar":"System Classpath","*********(redacted)/jackson-module-paranamer-2.10.0.jar":"System Classpath","*********(redacted)/commons-beanutils-1.9.3.jar":"System Classpath","*********(redacted)/woodstox-core-5.0.3.jar":"System Classpath","*********(redacted)/arrow-memory-0.15.1.jar":"System Classpath","*********(redacted)/activation-1.1.1.jar":"System Classpath","*********(redacted)/okhttp-2.7.5.jar":"System Classpath","*********(redacted)/jackson-mapper-asl-1.9.13.jar":"System Classpath","*********(redacted)/core-1.1.2.jar":"System Classpath","*********(redacted)/jetty-plus-9.4.28.v20200408.jar":"System Classpath","*********(redacted)/apache-spark/resource-managers/yarn/target/scala-2.12/classes/":"System Classpath","*********(redacted)/antlr4-runtime-4.7.1.jar":"System Classpath","*********(redacted)/commons-compress-1.8.1.jar":"System Classpath","*********(redacted)/spark-tags_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","*********(redacted)/orc-shims-1.5.10.jar":"System Classpath","*********(redacted)/opencsv-2.3.jar":"System Classpath","*********(redacted)/compress-lzf-1.0.3.jar":"System Classpath","*********(redacted)/jackson-core-asl-1.9.13.jar":"System Classpath","*********(redacted)/objenesis-2.5.1.jar":"System Classpath","*********(redacted)/apache-spark/conf/":"System Classpath","*********(redacted)/apache-spark/common/network-common/target/scala-2.12/classes/":"System Classpath","*********(redacted)/httpcore-4.4.4.jar":"System Classpath","*********(redacted)/jcip-annotations-1.0-1.jar":"System Classpath","*********(redacted)/apache-spark/launcher/target/scala-2.12/classes/":"System Classpath","*********(redacted)/apache-spark/resource-managers/mesos/target/scala-2.12/classes":"System Classpath","*********(redacted)/apache-spark/sql/hive-thriftserver/target/scala-2.12/classes/":"System Classpath","*********(redacted)/jetty-util-9.4.28.v20200408.jar":"System Classpath","*********(redacted)/jetty-server-9.4.28.v20200408.jar":"System Classpath","*********(redacted)/jetty-xml-9.4.28.v20200408.jar":"System Classpath","*********(redacted)/scala-collection-compat_2.12-2.1.1.jar":"System Classpath","*********(redacted)/javax.servlet-api-3.1.0.jar":"System Classpath","*********(redacted)/jackson-core-2.10.0.jar":"System Classpath","*********(redacted)/jetty-client-9.4.28.v20200408.jar":"System Classpath","*********(redacted)/netty-all-4.1.47.Final.jar":"System Classpath","*********(redacted)/arpack_combined_all-0.1.jar":"System Classpath","*********(redacted)/jersey-container-servlet-core-2.30.jar":"System Classpath","*********(redacted)/paranamer-2.8.jar":"System Classpath","*********(redacted)/spark-sql_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","*********(redacted)/hadoop-common-3.2.0.jar":"System Classpath","*********(redacted)/jersey-common-2.30.jar":"System Classpath","*********(redacted)/hk2-locator-2.6.1.jar":"System Classpath","*********(redacted)/okio-1.6.0.jar":"System Classpath","*********(redacted)/jetty-http-9.4.28.v20200408.jar":"System Classpath","*********(redacted)/jakarta.xml.bind-api-2.3.2.jar":"System Classpath","*********(redacted)/ivy-2.4.0.jar":"System Classpath","*********(redacted)/apache-spark/mllib/target/scala-2.12/classes/":"System Classpath","*********(redacted)/scala-library-2.12.10.jar":"System Classpath","*********(redacted)/scala-xml_2.12-1.2.0.jar":"System Classpath","*********(redacted)/apache-spark/streaming/target/scala-2.12/classes/":"System Classpath","*********(redacted)/spark-repl_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","*********(redacted)/apache-spark/common/tags/target/scala-2.12/classes/":"System Classpath","*********(redacted)/kerby-config-1.0.1.jar":"System Classpath","*********(redacted)/zookeeper-3.4.14.jar":"System Classpath","*********(redacted)/janino-3.1.2.jar":"System Classpath","*********(redacted)/breeze_2.12-1.0.jar":"System Classpath","*********(redacted)/hk2-api-2.6.1.jar":"System Classpath","*********(redacted)/hadoop-mapreduce-client-common-3.2.0.jar":"System Classpath","*********(redacted)/flatbuffers-java-1.9.0.jar":"System Classpath","*********(redacted)/curator-framework-2.13.0.jar":"System Classpath","*********(redacted)/metrics-jvm-4.1.1.jar":"System Classpath","*********(redacted)/stax2-api-3.1.4.jar":"System Classpath","*********(redacted)/json-smart-2.3.jar":"System Classpath","*********(redacted)/parquet-hadoop-1.10.1.jar":"System Classpath","*********(redacted)/kerb-common-1.0.1.jar":"System Classpath","*********(redacted)/breeze-macros_2.12-1.0.jar":"System Classpath","*********(redacted)/scala-compiler-2.12.10.jar":"System Classpath","*********(redacted)/guava-14.0.1.jar":"System Classpath","*********(redacted)/jul-to-slf4j-1.7.30.jar":"System Classpath","*********(redacted)/spark-assembly_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","*********(redacted)/spark-core_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","*********(redacted)/spark-graphx_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","*********(redacted)/jakarta.ws.rs-api-2.1.6.jar":"System Classpath","*********(redacted)/spark-unsafe_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","*********(redacted)/jetty-io-9.4.28.v20200408.jar":"System Classpath","*********(redacted)/JTransforms-3.1.jar":"System Classpath","*********(redacted)/commons-io-2.5.jar":"System Classpath","*********(redacted)/spark-catalyst_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","*********(redacted)/spire_2.12-0.17.0-M1.jar":"System Classpath","*********(redacted)/commons-cli-1.2.jar":"System Classpath","*********(redacted)/shims-0.7.45.jar":"System Classpath","*********(redacted)/spotbugs-annotations-3.1.9.jar":"System Classpath","*********(redacted)/kerb-simplekdc-1.0.1.jar":"System Classpath","*********(redacted)/commons-codec-1.11.jar":"System Classpath","*********(redacted)/jcl-over-slf4j-1.7.30.jar":"System Classpath","*********(redacted)/lz4-java-1.7.1.jar":"System Classpath","*********(redacted)/aopalliance-1.0.jar":"System Classpath","*********(redacted)/jackson-jaxrs-json-provider-2.9.5.jar":"System Classpath","*********(redacted)/json4s-jackson_2.12-3.6.6.jar":"System Classpath","*********(redacted)/commons-math3-3.4.1.jar":"System Classpath","*********(redacted)/shapeless_2.12-2.3.3.jar":"System Classpath","*********(redacted)/spark-mllib-local_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","*********(redacted)/dnsjava-2.1.7.jar":"System Classpath","*********(redacted)/unused-1.0.0.jar":"System Classpath","*********(redacted)/log4j-1.2.17.jar":"System Classpath","*********(redacted)/kerb-client-1.0.1.jar":"System Classpath","*********(redacted)/commons-lang-2.6.jar":"System Classpath","*********(redacted)/apache-spark/repl/target/scala-2.12/classes/":"System Classpath","*********(redacted)/kerb-util-1.0.1.jar":"System Classpath","*********(redacted)/slf4j-api-1.7.30.jar":"System Classpath","*********(redacted)/jackson-annotations-2.10.0.jar":"System Classpath","*********(redacted)/kerb-core-1.0.1.jar":"System Classpath","*********(redacted)/apache-spark/sql/catalyst/target/scala-2.12/classes/":"System Classpath","*********(redacted)/spire-macros_2.12-0.17.0-M1.jar":"System Classpath","*********(redacted)/apache-spark/examples/target/scala-2.12/classes/":"System Classpath","*********(redacted)/jakarta.annotation-api-1.3.5.jar":"System Classpath","*********(redacted)/json4s-scalap_2.12-3.6.6.jar":"System Classpath","*********(redacted)/apache-spark/core/target/scala-2.12/classes/":"System Classpath","*********(redacted)/arrow-format-0.15.1.jar":"System Classpath","*********(redacted)/kryo-shaded-4.0.2.jar":"System Classpath","*********(redacted)/scala-reflect-2.12.10.jar":"System Classpath","*********(redacted)/kerb-admin-1.0.1.jar":"System Classpath","*********(redacted)/hadoop-yarn-client-3.2.0.jar":"System Classpath","*********(redacted)/nimbus-jose-jwt-4.41.1.jar":"System Classpath","*********(redacted)/apache-spark/common/kvstore/target/scala-2.12/classes/":"System Classpath","*********(redacted)/oro-2.0.8.jar":"System Classpath","*********(redacted)/jakarta.inject-2.6.1.jar":"System Classpath","*********(redacted)/chill-java-0.9.5.jar":"System Classpath","*********(redacted)/cats-kernel_2.12-2.0.0-M4.jar":"System Classpath","*********(redacted)/jersey-container-servlet-2.30.jar":"System Classpath","*********(redacted)/py4j-0.10.9.jar":"System Classpath","*********(redacted)/parquet-format-2.4.0.jar":"System Classpath","*********(redacted)/apache-spark/sql/core/target/scala-2.12/classes/":"System Classpath","*********(redacted)/jline-2.14.6.jar":"System Classpath","*********(redacted)/JLargeArrays-1.5.jar":"System Classpath","*********(redacted)/kerby-asn1-1.0.1.jar":"System Classpath","*********(redacted)/jaxb-runtime-2.3.2.jar":"System Classpath","*********(redacted)/pmml-model-1.4.8.jar":"System Classpath","*********(redacted)/parquet-encoding-1.10.1.jar":"System Classpath","*********(redacted)/machinist_2.12-0.6.8.jar":"System Classpath","*********(redacted)/commons-compiler-3.1.2.jar":"System Classpath","*********(redacted)/kerb-crypto-1.0.1.jar":"System Classpath","*********(redacted)/aircompressor-0.10.jar":"System Classpath","*********(redacted)/leveldbjni-all-1.8.jar":"System Classpath","*********(redacted)/metrics-json-4.1.1.jar":"System Classpath","*********(redacted)/jackson-module-jaxb-annotations-2.9.5.jar":"System Classpath","*********(redacted)/jaxb-api-2.2.11.jar":"System Classpath","*********(redacted)/spire-util_2.12-0.17.0-M1.jar":"System Classpath","*********(redacted)/jetty-servlet-9.4.28.v20200408.jar":"System Classpath","*********(redacted)/spark-network-common_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","*********(redacted)/spark-streaming_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","*********(redacted)/json4s-core_2.12-3.6.6.jar":"System Classpath","*********(redacted)/stream-2.9.6.jar":"System Classpath","*********(redacted)/spark-sketch_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","*********(redacted)/chill_2.12-0.9.5.jar":"System Classpath","*********(redacted)/commons-configuration2-2.1.1.jar":"System Classpath","*********(redacted)/univocity-parsers-2.8.3.jar":"System Classpath","*********(redacted)/parquet-common-1.10.1.jar":"System Classpath","*********(redacted)/threeten-extra-1.5.0.jar":"System Classpath","*********(redacted)/token-provider-1.0.1.jar":"System Classpath","*********(redacted)/commons-text-1.6.jar":"System Classpath","*********(redacted)/apache-spark/common/sketch/target/scala-2.12/classes/":"System Classpath","*********(redacted)/javax.inject-1.jar":"System Classpath","*********(redacted)/httpclient-4.5.2.jar":"System Classpath","*********(redacted)/avro-ipc-1.8.2.jar":"System Classpath","*********(redacted)/curator-recipes-2.13.0.jar":"System Classpath","*********(redacted)/commons-collections-3.2.2.jar":"System Classpath","*********(redacted)/jersey-client-2.30.jar":"System Classpath","*********(redacted)/snappy-java-1.1.7.5.jar":"System Classpath","*********(redacted)/algebra_2.12-2.0.0-M2.jar":"System Classpath","*********(redacted)/javassist-3.25.0-GA.jar":"System Classpath","*********(redacted)/arrow-vector-0.15.1.jar":"System Classpath","*********(redacted)/avro-mapred-1.8.2-hadoop2.jar":"System Classpath","*********(redacted)/re2j-1.1.jar":"System Classpath","*********(redacted)/parquet-column-1.10.1.jar":"System Classpath","*********(redacted)/pyrolite-4.30.jar":"System Classpath","*********(redacted)/apache-spark/common/network-shuffle/target/scala-2.12/classes/":"System Classpath","*********(redacted)/apache-spark/common/network-yarn/target/scala-2.12/classes":"System Classpath","*********(redacted)/hadoop-yarn-api-3.2.0.jar":"System Classpath","*********(redacted)/macro-compat_2.12-1.1.1.jar":"System Classpath","*********(redacted)/spire-platform_2.12-0.17.0-M1.jar":"System Classpath","*********(redacted)/commons-lang3-3.10.jar":"System Classpath","*********(redacted)/json4s-ast_2.12-3.6.6.jar":"System Classpath","*********(redacted)/RoaringBitmap-0.7.45.jar":"System Classpath","*********(redacted)/orc-core-1.5.10.jar":"System Classpath","*********(redacted)/jackson-module-scala_2.12-2.10.0.jar":"System Classpath","*********(redacted)/hadoop-mapreduce-client-core-3.2.0.jar":"System Classpath"}}
+{"Event":"SparkListenerApplicationStart","App Name":"Spark shell","App ID":"app-20200706201101-0003","Timestamp":1594091460235,"User":"terryk"}
+{"Event":"SparkListenerExecutorAdded","Timestamp":1594091463318,"Executor ID":"0","Executor Info":{"Host":"127.0.0.1","Total Cores":16,"Log Urls":{"stdout":"http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stdout","stderr":"http://127.0.0.1:8081/logPage/?appId=app-20200706201101-0003&executorId=0&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}}
+{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"0","Host":"127.0.0.1","Port":64419},"Maximum Memory":384093388,"Timestamp":1594091463413,"Maximum Onheap Memory":384093388,"Maximum Offheap Memory":0}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":0,"description":"foreach at <console>:26","details":"org.apache.spark.sql.Dataset.foreach(Dataset.scala:2862)\n$line19.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:26)\n$line19.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:30)\n$line19.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:32)\n$line19.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:34)\n$line19.$read$$iw$$iw$$iw$$iw.<init>(<console>:36)\n$line19.$read$$iw$$iw$$iw.<init>(<console>:38)\n$line19.$read$$iw$$iw.<init>(<console>:40)\n$line19.$read$$iw.<init>(<console>:42)\n$line19.$read.<init>(<console>:44)\n$line19.$read$.<init>(<console>:48)\n$line19.$read$.<clinit>(<console>)\n$line19.$eval$.$print$lzycompute(<console>:7)\n$line19.$eval$.$print(<console>:6)\n$line19.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","physicalPlanDescription":"== Physical Plan ==\nDeserializeToObject (8)\n+- * BroadcastHashJoin Inner BuildLeft (7)\n   :- BroadcastExchange (5)\n   :  +- * BroadcastHashJoin Inner BuildRight (4)\n   :     :- * LocalTableScan (1)\n   :     +- BroadcastExchange (3)\n   :        +- LocalTableScan (2)\n   +- * LocalTableScan (6)\n\n\n(1) LocalTableScan [codegen id : 1]\nOutput [3]: [i1#10, j1#11, k1#12]\nArguments: [i1#10, j1#11, k1#12]\n\n(2) LocalTableScan\nOutput [3]: [i2#26, j2#27, k2#28]\nArguments: [i2#26, j2#27, k2#28]\n\n(3) BroadcastExchange\nInput [3]: [i2#26, j2#27, k2#28]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint))), [id=#23]\n\n(4) BroadcastHashJoin [codegen id : 1]\nLeft keys [1]: [i1#10]\nRight keys [1]: [i2#26]\nJoin condition: None\n\n(5) BroadcastExchange\nInput [6]: [i1#10, j1#11, k1#12, i2#26, j2#27, k2#28]\nArguments: HashedRelationBroadcastMode(List(cast(input[3, int, false] as bigint))), [id=#32]\n\n(6) LocalTableScan\nOutput [3]: [i3#42, j3#43, k3#44]\nArguments: [i3#42, j3#43, k3#44]\n\n(7) BroadcastHashJoin [codegen id : 2]\nLeft keys [1]: [i2#26]\nRight keys [1]: [i3#42]\nJoin condition: None\n\n(8) DeserializeToObject\nInput [9]: [i1#10, j1#11, k1#12, i2#26, j2#27, k2#28, i3#42, j3#43, k3#44]\nArguments: createexternalrow(i1#10, j1#11, k1#12.toString, i2#26, j2#27, k2#28.toString, i3#42, j3#43, k3#44.toString, StructField(i1,IntegerType,false), StructField(j1,IntegerType,false), StructField(k1,StringType,true), StructField(i2,IntegerType,false), StructField(j2,IntegerType,false), StructField(k2,StringType,true), StructField(i3,IntegerType,false), StructField(j3,IntegerType,false), StructField(k3,StringType,true)), obj#93: org.apache.spark.sql.Row\n\n","sparkPlanInfo":{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject createexternalrow(i1#10, j1#11, k1#12.toString, i2#26, j2#27, k2#28.toString, i3#42, j3#43, k3#44.toString, StructField(i1,IntegerType,false), StructField(j1,IntegerType,false), StructField(k1,StringType,true), StructField(i2,IntegerType,false), StructField(j2,IntegerType,false), StructField(k2,StringType,true), StructField(i3,IntegerType,false), StructField(j3,IntegerType,false), StructField(k3,StringType,true)), obj#93: org.apache.spark.sql.Row","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [i2#26], [i3#42], Inner, BuildLeft","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[3, int, false] as bigint))), [id=#32]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [i1#10], [i2#26], Inner, BuildRight","children":[{"nodeName":"LocalTableScan","simpleString":"LocalTableScan [i1#10, j1#11, k1#12]","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":8,"metricType":"sum"}]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint))), [id=#23]","children":[{"nodeName":"LocalTableScan","simpleString":"LocalTableScan [i2#26, j2#27, k2#28]","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":13,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"data size","accumulatorId":9,"metricType":"size"},{"name":"time to collect","accumulatorId":10,"metricType":"timing"},{"name":"time to build","accumulatorId":11,"metricType":"timing"},{"name":"time to broadcast","accumulatorId":12,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":7,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":6,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"data size","accumulatorId":2,"metricType":"size"},{"name":"time to collect","accumulatorId":3,"metricType":"timing"},{"name":"time to build","accumulatorId":4,"metricType":"timing"},{"name":"time to broadcast","accumulatorId":5,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"LocalTableScan","simpleString":"LocalTableScan [i3#42, j3#43, k3#44]","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":14,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":0,"metricType":"timing"}]}],"metadata":{},"metrics":[]},"time":1594091478577}
+{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1594091478844,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Number of Tasks":16,"RDD Info":[{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"DeserializeToObject\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"3\",\"name\":\"LocalTableScan\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"LocalTableScan\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:185)\njava.util.concurrent.FutureTask.run(FutureTask.java:266)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[0],"Properties":{"spark.driver.host":"localhost","spark.eventLog.enabled":"true","spark.job.interruptOnCancel":"true","spark.driver.port":"64413","spark.repl.class.uri":"spark://localhost:64413/classes","spark.jars":"","spark.repl.class.outputDir":"/private/var/folders/p_/5j1mtw1x0579b34vy4ztcbs40000gn/T/spark-0c713088-21d7-4b35-9b52-7a4dd4d15ae1/repl-361cef81-42cd-4875-a8a6-ce9dfe55682a","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"0\",\"name\":\"DeserializeToObject\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.job.description":"broadcast exchange (runId bb0234e5-4157-49a4-b40c-6d538d9f2ec8)","spark.executor.id":"driver","spark.jobGroup.id":"bb0234e5-4157-49a4-b40c-6d538d9f2ec8","spark.eventLog.logStageExecutorMetrics":"true","spark.submit.deployMode":"client","spark.master":"*********(redacted)","spark.home":"*********(redacted)","spark.sql.execution.id":"0","spark.sql.catalogImplementation":"in-memory","spark.app.id":"app-20200706201101-0003"}}
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Number of Tasks":16,"RDD Info":[{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"DeserializeToObject\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"3\",\"name\":\"LocalTableScan\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"LocalTableScan\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:185)\njava.util.concurrent.FutureTask.run(FutureTask.java:266)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1594091478860,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.driver.host":"localhost","spark.eventLog.enabled":"true","spark.job.interruptOnCancel":"true","spark.driver.port":"64413","spark.repl.class.uri":"spark://localhost:64413/classes","spark.jars":"","spark.repl.class.outputDir":"/private/var/folders/p_/5j1mtw1x0579b34vy4ztcbs40000gn/T/spark-0c713088-21d7-4b35-9b52-7a4dd4d15ae1/repl-361cef81-42cd-4875-a8a6-ce9dfe55682a","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"0\",\"name\":\"DeserializeToObject\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.job.description":"broadcast exchange (runId bb0234e5-4157-49a4-b40c-6d538d9f2ec8)","spark.executor.id":"driver","spark.jobGroup.id":"bb0234e5-4157-49a4-b40c-6d538d9f2ec8","spark.eventLog.logStageExecutorMetrics":"true","spark.submit.deployMode":"client","spark.master":"*********(redacted)","spark.home":"*********(redacted)","spark.sql.execution.id":"0","spark.sql.catalogImplementation":"in-memory","spark.app.id":"app-20200706201101-0003"}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1594091479253,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Launch Time":1594091479269,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":2,"Index":2,"Attempt":0,"Launch Time":1594091479269,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":3,"Index":3,"Attempt":0,"Launch Time":1594091479270,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":4,"Index":4,"Attempt":0,"Launch Time":1594091479270,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":5,"Index":5,"Attempt":0,"Launch Time":1594091479270,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":6,"Index":6,"Attempt":0,"Launch Time":1594091479271,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":7,"Index":7,"Attempt":0,"Launch Time":1594091479272,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":8,"Index":8,"Attempt":0,"Launch Time":1594091479272,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":9,"Index":9,"Attempt":0,"Launch Time":1594091479273,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":10,"Index":10,"Attempt":0,"Launch Time":1594091479273,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":11,"Index":11,"Attempt":0,"Launch Time":1594091479273,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":12,"Index":12,"Attempt":0,"Launch Time":1594091479274,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":13,"Index":13,"Attempt":0,"Launch Time":1594091479274,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":14,"Index":14,"Attempt":0,"Launch Time":1594091479274,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":15,"Index":15,"Attempt":0,"Launch Time":1594091479274,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":6,"Index":6,"Attempt":0,"Launch Time":1594091479271,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480364,"Failed":false,"Killed":false,"Accumulables":[{"ID":13,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":20,"Name":"internal.metrics.jvmGCTime","Update":109,"Value":109,"Internal":true,"Count Failed Values":true},{"ID":19,"Name":"internal.metrics.resultSize","Update":1387,"Value":1387,"Internal":true,"Count Failed Values":true},{"ID":18,"Name":"internal.metrics.executorCpuTime","Update":8922000,"Value":8922000,"Internal":true,"Count Failed Values":true},{"ID":17,"Name":"internal.metrics.executorRunTime","Update":80,"Value":80,"Internal":true,"Count Failed Values":true},{"ID":16,"Name":"internal.metrics.executorDeserializeCpuTime","Update":235295000,"Value":235295000,"Internal":true,"Count Failed Values":true},{"ID":15,"Name":"internal.metrics.executorDeserializeTime","Update":962,"Value":962,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":155100856,"JVMOffHeapMemory":64239224,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":6964,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":6964,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1852,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":7,"MinorGCTime":33,"MajorGCCount":3,"MajorGCTime":110},"Task Metrics":{"Executor Deserialize Time":962,"Executor Deserialize CPU Time":235295000,"Executor Run Time":80,"Executor CPU Time":8922000,"Peak Execution Memory":0,"Result Size":1387,"JVM GC Time":109,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":8,"Index":8,"Attempt":0,"Launch Time":1594091479272,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480367,"Failed":false,"Killed":false,"Accumulables":[{"ID":13,"Name":"number of output rows","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":20,"Name":"internal.metrics.jvmGCTime","Update":109,"Value":218,"Internal":true,"Count Failed Values":true},{"ID":19,"Name":"internal.metrics.resultSize","Update":1391,"Value":2778,"Internal":true,"Count Failed Values":true},{"ID":18,"Name":"internal.metrics.executorCpuTime","Update":8860000,"Value":17782000,"Internal":true,"Count Failed Values":true},{"ID":17,"Name":"internal.metrics.executorRunTime","Update":81,"Value":161,"Internal":true,"Count Failed Values":true},{"ID":16,"Name":"internal.metrics.executorDeserializeCpuTime","Update":213308000,"Value":448603000,"Internal":true,"Count Failed Values":true},{"ID":15,"Name":"internal.metrics.executorDeserializeTime","Update":963,"Value":1925,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":155100856,"JVMOffHeapMemory":64239224,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":6964,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":6964,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1852,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":7,"MinorGCTime":33,"MajorGCCount":3,"MajorGCTime":110},"Task Metrics":{"Executor Deserialize Time":963,"Executor Deserialize CPU Time":213308000,"Executor Run Time":81,"Executor CPU Time":8860000,"Peak Execution Memory":0,"Result Size":1391,"JVM GC Time":109,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":2,"Index":2,"Attempt":0,"Launch Time":1594091479269,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480367,"Failed":false,"Killed":false,"Accumulables":[{"ID":13,"Name":"number of output rows","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":20,"Name":"internal.metrics.jvmGCTime","Update":109,"Value":327,"Internal":true,"Count Failed Values":true},{"ID":19,"Name":"internal.metrics.resultSize","Update":1387,"Value":4165,"Internal":true,"Count Failed Values":true},{"ID":18,"Name":"internal.metrics.executorCpuTime","Update":10463000,"Value":28245000,"Internal":true,"Count Failed Values":true},{"ID":17,"Name":"internal.metrics.executorRunTime","Update":79,"Value":240,"Internal":true,"Count Failed Values":true},{"ID":16,"Name":"internal.metrics.executorDeserializeCpuTime","Update":228677000,"Value":677280000,"Internal":true,"Count Failed Values":true},{"ID":15,"Name":"internal.metrics.executorDeserializeTime","Update":962,"Value":2887,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":155100856,"JVMOffHeapMemory":64239224,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":6964,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":6964,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1852,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":7,"MinorGCTime":33,"MajorGCCount":3,"MajorGCTime":110},"Task Metrics":{"Executor Deserialize Time":962,"Executor Deserialize CPU Time":228677000,"Executor Run Time":79,"Executor CPU Time":10463000,"Peak Execution Memory":0,"Result Size":1387,"JVM GC Time":109,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":13,"Index":13,"Attempt":0,"Launch Time":1594091479274,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480367,"Failed":false,"Killed":false,"Accumulables":[{"ID":13,"Name":"number of output rows","Update":"1","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":20,"Name":"internal.metrics.jvmGCTime","Update":109,"Value":436,"Internal":true,"Count Failed Values":true},{"ID":19,"Name":"internal.metrics.resultSize","Update":1390,"Value":5555,"Internal":true,"Count Failed Values":true},{"ID":18,"Name":"internal.metrics.executorCpuTime","Update":8782000,"Value":37027000,"Internal":true,"Count Failed Values":true},{"ID":17,"Name":"internal.metrics.executorRunTime","Update":83,"Value":323,"Internal":true,"Count Failed Values":true},{"ID":16,"Name":"internal.metrics.executorDeserializeCpuTime","Update":196368000,"Value":873648000,"Internal":true,"Count Failed Values":true},{"ID":15,"Name":"internal.metrics.executorDeserializeTime","Update":962,"Value":3849,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":155100856,"JVMOffHeapMemory":64239224,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":6964,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":6964,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1852,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":7,"MinorGCTime":33,"MajorGCCount":3,"MajorGCTime":110},"Task Metrics":{"Executor Deserialize Time":962,"Executor Deserialize CPU Time":196368000,"Executor Run Time":83,"Executor CPU Time":8782000,"Peak Execution Memory":0,"Result Size":1390,"JVM GC Time":109,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1594091479253,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480368,"Failed":false,"Killed":false,"Accumulables":[{"ID":13,"Name":"number of output rows","Update":"1","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":20,"Name":"internal.metrics.jvmGCTime","Update":109,"Value":545,"Internal":true,"Count Failed Values":true},{"ID":19,"Name":"internal.metrics.resultSize","Update":1384,"Value":6939,"Internal":true,"Count Failed Values":true},{"ID":18,"Name":"internal.metrics.executorCpuTime","Update":10820000,"Value":47847000,"Internal":true,"Count Failed Values":true},{"ID":17,"Name":"internal.metrics.executorRunTime","Update":77,"Value":400,"Internal":true,"Count Failed Values":true},{"ID":16,"Name":"internal.metrics.executorDeserializeCpuTime","Update":221708000,"Value":1095356000,"Internal":true,"Count Failed Values":true},{"ID":15,"Name":"internal.metrics.executorDeserializeTime","Update":962,"Value":4811,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":155100856,"JVMOffHeapMemory":64239224,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":6964,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":6964,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1852,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":7,"MinorGCTime":33,"MajorGCCount":3,"MajorGCTime":110},"Task Metrics":{"Executor Deserialize Time":962,"Executor Deserialize CPU Time":221708000,"Executor Run Time":77,"Executor CPU Time":10820000,"Peak Execution Memory":0,"Result Size":1384,"JVM GC Time":109,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":11,"Index":11,"Attempt":0,"Launch Time":1594091479273,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480368,"Failed":false,"Killed":false,"Accumulables":[{"ID":13,"Name":"number of output rows","Update":"2","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":20,"Name":"internal.metrics.jvmGCTime","Update":109,"Value":654,"Internal":true,"Count Failed Values":true},{"ID":19,"Name":"internal.metrics.resultSize","Update":1406,"Value":8345,"Internal":true,"Count Failed Values":true},{"ID":18,"Name":"internal.metrics.executorCpuTime","Update":13213000,"Value":61060000,"Internal":true,"Count Failed Values":true},{"ID":17,"Name":"internal.metrics.executorRunTime","Update":78,"Value":478,"Internal":true,"Count Failed Values":true},{"ID":16,"Name":"internal.metrics.executorDeserializeCpuTime","Update":260380000,"Value":1355736000,"Internal":true,"Count Failed Values":true},{"ID":15,"Name":"internal.metrics.executorDeserializeTime","Update":963,"Value":5774,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":155100856,"JVMOffHeapMemory":64239224,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":6964,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":6964,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1852,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":7,"MinorGCTime":33,"MajorGCCount":3,"MajorGCTime":110},"Task Metrics":{"Executor Deserialize Time":963,"Executor Deserialize CPU Time":260380000,"Executor Run Time":78,"Executor CPU Time":13213000,"Peak Execution Memory":0,"Result Size":1406,"JVM GC Time":109,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":10,"Index":10,"Attempt":0,"Launch Time":1594091479273,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480368,"Failed":false,"Killed":false,"Accumulables":[{"ID":13,"Name":"number of output rows","Update":"1","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":20,"Name":"internal.metrics.jvmGCTime","Update":109,"Value":763,"Internal":true,"Count Failed Values":true},{"ID":19,"Name":"internal.metrics.resultSize","Update":1391,"Value":9736,"Internal":true,"Count Failed Values":true},{"ID":18,"Name":"internal.metrics.executorCpuTime","Update":9913000,"Value":70973000,"Internal":true,"Count Failed Values":true},{"ID":17,"Name":"internal.metrics.executorRunTime","Update":76,"Value":554,"Internal":true,"Count Failed Values":true},{"ID":16,"Name":"internal.metrics.executorDeserializeCpuTime","Update":210788000,"Value":1566524000,"Internal":true,"Count Failed Values":true},{"ID":15,"Name":"internal.metrics.executorDeserializeTime","Update":962,"Value":6736,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":155100856,"JVMOffHeapMemory":64239224,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":6964,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":6964,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1852,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":7,"MinorGCTime":33,"MajorGCCount":3,"MajorGCTime":110},"Task Metrics":{"Executor Deserialize Time":962,"Executor Deserialize CPU Time":210788000,"Executor Run Time":76,"Executor CPU Time":9913000,"Peak Execution Memory":0,"Result Size":1391,"JVM GC Time":109,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":14,"Index":14,"Attempt":0,"Launch Time":1594091479274,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480368,"Failed":false,"Killed":false,"Accumulables":[{"ID":13,"Name":"number of output rows","Update":"1","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":20,"Name":"internal.metrics.jvmGCTime","Update":109,"Value":872,"Internal":true,"Count Failed Values":true},{"ID":19,"Name":"internal.metrics.resultSize","Update":1391,"Value":11127,"Internal":true,"Count Failed Values":true},{"ID":18,"Name":"internal.metrics.executorCpuTime","Update":8784000,"Value":79757000,"Internal":true,"Count Failed Values":true},{"ID":17,"Name":"internal.metrics.executorRunTime","Update":79,"Value":633,"Internal":true,"Count Failed Values":true},{"ID":16,"Name":"internal.metrics.executorDeserializeCpuTime","Update":235620000,"Value":1802144000,"Internal":true,"Count Failed Values":true},{"ID":15,"Name":"internal.metrics.executorDeserializeTime","Update":964,"Value":7700,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":155100856,"JVMOffHeapMemory":64239224,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":6964,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":6964,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1852,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":7,"MinorGCTime":33,"MajorGCCount":3,"MajorGCTime":110},"Task Metrics":{"Executor Deserialize Time":964,"Executor Deserialize CPU Time":235620000,"Executor Run Time":79,"Executor CPU Time":8784000,"Peak Execution Memory":0,"Result Size":1391,"JVM GC Time":109,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":12,"Index":12,"Attempt":0,"Launch Time":1594091479274,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480369,"Failed":false,"Killed":false,"Accumulables":[{"ID":13,"Name":"number of output rows","Update":"1","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":20,"Name":"internal.metrics.jvmGCTime","Update":109,"Value":981,"Internal":true,"Count Failed Values":true},{"ID":19,"Name":"internal.metrics.resultSize","Update":1391,"Value":12518,"Internal":true,"Count Failed Values":true},{"ID":18,"Name":"internal.metrics.executorCpuTime","Update":12053000,"Value":91810000,"Internal":true,"Count Failed Values":true},{"ID":17,"Name":"internal.metrics.executorRunTime","Update":86,"Value":719,"Internal":true,"Count Failed Values":true},{"ID":16,"Name":"internal.metrics.executorDeserializeCpuTime","Update":215398000,"Value":2017542000,"Internal":true,"Count Failed Values":true},{"ID":15,"Name":"internal.metrics.executorDeserializeTime","Update":963,"Value":8663,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":155100856,"JVMOffHeapMemory":64239224,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":6964,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":6964,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1852,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":7,"MinorGCTime":33,"MajorGCCount":3,"MajorGCTime":110},"Task Metrics":{"Executor Deserialize Time":963,"Executor Deserialize CPU Time":215398000,"Executor Run Time":86,"Executor CPU Time":12053000,"Peak Execution Memory":0,"Result Size":1391,"JVM GC Time":109,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":3,"Index":3,"Attempt":0,"Launch Time":1594091479270,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480369,"Failed":false,"Killed":false,"Accumulables":[{"ID":13,"Name":"number of output rows","Update":"2","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":20,"Name":"internal.metrics.jvmGCTime","Update":109,"Value":1090,"Internal":true,"Count Failed Values":true},{"ID":19,"Name":"internal.metrics.resultSize","Update":1403,"Value":13921,"Internal":true,"Count Failed Values":true},{"ID":18,"Name":"internal.metrics.executorCpuTime","Update":9030000,"Value":100840000,"Internal":true,"Count Failed Values":true},{"ID":17,"Name":"internal.metrics.executorRunTime","Update":79,"Value":798,"Internal":true,"Count Failed Values":true},{"ID":16,"Name":"internal.metrics.executorDeserializeCpuTime","Update":196266000,"Value":2213808000,"Internal":true,"Count Failed Values":true},{"ID":15,"Name":"internal.metrics.executorDeserializeTime","Update":962,"Value":9625,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":155100856,"JVMOffHeapMemory":64239224,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":6964,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":6964,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1852,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":7,"MinorGCTime":33,"MajorGCCount":3,"MajorGCTime":110},"Task Metrics":{"Executor Deserialize Time":962,"Executor Deserialize CPU Time":196266000,"Executor Run Time":79,"Executor CPU Time":9030000,"Peak Execution Memory":0,"Result Size":1403,"JVM GC Time":109,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":9,"Index":9,"Attempt":0,"Launch Time":1594091479273,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480369,"Failed":false,"Killed":false,"Accumulables":[{"ID":13,"Name":"number of output rows","Update":"1","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":20,"Name":"internal.metrics.jvmGCTime","Update":109,"Value":1199,"Internal":true,"Count Failed Values":true},{"ID":19,"Name":"internal.metrics.resultSize","Update":1390,"Value":15311,"Internal":true,"Count Failed Values":true},{"ID":18,"Name":"internal.metrics.executorCpuTime","Update":10087000,"Value":110927000,"Internal":true,"Count Failed Values":true},{"ID":17,"Name":"internal.metrics.executorRunTime","Update":79,"Value":877,"Internal":true,"Count Failed Values":true},{"ID":16,"Name":"internal.metrics.executorDeserializeCpuTime","Update":195342000,"Value":2409150000,"Internal":true,"Count Failed Values":true},{"ID":15,"Name":"internal.metrics.executorDeserializeTime","Update":962,"Value":10587,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":155100856,"JVMOffHeapMemory":64239224,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":6964,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":6964,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1852,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":7,"MinorGCTime":33,"MajorGCCount":3,"MajorGCTime":110},"Task Metrics":{"Executor Deserialize Time":962,"Executor Deserialize CPU Time":195342000,"Executor Run Time":79,"Executor CPU Time":10087000,"Peak Execution Memory":0,"Result Size":1390,"JVM GC Time":109,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":15,"Index":15,"Attempt":0,"Launch Time":1594091479274,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480369,"Failed":false,"Killed":false,"Accumulables":[{"ID":13,"Name":"number of output rows","Update":"2","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":20,"Name":"internal.metrics.jvmGCTime","Update":109,"Value":1308,"Internal":true,"Count Failed Values":true},{"ID":19,"Name":"internal.metrics.resultSize","Update":1411,"Value":16722,"Internal":true,"Count Failed Values":true},{"ID":18,"Name":"internal.metrics.executorCpuTime","Update":12920000,"Value":123847000,"Internal":true,"Count Failed Values":true},{"ID":17,"Name":"internal.metrics.executorRunTime","Update":86,"Value":963,"Internal":true,"Count Failed Values":true},{"ID":16,"Name":"internal.metrics.executorDeserializeCpuTime","Update":236044000,"Value":2645194000,"Internal":true,"Count Failed Values":true},{"ID":15,"Name":"internal.metrics.executorDeserializeTime","Update":961,"Value":11548,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":155100856,"JVMOffHeapMemory":64239224,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":6964,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":6964,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1852,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":7,"MinorGCTime":33,"MajorGCCount":3,"MajorGCTime":110},"Task Metrics":{"Executor Deserialize Time":961,"Executor Deserialize CPU Time":236044000,"Executor Run Time":86,"Executor CPU Time":12920000,"Peak Execution Memory":0,"Result Size":1411,"JVM GC Time":109,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":5,"Index":5,"Attempt":0,"Launch Time":1594091479270,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480370,"Failed":false,"Killed":false,"Accumulables":[{"ID":13,"Name":"number of output rows","Update":"1","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":21,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":20,"Name":"internal.metrics.jvmGCTime","Update":109,"Value":1417,"Internal":true,"Count Failed Values":true},{"ID":19,"Name":"internal.metrics.resultSize","Update":1430,"Value":18152,"Internal":true,"Count Failed Values":true},{"ID":18,"Name":"internal.metrics.executorCpuTime","Update":10478000,"Value":134325000,"Internal":true,"Count Failed Values":true},{"ID":17,"Name":"internal.metrics.executorRunTime","Update":77,"Value":1040,"Internal":true,"Count Failed Values":true},{"ID":16,"Name":"internal.metrics.executorDeserializeCpuTime","Update":205925000,"Value":2851119000,"Internal":true,"Count Failed Values":true},{"ID":15,"Name":"internal.metrics.executorDeserializeTime","Update":962,"Value":12510,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":155100856,"JVMOffHeapMemory":64239224,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":6964,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":6964,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1852,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":7,"MinorGCTime":33,"MajorGCCount":3,"MajorGCTime":110},"Task Metrics":{"Executor Deserialize Time":962,"Executor Deserialize CPU Time":205925000,"Executor Run Time":77,"Executor CPU Time":10478000,"Peak Execution Memory":0,"Result Size":1430,"JVM GC Time":109,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":4,"Index":4,"Attempt":0,"Launch Time":1594091479270,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480370,"Failed":false,"Killed":false,"Accumulables":[{"ID":13,"Name":"number of output rows","Update":"1","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":20,"Name":"internal.metrics.jvmGCTime","Update":109,"Value":1526,"Internal":true,"Count Failed Values":true},{"ID":19,"Name":"internal.metrics.resultSize","Update":1387,"Value":19539,"Internal":true,"Count Failed Values":true},{"ID":18,"Name":"internal.metrics.executorCpuTime","Update":8972000,"Value":143297000,"Internal":true,"Count Failed Values":true},{"ID":17,"Name":"internal.metrics.executorRunTime","Update":87,"Value":1127,"Internal":true,"Count Failed Values":true},{"ID":16,"Name":"internal.metrics.executorDeserializeCpuTime","Update":206247000,"Value":3057366000,"Internal":true,"Count Failed Values":true},{"ID":15,"Name":"internal.metrics.executorDeserializeTime","Update":963,"Value":13473,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":155100856,"JVMOffHeapMemory":64239224,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":6964,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":6964,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1852,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":7,"MinorGCTime":33,"MajorGCCount":3,"MajorGCTime":110},"Task Metrics":{"Executor Deserialize Time":963,"Executor Deserialize CPU Time":206247000,"Executor Run Time":87,"Executor CPU Time":8972000,"Peak Execution Memory":0,"Result Size":1387,"JVM GC Time":109,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":7,"Index":7,"Attempt":0,"Launch Time":1594091479272,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480370,"Failed":false,"Killed":false,"Accumulables":[{"ID":13,"Name":"number of output rows","Update":"2","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":21,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":20,"Name":"internal.metrics.jvmGCTime","Update":109,"Value":1635,"Internal":true,"Count Failed Values":true},{"ID":19,"Name":"internal.metrics.resultSize","Update":1452,"Value":20991,"Internal":true,"Count Failed Values":true},{"ID":18,"Name":"internal.metrics.executorCpuTime","Update":20898000,"Value":164195000,"Internal":true,"Count Failed Values":true},{"ID":17,"Name":"internal.metrics.executorRunTime","Update":83,"Value":1210,"Internal":true,"Count Failed Values":true},{"ID":16,"Name":"internal.metrics.executorDeserializeCpuTime","Update":389356000,"Value":3446722000,"Internal":true,"Count Failed Values":true},{"ID":15,"Name":"internal.metrics.executorDeserializeTime","Update":963,"Value":14436,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":155100856,"JVMOffHeapMemory":64239224,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":6964,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":6964,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1852,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":7,"MinorGCTime":33,"MajorGCCount":3,"MajorGCTime":110},"Task Metrics":{"Executor Deserialize Time":963,"Executor Deserialize CPU Time":389356000,"Executor Run Time":83,"Executor CPU Time":20898000,"Peak Execution Memory":0,"Result Size":1452,"JVM GC Time":109,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Launch Time":1594091479269,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480370,"Failed":false,"Killed":false,"Accumulables":[{"ID":13,"Name":"number of output rows","Update":"1","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":20,"Name":"internal.metrics.jvmGCTime","Update":109,"Value":1744,"Internal":true,"Count Failed Values":true},{"ID":19,"Name":"internal.metrics.resultSize","Update":1384,"Value":22375,"Internal":true,"Count Failed Values":true},{"ID":18,"Name":"internal.metrics.executorCpuTime","Update":8668000,"Value":172863000,"Internal":true,"Count Failed Values":true},{"ID":17,"Name":"internal.metrics.executorRunTime","Update":82,"Value":1292,"Internal":true,"Count Failed Values":true},{"ID":16,"Name":"internal.metrics.executorDeserializeCpuTime","Update":222167000,"Value":3668889000,"Internal":true,"Count Failed Values":true},{"ID":15,"Name":"internal.metrics.executorDeserializeTime","Update":963,"Value":15399,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":155100856,"JVMOffHeapMemory":64239224,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":6964,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":6964,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1852,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":7,"MinorGCTime":33,"MajorGCCount":3,"MajorGCTime":110},"Task Metrics":{"Executor Deserialize Time":963,"Executor Deserialize CPU Time":222167000,"Executor Run Time":82,"Executor CPU Time":8668000,"Peak Execution Memory":0,"Result Size":1384,"JVM GC Time":109,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerStageExecutorMetrics","Executor ID":"0","Stage ID":0,"Stage Attempt ID":0,"Executor Metrics":{"JVMHeapMemory":155100856,"JVMOffHeapMemory":64239224,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":6964,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":6964,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1852,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":7,"MinorGCTime":33,"MajorGCCount":3,"MajorGCTime":110}}
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Number of Tasks":16,"RDD Info":[{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"DeserializeToObject\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"3\",\"name\":\"LocalTableScan\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"LocalTableScan\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:185)\njava.util.concurrent.FutureTask.run(FutureTask.java:266)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1594091478860,"Completion Time":1594091480381,"Accumulables":[{"ID":17,"Name":"internal.metrics.executorRunTime","Value":1292,"Internal":true,"Count Failed Values":true},{"ID":20,"Name":"internal.metrics.jvmGCTime","Value":1744,"Internal":true,"Count Failed Values":true},{"ID":13,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":16,"Name":"internal.metrics.executorDeserializeCpuTime","Value":3668889000,"Internal":true,"Count Failed Values":true},{"ID":19,"Name":"internal.metrics.resultSize","Value":22375,"Internal":true,"Count Failed Values":true},{"ID":18,"Name":"internal.metrics.executorCpuTime","Value":172863000,"Internal":true,"Count Failed Values":true},{"ID":21,"Name":"internal.metrics.resultSerializationTime","Value":3,"Internal":true,"Count Failed Values":true},{"ID":15,"Name":"internal.metrics.executorDeserializeTime","Value":15399,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}}
+{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1594091480385,"Job Result":{"Result":"JobSucceeded"}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[9,1048632],[10,1736],[11,37],[12,5]]}
+{"Event":"SparkListenerJobStart","Job ID":1,"Submission Time":1594091480498,"Stage Infos":[{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Number of Tasks":16,"RDD Info":[{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"DeserializeToObject\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"2\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:185)\njava.util.concurrent.FutureTask.run(FutureTask.java:266)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[1],"Properties":{"spark.driver.host":"localhost","spark.eventLog.enabled":"true","spark.job.interruptOnCancel":"true","spark.driver.port":"64413","spark.repl.class.uri":"spark://localhost:64413/classes","spark.jars":"","spark.repl.class.outputDir":"/private/var/folders/p_/5j1mtw1x0579b34vy4ztcbs40000gn/T/spark-0c713088-21d7-4b35-9b52-7a4dd4d15ae1/repl-361cef81-42cd-4875-a8a6-ce9dfe55682a","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"0\",\"name\":\"DeserializeToObject\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.job.description":"broadcast exchange (runId bac7289a-c1d8-4966-a6a3-d9f347e13a5d)","spark.executor.id":"driver","spark.jobGroup.id":"bac7289a-c1d8-4966-a6a3-d9f347e13a5d","spark.eventLog.logStageExecutorMetrics":"true","spark.submit.deployMode":"client","spark.master":"*********(redacted)","spark.home":"*********(redacted)","spark.sql.execution.id":"0","spark.sql.catalogImplementation":"in-memory","spark.app.id":"app-20200706201101-0003"}}
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Number of Tasks":16,"RDD Info":[{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"DeserializeToObject\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"2\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:185)\njava.util.concurrent.FutureTask.run(FutureTask.java:266)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1594091480499,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.driver.host":"localhost","spark.eventLog.enabled":"true","spark.job.interruptOnCancel":"true","spark.driver.port":"64413","spark.repl.class.uri":"spark://localhost:64413/classes","spark.jars":"","spark.repl.class.outputDir":"/private/var/folders/p_/5j1mtw1x0579b34vy4ztcbs40000gn/T/spark-0c713088-21d7-4b35-9b52-7a4dd4d15ae1/repl-361cef81-42cd-4875-a8a6-ce9dfe55682a","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"0\",\"name\":\"DeserializeToObject\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.job.description":"broadcast exchange (runId bac7289a-c1d8-4966-a6a3-d9f347e13a5d)","spark.executor.id":"driver","spark.jobGroup.id":"bac7289a-c1d8-4966-a6a3-d9f347e13a5d","spark.eventLog.logStageExecutorMetrics":"true","spark.submit.deployMode":"client","spark.master":"*********(redacted)","spark.home":"*********(redacted)","spark.sql.execution.id":"0","spark.sql.catalogImplementation":"in-memory","spark.app.id":"app-20200706201101-0003"}}
+{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":16,"Index":0,"Attempt":0,"Launch Time":1594091480502,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":17,"Index":1,"Attempt":0,"Launch Time":1594091480503,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":18,"Index":2,"Attempt":0,"Launch Time":1594091480503,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":19,"Index":3,"Attempt":0,"Launch Time":1594091480503,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":20,"Index":4,"Attempt":0,"Launch Time":1594091480503,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":21,"Index":5,"Attempt":0,"Launch Time":1594091480504,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":22,"Index":6,"Attempt":0,"Launch Time":1594091480504,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":23,"Index":7,"Attempt":0,"Launch Time":1594091480504,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":24,"Index":8,"Attempt":0,"Launch Time":1594091480504,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":25,"Index":9,"Attempt":0,"Launch Time":1594091480504,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":26,"Index":10,"Attempt":0,"Launch Time":1594091480505,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":27,"Index":11,"Attempt":0,"Launch Time":1594091480505,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":28,"Index":12,"Attempt":0,"Launch Time":1594091480506,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":29,"Index":13,"Attempt":0,"Launch Time":1594091480506,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":30,"Index":14,"Attempt":0,"Launch Time":1594091480506,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":31,"Index":15,"Attempt":0,"Launch Time":1594091480507,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":20,"Index":4,"Attempt":0,"Launch Time":1594091480503,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480921,"Failed":false,"Killed":false,"Accumulables":[{"ID":6,"Name":"duration","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":7,"Name":"number of output rows","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":8,"Name":"number of output rows","Update":"6","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":49,"Name":"internal.metrics.peakExecutionMemory","Update":1016,"Value":1016,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.resultSize","Update":1966,"Value":1966,"Internal":true,"Count Failed Values":true},{"ID":43,"Name":"internal.metrics.executorCpuTime","Update":3116000,"Value":3116000,"Internal":true,"Count Failed Values":true},{"ID":42,"Name":"internal.metrics.executorRunTime","Update":378,"Value":378,"Internal":true,"Count Failed Values":true},{"ID":41,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4258000,"Value":4258000,"Internal":true,"Count Failed Values":true},{"ID":40,"Name":"internal.metrics.executorDeserializeTime","Update":29,"Value":29,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":29,"Executor Deserialize CPU Time":4258000,"Executor Run Time":378,"Executor CPU Time":3116000,"Peak Execution Memory":1016,"Result Size":1966,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":17,"Index":1,"Attempt":0,"Launch Time":1594091480503,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480921,"Failed":false,"Killed":false,"Accumulables":[{"ID":6,"Name":"duration","Update":"20","Value":"40","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":7,"Name":"number of output rows","Update":"18","Value":"36","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":8,"Name":"number of output rows","Update":"6","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":49,"Name":"internal.metrics.peakExecutionMemory","Update":1016,"Value":2032,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.resultSize","Update":1951,"Value":3917,"Internal":true,"Count Failed Values":true},{"ID":43,"Name":"internal.metrics.executorCpuTime","Update":2895000,"Value":6011000,"Internal":true,"Count Failed Values":true},{"ID":42,"Name":"internal.metrics.executorRunTime","Update":379,"Value":757,"Internal":true,"Count Failed Values":true},{"ID":41,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2641000,"Value":6899000,"Internal":true,"Count Failed Values":true},{"ID":40,"Name":"internal.metrics.executorDeserializeTime","Update":29,"Value":58,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":29,"Executor Deserialize CPU Time":2641000,"Executor Run Time":379,"Executor CPU Time":2895000,"Peak Execution Memory":1016,"Result Size":1951,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":21,"Index":5,"Attempt":0,"Launch Time":1594091480504,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480923,"Failed":false,"Killed":false,"Accumulables":[{"ID":6,"Name":"duration","Update":"20","Value":"60","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":7,"Name":"number of output rows","Update":"18","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":8,"Name":"number of output rows","Update":"6","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":49,"Name":"internal.metrics.peakExecutionMemory","Update":1016,"Value":3048,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.resultSize","Update":1957,"Value":5874,"Internal":true,"Count Failed Values":true},{"ID":43,"Name":"internal.metrics.executorCpuTime","Update":4164000,"Value":10175000,"Internal":true,"Count Failed Values":true},{"ID":42,"Name":"internal.metrics.executorRunTime","Update":380,"Value":1137,"Internal":true,"Count Failed Values":true},{"ID":41,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2012000,"Value":8911000,"Internal":true,"Count Failed Values":true},{"ID":40,"Name":"internal.metrics.executorDeserializeTime","Update":28,"Value":86,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":28,"Executor Deserialize CPU Time":2012000,"Executor Run Time":380,"Executor CPU Time":4164000,"Peak Execution Memory":1016,"Result Size":1957,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":25,"Index":9,"Attempt":0,"Launch Time":1594091480504,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480923,"Failed":false,"Killed":false,"Accumulables":[{"ID":6,"Name":"duration","Update":"20","Value":"80","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":7,"Name":"number of output rows","Update":"18","Value":"72","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":8,"Name":"number of output rows","Update":"6","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":49,"Name":"internal.metrics.peakExecutionMemory","Update":1016,"Value":4064,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.resultSize","Update":1957,"Value":7831,"Internal":true,"Count Failed Values":true},{"ID":43,"Name":"internal.metrics.executorCpuTime","Update":4285000,"Value":14460000,"Internal":true,"Count Failed Values":true},{"ID":42,"Name":"internal.metrics.executorRunTime","Update":380,"Value":1517,"Internal":true,"Count Failed Values":true},{"ID":41,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2749000,"Value":11660000,"Internal":true,"Count Failed Values":true},{"ID":40,"Name":"internal.metrics.executorDeserializeTime","Update":26,"Value":112,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":26,"Executor Deserialize CPU Time":2749000,"Executor Run Time":380,"Executor CPU Time":4285000,"Peak Execution Memory":1016,"Result Size":1957,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":23,"Index":7,"Attempt":0,"Launch Time":1594091480504,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480924,"Failed":false,"Killed":false,"Accumulables":[{"ID":6,"Name":"duration","Update":"20","Value":"100","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":7,"Name":"number of output rows","Update":"21","Value":"93","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":8,"Name":"number of output rows","Update":"7","Value":"31","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":49,"Name":"internal.metrics.peakExecutionMemory","Update":1016,"Value":5080,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.resultSize","Update":2007,"Value":9838,"Internal":true,"Count Failed Values":true},{"ID":43,"Name":"internal.metrics.executorCpuTime","Update":16921000,"Value":31381000,"Internal":true,"Count Failed Values":true},{"ID":42,"Name":"internal.metrics.executorRunTime","Update":383,"Value":1900,"Internal":true,"Count Failed Values":true},{"ID":41,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3854000,"Value":15514000,"Internal":true,"Count Failed Values":true},{"ID":40,"Name":"internal.metrics.executorDeserializeTime","Update":26,"Value":138,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":26,"Executor Deserialize CPU Time":3854000,"Executor Run Time":383,"Executor CPU Time":16921000,"Peak Execution Memory":1016,"Result Size":2007,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":30,"Index":14,"Attempt":0,"Launch Time":1594091480506,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480925,"Failed":false,"Killed":false,"Accumulables":[{"ID":6,"Name":"duration","Update":"20","Value":"120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":7,"Name":"number of output rows","Update":"18","Value":"111","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":8,"Name":"number of output rows","Update":"6","Value":"37","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":49,"Name":"internal.metrics.peakExecutionMemory","Update":1016,"Value":6096,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.resultSize","Update":1955,"Value":11793,"Internal":true,"Count Failed Values":true},{"ID":43,"Name":"internal.metrics.executorCpuTime","Update":4798000,"Value":36179000,"Internal":true,"Count Failed Values":true},{"ID":42,"Name":"internal.metrics.executorRunTime","Update":382,"Value":2282,"Internal":true,"Count Failed Values":true},{"ID":41,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2959000,"Value":18473000,"Internal":true,"Count Failed Values":true},{"ID":40,"Name":"internal.metrics.executorDeserializeTime","Update":24,"Value":162,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":24,"Executor Deserialize CPU Time":2959000,"Executor Run Time":382,"Executor CPU Time":4798000,"Peak Execution Memory":1016,"Result Size":1955,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":27,"Index":11,"Attempt":0,"Launch Time":1594091480505,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480926,"Failed":false,"Killed":false,"Accumulables":[{"ID":6,"Name":"duration","Update":"20","Value":"140","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":7,"Name":"number of output rows","Update":"21","Value":"132","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":8,"Name":"number of output rows","Update":"7","Value":"44","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":49,"Name":"internal.metrics.peakExecutionMemory","Update":1016,"Value":7112,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.resultSize","Update":2015,"Value":13808,"Internal":true,"Count Failed Values":true},{"ID":43,"Name":"internal.metrics.executorCpuTime","Update":4850000,"Value":41029000,"Internal":true,"Count Failed Values":true},{"ID":42,"Name":"internal.metrics.executorRunTime","Update":382,"Value":2664,"Internal":true,"Count Failed Values":true},{"ID":41,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4278000,"Value":22751000,"Internal":true,"Count Failed Values":true},{"ID":40,"Name":"internal.metrics.executorDeserializeTime","Update":25,"Value":187,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":25,"Executor Deserialize CPU Time":4278000,"Executor Run Time":382,"Executor CPU Time":4850000,"Peak Execution Memory":1016,"Result Size":2015,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":19,"Index":3,"Attempt":0,"Launch Time":1594091480503,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480927,"Failed":false,"Killed":false,"Accumulables":[{"ID":6,"Name":"duration","Update":"21","Value":"161","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":7,"Name":"number of output rows","Update":"21","Value":"153","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":8,"Name":"number of output rows","Update":"7","Value":"51","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":49,"Name":"internal.metrics.peakExecutionMemory","Update":1016,"Value":8128,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.resultSize","Update":1998,"Value":15806,"Internal":true,"Count Failed Values":true},{"ID":43,"Name":"internal.metrics.executorCpuTime","Update":4100000,"Value":45129000,"Internal":true,"Count Failed Values":true},{"ID":42,"Name":"internal.metrics.executorRunTime","Update":383,"Value":3047,"Internal":true,"Count Failed Values":true},{"ID":41,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3622000,"Value":26373000,"Internal":true,"Count Failed Values":true},{"ID":40,"Name":"internal.metrics.executorDeserializeTime","Update":28,"Value":215,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":28,"Executor Deserialize CPU Time":3622000,"Executor Run Time":383,"Executor CPU Time":4100000,"Peak Execution Memory":1016,"Result Size":1998,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":26,"Index":10,"Attempt":0,"Launch Time":1594091480505,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480927,"Failed":false,"Killed":false,"Accumulables":[{"ID":6,"Name":"duration","Update":"21","Value":"182","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":7,"Name":"number of output rows","Update":"18","Value":"171","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":8,"Name":"number of output rows","Update":"6","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":49,"Name":"internal.metrics.peakExecutionMemory","Update":1016,"Value":9144,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.resultSize","Update":1973,"Value":17779,"Internal":true,"Count Failed Values":true},{"ID":43,"Name":"internal.metrics.executorCpuTime","Update":4350000,"Value":49479000,"Internal":true,"Count Failed Values":true},{"ID":42,"Name":"internal.metrics.executorRunTime","Update":383,"Value":3430,"Internal":true,"Count Failed Values":true},{"ID":41,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3471000,"Value":29844000,"Internal":true,"Count Failed Values":true},{"ID":40,"Name":"internal.metrics.executorDeserializeTime","Update":25,"Value":240,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":25,"Executor Deserialize CPU Time":3471000,"Executor Run Time":383,"Executor CPU Time":4350000,"Peak Execution Memory":1016,"Result Size":1973,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":28,"Index":12,"Attempt":0,"Launch Time":1594091480506,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480927,"Failed":false,"Killed":false,"Accumulables":[{"ID":6,"Name":"duration","Update":"20","Value":"202","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":7,"Name":"number of output rows","Update":"18","Value":"189","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":8,"Name":"number of output rows","Update":"6","Value":"63","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":49,"Name":"internal.metrics.peakExecutionMemory","Update":1016,"Value":10160,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.resultSize","Update":1966,"Value":19745,"Internal":true,"Count Failed Values":true},{"ID":43,"Name":"internal.metrics.executorCpuTime","Update":4682000,"Value":54161000,"Internal":true,"Count Failed Values":true},{"ID":42,"Name":"internal.metrics.executorRunTime","Update":384,"Value":3814,"Internal":true,"Count Failed Values":true},{"ID":41,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2652000,"Value":32496000,"Internal":true,"Count Failed Values":true},{"ID":40,"Name":"internal.metrics.executorDeserializeTime","Update":23,"Value":263,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":23,"Executor Deserialize CPU Time":2652000,"Executor Run Time":384,"Executor CPU Time":4682000,"Peak Execution Memory":1016,"Result Size":1966,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":31,"Index":15,"Attempt":0,"Launch Time":1594091480507,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480928,"Failed":false,"Killed":false,"Accumulables":[{"ID":6,"Name":"duration","Update":"21","Value":"223","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":7,"Name":"number of output rows","Update":"21","Value":"210","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":8,"Name":"number of output rows","Update":"7","Value":"70","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":49,"Name":"internal.metrics.peakExecutionMemory","Update":1016,"Value":11176,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.resultSize","Update":2008,"Value":21753,"Internal":true,"Count Failed Values":true},{"ID":43,"Name":"internal.metrics.executorCpuTime","Update":3954000,"Value":58115000,"Internal":true,"Count Failed Values":true},{"ID":42,"Name":"internal.metrics.executorRunTime","Update":382,"Value":4196,"Internal":true,"Count Failed Values":true},{"ID":41,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4289000,"Value":36785000,"Internal":true,"Count Failed Values":true},{"ID":40,"Name":"internal.metrics.executorDeserializeTime","Update":25,"Value":288,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":25,"Executor Deserialize CPU Time":4289000,"Executor Run Time":382,"Executor CPU Time":3954000,"Peak Execution Memory":1016,"Result Size":2008,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":18,"Index":2,"Attempt":0,"Launch Time":1594091480503,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480928,"Failed":false,"Killed":false,"Accumulables":[{"ID":6,"Name":"duration","Update":"20","Value":"243","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":7,"Name":"number of output rows","Update":"18","Value":"228","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":8,"Name":"number of output rows","Update":"6","Value":"76","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":49,"Name":"internal.metrics.peakExecutionMemory","Update":1016,"Value":12192,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.resultSize","Update":1978,"Value":23731,"Internal":true,"Count Failed Values":true},{"ID":43,"Name":"internal.metrics.executorCpuTime","Update":349926000,"Value":408041000,"Internal":true,"Count Failed Values":true},{"ID":42,"Name":"internal.metrics.executorRunTime","Update":385,"Value":4581,"Internal":true,"Count Failed Values":true},{"ID":41,"Name":"internal.metrics.executorDeserializeCpuTime","Update":14543000,"Value":51328000,"Internal":true,"Count Failed Values":true},{"ID":40,"Name":"internal.metrics.executorDeserializeTime","Update":27,"Value":315,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":27,"Executor Deserialize CPU Time":14543000,"Executor Run Time":385,"Executor CPU Time":349926000,"Peak Execution Memory":1016,"Result Size":1978,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":22,"Index":6,"Attempt":0,"Launch Time":1594091480504,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480928,"Failed":false,"Killed":false,"Accumulables":[{"ID":6,"Name":"duration","Update":"20","Value":"263","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":7,"Name":"number of output rows","Update":"18","Value":"246","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":8,"Name":"number of output rows","Update":"6","Value":"82","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":49,"Name":"internal.metrics.peakExecutionMemory","Update":1016,"Value":13208,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.resultSize","Update":1965,"Value":25696,"Internal":true,"Count Failed Values":true},{"ID":43,"Name":"internal.metrics.executorCpuTime","Update":4473000,"Value":412514000,"Internal":true,"Count Failed Values":true},{"ID":42,"Name":"internal.metrics.executorRunTime","Update":381,"Value":4962,"Internal":true,"Count Failed Values":true},{"ID":41,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3906000,"Value":55234000,"Internal":true,"Count Failed Values":true},{"ID":40,"Name":"internal.metrics.executorDeserializeTime","Update":27,"Value":342,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":27,"Executor Deserialize CPU Time":3906000,"Executor Run Time":381,"Executor CPU Time":4473000,"Peak Execution Memory":1016,"Result Size":1965,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":29,"Index":13,"Attempt":0,"Launch Time":1594091480506,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480929,"Failed":false,"Killed":false,"Accumulables":[{"ID":6,"Name":"duration","Update":"20","Value":"283","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":7,"Name":"number of output rows","Update":"18","Value":"264","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":8,"Name":"number of output rows","Update":"6","Value":"88","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":49,"Name":"internal.metrics.peakExecutionMemory","Update":1016,"Value":14224,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.resultSize","Update":1955,"Value":27651,"Internal":true,"Count Failed Values":true},{"ID":43,"Name":"internal.metrics.executorCpuTime","Update":6459000,"Value":418973000,"Internal":true,"Count Failed Values":true},{"ID":42,"Name":"internal.metrics.executorRunTime","Update":381,"Value":5343,"Internal":true,"Count Failed Values":true},{"ID":41,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3298000,"Value":58532000,"Internal":true,"Count Failed Values":true},{"ID":40,"Name":"internal.metrics.executorDeserializeTime","Update":24,"Value":366,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":24,"Executor Deserialize CPU Time":3298000,"Executor Run Time":381,"Executor CPU Time":6459000,"Peak Execution Memory":1016,"Result Size":1955,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":24,"Index":8,"Attempt":0,"Launch Time":1594091480504,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480929,"Failed":false,"Killed":false,"Accumulables":[{"ID":6,"Name":"duration","Update":"21","Value":"304","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":7,"Name":"number of output rows","Update":"18","Value":"282","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":8,"Name":"number of output rows","Update":"6","Value":"94","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":49,"Name":"internal.metrics.peakExecutionMemory","Update":1016,"Value":15240,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.resultSize","Update":1954,"Value":29605,"Internal":true,"Count Failed Values":true},{"ID":43,"Name":"internal.metrics.executorCpuTime","Update":4365000,"Value":423338000,"Internal":true,"Count Failed Values":true},{"ID":42,"Name":"internal.metrics.executorRunTime","Update":380,"Value":5723,"Internal":true,"Count Failed Values":true},{"ID":41,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2411000,"Value":60943000,"Internal":true,"Count Failed Values":true},{"ID":40,"Name":"internal.metrics.executorDeserializeTime","Update":28,"Value":394,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":28,"Executor Deserialize CPU Time":2411000,"Executor Run Time":380,"Executor CPU Time":4365000,"Peak Execution Memory":1016,"Result Size":1954,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":16,"Index":0,"Attempt":0,"Launch Time":1594091480502,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091480930,"Failed":false,"Killed":false,"Accumulables":[{"ID":6,"Name":"duration","Update":"20","Value":"324","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":7,"Name":"number of output rows","Update":"18","Value":"300","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":8,"Name":"number of output rows","Update":"6","Value":"100","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":49,"Name":"internal.metrics.peakExecutionMemory","Update":1016,"Value":16256,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.resultSize","Update":1941,"Value":31546,"Internal":true,"Count Failed Values":true},{"ID":43,"Name":"internal.metrics.executorCpuTime","Update":3111000,"Value":426449000,"Internal":true,"Count Failed Values":true},{"ID":42,"Name":"internal.metrics.executorRunTime","Update":382,"Value":6105,"Internal":true,"Count Failed Values":true},{"ID":41,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2723000,"Value":63666000,"Internal":true,"Count Failed Values":true},{"ID":40,"Name":"internal.metrics.executorDeserializeTime","Update":30,"Value":424,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":30,"Executor Deserialize CPU Time":2723000,"Executor Run Time":382,"Executor CPU Time":3111000,"Peak Execution Memory":1016,"Result Size":1941,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerStageExecutorMetrics","Executor ID":"0","Stage ID":1,"Stage Attempt ID":0,"Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0}}
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Number of Tasks":16,"RDD Info":[{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"DeserializeToObject\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"2\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:185)\njava.util.concurrent.FutureTask.run(FutureTask.java:266)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1594091480499,"Completion Time":1594091480930,"Accumulables":[{"ID":41,"Name":"internal.metrics.executorDeserializeCpuTime","Value":63666000,"Internal":true,"Count Failed Values":true},{"ID":8,"Name":"number of output rows","Value":"100","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"internal.metrics.resultSize","Value":31546,"Internal":true,"Count Failed Values":true},{"ID":49,"Name":"internal.metrics.peakExecutionMemory","Value":16256,"Internal":true,"Count Failed Values":true},{"ID":40,"Name":"internal.metrics.executorDeserializeTime","Value":424,"Internal":true,"Count Failed Values":true},{"ID":7,"Name":"number of output rows","Value":"300","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"internal.metrics.executorCpuTime","Value":426449000,"Internal":true,"Count Failed Values":true},{"ID":6,"Name":"duration","Value":"324","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"internal.metrics.executorRunTime","Value":6105,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}}
+{"Event":"SparkListenerJobEnd","Job ID":1,"Completion Time":1594091480930,"Job Result":{"Result":"JobSucceeded"}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[2,1048616],[3,2276],[4,13],[5,2]]}
+{"Event":"SparkListenerJobStart","Job ID":2,"Submission Time":1594091481039,"Stage Infos":[{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"foreach at <console>:26","Number of Tasks":16,"RDD Info":[{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"18\",\"name\":\"mapPartitions\"}","Callsite":"foreach at <console>:26","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"DeserializeToObject\"}","Callsite":"foreach at <console>:26","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"1\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"foreach at <console>:26","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"foreach at <console>:26","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"SQLExecutionRDD","Callsite":"foreach at <console>:26","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.foreach(Dataset.scala:2862)\n$line19.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:26)\n$line19.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:30)\n$line19.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:32)\n$line19.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:34)\n$line19.$read$$iw$$iw$$iw$$iw.<init>(<console>:36)\n$line19.$read$$iw$$iw$$iw.<init>(<console>:38)\n$line19.$read$$iw$$iw.<init>(<console>:40)\n$line19.$read$$iw.<init>(<console>:42)\n$line19.$read.<init>(<console>:44)\n$line19.$read$.<init>(<console>:48)\n$line19.$read$.<clinit>(<console>)\n$line19.$eval$.$print$lzycompute(<console>:7)\n$line19.$eval$.$print(<console>:6)\n$line19.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[2],"Properties":{"spark.driver.host":"localhost","spark.eventLog.enabled":"true","spark.driver.port":"64413","spark.repl.class.uri":"spark://localhost:64413/classes","spark.jars":"","spark.repl.class.outputDir":"/private/var/folders/p_/5j1mtw1x0579b34vy4ztcbs40000gn/T/spark-0c713088-21d7-4b35-9b52-7a4dd4d15ae1/repl-361cef81-42cd-4875-a8a6-ce9dfe55682a","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"19\",\"name\":\"foreach\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.executor.id":"driver","spark.eventLog.logStageExecutorMetrics":"true","spark.submit.deployMode":"client","spark.master":"*********(redacted)","spark.home":"*********(redacted)","spark.sql.execution.id":"0","spark.sql.catalogImplementation":"in-memory","spark.app.id":"app-20200706201101-0003"}}
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"foreach at <console>:26","Number of Tasks":16,"RDD Info":[{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"18\",\"name\":\"mapPartitions\"}","Callsite":"foreach at <console>:26","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"DeserializeToObject\"}","Callsite":"foreach at <console>:26","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"1\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"foreach at <console>:26","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"foreach at <console>:26","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"SQLExecutionRDD","Callsite":"foreach at <console>:26","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.foreach(Dataset.scala:2862)\n$line19.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:26)\n$line19.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:30)\n$line19.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:32)\n$line19.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:34)\n$line19.$read$$iw$$iw$$iw$$iw.<init>(<console>:36)\n$line19.$read$$iw$$iw$$iw.<init>(<console>:38)\n$line19.$read$$iw$$iw.<init>(<console>:40)\n$line19.$read$$iw.<init>(<console>:42)\n$line19.$read.<init>(<console>:44)\n$line19.$read$.<init>(<console>:48)\n$line19.$read$.<clinit>(<console>)\n$line19.$eval$.$print$lzycompute(<console>:7)\n$line19.$eval$.$print(<console>:6)\n$line19.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Submission Time":1594091481040,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.driver.host":"localhost","spark.eventLog.enabled":"true","spark.driver.port":"64413","spark.repl.class.uri":"spark://localhost:64413/classes","spark.jars":"","spark.repl.class.outputDir":"/private/var/folders/p_/5j1mtw1x0579b34vy4ztcbs40000gn/T/spark-0c713088-21d7-4b35-9b52-7a4dd4d15ae1/repl-361cef81-42cd-4875-a8a6-ce9dfe55682a","spark.app.name":"Spark shell","spark.rdd.scope":"{\"id\":\"19\",\"name\":\"foreach\"}","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.executor.id":"driver","spark.eventLog.logStageExecutorMetrics":"true","spark.submit.deployMode":"client","spark.master":"*********(redacted)","spark.home":"*********(redacted)","spark.sql.execution.id":"0","spark.sql.catalogImplementation":"in-memory","spark.app.id":"app-20200706201101-0003"}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":32,"Index":0,"Attempt":0,"Launch Time":1594091481077,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":33,"Index":1,"Attempt":0,"Launch Time":1594091481082,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":34,"Index":2,"Attempt":0,"Launch Time":1594091481087,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":35,"Index":3,"Attempt":0,"Launch Time":1594091481091,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":36,"Index":4,"Attempt":0,"Launch Time":1594091481095,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":37,"Index":5,"Attempt":0,"Launch Time":1594091481100,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":38,"Index":6,"Attempt":0,"Launch Time":1594091481104,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":39,"Index":7,"Attempt":0,"Launch Time":1594091481109,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":40,"Index":8,"Attempt":0,"Launch Time":1594091481112,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":41,"Index":9,"Attempt":0,"Launch Time":1594091481116,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":42,"Index":10,"Attempt":0,"Launch Time":1594091481120,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":43,"Index":11,"Attempt":0,"Launch Time":1594091481123,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":44,"Index":12,"Attempt":0,"Launch Time":1594091481126,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":45,"Index":13,"Attempt":0,"Launch Time":1594091481129,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":46,"Index":14,"Attempt":0,"Launch Time":1594091481132,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":47,"Index":15,"Attempt":0,"Launch Time":1594091481136,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":38,"Index":6,"Attempt":0,"Launch Time":1594091481104,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091482939,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"duration","Update":"1467","Value":"1467","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1,"Name":"number of output rows","Update":"375000","Value":"375000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":14,"Name":"number of output rows","Update":"6250","Value":"6250","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"internal.metrics.peakExecutionMemory","Update":24040,"Value":24040,"Internal":true,"Count Failed Values":true},{"ID":71,"Name":"internal.metrics.resultSerializationTime","Update":11,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":70,"Name":"internal.metrics.jvmGCTime","Update":62,"Value":62,"Internal":true,"Count Failed Values":true},{"ID":69,"Name":"internal.metrics.resultSize","Update":2158,"Value":2158,"Internal":true,"Count Failed Values":true},{"ID":68,"Name":"internal.metrics.executorCpuTime","Update":414110000,"Value":414110000,"Internal":true,"Count Failed Values":true},{"ID":67,"Name":"internal.metrics.executorRunTime","Update":1498,"Value":1498,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.executorDeserializeCpuTime","Update":60358000,"Value":60358000,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.executorDeserializeTime","Update":255,"Value":255,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":255,"Executor Deserialize CPU Time":60358000,"Executor Run Time":1498,"Executor CPU Time":414110000,"Peak Execution Memory":24040,"Result Size":2158,"JVM GC Time":62,"Result Serialization Time":11,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":47,"Index":15,"Attempt":0,"Launch Time":1594091481136,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091483014,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"duration","Update":"1563","Value":"3030","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1,"Name":"number of output rows","Update":"375000","Value":"750000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":14,"Name":"number of output rows","Update":"6250","Value":"12500","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"internal.metrics.peakExecutionMemory","Update":24040,"Value":48080,"Internal":true,"Count Failed Values":true},{"ID":70,"Name":"internal.metrics.jvmGCTime","Update":52,"Value":114,"Internal":true,"Count Failed Values":true},{"ID":69,"Name":"internal.metrics.resultSize","Update":2115,"Value":4273,"Internal":true,"Count Failed Values":true},{"ID":68,"Name":"internal.metrics.executorCpuTime","Update":1324251000,"Value":1738361000,"Internal":true,"Count Failed Values":true},{"ID":67,"Name":"internal.metrics.executorRunTime","Update":1594,"Value":3092,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.executorDeserializeCpuTime","Update":47496000,"Value":107854000,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.executorDeserializeTime","Update":213,"Value":468,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":213,"Executor Deserialize CPU Time":47496000,"Executor Run Time":1594,"Executor CPU Time":1324251000,"Peak Execution Memory":24040,"Result Size":2115,"JVM GC Time":52,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":37,"Index":5,"Attempt":0,"Launch Time":1594091481100,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091483015,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"duration","Update":"1563","Value":"4593","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1,"Name":"number of output rows","Update":"375000","Value":"1125000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":14,"Name":"number of output rows","Update":"6250","Value":"18750","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"internal.metrics.peakExecutionMemory","Update":24040,"Value":72120,"Internal":true,"Count Failed Values":true},{"ID":70,"Name":"internal.metrics.jvmGCTime","Update":62,"Value":176,"Internal":true,"Count Failed Values":true},{"ID":69,"Name":"internal.metrics.resultSize","Update":2115,"Value":6388,"Internal":true,"Count Failed Values":true},{"ID":68,"Name":"internal.metrics.executorCpuTime","Update":507192000,"Value":2245553000,"Internal":true,"Count Failed Values":true},{"ID":67,"Name":"internal.metrics.executorRunTime","Update":1596,"Value":4688,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.executorDeserializeCpuTime","Update":60890000,"Value":168744000,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.executorDeserializeTime","Update":256,"Value":724,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":256,"Executor Deserialize CPU Time":60890000,"Executor Run Time":1596,"Executor CPU Time":507192000,"Peak Execution Memory":24040,"Result Size":2115,"JVM GC Time":62,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":36,"Index":4,"Attempt":0,"Launch Time":1594091481095,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091483015,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"duration","Update":"1564","Value":"6157","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1,"Name":"number of output rows","Update":"375000","Value":"1500000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":14,"Name":"number of output rows","Update":"6250","Value":"25000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"internal.metrics.peakExecutionMemory","Update":24040,"Value":96160,"Internal":true,"Count Failed Values":true},{"ID":70,"Name":"internal.metrics.jvmGCTime","Update":62,"Value":238,"Internal":true,"Count Failed Values":true},{"ID":69,"Name":"internal.metrics.resultSize","Update":2115,"Value":8503,"Internal":true,"Count Failed Values":true},{"ID":68,"Name":"internal.metrics.executorCpuTime","Update":503010000,"Value":2748563000,"Internal":true,"Count Failed Values":true},{"ID":67,"Name":"internal.metrics.executorRunTime","Update":1596,"Value":6284,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.executorDeserializeCpuTime","Update":112849000,"Value":281593000,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.executorDeserializeTime","Update":260,"Value":984,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":260,"Executor Deserialize CPU Time":112849000,"Executor Run Time":1596,"Executor CPU Time":503010000,"Peak Execution Memory":24040,"Result Size":2115,"JVM GC Time":62,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":35,"Index":3,"Attempt":0,"Launch Time":1594091481091,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091483016,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"duration","Update":"1566","Value":"7723","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1,"Name":"number of output rows","Update":"375000","Value":"1875000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":14,"Name":"number of output rows","Update":"6250","Value":"31250","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"internal.metrics.peakExecutionMemory","Update":24040,"Value":120200,"Internal":true,"Count Failed Values":true},{"ID":70,"Name":"internal.metrics.jvmGCTime","Update":62,"Value":300,"Internal":true,"Count Failed Values":true},{"ID":69,"Name":"internal.metrics.resultSize","Update":2115,"Value":10618,"Internal":true,"Count Failed Values":true},{"ID":68,"Name":"internal.metrics.executorCpuTime","Update":502908000,"Value":3251471000,"Internal":true,"Count Failed Values":true},{"ID":67,"Name":"internal.metrics.executorRunTime","Update":1598,"Value":7882,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.executorDeserializeCpuTime","Update":62944000,"Value":344537000,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.executorDeserializeTime","Update":263,"Value":1247,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":263,"Executor Deserialize CPU Time":62944000,"Executor Run Time":1598,"Executor CPU Time":502908000,"Peak Execution Memory":24040,"Result Size":2115,"JVM GC Time":62,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":40,"Index":8,"Attempt":0,"Launch Time":1594091481112,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091483016,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"duration","Update":"1562","Value":"9285","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1,"Name":"number of output rows","Update":"375000","Value":"2250000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":14,"Name":"number of output rows","Update":"6250","Value":"37500","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"internal.metrics.peakExecutionMemory","Update":24040,"Value":144240,"Internal":true,"Count Failed Values":true},{"ID":70,"Name":"internal.metrics.jvmGCTime","Update":62,"Value":362,"Internal":true,"Count Failed Values":true},{"ID":69,"Name":"internal.metrics.resultSize","Update":2115,"Value":12733,"Internal":true,"Count Failed Values":true},{"ID":68,"Name":"internal.metrics.executorCpuTime","Update":510597000,"Value":3762068000,"Internal":true,"Count Failed Values":true},{"ID":67,"Name":"internal.metrics.executorRunTime","Update":1595,"Value":9477,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.executorDeserializeCpuTime","Update":69760000,"Value":414297000,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.executorDeserializeTime","Update":246,"Value":1493,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":246,"Executor Deserialize CPU Time":69760000,"Executor Run Time":1595,"Executor CPU Time":510597000,"Peak Execution Memory":24040,"Result Size":2115,"JVM GC Time":62,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":45,"Index":13,"Attempt":0,"Launch Time":1594091481129,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091483024,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"duration","Update":"1564","Value":"10849","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1,"Name":"number of output rows","Update":"375000","Value":"2625000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":14,"Name":"number of output rows","Update":"6250","Value":"43750","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"internal.metrics.peakExecutionMemory","Update":24040,"Value":168280,"Internal":true,"Count Failed Values":true},{"ID":70,"Name":"internal.metrics.jvmGCTime","Update":62,"Value":424,"Internal":true,"Count Failed Values":true},{"ID":69,"Name":"internal.metrics.resultSize","Update":2115,"Value":14848,"Internal":true,"Count Failed Values":true},{"ID":68,"Name":"internal.metrics.executorCpuTime","Update":495138000,"Value":4257206000,"Internal":true,"Count Failed Values":true},{"ID":67,"Name":"internal.metrics.executorRunTime","Update":1595,"Value":11072,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.executorDeserializeCpuTime","Update":54222000,"Value":468519000,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.executorDeserializeTime","Update":221,"Value":1714,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":221,"Executor Deserialize CPU Time":54222000,"Executor Run Time":1595,"Executor CPU Time":495138000,"Peak Execution Memory":24040,"Result Size":2115,"JVM GC Time":62,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":39,"Index":7,"Attempt":0,"Launch Time":1594091481109,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091483024,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"duration","Update":"1566","Value":"12415","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1,"Name":"number of output rows","Update":"375000","Value":"3000000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":14,"Name":"number of output rows","Update":"6250","Value":"50000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"internal.metrics.peakExecutionMemory","Update":24040,"Value":192320,"Internal":true,"Count Failed Values":true},{"ID":70,"Name":"internal.metrics.jvmGCTime","Update":62,"Value":486,"Internal":true,"Count Failed Values":true},{"ID":69,"Name":"internal.metrics.resultSize","Update":2115,"Value":16963,"Internal":true,"Count Failed Values":true},{"ID":68,"Name":"internal.metrics.executorCpuTime","Update":539451000,"Value":4796657000,"Internal":true,"Count Failed Values":true},{"ID":67,"Name":"internal.metrics.executorRunTime","Update":1596,"Value":12668,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.executorDeserializeCpuTime","Update":64380000,"Value":532899000,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.executorDeserializeTime","Update":254,"Value":1968,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":254,"Executor Deserialize CPU Time":64380000,"Executor Run Time":1596,"Executor CPU Time":539451000,"Peak Execution Memory":24040,"Result Size":2115,"JVM GC Time":62,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":33,"Index":1,"Attempt":0,"Launch Time":1594091481082,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091483025,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"duration","Update":"1566","Value":"13981","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1,"Name":"number of output rows","Update":"375000","Value":"3375000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":14,"Name":"number of output rows","Update":"6250","Value":"56250","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"internal.metrics.peakExecutionMemory","Update":24040,"Value":216360,"Internal":true,"Count Failed Values":true},{"ID":70,"Name":"internal.metrics.jvmGCTime","Update":62,"Value":548,"Internal":true,"Count Failed Values":true},{"ID":69,"Name":"internal.metrics.resultSize","Update":2115,"Value":19078,"Internal":true,"Count Failed Values":true},{"ID":68,"Name":"internal.metrics.executorCpuTime","Update":519178000,"Value":5315835000,"Internal":true,"Count Failed Values":true},{"ID":67,"Name":"internal.metrics.executorRunTime","Update":1597,"Value":14265,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.executorDeserializeCpuTime","Update":54442000,"Value":587341000,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.executorDeserializeTime","Update":267,"Value":2235,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":267,"Executor Deserialize CPU Time":54442000,"Executor Run Time":1597,"Executor CPU Time":519178000,"Peak Execution Memory":24040,"Result Size":2115,"JVM GC Time":62,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":34,"Index":2,"Attempt":0,"Launch Time":1594091481087,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091483026,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"duration","Update":"1574","Value":"15555","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1,"Name":"number of output rows","Update":"375000","Value":"3750000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":14,"Name":"number of output rows","Update":"6250","Value":"62500","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"internal.metrics.peakExecutionMemory","Update":24040,"Value":240400,"Internal":true,"Count Failed Values":true},{"ID":70,"Name":"internal.metrics.jvmGCTime","Update":66,"Value":614,"Internal":true,"Count Failed Values":true},{"ID":69,"Name":"internal.metrics.resultSize","Update":2115,"Value":21193,"Internal":true,"Count Failed Values":true},{"ID":68,"Name":"internal.metrics.executorCpuTime","Update":508433000,"Value":5824268000,"Internal":true,"Count Failed Values":true},{"ID":67,"Name":"internal.metrics.executorRunTime","Update":1606,"Value":15871,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.executorDeserializeCpuTime","Update":69492000,"Value":656833000,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.executorDeserializeTime","Update":265,"Value":2500,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":265,"Executor Deserialize CPU Time":69492000,"Executor Run Time":1606,"Executor CPU Time":508433000,"Peak Execution Memory":24040,"Result Size":2115,"JVM GC Time":66,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":43,"Index":11,"Attempt":0,"Launch Time":1594091481123,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091483029,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"duration","Update":"1578","Value":"17133","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1,"Name":"number of output rows","Update":"375000","Value":"4125000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":14,"Name":"number of output rows","Update":"6250","Value":"68750","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"internal.metrics.peakExecutionMemory","Update":24040,"Value":264440,"Internal":true,"Count Failed Values":true},{"ID":70,"Name":"internal.metrics.jvmGCTime","Update":66,"Value":680,"Internal":true,"Count Failed Values":true},{"ID":69,"Name":"internal.metrics.resultSize","Update":2115,"Value":23308,"Internal":true,"Count Failed Values":true},{"ID":68,"Name":"internal.metrics.executorCpuTime","Update":502120000,"Value":6326388000,"Internal":true,"Count Failed Values":true},{"ID":67,"Name":"internal.metrics.executorRunTime","Update":1609,"Value":17480,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.executorDeserializeCpuTime","Update":48849000,"Value":705682000,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.executorDeserializeTime","Update":225,"Value":2725,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":225,"Executor Deserialize CPU Time":48849000,"Executor Run Time":1609,"Executor CPU Time":502120000,"Peak Execution Memory":24040,"Result Size":2115,"JVM GC Time":66,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":41,"Index":9,"Attempt":0,"Launch Time":1594091481116,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091483032,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"duration","Update":"1583","Value":"18716","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1,"Name":"number of output rows","Update":"375000","Value":"4500000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":14,"Name":"number of output rows","Update":"6250","Value":"75000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"internal.metrics.peakExecutionMemory","Update":24040,"Value":288480,"Internal":true,"Count Failed Values":true},{"ID":70,"Name":"internal.metrics.jvmGCTime","Update":66,"Value":746,"Internal":true,"Count Failed Values":true},{"ID":69,"Name":"internal.metrics.resultSize","Update":2115,"Value":25423,"Internal":true,"Count Failed Values":true},{"ID":68,"Name":"internal.metrics.executorCpuTime","Update":489923000,"Value":6816311000,"Internal":true,"Count Failed Values":true},{"ID":67,"Name":"internal.metrics.executorRunTime","Update":1614,"Value":19094,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.executorDeserializeCpuTime","Update":55787000,"Value":761469000,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.executorDeserializeTime","Update":240,"Value":2965,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":240,"Executor Deserialize CPU Time":55787000,"Executor Run Time":1614,"Executor CPU Time":489923000,"Peak Execution Memory":24040,"Result Size":2115,"JVM GC Time":66,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":46,"Index":14,"Attempt":0,"Launch Time":1594091481132,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091483037,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"duration","Update":"1587","Value":"20303","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1,"Name":"number of output rows","Update":"375000","Value":"4875000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":14,"Name":"number of output rows","Update":"6250","Value":"81250","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"internal.metrics.peakExecutionMemory","Update":24040,"Value":312520,"Internal":true,"Count Failed Values":true},{"ID":70,"Name":"internal.metrics.jvmGCTime","Update":66,"Value":812,"Internal":true,"Count Failed Values":true},{"ID":69,"Name":"internal.metrics.resultSize","Update":2115,"Value":27538,"Internal":true,"Count Failed Values":true},{"ID":68,"Name":"internal.metrics.executorCpuTime","Update":490927000,"Value":7307238000,"Internal":true,"Count Failed Values":true},{"ID":67,"Name":"internal.metrics.executorRunTime","Update":1618,"Value":20712,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.executorDeserializeCpuTime","Update":51464000,"Value":812933000,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.executorDeserializeTime","Update":218,"Value":3183,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":218,"Executor Deserialize CPU Time":51464000,"Executor Run Time":1618,"Executor CPU Time":490927000,"Peak Execution Memory":24040,"Result Size":2115,"JVM GC Time":66,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":32,"Index":0,"Attempt":0,"Launch Time":1594091481077,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091483037,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"duration","Update":"1587","Value":"21890","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1,"Name":"number of output rows","Update":"375000","Value":"5250000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":14,"Name":"number of output rows","Update":"6250","Value":"87500","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"internal.metrics.peakExecutionMemory","Update":24040,"Value":336560,"Internal":true,"Count Failed Values":true},{"ID":70,"Name":"internal.metrics.jvmGCTime","Update":66,"Value":878,"Internal":true,"Count Failed Values":true},{"ID":69,"Name":"internal.metrics.resultSize","Update":2115,"Value":29653,"Internal":true,"Count Failed Values":true},{"ID":68,"Name":"internal.metrics.executorCpuTime","Update":496683000,"Value":7803921000,"Internal":true,"Count Failed Values":true},{"ID":67,"Name":"internal.metrics.executorRunTime","Update":1619,"Value":22331,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.executorDeserializeCpuTime","Update":56827000,"Value":869760000,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.executorDeserializeTime","Update":271,"Value":3454,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":271,"Executor Deserialize CPU Time":56827000,"Executor Run Time":1619,"Executor CPU Time":496683000,"Peak Execution Memory":24040,"Result Size":2115,"JVM GC Time":66,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":42,"Index":10,"Attempt":0,"Launch Time":1594091481120,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091483043,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"duration","Update":"1593","Value":"23483","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1,"Name":"number of output rows","Update":"375000","Value":"5625000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":14,"Name":"number of output rows","Update":"6250","Value":"93750","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"internal.metrics.peakExecutionMemory","Update":24040,"Value":360600,"Internal":true,"Count Failed Values":true},{"ID":70,"Name":"internal.metrics.jvmGCTime","Update":66,"Value":944,"Internal":true,"Count Failed Values":true},{"ID":69,"Name":"internal.metrics.resultSize","Update":2115,"Value":31768,"Internal":true,"Count Failed Values":true},{"ID":68,"Name":"internal.metrics.executorCpuTime","Update":508230000,"Value":8312151000,"Internal":true,"Count Failed Values":true},{"ID":67,"Name":"internal.metrics.executorRunTime","Update":1624,"Value":23955,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.executorDeserializeCpuTime","Update":58152000,"Value":927912000,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.executorDeserializeTime","Update":229,"Value":3683,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":229,"Executor Deserialize CPU Time":58152000,"Executor Run Time":1624,"Executor CPU Time":508230000,"Peak Execution Memory":24040,"Result Size":2115,"JVM GC Time":66,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":44,"Index":12,"Attempt":0,"Launch Time":1594091481126,"Executor ID":"0","Host":"127.0.0.1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1594091483043,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"duration","Update":"1593","Value":"25076","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1,"Name":"number of output rows","Update":"375000","Value":"6000000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":14,"Name":"number of output rows","Update":"6250","Value":"100000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"internal.metrics.peakExecutionMemory","Update":24040,"Value":384640,"Internal":true,"Count Failed Values":true},{"ID":70,"Name":"internal.metrics.jvmGCTime","Update":66,"Value":1010,"Internal":true,"Count Failed Values":true},{"ID":69,"Name":"internal.metrics.resultSize","Update":2115,"Value":33883,"Internal":true,"Count Failed Values":true},{"ID":68,"Name":"internal.metrics.executorCpuTime","Update":498187000,"Value":8810338000,"Internal":true,"Count Failed Values":true},{"ID":67,"Name":"internal.metrics.executorRunTime","Update":1624,"Value":25579,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.executorDeserializeCpuTime","Update":51988000,"Value":979900000,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.executorDeserializeTime","Update":222,"Value":3905,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":222,"Executor Deserialize CPU Time":51988000,"Executor Run Time":1624,"Executor CPU Time":498187000,"Peak Execution Memory":24040,"Result Size":2115,"JVM GC Time":66,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerStageExecutorMetrics","Executor ID":"driver","Stage ID":2,"Stage Attempt ID":0,"Executor Metrics":{"JVMHeapMemory":213367864,"JVMOffHeapMemory":189011656,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":2133349,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2133349,"OffHeapUnifiedMemory":0,"DirectPoolMemory":282024,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":13,"MinorGCTime":115,"MajorGCCount":4,"MajorGCTime":339}}
+{"Event":"SparkListenerStageExecutorMetrics","Executor ID":"0","Stage ID":2,"Stage Attempt ID":0,"Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0}}
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"foreach at <console>:26","Number of Tasks":16,"RDD Info":[{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"18\",\"name\":\"mapPartitions\"}","Callsite":"foreach at <console>:26","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"DeserializeToObject\"}","Callsite":"foreach at <console>:26","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"1\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"foreach at <console>:26","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"foreach at <console>:26","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"SQLExecutionRDD","Callsite":"foreach at <console>:26","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.foreach(Dataset.scala:2862)\n$line19.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:26)\n$line19.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:30)\n$line19.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:32)\n$line19.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:34)\n$line19.$read$$iw$$iw$$iw$$iw.<init>(<console>:36)\n$line19.$read$$iw$$iw$$iw.<init>(<console>:38)\n$line19.$read$$iw$$iw.<init>(<console>:40)\n$line19.$read$$iw.<init>(<console>:42)\n$line19.$read.<init>(<console>:44)\n$line19.$read$.<init>(<console>:48)\n$line19.$read$.<clinit>(<console>)\n$line19.$eval$.$print$lzycompute(<console>:7)\n$line19.$eval$.$print(<console>:6)\n$line19.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:745)","Submission Time":1594091481040,"Completion Time":1594091483044,"Accumulables":[{"ID":68,"Name":"internal.metrics.executorCpuTime","Value":8810338000,"Internal":true,"Count Failed Values":true},{"ID":71,"Name":"internal.metrics.resultSerializationTime","Value":11,"Internal":true,"Count Failed Values":true},{"ID":74,"Name":"internal.metrics.peakExecutionMemory","Value":384640,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.executorDeserializeTime","Value":3905,"Internal":true,"Count Failed Values":true},{"ID":14,"Name":"number of output rows","Value":"100000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"internal.metrics.executorRunTime","Value":25579,"Internal":true,"Count Failed Values":true},{"ID":70,"Name":"internal.metrics.jvmGCTime","Value":1010,"Internal":true,"Count Failed Values":true},{"ID":1,"Name":"number of output rows","Value":"6000000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":69,"Name":"internal.metrics.resultSize","Value":33883,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.executorDeserializeCpuTime","Value":979900000,"Internal":true,"Count Failed Values":true},{"ID":0,"Name":"duration","Value":"25076","Internal":true,"Count Failed Values":true,"Metadata":"sql"}],"Resource Profile Id":0}}
+{"Event":"SparkListenerJobEnd","Job ID":2,"Completion Time":1594091483044,"Job Result":{"Result":"JobSucceeded"}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":0,"time":1594091483045}
+{"Event":"SparkListenerApplicationEnd","Timestamp":1594091824231}
diff --git a/core/src/test/resources/spark-events/application_1555004656427_0144 b/core/src/test/resources/spark-events/application_1555004656427_0144
index 91dae7e09ee94..6868fe68b3540 100644
--- a/core/src/test/resources/spark-events/application_1555004656427_0144
+++ b/core/src/test/resources/spark-events/application_1555004656427_0144
@@ -2,6 +2,7 @@
 {"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"10.28.9.112","Port":37319},"Maximum Memory":384093388,"Timestamp":1562101345595,"Maximum Onheap Memory":384093388,"Maximum Offheap Memory":0}
 {"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/java-8-openjdk-amd64/jre","Java Version":"1.8.0_212 (Oracle Corporation)","Scala Version":"version 2.12.8"},"Spark Properties":{"spark.executor.resource.gpu.amount":"1","spark.yarn.dist.files":"file:///home/tgraves/getGpus","spark.driver.host":"10.28.9.112","spark.executor.resource.gpu.discoveryScript":"./getGpus","spark.eventLog.enabled":"true","spark.driver.port":"38895","spark.repl.class.uri":"spark://10.28.9.112:38895/classes","spark.jars":"","spark.repl.class.outputDir":"/tmp/spark-f3ed6acc-96b8-420c-9974-b5be5e562fc6/repl-16ea2b82-f737-4619-8f9c-0dada7cab190","spark.driver.resource.gpu.amount":"1","spark.app.name":"Spark shell","spark.scheduler.mode":"FIFO","spark.executor.instances":"2","spark.submit.pyFiles":"","spark.ui.showConsoleProgress":"true","spark.driver.resource.gpu.discoveryScript":"/home/tgraves/getGpus","spark.executor.id":"driver","spark.task.resource.gpu.amount":"1","spark.submit.deployMode":"client","spark.master":"yarn","spark.ui.filters":"org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter","spark.home":"/home/tgraves/workspace/tgravescs-spark","spark.eventLog.dir":"hdfs:///user/tgraves/eventlogdir","spark.sql.catalogImplementation":"in-memory","spark.driver.appUIAddress":"http://10.28.9.112:4040","spark.org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter.param.PROXY_HOSTS":"tomg-test","spark.org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter.param.PROXY_URI_BASES":"http://tomg-test:8088/proxy/application_1555004656427_0144","spark.app.id":"application_1555004656427_0144"},"Hadoop Properties":{"hadoop.service.shutdown.timeout":"30s","yarn.resourcemanager.amlauncher.thread-count":"50","yarn.sharedcache.enabled":"false","fs.s3a.connection.maximum":"15","yarn.nodemanager.numa-awareness.numactl.cmd":"/usr/bin/numactl","fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","yarn.timeline-service.timeline-client.number-of-async-entities-to-merge":"10","hadoop.security.kms.client.timeout":"60","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.jobhistory.loadedjob.tasks.max":"-1","mapreduce.framework.name":"yarn","yarn.sharedcache.uploader.server.thread-count":"50","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"30","hadoop.security.groups.cache.background.reload.threads":"3","yarn.resourcemanager.webapp.cross-origin.enabled":"false","fs.AbstractFileSystem.ftp.impl":"org.apache.hadoop.fs.ftp.FtpFs","hadoop.registry.secure":"false","hadoop.shell.safely.delete.limit.num.files":"100","dfs.bytes-per-checksum":"512","mapreduce.job.acl-view-job":" ","fs.s3a.s3guard.ddb.background.sleep":"25ms","fs.s3a.retry.limit":"${fs.s3a.attempts.maximum}","mapreduce.jobhistory.loadedjobs.cache.size":"5","fs.s3a.s3guard.ddb.table.create":"false","yarn.nodemanager.amrmproxy.enabled":"false","yarn.timeline-service.entity-group-fs-store.with-user-dir":"false","mapreduce.input.fileinputformat.split.minsize":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","yarn.sharedcache.checksum.algo.impl":"org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl","yarn.nodemanager.amrmproxy.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor","dfs.replication":"1","yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size":"10485760","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","mapreduce.task.profile.maps":"0-2","yarn.scheduler.include-port-in-node-name":"false","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","yarn.resourcemanager.node-removal-untracked.timeout-ms":"60000","mapreduce.am.max-attempts":"2","hadoop.security.kms.client.failover.sleep.base.millis":"100","mapreduce.jobhistory.webapp.https.address":"0.0.0.0:19890","yarn.node-labels.fs-store.impl.class":"org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore","yarn.nodemanager.collector-service.address":"${yarn.nodemanager.hostname}:8048","fs.trash.checkpoint.interval":"0","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","yarn.resourcemanager.node-ip-cache.expiry-interval-secs":"-1","hadoop.http.authentication.signature.secret.file":"*********(redacted)","hadoop.jetty.logs.serve.aliases":"true","yarn.resourcemanager.placement-constraints.handler":"disabled","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"1000","yarn.resourcemanager.placement-constraints.algorithm.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm","yarn.sharedcache.webapp.address":"0.0.0.0:8788","yarn.resourcemanager.delegation.token.renew-interval":"*********(redacted)","yarn.sharedcache.nm.uploader.replication.factor":"10","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","mapreduce.reduce.skip.proc-count.auto-incr":"true","hadoop.security.group.mapping.ldap.posix.attr.gid.name":"gidNumber","ipc.client.fallback-to-simple-auth-allowed":"false","yarn.nodemanager.resource.memory.enforced":"true","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","yarn.acl.reservation-enable":"false","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","yarn.app.mapreduce.am.hard-kill-timeout-ms":"10000","fs.s3a.etag.checksum.enabled":"false","yarn.nodemanager.container-metrics.enable":"true","yarn.timeline-service.client.fd-clean-interval-secs":"60","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","hadoop.common.configuration.version":"3.0.0","fs.s3a.s3guard.ddb.table.capacity.read":"500","yarn.nodemanager.remote-app-log-dir-suffix":"logs","yarn.nodemanager.windows-container.cpu-limit.enabled":"false","yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed":"false","file.blocksize":"67108864","hadoop.registry.zk.retry.ceiling.ms":"60000","mapreduce.reduce.env":"HADOOP_MAPRED_HOME=/home/tgraves/runspace/hadoop/lce/hadoop-3.1.2","yarn.scheduler.configuration.leveldb-store.path":"${hadoop.tmp.dir}/yarn/system/confstore","yarn.sharedcache.store.in-memory.initial-delay-mins":"10","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","mapreduce.map.skip.proc-count.auto-incr":"true","fs.s3a.committer.name":"file","mapreduce.task.profile.reduces":"0-2","hadoop.zk.num-retries":"1000","yarn.webapp.xfs-filter.enabled":"true","seq.io.sort.mb":"100","yarn.scheduler.configuration.max.version":"100","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"${yarn.resourcemanager.hostname}:8030","yarn.node-labels.enabled":"false","yarn.resourcemanager.webapp.ui-actions.enabled":"true","mapreduce.task.timeout":"600000","yarn.sharedcache.client-server.thread-count":"50","hadoop.security.groups.shell.command.timeout":"0s","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.nodemanager.elastic-memory-control.oom-handler":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.DefaultOOMHandler","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"hdfs://localhost:9000","yarn.minicluster.use-rpc":"false","yarn.app.mapreduce.am.env":"HADOOP_MAPRED_HOME=/home/tgraves/runspace/hadoop/lce/hadoop-3.1.2","fs.har.impl.disable.cache":"true","yarn.webapp.ui2.enable":"false","io.compression.codec.bzip2.library":"system-native","yarn.nodemanager.distributed-scheduling.enabled":"false","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"${yarn.resourcemanager.hostname}:8090","mapreduce.jobhistory.address":"0.0.0.0:10020","yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.is.minicluster":"false","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","fs.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A","mapreduce.task.combine.progress.records":"10000","yarn.resourcemanager.epoch.range":"0","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","fs.AbstractFileSystem.wasbs.impl":"org.apache.hadoop.fs.azure.Wasbs","yarn.timeline-service.entity-group-fs-store.cache-store-class":"org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore","fs.ftp.transfer.mode":"BLOCK_TRANSFER_MODE","ipc.server.log.slow.rpc":"false","yarn.resourcemanager.node-labels.provider.fetch-interval-ms":"1800000","yarn.router.webapp.https.address":"0.0.0.0:8091","yarn.nodemanager.webapp.cross-origin.enabled":"false","fs.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","yarn.resourcemanager.auto-update.containers":"false","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","yarn.scheduler.configuration.zk-store.parent-path":"/confstore","yarn.nodemanager.default-container-executor.log-dirs.permissions":"710","yarn.app.attempt.diagnostics.limit.kc":"64","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"-1","fs.AbstractFileSystem.abfs.impl":"org.apache.hadoop.fs.azurebfs.Abfs","yarn.timeline-service.writer.flush-interval-seconds":"60","fs.s3a.fast.upload.active.blocks":"4","hadoop.security.credential.clear-text-fallback":"true","yarn.nodemanager.collector-service.thread-count":"5","fs.azure.secure.mode":"false","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","yarn.resourcemanager.fs.state-store.num-retries":"0","yarn.resourcemanager.nodemanager-connect-retries":"10","yarn.nodemanager.log-aggregation.num-log-files-per-app":"30","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","fs.s3a.committer.magic.enabled":"false","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","yarn.system-metrics-publisher.enabled":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","mapreduce.task.io.sort.factor":"10","yarn.nodemanager.amrmproxy.client.thread-count":"25","ha.failover-controller.new-active.rpc-timeout.ms":"60000","yarn.nodemanager.container-localizer.java.opts":"-Xmx256m","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","yarn.nodemanager.windows-container.memory-limit.enabled":"false","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.minicluster.fixed.ports":"false","hadoop.tags.system":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n      ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.cluster.max-application-priority":"0","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","hadoop.caller.context.signature.max.size":"40","yarn.client.load.resource-types.from-server":"false","ha.zookeeper.session-timeout.ms":"10000","tfile.io.chunk.size":"1048576","fs.s3a.s3guard.ddb.table.capacity.write":"100","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.kms.client.failover.sleep.max.millis":"2000","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","yarn.scheduler.configuration.store.max-logs":"1000","yarn.nodemanager.node-attributes.provider.fetch-interval-ms":"600000","fs.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","yarn.nodemanager.local-cache.max-files-per-directory":"8192","hadoop.http.cross-origin.enabled":"false","hadoop.zk.acl":"world:anyone:rwcda","mapreduce.map.sort.spill.percent":"0.80","yarn.timeline-service.entity-group-fs-store.scan-interval-seconds":"60","yarn.node-attribute.fs-store.impl.class":"org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore","fs.s3a.retry.interval":"500ms","yarn.timeline-service.client.best-effort":"false","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","hadoop.security.group.mapping.ldap.posix.attr.uid.name":"uidNumber","fs.AbstractFileSystem.swebhdfs.impl":"org.apache.hadoop.fs.SWebHdfs","yarn.nodemanager.elastic-memory-control.timeout-sec":"5","mapreduce.ifile.readahead":"true","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","yarn.timeline-service.reader.webapp.address":"${yarn.timeline-service.webapp.address}","yarn.resourcemanager.placement-constraints.algorithm.pool-size":"1","yarn.timeline-service.hbase.coprocessor.jar.hdfs.location":"/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler","yarn.app.mapreduce.am.command-opts":"-Xmx1024m","mapreduce.cluster.local.dir":"${hadoop.tmp.dir}/mapred/local","io.mapfile.bloom.error.rate":"0.005","fs.client.resolve.topology.enabled":"false","yarn.nodemanager.runtime.linux.allowed-runtimes":"default","yarn.sharedcache.store.class":"org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","fs.s3a.metadatastore.impl":"org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore","io.skip.checksum.errors":"false","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","yarn.timeline-service.webapp.xfs-filter.xframe-options":"SAMEORIGIN","fs.s3a.connection.timeout":"200000","yarn.nodemanager.linux-container-executor.group":"tgraves","mapreduce.job.max.split.locations":"15","yarn.resourcemanager.nm-container-queuing.max-queue-length":"15","hadoop.registry.zk.session.timeout.ms":"60000","yarn.federation.cache-ttl.secs":"300","mapreduce.jvm.system-properties-to-log":"os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name","yarn.resourcemanager.opportunistic-container-allocation.nodes-used":"10","yarn.timeline-service.entity-group-fs-store.active-dir":"/tmp/entity-file-history/active","mapreduce.shuffle.transfer.buffer.size":"131072","yarn.timeline-service.client.retry-interval-ms":"1000","yarn.http.policy":"HTTP_ONLY","fs.s3a.socket.send.buffer":"8192","fs.AbstractFileSystem.abfss.impl":"org.apache.hadoop.fs.azurebfs.Abfss","yarn.sharedcache.uploader.server.address":"0.0.0.0:8046","yarn.resourcemanager.delegation-token.max-conf-size-bytes":"*********(redacted)","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","yarn.minicluster.yarn.nodemanager.resource.memory-mb":"4096","mapreduce.job.emit-timeline-data":"false","yarn.nodemanager.resource.system-reserved-memory-mb":"-1","hadoop.kerberos.min.seconds.before.relogin":"60","mapreduce.jobhistory.move.thread-count":"3","yarn.resourcemanager.admin.client.thread-count":"1","yarn.dispatcher.drain-events.timeout":"300000","fs.s3a.buffer.dir":"${hadoop.tmp.dir}/s3a","hadoop.ssl.enabled.protocols":"TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2","mapreduce.jobhistory.admin.address":"0.0.0.0:10033","yarn.log-aggregation-status.time-out.ms":"600000","fs.s3a.assumed.role.sts.endpoint.region":"us-west-1","mapreduce.shuffle.port":"13562","yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory":"10","yarn.nodemanager.health-checker.interval-ms":"600000","yarn.router.clientrm.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor","yarn.resourcemanager.zk-appid-node.split-index":"0","ftp.blocksize":"67108864","yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions":"read","yarn.router.rmadmin.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor","yarn.nodemanager.log-container-debug-info.enabled":"true","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms":"20","yarn.nodemanager.delete.debug-delay-sec":"0","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","yarn.nodemanager.container-retry-minimum-interval-ms":"1000","hadoop.security.groups.cache.secs":"300","yarn.federation.enabled":"false","fs.azure.local.sas.key.mode":"false","ipc.maximum.data.length":"67108864","mapreduce.shuffle.max.threads":"0","yarn.router.pipeline.cache-max-size":"25","yarn.resourcemanager.nm-container-queuing.load-comparator":"QUEUE_LENGTH","hadoop.security.authorization":"false","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","yarn.nodemanager.amrmproxy.ha.enable":"false","mapreduce.jobhistory.http.policy":"HTTP_ONLY","yarn.sharedcache.store.in-memory.check-period-mins":"720","hadoop.security.group.mapping.ldap.ssl":"false","yarn.client.application-client-protocol.poll-interval-ms":"200","yarn.scheduler.configuration.leveldb-store.compaction-interval-secs":"86400","yarn.timeline-service.writer.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl","ha.zookeeper.parent-znode":"/hadoop-ha","yarn.nodemanager.log-aggregation.policy.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","yarn.resourcemanager.placement-constraints.scheduler.pool-size":"1","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","yarn.nodemanager.recovery.supervised":"false","yarn.sharedcache.admin.thread-count":"1","yarn.resourcemanager.ha.automatic-failover.enabled":"true","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.resourcemanager.address":"${yarn.resourcemanager.hostname}:8032","ipc.client.ping":"true","mapreduce.task.local-fs.write-limit.bytes":"-1","fs.adl.oauth2.access.token.provider.type":"*********(redacted)","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","yarn.nodemanager.resource-plugins.gpu.docker-plugin":"nvidia-docker-v1","hadoop.ssl.enabled":"false","fs.s3a.multipart.purge":"false","yarn.scheduler.configuration.store.class":"file","yarn.resourcemanager.nm-container-queuing.queue-limit-stdev":"1.0f","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","ipc.client.bind.wildcard.addr":"false","yarn.resourcemanager.webapp.rest-csrf.enabled":"false","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"10","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","yarn.nodemanager.runtime.linux.docker.allowed-container-networks":"host,none,bridge","yarn.nodemanager.node-labels.resync-interval-ms":"120000","hadoop.tmp.dir":"/tmp/hadoop-${user.name}","mapreduce.job.maps":"2","mapreduce.jobhistory.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","yarn.rm.system-metrics-publisher.emit-container-events":"false","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","fs.s3a.s3guard.ddb.throttle.retry.interval":"100ms","fs.AbstractFileSystem.wasb.impl":"org.apache.hadoop.fs.azure.Wasb","mapreduce.client.submit.file.replication":"10","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"2147483647","yarn.resourcemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold":"1","yarn.nodemanager.webapp.rest-csrf.enabled":"false","fs.ftp.host.port":"21","ipc.ping.interval":"60000","yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size":"10","yarn.resourcemanager.admin.address":"${yarn.resourcemanager.hostname}:8033","file.client-write-packet-size":"65536","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"true","hadoop.security.key.default.bitlength":"128","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","yarn.nodemanager.log.deletion-threads-count":"4","yarn.webapp.filter-entity-list-by-user":"false","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"100","yarn.nodemanager.localizer.client.thread-count":"5","io.erasurecode.codec.rs.rawcoders":"rs_native,rs_java","io.erasurecode.codec.rs-legacy.rawcoders":"rs-legacy_java","yarn.sharedcache.admin.address":"0.0.0.0:8047","yarn.resourcemanager.placement-constraints.algorithm.iterator":"SERIAL","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec","mapreduce.job.cache.limit.max-resources-mb":"0","fs.s3a.connection.ssl.enabled":"true","yarn.nodemanager.process-kill-wait.ms":"5000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","hadoop.workaround.non.threadsafe.getpwuid":"true","fs.df.interval":"60000","yarn.scheduler.capacity.resource-calculator":"org.apache.hadoop.yarn.util.resource.DominantResourceCalculator","fs.s3a.multiobjectdelete.enable":"true","yarn.sharedcache.cleaner.resource-sleep-ms":"0","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","hadoop.shell.missing.defaultFs.warning":"false","io.file.buffer.size":"65536","hadoop.security.group.mapping.ldap.search.attr.member":"member","hadoop.security.random.device.file.path":"/dev/urandom","hadoop.security.sensitive-config-keys":"*********(redacted)","fs.s3a.s3guard.ddb.max.retries":"9","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","yarn.intermediate-data-encryption.enable":"false","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.nodemanager.container.stderr.pattern":"{*stderr*,*STDERR*}","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/tmp/hadoop-yarn/staging","mapreduce.reduce.shuffle.read.timeout":"180000","hadoop.http.cross-origin.max-age":"1800","io.erasurecode.codec.xor.rawcoders":"xor_native,xor_java","fs.s3a.connection.establish.timeout":"5000","mapreduce.job.running.map.limit":"0","yarn.minicluster.control-resource-monitoring":"false","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","yarn.federation.state-store.class":"org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore","mapreduce.reduce.log.level":"INFO","hadoop.security.dns.log-slow-lookups.threshold.ms":"1000","mapreduce.job.ubertask.enable":"false","adl.http.timeout":"-1","yarn.resourcemanager.placement-constraints.retry-attempts":"3","hadoop.caller.context.enabled":"false","yarn.nodemanager.vmem-pmem-ratio":"2.1","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","hadoop.zk.timeout-ms":"10000","fs.s3a.s3guard.cli.prune.age":"86400000","yarn.nodemanager.resource.pcores-vcores-multiplier":"1.0","yarn.nodemanager.runtime.linux.sandbox-mode":"disabled","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.s3a.committer.threads":"8","hadoop.zk.retry-interval-ms":"1000","hadoop.security.crypto.buffer.size":"8192","yarn.nodemanager.node-labels.provider.fetch-interval-ms":"600000","mapreduce.jobhistory.recovery.store.leveldb.path":"${hadoop.tmp.dir}/mapred/history/recoverystore","yarn.client.failover-retries-on-socket-timeouts":"0","yarn.nodemanager.resource.memory.enabled":"false","fs.azure.authorization.caching.enable":"true","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","mapreduce.job.finish-when-all-reducers-done":"true","hadoop.registry.jaas.context":"Client","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","io.map.index.interval":"128","yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms":"100","fs.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","mapreduce.job.counters.max":"120","mapreduce.jobhistory.webapp.rest-csrf.enabled":"false","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","yarn.timeline-service.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/timeline","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","hadoop.security.auth_to_local.mechanism":"hadoop","yarn.timeline-service.app-collector.linger-period.ms":"60000","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.reservation-system.planfollower.time-step":"1000","yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed":"true","yarn.webapp.api-service.enable":"false","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","fs.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","yarn.nodemanager.container.stderr.tail.bytes":"4096","hadoop.security.group.mapping.ldap.read.timeout.ms":"60000","mapreduce.map.env":"HADOOP_MAPRED_HOME=/home/tgraves/runspace/hadoop/lce/hadoop-3.1.2","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","mapreduce.outputcommitter.factory.scheme.s3a":"org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory","hadoop.security.groups.cache.background.reload":"false","yarn.nodemanager.container-monitor.enabled":"true","yarn.nodemanager.elastic-memory-control.enabled":"false","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","yarn.resourcemanager.rm.container-allocation.expiry-interval-ms":"600000","mapreduce.fileoutputcommitter.algorithm.version":"2","yarn.resourcemanager.work-preserving-recovery.enabled":"true","mapreduce.map.skip.maxrecords":"0","yarn.sharedcache.root-dir":"/sharedcache","fs.s3a.retry.throttle.limit":"${fs.s3a.attempts.maximum}","hadoop.http.authentication.type":"simple","mapreduce.job.cache.limit.max-resources":"0","mapreduce.task.userlog.limit.kb":"0","yarn.resourcemanager.scheduler.monitor.enable":"false","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","yarn.nodemanager.resource-monitor.interval-ms":"3000","yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices":"auto","mapreduce.job.sharedcache.mode":"disabled","yarn.nodemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.shuffle.listen.queue.size":"128","yarn.scheduler.configuration.mutation.acl-policy.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy","mapreduce.map.cpu.vcores":"1","yarn.log-aggregation.file-formats":"TFile","yarn.timeline-service.client.fd-retain-secs":"300","hadoop.user.group.static.mapping.overrides":"dr.who=;","fs.azure.sas.expiry.period":"90d","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"false","yarn.router.webapp.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST","yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage":"90.0","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","yarn.nodemanager.disk-validator":"basic","yarn.app.mapreduce.client.job.max-retries":"3","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","hadoop.security.group.mapping.ldap.connection.timeout.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","yarn.nodemanager.container-diagnostics-maximum-size":"10000","hadoop.registry.zk.retry.interval.ms":"1000","yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms":"1000","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","fs.s3a.assumed.role.session.duration":"30m","hadoop.security.group.mapping.ldap.conversion.rule":"none","hadoop.ssl.server.conf":"ssl-server.xml","fs.s3a.retry.throttle.interval":"1000ms","seq.io.sort.factor":"100","yarn.sharedcache.cleaner.initial-delay-mins":"10","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.nodemanager.runtime.linux.docker.capabilities":"CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE","yarn.acl.enable":"false","yarn.timeline-service.entity-group-fs-store.done-dir":"/tmp/entity-file-history/done/","mapreduce.task.profile":"false","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","mapreduce.jobhistory.always-scan-user-dir":"false","yarn.nodemanager.opportunistic-containers-use-pause-for-preemption":"false","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"nobody","yarn.timeline-service.reader.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold":"1","yarn.resourcemanager.configuration.file-system-based-store":"/yarn/conf","mapreduce.job.cache.limit.max-single-resource-mb":"0","yarn.nodemanager.runtime.linux.docker.stop.grace-period":"10","yarn.resourcemanager.resource-profiles.source-file":"resource-profiles.json","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"${yarn.log.dir}/userlogs","fs.automatic.close":"true","yarn.nodemanager.hostname":"0.0.0.0","yarn.nodemanager.resource.memory.cgroups.swappiness":"0","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","yarn.timeline-service.app-aggregation-interval-secs":"15","hadoop.security.group.mapping.ldap.search.filter.user":"(&(objectClass=user)(sAMAccountName={0}))","yarn.nodemanager.container-localizer.log.level":"INFO","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","mapreduce.job.ubertask.maxmaps":"9","fs.s3a.threads.keepalivetime":"60","mapreduce.jobhistory.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","yarn.resourcemanager.delegation.token.max-lifetime":"*********(redacted)","yarn.timeline-service.client.drain-entities.timeout.ms":"2000","yarn.nodemanager.resource-plugins.fpga.vendor-plugin.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin","yarn.timeline-service.entity-group-fs-store.summary-store":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.reduce.cpu.vcores":"1","mapreduce.job.encrypted-intermediate-data.buffer.kb":"128","fs.client.resolve.remote.symlinks":"true","yarn.nodemanager.webapp.https.address":"0.0.0.0:8044","hadoop.http.cross-origin.allowed-origins":"*","mapreduce.job.encrypted-intermediate-data":"false","yarn.timeline-service.entity-group-fs-store.retain-seconds":"604800","yarn.resourcemanager.metrics.runtime.buckets":"60,300,1440","yarn.timeline-service.generic-application-history.max-applications":"10000","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","yarn.node-labels.configuration-type":"centralized","fs.s3a.path.style.access":"false","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","yarn.sharedcache.store.in-memory.staleness-period-mins":"10080","fs.adl.impl":"org.apache.hadoop.fs.adl.AdlFileSystem","yarn.resourcemanager.nodemanager.minimum.version":"NONE","mapreduce.jobhistory.webapp.xfs-filter.xframe-options":"SAMEORIGIN","yarn.app.mapreduce.am.staging-dir.erasurecoding.enabled":"false","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","yarn.timeline-service.reader.webapp.https.address":"${yarn.timeline-service.webapp.https.address}","fs.ftp.data.connection.mode":"ACTIVE_LOCAL_DATA_CONNECTION_MODE","mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed":"true","yarn.scheduler.maximum-allocation-vcores":"4","hadoop.http.cross-origin.allowed-headers":"X-Requested-With,Content-Type,Accept,Origin","yarn.nodemanager.log-aggregation.compression-type":"none","yarn.timeline-service.version":"1.0f","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","mapreduce.reduce.maxattempts":"4","hadoop.security.dns.log-slow-lookups.enabled":"false","mapreduce.job.committer.setup.cleanup.needed":"true","mapreduce.job.running.reduce.limit":"0","ipc.maximum.response.length":"134217728","yarn.resourcemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","hadoop.caller.context.max.size":"128","yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed":"false","yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed":"false","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","fs.s3a.fast.upload.buffer":"disk","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","fs.s3a.max.total.tasks":"5","fs.s3a.readahead.range":"64K","hadoop.http.authentication.simple.anonymous.allowed":"true","yarn.nodemanager.aux-services.spark_shuffle.class":"org.apache.spark.network.yarn.YarnShuffleService","fs.s3a.attempts.maximum":"20","hadoop.registry.zk.connection.timeout.ms":"15000","yarn.resourcemanager.delegation-token-renewer.thread-count":"*********(redacted)","yarn.nodemanager.health-checker.script.timeout-ms":"1200000","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","yarn.resourcemanager.resource-profiles.enabled":"false","yarn.timeline-service.hbase-schema.prefix":"prod.","fs.azure.authorization":"false","mapreduce.map.log.level":"INFO","yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs":"20","mapreduce.output.fileoutputformat.compress.type":"RECORD","yarn.resourcemanager.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.timeline-service.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.ifile.readahead.bytes":"4194304","yarn.sharedcache.app-checker.class":"org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"false","yarn.nodemanager.resource.detect-hardware-capabilities":"false","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","hadoop.security.group.mapping.ldap.search.group.hierarchy.levels":"0","yarn.resourcemanager.fs.state-store.retry-interval-ms":"1000","file.stream-buffer-size":"4096","yarn.resourcemanager.application-timeouts.monitor.interval-ms":"3000","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","mapreduce.map.speculative":"true","mapreduce.job.speculative.retry-after-speculate":"15000","yarn.nodemanager.linux-container-executor.cgroups.mount":"true","yarn.app.mapreduce.am.container.log.backups":"0","yarn.app.mapreduce.am.log.level":"INFO","mapreduce.job.reduce.slowstart.completedmaps":"0.05","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices":"0,1","yarn.timeline-service.client.internal-timers-ttl-secs":"420","hadoop.http.logs.enabled":"true","fs.s3a.block.size":"32M","yarn.sharedcache.client-server.address":"0.0.0.0:8045","yarn.nodemanager.logaggregation.threadpool-size-max":"100","yarn.resourcemanager.hostname":"0.0.0.0","yarn.resourcemanager.delegation.key.update-interval":"86400000","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"-1","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","yarn.nodemanager.disk-health-checker.enable":"true","ipc.client.tcpnodelay":"true","ipc.client.rpc-timeout.ms":"0","yarn.nodemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","ipc.client.low-latency":"false","mapreduce.input.lineinputformat.linespermap":"1","yarn.router.interceptor.user.threadpool-size":"5","ipc.client.connect.max.retries.on.timeouts":"45","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.timeline-service.entity-group-fs-store.app-cache-size":"10","fs.s3a.socket.recv.buffer":"8192","yarn.resourcemanager.resource-tracker.address":"${yarn.resourcemanager.hostname}:8031","yarn.nodemanager.node-labels.provider.fetch-timeout-ms":"1200000","mapreduce.job.heap.memory-mb.ratio":"0.8","yarn.resourcemanager.leveldb-state-store.compaction-interval-secs":"3600","yarn.resourcemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","yarn.scheduler.configuration.fs.path":"file://${hadoop.tmp.dir}/yarn/system/schedconf","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"org.apache.hadoop.http.lib.StaticUserWebFilter","mapreduce.reduce.memory.mb":"-1","yarn.timeline-service.hostname":"0.0.0.0","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","yarn.nodemanager.container-metrics.period-ms":"-1","mapreduce.fileoutputcommitter.task.cleanup.enabled":"false","yarn.nodemanager.log.retain-seconds":"10800","yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds":"3600","yarn.resourcemanager.keytab":"/etc/krb5.keytab","hadoop.security.group.mapping.providers.combined":"true","mapreduce.reduce.merge.inmem.threshold":"1000","yarn.timeline-service.recovery.enabled":"false","fs.azure.saskey.usecontainersaskeyforallaccess":"true","yarn.sharedcache.nm.uploader.thread-count":"20","yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs":"3600","mapreduce.shuffle.ssl.enabled":"false","yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds":"259200000","fs.s3a.committer.staging.abort.pending.uploads":"true","yarn.nodemanager.opportunistic-containers-max-queue-length":"0","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","mapreduce.job.speculative.minimum-allowed-tasks":"10","yarn.log-aggregation.retain-seconds":"-1","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.http.cross-origin.allowed-methods":"GET,POST,HEAD","yarn.resourcemanager.opportunistic-container-allocation.enabled":"false","mapreduce.jobhistory.webapp.address":"0.0.0.0:19888","hadoop.system.tags":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n      ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.log-aggregation.file-controller.TFile.class":"org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"${yarn.resourcemanager.hostname}:8088","mapreduce.jobhistory.recovery.enable":"false","mapreduce.reduce.shuffle.parallelcopies":"5","fs.AbstractFileSystem.webhdfs.impl":"org.apache.hadoop.fs.WebHdfs","fs.trash.interval":"0","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","yarn.app.mapreduce.am.resource.mb":"1536","mapreduce.input.fileinputformat.list-status.num-threads":"1","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.resourcemanager.nm-container-queuing.min-queue-length":"5","yarn.nodemanager.resource.cpu-vcores":"-1","mapreduce.job.reduces":"1","fs.s3a.multipart.size":"100M","yarn.scheduler.minimum-allocation-vcores":"1","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","mapreduce.job.encrypted-intermediate-data-key-size-bits":"128","fs.s3a.metadatastore.authoritative":"false","yarn.nodemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","ha.health-monitor.sleep-after-disconnect.ms":"1000","yarn.app.mapreduce.shuffle.log.limit.kb":"0","hadoop.security.group.mapping":"org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback","yarn.client.application-client-protocol.poll-timeout-ms":"-1","mapreduce.jobhistory.jhist.format":"binary","yarn.resourcemanager.ha.enabled":"false","hadoop.http.staticuser.user":"dr.who","mapreduce.task.exit.timeout.check-interval-ms":"20000","mapreduce.jobhistory.intermediate-user-done-dir.permissions":"770","mapreduce.task.exit.timeout":"60000","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","yarn.resourcemanager.reservation-system.enable":"false","mapreduce.map.output.compress":"false","ha.zookeeper.acl":"world:anyone:rwcda","ipc.server.max.connections":"0","yarn.nodemanager.aux-services":"mapreduce_shuffle,spark_shuffle","yarn.nodemanager.runtime.linux.docker.default-container-network":"host","yarn.router.webapp.address":"0.0.0.0:8089","yarn.scheduler.maximum-allocation-mb":"8192","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.sharedcache.cleaner.period-mins":"1440","yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint":"http://localhost:3476/v1.0/docker/cli","yarn.app.mapreduce.am.container.log.limit.kb":"0","ipc.client.connect.retry.interval":"1000","yarn.timeline-service.http-cross-origin.enabled":"false","fs.wasbs.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure","yarn.federation.subcluster-resolver.class":"org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","mapreduce.jobhistory.cleaner.enable":"true","yarn.timeline-service.client.fd-flush-interval-secs":"10","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms":"1000","fs.s3a.committer.staging.tmp.path":"tmp/staging","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms":"10","mapreduce.job.end-notification.retry.attempts":"0","yarn.nodemanager.resource.count-logical-processors-as-cores":"false","hadoop.registry.zk.root":"/registry","adl.feature.ownerandgroup.enableupn":"false","yarn.resourcemanager.zk-max-znode-size.bytes":"1048576","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","yarn.nodemanager.localizer.cache.target-size-mb":"10240","fs.s3a.committer.staging.conflict-mode":"fail","mapreduce.client.libjars.wildcard":"true","fs.s3a.committer.staging.unique-filenames":"true","yarn.nodemanager.node-attributes.provider.fetch-timeout-ms":"1200000","fs.s3a.list.version":"2","ftp.client-write-packet-size":"65536","fs.AbstractFileSystem.adl.impl":"org.apache.hadoop.fs.adl.Adl","hadoop.security.key.default.cipher":"AES/CTR/NoPadding","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"86400","mapreduce.job.local-fs.single-disk-limit.check.interval-ms":"5000","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","yarn.nodemanager.amrmproxy.address":"0.0.0.0:8049","ipc.server.listen.queue.size":"128","map.sort.class":"org.apache.hadoop.util.QuickSort","fs.viewfs.rename.strategy":"SAME_MOUNTPOINT","hadoop.security.kms.client.authentication.retry-count":"1","fs.permissions.umask-mode":"022","fs.s3a.assumed.role.credentials.provider":"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider","yarn.nodemanager.vmem-check-enabled":"false","yarn.nodemanager.numa-awareness.enabled":"false","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","yarn.federation.registry.base-dir":"yarnfederation/","mapreduce.job.max.map":"-1","mapreduce.job.local-fs.single-disk-limit.bytes":"-1","mapreduce.job.ubertask.maxreduces":"1","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","mapreduce.jobhistory.jobname.limit":"50","mapreduce.application.classpath":"$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*","yarn.client.nodemanager-connect.retry-interval-ms":"10000","yarn.timeline-service.state-store-class":"org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ","yarn.sharedcache.nested-level":"3","yarn.timeline-service.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","fs.azure.user.agent.prefix":"unknown","yarn.resourcemanager.zk-delegation-token-node.split-index":"*********(redacted)","yarn.nodemanager.numa-awareness.read-topology":"false","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","yarn.registry.class":"org.apache.hadoop.registry.client.impl.FSRegistryOperationsService","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","yarn.scheduler.queue-placement-rules":"user-group","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","yarn.resourcemanager.recovery.enabled":"false","yarn.timeline-service.webapp.rest-csrf.enabled":"false"},"System Properties":{"java.io.tmpdir":"/tmp","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.version":"1.8","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Oracle Corporation","java.vm.specification.version":"1.8","user.home":"/home/tgraves","file.encoding.pkg":"sun.io","sun.nio.ch.bugLevel":"","sun.arch.data.model":"64","sun.boot.library.path":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64","user.dir":"/home/tgraves/workspace/tgravescs-spark","java.library.path":"/usr/java/packages/lib/amd64:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib","sun.cpu.isalist":"","sun.desktop":"gnome","os.arch":"amd64","java.vm.version":"25.212-b03","jetty.git.hash":"e1bc35120a6617ee3df052294e433f3a25ce7097","java.endorsed.dirs":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/endorsed","java.runtime.version":"1.8.0_212-8u212-b03-0ubuntu1.18.04.1-b03","java.vm.info":"mixed mode","java.ext.dirs":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/ext:/usr/java/packages/lib/ext","java.runtime.name":"OpenJDK Runtime Environment","file.separator":"/","java.class.version":"52.0","scala.usejavacp":"true","java.specification.name":"Java Platform API Specification","sun.boot.class.path":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/resources.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/rt.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/sunrsasign.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jsse.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jce.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/charsets.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jfr.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/classes","file.encoding":"UTF-8","user.timezone":"America/Chicago","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","os.version":"4.15.0-46-generic","sun.os.patch.level":"unknown","java.vm.specification.vendor":"Oracle Corporation","user.country":"US","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.url":"http://java.oracle.com/","java.awt.printerjob":"sun.print.PSPrinterJob","java.awt.graphicsenv":"sun.awt.X11GraphicsEnvironment","awt.toolkit":"sun.awt.X11.XToolkit","os.name":"Linux","java.vm.vendor":"Oracle Corporation","java.vendor.url.bug":"http://bugreport.sun.com/bugreport/","user.name":"tgraves","java.vm.name":"OpenJDK 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.SparkSubmit --master yarn --conf spark.executor.resource.gpu.amount=1 --conf spark.driver.resource.gpu.amount=1 --conf spark.eventLog.enabled=true --conf spark.eventLog.dir=hdfs:///user/tgraves/eventlogdir --conf spark.executor.resource.gpu.discoveryScript=./getGpus --conf spark.driver.resource.gpu.discoveryScript=/home/tgraves/getGpus --conf spark.task.resource.gpu.amount=1 --class org.apache.spark.repl.Main --name Spark shell --driver-cores 2 --num-executors 2 --files file:/home/tgraves/getGpus spark-shell","java.home":"/usr/lib/jvm/java-8-openjdk-amd64/jre","java.version":"1.8.0_212","sun.io.unicode.encoding":"UnicodeLittle"},"Classpath Entries":{"/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/stream-2.9.6.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/spark-repl_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/hadoop-yarn-registry-3.2.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/kerb-crypto-1.0.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/spark-tags_2.12-3.0.0-SNAPSHOT-tests.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/kerby-asn1-1.0.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/aircompressor-0.10.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jersey-container-servlet-2.22.2.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jersey-server-2.22.2.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jackson-jaxrs-base-2.9.5.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/guava-14.0.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/zjsonpatch-0.3.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/slf4j-log4j12-1.7.16.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/parquet-jackson-1.10.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/kerb-core-1.0.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/token-provider-1.0.1.jar":"*********(redacted)","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/commons-beanutils-1.9.3.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/metrics-graphite-3.1.5.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/metrics-jvm-3.1.5.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/istack-commons-runtime-3.0.8.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/minlog-1.3.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/dnsjava-2.1.7.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/hadoop-yarn-client-3.2.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/commons-net-3.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/macro-compat_2.12-1.1.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/shapeless_2.12-2.3.2.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/spark-streaming_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jackson-databind-2.9.9.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jetty-xml-9.4.18.v20190429.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/json-smart-2.3.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/guice-servlet-4.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/orc-shims-1.5.5.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/scala-xml_2.12-1.2.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jersey-common-2.22.2.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/parquet-format-2.4.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/spark-catalyst_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jackson-module-paranamer-2.9.9.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/aopalliance-repackaged-2.4.0-b34.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jtransforms-2.4.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/kerb-util-1.0.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/commons-codec-1.10.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/netty-all-4.1.30.Final.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/javax.ws.rs-api-2.0.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/spark-launcher_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/hadoop-client-3.2.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/xz-1.5.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/spark-yarn_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/kubernetes-model-4.1.2.jar":"System Classpath","/hadoopconfs/":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/ivy-2.4.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/avro-mapred-1.8.2-hadoop2.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/parquet-column-1.10.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/geronimo-jcache_1.0_spec-1.0-alpha-1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/hk2-api-2.4.0-b34.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/slf4j-api-1.7.16.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/spark-graph-api_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/leveldbjni-all-1.8.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/commons-lang-2.6.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/compress-lzf-1.0.3.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/nimbus-jose-jwt-4.41.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/re2j-1.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/hadoop-mapreduce-client-common-3.2.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/chill-java-0.9.3.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/kubernetes-model-common-4.1.2.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/kerb-client-1.0.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/automaton-1.11-8.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/joda-time-2.9.3.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/hadoop-yarn-api-3.2.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jersey-media-jaxb-2.22.2.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/conf/":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/commons-lang3-3.8.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/arpack_combined_all-0.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/spark-kvstore_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/core-1.1.2.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/arrow-format-0.12.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/accessors-smart-1.2.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/commons-crypto-1.0.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/okhttp-3.8.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/snakeyaml-1.23.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/breeze_2.12-0.13.2.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/orc-core-1.5.5.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/curator-client-2.13.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/spark-graphx_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/spark-core_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/antlr4-runtime-4.7.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/commons-text-1.6.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/activation-1.1.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/kerb-simplekdc-1.0.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jsr305-3.0.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/paranamer-2.8.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/commons-math3-3.4.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jackson-core-2.9.9.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/hadoop-mapreduce-client-jobclient-3.2.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/kerb-common-1.0.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/hadoop-hdfs-client-3.2.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jackson-module-scala_2.12-2.9.9.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/lz4-java-1.6.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/httpcore-4.4.10.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/kerb-server-1.0.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/commons-cli-1.2.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/parquet-hadoop-1.10.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/curator-recipes-2.13.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/logging-interceptor-3.12.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/kerby-pkix-1.0.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/woodstox-core-5.0.3.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/hadoop-annotations-3.2.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/json4s-ast_2.12-3.6.6.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/univocity-parsers-2.7.3.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/xbean-asm7-shaded-4.14.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/parquet-encoding-1.10.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/spark-sketch_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jersey-container-servlet-core-2.22.2.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/spark-graph_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/osgi-resource-locator-1.0.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/hk2-locator-2.4.0-b34.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jackson-dataformat-yaml-2.9.9.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/javax.inject-2.4.0-b34.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/aopalliance-1.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/commons-compiler-3.0.11.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/netty-3.9.9.Final.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/spark-mllib_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/scala-library-2.12.8.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/javax.inject-1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/hadoop-auth-3.2.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jackson-mapper-asl-1.9.13.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/spark-network-common_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/okio-1.13.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/spire-macros_2.12-0.13.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/arrow-memory-0.12.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/spark-sql_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/hppc-0.7.2.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/spire_2.12-0.13.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/arrow-vector-0.12.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/opencsv-2.3.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/commons-configuration2-2.1.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jackson-annotations-2.9.9.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/spark-mllib-local_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/pyrolite-4.23.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/json4s-jackson_2.12-3.6.6.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jersey-client-2.22.2.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/ehcache-3.3.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/spark-tags_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/machinist_2.12-0.6.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/hadoop-yarn-server-web-proxy-3.2.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/javassist-3.18.1-GA.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/orc-mapreduce-1.5.5.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/hive-storage-api-2.6.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/hadoop-mapreduce-client-core-3.2.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/log4j-1.2.17.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/htrace-core4-4.1.0-incubating.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/avro-ipc-1.8.2.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jcl-over-slf4j-1.7.16.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/spark-kubernetes_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/snappy-java-1.1.7.3.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/oro-2.0.8.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/kubernetes-client-4.1.2.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/generex-1.0.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/spark-cypher_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/RoaringBitmap-0.7.45.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/commons-io-2.4.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/commons-daemon-1.0.13.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/avro-1.8.2.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/chill_2.12-0.9.3.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/janino-3.0.11.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/hadoop-yarn-server-common-3.2.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/stax2-api-3.1.4.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/zookeeper-3.4.13.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/breeze-macros_2.12-0.13.2.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/spark-network-shuffle_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jaxb-api-2.2.11.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/spark-unsafe_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/guice-4.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/scala-compiler-2.12.8.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jsp-api-2.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/metrics-json-3.1.5.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/kerby-xdr-1.0.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/httpclient-4.5.6.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/shims-0.7.45.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/py4j-0.10.8.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jackson-module-jaxb-annotations-2.9.9.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/validation-api-1.1.0.Final.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/kerb-identity-1.0.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/scala-parser-combinators_2.12-1.1.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/audience-annotations-0.5.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jul-to-slf4j-1.7.16.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jcip-annotations-1.0-1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/objenesis-2.5.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/commons-compress-1.8.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/kerb-admin-1.0.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/okhttp-2.7.5.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/metrics-core-3.1.5.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/kryo-shaded-4.0.2.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jaxb-runtime-2.3.2.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/json4s-scalap_2.12-3.6.6.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jackson-core-asl-1.9.13.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/javax.servlet-api-3.1.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/kerby-config-1.0.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/flatbuffers-java-1.9.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/curator-framework-2.13.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jetty-webapp-9.4.18.v20190429.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/hadoop-yarn-common-3.2.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/gson-2.2.4.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/scala-reflect-2.12.8.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/hk2-utils-2.4.0-b34.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/hadoop-common-3.2.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/kerby-util-1.0.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jackson-jaxrs-json-provider-2.9.5.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/javax.annotation-api-1.2.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jakarta.xml.bind-api-2.3.2.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/parquet-common-1.10.1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/zstd-jni-1.4.0-1.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/jersey-guava-2.22.2.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/commons-collections-3.2.2.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/json4s-core_2.12-3.6.6.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/protobuf-java-2.5.0.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/okapi-shade-0.4.2.jar":"System Classpath","/home/tgraves/workspace/tgravescs-spark/assembly/target/scala-2.12/jars/mssql-jdbc-6.2.1.jre7.jar":"System Classpath"}}
 {"Event":"SparkListenerApplicationStart","App Name":"Spark shell","App ID":"application_1555004656427_0144","Timestamp":1562101337180,"User":"tgraves"}
+{"Event":"org.apache.spark.scheduler.SparkListenerMiscellaneousProcessAdded","time":1618846550218,"processId":"yarn-am","info":{"hostPort":"192.168.1.19:8042","cores":1,"logUrlInfo":{"stdout":"http://192.168.1.19:8042/node/containerlogs/container_1555004656427_0144_01_000001/test/stdout?start=-4096","stderr":"http://192.168.1.19:8042/node/containerlogs/container_1555004656427_0144_01_000001/test/stderr?start=-4096"}}}
 {"Event":"SparkListenerExecutorAdded","Timestamp":1562101348551,"Executor ID":"1","Executor Info":{"Host":"tomg-test","Total Cores":1,"Log Urls":{"stdout":"http://tomg-test:8042/node/containerlogs/container_1555004656427_0144_01_000002/tgraves/stdout?start=-4096","stderr":"http://tomg-test:8042/node/containerlogs/container_1555004656427_0144_01_000002/tgraves/stderr?start=-4096"},"Attributes":{"NM_HTTP_ADDRESS":"tomg-test:8042","USER":"tgraves","LOG_FILES":"stderr,stdout","NM_HTTP_PORT":"8042","CLUSTER_ID":"","NM_PORT":"43125","HTTP_SCHEME":"http://","NM_HOST":"tomg-test","CONTAINER_ID":"container_1555004656427_0144_01_000002"},"Resources":{"gpu":{"name":"gpu","addresses":["0","1","2","3","4","5","6","7","8","9","10","11","12"]}}}}
 {"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"1","Host":"tomg-test","Port":44873},"Maximum Memory":384093388,"Timestamp":1562101348609,"Maximum Onheap Memory":384093388,"Maximum Offheap Memory":0}
 {"Event":"SparkListenerExecutorAdded","Timestamp":1562101349256,"Executor ID":"2","Executor Info":{"Host":"tomg-test","Total Cores":1,"Log Urls":{"stdout":"http://tomg-test:8042/node/containerlogs/container_1555004656427_0144_01_000003/tgraves/stdout?start=-4096","stderr":"http://tomg-test:8042/node/containerlogs/container_1555004656427_0144_01_000003/tgraves/stderr?start=-4096"},"Attributes":{"NM_HTTP_ADDRESS":"tomg-test:8042","USER":"tgraves","LOG_FILES":"stderr,stdout","NM_HTTP_PORT":"8042","CLUSTER_ID":"","NM_PORT":"43125","HTTP_SCHEME":"http://","NM_HOST":"tomg-test","CONTAINER_ID":"container_1555004656427_0144_01_000003"},"Resources":{"gpu":{"name":"gpu","addresses":["0","1","2","3","4","5","6","7","8","9","10","11","12"]}}}}
diff --git a/core/src/test/resources/spark-events/application_1578436911597_0052 b/core/src/test/resources/spark-events/application_1578436911597_0052
new file mode 100644
index 0000000000000..c57481a348a89
--- /dev/null
+++ b/core/src/test/resources/spark-events/application_1578436911597_0052
@@ -0,0 +1,27 @@
+{"Event":"SparkListenerLogStart","Spark Version":"3.0.0-SNAPSHOT"}
+{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"cores":{"Resource Name":"cores","Amount":1,"Discovery Script":"","Vendor":""},"memory":{"Resource Name":"memory","Amount":1024,"Discovery Script":"","Vendor":""},"gpu":{"Resource Name":"gpu","Amount":1,"Discovery Script":"/home/tgraves/getGpus","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0},"gpu":{"Resource Name":"gpu","Amount":1.0}}}
+{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"10.10.10.10","Port":32957},"Maximum Memory":428762726,"Timestamp":1578764671818,"Maximum Onheap Memory":428762726,"Maximum Offheap Memory":0}
+{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/java-8-openjdk-amd64/jre","Java Version":"1.8.0_232 (Private Build)","Scala Version":"version 2.12.10"},"Spark Properties":{},"Hadoop Properties":{},"System Properties":{}, "Classpath Entries": {}}
+{"Event":"SparkListenerApplicationStart","App Name":"Spark shell","App ID":"application_1578436911597_0052","Timestamp":1578764662851,"User":"tgraves"}
+{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":1,"Executor Resource Requests":{"cores":{"Resource Name":"cores","Amount":4,"Discovery Script":"","Vendor":""},"gpu":{"Resource Name":"gpu","Amount":1,"Discovery Script":"./getGpus","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0},"gpu":{"Resource Name":"gpu","Amount":1.0}}}
+{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":2,"Executor Resource Requests":{"cores":{"Resource Name":"cores","Amount":2,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":2.0}}}
+{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":3,"Executor Resource Requests":{"cores":{"Resource Name":"cores","Amount":4,"Discovery Script":"","Vendor":""},"gpu":{"Resource Name":"gpu","Amount":1,"Discovery Script":"./getGpus","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":2.0},"gpu":{"Resource Name":"gpu","Amount":1.0}}}
+{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1578764765274,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"collect at <console>:29","Number of Tasks":6,"RDD Info":[{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"map\"}","Callsite":"map at <console>:31","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"parallelize at <console>:31","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.rdd.RDD.collect(RDD.scala:1004)\n$line37.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:29)\n$line37.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:33)\n$line37.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:35)\n$line37.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:37)\n$line37.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:39)\n$line37.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:41)\n$line37.$read$$iw$$iw$$iw$$iw.<init>(<console>:43)\n$line37.$read$$iw$$iw$$iw.<init>(<console>:45)\n$line37.$read$$iw$$iw.<init>(<console>:47)\n$line37.$read$$iw.<init>(<console>:49)\n$line37.$read.<init>(<console>:51)\n$line37.$read$.<init>(<console>:55)\n$line37.$read$.<clinit>(<console>)\n$line37.$eval$.$print$lzycompute(<console>:7)\n$line37.$eval$.$print(<console>:6)\n$line37.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)","Accumulables":[],"Resource Profile Id":3}],"Stage IDs":[0],"Properties":{"spark.rdd.scope":"{\"id\":\"2\",\"name\":\"collect\"}","spark.rdd.scope.noOverride":"true"}}
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"collect at <console>:29","Number of Tasks":6,"RDD Info":[{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"map\"}","Callsite":"map at <console>:31","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"parallelize at <console>:31","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.rdd.RDD.collect(RDD.scala:1004)\n$line37.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:29)\n$line37.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:33)\n$line37.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:35)\n$line37.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:37)\n$line37.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:39)\n$line37.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:41)\n$line37.$read$$iw$$iw$$iw$$iw.<init>(<console>:43)\n$line37.$read$$iw$$iw$$iw.<init>(<console>:45)\n$line37.$read$$iw$$iw.<init>(<console>:47)\n$line37.$read$$iw.<init>(<console>:49)\n$line37.$read.<init>(<console>:51)\n$line37.$read$.<init>(<console>:55)\n$line37.$read$.<clinit>(<console>)\n$line37.$eval$.$print$lzycompute(<console>:7)\n$line37.$eval$.$print(<console>:6)\n$line37.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)","Submission Time":1578764765293,"Accumulables":[],"Resource Profile Id":3},"Properties":{"spark.rdd.scope":"{\"id\":\"2\",\"name\":\"collect\"}","spark.rdd.scope.noOverride":"true"}}
+{"Event":"SparkListenerExecutorAdded","Timestamp":1578764769706,"Executor ID":"1","Executor Info":{"Host":"host1","Total Cores":4,"Log Urls":{"stdout":"http://host1:8042/node/containerlogs/container_1578436911597_0052_01_000002/tgraves/stdout?start=-4096","stderr":"http://host1:8042/node/containerlogs/container_1578436911597_0052_01_000002/tgraves/stderr?start=-4096"},"Attributes":{"NM_HTTP_ADDRESS":"host1:8042","USER":"tgraves","LOG_FILES":"stderr,stdout","NM_HTTP_PORT":"8042","CLUSTER_ID":"","NM_PORT":"37783","HTTP_SCHEME":"http://","NM_HOST":"host1","CONTAINER_ID":"container_1578436911597_0052_01_000002"},"Resources":{"gpu":{"name":"gpu","addresses":["0","1","2"]}},"Resource Profile Id":3}}
+{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"1","Host":"host1","Port":40787},"Maximum Memory":384093388,"Timestamp":1578764769796,"Maximum Onheap Memory":384093388,"Maximum Offheap Memory":0}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1578764769858,"Executor ID":"1","Host":"host1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Launch Time":1578764769877,"Executor ID":"1","Host":"host1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":2,"Index":2,"Attempt":0,"Launch Time":1578764770507,"Executor ID":"1","Host":"host1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":3,"Index":3,"Attempt":0,"Launch Time":1578764770509,"Executor ID":"1","Host":"host1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1578764769858,"Executor ID":"1","Host":"host1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1578764770512,"Failed":false,"Killed":false,"Accumulables":[{"ID":6,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":5,"Name":"internal.metrics.jvmGCTime","Update":49,"Value":49,"Internal":true,"Count Failed Values":true},{"ID":4,"Name":"internal.metrics.resultSize","Update":3706,"Value":3706,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"internal.metrics.executorCpuTime","Update":20740892,"Value":20740892,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"internal.metrics.executorRunTime","Update":32,"Value":32,"Internal":true,"Count Failed Values":true},{"ID":1,"Name":"internal.metrics.executorDeserializeCpuTime","Update":250921658,"Value":250921658,"Internal":true,"Count Failed Values":true},{"ID":0,"Name":"internal.metrics.executorDeserializeTime","Update":555,"Value":555,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":555,"Executor Deserialize CPU Time":250921658,"Executor Run Time":32,"Executor CPU Time":20740892,"Peak Execution Memory":0,"Result Size":3706,"JVM GC Time":49,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Launch Time":1578764769877,"Executor ID":"1","Host":"host1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1578764770515,"Failed":false,"Killed":false,"Accumulables":[{"ID":6,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":5,"Name":"internal.metrics.jvmGCTime","Update":49,"Value":98,"Internal":true,"Count Failed Values":true},{"ID":4,"Name":"internal.metrics.resultSize","Update":3722,"Value":7428,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"internal.metrics.executorCpuTime","Update":25185125,"Value":45926017,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"internal.metrics.executorRunTime","Update":32,"Value":64,"Internal":true,"Count Failed Values":true},{"ID":1,"Name":"internal.metrics.executorDeserializeCpuTime","Update":416274503,"Value":667196161,"Internal":true,"Count Failed Values":true},{"ID":0,"Name":"internal.metrics.executorDeserializeTime","Update":555,"Value":1110,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":555,"Executor Deserialize CPU Time":416274503,"Executor Run Time":32,"Executor CPU Time":25185125,"Peak Execution Memory":0,"Result Size":3722,"JVM GC Time":49,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":4,"Index":4,"Attempt":0,"Launch Time":1578764770525,"Executor ID":"1","Host":"host1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":2,"Index":2,"Attempt":0,"Launch Time":1578764770507,"Executor ID":"1","Host":"host1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1578764770526,"Failed":false,"Killed":false,"Accumulables":[{"ID":4,"Name":"internal.metrics.resultSize","Update":3636,"Value":11064,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"internal.metrics.executorCpuTime","Update":2203515,"Value":48129532,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"internal.metrics.executorRunTime","Update":2,"Value":66,"Internal":true,"Count Failed Values":true},{"ID":1,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2733237,"Value":669929398,"Internal":true,"Count Failed Values":true},{"ID":0,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":1112,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2733237,"Executor Run Time":2,"Executor CPU Time":2203515,"Peak Execution Memory":0,"Result Size":3636,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":5,"Index":5,"Attempt":0,"Launch Time":1578764770527,"Executor ID":"1","Host":"host1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":3,"Index":3,"Attempt":0,"Launch Time":1578764770509,"Executor ID":"1","Host":"host1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1578764770529,"Failed":false,"Killed":false,"Accumulables":[{"ID":4,"Name":"internal.metrics.resultSize","Update":3620,"Value":14684,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"internal.metrics.executorCpuTime","Update":2365599,"Value":50495131,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"internal.metrics.executorRunTime","Update":2,"Value":68,"Internal":true,"Count Failed Values":true},{"ID":1,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3387884,"Value":673317282,"Internal":true,"Count Failed Values":true},{"ID":0,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":1115,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3387884,"Executor Run Time":2,"Executor CPU Time":2365599,"Peak Execution Memory":0,"Result Size":3620,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":4,"Index":4,"Attempt":0,"Launch Time":1578764770525,"Executor ID":"1","Host":"host1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1578764770542,"Failed":false,"Killed":false,"Accumulables":[{"ID":4,"Name":"internal.metrics.resultSize","Update":3636,"Value":18320,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"internal.metrics.executorCpuTime","Update":2456346,"Value":52951477,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"internal.metrics.executorRunTime","Update":2,"Value":70,"Internal":true,"Count Failed Values":true},{"ID":1,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3502860,"Value":676820142,"Internal":true,"Count Failed Values":true},{"ID":0,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":1118,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3502860,"Executor Run Time":2,"Executor CPU Time":2456346,"Peak Execution Memory":0,"Result Size":3636,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":5,"Index":5,"Attempt":0,"Launch Time":1578764770527,"Executor ID":"1","Host":"host1","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1578764770542,"Failed":false,"Killed":false,"Accumulables":[{"ID":4,"Name":"internal.metrics.resultSize","Update":3636,"Value":21956,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"internal.metrics.executorCpuTime","Update":2162370,"Value":55113847,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"internal.metrics.executorRunTime","Update":2,"Value":72,"Internal":true,"Count Failed Values":true},{"ID":1,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3622437,"Value":680442579,"Internal":true,"Count Failed Values":true},{"ID":0,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":1121,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3622437,"Executor Run Time":2,"Executor CPU Time":2162370,"Peak Execution Memory":0,"Result Size":3636,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"collect at <console>:29","Number of Tasks":6,"RDD Info":[{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"map\"}","Callsite":"map at <console>:31","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"parallelize at <console>:31","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.rdd.RDD.collect(RDD.scala:1004)\n$line37.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:29)\n$line37.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:33)\n$line37.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:35)\n$line37.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:37)\n$line37.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:39)\n$line37.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:41)\n$line37.$read$$iw$$iw$$iw$$iw.<init>(<console>:43)\n$line37.$read$$iw$$iw$$iw.<init>(<console>:45)\n$line37.$read$$iw$$iw.<init>(<console>:47)\n$line37.$read$$iw.<init>(<console>:49)\n$line37.$read.<init>(<console>:51)\n$line37.$read$.<init>(<console>:55)\n$line37.$read$.<clinit>(<console>)\n$line37.$eval$.$print$lzycompute(<console>:7)\n$line37.$eval$.$print(<console>:6)\n$line37.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)","Submission Time":1578764765293,"Completion Time":1578764770543,"Accumulables":[{"ID":2,"Name":"internal.metrics.executorRunTime","Value":72,"Internal":true,"Count Failed Values":true},{"ID":5,"Name":"internal.metrics.jvmGCTime","Value":98,"Internal":true,"Count Failed Values":true},{"ID":4,"Name":"internal.metrics.resultSize","Value":21956,"Internal":true,"Count Failed Values":true},{"ID":1,"Name":"internal.metrics.executorDeserializeCpuTime","Value":680442579,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"internal.metrics.executorCpuTime","Value":55113847,"Internal":true,"Count Failed Values":true},{"ID":6,"Name":"internal.metrics.resultSerializationTime","Value":4,"Internal":true,"Count Failed Values":true},{"ID":0,"Name":"internal.metrics.executorDeserializeTime","Value":1121,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":3}}
+{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1578764770546,"Job Result":{"Result":"JobSucceeded"}}
+{"Event":"SparkListenerApplicationEnd","Timestamp":1578764802615}
diff --git a/core/src/test/scala/org/apache/spark/AccumulatorSuite.scala b/core/src/test/scala/org/apache/spark/AccumulatorSuite.scala
index a75cf3f0381df..e4dfa149a7d20 100644
--- a/core/src/test/scala/org/apache/spark/AccumulatorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/AccumulatorSuite.scala
@@ -25,8 +25,8 @@ import scala.collection.mutable.ArrayBuffer
 import scala.ref.WeakReference
 import scala.util.control.NonFatal
 
-import org.scalatest.Matchers
 import org.scalatest.exceptions.TestFailedException
+import org.scalatest.matchers.must.Matchers
 
 import org.apache.spark.scheduler._
 import org.apache.spark.serializer.JavaSerializer
@@ -157,7 +157,7 @@ private class SaveInfoListener extends SparkListener {
   def getCompletedStageInfos: Seq[StageInfo] = completedStageInfos.toArray.toSeq
   def getCompletedTaskInfos: Seq[TaskInfo] = completedTaskInfos.values.flatten.toSeq
   def getCompletedTaskInfos(stageId: StageId, stageAttemptId: StageAttemptId): Seq[TaskInfo] =
-    completedTaskInfos.getOrElse((stageId, stageAttemptId), Seq.empty[TaskInfo])
+    completedTaskInfos.getOrElse((stageId, stageAttemptId), Seq.empty[TaskInfo]).toSeq
 
   /**
    * If `jobCompletionCallback` is set, block until the next call has finished.
diff --git a/core/src/test/scala/org/apache/spark/BarrierStageOnSubmittedSuite.scala b/core/src/test/scala/org/apache/spark/BarrierStageOnSubmittedSuite.scala
index 435b927068e60..1ba13c2ef1897 100644
--- a/core/src/test/scala/org/apache/spark/BarrierStageOnSubmittedSuite.scala
+++ b/core/src/test/scala/org/apache/spark/BarrierStageOnSubmittedSuite.scala
@@ -19,9 +19,12 @@ package org.apache.spark
 
 import scala.concurrent.duration._
 
+import org.apache.spark.TestUtils.createTempScriptWithExpectedOutput
 import org.apache.spark.internal.config._
 import org.apache.spark.rdd.{PartitionPruningRDD, RDD}
+import org.apache.spark.resource.TestResourceIDs.{EXECUTOR_GPU_ID, TASK_GPU_ID, WORKER_GPU_ID}
 import org.apache.spark.scheduler.BarrierJobAllocationFailed._
+import org.apache.spark.scheduler.BarrierJobSlotsNumberCheckFailed
 import org.apache.spark.util.ThreadUtils
 
 /**
@@ -259,4 +262,37 @@ class BarrierStageOnSubmittedSuite extends SparkFunSuite with LocalSparkContext
     testSubmitJob(sc, rdd,
       message = ERROR_MESSAGE_BARRIER_REQUIRE_MORE_SLOTS_THAN_CURRENT_TOTAL_NUMBER)
   }
+
+  test("SPARK-32518: CoarseGrainedSchedulerBackend.maxNumConcurrentTasks should " +
+    "consider all kinds of resources for the barrier stage") {
+    withTempDir { dir =>
+      val discoveryScript = createTempScriptWithExpectedOutput(
+        dir, "gpuDiscoveryScript", """{"name": "gpu","addresses":["0"]}""")
+
+      val conf = new SparkConf()
+        // Setup a local cluster which would only has one executor with 2 CPUs and 1 GPU.
+        .setMaster("local-cluster[1, 2, 1024]")
+        .setAppName("test-cluster")
+        .set(WORKER_GPU_ID.amountConf, "1")
+        .set(WORKER_GPU_ID.discoveryScriptConf, discoveryScript)
+        .set(EXECUTOR_GPU_ID.amountConf, "1")
+        .set(TASK_GPU_ID.amountConf, "1")
+        // disable barrier stage retry to fail the application as soon as possible
+        .set(BARRIER_MAX_CONCURRENT_TASKS_CHECK_MAX_FAILURES, 1)
+      sc = new SparkContext(conf)
+      TestUtils.waitUntilExecutorsUp(sc, 1, 60000)
+
+      val exception = intercept[BarrierJobSlotsNumberCheckFailed] {
+        // Setup a barrier stage which contains 2 tasks and each task requires 1 CPU and 1 GPU.
+        // Therefore, the total resources requirement (2 CPUs and 2 GPUs) of this barrier stage
+        // can not be satisfied since the cluster only has 2 CPUs and 1 GPU in total.
+        sc.parallelize(Range(1, 10), 2)
+          .barrier()
+          .mapPartitions { iter => iter }
+          .collect()
+      }
+      assert(exception.getMessage.contains("[SPARK-24819]: Barrier execution " +
+        "mode does not allow run a barrier stage that requires more slots"))
+    }
+  }
 }
diff --git a/core/src/test/scala/org/apache/spark/CheckpointSuite.scala b/core/src/test/scala/org/apache/spark/CheckpointSuite.scala
index a69381d18e3b6..e42df0821589b 100644
--- a/core/src/test/scala/org/apache/spark/CheckpointSuite.scala
+++ b/core/src/test/scala/org/apache/spark/CheckpointSuite.scala
@@ -194,7 +194,7 @@ trait RDDCheckpointTester { self: SparkFunSuite =>
   /**
    * Serialize and deserialize an object. This is useful to verify the objects
    * contents after deserialization (e.g., the contents of an RDD split after
-   * it is sent to a slave along with a task)
+   * it is sent to an executor along with a task)
    */
   protected def serializeDeserialize[T](obj: T): T = {
     val bytes = Utils.serialize(obj)
@@ -635,12 +635,12 @@ class CheckpointStorageSuite extends SparkFunSuite with LocalSparkContext {
       // Verify that RDD is checkpointed
       assert(rdd.firstParent.isInstanceOf[ReliableCheckpointRDD[_]])
       val checkpointedRDD = rdd.firstParent.asInstanceOf[ReliableCheckpointRDD[_]]
-      val partiton = checkpointedRDD.partitions(0)
-      assert(!checkpointedRDD.cachedPreferredLocations.asMap.containsKey(partiton))
+      val partition = checkpointedRDD.partitions(0)
+      assert(!checkpointedRDD.cachedPreferredLocations.asMap.containsKey(partition))
 
-      val preferredLoc = checkpointedRDD.preferredLocations(partiton)
-      assert(checkpointedRDD.cachedPreferredLocations.asMap.containsKey(partiton))
-      assert(preferredLoc == checkpointedRDD.cachedPreferredLocations.get(partiton))
+      val preferredLoc = checkpointedRDD.preferredLocations(partition)
+      assert(checkpointedRDD.cachedPreferredLocations.asMap.containsKey(partition))
+      assert(preferredLoc == checkpointedRDD.cachedPreferredLocations.get(partition))
     }
   }
 
@@ -653,7 +653,7 @@ class CheckpointStorageSuite extends SparkFunSuite with LocalSparkContext {
       val rdd = sc.makeRDD(1 to 200, numSlices = 4).repartition(1).mapPartitions { iter =>
         iter.map { i =>
           if (i > 100 && TaskContext.get().stageAttemptNumber() == 0) {
-            // throw new SparkException("Make first attemp failed.")
+            // throw new SparkException("Make first attempt failed.")
             // Throw FetchFailedException to explicitly trigger stage resubmission.
             // A normal exception will only trigger task resubmission in the same stage.
             throw new FetchFailedException(null, 0, 0L, 0, 0, "Fake")
diff --git a/core/src/test/scala/org/apache/spark/ContextCleanerSuite.scala b/core/src/test/scala/org/apache/spark/ContextCleanerSuite.scala
index 92ed24408384f..5434e82c95b1b 100644
--- a/core/src/test/scala/org/apache/spark/ContextCleanerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ContextCleanerSuite.scala
@@ -291,14 +291,14 @@ class ContextCleanerSuite extends ContextCleanerSuiteBase {
     val shuffleIds = 0 until sc.newShuffleId
     val broadcastIds = broadcastBuffer.map(_.id)
 
-    val preGCTester = new CleanerTester(sc, rddIds, shuffleIds, broadcastIds)
+    val preGCTester = new CleanerTester(sc, rddIds, shuffleIds, broadcastIds.toSeq)
     runGC()
     intercept[Exception] {
       preGCTester.assertCleanup()(timeout(1.second))
     }
 
     // Test that GC triggers the cleanup of all variables after the dereferencing them
-    val postGCTester = new CleanerTester(sc, rddIds, shuffleIds, broadcastIds)
+    val postGCTester = new CleanerTester(sc, rddIds, shuffleIds, broadcastIds.toSeq)
     broadcastBuffer.clear()
     rddBuffer.clear()
     runGC()
@@ -309,7 +309,7 @@ class ContextCleanerSuite extends ContextCleanerSuiteBase {
     assert(sc.env.blockManager.master.getMatchingBlockIds({
       case BroadcastBlockId(`taskClosureBroadcastId`, _) => true
       case _ => false
-    }, askSlaves = true).isEmpty)
+    }, askStorageEndpoints = true).isEmpty)
   }
 
   test("automatically cleanup RDD + shuffle + broadcast in distributed mode") {
@@ -331,14 +331,14 @@ class ContextCleanerSuite extends ContextCleanerSuiteBase {
     val shuffleIds = 0 until sc.newShuffleId
     val broadcastIds = broadcastBuffer.map(_.id)
 
-    val preGCTester = new CleanerTester(sc, rddIds, shuffleIds, broadcastIds)
+    val preGCTester = new CleanerTester(sc, rddIds, shuffleIds, broadcastIds.toSeq)
     runGC()
     intercept[Exception] {
       preGCTester.assertCleanup()(timeout(1.second))
     }
 
     // Test that GC triggers the cleanup of all variables after the dereferencing them
-    val postGCTester = new CleanerTester(sc, rddIds, shuffleIds, broadcastIds)
+    val postGCTester = new CleanerTester(sc, rddIds, shuffleIds, broadcastIds.toSeq)
     broadcastBuffer.clear()
     rddBuffer.clear()
     runGC()
@@ -349,7 +349,7 @@ class ContextCleanerSuite extends ContextCleanerSuiteBase {
     assert(sc.env.blockManager.master.getMatchingBlockIds({
       case BroadcastBlockId(`taskClosureBroadcastId`, _) => true
       case _ => false
-    }, askSlaves = true).isEmpty)
+    }, askStorageEndpoints = true).isEmpty)
   }
 }
 
@@ -368,7 +368,7 @@ class CleanerTester(
 
   val toBeCleanedRDDIds = new HashSet[Int] ++= rddIds
   val toBeCleanedShuffleIds = new HashSet[Int] ++= shuffleIds
-  val toBeCleanedBroadcstIds = new HashSet[Long] ++= broadcastIds
+  val toBeCleanedBroadcastIds = new HashSet[Long] ++= broadcastIds
   val toBeCheckpointIds = new HashSet[Long] ++= checkpointIds
   val isDistributed = !sc.isLocal
 
@@ -384,7 +384,7 @@ class CleanerTester(
     }
 
     def broadcastCleaned(broadcastId: Long): Unit = {
-      toBeCleanedBroadcstIds.synchronized { toBeCleanedBroadcstIds -= broadcastId }
+      toBeCleanedBroadcastIds.synchronized { toBeCleanedBroadcastIds -= broadcastId }
       logInfo("Broadcast " + broadcastId + " cleaned")
     }
 
@@ -508,8 +508,8 @@ class CleanerTester(
     val s2 = toBeCleanedShuffleIds.synchronized {
       toBeCleanedShuffleIds.toSeq.sorted.mkString("[", ", ", "]")
     }
-    val s3 = toBeCleanedBroadcstIds.synchronized {
-      toBeCleanedBroadcstIds.toSeq.sorted.mkString("[", ", ", "]")
+    val s3 = toBeCleanedBroadcastIds.synchronized {
+      toBeCleanedBroadcastIds.toSeq.sorted.mkString("[", ", ", "]")
     }
     s"""
        |\tRDDs = $s1
@@ -521,14 +521,14 @@ class CleanerTester(
   private def isAllCleanedUp =
     toBeCleanedRDDIds.synchronized { toBeCleanedRDDIds.isEmpty } &&
     toBeCleanedShuffleIds.synchronized { toBeCleanedShuffleIds.isEmpty } &&
-    toBeCleanedBroadcstIds.synchronized { toBeCleanedBroadcstIds.isEmpty } &&
+    toBeCleanedBroadcastIds.synchronized { toBeCleanedBroadcastIds.isEmpty } &&
     toBeCheckpointIds.synchronized { toBeCheckpointIds.isEmpty }
 
   private def getRDDBlocks(rddId: Int): Seq[BlockId] = {
     blockManager.master.getMatchingBlockIds( _ match {
       case RDDBlockId(`rddId`, _) => true
       case _ => false
-    }, askSlaves = true)
+    }, askStorageEndpoints = true)
   }
 
   private def getShuffleBlocks(shuffleId: Int): Seq[BlockId] = {
@@ -536,14 +536,14 @@ class CleanerTester(
       case ShuffleBlockId(`shuffleId`, _, _) => true
       case ShuffleIndexBlockId(`shuffleId`, _, _) => true
       case _ => false
-    }, askSlaves = true)
+    }, askStorageEndpoints = true)
   }
 
   private def getBroadcastBlocks(broadcastId: Long): Seq[BlockId] = {
     blockManager.master.getMatchingBlockIds( _ match {
       case BroadcastBlockId(`broadcastId`, _) => true
       case _ => false
-    }, askSlaves = true)
+    }, askStorageEndpoints = true)
   }
 
   private def blockManager = sc.env.blockManager
diff --git a/core/src/test/scala/org/apache/spark/DistributedSuite.scala b/core/src/test/scala/org/apache/spark/DistributedSuite.scala
index 4d157b9607000..ce1df3adf6352 100644
--- a/core/src/test/scala/org/apache/spark/DistributedSuite.scala
+++ b/core/src/test/scala/org/apache/spark/DistributedSuite.scala
@@ -18,8 +18,8 @@
 package org.apache.spark
 
 import org.scalatest.Assertions._
-import org.scalatest.Matchers
 import org.scalatest.concurrent.{Signaler, ThreadSignaler, TimeLimits}
+import org.scalatest.matchers.must.Matchers
 import org.scalatest.time.{Millis, Span}
 
 import org.apache.spark.internal.config
@@ -38,18 +38,18 @@ class DistributedSuite extends SparkFunSuite with Matchers with LocalSparkContex
   // Necessary to make ScalaTest 3.x interrupt a thread on the JVM like ScalaTest 2.2.x
   implicit val defaultSignaler: Signaler = ThreadSignaler
 
-  val clusterUrl = "local-cluster[2,1,1024]"
+  val clusterUrl = "local-cluster[3,1,1024]"
 
   test("task throws not serializable exception") {
     // Ensures that executors do not crash when an exn is not serializable. If executors crash,
     // this test will hang. Correct behavior is that executors don't crash but fail tasks
     // and the scheduler throws a SparkException.
 
-    // numSlaves must be less than numPartitions
-    val numSlaves = 3
+    // numWorkers must be less than numPartitions
+    val numWorkers = 3
     val numPartitions = 10
 
-    sc = new SparkContext("local-cluster[%s,1,1024]".format(numSlaves), "test")
+    sc = new SparkContext("local-cluster[%s,1,1024]".format(numWorkers), "test")
     val data = sc.parallelize(1 to 100, numPartitions).
       map(x => throw new NotSerializableExn(new NotSerializableClass))
     intercept[SparkException] {
@@ -69,10 +69,10 @@ class DistributedSuite extends SparkFunSuite with Matchers with LocalSparkContex
     )
 
     masterStrings.foreach {
-      case LOCAL_CLUSTER_REGEX(numSlaves, coresPerSlave, memoryPerSlave) =>
-        assert(numSlaves.toInt == 2)
-        assert(coresPerSlave.toInt == 1)
-        assert(memoryPerSlave.toInt == 1024)
+      case LOCAL_CLUSTER_REGEX(numWorkers, coresPerWorker, memoryPerWorker) =>
+        assert(numWorkers.toInt == 2)
+        assert(coresPerWorker.toInt == 1)
+        assert(memoryPerWorker.toInt == 1024)
     }
   }
 
@@ -174,7 +174,7 @@ class DistributedSuite extends SparkFunSuite with Matchers with LocalSparkContex
 
   private def testCaching(conf: SparkConf, storageLevel: StorageLevel): Unit = {
     sc = new SparkContext(conf.setMaster(clusterUrl).setAppName("test"))
-    TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
+    TestUtils.waitUntilExecutorsUp(sc, 3, 60000)
     val data = sc.parallelize(1 to 1000, 10)
     val cachedData = data.persist(storageLevel)
     assert(cachedData.count === 1000)
@@ -206,7 +206,8 @@ class DistributedSuite extends SparkFunSuite with Matchers with LocalSparkContex
     "caching on disk" -> StorageLevel.DISK_ONLY,
     "caching in memory, replicated" -> StorageLevel.MEMORY_ONLY_2,
     "caching in memory, serialized, replicated" -> StorageLevel.MEMORY_ONLY_SER_2,
-    "caching on disk, replicated" -> StorageLevel.DISK_ONLY_2,
+    "caching on disk, replicated 2" -> StorageLevel.DISK_ONLY_2,
+    "caching on disk, replicated 3" -> StorageLevel.DISK_ONLY_3,
     "caching in memory and disk, replicated" -> StorageLevel.MEMORY_AND_DISK_2,
     "caching in memory and disk, serialized, replicated" -> StorageLevel.MEMORY_AND_DISK_SER_2
   ).foreach { case (testName, storageLevel) =>
@@ -227,7 +228,8 @@ class DistributedSuite extends SparkFunSuite with Matchers with LocalSparkContex
     assert(data.count() === size)
     assert(data.count() === size)
     // ensure only a subset of partitions were cached
-    val rddBlocks = sc.env.blockManager.master.getMatchingBlockIds(_.isRDD, askSlaves = true)
+    val rddBlocks = sc.env.blockManager.master.getMatchingBlockIds(_.isRDD,
+      askStorageEndpoints = true)
     assert(rddBlocks.size === 0, s"expected no RDD blocks, found ${rddBlocks.size}")
   }
 
@@ -244,7 +246,8 @@ class DistributedSuite extends SparkFunSuite with Matchers with LocalSparkContex
     assert(data.count() === size)
     assert(data.count() === size)
     // ensure only a subset of partitions were cached
-    val rddBlocks = sc.env.blockManager.master.getMatchingBlockIds(_.isRDD, askSlaves = true)
+    val rddBlocks = sc.env.blockManager.master.getMatchingBlockIds(_.isRDD,
+      askStorageEndpoints = true)
     assert(rddBlocks.size > 0, "no RDD blocks found")
     assert(rddBlocks.size < numPartitions, s"too many RDD blocks found, expected <$numPartitions")
   }
diff --git a/core/src/test/scala/org/apache/spark/ExecutorAllocationManagerSuite.scala b/core/src/test/scala/org/apache/spark/ExecutorAllocationManagerSuite.scala
index 807f0eb808f9b..2fb5140d38b2c 100644
--- a/core/src/test/scala/org/apache/spark/ExecutorAllocationManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ExecutorAllocationManagerSuite.scala
@@ -21,15 +21,16 @@ import java.util.concurrent.TimeUnit
 
 import scala.collection.mutable
 
-import org.mockito.ArgumentMatchers.{any, eq => meq}
-import org.mockito.Mockito.{mock, never, times, verify, when}
+import org.mockito.ArgumentMatchers.any
+import org.mockito.Mockito._
 import org.scalatest.PrivateMethodTester
 
 import org.apache.spark.executor.ExecutorMetrics
 import org.apache.spark.internal.config
-import org.apache.spark.internal.config.Tests.TEST_SCHEDULE_INTERVAL
+import org.apache.spark.internal.config.DECOMMISSION_ENABLED
+import org.apache.spark.internal.config.Tests.TEST_DYNAMIC_ALLOCATION_SCHEDULE_ENABLED
 import org.apache.spark.metrics.MetricsSystem
-import org.apache.spark.resource.{ExecutorResourceRequests, ResourceProfile, ResourceProfileBuilder, ResourceProfileManager, TaskResourceRequests}
+import org.apache.spark.resource._
 import org.apache.spark.resource.ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID
 import org.apache.spark.scheduler._
 import org.apache.spark.scheduler.cluster.ExecutorInfo
@@ -267,7 +268,7 @@ class ExecutorAllocationManagerSuite extends SparkFunSuite {
 
   test("add executors multiple profiles initial num same as needed") {
     // test when the initial number of executors equals the number needed for the first
-    // stage using a non default profile to make sure we request the intitial number
+    // stage using a non default profile to make sure we request the initial number
     // properly. Here initial is 2, each executor in ResourceProfile 1 can have 2 tasks
     // per executor, and start a stage with 4 tasks, which would need 2 executors.
     val clock = new ManualClock(8888L)
@@ -501,6 +502,175 @@ class ExecutorAllocationManagerSuite extends SparkFunSuite {
     assert(numExecutorsToAddForDefaultProfile(manager) === 1)
   }
 
+  test("SPARK-31418: one stage being unschedulable") {
+    val clock = new ManualClock()
+    val conf = createConf(0, 5, 0).set(config.EXECUTOR_CORES, 2)
+    val manager = createManager(conf, clock = clock)
+    val updatesNeeded =
+      new mutable.HashMap[ResourceProfile, ExecutorAllocationManager.TargetNumUpdates]
+
+    post(SparkListenerStageSubmitted(createStageInfo(0, 2)))
+
+    assert(addExecutorsToTargetForDefaultProfile(manager, updatesNeeded) === 1)
+    doUpdateRequest(manager, updatesNeeded.toMap, clock.getTimeMillis())
+    assert(addExecutorsToTargetForDefaultProfile(manager, updatesNeeded) === 0)
+
+    onExecutorAddedDefaultProfile(manager, "0")
+    val t1 = createTaskInfo(0, 0, executorId = s"0")
+    val t2 = createTaskInfo(1, 1, executorId = s"0")
+    post(SparkListenerTaskStart(0, 0, t1))
+    post(SparkListenerTaskStart(0, 0, t2))
+
+    assert(numExecutorsTarget(manager, defaultProfile.id) === 1)
+    assert(maxNumExecutorsNeededPerResourceProfile(manager, defaultProfile) == 1)
+
+    // Stage 0 becomes unschedulable due to excludeOnFailure
+    post(SparkListenerUnschedulableTaskSetAdded(0, 0))
+    clock.advance(1000)
+    manager invokePrivate _updateAndSyncNumExecutorsTarget(clock.nanoTime())
+    // Assert that we are getting additional executor to schedule unschedulable tasks
+    assert(numExecutorsTarget(manager, defaultProfile.id) === 2)
+    assert(maxNumExecutorsNeededPerResourceProfile(manager, defaultProfile) == 2)
+
+    // Add a new executor
+    onExecutorAddedDefaultProfile(manager, "1")
+    // Now once the task becomes schedulable, clear the unschedulableTaskSets
+    post(SparkListenerUnschedulableTaskSetRemoved(0, 0))
+    clock.advance(1000)
+    manager invokePrivate _updateAndSyncNumExecutorsTarget(clock.nanoTime())
+    assert(numExecutorsTarget(manager, defaultProfile.id) === 1)
+    assert(maxNumExecutorsNeededPerResourceProfile(manager, defaultProfile) == 1)
+  }
+
+  test("SPARK-31418: multiple stages being unschedulable") {
+    val clock = new ManualClock()
+    val conf = createConf(0, 10, 0).set(config.EXECUTOR_CORES, 2)
+    val manager = createManager(conf, clock = clock)
+    val updatesNeeded =
+      new mutable.HashMap[ResourceProfile, ExecutorAllocationManager.TargetNumUpdates]
+
+    post(SparkListenerStageSubmitted(createStageInfo(0, 2)))
+    post(SparkListenerStageSubmitted(createStageInfo(1, 2)))
+    post(SparkListenerStageSubmitted(createStageInfo(2, 2)))
+
+    assert(addExecutorsToTargetForDefaultProfile(manager, updatesNeeded) === 1)
+    doUpdateRequest(manager, updatesNeeded.toMap, clock.getTimeMillis())
+    assert(addExecutorsToTargetForDefaultProfile(manager, updatesNeeded) === 2)
+    doUpdateRequest(manager, updatesNeeded.toMap, clock.getTimeMillis())
+    assert(addExecutorsToTargetForDefaultProfile(manager, updatesNeeded) === 0)
+
+    // Add necessary executors
+    (0 to 2).foreach(execId => onExecutorAddedDefaultProfile(manager, execId.toString))
+
+    // Start all the tasks
+    (0 to 2).foreach {
+      i =>
+        val t1Info = createTaskInfo(0, (i * 2) + 1, executorId = s"${i / 2}")
+        val t2Info = createTaskInfo(1, (i * 2) + 2, executorId = s"${i / 2}")
+        post(SparkListenerTaskStart(i, 0, t1Info))
+        post(SparkListenerTaskStart(i, 0, t2Info))
+    }
+    assert(numExecutorsTarget(manager, defaultProfile.id) === 3)
+    assert(maxNumExecutorsNeededPerResourceProfile(manager, defaultProfile) == 3)
+
+    // Complete the stage 0 tasks.
+    val t1Info = createTaskInfo(0, 0, executorId = s"0")
+    val t2Info = createTaskInfo(1, 1, executorId = s"0")
+    post(SparkListenerTaskEnd(0, 0, null, Success, t1Info, new ExecutorMetrics, null))
+    post(SparkListenerTaskEnd(0, 0, null, Success, t2Info, new ExecutorMetrics, null))
+    post(SparkListenerStageCompleted(createStageInfo(0, 2)))
+
+    // Stage 1 and 2 becomes unschedulable now due to excludeOnFailure
+    post(SparkListenerUnschedulableTaskSetAdded(1, 0))
+    post(SparkListenerUnschedulableTaskSetAdded(2, 0))
+
+    clock.advance(1000)
+    manager invokePrivate _updateAndSyncNumExecutorsTarget(clock.nanoTime())
+    // Assert that we are getting additional executor to schedule unschedulable tasks
+    assert(numExecutorsTarget(manager, defaultProfile.id) === 4)
+    assert(maxNumExecutorsNeededPerResourceProfile(manager, defaultProfile) == 4)
+
+    // Add a new executor
+    onExecutorAddedDefaultProfile(manager, "3")
+
+    // Now once the task becomes schedulable, clear the unschedulableTaskSets
+    post(SparkListenerUnschedulableTaskSetRemoved(1, 0))
+    clock.advance(1000)
+    manager invokePrivate _updateAndSyncNumExecutorsTarget(clock.nanoTime())
+    assert(numExecutorsTarget(manager, defaultProfile.id) === 4)
+    assert(maxNumExecutorsNeededPerResourceProfile(manager, defaultProfile) == 5)
+  }
+
+  test("SPARK-31418: remove executors after unschedulable tasks end") {
+    val clock = new ManualClock()
+    val stage = createStageInfo(0, 10)
+    val conf = createConf(0, 6, 0).set(config.EXECUTOR_CORES, 2)
+    val manager = createManager(conf, clock = clock)
+    val updatesNeeded =
+      new mutable.HashMap[ResourceProfile, ExecutorAllocationManager.TargetNumUpdates]
+
+    post(SparkListenerStageSubmitted(stage))
+    assert(addExecutorsToTargetForDefaultProfile(manager, updatesNeeded) === 1)
+    doUpdateRequest(manager, updatesNeeded.toMap, clock.getTimeMillis())
+    assert(addExecutorsToTargetForDefaultProfile(manager, updatesNeeded) === 2)
+    doUpdateRequest(manager, updatesNeeded.toMap, clock.getTimeMillis())
+    assert(addExecutorsToTargetForDefaultProfile(manager, updatesNeeded) === 2)
+    doUpdateRequest(manager, updatesNeeded.toMap, clock.getTimeMillis())
+    assert(addExecutorsToTargetForDefaultProfile(manager, updatesNeeded) === 0)
+    doUpdateRequest(manager, updatesNeeded.toMap, clock.getTimeMillis())
+
+    (0 to 4).foreach(execId => onExecutorAddedDefaultProfile(manager, execId.toString))
+    (0 to 9).map { i => createTaskInfo(i, i, executorId = s"${i / 2}") }.foreach {
+      info => post(SparkListenerTaskStart(0, 0, info))
+    }
+    assert(numExecutorsTarget(manager, defaultProfile.id) === 5)
+    assert(maxNumExecutorsNeededPerResourceProfile(manager, defaultProfile) == 5)
+
+    // 8 tasks (0 - 7) finished
+    (0 to 7).map { i => createTaskInfo(i, i, executorId = s"${i / 2}") }.foreach {
+      info => post(SparkListenerTaskEnd(0, 0, null, Success, info, new ExecutorMetrics, null))
+    }
+    clock.advance(1000)
+    manager invokePrivate _updateAndSyncNumExecutorsTarget(clock.nanoTime())
+    assert(numExecutorsTarget(manager, defaultProfile.id) === 1)
+    assert(maxNumExecutorsNeededPerResourceProfile(manager, defaultProfile) == 1)
+    (0 to 3).foreach { i => assert(removeExecutorDefaultProfile(manager, i.toString)) }
+    (0 to 3).foreach { i => onExecutorRemoved(manager, i.toString) }
+
+    // Now due to executor being excluded, the task becomes unschedulable
+    post(SparkListenerUnschedulableTaskSetAdded(0, 0))
+    clock.advance(1000)
+    manager invokePrivate _updateAndSyncNumExecutorsTarget(clock.nanoTime())
+    assert(numExecutorsTarget(manager, defaultProfile.id) === 2)
+    assert(maxNumExecutorsNeededPerResourceProfile(manager, defaultProfile) == 2)
+
+    // New executor got added
+    onExecutorAddedDefaultProfile(manager, "5")
+
+    // Now once the task becomes schedulable, clear the unschedulableTaskSets
+    post(SparkListenerUnschedulableTaskSetRemoved(0, 0))
+    clock.advance(1000)
+    manager invokePrivate _updateAndSyncNumExecutorsTarget(clock.nanoTime())
+    assert(numExecutorsTarget(manager, defaultProfile.id) === 1)
+    assert(maxNumExecutorsNeededPerResourceProfile(manager, defaultProfile) == 1)
+    post(SparkListenerTaskEnd(0, 0, null, Success,
+      createTaskInfo(9, 9, "4"), new ExecutorMetrics, null))
+    // Unschedulable task successfully ran on the new executor provisioned
+    post(SparkListenerTaskEnd(0, 0, null, Success,
+      createTaskInfo(8, 8, "5"), new ExecutorMetrics, null))
+    clock.advance(1000)
+    manager invokePrivate _updateAndSyncNumExecutorsTarget(clock.nanoTime())
+    post(SparkListenerStageCompleted(stage))
+    clock.advance(1000)
+    manager invokePrivate _updateAndSyncNumExecutorsTarget(clock.nanoTime())
+    assert(numExecutorsTarget(manager, defaultProfile.id) === 0)
+    assert(maxNumExecutorsNeededPerResourceProfile(manager, defaultProfile) == 0)
+    assert(removeExecutorDefaultProfile(manager, "4"))
+    onExecutorRemoved(manager, "4")
+    assert(removeExecutorDefaultProfile(manager, "5"))
+    onExecutorRemoved(manager, "5")
+  }
+
   test("SPARK-30511 remove executors when speculative tasks end") {
     val clock = new ManualClock()
     val stage = createStageInfo(0, 40)
@@ -766,6 +936,53 @@ class ExecutorAllocationManagerSuite extends SparkFunSuite {
     assert(executorsPendingToRemove(manager).isEmpty)
   }
 
+  test("SPARK-33763: metrics to track dynamic allocation (decommissionEnabled=false)") {
+    val manager = createManager(createConf(3, 5, 3))
+    (1 to 5).map(_.toString).foreach { id => onExecutorAddedDefaultProfile(manager, id) }
+
+    assert(executorsPendingToRemove(manager).isEmpty)
+    assert(removeExecutorsDefaultProfile(manager, Seq("1", "2")) === Seq("1", "2"))
+    assert(executorsPendingToRemove(manager).contains("1"))
+    assert(executorsPendingToRemove(manager).contains("2"))
+
+    onExecutorRemoved(manager, "1", "driver requested exit")
+    assert(manager.executorAllocationManagerSource.driverKilled.getCount() === 1)
+    assert(manager.executorAllocationManagerSource.exitedUnexpectedly.getCount() === 0)
+
+    onExecutorRemoved(manager, "2", "another driver requested exit")
+    assert(manager.executorAllocationManagerSource.driverKilled.getCount() === 2)
+    assert(manager.executorAllocationManagerSource.exitedUnexpectedly.getCount() === 0)
+
+    onExecutorRemoved(manager, "3", "this will be an unexpected exit")
+    assert(manager.executorAllocationManagerSource.driverKilled.getCount() === 2)
+    assert(manager.executorAllocationManagerSource.exitedUnexpectedly.getCount() === 1)
+  }
+
+  test("SPARK-33763: metrics to track dynamic allocation (decommissionEnabled = true)") {
+    val manager = createManager(createConf(3, 5, 3, decommissioningEnabled = true))
+    (1 to 5).map(_.toString).foreach { id => onExecutorAddedDefaultProfile(manager, id) }
+
+    assert(executorsPendingToRemove(manager).isEmpty)
+    assert(removeExecutorsDefaultProfile(manager, Seq("1", "2")) === Seq("1", "2"))
+    assert(executorsDecommissioning(manager).contains("1"))
+    assert(executorsDecommissioning(manager).contains("2"))
+
+    onExecutorRemoved(manager, "1", ExecutorLossMessage.decommissionFinished)
+    assert(manager.executorAllocationManagerSource.gracefullyDecommissioned.getCount() === 1)
+    assert(manager.executorAllocationManagerSource.decommissionUnfinished.getCount() === 0)
+    assert(manager.executorAllocationManagerSource.exitedUnexpectedly.getCount() === 0)
+
+    onExecutorRemoved(manager, "2", "stopped before gracefully finished")
+    assert(manager.executorAllocationManagerSource.gracefullyDecommissioned.getCount() === 1)
+    assert(manager.executorAllocationManagerSource.decommissionUnfinished.getCount() === 1)
+    assert(manager.executorAllocationManagerSource.exitedUnexpectedly.getCount() === 0)
+
+    onExecutorRemoved(manager, "3", "this will be an unexpected exit")
+    assert(manager.executorAllocationManagerSource.gracefullyDecommissioned.getCount() === 1)
+    assert(manager.executorAllocationManagerSource.decommissionUnfinished.getCount() === 1)
+    assert(manager.executorAllocationManagerSource.exitedUnexpectedly.getCount() === 1)
+  }
+
   test("remove multiple executors") {
     val manager = createManager(createConf(5, 10, 5))
     (1 to 10).map(_.toString).foreach { id => onExecutorAddedDefaultProfile(manager, id) }
@@ -1101,6 +1318,68 @@ class ExecutorAllocationManagerSuite extends SparkFunSuite {
     assert(executorsPendingToRemove(manager).size === 6) // limit reached (1 executor remaining)
   }
 
+  test("mock polling loop remove with decommissioning") {
+    val clock = new ManualClock(2020L)
+    val manager = createManager(createConf(1, 20, 1, true), clock = clock)
+
+    // Remove idle executors on timeout
+    onExecutorAddedDefaultProfile(manager, "executor-1")
+    onExecutorAddedDefaultProfile(manager, "executor-2")
+    onExecutorAddedDefaultProfile(manager, "executor-3")
+    assert(executorsDecommissioning(manager).isEmpty)
+    assert(executorsPendingToRemove(manager).isEmpty)
+
+    // idle threshold not reached yet
+    clock.advance(executorIdleTimeout * 1000 / 2)
+    schedule(manager)
+    assert(manager.executorMonitor.timedOutExecutors().isEmpty)
+    assert(executorsPendingToRemove(manager).isEmpty)
+    assert(executorsDecommissioning(manager).isEmpty)
+
+    // idle threshold exceeded
+    clock.advance(executorIdleTimeout * 1000)
+    assert(manager.executorMonitor.timedOutExecutors().size === 3)
+    schedule(manager)
+    assert(executorsPendingToRemove(manager).isEmpty) // limit reached (1 executor remaining)
+    assert(executorsDecommissioning(manager).size === 2) // limit reached (1 executor remaining)
+
+    // Mark a subset as busy - only idle executors should be removed
+    onExecutorAddedDefaultProfile(manager, "executor-4")
+    onExecutorAddedDefaultProfile(manager, "executor-5")
+    onExecutorAddedDefaultProfile(manager, "executor-6")
+    onExecutorAddedDefaultProfile(manager, "executor-7")
+    assert(manager.executorMonitor.executorCount === 7)
+    assert(executorsPendingToRemove(manager).isEmpty) // no pending to be removed
+    assert(executorsDecommissioning(manager).size === 2) // 2 decommissioning
+    onExecutorBusy(manager, "executor-4")
+    onExecutorBusy(manager, "executor-5")
+    onExecutorBusy(manager, "executor-6") // 3 busy and 2 idle (of the 5 active ones)
+
+    // after scheduling, the previously timed out executor should be removed, since
+    // there are new active ones.
+    schedule(manager)
+    assert(executorsDecommissioning(manager).size === 3)
+
+    // advance the clock so that idle executors should time out and move to the pending list
+    clock.advance(executorIdleTimeout * 1000)
+    schedule(manager)
+    assert(executorsPendingToRemove(manager).size === 0)
+    assert(executorsDecommissioning(manager).size === 4)
+    assert(!executorsDecommissioning(manager).contains("executor-4"))
+    assert(!executorsDecommissioning(manager).contains("executor-5"))
+    assert(!executorsDecommissioning(manager).contains("executor-6"))
+
+    // Busy executors are now idle and should be removed
+    onExecutorIdle(manager, "executor-4")
+    onExecutorIdle(manager, "executor-5")
+    onExecutorIdle(manager, "executor-6")
+    schedule(manager)
+    assert(executorsDecommissioning(manager).size === 4)
+    clock.advance(executorIdleTimeout * 1000)
+    schedule(manager)
+    assert(executorsDecommissioning(manager).size === 6) // limit reached (1 executor remaining)
+  }
+
   test("listeners trigger add executors correctly") {
     val manager = createManager(createConf(1, 20, 1))
     assert(addTime(manager) === NOT_SET)
@@ -1356,7 +1635,7 @@ class ExecutorAllocationManagerSuite extends SparkFunSuite {
   test("SPARK-23365 Don't update target num executors when killing idle executors") {
     val clock = new ManualClock()
     val manager = createManager(
-      createConf(1, 2, 1).set(config.DYN_ALLOCATION_TESTING, false),
+      createConf(1, 2, 1),
       clock = clock)
 
     when(client.requestTotalExecutors(any(), any(), any())).thenReturn(true)
@@ -1384,19 +1663,17 @@ class ExecutorAllocationManagerSuite extends SparkFunSuite {
     clock.advance(1000)
     manager invokePrivate _updateAndSyncNumExecutorsTarget(clock.nanoTime())
     assert(numExecutorsTargetForDefaultProfileId(manager) === 1)
-    verify(client, never).killExecutors(any(), any(), any(), any())
+    assert(manager.executorMonitor.executorsPendingToRemove().isEmpty)
 
     // now we cross the idle timeout for executor-1, so we kill it.  the really important
     // thing here is that we do *not* ask the executor allocation client to adjust the target
     // number of executors down
-    when(client.killExecutors(Seq("executor-1"), false, false, false))
-      .thenReturn(Seq("executor-1"))
     clock.advance(3000)
     schedule(manager)
     assert(maxNumExecutorsNeededPerResourceProfile(manager, defaultProfile) === 1)
     assert(numExecutorsTargetForDefaultProfileId(manager) === 1)
     // here's the important verify -- we did kill the executors, but did not adjust the target count
-    verify(client).killExecutors(Seq("executor-1"), false, false, false)
+    assert(manager.executorMonitor.executorsPendingToRemove() === Set("executor-1"))
   }
 
   test("SPARK-26758 check executor target number after idle time out ") {
@@ -1419,7 +1696,8 @@ class ExecutorAllocationManagerSuite extends SparkFunSuite {
   private def createConf(
       minExecutors: Int = 1,
       maxExecutors: Int = 5,
-      initialExecutors: Int = 1): SparkConf = {
+      initialExecutors: Int = 1,
+      decommissioningEnabled: Boolean = false): SparkConf = {
     val sparkConf = new SparkConf()
       .set(config.DYN_ALLOCATION_ENABLED, true)
       .set(config.DYN_ALLOCATION_MIN_EXECUTORS, minExecutors)
@@ -1432,9 +1710,11 @@ class ExecutorAllocationManagerSuite extends SparkFunSuite {
       .set(config.DYN_ALLOCATION_EXECUTOR_IDLE_TIMEOUT.key, s"${executorIdleTimeout.toString}s")
       .set(config.SHUFFLE_SERVICE_ENABLED, true)
       .set(config.DYN_ALLOCATION_TESTING, true)
-      // SPARK-22864: effectively disable the allocation schedule by setting the period to a
-      // really long value.
-      .set(TEST_SCHEDULE_INTERVAL, 10000L)
+      // SPARK-22864/SPARK-32287: effectively disable the allocation schedule for the tests so that
+      // we won't result in the race condition between thread "spark-dynamic-executor-allocation"
+      // and thread "pool-1-thread-1-ScalaTest-running".
+      .set(TEST_DYNAMIC_ALLOCATION_SCHEDULE_ENABLED, false)
+      .set(DECOMMISSION_ENABLED, decommissioningEnabled)
     sparkConf
   }
 
@@ -1442,7 +1722,7 @@ class ExecutorAllocationManagerSuite extends SparkFunSuite {
       conf: SparkConf,
       clock: Clock = new SystemClock()): ExecutorAllocationManager = {
     ResourceProfile.reInitDefaultProfile(conf)
-    rpManager = new ResourceProfileManager(conf)
+    rpManager = new ResourceProfileManager(conf, listenerBus)
     val manager = new ExecutorAllocationManager(client, listenerBus, conf, clock = clock,
       resourceProfileManager = rpManager)
     managers += manager
@@ -1468,8 +1748,11 @@ class ExecutorAllocationManagerSuite extends SparkFunSuite {
     post(SparkListenerExecutorAdded(0L, id, execInfo))
   }
 
-  private def onExecutorRemoved(manager: ExecutorAllocationManager, id: String): Unit = {
-    post(SparkListenerExecutorRemoved(0L, id, null))
+  private def onExecutorRemoved(
+      manager: ExecutorAllocationManager,
+      id: String,
+      reason: String = null): Unit = {
+    post(SparkListenerExecutorRemoved(0L, id, reason))
   }
 
   private def onExecutorBusy(manager: ExecutorAllocationManager, id: String): Unit = {
@@ -1501,6 +1784,10 @@ class ExecutorAllocationManagerSuite extends SparkFunSuite {
   private def executorsPendingToRemove(manager: ExecutorAllocationManager): Set[String] = {
     manager.executorMonitor.executorsPendingToRemove()
   }
+
+  private def executorsDecommissioning(manager: ExecutorAllocationManager): Set[String] = {
+    manager.executorMonitor.executorsDecommissioning()
+  }
 }
 
 /**
diff --git a/core/src/test/scala/org/apache/spark/ExternalShuffleServiceSuite.scala b/core/src/test/scala/org/apache/spark/ExternalShuffleServiceSuite.scala
index c217419f4092e..48c1cc5906f30 100644
--- a/core/src/test/scala/org/apache/spark/ExternalShuffleServiceSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ExternalShuffleServiceSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark
 
 import org.scalatest.BeforeAndAfterAll
 import org.scalatest.concurrent.Eventually
+import org.scalatest.matchers.should.Matchers._
 import org.scalatest.time.SpanSugar._
 
 import org.apache.spark.internal.config
@@ -72,12 +73,12 @@ class ExternalShuffleServiceSuite extends ShuffleSuite with BeforeAndAfterAll wi
     sc.env.blockManager.externalShuffleServiceEnabled should equal(true)
     sc.env.blockManager.blockStoreClient.getClass should equal(classOf[ExternalBlockStoreClient])
 
-    // In a slow machine, one slave may register hundreds of milliseconds ahead of the other one.
-    // If we don't wait for all slaves, it's possible that only one executor runs all jobs. Then
+    // In a slow machine, one executor may register hundreds of milliseconds ahead of the other one.
+    // If we don't wait for all executors, it's possible that only one executor runs all jobs. Then
     // all shuffle blocks will be in this executor, ShuffleBlockFetcherIterator will directly fetch
     // local blocks from the local BlockManager and won't send requests to ExternalShuffleService.
     // In this case, we won't receive FetchFailed. And it will make this test fail.
-    // Therefore, we should wait until all slaves are up
+    // Therefore, we should wait until all executors are up
     TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
 
     val rdd = sc.parallelize(0 until 1000, 10)
@@ -99,50 +100,6 @@ class ExternalShuffleServiceSuite extends ShuffleSuite with BeforeAndAfterAll wi
     e.getMessage should include ("Fetch failure will not retry stage due to testing config")
   }
 
-  test("SPARK-27651: read host local shuffle blocks from disk and avoid network remote fetches") {
-    val confWithHostLocalRead =
-      conf.clone.set(config.SHUFFLE_HOST_LOCAL_DISK_READING_ENABLED, true)
-    confWithHostLocalRead.set(config.STORAGE_LOCAL_DISK_BY_EXECUTORS_CACHE_SIZE, 5)
-    sc = new SparkContext("local-cluster[2,1,1024]", "test", confWithHostLocalRead)
-    sc.getConf.get(config.SHUFFLE_HOST_LOCAL_DISK_READING_ENABLED) should equal(true)
-    sc.env.blockManager.externalShuffleServiceEnabled should equal(true)
-    sc.env.blockManager.hostLocalDirManager.isDefined should equal(true)
-    sc.env.blockManager.blockStoreClient.getClass should equal(classOf[ExternalBlockStoreClient])
-
-    // In a slow machine, one slave may register hundreds of milliseconds ahead of the other one.
-    // If we don't wait for all slaves, it's possible that only one executor runs all jobs. Then
-    // all shuffle blocks will be in this executor, ShuffleBlockFetcherIterator will directly fetch
-    // local blocks from the local BlockManager and won't send requests to ExternalShuffleService.
-    // In this case, we won't receive FetchFailed. And it will make this test fail.
-    // Therefore, we should wait until all slaves are up
-    TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
-
-    val rdd = sc.parallelize(0 until 1000, 10)
-      .map { i => (i, 1) }
-      .reduceByKey(_ + _)
-
-    rdd.count()
-    rdd.count()
-
-    val cachedExecutors = rdd.mapPartitions { _ =>
-      SparkEnv.get.blockManager.hostLocalDirManager.map { localDirManager =>
-        localDirManager.getCachedHostLocalDirs().keySet.iterator
-      }.getOrElse(Iterator.empty)
-    }.collect().toSet
-
-    // both executors are caching the dirs of the other one
-    cachedExecutors should equal(sc.getExecutorIds().toSet)
-
-    // Invalidate the registered executors, disallowing access to their shuffle blocks (without
-    // deleting the actual shuffle files, so we could access them without the shuffle service).
-    // As directories are already cached there is no request to external shuffle service.
-    rpcHandler.applicationRemoved(sc.conf.getAppId, false /* cleanupLocalDirs */)
-
-    // Now Spark will not receive FetchFailed as host local blocks are read from the cached local
-    // disk directly
-    rdd.collect().map(_._2).sum should equal(1000)
-  }
-
   test("SPARK-25888: using external shuffle service fetching disk persisted blocks") {
     val confWithRddFetchEnabled = conf.clone.set(config.SHUFFLE_SERVICE_FETCH_RDD_ENABLED, true)
     sc = new SparkContext("local-cluster[1,1,1024]", "test", confWithRddFetchEnabled)
diff --git a/core/src/test/scala/org/apache/spark/FileSuite.scala b/core/src/test/scala/org/apache/spark/FileSuite.scala
index e9ee6b5dfb665..f953bf4043f33 100644
--- a/core/src/test/scala/org/apache/spark/FileSuite.scala
+++ b/core/src/test/scala/org/apache/spark/FileSuite.scala
@@ -170,7 +170,7 @@ class FileSuite extends SparkFunSuite with LocalSparkContext {
     val nums = sc.makeRDD(1 to 3).map(x => (x, "a" * x)) // (1,a), (2,aa), (3,aaa)
     nums.saveAsSequenceFile(outputDir)
     // Similar to the tests above, we read a SequenceFile, but this time we pass type params
-    // that are convertable to Writable instead of calling sequenceFile[IntWritable, Text]
+    // that are convertible to Writable instead of calling sequenceFile[IntWritable, Text]
     val output1 = sc.sequenceFile[Int, String](outputDir)
     assert(output1.collect().toList === List((1, "a"), (2, "aa"), (3, "aaa")))
     // Also try having one type be a subclass of Writable and one not
diff --git a/core/src/test/scala/org/apache/spark/FutureActionSuite.scala b/core/src/test/scala/org/apache/spark/FutureActionSuite.scala
index 70b6309be7d53..737a856ba13f1 100644
--- a/core/src/test/scala/org/apache/spark/FutureActionSuite.scala
+++ b/core/src/test/scala/org/apache/spark/FutureActionSuite.scala
@@ -19,7 +19,9 @@ package org.apache.spark
 
 import scala.concurrent.duration.Duration
 
-import org.scalatest.{BeforeAndAfter, Matchers}
+import org.scalatest.BeforeAndAfter
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.util.ThreadUtils
 
diff --git a/core/src/test/scala/org/apache/spark/HeartbeatReceiverSuite.scala b/core/src/test/scala/org/apache/spark/HeartbeatReceiverSuite.scala
index 312691302b064..879ce5584061e 100644
--- a/core/src/test/scala/org/apache/spark/HeartbeatReceiverSuite.scala
+++ b/core/src/test/scala/org/apache/spark/HeartbeatReceiverSuite.scala
@@ -76,7 +76,7 @@ class HeartbeatReceiverSuite
     sc = spy(new SparkContext(conf))
     scheduler = mock(classOf[TaskSchedulerImpl])
     when(sc.taskScheduler).thenReturn(scheduler)
-    when(scheduler.nodeBlacklist).thenReturn(Predef.Set[String]())
+    when(scheduler.excludedNodes).thenReturn(Predef.Set[String]())
     when(scheduler.sc).thenReturn(sc)
     heartbeatReceiverClock = new ManualClock
     heartbeatReceiver = new HeartbeatReceiver(sc, heartbeatReceiverClock)
@@ -219,6 +219,24 @@ class HeartbeatReceiverSuite
     fakeSchedulerBackend.stop()
   }
 
+  test("SPARK-34273: Do not reregister BlockManager when SparkContext is stopped") {
+    val blockManagerId = BlockManagerId(executorId1, "localhost", 12345)
+
+    heartbeatReceiverRef.askSync[Boolean](TaskSchedulerIsSet)
+    val response = heartbeatReceiverRef.askSync[HeartbeatResponse](
+      Heartbeat(executorId1, Array.empty, blockManagerId, mutable.Map.empty))
+    assert(response.reregisterBlockManager)
+
+    try {
+      sc.stopped.set(true)
+      val response = heartbeatReceiverRef.askSync[HeartbeatResponse](
+        Heartbeat(executorId1, Array.empty, blockManagerId, mutable.Map.empty))
+      assert(!response.reregisterBlockManager)
+    } finally {
+      sc.stopped.set(false)
+    }
+  }
+
   /** Manually send a heartbeat and return the response. */
   private def triggerHeartbeat(
       executorId: String,
@@ -261,7 +279,7 @@ class HeartbeatReceiverSuite
     // We may receive undesired SparkListenerExecutorAdded from LocalSchedulerBackend,
     // so exclude it from the map. See SPARK-10800.
     heartbeatReceiver.invokePrivate(_executorLastSeen()).
-      filterKeys(_ != SparkContext.DRIVER_IDENTIFIER)
+      filterKeys(_ != SparkContext.DRIVER_IDENTIFIER).toMap
   }
 }
 
@@ -287,6 +305,8 @@ private class FakeSchedulerBackend(
     resourceProfileManager: ResourceProfileManager)
   extends CoarseGrainedSchedulerBackend(scheduler, rpcEnv) {
 
+  def this() = this(null, null, null, null)
+
   protected override def doRequestTotalExecutors(
       resourceProfileToTotalExecs: Map[ResourceProfile, Int]): Future[Boolean] = {
     clusterManagerEndpoint.ask[Boolean](
diff --git a/core/src/test/scala/org/apache/spark/InternalAccumulatorSuite.scala b/core/src/test/scala/org/apache/spark/InternalAccumulatorSuite.scala
index 5399d868f46f1..f2b81e5153ae4 100644
--- a/core/src/test/scala/org/apache/spark/InternalAccumulatorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/InternalAccumulatorSuite.scala
@@ -220,7 +220,7 @@ class InternalAccumulatorSuite extends SparkFunSuite with LocalSparkContext {
       super.registerAccumulatorForCleanup(a)
     }
 
-    def accumsRegisteredForCleanup: Seq[Long] = accumsRegistered.toArray
+    def accumsRegisteredForCleanup: Seq[Long] = accumsRegistered.toSeq
   }
 
 }
diff --git a/core/src/test/scala/org/apache/spark/JobCancellationSuite.scala b/core/src/test/scala/org/apache/spark/JobCancellationSuite.scala
index 94ad8d8880027..082a92ef41d3b 100644
--- a/core/src/test/scala/org/apache/spark/JobCancellationSuite.scala
+++ b/core/src/test/scala/org/apache/spark/JobCancellationSuite.scala
@@ -25,7 +25,7 @@ import scala.concurrent.Future
 import scala.concurrent.duration._
 
 import org.scalatest.BeforeAndAfter
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
 
 import org.apache.spark.internal.config._
 import org.apache.spark.internal.config.Deploy._
diff --git a/core/src/test/scala/org/apache/spark/MapOutputTrackerSuite.scala b/core/src/test/scala/org/apache/spark/MapOutputTrackerSuite.scala
index d5ee19bde8edf..f4b47e2bb0cdc 100644
--- a/core/src/test/scala/org/apache/spark/MapOutputTrackerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/MapOutputTrackerSuite.scala
@@ -21,22 +21,23 @@ import scala.collection.mutable.ArrayBuffer
 
 import org.mockito.ArgumentMatchers.any
 import org.mockito.Mockito._
+import org.roaringbitmap.RoaringBitmap
 
 import org.apache.spark.LocalSparkContext._
 import org.apache.spark.broadcast.BroadcastManager
 import org.apache.spark.internal.config._
 import org.apache.spark.internal.config.Network.{RPC_ASK_TIMEOUT, RPC_MESSAGE_MAX_SIZE}
+import org.apache.spark.internal.config.Tests.IS_TESTING
 import org.apache.spark.rpc.{RpcAddress, RpcCallContext, RpcEnv}
-import org.apache.spark.scheduler.{CompressedMapStatus, MapStatus}
+import org.apache.spark.scheduler.{CompressedMapStatus, MapStatus, MergeStatus}
 import org.apache.spark.shuffle.FetchFailedException
 import org.apache.spark.storage.{BlockManagerId, ShuffleBlockId}
 
-class MapOutputTrackerSuite extends SparkFunSuite {
+class MapOutputTrackerSuite extends SparkFunSuite with LocalSparkContext {
   private val conf = new SparkConf
 
   private def newTrackerMaster(sparkConf: SparkConf = conf) = {
-    val broadcastManager = new BroadcastManager(true, sparkConf,
-      new SecurityManager(sparkConf))
+    val broadcastManager = new BroadcastManager(true, sparkConf)
     new MapOutputTrackerMaster(sparkConf, broadcastManager, true)
   }
 
@@ -59,7 +60,7 @@ class MapOutputTrackerSuite extends SparkFunSuite {
     val tracker = newTrackerMaster()
     tracker.trackerEndpoint = rpcEnv.setupEndpoint(MapOutputTracker.ENDPOINT_NAME,
       new MapOutputTrackerMasterEndpoint(rpcEnv, tracker, conf))
-    tracker.registerShuffle(10, 2)
+    tracker.registerShuffle(10, 2, MergeStatus.SHUFFLE_PUSH_DUMMY_NUM_REDUCES)
     assert(tracker.containsShuffle(10))
     val size1000 = MapStatus.decompressSize(MapStatus.compressSize(1000L))
     val size10000 = MapStatus.decompressSize(MapStatus.compressSize(10000L))
@@ -83,7 +84,7 @@ class MapOutputTrackerSuite extends SparkFunSuite {
     val tracker = newTrackerMaster()
     tracker.trackerEndpoint = rpcEnv.setupEndpoint(MapOutputTracker.ENDPOINT_NAME,
       new MapOutputTrackerMasterEndpoint(rpcEnv, tracker, conf))
-    tracker.registerShuffle(10, 2)
+    tracker.registerShuffle(10, 2, MergeStatus.SHUFFLE_PUSH_DUMMY_NUM_REDUCES)
     val compressedSize1000 = MapStatus.compressSize(1000L)
     val compressedSize10000 = MapStatus.compressSize(10000L)
     tracker.registerMapOutput(10, 0, MapStatus(BlockManagerId("a", "hostA", 1000),
@@ -106,7 +107,7 @@ class MapOutputTrackerSuite extends SparkFunSuite {
     val tracker = newTrackerMaster()
     tracker.trackerEndpoint = rpcEnv.setupEndpoint(MapOutputTracker.ENDPOINT_NAME,
       new MapOutputTrackerMasterEndpoint(rpcEnv, tracker, conf))
-    tracker.registerShuffle(10, 2)
+    tracker.registerShuffle(10, 2, MergeStatus.SHUFFLE_PUSH_DUMMY_NUM_REDUCES)
     val compressedSize1000 = MapStatus.compressSize(1000L)
     val compressedSize10000 = MapStatus.compressSize(10000L)
     tracker.registerMapOutput(10, 0, MapStatus(BlockManagerId("a", "hostA", 1000),
@@ -136,21 +137,21 @@ class MapOutputTrackerSuite extends SparkFunSuite {
     masterTracker.trackerEndpoint = rpcEnv.setupEndpoint(MapOutputTracker.ENDPOINT_NAME,
       new MapOutputTrackerMasterEndpoint(rpcEnv, masterTracker, conf))
 
-    val slaveRpcEnv = createRpcEnv("spark-slave", hostname, 0, new SecurityManager(conf))
-    val slaveTracker = new MapOutputTrackerWorker(conf)
-    slaveTracker.trackerEndpoint =
-      slaveRpcEnv.setupEndpointRef(rpcEnv.address, MapOutputTracker.ENDPOINT_NAME)
+    val mapWorkerRpcEnv = createRpcEnv("spark-worker", hostname, 0, new SecurityManager(conf))
+    val mapWorkerTracker = new MapOutputTrackerWorker(conf)
+    mapWorkerTracker.trackerEndpoint =
+      mapWorkerRpcEnv.setupEndpointRef(rpcEnv.address, MapOutputTracker.ENDPOINT_NAME)
 
-    masterTracker.registerShuffle(10, 1)
-    slaveTracker.updateEpoch(masterTracker.getEpoch)
+    masterTracker.registerShuffle(10, 1, MergeStatus.SHUFFLE_PUSH_DUMMY_NUM_REDUCES)
+    mapWorkerTracker.updateEpoch(masterTracker.getEpoch)
     // This is expected to fail because no outputs have been registered for the shuffle.
-    intercept[FetchFailedException] { slaveTracker.getMapSizesByExecutorId(10, 0) }
+    intercept[FetchFailedException] { mapWorkerTracker.getMapSizesByExecutorId(10, 0) }
 
     val size1000 = MapStatus.decompressSize(MapStatus.compressSize(1000L))
     masterTracker.registerMapOutput(10, 0, MapStatus(
       BlockManagerId("a", "hostA", 1000), Array(1000L), 5))
-    slaveTracker.updateEpoch(masterTracker.getEpoch)
-    assert(slaveTracker.getMapSizesByExecutorId(10, 0).toSeq ===
+    mapWorkerTracker.updateEpoch(masterTracker.getEpoch)
+    assert(mapWorkerTracker.getMapSizesByExecutorId(10, 0).toSeq ===
       Seq((BlockManagerId("a", "hostA", 1000),
         ArrayBuffer((ShuffleBlockId(10, 5, 0), size1000, 0)))))
     assert(0 == masterTracker.getNumCachedSerializedBroadcast)
@@ -158,17 +159,17 @@ class MapOutputTrackerSuite extends SparkFunSuite {
     val masterTrackerEpochBeforeLossOfMapOutput = masterTracker.getEpoch
     masterTracker.unregisterMapOutput(10, 0, BlockManagerId("a", "hostA", 1000))
     assert(masterTracker.getEpoch > masterTrackerEpochBeforeLossOfMapOutput)
-    slaveTracker.updateEpoch(masterTracker.getEpoch)
-    intercept[FetchFailedException] { slaveTracker.getMapSizesByExecutorId(10, 0) }
+    mapWorkerTracker.updateEpoch(masterTracker.getEpoch)
+    intercept[FetchFailedException] { mapWorkerTracker.getMapSizesByExecutorId(10, 0) }
 
     // failure should be cached
-    intercept[FetchFailedException] { slaveTracker.getMapSizesByExecutorId(10, 0) }
+    intercept[FetchFailedException] { mapWorkerTracker.getMapSizesByExecutorId(10, 0) }
     assert(0 == masterTracker.getNumCachedSerializedBroadcast)
 
     masterTracker.stop()
-    slaveTracker.stop()
+    mapWorkerTracker.stop()
     rpcEnv.shutdown()
-    slaveRpcEnv.shutdown()
+    mapWorkerRpcEnv.shutdown()
   }
 
   test("remote fetch below max RPC message size") {
@@ -184,7 +185,7 @@ class MapOutputTrackerSuite extends SparkFunSuite {
       rpcEnv.setupEndpoint(MapOutputTracker.ENDPOINT_NAME, masterEndpoint)
 
     // Message size should be ~123B, and no exception should be thrown
-    masterTracker.registerShuffle(10, 1)
+    masterTracker.registerShuffle(10, 1, MergeStatus.SHUFFLE_PUSH_DUMMY_NUM_REDUCES)
     masterTracker.registerMapOutput(10, 0, MapStatus(
       BlockManagerId("88", "mph", 1000), Array.fill[Long](10)(0), 5))
     val senderAddress = RpcAddress("localhost", 12345)
@@ -218,7 +219,7 @@ class MapOutputTrackerSuite extends SparkFunSuite {
     // on hostA with output size 2
     // on hostA with output size 2
     // on hostB with output size 3
-    tracker.registerShuffle(10, 3)
+    tracker.registerShuffle(10, 3, MergeStatus.SHUFFLE_PUSH_DUMMY_NUM_REDUCES)
     tracker.registerMapOutput(10, 0, MapStatus(BlockManagerId("a", "hostA", 1000),
         Array(2L), 5))
     tracker.registerMapOutput(10, 1, MapStatus(BlockManagerId("a", "hostA", 1000),
@@ -261,7 +262,7 @@ class MapOutputTrackerSuite extends SparkFunSuite {
       // Frame size should be ~1.1MB, and MapOutputTrackerMasterEndpoint should throw exception.
       // Note that the size is hand-selected here because map output statuses are compressed before
       // being sent.
-      masterTracker.registerShuffle(20, 100)
+      masterTracker.registerShuffle(20, 100, MergeStatus.SHUFFLE_PUSH_DUMMY_NUM_REDUCES)
       (0 until 100).foreach { i =>
         masterTracker.registerMapOutput(20, i, new CompressedMapStatus(
           BlockManagerId("999", "mps", 1000), Array.fill[Long](4000000)(0), 5))
@@ -307,7 +308,7 @@ class MapOutputTrackerSuite extends SparkFunSuite {
     val tracker = newTrackerMaster()
     tracker.trackerEndpoint = rpcEnv.setupEndpoint(MapOutputTracker.ENDPOINT_NAME,
       new MapOutputTrackerMasterEndpoint(rpcEnv, tracker, conf))
-    tracker.registerShuffle(10, 2)
+    tracker.registerShuffle(10, 2, MergeStatus.SHUFFLE_PUSH_DUMMY_NUM_REDUCES)
 
     val size0 = MapStatus.decompressSize(MapStatus.compressSize(0L))
     val size1000 = MapStatus.decompressSize(MapStatus.compressSize(1000L))
@@ -317,7 +318,7 @@ class MapOutputTrackerSuite extends SparkFunSuite {
     tracker.registerMapOutput(10, 1, MapStatus(BlockManagerId("b", "hostB", 1000),
       Array(size10000, size0, size1000, size0), 6))
     assert(tracker.containsShuffle(10))
-    assert(tracker.getMapSizesByExecutorId(10, 0, 4).toSeq ===
+    assert(tracker.getMapSizesByExecutorId(10, 0, 2, 0, 4).toSeq ===
         Seq(
           (BlockManagerId("a", "hostA", 1000),
               Seq((ShuffleBlockId(10, 5, 1), size1000, 0),
@@ -333,4 +334,334 @@ class MapOutputTrackerSuite extends SparkFunSuite {
     rpcEnv.shutdown()
   }
 
+  test("SPARK-32921: master register and unregister merge result") {
+    conf.set(PUSH_BASED_SHUFFLE_ENABLED, true)
+    conf.set(IS_TESTING, true)
+    val rpcEnv = createRpcEnv("test")
+    val tracker = newTrackerMaster()
+    tracker.trackerEndpoint = rpcEnv.setupEndpoint(MapOutputTracker.ENDPOINT_NAME,
+      new MapOutputTrackerMasterEndpoint(rpcEnv, tracker, conf))
+    tracker.registerShuffle(10, 4, 2)
+    assert(tracker.containsShuffle(10))
+    val bitmap = new RoaringBitmap()
+    bitmap.add(0)
+    bitmap.add(1)
+
+    tracker.registerMergeResult(10, 0, MergeStatus(BlockManagerId("a", "hostA", 1000),
+      bitmap, 1000L))
+    tracker.registerMergeResult(10, 1, MergeStatus(BlockManagerId("b", "hostB", 1000),
+      bitmap, 1000L))
+    assert(tracker.getNumAvailableMergeResults(10) == 2)
+    tracker.unregisterMergeResult(10, 0, BlockManagerId("a", "hostA", 1000))
+    assert(tracker.getNumAvailableMergeResults(10) == 1)
+    tracker.stop()
+    rpcEnv.shutdown()
+  }
+
+  test("SPARK-32921: get map sizes with merged shuffle") {
+    conf.set(PUSH_BASED_SHUFFLE_ENABLED, true)
+    conf.set(IS_TESTING, true)
+    val hostname = "localhost"
+    val rpcEnv = createRpcEnv("spark", hostname, 0, new SecurityManager(conf))
+
+    val masterTracker = newTrackerMaster()
+    masterTracker.trackerEndpoint = rpcEnv.setupEndpoint(MapOutputTracker.ENDPOINT_NAME,
+      new MapOutputTrackerMasterEndpoint(rpcEnv, masterTracker, conf))
+
+    val slaveRpcEnv = createRpcEnv("spark-slave", hostname, 0, new SecurityManager(conf))
+    val slaveTracker = new MapOutputTrackerWorker(conf)
+    slaveTracker.trackerEndpoint =
+      slaveRpcEnv.setupEndpointRef(rpcEnv.address, MapOutputTracker.ENDPOINT_NAME)
+
+    masterTracker.registerShuffle(10, 4, 1)
+    slaveTracker.updateEpoch(masterTracker.getEpoch)
+    val bitmap = new RoaringBitmap()
+    bitmap.add(0)
+    bitmap.add(1)
+    bitmap.add(3)
+
+    val blockMgrId = BlockManagerId("a", "hostA", 1000)
+    masterTracker.registerMapOutput(10, 0, MapStatus(blockMgrId, Array(1000L), 0))
+    masterTracker.registerMapOutput(10, 1, MapStatus(blockMgrId, Array(1000L), 1))
+    masterTracker.registerMapOutput(10, 2, MapStatus(blockMgrId, Array(1000L), 2))
+    masterTracker.registerMapOutput(10, 3, MapStatus(blockMgrId, Array(1000L), 3))
+
+    masterTracker.registerMergeResult(10, 0, MergeStatus(blockMgrId,
+      bitmap, 3000L))
+    slaveTracker.updateEpoch(masterTracker.getEpoch)
+    val size1000 = MapStatus.decompressSize(MapStatus.compressSize(1000L))
+    assert(slaveTracker.getMapSizesByExecutorId(10, 0).toSeq ===
+      Seq((blockMgrId, ArrayBuffer((ShuffleBlockId(10, -1, 0), 3000, -1),
+        (ShuffleBlockId(10, 2, 0), size1000, 2)))))
+
+    masterTracker.stop()
+    slaveTracker.stop()
+    rpcEnv.shutdown()
+    slaveRpcEnv.shutdown()
+  }
+
+  test("SPARK-32921: get map statuses from merged shuffle") {
+    conf.set(PUSH_BASED_SHUFFLE_ENABLED, true)
+    conf.set(IS_TESTING, true)
+    val hostname = "localhost"
+    val rpcEnv = createRpcEnv("spark", hostname, 0, new SecurityManager(conf))
+
+    val masterTracker = newTrackerMaster()
+    masterTracker.trackerEndpoint = rpcEnv.setupEndpoint(MapOutputTracker.ENDPOINT_NAME,
+      new MapOutputTrackerMasterEndpoint(rpcEnv, masterTracker, conf))
+
+    val slaveRpcEnv = createRpcEnv("spark-slave", hostname, 0, new SecurityManager(conf))
+    val slaveTracker = new MapOutputTrackerWorker(conf)
+    slaveTracker.trackerEndpoint =
+      slaveRpcEnv.setupEndpointRef(rpcEnv.address, MapOutputTracker.ENDPOINT_NAME)
+
+    masterTracker.registerShuffle(10, 4, 1)
+    slaveTracker.updateEpoch(masterTracker.getEpoch)
+    // This is expected to fail because no outputs have been registered for the shuffle.
+    intercept[FetchFailedException] { slaveTracker.getMapSizesByExecutorId(10, 0) }
+    val bitmap = new RoaringBitmap()
+    bitmap.add(0)
+    bitmap.add(1)
+    bitmap.add(2)
+    bitmap.add(3)
+
+    val blockMgrId = BlockManagerId("a", "hostA", 1000)
+    masterTracker.registerMapOutput(10, 0, MapStatus(blockMgrId, Array(1000L), 0))
+    masterTracker.registerMapOutput(10, 1, MapStatus(blockMgrId, Array(1000L), 1))
+    masterTracker.registerMapOutput(10, 2, MapStatus(blockMgrId, Array(1000L), 2))
+    masterTracker.registerMapOutput(10, 3, MapStatus(blockMgrId, Array(1000L), 3))
+
+    masterTracker.registerMergeResult(10, 0, MergeStatus(blockMgrId,
+      bitmap, 4000L))
+    slaveTracker.updateEpoch(masterTracker.getEpoch)
+    val size1000 = MapStatus.decompressSize(MapStatus.compressSize(1000L))
+    assert(slaveTracker.getMapSizesForMergeResult(10, 0).toSeq ===
+      Seq((blockMgrId, ArrayBuffer((ShuffleBlockId(10, 0, 0), size1000, 0),
+        (ShuffleBlockId(10, 1, 0), size1000, 1), (ShuffleBlockId(10, 2, 0), size1000, 2),
+        (ShuffleBlockId(10, 3, 0), size1000, 3)))))
+    masterTracker.stop()
+    slaveTracker.stop()
+    rpcEnv.shutdown()
+    slaveRpcEnv.shutdown()
+  }
+
+  test("SPARK-32921: get map statuses for merged shuffle block chunks") {
+    conf.set(PUSH_BASED_SHUFFLE_ENABLED, true)
+    conf.set(IS_TESTING, true)
+    val hostname = "localhost"
+    val rpcEnv = createRpcEnv("spark", hostname, 0, new SecurityManager(conf))
+
+    val masterTracker = newTrackerMaster()
+    masterTracker.trackerEndpoint = rpcEnv.setupEndpoint(MapOutputTracker.ENDPOINT_NAME,
+      new MapOutputTrackerMasterEndpoint(rpcEnv, masterTracker, conf))
+
+    val slaveRpcEnv = createRpcEnv("spark-slave", hostname, 0, new SecurityManager(conf))
+    val slaveTracker = new MapOutputTrackerWorker(conf)
+    slaveTracker.trackerEndpoint =
+      slaveRpcEnv.setupEndpointRef(rpcEnv.address, MapOutputTracker.ENDPOINT_NAME)
+
+    masterTracker.registerShuffle(10, 4, 1)
+    slaveTracker.updateEpoch(masterTracker.getEpoch)
+
+    val blockMgrId = BlockManagerId("a", "hostA", 1000)
+    masterTracker.registerMapOutput(10, 0, MapStatus(blockMgrId, Array(1000L), 0))
+    masterTracker.registerMapOutput(10, 1, MapStatus(blockMgrId, Array(1000L), 1))
+    masterTracker.registerMapOutput(10, 2, MapStatus(blockMgrId, Array(1000L), 2))
+    masterTracker.registerMapOutput(10, 3, MapStatus(blockMgrId, Array(1000L), 3))
+
+    val chunkBitmap = new RoaringBitmap()
+    chunkBitmap.add(0)
+    chunkBitmap.add(2)
+    val size1000 = MapStatus.decompressSize(MapStatus.compressSize(1000L))
+    assert(slaveTracker.getMapSizesForMergeResult(10, 0, chunkBitmap).toSeq ===
+      Seq((blockMgrId, ArrayBuffer((ShuffleBlockId(10, 0, 0), size1000, 0),
+        (ShuffleBlockId(10, 2, 0), size1000, 2))))
+    )
+    masterTracker.stop()
+    slaveTracker.stop()
+    rpcEnv.shutdown()
+    slaveRpcEnv.shutdown()
+  }
+
+  test("SPARK-32921: getPreferredLocationsForShuffle with MergeStatus") {
+    val rpcEnv = createRpcEnv("test")
+    val tracker = newTrackerMaster()
+    sc = new SparkContext("local", "test", conf.clone())
+    tracker.trackerEndpoint = rpcEnv.setupEndpoint(MapOutputTracker.ENDPOINT_NAME,
+      new MapOutputTrackerMasterEndpoint(rpcEnv, tracker, conf))
+    // Setup 5 map tasks
+    // on hostA with output size 2
+    // on hostA with output size 2
+    // on hostB with output size 3
+    // on hostB with output size 3
+    // on hostC with output size 1
+    // on hostC with output size 1
+    tracker.registerShuffle(10, 6, 1)
+    tracker.registerMapOutput(10, 0, MapStatus(BlockManagerId("a", "hostA", 1000),
+      Array(2L), 5))
+    tracker.registerMapOutput(10, 1, MapStatus(BlockManagerId("a", "hostA", 1000),
+      Array(2L), 6))
+    tracker.registerMapOutput(10, 2, MapStatus(BlockManagerId("b", "hostB", 1000),
+      Array(3L), 7))
+    tracker.registerMapOutput(10, 3, MapStatus(BlockManagerId("b", "hostB", 1000),
+      Array(3L), 8))
+    tracker.registerMapOutput(10, 4, MapStatus(BlockManagerId("c", "hostC", 1000),
+      Array(1L), 9))
+    tracker.registerMapOutput(10, 5, MapStatus(BlockManagerId("c", "hostC", 1000),
+      Array(1L), 10))
+
+    val rdd = sc.parallelize(1 to 6, 6).map(num => (num, num).asInstanceOf[Product2[Int, Int]])
+    val mockShuffleDep = mock(classOf[ShuffleDependency[Int, Int, _]])
+    when(mockShuffleDep.shuffleId).thenReturn(10)
+    when(mockShuffleDep.partitioner).thenReturn(new HashPartitioner(1))
+    when(mockShuffleDep.rdd).thenReturn(rdd)
+
+    // Prepare a MergeStatus that merges 4 out of 5 blocks
+    val bitmap80 = new RoaringBitmap()
+    bitmap80.add(0)
+    bitmap80.add(1)
+    bitmap80.add(2)
+    bitmap80.add(3)
+    bitmap80.add(4)
+    tracker.registerMergeResult(10, 0, MergeStatus(BlockManagerId("a", "hostA", 1000),
+      bitmap80, 11))
+
+    val preferredLocs1 = tracker.getPreferredLocationsForShuffle(mockShuffleDep, 0)
+    assert(preferredLocs1.nonEmpty)
+    assert(preferredLocs1.length === 1)
+    assert(preferredLocs1.head === "hostA")
+
+    tracker.unregisterMergeResult(10, 0, BlockManagerId("a", "hostA", 1000))
+    // Prepare another MergeStatus that merges only 1 out of 5 blocks
+    val bitmap20 = new RoaringBitmap()
+    bitmap20.add(0)
+    tracker.registerMergeResult(10, 0, MergeStatus(BlockManagerId("a", "hostA", 1000),
+      bitmap20, 2))
+
+    val preferredLocs2 = tracker.getPreferredLocationsForShuffle(mockShuffleDep, 0)
+    assert(preferredLocs2.nonEmpty)
+    assert(preferredLocs2.length === 2)
+    assert(preferredLocs2 === Seq("hostA", "hostB"))
+
+    tracker.stop()
+    rpcEnv.shutdown()
+  }
+
+  test("SPARK-34939: remote fetch using broadcast if broadcasted value is destroyed") {
+    val newConf = new SparkConf
+    newConf.set(RPC_MESSAGE_MAX_SIZE, 1)
+    newConf.set(RPC_ASK_TIMEOUT, "1") // Fail fast
+    newConf.set(SHUFFLE_MAPOUTPUT_MIN_SIZE_FOR_BROADCAST, 10240L) // 10 KiB << 1MiB framesize
+
+    // needs TorrentBroadcast so need a SparkContext
+    withSpark(new SparkContext("local", "MapOutputTrackerSuite", newConf)) { sc =>
+      val masterTracker = sc.env.mapOutputTracker.asInstanceOf[MapOutputTrackerMaster]
+      val rpcEnv = sc.env.rpcEnv
+      val masterEndpoint = new MapOutputTrackerMasterEndpoint(rpcEnv, masterTracker, newConf)
+      rpcEnv.stop(masterTracker.trackerEndpoint)
+      rpcEnv.setupEndpoint(MapOutputTracker.ENDPOINT_NAME, masterEndpoint)
+
+      masterTracker.registerShuffle(20, 100, MergeStatus.SHUFFLE_PUSH_DUMMY_NUM_REDUCES)
+      (0 until 100).foreach { i =>
+        masterTracker.registerMapOutput(20, i, new CompressedMapStatus(
+          BlockManagerId("999", "mps", 1000), Array.fill[Long](4000000)(0), 5))
+      }
+
+      val mapWorkerRpcEnv = createRpcEnv("spark-worker", "localhost", 0, new SecurityManager(conf))
+      val mapWorkerTracker = new MapOutputTrackerWorker(conf)
+      mapWorkerTracker.trackerEndpoint =
+        mapWorkerRpcEnv.setupEndpointRef(rpcEnv.address, MapOutputTracker.ENDPOINT_NAME)
+
+      val fetchedBytes = mapWorkerTracker.trackerEndpoint
+        .askSync[Array[Byte]](GetMapOutputStatuses(20))
+      assert(fetchedBytes(0) == 1)
+
+      // Normally `unregisterMapOutput` triggers the destroy of broadcasted value.
+      // But the timing of destroying broadcasted value is indeterminate, we manually destroy
+      // it by blocking.
+      masterTracker.shuffleStatuses.get(20).foreach { shuffleStatus =>
+        shuffleStatus.cachedSerializedBroadcast.destroy(true)
+      }
+      val err = intercept[SparkException] {
+        MapOutputTracker.deserializeOutputStatuses[MapStatus](fetchedBytes, conf)
+      }
+      assert(err.getMessage.contains("Unable to deserialize broadcasted output statuses"))
+    }
+  }
+
+  test("SPARK-32921: test new protocol changes fetching both Map and Merge status in single RPC") {
+    val newConf = new SparkConf
+    newConf.set(RPC_MESSAGE_MAX_SIZE, 1)
+    newConf.set(RPC_ASK_TIMEOUT, "1") // Fail fast
+    newConf.set(SHUFFLE_MAPOUTPUT_MIN_SIZE_FOR_BROADCAST, 10240L) // 10 KiB << 1MiB framesize
+    newConf.set(PUSH_BASED_SHUFFLE_ENABLED, true)
+    newConf.set(IS_TESTING, true)
+
+    // needs TorrentBroadcast so need a SparkContext
+    withSpark(new SparkContext("local", "MapOutputTrackerSuite", newConf)) { sc =>
+      val masterTracker = sc.env.mapOutputTracker.asInstanceOf[MapOutputTrackerMaster]
+      val rpcEnv = sc.env.rpcEnv
+      val masterEndpoint = new MapOutputTrackerMasterEndpoint(rpcEnv, masterTracker, newConf)
+      rpcEnv.stop(masterTracker.trackerEndpoint)
+      rpcEnv.setupEndpoint(MapOutputTracker.ENDPOINT_NAME, masterEndpoint)
+      val bitmap1 = new RoaringBitmap()
+      bitmap1.add(1)
+
+      masterTracker.registerShuffle(20, 100, MergeStatus.SHUFFLE_PUSH_DUMMY_NUM_REDUCES)
+      (0 until 100).foreach { i =>
+        masterTracker.registerMapOutput(20, i, new CompressedMapStatus(
+          BlockManagerId("999", "mps", 1000), Array.fill[Long](4000000)(0), 5))
+      }
+      masterTracker.registerMergeResult(20, 0, MergeStatus(BlockManagerId("999", "mps", 1000),
+        bitmap1, 1000L))
+
+      val mapWorkerRpcEnv = createRpcEnv("spark-worker", "localhost", 0, new SecurityManager(conf))
+      val mapWorkerTracker = new MapOutputTrackerWorker(conf)
+      mapWorkerTracker.trackerEndpoint =
+        mapWorkerRpcEnv.setupEndpointRef(rpcEnv.address, MapOutputTracker.ENDPOINT_NAME)
+
+      val fetchedBytes = mapWorkerTracker.trackerEndpoint
+        .askSync[(Array[Byte], Array[Byte])](GetMapAndMergeResultStatuses(20))
+      assert(masterTracker.getNumAvailableMergeResults(20) == 1)
+      assert(masterTracker.getNumAvailableOutputs(20) == 100)
+
+      val mapOutput =
+        MapOutputTracker.deserializeOutputStatuses[MapStatus](fetchedBytes._1, newConf)
+      val mergeOutput =
+        MapOutputTracker.deserializeOutputStatuses[MergeStatus](fetchedBytes._2, newConf)
+      assert(mapOutput.length == 100)
+      assert(mergeOutput.length == 1)
+      mapWorkerTracker.stop()
+      masterTracker.stop()
+    }
+  }
+
+  test("SPARK-32921: unregister merge result if it is present and contains the map Id") {
+    val rpcEnv = createRpcEnv("test")
+    val tracker = newTrackerMaster()
+    tracker.trackerEndpoint = rpcEnv.setupEndpoint(MapOutputTracker.ENDPOINT_NAME,
+      new MapOutputTrackerMasterEndpoint(rpcEnv, tracker, conf))
+    tracker.registerShuffle(10, 4, 2)
+    assert(tracker.containsShuffle(10))
+    val bitmap1 = new RoaringBitmap()
+    bitmap1.add(0)
+    bitmap1.add(1)
+    tracker.registerMergeResult(10, 0, MergeStatus(BlockManagerId("a", "hostA", 1000),
+      bitmap1, 1000L))
+
+    val bitmap2 = new RoaringBitmap()
+    bitmap2.add(5)
+    bitmap2.add(6)
+    tracker.registerMergeResult(10, 1, MergeStatus(BlockManagerId("b", "hostB", 1000),
+      bitmap2, 1000L))
+    assert(tracker.getNumAvailableMergeResults(10) == 2)
+    tracker.unregisterMergeResult(10, 0, BlockManagerId("a", "hostA", 1000), Option(0))
+    assert(tracker.getNumAvailableMergeResults(10) == 1)
+    tracker.unregisterMergeResult(10, 1, BlockManagerId("b", "hostB", 1000), Option(1))
+    assert(tracker.getNumAvailableMergeResults(10) == 1)
+    tracker.unregisterMergeResult(10, 1, BlockManagerId("b", "hostB", 1000), Option(5))
+    assert(tracker.getNumAvailableMergeResults(10) == 0)
+    tracker.stop()
+    rpcEnv.shutdown()
+  }
 }
diff --git a/core/src/test/scala/org/apache/spark/MapStatusesSerDeserBenchmark.scala b/core/src/test/scala/org/apache/spark/MapStatusesSerDeserBenchmark.scala
index 78f1246295bf8..d8088239870ba 100644
--- a/core/src/test/scala/org/apache/spark/MapStatusesSerDeserBenchmark.scala
+++ b/core/src/test/scala/org/apache/spark/MapStatusesSerDeserBenchmark.scala
@@ -17,18 +17,16 @@
 
 package org.apache.spark
 
-import org.scalatest.Assertions._
-
 import org.apache.spark.benchmark.Benchmark
 import org.apache.spark.benchmark.BenchmarkBase
-import org.apache.spark.scheduler.CompressedMapStatus
+import org.apache.spark.scheduler.{CompressedMapStatus, MergeStatus}
 import org.apache.spark.storage.BlockManagerId
 
 /**
  * Benchmark for MapStatuses serialization & deserialization performance.
  * {{{
  *   To run this benchmark:
- *   1. without sbt: bin/spark-submit --class <this class> --jars <core test jar>
+ *   1. without sbt: bin/spark-submit --class <this class> <spark core test jar>
  *   2. build/sbt "core/test:runMain <this class>"
  *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "core/test:runMain <this class>"
  *      Results will be written to "benchmarks/MapStatusesSerDeserBenchmark-results.txt".
@@ -52,7 +50,7 @@ object MapStatusesSerDeserBenchmark extends BenchmarkBase {
 
     val shuffleId = 10
 
-    tracker.registerShuffle(shuffleId, numMaps)
+    tracker.registerShuffle(shuffleId, numMaps, MergeStatus.SHUFFLE_PUSH_DUMMY_NUM_REDUCES)
     val r = new scala.util.Random(912)
     (0 until numMaps).foreach { i =>
       tracker.registerMapOutput(shuffleId, i,
@@ -68,7 +66,7 @@ object MapStatusesSerDeserBenchmark extends BenchmarkBase {
     var serializedMapStatusSizes = 0
     var serializedBroadcastSizes = 0
 
-    val (serializedMapStatus, serializedBroadcast) = MapOutputTracker.serializeMapStatuses(
+    val (serializedMapStatus, serializedBroadcast) = MapOutputTracker.serializeOutputStatuses(
       shuffleStatus.mapStatuses, tracker.broadcastManager, tracker.isLocal, minBroadcastSize,
       sc.getConf)
     serializedMapStatusSizes = serializedMapStatus.length
@@ -77,12 +75,12 @@ object MapStatusesSerDeserBenchmark extends BenchmarkBase {
     }
 
     benchmark.addCase("Serialization") { _ =>
-      MapOutputTracker.serializeMapStatuses(shuffleStatus.mapStatuses, tracker.broadcastManager,
+      MapOutputTracker.serializeOutputStatuses(shuffleStatus.mapStatuses, tracker.broadcastManager,
         tracker.isLocal, minBroadcastSize, sc.getConf)
     }
 
     benchmark.addCase("Deserialization") { _ =>
-      val result = MapOutputTracker.deserializeMapStatuses(serializedMapStatus, sc.getConf)
+      val result = MapOutputTracker.deserializeOutputStatuses(serializedMapStatus, sc.getConf)
       assert(result.length == numMaps)
     }
 
diff --git a/core/src/test/scala/org/apache/spark/SharedSparkContext.scala b/core/src/test/scala/org/apache/spark/SharedSparkContext.scala
index bdeb631878350..7106a780b3256 100644
--- a/core/src/test/scala/org/apache/spark/SharedSparkContext.scala
+++ b/core/src/test/scala/org/apache/spark/SharedSparkContext.scala
@@ -27,7 +27,7 @@ trait SharedSparkContext extends BeforeAndAfterAll with BeforeAndAfterEach { sel
 
   def sc: SparkContext = _sc
 
-  var conf = new SparkConf(false)
+  val conf = new SparkConf(false)
 
   /**
    * Initialize the [[SparkContext]].  Generally, this is just called from beforeAll; however, in
diff --git a/core/src/test/scala/org/apache/spark/ShuffleSuite.scala b/core/src/test/scala/org/apache/spark/ShuffleSuite.scala
index 9e39271bdf9ee..126faec334e77 100644
--- a/core/src/test/scala/org/apache/spark/ShuffleSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ShuffleSuite.scala
@@ -17,18 +17,24 @@
 
 package org.apache.spark
 
+import java.io.File
 import java.util.{Locale, Properties}
-import java.util.concurrent.{Callable, CyclicBarrier, Executors, ExecutorService}
+import java.util.concurrent.{Callable, CyclicBarrier, Executors, ExecutorService }
 
-import org.scalatest.Matchers
+import scala.collection.JavaConverters._
+
+import org.apache.commons.io.FileUtils
+import org.apache.commons.io.filefilter.TrueFileFilter
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.ShuffleSuite.NonJavaSerializableClass
 import org.apache.spark.internal.config
 import org.apache.spark.internal.config.Tests.TEST_NO_STAGE_RETRY
 import org.apache.spark.memory.TaskMemoryManager
 import org.apache.spark.rdd.{CoGroupedRDD, OrderedRDDFunctions, RDD, ShuffledRDD, SubtractedRDD}
-import org.apache.spark.scheduler.{MapStatus, MyRDD, SparkListener, SparkListenerTaskEnd}
-import org.apache.spark.serializer.KryoSerializer
+import org.apache.spark.scheduler.{MapStatus, MergeStatus, MyRDD, SparkListener, SparkListenerTaskEnd}
+import org.apache.spark.serializer.{JavaSerializer, KryoSerializer}
 import org.apache.spark.shuffle.ShuffleWriter
 import org.apache.spark.storage.{ShuffleBlockId, ShuffleDataBlockId, ShuffleIndexBlockId}
 import org.apache.spark.util.MutablePair
@@ -182,7 +188,7 @@ abstract class ShuffleSuite extends SparkFunSuite with Matchers with LocalSparkC
     val pairs1: RDD[MutablePair[Int, Int]] = sc.parallelize(data1, 2)
     val pairs2: RDD[MutablePair[Int, String]] = sc.parallelize(data2, 2)
     val results = new CoGroupedRDD[Int](Seq(pairs1, pairs2), new HashPartitioner(2))
-      .map(p => (p._1, p._2.map(_.toArray)))
+      .map(p => (p._1, p._2.map(_.toSeq)))
       .collectAsMap()
 
     assert(results(1)(0).length === 3)
@@ -361,7 +367,7 @@ abstract class ShuffleSuite extends SparkFunSuite with Matchers with LocalSparkC
     val shuffleMapRdd = new MyRDD(sc, 1, Nil)
     val shuffleDep = new ShuffleDependency(shuffleMapRdd, new HashPartitioner(1))
     val shuffleHandle = manager.registerShuffle(0, shuffleDep)
-    mapTrackerMaster.registerShuffle(0, 1)
+    mapTrackerMaster.registerShuffle(0, 1, MergeStatus.SHUFFLE_PUSH_DUMMY_NUM_REDUCES)
 
     // first attempt -- its successful
     val context1 =
@@ -418,6 +424,31 @@ abstract class ShuffleSuite extends SparkFunSuite with Matchers with LocalSparkC
 
     manager.unregisterShuffle(0)
   }
+
+  test("SPARK-34541: shuffle can be removed") {
+    withTempDir { tmpDir =>
+      def getAllFiles: Set[File] =
+        FileUtils.listFiles(tmpDir, TrueFileFilter.INSTANCE, TrueFileFilter.INSTANCE).asScala.toSet
+      conf.set("spark.local.dir", tmpDir.getAbsolutePath)
+      sc = new SparkContext("local", "test", conf)
+      // For making the taskAttemptId starts from 1.
+      sc.parallelize(1 to 10).count()
+      val rdd = sc.parallelize(1 to 10, 1).map(x => (x, x))
+      // Create a shuffleRdd
+      val shuffledRdd = new ShuffledRDD[Int, Int, Int](rdd, new HashPartitioner(4))
+        .setSerializer(new JavaSerializer(conf))
+      val filesBeforeShuffle = getAllFiles
+      // Force the shuffle to be performed
+      shuffledRdd.count()
+      // Ensure that the shuffle actually created files that will need to be cleaned up
+      val filesCreatedByShuffle = getAllFiles -- filesBeforeShuffle
+      // Check that the cleanup actually removes the files
+      sc.env.blockManager.master.removeShuffle(0, blocking = true)
+      for (file <- filesCreatedByShuffle) {
+        assert (!file.exists(), s"Shuffle file $file was not cleaned up")
+      }
+    }
+  }
 }
 
 /**
diff --git a/core/src/test/scala/org/apache/spark/SortShuffleSuite.scala b/core/src/test/scala/org/apache/spark/SortShuffleSuite.scala
index 1a563621a5179..3d853ff4294be 100644
--- a/core/src/test/scala/org/apache/spark/SortShuffleSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SortShuffleSuite.scala
@@ -24,6 +24,7 @@ import scala.collection.JavaConverters._
 import org.apache.commons.io.FileUtils
 import org.apache.commons.io.filefilter.TrueFileFilter
 import org.scalatest.BeforeAndAfterAll
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.internal.config
 import org.apache.spark.rdd.ShuffledRDD
diff --git a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
index 3bc2061c4f2ad..7779fb2aeaf07 100644
--- a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
@@ -221,7 +221,7 @@ class SparkConfSuite extends SparkFunSuite with LocalSparkContext with ResetSyst
     conf.registerKryoClasses(Array(classOf[Class1]))
     assert(conf.get(KRYO_CLASSES_TO_REGISTER).toSet === Seq(classOf[Class1].getName).toSet)
 
-    conf.set(KRYO_USER_REGISTRATORS, classOf[CustomRegistrator].getName)
+    conf.set(KRYO_USER_REGISTRATORS, Seq(classOf[CustomRegistrator].getName))
 
     // Kryo doesn't expose a way to discover registered classes, but at least make sure this doesn't
     // blow up.
@@ -461,7 +461,7 @@ class SparkConfSuite extends SparkFunSuite with LocalSparkContext with ResetSyst
     val conf = new SparkConf()
     conf.set(TASK_GPU_ID.amountConf, "2")
     conf.set(TASK_FPGA_ID.amountConf, "0")
-    var taskResourceRequirement =
+    val taskResourceRequirement =
       parseResourceRequirements(conf, SPARK_TASK_PREFIX)
         .map(req => (req.resourceName, req.amount)).toMap
 
diff --git a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
index 30237fd576830..b663d1a77ce58 100644
--- a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
@@ -23,6 +23,7 @@ import java.nio.charset.StandardCharsets
 import java.util.concurrent.{CountDownLatch, Semaphore, TimeUnit}
 
 import scala.concurrent.duration._
+import scala.io.Source
 
 import com.google.common.io.Files
 import org.apache.hadoop.conf.Configuration
@@ -31,8 +32,9 @@ import org.apache.hadoop.io.{BytesWritable, LongWritable, Text}
 import org.apache.hadoop.mapred.TextInputFormat
 import org.apache.hadoop.mapreduce.lib.input.{TextInputFormat => NewTextInputFormat}
 import org.json4s.{DefaultFormats, Extraction}
-import org.scalatest.Matchers._
+import org.junit.Assert.{assertEquals, assertFalse}
 import org.scalatest.concurrent.Eventually
+import org.scalatest.matchers.must.Matchers._
 
 import org.apache.spark.TestUtils._
 import org.apache.spark.internal.config._
@@ -45,7 +47,6 @@ import org.apache.spark.scheduler.{SparkListener, SparkListenerExecutorMetricsUp
 import org.apache.spark.shuffle.FetchFailedException
 import org.apache.spark.util.{ThreadUtils, Utils}
 
-
 class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventually {
 
   test("Only one SparkContext may be active at a time") {
@@ -153,7 +154,86 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
           }
           x
         }).count()
-        assert(sc.listFiles().filter(_.contains("somesuffix1")).size == 1)
+        assert(sc.listFiles().count(_.contains("somesuffix1")) == 1)
+      } finally {
+        sc.stop()
+      }
+    }
+  }
+
+  test("SPARK-33530: basic case for addArchive and listArchives") {
+    withTempDir { dir =>
+      val file1 = File.createTempFile("someprefix1", "somesuffix1", dir)
+      val file2 = File.createTempFile("someprefix2", "somesuffix2", dir)
+      val file3 = File.createTempFile("someprefix3", "somesuffix3", dir)
+      val file4 = File.createTempFile("someprefix4", "somesuffix4", dir)
+
+      val jarFile = new File(dir, "test!@$jar.jar")
+      val zipFile = new File(dir, "test-zip.zip")
+      val relativePath1 =
+        s"${zipFile.getParent}/../${zipFile.getParentFile.getName}/${zipFile.getName}"
+      val relativePath2 =
+        s"${jarFile.getParent}/../${jarFile.getParentFile.getName}/${jarFile.getName}#zoo"
+
+      try {
+        Files.write("somewords1", file1, StandardCharsets.UTF_8)
+        Files.write("somewords22", file2, StandardCharsets.UTF_8)
+        Files.write("somewords333", file3, StandardCharsets.UTF_8)
+        Files.write("somewords4444", file4, StandardCharsets.UTF_8)
+        val length1 = file1.length()
+        val length2 = file2.length()
+        val length3 = file1.length()
+        val length4 = file2.length()
+
+        createJar(Seq(file1, file2), jarFile)
+        createJar(Seq(file3, file4), zipFile)
+
+        sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local"))
+        sc.addArchive(jarFile.getAbsolutePath)
+        sc.addArchive(relativePath1)
+        sc.addArchive(s"${jarFile.getAbsolutePath}#foo")
+        sc.addArchive(s"${zipFile.getAbsolutePath}#bar")
+        sc.addArchive(relativePath2)
+
+        sc.parallelize(Array(1), 1).map { x =>
+          val gotten1 = new File(SparkFiles.get(jarFile.getName))
+          val gotten2 = new File(SparkFiles.get(zipFile.getName))
+          val gotten3 = new File(SparkFiles.get("foo"))
+          val gotten4 = new File(SparkFiles.get("bar"))
+          val gotten5 = new File(SparkFiles.get("zoo"))
+
+          Seq(gotten1, gotten2, gotten3, gotten4, gotten5).foreach { gotten =>
+            if (!gotten.exists()) {
+              throw new SparkException(s"The archive doesn't exist: ${gotten.getAbsolutePath}")
+            }
+            if (!gotten.isDirectory) {
+              throw new SparkException(s"The archive was not unpacked: ${gotten.getAbsolutePath}")
+            }
+          }
+
+          // Jars
+          Seq(gotten1, gotten3, gotten5).foreach { gotten =>
+            val actualLength1 = new File(gotten, file1.getName).length()
+            val actualLength2 = new File(gotten, file2.getName).length()
+            if (actualLength1 != length1 || actualLength2 != length2) {
+              s"Unpacked files have different lengths $actualLength1 and $actualLength2. at " +
+                s"${gotten.getAbsolutePath}. They should be $length1 and $length2."
+            }
+          }
+
+          // Zip
+          Seq(gotten2, gotten4).foreach { gotten =>
+            val actualLength3 = new File(gotten, file1.getName).length()
+            val actualLength4 = new File(gotten, file2.getName).length()
+            if (actualLength3 != length3 || actualLength4 != length4) {
+              s"Unpacked files have different lengths $actualLength3 and $actualLength4. at " +
+                s"${gotten.getAbsolutePath}. They should be $length3 and $length4."
+            }
+          }
+          x
+        }.count()
+        assert(sc.listArchives().count(_.endsWith("test!@$jar.jar")) == 1)
+        assert(sc.listArchives().count(_.contains("test-zip.zip")) == 2)
       } finally {
         sc.stop()
       }
@@ -165,7 +245,7 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
     try {
       sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local"))
       sc.addJar(jarPath.toString)
-      assert(sc.listJars().filter(_.contains("TestUDTF.jar")).size == 1)
+      assert(sc.listJars().count(_.contains("TestUDTF.jar")) == 1)
     } finally {
       sc.stop()
     }
@@ -297,7 +377,7 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
       sc.addFile(file1.getAbsolutePath)
       def getAddedFileContents(): String = {
         sc.parallelize(Seq(0)).map { _ =>
-          scala.io.Source.fromFile(SparkFiles.get("file")).mkString
+          Utils.tryWithResource(Source.fromFile(SparkFiles.get("file")))(_.mkString)
         }.first()
       }
       assert(getAddedFileContents() === "old")
@@ -860,7 +940,7 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
       .setAppName("test-cluster")
     conf.set(TASK_GPU_ID.amountConf, "1")
 
-    var error = intercept[SparkException] {
+    val error = intercept[SparkException] {
       sc = new SparkContext(conf)
     }.getMessage()
 
@@ -875,7 +955,7 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
     conf.set(TASK_GPU_ID.amountConf, "2")
     conf.set(EXECUTOR_GPU_ID.amountConf, "1")
 
-    var error = intercept[SparkException] {
+    val error = intercept[SparkException] {
       sc = new SparkContext(conf)
     }.getMessage()
 
@@ -891,7 +971,7 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
     conf.set(TASK_GPU_ID.amountConf, "2")
     conf.set(EXECUTOR_GPU_ID.amountConf, "4")
 
-    var error = intercept[SparkException] {
+    val error = intercept[SparkException] {
       sc = new SparkContext(conf)
     }.getMessage()
 
@@ -934,6 +1014,277 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
       }
     }
   }
+
+  test("SPARK-32160: Disallow to create SparkContext in executors") {
+    sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+
+    val error = intercept[SparkException] {
+      sc.range(0, 1).foreach { _ =>
+        new SparkContext(new SparkConf().setAppName("test").setMaster("local"))
+      }
+    }.getMessage()
+
+    assert(error.contains("SparkContext should only be created and accessed on the driver."))
+  }
+
+  test("SPARK-32160: Allow to create SparkContext in executors if the config is set") {
+    sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+
+    sc.range(0, 1).foreach { _ =>
+      new SparkContext(new SparkConf().setAppName("test").setMaster("local")
+        .set(EXECUTOR_ALLOW_SPARK_CONTEXT, true)).stop()
+    }
+  }
+
+  test("SPARK-33084: Add jar support Ivy URI -- default transitive = true") {
+    sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+    sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0")
+    assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
+    assert(sc.listJars().exists(_.contains("commons-lang_commons-lang-2.6.jar")))
+  }
+
+  test("SPARK-33084: Add jar support Ivy URI -- invalid transitive use default false") {
+    sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+    sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=foo")
+    assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
+    assert(!sc.listJars().exists(_.contains("org.slf4j_slf4j-api-1.7.10.jar")))
+    assert(!sc.listJars().exists(_.contains("commons-lang_commons-lang-2.6.jar")))
+  }
+
+  test("SPARK-33084: Add jar support Ivy URI -- transitive=true will download dependency jars") {
+    val logAppender = new LogAppender("transitive=true will download dependency jars")
+    withLogAppender(logAppender) {
+      sc = new SparkContext(
+        new SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+      sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=true")
+      val dependencyJars = Array(
+        "org.apache.hive_hive-storage-api-2.7.0.jar",
+        "org.slf4j_slf4j-api-1.7.10.jar",
+        "commons-lang_commons-lang-2.6.jar")
+
+      dependencyJars.foreach(jar => assert(sc.listJars().exists(_.contains(jar))))
+
+      assert(logAppender.loggingEvents.count(_.getRenderedMessage.contains(
+        "Added dependency jars of Ivy URI " +
+          "ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=true")) == 1)
+
+      // test dependency jars exist
+      sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=true")
+      assert(logAppender.loggingEvents.count(_.getRenderedMessage.contains(
+        "The dependency jars of Ivy URI " +
+          "ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=true")) == 1)
+      val existMsg = logAppender.loggingEvents.filter(_.getRenderedMessage.contains(
+        "The dependency jars of Ivy URI " +
+          "ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=true"))
+        .head.getRenderedMessage
+      dependencyJars.foreach(jar => assert(existMsg.contains(jar)))
+    }
+  }
+
+  test("SPARK-34506: Add jar support Ivy URI -- transitive=false will not download " +
+    "dependency jars") {
+    sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+    sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=false")
+    assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
+    assert(!sc.listJars().exists(_.contains("commons-lang_commons-lang-2.6.jar")))
+  }
+
+  test("SPARK-34506: Add jar support Ivy URI -- test exclude param when transitive unspecified") {
+    sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+    sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?exclude=commons-lang:commons-lang")
+    assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
+    assert(sc.listJars().exists(_.contains("org.slf4j_slf4j-api-1.7.10.jar")))
+    assert(!sc.listJars().exists(_.contains("commons-lang_commons-lang-2.6.jar")))
+  }
+
+  test("SPARK-33084: Add jar support Ivy URI -- test exclude param when transitive=true") {
+    sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+    sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0" +
+      "?exclude=commons-lang:commons-lang&transitive=true")
+    assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
+    assert(sc.listJars().exists(_.contains("org.slf4j_slf4j-api-1.7.10.jar")))
+    assert(!sc.listJars().exists(_.contains("commons-lang_commons-lang-2.6.jar")))
+  }
+
+  test("SPARK-33084: Add jar support Ivy URI -- test different version") {
+    sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+    sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0")
+    sc.addJar("ivy://org.apache.hive:hive-storage-api:2.6.0")
+    assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
+    assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.6.0.jar")))
+  }
+
+  test("SPARK-33084: Add jar support Ivy URI -- test invalid param") {
+    val logAppender = new LogAppender("test log when have invalid parameter")
+    withLogAppender(logAppender) {
+      sc = new SparkContext(
+        new SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+      sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?" +
+        "invalidParam1=foo&invalidParam2=boo")
+      assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
+      assert(logAppender.loggingEvents.exists(_.getRenderedMessage.contains(
+        "Invalid parameters `invalidParam1,invalidParam2` found in Ivy URI query " +
+          "`invalidParam1=foo&invalidParam2=boo`.")))
+    }
+  }
+
+  test("SPARK-33084: Add jar support Ivy URI -- test multiple transitive params") {
+    sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+    // transitive=invalidValue will win and treated as false
+    sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?" +
+      "transitive=true&transitive=invalidValue")
+    assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
+    assert(!sc.listJars().exists(_.contains("commons-lang_commons-lang-2.6.jar")))
+
+    // transitive=true will win
+    sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?" +
+      "transitive=false&transitive=invalidValue&transitive=true")
+    assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
+    assert(sc.listJars().exists(_.contains("commons-lang_commons-lang-2.6.jar")))
+  }
+
+  test("SPARK-33084: Add jar support Ivy URI -- test param key case sensitive") {
+    sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+    sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=false")
+    assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
+    assert(!sc.listJars().exists(_.contains("commons-lang_commons-lang-2.6.jar")))
+
+    sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?TRANSITIVE=false")
+    assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
+    assert(sc.listJars().exists(_.contains("commons-lang_commons-lang-2.6.jar")))
+  }
+
+  test("SPARK-33084: Add jar support Ivy URI -- test transitive value case insensitive") {
+    sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+    sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=FALSE")
+    assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
+    assert(!sc.listJars().exists(_.contains("commons-lang_commons-lang-2.6.jar")))
+
+    sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=false")
+    assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
+    assert(!sc.listJars().exists(_.contains("commons-lang_commons-lang-2.6.jar")))
+  }
+
+  test("SPARK-34346: hadoop configuration priority for spark/hive/hadoop configs") {
+    val testKey = "hadoop.tmp.dir"
+    val bufferKey = "io.file.buffer.size"
+    val hadoopConf0 = new Configuration()
+    hadoopConf0.set(testKey, "/tmp/hive_zero")
+
+    val hiveConfFile = Utils.getContextOrSparkClassLoader.getResource("hive-site.xml")
+    assert(hiveConfFile != null)
+    hadoopConf0.addResource(hiveConfFile)
+    assert(hadoopConf0.get(testKey) === "/tmp/hive_zero")
+    assert(hadoopConf0.get(bufferKey) === "201811")
+
+    val sparkConf = new SparkConf()
+      .setAppName("test")
+      .setMaster("local")
+      .set(BUFFER_SIZE, 65536)
+    sc = new SparkContext(sparkConf)
+    assert(sc.hadoopConfiguration.get(testKey) === "/tmp/hive_one",
+      "hive configs have higher priority than hadoop ones ")
+    assert(sc.hadoopConfiguration.get(bufferKey).toInt === 65536,
+      "spark configs have higher priority than hive ones")
+
+    resetSparkContext()
+
+    sparkConf
+      .set("spark.hadoop.hadoop.tmp.dir", "/tmp/hive_two")
+      .set(s"spark.hadoop.$bufferKey", "20181117")
+    sc = new SparkContext(sparkConf)
+    assert(sc.hadoopConfiguration.get(testKey) === "/tmp/hive_two",
+      "spark.hadoop configs have higher priority than hive/hadoop ones")
+    assert(sc.hadoopConfiguration.get(bufferKey).toInt === 65536,
+      "spark configs have higher priority than spark.hadoop configs")
+  }
+
+  test("SPARK-34225: addFile/addJar shouldn't further encode URI if a URI form string is passed") {
+    withTempDir { dir =>
+      val jar1 = File.createTempFile("testprefix", "test jar.jar", dir)
+      val jarUrl1 = jar1.toURI.toString
+      val file1 = File.createTempFile("testprefix", "test file.txt", dir)
+      val fileUrl1 = file1.toURI.toString
+      val jar2 = File.createTempFile("testprefix", "test %20jar.jar", dir)
+      val file2 = File.createTempFile("testprefix", "test %20file.txt", dir)
+
+      try {
+        sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local"))
+        sc.addJar(jarUrl1)
+        sc.addFile(fileUrl1)
+        sc.addJar(jar2.toString)
+        sc.addFile(file2.toString)
+        sc.parallelize(Array(1), 1).map { x =>
+          val gottenJar1 = new File(SparkFiles.get(jar1.getName))
+          if (!gottenJar1.exists()) {
+            throw new SparkException("file doesn't exist : " + jar1)
+          }
+          val gottenFile1 = new File(SparkFiles.get(file1.getName))
+          if (!gottenFile1.exists()) {
+            throw new SparkException("file doesn't exist : " + file1)
+          }
+          val gottenJar2 = new File(SparkFiles.get(jar2.getName))
+          if (!gottenJar2.exists()) {
+            throw new SparkException("file doesn't exist : " + jar2)
+          }
+          val gottenFile2 = new File(SparkFiles.get(file2.getName))
+          if (!gottenFile2.exists()) {
+            throw new SparkException("file doesn't exist : " + file2)
+          }
+          x
+        }.collect()
+      } finally {
+        sc.stop()
+      }
+    }
+  }
+
+  test("SPARK-35383: Fill missing S3A magic committer configs if needed") {
+    val c1 = new SparkConf().setAppName("s3a-test").setMaster("local")
+    sc = new SparkContext(c1)
+    assertFalse(sc.getConf.contains("spark.hadoop.fs.s3a.committer.name"))
+
+    resetSparkContext()
+    val c2 = c1.clone.set("spark.hadoop.fs.s3a.bucket.mybucket.committer.magic.enabled", "false")
+    sc = new SparkContext(c2)
+    assertFalse(sc.getConf.contains("spark.hadoop.fs.s3a.committer.name"))
+
+    resetSparkContext()
+    val c3 = c1.clone.set("spark.hadoop.fs.s3a.bucket.mybucket.committer.magic.enabled", "true")
+    sc = new SparkContext(c3)
+    Seq(
+      "spark.hadoop.fs.s3a.committer.magic.enabled" -> "true",
+      "spark.hadoop.fs.s3a.committer.name" -> "magic",
+      "spark.hadoop.mapreduce.outputcommitter.factory.scheme.s3a" ->
+        "org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory",
+      "spark.sql.parquet.output.committer.class" ->
+        "org.apache.spark.internal.io.cloud.BindingParquetOutputCommitter",
+      "spark.sql.sources.commitProtocolClass" ->
+        "org.apache.spark.internal.io.cloud.PathOutputCommitProtocol"
+    ).foreach { case (k, v) =>
+      assertEquals(v, sc.getConf.get(k))
+    }
+
+    // Respect a user configuration
+    resetSparkContext()
+    val c4 = c1.clone
+      .set("spark.hadoop.fs.s3a.committer.magic.enabled", "false")
+      .set("spark.hadoop.fs.s3a.bucket.mybucket.committer.magic.enabled", "true")
+    sc = new SparkContext(c4)
+    Seq(
+      "spark.hadoop.fs.s3a.committer.magic.enabled" -> "false",
+      "spark.hadoop.fs.s3a.committer.name" -> null,
+      "spark.hadoop.mapreduce.outputcommitter.factory.scheme.s3a" -> null,
+      "spark.sql.parquet.output.committer.class" -> null,
+      "spark.sql.sources.commitProtocolClass" -> null
+    ).foreach { case (k, v) =>
+      if (v == null) {
+        assertFalse(sc.getConf.contains(k))
+      } else {
+        assertEquals(v, sc.getConf.get(k))
+      }
+    }
+  }
 }
 
 object SparkContextSuite {
diff --git a/core/src/test/scala/org/apache/spark/SparkFunSuite.scala b/core/src/test/scala/org/apache/spark/SparkFunSuite.scala
index ec641f8294b29..939e64d8e1767 100644
--- a/core/src/test/scala/org/apache/spark/SparkFunSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkFunSuite.scala
@@ -19,19 +19,22 @@ package org.apache.spark
 
 // scalastyle:off
 import java.io.File
+import java.nio.file.Path
 import java.util.{Locale, TimeZone}
 
-import org.apache.log4j.spi.LoggingEvent
-
 import scala.annotation.tailrec
+import scala.collection.mutable.ArrayBuffer
+
+import org.apache.commons.io.FileUtils
 import org.apache.log4j.{Appender, AppenderSkeleton, Level, Logger}
-import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll, BeforeAndAfterEach, FunSuite, Outcome}
+import org.apache.log4j.spi.LoggingEvent
+import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll, BeforeAndAfterEach, Failed, Outcome}
+import org.scalatest.funsuite.AnyFunSuite
+
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.Tests.IS_TESTING
 import org.apache.spark.util.{AccumulatorContext, Utils}
 
-import scala.collection.mutable.ArrayBuffer
-
 /**
  * Base abstract class for all unit tests in Spark for handling common functionality.
  *
@@ -57,13 +60,19 @@ import scala.collection.mutable.ArrayBuffer
  * }
  */
 abstract class SparkFunSuite
-  extends FunSuite
+  extends AnyFunSuite
   with BeforeAndAfterAll
   with BeforeAndAfterEach
   with ThreadAudit
   with Logging {
 // scalastyle:on
 
+  // Initialize the logger forcibly to let the logger log timestamp
+  // based on the local time zone depending on environments.
+  // The default time zone will be set to America/Los_Angeles later
+  // so this initialization is necessary here.
+  log
+
   // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
   TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
   // Add Locale setting
@@ -100,6 +109,28 @@ abstract class SparkFunSuite
     getTestResourceFile(file).getCanonicalPath
   }
 
+  protected final def copyAndGetResourceFile(fileName: String, suffix: String): File = {
+    val url = Thread.currentThread().getContextClassLoader.getResource(fileName)
+    // To avoid illegal accesses to a resource file inside jar
+    // (URISyntaxException might be thrown when accessing it),
+    // copy it into a temporary one for accessing it from the dependent module.
+    val file = File.createTempFile("test-resource", suffix)
+    file.deleteOnExit()
+    FileUtils.copyURLToFile(url, file)
+    file
+  }
+
+  /**
+   * Get a Path relative to the root project. It is assumed that a spark home is set.
+   */
+  protected final def getWorkspaceFilePath(first: String, more: String*): Path = {
+    if (!(sys.props.contains("spark.test.home") || sys.env.contains("SPARK_HOME"))) {
+      fail("spark.test.home or SPARK_HOME is not set.")
+    }
+    val sparkHome = sys.props.getOrElse("spark.test.home", sys.env("SPARK_HOME"))
+    java.nio.file.Paths.get(sparkHome, first +: more: _*)
+  }
+
   /**
    * Note: this method doesn't support `BeforeAndAfter`. You must use `BeforeAndAfterEach` to
    * set up and tear down resources.
@@ -141,6 +172,8 @@ abstract class SparkFunSuite
     }
   }
 
+  protected def logForFailedTest(): Unit = {}
+
   /**
    * Log the suite name and the test name before and after each test.
    *
@@ -154,7 +187,13 @@ abstract class SparkFunSuite
     val shortSuiteName = suiteName.replaceAll("org.apache.spark", "o.a.s")
     try {
       logInfo(s"\n\n===== TEST OUTPUT FOR $shortSuiteName: '$testName' =====\n")
-      test()
+      val outcome = test()
+      outcome match {
+        case _: Failed =>
+          logForFailedTest()
+        case _ =>
+      }
+      outcome
     } finally {
       logInfo(s"\n\n===== FINISHED $shortSuiteName: '$testName' =====\n")
     }
diff --git a/core/src/test/scala/org/apache/spark/StatusTrackerSuite.scala b/core/src/test/scala/org/apache/spark/StatusTrackerSuite.scala
index f527bbe718524..e6d3377120e56 100644
--- a/core/src/test/scala/org/apache/spark/StatusTrackerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/StatusTrackerSuite.scala
@@ -18,10 +18,10 @@
 package org.apache.spark
 
 import scala.concurrent.duration._
-import scala.language.implicitConversions
 
-import org.scalatest.Matchers
 import org.scalatest.concurrent.Eventually._
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.JobExecutionStatus._
 
diff --git a/core/src/test/scala/org/apache/spark/TempLocalSparkContext.scala b/core/src/test/scala/org/apache/spark/TempLocalSparkContext.scala
new file mode 100644
index 0000000000000..6d5fcd1edfb03
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/TempLocalSparkContext.scala
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark
+
+import _root_.io.netty.util.internal.logging.{InternalLoggerFactory, Slf4JLoggerFactory}
+import org.scalatest.BeforeAndAfterAll
+import org.scalatest.BeforeAndAfterEach
+import org.scalatest.Suite
+
+import org.apache.spark.internal.Logging
+import org.apache.spark.resource.ResourceProfile
+
+/**
+ * Manages a local `sc` `SparkContext` variable, correctly stopping it after each test.
+ *
+ * Note: this class is a copy of [[LocalSparkContext]]. Why copy it? Reduce conflict. Because
+ * many test suites use [[LocalSparkContext]] and overwrite some variable or function (e.g.
+ * sc of LocalSparkContext), there occurs conflict when we refactor the `sc` as a new function.
+ * After migrating all test suites that use [[LocalSparkContext]] to use
+ * [[TempLocalSparkContext]], we will delete the original [[LocalSparkContext]] and rename
+ * [[TempLocalSparkContext]] to [[LocalSparkContext]].
+ */
+trait TempLocalSparkContext extends BeforeAndAfterEach
+  with BeforeAndAfterAll with Logging { self: Suite =>
+
+  private var _conf: SparkConf = defaultSparkConf
+
+  @transient private var _sc: SparkContext = _
+
+  def conf: SparkConf = _conf
+
+  /**
+   * Currently, we are focusing on the reconstruction of LocalSparkContext, so this method
+   * was created temporarily. When the migration work is completed, this method will be
+   * renamed to `sc` and the variable `sc` will be deleted.
+   */
+  def sc: SparkContext = {
+    if (_sc == null) {
+      _sc = new SparkContext(_conf)
+    }
+    _sc
+  }
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    InternalLoggerFactory.setDefaultFactory(Slf4JLoggerFactory.INSTANCE)
+  }
+
+  override def afterEach(): Unit = {
+    try {
+      resetSparkContext()
+    } finally {
+      super.afterEach()
+    }
+  }
+
+  def resetSparkContext(): Unit = {
+    TempLocalSparkContext.stop(_sc)
+    ResourceProfile.clearDefaultProfile()
+    _sc = null
+    _conf = defaultSparkConf
+  }
+
+  private def defaultSparkConf: SparkConf = new SparkConf()
+    .setMaster("local[2]").setAppName(s"${this.getClass.getSimpleName}")
+}
+
+object TempLocalSparkContext {
+  def stop(sc: SparkContext): Unit = {
+    if (sc != null) {
+      sc.stop()
+    }
+    // To avoid RPC rebinding to the same port, since it doesn't unbind immediately on shutdown
+    System.clearProperty("spark.driver.port")
+  }
+
+  /** Runs `f` by passing in `sc` and ensures that `sc` is stopped. */
+  def withSpark[T](sc: SparkContext)(f: SparkContext => T): T = {
+    try {
+      f(sc)
+    } finally {
+      stop(sc)
+    }
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/ThreadAudit.scala b/core/src/test/scala/org/apache/spark/ThreadAudit.scala
index 44d1f220bf6b1..1e2917621fa79 100644
--- a/core/src/test/scala/org/apache/spark/ThreadAudit.scala
+++ b/core/src/test/scala/org/apache/spark/ThreadAudit.scala
@@ -26,7 +26,7 @@ import org.apache.spark.internal.Logging
  */
 trait ThreadAudit extends Logging {
 
-  val threadWhiteList = Set(
+  val threadExcludeList = Set(
     /**
      * Netty related internal threads.
      * These are excluded because their lifecycle is handled by the netty itself
@@ -108,7 +108,7 @@ trait ThreadAudit extends Logging {
 
     if (threadNamesSnapshot.nonEmpty) {
       val remainingThreadNames = runningThreadNames().diff(threadNamesSnapshot)
-        .filterNot { s => threadWhiteList.exists(s.matches(_)) }
+        .filterNot { s => threadExcludeList.exists(s.matches(_)) }
       if (remainingThreadNames.nonEmpty) {
         logWarning(s"\n\n===== POSSIBLE THREAD LEAK IN SUITE $shortSuiteName, " +
           s"thread names: ${remainingThreadNames.mkString(", ")} =====\n")
diff --git a/core/src/test/scala/org/apache/spark/api/python/PythonBroadcastSuite.scala b/core/src/test/scala/org/apache/spark/api/python/PythonBroadcastSuite.scala
index dffdd96cd2dcc..daf0151ad65a7 100644
--- a/core/src/test/scala/org/apache/spark/api/python/PythonBroadcastSuite.scala
+++ b/core/src/test/scala/org/apache/spark/api/python/PythonBroadcastSuite.scala
@@ -21,7 +21,8 @@ import java.io.{File, PrintWriter}
 
 import scala.io.Source
 
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.{SharedSparkContext, SparkConf, SparkFunSuite}
 import org.apache.spark.internal.config.Kryo._
diff --git a/core/src/test/scala/org/apache/spark/benchmark/Benchmark.scala b/core/src/test/scala/org/apache/spark/benchmark/Benchmark.scala
index 9629f5ab1a3dd..5511852ca176e 100644
--- a/core/src/test/scala/org/apache/spark/benchmark/Benchmark.scala
+++ b/core/src/test/scala/org/apache/spark/benchmark/Benchmark.scala
@@ -26,7 +26,6 @@ import scala.util.Try
 
 import org.apache.commons.io.output.TeeOutputStream
 import org.apache.commons.lang3.SystemUtils
-import org.scalatest.Assertions._
 
 import org.apache.spark.util.Utils
 
@@ -112,11 +111,12 @@ private[spark] class Benchmark(
     // The results are going to be processor specific so it is useful to include that.
     out.println(Benchmark.getJVMOSInfo())
     out.println(Benchmark.getProcessorName())
-    out.printf("%-40s %14s %14s %11s %12s %13s %10s\n", name + ":", "Best Time(ms)", "Avg Time(ms)", "Stdev(ms)", "Rate(M/s)",
-      "Per Row(ns)", "Relative")
-    out.println("-" * 120)
+    val nameLen = Math.max(40, Math.max(name.length, benchmarks.map(_.name.length).max))
+    out.printf(s"%-${nameLen}s %14s %14s %11s %12s %13s %10s\n",
+      name + ":", "Best Time(ms)", "Avg Time(ms)", "Stdev(ms)", "Rate(M/s)", "Per Row(ns)", "Relative")
+    out.println("-" * (nameLen + 80))
     results.zip(benchmarks).foreach { case (result, benchmark) =>
-      out.printf("%-40s %14s %14s %11s %12s %13s %10s\n",
+      out.printf(s"%-${nameLen}s %14s %14s %11s %12s %13s %10s\n",
         benchmark.name,
         "%5.0f" format result.bestMs,
         "%4.0f" format result.avgMs,
diff --git a/core/src/test/scala/org/apache/spark/benchmark/BenchmarkBase.scala b/core/src/test/scala/org/apache/spark/benchmark/BenchmarkBase.scala
index 55e34b32fe0d4..9ba2f0d04c901 100644
--- a/core/src/test/scala/org/apache/spark/benchmark/BenchmarkBase.scala
+++ b/core/src/test/scala/org/apache/spark/benchmark/BenchmarkBase.scala
@@ -19,9 +19,11 @@ package org.apache.spark.benchmark
 
 import java.io.{File, FileOutputStream, OutputStream}
 
+import org.apache.spark.internal.config.Tests.IS_TESTING
+
 /**
  * A base class for generate benchmark results to a file.
- * For JDK9+, JDK major version number is added to the file names to distingush the results.
+ * For JDK9+, JDK major version number is added to the file names to distinguish the results.
  */
 abstract class BenchmarkBase {
   var output: Option[OutputStream] = None
@@ -42,12 +44,25 @@ abstract class BenchmarkBase {
   }
 
   def main(args: Array[String]): Unit = {
+    // turning this on so the behavior between running benchmark via `spark-submit` or SBT will
+    // be consistent, also allow users to turn on/off certain behavior such as
+    // `spark.sql.codegen.factoryMode`
+    System.setProperty(IS_TESTING.key, "true")
     val regenerateBenchmarkFiles: Boolean = System.getenv("SPARK_GENERATE_BENCHMARK_FILES") == "1"
     if (regenerateBenchmarkFiles) {
       val version = System.getProperty("java.version").split("\\D+")(0).toInt
       val jdkString = if (version > 8) s"-jdk$version" else ""
-      val resultFileName = s"${this.getClass.getSimpleName.replace("$", "")}$jdkString-results.txt"
-      val file = new File(s"benchmarks/$resultFileName")
+      val resultFileName =
+        s"${this.getClass.getSimpleName.replace("$", "")}$jdkString$suffix-results.txt"
+      val prefix = Benchmarks.currentProjectRoot.map(_ + "/").getOrElse("")
+      val dir = new File(s"${prefix}benchmarks/")
+      if (!dir.exists()) {
+        // scalastyle:off println
+        println(s"Creating ${dir.getAbsolutePath} for benchmark results.")
+        // scalastyle:on println
+        dir.mkdirs()
+      }
+      val file = new File(dir, resultFileName)
       if (!file.exists()) {
         file.createNewFile()
       }
@@ -65,6 +80,8 @@ abstract class BenchmarkBase {
     afterAll()
   }
 
+  def suffix: String = ""
+
   /**
    * Any shutdown code to ensure a clean shutdown
    */
diff --git a/core/src/test/scala/org/apache/spark/benchmark/Benchmarks.scala b/core/src/test/scala/org/apache/spark/benchmark/Benchmarks.scala
new file mode 100644
index 0000000000000..2bb70bc75f6bb
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/benchmark/Benchmarks.scala
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.benchmark
+
+import java.io.File
+import java.lang.reflect.Modifier
+import java.nio.file.{FileSystems, Paths}
+import java.util.Locale
+
+import scala.collection.JavaConverters._
+import scala.util.Try
+
+import com.google.common.reflect.ClassPath
+
+/**
+ * Run all benchmarks. To run this benchmark, you should build Spark with either Maven or SBT.
+ * After that, you can run as below:
+ *
+ * {{{
+ *   1. with spark-submit
+ *      bin/spark-submit --class <this class>
+ *        --jars <all spark test jars>,<spark external package jars>
+ *        <spark core test jar> <glob pattern for class> <extra arguments>
+ *   2. generate result:
+ *      SPARK_GENERATE_BENCHMARK_FILES=1 bin/spark-submit --class <this class>
+ *        --jars <all spark test jars>,<spark external package jars>
+ *        <spark core test jar> <glob pattern for class> <extra arguments>
+ *      Results will be written to all corresponding files under "benchmarks/".
+ *      Notice that it detects the sub-project's directories from jar's paths so the provided jars
+ *      should be properly placed under target (Maven build) or target/scala-* (SBT) when you
+ *      generate the files.
+ * }}}
+ *
+ * You can use a command as below to find all the test jars.
+ * Make sure to do not select duplicated jars created by different versions of builds or tools.
+ * {{{
+ *   find . -name '*-SNAPSHOT-tests.jar' | paste -sd ',' -
+ * }}}
+ *
+ * The example below runs all benchmarks and generates the results:
+ * {{{
+ *   SPARK_GENERATE_BENCHMARK_FILES=1 bin/spark-submit --class \
+ *     org.apache.spark.benchmark.Benchmarks --jars \
+ *     "`find . -name '*-SNAPSHOT-tests.jar' -o -name '*avro*-SNAPSHOT.jar' | paste -sd ',' -`" \
+ *     "`find . -name 'spark-core*-SNAPSHOT-tests.jar'`" \
+ *     "*"
+ * }}}
+ *
+ * The example below runs all benchmarks under "org.apache.spark.sql.execution.datasources"
+ * {{{
+ *   bin/spark-submit --class \
+ *     org.apache.spark.benchmark.Benchmarks --jars \
+ *     "`find . -name '*-SNAPSHOT-tests.jar' -o -name '*avro*-SNAPSHOT.jar' | paste -sd ',' -`" \
+ *     "`find . -name 'spark-core*-SNAPSHOT-tests.jar'`" \
+ *     "org.apache.spark.sql.execution.datasources.*"
+ * }}}
+ */
+
+object Benchmarks {
+  var currentProjectRoot: Option[String] = None
+
+  def main(args: Array[String]): Unit = {
+    val isFailFast = sys.env.get(
+      "SPARK_BENCHMARK_FAILFAST").map(_.toLowerCase(Locale.ROOT).trim.toBoolean).getOrElse(true)
+    val numOfSplits = sys.env.get(
+      "SPARK_BENCHMARK_NUM_SPLITS").map(_.toLowerCase(Locale.ROOT).trim.toInt).getOrElse(1)
+    val currentSplit = sys.env.get(
+      "SPARK_BENCHMARK_CUR_SPLIT").map(_.toLowerCase(Locale.ROOT).trim.toInt - 1).getOrElse(0)
+    var numBenchmark = 0
+
+    var isBenchmarkFound = false
+    val benchmarkClasses = ClassPath.from(
+      Thread.currentThread.getContextClassLoader
+    ).getTopLevelClassesRecursive("org.apache.spark").asScala.toArray
+    val matcher = FileSystems.getDefault.getPathMatcher(s"glob:${args.head}")
+
+    benchmarkClasses.foreach { info =>
+      lazy val clazz = info.load
+      lazy val runBenchmark = clazz.getMethod("main", classOf[Array[String]])
+      // isAssignableFrom seems not working with the reflected class from Guava's
+      // getTopLevelClassesRecursive.
+      require(args.length > 0, "Benchmark class to run should be specified.")
+      if (
+          info.getName.endsWith("Benchmark") &&
+          // TODO(SPARK-34927): Support TPCDSQueryBenchmark in Benchmarks
+          !info.getName.endsWith("TPCDSQueryBenchmark") &&
+          matcher.matches(Paths.get(info.getName)) &&
+          Try(runBenchmark).isSuccess && // Does this has a main method?
+          !Modifier.isAbstract(clazz.getModifiers) // Is this a regular class?
+      ) {
+        numBenchmark += 1
+        if (numBenchmark % numOfSplits == currentSplit) {
+          isBenchmarkFound = true
+
+          val targetDirOrProjDir =
+            new File(clazz.getProtectionDomain.getCodeSource.getLocation.toURI)
+              .getParentFile.getParentFile
+
+          // The root path to be referred in each benchmark.
+          currentProjectRoot = Some {
+            if (targetDirOrProjDir.getName == "target") {
+              // SBT build
+              targetDirOrProjDir.getParentFile.getCanonicalPath
+            } else {
+              // Maven build
+              targetDirOrProjDir.getCanonicalPath
+            }
+          }
+
+          // scalastyle:off println
+          println(s"Running ${clazz.getName}:")
+          // scalastyle:on println
+          // Force GC to minimize the side effect.
+          System.gc()
+          try {
+            runBenchmark.invoke(null, args.tail.toArray)
+          } catch {
+            case e: Throwable if !isFailFast =>
+              // scalastyle:off println
+              println(s"${clazz.getName} failed with the exception below:")
+              // scalastyle:on println
+              e.printStackTrace()
+          }
+        }
+      }
+    }
+
+    if (!isBenchmarkFound) throw new RuntimeException("No benchmark found to run.")
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/broadcast/BroadcastSuite.scala b/core/src/test/scala/org/apache/spark/broadcast/BroadcastSuite.scala
index a6776ee077894..5e8b25f425166 100644
--- a/core/src/test/scala/org/apache/spark/broadcast/BroadcastSuite.scala
+++ b/core/src/test/scala/org/apache/spark/broadcast/BroadcastSuite.scala
@@ -68,14 +68,14 @@ class BroadcastSuite extends SparkFunSuite with LocalSparkContext with Encryptio
   }
 
   encryptionTest("Accessing TorrentBroadcast variables in a local cluster") { conf =>
-    val numSlaves = 4
+    val numWorkers = 4
     conf.set(SERIALIZER, "org.apache.spark.serializer.KryoSerializer")
     conf.set(config.BROADCAST_COMPRESS, true)
-    sc = new SparkContext("local-cluster[%d, 1, 1024]".format(numSlaves), "test", conf)
+    sc = new SparkContext("local-cluster[%d, 1, 1024]".format(numWorkers), "test", conf)
     val list = List[Int](1, 2, 3, 4)
     val broadcast = sc.broadcast(list)
-    val results = sc.parallelize(1 to numSlaves).map(x => (x, broadcast.value.sum))
-    assert(results.collect().toSet === (1 to numSlaves).map(x => (x, 10)).toSet)
+    val results = sc.parallelize(1 to numWorkers).map(x => (x, broadcast.value.sum))
+    assert(results.collect().toSet === (1 to numWorkers).map(x => (x, 10)).toSet)
   }
 
   test("TorrentBroadcast's blockifyObject and unblockifyObject are inverses") {
@@ -99,12 +99,12 @@ class BroadcastSuite extends SparkFunSuite with LocalSparkContext with Encryptio
   }
 
   test("Test Lazy Broadcast variables with TorrentBroadcast") {
-    val numSlaves = 2
-    sc = new SparkContext("local-cluster[%d, 1, 1024]".format(numSlaves), "test")
-    val rdd = sc.parallelize(1 to numSlaves)
+    val numWorkers = 2
+    sc = new SparkContext("local-cluster[%d, 1, 1024]".format(numWorkers), "test")
+    val rdd = sc.parallelize(1 to numWorkers)
     val results = new DummyBroadcastClass(rdd).doSomething()
 
-    assert(results.toSet === (1 to numSlaves).map(x => (x, false)).toSet)
+    assert(results.toSet === (1 to numWorkers).map(x => (x, false)).toSet)
   }
 
   test("Unpersisting TorrentBroadcast on executors only in local mode") {
@@ -196,27 +196,27 @@ class BroadcastSuite extends SparkFunSuite with LocalSparkContext with Encryptio
    */
   private def testUnpersistTorrentBroadcast(distributed: Boolean,
       removeFromDriver: Boolean): Unit = {
-    val numSlaves = if (distributed) 2 else 0
+    val numWorkers = if (distributed) 2 else 0
 
     // Verify that blocks are persisted only on the driver
     def afterCreation(broadcastId: Long, bmm: BlockManagerMaster): Unit = {
       var blockId = BroadcastBlockId(broadcastId)
-      var statuses = bmm.getBlockStatus(blockId, askSlaves = true)
+      var statuses = bmm.getBlockStatus(blockId, askStorageEndpoints = true)
       assert(statuses.size === 1)
 
       blockId = BroadcastBlockId(broadcastId, "piece0")
-      statuses = bmm.getBlockStatus(blockId, askSlaves = true)
+      statuses = bmm.getBlockStatus(blockId, askStorageEndpoints = true)
       assert(statuses.size === 1)
     }
 
     // Verify that blocks are persisted in both the executors and the driver
     def afterUsingBroadcast(broadcastId: Long, bmm: BlockManagerMaster): Unit = {
       var blockId = BroadcastBlockId(broadcastId)
-      val statuses = bmm.getBlockStatus(blockId, askSlaves = true)
-      assert(statuses.size === numSlaves + 1)
+      val statuses = bmm.getBlockStatus(blockId, askStorageEndpoints = true)
+      assert(statuses.size === numWorkers + 1)
 
       blockId = BroadcastBlockId(broadcastId, "piece0")
-      assert(statuses.size === numSlaves + 1)
+      assert(statuses.size === numWorkers + 1)
     }
 
     // Verify that blocks are unpersisted on all executors, and on all nodes if removeFromDriver
@@ -224,16 +224,16 @@ class BroadcastSuite extends SparkFunSuite with LocalSparkContext with Encryptio
     def afterUnpersist(broadcastId: Long, bmm: BlockManagerMaster): Unit = {
       var blockId = BroadcastBlockId(broadcastId)
       var expectedNumBlocks = if (removeFromDriver) 0 else 1
-      var statuses = bmm.getBlockStatus(blockId, askSlaves = true)
+      var statuses = bmm.getBlockStatus(blockId, askStorageEndpoints = true)
       assert(statuses.size === expectedNumBlocks)
 
       blockId = BroadcastBlockId(broadcastId, "piece0")
       expectedNumBlocks = if (removeFromDriver) 0 else 1
-      statuses = bmm.getBlockStatus(blockId, askSlaves = true)
+      statuses = bmm.getBlockStatus(blockId, askStorageEndpoints = true)
       assert(statuses.size === expectedNumBlocks)
     }
 
-    testUnpersistBroadcast(distributed, numSlaves, afterCreation,
+    testUnpersistBroadcast(distributed, numWorkers, afterCreation,
       afterUsingBroadcast, afterUnpersist, removeFromDriver)
   }
 
@@ -248,7 +248,7 @@ class BroadcastSuite extends SparkFunSuite with LocalSparkContext with Encryptio
    */
   private def testUnpersistBroadcast(
       distributed: Boolean,
-      numSlaves: Int,  // used only when distributed = true
+      numWorkers: Int,  // used only when distributed = true
       afterCreation: (Long, BlockManagerMaster) => Unit,
       afterUsingBroadcast: (Long, BlockManagerMaster) => Unit,
       afterUnpersist: (Long, BlockManagerMaster) => Unit,
@@ -256,10 +256,10 @@ class BroadcastSuite extends SparkFunSuite with LocalSparkContext with Encryptio
 
     sc = if (distributed) {
       val _sc =
-        new SparkContext("local-cluster[%d, 1, 1024]".format(numSlaves), "test")
+        new SparkContext("local-cluster[%d, 1, 1024]".format(numWorkers), "test")
       // Wait until all salves are up
       try {
-        TestUtils.waitUntilExecutorsUp(_sc, numSlaves, 60000)
+        TestUtils.waitUntilExecutorsUp(_sc, numWorkers, 60000)
         _sc
       } catch {
         case e: Throwable =>
@@ -278,7 +278,7 @@ class BroadcastSuite extends SparkFunSuite with LocalSparkContext with Encryptio
 
     // Use broadcast variable on all executors
     val partitions = 10
-    assert(partitions > numSlaves)
+    assert(partitions > numWorkers)
     val results = sc.parallelize(1 to partitions, partitions).map(x => (x, broadcast.value.sum))
     assert(results.collect().toSet === (1 to partitions).map(x => (x, list.sum)).toSet)
     afterUsingBroadcast(broadcast.id, blockManagerMaster)
diff --git a/core/src/test/scala/org/apache/spark/deploy/ClientSuite.scala b/core/src/test/scala/org/apache/spark/deploy/ClientSuite.scala
index 6a99dbca64f4b..792168834dea2 100644
--- a/core/src/test/scala/org/apache/spark/deploy/ClientSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/ClientSuite.scala
@@ -17,7 +17,8 @@
 
 package org.apache.spark.deploy
 
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.SparkFunSuite
 
diff --git a/core/src/test/scala/org/apache/spark/deploy/DecommissionWorkerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/DecommissionWorkerSuite.scala
new file mode 100644
index 0000000000000..c2486b9650d5c
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/deploy/DecommissionWorkerSuite.scala
@@ -0,0 +1,469 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.deploy
+
+import java.util.concurrent.{ConcurrentHashMap, ConcurrentLinkedQueue}
+import java.util.concurrent.atomic.AtomicBoolean
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+import scala.concurrent.duration._
+
+import org.scalatest.BeforeAndAfterEach
+import org.scalatest.concurrent.Eventually._
+
+import org.apache.spark._
+import org.apache.spark.deploy.DeployMessages.{DecommissionWorkers, MasterStateResponse, RequestMasterState}
+import org.apache.spark.deploy.master.{ApplicationInfo, Master, WorkerInfo}
+import org.apache.spark.deploy.worker.Worker
+import org.apache.spark.internal.{config, Logging}
+import org.apache.spark.network.TransportContext
+import org.apache.spark.network.netty.SparkTransportConf
+import org.apache.spark.network.shuffle.ExternalBlockHandler
+import org.apache.spark.rpc.{RpcAddress, RpcEnv}
+import org.apache.spark.scheduler._
+import org.apache.spark.shuffle.FetchFailedException
+import org.apache.spark.storage.BlockManagerId
+import org.apache.spark.util.Utils
+
+class DecommissionWorkerSuite
+  extends SparkFunSuite
+    with Logging
+    with LocalSparkContext
+    with BeforeAndAfterEach {
+
+  private var masterAndWorkerConf: SparkConf = null
+  private var masterAndWorkerSecurityManager: SecurityManager = null
+  private var masterRpcEnv: RpcEnv = null
+  private var master: Master = null
+  private var workerIdToRpcEnvs: mutable.HashMap[String, RpcEnv] = null
+  private var workers: mutable.ArrayBuffer[Worker] = null
+
+  override def beforeEach(): Unit = {
+    super.beforeEach()
+    masterAndWorkerConf = new SparkConf()
+      .set(config.DECOMMISSION_ENABLED, true)
+    masterAndWorkerSecurityManager = new SecurityManager(masterAndWorkerConf)
+    masterRpcEnv = RpcEnv.create(
+      Master.SYSTEM_NAME,
+      "localhost",
+      0,
+      masterAndWorkerConf,
+      masterAndWorkerSecurityManager)
+    master = makeMaster()
+    workerIdToRpcEnvs = mutable.HashMap.empty
+    workers = mutable.ArrayBuffer.empty
+  }
+
+  override def afterEach(): Unit = {
+    try {
+      masterRpcEnv.shutdown()
+      workerIdToRpcEnvs.values.foreach(_.shutdown())
+      workerIdToRpcEnvs.clear()
+      master.stop()
+      workers.foreach(_.stop())
+      workers.clear()
+      masterRpcEnv = null
+    } finally {
+      super.afterEach()
+    }
+  }
+
+  // Unlike TestUtils.withListener, it also waits for the job to be done
+  def withListener(sc: SparkContext, listener: RootStageAwareListener)
+                  (body: SparkListener => Unit): Unit = {
+    sc.addSparkListener(listener)
+    try {
+      body(listener)
+      sc.listenerBus.waitUntilEmpty()
+      listener.waitForJobDone()
+    } finally {
+      sc.listenerBus.removeListener(listener)
+    }
+  }
+
+  test("decommission workers should not result in job failure") {
+    val maxTaskFailures = 2
+    val numTimesToKillWorkers = maxTaskFailures + 1
+    val numWorkers = numTimesToKillWorkers + 1
+    createWorkers(numWorkers)
+
+    // Here we will have a single task job and we will keep decommissioning (and killing) the
+    // worker running that task K times. Where K is more than the maxTaskFailures. Since the worker
+    // is notified of the decommissioning, the task failures can be ignored and not fail
+    // the job.
+
+    sc = createSparkContext(config.TASK_MAX_FAILURES.key -> maxTaskFailures.toString)
+    val executorIdToWorkerInfo = getExecutorToWorkerAssignments
+    val taskIdsKilled = new ConcurrentHashMap[Long, Boolean]
+    val listener = new RootStageAwareListener {
+      override def handleRootTaskStart(taskStart: SparkListenerTaskStart): Unit = {
+        val taskInfo = taskStart.taskInfo
+        if (taskIdsKilled.size() < numTimesToKillWorkers) {
+          val workerInfo = executorIdToWorkerInfo(taskInfo.executorId)
+          decommissionWorkerOnMaster(workerInfo, "partition 0 must die")
+          killWorkerAfterTimeout(workerInfo, 1)
+          taskIdsKilled.put(taskInfo.taskId, true)
+        }
+      }
+    }
+    withListener(sc, listener) { _ =>
+      val jobResult = sc.parallelize(1 to 1, 1).map { _ =>
+        Thread.sleep(5 * 1000L); 1
+      }.count()
+      assert(jobResult === 1)
+    }
+    // single task job that gets to run numTimesToKillWorkers + 1 times.
+    assert(listener.getTasksFinished().size === numTimesToKillWorkers + 1)
+    listener.rootTasksStarted.asScala.foreach { taskInfo =>
+      assert(taskInfo.index == 0, s"Unknown task index ${taskInfo.index}")
+    }
+    listener.rootTasksEnded.asScala.foreach { taskInfo =>
+      assert(taskInfo.index === 0, s"Expected task index ${taskInfo.index} to be 0")
+      // If a task has been killed then it shouldn't be successful
+      val taskSuccessExpected = !taskIdsKilled.getOrDefault(taskInfo.taskId, false)
+      val taskSuccessActual = taskInfo.successful
+      assert(taskSuccessActual === taskSuccessExpected,
+        s"Expected task success $taskSuccessActual == $taskSuccessExpected")
+    }
+  }
+
+  test("decommission workers ensure that shuffle output is regenerated even with shuffle service") {
+    createWorkers(2)
+    val ss = new ExternalShuffleServiceHolder()
+
+    sc = createSparkContext(
+      config.Tests.TEST_NO_STAGE_RETRY.key -> "true",
+      config.SHUFFLE_MANAGER.key -> "sort",
+      config.SHUFFLE_SERVICE_ENABLED.key -> "true",
+      config.SHUFFLE_SERVICE_PORT.key -> ss.getPort.toString
+    )
+    TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
+
+    // Here we will create a 2 stage job: The first stage will have two tasks and the second stage
+    // will have one task. The two tasks in the first stage will be long and short. We decommission
+    // and kill the worker after the short task is done. Eventually the driver should get the
+    // executor lost signal for the short task executor. This should trigger regenerating
+    // the shuffle output since we cleanly decommissioned the executor, despite running with an
+    // external shuffle service.
+    try {
+      val executorIdToWorkerInfo = getExecutorToWorkerAssignments
+      val workerForTask0Decommissioned = new AtomicBoolean(false)
+      // single task job
+      val listener = new RootStageAwareListener {
+        override def handleRootTaskEnd(taskEnd: SparkListenerTaskEnd): Unit = {
+          val taskInfo = taskEnd.taskInfo
+          if (taskInfo.index == 0) {
+            if (workerForTask0Decommissioned.compareAndSet(false, true)) {
+              val workerInfo = executorIdToWorkerInfo(taskInfo.executorId)
+              decommissionWorkerOnMaster(workerInfo, "Kill early done map worker")
+              killWorkerAfterTimeout(workerInfo, 0)
+              logInfo(s"Killed the node ${workerInfo.hostPort} that was running the early task")
+            }
+          }
+        }
+      }
+      withListener(sc, listener) { _ =>
+        val jobResult = sc.parallelize(1 to 2, 2).mapPartitionsWithIndex((pid, _) => {
+          val sleepTimeSeconds = if (pid == 0) 1 else 10
+          Thread.sleep(sleepTimeSeconds * 1000L)
+          List(1).iterator
+        }, preservesPartitioning = true).repartition(1).sum()
+        assert(jobResult === 2)
+      }
+      val tasksSeen = listener.getTasksFinished()
+      // 4 tasks: 2 from first stage, one retry due to decom, one more from the second stage.
+      assert(tasksSeen.size === 4, s"Expected 4 tasks but got $tasksSeen")
+      listener.rootTasksStarted.asScala.foreach { taskInfo =>
+        assert(taskInfo.index <= 1, s"Expected ${taskInfo.index} <= 1")
+        assert(taskInfo.successful, s"Task ${taskInfo.index} should be successful")
+      }
+      val tasksEnded = listener.rootTasksEnded.asScala
+      tasksEnded.filter(_.index != 0).foreach { taskInfo =>
+        assert(taskInfo.attemptNumber === 0, "2nd task should succeed on 1st attempt")
+      }
+      val firstTaskAttempts = tasksEnded.filter(_.index == 0)
+      assert(firstTaskAttempts.size > 1, s"Task 0 should have multiple attempts")
+    } finally {
+      ss.close()
+    }
+  }
+
+  def testFetchFailures(initialSleepMillis: Int): Unit = {
+    createWorkers(2)
+    sc = createSparkContext(
+      config.Tests.TEST_NO_STAGE_RETRY.key -> "false",
+      "spark.test.executor.decommission.initial.sleep.millis" -> initialSleepMillis.toString,
+      config.UNREGISTER_OUTPUT_ON_HOST_ON_FETCH_FAILURE.key -> "true")
+    TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
+
+    val executorIdToWorkerInfo = getExecutorToWorkerAssignments
+    val executorToDecom = executorIdToWorkerInfo.keysIterator.next
+
+    // The task code below cannot call executorIdToWorkerInfo, so we need to pre-compute
+    // the worker to decom to force it to be serialized into the task.
+    val workerToDecom = executorIdToWorkerInfo(executorToDecom)
+
+    // The setup of this job is similar to the one above: 2 stage job with first stage having
+    // long and short tasks. Except that we want the shuffle output to be regenerated on a
+    // fetch failure instead of an executor lost. Since it is hard to "trigger a fetch failure",
+    // we manually raise the FetchFailed exception when the 2nd stage's task runs and require that
+    // fetch failure to trigger a recomputation.
+    logInfo(s"Will try to decommission the task running on executor $executorToDecom")
+    val listener = new RootStageAwareListener {
+      override def handleRootTaskEnd(taskEnd: SparkListenerTaskEnd): Unit = {
+        val taskInfo = taskEnd.taskInfo
+        if (taskInfo.executorId == executorToDecom && taskInfo.attemptNumber == 0 &&
+          taskEnd.stageAttemptId == 0 && taskEnd.stageId == 0) {
+          decommissionWorkerOnMaster(workerToDecom,
+            "decommission worker after task on it is done")
+        }
+      }
+    }
+    withListener(sc, listener) { _ =>
+      val jobResult = sc.parallelize(1 to 2, 2).mapPartitionsWithIndex((_, _) => {
+        val executorId = SparkEnv.get.executorId
+        val context = TaskContext.get()
+        // Only sleep in the first attempt to create the required window for decommissioning.
+        // Subsequent attempts don't need to be delayed to speed up the test.
+        if (context.attemptNumber() == 0 && context.stageAttemptNumber() == 0) {
+          val sleepTimeSeconds = if (executorId == executorToDecom) 10 else 1
+          Thread.sleep(sleepTimeSeconds * 1000L)
+        }
+        List(1).iterator
+      }, preservesPartitioning = true)
+        .repartition(1).mapPartitions(iter => {
+        val context = TaskContext.get()
+        if (context.attemptNumber == 0 && context.stageAttemptNumber() == 0) {
+          // Wait a bit for the decommissioning to be triggered in the listener
+          Thread.sleep(5000)
+          // MapIndex is explicitly -1 to force the entire host to be decommissioned
+          // However, this will cause both the tasks in the preceding stage since the host here is
+          // "localhost" (shortcoming of this single-machine unit test in that all the workers
+          // are actually on the same host)
+          throw new FetchFailedException(BlockManagerId(executorToDecom,
+            workerToDecom.host, workerToDecom.port), 0, 0, -1, 0, "Forcing fetch failure")
+        }
+        val sumVal: List[Int] = List(iter.sum)
+        sumVal.iterator
+      }, preservesPartitioning = true)
+        .sum()
+      assert(jobResult === 2)
+    }
+    // 6 tasks: 2 from first stage, 2 rerun again from first stage, 2nd stage attempt 1 and 2.
+    val tasksSeen = listener.getTasksFinished()
+    assert(tasksSeen.size === 6, s"Expected 6 tasks but got $tasksSeen")
+  }
+
+  test("decommission stalled workers ensure that fetch failures lead to rerun") {
+    testFetchFailures(3600 * 1000)
+  }
+
+  test("decommission eager workers ensure that fetch failures lead to rerun") {
+    testFetchFailures(0)
+  }
+
+  private abstract class RootStageAwareListener extends SparkListener {
+    private var rootStageId: Option[Int] = None
+    private val tasksFinished = new ConcurrentLinkedQueue[String]()
+    private val jobDone = new AtomicBoolean(false)
+    val rootTasksStarted = new ConcurrentLinkedQueue[TaskInfo]()
+    val rootTasksEnded = new ConcurrentLinkedQueue[TaskInfo]()
+
+    protected def isRootStageId(stageId: Int): Boolean =
+      (rootStageId.isDefined && rootStageId.get == stageId)
+
+    override def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted): Unit = {
+      if (stageSubmitted.stageInfo.parentIds.isEmpty && rootStageId.isEmpty) {
+        rootStageId = Some(stageSubmitted.stageInfo.stageId)
+      }
+    }
+
+    override def onJobEnd(jobEnd: SparkListenerJobEnd): Unit = {
+      jobEnd.jobResult match {
+        case JobSucceeded => jobDone.set(true)
+        case JobFailed(exception) => logError(s"Job failed", exception)
+      }
+    }
+
+    protected def handleRootTaskEnd(end: SparkListenerTaskEnd) = {}
+
+    protected def handleRootTaskStart(start: SparkListenerTaskStart) = {}
+
+    private def getSignature(taskInfo: TaskInfo, stageId: Int, stageAttemptId: Int):
+    String = {
+      s"${stageId}:${stageAttemptId}:" +
+        s"${taskInfo.index}:${taskInfo.attemptNumber}-${taskInfo.status}"
+    }
+
+    override def onTaskStart(taskStart: SparkListenerTaskStart): Unit = {
+      val signature = getSignature(taskStart.taskInfo, taskStart.stageId, taskStart.stageAttemptId)
+      logInfo(s"Task started: $signature")
+      if (isRootStageId(taskStart.stageId)) {
+        rootTasksStarted.add(taskStart.taskInfo)
+        handleRootTaskStart(taskStart)
+      }
+    }
+
+    override def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit = {
+      val taskSignature = getSignature(taskEnd.taskInfo, taskEnd.stageId, taskEnd.stageAttemptId)
+      logInfo(s"Task End $taskSignature")
+      tasksFinished.add(taskSignature)
+      if (isRootStageId(taskEnd.stageId)) {
+        rootTasksEnded.add(taskEnd.taskInfo)
+        handleRootTaskEnd(taskEnd)
+      }
+    }
+
+    def getTasksFinished(): Seq[String] = {
+      tasksFinished.asScala.toList
+    }
+
+    def waitForJobDone(): Unit = {
+      eventually(timeout(10.seconds), interval(100.milliseconds)) {
+        assert(jobDone.get(), "Job isn't successfully done yet")
+      }
+    }
+  }
+
+  private def getExecutorToWorkerAssignments: Map[String, WorkerInfo] = {
+    val executorIdToWorkerInfo = mutable.HashMap[String, WorkerInfo]()
+    master.workers.foreach { wi =>
+      assert(wi.executors.size <= 1, "There should be at most one executor per worker")
+      // Cast the executorId to string since the TaskInfo.executorId is a string
+      wi.executors.values.foreach { e =>
+        val executorIdString = e.id.toString
+        val oldWorkerInfo = executorIdToWorkerInfo.put(executorIdString, wi)
+        assert(oldWorkerInfo.isEmpty,
+          s"Executor $executorIdString already present on another worker ${oldWorkerInfo}")
+      }
+    }
+    executorIdToWorkerInfo.toMap
+  }
+
+  private def makeMaster(): Master = {
+    val master = new Master(
+      masterRpcEnv,
+      masterRpcEnv.address,
+      0,
+      masterAndWorkerSecurityManager,
+      masterAndWorkerConf)
+    masterRpcEnv.setupEndpoint(Master.ENDPOINT_NAME, master)
+    master
+  }
+
+  private def createWorkers(numWorkers: Int, cores: Int = 1, memory: Int = 1024): Unit = {
+    val workerRpcEnvs = (0 until numWorkers).map { i =>
+      RpcEnv.create(
+        Worker.SYSTEM_NAME + i,
+        "localhost",
+        0,
+        masterAndWorkerConf,
+        masterAndWorkerSecurityManager)
+    }
+    workers.clear()
+    val rpcAddressToRpcEnv: mutable.HashMap[RpcAddress, RpcEnv] = mutable.HashMap.empty
+    workerRpcEnvs.foreach { rpcEnv =>
+      val workDir = Utils.createTempDir(namePrefix = this.getClass.getSimpleName()).toString
+      val worker = new Worker(rpcEnv, 0, cores, memory, Array(masterRpcEnv.address),
+        Worker.ENDPOINT_NAME, workDir, masterAndWorkerConf, masterAndWorkerSecurityManager)
+      rpcEnv.setupEndpoint(Worker.ENDPOINT_NAME, worker)
+      workers.append(worker)
+      val oldRpcEnv = rpcAddressToRpcEnv.put(rpcEnv.address, rpcEnv)
+      logInfo(s"Created a worker at ${rpcEnv.address} with workdir $workDir")
+      assert(oldRpcEnv.isEmpty, s"Detected duplicate rpcEnv ${oldRpcEnv} for ${rpcEnv.address}")
+    }
+    workerIdToRpcEnvs.clear()
+    // Wait until all workers register with master successfully
+    eventually(timeout(1.minute), interval(1.seconds)) {
+      val workersOnMaster = getMasterState.workers
+      val numWorkersCurrently = workersOnMaster.length
+      logInfo(s"Waiting for $numWorkers workers to come up: So far $numWorkersCurrently")
+      assert(numWorkersCurrently === numWorkers)
+      workersOnMaster.foreach { workerInfo =>
+        val rpcAddress = RpcAddress(workerInfo.host, workerInfo.port)
+        val rpcEnv = rpcAddressToRpcEnv(rpcAddress)
+        assert(rpcEnv != null, s"Cannot find the worker for $rpcAddress")
+        val oldRpcEnv = workerIdToRpcEnvs.put(workerInfo.id, rpcEnv)
+        assert(oldRpcEnv.isEmpty, s"Detected duplicate rpcEnv ${oldRpcEnv} for worker " +
+          s"${workerInfo.id}")
+      }
+    }
+    logInfo(s"Created ${workers.size} workers")
+  }
+
+  private def getMasterState: MasterStateResponse = {
+    master.self.askSync[MasterStateResponse](RequestMasterState)
+  }
+
+  private def getApplications(): Seq[ApplicationInfo] = {
+    getMasterState.activeApps
+  }
+
+  def decommissionWorkerOnMaster(workerInfo: WorkerInfo, reason: String): Unit = {
+    logInfo(s"Trying to decommission worker ${workerInfo.id} for reason `$reason`")
+    master.self.send(DecommissionWorkers(Seq(workerInfo.id)))
+  }
+
+  def killWorkerAfterTimeout(workerInfo: WorkerInfo, secondsToWait: Int): Unit = {
+    val env = workerIdToRpcEnvs(workerInfo.id)
+    Thread.sleep(secondsToWait * 1000L)
+    env.shutdown()
+    env.awaitTermination()
+  }
+
+  def createSparkContext(extraConfs: (String, String)*): SparkContext = {
+    val conf = new SparkConf()
+      .setMaster(masterRpcEnv.address.toSparkURL)
+      .setAppName("test")
+      .setAll(extraConfs)
+    sc = new SparkContext(conf)
+    val appId = sc.applicationId
+    eventually(timeout(1.minute), interval(1.seconds)) {
+      val apps = getApplications()
+      assert(apps.size === 1)
+      assert(apps.head.id === appId)
+      assert(apps.head.getExecutorLimit === Int.MaxValue)
+    }
+    sc
+  }
+
+  private class ExternalShuffleServiceHolder() {
+    // The external shuffle service can start with default configs and not get polluted by the
+    // other configs used in this test.
+    private val transportConf = SparkTransportConf.fromSparkConf(new SparkConf(),
+      "shuffle", numUsableCores = 2)
+    private val rpcHandler = new ExternalBlockHandler(transportConf, null)
+    private val transportContext = new TransportContext(transportConf, rpcHandler)
+    private val server = transportContext.createServer()
+
+    def getPort: Int = server.getPort
+
+    def close(): Unit = {
+      Utils.tryLogNonFatalError {
+        server.close()
+      }
+      Utils.tryLogNonFatalError {
+        rpcHandler.close()
+      }
+      Utils.tryLogNonFatalError {
+        transportContext.close()
+      }
+    }
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/deploy/DeployTestUtils.scala b/core/src/test/scala/org/apache/spark/deploy/DeployTestUtils.scala
index 31f065ec55749..b182b11a0e85e 100644
--- a/core/src/test/scala/org/apache/spark/deploy/DeployTestUtils.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/DeployTestUtils.scala
@@ -102,6 +102,7 @@ private[deploy] object DeployTestUtils {
       createDriverDesc(),
       null,
       "spark://worker",
+      "http://publicAddress:80",
       new SecurityManager(conf))
   }
 
diff --git a/core/src/test/scala/org/apache/spark/deploy/ExternalShuffleServiceMetricsSuite.scala b/core/src/test/scala/org/apache/spark/deploy/ExternalShuffleServiceMetricsSuite.scala
index d681c13337e0d..ea4d252f0dbae 100644
--- a/core/src/test/scala/org/apache/spark/deploy/ExternalShuffleServiceMetricsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/ExternalShuffleServiceMetricsSuite.scala
@@ -61,7 +61,8 @@ class ExternalShuffleServiceMetricsSuite extends SparkFunSuite {
         "registeredExecutorsSize",
         "registerExecutorRequestLatencyMillis",
         "shuffle-server.usedDirectMemory",
-        "shuffle-server.usedHeapMemory")
+        "shuffle-server.usedHeapMemory",
+        "finalizeShuffleMergeLatencyMillis")
     )
   }
 }
diff --git a/core/src/test/scala/org/apache/spark/deploy/IvyTestUtils.scala b/core/src/test/scala/org/apache/spark/deploy/IvyTestUtils.scala
index 42b8cde650390..b986be03e965c 100644
--- a/core/src/test/scala/org/apache/spark/deploy/IvyTestUtils.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/IvyTestUtils.scala
@@ -317,7 +317,7 @@ private[deploy] object IvyTestUtils {
         val rFiles = createRFiles(root, className, artifact.groupId)
         allFiles.append(rFiles: _*)
       }
-      val jarFile = packJar(jarPath, artifact, allFiles, useIvyLayout, withR)
+      val jarFile = packJar(jarPath, artifact, allFiles.toSeq, useIvyLayout, withR)
       assert(jarFile.exists(), "Problem creating Jar file")
       val descriptor = createDescriptor(tempPath, artifact, dependencies, useIvyLayout)
       assert(descriptor.exists(), "Problem creating Pom file")
diff --git a/core/src/test/scala/org/apache/spark/deploy/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/deploy/JsonProtocolSuite.scala
index eeccf56cbf02e..7d3eb7c6b0f6e 100644
--- a/core/src/test/scala/org/apache/spark/deploy/JsonProtocolSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/JsonProtocolSuite.scala
@@ -106,6 +106,9 @@ object JsonConstants {
     """
       |{"id":"id","starttime":3,"name":"name",
       |"cores":0,"user":"%s",
+      |"memoryperexecutor":1234,
+      |"resourcesperexecutor":[{"name":"gpu",
+      |"amount":3},{"name":"fpga","amount":3}],
       |"memoryperslave":1234,
       |"resourcesperslave":[{"name":"gpu",
       |"amount":3},{"name":"fpga","amount":3}],
@@ -125,14 +128,15 @@ object JsonConstants {
       |:["3","4","5"]}},"resourcesused":{"gpu":
       |{"name":"gpu","addresses":[]},"fpga":
       |{"name":"fpga","addresses":[]}},"resourcesfree":
-      |{"gpu":{"name":"gpu","addresses":["2","1","0"]},
-      |"fpga":{"name":"fpga","addresses":["5","4","3"]}},
+      |{"gpu":{"name":"gpu","addresses":["0","1","2"]},
+      |"fpga":{"name":"fpga","addresses":["3","4","5"]}},
       |"state":"ALIVE","lastheartbeat":%d}
     """.format(currTimeInMillis).stripMargin
 
   val appDescJsonStr =
     """
-      |{"name":"name","cores":4,"memoryperslave":1234,"resourcesperslave":[],
+      |{"name":"name","cores":4,"memoryperexecutor":1234,"resourcesperexecutor":[],
+      |"memoryperslave":1234,"resourcesperslave":[],
       |"user":"%s","command":"Command(mainClass,List(arg1, arg2),Map(),List(),List(),List())"}
     """.format(System.getProperty("user.name", "<unknown>")).stripMargin
 
diff --git a/core/src/test/scala/org/apache/spark/deploy/LogUrlsStandaloneSuite.scala b/core/src/test/scala/org/apache/spark/deploy/LogUrlsStandaloneSuite.scala
index 84fc16979925b..5d60aad615583 100644
--- a/core/src/test/scala/org/apache/spark/deploy/LogUrlsStandaloneSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/LogUrlsStandaloneSuite.scala
@@ -25,7 +25,7 @@ import scala.io.Source
 import org.apache.spark.{LocalSparkContext, SparkContext, SparkFunSuite}
 import org.apache.spark.scheduler.{SparkListener, SparkListenerExecutorAdded}
 import org.apache.spark.scheduler.cluster.ExecutorInfo
-import org.apache.spark.util.SparkConfWithEnv
+import org.apache.spark.util.{SparkConfWithEnv, Utils}
 
 class LogUrlsStandaloneSuite extends SparkFunSuite with LocalSparkContext {
 
@@ -43,7 +43,7 @@ class LogUrlsStandaloneSuite extends SparkFunSuite with LocalSparkContext {
       assert(info.logUrlMap.nonEmpty)
       // Browse to each URL to check that it's valid
       info.logUrlMap.foreach { case (logType, logUrl) =>
-        val html = Source.fromURL(logUrl).mkString
+        val html = Utils.tryWithResource(Source.fromURL(logUrl))(_.mkString)
         assert(html.contains(s"$logType log page"))
       }
     }
diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
index fd2d1f56ed9b6..02efcaedd6c33 100644
--- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
@@ -29,8 +29,10 @@ import com.google.common.io.ByteStreams
 import org.apache.commons.io.FileUtils
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.{FileStatus, FSDataInputStream, Path}
-import org.scalatest.{BeforeAndAfterEach, Matchers}
+import org.scalatest.BeforeAndAfterEach
 import org.scalatest.concurrent.{Signaler, ThreadSignaler, TimeLimits}
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 import org.scalatest.time.Span
 import org.scalatest.time.SpanSugar._
 
@@ -45,7 +47,7 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config._
 import org.apache.spark.internal.config.UI._
 import org.apache.spark.launcher.SparkLauncher
-import org.apache.spark.util.{CommandLineUtils, ResetSystemProperties, Utils}
+import org.apache.spark.util.{CommandLineUtils, DependencyUtils, ResetSystemProperties, Utils}
 
 trait TestPrematureExit {
   suite: SparkFunSuite =>
@@ -333,6 +335,43 @@ class SparkSubmitSuite
     sys.props("SPARK_SUBMIT") should be ("true")
   }
 
+  test("SPARK-33530: handles standalone mode with archives") {
+    val clArgs = Seq(
+      "--master", "spark://localhost:1234",
+      "--executor-memory", "5g",
+      "--executor-cores", "5",
+      "--class", "org.SomeClass",
+      "--jars", "one.jar,two.jar,three.jar",
+      "--driver-memory", "4g",
+      "--files", "file1.txt,file2.txt",
+      "--archives", "archive1.zip,archive2.jar",
+      "--num-executors", "6",
+      "--name", "beauty",
+      "--conf", "spark.ui.enabled=false",
+      "thejar.jar",
+      "arg1", "arg2")
+    val appArgs = new SparkSubmitArguments(clArgs)
+    val (childArgs, classpath, conf, mainClass) = submit.prepareSubmitEnvironment(appArgs)
+    val childArgsStr = childArgs.mkString(" ")
+    childArgsStr should include ("arg1 arg2")
+    mainClass should be ("org.SomeClass")
+
+    classpath(0) should endWith ("thejar.jar")
+    classpath(1) should endWith ("one.jar")
+    classpath(2) should endWith ("two.jar")
+    classpath(3) should endWith ("three.jar")
+
+    conf.get("spark.executor.memory") should be ("5g")
+    conf.get("spark.driver.memory") should be ("4g")
+    conf.get("spark.executor.cores") should be ("5")
+    conf.get("spark.jars") should include regex (".*one.jar,.*two.jar,.*three.jar")
+    conf.get("spark.files") should include regex (".*file1.txt,.*file2.txt")
+    conf.get("spark.archives") should include regex (".*archive1.zip,.*archive2.jar")
+    conf.get("spark.app.name") should be ("beauty")
+    conf.get(UI_ENABLED) should be (false)
+    sys.props("SPARK_SUBMIT") should be ("true")
+  }
+
   test("handles standalone cluster mode") {
     testStandaloneCluster(useRest = true)
   }
@@ -568,7 +607,8 @@ class SparkSubmitSuite
       }
     }
 
-    val clArgs2 = Seq("--class", "org.SomeClass", "thejar.jar")
+    val dummyJarFile = TestUtils.createJarWithClasses(Seq.empty)
+    val clArgs2 = Seq("--class", "org.SomeClass", dummyJarFile.toString)
     val appArgs2 = new SparkSubmitArguments(clArgs2)
     val (_, _, conf2, _) = submit.prepareSubmitEnvironment(appArgs2)
     assert(!conf2.contains(UI_SHOW_CONSOLE_PROGRESS))
@@ -1066,8 +1106,8 @@ class SparkSubmitSuite
 
     // The path and filename are preserved.
     assert(outputUri.getPath.endsWith(new Path(sourceUri).getName))
-    assert(FileUtils.readFileToString(new File(outputUri.getPath)) ===
-      FileUtils.readFileToString(new File(sourceUri.getPath)))
+    assert(FileUtils.readFileToString(new File(outputUri.getPath), StandardCharsets.UTF_8) ===
+      FileUtils.readFileToString(new File(sourceUri.getPath), StandardCharsets.UTF_8))
   }
 
   private def deleteTempOutputFile(outputPath: String): Unit = {
@@ -1081,8 +1121,7 @@ class SparkSubmitSuite
     val sparkConf = new SparkConf(false)
     intercept[IOException] {
       DependencyUtils.downloadFile(
-        "abc:/my/file", Utils.createTempDir(), sparkConf, new Configuration(),
-        new SecurityManager(sparkConf))
+        "abc:/my/file", Utils.createTempDir(), sparkConf, new Configuration())
     }
   }
 
@@ -1092,19 +1131,17 @@ class SparkSubmitSuite
     val tmpDir = Utils.createTempDir()
     updateConfWithFakeS3Fs(hadoopConf)
     intercept[FileNotFoundException] {
-      DependencyUtils.downloadFile("s3a:/no/such/file", tmpDir, sparkConf, hadoopConf,
-        new SecurityManager(sparkConf))
+      DependencyUtils.downloadFile("s3a:/no/such/file", tmpDir, sparkConf, hadoopConf)
     }
   }
 
   test("downloadFile does not download local file") {
     val sparkConf = new SparkConf(false)
-    val secMgr = new SecurityManager(sparkConf)
     // empty path is considered as local file.
     val tmpDir = Files.createTempDirectory("tmp").toFile
-    assert(DependencyUtils.downloadFile("", tmpDir, sparkConf, new Configuration(), secMgr) === "")
-    assert(DependencyUtils.downloadFile("/local/file", tmpDir, sparkConf, new Configuration(),
-      secMgr) === "/local/file")
+    assert(DependencyUtils.downloadFile("", tmpDir, sparkConf, new Configuration()) === "")
+    assert(DependencyUtils.downloadFile(
+      "/local/file", tmpDir, sparkConf, new Configuration()) === "/local/file")
   }
 
   test("download one file to local") {
@@ -1112,13 +1149,12 @@ class SparkSubmitSuite
     val jarFile = File.createTempFile("test", ".jar")
     jarFile.deleteOnExit()
     val content = "hello, world"
-    FileUtils.write(jarFile, content)
+    FileUtils.write(jarFile, content, StandardCharsets.UTF_8)
     val hadoopConf = new Configuration()
     val tmpDir = Files.createTempDirectory("tmp").toFile
     updateConfWithFakeS3Fs(hadoopConf)
     val sourcePath = s"s3a://${jarFile.toURI.getPath}"
-    val outputPath = DependencyUtils.downloadFile(sourcePath, tmpDir, sparkConf, hadoopConf,
-      new SecurityManager(sparkConf))
+    val outputPath = DependencyUtils.downloadFile(sourcePath, tmpDir, sparkConf, hadoopConf)
     checkDownloadedFile(sourcePath, outputPath)
     deleteTempOutputFile(outputPath)
   }
@@ -1128,14 +1164,13 @@ class SparkSubmitSuite
     val jarFile = File.createTempFile("test", ".jar")
     jarFile.deleteOnExit()
     val content = "hello, world"
-    FileUtils.write(jarFile, content)
+    FileUtils.write(jarFile, content, StandardCharsets.UTF_8)
     val hadoopConf = new Configuration()
     val tmpDir = Files.createTempDirectory("tmp").toFile
     updateConfWithFakeS3Fs(hadoopConf)
     val sourcePaths = Seq("/local/file", s"s3a://${jarFile.toURI.getPath}")
     val outputPaths = DependencyUtils
-      .downloadFileList(sourcePaths.mkString(","), tmpDir, sparkConf, hadoopConf,
-        new SecurityManager(sparkConf))
+      .downloadFileList(sourcePaths.mkString(","), tmpDir, sparkConf, hadoopConf)
       .split(",")
 
     assert(outputPaths.length === sourcePaths.length)
@@ -1149,7 +1184,6 @@ class SparkSubmitSuite
     val fs = File.separator
     val sparkConf = new SparkConf(false)
     val hadoopConf = new Configuration()
-    val secMgr = new SecurityManager(sparkConf)
 
     val appJarName = "myApp.jar"
     val jar1Name = "myJar1.jar"
@@ -1157,8 +1191,7 @@ class SparkSubmitSuite
     val userJar = s"file:/path${fs}to${fs}app${fs}jar$fs$appJarName"
     val jars = s"file:/$jar1Name,file:/$appJarName,file:/$jar2Name"
 
-    val resolvedJars = DependencyUtils
-      .resolveAndDownloadJars(jars, userJar, sparkConf, hadoopConf, secMgr)
+    val resolvedJars = DependencyUtils.resolveAndDownloadJars(jars, userJar, sparkConf, hadoopConf)
 
     assert(!resolvedJars.contains(appJarName))
     assert(resolvedJars.contains(jar1Name) && resolvedJars.contains(jar2Name))
@@ -1210,17 +1243,97 @@ class SparkSubmitSuite
     testRemoteResources(enableHttpFs = true)
   }
 
-  test("force download from blacklisted schemes") {
-    testRemoteResources(enableHttpFs = true, blacklistSchemes = Seq("http"))
+  test("force download from forced schemes") {
+    testRemoteResources(enableHttpFs = true, forceDownloadSchemes = Seq("http"))
   }
 
   test("force download for all the schemes") {
-    testRemoteResources(enableHttpFs = true, blacklistSchemes = Seq("*"))
+    testRemoteResources(enableHttpFs = true, forceDownloadSchemes = Seq("*"))
+  }
+
+  test("SPARK-32119: Jars and files should be loaded when Executors launch for plugins") {
+    val tempDir = Utils.createTempDir()
+    val tempFileName = "test.txt"
+    val tempFile = new File(tempDir, tempFileName)
+
+    // scalastyle:off println
+    Utils.tryWithResource {
+      new PrintWriter(tempFile)
+    } { writer =>
+      writer.println("SparkPluginTest")
+    }
+    // scalastyle:on println
+
+    val sparkPluginCodeBody =
+      """
+        |@Override
+        |public org.apache.spark.api.plugin.ExecutorPlugin executorPlugin() {
+        |  return new TestExecutorPlugin();
+        |}
+        |
+        |@Override
+        |public org.apache.spark.api.plugin.DriverPlugin driverPlugin() { return null; }
+      """.stripMargin
+    val executorPluginCodeBody =
+      s"""
+        |@Override
+        |public void init(
+        |    org.apache.spark.api.plugin.PluginContext ctx,
+        |    java.util.Map<String, String> extraConf) {
+        |  String str = null;
+        |  try (java.io.BufferedReader reader =
+        |    new java.io.BufferedReader(new java.io.InputStreamReader(
+        |      new java.io.FileInputStream("$tempFileName")))) {
+        |    str = reader.readLine();
+        |  } catch (java.io.IOException e) {
+        |    throw new RuntimeException(e);
+        |  } finally {
+        |    assert str == "SparkPluginTest";
+        |  }
+        |}
+      """.stripMargin
+
+    val compiledExecutorPlugin = TestUtils.createCompiledClass(
+      "TestExecutorPlugin",
+      tempDir,
+      "",
+      null,
+      Seq.empty,
+      Seq("org.apache.spark.api.plugin.ExecutorPlugin"),
+      executorPluginCodeBody)
+
+    val thisClassPath =
+      sys.props("java.class.path").split(File.pathSeparator).map(p => new File(p).toURI.toURL)
+    val compiledSparkPlugin = TestUtils.createCompiledClass(
+      "TestSparkPlugin",
+      tempDir,
+      "",
+      null,
+      Seq(tempDir.toURI.toURL) ++ thisClassPath,
+      Seq("org.apache.spark.api.plugin.SparkPlugin"),
+      sparkPluginCodeBody)
+
+    val jarUrl = TestUtils.createJar(
+      Seq(compiledSparkPlugin, compiledExecutorPlugin),
+      new File(tempDir, "testplugin.jar"))
+
+    val unusedJar = TestUtils.createJarWithClasses(Seq.empty)
+    val unusedFile = Files.createTempFile(tempDir.toPath, "unused", null)
+    val args = Seq(
+      "--class", SimpleApplicationTest.getClass.getName.stripSuffix("$"),
+      "--name", "testApp",
+      "--master", "local-cluster[1,1,1024]",
+      "--conf", "spark.plugins=TestSparkPlugin",
+      "--conf", "spark.ui.enabled=false",
+      "--jars", jarUrl.toString + "," + unusedJar.toString,
+      "--files", tempFile.toString + "," + unusedFile.toString,
+      unusedJar.toString)
+    runSparkSubmit(args)
   }
 
   private def testRemoteResources(
       enableHttpFs: Boolean,
-      blacklistSchemes: Seq[String] = Nil): Unit = {
+      forceDownloadSchemes: Seq[String] = Nil): Unit = {
     val hadoopConf = new Configuration()
     updateConfWithFakeS3Fs(hadoopConf)
     if (enableHttpFs) {
@@ -1237,8 +1350,8 @@ class SparkSubmitSuite
     val tmpHttpJar = TestUtils.createJarWithFiles(Map("test.resource" -> "USER"), tmpDir)
     val tmpHttpJarPath = s"http://${new File(tmpHttpJar.toURI).getAbsolutePath}"
 
-    val forceDownloadArgs = if (blacklistSchemes.nonEmpty) {
-      Seq("--conf", s"spark.yarn.dist.forceDownloadSchemes=${blacklistSchemes.mkString(",")}")
+    val forceDownloadArgs = if (forceDownloadSchemes.nonEmpty) {
+      Seq("--conf", s"spark.yarn.dist.forceDownloadSchemes=${forceDownloadSchemes.mkString(",")}")
     } else {
       Nil
     }
@@ -1256,19 +1369,19 @@ class SparkSubmitSuite
 
     val jars = conf.get("spark.yarn.dist.jars").split(",").toSet
 
-    def isSchemeBlacklisted(scheme: String) = {
-      blacklistSchemes.contains("*") || blacklistSchemes.contains(scheme)
+    def isSchemeForcedDownload(scheme: String) = {
+      forceDownloadSchemes.contains("*") || forceDownloadSchemes.contains(scheme)
     }
 
-    if (!isSchemeBlacklisted("s3")) {
+    if (!isSchemeForcedDownload("s3")) {
       assert(jars.contains(tmpS3JarPath))
     }
 
-    if (enableHttpFs && blacklistSchemes.isEmpty) {
+    if (enableHttpFs && forceDownloadSchemes.isEmpty) {
       // If Http FS is supported by yarn service, the URI of remote http resource should
       // still be remote.
       assert(jars.contains(tmpHttpJarPath))
-    } else if (!enableHttpFs || isSchemeBlacklisted("http")) {
+    } else if (!enableHttpFs || isSchemeForcedDownload("http")) {
       // If Http FS is not supported by yarn service, or http scheme is configured to be force
       // downloading, the URI of remote http resource should be changed to a local one.
       val jarName = new File(tmpHttpJar.toURI).getName
diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala
index 31e6c730eadc0..b8ad85b4b953b 100644
--- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala
@@ -18,11 +18,13 @@
 package org.apache.spark.deploy
 
 import java.io.{File, OutputStream, PrintStream}
+import java.net.URI
 import java.nio.charset.StandardCharsets
+import java.nio.file.{Files, Paths}
 
+import scala.collection.JavaConverters._
 import scala.collection.mutable.ArrayBuffer
 
-import com.google.common.io.Files
 import org.apache.ivy.core.module.descriptor.MDArtifact
 import org.apache.ivy.core.settings.IvySettings
 import org.apache.ivy.plugins.resolver.{AbstractResolver, ChainResolver, FileSystemResolver, IBiblioResolver}
@@ -79,7 +81,7 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll {
 
   test("create additional resolvers") {
     val repos = "a/1,b/2,c/3"
-    val settings = SparkSubmitUtils.buildIvySettings(Option(repos), None)
+    val settings = SparkSubmitUtils.buildIvySettings(Option(repos), Some(tempIvyPath))
     val resolver = settings.getDefaultResolver.asInstanceOf[ChainResolver]
     assert(resolver.getResolvers.size() === 4)
     val expected = repos.split(",").map(r => s"$r/")
@@ -123,20 +125,17 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll {
   test("ivy path works correctly") {
     val md = SparkSubmitUtils.getModuleDescriptor
     val artifacts = for (i <- 0 until 3) yield new MDArtifact(md, s"jar-$i", "jar", "jar")
-    var jPaths = SparkSubmitUtils.resolveDependencyPaths(artifacts.toArray, new File(tempIvyPath))
-    for (i <- 0 until 3) {
-      val index = jPaths.indexOf(tempIvyPath)
-      assert(index >= 0)
-      jPaths = jPaths.substring(index + tempIvyPath.length)
-    }
+    val jPaths = SparkSubmitUtils.resolveDependencyPaths(artifacts.toArray, new File(tempIvyPath))
+    assert(jPaths.count(_.startsWith(tempIvyPath)) >= 3)
     val main = MavenCoordinate("my.awesome.lib", "mylib", "0.1")
     IvyTestUtils.withRepository(main, None, None) { repo =>
       // end to end
       val jarPath = SparkSubmitUtils.resolveMavenCoordinates(
         main.toString,
-        SparkSubmitUtils.buildIvySettings(Option(repo), Option(tempIvyPath)),
+        SparkSubmitUtils.buildIvySettings(Option(repo), Some(tempIvyPath)),
+        transitive = true,
         isTest = true)
-      assert(jarPath.indexOf(tempIvyPath) >= 0, "should use non-default ivy path")
+      assert(jarPath.forall(_.indexOf(tempIvyPath) >= 0), "should use non-default ivy path")
     }
   }
 
@@ -147,10 +146,11 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll {
     IvyTestUtils.withRepository(main, Some(dep), Some(SparkSubmitUtils.m2Path)) { repo =>
       val jarPath = SparkSubmitUtils.resolveMavenCoordinates(
         main.toString,
-        SparkSubmitUtils.buildIvySettings(None, None),
+        SparkSubmitUtils.buildIvySettings(None, Some(tempIvyPath)),
+        transitive = true,
         isTest = true)
-      assert(jarPath.indexOf("mylib") >= 0, "should find artifact")
-      assert(jarPath.indexOf("mydep") >= 0, "should find dependency")
+      assert(jarPath.exists(_.indexOf("mylib") >= 0), "should find artifact")
+      assert(jarPath.exists(_.indexOf("mydep") >= 0), "should find dependency")
     }
     // Local Ivy Repository
     val settings = new IvySettings
@@ -158,10 +158,11 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll {
     IvyTestUtils.withRepository(main, Some(dep), Some(ivyLocal), useIvyLayout = true) { repo =>
       val jarPath = SparkSubmitUtils.resolveMavenCoordinates(
         main.toString,
-        SparkSubmitUtils.buildIvySettings(None, None),
+        SparkSubmitUtils.buildIvySettings(None, Some(tempIvyPath)),
+        transitive = true,
         isTest = true)
-      assert(jarPath.indexOf("mylib") >= 0, "should find artifact")
-      assert(jarPath.indexOf("mydep") >= 0, "should find dependency")
+      assert(jarPath.exists(_.indexOf("mylib") >= 0), "should find artifact")
+      assert(jarPath.exists(_.indexOf("mydep") >= 0), "should find dependency")
     }
     // Local ivy repository with modified home
     val dummyIvyLocal = new File(tempIvyPath, "local" + File.separator)
@@ -171,10 +172,11 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll {
       val jarPath = SparkSubmitUtils.resolveMavenCoordinates(
         main.toString,
         SparkSubmitUtils.buildIvySettings(None, Some(tempIvyPath)),
+        transitive = true,
         isTest = true)
-      assert(jarPath.indexOf("mylib") >= 0, "should find artifact")
-      assert(jarPath.indexOf(tempIvyPath) >= 0, "should be in new ivy path")
-      assert(jarPath.indexOf("mydep") >= 0, "should find dependency")
+      assert(jarPath.exists(_.indexOf("mylib") >= 0), "should find artifact")
+      assert(jarPath.forall(_.indexOf(tempIvyPath) >= 0), "should be in new ivy path")
+      assert(jarPath.exists(_.indexOf("mydep") >= 0), "should find dependency")
     }
   }
 
@@ -182,7 +184,8 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll {
     intercept[RuntimeException] {
       SparkSubmitUtils.resolveMavenCoordinates(
       "a:b:c",
-      SparkSubmitUtils.buildIvySettings(None, None),
+      SparkSubmitUtils.buildIvySettings(None, Some(tempIvyPath)),
+        transitive = true,
       isTest = true)
     }
   }
@@ -194,16 +197,18 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll {
 
     val path = SparkSubmitUtils.resolveMavenCoordinates(
       coordinates,
-      SparkSubmitUtils.buildIvySettings(None, None),
+      SparkSubmitUtils.buildIvySettings(None, Some(tempIvyPath)),
+      transitive = true,
       isTest = true)
-    assert(path === "", "should return empty path")
+    assert(path.isEmpty, "should return empty path")
     val main = MavenCoordinate("org.apache.spark", "spark-streaming-kafka-assembly_2.12", "1.2.0")
     IvyTestUtils.withRepository(main, None, None) { repo =>
       val files = SparkSubmitUtils.resolveMavenCoordinates(
         coordinates + "," + main.toString,
-        SparkSubmitUtils.buildIvySettings(Some(repo), None),
+        SparkSubmitUtils.buildIvySettings(Some(repo), Some(tempIvyPath)),
+        transitive = true,
         isTest = true)
-      assert(files.indexOf(main.artifactId) >= 0, "Did not return artifact")
+      assert(files.forall(_.indexOf(main.artifactId) >= 0), "Did not return artifact")
     }
   }
 
@@ -213,11 +218,12 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll {
     IvyTestUtils.withRepository(main, Some(dep), None) { repo =>
       val files = SparkSubmitUtils.resolveMavenCoordinates(
         main.toString,
-        SparkSubmitUtils.buildIvySettings(Some(repo), None),
-        Seq("my.great.dep:mydep"),
+        SparkSubmitUtils.buildIvySettings(Some(repo), Some(tempIvyPath)),
+        exclusions = Seq("my.great.dep:mydep"),
+        transitive = true,
         isTest = true)
-      assert(files.indexOf(main.artifactId) >= 0, "Did not return artifact")
-      assert(files.indexOf("my.great.dep") < 0, "Returned excluded artifact")
+      assert(files.forall(_.indexOf(main.artifactId) >= 0), "Did not return artifact")
+      assert(files.forall(_.indexOf("my.great.dep") < 0), "Returned excluded artifact")
     }
   }
 
@@ -241,8 +247,8 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll {
          |</ivysettings>
          |""".stripMargin
 
-    val settingsFile = new File(tempIvyPath, "ivysettings.xml")
-    Files.write(settingsText, settingsFile, StandardCharsets.UTF_8)
+    val settingsFile = Paths.get(tempIvyPath, "ivysettings.xml")
+    Files.write(settingsFile, settingsText.getBytes(StandardCharsets.UTF_8))
     val settings = SparkSubmitUtils.loadIvySettings(settingsFile.toString, None, None)
     settings.setDefaultIvyUserDir(new File(tempIvyPath))  // NOTE - can't set this through file
 
@@ -250,10 +256,11 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll {
     testUtilSettings.setDefaultIvyUserDir(new File(tempIvyPath))
     IvyTestUtils.withRepository(main, Some(dep), Some(dummyIvyLocal), useIvyLayout = true,
       ivySettings = testUtilSettings) { repo =>
-      val jarPath = SparkSubmitUtils.resolveMavenCoordinates(main.toString, settings, isTest = true)
-      assert(jarPath.indexOf("mylib") >= 0, "should find artifact")
-      assert(jarPath.indexOf(tempIvyPath) >= 0, "should be in new ivy path")
-      assert(jarPath.indexOf("mydep") >= 0, "should find dependency")
+      val jarPath = SparkSubmitUtils.resolveMavenCoordinates(main.toString, settings,
+        transitive = true, isTest = true)
+      assert(jarPath.exists(_.indexOf("mylib") >= 0), "should find artifact")
+      assert(jarPath.forall(_.indexOf(tempIvyPath) >= 0), "should be in new ivy path")
+      assert(jarPath.exists(_.indexOf("mydep") >= 0), "should find dependency")
     }
   }
 
@@ -265,10 +272,36 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll {
       val jarPath = SparkSubmitUtils.resolveMavenCoordinates(
         main.toString,
         ivySettings,
+        transitive = true,
         isTest = true)
       val r = """.*org.apache.spark-spark-submit-parent-.*""".r
       assert(!ivySettings.getDefaultCache.listFiles.map(_.getName)
         .exists(r.findFirstIn(_).isDefined), "resolution files should be cleaned")
     }
   }
+
+  test("SPARK-34624: should ignore non-jar dependencies") {
+    val main = MavenCoordinate("my.great.lib", "mylib", "0.1")
+    val dep = "my.great.dep:mydep:0.1"
+
+    IvyTestUtils.withRepository(main, Some(dep), None) { repo =>
+      // IvyTestUtils.withRepository does not have an easy way for creating non-jar dependencies
+      // So we let it create the jar dependency in `mylib-0.1.pom`, and then modify the pom
+      // to change the type of the transitive to `pom`
+      val mainPom = Paths.get(URI.create(repo)).resolve("my/great/lib/mylib/0.1/mylib-0.1.pom")
+      val lines = Files.lines(mainPom).iterator.asScala
+        .map(l => if (l.trim == "<artifactId>mydep</artifactId>") s"$l<type>pom</type>" else l)
+        .toList
+      Files.write(mainPom, lines.asJava)
+
+      val ivySettings = SparkSubmitUtils.buildIvySettings(Some(repo), Some(tempIvyPath))
+      val jarPath = SparkSubmitUtils.resolveMavenCoordinates(
+        main.toString,
+        ivySettings,
+        transitive = true,
+        isTest = true)
+      assert(!jarPath.exists(_.indexOf("mydep") >= 0), "should not find pom dependency." +
+        s" Resolved jars are: $jarPath")
+    }
+  }
 }
diff --git a/core/src/test/scala/org/apache/spark/deploy/StandaloneDynamicAllocationSuite.scala b/core/src/test/scala/org/apache/spark/deploy/StandaloneDynamicAllocationSuite.scala
index 57cbda3c0620d..e47181719a9db 100644
--- a/core/src/test/scala/org/apache/spark/deploy/StandaloneDynamicAllocationSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/StandaloneDynamicAllocationSuite.scala
@@ -21,7 +21,7 @@ import scala.collection.mutable
 import scala.concurrent.duration._
 
 import org.mockito.ArgumentMatchers.any
-import org.mockito.Mockito.{mock, verify, when}
+import org.mockito.Mockito.{mock, when}
 import org.scalatest.{BeforeAndAfterAll, PrivateMethodTester}
 import org.scalatest.concurrent.Eventually._
 
@@ -497,19 +497,19 @@ class StandaloneDynamicAllocationSuite
     }
   }
 
-  test("executor registration on a blacklisted host must fail") {
+  test("executor registration on a excluded host must fail") {
     // The context isn't really used by the test, but it helps with creating a test scheduler,
     // since CoarseGrainedSchedulerBackend makes a lot of calls to the context instance.
-    sc = new SparkContext(appConf.set(config.BLACKLIST_ENABLED.key, "true"))
+    sc = new SparkContext(appConf.set(config.EXCLUDE_ON_FAILURE_ENABLED.key, "true"))
 
     val endpointRef = mock(classOf[RpcEndpointRef])
     val mockAddress = mock(classOf[RpcAddress])
     when(endpointRef.address).thenReturn(mockAddress)
-    val message = RegisterExecutor("one", endpointRef, "blacklisted-host", 10, Map.empty,
+    val message = RegisterExecutor("one", endpointRef, "excluded-host", 10, Map.empty,
       Map.empty, Map.empty, ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID)
 
     val taskScheduler = mock(classOf[TaskSchedulerImpl])
-    when(taskScheduler.nodeBlacklist()).thenReturn(Set("blacklisted-host"))
+    when(taskScheduler.excludedNodes()).thenReturn(Set("excluded-host"))
     when(taskScheduler.resourceOffers(any(), any[Boolean])).thenReturn(Nil)
     when(taskScheduler.sc).thenReturn(sc)
 
@@ -545,7 +545,7 @@ class StandaloneDynamicAllocationSuite
       // will not timeout anything related to executors.
       .set(config.Network.NETWORK_TIMEOUT.key, "2h")
       .set(config.EXECUTOR_HEARTBEAT_INTERVAL.key, "1h")
-      .set(config.STORAGE_BLOCKMANAGER_SLAVE_TIMEOUT.key, "1h")
+      .set(config.STORAGE_BLOCKMANAGER_HEARTBEAT_TIMEOUT.key, "1h")
   }
 
   /** Make a master to which our application will send executor requests. */
diff --git a/core/src/test/scala/org/apache/spark/deploy/client/AppClientSuite.scala b/core/src/test/scala/org/apache/spark/deploy/client/AppClientSuite.scala
index a3e39d7f53728..93c0aa000e207 100644
--- a/core/src/test/scala/org/apache/spark/deploy/client/AppClientSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/client/AppClientSuite.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.deploy.client
 
 import java.io.Closeable
-import java.util.concurrent.ConcurrentLinkedQueue
+import java.util.concurrent.{ConcurrentHashMap, ConcurrentLinkedQueue}
 
 import scala.concurrent.duration._
 
@@ -27,11 +27,12 @@ import org.scalatest.concurrent.{Eventually, ScalaFutures}
 
 import org.apache.spark._
 import org.apache.spark.deploy.{ApplicationDescription, Command}
-import org.apache.spark.deploy.DeployMessages.{MasterStateResponse, RequestMasterState}
+import org.apache.spark.deploy.DeployMessages.{MasterStateResponse, RequestMasterState, WorkerDecommissioning}
 import org.apache.spark.deploy.master.{ApplicationInfo, Master}
 import org.apache.spark.deploy.worker.Worker
 import org.apache.spark.internal.{config, Logging}
 import org.apache.spark.rpc.RpcEnv
+import org.apache.spark.scheduler.ExecutorDecommissionInfo
 import org.apache.spark.util.Utils
 
 /**
@@ -58,7 +59,7 @@ class AppClientSuite
    */
   override def beforeAll(): Unit = {
     super.beforeAll()
-    conf = new SparkConf().set(config.Worker.WORKER_DECOMMISSION_ENABLED.key, "true")
+    conf = new SparkConf().set(config.DECOMMISSION_ENABLED.key, "true")
     securityManager = new SecurityManager(conf)
     masterRpcEnv = RpcEnv.create(Master.SYSTEM_NAME, "localhost", 0, conf, securityManager)
     workerRpcEnvs = (0 until numWorkers).map { i =>
@@ -121,12 +122,19 @@ class AppClientSuite
 
       // Send a decommission self to all the workers
       // Note: normally the worker would send this on their own.
-      workers.foreach(worker => worker.decommissionSelf())
+      workers.foreach { worker =>
+        worker.decommissionSelf()
+        // send the notice to Master to tell the decommission of Workers
+        master.self.send(WorkerDecommissioning(worker.workerId, worker.self))
+      }
 
       // Decommissioning is async.
       eventually(timeout(1.seconds), interval(10.millis)) {
         // We only record decommissioning for the executor we've requested
-        assert(ci.listener.execDecommissionedList.size === 1)
+        assert(ci.listener.execDecommissionedMap.size === 1)
+        val decommissionInfo = ci.listener.execDecommissionedMap.get(executorId)
+        assert(decommissionInfo != null && decommissionInfo.workerHost.isDefined,
+          s"$executorId should have been decommissioned along with its worker")
       }
 
       // Send request to kill executor, verify request was made
@@ -215,7 +223,7 @@ class AppClientSuite
     val deadReasonList = new ConcurrentLinkedQueue[String]()
     val execAddedList = new ConcurrentLinkedQueue[String]()
     val execRemovedList = new ConcurrentLinkedQueue[String]()
-    val execDecommissionedList = new ConcurrentLinkedQueue[String]()
+    val execDecommissionedMap = new ConcurrentHashMap[String, ExecutorDecommissionInfo]()
 
     def connected(id: String): Unit = {
       connectedIdList.add(id)
@@ -241,12 +249,13 @@ class AppClientSuite
     }
 
     def executorRemoved(
-        id: String, message: String, exitStatus: Option[Int], workerLost: Boolean): Unit = {
+        id: String, message: String, exitStatus: Option[Int], workerHost: Option[String]): Unit = {
       execRemovedList.add(id)
     }
 
-    def executorDecommissioned(id: String, message: String): Unit = {
-      execDecommissionedList.add(id)
+    def executorDecommissioned(id: String, decommissionInfo: ExecutorDecommissionInfo): Unit = {
+      val previousDecommissionInfo = execDecommissionedMap.putIfAbsent(id, decommissionInfo)
+      assert(previousDecommissionInfo === null, s"Expected no previous decommission info for $id")
     }
 
     def workerRemoved(workerId: String, host: String, message: String): Unit = {}
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala
index 48bd088d07ff9..7cf533e58b658 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala
@@ -27,7 +27,8 @@ import org.eclipse.jetty.servlet.ServletContextHandler
 import org.mockito.ArgumentMatchers.any
 import org.mockito.Mockito._
 import org.mockito.invocation.InvocationOnMock
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 import org.scalatestplus.mockito.MockitoSugar
 
 import org.apache.spark.SparkFunSuite
@@ -54,7 +55,7 @@ class ApplicationCacheSuite extends SparkFunSuite with Logging with MockitoSugar
     var getAppUICount = 0L
     var attachCount = 0L
     var detachCount = 0L
-    var updateProbeCount = 0L
+    val updateProbeCount = 0L
 
     override def getAppUI(appId: String, attemptId: Option[String]): Option[LoadedAppUI] = {
       logDebug(s"getAppUI($appId, $attemptId)")
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/BasicEventFilterSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/BasicEventFilterSuite.scala
index 2da40dccba53e..5d40a0610eb6c 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/BasicEventFilterSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/BasicEventFilterSuite.scala
@@ -135,6 +135,8 @@ class BasicEventFilterSuite extends SparkFunSuite {
       SparkListenerStageExecutorMetrics(1.toString, 0, 0, new ExecutorMetrics)))
     assert(Some(false) === acceptFn(SparkListenerExecutorBlacklisted(0, 1.toString, 1)))
     assert(Some(false) === acceptFn(SparkListenerExecutorUnblacklisted(0, 1.toString)))
+    assert(Some(false) === acceptFn(SparkListenerExecutorExcluded(0, 1.toString, 1)))
+    assert(Some(false) === acceptFn(SparkListenerExecutorUnexcluded(0, 1.toString)))
     assert(Some(false) === acceptFn(createExecutorRemovedEvent(1)))
     val bmId = BlockManagerId(1.toString, "host1", 1)
     assert(Some(false) === acceptFn(SparkListenerBlockManagerAdded(0, bmId, 1)))
@@ -148,6 +150,10 @@ class BasicEventFilterSuite extends SparkFunSuite {
       SparkListenerStageExecutorMetrics(2.toString, 0, 0, new ExecutorMetrics)))
     assert(Some(true) === acceptFn(SparkListenerExecutorBlacklisted(0, 2.toString, 1)))
     assert(Some(true) === acceptFn(SparkListenerExecutorUnblacklisted(0, 2.toString)))
+    assert(None === acceptFn(SparkListenerNodeBlacklisted(0, "host1", 1)))
+    assert(None === acceptFn(SparkListenerNodeUnblacklisted(0, "host1")))
+    assert(Some(true) === acceptFn(SparkListenerExecutorExcluded(0, 2.toString, 1)))
+    assert(Some(true) === acceptFn(SparkListenerExecutorUnexcluded(0, 2.toString)))
     assert(Some(true) === acceptFn(createExecutorRemovedEvent(2)))
     val bmId2 = BlockManagerId(2.toString, "host1", 1)
     assert(Some(true) === acceptFn(SparkListenerBlockManagerAdded(0, bmId2, 1)))
@@ -164,8 +170,8 @@ class BasicEventFilterSuite extends SparkFunSuite {
     assert(None === acceptFn(SparkListenerEnvironmentUpdate(Map.empty)))
     assert(None === acceptFn(SparkListenerApplicationStart("1", Some("1"), 0, "user", None)))
     assert(None === acceptFn(SparkListenerApplicationEnd(1)))
-    assert(None === acceptFn(SparkListenerNodeBlacklisted(0, "host1", 1)))
-    assert(None === acceptFn(SparkListenerNodeUnblacklisted(0, "host1")))
+    assert(None === acceptFn(SparkListenerNodeExcluded(0, "host1", 1)))
+    assert(None === acceptFn(SparkListenerNodeUnexcluded(0, "host1")))
     assert(None === acceptFn(SparkListenerLogStart("testVersion")))
   }
 
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/ChromeUIHistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/ChromeUIHistoryServerSuite.scala
new file mode 100644
index 0000000000000..1fa2d0ab882c9
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/deploy/history/ChromeUIHistoryServerSuite.scala
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.deploy.history
+
+import org.openqa.selenium.WebDriver
+import org.openqa.selenium.chrome.{ChromeDriver, ChromeOptions}
+
+import org.apache.spark.tags.ChromeUITest
+
+/**
+ * Tests for HistoryServer with Chrome.
+ */
+@ChromeUITest
+class ChromeUIHistoryServerSuite
+  extends RealBrowserUIHistoryServerSuite("webdriver.chrome.driver") {
+
+  override var webDriver: WebDriver = _
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    val chromeOptions = new ChromeOptions
+    chromeOptions.addArguments("--headless", "--disable-gpu")
+    webDriver = new ChromeDriver(chromeOptions)
+  }
+
+  override def afterAll(): Unit = {
+    try {
+      if (webDriver != null) {
+        webDriver.quit()
+      }
+    } finally {
+      super.afterAll()
+    }
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileCompactorSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileCompactorSuite.scala
index 2a914023ec821..7d07af4d7246b 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileCompactorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileCompactorSuite.scala
@@ -23,10 +23,9 @@ import scala.io.{Codec, Source}
 import org.apache.hadoop.fs.{FileStatus, FileSystem, Path}
 import org.json4s.jackson.JsonMethods.parse
 
-import org.apache.spark.{SparkConf, SparkFunSuite, Success}
+import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.deploy.SparkHadoopUtil
 import org.apache.spark.deploy.history.EventLogTestHelper.writeEventsToRollingWriter
-import org.apache.spark.executor.ExecutorMetrics
 import org.apache.spark.scheduler._
 import org.apache.spark.scheduler.cluster.ExecutorInfo
 import org.apache.spark.status.ListenerEventsTestHelper._
@@ -219,10 +218,10 @@ class EventLogFileCompactorSuite extends SparkFunSuite {
       override def acceptFn(): PartialFunction[SparkListenerEvent, Boolean] = {
         case _: SparkListenerApplicationEnd => true
         case _: SparkListenerEnvironmentUpdate => true
-        case _: SparkListenerNodeBlacklisted => true
+        case _: SparkListenerNodeExcluded => true
         case _: SparkListenerBlockManagerAdded => false
         case _: SparkListenerApplicationStart => false
-        case _: SparkListenerNodeUnblacklisted => false
+        case _: SparkListenerNodeUnexcluded => false
       }
 
       override def statistics(): Option[EventFilter.FilterStatistics] = None
@@ -254,11 +253,11 @@ class EventLogFileCompactorSuite extends SparkFunSuite {
       // filterApplicationStart: Some(false) & Some(false) => filter out
       writeEventToWriter(writer, SparkListenerApplicationStart("app", None, 0, "user", None))
 
-      // filterNodeBlacklisted: None & Some(true) => filter in
-      expectedLines += writeEventToWriter(writer, SparkListenerNodeBlacklisted(0, "host1", 1))
+      // filterNodeExcluded: None & Some(true) => filter in
+      expectedLines += writeEventToWriter(writer, SparkListenerNodeExcluded(0, "host1", 1))
 
-      // filterNodeUnblacklisted: None & Some(false) => filter out
-      writeEventToWriter(writer, SparkListenerNodeUnblacklisted(0, "host1"))
+      // filterNodeUnexcluded: None & Some(false) => filter out
+      writeEventToWriter(writer, SparkListenerNodeUnexcluded(0, "host1"))
 
       // other events: None & None => filter in
       expectedLines += writeEventToWriter(writer, SparkListenerUnpersistRDD(0))
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileReadersSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileReadersSuite.scala
index 8eab2da1a37b7..7db30548fd668 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileReadersSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileReadersSuite.scala
@@ -216,7 +216,7 @@ class SingleFileEventLogFileReaderSuite extends EventLogFileReadersSuite {
     Utils.tryWithResource(new ZipInputStream(
         new ByteArrayInputStream(underlyingStream.toByteArray))) { is =>
 
-      var entry = is.getNextEntry
+      val entry = is.getNextEntry
       assert(entry != null)
       val actual = new String(ByteStreams.toByteArray(is), StandardCharsets.UTF_8)
       val expected = Files.toString(new File(logPath.toString), StandardCharsets.UTF_8)
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala
index 060b878fb8ef2..e6dd9ae4224d9 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala
@@ -99,7 +99,7 @@ abstract class EventLogFileWritersSuite extends SparkFunSuite with LocalSparkCon
     }
   }
 
-  test("spark.eventLog.compression.codec overrides spark.io.compression.codec") {
+  test("Use the defalut value of spark.eventLog.compression.codec") {
     val conf = new SparkConf
     conf.set(EVENT_LOG_COMPRESS, true)
     val hadoopConf = SparkHadoopUtil.get.newConfiguration(conf)
@@ -107,14 +107,8 @@ abstract class EventLogFileWritersSuite extends SparkFunSuite with LocalSparkCon
     val appId = "test"
     val appAttemptId = None
 
-    // The default value is `spark.io.compression.codec`.
     val writer = createWriter(appId, appAttemptId, testDirPath.toUri, conf, hadoopConf)
-    assert(writer.compressionCodecName.contains("lz4"))
-
-    // `spark.eventLog.compression.codec` overrides `spark.io.compression.codec`.
-    conf.set(EVENT_LOG_COMPRESSION_CODEC, "zstd")
-    val writer2 = createWriter(appId, appAttemptId, testDirPath.toUri, conf, hadoopConf)
-    assert(writer2.compressionCodecName.contains("zstd"))
+    assert(writer.compressionCodecName === EVENT_LOG_COMPRESSION_CODEC.defaultValue)
   }
 
   protected def readLinesFromEventLogFile(log: Path, fs: FileSystem): List[String] = {
@@ -213,7 +207,7 @@ class SingleEventLogFileWriterSuite extends EventLogFileWritersSuite {
       compressionCodecShortName)
 
     val finalLogPath = new Path(logPath)
-    assert(fileSystem.exists(finalLogPath) && fileSystem.isFile(finalLogPath))
+    assert(fileSystem.exists(finalLogPath) && fileSystem.getFileStatus(finalLogPath).isFile)
     assert(expectedLines === readLinesFromEventLogFile(finalLogPath, fileSystem))
   }
 }
@@ -357,10 +351,10 @@ class RollingEventLogFilesWriterSuite extends EventLogFileWritersSuite {
       expectedLines: Seq[String]): Unit = {
     val logDirPath = getAppEventLogDirPath(logBaseDir, appId, appAttemptId)
 
-    assert(fileSystem.exists(logDirPath) && fileSystem.isDirectory(logDirPath))
+    assert(fileSystem.exists(logDirPath) && fileSystem.getFileStatus(logDirPath).isDirectory)
 
     val appStatusFile = getAppStatusFilePath(logDirPath, appId, appAttemptId, inProgress = false)
-    assert(fileSystem.exists(appStatusFile) && fileSystem.isFile(appStatusFile))
+    assert(fileSystem.exists(appStatusFile) && fileSystem.getFileStatus(appStatusFile).isFile)
 
     val eventLogFiles = listEventLogFiles(logDirPath)
     val allLines = mutable.ArrayBuffer[String]()
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
index c2f34fc3a95ed..3b8677742ca16 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
@@ -34,8 +34,9 @@ import org.apache.hadoop.security.AccessControlException
 import org.json4s.jackson.JsonMethods._
 import org.mockito.ArgumentMatchers.{any, argThat}
 import org.mockito.Mockito.{doThrow, mock, spy, verify, when}
-import org.scalatest.Matchers
 import org.scalatest.concurrent.Eventually._
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.{JobExecutionStatus, SecurityManager, SPARK_VERSION, SparkConf, SparkFunSuite}
 import org.apache.spark.deploy.SparkHadoopUtil
@@ -43,7 +44,7 @@ import org.apache.spark.deploy.history.EventLogTestHelper._
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.DRIVER_LOG_DFS_DIR
 import org.apache.spark.internal.config.History._
-import org.apache.spark.internal.config.UI.{ADMIN_ACLS, ADMIN_ACLS_GROUPS, USER_GROUPS_MAPPING}
+import org.apache.spark.internal.config.UI.{ADMIN_ACLS, ADMIN_ACLS_GROUPS, UI_VIEW_ACLS, UI_VIEW_ACLS_GROUPS, USER_GROUPS_MAPPING}
 import org.apache.spark.io._
 import org.apache.spark.scheduler._
 import org.apache.spark.scheduler.cluster.ExecutorInfo
@@ -89,9 +90,13 @@ class FsHistoryProviderSuite extends SparkFunSuite with Matchers with Logging {
     }
   }
 
-  private def testAppLogParsing(inMemory: Boolean): Unit = {
+  test("SPARK-31608: parse application logs with HybridStore") {
+    testAppLogParsing(false, true)
+  }
+
+  private def testAppLogParsing(inMemory: Boolean, useHybridStore: Boolean = false): Unit = {
     val clock = new ManualClock(12345678)
-    val conf = createTestConf(inMemory = inMemory)
+    val conf = createTestConf(inMemory = inMemory, useHybridStore = useHybridStore)
     val provider = new FsHistoryProvider(conf, clock)
 
     // Write a new-style application log.
@@ -921,8 +926,8 @@ class FsHistoryProviderSuite extends SparkFunSuite with Matchers with Logging {
     oldProvider.listing.setMetadata(meta)
     oldProvider.stop()
 
-    val mistatchedVersionProvider = new FsHistoryProvider(conf)
-    assert(mistatchedVersionProvider.listing.count(classOf[ApplicationInfoWrapper]) === 0)
+    val mismatchedVersionProvider = new FsHistoryProvider(conf)
+    assert(mismatchedVersionProvider.listing.count(classOf[ApplicationInfoWrapper]) === 0)
   }
 
   test("invalidate cached UI") {
@@ -1117,7 +1122,7 @@ class FsHistoryProviderSuite extends SparkFunSuite with Matchers with Logging {
     }
   }
 
-  test("SPARK-24948: blacklist files we don't have read permission on") {
+  test("SPARK-24948: ignore files we don't have read permission on") {
     val clock = new ManualClock(1533132471)
     val provider = new FsHistoryProvider(createTestConf(), clock)
     val accessDenied = newLogFile("accessDenied", None, inProgress = false)
@@ -1137,17 +1142,17 @@ class FsHistoryProviderSuite extends SparkFunSuite with Matchers with Logging {
     updateAndCheck(mockedProvider) { list =>
       list.size should be(1)
     }
-    // Doing 2 times in order to check the blacklist filter too
+    // Doing 2 times in order to check the inaccessibleList filter too
     updateAndCheck(mockedProvider) { list =>
       list.size should be(1)
     }
     val accessDeniedPath = new Path(accessDenied.getPath)
-    assert(mockedProvider.isBlacklisted(accessDeniedPath))
+    assert(!mockedProvider.isAccessible(accessDeniedPath))
     clock.advance(24 * 60 * 60 * 1000 + 1) // add a bit more than 1d
     isReadable = true
     mockedProvider.cleanLogs()
     updateAndCheck(mockedProvider) { list =>
-      assert(!mockedProvider.isBlacklisted(accessDeniedPath))
+      assert(mockedProvider.isAccessible(accessDeniedPath))
       assert(list.exists(_.name == "accessDenied"))
       assert(list.exists(_.name == "accessGranted"))
       list.size should be(2)
@@ -1470,6 +1475,107 @@ class FsHistoryProviderSuite extends SparkFunSuite with Matchers with Logging {
     }
   }
 
+  test("SPARK-33146: don't let one bad rolling log folder prevent loading other applications") {
+    withTempDir { dir =>
+      val conf = createTestConf(true)
+      conf.set(HISTORY_LOG_DIR, dir.getAbsolutePath)
+      val hadoopConf = SparkHadoopUtil.newConfiguration(conf)
+      val fs = new Path(dir.getAbsolutePath).getFileSystem(hadoopConf)
+
+      val provider = new FsHistoryProvider(conf)
+
+      val writer = new RollingEventLogFilesWriter("app", None, dir.toURI, conf, hadoopConf)
+      writer.start()
+
+      writeEventsToRollingWriter(writer, Seq(
+        SparkListenerApplicationStart("app", Some("app"), 0, "user", None),
+        SparkListenerJobStart(1, 0, Seq.empty)), rollFile = false)
+      provider.checkForLogs()
+      provider.cleanLogs()
+      assert(dir.listFiles().size === 1)
+      assert(provider.getListing.length === 1)
+
+      // Manually delete the appstatus file to make an invalid rolling event log
+      val appStatusPath = RollingEventLogFilesWriter.getAppStatusFilePath(new Path(writer.logPath),
+        "app", None, true)
+      fs.delete(appStatusPath, false)
+      provider.checkForLogs()
+      provider.cleanLogs()
+      assert(provider.getListing.length === 0)
+
+      // Create a new application
+      val writer2 = new RollingEventLogFilesWriter("app2", None, dir.toURI, conf, hadoopConf)
+      writer2.start()
+      writeEventsToRollingWriter(writer2, Seq(
+        SparkListenerApplicationStart("app2", Some("app2"), 0, "user", None),
+        SparkListenerJobStart(1, 0, Seq.empty)), rollFile = false)
+
+      // Both folders exist but only one application found
+      provider.checkForLogs()
+      provider.cleanLogs()
+      assert(provider.getListing.length === 1)
+      assert(dir.listFiles().size === 2)
+
+      // Make sure a new provider sees the valid application
+      provider.stop()
+      val newProvider = new FsHistoryProvider(conf)
+      newProvider.checkForLogs()
+      assert(newProvider.getListing.length === 1)
+    }
+  }
+
+  test("SPARK-33215: check ui view permissions without retrieving ui") {
+    val conf = createTestConf()
+      .set(HISTORY_SERVER_UI_ACLS_ENABLE, true)
+      .set(HISTORY_SERVER_UI_ADMIN_ACLS, Seq("user1", "user2"))
+      .set(HISTORY_SERVER_UI_ADMIN_ACLS_GROUPS, Seq("group1"))
+      .set(USER_GROUPS_MAPPING, classOf[TestGroupsMappingProvider].getName)
+
+    val provider = new FsHistoryProvider(conf)
+    val log = newLogFile("app1", Some("attempt1"), inProgress = false)
+    writeFile(log, None,
+      SparkListenerApplicationStart("app1", Some("app1"), System.currentTimeMillis(),
+        "test", Some("attempt1")),
+      SparkListenerEnvironmentUpdate(Map(
+        "Spark Properties" -> List((UI_VIEW_ACLS.key, "user"), (UI_VIEW_ACLS_GROUPS.key, "group")),
+        "Hadoop Properties" -> Seq.empty,
+        "JVM Information" -> Seq.empty,
+        "System Properties" -> Seq.empty,
+        "Classpath Entries" -> Seq.empty
+      )),
+      SparkListenerApplicationEnd(System.currentTimeMillis()))
+
+    provider.checkForLogs()
+
+    // attempt2 doesn't exist
+    intercept[NoSuchElementException] {
+      provider.checkUIViewPermissions("app1", Some("attempt2"), "user1")
+    }
+    // app2 doesn't exist
+    intercept[NoSuchElementException] {
+      provider.checkUIViewPermissions("app2", Some("attempt1"), "user1")
+    }
+
+    // user1 and user2 are admins
+    assert(provider.checkUIViewPermissions("app1", Some("attempt1"), "user1"))
+    assert(provider.checkUIViewPermissions("app1", Some("attempt1"), "user2"))
+    // user3 is a member of admin group "group1"
+    assert(provider.checkUIViewPermissions("app1", Some("attempt1"), "user3"))
+    // test is the app owner
+    assert(provider.checkUIViewPermissions("app1", Some("attempt1"), "test"))
+    // user is in the app's view acls
+    assert(provider.checkUIViewPermissions("app1", Some("attempt1"), "user"))
+    // user5 is a member of the app's view acls group "group"
+    assert(provider.checkUIViewPermissions("app1", Some("attempt1"), "user5"))
+
+    // abc, user6, user7 don't have permissions
+    assert(!provider.checkUIViewPermissions("app1", Some("attempt1"), "abc"))
+    assert(!provider.checkUIViewPermissions("app1", Some("attempt1"), "user6"))
+    assert(!provider.checkUIViewPermissions("app1", Some("attempt1"), "user7"))
+
+    provider.stop()
+  }
+
   /**
    * Asks the provider to check for logs and calls a function to perform checks on the updated
    * app list. Example:
@@ -1508,7 +1614,9 @@ class FsHistoryProviderSuite extends SparkFunSuite with Matchers with Logging {
     new FileOutputStream(file).close()
   }
 
-  private def createTestConf(inMemory: Boolean = false): SparkConf = {
+  private def createTestConf(
+      inMemory: Boolean = false,
+      useHybridStore: Boolean = false): SparkConf = {
     val conf = new SparkConf()
       .set(HISTORY_LOG_DIR, testDir.getAbsolutePath())
       .set(FAST_IN_PROGRESS_PARSING, true)
@@ -1516,6 +1624,7 @@ class FsHistoryProviderSuite extends SparkFunSuite with Matchers with Logging {
     if (!inMemory) {
       conf.set(LOCAL_STORE_DIR, Utils.createTempDir().getAbsolutePath())
     }
+    conf.set(HYBRID_STORE_ENABLED, useHybridStore)
 
     conf
   }
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerDiskManagerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerDiskManagerSuite.scala
index f78469e132490..9004e86323691 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerDiskManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerDiskManagerSuite.scala
@@ -158,4 +158,56 @@ class HistoryServerDiskManagerSuite extends SparkFunSuite with BeforeAndAfter {
     assert(manager.approximateSize(50L, true) > 50L)
   }
 
+  test("SPARK-32024: update ApplicationStoreInfo.size during initializing") {
+    val manager = mockManager()
+    val leaseA = manager.lease(2)
+    doReturn(3L).when(manager).sizeOf(meq(leaseA.tmpPath))
+    val dstPathA = manager.appStorePath("app1", None)
+    doReturn(3L).when(manager).sizeOf(meq(dstPathA))
+    val dstA = leaseA.commit("app1", None)
+    assert(manager.free() === 0)
+    assert(manager.committed() === 3)
+    // Listing store tracks dstA now.
+    assert(store.read(classOf[ApplicationStoreInfo], dstA.getAbsolutePath).size === 3)
+
+    // Simulate: service restarts, new disk manager (manager1) is initialized.
+    val manager1 = mockManager()
+    // Simulate: event KVstore compaction before restart, directory size reduces.
+    doReturn(2L).when(manager1).sizeOf(meq(dstA))
+    doReturn(2L).when(manager1).sizeOf(meq(new File(testDir, "apps")))
+    manager1.initialize()
+    // "ApplicationStoreInfo.size" is updated for dstA.
+    assert(store.read(classOf[ApplicationStoreInfo], dstA.getAbsolutePath).size === 2)
+    assert(manager1.free() === 1)
+    // If "ApplicationStoreInfo.size" is not correctly updated, "IllegalStateException"
+    // would be thrown.
+    val leaseB = manager1.lease(2)
+    assert(manager1.free() === 1)
+    doReturn(2L).when(manager1).sizeOf(meq(leaseB.tmpPath))
+    val dstPathB = manager.appStorePath("app2", None)
+    doReturn(2L).when(manager1).sizeOf(meq(dstPathB))
+    val dstB = leaseB.commit("app2", None)
+    assert(manager1.committed() === 2)
+    // Listing store tracks dstB only, dstA is evicted by "makeRoom()".
+    assert(store.read(classOf[ApplicationStoreInfo], dstB.getAbsolutePath).size === 2)
+
+    val manager2 = mockManager()
+    // Simulate: cache entities are written after replaying, directory size increases.
+    doReturn(3L).when(manager2).sizeOf(meq(dstB))
+    doReturn(3L).when(manager2).sizeOf(meq(new File(testDir, "apps")))
+    manager2.initialize()
+    // "ApplicationStoreInfo.size" is updated for dstB.
+    assert(store.read(classOf[ApplicationStoreInfo], dstB.getAbsolutePath).size === 3)
+    assert(manager2.free() === 0)
+    val leaseC = manager2.lease(2)
+    doReturn(2L).when(manager2).sizeOf(meq(leaseC.tmpPath))
+    val dstPathC = manager.appStorePath("app3", None)
+    doReturn(2L).when(manager2).sizeOf(meq(dstPathC))
+    val dstC = leaseC.commit("app3", None)
+    assert(manager2.free() === 1)
+    assert(manager2.committed() === 2)
+    // Listing store tracks dstC only, dstB is evicted by "makeRoom()".
+    assert(store.read(classOf[ApplicationStoreInfo], dstC.getAbsolutePath).size === 2)
+  }
+
 }
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerMemoryManagerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerMemoryManagerSuite.scala
new file mode 100644
index 0000000000000..697f8f72624e5
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerMemoryManagerSuite.scala
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.deploy.history
+
+import org.apache.spark.{SparkConf, SparkFunSuite}
+import org.apache.spark.internal.config.History._
+
+class HistoryServerMemoryManagerSuite extends SparkFunSuite {
+
+  private val MAX_USAGE = 3L
+
+  test("lease and release memory") {
+    val conf = new SparkConf().set(MAX_IN_MEMORY_STORE_USAGE, MAX_USAGE)
+    val manager = new HistoryServerMemoryManager(conf)
+
+    // Memory usage estimation for non-compressed log file is filesize / 2
+    manager.lease("app1", None, 2L, None)
+    manager.lease("app2", None, 2L, None)
+    manager.lease("app3", None, 2L, None)
+    assert(manager.currentUsage.get === 3L)
+    assert(manager.active.size === 3)
+    assert(manager.active.get(("app1", None)) === Some(1L))
+
+    intercept[RuntimeException] {
+      manager.lease("app4", None, 2L, None)
+    }
+
+    // Releasing a non-existent app is a no-op
+    manager.release("app4", None)
+    assert(manager.currentUsage.get === 3L)
+
+    manager.release("app1", None)
+    assert(manager.currentUsage.get === 2L)
+    assert(manager.active.size === 2)
+
+    manager.lease("app4", None, 2L, None)
+    assert(manager.currentUsage.get === 3L)
+    assert(manager.active.size === 3)
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
index c55b29b15051d..5da5835054e9f 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
@@ -29,16 +29,16 @@ import scala.concurrent.duration._
 import com.google.common.io.{ByteStreams, Files}
 import org.apache.commons.io.{FileUtils, IOUtils}
 import org.apache.hadoop.fs.{FileStatus, FileSystem, Path}
-import org.eclipse.jetty.proxy.ProxyServlet
-import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder}
 import org.json4s.JsonAST._
 import org.json4s.jackson.JsonMethods
 import org.json4s.jackson.JsonMethods._
 import org.mockito.Mockito._
 import org.openqa.selenium.WebDriver
 import org.openqa.selenium.htmlunit.HtmlUnitDriver
-import org.scalatest.{BeforeAndAfter, Matchers}
+import org.scalatest.BeforeAndAfter
 import org.scalatest.concurrent.Eventually
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 import org.scalatestplus.mockito.MockitoSugar
 import org.scalatestplus.selenium.WebBrowser
 
@@ -138,7 +138,11 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
     "complete stage list json" -> "applications/local-1422981780767/stages?status=complete",
     "failed stage list json" -> "applications/local-1422981780767/stages?status=failed",
     "one stage json" -> "applications/local-1422981780767/stages/1",
+    "one stage json with details" ->
+      "applications/local-1422981780767/stages/1?details=true&taskStatus=success",
     "one stage attempt json" -> "applications/local-1422981780767/stages/1/0",
+    "one stage attempt json details with failed task" ->
+      "applications/local-1422981780767/stages/1/0?details=true&taskStatus=failed",
 
     "stage task summary w shuffle write"
       -> "applications/local-1430917381534/stages/0/0/taskSummary",
@@ -156,6 +160,12 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
       "applications/local-1430917381534/stages/0/0/taskList?sortBy=-runtime",
     "stage task list w/ sortBy short names: runtime" ->
       "applications/local-1430917381534/stages/0/0/taskList?sortBy=runtime",
+    "stage task list w/ status" ->
+      "applications/app-20161115172038-0000/stages/0/0/taskList?status=failed",
+    "stage task list w/ status & offset & length" ->
+      "applications/local-1430917381534/stages/0/0/taskList?status=success&offset=1&length=2",
+    "stage task list w/ status & sortBy short names: runtime" ->
+      "applications/local-1430917381534/stages/0/0/taskList?status=success&sortBy=runtime",
 
     "stage list with accumulable json" -> "applications/local-1426533911241/1/stages",
     "stage with accumulable json" -> "applications/local-1426533911241/1/stages/0/0",
@@ -163,20 +173,26 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
       "applications/local-1426533911241/1/stages/0/0/taskList",
     "stage task list from multi-attempt app json(2)" ->
       "applications/local-1426533911241/2/stages/0/0/taskList",
-    "blacklisting for stage" -> "applications/app-20180109111548-0000/stages/0/0",
-    "blacklisting node for stage" -> "applications/application_1516285256255_0012/stages/0/0",
+    "excludeOnFailure for stage" -> "applications/app-20180109111548-0000/stages/0/0",
+    "excludeOnFailure node for stage" -> "applications/application_1516285256255_0012/stages/0/0",
 
     "rdd list storage json" -> "applications/local-1422981780767/storage/rdd",
-    "executor node blacklisting" -> "applications/app-20161116163331-0000/executors",
-    "executor node blacklisting unblacklisting" -> "applications/app-20161115172038-0000/executors",
+    "executor node excludeOnFailure" -> "applications/app-20161116163331-0000/executors",
+    "executor node excludeOnFailure unexcluding" ->
+      "applications/app-20161115172038-0000/executors",
     "executor memory usage" -> "applications/app-20161116163331-0000/executors",
     "executor resource information" -> "applications/application_1555004656427_0144/executors",
+    "multiple resource profiles" -> "applications/application_1578436911597_0052/environment",
+    "stage list with peak metrics" -> "applications/app-20200706201101-0003/stages",
+    "stage with peak metrics" -> "applications/app-20200706201101-0003/stages/2/0",
+    "stage with summaries" -> "applications/app-20200706201101-0003/stages/2/0?withSummaries=true",
 
     "app environment" -> "applications/app-20161116163331-0000/environment",
 
     // Enable "spark.eventLog.logBlockUpdates.enabled", to get the storage information
     // in the history server.
-    "one rdd storage json" -> "applications/local-1422981780767/storage/rdd/0"
+    "one rdd storage json" -> "applications/local-1422981780767/storage/rdd/0",
+    "miscellaneous process" -> "applications/application_1555004656427_0144/allmiscellaneousprocess"
   )
 
   // run a bunch of characterization tests -- just verify the behavior is the same as what is saved
@@ -189,7 +205,8 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
       errOpt should be (None)
 
       val exp = IOUtils.toString(new FileInputStream(
-        new File(expRoot, HistoryServerSuite.sanitizePath(name) + "_expectation.json")))
+        new File(expRoot, HistoryServerSuite.sanitizePath(name) + "_expectation.json")),
+        StandardCharsets.UTF_8)
       // compare the ASTs so formatting differences don't cause failures
       import org.json4s._
       import org.json4s.jackson.JsonMethods._
@@ -304,18 +321,21 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
 
     val urlsThroughKnox = responseThroughKnox \\ "@href" map (_.toString)
     val siteRelativeLinksThroughKnox = urlsThroughKnox filter (_.startsWith("/"))
-    all (siteRelativeLinksThroughKnox) should startWith (knoxBaseUrl)
+    for (link <- siteRelativeLinksThroughKnox) {
+      link should startWith (knoxBaseUrl)
+    }
 
     val directRequest = mock[HttpServletRequest]
     val directResponse = page.render(directRequest)
 
     val directUrls = directResponse \\ "@href" map (_.toString)
     val directSiteRelativeLinks = directUrls filter (_.startsWith("/"))
-    all (directSiteRelativeLinks) should not startWith (knoxBaseUrl)
+    for (link <- directSiteRelativeLinks) {
+      link should not startWith (knoxBaseUrl)
+    }
   }
 
-  // TODO (SPARK-31723): re-enable it
-  ignore("static relative links are prefixed with uiRoot (spark.ui.proxyBase)") {
+  test("static relative links are prefixed with uiRoot (spark.ui.proxyBase)") {
     val uiRoot = Option(System.getenv("APPLICATION_WEB_PROXY_BASE")).getOrElse("/testwebproxybase")
     val page = new HistoryPage(server)
     val request = mock[HttpServletRequest]
@@ -327,7 +347,9 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
     // then
     val urls = response \\ "@href" map (_.toString)
     val siteRelativeLinks = urls filter (_.startsWith("/"))
-    all (siteRelativeLinks) should startWith (uiRoot)
+    for (link <- siteRelativeLinks) {
+      link should startWith (uiRoot)
+    }
   }
 
   test("/version api endpoint") {
@@ -335,66 +357,6 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
     assert(response.contains(SPARK_VERSION))
   }
 
-  test("ajax rendered relative links are prefixed with uiRoot (spark.ui.proxyBase)") {
-    val uiRoot = "/testwebproxybase"
-    System.setProperty("spark.ui.proxyBase", uiRoot)
-
-    stop()
-    init()
-
-    val port = server.boundPort
-
-    val servlet = new ProxyServlet {
-      override def rewriteTarget(request: HttpServletRequest): String = {
-        // servlet acts like a proxy that redirects calls made on
-        // spark.ui.proxyBase context path to the normal servlet handlers operating off "/"
-        val sb = request.getRequestURL()
-
-        if (request.getQueryString() != null) {
-          sb.append(s"?${request.getQueryString()}")
-        }
-
-        val proxyidx = sb.indexOf(uiRoot)
-        sb.delete(proxyidx, proxyidx + uiRoot.length).toString
-      }
-    }
-
-    val contextHandler = new ServletContextHandler
-    val holder = new ServletHolder(servlet)
-    contextHandler.setContextPath(uiRoot)
-    contextHandler.addServlet(holder, "/")
-    server.attachHandler(contextHandler)
-
-    implicit val webDriver: WebDriver = new HtmlUnitDriver(true)
-
-    try {
-      val url = s"http://localhost:$port"
-
-      go to s"$url$uiRoot"
-
-      // expect the ajax call to finish in 5 seconds
-      implicitlyWait(org.scalatest.time.Span(5, org.scalatest.time.Seconds))
-
-      // once this findAll call returns, we know the ajax load of the table completed
-      findAll(ClassNameQuery("odd"))
-
-      val links = findAll(TagNameQuery("a"))
-        .map(_.attribute("href"))
-        .filter(_.isDefined)
-        .map(_.get)
-        .filter(_.startsWith(url)).toList
-
-      // there are at least some URL links that were generated via javascript,
-      // and they all contain the spark.ui.proxyBase (uiRoot)
-      links.length should be > 4
-      all(links) should startWith(url + uiRoot)
-    } finally {
-      contextHandler.stop()
-      quit()
-    }
-
-  }
-
   /**
    * Verify that the security manager needed for the history server can be instantiated
    * when `spark.authenticate` is `true`, rather than raise an `IllegalArgumentException`.
@@ -630,6 +592,24 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
     }
   }
 
+  test("SPARK-33215: speed up event log download by skipping UI rebuild") {
+    val appId = "local-1430917381535"
+
+    stop()
+    init()
+
+    val port = server.boundPort
+    val testUrls = Seq(
+      s"http://localhost:$port/api/v1/applications/$appId/logs",
+      s"http://localhost:$port/api/v1/applications/$appId/1/logs",
+      s"http://localhost:$port/api/v1/applications/$appId/2/logs")
+
+    testUrls.foreach { url =>
+      TestUtils.httpResponseCode(new URL(url))
+    }
+    assert(server.cacheMetrics.loadCount.getCount === 0, "downloading event log shouldn't load ui")
+  }
+
   test("access history application defaults to the last attempt id") {
 
     def getRedirectUrl(url: URL): (Int, String) = {
@@ -705,6 +685,19 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
     val actualContentType = conn.getContentType
     assert(actualContentType === expectedContentType)
   }
+
+  test("Redirect to the root page when accessed to /history/") {
+    val port = server.boundPort
+    val url = new URL(s"http://localhost:$port/history/")
+    val conn = url.openConnection().asInstanceOf[HttpURLConnection]
+    conn.setRequestMethod("GET")
+    conn.setUseCaches(false)
+    conn.setDefaultUseCaches(false)
+    conn.setInstanceFollowRedirects(false)
+    conn.connect()
+    assert(conn.getResponseCode === 302)
+    assert(conn.getHeaderField("Location") === s"http://localhost:$port/")
+  }
 }
 
 object HistoryServerSuite {
@@ -727,7 +720,7 @@ object HistoryServerSuite {
 
   def getContentAndCode(url: URL): (Int, Option[String], Option[String]) = {
     val (code, in, errString) = connectAndGetInputStream(url)
-    val inString = in.map(IOUtils.toString)
+    val inString = in.map(IOUtils.toString(_, StandardCharsets.UTF_8))
     (code, inString, errString)
   }
 
@@ -743,7 +736,7 @@ object HistoryServerSuite {
     }
     val errString = try {
       val err = Option(connection.getErrorStream())
-      err.map(IOUtils.toString)
+      err.map(IOUtils.toString(_, StandardCharsets.UTF_8))
     } catch {
       case io: IOException => None
     }
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HybridStoreSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HybridStoreSuite.scala
new file mode 100644
index 0000000000000..fa57049b1a770
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/deploy/history/HybridStoreSuite.scala
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.deploy.history
+
+import java.io.File
+import java.util.NoSuchElementException
+import java.util.concurrent.LinkedBlockingQueue
+
+import org.apache.commons.io.FileUtils
+import org.scalatest.BeforeAndAfter
+import org.scalatest.concurrent.TimeLimits
+import org.scalatest.time.SpanSugar._
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.status.KVUtils._
+import org.apache.spark.util.kvstore._
+
+class HybridStoreSuite extends SparkFunSuite with BeforeAndAfter with TimeLimits {
+
+  private var db: LevelDB = _
+  private var dbpath: File = _
+
+  before {
+    dbpath = File.createTempFile("test.", ".ldb")
+    dbpath.delete()
+    db = new LevelDB(dbpath, new KVStoreScalaSerializer())
+  }
+
+  after {
+    if (db != null) {
+      db.close()
+    }
+    if (dbpath != null) {
+      FileUtils.deleteQuietly(dbpath)
+    }
+  }
+
+  test("test multiple objects write read delete") {
+    val store = createHybridStore()
+
+    val t1 = createCustomType1(1)
+    val t2 = createCustomType1(2)
+
+    intercept[NoSuchElementException] {
+      store.read(t1.getClass(), t1.key)
+    }
+
+    store.write(t1)
+    store.write(t2)
+    store.delete(t2.getClass(), t2.key)
+
+    Seq(false, true).foreach { switch =>
+      if (switch) switchHybridStore(store)
+
+      intercept[NoSuchElementException] {
+        store.read(t2.getClass(), t2.key)
+      }
+      assert(store.read(t1.getClass(), t1.key) === t1)
+      assert(store.count(t1.getClass()) === 1L)
+    }
+  }
+
+  test("test metadata") {
+    val store = createHybridStore()
+    assert(store.getMetadata(classOf[CustomType1]) === null)
+
+    val t1 = createCustomType1(1)
+    store.setMetadata(t1)
+    assert(store.getMetadata(classOf[CustomType1]) === t1)
+
+    // Switch to LevelDB and set a new metadata
+    switchHybridStore(store)
+
+    val t2 = createCustomType1(2)
+    store.setMetadata(t2)
+    assert(store.getMetadata(classOf[CustomType1]) === t2)
+  }
+
+  test("test update") {
+    val store = createHybridStore()
+    val t = createCustomType1(1)
+
+    store.write(t)
+    t.name = "name2"
+    store.write(t)
+
+    Seq(false, true).foreach { switch =>
+      if (switch) switchHybridStore(store)
+
+      assert(store.count(t.getClass()) === 1L)
+      assert(store.read(t.getClass(), t.key) === t)
+    }
+  }
+
+  test("test basic iteration") {
+    val store = createHybridStore()
+
+    val t1 = createCustomType1(1)
+    store.write(t1)
+    val t2 = createCustomType1(2)
+    store.write(t2)
+
+    Seq(false, true).foreach { switch =>
+      if (switch) switchHybridStore(store)
+
+      assert(store.view(t1.getClass()).iterator().next().id === t1.id)
+      assert(store.view(t1.getClass()).skip(1).iterator().next().id === t2.id)
+      assert(store.view(t1.getClass()).skip(1).max(1).iterator().next().id === t2.id)
+      assert(store.view(t1.getClass()).first(t1.key).max(1).iterator().next().id === t1.id)
+      assert(store.view(t1.getClass()).first(t2.key).max(1).iterator().next().id === t2.id)
+    }
+  }
+
+  test("test delete after switch") {
+    val store = createHybridStore()
+    val t = createCustomType1(1)
+    store.write(t)
+    switchHybridStore(store)
+    intercept[IllegalStateException] {
+      store.delete(t.getClass(), t.key)
+    }
+  }
+
+  test("test klassMap") {
+    val store = createHybridStore()
+    val t1 = createCustomType1(1)
+    store.write(t1)
+    assert(store.klassMap.size === 1)
+    val t2 = new CustomType2("key2")
+    store.write(t2)
+    assert(store.klassMap.size === 2)
+
+    switchHybridStore(store)
+    val t3 = new CustomType3("key3")
+    store.write(t3)
+    // Cannot put new klass to klassMap after the switching starts
+    assert(store.klassMap.size === 2)
+  }
+
+  private def createHybridStore(): HybridStore = {
+    val store = new HybridStore()
+    store.setLevelDB(db)
+    store
+  }
+
+  private def createCustomType1(i: Int): CustomType1 = {
+    new CustomType1("key" + i, "id" + i, "name" + i, i, "child" + i)
+  }
+
+  private def switchHybridStore(store: HybridStore): Unit = {
+    assert(store.getStore().isInstanceOf[InMemoryStore])
+    val listener = new SwitchListener()
+    store.switchToLevelDB(listener, "test", None)
+    failAfter(2.seconds) {
+      assert(listener.waitUntilDone())
+    }
+    while (!store.getStore().isInstanceOf[LevelDB]) {
+      Thread.sleep(10)
+    }
+  }
+
+  private class SwitchListener extends HybridStore.SwitchToLevelDBListener {
+
+    // Put true to the queue when switch succeeds, and false when fails.
+    private val results = new LinkedBlockingQueue[Boolean]()
+
+    override def onSwitchToLevelDBSuccess(): Unit = {
+      try {
+        results.put(true)
+      } catch {
+        case _: InterruptedException =>
+          // no-op
+      }
+    }
+
+    override def onSwitchToLevelDBFail(e: Exception): Unit = {
+      try {
+        results.put(false)
+      } catch {
+        case _: InterruptedException =>
+          // no-op
+      }
+    }
+
+    def waitUntilDone(): Boolean = {
+      results.take()
+    }
+  }
+}
+
+class CustomType1(
+    @KVIndexParam var key: String,
+    @KVIndexParam("id") var id: String,
+    @KVIndexParam(value = "name", copy = true) var name: String,
+    @KVIndexParam("int") var num: Int,
+    @KVIndexParam(value = "child", parent = "id") var child: String) {
+
+  override def equals(o: Any): Boolean = {
+    o match {
+      case t: CustomType1 =>
+        id.equals(t.id) && name.equals(t.name)
+      case _ => false
+    }
+  }
+
+  override def hashCode: Int = {
+    id.hashCode
+  }
+
+  override def toString: String = {
+    "CustomType1[key=" + key + ",id=" + id + ",name=" + name + ",num=" + num;
+  }
+}
+
+class CustomType2(@KVIndexParam var key: String) {}
+
+class CustomType3(@KVIndexParam var key: String) {}
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala
new file mode 100644
index 0000000000000..4a5c34f86753c
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.deploy.history
+
+import javax.servlet.http.HttpServletRequest
+
+import org.eclipse.jetty.proxy.ProxyServlet
+import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder}
+import org.openqa.selenium.WebDriver
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
+import org.scalatestplus.selenium.WebBrowser
+
+import org.apache.spark._
+import org.apache.spark.internal.config.{EVENT_LOG_STAGE_EXECUTOR_METRICS, EXECUTOR_PROCESS_TREE_METRICS_ENABLED}
+import org.apache.spark.internal.config.History.{HISTORY_LOG_DIR, LOCAL_STORE_DIR, UPDATE_INTERVAL_S}
+import org.apache.spark.internal.config.Tests.IS_TESTING
+import org.apache.spark.util.{ResetSystemProperties, Utils}
+
+/**
+ * Tests for HistoryServer with real web browsers.
+ */
+abstract class RealBrowserUIHistoryServerSuite(val driverProp: String)
+  extends SparkFunSuite with WebBrowser with Matchers with BeforeAndAfterAll
+  with BeforeAndAfterEach with ResetSystemProperties {
+
+  implicit var webDriver: WebDriver
+
+  private val driverPropPrefix = "spark.test."
+  private val logDir = getTestResourcePath("spark-events")
+  private val storeDir = Utils.createTempDir(namePrefix = "history")
+
+  private var provider: FsHistoryProvider = null
+  private var server: HistoryServer = null
+  private var port: Int = -1
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    assume(
+      sys.props(driverPropPrefix + driverProp) !== null,
+      "System property " + driverPropPrefix + driverProp +
+        " should be set to the corresponding driver path.")
+    sys.props(driverProp) = sys.props(driverPropPrefix + driverProp)
+  }
+
+  override def beforeEach(): Unit = {
+    super.beforeEach()
+    if (server == null) {
+      init()
+    }
+  }
+
+  override def afterAll(): Unit = {
+    sys.props.remove(driverProp)
+    super.afterAll()
+  }
+
+  def init(extraConf: (String, String)*): Unit = {
+    Utils.deleteRecursively(storeDir)
+    assert(storeDir.mkdir())
+    val conf = new SparkConf()
+      .set(HISTORY_LOG_DIR, logDir)
+      .set(UPDATE_INTERVAL_S.key, "0")
+      .set(IS_TESTING, true)
+      .set(LOCAL_STORE_DIR, storeDir.getAbsolutePath())
+      .set(EVENT_LOG_STAGE_EXECUTOR_METRICS, true)
+      .set(EXECUTOR_PROCESS_TREE_METRICS_ENABLED, true)
+    conf.setAll(extraConf)
+    provider = new FsHistoryProvider(conf)
+    provider.checkForLogs()
+    val securityManager = HistoryServer.createSecurityManager(conf)
+
+    server = new HistoryServer(conf, provider, securityManager, 18080)
+    server.initialize()
+    server.bind()
+    provider.start()
+    port = server.boundPort
+  }
+
+  def stop(): Unit = {
+    server.stop()
+    server = null
+  }
+
+  test("ajax rendered relative links are prefixed with uiRoot (spark.ui.proxyBase)") {
+    val uiRoot = "/testwebproxybase"
+    System.setProperty("spark.ui.proxyBase", uiRoot)
+
+    stop()
+    init()
+
+    val port = server.boundPort
+
+    val servlet = new ProxyServlet {
+      override def rewriteTarget(request: HttpServletRequest): String = {
+        // servlet acts like a proxy that redirects calls made on
+        // spark.ui.proxyBase context path to the normal servlet handlers operating off "/"
+        val sb = request.getRequestURL()
+
+        if (request.getQueryString() != null) {
+          sb.append(s"?${request.getQueryString()}")
+        }
+
+        val proxyidx = sb.indexOf(uiRoot)
+        sb.delete(proxyidx, proxyidx + uiRoot.length).toString
+      }
+    }
+
+    val contextHandler = new ServletContextHandler
+    val holder = new ServletHolder(servlet)
+    contextHandler.setContextPath(uiRoot)
+    contextHandler.addServlet(holder, "/")
+    server.attachHandler(contextHandler)
+
+    try {
+      val url = s"http://localhost:$port"
+
+      go to s"$url$uiRoot"
+
+      // expect the ajax call to finish in 5 seconds
+      implicitlyWait(org.scalatest.time.Span(5, org.scalatest.time.Seconds))
+
+      // once this findAll call returns, we know the ajax load of the table completed
+      findAll(ClassNameQuery("odd"))
+
+      val links = findAll(TagNameQuery("a"))
+        .map(_.attribute("href"))
+        .filter(_.isDefined)
+        .map(_.get)
+        .filter(_.startsWith(url)).toList
+
+      // there are at least some URL links that were generated via javascript,
+      // and they all contain the spark.ui.proxyBase (uiRoot)
+      links.length should be > 4
+      for (link <- links) {
+        link should startWith(url + uiRoot)
+      }
+    } finally {
+      contextHandler.stop()
+      quit()
+    }
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
index 0cf573c2490b3..7fd9af1389cb4 100644
--- a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
@@ -23,7 +23,7 @@ import java.util.concurrent.atomic.AtomicInteger
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable
-import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet}
+import scala.collection.mutable.{HashMap, HashSet}
 import scala.concurrent.duration._
 import scala.io.Source
 import scala.reflect.ClassTag
@@ -31,8 +31,10 @@ import scala.reflect.ClassTag
 import org.json4s._
 import org.json4s.jackson.JsonMethods._
 import org.mockito.Mockito.{mock, when}
-import org.scalatest.{BeforeAndAfter, Matchers, PrivateMethodTester}
+import org.scalatest.{BeforeAndAfter, PrivateMethodTester}
 import org.scalatest.concurrent.Eventually
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 import other.supplier.{CustomPersistenceEngine, CustomRecoveryModeFactory}
 
 import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
@@ -46,6 +48,7 @@ import org.apache.spark.resource.{ResourceInformation, ResourceRequirement}
 import org.apache.spark.resource.ResourceUtils.{FPGA, GPU}
 import org.apache.spark.rpc.{RpcAddress, RpcEndpoint, RpcEndpointRef, RpcEnv}
 import org.apache.spark.serializer
+import org.apache.spark.util.Utils
 
 object MockWorker {
   val counter = new AtomicInteger(10000)
@@ -56,7 +59,7 @@ class MockWorker(master: RpcEndpointRef, conf: SparkConf = new SparkConf) extend
   val id = seq.toString
   override val rpcEnv: RpcEnv = RpcEnv.create("worker", "localhost", seq,
     conf, new SecurityManager(conf))
-  var apps = new mutable.HashMap[String, String]()
+  val apps = new mutable.HashMap[String, String]()
   val driverIdToAppId = new mutable.HashMap[String, String]()
   def newDriver(driverId: String): RpcEndpointRef = {
     val name = s"driver_${drivers.size}"
@@ -70,6 +73,7 @@ class MockWorker(master: RpcEndpointRef, conf: SparkConf = new SparkConf) extend
     })
   }
 
+  var decommissioned = false
   var appDesc = DeployTestUtils.createAppDesc()
   val drivers = mutable.HashSet[String]()
   val driverResources = new mutable.HashMap[String, Map[String, Set[String]]]
@@ -94,6 +98,8 @@ class MockWorker(master: RpcEndpointRef, conf: SparkConf = new SparkConf) extend
         case None =>
       }
       driverIdToAppId.remove(driverId)
+    case DecommissionWorker =>
+      decommissioned = true
   }
 }
 
@@ -129,7 +135,7 @@ class MockExecutorLaunchFailWorker(master: Master, conf: SparkConf = new SparkCo
       assert(master.idToApp.contains(appId))
       appIdsToLaunchExecutor += appId
       failedCnt += 1
-      master.self.send(ExecutorStateChanged(appId, execId, ExecutorState.FAILED, None, None))
+      master.self.askSync(ExecutorStateChanged(appId, execId, ExecutorState.FAILED, None, None))
 
     case otherMsg => super.receive(otherMsg)
   }
@@ -138,6 +144,10 @@ class MockExecutorLaunchFailWorker(master: Master, conf: SparkConf = new SparkCo
 class MasterSuite extends SparkFunSuite
   with Matchers with Eventually with PrivateMethodTester with BeforeAndAfter {
 
+  // regex to extract worker links from the master webui HTML
+  // groups represent URL and worker ID
+  val WORKER_LINK_RE = """<a href="(.+?)">\s*(worker-.+?)\s*</a>""".r
+
   private var _master: Master = _
 
   after {
@@ -315,18 +325,31 @@ class MasterSuite extends SparkFunSuite
     val conf = new SparkConf()
     val localCluster = new LocalSparkCluster(2, 2, 512, conf)
     localCluster.start()
+    val masterUrl = s"http://localhost:${localCluster.masterWebUIPort}"
     try {
       eventually(timeout(5.seconds), interval(100.milliseconds)) {
-        val json = Source.fromURL(s"http://localhost:${localCluster.masterWebUIPort}/json")
-          .getLines().mkString("\n")
+        val json = Utils
+          .tryWithResource(Source.fromURL(s"$masterUrl/json"))(_.getLines().mkString("\n"))
         val JArray(workers) = (parse(json) \ "workers")
         workers.size should be (2)
         workers.foreach { workerSummaryJson =>
           val JString(workerWebUi) = workerSummaryJson \ "webuiaddress"
-          val workerResponse = parse(Source.fromURL(s"${workerWebUi}/json")
-            .getLines().mkString("\n"))
+          val workerResponse = parse(Utils
+            .tryWithResource(Source.fromURL(s"$workerWebUi/json"))(_.getLines().mkString("\n")))
           (workerResponse \ "cores").extract[Int] should be (2)
         }
+
+        val html = Utils
+          .tryWithResource(Source.fromURL(s"$masterUrl/"))(_.getLines().mkString("\n"))
+        html should include ("Spark Master at spark://")
+        val workerLinks = (WORKER_LINK_RE findAllMatchIn html).toList
+        workerLinks.size should be (2)
+        workerLinks foreach { case WORKER_LINK_RE(workerUrl, workerId) =>
+          val workerHtml = Utils
+            .tryWithResource(Source.fromURL(workerUrl))(_.getLines().mkString("\n"))
+          workerHtml should include ("Spark Worker at")
+          workerHtml should include ("Running Executors (0)")
+        }
       }
     } finally {
       localCluster.stop()
@@ -335,31 +358,106 @@ class MasterSuite extends SparkFunSuite
 
   test("master/worker web ui available with reverseProxy") {
     implicit val formats = org.json4s.DefaultFormats
-    val reverseProxyUrl = "http://localhost:8080"
+    val conf = new SparkConf()
+    conf.set(UI_REVERSE_PROXY, true)
+    val localCluster = new LocalSparkCluster(2, 2, 512, conf)
+    localCluster.start()
+    val masterUrl = s"http://localhost:${localCluster.masterWebUIPort}"
+    try {
+      eventually(timeout(5.seconds), interval(100.milliseconds)) {
+        val json = Utils
+          .tryWithResource(Source.fromURL(s"$masterUrl/json"))(_.getLines().mkString("\n"))
+        val JArray(workers) = (parse(json) \ "workers")
+        workers.size should be (2)
+        workers.foreach { workerSummaryJson =>
+          // the webuiaddress intentionally points to the local web ui.
+          // explicitly construct reverse proxy url targeting the master
+          val JString(workerId) = workerSummaryJson \ "id"
+          val url = s"$masterUrl/proxy/${workerId}/json"
+          val workerResponse = parse(
+            Utils.tryWithResource(Source.fromURL(url))(_.getLines().mkString("\n")))
+          (workerResponse \ "cores").extract[Int] should be (2)
+        }
+
+        val html = Utils
+          .tryWithResource(Source.fromURL(s"$masterUrl/"))(_.getLines().mkString("\n"))
+        html should include ("Spark Master at spark://")
+        html should include ("""href="/static""")
+        html should include ("""src="/static""")
+        verifyWorkerUI(html, masterUrl)
+      }
+    } finally {
+      localCluster.stop()
+      System.getProperties().remove("spark.ui.proxyBase")
+    }
+  }
+
+  test("master/worker web ui available behind front-end reverseProxy") {
+    implicit val formats = org.json4s.DefaultFormats
+    val reverseProxyUrl = "http://proxyhost:8080/path/to/spark"
     val conf = new SparkConf()
     conf.set(UI_REVERSE_PROXY, true)
     conf.set(UI_REVERSE_PROXY_URL, reverseProxyUrl)
     val localCluster = new LocalSparkCluster(2, 2, 512, conf)
     localCluster.start()
+    val masterUrl = s"http://localhost:${localCluster.masterWebUIPort}"
     try {
       eventually(timeout(5.seconds), interval(100.milliseconds)) {
-        val json = Source.fromURL(s"http://localhost:${localCluster.masterWebUIPort}/json")
-          .getLines().mkString("\n")
+        val json = Utils
+          .tryWithResource(Source.fromURL(s"$masterUrl/json"))(_.getLines().mkString("\n"))
         val JArray(workers) = (parse(json) \ "workers")
         workers.size should be (2)
         workers.foreach { workerSummaryJson =>
+          // the webuiaddress intentionally points to the local web ui.
+          // explicitly construct reverse proxy url targeting the master
           val JString(workerId) = workerSummaryJson \ "id"
-          val url = s"http://localhost:${localCluster.masterWebUIPort}/proxy/${workerId}/json"
-          val workerResponse = parse(Source.fromURL(url).getLines().mkString("\n"))
+          val url = s"$masterUrl/proxy/${workerId}/json"
+          val workerResponse = parse(Utils
+            .tryWithResource(Source.fromURL(url))(_.getLines().mkString("\n")))
           (workerResponse \ "cores").extract[Int] should be (2)
-          (workerResponse \ "masterwebuiurl").extract[String] should be (reverseProxyUrl)
+          (workerResponse \ "masterwebuiurl").extract[String] should be (reverseProxyUrl + "/")
         }
+
+        System.getProperty("spark.ui.proxyBase") should be (reverseProxyUrl)
+        val html = Utils
+          .tryWithResource(Source.fromURL(s"$masterUrl/"))(_.getLines().mkString("\n"))
+        html should include ("Spark Master at spark://")
+        verifyStaticResourcesServedByProxy(html, reverseProxyUrl)
+        verifyWorkerUI(html, masterUrl, reverseProxyUrl)
       }
     } finally {
       localCluster.stop()
+      System.getProperties().remove("spark.ui.proxyBase")
     }
   }
 
+  private def verifyWorkerUI(masterHtml: String, masterUrl: String,
+      reverseProxyUrl: String = ""): Unit = {
+    val workerLinks = (WORKER_LINK_RE findAllMatchIn masterHtml).toList
+    workerLinks.size should be (2)
+    workerLinks foreach {
+      case WORKER_LINK_RE(workerUrl, workerId) =>
+        workerUrl should be (s"$reverseProxyUrl/proxy/$workerId")
+        // there is no real front-end proxy as defined in $reverseProxyUrl
+        // construct url directly targeting the master
+        val url = s"$masterUrl/proxy/$workerId/"
+        System.setProperty("spark.ui.proxyBase", workerUrl)
+        val workerHtml = Utils
+          .tryWithResource(Source.fromURL(url))(_.getLines().mkString("\n"))
+        workerHtml should include ("Spark Worker at")
+        workerHtml should include ("Running Executors (0)")
+        verifyStaticResourcesServedByProxy(workerHtml, workerUrl)
+      case _ => fail  // make sure we don't accidentially skip the tests
+    }
+  }
+
+  private def verifyStaticResourcesServedByProxy(html: String, proxyUrl: String): Unit = {
+    html should not include ("""href="/static""")
+    html should include (s"""href="$proxyUrl/static""")
+    html should not include ("""src="/static""")
+    html should include (s"""src="$proxyUrl/static""")
+  }
+
   test("basic scheduling - spread out") {
     basicScheduling(spreadOut = true)
   }
@@ -689,7 +787,16 @@ class MasterSuite extends SparkFunSuite
     val master = makeAliveMaster()
     var worker: MockExecutorLaunchFailWorker = null
     try {
-      worker = new MockExecutorLaunchFailWorker(master)
+      val conf = new SparkConf()
+      // SPARK-32250: When running test on GitHub Action machine, the available processors in JVM
+      // is only 2, while on Jenkins it's 32. For this specific test, 2 available processors, which
+      // also decides number of threads in Dispatcher, is not enough to consume the messages. In
+      // the worst situation, MockExecutorLaunchFailWorker would occupy these 2 threads for
+      // handling messages LaunchDriver, LaunchExecutor at the same time but leave no thread for
+      // the driver to handle the message RegisteredApplication. At the end, it results in the dead
+      // lock situation. Therefore, we need to set more threads to avoid the dead lock.
+      conf.set(Network.RPC_NETTY_DISPATCHER_NUM_THREADS, 6)
+      worker = new MockExecutorLaunchFailWorker(master, conf)
       worker.rpcEnv.setupEndpoint("worker", worker)
       val workerRegMsg = RegisterWorker(
         worker.id,
@@ -725,6 +832,68 @@ class MasterSuite extends SparkFunSuite
     }
   }
 
+  def testWorkerDecommissioning(
+      numWorkers: Int,
+      numWorkersExpectedToDecom: Int,
+      hostnames: Seq[String]): Unit = {
+    val conf = new SparkConf()
+    val master = makeAliveMaster(conf)
+    val workers = (1 to numWorkers).map { idx =>
+      val worker = new MockWorker(master.self, conf)
+      worker.rpcEnv.setupEndpoint(s"worker-$idx", worker)
+      val workerReg = RegisterWorker(
+        worker.id,
+        "localhost",
+        worker.self.address.port,
+        worker.self,
+        10,
+        1024,
+        "http://localhost:8080",
+        RpcAddress("localhost", 10000))
+      master.self.send(workerReg)
+      worker
+    }
+
+    eventually(timeout(10.seconds)) {
+      val masterState = master.self.askSync[MasterStateResponse](RequestMasterState)
+      assert(masterState.workers.length === numWorkers)
+      assert(masterState.workers.forall(_.state == WorkerState.ALIVE))
+      assert(masterState.workers.map(_.id).toSet == workers.map(_.id).toSet)
+    }
+
+    val decomWorkersCount = master.self.askSync[Integer](DecommissionWorkersOnHosts(hostnames))
+    assert(decomWorkersCount === numWorkersExpectedToDecom)
+
+    // Decommissioning is actually async ... wait for the workers to actually be decommissioned by
+    // polling the master's state.
+    eventually(timeout(30.seconds)) {
+      val masterState = master.self.askSync[MasterStateResponse](RequestMasterState)
+      assert(masterState.workers.length === numWorkers)
+      val workersActuallyDecomed = masterState.workers
+        .filter(_.state == WorkerState.DECOMMISSIONED).map(_.id)
+      val decommissionedWorkers = workers.filter(w => workersActuallyDecomed.contains(w.id))
+      assert(workersActuallyDecomed.length === numWorkersExpectedToDecom)
+      assert(decommissionedWorkers.forall(_.decommissioned))
+    }
+
+    // Decommissioning a worker again should return the same answer since we want this call to be
+    // idempotent.
+    val decomWorkersCountAgain = master.self.askSync[Integer](DecommissionWorkersOnHosts(hostnames))
+    assert(decomWorkersCountAgain === numWorkersExpectedToDecom)
+  }
+
+  test("All workers on a host should be decommissioned") {
+    testWorkerDecommissioning(2, 2, Seq("LoCalHost", "localHOST"))
+  }
+
+  test("No workers should be decommissioned with invalid host") {
+    testWorkerDecommissioning(2, 0, Seq("NoSuchHost1", "NoSuchHost2"))
+  }
+
+  test("Only worker on host should be decommissioned") {
+    testWorkerDecommissioning(1, 1, Seq("lOcalHost", "NoSuchHost"))
+  }
+
   test("SPARK-19900: there should be a corresponding driver for the app after relaunching driver") {
     val conf = new SparkConf().set(WORKER_TIMEOUT, 1L)
     val master = makeAliveMaster(conf)
diff --git a/core/src/test/scala/org/apache/spark/deploy/master/ui/MasterWebUISuite.scala b/core/src/test/scala/org/apache/spark/deploy/master/ui/MasterWebUISuite.scala
index e2d7facdd77e0..be83ec12f92f5 100644
--- a/core/src/test/scala/org/apache/spark/deploy/master/ui/MasterWebUISuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/master/ui/MasterWebUISuite.scala
@@ -28,7 +28,7 @@ import org.mockito.Mockito.{mock, times, verify, when}
 import org.scalatest.BeforeAndAfterAll
 
 import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
-import org.apache.spark.deploy.DeployMessages.{KillDriverResponse, RequestKillDriver}
+import org.apache.spark.deploy.DeployMessages.{DecommissionWorkersOnHosts, KillDriverResponse, RequestKillDriver}
 import org.apache.spark.deploy.DeployTestUtils._
 import org.apache.spark.deploy.master._
 import org.apache.spark.rpc.{RpcEndpointRef, RpcEnv}
@@ -36,7 +36,7 @@ import org.apache.spark.rpc.{RpcEndpointRef, RpcEnv}
 
 class MasterWebUISuite extends SparkFunSuite with BeforeAndAfterAll {
 
-  val conf = new SparkConf
+  val conf = new SparkConf()
   val securityMgr = new SecurityManager(conf)
   val rpcEnv = mock(classOf[RpcEnv])
   val master = mock(classOf[Master])
@@ -88,10 +88,32 @@ class MasterWebUISuite extends SparkFunSuite with BeforeAndAfterAll {
     verify(masterEndpointRef, times(1)).ask[KillDriverResponse](RequestKillDriver(activeDriverId))
   }
 
-  private def convPostDataToString(data: Map[String, String]): String = {
+  private def testKillWorkers(hostnames: Seq[String]): Unit = {
+    val url = s"http://localhost:${masterWebUI.boundPort}/workers/kill/"
+    val body = convPostDataToString(hostnames.map(("host", _)))
+    val conn = sendHttpRequest(url, "POST", body)
+    // The master is mocked here, so cannot assert on the response code
+    conn.getResponseCode
+    // Verify that master was asked to kill driver with the correct id
+    verify(masterEndpointRef).askSync[Integer](DecommissionWorkersOnHosts(hostnames))
+  }
+
+  test("Kill one host") {
+    testKillWorkers(Seq("localhost"))
+  }
+
+  test("Kill multiple hosts") {
+    testKillWorkers(Seq("noSuchHost", "LocalHost"))
+  }
+
+  private def convPostDataToString(data: Seq[(String, String)]): String = {
     (for ((name, value) <- data) yield s"$name=$value").mkString("&")
   }
 
+  private def convPostDataToString(data: Map[String, String]): String = {
+    convPostDataToString(data.toSeq)
+  }
+
   /**
    * Send an HTTP request to the given URL using the method and the body specified.
    * Return the connection object.
diff --git a/core/src/test/scala/org/apache/spark/deploy/rest/SubmitRestProtocolSuite.scala b/core/src/test/scala/org/apache/spark/deploy/rest/SubmitRestProtocolSuite.scala
index 03102fd8c696c..9fdbf485e17d3 100644
--- a/core/src/test/scala/org/apache/spark/deploy/rest/SubmitRestProtocolSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/rest/SubmitRestProtocolSuite.scala
@@ -19,6 +19,8 @@ package org.apache.spark.deploy.rest
 
 import java.lang.Boolean
 
+import scala.util.Properties.versionNumberString
+
 import org.json4s.jackson.JsonMethods._
 
 import org.apache.spark.{SparkConf, SparkFunSuite}
@@ -96,6 +98,7 @@ class SubmitRestProtocolSuite extends SparkFunSuite {
     // optional fields
     conf.set(JARS, Seq("mayonnaise.jar", "ketchup.jar"))
     conf.set(FILES.key, "fireball.png")
+    conf.set(ARCHIVES.key, "fireballs.zip")
     conf.set("spark.driver.memory", s"${Utils.DEFAULT_DRIVER_MEM_MB}m")
     conf.set(DRIVER_CORES, 180)
     conf.set("spark.driver.extraJavaOptions", " -Dslices=5 -Dcolor=mostly_red")
@@ -232,7 +235,7 @@ class SubmitRestProtocolSuite extends SparkFunSuite {
       |}
     """.stripMargin
 
-  private val submitDriverRequestJson =
+  private lazy val submitDriverRequestJson = if (versionNumberString.startsWith("2.12")) {
     s"""
       |{
       |  "action" : "CreateSubmissionRequest",
@@ -244,6 +247,7 @@ class SubmitRestProtocolSuite extends SparkFunSuite {
       |  },
       |  "mainClass" : "org.apache.spark.examples.SparkPie",
       |  "sparkProperties" : {
+      |    "spark.archives" : "fireballs.zip",
       |    "spark.driver.extraLibraryPath" : "pickle.jar",
       |    "spark.jars" : "mayonnaise.jar,ketchup.jar",
       |    "spark.driver.supervise" : "false",
@@ -258,6 +262,34 @@ class SubmitRestProtocolSuite extends SparkFunSuite {
       |  }
       |}
     """.stripMargin
+  } else {
+    s"""
+      |{
+      |  "action" : "CreateSubmissionRequest",
+      |  "appArgs" : [ "two slices", "a hint of cinnamon" ],
+      |  "appResource" : "honey-walnut-cherry.jar",
+      |  "clientSparkVersion" : "1.2.3",
+      |  "environmentVariables" : {
+      |    "PATH" : "/dev/null"
+      |  },
+      |  "mainClass" : "org.apache.spark.examples.SparkPie",
+      |  "sparkProperties" : {
+      |    "spark.archives" : "fireballs.zip",
+      |    "spark.driver.extraLibraryPath" : "pickle.jar",
+      |    "spark.jars" : "mayonnaise.jar,ketchup.jar",
+      |    "spark.driver.supervise" : "false",
+      |    "spark.driver.memory" : "${Utils.DEFAULT_DRIVER_MEM_MB}m",
+      |    "spark.files" : "fireball.png",
+      |    "spark.driver.cores" : "180",
+      |    "spark.driver.extraJavaOptions" : " -Dslices=5 -Dcolor=mostly_red",
+      |    "spark.app.name" : "SparkPie",
+      |    "spark.cores.max" : "10000",
+      |    "spark.executor.memory" : "256m",
+      |    "spark.driver.extraClassPath" : "food-coloring.jar"
+      |  }
+      |}
+    """.stripMargin
+  }
 
   private val submitDriverResponseJson =
     """
diff --git a/core/src/test/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProviderSuite.scala b/core/src/test/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProviderSuite.scala
index 44f38e7043dcd..2cfc60f927e1c 100644
--- a/core/src/test/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProviderSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProviderSuite.scala
@@ -19,7 +19,8 @@ package org.apache.spark.deploy.security
 
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.Path
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.internal.config.{STAGING_DIR, SUBMIT_DEPLOY_MODE}
diff --git a/core/src/test/scala/org/apache/spark/deploy/worker/CommandUtilsSuite.scala b/core/src/test/scala/org/apache/spark/deploy/worker/CommandUtilsSuite.scala
index 2d3cc5d3abd65..28e35bc8183ba 100644
--- a/core/src/test/scala/org/apache/spark/deploy/worker/CommandUtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/worker/CommandUtilsSuite.scala
@@ -17,7 +17,9 @@
 
 package org.apache.spark.deploy.worker
 
-import org.scalatest.{Matchers, PrivateMethodTester}
+import org.scalatest.PrivateMethodTester
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
 import org.apache.spark.deploy.Command
diff --git a/core/src/test/scala/org/apache/spark/deploy/worker/DriverRunnerTest.scala b/core/src/test/scala/org/apache/spark/deploy/worker/DriverRunnerTest.scala
index c3b580e7ccac4..e429ddfd570de 100644
--- a/core/src/test/scala/org/apache/spark/deploy/worker/DriverRunnerTest.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/worker/DriverRunnerTest.scala
@@ -40,7 +40,8 @@ class DriverRunnerTest extends SparkFunSuite {
     val worker = mock(classOf[RpcEndpointRef])
     doNothing().when(worker).send(any())
     spy(new DriverRunner(conf, "driverId", new File("workDir"), new File("sparkHome"),
-      driverDescription, worker, "spark://1.2.3.4/worker/", new SecurityManager(conf)))
+      driverDescription, worker, "spark://1.2.3.4/worker/", "http://publicAddress:80",
+      new SecurityManager(conf)))
   }
 
   private def createProcessBuilderAndProcess(): (ProcessBuilderLike, Process) = {
diff --git a/core/src/test/scala/org/apache/spark/deploy/worker/WorkerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/worker/WorkerSuite.scala
index 2d3d0afe3f80c..e24ff051ee339 100644
--- a/core/src/test/scala/org/apache/spark/deploy/worker/WorkerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/worker/WorkerSuite.scala
@@ -29,8 +29,10 @@ import org.mockito.Answers.RETURNS_SMART_NULLS
 import org.mockito.ArgumentMatchers.any
 import org.mockito.Mockito._
 import org.mockito.invocation.InvocationOnMock
-import org.scalatest.{BeforeAndAfter, Matchers}
+import org.scalatest.BeforeAndAfter
 import org.scalatest.concurrent.Eventually.{eventually, interval, timeout}
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
 import org.apache.spark.TestUtils.{createTempJsonFile, createTempScriptWithExpectedOutput}
@@ -80,7 +82,7 @@ class WorkerSuite extends SparkFunSuite with Matchers with BeforeAndAfter {
   }
 
   before {
-    MockitoAnnotations.initMocks(this)
+    MockitoAnnotations.openMocks(this).close()
   }
 
   after {
@@ -340,7 +342,7 @@ class WorkerSuite extends SparkFunSuite with Matchers with BeforeAndAfter {
     testWorkDirCleanupAndRemoveMetadataWithConfig(true)
   }
 
-  test("WorkdDirCleanup cleans only app dirs when" +
+  test("WorkDirCleanup cleans only app dirs when" +
     "spark.shuffle.service.db.enabled=false") {
     testWorkDirCleanupAndRemoveMetadataWithConfig(false)
   }
diff --git a/core/src/test/scala/org/apache/spark/executor/CoarseGrainedExecutorBackendSuite.scala b/core/src/test/scala/org/apache/spark/executor/CoarseGrainedExecutorBackendSuite.scala
index e0b586074b89e..4909a586d31ae 100644
--- a/core/src/test/scala/org/apache/spark/executor/CoarseGrainedExecutorBackendSuite.scala
+++ b/core/src/test/scala/org/apache/spark/executor/CoarseGrainedExecutorBackendSuite.scala
@@ -61,7 +61,7 @@ class CoarseGrainedExecutorBackendSuite extends SparkFunSuite
       val testResourceArgs: JObject = ("" -> "")
       val ja = JArray(List(testResourceArgs))
       val f1 = createTempJsonFile(tmpDir, "resources", ja)
-      var error = intercept[SparkException] {
+      val error = intercept[SparkException] {
         val parsedResources = backend.parseOrFindResources(Some(f1))
       }.getMessage()
 
@@ -106,7 +106,7 @@ class CoarseGrainedExecutorBackendSuite extends SparkFunSuite
     testParsingMultipleResources(conf, ResourceProfile.getOrCreateDefaultProfile(conf))
   }
 
-  def testParsingMultipleResources(conf: SparkConf, resourceProfile: ResourceProfile) {
+  def testParsingMultipleResources(conf: SparkConf, resourceProfile: ResourceProfile): Unit = {
     val serializer = new JavaSerializer(conf)
     val env = createMockEnv(conf, serializer)
     // we don't really use this, just need it to get at the parser function
@@ -146,7 +146,7 @@ class CoarseGrainedExecutorBackendSuite extends SparkFunSuite
       val ja = Extraction.decompose(Seq(gpuArgs))
       val f1 = createTempJsonFile(tmpDir, "resources", ja)
 
-      var error = intercept[IllegalArgumentException] {
+      val error = intercept[IllegalArgumentException] {
         val parsedResources = backend.parseOrFindResources(Some(f1))
       }.getMessage()
 
@@ -160,7 +160,7 @@ class CoarseGrainedExecutorBackendSuite extends SparkFunSuite
       val ja = Extraction.decompose(Seq(fpga))
       val f1 = createTempJsonFile(tmpDir, "resources", ja)
 
-      var error = intercept[SparkException] {
+      val error = intercept[SparkException] {
         val parsedResources = backend.parseOrFindResources(Some(f1))
       }.getMessage()
 
@@ -199,7 +199,7 @@ class CoarseGrainedExecutorBackendSuite extends SparkFunSuite
       val ja = Extraction.decompose(Seq(gpuArgs))
       val f1 = createTempJsonFile(tmpDir, "resources", ja)
 
-      var error = intercept[IllegalArgumentException] {
+      val error = intercept[IllegalArgumentException] {
         val parsedResources = backend.parseOrFindResources(Some(f1))
       }.getMessage()
 
@@ -302,7 +302,7 @@ class CoarseGrainedExecutorBackendSuite extends SparkFunSuite
       // We don't really verify the data, just pass it around.
       val data = ByteBuffer.wrap(Array[Byte](1, 2, 3, 4))
       val taskDescription = new TaskDescription(taskId, 2, "1", "TASK 1000000", 19,
-        1, mutable.Map.empty, mutable.Map.empty, new Properties,
+        1, mutable.Map.empty, mutable.Map.empty, mutable.Map.empty, new Properties,
         Map(GPU -> new ResourceInformation(GPU, Array("0", "1"))), data)
       val serializedTaskDescription = TaskDescription.encode(taskDescription)
       backend.executor = mock[Executor]
diff --git a/core/src/test/scala/org/apache/spark/executor/ExecutorMetricsPollerSuite.scala b/core/src/test/scala/org/apache/spark/executor/ExecutorMetricsPollerSuite.scala
new file mode 100644
index 0000000000000..e471864ae240f
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/executor/ExecutorMetricsPollerSuite.scala
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.executor
+
+import org.apache.spark.{SparkConf, SparkFunSuite}
+import org.apache.spark.memory.TestMemoryManager
+
+class ExecutorMetricsPollerSuite extends SparkFunSuite {
+
+  test("SPARK-34779: stage entry shouldn't be removed before a heartbeat occurs") {
+    val testMemoryManager = new TestMemoryManager(new SparkConf())
+    val poller = new ExecutorMetricsPoller(testMemoryManager, 1000, None)
+
+    poller.onTaskStart(0L, 0, 0)
+    // stage (0, 0) has an active task, so it remains on stageTCMP after heartbeat.
+    assert(poller.getExecutorUpdates.size === 1)
+    assert(poller.stageTCMP.size === 1)
+    assert(poller.stageTCMP.get((0, 0)).count.get === 1)
+
+    poller.onTaskCompletion(0L, 0, 0)
+    // stage (0, 0) doesn't have active tasks, but its entry will be kept until next
+    // heartbeat.
+    assert(poller.stageTCMP.size === 1)
+    assert(poller.stageTCMP.get((0, 0)).count.get === 0)
+
+    // the next heartbeat will report the peak metrics of stage (0, 0) during the
+    // previous heartbeat interval, then remove it from stageTCMP.
+    assert(poller.getExecutorUpdates.size === 1)
+    assert(poller.stageTCMP.size === 0)
+
+    poller.stop
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
index 31049d104e63d..a237447b0fa2d 100644
--- a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.executor
 
 import java.io.{Externalizable, ObjectInput, ObjectOutput}
 import java.lang.Thread.UncaughtExceptionHandler
+import java.net.URL
 import java.nio.ByteBuffer
 import java.util.Properties
 import java.util.concurrent.{ConcurrentHashMap, CountDownLatch, TimeUnit}
@@ -28,6 +29,7 @@ import scala.collection.immutable
 import scala.collection.mutable.{ArrayBuffer, Map}
 import scala.concurrent.duration._
 
+import com.google.common.cache.{CacheBuilder, CacheLoader}
 import org.mockito.ArgumentCaptor
 import org.mockito.ArgumentMatchers.{any, eq => meq}
 import org.mockito.Mockito.{inOrder, verify, when}
@@ -43,7 +45,7 @@ import org.apache.spark.TaskState.TaskState
 import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.internal.config._
 import org.apache.spark.internal.config.UI._
-import org.apache.spark.memory.TestMemoryManager
+import org.apache.spark.memory.{SparkOutOfMemoryError, TestMemoryManager}
 import org.apache.spark.metrics.MetricsSystem
 import org.apache.spark.rdd.RDD
 import org.apache.spark.resource.ResourceInformation
@@ -52,7 +54,7 @@ import org.apache.spark.scheduler.{DirectTaskResult, FakeTask, ResultTask, Task,
 import org.apache.spark.serializer.{JavaSerializer, SerializerInstance, SerializerManager}
 import org.apache.spark.shuffle.FetchFailedException
 import org.apache.spark.storage.{BlockManager, BlockManagerId}
-import org.apache.spark.util.{LongAccumulator, UninterruptibleThread}
+import org.apache.spark.util.{LongAccumulator, SparkUncaughtExceptionHandler, ThreadUtils, UninterruptibleThread}
 
 class ExecutorSuite extends SparkFunSuite
     with LocalSparkContext with MockitoSugar with Eventually with PrivateMethodTester {
@@ -63,6 +65,33 @@ class ExecutorSuite extends SparkFunSuite
     super.afterEach()
   }
 
+  /**
+   * Creates an Executor with the provided arguments, is then passed to `f`
+   * and will be stopped after `f` returns.
+   */
+  def withExecutor(
+      executorId: String,
+      executorHostname: String,
+      env: SparkEnv,
+      userClassPath: Seq[URL] = Nil,
+      isLocal: Boolean = true,
+      uncaughtExceptionHandler: UncaughtExceptionHandler
+        = new SparkUncaughtExceptionHandler,
+      resources: immutable.Map[String, ResourceInformation]
+        = immutable.Map.empty[String, ResourceInformation])(f: Executor => Unit): Unit = {
+    var executor: Executor = null
+    try {
+      executor = new Executor(executorId, executorHostname, env, userClassPath, isLocal,
+        uncaughtExceptionHandler, resources)
+
+      f(executor)
+    } finally {
+      if (executor != null) {
+        executor.stop()
+      }
+    }
+  }
+
   test("SPARK-15963: Catch `TaskKilledException` correctly in Executor.TaskRunner") {
     // mock some objects to make Executor.launchTask() happy
     val conf = new SparkConf
@@ -115,10 +144,8 @@ class ExecutorSuite extends SparkFunSuite
         }
       })
 
-    var executor: Executor = null
-    try {
-      executor = new Executor("id", "localhost", env, userClassPath = Nil, isLocal = true,
-        resources = immutable.Map.empty[String, ResourceInformation])
+    withExecutor("id", "localhost", env) { executor =>
+
       // the task will be launched in a dedicated worker thread
       executor.launchTask(mockExecutorBackend, taskDescription)
 
@@ -138,11 +165,6 @@ class ExecutorSuite extends SparkFunSuite
       assert(executorSuiteHelper.testFailedReason.toErrorString === "TaskKilled (test)")
       assert(executorSuiteHelper.taskState === TaskState.KILLED)
     }
-    finally {
-      if (executor != null) {
-        executor.stop()
-      }
-    }
   }
 
   test("SPARK-19276: Handle FetchFailedExceptions that are hidden by user exceptions") {
@@ -248,31 +270,37 @@ class ExecutorSuite extends SparkFunSuite
     heartbeatZeroAccumulatorUpdateTest(false)
   }
 
+  private def withMockHeartbeatReceiverRef(executor: Executor)
+      (func: RpcEndpointRef => Unit): Unit = {
+    val executorClass = classOf[Executor]
+    val mockReceiverRef = mock[RpcEndpointRef]
+    val receiverRef = executorClass.getDeclaredField("heartbeatReceiverRef")
+    receiverRef.setAccessible(true)
+    receiverRef.set(executor, mockReceiverRef)
+
+    func(mockReceiverRef)
+  }
+
   private def withHeartbeatExecutor(confs: (String, String)*)
       (f: (Executor, ArrayBuffer[Heartbeat]) => Unit): Unit = {
     val conf = new SparkConf
     confs.foreach { case (k, v) => conf.set(k, v) }
     val serializer = new JavaSerializer(conf)
     val env = createMockEnv(conf, serializer)
-    val executor =
-      new Executor("id", "localhost", SparkEnv.get, userClassPath = Nil, isLocal = true,
-        resources = immutable.Map.empty[String, ResourceInformation])
-    val executorClass = classOf[Executor]
-
-    // Save all heartbeats sent into an ArrayBuffer for verification
-    val heartbeats = ArrayBuffer[Heartbeat]()
-    val mockReceiver = mock[RpcEndpointRef]
-    when(mockReceiver.askSync(any[Heartbeat], any[RpcTimeout])(any))
-      .thenAnswer((invocation: InvocationOnMock) => {
-        val args = invocation.getArguments()
-        heartbeats += args(0).asInstanceOf[Heartbeat]
-        HeartbeatResponse(false)
-      })
-    val receiverRef = executorClass.getDeclaredField("heartbeatReceiverRef")
-    receiverRef.setAccessible(true)
-    receiverRef.set(executor, mockReceiver)
-
-    f(executor, heartbeats)
+    withExecutor("id", "localhost", SparkEnv.get) { executor =>
+      withMockHeartbeatReceiverRef(executor) { mockReceiverRef =>
+        // Save all heartbeats sent into an ArrayBuffer for verification
+        val heartbeats = ArrayBuffer[Heartbeat]()
+        when(mockReceiverRef.askSync(any[Heartbeat], any[RpcTimeout])(any))
+          .thenAnswer((invocation: InvocationOnMock) => {
+            val args = invocation.getArguments()
+            heartbeats += args(0).asInstanceOf[Heartbeat]
+            HeartbeatResponse(false)
+          })
+
+        f(executor, heartbeats)
+      }
+    }
   }
 
   private def heartbeatZeroAccumulatorUpdateTest(dropZeroMetrics: Boolean): Unit = {
@@ -353,10 +381,7 @@ class ExecutorSuite extends SparkFunSuite
     val taskDescription = createResultTaskDescription(serializer, taskBinary, rdd, 0)
 
     val mockBackend = mock[ExecutorBackend]
-    var executor: Executor = null
-    try {
-      executor = new Executor("id", "localhost", SparkEnv.get, userClassPath = Nil, isLocal = true,
-        resources = immutable.Map.empty[String, ResourceInformation])
+    withExecutor("id", "localhost", SparkEnv.get) { executor =>
       executor.launchTask(mockBackend, taskDescription)
 
       // Ensure that the executor's metricsPoller is polled so that values are recorded for
@@ -367,10 +392,6 @@ class ExecutorSuite extends SparkFunSuite
       eventually(timeout(5.seconds), interval(10.milliseconds)) {
         assert(executor.numRunningTasks === 0)
       }
-    } finally {
-      if (executor != null) {
-        executor.stop()
-      }
     }
 
     // Verify that peak values for task metrics get sent in the TaskResult
@@ -402,6 +423,103 @@ class ExecutorSuite extends SparkFunSuite
     assert(taskMetrics.getMetricValue("JVMHeapMemory") > 0)
   }
 
+  test("SPARK-34949: do not re-register BlockManager when executor is shutting down") {
+    val reregisterInvoked = new AtomicBoolean(false)
+    val mockBlockManager = mock[BlockManager]
+    when(mockBlockManager.reregister()).thenAnswer { (_: InvocationOnMock) =>
+      reregisterInvoked.getAndSet(true)
+    }
+    val conf = new SparkConf(false).setAppName("test").setMaster("local[2]")
+    val mockEnv = createMockEnv(conf, new JavaSerializer(conf))
+    when(mockEnv.blockManager).thenReturn(mockBlockManager)
+
+    withExecutor("id", "localhost", mockEnv) { executor =>
+      withMockHeartbeatReceiverRef(executor) { mockReceiverRef =>
+        when(mockReceiverRef.askSync(any[Heartbeat], any[RpcTimeout])(any)).thenAnswer {
+          (_: InvocationOnMock) => HeartbeatResponse(reregisterBlockManager = true)
+        }
+        val reportHeartbeat = PrivateMethod[Unit](Symbol("reportHeartBeat"))
+        executor.invokePrivate(reportHeartbeat())
+        assert(reregisterInvoked.get(), "BlockManager.reregister should be invoked " +
+          "on HeartbeatResponse(reregisterBlockManager = true) when executor is not shutting down")
+
+        reregisterInvoked.getAndSet(false)
+        executor.stop()
+        executor.invokePrivate(reportHeartbeat())
+        assert(!reregisterInvoked.get(),
+          "BlockManager.reregister should not be invoked when executor is shutting down")
+      }
+    }
+  }
+
+  test("SPARK-33587: isFatalError") {
+    def errorInThreadPool(e: => Throwable): Throwable = {
+      intercept[Throwable] {
+        val taskPool = ThreadUtils.newDaemonFixedThreadPool(1, "test")
+        try {
+          val f = taskPool.submit(new java.util.concurrent.Callable[String] {
+            override def call(): String = throw e
+          })
+          f.get()
+        } finally {
+          taskPool.shutdown()
+        }
+      }
+    }
+
+    def errorInGuavaCache(e: => Throwable): Throwable = {
+      val cache = CacheBuilder.newBuilder()
+        .build(new CacheLoader[String, String] {
+          override def load(key: String): String = throw e
+        })
+      intercept[Throwable] {
+        cache.get("test")
+      }
+    }
+
+    def testThrowable(
+        e: => Throwable,
+        depthToCheck: Int,
+        isFatal: Boolean): Unit = {
+      import Executor.isFatalError
+      // `e`'s depth is 1 so `depthToCheck` needs to be at least 3 to detect fatal errors.
+      assert(isFatalError(e, depthToCheck) == (depthToCheck >= 1 && isFatal))
+      // `e`'s depth is 2 so `depthToCheck` needs to be at least 3 to detect fatal errors.
+      assert(isFatalError(errorInThreadPool(e), depthToCheck) == (depthToCheck >= 2 && isFatal))
+      assert(isFatalError(errorInGuavaCache(e), depthToCheck) == (depthToCheck >= 2 && isFatal))
+      assert(isFatalError(
+        new SparkException("foo", e),
+        depthToCheck) == (depthToCheck >= 2 && isFatal))
+      // `e`'s depth is 3 so `depthToCheck` needs to be at least 3 to detect fatal errors.
+      assert(isFatalError(
+        errorInThreadPool(errorInGuavaCache(e)),
+        depthToCheck) == (depthToCheck >= 3 && isFatal))
+      assert(isFatalError(
+        errorInGuavaCache(errorInThreadPool(e)),
+        depthToCheck) == (depthToCheck >= 3 && isFatal))
+      assert(isFatalError(
+        new SparkException("foo", new SparkException("foo", e)),
+        depthToCheck) == (depthToCheck >= 3 && isFatal))
+    }
+
+    for (depthToCheck <- 0 to 5) {
+      testThrowable(new OutOfMemoryError(), depthToCheck, isFatal = true)
+      testThrowable(new InterruptedException(), depthToCheck, isFatal = false)
+      testThrowable(new RuntimeException("test"), depthToCheck, isFatal = false)
+      testThrowable(new SparkOutOfMemoryError("test"), depthToCheck, isFatal = false)
+    }
+
+    // Verify we can handle the cycle in the exception chain
+    val e1 = new Exception("test1")
+    val e2 = new Exception("test2")
+    e1.initCause(e2)
+    e2.initCause(e1)
+    for (depthToCheck <- 0 to 5) {
+      testThrowable(e1, depthToCheck, isFatal = false)
+      testThrowable(e2, depthToCheck, isFatal = false)
+    }
+  }
+
   private def createMockEnv(conf: SparkConf, serializer: JavaSerializer): SparkEnv = {
     val mockEnv = mock[SparkEnv]
     val mockRpcEnv = mock[RpcEnv]
@@ -450,6 +568,7 @@ class ExecutorSuite extends SparkFunSuite
       partitionId = 0,
       addedFiles = Map[String, Long](),
       addedJars = Map[String, Long](),
+      addedArchives = Map[String, Long](),
       properties = new Properties,
       resources = immutable.Map[String, ResourceInformation](),
       serializedTask)
@@ -465,12 +584,11 @@ class ExecutorSuite extends SparkFunSuite
       poll: Boolean = false): (TaskFailedReason, UncaughtExceptionHandler) = {
     val mockBackend = mock[ExecutorBackend]
     val mockUncaughtExceptionHandler = mock[UncaughtExceptionHandler]
-    var executor: Executor = null
     val timedOut = new AtomicBoolean(false)
-    try {
-      executor = new Executor("id", "localhost", SparkEnv.get, userClassPath = Nil, isLocal = true,
-        uncaughtExceptionHandler = mockUncaughtExceptionHandler,
-        resources = immutable.Map.empty[String, ResourceInformation])
+
+    withExecutor("id", "localhost", SparkEnv.get,
+        uncaughtExceptionHandler = mockUncaughtExceptionHandler) { executor =>
+
       // the task will be launched in a dedicated worker thread
       executor.launchTask(mockBackend, taskDescription)
       if (killTask) {
@@ -483,7 +601,7 @@ class ExecutorSuite extends SparkFunSuite
               if (poll) {
                 executor.metricsPoller.poll()
               }
-              executor.killAllTasks(true, "Killed task, eg. because of speculative execution")
+              executor.killAllTasks(true, "Killed task, e.g. because of speculative execution")
             } else {
               timedOut.set(true)
             }
@@ -503,11 +621,8 @@ class ExecutorSuite extends SparkFunSuite
         assert(executor.numRunningTasks === 0)
       }
       assert(!timedOut.get(), "timed out waiting to be ready to kill tasks")
-    } finally {
-      if (executor != null) {
-        executor.stop()
-      }
     }
+
     val orderedMock = inOrder(mockBackend)
     val statusCaptor = ArgumentCaptor.forClass(classOf[ByteBuffer])
     orderedMock.verify(mockBackend)
diff --git a/core/src/test/scala/org/apache/spark/executor/ProcfsMetricsGetterSuite.scala b/core/src/test/scala/org/apache/spark/executor/ProcfsMetricsGetterSuite.scala
index 9836697e1647c..ff0374da1bcfe 100644
--- a/core/src/test/scala/org/apache/spark/executor/ProcfsMetricsGetterSuite.scala
+++ b/core/src/test/scala/org/apache/spark/executor/ProcfsMetricsGetterSuite.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.executor
 
+import org.mockito.Mockito.{spy, when}
+
 import org.apache.spark.SparkFunSuite
 
 
@@ -38,4 +40,26 @@ class ProcfsMetricsGetterSuite extends SparkFunSuite {
     assert(r.jvmVmemTotal == 4769947648L)
     assert(r.jvmRSSTotal == 262610944)
   }
+
+  test("SPARK-34845: partial metrics shouldn't be returned") {
+    val p = new ProcfsMetricsGetter(getTestResourcePath("ProcfsMetrics"))
+    val mockedP = spy(p)
+
+    var ptree: Set[Int] = Set(26109, 22763)
+    when(mockedP.computeProcessTree).thenReturn(ptree)
+    var r = mockedP.computeAllMetrics
+    assert(r.jvmVmemTotal == 4769947648L)
+    assert(r.jvmRSSTotal == 262610944)
+    assert(r.pythonVmemTotal == 360595456)
+    assert(r.pythonRSSTotal == 7831552)
+
+    // proc file of pid 22764 doesn't exist, so partial metrics shouldn't be returned
+    ptree = Set(26109, 22764, 22763)
+    when(mockedP.computeProcessTree).thenReturn(ptree)
+    r = mockedP.computeAllMetrics
+    assert(r.jvmVmemTotal == 0)
+    assert(r.jvmRSSTotal == 0)
+    assert(r.pythonVmemTotal == 0)
+    assert(r.pythonRSSTotal == 0)
+  }
 }
diff --git a/core/src/test/scala/org/apache/spark/input/WholeTextFileRecordReaderSuite.scala b/core/src/test/scala/org/apache/spark/input/WholeTextFileRecordReaderSuite.scala
index fab7aea6c47aa..f1d7053c34594 100644
--- a/core/src/test/scala/org/apache/spark/input/WholeTextFileRecordReaderSuite.scala
+++ b/core/src/test/scala/org/apache/spark/input/WholeTextFileRecordReaderSuite.scala
@@ -29,7 +29,6 @@ import org.scalatest.BeforeAndAfterAll
 
 import org.apache.spark.{SparkConf, SparkContext, SparkFunSuite}
 import org.apache.spark.internal.Logging
-import org.apache.spark.util.Utils
 
 /**
  * Tests the correctness of
diff --git a/core/src/test/scala/org/apache/spark/internal/io/FileCommitProtocolInstantiationSuite.scala b/core/src/test/scala/org/apache/spark/internal/io/FileCommitProtocolInstantiationSuite.scala
index 2bd32fc927e21..778f748f83950 100644
--- a/core/src/test/scala/org/apache/spark/internal/io/FileCommitProtocolInstantiationSuite.scala
+++ b/core/src/test/scala/org/apache/spark/internal/io/FileCommitProtocolInstantiationSuite.scala
@@ -75,7 +75,7 @@ class FileCommitProtocolInstantiationSuite extends SparkFunSuite {
 
   /**
    * Create a classic two-arg protocol instance.
-   * @param dynamic dyanmic partitioning mode
+   * @param dynamic dynamic partitioning mode
    * @return the instance
    */
   private def instantiateClassic(dynamic: Boolean): ClassicConstructorCommitProtocol = {
@@ -88,7 +88,7 @@ class FileCommitProtocolInstantiationSuite extends SparkFunSuite {
 
   /**
    * Create a three-arg protocol instance.
-   * @param dynamic dyanmic partitioning mode
+   * @param dynamic dynamic partitioning mode
    * @return the instance
    */
   private def instantiateNew(
diff --git a/core/src/test/scala/org/apache/spark/internal/io/SparkHadoopWriterUtilsSuite.scala b/core/src/test/scala/org/apache/spark/internal/io/SparkHadoopWriterUtilsSuite.scala
new file mode 100644
index 0000000000000..33b58ec9e6665
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/internal/io/SparkHadoopWriterUtilsSuite.scala
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.internal.io
+
+import java.util.Date
+
+import org.apache.hadoop.mapreduce.JobID
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.internal.io.SparkHadoopWriterUtils.createJobID
+
+/**
+ * Unit tests for functions in SparkHadoopWriterUtils.
+ */
+class SparkHadoopWriterUtilsSuite extends SparkFunSuite {
+
+  /**
+   * Core test of JobID generation:
+   * They are created.
+   * The job number is converted to the job ID.
+   * They round trip to string and back
+   * (which implies that the full string matches the regexp
+   * in the JobID class).
+   */
+  test("JobID Generation") {
+    val jobNumber = 1010
+    val j1 = createJobID(new Date(), jobNumber)
+    assert(jobNumber == j1.getId,
+      s"Job number mismatch in $j1")
+
+    val jobStr = j1.toString
+    // the string value begins with job_
+    assert(jobStr.startsWith("job_"),
+      s"wrong prefix of $jobStr")
+    // and the hadoop code can parse it
+    val j2 = roundTrip(j1)
+    assert(j1.getId == j2.getId, "Job ID mismatch")
+    assert(j1.getJtIdentifier == j2.getJtIdentifier, "Job identifier mismatch")
+  }
+
+  /**
+   * This is the problem surfacing in situations where committers expect
+   * Job IDs to be unique: if the timestamp is (exclusively) used
+   * then there will conflict in directories created.
+   */
+  test("JobIDs generated at same time are different") {
+    val now = new Date()
+    val j1 = createJobID(now, 1)
+    val j2 = createJobID(now, 1)
+    assert(j1.toString != j2.toString)
+  }
+
+  /**
+   * There's nothing explicitly in the Hadoop classes to stop
+   * job numbers being negative.
+   * There's some big assumptions in the FileOutputCommitter about attempt IDs
+   * being positive during any recovery operations; for safety the ID
+   * job number is validated.
+   */
+  test("JobIDs with negative job number") {
+    intercept[IllegalArgumentException] {
+      createJobID(new Date(), -1)
+    }
+  }
+
+  /**
+   * If someone ever does reinstate use of timestamps,
+   * make sure that the case of timestamp == 0 is handled.
+   */
+  test("JobIDs on Epoch are different") {
+    val j1 = createJobID(new Date(0), 0)
+    val j2 = createJobID(new Date(0), 0)
+    assert (j1.toString != j2.toString)
+  }
+
+  /**
+   * Do a round trip as a string and back again.
+   * This uses the JobID parser.
+   * @param jobID job ID
+   * @return the returned jobID
+   */
+  private def roundTrip(jobID: JobID): JobID = {
+    val parsedJobId = JobID.forName(jobID.toString)
+    assert(jobID == parsedJobId, "Round trip was inconsistent")
+    parsedJobId
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/internal/plugin/PluginContainerSuite.scala b/core/src/test/scala/org/apache/spark/internal/plugin/PluginContainerSuite.scala
index 7888796dd55e6..9ef81d30ff196 100644
--- a/core/src/test/scala/org/apache/spark/internal/plugin/PluginContainerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/internal/plugin/PluginContainerSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.internal.plugin
 import java.io.File
 import java.nio.charset.StandardCharsets
 import java.util.{Map => JMap}
+import java.util.concurrent.atomic.AtomicInteger
 
 import scala.collection.JavaConverters._
 import scala.concurrent.duration._
@@ -129,6 +130,40 @@ class PluginContainerSuite extends SparkFunSuite with BeforeAndAfterEach with Lo
     assert(TestSparkPlugin.driverPlugin != null)
   }
 
+  test("SPARK-33088: executor tasks trigger plugin calls") {
+    val conf = new SparkConf()
+      .setAppName(getClass().getName())
+      .set(SparkLauncher.SPARK_MASTER, "local[1]")
+      .set(PLUGINS, Seq(classOf[TestSparkPlugin].getName()))
+
+    sc = new SparkContext(conf)
+    sc.parallelize(1 to 10, 2).count()
+
+    assert(TestSparkPlugin.executorPlugin.numOnTaskStart.get() == 2)
+    assert(TestSparkPlugin.executorPlugin.numOnTaskSucceeded.get() == 2)
+    assert(TestSparkPlugin.executorPlugin.numOnTaskFailed.get() == 0)
+  }
+
+  test("SPARK-33088: executor failed tasks trigger plugin calls") {
+    val conf = new SparkConf()
+      .setAppName(getClass().getName())
+      .set(SparkLauncher.SPARK_MASTER, "local[2]")
+      .set(PLUGINS, Seq(classOf[TestSparkPlugin].getName()))
+
+    sc = new SparkContext(conf)
+    try {
+      sc.parallelize(1 to 10, 2).foreach(i => throw new RuntimeException)
+    } catch {
+      case t: Throwable => // ignore exception
+    }
+
+    eventually(timeout(10.seconds), interval(100.millis)) {
+      assert(TestSparkPlugin.executorPlugin.numOnTaskStart.get() == 2)
+      assert(TestSparkPlugin.executorPlugin.numOnTaskSucceeded.get() == 0)
+      assert(TestSparkPlugin.executorPlugin.numOnTaskFailed.get() == 2)
+    }
+  }
+
   test("plugin initialization in non-local mode") {
     val path = Utils.createTempDir()
 
@@ -309,6 +344,10 @@ private class TestDriverPlugin extends DriverPlugin {
 
 private class TestExecutorPlugin extends ExecutorPlugin {
 
+  val numOnTaskStart = new AtomicInteger(0)
+  val numOnTaskSucceeded = new AtomicInteger(0)
+  val numOnTaskFailed = new AtomicInteger(0)
+
   override def init(ctx: PluginContext, extraConf: JMap[String, String]): Unit = {
     ctx.metricRegistry().register("executorMetric", new Gauge[Int] {
       override def getValue(): Int = 84
@@ -316,6 +355,17 @@ private class TestExecutorPlugin extends ExecutorPlugin {
     TestSparkPlugin.executorContext = ctx
   }
 
+  override def onTaskStart(): Unit = {
+    numOnTaskStart.incrementAndGet()
+  }
+
+  override def onTaskSucceeded(): Unit = {
+    numOnTaskSucceeded.incrementAndGet()
+  }
+
+  override def onTaskFailed(failureReason: TaskFailedReason): Unit = {
+    numOnTaskFailed.incrementAndGet()
+  }
 }
 
 private object TestSparkPlugin {
diff --git a/core/src/test/scala/org/apache/spark/io/CompressionCodecSuite.scala b/core/src/test/scala/org/apache/spark/io/CompressionCodecSuite.scala
index 4b27396e6ae05..18520ff96a599 100644
--- a/core/src/test/scala/org/apache/spark/io/CompressionCodecSuite.scala
+++ b/core/src/test/scala/org/apache/spark/io/CompressionCodecSuite.scala
@@ -22,6 +22,7 @@ import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
 import com.google.common.io.ByteStreams
 
 import org.apache.spark.{SparkConf, SparkFunSuite}
+import org.apache.spark.internal.config.IO_COMPRESSION_ZSTD_BUFFERPOOL_ENABLED
 
 class CompressionCodecSuite extends SparkFunSuite {
   val conf = new SparkConf(false)
@@ -105,9 +106,12 @@ class CompressionCodecSuite extends SparkFunSuite {
   }
 
   test("zstd compression codec") {
-    val codec = CompressionCodec.createCodec(conf, classOf[ZStdCompressionCodec].getName)
-    assert(codec.getClass === classOf[ZStdCompressionCodec])
-    testCodec(codec)
+    Seq("true", "false").foreach { flag =>
+      val conf = new SparkConf(false).set(IO_COMPRESSION_ZSTD_BUFFERPOOL_ENABLED.key, flag)
+      val codec = CompressionCodec.createCodec(conf, classOf[ZStdCompressionCodec].getName)
+      assert(codec.getClass === classOf[ZStdCompressionCodec])
+      testCodec(codec)
+    }
   }
 
   test("zstd compression codec short form") {
diff --git a/core/src/test/scala/org/apache/spark/io/ZStandardBenchmark.scala b/core/src/test/scala/org/apache/spark/io/ZStandardBenchmark.scala
new file mode 100644
index 0000000000000..62a3d48d2ec16
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/io/ZStandardBenchmark.scala
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.io
+
+import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
+
+import org.apache.spark.SparkConf
+import org.apache.spark.benchmark.{Benchmark, BenchmarkBase}
+import org.apache.spark.internal.config.{IO_COMPRESSION_ZSTD_BUFFERPOOL_ENABLED, IO_COMPRESSION_ZSTD_BUFFERSIZE, IO_COMPRESSION_ZSTD_LEVEL}
+
+
+/**
+ * Benchmark for ZStandard codec performance.
+ * {{{
+ *   To run this benchmark:
+ *   1. without sbt: bin/spark-submit --class <this class> <spark core test jar>
+ *   2. build/sbt "core/test:runMain <this class>"
+ *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "core/test:runMain <this class>"
+ *      Results will be written to "benchmarks/ZStandardBenchmark-results.txt".
+ * }}}
+ */
+object ZStandardBenchmark extends BenchmarkBase {
+
+  val N = 10000
+  val numInteger = IO_COMPRESSION_ZSTD_BUFFERSIZE.defaultValue.get.toInt / 4
+
+  override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
+    val name = "Benchmark ZStandardCompressionCodec"
+    runBenchmark(name) {
+      val benchmark1 = new Benchmark(name, N, output = output)
+      compressionBenchmark(benchmark1, N)
+      benchmark1.run()
+
+      val benchmark2 = new Benchmark(name, N, output = output)
+      decompressionBenchmark(benchmark2, N)
+      benchmark2.run()
+    }
+  }
+
+  private def compressionBenchmark(benchmark: Benchmark, N: Int): Unit = {
+    Seq(false, true).foreach { enablePool =>
+      Seq(1, 2, 3).foreach { level =>
+        val conf = new SparkConf(false)
+          .set(IO_COMPRESSION_ZSTD_BUFFERPOOL_ENABLED, enablePool)
+          .set(IO_COMPRESSION_ZSTD_LEVEL, level)
+        val condition = if (enablePool) "with" else "without"
+        benchmark.addCase(s"Compression $N times at level $level $condition buffer pool") { _ =>
+          (1 until N).foreach { _ =>
+            val os = new ZStdCompressionCodec(conf)
+              .compressedOutputStream(new ByteArrayOutputStream())
+            for (i <- 1 until numInteger) {
+              os.write(i)
+            }
+            os.close()
+          }
+        }
+      }
+    }
+  }
+
+  private def decompressionBenchmark(benchmark: Benchmark, N: Int): Unit = {
+    Seq(false, true).foreach { enablePool =>
+      Seq(1, 2, 3).foreach { level =>
+        val conf = new SparkConf(false)
+          .set(IO_COMPRESSION_ZSTD_BUFFERPOOL_ENABLED, enablePool)
+          .set(IO_COMPRESSION_ZSTD_LEVEL, level)
+        val outputStream = new ByteArrayOutputStream()
+        val out = new ZStdCompressionCodec(conf).compressedOutputStream(outputStream)
+        for (i <- 1 until numInteger) {
+          out.write(i)
+        }
+        out.close()
+        val bytes = outputStream.toByteArray
+
+        val condition = if (enablePool) "with" else "without"
+        benchmark.addCase(s"Decompression $N times from level $level $condition buffer pool") { _ =>
+          (1 until N).foreach { _ =>
+            val bais = new ByteArrayInputStream(bytes)
+            val is = new ZStdCompressionCodec(conf).compressedInputStream(bais)
+            for (i <- 1 until numInteger) {
+              is.read()
+            }
+            is.close()
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/launcher/LauncherBackendSuite.scala b/core/src/test/scala/org/apache/spark/launcher/LauncherBackendSuite.scala
index edec968d0745a..473782ee28d1c 100644
--- a/core/src/test/scala/org/apache/spark/launcher/LauncherBackendSuite.scala
+++ b/core/src/test/scala/org/apache/spark/launcher/LauncherBackendSuite.scala
@@ -21,8 +21,9 @@ import java.util.concurrent.TimeUnit
 
 import scala.concurrent.duration._
 
-import org.scalatest.Matchers
 import org.scalatest.concurrent.Eventually._
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark._
 import org.apache.spark.internal.config.UI.UI_ENABLED
diff --git a/core/src/test/scala/org/apache/spark/memory/TestMemoryManager.scala b/core/src/test/scala/org/apache/spark/memory/TestMemoryManager.scala
index 60f67699f81be..987f383c9c4fa 100644
--- a/core/src/test/scala/org/apache/spark/memory/TestMemoryManager.scala
+++ b/core/src/test/scala/org/apache/spark/memory/TestMemoryManager.scala
@@ -119,6 +119,14 @@ class TestMemoryManager(conf: SparkConf)
     consequentOOM += n
   }
 
+  /**
+   * Undos the effects of [[markExecutionAsOutOfMemoryOnce]] and [[markconsequentOOM]] and lets
+   * calls to [[acquireExecutionMemory()]] (if there is enough memory available).
+   */
+  def resetConsequentOOM(): Unit = synchronized {
+    consequentOOM = 0
+  }
+
   def limit(avail: Long): Unit = synchronized {
     require(avail >= 0)
     available = avail
diff --git a/core/src/test/scala/org/apache/spark/metrics/InputOutputMetricsSuite.scala b/core/src/test/scala/org/apache/spark/metrics/InputOutputMetricsSuite.scala
index 330347299ab56..905bb8110736d 100644
--- a/core/src/test/scala/org/apache/spark/metrics/InputOutputMetricsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/metrics/InputOutputMetricsSuite.scala
@@ -213,7 +213,7 @@ class InputOutputMetricsSuite extends SparkFunSuite with SharedSparkContext
     }
 
     // Computing the amount of bytes read for a cartesian operation is a little involved.
-    // Cartesian interleaves reads between two partitions eg. p1 and p2.
+    // Cartesian interleaves reads between two partitions e.g. p1 and p2.
     // Here are the steps:
     //  1) First it creates an iterator for p1
     //  2) Creates an iterator for p2
diff --git a/core/src/test/scala/org/apache/spark/metrics/MetricsSystemSuite.scala b/core/src/test/scala/org/apache/spark/metrics/MetricsSystemSuite.scala
index 70b6c9a112142..31d8492510f06 100644
--- a/core/src/test/scala/org/apache/spark/metrics/MetricsSystemSuite.scala
+++ b/core/src/test/scala/org/apache/spark/metrics/MetricsSystemSuite.scala
@@ -40,7 +40,7 @@ class MetricsSystemSuite extends SparkFunSuite with BeforeAndAfter with PrivateM
   }
 
   test("MetricsSystem with default config") {
-    val metricsSystem = MetricsSystem.createMetricsSystem("default", conf, securityMgr)
+    val metricsSystem = MetricsSystem.createMetricsSystem("default", conf)
     metricsSystem.start()
     val sources = PrivateMethod[ArrayBuffer[Source]](Symbol("sources"))
     val sinks = PrivateMethod[ArrayBuffer[Sink]](Symbol("sinks"))
@@ -51,7 +51,7 @@ class MetricsSystemSuite extends SparkFunSuite with BeforeAndAfter with PrivateM
   }
 
   test("MetricsSystem with sources add") {
-    val metricsSystem = MetricsSystem.createMetricsSystem("test", conf, securityMgr)
+    val metricsSystem = MetricsSystem.createMetricsSystem("test", conf)
     metricsSystem.start()
     val sources = PrivateMethod[ArrayBuffer[Source]](Symbol("sources"))
     val sinks = PrivateMethod[ArrayBuffer[Sink]](Symbol("sinks"))
@@ -77,7 +77,7 @@ class MetricsSystemSuite extends SparkFunSuite with BeforeAndAfter with PrivateM
     conf.set("spark.executor.id", executorId)
 
     val instanceName = MetricsSystemInstances.DRIVER
-    val driverMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
+    val driverMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf)
 
     val metricName = driverMetricsSystem.buildRegistryName(source)
     assert(metricName === s"$appId.$executorId.${source.sourceName}")
@@ -93,7 +93,7 @@ class MetricsSystemSuite extends SparkFunSuite with BeforeAndAfter with PrivateM
     conf.set("spark.executor.id", executorId)
 
     val instanceName = MetricsSystemInstances.DRIVER
-    val driverMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
+    val driverMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf)
 
     val metricName = driverMetricsSystem.buildRegistryName(source)
     assert(metricName === source.sourceName)
@@ -109,7 +109,7 @@ class MetricsSystemSuite extends SparkFunSuite with BeforeAndAfter with PrivateM
     conf.set("spark.app.id", appId)
 
     val instanceName = MetricsSystemInstances.DRIVER
-    val driverMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
+    val driverMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf)
 
     val metricName = driverMetricsSystem.buildRegistryName(source)
     assert(metricName === source.sourceName)
@@ -127,7 +127,7 @@ class MetricsSystemSuite extends SparkFunSuite with BeforeAndAfter with PrivateM
     conf.set("spark.executor.id", executorId)
 
     val instanceName = MetricsSystemInstances.EXECUTOR
-    val executorMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
+    val executorMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf)
 
     val metricName = executorMetricsSystem.buildRegistryName(source)
     assert(metricName === s"$appId.$executorId.${source.sourceName}")
@@ -143,7 +143,7 @@ class MetricsSystemSuite extends SparkFunSuite with BeforeAndAfter with PrivateM
     conf.set("spark.executor.id", executorId)
 
     val instanceName = MetricsSystemInstances.EXECUTOR
-    val executorMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
+    val executorMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf)
 
     val metricName = executorMetricsSystem.buildRegistryName(source)
     assert(metricName === source.sourceName)
@@ -159,7 +159,7 @@ class MetricsSystemSuite extends SparkFunSuite with BeforeAndAfter with PrivateM
     conf.set("spark.app.id", appId)
 
     val instanceName = MetricsSystemInstances.EXECUTOR
-    val executorMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
+    val executorMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf)
 
     val metricName = executorMetricsSystem.buildRegistryName(source)
     assert(metricName === source.sourceName)
@@ -177,7 +177,7 @@ class MetricsSystemSuite extends SparkFunSuite with BeforeAndAfter with PrivateM
     conf.set("spark.executor.id", executorId)
 
     val instanceName = "testInstance"
-    val testMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
+    val testMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf)
 
     val metricName = testMetricsSystem.buildRegistryName(source)
 
@@ -201,7 +201,7 @@ class MetricsSystemSuite extends SparkFunSuite with BeforeAndAfter with PrivateM
     conf.set(METRICS_NAMESPACE, "${spark.app.name}")
 
     val instanceName = MetricsSystemInstances.EXECUTOR
-    val executorMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
+    val executorMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf)
 
     val metricName = executorMetricsSystem.buildRegistryName(source)
     assert(metricName === s"$appName.$executorId.${source.sourceName}")
@@ -219,7 +219,7 @@ class MetricsSystemSuite extends SparkFunSuite with BeforeAndAfter with PrivateM
     conf.set(METRICS_NAMESPACE, namespaceToResolve)
 
     val instanceName = MetricsSystemInstances.EXECUTOR
-    val executorMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
+    val executorMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf)
 
     val metricName = executorMetricsSystem.buildRegistryName(source)
     // If the user set the spark.metrics.namespace property to an expansion of another property
@@ -239,7 +239,7 @@ class MetricsSystemSuite extends SparkFunSuite with BeforeAndAfter with PrivateM
     conf.set(METRICS_NAMESPACE, "${spark.app.name}")
 
     val instanceName = MetricsSystemInstances.EXECUTOR
-    val executorMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
+    val executorMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf)
 
     val metricName = executorMetricsSystem.buildRegistryName(source)
     assert(metricName === source.sourceName)
@@ -260,7 +260,7 @@ class MetricsSystemSuite extends SparkFunSuite with BeforeAndAfter with PrivateM
     conf.set("spark.executor.id", executorId)
 
     val instanceName = "testInstance"
-    val testMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
+    val testMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf)
 
     val metricName = testMetricsSystem.buildRegistryName(source)
 
diff --git a/core/src/test/scala/org/apache/spark/metrics/sink/GraphiteSinkSuite.scala b/core/src/test/scala/org/apache/spark/metrics/sink/GraphiteSinkSuite.scala
index 2369218830215..cf34121fe73dc 100644
--- a/core/src/test/scala/org/apache/spark/metrics/sink/GraphiteSinkSuite.scala
+++ b/core/src/test/scala/org/apache/spark/metrics/sink/GraphiteSinkSuite.scala
@@ -23,7 +23,7 @@ import scala.collection.JavaConverters._
 
 import com.codahale.metrics._
 
-import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
+import org.apache.spark.SparkFunSuite
 
 class GraphiteSinkSuite extends SparkFunSuite {
 
@@ -32,9 +32,8 @@ class GraphiteSinkSuite extends SparkFunSuite {
     props.put("host", "127.0.0.1")
     props.put("port", "54321")
     val registry = new MetricRegistry
-    val securityMgr = new SecurityManager(new SparkConf(false))
 
-    val sink = new GraphiteSink(props, registry, securityMgr)
+    val sink = new GraphiteSink(props, registry)
 
     val gauge = new Gauge[Double] {
       override def getValue: Double = 1.23
@@ -55,9 +54,8 @@ class GraphiteSinkSuite extends SparkFunSuite {
     props.put("port", "54321")
     props.put("regex", "local-[0-9]+.driver.(CodeGenerator|BlockManager)")
     val registry = new MetricRegistry
-    val securityMgr = new SecurityManager(new SparkConf(false))
 
-    val sink = new GraphiteSink(props, registry, securityMgr)
+    val sink = new GraphiteSink(props, registry)
 
     val gauge = new Gauge[Double] {
       override def getValue: Double = 1.23
diff --git a/core/src/test/scala/org/apache/spark/metrics/sink/PrometheusServletSuite.scala b/core/src/test/scala/org/apache/spark/metrics/sink/PrometheusServletSuite.scala
new file mode 100644
index 0000000000000..4b5b41c14a21e
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/metrics/sink/PrometheusServletSuite.scala
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.metrics.sink
+
+import java.util.Properties
+
+import scala.collection.JavaConverters._
+
+import com.codahale.metrics.{Counter, Gauge, MetricRegistry}
+import org.scalatest.PrivateMethodTester
+
+import org.apache.spark.SparkFunSuite
+
+class PrometheusServletSuite extends SparkFunSuite with PrivateMethodTester {
+
+  test("register metrics") {
+    val sink = createPrometheusServlet()
+
+    val gauge = new Gauge[Double] {
+      override def getValue: Double = 5.0
+    }
+
+    val counter = new Counter
+    counter.inc(10)
+
+    sink.registry.register("gauge1", gauge)
+    sink.registry.register("gauge2", gauge)
+    sink.registry.register("counter1", counter)
+
+    val metricGaugeKeys = sink.registry.getGauges.keySet.asScala
+    assert(metricGaugeKeys.equals(Set("gauge1", "gauge2")),
+      "Should contain 2 gauges metrics registered")
+
+    val metricCounterKeys = sink.registry.getCounters.keySet.asScala
+    assert(metricCounterKeys.equals(Set("counter1")),
+      "Should contain 1 counter metric registered")
+
+    val gaugeValues = sink.registry.getGauges.values.asScala
+    assert(gaugeValues.size == 2)
+    gaugeValues.foreach(gauge => assert(gauge.getValue == 5.0))
+
+    val counterValues = sink.registry.getCounters.values.asScala
+    assert(counterValues.size == 1)
+    counterValues.foreach(counter => assert(counter.getCount == 10))
+  }
+
+  test("normalize key") {
+    val key = "local-1592132938718.driver.LiveListenerBus." +
+      "listenerProcessingTime.org.apache.spark.HeartbeatReceiver"
+    val sink = createPrometheusServlet()
+    val suffix = sink invokePrivate PrivateMethod[String]('normalizeKey)(key)
+    assert(suffix == "metrics_local_1592132938718_driver_LiveListenerBus_" +
+      "listenerProcessingTime_org_apache_spark_HeartbeatReceiver_")
+  }
+
+  private def createPrometheusServlet(): PrometheusServlet =
+    new PrometheusServlet(new Properties, new MetricRegistry)
+}
diff --git a/core/src/test/scala/org/apache/spark/metrics/sink/StatsdSinkSuite.scala b/core/src/test/scala/org/apache/spark/metrics/sink/StatsdSinkSuite.scala
index 0e21a36071c42..ff883633d5e7a 100644
--- a/core/src/test/scala/org/apache/spark/metrics/sink/StatsdSinkSuite.scala
+++ b/core/src/test/scala/org/apache/spark/metrics/sink/StatsdSinkSuite.scala
@@ -24,29 +24,43 @@ import java.util.concurrent.TimeUnit._
 
 import com.codahale.metrics._
 
-import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
+import org.apache.spark.SparkFunSuite
 import org.apache.spark.metrics.sink.StatsdSink._
 
 class StatsdSinkSuite extends SparkFunSuite {
-  private val securityMgr = new SecurityManager(new SparkConf(false))
   private val defaultProps = Map(
     STATSD_KEY_PREFIX -> "spark",
     STATSD_KEY_PERIOD -> "1",
     STATSD_KEY_UNIT -> "seconds",
     STATSD_KEY_HOST -> "127.0.0.1"
   )
-  private val socketTimeout = 30000 // milliseconds
-  private val socketBufferSize = 8192
+  // The maximum size of a single datagram packet payload. Payloads
+  // larger than this will be truncated.
+  private val maxPayloadSize = 256 // bytes
+
+  // The receive buffer must be large enough to hold all inflight
+  // packets. This includes any kernel and protocol overhead.
+  // This value was determined experimentally and should be
+  // increased if timeouts are seen.
+  private val socketMinRecvBufferSize = 16384 // bytes
+  private val socketTimeout = 30000           // milliseconds
 
   private def withSocketAndSink(testCode: (DatagramSocket, StatsdSink) => Any): Unit = {
     val socket = new DatagramSocket
-    socket.setReceiveBufferSize(socketBufferSize)
+
+    // Leave the receive buffer size untouched unless it is too
+    // small. If the receive buffer is too small packets will be
+    // silently dropped and receive operations will timeout.
+    if (socket.getReceiveBufferSize() < socketMinRecvBufferSize) {
+      socket.setReceiveBufferSize(socketMinRecvBufferSize)
+    }
+
     socket.setSoTimeout(socketTimeout)
     val props = new Properties
     defaultProps.foreach(e => props.put(e._1, e._2))
     props.put(STATSD_KEY_PORT, socket.getLocalPort.toString)
     val registry = new MetricRegistry
-    val sink = new StatsdSink(props, registry, securityMgr)
+    val sink = new StatsdSink(props, registry)
     try {
       testCode(socket, sink)
     } finally {
@@ -61,7 +75,7 @@ class StatsdSinkSuite extends SparkFunSuite {
       sink.registry.register("counter", counter)
       sink.report()
 
-      val p = new DatagramPacket(new Array[Byte](socketBufferSize), socketBufferSize)
+      val p = new DatagramPacket(new Array[Byte](maxPayloadSize), maxPayloadSize)
       socket.receive(p)
 
       val result = new String(p.getData, 0, p.getLength, UTF_8)
@@ -77,7 +91,7 @@ class StatsdSinkSuite extends SparkFunSuite {
       sink.registry.register("gauge", gauge)
       sink.report()
 
-      val p = new DatagramPacket(new Array[Byte](socketBufferSize), socketBufferSize)
+      val p = new DatagramPacket(new Array[Byte](maxPayloadSize), maxPayloadSize)
       socket.receive(p)
 
       val result = new String(p.getData, 0, p.getLength, UTF_8)
@@ -87,7 +101,7 @@ class StatsdSinkSuite extends SparkFunSuite {
 
   test("metrics StatsD sink with Histogram") {
     withSocketAndSink { (socket, sink) =>
-      val p = new DatagramPacket(new Array[Byte](socketBufferSize), socketBufferSize)
+      val p = new DatagramPacket(new Array[Byte](maxPayloadSize), maxPayloadSize)
       val histogram = new Histogram(new UniformReservoir)
       histogram.update(10)
       histogram.update(20)
@@ -121,7 +135,7 @@ class StatsdSinkSuite extends SparkFunSuite {
 
   test("metrics StatsD sink with Timer") {
     withSocketAndSink { (socket, sink) =>
-      val p = new DatagramPacket(new Array[Byte](socketBufferSize), socketBufferSize)
+      val p = new DatagramPacket(new Array[Byte](maxPayloadSize), maxPayloadSize)
       val timer = new Timer()
       timer.update(1, SECONDS)
       timer.update(2, SECONDS)
diff --git a/core/src/test/scala/org/apache/spark/metrics/source/SourceConfigSuite.scala b/core/src/test/scala/org/apache/spark/metrics/source/SourceConfigSuite.scala
index 8f5ab7419d4f7..7da1403ecd4b5 100644
--- a/core/src/test/scala/org/apache/spark/metrics/source/SourceConfigSuite.scala
+++ b/core/src/test/scala/org/apache/spark/metrics/source/SourceConfigSuite.scala
@@ -80,4 +80,16 @@ class SourceConfigSuite extends SparkFunSuite with LocalSparkContext {
     }
   }
 
+  test("SPARK-31711: Test executor source registration in local mode") {
+    val conf = new SparkConf()
+    val sc = new SparkContext("local", "test", conf)
+    try {
+      val metricsSystem = sc.env.metricsSystem
+
+      // Executor source should be registered
+      assert (metricsSystem.getSourcesByName("executor").nonEmpty)
+    } finally {
+      sc.stop()
+    }
+  }
 }
diff --git a/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferSecuritySuite.scala b/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferSecuritySuite.scala
index c726329ce8a84..13bb811b840d5 100644
--- a/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferSecuritySuite.scala
+++ b/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferSecuritySuite.scala
@@ -28,7 +28,8 @@ import scala.util.{Failure, Success, Try}
 
 import com.google.common.io.CharStreams
 import org.mockito.Mockito._
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 import org.scalatestplus.mockito.MockitoSugar
 
 import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
diff --git a/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferServiceSuite.scala b/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferServiceSuite.scala
index c804102e4ab2c..c8a8f37212a82 100644
--- a/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferServiceSuite.scala
+++ b/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferServiceSuite.scala
@@ -24,8 +24,10 @@ import scala.reflect.ClassTag
 import scala.util.Random
 
 import org.mockito.ArgumentMatchers.any
-import org.mockito.Mockito.{mock, times, verify, when}
-import org.scalatest._
+import org.mockito.Mockito.{mock, when}
+import org.scalatest.BeforeAndAfterEach
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.{ExecutorDeadException, SecurityManager, SparkConf, SparkFunSuite}
 import org.apache.spark.network.BlockDataManager
@@ -86,7 +88,7 @@ class NettyBlockTransferServiceSuite
   }
 
   test("SPARK-27637: test fetch block with executor dead") {
-    implicit val exectionContext = ExecutionContext.global
+    implicit val executionContext = ExecutionContext.global
     val port = 17634 + Random.nextInt(10000)
     logInfo("random port for test: " + port)
 
@@ -116,8 +118,8 @@ class NettyBlockTransferServiceSuite
       .thenAnswer(_ => {hitExecutorDeadException = true})
 
     service0 = createService(port, driverEndpointRef)
-    val clientFactoryField = service0.getClass.getField(
-      "org$apache$spark$network$netty$NettyBlockTransferService$$clientFactory")
+    val clientFactoryField = service0.getClass
+      .getSuperclass.getSuperclass.getDeclaredField("clientFactory")
     clientFactoryField.setAccessible(true)
     clientFactoryField.set(service0, clientFactory)
 
diff --git a/core/src/test/scala/org/apache/spark/rdd/CoalescedRDDBenchmark.scala b/core/src/test/scala/org/apache/spark/rdd/CoalescedRDDBenchmark.scala
index 617ca5a1a8bc4..b622e0b1d6e1c 100644
--- a/core/src/test/scala/org/apache/spark/rdd/CoalescedRDDBenchmark.scala
+++ b/core/src/test/scala/org/apache/spark/rdd/CoalescedRDDBenchmark.scala
@@ -29,7 +29,7 @@ import org.apache.spark.benchmark.{Benchmark, BenchmarkBase}
  * To run this benchmark:
  * {{{
  *   1. without sbt:
- *      bin/spark-submit --class <this class> --jars <spark core test jar>
+ *      bin/spark-submit --class <this class> <spark core test jar>
  *   2. build/sbt "core/test:runMain <this class>"
  *   3. generate result:
  *      SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "core/test:runMain <this class>"
@@ -78,4 +78,10 @@ object CoalescedRDDBenchmark extends BenchmarkBase {
       coalescedRDD(numIters)
     }
   }
+
+  override def afterAll(): Unit = {
+    if (sc != null) {
+      sc.stop()
+    }
+  }
 }
diff --git a/core/src/test/scala/org/apache/spark/rdd/JdbcRDDSuite.scala b/core/src/test/scala/org/apache/spark/rdd/JdbcRDDSuite.scala
index 05013fbc49b8e..a204502be74b6 100644
--- a/core/src/test/scala/org/apache/spark/rdd/JdbcRDDSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rdd/JdbcRDDSuite.scala
@@ -26,7 +26,8 @@ import org.apache.spark.util.Utils
 
 class JdbcRDDSuite extends SparkFunSuite with BeforeAndAfter with LocalSparkContext {
 
-  before {
+  override def beforeAll(): Unit = {
+    super.beforeAll()
     Utils.classForName("org.apache.derby.jdbc.EmbeddedDriver")
     val conn = DriverManager.getConnection("jdbc:derby:target/JdbcRDDSuiteDb;create=true")
     try {
@@ -96,7 +97,7 @@ class JdbcRDDSuite extends SparkFunSuite with BeforeAndAfter with LocalSparkCont
     assert(rdd.reduce(_ + _) === 5050)
   }
 
-  after {
+  override def afterAll(): Unit = {
     try {
       DriverManager.getConnection("jdbc:derby:target/JdbcRDDSuiteDb;shutdown=true")
     } catch {
@@ -104,5 +105,6 @@ class JdbcRDDSuite extends SparkFunSuite with BeforeAndAfter with LocalSparkCont
         // Normal single database shutdown
         // https://db.apache.org/derby/docs/10.2/ref/rrefexcept71493.html
     }
+    super.afterAll()
   }
 }
diff --git a/core/src/test/scala/org/apache/spark/rdd/LocalCheckpointSuite.scala b/core/src/test/scala/org/apache/spark/rdd/LocalCheckpointSuite.scala
index c942328acc8c1..9e3f27911019c 100644
--- a/core/src/test/scala/org/apache/spark/rdd/LocalCheckpointSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rdd/LocalCheckpointSuite.scala
@@ -44,6 +44,7 @@ class LocalCheckpointSuite extends SparkFunSuite with LocalSparkContext {
     assert(transform(StorageLevel.MEMORY_ONLY_SER_2) === StorageLevel.MEMORY_AND_DISK_SER_2)
     assert(transform(StorageLevel.DISK_ONLY) === StorageLevel.DISK_ONLY)
     assert(transform(StorageLevel.DISK_ONLY_2) === StorageLevel.DISK_ONLY_2)
+    assert(transform(StorageLevel.DISK_ONLY_3) === StorageLevel.DISK_ONLY_3)
     assert(transform(StorageLevel.MEMORY_AND_DISK) === StorageLevel.MEMORY_AND_DISK)
     assert(transform(StorageLevel.MEMORY_AND_DISK_SER) === StorageLevel.MEMORY_AND_DISK_SER)
     assert(transform(StorageLevel.MEMORY_AND_DISK_2) === StorageLevel.MEMORY_AND_DISK_2)
diff --git a/core/src/test/scala/org/apache/spark/rdd/PairRDDFunctionsSuite.scala b/core/src/test/scala/org/apache/spark/rdd/PairRDDFunctionsSuite.scala
index 2de4b109e40e9..a669993352fe7 100644
--- a/core/src/test/scala/org/apache/spark/rdd/PairRDDFunctionsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rdd/PairRDDFunctionsSuite.scala
@@ -28,7 +28,7 @@ import org.apache.hadoop.fs.FileSystem
 import org.apache.hadoop.mapred._
 import org.apache.hadoop.mapreduce.{Job => NewJob, JobContext => NewJobContext,
   OutputCommitter => NewOutputCommitter, OutputFormat => NewOutputFormat,
-  RecordWriter => NewRecordWriter, TaskAttemptContext => NewTaskAttempContext}
+  RecordWriter => NewRecordWriter, TaskAttemptContext => NewTaskAttemptContext}
 import org.apache.hadoop.util.Progressable
 import org.scalatest.Assertions
 
@@ -892,7 +892,7 @@ class FakeOutputFormat() extends OutputFormat[Integer, Integer]() {
  */
 class NewFakeWriter extends NewRecordWriter[Integer, Integer] {
 
-  def close(p1: NewTaskAttempContext): Unit = ()
+  def close(p1: NewTaskAttemptContext): Unit = ()
 
   def write(p1: Integer, p2: Integer): Unit = ()
 
@@ -901,24 +901,24 @@ class NewFakeWriter extends NewRecordWriter[Integer, Integer] {
 class NewFakeCommitter extends NewOutputCommitter {
   def setupJob(p1: NewJobContext): Unit = ()
 
-  def needsTaskCommit(p1: NewTaskAttempContext): Boolean = false
+  def needsTaskCommit(p1: NewTaskAttemptContext): Boolean = false
 
-  def setupTask(p1: NewTaskAttempContext): Unit = ()
+  def setupTask(p1: NewTaskAttemptContext): Unit = ()
 
-  def commitTask(p1: NewTaskAttempContext): Unit = ()
+  def commitTask(p1: NewTaskAttemptContext): Unit = ()
 
-  def abortTask(p1: NewTaskAttempContext): Unit = ()
+  def abortTask(p1: NewTaskAttemptContext): Unit = ()
 }
 
 class NewFakeFormat() extends NewOutputFormat[Integer, Integer]() {
 
   def checkOutputSpecs(p1: NewJobContext): Unit = ()
 
-  def getRecordWriter(p1: NewTaskAttempContext): NewRecordWriter[Integer, Integer] = {
+  def getRecordWriter(p1: NewTaskAttemptContext): NewRecordWriter[Integer, Integer] = {
     new NewFakeWriter()
   }
 
-  def getOutputCommitter(p1: NewTaskAttempContext): NewOutputCommitter = {
+  def getOutputCommitter(p1: NewTaskAttemptContext): NewOutputCommitter = {
     new NewFakeCommitter()
   }
 }
@@ -958,7 +958,7 @@ class FakeFormatWithCallback() extends FakeOutputFormat {
 }
 
 class NewFakeWriterWithCallback extends NewFakeWriter {
-  override def close(p1: NewTaskAttempContext): Unit = {
+  override def close(p1: NewTaskAttemptContext): Unit = {
     FakeWriterWithCallback.calledBy += "close"
   }
 
@@ -972,7 +972,7 @@ class NewFakeWriterWithCallback extends NewFakeWriter {
 }
 
 class NewFakeFormatWithCallback() extends NewFakeFormat {
-  override def getRecordWriter(p1: NewTaskAttempContext): NewRecordWriter[Integer, Integer] = {
+  override def getRecordWriter(p1: NewTaskAttemptContext): NewRecordWriter[Integer, Integer] = {
     new NewFakeWriterWithCallback()
   }
 }
@@ -982,27 +982,27 @@ class YetAnotherFakeCommitter extends NewOutputCommitter with Assertions {
     JobID.jobid = j.getJobID().getId
   }
 
-  def needsTaskCommit(t: NewTaskAttempContext): Boolean = false
+  def needsTaskCommit(t: NewTaskAttemptContext): Boolean = false
 
-  def setupTask(t: NewTaskAttempContext): Unit = {
+  def setupTask(t: NewTaskAttemptContext): Unit = {
     val jobId = t.getTaskAttemptID().getJobID().getId
     assert(jobId === JobID.jobid)
   }
 
-  def commitTask(t: NewTaskAttempContext): Unit = {}
+  def commitTask(t: NewTaskAttemptContext): Unit = {}
 
-  def abortTask(t: NewTaskAttempContext): Unit = {}
+  def abortTask(t: NewTaskAttemptContext): Unit = {}
 }
 
 class YetAnotherFakeFormat() extends NewOutputFormat[Integer, Integer]() {
 
   def checkOutputSpecs(j: NewJobContext): Unit = {}
 
-  def getRecordWriter(t: NewTaskAttempContext): NewRecordWriter[Integer, Integer] = {
+  def getRecordWriter(t: NewTaskAttemptContext): NewRecordWriter[Integer, Integer] = {
     new NewFakeWriter()
   }
 
-  def getOutputCommitter(t: NewTaskAttempContext): NewOutputCommitter = {
+  def getOutputCommitter(t: NewTaskAttemptContext): NewOutputCommitter = {
     new YetAnotherFakeCommitter()
   }
 }
@@ -1021,7 +1021,7 @@ class ConfigTestFormat() extends NewFakeFormat() with Configurable {
 
   def getConf: Configuration = null
 
-  override def getRecordWriter(p1: NewTaskAttempContext): NewRecordWriter[Integer, Integer] = {
+  override def getRecordWriter(p1: NewTaskAttemptContext): NewRecordWriter[Integer, Integer] = {
     assert(setConfCalled, "setConf was never called")
     super.getRecordWriter(p1)
   }
diff --git a/core/src/test/scala/org/apache/spark/rdd/ParallelCollectionSplitSuite.scala b/core/src/test/scala/org/apache/spark/rdd/ParallelCollectionSplitSuite.scala
index 10f4bbcf7f48b..879107350bb52 100644
--- a/core/src/test/scala/org/apache/spark/rdd/ParallelCollectionSplitSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rdd/ParallelCollectionSplitSuite.scala
@@ -140,7 +140,7 @@ class ParallelCollectionSplitSuite extends SparkFunSuite with Checkers {
       assert(slices(i).isInstanceOf[Range])
       val range = slices(i).asInstanceOf[Range]
       assert(range.start === i * (N / 40), "slice " + i + " start")
-      assert(range.end   === (i + 1) * (N / 40), "slice " + i + " end")
+      assert(range.last  === (i + 1) * (N / 40) - 1, "slice " + i + " end")
       assert(range.step  === 1, "slice " + i + " step")
     }
   }
diff --git a/core/src/test/scala/org/apache/spark/rdd/PipedRDDSuite.scala b/core/src/test/scala/org/apache/spark/rdd/PipedRDDSuite.scala
index 2da2854dfbcb9..5000011b3c5ee 100644
--- a/core/src/test/scala/org/apache/spark/rdd/PipedRDDSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rdd/PipedRDDSuite.scala
@@ -176,7 +176,8 @@ class PipedRDDSuite extends SparkFunSuite with SharedSparkContext with Eventuall
   }
 
   test("pipe with env variable") {
-    assume(TestUtils.testCommandAvailable(envCommand))
+    val executable = envCommand.split("\\s+", 2)(0)
+    assume(TestUtils.testCommandAvailable(executable))
     val nums = sc.makeRDD(Array(1, 2, 3, 4), 2)
     val piped = nums.pipe(s"$envCommand MY_TEST_ENV", Map("MY_TEST_ENV" -> "LALALA"))
     val c = piped.collect()
@@ -238,7 +239,8 @@ class PipedRDDSuite extends SparkFunSuite with SharedSparkContext with Eventuall
   }
 
   def testExportInputFile(varName: String): Unit = {
-    assume(TestUtils.testCommandAvailable(envCommand))
+    val executable = envCommand.split("\\s+", 2)(0)
+    assume(TestUtils.testCommandAvailable(executable))
     val nums = new HadoopRDD(sc, new JobConf(), classOf[TextInputFormat], classOf[LongWritable],
       classOf[Text], 2) {
       override def getPartitions: Array[Partition] = Array(generateFakeHadoopPartition())
diff --git a/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala b/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala
index 18154d861a731..6f49e108aea2e 100644
--- a/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala
@@ -237,10 +237,8 @@ class RDDSuite extends SparkFunSuite with SharedSparkContext with Eventually {
 
   test("aggregate") {
     val pairs = sc.makeRDD(Seq(("a", 1), ("b", 2), ("a", 2), ("c", 5), ("a", 3)))
-    type StringMap = HashMap[String, Int]
-    val emptyMap = new StringMap {
-      override def default(key: String): Int = 0
-    }
+    type StringMap = scala.collection.mutable.Map[String, Int]
+    val emptyMap = HashMap[String, Int]().withDefaultValue(0).asInstanceOf[StringMap]
     val mergeElement: (StringMap, (String, Int)) => StringMap = (map, pair) => {
       map(pair._1) += pair._2
       map
@@ -656,7 +654,7 @@ class RDDSuite extends SparkFunSuite with SharedSparkContext with Eventually {
   }
 
   test("top with predefined ordering") {
-    val nums = Array.range(1, 100000)
+    val nums = Seq.range(1, 100000)
     val ints = sc.makeRDD(scala.util.Random.shuffle(nums), 2)
     val topK = ints.top(5)
     assert(topK.size === 5)
@@ -862,6 +860,20 @@ class RDDSuite extends SparkFunSuite with SharedSparkContext with Eventually {
     assert(partitions(1) === Seq((1, 3), (3, 8), (3, 8)))
   }
 
+  test("SPARK-32384: repartitionAndSortWithinPartitions without shuffle") {
+    val data = sc.parallelize(Seq((0, 5), (3, 8), (2, 6), (0, 8), (3, 8), (1, 3)), 2)
+
+    val partitioner = new HashPartitioner(2)
+    val agged = data.reduceByKey(_ + _, 2)
+    assert(agged.partitioner == Some(partitioner))
+
+    val sorted = agged.repartitionAndSortWithinPartitions(partitioner)
+    assert(sorted.partitioner == Some(partitioner))
+
+    assert(sorted.dependencies.nonEmpty &&
+      sorted.dependencies.forall(_.isInstanceOf[OneToOneDependency[_]]))
+  }
+
   test("cartesian on empty RDD") {
     val a = sc.emptyRDD[Int]
     val b = sc.parallelize(1 to 3)
@@ -1098,13 +1110,13 @@ class RDDSuite extends SparkFunSuite with SharedSparkContext with Eventually {
     override def getPartitions: Array[Partition] = Array(new Partition {
       override def index: Int = 0
     })
-    override def getDependencies: Seq[Dependency[_]] = mutableDependencies
+    override def getDependencies: Seq[Dependency[_]] = mutableDependencies.toSeq
     def addDependency(dep: Dependency[_]): Unit = {
       mutableDependencies += dep
     }
   }
 
-  test("RDD.partitions() fails fast when partitions indicies are incorrect (SPARK-13021)") {
+  test("RDD.partitions() fails fast when partitions indices are incorrect (SPARK-13021)") {
     class BadRDD[T: ClassTag](prev: RDD[T]) extends RDD[T](prev) {
 
       override def compute(part: Partition, context: TaskContext): Iterator[T] = {
@@ -1298,19 +1310,15 @@ class SizeBasedCoalescer(val maxSize: Int) extends PartitionCoalescer with Seria
       val splitSize = fileSplit.getLength
       if (currentSum + splitSize < maxSize) {
         addPartition(partition, splitSize)
-        index += 1
-        if (index == partitions.size) {
-          updateGroups
-        }
       } else {
-        if (currentGroup.partitions.size == 0) {
-          addPartition(partition, splitSize)
-          index += 1
-        } else {
-          updateGroups
+        if (currentGroup.partitions.nonEmpty) {
+          updateGroups()
         }
+        addPartition(partition, splitSize)
       }
+      index += 1
     }
+    updateGroups()
     groups.toArray
   }
 }
diff --git a/core/src/test/scala/org/apache/spark/rdd/SortingSuite.scala b/core/src/test/scala/org/apache/spark/rdd/SortingSuite.scala
index d5f7d30a253fe..5b01b54a0a9f4 100644
--- a/core/src/test/scala/org/apache/spark/rdd/SortingSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rdd/SortingSuite.scala
@@ -17,7 +17,8 @@
 
 package org.apache.spark.rdd
 
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.{SharedSparkContext, SparkFunSuite}
 import org.apache.spark.internal.Logging
diff --git a/core/src/test/scala/org/apache/spark/resource/ResourceProfileManagerSuite.scala b/core/src/test/scala/org/apache/spark/resource/ResourceProfileManagerSuite.scala
index 004618a161b44..65e41986ff31f 100644
--- a/core/src/test/scala/org/apache/spark/resource/ResourceProfileManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/resource/ResourceProfileManagerSuite.scala
@@ -20,10 +20,11 @@ package org.apache.spark.resource
 import org.apache.spark.{SparkConf, SparkException, SparkFunSuite}
 import org.apache.spark.internal.config._
 import org.apache.spark.internal.config.Tests._
+import org.apache.spark.scheduler.LiveListenerBus
 
 class ResourceProfileManagerSuite extends SparkFunSuite {
 
-  override def beforeAll() {
+  override def beforeAll(): Unit = {
     try {
       ResourceProfile.clearDefaultProfile()
     } finally {
@@ -31,7 +32,7 @@ class ResourceProfileManagerSuite extends SparkFunSuite {
     }
   }
 
-  override def afterEach() {
+  override def afterEach(): Unit = {
     try {
       ResourceProfile.clearDefaultProfile()
     } finally {
@@ -39,13 +40,15 @@ class ResourceProfileManagerSuite extends SparkFunSuite {
     }
   }
 
+  val listenerBus = new LiveListenerBus(new SparkConf())
+
   test("ResourceProfileManager") {
     val conf = new SparkConf().set(EXECUTOR_CORES, 4)
-    val rpmanager = new ResourceProfileManager(conf)
+    val rpmanager = new ResourceProfileManager(conf, listenerBus)
     val defaultProf = rpmanager.defaultResourceProfile
     assert(defaultProf.id === ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID)
-    assert(defaultProf.executorResources.size === 2,
-      "Executor resources should contain cores and memory by default")
+    assert(defaultProf.executorResources.size === 3,
+      "Executor resources should contain cores, heap and offheap memory by default")
     assert(defaultProf.executorResources(ResourceProfile.CORES).amount === 4,
       s"Executor resources should have 4 cores")
   }
@@ -53,7 +56,7 @@ class ResourceProfileManagerSuite extends SparkFunSuite {
   test("isSupported yarn no dynamic allocation") {
     val conf = new SparkConf().setMaster("yarn").set(EXECUTOR_CORES, 4)
     conf.set(RESOURCE_PROFILE_MANAGER_TESTING.key, "true")
-    val rpmanager = new ResourceProfileManager(conf)
+    val rpmanager = new ResourceProfileManager(conf, listenerBus)
     // default profile should always work
     val defaultProf = rpmanager.defaultResourceProfile
     val rprof = new ResourceProfileBuilder()
@@ -64,14 +67,15 @@ class ResourceProfileManagerSuite extends SparkFunSuite {
       rpmanager.isSupported(immrprof)
     }.getMessage()
 
-    assert(error.contains("ResourceProfiles are only supported on YARN with dynamic allocation"))
+    assert(error.contains(
+      "ResourceProfiles are only supported on YARN and Kubernetes with dynamic allocation"))
   }
 
   test("isSupported yarn with dynamic allocation") {
     val conf = new SparkConf().setMaster("yarn").set(EXECUTOR_CORES, 4)
     conf.set(DYN_ALLOCATION_ENABLED, true)
     conf.set(RESOURCE_PROFILE_MANAGER_TESTING.key, "true")
-    val rpmanager = new ResourceProfileManager(conf)
+    val rpmanager = new ResourceProfileManager(conf, listenerBus)
     // default profile should always work
     val defaultProf = rpmanager.defaultResourceProfile
     val rprof = new ResourceProfileBuilder()
@@ -81,26 +85,42 @@ class ResourceProfileManagerSuite extends SparkFunSuite {
     assert(rpmanager.isSupported(immrprof) == true)
   }
 
-  test("isSupported yarn with local mode") {
+  test("isSupported k8s with dynamic allocation") {
+    val conf = new SparkConf().setMaster("k8s://foo").set(EXECUTOR_CORES, 4)
+    conf.set(DYN_ALLOCATION_ENABLED, true)
+    conf.set(DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED, true)
+    conf.set(RESOURCE_PROFILE_MANAGER_TESTING.key, "true")
+    val rpmanager = new ResourceProfileManager(conf, listenerBus)
+    // default profile should always work
+    val defaultProf = rpmanager.defaultResourceProfile
+    val rprof = new ResourceProfileBuilder()
+    val gpuExecReq =
+      new ExecutorResourceRequests().resource("gpu", 2, "someScript", "nvidia")
+    val immrprof = rprof.require(gpuExecReq).build
+    assert(rpmanager.isSupported(immrprof) == true)
+  }
+
+  test("isSupported with local mode") {
     val conf = new SparkConf().setMaster("local").set(EXECUTOR_CORES, 4)
     conf.set(RESOURCE_PROFILE_MANAGER_TESTING.key, "true")
-    val rpmanager = new ResourceProfileManager(conf)
+    val rpmanager = new ResourceProfileManager(conf, listenerBus)
     // default profile should always work
     val defaultProf = rpmanager.defaultResourceProfile
     val rprof = new ResourceProfileBuilder()
     val gpuExecReq =
       new ExecutorResourceRequests().resource("gpu", 2, "someScript")
     val immrprof = rprof.require(gpuExecReq).build
-    var error = intercept[SparkException] {
+    val error = intercept[SparkException] {
       rpmanager.isSupported(immrprof)
     }.getMessage()
 
-    assert(error.contains("ResourceProfiles are only supported on YARN with dynamic allocation"))
+    assert(error.contains(
+      "ResourceProfiles are only supported on YARN and Kubernetes with dynamic allocation"))
   }
 
   test("ResourceProfileManager has equivalent profile") {
     val conf = new SparkConf().set(EXECUTOR_CORES, 4)
-    val rpmanager = new ResourceProfileManager(conf)
+    val rpmanager = new ResourceProfileManager(conf, listenerBus)
     var rpAlreadyExist: Option[ResourceProfile] = None
     val checkId = 500
     for (i <- 1 to 1000) {
diff --git a/core/src/test/scala/org/apache/spark/resource/ResourceProfileSuite.scala b/core/src/test/scala/org/apache/spark/resource/ResourceProfileSuite.scala
index 29d3ef130ce57..27cc44a099de1 100644
--- a/core/src/test/scala/org/apache/spark/resource/ResourceProfileSuite.scala
+++ b/core/src/test/scala/org/apache/spark/resource/ResourceProfileSuite.scala
@@ -18,13 +18,13 @@
 package org.apache.spark.resource
 
 import org.apache.spark.{SparkConf, SparkFunSuite}
-import org.apache.spark.internal.config.{EXECUTOR_CORES, EXECUTOR_MEMORY, EXECUTOR_MEMORY_OVERHEAD}
+import org.apache.spark.internal.config._
 import org.apache.spark.internal.config.Python.PYSPARK_EXECUTOR_MEMORY
 import org.apache.spark.resource.TestResourceIDs._
 
 class ResourceProfileSuite extends SparkFunSuite {
 
-  override def beforeAll() {
+  override def beforeAll(): Unit = {
     try {
       ResourceProfile.clearDefaultProfile()
     } finally {
@@ -32,7 +32,7 @@ class ResourceProfileSuite extends SparkFunSuite {
     }
   }
 
-  override def afterEach() {
+  override def afterEach(): Unit = {
     try {
       ResourceProfile.clearDefaultProfile()
     } finally {
@@ -43,8 +43,8 @@ class ResourceProfileSuite extends SparkFunSuite {
   test("Default ResourceProfile") {
     val rprof = ResourceProfile.getOrCreateDefaultProfile(new SparkConf)
     assert(rprof.id === ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID)
-    assert(rprof.executorResources.size === 2,
-      "Executor resources should contain cores and memory by default")
+    assert(rprof.executorResources.size === 3,
+      "Executor resources should contain cores, heap and offheap memory by default")
     assert(rprof.executorResources(ResourceProfile.CORES).amount === 1,
       "Executor resources should have 1 core")
     assert(rprof.getExecutorCores.get === 1,
@@ -55,6 +55,8 @@ class ResourceProfileSuite extends SparkFunSuite {
       "pyspark memory empty if not specified")
     assert(rprof.executorResources.get(ResourceProfile.OVERHEAD_MEM) == None,
       "overhead memory empty if not specified")
+    assert(rprof.executorResources(ResourceProfile.OFFHEAP_MEM).amount === 0,
+      "Executor resources should have 0 offheap memory")
     assert(rprof.taskResources.size === 1,
       "Task resources should just contain cpus by default")
     assert(rprof.taskResources(ResourceProfile.CPUS).amount === 1,
@@ -69,14 +71,16 @@ class ResourceProfileSuite extends SparkFunSuite {
     conf.set(EXECUTOR_MEMORY_OVERHEAD.key, "1g")
     conf.set(EXECUTOR_MEMORY.key, "4g")
     conf.set(EXECUTOR_CORES.key, "4")
+    conf.set(MEMORY_OFFHEAP_ENABLED.key, "true")
+    conf.set(MEMORY_OFFHEAP_SIZE.key, "3m")
     conf.set(TASK_GPU_ID.amountConf, "1")
     conf.set(EXECUTOR_GPU_ID.amountConf, "1")
     conf.set(EXECUTOR_GPU_ID.discoveryScriptConf, "nameOfScript")
     val rprof = ResourceProfile.getOrCreateDefaultProfile(conf)
     assert(rprof.id === ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID)
     val execResources = rprof.executorResources
-    assert(execResources.size === 5, s"Executor resources should contain cores, pyspark " +
-      s"memory, memory overhead, memory, and gpu $execResources")
+    assert(execResources.size === 6, s"Executor resources should contain cores, pyspark " +
+      s"memory, memory overhead, memory, offHeap memory and gpu $execResources")
     assert(execResources.contains("gpu"), "Executor resources should have gpu")
     assert(rprof.executorResources(ResourceProfile.CORES).amount === 4,
       "Executor resources should have 4 core")
@@ -88,6 +92,8 @@ class ResourceProfileSuite extends SparkFunSuite {
       "pyspark memory empty if not specified")
     assert(rprof.executorResources(ResourceProfile.OVERHEAD_MEM).amount == 1024,
       "overhead memory empty if not specified")
+    assert(rprof.executorResources(ResourceProfile.OFFHEAP_MEM).amount == 3,
+      "Executor resources should have 3 offHeap memory")
     assert(rprof.taskResources.size === 2,
       "Task resources should just contain cpus and gpu")
     assert(rprof.taskResources.contains("gpu"), "Task resources should have gpu")
@@ -172,14 +178,14 @@ class ResourceProfileSuite extends SparkFunSuite {
 
     val ereqs = new ExecutorResourceRequests()
     ereqs.cores(2).memory("4096")
-    ereqs.memoryOverhead("2048").pysparkMemory("1024")
+    ereqs.memoryOverhead("2048").pysparkMemory("1024").offHeapMemory("3072")
     val treqs = new TaskResourceRequests()
     treqs.cpus(1)
 
     rprof.require(treqs)
     rprof.require(ereqs)
 
-    assert(rprof.executorResources.size === 5)
+    assert(rprof.executorResources.size === 6)
     assert(rprof.executorResources(ResourceProfile.CORES).amount === 2,
       "Executor resources should have 2 cores")
     assert(rprof.executorResources(ResourceProfile.MEMORY).amount === 4096,
@@ -188,6 +194,8 @@ class ResourceProfileSuite extends SparkFunSuite {
       "Executor resources should have 2048 overhead memory")
     assert(rprof.executorResources(ResourceProfile.PYSPARK_MEM).amount === 1024,
       "Executor resources should have 1024 pyspark memory")
+    assert(rprof.executorResources(ResourceProfile.OFFHEAP_MEM).amount === 3072,
+      "Executor resources should have 3072 offHeap memory")
 
     assert(rprof.taskResources.size === 2)
     assert(rprof.taskResources("cpus").amount === 1, "Task resources should have cpu")
@@ -217,7 +225,7 @@ class ResourceProfileSuite extends SparkFunSuite {
     val rprof = new ResourceProfileBuilder()
     val ereqs = new ExecutorResourceRequests()
     ereqs.memory("4g")
-    ereqs.memoryOverhead("2000m").pysparkMemory("512000k")
+    ereqs.memoryOverhead("2000m").pysparkMemory("512000k").offHeapMemory("1g")
     rprof.require(ereqs)
 
     assert(rprof.executorResources(ResourceProfile.MEMORY).amount === 4096,
@@ -226,6 +234,8 @@ class ResourceProfileSuite extends SparkFunSuite {
       "Executor resources should have 2000 overhead memory")
     assert(rprof.executorResources(ResourceProfile.PYSPARK_MEM).amount === 500,
       "Executor resources should have 512 pyspark memory")
+    assert(rprof.executorResources(ResourceProfile.OFFHEAP_MEM).amount === 1024,
+      "Executor resources should have 1024 offHeap memory")
   }
 
   test("Test TaskResourceRequest fractional") {
@@ -256,4 +266,32 @@ class ResourceProfileSuite extends SparkFunSuite {
     }.getMessage()
     assert(taskError.contains("The resource amount 0.7 must be either <= 0.5, or a whole number."))
   }
+
+  test("ResourceProfile has correct custom executor resources") {
+    val rprof = new ResourceProfileBuilder()
+    val eReq = new ExecutorResourceRequests()
+      .cores(2).memory("4096")
+      .memoryOverhead("2048").pysparkMemory("1024").offHeapMemory("3072")
+      .resource("gpu", 2)
+    rprof.require(eReq)
+
+    // Update this if new resource type added
+    assert(ResourceProfile.allSupportedExecutorResources.size === 5,
+      "Executor resources should have 5 supported resources")
+    assert(ResourceProfile.getCustomExecutorResources(rprof.build).size === 1,
+      "Executor resources should have 1 custom resource")
+  }
+
+  test("ResourceProfile has correct custom task resources") {
+    val rprof = new ResourceProfileBuilder()
+    val taskReq = new TaskResourceRequests()
+      .resource("gpu", 1)
+    val eReq = new ExecutorResourceRequests()
+      .cores(2).memory("4096")
+      .memoryOverhead("2048").pysparkMemory("1024").offHeapMemory("3072")
+    rprof.require(taskReq).require(eReq)
+
+    assert(ResourceProfile.getCustomTaskResources(rprof.build).size === 1,
+      "Task resources should have 1 custom resource")
+  }
 }
diff --git a/core/src/test/scala/org/apache/spark/resource/ResourceUtilsSuite.scala b/core/src/test/scala/org/apache/spark/resource/ResourceUtilsSuite.scala
index 278a72a7192d8..ffe5ff5787102 100644
--- a/core/src/test/scala/org/apache/spark/resource/ResourceUtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/resource/ResourceUtilsSuite.scala
@@ -26,10 +26,8 @@ import org.json4s.{DefaultFormats, Extraction}
 import org.apache.spark.{LocalSparkContext, SparkConf, SparkException, SparkFunSuite}
 import org.apache.spark.TestUtils._
 import org.apache.spark.internal.config._
-import org.apache.spark.internal.config.Tests._
 import org.apache.spark.resource.ResourceUtils._
 import org.apache.spark.resource.TestResourceIDs._
-import org.apache.spark.scheduler.LiveListenerBus
 import org.apache.spark.util.Utils
 
 class ResourceUtilsSuite extends SparkFunSuite
@@ -180,7 +178,7 @@ class ResourceUtilsSuite extends SparkFunSuite
   test("list resource ids") {
     val conf = new SparkConf
     conf.set(DRIVER_GPU_ID.amountConf, "2")
-    var resources = listResourceIds(conf, SPARK_DRIVER_PREFIX)
+    val resources = listResourceIds(conf, SPARK_DRIVER_PREFIX)
     assert(resources.size === 1, "should only have GPU for resource")
     assert(resources(0).resourceName == GPU, "name should be gpu")
 
@@ -223,7 +221,7 @@ class ResourceUtilsSuite extends SparkFunSuite
     val conf = new SparkConf
     assume(!(Utils.isWindows))
     withTempDir { dir =>
-      val gpuDiscovery = createTempScriptWithExpectedOutput(dir, "gpuDisocveryScript",
+      val gpuDiscovery = createTempScriptWithExpectedOutput(dir, "gpuDiscoveryScript",
         """{"name": "gpu", "addresses": ["0", "1"]}""")
       conf.set(DRIVER_GPU_ID.amountConf, "2")
       conf.set(DRIVER_GPU_ID.discoveryScriptConf, gpuDiscovery)
diff --git a/core/src/test/scala/org/apache/spark/rpc/RpcEnvSuite.scala b/core/src/test/scala/org/apache/spark/rpc/RpcEnvSuite.scala
index d25fd20340d48..bec96e523e9e5 100644
--- a/core/src/test/scala/org/apache/spark/rpc/RpcEnvSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rpc/RpcEnvSuite.scala
@@ -33,7 +33,7 @@ import org.mockito.Mockito.{mock, never, verify, when}
 import org.scalatest.BeforeAndAfterAll
 import org.scalatest.concurrent.Eventually._
 
-import org.apache.spark.{SecurityManager, SparkConf, SparkEnv, SparkException, SparkFunSuite}
+import org.apache.spark.{SparkConf, SparkEnv, SparkException, SparkFunSuite}
 import org.apache.spark.deploy.SparkHadoopUtil
 import org.apache.spark.internal.config._
 import org.apache.spark.util.{ThreadUtils, Utils}
@@ -901,7 +901,6 @@ abstract class RpcEnvSuite extends SparkFunSuite with BeforeAndAfterAll {
           }
         }
 
-        val sm = new SecurityManager(conf)
         val hc = SparkHadoopUtil.get.conf
 
         val files = Seq(
@@ -913,7 +912,7 @@ abstract class RpcEnvSuite extends SparkFunSuite with BeforeAndAfterAll {
           (subFile2, dir2Uri + "/file2"))
         files.foreach { case (f, uri) =>
           val destFile = new File(destDir, f.getName())
-          Utils.fetchFile(uri, destDir, conf, sm, hc, 0L, false)
+          Utils.fetchFile(uri, destDir, conf, hc, 0L, false)
           assert(Files.equal(f, destFile))
         }
 
@@ -921,7 +920,7 @@ abstract class RpcEnvSuite extends SparkFunSuite with BeforeAndAfterAll {
         Seq("files", "jars", "dir1").foreach { root =>
           intercept[Exception] {
             val uri = env.address.toSparkURL + s"/$root/doesNotExist"
-            Utils.fetchFile(uri, destDir, conf, sm, hc, 0L, false)
+            Utils.fetchFile(uri, destDir, conf, hc, 0L, false)
           }
         }
       }
diff --git a/core/src/test/scala/org/apache/spark/rpc/netty/InboxSuite.scala b/core/src/test/scala/org/apache/spark/rpc/netty/InboxSuite.scala
index c74c728b3e3f3..8b1c602cd8e58 100644
--- a/core/src/test/scala/org/apache/spark/rpc/netty/InboxSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rpc/netty/InboxSuite.scala
@@ -136,4 +136,17 @@ class InboxSuite extends SparkFunSuite {
 
     endpoint.verifySingleOnNetworkErrorMessage(cause, remoteAddress)
   }
+
+  test("SPARK-32738: should reduce the number of active threads when fatal error happens") {
+    val endpoint = mock(classOf[TestRpcEndpoint])
+    when(endpoint.receive).thenThrow(new OutOfMemoryError())
+
+    val dispatcher = mock(classOf[Dispatcher])
+    val inbox = new Inbox("name", endpoint)
+    inbox.post(OneWayMessage(null, "hi"))
+    intercept[OutOfMemoryError] {
+      inbox.process(dispatcher)
+    }
+    assert(inbox.getNumActiveThreads == 0)
+  }
 }
diff --git a/core/src/test/scala/org/apache/spark/rpc/netty/NettyRpcEnvSuite.scala b/core/src/test/scala/org/apache/spark/rpc/netty/NettyRpcEnvSuite.scala
index c2730f90ed982..fe6d0db837bda 100644
--- a/core/src/test/scala/org/apache/spark/rpc/netty/NettyRpcEnvSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rpc/netty/NettyRpcEnvSuite.scala
@@ -73,7 +73,7 @@ class NettyRpcEnvSuite extends RpcEnvSuite with MockitoSugar with TimeLimits {
 
     val nettyEnv = env.asInstanceOf[NettyRpcEnv]
     val client = mock[TransportClient]
-    val senderAddress = RpcAddress("locahost", 12345)
+    val senderAddress = RpcAddress("localhost", 12345)
     val receiverAddress = RpcEndpointAddress("localhost", 54321, "test")
     val receiver = new NettyRpcEndpointRef(nettyEnv.conf, receiverAddress, nettyEnv)
 
diff --git a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
index b5614b263ca94..4f97003e2ed59 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
@@ -19,28 +19,30 @@ package org.apache.spark.scheduler
 
 import java.io.File
 
+import scala.collection.mutable
 import scala.util.Random
 
 import org.scalatest.concurrent.Eventually
 import org.scalatest.time.SpanSugar._
 
 import org.apache.spark._
+import org.apache.spark.internal.config.LEGACY_LOCALITY_WAIT_RESET
 import org.apache.spark.internal.config.Tests.TEST_NO_STAGE_RETRY
 
 class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with Eventually {
 
-  def initLocalClusterSparkContext(numWorker: Int = 4): Unit = {
-    val conf = new SparkConf()
+  def initLocalClusterSparkContext(numWorker: Int = 4, conf: SparkConf = new SparkConf()): Unit = {
+    conf
       // Init local cluster here so each barrier task runs in a separated process, thus `barrier()`
       // call is actually useful.
       .setMaster(s"local-cluster[$numWorker, 1, 1024]")
       .setAppName("test-cluster")
       .set(TEST_NO_STAGE_RETRY, true)
     sc = new SparkContext(conf)
+    TestUtils.waitUntilExecutorsUp(sc, numWorker, 60000)
   }
 
-  // TODO (SPARK-31730): re-enable it
-  ignore("global sync by barrier() call") {
+  test("global sync by barrier() call") {
     initLocalClusterSparkContext()
     val rdd = sc.makeRDD(1 to 10, 4)
     val rdd2 = rdd.barrier().mapPartitions { it =>
@@ -57,10 +59,7 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
   }
 
   test("share messages with allGather() call") {
-    val conf = new SparkConf()
-      .setMaster("local-cluster[4, 1, 1024]")
-      .setAppName("test-cluster")
-    sc = new SparkContext(conf)
+    initLocalClusterSparkContext()
     val rdd = sc.makeRDD(1 to 10, 4)
     val rdd2 = rdd.barrier().mapPartitions { it =>
       val context = BarrierTaskContext.get()
@@ -69,19 +68,16 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
       // Pass partitionId message in
       val message: String = context.partitionId().toString
       val messages: Array[String] = context.allGather(message)
-      messages.toList.iterator
+      Iterator.single(messages.toList)
     }
-    // Take a sorted list of all the partitionId messages
-    val messages = rdd2.collect().head
-    // All the task partitionIds are shared
-    for((x, i) <- messages.view.zipWithIndex) assert(x.toString == i.toString)
+    val messages = rdd2.collect()
+    // All the task partitionIds are shared across all tasks
+    assert(messages.length === 4)
+    assert(messages.forall(_ == List("0", "1", "2", "3")))
   }
 
   test("throw exception if we attempt to synchronize with different blocking calls") {
-    val conf = new SparkConf()
-      .setMaster("local-cluster[4, 1, 1024]")
-      .setAppName("test-cluster")
-    sc = new SparkContext(conf)
+    initLocalClusterSparkContext()
     val rdd = sc.makeRDD(1 to 10, 4)
     val rdd2 = rdd.barrier().mapPartitions { it =>
       val context = BarrierTaskContext.get()
@@ -100,10 +96,7 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
   }
 
   test("successively sync with allGather and barrier") {
-    val conf = new SparkConf()
-      .setMaster("local-cluster[4, 1, 1024]")
-      .setAppName("test-cluster")
-    sc = new SparkContext(conf)
+    initLocalClusterSparkContext()
     val rdd = sc.makeRDD(1 to 10, 4)
     val rdd2 = rdd.barrier().mapPartitions { it =>
       val context = BarrierTaskContext.get()
@@ -129,8 +122,7 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
     assert(times2.max - times2.min <= 1000)
   }
 
-  // TODO (SPARK-31730): re-enable it
-  ignore("support multiple barrier() call within a single task") {
+  test("support multiple barrier() call within a single task") {
     initLocalClusterSparkContext()
     val rdd = sc.makeRDD(1 to 10, 4)
     val rdd2 = rdd.barrier().mapPartitions { it =>
@@ -198,7 +190,7 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
 
   test("throw exception if the number of barrier() calls are not the same on every task") {
     initLocalClusterSparkContext()
-    sc.conf.set("spark.barrier.sync.timeout", "1")
+    sc.conf.set("spark.barrier.sync.timeout", "5")
     val rdd = sc.makeRDD(1 to 10, 4)
     val rdd2 = rdd.barrier().mapPartitions { it =>
       val context = BarrierTaskContext.get()
@@ -221,7 +213,7 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
       rdd2.collect()
     }.getMessage
     assert(error.contains("The coordinator didn't get all barrier sync requests"))
-    assert(error.contains("within 1 second(s)"))
+    assert(error.contains("within 5 second(s)"))
   }
 
   def testBarrierTaskKilled(interruptOnKill: Boolean): Unit = {
@@ -283,19 +275,96 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
     testBarrierTaskKilled(interruptOnKill = true)
   }
 
-  test("SPARK-31485: barrier stage should fail if only partial tasks are launched") {
-    initLocalClusterSparkContext(2)
-    val rdd0 = sc.parallelize(Seq(0, 1, 2, 3), 2)
-    val dep = new OneToOneDependency[Int](rdd0)
-    // set up a barrier stage with 2 tasks and both tasks prefer executor 0 (only 1 core) for
-    // scheduling. So, one of tasks won't be scheduled in one round of resource offer.
-    val rdd = new MyRDD(sc, 2, List(dep), Seq(Seq("executor_h_0"), Seq("executor_h_0")))
-    val errorMsg = intercept[SparkException] {
+  test("SPARK-24818: disable legacy delay scheduling for barrier stage") {
+    val conf = new SparkConf().set(LEGACY_LOCALITY_WAIT_RESET, true)
+    initLocalClusterSparkContext(2, conf)
+    val taskLocality = new mutable.ArrayBuffer[TaskLocality.TaskLocality]()
+    val listener = new SparkListener {
+      override def onTaskStart(taskStart: SparkListenerTaskStart): Unit = {
+        taskLocality += taskStart.taskInfo.taskLocality
+      }
+    }
+
+    try {
+      sc.addSparkListener(listener)
+      val id = sc.getExecutorIds().head
+      val rdd0 = sc.parallelize(Seq(0, 1, 2, 3), 2)
+      val dep = new OneToOneDependency[Int](rdd0)
+      // set up a stage with 2 tasks and both tasks prefer the same executor (only 1 core)
+      // for scheduling. So, the first task can always get the best locality (PROCESS_LOCAL),
+      // but the second task may not get the best locality depends whether it's a barrier stage
+      // or not.
+      val rdd = new MyRDD(sc, 2, List(dep), Seq(Seq(s"executor_h_$id"), Seq(s"executor_h_$id"))) {
+        override def compute(split: Partition, context: TaskContext): Iterator[(Int, Int)] = {
+          Iterator.single((split.index, split.index + 1))
+        }
+      }
+
+      // run a barrier stage
       rdd.barrier().mapPartitions { iter =>
         BarrierTaskContext.get().barrier()
         iter
       }.collect()
-    }.getMessage
-    assert(errorMsg.contains("Fail resource offers for barrier stage"))
+
+      // The delay scheduling for barrier TaskSetManager has been disabled. So, the second task
+      // would not wait for any time but just launch at ANY locality level.
+      assert(taskLocality.sorted === Seq(TaskLocality.PROCESS_LOCAL, TaskLocality.ANY))
+      taskLocality.clear()
+
+      // run a common stage
+      rdd.mapPartitions { iter =>
+        iter
+      }.collect()
+      // The delay scheduling works for the common stage. So, the second task would be delayed
+      // in order to get the better locality.
+      assert(taskLocality.sorted === Seq(TaskLocality.PROCESS_LOCAL, TaskLocality.PROCESS_LOCAL))
+
+    } finally {
+      taskLocality.clear()
+      sc.removeSparkListener(listener)
+    }
+  }
+
+  test("SPARK-34069: Kill barrier tasks should respect SPARK_JOB_INTERRUPT_ON_CANCEL") {
+    sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local[2]"))
+    var index = 0
+    var checkDone = false
+    var startTime = 0L
+    val listener = new SparkListener {
+      override def onTaskStart(taskStart: SparkListenerTaskStart): Unit = {
+        if (startTime == 0) {
+          startTime = taskStart.taskInfo.launchTime
+        }
+      }
+
+      override def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit = {
+        if (index == 0) {
+          assert(taskEnd.reason.isInstanceOf[ExceptionFailure])
+          assert(System.currentTimeMillis() - taskEnd.taskInfo.launchTime < 1000)
+          index = 1
+        } else if (index == 1) {
+          assert(taskEnd.reason.isInstanceOf[TaskKilled])
+          assert(System.currentTimeMillis() - taskEnd.taskInfo.launchTime < 1000)
+          index = 2
+          checkDone = true
+        }
+      }
+    }
+    sc.addSparkListener(listener)
+    sc.setJobGroup("test", "", true)
+    sc.parallelize(Seq(1, 2), 2).barrier().mapPartitions { it =>
+      if (TaskContext.get().stageAttemptNumber() == 0) {
+        if (it.hasNext && it.next() == 1) {
+          throw new RuntimeException("failed")
+        } else {
+          Thread.sleep(5000)
+        }
+      }
+      it
+    }.groupBy(x => x).collect()
+    sc.listenerBus.waitUntilEmpty()
+    assert(checkDone)
+    // double check we kill task success
+    assert(System.currentTimeMillis() - startTime < 5000)
   }
 }
diff --git a/core/src/test/scala/org/apache/spark/scheduler/BlacklistIntegrationSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/BlacklistIntegrationSuite.scala
deleted file mode 100644
index 246d4b2f56ec9..0000000000000
--- a/core/src/test/scala/org/apache/spark/scheduler/BlacklistIntegrationSuite.scala
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.scheduler
-
-import org.apache.spark._
-import org.apache.spark.internal.config
-import org.apache.spark.internal.config.Tests._
-
-class BlacklistIntegrationSuite extends SchedulerIntegrationSuite[MultiExecutorMockBackend]{
-
-  val badHost = "host-0"
-
-  /**
-   * This backend just always fails if the task is executed on a bad host, but otherwise succeeds
-   * all tasks.
-   */
-  def badHostBackend(): Unit = {
-    val (taskDescription, _) = backend.beginTask()
-    val host = backend.executorIdToExecutor(taskDescription.executorId).host
-    if (host == badHost) {
-      backend.taskFailed(taskDescription, new RuntimeException("I'm a bad host!"))
-    } else {
-      backend.taskSuccess(taskDescription, 42)
-    }
-  }
-
-  // Test demonstrating the issue -- without a config change, the scheduler keeps scheduling
-  // according to locality preferences, and so the job fails
-  testScheduler("If preferred node is bad, without blacklist job will fail",
-    extraConfs = Seq(
-      config.BLACKLIST_ENABLED.key -> "false"
-  )) {
-    val rdd = new MockRDDWithLocalityPrefs(sc, 10, Nil, badHost)
-    withBackend(badHostBackend _) {
-      val jobFuture = submit(rdd, (0 until 10).toArray)
-      awaitJobTermination(jobFuture, duration)
-    }
-    assertDataStructuresEmpty(noFailure = false)
-  }
-
-  testScheduler(
-    "With default settings, job can succeed despite multiple bad executors on node",
-    extraConfs = Seq(
-      config.BLACKLIST_ENABLED.key -> "true",
-      config.TASK_MAX_FAILURES.key -> "4",
-      TEST_N_HOSTS.key -> "2",
-      TEST_N_EXECUTORS_HOST.key -> "5",
-      TEST_N_CORES_EXECUTOR.key -> "10"
-    )
-  ) {
-    // To reliably reproduce the failure that would occur without blacklisting, we have to use 1
-    // task.  That way, we ensure this 1 task gets rotated through enough bad executors on the host
-    // to fail the taskSet, before we have a bunch of different tasks fail in the executors so we
-    // blacklist them.
-    // But the point here is -- without blacklisting, we would never schedule anything on the good
-    // host-1 before we hit too many failures trying our preferred host-0.
-    val rdd = new MockRDDWithLocalityPrefs(sc, 1, Nil, badHost)
-    withBackend(badHostBackend _) {
-      val jobFuture = submit(rdd, (0 until 1).toArray)
-      awaitJobTermination(jobFuture, duration)
-    }
-    assertDataStructuresEmpty(noFailure = true)
-  }
-
-  // Here we run with the blacklist on, and the default config takes care of having this
-  // robust to one bad node.
-  testScheduler(
-    "Bad node with multiple executors, job will still succeed with the right confs",
-    extraConfs = Seq(
-       config.BLACKLIST_ENABLED.key -> "true",
-      // just to avoid this test taking too long
-      config.LOCALITY_WAIT.key -> "10ms"
-    )
-  ) {
-    val rdd = new MockRDDWithLocalityPrefs(sc, 10, Nil, badHost)
-    withBackend(badHostBackend _) {
-      val jobFuture = submit(rdd, (0 until 10).toArray)
-      awaitJobTermination(jobFuture, duration)
-    }
-    assert(results === (0 until 10).map { _ -> 42 }.toMap)
-    assertDataStructuresEmpty(noFailure = true)
-  }
-
-  // Make sure that if we've failed on all executors, but haven't hit task.maxFailures yet, we try
-  // to acquire a new executor and if we aren't able to get one, the job doesn't hang and we abort
-  testScheduler(
-    "SPARK-15865 Progress with fewer executors than maxTaskFailures",
-    extraConfs = Seq(
-      config.BLACKLIST_ENABLED.key -> "true",
-      TEST_N_HOSTS.key -> "2",
-      TEST_N_EXECUTORS_HOST.key -> "1",
-      TEST_N_CORES_EXECUTOR.key -> "1",
-      config.UNSCHEDULABLE_TASKSET_TIMEOUT.key -> "0s"
-    )
-  ) {
-    def runBackend(): Unit = {
-      val (taskDescription, _) = backend.beginTask()
-      backend.taskFailed(taskDescription, new RuntimeException("test task failure"))
-    }
-    withBackend(runBackend _) {
-      val jobFuture = submit(new MockRDD(sc, 10, Nil), (0 until 10).toArray)
-      awaitJobTermination(jobFuture, duration)
-      val pattern = (
-        s"""|Aborting TaskSet 0.0 because task .*
-            |cannot run anywhere due to node and executor blacklist""".stripMargin).r
-      assert(pattern.findFirstIn(failure.getMessage).isDefined,
-        s"Couldn't find $pattern in ${failure.getMessage()}")
-    }
-    assertDataStructuresEmpty(noFailure = false)
-  }
-}
-
-class MultiExecutorMockBackend(
-    conf: SparkConf,
-    taskScheduler: TaskSchedulerImpl) extends MockBackend(conf, taskScheduler) {
-
-  val nHosts = conf.get(TEST_N_HOSTS)
-  val nExecutorsPerHost = conf.get(TEST_N_EXECUTORS_HOST)
-  val nCoresPerExecutor = conf.get(TEST_N_CORES_EXECUTOR)
-
-  override val executorIdToExecutor: Map[String, ExecutorTaskStatus] = {
-    (0 until nHosts).flatMap { hostIdx =>
-      val hostName = "host-" + hostIdx
-      (0 until nExecutorsPerHost).map { subIdx =>
-        val executorId = (hostIdx * nExecutorsPerHost + subIdx).toString
-        executorId ->
-          ExecutorTaskStatus(host = hostName, executorId = executorId, nCoresPerExecutor)
-      }
-    }.toMap
-  }
-
-  override def defaultParallelism(): Int = nHosts * nExecutorsPerHost * nCoresPerExecutor
-}
-
-class MockRDDWithLocalityPrefs(
-    sc: SparkContext,
-    numPartitions: Int,
-    shuffleDeps: Seq[ShuffleDependency[Int, Int, Nothing]],
-    val preferredLoc: String) extends MockRDD(sc, numPartitions, shuffleDeps) {
-  override def getPreferredLocations(split: Partition): Seq[String] = {
-    Seq(preferredLoc)
-  }
-}
diff --git a/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala
deleted file mode 100644
index a1671a58f0d9b..0000000000000
--- a/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala
+++ /dev/null
@@ -1,608 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.scheduler
-
-import org.mockito.ArgumentMatchers.any
-import org.mockito.Mockito.{never, verify, when}
-import org.mockito.invocation.InvocationOnMock
-import org.scalatest.BeforeAndAfterEach
-import org.scalatestplus.mockito.MockitoSugar
-
-import org.apache.spark._
-import org.apache.spark.internal.config
-import org.apache.spark.util.ManualClock
-
-class BlacklistTrackerSuite extends SparkFunSuite with BeforeAndAfterEach with MockitoSugar
-    with LocalSparkContext {
-
-  private val clock = new ManualClock(0)
-
-  private var blacklist: BlacklistTracker = _
-  private var listenerBusMock: LiveListenerBus = _
-  private var scheduler: TaskSchedulerImpl = _
-  private var conf: SparkConf = _
-
-  override def beforeEach(): Unit = {
-    conf = new SparkConf().setAppName("test").setMaster("local")
-      .set(config.BLACKLIST_ENABLED.key, "true")
-    scheduler = mockTaskSchedWithConf(conf)
-
-    clock.setTime(0)
-
-    listenerBusMock = mock[LiveListenerBus]
-    blacklist = new BlacklistTracker(listenerBusMock, conf, None, clock)
-  }
-
-  override def afterEach(): Unit = {
-    if (blacklist != null) {
-      blacklist = null
-    }
-    if (scheduler != null) {
-      scheduler.stop()
-      scheduler = null
-    }
-    super.afterEach()
-  }
-
-  // All executors and hosts used in tests should be in this set, so that [[assertEquivalentToSet]]
-  // works.  Its OK if its got extraneous entries
-  val allExecutorAndHostIds = {
-    (('A' to 'Z')++ (1 to 100).map(_.toString))
-      .flatMap{ suffix =>
-        Seq(s"host$suffix", s"host-$suffix")
-      }
-  }.toSet
-
-  /**
-   * Its easier to write our tests as if we could directly look at the sets of nodes & executors in
-   * the blacklist.  However the api doesn't expose a set, so this is a simple way to test
-   * something similar, since we know the universe of values that might appear in these sets.
-   */
-  def assertEquivalentToSet(f: String => Boolean, expected: Set[String]): Unit = {
-    allExecutorAndHostIds.foreach { id =>
-      val actual = f(id)
-      val exp = expected.contains(id)
-      assert(actual === exp, raw"""for string "$id" """)
-    }
-  }
-
-  def mockTaskSchedWithConf(conf: SparkConf): TaskSchedulerImpl = {
-    sc = new SparkContext(conf)
-    val scheduler = mock[TaskSchedulerImpl]
-    when(scheduler.sc).thenReturn(sc)
-    when(scheduler.mapOutputTracker).thenReturn(
-      SparkEnv.get.mapOutputTracker.asInstanceOf[MapOutputTrackerMaster])
-    scheduler
-  }
-
-  def createTaskSetBlacklist(stageId: Int = 0): TaskSetBlacklist = {
-    new TaskSetBlacklist(listenerBusMock, conf, stageId, stageAttemptId = 0, clock = clock)
-  }
-
-  test("executors can be blacklisted with only a few failures per stage") {
-    // For many different stages, executor 1 fails a task, then executor 2 succeeds the task,
-    // and then the task set is done.  Not enough failures to blacklist the executor *within*
-    // any particular taskset, but we still blacklist the executor overall eventually.
-    // Also, we intentionally have a mix of task successes and failures -- there are even some
-    // successes after the executor is blacklisted.  The idea here is those tasks get scheduled
-    // before the executor is blacklisted.  We might get successes after blacklisting (because the
-    // executor might be flaky but not totally broken).  But successes should not unblacklist the
-    // executor.
-    val failuresUntilBlacklisted = conf.get(config.MAX_FAILURES_PER_EXEC)
-    var failuresSoFar = 0
-    (0 until failuresUntilBlacklisted * 10).foreach { stageId =>
-      val taskSetBlacklist = createTaskSetBlacklist(stageId)
-      if (stageId % 2 == 0) {
-        // fail one task in every other taskset
-        taskSetBlacklist.updateBlacklistForFailedTask(
-          "hostA", exec = "1", index = 0, failureReason = "testing")
-        failuresSoFar += 1
-      }
-      blacklist.updateBlacklistForSuccessfulTaskSet(stageId, 0, taskSetBlacklist.execToFailures)
-      assert(failuresSoFar == stageId / 2 + 1)
-      if (failuresSoFar < failuresUntilBlacklisted) {
-        assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set())
-      } else {
-        assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set("1"))
-        verify(listenerBusMock).post(
-          SparkListenerExecutorBlacklisted(0, "1", failuresUntilBlacklisted))
-      }
-    }
-  }
-
-  // If an executor has many task failures, but the task set ends up failing, it shouldn't be
-  // counted against the executor.
-  test("executors aren't blacklisted as a result of tasks in failed task sets") {
-    val failuresUntilBlacklisted = conf.get(config.MAX_FAILURES_PER_EXEC)
-    // for many different stages, executor 1 fails a task, and then the taskSet fails.
-    (0 until failuresUntilBlacklisted * 10).foreach { stage =>
-      val taskSetBlacklist = createTaskSetBlacklist(stage)
-      taskSetBlacklist.updateBlacklistForFailedTask(
-        "hostA", exec = "1", index = 0, failureReason = "testing")
-    }
-    assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set())
-  }
-
-  Seq(true, false).foreach { succeedTaskSet =>
-    val label = if (succeedTaskSet) "success" else "failure"
-    test(s"stage blacklist updates correctly on stage $label") {
-      // Within one taskset, an executor fails a few times, so it's blacklisted for the taskset.
-      // But if the taskset fails, we shouldn't blacklist the executor after the stage.
-      val taskSetBlacklist = createTaskSetBlacklist(0)
-      // We trigger enough failures for both the taskset blacklist, and the application blacklist.
-      val numFailures = math.max(conf.get(config.MAX_FAILURES_PER_EXEC),
-        conf.get(config.MAX_FAILURES_PER_EXEC_STAGE))
-      (0 until numFailures).foreach { index =>
-        taskSetBlacklist.updateBlacklistForFailedTask(
-          "hostA", exec = "1", index = index, failureReason = "testing")
-      }
-      assert(taskSetBlacklist.isExecutorBlacklistedForTaskSet("1"))
-      assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set())
-      if (succeedTaskSet) {
-        // The task set succeeded elsewhere, so we should count those failures against our executor,
-        // and it should be blacklisted for the entire application.
-        blacklist.updateBlacklistForSuccessfulTaskSet(0, 0, taskSetBlacklist.execToFailures)
-        assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set("1"))
-        verify(listenerBusMock).post(SparkListenerExecutorBlacklisted(0, "1", numFailures))
-      } else {
-        // The task set failed, so we don't count these failures against the executor for other
-        // stages.
-        assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set())
-      }
-    }
-  }
-
-  test("blacklisted executors and nodes get recovered with time") {
-    val taskSetBlacklist0 = createTaskSetBlacklist(stageId = 0)
-    // Fail 4 tasks in one task set on executor 1, so that executor gets blacklisted for the whole
-    // application.
-    (0 until 4).foreach { partition =>
-      taskSetBlacklist0.updateBlacklistForFailedTask(
-        "hostA", exec = "1", index = partition, failureReason = "testing")
-    }
-    blacklist.updateBlacklistForSuccessfulTaskSet(0, 0, taskSetBlacklist0.execToFailures)
-    assert(blacklist.nodeBlacklist() === Set())
-    assertEquivalentToSet(blacklist.isNodeBlacklisted(_), Set())
-    assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set("1"))
-    verify(listenerBusMock).post(SparkListenerExecutorBlacklisted(0, "1", 4))
-
-    val taskSetBlacklist1 = createTaskSetBlacklist(stageId = 1)
-    // Fail 4 tasks in one task set on executor 2, so that executor gets blacklisted for the whole
-    // application.  Since that's the second executor that is blacklisted on the same node, we also
-    // blacklist that node.
-    (0 until 4).foreach { partition =>
-      taskSetBlacklist1.updateBlacklistForFailedTask(
-        "hostA", exec = "2", index = partition, failureReason = "testing")
-    }
-    blacklist.updateBlacklistForSuccessfulTaskSet(0, 0, taskSetBlacklist1.execToFailures)
-    assert(blacklist.nodeBlacklist() === Set("hostA"))
-    assertEquivalentToSet(blacklist.isNodeBlacklisted(_), Set("hostA"))
-    verify(listenerBusMock).post(SparkListenerNodeBlacklisted(0, "hostA", 2))
-    assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set("1", "2"))
-    verify(listenerBusMock).post(SparkListenerExecutorBlacklisted(0, "2", 4))
-
-    // Advance the clock and then make sure hostA and executors 1 and 2 have been removed from the
-    // blacklist.
-    val timeout = blacklist.BLACKLIST_TIMEOUT_MILLIS + 1
-    clock.advance(timeout)
-    blacklist.applyBlacklistTimeout()
-    assert(blacklist.nodeBlacklist() === Set())
-    assertEquivalentToSet(blacklist.isNodeBlacklisted(_), Set())
-    assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set())
-    verify(listenerBusMock).post(SparkListenerExecutorUnblacklisted(timeout, "2"))
-    verify(listenerBusMock).post(SparkListenerExecutorUnblacklisted(timeout, "1"))
-    verify(listenerBusMock).post(SparkListenerNodeUnblacklisted(timeout, "hostA"))
-
-    // Fail one more task, but executor isn't put back into blacklist since the count of failures
-    // on that executor should have been reset to 0.
-    val taskSetBlacklist2 = createTaskSetBlacklist(stageId = 2)
-    taskSetBlacklist2.updateBlacklistForFailedTask(
-      "hostA", exec = "1", index = 0, failureReason = "testing")
-    blacklist.updateBlacklistForSuccessfulTaskSet(2, 0, taskSetBlacklist2.execToFailures)
-    assert(blacklist.nodeBlacklist() === Set())
-    assertEquivalentToSet(blacklist.isNodeBlacklisted(_), Set())
-    assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set())
-  }
-
-  test("blacklist can handle lost executors") {
-    // The blacklist should still work if an executor is killed completely.  We should still
-    // be able to blacklist the entire node.
-    val taskSetBlacklist0 = createTaskSetBlacklist(stageId = 0)
-    // Lets say that executor 1 dies completely.  We get some task failures, but
-    // the taskset then finishes successfully (elsewhere).
-    (0 until 4).foreach { partition =>
-      taskSetBlacklist0.updateBlacklistForFailedTask(
-        "hostA", exec = "1", index = partition, failureReason = "testing")
-    }
-    blacklist.handleRemovedExecutor("1")
-    blacklist.updateBlacklistForSuccessfulTaskSet(
-      stageId = 0,
-      stageAttemptId = 0,
-      taskSetBlacklist0.execToFailures)
-    assert(blacklist.isExecutorBlacklisted("1"))
-    verify(listenerBusMock).post(SparkListenerExecutorBlacklisted(0, "1", 4))
-    val t1 = blacklist.BLACKLIST_TIMEOUT_MILLIS / 2
-    clock.advance(t1)
-
-    // Now another executor gets spun up on that host, but it also dies.
-    val taskSetBlacklist1 = createTaskSetBlacklist(stageId = 1)
-    (0 until 4).foreach { partition =>
-      taskSetBlacklist1.updateBlacklistForFailedTask(
-        "hostA", exec = "2", index = partition, failureReason = "testing")
-    }
-    blacklist.handleRemovedExecutor("2")
-    blacklist.updateBlacklistForSuccessfulTaskSet(
-      stageId = 1,
-      stageAttemptId = 0,
-      taskSetBlacklist1.execToFailures)
-    // We've now had two bad executors on the hostA, so we should blacklist the entire node.
-    assert(blacklist.isExecutorBlacklisted("1"))
-    assert(blacklist.isExecutorBlacklisted("2"))
-    verify(listenerBusMock).post(SparkListenerExecutorBlacklisted(t1, "2", 4))
-    assert(blacklist.isNodeBlacklisted("hostA"))
-    verify(listenerBusMock).post(SparkListenerNodeBlacklisted(t1, "hostA", 2))
-
-    // Advance the clock so that executor 1 should no longer be explicitly blacklisted, but
-    // everything else should still be blacklisted.
-    val t2 = blacklist.BLACKLIST_TIMEOUT_MILLIS / 2 + 1
-    clock.advance(t2)
-    blacklist.applyBlacklistTimeout()
-    assert(!blacklist.isExecutorBlacklisted("1"))
-    verify(listenerBusMock).post(SparkListenerExecutorUnblacklisted(t1 + t2, "1"))
-    assert(blacklist.isExecutorBlacklisted("2"))
-    assert(blacklist.isNodeBlacklisted("hostA"))
-    // make sure we don't leak memory
-    assert(!blacklist.executorIdToBlacklistStatus.contains("1"))
-    assert(!blacklist.nodeToBlacklistedExecs("hostA").contains("1"))
-    // Advance the timeout again so now hostA should be removed from the blacklist.
-    clock.advance(t1)
-    blacklist.applyBlacklistTimeout()
-    assert(!blacklist.nodeIdToBlacklistExpiryTime.contains("hostA"))
-    verify(listenerBusMock).post(SparkListenerNodeUnblacklisted(t1 + t2 + t1, "hostA"))
-    // Even though unblacklisting a node implicitly unblacklists all of its executors,
-    // there will be no SparkListenerExecutorUnblacklisted sent here.
-  }
-
-  test("task failures expire with time") {
-    // Verifies that 2 failures within the timeout period cause an executor to be blacklisted, but
-    // if task failures are spaced out by more than the timeout period, the first failure is timed
-    // out, and the executor isn't blacklisted.
-    var stageId = 0
-
-    def failOneTaskInTaskSet(exec: String): Unit = {
-      val taskSetBlacklist = createTaskSetBlacklist(stageId = stageId)
-      taskSetBlacklist.updateBlacklistForFailedTask("host-" + exec, exec, 0, "testing")
-      blacklist.updateBlacklistForSuccessfulTaskSet(stageId, 0, taskSetBlacklist.execToFailures)
-      stageId += 1
-    }
-
-    failOneTaskInTaskSet(exec = "1")
-    // We have one sporadic failure on exec 2, but that's it.  Later checks ensure that we never
-    // blacklist executor 2 despite this one failure.
-    failOneTaskInTaskSet(exec = "2")
-    assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set())
-    assert(blacklist.nextExpiryTime === Long.MaxValue)
-
-    // We advance the clock past the expiry time.
-    clock.advance(blacklist.BLACKLIST_TIMEOUT_MILLIS + 1)
-    val t0 = clock.getTimeMillis()
-    blacklist.applyBlacklistTimeout()
-    assert(blacklist.nextExpiryTime === Long.MaxValue)
-    failOneTaskInTaskSet(exec = "1")
-
-    // Because the 2nd failure on executor 1 happened past the expiry time, nothing should have been
-    // blacklisted.
-    assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set())
-
-    // Now we add one more failure, within the timeout, and it should be counted.
-    clock.setTime(t0 + blacklist.BLACKLIST_TIMEOUT_MILLIS - 1)
-    val t1 = clock.getTimeMillis()
-    failOneTaskInTaskSet(exec = "1")
-    blacklist.applyBlacklistTimeout()
-    assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set("1"))
-    verify(listenerBusMock).post(SparkListenerExecutorBlacklisted(t1, "1", 2))
-    assert(blacklist.nextExpiryTime === t1 + blacklist.BLACKLIST_TIMEOUT_MILLIS)
-
-    // Add failures on executor 3, make sure it gets put on the blacklist.
-    clock.setTime(t1 + blacklist.BLACKLIST_TIMEOUT_MILLIS - 1)
-    val t2 = clock.getTimeMillis()
-    failOneTaskInTaskSet(exec = "3")
-    failOneTaskInTaskSet(exec = "3")
-    blacklist.applyBlacklistTimeout()
-    assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set("1", "3"))
-    verify(listenerBusMock).post(SparkListenerExecutorBlacklisted(t2, "3", 2))
-    assert(blacklist.nextExpiryTime === t1 + blacklist.BLACKLIST_TIMEOUT_MILLIS)
-
-    // Now we go past the timeout for executor 1, so it should be dropped from the blacklist.
-    clock.setTime(t1 + blacklist.BLACKLIST_TIMEOUT_MILLIS + 1)
-    blacklist.applyBlacklistTimeout()
-    assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set("3"))
-    verify(listenerBusMock).post(SparkListenerExecutorUnblacklisted(clock.getTimeMillis(), "1"))
-    assert(blacklist.nextExpiryTime === t2 + blacklist.BLACKLIST_TIMEOUT_MILLIS)
-
-    // Make sure that we update correctly when we go from having blacklisted executors to
-    // just having tasks with timeouts.
-    clock.setTime(t2 + blacklist.BLACKLIST_TIMEOUT_MILLIS - 1)
-    failOneTaskInTaskSet(exec = "4")
-    blacklist.applyBlacklistTimeout()
-    assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set("3"))
-    assert(blacklist.nextExpiryTime === t2 + blacklist.BLACKLIST_TIMEOUT_MILLIS)
-
-    clock.setTime(t2 + blacklist.BLACKLIST_TIMEOUT_MILLIS + 1)
-    blacklist.applyBlacklistTimeout()
-    assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set())
-    verify(listenerBusMock).post(SparkListenerExecutorUnblacklisted(clock.getTimeMillis(), "3"))
-    // we've got one task failure still, but we don't bother setting nextExpiryTime to it, to
-    // avoid wasting time checking for expiry of individual task failures.
-    assert(blacklist.nextExpiryTime === Long.MaxValue)
-  }
-
-  test("task failure timeout works as expected for long-running tasksets") {
-    // This ensures that we don't trigger spurious blacklisting for long tasksets, when the taskset
-    // finishes long after the task failures.  We create two tasksets, each with one failure.
-    // Individually they shouldn't cause any blacklisting since there is only one failure.
-    // Furthermore, we space the failures out so far that even when both tasksets have completed,
-    // we still don't trigger any blacklisting.
-    val taskSetBlacklist1 = createTaskSetBlacklist(stageId = 1)
-    val taskSetBlacklist2 = createTaskSetBlacklist(stageId = 2)
-    // Taskset1 has one failure immediately
-    taskSetBlacklist1.updateBlacklistForFailedTask("host-1", "1", 0, "testing")
-    // Then we have a *long* delay, much longer than the timeout, before any other failures or
-    // taskset completion
-    clock.advance(blacklist.BLACKLIST_TIMEOUT_MILLIS * 5)
-    // After the long delay, we have one failure on taskset 2, on the same executor
-    taskSetBlacklist2.updateBlacklistForFailedTask("host-1", "1", 0, "testing")
-    // Finally, we complete both tasksets.  Its important here to complete taskset2 *first*.  We
-    // want to make sure that when taskset 1 finishes, even though we've now got two task failures,
-    // we realize that the task failure we just added was well before the timeout.
-    clock.advance(1)
-    blacklist.updateBlacklistForSuccessfulTaskSet(stageId = 2, 0, taskSetBlacklist2.execToFailures)
-    clock.advance(1)
-    blacklist.updateBlacklistForSuccessfulTaskSet(stageId = 1, 0, taskSetBlacklist1.execToFailures)
-
-    // Make sure nothing was blacklisted
-    assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set())
-  }
-
-  test("only blacklist nodes for the application when enough executors have failed on that " +
-    "specific host") {
-    // we blacklist executors on two different hosts -- make sure that doesn't lead to any
-    // node blacklisting
-    val taskSetBlacklist0 = createTaskSetBlacklist(stageId = 0)
-    taskSetBlacklist0.updateBlacklistForFailedTask(
-      "hostA", exec = "1", index = 0, failureReason = "testing")
-    taskSetBlacklist0.updateBlacklistForFailedTask(
-      "hostA", exec = "1", index = 1, failureReason = "testing")
-    blacklist.updateBlacklistForSuccessfulTaskSet(0, 0, taskSetBlacklist0.execToFailures)
-    assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set("1"))
-    verify(listenerBusMock).post(SparkListenerExecutorBlacklisted(0, "1", 2))
-    assertEquivalentToSet(blacklist.isNodeBlacklisted(_), Set())
-
-    val taskSetBlacklist1 = createTaskSetBlacklist(stageId = 1)
-    taskSetBlacklist1.updateBlacklistForFailedTask(
-      "hostB", exec = "2", index = 0, failureReason = "testing")
-    taskSetBlacklist1.updateBlacklistForFailedTask(
-      "hostB", exec = "2", index = 1, failureReason = "testing")
-    blacklist.updateBlacklistForSuccessfulTaskSet(1, 0, taskSetBlacklist1.execToFailures)
-    assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set("1", "2"))
-    verify(listenerBusMock).post(SparkListenerExecutorBlacklisted(0, "2", 2))
-    assertEquivalentToSet(blacklist.isNodeBlacklisted(_), Set())
-
-    // Finally, blacklist another executor on the same node as the original blacklisted executor,
-    // and make sure this time we *do* blacklist the node.
-    val taskSetBlacklist2 = createTaskSetBlacklist(stageId = 0)
-    taskSetBlacklist2.updateBlacklistForFailedTask(
-      "hostA", exec = "3", index = 0, failureReason = "testing")
-    taskSetBlacklist2.updateBlacklistForFailedTask(
-      "hostA", exec = "3", index = 1, failureReason = "testing")
-    blacklist.updateBlacklistForSuccessfulTaskSet(0, 0, taskSetBlacklist2.execToFailures)
-    assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set("1", "2", "3"))
-    verify(listenerBusMock).post(SparkListenerExecutorBlacklisted(0, "3", 2))
-    assertEquivalentToSet(blacklist.isNodeBlacklisted(_), Set("hostA"))
-    verify(listenerBusMock).post(SparkListenerNodeBlacklisted(0, "hostA", 2))
-  }
-
-  test("blacklist still respects legacy configs") {
-    val conf = new SparkConf().setMaster("local")
-    assert(!BlacklistTracker.isBlacklistEnabled(conf))
-    conf.set(config.BLACKLIST_LEGACY_TIMEOUT_CONF, 5000L)
-    assert(BlacklistTracker.isBlacklistEnabled(conf))
-    assert(5000 === BlacklistTracker.getBlacklistTimeout(conf))
-    // the new conf takes precedence, though
-    conf.set(config.BLACKLIST_TIMEOUT_CONF, 1000L)
-    assert(1000 === BlacklistTracker.getBlacklistTimeout(conf))
-
-    // if you explicitly set the legacy conf to 0, that also would disable blacklisting
-    conf.set(config.BLACKLIST_LEGACY_TIMEOUT_CONF, 0L)
-    assert(!BlacklistTracker.isBlacklistEnabled(conf))
-    // but again, the new conf takes precedence
-    conf.set(config.BLACKLIST_ENABLED, true)
-    assert(BlacklistTracker.isBlacklistEnabled(conf))
-    assert(1000 === BlacklistTracker.getBlacklistTimeout(conf))
-  }
-
-  test("check blacklist configuration invariants") {
-    val conf = new SparkConf().setMaster("yarn").set(config.SUBMIT_DEPLOY_MODE, "cluster")
-    Seq(
-      (2, 2),
-      (2, 3)
-    ).foreach { case (maxTaskFailures, maxNodeAttempts) =>
-      conf.set(config.TASK_MAX_FAILURES, maxTaskFailures)
-      conf.set(config.MAX_TASK_ATTEMPTS_PER_NODE.key, maxNodeAttempts.toString)
-      val excMsg = intercept[IllegalArgumentException] {
-        BlacklistTracker.validateBlacklistConfs(conf)
-      }.getMessage()
-      assert(excMsg === s"${config.MAX_TASK_ATTEMPTS_PER_NODE.key} " +
-        s"( = ${maxNodeAttempts}) was >= ${config.TASK_MAX_FAILURES.key} " +
-        s"( = ${maxTaskFailures} ).  Though blacklisting is enabled, with this configuration, " +
-        s"Spark will not be robust to one bad node.  Decrease " +
-        s"${config.MAX_TASK_ATTEMPTS_PER_NODE.key}, increase ${config.TASK_MAX_FAILURES.key}, " +
-        s"or disable blacklisting with ${config.BLACKLIST_ENABLED.key}")
-    }
-
-    conf.remove(config.TASK_MAX_FAILURES)
-    conf.remove(config.MAX_TASK_ATTEMPTS_PER_NODE)
-
-    Seq(
-      config.MAX_TASK_ATTEMPTS_PER_EXECUTOR,
-      config.MAX_TASK_ATTEMPTS_PER_NODE,
-      config.MAX_FAILURES_PER_EXEC_STAGE,
-      config.MAX_FAILED_EXEC_PER_NODE_STAGE,
-      config.MAX_FAILURES_PER_EXEC,
-      config.MAX_FAILED_EXEC_PER_NODE,
-      config.BLACKLIST_TIMEOUT_CONF
-    ).foreach { config =>
-      conf.set(config.key, "0")
-      val excMsg = intercept[IllegalArgumentException] {
-        BlacklistTracker.validateBlacklistConfs(conf)
-      }.getMessage()
-      assert(excMsg.contains(s"${config.key} was 0, but must be > 0."))
-      conf.remove(config)
-    }
-  }
-
-  test("blacklisting kills executors, configured by BLACKLIST_KILL_ENABLED") {
-    val allocationClientMock = mock[ExecutorAllocationClient]
-    when(allocationClientMock.killExecutors(any(), any(), any(), any())).thenReturn(Seq("called"))
-    when(allocationClientMock.killExecutorsOnHost("hostA")).thenAnswer { (_: InvocationOnMock) =>
-      // To avoid a race between blacklisting and killing, it is important that the nodeBlacklist
-      // is updated before we ask the executor allocation client to kill all the executors
-      // on a particular host.
-      if (blacklist.nodeBlacklist.contains("hostA")) {
-        true
-      } else {
-        throw new IllegalStateException("hostA should be on the blacklist")
-      }
-    }
-    blacklist = new BlacklistTracker(listenerBusMock, conf, Some(allocationClientMock), clock)
-
-    // Disable auto-kill. Blacklist an executor and make sure killExecutors is not called.
-    conf.set(config.BLACKLIST_KILL_ENABLED, false)
-
-    val taskSetBlacklist0 = createTaskSetBlacklist(stageId = 0)
-    // Fail 4 tasks in one task set on executor 1, so that executor gets blacklisted for the whole
-    // application.
-    (0 until 4).foreach { partition =>
-      taskSetBlacklist0.updateBlacklistForFailedTask(
-        "hostA", exec = "1", index = partition, failureReason = "testing")
-    }
-    blacklist.updateBlacklistForSuccessfulTaskSet(0, 0, taskSetBlacklist0.execToFailures)
-
-    verify(allocationClientMock, never).killExecutor(any())
-
-    val taskSetBlacklist1 = createTaskSetBlacklist(stageId = 1)
-    // Fail 4 tasks in one task set on executor 2, so that executor gets blacklisted for the whole
-    // application.  Since that's the second executor that is blacklisted on the same node, we also
-    // blacklist that node.
-    (0 until 4).foreach { partition =>
-      taskSetBlacklist1.updateBlacklistForFailedTask(
-        "hostA", exec = "2", index = partition, failureReason = "testing")
-    }
-    blacklist.updateBlacklistForSuccessfulTaskSet(0, 0, taskSetBlacklist1.execToFailures)
-
-    verify(allocationClientMock, never).killExecutors(any(), any(), any(), any())
-    verify(allocationClientMock, never).killExecutorsOnHost(any())
-
-    // Enable auto-kill. Blacklist an executor and make sure killExecutors is called.
-    conf.set(config.BLACKLIST_KILL_ENABLED, true)
-    blacklist = new BlacklistTracker(listenerBusMock, conf, Some(allocationClientMock), clock)
-
-    val taskSetBlacklist2 = createTaskSetBlacklist(stageId = 0)
-    // Fail 4 tasks in one task set on executor 1, so that executor gets blacklisted for the whole
-    // application.
-    (0 until 4).foreach { partition =>
-      taskSetBlacklist2.updateBlacklistForFailedTask(
-        "hostA", exec = "1", index = partition, failureReason = "testing")
-    }
-    blacklist.updateBlacklistForSuccessfulTaskSet(0, 0, taskSetBlacklist2.execToFailures)
-
-    verify(allocationClientMock).killExecutors(Seq("1"), false, false, true)
-
-    val taskSetBlacklist3 = createTaskSetBlacklist(stageId = 1)
-    // Fail 4 tasks in one task set on executor 2, so that executor gets blacklisted for the whole
-    // application.  Since that's the second executor that is blacklisted on the same node, we also
-    // blacklist that node.
-    (0 until 4).foreach { partition =>
-      taskSetBlacklist3.updateBlacklistForFailedTask(
-        "hostA", exec = "2", index = partition, failureReason = "testing")
-    }
-    blacklist.updateBlacklistForSuccessfulTaskSet(0, 0, taskSetBlacklist3.execToFailures)
-
-    verify(allocationClientMock).killExecutors(Seq("2"), false, false, true)
-    verify(allocationClientMock).killExecutorsOnHost("hostA")
-  }
-
-  test("fetch failure blacklisting kills executors, configured by BLACKLIST_KILL_ENABLED") {
-    val allocationClientMock = mock[ExecutorAllocationClient]
-    when(allocationClientMock.killExecutors(any(), any(), any(), any())).thenReturn(Seq("called"))
-    when(allocationClientMock.killExecutorsOnHost("hostA")).thenAnswer { (_: InvocationOnMock) =>
-      // To avoid a race between blacklisting and killing, it is important that the nodeBlacklist
-      // is updated before we ask the executor allocation client to kill all the executors
-      // on a particular host.
-      if (blacklist.nodeBlacklist.contains("hostA")) {
-        true
-      } else {
-        throw new IllegalStateException("hostA should be on the blacklist")
-      }
-    }
-
-    conf.set(config.BLACKLIST_FETCH_FAILURE_ENABLED, true)
-    blacklist = new BlacklistTracker(listenerBusMock, conf, Some(allocationClientMock), clock)
-
-    // Disable auto-kill. Blacklist an executor and make sure killExecutors is not called.
-    conf.set(config.BLACKLIST_KILL_ENABLED, false)
-    blacklist.updateBlacklistForFetchFailure("hostA", exec = "1")
-
-    verify(allocationClientMock, never).killExecutors(any(), any(), any(), any())
-    verify(allocationClientMock, never).killExecutorsOnHost(any())
-
-    assert(blacklist.nodeToBlacklistedExecs.contains("hostA"))
-    assert(blacklist.nodeToBlacklistedExecs("hostA").contains("1"))
-
-    // Enable auto-kill. Blacklist an executor and make sure killExecutors is called.
-    conf.set(config.BLACKLIST_KILL_ENABLED, true)
-    blacklist = new BlacklistTracker(listenerBusMock, conf, Some(allocationClientMock), clock)
-    clock.advance(1000)
-    blacklist.updateBlacklistForFetchFailure("hostA", exec = "1")
-
-    verify(allocationClientMock).killExecutors(Seq("1"), false, false, true)
-    verify(allocationClientMock, never).killExecutorsOnHost(any())
-
-    assert(blacklist.executorIdToBlacklistStatus.contains("1"))
-    assert(blacklist.executorIdToBlacklistStatus("1").node === "hostA")
-    assert(blacklist.executorIdToBlacklistStatus("1").expiryTime ===
-      1000 + blacklist.BLACKLIST_TIMEOUT_MILLIS)
-    assert(blacklist.nextExpiryTime === 1000 + blacklist.BLACKLIST_TIMEOUT_MILLIS)
-    assert(blacklist.nodeIdToBlacklistExpiryTime.isEmpty)
-    assert(blacklist.nodeToBlacklistedExecs.contains("hostA"))
-    assert(blacklist.nodeToBlacklistedExecs("hostA").contains("1"))
-
-    // Enable external shuffle service to see if all the executors on this node will be killed.
-    conf.set(config.SHUFFLE_SERVICE_ENABLED, true)
-    clock.advance(1000)
-    blacklist.updateBlacklistForFetchFailure("hostA", exec = "2")
-
-    verify(allocationClientMock, never).killExecutors(Seq("2"), true, true)
-    verify(allocationClientMock).killExecutorsOnHost("hostA")
-
-    assert(blacklist.nodeIdToBlacklistExpiryTime.contains("hostA"))
-    assert(blacklist.nodeIdToBlacklistExpiryTime("hostA") ===
-      2000 + blacklist.BLACKLIST_TIMEOUT_MILLIS)
-    assert(blacklist.nextExpiryTime === 1000 + blacklist.BLACKLIST_TIMEOUT_MILLIS)
-  }
-}
diff --git a/core/src/test/scala/org/apache/spark/scheduler/CoarseGrainedSchedulerBackendSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/CoarseGrainedSchedulerBackendSuite.scala
index d648293fdbe06..3ce4ccf8d632e 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/CoarseGrainedSchedulerBackendSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/CoarseGrainedSchedulerBackendSuite.scala
@@ -99,8 +99,8 @@ class CoarseGrainedSchedulerBackendSuite extends SparkFunSuite with LocalSparkCo
       Thread.sleep(5000)
       iter
     }
-    var taskStarted = new AtomicBoolean(false)
-    var taskEnded = new AtomicBoolean(false)
+    val taskStarted = new AtomicBoolean(false)
+    val taskEnded = new AtomicBoolean(false)
     val listener = new SparkListener() {
       override def onTaskStart(taskStart: SparkListenerTaskStart): Unit = {
         taskStarted.set(true)
@@ -188,7 +188,6 @@ class CoarseGrainedSchedulerBackendSuite extends SparkFunSuite with LocalSparkCo
   }
 
   test("extra resources from executor") {
-    import TestUtils._
 
     val execCores = 3
     val conf = new SparkConf()
@@ -240,12 +239,13 @@ class CoarseGrainedSchedulerBackendSuite extends SparkFunSuite with LocalSparkCo
     var execResources = backend.getExecutorAvailableResources("1")
     assert(execResources(GPU).availableAddrs.sorted === Array("0", "1", "3"))
 
-    var exec3ResourceProfileId = backend.getExecutorResourceProfileId("3")
+    val exec3ResourceProfileId = backend.getExecutorResourceProfileId("3")
     assert(exec3ResourceProfileId === rp.id)
 
     val taskResources = Map(GPU -> new ResourceInformation(GPU, Array("0")))
-    var taskDescs: Seq[Seq[TaskDescription]] = Seq(Seq(new TaskDescription(1, 0, "1",
-      "t1", 0, 1, mutable.Map.empty[String, Long], mutable.Map.empty[String, Long],
+    val taskDescs: Seq[Seq[TaskDescription]] = Seq(Seq(new TaskDescription(1, 0, "1",
+      "t1", 0, 1, mutable.Map.empty[String, Long],
+      mutable.Map.empty[String, Long], mutable.Map.empty[String, Long],
       new Properties(), taskResources, bytebuffer)))
     val ts = backend.getTaskSchedulerImpl()
     when(ts.resourceOffers(any[IndexedSeq[WorkerOffer]], any[Boolean])).thenReturn(taskDescs)
@@ -300,7 +300,7 @@ private class CSMockExternalClusterManager extends ExternalClusterManager {
     when(ts.applicationId()).thenReturn("appid1")
     when(ts.applicationAttemptId()).thenReturn(Some("attempt1"))
     when(ts.schedulingMode).thenReturn(SchedulingMode.FIFO)
-    when(ts.nodeBlacklist()).thenReturn(Set.empty[String])
+    when(ts.excludedNodes()).thenReturn(Set.empty[String])
     ts
   }
 
diff --git a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
index 4c6033edff247..4c74e4fbb3728 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
@@ -19,12 +19,15 @@ package org.apache.spark.scheduler
 
 import java.util.Properties
 import java.util.concurrent.{CountDownLatch, TimeUnit}
-import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger, AtomicLong, AtomicReference}
+import java.util.concurrent.atomic.{AtomicBoolean, AtomicLong, AtomicReference}
 
 import scala.annotation.meta.param
 import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet, Map}
 import scala.util.control.NonFatal
 
+import org.mockito.Mockito.spy
+import org.mockito.Mockito.times
+import org.mockito.Mockito.verify
 import org.scalatest.concurrent.{Signaler, ThreadSignaler, TimeLimits}
 import org.scalatest.exceptions.TestFailedException
 import org.scalatest.time.SpanSugar._
@@ -39,7 +42,7 @@ import org.apache.spark.resource.ResourceUtils.{FPGA, GPU}
 import org.apache.spark.scheduler.SchedulingMode.SchedulingMode
 import org.apache.spark.shuffle.{FetchFailedException, MetadataFetchFailedException}
 import org.apache.spark.storage.{BlockId, BlockManagerId, BlockManagerMaster}
-import org.apache.spark.util.{AccumulatorContext, AccumulatorV2, CallSite, LongAccumulator, ThreadUtils, Utils}
+import org.apache.spark.util.{AccumulatorContext, AccumulatorV2, CallSite, LongAccumulator, Utils}
 
 class DAGSchedulerEventProcessLoopTester(dagScheduler: DAGScheduler)
   extends DAGSchedulerEventProcessLoop(dagScheduler) {
@@ -122,14 +125,14 @@ class MyRDD(
 
 class DAGSchedulerSuiteDummyException extends Exception
 
-class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLimits {
+class DAGSchedulerSuite extends SparkFunSuite with TempLocalSparkContext with TimeLimits {
 
   import DAGSchedulerSuite._
 
   // Necessary to make ScalaTest 3.x interrupt a thread on the JVM like ScalaTest 2.2.x
   implicit val defaultSignaler: Signaler = ThreadSignaler
 
-  val conf = new SparkConf
+  private var firstInit: Boolean = _
   /** Set of TaskSets the DAGScheduler has requested executed. */
   val taskSets = scala.collection.mutable.Buffer[TaskSet]()
 
@@ -169,10 +172,14 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     }
     override def setDAGScheduler(dagScheduler: DAGScheduler) = {}
     override def defaultParallelism() = 2
-    override def executorDecommission(executorId: String) = {}
     override def executorLost(executorId: String, reason: ExecutorLossReason): Unit = {}
     override def workerRemoved(workerId: String, host: String, message: String): Unit = {}
     override def applicationAttemptId(): Option[String] = None
+    override def executorDecommission(
+      executorId: String,
+      decommissionInfo: ExecutorDecommissionInfo): Unit = {}
+    override def getExecutorDecommissionState(
+      executorId: String): Option[ExecutorDecommissionState] = None
   }
 
   /**
@@ -235,6 +242,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
 
   var sparkListener: EventInfoRecordingListener = null
 
+  var blockManagerMaster: BlockManagerMaster = null
   var mapOutputTracker: MapOutputTrackerMaster = null
   var broadcastManager: BroadcastManager = null
   var securityMgr: SecurityManager = null
@@ -248,17 +256,18 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
    */
   val cacheLocations = new HashMap[(Int, Int), Seq[BlockManagerId]]
   // stub out BlockManagerMaster.getLocations to use our cacheLocations
-  val blockManagerMaster = new BlockManagerMaster(null, null, conf, true) {
-      override def getLocations(blockIds: Array[BlockId]): IndexedSeq[Seq[BlockManagerId]] = {
-        blockIds.map {
-          _.asRDDId.map(id => (id.rddId -> id.splitIndex)).flatMap(key => cacheLocations.get(key)).
-            getOrElse(Seq())
-        }.toIndexedSeq
-      }
-      override def removeExecutor(execId: String): Unit = {
-        // don't need to propagate to the driver, which we don't have
-      }
+  class MyBlockManagerMaster(conf: SparkConf) extends BlockManagerMaster(null, null, conf, true) {
+    override def getLocations(blockIds: Array[BlockId]): IndexedSeq[Seq[BlockManagerId]] = {
+      blockIds.map {
+        _.asRDDId.map { id => (id.rddId -> id.splitIndex)
+        }.flatMap { key => cacheLocations.get(key)
+        }.getOrElse(Seq())
+      }.toIndexedSeq
     }
+    override def removeExecutor(execId: String): Unit = {
+      // don't need to propagate to the driver, which we don't have
+    }
+  }
 
   /** The list of results that DAGScheduler has collected. */
   val results = new HashMap[Int, Any]()
@@ -276,13 +285,31 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     override def jobFailed(exception: Exception): Unit = { failure = exception }
   }
 
+  class MyMapOutputTrackerMaster(
+      conf: SparkConf,
+      broadcastManager: BroadcastManager)
+    extends MapOutputTrackerMaster(conf, broadcastManager, true) {
+
+    override def sendTracker(message: Any): Unit = {
+      // no-op, just so we can stop this to avoid leaking threads
+    }
+  }
+
   override def beforeEach(): Unit = {
     super.beforeEach()
-    init(new SparkConf())
+    firstInit = true
+  }
+
+  override def sc: SparkContext = {
+    val sc = super.sc
+    if (firstInit) {
+      init(sc)
+      firstInit = false
+    }
+    sc
   }
 
-  private def init(testConf: SparkConf): Unit = {
-    sc = new SparkContext("local[2]", "DAGSchedulerSuite", testConf)
+  private def init(sc: SparkContext): Unit = {
     sparkListener = new EventInfoRecordingListener
     failure = null
     sc.addSparkListener(sparkListener)
@@ -291,13 +318,10 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     cancelledStages.clear()
     cacheLocations.clear()
     results.clear()
-    securityMgr = new SecurityManager(conf)
-    broadcastManager = new BroadcastManager(true, conf, securityMgr)
-    mapOutputTracker = new MapOutputTrackerMaster(conf, broadcastManager, true) {
-      override def sendTracker(message: Any): Unit = {
-        // no-op, just so we can stop this to avoid leaking threads
-      }
-    }
+    securityMgr = new SecurityManager(sc.getConf)
+    broadcastManager = new BroadcastManager(true, sc.getConf)
+    mapOutputTracker = spy(new MyMapOutputTrackerMaster(sc.getConf, broadcastManager))
+    blockManagerMaster = spy(new MyBlockManagerMaster(sc.getConf))
     scheduler = new DAGScheduler(
       sc,
       taskScheduler,
@@ -337,6 +361,8 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
    * DAGScheduler event loop.
    */
   private def runEvent(event: DAGSchedulerEvent): Unit = {
+    // Ensure the initialization of various components
+    sc
     dagEventProcessLoopTester.post(event)
   }
 
@@ -349,9 +375,9 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     it.next.asInstanceOf[Tuple2[_, _]]._1
 
   /** Send the given CompletionEvent messages for the tasks in the TaskSet. */
-  private def complete(taskSet: TaskSet, results: Seq[(TaskEndReason, Any)]): Unit = {
-    assert(taskSet.tasks.size >= results.size)
-    for ((result, i) <- results.zipWithIndex) {
+  private def complete(taskSet: TaskSet, taskEndInfos: Seq[(TaskEndReason, Any)]): Unit = {
+    assert(taskSet.tasks.size >= taskEndInfos.size)
+    for ((result, i) <- taskEndInfos.zipWithIndex) {
       if (i < taskSet.tasks.size) {
         runEvent(makeCompletionEvent(taskSet.tasks(i), result._1, result._2))
       }
@@ -405,6 +431,15 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     runEvent(JobCancelled(jobId, None))
   }
 
+  /** Make some tasks in task set success and check results. */
+  private def completeAndCheckAnswer(
+      taskSet: TaskSet,
+      taskEndInfos: Seq[(TaskEndReason, Any)],
+      expected: Map[Int, Any]): Unit = {
+    complete(taskSet, taskEndInfos)
+    assert(this.results === expected)
+  }
+
   test("[SPARK-3353] parent stage should have lower stage id") {
     sc.parallelize(1 to 10).map(x => (x, x)).reduceByKey(_ + _, 4).count()
     val stageByOrderOfExecution = sparkListener.stageByOrderOfExecution
@@ -458,24 +493,19 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     assert(mapStageC.parents === List(mapStageA, mapStageB))
     assert(finalStage.parents === List(mapStageC))
 
-    complete(taskSets(0), Seq((Success, makeMapStatus("hostA", 1))))
-    complete(taskSets(1), Seq((Success, makeMapStatus("hostA", 1))))
-    complete(taskSets(2), Seq((Success, makeMapStatus("hostA", 1))))
-    complete(taskSets(3), Seq((Success, 42)))
-    assert(results === Map(0 -> 42))
+    completeShuffleMapStageSuccessfully(0, 0, 1)
+    completeShuffleMapStageSuccessfully(1, 0, 1)
+    completeShuffleMapStageSuccessfully(2, 0, 1)
+    completeAndCheckAnswer(taskSets(3), Seq((Success, 42)), Map(0 -> 42))
     assertDataStructuresEmpty()
   }
 
-  test("All shuffle files on the slave should be cleaned up when slave lost") {
-    // reset the test context with the right shuffle service config
-    afterEach()
-    val conf = new SparkConf()
+  test("All shuffle files on the storage endpoint should be cleaned up when it is lost") {
     conf.set(config.SHUFFLE_SERVICE_ENABLED.key, "true")
     conf.set("spark.files.fetchFailure.unRegisterOutputOnHost", "true")
-    init(conf)
-    runEvent(ExecutorAdded("exec-hostA1", "hostA"))
-    runEvent(ExecutorAdded("exec-hostA2", "hostA"))
-    runEvent(ExecutorAdded("exec-hostB", "hostB"))
+    runEvent(ExecutorAdded("hostA-exec1", "hostA"))
+    runEvent(ExecutorAdded("hostA-exec2", "hostA"))
+    runEvent(ExecutorAdded("hostB-exec", "hostB"))
     val firstRDD = new MyRDD(sc, 3, Nil)
     val firstShuffleDep = new ShuffleDependency(firstRDD, new HashPartitioner(3))
     val firstShuffleId = firstShuffleDep.shuffleId
@@ -488,20 +518,20 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     complete(taskSets(0), Seq(
       (Success,
         MapStatus(
-          BlockManagerId("exec-hostA1", "hostA", 12345), Array.fill[Long](1)(2), mapTaskId = 5)),
+          BlockManagerId("hostA-exec1", "hostA", 12345), Array.fill[Long](1)(2), mapTaskId = 5)),
       (Success,
         MapStatus(
-          BlockManagerId("exec-hostA2", "hostA", 12345), Array.fill[Long](1)(2), mapTaskId = 6)),
+          BlockManagerId("hostA-exec2", "hostA", 12345), Array.fill[Long](1)(2), mapTaskId = 6)),
       (Success, makeMapStatus("hostB", 1, mapTaskId = 7))
     ))
     // map stage2 completes successfully, with one task on each executor
     complete(taskSets(1), Seq(
       (Success,
         MapStatus(
-          BlockManagerId("exec-hostA1", "hostA", 12345), Array.fill[Long](1)(2), mapTaskId = 8)),
+          BlockManagerId("hostA-exec1", "hostA", 12345), Array.fill[Long](1)(2), mapTaskId = 8)),
       (Success,
         MapStatus(
-          BlockManagerId("exec-hostA2", "hostA", 12345), Array.fill[Long](1)(2), mapTaskId = 9)),
+          BlockManagerId("hostA-exec2", "hostA", 12345), Array.fill[Long](1)(2), mapTaskId = 9)),
       (Success, makeMapStatus("hostB", 1, mapTaskId = 10))
     ))
     // make sure our test setup is correct
@@ -509,19 +539,19 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     //  val initialMapStatus1 = mapOutputTracker.mapStatuses.get(0).get
     assert(initialMapStatus1.count(_ != null) === 3)
     assert(initialMapStatus1.map{_.location.executorId}.toSet ===
-      Set("exec-hostA1", "exec-hostA2", "exec-hostB"))
+      Set("hostA-exec1", "hostA-exec2", "hostB-exec"))
     assert(initialMapStatus1.map{_.mapId}.toSet === Set(5, 6, 7))
 
     val initialMapStatus2 = mapOutputTracker.shuffleStatuses(secondShuffleId).mapStatuses
     //  val initialMapStatus1 = mapOutputTracker.mapStatuses.get(0).get
     assert(initialMapStatus2.count(_ != null) === 3)
     assert(initialMapStatus2.map{_.location.executorId}.toSet ===
-      Set("exec-hostA1", "exec-hostA2", "exec-hostB"))
+      Set("hostA-exec1", "hostA-exec2", "hostB-exec"))
     assert(initialMapStatus2.map{_.mapId}.toSet === Set(8, 9, 10))
 
     // reduce stage fails with a fetch failure from one host
     complete(taskSets(2), Seq(
-      (FetchFailed(BlockManagerId("exec-hostA2", "hostA", 12345),
+      (FetchFailed(BlockManagerId("hostA-exec2", "hostA", 12345),
         firstShuffleId, 0L, 0, 0, "ignored"),
         null)
     ))
@@ -531,15 +561,61 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
 
     val mapStatus1 = mapOutputTracker.shuffleStatuses(firstShuffleId).mapStatuses
     assert(mapStatus1.count(_ != null) === 1)
-    assert(mapStatus1(2).location.executorId === "exec-hostB")
+    assert(mapStatus1(2).location.executorId === "hostB-exec")
     assert(mapStatus1(2).location.host === "hostB")
 
     val mapStatus2 = mapOutputTracker.shuffleStatuses(secondShuffleId).mapStatuses
     assert(mapStatus2.count(_ != null) === 1)
-    assert(mapStatus2(2).location.executorId === "exec-hostB")
+    assert(mapStatus2(2).location.executorId === "hostB-exec")
     assert(mapStatus2(2).location.host === "hostB")
   }
 
+  test("SPARK-32003: All shuffle files for executor should be cleaned up on fetch failure") {
+    conf.set(config.SHUFFLE_SERVICE_ENABLED.key, "true")
+
+    val shuffleMapRdd = new MyRDD(sc, 3, Nil)
+    val shuffleDep = new ShuffleDependency(shuffleMapRdd, new HashPartitioner(3))
+    val shuffleId = shuffleDep.shuffleId
+    val reduceRdd = new MyRDD(sc, 3, List(shuffleDep), tracker = mapOutputTracker)
+
+    submit(reduceRdd, Array(0, 1, 2))
+    // Map stage completes successfully,
+    // two tasks are run on an executor on hostA and one on an executor on hostB
+    completeShuffleMapStageSuccessfully(0, 0, 3, Seq("hostA", "hostA", "hostB"))
+    // Now the executor on hostA is lost
+    runEvent(ExecutorLost("hostA-exec", ExecutorExited(-100, false, "Container marked as failed")))
+    // Executor is removed but shuffle files are not unregistered
+    verify(blockManagerMaster, times(1)).removeExecutor("hostA-exec")
+    verify(mapOutputTracker, times(0)).removeOutputsOnExecutor("hostA-exec")
+
+    // The MapOutputTracker has all the shuffle files
+    val mapStatuses = mapOutputTracker.shuffleStatuses(shuffleId).mapStatuses
+    assert(mapStatuses.count(_ != null) === 3)
+    assert(mapStatuses.count(s => s != null && s.location.executorId == "hostA-exec") === 2)
+    assert(mapStatuses.count(s => s != null && s.location.executorId == "hostB-exec") === 1)
+
+    // Now a fetch failure from the lost executor occurs
+    complete(taskSets(1), Seq(
+      (FetchFailed(makeBlockManagerId("hostA"), shuffleId, 0L, 0, 0, "ignored"), null)
+    ))
+    // blockManagerMaster.removeExecutor is not called again
+    // but shuffle files are unregistered
+    verify(blockManagerMaster, times(1)).removeExecutor("hostA-exec")
+    verify(mapOutputTracker, times(1)).removeOutputsOnExecutor("hostA-exec")
+
+    // Shuffle files for hostA-exec should be lost
+    assert(mapStatuses.count(_ != null) === 1)
+    assert(mapStatuses.count(s => s != null && s.location.executorId == "hostA-exec") === 0)
+    assert(mapStatuses.count(s => s != null && s.location.executorId == "hostB-exec") === 1)
+
+    // Additional fetch failure from the executor does not result in further call to
+    // mapOutputTracker.removeOutputsOnExecutor
+    complete(taskSets(1), Seq(
+      (FetchFailed(makeBlockManagerId("hostA"), shuffleId, 0L, 1, 0, "ignored"), null)
+    ))
+    verify(mapOutputTracker, times(1)).removeOutputsOnExecutor("hostA-exec")
+  }
+
   test("zero split job") {
     var numResults = 0
     var failureReason: Option[Exception] = None
@@ -558,8 +634,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
 
   test("run trivial job") {
     submit(new MyRDD(sc, 1, Nil), Array(0))
-    complete(taskSets(0), List((Success, 42)))
-    assert(results === Map(0 -> 42))
+    completeAndCheckAnswer(taskSets(0), Seq((Success, 42)), Map(0 -> 42))
     assertDataStructuresEmpty()
   }
 
@@ -567,8 +642,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     val baseRdd = new MyRDD(sc, 1, Nil)
     val finalRdd = new MyRDD(sc, 1, List(new OneToOneDependency(baseRdd)))
     submit(finalRdd, Array(0))
-    complete(taskSets(0), Seq((Success, 42)))
-    assert(results === Map(0 -> 42))
+    completeAndCheckAnswer(taskSets(0), Seq((Success, 42)), Map(0 -> 42))
     assertDataStructuresEmpty()
   }
 
@@ -592,8 +666,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     submit(finalRdd, Array(0))
     val taskSet = taskSets(0)
     assertLocations(taskSet, Seq(Seq("hostA", "hostB")))
-    complete(taskSet, Seq((Success, 42)))
-    assert(results === Map(0 -> 42))
+    completeAndCheckAnswer(taskSet, Seq((Success, 42)), Map(0 -> 42))
     assertDataStructuresEmpty()
   }
 
@@ -710,10 +783,14 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
           accumUpdates: Array[(Long, Seq[AccumulatorV2[_, _]])],
           blockManagerId: BlockManagerId,
           executorUpdates: Map[(Int, Int), ExecutorMetrics]): Boolean = true
-      override def executorDecommission(executorId: String): Unit = {}
       override def executorLost(executorId: String, reason: ExecutorLossReason): Unit = {}
       override def workerRemoved(workerId: String, host: String, message: String): Unit = {}
       override def applicationAttemptId(): Option[String] = None
+      override def executorDecommission(
+        executorId: String,
+        decommissionInfo: ExecutorDecommissionInfo): Unit = {}
+      override def getExecutorDecommissionState(
+        executorId: String): Option[ExecutorDecommissionState] = None
     }
     val noKillScheduler = new DAGScheduler(
       sc,
@@ -729,8 +806,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     assert(failure === null)
 
     // When the task set completes normally, state should be correctly updated.
-    complete(taskSets(0), Seq((Success, 42)))
-    assert(results === Map(0 -> 42))
+    completeAndCheckAnswer(taskSets(0), Seq((Success, 42)), Map(0 -> 42))
     assertDataStructuresEmpty()
 
     assert(sparkListener.failedStages.isEmpty)
@@ -743,13 +819,10 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     val shuffleId = shuffleDep.shuffleId
     val reduceRdd = new MyRDD(sc, 1, List(shuffleDep), tracker = mapOutputTracker)
     submit(reduceRdd, Array(0))
-    complete(taskSets(0), Seq(
-      (Success, makeMapStatus("hostA", 1)),
-      (Success, makeMapStatus("hostB", 1))))
+    completeShuffleMapStageSuccessfully(0, 0, 1)
     assert(mapOutputTracker.getMapSizesByExecutorId(shuffleId, 0).map(_._1).toSet ===
       HashSet(makeBlockManagerId("hostA"), makeBlockManagerId("hostB")))
-    complete(taskSets(1), Seq((Success, 42)))
-    assert(results === Map(0 -> 42))
+    completeAndCheckAnswer(taskSets(1), Seq((Success, 42)), Map(0 -> 42))
     assertDataStructuresEmpty()
   }
 
@@ -759,15 +832,12 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     val shuffleId = shuffleDep.shuffleId
     val reduceRdd = new MyRDD(sc, 2, List(shuffleDep), tracker = mapOutputTracker)
     submit(reduceRdd, Array(0, 1))
-    complete(taskSets(0), Seq(
-      (Success, makeMapStatus("hostA", reduceRdd.partitions.length)),
-      (Success, makeMapStatus("hostB", reduceRdd.partitions.length))))
+    completeShuffleMapStageSuccessfully(0, 0, reduceRdd.partitions.length)
     // the 2nd ResultTask failed
     complete(taskSets(1), Seq(
       (Success, 42),
       (FetchFailed(makeBlockManagerId("hostA"), shuffleId, 0L, 0, 0, "ignored"), null)))
-    // this will get called
-    // blockManagerMaster.removeExecutor("exec-hostA")
+    verify(blockManagerMaster, times(1)).removeExecutor("hostA-exec")
     // ask the scheduler to try it again
     scheduler.resubmitFailedStages()
     // have the 2nd attempt pass
@@ -775,15 +845,14 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     // we can see both result blocks now
     assert(mapOutputTracker.getMapSizesByExecutorId(shuffleId, 0).map(_._1.host).toSet ===
       HashSet("hostA", "hostB"))
-    complete(taskSets(3), Seq((Success, 43)))
-    assert(results === Map(0 -> 42, 1 -> 43))
+    completeAndCheckAnswer(taskSets(3), Seq((Success, 43)), Map(0 -> 42, 1 -> 43))
     assertDataStructuresEmpty()
   }
 
   private val shuffleFileLossTests = Seq(
-    ("slave lost with shuffle service", SlaveLost("", false), true, false),
-    ("worker lost with shuffle service", SlaveLost("", true), true, true),
-    ("worker lost without shuffle service", SlaveLost("", true), false, true),
+    ("executor process lost with shuffle service", ExecutorProcessLost("", None), true, false),
+    ("worker lost with shuffle service", ExecutorProcessLost("", Some("hostA")), true, true),
+    ("worker lost without shuffle service", ExecutorProcessLost("", Some("hostA")), false, true),
     ("executor failure with shuffle service", ExecutorKilled, true, false),
     ("executor failure without shuffle service", ExecutorKilled, false, true))
 
@@ -794,11 +863,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
       "not lost"
     }
     test(s"shuffle files $maybeLost when $eventDescription") {
-      // reset the test context with the right shuffle service config
-      afterEach()
-      val conf = new SparkConf()
       conf.set(config.SHUFFLE_SERVICE_ENABLED.key, shuffleServiceOn.toString)
-      init(conf)
       assert(sc.env.blockManager.externalShuffleServiceEnabled == shuffleServiceOn)
 
       val shuffleMapRdd = new MyRDD(sc, 2, Nil)
@@ -806,15 +871,24 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
       val shuffleId = shuffleDep.shuffleId
       val reduceRdd = new MyRDD(sc, 1, List(shuffleDep), tracker = mapOutputTracker)
       submit(reduceRdd, Array(0))
-      complete(taskSets(0), Seq(
-        (Success, makeMapStatus("hostA", 1)),
-        (Success, makeMapStatus("hostB", 1))))
-      runEvent(ExecutorLost("exec-hostA", event))
+      completeShuffleMapStageSuccessfully(0, 0, 1)
+      val expectHostFileLoss = event match {
+        case ExecutorProcessLost(_, workerHost, _) => workerHost.isDefined
+        case _ => false
+      }
+      runEvent(ExecutorLost("hostA-exec", event))
+      verify(blockManagerMaster, times(1)).removeExecutor("hostA-exec")
       if (expectFileLoss) {
+        if (expectHostFileLoss) {
+          verify(mapOutputTracker, times(1)).removeOutputsOnHost("hostA")
+        } else {
+          verify(mapOutputTracker, times(1)).removeOutputsOnExecutor("hostA-exec")
+        }
         intercept[MetadataFetchFailedException] {
           mapOutputTracker.getMapSizesByExecutorId(shuffleId, 0)
         }
       } else {
+        verify(mapOutputTracker, times(0)).removeOutputsOnExecutor("hostA-exec")
         assert(mapOutputTracker.getMapSizesByExecutorId(shuffleId, 0).map(_._1).toSet ===
           HashSet(makeBlockManagerId("hostA"), makeBlockManagerId("hostB")))
       }
@@ -889,16 +963,28 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
    * @param stageId - The current stageId
    * @param attemptIdx - The current attempt count
    * @param numShufflePartitions - The number of partitions in the next stage
+   * @param hostNames - Host on which each task in the task set is executed
    */
   private def completeShuffleMapStageSuccessfully(
       stageId: Int,
       attemptIdx: Int,
-      numShufflePartitions: Int): Unit = {
-    val stageAttempt = taskSets.last
-    checkStageId(stageId, attemptIdx, stageAttempt)
+      numShufflePartitions: Int,
+      hostNames: Seq[String] = Seq.empty[String]): Unit = {
+    def compareStageAttempt(taskSet: TaskSet): Boolean = {
+      taskSet.stageId == stageId && taskSet.stageAttemptId == attemptIdx
+    }
+
+    val stageAttemptOpt = taskSets.find(compareStageAttempt(_))
+    assert(stageAttemptOpt.isDefined)
+    val stageAttempt = stageAttemptOpt.get
     complete(stageAttempt, stageAttempt.tasks.zipWithIndex.map {
       case (task, idx) =>
-        (Success, makeMapStatus("host" + ('A' + idx).toChar, numShufflePartitions))
+        val hostName = if (idx < hostNames.size) {
+          hostNames(idx)
+        } else {
+          s"host${('A' + idx).toChar}"
+        }
+        (Success, makeMapStatus(hostName, numShufflePartitions))
     }.toSeq)
   }
 
@@ -1140,9 +1226,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     val shuffleId = shuffleDep.shuffleId
     val reduceRdd = new MyRDD(sc, 2, List(shuffleDep), tracker = mapOutputTracker)
     submit(reduceRdd, Array(0, 1))
-    complete(taskSets(0), Seq(
-      (Success, makeMapStatus("hostA", reduceRdd.partitions.length)),
-      (Success, makeMapStatus("hostB", reduceRdd.partitions.length))))
+    completeShuffleMapStageSuccessfully(0, 0, reduceRdd.partitions.length)
     // The MapOutputTracker should know about both map output locations.
     assert(mapOutputTracker.getMapSizesByExecutorId(shuffleId, 0).map(_._1.host).toSet ===
       HashSet("hostA", "hostB"))
@@ -1169,9 +1253,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     val shuffleId = shuffleDep.shuffleId
     val reduceRdd = new MyRDD(sc, 2, List(shuffleDep), tracker = mapOutputTracker)
     submit(reduceRdd, Array(0, 1))
-    complete(taskSets(0), Seq(
-      (Success, makeMapStatus("hostA", reduceRdd.partitions.length)),
-      (Success, makeMapStatus("hostB", reduceRdd.partitions.length))))
+    completeShuffleMapStageSuccessfully(0, 0, reduceRdd.partitions.length)
     assert(mapOutputTracker.findMissingPartitions(shuffleId) === Some(Seq.empty))
 
     // The first result task fails, with a fetch failure for the output from the first mapper.
@@ -1231,9 +1313,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     submit(reduceRdd, Array(0, 1))
 
     // Complete the map stage.
-    complete(taskSets(0), Seq(
-      (Success, makeMapStatus("hostA", 2)),
-      (Success, makeMapStatus("hostA", 2))))
+    completeShuffleMapStageSuccessfully(0, 0, 2, hostNames = Seq("hostA", "hostA"))
     assert(mapOutputTracker.findMissingPartitions(shuffleId) === Some(Seq.empty))
 
     // The first ResultTask fails
@@ -1267,9 +1347,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     // The map stage should have been submitted.
     assert(countSubmittedMapStageAttempts() === 1)
 
-    complete(taskSets(0), Seq(
-      (Success, makeMapStatus("hostA", 2)),
-      (Success, makeMapStatus("hostB", 2))))
+    completeShuffleMapStageSuccessfully(0, 0, 2)
     // The MapOutputTracker should know about both map output locations.
     assert(mapOutputTracker.getMapSizesByExecutorId(shuffleId, 0).map(_._1.host).toSet ===
       HashSet("hostA", "hostB"))
@@ -1328,9 +1406,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     assert(countSubmittedMapStageAttempts() === 1)
 
     // Complete the map stage.
-    complete(taskSets(0), Seq(
-      (Success, makeMapStatus("hostA", 2)),
-      (Success, makeMapStatus("hostB", 2))))
+    completeShuffleMapStageSuccessfully(0, 0, 2)
 
     // The reduce stage should have been submitted.
     assert(countSubmittedReduceStageAttempts() === 1)
@@ -1415,7 +1491,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
 
     // pretend we were told hostA went away
     val oldEpoch = mapOutputTracker.getEpoch
-    runEvent(ExecutorLost("exec-hostA", ExecutorKilled))
+    runEvent(ExecutorLost("hostA-exec", ExecutorKilled))
     val newEpoch = mapOutputTracker.getEpoch
     assert(newEpoch > oldEpoch)
 
@@ -1458,8 +1534,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
       HashSet(makeBlockManagerId("hostB"), makeBlockManagerId("hostA")))
 
     // finish the next stage normally, which completes the job
-    complete(taskSets(1), Seq((Success, 42), (Success, 43)))
-    assert(results === Map(0 -> 42, 1 -> 43))
+    completeAndCheckAnswer(taskSets(1), Seq((Success, 42), (Success, 43)), Map(0 -> 42, 1 -> 43))
     assertDataStructuresEmpty()
   }
 
@@ -1538,14 +1613,11 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     submit(reduceRdd, Array(0))
 
     // things start out smoothly, stage 0 completes with no issues
-    complete(taskSets(0), Seq(
-      (Success, makeMapStatus("hostB", shuffleMapRdd.partitions.length)),
-      (Success, makeMapStatus("hostB", shuffleMapRdd.partitions.length)),
-      (Success, makeMapStatus("hostA", shuffleMapRdd.partitions.length))
-    ))
+    completeShuffleMapStageSuccessfully(
+      0, 0, shuffleMapRdd.partitions.length, Seq("hostB", "hostB", "hostA"))
 
     // then one executor dies, and a task fails in stage 1
-    runEvent(ExecutorLost("exec-hostA", ExecutorKilled))
+    runEvent(ExecutorLost("hostA-exec", ExecutorKilled))
     runEvent(makeCompletionEvent(
       taskSets(1).tasks(0),
       FetchFailed(null, firstShuffleId, 2L, 2, 0, "Fetch failed"),
@@ -1643,18 +1715,15 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
       makeMapStatus("hostA", reduceRdd.partitions.length)))
 
     // now that host goes down
-    runEvent(ExecutorLost("exec-hostA", ExecutorKilled))
+    runEvent(ExecutorLost("hostA-exec", ExecutorKilled))
 
     // so we resubmit those tasks
     runEvent(makeCompletionEvent(taskSets(0).tasks(0), Resubmitted, null))
     runEvent(makeCompletionEvent(taskSets(0).tasks(1), Resubmitted, null))
 
     // now complete everything on a different host
-    complete(taskSets(0), Seq(
-      (Success, makeMapStatus("hostB", reduceRdd.partitions.length)),
-      (Success, makeMapStatus("hostB", reduceRdd.partitions.length)),
-      (Success, makeMapStatus("hostB", reduceRdd.partitions.length))
-    ))
+    completeShuffleMapStageSuccessfully(
+      0, 0, reduceRdd.partitions.length, Seq("hostB", "hostB", "hostB"))
 
     // now we should submit stage 1, and the map output from stage 0 should be registered
 
@@ -1768,7 +1837,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     // correct behavior.
     val job1Id = 0  // TaskSet priority for Stages run with "job1" as the ActiveJob
     checkJobPropertiesAndPriority(taskSets(0), "job1", job1Id)
-    complete(taskSets(0), Seq((Success, makeMapStatus("hostA", 1))))
+    completeShuffleMapStageSuccessfully(0, 0, 1)
 
     shuffleDep1
   }
@@ -1785,7 +1854,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     // the stage.
     checkJobPropertiesAndPriority(taskSets(1), "job2", 1)
 
-    complete(taskSets(1), Seq((Success, makeMapStatus("hostA", 1))))
+    completeShuffleMapStageSuccessfully(1, 0, 1)
     assert(taskSets(2).properties != null)
     complete(taskSets(2), Seq((Success, 42)))
     assert(results === Map(0 -> 42))
@@ -1806,9 +1875,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
 
     // lets say there is a fetch failure in this task set, which makes us go back and
     // run stage 0, attempt 1
-    complete(taskSets(1), Seq(
-      (FetchFailed(makeBlockManagerId("hostA"),
-        shuffleDep1.shuffleId, 0L, 0, 0, "ignored"), null)))
+    completeNextStageWithFetchFailure(1, 0, shuffleDep1)
     scheduler.resubmitFailedStages()
 
     // stage 0, attempt 1 should have the properties of job2
@@ -1817,9 +1884,9 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     checkJobPropertiesAndPriority(taskSets(2), "job2", job2Id)
 
     // run the rest of the stages normally, checking that they have the correct properties
-    complete(taskSets(2), Seq((Success, makeMapStatus("hostA", 1))))
+    completeShuffleMapStageSuccessfully(0, 1, 1)
     checkJobPropertiesAndPriority(taskSets(3), "job2", job2Id)
-    complete(taskSets(3), Seq((Success, makeMapStatus("hostA", 1))))
+    completeShuffleMapStageSuccessfully(1, 1, 1)
     checkJobPropertiesAndPriority(taskSets(4), "job2", job2Id)
     complete(taskSets(4), Seq((Success, 42)))
     assert(results === Map(0 -> 42))
@@ -1841,10 +1908,8 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     val reduceRdd = new MyRDD(sc, 1, List(shuffleDep), tracker = mapOutputTracker)
     submit(reduceRdd, Array(0))
     // Tell the DAGScheduler that hostA was lost.
-    runEvent(ExecutorLost("exec-hostA", ExecutorKilled))
-    complete(taskSets(0), Seq(
-      (Success, makeMapStatus("hostA", 1)),
-      (Success, makeMapStatus("hostB", 1))))
+    runEvent(ExecutorLost("hostA-exec", ExecutorKilled))
+    completeShuffleMapStageSuccessfully(0, 0, 1)
 
     // At this point, no more tasks are running for the stage (and the TaskSetManager considers the
     // stage complete), but the tasks that ran on HostA need to be re-run, so the DAGScheduler
@@ -1880,19 +1945,13 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     val finalRdd = new MyRDD(sc, 1, List(shuffleDepTwo), tracker = mapOutputTracker)
     submit(finalRdd, Array(0))
     // have the first stage complete normally
-    complete(taskSets(0), Seq(
-      (Success, makeMapStatus("hostA", 2)),
-      (Success, makeMapStatus("hostB", 2))))
+    completeShuffleMapStageSuccessfully(0, 0, 2)
     // have the second stage complete normally
-    complete(taskSets(1), Seq(
-      (Success, makeMapStatus("hostA", 1)),
-      (Success, makeMapStatus("hostC", 1))))
+    completeShuffleMapStageSuccessfully(1, 0, 1, Seq("hostA", "hostC"))
     // fail the third stage because hostA went down
-    complete(taskSets(2), Seq(
-      (FetchFailed(makeBlockManagerId("hostA"),
-        shuffleDepTwo.shuffleId, 0L, 0, 0, "ignored"), null)))
+    completeNextStageWithFetchFailure(2, 0, shuffleDepTwo)
     // TODO assert this:
-    // blockManagerMaster.removeExecutor("exec-hostA")
+    // blockManagerMaster.removeExecutor("hostA-exec")
     // have DAGScheduler try again
     scheduler.resubmitFailedStages()
     complete(taskSets(3), Seq((Success, makeMapStatus("hostA", 2))))
@@ -1912,19 +1971,13 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     cacheLocations(shuffleTwoRdd.id -> 0) = Seq(makeBlockManagerId("hostD"))
     cacheLocations(shuffleTwoRdd.id -> 1) = Seq(makeBlockManagerId("hostC"))
     // complete stage 0
-    complete(taskSets(0), Seq(
-      (Success, makeMapStatus("hostA", 2)),
-      (Success, makeMapStatus("hostB", 2))))
+    completeShuffleMapStageSuccessfully(0, 0, 2)
     // complete stage 1
-    complete(taskSets(1), Seq(
-      (Success, makeMapStatus("hostA", 1)),
-      (Success, makeMapStatus("hostB", 1))))
+    completeShuffleMapStageSuccessfully(1, 0, 1)
     // pretend stage 2 failed because hostA went down
-    complete(taskSets(2), Seq(
-      (FetchFailed(makeBlockManagerId("hostA"),
-        shuffleDepTwo.shuffleId, 0L, 0, 0, "ignored"), null)))
+    completeNextStageWithFetchFailure(2, 0, shuffleDepTwo)
     // TODO assert this:
-    // blockManagerMaster.removeExecutor("exec-hostA")
+    // blockManagerMaster.removeExecutor("hostA-exec")
     // DAGScheduler should notice the cached copy of the second shuffle and try to get it rerun.
     scheduler.resubmitFailedStages()
     assertLocations(taskSets(3), Seq(Seq("hostD")))
@@ -2169,8 +2222,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     val shuffleId = shuffleDep.shuffleId
     val reduceRdd = new MyRDD(sc, 1, List(shuffleDep), tracker = mapOutputTracker)
     submit(reduceRdd, Array(0))
-    complete(taskSets(0), Seq(
-      (Success, makeMapStatus("hostA", 1))))
+    completeShuffleMapStageSuccessfully(0, 0, 1)
     assert(mapOutputTracker.getMapSizesByExecutorId(shuffleId, 0).map(_._1).toSet ===
       HashSet(makeBlockManagerId("hostA")))
 
@@ -2215,8 +2267,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     val shuffleId = shuffleDep.shuffleId
     val reduceRdd = new MyRDD(sc, 1, List(shuffleDep, narrowDep), tracker = mapOutputTracker)
     submit(reduceRdd, Array(0))
-    complete(taskSets(0), Seq(
-      (Success, makeMapStatus("hostA", 1))))
+    completeShuffleMapStageSuccessfully(0, 0, 1)
     assert(mapOutputTracker.getMapSizesByExecutorId(shuffleId, 0).map(_._1).toSet ===
       HashSet(makeBlockManagerId("hostA")))
 
@@ -2240,7 +2291,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     assert(stackTraceString.contains("org.apache.spark.rdd.RDD.count"))
 
     // should include the FunSuite setup:
-    assert(stackTraceString.contains("org.scalatest.FunSuite"))
+    assert(stackTraceString.contains("org.scalatest.funsuite.AnyFunSuite"))
   }
 
   test("catch errors in event loop") {
@@ -2319,9 +2370,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
 
     // Submit a map stage by itself
     submitMapStage(shuffleDep)
-    complete(taskSets(0), Seq(
-      (Success, makeMapStatus("hostA", reduceRdd.partitions.length)),
-      (Success, makeMapStatus("hostB", reduceRdd.partitions.length))))
+    completeShuffleMapStageSuccessfully(0, 0, reduceRdd.partitions.length)
     assert(results.size === 1)
     results.clear()
     assertDataStructuresEmpty()
@@ -2378,9 +2427,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
 
     // Complete the first stage
     assert(taskSets(0).stageId === 0)
-    complete(taskSets(0), Seq(
-      (Success, makeMapStatus("hostA", rdd1.partitions.length)),
-      (Success, makeMapStatus("hostB", rdd1.partitions.length))))
+    completeShuffleMapStageSuccessfully(0, 0, rdd1.partitions.length)
     assert(mapOutputTracker.getMapSizesByExecutorId(dep1.shuffleId, 0).map(_._1).toSet ===
       HashSet(makeBlockManagerId("hostA"), makeBlockManagerId("hostB")))
     assert(listener1.results.size === 1)
@@ -2442,9 +2489,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
 
     // Complete the stage0.
     assert(taskSets(0).stageId === 0)
-    complete(taskSets(0), Seq(
-      (Success, makeMapStatus("hostA", rdd1.partitions.length)),
-      (Success, makeMapStatus("hostB", rdd1.partitions.length))))
+    completeShuffleMapStageSuccessfully(0, 0, rdd1.partitions.length)
     assert(mapOutputTracker.getMapSizesByExecutorId(dep1.shuffleId, 0).map(_._1).toSet ===
         HashSet(makeBlockManagerId("hostA"), makeBlockManagerId("hostB")))
     assert(listener1.results.size === 1)
@@ -2503,7 +2548,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     // Pretend host A was lost. This will cause the TaskSetManager to resubmit task 0, because it
     // completed on hostA.
     val oldEpoch = mapOutputTracker.getEpoch
-    runEvent(ExecutorLost("exec-hostA", ExecutorKilled))
+    runEvent(ExecutorLost("hostA-exec", ExecutorKilled))
     val newEpoch = mapOutputTracker.getEpoch
     assert(newEpoch > oldEpoch)
 
@@ -2524,7 +2569,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     val newTaskSet = taskSets(1)
     // 2 tasks should have been re-submitted, for tasks 0 and 1 (which ran on hostA).
     assert(newTaskSet.tasks.size === 2)
-    // Complete task 0 from the original task set (i.e., not hte one that's currently active).
+    // Complete task 0 from the original task set (i.e., not the one that's currently active).
     // This should still be counted towards the job being complete (but there's still one
     // outstanding task).
     runEvent(makeCompletionEvent(newTaskSet.tasks(0), Success, makeMapStatus("hostB", 2)))
@@ -2651,9 +2696,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
 
     // Complete both tasks in rddA.
     assert(taskSets(0).stageId === 0 && taskSets(0).stageAttemptId === 0)
-    complete(taskSets(0), Seq(
-      (Success, makeMapStatus("hostA", 2)),
-      (Success, makeMapStatus("hostA", 2))))
+    completeShuffleMapStageSuccessfully(0, 0, 2, Seq("hostA", "hostA"))
 
     // Fetch failed for task(stageId=1, stageAttemptId=0, partitionId=0) running on hostA
     // and task(stageId=1, stageAttemptId=0, partitionId=1) is still running.
@@ -2827,15 +2870,11 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     submit(finalRdd, Array(0, 1))
 
     // Finish the first shuffle map stage.
-    complete(taskSets(0), Seq(
-      (Success, makeMapStatus("hostA", 2)),
-      (Success, makeMapStatus("hostB", 2))))
+    completeShuffleMapStageSuccessfully(0, 0, 2)
     assert(mapOutputTracker.findMissingPartitions(shuffleId1) === Some(Seq.empty))
 
     // Finish the second shuffle map stage.
-    complete(taskSets(1), Seq(
-      (Success, makeMapStatus("hostC", 2)),
-      (Success, makeMapStatus("hostD", 2))))
+    completeShuffleMapStageSuccessfully(1, 0, 2, Seq("hostC", "hostD"))
     assert(mapOutputTracker.findMissingPartitions(shuffleId2) === Some(Seq.empty))
 
     // The first task of the final stage failed with fetch failure
@@ -2847,11 +2886,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
   }
 
   test("SPARK-25341: abort stage while using old fetch protocol") {
-    // reset the test context with using old fetch protocol
-    afterEach()
-    val conf = new SparkConf()
     conf.set(config.SHUFFLE_USE_OLD_FETCH_PROTOCOL.key, "true")
-    init(conf)
     // Construct the scenario of indeterminate stage fetch failed.
     constructIndeterminateStageFetchFailed()
     // The job should fail because Spark can't rollback the shuffle map stage while
@@ -2894,14 +2929,10 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     assert(taskSets(4).tasks.length == 2)
 
     // Finish all stage.
-    complete(taskSets(4), Seq(
-      (Success, makeMapStatus("hostA", 2)),
-      (Success, makeMapStatus("hostB", 2))))
+    completeShuffleMapStageSuccessfully(0, 1, 2)
     assert(mapOutputTracker.findMissingPartitions(shuffleId1) === Some(Seq.empty))
 
-    complete(taskSets(5), Seq(
-      (Success, makeMapStatus("hostC", 2)),
-      (Success, makeMapStatus("hostD", 2))))
+    completeShuffleMapStageSuccessfully(1, 2, 2, Seq("hostC", "hostD"))
     assert(mapOutputTracker.findMissingPartitions(shuffleId2) === Some(Seq.empty))
 
     complete(taskSets(6), Seq((Success, 11), (Success, 12)))
@@ -2932,14 +2963,10 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     submit(finalRdd, Array(0, 1), properties = new Properties())
 
     // Finish the first 2 shuffle map stages.
-    complete(taskSets(0), Seq(
-      (Success, makeMapStatus("hostA", 2)),
-      (Success, makeMapStatus("hostB", 2))))
+    completeShuffleMapStageSuccessfully(0, 0, 2)
     assert(mapOutputTracker.findMissingPartitions(shuffleId1) === Some(Seq.empty))
 
-    complete(taskSets(1), Seq(
-      (Success, makeMapStatus("hostB", 2)),
-      (Success, makeMapStatus("hostD", 2))))
+    completeShuffleMapStageSuccessfully(1, 0, 2, Seq("hostB", "hostD"))
     assert(mapOutputTracker.findMissingPartitions(shuffleId2) === Some(Seq.empty))
 
     // Executor lost on hostB, both of stage 0 and 1 should be reran.
@@ -2959,9 +2986,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
       assert(taskSets(taskSetIndex).stageId == stageId)
       assert(taskSets(taskSetIndex).stageAttemptId == 1)
       assert(taskSets(taskSetIndex).tasks.length == 2)
-      complete(taskSets(taskSetIndex), Seq(
-        (Success, makeMapStatus("hostA", 2)),
-        (Success, makeMapStatus("hostB", 2))))
+      completeShuffleMapStageSuccessfully(stageId, 1, 2)
       assert(mapOutputTracker.findMissingPartitions(shuffleId) === Some(Seq.empty))
     }
 
@@ -3032,7 +3057,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     assertResultStageFailToRollback(shuffleMapRdd)
   }
 
-  private def assertResultStageNotRollbacked(mapRdd: MyRDD): Unit = {
+  private def assertResultStageNotRolledBack(mapRdd: MyRDD): Unit = {
     val shuffleDep = new ShuffleDependency(mapRdd, new HashPartitioner(2))
     val shuffleId = shuffleDep.shuffleId
     val finalRdd = new MyRDD(sc, 2, List(shuffleDep), tracker = mapOutputTracker)
@@ -3072,7 +3097,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
       val shuffleMapRdd = new MyCheckpointRDD(sc, 2, Nil, indeterminate = true)
       shuffleMapRdd.checkpoint()
       shuffleMapRdd.doCheckpoint()
-      assertResultStageNotRollbacked(shuffleMapRdd)
+      assertResultStageNotRolledBack(shuffleMapRdd)
     }
   }
 
@@ -3189,10 +3214,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
   }
 
   test("test 2 resource profile with merge conflict config true") {
-    afterEach()
-    val conf = new SparkConf()
     conf.set(config.RESOURCE_PROFILE_MERGE_CONFLICTS.key, "true")
-    init(conf)
 
     val ereqs = new ExecutorResourceRequests().cores(4)
     val treqs = new TaskResourceRequests().cpus(1)
@@ -3210,10 +3232,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
   }
 
   test("test multiple resource profiles created from merging use same rp") {
-    afterEach()
-    val conf = new SparkConf()
     conf.set(config.RESOURCE_PROFILE_MERGE_CONFLICTS.key, "true")
-    init(conf)
 
     val ereqs = new ExecutorResourceRequests().cores(4)
     val treqs = new TaskResourceRequests().cpus(1)
@@ -3267,7 +3286,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     assert(mergedRp.taskResources.get(GPU).get.amount == 1)
 
     val ereqs5 = new ExecutorResourceRequests().cores(1).memory("3g")
-      .memoryOverhead("1g").pysparkMemory("2g").resource(GPU, 1, "disc")
+      .memoryOverhead("1g").pysparkMemory("2g").offHeapMemory("4g").resource(GPU, 1, "disc")
     val treqs5 = new TaskResourceRequests().cpus(1).resource(GPU, 1)
     val rp5 = new ResourceProfile(ereqs5.requests, treqs5.requests)
     val ereqs6 = new ExecutorResourceRequests().cores(8).resource(FPGA, 2, "fdisc")
@@ -3277,7 +3296,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
 
     assert(mergedRp.getTaskCpus.get == 2)
     assert(mergedRp.getExecutorCores.get == 8)
-    assert(mergedRp.executorResources.size == 6)
+    assert(mergedRp.executorResources.size == 7)
     assert(mergedRp.taskResources.size == 3)
     assert(mergedRp.executorResources.get(GPU).get.amount == 1)
     assert(mergedRp.executorResources.get(GPU).get.discoveryScript == "disc")
@@ -3288,6 +3307,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     assert(mergedRp.executorResources.get(ResourceProfile.MEMORY).get.amount == 3072)
     assert(mergedRp.executorResources.get(ResourceProfile.PYSPARK_MEM).get.amount == 2048)
     assert(mergedRp.executorResources.get(ResourceProfile.OVERHEAD_MEM).get.amount == 1024)
+    assert(mergedRp.executorResources.get(ResourceProfile.OFFHEAP_MEM).get.amount == 4096)
 
     val ereqs7 = new ExecutorResourceRequests().cores(1).memory("3g")
       .resource(GPU, 4, "disc")
@@ -3306,10 +3326,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
   }
 
   test("test merge 3 resource profiles") {
-    afterEach()
-    val conf = new SparkConf()
     conf.set(config.RESOURCE_PROFILE_MERGE_CONFLICTS.key, "true")
-    init(conf)
     val ereqs = new ExecutorResourceRequests().cores(4)
     val treqs = new TaskResourceRequests().cpus(1)
     val rp1 = new ResourceProfile(ereqs.requests, treqs.requests)
@@ -3319,7 +3336,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     val ereqs3 = new ExecutorResourceRequests().cores(3)
     val treqs3 = new TaskResourceRequests().cpus(2)
     val rp3 = new ResourceProfile(ereqs3.requests, treqs3.requests)
-    var mergedRp = scheduler.mergeResourceProfilesForStage(HashSet(rp1, rp2, rp3))
+    val mergedRp = scheduler.mergeResourceProfilesForStage(HashSet(rp1, rp2, rp3))
 
     assert(mergedRp.getTaskCpus.get == 2)
     assert(mergedRp.getExecutorCores.get == 4)
@@ -3434,8 +3451,9 @@ object DAGSchedulerSuite {
   def makeMapStatus(host: String, reduces: Int, sizes: Byte = 2, mapTaskId: Long = -1): MapStatus =
     MapStatus(makeBlockManagerId(host), Array.fill[Long](reduces)(sizes), mapTaskId)
 
-  def makeBlockManagerId(host: String): BlockManagerId =
-    BlockManagerId("exec-" + host, host, 12345)
+  def makeBlockManagerId(host: String): BlockManagerId = {
+    BlockManagerId(host + "-exec", host, 12345)
+  }
 }
 
 object FailThisAttempt {
diff --git a/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
index 61ea21fa86c5a..240774d854c92 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.scheduler
 
 import java.io.{File, InputStream}
-import java.util.Arrays
+import java.util.{Arrays, Properties}
 
 import scala.collection.immutable.Map
 import scala.collection.mutable
@@ -36,6 +36,7 @@ import org.apache.spark.deploy.history.{EventLogFileReader, SingleEventLogFileWr
 import org.apache.spark.deploy.history.EventLogTestHelper._
 import org.apache.spark.executor.{ExecutorMetrics, TaskMetrics}
 import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config.{EVENT_LOG_DIR, EVENT_LOG_ENABLED}
 import org.apache.spark.io._
 import org.apache.spark.metrics.{ExecutorMetricType, MetricsSystem}
 import org.apache.spark.resource.ResourceProfile
@@ -89,17 +90,123 @@ class EventLoggingListenerSuite extends SparkFunSuite with LocalSparkContext wit
     val conf = getLoggingConf(testDirPath, None)
       .set(key, secretPassword)
     val hadoopconf = SparkHadoopUtil.get.newConfiguration(new SparkConf())
-    val eventLogger = new EventLoggingListener("test", None, testDirPath.toUri(), conf)
-    val envDetails = SparkEnv.environmentDetails(conf, hadoopconf, "FIFO", Seq.empty, Seq.empty)
+    val envDetails = SparkEnv.environmentDetails(
+      conf, hadoopconf, "FIFO", Seq.empty, Seq.empty, Seq.empty)
     val event = SparkListenerEnvironmentUpdate(envDetails)
-    val redactedProps = eventLogger.redactEvent(event).environmentDetails("Spark Properties").toMap
+    val redactedProps = EventLoggingListener
+      .redactEvent(conf, event).environmentDetails("Spark Properties").toMap
     assert(redactedProps(key) == "*********(redacted)")
   }
 
+  test("Spark-33504 sensitive attributes redaction in properties") {
+    val (secretKey, secretPassword) = ("spark.executorEnv.HADOOP_CREDSTORE_PASSWORD",
+      "secret_password")
+    val (customKey, customValue) = ("parse_token", "secret_password")
+
+    val conf = getLoggingConf(testDirPath, None).set(secretKey, secretPassword)
+
+    val properties = new Properties()
+    properties.setProperty(secretKey, secretPassword)
+    properties.setProperty(customKey, customValue)
+
+    val logName = "properties-reaction-test"
+    val eventLogger = new EventLoggingListener(logName, None, testDirPath.toUri(), conf)
+    val listenerBus = new LiveListenerBus(conf)
+
+    val stageId = 1
+    val jobId = 1
+    val stageInfo = new StageInfo(stageId, 0, stageId.toString, 0,
+      Seq.empty, Seq.empty, "details",
+      resourceProfileId = ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID)
+
+    val events = Array(SparkListenerStageSubmitted(stageInfo, properties),
+      SparkListenerJobStart(jobId, 0, Seq(stageInfo), properties))
+
+    eventLogger.start()
+    listenerBus.start(Mockito.mock(classOf[SparkContext]), Mockito.mock(classOf[MetricsSystem]))
+    listenerBus.addToEventLogQueue(eventLogger)
+    events.foreach(event => listenerBus.post(event))
+    listenerBus.stop()
+    eventLogger.stop()
+
+    val logData = EventLogFileReader.openEventLog(new Path(eventLogger.logWriter.logPath),
+      fileSystem)
+    try {
+      val lines = readLines(logData)
+      val logStart = SparkListenerLogStart(SPARK_VERSION)
+      assert(lines.size === 3)
+      assert(lines(0).contains("SparkListenerLogStart"))
+      assert(lines(1).contains("SparkListenerStageSubmitted"))
+      assert(lines(2).contains("SparkListenerJobStart"))
+
+      lines.foreach{
+        line => JsonProtocol.sparkEventFromJson(parse(line)) match {
+          case logStartEvent: SparkListenerLogStart =>
+            assert(logStartEvent == logStart)
+
+          case stageSubmittedEvent: SparkListenerStageSubmitted =>
+            assert(stageSubmittedEvent.properties.getProperty(secretKey) == "*********(redacted)")
+            assert(stageSubmittedEvent.properties.getProperty(customKey) ==  customValue)
+
+          case jobStartEvent : SparkListenerJobStart =>
+            assert(jobStartEvent.properties.getProperty(secretKey) == "*********(redacted)")
+            assert(jobStartEvent.properties.getProperty(customKey) ==  customValue)
+
+          case _ => assert(false)
+        }
+      }
+    } finally {
+      logData.close()
+    }
+  }
+
   test("Executor metrics update") {
     testStageExecutorMetricsEventLogging()
   }
 
+  test("SPARK-31764: isBarrier should be logged in event log") {
+    val conf = new SparkConf()
+    conf.set(EVENT_LOG_ENABLED, true)
+    conf.set(EVENT_LOG_DIR, testDirPath.toString)
+    val sc = new SparkContext("local", "test-SPARK-31764", conf)
+    val appId = sc.applicationId
+
+    sc.parallelize(1 to 10)
+      .barrier()
+      .mapPartitions(_.map(elem => (elem, elem)))
+      .filter(elem => elem._1 % 2 == 0)
+      .reduceByKey(_ + _)
+      .collect
+    sc.stop()
+
+    val eventLogStream = EventLogFileReader.openEventLog(new Path(testDirPath, appId), fileSystem)
+    val events = readLines(eventLogStream).map(line => JsonProtocol.sparkEventFromJson(parse(line)))
+    val jobStartEvents = events
+      .filter(event => event.isInstanceOf[SparkListenerJobStart])
+      .map(_.asInstanceOf[SparkListenerJobStart])
+
+    assert(jobStartEvents.size === 1)
+    val stageInfos = jobStartEvents.head.stageInfos
+    assert(stageInfos.size === 2)
+
+    val stage0 = stageInfos(0)
+    val rddInfosInStage0 = stage0.rddInfos
+    assert(rddInfosInStage0.size === 3)
+    val sortedRddInfosInStage0 = rddInfosInStage0.sortBy(_.scope.get.name)
+    assert(sortedRddInfosInStage0(0).scope.get.name === "filter")
+    assert(sortedRddInfosInStage0(0).isBarrier === true)
+    assert(sortedRddInfosInStage0(1).scope.get.name === "mapPartitions")
+    assert(sortedRddInfosInStage0(1).isBarrier === true)
+    assert(sortedRddInfosInStage0(2).scope.get.name === "parallelize")
+    assert(sortedRddInfosInStage0(2).isBarrier === false)
+
+    val stage1 = stageInfos(1)
+    val rddInfosInStage1 = stage1.rddInfos
+    assert(rddInfosInStage1.size === 1)
+    assert(rddInfosInStage1(0).scope.get.name === "reduceByKey")
+    assert(rddInfosInStage1(0).isBarrier === false) // reduceByKey
+  }
+
   /* ----------------- *
    * Actual test logic *
    * ----------------- */
@@ -281,7 +388,7 @@ class EventLoggingListenerSuite extends SparkFunSuite with LocalSparkContext wit
       8000L, 5000L, 7000L, 4000L, 6000L, 3000L, 10L, 90L, 2L, 20L)
 
     def max(a: Array[Long], b: Array[Long]): Array[Long] =
-      (a, b).zipped.map(Math.max)
+      (a, b).zipped.map(Math.max).toArray
 
     // calculated metric peaks per stage per executor
     // metrics sent during stage 0 for each executor
diff --git a/core/src/test/scala/org/apache/spark/scheduler/ExecutorResourceInfoSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/ExecutorResourceInfoSuite.scala
index 388d4e25a06cf..e392ff53e02c9 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/ExecutorResourceInfoSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/ExecutorResourceInfoSuite.scala
@@ -26,7 +26,7 @@ class ExecutorResourceInfoSuite extends SparkFunSuite {
 
   test("Track Executor Resource information") {
     // Init Executor Resource.
-    val info = new ExecutorResourceInfo(GPU, ArrayBuffer("0", "1", "2", "3"), 1)
+    val info = new ExecutorResourceInfo(GPU, Seq("0", "1", "2", "3"), 1)
     assert(info.availableAddrs.sorted sameElements Seq("0", "1", "2", "3"))
     assert(info.assignedAddrs.isEmpty)
 
@@ -43,7 +43,7 @@ class ExecutorResourceInfoSuite extends SparkFunSuite {
 
   test("Don't allow acquire address that is not available") {
     // Init Executor Resource.
-    val info = new ExecutorResourceInfo(GPU, ArrayBuffer("0", "1", "2", "3"), 1)
+    val info = new ExecutorResourceInfo(GPU, Seq("0", "1", "2", "3"), 1)
     // Acquire some addresses.
     info.acquire(Seq("0", "1"))
     assert(!info.availableAddrs.contains("1"))
@@ -56,7 +56,7 @@ class ExecutorResourceInfoSuite extends SparkFunSuite {
 
   test("Don't allow acquire address that doesn't exist") {
     // Init Executor Resource.
-    val info = new ExecutorResourceInfo(GPU, ArrayBuffer("0", "1", "2", "3"), 1)
+    val info = new ExecutorResourceInfo(GPU, Seq("0", "1", "2", "3"), 1)
     assert(!info.availableAddrs.contains("4"))
     // Acquire an address that doesn't exist
     val e = intercept[SparkException] {
@@ -67,7 +67,7 @@ class ExecutorResourceInfoSuite extends SparkFunSuite {
 
   test("Don't allow release address that is not assigned") {
     // Init Executor Resource.
-    val info = new ExecutorResourceInfo(GPU, ArrayBuffer("0", "1", "2", "3"), 1)
+    val info = new ExecutorResourceInfo(GPU, Seq("0", "1", "2", "3"), 1)
     // Acquire addresses
     info.acquire(Array("0", "1"))
     assert(!info.assignedAddrs.contains("2"))
@@ -80,7 +80,7 @@ class ExecutorResourceInfoSuite extends SparkFunSuite {
 
   test("Don't allow release address that doesn't exist") {
     // Init Executor Resource.
-    val info = new ExecutorResourceInfo(GPU, ArrayBuffer("0", "1", "2", "3"), 1)
+    val info = new ExecutorResourceInfo(GPU, Seq("0", "1", "2", "3"), 1)
     assert(!info.assignedAddrs.contains("4"))
     // Release an address that doesn't exist
     val e = intercept[SparkException] {
@@ -93,7 +93,7 @@ class ExecutorResourceInfoSuite extends SparkFunSuite {
     val slotSeq = Seq(10, 9, 8, 7, 6, 5, 4, 3, 2, 1)
     val addresses = ArrayBuffer("0", "1", "2", "3")
     slotSeq.foreach { slots =>
-      val info = new ExecutorResourceInfo(GPU, addresses, slots)
+      val info = new ExecutorResourceInfo(GPU, addresses.toSeq, slots)
       for (_ <- 0 until slots) {
         addresses.foreach(addr => info.acquire(Seq(addr)))
       }
diff --git a/core/src/test/scala/org/apache/spark/scheduler/ExternalClusterManagerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/ExternalClusterManagerSuite.scala
index 7ead51bc691fb..08191d09a9f2d 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/ExternalClusterManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/ExternalClusterManagerSuite.scala
@@ -90,7 +90,6 @@ private class DummyTaskScheduler extends TaskScheduler {
   override def notifyPartitionCompletion(stageId: Int, partitionId: Int): Unit = {}
   override def setDAGScheduler(dagScheduler: DAGScheduler): Unit = {}
   override def defaultParallelism(): Int = 2
-  override def executorDecommission(executorId: String): Unit = {}
   override def executorLost(executorId: String, reason: ExecutorLossReason): Unit = {}
   override def workerRemoved(workerId: String, host: String, message: String): Unit = {}
   override def applicationAttemptId(): Option[String] = None
@@ -99,4 +98,9 @@ private class DummyTaskScheduler extends TaskScheduler {
       accumUpdates: Array[(Long, Seq[AccumulatorV2[_, _]])],
       blockManagerId: BlockManagerId,
       executorMetrics: Map[(Int, Int), ExecutorMetrics]): Boolean = true
+  override def executorDecommission(
+    executorId: String,
+    decommissionInfo: ExecutorDecommissionInfo): Unit = {}
+  override def getExecutorDecommissionState(
+    executorId: String): Option[ExecutorDecommissionState] = None
 }
diff --git a/core/src/test/scala/org/apache/spark/scheduler/HealthTrackerIntegrationSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/HealthTrackerIntegrationSuite.scala
new file mode 100644
index 0000000000000..29a8f4be8b72b
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/scheduler/HealthTrackerIntegrationSuite.scala
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.scheduler
+
+import org.apache.spark._
+import org.apache.spark.internal.config
+import org.apache.spark.internal.config.Tests._
+
+class HealthTrackerIntegrationSuite extends SchedulerIntegrationSuite[MultiExecutorMockBackend]{
+
+  val badHost = "host-0"
+
+  /**
+   * This backend just always fails if the task is executed on a bad host, but otherwise succeeds
+   * all tasks.
+   */
+  def badHostBackend(): Unit = {
+    val (taskDescription, _) = backend.beginTask()
+    val host = backend.executorIdToExecutor(taskDescription.executorId).host
+    if (host == badHost) {
+      backend.taskFailed(taskDescription, new RuntimeException("I'm a bad host!"))
+    } else {
+      backend.taskSuccess(taskDescription, 42)
+    }
+  }
+
+  // Test demonstrating the issue -- without a config change, the scheduler keeps scheduling
+  // according to locality preferences, and so the job fails
+  testScheduler("If preferred node is bad, without excludeOnFailure job will fail",
+    extraConfs = Seq(
+      config.EXCLUDE_ON_FAILURE_ENABLED.key -> "false"
+  )) {
+    val rdd = new MockRDDWithLocalityPrefs(sc, 10, Nil, badHost)
+    withBackend(badHostBackend _) {
+      val jobFuture = submit(rdd, (0 until 10).toArray)
+      awaitJobTermination(jobFuture, duration)
+    }
+    assertDataStructuresEmpty(noFailure = false)
+  }
+
+  testScheduler(
+    "With default settings, job can succeed despite multiple bad executors on node",
+    extraConfs = Seq(
+      config.EXCLUDE_ON_FAILURE_ENABLED.key -> "true",
+      config.TASK_MAX_FAILURES.key -> "4",
+      TEST_N_HOSTS.key -> "2",
+      TEST_N_EXECUTORS_HOST.key -> "5",
+      TEST_N_CORES_EXECUTOR.key -> "10"
+    )
+  ) {
+    // To reliably reproduce the failure that would occur without exludeOnFailure, we have to use 1
+    // task.  That way, we ensure this 1 task gets rotated through enough bad executors on the host
+    // to fail the taskSet, before we have a bunch of different tasks fail in the executors so we
+    // exclude them.
+    // But the point here is -- without excludeOnFailure, we would never schedule anything on the
+    // good host-1 before we hit too many failures trying our preferred host-0.
+    val rdd = new MockRDDWithLocalityPrefs(sc, 1, Nil, badHost)
+    withBackend(badHostBackend _) {
+      val jobFuture = submit(rdd, (0 until 1).toArray)
+      awaitJobTermination(jobFuture, duration)
+    }
+    assertDataStructuresEmpty(noFailure = true)
+  }
+
+  // Here we run with the excludeOnFailure on, and the default config takes care of having this
+  // robust to one bad node.
+  testScheduler(
+    "Bad node with multiple executors, job will still succeed with the right confs",
+    extraConfs = Seq(
+       config.EXCLUDE_ON_FAILURE_ENABLED.key -> "true",
+      // just to avoid this test taking too long
+      config.LOCALITY_WAIT.key -> "10ms"
+    )
+  ) {
+    val rdd = new MockRDDWithLocalityPrefs(sc, 10, Nil, badHost)
+    withBackend(badHostBackend _) {
+      val jobFuture = submit(rdd, (0 until 10).toArray)
+      awaitJobTermination(jobFuture, duration)
+    }
+    assert(results === (0 until 10).map { _ -> 42 }.toMap)
+    assertDataStructuresEmpty(noFailure = true)
+  }
+
+  // Make sure that if we've failed on all executors, but haven't hit task.maxFailures yet, we try
+  // to acquire a new executor and if we aren't able to get one, the job doesn't hang and we abort
+  testScheduler(
+    "SPARK-15865 Progress with fewer executors than maxTaskFailures",
+    extraConfs = Seq(
+      config.EXCLUDE_ON_FAILURE_ENABLED.key -> "true",
+      TEST_N_HOSTS.key -> "2",
+      TEST_N_EXECUTORS_HOST.key -> "1",
+      TEST_N_CORES_EXECUTOR.key -> "1",
+      config.UNSCHEDULABLE_TASKSET_TIMEOUT.key -> "0s"
+    )
+  ) {
+    def runBackend(): Unit = {
+      val (taskDescription, _) = backend.beginTask()
+      backend.taskFailed(taskDescription, new RuntimeException("test task failure"))
+    }
+    withBackend(runBackend _) {
+      val jobFuture = submit(new MockRDD(sc, 10, Nil), (0 until 10).toArray)
+      awaitJobTermination(jobFuture, duration)
+      val pattern = (
+        s"""|Aborting TaskSet 0.0 because task .*
+            |cannot run anywhere due to node and executor excludeOnFailure""".stripMargin).r
+      assert(pattern.findFirstIn(failure.getMessage).isDefined,
+        s"Couldn't find $pattern in ${failure.getMessage()}")
+    }
+    assertDataStructuresEmpty(noFailure = false)
+  }
+}
+
+class MultiExecutorMockBackend(
+    conf: SparkConf,
+    taskScheduler: TaskSchedulerImpl) extends MockBackend(conf, taskScheduler) {
+
+  val nHosts = conf.get(TEST_N_HOSTS)
+  val nExecutorsPerHost = conf.get(TEST_N_EXECUTORS_HOST)
+  val nCoresPerExecutor = conf.get(TEST_N_CORES_EXECUTOR)
+
+  override val executorIdToExecutor: Map[String, ExecutorTaskStatus] = {
+    (0 until nHosts).flatMap { hostIdx =>
+      val hostName = "host-" + hostIdx
+      (0 until nExecutorsPerHost).map { subIdx =>
+        val executorId = (hostIdx * nExecutorsPerHost + subIdx).toString
+        executorId ->
+          ExecutorTaskStatus(host = hostName, executorId = executorId, nCoresPerExecutor)
+      }
+    }.toMap
+  }
+
+  override def defaultParallelism(): Int = nHosts * nExecutorsPerHost * nCoresPerExecutor
+}
+
+class MockRDDWithLocalityPrefs(
+    sc: SparkContext,
+    numPartitions: Int,
+    shuffleDeps: Seq[ShuffleDependency[Int, Int, Nothing]],
+    val preferredLoc: String) extends MockRDD(sc, numPartitions, shuffleDeps) {
+  override def getPreferredLocations(split: Partition): Seq[String] = {
+    Seq(preferredLoc)
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/scheduler/HealthTrackerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/HealthTrackerSuite.scala
new file mode 100644
index 0000000000000..5710be1a97174
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/scheduler/HealthTrackerSuite.scala
@@ -0,0 +1,660 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.scheduler
+
+import org.mockito.ArgumentMatchers.any
+import org.mockito.Mockito.{never, verify, when}
+import org.mockito.invocation.InvocationOnMock
+import org.scalatest.BeforeAndAfterEach
+import org.scalatestplus.mockito.MockitoSugar
+
+import org.apache.spark._
+import org.apache.spark.internal.config
+import org.apache.spark.util.ManualClock
+
+class HealthTrackerSuite extends SparkFunSuite with BeforeAndAfterEach with MockitoSugar
+    with LocalSparkContext {
+
+  private val clock = new ManualClock(0)
+
+  private var healthTracker: HealthTracker = _
+  private var listenerBusMock: LiveListenerBus = _
+  private var scheduler: TaskSchedulerImpl = _
+  private var conf: SparkConf = _
+
+  override def beforeEach(): Unit = {
+    conf = new SparkConf().setAppName("test").setMaster("local")
+      .set(config.EXCLUDE_ON_FAILURE_ENABLED.key, "true")
+    scheduler = mockTaskSchedWithConf(conf)
+
+    clock.setTime(0)
+
+    listenerBusMock = mock[LiveListenerBus]
+    healthTracker = new HealthTracker(listenerBusMock, conf, None, clock)
+  }
+
+  override def afterEach(): Unit = {
+    if (healthTracker != null) {
+      healthTracker = null
+    }
+    if (scheduler != null) {
+      scheduler.stop()
+      scheduler = null
+    }
+    super.afterEach()
+  }
+
+  // All executors and hosts used in tests should be in this set, so that [[assertEquivalentToSet]]
+  // works.  Its OK if its got extraneous entries
+  val allExecutorAndHostIds = {
+    (('A' to 'Z')++ (1 to 100).map(_.toString))
+      .flatMap{ suffix =>
+        Seq(s"host$suffix", s"host-$suffix")
+      }
+  }.toSet
+
+  /**
+   * Its easier to write our tests as if we could directly look at the sets of nodes & executors in
+   * the exclude.  However the api doesn't expose a set, so this is a simple way to test
+   * something similar, since we know the universe of values that might appear in these sets.
+   */
+  def assertEquivalentToSet(f: String => Boolean, expected: Set[String]): Unit = {
+    allExecutorAndHostIds.foreach { id =>
+      val actual = f(id)
+      val exp = expected.contains(id)
+      assert(actual === exp, raw"""for string "$id" """)
+    }
+  }
+
+  def mockTaskSchedWithConf(conf: SparkConf): TaskSchedulerImpl = {
+    sc = new SparkContext(conf)
+    val scheduler = mock[TaskSchedulerImpl]
+    when(scheduler.sc).thenReturn(sc)
+    when(scheduler.mapOutputTracker).thenReturn(
+      SparkEnv.get.mapOutputTracker.asInstanceOf[MapOutputTrackerMaster])
+    scheduler
+  }
+
+  def createTaskSetExcludelist(stageId: Int = 0): TaskSetExcludelist = {
+    new TaskSetExcludelist(listenerBusMock, conf, stageId, stageAttemptId = 0, clock = clock)
+  }
+
+  test("executors can be excluded with only a few failures per stage") {
+    // For many different stages, executor 1 fails a task, then executor 2 succeeds the task,
+    // and then the task set is done.  Not enough failures to exclude the executor *within*
+    // any particular taskset, but we still exclude the executor overall eventually.
+    // Also, we intentionally have a mix of task successes and failures -- there are even some
+    // successes after the executor is excluded.  The idea here is those tasks get scheduled
+    // before the executor is excluded.  We might get successes after excluding (because the
+    // executor might be flaky but not totally broken).  But successes should not unexclude the
+    // executor.
+    val failuresUntilExcludeed = conf.get(config.MAX_FAILURES_PER_EXEC)
+    var failuresSoFar = 0
+    (0 until failuresUntilExcludeed * 10).foreach { stageId =>
+      val taskSetExclude = createTaskSetExcludelist(stageId)
+      if (stageId % 2 == 0) {
+        // fail one task in every other taskset
+        taskSetExclude.updateExcludedForFailedTask(
+          "hostA", exec = "1", index = 0, failureReason = "testing")
+        failuresSoFar += 1
+      }
+      healthTracker.updateExcludedForSuccessfulTaskSet(stageId, 0, taskSetExclude.execToFailures)
+      assert(failuresSoFar == stageId / 2 + 1)
+      if (failuresSoFar < failuresUntilExcludeed) {
+        assertEquivalentToSet(healthTracker.isExecutorExcluded(_), Set())
+      } else {
+        assertEquivalentToSet(healthTracker.isExecutorExcluded(_), Set("1"))
+        verify(listenerBusMock).post(
+          SparkListenerExecutorExcluded(0, "1", failuresUntilExcludeed))
+        verify(listenerBusMock).post(
+          SparkListenerExecutorBlacklisted(0, "1", failuresUntilExcludeed))
+      }
+    }
+  }
+
+  // If an executor has many task failures, but the task set ends up failing, it shouldn't be
+  // counted against the executor.
+  test("executors aren't excluded as a result of tasks in failed task sets") {
+    val failuresUntilExcludeed = conf.get(config.MAX_FAILURES_PER_EXEC)
+    // for many different stages, executor 1 fails a task, and then the taskSet fails.
+    (0 until failuresUntilExcludeed * 10).foreach { stage =>
+      val taskSetExclude = createTaskSetExcludelist(stage)
+      taskSetExclude.updateExcludedForFailedTask(
+        "hostA", exec = "1", index = 0, failureReason = "testing")
+    }
+    assertEquivalentToSet(healthTracker.isExecutorExcluded(_), Set())
+  }
+
+  Seq(true, false).foreach { succeedTaskSet =>
+    val label = if (succeedTaskSet) "success" else "failure"
+    test(s"stage exclude updates correctly on stage $label") {
+      // Within one taskset, an executor fails a few times, so it's excluded for the taskset.
+      // But if the taskset fails, we shouldn't exclude the executor after the stage.
+      val taskSetExclude = createTaskSetExcludelist(0)
+      // We trigger enough failures for both the taskset exclude, and the application exclude.
+      val numFailures = math.max(conf.get(config.MAX_FAILURES_PER_EXEC),
+        conf.get(config.MAX_FAILURES_PER_EXEC_STAGE))
+      (0 until numFailures).foreach { index =>
+        taskSetExclude.updateExcludedForFailedTask(
+          "hostA", exec = "1", index = index, failureReason = "testing")
+      }
+      assert(taskSetExclude.isExecutorExcludedForTaskSet("1"))
+      assertEquivalentToSet(healthTracker.isExecutorExcluded(_), Set())
+      if (succeedTaskSet) {
+        // The task set succeeded elsewhere, so we should count those failures against our executor,
+        // and it should be excluded for the entire application.
+        healthTracker.updateExcludedForSuccessfulTaskSet(0, 0, taskSetExclude.execToFailures)
+        assertEquivalentToSet(healthTracker.isExecutorExcluded(_), Set("1"))
+        verify(listenerBusMock).post(SparkListenerExecutorExcluded(0, "1", numFailures))
+      } else {
+        // The task set failed, so we don't count these failures against the executor for other
+        // stages.
+        assertEquivalentToSet(healthTracker.isExecutorExcluded(_), Set())
+      }
+    }
+  }
+
+  test("excluded executors and nodes get recovered with time") {
+    val taskSetExclude0 = createTaskSetExcludelist(stageId = 0)
+    // Fail 4 tasks in one task set on executor 1, so that executor gets excluded for the whole
+    // application.
+    (0 until 4).foreach { partition =>
+      taskSetExclude0.updateExcludedForFailedTask(
+        "hostA", exec = "1", index = partition, failureReason = "testing")
+    }
+    healthTracker.updateExcludedForSuccessfulTaskSet(0, 0, taskSetExclude0.execToFailures)
+    assert(healthTracker.excludedNodeList() === Set())
+    assertEquivalentToSet(healthTracker.isNodeExcluded(_), Set())
+    assertEquivalentToSet(healthTracker.isExecutorExcluded(_), Set("1"))
+    verify(listenerBusMock).post(SparkListenerExecutorExcluded(0, "1", 4))
+    verify(listenerBusMock).post(SparkListenerExecutorBlacklisted(0, "1", 4))
+
+    val taskSetExclude1 = createTaskSetExcludelist(stageId = 1)
+    // Fail 4 tasks in one task set on executor 2, so that executor gets excluded for the whole
+    // application.  Since that's the second executor that is excluded on the same node, we also
+    // exclude that node.
+    (0 until 4).foreach { partition =>
+      taskSetExclude1.updateExcludedForFailedTask(
+        "hostA", exec = "2", index = partition, failureReason = "testing")
+    }
+    healthTracker.updateExcludedForSuccessfulTaskSet(0, 0, taskSetExclude1.execToFailures)
+    assert(healthTracker.excludedNodeList() === Set("hostA"))
+    assertEquivalentToSet(healthTracker.isNodeExcluded(_), Set("hostA"))
+    verify(listenerBusMock).post(SparkListenerNodeExcluded(0, "hostA", 2))
+    verify(listenerBusMock).post(SparkListenerNodeBlacklisted(0, "hostA", 2))
+    assertEquivalentToSet(healthTracker.isExecutorExcluded(_), Set("1", "2"))
+    verify(listenerBusMock).post(SparkListenerExecutorExcluded(0, "2", 4))
+    verify(listenerBusMock).post(SparkListenerExecutorBlacklisted(0, "2", 4))
+
+    // Advance the clock and then make sure hostA and executors 1 and 2 have been removed from the
+    // exclude.
+    val timeout = healthTracker.EXCLUDE_ON_FAILURE_TIMEOUT_MILLIS + 1
+    clock.advance(timeout)
+    healthTracker.applyExcludeOnFailureTimeout()
+    assert(healthTracker.excludedNodeList() === Set())
+    assertEquivalentToSet(healthTracker.isNodeExcluded(_), Set())
+    assertEquivalentToSet(healthTracker.isExecutorExcluded(_), Set())
+    verify(listenerBusMock).post(SparkListenerExecutorUnexcluded(timeout, "2"))
+    verify(listenerBusMock).post(SparkListenerExecutorUnexcluded(timeout, "1"))
+    verify(listenerBusMock).post(SparkListenerExecutorUnblacklisted(timeout, "2"))
+    verify(listenerBusMock).post(SparkListenerExecutorUnblacklisted(timeout, "1"))
+    verify(listenerBusMock).post(SparkListenerNodeUnexcluded(timeout, "hostA"))
+
+    // Fail one more task, but executor isn't put back into exclude since the count of failures
+    // on that executor should have been reset to 0.
+    val taskSetExclude2 = createTaskSetExcludelist(stageId = 2)
+    taskSetExclude2.updateExcludedForFailedTask(
+      "hostA", exec = "1", index = 0, failureReason = "testing")
+    healthTracker.updateExcludedForSuccessfulTaskSet(2, 0, taskSetExclude2.execToFailures)
+    assert(healthTracker.excludedNodeList() === Set())
+    assertEquivalentToSet(healthTracker.isNodeExcluded(_), Set())
+    assertEquivalentToSet(healthTracker.isExecutorExcluded(_), Set())
+  }
+
+  test("exclude can handle lost executors") {
+    // The exclude should still work if an executor is killed completely.  We should still
+    // be able to exclude the entire node.
+    val taskSetExclude0 = createTaskSetExcludelist(stageId = 0)
+    // Lets say that executor 1 dies completely.  We get some task failures, but
+    // the taskset then finishes successfully (elsewhere).
+    (0 until 4).foreach { partition =>
+      taskSetExclude0.updateExcludedForFailedTask(
+        "hostA", exec = "1", index = partition, failureReason = "testing")
+    }
+    healthTracker.handleRemovedExecutor("1")
+    healthTracker.updateExcludedForSuccessfulTaskSet(
+      stageId = 0,
+      stageAttemptId = 0,
+      taskSetExclude0.execToFailures)
+    assert(healthTracker.isExecutorExcluded("1"))
+    verify(listenerBusMock).post(SparkListenerExecutorExcluded(0, "1", 4))
+    val t1 = healthTracker.EXCLUDE_ON_FAILURE_TIMEOUT_MILLIS / 2
+    clock.advance(t1)
+
+    // Now another executor gets spun up on that host, but it also dies.
+    val taskSetExclude1 = createTaskSetExcludelist(stageId = 1)
+    (0 until 4).foreach { partition =>
+      taskSetExclude1.updateExcludedForFailedTask(
+        "hostA", exec = "2", index = partition, failureReason = "testing")
+    }
+    healthTracker.handleRemovedExecutor("2")
+    healthTracker.updateExcludedForSuccessfulTaskSet(
+      stageId = 1,
+      stageAttemptId = 0,
+      taskSetExclude1.execToFailures)
+    // We've now had two bad executors on the hostA, so we should exclude the entire node.
+    assert(healthTracker.isExecutorExcluded("1"))
+    assert(healthTracker.isExecutorExcluded("2"))
+    verify(listenerBusMock).post(SparkListenerExecutorExcluded(t1, "2", 4))
+    assert(healthTracker.isNodeExcluded("hostA"))
+    verify(listenerBusMock).post(SparkListenerNodeExcluded(t1, "hostA", 2))
+
+    // Advance the clock so that executor 1 should no longer be explicitly excluded, but
+    // everything else should still be excluded.
+    val t2 = healthTracker.EXCLUDE_ON_FAILURE_TIMEOUT_MILLIS / 2 + 1
+    clock.advance(t2)
+    healthTracker.applyExcludeOnFailureTimeout()
+    assert(!healthTracker.isExecutorExcluded("1"))
+    verify(listenerBusMock).post(SparkListenerExecutorUnexcluded(t1 + t2, "1"))
+    assert(healthTracker.isExecutorExcluded("2"))
+    assert(healthTracker.isNodeExcluded("hostA"))
+    // make sure we don't leak memory
+    assert(!healthTracker.executorIdToExcludedStatus.contains("1"))
+    assert(!healthTracker.nodeToExcludedExecs("hostA").contains("1"))
+    // Advance the timeout again so now hostA should be removed from the exclude.
+    clock.advance(t1)
+    healthTracker.applyExcludeOnFailureTimeout()
+    assert(!healthTracker.nodeIdToExcludedExpiryTime.contains("hostA"))
+    verify(listenerBusMock).post(SparkListenerNodeUnexcluded(t1 + t2 + t1, "hostA"))
+    // Even though unexcluding a node implicitly unexcludes all of its executors,
+    // there will be no SparkListenerExecutorUnexcluded sent here.
+  }
+
+  test("task failures expire with time") {
+    // Verifies that 2 failures within the timeout period cause an executor to be excluded, but
+    // if task failures are spaced out by more than the timeout period, the first failure is timed
+    // out, and the executor isn't excluded.
+    var stageId = 0
+
+    def failOneTaskInTaskSet(exec: String): Unit = {
+      val taskSetExclude = createTaskSetExcludelist(stageId = stageId)
+      taskSetExclude.updateExcludedForFailedTask("host-" + exec, exec, 0, "testing")
+      healthTracker.updateExcludedForSuccessfulTaskSet(stageId, 0, taskSetExclude.execToFailures)
+      stageId += 1
+    }
+
+    failOneTaskInTaskSet(exec = "1")
+    // We have one sporadic failure on exec 2, but that's it.  Later checks ensure that we never
+    // exclude executor 2 despite this one failure.
+    failOneTaskInTaskSet(exec = "2")
+    assertEquivalentToSet(healthTracker.isExecutorExcluded(_), Set())
+    assert(healthTracker.nextExpiryTime === Long.MaxValue)
+
+    // We advance the clock past the expiry time.
+    clock.advance(healthTracker.EXCLUDE_ON_FAILURE_TIMEOUT_MILLIS + 1)
+    val t0 = clock.getTimeMillis()
+    healthTracker.applyExcludeOnFailureTimeout()
+    assert(healthTracker.nextExpiryTime === Long.MaxValue)
+    failOneTaskInTaskSet(exec = "1")
+
+    // Because the 2nd failure on executor 1 happened past the expiry time, nothing should have been
+    // excluded.
+    assertEquivalentToSet(healthTracker.isExecutorExcluded(_), Set())
+
+    // Now we add one more failure, within the timeout, and it should be counted.
+    clock.setTime(t0 + healthTracker.EXCLUDE_ON_FAILURE_TIMEOUT_MILLIS - 1)
+    val t1 = clock.getTimeMillis()
+    failOneTaskInTaskSet(exec = "1")
+    healthTracker.applyExcludeOnFailureTimeout()
+    assertEquivalentToSet(healthTracker.isExecutorExcluded(_), Set("1"))
+    verify(listenerBusMock).post(SparkListenerExecutorExcluded(t1, "1", 2))
+    assert(healthTracker.nextExpiryTime === t1 + healthTracker.EXCLUDE_ON_FAILURE_TIMEOUT_MILLIS)
+
+    // Add failures on executor 3, make sure it gets put on the exclude.
+    clock.setTime(t1 + healthTracker.EXCLUDE_ON_FAILURE_TIMEOUT_MILLIS - 1)
+    val t2 = clock.getTimeMillis()
+    failOneTaskInTaskSet(exec = "3")
+    failOneTaskInTaskSet(exec = "3")
+    healthTracker.applyExcludeOnFailureTimeout()
+    assertEquivalentToSet(healthTracker.isExecutorExcluded(_), Set("1", "3"))
+    verify(listenerBusMock).post(SparkListenerExecutorExcluded(t2, "3", 2))
+    assert(healthTracker.nextExpiryTime === t1 + healthTracker.EXCLUDE_ON_FAILURE_TIMEOUT_MILLIS)
+
+    // Now we go past the timeout for executor 1, so it should be dropped from the exclude.
+    clock.setTime(t1 + healthTracker.EXCLUDE_ON_FAILURE_TIMEOUT_MILLIS + 1)
+    healthTracker.applyExcludeOnFailureTimeout()
+    assertEquivalentToSet(healthTracker.isExecutorExcluded(_), Set("3"))
+    verify(listenerBusMock).post(SparkListenerExecutorUnexcluded(clock.getTimeMillis(), "1"))
+    assert(healthTracker.nextExpiryTime === t2 + healthTracker.EXCLUDE_ON_FAILURE_TIMEOUT_MILLIS)
+
+    // Make sure that we update correctly when we go from having excluded executors to
+    // just having tasks with timeouts.
+    clock.setTime(t2 + healthTracker.EXCLUDE_ON_FAILURE_TIMEOUT_MILLIS - 1)
+    failOneTaskInTaskSet(exec = "4")
+    healthTracker.applyExcludeOnFailureTimeout()
+    assertEquivalentToSet(healthTracker.isExecutorExcluded(_), Set("3"))
+    assert(healthTracker.nextExpiryTime === t2 + healthTracker.EXCLUDE_ON_FAILURE_TIMEOUT_MILLIS)
+
+    clock.setTime(t2 + healthTracker.EXCLUDE_ON_FAILURE_TIMEOUT_MILLIS + 1)
+    healthTracker.applyExcludeOnFailureTimeout()
+    assertEquivalentToSet(healthTracker.isExecutorExcluded(_), Set())
+    verify(listenerBusMock).post(SparkListenerExecutorUnexcluded(clock.getTimeMillis(), "3"))
+    // we've got one task failure still, but we don't bother setting nextExpiryTime to it, to
+    // avoid wasting time checking for expiry of individual task failures.
+    assert(healthTracker.nextExpiryTime === Long.MaxValue)
+  }
+
+  test("task failure timeout works as expected for long-running tasksets") {
+    // This ensures that we don't trigger spurious excluding for long tasksets, when the taskset
+    // finishes long after the task failures.  We create two tasksets, each with one failure.
+    // Individually they shouldn't cause any excluding since there is only one failure.
+    // Furthermore, we space the failures out so far that even when both tasksets have completed,
+    // we still don't trigger any excluding.
+    val taskSetExclude1 = createTaskSetExcludelist(stageId = 1)
+    val taskSetExclude2 = createTaskSetExcludelist(stageId = 2)
+    // Taskset1 has one failure immediately
+    taskSetExclude1.updateExcludedForFailedTask("host-1", "1", 0, "testing")
+    // Then we have a *long* delay, much longer than the timeout, before any other failures or
+    // taskset completion
+    clock.advance(healthTracker.EXCLUDE_ON_FAILURE_TIMEOUT_MILLIS * 5)
+    // After the long delay, we have one failure on taskset 2, on the same executor
+    taskSetExclude2.updateExcludedForFailedTask("host-1", "1", 0, "testing")
+    // Finally, we complete both tasksets.  Its important here to complete taskset2 *first*.  We
+    // want to make sure that when taskset 1 finishes, even though we've now got two task failures,
+    // we realize that the task failure we just added was well before the timeout.
+    clock.advance(1)
+    healthTracker.updateExcludedForSuccessfulTaskSet(stageId = 2, 0, taskSetExclude2.execToFailures)
+    clock.advance(1)
+    healthTracker.updateExcludedForSuccessfulTaskSet(stageId = 1, 0, taskSetExclude1.execToFailures)
+
+    // Make sure nothing was excluded
+    assertEquivalentToSet(healthTracker.isExecutorExcluded(_), Set())
+  }
+
+  test("only exclude nodes for the application when enough executors have failed on that " +
+    "specific host") {
+    // we exclude executors on two different hosts -- make sure that doesn't lead to any
+    // node excluding
+    val taskSetExclude0 = createTaskSetExcludelist(stageId = 0)
+    taskSetExclude0.updateExcludedForFailedTask(
+      "hostA", exec = "1", index = 0, failureReason = "testing")
+    taskSetExclude0.updateExcludedForFailedTask(
+      "hostA", exec = "1", index = 1, failureReason = "testing")
+    healthTracker.updateExcludedForSuccessfulTaskSet(0, 0, taskSetExclude0.execToFailures)
+    assertEquivalentToSet(healthTracker.isExecutorExcluded(_), Set("1"))
+    verify(listenerBusMock).post(SparkListenerExecutorExcluded(0, "1", 2))
+    assertEquivalentToSet(healthTracker.isNodeExcluded(_), Set())
+
+    val taskSetExclude1 = createTaskSetExcludelist(stageId = 1)
+    taskSetExclude1.updateExcludedForFailedTask(
+      "hostB", exec = "2", index = 0, failureReason = "testing")
+    taskSetExclude1.updateExcludedForFailedTask(
+      "hostB", exec = "2", index = 1, failureReason = "testing")
+    healthTracker.updateExcludedForSuccessfulTaskSet(1, 0, taskSetExclude1.execToFailures)
+    assertEquivalentToSet(healthTracker.isExecutorExcluded(_), Set("1", "2"))
+    verify(listenerBusMock).post(SparkListenerExecutorExcluded(0, "2", 2))
+    assertEquivalentToSet(healthTracker.isNodeExcluded(_), Set())
+
+    // Finally, exclude another executor on the same node as the original excluded executor,
+    // and make sure this time we *do* exclude the node.
+    val taskSetExclude2 = createTaskSetExcludelist(stageId = 0)
+    taskSetExclude2.updateExcludedForFailedTask(
+      "hostA", exec = "3", index = 0, failureReason = "testing")
+    taskSetExclude2.updateExcludedForFailedTask(
+      "hostA", exec = "3", index = 1, failureReason = "testing")
+    healthTracker.updateExcludedForSuccessfulTaskSet(0, 0, taskSetExclude2.execToFailures)
+    assertEquivalentToSet(healthTracker.isExecutorExcluded(_), Set("1", "2", "3"))
+    verify(listenerBusMock).post(SparkListenerExecutorExcluded(0, "3", 2))
+    assertEquivalentToSet(healthTracker.isNodeExcluded(_), Set("hostA"))
+    verify(listenerBusMock).post(SparkListenerNodeExcluded(0, "hostA", 2))
+  }
+
+  test("exclude still respects legacy configs") {
+    val conf = new SparkConf().setMaster("local")
+    assert(!HealthTracker.isExcludeOnFailureEnabled(conf))
+    conf.set(config.EXCLUDE_ON_FAILURE_LEGACY_TIMEOUT_CONF, 5000L)
+    assert(HealthTracker.isExcludeOnFailureEnabled(conf))
+    assert(5000 === HealthTracker.getExludeOnFailureTimeout(conf))
+    // the new conf takes precedence, though
+    conf.set(config.EXCLUDE_ON_FAILURE_TIMEOUT_CONF, 1000L)
+    assert(1000 === HealthTracker.getExludeOnFailureTimeout(conf))
+
+    // if you explicitly set the legacy conf to 0, that also would disable excluding
+    conf.set(config.EXCLUDE_ON_FAILURE_LEGACY_TIMEOUT_CONF, 0L)
+    assert(!HealthTracker.isExcludeOnFailureEnabled(conf))
+    // but again, the new conf takes precedence
+    conf.set(config.EXCLUDE_ON_FAILURE_ENABLED, true)
+    assert(HealthTracker.isExcludeOnFailureEnabled(conf))
+    assert(1000 === HealthTracker.getExludeOnFailureTimeout(conf))
+  }
+
+  test("check exclude configuration invariants") {
+    val conf = new SparkConf().setMaster("yarn").set(config.SUBMIT_DEPLOY_MODE, "cluster")
+    Seq(
+      (2, 2),
+      (2, 3)
+    ).foreach { case (maxTaskFailures, maxNodeAttempts) =>
+      conf.set(config.TASK_MAX_FAILURES, maxTaskFailures)
+      conf.set(config.MAX_TASK_ATTEMPTS_PER_NODE.key, maxNodeAttempts.toString)
+      val excMsg = intercept[IllegalArgumentException] {
+        HealthTracker.validateExcludeOnFailureConfs(conf)
+      }.getMessage()
+      assert(excMsg === s"${config.MAX_TASK_ATTEMPTS_PER_NODE.key} " +
+        s"( = ${maxNodeAttempts}) was >= ${config.TASK_MAX_FAILURES.key} " +
+        s"( = ${maxTaskFailures} ). Though excludeOnFailure is enabled, with this " +
+        s"configuration, Spark will not be robust to one bad node. Decrease " +
+        s"${config.MAX_TASK_ATTEMPTS_PER_NODE.key}, increase ${config.TASK_MAX_FAILURES.key}, " +
+        s"or disable excludeOnFailure with ${config.EXCLUDE_ON_FAILURE_ENABLED.key}")
+    }
+
+    conf.remove(config.TASK_MAX_FAILURES)
+    conf.remove(config.MAX_TASK_ATTEMPTS_PER_NODE)
+
+    Seq(
+      config.MAX_TASK_ATTEMPTS_PER_EXECUTOR,
+      config.MAX_TASK_ATTEMPTS_PER_NODE,
+      config.MAX_FAILURES_PER_EXEC_STAGE,
+      config.MAX_FAILED_EXEC_PER_NODE_STAGE,
+      config.MAX_FAILURES_PER_EXEC,
+      config.MAX_FAILED_EXEC_PER_NODE,
+      config.EXCLUDE_ON_FAILURE_TIMEOUT_CONF
+    ).foreach { config =>
+      conf.set(config.key, "0")
+      val excMsg = intercept[IllegalArgumentException] {
+        HealthTracker.validateExcludeOnFailureConfs(conf)
+      }.getMessage()
+      assert(excMsg.contains(s"${config.key} was 0, but must be > 0."))
+      conf.remove(config)
+    }
+  }
+
+  test("excluding kills executors, configured by EXCLUDE_ON_FAILURE_KILL_ENABLED") {
+    val allocationClientMock = mock[ExecutorAllocationClient]
+    when(allocationClientMock.killExecutors(any(), any(), any(), any())).thenReturn(Seq("called"))
+    when(allocationClientMock.killExecutorsOnHost("hostA")).thenAnswer { (_: InvocationOnMock) =>
+      // To avoid a race between excluding and killing, it is important that the nodeExclude
+      // is updated before we ask the executor allocation client to kill all the executors
+      // on a particular host.
+      if (healthTracker.excludedNodeList().contains("hostA")) {
+        true
+      } else {
+        throw new IllegalStateException("hostA should be on the exclude")
+      }
+    }
+    healthTracker = new HealthTracker(listenerBusMock, conf, Some(allocationClientMock), clock)
+
+    // Disable auto-kill. Exclude an executor and make sure killExecutors is not called.
+    conf.set(config.EXCLUDE_ON_FAILURE_KILL_ENABLED, false)
+
+    val taskSetExclude0 = createTaskSetExcludelist(stageId = 0)
+    // Fail 4 tasks in one task set on executor 1, so that executor gets excluded for the whole
+    // application.
+    (0 until 4).foreach { partition =>
+      taskSetExclude0.updateExcludedForFailedTask(
+        "hostA", exec = "1", index = partition, failureReason = "testing")
+    }
+    healthTracker.updateExcludedForSuccessfulTaskSet(0, 0, taskSetExclude0.execToFailures)
+
+    verify(allocationClientMock, never).killExecutor(any())
+
+    val taskSetExclude1 = createTaskSetExcludelist(stageId = 1)
+    // Fail 4 tasks in one task set on executor 2, so that executor gets excluded for the whole
+    // application.  Since that's the second executor that is excluded on the same node, we also
+    // exclude that node.
+    (0 until 4).foreach { partition =>
+      taskSetExclude1.updateExcludedForFailedTask(
+        "hostA", exec = "2", index = partition, failureReason = "testing")
+    }
+    healthTracker.updateExcludedForSuccessfulTaskSet(0, 0, taskSetExclude1.execToFailures)
+
+    verify(allocationClientMock, never).killExecutors(any(), any(), any(), any())
+    verify(allocationClientMock, never).killExecutorsOnHost(any())
+
+    // Enable auto-kill. Exclude an executor and make sure killExecutors is called.
+    conf.set(config.EXCLUDE_ON_FAILURE_KILL_ENABLED, true)
+    healthTracker = new HealthTracker(listenerBusMock, conf, Some(allocationClientMock), clock)
+
+    val taskSetExclude2 = createTaskSetExcludelist(stageId = 0)
+    // Fail 4 tasks in one task set on executor 1, so that executor gets excluded for the whole
+    // application.
+    (0 until 4).foreach { partition =>
+      taskSetExclude2.updateExcludedForFailedTask(
+        "hostA", exec = "1", index = partition, failureReason = "testing")
+    }
+    healthTracker.updateExcludedForSuccessfulTaskSet(0, 0, taskSetExclude2.execToFailures)
+
+    verify(allocationClientMock).killExecutors(Seq("1"), false, false, true)
+
+    val taskSetExclude3 = createTaskSetExcludelist(stageId = 1)
+    // Fail 4 tasks in one task set on executor 2, so that executor gets excluded for the whole
+    // application.  Since that's the second executor that is excluded on the same node, we also
+    // exclude that node.
+    (0 until 4).foreach { partition =>
+      taskSetExclude3.updateExcludedForFailedTask(
+        "hostA", exec = "2", index = partition, failureReason = "testing")
+    }
+    healthTracker.updateExcludedForSuccessfulTaskSet(0, 0, taskSetExclude3.execToFailures)
+
+    verify(allocationClientMock).killExecutors(Seq("2"), false, false, true)
+    verify(allocationClientMock).killExecutorsOnHost("hostA")
+  }
+
+  test("excluding decommission and kills executors when enabled") {
+    val allocationClientMock = mock[ExecutorAllocationClient]
+
+    // verify we decommission when configured
+    conf.set(config.EXCLUDE_ON_FAILURE_KILL_ENABLED, true)
+    conf.set(config.DECOMMISSION_ENABLED.key, "true")
+    conf.set(config.EXCLUDE_ON_FAILURE_DECOMMISSION_ENABLED.key, "true")
+    conf.set(config.MAX_FAILURES_PER_EXEC.key, "1")
+    conf.set(config.MAX_FAILED_EXEC_PER_NODE.key, "2")
+    healthTracker = new HealthTracker(listenerBusMock, conf, Some(allocationClientMock), clock)
+
+    // Fail 4 tasks in one task set on executor 1, so that executor gets excluded for the whole
+    // application.
+    val taskSetExclude2 = createTaskSetExcludelist(stageId = 0)
+    (0 until 4).foreach { partition =>
+      taskSetExclude2.updateExcludedForFailedTask(
+        "hostA", exec = "1", index = partition, failureReason = "testing")
+    }
+    healthTracker.updateExcludedForSuccessfulTaskSet(0, 0, taskSetExclude2.execToFailures)
+
+    val msg1 =
+      "Killing excluded executor id 1 since spark.excludeOnFailure.killExcludedExecutors is set." +
+      " (actually decommissioning)"
+
+    verify(allocationClientMock).decommissionExecutor(
+      "1", ExecutorDecommissionInfo(msg1), false)
+
+    val taskSetExclude3 = createTaskSetExcludelist(stageId = 1)
+    // Fail 4 tasks in one task set on executor 2, so that executor gets excluded for the whole
+    // application.  Since that's the second executor that is excluded on the same node, we also
+    // exclude that node.
+    (0 until 4).foreach { partition =>
+      taskSetExclude3.updateExcludedForFailedTask(
+        "hostA", exec = "2", index = partition, failureReason = "testing")
+    }
+    healthTracker.updateExcludedForSuccessfulTaskSet(0, 0, taskSetExclude3.execToFailures)
+
+    val msg2 =
+      "Killing excluded executor id 2 since spark.excludeOnFailure.killExcludedExecutors is set." +
+      " (actually decommissioning)"
+    verify(allocationClientMock).decommissionExecutor(
+      "2", ExecutorDecommissionInfo(msg2), false, false)
+    verify(allocationClientMock).decommissionExecutorsOnHost("hostA")
+  }
+
+  test("fetch failure excluding kills executors, configured by EXCLUDE_ON_FAILURE_KILL_ENABLED") {
+    val allocationClientMock = mock[ExecutorAllocationClient]
+    when(allocationClientMock.killExecutors(any(), any(), any(), any())).thenReturn(Seq("called"))
+    when(allocationClientMock.killExecutorsOnHost("hostA")).thenAnswer { (_: InvocationOnMock) =>
+      // To avoid a race between excluding and killing, it is important that the nodeExclude
+      // is updated before we ask the executor allocation client to kill all the executors
+      // on a particular host.
+      if (healthTracker.excludedNodeList().contains("hostA")) {
+        true
+      } else {
+        throw new IllegalStateException("hostA should be on the exclude")
+      }
+    }
+
+    conf.set(config.EXCLUDE_ON_FAILURE_FETCH_FAILURE_ENABLED, true)
+    healthTracker = new HealthTracker(listenerBusMock, conf, Some(allocationClientMock), clock)
+
+    // Disable auto-kill. Exclude an executor and make sure killExecutors is not called.
+    conf.set(config.EXCLUDE_ON_FAILURE_KILL_ENABLED, false)
+    healthTracker.updateExcludedForFetchFailure("hostA", exec = "1")
+
+    verify(allocationClientMock, never).killExecutors(any(), any(), any(), any())
+    verify(allocationClientMock, never).killExecutorsOnHost(any())
+
+    assert(healthTracker.nodeToExcludedExecs.contains("hostA"))
+    assert(healthTracker.nodeToExcludedExecs("hostA").contains("1"))
+
+    // Enable auto-kill. Exclude an executor and make sure killExecutors is called.
+    conf.set(config.EXCLUDE_ON_FAILURE_KILL_ENABLED, true)
+    healthTracker = new HealthTracker(listenerBusMock, conf, Some(allocationClientMock), clock)
+    clock.advance(1000)
+    healthTracker.updateExcludedForFetchFailure("hostA", exec = "1")
+
+    verify(allocationClientMock).killExecutors(Seq("1"), false, false, true)
+    verify(allocationClientMock, never).killExecutorsOnHost(any())
+
+    assert(healthTracker.executorIdToExcludedStatus.contains("1"))
+    assert(healthTracker.executorIdToExcludedStatus("1").node === "hostA")
+    assert(healthTracker.executorIdToExcludedStatus("1").expiryTime ===
+      1000 + healthTracker.EXCLUDE_ON_FAILURE_TIMEOUT_MILLIS)
+    assert(healthTracker.nextExpiryTime === 1000 + healthTracker.EXCLUDE_ON_FAILURE_TIMEOUT_MILLIS)
+    assert(healthTracker.nodeIdToExcludedExpiryTime.isEmpty)
+    assert(healthTracker.nodeToExcludedExecs.contains("hostA"))
+    assert(healthTracker.nodeToExcludedExecs("hostA").contains("1"))
+
+    // Enable external shuffle service to see if all the executors on this node will be killed.
+    conf.set(config.SHUFFLE_SERVICE_ENABLED, true)
+    clock.advance(1000)
+    healthTracker.updateExcludedForFetchFailure("hostA", exec = "2")
+
+    verify(allocationClientMock, never).killExecutors(Seq("2"), true, true)
+    verify(allocationClientMock).killExecutorsOnHost("hostA")
+
+    assert(healthTracker.nodeIdToExcludedExpiryTime.contains("hostA"))
+    assert(healthTracker.nodeIdToExcludedExpiryTime("hostA") ===
+      2000 + healthTracker.EXCLUDE_ON_FAILURE_TIMEOUT_MILLIS)
+    assert(healthTracker.nextExpiryTime === 1000 + healthTracker.EXCLUDE_ON_FAILURE_TIMEOUT_MILLIS)
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/scheduler/PoolSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/PoolSuite.scala
index d9de976c789d4..fa2c5eaee8baf 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/PoolSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/PoolSuite.scala
@@ -20,10 +20,14 @@ package org.apache.spark.scheduler
 import java.io.FileNotFoundException
 import java.util.Properties
 
-import org.apache.spark.{LocalSparkContext, SparkConf, SparkContext, SparkFunSuite}
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.util.VersionInfo
+
+import org.apache.spark.{LocalSparkContext, SparkConf, SparkContext, SparkFunSuite, TestUtils}
 import org.apache.spark.internal.config.SCHEDULER_ALLOCATION_FILE
 import org.apache.spark.resource.ResourceProfile
 import org.apache.spark.scheduler.SchedulingMode._
+import org.apache.spark.util.Utils
 
 /**
  * Tests that pools and the associated scheduling algorithms for FIFO and fair scheduling work
@@ -87,7 +91,7 @@ class PoolSuite extends SparkFunSuite with LocalSparkContext {
     val taskScheduler = new TaskSchedulerImpl(sc)
 
     val rootPool = new Pool("", FAIR, 0, 0)
-    val schedulableBuilder = new FairSchedulableBuilder(rootPool, sc.conf)
+    val schedulableBuilder = new FairSchedulableBuilder(rootPool, sc)
     schedulableBuilder.buildPools()
 
     // Ensure that the XML file was read in correctly.
@@ -185,9 +189,10 @@ class PoolSuite extends SparkFunSuite with LocalSparkContext {
     val xmlPath = getClass.getClassLoader.getResource("fairscheduler-with-invalid-data.xml")
       .getFile()
     val conf = new SparkConf().set(SCHEDULER_ALLOCATION_FILE, xmlPath)
+    sc = new SparkContext(LOCAL, APP_NAME, conf)
 
     val rootPool = new Pool("", FAIR, 0, 0)
-    val schedulableBuilder = new FairSchedulableBuilder(rootPool, conf)
+    val schedulableBuilder = new FairSchedulableBuilder(rootPool, sc)
     schedulableBuilder.buildPools()
 
     verifyPool(rootPool, schedulableBuilder.DEFAULT_POOL_NAME, 0, 1, FIFO)
@@ -239,7 +244,7 @@ class PoolSuite extends SparkFunSuite with LocalSparkContext {
     val taskScheduler = new TaskSchedulerImpl(sc)
 
     val rootPool = new Pool("", SchedulingMode.FAIR, initMinShare = 0, initWeight = 0)
-    val schedulableBuilder = new FairSchedulableBuilder(rootPool, sc.conf)
+    val schedulableBuilder = new FairSchedulableBuilder(rootPool, sc)
     schedulableBuilder.buildPools()
 
     // Submit a new task set manager with pool properties set to null. This should result
@@ -267,7 +272,7 @@ class PoolSuite extends SparkFunSuite with LocalSparkContext {
     val taskScheduler = new TaskSchedulerImpl(sc)
 
     val rootPool = new Pool("", SchedulingMode.FAIR, initMinShare = 0, initWeight = 0)
-    val schedulableBuilder = new FairSchedulableBuilder(rootPool, sc.conf)
+    val schedulableBuilder = new FairSchedulableBuilder(rootPool, sc)
     schedulableBuilder.buildPools()
 
     assert(rootPool.getSchedulableByName(TEST_POOL) === null)
@@ -302,7 +307,7 @@ class PoolSuite extends SparkFunSuite with LocalSparkContext {
     sc = new SparkContext(LOCAL, APP_NAME, conf)
 
     val rootPool = new Pool("", SchedulingMode.FAIR, 0, 0)
-    val schedulableBuilder = new FairSchedulableBuilder(rootPool, sc.conf)
+    val schedulableBuilder = new FairSchedulableBuilder(rootPool, sc)
     schedulableBuilder.buildPools()
 
     verifyPool(rootPool, schedulableBuilder.DEFAULT_POOL_NAME, 0, 1, FIFO)
@@ -317,7 +322,7 @@ class PoolSuite extends SparkFunSuite with LocalSparkContext {
     sc = new SparkContext(LOCAL, APP_NAME, conf)
 
     val rootPool = new Pool("", SchedulingMode.FAIR, 0, 0)
-    val schedulableBuilder = new FairSchedulableBuilder(rootPool, sc.conf)
+    val schedulableBuilder = new FairSchedulableBuilder(rootPool, sc)
     schedulableBuilder.buildPools()
 
     verifyPool(rootPool, schedulableBuilder.DEFAULT_POOL_NAME, 0, 1, FIFO)
@@ -332,12 +337,36 @@ class PoolSuite extends SparkFunSuite with LocalSparkContext {
     sc = new SparkContext(LOCAL, APP_NAME, conf)
 
     val rootPool = new Pool("", SchedulingMode.FAIR, 0, 0)
-    val schedulableBuilder = new FairSchedulableBuilder(rootPool, sc.conf)
+    val schedulableBuilder = new FairSchedulableBuilder(rootPool, sc)
     intercept[FileNotFoundException] {
       schedulableBuilder.buildPools()
     }
   }
 
+  test("SPARK-35083: Support remote scheduler pool file") {
+    val hadoopVersion = VersionInfo.getVersion.split("\\.")
+    // HttpFileSystem supported since hadoop 2.9
+    assume(hadoopVersion.head.toInt >= 3 ||
+      (hadoopVersion.head.toInt == 2 && hadoopVersion(1).toInt >= 9))
+
+    val xmlPath = new Path(
+      Utils.getSparkClassLoader.getResource("fairscheduler-with-valid-data.xml").getFile)
+    TestUtils.withHttpServer(xmlPath.getParent.toUri.getPath) { baseURL =>
+      val conf = new SparkConf().set(SCHEDULER_ALLOCATION_FILE,
+        baseURL + "fairscheduler-with-valid-data.xml")
+      sc = new SparkContext(LOCAL, APP_NAME, conf)
+
+      val rootPool = new Pool("", SchedulingMode.FAIR, 0, 0)
+      val schedulableBuilder = new FairSchedulableBuilder(rootPool, sc)
+      schedulableBuilder.buildPools()
+
+      verifyPool(rootPool, schedulableBuilder.DEFAULT_POOL_NAME, 0, 1, FIFO)
+      verifyPool(rootPool, "pool1", 3, 1, FIFO)
+      verifyPool(rootPool, "pool2", 4, 2, FAIR)
+      verifyPool(rootPool, "pool3", 2, 3, FAIR)
+    }
+  }
+
   private def verifyPool(rootPool: Pool, poolName: String, expectedInitMinShare: Int,
                          expectedInitWeight: Int, expectedSchedulingMode: SchedulingMode): Unit = {
     val selectedPool = rootPool.getSchedulableByName(poolName)
diff --git a/core/src/test/scala/org/apache/spark/scheduler/ReplayListenerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/ReplayListenerSuite.scala
index e6fbf9b09d43d..cb50c7c959754 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/ReplayListenerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/ReplayListenerSuite.scala
@@ -255,7 +255,7 @@ class ReplayListenerSuite extends SparkFunSuite with BeforeAndAfter with LocalSp
 
   /*
    * This is a dummy input stream that wraps another input stream but ends prematurely when
-   * reading at the specified position, throwing an EOFExeption.
+   * reading at the specified position, throwing an EOFException.
    */
   private class EarlyEOFInputStream(in: InputStream, failAtPos: Int) extends InputStream {
     private val countDown = new AtomicInteger(failAtPos)
diff --git a/core/src/test/scala/org/apache/spark/scheduler/SchedulerIntegrationSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/SchedulerIntegrationSuite.scala
index 0874163b0e946..88d2868b957f9 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/SchedulerIntegrationSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/SchedulerIntegrationSuite.scala
@@ -44,7 +44,7 @@ import org.apache.spark.util.{CallSite, ThreadUtils, Utils}
  * TaskSetManagers.
  *
  * Test cases are configured by providing a set of jobs to submit, and then simulating interaction
- * with spark's executors via a mocked backend (eg., task completion, task failure, executors
+ * with spark's executors via a mocked backend (e.g., task completion, task failure, executors
  * disconnecting, etc.).
  */
 abstract class SchedulerIntegrationSuite[T <: MockBackend: ClassTag] extends SparkFunSuite
@@ -372,7 +372,7 @@ private[spark] abstract class MockBackend(
 
   /**
    * Accessed by both scheduling and backend thread, so should be protected by this.
-   * Most likely the only thing that needs to be protected are the inidividual ExecutorTaskStatus,
+   * Most likely the only thing that needs to be protected are the individual ExecutorTaskStatus,
    * but for simplicity in this mock just lock the whole backend.
    */
   def executorIdToExecutor: Map[String, ExecutorTaskStatus]
@@ -535,8 +535,8 @@ class BasicSchedulerIntegrationSuite extends SchedulerIntegrationSuite[SingleCor
    */
   testScheduler("super simple job") {
     def runBackend(): Unit = {
-      val (taskDescripition, _) = backend.beginTask()
-      backend.taskSuccess(taskDescripition, 42)
+      val (taskDescription, _) = backend.beginTask()
+      backend.taskSuccess(taskDescription, 42)
     }
     withBackend(runBackend _) {
       val jobFuture = submit(new MockRDD(sc, 10, Nil), (0 until 10).toArray)
diff --git a/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala
index d4e8d63b54e5f..d72744c5cc348 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala
@@ -24,7 +24,8 @@ import scala.collection.JavaConverters._
 import scala.collection.mutable
 
 import org.mockito.Mockito
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark._
 import org.apache.spark.executor.TaskMetrics
@@ -570,9 +571,9 @@ class SparkListenerSuite extends SparkFunSuite with LocalSparkContext with Match
     }
   }
 
-  test("event queue size can be configued through spark conf") {
+  test("event queue size can be configured through spark conf") {
     // configure the shared queue size to be 1, event log queue size to be 2,
-    // and listner bus event queue size to be 5
+    // and listener bus event queue size to be 5
     val conf = new SparkConf(false)
       .set(LISTENER_BUS_EVENT_QUEUE_CAPACITY, 5)
       .set(s"spark.scheduler.listenerbus.eventqueue.${SHARED_QUEUE}.capacity", "1")
@@ -592,7 +593,7 @@ class SparkListenerSuite extends SparkFunSuite with LocalSparkContext with Match
     // check the size of shared queue is 1 as configured
     assert(bus.getQueueCapacity(SHARED_QUEUE) == Some(1))
     // no specific size of status queue is configured,
-    // it shoud use the LISTENER_BUS_EVENT_QUEUE_CAPACITY
+    // it should use the LISTENER_BUS_EVENT_QUEUE_CAPACITY
     assert(bus.getQueueCapacity(APP_STATUS_QUEUE) == Some(5))
     // check the size of event log queue is 5 as configured
     assert(bus.getQueueCapacity(EVENT_LOG_QUEUE) == Some(2))
@@ -621,7 +622,7 @@ class SparkListenerSuite extends SparkFunSuite with LocalSparkContext with Match
     }
 
     override def onStageCompleted(stage: SparkListenerStageCompleted): Unit = {
-      stageInfos(stage.stageInfo) = taskInfoMetrics
+      stageInfos(stage.stageInfo) = taskInfoMetrics.toSeq
       taskInfoMetrics = mutable.Buffer.empty
     }
   }
diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala
index 394a2a9fbf7cb..8a7ff9eb6dcd3 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala
@@ -70,7 +70,7 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark
       0, 0, taskBinary, rdd.partitions(0), Seq.empty, 0, new Properties,
       closureSerializer.serialize(TaskMetrics.registered).array())
     intercept[RuntimeException] {
-      task.run(0, 0, null, null)
+      task.run(0, 0, null, null, Option.empty)
     }
     assert(TaskContextSuite.completed)
   }
@@ -92,7 +92,7 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark
       0, 0, taskBinary, rdd.partitions(0), Seq.empty, 0, new Properties,
       closureSerializer.serialize(TaskMetrics.registered).array())
     intercept[RuntimeException] {
-      task.run(0, 0, null, null)
+      task.run(0, 0, null, null, Option.empty)
     }
     assert(TaskContextSuite.lastError.getMessage == "damn error")
   }
diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskDescriptionSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskDescriptionSuite.scala
index 5839532f11666..98b5bada27646 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/TaskDescriptionSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/TaskDescriptionSuite.scala
@@ -33,6 +33,10 @@ class TaskDescriptionSuite extends SparkFunSuite {
     originalFiles.put("fileUrl1", 1824)
     originalFiles.put("fileUrl2", 2)
 
+    val originalArchives = new HashMap[String, Long]()
+    originalArchives.put("archiveUrl1", 1824)
+    originalArchives.put("archiveUrl2", 2)
+
     val originalJars = new HashMap[String, Long]()
     originalJars.put("jar1", 3)
 
@@ -70,6 +74,7 @@ class TaskDescriptionSuite extends SparkFunSuite {
       partitionId = 1,
       originalFiles,
       originalJars,
+      originalArchives,
       originalProperties,
       originalResources,
       taskBuffer
@@ -87,6 +92,7 @@ class TaskDescriptionSuite extends SparkFunSuite {
     assert(decodedTaskDescription.partitionId === originalTaskDescription.partitionId)
     assert(decodedTaskDescription.addedFiles.equals(originalFiles))
     assert(decodedTaskDescription.addedJars.equals(originalJars))
+    assert(decodedTaskDescription.addedArchives.equals(originalArchives))
     assert(decodedTaskDescription.properties.equals(originalTaskDescription.properties))
     assert(equalResources(decodedTaskDescription.resources, originalTaskDescription.resources))
     assert(decodedTaskDescription.serializedTask.equals(taskBuffer))
diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskResultGetterSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskResultGetterSuite.scala
index 2efe6da5e986f..ea44a2d948ca9 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/TaskResultGetterSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/TaskResultGetterSuite.scala
@@ -103,7 +103,7 @@ private class MyTaskResultGetter(env: SparkEnv, scheduler: TaskSchedulerImpl)
   // DirectTaskResults that we receive from the executors
   private val _taskResults = new ArrayBuffer[DirectTaskResult[_]]
 
-  def taskResults: Seq[DirectTaskResult[_]] = _taskResults
+  def taskResults: Seq[DirectTaskResult[_]] = _taskResults.toSeq
 
   override def enqueueSuccessfulTask(tsm: TaskSetManager, tid: Long, data: ByteBuffer): Unit = {
     // work on a copy since the super class still needs to use the buffer
diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskSchedulerImplSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskSchedulerImplSuite.scala
index a8541cb863478..53dc14cc12a6b 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/TaskSchedulerImplSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/TaskSchedulerImplSuite.scala
@@ -34,7 +34,7 @@ import org.apache.spark.internal.config
 import org.apache.spark.resource.{ExecutorResourceRequests, ResourceProfile, TaskResourceRequests}
 import org.apache.spark.resource.ResourceUtils._
 import org.apache.spark.resource.TestResourceIDs._
-import org.apache.spark.util.ManualClock
+import org.apache.spark.util.{Clock, ManualClock}
 
 class FakeSchedulerBackend extends SchedulerBackend {
   def start(): Unit = {}
@@ -51,11 +51,11 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
   var failedTaskSetReason: String = null
   var failedTaskSet = false
 
-  var blacklist: BlacklistTracker = null
+  var healthTracker: HealthTracker = null
   var taskScheduler: TaskSchedulerImpl = null
   var dagScheduler: DAGScheduler = null
 
-  val stageToMockTaskSetBlacklist = new HashMap[Int, TaskSetBlacklist]()
+  val stageToMockTaskSetExcludelist = new HashMap[Int, TaskSetExcludelist]()
   val stageToMockTaskSetManager = new HashMap[Int, TaskSetManager]()
 
   override def beforeEach(): Unit = {
@@ -63,7 +63,7 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
     failedTaskSet = false
     failedTaskSetException = None
     failedTaskSetReason = null
-    stageToMockTaskSetBlacklist.clear()
+    stageToMockTaskSetExcludelist.clear()
     stageToMockTaskSetManager.clear()
   }
 
@@ -91,14 +91,14 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
     val conf = new SparkConf().setMaster(master).setAppName("TaskSchedulerImplSuite")
     confs.foreach { case (k, v) => conf.set(k, v) }
     sc = new SparkContext(conf)
-    taskScheduler = new TaskSchedulerImpl(sc)
+    taskScheduler = new TaskSchedulerImpl(sc, sc.conf.get(config.TASK_MAX_FAILURES))
     setupHelper()
   }
 
-  def setupSchedulerWithMockTaskSetBlacklist(confs: (String, String)*): TaskSchedulerImpl = {
-    blacklist = mock[BlacklistTracker]
+  def setupSchedulerWithMockTaskSetExcludelist(confs: (String, String)*): TaskSchedulerImpl = {
+    healthTracker = mock[HealthTracker]
     val conf = new SparkConf().setMaster("local").setAppName("TaskSchedulerImplSuite")
-    conf.set(config.BLACKLIST_ENABLED, true)
+    conf.set(config.EXCLUDE_ON_FAILURE_ENABLED, true)
     confs.foreach { case (k, v) => conf.set(k, v) }
 
     sc = new SparkContext(conf)
@@ -106,16 +106,16 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
       new TaskSchedulerImpl(sc, sc.conf.get(config.TASK_MAX_FAILURES)) {
         override def createTaskSetManager(taskSet: TaskSet, maxFailures: Int): TaskSetManager = {
           val tsm = super.createTaskSetManager(taskSet, maxFailures)
-          // we need to create a spied tsm just so we can set the TaskSetBlacklist
+          // we need to create a spied tsm just so we can set the TaskSetExcludelist
           val tsmSpy = spy(tsm)
-          val taskSetBlacklist = mock[TaskSetBlacklist]
-          when(tsmSpy.taskSetBlacklistHelperOpt).thenReturn(Some(taskSetBlacklist))
+          val taskSetExcludelist = mock[TaskSetExcludelist]
+          when(tsmSpy.taskSetExcludelistHelperOpt).thenReturn(Some(taskSetExcludelist))
           stageToMockTaskSetManager(taskSet.stageId) = tsmSpy
-          stageToMockTaskSetBlacklist(taskSet.stageId) = taskSetBlacklist
+          stageToMockTaskSetExcludelist(taskSet.stageId) = taskSetExcludelist
           tsmSpy
         }
 
-        override private[scheduler] lazy val blacklistTrackerOpt = Some(blacklist)
+        override private[scheduler] lazy val healthTrackerOpt = Some(healthTracker)
       }
     setupHelper()
   }
@@ -140,6 +140,33 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
     taskScheduler
   }
 
+  test("SPARK-32653: Decommissioned host/executor should be considered as inactive") {
+    val scheduler = setupScheduler()
+    val exec0 = "exec0"
+    val exec1 = "exec1"
+    val exec2 = "exec2"
+    val host0 = "host0"
+    val host1 = "host1"
+    val workerOffers = IndexedSeq(
+      WorkerOffer(exec0, host0, 1),
+      WorkerOffer(exec1, host0, 1),
+      WorkerOffer(exec2, host1, 1))
+    scheduler.resourceOffers(workerOffers)
+    assert(Seq(exec0, exec1, exec2).forall(scheduler.isExecutorAlive))
+    assert(Seq(host0, host1).forall(scheduler.hasExecutorsAliveOnHost))
+    assert(scheduler.getExecutorsAliveOnHost(host0)
+      .exists(s => s.contains(exec0) && s.contains(exec1)))
+    assert(scheduler.getExecutorsAliveOnHost(host1).exists(_.contains(exec2)))
+
+    scheduler.executorDecommission(exec1, ExecutorDecommissionInfo("test", None))
+    scheduler.executorDecommission(exec2, ExecutorDecommissionInfo("test", Some(host1)))
+
+    assert(scheduler.isExecutorAlive(exec0))
+    assert(!Seq(exec1, exec2).exists(scheduler.isExecutorAlive))
+    assert(scheduler.hasExecutorsAliveOnHost(host0))
+    assert(!scheduler.hasExecutorsAliveOnHost(host1))
+  }
+
   test("Scheduler does not always schedule tasks on the same workers") {
     val taskScheduler = setupScheduler()
     val numFreeCores = 1
@@ -196,14 +223,15 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
     assert(!failedTaskSet)
   }
 
-  private def setupTaskSchedulerForLocalityTests(clock: ManualClock): TaskSchedulerImpl = {
-    val conf = new SparkConf()
+  private def setupTaskSchedulerForLocalityTests(
+      clock: ManualClock,
+      conf: SparkConf = new SparkConf()): TaskSchedulerImpl = {
     sc = new SparkContext("local", "TaskSchedulerImplSuite", conf)
     val taskScheduler = new TaskSchedulerImpl(sc,
       sc.conf.get(config.TASK_MAX_FAILURES),
       clock = clock) {
       override def createTaskSetManager(taskSet: TaskSet, maxTaskFailures: Int): TaskSetManager = {
-        new TaskSetManager(this, taskSet, maxTaskFailures, blacklistTrackerOpt, clock)
+        new TaskSetManager(this, taskSet, maxTaskFailures, healthTrackerOpt, clock)
       }
       override def shuffleOffers(offers: IndexedSeq[WorkerOffer]): IndexedSeq[WorkerOffer] = {
         // Don't shuffle the offers around for this test.  Instead, we'll just pass in all
@@ -641,7 +669,7 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
     assert(0 === taskDescriptions2.length)
 
     // provide the actual loss reason for executor0
-    taskScheduler.executorLost("executor0", SlaveLost("oops"))
+    taskScheduler.executorLost("executor0", ExecutorProcessLost("oops"))
 
     // executor0's tasks should have failed now that the loss reason is known, so offering more
     // resources should make them be scheduled on the new executor.
@@ -651,22 +679,22 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
     assert(!failedTaskSet)
   }
 
-  test("scheduled tasks obey task and stage blacklists") {
-    taskScheduler = setupSchedulerWithMockTaskSetBlacklist()
+  test("scheduled tasks obey task and stage excludelist") {
+    taskScheduler = setupSchedulerWithMockTaskSetExcludelist()
     (0 to 2).foreach {stageId =>
       val taskSet = FakeTask.createTaskSet(numTasks = 2, stageId = stageId, stageAttemptId = 0)
       taskScheduler.submitTasks(taskSet)
     }
 
-    // Setup our mock blacklist:
-    // * stage 0 is blacklisted on node "host1"
-    // * stage 1 is blacklisted on executor "executor3"
-    // * stage 0, partition 0 is blacklisted on executor 0
-    // (mocked methods default to returning false, ie. no blacklisting)
-    when(stageToMockTaskSetBlacklist(0).isNodeBlacklistedForTaskSet("host1")).thenReturn(true)
-    when(stageToMockTaskSetBlacklist(1).isExecutorBlacklistedForTaskSet("executor3"))
+    // Setup our mock excludelist:
+    // * stage 0 is excluded on node "host1"
+    // * stage 1 is excluded on executor "executor3"
+    // * stage 0, partition 0 is excluded on executor 0
+    // (mocked methods default to returning false, ie. no excluding)
+    when(stageToMockTaskSetExcludelist(0).isNodeExcludedForTaskSet("host1")).thenReturn(true)
+    when(stageToMockTaskSetExcludelist(1).isExecutorExcludedForTaskSet("executor3"))
       .thenReturn(true)
-    when(stageToMockTaskSetBlacklist(0).isExecutorBlacklistedForTask("executor0", 0))
+    when(stageToMockTaskSetExcludelist(0).isExecutorExcludedForTask("executor0", 0))
       .thenReturn(true)
 
     val offers = IndexedSeq(
@@ -678,21 +706,21 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
     val firstTaskAttempts = taskScheduler.resourceOffers(offers).flatten
     // We should schedule all tasks.
     assert(firstTaskAttempts.size === 6)
-    // Whenever we schedule a task, we must consult the node and executor blacklist.  (The test
+    // Whenever we schedule a task, we must consult the node and executor excludelist.  (The test
     // doesn't check exactly what checks are made because the offers get shuffled.)
     (0 to 2).foreach { stageId =>
-      verify(stageToMockTaskSetBlacklist(stageId), atLeast(1))
-        .isNodeBlacklistedForTaskSet(anyString())
-      verify(stageToMockTaskSetBlacklist(stageId), atLeast(1))
-        .isExecutorBlacklistedForTaskSet(anyString())
+      verify(stageToMockTaskSetExcludelist(stageId), atLeast(1))
+        .isNodeExcludedForTaskSet(anyString())
+      verify(stageToMockTaskSetExcludelist(stageId), atLeast(1))
+        .isExecutorExcludedForTaskSet(anyString())
     }
 
     def tasksForStage(stageId: Int): Seq[TaskDescription] = {
       firstTaskAttempts.filter{_.name.contains(s"stage $stageId")}
     }
     tasksForStage(0).foreach { task =>
-      // executors 1 & 2 blacklisted for node
-      // executor 0 blacklisted just for partition 0
+      // executors 1 & 2 excluded for node
+      // executor 0 excluded just for partition 0
       if (task.index == 0) {
         assert(task.executorId === "executor3")
       } else {
@@ -700,12 +728,12 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
       }
     }
     tasksForStage(1).foreach { task =>
-      // executor 3 blacklisted
+      // executor 3 excluded
       assert("executor3" != task.executorId)
     }
     // no restrictions on stage 2
 
-    // Finally, just make sure that we can still complete tasks as usual with blacklisting
+    // Finally, just make sure that we can still complete tasks as usual with exclusion
     // in effect.  Finish each of the tasksets -- taskset 0 & 1 complete successfully, taskset 2
     // fails.
     (0 to 2).foreach { stageId =>
@@ -743,23 +771,23 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
     }
 
     // the tasksSets complete, so the tracker should be notified of the successful ones
-    verify(blacklist, times(1)).updateBlacklistForSuccessfulTaskSet(
+    verify(healthTracker, times(1)).updateExcludedForSuccessfulTaskSet(
       stageId = 0,
       stageAttemptId = 0,
-      failuresByExec = stageToMockTaskSetBlacklist(0).execToFailures)
-    verify(blacklist, times(1)).updateBlacklistForSuccessfulTaskSet(
+      failuresByExec = stageToMockTaskSetExcludelist(0).execToFailures)
+    verify(healthTracker, times(1)).updateExcludedForSuccessfulTaskSet(
       stageId = 1,
       stageAttemptId = 0,
-      failuresByExec = stageToMockTaskSetBlacklist(1).execToFailures)
+      failuresByExec = stageToMockTaskSetExcludelist(1).execToFailures)
     // but we shouldn't update for the failed taskset
-    verify(blacklist, never).updateBlacklistForSuccessfulTaskSet(
+    verify(healthTracker, never).updateExcludedForSuccessfulTaskSet(
       stageId = meq(2),
       stageAttemptId = anyInt(),
       failuresByExec = any())
   }
 
-  test("scheduled tasks obey node and executor blacklists") {
-    taskScheduler = setupSchedulerWithMockTaskSetBlacklist()
+  test("scheduled tasks obey node and executor excludelists") {
+    taskScheduler = setupSchedulerWithMockTaskSetExcludelist()
     (0 to 2).foreach { stageId =>
       val taskSet = FakeTask.createTaskSet(numTasks = 2, stageId = stageId, stageAttemptId = 0)
       taskScheduler.submitTasks(taskSet)
@@ -773,13 +801,13 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
       new WorkerOffer("executor4", "host3", 1)
     )
 
-    // setup our mock blacklist:
-    // host1, executor0 & executor3 are completely blacklisted
+    // setup our mock excludelist:
+    // host1, executor0 & executor3 are completely excluded
     // This covers everything *except* one core on executor4 / host3, so that everything is still
     // schedulable.
-    when(blacklist.isNodeBlacklisted("host1")).thenReturn(true)
-    when(blacklist.isExecutorBlacklisted("executor0")).thenReturn(true)
-    when(blacklist.isExecutorBlacklisted("executor3")).thenReturn(true)
+    when(healthTracker.isNodeExcluded("host1")).thenReturn(true)
+    when(healthTracker.isExecutorExcluded("executor0")).thenReturn(true)
+    when(healthTracker.isExecutorExcluded("executor3")).thenReturn(true)
 
     val stageToTsm = (0 to 2).map { stageId =>
       val tsm = taskScheduler.taskSetManagerForAttempt(stageId, 0).get
@@ -791,12 +819,12 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
     assert(firstTaskAttempts.size === 1)
     assert(firstTaskAttempts.head.executorId === "executor4")
     ('0' until '2').foreach { hostNum =>
-      verify(blacklist, atLeast(1)).isNodeBlacklisted("host" + hostNum)
+      verify(healthTracker, atLeast(1)).isNodeExcluded("host" + hostNum)
     }
   }
 
-  test("abort stage when all executors are blacklisted and we cannot acquire new executor") {
-    taskScheduler = setupSchedulerWithMockTaskSetBlacklist()
+  test("abort stage when all executors are excluded and we cannot acquire new executor") {
+    taskScheduler = setupSchedulerWithMockTaskSetExcludelist()
     val taskSet = FakeTask.createTaskSet(numTasks = 10)
     taskScheduler.submitTasks(taskSet)
     val tsm = stageToMockTaskSetManager(0)
@@ -809,11 +837,11 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
       WorkerOffer("executor3", "host1", 2)
     ))
 
-    // now say our blacklist updates to blacklist a bunch of resources, but *not* everything
-    when(blacklist.isNodeBlacklisted("host1")).thenReturn(true)
-    when(blacklist.isExecutorBlacklisted("executor0")).thenReturn(true)
+    // now say our health tracker updates to exclude a bunch of resources, but *not* everything
+    when(healthTracker.isNodeExcluded("host1")).thenReturn(true)
+    when(healthTracker.isExecutorExcluded("executor0")).thenReturn(true)
 
-    // make an offer on the blacklisted resources.  We won't schedule anything, but also won't
+    // make an offer on the excluded resources.  We won't schedule anything, but also won't
     // abort yet, since we know of other resources that work
     assert(taskScheduler.resourceOffers(IndexedSeq(
       WorkerOffer("executor0", "host0", 2),
@@ -821,9 +849,9 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
     )).flatten.size === 0)
     assert(!tsm.isZombie)
 
-    // now update the blacklist so that everything really is blacklisted
-    when(blacklist.isExecutorBlacklisted("executor1")).thenReturn(true)
-    when(blacklist.isExecutorBlacklisted("executor2")).thenReturn(true)
+    // now update the health tracker so that everything really is excluded
+    when(healthTracker.isExecutorExcluded("executor1")).thenReturn(true)
+    when(healthTracker.isExecutorExcluded("executor2")).thenReturn(true)
     assert(taskScheduler.resourceOffers(IndexedSeq(
       WorkerOffer("executor0", "host0", 2),
       WorkerOffer("executor3", "host1", 2)
@@ -832,10 +860,10 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
     verify(tsm).abort(anyString(), any())
   }
 
-  test("SPARK-22148 abort timer should kick in when task is completely blacklisted & no new " +
+  test("SPARK-22148 abort timer should kick in when task is completely excluded & no new " +
       "executor can be acquired") {
     // set the abort timer to fail immediately
-    taskScheduler = setupSchedulerWithMockTaskSetBlacklist(
+    taskScheduler = setupSchedulerWithMockTaskSetExcludelist(
       config.UNSCHEDULABLE_TASKSET_TIMEOUT.key -> "0")
 
     // We have only 1 task remaining with 1 executor
@@ -851,10 +879,10 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
     // Fail the running task
     val failedTask = firstTaskAttempts.find(_.executorId == "executor0").get
     failTask(failedTask.taskId, TaskState.FAILED, UnknownReason, tsm)
-    when(tsm.taskSetBlacklistHelperOpt.get.isExecutorBlacklistedForTask(
+    when(tsm.taskSetExcludelistHelperOpt.get.isExecutorExcludedForTask(
       "executor0", failedTask.index)).thenReturn(true)
 
-    // make an offer on the blacklisted executor.  We won't schedule anything, and set the abort
+    // make an offer on the excluded executor.  We won't schedule anything, and set the abort
     // timer to kick in immediately
     assert(taskScheduler.resourceOffers(IndexedSeq(
       WorkerOffer("executor0", "host0", 1)
@@ -867,7 +895,7 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
   }
 
   test("SPARK-22148 try to acquire a new executor when task is unschedulable with 1 executor") {
-    taskScheduler = setupSchedulerWithMockTaskSetBlacklist(
+    taskScheduler = setupSchedulerWithMockTaskSetExcludelist(
       config.UNSCHEDULABLE_TASKSET_TIMEOUT.key -> "10")
 
     // We have only 1 task remaining with 1 executor
@@ -883,11 +911,11 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
     // Fail the running task
     val failedTask = firstTaskAttempts.head
     failTask(failedTask.taskId, TaskState.FAILED, UnknownReason, tsm)
-    when(tsm.taskSetBlacklistHelperOpt.get.isExecutorBlacklistedForTask(
+    when(tsm.taskSetExcludelistHelperOpt.get.isExecutorExcludedForTask(
       "executor0", failedTask.index)).thenReturn(true)
 
-    // make an offer on the blacklisted executor.  We won't schedule anything, and set the abort
-    // timer to expire if no new executors could be acquired. We kill the existing idle blacklisted
+    // make an offer on the excluded executor.  We won't schedule anything, and set the abort
+    // timer to expire if no new executors could be acquired. We kill the existing idle excluded
     // executor and try to acquire a new one.
     assert(taskScheduler.resourceOffers(IndexedSeq(
       WorkerOffer("executor0", "host0", 1)
@@ -903,12 +931,12 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
     assert(!tsm.isZombie)
   }
 
-  // This is to test a scenario where we have two taskSets completely blacklisted and on acquiring
+  // This is to test a scenario where we have two taskSets completely excluded and on acquiring
   // a new executor we don't want the abort timer for the second taskSet to expire and abort the job
   test("SPARK-22148 abort timer should clear unschedulableTaskSetToExpiryTime for all TaskSets") {
-    taskScheduler = setupSchedulerWithMockTaskSetBlacklist()
+    taskScheduler = setupSchedulerWithMockTaskSetExcludelist()
 
-    // We have 2 taskSets with 1 task remaining in each with 1 executor completely blacklisted
+    // We have 2 taskSets with 1 task remaining in each with 1 executor completely excluded
     val taskSet1 = FakeTask.createTaskSet(numTasks = 1, stageId = 0, stageAttemptId = 0)
     taskScheduler.submitTasks(taskSet1)
     val taskSet2 = FakeTask.createTaskSet(numTasks = 1, stageId = 1, stageAttemptId = 0)
@@ -925,7 +953,7 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
     // Fail the running task
     val failedTask = firstTaskAttempts.head
     failTask(failedTask.taskId, TaskState.FAILED, UnknownReason, tsm)
-    when(tsm.taskSetBlacklistHelperOpt.get.isExecutorBlacklistedForTask(
+    when(tsm.taskSetExcludelistHelperOpt.get.isExecutorExcludedForTask(
       "executor0", failedTask.index)).thenReturn(true)
 
     // make an offer. We will schedule the task from the second taskSet. Since a task was scheduled
@@ -939,10 +967,10 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
     val tsm2 = stageToMockTaskSetManager(1)
     val failedTask2 = secondTaskAttempts.head
     failTask(failedTask2.taskId, TaskState.FAILED, UnknownReason, tsm2)
-    when(tsm2.taskSetBlacklistHelperOpt.get.isExecutorBlacklistedForTask(
+    when(tsm2.taskSetExcludelistHelperOpt.get.isExecutorExcludedForTask(
       "executor0", failedTask2.index)).thenReturn(true)
 
-    // make an offer on the blacklisted executor.  We won't schedule anything, and set the abort
+    // make an offer on the excluded executor.  We won't schedule anything, and set the abort
     // timer for taskSet1 and taskSet2
     assert(taskScheduler.resourceOffers(IndexedSeq(
       WorkerOffer("executor0", "host0", 1)
@@ -964,9 +992,9 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
 
   // this test is to check that we don't abort a taskSet which is not being scheduled on other
   // executors as it is waiting on locality timeout and not being aborted because it is still not
-  // completely blacklisted.
-  test("SPARK-22148 Ensure we don't abort the taskSet if we haven't been completely blacklisted") {
-    taskScheduler = setupSchedulerWithMockTaskSetBlacklist(
+  // completely excluded.
+  test("SPARK-22148 Ensure we don't abort the taskSet if we haven't been completely excluded") {
+    taskScheduler = setupSchedulerWithMockTaskSetExcludelist(
       config.UNSCHEDULABLE_TASKSET_TIMEOUT.key -> "0",
       // This is to avoid any potential flakiness in the test because of large pauses in jenkins
       config.LOCALITY_WAIT.key -> "30s"
@@ -980,14 +1008,14 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
     val tsm = stageToMockTaskSetManager(0)
 
     // submit an offer with one executor
-    var taskAttempts = taskScheduler.resourceOffers(IndexedSeq(
+    val taskAttempts = taskScheduler.resourceOffers(IndexedSeq(
       WorkerOffer("executor0", "host0", 1)
     )).flatten
 
     // Fail the running task
     val failedTask = taskAttempts.head
     failTask(failedTask.taskId, TaskState.FAILED, UnknownReason, tsm)
-    when(tsm.taskSetBlacklistHelperOpt.get.isExecutorBlacklistedForTask(
+    when(tsm.taskSetExcludelistHelperOpt.get.isExecutorExcludedForTask(
       "executor0", failedTask.index)).thenReturn(true)
 
     // make an offer but we won't schedule anything yet as scheduler locality is still PROCESS_LOCAL
@@ -1000,32 +1028,68 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
     assert(!tsm.isZombie)
   }
 
+  test("SPARK-31418 abort timer should kick in when task is completely excluded &" +
+    "allocation manager could not acquire a new executor before the timeout") {
+    // set the abort timer to fail immediately
+    taskScheduler = setupSchedulerWithMockTaskSetExcludelist(
+      config.UNSCHEDULABLE_TASKSET_TIMEOUT.key -> "0",
+      config.DYN_ALLOCATION_ENABLED.key -> "true")
+
+    // We have 2 tasks remaining with 1 executor
+    val taskSet = FakeTask.createTaskSet(numTasks = 2)
+    taskScheduler.submitTasks(taskSet)
+    val tsm = stageToMockTaskSetManager(0)
+
+    // submit an offer with one executor
+    taskScheduler.resourceOffers(IndexedSeq(WorkerOffer("executor0", "host0", 2))).flatten
+
+    // Fail the running task
+    failTask(0, TaskState.FAILED, UnknownReason, tsm)
+    when(tsm.taskSetExcludelistHelperOpt.get.isExecutorExcludedForTask(
+      "executor0", 0)).thenReturn(true)
+
+    // If the executor is busy, then dynamic allocation should kick in and try
+    // to acquire additional executors to schedule the excluded task
+    assert(taskScheduler.isExecutorBusy("executor0"))
+
+    // make an offer on the excluded executor.  We won't schedule anything, and set the abort
+    // timer to kick in immediately
+    assert(taskScheduler.resourceOffers(IndexedSeq(
+      WorkerOffer("executor0", "host0", 1)
+    )).flatten.size === 0)
+    // Wait for the abort timer to kick in. Even though we configure the timeout to be 0, there is a
+    // slight delay as the abort timer is launched in a separate thread.
+    eventually(timeout(500.milliseconds)) {
+      assert(tsm.isZombie)
+    }
+  }
+
   /**
-   * Helper for performance tests.  Takes the explicitly blacklisted nodes and executors; verifies
-   * that the blacklists are used efficiently to ensure scheduling is not O(numPendingTasks).
+   * Helper for performance tests.  Takes the explicitly excluded nodes and executors; verifies
+   * that the excluded are used efficiently to ensure scheduling is not O(numPendingTasks).
    * Creates 1 offer on executor[1-3].  Executor1 & 2 are on host1, executor3 is on host2.  Passed
    * in nodes and executors should be on that list.
    */
-  private def testBlacklistPerformance(
+  private def testExcludelistPerformance(
       testName: String,
-      nodeBlacklist: Seq[String],
-      execBlacklist: Seq[String]): Unit = {
+      nodeExcludelist: Seq[String],
+      execExcludelist: Seq[String]): Unit = {
     // Because scheduling involves shuffling the order of offers around, we run this test a few
     // times to cover more possibilities.  There are only 3 offers, which means 6 permutations,
     // so 10 iterations is pretty good.
     (0 until 10).foreach { testItr =>
       test(s"$testName: iteration $testItr") {
-        // When an executor or node is blacklisted, we want to make sure that we don't try
-        // scheduling each pending task, one by one, to discover they are all blacklisted.  This is
+        // When an executor or node is excluded, we want to make sure that we don't try
+        // scheduling each pending task, one by one, to discover they are all excluded.  This is
         // important for performance -- if we did check each task one-by-one, then responding to a
         // resource offer (which is usually O(1)-ish) would become O(numPendingTasks), which would
         // slow down scheduler throughput and slow down scheduling even on healthy executors.
         // Here, we check a proxy for the runtime -- we make sure the scheduling is short-circuited
-        // at the node or executor blacklist, so we never check the per-task blacklist.  We also
-        // make sure we don't check the node & executor blacklist for the entire taskset
+        // at the node or executor excludelist, so we never check the per-task excludelist.  We also
+        // make sure we don't check the node & executor excludelist for the entire taskset
         // O(numPendingTasks) times.
 
-        taskScheduler = setupSchedulerWithMockTaskSetBlacklist()
+        taskScheduler = setupSchedulerWithMockTaskSetExcludelist()
         // we schedule 500 tasks so we can clearly distinguish anything that is O(numPendingTasks)
         val taskSet = FakeTask.createTaskSet(numTasks = 500, stageId = 0, stageAttemptId = 0)
         taskScheduler.submitTasks(taskSet)
@@ -1035,91 +1099,92 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
           new WorkerOffer("executor2", "host1", 1),
           new WorkerOffer("executor3", "host2", 1)
         )
-        // We should check the node & exec blacklists, but only O(numOffers), not O(numPendingTasks)
-        // times.  In the worst case, after shuffling, we offer our blacklisted resource first, and
-        // then offer other resources which do get used.  The taskset blacklist is consulted
-        // repeatedly as we offer resources to the taskset -- each iteration either schedules
-        // something, or it terminates that locality level, so the maximum number of checks is
-        // numCores + numLocalityLevels
+        // We should check the node & exec excludelists, but only O(numOffers),
+        // not O(numPendingTasks) times.  In the worst case, after shuffling,
+        // we offer our excluded resource first, and then offer other resources
+        // which do get used.  The taskset excludelist is consulted repeatedly
+        // as we offer resources to the taskset -- each iteration either schedules
+        // something, or it terminates that locality level, so the maximum number of
+        // checks is numCores + numLocalityLevels
         val numCoresOnAllOffers = offers.map(_.cores).sum
         val numLocalityLevels = TaskLocality.values.size
-        val maxBlacklistChecks = numCoresOnAllOffers + numLocalityLevels
+        val maxExcludelistChecks = numCoresOnAllOffers + numLocalityLevels
 
-        // Setup the blacklist
-        nodeBlacklist.foreach { node =>
-          when(stageToMockTaskSetBlacklist(0).isNodeBlacklistedForTaskSet(node)).thenReturn(true)
+        // Setup the excludelist
+        nodeExcludelist.foreach { node =>
+          when(stageToMockTaskSetExcludelist(0).isNodeExcludedForTaskSet(node)).thenReturn(true)
         }
-        execBlacklist.foreach { exec =>
-          when(stageToMockTaskSetBlacklist(0).isExecutorBlacklistedForTaskSet(exec))
+        execExcludelist.foreach { exec =>
+          when(stageToMockTaskSetExcludelist(0).isExecutorExcludedForTaskSet(exec))
             .thenReturn(true)
         }
 
-        // Figure out which nodes have any effective blacklisting on them.  This means all nodes
-        // that are explicitly blacklisted, plus those that have *any* executors blacklisted.
-        val nodesForBlacklistedExecutors = offers.filter { offer =>
-          execBlacklist.contains(offer.executorId)
+        // Figure out which nodes have any effective exclusions on them.  This means all nodes
+        // that are explicitly excluded, plus those that have *any* executors excluded.
+        val nodesForExcludedExecutors = offers.filter { offer =>
+          execExcludelist.contains(offer.executorId)
         }.map(_.host).distinct
-        val nodesWithAnyBlacklisting = (nodeBlacklist ++ nodesForBlacklistedExecutors).toSet
-        // Similarly, figure out which executors have any blacklisting.  This means all executors
-        // that are explicitly blacklisted, plus all executors on nodes that are blacklisted.
-        val execsForBlacklistedNodes = offers.filter { offer =>
-          nodeBlacklist.contains(offer.host)
+        val nodesWithAnyExclusions = (nodeExcludelist ++ nodesForExcludedExecutors).toSet
+        // Similarly, figure out which executors have any exclusions.  This means all executors
+        // that are explicitly excluded, plus all executors on nodes that are excluded.
+        val execsForExcludedNodes = offers.filter { offer =>
+          nodeExcludelist.contains(offer.host)
         }.map(_.executorId).toSeq
-        val executorsWithAnyBlacklisting = (execBlacklist ++ execsForBlacklistedNodes).toSet
+        val executorsWithAnyExclusions = (execExcludelist ++ execsForExcludedNodes).toSet
 
         // Schedule a taskset, and make sure our test setup is correct -- we are able to schedule
-        // a task on all executors that aren't blacklisted (whether that executor is a explicitly
-        // blacklisted, or implicitly blacklisted via the node blacklist).
+        // a task on all executors that aren't excluded (whether that executor is a explicitly
+        // excluded, or implicitly excluded via the node excludeOnFailures).
         val firstTaskAttempts = taskScheduler.resourceOffers(offers).flatten
-        assert(firstTaskAttempts.size === offers.size - executorsWithAnyBlacklisting.size)
+        assert(firstTaskAttempts.size === offers.size - executorsWithAnyExclusions.size)
 
-        // Now check that we haven't made too many calls to any of the blacklist methods.
-        // We should be checking our node blacklist, but it should be within the bound we defined
+        // Now check that we haven't made too many calls to any of the excludelist methods.
+        // We should be checking our node excludelist, but it should be within the bound we defined
         // above.
-        verify(stageToMockTaskSetBlacklist(0), atMost(maxBlacklistChecks))
-          .isNodeBlacklistedForTaskSet(anyString())
-        // We shouldn't ever consult the per-task blacklist for the nodes that have been blacklisted
-        // for the entire taskset, since the taskset level blacklisting should prevent scheduling
+        verify(stageToMockTaskSetExcludelist(0), atMost(maxExcludelistChecks))
+          .isNodeExcludedForTaskSet(anyString())
+        // We shouldn't ever consult the per-task excludelist for the nodes that have been excluded
+        // for the entire taskset, since the taskset level exclusions should prevent scheduling
         // from ever looking at specific tasks.
-        nodesWithAnyBlacklisting.foreach { node =>
-          verify(stageToMockTaskSetBlacklist(0), never)
-            .isNodeBlacklistedForTask(meq(node), anyInt())
+        nodesWithAnyExclusions.foreach { node =>
+          verify(stageToMockTaskSetExcludelist(0), never)
+            .isNodeExcludedForTask(meq(node), anyInt())
         }
-        executorsWithAnyBlacklisting.foreach { exec =>
-          // We should be checking our executor blacklist, but it should be within the bound defined
-          // above.  Its possible that this will be significantly fewer calls, maybe even 0, if
-          // there is also a node-blacklist which takes effect first.  But this assert is all we
-          // need to avoid an O(numPendingTask) slowdown.
-          verify(stageToMockTaskSetBlacklist(0), atMost(maxBlacklistChecks))
-            .isExecutorBlacklistedForTaskSet(exec)
-          // We shouldn't ever consult the per-task blacklist for executors that have been
-          // blacklisted for the entire taskset, since the taskset level blacklisting should prevent
+        executorsWithAnyExclusions.foreach { exec =>
+          // We should be checking our executor excludelist, but it should be within the bound
+          // defined above. Its possible that this will be significantly fewer calls, maybe even
+          // 0, if there is also a node-excludelist which takes effect first. But this assert is
+          // all we need to avoid an O(numPendingTask) slowdown.
+          verify(stageToMockTaskSetExcludelist(0), atMost(maxExcludelistChecks))
+            .isExecutorExcludedForTaskSet(exec)
+          // We shouldn't ever consult the per-task excludelist for executors that have been
+          // excluded for the entire taskset, since the taskset level exclusions should prevent
           // scheduling from ever looking at specific tasks.
-          verify(stageToMockTaskSetBlacklist(0), never)
-            .isExecutorBlacklistedForTask(meq(exec), anyInt())
+          verify(stageToMockTaskSetExcludelist(0), never)
+            .isExecutorExcludedForTask(meq(exec), anyInt())
         }
       }
     }
   }
 
-  testBlacklistPerformance(
-    testName = "Blacklisted node for entire task set prevents per-task blacklist checks",
-    nodeBlacklist = Seq("host1"),
-    execBlacklist = Seq())
+  testExcludelistPerformance(
+    testName = "Excluded node for entire task set prevents per-task exclusion checks",
+    nodeExcludelist = Seq("host1"),
+    execExcludelist = Seq())
 
-  testBlacklistPerformance(
-    testName = "Blacklisted executor for entire task set prevents per-task blacklist checks",
-    nodeBlacklist = Seq(),
-    execBlacklist = Seq("executor3")
+  testExcludelistPerformance(
+    testName = "Excluded executor for entire task set prevents per-task exclusion checks",
+    nodeExcludelist = Seq(),
+    execExcludelist = Seq("executor3")
   )
 
   test("abort stage if executor loss results in unschedulability from previously failed tasks") {
-    // Make sure we can detect when a taskset becomes unschedulable from a blacklisting.  This
+    // Make sure we can detect when a taskset becomes unschedulable from excludeOnFailure.  This
     // test explores a particular corner case -- you may have one task fail, but still be
     // schedulable on another executor.  However, that executor may fail later on, leaving the
     // first task with no place to run.
     val taskScheduler = setupScheduler(
-      config.BLACKLIST_ENABLED.key -> "true"
+      config.EXCLUDE_ON_FAILURE_ENABLED.key -> "true"
     )
 
     val taskSet = FakeTask.createTaskSet(2)
@@ -1141,7 +1206,7 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
 
     // Now we fail our second executor.  The other task can still run on executor1, so make an offer
     // on that executor, and make sure that the other task (not the failed one) is assigned there.
-    taskScheduler.executorLost("executor1", SlaveLost("oops"))
+    taskScheduler.executorLost("executor1", ExecutorProcessLost("oops"))
     val nextTaskAttempts =
       taskScheduler.resourceOffers(IndexedSeq(new WorkerOffer("executor0", "host0", 1))).flatten
     // Note: Its OK if some future change makes this already realize the taskset has become
@@ -1152,7 +1217,7 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
     assert(nextTaskAttempts.head.index != failedTask.index)
 
     // Now we should definitely realize that our task set is unschedulable, because the only
-    // task left can't be scheduled on any executors due to the blacklist.
+    // task left can't be scheduled on any executors due to the excludelist.
     taskScheduler.resourceOffers(IndexedSeq(new WorkerOffer("executor0", "host0", 1)))
     sc.listenerBus.waitUntilEmpty(100000)
     assert(tsm.isZombie)
@@ -1160,11 +1225,11 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
     val idx = failedTask.index
     assert(failedTaskSetReason === s"""
       |Aborting $taskSet because task $idx (partition $idx)
-      |cannot run anywhere due to node and executor blacklist.
+      |cannot run anywhere due to node and executor excludeOnFailure.
       |Most recent failure:
-      |${tsm.taskSetBlacklistHelperOpt.get.getLatestFailureReason}
+      |${tsm.taskSetExcludelistHelperOpt.get.getLatestFailureReason}
       |
-      |Blacklisting behavior can be configured via spark.blacklist.*.
+      |ExcludeOnFailure behavior can be configured via spark.excludeOnFailure.*.
       |""".stripMargin)
   }
 
@@ -1175,7 +1240,7 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
     // available and not bail on the job
 
     val taskScheduler = setupScheduler(
-      config.BLACKLIST_ENABLED.key -> "true"
+      config.EXCLUDE_ON_FAILURE_ENABLED.key -> "true"
     )
 
     val taskSet = FakeTask.createTaskSet(2, (0 until 2).map { _ => Seq(TaskLocation("host0")) }: _*)
@@ -1208,7 +1273,6 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
   test("SPARK-16106 locality levels updated if executor added to existing host") {
     val taskScheduler = setupScheduler()
 
-    taskScheduler.resourceOffers(IndexedSeq(new WorkerOffer("executor0", "host0", 1)))
     taskScheduler.submitTasks(FakeTask.createTaskSet(2, stageId = 0, stageAttemptId = 0,
       (0 until 2).map { _ => Seq(TaskLocation("host0", "executor2")) }: _*
     ))
@@ -1244,7 +1308,7 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
     assert(taskScheduler.getExecutorsAliveOnHost("host1") === Some(Set("executor1", "executor3")))
   }
 
-  test("scheduler checks for executors that can be expired from blacklist") {
+  test("scheduler checks for executors that can be expired from excludeOnFailure") {
     taskScheduler = setupScheduler()
 
     taskScheduler.submitTasks(FakeTask.createTaskSet(1, stageId = 0, stageAttemptId = 0))
@@ -1252,7 +1316,7 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
       new WorkerOffer("executor0", "host0", 1)
     )).flatten
 
-    verify(blacklist).applyBlacklistTimeout()
+    verify(healthTracker).applyExcludeOnFailureTimeout()
   }
 
   test("if an executor is lost then the state for its running tasks is cleaned up (SPARK-18553)") {
@@ -1274,7 +1338,7 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
     assert(1 === taskDescriptions.length)
 
     // mark executor0 as dead
-    taskScheduler.executorLost("executor0", SlaveLost())
+    taskScheduler.executorLost("executor0", ExecutorProcessLost())
     assert(!taskScheduler.isExecutorAlive("executor0"))
     assert(!taskScheduler.hasExecutorsAliveOnHost("host0"))
     assert(taskScheduler.getExecutorsAliveOnHost("host0").isEmpty)
@@ -1338,7 +1402,7 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
         offers
       }
       override def createTaskSetManager(taskSet: TaskSet, maxTaskFailures: Int): TaskSetManager = {
-        new TaskSetManager(this, taskSet, maxTaskFailures, blacklistTrackerOpt, clock)
+        new TaskSetManager(this, taskSet, maxTaskFailures, healthTrackerOpt, clock)
       }
     }
     // Need to initialize a DAGScheduler for the taskScheduler to use for callbacks.
@@ -1378,7 +1442,7 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
     val clock = new ManualClock()
     val taskScheduler = new TaskSchedulerImpl(sc) {
       override def createTaskSetManager(taskSet: TaskSet, maxTaskFailures: Int): TaskSetManager = {
-        new TaskSetManager(this, taskSet, maxTaskFailures, blacklistTrackerOpt, clock)
+        new TaskSetManager(this, taskSet, maxTaskFailures, healthTrackerOpt, clock)
       }
     }
     // Need to initialize a DAGScheduler for the taskScheduler to use for callbacks.
@@ -1700,7 +1764,7 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
 
     taskScheduler.submitTasks(taskSet)
     // Launch tasks on executor that satisfies resource requirements.
-    var taskDescriptions = taskScheduler.resourceOffers(singleCoreWorkerOffers).flatten
+    val taskDescriptions = taskScheduler.resourceOffers(singleCoreWorkerOffers).flatten
     assert(3 === taskDescriptions.length)
     assert(!failedTaskSet)
     assert(ArrayBuffer("0") === taskDescriptions(0).resources.get(GPU).get.addresses)
@@ -1767,6 +1831,170 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
     assert(2 == taskDescriptions.head.resources(GPU).addresses.size)
   }
 
+  private def setupSchedulerForDecommissionTests(clock: Clock, numTasks: Int): TaskSchedulerImpl = {
+    // one task per host
+    val numHosts = numTasks
+    val conf = new SparkConf()
+      .setMaster(s"local[$numHosts]")
+      .setAppName("TaskSchedulerImplSuite")
+      .set(config.CPUS_PER_TASK.key, "1")
+    sc = new SparkContext(conf)
+    val maxTaskFailures = sc.conf.get(config.TASK_MAX_FAILURES)
+    taskScheduler = new TaskSchedulerImpl(sc, maxTaskFailures, clock = clock) {
+      override def createTaskSetManager(taskSet: TaskSet, maxFailures: Int): TaskSetManager = {
+        val tsm = super.createTaskSetManager(taskSet, maxFailures)
+        // we need to create a spied tsm so that we can see the copies running
+        val tsmSpy = spy(tsm)
+        stageToMockTaskSetManager(taskSet.stageId) = tsmSpy
+        tsmSpy
+      }
+    }
+    setupHelper()
+    // Spawn the tasks on different executors/hosts
+    taskScheduler.submitTasks(FakeTask.createTaskSet(numTasks))
+    for (i <- 0 until numTasks) {
+      val executorId = s"executor$i"
+      val taskDescriptions = taskScheduler.resourceOffers(IndexedSeq(WorkerOffer(
+         executorId, s"host$i", 1))).flatten
+      assert(taskDescriptions.size === 1)
+      assert(taskDescriptions(0).executorId == executorId)
+      assert(taskDescriptions(0).index === i)
+    }
+    taskScheduler
+  }
+
+  test("scheduler should keep the decommission state where host was decommissioned") {
+    val clock = new ManualClock(10000L)
+    val scheduler = setupSchedulerForDecommissionTests(clock, 2)
+    val decomTime = clock.getTimeMillis()
+    scheduler.executorDecommission("executor0", ExecutorDecommissionInfo("0", None))
+    scheduler.executorDecommission("executor1", ExecutorDecommissionInfo("1", Some("host1")))
+
+    assert(scheduler.getExecutorDecommissionState("executor0")
+      === Some(ExecutorDecommissionState(decomTime, None)))
+    assert(scheduler.getExecutorDecommissionState("executor1")
+      === Some(ExecutorDecommissionState(decomTime, Some("host1"))))
+    assert(scheduler.getExecutorDecommissionState("executor2").isEmpty)
+  }
+
+  test("test full decommissioning flow") {
+    val clock = new ManualClock(10000L)
+    val scheduler = setupSchedulerForDecommissionTests(clock, 2)
+    val manager = stageToMockTaskSetManager(0)
+    // The task started should be running.
+    assert(manager.copiesRunning.take(2) === Array(1, 1))
+
+    // executor 0 is decommissioned after loosing
+    assert(scheduler.getExecutorDecommissionState("executor0").isEmpty)
+    scheduler.executorLost("executor0", ExecutorExited(0, false, "normal"))
+    assert(scheduler.getExecutorDecommissionState("executor0").isEmpty)
+    scheduler.executorDecommission("executor0", ExecutorDecommissionInfo("", None))
+    assert(scheduler.getExecutorDecommissionState("executor0").isEmpty)
+
+    // 0th task just died above
+    assert(manager.copiesRunning.take(2) === Array(0, 1))
+
+    assert(scheduler.executorsPendingDecommission.isEmpty)
+    clock.advance(5000)
+
+    // executor1 hasn't been decommissioned yet
+    assert(scheduler.getExecutorDecommissionState("executor1").isEmpty)
+
+    // executor 1 is decommissioned before loosing
+    scheduler.executorDecommission("executor1", ExecutorDecommissionInfo("", None))
+    assert(scheduler.getExecutorDecommissionState("executor1").isDefined)
+    clock.advance(2000)
+
+    // executor1 is eventually lost
+    scheduler.executorLost("executor1", ExecutorExited(0, false, "normal"))
+    assert(scheduler.executorsPendingDecommission.isEmpty)
+    // So now both the tasks are no longer running
+    assert(manager.copiesRunning.take(2) === Array(0, 0))
+    clock.advance(2000)
+
+    // Now give it some resources and both tasks should be rerun
+    val taskDescriptions = taskScheduler.resourceOffers(IndexedSeq(
+      WorkerOffer("executor2", "host2", 1), WorkerOffer("executor3", "host3", 1))).flatten
+    assert(taskDescriptions.size === 2)
+    assert(taskDescriptions.map(_.index).sorted == Seq(0, 1))
+    assert(manager.copiesRunning.take(2) === Array(1, 1))
+  }
+
+  test("SPARK-24818: test delay scheduling for barrier TaskSetManager") {
+    val clock = new ManualClock()
+    val conf = new SparkConf().set(config.LEGACY_LOCALITY_WAIT_RESET, false)
+    val sched = setupTaskSchedulerForLocalityTests(clock, conf)
+
+    // Call resourceOffers() first, so executor-0 can be used
+    // to calculate the locality levels of the TaskSetManager later
+    sched.resourceOffers(Seq(WorkerOffer("executor-0", "host1", 1, Some("host1"))).toIndexedSeq)
+
+    val prefLocs = Seq(TaskLocation("host1", "executor-0"))
+    val barrierTaskSet = FakeTask.createBarrierTaskSet(1, prefLocs)
+    sched.submitTasks(barrierTaskSet)
+
+    val tsm = sched.taskSetManagerForAttempt(0, 0).get
+    assert(tsm.myLocalityLevels ===
+      Array(TaskLocality.PROCESS_LOCAL, TaskLocality.NODE_LOCAL, TaskLocality.ANY))
+    val offers = Seq(WorkerOffer("executor-1", "host1", 1, Some("host1"))).toIndexedSeq
+    var tasks = sched.resourceOffers(offers).flatten
+    // The TaskSetManager prefers executor-0 for the PROCESS_LOCAL location but there's no
+    // available offer of executor-0 in this round, so task scheduling will be delayed first.
+    assert(tasks.length === 0)
+    // Advance the clock so the TaskSetManager can move to next locality level(NODE_LOCAL)
+    clock.advance(4000)
+    tasks = sched.resourceOffers(offers).flatten
+    assert(tasks.length === 1)
+    assert(tsm.taskInfos(tasks.head.taskId).taskLocality === TaskLocality.NODE_LOCAL)
+  }
+
+  test("SPARK-24818: test resource revert of barrier TaskSetManager") {
+    val clock = new ManualClock()
+    val conf = new SparkConf().set(config.LEGACY_LOCALITY_WAIT_RESET, false)
+    val sched = setupTaskSchedulerForLocalityTests(clock, conf)
+
+    // Call resourceOffers() first, so executors can be used
+    // to calculate the locality levels of the TaskSetManager later
+    sched.resourceOffers(Seq(WorkerOffer("executor-0", "host1", 1, Some("host1"))).toIndexedSeq)
+
+    val barrierTaskSet =
+      FakeTask.createBarrierTaskSet(2, 0, 0, 0, 0,
+        Seq(TaskLocation("host1", "executor-0")), Seq(TaskLocation("host1", "executor-1")))
+    val normalTaskSet = FakeTask.createTaskSet(2, 1, 0, 0, 0)
+
+    // Submit barrier task set first, so we can schedule it before the normal task set in order to
+    // test the resource revert behaviour of the barrier TaskSetManager
+    sched.submitTasks(barrierTaskSet)
+    sched.submitTasks(normalTaskSet)
+
+    val barrierTSM = sched.taskSetManagerForAttempt(0, 0).get
+    val normalTSM = sched.taskSetManagerForAttempt(1, 0).get
+    assert(barrierTSM.myLocalityLevels ===
+      Array(TaskLocality.PROCESS_LOCAL, TaskLocality.NODE_LOCAL, TaskLocality.ANY))
+    assert(normalTSM.myLocalityLevels ===  Array(TaskLocality.NO_PREF, TaskLocality.ANY))
+
+    // The barrier TaskSetManager can not launch all tasks because of delay scheduling.
+    // So it will revert assigned resources and let the normal TaskSetManager to schedule first.
+    var tasks = sched.resourceOffers(
+      Seq(WorkerOffer("executor-0", "host1", 1, Some("host1")),
+        WorkerOffer("executor-2", "host1", 1, Some("host1"))).toIndexedSeq).flatten
+    assert(tasks.length === 2)
+    var taskId = tasks.head.taskId
+    assert(!barrierTSM.runningTasksSet.contains(taskId))
+    assert(normalTSM.runningTasksSet.contains(taskId))
+
+    // Advance the clock so the TaskSetManager can move to next locality level(NODE_LOCAL)
+    // and launch all tasks.
+    clock.advance(4000)
+    tasks = sched.resourceOffers(
+      Seq(WorkerOffer("executor-0", "host1", 1, Some("host1")),
+        WorkerOffer("executor-2", "host1", 1, Some("host1"))).toIndexedSeq).flatten
+    assert(tasks.length === 2)
+    taskId = tasks.head.taskId
+    assert(barrierTSM.runningTasksSet.contains(taskId))
+    assert(!normalTSM.runningTasksSet.contains(taskId))
+  }
+
   /**
    * Used by tests to simulate a task failure. This calls the failure handler explicitly, to ensure
    * that all the state is updated when this method returns. Otherwise, there's no way to know when
diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskSetBlacklistSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskSetBlacklistSuite.scala
deleted file mode 100644
index ed97a4c206ca3..0000000000000
--- a/core/src/test/scala/org/apache/spark/scheduler/TaskSetBlacklistSuite.scala
+++ /dev/null
@@ -1,287 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.scheduler
-
-import org.mockito.ArgumentMatchers.isA
-import org.mockito.Mockito.{never, verify}
-import org.scalatest.BeforeAndAfterEach
-import org.scalatestplus.mockito.MockitoSugar
-
-import org.apache.spark.{SparkConf, SparkFunSuite}
-import org.apache.spark.internal.config
-import org.apache.spark.util.ManualClock
-
-class TaskSetBlacklistSuite extends SparkFunSuite with BeforeAndAfterEach with MockitoSugar {
-
-  private var listenerBusMock: LiveListenerBus = _
-
-  override def beforeEach(): Unit = {
-    listenerBusMock = mock[LiveListenerBus]
-    super.beforeEach()
-  }
-
-  test("Blacklisting tasks, executors, and nodes") {
-    val conf = new SparkConf().setAppName("test").setMaster("local")
-      .set(config.BLACKLIST_ENABLED.key, "true")
-    val clock = new ManualClock
-    val attemptId = 0
-    val taskSetBlacklist = new TaskSetBlacklist(
-      listenerBusMock, conf, stageId = 0, stageAttemptId = attemptId, clock = clock)
-
-    clock.setTime(0)
-    // We will mark task 0 & 1 failed on both executor 1 & 2.
-    // We should blacklist all executors on that host, for all tasks for the stage.  Note the API
-    // will return false for isExecutorBacklistedForTaskSet even when the node is blacklisted, so
-    // the executor is implicitly blacklisted (this makes sense with how the scheduler uses the
-    // blacklist)
-
-    // First, mark task 0 as failed on exec1.
-    // task 0 should be blacklisted on exec1, and nowhere else
-    taskSetBlacklist.updateBlacklistForFailedTask(
-      "hostA", exec = "exec1", index = 0, failureReason = "testing")
-    for {
-      executor <- (1 to 4).map(_.toString)
-      index <- 0 until 10
-    } {
-      val shouldBeBlacklisted = (executor == "exec1" && index == 0)
-      assert(taskSetBlacklist.isExecutorBlacklistedForTask(executor, index) === shouldBeBlacklisted)
-    }
-
-    assert(!taskSetBlacklist.isExecutorBlacklistedForTaskSet("exec1"))
-    verify(listenerBusMock, never())
-      .post(isA(classOf[SparkListenerExecutorBlacklistedForStage]))
-
-    assert(!taskSetBlacklist.isNodeBlacklistedForTaskSet("hostA"))
-    verify(listenerBusMock, never())
-      .post(isA(classOf[SparkListenerNodeBlacklistedForStage]))
-
-    // Mark task 1 failed on exec1 -- this pushes the executor into the blacklist
-    taskSetBlacklist.updateBlacklistForFailedTask(
-      "hostA", exec = "exec1", index = 1, failureReason = "testing")
-
-    assert(taskSetBlacklist.isExecutorBlacklistedForTaskSet("exec1"))
-    verify(listenerBusMock).post(
-      SparkListenerExecutorBlacklistedForStage(0, "exec1", 2, 0, attemptId))
-
-    assert(!taskSetBlacklist.isNodeBlacklistedForTaskSet("hostA"))
-    verify(listenerBusMock, never())
-      .post(isA(classOf[SparkListenerNodeBlacklistedForStage]))
-
-    // Mark one task as failed on exec2 -- not enough for any further blacklisting yet.
-    taskSetBlacklist.updateBlacklistForFailedTask(
-      "hostA", exec = "exec2", index = 0, failureReason = "testing")
-    assert(taskSetBlacklist.isExecutorBlacklistedForTaskSet("exec1"))
-
-    assert(!taskSetBlacklist.isExecutorBlacklistedForTaskSet("exec2"))
-
-    assert(!taskSetBlacklist.isNodeBlacklistedForTaskSet("hostA"))
-    verify(listenerBusMock, never())
-      .post(isA(classOf[SparkListenerNodeBlacklistedForStage]))
-
-    // Mark another task as failed on exec2 -- now we blacklist exec2, which also leads to
-    // blacklisting the entire node.
-    taskSetBlacklist.updateBlacklistForFailedTask(
-      "hostA", exec = "exec2", index = 1, failureReason = "testing")
-
-    assert(taskSetBlacklist.isExecutorBlacklistedForTaskSet("exec1"))
-
-    assert(taskSetBlacklist.isExecutorBlacklistedForTaskSet("exec2"))
-    verify(listenerBusMock).post(
-      SparkListenerExecutorBlacklistedForStage(0, "exec2", 2, 0, attemptId))
-
-    assert(taskSetBlacklist.isNodeBlacklistedForTaskSet("hostA"))
-    verify(listenerBusMock).post(
-      SparkListenerNodeBlacklistedForStage(0, "hostA", 2, 0, attemptId))
-
-    // Make sure the blacklist has the correct per-task && per-executor responses, over a wider
-    // range of inputs.
-    for {
-      executor <- (1 to 4).map(e => s"exec$e")
-      index <- 0 until 10
-    } {
-      withClue(s"exec = $executor; index = $index") {
-        val badExec = (executor == "exec1" || executor == "exec2")
-        val badIndex = (index == 0 || index == 1)
-        assert(
-          // this ignores whether the executor is blacklisted entirely for the taskset -- that is
-          // intentional, it keeps it fast and is sufficient for usage in the scheduler.
-          taskSetBlacklist.isExecutorBlacklistedForTask(executor, index) === (badExec && badIndex))
-        assert(taskSetBlacklist.isExecutorBlacklistedForTaskSet(executor) === badExec)
-        if (badExec) {
-          verify(listenerBusMock).post(
-            SparkListenerExecutorBlacklistedForStage(0, executor, 2, 0, attemptId))
-        }
-      }
-    }
-    assert(taskSetBlacklist.isNodeBlacklistedForTaskSet("hostA"))
-    val execToFailures = taskSetBlacklist.execToFailures
-    assert(execToFailures.keySet === Set("exec1", "exec2"))
-
-    Seq("exec1", "exec2").foreach { exec =>
-      assert(
-        execToFailures(exec).taskToFailureCountAndFailureTime === Map(
-          0 -> ((1, 0)),
-          1 -> ((1, 0))
-        )
-      )
-    }
-  }
-
-  test("multiple attempts for the same task count once") {
-    // Make sure that for blacklisting tasks, the node counts task attempts, not executors.  But for
-    // stage-level blacklisting, we count unique tasks.  The reason for this difference is, with
-    // task-attempt blacklisting, we want to make it easy to configure so that you ensure a node
-    // is blacklisted before the taskset is completely aborted because of spark.task.maxFailures.
-    // But with stage-blacklisting, we want to make sure we're not just counting one bad task
-    // that has failed many times.
-
-    val conf = new SparkConf().setMaster("local").setAppName("test")
-      .set(config.MAX_TASK_ATTEMPTS_PER_EXECUTOR, 2)
-      .set(config.MAX_TASK_ATTEMPTS_PER_NODE, 3)
-      .set(config.MAX_FAILURES_PER_EXEC_STAGE, 2)
-      .set(config.MAX_FAILED_EXEC_PER_NODE_STAGE, 3)
-    val clock = new ManualClock
-
-    val attemptId = 0
-    val taskSetBlacklist = new TaskSetBlacklist(
-      listenerBusMock, conf, stageId = 0, stageAttemptId = attemptId, clock = clock)
-
-    var time = 0
-    clock.setTime(time)
-    // Fail a task twice on hostA, exec:1
-    taskSetBlacklist.updateBlacklistForFailedTask(
-      "hostA", exec = "1", index = 0, failureReason = "testing")
-    taskSetBlacklist.updateBlacklistForFailedTask(
-      "hostA", exec = "1", index = 0, failureReason = "testing")
-    assert(taskSetBlacklist.isExecutorBlacklistedForTask("1", 0))
-    assert(!taskSetBlacklist.isNodeBlacklistedForTask("hostA", 0))
-
-    assert(!taskSetBlacklist.isExecutorBlacklistedForTaskSet("1"))
-    verify(listenerBusMock, never()).post(
-      SparkListenerExecutorBlacklistedForStage(time, "1", 2, 0, attemptId))
-
-    assert(!taskSetBlacklist.isNodeBlacklistedForTaskSet("hostA"))
-    verify(listenerBusMock, never()).post(
-      SparkListenerNodeBlacklistedForStage(time, "hostA", 2, 0, attemptId))
-
-    // Fail the same task once more on hostA, exec:2
-    time += 1
-    clock.setTime(time)
-    taskSetBlacklist.updateBlacklistForFailedTask(
-      "hostA", exec = "2", index = 0, failureReason = "testing")
-    assert(taskSetBlacklist.isNodeBlacklistedForTask("hostA", 0))
-
-    assert(!taskSetBlacklist.isExecutorBlacklistedForTaskSet("2"))
-    verify(listenerBusMock, never()).post(
-      SparkListenerExecutorBlacklistedForStage(time, "2", 2, 0, attemptId))
-
-    assert(!taskSetBlacklist.isNodeBlacklistedForTaskSet("hostA"))
-    verify(listenerBusMock, never()).post(
-      SparkListenerNodeBlacklistedForStage(time, "hostA", 2, 0, attemptId))
-
-    // Fail another task on hostA, exec:1.  Now that executor has failures on two different tasks,
-    // so its blacklisted
-    time += 1
-    clock.setTime(time)
-    taskSetBlacklist.updateBlacklistForFailedTask(
-      "hostA", exec = "1", index = 1, failureReason = "testing")
-
-    assert(taskSetBlacklist.isExecutorBlacklistedForTaskSet("1"))
-    verify(listenerBusMock)
-      .post(SparkListenerExecutorBlacklistedForStage(time, "1", 2, 0, attemptId))
-
-    assert(!taskSetBlacklist.isNodeBlacklistedForTaskSet("hostA"))
-    verify(listenerBusMock, never())
-      .post(isA(classOf[SparkListenerNodeBlacklistedForStage]))
-
-    // Fail a third task on hostA, exec:2, so that exec is blacklisted for the whole task set
-    time += 1
-    clock.setTime(time)
-    taskSetBlacklist.updateBlacklistForFailedTask(
-      "hostA", exec = "2", index = 2, failureReason = "testing")
-
-    assert(taskSetBlacklist.isExecutorBlacklistedForTaskSet("2"))
-    verify(listenerBusMock)
-      .post(SparkListenerExecutorBlacklistedForStage(time, "2", 2, 0, attemptId))
-
-    assert(!taskSetBlacklist.isNodeBlacklistedForTaskSet("hostA"))
-    verify(listenerBusMock, never())
-      .post(isA(classOf[SparkListenerNodeBlacklistedForStage]))
-
-    // Fail a fourth & fifth task on hostA, exec:3.  Now we've got three executors that are
-    // blacklisted for the taskset, so blacklist the whole node.
-    time += 1
-    clock.setTime(time)
-    taskSetBlacklist.updateBlacklistForFailedTask(
-      "hostA", exec = "3", index = 3, failureReason = "testing")
-    taskSetBlacklist.updateBlacklistForFailedTask(
-      "hostA", exec = "3", index = 4, failureReason = "testing")
-
-    assert(taskSetBlacklist.isExecutorBlacklistedForTaskSet("3"))
-    verify(listenerBusMock)
-      .post(SparkListenerExecutorBlacklistedForStage(time, "3", 2, 0, attemptId))
-
-    assert(taskSetBlacklist.isNodeBlacklistedForTaskSet("hostA"))
-    verify(listenerBusMock).post(
-      SparkListenerNodeBlacklistedForStage(time, "hostA", 3, 0, attemptId))
-  }
-
-  test("only blacklist nodes for the task set when all the blacklisted executors are all on " +
-    "same host") {
-    // we blacklist executors on two different hosts within one taskSet -- make sure that doesn't
-    // lead to any node blacklisting
-    val conf = new SparkConf().setAppName("test").setMaster("local")
-      .set(config.BLACKLIST_ENABLED.key, "true")
-    val clock = new ManualClock
-
-    val attemptId = 0
-    val taskSetBlacklist = new TaskSetBlacklist(
-      listenerBusMock, conf, stageId = 0, stageAttemptId = attemptId, clock = clock)
-    var time = 0
-    clock.setTime(time)
-    taskSetBlacklist.updateBlacklistForFailedTask(
-      "hostA", exec = "1", index = 0, failureReason = "testing")
-    taskSetBlacklist.updateBlacklistForFailedTask(
-      "hostA", exec = "1", index = 1, failureReason = "testing")
-
-    assert(taskSetBlacklist.isExecutorBlacklistedForTaskSet("1"))
-    verify(listenerBusMock)
-      .post(SparkListenerExecutorBlacklistedForStage(time, "1", 2, 0, attemptId))
-
-    assert(!taskSetBlacklist.isNodeBlacklistedForTaskSet("hostA"))
-    verify(listenerBusMock, never()).post(
-      SparkListenerNodeBlacklistedForStage(time, "hostA", 2, 0, attemptId))
-
-    time += 1
-    clock.setTime(time)
-    taskSetBlacklist.updateBlacklistForFailedTask(
-      "hostB", exec = "2", index = 0, failureReason = "testing")
-    taskSetBlacklist.updateBlacklistForFailedTask(
-      "hostB", exec = "2", index = 1, failureReason = "testing")
-    assert(taskSetBlacklist.isExecutorBlacklistedForTaskSet("1"))
-
-    assert(taskSetBlacklist.isExecutorBlacklistedForTaskSet("2"))
-    verify(listenerBusMock)
-      .post(SparkListenerExecutorBlacklistedForStage(time, "2", 2, 0, attemptId))
-
-    assert(!taskSetBlacklist.isNodeBlacklistedForTaskSet("hostA"))
-    assert(!taskSetBlacklist.isNodeBlacklistedForTaskSet("hostB"))
-    verify(listenerBusMock, never())
-      .post(isA(classOf[SparkListenerNodeBlacklistedForStage]))
-  }
-
-}
diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskSetExcludelistSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskSetExcludelistSuite.scala
new file mode 100644
index 0000000000000..d20768d7cd12b
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/scheduler/TaskSetExcludelistSuite.scala
@@ -0,0 +1,310 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.scheduler
+
+import org.mockito.ArgumentMatchers.isA
+import org.mockito.Mockito.{never, verify}
+import org.scalatest.BeforeAndAfterEach
+import org.scalatestplus.mockito.MockitoSugar
+
+import org.apache.spark.{SparkConf, SparkFunSuite}
+import org.apache.spark.internal.config
+import org.apache.spark.util.ManualClock
+
+class TaskSetExcludelistSuite extends SparkFunSuite with BeforeAndAfterEach with MockitoSugar {
+
+  private var listenerBusMock: LiveListenerBus = _
+
+  override def beforeEach(): Unit = {
+    listenerBusMock = mock[LiveListenerBus]
+    super.beforeEach()
+  }
+
+  test("Excluding tasks, executors, and nodes") {
+    val conf = new SparkConf().setAppName("test").setMaster("local")
+      .set(config.EXCLUDE_ON_FAILURE_ENABLED.key, "true")
+    val clock = new ManualClock
+    val attemptId = 0
+    val taskSetExcludelist = new TaskSetExcludelist(
+      listenerBusMock, conf, stageId = 0, stageAttemptId = attemptId, clock = clock)
+
+    clock.setTime(0)
+    // We will mark task 0 & 1 failed on both executor 1 & 2.
+    // We should exclude all executors on that host, for all tasks for the stage.  Note the API
+    // will return false for isExecutorBacklistedForTaskSet even when the node is excluded, so
+    // the executor is implicitly excluded (this makes sense with how the scheduler uses the
+    // exclude)
+
+    // First, mark task 0 as failed on exec1.
+    // task 0 should be excluded on exec1, and nowhere else
+    taskSetExcludelist.updateExcludedForFailedTask(
+      "hostA", exec = "exec1", index = 0, failureReason = "testing")
+    for {
+      executor <- (1 to 4).map(_.toString)
+      index <- 0 until 10
+    } {
+      val shouldBeExcluded = (executor == "exec1" && index == 0)
+      assert(taskSetExcludelist.isExecutorExcludedForTask(executor, index) === shouldBeExcluded)
+    }
+
+    assert(!taskSetExcludelist.isExecutorExcludedForTaskSet("exec1"))
+    verify(listenerBusMock, never())
+      .post(isA(classOf[SparkListenerExecutorExcludedForStage]))
+    verify(listenerBusMock, never())
+      .post(isA(classOf[SparkListenerExecutorBlacklistedForStage]))
+
+    assert(!taskSetExcludelist.isNodeExcludedForTaskSet("hostA"))
+    verify(listenerBusMock, never())
+      .post(isA(classOf[SparkListenerNodeExcludedForStage]))
+
+    // Mark task 1 failed on exec1 -- this pushes the executor into the exclude
+    taskSetExcludelist.updateExcludedForFailedTask(
+      "hostA", exec = "exec1", index = 1, failureReason = "testing")
+
+    assert(taskSetExcludelist.isExecutorExcludedForTaskSet("exec1"))
+    verify(listenerBusMock).post(
+      SparkListenerExecutorExcludedForStage(0, "exec1", 2, 0, attemptId))
+    verify(listenerBusMock).post(
+      SparkListenerExecutorBlacklistedForStage(0, "exec1", 2, 0, attemptId))
+
+
+    assert(!taskSetExcludelist.isNodeExcludedForTaskSet("hostA"))
+    verify(listenerBusMock, never())
+      .post(isA(classOf[SparkListenerNodeExcludedForStage]))
+    verify(listenerBusMock, never())
+      .post(isA(classOf[SparkListenerNodeBlacklistedForStage]))
+
+    // Mark one task as failed on exec2 -- not enough for any further excluding yet.
+    taskSetExcludelist.updateExcludedForFailedTask(
+      "hostA", exec = "exec2", index = 0, failureReason = "testing")
+    assert(taskSetExcludelist.isExecutorExcludedForTaskSet("exec1"))
+
+    assert(!taskSetExcludelist.isExecutorExcludedForTaskSet("exec2"))
+
+    assert(!taskSetExcludelist.isNodeExcludedForTaskSet("hostA"))
+    verify(listenerBusMock, never())
+      .post(isA(classOf[SparkListenerNodeExcludedForStage]))
+    verify(listenerBusMock, never())
+      .post(isA(classOf[SparkListenerNodeBlacklistedForStage]))
+
+    // Mark another task as failed on exec2 -- now we exclude exec2, which also leads to
+    // excluding the entire node.
+    taskSetExcludelist.updateExcludedForFailedTask(
+      "hostA", exec = "exec2", index = 1, failureReason = "testing")
+
+    assert(taskSetExcludelist.isExecutorExcludedForTaskSet("exec1"))
+
+    assert(taskSetExcludelist.isExecutorExcludedForTaskSet("exec2"))
+    verify(listenerBusMock).post(
+      SparkListenerExecutorExcludedForStage(0, "exec2", 2, 0, attemptId))
+    verify(listenerBusMock).post(
+      SparkListenerExecutorBlacklistedForStage(0, "exec2", 2, 0, attemptId))
+
+    assert(taskSetExcludelist.isNodeExcludedForTaskSet("hostA"))
+    verify(listenerBusMock).post(
+      SparkListenerNodeExcludedForStage(0, "hostA", 2, 0, attemptId))
+    verify(listenerBusMock).post(
+      SparkListenerNodeBlacklistedForStage(0, "hostA", 2, 0, attemptId))
+
+    // Make sure the exclude has the correct per-task && per-executor responses, over a wider
+    // range of inputs.
+    for {
+      executor <- (1 to 4).map(e => s"exec$e")
+      index <- 0 until 10
+    } {
+      withClue(s"exec = $executor; index = $index") {
+        val badExec = (executor == "exec1" || executor == "exec2")
+        val badIndex = (index == 0 || index == 1)
+        assert(
+          // this ignores whether the executor is excluded entirely for the taskset -- that is
+          // intentional, it keeps it fast and is sufficient for usage in the scheduler.
+          taskSetExcludelist.isExecutorExcludedForTask(executor, index) === (badExec && badIndex))
+        assert(taskSetExcludelist.isExecutorExcludedForTaskSet(executor) === badExec)
+        if (badExec) {
+          verify(listenerBusMock).post(
+            SparkListenerExecutorExcludedForStage(0, executor, 2, 0, attemptId))
+          verify(listenerBusMock).post(
+            SparkListenerExecutorBlacklistedForStage(0, executor, 2, 0, attemptId))
+        }
+      }
+    }
+    assert(taskSetExcludelist.isNodeExcludedForTaskSet("hostA"))
+    val execToFailures = taskSetExcludelist.execToFailures
+    assert(execToFailures.keySet === Set("exec1", "exec2"))
+
+    Seq("exec1", "exec2").foreach { exec =>
+      assert(
+        execToFailures(exec).taskToFailureCountAndFailureTime === Map(
+          0 -> ((1, 0)),
+          1 -> ((1, 0))
+        )
+      )
+    }
+  }
+
+  test("multiple attempts for the same task count once") {
+    // Make sure that for excluding tasks, the node counts task attempts, not executors.  But for
+    // stage-level excluding, we count unique tasks.  The reason for this difference is, with
+    // task-attempt excluding, we want to make it easy to configure so that you ensure a node
+    // is excluded before the taskset is completely aborted because of spark.task.maxFailures.
+    // But with stage-excluding, we want to make sure we're not just counting one bad task
+    // that has failed many times.
+
+    val conf = new SparkConf().setMaster("local").setAppName("test")
+      .set(config.MAX_TASK_ATTEMPTS_PER_EXECUTOR, 2)
+      .set(config.MAX_TASK_ATTEMPTS_PER_NODE, 3)
+      .set(config.MAX_FAILURES_PER_EXEC_STAGE, 2)
+      .set(config.MAX_FAILED_EXEC_PER_NODE_STAGE, 3)
+    val clock = new ManualClock
+
+    val attemptId = 0
+    val taskSetExcludlist = new TaskSetExcludelist(
+      listenerBusMock, conf, stageId = 0, stageAttemptId = attemptId, clock = clock)
+
+    var time = 0
+    clock.setTime(time)
+    // Fail a task twice on hostA, exec:1
+    taskSetExcludlist.updateExcludedForFailedTask(
+      "hostA", exec = "1", index = 0, failureReason = "testing")
+    taskSetExcludlist.updateExcludedForFailedTask(
+      "hostA", exec = "1", index = 0, failureReason = "testing")
+    assert(taskSetExcludlist.isExecutorExcludedForTask("1", 0))
+    assert(!taskSetExcludlist.isNodeExcludedForTask("hostA", 0))
+
+    assert(!taskSetExcludlist.isExecutorExcludedForTaskSet("1"))
+    verify(listenerBusMock, never()).post(
+      SparkListenerExecutorExcludedForStage(time, "1", 2, 0, attemptId))
+
+    assert(!taskSetExcludlist.isNodeExcludedForTaskSet("hostA"))
+    verify(listenerBusMock, never()).post(
+      SparkListenerNodeExcludedForStage(time, "hostA", 2, 0, attemptId))
+
+    // Fail the same task once more on hostA, exec:2
+    time += 1
+    clock.setTime(time)
+    taskSetExcludlist.updateExcludedForFailedTask(
+      "hostA", exec = "2", index = 0, failureReason = "testing")
+    assert(taskSetExcludlist.isNodeExcludedForTask("hostA", 0))
+
+    assert(!taskSetExcludlist.isExecutorExcludedForTaskSet("2"))
+    verify(listenerBusMock, never()).post(
+      SparkListenerExecutorExcludedForStage(time, "2", 2, 0, attemptId))
+
+    assert(!taskSetExcludlist.isNodeExcludedForTaskSet("hostA"))
+    verify(listenerBusMock, never()).post(
+      SparkListenerNodeExcludedForStage(time, "hostA", 2, 0, attemptId))
+
+    // Fail another task on hostA, exec:1.  Now that executor has failures on two different tasks,
+    // so its excluded
+    time += 1
+    clock.setTime(time)
+    taskSetExcludlist.updateExcludedForFailedTask(
+      "hostA", exec = "1", index = 1, failureReason = "testing")
+
+    assert(taskSetExcludlist.isExecutorExcludedForTaskSet("1"))
+    verify(listenerBusMock)
+      .post(SparkListenerExecutorExcludedForStage(time, "1", 2, 0, attemptId))
+
+    assert(!taskSetExcludlist.isNodeExcludedForTaskSet("hostA"))
+    verify(listenerBusMock, never())
+      .post(isA(classOf[SparkListenerNodeExcludedForStage]))
+
+    // Fail a third task on hostA, exec:2, so that exec is excluded for the whole task set
+    time += 1
+    clock.setTime(time)
+    taskSetExcludlist.updateExcludedForFailedTask(
+      "hostA", exec = "2", index = 2, failureReason = "testing")
+
+    assert(taskSetExcludlist.isExecutorExcludedForTaskSet("2"))
+    verify(listenerBusMock)
+      .post(SparkListenerExecutorExcludedForStage(time, "2", 2, 0, attemptId))
+
+    assert(!taskSetExcludlist.isNodeExcludedForTaskSet("hostA"))
+    verify(listenerBusMock, never())
+      .post(isA(classOf[SparkListenerNodeExcludedForStage]))
+
+    // Fail a fourth & fifth task on hostA, exec:3.  Now we've got three executors that are
+    // excluded for the taskset, so exclude the whole node.
+    time += 1
+    clock.setTime(time)
+    taskSetExcludlist.updateExcludedForFailedTask(
+      "hostA", exec = "3", index = 3, failureReason = "testing")
+    taskSetExcludlist.updateExcludedForFailedTask(
+      "hostA", exec = "3", index = 4, failureReason = "testing")
+
+    assert(taskSetExcludlist.isExecutorExcludedForTaskSet("3"))
+    verify(listenerBusMock)
+      .post(SparkListenerExecutorExcludedForStage(time, "3", 2, 0, attemptId))
+
+    assert(taskSetExcludlist.isNodeExcludedForTaskSet("hostA"))
+    verify(listenerBusMock).post(
+      SparkListenerNodeExcludedForStage(time, "hostA", 3, 0, attemptId))
+  }
+
+  test("only exclude nodes for the task set when all the excluded executors are all on " +
+    "same host") {
+    // we exclude executors on two different hosts within one taskSet -- make sure that doesn't
+    // lead to any node excluding
+    val conf = new SparkConf().setAppName("test").setMaster("local")
+      .set(config.EXCLUDE_ON_FAILURE_ENABLED.key, "true")
+    val clock = new ManualClock
+
+    val attemptId = 0
+    val taskSetExcludlist = new TaskSetExcludelist(
+      listenerBusMock, conf, stageId = 0, stageAttemptId = attemptId, clock = clock)
+    var time = 0
+    clock.setTime(time)
+    taskSetExcludlist.updateExcludedForFailedTask(
+      "hostA", exec = "1", index = 0, failureReason = "testing")
+    taskSetExcludlist.updateExcludedForFailedTask(
+      "hostA", exec = "1", index = 1, failureReason = "testing")
+
+    assert(taskSetExcludlist.isExecutorExcludedForTaskSet("1"))
+    verify(listenerBusMock)
+      .post(SparkListenerExecutorExcludedForStage(time, "1", 2, 0, attemptId))
+    verify(listenerBusMock)
+      .post(SparkListenerExecutorBlacklistedForStage(time, "1", 2, 0, attemptId))
+
+    assert(!taskSetExcludlist.isNodeExcludedForTaskSet("hostA"))
+    verify(listenerBusMock, never()).post(
+      SparkListenerNodeExcludedForStage(time, "hostA", 2, 0, attemptId))
+    verify(listenerBusMock, never()).post(
+      SparkListenerNodeBlacklistedForStage(time, "hostA", 2, 0, attemptId))
+
+    time += 1
+    clock.setTime(time)
+    taskSetExcludlist.updateExcludedForFailedTask(
+      "hostB", exec = "2", index = 0, failureReason = "testing")
+    taskSetExcludlist.updateExcludedForFailedTask(
+      "hostB", exec = "2", index = 1, failureReason = "testing")
+    assert(taskSetExcludlist.isExecutorExcludedForTaskSet("1"))
+
+    assert(taskSetExcludlist.isExecutorExcludedForTaskSet("2"))
+    verify(listenerBusMock)
+      .post(SparkListenerExecutorExcludedForStage(time, "2", 2, 0, attemptId))
+    verify(listenerBusMock)
+      .post(SparkListenerExecutorBlacklistedForStage(time, "2", 2, 0, attemptId))
+
+    assert(!taskSetExcludlist.isNodeExcludedForTaskSet("hostA"))
+    assert(!taskSetExcludlist.isNodeExcludedForTaskSet("hostB"))
+    verify(listenerBusMock, never())
+      .post(isA(classOf[SparkListenerNodeExcludedForStage]))
+    verify(listenerBusMock, never())
+      .post(isA(classOf[SparkListenerNodeBlacklistedForStage]))
+  }
+
+}
diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala
index 4978be3e04c1e..3841425fa5ae2 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala
@@ -31,7 +31,7 @@ import org.scalatest.Assertions._
 import org.scalatest.PrivateMethodTester
 import org.scalatest.concurrent.Eventually
 
-import org.apache.spark._
+import org.apache.spark.{FakeSchedulerBackend => _, _}
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config
 import org.apache.spark.internal.config.Tests.SKIP_VALIDATE_CORES_TESTING
@@ -41,7 +41,7 @@ import org.apache.spark.resource.TestResourceIDs._
 import org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend
 import org.apache.spark.serializer.SerializerInstance
 import org.apache.spark.storage.BlockManagerId
-import org.apache.spark.util.{AccumulatorV2, ManualClock}
+import org.apache.spark.util.{AccumulatorV2, Clock, ManualClock, SystemClock}
 
 class FakeDAGScheduler(sc: SparkContext, taskScheduler: FakeTaskScheduler)
   extends DAGScheduler(sc) {
@@ -109,8 +109,11 @@ object FakeRackUtil {
  * a list of "live" executors and their hostnames for isExecutorAlive and hasExecutorsAliveOnHost
  * to work, and these are required for locality in TaskSetManager.
  */
-class FakeTaskScheduler(sc: SparkContext, liveExecutors: (String, String)* /* execId, host */)
-  extends TaskSchedulerImpl(sc)
+class FakeTaskScheduler(
+    sc: SparkContext,
+    clock: Clock,
+    liveExecutors: (String, String)* /* execId, host */)
+  extends TaskSchedulerImpl(sc, sc.conf.get(config.TASK_MAX_FAILURES), clock = clock)
 {
   val startedTasks = new ArrayBuffer[Long]
   val endedTasks = new mutable.HashMap[Long, TaskEndReason]
@@ -120,6 +123,10 @@ class FakeTaskScheduler(sc: SparkContext, liveExecutors: (String, String)* /* ex
 
   val executors = new mutable.HashMap[String, String]
 
+  def this(sc: SparkContext, liveExecutors: (String, String)*) = {
+    this(sc, new SystemClock, liveExecutors: _*)
+  }
+
   // this must be initialized before addExecutor
   override val defaultRackValue: Option[String] = Some("default")
   for ((execId, host) <- liveExecutors) {
@@ -149,13 +156,12 @@ class FakeTaskScheduler(sc: SparkContext, liveExecutors: (String, String)* /* ex
 
   override def taskSetFinished(manager: TaskSetManager): Unit = finishedManagers += manager
 
-  override def isExecutorAlive(execId: String): Boolean = executors.contains(execId)
+  override def isExecutorAlive(execId: String): Boolean =
+    executors.contains(execId) && !isExecutorDecommissioned(execId)
 
-  override def hasExecutorsAliveOnHost(host: String): Boolean = executors.values.exists(_ == host)
-
-  override def hasHostAliveOnRack(rack: String): Boolean = {
-    hostsByRack.get(rack) != None
-  }
+  override def hasExecutorsAliveOnHost(host: String): Boolean =
+    !isHostDecommissioned(host) && executors
+      .exists { case (e, h) => h == host && !isExecutorDecommissioned(e) }
 
   def addExecutor(execId: String, host: String): Unit = {
     executors.put(execId, host)
@@ -371,25 +377,25 @@ class TaskSetManagerSuite
 
     // offers not accepted due to task set zombies are not delay schedule rejects
     manager.isZombie = true
-    val (taskDesciption, delayReject) = manager.resourceOffer("exec2", "host2", ANY)
-    assert(taskDesciption.isEmpty)
+    val (taskDescription, delayReject, _) = manager.resourceOffer("exec2", "host2", ANY)
+    assert(taskDescription.isEmpty)
     assert(delayReject === false)
     manager.isZombie = false
 
-    // offers not accepted due to blacklisting are not delay schedule rejects
+    // offers not accepted due to excludelist are not delay schedule rejects
     val tsmSpy = spy(manager)
-    val blacklist = mock(classOf[TaskSetBlacklist])
-    when(tsmSpy.taskSetBlacklistHelperOpt).thenReturn(Some(blacklist))
-    when(blacklist.isNodeBlacklistedForTaskSet(any())).thenReturn(true)
-    val (blacklistTask, blackListReject) = tsmSpy.resourceOffer("exec2", "host2", ANY)
-    assert(blacklistTask.isEmpty)
-    assert(blackListReject === false)
+    val excludelist = mock(classOf[TaskSetExcludelist])
+    when(tsmSpy.taskSetExcludelistHelperOpt).thenReturn(Some(excludelist))
+    when(excludelist.isNodeExcludedForTaskSet(any())).thenReturn(true)
+    val (task, taskReject, _) = tsmSpy.resourceOffer("exec2", "host2", ANY)
+    assert(task.isEmpty)
+    assert(taskReject === false)
 
     // After another delay, we can go ahead and launch that task non-locally
     assert(manager.resourceOffer("exec2", "host2", ANY)._1.get.index === 3)
 
     // offers not accepted due to no pending tasks are not delay schedule rejects
-    val (noPendingTask, noPendingReject) = manager.resourceOffer("exec2", "host2", ANY)
+    val (noPendingTask, noPendingReject, _) = manager.resourceOffer("exec2", "host2", ANY)
     assert(noPendingTask.isEmpty)
     assert(noPendingReject === false)
   }
@@ -415,7 +421,7 @@ class TaskSetManagerSuite
 
     // Now mark host2 as dead
     sched.removeExecutor("exec2")
-    manager.executorLost("exec2", "host2", SlaveLost())
+    manager.executorLost("exec2", "host2", ExecutorProcessLost())
 
     // nothing should be chosen
     assert(manager.resourceOffer("exec1", "host1", ANY)._1 === None)
@@ -473,11 +479,11 @@ class TaskSetManagerSuite
     }
   }
 
-  test("executors should be blacklisted after task failure, in spite of locality preferences") {
+  test("executors should be excluded after task failure, in spite of locality preferences") {
     val rescheduleDelay = 300L
     val conf = new SparkConf().
-      set(config.BLACKLIST_ENABLED, true).
-      set(config.BLACKLIST_TIMEOUT_CONF, rescheduleDelay).
+      set(config.EXCLUDE_ON_FAILURE_ENABLED, true).
+      set(config.EXCLUDE_ON_FAILURE_TIMEOUT_CONF, rescheduleDelay).
       // don't wait to jump locality levels in this test
       set(config.LOCALITY_WAIT.key, "0")
 
@@ -489,11 +495,11 @@ class TaskSetManagerSuite
     val taskSet = FakeTask.createTaskSet(1, Seq(TaskLocation("host1", "exec1")))
     val clock = new ManualClock
     clock.advance(1)
-    // We don't directly use the application blacklist, but its presence triggers blacklisting
+    // We don't directly use the application excludelist, but its presence triggers exclusion
     // within the taskset.
     val mockListenerBus = mock(classOf[LiveListenerBus])
-    val blacklistTrackerOpt = Some(new BlacklistTracker(mockListenerBus, conf, None, clock))
-    val manager = new TaskSetManager(sched, taskSet, 4, blacklistTrackerOpt, clock)
+    val healthTrackerOpt = Some(new HealthTracker(mockListenerBus, conf, None, clock))
+    val manager = new TaskSetManager(sched, taskSet, 4, healthTrackerOpt, clock)
 
     {
       val offerResult = manager.resourceOffer("exec1", "host1", PROCESS_LOCAL)._1
@@ -506,7 +512,7 @@ class TaskSetManagerSuite
       manager.handleFailedTask(offerResult.get.taskId, TaskState.FINISHED, TaskResultLost)
       assert(!sched.taskSetsFailed.contains(taskSet.id))
 
-      // Ensure scheduling on exec1 fails after failure 1 due to blacklist
+      // Ensure scheduling on exec1 fails after failure 1 due to executor being excluded
       assert(manager.resourceOffer("exec1", "host1", PROCESS_LOCAL)._1.isEmpty)
       assert(manager.resourceOffer("exec1", "host1", NODE_LOCAL)._1.isEmpty)
       assert(manager.resourceOffer("exec1", "host1", RACK_LOCAL)._1.isEmpty)
@@ -526,7 +532,7 @@ class TaskSetManagerSuite
       manager.handleFailedTask(offerResult.get.taskId, TaskState.FINISHED, TaskResultLost)
       assert(!sched.taskSetsFailed.contains(taskSet.id))
 
-      // Ensure scheduling on exec1.1 fails after failure 2 due to blacklist
+      // Ensure scheduling on exec1.1 fails after failure 2 due to executor being excluded
       assert(manager.resourceOffer("exec1.1", "host1", NODE_LOCAL)._1.isEmpty)
     }
 
@@ -542,12 +548,12 @@ class TaskSetManagerSuite
       manager.handleFailedTask(offerResult.get.taskId, TaskState.FINISHED, TaskResultLost)
       assert(!sched.taskSetsFailed.contains(taskSet.id))
 
-      // Ensure scheduling on exec2 fails after failure 3 due to blacklist
+      // Ensure scheduling on exec2 fails after failure 3 due to executor being excluded
       assert(manager.resourceOffer("exec2", "host2", ANY)._1.isEmpty)
     }
 
-    // Despite advancing beyond the time for expiring executors from within the blacklist,
-    // we *never* expire from *within* the stage blacklist
+    // Despite advancing beyond the time for expiring executors from within the excludelist,
+    // we *never* expire from *within* the stage excludelist
     clock.advance(rescheduleDelay)
 
     {
@@ -598,10 +604,10 @@ class TaskSetManagerSuite
       Array(PROCESS_LOCAL, NODE_LOCAL, NO_PREF, RACK_LOCAL, ANY)))
     // test if the valid locality is recomputed when the executor is lost
     sched.removeExecutor("execC")
-    manager.executorLost("execC", "host2", SlaveLost())
+    manager.executorLost("execC", "host2", ExecutorProcessLost())
     assert(manager.myLocalityLevels.sameElements(Array(NODE_LOCAL, NO_PREF, ANY)))
     sched.removeExecutor("execD")
-    manager.executorLost("execD", "host1", SlaveLost())
+    manager.executorLost("execD", "host1", ExecutorProcessLost())
     assert(manager.myLocalityLevels.sameElements(Array(NO_PREF, ANY)))
   }
 
@@ -620,7 +626,7 @@ class TaskSetManagerSuite
     manager.executorAdded()
     sched.addExecutor("execC", "host2")
     manager.executorAdded()
-    assert(manager.resourceOffer("exec1", "host1", ANY)._1.isDefined)
+    assert(manager.resourceOffer("execB", "host1", ANY)._1.isDefined)
     sched.removeExecutor("execA")
     manager.executorLost(
       "execA",
@@ -634,6 +640,78 @@ class TaskSetManagerSuite
     assert(sched.taskSetsFailed.contains(taskSet.id))
   }
 
+  test("SPARK-31837: Shift to the new highest locality level if there is when recomputeLocality") {
+    sc = new SparkContext("local", "test")
+    sched = new FakeTaskScheduler(sc)
+    val taskSet = FakeTask.createTaskSet(2,
+      Seq(TaskLocation("host1", "execA")),
+      Seq(TaskLocation("host1", "execA")))
+    val clock = new ManualClock()
+    val manager = new TaskSetManager(sched, taskSet, 1, clock = clock)
+    // before any executors are added to TaskScheduler, the manager's
+    // locality level only has ANY, so tasks can be scheduled anyway.
+    assert(manager.resourceOffer("execB", "host2", ANY)._1.isDefined)
+    sched.addExecutor("execA", "host1")
+    manager.executorAdded()
+    // after adding a new executor, the manager locality has PROCESS_LOCAL, NODE_LOCAL, ANY.
+    // And we'll shift to the new highest locality level, which is PROCESS_LOCAL in this case.
+    assert(manager.resourceOffer("execC", "host3", ANY)._1.isEmpty)
+    assert(manager.resourceOffer("execA", "host1", ANY)._1.isDefined)
+  }
+
+  test("SPARK-32653: Decommissioned host should not be used to calculate locality levels") {
+    sc = new SparkContext("local", "test")
+    sched = new FakeTaskScheduler(sc)
+    val backend = mock(classOf[SchedulerBackend])
+    doNothing().when(backend).reviveOffers()
+    sched.initialize(backend)
+
+    val exec0 = "exec0"
+    val exec1 = "exec1"
+    val host0 = "host0"
+    sched.addExecutor(exec0, host0)
+    sched.addExecutor(exec1, host0)
+
+    val taskSet = FakeTask.createTaskSet(2,
+      Seq(ExecutorCacheTaskLocation(host0, exec0)),
+      Seq(ExecutorCacheTaskLocation(host0, exec1)))
+    sched.submitTasks(taskSet)
+    val manager = sched.taskSetManagerForAttempt(0, 0).get
+
+    assert(manager.myLocalityLevels === Array(PROCESS_LOCAL, NODE_LOCAL, ANY))
+
+    // Decommission all executors on host0, to mimic CoarseGrainedSchedulerBackend.
+    sched.executorDecommission(exec0, ExecutorDecommissionInfo("test", Some(host0)))
+    sched.executorDecommission(exec1, ExecutorDecommissionInfo("test", Some(host0)))
+
+    assert(manager.myLocalityLevels === Array(ANY))
+  }
+
+  test("SPARK-32653: Decommissioned executor should not be used to calculate locality levels") {
+    sc = new SparkContext("local", "test")
+    sched = new FakeTaskScheduler(sc)
+    val backend = mock(classOf[SchedulerBackend])
+    doNothing().when(backend).reviveOffers()
+    sched.initialize(backend)
+
+    val exec0 = "exec0"
+    val exec1 = "exec1"
+    val host0 = "host0"
+    sched.addExecutor(exec0, host0)
+    sched.addExecutor(exec1, host0)
+
+    val taskSet = FakeTask.createTaskSet(1, Seq(ExecutorCacheTaskLocation(host0, exec0)))
+    sched.submitTasks(taskSet)
+    val manager = sched.taskSetManagerForAttempt(0, 0).get
+
+    assert(manager.myLocalityLevels === Array(PROCESS_LOCAL, NODE_LOCAL, ANY))
+
+    // Decommission the only executor (without the host) that the task is interested in running on.
+    sched.executorDecommission(exec0, ExecutorDecommissionInfo("test", None))
+
+    assert(manager.myLocalityLevels === Array(NODE_LOCAL, ANY))
+  }
+
   test("test RACK_LOCAL tasks") {
     // Assign host1 to rack1
     FakeRackUtil.assignHostToRack("host1", "rack1")
@@ -731,6 +809,14 @@ class TaskSetManagerSuite
     assert(thrown2.getMessage().contains("bigger than spark.driver.maxResultSize"))
   }
 
+  test("SPARK-32470: do not check total size of intermediate stages") {
+    val conf = new SparkConf().set(config.MAX_RESULT_SIZE.key, "20k")
+    sc = new SparkContext("local", "test", conf)
+    // final result is below limit.
+    val r = sc.makeRDD(0 until 2000, 2000).distinct(10).filter(_ == 0).collect()
+    assert(1 === r.size)
+  }
+
   test("[SPARK-13931] taskSetManager should not send Resubmitted tasks after being a zombie") {
     val conf = new SparkConf().set(config.SPECULATION_ENABLED, true)
     sc = new SparkContext("local", "test", conf)
@@ -795,7 +881,7 @@ class TaskSetManagerSuite
     assert(resubmittedTasks === 0)
     assert(manager.runningTasks === 1)
 
-    manager.executorLost("execB", "host2", new SlaveLost())
+    manager.executorLost("execB", "host2", new ExecutorProcessLost())
     assert(manager.runningTasks === 0)
     assert(resubmittedTasks === 0)
   }
@@ -904,7 +990,7 @@ class TaskSetManagerSuite
     // Make sure schedBackend.killTask(2, "exec3", true, "another attempt succeeded") gets called
     assert(killTaskCalled)
     // Host 3 Losts, there's only task 2.0 on it, which killed by task 2.1
-    manager.executorLost("exec3", "host3", SlaveLost())
+    manager.executorLost("exec3", "host3", ExecutorProcessLost())
     // Check the resubmittedTasks
     assert(resubmittedTasks === 0)
   }
@@ -1025,8 +1111,8 @@ class TaskSetManagerSuite
     assert(manager.resourceOffer("execB.2", "host2", ANY) !== None)
     sched.removeExecutor("execA")
     sched.removeExecutor("execB.2")
-    manager.executorLost("execA", "host1", SlaveLost())
-    manager.executorLost("execB.2", "host2", SlaveLost())
+    manager.executorLost("execA", "host1", ExecutorProcessLost())
+    manager.executorLost("execB.2", "host2", ExecutorProcessLost())
     clock.advance(LOCALITY_WAIT_MS * 4)
     sched.addExecutor("execC", "host3")
     manager.executorAdded()
@@ -1236,7 +1322,7 @@ class TaskSetManagerSuite
 
   test("SPARK-19868: DagScheduler only notified of taskEnd when state is ready") {
     // dagScheduler.taskEnded() is async, so it may *seem* ok to call it before we've set all
-    // appropriate state, eg. isZombie.   However, this sets up a race that could go the wrong way.
+    // appropriate state, e.g. isZombie.   However, this sets up a race that could go the wrong way.
     // This is a super-focused regression test which checks the zombie state as soon as
     // dagScheduler.taskEnded() is called, to ensure we haven't introduced a race.
     sc = new SparkContext("local", "test")
@@ -1272,20 +1358,20 @@ class TaskSetManagerSuite
     assert(manager3.name === "TaskSet_1.1")
   }
 
-  test("don't update blacklist for shuffle-fetch failures, preemption, denied commits, " +
+  test("don't update excludelist for shuffle-fetch failures, preemption, denied commits, " +
       "or killed tasks") {
     // Setup a taskset, and fail some tasks for a fetch failure, preemption, denied commit,
     // and killed task.
     val conf = new SparkConf().
-      set(config.BLACKLIST_ENABLED, true)
+      set(config.EXCLUDE_ON_FAILURE_ENABLED, true)
     sc = new SparkContext("local", "test", conf)
     sched = new FakeTaskScheduler(sc, ("exec1", "host1"), ("exec2", "host2"))
     val taskSet = FakeTask.createTaskSet(4)
     val tsm = new TaskSetManager(sched, taskSet, 4)
-    // we need a spy so we can attach our mock blacklist
+    // we need a spy so we can attach our mock excludelist
     val tsmSpy = spy(tsm)
-    val blacklist = mock(classOf[TaskSetBlacklist])
-    when(tsmSpy.taskSetBlacklistHelperOpt).thenReturn(Some(blacklist))
+    val excludelist = mock(classOf[TaskSetExcludelist])
+    when(tsmSpy.taskSetExcludelistHelperOpt).thenReturn(Some(excludelist))
 
     // make some offers to our taskset, to get tasks we will fail
     val taskDescs = Seq(
@@ -1306,23 +1392,23 @@ class TaskSetManagerSuite
       TaskCommitDenied(0, 2, 0))
     tsmSpy.handleFailedTask(taskDescs(3).taskId, TaskState.KILLED, TaskKilled("test"))
 
-    // Make sure that the blacklist ignored all of the task failures above, since they aren't
+    // Make sure that the excludelist ignored all of the task failures above, since they aren't
     // the fault of the executor where the task was running.
-    verify(blacklist, never())
-      .updateBlacklistForFailedTask(anyString(), anyString(), anyInt(), anyString())
+    verify(excludelist, never())
+      .updateExcludedForFailedTask(anyString(), anyString(), anyInt(), anyString())
   }
 
-  test("update application blacklist for shuffle-fetch") {
+  test("update application healthTracker for shuffle-fetch") {
     // Setup a taskset, and fail some one task for fetch failure.
     val conf = new SparkConf()
-      .set(config.BLACKLIST_ENABLED, true)
+      .set(config.EXCLUDE_ON_FAILURE_ENABLED, true)
       .set(config.SHUFFLE_SERVICE_ENABLED, true)
-      .set(config.BLACKLIST_FETCH_FAILURE_ENABLED, true)
+      .set(config.EXCLUDE_ON_FAILURE_FETCH_FAILURE_ENABLED, true)
     sc = new SparkContext("local", "test", conf)
     sched = new FakeTaskScheduler(sc, ("exec1", "host1"), ("exec2", "host2"))
     val taskSet = FakeTask.createTaskSet(4)
-    val blacklistTracker = new BlacklistTracker(sc, None)
-    val tsm = new TaskSetManager(sched, taskSet, 4, Some(blacklistTracker))
+    val healthTracker = new HealthTracker(sc, None)
+    val tsm = new TaskSetManager(sched, taskSet, 4, Some(healthTracker))
 
     // make some offers to our taskset, to get tasks we will fail
     val taskDescs = Seq(
@@ -1334,22 +1420,22 @@ class TaskSetManagerSuite
     }
     assert(taskDescs.size === 4)
 
-    assert(!blacklistTracker.isExecutorBlacklisted(taskDescs(0).executorId))
-    assert(!blacklistTracker.isNodeBlacklisted("host1"))
+    assert(!healthTracker.isExecutorExcluded(taskDescs(0).executorId))
+    assert(!healthTracker.isNodeExcluded("host1"))
 
     // Fail the task with fetch failure
     tsm.handleFailedTask(taskDescs(0).taskId, TaskState.FAILED,
       FetchFailed(BlockManagerId(taskDescs(0).executorId, "host1", 12345), 0, 0L, 0, 0, "ignored"))
 
-    assert(blacklistTracker.isNodeBlacklisted("host1"))
+    assert(healthTracker.isNodeExcluded("host1"))
   }
 
-  test("update blacklist before adding pending task to avoid race condition") {
-    // When a task fails, it should apply the blacklist policy prior to
+  test("update healthTracker before adding pending task to avoid race condition") {
+    // When a task fails, it should apply the excludeOnFailure policy prior to
     // retrying the task otherwise there's a race condition where run on
     // the same executor that it was intended to be black listed from.
     val conf = new SparkConf().
-      set(config.BLACKLIST_ENABLED, true)
+      set(config.EXCLUDE_ON_FAILURE_ENABLED, true)
 
     // Create a task with two executors.
     sc = new SparkContext("local", "test", conf)
@@ -1362,8 +1448,8 @@ class TaskSetManagerSuite
 
     val clock = new ManualClock
     val mockListenerBus = mock(classOf[LiveListenerBus])
-    val blacklistTracker = new BlacklistTracker(mockListenerBus, conf, None, clock)
-    val taskSetManager = new TaskSetManager(sched, taskSet, 1, Some(blacklistTracker))
+    val healthTracker = new HealthTracker(mockListenerBus, conf, None, clock)
+    val taskSetManager = new TaskSetManager(sched, taskSet, 1, Some(healthTracker))
     val taskSetManagerSpy = spy(taskSetManager)
 
     val taskDesc = taskSetManagerSpy.resourceOffer(exec, host, TaskLocality.ANY)._1
@@ -1372,8 +1458,8 @@ class TaskSetManagerSuite
     when(taskSetManagerSpy.addPendingTask(anyInt(), anyBoolean(), anyBoolean())).thenAnswer(
       (invocationOnMock: InvocationOnMock) => {
         val task: Int = invocationOnMock.getArgument(0)
-        assert(taskSetManager.taskSetBlacklistHelperOpt.get.
-          isExecutorBlacklistedForTask(exec, task))
+        assert(taskSetManager.taskSetExcludelistHelperOpt.get.
+          isExecutorExcludedForTask(exec, task))
       }
     )
 
@@ -1550,7 +1636,7 @@ class TaskSetManagerSuite
 
     assert(resubmittedTasks.isEmpty)
     // Host 2 Losts, meaning we lost the map output task4
-    manager.executorLost("exec2", "host2", SlaveLost())
+    manager.executorLost("exec2", "host2", ExecutorProcessLost())
     // Make sure that task with index 2 is re-submitted
     assert(resubmittedTasks.contains(2))
 
@@ -1651,7 +1737,7 @@ class TaskSetManagerSuite
     for (i <- 0 to 99) {
       locations += Seq(TaskLocation("host" + i))
     }
-    val taskSet = FakeTask.createTaskSet(100, locations: _*)
+    val taskSet = FakeTask.createTaskSet(100, locations.toSeq: _*)
     val clock = new ManualClock
     // make sure we only do one rack resolution call, for the entire batch of hosts, as this
     // can be expensive.  The FakeTaskScheduler calls rack resolution more than the real one
@@ -1682,7 +1768,6 @@ class TaskSetManagerSuite
   }
 
   test("TaskSetManager passes task resource along") {
-    import TestUtils._
 
     sc = new SparkContext("local", "test")
     sc.conf.set(TASK_GPU_ID.amountConf, "2")
@@ -1816,7 +1901,8 @@ class TaskSetManagerSuite
       speculationQuantile: Double,
       numTasks: Int,
       numExecutorCores: Int,
-      numCoresPerTask: Int): (TaskSetManager, ManualClock) = {
+      numCoresPerTask: Int,
+      speculationMinimumThreshold: Option[String]): (TaskSetManager, ManualClock) = {
     val conf = new SparkConf()
     conf.set(config.SPECULATION_ENABLED, true)
     conf.set(config.SPECULATION_QUANTILE.key, speculationQuantile.toString)
@@ -1826,6 +1912,9 @@ class TaskSetManagerSuite
     if (speculationThresholdOpt.isDefined) {
       conf.set(config.SPECULATION_TASK_DURATION_THRESHOLD.key, speculationThresholdOpt.get)
     }
+    if (speculationMinimumThreshold.isDefined) {
+      conf.set(config.SPECULATION_MIN_THRESHOLD.key, speculationMinimumThreshold.get)
+    }
     sc = new SparkContext("local", "test", conf)
     sched = new FakeTaskScheduler(sc, ("exec1", "host1"), ("exec2", "host2"))
     // Create a task set with the given number of tasks
@@ -1852,7 +1941,8 @@ class TaskSetManagerSuite
       speculationQuantile = 1.0,
       numTasks,
       numSlots,
-      numCoresPerTask = 1
+      numCoresPerTask = 1,
+      None
     )
 
     // if the time threshold has not been exceeded, no speculative run should be triggered
@@ -1892,6 +1982,113 @@ class TaskSetManagerSuite
     testSpeculationDurationThreshold(true, 2, 1)
   }
 
+  test("SPARK-21040: Check speculative tasks are launched when an executor is decommissioned" +
+    " and the tasks running on it cannot finish within EXECUTOR_DECOMMISSION_KILL_INTERVAL") {
+    sc = new SparkContext("local", "test")
+    val clock = new ManualClock()
+    sched = new FakeTaskScheduler(sc, clock,
+      ("exec1", "host1"), ("exec2", "host2"), ("exec3", "host3"))
+    sched.backend = mock(classOf[SchedulerBackend])
+    val taskSet = FakeTask.createTaskSet(4)
+    sc.conf.set(config.SPECULATION_ENABLED, true)
+    sc.conf.set(config.SPECULATION_MULTIPLIER, 1.5)
+    sc.conf.set(config.SPECULATION_QUANTILE, 0.5)
+    sc.conf.set(config.EXECUTOR_DECOMMISSION_KILL_INTERVAL.key, "5s")
+    val manager = sched.createTaskSetManager(taskSet, MAX_TASK_FAILURES)
+    val accumUpdatesByTask: Array[Seq[AccumulatorV2[_, _]]] = taskSet.tasks.map { task =>
+      task.metrics.internalAccums
+    }
+
+    // Start TASK 0,1 on exec1, TASK 2 on exec2
+    (0 until 2).foreach { _ =>
+      val taskOption = manager.resourceOffer("exec1", "host1", NO_PREF)._1
+      assert(taskOption.isDefined)
+      assert(taskOption.get.executorId === "exec1")
+    }
+    val taskOption2 = manager.resourceOffer("exec2", "host2", NO_PREF)._1
+    assert(taskOption2.isDefined)
+    assert(taskOption2.get.executorId === "exec2")
+
+    clock.advance(6*1000) // time = 6s
+    // Start TASK 3 on exec2 after some delay
+    val taskOption3 = manager.resourceOffer("exec2", "host2", NO_PREF)._1
+    assert(taskOption3.isDefined)
+    assert(taskOption3.get.executorId === "exec2")
+
+    assert(sched.startedTasks.toSet === Set(0, 1, 2, 3))
+
+    clock.advance(4*1000) // time = 10s
+    // Complete the first 2 tasks and leave the other 2 tasks in running
+    for (id <- Set(0, 1)) {
+      manager.handleSuccessfulTask(id, createTaskResult(id, accumUpdatesByTask(id)))
+      assert(sched.endedTasks(id) === Success)
+    }
+
+    // checkSpeculatableTasks checks that the task runtime is greater than the threshold for
+    // speculating. Since we use a SPECULATION_MULTIPLIER of 1.5, So tasks need to be running for
+    // > 15s for speculation
+    assert(!manager.checkSpeculatableTasks(0))
+    assert(sched.speculativeTasks.toSet === Set())
+
+    // decommission exec-2. All tasks running on exec-2 (i.e. TASK 2,3) will be now
+    // checked if they should be speculated.
+    // (TASK 2 -> 15, TASK 3 -> 15)
+    sched.executorDecommission("exec2", ExecutorDecommissionInfo("decom", None))
+    assert(sched.getExecutorDecommissionState("exec2").map(_.startTime) ===
+      Some(clock.getTimeMillis()))
+
+    assert(manager.checkSpeculatableTasks(0))
+    // TASK 2 started at t=0s, so it can still finish before t=15s (Median task runtime = 10s)
+    // TASK 3 started at t=6s, so it might not finish before t=15s. So TASK 3 should be part
+    // of speculativeTasks
+    assert(sched.speculativeTasks.toSet === Set(3))
+    assert(manager.copiesRunning(3) === 1)
+
+    // Offer resource to start the speculative attempt for the running task
+    val taskOption3New = manager.resourceOffer("exec3", "host3", NO_PREF)._1
+    // Offer more resources. Nothing should get scheduled now.
+    assert(manager.resourceOffer("exec3", "host3", NO_PREF)._1.isEmpty)
+    assert(taskOption3New.isDefined)
+
+    // Assert info about the newly launched speculative task
+    val speculativeTask3 = taskOption3New.get
+    assert(speculativeTask3.index === 3)
+    assert(speculativeTask3.taskId === 4)
+    assert(speculativeTask3.executorId === "exec3")
+    assert(speculativeTask3.attemptNumber === 1)
+
+    clock.advance(1*1000) // time = 11s
+    // Running checkSpeculatableTasks again should return false
+    assert(!manager.checkSpeculatableTasks(0))
+    assert(manager.copiesRunning(2) === 1)
+    assert(manager.copiesRunning(3) === 2)
+
+    clock.advance(5*1000) // time = 16s
+    // At t=16s, TASK 2 has been running for 16s. It is more than the
+    // SPECULATION_MULTIPLIER * medianRuntime = 1.5 * 10 = 15s. So now TASK 2 will
+    // be selected for speculation. Here we are verifying that regular speculation configs
+    // should still take effect even when a EXECUTOR_DECOMMISSION_KILL_INTERVAL is provided and
+    // corresponding executor is decommissioned
+    assert(manager.checkSpeculatableTasks(0))
+    assert(sched.speculativeTasks.toSet === Set(2, 3))
+    assert(manager.copiesRunning(2) === 1)
+    assert(manager.copiesRunning(3) === 2)
+    val taskOption2New = manager.resourceOffer("exec3", "host3", NO_PREF)._1
+    assert(taskOption2New.isDefined)
+    val speculativeTask2 = taskOption2New.get
+    // Ensure that TASK 2 is re-launched on exec3, host3
+    assert(speculativeTask2.index === 2)
+    assert(speculativeTask2.taskId === 5)
+    assert(speculativeTask2.executorId === "exec3")
+    assert(speculativeTask2.attemptNumber === 1)
+
+    assert(manager.copiesRunning(2) === 2)
+    assert(manager.copiesRunning(3) === 2)
+
+    // Offering additional resources should not lead to any speculative tasks being respawned
+    assert(manager.resourceOffer("exec1", "host1", ANY)._1.isEmpty)
+  }
+
   test("SPARK-29976 Regular speculation configs should still take effect even when a " +
       "threshold is provided") {
     val (manager, clock) = testSpeculationDurationSetup(
@@ -1899,7 +2096,8 @@ class TaskSetManagerSuite
       speculationQuantile = 0.5,
       numTasks = 2,
       numExecutorCores = 2,
-      numCoresPerTask = 1
+      numCoresPerTask = 1,
+      None
     )
 
     // Task duration can't be 0, advance 1 sec
@@ -2019,6 +2217,31 @@ class TaskSetManagerSuite
       assert(manager.invokePrivate(numFailures())(index1) === 1)
     }
   }
+
+  test("SPARK-33741 Test minimum amount of time a task runs " +
+    "before being considered for speculation") {
+    val (manager, clock) = testSpeculationDurationSetup(
+      None,
+      speculationQuantile = 0.5,
+      numTasks = 2,
+      numExecutorCores = 2,
+      numCoresPerTask = 1,
+      Some("3000") // spark.speculation.min.threshold
+    )
+    // Task duration can't be 0, advance 1 sec
+    clock.advance(1000)
+    // Mark one of the task succeeded, which should satisfy the quantile
+    manager.handleSuccessfulTask(0, createTaskResult(0))
+    // Advance 1 more second so the remaining task takes longer
+    clock.advance(1000)
+    manager.checkSpeculatableTasks(sched.MIN_TIME_TO_SPECULATION)
+    // The task is not considered as speculative task due to minimum threshold interval of 3s
+    assert(sched.speculativeTasks.size == 0)
+    clock.advance(2000)
+    manager.checkSpeculatableTasks(sched.MIN_TIME_TO_SPECULATION)
+    // After 3s have elapsed now the task is marked as speculative task
+    assert(sched.speculativeTasks.size == 1)
+  }
 }
 
 class FakeLongTasks(stageId: Int, partitionId: Int) extends FakeTask(stageId, partitionId) {
diff --git a/core/src/test/scala/org/apache/spark/scheduler/WorkerDecommissionExtendedSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/WorkerDecommissionExtendedSuite.scala
new file mode 100644
index 0000000000000..129eb8bf91051
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/scheduler/WorkerDecommissionExtendedSuite.scala
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.scheduler
+
+import scala.concurrent.duration._
+
+import org.scalatest.concurrent.Eventually.{eventually, interval, timeout}
+
+import org.apache.spark.{LocalSparkContext, SparkContext, SparkFunSuite, TestUtils}
+import org.apache.spark.LocalSparkContext.withSpark
+import org.apache.spark.internal.config.{DECOMMISSION_ENABLED, DYN_ALLOCATION_ENABLED, DYN_ALLOCATION_EXECUTOR_IDLE_TIMEOUT, DYN_ALLOCATION_INITIAL_EXECUTORS, DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED}
+import org.apache.spark.launcher.SparkLauncher.{EXECUTOR_MEMORY, SPARK_MASTER}
+import org.apache.spark.scheduler.cluster.StandaloneSchedulerBackend
+
+/** This test suite aims to test worker decommission with various configurations. */
+class WorkerDecommissionExtendedSuite extends SparkFunSuite with LocalSparkContext {
+  private val conf = new org.apache.spark.SparkConf()
+    .setAppName(getClass.getName)
+    .set(SPARK_MASTER, "local-cluster[5,1,512]")
+    .set(EXECUTOR_MEMORY, "512m")
+    .set(DYN_ALLOCATION_ENABLED, true)
+    .set(DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED, true)
+    .set(DYN_ALLOCATION_INITIAL_EXECUTORS, 5)
+    .set(DECOMMISSION_ENABLED, true)
+
+  test("Worker decommission and executor idle timeout") {
+    sc = new SparkContext(conf.set(DYN_ALLOCATION_EXECUTOR_IDLE_TIMEOUT.key, "10s"))
+    withSpark(sc) { sc =>
+      TestUtils.waitUntilExecutorsUp(sc, 5, 60000)
+      val rdd1 = sc.parallelize(1 to 10, 2)
+      val rdd2 = rdd1.map(x => (1, x))
+      val rdd3 = rdd2.reduceByKey(_ + _)
+      val rdd4 = rdd3.sortByKey()
+      assert(rdd4.count() === 1)
+      eventually(timeout(20.seconds), interval(1.seconds)) {
+        assert(sc.getExecutorIds().length < 5)
+      }
+    }
+  }
+
+  test("Decommission 4 executors from 5 executors in total") {
+    sc = new SparkContext(conf)
+    withSpark(sc) { sc =>
+      TestUtils.waitUntilExecutorsUp(sc, 5, 60000)
+      val rdd1 = sc.parallelize(1 to 100000, 200)
+      val rdd2 = rdd1.map(x => (x % 100, x))
+      val rdd3 = rdd2.reduceByKey(_ + _)
+      assert(rdd3.count() === 100)
+
+      val sched = sc.schedulerBackend.asInstanceOf[StandaloneSchedulerBackend]
+      sc.getExecutorIds().tail.foreach { id =>
+        sched.decommissionExecutor(id, ExecutorDecommissionInfo("", None),
+          adjustTargetNumExecutors = false)
+        assert(rdd3.sortByKey().collect().length === 100)
+      }
+    }
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/scheduler/WorkerDecommissionSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/WorkerDecommissionSuite.scala
index 8c6f86a6c0e88..1c2326db6dc99 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/WorkerDecommissionSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/WorkerDecommissionSuite.scala
@@ -19,19 +19,18 @@ package org.apache.spark.scheduler
 
 import java.util.concurrent.Semaphore
 
-import scala.concurrent.TimeoutException
 import scala.concurrent.duration._
 
-import org.apache.spark.{LocalSparkContext, SparkConf, SparkContext, SparkException, SparkFunSuite}
+import org.apache.spark.{LocalSparkContext, SparkConf, SparkContext, SparkFunSuite, TestUtils}
 import org.apache.spark.internal.config
 import org.apache.spark.scheduler.cluster.StandaloneSchedulerBackend
-import org.apache.spark.util.{RpcUtils, SerializableBuffer, ThreadUtils}
+import org.apache.spark.util.ThreadUtils
 
 class WorkerDecommissionSuite extends SparkFunSuite with LocalSparkContext {
 
   override def beforeEach(): Unit = {
-    val conf = new SparkConf().setAppName("test").setMaster("local")
-      .set(config.Worker.WORKER_DECOMMISSION_ENABLED, true)
+    val conf = new SparkConf().setAppName("test")
+      .set(config.DECOMMISSION_ENABLED, true)
 
     sc = new SparkContext("local-cluster[2, 1, 1024]", "test", conf)
   }
@@ -46,14 +45,13 @@ class WorkerDecommissionSuite extends SparkFunSuite with LocalSparkContext {
     assert(sleepyRdd.count() === 10)
   }
 
-  test("verify a task with all workers decommissioned succeeds") {
+  test("verify a running task with all workers decommissioned succeeds") {
+    // Wait for the executors to come up
+    TestUtils.waitUntilExecutorsUp(sc = sc,
+      numExecutors = 2,
+      timeout = 30000) // 30s
+
     val input = sc.parallelize(1 to 10)
-    // Do a count to wait for the executors to be registered.
-    input.count()
-    val sleepyRdd = input.mapPartitions{ x =>
-      Thread.sleep(50)
-      x
-    }
     // Listen for the job
     val sem = new Semaphore(0)
     sc.addSparkListener(new SparkListener {
@@ -61,24 +59,28 @@ class WorkerDecommissionSuite extends SparkFunSuite with LocalSparkContext {
         sem.release()
       }
     })
+
+    val sleepyRdd = input.mapPartitions{ x =>
+      Thread.sleep(5000) // 5s
+      x
+    }
     // Start the task.
     val asyncCount = sleepyRdd.countAsync()
     // Wait for the job to have started
     sem.acquire(1)
+    // Give it time to make it to the worker otherwise we'll block
+    Thread.sleep(2000) // 2s
     // Decommission all the executors, this should not halt the current task.
     // decom.sh message passing is tested manually.
     val sched = sc.schedulerBackend.asInstanceOf[StandaloneSchedulerBackend]
     val execs = sched.getExecutorIds()
-    execs.foreach(execId => sched.decommissionExecutor(execId))
-    val asyncCountResult = ThreadUtils.awaitResult(asyncCount, 10.seconds)
+    // Make the executors decommission, finish, exit, and not be replaced.
+    val execsAndDecomInfo = execs.map((_, ExecutorDecommissionInfo("", None))).toArray
+    sched.decommissionExecutors(
+      execsAndDecomInfo,
+      adjustTargetNumExecutors = true,
+      triggeredByExecutor = false)
+    val asyncCountResult = ThreadUtils.awaitResult(asyncCount, 20.seconds)
     assert(asyncCountResult === 10)
-    // Try and launch task after decommissioning, this should fail
-    val postDecommissioned = input.map(x => x)
-    val postDecomAsyncCount = postDecommissioned.countAsync()
-    val thrown = intercept[java.util.concurrent.TimeoutException]{
-      val result = ThreadUtils.awaitResult(postDecomAsyncCount, 10.seconds)
-    }
-    assert(postDecomAsyncCount.isCompleted === false,
-      "After exec decommission new task could not launch")
   }
 }
diff --git a/core/src/test/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitorSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitorSuite.scala
index 6d494796d5a26..69afdb57ef404 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitorSuite.scala
@@ -21,6 +21,7 @@ import java.util.concurrent.TimeUnit
 
 import scala.collection.mutable
 
+import com.codahale.metrics.Counter
 import org.mockito.ArgumentMatchers.any
 import org.mockito.Mockito.{doAnswer, mock, when}
 
@@ -57,6 +58,15 @@ class ExecutorMonitorSuite extends SparkFunSuite {
   // having to use mockito APIs directly in each test.
   private val knownExecs = mutable.HashSet[String]()
 
+  private def allocationManagerSource(): ExecutorAllocationManagerSource = {
+    val metricSource = mock(classOf[ExecutorAllocationManagerSource])
+    when(metricSource.driverKilled).thenReturn(new Counter)
+    when(metricSource.decommissionUnfinished).thenReturn(new Counter)
+    when(metricSource.gracefullyDecommissioned).thenReturn(new Counter)
+    when(metricSource.exitedUnexpectedly).thenReturn(new Counter)
+    metricSource
+  }
+
   override def beforeEach(): Unit = {
     super.beforeEach()
     knownExecs.clear()
@@ -65,7 +75,7 @@ class ExecutorMonitorSuite extends SparkFunSuite {
     when(client.isExecutorActive(any())).thenAnswer { invocation =>
       knownExecs.contains(invocation.getArguments()(0).asInstanceOf[String])
     }
-    monitor = new ExecutorMonitor(conf, client, null, clock)
+    monitor = new ExecutorMonitor(conf, client, null, clock, allocationManagerSource())
   }
 
   test("basic executor timeout") {
@@ -231,7 +241,7 @@ class ExecutorMonitorSuite extends SparkFunSuite {
     assert(monitor.timedOutExecutors(storageDeadline) ===  Seq("1"))
 
     conf.set(SHUFFLE_SERVICE_ENABLED, true).set(SHUFFLE_SERVICE_FETCH_RDD_ENABLED, true)
-    monitor = new ExecutorMonitor(conf, client, null, clock)
+    monitor = new ExecutorMonitor(conf, client, null, clock, allocationManagerSource())
 
     monitor.onExecutorAdded(SparkListenerExecutorAdded(clock.getTimeMillis(), "1", execInfo))
     monitor.onBlockUpdated(rddUpdate(1, 0, "1", level = StorageLevel.MEMORY_ONLY))
@@ -292,7 +302,7 @@ class ExecutorMonitorSuite extends SparkFunSuite {
   test("shuffle block tracking") {
     val bus = mockListenerBus()
     conf.set(DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED, true).set(SHUFFLE_SERVICE_ENABLED, false)
-    monitor = new ExecutorMonitor(conf, client, bus, clock)
+    monitor = new ExecutorMonitor(conf, client, bus, clock, allocationManagerSource())
 
     // 3 jobs: 2 and 3 share a shuffle, 1 has a separate shuffle.
     val stage1 = stageInfo(1, shuffleId = 0)
@@ -360,7 +370,7 @@ class ExecutorMonitorSuite extends SparkFunSuite {
   test("SPARK-28839: Avoids NPE in context cleaner when shuffle service is on") {
     val bus = mockListenerBus()
     conf.set(DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED, true).set(SHUFFLE_SERVICE_ENABLED, true)
-    monitor = new ExecutorMonitor(conf, client, bus, clock) {
+    monitor = new ExecutorMonitor(conf, client, bus, clock, allocationManagerSource()) {
       override def onOtherEvent(event: SparkListenerEvent): Unit = {
         throw new IllegalStateException("No event should be sent.")
       }
@@ -372,7 +382,7 @@ class ExecutorMonitorSuite extends SparkFunSuite {
   test("shuffle tracking with multiple executors and concurrent jobs") {
     val bus = mockListenerBus()
     conf.set(DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED, true).set(SHUFFLE_SERVICE_ENABLED, false)
-    monitor = new ExecutorMonitor(conf, client, bus, clock)
+    monitor = new ExecutorMonitor(conf, client, bus, clock, allocationManagerSource())
 
     monitor.onExecutorAdded(SparkListenerExecutorAdded(clock.getTimeMillis(), "1", execInfo))
     monitor.onExecutorAdded(SparkListenerExecutorAdded(clock.getTimeMillis(), "2", execInfo))
@@ -417,7 +427,7 @@ class ExecutorMonitorSuite extends SparkFunSuite {
       .set(DYN_ALLOCATION_SHUFFLE_TRACKING_TIMEOUT, Long.MaxValue)
       .set(DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED, true)
       .set(SHUFFLE_SERVICE_ENABLED, false)
-    monitor = new ExecutorMonitor(conf, client, null, clock)
+    monitor = new ExecutorMonitor(conf, client, null, clock, allocationManagerSource())
 
     // Generate events that will make executor 1 be idle, while still holding shuffle data.
     // The executor should not be eligible for removal since the timeout is basically "infinite".
diff --git a/core/src/test/scala/org/apache/spark/security/CryptoStreamUtilsSuite.scala b/core/src/test/scala/org/apache/spark/security/CryptoStreamUtilsSuite.scala
index abccb8e9bbf28..c3d96e7c42af8 100644
--- a/core/src/test/scala/org/apache/spark/security/CryptoStreamUtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/security/CryptoStreamUtilsSuite.scala
@@ -54,14 +54,14 @@ class CryptoStreamUtilsSuite extends SparkFunSuite {
 
   test("shuffle encryption key length should be 128 by default") {
     val conf = createConf()
-    var key = CryptoStreamUtils.createKey(conf)
+    val key = CryptoStreamUtils.createKey(conf)
     val actual = key.length * (java.lang.Byte.SIZE)
     assert(actual === 128)
   }
 
   test("create 256-bit key") {
     val conf = createConf(IO_ENCRYPTION_KEY_SIZE_BITS.key -> "256")
-    var key = CryptoStreamUtils.createKey(conf)
+    val key = CryptoStreamUtils.createKey(conf)
     val actual = key.length * (java.lang.Byte.SIZE)
     assert(actual === 256)
   }
diff --git a/core/src/test/scala/org/apache/spark/serializer/GenericAvroSerializerSuite.scala b/core/src/test/scala/org/apache/spark/serializer/GenericAvroSerializerSuite.scala
index 8610b18702ec0..54e4aebe54430 100644
--- a/core/src/test/scala/org/apache/spark/serializer/GenericAvroSerializerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/GenericAvroSerializerSuite.scala
@@ -22,53 +22,55 @@ import java.nio.ByteBuffer
 
 import com.esotericsoftware.kryo.io.{Input, Output}
 import org.apache.avro.{Schema, SchemaBuilder}
-import org.apache.avro.generic.GenericData.Record
+import org.apache.avro.generic.GenericData.{Array => AvroArray, EnumSymbol, Fixed, Record}
 
 import org.apache.spark.{SharedSparkContext, SparkFunSuite}
 import org.apache.spark.internal.config.SERIALIZER
 
 class GenericAvroSerializerSuite extends SparkFunSuite with SharedSparkContext {
-  conf.set(SERIALIZER, "org.apache.spark.serializer.KryoSerializer")
 
-  val schema : Schema = SchemaBuilder
+  override def beforeAll(): Unit = {
+    conf.set(SERIALIZER, "org.apache.spark.serializer.KryoSerializer")
+    super.beforeAll()
+  }
+
+  val recordSchema : Schema = SchemaBuilder
     .record("testRecord").fields()
     .requiredString("data")
     .endRecord()
-  val record = new Record(schema)
-  record.put("data", "test data")
+  val recordDatum = new Record(recordSchema)
+  recordDatum.put("data", "test data")
 
-  test("schema compression and decompression") {
-    val genericSer = new GenericAvroSerializer(conf.getAvroSchema)
-    assert(schema === genericSer.decompress(ByteBuffer.wrap(genericSer.compress(schema))))
-  }
+  val arraySchema = SchemaBuilder.array().items().`type`(recordSchema)
+  val arrayDatum = new AvroArray[Record](1, arraySchema)
+  arrayDatum.add(recordDatum)
 
-  test("record serialization and deserialization") {
-    val genericSer = new GenericAvroSerializer(conf.getAvroSchema)
+  val enumSchema = SchemaBuilder.enumeration("enum").symbols("A", "B")
+  val enumDatum = new EnumSymbol(enumSchema, "A")
 
-    val outputStream = new ByteArrayOutputStream()
-    val output = new Output(outputStream)
-    genericSer.serializeDatum(record, output)
-    output.flush()
-    output.close()
+  val fixedSchema = SchemaBuilder.fixed("fixed").size(4)
+  val fixedDatum = new Fixed(fixedSchema, "ABCD".getBytes)
 
-    val input = new Input(new ByteArrayInputStream(outputStream.toByteArray))
-    assert(genericSer.deserializeDatum(input) === record)
+  test("schema compression and decompression") {
+    val genericSer = new GenericAvroSerializer(conf.getAvroSchema)
+    assert(recordSchema ===
+      genericSer.decompress(ByteBuffer.wrap(genericSer.compress(recordSchema))))
   }
 
   test("uses schema fingerprint to decrease message size") {
-    val genericSerFull = new GenericAvroSerializer(conf.getAvroSchema)
+    val genericSerFull = new GenericAvroSerializer[Record](conf.getAvroSchema)
 
     val output = new Output(new ByteArrayOutputStream())
 
     val beginningNormalPosition = output.total()
-    genericSerFull.serializeDatum(record, output)
+    genericSerFull.serializeDatum(recordDatum, output)
     output.flush()
     val normalLength = output.total - beginningNormalPosition
 
-    conf.registerAvroSchemas(schema)
-    val genericSerFinger = new GenericAvroSerializer(conf.getAvroSchema)
+    conf.registerAvroSchemas(recordSchema)
+    val genericSerFinger = new GenericAvroSerializer[Record](conf.getAvroSchema)
     val beginningFingerprintPosition = output.total()
-    genericSerFinger.serializeDatum(record, output)
+    genericSerFinger.serializeDatum(recordDatum, output)
     val fingerprintLength = output.total - beginningFingerprintPosition
 
     assert(fingerprintLength < normalLength)
@@ -76,10 +78,36 @@ class GenericAvroSerializerSuite extends SparkFunSuite with SharedSparkContext {
 
   test("caches previously seen schemas") {
     val genericSer = new GenericAvroSerializer(conf.getAvroSchema)
-    val compressedSchema = genericSer.compress(schema)
+    val compressedSchema = genericSer.compress(recordSchema)
     val decompressedSchema = genericSer.decompress(ByteBuffer.wrap(compressedSchema))
 
-    assert(compressedSchema.eq(genericSer.compress(schema)))
+    assert(compressedSchema.eq(genericSer.compress(recordSchema)))
     assert(decompressedSchema.eq(genericSer.decompress(ByteBuffer.wrap(compressedSchema))))
   }
+
+  Seq(
+    ("Record", recordDatum),
+    ("Array", arrayDatum),
+    ("EnumSymbol", enumDatum),
+    ("Fixed", fixedDatum)
+  ).foreach { case (name, datum) =>
+    test(s"SPARK-34477: GenericData.$name serialization and deserialization") {
+      val genericSer = new GenericAvroSerializer[datum.type](conf.getAvroSchema)
+
+      val outputStream = new ByteArrayOutputStream()
+      val output = new Output(outputStream)
+      genericSer.serializeDatum(datum, output)
+      output.flush()
+      output.close()
+
+      val input = new Input(new ByteArrayInputStream(outputStream.toByteArray))
+      assert(genericSer.deserializeDatum(input) === datum)
+    }
+
+    test(s"SPARK-34477: GenericData.$name serialization and deserialization" +
+      " through KryoSerializer ") {
+      val rdd = sc.parallelize((0 until 10).map(_ => datum), 2)
+      assert(rdd.collect() sameElements Array.fill(10)(datum))
+    }
+  }
 }
diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoBenchmark.scala b/core/src/test/scala/org/apache/spark/serializer/KryoBenchmark.scala
index fd228cded783a..1c17d7b1392ab 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoBenchmark.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoBenchmark.scala
@@ -31,7 +31,7 @@ import org.apache.spark.serializer.KryoTest._
  * To run this benchmark:
  * {{{
  *   1. without sbt:
- *      bin/spark-submit --class <this class> --jars <spark core test jar>
+ *      bin/spark-submit --class <this class> <spark core test jar>
  *   2. build/sbt "core/test:runMain <this class>"
  *   3. generate result:
  *      SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "core/test:runMain <this class>"
@@ -125,7 +125,7 @@ object KryoBenchmark extends BenchmarkBase {
   def createSerializer(useUnsafe: Boolean): SerializerInstance = {
     val conf = new SparkConf()
     conf.set(SERIALIZER, "org.apache.spark.serializer.KryoSerializer")
-    conf.set(KRYO_USER_REGISTRATORS, classOf[MyRegistrator].getName)
+    conf.set(KRYO_USER_REGISTRATORS, Seq(classOf[MyRegistrator].getName))
     conf.set(KRYO_USE_UNSAFE, useUnsafe)
 
     new KryoSerializer(conf).newInstance()
diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerBenchmark.scala b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerBenchmark.scala
index 953b651c72a83..7f5dbff46b776 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerBenchmark.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerBenchmark.scala
@@ -34,7 +34,7 @@ import org.apache.spark.util.ThreadUtils
  * To run this benchmark:
  * {{{
  *   1. without sbt:
- *      bin/spark-submit --class <this class> --jars <spark core test jar>
+ *      bin/spark-submit --class <this class> <spark core test jar>
  *   2. build/sbt "core/test:runMain <this class>"
  *   3. generate result:
  *      SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "core/test:runMain <this class>"
@@ -76,7 +76,7 @@ object KryoSerializerBenchmark extends BenchmarkBase {
     conf.set(EXECUTOR_EXTRA_JAVA_OPTIONS,
       "-XX:+UseParallelGC -XX:-UseDynamicNumberOfGCThreads")
     conf.set(SERIALIZER, "org.apache.spark.serializer.KryoSerializer")
-    conf.set(KRYO_USER_REGISTRATORS, classOf[MyRegistrator].getName)
+    conf.set(KRYO_USER_REGISTRATORS, Seq(classOf[MyRegistrator].getName))
     conf.set(KRYO_USE_POOL, usePool)
 
     if (sc != null) {
diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerDistributedSuite.scala b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerDistributedSuite.scala
index d4fafab4a5d64..4acb4bbc779c3 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerDistributedSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerDistributedSuite.scala
@@ -29,9 +29,9 @@ class KryoSerializerDistributedSuite extends SparkFunSuite with LocalSparkContex
   test("kryo objects are serialised consistently in different processes") {
     val conf = new SparkConf(false)
       .set(config.SERIALIZER, "org.apache.spark.serializer.KryoSerializer")
-      .set(config.Kryo.KRYO_USER_REGISTRATORS, classOf[AppJarRegistrator].getName)
+      .set(config.Kryo.KRYO_USER_REGISTRATORS, Seq(classOf[AppJarRegistrator].getName))
       .set(config.TASK_MAX_FAILURES, 1)
-      .set(config.BLACKLIST_ENABLED, false)
+      .set(config.EXCLUDE_ON_FAILURE_ENABLED, false)
 
     val jar = TestUtils.createJarWithClasses(List(AppJarRegistrator.customClassName))
     conf.setJars(List(jar.getPath))
diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
index 4c47a67ee9ffc..229ef69973775 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
@@ -42,7 +42,7 @@ import org.apache.spark.util.ThreadUtils
 
 class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
   conf.set(SERIALIZER, "org.apache.spark.serializer.KryoSerializer")
-  conf.set(KRYO_USER_REGISTRATORS, classOf[MyRegistrator].getName)
+  conf.set(KRYO_USER_REGISTRATORS, Seq(classOf[MyRegistrator].getName))
   conf.set(KRYO_USE_UNSAFE, false)
 
   test("SPARK-7392 configuration limits") {
@@ -313,7 +313,7 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
     import org.apache.spark.SparkException
 
     val conf = new SparkConf(false)
-    conf.set(KRYO_USER_REGISTRATORS, "this.class.does.not.exist")
+    conf.set(KRYO_USER_REGISTRATORS, Seq("this.class.does.not.exist"))
 
     val thrown = intercept[SparkException](new KryoSerializer(conf).newInstance().serialize(1))
     assert(thrown.getMessage.contains("Failed to register classes with Kryo"))
@@ -412,7 +412,7 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
     val ser = new KryoSerializer(new SparkConf).newInstance().asInstanceOf[KryoSerializerInstance]
     assert(ser.getAutoReset)
     val conf = new SparkConf().set(KRYO_USER_REGISTRATORS,
-      classOf[RegistratorWithoutAutoReset].getName)
+      Seq(classOf[RegistratorWithoutAutoReset].getName))
     val ser2 = new KryoSerializer(conf).newInstance().asInstanceOf[KryoSerializerInstance]
     assert(!ser2.getAutoReset)
   }
@@ -443,7 +443,7 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
       .set(KRYO_REFERENCE_TRACKING, referenceTracking)
       .set(KRYO_USE_POOL, usePool)
     if (!autoReset) {
-      conf.set(KRYO_USER_REGISTRATORS, classOf[RegistratorWithoutAutoReset].getName)
+      conf.set(KRYO_USER_REGISTRATORS, Seq(classOf[RegistratorWithoutAutoReset].getName))
     }
     val ser = new KryoSerializer(conf)
     val serInstance = ser.newInstance().asInstanceOf[KryoSerializerInstance]
@@ -530,7 +530,7 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
 
 class KryoSerializerAutoResetDisabledSuite extends SparkFunSuite with SharedSparkContext {
   conf.set(SERIALIZER, classOf[KryoSerializer].getName)
-  conf.set(KRYO_USER_REGISTRATORS, classOf[RegistratorWithoutAutoReset].getName)
+  conf.set(KRYO_USER_REGISTRATORS, Seq(classOf[RegistratorWithoutAutoReset].getName))
   conf.set(KRYO_REFERENCE_TRACKING, true)
   conf.set(SHUFFLE_MANAGER, "sort")
   conf.set(SHUFFLE_SORT_BYPASS_MERGE_THRESHOLD, 200)
diff --git a/core/src/test/scala/org/apache/spark/serializer/SerializerPropertiesSuite.scala b/core/src/test/scala/org/apache/spark/serializer/SerializerPropertiesSuite.scala
index dad080c5fc161..9747f5780dd1e 100644
--- a/core/src/test/scala/org/apache/spark/serializer/SerializerPropertiesSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/SerializerPropertiesSuite.scala
@@ -52,7 +52,7 @@ class SerializerPropertiesSuite extends SparkFunSuite {
 
   test("KryoSerializer does not support relocation when auto-reset is disabled") {
     val conf = new SparkConf().set(KRYO_USER_REGISTRATORS,
-      classOf[RegistratorWithoutAutoReset].getName)
+      Seq(classOf[RegistratorWithoutAutoReset].getName))
     val ser = new KryoSerializer(conf)
     assert(!ser.newInstance().asInstanceOf[KryoSerializerInstance].getAutoReset())
     testSupportsRelocationOfSerializedObjects(ser, generateRandomItem)
diff --git a/core/src/test/scala/org/apache/spark/shuffle/BlockStoreShuffleReaderSuite.scala b/core/src/test/scala/org/apache/spark/shuffle/BlockStoreShuffleReaderSuite.scala
index a82f86a11c77e..d964b28df2983 100644
--- a/core/src/test/scala/org/apache/spark/shuffle/BlockStoreShuffleReaderSuite.scala
+++ b/core/src/test/scala/org/apache/spark/shuffle/BlockStoreShuffleReaderSuite.scala
@@ -104,7 +104,7 @@ class BlockStoreShuffleReaderSuite extends SparkFunSuite with LocalSparkContext
     // shuffle data to read.
     val mapOutputTracker = mock(classOf[MapOutputTracker])
     when(mapOutputTracker.getMapSizesByExecutorId(
-      shuffleId, reduceId, reduceId + 1)).thenReturn {
+      shuffleId, 0, numMaps, reduceId, reduceId + 1)).thenReturn {
       // Test a scenario where all data is local, to avoid creating a bunch of additional mocks
       // for the code to read data over the network.
       val shuffleBlockIdsAndSizes = (0 until numMaps).map { mapId =>
@@ -132,7 +132,7 @@ class BlockStoreShuffleReaderSuite extends SparkFunSuite with LocalSparkContext
     val taskContext = TaskContext.empty()
     val metrics = taskContext.taskMetrics.createTempShuffleReadMetrics()
     val blocksByAddress = mapOutputTracker.getMapSizesByExecutorId(
-      shuffleId, reduceId, reduceId + 1)
+      shuffleId, 0, numMaps, reduceId, reduceId + 1)
     val shuffleReader = new BlockStoreShuffleReader(
       shuffleHandle,
       blocksByAddress,
diff --git a/core/src/test/scala/org/apache/spark/shuffle/HostLocalShuffleReadingSuite.scala b/core/src/test/scala/org/apache/spark/shuffle/HostLocalShuffleReadingSuite.scala
new file mode 100644
index 0000000000000..8f0c4da88feb2
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/shuffle/HostLocalShuffleReadingSuite.scala
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.shuffle
+
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
+
+import org.apache.spark._
+import org.apache.spark.internal.config._
+import org.apache.spark.network.TransportContext
+import org.apache.spark.network.netty.{NettyBlockTransferService, SparkTransportConf}
+import org.apache.spark.network.server.TransportServer
+import org.apache.spark.network.shuffle.{ExternalBlockHandler, ExternalBlockStoreClient}
+import org.apache.spark.util.Utils
+
+/**
+ * This's an end to end test suite used to test the host local shuffle reading.
+ */
+class HostLocalShuffleReadingSuite extends SparkFunSuite with Matchers with LocalSparkContext {
+  var rpcHandler: ExternalBlockHandler = _
+  var server: TransportServer = _
+  var transportContext: TransportContext = _
+
+  override def afterEach(): Unit = {
+    Option(rpcHandler).foreach { handler =>
+      Utils.tryLogNonFatalError{
+        server.close()
+      }
+      Utils.tryLogNonFatalError{
+        handler.close()
+      }
+      Utils.tryLogNonFatalError{
+        transportContext.close()
+      }
+      server = null
+      rpcHandler = null
+      transportContext = null
+    }
+    super.afterEach()
+  }
+
+  Seq(true, false).foreach { isESSEnabled => /* ESS: external shuffle service */
+    val conf = new SparkConf()
+      .set(SHUFFLE_HOST_LOCAL_DISK_READING_ENABLED, true)
+
+    import scala.language.existentials
+    val (essStatus, blockStoreClientClass) = if (isESSEnabled) {
+      // LocalSparkCluster will disable the ExternalShuffleService by default. Therefore,
+      // we have to manually setup an server which embedded with ExternalBlockHandler to
+      // mimic a ExternalShuffleService. Then, executors on the Worker can successfully
+      // find a ExternalShuffleService to connect.
+      val transportConf = SparkTransportConf.fromSparkConf(conf, "shuffle", numUsableCores = 2)
+      rpcHandler = new ExternalBlockHandler(transportConf, null)
+      transportContext = new TransportContext(transportConf, rpcHandler)
+      server = transportContext.createServer()
+      conf.set(SHUFFLE_SERVICE_PORT, server.getPort)
+
+      ("enabled (SPARK-27651)", classOf[ExternalBlockStoreClient])
+    } else {
+      ("disabled (SPARK-32077)", classOf[NettyBlockTransferService])
+    }
+
+    test(s"host local shuffle reading with external shuffle service $essStatus") {
+      conf.set(SHUFFLE_SERVICE_ENABLED, isESSEnabled)
+        .set(STORAGE_LOCAL_DISK_BY_EXECUTORS_CACHE_SIZE, 5)
+      sc = new SparkContext("local-cluster[2,1,1024]", "test-host-local-shuffle-reading", conf)
+      // In a slow machine, one executor may register hundreds of milliseconds ahead of the other
+      // one. If we don't wait for all executors, it's possible that only one executor runs all
+      // jobs. Then all shuffle blocks will be in this executor, ShuffleBlockFetcherIterator will
+      // directly fetch local blocks from the local BlockManager and won't send requests to
+      // BlockStoreClient. In this case, we won't receive FetchFailed. And it will make this
+      // test fail. Therefore, we should wait until all executors are up
+      TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
+
+      sc.getConf.get(SHUFFLE_HOST_LOCAL_DISK_READING_ENABLED) should equal(true)
+      sc.env.blockManager.externalShuffleServiceEnabled should equal(isESSEnabled)
+      sc.env.blockManager.hostLocalDirManager.isDefined should equal(true)
+      sc.env.blockManager.blockStoreClient.getClass should equal(blockStoreClientClass)
+
+      val rdd = sc.parallelize(0 until 1000, 10)
+        .map { i =>
+          SparkEnv.get.blockManager.hostLocalDirManager.map { localDirManager =>
+            // No shuffle fetch yet. So the cache must be empty
+            assert(localDirManager.getCachedHostLocalDirs.isEmpty)
+          }
+         (i, 1)
+        }.reduceByKey(_ + _)
+
+      // raise a job and trigger the shuffle fetching during the job
+      assert(rdd.count() === 1000)
+
+      val cachedExecutors = rdd.mapPartitions { _ =>
+        SparkEnv.get.blockManager.hostLocalDirManager.map { localDirManager =>
+          localDirManager.getCachedHostLocalDirs.keySet.iterator
+        }.getOrElse(Iterator.empty)
+      }.collect().toSet
+
+      // both executors are caching the dirs of the other one
+      cachedExecutors should equal(sc.getExecutorIds().toSet)
+
+      Option(rpcHandler).foreach { handler =>
+        // Invalidate the registered executors, disallowing access to their shuffle blocks (without
+        // deleting the actual shuffle files, so we could access them without the shuffle service).
+        // As directories are already cached there is no request to external shuffle service.
+        handler.applicationRemoved(sc.conf.getAppId, false /* cleanupLocalDirs */)
+      }
+
+      val (local, remote) = rdd.map { case (_, _) =>
+        val shuffleReadMetrics = TaskContext.get().taskMetrics().shuffleReadMetrics
+        ((shuffleReadMetrics.localBytesRead, shuffleReadMetrics.localBlocksFetched),
+        (shuffleReadMetrics.remoteBytesRead, shuffleReadMetrics.remoteBlocksFetched))
+      }.collect().unzip
+      // Spark should read the shuffle data locally from the cached directories on the same host,
+      // so there's no remote fetching at all.
+      val (localBytesRead, localBlocksFetched) = local.unzip
+      val (remoteBytesRead, remoteBlocksFetched) = remote.unzip
+      assert(localBytesRead.sum > 0 && localBlocksFetched.sum > 0)
+      assert(remoteBytesRead.sum === 0 && remoteBlocksFetched.sum === 0)
+    }
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/shuffle/ShuffleBlockPusherSuite.scala b/core/src/test/scala/org/apache/spark/shuffle/ShuffleBlockPusherSuite.scala
new file mode 100644
index 0000000000000..4e51ba4b3a2a5
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/shuffle/ShuffleBlockPusherSuite.scala
@@ -0,0 +1,355 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.shuffle
+
+import java.io.File
+import java.net.ConnectException
+import java.nio.ByteBuffer
+import java.util.concurrent.LinkedBlockingQueue
+
+import scala.collection.mutable.ArrayBuffer
+
+import org.mockito.{Mock, MockitoAnnotations}
+import org.mockito.Answers.RETURNS_SMART_NULLS
+import org.mockito.ArgumentMatchers.any
+import org.mockito.Mockito._
+import org.mockito.invocation.InvocationOnMock
+import org.scalatest.BeforeAndAfterEach
+
+import org.apache.spark._
+import org.apache.spark.network.buffer.ManagedBuffer
+import org.apache.spark.network.shuffle.{BlockFetchingListener, BlockStoreClient}
+import org.apache.spark.network.shuffle.ErrorHandler.BlockPushErrorHandler
+import org.apache.spark.network.util.TransportConf
+import org.apache.spark.serializer.JavaSerializer
+import org.apache.spark.shuffle.ShuffleBlockPusher.PushRequest
+import org.apache.spark.storage._
+
+class ShuffleBlockPusherSuite extends SparkFunSuite with BeforeAndAfterEach {
+
+  @Mock(answer = RETURNS_SMART_NULLS) private var blockManager: BlockManager = _
+  @Mock(answer = RETURNS_SMART_NULLS) private var dependency: ShuffleDependency[Int, Int, Int] = _
+  @Mock(answer = RETURNS_SMART_NULLS) private var shuffleClient: BlockStoreClient = _
+
+  private var conf: SparkConf = _
+  private var pushedBlocks = new ArrayBuffer[String]
+
+  override def beforeEach(): Unit = {
+    super.beforeEach()
+    conf = new SparkConf(loadDefaults = false)
+    MockitoAnnotations.openMocks(this).close()
+    when(dependency.partitioner).thenReturn(new HashPartitioner(8))
+    when(dependency.serializer).thenReturn(new JavaSerializer(conf))
+    when(dependency.getMergerLocs).thenReturn(Seq(BlockManagerId("test-client", "test-client", 1)))
+    conf.set("spark.shuffle.push.based.enabled", "true")
+    conf.set("spark.shuffle.service.enabled", "true")
+    // Set the env because the shuffler writer gets the shuffle client instance from the env.
+    val mockEnv = mock(classOf[SparkEnv])
+    when(mockEnv.conf).thenReturn(conf)
+    when(mockEnv.blockManager).thenReturn(blockManager)
+    SparkEnv.set(mockEnv)
+    when(blockManager.blockStoreClient).thenReturn(shuffleClient)
+  }
+
+  override def afterEach(): Unit = {
+    pushedBlocks.clear()
+    super.afterEach()
+  }
+
+  private def interceptPushedBlocksForSuccess(): Unit = {
+    when(shuffleClient.pushBlocks(any(), any(), any(), any(), any()))
+      .thenAnswer((invocation: InvocationOnMock) => {
+        val blocks = invocation.getArguments()(2).asInstanceOf[Array[String]]
+        pushedBlocks ++= blocks
+        val managedBuffers = invocation.getArguments()(3).asInstanceOf[Array[ManagedBuffer]]
+        val blockFetchListener = invocation.getArguments()(4).asInstanceOf[BlockFetchingListener]
+        (blocks, managedBuffers).zipped.foreach((blockId, buffer) => {
+          blockFetchListener.onBlockFetchSuccess(blockId, buffer)
+        })
+      })
+  }
+
+  private def verifyPushRequests(
+      pushRequests: Seq[PushRequest],
+      expectedSizes: Seq[Int]): Unit = {
+    (pushRequests, expectedSizes).zipped.foreach((req, size) => {
+      assert(req.size == size)
+    })
+  }
+
+  test("A batch of blocks is limited by maxBlocksBatchSize") {
+    conf.set("spark.shuffle.push.maxBlockBatchSize", "1m")
+    conf.set("spark.shuffle.push.maxBlockSizeToPush", "2048k")
+    val blockPusher = new TestShuffleBlockPusher(conf)
+    val mergerLocs = dependency.getMergerLocs.map(loc => BlockManagerId("", loc.host, loc.port))
+    val largeBlockSize = 2 * 1024 * 1024
+    val pushRequests = blockPusher.prepareBlockPushRequests(5, 0, 0,
+      mock(classOf[File]), Array(2, 2, 2, largeBlockSize, largeBlockSize), mergerLocs,
+      mock(classOf[TransportConf]))
+    assert(pushRequests.length == 3)
+    verifyPushRequests(pushRequests, Seq(6, largeBlockSize, largeBlockSize))
+  }
+
+  test("Large blocks are excluded in the preparation") {
+    conf.set("spark.shuffle.push.maxBlockSizeToPush", "1k")
+    val blockPusher = new TestShuffleBlockPusher(conf)
+    val mergerLocs = dependency.getMergerLocs.map(loc => BlockManagerId("", loc.host, loc.port))
+    val pushRequests = blockPusher.prepareBlockPushRequests(5, 0, 0,
+      mock(classOf[File]), Array(2, 2, 2, 1028, 1024), mergerLocs, mock(classOf[TransportConf]))
+    assert(pushRequests.length == 2)
+    verifyPushRequests(pushRequests, Seq(6, 1024))
+  }
+
+  test("Number of blocks in a push request are limited by maxBlocksInFlightPerAddress ") {
+    conf.set("spark.reducer.maxBlocksInFlightPerAddress", "1")
+    val blockPusher = new TestShuffleBlockPusher(conf)
+    val mergerLocs = dependency.getMergerLocs.map(loc => BlockManagerId("", loc.host, loc.port))
+    val pushRequests = blockPusher.prepareBlockPushRequests(5, 0, 0,
+      mock(classOf[File]), Array(2, 2, 2, 2, 2), mergerLocs, mock(classOf[TransportConf]))
+    assert(pushRequests.length == 5)
+    verifyPushRequests(pushRequests, Seq(2, 2, 2, 2, 2))
+  }
+
+  test("Basic block push") {
+    interceptPushedBlocksForSuccess()
+    val blockPusher = new TestShuffleBlockPusher(conf)
+    blockPusher.initiateBlockPush(mock(classOf[File]),
+      Array.fill(dependency.partitioner.numPartitions) { 2 }, dependency, 0)
+    blockPusher.runPendingTasks()
+    verify(shuffleClient, times(1))
+      .pushBlocks(any(), any(), any(), any(), any())
+    assert(pushedBlocks.length == dependency.partitioner.numPartitions)
+    ShuffleBlockPusher.stop()
+  }
+
+  test("Large blocks are skipped for push") {
+    conf.set("spark.shuffle.push.maxBlockSizeToPush", "1k")
+    interceptPushedBlocksForSuccess()
+    val pusher = new TestShuffleBlockPusher(conf)
+    pusher.initiateBlockPush(
+      mock(classOf[File]), Array(2, 2, 2, 2, 2, 2, 2, 1100), dependency, 0)
+    pusher.runPendingTasks()
+    verify(shuffleClient, times(1))
+      .pushBlocks(any(), any(), any(), any(), any())
+    assert(pushedBlocks.length == dependency.partitioner.numPartitions - 1)
+    ShuffleBlockPusher.stop()
+  }
+
+  test("Number of blocks in flight per address are limited by maxBlocksInFlightPerAddress") {
+    conf.set("spark.reducer.maxBlocksInFlightPerAddress", "1")
+    interceptPushedBlocksForSuccess()
+    val pusher = new TestShuffleBlockPusher(conf)
+    pusher.initiateBlockPush(
+      mock(classOf[File]), Array.fill(dependency.partitioner.numPartitions) { 2 }, dependency, 0)
+    pusher.runPendingTasks()
+    verify(shuffleClient, times(8))
+      .pushBlocks(any(), any(), any(), any(), any())
+    assert(pushedBlocks.length == dependency.partitioner.numPartitions)
+    ShuffleBlockPusher.stop()
+  }
+
+  test("Hit maxBlocksInFlightPerAddress limit so that the blocks are deferred") {
+    conf.set("spark.reducer.maxBlocksInFlightPerAddress", "2")
+    var blockPendingResponse : String = null
+    var listener : BlockFetchingListener = null
+    when(shuffleClient.pushBlocks(any(), any(), any(), any(), any()))
+      .thenAnswer((invocation: InvocationOnMock) => {
+        val blocks = invocation.getArguments()(2).asInstanceOf[Array[String]]
+        pushedBlocks ++= blocks
+        val managedBuffers = invocation.getArguments()(3).asInstanceOf[Array[ManagedBuffer]]
+        val blockFetchListener = invocation.getArguments()(4).asInstanceOf[BlockFetchingListener]
+        // Expecting 2 blocks
+        assert(blocks.length == 2)
+        if (blockPendingResponse == null) {
+          blockPendingResponse = blocks(1)
+          listener = blockFetchListener
+          // Respond with success only for the first block which will cause all the rest of the
+          // blocks to be deferred
+          blockFetchListener.onBlockFetchSuccess(blocks(0), managedBuffers(0))
+        } else {
+          (blocks, managedBuffers).zipped.foreach((blockId, buffer) => {
+            blockFetchListener.onBlockFetchSuccess(blockId, buffer)
+          })
+        }
+      })
+    val pusher = new TestShuffleBlockPusher(conf)
+    pusher.initiateBlockPush(
+      mock(classOf[File]), Array.fill(dependency.partitioner.numPartitions) { 2 }, dependency, 0)
+    pusher.runPendingTasks()
+    verify(shuffleClient, times(1))
+      .pushBlocks(any(), any(), any(), any(), any())
+    assert(pushedBlocks.length == 2)
+    // this will trigger push of deferred blocks
+    listener.onBlockFetchSuccess(blockPendingResponse, mock(classOf[ManagedBuffer]))
+    pusher.runPendingTasks()
+    verify(shuffleClient, times(4))
+      .pushBlocks(any(), any(), any(), any(), any())
+    assert(pushedBlocks.length == 8)
+    ShuffleBlockPusher.stop()
+  }
+
+  test("Number of shuffle blocks grouped in a single push request is limited by " +
+      "maxBlockBatchSize") {
+    conf.set("spark.shuffle.push.maxBlockBatchSize", "1m")
+    interceptPushedBlocksForSuccess()
+    val pusher = new TestShuffleBlockPusher(conf)
+    pusher.initiateBlockPush(mock(classOf[File]),
+      Array.fill(dependency.partitioner.numPartitions) { 512 * 1024 }, dependency, 0)
+    pusher.runPendingTasks()
+    verify(shuffleClient, times(4))
+      .pushBlocks(any(), any(), any(), any(), any())
+    assert(pushedBlocks.length == dependency.partitioner.numPartitions)
+    ShuffleBlockPusher.stop()
+  }
+
+  test("Error retries") {
+    val pusher = new ShuffleBlockPusher(conf)
+    val errorHandler = pusher.createErrorHandler()
+    assert(
+      !errorHandler.shouldRetryError(new RuntimeException(
+        new IllegalArgumentException(BlockPushErrorHandler.TOO_LATE_MESSAGE_SUFFIX))))
+    assert(errorHandler.shouldRetryError(new RuntimeException(new ConnectException())))
+    assert(
+      errorHandler.shouldRetryError(new RuntimeException(new IllegalArgumentException(
+        BlockPushErrorHandler.BLOCK_APPEND_COLLISION_DETECTED_MSG_PREFIX))))
+    assert (errorHandler.shouldRetryError(new Throwable()))
+  }
+
+  test("Error logging") {
+    val pusher = new ShuffleBlockPusher(conf)
+    val errorHandler = pusher.createErrorHandler()
+    assert(
+      !errorHandler.shouldLogError(new RuntimeException(
+        new IllegalArgumentException(BlockPushErrorHandler.TOO_LATE_MESSAGE_SUFFIX))))
+    assert(!errorHandler.shouldLogError(new RuntimeException(
+      new IllegalArgumentException(
+        BlockPushErrorHandler.BLOCK_APPEND_COLLISION_DETECTED_MSG_PREFIX))))
+    assert(errorHandler.shouldLogError(new Throwable()))
+  }
+
+  test("Blocks are continued to push even when a block push fails with collision " +
+      "exception") {
+    conf.set("spark.reducer.maxBlocksInFlightPerAddress", "1")
+    val pusher = new TestShuffleBlockPusher(conf)
+    var failBlock: Boolean = true
+    when(shuffleClient.pushBlocks(any(), any(), any(), any(), any()))
+      .thenAnswer((invocation: InvocationOnMock) => {
+        val blocks = invocation.getArguments()(2).asInstanceOf[Array[String]]
+        val blockFetchListener = invocation.getArguments()(4).asInstanceOf[BlockFetchingListener]
+        blocks.foreach(blockId => {
+          if (failBlock) {
+            failBlock = false
+            // Fail the first block with the collision exception.
+            blockFetchListener.onBlockFetchFailure(blockId, new RuntimeException(
+              new IllegalArgumentException(
+                BlockPushErrorHandler.BLOCK_APPEND_COLLISION_DETECTED_MSG_PREFIX)))
+          } else {
+            pushedBlocks += blockId
+            blockFetchListener.onBlockFetchSuccess(blockId, mock(classOf[ManagedBuffer]))
+          }
+        })
+      })
+    pusher.initiateBlockPush(
+      mock(classOf[File]), Array.fill(dependency.partitioner.numPartitions) { 2 }, dependency, 0)
+    pusher.runPendingTasks()
+    verify(shuffleClient, times(8))
+      .pushBlocks(any(), any(), any(), any(), any())
+    assert(pushedBlocks.length == 7)
+  }
+
+  test("More blocks are not pushed when a block push fails with too late " +
+      "exception") {
+    conf.set("spark.reducer.maxBlocksInFlightPerAddress", "1")
+    val pusher = new TestShuffleBlockPusher(conf)
+    var failBlock: Boolean = true
+    when(shuffleClient.pushBlocks(any(), any(), any(), any(), any()))
+      .thenAnswer((invocation: InvocationOnMock) => {
+        val blocks = invocation.getArguments()(2).asInstanceOf[Array[String]]
+        val blockFetchListener = invocation.getArguments()(4).asInstanceOf[BlockFetchingListener]
+        blocks.foreach(blockId => {
+          if (failBlock) {
+            failBlock = false
+            // Fail the first block with the too late exception.
+            blockFetchListener.onBlockFetchFailure(blockId, new RuntimeException(
+              new IllegalArgumentException(BlockPushErrorHandler.TOO_LATE_MESSAGE_SUFFIX)))
+          } else {
+            pushedBlocks += blockId
+            blockFetchListener.onBlockFetchSuccess(blockId, mock(classOf[ManagedBuffer]))
+          }
+        })
+      })
+    pusher.initiateBlockPush(
+      mock(classOf[File]), Array.fill(dependency.partitioner.numPartitions) { 2 }, dependency, 0)
+    pusher.runPendingTasks()
+    verify(shuffleClient, times(1))
+      .pushBlocks(any(), any(), any(), any(), any())
+    assert(pushedBlocks.isEmpty)
+  }
+
+  test("Connect exceptions remove all the push requests for that host") {
+    when(dependency.getMergerLocs).thenReturn(
+      Seq(BlockManagerId("client1", "client1", 1), BlockManagerId("client2", "client2", 2)))
+    conf.set("spark.reducer.maxBlocksInFlightPerAddress", "2")
+    when(shuffleClient.pushBlocks(any(), any(), any(), any(), any()))
+      .thenAnswer((invocation: InvocationOnMock) => {
+        val blocks = invocation.getArguments()(2).asInstanceOf[Array[String]]
+        pushedBlocks ++= blocks
+        val blockFetchListener = invocation.getArguments()(4).asInstanceOf[BlockFetchingListener]
+        blocks.foreach(blockId => {
+          blockFetchListener.onBlockFetchFailure(
+            blockId, new RuntimeException(new ConnectException()))
+        })
+      })
+    val pusher = new TestShuffleBlockPusher(conf)
+    pusher.initiateBlockPush(
+      mock(classOf[File]), Array.fill(dependency.partitioner.numPartitions) { 2 }, dependency, 0)
+    pusher.runPendingTasks()
+    verify(shuffleClient, times(2))
+      .pushBlocks(any(), any(), any(), any(), any())
+    // 2 blocks for each merger locations
+    assert(pushedBlocks.length == 4)
+    assert(pusher.unreachableBlockMgrs.size == 2)
+  }
+
+  private class TestShuffleBlockPusher(conf: SparkConf) extends ShuffleBlockPusher(conf) {
+   private[this] val tasks = new LinkedBlockingQueue[Runnable]
+
+    override protected def submitTask(task: Runnable): Unit = {
+      tasks.add(task)
+    }
+
+    def runPendingTasks(): Unit = {
+      // This ensures that all the submitted tasks - updateStateAndCheckIfPushMore and pushUpToMax
+      // are run synchronously.
+      while (!tasks.isEmpty) {
+        tasks.take().run()
+      }
+    }
+
+    override protected def createRequestBuffer(
+        conf: TransportConf,
+        dataFile: File,
+        offset: Long,
+        length: Long): ManagedBuffer = {
+      val managedBuffer = mock(classOf[ManagedBuffer])
+      val byteBuffer = new Array[Byte](length.toInt)
+      when(managedBuffer.nioByteBuffer()).thenReturn(ByteBuffer.wrap(byteBuffer))
+      managedBuffer
+    }
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriterSuite.scala b/core/src/test/scala/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriterSuite.scala
index f8474022867f4..7fd0bf626fda1 100644
--- a/core/src/test/scala/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriterSuite.scala
+++ b/core/src/test/scala/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriterSuite.scala
@@ -59,7 +59,7 @@ class BypassMergeSortShuffleWriterSuite extends SparkFunSuite with BeforeAndAfte
 
   override def beforeEach(): Unit = {
     super.beforeEach()
-    MockitoAnnotations.initMocks(this)
+    MockitoAnnotations.openMocks(this).close()
     tempDir = Utils.createTempDir()
     outputFile = File.createTempFile("shuffle", null, tempDir)
     taskMetrics = new TaskMetrics
diff --git a/core/src/test/scala/org/apache/spark/shuffle/sort/IndexShuffleBlockResolverSuite.scala b/core/src/test/scala/org/apache/spark/shuffle/sort/IndexShuffleBlockResolverSuite.scala
index 27bb06b4e0636..da98ad3d1c982 100644
--- a/core/src/test/scala/org/apache/spark/shuffle/sort/IndexShuffleBlockResolverSuite.scala
+++ b/core/src/test/scala/org/apache/spark/shuffle/sort/IndexShuffleBlockResolverSuite.scala
@@ -27,7 +27,7 @@ import org.mockito.invocation.InvocationOnMock
 import org.scalatest.BeforeAndAfterEach
 
 import org.apache.spark.{SparkConf, SparkFunSuite}
-import org.apache.spark.shuffle.IndexShuffleBlockResolver
+import org.apache.spark.shuffle.{IndexShuffleBlockResolver, ShuffleBlockInfo}
 import org.apache.spark.storage._
 import org.apache.spark.util.Utils
 
@@ -43,11 +43,12 @@ class IndexShuffleBlockResolverSuite extends SparkFunSuite with BeforeAndAfterEa
   override def beforeEach(): Unit = {
     super.beforeEach()
     tempDir = Utils.createTempDir()
-    MockitoAnnotations.initMocks(this)
+    MockitoAnnotations.openMocks(this).close()
 
     when(blockManager.diskBlockManager).thenReturn(diskBlockManager)
     when(diskBlockManager.getFile(any[BlockId])).thenAnswer(
       (invocation: InvocationOnMock) => new File(tempDir, invocation.getArguments.head.toString))
+    when(diskBlockManager.localDirs).thenReturn(Array(tempDir))
   }
 
   override def afterEach(): Unit = {
@@ -155,4 +156,9 @@ class IndexShuffleBlockResolverSuite extends SparkFunSuite with BeforeAndAfterEa
       indexIn2.close()
     }
   }
+
+  test("SPARK-33198 getMigrationBlocks should not fail at missing files") {
+    val resolver = new IndexShuffleBlockResolver(conf, blockManager)
+    assert(resolver.getMigrationBlocks(ShuffleBlockInfo(Int.MaxValue, Long.MaxValue)).isEmpty)
+  }
 }
diff --git a/core/src/test/scala/org/apache/spark/shuffle/sort/SortShuffleManagerSuite.scala b/core/src/test/scala/org/apache/spark/shuffle/sort/SortShuffleManagerSuite.scala
index e5f3aab6a6a1a..f75db20808bee 100644
--- a/core/src/test/scala/org/apache/spark/shuffle/sort/SortShuffleManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/shuffle/sort/SortShuffleManagerSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.shuffle.sort
 import org.mockito.Mockito.{mock, when}
 import org.mockito.invocation.InvocationOnMock
 import org.mockito.stubbing.Answer
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
 
 import org.apache.spark._
 import org.apache.spark.serializer.{JavaSerializer, KryoSerializer, Serializer}
diff --git a/core/src/test/scala/org/apache/spark/shuffle/sort/SortShuffleWriterSuite.scala b/core/src/test/scala/org/apache/spark/shuffle/sort/SortShuffleWriterSuite.scala
index 4c5694fcf0305..4c679fd874c9b 100644
--- a/core/src/test/scala/org/apache/spark/shuffle/sort/SortShuffleWriterSuite.scala
+++ b/core/src/test/scala/org/apache/spark/shuffle/sort/SortShuffleWriterSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.shuffle.sort
 import org.mockito.{Mock, MockitoAnnotations}
 import org.mockito.Answers.RETURNS_SMART_NULLS
 import org.mockito.Mockito._
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
 
 import org.apache.spark.{Partitioner, SharedSparkContext, ShuffleDependency, SparkFunSuite}
 import org.apache.spark.memory.MemoryTestingUtils
@@ -46,7 +46,7 @@ class SortShuffleWriterSuite extends SparkFunSuite with SharedSparkContext with
 
   override def beforeEach(): Unit = {
     super.beforeEach()
-    MockitoAnnotations.initMocks(this)
+    MockitoAnnotations.openMocks(this).close()
     val partitioner = new Partitioner() {
       def numPartitions = numMaps
       def getPartition(key: Any) = Utils.nonNegativeMod(key.hashCode, numPartitions)
@@ -74,7 +74,6 @@ class SortShuffleWriterSuite extends SparkFunSuite with SharedSparkContext with
   test("write empty iterator") {
     val context = MemoryTestingUtils.fakeTaskContext(sc.env)
     val writer = new SortShuffleWriter[Int, Int, Int](
-      shuffleBlockResolver,
       shuffleHandle,
       mapId = 1,
       context,
@@ -92,7 +91,6 @@ class SortShuffleWriterSuite extends SparkFunSuite with SharedSparkContext with
     val context = MemoryTestingUtils.fakeTaskContext(sc.env)
     val records = List[(Int, Int)]((1, 2), (2, 3), (4, 4), (6, 5))
     val writer = new SortShuffleWriter[Int, Int, Int](
-      shuffleBlockResolver,
       shuffleHandle,
       mapId = 2,
       context,
diff --git a/core/src/test/scala/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriterSuite.scala b/core/src/test/scala/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriterSuite.scala
index f92455912f510..ef5c615bf7591 100644
--- a/core/src/test/scala/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriterSuite.scala
+++ b/core/src/test/scala/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriterSuite.scala
@@ -65,7 +65,7 @@ class LocalDiskShuffleMapOutputWriterSuite extends SparkFunSuite with BeforeAndA
   }
 
   override def beforeEach(): Unit = {
-    MockitoAnnotations.initMocks(this)
+    MockitoAnnotations.openMocks(this).close()
     tempDir = Utils.createTempDir()
     mergedOutputFile = File.createTempFile("mergedoutput", "", tempDir)
     tempFile = File.createTempFile("tempfile", "", tempDir)
@@ -136,7 +136,7 @@ class LocalDiskShuffleMapOutputWriterSuite extends SparkFunSuite with BeforeAndA
   }
 
   private def verifyWrittenRecords(): Unit = {
-    val committedLengths = mapOutputWriter.commitAllPartitions()
+    val committedLengths = mapOutputWriter.commitAllPartitions().getPartitionLengths
     assert(partitionSizesInMergedFile === partitionLengths)
     assert(committedLengths === partitionLengths)
     assert(mergedOutputFile.length() === partitionLengths.sum)
diff --git a/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala b/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
index 24eb1685f577a..424b328ce3106 100644
--- a/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
@@ -234,7 +234,7 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter {
 
     // Send two executor metrics update. Only update one metric to avoid a lot of boilerplate code.
     // The tasks are distributed among the two executors, so the executor-level metrics should
-    // hold half of the cummulative value of the metric being updated.
+    // hold half of the cumulative value of the metric being updated.
     Seq(1L, 2L).foreach { value =>
       s1Tasks.foreach { task =>
         val accum = new AccumulableInfo(1L, Some(InternalAccumulator.MEMORY_BYTES_SPILLED),
@@ -256,9 +256,9 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter {
       }
     }
 
-    // Blacklisting executor for stage
+    // Excluding executor for stage
     time += 1
-    listener.onExecutorBlacklistedForStage(SparkListenerExecutorBlacklistedForStage(
+    listener.onExecutorExcludedForStage(SparkListenerExecutorExcludedForStage(
       time = time,
       executorId = execIds.head,
       taskFailures = 2,
@@ -273,18 +273,21 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter {
 
     assert(executorStageSummaryWrappers.nonEmpty)
     executorStageSummaryWrappers.foreach { exec =>
-      // only the first executor is expected to be blacklisted
-      val expectedBlacklistedFlag = exec.executorId == execIds.head
-      assert(exec.info.isBlacklistedForStage === expectedBlacklistedFlag)
+      // only the first executor is expected to be excluded
+      val expectedExcludedFlag = exec.executorId == execIds.head
+      assert(exec.info.isBlacklistedForStage === expectedExcludedFlag)
+      assert(exec.info.isExcludedForStage === expectedExcludedFlag)
     }
 
     check[ExecutorSummaryWrapper](execIds.head) { exec =>
       assert(exec.info.blacklistedInStages === Set(stages.head.stageId))
+      assert(exec.info.excludedInStages === Set(stages.head.stageId))
+
     }
 
-    // Blacklisting node for stage
+    // Excluding node for stage
     time += 1
-    listener.onNodeBlacklistedForStage(SparkListenerNodeBlacklistedForStage(
+    listener.onNodeExcludedForStage(SparkListenerNodeExcludedForStage(
       time = time,
       hostId = "2.example.com", // this is where the second executor is hosted
       executorFailures = 1,
@@ -299,8 +302,10 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter {
 
     assert(executorStageSummaryWrappersForNode.nonEmpty)
     executorStageSummaryWrappersForNode.foreach { exec =>
-      // both executor is expected to be blacklisted
+      // both executor is expected to be excluded
       assert(exec.info.isBlacklistedForStage)
+      assert(exec.info.isExcludedForStage)
+
     }
 
     // Fail one of the tasks, re-start it.
@@ -450,6 +455,7 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter {
 
     check[ExecutorSummaryWrapper](execIds.head) { exec =>
       assert(exec.info.blacklistedInStages === Set())
+      assert(exec.info.excludedInStages === Set())
     }
 
     // Submit stage 2.
@@ -466,9 +472,9 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter {
       assert(stage.info.submissionTime === Some(new Date(stages.last.submissionTime.get)))
     }
 
-    // Blacklisting node for stage
+    // Excluding node for stage
     time += 1
-    listener.onNodeBlacklistedForStage(SparkListenerNodeBlacklistedForStage(
+    listener.onNodeExcludedForStage(SparkListenerNodeExcludedForStage(
       time = time,
       hostId = "1.example.com",
       executorFailures = 1,
@@ -477,6 +483,7 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter {
 
     check[ExecutorSummaryWrapper](execIds.head) { exec =>
       assert(exec.info.blacklistedInStages === Set(stages.last.stageId))
+      assert(exec.info.excludedInStages === Set(stages.last.stageId))
     }
 
     // Start and fail all tasks of stage 2.
@@ -628,30 +635,34 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter {
       assert(job.info.numSkippedTasks === s1Tasks.size)
     }
 
-    // Blacklist an executor.
+    // Exclude an executor.
     time += 1
-    listener.onExecutorBlacklisted(SparkListenerExecutorBlacklisted(time, "1", 42))
+    listener.onExecutorExcluded(SparkListenerExecutorExcluded(time, "1", 42))
     check[ExecutorSummaryWrapper]("1") { exec =>
       assert(exec.info.isBlacklisted)
+      assert(exec.info.isExcluded)
     }
 
     time += 1
-    listener.onExecutorUnblacklisted(SparkListenerExecutorUnblacklisted(time, "1"))
+    listener.onExecutorUnexcluded(SparkListenerExecutorUnexcluded(time, "1"))
     check[ExecutorSummaryWrapper]("1") { exec =>
       assert(!exec.info.isBlacklisted)
+      assert(!exec.info.isExcluded)
     }
 
-    // Blacklist a node.
+    // Exclude a node.
     time += 1
-    listener.onNodeBlacklisted(SparkListenerNodeBlacklisted(time, "1.example.com", 2))
+    listener.onNodeExcluded(SparkListenerNodeExcluded(time, "1.example.com", 2))
     check[ExecutorSummaryWrapper]("1") { exec =>
       assert(exec.info.isBlacklisted)
+      assert(exec.info.isExcluded)
     }
 
     time += 1
-    listener.onNodeUnblacklisted(SparkListenerNodeUnblacklisted(time, "1.example.com"))
+    listener.onNodeUnexcluded(SparkListenerNodeUnexcluded(time, "1.example.com"))
     check[ExecutorSummaryWrapper]("1") { exec =>
       assert(!exec.info.isBlacklisted)
+      assert(!exec.info.isExcluded)
     }
 
     // Stop executors.
@@ -1523,14 +1534,32 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter {
         assert(exec.info.id === id)
         exec.info.peakMemoryMetrics match {
           case Some(actual) =>
-            ExecutorMetricType.metricToOffset.foreach { metric =>
-              assert(actual.getMetricValue(metric._1) === metrics.getMetricValue(metric._1))
-            }
+            checkExecutorMetrics(metrics, actual)
           case _ =>
             assert(false)
         }
       }
     }
+
+    // check stage level executor metrics
+    val expectedStageValues = Map(
+      0 -> StageExecutorMetrics(
+        new ExecutorMetrics(Array(7000L, 80L, 50L, 20L, 50L, 10L, 100L, 30L,
+          80L, 40L, 9000L, 4000L, 8000L, 3000L, 7000L, 2000L)),
+        Map(
+          "1" -> new ExecutorMetrics(Array(5000L, 50L, 50L, 20L, 50L, 10L, 100L, 30L,
+            70L, 20L, 8000L, 4000L, 7000L, 3000L, 6000L, 2000L)),
+          "2" -> new ExecutorMetrics(Array(7000L, 80L, 50L, 20L, 10L, 10L, 50L, 30L,
+            80L, 40L, 9000L, 4000L, 8000L, 3000L, 7000L, 2000L)))),
+      1 -> StageExecutorMetrics(
+        new ExecutorMetrics(Array(7000L, 70L, 25L, 40L, 60L, 30L, 70L, 60L,
+          40L, 20L, 8000L, 5000L, 7000L, 4000L, 6000L, 3000L)),
+        Map(
+          "1" -> new ExecutorMetrics(Array(7000L, 70L, 20L, 30L, 60L, 30L, 70L, 55L,
+            30L, 0L, 5000L, 3000L, 4000L, 2000L, 3000L, 1000L)),
+          "2" -> new ExecutorMetrics(Array(5500L, 40L, 25L, 40L, 10L, 30L, 35L, 60L,
+            40L, 20L, 8000L, 5000L, 7000L, 4000L, 6000L, 3000L)))))
+    checkStageExecutorMetrics(expectedStageValues)
   }
 
   test("stage executor metrics") {
@@ -1573,14 +1602,74 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter {
         assert(exec.info.id === id)
         exec.info.peakMemoryMetrics match {
           case Some(actual) =>
-            ExecutorMetricType.metricToOffset.foreach { metric =>
-              assert(actual.getMetricValue(metric._1) === metrics.getMetricValue(metric._1))
-            }
+            checkExecutorMetrics(metrics, actual)
           case _ =>
             assert(false)
         }
       }
     }
+
+    // check stage level executor metrics
+    val expectedStageValues = Map(
+      0 -> StageExecutorMetrics(
+        new ExecutorMetrics(Array(7000L, 70L, 50L, 20L, 50L, 10L, 100L, 30L,
+          80L, 40L, 9000L, 4000L, 8000L, 3000L, 7000L, 2000L)),
+        Map(
+          "1" -> new ExecutorMetrics(Array(5000L, 50L, 50L, 20L, 50L, 10L, 100L, 30L,
+            70L, 20L, 8000L, 4000L, 7000L, 3000L, 6000L, 2000L)),
+          "2" -> new ExecutorMetrics(Array(7000L, 70L, 50L, 20L, 10L, 10L, 50L, 30L,
+            80L, 40L, 9000L, 4000L, 8000L, 3000L, 7000L, 2000L)))),
+      1 -> StageExecutorMetrics(
+        new ExecutorMetrics(Array(7000L, 80L, 50L, 40L, 60L, 30L, 80L, 60L,
+          50L, 40L, 8000L, 5000L, 7000L, 4000L, 6000L, 3000L)),
+        Map(
+          "1" -> new ExecutorMetrics(Array(7000L, 70L, 50L, 30L, 60L, 30L, 80L, 55L,
+            50L, 0L, 5000L, 3000L, 4000L, 2000L, 3000L, 1000L)),
+          "2" -> new ExecutorMetrics(Array(7000L, 80L, 50L, 40L, 10L, 30L, 50L, 60L,
+            40L, 40L, 8000L, 5000L, 7000L, 4000L, 6000L, 3000L)))))
+    checkStageExecutorMetrics(expectedStageValues)
+  }
+
+  /** expected stage executor metrics */
+  private case class StageExecutorMetrics(
+      peakExecutorMetrics: ExecutorMetrics,
+      executorMetrics: Map[String, ExecutorMetrics])
+
+  private def checkExecutorMetrics(expected: ExecutorMetrics, actual: ExecutorMetrics): Unit = {
+    ExecutorMetricType.metricToOffset.foreach { metric =>
+      assert(actual.getMetricValue(metric._1) === expected.getMetricValue(metric._1))
+    }
+  }
+
+  /** check stage level peak executor metric values, and executor peak values for each stage */
+  private def checkStageExecutorMetrics(expectedStageValues: Map[Int, StageExecutorMetrics]) = {
+    // check stage level peak executor metric values for each stage
+    for ((stageId, expectedMetrics) <- expectedStageValues) {
+      check[StageDataWrapper](Array(stageId, 0)) { stage =>
+        stage.info.peakExecutorMetrics match {
+          case Some(actual) =>
+            checkExecutorMetrics(expectedMetrics.peakExecutorMetrics, actual)
+          case None =>
+            assert(false)
+        }
+      }
+    }
+
+    // check peak executor metric values for each stage and executor
+    val stageExecSummaries = store.view(classOf[ExecutorStageSummaryWrapper]).asScala.toSeq
+    stageExecSummaries.foreach { exec =>
+      expectedStageValues.get(exec.stageId) match {
+        case Some(stageValue) =>
+          (stageValue.executorMetrics.get(exec.executorId), exec.info.peakMemoryMetrics) match {
+            case (Some(expected), Some(actual)) =>
+              checkExecutorMetrics(expected, actual)
+            case _ =>
+              assert(false)
+          }
+        case None =>
+          assert(false)
+      }
+    }
   }
 
   test("storage information on executor lost/down") {
@@ -1707,6 +1796,44 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter {
     }
   }
 
+  test("SPARK-34877 - check YarnAmInfoEvent is populated correctly") {
+    def checkInfoPopulated(listener: AppStatusListener,
+       logUrlMap: Map[String, String], processId: String): Unit = {
+      val yarnAmInfo = listener.liveMiscellaneousProcess.get(processId)
+      assert(yarnAmInfo.isDefined)
+      yarnAmInfo.foreach { info =>
+        assert(info.processId == processId)
+        assert(info.isActive)
+        assert(info.processLogs == logUrlMap)
+      }
+      check[ProcessSummaryWrapper](processId) { process =>
+        assert(process.info.id === processId)
+        assert(process.info.isActive)
+        assert(process.info.processLogs == logUrlMap)
+      }
+    }
+    val processId = "yarn-am"
+    val listener = new AppStatusListener(store, conf, true)
+    var stdout = "http:yarnAmHost:2453/con1/stdout"
+    var stderr = "http:yarnAmHost:2453/con2/stderr"
+    var logUrlMap: Map[String, String] = Map("stdout" -> stdout,
+      "stderr" -> stderr)
+    var hostport = "yarnAmHost:2453"
+    var info = new MiscellaneousProcessDetails(hostport, 1, logUrlMap)
+    listener.onOtherEvent(SparkListenerMiscellaneousProcessAdded(123678L, processId, info))
+    checkInfoPopulated(listener, logUrlMap, processId)
+
+    // Launch new AM in case of failure
+    // New container entry will be updated in this scenario
+    stdout = "http:yarnAmHost:2451/con1/stdout"
+    stderr = "http:yarnAmHost:2451/con2/stderr"
+    logUrlMap = Map("stdout" -> stdout,
+      "stderr" -> stderr)
+    hostport = "yarnAmHost:2451"
+    info = new MiscellaneousProcessDetails(hostport, 1, logUrlMap)
+    listener.onOtherEvent(SparkListenerMiscellaneousProcessAdded(123678L, processId, info))
+    checkInfoPopulated(listener, logUrlMap, processId)
+  }
 
   private def key(stage: StageInfo): Array[Int] = Array(stage.stageId, stage.attemptNumber)
 
diff --git a/core/src/test/scala/org/apache/spark/status/ElementTrackingStoreSuite.scala b/core/src/test/scala/org/apache/spark/status/ElementTrackingStoreSuite.scala
index 38e88e6a012c4..98d528b621605 100644
--- a/core/src/test/scala/org/apache/spark/status/ElementTrackingStoreSuite.scala
+++ b/core/src/test/scala/org/apache/spark/status/ElementTrackingStoreSuite.scala
@@ -20,8 +20,8 @@ package org.apache.spark.status
 import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger}
 
 import org.mockito.Mockito._
-import org.scalatest.Matchers._
 import org.scalatest.concurrent.Eventually
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.internal.config.Status._
@@ -35,8 +35,8 @@ class ElementTrackingStoreSuite extends SparkFunSuite with Eventually {
     val tracking = new ElementTrackingStore(store, new SparkConf()
       .set(ASYNC_TRACKING_ENABLED, true))
 
-    var done = new AtomicBoolean(false)
-    var type1 = new AtomicInteger(0)
+    val done = new AtomicBoolean(false)
+    val type1 = new AtomicInteger(0)
     var queued0: WriteQueueResult = null
     var queued1: WriteQueueResult = null
     var queued2: WriteQueueResult = null
diff --git a/core/src/test/scala/org/apache/spark/status/api/v1/ExecutorSummarySuite.scala b/core/src/test/scala/org/apache/spark/status/api/v1/ExecutorSummarySuite.scala
new file mode 100644
index 0000000000000..541a7821a51fb
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/status/api/v1/ExecutorSummarySuite.scala
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.status.api.v1
+
+import java.util.Date
+
+import com.fasterxml.jackson.core.`type`.TypeReference
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.module.scala.DefaultScalaModule
+
+import org.apache.spark.SparkFunSuite
+
+class ExecutorSummarySuite extends SparkFunSuite {
+
+  test("Check ExecutorSummary serialize and deserialize with empty peakMemoryMetrics") {
+    val mapper = new ObjectMapper().registerModule(DefaultScalaModule)
+    val executorSummary = new ExecutorSummary("id", "host:port", true, 1,
+      10, 10, 1, 1, 1,
+      0, 0, 1, 100,
+      1, 100, 100,
+      10, false, 20, new Date(1600984336352L),
+      Option.empty, Option.empty, Map(), Option.empty, Set(), Option.empty, Map(), Map(), 1,
+      false, Set())
+    val expectedJson = "{\"id\":\"id\",\"hostPort\":\"host:port\",\"isActive\":true," +
+      "\"rddBlocks\":1,\"memoryUsed\":10,\"diskUsed\":10,\"totalCores\":1,\"maxTasks\":1," +
+      "\"activeTasks\":1,\"failedTasks\":0,\"completedTasks\":0,\"totalTasks\":1," +
+      "\"totalDuration\":100,\"totalGCTime\":1,\"totalInputBytes\":100," +
+      "\"totalShuffleRead\":100,\"totalShuffleWrite\":10,\"isBlacklisted\":false," +
+      "\"maxMemory\":20,\"addTime\":1600984336352,\"removeTime\":null,\"removeReason\":null," +
+      "\"executorLogs\":{},\"memoryMetrics\":null,\"blacklistedInStages\":[]," +
+      "\"peakMemoryMetrics\":null,\"attributes\":{},\"resources\":{},\"resourceProfileId\":1," +
+      "\"isExcluded\":false,\"excludedInStages\":[]}"
+    val json = mapper.writeValueAsString(executorSummary)
+    assert(expectedJson.equals(json))
+    val deserializeExecutorSummary = mapper.readValue(json, new TypeReference[ExecutorSummary] {})
+    assert(deserializeExecutorSummary.peakMemoryMetrics == None)
+  }
+
+}
diff --git a/core/src/test/scala/org/apache/spark/status/api/v1/SimpleDateParamSuite.scala b/core/src/test/scala/org/apache/spark/status/api/v1/SimpleDateParamSuite.scala
index 18baeb1cb9c71..ff2060e278465 100644
--- a/core/src/test/scala/org/apache/spark/status/api/v1/SimpleDateParamSuite.scala
+++ b/core/src/test/scala/org/apache/spark/status/api/v1/SimpleDateParamSuite.scala
@@ -18,7 +18,8 @@ package org.apache.spark.status.api.v1
 
 import javax.ws.rs.WebApplicationException
 
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.SparkFunSuite
 
diff --git a/core/src/test/scala/org/apache/spark/storage/BlockIdSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockIdSuite.scala
index ef7b13875540f..d7009e6ed5799 100644
--- a/core/src/test/scala/org/apache/spark/storage/BlockIdSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/BlockIdSuite.scala
@@ -87,7 +87,7 @@ class BlockIdSuite extends SparkFunSuite {
     assert(id.shuffleId === 4)
     assert(id.mapId === 5)
     assert(id.reduceId === 6)
-    assert(!id.isShuffle)
+    assert(id.isShuffle)
     assertSame(id, BlockId(id.toString))
   }
 
@@ -100,7 +100,7 @@ class BlockIdSuite extends SparkFunSuite {
     assert(id.shuffleId === 7)
     assert(id.mapId === 8)
     assert(id.reduceId === 9)
-    assert(!id.isShuffle)
+    assert(id.isShuffle)
     assertSame(id, BlockId(id.toString))
   }
 
diff --git a/core/src/test/scala/org/apache/spark/storage/BlockInfoManagerSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockInfoManagerSuite.scala
index 9c0699bc981f8..d2bf385e10796 100644
--- a/core/src/test/scala/org/apache/spark/storage/BlockInfoManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/BlockInfoManagerSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.storage
 
 import java.util.Properties
 
-import scala.concurrent.{Await, ExecutionContext, Future}
+import scala.concurrent.{ExecutionContext, Future}
 import scala.language.implicitConversions
 import scala.reflect.ClassTag
 
diff --git a/core/src/test/scala/org/apache/spark/storage/BlockManagerDecommissionIntegrationSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockManagerDecommissionIntegrationSuite.scala
new file mode 100644
index 0000000000000..e461474294f38
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/storage/BlockManagerDecommissionIntegrationSuite.scala
@@ -0,0 +1,334 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.storage
+
+import java.util.concurrent.{ConcurrentHashMap, ConcurrentLinkedQueue, Semaphore, TimeUnit}
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+import scala.concurrent.duration._
+
+import org.scalatest.concurrent.Eventually
+
+import org.apache.spark._
+import org.apache.spark.internal.config
+import org.apache.spark.scheduler._
+import org.apache.spark.scheduler.cluster.StandaloneSchedulerBackend
+import org.apache.spark.util.{ResetSystemProperties, SystemClock, ThreadUtils}
+
+class BlockManagerDecommissionIntegrationSuite extends SparkFunSuite with LocalSparkContext
+    with ResetSystemProperties with Eventually {
+
+  val numExecs = 3
+  val numParts = 3
+  val TaskStarted = "TASK_STARTED"
+  val TaskEnded = "TASK_ENDED"
+  val JobEnded = "JOB_ENDED"
+
+  Seq(false, true).foreach { isEnabled =>
+    test(s"SPARK-32850: BlockManager decommission should respect the configuration " +
+      s"(enabled=${isEnabled})") {
+      val conf = new SparkConf()
+        .setAppName("test-blockmanager-decommissioner")
+        .setMaster("local-cluster[2, 1, 1024]")
+        .set(config.DECOMMISSION_ENABLED, true)
+        .set(config.STORAGE_DECOMMISSION_ENABLED, isEnabled)
+      sc = new SparkContext(conf)
+      TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
+      val executors = sc.getExecutorIds().toArray
+      val decommissionListener = new SparkListener {
+        override def onTaskStart(taskStart: SparkListenerTaskStart): Unit = {
+          // ensure Tasks launched at executors before they're marked as decommissioned by driver
+          Thread.sleep(3000)
+          sc.schedulerBackend.asInstanceOf[StandaloneSchedulerBackend]
+            .decommissionExecutors(
+              executors.map { id => (id, ExecutorDecommissionInfo("test")) },
+              true,
+              false)
+        }
+      }
+      sc.addSparkListener(decommissionListener)
+
+      val decommissionStatus: Seq[Boolean] = sc.parallelize(1 to 100, 2).mapPartitions { _ =>
+        val startTime = System.currentTimeMillis()
+        while (SparkEnv.get.blockManager.decommissioner.isEmpty &&
+          // wait at most 6 seconds for BlockManager to start to decommission (if enabled)
+          System.currentTimeMillis() - startTime < 6000) {
+          Thread.sleep(300)
+        }
+        val blockManagerDecommissionStatus =
+          if (SparkEnv.get.blockManager.decommissioner.isEmpty) false else true
+        Iterator.single(blockManagerDecommissionStatus)
+      }.collect()
+      assert(decommissionStatus.forall(_ == isEnabled))
+      sc.removeSparkListener(decommissionListener)
+    }
+  }
+
+  testRetry(s"verify that an already running task which is going to cache data succeeds " +
+    s"on a decommissioned executor after task start") {
+    runDecomTest(true, false, TaskStarted)
+  }
+
+  test(s"verify that an already running task which is going to cache data succeeds " +
+    s"on a decommissioned executor after one task ends but before job ends") {
+    runDecomTest(true, false, TaskEnded)
+  }
+
+  test(s"verify that shuffle blocks are migrated") {
+    runDecomTest(false, true, JobEnded)
+  }
+
+  test(s"verify that both migrations can work at the same time") {
+    runDecomTest(true, true, JobEnded)
+  }
+
+  private def runDecomTest(
+      persist: Boolean,
+      shuffle: Boolean,
+      whenToDecom: String): Unit = {
+    val migrateDuring = whenToDecom != JobEnded
+    val master = s"local-cluster[${numExecs}, 1, 1024]"
+    val conf = new SparkConf().setAppName("test").setMaster(master)
+      .set(config.DECOMMISSION_ENABLED, true)
+      .set(config.STORAGE_DECOMMISSION_ENABLED, true)
+      .set(config.STORAGE_DECOMMISSION_RDD_BLOCKS_ENABLED, persist)
+      .set(config.STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED, shuffle)
+      // Since we use the bus for testing we don't want to drop any messages
+      .set(config.LISTENER_BUS_EVENT_QUEUE_CAPACITY, 1000000)
+      // Just replicate blocks quickly during testing, there isn't another
+      // workload we need to worry about.
+      .set(config.STORAGE_DECOMMISSION_REPLICATION_REATTEMPT_INTERVAL, 10L)
+
+    if (whenToDecom == TaskStarted) {
+      // We are using accumulators below, make sure those are reported frequently.
+      conf.set(config.EXECUTOR_HEARTBEAT_INTERVAL.key, "10ms")
+    }
+    sc = new SparkContext(master, "test", conf)
+
+    // Wait for the executors to start
+    TestUtils.waitUntilExecutorsUp(sc = sc,
+      numExecutors = numExecs,
+      timeout = 60000) // 60s
+
+    val input = sc.parallelize(1 to numParts, numParts)
+    val accum = sc.longAccumulator("mapperRunAccumulator")
+
+    val sleepIntervalMs = whenToDecom match {
+      // Increase the window of time b/w task started and ended so that we can decom within that.
+      case TaskStarted => 10000
+      // Make one task take a really short time so that we can decommission right after it is
+      // done but before its peers are done.
+      case TaskEnded =>
+        if (TaskContext.getPartitionId() == 0) {
+          100
+        } else {
+          1000
+        }
+      // No sleep otherwise
+      case _ => 0
+    }
+
+    // Create a new RDD where we have sleep in each partition, we are also increasing
+    // the value of accumulator in each partition
+    val baseRdd = input.mapPartitions { x =>
+      accum.add(1)
+      if (sleepIntervalMs > 0) {
+        Thread.sleep(sleepIntervalMs)
+      }
+      x.map(y => (y, y))
+    }
+    val testRdd = shuffle match {
+      case true => baseRdd.reduceByKey(_ + _)
+      case false => baseRdd
+    }
+
+    // Listen for the job & block updates
+    val executorRemovedSem = new Semaphore(0)
+    val taskEndEvents = new ConcurrentLinkedQueue[SparkListenerTaskEnd]()
+    val executorsActuallyStarted = new ConcurrentHashMap[String, Boolean]()
+    val blocksUpdated = ArrayBuffer.empty[SparkListenerBlockUpdated]
+
+    def getCandidateExecutorToDecom: Option[String] = if (whenToDecom == TaskStarted) {
+      executorsActuallyStarted.keySet().asScala.headOption
+    } else {
+      taskEndEvents.asScala.filter(_.taskInfo.successful).map(_.taskInfo.executorId).headOption
+    }
+
+    sc.addSparkListener(new SparkListener {
+      override def onExecutorRemoved(execRemoved: SparkListenerExecutorRemoved): Unit = {
+        executorRemovedSem.release()
+      }
+
+      override def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit = {
+        taskEndEvents.add(taskEnd)
+      }
+
+      override def onBlockUpdated(blockUpdated: SparkListenerBlockUpdated): Unit = synchronized {
+        blocksUpdated.append(blockUpdated)
+      }
+
+      override def onExecutorMetricsUpdate(
+          executorMetricsUpdate: SparkListenerExecutorMetricsUpdate): Unit = {
+        val executorId = executorMetricsUpdate.execId
+        if (executorId != SparkContext.DRIVER_IDENTIFIER) {
+          val validUpdate = executorMetricsUpdate
+            .accumUpdates
+            .flatMap(_._4)
+            .exists { accumInfo =>
+              accumInfo.name == accum.name && accumInfo.update.exists(_.asInstanceOf[Long] >= 1)
+            }
+          if (validUpdate) {
+            executorsActuallyStarted.put(executorId, java.lang.Boolean.TRUE)
+          }
+        }
+      }
+    })
+
+    // Cache the RDD lazily
+    if (persist) {
+      testRdd.persist()
+    }
+
+    // Start the computation of RDD - this step will also cache the RDD
+    val asyncCount = testRdd.countAsync()
+
+    // Make sure the job is either mid run or otherwise has data to migrate.
+    if (migrateDuring) {
+      // Wait for one of the tasks to succeed and finish writing its blocks.
+      // This way we know that this executor had real data to migrate when it is subsequently
+      // decommissioned below.
+      val intervalMs = if (whenToDecom == TaskStarted) {
+        3.milliseconds
+      } else {
+        10.milliseconds
+      }
+      eventually(timeout(20.seconds), interval(intervalMs)) {
+        assert(getCandidateExecutorToDecom.isDefined)
+      }
+    } else {
+      ThreadUtils.awaitResult(asyncCount, 1.minute)
+    }
+
+    // Decommission one of the executors.
+    val sched = sc.schedulerBackend.asInstanceOf[StandaloneSchedulerBackend]
+
+    val execToDecommission = getCandidateExecutorToDecom.get
+    logInfo(s"Decommissioning executor ${execToDecommission}")
+
+    // Decommission executor and ensure it is not relaunched by setting adjustTargetNumExecutors
+    sched.decommissionExecutor(
+      execToDecommission,
+      ExecutorDecommissionInfo("", None),
+      adjustTargetNumExecutors = true)
+    val decomTime = new SystemClock().getTimeMillis()
+
+    // Wait for job to finish.
+    val asyncCountResult = ThreadUtils.awaitResult(asyncCount, 1.minute)
+    assert(asyncCountResult === numParts)
+    // All tasks finished, so accum should have been increased numParts times.
+    assert(accum.value === numParts)
+
+    sc.listenerBus.waitUntilEmpty()
+    val taskEndEventsCopy = taskEndEvents.asScala
+    if (shuffle) {
+      //  mappers & reducers which succeeded
+      assert(taskEndEventsCopy.count(_.reason == Success) === 2 * numParts,
+        s"Expected ${2 * numParts} tasks got ${taskEndEvents.size} (${taskEndEvents})")
+    } else {
+      // only mappers which executed successfully
+      assert(taskEndEventsCopy.count(_.reason == Success) === numParts,
+        s"Expected ${numParts} tasks got ${taskEndEvents.size} (${taskEndEvents})")
+    }
+
+    val minTaskEndTime = taskEndEventsCopy.map(_.taskInfo.finishTime).min
+    val maxTaskEndTime = taskEndEventsCopy.map(_.taskInfo.finishTime).max
+
+    // Verify that the decom time matched our expectations
+    val decomAssertMsg = s"$whenToDecom: decomTime: $decomTime, minTaskEnd: $minTaskEndTime," +
+      s" maxTaskEnd: $maxTaskEndTime"
+    assert(minTaskEndTime <= maxTaskEndTime, decomAssertMsg)
+    whenToDecom match {
+      case TaskStarted => assert(minTaskEndTime > decomTime, decomAssertMsg)
+      case TaskEnded => assert(minTaskEndTime <= decomTime &&
+        decomTime < maxTaskEndTime, decomAssertMsg)
+      case JobEnded => assert(maxTaskEndTime <= decomTime, decomAssertMsg)
+    }
+
+    // Wait for our respective blocks to have migrated
+    eventually(timeout(1.minute), interval(10.milliseconds)) {
+      if (persist) {
+        // One of our blocks should have moved.
+        val rddUpdates = blocksUpdated.filter { update =>
+          val blockId = update.blockUpdatedInfo.blockId
+          blockId.isRDD}
+        val blockLocs = rddUpdates.map { update =>
+          (update.blockUpdatedInfo.blockId.name,
+            update.blockUpdatedInfo.blockManagerId)}
+        val blocksToManagers = blockLocs.groupBy(_._1).mapValues(_.size)
+        assert(blocksToManagers.exists(_._2 > 1),
+          s"We should have a block that has been on multiple BMs in rdds:\n ${rddUpdates} from:\n" +
+          s"${blocksUpdated}\n but instead we got:\n ${blocksToManagers}")
+      }
+      // If we're migrating shuffles we look for any shuffle block updates
+      // as there is no block update on the initial shuffle block write.
+      if (shuffle) {
+        val numDataLocs = blocksUpdated.count { update =>
+          val blockId = update.blockUpdatedInfo.blockId
+          blockId.isInstanceOf[ShuffleDataBlockId]
+        }
+        val numIndexLocs = blocksUpdated.count { update =>
+          val blockId = update.blockUpdatedInfo.blockId
+          blockId.isInstanceOf[ShuffleIndexBlockId]
+        }
+        assert(numDataLocs === 1, s"Expect shuffle data block updates in ${blocksUpdated}")
+        assert(numIndexLocs === 1, s"Expect shuffle index block updates in ${blocksUpdated}")
+      }
+    }
+
+    // Since the RDD is cached or shuffled so further usage of same RDD should use the
+    // cached data. Original RDD partitions should not be recomputed i.e. accum
+    // should have same value like before
+    assert(testRdd.count() === numParts)
+    assert(accum.value === numParts)
+
+    val storageStatus = sc.env.blockManager.master.getStorageStatus
+    val execIdToBlocksMapping = storageStatus.map(
+      status => (status.blockManagerId.executorId, status.blocks)).toMap
+    // No cached blocks should be present on executor which was decommissioned
+    assert(
+      !execIdToBlocksMapping.contains(execToDecommission) ||
+      execIdToBlocksMapping(execToDecommission).keys.filter(_.isRDD).toSeq === Seq(),
+      "Cache blocks should be migrated")
+    if (persist) {
+      // There should still be all the RDD blocks cached
+      assert(execIdToBlocksMapping.values.flatMap(_.keys).count(_.isRDD) === numParts)
+    }
+
+    // Wait for the executor to be removed automatically after migration.
+    // This is set to a high value since github actions is sometimes high latency
+    // but I've never seen this go for more than a minute.
+    assert(executorRemovedSem.tryAcquire(1, 5L, TimeUnit.MINUTES))
+
+    // Since the RDD is cached or shuffled so further usage of same RDD should use the
+    // cached data. Original RDD partitions should not be recomputed i.e. accum
+    // should have same value like before
+    assert(testRdd.count() === numParts)
+    assert(accum.value === numParts)
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/storage/BlockManagerDecommissionSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockManagerDecommissionSuite.scala
deleted file mode 100644
index 7456ca7f02a2e..0000000000000
--- a/core/src/test/scala/org/apache/spark/storage/BlockManagerDecommissionSuite.scala
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.storage
-
-import java.util.concurrent.Semaphore
-
-import scala.collection.mutable.ArrayBuffer
-import scala.concurrent.duration._
-
-import org.apache.spark.{LocalSparkContext, SparkConf, SparkContext, SparkFunSuite, Success}
-import org.apache.spark.internal.config
-import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd, SparkListenerTaskStart}
-import org.apache.spark.scheduler.cluster.StandaloneSchedulerBackend
-import org.apache.spark.util.{ResetSystemProperties, ThreadUtils}
-
-class BlockManagerDecommissionSuite extends SparkFunSuite with LocalSparkContext
-    with ResetSystemProperties {
-
-  override def beforeEach(): Unit = {
-    val conf = new SparkConf().setAppName("test")
-      .set(config.Worker.WORKER_DECOMMISSION_ENABLED, true)
-      .set(config.STORAGE_DECOMMISSION_ENABLED, true)
-
-    sc = new SparkContext("local-cluster[2, 1, 1024]", "test", conf)
-  }
-
-  test(s"verify that an already running task which is going to cache data succeeds " +
-    s"on a decommissioned executor") {
-    // Create input RDD with 10 partitions
-    val input = sc.parallelize(1 to 10, 10)
-    val accum = sc.longAccumulator("mapperRunAccumulator")
-    // Do a count to wait for the executors to be registered.
-    input.count()
-
-    // Create a new RDD where we have sleep in each partition, we are also increasing
-    // the value of accumulator in each partition
-    val sleepyRdd = input.mapPartitions { x =>
-      Thread.sleep(500)
-      accum.add(1)
-      x
-    }
-
-    // Listen for the job
-    val sem = new Semaphore(0)
-    val taskEndEvents = ArrayBuffer.empty[SparkListenerTaskEnd]
-    sc.addSparkListener(new SparkListener {
-      override def onTaskStart(taskStart: SparkListenerTaskStart): Unit = {
-       sem.release()
-      }
-
-      override def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit = {
-        taskEndEvents.append(taskEnd)
-      }
-    })
-
-    // Cache the RDD lazily
-    sleepyRdd.persist()
-
-    // Start the computation of RDD - this step will also cache the RDD
-    val asyncCount = sleepyRdd.countAsync()
-
-    // Wait for the job to have started
-    sem.acquire(1)
-
-    // Give Spark a tiny bit to start the tasks after the listener says hello
-    Thread.sleep(100)
-    // Decommission one of the executor
-    val sched = sc.schedulerBackend.asInstanceOf[StandaloneSchedulerBackend]
-    val execs = sched.getExecutorIds()
-    assert(execs.size == 2, s"Expected 2 executors but found ${execs.size}")
-    val execToDecommission = execs.head
-    sched.decommissionExecutor(execToDecommission)
-
-    // Wait for job to finish
-    val asyncCountResult = ThreadUtils.awaitResult(asyncCount, 6.seconds)
-    assert(asyncCountResult === 10)
-    // All 10 tasks finished, so accum should have been increased 10 times
-    assert(accum.value === 10)
-
-    // All tasks should be successful, nothing should have failed
-    sc.listenerBus.waitUntilEmpty()
-    assert(taskEndEvents.size === 10) // 10 mappers
-    assert(taskEndEvents.map(_.reason).toSet === Set(Success))
-
-    // Since the RDD is cached, so further usage of same RDD should use the
-    // cached data. Original RDD partitions should not be recomputed i.e. accum
-    // should have same value like before
-    assert(sleepyRdd.count() === 10)
-    assert(accum.value === 10)
-  }
-}
diff --git a/core/src/test/scala/org/apache/spark/storage/BlockManagerDecommissionUnitSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockManagerDecommissionUnitSuite.scala
new file mode 100644
index 0000000000000..b7ac378b4c6cd
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/storage/BlockManagerDecommissionUnitSuite.scala
@@ -0,0 +1,308 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.storage
+
+import scala.concurrent.duration._
+
+import org.mockito.{ArgumentMatchers => mc}
+import org.mockito.Mockito.{atLeast => least, mock, times, verify, when}
+import org.scalatest.concurrent.Eventually._
+import org.scalatest.matchers.must.Matchers
+
+import org.apache.spark._
+import org.apache.spark.internal.config
+import org.apache.spark.network.BlockTransferService
+import org.apache.spark.network.buffer.ManagedBuffer
+import org.apache.spark.shuffle.{MigratableResolver, ShuffleBlockInfo}
+import org.apache.spark.storage.BlockManagerMessages.ReplicateBlock
+
+class BlockManagerDecommissionUnitSuite extends SparkFunSuite with Matchers {
+
+  private val bmPort = 12345
+
+  private val sparkConf = new SparkConf(false)
+    .set(config.STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED, true)
+    .set(config.STORAGE_DECOMMISSION_RDD_BLOCKS_ENABLED, true)
+    // Just replicate blocks quickly during testing, as there isn't another
+    // workload we need to worry about.
+    .set(config.STORAGE_DECOMMISSION_REPLICATION_REATTEMPT_INTERVAL, 10L)
+
+  private def registerShuffleBlocks(
+      mockMigratableShuffleResolver: MigratableResolver,
+      ids: Set[(Int, Long, Int)]): Unit = {
+
+    when(mockMigratableShuffleResolver.getStoredShuffles())
+      .thenReturn(ids.map(triple => ShuffleBlockInfo(triple._1, triple._2)).toSeq)
+
+    ids.foreach { case (shuffleId: Int, mapId: Long, reduceId: Int) =>
+      when(mockMigratableShuffleResolver.getMigrationBlocks(mc.any()))
+        .thenReturn(List(
+          (ShuffleIndexBlockId(shuffleId, mapId, reduceId), mock(classOf[ManagedBuffer])),
+          (ShuffleDataBlockId(shuffleId, mapId, reduceId), mock(classOf[ManagedBuffer]))))
+    }
+  }
+
+  /**
+   * Validate a given configuration with the mocks.
+   * The fail variable controls if we expect migration to fail, in which case we expect
+   * a constant Long.MaxValue timestamp.
+   */
+  private def validateDecommissionTimestamps(conf: SparkConf, bm: BlockManager,
+      fail: Boolean = false, assertDone: Boolean = true) = {
+    // Verify the decommissioning manager timestamps and status
+    val bmDecomManager = new BlockManagerDecommissioner(conf, bm)
+    validateDecommissionTimestampsOnManager(bmDecomManager, fail, assertDone)
+  }
+
+  private def validateDecommissionTimestampsOnManager(bmDecomManager: BlockManagerDecommissioner,
+      fail: Boolean = false, assertDone: Boolean = true, numShuffles: Option[Int] = None) = {
+    var previousTime: Option[Long] = None
+    try {
+      bmDecomManager.start()
+      eventually(timeout(100.second), interval(10.milliseconds)) {
+        val (currentTime, done) = bmDecomManager.lastMigrationInfo()
+        assert(!assertDone || done)
+        // Make sure the time stamp starts moving forward.
+        if (!fail) {
+          previousTime match {
+            case None =>
+              previousTime = Some(currentTime)
+              assert(false)
+            case Some(t) =>
+              assert(t < currentTime)
+          }
+        } else {
+          // If we expect migration to fail we should get the max value quickly.
+          assert(currentTime === Long.MaxValue)
+        }
+        numShuffles.foreach { s =>
+          assert(bmDecomManager.numMigratedShuffles.get() === s)
+        }
+      }
+      if (!fail) {
+        // Wait 5 seconds and assert times keep moving forward.
+        Thread.sleep(5000)
+        val (currentTime, done) = bmDecomManager.lastMigrationInfo()
+        assert((!assertDone || done) && currentTime > previousTime.get)
+      }
+    } finally {
+      bmDecomManager.stop()
+    }
+  }
+
+  test("test that with no blocks we finish migration") {
+    // Set up the mocks so we return empty
+    val bm = mock(classOf[BlockManager])
+    val migratableShuffleBlockResolver = mock(classOf[MigratableResolver])
+    when(migratableShuffleBlockResolver.getStoredShuffles())
+      .thenReturn(Seq())
+    when(bm.migratableResolver).thenReturn(migratableShuffleBlockResolver)
+    when(bm.getMigratableRDDBlocks())
+      .thenReturn(Seq())
+    when(bm.getPeers(mc.any()))
+      .thenReturn(Seq(BlockManagerId("exec2", "host2", 12345)))
+
+    // Verify the decom manager handles this correctly
+    validateDecommissionTimestamps(sparkConf, bm)
+  }
+
+  test("block decom manager with no migrations configured") {
+    val bm = mock(classOf[BlockManager])
+    val migratableShuffleBlockResolver = mock(classOf[MigratableResolver])
+    registerShuffleBlocks(migratableShuffleBlockResolver, Set((1, 1L, 1)))
+    when(bm.migratableResolver).thenReturn(migratableShuffleBlockResolver)
+    when(bm.getMigratableRDDBlocks())
+      .thenReturn(Seq())
+    when(bm.getPeers(mc.any()))
+      .thenReturn(Seq(BlockManagerId("exec2", "host2", 12345)))
+
+    val badConf = new SparkConf(false)
+      .set(config.STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED, false)
+      .set(config.STORAGE_DECOMMISSION_RDD_BLOCKS_ENABLED, false)
+      .set(config.STORAGE_DECOMMISSION_REPLICATION_REATTEMPT_INTERVAL, 10L)
+    // Verify the decom manager handles this correctly
+    validateDecommissionTimestamps(badConf, bm, fail = true)
+  }
+
+  test("block decom manager with no peers") {
+    // Set up the mocks so we return one shuffle block
+    val bm = mock(classOf[BlockManager])
+    val migratableShuffleBlockResolver = mock(classOf[MigratableResolver])
+    registerShuffleBlocks(migratableShuffleBlockResolver, Set((1, 1L, 1)))
+    when(bm.migratableResolver).thenReturn(migratableShuffleBlockResolver)
+    when(bm.getMigratableRDDBlocks())
+      .thenReturn(Seq())
+    when(bm.getPeers(mc.any()))
+      .thenReturn(Seq())
+
+    // Verify the decom manager handles this correctly
+    validateDecommissionTimestamps(sparkConf, bm, fail = true)
+  }
+
+
+  test("block decom manager with only shuffle files time moves forward") {
+    // Set up the mocks so we return one shuffle block
+    val bm = mock(classOf[BlockManager])
+    val migratableShuffleBlockResolver = mock(classOf[MigratableResolver])
+    registerShuffleBlocks(migratableShuffleBlockResolver, Set((1, 1L, 1)))
+    when(bm.migratableResolver).thenReturn(migratableShuffleBlockResolver)
+    when(bm.getMigratableRDDBlocks())
+      .thenReturn(Seq())
+    when(bm.getPeers(mc.any()))
+      .thenReturn(Seq(BlockManagerId("exec2", "host2", 12345)))
+
+    // Verify the decom manager handles this correctly
+    validateDecommissionTimestamps(sparkConf, bm)
+  }
+
+  test("block decom manager does not re-add removed shuffle files") {
+    // Set up the mocks so we return one shuffle block
+    val bm = mock(classOf[BlockManager])
+    val migratableShuffleBlockResolver = mock(classOf[MigratableResolver])
+    registerShuffleBlocks(migratableShuffleBlockResolver, Set())
+    when(bm.migratableResolver).thenReturn(migratableShuffleBlockResolver)
+    when(bm.getMigratableRDDBlocks())
+      .thenReturn(Seq())
+    when(bm.getPeers(mc.any()))
+      .thenReturn(Seq(BlockManagerId("exec2", "host2", 12345)))
+    val bmDecomManager = new BlockManagerDecommissioner(sparkConf, bm)
+    bmDecomManager.migratingShuffles += ShuffleBlockInfo(10, 10)
+
+    validateDecommissionTimestampsOnManager(bmDecomManager, fail = false, assertDone = false)
+  }
+
+  test("block decom manager handles IO failures") {
+    // Set up the mocks so we return one shuffle block
+    val bm = mock(classOf[BlockManager])
+    val migratableShuffleBlockResolver = mock(classOf[MigratableResolver])
+    registerShuffleBlocks(migratableShuffleBlockResolver, Set((1, 1L, 1)))
+    when(bm.migratableResolver).thenReturn(migratableShuffleBlockResolver)
+    when(bm.getMigratableRDDBlocks())
+      .thenReturn(Seq())
+    when(bm.getPeers(mc.any()))
+      .thenReturn(Seq(BlockManagerId("exec2", "host2", 12345)))
+
+    val blockTransferService = mock(classOf[BlockTransferService])
+    // Simulate an ambiguous IO error (e.g. block could be gone, connection failed, etc.)
+    when(blockTransferService.uploadBlockSync(
+      mc.any(), mc.any(), mc.any(), mc.any(), mc.any(), mc.any(), mc.isNull())).thenThrow(
+      new java.io.IOException("boop")
+    )
+
+    when(bm.blockTransferService).thenReturn(blockTransferService)
+
+    // Verify the decom manager handles this correctly
+    val bmDecomManager = new BlockManagerDecommissioner(sparkConf, bm)
+    validateDecommissionTimestampsOnManager(bmDecomManager, fail = false)
+  }
+
+  test("block decom manager short circuits removed blocks") {
+    // Set up the mocks so we return one shuffle block
+    val bm = mock(classOf[BlockManager])
+    val migratableShuffleBlockResolver = mock(classOf[MigratableResolver])
+    // First call get blocks, then empty list simulating a delete.
+    when(migratableShuffleBlockResolver.getStoredShuffles())
+      .thenReturn(Seq(ShuffleBlockInfo(1, 1)))
+      .thenReturn(Seq())
+    when(migratableShuffleBlockResolver.getMigrationBlocks(mc.any()))
+      .thenReturn(List(
+        (ShuffleIndexBlockId(1, 1, 1), mock(classOf[ManagedBuffer])),
+        (ShuffleDataBlockId(1, 1, 1), mock(classOf[ManagedBuffer]))))
+      .thenReturn(List())
+
+    when(bm.migratableResolver).thenReturn(migratableShuffleBlockResolver)
+    when(bm.getMigratableRDDBlocks())
+      .thenReturn(Seq())
+    when(bm.getPeers(mc.any()))
+      .thenReturn(Seq(BlockManagerId("exec2", "host2", 12345)))
+
+    val blockTransferService = mock(classOf[BlockTransferService])
+    // Simulate an ambiguous IO error (e.g. block could be gone, connection failed, etc.)
+    when(blockTransferService.uploadBlockSync(
+      mc.any(), mc.any(), mc.any(), mc.any(), mc.any(), mc.any(), mc.isNull())).thenThrow(
+      new java.io.IOException("boop")
+    )
+
+    when(bm.blockTransferService).thenReturn(blockTransferService)
+
+    // Verify the decom manager handles this correctly
+    val bmDecomManager = new BlockManagerDecommissioner(sparkConf, bm)
+    validateDecommissionTimestampsOnManager(bmDecomManager, fail = false,
+      numShuffles = Some(1))
+  }
+
+  test("test shuffle and cached rdd migration without any error") {
+    val blockTransferService = mock(classOf[BlockTransferService])
+    val bm = mock(classOf[BlockManager])
+
+    val storedBlockId1 = RDDBlockId(0, 0)
+    val storedBlock1 =
+      new ReplicateBlock(storedBlockId1, Seq(BlockManagerId("replicaHolder", "host1", bmPort)), 1)
+
+    val migratableShuffleBlockResolver = mock(classOf[MigratableResolver])
+    registerShuffleBlocks(migratableShuffleBlockResolver, Set((1, 1L, 1)))
+    when(bm.getPeers(mc.any()))
+      .thenReturn(Seq(BlockManagerId("exec2", "host2", 12345)))
+
+    when(bm.blockTransferService).thenReturn(blockTransferService)
+    when(bm.migratableResolver).thenReturn(migratableShuffleBlockResolver)
+    when(bm.getMigratableRDDBlocks())
+      .thenReturn(Seq(storedBlock1))
+
+    val bmDecomManager = new BlockManagerDecommissioner(sparkConf, bm)
+
+    try {
+      bmDecomManager.start()
+
+      var previousRDDTime: Option[Long] = None
+      var previousShuffleTime: Option[Long] = None
+
+      // We don't check that all blocks are migrated because out mock is always returning an RDD.
+      eventually(timeout(100.second), interval(10.milliseconds)) {
+        assert(bmDecomManager.shufflesToMigrate.isEmpty === true)
+        assert(bmDecomManager.numMigratedShuffles.get() === 1)
+        verify(bm, least(1)).replicateBlock(
+          mc.eq(storedBlockId1), mc.any(), mc.any(), mc.eq(Some(3)))
+        verify(blockTransferService, times(2))
+          .uploadBlockSync(mc.eq("host2"), mc.eq(bmPort), mc.eq("exec2"), mc.any(), mc.any(),
+            mc.eq(StorageLevel.DISK_ONLY), mc.isNull())
+        // Since we never "finish" the RDD blocks, make sure the time is always moving forward.
+        assert(bmDecomManager.rddBlocksLeft)
+        previousRDDTime match {
+          case None =>
+            previousRDDTime = Some(bmDecomManager.lastRDDMigrationTime)
+            assert(false)
+          case Some(t) =>
+            assert(bmDecomManager.lastRDDMigrationTime > t)
+        }
+        // Since we do eventually finish the shuffle blocks make sure the shuffle blocks complete
+        // and that the time keeps moving forward.
+        assert(!bmDecomManager.shuffleBlocksLeft)
+        previousShuffleTime match {
+          case None =>
+            previousShuffleTime = Some(bmDecomManager.lastShuffleMigrationTime)
+            assert(false)
+          case Some(t) =>
+            assert(bmDecomManager.lastShuffleMigrationTime > t)
+        }
+      }
+    } finally {
+        bmDecomManager.stop()
+    }
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/storage/BlockManagerInfoSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockManagerInfoSuite.scala
index 01e3d6a46e709..3f5ffaa732f25 100644
--- a/core/src/test/scala/org/apache/spark/storage/BlockManagerInfoSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/BlockManagerInfoSuite.scala
@@ -33,7 +33,7 @@ class BlockManagerInfoSuite extends SparkFunSuite {
         timeMs = 300,
         maxOnHeapMem = 10000,
         maxOffHeapMem = 20000,
-        slaveEndpoint = null,
+        storageEndpoint = null,
         if (svcEnabled) Some(new JHashMap[BlockId, BlockStatus]) else None)
       test(s"$testName externalShuffleServiceEnabled=$svcEnabled") {
         f(svcEnabled, bmInfo)
diff --git a/core/src/test/scala/org/apache/spark/storage/BlockManagerReplicationSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockManagerReplicationSuite.scala
index 660bfcfc48267..495747b2c7c11 100644
--- a/core/src/test/scala/org/apache/spark/storage/BlockManagerReplicationSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/BlockManagerReplicationSuite.scala
@@ -26,8 +26,9 @@ import scala.language.implicitConversions
 
 import org.mockito.ArgumentMatchers.any
 import org.mockito.Mockito.{doAnswer, mock, spy, when}
-import org.scalatest.{BeforeAndAfter, Matchers}
+import org.scalatest.BeforeAndAfter
 import org.scalatest.concurrent.Eventually._
+import org.scalatest.matchers.must.Matchers
 
 import org.apache.spark._
 import org.apache.spark.broadcast.BroadcastManager
@@ -54,7 +55,7 @@ trait BlockManagerReplicationBehavior extends SparkFunSuite
   protected var rpcEnv: RpcEnv = null
   protected var master: BlockManagerMaster = null
   protected lazy val securityMgr = new SecurityManager(conf)
-  protected lazy val bcastManager = new BroadcastManager(true, conf, securityMgr)
+  protected lazy val bcastManager = new BroadcastManager(true, conf)
   protected lazy val mapOutputTracker = new MapOutputTrackerMaster(conf, bcastManager, true)
   protected lazy val shuffleManager = new SortShuffleManager(conf)
 
@@ -94,8 +95,6 @@ trait BlockManagerReplicationBehavior extends SparkFunSuite
     conf.set(MEMORY_STORAGE_FRACTION, 0.999)
     conf.set(STORAGE_UNROLL_MEMORY_THRESHOLD, 512L)
 
-    // to make a replication attempt to inactive store fail fast
-    conf.set("spark.core.connection.ack.wait.timeout", "1s")
     // to make cached peers refresh frequently
     conf.set(STORAGE_CACHED_PEERS_TTL, 10)
 
@@ -103,7 +102,7 @@ trait BlockManagerReplicationBehavior extends SparkFunSuite
     val blockManagerInfo = new mutable.HashMap[BlockManagerId, BlockManagerInfo]()
     master = new BlockManagerMaster(rpcEnv.setupEndpoint("blockmanager",
       new BlockManagerMasterEndpoint(rpcEnv, true, conf,
-        new LiveListenerBus(conf), None, blockManagerInfo)),
+        new LiveListenerBus(conf), None, blockManagerInfo, mapOutputTracker)),
       rpcEnv.setupEndpoint("blockmanagerHeartbeat",
       new BlockManagerMasterHeartbeatEndpoint(rpcEnv, true, blockManagerInfo)), conf, true)
     allStores.clear()
diff --git a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
index bfef8f1ab29d8..707e1684f78fd 100644
--- a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
@@ -19,11 +19,12 @@ package org.apache.spark.storage
 
 import java.io.File
 import java.nio.ByteBuffer
+import java.nio.file.Files
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable
 import scala.collection.mutable.ArrayBuffer
-import scala.concurrent.Future
+import scala.concurrent.{Future, TimeoutException}
 import scala.concurrent.duration._
 import scala.language.implicitConversions
 import scala.reflect.ClassTag
@@ -31,9 +32,11 @@ import scala.reflect.ClassTag
 import org.apache.commons.lang3.RandomUtils
 import org.mockito.{ArgumentCaptor, ArgumentMatchers => mc}
 import org.mockito.Mockito.{doAnswer, mock, never, spy, times, verify, when}
-import org.scalatest._
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, PrivateMethodTester}
 import org.scalatest.concurrent.{Signaler, ThreadSignaler, TimeLimits}
 import org.scalatest.concurrent.Eventually._
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark._
 import org.apache.spark.broadcast.BroadcastManager
@@ -49,10 +52,12 @@ import org.apache.spark.network.netty.{NettyBlockTransferService, SparkTransport
 import org.apache.spark.network.server.{NoOpRpcHandler, TransportServer, TransportServerBootstrap}
 import org.apache.spark.network.shuffle.{BlockFetchingListener, DownloadFileManager, ExecutorDiskUtils, ExternalBlockStoreClient}
 import org.apache.spark.network.shuffle.protocol.{BlockTransferMessage, RegisterExecutor}
-import org.apache.spark.rpc.RpcEnv
-import org.apache.spark.scheduler.{LiveListenerBus, SparkListenerBlockUpdated}
+import org.apache.spark.rpc.{RpcCallContext, RpcEndpoint, RpcEnv}
+import org.apache.spark.scheduler.{LiveListenerBus, MapStatus, MergeStatus, SparkListenerBlockUpdated}
+import org.apache.spark.scheduler.cluster.{CoarseGrainedClusterMessages, CoarseGrainedSchedulerBackend}
 import org.apache.spark.security.{CryptoStreamUtils, EncryptionFunSuite}
 import org.apache.spark.serializer.{JavaSerializer, KryoSerializer, SerializerManager}
+import org.apache.spark.shuffle.{MigratableResolver, ShuffleBlockInfo, ShuffleBlockResolver, ShuffleManager}
 import org.apache.spark.shuffle.sort.SortShuffleManager
 import org.apache.spark.storage.BlockManagerMessages._
 import org.apache.spark.util._
@@ -60,7 +65,7 @@ import org.apache.spark.util.io.ChunkedByteBuffer
 
 class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterEach
   with PrivateMethodTester with LocalSparkContext with ResetSystemProperties
-  with EncryptionFunSuite with TimeLimits {
+  with EncryptionFunSuite with TimeLimits with BeforeAndAfterAll {
 
   import BlockManagerSuite._
 
@@ -69,11 +74,12 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
 
   var conf: SparkConf = null
   val allStores = ArrayBuffer[BlockManager]()
+  val sortShuffleManagers = ArrayBuffer[SortShuffleManager]()
   var rpcEnv: RpcEnv = null
   var master: BlockManagerMaster = null
   var liveListenerBus: LiveListenerBus = null
   val securityMgr = new SecurityManager(new SparkConf(false))
-  val bcastManager = new BroadcastManager(true, new SparkConf(false), securityMgr)
+  val bcastManager = new BroadcastManager(true, new SparkConf(false))
   val mapOutputTracker = new MapOutputTrackerMaster(new SparkConf(false), bcastManager, true)
   val shuffleManager = new SortShuffleManager(new SparkConf(false))
 
@@ -93,6 +99,14 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
       .set(MEMORY_STORAGE_FRACTION, 0.999)
       .set(Kryo.KRYO_SERIALIZER_BUFFER_SIZE.key, "1m")
       .set(STORAGE_UNROLL_MEMORY_THRESHOLD, 512L)
+      .set(Network.RPC_ASK_TIMEOUT, "5s")
+      .set(PUSH_BASED_SHUFFLE_ENABLED, true)
+  }
+
+  private def makeSortShuffleManager(conf: Option[SparkConf] = None): SortShuffleManager = {
+    val newMgr = new SortShuffleManager(conf.getOrElse(new SparkConf(false)))
+    sortShuffleManagers += newMgr
+    newMgr
   }
 
   private def makeBlockManager(
@@ -100,7 +114,8 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
       name: String = SparkContext.DRIVER_IDENTIFIER,
       master: BlockManagerMaster = this.master,
       transferService: Option[BlockTransferService] = Option.empty,
-      testConf: Option[SparkConf] = None): BlockManager = {
+      testConf: Option[SparkConf] = None,
+      shuffleManager: ShuffleManager = shuffleManager): BlockManager = {
     val bmConf = testConf.map(_.setAll(conf.getAll)).getOrElse(conf)
     bmConf.set(TEST_MEMORY, maxMem)
     bmConf.set(MEMORY_OFFHEAP_SIZE, maxMem)
@@ -130,15 +145,35 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
     blockManager
   }
 
+  // Save modified system properties so that we can restore them after tests.
+  val originalArch = System.getProperty("os.arch")
+  val originalCompressedOops = System.getProperty(TEST_USE_COMPRESSED_OOPS_KEY)
+
+  def reinitializeSizeEstimator(arch: String, useCompressedOops: String): Unit = {
+    def set(k: String, v: String): Unit = {
+      if (v == null) {
+        System.clearProperty(k)
+      } else {
+        System.setProperty(k, v)
+      }
+    }
+    set("os.arch", arch)
+    set(TEST_USE_COMPRESSED_OOPS_KEY, useCompressedOops)
+    val initialize = PrivateMethod[Unit](Symbol("initialize"))
+    SizeEstimator invokePrivate initialize()
+  }
+
   override def beforeEach(): Unit = {
     super.beforeEach()
     // Set the arch to 64-bit and compressedOops to true to get a deterministic test-case
-    System.setProperty("os.arch", "amd64")
+    reinitializeSizeEstimator("amd64", "true")
     conf = new SparkConf(false)
     init(conf)
 
-    rpcEnv = RpcEnv.create("test", "localhost", 0, conf, securityMgr)
+    rpcEnv = RpcEnv.create("test", conf.get(config.DRIVER_HOST_ADDRESS),
+      conf.get(config.DRIVER_PORT), conf, securityMgr)
     conf.set(DRIVER_PORT, rpcEnv.address.port)
+    conf.set(DRIVER_HOST_ADDRESS, rpcEnv.address.host)
 
     // Mock SparkContext to reduce the memory usage of tests. It's fine since the only reason we
     // need to create a SparkContext is to initialize LiveListenerBus.
@@ -149,19 +184,21 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
     liveListenerBus = spy(new LiveListenerBus(conf))
     master = spy(new BlockManagerMaster(rpcEnv.setupEndpoint("blockmanager",
       new BlockManagerMasterEndpoint(rpcEnv, true, conf,
-        liveListenerBus, None, blockManagerInfo)),
+        liveListenerBus, None, blockManagerInfo, mapOutputTracker)),
       rpcEnv.setupEndpoint("blockmanagerHeartbeat",
       new BlockManagerMasterHeartbeatEndpoint(rpcEnv, true, blockManagerInfo)), conf, true))
-
-    val initialize = PrivateMethod[Unit](Symbol("initialize"))
-    SizeEstimator invokePrivate initialize()
   }
 
   override def afterEach(): Unit = {
+    // Restore system properties and SizeEstimator to their original states.
+    reinitializeSizeEstimator(originalArch, originalCompressedOops)
+
     try {
       conf = null
       allStores.foreach(_.stop())
       allStores.clear()
+      sortShuffleManagers.foreach(_.stop())
+      sortShuffleManagers.clear()
       rpcEnv.shutdown()
       rpcEnv.awaitTermination()
       rpcEnv = null
@@ -172,11 +209,121 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
     }
   }
 
+  override def afterAll(): Unit = {
+    try {
+      // Cleanup the reused items.
+      Option(bcastManager).foreach(_.stop())
+      Option(mapOutputTracker).foreach(_.stop())
+      Option(shuffleManager).foreach(_.stop())
+    } finally {
+      super.afterAll()
+    }
+  }
+
   private def stopBlockManager(blockManager: BlockManager): Unit = {
     allStores -= blockManager
     blockManager.stop()
   }
 
+  /**
+   * Setup driverEndpoint, executor-1(BlockManager), executor-2(BlockManager) to simulate
+   * the real cluster before the tests. Any requests from driver to executor-1 will be responded
+   * in time. However, any requests from driver to executor-2 will be timeouted, in order to test
+   * the specific handling of `TimeoutException`, which is raised at driver side.
+   *
+   * And, when `withLost` is true, we will not register the executor-2 to the driver. Therefore,
+   * it behaves like a lost executor in terms of driver's view. When `withLost` is false, we'll
+   * register the executor-2 normally.
+   */
+  private def setupBlockManagerMasterWithBlocks(withLost: Boolean): Unit = {
+    // set up a simple DriverEndpoint which simply adds executorIds and
+    // checks whether a certain executorId has been added before.
+    val driverEndpoint = rpcEnv.setupEndpoint(CoarseGrainedSchedulerBackend.ENDPOINT_NAME,
+      new RpcEndpoint {
+        private val executorSet = mutable.HashSet[String]()
+        override val rpcEnv: RpcEnv = BlockManagerSuite.this.rpcEnv
+        override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
+          case CoarseGrainedClusterMessages.RegisterExecutor(executorId, _, _, _, _, _, _, _) =>
+            executorSet += executorId
+            context.reply(true)
+          case CoarseGrainedClusterMessages.IsExecutorAlive(executorId) =>
+            context.reply(executorSet.contains(executorId))
+        }
+      }
+    )
+
+    def createAndRegisterBlockManager(timeout: Boolean): BlockManagerId = {
+      val id = if (timeout) "timeout" else "normal"
+      val bmRef = rpcEnv.setupEndpoint(s"bm-$id", new RpcEndpoint {
+        override val rpcEnv: RpcEnv = BlockManagerSuite.this.rpcEnv
+        private def reply[T](context: RpcCallContext, response: T): Unit = {
+          if (timeout) {
+            Thread.sleep(conf.getTimeAsMs(Network.RPC_ASK_TIMEOUT.key) + 1000)
+          }
+          context.reply(response)
+        }
+
+        override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
+          case RemoveRdd(_) => reply(context, 1)
+          case RemoveBroadcast(_, _) => reply(context, 1)
+          case RemoveShuffle(_) => reply(context, true)
+        }
+      })
+      val bmId = BlockManagerId(s"exec-$id", "localhost", 1234, None)
+      master.registerBlockManager(bmId, Array.empty, 2000, 0, bmRef)
+    }
+
+    // set up normal bm1
+    val bm1Id = createAndRegisterBlockManager(false)
+    // set up bm2, which intentionally takes more time than RPC_ASK_TIMEOUT to
+    // remove rdd/broadcast/shuffle in order to raise timeout error
+    val bm2Id = createAndRegisterBlockManager(true)
+
+    driverEndpoint.askSync[Boolean](CoarseGrainedClusterMessages.RegisterExecutor(
+      bm1Id.executorId, null, bm1Id.host, 1, Map.empty, Map.empty,
+      Map.empty, 0))
+
+    if (!withLost) {
+      driverEndpoint.askSync[Boolean](CoarseGrainedClusterMessages.RegisterExecutor(
+        bm2Id.executorId, null, bm1Id.host, 1, Map.empty, Map.empty, Map.empty, 0))
+    }
+
+    eventually(timeout(5.seconds)) {
+      // make sure both bm1 and bm2 are registered at driver side BlockManagerMaster
+      verify(master, times(2))
+        .registerBlockManager(mc.any(), mc.any(), mc.any(), mc.any(), mc.any())
+      assert(driverEndpoint.askSync[Boolean](
+        CoarseGrainedClusterMessages.IsExecutorAlive(bm1Id.executorId)))
+      assert(driverEndpoint.askSync[Boolean](
+        CoarseGrainedClusterMessages.IsExecutorAlive(bm2Id.executorId)) === !withLost)
+    }
+
+    // update RDD block info for bm1 and bm2 (Broadcast and shuffle don't report block
+    // locations to BlockManagerMaster)
+    master.updateBlockInfo(bm1Id, RDDBlockId(0, 0), StorageLevel.MEMORY_ONLY, 100, 0)
+    master.updateBlockInfo(bm2Id, RDDBlockId(0, 1), StorageLevel.MEMORY_ONLY, 100, 0)
+  }
+
+  test("SPARK-32091: count failures from active executors when remove rdd/broadcast/shuffle") {
+    setupBlockManagerMasterWithBlocks(false)
+    // fail because bm2 will timeout and it's not lost anymore
+    assert(intercept[Exception](master.removeRdd(0, true))
+      .getCause.isInstanceOf[TimeoutException])
+    assert(intercept[Exception](master.removeBroadcast(0, true, true))
+      .getCause.isInstanceOf[TimeoutException])
+    assert(intercept[Exception](master.removeShuffle(0, true))
+      .getCause.isInstanceOf[TimeoutException])
+  }
+
+  test("SPARK-32091: ignore failures from lost executors when remove rdd/broadcast/shuffle") {
+    setupBlockManagerMasterWithBlocks(true)
+    // succeed because bm1 will remove rdd/broadcast successfully and bm2 will
+    // timeout but ignored as it's lost
+    master.removeRdd(0, true)
+    master.removeBroadcast(0, true, true)
+    master.removeShuffle(0, true)
+  }
+
   test("StorageLevel object caching") {
     val level1 = StorageLevel(false, false, false, 3)
     // this should return the same object as level1
@@ -1271,12 +1418,12 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
     assert(store.master.getLocations("list1").size === 0)
     assert(store.master.getLocations("list2").size === 1)
     assert(store.master.getLocations("list3").size === 1)
-    assert(store.master.getBlockStatus("list1", askSlaves = false).size === 0)
-    assert(store.master.getBlockStatus("list2", askSlaves = false).size === 1)
-    assert(store.master.getBlockStatus("list3", askSlaves = false).size === 1)
-    assert(store.master.getBlockStatus("list1", askSlaves = true).size === 0)
-    assert(store.master.getBlockStatus("list2", askSlaves = true).size === 1)
-    assert(store.master.getBlockStatus("list3", askSlaves = true).size === 1)
+    assert(store.master.getBlockStatus("list1", askStorageEndpoints = false).size === 0)
+    assert(store.master.getBlockStatus("list2", askStorageEndpoints = false).size === 1)
+    assert(store.master.getBlockStatus("list3", askStorageEndpoints = false).size === 1)
+    assert(store.master.getBlockStatus("list1", askStorageEndpoints = true).size === 0)
+    assert(store.master.getBlockStatus("list2", askStorageEndpoints = true).size === 1)
+    assert(store.master.getBlockStatus("list3", askStorageEndpoints = true).size === 1)
 
     // This time don't tell master and see what happens. By LRU, only list5 and list6 remains.
     store.putIterator(
@@ -1287,17 +1434,17 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
       "list6", list.iterator, StorageLevel.MEMORY_ONLY, tellMaster = false)
 
     // getLocations should return nothing because the master is not informed
-    // getBlockStatus without asking slaves should have the same result
-    // getBlockStatus with asking slaves, however, should return the actual block statuses
+    // getBlockStatus without asking storage endpoints should have the same result
+    // getBlockStatus with asking storage endpoints, however, should return the actual statuses
     assert(store.master.getLocations("list4").size === 0)
     assert(store.master.getLocations("list5").size === 0)
     assert(store.master.getLocations("list6").size === 0)
-    assert(store.master.getBlockStatus("list4", askSlaves = false).size === 0)
-    assert(store.master.getBlockStatus("list5", askSlaves = false).size === 0)
-    assert(store.master.getBlockStatus("list6", askSlaves = false).size === 0)
-    assert(store.master.getBlockStatus("list4", askSlaves = true).size === 0)
-    assert(store.master.getBlockStatus("list5", askSlaves = true).size === 1)
-    assert(store.master.getBlockStatus("list6", askSlaves = true).size === 1)
+    assert(store.master.getBlockStatus("list4", askStorageEndpoints = false).size === 0)
+    assert(store.master.getBlockStatus("list5", askStorageEndpoints = false).size === 0)
+    assert(store.master.getBlockStatus("list6", askStorageEndpoints = false).size === 0)
+    assert(store.master.getBlockStatus("list4", askStorageEndpoints = true).size === 0)
+    assert(store.master.getBlockStatus("list5", askStorageEndpoints = true).size === 1)
+    assert(store.master.getBlockStatus("list6", askStorageEndpoints = true).size === 1)
   }
 
   test("get matching blocks") {
@@ -1313,9 +1460,11 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
       "list3", list.iterator, StorageLevel.MEMORY_AND_DISK, tellMaster = true)
 
     // getLocations and getBlockStatus should yield the same locations
-    assert(store.master.getMatchingBlockIds(_.toString.contains("list"), askSlaves = false).size
+    assert(store.master.getMatchingBlockIds(
+      _.toString.contains("list"), askStorageEndpoints = false).size
       === 3)
-    assert(store.master.getMatchingBlockIds(_.toString.contains("list1"), askSlaves = false).size
+    assert(store.master.getMatchingBlockIds(
+      _.toString.contains("list1"), askStorageEndpoints = false).size
       === 1)
 
     // insert some more blocks
@@ -1327,9 +1476,13 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
       "newlist3", list.iterator, StorageLevel.MEMORY_AND_DISK, tellMaster = false)
 
     // getLocations and getBlockStatus should yield the same locations
-    assert(store.master.getMatchingBlockIds(_.toString.contains("newlist"), askSlaves = false).size
+    assert(
+      store.master.getMatchingBlockIds(
+        _.toString.contains("newlist"), askStorageEndpoints = false).size
       === 1)
-    assert(store.master.getMatchingBlockIds(_.toString.contains("newlist"), askSlaves = true).size
+    assert(
+      store.master.getMatchingBlockIds(
+        _.toString.contains("newlist"), askStorageEndpoints = true).size
       === 3)
 
     val blockIds = Seq(RDDBlockId(1, 0), RDDBlockId(1, 1), RDDBlockId(2, 0))
@@ -1340,7 +1493,7 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
     val matchedBlockIds = store.master.getMatchingBlockIds(_ match {
       case RDDBlockId(1, _) => true
       case _ => false
-    }, askSlaves = true)
+    }, askStorageEndpoints = true)
     assert(matchedBlockIds.toSet === Set(RDDBlockId(1, 0), RDDBlockId(1, 1)))
   }
 
@@ -1559,12 +1712,12 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
     val externalShuffleServicePort = StorageUtils.externalShuffleServicePort(conf)
     val port = store.blockTransferService.port
     val rack = Some("rack")
-    val blockManagerWithTopolgyInfo = BlockManagerId(
+    val blockManagerWithTopologyInfo = BlockManagerId(
       store.blockManagerId.executorId,
       store.blockManagerId.host,
       store.blockManagerId.port,
       rack)
-    store.blockManagerId = blockManagerWithTopolgyInfo
+    store.blockManagerId = blockManagerWithTopologyInfo
     val locations = Seq(
       BlockManagerId("executor4", otherHost, externalShuffleServicePort, rack),
       BlockManagerId("executor3", otherHost, port, rack),
@@ -1706,13 +1859,26 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
     verify(liveListenerBus, never()).post(SparkListenerBlockUpdated(BlockUpdatedInfo(updateInfo)))
   }
 
+  test("we reject putting blocks when we have the wrong shuffle resolver") {
+    val badShuffleManager = mock(classOf[ShuffleManager])
+    val badShuffleResolver = mock(classOf[ShuffleBlockResolver])
+    when(badShuffleManager.shuffleBlockResolver).thenReturn(badShuffleResolver)
+    val shuffleBlockId = ShuffleDataBlockId(0, 0, 0)
+    val bm = makeBlockManager(100, "exec1", shuffleManager = badShuffleManager)
+    val message = "message"
+    val exception = intercept[SparkException] {
+      bm.putBlockDataAsStream(shuffleBlockId, StorageLevel.DISK_ONLY, ClassTag(message.getClass))
+    }
+    assert(exception.getMessage.contains("unsupported shuffle resolver"))
+  }
+
   test("test decommission block manager should not be part of peers") {
     val exec1 = "exec1"
     val exec2 = "exec2"
     val exec3 = "exec3"
-    val store1 = makeBlockManager(800, exec1)
-    val store2 = makeBlockManager(800, exec2)
-    val store3 = makeBlockManager(800, exec3)
+    val store1 = makeBlockManager(1000, exec1)
+    val store2 = makeBlockManager(1000, exec2)
+    val store3 = makeBlockManager(1000, exec3)
 
     assert(master.getPeers(store3.blockManagerId).map(_.executorId).toSet === Set(exec1, exec2))
 
@@ -1727,9 +1893,9 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
   }
 
   test("test decommissionRddCacheBlocks should offload all cached blocks") {
-    val store1 = makeBlockManager(800, "exec1")
-    val store2 = makeBlockManager(800, "exec2")
-    val store3 = makeBlockManager(800, "exec3")
+    val store1 = makeBlockManager(1000, "exec1")
+    val store2 = makeBlockManager(1000, "exec2")
+    val store3 = makeBlockManager(1000, "exec3")
 
     val data = new Array[Byte](4)
     val blockId = rdd(0, 0)
@@ -1737,7 +1903,8 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
     assert(master.getLocations(blockId).size === 2)
     assert(master.getLocations(blockId).contains(store1.blockManagerId))
 
-    store1.decommissionRddCacheBlocks()
+    val decomManager = new BlockManagerDecommissioner(conf, store1)
+    decomManager.decommissionRddCacheBlocks()
     assert(master.getLocations(blockId).size === 2)
     assert(master.getLocations(blockId).toSet === Set(store2.blockManagerId,
       store3.blockManagerId))
@@ -1757,14 +1924,159 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
     assert(master.getLocations(blockIdLarge) === Seq(store1.blockManagerId))
     assert(master.getLocations(blockIdSmall) === Seq(store1.blockManagerId))
 
-    store1.decommissionRddCacheBlocks()
+    val decomManager = new BlockManagerDecommissioner(conf, store1)
+    decomManager.decommissionRddCacheBlocks()
     // Smaller block offloaded to store2
     assert(master.getLocations(blockIdSmall) === Seq(store2.blockManagerId))
     // Larger block still present in store1 as it can't be offloaded
     assert(master.getLocations(blockIdLarge) === Seq(store1.blockManagerId))
   }
 
-  class MockBlockTransferService(val maxFailures: Int) extends BlockTransferService {
+  private def testShuffleBlockDecommissioning(maxShuffleSize: Option[Int], willReject: Boolean) = {
+    maxShuffleSize.foreach{ size =>
+      conf.set(STORAGE_DECOMMISSION_SHUFFLE_MAX_DISK_SIZE.key, s"${size}b")
+    }
+    val shuffleManager1 = makeSortShuffleManager(Some(conf))
+    val bm1 = makeBlockManager(3500, "exec1", shuffleManager = shuffleManager1)
+    shuffleManager1.shuffleBlockResolver._blockManager = bm1
+
+    val shuffleManager2 = makeSortShuffleManager(Some(conf))
+    val bm2 = makeBlockManager(3500, "exec2", shuffleManager = shuffleManager2)
+    shuffleManager2.shuffleBlockResolver._blockManager = bm2
+
+    val blockSize = 5
+    val shuffleDataBlockContent = Array[Byte](0, 1, 2, 3, 4)
+    val shuffleData = ShuffleDataBlockId(0, 0, 0)
+    val shuffleData2 = ShuffleDataBlockId(1, 0, 0)
+    Files.write(bm1.diskBlockManager.getFile(shuffleData).toPath(), shuffleDataBlockContent)
+    Files.write(bm2.diskBlockManager.getFile(shuffleData2).toPath(), shuffleDataBlockContent)
+    val shuffleIndexBlockContent = Array[Byte](5, 6, 7, 8, 9)
+    val shuffleIndex = ShuffleIndexBlockId(0, 0, 0)
+    val shuffleIndex2 = ShuffleIndexBlockId(1, 0, 0)
+    Files.write(bm1.diskBlockManager.getFile(shuffleIndex).toPath(), shuffleIndexBlockContent)
+    Files.write(bm2.diskBlockManager.getFile(shuffleIndex2).toPath(), shuffleIndexBlockContent)
+
+    mapOutputTracker.registerShuffle(0, 1, MergeStatus.SHUFFLE_PUSH_DUMMY_NUM_REDUCES)
+    val decomManager = new BlockManagerDecommissioner(conf, bm1)
+    try {
+      mapOutputTracker.registerMapOutput(0, 0, MapStatus(bm1.blockManagerId, Array(blockSize), 0))
+      assert(mapOutputTracker.shuffleStatuses(0).mapStatuses(0).location === bm1.blockManagerId)
+
+      val env = mock(classOf[SparkEnv])
+      when(env.conf).thenReturn(conf)
+      SparkEnv.set(env)
+
+      decomManager.refreshOffloadingShuffleBlocks()
+
+      if (willReject) {
+        eventually(timeout(1.second), interval(10.milliseconds)) {
+          assert(mapOutputTracker.shuffleStatuses(0).mapStatuses(0).location === bm2.blockManagerId)
+        }
+        assert(Files.readAllBytes(bm2.diskBlockManager.getFile(shuffleData).toPath())
+          === shuffleDataBlockContent)
+        assert(Files.readAllBytes(bm2.diskBlockManager.getFile(shuffleIndex).toPath())
+          === shuffleIndexBlockContent)
+      } else {
+        Thread.sleep(1000)
+        assert(mapOutputTracker.shuffleStatuses(0).mapStatuses(0).location === bm1.blockManagerId)
+      }
+    } finally {
+      mapOutputTracker.unregisterShuffle(0)
+      // Avoid thread leak
+      decomManager.stopOffloadingShuffleBlocks()
+    }
+  }
+
+  test("test migration of shuffle blocks during decommissioning - no limit") {
+    testShuffleBlockDecommissioning(None, true)
+  }
+
+  test("test migration of shuffle blocks during decommissioning - larger limit") {
+    testShuffleBlockDecommissioning(Some(10000), true)
+  }
+
+  test("[SPARK-34363]test migration of shuffle blocks during decommissioning - small limit") {
+    testShuffleBlockDecommissioning(Some(1), false)
+  }
+
+  test("SPARK-32919: Shuffle push merger locations should be bounded with in" +
+    " spark.shuffle.push.retainedMergerLocations") {
+    assert(master.getShufflePushMergerLocations(10, Set.empty).isEmpty)
+    makeBlockManager(100, "execA",
+      transferService = Some(new MockBlockTransferService(10, "hostA")))
+    makeBlockManager(100, "execB",
+      transferService = Some(new MockBlockTransferService(10, "hostB")))
+    makeBlockManager(100, "execC",
+      transferService = Some(new MockBlockTransferService(10, "hostC")))
+    makeBlockManager(100, "execD",
+      transferService = Some(new MockBlockTransferService(10, "hostD")))
+    makeBlockManager(100, "execE",
+      transferService = Some(new MockBlockTransferService(10, "hostA")))
+    assert(master.getShufflePushMergerLocations(10, Set.empty).size == 4)
+    assert(master.getShufflePushMergerLocations(10, Set.empty).map(_.host).sorted ===
+      Seq("hostC", "hostD", "hostA", "hostB").sorted)
+    assert(master.getShufflePushMergerLocations(10, Set("hostB")).size == 3)
+  }
+
+  test("SPARK-32919: Prefer active executor locations for shuffle push mergers") {
+    makeBlockManager(100, "execA",
+      transferService = Some(new MockBlockTransferService(10, "hostA")))
+    makeBlockManager(100, "execB",
+      transferService = Some(new MockBlockTransferService(10, "hostB")))
+    makeBlockManager(100, "execC",
+      transferService = Some(new MockBlockTransferService(10, "hostC")))
+    makeBlockManager(100, "execD",
+      transferService = Some(new MockBlockTransferService(10, "hostD")))
+    makeBlockManager(100, "execE",
+      transferService = Some(new MockBlockTransferService(10, "hostA")))
+    assert(master.getShufflePushMergerLocations(5, Set.empty).size == 4)
+
+    master.removeExecutor("execA")
+    master.removeExecutor("execE")
+
+    assert(master.getShufflePushMergerLocations(3, Set.empty).size == 3)
+    assert(master.getShufflePushMergerLocations(3, Set.empty).map(_.host).sorted ===
+      Seq("hostC", "hostB", "hostD").sorted)
+    assert(master.getShufflePushMergerLocations(4, Set.empty).map(_.host).sorted ===
+      Seq("hostB", "hostA", "hostC", "hostD").sorted)
+  }
+
+  test("SPARK-33387 Support ordered shuffle block migration") {
+    val blocks: Seq[ShuffleBlockInfo] = Seq(
+      ShuffleBlockInfo(1, 0L),
+      ShuffleBlockInfo(0, 1L),
+      ShuffleBlockInfo(0, 0L),
+      ShuffleBlockInfo(1, 1L))
+    val sortedBlocks = blocks.sortBy(b => (b.shuffleId, b.mapId))
+
+    val resolver = mock(classOf[MigratableResolver])
+    when(resolver.getStoredShuffles).thenReturn(blocks)
+
+    val bm = mock(classOf[BlockManager])
+    when(bm.migratableResolver).thenReturn(resolver)
+    when(bm.getPeers(mc.any())).thenReturn(Seq.empty)
+
+    val decomManager = new BlockManagerDecommissioner(conf, bm)
+    decomManager.refreshOffloadingShuffleBlocks()
+
+    assert(sortedBlocks.sameElements(decomManager.shufflesToMigrate.asScala.map(_._1)))
+  }
+
+  test("SPARK-34193: Potential race condition during decommissioning with TorrentBroadcast") {
+    // Validate that we allow putting of broadcast blocks during decommissioning
+    val exec1 = "exec1"
+
+    val store = makeBlockManager(1000, exec1)
+    master.decommissionBlockManagers(Seq(exec1))
+    val a = new Array[Byte](1)
+    // Put a broadcast block, no exception
+    val broadcast0BlockId = BroadcastBlockId(0)
+    store.putSingle(broadcast0BlockId, a, StorageLevel.DISK_ONLY)
+  }
+
+  class MockBlockTransferService(
+      val maxFailures: Int,
+      override val hostName: String = "MockBlockTransferServiceHost") extends BlockTransferService {
     var numCalls = 0
     var tempFileManager: DownloadFileManager = null
 
@@ -1782,8 +2094,6 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
 
     override def close(): Unit = {}
 
-    override def hostName: String = { "MockBlockTransferServiceHost" }
-
     override def port: Int = { 63332 }
 
     override def uploadBlock(
diff --git a/core/src/test/scala/org/apache/spark/storage/BlockReplicationPolicySuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockReplicationPolicySuite.scala
index 4000218e71a8b..9ce4acc75ec43 100644
--- a/core/src/test/scala/org/apache/spark/storage/BlockReplicationPolicySuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/BlockReplicationPolicySuite.scala
@@ -21,7 +21,8 @@ import scala.collection.mutable
 import scala.language.implicitConversions
 import scala.util.Random
 
-import org.scalatest.{BeforeAndAfter, Matchers}
+import org.scalatest.BeforeAndAfter
+import org.scalatest.matchers.must.Matchers
 
 import org.apache.spark.{LocalSparkContext, SparkFunSuite}
 
diff --git a/core/src/test/scala/org/apache/spark/storage/FallbackStorageSuite.scala b/core/src/test/scala/org/apache/spark/storage/FallbackStorageSuite.scala
new file mode 100644
index 0000000000000..3e225fe7d2470
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/storage/FallbackStorageSuite.scala
@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.storage
+
+import java.io.{DataOutputStream, File, FileOutputStream, IOException}
+import java.nio.file.Files
+
+import scala.concurrent.duration._
+
+import org.apache.hadoop.conf.Configuration
+import org.mockito.{ArgumentMatchers => mc}
+import org.mockito.Mockito.{mock, times, verify, when}
+import org.scalatest.concurrent.Eventually.{eventually, interval, timeout}
+
+import org.apache.spark.{LocalSparkContext, SparkConf, SparkContext, SparkFunSuite, TestUtils}
+import org.apache.spark.LocalSparkContext.withSpark
+import org.apache.spark.internal.config._
+import org.apache.spark.launcher.SparkLauncher.{EXECUTOR_MEMORY, SPARK_MASTER}
+import org.apache.spark.network.BlockTransferService
+import org.apache.spark.network.buffer.ManagedBuffer
+import org.apache.spark.scheduler.ExecutorDecommissionInfo
+import org.apache.spark.scheduler.cluster.StandaloneSchedulerBackend
+import org.apache.spark.shuffle.{IndexShuffleBlockResolver, ShuffleBlockInfo}
+import org.apache.spark.shuffle.IndexShuffleBlockResolver.NOOP_REDUCE_ID
+import org.apache.spark.util.Utils.tryWithResource
+
+class FallbackStorageSuite extends SparkFunSuite with LocalSparkContext {
+
+  def getSparkConf(initialExecutor: Int = 1, minExecutor: Int = 1): SparkConf = {
+    new SparkConf(false)
+      .setAppName(getClass.getName)
+      .set(SPARK_MASTER, s"local-cluster[$initialExecutor,1,1024]")
+      .set(EXECUTOR_MEMORY, "1g")
+      .set(UI.UI_ENABLED, false)
+      .set(DYN_ALLOCATION_ENABLED, true)
+      .set(DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED, true)
+      .set(DYN_ALLOCATION_INITIAL_EXECUTORS, initialExecutor)
+      .set(DYN_ALLOCATION_MIN_EXECUTORS, minExecutor)
+      .set(DECOMMISSION_ENABLED, true)
+      .set(STORAGE_DECOMMISSION_ENABLED, true)
+      .set(STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED, true)
+      .set(STORAGE_DECOMMISSION_FALLBACK_STORAGE_PATH,
+         Files.createTempDirectory("tmp").toFile.getAbsolutePath + "/")
+  }
+
+  test("fallback storage APIs - copy/exists") {
+    val conf = new SparkConf(false)
+      .set("spark.app.id", "testId")
+      .set(SHUFFLE_COMPRESS, false)
+      .set(STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED, true)
+      .set(STORAGE_DECOMMISSION_FALLBACK_STORAGE_PATH,
+        Files.createTempDirectory("tmp").toFile.getAbsolutePath + "/")
+    val fallbackStorage = new FallbackStorage(conf)
+    val bmm = new BlockManagerMaster(new NoopRpcEndpointRef(conf), null, conf, false)
+
+    val bm = mock(classOf[BlockManager])
+    val dbm = new DiskBlockManager(conf, false)
+    when(bm.diskBlockManager).thenReturn(dbm)
+    when(bm.master).thenReturn(bmm)
+    val resolver = new IndexShuffleBlockResolver(conf, bm)
+    when(bm.migratableResolver).thenReturn(resolver)
+
+    resolver.getIndexFile(1, 1L).createNewFile()
+    resolver.getDataFile(1, 1L).createNewFile()
+
+    val indexFile = resolver.getIndexFile(1, 2L)
+    tryWithResource(new FileOutputStream(indexFile)) { fos =>
+      tryWithResource(new DataOutputStream(fos)) { dos =>
+        dos.writeLong(0)
+        dos.writeLong(4)
+      }
+    }
+
+    val dataFile = resolver.getDataFile(1, 2L)
+    tryWithResource(new FileOutputStream(dataFile)) { fos =>
+      tryWithResource(new DataOutputStream(fos)) { dos =>
+        dos.writeLong(0)
+      }
+    }
+
+    fallbackStorage.copy(ShuffleBlockInfo(1, 1L), bm)
+    fallbackStorage.copy(ShuffleBlockInfo(1, 2L), bm)
+
+    assert(fallbackStorage.exists(1, ShuffleIndexBlockId(1, 1L, NOOP_REDUCE_ID).name))
+    assert(fallbackStorage.exists(1, ShuffleDataBlockId(1, 1L, NOOP_REDUCE_ID).name))
+    assert(fallbackStorage.exists(1, ShuffleIndexBlockId(1, 2L, NOOP_REDUCE_ID).name))
+    assert(fallbackStorage.exists(1, ShuffleDataBlockId(1, 2L, NOOP_REDUCE_ID).name))
+
+    // The files for shuffle 1 and map 1 are empty intentionally.
+    intercept[java.io.EOFException] {
+      FallbackStorage.read(conf, ShuffleBlockId(1, 1L, 0))
+    }
+    FallbackStorage.read(conf, ShuffleBlockId(1, 2L, 0))
+  }
+
+  test("SPARK-34142: fallback storage API - cleanUp") {
+    withTempDir { dir =>
+      Seq(true, false).foreach { cleanUp =>
+        val appId = s"test$cleanUp"
+        val conf = new SparkConf(false)
+          .set("spark.app.id", appId)
+          .set(STORAGE_DECOMMISSION_FALLBACK_STORAGE_PATH, dir.getAbsolutePath + "/")
+          .set(STORAGE_DECOMMISSION_FALLBACK_STORAGE_CLEANUP, cleanUp)
+
+        val location = new File(dir, appId)
+        assert(location.mkdir())
+        assert(location.exists())
+        FallbackStorage.cleanUp(conf, new Configuration())
+        assert(location.exists() != cleanUp)
+      }
+    }
+  }
+
+  test("migrate shuffle data to fallback storage") {
+    val conf = new SparkConf(false)
+      .set("spark.app.id", "testId")
+      .set(STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED, true)
+      .set(STORAGE_DECOMMISSION_FALLBACK_STORAGE_PATH,
+        Files.createTempDirectory("tmp").toFile.getAbsolutePath + "/")
+
+    val ids = Set((1, 1L, 1))
+    val bm = mock(classOf[BlockManager])
+    val dbm = new DiskBlockManager(conf, false)
+    when(bm.diskBlockManager).thenReturn(dbm)
+    val indexShuffleBlockResolver = new IndexShuffleBlockResolver(conf, bm)
+    val indexFile = indexShuffleBlockResolver.getIndexFile(1, 1L)
+    val dataFile = indexShuffleBlockResolver.getDataFile(1, 1L)
+    indexFile.createNewFile()
+    dataFile.createNewFile()
+
+    val resolver = mock(classOf[IndexShuffleBlockResolver])
+    when(resolver.getStoredShuffles())
+      .thenReturn(ids.map(triple => ShuffleBlockInfo(triple._1, triple._2)).toSeq)
+    ids.foreach { case (shuffleId: Int, mapId: Long, reduceId: Int) =>
+      when(resolver.getMigrationBlocks(mc.any()))
+        .thenReturn(List(
+          (ShuffleIndexBlockId(shuffleId, mapId, reduceId), mock(classOf[ManagedBuffer])),
+          (ShuffleDataBlockId(shuffleId, mapId, reduceId), mock(classOf[ManagedBuffer]))))
+      when(resolver.getIndexFile(shuffleId, mapId)).thenReturn(indexFile)
+      when(resolver.getDataFile(shuffleId, mapId)).thenReturn(dataFile)
+    }
+
+    when(bm.getPeers(mc.any()))
+      .thenReturn(Seq(FallbackStorage.FALLBACK_BLOCK_MANAGER_ID))
+    val bmm = new BlockManagerMaster(new NoopRpcEndpointRef(conf), null, conf, false)
+    when(bm.master).thenReturn(bmm)
+    val blockTransferService = mock(classOf[BlockTransferService])
+    when(blockTransferService.uploadBlockSync(mc.any(), mc.any(), mc.any(), mc.any(), mc.any(),
+      mc.any(), mc.any())).thenThrow(new IOException)
+    when(bm.blockTransferService).thenReturn(blockTransferService)
+    when(bm.migratableResolver).thenReturn(resolver)
+    when(bm.getMigratableRDDBlocks()).thenReturn(Seq())
+
+    val decommissioner = new BlockManagerDecommissioner(conf, bm)
+
+    try {
+      decommissioner.start()
+      val fallbackStorage = new FallbackStorage(conf)
+      eventually(timeout(10.second), interval(1.seconds)) {
+        // uploadBlockSync is not used
+        verify(blockTransferService, times(1))
+          .uploadBlockSync(mc.any(), mc.any(), mc.any(), mc.any(), mc.any(), mc.any(), mc.any())
+
+        Seq("shuffle_1_1_0.index", "shuffle_1_1_0.data").foreach { filename =>
+          assert(fallbackStorage.exists(shuffleId = 1, filename))
+        }
+      }
+    } finally {
+      decommissioner.stop()
+    }
+  }
+
+  test("Upload from all decommissioned executors") {
+    sc = new SparkContext(getSparkConf(2, 2))
+    withSpark(sc) { sc =>
+      TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
+      val rdd1 = sc.parallelize(1 to 10, 10)
+      val rdd2 = rdd1.map(x => (x % 2, 1))
+      val rdd3 = rdd2.reduceByKey(_ + _)
+      assert(rdd3.count() === 2)
+
+      // Decommission all
+      val sched = sc.schedulerBackend.asInstanceOf[StandaloneSchedulerBackend]
+      sc.getExecutorIds().foreach {
+        sched.decommissionExecutor(_, ExecutorDecommissionInfo(""), false)
+      }
+
+      val files = Seq("shuffle_0_0_0.index", "shuffle_0_0_0.data")
+      val fallbackStorage = new FallbackStorage(sc.getConf)
+      // Uploading is not started yet.
+      files.foreach { file => assert(!fallbackStorage.exists(0, file)) }
+
+      // Uploading is completed on decommissioned executors
+      eventually(timeout(20.seconds), interval(1.seconds)) {
+        files.foreach { file => assert(fallbackStorage.exists(0, file)) }
+      }
+
+      // All executors are still alive.
+      assert(sc.getExecutorIds().size == 2)
+    }
+  }
+
+  test("Upload multi stages") {
+    sc = new SparkContext(getSparkConf())
+    withSpark(sc) { sc =>
+      TestUtils.waitUntilExecutorsUp(sc, 1, 60000)
+      val rdd1 = sc.parallelize(1 to 10, 2)
+      val rdd2 = rdd1.map(x => (x % 2, 1))
+      val rdd3 = rdd2.reduceByKey(_ + _)
+      val rdd4 = rdd3.sortByKey()
+      assert(rdd4.count() === 2)
+
+      val shuffle0_files = Seq(
+        "shuffle_0_0_0.index", "shuffle_0_0_0.data",
+        "shuffle_0_1_0.index", "shuffle_0_1_0.data")
+      val shuffle1_files = Seq(
+        "shuffle_1_4_0.index", "shuffle_1_4_0.data",
+        "shuffle_1_5_0.index", "shuffle_1_5_0.data")
+      val fallbackStorage = new FallbackStorage(sc.getConf)
+      shuffle0_files.foreach { file => assert(!fallbackStorage.exists(0, file)) }
+      shuffle1_files.foreach { file => assert(!fallbackStorage.exists(1, file)) }
+
+      // Decommission all
+      val sched = sc.schedulerBackend.asInstanceOf[StandaloneSchedulerBackend]
+      sc.getExecutorIds().foreach {
+        sched.decommissionExecutor(_, ExecutorDecommissionInfo(""), false)
+      }
+
+      eventually(timeout(10.seconds), interval(1.seconds)) {
+        shuffle0_files.foreach { file => assert(fallbackStorage.exists(0, file)) }
+        shuffle1_files.foreach { file => assert(fallbackStorage.exists(1, file)) }
+      }
+    }
+  }
+
+  Seq("lz4", "lzf", "snappy", "zstd").foreach { codec =>
+    test(s"$codec - Newly added executors should access old data from remote storage") {
+      sc = new SparkContext(getSparkConf(2, 0).set(IO_COMPRESSION_CODEC, codec))
+      withSpark(sc) { sc =>
+        TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
+        val rdd1 = sc.parallelize(1 to 10, 2)
+        val rdd2 = rdd1.map(x => (x % 2, 1))
+        val rdd3 = rdd2.reduceByKey(_ + _)
+        assert(rdd3.collect() === Array((0, 5), (1, 5)))
+
+        // Decommission all
+        val sched = sc.schedulerBackend.asInstanceOf[StandaloneSchedulerBackend]
+        sc.getExecutorIds().foreach {
+          sched.decommissionExecutor(_, ExecutorDecommissionInfo(""), false)
+        }
+
+        // Make it sure that fallback storage are ready
+        val fallbackStorage = new FallbackStorage(sc.getConf)
+        eventually(timeout(10.seconds), interval(1.seconds)) {
+          Seq(
+            "shuffle_0_0_0.index", "shuffle_0_0_0.data",
+            "shuffle_0_1_0.index", "shuffle_0_1_0.data").foreach { file =>
+            assert(fallbackStorage.exists(0, file))
+          }
+        }
+
+        // Since the data is safe, force to shrink down to zero executor
+        sc.getExecutorIds().foreach { id =>
+          sched.killExecutor(id)
+        }
+        eventually(timeout(20.seconds), interval(1.seconds)) {
+          assert(sc.getExecutorIds().isEmpty)
+        }
+
+        // Dynamic allocation will start new executors
+        assert(rdd3.collect() === Array((0, 5), (1, 5)))
+        assert(rdd3.sortByKey().count() == 2)
+        assert(sc.getExecutorIds().nonEmpty)
+      }
+    }
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/storage/FlatmapIteratorSuite.scala b/core/src/test/scala/org/apache/spark/storage/FlatmapIteratorSuite.scala
index fc16fe362882c..e719c722d01d3 100644
--- a/core/src/test/scala/org/apache/spark/storage/FlatmapIteratorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/FlatmapIteratorSuite.scala
@@ -37,7 +37,7 @@ class FlatmapIteratorSuite extends SparkFunSuite with LocalSparkContext {
     val expand_size = 100
     val data = sc.parallelize((1 to 5).toSeq).
       flatMap( x => Stream.range(0, expand_size))
-    var persisted = data.persist(StorageLevel.DISK_ONLY)
+    val persisted = data.persist(StorageLevel.DISK_ONLY)
     assert(persisted.count()===500)
     assert(persisted.filter(_==1).count()===5)
   }
@@ -48,7 +48,7 @@ class FlatmapIteratorSuite extends SparkFunSuite with LocalSparkContext {
     val expand_size = 100
     val data = sc.parallelize((1 to 5).toSeq).
       flatMap(x => Stream.range(0, expand_size))
-    var persisted = data.persist(StorageLevel.MEMORY_ONLY)
+    val persisted = data.persist(StorageLevel.MEMORY_ONLY)
     assert(persisted.count()===500)
     assert(persisted.filter(_==1).count()===5)
   }
diff --git a/core/src/test/scala/org/apache/spark/storage/MemoryStoreSuite.scala b/core/src/test/scala/org/apache/spark/storage/MemoryStoreSuite.scala
index ccd7e4b62ad9e..d6a4e5bb2b930 100644
--- a/core/src/test/scala/org/apache/spark/storage/MemoryStoreSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/MemoryStoreSuite.scala
@@ -26,6 +26,7 @@ import org.scalatest._
 
 import org.apache.spark._
 import org.apache.spark.internal.config._
+import org.apache.spark.internal.config.Tests.TEST_USE_COMPRESSED_OOPS_KEY
 import org.apache.spark.memory.{MemoryMode, UnifiedMemoryManager}
 import org.apache.spark.serializer.{KryoSerializer, SerializerManager}
 import org.apache.spark.storage.memory.{BlockEvictionHandler, MemoryStore, PartiallySerializedBlock, PartiallyUnrolledIterator}
@@ -38,7 +39,7 @@ class MemoryStoreSuite
   with BeforeAndAfterEach
   with ResetSystemProperties {
 
-  var conf: SparkConf = new SparkConf(false)
+  val conf: SparkConf = new SparkConf(false)
     .set(STORAGE_UNROLL_MEMORY_THRESHOLD, 512L)
 
   // Reuse a serializer across tests to avoid creating a new thread-local buffer on each test
@@ -51,12 +52,34 @@ class MemoryStoreSuite
   implicit def StringToBlockId(value: String): BlockId = new TestBlockId(value)
   def rdd(rddId: Int, splitId: Int): RDDBlockId = RDDBlockId(rddId, splitId)
 
+  // Save modified system properties so that we can restore them after tests.
+  val originalArch = System.getProperty("os.arch")
+  val originalCompressedOops = System.getProperty(TEST_USE_COMPRESSED_OOPS_KEY)
+
+  def reinitializeSizeEstimator(arch: String, useCompressedOops: String): Unit = {
+    def set(k: String, v: String): Unit = {
+      if (v == null) {
+        System.clearProperty(k)
+      } else {
+        System.setProperty(k, v)
+      }
+    }
+    set("os.arch", arch)
+    set(TEST_USE_COMPRESSED_OOPS_KEY, useCompressedOops)
+    val initialize = PrivateMethod[Unit](Symbol("initialize"))
+    SizeEstimator invokePrivate initialize()
+  }
+
   override def beforeEach(): Unit = {
     super.beforeEach()
     // Set the arch to 64-bit and compressedOops to true to get a deterministic test-case
-    System.setProperty("os.arch", "amd64")
-    val initialize = PrivateMethod[Unit](Symbol("initialize"))
-    SizeEstimator invokePrivate initialize()
+    reinitializeSizeEstimator("amd64", "true")
+  }
+
+  override def afterEach(): Unit = {
+    super.afterEach()
+    // Restore system properties and SizeEstimator to their original states.
+    reinitializeSizeEstimator(originalArch, originalCompressedOops)
   }
 
   def makeMemoryStore(maxMem: Long): (MemoryStore, BlockInfoManager) = {
diff --git a/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala b/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala
index 43917a5b83bb0..99c43b12d6553 100644
--- a/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala
@@ -66,6 +66,16 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
     transfer
   }
 
+  private def createMockBlockManager(): BlockManager = {
+    val blockManager = mock(classOf[BlockManager])
+    val localBmId = BlockManagerId("test-client", "test-local-host", 1)
+    doReturn(localBmId).when(blockManager).blockManagerId
+    // By default, the mock BlockManager returns None for hostLocalDirManager. One could
+    // still use initHostLocalDirManager() to specify a custom hostLocalDirManager.
+    doReturn(None).when(blockManager).hostLocalDirManager
+    blockManager
+  }
+
   private def initHostLocalDirManager(
       blockManager: BlockManager,
       hostLocalDirs: Map[String, Array[String]]): Unit = {
@@ -73,9 +83,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
     val hostLocalDirManager = new HostLocalDirManager(
       futureExecutionContext = global,
       cacheSize = 1,
-      externalBlockStoreClient = mockExternalBlockStoreClient,
-      host = "localhost",
-      externalShuffleServicePort = 7337)
+      blockStoreClient = mockExternalBlockStoreClient)
 
     when(blockManager.hostLocalDirManager).thenReturn(Some(hostLocalDirManager))
     when(mockExternalBlockStoreClient.getHostLocalDirs(any(), any(), any(), any()))
@@ -116,9 +124,8 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
   }
 
   test("successful 3 local + 4 host local + 2 remote reads") {
-    val blockManager = mock(classOf[BlockManager])
-    val localBmId = BlockManagerId("test-local-client", "test-local-host", 1)
-    doReturn(localBmId).when(blockManager).blockManagerId
+    val blockManager = createMockBlockManager()
+    val localBmId = blockManager.blockManagerId
 
     // Make sure blockManager.getBlockData would return the blocks
     val localBlocks = Map[BlockId, ManagedBuffer](
@@ -197,13 +204,11 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
 
     // 2 remote blocks are read from the same block manager
     verify(transfer, times(1)).fetchBlocks(any(), any(), any(), any(), any(), any())
-    assert(blockManager.hostLocalDirManager.get.getCachedHostLocalDirs().size === 1)
+    assert(blockManager.hostLocalDirManager.get.getCachedHostLocalDirs.size === 1)
   }
 
   test("error during accessing host local dirs for executors") {
-    val blockManager = mock(classOf[BlockManager])
-    val localBmId = BlockManagerId("test-local-client", "test-local-host", 1)
-    doReturn(localBmId).when(blockManager).blockManagerId
+    val blockManager = createMockBlockManager()
     val hostLocalBlocks = Map[BlockId, ManagedBuffer](
       ShuffleBlockId(0, 1, 0) -> createMockManagedBuffer())
 
@@ -218,9 +223,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
     val hostLocalDirManager = new HostLocalDirManager(
       futureExecutionContext = global,
       cacheSize = 1,
-      externalBlockStoreClient = mockExternalBlockStoreClient,
-      host = "localhost",
-      externalShuffleServicePort = 7337)
+      blockStoreClient = mockExternalBlockStoreClient)
 
     when(blockManager.hostLocalDirManager).thenReturn(Some(hostLocalDirManager))
     when(mockExternalBlockStoreClient.getHostLocalDirs(any(), any(), any(), any()))
@@ -256,10 +259,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
   }
 
   test("Hit maxBytesInFlight limitation before maxBlocksInFlightPerAddress") {
-    val blockManager = mock(classOf[BlockManager])
-    val localBmId = BlockManagerId("test-client", "test-local-host", 1)
-    doReturn(localBmId).when(blockManager).blockManagerId
-
+    val blockManager = createMockBlockManager()
     val remoteBmId1 = BlockManagerId("test-remote-client-1", "test-remote-host1", 1)
     val remoteBmId2 = BlockManagerId("test-remote-client-2", "test-remote-host2", 2)
     val blockId1 = ShuffleBlockId(0, 1, 0)
@@ -301,10 +301,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
   }
 
   test("Hit maxBlocksInFlightPerAddress limitation before maxBytesInFlight") {
-    val blockManager = mock(classOf[BlockManager])
-    val localBmId = BlockManagerId("test-client", "test-local-host", 1)
-    doReturn(localBmId).when(blockManager).blockManagerId
-
+    val blockManager = createMockBlockManager()
     val remoteBmId = BlockManagerId("test-remote-client-1", "test-remote-host", 2)
     val blockId1 = ShuffleBlockId(0, 1, 0)
     val blockId2 = ShuffleBlockId(0, 2, 0)
@@ -348,10 +345,8 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
   }
 
   test("fetch continuous blocks in batch successful 3 local + 4 host local + 2 remote reads") {
-    val blockManager = mock(classOf[BlockManager])
-    val localBmId = BlockManagerId("test-client", "test-local-host", 1)
-    doReturn(localBmId).when(blockManager).blockManagerId
-
+    val blockManager = createMockBlockManager()
+    val localBmId = blockManager.blockManagerId
     // Make sure blockManager.getBlockData would return the merged block
     val localBlocks = Seq[BlockId](
       ShuffleBlockId(0, 0, 0),
@@ -431,14 +426,11 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
     verify(blockManager, times(1))
       .getHostLocalShuffleData(any(), meq(Array("local-dir")))
 
-    assert(blockManager.hostLocalDirManager.get.getCachedHostLocalDirs().size === 1)
+    assert(blockManager.hostLocalDirManager.get.getCachedHostLocalDirs.size === 1)
   }
 
   test("fetch continuous blocks in batch should respect maxBytesInFlight") {
-    val blockManager = mock(classOf[BlockManager])
-    val localBmId = BlockManagerId("test-client", "test-local-host", 1)
-    doReturn(localBmId).when(blockManager).blockManagerId
-
+    val blockManager = createMockBlockManager()
     // Make sure remote blocks would return the merged block
     val remoteBmId1 = BlockManagerId("test-client-1", "test-client-1", 1)
     val remoteBmId2 = BlockManagerId("test-client-2", "test-client-2", 2)
@@ -494,10 +486,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
   }
 
   test("fetch continuous blocks in batch should respect maxBlocksInFlightPerAddress") {
-    val blockManager = mock(classOf[BlockManager])
-    val localBmId = BlockManagerId("test-client", "test-local-host", 1)
-    doReturn(localBmId).when(blockManager).blockManagerId
-
+    val blockManager = createMockBlockManager()
     // Make sure remote blocks would return the merged block
     val remoteBmId = BlockManagerId("test-client-1", "test-client-1", 1)
     val remoteBlocks = Seq(
@@ -549,10 +538,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
   }
 
   test("release current unexhausted buffer in case the task completes early") {
-    val blockManager = mock(classOf[BlockManager])
-    val localBmId = BlockManagerId("test-client", "test-client", 1)
-    doReturn(localBmId).when(blockManager).blockManagerId
-
+    val blockManager = createMockBlockManager()
     // Make sure remote blocks would return
     val remoteBmId = BlockManagerId("test-client-1", "test-client-1", 2)
     val blocks = Map[BlockId, ManagedBuffer](
@@ -617,10 +603,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
   }
 
   test("fail all blocks if any of the remote request fails") {
-    val blockManager = mock(classOf[BlockManager])
-    val localBmId = BlockManagerId("test-client", "test-client", 1)
-    doReturn(localBmId).when(blockManager).blockManagerId
-
+    val blockManager = createMockBlockManager()
     // Make sure remote blocks would return
     val remoteBmId = BlockManagerId("test-client-1", "test-client-1", 2)
     val blocks = Map[BlockId, ManagedBuffer](
@@ -707,10 +690,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
   }
 
   test("retry corrupt blocks") {
-    val blockManager = mock(classOf[BlockManager])
-    val localBmId = BlockManagerId("test-client", "test-client", 1)
-    doReturn(localBmId).when(blockManager).blockManagerId
-
+    val blockManager = createMockBlockManager()
     // Make sure remote blocks would return
     val remoteBmId = BlockManagerId("test-client-1", "test-client-1", 2)
     val blocks = Map[BlockId, ManagedBuffer](
@@ -785,9 +765,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
 
   test("big blocks are also checked for corruption") {
     val streamLength = 10000L
-    val blockManager = mock(classOf[BlockManager])
-    val localBlockManagerId = BlockManagerId("local-client", "local-client", 1)
-    doReturn(localBlockManagerId).when(blockManager).blockManagerId
+    val blockManager = createMockBlockManager()
 
     // This stream will throw IOException when the first byte is read
     val corruptBuffer1 = mockCorruptBuffer(streamLength, 0)
@@ -906,10 +884,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
   }
 
   test("retry corrupt blocks (disabled)") {
-    val blockManager = mock(classOf[BlockManager])
-    val localBmId = BlockManagerId("test-client", "test-client", 1)
-    doReturn(localBmId).when(blockManager).blockManagerId
-
+    val blockManager = createMockBlockManager()
     // Make sure remote blocks would return
     val remoteBmId = BlockManagerId("test-client-1", "test-client-1", 2)
     val blocks = Map[BlockId, ManagedBuffer](
@@ -971,10 +946,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
 
   test("Blocks should be shuffled to disk when size of the request is above the" +
     " threshold(maxReqSizeShuffleToMem).") {
-    val blockManager = mock(classOf[BlockManager])
-    val localBmId = BlockManagerId("test-client", "test-client", 1)
-    doReturn(localBmId).when(blockManager).blockManagerId
-
+    val blockManager = createMockBlockManager()
     val diskBlockManager = mock(classOf[DiskBlockManager])
     val tmpDir = Utils.createTempDir()
     doReturn{
@@ -1036,10 +1008,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
   }
 
   test("fail zero-size blocks") {
-    val blockManager = mock(classOf[BlockManager])
-    val localBmId = BlockManagerId("test-client", "test-client", 1)
-    doReturn(localBmId).when(blockManager).blockManagerId
-
+    val blockManager = createMockBlockManager()
     // Make sure remote blocks would return
     val remoteBmId = BlockManagerId("test-client-1", "test-client-1", 2)
     val blocks = Map[BlockId, ManagedBuffer](
@@ -1047,7 +1016,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
       ShuffleBlockId(0, 1, 0) -> createMockManagedBuffer()
     )
 
-    val transfer = createMockTransfer(blocks.mapValues(_ => createMockManagedBuffer(0)))
+    val transfer = createMockTransfer(blocks.mapValues(_ => createMockManagedBuffer(0)).toMap)
 
     val blocksByAddress = Seq[(BlockManagerId, Seq[(BlockId, Long, Int)])](
       (remoteBmId, blocks.keys.map(blockId => (blockId, 1L, 0)).toSeq))
diff --git a/core/src/test/scala/org/apache/spark/storage/TopologyMapperSuite.scala b/core/src/test/scala/org/apache/spark/storage/TopologyMapperSuite.scala
index 0bc26adeeb443..03e27bfaf2d45 100644
--- a/core/src/test/scala/org/apache/spark/storage/TopologyMapperSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/TopologyMapperSuite.scala
@@ -19,7 +19,8 @@ package org.apache.spark.storage
 
 import java.io.{File, FileOutputStream}
 
-import org.scalatest.{BeforeAndAfter, Matchers}
+import org.scalatest.BeforeAndAfter
+import org.scalatest.matchers.must.Matchers
 
 import org.apache.spark._
 import org.apache.spark.internal.config.STORAGE_REPLICATION_TOPOLOGY_FILE
diff --git a/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala
new file mode 100644
index 0000000000000..459af6748e0e0
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ui
+
+import org.openqa.selenium.{JavascriptExecutor, WebDriver}
+import org.openqa.selenium.chrome.{ChromeDriver, ChromeOptions}
+
+import org.apache.spark.tags.ChromeUITest
+
+/**
+ * Selenium tests for the Spark Web UI with Chrome.
+ */
+@ChromeUITest
+class ChromeUISeleniumSuite extends RealBrowserUISeleniumSuite("webdriver.chrome.driver") {
+
+  override var webDriver: WebDriver with JavascriptExecutor = _
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    val chromeOptions = new ChromeOptions
+    chromeOptions.addArguments("--headless", "--disable-gpu")
+    webDriver = new ChromeDriver(chromeOptions)
+  }
+
+  override def afterAll(): Unit = {
+    try {
+      if (webDriver != null) {
+        webDriver.quit()
+      }
+    } finally {
+      super.afterAll()
+    }
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala
new file mode 100644
index 0000000000000..3f296acdeb326
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ui
+
+import org.openqa.selenium.{By, JavascriptExecutor, WebDriver}
+import org.scalatest.BeforeAndAfterAll
+import org.scalatest.concurrent.Eventually._
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
+import org.scalatest.time.SpanSugar._
+import org.scalatestplus.selenium.WebBrowser
+
+import org.apache.spark._
+import org.apache.spark.LocalSparkContext.withSpark
+import org.apache.spark.internal.config.MEMORY_OFFHEAP_SIZE
+import org.apache.spark.internal.config.UI.{UI_ENABLED, UI_KILL_ENABLED, UI_PORT}
+import org.apache.spark.util.CallSite
+
+/**
+ * Selenium tests for the Spark Web UI with real web browsers.
+ */
+abstract class RealBrowserUISeleniumSuite(val driverProp: String)
+  extends SparkFunSuite with WebBrowser with Matchers with BeforeAndAfterAll {
+
+  implicit var webDriver: WebDriver with JavascriptExecutor
+  private val driverPropPrefix = "spark.test."
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    assume(
+      sys.props(driverPropPrefix + driverProp) !== null,
+      "System property " + driverPropPrefix + driverProp +
+        " should be set to the corresponding driver path.")
+    sys.props(driverProp) = sys.props(driverPropPrefix + driverProp)
+  }
+
+  override def afterAll(): Unit = {
+    sys.props.remove(driverProp)
+    super.afterAll()
+  }
+
+  test("SPARK-31534: text for tooltip should be escaped") {
+    withSpark(newSparkContext()) { sc =>
+      sc.setLocalProperty(CallSite.LONG_FORM, "collect at <console>:25")
+      sc.setLocalProperty(CallSite.SHORT_FORM, "collect at <console>:25")
+      sc.parallelize(1 to 10).collect
+
+      eventually(timeout(10.seconds), interval(50.milliseconds)) {
+        goToUi(sc, "/jobs")
+
+        val jobDesc =
+          webDriver.findElement(By.cssSelector("div[class='application-timeline-content']"))
+        jobDesc.getAttribute("data-title") should include  ("collect at &lt;console&gt;:25")
+
+        goToUi(sc, "/jobs/job/?id=0")
+        webDriver.get(sc.ui.get.webUrl.stripSuffix("/") + "/jobs/job/?id=0")
+        val stageDesc = webDriver.findElement(By.cssSelector("div[class='job-timeline-content']"))
+        stageDesc.getAttribute("data-title") should include ("collect at &lt;console&gt;:25")
+
+        // Open DAG Viz.
+        webDriver.findElement(By.id("job-dag-viz")).click()
+        val nodeDesc = webDriver.findElement(By.cssSelector("g[class='node_0 node']"))
+        nodeDesc.getAttribute("name") should include ("collect at &lt;console&gt;:25")
+      }
+    }
+  }
+
+  test("SPARK-31882: Link URL for Stage DAGs should not depend on paged table.") {
+    withSpark(newSparkContext()) { sc =>
+      sc.parallelize(1 to 100).map(v => (v, v)).repartition(10).reduceByKey(_ + _).collect
+
+      eventually(timeout(10.seconds), interval(50.microseconds)) {
+        val pathWithPagedTable =
+          "/jobs/job/?id=0&completedStage.page=2&completedStage.sort=Stage+Id&" +
+            "completedStage.desc=true&completedStage.pageSize=1#completed"
+        goToUi(sc, pathWithPagedTable)
+
+        // Open DAG Viz.
+        webDriver.findElement(By.id("job-dag-viz")).click()
+        val stages = webDriver.findElements(By.cssSelector("svg[class='job'] > a"))
+        stages.size() should be (3)
+
+        stages.get(0).getAttribute("href") should include ("/stages/stage/?id=0&attempt=0")
+        stages.get(1).getAttribute("href") should include ("/stages/stage/?id=1&attempt=0")
+        stages.get(2).getAttribute("href") should include ("/stages/stage/?id=2&attempt=0")
+      }
+    }
+  }
+
+  test("SPARK-31886: Color barrier execution mode RDD correctly") {
+    withSpark(newSparkContext()) { sc =>
+      sc.parallelize(1 to 10).barrier.mapPartitions(identity).repartition(1).collect()
+
+      eventually(timeout(10.seconds), interval(50.milliseconds)) {
+        goToUi(sc, "/jobs/job/?id=0")
+        webDriver.findElement(By.id("job-dag-viz")).click()
+
+        val stage0 = webDriver.findElement(By.cssSelector("g[id='graph_0']"))
+        val stage1 = webDriver.findElement(By.cssSelector("g[id='graph_1']"))
+        val barrieredOps = webDriver.findElements(By.className("barrier-rdd")).iterator()
+
+        while (barrieredOps.hasNext) {
+          val barrieredOpId = barrieredOps.next().getAttribute("innerHTML")
+          val foundInStage0 =
+            stage0.findElements(
+              By.cssSelector("g.barrier.cluster.cluster_" + barrieredOpId))
+          assert(foundInStage0.size === 1)
+
+          val foundInStage1 =
+            stage1.findElements(
+              By.cssSelector("g.barrier.cluster.cluster_" + barrieredOpId))
+          assert(foundInStage1.size === 0)
+        }
+      }
+    }
+  }
+
+  test("Search text for paged tables should not be saved") {
+    withSpark(newSparkContext()) { sc =>
+      sc.parallelize(1 to 10).collect
+
+      eventually(timeout(10.seconds), interval(1.seconds)) {
+        val taskSearchBox = "$(\"input[aria-controls='active-tasks-table']\")"
+        goToUi(sc, "/stages/stage/?id=0&attempt=0")
+        // Wait for ajax loading done.
+        Thread.sleep(20)
+        setValueToSearchBox(taskSearchBox, "task1")
+        val taskSearchText = getTextFromSearchBox(taskSearchBox)
+        assert(taskSearchText === "task1")
+
+        val executorSearchBox = "$(\"input[aria-controls='active-executors-table']\")"
+        goToUi(sc, "/executors")
+        Thread.sleep(20)
+        setValueToSearchBox(executorSearchBox, "executor1")
+        val executorSearchText = getTextFromSearchBox(executorSearchBox)
+        assert(executorSearchText === "executor1")
+
+        goToUi(sc, "/stages/stage/?id=0&attempt=0")
+        Thread.sleep(20)
+        val revisitTaskSearchText = getTextFromSearchBox(taskSearchBox)
+        assert(revisitTaskSearchText === "")
+
+        goToUi(sc, "/executors")
+        Thread.sleep(20)
+        val revisitExecutorSearchText = getTextFromSearchBox(executorSearchBox)
+        assert(revisitExecutorSearchText === "")
+      }
+    }
+
+    def setValueToSearchBox(searchBox: String, text: String): Unit = {
+      webDriver.executeScript(s"$searchBox.val('$text');")
+      fireDataTable(searchBox)
+    }
+
+    def getTextFromSearchBox(searchBox: String): String = {
+      webDriver.executeScript(s"return $searchBox.val();").toString
+    }
+
+    def fireDataTable(searchBox: String): Unit = {
+      webDriver.executeScript(
+        s"""
+           |var keyEvent = $$.Event('keyup');
+           |// 13 means enter key.
+           |keyEvent.keyCode = keyEvent.which = 13;
+           |$searchBox.trigger(keyEvent);
+         """.stripMargin)
+    }
+  }
+
+  /**
+   * Create a test SparkContext with the SparkUI enabled.
+   * It is safe to `get` the SparkUI directly from the SparkContext returned here.
+   */
+  private def newSparkContext(
+      killEnabled: Boolean = true,
+      master: String = "local",
+      additionalConfs: Map[String, String] = Map.empty): SparkContext = {
+    val conf = new SparkConf()
+      .setMaster(master)
+      .setAppName("test")
+      .set(UI_ENABLED, true)
+      .set(UI_PORT, 0)
+      .set(UI_KILL_ENABLED, killEnabled)
+      .set(MEMORY_OFFHEAP_SIZE.key, "64m")
+    additionalConfs.foreach { case (k, v) => conf.set(k, v) }
+    val sc = new SparkContext(conf)
+    assert(sc.ui.isDefined)
+    sc
+  }
+
+  def goToUi(sc: SparkContext, path: String): Unit = {
+    goToUi(sc.ui.get, path)
+  }
+
+  def goToUi(ui: SparkUI, path: String): Unit = {
+    go to (ui.webUrl.stripSuffix("/") + path)
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala b/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala
index 7711934cbe8a6..9f0b73f8132f8 100644
--- a/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala
@@ -17,7 +17,6 @@
 
 package org.apache.spark.ui
 
-import java.util.Locale
 import javax.servlet.http.HttpServletRequest
 
 import scala.xml.Node
@@ -92,12 +91,15 @@ class StagePageSuite extends SparkFunSuite with LocalSparkContext {
         accumulatorUpdates = Seq(new UIAccumulableInfo(0L, "acc", None, "value")),
         tasks = None,
         executorSummary = None,
-        killedTasksSummary = Map.empty
+        killedTasksSummary = Map.empty,
+        ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID,
+        peakExecutorMetrics = None,
+        taskMetricsDistributions = None,
+        executorMetricsDistributions = None
       )
       val taskTable = new TaskPagedTable(
         stageData,
         basePath = "/a/b/c",
-        currentTime = 0,
         pageSize = 10,
         sortColumn = "Index",
         desc = false,
diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
index 3ec9385116408..015f299fc6bdf 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
@@ -24,16 +24,18 @@ import javax.servlet.http.{HttpServletRequest, HttpServletResponse}
 import scala.io.Source
 import scala.xml.Node
 
+import com.gargoylesoftware.css.parser.CSSParseException
 import com.gargoylesoftware.htmlunit.DefaultCssErrorHandler
 import org.json4s._
 import org.json4s.jackson.JsonMethods
 import org.openqa.selenium.{By, WebDriver}
 import org.openqa.selenium.htmlunit.HtmlUnitDriver
-import org.scalatest._
+import org.scalatest.BeforeAndAfterAll
 import org.scalatest.concurrent.Eventually._
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 import org.scalatest.time.SpanSugar._
 import org.scalatestplus.selenium.WebBrowser
-import org.w3c.css.sac.CSSParseException
 
 import org.apache.spark._
 import org.apache.spark.LocalSparkContext._
@@ -44,28 +46,32 @@ import org.apache.spark.internal.config.Status._
 import org.apache.spark.internal.config.UI._
 import org.apache.spark.shuffle.FetchFailedException
 import org.apache.spark.status.api.v1.{JacksonMessageWriter, RDDDataDistribution, StageStatus}
-import org.apache.spark.util.CallSite
+import org.apache.spark.util.Utils
 
 private[spark] class SparkUICssErrorHandler extends DefaultCssErrorHandler {
 
-  private val cssWhiteList = List("bootstrap.min.css", "vis-timeline-graph2d.min.css")
+  /**
+   * Some libraries have warn/error messages that are too noisy for the tests; exclude them from
+   * normal error handling to avoid logging these.
+   */
+  private val cssExcludeList = List("bootstrap.min.css", "vis-timeline-graph2d.min.css")
 
-  private def isInWhileList(uri: String): Boolean = cssWhiteList.exists(uri.endsWith)
+  private def isInExcludeList(uri: String): Boolean = cssExcludeList.exists(uri.endsWith)
 
   override def warning(e: CSSParseException): Unit = {
-    if (!isInWhileList(e.getURI)) {
+    if (!isInExcludeList(e.getURI)) {
       super.warning(e)
     }
   }
 
   override def fatalError(e: CSSParseException): Unit = {
-    if (!isInWhileList(e.getURI)) {
+    if (!isInExcludeList(e.getURI)) {
       super.fatalError(e)
     }
   }
 
   override def error(e: CSSParseException): Unit = {
-    if (!isInWhileList(e.getURI)) {
+    if (!isInExcludeList(e.getURI)) {
       super.error(e)
     }
   }
@@ -118,6 +124,27 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
     sc
   }
 
+  test("all jobs page should be rendered even though we configure the scheduling mode to fair") {
+    // Regression test for SPARK-33991
+    val conf = Map("spark.scheduler.mode" -> "fair")
+    withSpark(newSparkContext(additionalConfs = conf)) { sc =>
+      val rdd = sc.parallelize(0 to 100, 100).repartition(10).cache()
+      rdd.count()
+
+      eventually(timeout(5.seconds), interval(50.milliseconds)) {
+        goToUi(sc, "/jobs")
+        // The completed jobs table should have one row. The first row will be the most recent job:
+        val firstRow = find(cssSelector("tbody tr")).get.underlying
+        val firstRowColumns = firstRow.findElements(By.tagName("td"))
+        // if first row can get the id 0, then the page is rendered and the scheduling mode is
+        // displayed with no error when we visit http://localhost:4040/jobs/ even though
+        // we configure the scheduling mode like spark.scheduler.mode=fair
+        // instead of spark.scheculer.mode=FAIR
+        firstRowColumns.get(0).getText should be ("0")
+      }
+    }
+  }
+
   test("effects of unpersist() / persist() should be reflected") {
     // Regression test for SPARK-2527
     withSpark(newSparkContext()) { sc =>
@@ -683,8 +710,8 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
       rdd.count()
 
       eventually(timeout(5.seconds), interval(100.milliseconds)) {
-        val stage0 = Source.fromURL(sc.ui.get.webUrl +
-          "/stages/stage/?id=0&attempt=0&expandDagViz=true").mkString
+        val stage0 = Utils.tryWithResource(Source.fromURL(sc.ui.get.webUrl +
+          "/stages/stage/?id=0&attempt=0&expandDagViz=true"))(_.mkString)
         assert(stage0.contains("digraph G {\n  subgraph clusterstage_0 {\n    " +
           "label=&quot;Stage 0&quot;;\n    subgraph "))
         assert(stage0.contains("{\n      label=&quot;parallelize&quot;;\n      " +
@@ -694,8 +721,8 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
         assert(stage0.contains("{\n      label=&quot;groupBy&quot;;\n      " +
           "2 [labelType=&quot;html&quot; label=&quot;MapPartitionsRDD [2]"))
 
-        val stage1 = Source.fromURL(sc.ui.get.webUrl +
-          "/stages/stage/?id=1&attempt=0&expandDagViz=true").mkString
+        val stage1 = Utils.tryWithResource(Source.fromURL(sc.ui.get.webUrl +
+          "/stages/stage/?id=1&attempt=0&expandDagViz=true"))(_.mkString)
         assert(stage1.contains("digraph G {\n  subgraph clusterstage_1 {\n    " +
           "label=&quot;Stage 1&quot;;\n    subgraph "))
         assert(stage1.contains("{\n      label=&quot;groupBy&quot;;\n      " +
@@ -705,8 +732,8 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
         assert(stage1.contains("{\n      label=&quot;groupBy&quot;;\n      " +
           "5 [labelType=&quot;html&quot; label=&quot;MapPartitionsRDD [5]"))
 
-        val stage2 = Source.fromURL(sc.ui.get.webUrl +
-          "/stages/stage/?id=2&attempt=0&expandDagViz=true").mkString
+        val stage2 = Utils.tryWithResource(Source.fromURL(sc.ui.get.webUrl +
+          "/stages/stage/?id=2&attempt=0&expandDagViz=true"))(_.mkString)
         assert(stage2.contains("digraph G {\n  subgraph clusterstage_2 {\n    " +
           "label=&quot;Stage 2&quot;;\n    subgraph "))
         assert(stage2.contains("{\n      label=&quot;groupBy&quot;;\n      " +
@@ -773,33 +800,6 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
     }
   }
 
-  test("SPARK-31534: text for tooltip should be escaped") {
-    withSpark(newSparkContext()) { sc =>
-      sc.setLocalProperty(CallSite.LONG_FORM, "collect at <console>:25")
-      sc.setLocalProperty(CallSite.SHORT_FORM, "collect at <console>:25")
-      sc.parallelize(1 to 10).collect
-
-      val driver = webDriver.asInstanceOf[HtmlUnitDriver]
-      driver.setJavascriptEnabled(true)
-
-      eventually(timeout(10.seconds), interval(50.milliseconds)) {
-        goToUi(sc, "/jobs")
-        val jobDesc =
-          driver.findElement(By.cssSelector("div[class='application-timeline-content']"))
-        jobDesc.getAttribute("data-title") should include  ("collect at &lt;console&gt;:25")
-
-        goToUi(sc, "/jobs/job/?id=0")
-        val stageDesc = driver.findElement(By.cssSelector("div[class='job-timeline-content']"))
-        stageDesc.getAttribute("data-title") should include ("collect at &lt;console&gt;:25")
-
-        // Open DAG Viz.
-        driver.findElement(By.id("job-dag-viz")).click()
-        val nodeDesc = driver.findElement(By.cssSelector("g[class='node_0 node']"))
-        nodeDesc.getAttribute("name") should include ("collect at &lt;console&gt;:25")
-      }
-    }
-  }
-
   def goToUi(sc: SparkContext, path: String): Unit = {
     goToUi(sc.ui.get, path)
   }
diff --git a/core/src/test/scala/org/apache/spark/ui/UISuite.scala b/core/src/test/scala/org/apache/spark/ui/UISuite.scala
index 2ad4a634cd9a7..45a2ce03eef8a 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISuite.scala
@@ -76,7 +76,7 @@ class UISuite extends SparkFunSuite {
     withSpark(newSparkContext()) { sc =>
       // test if the ui is visible, and all the expected tabs are visible
       eventually(timeout(10.seconds), interval(50.milliseconds)) {
-        val html = Source.fromURL(sc.ui.get.webUrl).mkString
+        val html = Utils.tryWithResource(Source.fromURL(sc.ui.get.webUrl))(_.mkString)
         assert(!html.contains("random data that should not be present"))
         assert(html.toLowerCase(Locale.ROOT).contains("stages"))
         assert(html.toLowerCase(Locale.ROOT).contains("storage"))
@@ -90,7 +90,7 @@ class UISuite extends SparkFunSuite {
     withSpark(newSparkContext()) { sc =>
       // test if visible from http://localhost:4040
       eventually(timeout(10.seconds), interval(50.milliseconds)) {
-        val html = Source.fromURL("http://localhost:4040").mkString
+        val html = Utils.tryWithResource(Source.fromURL("http://localhost:4040"))(_.mkString)
         assert(html.toLowerCase(Locale.ROOT).contains("stages"))
       }
     }
@@ -216,6 +216,15 @@ class UISuite extends SparkFunSuite {
     assert(rewrittenURI === null)
   }
 
+  test("SPARK-33611: Avoid encoding twice on the query parameter of proxy rewrittenURI") {
+    val prefix = "/worker-id"
+    val target = "http://localhost:8081"
+    val path = "/worker-id/json"
+    val rewrittenURI =
+      JettyUtils.createProxyURI(prefix, target, path, "order%5B0%5D%5Bcolumn%5D=0")
+    assert(rewrittenURI.toString === "http://localhost:8081/json?order%5B0%5D%5Bcolumn%5D=0")
+  }
+
   test("verify rewriting location header for reverse proxy") {
     val clientRequest = mock(classOf[HttpServletRequest])
     var headerValue = "http://localhost:4040/jobs"
@@ -262,6 +271,27 @@ class UISuite extends SparkFunSuite {
     }
   }
 
+  test("SPARK-32467: Avoid encoding URL twice on https redirect") {
+    val (conf, securityMgr, sslOptions) = sslEnabledConf()
+    val serverInfo = JettyUtils.startJettyServer("0.0.0.0", 0, sslOptions, conf)
+    try {
+      val serverAddr = s"http://localhost:${serverInfo.boundPort}"
+
+      val (_, ctx) = newContext("/ctx1")
+      serverInfo.addHandler(ctx, securityMgr)
+
+      TestUtils.withHttpConnection(new URL(s"$serverAddr/ctx%281%29?a%5B0%5D=b")) { conn =>
+        assert(conn.getResponseCode() === HttpServletResponse.SC_FOUND)
+        val location = Option(conn.getHeaderFields().get("Location"))
+          .map(_.get(0)).orNull
+        val expectedLocation = s"https://localhost:${serverInfo.securePort.get}/ctx(1)?a[0]=b"
+        assert(location == expectedLocation)
+      }
+    } finally {
+      stopServer(serverInfo)
+    }
+  }
+
   test("http -> https redirect applies to all URIs") {
     val (conf, securityMgr, sslOptions) = sslEnabledConf()
     val serverInfo = JettyUtils.startJettyServer("0.0.0.0", 0, sslOptions, conf)
@@ -326,20 +356,9 @@ class UISuite extends SparkFunSuite {
     try {
       val serverAddr = s"http://localhost:${serverInfo.boundPort}"
 
-      val (_, ctx) = newContext("/ctx1")
-      serverInfo.addHandler(ctx, securityMgr)
-
       val redirect = JettyUtils.createRedirectHandler("/src", "/dst")
       serverInfo.addHandler(redirect, securityMgr)
 
-      // Test Jetty's built-in redirect to add the trailing slash to the context path.
-      TestUtils.withHttpConnection(new URL(s"$serverAddr/ctx1")) { conn =>
-        assert(conn.getResponseCode() === HttpServletResponse.SC_FOUND)
-        val location = Option(conn.getHeaderFields().get("Location"))
-          .map(_.get(0)).orNull
-        assert(location === s"$proxyRoot/ctx1/")
-      }
-
       // Test with a URL handled by the added redirect handler, and also including a path prefix.
       val headers = Seq("X-Forwarded-Context" -> "/prefix")
       TestUtils.withHttpConnection(
@@ -365,6 +384,28 @@ class UISuite extends SparkFunSuite {
     }
   }
 
+  test("SPARK-34449: Jetty 9.4.35.v20201120 and later no longer return status code 302 " +
+       " and handle internally when request URL ends with a context path without trailing '/'") {
+    val proxyRoot = "https://proxy.example.com:443/prefix"
+    val (conf, securityMgr, sslOptions) = sslDisabledConf()
+    conf.set(UI.PROXY_REDIRECT_URI, proxyRoot)
+    val serverInfo = JettyUtils.startJettyServer("0.0.0.0", 0, sslOptions, conf)
+
+    try {
+      val (_, ctx) = newContext("/ctx")
+      serverInfo.addHandler(ctx, securityMgr)
+      val urlStr = s"http://localhost:${serverInfo.boundPort}/ctx"
+
+      assert(TestUtils.httpResponseCode(new URL(urlStr + "/")) === HttpServletResponse.SC_OK)
+
+      // If the following assertion fails when we upgrade Jetty, it seems to change the behavior of
+      // handling context path which doesn't have the trailing slash.
+      assert(TestUtils.httpResponseCode(new URL(urlStr)) === HttpServletResponse.SC_OK)
+    } finally {
+      stopServer(serverInfo)
+    }
+  }
+
   /**
    * Create a new context handler for the given path, with a single servlet that responds to
    * requests in `$path/root`.
@@ -373,7 +414,9 @@ class UISuite extends SparkFunSuite {
     val servlet = new CapturingServlet()
     val ctx = new ServletContextHandler()
     ctx.setContextPath(path)
-    ctx.addServlet(new ServletHolder(servlet), "/root")
+    val servletHolder = new ServletHolder(servlet)
+    ctx.addServlet(servletHolder, "/root")
+    ctx.addServlet(servletHolder, "/")
     (servlet, ctx)
   }
 
diff --git a/core/src/test/scala/org/apache/spark/ui/UIUtilsSuite.scala b/core/src/test/scala/org/apache/spark/ui/UIUtilsSuite.scala
index d4368c882d660..9d040bb4e1ec7 100644
--- a/core/src/test/scala/org/apache/spark/ui/UIUtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UIUtilsSuite.scala
@@ -113,7 +113,8 @@ class UIUtilsSuite extends SparkFunSuite {
   test("SPARK-11906: Progress bar should not overflow because of speculative tasks") {
     val generated = makeProgressBar(2, 3, 0, 0, Map.empty, 4).head.child.filter(_.label == "div")
     val expected = Seq(
-      <div class="progress-bar" style="width: 75.0%"></div>
+      <div class="progress-bar progress-completed" style="width: 75.0%"></div>,
+      <div class="progress-bar progress-started" style="width: 25.0%"></div>
     )
     assert(generated.sameElements(expected),
       s"\nRunning progress bar should round down\n\nExpected:\n$expected\nGenerated:\n$generated")
diff --git a/core/src/test/scala/org/apache/spark/ui/scope/RDDOperationGraphSuite.scala b/core/src/test/scala/org/apache/spark/ui/scope/RDDOperationGraphSuite.scala
index e335451f1e484..ba83bfe9554f7 100644
--- a/core/src/test/scala/org/apache/spark/ui/scope/RDDOperationGraphSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/scope/RDDOperationGraphSuite.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.ui.scope
 
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.rdd.DeterministicLevel
 
 class RDDOperationGraphSuite extends SparkFunSuite {
   test("Test simple cluster equals") {
@@ -25,7 +26,8 @@ class RDDOperationGraphSuite extends SparkFunSuite {
     val c1 = new RDDOperationCluster("1", false, "Bender")
     val c2 = new RDDOperationCluster("2", false, "Hal")
     c1.attachChildCluster(c2)
-    c1.attachChildNode(new RDDOperationNode(3, "Marvin", false, false, "collect!"))
+    c1.attachChildNode(new RDDOperationNode(3, "Marvin", false, false, "collect!",
+      DeterministicLevel.DETERMINATE))
 
     // create an equal cluster, but without the child node
     val c1copy = new RDDOperationCluster("1", false, "Bender")
diff --git a/core/src/test/scala/org/apache/spark/util/ClosureCleanerSuite.scala b/core/src/test/scala/org/apache/spark/util/ClosureCleanerSuite.scala
index b0520c7ab1b1f..cef0d8c1de02b 100644
--- a/core/src/test/scala/org/apache/spark/util/ClosureCleanerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/ClosureCleanerSuite.scala
@@ -147,7 +147,7 @@ object TestObject {
 }
 
 class TestClass extends Serializable {
-  var x = 5
+  val x = 5
 
   def getX: Int = x
 
@@ -179,7 +179,7 @@ class TestClassWithoutFieldAccess {
 
   def run(): Int = {
     var nonSer2 = new NonSerializable
-    var x = 5
+    val x = 5
     withSpark(new SparkContext("local", "test")) { sc =>
       val nums = sc.parallelize(Array(1, 2, 3, 4))
       nums.map(_ + x).reduce(_ + _)
@@ -218,10 +218,10 @@ object TestObjectWithNesting {
     var answer = 0
     withSpark(new SparkContext("local", "test")) { sc =>
       val nums = sc.parallelize(Array(1, 2, 3, 4))
-      var y = 1
+      val y = 1
       for (i <- 1 to 4) {
         var nonSer2 = new NonSerializable
-        var x = i
+        val x = i
         answer += nums.map(_ + x + y).reduce(_ + _)
       }
       answer
@@ -239,7 +239,7 @@ class TestClassWithNesting(val y: Int) extends Serializable {
       val nums = sc.parallelize(Array(1, 2, 3, 4))
       for (i <- 1 to 4) {
         var nonSer2 = new NonSerializable
-        var x = i
+        val x = i
         answer += nums.map(_ + x + getY).reduce(_ + _)
       }
       answer
@@ -339,7 +339,7 @@ private object TestUserClosuresActuallyCleaned {
 
 class TestCreateNullValue {
 
-  var x = 5
+  val x = 5
 
   def getX: Int = x
 
diff --git a/core/src/test/scala/org/apache/spark/util/DependencyUtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/DependencyUtilsSuite.scala
new file mode 100644
index 0000000000000..bf8edeff37c25
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/util/DependencyUtilsSuite.scala
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util
+
+import java.net.URI
+
+import org.apache.spark.SparkFunSuite
+
+class DependencyUtilsSuite extends SparkFunSuite {
+
+  test("SPARK-33084: Add jar support Ivy URI -- test invalid ivy uri") {
+    val e1 = intercept[IllegalArgumentException] {
+      DependencyUtils.resolveMavenDependencies(URI.create("ivy://"))
+    }.getMessage
+    assert(e1.contains("Expected authority at index 6: ivy://"))
+
+    val e2 = intercept[IllegalArgumentException] {
+      DependencyUtils.resolveMavenDependencies(URI.create("ivy://org.apache.test:test-test"))
+    }.getMessage
+    assert(e2.contains("Invalid Ivy URI authority in uri ivy://org.apache.test:test-test: " +
+      "Expected 'org:module:version', found org.apache.test:test-test."))
+
+    val e3 = intercept[IllegalArgumentException] {
+      DependencyUtils.resolveMavenDependencies(
+        URI.create("ivy://org.apache.test:test-test:1.0.0?foo="))
+    }.getMessage
+    assert(e3.contains("Invalid query string in Ivy URI " +
+      "ivy://org.apache.test:test-test:1.0.0?foo=:"))
+
+    val e4 = intercept[IllegalArgumentException] {
+      DependencyUtils.resolveMavenDependencies(
+        URI.create("ivy://org.apache.test:test-test:1.0.0?bar=&baz=foo"))
+    }.getMessage
+    assert(e4.contains("Invalid query string in Ivy URI " +
+      "ivy://org.apache.test:test-test:1.0.0?bar=&baz=foo: bar=&baz=foo"))
+
+    val e5 = intercept[IllegalArgumentException] {
+      DependencyUtils.resolveMavenDependencies(
+        URI.create("ivy://org.apache.test:test-test:1.0.0?exclude=org.apache"))
+    }.getMessage
+    assert(e5.contains("Invalid exclude string in Ivy URI " +
+      "ivy://org.apache.test:test-test:1.0.0?exclude=org.apache: " +
+      "expected 'org:module,org:module,..', found org.apache"))
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/util/DistributionSuite.scala b/core/src/test/scala/org/apache/spark/util/DistributionSuite.scala
index d3a95e399c289..0ffa9842e16f5 100644
--- a/core/src/test/scala/org/apache/spark/util/DistributionSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/DistributionSuite.scala
@@ -17,7 +17,8 @@
 
 package org.apache.spark.util
 
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.SparkFunSuite
 
diff --git a/core/src/test/scala/org/apache/spark/util/FileAppenderSuite.scala b/core/src/test/scala/org/apache/spark/util/FileAppenderSuite.scala
index 21e69550785a4..12d97573ff6ee 100644
--- a/core/src/test/scala/org/apache/spark/util/FileAppenderSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/FileAppenderSuite.scala
@@ -337,7 +337,7 @@ class FileAppenderSuite extends SparkFunSuite with BeforeAndAfter with Logging {
     assert(generatedFiles.size > 1)
     if (isCompressed) {
       assert(
-        generatedFiles.filter(_.getName.endsWith(RollingFileAppender.GZIP_LOG_SUFFIX)).size > 0)
+        generatedFiles.exists(_.getName.endsWith(RollingFileAppender.GZIP_LOG_SUFFIX)))
     }
     val allText = generatedFiles.map { file =>
       if (file.getName.endsWith(RollingFileAppender.GZIP_LOG_SUFFIX)) {
diff --git a/core/src/test/scala/org/apache/spark/util/HadoopFSUtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/HadoopFSUtilsSuite.scala
new file mode 100644
index 0000000000000..ba91eabc1cab1
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/util/HadoopFSUtilsSuite.scala
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util
+
+import org.apache.spark.SparkFunSuite
+
+class HadoopFSUtilsSuite extends SparkFunSuite {
+  test("HadoopFSUtils - file filtering") {
+    assert(!HadoopFSUtils.shouldFilterOutPathName("abcd"))
+    assert(HadoopFSUtils.shouldFilterOutPathName(".ab"))
+    assert(HadoopFSUtils.shouldFilterOutPathName("_cd"))
+    assert(!HadoopFSUtils.shouldFilterOutPathName("_metadata"))
+    assert(!HadoopFSUtils.shouldFilterOutPathName("_common_metadata"))
+    assert(HadoopFSUtils.shouldFilterOutPathName("_ab_metadata"))
+    assert(HadoopFSUtils.shouldFilterOutPathName("_cd_common_metadata"))
+    assert(HadoopFSUtils.shouldFilterOutPathName("a._COPYING_"))
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
index eb7f3079bee36..8dbfe53db6271 100644
--- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
@@ -31,8 +31,8 @@ import org.scalatest.exceptions.TestFailedException
 import org.apache.spark._
 import org.apache.spark.executor._
 import org.apache.spark.metrics.ExecutorMetricType
-import org.apache.spark.rdd.RDDOperationScope
-import org.apache.spark.resource.{ResourceInformation, ResourceProfile, ResourceUtils}
+import org.apache.spark.rdd.{DeterministicLevel, RDDOperationScope}
+import org.apache.spark.resource._
 import org.apache.spark.scheduler._
 import org.apache.spark.scheduler.cluster.ExecutorInfo
 import org.apache.spark.shuffle.MetadataFetchFailedException
@@ -92,14 +92,20 @@ class JsonProtocolSuite extends SparkFunSuite {
       42L, "Garfield", Some("appAttempt"), Some(logUrlMap))
     val applicationEnd = SparkListenerApplicationEnd(42L)
     val executorAdded = SparkListenerExecutorAdded(executorAddedTime, "exec1",
-      new ExecutorInfo("Hostee.awesome.com", 11, logUrlMap, attributes, resources.toMap))
+      new ExecutorInfo("Hostee.awesome.com", 11, logUrlMap, attributes, resources.toMap, 4))
     val executorRemoved = SparkListenerExecutorRemoved(executorRemovedTime, "exec2", "test reason")
-    val executorBlacklisted = SparkListenerExecutorBlacklisted(executorBlacklistedTime, "exec1", 22)
+    val executorBlacklisted = SparkListenerExecutorBlacklisted(executorExcludedTime, "exec1", 22)
     val executorUnblacklisted =
-      SparkListenerExecutorUnblacklisted(executorUnblacklistedTime, "exec1")
-    val nodeBlacklisted = SparkListenerNodeBlacklisted(nodeBlacklistedTime, "node1", 33)
+      SparkListenerExecutorUnblacklisted(executorUnexcludedTime, "exec1")
+    val nodeBlacklisted = SparkListenerNodeBlacklisted(nodeExcludedTime, "node1", 33)
+    val executorExcluded = SparkListenerExecutorExcluded(executorExcludedTime, "exec1", 22)
+    val executorUnexcluded =
+      SparkListenerExecutorUnexcluded(executorUnexcludedTime, "exec1")
+    val nodeExcluded = SparkListenerNodeExcluded(nodeExcludedTime, "node1", 33)
     val nodeUnblacklisted =
-      SparkListenerNodeUnblacklisted(nodeUnblacklistedTime, "node1")
+      SparkListenerNodeUnblacklisted(nodeUnexcludedTime, "node1")
+    val nodeUnexcluded =
+      SparkListenerNodeUnexcluded(nodeUnexcludedTime, "node1")
     val executorMetricsUpdate = {
       // Use custom accum ID for determinism
       val accumUpdates =
@@ -119,6 +125,14 @@ class JsonProtocolSuite extends SparkFunSuite {
       SparkListenerStageExecutorMetrics("1", 2, 3,
         new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L,
           321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L, 10L, 90L, 2L, 20L)))
+    val rprofBuilder = new ResourceProfileBuilder()
+    val taskReq = new TaskResourceRequests().cpus(1).resource("gpu", 1)
+    val execReq =
+      new ExecutorResourceRequests().cores(2).resource("gpu", 2, "myscript")
+    rprofBuilder.require(taskReq).require(execReq)
+    val resourceProfile = rprofBuilder.build
+    resourceProfile.setResourceProfileId(21)
+    val resourceProfileAdded = SparkListenerResourceProfileAdded(resourceProfile)
     testEvent(stageSubmitted, stageSubmittedJsonString)
     testEvent(stageCompleted, stageCompletedJsonString)
     testEvent(taskStart, taskStartJsonString)
@@ -139,17 +153,22 @@ class JsonProtocolSuite extends SparkFunSuite {
     testEvent(executorRemoved, executorRemovedJsonString)
     testEvent(executorBlacklisted, executorBlacklistedJsonString)
     testEvent(executorUnblacklisted, executorUnblacklistedJsonString)
+    testEvent(executorExcluded, executorExcludedJsonString)
+    testEvent(executorUnexcluded, executorUnexcludedJsonString)
     testEvent(nodeBlacklisted, nodeBlacklistedJsonString)
     testEvent(nodeUnblacklisted, nodeUnblacklistedJsonString)
+    testEvent(nodeExcluded, nodeExcludedJsonString)
+    testEvent(nodeUnexcluded, nodeUnexcludedJsonString)
     testEvent(executorMetricsUpdate, executorMetricsUpdateJsonString)
     testEvent(blockUpdated, blockUpdatedJsonString)
     testEvent(stageExecutorMetrics, stageExecutorMetricsJsonString)
+    testEvent(resourceProfileAdded, resourceProfileJsonString)
   }
 
   test("Dependent Classes") {
     val logUrlMap = Map("stderr" -> "mystderr", "stdout" -> "mystdout").toMap
     val attributes = Map("ContainerId" -> "ct1", "User" -> "spark").toMap
-    testRDDInfo(makeRddInfo(2, 3, 4, 5L, 6L))
+    testRDDInfo(makeRddInfo(2, 3, 4, 5L, 6L, DeterministicLevel.DETERMINATE))
     testStageInfo(makeStageInfo(10, 20, 30, 40L, 50L))
     testTaskInfo(makeTaskInfo(999L, 888, 55, 777L, false))
     testTaskMetrics(makeTaskMetrics(
@@ -161,6 +180,7 @@ class JsonProtocolSuite extends SparkFunSuite {
     testStorageLevel(StorageLevel.NONE)
     testStorageLevel(StorageLevel.DISK_ONLY)
     testStorageLevel(StorageLevel.DISK_ONLY_2)
+    testStorageLevel(StorageLevel.DISK_ONLY_3)
     testStorageLevel(StorageLevel.MEMORY_ONLY)
     testStorageLevel(StorageLevel.MEMORY_ONLY_2)
     testStorageLevel(StorageLevel.MEMORY_ONLY_SER)
@@ -231,6 +251,20 @@ class JsonProtocolSuite extends SparkFunSuite {
     assert(0 === newInfo.accumulables.size)
   }
 
+  test("StageInfo resourceProfileId") {
+    val info = makeStageInfo(1, 2, 3, 4L, 5L, 5)
+    val json = JsonProtocol.stageInfoToJson(info)
+
+    // Fields added after 1.0.0.
+    assert(info.details.nonEmpty)
+    assert(info.resourceProfileId === 5)
+
+    val newInfo = JsonProtocol.stageInfoFromJson(json)
+
+    assert(info.name === newInfo.name)
+    assert(5 === newInfo.resourceProfileId)
+  }
+
   test("InputMetrics backward compatibility") {
     // InputMetrics were added after 1.0.1.
     val metrics = makeTaskMetrics(1L, 2L, 3L, 4L, 5, 6, hasHadoopInput = true, hasOutput = false)
@@ -304,6 +338,17 @@ class JsonProtocolSuite extends SparkFunSuite {
     assert(expectedFetchFailed === JsonProtocol.taskEndReasonFromJson(oldEvent))
   }
 
+  test("SPARK-32124: FetchFailed Map Index backwards compatibility") {
+    // FetchFailed in Spark 2.4.0 does not have "Map Index" property.
+    val fetchFailed = FetchFailed(BlockManagerId("With or", "without you", 15), 17, 16L, 18, 19,
+      "ignored")
+    val oldEvent = JsonProtocol.taskEndReasonToJson(fetchFailed)
+      .removeField({ _._1 == "Map Index" })
+    val expectedFetchFailed = FetchFailed(BlockManagerId("With or", "without you", 15), 17, 16L,
+      Int.MinValue, 19, "ignored")
+    assert(expectedFetchFailed === JsonProtocol.taskEndReasonFromJson(oldEvent))
+  }
+
   test("ShuffleReadMetrics: Local bytes read backwards compatibility") {
     // Metrics about local shuffle bytes read were added in 1.3.1.
     val metrics = makeTaskMetrics(1L, 2L, 3L, 4L, 5, 6,
@@ -461,9 +506,9 @@ class JsonProtocolSuite extends SparkFunSuite {
     val oldExecutorMetricsJson =
       JsonProtocol.executorMetricsToJson(executorMetrics)
         .removeField( _._1 == "MappedPoolMemory")
-    val exepectedExecutorMetrics = new ExecutorMetrics(Array(12L, 23L, 45L, 67L,
+    val expectedExecutorMetrics = new ExecutorMetrics(Array(12L, 23L, 45L, 67L,
       78L, 89L, 90L, 123L, 456L, 0L, 40L, 20L, 20L, 10L, 20L, 10L))
-    assertEquals(exepectedExecutorMetrics,
+    assertEquals(expectedExecutorMetrics,
       JsonProtocol.executorMetricsFromJson(oldExecutorMetricsJson))
   }
 
@@ -484,6 +529,54 @@ class JsonProtocolSuite extends SparkFunSuite {
     testAccumValue(Some("anything"), 123, JString("123"))
   }
 
+  /** Create an AccumulableInfo and verify we can serialize and deserialize it. */
+  private def testAccumulableInfo(
+      name: String,
+      value: Option[Any],
+      expectedValue: Option[Any]): Unit = {
+    val isInternal = name.startsWith(InternalAccumulator.METRICS_PREFIX)
+    val accum = AccumulableInfo(
+      123L,
+      Some(name),
+      update = value,
+      value = value,
+      internal = isInternal,
+      countFailedValues = false)
+    val json = JsonProtocol.accumulableInfoToJson(accum)
+    val newAccum = JsonProtocol.accumulableInfoFromJson(json)
+    assert(newAccum == accum.copy(update = expectedValue, value = expectedValue))
+  }
+
+  test("SPARK-31923: unexpected value type of internal accumulator") {
+    // Because a user may use `METRICS_PREFIX` in an accumulator name, we should test unexpected
+    // types to make sure we don't crash.
+    import InternalAccumulator.METRICS_PREFIX
+    testAccumulableInfo(
+      METRICS_PREFIX + "fooString",
+      value = Some("foo"),
+      expectedValue = None)
+    testAccumulableInfo(
+      METRICS_PREFIX + "fooList",
+      value = Some(java.util.Arrays.asList("string")),
+      expectedValue = Some(java.util.Collections.emptyList())
+    )
+    val blocks = Seq(
+      (TestBlockId("block1"), BlockStatus(StorageLevel.MEMORY_ONLY, 1L, 2L)),
+      (TestBlockId("block2"), BlockStatus(StorageLevel.DISK_ONLY, 3L, 4L)))
+    testAccumulableInfo(
+      METRICS_PREFIX + "fooList",
+      value = Some(java.util.Arrays.asList(
+        "string",
+        blocks(0),
+        blocks(1))),
+      expectedValue = Some(blocks.asJava)
+    )
+    testAccumulableInfo(
+      METRICS_PREFIX + "fooSet",
+      value = Some(Set("foo")),
+      expectedValue = None)
+  }
+
   test("SPARK-30936: forwards compatibility - ignore unknown fields") {
     val expected = TestListenerEvent("foo", 123)
     val unknownFieldsJson =
@@ -515,10 +608,10 @@ private[spark] object JsonProtocolSuite extends Assertions {
   private val jobCompletionTime = 1421191296660L
   private val executorAddedTime = 1421458410000L
   private val executorRemovedTime = 1421458922000L
-  private val executorBlacklistedTime = 1421458932000L
-  private val executorUnblacklistedTime = 1421458942000L
-  private val nodeBlacklistedTime = 1421458952000L
-  private val nodeUnblacklistedTime = 1421458962000L
+  private val executorExcludedTime = 1421458932000L
+  private val executorUnexcludedTime = 1421458942000L
+  private val nodeExcludedTime = 1421458952000L
+  private val nodeUnexcludedTime = 1421458962000L
 
   private def testEvent(event: SparkListenerEvent, jsonString: String): Unit = {
     val actualJsonString = compact(render(JsonProtocol.sparkEventToJson(event)))
@@ -865,6 +958,10 @@ private[spark] object JsonProtocolSuite extends Assertions {
     assert(ste1.getFileName === ste2.getFileName)
   }
 
+  private def assertEquals(rp1: ResourceProfile, rp2: ResourceProfile): Unit = {
+    assert(rp1 === rp2)
+  }
+
   /** ----------------------------------- *
    | Util methods for constructing events |
    * ------------------------------------ */
@@ -881,24 +978,38 @@ private[spark] object JsonProtocolSuite extends Assertions {
   private val stackTrace = {
     Array[StackTraceElement](
       new StackTraceElement("Apollo", "Venus", "Mercury", 42),
-      new StackTraceElement("Afollo", "Vemus", "Mercurry", 420),
-      new StackTraceElement("Ayollo", "Vesus", "Blackberry", 4200)
+      new StackTraceElement("Afollo", "Vemus", "Mercurry", 420), /* odd spellings intentional */
+      new StackTraceElement("Ayollo", "Vesus", "Blackberry", 4200) /* odd spellings intentional */
     )
   }
 
-  private def makeRddInfo(a: Int, b: Int, c: Int, d: Long, e: Long) = {
+  private def makeRddInfo(a: Int, b: Int, c: Int, d: Long, e: Long,
+      deterministic: DeterministicLevel.Value) = {
     val r =
-      new RDDInfo(a, "mayor", b, StorageLevel.MEMORY_AND_DISK, false, Seq(1, 4, 7), a.toString)
+      new RDDInfo(a, "mayor", b, StorageLevel.MEMORY_AND_DISK, false, Seq(1, 4, 7), a.toString,
+        outputDeterministicLevel = deterministic)
     r.numCachedPartitions = c
     r.memSize = d
     r.diskSize = e
     r
   }
 
-  private def makeStageInfo(a: Int, b: Int, c: Int, d: Long, e: Long) = {
-    val rddInfos = (0 until a % 5).map { i => makeRddInfo(a + i, b + i, c + i, d + i, e + i) }
+  private def makeStageInfo(
+      a: Int,
+      b: Int,
+      c: Int,
+      d: Long,
+      e: Long,
+      rpId: Int = ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID) = {
+    val rddInfos = (0 until a % 5).map { i =>
+      if (i == (a % 5) - 1) {
+        makeRddInfo(a + i, b + i, c + i, d + i, e + i, DeterministicLevel.INDETERMINATE)
+      } else {
+        makeRddInfo(a + i, b + i, c + i, d + i, e + i, DeterministicLevel.DETERMINATE)
+      }
+    }
     val stageInfo = new StageInfo(a, 0, "greetings", b, rddInfos, Seq(100, 200, 300), "details",
-      resourceProfileId = ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID)
+      resourceProfileId = rpId)
     val (acc1, acc2) = (makeAccumulableInfo(1), makeAccumulableInfo(2))
     stageInfo.accumulables(acc1.id) = acc1
     stageInfo.accumulables(acc2.id) = acc2
@@ -1019,22 +1130,23 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |    "Details": "details",
       |    "Accumulables": [
       |      {
-      |        "ID": 2,
-      |        "Name": "Accumulable2",
-      |        "Update": "delta2",
-      |        "Value": "val2",
-      |        "Internal": false,
-      |        "Count Failed Values": false
-      |      },
-      |      {
       |        "ID": 1,
       |        "Name": "Accumulable1",
       |        "Update": "delta1",
       |        "Value": "val1",
       |        "Internal": false,
       |        "Count Failed Values": false
+      |      },
+      |      {
+      |        "ID": 2,
+      |        "Name": "Accumulable2",
+      |        "Update": "delta2",
+      |        "Value": "val2",
+      |        "Internal": false,
+      |        "Count Failed Values": false
       |      }
-      |    ]
+      |    ],
+      |    "Resource Profile Id" : 0
       |  },
       |  "Properties": {
       |    "France": "Paris",
@@ -1066,6 +1178,8 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |          "Deserialized": true,
       |          "Replication": 1
       |        },
+      |        "Barrier" : false,
+      |        "DeterministicLevel" : "INDETERMINATE",
       |        "Number of Partitions": 201,
       |        "Number of Cached Partitions": 301,
       |        "Memory Size": 401,
@@ -1076,22 +1190,23 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |    "Details": "details",
       |    "Accumulables": [
       |      {
-      |        "ID": 2,
-      |        "Name": "Accumulable2",
-      |        "Update": "delta2",
-      |        "Value": "val2",
-      |        "Internal": false,
-      |        "Count Failed Values": false
-      |      },
-      |      {
       |        "ID": 1,
       |        "Name": "Accumulable1",
       |        "Update": "delta1",
       |        "Value": "val1",
       |        "Internal": false,
       |        "Count Failed Values": false
+      |      },
+      |      {
+      |        "ID": 2,
+      |        "Name": "Accumulable2",
+      |        "Update": "delta2",
+      |        "Value": "val2",
+      |        "Internal": false,
+      |        "Count Failed Values": false
       |      }
-      |    ]
+      |    ],
+      |    "Resource Profile Id" : 0
       |  }
       |}
     """.stripMargin
@@ -1588,6 +1703,8 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
+      |          "DeterministicLevel" : "INDETERMINATE",
       |          "Number of Partitions": 200,
       |          "Number of Cached Partitions": 300,
       |          "Memory Size": 400,
@@ -1598,22 +1715,23 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |      "Details": "details",
       |      "Accumulables": [
       |        {
-      |          "ID": 2,
-      |          "Name": "Accumulable2",
-      |          "Update": "delta2",
-      |          "Value": "val2",
-      |          "Internal": false,
-      |          "Count Failed Values": false
-      |        },
-      |        {
       |          "ID": 1,
       |          "Name": "Accumulable1",
       |          "Update": "delta1",
       |          "Value": "val1",
       |          "Internal": false,
       |          "Count Failed Values": false
+      |        },
+      |        {
+      |          "ID": 2,
+      |          "Name": "Accumulable2",
+      |          "Update": "delta2",
+      |          "Value": "val2",
+      |          "Internal": false,
+      |          "Count Failed Values": false
       |        }
-      |      ]
+      |      ],
+      |      "Resource Profile Id" : 0
       |    },
       |    {
       |      "Stage ID": 2,
@@ -1632,6 +1750,8 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
+      |          "DeterministicLevel" : "DETERMINATE",
       |          "Number of Partitions": 400,
       |          "Number of Cached Partitions": 600,
       |          "Memory Size": 800,
@@ -1648,6 +1768,8 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
+      |          "DeterministicLevel" : "INDETERMINATE",
       |          "Number of Partitions": 401,
       |          "Number of Cached Partitions": 601,
       |          "Memory Size": 801,
@@ -1658,22 +1780,23 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |      "Details": "details",
       |      "Accumulables": [
       |        {
-      |          "ID": 2,
-      |          "Name": "Accumulable2",
-      |          "Update": "delta2",
-      |          "Value": "val2",
-      |          "Internal": false,
-      |          "Count Failed Values": false
-      |        },
-      |        {
       |          "ID": 1,
       |          "Name": "Accumulable1",
       |          "Update": "delta1",
       |          "Value": "val1",
       |          "Internal": false,
       |          "Count Failed Values": false
+      |        },
+      |        {
+      |          "ID": 2,
+      |          "Name": "Accumulable2",
+      |          "Update": "delta2",
+      |          "Value": "val2",
+      |          "Internal": false,
+      |          "Count Failed Values": false
       |        }
-      |      ]
+      |      ],
+      |      "Resource Profile Id" : 0
       |    },
       |    {
       |      "Stage ID": 3,
@@ -1692,6 +1815,8 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
+      |          "DeterministicLevel" : "DETERMINATE",
       |          "Number of Partitions": 600,
       |          "Number of Cached Partitions": 900,
       |          "Memory Size": 1200,
@@ -1708,6 +1833,8 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
+      |          "DeterministicLevel" : "DETERMINATE",
       |          "Number of Partitions": 601,
       |          "Number of Cached Partitions": 901,
       |          "Memory Size": 1201,
@@ -1724,6 +1851,8 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
+      |          "DeterministicLevel" : "INDETERMINATE",
       |          "Number of Partitions": 602,
       |          "Number of Cached Partitions": 902,
       |          "Memory Size": 1202,
@@ -1734,22 +1863,23 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |      "Details": "details",
       |      "Accumulables": [
       |        {
-      |          "ID": 2,
-      |          "Name": "Accumulable2",
-      |          "Update": "delta2",
-      |          "Value": "val2",
-      |          "Internal": false,
-      |          "Count Failed Values": false
-      |        },
-      |        {
       |          "ID": 1,
       |          "Name": "Accumulable1",
       |          "Update": "delta1",
       |          "Value": "val1",
       |          "Internal": false,
       |          "Count Failed Values": false
+      |        },
+      |        {
+      |          "ID": 2,
+      |          "Name": "Accumulable2",
+      |          "Update": "delta2",
+      |          "Value": "val2",
+      |          "Internal": false,
+      |          "Count Failed Values": false
       |        }
-      |      ]
+      |      ],
+      |      "Resource Profile Id" : 0
       |    },
       |    {
       |      "Stage ID": 4,
@@ -1768,6 +1898,8 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
+      |          "DeterministicLevel" : "DETERMINATE",
       |          "Number of Partitions": 800,
       |          "Number of Cached Partitions": 1200,
       |          "Memory Size": 1600,
@@ -1784,6 +1916,8 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
+      |          "DeterministicLevel" : "DETERMINATE",
       |          "Number of Partitions": 801,
       |          "Number of Cached Partitions": 1201,
       |          "Memory Size": 1601,
@@ -1800,6 +1934,8 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
+      |          "DeterministicLevel" : "DETERMINATE",
       |          "Number of Partitions": 802,
       |          "Number of Cached Partitions": 1202,
       |          "Memory Size": 1602,
@@ -1816,6 +1952,8 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
+      |          "DeterministicLevel" : "INDETERMINATE",
       |          "Number of Partitions": 803,
       |          "Number of Cached Partitions": 1203,
       |          "Memory Size": 1603,
@@ -1826,22 +1964,23 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |      "Details": "details",
       |      "Accumulables": [
       |        {
-      |          "ID": 2,
-      |          "Name": "Accumulable2",
-      |          "Update": "delta2",
-      |          "Value": "val2",
-      |          "Internal": false,
-      |          "Count Failed Values": false
-      |        },
-      |        {
       |          "ID": 1,
       |          "Name": "Accumulable1",
       |          "Update": "delta1",
       |          "Value": "val1",
       |          "Internal": false,
       |          "Count Failed Values": false
+      |        },
+      |        {
+      |          "ID": 2,
+      |          "Name": "Accumulable2",
+      |          "Update": "delta2",
+      |          "Value": "val2",
+      |          "Internal": false,
+      |          "Count Failed Values": false
       |        }
-      |      ]
+      |      ],
+      |      "Resource Profile Id" : 0
       |    }
       |  ],
       |  "Stage IDs": [
@@ -1988,7 +2127,8 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |        "name" : "gpu",
       |        "addresses" : [ "0", "1" ]
       |      }
-      |    }
+      |    },
+      |    "Resource Profile Id": 4
       |  }
       |}
     """.stripMargin
@@ -2304,36 +2444,102 @@ private[spark] object JsonProtocolSuite extends Assertions {
     s"""
       |{
       |  "Event" : "org.apache.spark.scheduler.SparkListenerExecutorBlacklisted",
-      |  "time" : ${executorBlacklistedTime},
+      |  "time" : ${executorExcludedTime},
       |  "executorId" : "exec1",
       |  "taskFailures" : 22
       |}
     """.stripMargin
+  private val executorExcludedJsonString =
+    s"""
+       |{
+       |  "Event" : "org.apache.spark.scheduler.SparkListenerExecutorExcluded",
+       |  "time" : ${executorExcludedTime},
+       |  "executorId" : "exec1",
+       |  "taskFailures" : 22
+       |}
+    """.stripMargin
   private val executorUnblacklistedJsonString =
     s"""
       |{
       |  "Event" : "org.apache.spark.scheduler.SparkListenerExecutorUnblacklisted",
-      |  "time" : ${executorUnblacklistedTime},
+      |  "time" : ${executorUnexcludedTime},
       |  "executorId" : "exec1"
       |}
     """.stripMargin
+  private val executorUnexcludedJsonString =
+    s"""
+       |{
+       |  "Event" : "org.apache.spark.scheduler.SparkListenerExecutorUnexcluded",
+       |  "time" : ${executorUnexcludedTime},
+       |  "executorId" : "exec1"
+       |}
+    """.stripMargin
   private val nodeBlacklistedJsonString =
     s"""
       |{
       |  "Event" : "org.apache.spark.scheduler.SparkListenerNodeBlacklisted",
-      |  "time" : ${nodeBlacklistedTime},
+      |  "time" : ${nodeExcludedTime},
       |  "hostId" : "node1",
       |  "executorFailures" : 33
       |}
     """.stripMargin
+  private val nodeExcludedJsonString =
+    s"""
+       |{
+       |  "Event" : "org.apache.spark.scheduler.SparkListenerNodeExcluded",
+       |  "time" : ${nodeExcludedTime},
+       |  "hostId" : "node1",
+       |  "executorFailures" : 33
+       |}
+    """.stripMargin
   private val nodeUnblacklistedJsonString =
     s"""
       |{
       |  "Event" : "org.apache.spark.scheduler.SparkListenerNodeUnblacklisted",
-      |  "time" : ${nodeUnblacklistedTime},
+      |  "time" : ${nodeUnexcludedTime},
       |  "hostId" : "node1"
       |}
     """.stripMargin
+  private val nodeUnexcludedJsonString =
+    s"""
+       |{
+       |  "Event" : "org.apache.spark.scheduler.SparkListenerNodeUnexcluded",
+       |  "time" : ${nodeUnexcludedTime},
+       |  "hostId" : "node1"
+       |}
+    """.stripMargin
+  private val resourceProfileJsonString =
+    """
+      |{
+      |  "Event":"SparkListenerResourceProfileAdded",
+      |  "Resource Profile Id":21,
+      |  "Executor Resource Requests":{
+      |    "cores" : {
+      |      "Resource Name":"cores",
+      |      "Amount":2,
+      |      "Discovery Script":"",
+      |      "Vendor":""
+      |    },
+      |    "gpu":{
+      |      "Resource Name":"gpu",
+      |      "Amount":2,
+      |      "Discovery Script":"myscript",
+      |      "Vendor":""
+      |    }
+      |  },
+      |  "Task Resource Requests":{
+      |    "cpus":{
+      |      "Resource Name":"cpus",
+      |      "Amount":1.0
+      |    },
+      |    "gpu":{
+      |      "Resource Name":"gpu",
+      |      "Amount":1.0
+      |    }
+      |  }
+      |}
+    """.stripMargin
+
 }
 
 case class TestListenerEvent(foo: String, bar: Int) extends SparkListenerEvent
diff --git a/core/src/test/scala/org/apache/spark/util/MutableURLClassLoaderSuite.scala b/core/src/test/scala/org/apache/spark/util/MutableURLClassLoaderSuite.scala
index 597e0b9597c38..9435b5acd2224 100644
--- a/core/src/test/scala/org/apache/spark/util/MutableURLClassLoaderSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/MutableURLClassLoaderSuite.scala
@@ -21,7 +21,8 @@ import java.net.URLClassLoader
 
 import scala.collection.JavaConverters._
 
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.{SparkContext, SparkException, SparkFunSuite, TestUtils}
 
@@ -111,9 +112,9 @@ class MutableURLClassLoaderSuite extends SparkFunSuite with Matchers {
     val res1 = classLoader.getResources("resource1").asScala.toList
     assert(res1.size === 2)
     assert(classLoader.getResources("resource2").asScala.size === 1)
-
-    res1.map(scala.io.Source.fromURL(_).mkString) should contain inOrderOnly
-      ("resource1Contents-child", "resource1Contents-parent")
+    res1.map { res =>
+      Utils.tryWithResource(scala.io.Source.fromURL(res))(_.mkString)
+    } should contain inOrderOnly("resource1Contents-child", "resource1Contents-parent")
     classLoader.close()
     parentLoader.close()
   }
diff --git a/core/src/test/scala/org/apache/spark/util/NextIteratorSuite.scala b/core/src/test/scala/org/apache/spark/util/NextIteratorSuite.scala
index 1efd399b5db68..4909d5f8325cc 100644
--- a/core/src/test/scala/org/apache/spark/util/NextIteratorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/NextIteratorSuite.scala
@@ -21,7 +21,8 @@ import java.util.NoSuchElementException
 
 import scala.collection.mutable.Buffer
 
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.SparkFunSuite
 
diff --git a/core/src/test/scala/org/apache/spark/util/PropertiesCloneBenchmark.scala b/core/src/test/scala/org/apache/spark/util/PropertiesCloneBenchmark.scala
index baacc7527a806..ff4a4941b6b9e 100644
--- a/core/src/test/scala/org/apache/spark/util/PropertiesCloneBenchmark.scala
+++ b/core/src/test/scala/org/apache/spark/util/PropertiesCloneBenchmark.scala
@@ -26,11 +26,11 @@ import org.apache.spark.benchmark.{Benchmark, BenchmarkBase}
 
 
 /**
- * Benchmark for Kryo Unsafe vs safe Serialization.
+ * Benchmark for SerializationUtils.clone vs Utils.cloneProperties.
  * To run this benchmark:
  * {{{
  *   1. without sbt:
- *      bin/spark-submit --class <this class> --jars <spark core test jar>
+ *      bin/spark-submit --class <this class> <spark core test jar>
  *   2. build/sbt "core/test:runMain <this class>"
  *   3. generate result:
  *      SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "core/test:runMain <this class>"
diff --git a/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala b/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala
index d4f2053e0b2f4..d669f2c655abb 100644
--- a/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala
@@ -73,15 +73,35 @@ class SizeEstimatorSuite
   with PrivateMethodTester
   with ResetSystemProperties {
 
+  // Save modified system properties so that we can restore them after tests.
+  val originalArch = System.getProperty("os.arch")
+  val originalCompressedOops = System.getProperty(TEST_USE_COMPRESSED_OOPS_KEY)
+
+  def reinitializeSizeEstimator(arch: String, useCompressedOops: String): Unit = {
+    def set(k: String, v: String): Unit = {
+      if (v == null) {
+        System.clearProperty(k)
+      } else {
+        System.setProperty(k, v)
+      }
+    }
+    set("os.arch", arch)
+    set(TEST_USE_COMPRESSED_OOPS_KEY, useCompressedOops)
+    val initialize = PrivateMethod[Unit](Symbol("initialize"))
+    SizeEstimator invokePrivate initialize()
+  }
+
   override def beforeEach(): Unit = {
-    // Set the arch to 64-bit and compressedOops to true to get a deterministic test-case
     super.beforeEach()
-    System.setProperty("os.arch", "amd64")
-    System.setProperty(TEST_USE_COMPRESSED_OOPS_KEY, "true")
+    // Set the arch to 64-bit and compressedOops to true so that SizeEstimator
+    // provides identical results across all systems in these tests.
+    reinitializeSizeEstimator("amd64", "true")
   }
 
   override def afterEach(): Unit = {
     super.afterEach()
+    // Restore system properties and SizeEstimator to their original states.
+    reinitializeSizeEstimator(originalArch, originalCompressedOops)
   }
 
   test("simple classes") {
@@ -178,11 +198,7 @@ class SizeEstimatorSuite
   }
 
   test("32-bit arch") {
-    System.setProperty("os.arch", "x86")
-
-    val initialize = PrivateMethod[Unit](Symbol("initialize"))
-    SizeEstimator invokePrivate initialize()
-
+    reinitializeSizeEstimator("x86", "true")
     assertResult(40)(SizeEstimator.estimate(DummyString("")))
     assertResult(48)(SizeEstimator.estimate(DummyString("a")))
     assertResult(48)(SizeEstimator.estimate(DummyString("ab")))
@@ -192,11 +208,7 @@ class SizeEstimatorSuite
   // NOTE: The String class definition varies across JDK versions (1.6 vs. 1.7) and vendors
   // (Sun vs IBM). Use a DummyString class to make tests deterministic.
   test("64-bit arch with no compressed oops") {
-    System.setProperty("os.arch", "amd64")
-    System.setProperty(TEST_USE_COMPRESSED_OOPS_KEY, "false")
-    val initialize = PrivateMethod[Unit](Symbol("initialize"))
-    SizeEstimator invokePrivate initialize()
-
+    reinitializeSizeEstimator("amd64", "false")
     assertResult(56)(SizeEstimator.estimate(DummyString("")))
     assertResult(64)(SizeEstimator.estimate(DummyString("a")))
     assertResult(64)(SizeEstimator.estimate(DummyString("ab")))
@@ -214,14 +226,13 @@ class SizeEstimatorSuite
   }
 
   test("class field blocks rounding on 64-bit VM without useCompressedOops") {
+    reinitializeSizeEstimator("amd64", "false")
     assertResult(24)(SizeEstimator.estimate(new DummyClass5))
     assertResult(32)(SizeEstimator.estimate(new DummyClass6))
   }
 
   test("check 64-bit detection for s390x arch") {
-    System.setProperty("os.arch", "s390x")
-    val initialize = PrivateMethod[Unit](Symbol("initialize"))
-    SizeEstimator invokePrivate initialize()
+    reinitializeSizeEstimator("s390x", "true")
     // Class should be 32 bytes on s390x if recognised as 64 bit platform
     assertResult(32)(SizeEstimator.estimate(new DummyClass7))
   }
diff --git a/core/src/test/scala/org/apache/spark/util/SparkUncaughtExceptionHandlerSuite.scala b/core/src/test/scala/org/apache/spark/util/SparkUncaughtExceptionHandlerSuite.scala
index 90741a6bde7f0..9e23b25493dfe 100644
--- a/core/src/test/scala/org/apache/spark/util/SparkUncaughtExceptionHandlerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/SparkUncaughtExceptionHandlerSuite.scala
@@ -80,7 +80,7 @@ object ThrowableThrower {
 
   // a thread that uses SparkUncaughtExceptionHandler and throws a Throwable by name
   class ThrowerThread(name: String, exitOnUncaughtException: Boolean) extends Thread {
-    override def run() {
+    override def run(): Unit = {
       Thread.setDefaultUncaughtExceptionHandler(
         new SparkUncaughtExceptionHandler(exitOnUncaughtException))
       throw ThrowableTypes.getThrowableByName(name)
diff --git a/core/src/test/scala/org/apache/spark/util/UninterruptibleThreadRunnerSuite.scala b/core/src/test/scala/org/apache/spark/util/UninterruptibleThreadRunnerSuite.scala
new file mode 100644
index 0000000000000..40312beacdff6
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/util/UninterruptibleThreadRunnerSuite.scala
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util
+
+import org.apache.spark.SparkFunSuite
+
+class UninterruptibleThreadRunnerSuite extends SparkFunSuite {
+  private var runner: UninterruptibleThreadRunner = null
+
+  override def beforeEach(): Unit = {
+    runner = new UninterruptibleThreadRunner("ThreadName")
+  }
+
+  override def afterEach(): Unit = {
+    runner.shutdown()
+  }
+
+  test("runUninterruptibly should switch to UninterruptibleThread") {
+    assert(!Thread.currentThread().isInstanceOf[UninterruptibleThread])
+    var isUninterruptibleThread = false
+    runner.runUninterruptibly {
+      isUninterruptibleThread = Thread.currentThread().isInstanceOf[UninterruptibleThread]
+    }
+    assert(isUninterruptibleThread, "The runner task must run in UninterruptibleThread")
+  }
+
+  test("runUninterruptibly should not add new UninterruptibleThread") {
+    var isInitialUninterruptibleThread = false
+    var isRunnerUninterruptibleThread = false
+    val t = new UninterruptibleThread("test") {
+      override def run(): Unit = {
+        runUninterruptibly {
+          val initialThread = Thread.currentThread()
+          isInitialUninterruptibleThread = initialThread.isInstanceOf[UninterruptibleThread]
+          runner.runUninterruptibly {
+            val runnerThread = Thread.currentThread()
+            isRunnerUninterruptibleThread = runnerThread.isInstanceOf[UninterruptibleThread]
+            assert(runnerThread.eq(initialThread))
+          }
+        }
+      }
+    }
+    t.start()
+    t.join()
+    assert(isInitialUninterruptibleThread,
+      "The initiator must already run in UninterruptibleThread")
+    assert(isRunnerUninterruptibleThread, "The runner task must run in UninterruptibleThread")
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
index 931eb6b5413f7..404cc343c8d9c 100644
--- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
@@ -18,8 +18,7 @@
 package org.apache.spark.util
 
 import java.io.{ByteArrayInputStream, ByteArrayOutputStream, DataOutput, DataOutputStream, File,
-  FileOutputStream, InputStream, PrintStream, SequenceInputStream}
-import java.lang.{Double => JDouble, Float => JFloat}
+  FileOutputStream, PrintStream, SequenceInputStream}
 import java.lang.reflect.Field
 import java.net.{BindException, ServerSocket, URI}
 import java.nio.{ByteBuffer, ByteOrder}
@@ -42,6 +41,7 @@ import org.apache.hadoop.fs.Path
 import org.apache.spark.{SparkConf, SparkException, SparkFunSuite, TaskContext}
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config._
+import org.apache.spark.internal.config.Tests.IS_TESTING
 import org.apache.spark.network.util.ByteUnit
 import org.apache.spark.scheduler.SparkListener
 import org.apache.spark.util.io.ChunkedByteBufferInputStream
@@ -1024,11 +1024,13 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging {
     // Set some secret keys
     val secretKeys = Seq(
       "spark.executorEnv.HADOOP_CREDSTORE_PASSWORD",
+      "spark.hadoop.fs.s3a.access.key",
       "spark.my.password",
       "spark.my.sECreT")
     secretKeys.foreach { key => sparkConf.set(key, "sensitive_value") }
     // Set a non-secret key
     sparkConf.set("spark.regular.property", "regular_value")
+    sparkConf.set("spark.hadoop.fs.s3a.access_key", "regular_value")
     // Set a property with a regular key but secret in the value
     sparkConf.set("spark.sensitive.property", "has_secret_in_value")
 
@@ -1039,7 +1041,8 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging {
     secretKeys.foreach { key => assert(redactedConf(key) === Utils.REDACTION_REPLACEMENT_TEXT) }
     assert(redactedConf("spark.regular.property") === "regular_value")
     assert(redactedConf("spark.sensitive.property") === Utils.REDACTION_REPLACEMENT_TEXT)
-
+    assert(redactedConf("spark.hadoop.fs.s3a.access.key") === Utils.REDACTION_REPLACEMENT_TEXT)
+    assert(redactedConf("spark.hadoop.fs.s3a.access_key") === "regular_value")
   }
 
   test("redact sensitive information in command line args") {
@@ -1301,6 +1304,150 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging {
       assert(Utils.trimExceptCRLF(s"b${s}b") === s"b${s}b")
     }
   }
+
+  test("pathsToMetadata") {
+    val paths = (0 to 4).map(i => new Path(s"path$i"))
+    assert(Utils.buildLocationMetadata(paths, 10) == "(5 paths)[...]")
+    // 11 is the minimum threshold to print at least one path
+    assert(Utils.buildLocationMetadata(paths, 11) == "(5 paths)[path0, ...]")
+    // 11 + 5 + 2 = 18 is the minimum threshold to print two paths
+    assert(Utils.buildLocationMetadata(paths, 18) == "(5 paths)[path0, path1, ...]")
+  }
+
+  test("checkHost supports both IPV4 and IPV6") {
+    // IPV4 ips
+    Utils.checkHost("0.0.0.0")
+    var e: AssertionError = intercept[AssertionError] {
+      Utils.checkHost("0.0.0.0:0")
+    }
+    assert(e.getMessage.contains("Expected hostname or IP but got 0.0.0.0:0"))
+    e = intercept[AssertionError] {
+      Utils.checkHost("0.0.0.0:")
+    }
+    assert(e.getMessage.contains("Expected hostname or IP but got 0.0.0.0:"))
+    // IPV6 ips
+    Utils.checkHost("[::1]")
+    e = intercept[AssertionError] {
+      Utils.checkHost("[::1]:0")
+    }
+    assert(e.getMessage.contains("Expected hostname or IPv6 IP enclosed in [] but got [::1]:0"))
+    e = intercept[AssertionError] {
+      Utils.checkHost("[::1]:")
+    }
+    assert(e.getMessage.contains("Expected hostname or IPv6 IP enclosed in [] but got [::1]:"))
+    // hostname
+    Utils.checkHost("localhost")
+    e = intercept[AssertionError] {
+      Utils.checkHost("localhost:0")
+    }
+    assert(e.getMessage.contains("Expected hostname or IP but got localhost:0"))
+    e = intercept[AssertionError] {
+      Utils.checkHost("localhost:")
+    }
+    assert(e.getMessage.contains("Expected hostname or IP but got localhost:"))
+  }
+
+  test("checkHostPort support IPV6 and IPV4") {
+    // IPV4 ips
+    Utils.checkHostPort("0.0.0.0:0")
+    var e: AssertionError = intercept[AssertionError] {
+      Utils.checkHostPort("0.0.0.0")
+    }
+    assert(e.getMessage.contains("Expected host and port but got 0.0.0.0"))
+
+    // IPV6 ips
+    Utils.checkHostPort("[::1]:0")
+    e = intercept[AssertionError] {
+      Utils.checkHostPort("[::1]")
+    }
+    assert(e.getMessage.contains("Expected host and port but got [::1]"))
+
+    // hostname
+    Utils.checkHostPort("localhost:0")
+    e = intercept[AssertionError] {
+      Utils.checkHostPort("localhost")
+    }
+    assert(e.getMessage.contains("Expected host and port but got localhost"))
+  }
+
+  test("parseHostPort support IPV6 and IPV4") {
+    // IPV4 ips
+    var hostnamePort = Utils.parseHostPort("0.0.0.0:80")
+    assert(hostnamePort._1.equals("0.0.0.0"))
+    assert(hostnamePort._2 === 80)
+
+    hostnamePort = Utils.parseHostPort("0.0.0.0")
+    assert(hostnamePort._1.equals("0.0.0.0"))
+    assert(hostnamePort._2 === 0)
+
+    hostnamePort = Utils.parseHostPort("0.0.0.0:")
+    assert(hostnamePort._1.equals("0.0.0.0"))
+    assert(hostnamePort._2 === 0)
+
+    // IPV6 ips
+    hostnamePort = Utils.parseHostPort("[::1]:80")
+    assert(hostnamePort._1.equals("[::1]"))
+    assert(hostnamePort._2 === 80)
+
+    hostnamePort = Utils.parseHostPort("[::1]")
+    assert(hostnamePort._1.equals("[::1]"))
+    assert(hostnamePort._2 === 0)
+
+    hostnamePort = Utils.parseHostPort("[::1]:")
+    assert(hostnamePort._1.equals("[::1]"))
+    assert(hostnamePort._2 === 0)
+
+    // hostname
+    hostnamePort = Utils.parseHostPort("localhost:80")
+    assert(hostnamePort._1.equals("localhost"))
+    assert(hostnamePort._2 === 80)
+
+    hostnamePort = Utils.parseHostPort("localhost")
+    assert(hostnamePort._1.equals("localhost"))
+    assert(hostnamePort._2 === 0)
+
+    hostnamePort = Utils.parseHostPort("localhost:")
+    assert(hostnamePort._1.equals("localhost"))
+    assert(hostnamePort._2 === 0)
+  }
+
+  test("executorOffHeapMemorySizeAsMb when MEMORY_OFFHEAP_ENABLED is false") {
+    val executorOffHeapMemory = Utils.executorOffHeapMemorySizeAsMb(new SparkConf())
+    assert(executorOffHeapMemory == 0)
+  }
+
+  test("executorOffHeapMemorySizeAsMb when MEMORY_OFFHEAP_ENABLED is true") {
+    val offHeapMemoryInMB = 50
+    val offHeapMemory: Long = offHeapMemoryInMB * 1024 * 1024
+    val sparkConf = new SparkConf()
+      .set(MEMORY_OFFHEAP_ENABLED, true)
+      .set(MEMORY_OFFHEAP_SIZE, offHeapMemory)
+    val executorOffHeapMemory = Utils.executorOffHeapMemorySizeAsMb(sparkConf)
+    assert(executorOffHeapMemory == offHeapMemoryInMB)
+  }
+
+  test("executorMemoryOverhead when MEMORY_OFFHEAP_ENABLED is true, " +
+    "but MEMORY_OFFHEAP_SIZE not config scene") {
+    val sparkConf = new SparkConf()
+      .set(MEMORY_OFFHEAP_ENABLED, true)
+    val expected =
+      s"${MEMORY_OFFHEAP_SIZE.key} must be > 0 when ${MEMORY_OFFHEAP_ENABLED.key} == true"
+    val message = intercept[IllegalArgumentException] {
+      Utils.executorOffHeapMemorySizeAsMb(sparkConf)
+    }.getMessage
+    assert(message.contains(expected))
+  }
+
+  test("isPushBasedShuffleEnabled when both PUSH_BASED_SHUFFLE_ENABLED" +
+    " and SHUFFLE_SERVICE_ENABLED are true") {
+    val conf = new SparkConf()
+    assert(Utils.isPushBasedShuffleEnabled(conf) === false)
+    conf.set(PUSH_BASED_SHUFFLE_ENABLED, true)
+    conf.set(IS_TESTING, false)
+    assert(Utils.isPushBasedShuffleEnabled(conf) === false)
+    conf.set(SHUFFLE_SERVICE_ENABLED, true)
+    assert(Utils.isPushBasedShuffleEnabled(conf) === true)
+  }
 }
 
 private class SimpleExtension
diff --git a/core/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala
index 56623ebea1651..ff68dd150973d 100644
--- a/core/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala
@@ -98,4 +98,18 @@ class VersionUtilsSuite extends SparkFunSuite {
       }
     }
   }
+
+  test("SPARK-33212: retrieve major/minor/patch version parts") {
+    assert(VersionUtils.majorMinorPatchVersion("3.2.2").contains((3, 2, 2)))
+    assert(VersionUtils.majorMinorPatchVersion("3.2.2.4").contains((3, 2, 2)))
+    assert(VersionUtils.majorMinorPatchVersion("3.2.2-SNAPSHOT").contains((3, 2, 2)))
+    assert(VersionUtils.majorMinorPatchVersion("3.2.2.4XXX").contains((3, 2, 2)))
+    assert(VersionUtils.majorMinorPatchVersion("3.2").contains((3, 2, 0)))
+    assert(VersionUtils.majorMinorPatchVersion("3").contains((3, 0, 0)))
+
+    // illegal cases
+    Seq("ABC", "3X", "3.2-SNAPSHOT", "3.2ABC", "3-ABC", "3.2.4XYZ").foreach { version =>
+      assert(VersionUtils.majorMinorPatchVersion(version).isEmpty, s"version $version")
+    }
+  }
 }
diff --git a/core/src/test/scala/org/apache/spark/util/collection/ExternalAppendOnlyMapSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/ExternalAppendOnlyMapSuite.scala
index 0b4e1494bf300..81a145906d33c 100644
--- a/core/src/test/scala/org/apache/spark/util/collection/ExternalAppendOnlyMapSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/collection/ExternalAppendOnlyMapSuite.scala
@@ -21,8 +21,8 @@ import scala.collection.mutable.ArrayBuffer
 import scala.concurrent.duration._
 import scala.ref.WeakReference
 
-import org.scalatest.Matchers
 import org.scalatest.concurrent.Eventually
+import org.scalatest.matchers.must.Matchers
 
 import org.apache.spark._
 import org.apache.spark.internal.config._
@@ -220,13 +220,13 @@ class ExternalAppendOnlyMapSuite extends SparkFunSuite
     testSimpleSpilling()
   }
 
-  test("spilling with compression") {
+  private def testSimpleSpillingForAllCodecs(encrypt: Boolean) {
     // Keep track of which compression codec we're using to report in test failure messages
     var lastCompressionCodec: Option[String] = None
     try {
       allCompressionCodecs.foreach { c =>
         lastCompressionCodec = Some(c)
-        testSimpleSpilling(Some(c))
+        testSimpleSpilling(Some(c), encrypt)
       }
     } catch {
       // Include compression codec used in test failure message
@@ -241,8 +241,12 @@ class ExternalAppendOnlyMapSuite extends SparkFunSuite
     }
   }
 
+  test("spilling with compression") {
+    testSimpleSpillingForAllCodecs(encrypt = false)
+  }
+
   test("spilling with compression and encryption") {
-    testSimpleSpilling(Some(CompressionCodec.DEFAULT_COMPRESSION_CODEC), encrypt = true)
+    testSimpleSpillingForAllCodecs(encrypt = true)
   }
 
   /**
diff --git a/core/src/test/scala/org/apache/spark/util/collection/ImmutableBitSetSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/ImmutableBitSetSuite.scala
new file mode 100644
index 0000000000000..333d5cb431221
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/util/collection/ImmutableBitSetSuite.scala
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util.collection
+
+import org.apache.spark.SparkFunSuite
+
+class ImmutableBitSetSuite extends SparkFunSuite {
+
+  test("basic get") {
+    val bitset = new ImmutableBitSet(100, 0, 9, 1, 10, 90, 96)
+    val setBits = Seq(0, 9, 1, 10, 90, 96)
+    for (i <- 0 until 100) {
+      if (setBits.contains(i)) {
+        assert(bitset.get(i))
+      } else {
+        assert(!bitset.get(i))
+      }
+    }
+    assert(bitset.cardinality() === setBits.size)
+  }
+
+  test("nextSetBit") {
+    val bitset = new ImmutableBitSet(100, 0, 9, 1, 10, 90, 96)
+
+    assert(bitset.nextSetBit(0) === 0)
+    assert(bitset.nextSetBit(1) === 1)
+    assert(bitset.nextSetBit(2) === 9)
+    assert(bitset.nextSetBit(9) === 9)
+    assert(bitset.nextSetBit(10) === 10)
+    assert(bitset.nextSetBit(11) === 90)
+    assert(bitset.nextSetBit(80) === 90)
+    assert(bitset.nextSetBit(91) === 96)
+    assert(bitset.nextSetBit(96) === 96)
+    assert(bitset.nextSetBit(97) === -1)
+  }
+
+  test( "xor len(bitsetX) < len(bitsetY)" ) {
+    val bitsetX = new ImmutableBitSet(60, 0, 2, 3, 37, 41)
+    val bitsetY = new ImmutableBitSet(100, 0, 1, 3, 37, 38, 41, 85)
+
+    val bitsetXor = bitsetX ^ bitsetY
+
+    assert(bitsetXor.nextSetBit(0) === 1)
+    assert(bitsetXor.nextSetBit(1) === 1)
+    assert(bitsetXor.nextSetBit(2) === 2)
+    assert(bitsetXor.nextSetBit(3) === 38)
+    assert(bitsetXor.nextSetBit(38) === 38)
+    assert(bitsetXor.nextSetBit(39) === 85)
+    assert(bitsetXor.nextSetBit(42) === 85)
+    assert(bitsetXor.nextSetBit(85) === 85)
+    assert(bitsetXor.nextSetBit(86) === -1)
+
+  }
+
+  test( "xor len(bitsetX) > len(bitsetY)" ) {
+    val bitsetX = new ImmutableBitSet(100, 0, 1, 3, 37, 38, 41, 85)
+    val bitsetY = new ImmutableBitSet(60, 0, 2, 3, 37, 41)
+
+    val bitsetXor = bitsetX ^ bitsetY
+
+    assert(bitsetXor.nextSetBit(0) === 1)
+    assert(bitsetXor.nextSetBit(1) === 1)
+    assert(bitsetXor.nextSetBit(2) === 2)
+    assert(bitsetXor.nextSetBit(3) === 38)
+    assert(bitsetXor.nextSetBit(38) === 38)
+    assert(bitsetXor.nextSetBit(39) === 85)
+    assert(bitsetXor.nextSetBit(42) === 85)
+    assert(bitsetXor.nextSetBit(85) === 85)
+    assert(bitsetXor.nextSetBit(86) === -1)
+
+  }
+
+  test( "andNot len(bitsetX) < len(bitsetY)" ) {
+    val bitsetX = new ImmutableBitSet(60, 0, 2, 3, 37, 41, 48)
+    val bitsetY = new ImmutableBitSet(100, 0, 1, 3, 37, 38, 41, 85)
+
+    val bitsetDiff = bitsetX.andNot( bitsetY )
+
+    assert(bitsetDiff.nextSetBit(0) === 2)
+    assert(bitsetDiff.nextSetBit(1) === 2)
+    assert(bitsetDiff.nextSetBit(2) === 2)
+    assert(bitsetDiff.nextSetBit(3) === 48)
+    assert(bitsetDiff.nextSetBit(48) === 48)
+    assert(bitsetDiff.nextSetBit(49) === -1)
+    assert(bitsetDiff.nextSetBit(65) === -1)
+  }
+
+  test( "andNot len(bitsetX) > len(bitsetY)" ) {
+    val bitsetX = new ImmutableBitSet(100, 0, 1, 3, 37, 38, 41, 85)
+    val bitsetY = new ImmutableBitSet(60, 0, 2, 3, 37, 41, 48)
+
+    val bitsetDiff = bitsetX.andNot( bitsetY )
+
+    assert(bitsetDiff.nextSetBit(0) === 1)
+    assert(bitsetDiff.nextSetBit(1) === 1)
+    assert(bitsetDiff.nextSetBit(2) === 38)
+    assert(bitsetDiff.nextSetBit(3) === 38)
+    assert(bitsetDiff.nextSetBit(38) === 38)
+    assert(bitsetDiff.nextSetBit(39) === 85)
+    assert(bitsetDiff.nextSetBit(85) === 85)
+    assert(bitsetDiff.nextSetBit(86) === -1)
+  }
+
+  test( "immutability" ) {
+    val bitset = new ImmutableBitSet(100)
+    intercept[UnsupportedOperationException] {
+      bitset.set(1)
+    }
+    intercept[UnsupportedOperationException] {
+      bitset.setUntil(10)
+    }
+    intercept[UnsupportedOperationException] {
+      bitset.unset(1)
+    }
+    intercept[UnsupportedOperationException] {
+      bitset.clear()
+    }
+    intercept[UnsupportedOperationException] {
+      bitset.clearUntil(10)
+    }
+    intercept[UnsupportedOperationException] {
+      bitset.union(new ImmutableBitSet(100))
+    }
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/util/collection/OpenHashMapSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/OpenHashMapSuite.scala
index 68bcc5e5a5092..08fed93364060 100644
--- a/core/src/test/scala/org/apache/spark/util/collection/OpenHashMapSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/collection/OpenHashMapSuite.scala
@@ -19,7 +19,8 @@ package org.apache.spark.util.collection
 
 import scala.collection.mutable.HashSet
 
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.util.SizeEstimator
diff --git a/core/src/test/scala/org/apache/spark/util/collection/OpenHashSetSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/OpenHashSetSuite.scala
index 44d2118d77945..89a308556d5df 100644
--- a/core/src/test/scala/org/apache/spark/util/collection/OpenHashSetSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/collection/OpenHashSetSuite.scala
@@ -17,7 +17,8 @@
 
 package org.apache.spark.util.collection
 
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.util.SizeEstimator
diff --git a/core/src/test/scala/org/apache/spark/util/collection/PrimitiveKeyOpenHashMapSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/PrimitiveKeyOpenHashMapSuite.scala
index f5ee428020fd4..3b42300ccdd6a 100644
--- a/core/src/test/scala/org/apache/spark/util/collection/PrimitiveKeyOpenHashMapSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/collection/PrimitiveKeyOpenHashMapSuite.scala
@@ -19,7 +19,8 @@ package org.apache.spark.util.collection
 
 import scala.collection.mutable.HashSet
 
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.util.SizeEstimator
diff --git a/core/src/test/scala/org/apache/spark/util/collection/SorterSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/SorterSuite.scala
index bb03f0d3cdc20..82b4f2eac6248 100644
--- a/core/src/test/scala/org/apache/spark/util/collection/SorterSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/collection/SorterSuite.scala
@@ -39,8 +39,8 @@ class SorterSuite extends SparkFunSuite with Logging {
     new Sorter(new KeyReuseIntArraySortDataFormat)
       .sort(data2, 0, data2.length, Ordering[IntWrapper])
 
-    assert(data0.view === data1.view)
-    assert(data0.view === data2.view)
+    assert(data0 === data1)
+    assert(data0 === data2)
   }
 
   test("KVArraySorter") {
diff --git a/core/src/test/scala/org/apache/spark/util/collection/unsafe/sort/RadixSortSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/unsafe/sort/RadixSortSuite.scala
index 9ae6a8ef879f3..b3e5e0a73dd9f 100644
--- a/core/src/test/scala/org/apache/spark/util/collection/unsafe/sort/RadixSortSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/collection/unsafe/sort/RadixSortSuite.scala
@@ -157,7 +157,7 @@ class RadixSortSuite extends SparkFunSuite with Logging {
         buffer, N, sortType.startByteIdx, sortType.endByteIdx,
         sortType.descending, sortType.signed)
       val result = collectToArray(buffer, outOffset, N)
-      assert(ref.view == result.view)
+      assert(ref === result)
     }
 
     test("sort key prefix " + sortType.name) {
@@ -169,7 +169,7 @@ class RadixSortSuite extends SparkFunSuite with Logging {
         sortType.descending, sortType.signed)
       val res1 = collectToArray(buf1, 0, N * 2)
       val res2 = collectToArray(buf2, outOffset, N * 2)
-      assert(res1.view == res2.view)
+      assert(res1 === res2)
     }
 
     fuzzTest(s"fuzz test ${sortType.name} with random bitmasks") { seed =>
@@ -181,7 +181,7 @@ class RadixSortSuite extends SparkFunSuite with Logging {
         buffer, N, sortType.startByteIdx, sortType.endByteIdx,
         sortType.descending, sortType.signed)
       val result = collectToArray(buffer, outOffset, N)
-      assert(ref.view == result.view)
+      assert(ref === result)
     }
 
     fuzzTest(s"fuzz test key prefix ${sortType.name} with random bitmasks") { seed =>
@@ -194,7 +194,7 @@ class RadixSortSuite extends SparkFunSuite with Logging {
         sortType.descending, sortType.signed)
       val res1 = collectToArray(buf1, 0, N * 2)
       val res2 = collectToArray(buf2, outOffset, N * 2)
-      assert(res1.view == res2.view)
+      assert(res1 ===res2)
     }
   }
 }
diff --git a/core/src/test/scala/org/apache/spark/util/random/RandomSamplerSuite.scala b/core/src/test/scala/org/apache/spark/util/random/RandomSamplerSuite.scala
index fef514e0c4e37..eb1aab645f66e 100644
--- a/core/src/test/scala/org/apache/spark/util/random/RandomSamplerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/random/RandomSamplerSuite.scala
@@ -22,7 +22,8 @@ import java.util.Random
 import scala.collection.mutable.ArrayBuffer
 
 import org.apache.commons.math3.distribution.PoissonDistribution
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.SparkFunSuite
 
@@ -322,7 +323,7 @@ class RandomSamplerSuite extends SparkFunSuite with Matchers {
     RandomSampler.defaultMaxGapSamplingFraction should be (0.4)
 
     var d: Double = 0.0
-    var sampler = new BernoulliSampler[Int](0.1)
+    val sampler = new BernoulliSampler[Int](0.1)
     sampler.setSeed(rngSeed.nextLong)
 
     // Array iterator (indexable type)
@@ -546,7 +547,7 @@ class RandomSamplerSuite extends SparkFunSuite with Matchers {
     RandomSampler.defaultMaxGapSamplingFraction should be (0.4)
 
     var d: Double = 0.0
-    var sampler = new PoissonSampler[Int](0.1)
+    val sampler = new PoissonSampler[Int](0.1)
     sampler.setSeed(rngSeed.nextLong)
 
     // Array iterator (indexable type)
diff --git a/core/src/test/scala/org/apache/spark/util/random/XORShiftRandomBenchmark.scala b/core/src/test/scala/org/apache/spark/util/random/XORShiftRandomBenchmark.scala
index b0563fdb2badb..7fd63a2c3518f 100644
--- a/core/src/test/scala/org/apache/spark/util/random/XORShiftRandomBenchmark.scala
+++ b/core/src/test/scala/org/apache/spark/util/random/XORShiftRandomBenchmark.scala
@@ -27,7 +27,7 @@ import org.apache.spark.util.Utils.times
  * To run this benchmark:
  * {{{
  *   1. without sbt:
- *      bin/spark-submit --class <this class> --jars <spark core test jar>
+ *      bin/spark-submit --class <this class> <spark core test jar>
  *   2. build/sbt "core/test:runMain <this class>"
  *   3. generate result:
  *      SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "core/test:runMain <this class>"
diff --git a/core/src/test/scala/org/apache/spark/util/random/XORShiftRandomSuite.scala b/core/src/test/scala/org/apache/spark/util/random/XORShiftRandomSuite.scala
index df3483830ca9c..267ab1020a185 100644
--- a/core/src/test/scala/org/apache/spark/util/random/XORShiftRandomSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/random/XORShiftRandomSuite.scala
@@ -18,7 +18,8 @@
 package org.apache.spark.util.random
 
 import org.apache.commons.math3.stat.inference.ChiSquareTest
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.util.Utils.times
diff --git a/dev/.rat-excludes b/dev/.rat-excludes
index f997c9b3f9f24..6c809f43418cd 100644
--- a/dev/.rat-excludes
+++ b/dev/.rat-excludes
@@ -25,15 +25,13 @@ bootstrap.bundle.min.js
 bootstrap.bundle.min.js.map
 bootstrap.min.css
 bootstrap.min.css.map
-jquery-3.4.1.min.js
+jquery-3.5.1.min.js
 d3.min.js
 dagre-d3.min.js
 graphlib-dot.min.js
 sorttable.js
 vis-timeline-graph2d.min.js
-vis-timeline-graph2d.min.js.map
 vis-timeline-graph2d.min.css
-vis-timeline-graph2d.min.css.map
 dataTables.bootstrap4.1.10.20.min.css
 dataTables.bootstrap4.1.10.20.min.js
 dataTables.rowsGroup.js
@@ -44,14 +42,13 @@ jquery.dataTables.1.10.20.min.js
 jquery.mustache.js
 jsonFormatter.min.css
 jsonFormatter.min.js
-.*avsc
-.*txt
-.*json
-.*data
-.*log
-pyspark-coverage-site/
-cloudpickle.py
-heapq3.py
+.*\.avsc
+.*\.txt
+.*\.json
+.*\.data
+.*\.log
+pyspark-coverage-site/*
+cloudpickle/*
 join.py
 SparkExprTyper.scala
 SparkILoop.scala
@@ -101,17 +98,17 @@ local-1430917381535_2
 DESCRIPTION
 NAMESPACE
 test_support/*
-.*Rd
+.*\.Rd
 help/*
 html/*
 INDEX
 .lintr
 gen-java.*
-.*avpr
-.*parquet
+.*\.avpr
+.*\.parquet
 spark-deps-.*
-.*csv
-.*tsv
+.*\.csv
+.*\.tsv
 .*\.sql
 .Rbuildignore
 META-INF/*
@@ -124,3 +121,17 @@ vote.tmpl
 SessionManager.java
 SessionHandler.java
 GangliaReporter.java
+application_1578436911597_0052
+config.properties
+local-1596020211915
+app-20200706201101-0003
+py.typed
+_metadata
+_SUCCESS
+part-00000
+.*\.res
+flights_tiny.txt.1
+over1k
+over10k
+exported_table/*
+node_modules
\ No newline at end of file
diff --git a/dev/.scalafmt.conf b/dev/.scalafmt.conf
index 9a8813e3b3eed..9598540752ebd 100644
--- a/dev/.scalafmt.conf
+++ b/dev/.scalafmt.conf
@@ -25,4 +25,4 @@ optIn = {
 danglingParentheses = false
 docstrings = JavaDoc
 maxColumn = 98
-
+newlines.topLevelStatements = [before,after]
diff --git a/dev/appveyor-guide.md b/dev/appveyor-guide.md
index a8c0c1ef23ac3..c68b5de9e61d0 100644
--- a/dev/appveyor-guide.md
+++ b/dev/appveyor-guide.md
@@ -33,22 +33,22 @@ Currently, SparkR on Windows is being tested with [AppVeyor](https://ci.appveyor
     
   <img width="379" alt="2016-09-04 11 07 58" src="https://cloud.githubusercontent.com/assets/6477701/18228810/2f674e5e-7299-11e6-929d-5c2dff269ddc.png">
 
-- Click "Github".
+- Click "GitHub".
 
   <img width="360" alt="2016-09-04 11 08 10" src="https://cloud.githubusercontent.com/assets/6477701/18228811/344263a0-7299-11e6-90b7-9b1c7b6b8b01.png">
 
 
-#### After signing up, go to profile to link Github and AppVeyor.
+#### After signing up, go to profile to link GitHub and AppVeyor.
 
 - Click your account and then click "Profile".
 
   <img width="204" alt="2016-09-04 11 09 43" src="https://cloud.githubusercontent.com/assets/6477701/18228803/12a4b810-7299-11e6-9140-5cfc277297b1.png">
 
-- Enable the link with GitHub via clicking "Link Github account".
+- Enable the link with GitHub via clicking "Link GitHub account".
 
   <img width="256" alt="2016-09-04 11 09 52" src="https://cloud.githubusercontent.com/assets/6477701/18228808/23861584-7299-11e6-9352-640a9c747c83.png">
 
-- Click "Authorize application" in Github site.
+- Click "Authorize application" in GitHub site.
 
 <img width="491" alt="2016-09-04 11 10 05" src="https://cloud.githubusercontent.com/assets/6477701/18228814/5cc239e0-7299-11e6-8aeb-71305e22d930.png">
 
@@ -63,11 +63,11 @@ Currently, SparkR on Windows is being tested with [AppVeyor](https://ci.appveyor
   
   <img width="144" alt="2016-08-30 12 16 35" src="https://cloud.githubusercontent.com/assets/6477701/18075026/3ee57bc6-6eac-11e6-826e-5dd09aeb0e7c.png">
 
-- Since we will use Github here, click the "GITHUB" button and then click "Authorize Github" so that AppVeyor can access the Github logs (e.g. commits).
+- Since we will use GitHub here, click the "GITHUB" button and then click "Authorize GitHub" so that AppVeyor can access the GitHub logs (e.g. commits).
     
   <img width="517" alt="2016-09-04 11 10 22" src="https://cloud.githubusercontent.com/assets/6477701/18228819/9a4d5722-7299-11e6-900c-c5ff6b0450b1.png">
 
-- Click "Authorize application" from Github (the above step will pop up this page).
+- Click "Authorize application" from GitHub (the above step will pop up this page).
 
   <img width="484" alt="2016-09-04 11 10 27" src="https://cloud.githubusercontent.com/assets/6477701/18228820/a7cfce02-7299-11e6-8ec0-1dd7807eecb7.png">
 
diff --git a/dev/appveyor-install-dependencies.ps1 b/dev/appveyor-install-dependencies.ps1
index 4be420b4e8594..fb4cc22de35f4 100644
--- a/dev/appveyor-install-dependencies.ps1
+++ b/dev/appveyor-install-dependencies.ps1
@@ -19,7 +19,7 @@ $CRAN = "https://cloud.r-project.org"
 
 Function InstallR {
   if ( -not(Test-Path Env:\R_ARCH) ) {
-    $arch = "i386"
+    $arch = "x64"
   }
   Else {
     $arch = $env:R_ARCH
@@ -67,7 +67,7 @@ Function InstallRtools {
   Else {
     $gccPath = $env:GCC_PATH
   }
-  $env:PATH = $RtoolsDrive + '\Rtools40\bin;' + $RtoolsDrive + '\Rtools40\MinGW$(WIN)\bin;' + $RtoolsDrive + '\Rtools40\' + $gccPath + '\bin;' + $env:PATH
+  $env:PATH = $RtoolsDrive + '\Rtools40\bin;' + $RtoolsDrive + '\Rtools40\mingw64\bin;' + $RtoolsDrive + '\Rtools40\' + $gccPath + '\bin;' + $env:PATH
   $env:BINPREF=$RtoolsDrive + '/Rtools40/mingw$(WIN)/bin/'
 }
 
@@ -95,28 +95,28 @@ $env:MAVEN_OPTS = "-Xmx2g -XX:ReservedCodeCacheSize=1g"
 Pop-Location
 
 # ========================== Hadoop bin package
-# This must match the version at https://github.com/steveloughran/winutils/tree/master/hadoop-2.7.1
-$hadoopVer = "2.7.1"
+# This must match the version at https://github.com/cdarlint/winutils/tree/master/hadoop-3.2.0
+$hadoopVer = "3.2.0"
 $hadoopPath = "$tools\hadoop"
 if (!(Test-Path $hadoopPath)) {
     New-Item -ItemType Directory -Force -Path $hadoopPath | Out-Null
 }
 Push-Location $hadoopPath
 
-Start-FileDownload "https://github.com/steveloughran/winutils/archive/master.zip" "winutils-master.zip"
+Start-FileDownload "https://codeload.github.com/cdarlint/winutils/zip/master" "winutils-master.zip"
 
 # extract
 Invoke-Expression "7z.exe x winutils-master.zip"
 
 # add hadoop bin to environment variables
-$env:HADOOP_HOME = "$hadoopPath/winutils-master/hadoop-$hadoopVer"
-$env:Path += ";$env:HADOOP_HOME\bin"
+$env:HADOOP_HOME = "$hadoopPath\winutils-master\hadoop-$hadoopVer"
+$env:PATH = "$env:HADOOP_HOME\bin;" + $env:PATH
 
 Pop-Location
 
 # ========================== R
-$rVer = "3.6.2"
-$rToolsVer = "4.0.0"
+$rVer = "4.0.2"
+$rToolsVer = "4.0.2"
 
 InstallR
 InstallRtools
diff --git a/dev/change-scala-version.sh b/dev/change-scala-version.sh
index 06411b9b12a0d..48b7f6475117a 100755
--- a/dev/change-scala-version.sh
+++ b/dev/change-scala-version.sh
@@ -60,6 +60,17 @@ BASEDIR=$(dirname $0)/..
 find "$BASEDIR" -name 'pom.xml' -not -path '*target*' -print \
   -exec bash -c "sed_i 's/\(artifactId.*\)_'$FROM_VERSION'/\1_'$TO_VERSION'/g' {}" \;
 
+# dependency:get is workaround for SPARK-34762 to download the JAR file of commons-cli.
+# Without this, build with Scala 2.13 using SBT will fail because the help plugin used below downloads only the POM file.
+COMMONS_CLI_VERSION=`build/mvn help:evaluate -Dexpression=commons-cli.version -q -DforceStdout`
+build/mvn dependency:get -Dartifact=commons-cli:commons-cli:${COMMONS_CLI_VERSION} -q
+
+# Update <scala.version> in parent POM
+# First find the right full version from the profile's build
+SCALA_VERSION=`build/mvn help:evaluate -Pscala-${TO_VERSION} -Dexpression=scala.version -q -DforceStdout`
+sed_i '1,/<scala\.version>[0-9]*\.[0-9]*\.[0-9]*</s/<scala\.version>[0-9]*\.[0-9]*\.[0-9]*</<scala.version>'$SCALA_VERSION'</' \
+  "$BASEDIR/pom.xml"
+
 # Also update <scala.binary.version> in parent POM
 # Match any scala binary version to ensure idempotency
 sed_i '1,/<scala\.binary\.version>[0-9]*\.[0-9]*</s/<scala\.binary\.version>[0-9]*\.[0-9]*</<scala.binary.version>'$TO_VERSION'</' \
@@ -67,4 +78,9 @@ sed_i '1,/<scala\.binary\.version>[0-9]*\.[0-9]*</s/<scala\.binary\.version>[0-9
 
 # Update source of scaladocs
 echo "$BASEDIR/docs/_plugins/copy_api_dirs.rb"
+if [ $TO_VERSION = "2.13" ]; then
+  sed_i '/\-Pscala-'$TO_VERSION'/!s:build/sbt:build/sbt \-Pscala\-'$TO_VERSION':' "$BASEDIR/docs/_plugins/copy_api_dirs.rb"
+else
+  sed_i 's:build/sbt \-Pscala\-'$FROM_VERSION':build/sbt:' "$BASEDIR/docs/_plugins/copy_api_dirs.rb"
+fi
 sed_i 's/scala\-'$FROM_VERSION'/scala\-'$TO_VERSION'/' "$BASEDIR/docs/_plugins/copy_api_dirs.rb"
diff --git a/dev/check-license b/dev/check-license
index 0cc17ffe55c67..bd255954d6db4 100755
--- a/dev/check-license
+++ b/dev/check-license
@@ -67,7 +67,7 @@ mkdir -p "$FWDIR"/lib
     exit 1
 }
 
-mkdir target
+mkdir -p target
 $java_cmd -jar "$rat_jar" -E "$FWDIR"/dev/.rat-excludes -d "$FWDIR" > target/rat-results.txt
 
 if [ $? -ne 0 ]; then
diff --git a/dev/create-release/do-release-docker.sh b/dev/create-release/do-release-docker.sh
index 2f794c0e0a174..f1632f01686c7 100755
--- a/dev/create-release/do-release-docker.sh
+++ b/dev/create-release/do-release-docker.sh
@@ -91,6 +91,11 @@ for f in "$SELF"/*; do
   fi
 done
 
+# Add the fallback version of Gemfile, Gemfile.lock and .bundle/config to the local directory.
+cp "$SELF/../../docs/Gemfile" "$WORKDIR"
+cp "$SELF/../../docs/Gemfile.lock" "$WORKDIR"
+cp -r "$SELF/../../docs/.bundle" "$WORKDIR"
+
 GPG_KEY_FILE="$WORKDIR/gpg.key"
 fcreate_secure "$GPG_KEY_FILE"
 $GPG --export-secret-key --armor --pinentry-mode loopback --passphrase "$GPG_PASSPHRASE" "$GPG_KEY" > "$GPG_KEY_FILE"
diff --git a/dev/create-release/generate-contributors.py b/dev/create-release/generate-contributors.py
index d9135173419ae..75965a6a26201 100755
--- a/dev/create-release/generate-contributors.py
+++ b/dev/create-release/generate-contributors.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
@@ -22,7 +22,9 @@
 import re
 import sys
 
-from releaseutils import *
+from releaseutils import tag_exists, get_commits, yesOrNoPrompt, get_date, \
+    is_valid_author, capitalize_author, JIRA, find_components, translate_issue_type, \
+    translate_component, CORE_COMPONENT, contributors_file_name, nice_join
 
 # You must set the following before use!
 JIRA_API_BASE = os.environ.get("JIRA_API_BASE", "https://issues.apache.org/jira")
@@ -31,10 +33,10 @@
 
 # If the release tags are not provided, prompt the user to provide them
 while not tag_exists(RELEASE_TAG):
-    RELEASE_TAG = raw_input("Please provide a valid release tag: ")
+    RELEASE_TAG = input("Please provide a valid release tag: ")
 while not tag_exists(PREVIOUS_RELEASE_TAG):
     print("Please specify the previous release tag.")
-    PREVIOUS_RELEASE_TAG = raw_input(
+    PREVIOUS_RELEASE_TAG = input(
         "For instance, if you are releasing v1.2.0, you should specify v1.1.0: ")
 
 # Gather commits found in the new tag but not in the old tag.
@@ -234,7 +236,7 @@ def populate(issue_type, components):
 # e.g. * Andrew Or -- Bug fixes in Windows, Core, and Web UI; improvements in Core
 # e.g. * Tathagata Das -- Bug fixes and new features in Streaming
 contributors_file = open(contributors_file_name, "w")
-authors = author_info.keys()
+authors = list(author_info.keys())
 authors.sort()
 for author in authors:
     contribution = ""
diff --git a/dev/create-release/known_translations b/dev/create-release/known_translations
index 65c00cce8c9c6..3b599b98c199c 100644
--- a/dev/create-release/known_translations
+++ b/dev/create-release/known_translations
@@ -1,263 +1,418 @@
 # This is a mapping of names to be translated through translate-contributors.py
-# The format expected on each line should be: <old name> - <new name>
+# The format expected on each line should be: <GitHub ID> - <full name>
+012huang - Weiyi Huang
+07ARB - Ankit Raj Boudh
+10110346 - Xian Liu
+979969786 - Yuming Wang
+Achuth17 - Achuth Narayan Rajagopal
+Adamyuanyuan - Adam Wang
+AiHe - Ai He
+Andrew-Crosby - Andrew Crosby
+AngersZhuuuu - Yi Zhu
+BenFradet - Ben Fradet
+Bilna - Bilna P
+ClassNotFoundExp - Fu Xing
 CodingCat - Nan Zhu
 CrazyJvm - Chao Chen
+Deegue - Yizhong Zhang
+DoingDone9 - Doing Done
+DylanGuedes - Dylan Guedes
+Earne - Ernest
 EugenCepoi - Eugen Cepoi
+FavioVazquez - Favio Vazquez
+FlytxtRnD - Meethu Mathew
+GenTang - Gen TANG
 GraceH - Jie Huang
+Gschiavon - German Schiavon Matteo
+GuoPhilipse - Philipse Guo
+HeartSaVioR - Jungtaek Lim
+Hellsen83 - Erik Christiansen
+HyukjinKwon - Hyukjin Kwon
+Icysandwich - Icysandwich
+JDrit - Joseph Batchik
+JasonWayne - Wenjie Wu
+JaysonSunshine - Jayson Sunshine
 JerryLead - Lijie Xu
+JiahuiJiang - Jiahui Jiang
+JkSelf - Ke Jia
+JoanFM - Joan Fontanals
+JoshRosen - Josh Rosen
+JuhongPark - Juhong Park
+JulienPeloton - Julien Peloton
+KaiXinXiaoLei - KaiXinXIaoLei
+KevinGrealish - Kevin Grealish
+KevinZwx - Kevin Zhang
+Koraseg - Artem Kupchinskiy
+KyleLi1985 - Liang Li
+LantaoJin - Lantao Jin
+Lemonjing - Rann Tao
 Leolh - Liu Hao
 Lewuathe - Kai Sasaki
+LiShuMing - Shuming Li
+LinhongLiu - Linhong Liu
+Liuchang0812 - Liu Chang
+LucaCanali - Luca Canali
+LuciferYang - Yang Jie
+MasterDDT - Mitesh Patel
+MaxGekk - Maxim Gekk
+MechCoder - Manoj Kumar
+NamelessAnalyst - NamelessAnalyst
+Ngone51 - Yi Wu
+OopsOutOfMemory - Sheng Li
+PavithraRamachandran - Pavithra Ramachandran
+Peishen-Jia - Peishen Jia
 RongGu - Rong Gu
+Rosstin - Rosstin Murphy
+SaintBacchus - Huang Zhaowei
+Sephiroth-Lin - Sephiroth Lin
 Shiti - Shiti Saxena
+SongYadong - Yadong Song
+TJX2014 - Jinxin Tang
+TigerYang414 - David Yang
+TomokoKomiyama - Tomoko Komiyama
+TopGunViper - TopGunViper
+Udbhav30 - Udbhav Agrawal
 Victsm - Min Shen
+VinceShieh - Vincent Xie
+WangGuangxin - Guangxin Wang
 WangTaoTheTonic - Wang Tao
+WeichenXu123 - Weichen Xu
+William1104 - William Wong
+XD-DENG - Xiaodong Deng
 XuTingjun - Tingjun Xu
 YanTangZhai - Yantang Zhai
+YongjinZhou - Yongjin Zhou
+Yunni - Yun Ni
+aai95 - Aleksei Izmalkin
+aaruna - Aaruna Godthi
+actuaryzhang - Wayne Zhang
+adrian555 - Weiqiang Zhuang
+ajithme - Ajith S
+akonopko - Alexander Konopko
 alexdebrie - Alex DeBrie
+alicegugu - Gu Huiqin Alice
+allisonwang-db - Allison Wang
 alokito - Alok Saldanha
+alyaxey - Alex Slusarenko
+amanomer - Aman Omer
+ameyc - Amey Chaugule
+anabranch - Bill Chambers
 anantasty - Anant Asthana
+ancasarb - Anca Sarb
+anchovYu - Xinyi Yu
 andrewor14 - Andrew Or
 aniketbhatnagar - Aniket Bhatnagar
+animeshbaranawal - Animesh Baranawal
+ankuriitg - Ankur Gupta
+aokolnychyi - Anton Okolnychyi
 arahuja - Arun Ahuja
+arucard21 - Riaas Mokiem
+ashangit - Nicolas Fraison
+attilapiros - Attila Zsolt Piros
+avkgh - Aleksandr Kashkirov
+avulanov - Alexander Ulanov
+ayudovin - Artsiom Yudovin
+azagrebin - Andrey Zagrebin
+baishuo - Shuo Bai
+bartosz25 - Bartosz Konieczny
+beliefer - Jiaan Geng
+bettermouse - Chen Hao
+biglobster - Liang Ke
+bravo-zhang - Bravo Zhang
 brkyvz - Burak Yavuz
+bscan - Brian Scannell
+bzz - Alexander Bezzubov
+cafreeman - Chris Freeman
+caneGuy - Kang Zhou
+cchung100m - Neo Chien
+cclauss - Christian Clauss
+cenyuhai - Yuhai Cen
+chakravarthiT - Chakravarthi
+chandulal - Chandu Kavar
+chaoslawful - Xiaozhe Wang
 chesterxgchen - Chester Chen
 chiragaggarwal - Chirag Aggarwal
+chitralverma - Chitral Verma
 chouqin - Qiping Li
+cjn082030 - Juanni Chen
+cloud-fan - Wenchen Fan
+cluo512 - Chuan Luo
 cocoatomo - Tomohiko K.
+codeatri - Neha Patil
+codeborui - codeborui
 coderfi - Fairiz Azizi
 coderxiang - Shuo Xiang
+codlife - Jianfei Wang
+colinmjj - Colin Ma
+crafty-coder - Carlos Pena
+cxzl25 - Shaoyun Chen
+cyq89051127 - Yongqiang Chai
+darrentirto - Darren Tirto
+david-weiluo-ren - Weiluo (David) Ren
+daviddingly - Xiaoyuan Ding
+davidvrba - David Vrba
 davies - Davies Liu
+dding3 - Ding Ding
+debugger87 - Chaozhong Yang
+deepyaman - Deepyaman Datta
+denglingang - Lingang Deng
+dengziming - dengziming
+deshanxiao - deshanxiao
+dima-asana - Dima Kamalov
+dlindelof - David Lindelof
+dobashim - Masaru Dobashi
+dongjoon-hyun - Dongjoon Hyun
+e-dorigatti - Emilio Dorigatti
+eatoncys - Yanshan Chen
+ehnalis - Zoltan Zvara
+emres - Emre Sevinc
 epahomov - Egor Pahomov
+eric-maynard - Eric Maynard
 falaki - Hossein Falaki
-freeman-lab - Jeremy Freeman
-industrial-sloth - Jascha Swisher
-jackylk - Jacky Li
-jayunit100 - Jay Vyas
-jerryshao - Saisai Shao
-jkbradley - Joseph Bradley
-lianhuiwang - Lianhui Wang
-lirui-intel - Rui Li
-luluorta - Lu Lu
-luogankun - Gankun Luo
-maji2014 - Derek Ma
-mccheah - Matthew Cheah
-mengxr - Xiangrui Meng
-nartz - Nathan Artz
-odedz - Oded Zimerman
-ravipesala - Ravindra Pesala
-roxchkplusony - Victor Tso
-scwf - Wang Fei
-shimingfei - Shiming Fei
-surq - Surong Quan
-suyanNone - Su Yan
-tedyu - Ted Yu
-tigerquoll - Dale Richardson
-wangxiaojing - Xiaojing Wang
-watermen - Yadong Qi
-witgo - Guoqiang Li
-xinyunh - Xinyun Huang
-zsxwing - Shixiong Zhu
-Bilna - Bilna P
-DoingDone9 - Doing Done
-Earne - Ernest
-FlytxtRnD - Meethu Mathew
-GenTang - Gen TANG
-JoshRosen - Josh Rosen
-MechCoder - Manoj Kumar
-OopsOutOfMemory - Sheng Li
-Peishen-Jia - Peishen Jia
-SaintBacchus - Huang Zhaowei
-azagrebin - Andrey Zagrebin
-bzz - Alexander Bezzubov
+fan31415 - Yijie Fan
+fe2s - Oleksiy Dyagilev
+felixalbani - Felix Albani
+felixcheung - Felix Cheung
+feynmanliang - Feynman Liang
+fidato13 - Tarun Kumar
+fitermay - Yuli Fiterman
+fjh100456 - Jinhua Fu
 fjiang6 - Fan Jiang
+francis0407 - Mingcong Han
+freeman-lab - Jeremy Freeman
+frreiss - Fred Reiss
+fuwhu - Fuwang Hu
 gasparms - Gaspar Munoz
+gatorsmile - Xiao Li
+gchen - Guancheng Chen
+gss2002 - Greg Senia
 guowei2 - Guo Wei
+guoxiaolongzte - Xiaolong Guo
+haiyangsea - Haiyang Sea
+hayashidac - Chie Hayashida
+hddong - Dongdong Hong
+heary-cao - Xuewen Cao
+hehuiyuan - hehuiyuan
+helenyugithub - Helen Yu
 hhbyyh - Yuhao Yang
+highmoutain - highmoutain
+hlin09 - Hao Lin
+hqzizania - Qian Huang
 hseagle - Peng Xu
+httfighter - Tiantian Han
+huangtianhua - huangtianhua
+huangweizhe123 - Weizhe Huang
+hvanhovell - Herman Van Hovell
+iRakson - Rakesh Raushan
+igorcalabria - Igor Calabria
+imback82 - Terry Kim
+industrial-sloth - Jascha Swisher
+invkrh - Hao Ren
+itholic - Haejoon Lee
+ivoson - Tengfei Huang
+jackylk - Jacky Li
+jagadeesanas2 - Jagadeesan A S
 javadba - Stephen Boesch
+javierivanov - Javier Fuentes
+jayunit100 - Jay Vyas
 jbencook - Ben Cook
-kul - Kuldeep
-ligangty - Gang Li
-marsishandsome - Liangliang Gu
-medale - Markus Dale
-nemccarthy - Nathan McCarthy
-nxwhite-str - Nate Crosswhite
-seayi - Xiaohua Yi
-tianyi - Yi Tian
-uncleGen - Uncle Gen
-viper-kun - Xu Kun
-x1- - Yuri Saito
-zapletal-martin - Martin Zapletal
-zuxqoj - Shekhar Bansal
-mingyukim - Mingyu Kim
-sigmoidanalytics - Mayur Rustagi
-AiHe - Ai He
-BenFradet - Ben Fradet
-FavioVazquez - Favio Vazquez
-JaysonSunshine - Jayson Sunshine
-Liuchang0812 - Liu Chang
-Sephiroth-Lin - Sephiroth Lin
-dobashim - Masaru Dobashi
-ehnalis - Zoltan Zvara
-emres - Emre Sevinc
-gchen - Guancheng Chen
-haiyangsea - Haiyang Sea
-hlin09 - Hao Lin
-hqzizania - Qian Huang
 jeanlyn - Jean Lyn
 jerluc - Jeremy A. Lucas
+jerryshao - Saisai Shao
+jiangxb1987 - Jiang Xingbo
+jinxing64 - Jin Xing
+jisookim0513 - Jisoo Kim
+jkbradley - Joseph Bradley
+joelgenter - Joel Genter
+josepablocam - Jose Cambronero
 jrabary - Jaonary Rabarisoa
 judynash - Judy Nash
+junyangq - Junyang Qian
+kai-zeng - Kai Zeng
 kaka1992 - Chen Song
+ketank-new - Ketan Kunde
+krishnakalyan3 - Krishna Kalyan
 ksonj - Kalle Jepsen
+kul - Kuldeep
 kuromatsu-nobuyuki - Nobuyuki Kuromatsu
+laskfla - Keith Sun
 lazyman500 - Dong Xu
+lcqzte10192193 - Chaoqun Li
 leahmcguire - Leah McGuire
-mbittmann - Mark Bittmann
-mbonaci - Marko Bonaci
-meawoppl - Matthew Goodman
-nyaapa - Arsenii Krasikov
-phatak-dev - Madhukara Phatak
-prabeesh - Prabeesh K
-rakeshchalasani - Rakesh Chalasani
-rekhajoshm - Rekha Joshi
-sisihj - June He
-szheng79 - Shuai Zheng
-texasmichelle - Michelle Casbon
-vinodkc - Vinod KC
-yongtang - Yong Tang
-ypcat - Pei-Lun Lee
-zhichao-li - Zhichao Li
-zzcclp - Zhichao Zhang
-979969786 - Yuming Wang
-Rosstin - Rosstin Murphy
-ameyc - Amey Chaugule
-animeshbaranawal - Animesh Baranawal
-cafreeman - Chris Freeman
+leanken - Leanken Lin
 lee19 - Lee
-lockwobr - Brian Lockwood
-navis - Navis Ryu
-pparkkin - Paavo Parkkinen
-HyukjinKwon - Hyukjin Kwon
-JDrit - Joseph Batchik
-JuhongPark - Juhong Park
-KaiXinXiaoLei - KaiXinXIaoLei
-NamelessAnalyst - NamelessAnalyst
-alyaxey - Alex Slusarenko
-baishuo - Shuo Bai
-fe2s - Oleksiy Dyagilev
-felixcheung - Felix Cheung
-feynmanliang - Feynman Liang
-josepablocam - Jose Cambronero
-kai-zeng - Kai Zeng
-mosessky - mosessky
-msannell - Michael Sannella
-nishkamravi2 - Nishkam Ravi
-noel-smith - Noel Smith
-petz2000 - Patrick Baier
-qiansl127 - Shilei Qian
-rahulpalamuttam - Rahul Palamuttam
-rowan000 - Rowan Chattaway
-sarutak - Kousuke Saruta
-sethah - Seth Hendrickson
-small-wang - Wang Wei
-stanzhai - Stan Zhai
-tien-dungle - Tien-Dung Le
-xuchenCN - Xu Chen
-zhangjiajin - Zhang JiaJin
-ClassNotFoundExp - Fu Xing
-KevinGrealish - Kevin Grealish
-MasterDDT - Mitesh Patel
-VinceShieh - Vincent Xie
-WeichenXu123 - Weichen Xu
-Yunni - Yun Ni
-actuaryzhang - Wayne Zhang
-alicegugu - Gu Huiqin Alice
-anabranch - Bill Chambers
-ashangit - Nicolas Fraison
-avulanov - Alexander Ulanov
-biglobster - Liang Ke
-cenyuhai - Yuhai Cen
-codlife - Jianfei Wang
-david-weiluo-ren - Weiluo (David) Ren
-dding3 - Ding Ding
-fidato13 - Tarun Kumar
-frreiss - Fred Reiss
-gatorsmile - Xiao Li
-hayashidac - Chie Hayashida
-invkrh - Hao Ren
-jagadeesanas2 - Jagadeesan A S
-jiangxb1987 - Jiang Xingbo
-jisookim0513 - Jisoo Kim
-junyangq - Junyang Qian
-krishnakalyan3 - Krishna Kalyan
-linbojin - Linbo Jin
-mpjlu - Peng Meng
-neggert - Nic Eggert
-petermaxlee - Peter Lee
-phalodi - Sandeep Purohit
-pkch - pkch
-priyankagargnitk - Priyanka Garg
-sharkdtu - Xiaogang Tu
-shenh062326 - Shen Hong
-aokolnychyi - Anton Okolnychyi
+leoluan2009 - Xuedong Luan
+liangxs - Xuesen Liang
+lianhuiwang - Lianhui Wang
+lidinghao - Li Hao
+ligangty - Gang Li
 linbojin - Linbo Jin
-lw-lin - Liwei Lin
-10110346 - Xian Liu
-Achuth17 - Achuth Narayan Rajagopal
-Adamyuanyuan - Adam Wang
-DylanGuedes - Dylan Guedes
-JiahuiJiang - Jiahui Jiang
-KevinZwx - Kevin Zhang
-LantaoJin - Lantao Jin
-Lemonjing - Rann Tao
-LucaCanali - Luca Canali
-XD-DENG - Xiaodong Deng
-aai95 - Aleksei Izmalkin
-akonopko - Alexander Konopko
-ankuriitg - Ankur Gupta
-arucard21 - Riaas Mokiem
-attilapiros - Attila Zsolt Piros
-bravo-zhang - Bravo Zhang
-caneGuy - Kang Zhou
-chaoslawful - Xiaozhe Wang
-cluo512 - Chuan Luo
-codeatri - Neha Patil
-crafty-coder - Carlos Pena
-debugger87 - Chaozhong Yang
-e-dorigatti - Emilio Dorigatti
-eric-maynard - Eric Maynard
-felixalbani - Felix Albani
-fjh100456 - Jinhua Fu
-guoxiaolongzte - Xiaolong Guo
-heary-cao - Xuewen Cao
-huangweizhe123 - Weizhe Huang
-ivoson - Tengfei Huang
-jinxing64 - Jin Xing
+linehrr - Ryne Yang
+linzebing - Zebing Lin
+lipzhu - Lipeng Zhu
+lirui-intel - Rui Li
 liu-zhaokun - Zhaokun Liu
+liucht-inspur - liucht-inspur
+liupc - Pengcheng Liu
 liutang123 - Lijia Liu
+liwensun - Liwen Sun
+lockwobr - Brian Lockwood
+luluorta - Lu Lu
+luogankun - Gankun Luo
+lw-lin - Liwei Lin
+maji2014 - Derek Ma
+manuzhang - Manu Zhang
+mareksimunek - Marek Simunek
 maropu - Takeshi Yamamuro
+marsishandsome - Liangliang Gu
 maryannxue - Maryann Xue
+masa3141 - Masahiro Kazama
+mbittmann - Mark Bittmann
+mbonaci - Marko Bonaci
+mccheah - Matthew Cheah
 mcteo - Thomas Dunne
+mdianjun - Dianjun Ma
+meawoppl - Matthew Goodman
+medale - Markus Dale
+mengxr - Xiangrui Meng
+merrily01 - Ruilei Ma
+mingyukim - Mingyu Kim
 mn-mikke - Marek Novotny
+mob-ai - mob-ai
+mosessky - mosessky
+mpjlu - Peng Meng
+msannell - Michael Sannella
+mu5358271 - Shuheng Dai
+mwlon - Martin Loncaric
 myroslavlisniak - Myroslav Lisniak
+nandorKollar - Nandor Kollar
+nartz - Nathan Artz
+navis - Navis Ryu
+neggert - Nic Eggert
+nemccarthy - Nathan McCarthy
+nishkamravi2 - Nishkam Ravi
+noel-smith - Noel Smith
+nooberfsh - nooberfsh
 npoggi - Nicolas Poggi
+nxwhite-str - Nate Crosswhite
+nyaapa - Arsenii Krasikov
+odedz - Oded Zimerman
+oleg-smith - Oleg Kuznetsov
+ozancicek - Ozan Cicekci
+pengbo - Peng Bo
+petermaxlee - Peter Lee
+petz2000 - Patrick Baier
 pgandhi999 - Parth Gandhi
+phalodi - Sandeep Purohit
+phatak-dev - Madhukara Phatak
+pkch - pkch
+planga82 - Pablo Langa Blanco
+pparkkin - Paavo Parkkinen
+prabeesh - Prabeesh K
+praneetsharma - Praneet Sharma
+priyankagargnitk - Priyanka Garg
+ptkool - Michael Styles
+qb-tarushg - Tarush Grover
+qiansl127 - Shilei Qian
+rahulpalamuttam - Rahul Palamuttam
+rakeshchalasani - Rakesh Chalasani
+ravipesala - Ravindra Pesala
+redsanket - Sanket Reddy
+redsk - Nicola Bova
+rekhajoshm - Rekha Joshi
 rimolive - Ricardo Martinelli De Oliveira
+roland1982 - Roland Pogonyi
+rongma1997 - Rong Ma
+rowan000 - Rowan Chattaway
+roxchkplusony - Victor Tso
+rrusso2007 - Rob Russo
 sadhen - Darcy Shen
+samsetegne - Samuel L. Setegne
 sandeep-katta - Sandeep Katta
+sangramga - Sangram Gaikwad
+sarthfrey - Sarth Frey
+sarutak - Kousuke Saruta
+scwf - Wang Fei
 seancxmao - Chenxiao Mao
+seayi - Xiaohua Yi
+seayoun - Haiyang Yu
 sel - Steve Larkin
+sethah - Seth Hendrickson
+sev7e0 - Jiaqi Li
+shahidki31 - Shahid K I
+sharangk - Sharanabasappa G Keriwaddi
+sharkdtu - Xiaogang Tu
+sheepstop - Ting Yang
+shenh062326 - Shen Hong
 shimamoto - Takako Shimamoto
+shimingfei - Shiming Fei
+shivsood - Shiv Prashant Sood
 shivusondur - Shivakumar Sondur
+sigmoidanalytics - Mayur Rustagi
+sisihj - June He
+sitegui - Guilherme Souza
 skonto - Stavros Kontopoulos
+slamke - Sun Ke
+small-wang - Wang Wei
+southernriver - Liang Chen
+squito - Imran Rashid
+stanzhai - Stan Zhai
+stczwd - Jackey Lee
+sujith71955 - Sujith Chacko
+surq - Surong Quan
+suxingfate - Xinglong Wang
+suyanNone - Su Yan
+szheng79 - Shuai Zheng
+tanelk - Tanel Kiis
+tedyu - Ted Yu
+teeyog - Yong Tian
+texasmichelle - Michelle Casbon
+tianyi - Yi Tian
+tien-dungle - Tien-Dung Le
+tigerquoll - Dale Richardson
+tinhto-000 - Tin Hang To
+tools4origins - tools4origins
+triplesheep - triplesheep
 trystanleftwich - Trystan Leftwich
+turboFei - Fei Wang
 ueshin - Takuya Ueshin
+ulysses-you - Xiduo You
+uncleGen - Uncle Gen
+uzadude - Ohad Raviv
 uzmijnlm - Weizhe Huang
+vinodkc - Vinod KC
+viper-kun - Xu Kun
+wackxu - Shiwei Xu
+wangjiaochun - Jiaochun Wang
+wangshisan - wangshisan
+wangxiaojing - Xiaojing Wang
+watermen - Yadong Qi
+weixiuli - XiuLi Wei
+wenfang6 - wenfang6
+wenxuanguan - wenxuanguan
+williamhyun - William Hyun
+windpiger - Song Jun
+witgo - Guoqiang Li
+woudygao - Woudy Gao
+x1- - Yuri Saito
+xianyinxin - Xianyin Xin
+xinyunh - Xinyun Huang
 xuanyuanking - Yuanjian Li
 xubo245 - Bo Xu
+xuchenCN - Xu Chen
 xueyumusic - Xue Yu
 yanlin-Lynn - Yanlin Wang
+yongtang - Yong Tang
+ypcat - Pei-Lun Lee
 yucai - Yucai Yu
+yunzoud - Yun Zou
+zapletal-martin - Martin Zapletal
+zero323 - Maciej Szymkiewicz
+zhangjiajin - Zhang JiaJin
 zhengruifeng - Ruifeng Zheng
+zhichao-li - Zhichao Li
+zjf2012 - Jiafu Zhang
+zsxwing - Shixiong Zhu
 zuotingbing - Tingbing Zuo
+zuxqoj - Shekhar Bansal
+zzcclp - Zhichao Zhang
diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh
index 655b079ac28bf..f406be9548c31 100755
--- a/dev/create-release/release-build.sh
+++ b/dev/create-release/release-build.sh
@@ -97,6 +97,7 @@ git clone "$ASF_REPO"
 cd spark
 git checkout $GIT_REF
 git_hash=`git rev-parse --short HEAD`
+export GIT_HASH=$git_hash
 echo "Checked out Spark git hash $git_hash"
 
 if [ -z "$SPARK_VERSION" ]; then
@@ -114,16 +115,10 @@ if [[ $SPARK_VERSION > "2.3" ]]; then
   BASE_PROFILES="$BASE_PROFILES -Pkubernetes"
 fi
 
-# TODO: revisit for Scala 2.13
-
-PUBLISH_SCALA_2_11=1
-SCALA_2_11_PROFILES="-Pscala-2.11"
-if [[ $SPARK_VERSION > "2.3" ]]; then
-  if [[ $SPARK_VERSION < "3.0." ]]; then
-    SCALA_2_11_PROFILES="-Pkafka-0-8 -Pflume $SCALA_2_11_PROFILES"
-  else
-    PUBLISH_SCALA_2_11=0
-  fi
+PUBLISH_SCALA_2_13=1
+SCALA_2_13_PROFILES="-Pscala-2.13"
+if [[ $SPARK_VERSION < "3.2" ]]; then
+  PUBLISH_SCALA_2_13=0
 fi
 
 PUBLISH_SCALA_2_12=0
@@ -164,9 +159,11 @@ fi
 DEST_DIR_NAME="$SPARK_PACKAGE_VERSION"
 
 git clean -d -f -x
-rm .gitignore
+rm -f .gitignore
 cd ..
 
+export MAVEN_OPTS="-Xmx12g"
+
 if [[ "$1" == "package" ]]; then
   # Source and binary tarballs
   echo "Packaging release source tarballs"
@@ -174,20 +171,17 @@ if [[ "$1" == "package" ]]; then
 
   # For source release in v2.4+, exclude copy of binary license/notice
   if [[ $SPARK_VERSION > "2.4" ]]; then
-    rm spark-$SPARK_VERSION/LICENSE-binary
-    rm spark-$SPARK_VERSION/NOTICE-binary
-    rm -r spark-$SPARK_VERSION/licenses-binary
+    rm -f spark-$SPARK_VERSION/LICENSE-binary
+    rm -f spark-$SPARK_VERSION/NOTICE-binary
+    rm -rf spark-$SPARK_VERSION/licenses-binary
   fi
 
   tar cvzf spark-$SPARK_VERSION.tgz --exclude spark-$SPARK_VERSION/.git spark-$SPARK_VERSION
   echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --armour --output spark-$SPARK_VERSION.tgz.asc \
     --detach-sig spark-$SPARK_VERSION.tgz
-  echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --print-md \
-    SHA512 spark-$SPARK_VERSION.tgz > spark-$SPARK_VERSION.tgz.sha512
+  shasum -a 512 spark-$SPARK_VERSION.tgz > spark-$SPARK_VERSION.tgz.sha512
   rm -rf spark-$SPARK_VERSION
 
-  ZINC_PORT=3035
-
   # Updated for each binary build
   make_binary_release() {
     NAME=$1
@@ -205,17 +199,12 @@ if [[ "$1" == "package" ]]; then
       R_FLAG="--r"
     fi
 
-    # We increment the Zinc port each time to avoid OOM's and other craziness if multiple builds
-    # share the same Zinc server.
-    ZINC_PORT=$((ZINC_PORT + 1))
-
     echo "Building binary dist $NAME"
     cp -r spark spark-$SPARK_VERSION-bin-$NAME
     cd spark-$SPARK_VERSION-bin-$NAME
 
     ./dev/change-scala-version.sh $SCALA_VERSION
 
-    export ZINC_PORT=$ZINC_PORT
     echo "Creating distribution: $NAME ($FLAGS)"
 
     # Write out the VERSION to PySpark version info we rewrite the - into a . and SNAPSHOT
@@ -228,8 +217,7 @@ if [[ "$1" == "package" ]]; then
 
     echo "Creating distribution"
     ./dev/make-distribution.sh --name $NAME --mvn $MVN_HOME/bin/mvn --tgz \
-      $PIP_FLAG $R_FLAG $FLAGS \
-      -DzincPort=$ZINC_PORT 2>&1 >  ../binary-release-$NAME.log
+      $PIP_FLAG $R_FLAG $FLAGS 2>&1 >  ../binary-release-$NAME.log
     cd ..
 
     if [[ -n $R_FLAG ]]; then
@@ -275,26 +263,28 @@ if [[ "$1" == "package" ]]; then
   # In dry run mode, only build the first one. The keys in BINARY_PKGS_ARGS are used as the
   # list of packages to be built, so it's ok for things to be missing in BINARY_PKGS_EXTRA.
 
+  # NOTE: Don't forget to update the valid combinations of distributions at
+  #   'python/pyspark/install.py' and 'python/docs/source/getting_started/install.rst'
+  #   if you're changing them.
   declare -A BINARY_PKGS_ARGS
-  BINARY_PKGS_ARGS["hadoop2.7"]="-Phadoop-2.7 $HIVE_PROFILES"
+  BINARY_PKGS_ARGS["hadoop3.2"]="-Phadoop-3.2 $HIVE_PROFILES"
   if ! is_dry_run; then
     BINARY_PKGS_ARGS["without-hadoop"]="-Phadoop-provided"
     if [[ $SPARK_VERSION < "3.0." ]]; then
       BINARY_PKGS_ARGS["hadoop2.6"]="-Phadoop-2.6 $HIVE_PROFILES"
     else
-      BINARY_PKGS_ARGS["hadoop2.7-hive1.2"]="-Phadoop-2.7 -Phive-1.2 $HIVE_PROFILES"
-      BINARY_PKGS_ARGS["hadoop3.2"]="-Phadoop-3.2 $HIVE_PROFILES"
+      BINARY_PKGS_ARGS["hadoop2.7"]="-Phadoop-2.7 $HIVE_PROFILES"
     fi
   fi
 
   declare -A BINARY_PKGS_EXTRA
-  BINARY_PKGS_EXTRA["hadoop2.7"]="withpip,withr"
+  BINARY_PKGS_EXTRA["hadoop3.2"]="withpip,withr"
 
-  if [[ $PUBLISH_SCALA_2_11 = 1 ]]; then
-    key="without-hadoop-scala-2.11"
-    args="-Phadoop-provided"
+  if [[ $PUBLISH_SCALA_2_13 = 1 ]]; then
+    key="hadoop3.2-scala2.13"
+    args="-Phadoop-3.2 $HIVE_PROFILES"
     extra=""
-    if ! make_binary_release "$key" "$SCALA_2_11_PROFILES $args" "$extra" "2.11"; then
+    if ! make_binary_release "$key" "$SCALA_2_13_PROFILES $args" "$extra" "2.13"; then
       error "Failed to build $key package. Check logs for details."
     fi
   fi
@@ -338,7 +328,13 @@ if [[ "$1" == "docs" ]]; then
   echo "Building Spark docs"
   cd docs
   # TODO: Make configurable to add this: PRODUCTION=1
-  PRODUCTION=1 RELEASE_VERSION="$SPARK_VERSION" jekyll build
+  if [ ! -f "Gemfile" ]; then
+    cp "$SELF/Gemfile" .
+    cp "$SELF/Gemfile.lock" .
+    cp -r "$SELF/.bundle" .
+  fi
+  bundle install
+  PRODUCTION=1 RELEASE_VERSION="$SPARK_VERSION" bundle exec jekyll build
   cd ..
   cd ..
 
@@ -379,10 +375,12 @@ if [[ "$1" == "publish-snapshot" ]]; then
   echo "<password>$ASF_PASSWORD</password>" >> $tmp_settings
   echo "</server></servers></settings>" >> $tmp_settings
 
-  # Generate random point for Zinc
-  export ZINC_PORT=$(python -S -c "import random; print random.randrange(3030,4030)")
+  $MVN --settings $tmp_settings -DskipTests $SCALA_2_12_PROFILES $PUBLISH_PROFILES clean deploy
 
-  $MVN -DzincPort=$ZINC_PORT --settings $tmp_settings -DskipTests $SCALA_2_12_PROFILES $PUBLISH_PROFILES deploy
+  if [[ $PUBLISH_SCALA_2_13 = 1 ]]; then
+    ./dev/change-scala-version.sh 2.13
+    $MVN --settings $tmp_settings -DskipTests $SCALA_2_13_PROFILES $PUBLISH_PROFILES clean deploy
+  fi
 
   rm $tmp_settings
   cd ..
@@ -411,21 +409,16 @@ if [[ "$1" == "publish-release" ]]; then
 
   tmp_repo=$(mktemp -d spark-repo-XXXXX)
 
-  # Generate random point for Zinc
-  export ZINC_PORT=$(python -S -c "import random; print random.randrange(3030,4030)")
-
-  # TODO: revisit for Scala 2.13 support
-
-  if [[ $PUBLISH_SCALA_2_11 = 1 ]]; then
-    ./dev/change-scala-version.sh 2.11
-    $MVN -DzincPort=$ZINC_PORT -Dmaven.repo.local=$tmp_repo -DskipTests \
-      $SCALA_2_11_PROFILES $PUBLISH_PROFILES clean install
+  if [[ $PUBLISH_SCALA_2_13 = 1 ]]; then
+    ./dev/change-scala-version.sh 2.13
+    $MVN -Dmaven.repo.local=$tmp_repo -DskipTests \
+      $SCALA_2_13_PROFILES $PUBLISH_PROFILES clean install
   fi
 
   if [[ $PUBLISH_SCALA_2_12 = 1 ]]; then
     ./dev/change-scala-version.sh 2.12
-    $MVN -DzincPort=$((ZINC_PORT + 2)) -Dmaven.repo.local=$tmp_repo -DskipTests \
-      $SCALA_2_11_PROFILES $PUBLISH_PROFILES clean install
+    $MVN -Dmaven.repo.local=$tmp_repo -DskipTests \
+      $SCALA_2_12_PROFILES $PUBLISH_PROFILES clean install
   fi
 
   pushd $tmp_repo/org/apache/spark
@@ -451,7 +444,7 @@ if [[ "$1" == "publish-release" ]]; then
 
   if ! is_dry_run; then
     nexus_upload=$NEXUS_ROOT/deployByRepositoryId/$staged_repo_id
-    echo "Uplading files to $nexus_upload"
+    echo "Uploading files to $nexus_upload"
     for file in $(find . -type f)
     do
       # strip leading ./
diff --git a/dev/create-release/release-tag.sh b/dev/create-release/release-tag.sh
index 39856a9955955..a9a518f9e10d7 100755
--- a/dev/create-release/release-tag.sh
+++ b/dev/create-release/release-tag.sh
@@ -64,8 +64,12 @@ init_maven_sbt
 
 ASF_SPARK_REPO="gitbox.apache.org/repos/asf/spark.git"
 
+function uriencode { jq -nSRr --arg v "$1" '$v|@uri'; }
+
+declare -r ENCODED_ASF_PASSWORD=$(uriencode "$ASF_PASSWORD")
+
 rm -rf spark
-git clone "https://$ASF_USERNAME:$ASF_PASSWORD@$ASF_SPARK_REPO" -b $GIT_BRANCH
+git clone "https://$ASF_USERNAME:$ENCODED_ASF_PASSWORD@$ASF_SPARK_REPO" -b $GIT_BRANCH
 cd spark
 
 git config user.name "$GIT_NAME"
diff --git a/dev/create-release/release-util.sh b/dev/create-release/release-util.sh
index af9ed201b3b47..7961eed850891 100755
--- a/dev/create-release/release-util.sh
+++ b/dev/create-release/release-util.sh
@@ -228,7 +228,7 @@ function init_maven_sbt {
   if [[ $JAVA_VERSION < "1.8." ]]; then
     # Needed for maven central when using Java 7.
     SBT_OPTS="-Dhttps.protocols=TLSv1.1,TLSv1.2"
-    MVN_EXTRA_OPTS="-Dhttps.protocols=TLSv1.1,TLSv1.2"
+    MVN_EXTRA_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g -Dhttps.protocols=TLSv1.1,TLSv1.2"
     MVN="$MVN $MVN_EXTRA_OPTS"
   fi
   export MVN MVN_EXTRA_OPTS SBT_OPTS
diff --git a/dev/create-release/releaseutils.py b/dev/create-release/releaseutils.py
index a5a26ae8f5354..94e255bd440b8 100755
--- a/dev/create-release/releaseutils.py
+++ b/dev/create-release/releaseutils.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
@@ -23,7 +23,7 @@
 from subprocess import Popen, PIPE
 
 try:
-    from jira.client import JIRA
+    from jira.client import JIRA  # noqa: F401
     # Old versions have JIRAError in exceptions package, new (0.5+) in utils.
     try:
         from jira.exceptions import JIRAError
@@ -31,26 +31,17 @@
         from jira.utils import JIRAError
 except ImportError:
     print("This tool requires the jira-python library")
-    print("Install using 'sudo pip install jira'")
+    print("Install using 'sudo pip3 install jira'")
     sys.exit(-1)
 
 try:
-    from github import Github
+    from github import Github  # noqa: F401
     from github import GithubException
 except ImportError:
     print("This tool requires the PyGithub library")
     print("Install using 'sudo pip install PyGithub'")
     sys.exit(-1)
 
-try:
-    import unidecode
-except ImportError:
-    print("This tool requires the unidecode library to decode obscure github usernames")
-    print("Install using 'sudo pip install unidecode'")
-    sys.exit(-1)
-
-if sys.version < '3':
-    input = raw_input  # noqa
 
 # Contributors list file name
 contributors_file_name = "contributors.txt"
@@ -66,11 +57,11 @@ def yesOrNoPrompt(msg):
 
 # Utility functions run git commands (written with Git 1.8.5)
 def run_cmd(cmd):
-    return Popen(cmd, stdout=PIPE).communicate()[0]
+    return Popen(cmd, stdout=PIPE).communicate()[0].decode("utf8")
 
 
 def run_cmd_error(cmd):
-    return Popen(cmd, stdout=PIPE, stderr=PIPE).communicate()[1]
+    return Popen(cmd, stdout=PIPE, stderr=PIPE).communicate()[1].decode("utf8")
 
 
 def get_date(commit_hash):
@@ -112,7 +103,7 @@ def __str__(self):
 # Under the hood, this runs a `git log` on that tag and parses the fields
 # from the command output to construct a list of Commit objects. Note that
 # because certain fields reside in the commit description and cannot be parsed
-# through the Github API itself, we need to do some intelligent regex parsing
+# through the GitHub API itself, we need to do some intelligent regex parsing
 # to extract those fields.
 #
 # This is written using Git 1.8.5.
@@ -142,7 +133,7 @@ def get_commits(tag):
             sys.exit("Unexpected format in commit: %s" % commit_digest)
         [_hash, author, title] = commit_digest.split(field_end_marker)
         # The PR number and github username is in the commit message
-        # itself and cannot be accessed through any Github API
+        # itself and cannot be accessed through any GitHub API
         pr_number = None
         match = re.search("Closes #([0-9]+) from ([^/\\s]+)/", commit_body)
         if match:
@@ -151,12 +142,7 @@ def get_commits(tag):
             # username so we can translate it properly later
             if not is_valid_author(author):
                 author = github_username
-        # Guard against special characters
-        try:               # Python 2
-            author = unicode(author, "UTF-8")
-        except NameError:  # Python 3
-            author = str(author)
-        author = unidecode.unidecode(author).strip()
+        author = author.strip()
         commit = Commit(_hash, author, title, pr_number)
         commits.append(commit)
     return commits
@@ -257,7 +243,7 @@ def nice_join(str_list):
         return ", ".join(str_list[:-1]) + ", and " + str_list[-1]
 
 
-# Return the full name of the specified user on Github
+# Return the full name of the specified user on GitHub
 # If the user doesn't exist, return None
 def get_github_name(author, github_client):
     if github_client:
diff --git a/dev/create-release/spark-rm/Dockerfile b/dev/create-release/spark-rm/Dockerfile
index 540dc90f42817..2751f3a1800bd 100644
--- a/dev/create-release/spark-rm/Dockerfile
+++ b/dev/create-release/spark-rm/Dockerfile
@@ -15,16 +15,20 @@
 # limitations under the License.
 #
 
-# Image for building Spark releases. Based on Ubuntu 18.04.
+# Image for building Spark releases. Based on Ubuntu 20.04.
 #
 # Includes:
 # * Java 8
 # * Ivy
-# * Python (2.7.15/3.6.7)
-# * R-base/R-base-dev (3.6.1)
-# * Ruby 2.3 build utilities
+# * Python (3.8.5)
+# * R-base/R-base-dev (4.0.3)
+# * Ruby (2.7.0)
+#
+# You can test it as below:
+#   cd dev/create-release/spark-rm
+#   docker build -t spark-rm --build-arg UID=$UID .
 
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 
 # For apt to be noninteractive
 ENV DEBIAN_FRONTEND noninteractive
@@ -33,8 +37,11 @@ ENV DEBCONF_NONINTERACTIVE_SEEN true
 # These arguments are just for reuse and not really meant to be customized.
 ARG APT_INSTALL="apt-get install --no-install-recommends -y"
 
-ARG PIP_PKGS="sphinx==2.3.1 mkdocs==1.0.4 numpy==1.18.1"
-ARG GEM_PKGS="jekyll:4.0.0 jekyll-redirect-from:0.16.0 rouge:3.15.0"
+# TODO(SPARK-32407): Sphinx 3.1+ does not correctly index nested classes.
+#   See also https://github.com/sphinx-doc/sphinx/issues/7551.
+#   We should use the latest Sphinx version once this is fixed.
+ARG PIP_PKGS="sphinx==3.0.4 mkdocs==1.1.2 numpy==1.19.4 pydata_sphinx_theme==0.4.1 ipython==7.19.0 nbsphinx==0.8.0 numpydoc==1.1.0"
+ARG GEM_PKGS="bundler:2.2.9"
 
 # Install extra needed repos and refresh.
 # - CRAN repo
@@ -43,7 +50,7 @@ ARG GEM_PKGS="jekyll:4.0.0 jekyll-redirect-from:0.16.0 rouge:3.15.0"
 # This is all in a single "RUN" command so that if anything changes, "apt update" is run to fetch
 # the most current package versions (instead of potentially using old versions cached by docker).
 RUN apt-get clean && apt-get update && $APT_INSTALL gnupg ca-certificates && \
-  echo 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' >> /etc/apt/sources.list && \
+  echo 'deb https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/' >> /etc/apt/sources.list && \
   gpg --keyserver keyserver.ubuntu.com --recv-key E298A3A825C0D65DFD57CBB651716619E084DAB9 && \
   gpg -a --export E084DAB9 | apt-key add - && \
   apt-get clean && \
@@ -51,7 +58,6 @@ RUN apt-get clean && apt-get update && $APT_INSTALL gnupg ca-certificates && \
   apt-get clean && \
   apt-get update && \
   $APT_INSTALL software-properties-common && \
-  apt-add-repository -y ppa:brightbox/ruby-ng && \
   apt-get update && \
   # Install openjdk 8.
   $APT_INSTALL openjdk-8-jdk && \
@@ -59,23 +65,23 @@ RUN apt-get clean && apt-get update && $APT_INSTALL gnupg ca-certificates && \
   # Install build / source control tools
   $APT_INSTALL curl wget git maven ivy subversion make gcc lsof libffi-dev \
     pandoc pandoc-citeproc libssl-dev libcurl4-openssl-dev libxml2-dev && \
-  curl -sL https://deb.nodesource.com/setup_11.x | bash && \
+  curl -sL https://deb.nodesource.com/setup_12.x | bash && \
   $APT_INSTALL nodejs && \
   # Install needed python packages. Use pip for installing packages (for consistency).
-  $APT_INSTALL libpython3-dev python3-pip python3-setuptools && \
-  # Change default python version to python3.
-  update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1 && \
-  update-alternatives --install /usr/bin/python python /usr/bin/python3.6 2 && \
-  update-alternatives --set python /usr/bin/python3.6 && \
+  $APT_INSTALL python3-pip python3-setuptools && \
+  # qpdf is required for CRAN checks to pass.
+  $APT_INSTALL qpdf jq && \
   pip3 install $PIP_PKGS && \
   # Install R packages and dependencies used when building.
   # R depends on pandoc*, libssl (which are installed above).
+  # Note that PySpark doc generation also needs pandoc due to nbsphinx
   $APT_INSTALL r-base r-base-dev && \
+  $APT_INSTALL libcurl4-openssl-dev libgit2-dev libssl-dev libxml2-dev && \
   $APT_INSTALL texlive-latex-base texlive texlive-fonts-extra texinfo qpdf && \
   Rscript -e "install.packages(c('curl', 'xml2', 'httr', 'devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2', 'e1071', 'survival'), repos='https://cloud.r-project.org/')" && \
   Rscript -e "devtools::install_github('jimhester/lintr')" && \
   # Install tools needed to build the documentation.
-  $APT_INSTALL ruby2.5 ruby2.5-dev && \
+  $APT_INSTALL ruby2.7 ruby2.7-dev && \
   gem install --no-document $GEM_PKGS
 
 WORKDIR /opt/spark-rm/output
diff --git a/dev/create-release/translate-contributors.py b/dev/create-release/translate-contributors.py
index be30e6ad30b24..6af975916ec49 100755
--- a/dev/create-release/translate-contributors.py
+++ b/dev/create-release/translate-contributors.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
@@ -17,7 +17,7 @@
 
 # This script translates invalid authors in the contributors list generated
 # by generate-contributors.py. When the script encounters an author name that
-# is considered invalid, it searches Github and JIRA in an attempt to search
+# is considered invalid, it searches GitHub and JIRA in an attempt to search
 # for replacements. This tool runs in two modes:
 #
 # (1) Interactive mode: For each invalid author name, this script presents
@@ -31,17 +31,18 @@
 import os
 import sys
 
-from releaseutils import *
+from releaseutils import JIRA, JIRAError, get_jira_name, Github, get_github_name, \
+    contributors_file_name, is_valid_author, capitalize_author, yesOrNoPrompt
 
 # You must set the following before use!
 JIRA_API_BASE = os.environ.get("JIRA_API_BASE", "https://issues.apache.org/jira")
 JIRA_USERNAME = os.environ.get("JIRA_USERNAME", None)
 JIRA_PASSWORD = os.environ.get("JIRA_PASSWORD", None)
-GITHUB_API_TOKEN = os.environ.get("GITHUB_API_TOKEN", None)
+GITHUB_OAUTH_KEY = os.environ.get("GITHUB_OAUTH_KEY", os.environ.get("GITHUB_API_TOKEN", None))
 if not JIRA_USERNAME or not JIRA_PASSWORD:
     sys.exit("Both JIRA_USERNAME and JIRA_PASSWORD must be set")
-if not GITHUB_API_TOKEN:
-    sys.exit("GITHUB_API_TOKEN must be set")
+if not GITHUB_OAUTH_KEY:
+    sys.exit("GITHUB_OAUTH_KEY must be set")
 
 # Write new contributors list to <old_file_name>.final
 if not os.path.isfile(contributors_file_name):
@@ -60,10 +61,10 @@
 if INTERACTIVE_MODE:
     print("Running in interactive mode. To disable this, provide the --non-interactive flag.")
 
-# Setup Github and JIRA clients
+# Setup GitHub and JIRA clients
 jira_options = {"server": JIRA_API_BASE}
 jira_client = JIRA(options=jira_options, basic_auth=(JIRA_USERNAME, JIRA_PASSWORD))
-github_client = Github(GITHUB_API_TOKEN)
+github_client = Github(GITHUB_OAUTH_KEY)
 
 # Load known author translations that are cached locally
 known_translations = {}
@@ -81,11 +82,11 @@
 
 # Generate candidates for the given author. This should only be called if the given author
 # name does not represent a full name as this operation is somewhat expensive. Under the
-# hood, it makes several calls to the Github and JIRA API servers to find the candidates.
+# hood, it makes several calls to the GitHub and JIRA API servers to find the candidates.
 #
 # This returns a list of (candidate name, source) 2-tuples. E.g.
 # [
-#   (NOT_FOUND, "No full name found for Github user andrewor14"),
+#   (NOT_FOUND, "No full name found for GitHub user andrewor14"),
 #   ("Andrew Or", "Full name of JIRA user andrewor14"),
 #   ("Andrew Orso", "Full name of SPARK-1444 assignee andrewor14"),
 #   ("Andrew Ordall", "Full name of SPARK-1663 assignee andrewor14"),
@@ -96,12 +97,12 @@
 
 def generate_candidates(author, issues):
     candidates = []
-    # First check for full name of Github user
+    # First check for full name of GitHub user
     github_name = get_github_name(author, github_client)
     if github_name:
-        candidates.append((github_name, "Full name of Github user %s" % author))
+        candidates.append((github_name, "Full name of GitHub user %s" % author))
     else:
-        candidates.append((NOT_FOUND, "No full name found for Github user %s" % author))
+        candidates.append((NOT_FOUND, "No full name found for GitHub user %s" % author))
     # Then do the same for JIRA user
     jira_name = get_jira_name(author, jira_client)
     if jira_name:
@@ -131,19 +132,12 @@ def generate_candidates(author, issues):
                     (NOT_FOUND, "No full name found for %s assignee %s" % (issue, user_name)))
         else:
             candidates.append((NOT_FOUND, "No assignee found for %s" % issue))
-    # Guard against special characters in candidate names
-    # Note that the candidate name may already be in unicode (JIRA returns this)
     for i, (candidate, source) in enumerate(candidates):
-        try:
-            candidate = unicode(candidate, "UTF-8")
-        except TypeError:
-            # already in unicode
-            pass
-        candidate = unidecode.unidecode(candidate).strip()
+        candidate = candidate.strip()
         candidates[i] = (candidate, source)
     return candidates
 
-# Translate each invalid author by searching for possible candidates from Github and JIRA
+# Translate each invalid author by searching for possible candidates from GitHub and JIRA
 # In interactive mode, this script presents the user with a list of choices and have the user
 # select from this list. Additionally, the user may also choose to enter a custom name.
 # In non-interactive mode, this script picks the first valid author name from the candidates
@@ -172,12 +166,12 @@ def generate_candidates(author, issues):
         issues = temp_author.split("/")[1:]
         candidates = generate_candidates(author, issues)
         # Print out potential replacement candidates along with the sources, e.g.
-        #   [X] No full name found for Github user andrewor14
+        #   [X] No full name found for GitHub user andrewor14
         #   [X] No assignee found for SPARK-1763
         #   [0] Andrew Or - Full name of JIRA user andrewor14
         #   [1] Andrew Orso - Full name of SPARK-1444 assignee andrewor14
         #   [2] Andrew Ordall - Full name of SPARK-1663 assignee andrewor14
-        #   [3] andrewor14 - Raw Github username
+        #   [3] andrewor14 - Raw GitHub username
         #   [4] Custom
         candidate_names = []
         bad_prompts = []  # Prompts that can't actually be selected; print these first.
@@ -199,15 +193,15 @@ def generate_candidates(author, issues):
             print(p)
         # In interactive mode, additionally provide "custom" option and await user response
         if INTERACTIVE_MODE:
-            print("    [%d] %s - Raw Github username" % (raw_index, author))
+            print("    [%d] %s - Raw GitHub username" % (raw_index, author))
             print("    [%d] Custom" % custom_index)
-            response = raw_input("    Your choice: ")
+            response = input("    Your choice: ")
             last_index = custom_index
             while not response.isdigit() or int(response) > last_index:
-                response = raw_input("    Please enter an integer between 0 and %d: " % last_index)
+                response = input("    Please enter an integer between 0 and %d: " % last_index)
             response = int(response)
             if response == custom_index:
-                new_author = raw_input("    Please type a custom name for this author: ")
+                new_author = input("    Please type a custom name for this author: ")
             elif response != raw_index:
                 new_author = candidate_names[response]
         # In non-interactive mode, just pick the first candidate
diff --git a/dev/create-release/vote.tmpl b/dev/create-release/vote.tmpl
index aa22a9c516535..b5e53345f27b1 100644
--- a/dev/create-release/vote.tmpl
+++ b/dev/create-release/vote.tmpl
@@ -37,8 +37,9 @@ an existing Spark workload and running on this release candidate, then
 reporting any regressions.
 
 If you're working in PySpark you can set up a virtual env and install
-the current RC and see if anything important breaks, in the Java/Scala
-you can add the staging repository to your projects resolvers and test
+the current RC via "pip install https://dist.apache.org/repos/dist/dev/spark/{tag}-bin/pyspark-{version}.tar.gz"
+and see if anything important breaks.
+In the Java/Scala, you can add the staging repository to your projects resolvers and test
 with the RC (make sure to clean up the artifact cache before/after so
 you don't end up building with a out of date RC going forward).
 
@@ -62,4 +63,4 @@ In order to make timely releases, we will typically not hold the
 release unless the bug in question is a regression from the previous
 release. That being said, if there is something which is a regression
 that has not been correctly targeted please ping me or a committer to
-help target the issue.
\ No newline at end of file
+help target the issue.
diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-1.2 b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
deleted file mode 100644
index b375629c37433..0000000000000
--- a/dev/deps/spark-deps-hadoop-2.7-hive-1.2
+++ /dev/null
@@ -1,210 +0,0 @@
-JLargeArrays/1.5//JLargeArrays-1.5.jar
-JTransforms/3.1//JTransforms-3.1.jar
-JavaEWAH/0.3.2//JavaEWAH-0.3.2.jar
-RoaringBitmap/0.7.45//RoaringBitmap-0.7.45.jar
-ST4/4.0.4//ST4-4.0.4.jar
-activation/1.1.1//activation-1.1.1.jar
-aircompressor/0.10//aircompressor-0.10.jar
-algebra_2.12/2.0.0-M2//algebra_2.12-2.0.0-M2.jar
-antlr-runtime/3.4//antlr-runtime-3.4.jar
-antlr/2.7.7//antlr-2.7.7.jar
-antlr4-runtime/4.7.1//antlr4-runtime-4.7.1.jar
-aopalliance-repackaged/2.6.1//aopalliance-repackaged-2.6.1.jar
-aopalliance/1.0//aopalliance-1.0.jar
-apache-log4j-extras/1.2.17//apache-log4j-extras-1.2.17.jar
-apacheds-i18n/2.0.0-M15//apacheds-i18n-2.0.0-M15.jar
-apacheds-kerberos-codec/2.0.0-M15//apacheds-kerberos-codec-2.0.0-M15.jar
-api-asn1-api/1.0.0-M20//api-asn1-api-1.0.0-M20.jar
-api-util/1.0.0-M20//api-util-1.0.0-M20.jar
-arpack_combined_all/0.1//arpack_combined_all-0.1.jar
-arrow-format/0.15.1//arrow-format-0.15.1.jar
-arrow-memory/0.15.1//arrow-memory-0.15.1.jar
-arrow-vector/0.15.1//arrow-vector-0.15.1.jar
-audience-annotations/0.5.0//audience-annotations-0.5.0.jar
-automaton/1.11-8//automaton-1.11-8.jar
-avro-ipc/1.8.2//avro-ipc-1.8.2.jar
-avro-mapred/1.8.2/hadoop2/avro-mapred-1.8.2-hadoop2.jar
-avro/1.8.2//avro-1.8.2.jar
-bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar
-breeze-macros_2.12/1.0//breeze-macros_2.12-1.0.jar
-breeze_2.12/1.0//breeze_2.12-1.0.jar
-cats-kernel_2.12/2.0.0-M4//cats-kernel_2.12-2.0.0-M4.jar
-chill-java/0.9.5//chill-java-0.9.5.jar
-chill_2.12/0.9.5//chill_2.12-0.9.5.jar
-commons-beanutils/1.9.4//commons-beanutils-1.9.4.jar
-commons-cli/1.2//commons-cli-1.2.jar
-commons-codec/1.10//commons-codec-1.10.jar
-commons-collections/3.2.2//commons-collections-3.2.2.jar
-commons-compiler/3.0.16//commons-compiler-3.0.16.jar
-commons-compress/1.8.1//commons-compress-1.8.1.jar
-commons-configuration/1.6//commons-configuration-1.6.jar
-commons-crypto/1.0.0//commons-crypto-1.0.0.jar
-commons-dbcp/1.4//commons-dbcp-1.4.jar
-commons-digester/1.8//commons-digester-1.8.jar
-commons-httpclient/3.1//commons-httpclient-3.1.jar
-commons-io/2.4//commons-io-2.4.jar
-commons-lang/2.6//commons-lang-2.6.jar
-commons-lang3/3.9//commons-lang3-3.9.jar
-commons-logging/1.1.3//commons-logging-1.1.3.jar
-commons-math3/3.4.1//commons-math3-3.4.1.jar
-commons-net/3.1//commons-net-3.1.jar
-commons-pool/1.5.4//commons-pool-1.5.4.jar
-commons-text/1.6//commons-text-1.6.jar
-compress-lzf/1.0.3//compress-lzf-1.0.3.jar
-core/1.1.2//core-1.1.2.jar
-curator-client/2.7.1//curator-client-2.7.1.jar
-curator-framework/2.7.1//curator-framework-2.7.1.jar
-curator-recipes/2.7.1//curator-recipes-2.7.1.jar
-datanucleus-api-jdo/3.2.6//datanucleus-api-jdo-3.2.6.jar
-datanucleus-core/3.2.10//datanucleus-core-3.2.10.jar
-datanucleus-rdbms/3.2.9//datanucleus-rdbms-3.2.9.jar
-derby/10.12.1.1//derby-10.12.1.1.jar
-flatbuffers-java/1.9.0//flatbuffers-java-1.9.0.jar
-generex/1.0.2//generex-1.0.2.jar
-gson/2.2.4//gson-2.2.4.jar
-guava/14.0.1//guava-14.0.1.jar
-guice-servlet/3.0//guice-servlet-3.0.jar
-guice/3.0//guice-3.0.jar
-hadoop-annotations/2.7.4//hadoop-annotations-2.7.4.jar
-hadoop-auth/2.7.4//hadoop-auth-2.7.4.jar
-hadoop-client/2.7.4//hadoop-client-2.7.4.jar
-hadoop-common/2.7.4//hadoop-common-2.7.4.jar
-hadoop-hdfs/2.7.4//hadoop-hdfs-2.7.4.jar
-hadoop-mapreduce-client-app/2.7.4//hadoop-mapreduce-client-app-2.7.4.jar
-hadoop-mapreduce-client-common/2.7.4//hadoop-mapreduce-client-common-2.7.4.jar
-hadoop-mapreduce-client-core/2.7.4//hadoop-mapreduce-client-core-2.7.4.jar
-hadoop-mapreduce-client-jobclient/2.7.4//hadoop-mapreduce-client-jobclient-2.7.4.jar
-hadoop-mapreduce-client-shuffle/2.7.4//hadoop-mapreduce-client-shuffle-2.7.4.jar
-hadoop-yarn-api/2.7.4//hadoop-yarn-api-2.7.4.jar
-hadoop-yarn-client/2.7.4//hadoop-yarn-client-2.7.4.jar
-hadoop-yarn-common/2.7.4//hadoop-yarn-common-2.7.4.jar
-hadoop-yarn-server-common/2.7.4//hadoop-yarn-server-common-2.7.4.jar
-hadoop-yarn-server-web-proxy/2.7.4//hadoop-yarn-server-web-proxy-2.7.4.jar
-hk2-api/2.6.1//hk2-api-2.6.1.jar
-hk2-locator/2.6.1//hk2-locator-2.6.1.jar
-hk2-utils/2.6.1//hk2-utils-2.6.1.jar
-htrace-core/3.1.0-incubating//htrace-core-3.1.0-incubating.jar
-httpclient/4.5.6//httpclient-4.5.6.jar
-httpcore/4.4.12//httpcore-4.4.12.jar
-istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar
-ivy/2.4.0//ivy-2.4.0.jar
-jackson-annotations/2.10.0//jackson-annotations-2.10.0.jar
-jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
-jackson-core/2.10.0//jackson-core-2.10.0.jar
-jackson-databind/2.10.0//jackson-databind-2.10.0.jar
-jackson-dataformat-yaml/2.10.0//jackson-dataformat-yaml-2.10.0.jar
-jackson-jaxrs/1.9.13//jackson-jaxrs-1.9.13.jar
-jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
-jackson-module-jaxb-annotations/2.10.0//jackson-module-jaxb-annotations-2.10.0.jar
-jackson-module-paranamer/2.10.0//jackson-module-paranamer-2.10.0.jar
-jackson-module-scala_2.12/2.10.0//jackson-module-scala_2.12-2.10.0.jar
-jackson-xc/1.9.13//jackson-xc-1.9.13.jar
-jakarta.activation-api/1.2.1//jakarta.activation-api-1.2.1.jar
-jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar
-jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar
-jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar
-jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar
-jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
-janino/3.0.16//janino-3.0.16.jar
-javassist/3.25.0-GA//javassist-3.25.0-GA.jar
-javax.inject/1//javax.inject-1.jar
-javax.servlet-api/3.1.0//javax.servlet-api-3.1.0.jar
-javolution/5.5.1//javolution-5.5.1.jar
-jaxb-api/2.2.2//jaxb-api-2.2.2.jar
-jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar
-jcl-over-slf4j/1.7.30//jcl-over-slf4j-1.7.30.jar
-jdo-api/3.0.1//jdo-api-3.0.1.jar
-jersey-client/2.30//jersey-client-2.30.jar
-jersey-common/2.30//jersey-common-2.30.jar
-jersey-container-servlet-core/2.30//jersey-container-servlet-core-2.30.jar
-jersey-container-servlet/2.30//jersey-container-servlet-2.30.jar
-jersey-hk2/2.30//jersey-hk2-2.30.jar
-jersey-media-jaxb/2.30//jersey-media-jaxb-2.30.jar
-jersey-server/2.30//jersey-server-2.30.jar
-jetty-sslengine/6.1.26//jetty-sslengine-6.1.26.jar
-jetty-util/6.1.26//jetty-util-6.1.26.jar
-jetty/6.1.26//jetty-6.1.26.jar
-jline/2.14.6//jline-2.14.6.jar
-joda-time/2.10.5//joda-time-2.10.5.jar
-jodd-core/3.5.2//jodd-core-3.5.2.jar
-jpam/1.1//jpam-1.1.jar
-json4s-ast_2.12/3.6.6//json4s-ast_2.12-3.6.6.jar
-json4s-core_2.12/3.6.6//json4s-core_2.12-3.6.6.jar
-json4s-jackson_2.12/3.6.6//json4s-jackson_2.12-3.6.6.jar
-json4s-scalap_2.12/3.6.6//json4s-scalap_2.12-3.6.6.jar
-jsp-api/2.1//jsp-api-2.1.jar
-jsr305/3.0.0//jsr305-3.0.0.jar
-jta/1.1//jta-1.1.jar
-jul-to-slf4j/1.7.30//jul-to-slf4j-1.7.30.jar
-kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar
-kubernetes-client/4.7.1//kubernetes-client-4.7.1.jar
-kubernetes-model-common/4.7.1//kubernetes-model-common-4.7.1.jar
-kubernetes-model/4.7.1//kubernetes-model-4.7.1.jar
-leveldbjni-all/1.8//leveldbjni-all-1.8.jar
-libfb303/0.9.3//libfb303-0.9.3.jar
-libthrift/0.12.0//libthrift-0.12.0.jar
-log4j/1.2.17//log4j-1.2.17.jar
-logging-interceptor/3.12.6//logging-interceptor-3.12.6.jar
-lz4-java/1.7.1//lz4-java-1.7.1.jar
-machinist_2.12/0.6.8//machinist_2.12-0.6.8.jar
-macro-compat_2.12/1.1.1//macro-compat_2.12-1.1.1.jar
-mesos/1.4.0/shaded-protobuf/mesos-1.4.0-shaded-protobuf.jar
-metrics-core/4.1.1//metrics-core-4.1.1.jar
-metrics-graphite/4.1.1//metrics-graphite-4.1.1.jar
-metrics-jmx/4.1.1//metrics-jmx-4.1.1.jar
-metrics-json/4.1.1//metrics-json-4.1.1.jar
-metrics-jvm/4.1.1//metrics-jvm-4.1.1.jar
-minlog/1.3.0//minlog-1.3.0.jar
-netty-all/4.1.47.Final//netty-all-4.1.47.Final.jar
-objenesis/2.5.1//objenesis-2.5.1.jar
-okhttp/3.12.6//okhttp-3.12.6.jar
-okio/1.15.0//okio-1.15.0.jar
-opencsv/2.3//opencsv-2.3.jar
-orc-core/1.5.10/nohive/orc-core-1.5.10-nohive.jar
-orc-mapreduce/1.5.10/nohive/orc-mapreduce-1.5.10-nohive.jar
-orc-shims/1.5.10//orc-shims-1.5.10.jar
-oro/2.0.8//oro-2.0.8.jar
-osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
-paranamer/2.8//paranamer-2.8.jar
-parquet-column/1.10.1//parquet-column-1.10.1.jar
-parquet-common/1.10.1//parquet-common-1.10.1.jar
-parquet-encoding/1.10.1//parquet-encoding-1.10.1.jar
-parquet-format/2.4.0//parquet-format-2.4.0.jar
-parquet-hadoop-bundle/1.6.0//parquet-hadoop-bundle-1.6.0.jar
-parquet-hadoop/1.10.1//parquet-hadoop-1.10.1.jar
-parquet-jackson/1.10.1//parquet-jackson-1.10.1.jar
-protobuf-java/2.5.0//protobuf-java-2.5.0.jar
-py4j/0.10.9//py4j-0.10.9.jar
-pyrolite/4.30//pyrolite-4.30.jar
-scala-collection-compat_2.12/2.1.1//scala-collection-compat_2.12-2.1.1.jar
-scala-compiler/2.12.10//scala-compiler-2.12.10.jar
-scala-library/2.12.10//scala-library-2.12.10.jar
-scala-parser-combinators_2.12/1.1.2//scala-parser-combinators_2.12-1.1.2.jar
-scala-reflect/2.12.10//scala-reflect-2.12.10.jar
-scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar
-shapeless_2.12/2.3.3//shapeless_2.12-2.3.3.jar
-shims/0.7.45//shims-0.7.45.jar
-slf4j-api/1.7.30//slf4j-api-1.7.30.jar
-slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
-snakeyaml/1.24//snakeyaml-1.24.jar
-snappy-java/1.1.7.5//snappy-java-1.1.7.5.jar
-snappy/0.2//snappy-0.2.jar
-spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar
-spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar
-spire-util_2.12/0.17.0-M1//spire-util_2.12-0.17.0-M1.jar
-spire_2.12/0.17.0-M1//spire_2.12-0.17.0-M1.jar
-stax-api/1.0-2//stax-api-1.0-2.jar
-stax-api/1.0.1//stax-api-1.0.1.jar
-stream/2.9.6//stream-2.9.6.jar
-stringtemplate/3.2.1//stringtemplate-3.2.1.jar
-super-csv/2.2.0//super-csv-2.2.0.jar
-threeten-extra/1.5.0//threeten-extra-1.5.0.jar
-univocity-parsers/2.8.3//univocity-parsers-2.8.3.jar
-xbean-asm7-shaded/4.15//xbean-asm7-shaded-4.15.jar
-xercesImpl/2.12.0//xercesImpl-2.12.0.jar
-xml-apis/1.4.01//xml-apis-1.4.01.jar
-xmlenc/0.52//xmlenc-0.52.jar
-xz/1.5//xz-1.5.jar
-zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar
-zookeeper/3.4.14//zookeeper-3.4.14.jar
-zstd-jni/1.4.4-3//zstd-jni-1.4.4-3.jar
diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
index 093924f58cb20..d7c64af0c54e4 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
@@ -1,28 +1,32 @@
 HikariCP/2.5.1//HikariCP-2.5.1.jar
 JLargeArrays/1.5//JLargeArrays-1.5.jar
 JTransforms/3.1//JTransforms-3.1.jar
-RoaringBitmap/0.7.45//RoaringBitmap-0.7.45.jar
+RoaringBitmap/0.9.0//RoaringBitmap-0.9.0.jar
 ST4/4.0.4//ST4-4.0.4.jar
 activation/1.1.1//activation-1.1.1.jar
-aircompressor/0.10//aircompressor-0.10.jar
+aircompressor/0.16//aircompressor-0.16.jar
 algebra_2.12/2.0.0-M2//algebra_2.12-2.0.0-M2.jar
+annotations/17.0.0//annotations-17.0.0.jar
 antlr-runtime/3.5.2//antlr-runtime-3.5.2.jar
-antlr4-runtime/4.7.1//antlr4-runtime-4.7.1.jar
+antlr4-runtime/4.8-1//antlr4-runtime-4.8-1.jar
 aopalliance-repackaged/2.6.1//aopalliance-repackaged-2.6.1.jar
 aopalliance/1.0//aopalliance-1.0.jar
 apacheds-i18n/2.0.0-M15//apacheds-i18n-2.0.0-M15.jar
 apacheds-kerberos-codec/2.0.0-M15//apacheds-kerberos-codec-2.0.0-M15.jar
 api-asn1-api/1.0.0-M20//api-asn1-api-1.0.0-M20.jar
 api-util/1.0.0-M20//api-util-1.0.0-M20.jar
+arpack/2.2.0//arpack-2.2.0.jar
 arpack_combined_all/0.1//arpack_combined_all-0.1.jar
-arrow-format/0.15.1//arrow-format-0.15.1.jar
-arrow-memory/0.15.1//arrow-memory-0.15.1.jar
-arrow-vector/0.15.1//arrow-vector-0.15.1.jar
+arrow-format/2.0.0//arrow-format-2.0.0.jar
+arrow-memory-core/2.0.0//arrow-memory-core-2.0.0.jar
+arrow-memory-netty/2.0.0//arrow-memory-netty-2.0.0.jar
+arrow-vector/2.0.0//arrow-vector-2.0.0.jar
 audience-annotations/0.5.0//audience-annotations-0.5.0.jar
 automaton/1.11-8//automaton-1.11-8.jar
-avro-ipc/1.8.2//avro-ipc-1.8.2.jar
-avro-mapred/1.8.2/hadoop2/avro-mapred-1.8.2-hadoop2.jar
-avro/1.8.2//avro-1.8.2.jar
+avro-ipc/1.10.2//avro-ipc-1.10.2.jar
+avro-mapred/1.10.2//avro-mapred-1.10.2.jar
+avro/1.10.2//avro-1.10.2.jar
+blas/2.2.0//blas-2.2.0.jar
 bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar
 breeze-macros_2.12/1.0//breeze-macros_2.12-1.0.jar
 breeze_2.12/1.0//breeze_2.12-1.0.jar
@@ -31,18 +35,18 @@ chill-java/0.9.5//chill-java-0.9.5.jar
 chill_2.12/0.9.5//chill_2.12-0.9.5.jar
 commons-beanutils/1.9.4//commons-beanutils-1.9.4.jar
 commons-cli/1.2//commons-cli-1.2.jar
-commons-codec/1.10//commons-codec-1.10.jar
+commons-codec/1.15//commons-codec-1.15.jar
 commons-collections/3.2.2//commons-collections-3.2.2.jar
-commons-compiler/3.0.16//commons-compiler-3.0.16.jar
-commons-compress/1.8.1//commons-compress-1.8.1.jar
+commons-compiler/3.1.4//commons-compiler-3.1.4.jar
+commons-compress/1.20//commons-compress-1.20.jar
 commons-configuration/1.6//commons-configuration-1.6.jar
-commons-crypto/1.0.0//commons-crypto-1.0.0.jar
+commons-crypto/1.1.0//commons-crypto-1.1.0.jar
 commons-dbcp/1.4//commons-dbcp-1.4.jar
 commons-digester/1.8//commons-digester-1.8.jar
 commons-httpclient/3.1//commons-httpclient-3.1.jar
 commons-io/2.4//commons-io-2.4.jar
 commons-lang/2.6//commons-lang-2.6.jar
-commons-lang3/3.9//commons-lang3-3.9.jar
+commons-lang3/3.12.0//commons-lang3-3.12.0.jar
 commons-logging/1.1.3//commons-logging-1.1.3.jar
 commons-math3/3.4.1//commons-math3-3.4.1.jar
 commons-net/3.1//commons-net-3.1.jar
@@ -56,7 +60,7 @@ curator-recipes/2.7.1//curator-recipes-2.7.1.jar
 datanucleus-api-jdo/4.2.4//datanucleus-api-jdo-4.2.4.jar
 datanucleus-core/4.1.17//datanucleus-core-4.1.17.jar
 datanucleus-rdbms/4.1.19//datanucleus-rdbms-4.1.19.jar
-derby/10.12.1.1//derby-10.12.1.1.jar
+derby/10.14.2.0//derby-10.14.2.0.jar
 dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar
 flatbuffers-java/1.9.0//flatbuffers-java-1.9.0.jar
 generex/1.0.2//generex-1.0.2.jar
@@ -79,62 +83,60 @@ hadoop-yarn-client/2.7.4//hadoop-yarn-client-2.7.4.jar
 hadoop-yarn-common/2.7.4//hadoop-yarn-common-2.7.4.jar
 hadoop-yarn-server-common/2.7.4//hadoop-yarn-server-common-2.7.4.jar
 hadoop-yarn-server-web-proxy/2.7.4//hadoop-yarn-server-web-proxy-2.7.4.jar
-hive-beeline/2.3.7//hive-beeline-2.3.7.jar
-hive-cli/2.3.7//hive-cli-2.3.7.jar
-hive-common/2.3.7//hive-common-2.3.7.jar
-hive-exec/2.3.7/core/hive-exec-2.3.7-core.jar
-hive-jdbc/2.3.7//hive-jdbc-2.3.7.jar
-hive-llap-common/2.3.7//hive-llap-common-2.3.7.jar
-hive-metastore/2.3.7//hive-metastore-2.3.7.jar
-hive-serde/2.3.7//hive-serde-2.3.7.jar
-hive-shims-0.23/2.3.7//hive-shims-0.23-2.3.7.jar
-hive-shims-common/2.3.7//hive-shims-common-2.3.7.jar
-hive-shims-scheduler/2.3.7//hive-shims-scheduler-2.3.7.jar
-hive-shims/2.3.7//hive-shims-2.3.7.jar
-hive-storage-api/2.7.1//hive-storage-api-2.7.1.jar
-hive-vector-code-gen/2.3.7//hive-vector-code-gen-2.3.7.jar
+hive-beeline/2.3.8//hive-beeline-2.3.8.jar
+hive-cli/2.3.8//hive-cli-2.3.8.jar
+hive-common/2.3.8//hive-common-2.3.8.jar
+hive-exec/2.3.8/core/hive-exec-2.3.8-core.jar
+hive-jdbc/2.3.8//hive-jdbc-2.3.8.jar
+hive-llap-common/2.3.8//hive-llap-common-2.3.8.jar
+hive-metastore/2.3.8//hive-metastore-2.3.8.jar
+hive-serde/2.3.8//hive-serde-2.3.8.jar
+hive-service-rpc/3.1.2//hive-service-rpc-3.1.2.jar
+hive-shims-0.23/2.3.8//hive-shims-0.23-2.3.8.jar
+hive-shims-common/2.3.8//hive-shims-common-2.3.8.jar
+hive-shims-scheduler/2.3.8//hive-shims-scheduler-2.3.8.jar
+hive-shims/2.3.8//hive-shims-2.3.8.jar
+hive-storage-api/2.7.2//hive-storage-api-2.7.2.jar
+hive-vector-code-gen/2.3.8//hive-vector-code-gen-2.3.8.jar
 hk2-api/2.6.1//hk2-api-2.6.1.jar
 hk2-locator/2.6.1//hk2-locator-2.6.1.jar
 hk2-utils/2.6.1//hk2-utils-2.6.1.jar
 htrace-core/3.1.0-incubating//htrace-core-3.1.0-incubating.jar
-httpclient/4.5.6//httpclient-4.5.6.jar
+httpclient/4.5.13//httpclient-4.5.13.jar
 httpcore/4.4.12//httpcore-4.4.12.jar
 istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar
 ivy/2.4.0//ivy-2.4.0.jar
-jackson-annotations/2.10.0//jackson-annotations-2.10.0.jar
+jackson-annotations/2.12.2//jackson-annotations-2.12.2.jar
 jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
-jackson-core/2.10.0//jackson-core-2.10.0.jar
-jackson-databind/2.10.0//jackson-databind-2.10.0.jar
-jackson-dataformat-yaml/2.10.0//jackson-dataformat-yaml-2.10.0.jar
+jackson-core/2.12.2//jackson-core-2.12.2.jar
+jackson-databind/2.12.2//jackson-databind-2.12.2.jar
+jackson-dataformat-yaml/2.12.2//jackson-dataformat-yaml-2.12.2.jar
+jackson-datatype-jsr310/2.11.2//jackson-datatype-jsr310-2.11.2.jar
 jackson-jaxrs/1.9.13//jackson-jaxrs-1.9.13.jar
 jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
-jackson-module-jaxb-annotations/2.10.0//jackson-module-jaxb-annotations-2.10.0.jar
-jackson-module-paranamer/2.10.0//jackson-module-paranamer-2.10.0.jar
-jackson-module-scala_2.12/2.10.0//jackson-module-scala_2.12-2.10.0.jar
+jackson-module-scala_2.12/2.12.2//jackson-module-scala_2.12-2.12.2.jar
 jackson-xc/1.9.13//jackson-xc-1.9.13.jar
-jakarta.activation-api/1.2.1//jakarta.activation-api-1.2.1.jar
 jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar
 jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar
+jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar
 jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar
 jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar
 jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
-janino/3.0.16//janino-3.0.16.jar
+janino/3.1.4//janino-3.1.4.jar
 javassist/3.25.0-GA//javassist-3.25.0-GA.jar
 javax.inject/1//javax.inject-1.jar
 javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar
-javax.servlet-api/3.1.0//javax.servlet-api-3.1.0.jar
 javolution/5.5.1//javolution-5.5.1.jar
-jaxb-api/2.2.2//jaxb-api-2.2.2.jar
+jaxb-api/2.2.11//jaxb-api-2.2.11.jar
 jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar
 jcl-over-slf4j/1.7.30//jcl-over-slf4j-1.7.30.jar
 jdo-api/3.0.1//jdo-api-3.0.1.jar
-jersey-client/2.30//jersey-client-2.30.jar
-jersey-common/2.30//jersey-common-2.30.jar
-jersey-container-servlet-core/2.30//jersey-container-servlet-core-2.30.jar
-jersey-container-servlet/2.30//jersey-container-servlet-2.30.jar
-jersey-hk2/2.30//jersey-hk2-2.30.jar
-jersey-media-jaxb/2.30//jersey-media-jaxb-2.30.jar
-jersey-server/2.30//jersey-server-2.30.jar
+jersey-client/2.34//jersey-client-2.34.jar
+jersey-common/2.34//jersey-common-2.34.jar
+jersey-container-servlet-core/2.34//jersey-container-servlet-core-2.34.jar
+jersey-container-servlet/2.34//jersey-container-servlet-2.34.jar
+jersey-hk2/2.34//jersey-hk2-2.34.jar
+jersey-server/2.34//jersey-server-2.34.jar
 jetty-sslengine/6.1.26//jetty-sslengine-6.1.26.jar
 jetty-util/6.1.26//jetty-util-6.1.26.jar
 jetty/6.1.26//jetty-6.1.26.jar
@@ -143,23 +145,41 @@ joda-time/2.10.5//joda-time-2.10.5.jar
 jodd-core/3.5.2//jodd-core-3.5.2.jar
 jpam/1.1//jpam-1.1.jar
 json/1.8//json-1.8.jar
-json4s-ast_2.12/3.6.6//json4s-ast_2.12-3.6.6.jar
-json4s-core_2.12/3.6.6//json4s-core_2.12-3.6.6.jar
-json4s-jackson_2.12/3.6.6//json4s-jackson_2.12-3.6.6.jar
-json4s-scalap_2.12/3.6.6//json4s-scalap_2.12-3.6.6.jar
+json4s-ast_2.12/3.7.0-M5//json4s-ast_2.12-3.7.0-M5.jar
+json4s-core_2.12/3.7.0-M5//json4s-core_2.12-3.7.0-M5.jar
+json4s-jackson_2.12/3.7.0-M5//json4s-jackson_2.12-3.7.0-M5.jar
+json4s-scalap_2.12/3.7.0-M5//json4s-scalap_2.12-3.7.0-M5.jar
 jsp-api/2.1//jsp-api-2.1.jar
 jsr305/3.0.0//jsr305-3.0.0.jar
 jta/1.1//jta-1.1.jar
 jul-to-slf4j/1.7.30//jul-to-slf4j-1.7.30.jar
 kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar
-kubernetes-client/4.7.1//kubernetes-client-4.7.1.jar
-kubernetes-model-common/4.7.1//kubernetes-model-common-4.7.1.jar
-kubernetes-model/4.7.1//kubernetes-model-4.7.1.jar
+kubernetes-client/5.3.1//kubernetes-client-5.3.1.jar
+kubernetes-model-admissionregistration/5.3.1//kubernetes-model-admissionregistration-5.3.1.jar
+kubernetes-model-apiextensions/5.3.1//kubernetes-model-apiextensions-5.3.1.jar
+kubernetes-model-apps/5.3.1//kubernetes-model-apps-5.3.1.jar
+kubernetes-model-autoscaling/5.3.1//kubernetes-model-autoscaling-5.3.1.jar
+kubernetes-model-batch/5.3.1//kubernetes-model-batch-5.3.1.jar
+kubernetes-model-certificates/5.3.1//kubernetes-model-certificates-5.3.1.jar
+kubernetes-model-common/5.3.1//kubernetes-model-common-5.3.1.jar
+kubernetes-model-coordination/5.3.1//kubernetes-model-coordination-5.3.1.jar
+kubernetes-model-core/5.3.1//kubernetes-model-core-5.3.1.jar
+kubernetes-model-discovery/5.3.1//kubernetes-model-discovery-5.3.1.jar
+kubernetes-model-events/5.3.1//kubernetes-model-events-5.3.1.jar
+kubernetes-model-extensions/5.3.1//kubernetes-model-extensions-5.3.1.jar
+kubernetes-model-metrics/5.3.1//kubernetes-model-metrics-5.3.1.jar
+kubernetes-model-networking/5.3.1//kubernetes-model-networking-5.3.1.jar
+kubernetes-model-node/5.3.1//kubernetes-model-node-5.3.1.jar
+kubernetes-model-policy/5.3.1//kubernetes-model-policy-5.3.1.jar
+kubernetes-model-rbac/5.3.1//kubernetes-model-rbac-5.3.1.jar
+kubernetes-model-scheduling/5.3.1//kubernetes-model-scheduling-5.3.1.jar
+kubernetes-model-storageclass/5.3.1//kubernetes-model-storageclass-5.3.1.jar
+lapack/2.2.0//lapack-2.2.0.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 libfb303/0.9.3//libfb303-0.9.3.jar
 libthrift/0.12.0//libthrift-0.12.0.jar
 log4j/1.2.17//log4j-1.2.17.jar
-logging-interceptor/3.12.6//logging-interceptor-3.12.6.jar
+logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar
 lz4-java/1.7.1//lz4-java-1.7.1.jar
 machinist_2.12/0.6.8//machinist_2.12-0.6.8.jar
 macro-compat_2.12/1.1.1//macro-compat_2.12-1.1.1.jar
@@ -170,25 +190,25 @@ metrics-jmx/4.1.1//metrics-jmx-4.1.1.jar
 metrics-json/4.1.1//metrics-json-4.1.1.jar
 metrics-jvm/4.1.1//metrics-jvm-4.1.1.jar
 minlog/1.3.0//minlog-1.3.0.jar
-netty-all/4.1.47.Final//netty-all-4.1.47.Final.jar
-objenesis/2.5.1//objenesis-2.5.1.jar
-okhttp/3.12.6//okhttp-3.12.6.jar
-okio/1.15.0//okio-1.15.0.jar
+netty-all/4.1.63.Final//netty-all-4.1.63.Final.jar
+objenesis/2.6//objenesis-2.6.jar
+okhttp/3.12.12//okhttp-3.12.12.jar
+okio/1.14.0//okio-1.14.0.jar
 opencsv/2.3//opencsv-2.3.jar
-orc-core/1.5.10//orc-core-1.5.10.jar
-orc-mapreduce/1.5.10//orc-mapreduce-1.5.10.jar
-orc-shims/1.5.10//orc-shims-1.5.10.jar
+orc-core/1.6.7//orc-core-1.6.7.jar
+orc-mapreduce/1.6.7//orc-mapreduce-1.6.7.jar
+orc-shims/1.6.7//orc-shims-1.6.7.jar
 oro/2.0.8//oro-2.0.8.jar
 osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
 paranamer/2.8//paranamer-2.8.jar
-parquet-column/1.10.1//parquet-column-1.10.1.jar
-parquet-common/1.10.1//parquet-common-1.10.1.jar
-parquet-encoding/1.10.1//parquet-encoding-1.10.1.jar
-parquet-format/2.4.0//parquet-format-2.4.0.jar
-parquet-hadoop/1.10.1//parquet-hadoop-1.10.1.jar
-parquet-jackson/1.10.1//parquet-jackson-1.10.1.jar
+parquet-column/1.12.0//parquet-column-1.12.0.jar
+parquet-common/1.12.0//parquet-common-1.12.0.jar
+parquet-encoding/1.12.0//parquet-encoding-1.12.0.jar
+parquet-format-structures/1.12.0//parquet-format-structures-1.12.0.jar
+parquet-hadoop/1.12.0//parquet-hadoop-1.12.0.jar
+parquet-jackson/1.12.0//parquet-jackson-1.12.0.jar
 protobuf-java/2.5.0//protobuf-java-2.5.0.jar
-py4j/0.10.9//py4j-0.10.9.jar
+py4j/0.10.9.2//py4j-0.10.9.2.jar
 pyrolite/4.30//pyrolite-4.30.jar
 scala-collection-compat_2.12/2.1.1//scala-collection-compat_2.12-2.1.1.jar
 scala-compiler/2.12.10//scala-compiler-2.12.10.jar
@@ -197,28 +217,28 @@ scala-parser-combinators_2.12/1.1.2//scala-parser-combinators_2.12-1.1.2.jar
 scala-reflect/2.12.10//scala-reflect-2.12.10.jar
 scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar
 shapeless_2.12/2.3.3//shapeless_2.12-2.3.3.jar
-shims/0.7.45//shims-0.7.45.jar
+shims/0.9.0//shims-0.9.0.jar
 slf4j-api/1.7.30//slf4j-api-1.7.30.jar
 slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
-snakeyaml/1.24//snakeyaml-1.24.jar
-snappy-java/1.1.7.5//snappy-java-1.1.7.5.jar
+snakeyaml/1.27//snakeyaml-1.27.jar
+snappy-java/1.1.8.4//snappy-java-1.1.8.4.jar
 spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar
 spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar
 spire-util_2.12/0.17.0-M1//spire-util_2.12-0.17.0-M1.jar
 spire_2.12/0.17.0-M1//spire_2.12-0.17.0-M1.jar
-stax-api/1.0-2//stax-api-1.0-2.jar
 stax-api/1.0.1//stax-api-1.0.1.jar
 stream/2.9.6//stream-2.9.6.jar
 super-csv/2.2.0//super-csv-2.2.0.jar
 threeten-extra/1.5.0//threeten-extra-1.5.0.jar
 transaction-api/1.1//transaction-api-1.1.jar
-univocity-parsers/2.8.3//univocity-parsers-2.8.3.jar
+univocity-parsers/2.9.1//univocity-parsers-2.9.1.jar
 velocity/1.5//velocity-1.5.jar
 xbean-asm7-shaded/4.15//xbean-asm7-shaded-4.15.jar
 xercesImpl/2.12.0//xercesImpl-2.12.0.jar
 xml-apis/1.4.01//xml-apis-1.4.01.jar
 xmlenc/0.52//xmlenc-0.52.jar
-xz/1.5//xz-1.5.jar
+xz/1.8//xz-1.8.jar
 zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar
-zookeeper/3.4.14//zookeeper-3.4.14.jar
-zstd-jni/1.4.4-3//zstd-jni-1.4.4-3.jar
+zookeeper-jute/3.6.2//zookeeper-jute-3.6.2.jar
+zookeeper/3.6.2//zookeeper-3.6.2.jar
+zstd-jni/1.4.9-1//zstd-jni-1.4.9-1.jar
diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
index 2db8d3e74d0e6..eda1a0b24e2af 100644
--- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
@@ -1,45 +1,44 @@
 HikariCP/2.5.1//HikariCP-2.5.1.jar
 JLargeArrays/1.5//JLargeArrays-1.5.jar
 JTransforms/3.1//JTransforms-3.1.jar
-RoaringBitmap/0.7.45//RoaringBitmap-0.7.45.jar
+RoaringBitmap/0.9.0//RoaringBitmap-0.9.0.jar
 ST4/4.0.4//ST4-4.0.4.jar
-accessors-smart/1.2//accessors-smart-1.2.jar
 activation/1.1.1//activation-1.1.1.jar
-aircompressor/0.10//aircompressor-0.10.jar
+aircompressor/0.16//aircompressor-0.16.jar
 algebra_2.12/2.0.0-M2//algebra_2.12-2.0.0-M2.jar
+annotations/17.0.0//annotations-17.0.0.jar
 antlr-runtime/3.5.2//antlr-runtime-3.5.2.jar
-antlr4-runtime/4.7.1//antlr4-runtime-4.7.1.jar
+antlr4-runtime/4.8-1//antlr4-runtime-4.8-1.jar
 aopalliance-repackaged/2.6.1//aopalliance-repackaged-2.6.1.jar
-aopalliance/1.0//aopalliance-1.0.jar
+arpack/2.2.0//arpack-2.2.0.jar
 arpack_combined_all/0.1//arpack_combined_all-0.1.jar
-arrow-format/0.15.1//arrow-format-0.15.1.jar
-arrow-memory/0.15.1//arrow-memory-0.15.1.jar
-arrow-vector/0.15.1//arrow-vector-0.15.1.jar
+arrow-format/2.0.0//arrow-format-2.0.0.jar
+arrow-memory-core/2.0.0//arrow-memory-core-2.0.0.jar
+arrow-memory-netty/2.0.0//arrow-memory-netty-2.0.0.jar
+arrow-vector/2.0.0//arrow-vector-2.0.0.jar
 audience-annotations/0.5.0//audience-annotations-0.5.0.jar
 automaton/1.11-8//automaton-1.11-8.jar
-avro-ipc/1.8.2//avro-ipc-1.8.2.jar
-avro-mapred/1.8.2/hadoop2/avro-mapred-1.8.2-hadoop2.jar
-avro/1.8.2//avro-1.8.2.jar
+avro-ipc/1.10.2//avro-ipc-1.10.2.jar
+avro-mapred/1.10.2//avro-mapred-1.10.2.jar
+avro/1.10.2//avro-1.10.2.jar
+blas/2.2.0//blas-2.2.0.jar
 bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar
 breeze-macros_2.12/1.0//breeze-macros_2.12-1.0.jar
 breeze_2.12/1.0//breeze_2.12-1.0.jar
 cats-kernel_2.12/2.0.0-M4//cats-kernel_2.12-2.0.0-M4.jar
 chill-java/0.9.5//chill-java-0.9.5.jar
 chill_2.12/0.9.5//chill_2.12-0.9.5.jar
-commons-beanutils/1.9.4//commons-beanutils-1.9.4.jar
 commons-cli/1.2//commons-cli-1.2.jar
-commons-codec/1.10//commons-codec-1.10.jar
+commons-codec/1.15//commons-codec-1.15.jar
 commons-collections/3.2.2//commons-collections-3.2.2.jar
-commons-compiler/3.0.16//commons-compiler-3.0.16.jar
-commons-compress/1.8.1//commons-compress-1.8.1.jar
-commons-configuration2/2.1.1//commons-configuration2-2.1.1.jar
-commons-crypto/1.0.0//commons-crypto-1.0.0.jar
-commons-daemon/1.0.13//commons-daemon-1.0.13.jar
+commons-compiler/3.1.4//commons-compiler-3.1.4.jar
+commons-compress/1.20//commons-compress-1.20.jar
+commons-crypto/1.1.0//commons-crypto-1.1.0.jar
 commons-dbcp/1.4//commons-dbcp-1.4.jar
 commons-httpclient/3.1//commons-httpclient-3.1.jar
-commons-io/2.4//commons-io-2.4.jar
+commons-io/2.8.0//commons-io-2.8.0.jar
 commons-lang/2.6//commons-lang-2.6.jar
-commons-lang3/3.9//commons-lang3-3.9.jar
+commons-lang3/3.12.0//commons-lang3-3.12.0.jar
 commons-logging/1.1.3//commons-logging-1.1.3.jar
 commons-math3/3.4.1//commons-math3-3.4.1.jar
 commons-net/3.1//commons-net-3.1.jar
@@ -53,125 +52,105 @@ curator-recipes/2.13.0//curator-recipes-2.13.0.jar
 datanucleus-api-jdo/4.2.4//datanucleus-api-jdo-4.2.4.jar
 datanucleus-core/4.1.17//datanucleus-core-4.1.17.jar
 datanucleus-rdbms/4.1.19//datanucleus-rdbms-4.1.19.jar
-derby/10.12.1.1//derby-10.12.1.1.jar
-dnsjava/2.1.7//dnsjava-2.1.7.jar
+derby/10.14.2.0//derby-10.14.2.0.jar
 dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar
-ehcache/3.3.1//ehcache-3.3.1.jar
 flatbuffers-java/1.9.0//flatbuffers-java-1.9.0.jar
 generex/1.0.2//generex-1.0.2.jar
-geronimo-jcache_1.0_spec/1.0-alpha-1//geronimo-jcache_1.0_spec-1.0-alpha-1.jar
 gson/2.2.4//gson-2.2.4.jar
 guava/14.0.1//guava-14.0.1.jar
-guice-servlet/4.0//guice-servlet-4.0.jar
-guice/4.0//guice-4.0.jar
-hadoop-annotations/3.2.0//hadoop-annotations-3.2.0.jar
-hadoop-auth/3.2.0//hadoop-auth-3.2.0.jar
-hadoop-client/3.2.0//hadoop-client-3.2.0.jar
-hadoop-common/3.2.0//hadoop-common-3.2.0.jar
-hadoop-hdfs-client/3.2.0//hadoop-hdfs-client-3.2.0.jar
-hadoop-mapreduce-client-common/3.2.0//hadoop-mapreduce-client-common-3.2.0.jar
-hadoop-mapreduce-client-core/3.2.0//hadoop-mapreduce-client-core-3.2.0.jar
-hadoop-mapreduce-client-jobclient/3.2.0//hadoop-mapreduce-client-jobclient-3.2.0.jar
-hadoop-yarn-api/3.2.0//hadoop-yarn-api-3.2.0.jar
-hadoop-yarn-client/3.2.0//hadoop-yarn-client-3.2.0.jar
-hadoop-yarn-common/3.2.0//hadoop-yarn-common-3.2.0.jar
-hadoop-yarn-registry/3.2.0//hadoop-yarn-registry-3.2.0.jar
-hadoop-yarn-server-common/3.2.0//hadoop-yarn-server-common-3.2.0.jar
-hadoop-yarn-server-web-proxy/3.2.0//hadoop-yarn-server-web-proxy-3.2.0.jar
-hive-beeline/2.3.7//hive-beeline-2.3.7.jar
-hive-cli/2.3.7//hive-cli-2.3.7.jar
-hive-common/2.3.7//hive-common-2.3.7.jar
-hive-exec/2.3.7/core/hive-exec-2.3.7-core.jar
-hive-jdbc/2.3.7//hive-jdbc-2.3.7.jar
-hive-llap-common/2.3.7//hive-llap-common-2.3.7.jar
-hive-metastore/2.3.7//hive-metastore-2.3.7.jar
-hive-serde/2.3.7//hive-serde-2.3.7.jar
-hive-shims-0.23/2.3.7//hive-shims-0.23-2.3.7.jar
-hive-shims-common/2.3.7//hive-shims-common-2.3.7.jar
-hive-shims-scheduler/2.3.7//hive-shims-scheduler-2.3.7.jar
-hive-shims/2.3.7//hive-shims-2.3.7.jar
-hive-storage-api/2.7.1//hive-storage-api-2.7.1.jar
-hive-vector-code-gen/2.3.7//hive-vector-code-gen-2.3.7.jar
+hadoop-client-api/3.2.2//hadoop-client-api-3.2.2.jar
+hadoop-client-runtime/3.2.2//hadoop-client-runtime-3.2.2.jar
+hadoop-yarn-server-web-proxy/3.2.2//hadoop-yarn-server-web-proxy-3.2.2.jar
+hive-beeline/2.3.8//hive-beeline-2.3.8.jar
+hive-cli/2.3.8//hive-cli-2.3.8.jar
+hive-common/2.3.8//hive-common-2.3.8.jar
+hive-exec/2.3.8/core/hive-exec-2.3.8-core.jar
+hive-jdbc/2.3.8//hive-jdbc-2.3.8.jar
+hive-llap-common/2.3.8//hive-llap-common-2.3.8.jar
+hive-metastore/2.3.8//hive-metastore-2.3.8.jar
+hive-serde/2.3.8//hive-serde-2.3.8.jar
+hive-service-rpc/3.1.2//hive-service-rpc-3.1.2.jar
+hive-shims-0.23/2.3.8//hive-shims-0.23-2.3.8.jar
+hive-shims-common/2.3.8//hive-shims-common-2.3.8.jar
+hive-shims-scheduler/2.3.8//hive-shims-scheduler-2.3.8.jar
+hive-shims/2.3.8//hive-shims-2.3.8.jar
+hive-storage-api/2.7.2//hive-storage-api-2.7.2.jar
+hive-vector-code-gen/2.3.8//hive-vector-code-gen-2.3.8.jar
 hk2-api/2.6.1//hk2-api-2.6.1.jar
 hk2-locator/2.6.1//hk2-locator-2.6.1.jar
 hk2-utils/2.6.1//hk2-utils-2.6.1.jar
 htrace-core4/4.1.0-incubating//htrace-core4-4.1.0-incubating.jar
-httpclient/4.5.6//httpclient-4.5.6.jar
+httpclient/4.5.13//httpclient-4.5.13.jar
 httpcore/4.4.12//httpcore-4.4.12.jar
 istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar
 ivy/2.4.0//ivy-2.4.0.jar
-jackson-annotations/2.10.0//jackson-annotations-2.10.0.jar
+jackson-annotations/2.12.2//jackson-annotations-2.12.2.jar
 jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
-jackson-core/2.10.0//jackson-core-2.10.0.jar
-jackson-databind/2.10.0//jackson-databind-2.10.0.jar
-jackson-dataformat-yaml/2.10.0//jackson-dataformat-yaml-2.10.0.jar
-jackson-jaxrs-base/2.9.5//jackson-jaxrs-base-2.9.5.jar
-jackson-jaxrs-json-provider/2.9.5//jackson-jaxrs-json-provider-2.9.5.jar
+jackson-core/2.12.2//jackson-core-2.12.2.jar
+jackson-databind/2.12.2//jackson-databind-2.12.2.jar
+jackson-dataformat-yaml/2.12.2//jackson-dataformat-yaml-2.12.2.jar
+jackson-datatype-jsr310/2.11.2//jackson-datatype-jsr310-2.11.2.jar
 jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
-jackson-module-jaxb-annotations/2.10.0//jackson-module-jaxb-annotations-2.10.0.jar
-jackson-module-paranamer/2.10.0//jackson-module-paranamer-2.10.0.jar
-jackson-module-scala_2.12/2.10.0//jackson-module-scala_2.12-2.10.0.jar
-jakarta.activation-api/1.2.1//jakarta.activation-api-1.2.1.jar
+jackson-module-scala_2.12/2.12.2//jackson-module-scala_2.12-2.12.2.jar
 jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar
 jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar
+jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar
 jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar
 jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar
 jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
-janino/3.0.16//janino-3.0.16.jar
+janino/3.1.4//janino-3.1.4.jar
 javassist/3.25.0-GA//javassist-3.25.0-GA.jar
-javax.inject/1//javax.inject-1.jar
 javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar
-javax.servlet-api/3.1.0//javax.servlet-api-3.1.0.jar
 javolution/5.5.1//javolution-5.5.1.jar
 jaxb-api/2.2.11//jaxb-api-2.2.11.jar
 jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar
-jcip-annotations/1.0-1//jcip-annotations-1.0-1.jar
 jcl-over-slf4j/1.7.30//jcl-over-slf4j-1.7.30.jar
 jdo-api/3.0.1//jdo-api-3.0.1.jar
-jersey-client/2.30//jersey-client-2.30.jar
-jersey-common/2.30//jersey-common-2.30.jar
-jersey-container-servlet-core/2.30//jersey-container-servlet-core-2.30.jar
-jersey-container-servlet/2.30//jersey-container-servlet-2.30.jar
-jersey-hk2/2.30//jersey-hk2-2.30.jar
-jersey-media-jaxb/2.30//jersey-media-jaxb-2.30.jar
-jersey-server/2.30//jersey-server-2.30.jar
+jersey-client/2.34//jersey-client-2.34.jar
+jersey-common/2.34//jersey-common-2.34.jar
+jersey-container-servlet-core/2.34//jersey-container-servlet-core-2.34.jar
+jersey-container-servlet/2.34//jersey-container-servlet-2.34.jar
+jersey-hk2/2.34//jersey-hk2-2.34.jar
+jersey-server/2.34//jersey-server-2.34.jar
 jline/2.14.6//jline-2.14.6.jar
 joda-time/2.10.5//joda-time-2.10.5.jar
 jodd-core/3.5.2//jodd-core-3.5.2.jar
 jpam/1.1//jpam-1.1.jar
-json-smart/2.3//json-smart-2.3.jar
 json/1.8//json-1.8.jar
-json4s-ast_2.12/3.6.6//json4s-ast_2.12-3.6.6.jar
-json4s-core_2.12/3.6.6//json4s-core_2.12-3.6.6.jar
-json4s-jackson_2.12/3.6.6//json4s-jackson_2.12-3.6.6.jar
-json4s-scalap_2.12/3.6.6//json4s-scalap_2.12-3.6.6.jar
-jsp-api/2.1//jsp-api-2.1.jar
+json4s-ast_2.12/3.7.0-M5//json4s-ast_2.12-3.7.0-M5.jar
+json4s-core_2.12/3.7.0-M5//json4s-core_2.12-3.7.0-M5.jar
+json4s-jackson_2.12/3.7.0-M5//json4s-jackson_2.12-3.7.0-M5.jar
+json4s-scalap_2.12/3.7.0-M5//json4s-scalap_2.12-3.7.0-M5.jar
 jsr305/3.0.0//jsr305-3.0.0.jar
 jta/1.1//jta-1.1.jar
 jul-to-slf4j/1.7.30//jul-to-slf4j-1.7.30.jar
-kerb-admin/1.0.1//kerb-admin-1.0.1.jar
-kerb-client/1.0.1//kerb-client-1.0.1.jar
-kerb-common/1.0.1//kerb-common-1.0.1.jar
-kerb-core/1.0.1//kerb-core-1.0.1.jar
-kerb-crypto/1.0.1//kerb-crypto-1.0.1.jar
-kerb-identity/1.0.1//kerb-identity-1.0.1.jar
-kerb-server/1.0.1//kerb-server-1.0.1.jar
-kerb-simplekdc/1.0.1//kerb-simplekdc-1.0.1.jar
-kerb-util/1.0.1//kerb-util-1.0.1.jar
-kerby-asn1/1.0.1//kerby-asn1-1.0.1.jar
-kerby-config/1.0.1//kerby-config-1.0.1.jar
-kerby-pkix/1.0.1//kerby-pkix-1.0.1.jar
-kerby-util/1.0.1//kerby-util-1.0.1.jar
-kerby-xdr/1.0.1//kerby-xdr-1.0.1.jar
 kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar
-kubernetes-client/4.7.1//kubernetes-client-4.7.1.jar
-kubernetes-model-common/4.7.1//kubernetes-model-common-4.7.1.jar
-kubernetes-model/4.7.1//kubernetes-model-4.7.1.jar
+kubernetes-client/5.3.1//kubernetes-client-5.3.1.jar
+kubernetes-model-admissionregistration/5.3.1//kubernetes-model-admissionregistration-5.3.1.jar
+kubernetes-model-apiextensions/5.3.1//kubernetes-model-apiextensions-5.3.1.jar
+kubernetes-model-apps/5.3.1//kubernetes-model-apps-5.3.1.jar
+kubernetes-model-autoscaling/5.3.1//kubernetes-model-autoscaling-5.3.1.jar
+kubernetes-model-batch/5.3.1//kubernetes-model-batch-5.3.1.jar
+kubernetes-model-certificates/5.3.1//kubernetes-model-certificates-5.3.1.jar
+kubernetes-model-common/5.3.1//kubernetes-model-common-5.3.1.jar
+kubernetes-model-coordination/5.3.1//kubernetes-model-coordination-5.3.1.jar
+kubernetes-model-core/5.3.1//kubernetes-model-core-5.3.1.jar
+kubernetes-model-discovery/5.3.1//kubernetes-model-discovery-5.3.1.jar
+kubernetes-model-events/5.3.1//kubernetes-model-events-5.3.1.jar
+kubernetes-model-extensions/5.3.1//kubernetes-model-extensions-5.3.1.jar
+kubernetes-model-metrics/5.3.1//kubernetes-model-metrics-5.3.1.jar
+kubernetes-model-networking/5.3.1//kubernetes-model-networking-5.3.1.jar
+kubernetes-model-node/5.3.1//kubernetes-model-node-5.3.1.jar
+kubernetes-model-policy/5.3.1//kubernetes-model-policy-5.3.1.jar
+kubernetes-model-rbac/5.3.1//kubernetes-model-rbac-5.3.1.jar
+kubernetes-model-scheduling/5.3.1//kubernetes-model-scheduling-5.3.1.jar
+kubernetes-model-storageclass/5.3.1//kubernetes-model-storageclass-5.3.1.jar
+lapack/2.2.0//lapack-2.2.0.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 libfb303/0.9.3//libfb303-0.9.3.jar
 libthrift/0.12.0//libthrift-0.12.0.jar
 log4j/1.2.17//log4j-1.2.17.jar
-logging-interceptor/3.12.6//logging-interceptor-3.12.6.jar
+logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar
 lz4-java/1.7.1//lz4-java-1.7.1.jar
 machinist_2.12/0.6.8//machinist_2.12-0.6.8.jar
 macro-compat_2.12/1.1.1//macro-compat_2.12-1.1.1.jar
@@ -182,30 +161,26 @@ metrics-jmx/4.1.1//metrics-jmx-4.1.1.jar
 metrics-json/4.1.1//metrics-json-4.1.1.jar
 metrics-jvm/4.1.1//metrics-jvm-4.1.1.jar
 minlog/1.3.0//minlog-1.3.0.jar
-mssql-jdbc/6.2.1.jre7//mssql-jdbc-6.2.1.jre7.jar
-netty-all/4.1.47.Final//netty-all-4.1.47.Final.jar
-nimbus-jose-jwt/4.41.1//nimbus-jose-jwt-4.41.1.jar
-objenesis/2.5.1//objenesis-2.5.1.jar
-okhttp/2.7.5//okhttp-2.7.5.jar
-okhttp/3.12.6//okhttp-3.12.6.jar
-okio/1.15.0//okio-1.15.0.jar
+netty-all/4.1.63.Final//netty-all-4.1.63.Final.jar
+objenesis/2.6//objenesis-2.6.jar
+okhttp/3.12.12//okhttp-3.12.12.jar
+okio/1.14.0//okio-1.14.0.jar
 opencsv/2.3//opencsv-2.3.jar
-orc-core/1.5.10//orc-core-1.5.10.jar
-orc-mapreduce/1.5.10//orc-mapreduce-1.5.10.jar
-orc-shims/1.5.10//orc-shims-1.5.10.jar
+orc-core/1.6.7//orc-core-1.6.7.jar
+orc-mapreduce/1.6.7//orc-mapreduce-1.6.7.jar
+orc-shims/1.6.7//orc-shims-1.6.7.jar
 oro/2.0.8//oro-2.0.8.jar
 osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
 paranamer/2.8//paranamer-2.8.jar
-parquet-column/1.10.1//parquet-column-1.10.1.jar
-parquet-common/1.10.1//parquet-common-1.10.1.jar
-parquet-encoding/1.10.1//parquet-encoding-1.10.1.jar
-parquet-format/2.4.0//parquet-format-2.4.0.jar
-parquet-hadoop/1.10.1//parquet-hadoop-1.10.1.jar
-parquet-jackson/1.10.1//parquet-jackson-1.10.1.jar
+parquet-column/1.12.0//parquet-column-1.12.0.jar
+parquet-common/1.12.0//parquet-common-1.12.0.jar
+parquet-encoding/1.12.0//parquet-encoding-1.12.0.jar
+parquet-format-structures/1.12.0//parquet-format-structures-1.12.0.jar
+parquet-hadoop/1.12.0//parquet-hadoop-1.12.0.jar
+parquet-jackson/1.12.0//parquet-jackson-1.12.0.jar
 protobuf-java/2.5.0//protobuf-java-2.5.0.jar
-py4j/0.10.9//py4j-0.10.9.jar
+py4j/0.10.9.2//py4j-0.10.9.2.jar
 pyrolite/4.30//pyrolite-4.30.jar
-re2j/1.1//re2j-1.1.jar
 scala-collection-compat_2.12/2.1.1//scala-collection-compat_2.12-2.1.1.jar
 scala-compiler/2.12.10//scala-compiler-2.12.10.jar
 scala-library/2.12.10//scala-library-2.12.10.jar
@@ -213,27 +188,25 @@ scala-parser-combinators_2.12/1.1.2//scala-parser-combinators_2.12-1.1.2.jar
 scala-reflect/2.12.10//scala-reflect-2.12.10.jar
 scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar
 shapeless_2.12/2.3.3//shapeless_2.12-2.3.3.jar
-shims/0.7.45//shims-0.7.45.jar
+shims/0.9.0//shims-0.9.0.jar
 slf4j-api/1.7.30//slf4j-api-1.7.30.jar
 slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
-snakeyaml/1.24//snakeyaml-1.24.jar
-snappy-java/1.1.7.5//snappy-java-1.1.7.5.jar
+snakeyaml/1.27//snakeyaml-1.27.jar
+snappy-java/1.1.8.4//snappy-java-1.1.8.4.jar
 spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar
 spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar
 spire-util_2.12/0.17.0-M1//spire-util_2.12-0.17.0-M1.jar
 spire_2.12/0.17.0-M1//spire_2.12-0.17.0-M1.jar
 stax-api/1.0.1//stax-api-1.0.1.jar
-stax2-api/3.1.4//stax2-api-3.1.4.jar
 stream/2.9.6//stream-2.9.6.jar
 super-csv/2.2.0//super-csv-2.2.0.jar
 threeten-extra/1.5.0//threeten-extra-1.5.0.jar
-token-provider/1.0.1//token-provider-1.0.1.jar
 transaction-api/1.1//transaction-api-1.1.jar
-univocity-parsers/2.8.3//univocity-parsers-2.8.3.jar
+univocity-parsers/2.9.1//univocity-parsers-2.9.1.jar
 velocity/1.5//velocity-1.5.jar
-woodstox-core/5.0.3//woodstox-core-5.0.3.jar
 xbean-asm7-shaded/4.15//xbean-asm7-shaded-4.15.jar
-xz/1.5//xz-1.5.jar
+xz/1.8//xz-1.8.jar
 zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar
-zookeeper/3.4.14//zookeeper-3.4.14.jar
-zstd-jni/1.4.4-3//zstd-jni-1.4.4-3.jar
+zookeeper-jute/3.6.2//zookeeper-jute-3.6.2.jar
+zookeeper/3.6.2//zookeeper-3.6.2.jar
+zstd-jni/1.4.9-1//zstd-jni-1.4.9-1.jar
diff --git a/dev/eslint.json b/dev/eslint.json
new file mode 100644
index 0000000000000..ee1fd3dcc6e71
--- /dev/null
+++ b/dev/eslint.json
@@ -0,0 +1,24 @@
+{
+  "env": {
+    "browser": true,
+    "es6": true
+  },
+  "extends": "eslint:recommended",
+  "rules": {
+    "indent": [
+      "error",
+      2,
+      {
+        "SwitchCase": 1,
+        "MemberExpression": "off"
+        }
+    ],
+    "no-unused-vars": ["error", {"argsIgnorePattern": "^_ignored_.*"}]
+  },
+  "ignorePatterns": [
+    "*.min.js",
+    "sorttable.js",
+    "jquery.mustache.js",
+    "dataTables.rowsGroup.js"
+  ]
+}
diff --git a/dev/github_jira_sync.py b/dev/github_jira_sync.py
index b444b74d4027c..27451bba905dd 100755
--- a/dev/github_jira_sync.py
+++ b/dev/github_jira_sync.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
@@ -16,26 +16,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# Utility for updating JIRA's with information about Github pull requests
+# Utility for updating JIRA's with information about GitHub pull requests
 
 import json
 import os
 import re
 import sys
-if sys.version < '3':
-    from urllib2 import urlopen
-    from urllib2 import Request
-    from urllib2 import HTTPError
-else:
-    from urllib.request import urlopen
-    from urllib.request import Request
-    from urllib.error import HTTPError
+from urllib.request import urlopen
+from urllib.request import Request
+from urllib.error import HTTPError
 
 try:
     import jira.client
 except ImportError:
     print("This tool requires the jira-python library")
-    print("Install using 'sudo pip install jira'")
+    print("Install using 'sudo pip3 install jira'")
     sys.exit(-1)
 
 # User facing configs
@@ -147,9 +142,9 @@ def reset_pr_labels(pr_num, jira_components):
 jira_prs = get_jira_prs()
 
 previous_max = get_max_pr()
-print("Retrieved %s JIRA PR's from Github" % len(jira_prs))
+print("Retrieved %s JIRA PR's from GitHub" % len(jira_prs))
 jira_prs = [(k, v) for k, v in jira_prs if int(v['number']) > previous_max]
-print("%s PR's remain after excluding visted ones" % len(jira_prs))
+print("%s PR's remain after excluding visited ones" % len(jira_prs))
 
 num_updates = 0
 considered = []
@@ -162,7 +157,7 @@ def reset_pr_labels(pr_num, jira_components):
     considered = considered + [pr_num]
 
     url = pr['html_url']
-    title = "[Github] Pull Request #%s (%s)" % (pr['number'], pr['user']['login'])
+    title = "[GitHub] Pull Request #%s (%s)" % (pr['number'], pr['user']['login'])
     try:
         page = get_json(get_url(JIRA_API_BASE + "/rest/api/2/issue/" + issue + "/remotelink"))
         existing_links = map(lambda l: l['object']['url'], page)
@@ -179,7 +174,7 @@ def reset_pr_labels(pr_num, jira_components):
     destination = {"title": title, "url": url, "icon": icon}
     # For all possible fields see:
     # https://developer.atlassian.com/display/JIRADEV/Fields+in+Remote+Issue+Links
-    # application = {"name": "Github pull requests", "type": "org.apache.spark.jira.github"}
+    # application = {"name": "GitHub pull requests", "type": "org.apache.spark.jira.github"}
     jira_client.add_remote_link(issue, destination)
 
     comment = "User '%s' has created a pull request for this issue:" % pr['user']['login']
diff --git a/dev/lint-js b/dev/lint-js
new file mode 100755
index 0000000000000..ce06e282192a0
--- /dev/null
+++ b/dev/lint-js
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -o pipefail
+
+SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
+SPARK_ROOT_DIR="$(dirname $SCRIPT_DIR)"
+LINT_JS_REPORT_FILE_NAME="$SPARK_ROOT_DIR/dev/lint-js-report.log"
+LINT_TARGET_FILES=(
+  "$SPARK_ROOT_DIR/core/src/main/resources/org/apache/spark/ui/static/"
+  "$SPARK_ROOT_DIR/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/"
+  "$SPARK_ROOT_DIR/docs/js"
+)
+
+if ! type "npm" > /dev/null; then
+  echo "ERROR: You should install npm"
+  exit 1
+fi
+
+if ! type "npx" > /dev/null; then
+  echo "ERROR: You should install npx"
+  exit 1
+fi
+
+cd $SCRIPT_DIR
+
+if ! npm ls eslint > /dev/null; then
+  npm ci eslint
+fi
+
+npx eslint -c "$SPARK_ROOT_DIR/dev/eslint.json" $LINT_TARGET_FILES | tee "$LINT_JS_REPORT_FILE_NAME"
+lint_status=$?
+
+if [ "$lint_status" = "0" ] ; then
+   echo "lint-js checks passed."
+else
+  echo "lint-js checks failed."
+fi
+
+exit "$lint_status"
diff --git a/dev/lint-python b/dev/lint-python
index d5491f2447176..28df2a02ae25b 100755
--- a/dev/lint-python
+++ b/dev/lint-python
@@ -17,10 +17,11 @@
 #
 # define test binaries + versions
 FLAKE8_BUILD="flake8"
+# TODO(SPARK-34943): minimum version should be 3.8+
 MINIMUM_FLAKE8="3.5.0"
-
+MYPY_BUILD="mypy"
 PYCODESTYLE_BUILD="pycodestyle"
-MINIMUM_PYCODESTYLE="2.4.0"
+MINIMUM_PYCODESTYLE="2.7.0"
 
 SPHINX_BUILD="sphinx-build"
 
@@ -122,6 +123,31 @@ function pycodestyle_test {
     fi
 }
 
+function mypy_test {
+    local MYPY_REPORT=
+    local MYPY_STATUS=
+
+    # TODO(SPARK-32797): Install mypy on the Jenkins CI workers
+    if ! hash "$MYPY_BUILD" 2> /dev/null; then
+        echo "The $MYPY_BUILD command was not found. Skipping for now."
+        return
+    fi
+
+    echo "starting $MYPY_BUILD test..."
+    MYPY_REPORT=$( ($MYPY_BUILD --config-file python/mypy.ini python/pyspark) 2>&1)
+    MYPY_STATUS=$?
+
+    if [ "$MYPY_STATUS" -ne 0 ]; then
+        echo "mypy checks failed:"
+        echo "$MYPY_REPORT"
+        echo "$MYPY_STATUS"
+        exit "$MYPY_STATUS"
+    else
+        echo "mypy checks passed."
+        echo
+    fi
+}
+
 function flake8_test {
     local FLAKE8_VERSION=
     local EXPECTED_FLAKE8=
@@ -147,8 +173,7 @@ flake8 checks failed."
     fi
 
     echo "starting $FLAKE8_BUILD test..."
-    FLAKE8_REPORT=$( ($FLAKE8_BUILD . --count --select=E901,E999,F821,F822,F823 \
-                     --max-line-length=100 --show-source --statistics) 2>&1)
+    FLAKE8_REPORT=$( ($FLAKE8_BUILD --append-config dev/tox.ini --count --show-source --statistics .) 2>&1)
     FLAKE8_STATUS=$?
 
     if [ "$FLAKE8_STATUS" -ne 0 ]; then
@@ -168,7 +193,55 @@ function sphinx_test {
 
     # Check that the documentation builds acceptably, skip check if sphinx is not installed.
     if ! hash "$SPHINX_BUILD" 2> /dev/null; then
-        echo "The $SPHINX_BUILD command was not found. Skipping pydoc checks for now."
+        echo "The $SPHINX_BUILD command was not found. Skipping Sphinx build for now."
+        echo
+        return
+    fi
+
+    PYTHON_HAS_SPHINX=$("$PYTHON_EXECUTABLE" -c 'import importlib.util; print(importlib.util.find_spec("sphinx") is not None)')
+    if [[ "$PYTHON_HAS_SPHINX" == "False" ]]; then
+        echo "$PYTHON_EXECUTABLE does not have Sphinx installed. Skipping Sphinx build for now."
+        echo
+        return
+    fi
+
+    # TODO(SPARK-32407): Sphinx 3.1+ does not correctly index nested classes.
+    #   See also https://github.com/sphinx-doc/sphinx/issues/7551.
+    PYTHON_HAS_SPHINX_3_0=$("$PYTHON_EXECUTABLE" -c 'from distutils.version import LooseVersion; import sphinx; print(LooseVersion(sphinx.__version__) < LooseVersion("3.1.0"))')
+    if [[ "$PYTHON_HAS_SPHINX_3_0" == "False" ]]; then
+        echo "$PYTHON_EXECUTABLE has Sphinx 3.1+ installed but it requires lower than 3.1. Skipping Sphinx build for now."
+        echo
+        return
+    fi
+
+    # TODO(SPARK-32391): Install pydata_sphinx_theme in Jenkins machines
+    PYTHON_HAS_THEME=$("$PYTHON_EXECUTABLE" -c 'import importlib.util; print(importlib.util.find_spec("pydata_sphinx_theme") is not None)')
+    if [[ "$PYTHON_HAS_THEME" == "False" ]]; then
+        echo "$PYTHON_EXECUTABLE does not have pydata_sphinx_theme installed. Skipping Sphinx build for now."
+        echo
+        return
+    fi
+
+    # TODO(SPARK-32666): Install nbsphinx in Jenkins machines
+    PYTHON_HAS_NBSPHINX=$("$PYTHON_EXECUTABLE" -c 'import importlib.util; print(importlib.util.find_spec("nbsphinx") is not None)')
+    if [[ "$PYTHON_HAS_NBSPHINX" == "False" ]]; then
+        echo "$PYTHON_EXECUTABLE does not have nbsphinx installed. Skipping Sphinx build for now."
+        echo
+        return
+    fi
+
+    # TODO(SPARK-32666): Install ipython in Jenkins machines
+    PYTHON_HAS_IPYTHON=$("$PYTHON_EXECUTABLE" -c 'import importlib.util; print(importlib.util.find_spec("IPython") is not None)')
+    if [[ "$PYTHON_HAS_IPYTHON" == "False" ]]; then
+        echo "$PYTHON_EXECUTABLE does not have ipython installed. Skipping Sphinx build for now."
+        echo
+        return
+    fi
+
+    # TODO(SPARK-33242): Install numpydoc in Jenkins machines
+    PYTHON_HAS_NUMPYDOC=$("$PYTHON_EXECUTABLE" -c 'import importlib.util; print(importlib.util.find_spec("numpydoc") is not None)')
+    if [[ "$PYTHON_HAS_NUMPYDOC" == "False" ]]; then
+        echo "$PYTHON_EXECUTABLE does not have numpydoc installed. Skipping Sphinx build for now."
         echo
         return
     fi
@@ -202,11 +275,13 @@ SPARK_ROOT_DIR="$(dirname "${SCRIPT_DIR}")"
 
 pushd "$SPARK_ROOT_DIR" &> /dev/null
 
-PYTHON_SOURCE="$(find . -name "*.py")"
+# skipping local ruby bundle directory from the search
+PYTHON_SOURCE="$(find . -path ./docs/.local_ruby_bundle -prune -false -o -name "*.py")"
 
 compile_python_test "$PYTHON_SOURCE"
 pycodestyle_test "$PYTHON_SOURCE"
 flake8_test
+mypy_test
 sphinx_test
 
 echo
diff --git a/dev/merge_spark_pr.py b/dev/merge_spark_pr.py
index 967cdace60dc9..2a40618228c5f 100755
--- a/dev/merge_spark_pr.py
+++ b/dev/merge_spark_pr.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
@@ -31,15 +31,9 @@
 import subprocess
 import sys
 import traceback
-if sys.version < '3':
-    input = raw_input  # noqa
-    from urllib2 import urlopen
-    from urllib2 import Request
-    from urllib2 import HTTPError
-else:
-    from urllib.request import urlopen
-    from urllib.request import Request
-    from urllib.error import HTTPError
+from urllib.request import urlopen
+from urllib.request import Request
+from urllib.error import HTTPError
 
 try:
     import jira.client
@@ -565,7 +559,7 @@ def main():
             print("JIRA_USERNAME and JIRA_PASSWORD not set")
             print("Exiting without trying to close the associated JIRA.")
     else:
-        print("Could not find jira-python library. Run 'sudo pip install jira' to install.")
+        print("Could not find jira-python library. Run 'sudo pip3 install jira' to install.")
         print("Exiting without trying to close the associated JIRA.")
 
 if __name__ == "__main__":
diff --git a/dev/mima b/dev/mima
index f324c5c00a45c..d214bb96e09a3 100755
--- a/dev/mima
+++ b/dev/mima
@@ -25,8 +25,8 @@ FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
 cd "$FWDIR"
 
 SPARK_PROFILES=${1:-"-Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"}
-TOOLS_CLASSPATH="$(build/sbt -DcopyDependencies=false "export tools/fullClasspath" | tail -n1)"
-OLD_DEPS_CLASSPATH="$(build/sbt -DcopyDependencies=false $SPARK_PROFILES "export oldDeps/fullClasspath" | tail -n1)"
+TOOLS_CLASSPATH="$(build/sbt -DcopyDependencies=false "export tools/fullClasspath" | grep jar | tail -n1)"
+OLD_DEPS_CLASSPATH="$(build/sbt -DcopyDependencies=false $SPARK_PROFILES "export oldDeps/fullClasspath" | grep jar | tail -n1)"
 
 rm -f .generated-mima*
 
diff --git a/dev/package-lock.json b/dev/package-lock.json
new file mode 100644
index 0000000000000..a57f45bcf7184
--- /dev/null
+++ b/dev/package-lock.json
@@ -0,0 +1,979 @@
+{
+    "requires": true,
+    "lockfileVersion": 1,
+    "dependencies": {
+        "@babel/code-frame": {
+            "version": "7.12.11",
+            "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.12.11.tgz",
+            "integrity": "sha512-Zt1yodBx1UcyiePMSkWnU4hPqhwq7hGi2nFL1LeA3EUl+q2LQx16MISgJ0+z7dnmgvP9QtIleuETGOiOH1RcIw==",
+            "dev": true,
+            "requires": {
+                "@babel/highlight": "^7.10.4"
+            }
+        },
+        "@babel/helper-validator-identifier": {
+            "version": "7.14.0",
+            "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.14.0.tgz",
+            "integrity": "sha512-V3ts7zMSu5lfiwWDVWzRDGIN+lnCEUdaXgtVHJgLb1rGaA6jMrtB9EmE7L18foXJIE8Un/A/h6NJfGQp/e1J4A==",
+            "dev": true
+        },
+        "@babel/highlight": {
+            "version": "7.14.0",
+            "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.14.0.tgz",
+            "integrity": "sha512-YSCOwxvTYEIMSGaBQb5kDDsCopDdiUGsqpatp3fOlI4+2HQSkTmEVWnVuySdAC5EWCqSWWTv0ib63RjR7dTBdg==",
+            "dev": true,
+            "requires": {
+                "@babel/helper-validator-identifier": "^7.14.0",
+                "chalk": "^2.0.0",
+                "js-tokens": "^4.0.0"
+            },
+            "dependencies": {
+                "chalk": {
+                    "version": "2.4.2",
+                    "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
+                    "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
+                    "dev": true,
+                    "requires": {
+                        "ansi-styles": "^3.2.1",
+                        "escape-string-regexp": "^1.0.5",
+                        "supports-color": "^5.3.0"
+                    }
+                }
+            }
+        },
+        "@eslint/eslintrc": {
+            "version": "0.4.0",
+            "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.0.tgz",
+            "integrity": "sha512-2ZPCc+uNbjV5ERJr+aKSPRwZgKd2z11x0EgLvb1PURmUrn9QNRXFqje0Ldq454PfAVyaJYyrDvvIKSFP4NnBog==",
+            "dev": true,
+            "requires": {
+                "ajv": "^6.12.4",
+                "debug": "^4.1.1",
+                "espree": "^7.3.0",
+                "globals": "^12.1.0",
+                "ignore": "^4.0.6",
+                "import-fresh": "^3.2.1",
+                "js-yaml": "^3.13.1",
+                "minimatch": "^3.0.4",
+                "strip-json-comments": "^3.1.1"
+            },
+            "dependencies": {
+                "globals": {
+                    "version": "12.4.0",
+                    "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz",
+                    "integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==",
+                    "dev": true,
+                    "requires": {
+                        "type-fest": "^0.8.1"
+                    }
+                }
+            }
+        },
+        "acorn": {
+            "version": "7.4.1",
+            "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz",
+            "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==",
+            "dev": true
+        },
+        "acorn-jsx": {
+            "version": "5.3.1",
+            "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.1.tgz",
+            "integrity": "sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng==",
+            "dev": true
+        },
+        "ajv": {
+            "version": "6.12.6",
+            "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
+            "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
+            "dev": true,
+            "requires": {
+                "fast-deep-equal": "^3.1.1",
+                "fast-json-stable-stringify": "^2.0.0",
+                "json-schema-traverse": "^0.4.1",
+                "uri-js": "^4.2.2"
+            }
+        },
+        "ansi-colors": {
+            "version": "4.1.1",
+            "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz",
+            "integrity": "sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==",
+            "dev": true
+        },
+        "ansi-regex": {
+            "version": "5.0.0",
+            "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
+            "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
+            "dev": true
+        },
+        "ansi-styles": {
+            "version": "3.2.1",
+            "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
+            "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
+            "dev": true,
+            "requires": {
+                "color-convert": "^1.9.0"
+            }
+        },
+        "argparse": {
+            "version": "1.0.10",
+            "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
+            "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
+            "dev": true,
+            "requires": {
+                "sprintf-js": "~1.0.2"
+            }
+        },
+        "astral-regex": {
+            "version": "2.0.0",
+            "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz",
+            "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==",
+            "dev": true
+        },
+        "balanced-match": {
+            "version": "1.0.2",
+            "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
+            "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
+            "dev": true
+        },
+        "brace-expansion": {
+            "version": "1.1.11",
+            "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+            "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+            "dev": true,
+            "requires": {
+                "balanced-match": "^1.0.0",
+                "concat-map": "0.0.1"
+            }
+        },
+        "callsites": {
+            "version": "3.1.0",
+            "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
+            "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
+            "dev": true
+        },
+        "chalk": {
+            "version": "4.1.1",
+            "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz",
+            "integrity": "sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg==",
+            "dev": true,
+            "requires": {
+                "ansi-styles": "^4.1.0",
+                "supports-color": "^7.1.0"
+            },
+            "dependencies": {
+                "ansi-styles": {
+                    "version": "4.3.0",
+                    "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+                    "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+                    "dev": true,
+                    "requires": {
+                        "color-convert": "^2.0.1"
+                    }
+                },
+                "color-convert": {
+                    "version": "2.0.1",
+                    "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+                    "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+                    "dev": true,
+                    "requires": {
+                        "color-name": "~1.1.4"
+                    }
+                },
+                "color-name": {
+                    "version": "1.1.4",
+                    "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+                    "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+                    "dev": true
+                },
+                "has-flag": {
+                    "version": "4.0.0",
+                    "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+                    "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+                    "dev": true
+                },
+                "supports-color": {
+                    "version": "7.2.0",
+                    "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+                    "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+                    "dev": true,
+                    "requires": {
+                        "has-flag": "^4.0.0"
+                    }
+                }
+            }
+        },
+        "color-convert": {
+            "version": "1.9.3",
+            "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
+            "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
+            "dev": true,
+            "requires": {
+                "color-name": "1.1.3"
+            }
+        },
+        "color-name": {
+            "version": "1.1.3",
+            "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
+            "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=",
+            "dev": true
+        },
+        "concat-map": {
+            "version": "0.0.1",
+            "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
+            "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
+            "dev": true
+        },
+        "cross-spawn": {
+            "version": "7.0.3",
+            "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
+            "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
+            "dev": true,
+            "requires": {
+                "path-key": "^3.1.0",
+                "shebang-command": "^2.0.0",
+                "which": "^2.0.1"
+            }
+        },
+        "debug": {
+            "version": "4.3.1",
+            "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz",
+            "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==",
+            "dev": true,
+            "requires": {
+                "ms": "2.1.2"
+            }
+        },
+        "deep-is": {
+            "version": "0.1.3",
+            "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz",
+            "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=",
+            "dev": true
+        },
+        "doctrine": {
+            "version": "3.0.0",
+            "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz",
+            "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==",
+            "dev": true,
+            "requires": {
+                "esutils": "^2.0.2"
+            }
+        },
+        "emoji-regex": {
+            "version": "8.0.0",
+            "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+            "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
+            "dev": true
+        },
+        "enquirer": {
+            "version": "2.3.6",
+            "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz",
+            "integrity": "sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg==",
+            "dev": true,
+            "requires": {
+                "ansi-colors": "^4.1.1"
+            }
+        },
+        "escape-string-regexp": {
+            "version": "1.0.5",
+            "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
+            "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=",
+            "dev": true
+        },
+        "eslint": {
+            "version": "7.25.0",
+            "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.25.0.tgz",
+            "integrity": "sha512-TVpSovpvCNpLURIScDRB6g5CYu/ZFq9GfX2hLNIV4dSBKxIWojeDODvYl3t0k0VtMxYeR8OXPCFE5+oHMlGfhw==",
+            "dev": true,
+            "requires": {
+                "@babel/code-frame": "7.12.11",
+                "@eslint/eslintrc": "^0.4.0",
+                "ajv": "^6.10.0",
+                "chalk": "^4.0.0",
+                "cross-spawn": "^7.0.2",
+                "debug": "^4.0.1",
+                "doctrine": "^3.0.0",
+                "enquirer": "^2.3.5",
+                "eslint-scope": "^5.1.1",
+                "eslint-utils": "^2.1.0",
+                "eslint-visitor-keys": "^2.0.0",
+                "espree": "^7.3.1",
+                "esquery": "^1.4.0",
+                "esutils": "^2.0.2",
+                "file-entry-cache": "^6.0.1",
+                "functional-red-black-tree": "^1.0.1",
+                "glob-parent": "^5.0.0",
+                "globals": "^13.6.0",
+                "ignore": "^4.0.6",
+                "import-fresh": "^3.0.0",
+                "imurmurhash": "^0.1.4",
+                "is-glob": "^4.0.0",
+                "js-yaml": "^3.13.1",
+                "json-stable-stringify-without-jsonify": "^1.0.1",
+                "levn": "^0.4.1",
+                "lodash": "^4.17.21",
+                "minimatch": "^3.0.4",
+                "natural-compare": "^1.4.0",
+                "optionator": "^0.9.1",
+                "progress": "^2.0.0",
+                "regexpp": "^3.1.0",
+                "semver": "^7.2.1",
+                "strip-ansi": "^6.0.0",
+                "strip-json-comments": "^3.1.0",
+                "table": "^6.0.4",
+                "text-table": "^0.2.0",
+                "v8-compile-cache": "^2.0.3"
+            }
+        },
+        "eslint-scope": {
+            "version": "5.1.1",
+            "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz",
+            "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==",
+            "dev": true,
+            "requires": {
+                "esrecurse": "^4.3.0",
+                "estraverse": "^4.1.1"
+            }
+        },
+        "eslint-utils": {
+            "version": "2.1.0",
+            "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.1.0.tgz",
+            "integrity": "sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==",
+            "dev": true,
+            "requires": {
+                "eslint-visitor-keys": "^1.1.0"
+            },
+            "dependencies": {
+                "eslint-visitor-keys": {
+                    "version": "1.3.0",
+                    "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
+                    "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
+                    "dev": true
+                }
+            }
+        },
+        "eslint-visitor-keys": {
+            "version": "2.1.0",
+            "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz",
+            "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==",
+            "dev": true
+        },
+        "espree": {
+            "version": "7.3.1",
+            "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.1.tgz",
+            "integrity": "sha512-v3JCNCE64umkFpmkFGqzVKsOT0tN1Zr+ueqLZfpV1Ob8e+CEgPWa+OxCoGH3tnhimMKIaBm4m/vaRpJ/krRz2g==",
+            "dev": true,
+            "requires": {
+                "acorn": "^7.4.0",
+                "acorn-jsx": "^5.3.1",
+                "eslint-visitor-keys": "^1.3.0"
+            },
+            "dependencies": {
+                "eslint-visitor-keys": {
+                    "version": "1.3.0",
+                    "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
+                    "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
+                    "dev": true
+                }
+            }
+        },
+        "esprima": {
+            "version": "4.0.1",
+            "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
+            "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
+            "dev": true
+        },
+        "esquery": {
+            "version": "1.4.0",
+            "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz",
+            "integrity": "sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w==",
+            "dev": true,
+            "requires": {
+                "estraverse": "^5.1.0"
+            },
+            "dependencies": {
+                "estraverse": {
+                    "version": "5.2.0",
+                    "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz",
+                    "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==",
+                    "dev": true
+                }
+            }
+        },
+        "esrecurse": {
+            "version": "4.3.0",
+            "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz",
+            "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==",
+            "dev": true,
+            "requires": {
+                "estraverse": "^5.2.0"
+            },
+            "dependencies": {
+                "estraverse": {
+                    "version": "5.2.0",
+                    "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz",
+                    "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==",
+                    "dev": true
+                }
+            }
+        },
+        "estraverse": {
+            "version": "4.3.0",
+            "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz",
+            "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==",
+            "dev": true
+        },
+        "esutils": {
+            "version": "2.0.3",
+            "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz",
+            "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==",
+            "dev": true
+        },
+        "fast-deep-equal": {
+            "version": "3.1.3",
+            "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
+            "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
+            "dev": true
+        },
+        "fast-json-stable-stringify": {
+            "version": "2.1.0",
+            "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
+            "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==",
+            "dev": true
+        },
+        "fast-levenshtein": {
+            "version": "2.0.6",
+            "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz",
+            "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=",
+            "dev": true
+        },
+        "file-entry-cache": {
+            "version": "6.0.1",
+            "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz",
+            "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==",
+            "dev": true,
+            "requires": {
+                "flat-cache": "^3.0.4"
+            }
+        },
+        "flat-cache": {
+            "version": "3.0.4",
+            "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz",
+            "integrity": "sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg==",
+            "dev": true,
+            "requires": {
+                "flatted": "^3.1.0",
+                "rimraf": "^3.0.2"
+            }
+        },
+        "flatted": {
+            "version": "3.1.1",
+            "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.1.1.tgz",
+            "integrity": "sha512-zAoAQiudy+r5SvnSw3KJy5os/oRJYHzrzja/tBDqrZtNhUw8bt6y8OBzMWcjWr+8liV8Eb6yOhw8WZ7VFZ5ZzA==",
+            "dev": true
+        },
+        "fs.realpath": {
+            "version": "1.0.0",
+            "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
+            "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
+            "dev": true
+        },
+        "functional-red-black-tree": {
+            "version": "1.0.1",
+            "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz",
+            "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=",
+            "dev": true
+        },
+        "glob": {
+            "version": "7.1.6",
+            "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
+            "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
+            "dev": true,
+            "requires": {
+                "fs.realpath": "^1.0.0",
+                "inflight": "^1.0.4",
+                "inherits": "2",
+                "minimatch": "^3.0.4",
+                "once": "^1.3.0",
+                "path-is-absolute": "^1.0.0"
+            }
+        },
+        "glob-parent": {
+            "version": "5.1.2",
+            "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
+            "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
+            "dev": true,
+            "requires": {
+                "is-glob": "^4.0.1"
+            }
+        },
+        "globals": {
+            "version": "13.8.0",
+            "resolved": "https://registry.npmjs.org/globals/-/globals-13.8.0.tgz",
+            "integrity": "sha512-rHtdA6+PDBIjeEvA91rpqzEvk/k3/i7EeNQiryiWuJH0Hw9cpyJMAt2jtbAwUaRdhD+573X4vWw6IcjKPasi9Q==",
+            "dev": true,
+            "requires": {
+                "type-fest": "^0.20.2"
+            },
+            "dependencies": {
+                "type-fest": {
+                    "version": "0.20.2",
+                    "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz",
+                    "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==",
+                    "dev": true
+                }
+            }
+        },
+        "has-flag": {
+            "version": "3.0.0",
+            "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
+            "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
+            "dev": true
+        },
+        "ignore": {
+            "version": "4.0.6",
+            "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz",
+            "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==",
+            "dev": true
+        },
+        "import-fresh": {
+            "version": "3.3.0",
+            "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz",
+            "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==",
+            "dev": true,
+            "requires": {
+                "parent-module": "^1.0.0",
+                "resolve-from": "^4.0.0"
+            }
+        },
+        "imurmurhash": {
+            "version": "0.1.4",
+            "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
+            "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=",
+            "dev": true
+        },
+        "inflight": {
+            "version": "1.0.6",
+            "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
+            "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
+            "dev": true,
+            "requires": {
+                "once": "^1.3.0",
+                "wrappy": "1"
+            }
+        },
+        "inherits": {
+            "version": "2.0.4",
+            "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
+            "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
+            "dev": true
+        },
+        "is-extglob": {
+            "version": "2.1.1",
+            "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
+            "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=",
+            "dev": true
+        },
+        "is-fullwidth-code-point": {
+            "version": "3.0.0",
+            "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+            "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+            "dev": true
+        },
+        "is-glob": {
+            "version": "4.0.1",
+            "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz",
+            "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==",
+            "dev": true,
+            "requires": {
+                "is-extglob": "^2.1.1"
+            }
+        },
+        "isexe": {
+            "version": "2.0.0",
+            "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
+            "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=",
+            "dev": true
+        },
+        "js-tokens": {
+            "version": "4.0.0",
+            "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
+            "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
+            "dev": true
+        },
+        "js-yaml": {
+            "version": "3.14.1",
+            "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz",
+            "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==",
+            "dev": true,
+            "requires": {
+                "argparse": "^1.0.7",
+                "esprima": "^4.0.0"
+            }
+        },
+        "json-schema-traverse": {
+            "version": "0.4.1",
+            "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
+            "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
+            "dev": true
+        },
+        "json-stable-stringify-without-jsonify": {
+            "version": "1.0.1",
+            "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz",
+            "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=",
+            "dev": true
+        },
+        "levn": {
+            "version": "0.4.1",
+            "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz",
+            "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==",
+            "dev": true,
+            "requires": {
+                "prelude-ls": "^1.2.1",
+                "type-check": "~0.4.0"
+            }
+        },
+        "lodash": {
+            "version": "4.17.21",
+            "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
+            "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==",
+            "dev": true
+        },
+        "lodash.clonedeep": {
+            "version": "4.5.0",
+            "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz",
+            "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=",
+            "dev": true
+        },
+        "lodash.flatten": {
+            "version": "4.4.0",
+            "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz",
+            "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=",
+            "dev": true
+        },
+        "lodash.truncate": {
+            "version": "4.4.2",
+            "resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz",
+            "integrity": "sha1-WjUNoLERO4N+z//VgSy+WNbq4ZM=",
+            "dev": true
+        },
+        "lru-cache": {
+            "version": "6.0.0",
+            "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+            "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+            "dev": true,
+            "requires": {
+                "yallist": "^4.0.0"
+            }
+        },
+        "minimatch": {
+            "version": "3.0.4",
+            "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
+            "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
+            "dev": true,
+            "requires": {
+                "brace-expansion": "^1.1.7"
+            }
+        },
+        "ms": {
+            "version": "2.1.2",
+            "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+            "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
+            "dev": true
+        },
+        "natural-compare": {
+            "version": "1.4.0",
+            "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
+            "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=",
+            "dev": true
+        },
+        "once": {
+            "version": "1.4.0",
+            "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+            "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
+            "dev": true,
+            "requires": {
+                "wrappy": "1"
+            }
+        },
+        "optionator": {
+            "version": "0.9.1",
+            "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz",
+            "integrity": "sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==",
+            "dev": true,
+            "requires": {
+                "deep-is": "^0.1.3",
+                "fast-levenshtein": "^2.0.6",
+                "levn": "^0.4.1",
+                "prelude-ls": "^1.2.1",
+                "type-check": "^0.4.0",
+                "word-wrap": "^1.2.3"
+            }
+        },
+        "parent-module": {
+            "version": "1.0.1",
+            "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
+            "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==",
+            "dev": true,
+            "requires": {
+                "callsites": "^3.0.0"
+            }
+        },
+        "path-is-absolute": {
+            "version": "1.0.1",
+            "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
+            "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
+            "dev": true
+        },
+        "path-key": {
+            "version": "3.1.1",
+            "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
+            "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
+            "dev": true
+        },
+        "prelude-ls": {
+            "version": "1.2.1",
+            "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz",
+            "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==",
+            "dev": true
+        },
+        "progress": {
+            "version": "2.0.3",
+            "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
+            "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==",
+            "dev": true
+        },
+        "punycode": {
+            "version": "2.1.1",
+            "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
+            "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==",
+            "dev": true
+        },
+        "regexpp": {
+            "version": "3.1.0",
+            "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.1.0.tgz",
+            "integrity": "sha512-ZOIzd8yVsQQA7j8GCSlPGXwg5PfmA1mrq0JP4nGhh54LaKN3xdai/vHUDu74pKwV8OxseMS65u2NImosQcSD0Q==",
+            "dev": true
+        },
+        "require-from-string": {
+            "version": "2.0.2",
+            "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz",
+            "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==",
+            "dev": true
+        },
+        "resolve-from": {
+            "version": "4.0.0",
+            "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
+            "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==",
+            "dev": true
+        },
+        "rimraf": {
+            "version": "3.0.2",
+            "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
+            "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
+            "dev": true,
+            "requires": {
+                "glob": "^7.1.3"
+            }
+        },
+        "semver": {
+            "version": "7.3.5",
+            "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz",
+            "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==",
+            "dev": true,
+            "requires": {
+                "lru-cache": "^6.0.0"
+            }
+        },
+        "shebang-command": {
+            "version": "2.0.0",
+            "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
+            "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
+            "dev": true,
+            "requires": {
+                "shebang-regex": "^3.0.0"
+            }
+        },
+        "shebang-regex": {
+            "version": "3.0.0",
+            "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
+            "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
+            "dev": true
+        },
+        "slice-ansi": {
+            "version": "4.0.0",
+            "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz",
+            "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==",
+            "dev": true,
+            "requires": {
+                "ansi-styles": "^4.0.0",
+                "astral-regex": "^2.0.0",
+                "is-fullwidth-code-point": "^3.0.0"
+            },
+            "dependencies": {
+                "ansi-styles": {
+                    "version": "4.3.0",
+                    "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+                    "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+                    "dev": true,
+                    "requires": {
+                        "color-convert": "^2.0.1"
+                    }
+                },
+                "color-convert": {
+                    "version": "2.0.1",
+                    "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+                    "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+                    "dev": true,
+                    "requires": {
+                        "color-name": "~1.1.4"
+                    }
+                },
+                "color-name": {
+                    "version": "1.1.4",
+                    "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+                    "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+                    "dev": true
+                }
+            }
+        },
+        "sprintf-js": {
+            "version": "1.0.3",
+            "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
+            "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=",
+            "dev": true
+        },
+        "string-width": {
+            "version": "4.2.2",
+            "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz",
+            "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==",
+            "dev": true,
+            "requires": {
+                "emoji-regex": "^8.0.0",
+                "is-fullwidth-code-point": "^3.0.0",
+                "strip-ansi": "^6.0.0"
+            }
+        },
+        "strip-ansi": {
+            "version": "6.0.0",
+            "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
+            "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
+            "dev": true,
+            "requires": {
+                "ansi-regex": "^5.0.0"
+            }
+        },
+        "strip-json-comments": {
+            "version": "3.1.1",
+            "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz",
+            "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==",
+            "dev": true
+        },
+        "supports-color": {
+            "version": "5.5.0",
+            "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
+            "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
+            "dev": true,
+            "requires": {
+                "has-flag": "^3.0.0"
+            }
+        },
+        "table": {
+            "version": "6.6.0",
+            "resolved": "https://registry.npmjs.org/table/-/table-6.6.0.tgz",
+            "integrity": "sha512-iZMtp5tUvcnAdtHpZTWLPF0M7AgiQsURR2DwmxnJwSy8I3+cY+ozzVvYha3BOLG2TB+L0CqjIz+91htuj6yCXg==",
+            "dev": true,
+            "requires": {
+                "ajv": "^8.0.1",
+                "lodash.clonedeep": "^4.5.0",
+                "lodash.flatten": "^4.4.0",
+                "lodash.truncate": "^4.4.2",
+                "slice-ansi": "^4.0.0",
+                "string-width": "^4.2.0",
+                "strip-ansi": "^6.0.0"
+            },
+            "dependencies": {
+                "ajv": {
+                    "version": "8.2.0",
+                    "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.2.0.tgz",
+                    "integrity": "sha512-WSNGFuyWd//XO8n/m/EaOlNLtO0yL8EXT/74LqT4khdhpZjP7lkj/kT5uwRmGitKEVp/Oj7ZUHeGfPtgHhQ5CA==",
+                    "dev": true,
+                    "requires": {
+                        "fast-deep-equal": "^3.1.1",
+                        "json-schema-traverse": "^1.0.0",
+                        "require-from-string": "^2.0.2",
+                        "uri-js": "^4.2.2"
+                    }
+                },
+                "json-schema-traverse": {
+                    "version": "1.0.0",
+                    "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
+                    "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",
+                    "dev": true
+                }
+            }
+        },
+        "text-table": {
+            "version": "0.2.0",
+            "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",
+            "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=",
+            "dev": true
+        },
+        "type-check": {
+            "version": "0.4.0",
+            "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
+            "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==",
+            "dev": true,
+            "requires": {
+                "prelude-ls": "^1.2.1"
+            }
+        },
+        "type-fest": {
+            "version": "0.8.1",
+            "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz",
+            "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==",
+            "dev": true
+        },
+        "uri-js": {
+            "version": "4.4.1",
+            "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz",
+            "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==",
+            "dev": true,
+            "requires": {
+                "punycode": "^2.1.0"
+            }
+        },
+        "v8-compile-cache": {
+            "version": "2.3.0",
+            "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz",
+            "integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==",
+            "dev": true
+        },
+        "which": {
+            "version": "2.0.2",
+            "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
+            "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
+            "dev": true,
+            "requires": {
+                "isexe": "^2.0.0"
+            }
+        },
+        "word-wrap": {
+            "version": "1.2.3",
+            "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",
+            "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==",
+            "dev": true
+        },
+        "wrappy": {
+            "version": "1.0.2",
+            "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+            "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
+            "dev": true
+        },
+        "yallist": {
+            "version": "4.0.0",
+            "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+            "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+            "dev": true
+        }
+    }
+}
diff --git a/dev/package.json b/dev/package.json
new file mode 100644
index 0000000000000..0391a3983f78f
--- /dev/null
+++ b/dev/package.json
@@ -0,0 +1,5 @@
+{
+  "devDependencies": {
+    "eslint": "^7.25.0"
+  }
+}
diff --git a/dev/pip-sanity-check.py b/dev/pip-sanity-check.py
index e9f10233b12b7..469e27b78b40d 100644
--- a/dev/pip-sanity-check.py
+++ b/dev/pip-sanity-check.py
@@ -16,7 +16,6 @@
 #
 
 from pyspark.sql import SparkSession
-from pyspark.mllib.linalg import *
 import sys
 
 if __name__ == "__main__":
diff --git a/dev/requirements.txt b/dev/requirements.txt
index baea9213dbc97..f0bdc797b7033 100644
--- a/dev/requirements.txt
+++ b/dev/requirements.txt
@@ -1,5 +1,8 @@
 flake8==3.5.0
-jira==1.0.3
+jira==2.0.0
 PyGithub==1.26.0
-Unidecode==0.04.19
 sphinx
+pydata_sphinx_theme
+ipython
+nbsphinx
+numpydoc
diff --git a/dev/run-pip-tests b/dev/run-pip-tests
index 470f21e69d46a..cb64e88318a89 100755
--- a/dev/run-pip-tests
+++ b/dev/run-pip-tests
@@ -63,11 +63,15 @@ fi
 PYSPARK_VERSION=$(python3 -c "exec(open('python/pyspark/version.py').read());print(__version__)")
 PYSPARK_DIST="$FWDIR/python/dist/pyspark-$PYSPARK_VERSION.tar.gz"
 # The pip install options we use for all the pip commands
-PIP_OPTIONS="--upgrade --no-cache-dir --force-reinstall "
+PIP_OPTIONS="--upgrade --no-cache-dir --force-reinstall"
 # Test both regular user and edit/dev install modes.
 PIP_COMMANDS=("pip install $PIP_OPTIONS $PYSPARK_DIST"
 	      "pip install $PIP_OPTIONS -e python/")
 
+# Jenkins has PySpark installed under user sitepackages shared for some reasons.
+# In this test, explicitly exclude user sitepackages to prevent side effects
+export PYTHONNOUSERSITE=1
+
 for python in "${PYTHON_EXECS[@]}"; do
   for install_command in "${PIP_COMMANDS[@]}"; do
     echo "Testing pip installation with python $python"
@@ -77,7 +81,7 @@ for python in "${PYTHON_EXECS[@]}"; do
     rm -rf "$VIRTUALENV_PATH"
     if [ -n "$USE_CONDA" ]; then
       conda create -y -p "$VIRTUALENV_PATH" python=$python numpy pandas pip setuptools
-      source activate "$VIRTUALENV_PATH"
+      source activate "$VIRTUALENV_PATH" || conda activate "$VIRTUALENV_PATH"
     else
       mkdir -p "$VIRTUALENV_PATH"
       virtualenv --python=$python "$VIRTUALENV_PATH"
@@ -120,7 +124,7 @@ for python in "${PYTHON_EXECS[@]}"; do
 
     # conda / virtualenv environments need to be deactivated differently
     if [ -n "$USE_CONDA" ]; then
-      source deactivate
+      source deactivate || conda deactivate
     else
       deactivate
     fi
diff --git a/dev/run-tests-jenkins b/dev/run-tests-jenkins
index c3adc696a5122..c155d4ea3f076 100755
--- a/dev/run-tests-jenkins
+++ b/dev/run-tests-jenkins
@@ -26,6 +26,7 @@ FWDIR="$( cd "$( dirname "$0" )/.." && pwd )"
 cd "$FWDIR"
 
 export PATH=/home/anaconda/envs/py36/bin:$PATH
+export LANG="en_US.UTF-8"
 
 PYTHON_VERSION_CHECK=$(python3 -c 'import sys; print(sys.version_info < (3, 6, 0))')
 if [[ "$PYTHON_VERSION_CHECK" == "True" ]]; then
diff --git a/dev/run-tests-jenkins.py b/dev/run-tests-jenkins.py
index 72e32d4e16e14..4309a74773e89 100755
--- a/dev/run-tests-jenkins.py
+++ b/dev/run-tests-jenkins.py
@@ -22,15 +22,9 @@
 import json
 import functools
 import subprocess
-if sys.version < '3':
-    from urllib2 import urlopen
-    from urllib2 import Request
-    from urllib2 import HTTPError, URLError
-else:
-    from urllib.request import urlopen
-    from urllib.request import Request
-    from urllib.error import HTTPError, URLError
-
+from urllib.request import urlopen
+from urllib.request import Request
+from urllib.error import HTTPError, URLError
 
 from sparktestsupport import SPARK_HOME, ERROR_CODES
 from sparktestsupport.shellutils import run_cmd
@@ -44,7 +38,7 @@ def print_err(msg):
 
 
 def post_message_to_github(msg, ghprb_pull_id):
-    print("Attempting to post to Github...")
+    print("Attempting to post to GitHub...")
 
     api_url = os.getenv("GITHUB_API_BASE", "https://api.github.com/repos/apache/spark")
     url = api_url + "/issues/" + ghprb_pull_id + "/comments"
@@ -63,12 +57,12 @@ def post_message_to_github(msg, ghprb_pull_id):
         if response.getcode() == 201:
             print(" > Post successful.")
     except HTTPError as http_e:
-        print_err("Failed to post message to Github.")
+        print_err("Failed to post message to GitHub.")
         print_err(" > http_code: %s" % http_e.code)
         print_err(" > api_response: %s" % http_e.read())
         print_err(" > data: %s" % posted_message)
     except URLError as url_e:
-        print_err("Failed to post message to Github.")
+        print_err("Failed to post message to GitHub.")
         print_err(" > urllib_status: %s" % url_e.reason[1])
         print_err(" > data: %s" % posted_message)
 
@@ -95,7 +89,7 @@ def run_pr_checks(pr_tests, ghprb_actual_commit, sha1):
     """
     Executes a set of pull request checks to ease development and report issues with various
     components such as style, linting, dependencies, compatibilities, etc.
-    @return a list of messages to post back to Github
+    @return a list of messages to post back to GitHub
     """
     # Ensure we save off the current HEAD to revert to
     current_pr_head = run_cmd(['git', 'rev-parse', 'HEAD'], return_output=True).strip()
@@ -115,7 +109,7 @@ def run_tests(tests_timeout):
     """
     Runs the `dev/run-tests` script and responds with the correct error message
     under the various failure scenarios.
-    @return a tuple containing the test result code and the result note to post to Github
+    @return a tuple containing the test result code and the result note to post to GitHub
     """
 
     test_result_code = subprocess.Popen(['timeout',
@@ -181,8 +175,6 @@ def main():
     if "test-hadoop3.2" in ghprb_pull_title:
         os.environ["AMPLAB_JENKINS_BUILD_PROFILE"] = "hadoop3.2"
     # Switch the Hive profile based on the PR title:
-    if "test-hive1.2" in ghprb_pull_title:
-        os.environ["AMPLAB_JENKINS_BUILD_HIVE_PROFILE"] = "hive1.2"
     if "test-hive2.3" in ghprb_pull_title:
         os.environ["AMPLAB_JENKINS_BUILD_HIVE_PROFILE"] = "hive2.3"
 
@@ -198,7 +190,7 @@ def main():
     # format: http://linux.die.net/man/1/timeout
     # must be less than the timeout configured on Jenkins. Usually Jenkins's timeout is higher
     # then this. Please consult with the build manager or a committer when it should be increased.
-    tests_timeout = "400m"
+    tests_timeout = "500m"
 
     # Array to capture all test names to run on the pull request. These tests are represented
     # by their file equivalents in the dev/tests/ directory.
@@ -206,16 +198,16 @@ def main():
     # To write a PR test:
     #   * the file must reside within the dev/tests directory
     #   * be an executable bash script
-    #   * accept three arguments on the command line, the first being the Github PR long commit
-    #     hash, the second the Github SHA1 hash, and the final the current PR hash
+    #   * accept three arguments on the command line, the first being the GitHub PR long commit
+    #     hash, the second the GitHub SHA1 hash, and the final the current PR hash
     #   * and, lastly, return string output to be included in the pr message output that will
-    #     be posted to Github
+    #     be posted to GitHub
     pr_tests = [
         "pr_merge_ability",
         "pr_public_classes"
     ]
 
-    # `bind_message_base` returns a function to generate messages for Github posting
+    # `bind_message_base` returns a function to generate messages for GitHub posting
     github_message = functools.partial(pr_message,
                                        build_display_name,
                                        build_url,
diff --git a/dev/run-tests.py b/dev/run-tests.py
index 5255a77ec2081..d5d3445bd6125 100755
--- a/dev/run-tests.py
+++ b/dev/run-tests.py
@@ -20,13 +20,11 @@
 import itertools
 from argparse import ArgumentParser
 import os
-import random
 import re
 import sys
 import subprocess
 import glob
 import shutil
-from collections import namedtuple
 
 from sparktestsupport import SPARK_HOME, USER_HOME, ERROR_CODES
 from sparktestsupport.shellutils import exit_from_command_with_retcode, run_cmd, rm_r, which
@@ -43,19 +41,21 @@ def determine_modules_for_files(filenames):
     """
     Given a list of filenames, return the set of modules that contain those files.
     If a file is not associated with a more specific submodule, then this method will consider that
-    file to belong to the 'root' module. GitHub Action and Appveyor files are ignored.
+    file to belong to the 'root' module. `.github` directory is counted only in GitHub Actions,
+    and `appveyor.yml` is always ignored because this file is dedicated only to AppVeyor builds.
 
     >>> sorted(x.name for x in determine_modules_for_files(["python/pyspark/a.py", "sql/core/foo"]))
     ['pyspark-core', 'sql']
     >>> [x.name for x in determine_modules_for_files(["file_not_matched_by_any_subproject"])]
     ['root']
-    >>> [x.name for x in determine_modules_for_files( \
-            [".github/workflows/master.yml", "appveyor.yml"])]
+    >>> [x.name for x in determine_modules_for_files(["appveyor.yml"])]
     []
     """
     changed_modules = set()
     for filename in filenames:
-        if filename in (".github/workflows/master.yml", "appveyor.yml"):
+        if filename in ("appveyor.yml",):
+            continue
+        if ("GITHUB_ACTIONS" not in os.environ) and filename.startswith(".github"):
             continue
         matched_at_least_one_module = False
         for module in modules.all_modules:
@@ -101,28 +101,52 @@ def setup_test_environ(environ):
         os.environ[k] = v
 
 
-def determine_modules_to_test(changed_modules):
+def determine_modules_to_test(changed_modules, deduplicated=True):
     """
     Given a set of modules that have changed, compute the transitive closure of those modules'
     dependent modules in order to determine the set of modules that should be tested.
 
     Returns a topologically-sorted list of modules (ties are broken by sorting on module names).
+    If ``deduplicated`` is disabled, the modules are returned without tacking the deduplication
+    by dependencies into account.
 
     >>> [x.name for x in determine_modules_to_test([modules.root])]
     ['root']
     >>> [x.name for x in determine_modules_to_test([modules.build])]
     ['root']
+    >>> [x.name for x in determine_modules_to_test([modules.core])]
+    ['root']
+    >>> [x.name for x in determine_modules_to_test([modules.launcher])]
+    ['root']
     >>> [x.name for x in determine_modules_to_test([modules.graphx])]
     ['graphx', 'examples']
-    >>> x = [x.name for x in determine_modules_to_test([modules.sql])]
-    >>> x # doctest: +NORMALIZE_WHITESPACE
+    >>> [x.name for x in determine_modules_to_test([modules.sql])]
+    ... # doctest: +NORMALIZE_WHITESPACE
     ['sql', 'avro', 'hive', 'mllib', 'sql-kafka-0-10', 'examples', 'hive-thriftserver',
-     'pyspark-sql', 'repl', 'sparkr', 'pyspark-mllib', 'pyspark-ml']
+     'pyspark-sql', 'repl', 'sparkr', 'pyspark-mllib', 'pyspark-pandas', 'pyspark-ml']
+    >>> sorted([x.name for x in determine_modules_to_test(
+    ...     [modules.sparkr, modules.sql], deduplicated=False)])
+    ... # doctest: +NORMALIZE_WHITESPACE
+    ['avro', 'examples', 'hive', 'hive-thriftserver', 'mllib', 'pyspark-ml',
+     'pyspark-mllib', 'pyspark-pandas', 'pyspark-sql', 'repl', 'sparkr', 'sql', 'sql-kafka-0-10']
+    >>> sorted([x.name for x in determine_modules_to_test(
+    ...     [modules.sql, modules.core], deduplicated=False)])
+    ... # doctest: +NORMALIZE_WHITESPACE
+    ['avro', 'catalyst', 'core', 'examples', 'graphx', 'hive', 'hive-thriftserver',
+     'mllib', 'mllib-local', 'pyspark-core', 'pyspark-ml', 'pyspark-mllib', 'pyspark-pandas',
+     'pyspark-resource', 'pyspark-sql', 'pyspark-streaming', 'repl', 'root',
+     'sparkr', 'sql', 'sql-kafka-0-10', 'streaming', 'streaming-kafka-0-10',
+     'streaming-kinesis-asl']
     """
     modules_to_test = set()
     for module in changed_modules:
-        modules_to_test = modules_to_test.union(determine_modules_to_test(module.dependent_modules))
+        modules_to_test = modules_to_test.union(
+            determine_modules_to_test(module.dependent_modules, deduplicated))
     modules_to_test = modules_to_test.union(set(changed_modules))
+
+    if not deduplicated:
+        return modules_to_test
+
     # If we need to run all of the tests, then we should short-circuit and return 'root'
     if modules.root in modules_to_test:
         return [modules.root]
@@ -215,37 +239,28 @@ def run_sparkr_style_checks():
 
 def build_spark_documentation():
     set_title_and_block("Building Spark Documentation", "BLOCK_DOCUMENTATION")
-    os.environ["PRODUCTION"] = "1 jekyll build"
+    os.environ["PRODUCTION"] = "1"
 
     os.chdir(os.path.join(SPARK_HOME, "docs"))
 
-    jekyll_bin = which("jekyll")
+    bundle_bin = which("bundle")
 
-    if not jekyll_bin:
-        print("[error] Cannot find a version of `jekyll` on the system; please",
-              " install one and retry to build documentation.")
+    if not bundle_bin:
+        print("[error] Cannot find a version of `bundle` on the system; please",
+              " install one with `gem install bundler` and retry to build documentation.")
         sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
     else:
-        run_cmd([jekyll_bin, "build"])
+        run_cmd([bundle_bin, "install"])
+        run_cmd([bundle_bin, "exec", "jekyll", "build"])
 
     os.chdir(SPARK_HOME)
 
 
-def get_zinc_port():
-    """
-    Get a randomized port on which to start Zinc
-    """
-    return random.randrange(3030, 4030)
-
-
 def exec_maven(mvn_args=()):
     """Will call Maven in the current directory with the list of mvn_args passed
     in and returns the subprocess for any further processing"""
 
-    zinc_port = get_zinc_port()
-    os.environ["ZINC_PORT"] = "%s" % zinc_port
-    zinc_flag = "-DzincPort=%s" % zinc_port
-    flags = [os.path.join(SPARK_HOME, "build", "mvn"), zinc_flag]
+    flags = [os.path.join(SPARK_HOME, "build", "mvn")]
     run_cmd(flags + mvn_args)
 
 
@@ -303,7 +318,6 @@ def get_hive_profiles(hive_version):
     """
 
     sbt_maven_hive_profiles = {
-        "hive1.2": ["-Phive-1.2"],
         "hive2.3": ["-Phive-2.3"],
     }
 
@@ -363,7 +377,8 @@ def build_spark_assembly_sbt(extra_profiles, checkstyle=False):
     if checkstyle:
         run_java_style_checks(build_profiles)
 
-    build_spark_unidoc_sbt(extra_profiles)
+    if not os.environ.get("AMPLAB_JENKINS"):
+        build_spark_unidoc_sbt(extra_profiles)
 
 
 def build_apache_spark(build_tool, extra_profiles):
@@ -415,7 +430,7 @@ def run_scala_tests_sbt(test_modules, test_profiles):
     exec_sbt(profiles_and_goals)
 
 
-def run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags):
+def run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags, included_tags):
     """Function to properly execute all tests passed in as a set from the
     `determine_test_suites` function"""
     set_title_and_block("Running Spark unit tests", "BLOCK_SPARK_UNIT_TESTS")
@@ -425,6 +440,8 @@ def run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags):
     test_profiles = extra_profiles + \
         list(set(itertools.chain.from_iterable(m.build_profile_flags for m in test_modules)))
 
+    if included_tags:
+        test_profiles += ['-Dtest.include.tags=' + ",".join(included_tags)]
     if excluded_tags:
         test_profiles += ['-Dtest.exclude.tags=' + ",".join(excluded_tags)]
 
@@ -456,6 +473,12 @@ def run_python_tests(test_modules, parallelism, with_coverage=False):
     if test_modules != [modules.root]:
         command.append("--modules=%s" % ','.join(m.name for m in test_modules))
     command.append("--parallelism=%i" % parallelism)
+    if "GITHUB_ACTIONS" in os.environ:
+        # See SPARK-33565. Python 3.8 was temporarily removed as its default Python executables
+        # to test because of Jenkins environment issue. Once Jenkins has Python 3.8 to test,
+        # we should remove this change back and add python3.8 into python/run-tests.py script.
+        command.append("--python-executable=%s" % ','.join(
+            x for x in ["python3.6", "python3.8", "pypy3"] if which(x)))
     run_cmd(command)
 
     if with_coverage:
@@ -488,10 +511,13 @@ def post_python_tests_results():
         # 6. Commit current HTMLs.
         run_cmd([
             "git",
+            "-c",
+            "user.name='Apache Spark Test Account'",
+            "-c",
+            "user.email='sparktestacc@gmail.com'",
             "commit",
             "-am",
-            "Coverage report at latest commit in Apache Spark",
-            '--author="Apache Spark Test Account <sparktestacc@gmail.com>"'])
+            "Coverage report at latest commit in Apache Spark"])
         # 7. Delete the old branch.
         run_cmd(["git", "branch", "-D", "gh-pages"])
         # 8. Rename the temporary branch to master.
@@ -532,6 +558,24 @@ def parse_opts():
         "-p", "--parallelism", type=int, default=8,
         help="The number of suites to test in parallel (default %(default)d)"
     )
+    parser.add_argument(
+        "-m", "--modules", type=str,
+        default=None,
+        help="A comma-separated list of modules to test "
+             "(default: %s)" % ",".join(sorted([m.name for m in modules.all_modules]))
+    )
+    parser.add_argument(
+        "-e", "--excluded-tags", type=str,
+        default=None,
+        help="A comma-separated list of tags to exclude in the tests, "
+             "e.g., org.apache.spark.tags.ExtendedHiveTest "
+    )
+    parser.add_argument(
+        "-i", "--included-tags", type=str,
+        default=None,
+        help="A comma-separated list of tags to include in the tests, "
+             "e.g., org.apache.spark.tags.ExtendedHiveTest "
+    )
 
     args, unknown = parser.parse_known_args()
     if unknown:
@@ -564,17 +608,26 @@ def main():
               " install one and retry.")
         sys.exit(2)
 
-    # install SparkR
-    if which("R"):
-        run_cmd([os.path.join(SPARK_HOME, "R", "install-dev.sh")])
-    else:
-        print("Cannot install SparkR as R was not found in PATH")
+    # Install SparkR
+    should_only_test_modules = opts.modules is not None
+    test_modules = []
+    if should_only_test_modules:
+        str_test_modules = [m.strip() for m in opts.modules.split(",")]
+        test_modules = [m for m in modules.all_modules if m.name in str_test_modules]
+
+    if not should_only_test_modules or modules.sparkr in test_modules:
+        # If tests modules are specified, we will not run R linter.
+        # SparkR needs the manual SparkR installation.
+        if which("R"):
+            run_cmd([os.path.join(SPARK_HOME, "R", "install-dev.sh")])
+        else:
+            print("Cannot install SparkR as R was not found in PATH")
 
     if os.environ.get("AMPLAB_JENKINS"):
         # if we're on the Amplab Jenkins build servers setup variables
         # to reflect the environment settings
         build_tool = os.environ.get("AMPLAB_JENKINS_BUILD_TOOL", "sbt")
-        hadoop_version = os.environ.get("AMPLAB_JENKINS_BUILD_PROFILE", "hadoop2.7")
+        hadoop_version = os.environ.get("AMPLAB_JENKINS_BUILD_PROFILE", "hadoop3.2")
         hive_version = os.environ.get("AMPLAB_JENKINS_BUILD_HIVE_PROFILE", "hive2.3")
         test_env = "amplab_jenkins"
         # add path for Python3 in Jenkins if we're calling from a Jenkins machine
@@ -582,27 +635,69 @@ def main():
         # /home/jenkins/anaconda2/envs/py36/bin
         os.environ["PATH"] = "/home/anaconda/envs/py36/bin:" + os.environ.get("PATH")
     else:
-        # else we're running locally and can use local settings
+        # else we're running locally or GitHub Actions.
         build_tool = "sbt"
-        hadoop_version = os.environ.get("HADOOP_PROFILE", "hadoop2.7")
+        hadoop_version = os.environ.get("HADOOP_PROFILE", "hadoop3.2")
         hive_version = os.environ.get("HIVE_PROFILE", "hive2.3")
-        test_env = "local"
+        if "GITHUB_ACTIONS" in os.environ:
+            test_env = "github_actions"
+        else:
+            test_env = "local"
 
     print("[info] Using build tool", build_tool, "with Hadoop profile", hadoop_version,
           "and Hive profile", hive_version, "under environment", test_env)
     extra_profiles = get_hadoop_profiles(hadoop_version) + get_hive_profiles(hive_version)
 
-    changed_modules = None
-    changed_files = None
-    if test_env == "amplab_jenkins" and os.environ.get("AMP_JENKINS_PRB"):
+    changed_modules = []
+    changed_files = []
+    included_tags = []
+    excluded_tags = []
+    if should_only_test_modules:
+        # If we're running the tests in GitHub Actions, attempt to detect and test
+        # only the affected modules.
+        if test_env == "github_actions":
+            if os.environ["APACHE_SPARK_REF"] != "":
+                # Fork repository
+                changed_files = identify_changed_files_from_git_commits(
+                    "HEAD", target_ref=os.environ["APACHE_SPARK_REF"])
+            else:
+                # Build for each commit.
+                changed_files = identify_changed_files_from_git_commits(
+                    os.environ["GITHUB_SHA"], target_ref=os.environ["GITHUB_PREV_SHA"])
+
+            modules_to_test = determine_modules_to_test(
+                determine_modules_for_files(changed_files), deduplicated=False)
+
+            if modules.root not in modules_to_test:
+                # If root module is not found, only test the intersected modules.
+                # If root module is found, just run the modules as specified initially.
+                test_modules = list(set(modules_to_test).intersection(test_modules))
+
+        changed_modules = test_modules
+        if len(changed_modules) == 0:
+            print("[info] There are no modules to test, exiting without testing.")
+            return
+
+    # If we're running the tests in AMPLab Jenkins, calculate the diff from the targeted branch, and
+    # detect modules to test.
+    elif test_env == "amplab_jenkins" and os.environ.get("AMP_JENKINS_PRB"):
         target_branch = os.environ["ghprbTargetBranch"]
         changed_files = identify_changed_files_from_git_commits("HEAD", target_branch=target_branch)
         changed_modules = determine_modules_for_files(changed_files)
+        test_modules = determine_modules_to_test(changed_modules)
         excluded_tags = determine_tags_to_exclude(changed_modules)
 
+    # If there is no changed module found, tests all.
     if not changed_modules:
         changed_modules = [modules.root]
-        excluded_tags = []
+    if not test_modules:
+        test_modules = determine_modules_to_test(changed_modules)
+
+    if opts.excluded_tags:
+        excluded_tags.extend([t.strip() for t in opts.excluded_tags.split(",")])
+    if opts.included_tags:
+        included_tags.extend([t.strip() for t in opts.included_tags.split(",")])
+
     print("[info] Found the following changed modules:",
           ", ".join(x.name for x in changed_modules))
 
@@ -615,40 +710,39 @@ def main():
         test_environ.update(m.environ)
     setup_test_environ(test_environ)
 
-    test_modules = determine_modules_to_test(changed_modules)
-
-    # license checks
-    run_apache_rat_checks()
-
-    # style checks
-    if not changed_files or any(f.endswith(".scala")
-                                or f.endswith("scalastyle-config.xml")
-                                for f in changed_files):
-        run_scala_style_checks(extra_profiles)
     should_run_java_style_checks = False
-    if not changed_files or any(f.endswith(".java")
-                                or f.endswith("checkstyle.xml")
-                                or f.endswith("checkstyle-suppressions.xml")
-                                for f in changed_files):
-        # Run SBT Checkstyle after the build to prevent a side-effect to the build.
-        should_run_java_style_checks = True
-    if not changed_files or any(f.endswith("lint-python")
-                                or f.endswith("tox.ini")
-                                or f.endswith(".py")
-                                for f in changed_files):
-        run_python_style_checks()
-    if not changed_files or any(f.endswith(".R")
-                                or f.endswith("lint-r")
-                                or f.endswith(".lintr")
-                                for f in changed_files):
-        run_sparkr_style_checks()
+    if not should_only_test_modules:
+        # license checks
+        run_apache_rat_checks()
+
+        # style checks
+        if not changed_files or any(f.endswith(".scala")
+                                    or f.endswith("scalastyle-config.xml")
+                                    for f in changed_files):
+            run_scala_style_checks(extra_profiles)
+        if not changed_files or any(f.endswith(".java")
+                                    or f.endswith("checkstyle.xml")
+                                    or f.endswith("checkstyle-suppressions.xml")
+                                    for f in changed_files):
+            # Run SBT Checkstyle after the build to prevent a side-effect to the build.
+            should_run_java_style_checks = True
+        if not changed_files or any(f.endswith("lint-python")
+                                    or f.endswith("tox.ini")
+                                    or f.endswith(".py")
+                                    for f in changed_files):
+            run_python_style_checks()
+        if not changed_files or any(f.endswith(".R")
+                                    or f.endswith("lint-r")
+                                    or f.endswith(".lintr")
+                                    for f in changed_files):
+            run_sparkr_style_checks()
 
     # determine if docs were changed and if we're inside the amplab environment
-    # note - the below commented out until *all* Jenkins workers can get `jekyll` installed
+    # note - the below commented out until *all* Jenkins workers can get the Bundler gem installed
     # if "DOCS" in changed_modules and test_env == "amplab_jenkins":
     #    build_spark_documentation()
 
-    if any(m.should_run_build_tests for m in test_modules):
+    if any(m.should_run_build_tests for m in test_modules) and test_env != "amplab_jenkins":
         run_build_tests()
 
     # spark build
@@ -663,7 +757,7 @@ def main():
         build_spark_assembly_sbt(extra_profiles, should_run_java_style_checks)
 
     # run the test suites
-    run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags)
+    run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags, included_tags)
 
     modules_with_python_tests = [m for m in test_modules if m.python_test_goals]
     if modules_with_python_tests:
diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index 85e6a1e9fadac..ab65ccd7d3d11 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -31,9 +31,10 @@ class Module(object):
     files have changed.
     """
 
-    def __init__(self, name, dependencies, source_file_regexes, build_profile_flags=(), environ={},
-                 sbt_test_goals=(), python_test_goals=(), blacklisted_python_implementations=(),
-                 test_tags=(), should_run_r_tests=False, should_run_build_tests=False):
+    def __init__(self, name, dependencies, source_file_regexes, build_profile_flags=(),
+                 environ=None, sbt_test_goals=(), python_test_goals=(),
+                 excluded_python_implementations=(), test_tags=(), should_run_r_tests=False,
+                 should_run_build_tests=False):
         """
         Define a new module.
 
@@ -49,7 +50,7 @@ def __init__(self, name, dependencies, source_file_regexes, build_profile_flags=
             module are changed.
         :param sbt_test_goals: A set of SBT test goals for testing this module.
         :param python_test_goals: A set of Python test goals for testing this module.
-        :param blacklisted_python_implementations: A set of Python implementations that are not
+        :param excluded_python_implementations: A set of Python implementations that are not
             supported by this module's Python components. The values in this set should match
             strings returned by Python's `platform.python_implementation()`.
         :param test_tags A set of tags that will be excluded when running unit tests if the module
@@ -62,9 +63,9 @@ def __init__(self, name, dependencies, source_file_regexes, build_profile_flags=
         self.source_file_prefixes = source_file_regexes
         self.sbt_test_goals = sbt_test_goals
         self.build_profile_flags = build_profile_flags
-        self.environ = environ
+        self.environ = environ or {}
         self.python_test_goals = python_test_goals
-        self.blacklisted_python_implementations = blacklisted_python_implementations
+        self.excluded_python_implementations = excluded_python_implementations
         self.test_tags = test_tags
         self.should_run_r_tests = should_run_r_tests
         self.should_run_build_tests = should_run_build_tests
@@ -100,9 +101,75 @@ def __hash__(self):
     ]
 )
 
+kvstore = Module(
+    name="kvstore",
+    dependencies=[tags],
+    source_file_regexes=[
+        "common/kvstore/",
+    ],
+    sbt_test_goals=[
+        "kvstore/test",
+    ],
+)
+
+network_common = Module(
+    name="network-common",
+    dependencies=[tags],
+    source_file_regexes=[
+        "common/network-common/",
+    ],
+    sbt_test_goals=[
+        "network-common/test",
+    ],
+)
+
+network_shuffle = Module(
+    name="network-shuffle",
+    dependencies=[tags],
+    source_file_regexes=[
+        "common/network-shuffle/",
+    ],
+    sbt_test_goals=[
+        "network-shuffle/test",
+    ],
+)
+
+unsafe = Module(
+    name="unsafe",
+    dependencies=[tags],
+    source_file_regexes=[
+        "common/unsafe",
+    ],
+    sbt_test_goals=[
+        "unsafe/test",
+    ],
+)
+
+launcher = Module(
+    name="launcher",
+    dependencies=[tags],
+    source_file_regexes=[
+        "launcher/",
+    ],
+    sbt_test_goals=[
+        "launcher/test",
+    ],
+)
+
+core = Module(
+    name="core",
+    dependencies=[kvstore, network_common, network_shuffle, unsafe, launcher],
+    source_file_regexes=[
+        "core/",
+    ],
+    sbt_test_goals=[
+        "core/test",
+    ],
+)
+
 catalyst = Module(
     name="catalyst",
-    dependencies=[tags],
+    dependencies=[tags, core],
     source_file_regexes=[
         "sql/catalyst/",
     ],
@@ -111,7 +178,6 @@ def __hash__(self):
     ],
 )
 
-
 sql = Module(
     name="sql",
     dependencies=[catalyst],
@@ -123,7 +189,6 @@ def __hash__(self):
     ],
 )
 
-
 hive = Module(
     name="hive",
     dependencies=[sql],
@@ -142,7 +207,6 @@ def __hash__(self):
     ]
 )
 
-
 repl = Module(
     name="repl",
     dependencies=[hive],
@@ -154,7 +218,6 @@ def __hash__(self):
     ],
 )
 
-
 hive_thriftserver = Module(
     name="hive-thriftserver",
     dependencies=[hive],
@@ -192,7 +255,6 @@ def __hash__(self):
     ]
 )
 
-
 sketch = Module(
     name="sketch",
     dependencies=[tags],
@@ -204,10 +266,9 @@ def __hash__(self):
     ]
 )
 
-
 graphx = Module(
     name="graphx",
-    dependencies=[tags],
+    dependencies=[tags, core],
     source_file_regexes=[
         "graphx/",
     ],
@@ -216,10 +277,9 @@ def __hash__(self):
     ]
 )
 
-
 streaming = Module(
     name="streaming",
-    dependencies=[tags],
+    dependencies=[tags, core],
     source_file_regexes=[
         "streaming",
     ],
@@ -235,7 +295,7 @@ def __hash__(self):
 # fail other PRs.
 streaming_kinesis_asl = Module(
     name="streaming-kinesis-asl",
-    dependencies=[tags],
+    dependencies=[tags, core],
     source_file_regexes=[
         "external/kinesis-asl/",
         "external/kinesis-asl-assembly/",
@@ -254,21 +314,23 @@ def __hash__(self):
 
 streaming_kafka_0_10 = Module(
     name="streaming-kafka-0-10",
-    dependencies=[streaming],
+    dependencies=[streaming, core],
     source_file_regexes=[
         # The ending "/" is necessary otherwise it will include "sql-kafka" codes
         "external/kafka-0-10/",
         "external/kafka-0-10-assembly",
+        "external/kafka-0-10-token-provider",
     ],
     sbt_test_goals=[
         "streaming-kafka-0-10/test",
+        "token-provider-kafka-0-10/test"
     ]
 )
 
 
 mllib_local = Module(
     name="mllib-local",
-    dependencies=[tags],
+    dependencies=[tags, core],
     source_file_regexes=[
         "mllib-local",
     ],
@@ -302,10 +364,9 @@ def __hash__(self):
     ]
 )
 
-
 pyspark_core = Module(
     name="pyspark-core",
-    dependencies=[],
+    dependencies=[core],
     source_file_regexes=[
         "python/(?!pyspark/(ml|mllib|sql|streaming))"
     ],
@@ -326,6 +387,7 @@ def __hash__(self):
         "pyspark.tests.test_conf",
         "pyspark.tests.test_context",
         "pyspark.tests.test_daemon",
+        "pyspark.tests.test_install_spark",
         "pyspark.tests.test_join",
         "pyspark.tests.test_profiler",
         "pyspark.tests.test_rdd",
@@ -339,7 +401,6 @@ def __hash__(self):
     ]
 )
 
-
 pyspark_sql = Module(
     name="pyspark-sql",
     dependencies=[pyspark_core, hive, avro],
@@ -465,7 +526,7 @@ def __hash__(self):
         "pyspark.mllib.tests.test_streaming_algorithms",
         "pyspark.mllib.tests.test_util",
     ],
-    blacklisted_python_implementations=[
+    excluded_python_implementations=[
         "PyPy"  # Skip these tests under PyPy since they require numpy and it isn't available there
     ]
 )
@@ -504,13 +565,100 @@ def __hash__(self):
         "pyspark.ml.tests.test_stat",
         "pyspark.ml.tests.test_training_summary",
         "pyspark.ml.tests.test_tuning",
+        "pyspark.ml.tests.test_util",
         "pyspark.ml.tests.test_wrapper",
     ],
-    blacklisted_python_implementations=[
+    excluded_python_implementations=[
         "PyPy"  # Skip these tests under PyPy since they require numpy and it isn't available there
     ]
 )
 
+pyspark_pandas = Module(
+    name="pyspark-pandas",
+    dependencies=[pyspark_core, pyspark_sql],
+    source_file_regexes=[
+        "python/pyspark/pandas/"
+    ],
+    python_test_goals=[
+        # doctests
+        "pyspark.pandas.accessors",
+        "pyspark.pandas.base",
+        "pyspark.pandas.categorical",
+        "pyspark.pandas.config",
+        "pyspark.pandas.datetimes",
+        "pyspark.pandas.exceptions",
+        "pyspark.pandas.extensions",
+        "pyspark.pandas.frame",
+        "pyspark.pandas.generic",
+        "pyspark.pandas.groupby",
+        "pyspark.pandas.indexing",
+        "pyspark.pandas.internal",
+        "pyspark.pandas.ml",
+        "pyspark.pandas.mlflow",
+        "pyspark.pandas.namespace",
+        "pyspark.pandas.numpy_compat",
+        "pyspark.pandas.series",
+        "pyspark.pandas.sql_processor",
+        "pyspark.pandas.strings",
+        "pyspark.pandas.utils",
+        "pyspark.pandas.window",
+        "pyspark.pandas.indexes.base",
+        "pyspark.pandas.indexes.category",
+        "pyspark.pandas.indexes.datetimes",
+        "pyspark.pandas.indexes.multi",
+        "pyspark.pandas.indexes.numeric",
+        "pyspark.pandas.spark.accessors",
+        "pyspark.pandas.spark.utils",
+        "pyspark.pandas.typedef.typehints",
+        # unittests
+        "pyspark.pandas.tests.indexes.test_base",
+        "pyspark.pandas.tests.indexes.test_category",
+        "pyspark.pandas.tests.indexes.test_datetime",
+        "pyspark.pandas.tests.plot.test_frame_plot",
+        "pyspark.pandas.tests.plot.test_frame_plot_matplotlib",
+        "pyspark.pandas.tests.plot.test_frame_plot_plotly",
+        "pyspark.pandas.tests.plot.test_series_plot",
+        "pyspark.pandas.tests.plot.test_series_plot_matplotlib",
+        "pyspark.pandas.tests.plot.test_series_plot_plotly",
+        "pyspark.pandas.tests.test_categorical",
+        "pyspark.pandas.tests.test_config",
+        "pyspark.pandas.tests.test_csv",
+        "pyspark.pandas.tests.test_dataframe",
+        "pyspark.pandas.tests.test_dataframe_conversion",
+        "pyspark.pandas.tests.test_dataframe_spark_io",
+        "pyspark.pandas.tests.test_default_index",
+        "pyspark.pandas.tests.test_expanding",
+        "pyspark.pandas.tests.test_extension",
+        "pyspark.pandas.tests.test_frame_spark",
+        "pyspark.pandas.tests.test_groupby",
+        "pyspark.pandas.tests.test_indexing",
+        "pyspark.pandas.tests.test_indexops_spark",
+        "pyspark.pandas.tests.test_internal",
+        "pyspark.pandas.tests.test_namespace",
+        "pyspark.pandas.tests.test_numpy_compat",
+        "pyspark.pandas.tests.test_ops_on_diff_frames",
+        "pyspark.pandas.tests.test_ops_on_diff_frames_groupby",
+        "pyspark.pandas.tests.test_ops_on_diff_frames_groupby_expanding",
+        "pyspark.pandas.tests.test_ops_on_diff_frames_groupby_rolling",
+        "pyspark.pandas.tests.test_repr",
+        "pyspark.pandas.tests.test_reshape",
+        "pyspark.pandas.tests.test_rolling",
+        "pyspark.pandas.tests.test_series",
+        "pyspark.pandas.tests.test_series_conversion",
+        "pyspark.pandas.tests.test_series_datetime",
+        "pyspark.pandas.tests.test_series_string",
+        "pyspark.pandas.tests.test_sql",
+        "pyspark.pandas.tests.test_stats",
+        "pyspark.pandas.tests.test_typedef",
+        "pyspark.pandas.tests.test_utils",
+        "pyspark.pandas.tests.test_window",
+    ],
+    excluded_python_implementations=[
+        "PyPy"  # Skip these tests under PyPy since they require numpy, pandas, and pyarrow and
+                # they aren't available there
+    ]
+)
+
 sparkr = Module(
     name="sparkr",
     dependencies=[hive, mllib],
@@ -593,7 +741,7 @@ def __hash__(self):
 # No other modules should directly depend on this module.
 root = Module(
     name="root",
-    dependencies=[build],  # Changes to build should trigger all tests.
+    dependencies=[build, core],  # Changes to build should trigger all tests.
     source_file_regexes=[],
     # In order to run all of the tests, enable every test profile:
     build_profile_flags=list(set(
diff --git a/dev/sparktestsupport/toposort.py b/dev/sparktestsupport/toposort.py
index 8b2688d20039f..6785e481b56b5 100644
--- a/dev/sparktestsupport/toposort.py
+++ b/dev/sparktestsupport/toposort.py
@@ -24,8 +24,7 @@
 #    Moved functools import to the top of the file.
 #    Changed assert to a ValueError.
 #    Changed iter[items|keys] to [items|keys], for python 3
-#     compatibility. I don't think it matters for python 2 these are
-#     now lists instead of iterables.
+#     compatibility.
 #    Copy the input so as to leave it unmodified.
 #    Renamed function from toposort2 to toposort.
 #    Handle empty input.
diff --git a/dev/test-dependencies.sh b/dev/test-dependencies.sh
index b3e68bed1d1e7..e9e9227d239e1 100755
--- a/dev/test-dependencies.sh
+++ b/dev/test-dependencies.sh
@@ -29,10 +29,9 @@ export LC_ALL=C
 # TODO: This would be much nicer to do in SBT, once SBT supports Maven-style resolution.
 
 # NOTE: These should match those in the release publishing script
-HADOOP2_MODULE_PROFILES="-Phive-thriftserver -Pmesos -Pkubernetes -Pyarn -Phive"
+HADOOP_MODULE_PROFILES="-Phive-thriftserver -Pmesos -Pkubernetes -Pyarn -Phive"
 MVN="build/mvn"
 HADOOP_HIVE_PROFILES=(
-    hadoop-2.7-hive-1.2
     hadoop-2.7-hive-2.3
     hadoop-3.2-hive-2.3
 )
@@ -71,22 +70,19 @@ for HADOOP_HIVE_PROFILE in "${HADOOP_HIVE_PROFILES[@]}"; do
   if [[ $HADOOP_HIVE_PROFILE == **hadoop-3.2-hive-2.3** ]]; then
     HADOOP_PROFILE=hadoop-3.2
     HIVE_PROFILE=hive-2.3
-  elif [[ $HADOOP_HIVE_PROFILE == **hadoop-2.7-hive-2.3** ]]; then
-    HADOOP_PROFILE=hadoop-2.7
-    HIVE_PROFILE=hive-2.3
   else
     HADOOP_PROFILE=hadoop-2.7
-    HIVE_PROFILE=hive-1.2
+    HIVE_PROFILE=hive-2.3
   fi
   echo "Performing Maven install for $HADOOP_HIVE_PROFILE"
-  $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE -P$HIVE_PROFILE jar:jar jar:test-jar install:install clean -q
+  $MVN $HADOOP_MODULE_PROFILES -P$HADOOP_PROFILE -P$HIVE_PROFILE jar:jar jar:test-jar install:install clean -q
 
   echo "Performing Maven validate for $HADOOP_HIVE_PROFILE"
-  $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE -P$HIVE_PROFILE validate -q
+  $MVN $HADOOP_MODULE_PROFILES -P$HADOOP_PROFILE -P$HIVE_PROFILE validate -q
 
   echo "Generating dependency manifest for $HADOOP_HIVE_PROFILE"
   mkdir -p dev/pr-deps
-  $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE -P$HIVE_PROFILE dependency:build-classpath -pl assembly -am \
+  $MVN $HADOOP_MODULE_PROFILES -P$HADOOP_PROFILE -P$HIVE_PROFILE dependency:build-classpath -pl assembly -am \
     | grep "Dependencies classpath:" -A 1 \
     | tail -n 1 | tr ":" "\n" | awk -F '/' '{
       # For each dependency classpath, we fetch the last three parts split by "/": artifact id, version, and jar name.
diff --git a/dev/tests/pr_merge_ability.sh b/dev/tests/pr_merge_ability.sh
index 25fdbccac4dd8..a32667730f76c 100755
--- a/dev/tests/pr_merge_ability.sh
+++ b/dev/tests/pr_merge_ability.sh
@@ -22,7 +22,7 @@
 # another branch and returning results to be published. More details can be
 # found at dev/run-tests-jenkins.
 #
-# Arg1: The Github Pull Request Actual Commit
+# Arg1: The GitHub Pull Request Actual Commit
 # known as `ghprbActualCommit` in `run-tests-jenkins`
 # Arg2: The SHA1 hash
 # known as `sha1` in `run-tests-jenkins`
diff --git a/dev/tests/pr_public_classes.sh b/dev/tests/pr_public_classes.sh
index 479d1851fe0b8..ad1ad5e736594 100755
--- a/dev/tests/pr_public_classes.sh
+++ b/dev/tests/pr_public_classes.sh
@@ -22,7 +22,7 @@
 # another branch and returning results to be published. More details can be
 # found at dev/run-tests-jenkins.
 #
-# Arg1: The Github Pull Request Actual Commit
+# Arg1: The GitHub Pull Request Actual Commit
 # known as `ghprbActualCommit` in `run-tests-jenkins`
 
 ghprbActualCommit="$1"
diff --git a/dev/tox.ini b/dev/tox.ini
index 3ee13c09b5de7..68e875f4c54ed 100644
--- a/dev/tox.ini
+++ b/dev/tox.ini
@@ -16,4 +16,9 @@
 [pycodestyle]
 ignore=E226,E241,E305,E402,E722,E731,E741,W503,W504
 max-line-length=100
-exclude=cloudpickle.py,heapq3.py,shared.py,python/docs/conf.py,work/*/*.py,python/.eggs/*,dist/*,.git/*
+exclude=*/target/*,python/pyspark/cloudpickle/*.py,shared.py,python/docs/source/conf.py,work/*/*.py,python/.eggs/*,dist/*,.git/*
+
+[flake8]
+select = E901,E999,F821,F822,F823,F401,F405,B006
+exclude = */target/*,python/pyspark/cloudpickle/*.py,shared.py*,python/docs/source/conf.py,work/*/*.py,python/.eggs/*,dist/*,.git/*,python/out,python/pyspark/sql/pandas/functions.pyi,python/pyspark/sql/column.pyi,python/pyspark/worker.pyi,python/pyspark/java_gateway.pyi
+max-line-length = 100
diff --git a/docs/.bundle/config b/docs/.bundle/config
new file mode 100644
index 0000000000000..b13821f801858
--- /dev/null
+++ b/docs/.bundle/config
@@ -0,0 +1,2 @@
+---
+BUNDLE_PATH: ".local_ruby_bundle"
diff --git a/docs/Gemfile b/docs/Gemfile
new file mode 100644
index 0000000000000..fa2f23d71cd77
--- /dev/null
+++ b/docs/Gemfile
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+source "https://rubygems.org"
+
+gem "jekyll", "4.2.0"
+gem "rouge", "3.26.0"
+gem "jekyll-redirect-from", "0.16.0"
+gem "webrick", "1.7"
diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock
new file mode 100644
index 0000000000000..54722d6351bf4
--- /dev/null
+++ b/docs/Gemfile.lock
@@ -0,0 +1,73 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    colorator (1.1.0)
+    concurrent-ruby (1.1.8)
+    em-websocket (0.5.2)
+      eventmachine (>= 0.12.9)
+      http_parser.rb (~> 0.6.0)
+    eventmachine (1.2.7)
+    ffi (1.14.2)
+    forwardable-extended (2.6.0)
+    http_parser.rb (0.6.0)
+    i18n (1.8.9)
+      concurrent-ruby (~> 1.0)
+    jekyll (4.2.0)
+      addressable (~> 2.4)
+      colorator (~> 1.0)
+      em-websocket (~> 0.5)
+      i18n (~> 1.0)
+      jekyll-sass-converter (~> 2.0)
+      jekyll-watch (~> 2.0)
+      kramdown (~> 2.3)
+      kramdown-parser-gfm (~> 1.0)
+      liquid (~> 4.0)
+      mercenary (~> 0.4.0)
+      pathutil (~> 0.9)
+      rouge (~> 3.0)
+      safe_yaml (~> 1.0)
+      terminal-table (~> 2.0)
+    jekyll-redirect-from (0.16.0)
+      jekyll (>= 3.3, < 5.0)
+    jekyll-sass-converter (2.1.0)
+      sassc (> 2.0.1, < 3.0)
+    jekyll-watch (2.2.1)
+      listen (~> 3.0)
+    kramdown (2.3.0)
+      rexml
+    kramdown-parser-gfm (1.1.0)
+      kramdown (~> 2.0)
+    liquid (4.0.3)
+    listen (3.4.1)
+      rb-fsevent (~> 0.10, >= 0.10.3)
+      rb-inotify (~> 0.9, >= 0.9.10)
+    mercenary (0.4.0)
+    pathutil (0.16.2)
+      forwardable-extended (~> 2.6)
+    public_suffix (4.0.6)
+    rb-fsevent (0.10.4)
+    rb-inotify (0.10.1)
+      ffi (~> 1.0)
+    rexml (3.2.4)
+    rouge (3.26.0)
+    safe_yaml (1.0.5)
+    sassc (2.4.0)
+      ffi (~> 1.9)
+    terminal-table (2.0.0)
+      unicode-display_width (~> 1.1, >= 1.1.1)
+    unicode-display_width (1.7.0)
+    webrick (1.7.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  jekyll (= 4.2.0)
+  jekyll-redirect-from (= 0.16.0)
+  rouge (= 3.26.0)
+  webrick (= 1.7)
+
+BUNDLED WITH
+   2.2.9
diff --git a/docs/README.md b/docs/README.md
index 22039871cf63d..410a415b2882d 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -33,10 +33,17 @@ Python, R and SQL.
 
 You need to have [Ruby](https://www.ruby-lang.org/en/documentation/installation/) and
 [Python](https://docs.python.org/2/using/unix.html#getting-and-installing-the-latest-version-of-python)
-installed. Also install the following libraries:
+installed. Make sure the `bundle` command is available, if not install the Gem containing it:
 
 ```sh
-$ sudo gem install jekyll jekyll-redirect-from rouge
+$ sudo gem install bundler
+```
+
+After this all the required ruby dependencies can be installed from the `docs/` directory via the Bundler:
+
+```sh
+$ cd docs
+$ bundle install
 ```
 
 Note: If you are on a system with both Ruby 1.9 and Ruby 2.0 you may need to replace gem with gem2.0.
@@ -48,17 +55,22 @@ and install these libraries:
 
 ```sh
 $ sudo Rscript -e 'install.packages(c("knitr", "devtools", "testthat", "rmarkdown"), repos="https://cloud.r-project.org/")'
-$ sudo Rscript -e 'devtools::install_version("roxygen2", version = "5.0.1", repos="https://cloud.r-project.org/")'
+$ sudo Rscript -e 'devtools::install_version("roxygen2", version = "7.1.1", repos="https://cloud.r-project.org/")'
 ```
 
-Note: Other versions of roxygen2 might work in SparkR documentation generation but `RoxygenNote` field in `$SPARK_HOME/R/pkg/DESCRIPTION` is 5.0.1, which is updated if the version is mismatched.
+Note: Other versions of roxygen2 might work in SparkR documentation generation but `RoxygenNote` field in `$SPARK_HOME/R/pkg/DESCRIPTION` is 7.1.1, which is updated if the version is mismatched.
 
 ### API Documentation
 
 To generate API docs for any language, you'll need to install these libraries:
 
+<!--
+TODO(SPARK-32407): Sphinx 3.1+ does not correctly index nested classes.
+See also https://github.com/sphinx-doc/sphinx/issues/7551.
+-->
+
 ```sh
-$ sudo pip install sphinx mkdocs numpy
+$ sudo pip install 'sphinx<3.1.0' mkdocs numpy pydata_sphinx_theme ipython nbsphinx numpydoc
 ```
 
 ## Generating the Documentation HTML
@@ -72,26 +84,26 @@ you have checked out or downloaded.
 In this directory you will find text files formatted using Markdown, with an ".md" suffix. You can
 read those text files directly if you want. Start with `index.md`.
 
-Execute `jekyll build` from the `docs/` directory to compile the site. Compiling the site with
+Execute `bundle exec jekyll build` from the `docs/` directory to compile the site. Compiling the site with
 Jekyll will create a directory called `_site` containing `index.html` as well as the rest of the
 compiled files.
 
 ```sh
 $ cd docs
-$ jekyll build
+$ bundle exec jekyll build
 ```
 
 You can modify the default Jekyll build as follows:
 
 ```sh
 # Skip generating API docs (which takes a while)
-$ SKIP_API=1 jekyll build
+$ SKIP_API=1 bundle exec jekyll build
 
 # Serve content locally on port 4000
-$ jekyll serve --watch
+$ bundle exec jekyll serve --watch
 
 # Build the site with extra features used on the live page
-$ PRODUCTION=1 jekyll build
+$ PRODUCTION=1 bundle exec jekyll build
 ```
 
 ## API Docs (Scaladoc, Javadoc, Sphinx, roxygen2, MkDocs)
@@ -104,7 +116,7 @@ public in `__init__.py`. The SparkR docs can be built by running `$SPARK_HOME/R/
 the SQL docs can be built by running `$SPARK_HOME/sql/create-docs.sh`
 after [building Spark](https://github.com/apache/spark#building-spark) first.
 
-When you run `jekyll build` in the `docs` directory, it will also copy over the scaladoc and javadoc for the various
+When you run `bundle exec jekyll build` in the `docs` directory, it will also copy over the scaladoc and javadoc for the various
 Spark subprojects into the `docs` directory (and then also into the `_site` directory). We use a
 jekyll plugin to run `./build/sbt unidoc` before building the site so if you haven't run it (recently) it
 may take some time as it generates all of the scaladoc and javadoc using [Unidoc](https://github.com/sbt/sbt-unidoc).
@@ -113,12 +125,12 @@ using [roxygen2](https://cran.r-project.org/web/packages/roxygen2/index.html) an
 using [MkDocs](https://www.mkdocs.org/).
 
 NOTE: To skip the step of building and copying over the Scala, Java, Python, R and SQL API docs, run `SKIP_API=1
-jekyll build`. In addition, `SKIP_SCALADOC=1`, `SKIP_PYTHONDOC=1`, `SKIP_RDOC=1` and `SKIP_SQLDOC=1` can be used
+bundle exec jekyll build`. In addition, `SKIP_SCALADOC=1`, `SKIP_PYTHONDOC=1`, `SKIP_RDOC=1` and `SKIP_SQLDOC=1` can be used
 to skip a single step of the corresponding language. `SKIP_SCALADOC` indicates skipping both the Scala and Java docs.
 
 ### Automatically Rebuilding API Docs
 
-`jekyll serve --watch` will only watch what's in `docs/`, and it won't follow symlinks. That means it won't monitor your API docs under `python/docs` or elsewhere.
+`bundle exec jekyll serve --watch` will only watch what's in `docs/`, and it won't follow symlinks. That means it won't monitor your API docs under `python/docs` or elsewhere.
 
 To work around this limitation for Python, install [`entr`](http://eradman.com/entrproject/) and run the following in a separate shell:
 
diff --git a/docs/_config.yml b/docs/_config.yml
index f82394ed63694..a8d42e483d17d 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -8,16 +8,38 @@ gems:
 kramdown:
   entity_output: numeric
 
+plugins:
+  - jekyll-redirect-from
+
 include:
   - _static
   - _modules
+  - _images
+  - _sources
 
 # These allow the documentation to be updated with newer releases
 # of Spark, Scala, and Mesos.
-SPARK_VERSION: 3.1.0-SNAPSHOT
-SPARK_VERSION_SHORT: 3.1.0
+SPARK_VERSION: 3.2.0-SNAPSHOT
+SPARK_VERSION_SHORT: 3.2.0
 SCALA_BINARY_VERSION: "2.12"
 SCALA_VERSION: "2.12.10"
 MESOS_VERSION: 1.0.0
 SPARK_ISSUE_TRACKER_URL: https://issues.apache.org/jira/browse/SPARK
 SPARK_GITHUB_URL: https://github.com/apache/spark
+# Before a new release, we should:
+#   1. update the `version` array for the new Spark documentation
+#      on https://github.com/algolia/docsearch-configs/blob/master/configs/apache_spark.json.
+#   2. update the value of `facetFilters.version` in `algoliaOptions` on the new release branch.
+# Otherwise, after release, the search results are always based on the latest documentation
+# (https://spark.apache.org/docs/latest/) even when visiting the documentation of previous releases.
+DOCSEARCH_SCRIPT: |
+  docsearch({
+      apiKey: 'b18ca3732c502995563043aa17bc6ecb',
+      indexName: 'apache_spark',
+      inputSelector: '#docsearch-input',
+      enhancedSearchInput: true,
+      algoliaOptions: {
+        'facetFilters': ["version:latest"]
+      },
+      debug: false // Set debug to true if you want to inspect the dropdown
+  });
diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index 57fc493dad2f2..e7b22c43e732c 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -32,6 +32,10 @@
       url: sql-data-sources-orc.html
     - text: JSON Files
       url: sql-data-sources-json.html
+    - text: CSV Files
+      url: sql-data-sources-csv.html
+    - text: Text Files
+      url: sql-data-sources-text.html
     - text: Hive Tables
       url: sql-data-sources-hive-tables.html
     - text: JDBC To Other Databases
@@ -51,6 +55,10 @@
       url: sql-performance-tuning.html#other-configuration-options
     - text: Join Strategy Hints for SQL Queries
       url: sql-performance-tuning.html#join-strategy-hints-for-sql-queries
+    - text: Coalesce Hints for SQL Queries
+      url: sql-performance-tuning.html#coalesce-hints-for-sql-queries
+    - text: Adaptive Query Execution
+      url: sql-performance-tuning.html#adaptive-query-execution
 - text: Distributed SQL Engine
   url: sql-distributed-sql-engine.html
   subitems:
@@ -60,30 +68,11 @@
       url: sql-distributed-sql-engine.html#running-the-spark-sql-cli
 - text: PySpark Usage Guide for Pandas with Apache Arrow
   url: sql-pyspark-pandas-with-arrow.html
-  subitems:
-    - text: Apache Arrow in Spark
-      url: sql-pyspark-pandas-with-arrow.html#apache-arrow-in-spark
-    - text: "Enabling for Conversion to/from Pandas"
-      url: sql-pyspark-pandas-with-arrow.html#enabling-for-conversion-tofrom-pandas
-    - text: "Pandas UDFs (a.k.a. Vectorized UDFs)"
-      url: sql-pyspark-pandas-with-arrow.html#pandas-udfs-aka-vectorized-udfs
-    - text: "Pandas Function APIs"
-      url: sql-pyspark-pandas-with-arrow.html#pandas-function-apis
-    - text: Usage Notes
-      url: sql-pyspark-pandas-with-arrow.html#usage-notes
 - text: Migration Guide
   url: sql-migration-old.html
 - text: SQL Reference
   url: sql-ref.html
   subitems:
-    - text: Data Types
-      url: sql-ref-datatypes.html
-    - text: Identifiers
-      url: sql-ref-identifier.html
-    - text: Literals
-      url: sql-ref-literals.html
-    - text: Null Semantics
-      url: sql-ref-null-semantics.html
     - text: ANSI Compliance
       url: sql-ref-ansi-compliance.html
       subitems:
@@ -93,6 +82,27 @@
           url: sql-ref-ansi-compliance.html#type-conversion
         - text: SQL Keywords
           url: sql-ref-ansi-compliance.html#sql-keywords
+    - text: Data Types
+      url: sql-ref-datatypes.html
+    - text: Datetime Pattern
+      url: sql-ref-datetime-pattern.html
+    - text: Functions
+      url: sql-ref-functions.html
+      subitems:
+      - text: Built-in Functions
+        url: sql-ref-functions-builtin.html
+      - text: Scalar UDFs (User-Defined Functions)
+        url: sql-ref-functions-udf-scalar.html
+      - text: UDAFs (User-Defined Aggregate Functions)
+        url: sql-ref-functions-udf-aggregate.html
+      - text: Integration with Hive UDFs/UDAFs/UDTFs
+        url: sql-ref-functions-udf-hive.html
+    - text: Identifiers
+      url: sql-ref-identifier.html
+    - text: Literals
+      url: sql-ref-literals.html
+    - text: Null Semantics
+      url: sql-ref-null-semantics.html
     - text: SQL Syntax
       url: sql-ref-syntax.html
       subitems:
@@ -126,7 +136,7 @@
             - text: REPAIR TABLE
               url: sql-ref-syntax-ddl-repair-table.html
             - text: USE DATABASE
-              url: sql-ref-syntax-qry-select-usedb.html
+              url: sql-ref-syntax-ddl-usedb.html
         - text: Data Manipulation Statements
           url: sql-ref-syntax-dml.html
           subitems:
@@ -158,22 +168,32 @@
                   url: sql-ref-syntax-qry-select-limit.html
                 - text: Common Table Expression
                   url: sql-ref-syntax-qry-select-cte.html
+                - text: Hints
+                  url: sql-ref-syntax-qry-select-hints.html
                 - text: Inline Table
                   url: sql-ref-syntax-qry-select-inline-table.html
+                - text: File
+                  url: sql-ref-syntax-qry-select-file.html
                 - text: JOIN
                   url: sql-ref-syntax-qry-select-join.html
-                - text: Join Hints
-                  url: sql-ref-syntax-qry-select-hints.html
                 - text: LIKE Predicate
                   url: sql-ref-syntax-qry-select-like.html
                 - text: Set Operators
                   url: sql-ref-syntax-qry-select-setops.html
                 - text: TABLESAMPLE
-                  url: sql-ref-syntax-qry-sampling.html
+                  url: sql-ref-syntax-qry-select-sampling.html
                 - text: Table-valued Function
                   url: sql-ref-syntax-qry-select-tvf.html
                 - text: Window Function
-                  url: sql-ref-syntax-qry-window.html
+                  url: sql-ref-syntax-qry-select-window.html
+                - text: CASE Clause
+                  url: sql-ref-syntax-qry-select-case.html
+                - text: LATERAL VIEW Clause
+                  url: sql-ref-syntax-qry-select-lateral-view.html
+                - text: PIVOT Clause
+                  url: sql-ref-syntax-qry-select-pivot.html
+                - text: TRANSFORM Clause
+                  url: sql-ref-syntax-qry-select-transform.html
             - text: EXPLAIN
               url: sql-ref-syntax-qry-explain.html
         - text: Auxiliary Statements
@@ -184,6 +204,8 @@
               subitems: 
                 - text: ANALYZE TABLE
                   url: sql-ref-syntax-aux-analyze-table.html
+                - text: ANALYZE TABLES
+                  url: sql-ref-syntax-aux-analyze-tables.html
             - text: CACHE
               url: sql-ref-syntax-aux-cache.html
               subitems:
@@ -194,7 +216,9 @@
                 - text: CLEAR CACHE
                   url: sql-ref-syntax-aux-cache-clear-cache.html
                 - text: REFRESH TABLE
-                  url: sql-ref-syntax-aux-refresh-table.html
+                  url: sql-ref-syntax-aux-cache-refresh-table.html
+                - text: REFRESH FUNCTION
+                  url: sql-ref-syntax-aux-cache-refresh-function.html
                 - text: REFRESH
                   url: sql-ref-syntax-aux-cache-refresh.html
             - text: DESCRIBE
@@ -236,6 +260,8 @@
                   url: sql-ref-syntax-aux-conf-mgmt-set.html
                 - text: RESET
                   url: sql-ref-syntax-aux-conf-mgmt-reset.html
+                - text: SET TIME ZONE
+                  url: sql-ref-syntax-aux-conf-mgmt-set-timezone.html
             - text: RESOURCE MANAGEMENT
               url: sql-ref-syntax-aux-resource-mgmt.html
               subitems:
@@ -243,20 +269,11 @@
                   url: sql-ref-syntax-aux-resource-mgmt-add-file.html
                 - text: ADD JAR
                   url: sql-ref-syntax-aux-resource-mgmt-add-jar.html
+                - text: ADD ARCHIVE
+                  url: sql-ref-syntax-aux-resource-mgmt-add-archive.html
                 - text: LIST FILE
                   url: sql-ref-syntax-aux-resource-mgmt-list-file.html
                 - text: LIST JAR
                   url: sql-ref-syntax-aux-resource-mgmt-list-jar.html
-    - text: Functions
-      url: sql-ref-functions.html
-      subitems:
-      - text: Built-in Functions
-        url: sql-ref-functions-builtin.html
-      - text: Scalar UDFs (User-Defined Functions)
-        url: sql-ref-functions-udf-scalar.html
-      - text: UDAFs (User-Defined Aggregate Functions)
-        url: sql-ref-functions-udf-aggregate.html
-      - text: Integration with Hive UDFs/UDAFs/UDTFs
-        url: sql-ref-functions-udf-hive.html
-    - text: Datetime Pattern
-      url: sql-ref-datetime-pattern.html
+                - text: LIST ARCHIVE
+                  url: sql-ref-syntax-aux-resource-mgmt-list-archive.html
diff --git a/docs/_layouts/global.html b/docs/_layouts/global.html
index d05ac6bbe129d..f10d46763cf76 100755
--- a/docs/_layouts/global.html
+++ b/docs/_layouts/global.html
@@ -25,12 +25,13 @@
             }
         </style>
         <meta name="viewport" content="width=device-width">
-        <link rel="stylesheet" href="css/bootstrap-responsive.min.css">
         <link rel="stylesheet" href="css/main.css">
 
         <script src="js/vendor/modernizr-2.6.1-respond-1.1.0.min.js"></script>
 
         <link rel="stylesheet" href="css/pygments-default.css">
+        <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/docsearch.js@2/dist/cdn/docsearch.min.css" />
+        <link rel="stylesheet" href="css/docsearch.css">
 
         {% production %}
         <!-- Google analytics script -->
@@ -55,75 +56,87 @@
 
         <!-- This code is taken from http://twitter.github.com/bootstrap/examples/hero.html -->
 
-        <div class="navbar navbar-fixed-top" id="topbar">
-            <div class="navbar-inner">
-                <div class="container">
-                    <div class="brand"><a href="index.html">
-                      <img src="img/spark-logo-hd.png" style="height:50px;"/></a><span class="version">{{site.SPARK_VERSION_SHORT}}</span>
+        <nav class="navbar fixed-top navbar-expand-md navbar-light bg-light" id="topbar">
+            <div class="container">
+                <div class="navbar-header">
+                    <div class="navbar-brand"><a href="index.html">
+                        <img src="img/spark-logo-hd.png" style="height:50px;"/></a><span class="version">{{site.SPARK_VERSION_SHORT}}</span>
                     </div>
-                    <ul class="nav">
+                </div>
+                <button class="navbar-toggler" type="button" data-toggle="collapse"
+                        data-target="#navbarCollapse" aria-controls="navbarCollapse"
+                        aria-expanded="false" aria-label="Toggle navigation">
+                    <span class="navbar-toggler-icon"></span>
+                </button>
+                <div class="collapse navbar-collapse" id="navbarCollapse">
+                    <ul class="navbar-nav">
                         <!--TODO(andyk): Add class="active" attribute to li some how.-->
-                        <li><a href="index.html">Overview</a></li>
-
-                        <li class="dropdown">
-                            <a href="#" class="dropdown-toggle" data-toggle="dropdown">Programming Guides<b class="caret"></b></a>
-                            <ul class="dropdown-menu">
-                                <li><a href="quick-start.html">Quick Start</a></li>
-                                <li><a href="rdd-programming-guide.html">RDDs, Accumulators, Broadcasts Vars</a></li>
-                                <li><a href="sql-programming-guide.html">SQL, DataFrames, and Datasets</a></li>
-                                <li><a href="structured-streaming-programming-guide.html">Structured Streaming</a></li>
-                                <li><a href="streaming-programming-guide.html">Spark Streaming (DStreams)</a></li>
-                                <li><a href="ml-guide.html">MLlib (Machine Learning)</a></li>
-                                <li><a href="graphx-programming-guide.html">GraphX (Graph Processing)</a></li>
-                                <li><a href="sparkr.html">SparkR (R on Spark)</a></li>
-                            </ul>
+                        <li class="nav-item"><a href="index.html" class="nav-link">Overview</a></li>
+
+                        <li class="nav-item dropdown">
+                            <a href="#" class="nav-link dropdown-toggle" id="navbarQuickStart" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Programming Guides</a>
+                            <div class="dropdown-menu" aria-labelledby="navbarQuickStart">
+                                <a class="dropdown-item" href="quick-start.html">Quick Start</a>
+                                <a class="dropdown-item" href="rdd-programming-guide.html">RDDs, Accumulators, Broadcasts Vars</a>
+                                <a class="dropdown-item" href="sql-programming-guide.html">SQL, DataFrames, and Datasets</a>
+                                <a class="dropdown-item" href="structured-streaming-programming-guide.html">Structured Streaming</a>
+                                <a class="dropdown-item" href="streaming-programming-guide.html">Spark Streaming (DStreams)</a>
+                                <a class="dropdown-item" href="ml-guide.html">MLlib (Machine Learning)</a>
+                                <a class="dropdown-item" href="graphx-programming-guide.html">GraphX (Graph Processing)</a>
+                                <a class="dropdown-item" href="sparkr.html">SparkR (R on Spark)</a>
+                                <a class="dropdown-item" href="api/python/getting_started/index.html">PySpark (Python on Spark)</a>
+                            </div>
+                        </li>
+
+                        <li class="nav-item dropdown">
+                            <a href="#" class="nav-link dropdown-toggle" id="navbarAPIDocs" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">API Docs</a>
+                            <div class="dropdown-menu" aria-labelledby="navbarAPIDocs">
+                                <a class="dropdown-item" href="api/scala/org/apache/spark/index.html">Scala</a>
+                                <a class="dropdown-item" href="api/java/index.html">Java</a>
+                                <a class="dropdown-item" href="api/python/index.html">Python</a>
+                                <a class="dropdown-item" href="api/R/index.html">R</a>
+                                <a class="dropdown-item" href="api/sql/index.html">SQL, Built-in Functions</a>
+                            </div>
                         </li>
 
-                        <li class="dropdown">
-                            <a href="#" class="dropdown-toggle" data-toggle="dropdown">API Docs<b class="caret"></b></a>
-                            <ul class="dropdown-menu">
-                                <li><a href="api/scala/org/apache/spark/index.html">Scala</a></li>
-                                <li><a href="api/java/index.html">Java</a></li>
-                                <li><a href="api/python/index.html">Python</a></li>
-                                <li><a href="api/R/index.html">R</a></li>
-                                <li><a href="api/sql/index.html">SQL, Built-in Functions</a></li>
-                            </ul>
+                        <li class="nav-item dropdown">
+                            <a href="#" class="nav-link dropdown-toggle" id="navbarDeploying" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Deploying</a>
+                            <div class="dropdown-menu" aria-labelledby="navbarDeploying">
+                                <a class="dropdown-item" href="cluster-overview.html">Overview</a>
+                                <a class="dropdown-item" href="submitting-applications.html">Submitting Applications</a>
+                                <div class="dropdown-divider"></div>
+                                <a class="dropdown-item" href="spark-standalone.html">Spark Standalone</a>
+                                <a class="dropdown-item" href="running-on-mesos.html">Mesos</a>
+                                <a class="dropdown-item" href="running-on-yarn.html">YARN</a>
+                                <a class="dropdown-item" href="running-on-kubernetes.html">Kubernetes</a>
+                            </div>
                         </li>
 
-                        <li class="dropdown">
-                            <a href="#" class="dropdown-toggle" data-toggle="dropdown">Deploying<b class="caret"></b></a>
-                            <ul class="dropdown-menu">
-                                <li><a href="cluster-overview.html">Overview</a></li>
-                                <li><a href="submitting-applications.html">Submitting Applications</a></li>
-                                <li class="divider"></li>
-                                <li><a href="spark-standalone.html">Spark Standalone</a></li>
-                                <li><a href="running-on-mesos.html">Mesos</a></li>
-                                <li><a href="running-on-yarn.html">YARN</a></li>
-                                <li><a href="running-on-kubernetes.html">Kubernetes</a></li>
-                            </ul>
+                        <li class="nav-item dropdown">
+                            <a href="#" class="nav-link dropdown-toggle" id="navbarMore" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">More</a>
+                            <div class="dropdown-menu" aria-labelledby="navbarMore">
+                                <a class="dropdown-item" href="configuration.html">Configuration</a>
+                                <a class="dropdown-item" href="monitoring.html">Monitoring</a>
+                                <a class="dropdown-item" href="tuning.html">Tuning Guide</a>
+                                <a class="dropdown-item" href="job-scheduling.html">Job Scheduling</a>
+                                <a class="dropdown-item" href="security.html">Security</a>
+                                <a class="dropdown-item" href="hardware-provisioning.html">Hardware Provisioning</a>
+                                <a class="dropdown-item" href="migration-guide.html">Migration Guide</a>
+                                <div class="dropdown-divider"></div>
+                                <a class="dropdown-item" href="building-spark.html">Building Spark</a>
+                                <a class="dropdown-item" href="https://spark.apache.org/contributing.html">Contributing to Spark</a>
+                                <a class="dropdown-item" href="https://spark.apache.org/third-party-projects.html">Third Party Projects</a>
+                            </div>
                         </li>
 
-                        <li class="dropdown">
-                            <a href="api.html" class="dropdown-toggle" data-toggle="dropdown">More<b class="caret"></b></a>
-                            <ul class="dropdown-menu">
-                                <li><a href="configuration.html">Configuration</a></li>
-                                <li><a href="monitoring.html">Monitoring</a></li>
-                                <li><a href="tuning.html">Tuning Guide</a></li>
-                                <li><a href="job-scheduling.html">Job Scheduling</a></li>
-                                <li><a href="security.html">Security</a></li>
-                                <li><a href="hardware-provisioning.html">Hardware Provisioning</a></li>
-                                <li><a href="migration-guide.html">Migration Guide</a></li>
-                                <li class="divider"></li>
-                                <li><a href="building-spark.html">Building Spark</a></li>
-                                <li><a href="https://spark.apache.org/contributing.html">Contributing to Spark</a></li>
-                                <li><a href="https://spark.apache.org/third-party-projects.html">Third Party Projects</a></li>
-                            </ul>
+                        <li class="nav-item">
+                            <input type="text" id="docsearch-input" placeholder="Search the docs…">
                         </li>
                     </ul>
-                    <!--<p class="navbar-text pull-right"><span class="version-text">v{{site.SPARK_VERSION_SHORT}}</span></p>-->
+                    <!--<span class="navbar-text navbar-right"><span class="version-text">v{{site.SPARK_VERSION_SHORT}}</span></span>-->
                 </div>
             </div>
-        </div>
+        </nav>
 
         <div class="container-wrapper">
 
@@ -137,7 +150,7 @@
                 {% endif %}
                 <input id="nav-trigger" class="nav-trigger" checked type="checkbox">
                 <label for="nav-trigger"></label>
-                <div class="content-with-sidebar" id="content">
+                <div class="content-with-sidebar mr-3" id="content">
                     {% if page.displayTitle %}
                         <h1 class="title">{{ page.displayTitle }}</h1>
                     {% else %}
@@ -148,7 +161,7 @@ <h1 class="title">{{ page.title }}</h1>
 
                 </div>
             {% else %}
-                <div class="content" id="content">
+                <div class="content mr-3" id="content">
                     {% if page.displayTitle %}
                         <h1 class="title">{{ page.displayTitle }}</h1>
                     {% else %}
@@ -162,10 +175,21 @@ <h1 class="title">{{ page.title }}</h1>
              <!-- /container -->
         </div>
 
-        <script src="js/vendor/jquery-3.4.1.min.js"></script>
-        <script src="js/vendor/bootstrap.min.js"></script>
+        <script src="js/vendor/jquery-3.5.1.min.js"></script>
+        <script src="js/vendor/bootstrap.bundle.min.js"></script>
         <script src="js/vendor/anchor.min.js"></script>
         <script src="js/main.js"></script>
+        <script type="text/javascript" src="https://cdn.jsdelivr.net/npm/docsearch.js@2/dist/cdn/docsearch.min.js"></script>
+        <script type="text/javascript">
+            // DocSearch is entirely free and automated. DocSearch is built in two parts:
+            // 1. a crawler which we run on our own infrastructure every 24 hours. It follows every link
+            //    in your website and extract content from every page it traverses. It then pushes this
+            //    content to an Algolia index.
+            // 2. a JavaScript snippet to be inserted in your website that will bind this Algolia index
+            //    to your search input and display its results in a dropdown UI. If you want to find more
+            //    details on how works DocSearch, check the docs of DocSearch.
+            {{site.DOCSEARCH_SCRIPT}}
+        </script>
 
         <!-- MathJax Section -->
         <script type="text/x-mathjax-config">
diff --git a/docs/_plugins/copy_api_dirs.rb b/docs/_plugins/copy_api_dirs.rb
index 8e2a06e4bc9a8..17da22bf8a433 100644
--- a/docs/_plugins/copy_api_dirs.rb
+++ b/docs/_plugins/copy_api_dirs.rb
@@ -126,8 +126,8 @@
     puts "Making directory api/python"
     mkdir_p "api/python"
 
-    puts "cp -r ../python/docs/_build/html/. api/python"
-    cp_r("../python/docs/_build/html/.", "api/python")
+    puts "cp -r ../python/docs/build/html/. api/python"
+    cp_r("../python/docs/build/html/.", "api/python")
   end
 
   if not (ENV['SKIP_RDOC'] == '1')
diff --git a/docs/_plugins/include_example.rb b/docs/_plugins/include_example.rb
index 6b4b1c652a81b..7d0e78738095e 100644
--- a/docs/_plugins/include_example.rb
+++ b/docs/_plugins/include_example.rb
@@ -66,10 +66,10 @@ def render(context)
       rendered_code + hint
     end
 
-    # Trim the code block so as to have the same indention, regardless of their positions in the
+    # Trim the code block so as to have the same indentation, regardless of their positions in the
     # code file.
     def trim_codeblock(lines)
-      # Select the minimum indention of the current code block.
+      # Select the minimum indentation of the current code block.
       min_start_spaces = lines
         .select { |l| l.strip.size !=0 }
         .map { |l| l[/\A */].size }
diff --git a/docs/api.md b/docs/api.md
deleted file mode 100644
index 241d552d14329..0000000000000
--- a/docs/api.md
+++ /dev/null
@@ -1,27 +0,0 @@
----
-layout: global
-title: Spark API Documentation
-license: |
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
- 
-     http://www.apache.org/licenses/LICENSE-2.0
- 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
----
-
-Here you can read API docs for Spark and its submodules.
-
-- [Spark Scala API (Scaladoc)](api/scala/index.html)
-- [Spark Java API (Javadoc)](api/java/index.html)
-- [Spark Python API (Sphinx)](api/python/index.html)
-- [Spark R API (Roxygen2)](api/R/index.html)
-- [Spark SQL, Built-in Functions (MkDocs)](api/sql/index.html)
diff --git a/docs/building-spark.md b/docs/building-spark.md
index 3d12a60e2b974..8e1c84a37b436 100644
--- a/docs/building-spark.md
+++ b/docs/building-spark.md
@@ -51,7 +51,7 @@ You can fix these problems by setting the `MAVEN_OPTS` variable as discussed bef
 
 ### build/mvn
 
-Spark now comes packaged with a self-contained Maven installation to ease building and deployment of Spark from source located under the `build/` directory. This script will automatically download and setup all necessary build requirements ([Maven](https://maven.apache.org/), [Scala](https://www.scala-lang.org/), and [Zinc](https://github.com/typesafehub/zinc)) locally within the `build/` directory itself. It honors any `mvn` binary if present already, however, will pull down its own copy of Scala and Zinc regardless to ensure proper version requirements are met. `build/mvn` execution acts as a pass through to the `mvn` call allowing easy transition from previous build methods. As an example, one can build a version of Spark as follows:
+Spark now comes packaged with a self-contained Maven installation to ease building and deployment of Spark from source located under the `build/` directory. This script will automatically download and setup all necessary build requirements ([Maven](https://maven.apache.org/), [Scala](https://www.scala-lang.org/)) locally within the `build/` directory itself. It honors any `mvn` binary if present already, however, will pull down its own copy of Scala regardless to ensure proper version requirements are met. `build/mvn` execution acts as a pass through to the `mvn` call allowing easy transition from previous build methods. As an example, one can build a version of Spark as follows:
 
     ./build/mvn -DskipTests clean package
 
@@ -83,9 +83,9 @@ Example:
 
 To enable Hive integration for Spark SQL along with its JDBC server and CLI,
 add the `-Phive` and `-Phive-thriftserver` profiles to your existing build options.
-By default Spark will build with Hive 2.3.7.
+By default Spark will build with Hive 2.3.8.
 
-    # With Hive 2.3.7 support
+    # With Hive 2.3.8 support
     ./build/mvn -Pyarn -Phive -Phive-thriftserver -DskipTests clean package
 
 ## Packaging without Hadoop Dependencies for YARN
@@ -163,9 +163,8 @@ For the meanings of these two options, please carefully read the [Setting up Mav
 
 ## Speeding up Compilation
 
-Developers who compile Spark frequently may want to speed up compilation; e.g., by using Zinc
-(for developers who build with Maven) or by avoiding re-compilation of the assembly JAR (for
-developers who build with SBT).  For more information about how to do this, refer to the
+Developers who compile Spark frequently may want to speed up compilation; e.g., by avoiding re-compilation of the
+assembly JAR (for developers who build with SBT).  For more information about how to do this, refer to the
 [Useful Developer Tools page](https://spark.apache.org/developer-tools.html#reducing-build-times).
 
 ## Encrypted Filesystems
@@ -265,17 +264,15 @@ Change the major Scala version using (e.g. 2.13):
 
     ./dev/change-scala-version.sh 2.13
 
-For Maven, please enable the profile (e.g. 2.13):
+Enable the profile (e.g. 2.13):
 
+    # For Maven
     ./build/mvn -Pscala-2.13 compile
 
-For SBT, specify a complete scala version using (e.g. 2.13.0):
+    # For sbt
+    ./build/sbt -Pscala-2.13 compile
 
-    ./build/sbt -Dscala.version=2.13.0
-
-Otherwise, the sbt-pom-reader plugin will use the `scala.version` specified in the spark-parent pom.
-
-## Running Jenkins tests with Github Enterprise
+## Running Jenkins tests with GitHub Enterprise
 
 To run tests with Jenkins:
 
diff --git a/docs/cloud-integration.md b/docs/cloud-integration.md
index b2a3e77f1ee9d..22925a54a198f 100644
--- a/docs/cloud-integration.md
+++ b/docs/cloud-integration.md
@@ -49,7 +49,6 @@ They cannot be used as a direct replacement for a cluster filesystem such as HDF
 
 Key differences are:
 
-* Changes to stored objects may not be immediately visible, both in directory listings and actual data access.
 * The means by which directories are emulated may make working with them slow.
 * Rename operations may be very slow and, on failure, leave the store in an unknown state.
 * Seeking within a file may require new HTTP calls, hurting performance. 
@@ -58,7 +57,6 @@ How does this affect Spark?
 
 1. Reading and writing data can be significantly slower than working with a normal filesystem.
 1. Some directory structures may be very inefficient to scan during query split calculation.
-1. The output of work may not be immediately visible to a follow-on query.
 1. The rename-based algorithm by which Spark normally commits work when saving an RDD, DataFrame or Dataset
  is potentially both slow and unreliable.
 
@@ -66,8 +64,28 @@ For these reasons, it is not always safe to use an object store as a direct dest
 an intermediate store in a chain of queries. Consult the documentation of the object store and its
 connector to determine which uses are considered safe.
 
-In particular: *without some form of consistency layer, Amazon S3 cannot
-be safely used as the direct destination of work with the normal rename-based committer.*
+### Consistency
+
+As of 2021, the object stores of Amazon (S3), Google Cloud (GCS) and Microsoft (Azure Storage, ADLS Gen1, ADLS Gen2) are all *consistent*.
+
+This means that as soon as a file is written/updated it can be listed, viewed and opened by other processes
+-and the latest version will be retrieved. This was a known issue with AWS S3, especially with 404 caching
+of HEAD requests made before an object was created.
+
+Even so: none of the store connectors provide any guarantees as to how their clients cope with objects
+which are overwritten while a stream is reading them. Do not assume that the old file can be safely
+read, nor that there is any bounded time period for changes to become visible -or indeed, that
+the clients will not simply fail if a file being read is overwritten.
+
+For this reason: avoid overwriting files where it is known/likely that other clients
+will be actively reading them.
+
+Other object stores are *inconsistent*
+
+This includes [OpenStack Swift](https://docs.openstack.org/swift/latest/).
+
+Such stores are not always safe to use as a destination of work -consult
+each store's specific documentation. 
 
 ### Installation
 
@@ -103,7 +121,7 @@ for talking to cloud infrastructures, in which case this module may not be neede
 Spark jobs must authenticate with the object stores to access data within them.
 
 1. When Spark is running in a cloud infrastructure, the credentials are usually automatically set up.
-1. `spark-submit` reads the `AWS_ACCESS_KEY`, `AWS_SECRET_KEY`
+1. `spark-submit` reads the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`
 and `AWS_SESSION_TOKEN` environment variables and sets the associated authentication options
 for the `s3n` and `s3a` connectors to Amazon S3.
 1. In a Hadoop cluster, settings may be set in the `core-site.xml` file.
@@ -163,10 +181,15 @@ different stores and connectors when renaming directories:
 | Amazon S3     | s3a       | Unsafe                  | O(data) |
 | Azure Storage | wasb      | Safe                    | O(files) |
 | Azure Datalake Gen 2 | abfs | Safe                  | O(1) |
-| Google GCS    | gs        | Safe                    | O(1) |
+| Google Cloud Storage | gs        | Mixed                    | O(files) |
 
-As storing temporary files can run up charges; delete
+1. As storing temporary files can run up charges; delete
 directories called `"_temporary"` on a regular basis.
+1. For AWS S3, set a limit on how long multipart uploads can remain outstanding.
+This avoids incurring bills from incompleted uploads.
+1. For Google cloud, directory rename is file-by-file. Consider using the v2 committer
+and only write code which generates idemportent output -including filenames,
+as it is *no more unsafe* than the v1 committer, and faster.
 
 ### Parquet I/O Settings
 
@@ -245,17 +268,24 @@ mydataframe.write.format("parquet").save("s3a://bucket/destination")
 
 More details on these committers can be found in the latest Hadoop documentation.
 
+Note: depending upon the committer used, in-progress statistics may be
+under-reported with Hadoop versions before 3.3.1.
+
 ## Further Reading
 
 Here is the documentation on the standard connectors both from Apache and the cloud providers.
 
 * [OpenStack Swift](https://hadoop.apache.org/docs/current/hadoop-openstack/index.html).
-* [Azure Blob Storage and Azure Datalake Gen 2](https://hadoop.apache.org/docs/current/hadoop-aws/tools/hadoop-aws/index.html).
+* [Azure Blob Storage](https://hadoop.apache.org/docs/current/hadoop-azure/index.html).
+* [Azure Blob Filesystem (ABFS) and Azure Datalake Gen 2](https://hadoop.apache.org/docs/current/hadoop-azure/abfs.html).
 * [Azure Data Lake Gen 1](https://hadoop.apache.org/docs/current/hadoop-azure-datalake/index.html).
+* [Amazon S3 Strong Consistency](https://aws.amazon.com/s3/consistency/)
 * [Hadoop-AWS module (Hadoop 3.x)](https://hadoop.apache.org/docs/current3/hadoop-aws/tools/hadoop-aws/index.html).
-* [Amazon S3 via S3A and S3N (Hadoop 2.x)](https://hadoop.apache.org/docs/current/hadoop-aws/tools/hadoop-aws/index.html).
-* [Amazon EMR File System (EMRFS)](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-fs.html). From Amazon
-* [Google Cloud Storage Connector for Spark and Hadoop](https://cloud.google.com/hadoop/google-cloud-storage-connector). From Google
+* [Amazon S3 via S3A and S3N (Hadoop 2.x)](https://hadoop.apache.org/docs/current2/hadoop-aws/tools/hadoop-aws/index.html).
+* [Amazon EMR File System (EMRFS)](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-fs.html). From Amazon.
+* [Using the EMRFS S3-optimized Committer](https://docs.amazonaws.cn/en_us/emr/latest/ReleaseGuide/emr-spark-s3-optimized-committer.html)
+* [Google Cloud Storage Connector for Spark and Hadoop](https://cloud.google.com/dataproc/docs/concepts/connectors/cloud-storage). From Google.
 * [The Azure Blob Filesystem driver (ABFS)](https://docs.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-abfs-driver)
-* IBM Cloud Object Storage connector for Apache Spark: [Stocator](https://github.com/CODAIT/stocator), [IBM Object Storage](https://www.ibm.com/cloud/object-storage), [how-to-use-connector](https://developer.ibm.com/code/2018/08/16/installing-running-stocator-apache-spark-ibm-cloud-object-storage). From IBM
-
+* IBM Cloud Object Storage connector for Apache Spark: [Stocator](https://github.com/CODAIT/stocator),
+  [IBM Object Storage](https://www.ibm.com/cloud/object-storage). From IBM.
+* [Using JindoFS SDK to access Alibaba Cloud OSS](https://github.com/aliyun/alibabacloud-jindofs).
diff --git a/docs/cluster-overview.md b/docs/cluster-overview.md
index cb6d3a5c1f7ce..69ae1b6cdb083 100644
--- a/docs/cluster-overview.md
+++ b/docs/cluster-overview.md
@@ -65,7 +65,7 @@ The system currently supports several cluster managers:
 * [Standalone](spark-standalone.html) -- a simple cluster manager included with Spark that makes it
   easy to set up a cluster.
 * [Apache Mesos](running-on-mesos.html) -- a general cluster manager that can also run Hadoop MapReduce
-  and service applications.
+  and service applications. (Deprecated)
 * [Hadoop YARN](running-on-yarn.html) -- the resource manager in Hadoop 2.
 * [Kubernetes](running-on-kubernetes.html) -- an open-source system for automating deployment, scaling,
   and management of containerized applications.
diff --git a/docs/configuration.md b/docs/configuration.md
index fce04b940594b..520ede3ef53dd 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -274,10 +274,9 @@ of the most common options to set are:
  <td><code>spark.executor.memoryOverhead</code></td>
   <td>executorMemory * 0.10, with minimum of 384 </td>
   <td>
-    Amount of additional memory to be allocated per executor process in cluster mode, in MiB unless
-    otherwise specified. This is memory that accounts for things like VM overheads, interned strings,
-    other native overheads, etc. This tends to grow with the executor size (typically 6-10%).
-    This option is currently supported on YARN and Kubernetes.
+    Amount of additional memory to be allocated per executor process, in MiB unless otherwise specified.
+    This is memory that accounts for things like VM overheads, interned strings, other native overheads, etc.
+    This tends to grow with the executor size (typically 6-10%). This option is currently supported on YARN and Kubernetes.
     <br/>
     <em>Note:</em> Additional memory includes PySpark executor memory 
     (when <code>spark.executor.pyspark.memory</code> is not configured) and memory used by other
@@ -772,7 +771,12 @@ Apart from these, the following properties are also available, and may be useful
     option <code>--repositories</code> or <code>spark.jars.repositories</code> will also be included.
     Useful for allowing Spark to resolve artifacts from behind a firewall e.g. via an in-house
     artifact server like Artifactory. Details on the settings file format can be
-    found at <a href="http://ant.apache.org/ivy/history/latest-milestone/settings.html">Settings Files</a>
+    found at <a href="http://ant.apache.org/ivy/history/latest-milestone/settings.html">Settings Files</a>.
+    Only paths with <code>file://</code> scheme are supported. Paths without a scheme are assumed to have
+    a <code>file://</code> scheme.
+    <p/>
+    When running in YARN cluster mode, this file will also be localized to the remote driver for dependency
+    resolution within <code>SparkContext#addJar</code>
   </td>
   <td>2.2.0</td>
 </tr>
@@ -785,6 +789,17 @@ Apart from these, the following properties are also available, and may be useful
   </td>
   <td>2.3.0</td>
 </tr>
+<tr>
+  <td><code>spark.archives</code></td>
+  <td></td>
+  <td>
+    Comma-separated list of archives to be extracted into the working directory of each executor.
+    .jar, .tar.gz, .tgz and .zip are supported. You can specify the directory name to unpack via
+    adding <code>#</code> after the file name to unpack, for example, <code>file.zip#directory</code>.
+    This configuration is experimental.
+  </td>
+  <td>3.1.0</td>
+</tr>
 <tr>
   <td><code>spark.pyspark.driver.python</code></td>
   <td></td>
@@ -913,13 +928,22 @@ Apart from these, the following properties are also available, and may be useful
   </td>
   <td>1.1.1</td>
 </tr>
+<tr>
+  <td><code>spark.shuffle.io.connectionTimeout</code></td>
+  <td>value of <code>spark.network.timeout</code></td>
+  <td>
+    Timeout for the established connections between shuffle servers and clients to be marked
+    as idled and closed if there are still outstanding fetch requests but no traffic no the channel
+    for at least `connectionTimeout`.
+  </td>
+  <td>1.2.0</td>
+</tr>
 <tr>
   <td><code>spark.shuffle.service.enabled</code></td>
   <td>false</td>
   <td>
     Enables the external shuffle service. This service preserves the shuffle files written by
-    executors so the executors can be safely removed. This must be enabled if
-    <code>spark.dynamicAllocation.enabled</code> is "true". The external shuffle service
+    executors so the executors can be safely removed. The external shuffle service
     must be set up in order to enable it. See
     <a href="job-scheduling.html#configuration-and-setup">dynamic allocation
     configuration and setup documentation</a> for more information.
@@ -998,6 +1022,16 @@ Apart from these, the following properties are also available, and may be useful
   </td>
   <td>2.3.0</td>
 </tr>
+<tr>
+  <td><code>spark.files.io.connectionTimeout</code></td>
+  <td>value of <code>spark.network.timeout</code></td>
+  <td>
+    Timeout for the established connections for fetching files in Spark RPC environments to be marked
+    as idled and closed if there are still outstanding files being downloaded but no traffic no the channel
+    for at least `connectionTimeout`.
+  </td>
+  <td>1.6.0</td>
+</tr>
 </table>
 
 ### Spark UI
@@ -1031,10 +1065,15 @@ Apart from these, the following properties are also available, and may be useful
 </tr>
 <tr>
   <td><code>spark.eventLog.compression.codec</code></td>
-  <td></td>
+  <td>zstd</td>
   <td>
-    The codec to compress logged events. If this is not given,
-    <code>spark.io.compression.codec</code> will be used.
+    The codec to compress logged events. By default, Spark provides four codecs:
+    <code>lz4</code>, <code>lzf</code>, <code>snappy</code>, and <code>zstd</code>.
+    You can also use fully qualified class names to specify the codec, e.g.
+    <code>org.apache.spark.io.LZ4CompressionCodec</code>,
+    <code>org.apache.spark.io.LZFCompressionCodec</code>,
+    <code>org.apache.spark.io.SnappyCompressionCodec</code>,
+    and <code>org.apache.spark.io.ZStdCompressionCodec</code>.
   </td>
   <td>3.0.0</td>
 </tr>
@@ -1193,8 +1232,29 @@ Apart from these, the following properties are also available, and may be useful
   <td><code>spark.ui.reverseProxyUrl</code></td>
   <td></td>
   <td>
-    This is the URL where your proxy is running. This URL is for proxy which is running in front of Spark Master. This is useful when running proxy for authentication e.g. OAuth proxy. Make sure this is a complete URL including scheme (http/https) and port to reach your proxy.
-  </td>
+    If the Spark UI should be served through another front-end reverse proxy, this is the URL
+    for accessing the Spark master UI through that reverse proxy.
+    This is useful when running proxy for authentication e.g. an OAuth proxy. The URL may contain
+    a path prefix, like <code>http://mydomain.com/path/to/spark/</code>, allowing you to serve the
+    UI for multiple Spark clusters and other web applications through the same virtual host and
+    port.
+    Normally, this should be an absolute URL including scheme (http/https), host and port.
+    It is possible to specify a relative URL starting with "/" here. In this case, all URLs
+    generated by the Spark UI and Spark REST APIs will be server-relative links -- this will still
+    work, as the entire Spark UI is served through the same host and port.
+    <br/>The setting affects link generation in the Spark UI, but the front-end reverse proxy
+    is responsible for
+    <ul>
+      <li>stripping a path prefix before forwarding the request,</li>
+      <li>rewriting redirects which point directly to the Spark master,</li>
+      <li>redirecting access from <code>http://mydomain.com/path/to/spark</code> to
+      <code>http://mydomain.com/path/to/spark/</code> (trailing slash after path prefix); otherwise
+      relative links on the master page do not work correctly.</li>
+    </ul>
+    This setting affects all the workers and application UIs running in the cluster and must be set
+    identically on all the workers, drivers and masters. In is only effective when
+    <code>spark.ui.reverseProxy</code> is turned on. This setting is not needed when the Spark
+    master web UI is directly reachable.  </td>
   <td>2.1.0</td>
 </tr>
 <tr>
@@ -1305,6 +1365,38 @@ Apart from these, the following properties are also available, and may be useful
   </td>
   <td>2.2.3</td>
 </tr>
+<tr>
+  <td><code>spark.ui.timeline.executors.maximum</code></td>
+  <td>250</td>
+  <td>
+    The maximum number of executors shown in the event timeline.
+  </td>
+  <td>3.2.0</td>
+</tr>
+<tr>
+  <td><code>spark.ui.timeline.jobs.maximum</code></td>
+  <td>500</td>
+  <td>
+    The maximum number of jobs shown in the event timeline.
+  </td>
+  <td>3.2.0</td>
+</tr>
+<tr>
+  <td><code>spark.ui.timeline.stages.maximum</code></td>
+  <td>500</td>
+  <td>
+    The maximum number of stages shown in the event timeline.
+  </td>
+  <td>3.2.0</td>
+</tr>
+<tr>
+  <td><code>spark.ui.timeline.tasks.maximum</code></td>
+  <td>1000</td>
+  <td>
+    The maximum number of tasks shown in the event timeline.
+  </td>
+  <td>1.4.0</td>
+</tr>
 </table>
 
 ### Compression and Serialization
@@ -1764,8 +1856,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>1</td>
   <td>
     The file output committer algorithm version, valid algorithm version number: 1 or 2.
-    Version 2 may have better performance, but version 1 may handle failures better in certain situations,
-    as per <a href="https://issues.apache.org/jira/browse/MAPREDUCE-4815">MAPREDUCE-4815</a>.
+    Note that 2 may cause a correctness issue like MAPREDUCE-7282.
   </td>
   <td>2.2.0</td>
 </tr>
@@ -1889,8 +1980,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>120s</td>
   <td>
     Default timeout for all network interactions. This config will be used in place of
-    <code>spark.core.connection.ack.wait.timeout</code>,
-    <code>spark.storage.blockManagerSlaveTimeoutMs</code>,
+    <code>spark.storage.blockManagerHeartbeatTimeoutMs</code>,
     <code>spark.shuffle.io.connectionTimeout</code>, <code>spark.rpc.askTimeout</code> or
     <code>spark.rpc.lookupTimeout</code> if they are not configured.
   </td>
@@ -1952,16 +2042,6 @@ Apart from these, the following properties are also available, and may be useful
   </td>
   <td>1.4.0</td>
 </tr>
-<tr>
-  <td><code>spark.core.connection.ack.wait.timeout</code></td>
-  <td><code>spark.network.timeout</code></td>
-  <td>
-    How long for the connection to wait for ack to occur before timing
-    out and giving up. To avoid unwilling timeout caused by long pause like GC,
-    you can set larger value.
-  </td>
-  <td>1.1.1</td>
-</tr>
 <tr>
   <td><code>spark.network.maxRemoteBlockSizeFetchToMem</code></td>
   <td>200m</td>
@@ -1974,6 +2054,16 @@ Apart from these, the following properties are also available, and may be useful
   </td>
   <td>3.0.0</td>
 </tr>
+<tr>
+  <td><code>spark.rpc.io.connectionTimeout</code></td>
+  <td>value of <code>spark.network.timeout</code></td>
+  <td>
+    Timeout for the established connections between RPC peers to be marked as idled and closed
+    if there are outstanding RPC requests but no traffic on the channel for at least
+    `connectionTimeout`.
+  </td>
+  <td>1.2.0</td>
+</tr>
 </table>
 
 ### Scheduling
@@ -2147,113 +2237,113 @@ Apart from these, the following properties are also available, and may be useful
   <td>3.1.0</td>
 </tr>
 <tr>
-  <td><code>spark.scheduler.blacklist.unschedulableTaskSetTimeout</code></td>
+  <td><code>spark.scheduler.excludeOnFailure.unschedulableTaskSetTimeout</code></td>
   <td>120s</td>
   <td>
     The timeout in seconds to wait to acquire a new executor and schedule a task before aborting a
-    TaskSet which is unschedulable because of being completely blacklisted.
+    TaskSet which is unschedulable because all executors are excluded due to task failures.
   </td>
   <td>2.4.1</td>
 </tr>
 <tr>
-  <td><code>spark.blacklist.enabled</code></td>
+  <td><code>spark.excludeOnFailure.enabled</code></td>
   <td>
     false
   </td>
   <td>
-    If set to "true", prevent Spark from scheduling tasks on executors that have been blacklisted
-    due to too many task failures. The blacklisting algorithm can be further controlled by the
-    other "spark.blacklist" configuration options.
+    If set to "true", prevent Spark from scheduling tasks on executors that have been excluded
+    due to too many task failures. The algorithm used to exclude executors and nodes can be further
+    controlled by the other "spark.excludeOnFailure" configuration options.
   </td>
   <td>2.1.0</td>
 </tr>
 <tr>
-  <td><code>spark.blacklist.timeout</code></td>
+  <td><code>spark.excludeOnFailure.timeout</code></td>
   <td>1h</td>
   <td>
-    (Experimental) How long a node or executor is blacklisted for the entire application, before it
-    is unconditionally removed from the blacklist to attempt running new tasks.
+    (Experimental) How long a node or executor is excluded for the entire application, before it
+    is unconditionally removed from the excludelist to attempt running new tasks.
   </td>
   <td>2.1.0</td>
 </tr>
 <tr>
-  <td><code>spark.blacklist.task.maxTaskAttemptsPerExecutor</code></td>
+  <td><code>spark.excludeOnFailure.task.maxTaskAttemptsPerExecutor</code></td>
   <td>1</td>
   <td>
     (Experimental) For a given task, how many times it can be retried on one executor before the
-    executor is blacklisted for that task.
+    executor is excluded for that task.
   </td>
   <td>2.1.0</td>
 </tr>
 <tr>
-  <td><code>spark.blacklist.task.maxTaskAttemptsPerNode</code></td>
+  <td><code>spark.excludeOnFailure.task.maxTaskAttemptsPerNode</code></td>
   <td>2</td>
   <td>
     (Experimental) For a given task, how many times it can be retried on one node, before the entire
-    node is blacklisted for that task.
+    node is excluded for that task.
   </td>
   <td>2.1.0</td>
 </tr>
 <tr>
-  <td><code>spark.blacklist.stage.maxFailedTasksPerExecutor</code></td>
+  <td><code>spark.excludeOnFailure.stage.maxFailedTasksPerExecutor</code></td>
   <td>2</td>
   <td>
     (Experimental) How many different tasks must fail on one executor, within one stage, before the
-    executor is blacklisted for that stage.
+    executor is excluded for that stage.
   </td>
   <td>2.1.0</td>
 </tr>
 <tr>
-  <td><code>spark.blacklist.stage.maxFailedExecutorsPerNode</code></td>
+  <td><code>spark.excludeOnFailure.stage.maxFailedExecutorsPerNode</code></td>
   <td>2</td>
   <td>
-    (Experimental) How many different executors are marked as blacklisted for a given stage, before
+    (Experimental) How many different executors are marked as excluded for a given stage, before
     the entire node is marked as failed for the stage.
   </td>
   <td>2.1.0</td>
 </tr>
 <tr>
-  <td><code>spark.blacklist.application.maxFailedTasksPerExecutor</code></td>
+  <td><code>spark.excludeOnFailure.application.maxFailedTasksPerExecutor</code></td>
   <td>2</td>
   <td>
     (Experimental) How many different tasks must fail on one executor, in successful task sets,
-    before the executor is blacklisted for the entire application.  Blacklisted executors will
+    before the executor is excluded for the entire application.  Excluded executors will
     be automatically added back to the pool of available resources after the timeout specified by
-    <code>spark.blacklist.timeout</code>.  Note that with dynamic allocation, though, the executors
+    <code>spark.excludeOnFailure.timeout</code>.  Note that with dynamic allocation, though, the executors
     may get marked as idle and be reclaimed by the cluster manager.
   </td>
   <td>2.2.0</td>
 </tr>
 <tr>
-  <td><code>spark.blacklist.application.maxFailedExecutorsPerNode</code></td>
+  <td><code>spark.excludeOnFailure.application.maxFailedExecutorsPerNode</code></td>
   <td>2</td>
   <td>
-    (Experimental) How many different executors must be blacklisted for the entire application,
-    before the node is blacklisted for the entire application.  Blacklisted nodes will
+    (Experimental) How many different executors must be excluded for the entire application,
+    before the node is excluded for the entire application.  Excluded nodes will
     be automatically added back to the pool of available resources after the timeout specified by
-    <code>spark.blacklist.timeout</code>.  Note that with dynamic allocation, though, the executors
-    on the node may get marked as idle and be reclaimed by the cluster manager.
+    <code>spark.excludeOnFailure.timeout</code>.  Note that with dynamic allocation, though, the
+    executors on the node may get marked as idle and be reclaimed by the cluster manager.
   </td>
   <td>2.2.0</td>
 </tr>
 <tr>
-  <td><code>spark.blacklist.killBlacklistedExecutors</code></td>
+  <td><code>spark.excludeOnFailure.killExcludedExecutors</code></td>
   <td>false</td>
   <td>
     (Experimental) If set to "true", allow Spark to automatically kill the executors 
-    when they are blacklisted on fetch failure or blacklisted for the entire application, 
-    as controlled by spark.blacklist.application.*. Note that, when an entire node is added 
-    to the blacklist, all of the executors on that node will be killed.
+    when they are excluded on fetch failure or excluded for the entire application, 
+    as controlled by spark.killExcludedExecutors.application.*. Note that, when an entire node is added 
+    excluded, all of the executors on that node will be killed.
   </td>
   <td>2.2.0</td>
 </tr>
 <tr>
-  <td><code>spark.blacklist.application.fetchFailure.enabled</code></td>
+  <td><code>spark.excludeOnFailure.application.fetchFailure.enabled</code></td>
   <td>false</td>
   <td>
-    (Experimental) If set to "true", Spark will blacklist the executor immediately when a fetch
+    (Experimental) If set to "true", Spark will exclude the executor immediately when a fetch
     failure happens. If external shuffle service is enabled, then the whole node will be
-    blacklisted.
+    excluded.
   </td>
   <td>2.3.0</td>
 </tr>
@@ -2290,6 +2380,15 @@ Apart from these, the following properties are also available, and may be useful
   </td>
   <td>0.6.0</td>
 </tr>
+<tr>
+  <td><code>spark.speculation.min.threshold</code></td>
+  <td>100ms</td>
+  <td>
+    Minimum amount of time a task runs before being considered for speculation.
+    This can be used to avoid launching speculative copies of tasks that are very short.
+  </td>
+  <td>3.2.0</td>
+</tr>
 <tr>
   <td><code>spark.speculation.task.duration.threshold</code></td>
   <td>None</td>
@@ -2917,7 +3016,7 @@ The following variables can be set in `spark-env.sh`:
   </tr>
   <tr>
     <td><code>PYSPARK_PYTHON</code></td>
-    <td>Python binary executable to use for PySpark in both driver and workers (default is <code>python2.7</code> if available, otherwise <code>python</code>).
+    <td>Python binary executable to use for PySpark in both driver and workers (default is <code>python3</code> if available, otherwise <code>python</code>).
     Property <code>spark.pyspark.python</code> take precedence if it is set</td>
   </tr>
   <tr>
@@ -2955,6 +3054,12 @@ Spark uses [log4j](http://logging.apache.org/log4j/) for logging. You can config
 `log4j.properties` file in the `conf` directory. One way to start is to copy the existing
 `log4j.properties.template` located there.
 
+By default, Spark adds 1 record to the MDC (Mapped Diagnostic Context): `mdc.taskName`, which shows something
+like `task 1.0 in stage 0.0`. You can add `%X{mdc.taskName}` to your patternLayout in
+order to print it in the logs.
+Moreover, you can use `spark.sparkContext.setLocalProperty(s"mdc.$name", "value")` to add user specific data into MDC.
+The key in MDC will be the string of "mdc.$name".
+
 # Overriding configuration directory
 
 To specify a different configuration directory other than the default "SPARK_HOME/conf",
@@ -3022,3 +3127,10 @@ There are configurations available to request resources for the driver: <code>sp
 Spark will use the configurations specified to first request containers with the corresponding resources from the cluster manager. Once it gets the container, Spark launches an Executor in that container which will discover what resources the container has and the addresses associated with each resource. The Executor will register with the Driver and report back the resources available to that Executor. The Spark scheduler can then schedule tasks to each Executor and assign specific resource addresses based on the resource requirements the user specified. The user can see the resources assigned to a task using the <code>TaskContext.get().resources</code> api. On the driver, the user can see the resources assigned with the SparkContext <code>resources</code> call. It's then up to the user to use the assignedaddresses to do the processing they want or pass those into the ML/AI framework they are using.
 
 See your cluster manager specific page for requirements and details on each of - [YARN](running-on-yarn.html#resource-allocation-and-configuration-overview), [Kubernetes](running-on-kubernetes.html#resource-allocation-and-configuration-overview) and [Standalone Mode](spark-standalone.html#resource-allocation-and-configuration-overview). It is currently not available with Mesos or local mode. And please also note that local-cluster mode with multiple workers is not supported(see Standalone documentation).
+
+# Stage Level Scheduling Overview
+
+The stage level scheduling feature allows users to specify task and executor resource requirements at the stage level. This allows for different stages to run with executors that have different resources. A prime example of this is one ETL stage runs with executors with just CPUs, the next stage is an ML stage that needs GPUs. Stage level scheduling allows for user to request different executors that have GPUs when the ML stage runs rather then having to acquire executors with GPUs at the start of the application and them be idle while the ETL stage is being run.
+This is only available for the RDD API in Scala, Java, and Python.  It is available on YARN and Kubernetes when dynamic allocation is enabled. See the [YARN](running-on-yarn.html#stage-level-scheduling-overview) page or [Kubernetes](running-on-kubernetes.html#stage-level-scheduling-overview) page for more implementation details.
+
+See the `RDD.withResources` and `ResourceProfileBuilder` API's for using this feature. The current implementation acquires new executors for each `ResourceProfile`  created and currently has to be an exact match. Spark does not try to fit tasks into an executor that require a different ResourceProfile than the executor was created with. Executors that are not in use will idle timeout with the dynamic allocation logic. The default configuration for this feature is to only allow one ResourceProfile per stage. If the user associates more then 1 ResourceProfile to an RDD, Spark will throw an exception by default. See config `spark.scheduler.resource.profileMergeConflicts` to control that behavior. The current merge strategy Spark implements when `spark.scheduler.resource.profileMergeConflicts` is enabled is a simple max of each resource within the conflicting ResourceProfiles. Spark will create a new ResourceProfile with the max of each of the resources.
diff --git a/docs/core-migration-guide.md b/docs/core-migration-guide.md
index 63baef145f01e..1dee5029d8327 100644
--- a/docs/core-migration-guide.md
+++ b/docs/core-migration-guide.md
@@ -22,6 +22,26 @@ license: |
 * Table of contents
 {:toc}
 
+## Upgrading from Core 3.1 to 3.2
+
+- Since Spark 3.2, `spark.hadoopRDD.ignoreEmptySplits` is set to `true` by default which means Spark will not create empty partitions for empty input splits. To restore the behavior before Spark 3.2, you can set `spark.hadoopRDD.ignoreEmptySplits` to `false`.
+
+- Since Spark 3.2, `spark.eventLog.compression.codec` is set to `zstd` by default which means Spark will not fallback to use `spark.io.compression.codec` anymore.
+
+- Since Spark 3.2, `spark.storage.replication.proactive` is enabled by default which means Spark tries to replenish in case of the loss of cached RDD block replicas due to executor failures. To restore the behavior before Spark 3.2, you can set `spark.storage.replication.proactive` to `false`.
+
+- In Spark 3.2, `spark.launcher.childConectionTimeout` is deprecated (typo) though still works. Use `spark.launcher.childConnectionTimeout` instead.
+
+- In Spark 3.2, support for Apache Mesos as a resource manager is deprecated and will be removed in a future version. 
+
+- In Spark 3.2, Spark will delete K8s driver service resource when the application terminates by itself. To restore the behavior before Spark 3.2, you can set `spark.kubernetes.driver.service.deleteOnTermination` to `false`.
+
+## Upgrading from Core 3.0 to 3.1
+
+- In Spark 3.0 and below, `SparkContext` can be created in executors. Since Spark 3.1, an exception will be thrown when creating `SparkContext` in executors. You can allow it by setting the configuration `spark.executor.allowSparkContext` when creating `SparkContext` in executors.
+
+- In Spark 3.0 and below, Spark propagated the Hadoop classpath from `yarn.application.classpath` and `mapreduce.application.classpath` into the Spark application submitted to YARN when Spark distribution is with the built-in Hadoop. Since Spark 3.1, it does not propagate anymore when the Spark distribution is with the built-in Hadoop in order to prevent the failure from the different transitive dependencies picked up from the Hadoop cluster such as Guava and Jackson. To restore the behavior before Spark 3.1, you can set `spark.yarn.populateHadoopClasspath` to `true`.
+
 ## Upgrading from Core 2.4 to 3.0
 
 - The `org.apache.spark.ExecutorPlugin` interface and related configuration has been replaced with
diff --git a/docs/css/bootstrap-responsive.css b/docs/css/bootstrap-responsive.css
deleted file mode 100644
index daafa9180839e..0000000000000
--- a/docs/css/bootstrap-responsive.css
+++ /dev/null
@@ -1,1040 +0,0 @@
-/*!
- * Bootstrap Responsive v2.1.0
- *
- * Copyright 2012 Twitter, Inc
- * Licensed under the Apache License v2.0
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Designed and built with all the love in the world @twitter by @mdo and @fat.
- */
-
-.clearfix {
-  *zoom: 1;
-}
-
-.clearfix:before,
-.clearfix:after {
-  display: table;
-  line-height: 0;
-  content: "";
-}
-
-.clearfix:after {
-  clear: both;
-}
-
-.hide-text {
-  font: 0/0 a;
-  color: transparent;
-  text-shadow: none;
-  background-color: transparent;
-  border: 0;
-}
-
-.input-block-level {
-  display: block;
-  width: 100%;
-  min-height: 30px;
-  -webkit-box-sizing: border-box;
-     -moz-box-sizing: border-box;
-          box-sizing: border-box;
-}
-
-.hidden {
-  display: none;
-  visibility: hidden;
-}
-
-.visible-phone {
-  display: none !important;
-}
-
-.visible-tablet {
-  display: none !important;
-}
-
-.hidden-desktop {
-  display: none !important;
-}
-
-.visible-desktop {
-  display: inherit !important;
-}
-
-@media (min-width: 768px) and (max-width: 979px) {
-  .hidden-desktop {
-    display: inherit !important;
-  }
-  .visible-desktop {
-    display: none !important ;
-  }
-  .visible-tablet {
-    display: inherit !important;
-  }
-  .hidden-tablet {
-    display: none !important;
-  }
-}
-
-@media (max-width: 767px) {
-  .hidden-desktop {
-    display: inherit !important;
-  }
-  .visible-desktop {
-    display: none !important;
-  }
-  .visible-phone {
-    display: inherit !important;
-  }
-  .hidden-phone {
-    display: none !important;
-  }
-}
-
-@media (min-width: 1200px) {
-  .row {
-    margin-left: -30px;
-    *zoom: 1;
-  }
-  .row:before,
-  .row:after {
-    display: table;
-    line-height: 0;
-    content: "";
-  }
-  .row:after {
-    clear: both;
-  }
-  [class*="span"] {
-    float: left;
-    margin-left: 30px;
-  }
-  .container,
-  .navbar-static-top .container,
-  .navbar-fixed-top .container,
-  .navbar-fixed-bottom .container {
-    width: 1170px;
-  }
-  .span12 {
-    width: 1170px;
-  }
-  .span11 {
-    width: 1070px;
-  }
-  .span10 {
-    width: 970px;
-  }
-  .span9 {
-    width: 870px;
-  }
-  .span8 {
-    width: 770px;
-  }
-  .span7 {
-    width: 670px;
-  }
-  .span6 {
-    width: 570px;
-  }
-  .span5 {
-    width: 470px;
-  }
-  .span4 {
-    width: 370px;
-  }
-  .span3 {
-    width: 270px;
-  }
-  .span2 {
-    width: 170px;
-  }
-  .span1 {
-    width: 70px;
-  }
-  .offset12 {
-    margin-left: 1230px;
-  }
-  .offset11 {
-    margin-left: 1130px;
-  }
-  .offset10 {
-    margin-left: 1030px;
-  }
-  .offset9 {
-    margin-left: 930px;
-  }
-  .offset8 {
-    margin-left: 830px;
-  }
-  .offset7 {
-    margin-left: 730px;
-  }
-  .offset6 {
-    margin-left: 630px;
-  }
-  .offset5 {
-    margin-left: 530px;
-  }
-  .offset4 {
-    margin-left: 430px;
-  }
-  .offset3 {
-    margin-left: 330px;
-  }
-  .offset2 {
-    margin-left: 230px;
-  }
-  .offset1 {
-    margin-left: 130px;
-  }
-  .row-fluid {
-    width: 100%;
-    *zoom: 1;
-  }
-  .row-fluid:before,
-  .row-fluid:after {
-    display: table;
-    line-height: 0;
-    content: "";
-  }
-  .row-fluid:after {
-    clear: both;
-  }
-  .row-fluid [class*="span"] {
-    display: block;
-    float: left;
-    width: 100%;
-    min-height: 30px;
-    margin-left: 2.564102564102564%;
-    *margin-left: 2.5109110747408616%;
-    -webkit-box-sizing: border-box;
-       -moz-box-sizing: border-box;
-            box-sizing: border-box;
-  }
-  .row-fluid [class*="span"]:first-child {
-    margin-left: 0;
-  }
-  .row-fluid .span12 {
-    width: 100%;
-    *width: 99.94680851063829%;
-  }
-  .row-fluid .span11 {
-    width: 91.45299145299145%;
-    *width: 91.39979996362975%;
-  }
-  .row-fluid .span10 {
-    width: 82.90598290598291%;
-    *width: 82.8527914166212%;
-  }
-  .row-fluid .span9 {
-    width: 74.35897435897436%;
-    *width: 74.30578286961266%;
-  }
-  .row-fluid .span8 {
-    width: 65.81196581196582%;
-    *width: 65.75877432260411%;
-  }
-  .row-fluid .span7 {
-    width: 57.26495726495726%;
-    *width: 57.21176577559556%;
-  }
-  .row-fluid .span6 {
-    width: 48.717948717948715%;
-    *width: 48.664757228587014%;
-  }
-  .row-fluid .span5 {
-    width: 40.17094017094017%;
-    *width: 40.11774868157847%;
-  }
-  .row-fluid .span4 {
-    width: 31.623931623931625%;
-    *width: 31.570740134569924%;
-  }
-  .row-fluid .span3 {
-    width: 23.076923076923077%;
-    *width: 23.023731587561375%;
-  }
-  .row-fluid .span2 {
-    width: 14.52991452991453%;
-    *width: 14.476723040552828%;
-  }
-  .row-fluid .span1 {
-    width: 5.982905982905983%;
-    *width: 5.929714493544281%;
-  }
-  .row-fluid .offset12 {
-    margin-left: 105.12820512820512%;
-    *margin-left: 105.02182214948171%;
-  }
-  .row-fluid .offset12:first-child {
-    margin-left: 102.56410256410257%;
-    *margin-left: 102.45771958537915%;
-  }
-  .row-fluid .offset11 {
-    margin-left: 96.58119658119658%;
-    *margin-left: 96.47481360247316%;
-  }
-  .row-fluid .offset11:first-child {
-    margin-left: 94.01709401709402%;
-    *margin-left: 93.91071103837061%;
-  }
-  .row-fluid .offset10 {
-    margin-left: 88.03418803418803%;
-    *margin-left: 87.92780505546462%;
-  }
-  .row-fluid .offset10:first-child {
-    margin-left: 85.47008547008548%;
-    *margin-left: 85.36370249136206%;
-  }
-  .row-fluid .offset9 {
-    margin-left: 79.48717948717949%;
-    *margin-left: 79.38079650845607%;
-  }
-  .row-fluid .offset9:first-child {
-    margin-left: 76.92307692307693%;
-    *margin-left: 76.81669394435352%;
-  }
-  .row-fluid .offset8 {
-    margin-left: 70.94017094017094%;
-    *margin-left: 70.83378796144753%;
-  }
-  .row-fluid .offset8:first-child {
-    margin-left: 68.37606837606839%;
-    *margin-left: 68.26968539734497%;
-  }
-  .row-fluid .offset7 {
-    margin-left: 62.393162393162385%;
-    *margin-left: 62.28677941443899%;
-  }
-  .row-fluid .offset7:first-child {
-    margin-left: 59.82905982905982%;
-    *margin-left: 59.72267685033642%;
-  }
-  .row-fluid .offset6 {
-    margin-left: 53.84615384615384%;
-    *margin-left: 53.739770867430444%;
-  }
-  .row-fluid .offset6:first-child {
-    margin-left: 51.28205128205128%;
-    *margin-left: 51.175668303327875%;
-  }
-  .row-fluid .offset5 {
-    margin-left: 45.299145299145295%;
-    *margin-left: 45.1927623204219%;
-  }
-  .row-fluid .offset5:first-child {
-    margin-left: 42.73504273504273%;
-    *margin-left: 42.62865975631933%;
-  }
-  .row-fluid .offset4 {
-    margin-left: 36.75213675213675%;
-    *margin-left: 36.645753773413354%;
-  }
-  .row-fluid .offset4:first-child {
-    margin-left: 34.18803418803419%;
-    *margin-left: 34.081651209310785%;
-  }
-  .row-fluid .offset3 {
-    margin-left: 28.205128205128204%;
-    *margin-left: 28.0987452264048%;
-  }
-  .row-fluid .offset3:first-child {
-    margin-left: 25.641025641025642%;
-    *margin-left: 25.53464266230224%;
-  }
-  .row-fluid .offset2 {
-    margin-left: 19.65811965811966%;
-    *margin-left: 19.551736679396257%;
-  }
-  .row-fluid .offset2:first-child {
-    margin-left: 17.094017094017094%;
-    *margin-left: 16.98763411529369%;
-  }
-  .row-fluid .offset1 {
-    margin-left: 11.11111111111111%;
-    *margin-left: 11.004728132387708%;
-  }
-  .row-fluid .offset1:first-child {
-    margin-left: 8.547008547008547%;
-    *margin-left: 8.440625568285142%;
-  }
-  input,
-  textarea,
-  .uneditable-input {
-    margin-left: 0;
-  }
-  .controls-row [class*="span"] + [class*="span"] {
-    margin-left: 30px;
-  }
-  input.span12,
-  textarea.span12,
-  .uneditable-input.span12 {
-    width: 1156px;
-  }
-  input.span11,
-  textarea.span11,
-  .uneditable-input.span11 {
-    width: 1056px;
-  }
-  input.span10,
-  textarea.span10,
-  .uneditable-input.span10 {
-    width: 956px;
-  }
-  input.span9,
-  textarea.span9,
-  .uneditable-input.span9 {
-    width: 856px;
-  }
-  input.span8,
-  textarea.span8,
-  .uneditable-input.span8 {
-    width: 756px;
-  }
-  input.span7,
-  textarea.span7,
-  .uneditable-input.span7 {
-    width: 656px;
-  }
-  input.span6,
-  textarea.span6,
-  .uneditable-input.span6 {
-    width: 556px;
-  }
-  input.span5,
-  textarea.span5,
-  .uneditable-input.span5 {
-    width: 456px;
-  }
-  input.span4,
-  textarea.span4,
-  .uneditable-input.span4 {
-    width: 356px;
-  }
-  input.span3,
-  textarea.span3,
-  .uneditable-input.span3 {
-    width: 256px;
-  }
-  input.span2,
-  textarea.span2,
-  .uneditable-input.span2 {
-    width: 156px;
-  }
-  input.span1,
-  textarea.span1,
-  .uneditable-input.span1 {
-    width: 56px;
-  }
-  .thumbnails {
-    margin-left: -30px;
-  }
-  .thumbnails > li {
-    margin-left: 30px;
-  }
-  .row-fluid .thumbnails {
-    margin-left: 0;
-  }
-}
-
-@media (min-width: 768px) and (max-width: 979px) {
-  .row {
-    margin-left: -20px;
-    *zoom: 1;
-  }
-  .row:before,
-  .row:after {
-    display: table;
-    line-height: 0;
-    content: "";
-  }
-  .row:after {
-    clear: both;
-  }
-  [class*="span"] {
-    float: left;
-    margin-left: 20px;
-  }
-  .container,
-  .navbar-static-top .container,
-  .navbar-fixed-top .container,
-  .navbar-fixed-bottom .container {
-    width: 724px;
-  }
-  .span12 {
-    width: 724px;
-  }
-  .span11 {
-    width: 662px;
-  }
-  .span10 {
-    width: 600px;
-  }
-  .span9 {
-    width: 538px;
-  }
-  .span8 {
-    width: 476px;
-  }
-  .span7 {
-    width: 414px;
-  }
-  .span6 {
-    width: 352px;
-  }
-  .span5 {
-    width: 290px;
-  }
-  .span4 {
-    width: 228px;
-  }
-  .span3 {
-    width: 166px;
-  }
-  .span2 {
-    width: 104px;
-  }
-  .span1 {
-    width: 42px;
-  }
-  .offset12 {
-    margin-left: 764px;
-  }
-  .offset11 {
-    margin-left: 702px;
-  }
-  .offset10 {
-    margin-left: 640px;
-  }
-  .offset9 {
-    margin-left: 578px;
-  }
-  .offset8 {
-    margin-left: 516px;
-  }
-  .offset7 {
-    margin-left: 454px;
-  }
-  .offset6 {
-    margin-left: 392px;
-  }
-  .offset5 {
-    margin-left: 330px;
-  }
-  .offset4 {
-    margin-left: 268px;
-  }
-  .offset3 {
-    margin-left: 206px;
-  }
-  .offset2 {
-    margin-left: 144px;
-  }
-  .offset1 {
-    margin-left: 82px;
-  }
-  .row-fluid {
-    width: 100%;
-    *zoom: 1;
-  }
-  .row-fluid:before,
-  .row-fluid:after {
-    display: table;
-    line-height: 0;
-    content: "";
-  }
-  .row-fluid:after {
-    clear: both;
-  }
-  .row-fluid [class*="span"] {
-    display: block;
-    float: left;
-    width: 100%;
-    min-height: 30px;
-    margin-left: 2.7624309392265194%;
-    *margin-left: 2.709239449864817%;
-    -webkit-box-sizing: border-box;
-       -moz-box-sizing: border-box;
-            box-sizing: border-box;
-  }
-  .row-fluid [class*="span"]:first-child {
-    margin-left: 0;
-  }
-  .row-fluid .span12 {
-    width: 100%;
-    *width: 99.94680851063829%;
-  }
-  .row-fluid .span11 {
-    width: 91.43646408839778%;
-    *width: 91.38327259903608%;
-  }
-  .row-fluid .span10 {
-    width: 82.87292817679558%;
-    *width: 82.81973668743387%;
-  }
-  .row-fluid .span9 {
-    width: 74.30939226519337%;
-    *width: 74.25620077583166%;
-  }
-  .row-fluid .span8 {
-    width: 65.74585635359117%;
-    *width: 65.69266486422946%;
-  }
-  .row-fluid .span7 {
-    width: 57.18232044198895%;
-    *width: 57.12912895262725%;
-  }
-  .row-fluid .span6 {
-    width: 48.61878453038674%;
-    *width: 48.56559304102504%;
-  }
-  .row-fluid .span5 {
-    width: 40.05524861878453%;
-    *width: 40.00205712942283%;
-  }
-  .row-fluid .span4 {
-    width: 31.491712707182323%;
-    *width: 31.43852121782062%;
-  }
-  .row-fluid .span3 {
-    width: 22.92817679558011%;
-    *width: 22.87498530621841%;
-  }
-  .row-fluid .span2 {
-    width: 14.3646408839779%;
-    *width: 14.311449394616199%;
-  }
-  .row-fluid .span1 {
-    width: 5.801104972375691%;
-    *width: 5.747913483013988%;
-  }
-  .row-fluid .offset12 {
-    margin-left: 105.52486187845304%;
-    *margin-left: 105.41847889972962%;
-  }
-  .row-fluid .offset12:first-child {
-    margin-left: 102.76243093922652%;
-    *margin-left: 102.6560479605031%;
-  }
-  .row-fluid .offset11 {
-    margin-left: 96.96132596685082%;
-    *margin-left: 96.8549429881274%;
-  }
-  .row-fluid .offset11:first-child {
-    margin-left: 94.1988950276243%;
-    *margin-left: 94.09251204890089%;
-  }
-  .row-fluid .offset10 {
-    margin-left: 88.39779005524862%;
-    *margin-left: 88.2914070765252%;
-  }
-  .row-fluid .offset10:first-child {
-    margin-left: 85.6353591160221%;
-    *margin-left: 85.52897613729868%;
-  }
-  .row-fluid .offset9 {
-    margin-left: 79.8342541436464%;
-    *margin-left: 79.72787116492299%;
-  }
-  .row-fluid .offset9:first-child {
-    margin-left: 77.07182320441989%;
-    *margin-left: 76.96544022569647%;
-  }
-  .row-fluid .offset8 {
-    margin-left: 71.2707182320442%;
-    *margin-left: 71.16433525332079%;
-  }
-  .row-fluid .offset8:first-child {
-    margin-left: 68.50828729281768%;
-    *margin-left: 68.40190431409427%;
-  }
-  .row-fluid .offset7 {
-    margin-left: 62.70718232044199%;
-    *margin-left: 62.600799341718584%;
-  }
-  .row-fluid .offset7:first-child {
-    margin-left: 59.94475138121547%;
-    *margin-left: 59.838368402492065%;
-  }
-  .row-fluid .offset6 {
-    margin-left: 54.14364640883978%;
-    *margin-left: 54.037263430116376%;
-  }
-  .row-fluid .offset6:first-child {
-    margin-left: 51.38121546961326%;
-    *margin-left: 51.27483249088986%;
-  }
-  .row-fluid .offset5 {
-    margin-left: 45.58011049723757%;
-    *margin-left: 45.47372751851417%;
-  }
-  .row-fluid .offset5:first-child {
-    margin-left: 42.81767955801105%;
-    *margin-left: 42.71129657928765%;
-  }
-  .row-fluid .offset4 {
-    margin-left: 37.01657458563536%;
-    *margin-left: 36.91019160691196%;
-  }
-  .row-fluid .offset4:first-child {
-    margin-left: 34.25414364640884%;
-    *margin-left: 34.14776066768544%;
-  }
-  .row-fluid .offset3 {
-    margin-left: 28.45303867403315%;
-    *margin-left: 28.346655695309746%;
-  }
-  .row-fluid .offset3:first-child {
-    margin-left: 25.69060773480663%;
-    *margin-left: 25.584224756083227%;
-  }
-  .row-fluid .offset2 {
-    margin-left: 19.88950276243094%;
-    *margin-left: 19.783119783707537%;
-  }
-  .row-fluid .offset2:first-child {
-    margin-left: 17.12707182320442%;
-    *margin-left: 17.02068884448102%;
-  }
-  .row-fluid .offset1 {
-    margin-left: 11.32596685082873%;
-    *margin-left: 11.219583872105325%;
-  }
-  .row-fluid .offset1:first-child {
-    margin-left: 8.56353591160221%;
-    *margin-left: 8.457152932878806%;
-  }
-  input,
-  textarea,
-  .uneditable-input {
-    margin-left: 0;
-  }
-  .controls-row [class*="span"] + [class*="span"] {
-    margin-left: 20px;
-  }
-  input.span12,
-  textarea.span12,
-  .uneditable-input.span12 {
-    width: 710px;
-  }
-  input.span11,
-  textarea.span11,
-  .uneditable-input.span11 {
-    width: 648px;
-  }
-  input.span10,
-  textarea.span10,
-  .uneditable-input.span10 {
-    width: 586px;
-  }
-  input.span9,
-  textarea.span9,
-  .uneditable-input.span9 {
-    width: 524px;
-  }
-  input.span8,
-  textarea.span8,
-  .uneditable-input.span8 {
-    width: 462px;
-  }
-  input.span7,
-  textarea.span7,
-  .uneditable-input.span7 {
-    width: 400px;
-  }
-  input.span6,
-  textarea.span6,
-  .uneditable-input.span6 {
-    width: 338px;
-  }
-  input.span5,
-  textarea.span5,
-  .uneditable-input.span5 {
-    width: 276px;
-  }
-  input.span4,
-  textarea.span4,
-  .uneditable-input.span4 {
-    width: 214px;
-  }
-  input.span3,
-  textarea.span3,
-  .uneditable-input.span3 {
-    width: 152px;
-  }
-  input.span2,
-  textarea.span2,
-  .uneditable-input.span2 {
-    width: 90px;
-  }
-  input.span1,
-  textarea.span1,
-  .uneditable-input.span1 {
-    width: 28px;
-  }
-}
-
-@media (max-width: 767px) {
-  body {
-    padding-right: 20px;
-    padding-left: 20px;
-  }
-  .navbar-fixed-top,
-  .navbar-fixed-bottom {
-    margin-right: -20px;
-    margin-left: -20px;
-  }
-  .container-fluid {
-    padding: 0;
-  }
-  .dl-horizontal dt {
-    float: none;
-    width: auto;
-    clear: none;
-    text-align: left;
-  }
-  .dl-horizontal dd {
-    margin-left: 0;
-  }
-  .container {
-    width: auto;
-  }
-  .row-fluid {
-    width: 100%;
-  }
-  .row,
-  .thumbnails {
-    margin-left: 0;
-  }
-  .thumbnails > li {
-    float: none;
-    margin-left: 0;
-  }
-  [class*="span"],
-  .row-fluid [class*="span"] {
-    display: block;
-    float: none;
-    width: auto;
-    margin-left: 0;
-  }
-  .span12,
-  .row-fluid .span12 {
-    width: 100%;
-    -webkit-box-sizing: border-box;
-       -moz-box-sizing: border-box;
-            box-sizing: border-box;
-  }
-  .input-large,
-  .input-xlarge,
-  .input-xxlarge,
-  input[class*="span"],
-  select[class*="span"],
-  textarea[class*="span"],
-  .uneditable-input {
-    display: block;
-    width: 100%;
-    min-height: 30px;
-    -webkit-box-sizing: border-box;
-       -moz-box-sizing: border-box;
-            box-sizing: border-box;
-  }
-  .input-prepend input,
-  .input-append input,
-  .input-prepend input[class*="span"],
-  .input-append input[class*="span"] {
-    display: inline-block;
-    width: auto;
-  }
-  .modal {
-    position: fixed;
-    top: 20px;
-    right: 20px;
-    left: 20px;
-    width: auto;
-    margin: 0;
-  }
-  .modal.fade.in {
-    top: auto;
-  }
-}
-
-@media (max-width: 480px) {
-  .nav-collapse {
-    -webkit-transform: translate3d(0, 0, 0);
-  }
-  .page-header h1 small {
-    display: block;
-    line-height: 20px;
-  }
-  input[type="checkbox"],
-  input[type="radio"] {
-    border: 1px solid #ccc;
-  }
-  .form-horizontal .control-group > label {
-    float: none;
-    width: auto;
-    padding-top: 0;
-    text-align: left;
-  }
-  .form-horizontal .controls {
-    margin-left: 0;
-  }
-  .form-horizontal .control-list {
-    padding-top: 0;
-  }
-  .form-horizontal .form-actions {
-    padding-right: 10px;
-    padding-left: 10px;
-  }
-  .modal {
-    top: 10px;
-    right: 10px;
-    left: 10px;
-  }
-  .modal-header .close {
-    padding: 10px;
-    margin: -10px;
-  }
-  .carousel-caption {
-    position: static;
-  }
-}
-
-@media (max-width: 979px) {
-  body {
-    padding-top: 0;
-  }
-  .navbar-fixed-top,
-  .navbar-fixed-bottom {
-    position: static;
-  }
-  .navbar-fixed-top {
-    margin-bottom: 20px;
-  }
-  .navbar-fixed-bottom {
-    margin-top: 20px;
-  }
-  .navbar-fixed-top .navbar-inner,
-  .navbar-fixed-bottom .navbar-inner {
-    padding: 5px;
-  }
-  .navbar .container {
-    width: auto;
-    padding: 0;
-  }
-  .navbar .brand {
-    padding-right: 10px;
-    padding-left: 10px;
-    margin: 0 0 0 -5px;
-  }
-  .nav-collapse {
-    clear: both;
-  }
-  .nav-collapse .nav {
-    float: none;
-    margin: 0 0 10px;
-  }
-  .nav-collapse .nav > li {
-    float: none;
-  }
-  .nav-collapse .nav > li > a {
-    margin-bottom: 2px;
-  }
-  .nav-collapse .nav > .divider-vertical {
-    display: none;
-  }
-  .nav-collapse .nav .nav-header {
-    color: #555555;
-    text-shadow: none;
-  }
-  .nav-collapse .nav > li > a,
-  .nav-collapse .dropdown-menu a {
-    padding: 9px 15px;
-    font-weight: bold;
-    color: #555555;
-    -webkit-border-radius: 3px;
-       -moz-border-radius: 3px;
-            border-radius: 3px;
-  }
-  .nav-collapse .btn {
-    padding: 4px 10px 4px;
-    font-weight: normal;
-    -webkit-border-radius: 4px;
-       -moz-border-radius: 4px;
-            border-radius: 4px;
-  }
-  .nav-collapse .dropdown-menu li + li a {
-    margin-bottom: 2px;
-  }
-  .nav-collapse .nav > li > a:hover,
-  .nav-collapse .dropdown-menu a:hover {
-    background-color: #f2f2f2;
-  }
-  .navbar-inverse .nav-collapse .nav > li > a:hover,
-  .navbar-inverse .nav-collapse .dropdown-menu a:hover {
-    background-color: #111111;
-  }
-  .nav-collapse.in .btn-group {
-    padding: 0;
-    margin-top: 5px;
-  }
-  .nav-collapse .dropdown-menu {
-    position: static;
-    top: auto;
-    left: auto;
-    display: block;
-    float: none;
-    max-width: none;
-    padding: 0;
-    margin: 0 15px;
-    background-color: transparent;
-    border: none;
-    -webkit-border-radius: 0;
-       -moz-border-radius: 0;
-            border-radius: 0;
-    -webkit-box-shadow: none;
-       -moz-box-shadow: none;
-            box-shadow: none;
-  }
-  .nav-collapse .dropdown-menu:before,
-  .nav-collapse .dropdown-menu:after {
-    display: none;
-  }
-  .nav-collapse .dropdown-menu .divider {
-    display: none;
-  }
-  .nav-collapse .navbar-form,
-  .nav-collapse .navbar-search {
-    float: none;
-    padding: 10px 15px;
-    margin: 10px 0;
-    border-top: 1px solid #f2f2f2;
-    border-bottom: 1px solid #f2f2f2;
-    -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1);
-       -moz-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1);
-            box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1);
-  }
-  .navbar .nav-collapse .nav.pull-right {
-    float: none;
-    margin-left: 0;
-  }
-  .nav-collapse,
-  .nav-collapse.collapse {
-    height: 0;
-    overflow: hidden;
-  }
-  .navbar .btn-navbar {
-    display: block;
-  }
-  .navbar-static .navbar-inner {
-    padding-right: 10px;
-    padding-left: 10px;
-  }
-}
-
-@media (min-width: 980px) {
-  .nav-collapse.collapse {
-    height: auto !important;
-    overflow: visible !important;
-  }
-}
diff --git a/docs/css/bootstrap-responsive.min.css b/docs/css/bootstrap-responsive.min.css
deleted file mode 100644
index ab59da3427c7a..0000000000000
--- a/docs/css/bootstrap-responsive.min.css
+++ /dev/null
@@ -1,9 +0,0 @@
-/*!
- * Bootstrap Responsive v2.1.0
- *
- * Copyright 2012 Twitter, Inc
- * Licensed under the Apache License v2.0
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Designed and built with all the love in the world @twitter by @mdo and @fat.
- */.clearfix{*zoom:1}.clearfix:before,.clearfix:after{display:table;line-height:0;content:""}.clearfix:after{clear:both}.hide-text{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.input-block-level{display:block;width:100%;min-height:30px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.hidden{display:none;visibility:hidden}.visible-phone{display:none!important}.visible-tablet{display:none!important}.hidden-desktop{display:none!important}.visible-desktop{display:inherit!important}@media(min-width:768px) and (max-width:979px){.hidden-desktop{display:inherit!important}.visible-desktop{display:none!important}.visible-tablet{display:inherit!important}.hidden-tablet{display:none!important}}@media(max-width:767px){.hidden-desktop{display:inherit!important}.visible-desktop{display:none!important}.visible-phone{display:inherit!important}.hidden-phone{display:none!important}}@media(min-width:1200px){.row{margin-left:-30px;*zoom:1}.row:before,.row:after{display:table;line-height:0;content:""}.row:after{clear:both}[class*="span"]{float:left;margin-left:30px}.container,.navbar-static-top .container,.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:1170px}.span12{width:1170px}.span11{width:1070px}.span10{width:970px}.span9{width:870px}.span8{width:770px}.span7{width:670px}.span6{width:570px}.span5{width:470px}.span4{width:370px}.span3{width:270px}.span2{width:170px}.span1{width:70px}.offset12{margin-left:1230px}.offset11{margin-left:1130px}.offset10{margin-left:1030px}.offset9{margin-left:930px}.offset8{margin-left:830px}.offset7{margin-left:730px}.offset6{margin-left:630px}.offset5{margin-left:530px}.offset4{margin-left:430px}.offset3{margin-left:330px}.offset2{margin-left:230px}.offset1{margin-left:130px}.row-fluid{width:100%;*zoom:1}.row-fluid:before,.row-fluid:after{display:table;line-height:0;content:""}.row-fluid:after{clear:both}.row-fluid [class*="span"]{display:block;float:left;width:100%;min-height:30px;margin-left:2.564102564102564%;*margin-left:2.5109110747408616%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.row-fluid [class*="span"]:first-child{margin-left:0}.row-fluid .span12{width:100%;*width:99.94680851063829%}.row-fluid .span11{width:91.45299145299145%;*width:91.39979996362975%}.row-fluid .span10{width:82.90598290598291%;*width:82.8527914166212%}.row-fluid .span9{width:74.35897435897436%;*width:74.30578286961266%}.row-fluid .span8{width:65.81196581196582%;*width:65.75877432260411%}.row-fluid .span7{width:57.26495726495726%;*width:57.21176577559556%}.row-fluid .span6{width:48.717948717948715%;*width:48.664757228587014%}.row-fluid .span5{width:40.17094017094017%;*width:40.11774868157847%}.row-fluid .span4{width:31.623931623931625%;*width:31.570740134569924%}.row-fluid .span3{width:23.076923076923077%;*width:23.023731587561375%}.row-fluid .span2{width:14.52991452991453%;*width:14.476723040552828%}.row-fluid .span1{width:5.982905982905983%;*width:5.929714493544281%}.row-fluid .offset12{margin-left:105.12820512820512%;*margin-left:105.02182214948171%}.row-fluid .offset12:first-child{margin-left:102.56410256410257%;*margin-left:102.45771958537915%}.row-fluid .offset11{margin-left:96.58119658119658%;*margin-left:96.47481360247316%}.row-fluid .offset11:first-child{margin-left:94.01709401709402%;*margin-left:93.91071103837061%}.row-fluid .offset10{margin-left:88.03418803418803%;*margin-left:87.92780505546462%}.row-fluid .offset10:first-child{margin-left:85.47008547008548%;*margin-left:85.36370249136206%}.row-fluid .offset9{margin-left:79.48717948717949%;*margin-left:79.38079650845607%}.row-fluid .offset9:first-child{margin-left:76.92307692307693%;*margin-left:76.81669394435352%}.row-fluid .offset8{margin-left:70.94017094017094%;*margin-left:70.83378796144753%}.row-fluid .offset8:first-child{margin-left:68.37606837606839%;*margin-left:68.26968539734497%}.row-fluid .offset7{margin-left:62.393162393162385%;*margin-left:62.28677941443899%}.row-fluid .offset7:first-child{margin-left:59.82905982905982%;*margin-left:59.72267685033642%}.row-fluid .offset6{margin-left:53.84615384615384%;*margin-left:53.739770867430444%}.row-fluid .offset6:first-child{margin-left:51.28205128205128%;*margin-left:51.175668303327875%}.row-fluid .offset5{margin-left:45.299145299145295%;*margin-left:45.1927623204219%}.row-fluid .offset5:first-child{margin-left:42.73504273504273%;*margin-left:42.62865975631933%}.row-fluid .offset4{margin-left:36.75213675213675%;*margin-left:36.645753773413354%}.row-fluid .offset4:first-child{margin-left:34.18803418803419%;*margin-left:34.081651209310785%}.row-fluid .offset3{margin-left:28.205128205128204%;*margin-left:28.0987452264048%}.row-fluid .offset3:first-child{margin-left:25.641025641025642%;*margin-left:25.53464266230224%}.row-fluid .offset2{margin-left:19.65811965811966%;*margin-left:19.551736679396257%}.row-fluid .offset2:first-child{margin-left:17.094017094017094%;*margin-left:16.98763411529369%}.row-fluid .offset1{margin-left:11.11111111111111%;*margin-left:11.004728132387708%}.row-fluid .offset1:first-child{margin-left:8.547008547008547%;*margin-left:8.440625568285142%}input,textarea,.uneditable-input{margin-left:0}.controls-row [class*="span"]+[class*="span"]{margin-left:30px}input.span12,textarea.span12,.uneditable-input.span12{width:1156px}input.span11,textarea.span11,.uneditable-input.span11{width:1056px}input.span10,textarea.span10,.uneditable-input.span10{width:956px}input.span9,textarea.span9,.uneditable-input.span9{width:856px}input.span8,textarea.span8,.uneditable-input.span8{width:756px}input.span7,textarea.span7,.uneditable-input.span7{width:656px}input.span6,textarea.span6,.uneditable-input.span6{width:556px}input.span5,textarea.span5,.uneditable-input.span5{width:456px}input.span4,textarea.span4,.uneditable-input.span4{width:356px}input.span3,textarea.span3,.uneditable-input.span3{width:256px}input.span2,textarea.span2,.uneditable-input.span2{width:156px}input.span1,textarea.span1,.uneditable-input.span1{width:56px}.thumbnails{margin-left:-30px}.thumbnails>li{margin-left:30px}.row-fluid .thumbnails{margin-left:0}}@media(min-width:768px) and (max-width:979px){.row{margin-left:-20px;*zoom:1}.row:before,.row:after{display:table;line-height:0;content:""}.row:after{clear:both}[class*="span"]{float:left;margin-left:20px}.container,.navbar-static-top .container,.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:724px}.span12{width:724px}.span11{width:662px}.span10{width:600px}.span9{width:538px}.span8{width:476px}.span7{width:414px}.span6{width:352px}.span5{width:290px}.span4{width:228px}.span3{width:166px}.span2{width:104px}.span1{width:42px}.offset12{margin-left:764px}.offset11{margin-left:702px}.offset10{margin-left:640px}.offset9{margin-left:578px}.offset8{margin-left:516px}.offset7{margin-left:454px}.offset6{margin-left:392px}.offset5{margin-left:330px}.offset4{margin-left:268px}.offset3{margin-left:206px}.offset2{margin-left:144px}.offset1{margin-left:82px}.row-fluid{width:100%;*zoom:1}.row-fluid:before,.row-fluid:after{display:table;line-height:0;content:""}.row-fluid:after{clear:both}.row-fluid [class*="span"]{display:block;float:left;width:100%;min-height:30px;margin-left:2.7624309392265194%;*margin-left:2.709239449864817%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.row-fluid [class*="span"]:first-child{margin-left:0}.row-fluid .span12{width:100%;*width:99.94680851063829%}.row-fluid .span11{width:91.43646408839778%;*width:91.38327259903608%}.row-fluid .span10{width:82.87292817679558%;*width:82.81973668743387%}.row-fluid .span9{width:74.30939226519337%;*width:74.25620077583166%}.row-fluid .span8{width:65.74585635359117%;*width:65.69266486422946%}.row-fluid .span7{width:57.18232044198895%;*width:57.12912895262725%}.row-fluid .span6{width:48.61878453038674%;*width:48.56559304102504%}.row-fluid .span5{width:40.05524861878453%;*width:40.00205712942283%}.row-fluid .span4{width:31.491712707182323%;*width:31.43852121782062%}.row-fluid .span3{width:22.92817679558011%;*width:22.87498530621841%}.row-fluid .span2{width:14.3646408839779%;*width:14.311449394616199%}.row-fluid .span1{width:5.801104972375691%;*width:5.747913483013988%}.row-fluid .offset12{margin-left:105.52486187845304%;*margin-left:105.41847889972962%}.row-fluid .offset12:first-child{margin-left:102.76243093922652%;*margin-left:102.6560479605031%}.row-fluid .offset11{margin-left:96.96132596685082%;*margin-left:96.8549429881274%}.row-fluid .offset11:first-child{margin-left:94.1988950276243%;*margin-left:94.09251204890089%}.row-fluid .offset10{margin-left:88.39779005524862%;*margin-left:88.2914070765252%}.row-fluid .offset10:first-child{margin-left:85.6353591160221%;*margin-left:85.52897613729868%}.row-fluid .offset9{margin-left:79.8342541436464%;*margin-left:79.72787116492299%}.row-fluid .offset9:first-child{margin-left:77.07182320441989%;*margin-left:76.96544022569647%}.row-fluid .offset8{margin-left:71.2707182320442%;*margin-left:71.16433525332079%}.row-fluid .offset8:first-child{margin-left:68.50828729281768%;*margin-left:68.40190431409427%}.row-fluid .offset7{margin-left:62.70718232044199%;*margin-left:62.600799341718584%}.row-fluid .offset7:first-child{margin-left:59.94475138121547%;*margin-left:59.838368402492065%}.row-fluid .offset6{margin-left:54.14364640883978%;*margin-left:54.037263430116376%}.row-fluid .offset6:first-child{margin-left:51.38121546961326%;*margin-left:51.27483249088986%}.row-fluid .offset5{margin-left:45.58011049723757%;*margin-left:45.47372751851417%}.row-fluid .offset5:first-child{margin-left:42.81767955801105%;*margin-left:42.71129657928765%}.row-fluid .offset4{margin-left:37.01657458563536%;*margin-left:36.91019160691196%}.row-fluid .offset4:first-child{margin-left:34.25414364640884%;*margin-left:34.14776066768544%}.row-fluid .offset3{margin-left:28.45303867403315%;*margin-left:28.346655695309746%}.row-fluid .offset3:first-child{margin-left:25.69060773480663%;*margin-left:25.584224756083227%}.row-fluid .offset2{margin-left:19.88950276243094%;*margin-left:19.783119783707537%}.row-fluid .offset2:first-child{margin-left:17.12707182320442%;*margin-left:17.02068884448102%}.row-fluid .offset1{margin-left:11.32596685082873%;*margin-left:11.219583872105325%}.row-fluid .offset1:first-child{margin-left:8.56353591160221%;*margin-left:8.457152932878806%}input,textarea,.uneditable-input{margin-left:0}.controls-row [class*="span"]+[class*="span"]{margin-left:20px}input.span12,textarea.span12,.uneditable-input.span12{width:710px}input.span11,textarea.span11,.uneditable-input.span11{width:648px}input.span10,textarea.span10,.uneditable-input.span10{width:586px}input.span9,textarea.span9,.uneditable-input.span9{width:524px}input.span8,textarea.span8,.uneditable-input.span8{width:462px}input.span7,textarea.span7,.uneditable-input.span7{width:400px}input.span6,textarea.span6,.uneditable-input.span6{width:338px}input.span5,textarea.span5,.uneditable-input.span5{width:276px}input.span4,textarea.span4,.uneditable-input.span4{width:214px}input.span3,textarea.span3,.uneditable-input.span3{width:152px}input.span2,textarea.span2,.uneditable-input.span2{width:90px}input.span1,textarea.span1,.uneditable-input.span1{width:28px}}@media(max-width:767px){body{padding-right:20px;padding-left:20px}.navbar-fixed-top,.navbar-fixed-bottom{margin-right:-20px;margin-left:-20px}.container-fluid{padding:0}.dl-horizontal dt{float:none;width:auto;clear:none;text-align:left}.dl-horizontal dd{margin-left:0}.container{width:auto}.row-fluid{width:100%}.row,.thumbnails{margin-left:0}.thumbnails>li{float:none;margin-left:0}[class*="span"],.row-fluid [class*="span"]{display:block;float:none;width:auto;margin-left:0}.span12,.row-fluid .span12{width:100%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.input-large,.input-xlarge,.input-xxlarge,input[class*="span"],select[class*="span"],textarea[class*="span"],.uneditable-input{display:block;width:100%;min-height:30px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.input-prepend input,.input-append input,.input-prepend input[class*="span"],.input-append input[class*="span"]{display:inline-block;width:auto}.modal{position:fixed;top:20px;right:20px;left:20px;width:auto;margin:0}.modal.fade.in{top:auto}}@media(max-width:480px){.nav-collapse{-webkit-transform:translate3d(0,0,0)}.page-header h1 small{display:block;line-height:20px}input[type="checkbox"],input[type="radio"]{border:1px solid #ccc}.form-horizontal .control-group>label{float:none;width:auto;padding-top:0;text-align:left}.form-horizontal .controls{margin-left:0}.form-horizontal .control-list{padding-top:0}.form-horizontal .form-actions{padding-right:10px;padding-left:10px}.modal{top:10px;right:10px;left:10px}.modal-header .close{padding:10px;margin:-10px}.carousel-caption{position:static}}@media(max-width:979px){body{padding-top:0}.navbar-fixed-top,.navbar-fixed-bottom{position:static}.navbar-fixed-top{margin-bottom:20px}.navbar-fixed-bottom{margin-top:20px}.navbar-fixed-top .navbar-inner,.navbar-fixed-bottom .navbar-inner{padding:5px}.navbar .container{width:auto;padding:0}.navbar .brand{padding-right:10px;padding-left:10px;margin:0 0 0 -5px}.nav-collapse{clear:both}.nav-collapse .nav{float:none;margin:0 0 10px}.nav-collapse .nav>li{float:none}.nav-collapse .nav>li>a{margin-bottom:2px}.nav-collapse .nav>.divider-vertical{display:none}.nav-collapse .nav .nav-header{color:#555;text-shadow:none}.nav-collapse .nav>li>a,.nav-collapse .dropdown-menu a{padding:9px 15px;font-weight:bold;color:#555;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}.nav-collapse .btn{padding:4px 10px 4px;font-weight:normal;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.nav-collapse .dropdown-menu li+li a{margin-bottom:2px}.nav-collapse .nav>li>a:hover,.nav-collapse .dropdown-menu a:hover{background-color:#f2f2f2}.navbar-inverse .nav-collapse .nav>li>a:hover,.navbar-inverse .nav-collapse .dropdown-menu a:hover{background-color:#111}.nav-collapse.in .btn-group{padding:0;margin-top:5px}.nav-collapse .dropdown-menu{position:static;top:auto;left:auto;display:block;float:none;max-width:none;padding:0;margin:0 15px;background-color:transparent;border:0;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none}.nav-collapse .dropdown-menu:before,.nav-collapse .dropdown-menu:after{display:none}.nav-collapse .dropdown-menu .divider{display:none}.nav-collapse .navbar-form,.nav-collapse .navbar-search{float:none;padding:10px 15px;margin:10px 0;border-top:1px solid #f2f2f2;border-bottom:1px solid #f2f2f2;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1);-moz-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1);box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1)}.navbar .nav-collapse .nav.pull-right{float:none;margin-left:0}.nav-collapse,.nav-collapse.collapse{height:0;overflow:hidden}.navbar .btn-navbar{display:block}.navbar-static .navbar-inner{padding-right:10px;padding-left:10px}}@media(min-width:980px){.nav-collapse.collapse{height:auto!important;overflow:visible!important}}
diff --git a/docs/css/bootstrap.css b/docs/css/bootstrap.css
deleted file mode 100644
index b51ef7b42b438..0000000000000
--- a/docs/css/bootstrap.css
+++ /dev/null
@@ -1,5624 +0,0 @@
-/*!
- * Bootstrap v2.1.0
- *
- * Copyright 2012 Twitter, Inc
- * Licensed under the Apache License v2.0
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Designed and built with all the love in the world @twitter by @mdo and @fat.
- */
-
-article,
-aside,
-details,
-figcaption,
-figure,
-footer,
-header,
-hgroup,
-nav,
-section {
-  display: block;
-}
-
-audio,
-canvas,
-video {
-  display: inline-block;
-  *display: inline;
-  *zoom: 1;
-}
-
-audio:not([controls]) {
-  display: none;
-}
-
-html {
-  font-size: 100%;
-  -webkit-text-size-adjust: 100%;
-      -ms-text-size-adjust: 100%;
-}
-
-a:focus {
-  outline: thin dotted #333;
-  outline: 5px auto -webkit-focus-ring-color;
-  outline-offset: -2px;
-}
-
-a:hover,
-a:active {
-  outline: 0;
-}
-
-sub,
-sup {
-  position: relative;
-  font-size: 75%;
-  line-height: 0;
-  vertical-align: baseline;
-}
-
-sup {
-  top: -0.5em;
-}
-
-sub {
-  bottom: -0.25em;
-}
-
-img {
-  height: auto;
-  max-width: 100%;
-  vertical-align: middle;
-  border: 0;
-  -ms-interpolation-mode: bicubic;
-}
-
-#map_canvas img {
-  max-width: none;
-}
-
-button,
-input,
-select,
-textarea {
-  margin: 0;
-  font-size: 100%;
-  vertical-align: middle;
-}
-
-button,
-input {
-  *overflow: visible;
-  line-height: normal;
-}
-
-button::-moz-focus-inner,
-input::-moz-focus-inner {
-  padding: 0;
-  border: 0;
-}
-
-button,
-input[type="button"],
-input[type="reset"],
-input[type="submit"] {
-  cursor: pointer;
-  -webkit-appearance: button;
-}
-
-input[type="search"] {
-  -webkit-box-sizing: content-box;
-     -moz-box-sizing: content-box;
-          box-sizing: content-box;
-  -webkit-appearance: textfield;
-}
-
-input[type="search"]::-webkit-search-decoration,
-input[type="search"]::-webkit-search-cancel-button {
-  -webkit-appearance: none;
-}
-
-textarea {
-  overflow: auto;
-  vertical-align: top;
-}
-
-.clearfix {
-  *zoom: 1;
-}
-
-.clearfix:before,
-.clearfix:after {
-  display: table;
-  line-height: 0;
-  content: "";
-}
-
-.clearfix:after {
-  clear: both;
-}
-
-.hide-text {
-  font: 0/0 a;
-  color: transparent;
-  text-shadow: none;
-  background-color: transparent;
-  border: 0;
-}
-
-.input-block-level {
-  display: block;
-  width: 100%;
-  min-height: 30px;
-  -webkit-box-sizing: border-box;
-     -moz-box-sizing: border-box;
-          box-sizing: border-box;
-}
-
-body {
-  margin: 0;
-  font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
-  font-size: 14px;
-  line-height: 20px;
-  color: #333333;
-  background-color: #ffffff;
-}
-
-a {
-  color: #0088cc;
-  text-decoration: none;
-}
-
-a:hover {
-  color: #005580;
-  text-decoration: underline;
-}
-
-.img-rounded {
-  -webkit-border-radius: 6px;
-     -moz-border-radius: 6px;
-          border-radius: 6px;
-}
-
-.img-polaroid {
-  padding: 4px;
-  background-color: #fff;
-  border: 1px solid #ccc;
-  border: 1px solid rgba(0, 0, 0, 0.2);
-  -webkit-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
-     -moz-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
-          box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
-}
-
-.img-circle {
-  -webkit-border-radius: 500px;
-     -moz-border-radius: 500px;
-          border-radius: 500px;
-}
-
-.row {
-  margin-left: -20px;
-  *zoom: 1;
-}
-
-.row:before,
-.row:after {
-  display: table;
-  line-height: 0;
-  content: "";
-}
-
-.row:after {
-  clear: both;
-}
-
-[class*="span"] {
-  float: left;
-  margin-left: 20px;
-}
-
-.container,
-.navbar-static-top .container,
-.navbar-fixed-top .container,
-.navbar-fixed-bottom .container {
-  width: 940px;
-}
-
-.span12 {
-  width: 940px;
-}
-
-.span11 {
-  width: 860px;
-}
-
-.span10 {
-  width: 780px;
-}
-
-.span9 {
-  width: 700px;
-}
-
-.span8 {
-  width: 620px;
-}
-
-.span7 {
-  width: 540px;
-}
-
-.span6 {
-  width: 460px;
-}
-
-.span5 {
-  width: 380px;
-}
-
-.span4 {
-  width: 300px;
-}
-
-.span3 {
-  width: 220px;
-}
-
-.span2 {
-  width: 140px;
-}
-
-.span1 {
-  width: 60px;
-}
-
-.offset12 {
-  margin-left: 980px;
-}
-
-.offset11 {
-  margin-left: 900px;
-}
-
-.offset10 {
-  margin-left: 820px;
-}
-
-.offset9 {
-  margin-left: 740px;
-}
-
-.offset8 {
-  margin-left: 660px;
-}
-
-.offset7 {
-  margin-left: 580px;
-}
-
-.offset6 {
-  margin-left: 500px;
-}
-
-.offset5 {
-  margin-left: 420px;
-}
-
-.offset4 {
-  margin-left: 340px;
-}
-
-.offset3 {
-  margin-left: 260px;
-}
-
-.offset2 {
-  margin-left: 180px;
-}
-
-.offset1 {
-  margin-left: 100px;
-}
-
-.row-fluid {
-  width: 100%;
-  *zoom: 1;
-}
-
-.row-fluid:before,
-.row-fluid:after {
-  display: table;
-  line-height: 0;
-  content: "";
-}
-
-.row-fluid:after {
-  clear: both;
-}
-
-.row-fluid [class*="span"] {
-  display: block;
-  float: left;
-  width: 100%;
-  min-height: 30px;
-  margin-left: 2.127659574468085%;
-  *margin-left: 2.074468085106383%;
-  -webkit-box-sizing: border-box;
-     -moz-box-sizing: border-box;
-          box-sizing: border-box;
-}
-
-.row-fluid [class*="span"]:first-child {
-  margin-left: 0;
-}
-
-.row-fluid .span12 {
-  width: 100%;
-  *width: 99.94680851063829%;
-}
-
-.row-fluid .span11 {
-  width: 91.48936170212765%;
-  *width: 91.43617021276594%;
-}
-
-.row-fluid .span10 {
-  width: 82.97872340425532%;
-  *width: 82.92553191489361%;
-}
-
-.row-fluid .span9 {
-  width: 74.46808510638297%;
-  *width: 74.41489361702126%;
-}
-
-.row-fluid .span8 {
-  width: 65.95744680851064%;
-  *width: 65.90425531914893%;
-}
-
-.row-fluid .span7 {
-  width: 57.44680851063829%;
-  *width: 57.39361702127659%;
-}
-
-.row-fluid .span6 {
-  width: 48.93617021276595%;
-  *width: 48.88297872340425%;
-}
-
-.row-fluid .span5 {
-  width: 40.42553191489362%;
-  *width: 40.37234042553192%;
-}
-
-.row-fluid .span4 {
-  width: 31.914893617021278%;
-  *width: 31.861702127659576%;
-}
-
-.row-fluid .span3 {
-  width: 23.404255319148934%;
-  *width: 23.351063829787233%;
-}
-
-.row-fluid .span2 {
-  width: 14.893617021276595%;
-  *width: 14.840425531914894%;
-}
-
-.row-fluid .span1 {
-  width: 6.382978723404255%;
-  *width: 6.329787234042553%;
-}
-
-.row-fluid .offset12 {
-  margin-left: 104.25531914893617%;
-  *margin-left: 104.14893617021275%;
-}
-
-.row-fluid .offset12:first-child {
-  margin-left: 102.12765957446808%;
-  *margin-left: 102.02127659574467%;
-}
-
-.row-fluid .offset11 {
-  margin-left: 95.74468085106382%;
-  *margin-left: 95.6382978723404%;
-}
-
-.row-fluid .offset11:first-child {
-  margin-left: 93.61702127659574%;
-  *margin-left: 93.51063829787232%;
-}
-
-.row-fluid .offset10 {
-  margin-left: 87.23404255319149%;
-  *margin-left: 87.12765957446807%;
-}
-
-.row-fluid .offset10:first-child {
-  margin-left: 85.1063829787234%;
-  *margin-left: 84.99999999999999%;
-}
-
-.row-fluid .offset9 {
-  margin-left: 78.72340425531914%;
-  *margin-left: 78.61702127659572%;
-}
-
-.row-fluid .offset9:first-child {
-  margin-left: 76.59574468085106%;
-  *margin-left: 76.48936170212764%;
-}
-
-.row-fluid .offset8 {
-  margin-left: 70.2127659574468%;
-  *margin-left: 70.10638297872339%;
-}
-
-.row-fluid .offset8:first-child {
-  margin-left: 68.08510638297872%;
-  *margin-left: 67.9787234042553%;
-}
-
-.row-fluid .offset7 {
-  margin-left: 61.70212765957446%;
-  *margin-left: 61.59574468085106%;
-}
-
-.row-fluid .offset7:first-child {
-  margin-left: 59.574468085106375%;
-  *margin-left: 59.46808510638297%;
-}
-
-.row-fluid .offset6 {
-  margin-left: 53.191489361702125%;
-  *margin-left: 53.085106382978715%;
-}
-
-.row-fluid .offset6:first-child {
-  margin-left: 51.063829787234035%;
-  *margin-left: 50.95744680851063%;
-}
-
-.row-fluid .offset5 {
-  margin-left: 44.68085106382979%;
-  *margin-left: 44.57446808510638%;
-}
-
-.row-fluid .offset5:first-child {
-  margin-left: 42.5531914893617%;
-  *margin-left: 42.4468085106383%;
-}
-
-.row-fluid .offset4 {
-  margin-left: 36.170212765957444%;
-  *margin-left: 36.06382978723405%;
-}
-
-.row-fluid .offset4:first-child {
-  margin-left: 34.04255319148936%;
-  *margin-left: 33.93617021276596%;
-}
-
-.row-fluid .offset3 {
-  margin-left: 27.659574468085104%;
-  *margin-left: 27.5531914893617%;
-}
-
-.row-fluid .offset3:first-child {
-  margin-left: 25.53191489361702%;
-  *margin-left: 25.425531914893618%;
-}
-
-.row-fluid .offset2 {
-  margin-left: 19.148936170212764%;
-  *margin-left: 19.04255319148936%;
-}
-
-.row-fluid .offset2:first-child {
-  margin-left: 17.02127659574468%;
-  *margin-left: 16.914893617021278%;
-}
-
-.row-fluid .offset1 {
-  margin-left: 10.638297872340425%;
-  *margin-left: 10.53191489361702%;
-}
-
-.row-fluid .offset1:first-child {
-  margin-left: 8.51063829787234%;
-  *margin-left: 8.404255319148938%;
-}
-
-[class*="span"].hide,
-.row-fluid [class*="span"].hide {
-  display: none;
-}
-
-[class*="span"].pull-right,
-.row-fluid [class*="span"].pull-right {
-  float: right;
-}
-
-.container {
-  margin-right: auto;
-  margin-left: auto;
-  *zoom: 1;
-}
-
-.container:before,
-.container:after {
-  display: table;
-  line-height: 0;
-  content: "";
-}
-
-.container:after {
-  clear: both;
-}
-
-.container-fluid {
-  padding-right: 20px;
-  padding-left: 20px;
-  *zoom: 1;
-}
-
-.container-fluid:before,
-.container-fluid:after {
-  display: table;
-  line-height: 0;
-  content: "";
-}
-
-.container-fluid:after {
-  clear: both;
-}
-
-p {
-  margin: 0 0 10px;
-}
-
-.lead {
-  margin-bottom: 20px;
-  font-size: 20px;
-  font-weight: 200;
-  line-height: 30px;
-}
-
-small {
-  font-size: 85%;
-}
-
-strong {
-  font-weight: bold;
-}
-
-em {
-  font-style: italic;
-}
-
-cite {
-  font-style: normal;
-}
-
-.muted {
-  color: #999999;
-}
-
-h1,
-h2,
-h3,
-h4,
-h5,
-h6 {
-  margin: 10px 0;
-  font-family: inherit;
-  font-weight: bold;
-  line-height: 1;
-  color: inherit;
-  text-rendering: optimizelegibility;
-}
-
-h1 small,
-h2 small,
-h3 small,
-h4 small,
-h5 small,
-h6 small {
-  font-weight: normal;
-  line-height: 1;
-  color: #999999;
-}
-
-h1 {
-  font-size: 36px;
-  line-height: 40px;
-}
-
-h2 {
-  font-size: 30px;
-  line-height: 40px;
-}
-
-h3 {
-  font-size: 24px;
-  line-height: 40px;
-}
-
-h4 {
-  font-size: 18px;
-  line-height: 20px;
-}
-
-h5 {
-  font-size: 14px;
-  line-height: 20px;
-}
-
-h6 {
-  font-size: 12px;
-  line-height: 20px;
-}
-
-h1 small {
-  font-size: 24px;
-}
-
-h2 small {
-  font-size: 18px;
-}
-
-h3 small {
-  font-size: 14px;
-}
-
-h4 small {
-  font-size: 14px;
-}
-
-.page-header {
-  padding-bottom: 9px;
-  margin: 20px 0 30px;
-  border-bottom: 1px solid #eeeeee;
-}
-
-ul,
-ol {
-  padding: 0;
-  margin: 0 0 10px 25px;
-}
-
-ul ul,
-ul ol,
-ol ol,
-ol ul {
-  margin-bottom: 0;
-}
-
-li {
-  line-height: 20px;
-}
-
-ul.unstyled,
-ol.unstyled {
-  margin-left: 0;
-  list-style: none;
-}
-
-dl {
-  margin-bottom: 20px;
-}
-
-dt,
-dd {
-  line-height: 20px;
-}
-
-dt {
-  font-weight: bold;
-}
-
-dd {
-  margin-left: 10px;
-}
-
-.dl-horizontal dt {
-  float: left;
-  width: 120px;
-  overflow: hidden;
-  clear: left;
-  text-align: right;
-  text-overflow: ellipsis;
-  white-space: nowrap;
-}
-
-.dl-horizontal dd {
-  margin-left: 130px;
-}
-
-hr {
-  margin: 20px 0;
-  border: 0;
-  border-top: 1px solid #eeeeee;
-  border-bottom: 1px solid #ffffff;
-}
-
-abbr[title] {
-  cursor: help;
-  border-bottom: 1px dotted #999999;
-}
-
-abbr.initialism {
-  font-size: 90%;
-  text-transform: uppercase;
-}
-
-blockquote {
-  padding: 0 0 0 15px;
-  margin: 0 0 20px;
-  border-left: 5px solid #eeeeee;
-}
-
-blockquote p {
-  margin-bottom: 0;
-  font-size: 16px;
-  font-weight: 300;
-  line-height: 25px;
-}
-
-blockquote small {
-  display: block;
-  line-height: 20px;
-  color: #999999;
-}
-
-blockquote small:before {
-  content: '\2014 \00A0';
-}
-
-blockquote.pull-right {
-  float: right;
-  padding-right: 15px;
-  padding-left: 0;
-  border-right: 5px solid #eeeeee;
-  border-left: 0;
-}
-
-blockquote.pull-right p,
-blockquote.pull-right small {
-  text-align: right;
-}
-
-blockquote.pull-right small:before {
-  content: '';
-}
-
-blockquote.pull-right small:after {
-  content: '\00A0 \2014';
-}
-
-q:before,
-q:after,
-blockquote:before,
-blockquote:after {
-  content: "";
-}
-
-address {
-  display: block;
-  margin-bottom: 20px;
-  font-style: normal;
-  line-height: 20px;
-}
-
-code,
-pre {
-  padding: 0 3px 2px;
-  font-family: Monaco, Menlo, Consolas, "Courier New", monospace;
-  font-size: 12px;
-  color: #333333;
-  -webkit-border-radius: 3px;
-     -moz-border-radius: 3px;
-          border-radius: 3px;
-}
-
-code {
-  padding: 2px 4px;
-  color: #d14;
-  background-color: #f7f7f9;
-  border: 1px solid #e1e1e8;
-}
-
-pre {
-  display: block;
-  padding: 9.5px;
-  margin: 0 0 10px;
-  font-size: 13px;
-  line-height: 20px;
-  word-break: break-all;
-  word-wrap: break-word;
-  white-space: pre;
-  white-space: pre-wrap;
-  background-color: #f5f5f5;
-  border: 1px solid #ccc;
-  border: 1px solid rgba(0, 0, 0, 0.15);
-  -webkit-border-radius: 4px;
-     -moz-border-radius: 4px;
-          border-radius: 4px;
-}
-
-pre.prettyprint {
-  margin-bottom: 20px;
-}
-
-pre code {
-  padding: 0;
-  color: inherit;
-  background-color: transparent;
-  border: 0;
-}
-
-.pre-scrollable {
-  max-height: 340px;
-  overflow-y: scroll;
-}
-
-form {
-  margin: 0 0 20px;
-}
-
-fieldset {
-  padding: 0;
-  margin: 0;
-  border: 0;
-}
-
-legend {
-  display: block;
-  width: 100%;
-  padding: 0;
-  margin-bottom: 20px;
-  font-size: 21px;
-  line-height: 40px;
-  color: #333333;
-  border: 0;
-  border-bottom: 1px solid #e5e5e5;
-}
-
-legend small {
-  font-size: 15px;
-  color: #999999;
-}
-
-label,
-input,
-button,
-select,
-textarea {
-  font-size: 14px;
-  font-weight: normal;
-  line-height: 20px;
-}
-
-input,
-button,
-select,
-textarea {
-  font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
-}
-
-label {
-  display: block;
-  margin-bottom: 5px;
-}
-
-select,
-textarea,
-input[type="text"],
-input[type="password"],
-input[type="datetime"],
-input[type="datetime-local"],
-input[type="date"],
-input[type="month"],
-input[type="time"],
-input[type="week"],
-input[type="number"],
-input[type="email"],
-input[type="url"],
-input[type="search"],
-input[type="tel"],
-input[type="color"],
-.uneditable-input {
-  display: inline-block;
-  height: 20px;
-  padding: 4px 6px;
-  margin-bottom: 9px;
-  font-size: 14px;
-  line-height: 20px;
-  color: #555555;
-  -webkit-border-radius: 3px;
-     -moz-border-radius: 3px;
-          border-radius: 3px;
-}
-
-input,
-textarea {
-  width: 210px;
-}
-
-textarea {
-  height: auto;
-}
-
-textarea,
-input[type="text"],
-input[type="password"],
-input[type="datetime"],
-input[type="datetime-local"],
-input[type="date"],
-input[type="month"],
-input[type="time"],
-input[type="week"],
-input[type="number"],
-input[type="email"],
-input[type="url"],
-input[type="search"],
-input[type="tel"],
-input[type="color"],
-.uneditable-input {
-  background-color: #ffffff;
-  border: 1px solid #cccccc;
-  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
-     -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
-          box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
-  -webkit-transition: border linear 0.2s, box-shadow linear 0.2s;
-     -moz-transition: border linear 0.2s, box-shadow linear 0.2s;
-       -o-transition: border linear 0.2s, box-shadow linear 0.2s;
-          transition: border linear 0.2s, box-shadow linear 0.2s;
-}
-
-textarea:focus,
-input[type="text"]:focus,
-input[type="password"]:focus,
-input[type="datetime"]:focus,
-input[type="datetime-local"]:focus,
-input[type="date"]:focus,
-input[type="month"]:focus,
-input[type="time"]:focus,
-input[type="week"]:focus,
-input[type="number"]:focus,
-input[type="email"]:focus,
-input[type="url"]:focus,
-input[type="search"]:focus,
-input[type="tel"]:focus,
-input[type="color"]:focus,
-.uneditable-input:focus {
-  border-color: rgba(82, 168, 236, 0.8);
-  outline: 0;
-  outline: thin dotted \9;
-  /* IE6-9 */
-
-  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
-     -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
-          box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
-}
-
-input[type="radio"],
-input[type="checkbox"] {
-  margin: 4px 0 0;
-  margin-top: 1px \9;
-  *margin-top: 0;
-  line-height: normal;
-  cursor: pointer;
-}
-
-input[type="file"],
-input[type="image"],
-input[type="submit"],
-input[type="reset"],
-input[type="button"],
-input[type="radio"],
-input[type="checkbox"] {
-  width: auto;
-}
-
-select,
-input[type="file"] {
-  height: 30px;
-  /* In IE7, the height of the select element cannot be changed by height, only font-size */
-
-  *margin-top: 4px;
-  /* For IE7, add top margin to align select with labels */
-
-  line-height: 30px;
-}
-
-select {
-  width: 220px;
-  background-color: #ffffff;
-  border: 1px solid #bbb;
-}
-
-select[multiple],
-select[size] {
-  height: auto;
-}
-
-select:focus,
-input[type="file"]:focus,
-input[type="radio"]:focus,
-input[type="checkbox"]:focus {
-  outline: thin dotted #333;
-  outline: 5px auto -webkit-focus-ring-color;
-  outline-offset: -2px;
-}
-
-.uneditable-input,
-.uneditable-textarea {
-  color: #999999;
-  cursor: not-allowed;
-  background-color: #fcfcfc;
-  border-color: #cccccc;
-  -webkit-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025);
-     -moz-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025);
-          box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025);
-}
-
-.uneditable-input {
-  overflow: hidden;
-  white-space: nowrap;
-}
-
-.uneditable-textarea {
-  width: auto;
-  height: auto;
-}
-
-input:-moz-placeholder,
-textarea:-moz-placeholder {
-  color: #999999;
-}
-
-input:-ms-input-placeholder,
-textarea:-ms-input-placeholder {
-  color: #999999;
-}
-
-input::-webkit-input-placeholder,
-textarea::-webkit-input-placeholder {
-  color: #999999;
-}
-
-.radio,
-.checkbox {
-  min-height: 18px;
-  padding-left: 18px;
-}
-
-.radio input[type="radio"],
-.checkbox input[type="checkbox"] {
-  float: left;
-  margin-left: -18px;
-}
-
-.controls > .radio:first-child,
-.controls > .checkbox:first-child {
-  padding-top: 5px;
-}
-
-.radio.inline,
-.checkbox.inline {
-  display: inline-block;
-  padding-top: 5px;
-  margin-bottom: 0;
-  vertical-align: middle;
-}
-
-.radio.inline + .radio.inline,
-.checkbox.inline + .checkbox.inline {
-  margin-left: 10px;
-}
-
-.input-mini {
-  width: 60px;
-}
-
-.input-small {
-  width: 90px;
-}
-
-.input-medium {
-  width: 150px;
-}
-
-.input-large {
-  width: 210px;
-}
-
-.input-xlarge {
-  width: 270px;
-}
-
-.input-xxlarge {
-  width: 530px;
-}
-
-input[class*="span"],
-select[class*="span"],
-textarea[class*="span"],
-.uneditable-input[class*="span"],
-.row-fluid input[class*="span"],
-.row-fluid select[class*="span"],
-.row-fluid textarea[class*="span"],
-.row-fluid .uneditable-input[class*="span"] {
-  float: none;
-  margin-left: 0;
-}
-
-.input-append input[class*="span"],
-.input-append .uneditable-input[class*="span"],
-.input-prepend input[class*="span"],
-.input-prepend .uneditable-input[class*="span"],
-.row-fluid input[class*="span"],
-.row-fluid select[class*="span"],
-.row-fluid textarea[class*="span"],
-.row-fluid .uneditable-input[class*="span"],
-.row-fluid .input-prepend [class*="span"],
-.row-fluid .input-append [class*="span"] {
-  display: inline-block;
-}
-
-input,
-textarea,
-.uneditable-input {
-  margin-left: 0;
-}
-
-.controls-row [class*="span"] + [class*="span"] {
-  margin-left: 20px;
-}
-
-input.span12,
-textarea.span12,
-.uneditable-input.span12 {
-  width: 926px;
-}
-
-input.span11,
-textarea.span11,
-.uneditable-input.span11 {
-  width: 846px;
-}
-
-input.span10,
-textarea.span10,
-.uneditable-input.span10 {
-  width: 766px;
-}
-
-input.span9,
-textarea.span9,
-.uneditable-input.span9 {
-  width: 686px;
-}
-
-input.span8,
-textarea.span8,
-.uneditable-input.span8 {
-  width: 606px;
-}
-
-input.span7,
-textarea.span7,
-.uneditable-input.span7 {
-  width: 526px;
-}
-
-input.span6,
-textarea.span6,
-.uneditable-input.span6 {
-  width: 446px;
-}
-
-input.span5,
-textarea.span5,
-.uneditable-input.span5 {
-  width: 366px;
-}
-
-input.span4,
-textarea.span4,
-.uneditable-input.span4 {
-  width: 286px;
-}
-
-input.span3,
-textarea.span3,
-.uneditable-input.span3 {
-  width: 206px;
-}
-
-input.span2,
-textarea.span2,
-.uneditable-input.span2 {
-  width: 126px;
-}
-
-input.span1,
-textarea.span1,
-.uneditable-input.span1 {
-  width: 46px;
-}
-
-.controls-row {
-  *zoom: 1;
-}
-
-.controls-row:before,
-.controls-row:after {
-  display: table;
-  line-height: 0;
-  content: "";
-}
-
-.controls-row:after {
-  clear: both;
-}
-
-.controls-row [class*="span"] {
-  float: left;
-}
-
-input[disabled],
-select[disabled],
-textarea[disabled],
-input[readonly],
-select[readonly],
-textarea[readonly] {
-  cursor: not-allowed;
-  background-color: #eeeeee;
-}
-
-input[type="radio"][disabled],
-input[type="checkbox"][disabled],
-input[type="radio"][readonly],
-input[type="checkbox"][readonly] {
-  background-color: transparent;
-}
-
-.control-group.warning > label,
-.control-group.warning .help-block,
-.control-group.warning .help-inline {
-  color: #c09853;
-}
-
-.control-group.warning .checkbox,
-.control-group.warning .radio,
-.control-group.warning input,
-.control-group.warning select,
-.control-group.warning textarea {
-  color: #c09853;
-  border-color: #c09853;
-  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
-     -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
-          box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
-}
-
-.control-group.warning .checkbox:focus,
-.control-group.warning .radio:focus,
-.control-group.warning input:focus,
-.control-group.warning select:focus,
-.control-group.warning textarea:focus {
-  border-color: #a47e3c;
-  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #dbc59e;
-     -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #dbc59e;
-          box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #dbc59e;
-}
-
-.control-group.warning .input-prepend .add-on,
-.control-group.warning .input-append .add-on {
-  color: #c09853;
-  background-color: #fcf8e3;
-  border-color: #c09853;
-}
-
-.control-group.error > label,
-.control-group.error .help-block,
-.control-group.error .help-inline {
-  color: #b94a48;
-}
-
-.control-group.error .checkbox,
-.control-group.error .radio,
-.control-group.error input,
-.control-group.error select,
-.control-group.error textarea {
-  color: #b94a48;
-  border-color: #b94a48;
-  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
-     -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
-          box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
-}
-
-.control-group.error .checkbox:focus,
-.control-group.error .radio:focus,
-.control-group.error input:focus,
-.control-group.error select:focus,
-.control-group.error textarea:focus {
-  border-color: #953b39;
-  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #d59392;
-     -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #d59392;
-          box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #d59392;
-}
-
-.control-group.error .input-prepend .add-on,
-.control-group.error .input-append .add-on {
-  color: #b94a48;
-  background-color: #f2dede;
-  border-color: #b94a48;
-}
-
-.control-group.success > label,
-.control-group.success .help-block,
-.control-group.success .help-inline {
-  color: #468847;
-}
-
-.control-group.success .checkbox,
-.control-group.success .radio,
-.control-group.success input,
-.control-group.success select,
-.control-group.success textarea {
-  color: #468847;
-  border-color: #468847;
-  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
-     -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
-          box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
-}
-
-.control-group.success .checkbox:focus,
-.control-group.success .radio:focus,
-.control-group.success input:focus,
-.control-group.success select:focus,
-.control-group.success textarea:focus {
-  border-color: #356635;
-  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7aba7b;
-     -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7aba7b;
-          box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7aba7b;
-}
-
-.control-group.success .input-prepend .add-on,
-.control-group.success .input-append .add-on {
-  color: #468847;
-  background-color: #dff0d8;
-  border-color: #468847;
-}
-
-input:focus:required:invalid,
-textarea:focus:required:invalid,
-select:focus:required:invalid {
-  color: #b94a48;
-  border-color: #ee5f5b;
-}
-
-input:focus:required:invalid:focus,
-textarea:focus:required:invalid:focus,
-select:focus:required:invalid:focus {
-  border-color: #e9322d;
-  -webkit-box-shadow: 0 0 6px #f8b9b7;
-     -moz-box-shadow: 0 0 6px #f8b9b7;
-          box-shadow: 0 0 6px #f8b9b7;
-}
-
-.form-actions {
-  padding: 19px 20px 20px;
-  margin-top: 20px;
-  margin-bottom: 20px;
-  background-color: #f5f5f5;
-  border-top: 1px solid #e5e5e5;
-  *zoom: 1;
-}
-
-.form-actions:before,
-.form-actions:after {
-  display: table;
-  line-height: 0;
-  content: "";
-}
-
-.form-actions:after {
-  clear: both;
-}
-
-.help-block,
-.help-inline {
-  color: #595959;
-}
-
-.help-block {
-  display: block;
-  margin-bottom: 10px;
-}
-
-.help-inline {
-  display: inline-block;
-  *display: inline;
-  padding-left: 5px;
-  vertical-align: middle;
-  *zoom: 1;
-}
-
-.input-append,
-.input-prepend {
-  margin-bottom: 5px;
-  font-size: 0;
-  white-space: nowrap;
-}
-
-.input-append input,
-.input-prepend input,
-.input-append select,
-.input-prepend select,
-.input-append .uneditable-input,
-.input-prepend .uneditable-input {
-  position: relative;
-  margin-bottom: 0;
-  *margin-left: 0;
-  font-size: 14px;
-  vertical-align: top;
-  -webkit-border-radius: 0 3px 3px 0;
-     -moz-border-radius: 0 3px 3px 0;
-          border-radius: 0 3px 3px 0;
-}
-
-.input-append input:focus,
-.input-prepend input:focus,
-.input-append select:focus,
-.input-prepend select:focus,
-.input-append .uneditable-input:focus,
-.input-prepend .uneditable-input:focus {
-  z-index: 2;
-}
-
-.input-append .add-on,
-.input-prepend .add-on {
-  display: inline-block;
-  width: auto;
-  height: 20px;
-  min-width: 16px;
-  padding: 4px 5px;
-  font-size: 14px;
-  font-weight: normal;
-  line-height: 20px;
-  text-align: center;
-  text-shadow: 0 1px 0 #ffffff;
-  background-color: #eeeeee;
-  border: 1px solid #ccc;
-}
-
-.input-append .add-on,
-.input-prepend .add-on,
-.input-append .btn,
-.input-prepend .btn {
-  margin-left: -1px;
-  vertical-align: top;
-  -webkit-border-radius: 0;
-     -moz-border-radius: 0;
-          border-radius: 0;
-}
-
-.input-append .active,
-.input-prepend .active {
-  background-color: #a9dba9;
-  border-color: #46a546;
-}
-
-.input-prepend .add-on,
-.input-prepend .btn {
-  margin-right: -1px;
-}
-
-.input-prepend .add-on:first-child,
-.input-prepend .btn:first-child {
-  -webkit-border-radius: 3px 0 0 3px;
-     -moz-border-radius: 3px 0 0 3px;
-          border-radius: 3px 0 0 3px;
-}
-
-.input-append input,
-.input-append select,
-.input-append .uneditable-input {
-  -webkit-border-radius: 3px 0 0 3px;
-     -moz-border-radius: 3px 0 0 3px;
-          border-radius: 3px 0 0 3px;
-}
-
-.input-append .add-on:last-child,
-.input-append .btn:last-child {
-  -webkit-border-radius: 0 3px 3px 0;
-     -moz-border-radius: 0 3px 3px 0;
-          border-radius: 0 3px 3px 0;
-}
-
-.input-prepend.input-append input,
-.input-prepend.input-append select,
-.input-prepend.input-append .uneditable-input {
-  -webkit-border-radius: 0;
-     -moz-border-radius: 0;
-          border-radius: 0;
-}
-
-.input-prepend.input-append .add-on:first-child,
-.input-prepend.input-append .btn:first-child {
-  margin-right: -1px;
-  -webkit-border-radius: 3px 0 0 3px;
-     -moz-border-radius: 3px 0 0 3px;
-          border-radius: 3px 0 0 3px;
-}
-
-.input-prepend.input-append .add-on:last-child,
-.input-prepend.input-append .btn:last-child {
-  margin-left: -1px;
-  -webkit-border-radius: 0 3px 3px 0;
-     -moz-border-radius: 0 3px 3px 0;
-          border-radius: 0 3px 3px 0;
-}
-
-input.search-query {
-  padding-right: 14px;
-  padding-right: 4px \9;
-  padding-left: 14px;
-  padding-left: 4px \9;
-  /* IE7-8 doesn't have border-radius, so don't indent the padding */
-
-  margin-bottom: 0;
-  -webkit-border-radius: 15px;
-     -moz-border-radius: 15px;
-          border-radius: 15px;
-}
-
-/* Allow for input prepend/append in search forms */
-
-.form-search .input-append .search-query,
-.form-search .input-prepend .search-query {
-  -webkit-border-radius: 0;
-     -moz-border-radius: 0;
-          border-radius: 0;
-}
-
-.form-search .input-append .search-query {
-  -webkit-border-radius: 14px 0 0 14px;
-     -moz-border-radius: 14px 0 0 14px;
-          border-radius: 14px 0 0 14px;
-}
-
-.form-search .input-append .btn {
-  -webkit-border-radius: 0 14px 14px 0;
-     -moz-border-radius: 0 14px 14px 0;
-          border-radius: 0 14px 14px 0;
-}
-
-.form-search .input-prepend .search-query {
-  -webkit-border-radius: 0 14px 14px 0;
-     -moz-border-radius: 0 14px 14px 0;
-          border-radius: 0 14px 14px 0;
-}
-
-.form-search .input-prepend .btn {
-  -webkit-border-radius: 14px 0 0 14px;
-     -moz-border-radius: 14px 0 0 14px;
-          border-radius: 14px 0 0 14px;
-}
-
-.form-search input,
-.form-inline input,
-.form-horizontal input,
-.form-search textarea,
-.form-inline textarea,
-.form-horizontal textarea,
-.form-search select,
-.form-inline select,
-.form-horizontal select,
-.form-search .help-inline,
-.form-inline .help-inline,
-.form-horizontal .help-inline,
-.form-search .uneditable-input,
-.form-inline .uneditable-input,
-.form-horizontal .uneditable-input,
-.form-search .input-prepend,
-.form-inline .input-prepend,
-.form-horizontal .input-prepend,
-.form-search .input-append,
-.form-inline .input-append,
-.form-horizontal .input-append {
-  display: inline-block;
-  *display: inline;
-  margin-bottom: 0;
-  vertical-align: middle;
-  *zoom: 1;
-}
-
-.form-search .hide,
-.form-inline .hide,
-.form-horizontal .hide {
-  display: none;
-}
-
-.form-search label,
-.form-inline label,
-.form-search .btn-group,
-.form-inline .btn-group {
-  display: inline-block;
-}
-
-.form-search .input-append,
-.form-inline .input-append,
-.form-search .input-prepend,
-.form-inline .input-prepend {
-  margin-bottom: 0;
-}
-
-.form-search .radio,
-.form-search .checkbox,
-.form-inline .radio,
-.form-inline .checkbox {
-  padding-left: 0;
-  margin-bottom: 0;
-  vertical-align: middle;
-}
-
-.form-search .radio input[type="radio"],
-.form-search .checkbox input[type="checkbox"],
-.form-inline .radio input[type="radio"],
-.form-inline .checkbox input[type="checkbox"] {
-  float: left;
-  margin-right: 3px;
-  margin-left: 0;
-}
-
-.control-group {
-  margin-bottom: 10px;
-}
-
-legend + .control-group {
-  margin-top: 20px;
-  -webkit-margin-top-collapse: separate;
-}
-
-.form-horizontal .control-group {
-  margin-bottom: 20px;
-  *zoom: 1;
-}
-
-.form-horizontal .control-group:before,
-.form-horizontal .control-group:after {
-  display: table;
-  line-height: 0;
-  content: "";
-}
-
-.form-horizontal .control-group:after {
-  clear: both;
-}
-
-.form-horizontal .control-label {
-  float: left;
-  width: 140px;
-  padding-top: 5px;
-  text-align: right;
-}
-
-.form-horizontal .controls {
-  *display: inline-block;
-  *padding-left: 20px;
-  margin-left: 160px;
-  *margin-left: 0;
-}
-
-.form-horizontal .controls:first-child {
-  *padding-left: 160px;
-}
-
-.form-horizontal .help-block {
-  margin-top: 10px;
-  margin-bottom: 0;
-}
-
-.form-horizontal .form-actions {
-  padding-left: 160px;
-}
-
-table {
-  max-width: 100%;
-  background-color: transparent;
-  border-collapse: collapse;
-  border-spacing: 0;
-}
-
-.table {
-  width: 100%;
-  margin-bottom: 20px;
-}
-
-.table th,
-.table td {
-  padding: 8px;
-  line-height: 20px;
-  text-align: left;
-  vertical-align: top;
-  border-top: 1px solid #dddddd;
-}
-
-.table th {
-  font-weight: bold;
-}
-
-.table thead th {
-  vertical-align: bottom;
-}
-
-.table caption + thead tr:first-child th,
-.table caption + thead tr:first-child td,
-.table colgroup + thead tr:first-child th,
-.table colgroup + thead tr:first-child td,
-.table thead:first-child tr:first-child th,
-.table thead:first-child tr:first-child td {
-  border-top: 0;
-}
-
-.table tbody + tbody {
-  border-top: 2px solid #dddddd;
-}
-
-.table-condensed th,
-.table-condensed td {
-  padding: 4px 5px;
-}
-
-.table-bordered {
-  border: 1px solid #dddddd;
-  border-collapse: separate;
-  *border-collapse: collapse;
-  border-left: 0;
-  -webkit-border-radius: 4px;
-     -moz-border-radius: 4px;
-          border-radius: 4px;
-}
-
-.table-bordered th,
-.table-bordered td {
-  border-left: 1px solid #dddddd;
-}
-
-.table-bordered caption + thead tr:first-child th,
-.table-bordered caption + tbody tr:first-child th,
-.table-bordered caption + tbody tr:first-child td,
-.table-bordered colgroup + thead tr:first-child th,
-.table-bordered colgroup + tbody tr:first-child th,
-.table-bordered colgroup + tbody tr:first-child td,
-.table-bordered thead:first-child tr:first-child th,
-.table-bordered tbody:first-child tr:first-child th,
-.table-bordered tbody:first-child tr:first-child td {
-  border-top: 0;
-}
-
-.table-bordered thead:first-child tr:first-child th:first-child,
-.table-bordered tbody:first-child tr:first-child td:first-child {
-  -webkit-border-top-left-radius: 4px;
-          border-top-left-radius: 4px;
-  -moz-border-radius-topleft: 4px;
-}
-
-.table-bordered thead:first-child tr:first-child th:last-child,
-.table-bordered tbody:first-child tr:first-child td:last-child {
-  -webkit-border-top-right-radius: 4px;
-          border-top-right-radius: 4px;
-  -moz-border-radius-topright: 4px;
-}
-
-.table-bordered thead:last-child tr:last-child th:first-child,
-.table-bordered tbody:last-child tr:last-child td:first-child,
-.table-bordered tfoot:last-child tr:last-child td:first-child {
-  -webkit-border-radius: 0 0 0 4px;
-     -moz-border-radius: 0 0 0 4px;
-          border-radius: 0 0 0 4px;
-  -webkit-border-bottom-left-radius: 4px;
-          border-bottom-left-radius: 4px;
-  -moz-border-radius-bottomleft: 4px;
-}
-
-.table-bordered thead:last-child tr:last-child th:last-child,
-.table-bordered tbody:last-child tr:last-child td:last-child,
-.table-bordered tfoot:last-child tr:last-child td:last-child {
-  -webkit-border-bottom-right-radius: 4px;
-          border-bottom-right-radius: 4px;
-  -moz-border-radius-bottomright: 4px;
-}
-
-.table-bordered caption + thead tr:first-child th:first-child,
-.table-bordered caption + tbody tr:first-child td:first-child,
-.table-bordered colgroup + thead tr:first-child th:first-child,
-.table-bordered colgroup + tbody tr:first-child td:first-child {
-  -webkit-border-top-left-radius: 4px;
-          border-top-left-radius: 4px;
-  -moz-border-radius-topleft: 4px;
-}
-
-.table-bordered caption + thead tr:first-child th:last-child,
-.table-bordered caption + tbody tr:first-child td:last-child,
-.table-bordered colgroup + thead tr:first-child th:last-child,
-.table-bordered colgroup + tbody tr:first-child td:last-child {
-  -webkit-border-top-right-radius: 4px;
-          border-top-right-radius: 4px;
-  -moz-border-right-topleft: 4px;
-}
-
-.table-striped tbody tr:nth-child(odd) td,
-.table-striped tbody tr:nth-child(odd) th {
-  background-color: #f9f9f9;
-}
-
-.table-hover tbody tr:hover td,
-.table-hover tbody tr:hover th {
-  background-color: #f5f5f5;
-}
-
-table [class*=span],
-.row-fluid table [class*=span] {
-  display: table-cell;
-  float: none;
-  margin-left: 0;
-}
-
-table .span1 {
-  float: none;
-  width: 44px;
-  margin-left: 0;
-}
-
-table .span2 {
-  float: none;
-  width: 124px;
-  margin-left: 0;
-}
-
-table .span3 {
-  float: none;
-  width: 204px;
-  margin-left: 0;
-}
-
-table .span4 {
-  float: none;
-  width: 284px;
-  margin-left: 0;
-}
-
-table .span5 {
-  float: none;
-  width: 364px;
-  margin-left: 0;
-}
-
-table .span6 {
-  float: none;
-  width: 444px;
-  margin-left: 0;
-}
-
-table .span7 {
-  float: none;
-  width: 524px;
-  margin-left: 0;
-}
-
-table .span8 {
-  float: none;
-  width: 604px;
-  margin-left: 0;
-}
-
-table .span9 {
-  float: none;
-  width: 684px;
-  margin-left: 0;
-}
-
-table .span10 {
-  float: none;
-  width: 764px;
-  margin-left: 0;
-}
-
-table .span11 {
-  float: none;
-  width: 844px;
-  margin-left: 0;
-}
-
-table .span12 {
-  float: none;
-  width: 924px;
-  margin-left: 0;
-}
-
-table .span13 {
-  float: none;
-  width: 1004px;
-  margin-left: 0;
-}
-
-table .span14 {
-  float: none;
-  width: 1084px;
-  margin-left: 0;
-}
-
-table .span15 {
-  float: none;
-  width: 1164px;
-  margin-left: 0;
-}
-
-table .span16 {
-  float: none;
-  width: 1244px;
-  margin-left: 0;
-}
-
-table .span17 {
-  float: none;
-  width: 1324px;
-  margin-left: 0;
-}
-
-table .span18 {
-  float: none;
-  width: 1404px;
-  margin-left: 0;
-}
-
-table .span19 {
-  float: none;
-  width: 1484px;
-  margin-left: 0;
-}
-
-table .span20 {
-  float: none;
-  width: 1564px;
-  margin-left: 0;
-}
-
-table .span21 {
-  float: none;
-  width: 1644px;
-  margin-left: 0;
-}
-
-table .span22 {
-  float: none;
-  width: 1724px;
-  margin-left: 0;
-}
-
-table .span23 {
-  float: none;
-  width: 1804px;
-  margin-left: 0;
-}
-
-table .span24 {
-  float: none;
-  width: 1884px;
-  margin-left: 0;
-}
-
-.table tbody tr.success td {
-  background-color: #dff0d8;
-}
-
-.table tbody tr.error td {
-  background-color: #f2dede;
-}
-
-.table tbody tr.info td {
-  background-color: #d9edf7;
-}
-
-[class^="icon-"],
-[class*=" icon-"] {
-  display: inline-block;
-  width: 14px;
-  height: 14px;
-  margin-top: 1px;
-  *margin-right: .3em;
-  line-height: 14px;
-  vertical-align: text-top;
-  background-image: url("../img/glyphicons-halflings.png");
-  background-position: 14px 14px;
-  background-repeat: no-repeat;
-}
-
-/* White icons with optional class, or on hover/active states of certain elements */
-
-.icon-white,
-.nav > .active > a > [class^="icon-"],
-.nav > .active > a > [class*=" icon-"],
-.dropdown-menu > li > a:hover > [class^="icon-"],
-.dropdown-menu > li > a:hover > [class*=" icon-"],
-.dropdown-menu > .active > a > [class^="icon-"],
-.dropdown-menu > .active > a > [class*=" icon-"] {
-  background-image: url("../img/glyphicons-halflings-white.png");
-}
-
-.icon-glass {
-  background-position: 0      0;
-}
-
-.icon-music {
-  background-position: -24px 0;
-}
-
-.icon-search {
-  background-position: -48px 0;
-}
-
-.icon-envelope {
-  background-position: -72px 0;
-}
-
-.icon-heart {
-  background-position: -96px 0;
-}
-
-.icon-star {
-  background-position: -120px 0;
-}
-
-.icon-star-empty {
-  background-position: -144px 0;
-}
-
-.icon-user {
-  background-position: -168px 0;
-}
-
-.icon-film {
-  background-position: -192px 0;
-}
-
-.icon-th-large {
-  background-position: -216px 0;
-}
-
-.icon-th {
-  background-position: -240px 0;
-}
-
-.icon-th-list {
-  background-position: -264px 0;
-}
-
-.icon-ok {
-  background-position: -288px 0;
-}
-
-.icon-remove {
-  background-position: -312px 0;
-}
-
-.icon-zoom-in {
-  background-position: -336px 0;
-}
-
-.icon-zoom-out {
-  background-position: -360px 0;
-}
-
-.icon-off {
-  background-position: -384px 0;
-}
-
-.icon-signal {
-  background-position: -408px 0;
-}
-
-.icon-cog {
-  background-position: -432px 0;
-}
-
-.icon-trash {
-  background-position: -456px 0;
-}
-
-.icon-home {
-  background-position: 0 -24px;
-}
-
-.icon-file {
-  background-position: -24px -24px;
-}
-
-.icon-time {
-  background-position: -48px -24px;
-}
-
-.icon-road {
-  background-position: -72px -24px;
-}
-
-.icon-download-alt {
-  background-position: -96px -24px;
-}
-
-.icon-download {
-  background-position: -120px -24px;
-}
-
-.icon-upload {
-  background-position: -144px -24px;
-}
-
-.icon-inbox {
-  background-position: -168px -24px;
-}
-
-.icon-play-circle {
-  background-position: -192px -24px;
-}
-
-.icon-repeat {
-  background-position: -216px -24px;
-}
-
-.icon-refresh {
-  background-position: -240px -24px;
-}
-
-.icon-list-alt {
-  background-position: -264px -24px;
-}
-
-.icon-lock {
-  background-position: -287px -24px;
-}
-
-.icon-flag {
-  background-position: -312px -24px;
-}
-
-.icon-headphones {
-  background-position: -336px -24px;
-}
-
-.icon-volume-off {
-  background-position: -360px -24px;
-}
-
-.icon-volume-down {
-  background-position: -384px -24px;
-}
-
-.icon-volume-up {
-  background-position: -408px -24px;
-}
-
-.icon-qrcode {
-  background-position: -432px -24px;
-}
-
-.icon-barcode {
-  background-position: -456px -24px;
-}
-
-.icon-tag {
-  background-position: 0 -48px;
-}
-
-.icon-tags {
-  background-position: -25px -48px;
-}
-
-.icon-book {
-  background-position: -48px -48px;
-}
-
-.icon-bookmark {
-  background-position: -72px -48px;
-}
-
-.icon-print {
-  background-position: -96px -48px;
-}
-
-.icon-camera {
-  background-position: -120px -48px;
-}
-
-.icon-font {
-  background-position: -144px -48px;
-}
-
-.icon-bold {
-  background-position: -167px -48px;
-}
-
-.icon-italic {
-  background-position: -192px -48px;
-}
-
-.icon-text-height {
-  background-position: -216px -48px;
-}
-
-.icon-text-width {
-  background-position: -240px -48px;
-}
-
-.icon-align-left {
-  background-position: -264px -48px;
-}
-
-.icon-align-center {
-  background-position: -288px -48px;
-}
-
-.icon-align-right {
-  background-position: -312px -48px;
-}
-
-.icon-align-justify {
-  background-position: -336px -48px;
-}
-
-.icon-list {
-  background-position: -360px -48px;
-}
-
-.icon-indent-left {
-  background-position: -384px -48px;
-}
-
-.icon-indent-right {
-  background-position: -408px -48px;
-}
-
-.icon-facetime-video {
-  background-position: -432px -48px;
-}
-
-.icon-picture {
-  background-position: -456px -48px;
-}
-
-.icon-pencil {
-  background-position: 0 -72px;
-}
-
-.icon-map-marker {
-  background-position: -24px -72px;
-}
-
-.icon-adjust {
-  background-position: -48px -72px;
-}
-
-.icon-tint {
-  background-position: -72px -72px;
-}
-
-.icon-edit {
-  background-position: -96px -72px;
-}
-
-.icon-share {
-  background-position: -120px -72px;
-}
-
-.icon-check {
-  background-position: -144px -72px;
-}
-
-.icon-move {
-  background-position: -168px -72px;
-}
-
-.icon-step-backward {
-  background-position: -192px -72px;
-}
-
-.icon-fast-backward {
-  background-position: -216px -72px;
-}
-
-.icon-backward {
-  background-position: -240px -72px;
-}
-
-.icon-play {
-  background-position: -264px -72px;
-}
-
-.icon-pause {
-  background-position: -288px -72px;
-}
-
-.icon-stop {
-  background-position: -312px -72px;
-}
-
-.icon-forward {
-  background-position: -336px -72px;
-}
-
-.icon-fast-forward {
-  background-position: -360px -72px;
-}
-
-.icon-step-forward {
-  background-position: -384px -72px;
-}
-
-.icon-eject {
-  background-position: -408px -72px;
-}
-
-.icon-chevron-left {
-  background-position: -432px -72px;
-}
-
-.icon-chevron-right {
-  background-position: -456px -72px;
-}
-
-.icon-plus-sign {
-  background-position: 0 -96px;
-}
-
-.icon-minus-sign {
-  background-position: -24px -96px;
-}
-
-.icon-remove-sign {
-  background-position: -48px -96px;
-}
-
-.icon-ok-sign {
-  background-position: -72px -96px;
-}
-
-.icon-question-sign {
-  background-position: -96px -96px;
-}
-
-.icon-info-sign {
-  background-position: -120px -96px;
-}
-
-.icon-screenshot {
-  background-position: -144px -96px;
-}
-
-.icon-remove-circle {
-  background-position: -168px -96px;
-}
-
-.icon-ok-circle {
-  background-position: -192px -96px;
-}
-
-.icon-ban-circle {
-  background-position: -216px -96px;
-}
-
-.icon-arrow-left {
-  background-position: -240px -96px;
-}
-
-.icon-arrow-right {
-  background-position: -264px -96px;
-}
-
-.icon-arrow-up {
-  background-position: -289px -96px;
-}
-
-.icon-arrow-down {
-  background-position: -312px -96px;
-}
-
-.icon-share-alt {
-  background-position: -336px -96px;
-}
-
-.icon-resize-full {
-  background-position: -360px -96px;
-}
-
-.icon-resize-small {
-  background-position: -384px -96px;
-}
-
-.icon-plus {
-  background-position: -408px -96px;
-}
-
-.icon-minus {
-  background-position: -433px -96px;
-}
-
-.icon-asterisk {
-  background-position: -456px -96px;
-}
-
-.icon-exclamation-sign {
-  background-position: 0 -120px;
-}
-
-.icon-gift {
-  background-position: -24px -120px;
-}
-
-.icon-leaf {
-  background-position: -48px -120px;
-}
-
-.icon-fire {
-  background-position: -72px -120px;
-}
-
-.icon-eye-open {
-  background-position: -96px -120px;
-}
-
-.icon-eye-close {
-  background-position: -120px -120px;
-}
-
-.icon-warning-sign {
-  background-position: -144px -120px;
-}
-
-.icon-plane {
-  background-position: -168px -120px;
-}
-
-.icon-calendar {
-  background-position: -192px -120px;
-}
-
-.icon-random {
-  width: 16px;
-  background-position: -216px -120px;
-}
-
-.icon-comment {
-  background-position: -240px -120px;
-}
-
-.icon-magnet {
-  background-position: -264px -120px;
-}
-
-.icon-chevron-up {
-  background-position: -288px -120px;
-}
-
-.icon-chevron-down {
-  background-position: -313px -119px;
-}
-
-.icon-retweet {
-  background-position: -336px -120px;
-}
-
-.icon-shopping-cart {
-  background-position: -360px -120px;
-}
-
-.icon-folder-close {
-  background-position: -384px -120px;
-}
-
-.icon-folder-open {
-  width: 16px;
-  background-position: -408px -120px;
-}
-
-.icon-resize-vertical {
-  background-position: -432px -119px;
-}
-
-.icon-resize-horizontal {
-  background-position: -456px -118px;
-}
-
-.icon-hdd {
-  background-position: 0 -144px;
-}
-
-.icon-bullhorn {
-  background-position: -24px -144px;
-}
-
-.icon-bell {
-  background-position: -48px -144px;
-}
-
-.icon-certificate {
-  background-position: -72px -144px;
-}
-
-.icon-thumbs-up {
-  background-position: -96px -144px;
-}
-
-.icon-thumbs-down {
-  background-position: -120px -144px;
-}
-
-.icon-hand-right {
-  background-position: -144px -144px;
-}
-
-.icon-hand-left {
-  background-position: -168px -144px;
-}
-
-.icon-hand-up {
-  background-position: -192px -144px;
-}
-
-.icon-hand-down {
-  background-position: -216px -144px;
-}
-
-.icon-circle-arrow-right {
-  background-position: -240px -144px;
-}
-
-.icon-circle-arrow-left {
-  background-position: -264px -144px;
-}
-
-.icon-circle-arrow-up {
-  background-position: -288px -144px;
-}
-
-.icon-circle-arrow-down {
-  background-position: -312px -144px;
-}
-
-.icon-globe {
-  background-position: -336px -144px;
-}
-
-.icon-wrench {
-  background-position: -360px -144px;
-}
-
-.icon-tasks {
-  background-position: -384px -144px;
-}
-
-.icon-filter {
-  background-position: -408px -144px;
-}
-
-.icon-briefcase {
-  background-position: -432px -144px;
-}
-
-.icon-fullscreen {
-  background-position: -456px -144px;
-}
-
-.dropup,
-.dropdown {
-  position: relative;
-}
-
-.dropdown-toggle {
-  *margin-bottom: -3px;
-}
-
-.dropdown-toggle:active,
-.open .dropdown-toggle {
-  outline: 0;
-}
-
-.caret {
-  display: inline-block;
-  width: 0;
-  height: 0;
-  vertical-align: top;
-  border-top: 4px solid #000000;
-  border-right: 4px solid transparent;
-  border-left: 4px solid transparent;
-  content: "";
-}
-
-.dropdown .caret {
-  margin-top: 8px;
-  margin-left: 2px;
-}
-
-.dropdown-menu {
-  position: absolute;
-  top: 100%;
-  left: 0;
-  z-index: 1000;
-  display: none;
-  float: left;
-  min-width: 160px;
-  padding: 5px 0;
-  margin: 2px 0 0;
-  list-style: none;
-  background-color: #ffffff;
-  border: 1px solid #ccc;
-  border: 1px solid rgba(0, 0, 0, 0.2);
-  *border-right-width: 2px;
-  *border-bottom-width: 2px;
-  -webkit-border-radius: 6px;
-     -moz-border-radius: 6px;
-          border-radius: 6px;
-  -webkit-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
-     -moz-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
-          box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
-  -webkit-background-clip: padding-box;
-     -moz-background-clip: padding;
-          background-clip: padding-box;
-}
-
-.dropdown-menu.pull-right {
-  right: 0;
-  left: auto;
-}
-
-.dropdown-menu .divider {
-  *width: 100%;
-  height: 1px;
-  margin: 9px 1px;
-  *margin: -5px 0 5px;
-  overflow: hidden;
-  background-color: #e5e5e5;
-  border-bottom: 1px solid #ffffff;
-}
-
-.dropdown-menu a {
-  display: block;
-  padding: 3px 20px;
-  clear: both;
-  font-weight: normal;
-  line-height: 20px;
-  color: #333333;
-  white-space: nowrap;
-}
-
-.dropdown-menu li > a:hover,
-.dropdown-menu li > a:focus,
-.dropdown-submenu:hover > a {
-  color: #ffffff;
-  text-decoration: none;
-  background-color: #0088cc;
-  background-color: #0081c2;
-  background-image: -moz-linear-gradient(top, #0088cc, #0077b3);
-  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#0088cc), to(#0077b3));
-  background-image: -webkit-linear-gradient(top, #0088cc, #0077b3);
-  background-image: -o-linear-gradient(top, #0088cc, #0077b3);
-  background-image: linear-gradient(to bottom, #0088cc, #0077b3);
-  background-repeat: repeat-x;
-  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff0088cc', endColorstr='#ff0077b3', GradientType=0);
-}
-
-.dropdown-menu .active > a,
-.dropdown-menu .active > a:hover {
-  color: #ffffff;
-  text-decoration: none;
-  background-color: #0098cc;
-  background-color: #0098cc;
-  background-image: linear-gradient(to bottom, #0098cc, #0087b3);
-  background-image: -moz-linear-gradient(top, #0098cc, #0087b3);
-  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#0098cc), to(#0087b3));
-  background-image: -webkit-linear-gradient(top, #0098cc, #0087b3);
-  background-image: -o-linear-gradient(top, #0098cc, #0087b3);
-  background-repeat: repeat-x;
-  outline: 0;
-  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff0098cc', endColorstr='#ff0087b3', GradientType=0);
-}
-
-.dropdown-menu .disabled > a,
-.dropdown-menu .disabled > a:hover {
-  color: #999999;
-}
-
-.dropdown-menu .disabled > a:hover {
-  text-decoration: none;
-  cursor: default;
-  background-color: transparent;
-}
-
-.open {
-  *z-index: 1000;
-}
-
-.open > .dropdown-menu {
-  display: block;
-}
-
-.pull-right > .dropdown-menu {
-  right: 0;
-  left: auto;
-}
-
-.dropup .caret,
-.navbar-fixed-bottom .dropdown .caret {
-  border-top: 0;
-  border-bottom: 4px solid #000000;
-  content: "\2191";
-}
-
-.dropup .dropdown-menu,
-.navbar-fixed-bottom .dropdown .dropdown-menu {
-  top: auto;
-  bottom: 100%;
-  margin-bottom: 1px;
-}
-
-.dropdown-submenu {
-  position: relative;
-}
-
-.dropdown-submenu > .dropdown-menu {
-  top: 0;
-  left: 100%;
-  margin-top: -6px;
-  margin-left: -1px;
-  -webkit-border-radius: 0 6px 6px 6px;
-     -moz-border-radius: 0 6px 6px 6px;
-          border-radius: 0 6px 6px 6px;
-}
-
-.dropdown-submenu:hover .dropdown-menu {
-  display: block;
-}
-
-.dropdown-submenu > a:after {
-  display: block;
-  float: right;
-  width: 0;
-  height: 0;
-  margin-top: 5px;
-  margin-right: -10px;
-  border-color: transparent;
-  border-left-color: #cccccc;
-  border-style: solid;
-  border-width: 5px 0 5px 5px;
-  content: " ";
-}
-
-.dropdown-submenu:hover > a:after {
-  border-left-color: #ffffff;
-}
-
-.dropdown .dropdown-menu .nav-header {
-  padding-right: 20px;
-  padding-left: 20px;
-}
-
-.typeahead {
-  margin-top: 2px;
-  -webkit-border-radius: 4px;
-     -moz-border-radius: 4px;
-          border-radius: 4px;
-}
-
-.well {
-  min-height: 20px;
-  padding: 19px;
-  margin-bottom: 20px;
-  background-color: #f5f5f5;
-  border: 1px solid #e3e3e3;
-  -webkit-border-radius: 4px;
-     -moz-border-radius: 4px;
-          border-radius: 4px;
-  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05);
-     -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05);
-          box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05);
-}
-
-.well blockquote {
-  border-color: #ddd;
-  border-color: rgba(0, 0, 0, 0.15);
-}
-
-.well-large {
-  padding: 24px;
-  -webkit-border-radius: 6px;
-     -moz-border-radius: 6px;
-          border-radius: 6px;
-}
-
-.well-small {
-  padding: 9px;
-  -webkit-border-radius: 3px;
-     -moz-border-radius: 3px;
-          border-radius: 3px;
-}
-
-.fade {
-  opacity: 0;
-  -webkit-transition: opacity 0.15s linear;
-     -moz-transition: opacity 0.15s linear;
-       -o-transition: opacity 0.15s linear;
-          transition: opacity 0.15s linear;
-}
-
-.fade.in {
-  opacity: 1;
-}
-
-.collapse {
-  position: relative;
-  height: 0;
-  overflow: hidden;
-  overflow: visible \9;
-  -webkit-transition: height 0.35s ease;
-     -moz-transition: height 0.35s ease;
-       -o-transition: height 0.35s ease;
-          transition: height 0.35s ease;
-}
-
-.collapse.in {
-  height: auto;
-}
-
-.close {
-  float: right;
-  font-size: 20px;
-  font-weight: bold;
-  line-height: 20px;
-  color: #000000;
-  text-shadow: 0 1px 0 #ffffff;
-  opacity: 0.2;
-  filter: alpha(opacity=20);
-}
-
-.close:hover {
-  color: #000000;
-  text-decoration: none;
-  cursor: pointer;
-  opacity: 0.4;
-  filter: alpha(opacity=40);
-}
-
-button.close {
-  padding: 0;
-  cursor: pointer;
-  background: transparent;
-  border: 0;
-  -webkit-appearance: none;
-}
-
-.btn {
-  display: inline-block;
-  *display: inline;
-  padding: 4px 14px;
-  margin-bottom: 0;
-  *margin-left: .3em;
-  font-size: 14px;
-  line-height: 20px;
-  *line-height: 20px;
-  color: #333333;
-  text-align: center;
-  text-shadow: 0 1px 1px rgba(255, 255, 255, 0.75);
-  vertical-align: middle;
-  cursor: pointer;
-  background-color: #f5f5f5;
-  *background-color: #e6e6e6;
-  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ffffff), to(#e6e6e6));
-  background-image: -webkit-linear-gradient(top, #ffffff, #e6e6e6);
-  background-image: -o-linear-gradient(top, #ffffff, #e6e6e6);
-  background-image: linear-gradient(to bottom, #ffffff, #e6e6e6);
-  background-image: -moz-linear-gradient(top, #ffffff, #e6e6e6);
-  background-repeat: repeat-x;
-  border: 1px solid #bbbbbb;
-  *border: 0;
-  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
-  border-color: #e6e6e6 #e6e6e6 #bfbfbf;
-  border-bottom-color: #a2a2a2;
-  -webkit-border-radius: 4px;
-     -moz-border-radius: 4px;
-          border-radius: 4px;
-  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ffffffff', endColorstr='#ffe6e6e6', GradientType=0);
-  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
-  *zoom: 1;
-  -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
-     -moz-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
-          box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
-}
-
-.btn:hover,
-.btn:active,
-.btn.active,
-.btn.disabled,
-.btn[disabled] {
-  color: #333333;
-  background-color: #e6e6e6;
-  *background-color: #d9d9d9;
-}
-
-.btn:active,
-.btn.active {
-  background-color: #cccccc \9;
-}
-
-.btn:first-child {
-  *margin-left: 0;
-}
-
-.btn:hover {
-  color: #333333;
-  text-decoration: none;
-  background-color: #e6e6e6;
-  *background-color: #d9d9d9;
-  /* Buttons in IE7 don't get borders, so darken on hover */
-
-  background-position: 0 -15px;
-  -webkit-transition: background-position 0.1s linear;
-     -moz-transition: background-position 0.1s linear;
-       -o-transition: background-position 0.1s linear;
-          transition: background-position 0.1s linear;
-}
-
-.btn:focus {
-  outline: thin dotted #333;
-  outline: 5px auto -webkit-focus-ring-color;
-  outline-offset: -2px;
-}
-
-.btn.active,
-.btn:active {
-  background-color: #e6e6e6;
-  background-color: #d9d9d9 \9;
-  background-image: none;
-  outline: 0;
-  -webkit-box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
-     -moz-box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
-          box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
-}
-
-.btn.disabled,
-.btn[disabled] {
-  cursor: default;
-  background-color: #e6e6e6;
-  background-image: none;
-  opacity: 0.65;
-  filter: alpha(opacity=65);
-  -webkit-box-shadow: none;
-     -moz-box-shadow: none;
-          box-shadow: none;
-}
-
-.btn-large {
-  padding: 9px 14px;
-  font-size: 16px;
-  line-height: normal;
-  -webkit-border-radius: 5px;
-     -moz-border-radius: 5px;
-          border-radius: 5px;
-}
-
-.btn-large [class^="icon-"] {
-  margin-top: 2px;
-}
-
-.btn-small {
-  padding: 3px 9px;
-  font-size: 12px;
-  line-height: 18px;
-}
-
-.btn-small [class^="icon-"] {
-  margin-top: 0;
-}
-
-.btn-mini {
-  padding: 2px 6px;
-  font-size: 11px;
-  line-height: 16px;
-}
-
-.btn-block {
-  display: block;
-  width: 100%;
-  padding-right: 0;
-  padding-left: 0;
-  -webkit-box-sizing: border-box;
-     -moz-box-sizing: border-box;
-          box-sizing: border-box;
-}
-
-.btn-block + .btn-block {
-  margin-top: 5px;
-}
-
-.btn-primary.active,
-.btn-warning.active,
-.btn-danger.active,
-.btn-success.active,
-.btn-info.active,
-.btn-inverse.active {
-  color: rgba(255, 255, 255, 0.75);
-}
-
-.btn {
-  border-color: #c5c5c5;
-  border-color: rgba(0, 0, 0, 0.15) rgba(0, 0, 0, 0.15) rgba(0, 0, 0, 0.25);
-}
-
-.btn-primary {
-  color: #ffffff;
-  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
-  background-color: #006dcc;
-  *background-color: #0044cc;
-  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#0088cc), to(#0044cc));
-  background-image: -webkit-linear-gradient(top, #0088cc, #0044cc);
-  background-image: -o-linear-gradient(top, #0088cc, #0044cc);
-  background-image: linear-gradient(to bottom, #0088cc, #0044cc);
-  background-image: -moz-linear-gradient(top, #0088cc, #0044cc);
-  background-repeat: repeat-x;
-  border-color: #0044cc #0044cc #002a80;
-  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
-  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff0088cc', endColorstr='#ff0044cc', GradientType=0);
-  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
-}
-
-.btn-primary:hover,
-.btn-primary:active,
-.btn-primary.active,
-.btn-primary.disabled,
-.btn-primary[disabled] {
-  color: #ffffff;
-  background-color: #0044cc;
-  *background-color: #003bb3;
-}
-
-.btn-primary:active,
-.btn-primary.active {
-  background-color: #003399 \9;
-}
-
-.btn-warning {
-  color: #ffffff;
-  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
-  background-color: #faa732;
-  *background-color: #f89406;
-  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#fbb450), to(#f89406));
-  background-image: -webkit-linear-gradient(top, #fbb450, #f89406);
-  background-image: -o-linear-gradient(top, #fbb450, #f89406);
-  background-image: linear-gradient(to bottom, #fbb450, #f89406);
-  background-image: -moz-linear-gradient(top, #fbb450, #f89406);
-  background-repeat: repeat-x;
-  border-color: #f89406 #f89406 #ad6704;
-  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
-  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#fffbb450', endColorstr='#fff89406', GradientType=0);
-  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
-}
-
-.btn-warning:hover,
-.btn-warning:active,
-.btn-warning.active,
-.btn-warning.disabled,
-.btn-warning[disabled] {
-  color: #ffffff;
-  background-color: #f89406;
-  *background-color: #df8505;
-}
-
-.btn-warning:active,
-.btn-warning.active {
-  background-color: #c67605 \9;
-}
-
-.btn-danger {
-  color: #ffffff;
-  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
-  background-color: #da4f49;
-  *background-color: #bd362f;
-  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ee5f5b), to(#bd362f));
-  background-image: -webkit-linear-gradient(top, #ee5f5b, #bd362f);
-  background-image: -o-linear-gradient(top, #ee5f5b, #bd362f);
-  background-image: linear-gradient(to bottom, #ee5f5b, #bd362f);
-  background-image: -moz-linear-gradient(top, #ee5f5b, #bd362f);
-  background-repeat: repeat-x;
-  border-color: #bd362f #bd362f #802420;
-  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
-  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ffee5f5b', endColorstr='#ffbd362f', GradientType=0);
-  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
-}
-
-.btn-danger:hover,
-.btn-danger:active,
-.btn-danger.active,
-.btn-danger.disabled,
-.btn-danger[disabled] {
-  color: #ffffff;
-  background-color: #bd362f;
-  *background-color: #a9302a;
-}
-
-.btn-danger:active,
-.btn-danger.active {
-  background-color: #942a25 \9;
-}
-
-.btn-success {
-  color: #ffffff;
-  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
-  background-color: #5bb75b;
-  *background-color: #51a351;
-  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#62c462), to(#51a351));
-  background-image: -webkit-linear-gradient(top, #62c462, #51a351);
-  background-image: -o-linear-gradient(top, #62c462, #51a351);
-  background-image: linear-gradient(to bottom, #62c462, #51a351);
-  background-image: -moz-linear-gradient(top, #62c462, #51a351);
-  background-repeat: repeat-x;
-  border-color: #51a351 #51a351 #387038;
-  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
-  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff62c462', endColorstr='#ff51a351', GradientType=0);
-  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
-}
-
-.btn-success:hover,
-.btn-success:active,
-.btn-success.active,
-.btn-success.disabled,
-.btn-success[disabled] {
-  color: #ffffff;
-  background-color: #51a351;
-  *background-color: #499249;
-}
-
-.btn-success:active,
-.btn-success.active {
-  background-color: #408140 \9;
-}
-
-.btn-info {
-  color: #ffffff;
-  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
-  background-color: #49afcd;
-  *background-color: #2f96b4;
-  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#5bc0de), to(#2f96b4));
-  background-image: -webkit-linear-gradient(top, #5bc0de, #2f96b4);
-  background-image: -o-linear-gradient(top, #5bc0de, #2f96b4);
-  background-image: linear-gradient(to bottom, #5bc0de, #2f96b4);
-  background-image: -moz-linear-gradient(top, #5bc0de, #2f96b4);
-  background-repeat: repeat-x;
-  border-color: #2f96b4 #2f96b4 #1f6377;
-  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
-  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff5bc0de', endColorstr='#ff2f96b4', GradientType=0);
-  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
-}
-
-.btn-info:hover,
-.btn-info:active,
-.btn-info.active,
-.btn-info.disabled,
-.btn-info[disabled] {
-  color: #ffffff;
-  background-color: #2f96b4;
-  *background-color: #2a85a0;
-}
-
-.btn-info:active,
-.btn-info.active {
-  background-color: #24748c \9;
-}
-
-.btn-inverse {
-  color: #ffffff;
-  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
-  background-color: #363636;
-  *background-color: #222222;
-  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#444444), to(#222222));
-  background-image: -webkit-linear-gradient(top, #444444, #222222);
-  background-image: -o-linear-gradient(top, #444444, #222222);
-  background-image: linear-gradient(to bottom, #444444, #222222);
-  background-image: -moz-linear-gradient(top, #444444, #222222);
-  background-repeat: repeat-x;
-  border-color: #222222 #222222 #000000;
-  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
-  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff444444', endColorstr='#ff222222', GradientType=0);
-  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
-}
-
-.btn-inverse:hover,
-.btn-inverse:active,
-.btn-inverse.active,
-.btn-inverse.disabled,
-.btn-inverse[disabled] {
-  color: #ffffff;
-  background-color: #222222;
-  *background-color: #151515;
-}
-
-.btn-inverse:active,
-.btn-inverse.active {
-  background-color: #080808 \9;
-}
-
-button.btn,
-input[type="submit"].btn {
-  *padding-top: 3px;
-  *padding-bottom: 3px;
-}
-
-button.btn::-moz-focus-inner,
-input[type="submit"].btn::-moz-focus-inner {
-  padding: 0;
-  border: 0;
-}
-
-button.btn.btn-large,
-input[type="submit"].btn.btn-large {
-  *padding-top: 7px;
-  *padding-bottom: 7px;
-}
-
-button.btn.btn-small,
-input[type="submit"].btn.btn-small {
-  *padding-top: 3px;
-  *padding-bottom: 3px;
-}
-
-button.btn.btn-mini,
-input[type="submit"].btn.btn-mini {
-  *padding-top: 1px;
-  *padding-bottom: 1px;
-}
-
-.btn-link,
-.btn-link:active {
-  background-color: transparent;
-  background-image: none;
-  -webkit-box-shadow: none;
-     -moz-box-shadow: none;
-          box-shadow: none;
-}
-
-.btn-link {
-  color: #0088cc;
-  cursor: pointer;
-  border-color: transparent;
-  -webkit-border-radius: 0;
-     -moz-border-radius: 0;
-          border-radius: 0;
-}
-
-.btn-link:hover {
-  color: #005580;
-  text-decoration: underline;
-  background-color: transparent;
-}
-
-.btn-group {
-  position: relative;
-  *margin-left: .3em;
-  font-size: 0;
-  white-space: nowrap;
-}
-
-.btn-group:first-child {
-  *margin-left: 0;
-}
-
-.btn-group + .btn-group {
-  margin-left: 5px;
-}
-
-.btn-toolbar {
-  margin-top: 10px;
-  margin-bottom: 10px;
-  font-size: 0;
-}
-
-.btn-toolbar .btn-group {
-  display: inline-block;
-  *display: inline;
-  /* IE7 inline-block hack */
-
-  *zoom: 1;
-}
-
-.btn-toolbar .btn + .btn,
-.btn-toolbar .btn-group + .btn,
-.btn-toolbar .btn + .btn-group {
-  margin-left: 5px;
-}
-
-.btn-group > .btn {
-  position: relative;
-  -webkit-border-radius: 0;
-     -moz-border-radius: 0;
-          border-radius: 0;
-}
-
-.btn-group > .btn + .btn {
-  margin-left: -1px;
-}
-
-.btn-group > .btn,
-.btn-group > .dropdown-menu {
-  font-size: 14px;
-}
-
-.btn-group > .btn-mini {
-  font-size: 11px;
-}
-
-.btn-group > .btn-small {
-  font-size: 12px;
-}
-
-.btn-group > .btn-large {
-  font-size: 16px;
-}
-
-.btn-group > .btn:first-child {
-  margin-left: 0;
-  -webkit-border-bottom-left-radius: 4px;
-          border-bottom-left-radius: 4px;
-  -webkit-border-top-left-radius: 4px;
-          border-top-left-radius: 4px;
-  -moz-border-radius-bottomleft: 4px;
-  -moz-border-radius-topleft: 4px;
-}
-
-.btn-group > .btn:last-child,
-.btn-group > .dropdown-toggle {
-  -webkit-border-top-right-radius: 4px;
-          border-top-right-radius: 4px;
-  -webkit-border-bottom-right-radius: 4px;
-          border-bottom-right-radius: 4px;
-  -moz-border-radius-topright: 4px;
-  -moz-border-radius-bottomright: 4px;
-}
-
-.btn-group > .btn.large:first-child {
-  margin-left: 0;
-  -webkit-border-bottom-left-radius: 6px;
-          border-bottom-left-radius: 6px;
-  -webkit-border-top-left-radius: 6px;
-          border-top-left-radius: 6px;
-  -moz-border-radius-bottomleft: 6px;
-  -moz-border-radius-topleft: 6px;
-}
-
-.btn-group > .btn.large:last-child,
-.btn-group > .large.dropdown-toggle {
-  -webkit-border-top-right-radius: 6px;
-          border-top-right-radius: 6px;
-  -webkit-border-bottom-right-radius: 6px;
-          border-bottom-right-radius: 6px;
-  -moz-border-radius-topright: 6px;
-  -moz-border-radius-bottomright: 6px;
-}
-
-.btn-group > .btn:hover,
-.btn-group > .btn:focus,
-.btn-group > .btn:active,
-.btn-group > .btn.active {
-  z-index: 2;
-}
-
-.btn-group .dropdown-toggle:active,
-.btn-group.open .dropdown-toggle {
-  outline: 0;
-}
-
-.btn-group > .btn + .dropdown-toggle {
-  *padding-top: 5px;
-  padding-right: 8px;
-  *padding-bottom: 5px;
-  padding-left: 8px;
-  -webkit-box-shadow: inset 1px 0 0 rgba(255, 255, 255, 0.125), inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
-     -moz-box-shadow: inset 1px 0 0 rgba(255, 255, 255, 0.125), inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
-          box-shadow: inset 1px 0 0 rgba(255, 255, 255, 0.125), inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
-}
-
-.btn-group > .btn-mini + .dropdown-toggle {
-  *padding-top: 2px;
-  padding-right: 5px;
-  *padding-bottom: 2px;
-  padding-left: 5px;
-}
-
-.btn-group > .btn-small + .dropdown-toggle {
-  *padding-top: 5px;
-  *padding-bottom: 4px;
-}
-
-.btn-group > .btn-large + .dropdown-toggle {
-  *padding-top: 7px;
-  padding-right: 12px;
-  *padding-bottom: 7px;
-  padding-left: 12px;
-}
-
-.btn-group.open .dropdown-toggle {
-  background-image: none;
-  -webkit-box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
-     -moz-box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
-          box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
-}
-
-.btn-group.open .btn.dropdown-toggle {
-  background-color: #e6e6e6;
-}
-
-.btn-group.open .btn-primary.dropdown-toggle {
-  background-color: #0044cc;
-}
-
-.btn-group.open .btn-warning.dropdown-toggle {
-  background-color: #f89406;
-}
-
-.btn-group.open .btn-danger.dropdown-toggle {
-  background-color: #bd362f;
-}
-
-.btn-group.open .btn-success.dropdown-toggle {
-  background-color: #51a351;
-}
-
-.btn-group.open .btn-info.dropdown-toggle {
-  background-color: #2f96b4;
-}
-
-.btn-group.open .btn-inverse.dropdown-toggle {
-  background-color: #222222;
-}
-
-.btn .caret {
-  margin-top: 8px;
-  margin-left: 0;
-}
-
-.btn-mini .caret,
-.btn-small .caret,
-.btn-large .caret {
-  margin-top: 6px;
-}
-
-.btn-large .caret {
-  border-top-width: 5px;
-  border-right-width: 5px;
-  border-left-width: 5px;
-}
-
-.dropup .btn-large .caret {
-  border-top: 0;
-  border-bottom: 5px solid #000000;
-}
-
-.btn-primary .caret,
-.btn-warning .caret,
-.btn-danger .caret,
-.btn-info .caret,
-.btn-success .caret,
-.btn-inverse .caret {
-  border-top-color: #ffffff;
-  border-bottom-color: #ffffff;
-}
-
-.btn-group-vertical {
-  display: inline-block;
-  *display: inline;
-  /* IE7 inline-block hack */
-
-  *zoom: 1;
-}
-
-.btn-group-vertical .btn {
-  display: block;
-  float: none;
-  width: 100%;
-  -webkit-border-radius: 0;
-     -moz-border-radius: 0;
-          border-radius: 0;
-}
-
-.btn-group-vertical .btn + .btn {
-  margin-top: -1px;
-  margin-left: 0;
-}
-
-.btn-group-vertical .btn:first-child {
-  -webkit-border-radius: 4px 4px 0 0;
-     -moz-border-radius: 4px 4px 0 0;
-          border-radius: 4px 4px 0 0;
-}
-
-.btn-group-vertical .btn:last-child {
-  -webkit-border-radius: 0 0 4px 4px;
-     -moz-border-radius: 0 0 4px 4px;
-          border-radius: 0 0 4px 4px;
-}
-
-.btn-group-vertical .btn-large:first-child {
-  -webkit-border-radius: 6px 6px 0 0;
-     -moz-border-radius: 6px 6px 0 0;
-          border-radius: 6px 6px 0 0;
-}
-
-.btn-group-vertical .btn-large:last-child {
-  -webkit-border-radius: 0 0 6px 6px;
-     -moz-border-radius: 0 0 6px 6px;
-          border-radius: 0 0 6px 6px;
-}
-
-.alert {
-  padding: 8px 35px 8px 14px;
-  margin-bottom: 20px;
-  color: #c09853;
-  text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5);
-  background-color: #fcf8e3;
-  border: 1px solid #fbeed5;
-  -webkit-border-radius: 4px;
-     -moz-border-radius: 4px;
-          border-radius: 4px;
-}
-
-.alert h4 {
-  margin: 0;
-}
-
-.alert .close {
-  position: relative;
-  top: -2px;
-  right: -21px;
-  line-height: 20px;
-}
-
-.alert-success {
-  color: #468847;
-  background-color: #dff0d8;
-  border-color: #d6e9c6;
-}
-
-.alert-danger,
-.alert-error {
-  color: #b94a48;
-  background-color: #f2dede;
-  border-color: #eed3d7;
-}
-
-.alert-info {
-  color: #3a87ad;
-  background-color: #d9edf7;
-  border-color: #bce8f1;
-}
-
-.alert-block {
-  padding-top: 14px;
-  padding-bottom: 14px;
-}
-
-.alert-block > p,
-.alert-block > ul {
-  margin-bottom: 0;
-}
-
-.alert-block p + p {
-  margin-top: 5px;
-}
-
-.nav {
-  margin-bottom: 20px;
-  margin-left: 0;
-  list-style: none;
-}
-
-.nav > li > a {
-  display: block;
-}
-
-.nav > li > a:hover {
-  text-decoration: none;
-  background-color: #eeeeee;
-}
-
-.nav > .pull-right {
-  float: right;
-}
-
-.nav-header {
-  display: block;
-  padding: 3px 15px;
-  font-size: 11px;
-  font-weight: bold;
-  line-height: 20px;
-  color: #999999;
-  text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5);
-  text-transform: uppercase;
-}
-
-.nav li + .nav-header {
-  margin-top: 9px;
-}
-
-.nav-list {
-  padding-right: 15px;
-  padding-left: 15px;
-  margin-bottom: 0;
-}
-
-.nav-list > li > a,
-.nav-list .nav-header {
-  margin-right: -15px;
-  margin-left: -15px;
-  text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5);
-}
-
-.nav-list > li > a {
-  padding: 3px 15px;
-}
-
-.nav-list > .active > a,
-.nav-list > .active > a:hover {
-  color: #ffffff;
-  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.2);
-  background-color: #0088cc;
-}
-
-.nav-list [class^="icon-"] {
-  margin-right: 2px;
-}
-
-.nav-list .divider {
-  *width: 100%;
-  height: 1px;
-  margin: 9px 1px;
-  *margin: -5px 0 5px;
-  overflow: hidden;
-  background-color: #e5e5e5;
-  border-bottom: 1px solid #ffffff;
-}
-
-.nav-tabs,
-.nav-pills {
-  *zoom: 1;
-}
-
-.nav-tabs:before,
-.nav-pills:before,
-.nav-tabs:after,
-.nav-pills:after {
-  display: table;
-  line-height: 0;
-  content: "";
-}
-
-.nav-tabs:after,
-.nav-pills:after {
-  clear: both;
-}
-
-.nav-tabs > li,
-.nav-pills > li {
-  float: left;
-}
-
-.nav-tabs > li > a,
-.nav-pills > li > a {
-  padding-right: 12px;
-  padding-left: 12px;
-  margin-right: 2px;
-  line-height: 14px;
-}
-
-.nav-tabs {
-  border-bottom: 1px solid #ddd;
-}
-
-.nav-tabs > li {
-  margin-bottom: -1px;
-}
-
-.nav-tabs > li > a {
-  padding-top: 8px;
-  padding-bottom: 8px;
-  line-height: 20px;
-  border: 1px solid transparent;
-  -webkit-border-radius: 4px 4px 0 0;
-     -moz-border-radius: 4px 4px 0 0;
-          border-radius: 4px 4px 0 0;
-}
-
-.nav-tabs > li > a:hover {
-  border-color: #eeeeee #eeeeee #dddddd;
-}
-
-.nav-tabs > .active > a,
-.nav-tabs > .active > a:hover {
-  color: #555555;
-  cursor: default;
-  background-color: #ffffff;
-  border: 1px solid #ddd;
-  border-bottom-color: transparent;
-}
-
-.nav-pills > li > a {
-  padding-top: 8px;
-  padding-bottom: 8px;
-  margin-top: 2px;
-  margin-bottom: 2px;
-  -webkit-border-radius: 5px;
-     -moz-border-radius: 5px;
-          border-radius: 5px;
-}
-
-.nav-pills > .active > a,
-.nav-pills > .active > a:hover {
-  color: #ffffff;
-  background-color: #0088cc;
-}
-
-.nav-stacked > li {
-  float: none;
-}
-
-.nav-stacked > li > a {
-  margin-right: 0;
-}
-
-.nav-tabs.nav-stacked {
-  border-bottom: 0;
-}
-
-.nav-tabs.nav-stacked > li > a {
-  border: 1px solid #ddd;
-  -webkit-border-radius: 0;
-     -moz-border-radius: 0;
-          border-radius: 0;
-}
-
-.nav-tabs.nav-stacked > li:first-child > a {
-  -webkit-border-top-right-radius: 4px;
-          border-top-right-radius: 4px;
-  -webkit-border-top-left-radius: 4px;
-          border-top-left-radius: 4px;
-  -moz-border-radius-topright: 4px;
-  -moz-border-radius-topleft: 4px;
-}
-
-.nav-tabs.nav-stacked > li:last-child > a {
-  -webkit-border-bottom-right-radius: 4px;
-          border-bottom-right-radius: 4px;
-  -webkit-border-bottom-left-radius: 4px;
-          border-bottom-left-radius: 4px;
-  -moz-border-radius-bottomright: 4px;
-  -moz-border-radius-bottomleft: 4px;
-}
-
-.nav-tabs.nav-stacked > li > a:hover {
-  z-index: 2;
-  border-color: #ddd;
-}
-
-.nav-pills.nav-stacked > li > a {
-  margin-bottom: 3px;
-}
-
-.nav-pills.nav-stacked > li:last-child > a {
-  margin-bottom: 1px;
-}
-
-.nav-tabs .dropdown-menu {
-  -webkit-border-radius: 0 0 6px 6px;
-     -moz-border-radius: 0 0 6px 6px;
-          border-radius: 0 0 6px 6px;
-}
-
-.nav-pills .dropdown-menu {
-  -webkit-border-radius: 6px;
-     -moz-border-radius: 6px;
-          border-radius: 6px;
-}
-
-.nav .dropdown-toggle .caret {
-  margin-top: 6px;
-  border-top-color: #0088cc;
-  border-bottom-color: #0088cc;
-}
-
-.nav .dropdown-toggle:hover .caret {
-  border-top-color: #005580;
-  border-bottom-color: #005580;
-}
-
-/* move down carets for tabs */
-
-.nav-tabs .dropdown-toggle .caret {
-  margin-top: 8px;
-}
-
-.nav .active .dropdown-toggle .caret {
-  border-top-color: #fff;
-  border-bottom-color: #fff;
-}
-
-.nav-tabs .active .dropdown-toggle .caret {
-  border-top-color: #555555;
-  border-bottom-color: #555555;
-}
-
-.nav > .dropdown.active > a:hover {
-  cursor: pointer;
-}
-
-.nav-tabs .open .dropdown-toggle,
-.nav-pills .open .dropdown-toggle,
-.nav > li.dropdown.open.active > a:hover {
-  color: #ffffff;
-  background-color: #999999;
-  border-color: #999999;
-}
-
-.nav li.dropdown.open .caret,
-.nav li.dropdown.open.active .caret,
-.nav li.dropdown.open a:hover .caret {
-  border-top-color: #ffffff;
-  border-bottom-color: #ffffff;
-  opacity: 1;
-  filter: alpha(opacity=100);
-}
-
-.tabs-stacked .open > a:hover {
-  border-color: #999999;
-}
-
-.tabbable {
-  *zoom: 1;
-}
-
-.tabbable:before,
-.tabbable:after {
-  display: table;
-  line-height: 0;
-  content: "";
-}
-
-.tabbable:after {
-  clear: both;
-}
-
-.tab-content {
-  overflow: auto;
-}
-
-.tabs-below > .nav-tabs,
-.tabs-right > .nav-tabs,
-.tabs-left > .nav-tabs {
-  border-bottom: 0;
-}
-
-.tab-content > .tab-pane,
-.pill-content > .pill-pane {
-  display: none;
-}
-
-.tab-content > .active,
-.pill-content > .active {
-  display: block;
-}
-
-.tabs-below > .nav-tabs {
-  border-top: 1px solid #ddd;
-}
-
-.tabs-below > .nav-tabs > li {
-  margin-top: -1px;
-  margin-bottom: 0;
-}
-
-.tabs-below > .nav-tabs > li > a {
-  -webkit-border-radius: 0 0 4px 4px;
-     -moz-border-radius: 0 0 4px 4px;
-          border-radius: 0 0 4px 4px;
-}
-
-.tabs-below > .nav-tabs > li > a:hover {
-  border-top-color: #ddd;
-  border-bottom-color: transparent;
-}
-
-.tabs-below > .nav-tabs > .active > a,
-.tabs-below > .nav-tabs > .active > a:hover {
-  border-color: transparent #ddd #ddd #ddd;
-}
-
-.tabs-left > .nav-tabs > li,
-.tabs-right > .nav-tabs > li {
-  float: none;
-}
-
-.tabs-left > .nav-tabs > li > a,
-.tabs-right > .nav-tabs > li > a {
-  min-width: 74px;
-  margin-right: 0;
-  margin-bottom: 3px;
-}
-
-.tabs-left > .nav-tabs {
-  float: left;
-  margin-right: 19px;
-  border-right: 1px solid #ddd;
-}
-
-.tabs-left > .nav-tabs > li > a {
-  margin-right: -1px;
-  -webkit-border-radius: 4px 0 0 4px;
-     -moz-border-radius: 4px 0 0 4px;
-          border-radius: 4px 0 0 4px;
-}
-
-.tabs-left > .nav-tabs > li > a:hover {
-  border-color: #eeeeee #dddddd #eeeeee #eeeeee;
-}
-
-.tabs-left > .nav-tabs .active > a,
-.tabs-left > .nav-tabs .active > a:hover {
-  border-color: #ddd transparent #ddd #ddd;
-  *border-right-color: #ffffff;
-}
-
-.tabs-right > .nav-tabs {
-  float: right;
-  margin-left: 19px;
-  border-left: 1px solid #ddd;
-}
-
-.tabs-right > .nav-tabs > li > a {
-  margin-left: -1px;
-  -webkit-border-radius: 0 4px 4px 0;
-     -moz-border-radius: 0 4px 4px 0;
-          border-radius: 0 4px 4px 0;
-}
-
-.tabs-right > .nav-tabs > li > a:hover {
-  border-color: #eeeeee #eeeeee #eeeeee #dddddd;
-}
-
-.tabs-right > .nav-tabs .active > a,
-.tabs-right > .nav-tabs .active > a:hover {
-  border-color: #ddd #ddd #ddd transparent;
-  *border-left-color: #ffffff;
-}
-
-.nav > .disabled > a {
-  color: #999999;
-}
-
-.nav > .disabled > a:hover {
-  text-decoration: none;
-  cursor: default;
-  background-color: transparent;
-}
-
-.navbar {
-  *position: relative;
-  *z-index: 2;
-  margin-bottom: 20px;
-  overflow: visible;
-  color: #555555;
-}
-
-.navbar-inner {
-  min-height: 40px;
-  padding-right: 20px;
-  padding-left: 20px;
-  background-color: #fadafa;
-  background-image: -moz-linear-gradient(top, #ffddff, #f2d2f2);
-  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ffddff), to(#f2d2f2));
-  background-image: -webkit-linear-gradient(top, #ffddff, #f2d2f2);
-  background-image: -o-linear-gradient(top, #ffddff, #f2d2f2);
-  background-image: linear-gradient(to bottom, #ffddff, #f2d2f2);
-  background-repeat: repeat-x;
-  border: 1px solid #d4d4d4;
-  -webkit-border-radius: 4px;
-     -moz-border-radius: 4px;
-          border-radius: 4px;
-  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ffffddff', endColorstr='#fff2d2f2', GradientType=0);
-  -webkit-box-shadow: 0 1px 4px rgba(0, 0, 0, 0.065);
-     -moz-box-shadow: 0 1px 4px rgba(0, 0, 0, 0.065);
-          box-shadow: 0 1px 4px rgba(0, 0, 0, 0.065);
-}
-
-.navbar .container {
-  width: auto;
-}
-
-.nav-collapse.collapse {
-  height: auto;
-}
-
-.navbar .brand {
-  display: block;
-  float: left;
-  padding: 10px 20px 10px;
-  margin-left: -20px;
-  font-size: 20px;
-  font-weight: 200;
-  color: #555555;
-  text-shadow: 0 1px 0 #ffffff;
-}
-
-.navbar .brand:hover {
-  text-decoration: none;
-}
-
-.navbar-text {
-  margin-bottom: 0;
-  line-height: 40px;
-}
-
-.navbar-link {
-  color: #555555;
-}
-
-.navbar-link:hover {
-  color: #333333;
-}
-
-.navbar .divider-vertical {
-  height: 40px;
-  margin: 0 9px;
-  border-right: 1px solid #ffffff;
-  border-left: 1px solid #f2f2f2;
-}
-
-.navbar .btn,
-.navbar .btn-group {
-  margin-top: 6px;
-}
-
-.navbar .btn-group .btn {
-  margin: 0;
-}
-
-.navbar-form {
-  margin-bottom: 0;
-  *zoom: 1;
-}
-
-.navbar-form:before,
-.navbar-form:after {
-  display: table;
-  line-height: 0;
-  content: "";
-}
-
-.navbar-form:after {
-  clear: both;
-}
-
-.navbar-form input,
-.navbar-form select,
-.navbar-form .radio,
-.navbar-form .checkbox {
-  margin-top: 5px;
-}
-
-.navbar-form input,
-.navbar-form select,
-.navbar-form .btn {
-  display: inline-block;
-  margin-bottom: 0;
-}
-
-.navbar-form input[type="image"],
-.navbar-form input[type="checkbox"],
-.navbar-form input[type="radio"] {
-  margin-top: 3px;
-}
-
-.navbar-form .input-append,
-.navbar-form .input-prepend {
-  margin-top: 6px;
-  white-space: nowrap;
-}
-
-.navbar-form .input-append input,
-.navbar-form .input-prepend input {
-  margin-top: 0;
-}
-
-.navbar-search {
-  position: relative;
-  float: left;
-  margin-top: 5px;
-  margin-bottom: 0;
-}
-
-.navbar-search .search-query {
-  padding: 4px 14px;
-  margin-bottom: 0;
-  font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
-  font-size: 13px;
-  font-weight: normal;
-  line-height: 1;
-  -webkit-border-radius: 15px;
-     -moz-border-radius: 15px;
-          border-radius: 15px;
-}
-
-.navbar-static-top {
-  position: static;
-  width: 100%;
-  margin-bottom: 0;
-}
-
-.navbar-static-top .navbar-inner {
-  -webkit-border-radius: 0;
-     -moz-border-radius: 0;
-          border-radius: 0;
-}
-
-.navbar-fixed-top,
-.navbar-fixed-bottom {
-  position: fixed;
-  right: 0;
-  left: 0;
-  z-index: 1030;
-  margin-bottom: 0;
-}
-
-.navbar-fixed-top .navbar-inner,
-.navbar-fixed-bottom .navbar-inner,
-.navbar-static-top .navbar-inner {
-  border: 0;
-}
-
-.navbar-fixed-top .navbar-inner,
-.navbar-fixed-bottom .navbar-inner {
-  padding-right: 0;
-  padding-left: 0;
-  -webkit-border-radius: 0;
-     -moz-border-radius: 0;
-          border-radius: 0;
-}
-
-.navbar-static-top .container,
-.navbar-fixed-top .container,
-.navbar-fixed-bottom .container {
-  width: 940px;
-}
-
-.navbar-fixed-top {
-  top: 0;
-}
-
-.navbar-fixed-top .navbar-inner,
-.navbar-static-top .navbar-inner {
-  -webkit-box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.1), 0 1px 10px rgba(0, 0, 0, 0.1);
-     -moz-box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.1), 0 1px 10px rgba(0, 0, 0, 0.1);
-          box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.1), 0 1px 10px rgba(0, 0, 0, 0.1);
-}
-
-.navbar-fixed-bottom {
-  bottom: 0;
-}
-
-.navbar-fixed-bottom .navbar-inner {
-  -webkit-box-shadow: inset 0 1px 0 rgba(0, 0, 0, 0.1), 0 -1px 10px rgba(0, 0, 0, 0.1);
-     -moz-box-shadow: inset 0 1px 0 rgba(0, 0, 0, 0.1), 0 -1px 10px rgba(0, 0, 0, 0.1);
-          box-shadow: inset 0 1px 0 rgba(0, 0, 0, 0.1), 0 -1px 10px rgba(0, 0, 0, 0.1);
-}
-
-.navbar .nav {
-  position: relative;
-  left: 0;
-  display: block;
-  float: left;
-  margin: 0 10px 0 0;
-}
-
-.navbar .nav.pull-right {
-  float: right;
-}
-
-.navbar .nav > li {
-  float: left;
-}
-
-.navbar .nav > li > a {
-  float: none;
-  padding: 10px 15px 10px;
-  color: #555555;
-  text-decoration: none;
-  text-shadow: 0 1px 0 #ffffff;
-}
-
-.navbar .nav .dropdown-toggle .caret {
-  margin-top: 8px;
-}
-
-.navbar .nav > li > a:focus,
-.navbar .nav > li > a:hover {
-  color: #333333;
-  text-decoration: none;
-  background-color: transparent;
-}
-
-.navbar .nav > .active > a,
-.navbar .nav > .active > a:hover,
-.navbar .nav > .active > a:focus {
-  color: #555555;
-  text-decoration: none;
-  background-color: #e5e5e5;
-  -webkit-box-shadow: inset 0 3px 8px rgba(0, 0, 0, 0.125);
-     -moz-box-shadow: inset 0 3px 8px rgba(0, 0, 0, 0.125);
-          box-shadow: inset 0 3px 8px rgba(0, 0, 0, 0.125);
-}
-
-.navbar .btn-navbar {
-  display: none;
-  float: right;
-  padding: 7px 10px;
-  margin-right: 5px;
-  margin-left: 5px;
-  color: #ffffff;
-  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
-  background-color: #ededed;
-  *background-color: #e5e5e5;
-  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#f2f2f2), to(#e5e5e5));
-  background-image: -webkit-linear-gradient(top, #f2f2f2, #e5e5e5);
-  background-image: -o-linear-gradient(top, #f2f2f2, #e5e5e5);
-  background-image: linear-gradient(to bottom, #f2f2f2, #e5e5e5);
-  background-image: -moz-linear-gradient(top, #f2f2f2, #e5e5e5);
-  background-repeat: repeat-x;
-  border-color: #e5e5e5 #e5e5e5 #bfbfbf;
-  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
-  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#fff2f2f2', endColorstr='#ffe5e5e5', GradientType=0);
-  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
-  -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.075);
-     -moz-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.075);
-          box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.075);
-}
-
-.navbar .btn-navbar:hover,
-.navbar .btn-navbar:active,
-.navbar .btn-navbar.active,
-.navbar .btn-navbar.disabled,
-.navbar .btn-navbar[disabled] {
-  color: #ffffff;
-  background-color: #e5e5e5;
-  *background-color: #d9d9d9;
-}
-
-.navbar .btn-navbar:active,
-.navbar .btn-navbar.active {
-  background-color: #cccccc \9;
-}
-
-.navbar .btn-navbar .icon-bar {
-  display: block;
-  width: 18px;
-  height: 2px;
-  background-color: #f5f5f5;
-  -webkit-border-radius: 1px;
-     -moz-border-radius: 1px;
-          border-radius: 1px;
-  -webkit-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25);
-     -moz-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25);
-          box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25);
-}
-
-.btn-navbar .icon-bar + .icon-bar {
-  margin-top: 3px;
-}
-
-.navbar .nav > li > .dropdown-menu:before {
-  position: absolute;
-  top: -7px;
-  left: 9px;
-  display: inline-block;
-  border-right: 7px solid transparent;
-  border-bottom: 7px solid #ccc;
-  border-left: 7px solid transparent;
-  border-bottom-color: rgba(0, 0, 0, 0.2);
-  content: '';
-}
-
-.navbar .nav > li > .dropdown-menu:after {
-  position: absolute;
-  top: -6px;
-  left: 10px;
-  display: inline-block;
-  border-right: 6px solid transparent;
-  border-bottom: 6px solid #ffffff;
-  border-left: 6px solid transparent;
-  content: '';
-}
-
-.navbar-fixed-bottom .nav > li > .dropdown-menu:before {
-  top: auto;
-  bottom: -7px;
-  border-top: 7px solid #ccc;
-  border-bottom: 0;
-  border-top-color: rgba(0, 0, 0, 0.2);
-}
-
-.navbar-fixed-bottom .nav > li > .dropdown-menu:after {
-  top: auto;
-  bottom: -6px;
-  border-top: 6px solid #ffffff;
-  border-bottom: 0;
-}
-
-.navbar .nav li.dropdown.open > .dropdown-toggle,
-.navbar .nav li.dropdown.active > .dropdown-toggle,
-.navbar .nav li.dropdown.open.active > .dropdown-toggle {
-  color: #555555;
-  background-color: #e5e5e5;
-}
-
-.navbar .nav li.dropdown > .dropdown-toggle .caret {
-  border-top-color: #555555;
-  border-bottom-color: #555555;
-}
-
-.navbar .nav li.dropdown.open > .dropdown-toggle .caret,
-.navbar .nav li.dropdown.active > .dropdown-toggle .caret,
-.navbar .nav li.dropdown.open.active > .dropdown-toggle .caret {
-  border-top-color: #555555;
-  border-bottom-color: #555555;
-}
-
-.navbar .pull-right > li > .dropdown-menu,
-.navbar .nav > li > .dropdown-menu.pull-right {
-  right: 0;
-  left: auto;
-}
-
-.navbar .pull-right > li > .dropdown-menu:before,
-.navbar .nav > li > .dropdown-menu.pull-right:before {
-  right: 12px;
-  left: auto;
-}
-
-.navbar .pull-right > li > .dropdown-menu:after,
-.navbar .nav > li > .dropdown-menu.pull-right:after {
-  right: 13px;
-  left: auto;
-}
-
-.navbar .pull-right > li > .dropdown-menu .dropdown-menu,
-.navbar .nav > li > .dropdown-menu.pull-right .dropdown-menu {
-  right: 100%;
-  left: auto;
-  margin-right: -1px;
-  margin-left: 0;
-  -webkit-border-radius: 6px 0 6px 6px;
-     -moz-border-radius: 6px 0 6px 6px;
-          border-radius: 6px 0 6px 6px;
-}
-
-.navbar-inverse {
-  color: #999999;
-}
-
-.navbar-inverse .navbar-inner {
-  background-color: #1b1b1b;
-  background-image: -moz-linear-gradient(top, #222222, #111111);
-  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#222222), to(#111111));
-  background-image: -webkit-linear-gradient(top, #222222, #111111);
-  background-image: -o-linear-gradient(top, #222222, #111111);
-  background-image: linear-gradient(to bottom, #222222, #111111);
-  background-repeat: repeat-x;
-  border-color: #252525;
-  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff222222', endColorstr='#ff111111', GradientType=0);
-}
-
-.navbar-inverse .brand,
-.navbar-inverse .nav > li > a {
-  color: #999999;
-  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
-}
-
-.navbar-inverse .brand:hover,
-.navbar-inverse .nav > li > a:hover {
-  color: #ffffff;
-}
-
-.navbar-inverse .nav > li > a:focus,
-.navbar-inverse .nav > li > a:hover {
-  color: #ffffff;
-  background-color: transparent;
-}
-
-.navbar-inverse .nav .active > a,
-.navbar-inverse .nav .active > a:hover,
-.navbar-inverse .nav .active > a:focus {
-  color: #ffffff;
-  background-color: #111111;
-}
-
-.navbar-inverse .navbar-link {
-  color: #999999;
-}
-
-.navbar-inverse .navbar-link:hover {
-  color: #ffffff;
-}
-
-.navbar-inverse .divider-vertical {
-  border-right-color: #222222;
-  border-left-color: #111111;
-}
-
-.navbar-inverse .nav li.dropdown.open > .dropdown-toggle,
-.navbar-inverse .nav li.dropdown.active > .dropdown-toggle,
-.navbar-inverse .nav li.dropdown.open.active > .dropdown-toggle {
-  color: #ffffff;
-  background-color: #111111;
-}
-
-.navbar-inverse .nav li.dropdown > .dropdown-toggle .caret {
-  border-top-color: #999999;
-  border-bottom-color: #999999;
-}
-
-.navbar-inverse .nav li.dropdown.open > .dropdown-toggle .caret,
-.navbar-inverse .nav li.dropdown.active > .dropdown-toggle .caret,
-.navbar-inverse .nav li.dropdown.open.active > .dropdown-toggle .caret {
-  border-top-color: #ffffff;
-  border-bottom-color: #ffffff;
-}
-
-.navbar-inverse .navbar-search .search-query {
-  color: #ffffff;
-  background-color: #515151;
-  border-color: #111111;
-  -webkit-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1), 0 1px 0 rgba(255, 255, 255, 0.15);
-     -moz-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1), 0 1px 0 rgba(255, 255, 255, 0.15);
-          box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1), 0 1px 0 rgba(255, 255, 255, 0.15);
-  -webkit-transition: none;
-     -moz-transition: none;
-       -o-transition: none;
-          transition: none;
-}
-
-.navbar-inverse .navbar-search .search-query:-moz-placeholder {
-  color: #cccccc;
-}
-
-.navbar-inverse .navbar-search .search-query:-ms-input-placeholder {
-  color: #cccccc;
-}
-
-.navbar-inverse .navbar-search .search-query::-webkit-input-placeholder {
-  color: #cccccc;
-}
-
-.navbar-inverse .navbar-search .search-query:focus,
-.navbar-inverse .navbar-search .search-query.focused {
-  padding: 5px 15px;
-  color: #333333;
-  text-shadow: 0 1px 0 #ffffff;
-  background-color: #ffffff;
-  border: 0;
-  outline: 0;
-  -webkit-box-shadow: 0 0 3px rgba(0, 0, 0, 0.15);
-     -moz-box-shadow: 0 0 3px rgba(0, 0, 0, 0.15);
-          box-shadow: 0 0 3px rgba(0, 0, 0, 0.15);
-}
-
-.navbar-inverse .btn-navbar {
-  color: #ffffff;
-  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
-  background-color: #0e0e0e;
-  *background-color: #040404;
-  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#151515), to(#040404));
-  background-image: -webkit-linear-gradient(top, #151515, #040404);
-  background-image: -o-linear-gradient(top, #151515, #040404);
-  background-image: linear-gradient(to bottom, #151515, #040404);
-  background-image: -moz-linear-gradient(top, #151515, #040404);
-  background-repeat: repeat-x;
-  border-color: #040404 #040404 #000000;
-  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
-  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff151515', endColorstr='#ff040404', GradientType=0);
-  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
-}
-
-.navbar-inverse .btn-navbar:hover,
-.navbar-inverse .btn-navbar:active,
-.navbar-inverse .btn-navbar.active,
-.navbar-inverse .btn-navbar.disabled,
-.navbar-inverse .btn-navbar[disabled] {
-  color: #ffffff;
-  background-color: #040404;
-  *background-color: #000000;
-}
-
-.navbar-inverse .btn-navbar:active,
-.navbar-inverse .btn-navbar.active {
-  background-color: #000000 \9;
-}
-
-.breadcrumb {
-  padding: 8px 15px;
-  margin: 0 0 20px;
-  list-style: none;
-  background-color: #f5f5f5;
-  -webkit-border-radius: 4px;
-     -moz-border-radius: 4px;
-          border-radius: 4px;
-}
-
-.breadcrumb li {
-  display: inline-block;
-  *display: inline;
-  text-shadow: 0 1px 0 #ffffff;
-  *zoom: 1;
-}
-
-.breadcrumb .divider {
-  padding: 0 5px;
-  color: #ccc;
-}
-
-.breadcrumb .active {
-  color: #999999;
-}
-
-.pagination {
-  height: 40px;
-  margin: 20px 0;
-}
-
-.pagination ul {
-  display: inline-block;
-  *display: inline;
-  margin-bottom: 0;
-  margin-left: 0;
-  -webkit-border-radius: 3px;
-     -moz-border-radius: 3px;
-          border-radius: 3px;
-  *zoom: 1;
-  -webkit-box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05);
-     -moz-box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05);
-          box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05);
-}
-
-.pagination li {
-  display: inline;
-}
-
-.pagination a,
-.pagination span {
-  float: left;
-  padding: 0 14px;
-  line-height: 38px;
-  text-decoration: none;
-  background-color: #ffffff;
-  border: 1px solid #dddddd;
-  border-left-width: 0;
-}
-
-.pagination a:hover,
-.pagination .active a,
-.pagination .active span {
-  background-color: #f5f5f5;
-}
-
-.pagination .active a,
-.pagination .active span {
-  color: #999999;
-  cursor: default;
-}
-
-.pagination .disabled span,
-.pagination .disabled a,
-.pagination .disabled a:hover {
-  color: #999999;
-  cursor: default;
-  background-color: transparent;
-}
-
-.pagination li:first-child a,
-.pagination li:first-child span {
-  border-left-width: 1px;
-  -webkit-border-radius: 3px 0 0 3px;
-     -moz-border-radius: 3px 0 0 3px;
-          border-radius: 3px 0 0 3px;
-}
-
-.pagination li:last-child a,
-.pagination li:last-child span {
-  -webkit-border-radius: 0 3px 3px 0;
-     -moz-border-radius: 0 3px 3px 0;
-          border-radius: 0 3px 3px 0;
-}
-
-.pagination-centered {
-  text-align: center;
-}
-
-.pagination-right {
-  text-align: right;
-}
-
-.pager {
-  margin: 20px 0;
-  text-align: center;
-  list-style: none;
-  *zoom: 1;
-}
-
-.pager:before,
-.pager:after {
-  display: table;
-  line-height: 0;
-  content: "";
-}
-
-.pager:after {
-  clear: both;
-}
-
-.pager li {
-  display: inline;
-}
-
-.pager a {
-  display: inline-block;
-  padding: 5px 14px;
-  background-color: #fff;
-  border: 1px solid #ddd;
-  -webkit-border-radius: 15px;
-     -moz-border-radius: 15px;
-          border-radius: 15px;
-}
-
-.pager a:hover {
-  text-decoration: none;
-  background-color: #f5f5f5;
-}
-
-.pager .next a {
-  float: right;
-}
-
-.pager .previous a {
-  float: left;
-}
-
-.pager .disabled a,
-.pager .disabled a:hover {
-  color: #999999;
-  cursor: default;
-  background-color: #fff;
-}
-
-.modal-open .dropdown-menu {
-  z-index: 2050;
-}
-
-.modal-open .dropdown.open {
-  *z-index: 2050;
-}
-
-.modal-open .popover {
-  z-index: 2060;
-}
-
-.modal-open .tooltip {
-  z-index: 2080;
-}
-
-.modal-backdrop {
-  position: fixed;
-  top: 0;
-  right: 0;
-  bottom: 0;
-  left: 0;
-  z-index: 1040;
-  background-color: #000000;
-}
-
-.modal-backdrop.fade {
-  opacity: 0;
-}
-
-.modal-backdrop,
-.modal-backdrop.fade.in {
-  opacity: 0.8;
-  filter: alpha(opacity=80);
-}
-
-.modal {
-  position: fixed;
-  top: 50%;
-  left: 50%;
-  z-index: 1050;
-  width: 560px;
-  margin: -250px 0 0 -280px;
-  overflow: auto;
-  background-color: #ffffff;
-  border: 1px solid #999;
-  border: 1px solid rgba(0, 0, 0, 0.3);
-  *border: 1px solid #999;
-  -webkit-border-radius: 6px;
-     -moz-border-radius: 6px;
-          border-radius: 6px;
-  -webkit-box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
-     -moz-box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
-          box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
-  -webkit-background-clip: padding-box;
-     -moz-background-clip: padding-box;
-          background-clip: padding-box;
-}
-
-.modal.fade {
-  top: -25%;
-  -webkit-transition: opacity 0.3s linear, top 0.3s ease-out;
-     -moz-transition: opacity 0.3s linear, top 0.3s ease-out;
-       -o-transition: opacity 0.3s linear, top 0.3s ease-out;
-          transition: opacity 0.3s linear, top 0.3s ease-out;
-}
-
-.modal.fade.in {
-  top: 50%;
-}
-
-.modal-header {
-  padding: 9px 15px;
-  border-bottom: 1px solid #eee;
-}
-
-.modal-header .close {
-  margin-top: 2px;
-}
-
-.modal-header h3 {
-  margin: 0;
-  line-height: 30px;
-}
-
-.modal-body {
-  max-height: 400px;
-  padding: 15px;
-  overflow-y: auto;
-}
-
-.modal-form {
-  margin-bottom: 0;
-}
-
-.modal-footer {
-  padding: 14px 15px 15px;
-  margin-bottom: 0;
-  text-align: right;
-  background-color: #f5f5f5;
-  border-top: 1px solid #ddd;
-  -webkit-border-radius: 0 0 6px 6px;
-     -moz-border-radius: 0 0 6px 6px;
-          border-radius: 0 0 6px 6px;
-  *zoom: 1;
-  -webkit-box-shadow: inset 0 1px 0 #ffffff;
-     -moz-box-shadow: inset 0 1px 0 #ffffff;
-          box-shadow: inset 0 1px 0 #ffffff;
-}
-
-.modal-footer:before,
-.modal-footer:after {
-  display: table;
-  line-height: 0;
-  content: "";
-}
-
-.modal-footer:after {
-  clear: both;
-}
-
-.modal-footer .btn + .btn {
-  margin-bottom: 0;
-  margin-left: 5px;
-}
-
-.modal-footer .btn-group .btn + .btn {
-  margin-left: -1px;
-}
-
-.tooltip {
-  position: absolute;
-  z-index: 1030;
-  display: block;
-  padding: 5px;
-  font-size: 11px;
-  opacity: 0;
-  filter: alpha(opacity=0);
-  visibility: visible;
-}
-
-.tooltip.in {
-  opacity: 0.8;
-  filter: alpha(opacity=80);
-}
-
-.tooltip.top {
-  margin-top: -3px;
-}
-
-.tooltip.right {
-  margin-left: 3px;
-}
-
-.tooltip.bottom {
-  margin-top: 3px;
-}
-
-.tooltip.left {
-  margin-left: -3px;
-}
-
-.tooltip-inner {
-  max-width: 200px;
-  padding: 3px 8px;
-  color: #ffffff;
-  text-align: center;
-  text-decoration: none;
-  background-color: #000000;
-  -webkit-border-radius: 4px;
-     -moz-border-radius: 4px;
-          border-radius: 4px;
-}
-
-.tooltip-arrow {
-  position: absolute;
-  width: 0;
-  height: 0;
-  border-color: transparent;
-  border-style: solid;
-}
-
-.tooltip.top .tooltip-arrow {
-  bottom: 0;
-  left: 50%;
-  margin-left: -5px;
-  border-top-color: #000000;
-  border-width: 5px 5px 0;
-}
-
-.tooltip.right .tooltip-arrow {
-  top: 50%;
-  left: 0;
-  margin-top: -5px;
-  border-right-color: #000000;
-  border-width: 5px 5px 5px 0;
-}
-
-.tooltip.left .tooltip-arrow {
-  top: 50%;
-  right: 0;
-  margin-top: -5px;
-  border-left-color: #000000;
-  border-width: 5px 0 5px 5px;
-}
-
-.tooltip.bottom .tooltip-arrow {
-  top: 0;
-  left: 50%;
-  margin-left: -5px;
-  border-bottom-color: #000000;
-  border-width: 0 5px 5px;
-}
-
-.popover {
-  position: absolute;
-  top: 0;
-  left: 0;
-  z-index: 1010;
-  display: none;
-  width: 236px;
-  padding: 1px;
-  background-color: #ffffff;
-  border: 1px solid #ccc;
-  border: 1px solid rgba(0, 0, 0, 0.2);
-  -webkit-border-radius: 6px;
-     -moz-border-radius: 6px;
-          border-radius: 6px;
-  -webkit-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
-     -moz-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
-          box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
-  -webkit-background-clip: padding-box;
-     -moz-background-clip: padding;
-          background-clip: padding-box;
-}
-
-.popover.top {
-  margin-bottom: 10px;
-}
-
-.popover.right {
-  margin-left: 10px;
-}
-
-.popover.bottom {
-  margin-top: 10px;
-}
-
-.popover.left {
-  margin-right: 10px;
-}
-
-.popover-title {
-  padding: 8px 14px;
-  margin: 0;
-  font-size: 14px;
-  font-weight: normal;
-  line-height: 18px;
-  background-color: #f7f7f7;
-  border-bottom: 1px solid #ebebeb;
-  -webkit-border-radius: 5px 5px 0 0;
-     -moz-border-radius: 5px 5px 0 0;
-          border-radius: 5px 5px 0 0;
-}
-
-.popover-content {
-  padding: 9px 14px;
-}
-
-.popover-content p,
-.popover-content ul,
-.popover-content ol {
-  margin-bottom: 0;
-}
-
-.popover .arrow,
-.popover .arrow:after {
-  position: absolute;
-  display: inline-block;
-  width: 0;
-  height: 0;
-  border-color: transparent;
-  border-style: solid;
-}
-
-.popover .arrow:after {
-  z-index: -1;
-  content: "";
-}
-
-.popover.top .arrow {
-  bottom: -10px;
-  left: 50%;
-  margin-left: -10px;
-  border-top-color: #ffffff;
-  border-width: 10px 10px 0;
-}
-
-.popover.top .arrow:after {
-  bottom: -1px;
-  left: -11px;
-  border-top-color: rgba(0, 0, 0, 0.25);
-  border-width: 11px 11px 0;
-}
-
-.popover.right .arrow {
-  top: 50%;
-  left: -10px;
-  margin-top: -10px;
-  border-right-color: #ffffff;
-  border-width: 10px 10px 10px 0;
-}
-
-.popover.right .arrow:after {
-  bottom: -11px;
-  left: -1px;
-  border-right-color: rgba(0, 0, 0, 0.25);
-  border-width: 11px 11px 11px 0;
-}
-
-.popover.bottom .arrow {
-  top: -10px;
-  left: 50%;
-  margin-left: -10px;
-  border-bottom-color: #ffffff;
-  border-width: 0 10px 10px;
-}
-
-.popover.bottom .arrow:after {
-  top: -1px;
-  left: -11px;
-  border-bottom-color: rgba(0, 0, 0, 0.25);
-  border-width: 0 11px 11px;
-}
-
-.popover.left .arrow {
-  top: 50%;
-  right: -10px;
-  margin-top: -10px;
-  border-left-color: #ffffff;
-  border-width: 10px 0 10px 10px;
-}
-
-.popover.left .arrow:after {
-  right: -1px;
-  bottom: -11px;
-  border-left-color: rgba(0, 0, 0, 0.25);
-  border-width: 11px 0 11px 11px;
-}
-
-.thumbnails {
-  margin-left: -20px;
-  list-style: none;
-  *zoom: 1;
-}
-
-.thumbnails:before,
-.thumbnails:after {
-  display: table;
-  line-height: 0;
-  content: "";
-}
-
-.thumbnails:after {
-  clear: both;
-}
-
-.row-fluid .thumbnails {
-  margin-left: 0;
-}
-
-.thumbnails > li {
-  float: left;
-  margin-bottom: 20px;
-  margin-left: 20px;
-}
-
-.thumbnail {
-  display: block;
-  padding: 4px;
-  line-height: 20px;
-  border: 1px solid #ddd;
-  -webkit-border-radius: 4px;
-     -moz-border-radius: 4px;
-          border-radius: 4px;
-  -webkit-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.055);
-     -moz-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.055);
-          box-shadow: 0 1px 3px rgba(0, 0, 0, 0.055);
-  -webkit-transition: all 0.2s ease-in-out;
-     -moz-transition: all 0.2s ease-in-out;
-       -o-transition: all 0.2s ease-in-out;
-          transition: all 0.2s ease-in-out;
-}
-
-a.thumbnail:hover {
-  border-color: #0088cc;
-  -webkit-box-shadow: 0 1px 4px rgba(0, 105, 214, 0.25);
-     -moz-box-shadow: 0 1px 4px rgba(0, 105, 214, 0.25);
-          box-shadow: 0 1px 4px rgba(0, 105, 214, 0.25);
-}
-
-.thumbnail > img {
-  display: block;
-  max-width: 100%;
-  margin-right: auto;
-  margin-left: auto;
-}
-
-.thumbnail .caption {
-  padding: 9px;
-  color: #555555;
-}
-
-.label,
-.badge {
-  font-size: 11.844px;
-  font-weight: bold;
-  line-height: 14px;
-  color: #ffffff;
-  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
-  white-space: nowrap;
-  vertical-align: baseline;
-  background-color: #999999;
-}
-
-.label {
-  padding: 1px 4px 2px;
-  -webkit-border-radius: 3px;
-     -moz-border-radius: 3px;
-          border-radius: 3px;
-}
-
-.badge {
-  padding: 1px 9px 2px;
-  -webkit-border-radius: 9px;
-     -moz-border-radius: 9px;
-          border-radius: 9px;
-}
-
-a.label:hover,
-a.badge:hover {
-  color: #ffffff;
-  text-decoration: none;
-  cursor: pointer;
-}
-
-.label-important,
-.badge-important {
-  background-color: #b94a48;
-}
-
-.label-important[href],
-.badge-important[href] {
-  background-color: #953b39;
-}
-
-.label-warning,
-.badge-warning {
-  background-color: #f89406;
-}
-
-.label-warning[href],
-.badge-warning[href] {
-  background-color: #c67605;
-}
-
-.label-success,
-.badge-success {
-  background-color: #468847;
-}
-
-.label-success[href],
-.badge-success[href] {
-  background-color: #356635;
-}
-
-.label-info,
-.badge-info {
-  background-color: #3a87ad;
-}
-
-.label-info[href],
-.badge-info[href] {
-  background-color: #2d6987;
-}
-
-.label-inverse,
-.badge-inverse {
-  background-color: #333333;
-}
-
-.label-inverse[href],
-.badge-inverse[href] {
-  background-color: #1a1a1a;
-}
-
-.btn .label,
-.btn .badge {
-  position: relative;
-  top: -1px;
-}
-
-.btn-mini .label,
-.btn-mini .badge {
-  top: 0;
-}
-
-@-webkit-keyframes progress-bar-stripes {
-  from {
-    background-position: 40px 0;
-  }
-  to {
-    background-position: 0 0;
-  }
-}
-
-@-moz-keyframes progress-bar-stripes {
-  from {
-    background-position: 40px 0;
-  }
-  to {
-    background-position: 0 0;
-  }
-}
-
-@-ms-keyframes progress-bar-stripes {
-  from {
-    background-position: 40px 0;
-  }
-  to {
-    background-position: 0 0;
-  }
-}
-
-@-o-keyframes progress-bar-stripes {
-  from {
-    background-position: 0 0;
-  }
-  to {
-    background-position: 40px 0;
-  }
-}
-
-@keyframes progress-bar-stripes {
-  from {
-    background-position: 40px 0;
-  }
-  to {
-    background-position: 0 0;
-  }
-}
-
-.progress {
-  height: 20px;
-  margin-bottom: 20px;
-  overflow: hidden;
-  background-color: #f7f7f7;
-  background-image: -moz-linear-gradient(top, #f5f5f5, #f9f9f9);
-  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#f5f5f5), to(#f9f9f9));
-  background-image: -webkit-linear-gradient(top, #f5f5f5, #f9f9f9);
-  background-image: -o-linear-gradient(top, #f5f5f5, #f9f9f9);
-  background-image: linear-gradient(to bottom, #f5f5f5, #f9f9f9);
-  background-repeat: repeat-x;
-  -webkit-border-radius: 4px;
-     -moz-border-radius: 4px;
-          border-radius: 4px;
-  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#fff5f5f5', endColorstr='#fff9f9f9', GradientType=0);
-  -webkit-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1);
-     -moz-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1);
-          box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1);
-}
-
-.progress .bar {
-  float: left;
-  width: 0;
-  height: 100%;
-  font-size: 12px;
-  color: #ffffff;
-  text-align: center;
-  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
-  background-color: #0e90d2;
-  background-image: -moz-linear-gradient(top, #149bdf, #0480be);
-  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#149bdf), to(#0480be));
-  background-image: -webkit-linear-gradient(top, #149bdf, #0480be);
-  background-image: -o-linear-gradient(top, #149bdf, #0480be);
-  background-image: linear-gradient(to bottom, #149bdf, #0480be);
-  background-repeat: repeat-x;
-  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff149bdf', endColorstr='#ff0480be', GradientType=0);
-  -webkit-box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15);
-     -moz-box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15);
-          box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15);
-  -webkit-box-sizing: border-box;
-     -moz-box-sizing: border-box;
-          box-sizing: border-box;
-  -webkit-transition: width 0.6s ease;
-     -moz-transition: width 0.6s ease;
-       -o-transition: width 0.6s ease;
-          transition: width 0.6s ease;
-}
-
-.progress .bar + .bar {
-  -webkit-box-shadow: inset 1px 0 0 rgba(0, 0, 0, 0.15), inset 0 -1px 0 rgba(0, 0, 0, 0.15);
-     -moz-box-shadow: inset 1px 0 0 rgba(0, 0, 0, 0.15), inset 0 -1px 0 rgba(0, 0, 0, 0.15);
-          box-shadow: inset 1px 0 0 rgba(0, 0, 0, 0.15), inset 0 -1px 0 rgba(0, 0, 0, 0.15);
-}
-
-.progress-striped .bar {
-  background-color: #149bdf;
-  background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
-  background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
-  background-image: -moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
-  background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
-  background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
-  -webkit-background-size: 40px 40px;
-     -moz-background-size: 40px 40px;
-       -o-background-size: 40px 40px;
-          background-size: 40px 40px;
-}
-
-.progress.active .bar {
-  -webkit-animation: progress-bar-stripes 2s linear infinite;
-     -moz-animation: progress-bar-stripes 2s linear infinite;
-      -ms-animation: progress-bar-stripes 2s linear infinite;
-       -o-animation: progress-bar-stripes 2s linear infinite;
-          animation: progress-bar-stripes 2s linear infinite;
-}
-
-.progress-danger .bar,
-.progress .bar-danger {
-  background-color: #dd514c;
-  background-image: -moz-linear-gradient(top, #ee5f5b, #c43c35);
-  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ee5f5b), to(#c43c35));
-  background-image: -webkit-linear-gradient(top, #ee5f5b, #c43c35);
-  background-image: -o-linear-gradient(top, #ee5f5b, #c43c35);
-  background-image: linear-gradient(to bottom, #ee5f5b, #c43c35);
-  background-repeat: repeat-x;
-  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ffee5f5b', endColorstr='#ffc43c35', GradientType=0);
-}
-
-.progress-danger.progress-striped .bar,
-.progress-striped .bar-danger {
-  background-color: #ee5f5b;
-  background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
-  background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
-  background-image: -moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
-  background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
-  background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
-}
-
-.progress-success .bar,
-.progress .bar-success {
-  background-color: #5eb95e;
-  background-image: -moz-linear-gradient(top, #62c462, #57a957);
-  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#62c462), to(#57a957));
-  background-image: -webkit-linear-gradient(top, #62c462, #57a957);
-  background-image: -o-linear-gradient(top, #62c462, #57a957);
-  background-image: linear-gradient(to bottom, #62c462, #57a957);
-  background-repeat: repeat-x;
-  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff62c462', endColorstr='#ff57a957', GradientType=0);
-}
-
-.progress-success.progress-striped .bar,
-.progress-striped .bar-success {
-  background-color: #62c462;
-  background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
-  background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
-  background-image: -moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
-  background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
-  background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
-}
-
-.progress-info .bar,
-.progress .bar-info {
-  background-color: #4bb1cf;
-  background-image: -moz-linear-gradient(top, #5bc0de, #339bb9);
-  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#5bc0de), to(#339bb9));
-  background-image: -webkit-linear-gradient(top, #5bc0de, #339bb9);
-  background-image: -o-linear-gradient(top, #5bc0de, #339bb9);
-  background-image: linear-gradient(to bottom, #5bc0de, #339bb9);
-  background-repeat: repeat-x;
-  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff5bc0de', endColorstr='#ff339bb9', GradientType=0);
-}
-
-.progress-info.progress-striped .bar,
-.progress-striped .bar-info {
-  background-color: #5bc0de;
-  background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
-  background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
-  background-image: -moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
-  background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
-  background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
-}
-
-.progress-warning .bar,
-.progress .bar-warning {
-  background-color: #faa732;
-  background-image: -moz-linear-gradient(top, #fbb450, #f89406);
-  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#fbb450), to(#f89406));
-  background-image: -webkit-linear-gradient(top, #fbb450, #f89406);
-  background-image: -o-linear-gradient(top, #fbb450, #f89406);
-  background-image: linear-gradient(to bottom, #fbb450, #f89406);
-  background-repeat: repeat-x;
-  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#fffbb450', endColorstr='#fff89406', GradientType=0);
-}
-
-.progress-warning.progress-striped .bar,
-.progress-striped .bar-warning {
-  background-color: #fbb450;
-  background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
-  background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
-  background-image: -moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
-  background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
-  background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
-}
-
-.accordion {
-  margin-bottom: 20px;
-}
-
-.accordion-group {
-  margin-bottom: 2px;
-  border: 1px solid #e5e5e5;
-  -webkit-border-radius: 4px;
-     -moz-border-radius: 4px;
-          border-radius: 4px;
-}
-
-.accordion-heading {
-  border-bottom: 0;
-}
-
-.accordion-heading .accordion-toggle {
-  display: block;
-  padding: 8px 15px;
-}
-
-.accordion-toggle {
-  cursor: pointer;
-}
-
-.accordion-inner {
-  padding: 9px 15px;
-  border-top: 1px solid #e5e5e5;
-}
-
-.carousel {
-  position: relative;
-  margin-bottom: 20px;
-  line-height: 1;
-}
-
-.carousel-inner {
-  position: relative;
-  width: 100%;
-  overflow: hidden;
-}
-
-.carousel .item {
-  position: relative;
-  display: none;
-  -webkit-transition: 0.6s ease-in-out left;
-     -moz-transition: 0.6s ease-in-out left;
-       -o-transition: 0.6s ease-in-out left;
-          transition: 0.6s ease-in-out left;
-}
-
-.carousel .item > img {
-  display: block;
-  line-height: 1;
-}
-
-.carousel .active,
-.carousel .next,
-.carousel .prev {
-  display: block;
-}
-
-.carousel .active {
-  left: 0;
-}
-
-.carousel .next,
-.carousel .prev {
-  position: absolute;
-  top: 0;
-  width: 100%;
-}
-
-.carousel .next {
-  left: 100%;
-}
-
-.carousel .prev {
-  left: -100%;
-}
-
-.carousel .next.left,
-.carousel .prev.right {
-  left: 0;
-}
-
-.carousel .active.left {
-  left: -100%;
-}
-
-.carousel .active.right {
-  left: 100%;
-}
-
-.carousel-control {
-  position: absolute;
-  top: 40%;
-  left: 15px;
-  width: 40px;
-  height: 40px;
-  margin-top: -20px;
-  font-size: 60px;
-  font-weight: 100;
-  line-height: 30px;
-  color: #ffffff;
-  text-align: center;
-  background: #222222;
-  border: 3px solid #ffffff;
-  -webkit-border-radius: 23px;
-     -moz-border-radius: 23px;
-          border-radius: 23px;
-  opacity: 0.5;
-  filter: alpha(opacity=50);
-}
-
-.carousel-control.right {
-  right: 15px;
-  left: auto;
-}
-
-.carousel-control:hover {
-  color: #ffffff;
-  text-decoration: none;
-  opacity: 0.9;
-  filter: alpha(opacity=90);
-}
-
-.carousel-caption {
-  position: absolute;
-  right: 0;
-  bottom: 0;
-  left: 0;
-  padding: 15px;
-  background: #333333;
-  background: rgba(0, 0, 0, 0.75);
-}
-
-.carousel-caption h4,
-.carousel-caption p {
-  line-height: 20px;
-  color: #ffffff;
-}
-
-.carousel-caption h4 {
-  margin: 0 0 5px;
-}
-
-.carousel-caption p {
-  margin-bottom: 0;
-}
-
-.hero-unit {
-  padding: 60px;
-  margin-bottom: 30px;
-  background-color: #eeeeee;
-  -webkit-border-radius: 6px;
-     -moz-border-radius: 6px;
-          border-radius: 6px;
-}
-
-.hero-unit h1 {
-  margin-bottom: 0;
-  font-size: 60px;
-  line-height: 1;
-  letter-spacing: -1px;
-  color: inherit;
-}
-
-.hero-unit p {
-  font-size: 18px;
-  font-weight: 200;
-  line-height: 30px;
-  color: inherit;
-}
-
-.pull-right {
-  float: right;
-}
-
-.pull-left {
-  float: left;
-}
-
-.hide {
-  display: none;
-}
-
-.show {
-  display: block;
-}
-
-.invisible {
-  visibility: hidden;
-}
-
-.affix {
-  position: fixed;
-}
diff --git a/docs/css/bootstrap.min.css b/docs/css/bootstrap.min.css
index 3fa12ac2b2081..86b6845bc3420 100644
--- a/docs/css/bootstrap.min.css
+++ b/docs/css/bootstrap.min.css
@@ -1,9 +1,7 @@
 /*!
- * Bootstrap v2.1.0
- *
- * Copyright 2012 Twitter, Inc
- * Licensed under the Apache License v2.0
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Designed and built with all the love in the world @twitter by @mdo and @fat.
- */article,aside,details,figcaption,figure,footer,header,hgroup,nav,section{display:block}audio,canvas,video{display:inline-block;*display:inline;*zoom:1}audio:not([controls]){display:none}html{font-size:100%;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%}a:focus{outline:thin dotted #333;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}a:hover,a:active{outline:0}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sup{top:-0.5em}sub{bottom:-0.25em}img{height:auto;max-width:100%;vertical-align:middle;border:0;-ms-interpolation-mode:bicubic}#map_canvas img{max-width:none}button,input,select,textarea{margin:0;font-size:100%;vertical-align:middle}button,input{*overflow:visible;line-height:normal}button::-moz-focus-inner,input::-moz-focus-inner{padding:0;border:0}button,input[type="button"],input[type="reset"],input[type="submit"]{cursor:pointer;-webkit-appearance:button}input[type="search"]{-webkit-box-sizing:content-box;-moz-box-sizing:content-box;box-sizing:content-box;-webkit-appearance:textfield}input[type="search"]::-webkit-search-decoration,input[type="search"]::-webkit-search-cancel-button{-webkit-appearance:none}textarea{overflow:auto;vertical-align:top}.clearfix{*zoom:1}.clearfix:before,.clearfix:after{display:table;line-height:0;content:""}.clearfix:after{clear:both}.hide-text{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.input-block-level{display:block;width:100%;min-height:30px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}body{margin:0;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:14px;line-height:20px;color:#333;background-color:#fff}a{color:#08c;text-decoration:none}a:hover{color:#005580;text-decoration:underline}.img-rounded{-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px}.img-polaroid{padding:4px;background-color:#fff;border:1px solid #ccc;border:1px solid rgba(0,0,0,0.2);-webkit-box-shadow:0 1px 3px rgba(0,0,0,0.1);-moz-box-shadow:0 1px 3px rgba(0,0,0,0.1);box-shadow:0 1px 3px rgba(0,0,0,0.1)}.img-circle{-webkit-border-radius:500px;-moz-border-radius:500px;border-radius:500px}.row{margin-left:-20px;*zoom:1}.row:before,.row:after{display:table;line-height:0;content:""}.row:after{clear:both}[class*="span"]{float:left;margin-left:20px}.container,.navbar-static-top .container,.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:940px}.span12{width:940px}.span11{width:860px}.span10{width:780px}.span9{width:700px}.span8{width:620px}.span7{width:540px}.span6{width:460px}.span5{width:380px}.span4{width:300px}.span3{width:220px}.span2{width:140px}.span1{width:60px}.offset12{margin-left:980px}.offset11{margin-left:900px}.offset10{margin-left:820px}.offset9{margin-left:740px}.offset8{margin-left:660px}.offset7{margin-left:580px}.offset6{margin-left:500px}.offset5{margin-left:420px}.offset4{margin-left:340px}.offset3{margin-left:260px}.offset2{margin-left:180px}.offset1{margin-left:100px}.row-fluid{width:100%;*zoom:1}.row-fluid:before,.row-fluid:after{display:table;line-height:0;content:""}.row-fluid:after{clear:both}.row-fluid [class*="span"]{display:block;float:left;width:100%;min-height:30px;margin-left:2.127659574468085%;*margin-left:2.074468085106383%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.row-fluid [class*="span"]:first-child{margin-left:0}.row-fluid .span12{width:100%;*width:99.94680851063829%}.row-fluid .span11{width:91.48936170212765%;*width:91.43617021276594%}.row-fluid .span10{width:82.97872340425532%;*width:82.92553191489361%}.row-fluid .span9{width:74.46808510638297%;*width:74.41489361702126%}.row-fluid .span8{width:65.95744680851064%;*width:65.90425531914893%}.row-fluid .span7{width:57.44680851063829%;*width:57.39361702127659%}.row-fluid .span6{width:48.93617021276595%;*width:48.88297872340425%}.row-fluid .span5{width:40.42553191489362%;*width:40.37234042553192%}.row-fluid .span4{width:31.914893617021278%;*width:31.861702127659576%}.row-fluid .span3{width:23.404255319148934%;*width:23.351063829787233%}.row-fluid .span2{width:14.893617021276595%;*width:14.840425531914894%}.row-fluid .span1{width:6.382978723404255%;*width:6.329787234042553%}.row-fluid .offset12{margin-left:104.25531914893617%;*margin-left:104.14893617021275%}.row-fluid .offset12:first-child{margin-left:102.12765957446808%;*margin-left:102.02127659574467%}.row-fluid .offset11{margin-left:95.74468085106382%;*margin-left:95.6382978723404%}.row-fluid .offset11:first-child{margin-left:93.61702127659574%;*margin-left:93.51063829787232%}.row-fluid .offset10{margin-left:87.23404255319149%;*margin-left:87.12765957446807%}.row-fluid .offset10:first-child{margin-left:85.1063829787234%;*margin-left:84.99999999999999%}.row-fluid .offset9{margin-left:78.72340425531914%;*margin-left:78.61702127659572%}.row-fluid .offset9:first-child{margin-left:76.59574468085106%;*margin-left:76.48936170212764%}.row-fluid .offset8{margin-left:70.2127659574468%;*margin-left:70.10638297872339%}.row-fluid .offset8:first-child{margin-left:68.08510638297872%;*margin-left:67.9787234042553%}.row-fluid .offset7{margin-left:61.70212765957446%;*margin-left:61.59574468085106%}.row-fluid .offset7:first-child{margin-left:59.574468085106375%;*margin-left:59.46808510638297%}.row-fluid .offset6{margin-left:53.191489361702125%;*margin-left:53.085106382978715%}.row-fluid .offset6:first-child{margin-left:51.063829787234035%;*margin-left:50.95744680851063%}.row-fluid .offset5{margin-left:44.68085106382979%;*margin-left:44.57446808510638%}.row-fluid .offset5:first-child{margin-left:42.5531914893617%;*margin-left:42.4468085106383%}.row-fluid .offset4{margin-left:36.170212765957444%;*margin-left:36.06382978723405%}.row-fluid .offset4:first-child{margin-left:34.04255319148936%;*margin-left:33.93617021276596%}.row-fluid .offset3{margin-left:27.659574468085104%;*margin-left:27.5531914893617%}.row-fluid .offset3:first-child{margin-left:25.53191489361702%;*margin-left:25.425531914893618%}.row-fluid .offset2{margin-left:19.148936170212764%;*margin-left:19.04255319148936%}.row-fluid .offset2:first-child{margin-left:17.02127659574468%;*margin-left:16.914893617021278%}.row-fluid .offset1{margin-left:10.638297872340425%;*margin-left:10.53191489361702%}.row-fluid .offset1:first-child{margin-left:8.51063829787234%;*margin-left:8.404255319148938%}[class*="span"].hide,.row-fluid [class*="span"].hide{display:none}[class*="span"].pull-right,.row-fluid [class*="span"].pull-right{float:right}.container{margin-right:auto;margin-left:auto;*zoom:1}.container:before,.container:after{display:table;line-height:0;content:""}.container:after{clear:both}.container-fluid{padding-right:20px;padding-left:20px;*zoom:1}.container-fluid:before,.container-fluid:after{display:table;line-height:0;content:""}.container-fluid:after{clear:both}p{margin:0 0 10px}.lead{margin-bottom:20px;font-size:20px;font-weight:200;line-height:30px}small{font-size:85%}strong{font-weight:bold}em{font-style:italic}cite{font-style:normal}.muted{color:#999}h1,h2,h3,h4,h5,h6{margin:10px 0;font-family:inherit;font-weight:bold;line-height:1;color:inherit;text-rendering:optimizelegibility}h1 small,h2 small,h3 small,h4 small,h5 small,h6 small{font-weight:normal;line-height:1;color:#999}h1{font-size:36px;line-height:40px}h2{font-size:30px;line-height:40px}h3{font-size:24px;line-height:40px}h4{font-size:18px;line-height:20px}h5{font-size:14px;line-height:20px}h6{font-size:12px;line-height:20px}h1 small{font-size:24px}h2 small{font-size:18px}h3 small{font-size:14px}h4 small{font-size:14px}.page-header{padding-bottom:9px;margin:20px 0 30px;border-bottom:1px solid #eee}ul,ol{padding:0;margin:0 0 10px 25px}ul ul,ul ol,ol ol,ol ul{margin-bottom:0}li{line-height:20px}ul.unstyled,ol.unstyled{margin-left:0;list-style:none}dl{margin-bottom:20px}dt,dd{line-height:20px}dt{font-weight:bold}dd{margin-left:10px}.dl-horizontal dt{float:left;width:120px;overflow:hidden;clear:left;text-align:right;text-overflow:ellipsis;white-space:nowrap}.dl-horizontal dd{margin-left:130px}hr{margin:20px 0;border:0;border-top:1px solid #eee;border-bottom:1px solid #fff}abbr[title]{cursor:help;border-bottom:1px dotted #999}abbr.initialism{font-size:90%;text-transform:uppercase}blockquote{padding:0 0 0 15px;margin:0 0 20px;border-left:5px solid #eee}blockquote p{margin-bottom:0;font-size:16px;font-weight:300;line-height:25px}blockquote small{display:block;line-height:20px;color:#999}blockquote small:before{content:'\2014 \00A0'}blockquote.pull-right{float:right;padding-right:15px;padding-left:0;border-right:5px solid #eee;border-left:0}blockquote.pull-right p,blockquote.pull-right small{text-align:right}blockquote.pull-right small:before{content:''}blockquote.pull-right small:after{content:'\00A0 \2014'}q:before,q:after,blockquote:before,blockquote:after{content:""}address{display:block;margin-bottom:20px;font-style:normal;line-height:20px}code,pre{padding:0 3px 2px;font-family:Monaco,Menlo,Consolas,"Courier New",monospace;font-size:12px;color:#333;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}code{padding:2px 4px;color:#d14;background-color:#f7f7f9;border:1px solid #e1e1e8}pre{display:block;padding:9.5px;margin:0 0 10px;font-size:13px;line-height:20px;word-break:break-all;word-wrap:break-word;white-space:pre;white-space:pre-wrap;background-color:#f5f5f5;border:1px solid #ccc;border:1px solid rgba(0,0,0,0.15);-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}pre.prettyprint{margin-bottom:20px}pre code{padding:0;color:inherit;background-color:transparent;border:0}.pre-scrollable{max-height:340px;overflow-y:scroll}form{margin:0 0 20px}fieldset{padding:0;margin:0;border:0}legend{display:block;width:100%;padding:0;margin-bottom:20px;font-size:21px;line-height:40px;color:#333;border:0;border-bottom:1px solid #e5e5e5}legend small{font-size:15px;color:#999}label,input,button,select,textarea{font-size:14px;font-weight:normal;line-height:20px}input,button,select,textarea{font-family:"Helvetica Neue",Helvetica,Arial,sans-serif}label{display:block;margin-bottom:5px}select,textarea,input[type="text"],input[type="password"],input[type="datetime"],input[type="datetime-local"],input[type="date"],input[type="month"],input[type="time"],input[type="week"],input[type="number"],input[type="email"],input[type="url"],input[type="search"],input[type="tel"],input[type="color"],.uneditable-input{display:inline-block;height:20px;padding:4px 6px;margin-bottom:9px;font-size:14px;line-height:20px;color:#555;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}input,textarea{width:210px}textarea{height:auto}textarea,input[type="text"],input[type="password"],input[type="datetime"],input[type="datetime-local"],input[type="date"],input[type="month"],input[type="time"],input[type="week"],input[type="number"],input[type="email"],input[type="url"],input[type="search"],input[type="tel"],input[type="color"],.uneditable-input{background-color:#fff;border:1px solid #ccc;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-webkit-transition:border linear .2s,box-shadow linear .2s;-moz-transition:border linear .2s,box-shadow linear .2s;-o-transition:border linear .2s,box-shadow linear .2s;transition:border linear .2s,box-shadow linear .2s}textarea:focus,input[type="text"]:focus,input[type="password"]:focus,input[type="datetime"]:focus,input[type="datetime-local"]:focus,input[type="date"]:focus,input[type="month"]:focus,input[type="time"]:focus,input[type="week"]:focus,input[type="number"]:focus,input[type="email"]:focus,input[type="url"]:focus,input[type="search"]:focus,input[type="tel"]:focus,input[type="color"]:focus,.uneditable-input:focus{border-color:rgba(82,168,236,0.8);outline:0;outline:thin dotted \9;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 8px rgba(82,168,236,0.6);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 8px rgba(82,168,236,0.6);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 8px rgba(82,168,236,0.6)}input[type="radio"],input[type="checkbox"]{margin:4px 0 0;margin-top:1px \9;*margin-top:0;line-height:normal;cursor:pointer}input[type="file"],input[type="image"],input[type="submit"],input[type="reset"],input[type="button"],input[type="radio"],input[type="checkbox"]{width:auto}select,input[type="file"]{height:30px;*margin-top:4px;line-height:30px}select{width:220px;background-color:#fff;border:1px solid #bbb}select[multiple],select[size]{height:auto}select:focus,input[type="file"]:focus,input[type="radio"]:focus,input[type="checkbox"]:focus{outline:thin dotted #333;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}.uneditable-input,.uneditable-textarea{color:#999;cursor:not-allowed;background-color:#fcfcfc;border-color:#ccc;-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,0.025);-moz-box-shadow:inset 0 1px 2px rgba(0,0,0,0.025);box-shadow:inset 0 1px 2px rgba(0,0,0,0.025)}.uneditable-input{overflow:hidden;white-space:nowrap}.uneditable-textarea{width:auto;height:auto}input:-moz-placeholder,textarea:-moz-placeholder{color:#999}input:-ms-input-placeholder,textarea:-ms-input-placeholder{color:#999}input::-webkit-input-placeholder,textarea::-webkit-input-placeholder{color:#999}.radio,.checkbox{min-height:18px;padding-left:18px}.radio input[type="radio"],.checkbox input[type="checkbox"]{float:left;margin-left:-18px}.controls>.radio:first-child,.controls>.checkbox:first-child{padding-top:5px}.radio.inline,.checkbox.inline{display:inline-block;padding-top:5px;margin-bottom:0;vertical-align:middle}.radio.inline+.radio.inline,.checkbox.inline+.checkbox.inline{margin-left:10px}.input-mini{width:60px}.input-small{width:90px}.input-medium{width:150px}.input-large{width:210px}.input-xlarge{width:270px}.input-xxlarge{width:530px}input[class*="span"],select[class*="span"],textarea[class*="span"],.uneditable-input[class*="span"],.row-fluid input[class*="span"],.row-fluid select[class*="span"],.row-fluid textarea[class*="span"],.row-fluid .uneditable-input[class*="span"]{float:none;margin-left:0}.input-append input[class*="span"],.input-append .uneditable-input[class*="span"],.input-prepend input[class*="span"],.input-prepend .uneditable-input[class*="span"],.row-fluid input[class*="span"],.row-fluid select[class*="span"],.row-fluid textarea[class*="span"],.row-fluid .uneditable-input[class*="span"],.row-fluid .input-prepend [class*="span"],.row-fluid .input-append [class*="span"]{display:inline-block}input,textarea,.uneditable-input{margin-left:0}.controls-row [class*="span"]+[class*="span"]{margin-left:20px}input.span12,textarea.span12,.uneditable-input.span12{width:926px}input.span11,textarea.span11,.uneditable-input.span11{width:846px}input.span10,textarea.span10,.uneditable-input.span10{width:766px}input.span9,textarea.span9,.uneditable-input.span9{width:686px}input.span8,textarea.span8,.uneditable-input.span8{width:606px}input.span7,textarea.span7,.uneditable-input.span7{width:526px}input.span6,textarea.span6,.uneditable-input.span6{width:446px}input.span5,textarea.span5,.uneditable-input.span5{width:366px}input.span4,textarea.span4,.uneditable-input.span4{width:286px}input.span3,textarea.span3,.uneditable-input.span3{width:206px}input.span2,textarea.span2,.uneditable-input.span2{width:126px}input.span1,textarea.span1,.uneditable-input.span1{width:46px}.controls-row{*zoom:1}.controls-row:before,.controls-row:after{display:table;line-height:0;content:""}.controls-row:after{clear:both}.controls-row [class*="span"]{float:left}input[disabled],select[disabled],textarea[disabled],input[readonly],select[readonly],textarea[readonly]{cursor:not-allowed;background-color:#eee}input[type="radio"][disabled],input[type="checkbox"][disabled],input[type="radio"][readonly],input[type="checkbox"][readonly]{background-color:transparent}.control-group.warning>label,.control-group.warning .help-block,.control-group.warning .help-inline{color:#c09853}.control-group.warning .checkbox,.control-group.warning .radio,.control-group.warning input,.control-group.warning select,.control-group.warning textarea{color:#c09853;border-color:#c09853;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075)}.control-group.warning .checkbox:focus,.control-group.warning .radio:focus,.control-group.warning input:focus,.control-group.warning select:focus,.control-group.warning textarea:focus{border-color:#a47e3c;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #dbc59e;-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #dbc59e;box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #dbc59e}.control-group.warning .input-prepend .add-on,.control-group.warning .input-append .add-on{color:#c09853;background-color:#fcf8e3;border-color:#c09853}.control-group.error>label,.control-group.error .help-block,.control-group.error .help-inline{color:#b94a48}.control-group.error .checkbox,.control-group.error .radio,.control-group.error input,.control-group.error select,.control-group.error textarea{color:#b94a48;border-color:#b94a48;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075)}.control-group.error .checkbox:focus,.control-group.error .radio:focus,.control-group.error input:focus,.control-group.error select:focus,.control-group.error textarea:focus{border-color:#953b39;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #d59392;-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #d59392;box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #d59392}.control-group.error .input-prepend .add-on,.control-group.error .input-append .add-on{color:#b94a48;background-color:#f2dede;border-color:#b94a48}.control-group.success>label,.control-group.success .help-block,.control-group.success .help-inline{color:#468847}.control-group.success .checkbox,.control-group.success .radio,.control-group.success input,.control-group.success select,.control-group.success textarea{color:#468847;border-color:#468847;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075)}.control-group.success .checkbox:focus,.control-group.success .radio:focus,.control-group.success input:focus,.control-group.success select:focus,.control-group.success textarea:focus{border-color:#356635;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #7aba7b;-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #7aba7b;box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #7aba7b}.control-group.success .input-prepend .add-on,.control-group.success .input-append .add-on{color:#468847;background-color:#dff0d8;border-color:#468847}input:focus:required:invalid,textarea:focus:required:invalid,select:focus:required:invalid{color:#b94a48;border-color:#ee5f5b}input:focus:required:invalid:focus,textarea:focus:required:invalid:focus,select:focus:required:invalid:focus{border-color:#e9322d;-webkit-box-shadow:0 0 6px #f8b9b7;-moz-box-shadow:0 0 6px #f8b9b7;box-shadow:0 0 6px #f8b9b7}.form-actions{padding:19px 20px 20px;margin-top:20px;margin-bottom:20px;background-color:#f5f5f5;border-top:1px solid #e5e5e5;*zoom:1}.form-actions:before,.form-actions:after{display:table;line-height:0;content:""}.form-actions:after{clear:both}.help-block,.help-inline{color:#595959}.help-block{display:block;margin-bottom:10px}.help-inline{display:inline-block;*display:inline;padding-left:5px;vertical-align:middle;*zoom:1}.input-append,.input-prepend{margin-bottom:5px;font-size:0;white-space:nowrap}.input-append input,.input-prepend input,.input-append select,.input-prepend select,.input-append .uneditable-input,.input-prepend .uneditable-input{position:relative;margin-bottom:0;*margin-left:0;font-size:14px;vertical-align:top;-webkit-border-radius:0 3px 3px 0;-moz-border-radius:0 3px 3px 0;border-radius:0 3px 3px 0}.input-append input:focus,.input-prepend input:focus,.input-append select:focus,.input-prepend select:focus,.input-append .uneditable-input:focus,.input-prepend .uneditable-input:focus{z-index:2}.input-append .add-on,.input-prepend .add-on{display:inline-block;width:auto;height:20px;min-width:16px;padding:4px 5px;font-size:14px;font-weight:normal;line-height:20px;text-align:center;text-shadow:0 1px 0 #fff;background-color:#eee;border:1px solid #ccc}.input-append .add-on,.input-prepend .add-on,.input-append .btn,.input-prepend .btn{margin-left:-1px;vertical-align:top;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.input-append .active,.input-prepend .active{background-color:#a9dba9;border-color:#46a546}.input-prepend .add-on,.input-prepend .btn{margin-right:-1px}.input-prepend .add-on:first-child,.input-prepend .btn:first-child{-webkit-border-radius:3px 0 0 3px;-moz-border-radius:3px 0 0 3px;border-radius:3px 0 0 3px}.input-append input,.input-append select,.input-append .uneditable-input{-webkit-border-radius:3px 0 0 3px;-moz-border-radius:3px 0 0 3px;border-radius:3px 0 0 3px}.input-append .add-on:last-child,.input-append .btn:last-child{-webkit-border-radius:0 3px 3px 0;-moz-border-radius:0 3px 3px 0;border-radius:0 3px 3px 0}.input-prepend.input-append input,.input-prepend.input-append select,.input-prepend.input-append .uneditable-input{-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.input-prepend.input-append .add-on:first-child,.input-prepend.input-append .btn:first-child{margin-right:-1px;-webkit-border-radius:3px 0 0 3px;-moz-border-radius:3px 0 0 3px;border-radius:3px 0 0 3px}.input-prepend.input-append .add-on:last-child,.input-prepend.input-append .btn:last-child{margin-left:-1px;-webkit-border-radius:0 3px 3px 0;-moz-border-radius:0 3px 3px 0;border-radius:0 3px 3px 0}input.search-query{padding-right:14px;padding-right:4px \9;padding-left:14px;padding-left:4px \9;margin-bottom:0;-webkit-border-radius:15px;-moz-border-radius:15px;border-radius:15px}.form-search .input-append .search-query,.form-search .input-prepend .search-query{-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.form-search .input-append .search-query{-webkit-border-radius:14px 0 0 14px;-moz-border-radius:14px 0 0 14px;border-radius:14px 0 0 14px}.form-search .input-append .btn{-webkit-border-radius:0 14px 14px 0;-moz-border-radius:0 14px 14px 0;border-radius:0 14px 14px 0}.form-search .input-prepend .search-query{-webkit-border-radius:0 14px 14px 0;-moz-border-radius:0 14px 14px 0;border-radius:0 14px 14px 0}.form-search .input-prepend .btn{-webkit-border-radius:14px 0 0 14px;-moz-border-radius:14px 0 0 14px;border-radius:14px 0 0 14px}.form-search input,.form-inline input,.form-horizontal input,.form-search textarea,.form-inline textarea,.form-horizontal textarea,.form-search select,.form-inline select,.form-horizontal select,.form-search .help-inline,.form-inline .help-inline,.form-horizontal .help-inline,.form-search .uneditable-input,.form-inline .uneditable-input,.form-horizontal .uneditable-input,.form-search .input-prepend,.form-inline .input-prepend,.form-horizontal .input-prepend,.form-search .input-append,.form-inline .input-append,.form-horizontal .input-append{display:inline-block;*display:inline;margin-bottom:0;vertical-align:middle;*zoom:1}.form-search .hide,.form-inline .hide,.form-horizontal .hide{display:none}.form-search label,.form-inline label,.form-search .btn-group,.form-inline .btn-group{display:inline-block}.form-search .input-append,.form-inline .input-append,.form-search .input-prepend,.form-inline .input-prepend{margin-bottom:0}.form-search .radio,.form-search .checkbox,.form-inline .radio,.form-inline .checkbox{padding-left:0;margin-bottom:0;vertical-align:middle}.form-search .radio input[type="radio"],.form-search .checkbox input[type="checkbox"],.form-inline .radio input[type="radio"],.form-inline .checkbox input[type="checkbox"]{float:left;margin-right:3px;margin-left:0}.control-group{margin-bottom:10px}legend+.control-group{margin-top:20px;-webkit-margin-top-collapse:separate}.form-horizontal .control-group{margin-bottom:20px;*zoom:1}.form-horizontal .control-group:before,.form-horizontal .control-group:after{display:table;line-height:0;content:""}.form-horizontal .control-group:after{clear:both}.form-horizontal .control-label{float:left;width:140px;padding-top:5px;text-align:right}.form-horizontal .controls{*display:inline-block;*padding-left:20px;margin-left:160px;*margin-left:0}.form-horizontal .controls:first-child{*padding-left:160px}.form-horizontal .help-block{margin-top:10px;margin-bottom:0}.form-horizontal .form-actions{padding-left:160px}table{max-width:100%;background-color:transparent;border-collapse:collapse;border-spacing:0}.table{width:100%;margin-bottom:20px}.table th,.table td{padding:8px;line-height:20px;text-align:left;vertical-align:top;border-top:1px solid #ddd}.table th{font-weight:bold}.table thead th{vertical-align:bottom}.table caption+thead tr:first-child th,.table caption+thead tr:first-child td,.table colgroup+thead tr:first-child th,.table colgroup+thead tr:first-child td,.table thead:first-child tr:first-child th,.table thead:first-child tr:first-child td{border-top:0}.table tbody+tbody{border-top:2px solid #ddd}.table-condensed th,.table-condensed td{padding:4px 5px}.table-bordered{border:1px solid #ddd;border-collapse:separate;*border-collapse:collapse;border-left:0;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.table-bordered th,.table-bordered td{border-left:1px solid #ddd}.table-bordered caption+thead tr:first-child th,.table-bordered caption+tbody tr:first-child th,.table-bordered caption+tbody tr:first-child td,.table-bordered colgroup+thead tr:first-child th,.table-bordered colgroup+tbody tr:first-child th,.table-bordered colgroup+tbody tr:first-child td,.table-bordered thead:first-child tr:first-child th,.table-bordered tbody:first-child tr:first-child th,.table-bordered tbody:first-child tr:first-child td{border-top:0}.table-bordered thead:first-child tr:first-child th:first-child,.table-bordered tbody:first-child tr:first-child td:first-child{-webkit-border-top-left-radius:4px;border-top-left-radius:4px;-moz-border-radius-topleft:4px}.table-bordered thead:first-child tr:first-child th:last-child,.table-bordered tbody:first-child tr:first-child td:last-child{-webkit-border-top-right-radius:4px;border-top-right-radius:4px;-moz-border-radius-topright:4px}.table-bordered thead:last-child tr:last-child th:first-child,.table-bordered tbody:last-child tr:last-child td:first-child,.table-bordered tfoot:last-child tr:last-child td:first-child{-webkit-border-radius:0 0 0 4px;-moz-border-radius:0 0 0 4px;border-radius:0 0 0 4px;-webkit-border-bottom-left-radius:4px;border-bottom-left-radius:4px;-moz-border-radius-bottomleft:4px}.table-bordered thead:last-child tr:last-child th:last-child,.table-bordered tbody:last-child tr:last-child td:last-child,.table-bordered tfoot:last-child tr:last-child td:last-child{-webkit-border-bottom-right-radius:4px;border-bottom-right-radius:4px;-moz-border-radius-bottomright:4px}.table-bordered caption+thead tr:first-child th:first-child,.table-bordered caption+tbody tr:first-child td:first-child,.table-bordered colgroup+thead tr:first-child th:first-child,.table-bordered colgroup+tbody tr:first-child td:first-child{-webkit-border-top-left-radius:4px;border-top-left-radius:4px;-moz-border-radius-topleft:4px}.table-bordered caption+thead tr:first-child th:last-child,.table-bordered caption+tbody tr:first-child td:last-child,.table-bordered colgroup+thead tr:first-child th:last-child,.table-bordered colgroup+tbody tr:first-child td:last-child{-webkit-border-top-right-radius:4px;border-top-right-radius:4px;-moz-border-right-topleft:4px}.table-striped tbody tr:nth-child(odd) td,.table-striped tbody tr:nth-child(odd) th{background-color:#f9f9f9}.table-hover tbody tr:hover td,.table-hover tbody tr:hover th{background-color:#f5f5f5}table [class*=span],.row-fluid table [class*=span]{display:table-cell;float:none;margin-left:0}table .span1{float:none;width:44px;margin-left:0}table .span2{float:none;width:124px;margin-left:0}table .span3{float:none;width:204px;margin-left:0}table .span4{float:none;width:284px;margin-left:0}table .span5{float:none;width:364px;margin-left:0}table .span6{float:none;width:444px;margin-left:0}table .span7{float:none;width:524px;margin-left:0}table .span8{float:none;width:604px;margin-left:0}table .span9{float:none;width:684px;margin-left:0}table .span10{float:none;width:764px;margin-left:0}table .span11{float:none;width:844px;margin-left:0}table .span12{float:none;width:924px;margin-left:0}table .span13{float:none;width:1004px;margin-left:0}table .span14{float:none;width:1084px;margin-left:0}table .span15{float:none;width:1164px;margin-left:0}table .span16{float:none;width:1244px;margin-left:0}table .span17{float:none;width:1324px;margin-left:0}table .span18{float:none;width:1404px;margin-left:0}table .span19{float:none;width:1484px;margin-left:0}table .span20{float:none;width:1564px;margin-left:0}table .span21{float:none;width:1644px;margin-left:0}table .span22{float:none;width:1724px;margin-left:0}table .span23{float:none;width:1804px;margin-left:0}table .span24{float:none;width:1884px;margin-left:0}.table tbody tr.success td{background-color:#dff0d8}.table tbody tr.error td{background-color:#f2dede}.table tbody tr.info td{background-color:#d9edf7}[class^="icon-"],[class*=" icon-"]{display:inline-block;width:14px;height:14px;margin-top:1px;*margin-right:.3em;line-height:14px;vertical-align:text-top;background-image:url("../img/glyphicons-halflings.png");background-position:14px 14px;background-repeat:no-repeat}.icon-white,.nav>.active>a>[class^="icon-"],.nav>.active>a>[class*=" icon-"],.dropdown-menu>li>a:hover>[class^="icon-"],.dropdown-menu>li>a:hover>[class*=" icon-"],.dropdown-menu>.active>a>[class^="icon-"],.dropdown-menu>.active>a>[class*=" icon-"]{background-image:url("../img/glyphicons-halflings-white.png")}.icon-glass{background-position:0 0}.icon-music{background-position:-24px 0}.icon-search{background-position:-48px 0}.icon-envelope{background-position:-72px 0}.icon-heart{background-position:-96px 0}.icon-star{background-position:-120px 0}.icon-star-empty{background-position:-144px 0}.icon-user{background-position:-168px 0}.icon-film{background-position:-192px 0}.icon-th-large{background-position:-216px 0}.icon-th{background-position:-240px 0}.icon-th-list{background-position:-264px 0}.icon-ok{background-position:-288px 0}.icon-remove{background-position:-312px 0}.icon-zoom-in{background-position:-336px 0}.icon-zoom-out{background-position:-360px 0}.icon-off{background-position:-384px 0}.icon-signal{background-position:-408px 0}.icon-cog{background-position:-432px 0}.icon-trash{background-position:-456px 0}.icon-home{background-position:0 -24px}.icon-file{background-position:-24px -24px}.icon-time{background-position:-48px -24px}.icon-road{background-position:-72px -24px}.icon-download-alt{background-position:-96px -24px}.icon-download{background-position:-120px -24px}.icon-upload{background-position:-144px -24px}.icon-inbox{background-position:-168px -24px}.icon-play-circle{background-position:-192px -24px}.icon-repeat{background-position:-216px -24px}.icon-refresh{background-position:-240px -24px}.icon-list-alt{background-position:-264px -24px}.icon-lock{background-position:-287px -24px}.icon-flag{background-position:-312px -24px}.icon-headphones{background-position:-336px -24px}.icon-volume-off{background-position:-360px -24px}.icon-volume-down{background-position:-384px -24px}.icon-volume-up{background-position:-408px -24px}.icon-qrcode{background-position:-432px -24px}.icon-barcode{background-position:-456px -24px}.icon-tag{background-position:0 -48px}.icon-tags{background-position:-25px -48px}.icon-book{background-position:-48px -48px}.icon-bookmark{background-position:-72px -48px}.icon-print{background-position:-96px -48px}.icon-camera{background-position:-120px -48px}.icon-font{background-position:-144px -48px}.icon-bold{background-position:-167px -48px}.icon-italic{background-position:-192px -48px}.icon-text-height{background-position:-216px -48px}.icon-text-width{background-position:-240px -48px}.icon-align-left{background-position:-264px -48px}.icon-align-center{background-position:-288px -48px}.icon-align-right{background-position:-312px -48px}.icon-align-justify{background-position:-336px -48px}.icon-list{background-position:-360px -48px}.icon-indent-left{background-position:-384px -48px}.icon-indent-right{background-position:-408px -48px}.icon-facetime-video{background-position:-432px -48px}.icon-picture{background-position:-456px -48px}.icon-pencil{background-position:0 -72px}.icon-map-marker{background-position:-24px -72px}.icon-adjust{background-position:-48px -72px}.icon-tint{background-position:-72px -72px}.icon-edit{background-position:-96px -72px}.icon-share{background-position:-120px -72px}.icon-check{background-position:-144px -72px}.icon-move{background-position:-168px -72px}.icon-step-backward{background-position:-192px -72px}.icon-fast-backward{background-position:-216px -72px}.icon-backward{background-position:-240px -72px}.icon-play{background-position:-264px -72px}.icon-pause{background-position:-288px -72px}.icon-stop{background-position:-312px -72px}.icon-forward{background-position:-336px -72px}.icon-fast-forward{background-position:-360px -72px}.icon-step-forward{background-position:-384px -72px}.icon-eject{background-position:-408px -72px}.icon-chevron-left{background-position:-432px -72px}.icon-chevron-right{background-position:-456px -72px}.icon-plus-sign{background-position:0 -96px}.icon-minus-sign{background-position:-24px -96px}.icon-remove-sign{background-position:-48px -96px}.icon-ok-sign{background-position:-72px -96px}.icon-question-sign{background-position:-96px -96px}.icon-info-sign{background-position:-120px -96px}.icon-screenshot{background-position:-144px -96px}.icon-remove-circle{background-position:-168px -96px}.icon-ok-circle{background-position:-192px -96px}.icon-ban-circle{background-position:-216px -96px}.icon-arrow-left{background-position:-240px -96px}.icon-arrow-right{background-position:-264px -96px}.icon-arrow-up{background-position:-289px -96px}.icon-arrow-down{background-position:-312px -96px}.icon-share-alt{background-position:-336px -96px}.icon-resize-full{background-position:-360px -96px}.icon-resize-small{background-position:-384px -96px}.icon-plus{background-position:-408px -96px}.icon-minus{background-position:-433px -96px}.icon-asterisk{background-position:-456px -96px}.icon-exclamation-sign{background-position:0 -120px}.icon-gift{background-position:-24px -120px}.icon-leaf{background-position:-48px -120px}.icon-fire{background-position:-72px -120px}.icon-eye-open{background-position:-96px -120px}.icon-eye-close{background-position:-120px -120px}.icon-warning-sign{background-position:-144px -120px}.icon-plane{background-position:-168px -120px}.icon-calendar{background-position:-192px -120px}.icon-random{width:16px;background-position:-216px -120px}.icon-comment{background-position:-240px -120px}.icon-magnet{background-position:-264px -120px}.icon-chevron-up{background-position:-288px -120px}.icon-chevron-down{background-position:-313px -119px}.icon-retweet{background-position:-336px -120px}.icon-shopping-cart{background-position:-360px -120px}.icon-folder-close{background-position:-384px -120px}.icon-folder-open{width:16px;background-position:-408px -120px}.icon-resize-vertical{background-position:-432px -119px}.icon-resize-horizontal{background-position:-456px -118px}.icon-hdd{background-position:0 -144px}.icon-bullhorn{background-position:-24px -144px}.icon-bell{background-position:-48px -144px}.icon-certificate{background-position:-72px -144px}.icon-thumbs-up{background-position:-96px -144px}.icon-thumbs-down{background-position:-120px -144px}.icon-hand-right{background-position:-144px -144px}.icon-hand-left{background-position:-168px -144px}.icon-hand-up{background-position:-192px -144px}.icon-hand-down{background-position:-216px -144px}.icon-circle-arrow-right{background-position:-240px -144px}.icon-circle-arrow-left{background-position:-264px -144px}.icon-circle-arrow-up{background-position:-288px -144px}.icon-circle-arrow-down{background-position:-312px -144px}.icon-globe{background-position:-336px -144px}.icon-wrench{background-position:-360px -144px}.icon-tasks{background-position:-384px -144px}.icon-filter{background-position:-408px -144px}.icon-briefcase{background-position:-432px -144px}.icon-fullscreen{background-position:-456px -144px}.dropup,.dropdown{position:relative}.dropdown-toggle{*margin-bottom:-3px}.dropdown-toggle:active,.open .dropdown-toggle{outline:0}.caret{display:inline-block;width:0;height:0;vertical-align:top;border-top:4px solid #000;border-right:4px solid transparent;border-left:4px solid transparent;content:""}.dropdown .caret{margin-top:8px;margin-left:2px}.dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:160px;padding:5px 0;margin:2px 0 0;list-style:none;background-color:#fff;border:1px solid #ccc;border:1px solid rgba(0,0,0,0.2);*border-right-width:2px;*border-bottom-width:2px;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px;-webkit-box-shadow:0 5px 10px rgba(0,0,0,0.2);-moz-box-shadow:0 5px 10px rgba(0,0,0,0.2);box-shadow:0 5px 10px rgba(0,0,0,0.2);-webkit-background-clip:padding-box;-moz-background-clip:padding;background-clip:padding-box}.dropdown-menu.pull-right{right:0;left:auto}.dropdown-menu .divider{*width:100%;height:1px;margin:9px 1px;*margin:-5px 0 5px;overflow:hidden;background-color:#e5e5e5;border-bottom:1px solid #fff}.dropdown-menu a{display:block;padding:3px 20px;clear:both;font-weight:normal;line-height:20px;color:#333;white-space:nowrap}.dropdown-menu li>a:hover,.dropdown-menu li>a:focus,.dropdown-submenu:hover>a{color:#fff;text-decoration:none;background-color:#0088cc;background-color:#0088cc;background-image:-moz-linear-gradient(top,#0088cc,#0087b3);background-image:-webkit-gradient(linear,0 0,0 100%,from(#0088cc),to(#0087b3));background-image:-webkit-linear-gradient(top,#0088cc,#0087b3);background-image:-o-linear-gradient(top,#0088cc,#0087b3);background-image:linear-gradient(to bottom,#0088cc,#0087b3);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff0088cc',endColorstr='#ff0087b3',GradientType=0)}.dropdown-menu .active>a,.dropdown-menu .active>a:hover{color:#fff;text-decoration:none;background-color:#0088cc;background-color:#0081c2;background-image:linear-gradient(to bottom,#0088cc,#0087b3);background-image:-moz-linear-gradient(top,#0088cc,#0087b3);background-image:-webkit-gradient(linear,0 0,0 100%,from(#0088cc),to(#0087b3));background-image:-webkit-linear-gradient(top,#0088cc,#0087b3);background-image:-o-linear-gradient(top,#0088cc,#0087b3);background-repeat:repeat-x;outline:0;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff0088cc',endColorstr='#ff0087b3',GradientType=0)}.dropdown-menu .disabled>a,.dropdown-menu .disabled>a:hover{color:#999}.dropdown-menu .disabled>a:hover{text-decoration:none;cursor:default;background-color:transparent}.open{*z-index:1000}.open>.dropdown-menu{display:block}.pull-right>.dropdown-menu{right:0;left:auto}.dropup .caret,.navbar-fixed-bottom .dropdown .caret{border-top:0;border-bottom:4px solid #000;content:"\2191"}.dropup .dropdown-menu,.navbar-fixed-bottom .dropdown .dropdown-menu{top:auto;bottom:100%;margin-bottom:1px}.dropdown-submenu{position:relative}.dropdown-submenu>.dropdown-menu{top:0;left:100%;margin-top:-6px;margin-left:-1px;-webkit-border-radius:0 6px 6px 6px;-moz-border-radius:0 6px 6px 6px;border-radius:0 6px 6px 6px}.dropdown-submenu:hover .dropdown-menu{display:block}.dropdown-submenu>a:after{display:block;float:right;width:0;height:0;margin-top:5px;margin-right:-10px;border-color:transparent;border-left-color:#ccc;border-style:solid;border-width:5px 0 5px 5px;content:" "}.dropdown-submenu:hover>a:after{border-left-color:#fff}.dropdown .dropdown-menu .nav-header{padding-right:20px;padding-left:20px}.typeahead{margin-top:2px;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.well{min-height:20px;padding:19px;margin-bottom:20px;background-color:#f5f5f5;border:1px solid #e3e3e3;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.05);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.05);box-shadow:inset 0 1px 1px rgba(0,0,0,0.05)}.well blockquote{border-color:#ddd;border-color:rgba(0,0,0,0.15)}.well-large{padding:24px;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px}.well-small{padding:9px;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}.fade{opacity:0;-webkit-transition:opacity .15s linear;-moz-transition:opacity .15s linear;-o-transition:opacity .15s linear;transition:opacity .15s linear}.fade.in{opacity:1}.collapse{position:relative;height:0;overflow:hidden;overflow:visible \9;-webkit-transition:height .35s ease;-moz-transition:height .35s ease;-o-transition:height .35s ease;transition:height .35s ease}.collapse.in{height:auto}.close{float:right;font-size:20px;font-weight:bold;line-height:20px;color:#000;text-shadow:0 1px 0 #fff;opacity:.2;filter:alpha(opacity=20)}.close:hover{color:#000;text-decoration:none;cursor:pointer;opacity:.4;filter:alpha(opacity=40)}button.close{padding:0;cursor:pointer;background:transparent;border:0;-webkit-appearance:none}.btn{display:inline-block;*display:inline;padding:4px 14px;margin-bottom:0;*margin-left:.3em;font-size:14px;line-height:20px;*line-height:20px;color:#333;text-align:center;text-shadow:0 1px 1px rgba(255,255,255,0.75);vertical-align:middle;cursor:pointer;background-color:#f5f5f5;*background-color:#e6e6e6;background-image:-webkit-gradient(linear,0 0,0 100%,from(#fff),to(#e6e6e6));background-image:-webkit-linear-gradient(top,#fff,#e6e6e6);background-image:-o-linear-gradient(top,#fff,#e6e6e6);background-image:linear-gradient(to bottom,#fff,#e6e6e6);background-image:-moz-linear-gradient(top,#fff,#e6e6e6);background-repeat:repeat-x;border:1px solid #bbb;*border:0;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);border-color:#e6e6e6 #e6e6e6 #bfbfbf;border-bottom-color:#a2a2a2;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ffffffff',endColorstr='#ffe6e6e6',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false);*zoom:1;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);box-shadow:inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05)}.btn:hover,.btn:active,.btn.active,.btn.disabled,.btn[disabled]{color:#333;background-color:#e6e6e6;*background-color:#d9d9d9}.btn:active,.btn.active{background-color:#ccc \9}.btn:first-child{*margin-left:0}.btn:hover{color:#333;text-decoration:none;background-color:#e6e6e6;*background-color:#d9d9d9;background-position:0 -15px;-webkit-transition:background-position .1s linear;-moz-transition:background-position .1s linear;-o-transition:background-position .1s linear;transition:background-position .1s linear}.btn:focus{outline:thin dotted #333;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}.btn.active,.btn:active{background-color:#e6e6e6;background-color:#d9d9d9 \9;background-image:none;outline:0;-webkit-box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05)}.btn.disabled,.btn[disabled]{cursor:default;background-color:#e6e6e6;background-image:none;opacity:.65;filter:alpha(opacity=65);-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none}.btn-large{padding:9px 14px;font-size:16px;line-height:normal;-webkit-border-radius:5px;-moz-border-radius:5px;border-radius:5px}.btn-large [class^="icon-"]{margin-top:2px}.btn-small{padding:3px 9px;font-size:12px;line-height:18px}.btn-small [class^="icon-"]{margin-top:0}.btn-mini{padding:2px 6px;font-size:11px;line-height:16px}.btn-block{display:block;width:100%;padding-right:0;padding-left:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.btn-block+.btn-block{margin-top:5px}.btn-primary.active,.btn-warning.active,.btn-danger.active,.btn-success.active,.btn-info.active,.btn-inverse.active{color:rgba(255,255,255,0.75)}.btn{border-color:#c5c5c5;border-color:rgba(0,0,0,0.15) rgba(0,0,0,0.15) rgba(0,0,0,0.25)}.btn-primary{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#006dcc;*background-color:#04c;background-image:-webkit-gradient(linear,0 0,0 100%,from(#08c),to(#04c));background-image:-webkit-linear-gradient(top,#08c,#04c);background-image:-o-linear-gradient(top,#08c,#04c);background-image:linear-gradient(to bottom,#08c,#04c);background-image:-moz-linear-gradient(top,#08c,#04c);background-repeat:repeat-x;border-color:#04c #04c #002a80;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff0088cc',endColorstr='#ff0044cc',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-primary:hover,.btn-primary:active,.btn-primary.active,.btn-primary.disabled,.btn-primary[disabled]{color:#fff;background-color:#04c;*background-color:#003bb3}.btn-primary:active,.btn-primary.active{background-color:#039 \9}.btn-warning{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#faa732;*background-color:#f89406;background-image:-webkit-gradient(linear,0 0,0 100%,from(#fbb450),to(#f89406));background-image:-webkit-linear-gradient(top,#fbb450,#f89406);background-image:-o-linear-gradient(top,#fbb450,#f89406);background-image:linear-gradient(to bottom,#fbb450,#f89406);background-image:-moz-linear-gradient(top,#fbb450,#f89406);background-repeat:repeat-x;border-color:#f89406 #f89406 #ad6704;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#fffbb450',endColorstr='#fff89406',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-warning:hover,.btn-warning:active,.btn-warning.active,.btn-warning.disabled,.btn-warning[disabled]{color:#fff;background-color:#f89406;*background-color:#df8505}.btn-warning:active,.btn-warning.active{background-color:#c67605 \9}.btn-danger{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#da4f49;*background-color:#bd362f;background-image:-webkit-gradient(linear,0 0,0 100%,from(#ee5f5b),to(#bd362f));background-image:-webkit-linear-gradient(top,#ee5f5b,#bd362f);background-image:-o-linear-gradient(top,#ee5f5b,#bd362f);background-image:linear-gradient(to bottom,#ee5f5b,#bd362f);background-image:-moz-linear-gradient(top,#ee5f5b,#bd362f);background-repeat:repeat-x;border-color:#bd362f #bd362f #802420;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ffee5f5b',endColorstr='#ffbd362f',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-danger:hover,.btn-danger:active,.btn-danger.active,.btn-danger.disabled,.btn-danger[disabled]{color:#fff;background-color:#bd362f;*background-color:#a9302a}.btn-danger:active,.btn-danger.active{background-color:#942a25 \9}.btn-success{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#5bb75b;*background-color:#51a351;background-image:-webkit-gradient(linear,0 0,0 100%,from(#62c462),to(#51a351));background-image:-webkit-linear-gradient(top,#62c462,#51a351);background-image:-o-linear-gradient(top,#62c462,#51a351);background-image:linear-gradient(to bottom,#62c462,#51a351);background-image:-moz-linear-gradient(top,#62c462,#51a351);background-repeat:repeat-x;border-color:#51a351 #51a351 #387038;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff62c462',endColorstr='#ff51a351',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-success:hover,.btn-success:active,.btn-success.active,.btn-success.disabled,.btn-success[disabled]{color:#fff;background-color:#51a351;*background-color:#499249}.btn-success:active,.btn-success.active{background-color:#408140 \9}.btn-info{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#49afcd;*background-color:#2f96b4;background-image:-webkit-gradient(linear,0 0,0 100%,from(#5bc0de),to(#2f96b4));background-image:-webkit-linear-gradient(top,#5bc0de,#2f96b4);background-image:-o-linear-gradient(top,#5bc0de,#2f96b4);background-image:linear-gradient(to bottom,#5bc0de,#2f96b4);background-image:-moz-linear-gradient(top,#5bc0de,#2f96b4);background-repeat:repeat-x;border-color:#2f96b4 #2f96b4 #1f6377;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff5bc0de',endColorstr='#ff2f96b4',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-info:hover,.btn-info:active,.btn-info.active,.btn-info.disabled,.btn-info[disabled]{color:#fff;background-color:#2f96b4;*background-color:#2a85a0}.btn-info:active,.btn-info.active{background-color:#24748c \9}.btn-inverse{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#363636;*background-color:#222;background-image:-webkit-gradient(linear,0 0,0 100%,from(#444),to(#222));background-image:-webkit-linear-gradient(top,#444,#222);background-image:-o-linear-gradient(top,#444,#222);background-image:linear-gradient(to bottom,#444,#222);background-image:-moz-linear-gradient(top,#444,#222);background-repeat:repeat-x;border-color:#222 #222 #000;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff444444',endColorstr='#ff222222',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-inverse:hover,.btn-inverse:active,.btn-inverse.active,.btn-inverse.disabled,.btn-inverse[disabled]{color:#fff;background-color:#222;*background-color:#151515}.btn-inverse:active,.btn-inverse.active{background-color:#080808 \9}button.btn,input[type="submit"].btn{*padding-top:3px;*padding-bottom:3px}button.btn::-moz-focus-inner,input[type="submit"].btn::-moz-focus-inner{padding:0;border:0}button.btn.btn-large,input[type="submit"].btn.btn-large{*padding-top:7px;*padding-bottom:7px}button.btn.btn-small,input[type="submit"].btn.btn-small{*padding-top:3px;*padding-bottom:3px}button.btn.btn-mini,input[type="submit"].btn.btn-mini{*padding-top:1px;*padding-bottom:1px}.btn-link,.btn-link:active{background-color:transparent;background-image:none;-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none}.btn-link{color:#08c;cursor:pointer;border-color:transparent;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.btn-link:hover{color:#005580;text-decoration:underline;background-color:transparent}.btn-group{position:relative;*margin-left:.3em;font-size:0;white-space:nowrap}.btn-group:first-child{*margin-left:0}.btn-group+.btn-group{margin-left:5px}.btn-toolbar{margin-top:10px;margin-bottom:10px;font-size:0}.btn-toolbar .btn-group{display:inline-block;*display:inline;*zoom:1}.btn-toolbar .btn+.btn,.btn-toolbar .btn-group+.btn,.btn-toolbar .btn+.btn-group{margin-left:5px}.btn-group>.btn{position:relative;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.btn-group>.btn+.btn{margin-left:-1px}.btn-group>.btn,.btn-group>.dropdown-menu{font-size:14px}.btn-group>.btn-mini{font-size:11px}.btn-group>.btn-small{font-size:12px}.btn-group>.btn-large{font-size:16px}.btn-group>.btn:first-child{margin-left:0;-webkit-border-bottom-left-radius:4px;border-bottom-left-radius:4px;-webkit-border-top-left-radius:4px;border-top-left-radius:4px;-moz-border-radius-bottomleft:4px;-moz-border-radius-topleft:4px}.btn-group>.btn:last-child,.btn-group>.dropdown-toggle{-webkit-border-top-right-radius:4px;border-top-right-radius:4px;-webkit-border-bottom-right-radius:4px;border-bottom-right-radius:4px;-moz-border-radius-topright:4px;-moz-border-radius-bottomright:4px}.btn-group>.btn.large:first-child{margin-left:0;-webkit-border-bottom-left-radius:6px;border-bottom-left-radius:6px;-webkit-border-top-left-radius:6px;border-top-left-radius:6px;-moz-border-radius-bottomleft:6px;-moz-border-radius-topleft:6px}.btn-group>.btn.large:last-child,.btn-group>.large.dropdown-toggle{-webkit-border-top-right-radius:6px;border-top-right-radius:6px;-webkit-border-bottom-right-radius:6px;border-bottom-right-radius:6px;-moz-border-radius-topright:6px;-moz-border-radius-bottomright:6px}.btn-group>.btn:hover,.btn-group>.btn:focus,.btn-group>.btn:active,.btn-group>.btn.active{z-index:2}.btn-group .dropdown-toggle:active,.btn-group.open .dropdown-toggle{outline:0}.btn-group>.btn+.dropdown-toggle{*padding-top:5px;padding-right:8px;*padding-bottom:5px;padding-left:8px;-webkit-box-shadow:inset 1px 0 0 rgba(255,255,255,0.125),inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:inset 1px 0 0 rgba(255,255,255,0.125),inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);box-shadow:inset 1px 0 0 rgba(255,255,255,0.125),inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05)}.btn-group>.btn-mini+.dropdown-toggle{*padding-top:2px;padding-right:5px;*padding-bottom:2px;padding-left:5px}.btn-group>.btn-small+.dropdown-toggle{*padding-top:5px;*padding-bottom:4px}.btn-group>.btn-large+.dropdown-toggle{*padding-top:7px;padding-right:12px;*padding-bottom:7px;padding-left:12px}.btn-group.open .dropdown-toggle{background-image:none;-webkit-box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05)}.btn-group.open .btn.dropdown-toggle{background-color:#e6e6e6}.btn-group.open .btn-primary.dropdown-toggle{background-color:#04c}.btn-group.open .btn-warning.dropdown-toggle{background-color:#f89406}.btn-group.open .btn-danger.dropdown-toggle{background-color:#bd362f}.btn-group.open .btn-success.dropdown-toggle{background-color:#51a351}.btn-group.open .btn-info.dropdown-toggle{background-color:#2f96b4}.btn-group.open .btn-inverse.dropdown-toggle{background-color:#222}.btn .caret{margin-top:8px;margin-left:0}.btn-mini .caret,.btn-small .caret,.btn-large .caret{margin-top:6px}.btn-large .caret{border-top-width:5px;border-right-width:5px;border-left-width:5px}.dropup .btn-large .caret{border-top:0;border-bottom:5px solid #000}.btn-primary .caret,.btn-warning .caret,.btn-danger .caret,.btn-info .caret,.btn-success .caret,.btn-inverse .caret{border-top-color:#fff;border-bottom-color:#fff}.btn-group-vertical{display:inline-block;*display:inline;*zoom:1}.btn-group-vertical .btn{display:block;float:none;width:100%;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.btn-group-vertical .btn+.btn{margin-top:-1px;margin-left:0}.btn-group-vertical .btn:first-child{-webkit-border-radius:4px 4px 0 0;-moz-border-radius:4px 4px 0 0;border-radius:4px 4px 0 0}.btn-group-vertical .btn:last-child{-webkit-border-radius:0 0 4px 4px;-moz-border-radius:0 0 4px 4px;border-radius:0 0 4px 4px}.btn-group-vertical .btn-large:first-child{-webkit-border-radius:6px 6px 0 0;-moz-border-radius:6px 6px 0 0;border-radius:6px 6px 0 0}.btn-group-vertical .btn-large:last-child{-webkit-border-radius:0 0 6px 6px;-moz-border-radius:0 0 6px 6px;border-radius:0 0 6px 6px}.alert{padding:8px 35px 8px 14px;margin-bottom:20px;color:#c09853;text-shadow:0 1px 0 rgba(255,255,255,0.5);background-color:#fcf8e3;border:1px solid #fbeed5;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.alert h4{margin:0}.alert .close{position:relative;top:-2px;right:-21px;line-height:20px}.alert-success{color:#468847;background-color:#dff0d8;border-color:#d6e9c6}.alert-danger,.alert-error{color:#b94a48;background-color:#f2dede;border-color:#eed3d7}.alert-info{color:#3a87ad;background-color:#d9edf7;border-color:#bce8f1}.alert-block{padding-top:14px;padding-bottom:14px}.alert-block>p,.alert-block>ul{margin-bottom:0}.alert-block p+p{margin-top:5px}.nav{margin-bottom:20px;margin-left:0;list-style:none}.nav>li>a{display:block}.nav>li>a:hover{text-decoration:none;background-color:#eee}.nav>.pull-right{float:right}.nav-header{display:block;padding:3px 15px;font-size:11px;font-weight:bold;line-height:20px;color:#999;text-shadow:0 1px 0 rgba(255,255,255,0.5);text-transform:uppercase}.nav li+.nav-header{margin-top:9px}.nav-list{padding-right:15px;padding-left:15px;margin-bottom:0}.nav-list>li>a,.nav-list .nav-header{margin-right:-15px;margin-left:-15px;text-shadow:0 1px 0 rgba(255,255,255,0.5)}.nav-list>li>a{padding:3px 15px}.nav-list>.active>a,.nav-list>.active>a:hover{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.2);background-color:#08c}.nav-list [class^="icon-"]{margin-right:2px}.nav-list .divider{*width:100%;height:1px;margin:9px 1px;*margin:-5px 0 5px;overflow:hidden;background-color:#e5e5e5;border-bottom:1px solid #fff}.nav-tabs,.nav-pills{*zoom:1}.nav-tabs:before,.nav-pills:before,.nav-tabs:after,.nav-pills:after{display:table;line-height:0;content:""}.nav-tabs:after,.nav-pills:after{clear:both}.nav-tabs>li,.nav-pills>li{float:left}.nav-tabs>li>a,.nav-pills>li>a{padding-right:12px;padding-left:12px;margin-right:2px;line-height:14px}.nav-tabs{border-bottom:1px solid #ddd}.nav-tabs>li{margin-bottom:-1px}.nav-tabs>li>a{padding-top:8px;padding-bottom:8px;line-height:20px;border:1px solid transparent;-webkit-border-radius:4px 4px 0 0;-moz-border-radius:4px 4px 0 0;border-radius:4px 4px 0 0}.nav-tabs>li>a:hover{border-color:#eee #eee #ddd}.nav-tabs>.active>a,.nav-tabs>.active>a:hover{color:#555;cursor:default;background-color:#fff;border:1px solid #ddd;border-bottom-color:transparent}.nav-pills>li>a{padding-top:8px;padding-bottom:8px;margin-top:2px;margin-bottom:2px;-webkit-border-radius:5px;-moz-border-radius:5px;border-radius:5px}.nav-pills>.active>a,.nav-pills>.active>a:hover{color:#fff;background-color:#08c}.nav-stacked>li{float:none}.nav-stacked>li>a{margin-right:0}.nav-tabs.nav-stacked{border-bottom:0}.nav-tabs.nav-stacked>li>a{border:1px solid #ddd;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.nav-tabs.nav-stacked>li:first-child>a{-webkit-border-top-right-radius:4px;border-top-right-radius:4px;-webkit-border-top-left-radius:4px;border-top-left-radius:4px;-moz-border-radius-topright:4px;-moz-border-radius-topleft:4px}.nav-tabs.nav-stacked>li:last-child>a{-webkit-border-bottom-right-radius:4px;border-bottom-right-radius:4px;-webkit-border-bottom-left-radius:4px;border-bottom-left-radius:4px;-moz-border-radius-bottomright:4px;-moz-border-radius-bottomleft:4px}.nav-tabs.nav-stacked>li>a:hover{z-index:2;border-color:#ddd}.nav-pills.nav-stacked>li>a{margin-bottom:3px}.nav-pills.nav-stacked>li:last-child>a{margin-bottom:1px}.nav-tabs .dropdown-menu{-webkit-border-radius:0 0 6px 6px;-moz-border-radius:0 0 6px 6px;border-radius:0 0 6px 6px}.nav-pills .dropdown-menu{-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px}.nav .dropdown-toggle .caret{margin-top:6px;border-top-color:#08c;border-bottom-color:#08c}.nav .dropdown-toggle:hover .caret{border-top-color:#005580;border-bottom-color:#005580}.nav-tabs .dropdown-toggle .caret{margin-top:8px}.nav .active .dropdown-toggle .caret{border-top-color:#fff;border-bottom-color:#fff}.nav-tabs .active .dropdown-toggle .caret{border-top-color:#555;border-bottom-color:#555}.nav>.dropdown.active>a:hover{cursor:pointer}.nav-tabs .open .dropdown-toggle,.nav-pills .open .dropdown-toggle,.nav>li.dropdown.open.active>a:hover{color:#fff;background-color:#999;border-color:#999}.nav li.dropdown.open .caret,.nav li.dropdown.open.active .caret,.nav li.dropdown.open a:hover .caret{border-top-color:#fff;border-bottom-color:#fff;opacity:1;filter:alpha(opacity=100)}.tabs-stacked .open>a:hover{border-color:#999}.tabbable{*zoom:1}.tabbable:before,.tabbable:after{display:table;line-height:0;content:""}.tabbable:after{clear:both}.tab-content{overflow:auto}.tabs-below>.nav-tabs,.tabs-right>.nav-tabs,.tabs-left>.nav-tabs{border-bottom:0}.tab-content>.tab-pane,.pill-content>.pill-pane{display:none}.tab-content>.active,.pill-content>.active{display:block}.tabs-below>.nav-tabs{border-top:1px solid #ddd}.tabs-below>.nav-tabs>li{margin-top:-1px;margin-bottom:0}.tabs-below>.nav-tabs>li>a{-webkit-border-radius:0 0 4px 4px;-moz-border-radius:0 0 4px 4px;border-radius:0 0 4px 4px}.tabs-below>.nav-tabs>li>a:hover{border-top-color:#ddd;border-bottom-color:transparent}.tabs-below>.nav-tabs>.active>a,.tabs-below>.nav-tabs>.active>a:hover{border-color:transparent #ddd #ddd #ddd}.tabs-left>.nav-tabs>li,.tabs-right>.nav-tabs>li{float:none}.tabs-left>.nav-tabs>li>a,.tabs-right>.nav-tabs>li>a{min-width:74px;margin-right:0;margin-bottom:3px}.tabs-left>.nav-tabs{float:left;margin-right:19px;border-right:1px solid #ddd}.tabs-left>.nav-tabs>li>a{margin-right:-1px;-webkit-border-radius:4px 0 0 4px;-moz-border-radius:4px 0 0 4px;border-radius:4px 0 0 4px}.tabs-left>.nav-tabs>li>a:hover{border-color:#eee #ddd #eee #eee}.tabs-left>.nav-tabs .active>a,.tabs-left>.nav-tabs .active>a:hover{border-color:#ddd transparent #ddd #ddd;*border-right-color:#fff}.tabs-right>.nav-tabs{float:right;margin-left:19px;border-left:1px solid #ddd}.tabs-right>.nav-tabs>li>a{margin-left:-1px;-webkit-border-radius:0 4px 4px 0;-moz-border-radius:0 4px 4px 0;border-radius:0 4px 4px 0}.tabs-right>.nav-tabs>li>a:hover{border-color:#eee #eee #eee #ddd}.tabs-right>.nav-tabs .active>a,.tabs-right>.nav-tabs .active>a:hover{border-color:#ddd #ddd #ddd transparent;*border-left-color:#fff}.nav>.disabled>a{color:#999}.nav>.disabled>a:hover{text-decoration:none;cursor:default;background-color:transparent}.navbar{*position:relative;*z-index:2;margin-bottom:20px;overflow:visible;color:#555}.navbar-inner{min-height:40px;padding-right:20px;padding-left:20px;background-color:#fafafa;background-image:-moz-linear-gradient(top,#fff,#e2f1f8);background-image:-webkit-gradient(linear,0 0,0 100%,from(#fff),to(#e2f1f8));background-image:-webkit-linear-gradient(top,#fff,#e2f1f8);background-image:-o-linear-gradient(top,#fff,#e2f1f8);background-image:linear-gradient(to bottom,#fff,#e2f1f8);background-repeat:repeat-x;border:1px solid #d4d4d4;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ffffffff',endColorstr='#ffe2f1f8',GradientType=0);-webkit-box-shadow:0 1px 4px rgba(0,0,0,0.065);-moz-box-shadow:0 1px 4px rgba(0,0,0,0.065);box-shadow:0 1px 4px rgba(0,0,0,0.065)}.navbar .container{width:auto}.nav-collapse.collapse{height:auto}.navbar .brand{display:block;float:left;padding:10px 20px 10px;margin-left:-20px;font-size:20px;font-weight:200;color:#555;text-shadow:0 1px 0 #fff}.navbar .brand:hover{text-decoration:none}.navbar-text{margin-bottom:0;line-height:40px}.navbar-link{color:#555}.navbar-link:hover{color:#333}.navbar .divider-vertical{height:40px;margin:0 9px;border-right:1px solid #fff;border-left:1px solid #f2f2f2}.navbar .btn,.navbar .btn-group{margin-top:6px}.navbar .btn-group .btn{margin:0}.navbar-form{margin-bottom:0;*zoom:1}.navbar-form:before,.navbar-form:after{display:table;line-height:0;content:""}.navbar-form:after{clear:both}.navbar-form input,.navbar-form select,.navbar-form .radio,.navbar-form .checkbox{margin-top:5px}.navbar-form input,.navbar-form select,.navbar-form .btn{display:inline-block;margin-bottom:0}.navbar-form input[type="image"],.navbar-form input[type="checkbox"],.navbar-form input[type="radio"]{margin-top:3px}.navbar-form .input-append,.navbar-form .input-prepend{margin-top:6px;white-space:nowrap}.navbar-form .input-append input,.navbar-form .input-prepend input{margin-top:0}.navbar-search{position:relative;float:left;margin-top:5px;margin-bottom:0}.navbar-search .search-query{padding:4px 14px;margin-bottom:0;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:13px;font-weight:normal;line-height:1;-webkit-border-radius:15px;-moz-border-radius:15px;border-radius:15px}.navbar-static-top{position:static;width:100%;margin-bottom:0}.navbar-static-top .navbar-inner{-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.navbar-fixed-top,.navbar-fixed-bottom{position:fixed;right:0;left:0;z-index:1030;margin-bottom:0}.navbar-fixed-top .navbar-inner,.navbar-fixed-bottom .navbar-inner,.navbar-static-top .navbar-inner{border:0}.navbar-fixed-top .navbar-inner,.navbar-fixed-bottom .navbar-inner{padding-right:0;padding-left:0;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.navbar-static-top .container,.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:940px}.navbar-fixed-top{top:0}.navbar-fixed-top .navbar-inner,.navbar-static-top .navbar-inner{-webkit-box-shadow:inset 0 -1px 0 rgba(0,0,0,0.1),0 1px 10px rgba(0,0,0,0.1);-moz-box-shadow:inset 0 -1px 0 rgba(0,0,0,0.1),0 1px 10px rgba(0,0,0,0.1);box-shadow:inset 0 -1px 0 rgba(0,0,0,0.1),0 1px 10px rgba(0,0,0,0.1)}.navbar-fixed-bottom{bottom:0}.navbar-fixed-bottom .navbar-inner{-webkit-box-shadow:inset 0 1px 0 rgba(0,0,0,0.1),0 -1px 10px rgba(0,0,0,0.1);-moz-box-shadow:inset 0 1px 0 rgba(0,0,0,0.1),0 -1px 10px rgba(0,0,0,0.1);box-shadow:inset 0 1px 0 rgba(0,0,0,0.1),0 -1px 10px rgba(0,0,0,0.1)}.navbar .nav{position:relative;left:0;display:block;float:left;margin:0 10px 0 0}.navbar .nav.pull-right{float:right}.navbar .nav>li{float:left}.navbar .nav>li>a{float:none;padding:10px 15px 10px;color:#555;text-decoration:none;text-shadow:0 1px 0 #fff}.navbar .nav .dropdown-toggle .caret{margin-top:8px}.navbar .nav>li>a:focus,.navbar .nav>li>a:hover{color:#333;text-decoration:none;background-color:transparent}.navbar .nav>.active>a,.navbar .nav>.active>a:hover,.navbar .nav>.active>a:focus{color:#555;text-decoration:none;background-color:#e5e5e5;-webkit-box-shadow:inset 0 3px 8px rgba(0,0,0,0.125);-moz-box-shadow:inset 0 3px 8px rgba(0,0,0,0.125);box-shadow:inset 0 3px 8px rgba(0,0,0,0.125)}.navbar .btn-navbar{display:none;float:right;padding:7px 10px;margin-right:5px;margin-left:5px;color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#ededed;*background-color:#e5e5e5;background-image:-webkit-gradient(linear,0 0,0 100%,from(#f2f2f2),to(#e5e5e5));background-image:-webkit-linear-gradient(top,#f2f2f2,#e5e5e5);background-image:-o-linear-gradient(top,#f2f2f2,#e5e5e5);background-image:linear-gradient(to bottom,#f2f2f2,#e5e5e5);background-image:-moz-linear-gradient(top,#f2f2f2,#e5e5e5);background-repeat:repeat-x;border-color:#e5e5e5 #e5e5e5 #bfbfbf;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#fff2f2f2',endColorstr='#ffe5e5e5',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false);-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.075);-moz-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.075);box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.075)}.navbar .btn-navbar:hover,.navbar .btn-navbar:active,.navbar .btn-navbar.active,.navbar .btn-navbar.disabled,.navbar .btn-navbar[disabled]{color:#fff;background-color:#e5e5e5;*background-color:#d9d9d9}.navbar .btn-navbar:active,.navbar .btn-navbar.active{background-color:#ccc \9}.navbar .btn-navbar .icon-bar{display:block;width:18px;height:2px;background-color:#f5f5f5;-webkit-border-radius:1px;-moz-border-radius:1px;border-radius:1px;-webkit-box-shadow:0 1px 0 rgba(0,0,0,0.25);-moz-box-shadow:0 1px 0 rgba(0,0,0,0.25);box-shadow:0 1px 0 rgba(0,0,0,0.25)}.btn-navbar .icon-bar+.icon-bar{margin-top:3px}.navbar .nav>li>.dropdown-menu:before{position:absolute;top:-7px;left:9px;display:inline-block;border-right:7px solid transparent;border-bottom:7px solid #ccc;border-left:7px solid transparent;border-bottom-color:rgba(0,0,0,0.2);content:''}.navbar .nav>li>.dropdown-menu:after{position:absolute;top:-6px;left:10px;display:inline-block;border-right:6px solid transparent;border-bottom:6px solid #fff;border-left:6px solid transparent;content:''}.navbar-fixed-bottom .nav>li>.dropdown-menu:before{top:auto;bottom:-7px;border-top:7px solid #ccc;border-bottom:0;border-top-color:rgba(0,0,0,0.2)}.navbar-fixed-bottom .nav>li>.dropdown-menu:after{top:auto;bottom:-6px;border-top:6px solid #fff;border-bottom:0}.navbar .nav li.dropdown.open>.dropdown-toggle,.navbar .nav li.dropdown.active>.dropdown-toggle,.navbar .nav li.dropdown.open.active>.dropdown-toggle{color:#555;background-color:#e5e5e5}.navbar .nav li.dropdown>.dropdown-toggle .caret{border-top-color:#555;border-bottom-color:#555}.navbar .nav li.dropdown.open>.dropdown-toggle .caret,.navbar .nav li.dropdown.active>.dropdown-toggle .caret,.navbar .nav li.dropdown.open.active>.dropdown-toggle .caret{border-top-color:#555;border-bottom-color:#555}.navbar .pull-right>li>.dropdown-menu,.navbar .nav>li>.dropdown-menu.pull-right{right:0;left:auto}.navbar .pull-right>li>.dropdown-menu:before,.navbar .nav>li>.dropdown-menu.pull-right:before{right:12px;left:auto}.navbar .pull-right>li>.dropdown-menu:after,.navbar .nav>li>.dropdown-menu.pull-right:after{right:13px;left:auto}.navbar .pull-right>li>.dropdown-menu .dropdown-menu,.navbar .nav>li>.dropdown-menu.pull-right .dropdown-menu{right:100%;left:auto;margin-right:-1px;margin-left:0;-webkit-border-radius:6px 0 6px 6px;-moz-border-radius:6px 0 6px 6px;border-radius:6px 0 6px 6px}.navbar-inverse{color:#999}.navbar-inverse .navbar-inner{background-color:#1b1b1b;background-image:-moz-linear-gradient(top,#222,#111);background-image:-webkit-gradient(linear,0 0,0 100%,from(#222),to(#111));background-image:-webkit-linear-gradient(top,#222,#111);background-image:-o-linear-gradient(top,#222,#111);background-image:linear-gradient(to bottom,#222,#111);background-repeat:repeat-x;border-color:#252525;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff222222',endColorstr='#ff111111',GradientType=0)}.navbar-inverse .brand,.navbar-inverse .nav>li>a{color:#999;text-shadow:0 -1px 0 rgba(0,0,0,0.25)}.navbar-inverse .brand:hover,.navbar-inverse .nav>li>a:hover{color:#fff}.navbar-inverse .nav>li>a:focus,.navbar-inverse .nav>li>a:hover{color:#fff;background-color:transparent}.navbar-inverse .nav .active>a,.navbar-inverse .nav .active>a:hover,.navbar-inverse .nav .active>a:focus{color:#fff;background-color:#111}.navbar-inverse .navbar-link{color:#999}.navbar-inverse .navbar-link:hover{color:#fff}.navbar-inverse .divider-vertical{border-right-color:#222;border-left-color:#111}.navbar-inverse .nav li.dropdown.open>.dropdown-toggle,.navbar-inverse .nav li.dropdown.active>.dropdown-toggle,.navbar-inverse .nav li.dropdown.open.active>.dropdown-toggle{color:#fff;background-color:#111}.navbar-inverse .nav li.dropdown>.dropdown-toggle .caret{border-top-color:#999;border-bottom-color:#999}.navbar-inverse .nav li.dropdown.open>.dropdown-toggle .caret,.navbar-inverse .nav li.dropdown.active>.dropdown-toggle .caret,.navbar-inverse .nav li.dropdown.open.active>.dropdown-toggle .caret{border-top-color:#fff;border-bottom-color:#fff}.navbar-inverse .navbar-search .search-query{color:#fff;background-color:#515151;border-color:#111;-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1),0 1px 0 rgba(255,255,255,0.15);-moz-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1),0 1px 0 rgba(255,255,255,0.15);box-shadow:inset 0 1px 2px rgba(0,0,0,0.1),0 1px 0 rgba(255,255,255,0.15);-webkit-transition:none;-moz-transition:none;-o-transition:none;transition:none}.navbar-inverse .navbar-search .search-query:-moz-placeholder{color:#ccc}.navbar-inverse .navbar-search .search-query:-ms-input-placeholder{color:#ccc}.navbar-inverse .navbar-search .search-query::-webkit-input-placeholder{color:#ccc}.navbar-inverse .navbar-search .search-query:focus,.navbar-inverse .navbar-search .search-query.focused{padding:5px 15px;color:#333;text-shadow:0 1px 0 #fff;background-color:#fff;border:0;outline:0;-webkit-box-shadow:0 0 3px rgba(0,0,0,0.15);-moz-box-shadow:0 0 3px rgba(0,0,0,0.15);box-shadow:0 0 3px rgba(0,0,0,0.15)}.navbar-inverse .btn-navbar{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#0e0e0e;*background-color:#040404;background-image:-webkit-gradient(linear,0 0,0 100%,from(#151515),to(#040404));background-image:-webkit-linear-gradient(top,#151515,#040404);background-image:-o-linear-gradient(top,#151515,#040404);background-image:linear-gradient(to bottom,#151515,#040404);background-image:-moz-linear-gradient(top,#151515,#040404);background-repeat:repeat-x;border-color:#040404 #040404 #000;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff151515',endColorstr='#ff040404',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.navbar-inverse .btn-navbar:hover,.navbar-inverse .btn-navbar:active,.navbar-inverse .btn-navbar.active,.navbar-inverse .btn-navbar.disabled,.navbar-inverse .btn-navbar[disabled]{color:#fff;background-color:#040404;*background-color:#000}.navbar-inverse .btn-navbar:active,.navbar-inverse .btn-navbar.active{background-color:#000 \9}.breadcrumb{padding:8px 15px;margin:0 0 20px;list-style:none;background-color:#f5f5f5;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.breadcrumb li{display:inline-block;*display:inline;text-shadow:0 1px 0 #fff;*zoom:1}.breadcrumb .divider{padding:0 5px;color:#ccc}.breadcrumb .active{color:#999}.pagination{height:40px;margin:20px 0}.pagination ul{display:inline-block;*display:inline;margin-bottom:0;margin-left:0;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px;*zoom:1;-webkit-box-shadow:0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:0 1px 2px rgba(0,0,0,0.05);box-shadow:0 1px 2px rgba(0,0,0,0.05)}.pagination li{display:inline}.pagination a,.pagination span{float:left;padding:0 14px;line-height:38px;text-decoration:none;background-color:#fff;border:1px solid #ddd;border-left-width:0}.pagination a:hover,.pagination .active a,.pagination .active span{background-color:#f5f5f5}.pagination .active a,.pagination .active span{color:#999;cursor:default}.pagination .disabled span,.pagination .disabled a,.pagination .disabled a:hover{color:#999;cursor:default;background-color:transparent}.pagination li:first-child a,.pagination li:first-child span{border-left-width:1px;-webkit-border-radius:3px 0 0 3px;-moz-border-radius:3px 0 0 3px;border-radius:3px 0 0 3px}.pagination li:last-child a,.pagination li:last-child span{-webkit-border-radius:0 3px 3px 0;-moz-border-radius:0 3px 3px 0;border-radius:0 3px 3px 0}.pagination-centered{text-align:center}.pagination-right{text-align:right}.pager{margin:20px 0;text-align:center;list-style:none;*zoom:1}.pager:before,.pager:after{display:table;line-height:0;content:""}.pager:after{clear:both}.pager li{display:inline}.pager a{display:inline-block;padding:5px 14px;background-color:#fff;border:1px solid #ddd;-webkit-border-radius:15px;-moz-border-radius:15px;border-radius:15px}.pager a:hover{text-decoration:none;background-color:#f5f5f5}.pager .next a{float:right}.pager .previous a{float:left}.pager .disabled a,.pager .disabled a:hover{color:#999;cursor:default;background-color:#fff}.modal-open .dropdown-menu{z-index:2050}.modal-open .dropdown.open{*z-index:2050}.modal-open .popover{z-index:2060}.modal-open .tooltip{z-index:2080}.modal-backdrop{position:fixed;top:0;right:0;bottom:0;left:0;z-index:1040;background-color:#000}.modal-backdrop.fade{opacity:0}.modal-backdrop,.modal-backdrop.fade.in{opacity:.8;filter:alpha(opacity=80)}.modal{position:fixed;top:50%;left:50%;z-index:1050;width:560px;margin:-250px 0 0 -280px;overflow:auto;background-color:#fff;border:1px solid #999;border:1px solid rgba(0,0,0,0.3);*border:1px solid #999;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px;-webkit-box-shadow:0 3px 7px rgba(0,0,0,0.3);-moz-box-shadow:0 3px 7px rgba(0,0,0,0.3);box-shadow:0 3px 7px rgba(0,0,0,0.3);-webkit-background-clip:padding-box;-moz-background-clip:padding-box;background-clip:padding-box}.modal.fade{top:-25%;-webkit-transition:opacity .3s linear,top .3s ease-out;-moz-transition:opacity .3s linear,top .3s ease-out;-o-transition:opacity .3s linear,top .3s ease-out;transition:opacity .3s linear,top .3s ease-out}.modal.fade.in{top:50%}.modal-header{padding:9px 15px;border-bottom:1px solid #eee}.modal-header .close{margin-top:2px}.modal-header h3{margin:0;line-height:30px}.modal-body{max-height:400px;padding:15px;overflow-y:auto}.modal-form{margin-bottom:0}.modal-footer{padding:14px 15px 15px;margin-bottom:0;text-align:right;background-color:#f5f5f5;border-top:1px solid #ddd;-webkit-border-radius:0 0 6px 6px;-moz-border-radius:0 0 6px 6px;border-radius:0 0 6px 6px;*zoom:1;-webkit-box-shadow:inset 0 1px 0 #fff;-moz-box-shadow:inset 0 1px 0 #fff;box-shadow:inset 0 1px 0 #fff}.modal-footer:before,.modal-footer:after{display:table;line-height:0;content:""}.modal-footer:after{clear:both}.modal-footer .btn+.btn{margin-bottom:0;margin-left:5px}.modal-footer .btn-group .btn+.btn{margin-left:-1px}.tooltip{position:absolute;z-index:1030;display:block;padding:5px;font-size:11px;opacity:0;filter:alpha(opacity=0);visibility:visible}.tooltip.in{opacity:.8;filter:alpha(opacity=80)}.tooltip.top{margin-top:-3px}.tooltip.right{margin-left:3px}.tooltip.bottom{margin-top:3px}.tooltip.left{margin-left:-3px}.tooltip-inner{max-width:200px;padding:3px 8px;color:#fff;text-align:center;text-decoration:none;background-color:#000;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.tooltip-arrow{position:absolute;width:0;height:0;border-color:transparent;border-style:solid}.tooltip.top .tooltip-arrow{bottom:0;left:50%;margin-left:-5px;border-top-color:#000;border-width:5px 5px 0}.tooltip.right .tooltip-arrow{top:50%;left:0;margin-top:-5px;border-right-color:#000;border-width:5px 5px 5px 0}.tooltip.left .tooltip-arrow{top:50%;right:0;margin-top:-5px;border-left-color:#000;border-width:5px 0 5px 5px}.tooltip.bottom .tooltip-arrow{top:0;left:50%;margin-left:-5px;border-bottom-color:#000;border-width:0 5px 5px}.popover{position:absolute;top:0;left:0;z-index:1010;display:none;width:236px;padding:1px;background-color:#fff;border:1px solid #ccc;border:1px solid rgba(0,0,0,0.2);-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px;-webkit-box-shadow:0 5px 10px rgba(0,0,0,0.2);-moz-box-shadow:0 5px 10px rgba(0,0,0,0.2);box-shadow:0 5px 10px rgba(0,0,0,0.2);-webkit-background-clip:padding-box;-moz-background-clip:padding;background-clip:padding-box}.popover.top{margin-bottom:10px}.popover.right{margin-left:10px}.popover.bottom{margin-top:10px}.popover.left{margin-right:10px}.popover-title{padding:8px 14px;margin:0;font-size:14px;font-weight:normal;line-height:18px;background-color:#f7f7f7;border-bottom:1px solid #ebebeb;-webkit-border-radius:5px 5px 0 0;-moz-border-radius:5px 5px 0 0;border-radius:5px 5px 0 0}.popover-content{padding:9px 14px}.popover-content p,.popover-content ul,.popover-content ol{margin-bottom:0}.popover .arrow,.popover .arrow:after{position:absolute;display:inline-block;width:0;height:0;border-color:transparent;border-style:solid}.popover .arrow:after{z-index:-1;content:""}.popover.top .arrow{bottom:-10px;left:50%;margin-left:-10px;border-top-color:#fff;border-width:10px 10px 0}.popover.top .arrow:after{bottom:-1px;left:-11px;border-top-color:rgba(0,0,0,0.25);border-width:11px 11px 0}.popover.right .arrow{top:50%;left:-10px;margin-top:-10px;border-right-color:#fff;border-width:10px 10px 10px 0}.popover.right .arrow:after{bottom:-11px;left:-1px;border-right-color:rgba(0,0,0,0.25);border-width:11px 11px 11px 0}.popover.bottom .arrow{top:-10px;left:50%;margin-left:-10px;border-bottom-color:#fff;border-width:0 10px 10px}.popover.bottom .arrow:after{top:-1px;left:-11px;border-bottom-color:rgba(0,0,0,0.25);border-width:0 11px 11px}.popover.left .arrow{top:50%;right:-10px;margin-top:-10px;border-left-color:#fff;border-width:10px 0 10px 10px}.popover.left .arrow:after{right:-1px;bottom:-11px;border-left-color:rgba(0,0,0,0.25);border-width:11px 0 11px 11px}.thumbnails{margin-left:-20px;list-style:none;*zoom:1}.thumbnails:before,.thumbnails:after{display:table;line-height:0;content:""}.thumbnails:after{clear:both}.row-fluid .thumbnails{margin-left:0}.thumbnails>li{float:left;margin-bottom:20px;margin-left:20px}.thumbnail{display:block;padding:4px;line-height:20px;border:1px solid #ddd;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;-webkit-box-shadow:0 1px 3px rgba(0,0,0,0.055);-moz-box-shadow:0 1px 3px rgba(0,0,0,0.055);box-shadow:0 1px 3px rgba(0,0,0,0.055);-webkit-transition:all .2s ease-in-out;-moz-transition:all .2s ease-in-out;-o-transition:all .2s ease-in-out;transition:all .2s ease-in-out}a.thumbnail:hover{border-color:#08c;-webkit-box-shadow:0 1px 4px rgba(0,105,214,0.25);-moz-box-shadow:0 1px 4px rgba(0,105,214,0.25);box-shadow:0 1px 4px rgba(0,105,214,0.25)}.thumbnail>img{display:block;max-width:100%;margin-right:auto;margin-left:auto}.thumbnail .caption{padding:9px;color:#555}.label,.badge{font-size:11.844px;font-weight:bold;line-height:14px;color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);white-space:nowrap;vertical-align:baseline;background-color:#999}.label{padding:1px 4px 2px;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}.badge{padding:1px 9px 2px;-webkit-border-radius:9px;-moz-border-radius:9px;border-radius:9px}a.label:hover,a.badge:hover{color:#fff;text-decoration:none;cursor:pointer}.label-important,.badge-important{background-color:#b94a48}.label-important[href],.badge-important[href]{background-color:#953b39}.label-warning,.badge-warning{background-color:#f89406}.label-warning[href],.badge-warning[href]{background-color:#c67605}.label-success,.badge-success{background-color:#468847}.label-success[href],.badge-success[href]{background-color:#356635}.label-info,.badge-info{background-color:#3a87ad}.label-info[href],.badge-info[href]{background-color:#2d6987}.label-inverse,.badge-inverse{background-color:#333}.label-inverse[href],.badge-inverse[href]{background-color:#1a1a1a}.btn .label,.btn .badge{position:relative;top:-1px}.btn-mini .label,.btn-mini .badge{top:0}@-webkit-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@-moz-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@-ms-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@-o-keyframes progress-bar-stripes{from{background-position:0 0}to{background-position:40px 0}}@keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}.progress{height:20px;margin-bottom:20px;overflow:hidden;background-color:#f7f7f7;background-image:-moz-linear-gradient(top,#f5f5f5,#f9f9f9);background-image:-webkit-gradient(linear,0 0,0 100%,from(#f5f5f5),to(#f9f9f9));background-image:-webkit-linear-gradient(top,#f5f5f5,#f9f9f9);background-image:-o-linear-gradient(top,#f5f5f5,#f9f9f9);background-image:linear-gradient(to bottom,#f5f5f5,#f9f9f9);background-repeat:repeat-x;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#fff5f5f5',endColorstr='#fff9f9f9',GradientType=0);-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1);-moz-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1);box-shadow:inset 0 1px 2px rgba(0,0,0,0.1)}.progress .bar{float:left;width:0;height:100%;font-size:12px;color:#fff;text-align:center;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#0e90d2;background-image:-moz-linear-gradient(top,#149bdf,#0480be);background-image:-webkit-gradient(linear,0 0,0 100%,from(#149bdf),to(#0480be));background-image:-webkit-linear-gradient(top,#149bdf,#0480be);background-image:-o-linear-gradient(top,#149bdf,#0480be);background-image:linear-gradient(to bottom,#149bdf,#0480be);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff149bdf',endColorstr='#ff0480be',GradientType=0);-webkit-box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);-moz-box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;-webkit-transition:width .6s ease;-moz-transition:width .6s ease;-o-transition:width .6s ease;transition:width .6s ease}.progress .bar+.bar{-webkit-box-shadow:inset 1px 0 0 rgba(0,0,0,0.15),inset 0 -1px 0 rgba(0,0,0,0.15);-moz-box-shadow:inset 1px 0 0 rgba(0,0,0,0.15),inset 0 -1px 0 rgba(0,0,0,0.15);box-shadow:inset 1px 0 0 rgba(0,0,0,0.15),inset 0 -1px 0 rgba(0,0,0,0.15)}.progress-striped .bar{background-color:#149bdf;background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);-webkit-background-size:40px 40px;-moz-background-size:40px 40px;-o-background-size:40px 40px;background-size:40px 40px}.progress.active .bar{-webkit-animation:progress-bar-stripes 2s linear infinite;-moz-animation:progress-bar-stripes 2s linear infinite;-ms-animation:progress-bar-stripes 2s linear infinite;-o-animation:progress-bar-stripes 2s linear infinite;animation:progress-bar-stripes 2s linear infinite}.progress-danger .bar,.progress .bar-danger{background-color:#dd514c;background-image:-moz-linear-gradient(top,#ee5f5b,#c43c35);background-image:-webkit-gradient(linear,0 0,0 100%,from(#ee5f5b),to(#c43c35));background-image:-webkit-linear-gradient(top,#ee5f5b,#c43c35);background-image:-o-linear-gradient(top,#ee5f5b,#c43c35);background-image:linear-gradient(to bottom,#ee5f5b,#c43c35);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ffee5f5b',endColorstr='#ffc43c35',GradientType=0)}.progress-danger.progress-striped .bar,.progress-striped .bar-danger{background-color:#ee5f5b;background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent)}.progress-success .bar,.progress .bar-success{background-color:#5eb95e;background-image:-moz-linear-gradient(top,#62c462,#57a957);background-image:-webkit-gradient(linear,0 0,0 100%,from(#62c462),to(#57a957));background-image:-webkit-linear-gradient(top,#62c462,#57a957);background-image:-o-linear-gradient(top,#62c462,#57a957);background-image:linear-gradient(to bottom,#62c462,#57a957);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff62c462',endColorstr='#ff57a957',GradientType=0)}.progress-success.progress-striped .bar,.progress-striped .bar-success{background-color:#62c462;background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent)}.progress-info .bar,.progress .bar-info{background-color:#4bb1cf;background-image:-moz-linear-gradient(top,#5bc0de,#339bb9);background-image:-webkit-gradient(linear,0 0,0 100%,from(#5bc0de),to(#339bb9));background-image:-webkit-linear-gradient(top,#5bc0de,#339bb9);background-image:-o-linear-gradient(top,#5bc0de,#339bb9);background-image:linear-gradient(to bottom,#5bc0de,#339bb9);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff5bc0de',endColorstr='#ff339bb9',GradientType=0)}.progress-info.progress-striped .bar,.progress-striped .bar-info{background-color:#5bc0de;background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent)}.progress-warning .bar,.progress .bar-warning{background-color:#faa732;background-image:-moz-linear-gradient(top,#fbb450,#f89406);background-image:-webkit-gradient(linear,0 0,0 100%,from(#fbb450),to(#f89406));background-image:-webkit-linear-gradient(top,#fbb450,#f89406);background-image:-o-linear-gradient(top,#fbb450,#f89406);background-image:linear-gradient(to bottom,#fbb450,#f89406);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#fffbb450',endColorstr='#fff89406',GradientType=0)}.progress-warning.progress-striped .bar,.progress-striped .bar-warning{background-color:#fbb450;background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent)}.accordion{margin-bottom:20px}.accordion-group{margin-bottom:2px;border:1px solid #e5e5e5;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.accordion-heading{border-bottom:0}.accordion-heading .accordion-toggle{display:block;padding:8px 15px}.accordion-toggle{cursor:pointer}.accordion-inner{padding:9px 15px;border-top:1px solid #e5e5e5}.carousel{position:relative;margin-bottom:20px;line-height:1}.carousel-inner{position:relative;width:100%;overflow:hidden}.carousel .item{position:relative;display:none;-webkit-transition:.6s ease-in-out left;-moz-transition:.6s ease-in-out left;-o-transition:.6s ease-in-out left;transition:.6s ease-in-out left}.carousel .item>img{display:block;line-height:1}.carousel .active,.carousel .next,.carousel .prev{display:block}.carousel .active{left:0}.carousel .next,.carousel .prev{position:absolute;top:0;width:100%}.carousel .next{left:100%}.carousel .prev{left:-100%}.carousel .next.left,.carousel .prev.right{left:0}.carousel .active.left{left:-100%}.carousel .active.right{left:100%}.carousel-control{position:absolute;top:40%;left:15px;width:40px;height:40px;margin-top:-20px;font-size:60px;font-weight:100;line-height:30px;color:#fff;text-align:center;background:#222;border:3px solid #fff;-webkit-border-radius:23px;-moz-border-radius:23px;border-radius:23px;opacity:.5;filter:alpha(opacity=50)}.carousel-control.right{right:15px;left:auto}.carousel-control:hover{color:#fff;text-decoration:none;opacity:.9;filter:alpha(opacity=90)}.carousel-caption{position:absolute;right:0;bottom:0;left:0;padding:15px;background:#333;background:rgba(0,0,0,0.75)}.carousel-caption h4,.carousel-caption p{line-height:20px;color:#fff}.carousel-caption h4{margin:0 0 5px}.carousel-caption p{margin-bottom:0}.hero-unit{padding:60px;margin-bottom:30px;background-color:#eee;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px}.hero-unit h1{margin-bottom:0;font-size:60px;line-height:1;letter-spacing:-1px;color:inherit}.hero-unit p{font-size:18px;font-weight:200;line-height:30px;color:inherit}.pull-right{float:right}.pull-left{float:left}.hide{display:none}.show{display:block}.invisible{visibility:hidden}.affix{position:fixed}
+ * Bootstrap v4.4.1 (https://getbootstrap.com/)
+ * Copyright 2011-2019 The Bootstrap Authors
+ * Copyright 2011-2019 Twitter, Inc.
+ * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
+ */:root{--blue:#007bff;--indigo:#6610f2;--purple:#6f42c1;--pink:#e83e8c;--red:#dc3545;--orange:#fd7e14;--yellow:#ffc107;--green:#28a745;--teal:#20c997;--cyan:#17a2b8;--white:#fff;--gray:#6c757d;--gray-dark:#343a40;--primary:#007bff;--secondary:#6c757d;--success:#28a745;--info:#17a2b8;--warning:#ffc107;--danger:#dc3545;--light:#f8f9fa;--dark:#343a40;--breakpoint-xs:0;--breakpoint-sm:576px;--breakpoint-md:768px;--breakpoint-lg:992px;--breakpoint-xl:1200px;--font-family-sans-serif:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";--font-family-monospace:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace}*,::after,::before{box-sizing:border-box}html{font-family:sans-serif;line-height:1.15;-webkit-text-size-adjust:100%;-webkit-tap-highlight-color:transparent}article,aside,figcaption,figure,footer,header,hgroup,main,nav,section{display:block}body{margin:0;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-size:1rem;font-weight:400;line-height:1.5;color:#212529;text-align:left;background-color:#fff}[tabindex="-1"]:focus:not(:focus-visible){outline:0!important}hr{box-sizing:content-box;height:0;overflow:visible}h1,h2,h3,h4,h5,h6{margin-top:0;margin-bottom:.5rem}p{margin-top:0;margin-bottom:1rem}abbr[data-original-title],abbr[title]{text-decoration:underline;-webkit-text-decoration:underline dotted;text-decoration:underline dotted;cursor:help;border-bottom:0;-webkit-text-decoration-skip-ink:none;text-decoration-skip-ink:none}address{margin-bottom:1rem;font-style:normal;line-height:inherit}dl,ol,ul{margin-top:0;margin-bottom:1rem}ol ol,ol ul,ul ol,ul ul{margin-bottom:0}dt{font-weight:700}dd{margin-bottom:.5rem;margin-left:0}blockquote{margin:0 0 1rem}b,strong{font-weight:bolder}small{font-size:80%}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}a{color:#007bff;text-decoration:none;background-color:transparent}a:hover{color:#0056b3;text-decoration:underline}a:not([href]){color:inherit;text-decoration:none}a:not([href]):hover{color:inherit;text-decoration:none}code,kbd,pre,samp{font-family:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;font-size:1em}pre{margin-top:0;margin-bottom:1rem;overflow:auto}figure{margin:0 0 1rem}img{vertical-align:middle;border-style:none}svg{overflow:hidden;vertical-align:middle}table{border-collapse:collapse}caption{padding-top:.75rem;padding-bottom:.75rem;color:#6c757d;text-align:left;caption-side:bottom}th{text-align:inherit}label{display:inline-block;margin-bottom:.5rem}button{border-radius:0}button:focus{outline:1px dotted;outline:5px auto -webkit-focus-ring-color}button,input,optgroup,select,textarea{margin:0;font-family:inherit;font-size:inherit;line-height:inherit}button,input{overflow:visible}button,select{text-transform:none}select{word-wrap:normal}[type=button],[type=reset],[type=submit],button{-webkit-appearance:button}[type=button]:not(:disabled),[type=reset]:not(:disabled),[type=submit]:not(:disabled),button:not(:disabled){cursor:pointer}[type=button]::-moz-focus-inner,[type=reset]::-moz-focus-inner,[type=submit]::-moz-focus-inner,button::-moz-focus-inner{padding:0;border-style:none}input[type=checkbox],input[type=radio]{box-sizing:border-box;padding:0}input[type=date],input[type=datetime-local],input[type=month],input[type=time]{-webkit-appearance:listbox}textarea{overflow:auto;resize:vertical}fieldset{min-width:0;padding:0;margin:0;border:0}legend{display:block;width:100%;max-width:100%;padding:0;margin-bottom:.5rem;font-size:1.5rem;line-height:inherit;color:inherit;white-space:normal}progress{vertical-align:baseline}[type=number]::-webkit-inner-spin-button,[type=number]::-webkit-outer-spin-button{height:auto}[type=search]{outline-offset:-2px;-webkit-appearance:none}[type=search]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{font:inherit;-webkit-appearance:button}output{display:inline-block}summary{display:list-item;cursor:pointer}template{display:none}[hidden]{display:none!important}.h1,.h2,.h3,.h4,.h5,.h6,h1,h2,h3,h4,h5,h6{margin-bottom:.5rem;font-weight:500;line-height:1.2}.h1,h1{font-size:2.5rem}.h2,h2{font-size:2rem}.h3,h3{font-size:1.75rem}.h4,h4{font-size:1.5rem}.h5,h5{font-size:1.25rem}.h6,h6{font-size:1rem}.lead{font-size:1.25rem;font-weight:300}.display-1{font-size:6rem;font-weight:300;line-height:1.2}.display-2{font-size:5.5rem;font-weight:300;line-height:1.2}.display-3{font-size:4.5rem;font-weight:300;line-height:1.2}.display-4{font-size:3.5rem;font-weight:300;line-height:1.2}hr{margin-top:1rem;margin-bottom:1rem;border:0;border-top:1px solid rgba(0,0,0,.1)}.small,small{font-size:80%;font-weight:400}.mark,mark{padding:.2em;background-color:#fcf8e3}.list-unstyled{padding-left:0;list-style:none}.list-inline{padding-left:0;list-style:none}.list-inline-item{display:inline-block}.list-inline-item:not(:last-child){margin-right:.5rem}.initialism{font-size:90%;text-transform:uppercase}.blockquote{margin-bottom:1rem;font-size:1.25rem}.blockquote-footer{display:block;font-size:80%;color:#6c757d}.blockquote-footer::before{content:"\2014\00A0"}.img-fluid{max-width:100%;height:auto}.img-thumbnail{padding:.25rem;background-color:#fff;border:1px solid #dee2e6;border-radius:.25rem;max-width:100%;height:auto}.figure{display:inline-block}.figure-img{margin-bottom:.5rem;line-height:1}.figure-caption{font-size:90%;color:#6c757d}code{font-size:87.5%;color:#e83e8c;word-wrap:break-word}a>code{color:inherit}kbd{padding:.2rem .4rem;font-size:87.5%;color:#fff;background-color:#212529;border-radius:.2rem}kbd kbd{padding:0;font-size:100%;font-weight:700}pre{display:block;font-size:87.5%;color:#212529}pre code{font-size:inherit;color:inherit;word-break:normal}.pre-scrollable{max-height:340px;overflow-y:scroll}.container{width:100%;padding-right:15px;padding-left:15px;margin-right:auto;margin-left:auto}@media (min-width:576px){.container{max-width:540px}}@media (min-width:768px){.container{max-width:720px}}@media (min-width:992px){.container{max-width:960px}}@media (min-width:1200px){.container{max-width:1140px}}.container-fluid,.container-lg,.container-md,.container-sm,.container-xl{width:100%;padding-right:15px;padding-left:15px;margin-right:auto;margin-left:auto}@media (min-width:576px){.container,.container-sm{max-width:540px}}@media (min-width:768px){.container,.container-md,.container-sm{max-width:720px}}@media (min-width:992px){.container,.container-lg,.container-md,.container-sm{max-width:960px}}@media (min-width:1200px){.container,.container-lg,.container-md,.container-sm,.container-xl{max-width:1140px}}.row{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;margin-right:-15px;margin-left:-15px}.no-gutters{margin-right:0;margin-left:0}.no-gutters>.col,.no-gutters>[class*=col-]{padding-right:0;padding-left:0}.col,.col-1,.col-10,.col-11,.col-12,.col-2,.col-3,.col-4,.col-5,.col-6,.col-7,.col-8,.col-9,.col-auto,.col-lg,.col-lg-1,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9,.col-lg-auto,.col-md,.col-md-1,.col-md-10,.col-md-11,.col-md-12,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9,.col-md-auto,.col-sm,.col-sm-1,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9,.col-sm-auto,.col-xl,.col-xl-1,.col-xl-10,.col-xl-11,.col-xl-12,.col-xl-2,.col-xl-3,.col-xl-4,.col-xl-5,.col-xl-6,.col-xl-7,.col-xl-8,.col-xl-9,.col-xl-auto{position:relative;width:100%;padding-right:15px;padding-left:15px}.col{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;max-width:100%}.row-cols-1>*{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.row-cols-2>*{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.row-cols-3>*{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.row-cols-4>*{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.row-cols-5>*{-ms-flex:0 0 20%;flex:0 0 20%;max-width:20%}.row-cols-6>*{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-first{-ms-flex-order:-1;order:-1}.order-last{-ms-flex-order:13;order:13}.order-0{-ms-flex-order:0;order:0}.order-1{-ms-flex-order:1;order:1}.order-2{-ms-flex-order:2;order:2}.order-3{-ms-flex-order:3;order:3}.order-4{-ms-flex-order:4;order:4}.order-5{-ms-flex-order:5;order:5}.order-6{-ms-flex-order:6;order:6}.order-7{-ms-flex-order:7;order:7}.order-8{-ms-flex-order:8;order:8}.order-9{-ms-flex-order:9;order:9}.order-10{-ms-flex-order:10;order:10}.order-11{-ms-flex-order:11;order:11}.order-12{-ms-flex-order:12;order:12}.offset-1{margin-left:8.333333%}.offset-2{margin-left:16.666667%}.offset-3{margin-left:25%}.offset-4{margin-left:33.333333%}.offset-5{margin-left:41.666667%}.offset-6{margin-left:50%}.offset-7{margin-left:58.333333%}.offset-8{margin-left:66.666667%}.offset-9{margin-left:75%}.offset-10{margin-left:83.333333%}.offset-11{margin-left:91.666667%}@media (min-width:576px){.col-sm{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;max-width:100%}.row-cols-sm-1>*{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.row-cols-sm-2>*{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.row-cols-sm-3>*{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.row-cols-sm-4>*{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.row-cols-sm-5>*{-ms-flex:0 0 20%;flex:0 0 20%;max-width:20%}.row-cols-sm-6>*{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-sm-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-sm-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-sm-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-sm-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-sm-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-sm-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-sm-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-sm-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-sm-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-sm-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-sm-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-sm-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-sm-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-sm-first{-ms-flex-order:-1;order:-1}.order-sm-last{-ms-flex-order:13;order:13}.order-sm-0{-ms-flex-order:0;order:0}.order-sm-1{-ms-flex-order:1;order:1}.order-sm-2{-ms-flex-order:2;order:2}.order-sm-3{-ms-flex-order:3;order:3}.order-sm-4{-ms-flex-order:4;order:4}.order-sm-5{-ms-flex-order:5;order:5}.order-sm-6{-ms-flex-order:6;order:6}.order-sm-7{-ms-flex-order:7;order:7}.order-sm-8{-ms-flex-order:8;order:8}.order-sm-9{-ms-flex-order:9;order:9}.order-sm-10{-ms-flex-order:10;order:10}.order-sm-11{-ms-flex-order:11;order:11}.order-sm-12{-ms-flex-order:12;order:12}.offset-sm-0{margin-left:0}.offset-sm-1{margin-left:8.333333%}.offset-sm-2{margin-left:16.666667%}.offset-sm-3{margin-left:25%}.offset-sm-4{margin-left:33.333333%}.offset-sm-5{margin-left:41.666667%}.offset-sm-6{margin-left:50%}.offset-sm-7{margin-left:58.333333%}.offset-sm-8{margin-left:66.666667%}.offset-sm-9{margin-left:75%}.offset-sm-10{margin-left:83.333333%}.offset-sm-11{margin-left:91.666667%}}@media (min-width:768px){.col-md{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;max-width:100%}.row-cols-md-1>*{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.row-cols-md-2>*{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.row-cols-md-3>*{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.row-cols-md-4>*{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.row-cols-md-5>*{-ms-flex:0 0 20%;flex:0 0 20%;max-width:20%}.row-cols-md-6>*{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-md-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-md-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-md-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-md-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-md-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-md-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-md-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-md-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-md-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-md-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-md-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-md-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-md-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-md-first{-ms-flex-order:-1;order:-1}.order-md-last{-ms-flex-order:13;order:13}.order-md-0{-ms-flex-order:0;order:0}.order-md-1{-ms-flex-order:1;order:1}.order-md-2{-ms-flex-order:2;order:2}.order-md-3{-ms-flex-order:3;order:3}.order-md-4{-ms-flex-order:4;order:4}.order-md-5{-ms-flex-order:5;order:5}.order-md-6{-ms-flex-order:6;order:6}.order-md-7{-ms-flex-order:7;order:7}.order-md-8{-ms-flex-order:8;order:8}.order-md-9{-ms-flex-order:9;order:9}.order-md-10{-ms-flex-order:10;order:10}.order-md-11{-ms-flex-order:11;order:11}.order-md-12{-ms-flex-order:12;order:12}.offset-md-0{margin-left:0}.offset-md-1{margin-left:8.333333%}.offset-md-2{margin-left:16.666667%}.offset-md-3{margin-left:25%}.offset-md-4{margin-left:33.333333%}.offset-md-5{margin-left:41.666667%}.offset-md-6{margin-left:50%}.offset-md-7{margin-left:58.333333%}.offset-md-8{margin-left:66.666667%}.offset-md-9{margin-left:75%}.offset-md-10{margin-left:83.333333%}.offset-md-11{margin-left:91.666667%}}@media (min-width:992px){.col-lg{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;max-width:100%}.row-cols-lg-1>*{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.row-cols-lg-2>*{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.row-cols-lg-3>*{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.row-cols-lg-4>*{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.row-cols-lg-5>*{-ms-flex:0 0 20%;flex:0 0 20%;max-width:20%}.row-cols-lg-6>*{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-lg-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-lg-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-lg-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-lg-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-lg-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-lg-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-lg-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-lg-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-lg-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-lg-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-lg-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-lg-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-lg-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-lg-first{-ms-flex-order:-1;order:-1}.order-lg-last{-ms-flex-order:13;order:13}.order-lg-0{-ms-flex-order:0;order:0}.order-lg-1{-ms-flex-order:1;order:1}.order-lg-2{-ms-flex-order:2;order:2}.order-lg-3{-ms-flex-order:3;order:3}.order-lg-4{-ms-flex-order:4;order:4}.order-lg-5{-ms-flex-order:5;order:5}.order-lg-6{-ms-flex-order:6;order:6}.order-lg-7{-ms-flex-order:7;order:7}.order-lg-8{-ms-flex-order:8;order:8}.order-lg-9{-ms-flex-order:9;order:9}.order-lg-10{-ms-flex-order:10;order:10}.order-lg-11{-ms-flex-order:11;order:11}.order-lg-12{-ms-flex-order:12;order:12}.offset-lg-0{margin-left:0}.offset-lg-1{margin-left:8.333333%}.offset-lg-2{margin-left:16.666667%}.offset-lg-3{margin-left:25%}.offset-lg-4{margin-left:33.333333%}.offset-lg-5{margin-left:41.666667%}.offset-lg-6{margin-left:50%}.offset-lg-7{margin-left:58.333333%}.offset-lg-8{margin-left:66.666667%}.offset-lg-9{margin-left:75%}.offset-lg-10{margin-left:83.333333%}.offset-lg-11{margin-left:91.666667%}}@media (min-width:1200px){.col-xl{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;max-width:100%}.row-cols-xl-1>*{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.row-cols-xl-2>*{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.row-cols-xl-3>*{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.row-cols-xl-4>*{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.row-cols-xl-5>*{-ms-flex:0 0 20%;flex:0 0 20%;max-width:20%}.row-cols-xl-6>*{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-xl-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-xl-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-xl-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-xl-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-xl-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-xl-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-xl-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-xl-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-xl-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-xl-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-xl-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-xl-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-xl-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-xl-first{-ms-flex-order:-1;order:-1}.order-xl-last{-ms-flex-order:13;order:13}.order-xl-0{-ms-flex-order:0;order:0}.order-xl-1{-ms-flex-order:1;order:1}.order-xl-2{-ms-flex-order:2;order:2}.order-xl-3{-ms-flex-order:3;order:3}.order-xl-4{-ms-flex-order:4;order:4}.order-xl-5{-ms-flex-order:5;order:5}.order-xl-6{-ms-flex-order:6;order:6}.order-xl-7{-ms-flex-order:7;order:7}.order-xl-8{-ms-flex-order:8;order:8}.order-xl-9{-ms-flex-order:9;order:9}.order-xl-10{-ms-flex-order:10;order:10}.order-xl-11{-ms-flex-order:11;order:11}.order-xl-12{-ms-flex-order:12;order:12}.offset-xl-0{margin-left:0}.offset-xl-1{margin-left:8.333333%}.offset-xl-2{margin-left:16.666667%}.offset-xl-3{margin-left:25%}.offset-xl-4{margin-left:33.333333%}.offset-xl-5{margin-left:41.666667%}.offset-xl-6{margin-left:50%}.offset-xl-7{margin-left:58.333333%}.offset-xl-8{margin-left:66.666667%}.offset-xl-9{margin-left:75%}.offset-xl-10{margin-left:83.333333%}.offset-xl-11{margin-left:91.666667%}}.table{width:100%;margin-bottom:1rem;color:#212529}.table td,.table th{padding:.75rem;vertical-align:top;border-top:1px solid #dee2e6}.table thead th{vertical-align:bottom;border-bottom:2px solid #dee2e6}.table tbody+tbody{border-top:2px solid #dee2e6}.table-sm td,.table-sm th{padding:.3rem}.table-bordered{border:1px solid #dee2e6}.table-bordered td,.table-bordered th{border:1px solid #dee2e6}.table-bordered thead td,.table-bordered thead th{border-bottom-width:2px}.table-borderless tbody+tbody,.table-borderless td,.table-borderless th,.table-borderless thead th{border:0}.table-striped tbody tr:nth-of-type(odd){background-color:rgba(0,0,0,.05)}.table-hover tbody tr:hover{color:#212529;background-color:rgba(0,0,0,.075)}.table-primary,.table-primary>td,.table-primary>th{background-color:#b8daff}.table-primary tbody+tbody,.table-primary td,.table-primary th,.table-primary thead th{border-color:#7abaff}.table-hover .table-primary:hover{background-color:#9fcdff}.table-hover .table-primary:hover>td,.table-hover .table-primary:hover>th{background-color:#9fcdff}.table-secondary,.table-secondary>td,.table-secondary>th{background-color:#d6d8db}.table-secondary tbody+tbody,.table-secondary td,.table-secondary th,.table-secondary thead th{border-color:#b3b7bb}.table-hover .table-secondary:hover{background-color:#c8cbcf}.table-hover .table-secondary:hover>td,.table-hover .table-secondary:hover>th{background-color:#c8cbcf}.table-success,.table-success>td,.table-success>th{background-color:#c3e6cb}.table-success tbody+tbody,.table-success td,.table-success th,.table-success thead th{border-color:#8fd19e}.table-hover .table-success:hover{background-color:#b1dfbb}.table-hover .table-success:hover>td,.table-hover .table-success:hover>th{background-color:#b1dfbb}.table-info,.table-info>td,.table-info>th{background-color:#bee5eb}.table-info tbody+tbody,.table-info td,.table-info th,.table-info thead th{border-color:#86cfda}.table-hover .table-info:hover{background-color:#abdde5}.table-hover .table-info:hover>td,.table-hover .table-info:hover>th{background-color:#abdde5}.table-warning,.table-warning>td,.table-warning>th{background-color:#ffeeba}.table-warning tbody+tbody,.table-warning td,.table-warning th,.table-warning thead th{border-color:#ffdf7e}.table-hover .table-warning:hover{background-color:#ffe8a1}.table-hover .table-warning:hover>td,.table-hover .table-warning:hover>th{background-color:#ffe8a1}.table-danger,.table-danger>td,.table-danger>th{background-color:#f5c6cb}.table-danger tbody+tbody,.table-danger td,.table-danger th,.table-danger thead th{border-color:#ed969e}.table-hover .table-danger:hover{background-color:#f1b0b7}.table-hover .table-danger:hover>td,.table-hover .table-danger:hover>th{background-color:#f1b0b7}.table-light,.table-light>td,.table-light>th{background-color:#fdfdfe}.table-light tbody+tbody,.table-light td,.table-light th,.table-light thead th{border-color:#fbfcfc}.table-hover .table-light:hover{background-color:#ececf6}.table-hover .table-light:hover>td,.table-hover .table-light:hover>th{background-color:#ececf6}.table-dark,.table-dark>td,.table-dark>th{background-color:#c6c8ca}.table-dark tbody+tbody,.table-dark td,.table-dark th,.table-dark thead th{border-color:#95999c}.table-hover .table-dark:hover{background-color:#b9bbbe}.table-hover .table-dark:hover>td,.table-hover .table-dark:hover>th{background-color:#b9bbbe}.table-active,.table-active>td,.table-active>th{background-color:rgba(0,0,0,.075)}.table-hover .table-active:hover{background-color:rgba(0,0,0,.075)}.table-hover .table-active:hover>td,.table-hover .table-active:hover>th{background-color:rgba(0,0,0,.075)}.table .thead-dark th{color:#fff;background-color:#343a40;border-color:#454d55}.table .thead-light th{color:#495057;background-color:#e9ecef;border-color:#dee2e6}.table-dark{color:#fff;background-color:#343a40}.table-dark td,.table-dark th,.table-dark thead th{border-color:#454d55}.table-dark.table-bordered{border:0}.table-dark.table-striped tbody tr:nth-of-type(odd){background-color:rgba(255,255,255,.05)}.table-dark.table-hover tbody tr:hover{color:#fff;background-color:rgba(255,255,255,.075)}@media (max-width:575.98px){.table-responsive-sm{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-sm>.table-bordered{border:0}}@media (max-width:767.98px){.table-responsive-md{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-md>.table-bordered{border:0}}@media (max-width:991.98px){.table-responsive-lg{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-lg>.table-bordered{border:0}}@media (max-width:1199.98px){.table-responsive-xl{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-xl>.table-bordered{border:0}}.table-responsive{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive>.table-bordered{border:0}.form-control{display:block;width:100%;height:calc(1.5em + .75rem + 2px);padding:.375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:#495057;background-color:#fff;background-clip:padding-box;border:1px solid #ced4da;border-radius:.25rem;transition:border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.form-control{transition:none}}.form-control::-ms-expand{background-color:transparent;border:0}.form-control:-moz-focusring{color:transparent;text-shadow:0 0 0 #495057}.form-control:focus{color:#495057;background-color:#fff;border-color:#80bdff;outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.form-control::-webkit-input-placeholder{color:#6c757d;opacity:1}.form-control::-moz-placeholder{color:#6c757d;opacity:1}.form-control:-ms-input-placeholder{color:#6c757d;opacity:1}.form-control::-ms-input-placeholder{color:#6c757d;opacity:1}.form-control::placeholder{color:#6c757d;opacity:1}.form-control:disabled,.form-control[readonly]{background-color:#e9ecef;opacity:1}select.form-control:focus::-ms-value{color:#495057;background-color:#fff}.form-control-file,.form-control-range{display:block;width:100%}.col-form-label{padding-top:calc(.375rem + 1px);padding-bottom:calc(.375rem + 1px);margin-bottom:0;font-size:inherit;line-height:1.5}.col-form-label-lg{padding-top:calc(.5rem + 1px);padding-bottom:calc(.5rem + 1px);font-size:1.25rem;line-height:1.5}.col-form-label-sm{padding-top:calc(.25rem + 1px);padding-bottom:calc(.25rem + 1px);font-size:.875rem;line-height:1.5}.form-control-plaintext{display:block;width:100%;padding:.375rem 0;margin-bottom:0;font-size:1rem;line-height:1.5;color:#212529;background-color:transparent;border:solid transparent;border-width:1px 0}.form-control-plaintext.form-control-lg,.form-control-plaintext.form-control-sm{padding-right:0;padding-left:0}.form-control-sm{height:calc(1.5em + .5rem + 2px);padding:.25rem .5rem;font-size:.875rem;line-height:1.5;border-radius:.2rem}.form-control-lg{height:calc(1.5em + 1rem + 2px);padding:.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:.3rem}select.form-control[multiple],select.form-control[size]{height:auto}textarea.form-control{height:auto}.form-group{margin-bottom:1rem}.form-text{display:block;margin-top:.25rem}.form-row{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;margin-right:-5px;margin-left:-5px}.form-row>.col,.form-row>[class*=col-]{padding-right:5px;padding-left:5px}.form-check{position:relative;display:block;padding-left:1.25rem}.form-check-input{position:absolute;margin-top:.3rem;margin-left:-1.25rem}.form-check-input:disabled~.form-check-label,.form-check-input[disabled]~.form-check-label{color:#6c757d}.form-check-label{margin-bottom:0}.form-check-inline{display:-ms-inline-flexbox;display:inline-flex;-ms-flex-align:center;align-items:center;padding-left:0;margin-right:.75rem}.form-check-inline .form-check-input{position:static;margin-top:0;margin-right:.3125rem;margin-left:0}.valid-feedback{display:none;width:100%;margin-top:.25rem;font-size:80%;color:#28a745}.valid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:.25rem .5rem;margin-top:.1rem;font-size:.875rem;line-height:1.5;color:#fff;background-color:rgba(40,167,69,.9);border-radius:.25rem}.is-valid~.valid-feedback,.is-valid~.valid-tooltip,.was-validated :valid~.valid-feedback,.was-validated :valid~.valid-tooltip{display:block}.form-control.is-valid,.was-validated .form-control:valid{border-color:#28a745;padding-right:calc(1.5em + .75rem);background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath fill='%2328a745' d='M2.3 6.73L.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e");background-repeat:no-repeat;background-position:right calc(.375em + .1875rem) center;background-size:calc(.75em + .375rem) calc(.75em + .375rem)}.form-control.is-valid:focus,.was-validated .form-control:valid:focus{border-color:#28a745;box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.was-validated textarea.form-control:valid,textarea.form-control.is-valid{padding-right:calc(1.5em + .75rem);background-position:top calc(.375em + .1875rem) right calc(.375em + .1875rem)}.custom-select.is-valid,.was-validated .custom-select:valid{border-color:#28a745;padding-right:calc(.75em + 2.3125rem);background:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='5' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e") no-repeat right .75rem center/8px 10px,url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath fill='%2328a745' d='M2.3 6.73L.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e") #fff no-repeat center right 1.75rem/calc(.75em + .375rem) calc(.75em + .375rem)}.custom-select.is-valid:focus,.was-validated .custom-select:valid:focus{border-color:#28a745;box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.form-check-input.is-valid~.form-check-label,.was-validated .form-check-input:valid~.form-check-label{color:#28a745}.form-check-input.is-valid~.valid-feedback,.form-check-input.is-valid~.valid-tooltip,.was-validated .form-check-input:valid~.valid-feedback,.was-validated .form-check-input:valid~.valid-tooltip{display:block}.custom-control-input.is-valid~.custom-control-label,.was-validated .custom-control-input:valid~.custom-control-label{color:#28a745}.custom-control-input.is-valid~.custom-control-label::before,.was-validated .custom-control-input:valid~.custom-control-label::before{border-color:#28a745}.custom-control-input.is-valid:checked~.custom-control-label::before,.was-validated .custom-control-input:valid:checked~.custom-control-label::before{border-color:#34ce57;background-color:#34ce57}.custom-control-input.is-valid:focus~.custom-control-label::before,.was-validated .custom-control-input:valid:focus~.custom-control-label::before{box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.custom-control-input.is-valid:focus:not(:checked)~.custom-control-label::before,.was-validated .custom-control-input:valid:focus:not(:checked)~.custom-control-label::before{border-color:#28a745}.custom-file-input.is-valid~.custom-file-label,.was-validated .custom-file-input:valid~.custom-file-label{border-color:#28a745}.custom-file-input.is-valid:focus~.custom-file-label,.was-validated .custom-file-input:valid:focus~.custom-file-label{border-color:#28a745;box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.invalid-feedback{display:none;width:100%;margin-top:.25rem;font-size:80%;color:#dc3545}.invalid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:.25rem .5rem;margin-top:.1rem;font-size:.875rem;line-height:1.5;color:#fff;background-color:rgba(220,53,69,.9);border-radius:.25rem}.is-invalid~.invalid-feedback,.is-invalid~.invalid-tooltip,.was-validated :invalid~.invalid-feedback,.was-validated :invalid~.invalid-tooltip{display:block}.form-control.is-invalid,.was-validated .form-control:invalid{border-color:#dc3545;padding-right:calc(1.5em + .75rem);background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' fill='none' stroke='%23dc3545' viewBox='0 0 12 12'%3e%3ccircle cx='6' cy='6' r='4.5'/%3e%3cpath stroke-linejoin='round' d='M5.8 3.6h.4L6 6.5z'/%3e%3ccircle cx='6' cy='8.2' r='.6' fill='%23dc3545' stroke='none'/%3e%3c/svg%3e");background-repeat:no-repeat;background-position:right calc(.375em + .1875rem) center;background-size:calc(.75em + .375rem) calc(.75em + .375rem)}.form-control.is-invalid:focus,.was-validated .form-control:invalid:focus{border-color:#dc3545;box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.was-validated textarea.form-control:invalid,textarea.form-control.is-invalid{padding-right:calc(1.5em + .75rem);background-position:top calc(.375em + .1875rem) right calc(.375em + .1875rem)}.custom-select.is-invalid,.was-validated .custom-select:invalid{border-color:#dc3545;padding-right:calc(.75em + 2.3125rem);background:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='5' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e") no-repeat right .75rem center/8px 10px,url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' fill='none' stroke='%23dc3545' viewBox='0 0 12 12'%3e%3ccircle cx='6' cy='6' r='4.5'/%3e%3cpath stroke-linejoin='round' d='M5.8 3.6h.4L6 6.5z'/%3e%3ccircle cx='6' cy='8.2' r='.6' fill='%23dc3545' stroke='none'/%3e%3c/svg%3e") #fff no-repeat center right 1.75rem/calc(.75em + .375rem) calc(.75em + .375rem)}.custom-select.is-invalid:focus,.was-validated .custom-select:invalid:focus{border-color:#dc3545;box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.form-check-input.is-invalid~.form-check-label,.was-validated .form-check-input:invalid~.form-check-label{color:#dc3545}.form-check-input.is-invalid~.invalid-feedback,.form-check-input.is-invalid~.invalid-tooltip,.was-validated .form-check-input:invalid~.invalid-feedback,.was-validated .form-check-input:invalid~.invalid-tooltip{display:block}.custom-control-input.is-invalid~.custom-control-label,.was-validated .custom-control-input:invalid~.custom-control-label{color:#dc3545}.custom-control-input.is-invalid~.custom-control-label::before,.was-validated .custom-control-input:invalid~.custom-control-label::before{border-color:#dc3545}.custom-control-input.is-invalid:checked~.custom-control-label::before,.was-validated .custom-control-input:invalid:checked~.custom-control-label::before{border-color:#e4606d;background-color:#e4606d}.custom-control-input.is-invalid:focus~.custom-control-label::before,.was-validated .custom-control-input:invalid:focus~.custom-control-label::before{box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.custom-control-input.is-invalid:focus:not(:checked)~.custom-control-label::before,.was-validated .custom-control-input:invalid:focus:not(:checked)~.custom-control-label::before{border-color:#dc3545}.custom-file-input.is-invalid~.custom-file-label,.was-validated .custom-file-input:invalid~.custom-file-label{border-color:#dc3545}.custom-file-input.is-invalid:focus~.custom-file-label,.was-validated .custom-file-input:invalid:focus~.custom-file-label{border-color:#dc3545;box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.form-inline{display:-ms-flexbox;display:flex;-ms-flex-flow:row wrap;flex-flow:row wrap;-ms-flex-align:center;align-items:center}.form-inline .form-check{width:100%}@media (min-width:576px){.form-inline label{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center;margin-bottom:0}.form-inline .form-group{display:-ms-flexbox;display:flex;-ms-flex:0 0 auto;flex:0 0 auto;-ms-flex-flow:row wrap;flex-flow:row wrap;-ms-flex-align:center;align-items:center;margin-bottom:0}.form-inline .form-control{display:inline-block;width:auto;vertical-align:middle}.form-inline .form-control-plaintext{display:inline-block}.form-inline .custom-select,.form-inline .input-group{width:auto}.form-inline .form-check{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center;width:auto;padding-left:0}.form-inline .form-check-input{position:relative;-ms-flex-negative:0;flex-shrink:0;margin-top:0;margin-right:.25rem;margin-left:0}.form-inline .custom-control{-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center}.form-inline .custom-control-label{margin-bottom:0}}.btn{display:inline-block;font-weight:400;color:#212529;text-align:center;vertical-align:middle;cursor:pointer;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;background-color:transparent;border:1px solid transparent;padding:.375rem .75rem;font-size:1rem;line-height:1.5;border-radius:.25rem;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.btn{transition:none}}.btn:hover{color:#212529;text-decoration:none}.btn.focus,.btn:focus{outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.btn.disabled,.btn:disabled{opacity:.65}a.btn.disabled,fieldset:disabled a.btn{pointer-events:none}.btn-primary{color:#fff;background-color:#007bff;border-color:#007bff}.btn-primary:hover{color:#fff;background-color:#0069d9;border-color:#0062cc}.btn-primary.focus,.btn-primary:focus{color:#fff;background-color:#0069d9;border-color:#0062cc;box-shadow:0 0 0 .2rem rgba(38,143,255,.5)}.btn-primary.disabled,.btn-primary:disabled{color:#fff;background-color:#007bff;border-color:#007bff}.btn-primary:not(:disabled):not(.disabled).active,.btn-primary:not(:disabled):not(.disabled):active,.show>.btn-primary.dropdown-toggle{color:#fff;background-color:#0062cc;border-color:#005cbf}.btn-primary:not(:disabled):not(.disabled).active:focus,.btn-primary:not(:disabled):not(.disabled):active:focus,.show>.btn-primary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(38,143,255,.5)}.btn-secondary{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-secondary:hover{color:#fff;background-color:#5a6268;border-color:#545b62}.btn-secondary.focus,.btn-secondary:focus{color:#fff;background-color:#5a6268;border-color:#545b62;box-shadow:0 0 0 .2rem rgba(130,138,145,.5)}.btn-secondary.disabled,.btn-secondary:disabled{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-secondary:not(:disabled):not(.disabled).active,.btn-secondary:not(:disabled):not(.disabled):active,.show>.btn-secondary.dropdown-toggle{color:#fff;background-color:#545b62;border-color:#4e555b}.btn-secondary:not(:disabled):not(.disabled).active:focus,.btn-secondary:not(:disabled):not(.disabled):active:focus,.show>.btn-secondary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(130,138,145,.5)}.btn-success{color:#fff;background-color:#28a745;border-color:#28a745}.btn-success:hover{color:#fff;background-color:#218838;border-color:#1e7e34}.btn-success.focus,.btn-success:focus{color:#fff;background-color:#218838;border-color:#1e7e34;box-shadow:0 0 0 .2rem rgba(72,180,97,.5)}.btn-success.disabled,.btn-success:disabled{color:#fff;background-color:#28a745;border-color:#28a745}.btn-success:not(:disabled):not(.disabled).active,.btn-success:not(:disabled):not(.disabled):active,.show>.btn-success.dropdown-toggle{color:#fff;background-color:#1e7e34;border-color:#1c7430}.btn-success:not(:disabled):not(.disabled).active:focus,.btn-success:not(:disabled):not(.disabled):active:focus,.show>.btn-success.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(72,180,97,.5)}.btn-info{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-info:hover{color:#fff;background-color:#138496;border-color:#117a8b}.btn-info.focus,.btn-info:focus{color:#fff;background-color:#138496;border-color:#117a8b;box-shadow:0 0 0 .2rem rgba(58,176,195,.5)}.btn-info.disabled,.btn-info:disabled{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-info:not(:disabled):not(.disabled).active,.btn-info:not(:disabled):not(.disabled):active,.show>.btn-info.dropdown-toggle{color:#fff;background-color:#117a8b;border-color:#10707f}.btn-info:not(:disabled):not(.disabled).active:focus,.btn-info:not(:disabled):not(.disabled):active:focus,.show>.btn-info.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(58,176,195,.5)}.btn-warning{color:#212529;background-color:#ffc107;border-color:#ffc107}.btn-warning:hover{color:#212529;background-color:#e0a800;border-color:#d39e00}.btn-warning.focus,.btn-warning:focus{color:#212529;background-color:#e0a800;border-color:#d39e00;box-shadow:0 0 0 .2rem rgba(222,170,12,.5)}.btn-warning.disabled,.btn-warning:disabled{color:#212529;background-color:#ffc107;border-color:#ffc107}.btn-warning:not(:disabled):not(.disabled).active,.btn-warning:not(:disabled):not(.disabled):active,.show>.btn-warning.dropdown-toggle{color:#212529;background-color:#d39e00;border-color:#c69500}.btn-warning:not(:disabled):not(.disabled).active:focus,.btn-warning:not(:disabled):not(.disabled):active:focus,.show>.btn-warning.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(222,170,12,.5)}.btn-danger{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-danger:hover{color:#fff;background-color:#c82333;border-color:#bd2130}.btn-danger.focus,.btn-danger:focus{color:#fff;background-color:#c82333;border-color:#bd2130;box-shadow:0 0 0 .2rem rgba(225,83,97,.5)}.btn-danger.disabled,.btn-danger:disabled{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-danger:not(:disabled):not(.disabled).active,.btn-danger:not(:disabled):not(.disabled):active,.show>.btn-danger.dropdown-toggle{color:#fff;background-color:#bd2130;border-color:#b21f2d}.btn-danger:not(:disabled):not(.disabled).active:focus,.btn-danger:not(:disabled):not(.disabled):active:focus,.show>.btn-danger.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(225,83,97,.5)}.btn-light{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-light:hover{color:#212529;background-color:#e2e6ea;border-color:#dae0e5}.btn-light.focus,.btn-light:focus{color:#212529;background-color:#e2e6ea;border-color:#dae0e5;box-shadow:0 0 0 .2rem rgba(216,217,219,.5)}.btn-light.disabled,.btn-light:disabled{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-light:not(:disabled):not(.disabled).active,.btn-light:not(:disabled):not(.disabled):active,.show>.btn-light.dropdown-toggle{color:#212529;background-color:#dae0e5;border-color:#d3d9df}.btn-light:not(:disabled):not(.disabled).active:focus,.btn-light:not(:disabled):not(.disabled):active:focus,.show>.btn-light.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(216,217,219,.5)}.btn-dark{color:#fff;background-color:#343a40;border-color:#343a40}.btn-dark:hover{color:#fff;background-color:#23272b;border-color:#1d2124}.btn-dark.focus,.btn-dark:focus{color:#fff;background-color:#23272b;border-color:#1d2124;box-shadow:0 0 0 .2rem rgba(82,88,93,.5)}.btn-dark.disabled,.btn-dark:disabled{color:#fff;background-color:#343a40;border-color:#343a40}.btn-dark:not(:disabled):not(.disabled).active,.btn-dark:not(:disabled):not(.disabled):active,.show>.btn-dark.dropdown-toggle{color:#fff;background-color:#1d2124;border-color:#171a1d}.btn-dark:not(:disabled):not(.disabled).active:focus,.btn-dark:not(:disabled):not(.disabled):active:focus,.show>.btn-dark.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(82,88,93,.5)}.btn-outline-primary{color:#007bff;border-color:#007bff}.btn-outline-primary:hover{color:#fff;background-color:#007bff;border-color:#007bff}.btn-outline-primary.focus,.btn-outline-primary:focus{box-shadow:0 0 0 .2rem rgba(0,123,255,.5)}.btn-outline-primary.disabled,.btn-outline-primary:disabled{color:#007bff;background-color:transparent}.btn-outline-primary:not(:disabled):not(.disabled).active,.btn-outline-primary:not(:disabled):not(.disabled):active,.show>.btn-outline-primary.dropdown-toggle{color:#fff;background-color:#007bff;border-color:#007bff}.btn-outline-primary:not(:disabled):not(.disabled).active:focus,.btn-outline-primary:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-primary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(0,123,255,.5)}.btn-outline-secondary{color:#6c757d;border-color:#6c757d}.btn-outline-secondary:hover{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-outline-secondary.focus,.btn-outline-secondary:focus{box-shadow:0 0 0 .2rem rgba(108,117,125,.5)}.btn-outline-secondary.disabled,.btn-outline-secondary:disabled{color:#6c757d;background-color:transparent}.btn-outline-secondary:not(:disabled):not(.disabled).active,.btn-outline-secondary:not(:disabled):not(.disabled):active,.show>.btn-outline-secondary.dropdown-toggle{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-outline-secondary:not(:disabled):not(.disabled).active:focus,.btn-outline-secondary:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-secondary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(108,117,125,.5)}.btn-outline-success{color:#28a745;border-color:#28a745}.btn-outline-success:hover{color:#fff;background-color:#28a745;border-color:#28a745}.btn-outline-success.focus,.btn-outline-success:focus{box-shadow:0 0 0 .2rem rgba(40,167,69,.5)}.btn-outline-success.disabled,.btn-outline-success:disabled{color:#28a745;background-color:transparent}.btn-outline-success:not(:disabled):not(.disabled).active,.btn-outline-success:not(:disabled):not(.disabled):active,.show>.btn-outline-success.dropdown-toggle{color:#fff;background-color:#28a745;border-color:#28a745}.btn-outline-success:not(:disabled):not(.disabled).active:focus,.btn-outline-success:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-success.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(40,167,69,.5)}.btn-outline-info{color:#17a2b8;border-color:#17a2b8}.btn-outline-info:hover{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-outline-info.focus,.btn-outline-info:focus{box-shadow:0 0 0 .2rem rgba(23,162,184,.5)}.btn-outline-info.disabled,.btn-outline-info:disabled{color:#17a2b8;background-color:transparent}.btn-outline-info:not(:disabled):not(.disabled).active,.btn-outline-info:not(:disabled):not(.disabled):active,.show>.btn-outline-info.dropdown-toggle{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-outline-info:not(:disabled):not(.disabled).active:focus,.btn-outline-info:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-info.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(23,162,184,.5)}.btn-outline-warning{color:#ffc107;border-color:#ffc107}.btn-outline-warning:hover{color:#212529;background-color:#ffc107;border-color:#ffc107}.btn-outline-warning.focus,.btn-outline-warning:focus{box-shadow:0 0 0 .2rem rgba(255,193,7,.5)}.btn-outline-warning.disabled,.btn-outline-warning:disabled{color:#ffc107;background-color:transparent}.btn-outline-warning:not(:disabled):not(.disabled).active,.btn-outline-warning:not(:disabled):not(.disabled):active,.show>.btn-outline-warning.dropdown-toggle{color:#212529;background-color:#ffc107;border-color:#ffc107}.btn-outline-warning:not(:disabled):not(.disabled).active:focus,.btn-outline-warning:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-warning.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(255,193,7,.5)}.btn-outline-danger{color:#dc3545;border-color:#dc3545}.btn-outline-danger:hover{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-outline-danger.focus,.btn-outline-danger:focus{box-shadow:0 0 0 .2rem rgba(220,53,69,.5)}.btn-outline-danger.disabled,.btn-outline-danger:disabled{color:#dc3545;background-color:transparent}.btn-outline-danger:not(:disabled):not(.disabled).active,.btn-outline-danger:not(:disabled):not(.disabled):active,.show>.btn-outline-danger.dropdown-toggle{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-outline-danger:not(:disabled):not(.disabled).active:focus,.btn-outline-danger:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-danger.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(220,53,69,.5)}.btn-outline-light{color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light:hover{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light.focus,.btn-outline-light:focus{box-shadow:0 0 0 .2rem rgba(248,249,250,.5)}.btn-outline-light.disabled,.btn-outline-light:disabled{color:#f8f9fa;background-color:transparent}.btn-outline-light:not(:disabled):not(.disabled).active,.btn-outline-light:not(:disabled):not(.disabled):active,.show>.btn-outline-light.dropdown-toggle{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light:not(:disabled):not(.disabled).active:focus,.btn-outline-light:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-light.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(248,249,250,.5)}.btn-outline-dark{color:#343a40;border-color:#343a40}.btn-outline-dark:hover{color:#fff;background-color:#343a40;border-color:#343a40}.btn-outline-dark.focus,.btn-outline-dark:focus{box-shadow:0 0 0 .2rem rgba(52,58,64,.5)}.btn-outline-dark.disabled,.btn-outline-dark:disabled{color:#343a40;background-color:transparent}.btn-outline-dark:not(:disabled):not(.disabled).active,.btn-outline-dark:not(:disabled):not(.disabled):active,.show>.btn-outline-dark.dropdown-toggle{color:#fff;background-color:#343a40;border-color:#343a40}.btn-outline-dark:not(:disabled):not(.disabled).active:focus,.btn-outline-dark:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-dark.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(52,58,64,.5)}.btn-link{font-weight:400;color:#007bff;text-decoration:none}.btn-link:hover{color:#0056b3;text-decoration:underline}.btn-link.focus,.btn-link:focus{text-decoration:underline;box-shadow:none}.btn-link.disabled,.btn-link:disabled{color:#6c757d;pointer-events:none}.btn-group-lg>.btn,.btn-lg{padding:.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:.3rem}.btn-group-sm>.btn,.btn-sm{padding:.25rem .5rem;font-size:.875rem;line-height:1.5;border-radius:.2rem}.btn-block{display:block;width:100%}.btn-block+.btn-block{margin-top:.5rem}input[type=button].btn-block,input[type=reset].btn-block,input[type=submit].btn-block{width:100%}.fade{transition:opacity .15s linear}@media (prefers-reduced-motion:reduce){.fade{transition:none}}.fade:not(.show){opacity:0}.collapse:not(.show){display:none}.collapsing{position:relative;height:0;overflow:hidden;transition:height .35s ease}@media (prefers-reduced-motion:reduce){.collapsing{transition:none}}.dropdown,.dropleft,.dropright,.dropup{position:relative}.dropdown-toggle{white-space:nowrap}.dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:.3em solid;border-right:.3em solid transparent;border-bottom:0;border-left:.3em solid transparent}.dropdown-toggle:empty::after{margin-left:0}.dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:10rem;padding:.5rem 0;margin:.125rem 0 0;font-size:1rem;color:#212529;text-align:left;list-style:none;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.15);border-radius:.25rem}.dropdown-menu-left{right:auto;left:0}.dropdown-menu-right{right:0;left:auto}@media (min-width:576px){.dropdown-menu-sm-left{right:auto;left:0}.dropdown-menu-sm-right{right:0;left:auto}}@media (min-width:768px){.dropdown-menu-md-left{right:auto;left:0}.dropdown-menu-md-right{right:0;left:auto}}@media (min-width:992px){.dropdown-menu-lg-left{right:auto;left:0}.dropdown-menu-lg-right{right:0;left:auto}}@media (min-width:1200px){.dropdown-menu-xl-left{right:auto;left:0}.dropdown-menu-xl-right{right:0;left:auto}}.dropup .dropdown-menu{top:auto;bottom:100%;margin-top:0;margin-bottom:.125rem}.dropup .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:0;border-right:.3em solid transparent;border-bottom:.3em solid;border-left:.3em solid transparent}.dropup .dropdown-toggle:empty::after{margin-left:0}.dropright .dropdown-menu{top:0;right:auto;left:100%;margin-top:0;margin-left:.125rem}.dropright .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:.3em solid transparent;border-right:0;border-bottom:.3em solid transparent;border-left:.3em solid}.dropright .dropdown-toggle:empty::after{margin-left:0}.dropright .dropdown-toggle::after{vertical-align:0}.dropleft .dropdown-menu{top:0;right:100%;left:auto;margin-top:0;margin-right:.125rem}.dropleft .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:""}.dropleft .dropdown-toggle::after{display:none}.dropleft .dropdown-toggle::before{display:inline-block;margin-right:.255em;vertical-align:.255em;content:"";border-top:.3em solid transparent;border-right:.3em solid;border-bottom:.3em solid transparent}.dropleft .dropdown-toggle:empty::after{margin-left:0}.dropleft .dropdown-toggle::before{vertical-align:0}.dropdown-menu[x-placement^=bottom],.dropdown-menu[x-placement^=left],.dropdown-menu[x-placement^=right],.dropdown-menu[x-placement^=top]{right:auto;bottom:auto}.dropdown-divider{height:0;margin:.5rem 0;overflow:hidden;border-top:1px solid #e9ecef}.dropdown-item{display:block;width:100%;padding:.25rem 1.5rem;clear:both;font-weight:400;color:#212529;text-align:inherit;white-space:nowrap;background-color:transparent;border:0}.dropdown-item:focus,.dropdown-item:hover{color:#16181b;text-decoration:none;background-color:#f8f9fa}.dropdown-item.active,.dropdown-item:active{color:#fff;text-decoration:none;background-color:#007bff}.dropdown-item.disabled,.dropdown-item:disabled{color:#6c757d;pointer-events:none;background-color:transparent}.dropdown-menu.show{display:block}.dropdown-header{display:block;padding:.5rem 1.5rem;margin-bottom:0;font-size:.875rem;color:#6c757d;white-space:nowrap}.dropdown-item-text{display:block;padding:.25rem 1.5rem;color:#212529}.btn-group,.btn-group-vertical{position:relative;display:-ms-inline-flexbox;display:inline-flex;vertical-align:middle}.btn-group-vertical>.btn,.btn-group>.btn{position:relative;-ms-flex:1 1 auto;flex:1 1 auto}.btn-group-vertical>.btn:hover,.btn-group>.btn:hover{z-index:1}.btn-group-vertical>.btn.active,.btn-group-vertical>.btn:active,.btn-group-vertical>.btn:focus,.btn-group>.btn.active,.btn-group>.btn:active,.btn-group>.btn:focus{z-index:1}.btn-toolbar{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-pack:start;justify-content:flex-start}.btn-toolbar .input-group{width:auto}.btn-group>.btn-group:not(:first-child),.btn-group>.btn:not(:first-child){margin-left:-1px}.btn-group>.btn-group:not(:last-child)>.btn,.btn-group>.btn:not(:last-child):not(.dropdown-toggle){border-top-right-radius:0;border-bottom-right-radius:0}.btn-group>.btn-group:not(:first-child)>.btn,.btn-group>.btn:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.dropdown-toggle-split{padding-right:.5625rem;padding-left:.5625rem}.dropdown-toggle-split::after,.dropright .dropdown-toggle-split::after,.dropup .dropdown-toggle-split::after{margin-left:0}.dropleft .dropdown-toggle-split::before{margin-right:0}.btn-group-sm>.btn+.dropdown-toggle-split,.btn-sm+.dropdown-toggle-split{padding-right:.375rem;padding-left:.375rem}.btn-group-lg>.btn+.dropdown-toggle-split,.btn-lg+.dropdown-toggle-split{padding-right:.75rem;padding-left:.75rem}.btn-group-vertical{-ms-flex-direction:column;flex-direction:column;-ms-flex-align:start;align-items:flex-start;-ms-flex-pack:center;justify-content:center}.btn-group-vertical>.btn,.btn-group-vertical>.btn-group{width:100%}.btn-group-vertical>.btn-group:not(:first-child),.btn-group-vertical>.btn:not(:first-child){margin-top:-1px}.btn-group-vertical>.btn-group:not(:last-child)>.btn,.btn-group-vertical>.btn:not(:last-child):not(.dropdown-toggle){border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn-group:not(:first-child)>.btn,.btn-group-vertical>.btn:not(:first-child){border-top-left-radius:0;border-top-right-radius:0}.btn-group-toggle>.btn,.btn-group-toggle>.btn-group>.btn{margin-bottom:0}.btn-group-toggle>.btn input[type=checkbox],.btn-group-toggle>.btn input[type=radio],.btn-group-toggle>.btn-group>.btn input[type=checkbox],.btn-group-toggle>.btn-group>.btn input[type=radio]{position:absolute;clip:rect(0,0,0,0);pointer-events:none}.input-group{position:relative;display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-align:stretch;align-items:stretch;width:100%}.input-group>.custom-file,.input-group>.custom-select,.input-group>.form-control,.input-group>.form-control-plaintext{position:relative;-ms-flex:1 1 0%;flex:1 1 0%;min-width:0;margin-bottom:0}.input-group>.custom-file+.custom-file,.input-group>.custom-file+.custom-select,.input-group>.custom-file+.form-control,.input-group>.custom-select+.custom-file,.input-group>.custom-select+.custom-select,.input-group>.custom-select+.form-control,.input-group>.form-control+.custom-file,.input-group>.form-control+.custom-select,.input-group>.form-control+.form-control,.input-group>.form-control-plaintext+.custom-file,.input-group>.form-control-plaintext+.custom-select,.input-group>.form-control-plaintext+.form-control{margin-left:-1px}.input-group>.custom-file .custom-file-input:focus~.custom-file-label,.input-group>.custom-select:focus,.input-group>.form-control:focus{z-index:3}.input-group>.custom-file .custom-file-input:focus{z-index:4}.input-group>.custom-select:not(:last-child),.input-group>.form-control:not(:last-child){border-top-right-radius:0;border-bottom-right-radius:0}.input-group>.custom-select:not(:first-child),.input-group>.form-control:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.input-group>.custom-file{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center}.input-group>.custom-file:not(:last-child) .custom-file-label,.input-group>.custom-file:not(:last-child) .custom-file-label::after{border-top-right-radius:0;border-bottom-right-radius:0}.input-group>.custom-file:not(:first-child) .custom-file-label{border-top-left-radius:0;border-bottom-left-radius:0}.input-group-append,.input-group-prepend{display:-ms-flexbox;display:flex}.input-group-append .btn,.input-group-prepend .btn{position:relative;z-index:2}.input-group-append .btn:focus,.input-group-prepend .btn:focus{z-index:3}.input-group-append .btn+.btn,.input-group-append .btn+.input-group-text,.input-group-append .input-group-text+.btn,.input-group-append .input-group-text+.input-group-text,.input-group-prepend .btn+.btn,.input-group-prepend .btn+.input-group-text,.input-group-prepend .input-group-text+.btn,.input-group-prepend .input-group-text+.input-group-text{margin-left:-1px}.input-group-prepend{margin-right:-1px}.input-group-append{margin-left:-1px}.input-group-text{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;padding:.375rem .75rem;margin-bottom:0;font-size:1rem;font-weight:400;line-height:1.5;color:#495057;text-align:center;white-space:nowrap;background-color:#e9ecef;border:1px solid #ced4da;border-radius:.25rem}.input-group-text input[type=checkbox],.input-group-text input[type=radio]{margin-top:0}.input-group-lg>.custom-select,.input-group-lg>.form-control:not(textarea){height:calc(1.5em + 1rem + 2px)}.input-group-lg>.custom-select,.input-group-lg>.form-control,.input-group-lg>.input-group-append>.btn,.input-group-lg>.input-group-append>.input-group-text,.input-group-lg>.input-group-prepend>.btn,.input-group-lg>.input-group-prepend>.input-group-text{padding:.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:.3rem}.input-group-sm>.custom-select,.input-group-sm>.form-control:not(textarea){height:calc(1.5em + .5rem + 2px)}.input-group-sm>.custom-select,.input-group-sm>.form-control,.input-group-sm>.input-group-append>.btn,.input-group-sm>.input-group-append>.input-group-text,.input-group-sm>.input-group-prepend>.btn,.input-group-sm>.input-group-prepend>.input-group-text{padding:.25rem .5rem;font-size:.875rem;line-height:1.5;border-radius:.2rem}.input-group-lg>.custom-select,.input-group-sm>.custom-select{padding-right:1.75rem}.input-group>.input-group-append:last-child>.btn:not(:last-child):not(.dropdown-toggle),.input-group>.input-group-append:last-child>.input-group-text:not(:last-child),.input-group>.input-group-append:not(:last-child)>.btn,.input-group>.input-group-append:not(:last-child)>.input-group-text,.input-group>.input-group-prepend>.btn,.input-group>.input-group-prepend>.input-group-text{border-top-right-radius:0;border-bottom-right-radius:0}.input-group>.input-group-append>.btn,.input-group>.input-group-append>.input-group-text,.input-group>.input-group-prepend:first-child>.btn:not(:first-child),.input-group>.input-group-prepend:first-child>.input-group-text:not(:first-child),.input-group>.input-group-prepend:not(:first-child)>.btn,.input-group>.input-group-prepend:not(:first-child)>.input-group-text{border-top-left-radius:0;border-bottom-left-radius:0}.custom-control{position:relative;display:block;min-height:1.5rem;padding-left:1.5rem}.custom-control-inline{display:-ms-inline-flexbox;display:inline-flex;margin-right:1rem}.custom-control-input{position:absolute;left:0;z-index:-1;width:1rem;height:1.25rem;opacity:0}.custom-control-input:checked~.custom-control-label::before{color:#fff;border-color:#007bff;background-color:#007bff}.custom-control-input:focus~.custom-control-label::before{box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.custom-control-input:focus:not(:checked)~.custom-control-label::before{border-color:#80bdff}.custom-control-input:not(:disabled):active~.custom-control-label::before{color:#fff;background-color:#b3d7ff;border-color:#b3d7ff}.custom-control-input:disabled~.custom-control-label,.custom-control-input[disabled]~.custom-control-label{color:#6c757d}.custom-control-input:disabled~.custom-control-label::before,.custom-control-input[disabled]~.custom-control-label::before{background-color:#e9ecef}.custom-control-label{position:relative;margin-bottom:0;vertical-align:top}.custom-control-label::before{position:absolute;top:.25rem;left:-1.5rem;display:block;width:1rem;height:1rem;pointer-events:none;content:"";background-color:#fff;border:#adb5bd solid 1px}.custom-control-label::after{position:absolute;top:.25rem;left:-1.5rem;display:block;width:1rem;height:1rem;content:"";background:no-repeat 50%/50% 50%}.custom-checkbox .custom-control-label::before{border-radius:.25rem}.custom-checkbox .custom-control-input:checked~.custom-control-label::after{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath fill='%23fff' d='M6.564.75l-3.59 3.612-1.538-1.55L0 4.26l2.974 2.99L8 2.193z'/%3e%3c/svg%3e")}.custom-checkbox .custom-control-input:indeterminate~.custom-control-label::before{border-color:#007bff;background-color:#007bff}.custom-checkbox .custom-control-input:indeterminate~.custom-control-label::after{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='4' viewBox='0 0 4 4'%3e%3cpath stroke='%23fff' d='M0 2h4'/%3e%3c/svg%3e")}.custom-checkbox .custom-control-input:disabled:checked~.custom-control-label::before{background-color:rgba(0,123,255,.5)}.custom-checkbox .custom-control-input:disabled:indeterminate~.custom-control-label::before{background-color:rgba(0,123,255,.5)}.custom-radio .custom-control-label::before{border-radius:50%}.custom-radio .custom-control-input:checked~.custom-control-label::after{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='%23fff'/%3e%3c/svg%3e")}.custom-radio .custom-control-input:disabled:checked~.custom-control-label::before{background-color:rgba(0,123,255,.5)}.custom-switch{padding-left:2.25rem}.custom-switch .custom-control-label::before{left:-2.25rem;width:1.75rem;pointer-events:all;border-radius:.5rem}.custom-switch .custom-control-label::after{top:calc(.25rem + 2px);left:calc(-2.25rem + 2px);width:calc(1rem - 4px);height:calc(1rem - 4px);background-color:#adb5bd;border-radius:.5rem;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out,-webkit-transform .15s ease-in-out;transition:transform .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:transform .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out,-webkit-transform .15s ease-in-out}@media (prefers-reduced-motion:reduce){.custom-switch .custom-control-label::after{transition:none}}.custom-switch .custom-control-input:checked~.custom-control-label::after{background-color:#fff;-webkit-transform:translateX(.75rem);transform:translateX(.75rem)}.custom-switch .custom-control-input:disabled:checked~.custom-control-label::before{background-color:rgba(0,123,255,.5)}.custom-select{display:inline-block;width:100%;height:calc(1.5em + .75rem + 2px);padding:.375rem 1.75rem .375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:#495057;vertical-align:middle;background:#fff url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='5' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e") no-repeat right .75rem center/8px 10px;border:1px solid #ced4da;border-radius:.25rem;-webkit-appearance:none;-moz-appearance:none;appearance:none}.custom-select:focus{border-color:#80bdff;outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.custom-select:focus::-ms-value{color:#495057;background-color:#fff}.custom-select[multiple],.custom-select[size]:not([size="1"]){height:auto;padding-right:.75rem;background-image:none}.custom-select:disabled{color:#6c757d;background-color:#e9ecef}.custom-select::-ms-expand{display:none}.custom-select:-moz-focusring{color:transparent;text-shadow:0 0 0 #495057}.custom-select-sm{height:calc(1.5em + .5rem + 2px);padding-top:.25rem;padding-bottom:.25rem;padding-left:.5rem;font-size:.875rem}.custom-select-lg{height:calc(1.5em + 1rem + 2px);padding-top:.5rem;padding-bottom:.5rem;padding-left:1rem;font-size:1.25rem}.custom-file{position:relative;display:inline-block;width:100%;height:calc(1.5em + .75rem + 2px);margin-bottom:0}.custom-file-input{position:relative;z-index:2;width:100%;height:calc(1.5em + .75rem + 2px);margin:0;opacity:0}.custom-file-input:focus~.custom-file-label{border-color:#80bdff;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.custom-file-input:disabled~.custom-file-label,.custom-file-input[disabled]~.custom-file-label{background-color:#e9ecef}.custom-file-input:lang(en)~.custom-file-label::after{content:"Browse"}.custom-file-input~.custom-file-label[data-browse]::after{content:attr(data-browse)}.custom-file-label{position:absolute;top:0;right:0;left:0;z-index:1;height:calc(1.5em + .75rem + 2px);padding:.375rem .75rem;font-weight:400;line-height:1.5;color:#495057;background-color:#fff;border:1px solid #ced4da;border-radius:.25rem}.custom-file-label::after{position:absolute;top:0;right:0;bottom:0;z-index:3;display:block;height:calc(1.5em + .75rem);padding:.375rem .75rem;line-height:1.5;color:#495057;content:"Browse";background-color:#e9ecef;border-left:inherit;border-radius:0 .25rem .25rem 0}.custom-range{width:100%;height:1.4rem;padding:0;background-color:transparent;-webkit-appearance:none;-moz-appearance:none;appearance:none}.custom-range:focus{outline:0}.custom-range:focus::-webkit-slider-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .2rem rgba(0,123,255,.25)}.custom-range:focus::-moz-range-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .2rem rgba(0,123,255,.25)}.custom-range:focus::-ms-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .2rem rgba(0,123,255,.25)}.custom-range::-moz-focus-outer{border:0}.custom-range::-webkit-slider-thumb{width:1rem;height:1rem;margin-top:-.25rem;background-color:#007bff;border:0;border-radius:1rem;-webkit-transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;-webkit-appearance:none;appearance:none}@media (prefers-reduced-motion:reduce){.custom-range::-webkit-slider-thumb{-webkit-transition:none;transition:none}}.custom-range::-webkit-slider-thumb:active{background-color:#b3d7ff}.custom-range::-webkit-slider-runnable-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:#dee2e6;border-color:transparent;border-radius:1rem}.custom-range::-moz-range-thumb{width:1rem;height:1rem;background-color:#007bff;border:0;border-radius:1rem;-moz-transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;-moz-appearance:none;appearance:none}@media (prefers-reduced-motion:reduce){.custom-range::-moz-range-thumb{-moz-transition:none;transition:none}}.custom-range::-moz-range-thumb:active{background-color:#b3d7ff}.custom-range::-moz-range-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:#dee2e6;border-color:transparent;border-radius:1rem}.custom-range::-ms-thumb{width:1rem;height:1rem;margin-top:0;margin-right:.2rem;margin-left:.2rem;background-color:#007bff;border:0;border-radius:1rem;-ms-transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;appearance:none}@media (prefers-reduced-motion:reduce){.custom-range::-ms-thumb{-ms-transition:none;transition:none}}.custom-range::-ms-thumb:active{background-color:#b3d7ff}.custom-range::-ms-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:transparent;border-color:transparent;border-width:.5rem}.custom-range::-ms-fill-lower{background-color:#dee2e6;border-radius:1rem}.custom-range::-ms-fill-upper{margin-right:15px;background-color:#dee2e6;border-radius:1rem}.custom-range:disabled::-webkit-slider-thumb{background-color:#adb5bd}.custom-range:disabled::-webkit-slider-runnable-track{cursor:default}.custom-range:disabled::-moz-range-thumb{background-color:#adb5bd}.custom-range:disabled::-moz-range-track{cursor:default}.custom-range:disabled::-ms-thumb{background-color:#adb5bd}.custom-control-label::before,.custom-file-label,.custom-select{transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.custom-control-label::before,.custom-file-label,.custom-select{transition:none}}.nav{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;padding-left:0;margin-bottom:0;list-style:none}.nav-link{display:block;padding:.5rem 1rem}.nav-link:focus,.nav-link:hover{text-decoration:none}.nav-link.disabled{color:#6c757d;pointer-events:none;cursor:default}.nav-tabs{border-bottom:1px solid #dee2e6}.nav-tabs .nav-item{margin-bottom:-1px}.nav-tabs .nav-link{border:1px solid transparent;border-top-left-radius:.25rem;border-top-right-radius:.25rem}.nav-tabs .nav-link:focus,.nav-tabs .nav-link:hover{border-color:#e9ecef #e9ecef #dee2e6}.nav-tabs .nav-link.disabled{color:#6c757d;background-color:transparent;border-color:transparent}.nav-tabs .nav-item.show .nav-link,.nav-tabs .nav-link.active{color:#495057;background-color:#fff;border-color:#dee2e6 #dee2e6 #fff}.nav-tabs .dropdown-menu{margin-top:-1px;border-top-left-radius:0;border-top-right-radius:0}.nav-pills .nav-link{border-radius:.25rem}.nav-pills .nav-link.active,.nav-pills .show>.nav-link{color:#fff;background-color:#007bff}.nav-fill .nav-item{-ms-flex:1 1 auto;flex:1 1 auto;text-align:center}.nav-justified .nav-item{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;text-align:center}.tab-content>.tab-pane{display:none}.tab-content>.active{display:block}.navbar{position:relative;display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-align:center;align-items:center;-ms-flex-pack:justify;justify-content:space-between;padding:.5rem 1rem}.navbar .container,.navbar .container-fluid,.navbar .container-lg,.navbar .container-md,.navbar .container-sm,.navbar .container-xl{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-align:center;align-items:center;-ms-flex-pack:justify;justify-content:space-between}.navbar-brand{display:inline-block;padding-top:.3125rem;padding-bottom:.3125rem;margin-right:1rem;font-size:1.25rem;line-height:inherit;white-space:nowrap}.navbar-brand:focus,.navbar-brand:hover{text-decoration:none}.navbar-nav{display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;padding-left:0;margin-bottom:0;list-style:none}.navbar-nav .nav-link{padding-right:0;padding-left:0}.navbar-nav .dropdown-menu{position:static;float:none}.navbar-text{display:inline-block;padding-top:.5rem;padding-bottom:.5rem}.navbar-collapse{-ms-flex-preferred-size:100%;flex-basis:100%;-ms-flex-positive:1;flex-grow:1;-ms-flex-align:center;align-items:center}.navbar-toggler{padding:.25rem .75rem;font-size:1.25rem;line-height:1;background-color:transparent;border:1px solid transparent;border-radius:.25rem}.navbar-toggler:focus,.navbar-toggler:hover{text-decoration:none}.navbar-toggler-icon{display:inline-block;width:1.5em;height:1.5em;vertical-align:middle;content:"";background:no-repeat center center;background-size:100% 100%}@media (max-width:575.98px){.navbar-expand-sm>.container,.navbar-expand-sm>.container-fluid,.navbar-expand-sm>.container-lg,.navbar-expand-sm>.container-md,.navbar-expand-sm>.container-sm,.navbar-expand-sm>.container-xl{padding-right:0;padding-left:0}}@media (min-width:576px){.navbar-expand-sm{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-sm .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-sm .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-sm .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-sm>.container,.navbar-expand-sm>.container-fluid,.navbar-expand-sm>.container-lg,.navbar-expand-sm>.container-md,.navbar-expand-sm>.container-sm,.navbar-expand-sm>.container-xl{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-sm .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-sm .navbar-toggler{display:none}}@media (max-width:767.98px){.navbar-expand-md>.container,.navbar-expand-md>.container-fluid,.navbar-expand-md>.container-lg,.navbar-expand-md>.container-md,.navbar-expand-md>.container-sm,.navbar-expand-md>.container-xl{padding-right:0;padding-left:0}}@media (min-width:768px){.navbar-expand-md{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-md .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-md .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-md .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-md>.container,.navbar-expand-md>.container-fluid,.navbar-expand-md>.container-lg,.navbar-expand-md>.container-md,.navbar-expand-md>.container-sm,.navbar-expand-md>.container-xl{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-md .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-md .navbar-toggler{display:none}}@media (max-width:991.98px){.navbar-expand-lg>.container,.navbar-expand-lg>.container-fluid,.navbar-expand-lg>.container-lg,.navbar-expand-lg>.container-md,.navbar-expand-lg>.container-sm,.navbar-expand-lg>.container-xl{padding-right:0;padding-left:0}}@media (min-width:992px){.navbar-expand-lg{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-lg .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-lg .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-lg .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-lg>.container,.navbar-expand-lg>.container-fluid,.navbar-expand-lg>.container-lg,.navbar-expand-lg>.container-md,.navbar-expand-lg>.container-sm,.navbar-expand-lg>.container-xl{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-lg .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-lg .navbar-toggler{display:none}}@media (max-width:1199.98px){.navbar-expand-xl>.container,.navbar-expand-xl>.container-fluid,.navbar-expand-xl>.container-lg,.navbar-expand-xl>.container-md,.navbar-expand-xl>.container-sm,.navbar-expand-xl>.container-xl{padding-right:0;padding-left:0}}@media (min-width:1200px){.navbar-expand-xl{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-xl .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-xl .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-xl .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-xl>.container,.navbar-expand-xl>.container-fluid,.navbar-expand-xl>.container-lg,.navbar-expand-xl>.container-md,.navbar-expand-xl>.container-sm,.navbar-expand-xl>.container-xl{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-xl .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-xl .navbar-toggler{display:none}}.navbar-expand{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand>.container,.navbar-expand>.container-fluid,.navbar-expand>.container-lg,.navbar-expand>.container-md,.navbar-expand>.container-sm,.navbar-expand>.container-xl{padding-right:0;padding-left:0}.navbar-expand .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand .navbar-nav .dropdown-menu{position:absolute}.navbar-expand .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand>.container,.navbar-expand>.container-fluid,.navbar-expand>.container-lg,.navbar-expand>.container-md,.navbar-expand>.container-sm,.navbar-expand>.container-xl{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand .navbar-toggler{display:none}.navbar-light .navbar-brand{color:rgba(0,0,0,.9)}.navbar-light .navbar-brand:focus,.navbar-light .navbar-brand:hover{color:rgba(0,0,0,.9)}.navbar-light .navbar-nav .nav-link{color:rgba(0,0,0,.5)}.navbar-light .navbar-nav .nav-link:focus,.navbar-light .navbar-nav .nav-link:hover{color:rgba(0,0,0,.7)}.navbar-light .navbar-nav .nav-link.disabled{color:rgba(0,0,0,.3)}.navbar-light .navbar-nav .active>.nav-link,.navbar-light .navbar-nav .nav-link.active,.navbar-light .navbar-nav .nav-link.show,.navbar-light .navbar-nav .show>.nav-link{color:rgba(0,0,0,.9)}.navbar-light .navbar-toggler{color:rgba(0,0,0,.5);border-color:rgba(0,0,0,.1)}.navbar-light .navbar-toggler-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='30' height='30' viewBox='0 0 30 30'%3e%3cpath stroke='rgba(0, 0, 0, 0.5)' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e")}.navbar-light .navbar-text{color:rgba(0,0,0,.5)}.navbar-light .navbar-text a{color:rgba(0,0,0,.9)}.navbar-light .navbar-text a:focus,.navbar-light .navbar-text a:hover{color:rgba(0,0,0,.9)}.navbar-dark .navbar-brand{color:#fff}.navbar-dark .navbar-brand:focus,.navbar-dark .navbar-brand:hover{color:#fff}.navbar-dark .navbar-nav .nav-link{color:rgba(255,255,255,.5)}.navbar-dark .navbar-nav .nav-link:focus,.navbar-dark .navbar-nav .nav-link:hover{color:rgba(255,255,255,.75)}.navbar-dark .navbar-nav .nav-link.disabled{color:rgba(255,255,255,.25)}.navbar-dark .navbar-nav .active>.nav-link,.navbar-dark .navbar-nav .nav-link.active,.navbar-dark .navbar-nav .nav-link.show,.navbar-dark .navbar-nav .show>.nav-link{color:#fff}.navbar-dark .navbar-toggler{color:rgba(255,255,255,.5);border-color:rgba(255,255,255,.1)}.navbar-dark .navbar-toggler-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='30' height='30' viewBox='0 0 30 30'%3e%3cpath stroke='rgba(255, 255, 255, 0.5)' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e")}.navbar-dark .navbar-text{color:rgba(255,255,255,.5)}.navbar-dark .navbar-text a{color:#fff}.navbar-dark .navbar-text a:focus,.navbar-dark .navbar-text a:hover{color:#fff}.card{position:relative;display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;min-width:0;word-wrap:break-word;background-color:#fff;background-clip:border-box;border:1px solid rgba(0,0,0,.125);border-radius:.25rem}.card>hr{margin-right:0;margin-left:0}.card>.list-group:first-child .list-group-item:first-child{border-top-left-radius:.25rem;border-top-right-radius:.25rem}.card>.list-group:last-child .list-group-item:last-child{border-bottom-right-radius:.25rem;border-bottom-left-radius:.25rem}.card-body{-ms-flex:1 1 auto;flex:1 1 auto;min-height:1px;padding:1.25rem}.card-title{margin-bottom:.75rem}.card-subtitle{margin-top:-.375rem;margin-bottom:0}.card-text:last-child{margin-bottom:0}.card-link:hover{text-decoration:none}.card-link+.card-link{margin-left:1.25rem}.card-header{padding:.75rem 1.25rem;margin-bottom:0;background-color:rgba(0,0,0,.03);border-bottom:1px solid rgba(0,0,0,.125)}.card-header:first-child{border-radius:calc(.25rem - 1px) calc(.25rem - 1px) 0 0}.card-header+.list-group .list-group-item:first-child{border-top:0}.card-footer{padding:.75rem 1.25rem;background-color:rgba(0,0,0,.03);border-top:1px solid rgba(0,0,0,.125)}.card-footer:last-child{border-radius:0 0 calc(.25rem - 1px) calc(.25rem - 1px)}.card-header-tabs{margin-right:-.625rem;margin-bottom:-.75rem;margin-left:-.625rem;border-bottom:0}.card-header-pills{margin-right:-.625rem;margin-left:-.625rem}.card-img-overlay{position:absolute;top:0;right:0;bottom:0;left:0;padding:1.25rem}.card-img,.card-img-bottom,.card-img-top{-ms-flex-negative:0;flex-shrink:0;width:100%}.card-img,.card-img-top{border-top-left-radius:calc(.25rem - 1px);border-top-right-radius:calc(.25rem - 1px)}.card-img,.card-img-bottom{border-bottom-right-radius:calc(.25rem - 1px);border-bottom-left-radius:calc(.25rem - 1px)}.card-deck .card{margin-bottom:15px}@media (min-width:576px){.card-deck{display:-ms-flexbox;display:flex;-ms-flex-flow:row wrap;flex-flow:row wrap;margin-right:-15px;margin-left:-15px}.card-deck .card{-ms-flex:1 0 0%;flex:1 0 0%;margin-right:15px;margin-bottom:0;margin-left:15px}}.card-group>.card{margin-bottom:15px}@media (min-width:576px){.card-group{display:-ms-flexbox;display:flex;-ms-flex-flow:row wrap;flex-flow:row wrap}.card-group>.card{-ms-flex:1 0 0%;flex:1 0 0%;margin-bottom:0}.card-group>.card+.card{margin-left:0;border-left:0}.card-group>.card:not(:last-child){border-top-right-radius:0;border-bottom-right-radius:0}.card-group>.card:not(:last-child) .card-header,.card-group>.card:not(:last-child) .card-img-top{border-top-right-radius:0}.card-group>.card:not(:last-child) .card-footer,.card-group>.card:not(:last-child) .card-img-bottom{border-bottom-right-radius:0}.card-group>.card:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.card-group>.card:not(:first-child) .card-header,.card-group>.card:not(:first-child) .card-img-top{border-top-left-radius:0}.card-group>.card:not(:first-child) .card-footer,.card-group>.card:not(:first-child) .card-img-bottom{border-bottom-left-radius:0}}.card-columns .card{margin-bottom:.75rem}@media (min-width:576px){.card-columns{-webkit-column-count:3;-moz-column-count:3;column-count:3;-webkit-column-gap:1.25rem;-moz-column-gap:1.25rem;column-gap:1.25rem;orphans:1;widows:1}.card-columns .card{display:inline-block;width:100%}}.accordion>.card{overflow:hidden}.accordion>.card:not(:last-of-type){border-bottom:0;border-bottom-right-radius:0;border-bottom-left-radius:0}.accordion>.card:not(:first-of-type){border-top-left-radius:0;border-top-right-radius:0}.accordion>.card>.card-header{border-radius:0;margin-bottom:-1px}.breadcrumb{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;padding:.75rem 1rem;margin-bottom:1rem;list-style:none;background-color:#e9ecef;border-radius:.25rem}.breadcrumb-item+.breadcrumb-item{padding-left:.5rem}.breadcrumb-item+.breadcrumb-item::before{display:inline-block;padding-right:.5rem;color:#6c757d;content:"/"}.breadcrumb-item+.breadcrumb-item:hover::before{text-decoration:underline}.breadcrumb-item+.breadcrumb-item:hover::before{text-decoration:none}.breadcrumb-item.active{color:#6c757d}.pagination{display:-ms-flexbox;display:flex;padding-left:0;list-style:none;border-radius:.25rem}.page-link{position:relative;display:block;padding:.5rem .75rem;margin-left:-1px;line-height:1.25;color:#007bff;background-color:#fff;border:1px solid #dee2e6}.page-link:hover{z-index:2;color:#0056b3;text-decoration:none;background-color:#e9ecef;border-color:#dee2e6}.page-link:focus{z-index:3;outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.page-item:first-child .page-link{margin-left:0;border-top-left-radius:.25rem;border-bottom-left-radius:.25rem}.page-item:last-child .page-link{border-top-right-radius:.25rem;border-bottom-right-radius:.25rem}.page-item.active .page-link{z-index:3;color:#fff;background-color:#007bff;border-color:#007bff}.page-item.disabled .page-link{color:#6c757d;pointer-events:none;cursor:auto;background-color:#fff;border-color:#dee2e6}.pagination-lg .page-link{padding:.75rem 1.5rem;font-size:1.25rem;line-height:1.5}.pagination-lg .page-item:first-child .page-link{border-top-left-radius:.3rem;border-bottom-left-radius:.3rem}.pagination-lg .page-item:last-child .page-link{border-top-right-radius:.3rem;border-bottom-right-radius:.3rem}.pagination-sm .page-link{padding:.25rem .5rem;font-size:.875rem;line-height:1.5}.pagination-sm .page-item:first-child .page-link{border-top-left-radius:.2rem;border-bottom-left-radius:.2rem}.pagination-sm .page-item:last-child .page-link{border-top-right-radius:.2rem;border-bottom-right-radius:.2rem}.badge{display:inline-block;padding:.25em .4em;font-size:75%;font-weight:700;line-height:1;text-align:center;white-space:nowrap;vertical-align:baseline;border-radius:.25rem;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.badge{transition:none}}a.badge:focus,a.badge:hover{text-decoration:none}.badge:empty{display:none}.btn .badge{position:relative;top:-1px}.badge-pill{padding-right:.6em;padding-left:.6em;border-radius:10rem}.badge-primary{color:#fff;background-color:#007bff}a.badge-primary:focus,a.badge-primary:hover{color:#fff;background-color:#0062cc}a.badge-primary.focus,a.badge-primary:focus{outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.5)}.badge-secondary{color:#fff;background-color:#6c757d}a.badge-secondary:focus,a.badge-secondary:hover{color:#fff;background-color:#545b62}a.badge-secondary.focus,a.badge-secondary:focus{outline:0;box-shadow:0 0 0 .2rem rgba(108,117,125,.5)}.badge-success{color:#fff;background-color:#28a745}a.badge-success:focus,a.badge-success:hover{color:#fff;background-color:#1e7e34}a.badge-success.focus,a.badge-success:focus{outline:0;box-shadow:0 0 0 .2rem rgba(40,167,69,.5)}.badge-info{color:#fff;background-color:#17a2b8}a.badge-info:focus,a.badge-info:hover{color:#fff;background-color:#117a8b}a.badge-info.focus,a.badge-info:focus{outline:0;box-shadow:0 0 0 .2rem rgba(23,162,184,.5)}.badge-warning{color:#212529;background-color:#ffc107}a.badge-warning:focus,a.badge-warning:hover{color:#212529;background-color:#d39e00}a.badge-warning.focus,a.badge-warning:focus{outline:0;box-shadow:0 0 0 .2rem rgba(255,193,7,.5)}.badge-danger{color:#fff;background-color:#dc3545}a.badge-danger:focus,a.badge-danger:hover{color:#fff;background-color:#bd2130}a.badge-danger.focus,a.badge-danger:focus{outline:0;box-shadow:0 0 0 .2rem rgba(220,53,69,.5)}.badge-light{color:#212529;background-color:#f8f9fa}a.badge-light:focus,a.badge-light:hover{color:#212529;background-color:#dae0e5}a.badge-light.focus,a.badge-light:focus{outline:0;box-shadow:0 0 0 .2rem rgba(248,249,250,.5)}.badge-dark{color:#fff;background-color:#343a40}a.badge-dark:focus,a.badge-dark:hover{color:#fff;background-color:#1d2124}a.badge-dark.focus,a.badge-dark:focus{outline:0;box-shadow:0 0 0 .2rem rgba(52,58,64,.5)}.jumbotron{padding:2rem 1rem;margin-bottom:2rem;background-color:#e9ecef;border-radius:.3rem}@media (min-width:576px){.jumbotron{padding:4rem 2rem}}.jumbotron-fluid{padding-right:0;padding-left:0;border-radius:0}.alert{position:relative;padding:.75rem 1.25rem;margin-bottom:1rem;border:1px solid transparent;border-radius:.25rem}.alert-heading{color:inherit}.alert-link{font-weight:700}.alert-dismissible{padding-right:4rem}.alert-dismissible .close{position:absolute;top:0;right:0;padding:.75rem 1.25rem;color:inherit}.alert-primary{color:#004085;background-color:#cce5ff;border-color:#b8daff}.alert-primary hr{border-top-color:#9fcdff}.alert-primary .alert-link{color:#002752}.alert-secondary{color:#383d41;background-color:#e2e3e5;border-color:#d6d8db}.alert-secondary hr{border-top-color:#c8cbcf}.alert-secondary .alert-link{color:#202326}.alert-success{color:#155724;background-color:#d4edda;border-color:#c3e6cb}.alert-success hr{border-top-color:#b1dfbb}.alert-success .alert-link{color:#0b2e13}.alert-info{color:#0c5460;background-color:#d1ecf1;border-color:#bee5eb}.alert-info hr{border-top-color:#abdde5}.alert-info .alert-link{color:#062c33}.alert-warning{color:#856404;background-color:#fff3cd;border-color:#ffeeba}.alert-warning hr{border-top-color:#ffe8a1}.alert-warning .alert-link{color:#533f03}.alert-danger{color:#721c24;background-color:#f8d7da;border-color:#f5c6cb}.alert-danger hr{border-top-color:#f1b0b7}.alert-danger .alert-link{color:#491217}.alert-light{color:#818182;background-color:#fefefe;border-color:#fdfdfe}.alert-light hr{border-top-color:#ececf6}.alert-light .alert-link{color:#686868}.alert-dark{color:#1b1e21;background-color:#d6d8d9;border-color:#c6c8ca}.alert-dark hr{border-top-color:#b9bbbe}.alert-dark .alert-link{color:#040505}@-webkit-keyframes progress-bar-stripes{from{background-position:1rem 0}to{background-position:0 0}}@keyframes progress-bar-stripes{from{background-position:1rem 0}to{background-position:0 0}}.progress{display:-ms-flexbox;display:flex;height:1rem;overflow:hidden;font-size:.75rem;background-color:#e9ecef;border-radius:.25rem}.progress-bar{display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;-ms-flex-pack:center;justify-content:center;overflow:hidden;color:#fff;text-align:center;white-space:nowrap;background-color:#007bff;transition:width .6s ease}@media (prefers-reduced-motion:reduce){.progress-bar{transition:none}}.progress-bar-striped{background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-size:1rem 1rem}.progress-bar-animated{-webkit-animation:progress-bar-stripes 1s linear infinite;animation:progress-bar-stripes 1s linear infinite}@media (prefers-reduced-motion:reduce){.progress-bar-animated{-webkit-animation:none;animation:none}}.media{display:-ms-flexbox;display:flex;-ms-flex-align:start;align-items:flex-start}.media-body{-ms-flex:1;flex:1}.list-group{display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;padding-left:0;margin-bottom:0}.list-group-item-action{width:100%;color:#495057;text-align:inherit}.list-group-item-action:focus,.list-group-item-action:hover{z-index:1;color:#495057;text-decoration:none;background-color:#f8f9fa}.list-group-item-action:active{color:#212529;background-color:#e9ecef}.list-group-item{position:relative;display:block;padding:.75rem 1.25rem;background-color:#fff;border:1px solid rgba(0,0,0,.125)}.list-group-item:first-child{border-top-left-radius:.25rem;border-top-right-radius:.25rem}.list-group-item:last-child{border-bottom-right-radius:.25rem;border-bottom-left-radius:.25rem}.list-group-item.disabled,.list-group-item:disabled{color:#6c757d;pointer-events:none;background-color:#fff}.list-group-item.active{z-index:2;color:#fff;background-color:#007bff;border-color:#007bff}.list-group-item+.list-group-item{border-top-width:0}.list-group-item+.list-group-item.active{margin-top:-1px;border-top-width:1px}.list-group-horizontal{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal .list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal .list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal .list-group-item.active{margin-top:0}.list-group-horizontal .list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal .list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}@media (min-width:576px){.list-group-horizontal-sm{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal-sm .list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-sm .list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-sm .list-group-item.active{margin-top:0}.list-group-horizontal-sm .list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-sm .list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}@media (min-width:768px){.list-group-horizontal-md{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal-md .list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-md .list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-md .list-group-item.active{margin-top:0}.list-group-horizontal-md .list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-md .list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}@media (min-width:992px){.list-group-horizontal-lg{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal-lg .list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-lg .list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-lg .list-group-item.active{margin-top:0}.list-group-horizontal-lg .list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-lg .list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}@media (min-width:1200px){.list-group-horizontal-xl{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal-xl .list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-xl .list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-xl .list-group-item.active{margin-top:0}.list-group-horizontal-xl .list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-xl .list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}.list-group-flush .list-group-item{border-right-width:0;border-left-width:0;border-radius:0}.list-group-flush .list-group-item:first-child{border-top-width:0}.list-group-flush:last-child .list-group-item:last-child{border-bottom-width:0}.list-group-item-primary{color:#004085;background-color:#b8daff}.list-group-item-primary.list-group-item-action:focus,.list-group-item-primary.list-group-item-action:hover{color:#004085;background-color:#9fcdff}.list-group-item-primary.list-group-item-action.active{color:#fff;background-color:#004085;border-color:#004085}.list-group-item-secondary{color:#383d41;background-color:#d6d8db}.list-group-item-secondary.list-group-item-action:focus,.list-group-item-secondary.list-group-item-action:hover{color:#383d41;background-color:#c8cbcf}.list-group-item-secondary.list-group-item-action.active{color:#fff;background-color:#383d41;border-color:#383d41}.list-group-item-success{color:#155724;background-color:#c3e6cb}.list-group-item-success.list-group-item-action:focus,.list-group-item-success.list-group-item-action:hover{color:#155724;background-color:#b1dfbb}.list-group-item-success.list-group-item-action.active{color:#fff;background-color:#155724;border-color:#155724}.list-group-item-info{color:#0c5460;background-color:#bee5eb}.list-group-item-info.list-group-item-action:focus,.list-group-item-info.list-group-item-action:hover{color:#0c5460;background-color:#abdde5}.list-group-item-info.list-group-item-action.active{color:#fff;background-color:#0c5460;border-color:#0c5460}.list-group-item-warning{color:#856404;background-color:#ffeeba}.list-group-item-warning.list-group-item-action:focus,.list-group-item-warning.list-group-item-action:hover{color:#856404;background-color:#ffe8a1}.list-group-item-warning.list-group-item-action.active{color:#fff;background-color:#856404;border-color:#856404}.list-group-item-danger{color:#721c24;background-color:#f5c6cb}.list-group-item-danger.list-group-item-action:focus,.list-group-item-danger.list-group-item-action:hover{color:#721c24;background-color:#f1b0b7}.list-group-item-danger.list-group-item-action.active{color:#fff;background-color:#721c24;border-color:#721c24}.list-group-item-light{color:#818182;background-color:#fdfdfe}.list-group-item-light.list-group-item-action:focus,.list-group-item-light.list-group-item-action:hover{color:#818182;background-color:#ececf6}.list-group-item-light.list-group-item-action.active{color:#fff;background-color:#818182;border-color:#818182}.list-group-item-dark{color:#1b1e21;background-color:#c6c8ca}.list-group-item-dark.list-group-item-action:focus,.list-group-item-dark.list-group-item-action:hover{color:#1b1e21;background-color:#b9bbbe}.list-group-item-dark.list-group-item-action.active{color:#fff;background-color:#1b1e21;border-color:#1b1e21}.close{float:right;font-size:1.5rem;font-weight:700;line-height:1;color:#000;text-shadow:0 1px 0 #fff;opacity:.5}.close:hover{color:#000;text-decoration:none}.close:not(:disabled):not(.disabled):focus,.close:not(:disabled):not(.disabled):hover{opacity:.75}button.close{padding:0;background-color:transparent;border:0;-webkit-appearance:none;-moz-appearance:none;appearance:none}a.close.disabled{pointer-events:none}.toast{max-width:350px;overflow:hidden;font-size:.875rem;background-color:rgba(255,255,255,.85);background-clip:padding-box;border:1px solid rgba(0,0,0,.1);box-shadow:0 .25rem .75rem rgba(0,0,0,.1);-webkit-backdrop-filter:blur(10px);backdrop-filter:blur(10px);opacity:0;border-radius:.25rem}.toast:not(:last-child){margin-bottom:.75rem}.toast.showing{opacity:1}.toast.show{display:block;opacity:1}.toast.hide{display:none}.toast-header{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;padding:.25rem .75rem;color:#6c757d;background-color:rgba(255,255,255,.85);background-clip:padding-box;border-bottom:1px solid rgba(0,0,0,.05)}.toast-body{padding:.75rem}.modal-open{overflow:hidden}.modal-open .modal{overflow-x:hidden;overflow-y:auto}.modal{position:fixed;top:0;left:0;z-index:1050;display:none;width:100%;height:100%;overflow:hidden;outline:0}.modal-dialog{position:relative;width:auto;margin:.5rem;pointer-events:none}.modal.fade .modal-dialog{transition:-webkit-transform .3s ease-out;transition:transform .3s ease-out;transition:transform .3s ease-out,-webkit-transform .3s ease-out;-webkit-transform:translate(0,-50px);transform:translate(0,-50px)}@media (prefers-reduced-motion:reduce){.modal.fade .modal-dialog{transition:none}}.modal.show .modal-dialog{-webkit-transform:none;transform:none}.modal.modal-static .modal-dialog{-webkit-transform:scale(1.02);transform:scale(1.02)}.modal-dialog-scrollable{display:-ms-flexbox;display:flex;max-height:calc(100% - 1rem)}.modal-dialog-scrollable .modal-content{max-height:calc(100vh - 1rem);overflow:hidden}.modal-dialog-scrollable .modal-footer,.modal-dialog-scrollable .modal-header{-ms-flex-negative:0;flex-shrink:0}.modal-dialog-scrollable .modal-body{overflow-y:auto}.modal-dialog-centered{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;min-height:calc(100% - 1rem)}.modal-dialog-centered::before{display:block;height:calc(100vh - 1rem);content:""}.modal-dialog-centered.modal-dialog-scrollable{-ms-flex-direction:column;flex-direction:column;-ms-flex-pack:center;justify-content:center;height:100%}.modal-dialog-centered.modal-dialog-scrollable .modal-content{max-height:none}.modal-dialog-centered.modal-dialog-scrollable::before{content:none}.modal-content{position:relative;display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;width:100%;pointer-events:auto;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.2);border-radius:.3rem;outline:0}.modal-backdrop{position:fixed;top:0;left:0;z-index:1040;width:100vw;height:100vh;background-color:#000}.modal-backdrop.fade{opacity:0}.modal-backdrop.show{opacity:.5}.modal-header{display:-ms-flexbox;display:flex;-ms-flex-align:start;align-items:flex-start;-ms-flex-pack:justify;justify-content:space-between;padding:1rem 1rem;border-bottom:1px solid #dee2e6;border-top-left-radius:calc(.3rem - 1px);border-top-right-radius:calc(.3rem - 1px)}.modal-header .close{padding:1rem 1rem;margin:-1rem -1rem -1rem auto}.modal-title{margin-bottom:0;line-height:1.5}.modal-body{position:relative;-ms-flex:1 1 auto;flex:1 1 auto;padding:1rem}.modal-footer{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-align:center;align-items:center;-ms-flex-pack:end;justify-content:flex-end;padding:.75rem;border-top:1px solid #dee2e6;border-bottom-right-radius:calc(.3rem - 1px);border-bottom-left-radius:calc(.3rem - 1px)}.modal-footer>*{margin:.25rem}.modal-scrollbar-measure{position:absolute;top:-9999px;width:50px;height:50px;overflow:scroll}@media (min-width:576px){.modal-dialog{max-width:500px;margin:1.75rem auto}.modal-dialog-scrollable{max-height:calc(100% - 3.5rem)}.modal-dialog-scrollable .modal-content{max-height:calc(100vh - 3.5rem)}.modal-dialog-centered{min-height:calc(100% - 3.5rem)}.modal-dialog-centered::before{height:calc(100vh - 3.5rem)}.modal-sm{max-width:300px}}@media (min-width:992px){.modal-lg,.modal-xl{max-width:800px}}@media (min-width:1200px){.modal-xl{max-width:1140px}}.tooltip{position:absolute;z-index:1070;display:block;margin:0;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;white-space:normal;line-break:auto;font-size:.875rem;word-wrap:break-word;opacity:0}.tooltip.show{opacity:.9}.tooltip .arrow{position:absolute;display:block;width:.8rem;height:.4rem}.tooltip .arrow::before{position:absolute;content:"";border-color:transparent;border-style:solid}.bs-tooltip-auto[x-placement^=top],.bs-tooltip-top{padding:.4rem 0}.bs-tooltip-auto[x-placement^=top] .arrow,.bs-tooltip-top .arrow{bottom:0}.bs-tooltip-auto[x-placement^=top] .arrow::before,.bs-tooltip-top .arrow::before{top:0;border-width:.4rem .4rem 0;border-top-color:#000}.bs-tooltip-auto[x-placement^=right],.bs-tooltip-right{padding:0 .4rem}.bs-tooltip-auto[x-placement^=right] .arrow,.bs-tooltip-right .arrow{left:0;width:.4rem;height:.8rem}.bs-tooltip-auto[x-placement^=right] .arrow::before,.bs-tooltip-right .arrow::before{right:0;border-width:.4rem .4rem .4rem 0;border-right-color:#000}.bs-tooltip-auto[x-placement^=bottom],.bs-tooltip-bottom{padding:.4rem 0}.bs-tooltip-auto[x-placement^=bottom] .arrow,.bs-tooltip-bottom .arrow{top:0}.bs-tooltip-auto[x-placement^=bottom] .arrow::before,.bs-tooltip-bottom .arrow::before{bottom:0;border-width:0 .4rem .4rem;border-bottom-color:#000}.bs-tooltip-auto[x-placement^=left],.bs-tooltip-left{padding:0 .4rem}.bs-tooltip-auto[x-placement^=left] .arrow,.bs-tooltip-left .arrow{right:0;width:.4rem;height:.8rem}.bs-tooltip-auto[x-placement^=left] .arrow::before,.bs-tooltip-left .arrow::before{left:0;border-width:.4rem 0 .4rem .4rem;border-left-color:#000}.tooltip-inner{max-width:200px;padding:.25rem .5rem;color:#fff;text-align:center;background-color:#000;border-radius:.25rem}.popover{position:absolute;top:0;left:0;z-index:1060;display:block;max-width:276px;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;white-space:normal;line-break:auto;font-size:.875rem;word-wrap:break-word;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.2);border-radius:.3rem}.popover .arrow{position:absolute;display:block;width:1rem;height:.5rem;margin:0 .3rem}.popover .arrow::after,.popover .arrow::before{position:absolute;display:block;content:"";border-color:transparent;border-style:solid}.bs-popover-auto[x-placement^=top],.bs-popover-top{margin-bottom:.5rem}.bs-popover-auto[x-placement^=top]>.arrow,.bs-popover-top>.arrow{bottom:calc(-.5rem - 1px)}.bs-popover-auto[x-placement^=top]>.arrow::before,.bs-popover-top>.arrow::before{bottom:0;border-width:.5rem .5rem 0;border-top-color:rgba(0,0,0,.25)}.bs-popover-auto[x-placement^=top]>.arrow::after,.bs-popover-top>.arrow::after{bottom:1px;border-width:.5rem .5rem 0;border-top-color:#fff}.bs-popover-auto[x-placement^=right],.bs-popover-right{margin-left:.5rem}.bs-popover-auto[x-placement^=right]>.arrow,.bs-popover-right>.arrow{left:calc(-.5rem - 1px);width:.5rem;height:1rem;margin:.3rem 0}.bs-popover-auto[x-placement^=right]>.arrow::before,.bs-popover-right>.arrow::before{left:0;border-width:.5rem .5rem .5rem 0;border-right-color:rgba(0,0,0,.25)}.bs-popover-auto[x-placement^=right]>.arrow::after,.bs-popover-right>.arrow::after{left:1px;border-width:.5rem .5rem .5rem 0;border-right-color:#fff}.bs-popover-auto[x-placement^=bottom],.bs-popover-bottom{margin-top:.5rem}.bs-popover-auto[x-placement^=bottom]>.arrow,.bs-popover-bottom>.arrow{top:calc(-.5rem - 1px)}.bs-popover-auto[x-placement^=bottom]>.arrow::before,.bs-popover-bottom>.arrow::before{top:0;border-width:0 .5rem .5rem .5rem;border-bottom-color:rgba(0,0,0,.25)}.bs-popover-auto[x-placement^=bottom]>.arrow::after,.bs-popover-bottom>.arrow::after{top:1px;border-width:0 .5rem .5rem .5rem;border-bottom-color:#fff}.bs-popover-auto[x-placement^=bottom] .popover-header::before,.bs-popover-bottom .popover-header::before{position:absolute;top:0;left:50%;display:block;width:1rem;margin-left:-.5rem;content:"";border-bottom:1px solid #f7f7f7}.bs-popover-auto[x-placement^=left],.bs-popover-left{margin-right:.5rem}.bs-popover-auto[x-placement^=left]>.arrow,.bs-popover-left>.arrow{right:calc(-.5rem - 1px);width:.5rem;height:1rem;margin:.3rem 0}.bs-popover-auto[x-placement^=left]>.arrow::before,.bs-popover-left>.arrow::before{right:0;border-width:.5rem 0 .5rem .5rem;border-left-color:rgba(0,0,0,.25)}.bs-popover-auto[x-placement^=left]>.arrow::after,.bs-popover-left>.arrow::after{right:1px;border-width:.5rem 0 .5rem .5rem;border-left-color:#fff}.popover-header{padding:.5rem .75rem;margin-bottom:0;font-size:1rem;background-color:#f7f7f7;border-bottom:1px solid #ebebeb;border-top-left-radius:calc(.3rem - 1px);border-top-right-radius:calc(.3rem - 1px)}.popover-header:empty{display:none}.popover-body{padding:.5rem .75rem;color:#212529}.carousel{position:relative}.carousel.pointer-event{-ms-touch-action:pan-y;touch-action:pan-y}.carousel-inner{position:relative;width:100%;overflow:hidden}.carousel-inner::after{display:block;clear:both;content:""}.carousel-item{position:relative;display:none;float:left;width:100%;margin-right:-100%;-webkit-backface-visibility:hidden;backface-visibility:hidden;transition:-webkit-transform .6s ease-in-out;transition:transform .6s ease-in-out;transition:transform .6s ease-in-out,-webkit-transform .6s ease-in-out}@media (prefers-reduced-motion:reduce){.carousel-item{transition:none}}.carousel-item-next,.carousel-item-prev,.carousel-item.active{display:block}.active.carousel-item-right,.carousel-item-next:not(.carousel-item-left){-webkit-transform:translateX(100%);transform:translateX(100%)}.active.carousel-item-left,.carousel-item-prev:not(.carousel-item-right){-webkit-transform:translateX(-100%);transform:translateX(-100%)}.carousel-fade .carousel-item{opacity:0;transition-property:opacity;-webkit-transform:none;transform:none}.carousel-fade .carousel-item-next.carousel-item-left,.carousel-fade .carousel-item-prev.carousel-item-right,.carousel-fade .carousel-item.active{z-index:1;opacity:1}.carousel-fade .active.carousel-item-left,.carousel-fade .active.carousel-item-right{z-index:0;opacity:0;transition:opacity 0s .6s}@media (prefers-reduced-motion:reduce){.carousel-fade .active.carousel-item-left,.carousel-fade .active.carousel-item-right{transition:none}}.carousel-control-next,.carousel-control-prev{position:absolute;top:0;bottom:0;z-index:1;display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center;width:15%;color:#fff;text-align:center;opacity:.5;transition:opacity .15s ease}@media (prefers-reduced-motion:reduce){.carousel-control-next,.carousel-control-prev{transition:none}}.carousel-control-next:focus,.carousel-control-next:hover,.carousel-control-prev:focus,.carousel-control-prev:hover{color:#fff;text-decoration:none;outline:0;opacity:.9}.carousel-control-prev{left:0}.carousel-control-next{right:0}.carousel-control-next-icon,.carousel-control-prev-icon{display:inline-block;width:20px;height:20px;background:no-repeat 50%/100% 100%}.carousel-control-prev-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath d='M5.25 0l-4 4 4 4 1.5-1.5L4.25 4l2.5-2.5L5.25 0z'/%3e%3c/svg%3e")}.carousel-control-next-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath d='M2.75 0l-1.5 1.5L3.75 4l-2.5 2.5L2.75 8l4-4-4-4z'/%3e%3c/svg%3e")}.carousel-indicators{position:absolute;right:0;bottom:0;left:0;z-index:15;display:-ms-flexbox;display:flex;-ms-flex-pack:center;justify-content:center;padding-left:0;margin-right:15%;margin-left:15%;list-style:none}.carousel-indicators li{box-sizing:content-box;-ms-flex:0 1 auto;flex:0 1 auto;width:30px;height:3px;margin-right:3px;margin-left:3px;text-indent:-999px;cursor:pointer;background-color:#fff;background-clip:padding-box;border-top:10px solid transparent;border-bottom:10px solid transparent;opacity:.5;transition:opacity .6s ease}@media (prefers-reduced-motion:reduce){.carousel-indicators li{transition:none}}.carousel-indicators .active{opacity:1}.carousel-caption{position:absolute;right:15%;bottom:20px;left:15%;z-index:10;padding-top:20px;padding-bottom:20px;color:#fff;text-align:center}@-webkit-keyframes spinner-border{to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}@keyframes spinner-border{to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}.spinner-border{display:inline-block;width:2rem;height:2rem;vertical-align:text-bottom;border:.25em solid currentColor;border-right-color:transparent;border-radius:50%;-webkit-animation:spinner-border .75s linear infinite;animation:spinner-border .75s linear infinite}.spinner-border-sm{width:1rem;height:1rem;border-width:.2em}@-webkit-keyframes spinner-grow{0%{-webkit-transform:scale(0);transform:scale(0)}50%{opacity:1}}@keyframes spinner-grow{0%{-webkit-transform:scale(0);transform:scale(0)}50%{opacity:1}}.spinner-grow{display:inline-block;width:2rem;height:2rem;vertical-align:text-bottom;background-color:currentColor;border-radius:50%;opacity:0;-webkit-animation:spinner-grow .75s linear infinite;animation:spinner-grow .75s linear infinite}.spinner-grow-sm{width:1rem;height:1rem}.align-baseline{vertical-align:baseline!important}.align-top{vertical-align:top!important}.align-middle{vertical-align:middle!important}.align-bottom{vertical-align:bottom!important}.align-text-bottom{vertical-align:text-bottom!important}.align-text-top{vertical-align:text-top!important}.bg-primary{background-color:#007bff!important}a.bg-primary:focus,a.bg-primary:hover,button.bg-primary:focus,button.bg-primary:hover{background-color:#0062cc!important}.bg-secondary{background-color:#6c757d!important}a.bg-secondary:focus,a.bg-secondary:hover,button.bg-secondary:focus,button.bg-secondary:hover{background-color:#545b62!important}.bg-success{background-color:#28a745!important}a.bg-success:focus,a.bg-success:hover,button.bg-success:focus,button.bg-success:hover{background-color:#1e7e34!important}.bg-info{background-color:#17a2b8!important}a.bg-info:focus,a.bg-info:hover,button.bg-info:focus,button.bg-info:hover{background-color:#117a8b!important}.bg-warning{background-color:#ffc107!important}a.bg-warning:focus,a.bg-warning:hover,button.bg-warning:focus,button.bg-warning:hover{background-color:#d39e00!important}.bg-danger{background-color:#dc3545!important}a.bg-danger:focus,a.bg-danger:hover,button.bg-danger:focus,button.bg-danger:hover{background-color:#bd2130!important}.bg-light{background-color:#f8f9fa!important}a.bg-light:focus,a.bg-light:hover,button.bg-light:focus,button.bg-light:hover{background-color:#dae0e5!important}.bg-dark{background-color:#343a40!important}a.bg-dark:focus,a.bg-dark:hover,button.bg-dark:focus,button.bg-dark:hover{background-color:#1d2124!important}.bg-white{background-color:#fff!important}.bg-transparent{background-color:transparent!important}.border{border:1px solid #dee2e6!important}.border-top{border-top:1px solid #dee2e6!important}.border-right{border-right:1px solid #dee2e6!important}.border-bottom{border-bottom:1px solid #dee2e6!important}.border-left{border-left:1px solid #dee2e6!important}.border-0{border:0!important}.border-top-0{border-top:0!important}.border-right-0{border-right:0!important}.border-bottom-0{border-bottom:0!important}.border-left-0{border-left:0!important}.border-primary{border-color:#007bff!important}.border-secondary{border-color:#6c757d!important}.border-success{border-color:#28a745!important}.border-info{border-color:#17a2b8!important}.border-warning{border-color:#ffc107!important}.border-danger{border-color:#dc3545!important}.border-light{border-color:#f8f9fa!important}.border-dark{border-color:#343a40!important}.border-white{border-color:#fff!important}.rounded-sm{border-radius:.2rem!important}.rounded{border-radius:.25rem!important}.rounded-top{border-top-left-radius:.25rem!important;border-top-right-radius:.25rem!important}.rounded-right{border-top-right-radius:.25rem!important;border-bottom-right-radius:.25rem!important}.rounded-bottom{border-bottom-right-radius:.25rem!important;border-bottom-left-radius:.25rem!important}.rounded-left{border-top-left-radius:.25rem!important;border-bottom-left-radius:.25rem!important}.rounded-lg{border-radius:.3rem!important}.rounded-circle{border-radius:50%!important}.rounded-pill{border-radius:50rem!important}.rounded-0{border-radius:0!important}.clearfix::after{display:block;clear:both;content:""}.d-none{display:none!important}.d-inline{display:inline!important}.d-inline-block{display:inline-block!important}.d-block{display:block!important}.d-table{display:table!important}.d-table-row{display:table-row!important}.d-table-cell{display:table-cell!important}.d-flex{display:-ms-flexbox!important;display:flex!important}.d-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}@media (min-width:576px){.d-sm-none{display:none!important}.d-sm-inline{display:inline!important}.d-sm-inline-block{display:inline-block!important}.d-sm-block{display:block!important}.d-sm-table{display:table!important}.d-sm-table-row{display:table-row!important}.d-sm-table-cell{display:table-cell!important}.d-sm-flex{display:-ms-flexbox!important;display:flex!important}.d-sm-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:768px){.d-md-none{display:none!important}.d-md-inline{display:inline!important}.d-md-inline-block{display:inline-block!important}.d-md-block{display:block!important}.d-md-table{display:table!important}.d-md-table-row{display:table-row!important}.d-md-table-cell{display:table-cell!important}.d-md-flex{display:-ms-flexbox!important;display:flex!important}.d-md-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:992px){.d-lg-none{display:none!important}.d-lg-inline{display:inline!important}.d-lg-inline-block{display:inline-block!important}.d-lg-block{display:block!important}.d-lg-table{display:table!important}.d-lg-table-row{display:table-row!important}.d-lg-table-cell{display:table-cell!important}.d-lg-flex{display:-ms-flexbox!important;display:flex!important}.d-lg-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:1200px){.d-xl-none{display:none!important}.d-xl-inline{display:inline!important}.d-xl-inline-block{display:inline-block!important}.d-xl-block{display:block!important}.d-xl-table{display:table!important}.d-xl-table-row{display:table-row!important}.d-xl-table-cell{display:table-cell!important}.d-xl-flex{display:-ms-flexbox!important;display:flex!important}.d-xl-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media print{.d-print-none{display:none!important}.d-print-inline{display:inline!important}.d-print-inline-block{display:inline-block!important}.d-print-block{display:block!important}.d-print-table{display:table!important}.d-print-table-row{display:table-row!important}.d-print-table-cell{display:table-cell!important}.d-print-flex{display:-ms-flexbox!important;display:flex!important}.d-print-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}.embed-responsive{position:relative;display:block;width:100%;padding:0;overflow:hidden}.embed-responsive::before{display:block;content:""}.embed-responsive .embed-responsive-item,.embed-responsive embed,.embed-responsive iframe,.embed-responsive object,.embed-responsive video{position:absolute;top:0;bottom:0;left:0;width:100%;height:100%;border:0}.embed-responsive-21by9::before{padding-top:42.857143%}.embed-responsive-16by9::before{padding-top:56.25%}.embed-responsive-4by3::before{padding-top:75%}.embed-responsive-1by1::before{padding-top:100%}.flex-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-center{-ms-flex-align:center!important;align-items:center!important}.align-items-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}@media (min-width:576px){.flex-sm-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-sm-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-sm-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-sm-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-sm-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-sm-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-sm-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-sm-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-sm-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-sm-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-sm-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-sm-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-sm-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-sm-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-sm-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-sm-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-sm-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-sm-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-sm-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-sm-center{-ms-flex-align:center!important;align-items:center!important}.align-items-sm-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-sm-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-sm-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-sm-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-sm-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-sm-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-sm-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-sm-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-sm-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-sm-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-sm-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-sm-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-sm-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-sm-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}@media (min-width:768px){.flex-md-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-md-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-md-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-md-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-md-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-md-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-md-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-md-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-md-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-md-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-md-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-md-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-md-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-md-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-md-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-md-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-md-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-md-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-md-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-md-center{-ms-flex-align:center!important;align-items:center!important}.align-items-md-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-md-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-md-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-md-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-md-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-md-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-md-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-md-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-md-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-md-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-md-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-md-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-md-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-md-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}@media (min-width:992px){.flex-lg-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-lg-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-lg-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-lg-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-lg-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-lg-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-lg-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-lg-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-lg-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-lg-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-lg-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-lg-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-lg-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-lg-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-lg-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-lg-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-lg-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-lg-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-lg-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-lg-center{-ms-flex-align:center!important;align-items:center!important}.align-items-lg-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-lg-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-lg-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-lg-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-lg-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-lg-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-lg-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-lg-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-lg-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-lg-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-lg-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-lg-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-lg-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-lg-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}@media (min-width:1200px){.flex-xl-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-xl-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-xl-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-xl-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-xl-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-xl-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-xl-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-xl-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-xl-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-xl-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-xl-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-xl-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-xl-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-xl-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-xl-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-xl-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-xl-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-xl-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-xl-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-xl-center{-ms-flex-align:center!important;align-items:center!important}.align-items-xl-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-xl-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-xl-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-xl-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-xl-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-xl-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-xl-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-xl-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-xl-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-xl-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-xl-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-xl-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-xl-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-xl-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}.float-left{float:left!important}.float-right{float:right!important}.float-none{float:none!important}@media (min-width:576px){.float-sm-left{float:left!important}.float-sm-right{float:right!important}.float-sm-none{float:none!important}}@media (min-width:768px){.float-md-left{float:left!important}.float-md-right{float:right!important}.float-md-none{float:none!important}}@media (min-width:992px){.float-lg-left{float:left!important}.float-lg-right{float:right!important}.float-lg-none{float:none!important}}@media (min-width:1200px){.float-xl-left{float:left!important}.float-xl-right{float:right!important}.float-xl-none{float:none!important}}.overflow-auto{overflow:auto!important}.overflow-hidden{overflow:hidden!important}.position-static{position:static!important}.position-relative{position:relative!important}.position-absolute{position:absolute!important}.position-fixed{position:fixed!important}.position-sticky{position:-webkit-sticky!important;position:sticky!important}.fixed-top{position:fixed;top:0;right:0;left:0;z-index:1030}.fixed-bottom{position:fixed;right:0;bottom:0;left:0;z-index:1030}@supports ((position:-webkit-sticky) or (position:sticky)){.sticky-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;overflow:visible;clip:auto;white-space:normal}.shadow-sm{box-shadow:0 .125rem .25rem rgba(0,0,0,.075)!important}.shadow{box-shadow:0 .5rem 1rem rgba(0,0,0,.15)!important}.shadow-lg{box-shadow:0 1rem 3rem rgba(0,0,0,.175)!important}.shadow-none{box-shadow:none!important}.w-25{width:25%!important}.w-50{width:50%!important}.w-75{width:75%!important}.w-100{width:100%!important}.w-auto{width:auto!important}.h-25{height:25%!important}.h-50{height:50%!important}.h-75{height:75%!important}.h-100{height:100%!important}.h-auto{height:auto!important}.mw-100{max-width:100%!important}.mh-100{max-height:100%!important}.min-vw-100{min-width:100vw!important}.min-vh-100{min-height:100vh!important}.vw-100{width:100vw!important}.vh-100{height:100vh!important}.stretched-link::after{position:absolute;top:0;right:0;bottom:0;left:0;z-index:1;pointer-events:auto;content:"";background-color:rgba(0,0,0,0)}.m-0{margin:0!important}.mt-0,.my-0{margin-top:0!important}.mr-0,.mx-0{margin-right:0!important}.mb-0,.my-0{margin-bottom:0!important}.ml-0,.mx-0{margin-left:0!important}.m-1{margin:.25rem!important}.mt-1,.my-1{margin-top:.25rem!important}.mr-1,.mx-1{margin-right:.25rem!important}.mb-1,.my-1{margin-bottom:.25rem!important}.ml-1,.mx-1{margin-left:.25rem!important}.m-2{margin:.5rem!important}.mt-2,.my-2{margin-top:.5rem!important}.mr-2,.mx-2{margin-right:.5rem!important}.mb-2,.my-2{margin-bottom:.5rem!important}.ml-2,.mx-2{margin-left:.5rem!important}.m-3{margin:1rem!important}.mt-3,.my-3{margin-top:1rem!important}.mr-3,.mx-3{margin-right:1rem!important}.mb-3,.my-3{margin-bottom:1rem!important}.ml-3,.mx-3{margin-left:1rem!important}.m-4{margin:1.5rem!important}.mt-4,.my-4{margin-top:1.5rem!important}.mr-4,.mx-4{margin-right:1.5rem!important}.mb-4,.my-4{margin-bottom:1.5rem!important}.ml-4,.mx-4{margin-left:1.5rem!important}.m-5{margin:3rem!important}.mt-5,.my-5{margin-top:3rem!important}.mr-5,.mx-5{margin-right:3rem!important}.mb-5,.my-5{margin-bottom:3rem!important}.ml-5,.mx-5{margin-left:3rem!important}.p-0{padding:0!important}.pt-0,.py-0{padding-top:0!important}.pr-0,.px-0{padding-right:0!important}.pb-0,.py-0{padding-bottom:0!important}.pl-0,.px-0{padding-left:0!important}.p-1{padding:.25rem!important}.pt-1,.py-1{padding-top:.25rem!important}.pr-1,.px-1{padding-right:.25rem!important}.pb-1,.py-1{padding-bottom:.25rem!important}.pl-1,.px-1{padding-left:.25rem!important}.p-2{padding:.5rem!important}.pt-2,.py-2{padding-top:.5rem!important}.pr-2,.px-2{padding-right:.5rem!important}.pb-2,.py-2{padding-bottom:.5rem!important}.pl-2,.px-2{padding-left:.5rem!important}.p-3{padding:1rem!important}.pt-3,.py-3{padding-top:1rem!important}.pr-3,.px-3{padding-right:1rem!important}.pb-3,.py-3{padding-bottom:1rem!important}.pl-3,.px-3{padding-left:1rem!important}.p-4{padding:1.5rem!important}.pt-4,.py-4{padding-top:1.5rem!important}.pr-4,.px-4{padding-right:1.5rem!important}.pb-4,.py-4{padding-bottom:1.5rem!important}.pl-4,.px-4{padding-left:1.5rem!important}.p-5{padding:3rem!important}.pt-5,.py-5{padding-top:3rem!important}.pr-5,.px-5{padding-right:3rem!important}.pb-5,.py-5{padding-bottom:3rem!important}.pl-5,.px-5{padding-left:3rem!important}.m-n1{margin:-.25rem!important}.mt-n1,.my-n1{margin-top:-.25rem!important}.mr-n1,.mx-n1{margin-right:-.25rem!important}.mb-n1,.my-n1{margin-bottom:-.25rem!important}.ml-n1,.mx-n1{margin-left:-.25rem!important}.m-n2{margin:-.5rem!important}.mt-n2,.my-n2{margin-top:-.5rem!important}.mr-n2,.mx-n2{margin-right:-.5rem!important}.mb-n2,.my-n2{margin-bottom:-.5rem!important}.ml-n2,.mx-n2{margin-left:-.5rem!important}.m-n3{margin:-1rem!important}.mt-n3,.my-n3{margin-top:-1rem!important}.mr-n3,.mx-n3{margin-right:-1rem!important}.mb-n3,.my-n3{margin-bottom:-1rem!important}.ml-n3,.mx-n3{margin-left:-1rem!important}.m-n4{margin:-1.5rem!important}.mt-n4,.my-n4{margin-top:-1.5rem!important}.mr-n4,.mx-n4{margin-right:-1.5rem!important}.mb-n4,.my-n4{margin-bottom:-1.5rem!important}.ml-n4,.mx-n4{margin-left:-1.5rem!important}.m-n5{margin:-3rem!important}.mt-n5,.my-n5{margin-top:-3rem!important}.mr-n5,.mx-n5{margin-right:-3rem!important}.mb-n5,.my-n5{margin-bottom:-3rem!important}.ml-n5,.mx-n5{margin-left:-3rem!important}.m-auto{margin:auto!important}.mt-auto,.my-auto{margin-top:auto!important}.mr-auto,.mx-auto{margin-right:auto!important}.mb-auto,.my-auto{margin-bottom:auto!important}.ml-auto,.mx-auto{margin-left:auto!important}@media (min-width:576px){.m-sm-0{margin:0!important}.mt-sm-0,.my-sm-0{margin-top:0!important}.mr-sm-0,.mx-sm-0{margin-right:0!important}.mb-sm-0,.my-sm-0{margin-bottom:0!important}.ml-sm-0,.mx-sm-0{margin-left:0!important}.m-sm-1{margin:.25rem!important}.mt-sm-1,.my-sm-1{margin-top:.25rem!important}.mr-sm-1,.mx-sm-1{margin-right:.25rem!important}.mb-sm-1,.my-sm-1{margin-bottom:.25rem!important}.ml-sm-1,.mx-sm-1{margin-left:.25rem!important}.m-sm-2{margin:.5rem!important}.mt-sm-2,.my-sm-2{margin-top:.5rem!important}.mr-sm-2,.mx-sm-2{margin-right:.5rem!important}.mb-sm-2,.my-sm-2{margin-bottom:.5rem!important}.ml-sm-2,.mx-sm-2{margin-left:.5rem!important}.m-sm-3{margin:1rem!important}.mt-sm-3,.my-sm-3{margin-top:1rem!important}.mr-sm-3,.mx-sm-3{margin-right:1rem!important}.mb-sm-3,.my-sm-3{margin-bottom:1rem!important}.ml-sm-3,.mx-sm-3{margin-left:1rem!important}.m-sm-4{margin:1.5rem!important}.mt-sm-4,.my-sm-4{margin-top:1.5rem!important}.mr-sm-4,.mx-sm-4{margin-right:1.5rem!important}.mb-sm-4,.my-sm-4{margin-bottom:1.5rem!important}.ml-sm-4,.mx-sm-4{margin-left:1.5rem!important}.m-sm-5{margin:3rem!important}.mt-sm-5,.my-sm-5{margin-top:3rem!important}.mr-sm-5,.mx-sm-5{margin-right:3rem!important}.mb-sm-5,.my-sm-5{margin-bottom:3rem!important}.ml-sm-5,.mx-sm-5{margin-left:3rem!important}.p-sm-0{padding:0!important}.pt-sm-0,.py-sm-0{padding-top:0!important}.pr-sm-0,.px-sm-0{padding-right:0!important}.pb-sm-0,.py-sm-0{padding-bottom:0!important}.pl-sm-0,.px-sm-0{padding-left:0!important}.p-sm-1{padding:.25rem!important}.pt-sm-1,.py-sm-1{padding-top:.25rem!important}.pr-sm-1,.px-sm-1{padding-right:.25rem!important}.pb-sm-1,.py-sm-1{padding-bottom:.25rem!important}.pl-sm-1,.px-sm-1{padding-left:.25rem!important}.p-sm-2{padding:.5rem!important}.pt-sm-2,.py-sm-2{padding-top:.5rem!important}.pr-sm-2,.px-sm-2{padding-right:.5rem!important}.pb-sm-2,.py-sm-2{padding-bottom:.5rem!important}.pl-sm-2,.px-sm-2{padding-left:.5rem!important}.p-sm-3{padding:1rem!important}.pt-sm-3,.py-sm-3{padding-top:1rem!important}.pr-sm-3,.px-sm-3{padding-right:1rem!important}.pb-sm-3,.py-sm-3{padding-bottom:1rem!important}.pl-sm-3,.px-sm-3{padding-left:1rem!important}.p-sm-4{padding:1.5rem!important}.pt-sm-4,.py-sm-4{padding-top:1.5rem!important}.pr-sm-4,.px-sm-4{padding-right:1.5rem!important}.pb-sm-4,.py-sm-4{padding-bottom:1.5rem!important}.pl-sm-4,.px-sm-4{padding-left:1.5rem!important}.p-sm-5{padding:3rem!important}.pt-sm-5,.py-sm-5{padding-top:3rem!important}.pr-sm-5,.px-sm-5{padding-right:3rem!important}.pb-sm-5,.py-sm-5{padding-bottom:3rem!important}.pl-sm-5,.px-sm-5{padding-left:3rem!important}.m-sm-n1{margin:-.25rem!important}.mt-sm-n1,.my-sm-n1{margin-top:-.25rem!important}.mr-sm-n1,.mx-sm-n1{margin-right:-.25rem!important}.mb-sm-n1,.my-sm-n1{margin-bottom:-.25rem!important}.ml-sm-n1,.mx-sm-n1{margin-left:-.25rem!important}.m-sm-n2{margin:-.5rem!important}.mt-sm-n2,.my-sm-n2{margin-top:-.5rem!important}.mr-sm-n2,.mx-sm-n2{margin-right:-.5rem!important}.mb-sm-n2,.my-sm-n2{margin-bottom:-.5rem!important}.ml-sm-n2,.mx-sm-n2{margin-left:-.5rem!important}.m-sm-n3{margin:-1rem!important}.mt-sm-n3,.my-sm-n3{margin-top:-1rem!important}.mr-sm-n3,.mx-sm-n3{margin-right:-1rem!important}.mb-sm-n3,.my-sm-n3{margin-bottom:-1rem!important}.ml-sm-n3,.mx-sm-n3{margin-left:-1rem!important}.m-sm-n4{margin:-1.5rem!important}.mt-sm-n4,.my-sm-n4{margin-top:-1.5rem!important}.mr-sm-n4,.mx-sm-n4{margin-right:-1.5rem!important}.mb-sm-n4,.my-sm-n4{margin-bottom:-1.5rem!important}.ml-sm-n4,.mx-sm-n4{margin-left:-1.5rem!important}.m-sm-n5{margin:-3rem!important}.mt-sm-n5,.my-sm-n5{margin-top:-3rem!important}.mr-sm-n5,.mx-sm-n5{margin-right:-3rem!important}.mb-sm-n5,.my-sm-n5{margin-bottom:-3rem!important}.ml-sm-n5,.mx-sm-n5{margin-left:-3rem!important}.m-sm-auto{margin:auto!important}.mt-sm-auto,.my-sm-auto{margin-top:auto!important}.mr-sm-auto,.mx-sm-auto{margin-right:auto!important}.mb-sm-auto,.my-sm-auto{margin-bottom:auto!important}.ml-sm-auto,.mx-sm-auto{margin-left:auto!important}}@media (min-width:768px){.m-md-0{margin:0!important}.mt-md-0,.my-md-0{margin-top:0!important}.mr-md-0,.mx-md-0{margin-right:0!important}.mb-md-0,.my-md-0{margin-bottom:0!important}.ml-md-0,.mx-md-0{margin-left:0!important}.m-md-1{margin:.25rem!important}.mt-md-1,.my-md-1{margin-top:.25rem!important}.mr-md-1,.mx-md-1{margin-right:.25rem!important}.mb-md-1,.my-md-1{margin-bottom:.25rem!important}.ml-md-1,.mx-md-1{margin-left:.25rem!important}.m-md-2{margin:.5rem!important}.mt-md-2,.my-md-2{margin-top:.5rem!important}.mr-md-2,.mx-md-2{margin-right:.5rem!important}.mb-md-2,.my-md-2{margin-bottom:.5rem!important}.ml-md-2,.mx-md-2{margin-left:.5rem!important}.m-md-3{margin:1rem!important}.mt-md-3,.my-md-3{margin-top:1rem!important}.mr-md-3,.mx-md-3{margin-right:1rem!important}.mb-md-3,.my-md-3{margin-bottom:1rem!important}.ml-md-3,.mx-md-3{margin-left:1rem!important}.m-md-4{margin:1.5rem!important}.mt-md-4,.my-md-4{margin-top:1.5rem!important}.mr-md-4,.mx-md-4{margin-right:1.5rem!important}.mb-md-4,.my-md-4{margin-bottom:1.5rem!important}.ml-md-4,.mx-md-4{margin-left:1.5rem!important}.m-md-5{margin:3rem!important}.mt-md-5,.my-md-5{margin-top:3rem!important}.mr-md-5,.mx-md-5{margin-right:3rem!important}.mb-md-5,.my-md-5{margin-bottom:3rem!important}.ml-md-5,.mx-md-5{margin-left:3rem!important}.p-md-0{padding:0!important}.pt-md-0,.py-md-0{padding-top:0!important}.pr-md-0,.px-md-0{padding-right:0!important}.pb-md-0,.py-md-0{padding-bottom:0!important}.pl-md-0,.px-md-0{padding-left:0!important}.p-md-1{padding:.25rem!important}.pt-md-1,.py-md-1{padding-top:.25rem!important}.pr-md-1,.px-md-1{padding-right:.25rem!important}.pb-md-1,.py-md-1{padding-bottom:.25rem!important}.pl-md-1,.px-md-1{padding-left:.25rem!important}.p-md-2{padding:.5rem!important}.pt-md-2,.py-md-2{padding-top:.5rem!important}.pr-md-2,.px-md-2{padding-right:.5rem!important}.pb-md-2,.py-md-2{padding-bottom:.5rem!important}.pl-md-2,.px-md-2{padding-left:.5rem!important}.p-md-3{padding:1rem!important}.pt-md-3,.py-md-3{padding-top:1rem!important}.pr-md-3,.px-md-3{padding-right:1rem!important}.pb-md-3,.py-md-3{padding-bottom:1rem!important}.pl-md-3,.px-md-3{padding-left:1rem!important}.p-md-4{padding:1.5rem!important}.pt-md-4,.py-md-4{padding-top:1.5rem!important}.pr-md-4,.px-md-4{padding-right:1.5rem!important}.pb-md-4,.py-md-4{padding-bottom:1.5rem!important}.pl-md-4,.px-md-4{padding-left:1.5rem!important}.p-md-5{padding:3rem!important}.pt-md-5,.py-md-5{padding-top:3rem!important}.pr-md-5,.px-md-5{padding-right:3rem!important}.pb-md-5,.py-md-5{padding-bottom:3rem!important}.pl-md-5,.px-md-5{padding-left:3rem!important}.m-md-n1{margin:-.25rem!important}.mt-md-n1,.my-md-n1{margin-top:-.25rem!important}.mr-md-n1,.mx-md-n1{margin-right:-.25rem!important}.mb-md-n1,.my-md-n1{margin-bottom:-.25rem!important}.ml-md-n1,.mx-md-n1{margin-left:-.25rem!important}.m-md-n2{margin:-.5rem!important}.mt-md-n2,.my-md-n2{margin-top:-.5rem!important}.mr-md-n2,.mx-md-n2{margin-right:-.5rem!important}.mb-md-n2,.my-md-n2{margin-bottom:-.5rem!important}.ml-md-n2,.mx-md-n2{margin-left:-.5rem!important}.m-md-n3{margin:-1rem!important}.mt-md-n3,.my-md-n3{margin-top:-1rem!important}.mr-md-n3,.mx-md-n3{margin-right:-1rem!important}.mb-md-n3,.my-md-n3{margin-bottom:-1rem!important}.ml-md-n3,.mx-md-n3{margin-left:-1rem!important}.m-md-n4{margin:-1.5rem!important}.mt-md-n4,.my-md-n4{margin-top:-1.5rem!important}.mr-md-n4,.mx-md-n4{margin-right:-1.5rem!important}.mb-md-n4,.my-md-n4{margin-bottom:-1.5rem!important}.ml-md-n4,.mx-md-n4{margin-left:-1.5rem!important}.m-md-n5{margin:-3rem!important}.mt-md-n5,.my-md-n5{margin-top:-3rem!important}.mr-md-n5,.mx-md-n5{margin-right:-3rem!important}.mb-md-n5,.my-md-n5{margin-bottom:-3rem!important}.ml-md-n5,.mx-md-n5{margin-left:-3rem!important}.m-md-auto{margin:auto!important}.mt-md-auto,.my-md-auto{margin-top:auto!important}.mr-md-auto,.mx-md-auto{margin-right:auto!important}.mb-md-auto,.my-md-auto{margin-bottom:auto!important}.ml-md-auto,.mx-md-auto{margin-left:auto!important}}@media (min-width:992px){.m-lg-0{margin:0!important}.mt-lg-0,.my-lg-0{margin-top:0!important}.mr-lg-0,.mx-lg-0{margin-right:0!important}.mb-lg-0,.my-lg-0{margin-bottom:0!important}.ml-lg-0,.mx-lg-0{margin-left:0!important}.m-lg-1{margin:.25rem!important}.mt-lg-1,.my-lg-1{margin-top:.25rem!important}.mr-lg-1,.mx-lg-1{margin-right:.25rem!important}.mb-lg-1,.my-lg-1{margin-bottom:.25rem!important}.ml-lg-1,.mx-lg-1{margin-left:.25rem!important}.m-lg-2{margin:.5rem!important}.mt-lg-2,.my-lg-2{margin-top:.5rem!important}.mr-lg-2,.mx-lg-2{margin-right:.5rem!important}.mb-lg-2,.my-lg-2{margin-bottom:.5rem!important}.ml-lg-2,.mx-lg-2{margin-left:.5rem!important}.m-lg-3{margin:1rem!important}.mt-lg-3,.my-lg-3{margin-top:1rem!important}.mr-lg-3,.mx-lg-3{margin-right:1rem!important}.mb-lg-3,.my-lg-3{margin-bottom:1rem!important}.ml-lg-3,.mx-lg-3{margin-left:1rem!important}.m-lg-4{margin:1.5rem!important}.mt-lg-4,.my-lg-4{margin-top:1.5rem!important}.mr-lg-4,.mx-lg-4{margin-right:1.5rem!important}.mb-lg-4,.my-lg-4{margin-bottom:1.5rem!important}.ml-lg-4,.mx-lg-4{margin-left:1.5rem!important}.m-lg-5{margin:3rem!important}.mt-lg-5,.my-lg-5{margin-top:3rem!important}.mr-lg-5,.mx-lg-5{margin-right:3rem!important}.mb-lg-5,.my-lg-5{margin-bottom:3rem!important}.ml-lg-5,.mx-lg-5{margin-left:3rem!important}.p-lg-0{padding:0!important}.pt-lg-0,.py-lg-0{padding-top:0!important}.pr-lg-0,.px-lg-0{padding-right:0!important}.pb-lg-0,.py-lg-0{padding-bottom:0!important}.pl-lg-0,.px-lg-0{padding-left:0!important}.p-lg-1{padding:.25rem!important}.pt-lg-1,.py-lg-1{padding-top:.25rem!important}.pr-lg-1,.px-lg-1{padding-right:.25rem!important}.pb-lg-1,.py-lg-1{padding-bottom:.25rem!important}.pl-lg-1,.px-lg-1{padding-left:.25rem!important}.p-lg-2{padding:.5rem!important}.pt-lg-2,.py-lg-2{padding-top:.5rem!important}.pr-lg-2,.px-lg-2{padding-right:.5rem!important}.pb-lg-2,.py-lg-2{padding-bottom:.5rem!important}.pl-lg-2,.px-lg-2{padding-left:.5rem!important}.p-lg-3{padding:1rem!important}.pt-lg-3,.py-lg-3{padding-top:1rem!important}.pr-lg-3,.px-lg-3{padding-right:1rem!important}.pb-lg-3,.py-lg-3{padding-bottom:1rem!important}.pl-lg-3,.px-lg-3{padding-left:1rem!important}.p-lg-4{padding:1.5rem!important}.pt-lg-4,.py-lg-4{padding-top:1.5rem!important}.pr-lg-4,.px-lg-4{padding-right:1.5rem!important}.pb-lg-4,.py-lg-4{padding-bottom:1.5rem!important}.pl-lg-4,.px-lg-4{padding-left:1.5rem!important}.p-lg-5{padding:3rem!important}.pt-lg-5,.py-lg-5{padding-top:3rem!important}.pr-lg-5,.px-lg-5{padding-right:3rem!important}.pb-lg-5,.py-lg-5{padding-bottom:3rem!important}.pl-lg-5,.px-lg-5{padding-left:3rem!important}.m-lg-n1{margin:-.25rem!important}.mt-lg-n1,.my-lg-n1{margin-top:-.25rem!important}.mr-lg-n1,.mx-lg-n1{margin-right:-.25rem!important}.mb-lg-n1,.my-lg-n1{margin-bottom:-.25rem!important}.ml-lg-n1,.mx-lg-n1{margin-left:-.25rem!important}.m-lg-n2{margin:-.5rem!important}.mt-lg-n2,.my-lg-n2{margin-top:-.5rem!important}.mr-lg-n2,.mx-lg-n2{margin-right:-.5rem!important}.mb-lg-n2,.my-lg-n2{margin-bottom:-.5rem!important}.ml-lg-n2,.mx-lg-n2{margin-left:-.5rem!important}.m-lg-n3{margin:-1rem!important}.mt-lg-n3,.my-lg-n3{margin-top:-1rem!important}.mr-lg-n3,.mx-lg-n3{margin-right:-1rem!important}.mb-lg-n3,.my-lg-n3{margin-bottom:-1rem!important}.ml-lg-n3,.mx-lg-n3{margin-left:-1rem!important}.m-lg-n4{margin:-1.5rem!important}.mt-lg-n4,.my-lg-n4{margin-top:-1.5rem!important}.mr-lg-n4,.mx-lg-n4{margin-right:-1.5rem!important}.mb-lg-n4,.my-lg-n4{margin-bottom:-1.5rem!important}.ml-lg-n4,.mx-lg-n4{margin-left:-1.5rem!important}.m-lg-n5{margin:-3rem!important}.mt-lg-n5,.my-lg-n5{margin-top:-3rem!important}.mr-lg-n5,.mx-lg-n5{margin-right:-3rem!important}.mb-lg-n5,.my-lg-n5{margin-bottom:-3rem!important}.ml-lg-n5,.mx-lg-n5{margin-left:-3rem!important}.m-lg-auto{margin:auto!important}.mt-lg-auto,.my-lg-auto{margin-top:auto!important}.mr-lg-auto,.mx-lg-auto{margin-right:auto!important}.mb-lg-auto,.my-lg-auto{margin-bottom:auto!important}.ml-lg-auto,.mx-lg-auto{margin-left:auto!important}}@media (min-width:1200px){.m-xl-0{margin:0!important}.mt-xl-0,.my-xl-0{margin-top:0!important}.mr-xl-0,.mx-xl-0{margin-right:0!important}.mb-xl-0,.my-xl-0{margin-bottom:0!important}.ml-xl-0,.mx-xl-0{margin-left:0!important}.m-xl-1{margin:.25rem!important}.mt-xl-1,.my-xl-1{margin-top:.25rem!important}.mr-xl-1,.mx-xl-1{margin-right:.25rem!important}.mb-xl-1,.my-xl-1{margin-bottom:.25rem!important}.ml-xl-1,.mx-xl-1{margin-left:.25rem!important}.m-xl-2{margin:.5rem!important}.mt-xl-2,.my-xl-2{margin-top:.5rem!important}.mr-xl-2,.mx-xl-2{margin-right:.5rem!important}.mb-xl-2,.my-xl-2{margin-bottom:.5rem!important}.ml-xl-2,.mx-xl-2{margin-left:.5rem!important}.m-xl-3{margin:1rem!important}.mt-xl-3,.my-xl-3{margin-top:1rem!important}.mr-xl-3,.mx-xl-3{margin-right:1rem!important}.mb-xl-3,.my-xl-3{margin-bottom:1rem!important}.ml-xl-3,.mx-xl-3{margin-left:1rem!important}.m-xl-4{margin:1.5rem!important}.mt-xl-4,.my-xl-4{margin-top:1.5rem!important}.mr-xl-4,.mx-xl-4{margin-right:1.5rem!important}.mb-xl-4,.my-xl-4{margin-bottom:1.5rem!important}.ml-xl-4,.mx-xl-4{margin-left:1.5rem!important}.m-xl-5{margin:3rem!important}.mt-xl-5,.my-xl-5{margin-top:3rem!important}.mr-xl-5,.mx-xl-5{margin-right:3rem!important}.mb-xl-5,.my-xl-5{margin-bottom:3rem!important}.ml-xl-5,.mx-xl-5{margin-left:3rem!important}.p-xl-0{padding:0!important}.pt-xl-0,.py-xl-0{padding-top:0!important}.pr-xl-0,.px-xl-0{padding-right:0!important}.pb-xl-0,.py-xl-0{padding-bottom:0!important}.pl-xl-0,.px-xl-0{padding-left:0!important}.p-xl-1{padding:.25rem!important}.pt-xl-1,.py-xl-1{padding-top:.25rem!important}.pr-xl-1,.px-xl-1{padding-right:.25rem!important}.pb-xl-1,.py-xl-1{padding-bottom:.25rem!important}.pl-xl-1,.px-xl-1{padding-left:.25rem!important}.p-xl-2{padding:.5rem!important}.pt-xl-2,.py-xl-2{padding-top:.5rem!important}.pr-xl-2,.px-xl-2{padding-right:.5rem!important}.pb-xl-2,.py-xl-2{padding-bottom:.5rem!important}.pl-xl-2,.px-xl-2{padding-left:.5rem!important}.p-xl-3{padding:1rem!important}.pt-xl-3,.py-xl-3{padding-top:1rem!important}.pr-xl-3,.px-xl-3{padding-right:1rem!important}.pb-xl-3,.py-xl-3{padding-bottom:1rem!important}.pl-xl-3,.px-xl-3{padding-left:1rem!important}.p-xl-4{padding:1.5rem!important}.pt-xl-4,.py-xl-4{padding-top:1.5rem!important}.pr-xl-4,.px-xl-4{padding-right:1.5rem!important}.pb-xl-4,.py-xl-4{padding-bottom:1.5rem!important}.pl-xl-4,.px-xl-4{padding-left:1.5rem!important}.p-xl-5{padding:3rem!important}.pt-xl-5,.py-xl-5{padding-top:3rem!important}.pr-xl-5,.px-xl-5{padding-right:3rem!important}.pb-xl-5,.py-xl-5{padding-bottom:3rem!important}.pl-xl-5,.px-xl-5{padding-left:3rem!important}.m-xl-n1{margin:-.25rem!important}.mt-xl-n1,.my-xl-n1{margin-top:-.25rem!important}.mr-xl-n1,.mx-xl-n1{margin-right:-.25rem!important}.mb-xl-n1,.my-xl-n1{margin-bottom:-.25rem!important}.ml-xl-n1,.mx-xl-n1{margin-left:-.25rem!important}.m-xl-n2{margin:-.5rem!important}.mt-xl-n2,.my-xl-n2{margin-top:-.5rem!important}.mr-xl-n2,.mx-xl-n2{margin-right:-.5rem!important}.mb-xl-n2,.my-xl-n2{margin-bottom:-.5rem!important}.ml-xl-n2,.mx-xl-n2{margin-left:-.5rem!important}.m-xl-n3{margin:-1rem!important}.mt-xl-n3,.my-xl-n3{margin-top:-1rem!important}.mr-xl-n3,.mx-xl-n3{margin-right:-1rem!important}.mb-xl-n3,.my-xl-n3{margin-bottom:-1rem!important}.ml-xl-n3,.mx-xl-n3{margin-left:-1rem!important}.m-xl-n4{margin:-1.5rem!important}.mt-xl-n4,.my-xl-n4{margin-top:-1.5rem!important}.mr-xl-n4,.mx-xl-n4{margin-right:-1.5rem!important}.mb-xl-n4,.my-xl-n4{margin-bottom:-1.5rem!important}.ml-xl-n4,.mx-xl-n4{margin-left:-1.5rem!important}.m-xl-n5{margin:-3rem!important}.mt-xl-n5,.my-xl-n5{margin-top:-3rem!important}.mr-xl-n5,.mx-xl-n5{margin-right:-3rem!important}.mb-xl-n5,.my-xl-n5{margin-bottom:-3rem!important}.ml-xl-n5,.mx-xl-n5{margin-left:-3rem!important}.m-xl-auto{margin:auto!important}.mt-xl-auto,.my-xl-auto{margin-top:auto!important}.mr-xl-auto,.mx-xl-auto{margin-right:auto!important}.mb-xl-auto,.my-xl-auto{margin-bottom:auto!important}.ml-xl-auto,.mx-xl-auto{margin-left:auto!important}}.text-monospace{font-family:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace!important}.text-justify{text-align:justify!important}.text-wrap{white-space:normal!important}.text-nowrap{white-space:nowrap!important}.text-truncate{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.text-left{text-align:left!important}.text-right{text-align:right!important}.text-center{text-align:center!important}@media (min-width:576px){.text-sm-left{text-align:left!important}.text-sm-right{text-align:right!important}.text-sm-center{text-align:center!important}}@media (min-width:768px){.text-md-left{text-align:left!important}.text-md-right{text-align:right!important}.text-md-center{text-align:center!important}}@media (min-width:992px){.text-lg-left{text-align:left!important}.text-lg-right{text-align:right!important}.text-lg-center{text-align:center!important}}@media (min-width:1200px){.text-xl-left{text-align:left!important}.text-xl-right{text-align:right!important}.text-xl-center{text-align:center!important}}.text-lowercase{text-transform:lowercase!important}.text-uppercase{text-transform:uppercase!important}.text-capitalize{text-transform:capitalize!important}.font-weight-light{font-weight:300!important}.font-weight-lighter{font-weight:lighter!important}.font-weight-normal{font-weight:400!important}.font-weight-bold{font-weight:700!important}.font-weight-bolder{font-weight:bolder!important}.font-italic{font-style:italic!important}.text-white{color:#fff!important}.text-primary{color:#007bff!important}a.text-primary:focus,a.text-primary:hover{color:#0056b3!important}.text-secondary{color:#6c757d!important}a.text-secondary:focus,a.text-secondary:hover{color:#494f54!important}.text-success{color:#28a745!important}a.text-success:focus,a.text-success:hover{color:#19692c!important}.text-info{color:#17a2b8!important}a.text-info:focus,a.text-info:hover{color:#0f6674!important}.text-warning{color:#ffc107!important}a.text-warning:focus,a.text-warning:hover{color:#ba8b00!important}.text-danger{color:#dc3545!important}a.text-danger:focus,a.text-danger:hover{color:#a71d2a!important}.text-light{color:#f8f9fa!important}a.text-light:focus,a.text-light:hover{color:#cbd3da!important}.text-dark{color:#343a40!important}a.text-dark:focus,a.text-dark:hover{color:#121416!important}.text-body{color:#212529!important}.text-muted{color:#6c757d!important}.text-black-50{color:rgba(0,0,0,.5)!important}.text-white-50{color:rgba(255,255,255,.5)!important}.text-hide{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.text-decoration-none{text-decoration:none!important}.text-break{word-break:break-word!important;overflow-wrap:break-word!important}.text-reset{color:inherit!important}.visible{visibility:visible!important}.invisible{visibility:hidden!important}@media print{*,::after,::before{text-shadow:none!important;box-shadow:none!important}a:not(.btn){text-decoration:underline}abbr[title]::after{content:" (" attr(title) ")"}pre{white-space:pre-wrap!important}blockquote,pre{border:1px solid #adb5bd;page-break-inside:avoid}thead{display:table-header-group}img,tr{page-break-inside:avoid}h2,h3,p{orphans:3;widows:3}h2,h3{page-break-after:avoid}@page{size:a3}body{min-width:992px!important}.container{min-width:992px!important}.navbar{display:none}.badge{border:1px solid #000}.table{border-collapse:collapse!important}.table td,.table th{background-color:#fff!important}.table-bordered td,.table-bordered th{border:1px solid #dee2e6!important}.table-dark{color:inherit}.table-dark tbody+tbody,.table-dark td,.table-dark th,.table-dark thead th{border-color:#dee2e6}.table .thead-dark th{color:inherit;border-color:#dee2e6}}
+/*# sourceMappingURL=bootstrap.min.css.map */
\ No newline at end of file
diff --git a/docs/css/bootstrap.min.css.map b/docs/css/bootstrap.min.css.map
new file mode 100644
index 0000000000000..b939eb6f394c2
--- /dev/null
+++ b/docs/css/bootstrap.min.css.map
@@ -0,0 +1 @@
+{"version":3,"sources":["../../scss/bootstrap.scss","../../scss/_root.scss","../../scss/_reboot.scss","dist/css/bootstrap.css","../../scss/vendor/_rfs.scss","bootstrap.css","../../scss/mixins/_hover.scss","../../scss/_type.scss","../../scss/mixins/_lists.scss","../../scss/_images.scss","../../scss/mixins/_image.scss","../../scss/mixins/_border-radius.scss","../../scss/_code.scss","../../scss/_grid.scss","../../scss/mixins/_grid.scss","../../scss/mixins/_breakpoints.scss","../../scss/mixins/_grid-framework.scss","../../scss/_tables.scss","../../scss/mixins/_table-row.scss","../../scss/_forms.scss","../../scss/mixins/_transition.scss","../../scss/mixins/_forms.scss","../../scss/mixins/_gradients.scss","../../scss/_buttons.scss","../../scss/mixins/_buttons.scss","../../scss/_transitions.scss","../../scss/_dropdown.scss","../../scss/mixins/_caret.scss","../../scss/mixins/_nav-divider.scss","../../scss/_button-group.scss","../../scss/_input-group.scss","../../scss/_custom-forms.scss","../../scss/_nav.scss","../../scss/_navbar.scss","../../scss/_card.scss","../../scss/_breadcrumb.scss","../../scss/_pagination.scss","../../scss/mixins/_pagination.scss","../../scss/_badge.scss","../../scss/mixins/_badge.scss","../../scss/_jumbotron.scss","../../scss/_alert.scss","../../scss/mixins/_alert.scss","../../scss/_progress.scss","../../scss/_media.scss","../../scss/_list-group.scss","../../scss/mixins/_list-group.scss","../../scss/_close.scss","../../scss/_toasts.scss","../../scss/_modal.scss","../../scss/_tooltip.scss","../../scss/mixins/_reset-text.scss","../../scss/_popover.scss","../../scss/_carousel.scss","../../scss/mixins/_clearfix.scss","../../scss/_spinners.scss","../../scss/utilities/_align.scss","../../scss/mixins/_background-variant.scss","../../scss/utilities/_background.scss","../../scss/utilities/_borders.scss","../../scss/utilities/_display.scss","../../scss/utilities/_embed.scss","../../scss/utilities/_flex.scss","../../scss/utilities/_float.scss","../../scss/utilities/_overflow.scss","../../scss/utilities/_position.scss","../../scss/utilities/_screenreaders.scss","../../scss/mixins/_screen-reader.scss","../../scss/utilities/_shadows.scss","../../scss/utilities/_sizing.scss","../../scss/utilities/_stretched-link.scss","../../scss/utilities/_spacing.scss","../../scss/utilities/_text.scss","../../scss/mixins/_text-truncate.scss","../../scss/mixins/_text-emphasis.scss","../../scss/mixins/_text-hide.scss","../../scss/utilities/_visibility.scss","../../scss/_print.scss"],"names":[],"mappings":"AAAA;;;;;ACCA,MAGI,OAAA,QAAA,SAAA,QAAA,SAAA,QAAA,OAAA,QAAA,MAAA,QAAA,SAAA,QAAA,SAAA,QAAA,QAAA,QAAA,OAAA,QAAA,OAAA,QAAA,QAAA,KAAA,OAAA,QAAA,YAAA,QAIA,UAAA,QAAA,YAAA,QAAA,UAAA,QAAA,OAAA,QAAA,UAAA,QAAA,SAAA,QAAA,QAAA,QAAA,OAAA,QAIA,gBAAA,EAAA,gBAAA,MAAA,gBAAA,MAAA,gBAAA,MAAA,gBAAA,OAKF,yBAAA,aAAA,CAAA,kBAAA,CAAA,UAAA,CAAA,MAAA,CAAA,gBAAA,CAAA,KAAA,CAAA,WAAA,CAAA,UAAA,CAAA,mBAAA,CAAA,gBAAA,CAAA,iBAAA,CAAA,mBACA,wBAAA,cAAA,CAAA,KAAA,CAAA,MAAA,CAAA,QAAA,CAAA,iBAAA,CAAA,aAAA,CAAA,UCAF,ECqBA,QADA,SDjBE,WAAA,WAGF,KACE,YAAA,WACA,YAAA,KACA,yBAAA,KACA,4BAAA,YAMF,QAAA,MAAA,WAAA,OAAA,OAAA,OAAA,OAAA,KAAA,IAAA,QACE,QAAA,MAUF,KACE,OAAA,EACA,YAAA,aAAA,CAAA,kBAAA,CAAA,UAAA,CAAA,MAAA,CAAA,gBAAA,CAAA,KAAA,CAAA,WAAA,CAAA,UAAA,CAAA,mBAAA,CAAA,gBAAA,CAAA,iBAAA,CAAA,mBEgFI,UAAA,KF9EJ,YAAA,IACA,YAAA,IACA,MAAA,QACA,WAAA,KACA,iBAAA,KGYF,0CHCE,QAAA,YASF,GACE,WAAA,YACA,OAAA,EACA,SAAA,QAaF,GAAA,GAAA,GAAA,GAAA,GAAA,GACE,WAAA,EACA,cAAA,MAOF,EACE,WAAA,EACA,cAAA,KChBF,0BD2BA,YAEE,gBAAA,UACA,wBAAA,UAAA,OAAA,gBAAA,UAAA,OACA,OAAA,KACA,cAAA,EACA,iCAAA,KAAA,yBAAA,KAGF,QACE,cAAA,KACA,WAAA,OACA,YAAA,QCrBF,GDwBA,GCzBA,GD4BE,WAAA,EACA,cAAA,KAGF,MCxBA,MACA,MAFA,MD6BE,cAAA,EAGF,GACE,YAAA,IAGF,GACE,cAAA,MACA,YAAA,EAGF,WACE,OAAA,EAAA,EAAA,KAGF,ECzBA,OD2BE,YAAA,OAGF,MExFI,UAAA,IFiGJ,IC9BA,IDgCE,SAAA,SEnGE,UAAA,IFqGF,YAAA,EACA,eAAA,SAGF,IAAM,OAAA,OACN,IAAM,IAAA,MAON,EACE,MAAA,QACA,gBAAA,KACA,iBAAA,YIhLA,QJmLE,MAAA,QACA,gBAAA,UASJ,cACE,MAAA,QACA,gBAAA,KI/LA,oBJkME,MAAA,QACA,gBAAA,KC/BJ,KACA,IDuCA,ICtCA,KD0CE,YAAA,cAAA,CAAA,KAAA,CAAA,MAAA,CAAA,QAAA,CAAA,iBAAA,CAAA,aAAA,CAAA,UEpJE,UAAA,IFwJJ,IAEE,WAAA,EAEA,cAAA,KAEA,SAAA,KAQF,OAEE,OAAA,EAAA,EAAA,KAQF,IACE,eAAA,OACA,aAAA,KAGF,IAGE,SAAA,OACA,eAAA,OAQF,MACE,gBAAA,SAGF,QACE,YAAA,OACA,eAAA,OACA,MAAA,QACA,WAAA,KACA,aAAA,OAGF,GAGE,WAAA,QAQF,MAEE,QAAA,aACA,cAAA,MAMF,OAEE,cAAA,EAOF,aACE,QAAA,IAAA,OACA,QAAA,IAAA,KAAA,yBC1EF,OD6EA,MC3EA,SADA,OAEA,SD+EE,OAAA,EACA,YAAA,QErPE,UAAA,QFuPF,YAAA,QAGF,OC7EA,MD+EE,SAAA,QAGF,OC7EA,OD+EE,eAAA,KAMF,OACE,UAAA,OC7EF,cACA,aACA,cDkFA,OAIE,mBAAA,OCjFF,6BACA,4BACA,6BDoFE,sBAKI,OAAA,QCpFN,gCACA,+BACA,gCDwFA,yBAIE,QAAA,EACA,aAAA,KCvFF,qBD0FA,kBAEE,WAAA,WACA,QAAA,EAIF,iBC1FA,2BACA,kBAFA,iBDoGE,mBAAA,QAGF,SACE,SAAA,KAEA,OAAA,SAGF,SAME,UAAA,EAEA,QAAA,EACA,OAAA,EACA,OAAA,EAKF,OACE,QAAA,MACA,MAAA,KACA,UAAA,KACA,QAAA,EACA,cAAA,MEjSI,UAAA,OFmSJ,YAAA,QACA,MAAA,QACA,YAAA,OAGF,SACE,eAAA,SGzGF,yCFGA,yCD4GE,OAAA,KG1GF,cHkHE,eAAA,KACA,mBAAA,KG9GF,yCHsHE,mBAAA,KAQF,6BACE,KAAA,QACA,mBAAA,OAOF,OACE,QAAA,aAGF,QACE,QAAA,UACA,OAAA,QAGF,SACE,QAAA,KG3HF,SHiIE,QAAA,eC1HF,IAAK,IAAK,IAAK,IAAK,IAAK,IIhWzB,GAAA,GAAA,GAAA,GAAA,GAAA,GAEE,cAAA,MAEA,YAAA,IACA,YAAA,IAIF,IAAA,GHgHM,UAAA,OG/GN,IAAA,GH+GM,UAAA,KG9GN,IAAA,GH8GM,UAAA,QG7GN,IAAA,GH6GM,UAAA,OG5GN,IAAA,GH4GM,UAAA,QG3GN,IAAA,GH2GM,UAAA,KGzGN,MHyGM,UAAA,QGvGJ,YAAA,IAIF,WHmGM,UAAA,KGjGJ,YAAA,IACA,YAAA,IAEF,WH8FM,UAAA,OG5FJ,YAAA,IACA,YAAA,IAEF,WHyFM,UAAA,OGvFJ,YAAA,IACA,YAAA,IAEF,WHoFM,UAAA,OGlFJ,YAAA,IACA,YAAA,IL6BF,GKpBE,WAAA,KACA,cAAA,KACA,OAAA,EACA,WAAA,IAAA,MAAA,eJ+WF,OIvWA,MHMI,UAAA,IGHF,YAAA,IJ0WF,MIvWA,KAEE,QAAA,KACA,iBAAA,QAQF,eC/EE,aAAA,EACA,WAAA,KDmFF,aCpFE,aAAA,EACA,WAAA,KDsFF,kBACE,QAAA,aADF,mCAII,aAAA,MAUJ,YHjCI,UAAA,IGmCF,eAAA,UAIF,YACE,cAAA,KHeI,UAAA,QGXN,mBACE,QAAA,MH7CE,UAAA,IG+CF,MAAA,QAHF,2BAMI,QAAA,aEnHJ,WCIE,UAAA,KAGA,OAAA,KDDF,eACE,QAAA,OACA,iBAAA,KACA,OAAA,IAAA,MAAA,QEXE,cAAA,ODMF,UAAA,KAGA,OAAA,KDcF,QAEE,QAAA,aAGF,YACE,cAAA,MACA,YAAA,EAGF,gBLkCI,UAAA,IKhCF,MAAA,QGvCF,KRuEI,UAAA,MQrEF,MAAA,QACA,UAAA,WAGA,OACE,MAAA,QAKJ,IACE,QAAA,MAAA,MR0DE,UAAA,MQxDF,MAAA,KACA,iBAAA,QDZE,cAAA,MCQJ,QASI,QAAA,ERkDA,UAAA,KQhDA,YAAA,IVwMJ,IUjME,QAAA,MRyCE,UAAA,MQvCF,MAAA,QAHF,SR0CI,UAAA,QQlCA,MAAA,QACA,WAAA,OAKJ,gBACE,WAAA,MACA,WAAA,OCxCA,WCDA,MAAA,KACA,cAAA,KACA,aAAA,KACA,aAAA,KACA,YAAA,KCmDE,yBFtDF,WCWI,UAAA,OC2CF,yBFtDF,WCWI,UAAA,OC2CF,yBFtDF,WCWI,UAAA,OC2CF,0BFtDF,WCWI,UAAA,QDLJ,iBAAA,cAAA,cAAA,cAAA,cCPA,MAAA,KACA,cAAA,KACA,aAAA,KACA,aAAA,KACA,YAAA,KCmDE,yBFrCE,WAAA,cACE,UAAA,OEoCJ,yBFrCE,WAAA,cAAA,cACE,UAAA,OEoCJ,yBFrCE,WAAA,cAAA,cAAA,cACE,UAAA,OEoCJ,0BFrCE,WAAA,cAAA,cAAA,cAAA,cACE,UAAA,QAoBN,KCrBA,QAAA,YAAA,QAAA,KACA,cAAA,KAAA,UAAA,KACA,aAAA,MACA,YAAA,MDwBA,YACE,aAAA,EACA,YAAA,EAFF,iBV4jBF,0BUtjBM,cAAA,EACA,aAAA,EGlDJ,KAAA,OAAA,QAAA,QAAA,QAAA,OAAA,OAAA,OAAA,OAAA,OAAA,OAAA,OAAA,Ob6mBF,UAEqJ,QAAvI,UAAmG,WAAY,WAAY,WAAhH,UAAW,UAAW,UAAW,UAAW,UAAW,UAAW,UAAW,UACtG,aAFqJ,QAAvI,UAAmG,WAAY,WAAY,WAAhH,UAAW,UAAW,UAAW,UAAW,UAAW,UAAW,UAAW,UACtG,aAFkJ,QAAvI,UAAmG,WAAY,WAAY,WAAhH,UAAW,UAAW,UAAW,UAAW,UAAW,UAAW,UAAW,UACnG,aAEqJ,QAAvI,UAAmG,WAAY,WAAY,WAAhH,UAAW,UAAW,UAAW,UAAW,UAAW,UAAW,UAAW,UACtG,aahnBI,SAAA,SACA,MAAA,KACA,cAAA,KACA,aAAA,KAmBE,KACE,wBAAA,EAAA,WAAA,EACA,kBAAA,EAAA,UAAA,EACA,UAAA,KAIA,cF4BJ,SAAA,EAAA,EAAA,KAAA,KAAA,EAAA,EAAA,KACA,UAAA,KE7BI,cF4BJ,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IACA,UAAA,IE7BI,cF4BJ,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WACA,UAAA,WE7BI,cF4BJ,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IACA,UAAA,IE7BI,cF4BJ,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IACA,UAAA,IE7BI,cF4BJ,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WACA,UAAA,WExBE,UFMJ,SAAA,EAAA,EAAA,KAAA,KAAA,EAAA,EAAA,KACA,MAAA,KACA,UAAA,KEHM,OFPN,SAAA,EAAA,EAAA,UAAA,KAAA,EAAA,EAAA,UAIA,UAAA,UEGM,OFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,OFPN,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IAIA,UAAA,IEGM,OFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,OFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,OFPN,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IAIA,UAAA,IEGM,OFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,OFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,OFPN,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IAIA,UAAA,IEGM,QFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,QFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,QFPN,SAAA,EAAA,EAAA,KAAA,KAAA,EAAA,EAAA,KAIA,UAAA,KEQI,aAAwB,eAAA,GAAA,MAAA,GAExB,YAAuB,eAAA,GAAA,MAAA,GAGrB,SAAwB,eAAA,EAAA,MAAA,EAAxB,SAAwB,eAAA,EAAA,MAAA,EAAxB,SAAwB,eAAA,EAAA,MAAA,EAAxB,SAAwB,eAAA,EAAA,MAAA,EAAxB,SAAwB,eAAA,EAAA,MAAA,EAAxB,SAAwB,eAAA,EAAA,MAAA,EAAxB,SAAwB,eAAA,EAAA,MAAA,EAAxB,SAAwB,eAAA,EAAA,MAAA,EAAxB,SAAwB,eAAA,EAAA,MAAA,EAAxB,SAAwB,eAAA,EAAA,MAAA,EAAxB,UAAwB,eAAA,GAAA,MAAA,GAAxB,UAAwB,eAAA,GAAA,MAAA,GAAxB,UAAwB,eAAA,GAAA,MAAA,GAMtB,UFRR,YAAA,UEQQ,UFRR,YAAA,WEQQ,UFRR,YAAA,IEQQ,UFRR,YAAA,WEQQ,UFRR,YAAA,WEQQ,UFRR,YAAA,IEQQ,UFRR,YAAA,WEQQ,UFRR,YAAA,WEQQ,UFRR,YAAA,IEQQ,WFRR,YAAA,WEQQ,WFRR,YAAA,WCKE,yBC9BE,QACE,wBAAA,EAAA,WAAA,EACA,kBAAA,EAAA,UAAA,EACA,UAAA,KAIA,iBF4BJ,SAAA,EAAA,EAAA,KAAA,KAAA,EAAA,EAAA,KACA,UAAA,KE7BI,iBF4BJ,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IACA,UAAA,IE7BI,iBF4BJ,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WACA,UAAA,WE7BI,iBF4BJ,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IACA,UAAA,IE7BI,iBF4BJ,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IACA,UAAA,IE7BI,iBF4BJ,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WACA,UAAA,WExBE,aFMJ,SAAA,EAAA,EAAA,KAAA,KAAA,EAAA,EAAA,KACA,MAAA,KACA,UAAA,KEHM,UFPN,SAAA,EAAA,EAAA,UAAA,KAAA,EAAA,EAAA,UAIA,UAAA,UEGM,UFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,UFPN,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IAIA,UAAA,IEGM,UFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,UFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,UFPN,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IAIA,UAAA,IEGM,UFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,UFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,UFPN,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IAIA,UAAA,IEGM,WFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,WFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,WFPN,SAAA,EAAA,EAAA,KAAA,KAAA,EAAA,EAAA,KAIA,UAAA,KEQI,gBAAwB,eAAA,GAAA,MAAA,GAExB,eAAuB,eAAA,GAAA,MAAA,GAGrB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,aAAwB,eAAA,GAAA,MAAA,GAAxB,aAAwB,eAAA,GAAA,MAAA,GAAxB,aAAwB,eAAA,GAAA,MAAA,GAMtB,aFRR,YAAA,EEQQ,aFRR,YAAA,UEQQ,aFRR,YAAA,WEQQ,aFRR,YAAA,IEQQ,aFRR,YAAA,WEQQ,aFRR,YAAA,WEQQ,aFRR,YAAA,IEQQ,aFRR,YAAA,WEQQ,aFRR,YAAA,WEQQ,aFRR,YAAA,IEQQ,cFRR,YAAA,WEQQ,cFRR,YAAA,YCKE,yBC9BE,QACE,wBAAA,EAAA,WAAA,EACA,kBAAA,EAAA,UAAA,EACA,UAAA,KAIA,iBF4BJ,SAAA,EAAA,EAAA,KAAA,KAAA,EAAA,EAAA,KACA,UAAA,KE7BI,iBF4BJ,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IACA,UAAA,IE7BI,iBF4BJ,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WACA,UAAA,WE7BI,iBF4BJ,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IACA,UAAA,IE7BI,iBF4BJ,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IACA,UAAA,IE7BI,iBF4BJ,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WACA,UAAA,WExBE,aFMJ,SAAA,EAAA,EAAA,KAAA,KAAA,EAAA,EAAA,KACA,MAAA,KACA,UAAA,KEHM,UFPN,SAAA,EAAA,EAAA,UAAA,KAAA,EAAA,EAAA,UAIA,UAAA,UEGM,UFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,UFPN,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IAIA,UAAA,IEGM,UFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,UFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,UFPN,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IAIA,UAAA,IEGM,UFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,UFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,UFPN,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IAIA,UAAA,IEGM,WFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,WFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,WFPN,SAAA,EAAA,EAAA,KAAA,KAAA,EAAA,EAAA,KAIA,UAAA,KEQI,gBAAwB,eAAA,GAAA,MAAA,GAExB,eAAuB,eAAA,GAAA,MAAA,GAGrB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,aAAwB,eAAA,GAAA,MAAA,GAAxB,aAAwB,eAAA,GAAA,MAAA,GAAxB,aAAwB,eAAA,GAAA,MAAA,GAMtB,aFRR,YAAA,EEQQ,aFRR,YAAA,UEQQ,aFRR,YAAA,WEQQ,aFRR,YAAA,IEQQ,aFRR,YAAA,WEQQ,aFRR,YAAA,WEQQ,aFRR,YAAA,IEQQ,aFRR,YAAA,WEQQ,aFRR,YAAA,WEQQ,aFRR,YAAA,IEQQ,cFRR,YAAA,WEQQ,cFRR,YAAA,YCKE,yBC9BE,QACE,wBAAA,EAAA,WAAA,EACA,kBAAA,EAAA,UAAA,EACA,UAAA,KAIA,iBF4BJ,SAAA,EAAA,EAAA,KAAA,KAAA,EAAA,EAAA,KACA,UAAA,KE7BI,iBF4BJ,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IACA,UAAA,IE7BI,iBF4BJ,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WACA,UAAA,WE7BI,iBF4BJ,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IACA,UAAA,IE7BI,iBF4BJ,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IACA,UAAA,IE7BI,iBF4BJ,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WACA,UAAA,WExBE,aFMJ,SAAA,EAAA,EAAA,KAAA,KAAA,EAAA,EAAA,KACA,MAAA,KACA,UAAA,KEHM,UFPN,SAAA,EAAA,EAAA,UAAA,KAAA,EAAA,EAAA,UAIA,UAAA,UEGM,UFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,UFPN,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IAIA,UAAA,IEGM,UFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,UFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,UFPN,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IAIA,UAAA,IEGM,UFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,UFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,UFPN,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IAIA,UAAA,IEGM,WFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,WFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,WFPN,SAAA,EAAA,EAAA,KAAA,KAAA,EAAA,EAAA,KAIA,UAAA,KEQI,gBAAwB,eAAA,GAAA,MAAA,GAExB,eAAuB,eAAA,GAAA,MAAA,GAGrB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,aAAwB,eAAA,GAAA,MAAA,GAAxB,aAAwB,eAAA,GAAA,MAAA,GAAxB,aAAwB,eAAA,GAAA,MAAA,GAMtB,aFRR,YAAA,EEQQ,aFRR,YAAA,UEQQ,aFRR,YAAA,WEQQ,aFRR,YAAA,IEQQ,aFRR,YAAA,WEQQ,aFRR,YAAA,WEQQ,aFRR,YAAA,IEQQ,aFRR,YAAA,WEQQ,aFRR,YAAA,WEQQ,aFRR,YAAA,IEQQ,cFRR,YAAA,WEQQ,cFRR,YAAA,YCKE,0BC9BE,QACE,wBAAA,EAAA,WAAA,EACA,kBAAA,EAAA,UAAA,EACA,UAAA,KAIA,iBF4BJ,SAAA,EAAA,EAAA,KAAA,KAAA,EAAA,EAAA,KACA,UAAA,KE7BI,iBF4BJ,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IACA,UAAA,IE7BI,iBF4BJ,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WACA,UAAA,WE7BI,iBF4BJ,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IACA,UAAA,IE7BI,iBF4BJ,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IACA,UAAA,IE7BI,iBF4BJ,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WACA,UAAA,WExBE,aFMJ,SAAA,EAAA,EAAA,KAAA,KAAA,EAAA,EAAA,KACA,MAAA,KACA,UAAA,KEHM,UFPN,SAAA,EAAA,EAAA,UAAA,KAAA,EAAA,EAAA,UAIA,UAAA,UEGM,UFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,UFPN,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IAIA,UAAA,IEGM,UFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,UFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,UFPN,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IAIA,UAAA,IEGM,UFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,UFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,UFPN,SAAA,EAAA,EAAA,IAAA,KAAA,EAAA,EAAA,IAIA,UAAA,IEGM,WFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,WFPN,SAAA,EAAA,EAAA,WAAA,KAAA,EAAA,EAAA,WAIA,UAAA,WEGM,WFPN,SAAA,EAAA,EAAA,KAAA,KAAA,EAAA,EAAA,KAIA,UAAA,KEQI,gBAAwB,eAAA,GAAA,MAAA,GAExB,eAAuB,eAAA,GAAA,MAAA,GAGrB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,YAAwB,eAAA,EAAA,MAAA,EAAxB,aAAwB,eAAA,GAAA,MAAA,GAAxB,aAAwB,eAAA,GAAA,MAAA,GAAxB,aAAwB,eAAA,GAAA,MAAA,GAMtB,aFRR,YAAA,EEQQ,aFRR,YAAA,UEQQ,aFRR,YAAA,WEQQ,aFRR,YAAA,IEQQ,aFRR,YAAA,WEQQ,aFRR,YAAA,WEQQ,aFRR,YAAA,IEQQ,aFRR,YAAA,WEQQ,aFRR,YAAA,WEQQ,aFRR,YAAA,IEQQ,cFRR,YAAA,WEQQ,cFRR,YAAA,YGnDF,OACE,MAAA,KACA,cAAA,KACA,MAAA,QdypDF,Uc5pDA,UAQI,QAAA,OACA,eAAA,IACA,WAAA,IAAA,MAAA,QAVJ,gBAcI,eAAA,OACA,cAAA,IAAA,MAAA,QAfJ,mBAmBI,WAAA,IAAA,MAAA,QdypDJ,achpDA,aAGI,QAAA,MASJ,gBACE,OAAA,IAAA,MAAA,Qd4oDF,mBc7oDA,mBAKI,OAAA,IAAA,MAAA,Qd6oDJ,yBclpDA,yBAWM,oBAAA,Id8oDN,8BAFA,qBcvoDA,qBdwoDA,2BcnoDI,OAAA,EAQJ,yCAEI,iBAAA,gBX/DF,4BW2EI,MAAA,QACA,iBAAA,iBCnFJ,ef+sDF,kBADA,kBe1sDM,iBAAA,QfktDN,2BAFA,kBeptDE,kBfqtDF,wBezsDQ,aAAA,QZLN,kCYiBM,iBAAA,QALN,qCf4sDF,qCensDU,iBAAA,QA5BR,iBfquDF,oBADA,oBehuDM,iBAAA,QfwuDN,6BAFA,oBe1uDE,oBf2uDF,0Be/tDQ,aAAA,QZLN,oCYiBM,iBAAA,QALN,uCfkuDF,uCeztDU,iBAAA,QA5BR,ef2vDF,kBADA,kBetvDM,iBAAA,Qf8vDN,2BAFA,kBehwDE,kBfiwDF,wBervDQ,aAAA,QZLN,kCYiBM,iBAAA,QALN,qCfwvDF,qCe/uDU,iBAAA,QA5BR,YfixDF,eADA,ee5wDM,iBAAA,QfoxDN,wBAFA,eetxDE,efuxDF,qBe3wDQ,aAAA,QZLN,+BYiBM,iBAAA,QALN,kCf8wDF,kCerwDU,iBAAA,QA5BR,efuyDF,kBADA,kBelyDM,iBAAA,Qf0yDN,2BAFA,kBe5yDE,kBf6yDF,wBejyDQ,aAAA,QZLN,kCYiBM,iBAAA,QALN,qCfoyDF,qCe3xDU,iBAAA,QA5BR,cf6zDF,iBADA,iBexzDM,iBAAA,Qfg0DN,0BAFA,iBel0DE,iBfm0DF,uBevzDQ,aAAA,QZLN,iCYiBM,iBAAA,QALN,oCf0zDF,oCejzDU,iBAAA,QA5BR,afm1DF,gBADA,gBe90DM,iBAAA,Qfs1DN,yBAFA,gBex1DE,gBfy1DF,sBe70DQ,aAAA,QZLN,gCYiBM,iBAAA,QALN,mCfg1DF,mCev0DU,iBAAA,QA5BR,Yfy2DF,eADA,eep2DM,iBAAA,Qf42DN,wBAFA,ee92DE,ef+2DF,qBen2DQ,aAAA,QZLN,+BYiBM,iBAAA,QALN,kCfs2DF,kCe71DU,iBAAA,QA5BR,cf+3DF,iBADA,iBe13DM,iBAAA,iBZGJ,iCYiBM,iBAAA,iBALN,oCfq3DF,oCe52DU,iBAAA,iBD8EV,sBAGM,MAAA,KACA,iBAAA,QACA,aAAA,QALN,uBAWM,MAAA,QACA,iBAAA,QACA,aAAA,QAKN,YACE,MAAA,KACA,iBAAA,QdgyDF,eclyDA,edmyDA,qBc5xDI,aAAA,QAPJ,2BAWI,OAAA,EAXJ,oDAgBM,iBAAA,sBXrIJ,uCW4IM,MAAA,KACA,iBAAA,uBFhFJ,4BEiGA,qBAEI,QAAA,MACA,MAAA,KACA,WAAA,KACA,2BAAA,MALH,qCASK,OAAA,GF1GN,4BEiGA,qBAEI,QAAA,MACA,MAAA,KACA,WAAA,KACA,2BAAA,MALH,qCASK,OAAA,GF1GN,4BEiGA,qBAEI,QAAA,MACA,MAAA,KACA,WAAA,KACA,2BAAA,MALH,qCASK,OAAA,GF1GN,6BEiGA,qBAEI,QAAA,MACA,MAAA,KACA,WAAA,KACA,2BAAA,MALH,qCASK,OAAA,GAdV,kBAOQ,QAAA,MACA,MAAA,KACA,WAAA,KACA,2BAAA,MAVR,kCAcU,OAAA,EE7KV,cACE,QAAA,MACA,MAAA,KACA,OAAA,2BACA,QAAA,QAAA,OfqHI,UAAA,KelHJ,YAAA,IACA,YAAA,IACA,MAAA,QACA,iBAAA,KACA,gBAAA,YACA,OAAA,IAAA,MAAA,QRbE,cAAA,OSCE,WAAA,aAAA,KAAA,WAAA,CAAA,WAAA,KAAA,YAKF,uCDLJ,cCMM,WAAA,MDNN,0BAsBI,iBAAA,YACA,OAAA,EAvBJ,6BA4BI,MAAA,YACA,YAAA,EAAA,EAAA,EAAA,QEtBF,oBACE,MAAA,QACA,iBAAA,KACA,aAAA,QACA,QAAA,EAKE,WAAA,EAAA,EAAA,EAAA,MAAA,oBFhBN,yCAqCI,MAAA,QAEA,QAAA,EAvCJ,gCAqCI,MAAA,QAEA,QAAA,EAvCJ,oCAqCI,MAAA,QAEA,QAAA,EAvCJ,qCAqCI,MAAA,QAEA,QAAA,EAvCJ,2BAqCI,MAAA,QAEA,QAAA,EAvCJ,uBAAA,wBAiDI,iBAAA,QAEA,QAAA,EAIJ,qCAOI,MAAA,QACA,iBAAA,KAKJ,mBhBk/DA,oBgBh/DE,QAAA,MACA,MAAA,KAUF,gBACE,YAAA,oBACA,eAAA,oBACA,cAAA,EflBE,UAAA,QeoBF,YAAA,IAGF,mBACE,YAAA,kBACA,eAAA,kBf8BI,UAAA,Qe5BJ,YAAA,IAGF,mBACE,YAAA,mBACA,eAAA,mBfuBI,UAAA,QerBJ,YAAA,IASF,wBACE,QAAA,MACA,MAAA,KACA,QAAA,QAAA,EACA,cAAA,EfQI,UAAA,KeNJ,YAAA,IACA,MAAA,QACA,iBAAA,YACA,OAAA,MAAA,YACA,aAAA,IAAA,EAVF,wCAAA,wCAcI,cAAA,EACA,aAAA,EAYJ,iBACE,OAAA,0BACA,QAAA,OAAA,MfjBI,UAAA,QemBJ,YAAA,IR7IE,cAAA,MQiJJ,iBACE,OAAA,yBACA,QAAA,MAAA,KfzBI,UAAA,Qe2BJ,YAAA,IRrJE,cAAA,MQ0JJ,8BAAA,0BAGI,OAAA,KAIJ,sBACE,OAAA,KAQF,YACE,cAAA,KAGF,WACE,QAAA,MACA,WAAA,OAQF,UACE,QAAA,YAAA,QAAA,KACA,cAAA,KAAA,UAAA,KACA,aAAA,KACA,YAAA,KAJF,ehBu9DA,wBgB/8DI,cAAA,IACA,aAAA,IASJ,YACE,SAAA,SACA,QAAA,MACA,aAAA,QAGF,kBACE,SAAA,SACA,WAAA,MACA,YAAA,ShB88DF,6CgBj9DA,8CAQI,MAAA,QAIJ,kBACE,cAAA,EAGF,mBACE,QAAA,mBAAA,QAAA,YACA,eAAA,OAAA,YAAA,OACA,aAAA,EACA,aAAA,OAJF,qCAQI,SAAA,OACA,WAAA,EACA,aAAA,SACA,YAAA,EEpMF,gBACE,QAAA,KACA,MAAA,KACA,WAAA,OjByBA,UAAA,IiBvBA,MAAA,QAGF,eACE,SAAA,SACA,IAAA,KACA,QAAA,EACA,QAAA,KACA,UAAA,KACA,QAAA,OAAA,MACA,WAAA,MjBoEE,UAAA,QiBlEF,YAAA,IACA,MAAA,KACA,iBAAA,mBV1DA,cAAA,ORktEJ,0BACA,yBkBxrEI,sClBsrEJ,qCkBhpEM,QAAA,MAtCF,uBAAA,mCA4CE,aAAA,QAGE,cAAA,qBACA,iBAAA,gQACA,kBAAA,UACA,oBAAA,MAAA,wBAAA,OACA,gBAAA,sBAAA,sBAnDJ,6BAAA,yCAuDI,aAAA,QACA,WAAA,EAAA,EAAA,EAAA,MAAA,oBAxDJ,2CAAA,+BAiEI,cAAA,qBACA,oBAAA,IAAA,wBAAA,MAAA,wBAlEJ,wBAAA,oCAyEE,aAAA,QAGE,cAAA,wBACA,WAAA,+KAAA,UAAA,MAAA,OAAA,MAAA,CAAA,IAAA,IAAA,CAAA,gQAAA,KAAA,UAAA,OAAA,MAAA,OAAA,CAAA,sBAAA,sBA7EJ,8BAAA,0CAiFI,aAAA,QACA,WAAA,EAAA,EAAA,EAAA,MAAA,oBAlFJ,6CAAA,yDA0FI,MAAA,QlBqoEiD,2CACzD,0CkBhuEI,uDlB+tEJ,sDkBhoEQ,QAAA,MA/FJ,qDAAA,iEAuGI,MAAA,QAvGJ,6DAAA,yEA0GM,aAAA,QA1GN,qEAAA,iFAgHM,aAAA,QC1IN,iBAAA,QD0BA,mEAAA,+EAuHM,WAAA,EAAA,EAAA,EAAA,MAAA,oBAvHN,iFAAA,6FA2HM,aAAA,QA3HN,+CAAA,2DAqII,aAAA,QArIJ,qDAAA,iEA0IM,aAAA,QACA,WAAA,EAAA,EAAA,EAAA,MAAA,oBA/HR,kBACE,QAAA,KACA,MAAA,KACA,WAAA,OjByBA,UAAA,IiBvBA,MAAA,QAGF,iBACE,SAAA,SACA,IAAA,KACA,QAAA,EACA,QAAA,KACA,UAAA,KACA,QAAA,OAAA,MACA,WAAA,MjBoEE,UAAA,QiBlEF,YAAA,IACA,MAAA,KACA,iBAAA,mBV1DA,cAAA,ORszEJ,8BACA,6BkB5xEI,0ClB0xEJ,yCkBpvEM,QAAA,MAtCF,yBAAA,qCA4CE,aAAA,QAGE,cAAA,qBACA,iBAAA,2TACA,kBAAA,UACA,oBAAA,MAAA,wBAAA,OACA,gBAAA,sBAAA,sBAnDJ,+BAAA,2CAuDI,aAAA,QACA,WAAA,EAAA,EAAA,EAAA,MAAA,oBAxDJ,6CAAA,iCAiEI,cAAA,qBACA,oBAAA,IAAA,wBAAA,MAAA,wBAlEJ,0BAAA,sCAyEE,aAAA,QAGE,cAAA,wBACA,WAAA,+KAAA,UAAA,MAAA,OAAA,MAAA,CAAA,IAAA,IAAA,CAAA,2TAAA,KAAA,UAAA,OAAA,MAAA,OAAA,CAAA,sBAAA,sBA7EJ,gCAAA,4CAiFI,aAAA,QACA,WAAA,EAAA,EAAA,EAAA,MAAA,oBAlFJ,+CAAA,2DA0FI,MAAA,QlByuEqD,+CAC7D,8CkBp0EI,2DlBm0EJ,0DkBpuEQ,QAAA,MA/FJ,uDAAA,mEAuGI,MAAA,QAvGJ,+DAAA,2EA0GM,aAAA,QA1GN,uEAAA,mFAgHM,aAAA,QC1IN,iBAAA,QD0BA,qEAAA,iFAuHM,WAAA,EAAA,EAAA,EAAA,MAAA,oBAvHN,mFAAA,+FA2HM,aAAA,QA3HN,iDAAA,6DAqII,aAAA,QArIJ,uDAAA,mEA0IM,aAAA,QACA,WAAA,EAAA,EAAA,EAAA,MAAA,oBF8FV,aACE,QAAA,YAAA,QAAA,KACA,cAAA,IAAA,KAAA,UAAA,IAAA,KACA,eAAA,OAAA,YAAA,OAHF,yBASI,MAAA,KJtNA,yBI6MJ,mBAeM,QAAA,YAAA,QAAA,KACA,eAAA,OAAA,YAAA,OACA,cAAA,OAAA,gBAAA,OACA,cAAA,EAlBN,yBAuBM,QAAA,YAAA,QAAA,KACA,SAAA,EAAA,EAAA,KAAA,KAAA,EAAA,EAAA,KACA,cAAA,IAAA,KAAA,UAAA,IAAA,KACA,eAAA,OAAA,YAAA,OACA,cAAA,EA3BN,2BAgCM,QAAA,aACA,MAAA,KACA,eAAA,OAlCN,qCAuCM,QAAA,ahBioEJ,4BgBxqEF,0BA4CM,MAAA,KA5CN,yBAkDM,QAAA,YAAA,QAAA,KACA,eAAA,OAAA,YAAA,OACA,cAAA,OAAA,gBAAA,OACA,MAAA,KACA,aAAA,EAtDN,+BAyDM,SAAA,SACA,kBAAA,EAAA,YAAA,EACA,WAAA,EACA,aAAA,OACA,YAAA,EA7DN,6BAiEM,eAAA,OAAA,YAAA,OACA,cAAA,OAAA,gBAAA,OAlEN,mCAqEM,cAAA,GIxUN,KACE,QAAA,aAEA,YAAA,IACA,MAAA,QACA,WAAA,OAEA,eAAA,OACA,OAAA,QACA,oBAAA,KAAA,iBAAA,KAAA,gBAAA,KAAA,YAAA,KACA,iBAAA,YACA,OAAA,IAAA,MAAA,YCuFA,QAAA,QAAA,OpBuBI,UAAA,KoBrBJ,YAAA,IbrGE,cAAA,OSCE,WAAA,MAAA,KAAA,WAAA,CAAA,iBAAA,KAAA,WAAA,CAAA,aAAA,KAAA,WAAA,CAAA,WAAA,KAAA,YAKF,uCGLJ,KHMM,WAAA,MdAJ,WiBUE,MAAA,QACA,gBAAA,KAjBJ,WAAA,WAsBI,QAAA,EACA,WAAA,EAAA,EAAA,EAAA,MAAA,oBAvBJ,cAAA,cA6BI,QAAA,IAeJ,epBw8EA,wBoBt8EE,eAAA,KASA,aCvDA,MAAA,KFAE,iBAAA,QEEF,aAAA,QlBIA,mBkBAE,MAAA,KFNA,iBAAA,QEQA,aAAA,QAGF,mBAAA,mBAEE,MAAA,KFbA,iBAAA,QEeA,aAAA,QAKE,WAAA,EAAA,EAAA,EAAA,MAAA,oBAKJ,sBAAA,sBAEE,MAAA,KACA,iBAAA,QACA,aAAA,QAOF,kDAAA,kDrB8+EF,mCqB3+EI,MAAA,KACA,iBAAA,QAIA,aAAA,QAEA,wDAAA,wDrB2+EJ,yCqBt+EQ,WAAA,EAAA,EAAA,EAAA,MAAA,oBDIN,eCvDA,MAAA,KFAE,iBAAA,QEEF,aAAA,QlBIA,qBkBAE,MAAA,KFNA,iBAAA,QEQA,aAAA,QAGF,qBAAA,qBAEE,MAAA,KFbA,iBAAA,QEeA,aAAA,QAKE,WAAA,EAAA,EAAA,EAAA,MAAA,qBAKJ,wBAAA,wBAEE,MAAA,KACA,iBAAA,QACA,aAAA,QAOF,oDAAA,oDrBmhFF,qCqBhhFI,MAAA,KACA,iBAAA,QAIA,aAAA,QAEA,0DAAA,0DrBghFJ,2CqB3gFQ,WAAA,EAAA,EAAA,EAAA,MAAA,qBDIN,aCvDA,MAAA,KFAE,iBAAA,QEEF,aAAA,QlBIA,mBkBAE,MAAA,KFNA,iBAAA,QEQA,aAAA,QAGF,mBAAA,mBAEE,MAAA,KFbA,iBAAA,QEeA,aAAA,QAKE,WAAA,EAAA,EAAA,EAAA,MAAA,mBAKJ,sBAAA,sBAEE,MAAA,KACA,iBAAA,QACA,aAAA,QAOF,kDAAA,kDrBwjFF,mCqBrjFI,MAAA,KACA,iBAAA,QAIA,aAAA,QAEA,wDAAA,wDrBqjFJ,yCqBhjFQ,WAAA,EAAA,EAAA,EAAA,MAAA,mBDIN,UCvDA,MAAA,KFAE,iBAAA,QEEF,aAAA,QlBIA,gBkBAE,MAAA,KFNA,iBAAA,QEQA,aAAA,QAGF,gBAAA,gBAEE,MAAA,KFbA,iBAAA,QEeA,aAAA,QAKE,WAAA,EAAA,EAAA,EAAA,MAAA,oBAKJ,mBAAA,mBAEE,MAAA,KACA,iBAAA,QACA,aAAA,QAOF,+CAAA,+CrB6lFF,gCqB1lFI,MAAA,KACA,iBAAA,QAIA,aAAA,QAEA,qDAAA,qDrB0lFJ,sCqBrlFQ,WAAA,EAAA,EAAA,EAAA,MAAA,oBDIN,aCvDA,MAAA,QFAE,iBAAA,QEEF,aAAA,QlBIA,mBkBAE,MAAA,QFNA,iBAAA,QEQA,aAAA,QAGF,mBAAA,mBAEE,MAAA,QFbA,iBAAA,QEeA,aAAA,QAKE,WAAA,EAAA,EAAA,EAAA,MAAA,oBAKJ,sBAAA,sBAEE,MAAA,QACA,iBAAA,QACA,aAAA,QAOF,kDAAA,kDrBkoFF,mCqB/nFI,MAAA,QACA,iBAAA,QAIA,aAAA,QAEA,wDAAA,wDrB+nFJ,yCqB1nFQ,WAAA,EAAA,EAAA,EAAA,MAAA,oBDIN,YCvDA,MAAA,KFAE,iBAAA,QEEF,aAAA,QlBIA,kBkBAE,MAAA,KFNA,iBAAA,QEQA,aAAA,QAGF,kBAAA,kBAEE,MAAA,KFbA,iBAAA,QEeA,aAAA,QAKE,WAAA,EAAA,EAAA,EAAA,MAAA,mBAKJ,qBAAA,qBAEE,MAAA,KACA,iBAAA,QACA,aAAA,QAOF,iDAAA,iDrBuqFF,kCqBpqFI,MAAA,KACA,iBAAA,QAIA,aAAA,QAEA,uDAAA,uDrBoqFJ,wCqB/pFQ,WAAA,EAAA,EAAA,EAAA,MAAA,mBDIN,WCvDA,MAAA,QFAE,iBAAA,QEEF,aAAA,QlBIA,iBkBAE,MAAA,QFNA,iBAAA,QEQA,aAAA,QAGF,iBAAA,iBAEE,MAAA,QFbA,iBAAA,QEeA,aAAA,QAKE,WAAA,EAAA,EAAA,EAAA,MAAA,qBAKJ,oBAAA,oBAEE,MAAA,QACA,iBAAA,QACA,aAAA,QAOF,gDAAA,gDrB4sFF,iCqBzsFI,MAAA,QACA,iBAAA,QAIA,aAAA,QAEA,sDAAA,sDrBysFJ,uCqBpsFQ,WAAA,EAAA,EAAA,EAAA,MAAA,qBDIN,UCvDA,MAAA,KFAE,iBAAA,QEEF,aAAA,QlBIA,gBkBAE,MAAA,KFNA,iBAAA,QEQA,aAAA,QAGF,gBAAA,gBAEE,MAAA,KFbA,iBAAA,QEeA,aAAA,QAKE,WAAA,EAAA,EAAA,EAAA,MAAA,kBAKJ,mBAAA,mBAEE,MAAA,KACA,iBAAA,QACA,aAAA,QAOF,+CAAA,+CrBivFF,gCqB9uFI,MAAA,KACA,iBAAA,QAIA,aAAA,QAEA,qDAAA,qDrB8uFJ,sCqBzuFQ,WAAA,EAAA,EAAA,EAAA,MAAA,kBDUN,qBCHA,MAAA,QACA,aAAA,QlBrDA,2BkBwDE,MAAA,KACA,iBAAA,QACA,aAAA,QAGF,2BAAA,2BAEE,WAAA,EAAA,EAAA,EAAA,MAAA,mBAGF,8BAAA,8BAEE,MAAA,QACA,iBAAA,YAGF,0DAAA,0DrBuuFF,2CqBpuFI,MAAA,KACA,iBAAA,QACA,aAAA,QAEA,gEAAA,gErBuuFJ,iDqBluFQ,WAAA,EAAA,EAAA,EAAA,MAAA,mBD7BN,uBCHA,MAAA,QACA,aAAA,QlBrDA,6BkBwDE,MAAA,KACA,iBAAA,QACA,aAAA,QAGF,6BAAA,6BAEE,WAAA,EAAA,EAAA,EAAA,MAAA,qBAGF,gCAAA,gCAEE,MAAA,QACA,iBAAA,YAGF,4DAAA,4DrBuwFF,6CqBpwFI,MAAA,KACA,iBAAA,QACA,aAAA,QAEA,kEAAA,kErBuwFJ,mDqBlwFQ,WAAA,EAAA,EAAA,EAAA,MAAA,qBD7BN,qBCHA,MAAA,QACA,aAAA,QlBrDA,2BkBwDE,MAAA,KACA,iBAAA,QACA,aAAA,QAGF,2BAAA,2BAEE,WAAA,EAAA,EAAA,EAAA,MAAA,mBAGF,8BAAA,8BAEE,MAAA,QACA,iBAAA,YAGF,0DAAA,0DrBuyFF,2CqBpyFI,MAAA,KACA,iBAAA,QACA,aAAA,QAEA,gEAAA,gErBuyFJ,iDqBlyFQ,WAAA,EAAA,EAAA,EAAA,MAAA,mBD7BN,kBCHA,MAAA,QACA,aAAA,QlBrDA,wBkBwDE,MAAA,KACA,iBAAA,QACA,aAAA,QAGF,wBAAA,wBAEE,WAAA,EAAA,EAAA,EAAA,MAAA,oBAGF,2BAAA,2BAEE,MAAA,QACA,iBAAA,YAGF,uDAAA,uDrBu0FF,wCqBp0FI,MAAA,KACA,iBAAA,QACA,aAAA,QAEA,6DAAA,6DrBu0FJ,8CqBl0FQ,WAAA,EAAA,EAAA,EAAA,MAAA,oBD7BN,qBCHA,MAAA,QACA,aAAA,QlBrDA,2BkBwDE,MAAA,QACA,iBAAA,QACA,aAAA,QAGF,2BAAA,2BAEE,WAAA,EAAA,EAAA,EAAA,MAAA,mBAGF,8BAAA,8BAEE,MAAA,QACA,iBAAA,YAGF,0DAAA,0DrBu2FF,2CqBp2FI,MAAA,QACA,iBAAA,QACA,aAAA,QAEA,gEAAA,gErBu2FJ,iDqBl2FQ,WAAA,EAAA,EAAA,EAAA,MAAA,mBD7BN,oBCHA,MAAA,QACA,aAAA,QlBrDA,0BkBwDE,MAAA,KACA,iBAAA,QACA,aAAA,QAGF,0BAAA,0BAEE,WAAA,EAAA,EAAA,EAAA,MAAA,mBAGF,6BAAA,6BAEE,MAAA,QACA,iBAAA,YAGF,yDAAA,yDrBu4FF,0CqBp4FI,MAAA,KACA,iBAAA,QACA,aAAA,QAEA,+DAAA,+DrBu4FJ,gDqBl4FQ,WAAA,EAAA,EAAA,EAAA,MAAA,mBD7BN,mBCHA,MAAA,QACA,aAAA,QlBrDA,yBkBwDE,MAAA,QACA,iBAAA,QACA,aAAA,QAGF,yBAAA,yBAEE,WAAA,EAAA,EAAA,EAAA,MAAA,qBAGF,4BAAA,4BAEE,MAAA,QACA,iBAAA,YAGF,wDAAA,wDrBu6FF,yCqBp6FI,MAAA,QACA,iBAAA,QACA,aAAA,QAEA,8DAAA,8DrBu6FJ,+CqBl6FQ,WAAA,EAAA,EAAA,EAAA,MAAA,qBD7BN,kBCHA,MAAA,QACA,aAAA,QlBrDA,wBkBwDE,MAAA,KACA,iBAAA,QACA,aAAA,QAGF,wBAAA,wBAEE,WAAA,EAAA,EAAA,EAAA,MAAA,kBAGF,2BAAA,2BAEE,MAAA,QACA,iBAAA,YAGF,uDAAA,uDrBu8FF,wCqBp8FI,MAAA,KACA,iBAAA,QACA,aAAA,QAEA,6DAAA,6DrBu8FJ,8CqBl8FQ,WAAA,EAAA,EAAA,EAAA,MAAA,kBDlBR,UACE,YAAA,IACA,MAAA,QACA,gBAAA,KjBrEA,gBiBwEE,MAAA,QACA,gBAAA,UAPJ,gBAAA,gBAYI,gBAAA,UACA,WAAA,KAbJ,mBAAA,mBAkBI,MAAA,QACA,eAAA,KAWJ,mBAAA,QCJE,QAAA,MAAA,KpBuBI,UAAA,QoBrBJ,YAAA,IbrGE,cAAA,MY2GJ,mBAAA,QCRE,QAAA,OAAA,MpBuBI,UAAA,QoBrBJ,YAAA,IbrGE,cAAA,MYoHJ,WACE,QAAA,MACA,MAAA,KAFF,sBAMI,WAAA,MpBq9FJ,6BADA,4BoB/8FA,6BAII,MAAA,KExIJ,MLMM,WAAA,QAAA,KAAA,OAKF,uCKXJ,MLYM,WAAA,MKZN,iBAII,QAAA,EAIJ,qBAEI,QAAA,KAIJ,YACE,SAAA,SACA,OAAA,EACA,SAAA,OLXI,WAAA,OAAA,KAAA,KAKF,uCKGJ,YLFM,WAAA,MjB6mGN,UACA,UAFA,WuBvnGA,QAIE,SAAA,SAGF,iBACE,YAAA,OCoBE,wBACE,QAAA,aACA,YAAA,OACA,eAAA,OACA,QAAA,GAhCJ,WAAA,KAAA,MACA,aAAA,KAAA,MAAA,YACA,cAAA,EACA,YAAA,KAAA,MAAA,YAqDE,8BACE,YAAA,ED1CN,eACE,SAAA,SACA,IAAA,KACA,KAAA,EACA,QAAA,KACA,QAAA,KACA,MAAA,KACA,UAAA,MACA,QAAA,MAAA,EACA,OAAA,QAAA,EAAA,EtBsGI,UAAA,KsBpGJ,MAAA,QACA,WAAA,KACA,WAAA,KACA,iBAAA,KACA,gBAAA,YACA,OAAA,IAAA,MAAA,gBf3BE,cAAA,OeoCA,oBACE,MAAA,KACA,KAAA,EAGF,qBACE,MAAA,EACA,KAAA,KXYF,yBWnBA,uBACE,MAAA,KACA,KAAA,EAGF,wBACE,MAAA,EACA,KAAA,MXYF,yBWnBA,uBACE,MAAA,KACA,KAAA,EAGF,wBACE,MAAA,EACA,KAAA,MXYF,yBWnBA,uBACE,MAAA,KACA,KAAA,EAGF,wBACE,MAAA,EACA,KAAA,MXYF,0BWnBA,uBACE,MAAA,KACA,KAAA,EAGF,wBACE,MAAA,EACA,KAAA,MAON,uBAEI,IAAA,KACA,OAAA,KACA,WAAA,EACA,cAAA,QC/BA,gCACE,QAAA,aACA,YAAA,OACA,eAAA,OACA,QAAA,GAzBJ,WAAA,EACA,aAAA,KAAA,MAAA,YACA,cAAA,KAAA,MACA,YAAA,KAAA,MAAA,YA8CE,sCACE,YAAA,EDUN,0BAEI,IAAA,EACA,MAAA,KACA,KAAA,KACA,WAAA,EACA,YAAA,QC7CA,mCACE,QAAA,aACA,YAAA,OACA,eAAA,OACA,QAAA,GAlBJ,WAAA,KAAA,MAAA,YACA,aAAA,EACA,cAAA,KAAA,MAAA,YACA,YAAA,KAAA,MAuCE,yCACE,YAAA,EA7BF,mCDmDE,eAAA,EAKN,yBAEI,IAAA,EACA,MAAA,KACA,KAAA,KACA,WAAA,EACA,aAAA,QC9DA,kCACE,QAAA,aACA,YAAA,OACA,eAAA,OACA,QAAA,GAJF,kCAgBI,QAAA,KAGF,mCACE,QAAA,aACA,aAAA,OACA,eAAA,OACA,QAAA,GA9BN,WAAA,KAAA,MAAA,YACA,aAAA,KAAA,MACA,cAAA,KAAA,MAAA,YAiCE,wCACE,YAAA,EAVA,mCDiDA,eAAA,EAON,oCAAA,kCAAA,mCAAA,iCAKI,MAAA,KACA,OAAA,KAKJ,kBE9GE,OAAA,EACA,OAAA,MAAA,EACA,SAAA,OACA,WAAA,IAAA,MAAA,QFkHF,eACE,QAAA,MACA,MAAA,KACA,QAAA,OAAA,OACA,MAAA,KACA,YAAA,IACA,MAAA,QACA,WAAA,QACA,YAAA,OACA,iBAAA,YACA,OAAA,EpBpHA,qBAAA,qBoBmIE,MAAA,QACA,gBAAA,KJ9IA,iBAAA,QIoHJ,sBAAA,sBAgCI,MAAA,KACA,gBAAA,KJrJA,iBAAA,QIoHJ,wBAAA,wBAuCI,MAAA,QACA,eAAA,KACA,iBAAA,YAQJ,oBACE,QAAA,MAIF,iBACE,QAAA,MACA,QAAA,MAAA,OACA,cAAA,EtBpDI,UAAA,QsBsDJ,MAAA,QACA,YAAA,OAIF,oBACE,QAAA,MACA,QAAA,OAAA,OACA,MAAA,QG1LF,W1B62GA,oB0B32GE,SAAA,SACA,QAAA,mBAAA,QAAA,YACA,eAAA,O1Bi3GF,yB0Br3GA,gBAOI,SAAA,SACA,SAAA,EAAA,EAAA,KAAA,KAAA,EAAA,EAAA,K1Bo3GJ,+BGn3GE,sBuBII,QAAA,E1Bs3GN,gCADA,gCADA,+B0Bj4GA,uBAAA,uBAAA,sBAkBM,QAAA,EAMN,aACE,QAAA,YAAA,QAAA,KACA,cAAA,KAAA,UAAA,KACA,cAAA,MAAA,gBAAA,WAHF,0BAMI,MAAA,K1Bu3GJ,wC0Bn3GA,kCAII,YAAA,K1Bo3GJ,4C0Bx3GA,uDlBhBI,wBAAA,EACA,2BAAA,ER64GJ,6C0B93GA,kClBFI,uBAAA,EACA,0BAAA,EkBgCJ,uBACE,cAAA,SACA,aAAA,SAFF,8B1B22GA,yCADA,sC0Bn2GI,YAAA,EAGF,yCACE,aAAA,EAIJ,0CAAA,+BACE,cAAA,QACA,aAAA,QAGF,0CAAA,+BACE,cAAA,OACA,aAAA,OAoBF,oBACE,mBAAA,OAAA,eAAA,OACA,eAAA,MAAA,YAAA,WACA,cAAA,OAAA,gBAAA,OAHF,yB1B61GA,+B0Bt1GI,MAAA,K1B21GJ,iD0Bl2GA,2CAYI,WAAA,K1B21GJ,qD0Bv2GA,gElBlFI,2BAAA,EACA,0BAAA,ER87GJ,sD0B72GA,2ClBhGI,uBAAA,EACA,wBAAA,EkBuIJ,uB1B20GA,kC0Bx0GI,cAAA,E1B60GJ,4C0Bh1GA,yC1Bk1GA,uDADA,oD0B10GM,SAAA,SACA,KAAA,cACA,eAAA,KCzJN,aACE,SAAA,SACA,QAAA,YAAA,QAAA,KACA,cAAA,KAAA,UAAA,KACA,eAAA,QAAA,YAAA,QACA,MAAA,K3Bi/GF,0BADA,4B2Br/GA,2B3Bo/GA,qC2Bz+GI,SAAA,SACA,SAAA,EAAA,EAAA,GAAA,KAAA,EAAA,EAAA,GACA,UAAA,EACA,cAAA,E3B2/GJ,uCADA,yCADA,wCADA,yCADA,2CADA,0CAJA,wCADA,0C2B//GA,yC3BmgHA,kDADA,oDADA,mD2B9+GM,YAAA,K3B4/GN,sEADA,kC2B9gHA,iCA2BI,QAAA,EA3BJ,mDAgCI,QAAA,E3Bw/GJ,6C2BxhHA,4CnBeI,wBAAA,EACA,2BAAA,ER8gHJ,8C2B9hHA,6CnB6BI,uBAAA,EACA,0BAAA,EmB9BJ,0BA4CI,QAAA,YAAA,QAAA,KACA,eAAA,OAAA,YAAA,OA7CJ,8D3B2iHA,qEQ5hHI,wBAAA,EACA,2BAAA,EmBhBJ,+DnB6BI,uBAAA,EACA,0BAAA,ERwhHJ,oB2B1/GA,qBAEE,QAAA,YAAA,QAAA,K3B8/GF,yB2BhgHA,0BAQI,SAAA,SACA,QAAA,E3B6/GJ,+B2BtgHA,gCAYM,QAAA,E3BkgHN,8BACA,2CAEA,2CADA,wD2BhhHA,+B3B2gHA,4CAEA,4CADA,yD2Bx/GI,YAAA,KAIJ,qBAAuB,aAAA,KACvB,oBAAsB,YAAA,KAQtB,kBACE,QAAA,YAAA,QAAA,KACA,eAAA,OAAA,YAAA,OACA,QAAA,QAAA,OACA,cAAA,E1BwBI,UAAA,K0BtBJ,YAAA,IACA,YAAA,IACA,MAAA,QACA,WAAA,OACA,YAAA,OACA,iBAAA,QACA,OAAA,IAAA,MAAA,QnB1GE,cAAA,OR6mHJ,uC2B/gHA,oCAkBI,WAAA,E3BkgHJ,+B2Bx/GA,4CAEE,OAAA,yB3B2/GF,+B2Bx/GA,8B3B4/GA,yCAFA,sDACA,0CAFA,uD2Bn/GE,QAAA,MAAA,K1BXI,UAAA,Q0BaJ,YAAA,InBvIE,cAAA,MRmoHJ,+B2Bx/GA,4CAEE,OAAA,0B3B2/GF,+B2Bx/GA,8B3B4/GA,yCAFA,sDACA,0CAFA,uD2Bn/GE,QAAA,OAAA,M1B5BI,UAAA,Q0B8BJ,YAAA,InBxJE,cAAA,MmB4JJ,+B3Bw/GA,+B2Bt/GE,cAAA,Q3B8/GF,wFACA,+EAHA,uDACA,oE2Bl/GA,uC3Bg/GA,oDQzoHI,wBAAA,EACA,2BAAA,EmBiKJ,sC3Bi/GA,mDAGA,qEACA,kFAHA,yDACA,sEQvoHI,uBAAA,EACA,0BAAA,EoB3BJ,gBACE,SAAA,SACA,QAAA,MACA,WAAA,OACA,aAAA,OAGF,uBACE,QAAA,mBAAA,QAAA,YACA,aAAA,KAGF,sBACE,SAAA,SACA,KAAA,EACA,QAAA,GACA,MAAA,KACA,OAAA,QACA,QAAA,EANF,4DASI,MAAA,KACA,aAAA,QTzBA,iBAAA,QSeJ,0DAoBM,WAAA,EAAA,EAAA,EAAA,MAAA,oBApBN,wEAyBI,aAAA,QAzBJ,0EA6BI,MAAA,KACA,iBAAA,QACA,aAAA,QA/BJ,qDAAA,sDAuCM,MAAA,QAvCN,6DAAA,8DA0CQ,iBAAA,QAUR,sBACE,SAAA,SACA,cAAA,EAEA,eAAA,IAJF,8BASI,SAAA,SACA,IAAA,OACA,KAAA,QACA,QAAA,MACA,MAAA,KACA,OAAA,KACA,eAAA,KACA,QAAA,GACA,iBAAA,KACA,OAAA,QAAA,MAAA,IAlBJ,6BAwBI,SAAA,SACA,IAAA,OACA,KAAA,QACA,QAAA,MACA,MAAA,KACA,OAAA,KACA,QAAA,GACA,WAAA,UAAA,GAAA,CAAA,IAAA,IASJ,+CpB5GI,cAAA,OoB4GJ,4EAOM,iBAAA,iNAPN,mFAaM,aAAA,QTxHF,iBAAA,QS2GJ,kFAkBM,iBAAA,8JAlBN,sFAwBM,iBAAA,mBAxBN,4FA2BM,iBAAA,mBASN,4CAGI,cAAA,IAHJ,yEAQM,iBAAA,6JARN,mFAcM,iBAAA,mBAUN,eACE,aAAA,QADF,6CAKM,KAAA,SACA,MAAA,QACA,eAAA,IAEA,cAAA,MATN,4CAaM,IAAA,mBACA,KAAA,qBACA,MAAA,iBACA,OAAA,iBACA,iBAAA,QAEA,cAAA,MX1LA,WAAA,iBAAA,KAAA,WAAA,CAAA,aAAA,KAAA,WAAA,CAAA,WAAA,KAAA,WAAA,CAAA,kBAAA,KAAA,YAAA,WAAA,UAAA,KAAA,WAAA,CAAA,iBAAA,KAAA,WAAA,CAAA,aAAA,KAAA,WAAA,CAAA,WAAA,KAAA,YAAA,WAAA,UAAA,KAAA,WAAA,CAAA,iBAAA,KAAA,WAAA,CAAA,aAAA,KAAA,WAAA,CAAA,WAAA,KAAA,WAAA,CAAA,kBAAA,KAAA,YAKF,uCWkKJ,4CXjKM,WAAA,MWiKN,0EA0BM,iBAAA,KACA,kBAAA,mBAAA,UAAA,mBA3BN,oFAiCM,iBAAA,mBAYN,eACE,QAAA,aACA,MAAA,KACA,OAAA,2BACA,QAAA,QAAA,QAAA,QAAA,O3B/FI,UAAA,K2BkGJ,YAAA,IACA,YAAA,IACA,MAAA,QACA,eAAA,OACA,WAAA,KAAA,+KAAA,UAAA,MAAA,OAAA,MAAA,CAAA,IAAA,KACA,OAAA,IAAA,MAAA,QpBjOE,cAAA,OoBoOF,mBAAA,KAAA,gBAAA,KAAA,WAAA,KAfF,qBAkBI,aAAA,QACA,QAAA,EAIE,WAAA,EAAA,EAAA,EAAA,MAAA,oBAvBN,gCAgCM,MAAA,QACA,iBAAA,KAjCN,yBAAA,qCAuCI,OAAA,KACA,cAAA,OACA,iBAAA,KAzCJ,wBA6CI,MAAA,QACA,iBAAA,QA9CJ,2BAmDI,QAAA,KAnDJ,8BAwDI,MAAA,YACA,YAAA,EAAA,EAAA,EAAA,QAIJ,kBACE,OAAA,0BACA,YAAA,OACA,eAAA,OACA,aAAA,M3B5JI,UAAA,Q2BgKN,kBACE,OAAA,yBACA,YAAA,MACA,eAAA,MACA,aAAA,K3BpKI,UAAA,Q2B6KN,aACE,SAAA,SACA,QAAA,aACA,MAAA,KACA,OAAA,2BACA,cAAA,EAGF,mBACE,SAAA,SACA,QAAA,EACA,MAAA,KACA,OAAA,2BACA,OAAA,EACA,QAAA,EANF,4CASI,aAAA,QACA,WAAA,EAAA,EAAA,EAAA,MAAA,oB5BumHJ,+C4BjnHA,gDAgBI,iBAAA,QAhBJ,sDAqBM,QAAA,SArBN,0DA0BI,QAAA,kBAIJ,mBACE,SAAA,SACA,IAAA,EACA,MAAA,EACA,KAAA,EACA,QAAA,EACA,OAAA,2BACA,QAAA,QAAA,OAEA,YAAA,IACA,YAAA,IACA,MAAA,QACA,iBAAA,KACA,OAAA,IAAA,MAAA,QpB1VE,cAAA,OoB6UJ,0BAkBI,SAAA,SACA,IAAA,EACA,MAAA,EACA,OAAA,EACA,QAAA,EACA,QAAA,MACA,OAAA,qBACA,QAAA,QAAA,OACA,YAAA,IACA,MAAA,QACA,QAAA,STxWA,iBAAA,QS0WA,YAAA,QpB3WA,cAAA,EAAA,OAAA,OAAA,EoBsXJ,cACE,MAAA,KACA,OAAA,OACA,QAAA,EACA,iBAAA,YACA,mBAAA,KAAA,gBAAA,KAAA,WAAA,KALF,oBAQI,QAAA,EARJ,0CAY8B,WAAA,EAAA,EAAA,EAAA,IAAA,IAAA,CAAA,EAAA,EAAA,EAAA,MAAA,oBAZ9B,sCAa8B,WAAA,EAAA,EAAA,EAAA,IAAA,IAAA,CAAA,EAAA,EAAA,EAAA,MAAA,oBAb9B,+BAc8B,WAAA,EAAA,EAAA,EAAA,IAAA,IAAA,CAAA,EAAA,EAAA,EAAA,MAAA,oBAd9B,gCAkBI,OAAA,EAlBJ,oCAsBI,MAAA,KACA,OAAA,KACA,WAAA,QT7YA,iBAAA,QS+YA,OAAA,EpBhZA,cAAA,KSCE,mBAAA,iBAAA,KAAA,WAAA,CAAA,aAAA,KAAA,WAAA,CAAA,WAAA,KAAA,YAAA,WAAA,iBAAA,KAAA,WAAA,CAAA,aAAA,KAAA,WAAA,CAAA,WAAA,KAAA,YWmZF,mBAAA,KAAA,WAAA,KX9YA,uCWgXJ,oCX/WM,mBAAA,KAAA,WAAA,MW+WN,2CTrXI,iBAAA,QSqXJ,6CAsCI,MAAA,KACA,OAAA,MACA,MAAA,YACA,OAAA,QACA,iBAAA,QACA,aAAA,YpBjaA,cAAA,KoBsXJ,gCAiDI,MAAA,KACA,OAAA,KTvaA,iBAAA,QSyaA,OAAA,EpB1aA,cAAA,KSCE,gBAAA,iBAAA,KAAA,WAAA,CAAA,aAAA,KAAA,WAAA,CAAA,WAAA,KAAA,YAAA,WAAA,iBAAA,KAAA,WAAA,CAAA,aAAA,KAAA,WAAA,CAAA,WAAA,KAAA,YW6aF,gBAAA,KAAA,WAAA,KXxaA,uCWgXJ,gCX/WM,gBAAA,KAAA,WAAA,MW+WN,uCTrXI,iBAAA,QSqXJ,gCAgEI,MAAA,KACA,OAAA,MACA,MAAA,YACA,OAAA,QACA,iBAAA,QACA,aAAA,YpB3bA,cAAA,KoBsXJ,yBA2EI,MAAA,KACA,OAAA,KACA,WAAA,EACA,aAAA,MACA,YAAA,MTpcA,iBAAA,QSscA,OAAA,EpBvcA,cAAA,KSCE,eAAA,iBAAA,KAAA,WAAA,CAAA,aAAA,KAAA,WAAA,CAAA,WAAA,KAAA,YAAA,WAAA,iBAAA,KAAA,WAAA,CAAA,aAAA,KAAA,WAAA,CAAA,WAAA,KAAA,YW0cF,WAAA,KXrcA,uCWgXJ,yBX/WM,eAAA,KAAA,WAAA,MW+WN,gCTrXI,iBAAA,QSqXJ,yBA6FI,MAAA,KACA,OAAA,MACA,MAAA,YACA,OAAA,QACA,iBAAA,YACA,aAAA,YACA,aAAA,MAnGJ,8BAwGI,iBAAA,QpB9dA,cAAA,KoBsXJ,8BA6GI,aAAA,KACA,iBAAA,QpBpeA,cAAA,KoBsXJ,6CAoHM,iBAAA,QApHN,sDAwHM,OAAA,QAxHN,yCA4HM,iBAAA,QA5HN,yCAgIM,OAAA,QAhIN,kCAoIM,iBAAA,QAKN,8B5BknHA,mBACA,eiBjnIM,WAAA,iBAAA,KAAA,WAAA,CAAA,aAAA,KAAA,WAAA,CAAA,WAAA,KAAA,YAKF,uCWyfJ,8B5BynHE,mBACA,eiBlnII,WAAA,MYPN,KACE,QAAA,YAAA,QAAA,KACA,cAAA,KAAA,UAAA,KACA,aAAA,EACA,cAAA,EACA,WAAA,KAGF,UACE,QAAA,MACA,QAAA,MAAA,K1BCA,gBAAA,gB0BEE,gBAAA,KALJ,mBAUI,MAAA,QACA,eAAA,KACA,OAAA,QAQJ,UACE,cAAA,IAAA,MAAA,QADF,oBAII,cAAA,KAJJ,oBAQI,OAAA,IAAA,MAAA,YrB3BA,uBAAA,OACA,wBAAA,OLCF,0BAAA,0B0B6BI,aAAA,QAAA,QAAA,QAZN,6BAgBM,MAAA,QACA,iBAAA,YACA,aAAA,Y7BkoIN,mC6BppIA,2BAwBI,MAAA,QACA,iBAAA,KACA,aAAA,QAAA,QAAA,KA1BJ,yBA+BI,WAAA,KrBlDA,uBAAA,EACA,wBAAA,EqB4DJ,qBrBtEI,cAAA,OqBsEJ,4B7B2nIA,2B6BpnII,MAAA,KACA,iBAAA,QASJ,oBAEI,SAAA,EAAA,EAAA,KAAA,KAAA,EAAA,EAAA,KACA,WAAA,OAIJ,yBAEI,wBAAA,EAAA,WAAA,EACA,kBAAA,EAAA,UAAA,EACA,WAAA,OASJ,uBAEI,QAAA,KAFJ,qBAKI,QAAA,MCpGJ,QACE,SAAA,SACA,QAAA,YAAA,QAAA,KACA,cAAA,KAAA,UAAA,KACA,eAAA,OAAA,YAAA,OACA,cAAA,QAAA,gBAAA,cACA,QAAA,MAAA,KANF,mB9B8tIA,yBAAwE,sBAAvB,sBAAvB,sBAAqE,sB8BntI3F,QAAA,YAAA,QAAA,KACA,cAAA,KAAA,UAAA,KACA,eAAA,OAAA,YAAA,OACA,cAAA,QAAA,gBAAA,cAoBJ,cACE,QAAA,aACA,YAAA,SACA,eAAA,SACA,aAAA,K7BwEI,UAAA,Q6BtEJ,YAAA,QACA,YAAA,O3B1CA,oBAAA,oB2B6CE,gBAAA,KASJ,YACE,QAAA,YAAA,QAAA,KACA,mBAAA,OAAA,eAAA,OACA,aAAA,EACA,cAAA,EACA,WAAA,KALF,sBAQI,cAAA,EACA,aAAA,EATJ,2BAaI,SAAA,OACA,MAAA,KASJ,aACE,QAAA,aACA,YAAA,MACA,eAAA,MAYF,iBACE,wBAAA,KAAA,WAAA,KACA,kBAAA,EAAA,UAAA,EAGA,eAAA,OAAA,YAAA,OAIF,gBACE,QAAA,OAAA,O7BSI,UAAA,Q6BPJ,YAAA,EACA,iBAAA,YACA,OAAA,IAAA,MAAA,YtBrHE,cAAA,OLWF,sBAAA,sB2B8GE,gBAAA,KAMJ,qBACE,QAAA,aACA,MAAA,MACA,OAAA,MACA,eAAA,OACA,QAAA,GACA,WAAA,UAAA,OAAA,OACA,gBAAA,KAAA,KlBlEE,4BkB4EC,6B9B+qIH,mCAA4G,gCAAnC,gCAAnC,gCAAyG,gC8B5qIvI,cAAA,EACA,aAAA,GlB7FN,yBkByFA,kBAoBI,cAAA,IAAA,OAAA,UAAA,IAAA,OACA,cAAA,MAAA,gBAAA,WArBH,8BAwBK,mBAAA,IAAA,eAAA,IAxBL,6CA2BO,SAAA,SA3BP,wCA+BO,cAAA,MACA,aAAA,MAhCP,6B9BwsIH,mCAA4G,gCAAnC,gCAAnC,gCAAyG,gC8BlqIvI,cAAA,OAAA,UAAA,OAtCL,mCAqDK,QAAA,sBAAA,QAAA,eAGA,wBAAA,KAAA,WAAA,KAxDL,kCA4DK,QAAA,MlBxIN,4BkB4EC,6B9BytIH,mCAA4G,gCAAnC,gCAAnC,gCAAyG,gC8BttIvI,cAAA,EACA,aAAA,GlB7FN,yBkByFA,kBAoBI,cAAA,IAAA,OAAA,UAAA,IAAA,OACA,cAAA,MAAA,gBAAA,WArBH,8BAwBK,mBAAA,IAAA,eAAA,IAxBL,6CA2BO,SAAA,SA3BP,wCA+BO,cAAA,MACA,aAAA,MAhCP,6B9BkvIH,mCAA4G,gCAAnC,gCAAnC,gCAAyG,gC8B5sIvI,cAAA,OAAA,UAAA,OAtCL,mCAqDK,QAAA,sBAAA,QAAA,eAGA,wBAAA,KAAA,WAAA,KAxDL,kCA4DK,QAAA,MlBxIN,4BkB4EC,6B9BmwIH,mCAA4G,gCAAnC,gCAAnC,gCAAyG,gC8BhwIvI,cAAA,EACA,aAAA,GlB7FN,yBkByFA,kBAoBI,cAAA,IAAA,OAAA,UAAA,IAAA,OACA,cAAA,MAAA,gBAAA,WArBH,8BAwBK,mBAAA,IAAA,eAAA,IAxBL,6CA2BO,SAAA,SA3BP,wCA+BO,cAAA,MACA,aAAA,MAhCP,6B9B4xIH,mCAA4G,gCAAnC,gCAAnC,gCAAyG,gC8BtvIvI,cAAA,OAAA,UAAA,OAtCL,mCAqDK,QAAA,sBAAA,QAAA,eAGA,wBAAA,KAAA,WAAA,KAxDL,kCA4DK,QAAA,MlBxIN,6BkB4EC,6B9B6yIH,mCAA4G,gCAAnC,gCAAnC,gCAAyG,gC8B1yIvI,cAAA,EACA,aAAA,GlB7FN,0BkByFA,kBAoBI,cAAA,IAAA,OAAA,UAAA,IAAA,OACA,cAAA,MAAA,gBAAA,WArBH,8BAwBK,mBAAA,IAAA,eAAA,IAxBL,6CA2BO,SAAA,SA3BP,wCA+BO,cAAA,MACA,aAAA,MAhCP,6B9Bs0IH,mCAA4G,gCAAnC,gCAAnC,gCAAyG,gC8BhyIvI,cAAA,OAAA,UAAA,OAtCL,mCAqDK,QAAA,sBAAA,QAAA,eAGA,wBAAA,KAAA,WAAA,KAxDL,kCA4DK,QAAA,MAjEV,eAyBQ,cAAA,IAAA,OAAA,UAAA,IAAA,OACA,cAAA,MAAA,gBAAA,WA1BR,0B9Bk2IA,gCAAmG,6BAAhC,6BAAhC,6BAAgG,6B8B11IzH,cAAA,EACA,aAAA,EATV,2BA6BU,mBAAA,IAAA,eAAA,IA7BV,0CAgCY,SAAA,SAhCZ,qCAoCY,cAAA,MACA,aAAA,MArCZ,0B9Bs3IA,gCAAmG,6BAAhC,6BAAhC,6BAAgG,6B8B30IzH,cAAA,OAAA,UAAA,OA3CV,gCA0DU,QAAA,sBAAA,QAAA,eAGA,wBAAA,KAAA,WAAA,KA7DV,+BAiEU,QAAA,KAaV,4BAEI,MAAA,e3BhNF,kCAAA,kC2BmNI,MAAA,eALN,oCAWM,MAAA,e3BzNJ,0CAAA,0C2B4NM,MAAA,eAdR,6CAkBQ,MAAA,e9B2zIR,4CAEA,2CADA,yC8B90IA,0CA0BM,MAAA,eA1BN,8BA+BI,MAAA,eACA,aAAA,eAhCJ,mCAoCI,iBAAA,8PApCJ,2BAwCI,MAAA,eAxCJ,6BA0CM,MAAA,e3BxPJ,mCAAA,mC2B2PM,MAAA,eAOR,2BAEI,MAAA,K3BpQF,iCAAA,iC2BuQI,MAAA,KALN,mCAWM,MAAA,qB3B7QJ,yCAAA,yC2BgRM,MAAA,sBAdR,4CAkBQ,MAAA,sB9BuzIR,2CAEA,0CADA,wC8B10IA,yCA0BM,MAAA,KA1BN,6BA+BI,MAAA,qBACA,aAAA,qBAhCJ,kCAoCI,iBAAA,oQApCJ,0BAwCI,MAAA,qBAxCJ,4BA0CM,MAAA,K3B5SJ,kCAAA,kC2B+SM,MAAA,KC3TR,MACE,SAAA,SACA,QAAA,YAAA,QAAA,KACA,mBAAA,OAAA,eAAA,OACA,UAAA,EAEA,UAAA,WACA,iBAAA,KACA,gBAAA,WACA,OAAA,IAAA,MAAA,iBvBRE,cAAA,OuBDJ,SAaI,aAAA,EACA,YAAA,EAdJ,2DvBUI,uBAAA,OACA,wBAAA,OuBXJ,yDvBwBI,2BAAA,OACA,0BAAA,OuBKJ,WAGE,SAAA,EAAA,EAAA,KAAA,KAAA,EAAA,EAAA,KAGA,WAAA,IACA,QAAA,QAIF,YACE,cAAA,OAGF,eACE,WAAA,SACA,cAAA,EAGF,sBACE,cAAA,E5B3CA,iB4BgDE,gBAAA,KAFJ,sBAMI,YAAA,QAQJ,aACE,QAAA,OAAA,QACA,cAAA,EAEA,iBAAA,gBACA,cAAA,IAAA,MAAA,iBALF,yBvBnEI,cAAA,mBAAA,mBAAA,EAAA,EuBmEJ,sDAaM,WAAA,EAKN,aACE,QAAA,OAAA,QACA,iBAAA,gBACA,WAAA,IAAA,MAAA,iBAHF,wBvBrFI,cAAA,EAAA,EAAA,mBAAA,mBuBoGJ,kBACE,aAAA,SACA,cAAA,QACA,YAAA,SACA,cAAA,EAGF,mBACE,aAAA,SACA,YAAA,SAIF,kBACE,SAAA,SACA,IAAA,EACA,MAAA,EACA,OAAA,EACA,KAAA,EACA,QAAA,QAGF,U/BgmJA,iBADA,c+B5lJE,kBAAA,EAAA,YAAA,EACA,MAAA,KAGF,U/BgmJA,cQxtJI,uBAAA,mBACA,wBAAA,mBuB4HJ,U/BimJA,iBQhtJI,2BAAA,mBACA,0BAAA,mBuBsHJ,iBAEI,cAAA,KnBzFA,yBmBuFJ,WAMI,QAAA,YAAA,QAAA,KACA,cAAA,IAAA,KAAA,UAAA,IAAA,KACA,aAAA,MACA,YAAA,MATJ,iBAaM,SAAA,EAAA,EAAA,GAAA,KAAA,EAAA,EAAA,GACA,aAAA,KACA,cAAA,EACA,YAAA,MAUN,kBAII,cAAA,KnBrHA,yBmBiHJ,YAQI,QAAA,YAAA,QAAA,KACA,cAAA,IAAA,KAAA,UAAA,IAAA,KATJ,kBAcM,SAAA,EAAA,EAAA,GAAA,KAAA,EAAA,EAAA,GACA,cAAA,EAfN,wBAkBQ,YAAA,EACA,YAAA,EAnBR,mCvBxJI,wBAAA,EACA,2BAAA,ER0wJF,gD+BnnJF,iDA8BY,wBAAA,E/BylJV,gD+BvnJF,oDAmCY,2BAAA,EAnCZ,oCvB1II,uBAAA,EACA,0BAAA,ERwwJF,iD+B/nJF,kDA6CY,uBAAA,E/BslJV,iD+BnoJF,qDAkDY,0BAAA,GAaZ,oBAEI,cAAA,OnBlLA,yBmBgLJ,cAMI,qBAAA,EAAA,kBAAA,EAAA,aAAA,EACA,mBAAA,QAAA,gBAAA,QAAA,WAAA,QACA,QAAA,EACA,OAAA,EATJ,oBAYM,QAAA,aACA,MAAA,MAUN,iBAEI,SAAA,OAFJ,oCAKM,cAAA,EvB5OF,2BAAA,EACA,0BAAA,EuBsOJ,qCvBrPI,uBAAA,EACA,wBAAA,EuBoPJ,8BvB9PI,cAAA,EuB6QE,cAAA,KClRN,YACE,QAAA,YAAA,QAAA,KACA,cAAA,KAAA,UAAA,KACA,QAAA,OAAA,KACA,cAAA,KAEA,WAAA,KACA,iBAAA,QxBFE,cAAA,OwBMJ,kCAGI,aAAA,MAHJ,0CAMM,QAAA,aACA,cAAA,MACA,MAAA,QACA,QAAA,IATN,gDAoBI,gBAAA,UApBJ,gDAwBI,gBAAA,KAxBJ,wBA4BI,MAAA,QCvCJ,YACE,QAAA,YAAA,QAAA,K5BGA,aAAA,EACA,WAAA,KGAE,cAAA,OyBCJ,WACE,SAAA,SACA,QAAA,MACA,QAAA,MAAA,OACA,YAAA,KACA,YAAA,KACA,MAAA,QACA,iBAAA,KACA,OAAA,IAAA,MAAA,QARF,iBAWI,QAAA,EACA,MAAA,QACA,gBAAA,KACA,iBAAA,QACA,aAAA,QAfJ,iBAmBI,QAAA,EACA,QAAA,EACA,WAAA,EAAA,EAAA,EAAA,MAAA,oBAIJ,kCAGM,YAAA,EzBCF,uBAAA,OACA,0BAAA,OyBLJ,iCzBVI,wBAAA,OACA,2BAAA,OyBSJ,6BAcI,QAAA,EACA,MAAA,KACA,iBAAA,QACA,aAAA,QAjBJ,+BAqBI,MAAA,QACA,eAAA,KAEA,OAAA,KACA,iBAAA,KACA,aAAA,QCtDF,0BACE,QAAA,OAAA,OjC2HE,UAAA,QiCzHF,YAAA,IAKE,iD1BwBF,uBAAA,MACA,0BAAA,M0BpBE,gD1BKF,wBAAA,MACA,2BAAA,M0BnBF,0BACE,QAAA,OAAA,MjC2HE,UAAA,QiCzHF,YAAA,IAKE,iD1BwBF,uBAAA,MACA,0BAAA,M0BpBE,gD1BKF,wBAAA,MACA,2BAAA,M2BjBJ,OACE,QAAA,aACA,QAAA,MAAA,KlCiEE,UAAA,IkC/DF,YAAA,IACA,YAAA,EACA,WAAA,OACA,YAAA,OACA,eAAA,S3BRE,cAAA,OSCE,WAAA,MAAA,KAAA,WAAA,CAAA,iBAAA,KAAA,WAAA,CAAA,aAAA,KAAA,WAAA,CAAA,WAAA,KAAA,YAKF,uCkBNJ,OlBOM,WAAA,MdIJ,cAAA,cgCGI,gBAAA,KAdN,aAoBI,QAAA,KAKJ,YACE,SAAA,SACA,IAAA,KAOF,YACE,cAAA,KACA,aAAA,K3BpCE,cAAA,M2B6CF,eCjDA,MAAA,KACA,iBAAA,QjCcA,sBAAA,sBiCVI,MAAA,KACA,iBAAA,QAHI,sBAAA,sBAQJ,QAAA,EACA,WAAA,EAAA,EAAA,EAAA,MAAA,mBDqCJ,iBCjDA,MAAA,KACA,iBAAA,QjCcA,wBAAA,wBiCVI,MAAA,KACA,iBAAA,QAHI,wBAAA,wBAQJ,QAAA,EACA,WAAA,EAAA,EAAA,EAAA,MAAA,qBDqCJ,eCjDA,MAAA,KACA,iBAAA,QjCcA,sBAAA,sBiCVI,MAAA,KACA,iBAAA,QAHI,sBAAA,sBAQJ,QAAA,EACA,WAAA,EAAA,EAAA,EAAA,MAAA,mBDqCJ,YCjDA,MAAA,KACA,iBAAA,QjCcA,mBAAA,mBiCVI,MAAA,KACA,iBAAA,QAHI,mBAAA,mBAQJ,QAAA,EACA,WAAA,EAAA,EAAA,EAAA,MAAA,oBDqCJ,eCjDA,MAAA,QACA,iBAAA,QjCcA,sBAAA,sBiCVI,MAAA,QACA,iBAAA,QAHI,sBAAA,sBAQJ,QAAA,EACA,WAAA,EAAA,EAAA,EAAA,MAAA,mBDqCJ,cCjDA,MAAA,KACA,iBAAA,QjCcA,qBAAA,qBiCVI,MAAA,KACA,iBAAA,QAHI,qBAAA,qBAQJ,QAAA,EACA,WAAA,EAAA,EAAA,EAAA,MAAA,mBDqCJ,aCjDA,MAAA,QACA,iBAAA,QjCcA,oBAAA,oBiCVI,MAAA,QACA,iBAAA,QAHI,oBAAA,oBAQJ,QAAA,EACA,WAAA,EAAA,EAAA,EAAA,MAAA,qBDqCJ,YCjDA,MAAA,KACA,iBAAA,QjCcA,mBAAA,mBiCVI,MAAA,KACA,iBAAA,QAHI,mBAAA,mBAQJ,QAAA,EACA,WAAA,EAAA,EAAA,EAAA,MAAA,kBCbN,WACE,QAAA,KAAA,KACA,cAAA,KAEA,iBAAA,Q7BCE,cAAA,MIuDA,yByB5DJ,WAQI,QAAA,KAAA,MAIJ,iBACE,cAAA,EACA,aAAA,E7BTE,cAAA,E8BDJ,OACE,SAAA,SACA,QAAA,OAAA,QACA,cAAA,KACA,OAAA,IAAA,MAAA,Y9BHE,cAAA,O8BQJ,eAEE,MAAA,QAIF,YACE,YAAA,IAQF,mBACE,cAAA,KADF,0BAKI,SAAA,SACA,IAAA,EACA,MAAA,EACA,QAAA,OAAA,QACA,MAAA,QAUF,eC9CA,MAAA,QpBKE,iBAAA,QoBHF,aAAA,QAEA,kBACE,iBAAA,QAGF,2BACE,MAAA,QDqCF,iBC9CA,MAAA,QpBKE,iBAAA,QoBHF,aAAA,QAEA,oBACE,iBAAA,QAGF,6BACE,MAAA,QDqCF,eC9CA,MAAA,QpBKE,iBAAA,QoBHF,aAAA,QAEA,kBACE,iBAAA,QAGF,2BACE,MAAA,QDqCF,YC9CA,MAAA,QpBKE,iBAAA,QoBHF,aAAA,QAEA,eACE,iBAAA,QAGF,wBACE,MAAA,QDqCF,eC9CA,MAAA,QpBKE,iBAAA,QoBHF,aAAA,QAEA,kBACE,iBAAA,QAGF,2BACE,MAAA,QDqCF,cC9CA,MAAA,QpBKE,iBAAA,QoBHF,aAAA,QAEA,iBACE,iBAAA,QAGF,0BACE,MAAA,QDqCF,aC9CA,MAAA,QpBKE,iBAAA,QoBHF,aAAA,QAEA,gBACE,iBAAA,QAGF,yBACE,MAAA,QDqCF,YC9CA,MAAA,QpBKE,iBAAA,QoBHF,aAAA,QAEA,eACE,iBAAA,QAGF,wBACE,MAAA,QCRF,wCACE,KAAO,oBAAA,KAAA,EACP,GAAK,oBAAA,EAAA,GAFP,gCACE,KAAO,oBAAA,KAAA,EACP,GAAK,oBAAA,EAAA,GAIT,UACE,QAAA,YAAA,QAAA,KACA,OAAA,KACA,SAAA,OvCoHI,UAAA,OuClHJ,iBAAA,QhCRE,cAAA,OgCaJ,cACE,QAAA,YAAA,QAAA,KACA,mBAAA,OAAA,eAAA,OACA,cAAA,OAAA,gBAAA,OACA,SAAA,OACA,MAAA,KACA,WAAA,OACA,YAAA,OACA,iBAAA,QvBpBI,WAAA,MAAA,IAAA,KAKF,uCuBOJ,cvBNM,WAAA,MuBkBN,sBrBaE,iBAAA,iKqBXA,gBAAA,KAAA,KAIA,uBACE,kBAAA,qBAAA,GAAA,OAAA,SAAA,UAAA,qBAAA,GAAA,OAAA,SAGE,uCAJJ,uBAKM,kBAAA,KAAA,UAAA,MCzCR,OACE,QAAA,YAAA,QAAA,KACA,eAAA,MAAA,YAAA,WAGF,YACE,SAAA,EAAA,KAAA,ECFF,YACE,QAAA,YAAA,QAAA,KACA,mBAAA,OAAA,eAAA,OAGA,aAAA,EACA,cAAA,EASF,wBACE,MAAA,KACA,MAAA,QACA,WAAA,QvCNA,8BAAA,8BuCUE,QAAA,EACA,MAAA,QACA,gBAAA,KACA,iBAAA,QAVJ,+BAcI,MAAA,QACA,iBAAA,QASJ,iBACE,SAAA,SACA,QAAA,MACA,QAAA,OAAA,QAEA,iBAAA,KACA,OAAA,IAAA,MAAA,iBANF,6BlC7BI,uBAAA,OACA,wBAAA,OkC4BJ,4BlCfI,2BAAA,OACA,0BAAA,OkCcJ,0BAAA,0BAkBI,MAAA,QACA,eAAA,KACA,iBAAA,KApBJ,wBAyBI,QAAA,EACA,MAAA,KACA,iBAAA,QACA,aAAA,QA5BJ,kCAgCI,iBAAA,EAhCJ,yCAmCM,WAAA,KACA,iBAAA,IAcF,uBACE,mBAAA,IAAA,eAAA,IADF,oDlCjCA,0BAAA,OAZA,wBAAA,EkC6CA,mDlC7CA,wBAAA,OAYA,0BAAA,EkCiCA,+CAeM,WAAA,EAfN,yDAmBM,iBAAA,IACA,kBAAA,EApBN,gEAuBQ,YAAA,KACA,kBAAA,I9BzDR,yB8BiCA,0BACE,mBAAA,IAAA,eAAA,IADF,uDlCjCA,0BAAA,OAZA,wBAAA,EkC6CA,sDlC7CA,wBAAA,OAYA,0BAAA,EkCiCA,kDAeM,WAAA,EAfN,4DAmBM,iBAAA,IACA,kBAAA,EApBN,mEAuBQ,YAAA,KACA,kBAAA,K9BzDR,yB8BiCA,0BACE,mBAAA,IAAA,eAAA,IADF,uDlCjCA,0BAAA,OAZA,wBAAA,EkC6CA,sDlC7CA,wBAAA,OAYA,0BAAA,EkCiCA,kDAeM,WAAA,EAfN,4DAmBM,iBAAA,IACA,kBAAA,EApBN,mEAuBQ,YAAA,KACA,kBAAA,K9BzDR,yB8BiCA,0BACE,mBAAA,IAAA,eAAA,IADF,uDlCjCA,0BAAA,OAZA,wBAAA,EkC6CA,sDlC7CA,wBAAA,OAYA,0BAAA,EkCiCA,kDAeM,WAAA,EAfN,4DAmBM,iBAAA,IACA,kBAAA,EApBN,mEAuBQ,YAAA,KACA,kBAAA,K9BzDR,0B8BiCA,0BACE,mBAAA,IAAA,eAAA,IADF,uDlCjCA,0BAAA,OAZA,wBAAA,EkC6CA,sDlC7CA,wBAAA,OAYA,0BAAA,EkCiCA,kDAeM,WAAA,EAfN,4DAmBM,iBAAA,IACA,kBAAA,EApBN,mEAuBQ,YAAA,KACA,kBAAA,KAcZ,mCAEI,mBAAA,EACA,kBAAA,ElCjIA,cAAA,EkC8HJ,+CAOM,iBAAA,EAPN,yDAaM,oBAAA,EC7IJ,yBACE,MAAA,QACA,iBAAA,QxCWF,sDAAA,sDwCPM,MAAA,QACA,iBAAA,QAPN,uDAWM,MAAA,KACA,iBAAA,QACA,aAAA,QAbN,2BACE,MAAA,QACA,iBAAA,QxCWF,wDAAA,wDwCPM,MAAA,QACA,iBAAA,QAPN,yDAWM,MAAA,KACA,iBAAA,QACA,aAAA,QAbN,yBACE,MAAA,QACA,iBAAA,QxCWF,sDAAA,sDwCPM,MAAA,QACA,iBAAA,QAPN,uDAWM,MAAA,KACA,iBAAA,QACA,aAAA,QAbN,sBACE,MAAA,QACA,iBAAA,QxCWF,mDAAA,mDwCPM,MAAA,QACA,iBAAA,QAPN,oDAWM,MAAA,KACA,iBAAA,QACA,aAAA,QAbN,yBACE,MAAA,QACA,iBAAA,QxCWF,sDAAA,sDwCPM,MAAA,QACA,iBAAA,QAPN,uDAWM,MAAA,KACA,iBAAA,QACA,aAAA,QAbN,wBACE,MAAA,QACA,iBAAA,QxCWF,qDAAA,qDwCPM,MAAA,QACA,iBAAA,QAPN,sDAWM,MAAA,KACA,iBAAA,QACA,aAAA,QAbN,uBACE,MAAA,QACA,iBAAA,QxCWF,oDAAA,oDwCPM,MAAA,QACA,iBAAA,QAPN,qDAWM,MAAA,KACA,iBAAA,QACA,aAAA,QAbN,sBACE,MAAA,QACA,iBAAA,QxCWF,mDAAA,mDwCPM,MAAA,QACA,iBAAA,QAPN,oDAWM,MAAA,KACA,iBAAA,QACA,aAAA,QChBR,OACE,MAAA,M3C8HI,UAAA,O2C5HJ,YAAA,IACA,YAAA,EACA,MAAA,KACA,YAAA,EAAA,IAAA,EAAA,KACA,QAAA,GzCKA,ayCDE,MAAA,KACA,gBAAA,KzCIF,2CAAA,2CyCCI,QAAA,IAWN,aACE,QAAA,EACA,iBAAA,YACA,OAAA,EACA,mBAAA,KAAA,gBAAA,KAAA,WAAA,KAMF,iBACE,eAAA,KCvCF,OACE,UAAA,MACA,SAAA,O5C6HI,UAAA,Q4C1HJ,iBAAA,sBACA,gBAAA,YACA,OAAA,IAAA,MAAA,eACA,WAAA,EAAA,OAAA,OAAA,eACA,wBAAA,WAAA,gBAAA,WACA,QAAA,ErCLE,cAAA,OqCLJ,wBAcI,cAAA,OAdJ,eAkBI,QAAA,EAlBJ,YAsBI,QAAA,MACA,QAAA,EAvBJ,YA2BI,QAAA,KAIJ,cACE,QAAA,YAAA,QAAA,KACA,eAAA,OAAA,YAAA,OACA,QAAA,OAAA,OACA,MAAA,QACA,iBAAA,sBACA,gBAAA,YACA,cAAA,IAAA,MAAA,gBAGF,YACE,QAAA,OCpCF,YAEE,SAAA,OAFF,mBAKI,WAAA,OACA,WAAA,KAKJ,OACE,SAAA,MACA,IAAA,EACA,KAAA,EACA,QAAA,KACA,QAAA,KACA,MAAA,KACA,OAAA,KACA,SAAA,OAGA,QAAA,EAOF,cACE,SAAA,SACA,MAAA,KACA,OAAA,MAEA,eAAA,KAGA,0B7BrCI,WAAA,kBAAA,IAAA,SAAA,WAAA,UAAA,IAAA,SAAA,WAAA,UAAA,IAAA,QAAA,CAAA,kBAAA,IAAA,S6BuCF,kBAAA,mBAAA,UAAA,mB7BlCA,uC6BgCF,0B7B/BI,WAAA,M6BmCJ,0BACE,kBAAA,KAAA,UAAA,KAIF,kCACE,kBAAA,YAAA,UAAA,YAIJ,yBACE,QAAA,YAAA,QAAA,KACA,WAAA,kBAFF,wCAKI,WAAA,mBACA,SAAA,O9C6wLJ,uC8CnxLA,uCAWI,kBAAA,EAAA,YAAA,EAXJ,qCAeI,WAAA,KAIJ,uBACE,QAAA,YAAA,QAAA,KACA,eAAA,OAAA,YAAA,OACA,WAAA,kBAHF,+BAOI,QAAA,MACA,OAAA,mBACA,QAAA,GATJ,+CAcI,mBAAA,OAAA,eAAA,OACA,cAAA,OAAA,gBAAA,OACA,OAAA,KAhBJ,8DAmBM,WAAA,KAnBN,uDAuBM,QAAA,KAMN,eACE,SAAA,SACA,QAAA,YAAA,QAAA,KACA,mBAAA,OAAA,eAAA,OACA,MAAA,KAGA,eAAA,KACA,iBAAA,KACA,gBAAA,YACA,OAAA,IAAA,MAAA,etC9GE,cAAA,MsCkHF,QAAA,EAIF,gBACE,SAAA,MACA,IAAA,EACA,KAAA,EACA,QAAA,KACA,MAAA,MACA,OAAA,MACA,iBAAA,KAPF,qBAUW,QAAA,EAVX,qBAWW,QAAA,GAKX,cACE,QAAA,YAAA,QAAA,KACA,eAAA,MAAA,YAAA,WACA,cAAA,QAAA,gBAAA,cACA,QAAA,KAAA,KACA,cAAA,IAAA,MAAA,QtClIE,uBAAA,kBACA,wBAAA,kBsC4HJ,qBASI,QAAA,KAAA,KAEA,OAAA,MAAA,MAAA,MAAA,KAKJ,aACE,cAAA,EACA,YAAA,IAKF,YACE,SAAA,SAGA,SAAA,EAAA,EAAA,KAAA,KAAA,EAAA,EAAA,KACA,QAAA,KAIF,cACE,QAAA,YAAA,QAAA,KACA,cAAA,KAAA,UAAA,KACA,eAAA,OAAA,YAAA,OACA,cAAA,IAAA,gBAAA,SACA,QAAA,OACA,WAAA,IAAA,MAAA,QtCrJE,2BAAA,kBACA,0BAAA,kBsC8IJ,gBAcI,OAAA,OAKJ,yBACE,SAAA,SACA,IAAA,QACA,MAAA,KACA,OAAA,KACA,SAAA,OlCvIE,yBkCzBJ,cAuKI,UAAA,MACA,OAAA,QAAA,KAlJJ,yBAsJI,WAAA,oBAtJJ,wCAyJM,WAAA,qBAtIN,uBA2II,WAAA,oBA3IJ,+BA8IM,OAAA,qBAQJ,UAAY,UAAA,OlCtKV,yBkC0KF,U9C+vLA,U8C7vLE,UAAA,OlC5KA,0BkCiLF,UAAY,UAAA,QC5Od,SACE,SAAA,SACA,QAAA,KACA,QAAA,MACA,OAAA,ECJA,YAAA,aAAA,CAAA,kBAAA,CAAA,UAAA,CAAA,MAAA,CAAA,gBAAA,CAAA,KAAA,CAAA,WAAA,CAAA,UAAA,CAAA,mBAAA,CAAA,gBAAA,CAAA,iBAAA,CAAA,mBAEA,WAAA,OACA,YAAA,IACA,YAAA,IACA,WAAA,KACA,WAAA,MACA,gBAAA,KACA,YAAA,KACA,eAAA,KACA,eAAA,OACA,WAAA,OACA,aAAA,OACA,YAAA,OACA,WAAA,K/CgHI,UAAA,Q8CpHJ,UAAA,WACA,QAAA,EAXF,cAaW,QAAA,GAbX,gBAgBI,SAAA,SACA,QAAA,MACA,MAAA,MACA,OAAA,MAnBJ,wBAsBM,SAAA,SACA,QAAA,GACA,aAAA,YACA,aAAA,MAKN,mCAAA,gBACE,QAAA,MAAA,EADF,0CAAA,uBAII,OAAA,EAJJ,kDAAA,+BAOM,IAAA,EACA,aAAA,MAAA,MAAA,EACA,iBAAA,KAKN,qCAAA,kBACE,QAAA,EAAA,MADF,4CAAA,yBAII,KAAA,EACA,MAAA,MACA,OAAA,MANJ,oDAAA,iCASM,MAAA,EACA,aAAA,MAAA,MAAA,MAAA,EACA,mBAAA,KAKN,sCAAA,mBACE,QAAA,MAAA,EADF,6CAAA,0BAII,IAAA,EAJJ,qDAAA,kCAOM,OAAA,EACA,aAAA,EAAA,MAAA,MACA,oBAAA,KAKN,oCAAA,iBACE,QAAA,EAAA,MADF,2CAAA,wBAII,MAAA,EACA,MAAA,MACA,OAAA,MANJ,mDAAA,gCASM,KAAA,EACA,aAAA,MAAA,EAAA,MAAA,MACA,kBAAA,KAqBN,eACE,UAAA,MACA,QAAA,OAAA,MACA,MAAA,KACA,WAAA,OACA,iBAAA,KvC3GE,cAAA,OyCLJ,SACE,SAAA,SACA,IAAA,EACA,KAAA,EACA,QAAA,KACA,QAAA,MACA,UAAA,MDLA,YAAA,aAAA,CAAA,kBAAA,CAAA,UAAA,CAAA,MAAA,CAAA,gBAAA,CAAA,KAAA,CAAA,WAAA,CAAA,UAAA,CAAA,mBAAA,CAAA,gBAAA,CAAA,iBAAA,CAAA,mBAEA,WAAA,OACA,YAAA,IACA,YAAA,IACA,WAAA,KACA,WAAA,MACA,gBAAA,KACA,YAAA,KACA,eAAA,KACA,eAAA,OACA,WAAA,OACA,aAAA,OACA,YAAA,OACA,WAAA,K/CgHI,UAAA,QgDnHJ,UAAA,WACA,iBAAA,KACA,gBAAA,YACA,OAAA,IAAA,MAAA,ezCVE,cAAA,MyCLJ,gBAoBI,SAAA,SACA,QAAA,MACA,MAAA,KACA,OAAA,MACA,OAAA,EAAA,MAxBJ,uBAAA,wBA4BM,SAAA,SACA,QAAA,MACA,QAAA,GACA,aAAA,YACA,aAAA,MAKN,mCAAA,gBACE,cAAA,MADF,0CAAA,uBAII,OAAA,mBAJJ,kDAAA,+BAOM,OAAA,EACA,aAAA,MAAA,MAAA,EACA,iBAAA,gBATN,iDAAA,8BAaM,OAAA,IACA,aAAA,MAAA,MAAA,EACA,iBAAA,KAKN,qCAAA,kBACE,YAAA,MADF,4CAAA,yBAII,KAAA,mBACA,MAAA,MACA,OAAA,KACA,OAAA,MAAA,EAPJ,oDAAA,iCAUM,KAAA,EACA,aAAA,MAAA,MAAA,MAAA,EACA,mBAAA,gBAZN,mDAAA,gCAgBM,KAAA,IACA,aAAA,MAAA,MAAA,MAAA,EACA,mBAAA,KAKN,sCAAA,mBACE,WAAA,MADF,6CAAA,0BAII,IAAA,mBAJJ,qDAAA,kCAOM,IAAA,EACA,aAAA,EAAA,MAAA,MAAA,MACA,oBAAA,gBATN,oDAAA,iCAaM,IAAA,IACA,aAAA,EAAA,MAAA,MAAA,MACA,oBAAA,KAfN,8DAAA,2CAqBI,SAAA,SACA,IAAA,EACA,KAAA,IACA,QAAA,MACA,MAAA,KACA,YAAA,OACA,QAAA,GACA,cAAA,IAAA,MAAA,QAIJ,oCAAA,iBACE,aAAA,MADF,2CAAA,wBAII,MAAA,mBACA,MAAA,MACA,OAAA,KACA,OAAA,MAAA,EAPJ,mDAAA,gCAUM,MAAA,EACA,aAAA,MAAA,EAAA,MAAA,MACA,kBAAA,gBAZN,kDAAA,+BAgBM,MAAA,IACA,aAAA,MAAA,EAAA,MAAA,MACA,kBAAA,KAsBN,gBACE,QAAA,MAAA,OACA,cAAA,EhD3BI,UAAA,KgD8BJ,iBAAA,QACA,cAAA,IAAA,MAAA,QzChJE,uBAAA,kBACA,wBAAA,kByCyIJ,sBAUI,QAAA,KAIJ,cACE,QAAA,MAAA,OACA,MAAA,QC3JF,UACE,SAAA,SAGF,wBACE,iBAAA,MAAA,aAAA,MAGF,gBACE,SAAA,SACA,MAAA,KACA,SAAA,OCvBA,uBACE,QAAA,MACA,MAAA,KACA,QAAA,GDwBJ,eACE,SAAA,SACA,QAAA,KACA,MAAA,KACA,MAAA,KACA,aAAA,MACA,4BAAA,OAAA,oBAAA,OjC5BI,WAAA,kBAAA,IAAA,YAAA,WAAA,UAAA,IAAA,YAAA,WAAA,UAAA,IAAA,WAAA,CAAA,kBAAA,IAAA,YAKF,uCiCiBJ,ejChBM,WAAA,MjB6xMN,oBACA,oBkDpwMA,sBAGE,QAAA,MlDswMF,4BkDnwMA,6CAEE,kBAAA,iBAAA,UAAA,iBlDuwMF,2BkDpwMA,8CAEE,kBAAA,kBAAA,UAAA,kBAQF,8BAEI,QAAA,EACA,oBAAA,QACA,kBAAA,KAAA,UAAA,KlDmwMJ,sDACA,uDkDxwMA,qCAUI,QAAA,EACA,QAAA,EAXJ,0ClD8wMA,2CkD9vMI,QAAA,EACA,QAAA,EjCtEE,WAAA,QAAA,GAAA,IAKF,uCiCgDJ,0ClDsxME,2CiBr0MI,WAAA,MjB20MN,uBkDjwMA,uBAEE,SAAA,SACA,IAAA,EACA,OAAA,EACA,QAAA,EAEA,QAAA,YAAA,QAAA,KACA,eAAA,OAAA,YAAA,OACA,cAAA,OAAA,gBAAA,OACA,MAAA,IACA,MAAA,KACA,WAAA,OACA,QAAA,GjC7FI,WAAA,QAAA,KAAA,KAKF,uCjBg2MF,uBkDrxMF,uBjC1EM,WAAA,MjBs2MN,6BADA,6BGj2ME,6BAAA,6B+CwFE,MAAA,KACA,gBAAA,KACA,QAAA,EACA,QAAA,GAGJ,uBACE,KAAA,EAKF,uBACE,MAAA,ElD6wMF,4BkDtwMA,4BAEE,QAAA,aACA,MAAA,KACA,OAAA,KACA,WAAA,UAAA,GAAA,CAAA,KAAA,KAEF,4BACE,iBAAA,qMAEF,4BACE,iBAAA,sMASF,qBACE,SAAA,SACA,MAAA,EACA,OAAA,EACA,KAAA,EACA,QAAA,GACA,QAAA,YAAA,QAAA,KACA,cAAA,OAAA,gBAAA,OACA,aAAA,EAEA,aAAA,IACA,YAAA,IACA,WAAA,KAZF,wBAeI,WAAA,YACA,SAAA,EAAA,EAAA,KAAA,KAAA,EAAA,EAAA,KACA,MAAA,KACA,OAAA,IACA,aAAA,IACA,YAAA,IACA,YAAA,OACA,OAAA,QACA,iBAAA,KACA,gBAAA,YAEA,WAAA,KAAA,MAAA,YACA,cAAA,KAAA,MAAA,YACA,QAAA,GjCtKE,WAAA,QAAA,IAAA,KAKF,uCiCqIJ,wBjCpIM,WAAA,MiCoIN,6BAiCI,QAAA,EASJ,kBACE,SAAA,SACA,MAAA,IACA,OAAA,KACA,KAAA,IACA,QAAA,GACA,YAAA,KACA,eAAA,KACA,MAAA,KACA,WAAA,OE/LF,kCACE,GAAK,kBAAA,eAAA,UAAA,gBADP,0BACE,GAAK,kBAAA,eAAA,UAAA,gBAGP,gBACE,QAAA,aACA,MAAA,KACA,OAAA,KACA,eAAA,YACA,OAAA,MAAA,MAAA,aACA,mBAAA,YAEA,cAAA,IACA,kBAAA,eAAA,KAAA,OAAA,SAAA,UAAA,eAAA,KAAA,OAAA,SAGF,mBACE,MAAA,KACA,OAAA,KACA,aAAA,KAOF,gCACE,GACE,kBAAA,SAAA,UAAA,SAEF,IACE,QAAA,GALJ,wBACE,GACE,kBAAA,SAAA,UAAA,SAEF,IACE,QAAA,GAIJ,cACE,QAAA,aACA,MAAA,KACA,OAAA,KACA,eAAA,YACA,iBAAA,aAEA,cAAA,IACA,QAAA,EACA,kBAAA,aAAA,KAAA,OAAA,SAAA,UAAA,aAAA,KAAA,OAAA,SAGF,iBACE,MAAA,KACA,OAAA,KCnDF,gBAAqB,eAAA,mBACrB,WAAqB,eAAA,cACrB,cAAqB,eAAA,iBACrB,cAAqB,eAAA,iBACrB,mBAAqB,eAAA,sBACrB,gBAAqB,eAAA,mBCFnB,YACE,iBAAA,kBnDUF,mBAAA,mBH4hNF,wBADA,wBsDhiNM,iBAAA,kBANJ,cACE,iBAAA,kBnDUF,qBAAA,qBHsiNF,0BADA,0BsD1iNM,iBAAA,kBANJ,YACE,iBAAA,kBnDUF,mBAAA,mBHgjNF,wBADA,wBsDpjNM,iBAAA,kBANJ,SACE,iBAAA,kBnDUF,gBAAA,gBH0jNF,qBADA,qBsD9jNM,iBAAA,kBANJ,YACE,iBAAA,kBnDUF,mBAAA,mBHokNF,wBADA,wBsDxkNM,iBAAA,kBANJ,WACE,iBAAA,kBnDUF,kBAAA,kBH8kNF,uBADA,uBsDllNM,iBAAA,kBANJ,UACE,iBAAA,kBnDUF,iBAAA,iBHwlNF,sBADA,sBsD5lNM,iBAAA,kBANJ,SACE,iBAAA,kBnDUF,gBAAA,gBHkmNF,qBADA,qBsDtmNM,iBAAA,kBCCN,UACE,iBAAA,eAGF,gBACE,iBAAA,sBCXF,QAAkB,OAAA,IAAA,MAAA,kBAClB,YAAkB,WAAA,IAAA,MAAA,kBAClB,cAAkB,aAAA,IAAA,MAAA,kBAClB,eAAkB,cAAA,IAAA,MAAA,kBAClB,aAAkB,YAAA,IAAA,MAAA,kBAElB,UAAmB,OAAA,YACnB,cAAmB,WAAA,YACnB,gBAAmB,aAAA,YACnB,iBAAmB,cAAA,YACnB,eAAmB,YAAA,YAGjB,gBACE,aAAA,kBADF,kBACE,aAAA,kBADF,gBACE,aAAA,kBADF,aACE,aAAA,kBADF,gBACE,aAAA,kBADF,eACE,aAAA,kBADF,cACE,aAAA,kBADF,aACE,aAAA,kBAIJ,cACE,aAAA,eAOF,YACE,cAAA,gBAGF,SACE,cAAA,iBAGF,aACE,uBAAA,iBACA,wBAAA,iBAGF,eACE,wBAAA,iBACA,2BAAA,iBAGF,gBACE,2BAAA,iBACA,0BAAA,iBAGF,cACE,uBAAA,iBACA,0BAAA,iBAGF,YACE,cAAA,gBAGF,gBACE,cAAA,cAGF,cACE,cAAA,gBAGF,WACE,cAAA,YLxEA,iBACE,QAAA,MACA,MAAA,KACA,QAAA,GMOE,QAAwB,QAAA,eAAxB,UAAwB,QAAA,iBAAxB,gBAAwB,QAAA,uBAAxB,SAAwB,QAAA,gBAAxB,SAAwB,QAAA,gBAAxB,aAAwB,QAAA,oBAAxB,cAAwB,QAAA,qBAAxB,QAAwB,QAAA,sBAAA,QAAA,eAAxB,eAAwB,QAAA,6BAAA,QAAA,sB7CiD1B,yB6CjDE,WAAwB,QAAA,eAAxB,aAAwB,QAAA,iBAAxB,mBAAwB,QAAA,uBAAxB,YAAwB,QAAA,gBAAxB,YAAwB,QAAA,gBAAxB,gBAAwB,QAAA,oBAAxB,iBAAwB,QAAA,qBAAxB,WAAwB,QAAA,sBAAA,QAAA,eAAxB,kBAAwB,QAAA,6BAAA,QAAA,uB7CiD1B,yB6CjDE,WAAwB,QAAA,eAAxB,aAAwB,QAAA,iBAAxB,mBAAwB,QAAA,uBAAxB,YAAwB,QAAA,gBAAxB,YAAwB,QAAA,gBAAxB,gBAAwB,QAAA,oBAAxB,iBAAwB,QAAA,qBAAxB,WAAwB,QAAA,sBAAA,QAAA,eAAxB,kBAAwB,QAAA,6BAAA,QAAA,uB7CiD1B,yB6CjDE,WAAwB,QAAA,eAAxB,aAAwB,QAAA,iBAAxB,mBAAwB,QAAA,uBAAxB,YAAwB,QAAA,gBAAxB,YAAwB,QAAA,gBAAxB,gBAAwB,QAAA,oBAAxB,iBAAwB,QAAA,qBAAxB,WAAwB,QAAA,sBAAA,QAAA,eAAxB,kBAAwB,QAAA,6BAAA,QAAA,uB7CiD1B,0B6CjDE,WAAwB,QAAA,eAAxB,aAAwB,QAAA,iBAAxB,mBAAwB,QAAA,uBAAxB,YAAwB,QAAA,gBAAxB,YAAwB,QAAA,gBAAxB,gBAAwB,QAAA,oBAAxB,iBAAwB,QAAA,qBAAxB,WAAwB,QAAA,sBAAA,QAAA,eAAxB,kBAAwB,QAAA,6BAAA,QAAA,uBAU9B,aAEI,cAAqB,QAAA,eAArB,gBAAqB,QAAA,iBAArB,sBAAqB,QAAA,uBAArB,eAAqB,QAAA,gBAArB,eAAqB,QAAA,gBAArB,mBAAqB,QAAA,oBAArB,oBAAqB,QAAA,qBAArB,cAAqB,QAAA,sBAAA,QAAA,eAArB,qBAAqB,QAAA,6BAAA,QAAA,uBCrBzB,kBACE,SAAA,SACA,QAAA,MACA,MAAA,KACA,QAAA,EACA,SAAA,OALF,0BAQI,QAAA,MACA,QAAA,GATJ,yC1D+8NA,wBADA,yBAEA,yBACA,wB0Dh8NI,SAAA,SACA,IAAA,EACA,OAAA,EACA,KAAA,EACA,MAAA,KACA,OAAA,KACA,OAAA,EAQF,gCAEI,YAAA,WAFJ,gCAEI,YAAA,OAFJ,+BAEI,YAAA,IAFJ,+BAEI,YAAA,KCzBF,UAAgC,mBAAA,cAAA,eAAA,cAChC,aAAgC,mBAAA,iBAAA,eAAA,iBAChC,kBAAgC,mBAAA,sBAAA,eAAA,sBAChC,qBAAgC,mBAAA,yBAAA,eAAA,yBAEhC,WAA8B,cAAA,eAAA,UAAA,eAC9B,aAA8B,cAAA,iBAAA,UAAA,iBAC9B,mBAA8B,cAAA,uBAAA,UAAA,uBAC9B,WAA8B,SAAA,EAAA,EAAA,eAAA,KAAA,EAAA,EAAA,eAC9B,aAA8B,kBAAA,YAAA,UAAA,YAC9B,aAA8B,kBAAA,YAAA,UAAA,YAC9B,eAA8B,kBAAA,YAAA,YAAA,YAC9B,eAA8B,kBAAA,YAAA,YAAA,YAE9B,uBAAoC,cAAA,gBAAA,gBAAA,qBACpC,qBAAoC,cAAA,cAAA,gBAAA,mBACpC,wBAAoC,cAAA,iBAAA,gBAAA,iBACpC,yBAAoC,cAAA,kBAAA,gBAAA,wBACpC,wBAAoC,cAAA,qBAAA,gBAAA,uBAEpC,mBAAiC,eAAA,gBAAA,YAAA,qBACjC,iBAAiC,eAAA,cAAA,YAAA,mBACjC,oBAAiC,eAAA,iBAAA,YAAA,iBACjC,sBAAiC,eAAA,mBAAA,YAAA,mBACjC,qBAAiC,eAAA,kBAAA,YAAA,kBAEjC,qBAAkC,mBAAA,gBAAA,cAAA,qBAClC,mBAAkC,mBAAA,cAAA,cAAA,mBAClC,sBAAkC,mBAAA,iBAAA,cAAA,iBAClC,uBAAkC,mBAAA,kBAAA,cAAA,wBAClC,sBAAkC,mBAAA,qBAAA,cAAA,uBAClC,uBAAkC,mBAAA,kBAAA,cAAA,kBAElC,iBAAgC,oBAAA,eAAA,WAAA,eAChC,kBAAgC,oBAAA,gBAAA,WAAA,qBAChC,gBAAgC,oBAAA,cAAA,WAAA,mBAChC,mBAAgC,oBAAA,iBAAA,WAAA,iBAChC,qBAAgC,oBAAA,mBAAA,WAAA,mBAChC,oBAAgC,oBAAA,kBAAA,WAAA,kB/CYhC,yB+ClDA,aAAgC,mBAAA,cAAA,eAAA,cAChC,gBAAgC,mBAAA,iBAAA,eAAA,iBAChC,qBAAgC,mBAAA,sBAAA,eAAA,sBAChC,wBAAgC,mBAAA,yBAAA,eAAA,yBAEhC,cAA8B,cAAA,eAAA,UAAA,eAC9B,gBAA8B,cAAA,iBAAA,UAAA,iBAC9B,sBAA8B,cAAA,uBAAA,UAAA,uBAC9B,cAA8B,SAAA,EAAA,EAAA,eAAA,KAAA,EAAA,EAAA,eAC9B,gBAA8B,kBAAA,YAAA,UAAA,YAC9B,gBAA8B,kBAAA,YAAA,UAAA,YAC9B,kBAA8B,kBAAA,YAAA,YAAA,YAC9B,kBAA8B,kBAAA,YAAA,YAAA,YAE9B,0BAAoC,cAAA,gBAAA,gBAAA,qBACpC,wBAAoC,cAAA,cAAA,gBAAA,mBACpC,2BAAoC,cAAA,iBAAA,gBAAA,iBACpC,4BAAoC,cAAA,kBAAA,gBAAA,wBACpC,2BAAoC,cAAA,qBAAA,gBAAA,uBAEpC,sBAAiC,eAAA,gBAAA,YAAA,qBACjC,oBAAiC,eAAA,cAAA,YAAA,mBACjC,uBAAiC,eAAA,iBAAA,YAAA,iBACjC,yBAAiC,eAAA,mBAAA,YAAA,mBACjC,wBAAiC,eAAA,kBAAA,YAAA,kBAEjC,wBAAkC,mBAAA,gBAAA,cAAA,qBAClC,sBAAkC,mBAAA,cAAA,cAAA,mBAClC,yBAAkC,mBAAA,iBAAA,cAAA,iBAClC,0BAAkC,mBAAA,kBAAA,cAAA,wBAClC,yBAAkC,mBAAA,qBAAA,cAAA,uBAClC,0BAAkC,mBAAA,kBAAA,cAAA,kBAElC,oBAAgC,oBAAA,eAAA,WAAA,eAChC,qBAAgC,oBAAA,gBAAA,WAAA,qBAChC,mBAAgC,oBAAA,cAAA,WAAA,mBAChC,sBAAgC,oBAAA,iBAAA,WAAA,iBAChC,wBAAgC,oBAAA,mBAAA,WAAA,mBAChC,uBAAgC,oBAAA,kBAAA,WAAA,mB/CYhC,yB+ClDA,aAAgC,mBAAA,cAAA,eAAA,cAChC,gBAAgC,mBAAA,iBAAA,eAAA,iBAChC,qBAAgC,mBAAA,sBAAA,eAAA,sBAChC,wBAAgC,mBAAA,yBAAA,eAAA,yBAEhC,cAA8B,cAAA,eAAA,UAAA,eAC9B,gBAA8B,cAAA,iBAAA,UAAA,iBAC9B,sBAA8B,cAAA,uBAAA,UAAA,uBAC9B,cAA8B,SAAA,EAAA,EAAA,eAAA,KAAA,EAAA,EAAA,eAC9B,gBAA8B,kBAAA,YAAA,UAAA,YAC9B,gBAA8B,kBAAA,YAAA,UAAA,YAC9B,kBAA8B,kBAAA,YAAA,YAAA,YAC9B,kBAA8B,kBAAA,YAAA,YAAA,YAE9B,0BAAoC,cAAA,gBAAA,gBAAA,qBACpC,wBAAoC,cAAA,cAAA,gBAAA,mBACpC,2BAAoC,cAAA,iBAAA,gBAAA,iBACpC,4BAAoC,cAAA,kBAAA,gBAAA,wBACpC,2BAAoC,cAAA,qBAAA,gBAAA,uBAEpC,sBAAiC,eAAA,gBAAA,YAAA,qBACjC,oBAAiC,eAAA,cAAA,YAAA,mBACjC,uBAAiC,eAAA,iBAAA,YAAA,iBACjC,yBAAiC,eAAA,mBAAA,YAAA,mBACjC,wBAAiC,eAAA,kBAAA,YAAA,kBAEjC,wBAAkC,mBAAA,gBAAA,cAAA,qBAClC,sBAAkC,mBAAA,cAAA,cAAA,mBAClC,yBAAkC,mBAAA,iBAAA,cAAA,iBAClC,0BAAkC,mBAAA,kBAAA,cAAA,wBAClC,yBAAkC,mBAAA,qBAAA,cAAA,uBAClC,0BAAkC,mBAAA,kBAAA,cAAA,kBAElC,oBAAgC,oBAAA,eAAA,WAAA,eAChC,qBAAgC,oBAAA,gBAAA,WAAA,qBAChC,mBAAgC,oBAAA,cAAA,WAAA,mBAChC,sBAAgC,oBAAA,iBAAA,WAAA,iBAChC,wBAAgC,oBAAA,mBAAA,WAAA,mBAChC,uBAAgC,oBAAA,kBAAA,WAAA,mB/CYhC,yB+ClDA,aAAgC,mBAAA,cAAA,eAAA,cAChC,gBAAgC,mBAAA,iBAAA,eAAA,iBAChC,qBAAgC,mBAAA,sBAAA,eAAA,sBAChC,wBAAgC,mBAAA,yBAAA,eAAA,yBAEhC,cAA8B,cAAA,eAAA,UAAA,eAC9B,gBAA8B,cAAA,iBAAA,UAAA,iBAC9B,sBAA8B,cAAA,uBAAA,UAAA,uBAC9B,cAA8B,SAAA,EAAA,EAAA,eAAA,KAAA,EAAA,EAAA,eAC9B,gBAA8B,kBAAA,YAAA,UAAA,YAC9B,gBAA8B,kBAAA,YAAA,UAAA,YAC9B,kBAA8B,kBAAA,YAAA,YAAA,YAC9B,kBAA8B,kBAAA,YAAA,YAAA,YAE9B,0BAAoC,cAAA,gBAAA,gBAAA,qBACpC,wBAAoC,cAAA,cAAA,gBAAA,mBACpC,2BAAoC,cAAA,iBAAA,gBAAA,iBACpC,4BAAoC,cAAA,kBAAA,gBAAA,wBACpC,2BAAoC,cAAA,qBAAA,gBAAA,uBAEpC,sBAAiC,eAAA,gBAAA,YAAA,qBACjC,oBAAiC,eAAA,cAAA,YAAA,mBACjC,uBAAiC,eAAA,iBAAA,YAAA,iBACjC,yBAAiC,eAAA,mBAAA,YAAA,mBACjC,wBAAiC,eAAA,kBAAA,YAAA,kBAEjC,wBAAkC,mBAAA,gBAAA,cAAA,qBAClC,sBAAkC,mBAAA,cAAA,cAAA,mBAClC,yBAAkC,mBAAA,iBAAA,cAAA,iBAClC,0BAAkC,mBAAA,kBAAA,cAAA,wBAClC,yBAAkC,mBAAA,qBAAA,cAAA,uBAClC,0BAAkC,mBAAA,kBAAA,cAAA,kBAElC,oBAAgC,oBAAA,eAAA,WAAA,eAChC,qBAAgC,oBAAA,gBAAA,WAAA,qBAChC,mBAAgC,oBAAA,cAAA,WAAA,mBAChC,sBAAgC,oBAAA,iBAAA,WAAA,iBAChC,wBAAgC,oBAAA,mBAAA,WAAA,mBAChC,uBAAgC,oBAAA,kBAAA,WAAA,mB/CYhC,0B+ClDA,aAAgC,mBAAA,cAAA,eAAA,cAChC,gBAAgC,mBAAA,iBAAA,eAAA,iBAChC,qBAAgC,mBAAA,sBAAA,eAAA,sBAChC,wBAAgC,mBAAA,yBAAA,eAAA,yBAEhC,cAA8B,cAAA,eAAA,UAAA,eAC9B,gBAA8B,cAAA,iBAAA,UAAA,iBAC9B,sBAA8B,cAAA,uBAAA,UAAA,uBAC9B,cAA8B,SAAA,EAAA,EAAA,eAAA,KAAA,EAAA,EAAA,eAC9B,gBAA8B,kBAAA,YAAA,UAAA,YAC9B,gBAA8B,kBAAA,YAAA,UAAA,YAC9B,kBAA8B,kBAAA,YAAA,YAAA,YAC9B,kBAA8B,kBAAA,YAAA,YAAA,YAE9B,0BAAoC,cAAA,gBAAA,gBAAA,qBACpC,wBAAoC,cAAA,cAAA,gBAAA,mBACpC,2BAAoC,cAAA,iBAAA,gBAAA,iBACpC,4BAAoC,cAAA,kBAAA,gBAAA,wBACpC,2BAAoC,cAAA,qBAAA,gBAAA,uBAEpC,sBAAiC,eAAA,gBAAA,YAAA,qBACjC,oBAAiC,eAAA,cAAA,YAAA,mBACjC,uBAAiC,eAAA,iBAAA,YAAA,iBACjC,yBAAiC,eAAA,mBAAA,YAAA,mBACjC,wBAAiC,eAAA,kBAAA,YAAA,kBAEjC,wBAAkC,mBAAA,gBAAA,cAAA,qBAClC,sBAAkC,mBAAA,cAAA,cAAA,mBAClC,yBAAkC,mBAAA,iBAAA,cAAA,iBAClC,0BAAkC,mBAAA,kBAAA,cAAA,wBAClC,yBAAkC,mBAAA,qBAAA,cAAA,uBAClC,0BAAkC,mBAAA,kBAAA,cAAA,kBAElC,oBAAgC,oBAAA,eAAA,WAAA,eAChC,qBAAgC,oBAAA,gBAAA,WAAA,qBAChC,mBAAgC,oBAAA,cAAA,WAAA,mBAChC,sBAAgC,oBAAA,iBAAA,WAAA,iBAChC,wBAAgC,oBAAA,mBAAA,WAAA,mBAChC,uBAAgC,oBAAA,kBAAA,WAAA,mBC1ChC,YAAwB,MAAA,eACxB,aAAwB,MAAA,gBACxB,YAAwB,MAAA,ehDoDxB,yBgDtDA,eAAwB,MAAA,eACxB,gBAAwB,MAAA,gBACxB,eAAwB,MAAA,gBhDoDxB,yBgDtDA,eAAwB,MAAA,eACxB,gBAAwB,MAAA,gBACxB,eAAwB,MAAA,gBhDoDxB,yBgDtDA,eAAwB,MAAA,eACxB,gBAAwB,MAAA,gBACxB,eAAwB,MAAA,gBhDoDxB,0BgDtDA,eAAwB,MAAA,eACxB,gBAAwB,MAAA,gBACxB,eAAwB,MAAA,gBCL1B,eAAsB,SAAA,eAAtB,iBAAsB,SAAA,iBCCtB,iBAAyB,SAAA,iBAAzB,mBAAyB,SAAA,mBAAzB,mBAAyB,SAAA,mBAAzB,gBAAyB,SAAA,gBAAzB,iBAAyB,SAAA,yBAAA,SAAA,iBAK3B,WACE,SAAA,MACA,IAAA,EACA,MAAA,EACA,KAAA,EACA,QAAA,KAGF,cACE,SAAA,MACA,MAAA,EACA,OAAA,EACA,KAAA,EACA,QAAA,KAI4B,2DAD9B,YAEI,SAAA,eAAA,SAAA,OACA,IAAA,EACA,QAAA,MCzBJ,SCEE,SAAA,SACA,MAAA,IACA,OAAA,IACA,QAAA,EACA,OAAA,KACA,SAAA,OACA,KAAA,cACA,YAAA,OACA,OAAA,EAUA,0BAAA,yBAEE,SAAA,OACA,MAAA,KACA,OAAA,KACA,SAAA,QACA,KAAA,KACA,YAAA,OC7BJ,WAAa,WAAA,EAAA,QAAA,OAAA,2BACb,QAAU,WAAA,EAAA,MAAA,KAAA,0BACV,WAAa,WAAA,EAAA,KAAA,KAAA,2BACb,aAAe,WAAA,eCCX,MAAuB,MAAA,cAAvB,MAAuB,MAAA,cAAvB,MAAuB,MAAA,cAAvB,OAAuB,MAAA,eAAvB,QAAuB,MAAA,eAAvB,MAAuB,OAAA,cAAvB,MAAuB,OAAA,cAAvB,MAAuB,OAAA,cAAvB,OAAuB,OAAA,eAAvB,QAAuB,OAAA,eAI3B,QAAU,UAAA,eACV,QAAU,WAAA,eAIV,YAAc,UAAA,gBACd,YAAc,WAAA,gBAEd,QAAU,MAAA,gBACV,QAAU,OAAA,gBCfV,uBAEI,SAAA,SACA,IAAA,EACA,MAAA,EACA,OAAA,EACA,KAAA,EACA,QAAA,EAEA,eAAA,KACA,QAAA,GAEA,iBAAA,cCNI,KAAgC,OAAA,YAChC,MpEg6PR,MoE95PU,WAAA,YAEF,MpEi6PR,MoE/5PU,aAAA,YAEF,MpEk6PR,MoEh6PU,cAAA,YAEF,MpEm6PR,MoEj6PU,YAAA,YAfF,KAAgC,OAAA,iBAChC,MpEw7PR,MoEt7PU,WAAA,iBAEF,MpEy7PR,MoEv7PU,aAAA,iBAEF,MpE07PR,MoEx7PU,cAAA,iBAEF,MpE27PR,MoEz7PU,YAAA,iBAfF,KAAgC,OAAA,gBAChC,MpEg9PR,MoE98PU,WAAA,gBAEF,MpEi9PR,MoE/8PU,aAAA,gBAEF,MpEk9PR,MoEh9PU,cAAA,gBAEF,MpEm9PR,MoEj9PU,YAAA,gBAfF,KAAgC,OAAA,eAChC,MpEw+PR,MoEt+PU,WAAA,eAEF,MpEy+PR,MoEv+PU,aAAA,eAEF,MpE0+PR,MoEx+PU,cAAA,eAEF,MpE2+PR,MoEz+PU,YAAA,eAfF,KAAgC,OAAA,iBAChC,MpEggQR,MoE9/PU,WAAA,iBAEF,MpEigQR,MoE//PU,aAAA,iBAEF,MpEkgQR,MoEhgQU,cAAA,iBAEF,MpEmgQR,MoEjgQU,YAAA,iBAfF,KAAgC,OAAA,eAChC,MpEwhQR,MoEthQU,WAAA,eAEF,MpEyhQR,MoEvhQU,aAAA,eAEF,MpE0hQR,MoExhQU,cAAA,eAEF,MpE2hQR,MoEzhQU,YAAA,eAfF,KAAgC,QAAA,YAChC,MpEgjQR,MoE9iQU,YAAA,YAEF,MpEijQR,MoE/iQU,cAAA,YAEF,MpEkjQR,MoEhjQU,eAAA,YAEF,MpEmjQR,MoEjjQU,aAAA,YAfF,KAAgC,QAAA,iBAChC,MpEwkQR,MoEtkQU,YAAA,iBAEF,MpEykQR,MoEvkQU,cAAA,iBAEF,MpE0kQR,MoExkQU,eAAA,iBAEF,MpE2kQR,MoEzkQU,aAAA,iBAfF,KAAgC,QAAA,gBAChC,MpEgmQR,MoE9lQU,YAAA,gBAEF,MpEimQR,MoE/lQU,cAAA,gBAEF,MpEkmQR,MoEhmQU,eAAA,gBAEF,MpEmmQR,MoEjmQU,aAAA,gBAfF,KAAgC,QAAA,eAChC,MpEwnQR,MoEtnQU,YAAA,eAEF,MpEynQR,MoEvnQU,cAAA,eAEF,MpE0nQR,MoExnQU,eAAA,eAEF,MpE2nQR,MoEznQU,aAAA,eAfF,KAAgC,QAAA,iBAChC,MpEgpQR,MoE9oQU,YAAA,iBAEF,MpEipQR,MoE/oQU,cAAA,iBAEF,MpEkpQR,MoEhpQU,eAAA,iBAEF,MpEmpQR,MoEjpQU,aAAA,iBAfF,KAAgC,QAAA,eAChC,MpEwqQR,MoEtqQU,YAAA,eAEF,MpEyqQR,MoEvqQU,cAAA,eAEF,MpE0qQR,MoExqQU,eAAA,eAEF,MpE2qQR,MoEzqQU,aAAA,eAQF,MAAwB,OAAA,kBACxB,OpEyqQR,OoEvqQU,WAAA,kBAEF,OpE0qQR,OoExqQU,aAAA,kBAEF,OpE2qQR,OoEzqQU,cAAA,kBAEF,OpE4qQR,OoE1qQU,YAAA,kBAfF,MAAwB,OAAA,iBACxB,OpEisQR,OoE/rQU,WAAA,iBAEF,OpEksQR,OoEhsQU,aAAA,iBAEF,OpEmsQR,OoEjsQU,cAAA,iBAEF,OpEosQR,OoElsQU,YAAA,iBAfF,MAAwB,OAAA,gBACxB,OpEytQR,OoEvtQU,WAAA,gBAEF,OpE0tQR,OoExtQU,aAAA,gBAEF,OpE2tQR,OoEztQU,cAAA,gBAEF,OpE4tQR,OoE1tQU,YAAA,gBAfF,MAAwB,OAAA,kBACxB,OpEivQR,OoE/uQU,WAAA,kBAEF,OpEkvQR,OoEhvQU,aAAA,kBAEF,OpEmvQR,OoEjvQU,cAAA,kBAEF,OpEovQR,OoElvQU,YAAA,kBAfF,MAAwB,OAAA,gBACxB,OpEywQR,OoEvwQU,WAAA,gBAEF,OpE0wQR,OoExwQU,aAAA,gBAEF,OpE2wQR,OoEzwQU,cAAA,gBAEF,OpE4wQR,OoE1wQU,YAAA,gBAMN,QAAmB,OAAA,eACnB,SpE4wQJ,SoE1wQM,WAAA,eAEF,SpE6wQJ,SoE3wQM,aAAA,eAEF,SpE8wQJ,SoE5wQM,cAAA,eAEF,SpE+wQJ,SoE7wQM,YAAA,exDTF,yBwDlDI,QAAgC,OAAA,YAChC,SpEg1QN,SoE90QQ,WAAA,YAEF,SpEg1QN,SoE90QQ,aAAA,YAEF,SpEg1QN,SoE90QQ,cAAA,YAEF,SpEg1QN,SoE90QQ,YAAA,YAfF,QAAgC,OAAA,iBAChC,SpEm2QN,SoEj2QQ,WAAA,iBAEF,SpEm2QN,SoEj2QQ,aAAA,iBAEF,SpEm2QN,SoEj2QQ,cAAA,iBAEF,SpEm2QN,SoEj2QQ,YAAA,iBAfF,QAAgC,OAAA,gBAChC,SpEs3QN,SoEp3QQ,WAAA,gBAEF,SpEs3QN,SoEp3QQ,aAAA,gBAEF,SpEs3QN,SoEp3QQ,cAAA,gBAEF,SpEs3QN,SoEp3QQ,YAAA,gBAfF,QAAgC,OAAA,eAChC,SpEy4QN,SoEv4QQ,WAAA,eAEF,SpEy4QN,SoEv4QQ,aAAA,eAEF,SpEy4QN,SoEv4QQ,cAAA,eAEF,SpEy4QN,SoEv4QQ,YAAA,eAfF,QAAgC,OAAA,iBAChC,SpE45QN,SoE15QQ,WAAA,iBAEF,SpE45QN,SoE15QQ,aAAA,iBAEF,SpE45QN,SoE15QQ,cAAA,iBAEF,SpE45QN,SoE15QQ,YAAA,iBAfF,QAAgC,OAAA,eAChC,SpE+6QN,SoE76QQ,WAAA,eAEF,SpE+6QN,SoE76QQ,aAAA,eAEF,SpE+6QN,SoE76QQ,cAAA,eAEF,SpE+6QN,SoE76QQ,YAAA,eAfF,QAAgC,QAAA,YAChC,SpEk8QN,SoEh8QQ,YAAA,YAEF,SpEk8QN,SoEh8QQ,cAAA,YAEF,SpEk8QN,SoEh8QQ,eAAA,YAEF,SpEk8QN,SoEh8QQ,aAAA,YAfF,QAAgC,QAAA,iBAChC,SpEq9QN,SoEn9QQ,YAAA,iBAEF,SpEq9QN,SoEn9QQ,cAAA,iBAEF,SpEq9QN,SoEn9QQ,eAAA,iBAEF,SpEq9QN,SoEn9QQ,aAAA,iBAfF,QAAgC,QAAA,gBAChC,SpEw+QN,SoEt+QQ,YAAA,gBAEF,SpEw+QN,SoEt+QQ,cAAA,gBAEF,SpEw+QN,SoEt+QQ,eAAA,gBAEF,SpEw+QN,SoEt+QQ,aAAA,gBAfF,QAAgC,QAAA,eAChC,SpE2/QN,SoEz/QQ,YAAA,eAEF,SpE2/QN,SoEz/QQ,cAAA,eAEF,SpE2/QN,SoEz/QQ,eAAA,eAEF,SpE2/QN,SoEz/QQ,aAAA,eAfF,QAAgC,QAAA,iBAChC,SpE8gRN,SoE5gRQ,YAAA,iBAEF,SpE8gRN,SoE5gRQ,cAAA,iBAEF,SpE8gRN,SoE5gRQ,eAAA,iBAEF,SpE8gRN,SoE5gRQ,aAAA,iBAfF,QAAgC,QAAA,eAChC,SpEiiRN,SoE/hRQ,YAAA,eAEF,SpEiiRN,SoE/hRQ,cAAA,eAEF,SpEiiRN,SoE/hRQ,eAAA,eAEF,SpEiiRN,SoE/hRQ,aAAA,eAQF,SAAwB,OAAA,kBACxB,UpE6hRN,UoE3hRQ,WAAA,kBAEF,UpE6hRN,UoE3hRQ,aAAA,kBAEF,UpE6hRN,UoE3hRQ,cAAA,kBAEF,UpE6hRN,UoE3hRQ,YAAA,kBAfF,SAAwB,OAAA,iBACxB,UpEgjRN,UoE9iRQ,WAAA,iBAEF,UpEgjRN,UoE9iRQ,aAAA,iBAEF,UpEgjRN,UoE9iRQ,cAAA,iBAEF,UpEgjRN,UoE9iRQ,YAAA,iBAfF,SAAwB,OAAA,gBACxB,UpEmkRN,UoEjkRQ,WAAA,gBAEF,UpEmkRN,UoEjkRQ,aAAA,gBAEF,UpEmkRN,UoEjkRQ,cAAA,gBAEF,UpEmkRN,UoEjkRQ,YAAA,gBAfF,SAAwB,OAAA,kBACxB,UpEslRN,UoEplRQ,WAAA,kBAEF,UpEslRN,UoEplRQ,aAAA,kBAEF,UpEslRN,UoEplRQ,cAAA,kBAEF,UpEslRN,UoEplRQ,YAAA,kBAfF,SAAwB,OAAA,gBACxB,UpEymRN,UoEvmRQ,WAAA,gBAEF,UpEymRN,UoEvmRQ,aAAA,gBAEF,UpEymRN,UoEvmRQ,cAAA,gBAEF,UpEymRN,UoEvmRQ,YAAA,gBAMN,WAAmB,OAAA,eACnB,YpEumRF,YoErmRI,WAAA,eAEF,YpEumRF,YoErmRI,aAAA,eAEF,YpEumRF,YoErmRI,cAAA,eAEF,YpEumRF,YoErmRI,YAAA,gBxDTF,yBwDlDI,QAAgC,OAAA,YAChC,SpEyqRN,SoEvqRQ,WAAA,YAEF,SpEyqRN,SoEvqRQ,aAAA,YAEF,SpEyqRN,SoEvqRQ,cAAA,YAEF,SpEyqRN,SoEvqRQ,YAAA,YAfF,QAAgC,OAAA,iBAChC,SpE4rRN,SoE1rRQ,WAAA,iBAEF,SpE4rRN,SoE1rRQ,aAAA,iBAEF,SpE4rRN,SoE1rRQ,cAAA,iBAEF,SpE4rRN,SoE1rRQ,YAAA,iBAfF,QAAgC,OAAA,gBAChC,SpE+sRN,SoE7sRQ,WAAA,gBAEF,SpE+sRN,SoE7sRQ,aAAA,gBAEF,SpE+sRN,SoE7sRQ,cAAA,gBAEF,SpE+sRN,SoE7sRQ,YAAA,gBAfF,QAAgC,OAAA,eAChC,SpEkuRN,SoEhuRQ,WAAA,eAEF,SpEkuRN,SoEhuRQ,aAAA,eAEF,SpEkuRN,SoEhuRQ,cAAA,eAEF,SpEkuRN,SoEhuRQ,YAAA,eAfF,QAAgC,OAAA,iBAChC,SpEqvRN,SoEnvRQ,WAAA,iBAEF,SpEqvRN,SoEnvRQ,aAAA,iBAEF,SpEqvRN,SoEnvRQ,cAAA,iBAEF,SpEqvRN,SoEnvRQ,YAAA,iBAfF,QAAgC,OAAA,eAChC,SpEwwRN,SoEtwRQ,WAAA,eAEF,SpEwwRN,SoEtwRQ,aAAA,eAEF,SpEwwRN,SoEtwRQ,cAAA,eAEF,SpEwwRN,SoEtwRQ,YAAA,eAfF,QAAgC,QAAA,YAChC,SpE2xRN,SoEzxRQ,YAAA,YAEF,SpE2xRN,SoEzxRQ,cAAA,YAEF,SpE2xRN,SoEzxRQ,eAAA,YAEF,SpE2xRN,SoEzxRQ,aAAA,YAfF,QAAgC,QAAA,iBAChC,SpE8yRN,SoE5yRQ,YAAA,iBAEF,SpE8yRN,SoE5yRQ,cAAA,iBAEF,SpE8yRN,SoE5yRQ,eAAA,iBAEF,SpE8yRN,SoE5yRQ,aAAA,iBAfF,QAAgC,QAAA,gBAChC,SpEi0RN,SoE/zRQ,YAAA,gBAEF,SpEi0RN,SoE/zRQ,cAAA,gBAEF,SpEi0RN,SoE/zRQ,eAAA,gBAEF,SpEi0RN,SoE/zRQ,aAAA,gBAfF,QAAgC,QAAA,eAChC,SpEo1RN,SoEl1RQ,YAAA,eAEF,SpEo1RN,SoEl1RQ,cAAA,eAEF,SpEo1RN,SoEl1RQ,eAAA,eAEF,SpEo1RN,SoEl1RQ,aAAA,eAfF,QAAgC,QAAA,iBAChC,SpEu2RN,SoEr2RQ,YAAA,iBAEF,SpEu2RN,SoEr2RQ,cAAA,iBAEF,SpEu2RN,SoEr2RQ,eAAA,iBAEF,SpEu2RN,SoEr2RQ,aAAA,iBAfF,QAAgC,QAAA,eAChC,SpE03RN,SoEx3RQ,YAAA,eAEF,SpE03RN,SoEx3RQ,cAAA,eAEF,SpE03RN,SoEx3RQ,eAAA,eAEF,SpE03RN,SoEx3RQ,aAAA,eAQF,SAAwB,OAAA,kBACxB,UpEs3RN,UoEp3RQ,WAAA,kBAEF,UpEs3RN,UoEp3RQ,aAAA,kBAEF,UpEs3RN,UoEp3RQ,cAAA,kBAEF,UpEs3RN,UoEp3RQ,YAAA,kBAfF,SAAwB,OAAA,iBACxB,UpEy4RN,UoEv4RQ,WAAA,iBAEF,UpEy4RN,UoEv4RQ,aAAA,iBAEF,UpEy4RN,UoEv4RQ,cAAA,iBAEF,UpEy4RN,UoEv4RQ,YAAA,iBAfF,SAAwB,OAAA,gBACxB,UpE45RN,UoE15RQ,WAAA,gBAEF,UpE45RN,UoE15RQ,aAAA,gBAEF,UpE45RN,UoE15RQ,cAAA,gBAEF,UpE45RN,UoE15RQ,YAAA,gBAfF,SAAwB,OAAA,kBACxB,UpE+6RN,UoE76RQ,WAAA,kBAEF,UpE+6RN,UoE76RQ,aAAA,kBAEF,UpE+6RN,UoE76RQ,cAAA,kBAEF,UpE+6RN,UoE76RQ,YAAA,kBAfF,SAAwB,OAAA,gBACxB,UpEk8RN,UoEh8RQ,WAAA,gBAEF,UpEk8RN,UoEh8RQ,aAAA,gBAEF,UpEk8RN,UoEh8RQ,cAAA,gBAEF,UpEk8RN,UoEh8RQ,YAAA,gBAMN,WAAmB,OAAA,eACnB,YpEg8RF,YoE97RI,WAAA,eAEF,YpEg8RF,YoE97RI,aAAA,eAEF,YpEg8RF,YoE97RI,cAAA,eAEF,YpEg8RF,YoE97RI,YAAA,gBxDTF,yBwDlDI,QAAgC,OAAA,YAChC,SpEkgSN,SoEhgSQ,WAAA,YAEF,SpEkgSN,SoEhgSQ,aAAA,YAEF,SpEkgSN,SoEhgSQ,cAAA,YAEF,SpEkgSN,SoEhgSQ,YAAA,YAfF,QAAgC,OAAA,iBAChC,SpEqhSN,SoEnhSQ,WAAA,iBAEF,SpEqhSN,SoEnhSQ,aAAA,iBAEF,SpEqhSN,SoEnhSQ,cAAA,iBAEF,SpEqhSN,SoEnhSQ,YAAA,iBAfF,QAAgC,OAAA,gBAChC,SpEwiSN,SoEtiSQ,WAAA,gBAEF,SpEwiSN,SoEtiSQ,aAAA,gBAEF,SpEwiSN,SoEtiSQ,cAAA,gBAEF,SpEwiSN,SoEtiSQ,YAAA,gBAfF,QAAgC,OAAA,eAChC,SpE2jSN,SoEzjSQ,WAAA,eAEF,SpE2jSN,SoEzjSQ,aAAA,eAEF,SpE2jSN,SoEzjSQ,cAAA,eAEF,SpE2jSN,SoEzjSQ,YAAA,eAfF,QAAgC,OAAA,iBAChC,SpE8kSN,SoE5kSQ,WAAA,iBAEF,SpE8kSN,SoE5kSQ,aAAA,iBAEF,SpE8kSN,SoE5kSQ,cAAA,iBAEF,SpE8kSN,SoE5kSQ,YAAA,iBAfF,QAAgC,OAAA,eAChC,SpEimSN,SoE/lSQ,WAAA,eAEF,SpEimSN,SoE/lSQ,aAAA,eAEF,SpEimSN,SoE/lSQ,cAAA,eAEF,SpEimSN,SoE/lSQ,YAAA,eAfF,QAAgC,QAAA,YAChC,SpEonSN,SoElnSQ,YAAA,YAEF,SpEonSN,SoElnSQ,cAAA,YAEF,SpEonSN,SoElnSQ,eAAA,YAEF,SpEonSN,SoElnSQ,aAAA,YAfF,QAAgC,QAAA,iBAChC,SpEuoSN,SoEroSQ,YAAA,iBAEF,SpEuoSN,SoEroSQ,cAAA,iBAEF,SpEuoSN,SoEroSQ,eAAA,iBAEF,SpEuoSN,SoEroSQ,aAAA,iBAfF,QAAgC,QAAA,gBAChC,SpE0pSN,SoExpSQ,YAAA,gBAEF,SpE0pSN,SoExpSQ,cAAA,gBAEF,SpE0pSN,SoExpSQ,eAAA,gBAEF,SpE0pSN,SoExpSQ,aAAA,gBAfF,QAAgC,QAAA,eAChC,SpE6qSN,SoE3qSQ,YAAA,eAEF,SpE6qSN,SoE3qSQ,cAAA,eAEF,SpE6qSN,SoE3qSQ,eAAA,eAEF,SpE6qSN,SoE3qSQ,aAAA,eAfF,QAAgC,QAAA,iBAChC,SpEgsSN,SoE9rSQ,YAAA,iBAEF,SpEgsSN,SoE9rSQ,cAAA,iBAEF,SpEgsSN,SoE9rSQ,eAAA,iBAEF,SpEgsSN,SoE9rSQ,aAAA,iBAfF,QAAgC,QAAA,eAChC,SpEmtSN,SoEjtSQ,YAAA,eAEF,SpEmtSN,SoEjtSQ,cAAA,eAEF,SpEmtSN,SoEjtSQ,eAAA,eAEF,SpEmtSN,SoEjtSQ,aAAA,eAQF,SAAwB,OAAA,kBACxB,UpE+sSN,UoE7sSQ,WAAA,kBAEF,UpE+sSN,UoE7sSQ,aAAA,kBAEF,UpE+sSN,UoE7sSQ,cAAA,kBAEF,UpE+sSN,UoE7sSQ,YAAA,kBAfF,SAAwB,OAAA,iBACxB,UpEkuSN,UoEhuSQ,WAAA,iBAEF,UpEkuSN,UoEhuSQ,aAAA,iBAEF,UpEkuSN,UoEhuSQ,cAAA,iBAEF,UpEkuSN,UoEhuSQ,YAAA,iBAfF,SAAwB,OAAA,gBACxB,UpEqvSN,UoEnvSQ,WAAA,gBAEF,UpEqvSN,UoEnvSQ,aAAA,gBAEF,UpEqvSN,UoEnvSQ,cAAA,gBAEF,UpEqvSN,UoEnvSQ,YAAA,gBAfF,SAAwB,OAAA,kBACxB,UpEwwSN,UoEtwSQ,WAAA,kBAEF,UpEwwSN,UoEtwSQ,aAAA,kBAEF,UpEwwSN,UoEtwSQ,cAAA,kBAEF,UpEwwSN,UoEtwSQ,YAAA,kBAfF,SAAwB,OAAA,gBACxB,UpE2xSN,UoEzxSQ,WAAA,gBAEF,UpE2xSN,UoEzxSQ,aAAA,gBAEF,UpE2xSN,UoEzxSQ,cAAA,gBAEF,UpE2xSN,UoEzxSQ,YAAA,gBAMN,WAAmB,OAAA,eACnB,YpEyxSF,YoEvxSI,WAAA,eAEF,YpEyxSF,YoEvxSI,aAAA,eAEF,YpEyxSF,YoEvxSI,cAAA,eAEF,YpEyxSF,YoEvxSI,YAAA,gBxDTF,0BwDlDI,QAAgC,OAAA,YAChC,SpE21SN,SoEz1SQ,WAAA,YAEF,SpE21SN,SoEz1SQ,aAAA,YAEF,SpE21SN,SoEz1SQ,cAAA,YAEF,SpE21SN,SoEz1SQ,YAAA,YAfF,QAAgC,OAAA,iBAChC,SpE82SN,SoE52SQ,WAAA,iBAEF,SpE82SN,SoE52SQ,aAAA,iBAEF,SpE82SN,SoE52SQ,cAAA,iBAEF,SpE82SN,SoE52SQ,YAAA,iBAfF,QAAgC,OAAA,gBAChC,SpEi4SN,SoE/3SQ,WAAA,gBAEF,SpEi4SN,SoE/3SQ,aAAA,gBAEF,SpEi4SN,SoE/3SQ,cAAA,gBAEF,SpEi4SN,SoE/3SQ,YAAA,gBAfF,QAAgC,OAAA,eAChC,SpEo5SN,SoEl5SQ,WAAA,eAEF,SpEo5SN,SoEl5SQ,aAAA,eAEF,SpEo5SN,SoEl5SQ,cAAA,eAEF,SpEo5SN,SoEl5SQ,YAAA,eAfF,QAAgC,OAAA,iBAChC,SpEu6SN,SoEr6SQ,WAAA,iBAEF,SpEu6SN,SoEr6SQ,aAAA,iBAEF,SpEu6SN,SoEr6SQ,cAAA,iBAEF,SpEu6SN,SoEr6SQ,YAAA,iBAfF,QAAgC,OAAA,eAChC,SpE07SN,SoEx7SQ,WAAA,eAEF,SpE07SN,SoEx7SQ,aAAA,eAEF,SpE07SN,SoEx7SQ,cAAA,eAEF,SpE07SN,SoEx7SQ,YAAA,eAfF,QAAgC,QAAA,YAChC,SpE68SN,SoE38SQ,YAAA,YAEF,SpE68SN,SoE38SQ,cAAA,YAEF,SpE68SN,SoE38SQ,eAAA,YAEF,SpE68SN,SoE38SQ,aAAA,YAfF,QAAgC,QAAA,iBAChC,SpEg+SN,SoE99SQ,YAAA,iBAEF,SpEg+SN,SoE99SQ,cAAA,iBAEF,SpEg+SN,SoE99SQ,eAAA,iBAEF,SpEg+SN,SoE99SQ,aAAA,iBAfF,QAAgC,QAAA,gBAChC,SpEm/SN,SoEj/SQ,YAAA,gBAEF,SpEm/SN,SoEj/SQ,cAAA,gBAEF,SpEm/SN,SoEj/SQ,eAAA,gBAEF,SpEm/SN,SoEj/SQ,aAAA,gBAfF,QAAgC,QAAA,eAChC,SpEsgTN,SoEpgTQ,YAAA,eAEF,SpEsgTN,SoEpgTQ,cAAA,eAEF,SpEsgTN,SoEpgTQ,eAAA,eAEF,SpEsgTN,SoEpgTQ,aAAA,eAfF,QAAgC,QAAA,iBAChC,SpEyhTN,SoEvhTQ,YAAA,iBAEF,SpEyhTN,SoEvhTQ,cAAA,iBAEF,SpEyhTN,SoEvhTQ,eAAA,iBAEF,SpEyhTN,SoEvhTQ,aAAA,iBAfF,QAAgC,QAAA,eAChC,SpE4iTN,SoE1iTQ,YAAA,eAEF,SpE4iTN,SoE1iTQ,cAAA,eAEF,SpE4iTN,SoE1iTQ,eAAA,eAEF,SpE4iTN,SoE1iTQ,aAAA,eAQF,SAAwB,OAAA,kBACxB,UpEwiTN,UoEtiTQ,WAAA,kBAEF,UpEwiTN,UoEtiTQ,aAAA,kBAEF,UpEwiTN,UoEtiTQ,cAAA,kBAEF,UpEwiTN,UoEtiTQ,YAAA,kBAfF,SAAwB,OAAA,iBACxB,UpE2jTN,UoEzjTQ,WAAA,iBAEF,UpE2jTN,UoEzjTQ,aAAA,iBAEF,UpE2jTN,UoEzjTQ,cAAA,iBAEF,UpE2jTN,UoEzjTQ,YAAA,iBAfF,SAAwB,OAAA,gBACxB,UpE8kTN,UoE5kTQ,WAAA,gBAEF,UpE8kTN,UoE5kTQ,aAAA,gBAEF,UpE8kTN,UoE5kTQ,cAAA,gBAEF,UpE8kTN,UoE5kTQ,YAAA,gBAfF,SAAwB,OAAA,kBACxB,UpEimTN,UoE/lTQ,WAAA,kBAEF,UpEimTN,UoE/lTQ,aAAA,kBAEF,UpEimTN,UoE/lTQ,cAAA,kBAEF,UpEimTN,UoE/lTQ,YAAA,kBAfF,SAAwB,OAAA,gBACxB,UpEonTN,UoElnTQ,WAAA,gBAEF,UpEonTN,UoElnTQ,aAAA,gBAEF,UpEonTN,UoElnTQ,cAAA,gBAEF,UpEonTN,UoElnTQ,YAAA,gBAMN,WAAmB,OAAA,eACnB,YpEknTF,YoEhnTI,WAAA,eAEF,YpEknTF,YoEhnTI,aAAA,eAEF,YpEknTF,YoEhnTI,cAAA,eAEF,YpEknTF,YoEhnTI,YAAA,gBC/DN,gBAAkB,YAAA,cAAA,CAAA,KAAA,CAAA,MAAA,CAAA,QAAA,CAAA,iBAAA,CAAA,aAAA,CAAA,oBAIlB,cAAiB,WAAA,kBACjB,WAAiB,YAAA,iBACjB,aAAiB,YAAA,iBACjB,eCTE,SAAA,OACA,cAAA,SACA,YAAA,ODeE,WAAwB,WAAA,eACxB,YAAwB,WAAA,gBACxB,aAAwB,WAAA,iBzDqCxB,yByDvCA,cAAwB,WAAA,eACxB,eAAwB,WAAA,gBACxB,gBAAwB,WAAA,kBzDqCxB,yByDvCA,cAAwB,WAAA,eACxB,eAAwB,WAAA,gBACxB,gBAAwB,WAAA,kBzDqCxB,yByDvCA,cAAwB,WAAA,eACxB,eAAwB,WAAA,gBACxB,gBAAwB,WAAA,kBzDqCxB,0ByDvCA,cAAwB,WAAA,eACxB,eAAwB,WAAA,gBACxB,gBAAwB,WAAA,kBAM5B,gBAAmB,eAAA,oBACnB,gBAAmB,eAAA,oBACnB,iBAAmB,eAAA,qBAInB,mBAAuB,YAAA,cACvB,qBAAuB,YAAA,kBACvB,oBAAuB,YAAA,cACvB,kBAAuB,YAAA,cACvB,oBAAuB,YAAA,iBACvB,aAAuB,WAAA,iBAIvB,YAAc,MAAA,eEvCZ,cACE,MAAA,kBpEUF,qBAAA,qBoELM,MAAA,kBANN,gBACE,MAAA,kBpEUF,uBAAA,uBoELM,MAAA,kBANN,cACE,MAAA,kBpEUF,qBAAA,qBoELM,MAAA,kBANN,WACE,MAAA,kBpEUF,kBAAA,kBoELM,MAAA,kBANN,cACE,MAAA,kBpEUF,qBAAA,qBoELM,MAAA,kBANN,aACE,MAAA,kBpEUF,oBAAA,oBoELM,MAAA,kBANN,YACE,MAAA,kBpEUF,mBAAA,mBoELM,MAAA,kBANN,WACE,MAAA,kBpEUF,kBAAA,kBoELM,MAAA,kBFuCR,WAAa,MAAA,kBACb,YAAc,MAAA,kBAEd,eAAiB,MAAA,yBACjB,eAAiB,MAAA,+BAIjB,WGvDE,KAAA,CAAA,CAAA,EAAA,EACA,MAAA,YACA,YAAA,KACA,iBAAA,YACA,OAAA,EHuDF,sBAAwB,gBAAA,eAExB,YACE,WAAA,qBACA,cAAA,qBAKF,YAAc,MAAA,kBIjEd,SACE,WAAA,kBAGF,WACE,WAAA,iBCAA,a3EOF,ECk5TE,QADA,S0El5TI,YAAA,eAEA,WAAA,eAGF,YAEI,gBAAA,UASJ,mBACE,QAAA,KAAA,YAAA,I3E8LN,I2E/KM,YAAA,mB1Ei4TJ,W0E/3TE,IAEE,OAAA,IAAA,MAAA,QACA,kBAAA,MAQF,MACE,QAAA,mB1E23TJ,I0Ex3TE,GAEE,kBAAA,M1E03TJ,GACA,G0Ex3TE,EAGE,QAAA,EACA,OAAA,EAGF,G1Es3TF,G0Ep3TI,iBAAA,MAQF,MACE,KAAA,G3E5CN,K2E+CM,UAAA,gBhEtFJ,WgEyFI,UAAA,gB5C9EN,Q4CmFM,QAAA,KvC/FN,OuCkGM,OAAA,IAAA,MAAA,K5DnGN,O4DuGM,gBAAA,mBADF,U1Eg3TF,U0E32TM,iBAAA,e1E+2TN,mBcl7TF,mB4D0EQ,OAAA,IAAA,MAAA,kB5DWR,Y4DNM,MAAA,Q1E42TJ,wBAFA,eeh+TA,efi+TA,qB0Er2TM,aAAA,Q5DlBR,sB4DuBM,MAAA,QACA,aAAA","sourcesContent":["/*!\n * Bootstrap v4.4.1 (https://getbootstrap.com/)\n * Copyright 2011-2019 The Bootstrap Authors\n * Copyright 2011-2019 Twitter, Inc.\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)\n */\n\n@import \"functions\";\n@import \"variables\";\n@import \"mixins\";\n@import \"root\";\n@import \"reboot\";\n@import \"type\";\n@import \"images\";\n@import \"code\";\n@import \"grid\";\n@import \"tables\";\n@import \"forms\";\n@import \"buttons\";\n@import \"transitions\";\n@import \"dropdown\";\n@import \"button-group\";\n@import \"input-group\";\n@import \"custom-forms\";\n@import \"nav\";\n@import \"navbar\";\n@import \"card\";\n@import \"breadcrumb\";\n@import \"pagination\";\n@import \"badge\";\n@import \"jumbotron\";\n@import \"alert\";\n@import \"progress\";\n@import \"media\";\n@import \"list-group\";\n@import \"close\";\n@import \"toasts\";\n@import \"modal\";\n@import \"tooltip\";\n@import \"popover\";\n@import \"carousel\";\n@import \"spinners\";\n@import \"utilities\";\n@import \"print\";\n","// Do not forget to update getting-started/theming.md!\n:root {\n  // Custom variable values only support SassScript inside `#{}`.\n  @each $color, $value in $colors {\n    --#{$color}: #{$value};\n  }\n\n  @each $color, $value in $theme-colors {\n    --#{$color}: #{$value};\n  }\n\n  @each $bp, $value in $grid-breakpoints {\n    --breakpoint-#{$bp}: #{$value};\n  }\n\n  // Use `inspect` for lists so that quoted items keep the quotes.\n  // See https://github.com/sass/sass/issues/2383#issuecomment-336349172\n  --font-family-sans-serif: #{inspect($font-family-sans-serif)};\n  --font-family-monospace: #{inspect($font-family-monospace)};\n}\n","// stylelint-disable at-rule-no-vendor-prefix, declaration-no-important, selector-no-qualifying-type, property-no-vendor-prefix\n\n// Reboot\n//\n// Normalization of HTML elements, manually forked from Normalize.css to remove\n// styles targeting irrelevant browsers while applying new styles.\n//\n// Normalize is licensed MIT. https://github.com/necolas/normalize.css\n\n\n// Document\n//\n// 1. Change from `box-sizing: content-box` so that `width` is not affected by `padding` or `border`.\n// 2. Change the default font family in all browsers.\n// 3. Correct the line height in all browsers.\n// 4. Prevent adjustments of font size after orientation changes in IE on Windows Phone and in iOS.\n// 5. Change the default tap highlight to be completely transparent in iOS.\n\n*,\n*::before,\n*::after {\n  box-sizing: border-box; // 1\n}\n\nhtml {\n  font-family: sans-serif; // 2\n  line-height: 1.15; // 3\n  -webkit-text-size-adjust: 100%; // 4\n  -webkit-tap-highlight-color: rgba($black, 0); // 5\n}\n\n// Shim for \"new\" HTML5 structural elements to display correctly (IE10, older browsers)\n// TODO: remove in v5\n// stylelint-disable-next-line selector-list-comma-newline-after\narticle, aside, figcaption, figure, footer, header, hgroup, main, nav, section {\n  display: block;\n}\n\n// Body\n//\n// 1. Remove the margin in all browsers.\n// 2. As a best practice, apply a default `background-color`.\n// 3. Set an explicit initial text-align value so that we can later use\n//    the `inherit` value on things like `<th>` elements.\n\nbody {\n  margin: 0; // 1\n  font-family: $font-family-base;\n  @include font-size($font-size-base);\n  font-weight: $font-weight-base;\n  line-height: $line-height-base;\n  color: $body-color;\n  text-align: left; // 3\n  background-color: $body-bg; // 2\n}\n\n// Future-proof rule: in browsers that support :focus-visible, suppress the focus outline\n// on elements that programmatically receive focus but wouldn't normally show a visible\n// focus outline. In general, this would mean that the outline is only applied if the\n// interaction that led to the element receiving programmatic focus was a keyboard interaction,\n// or the browser has somehow determined that the user is primarily a keyboard user and/or\n// wants focus outlines to always be presented.\n//\n// See https://developer.mozilla.org/en-US/docs/Web/CSS/:focus-visible\n// and https://developer.paciellogroup.com/blog/2018/03/focus-visible-and-backwards-compatibility/\n[tabindex=\"-1\"]:focus:not(:focus-visible) {\n  outline: 0 !important;\n}\n\n\n// Content grouping\n//\n// 1. Add the correct box sizing in Firefox.\n// 2. Show the overflow in Edge and IE.\n\nhr {\n  box-sizing: content-box; // 1\n  height: 0; // 1\n  overflow: visible; // 2\n}\n\n\n//\n// Typography\n//\n\n// Remove top margins from headings\n//\n// By default, `<h1>`-`<h6>` all receive top and bottom margins. We nuke the top\n// margin for easier control within type scales as it avoids margin collapsing.\n// stylelint-disable-next-line selector-list-comma-newline-after\nh1, h2, h3, h4, h5, h6 {\n  margin-top: 0;\n  margin-bottom: $headings-margin-bottom;\n}\n\n// Reset margins on paragraphs\n//\n// Similarly, the top margin on `<p>`s get reset. However, we also reset the\n// bottom margin to use `rem` units instead of `em`.\np {\n  margin-top: 0;\n  margin-bottom: $paragraph-margin-bottom;\n}\n\n// Abbreviations\n//\n// 1. Duplicate behavior to the data-* attribute for our tooltip plugin\n// 2. Add the correct text decoration in Chrome, Edge, IE, Opera, and Safari.\n// 3. Add explicit cursor to indicate changed behavior.\n// 4. Remove the bottom border in Firefox 39-.\n// 5. Prevent the text-decoration to be skipped.\n\nabbr[title],\nabbr[data-original-title] { // 1\n  text-decoration: underline; // 2\n  text-decoration: underline dotted; // 2\n  cursor: help; // 3\n  border-bottom: 0; // 4\n  text-decoration-skip-ink: none; // 5\n}\n\naddress {\n  margin-bottom: 1rem;\n  font-style: normal;\n  line-height: inherit;\n}\n\nol,\nul,\ndl {\n  margin-top: 0;\n  margin-bottom: 1rem;\n}\n\nol ol,\nul ul,\nol ul,\nul ol {\n  margin-bottom: 0;\n}\n\ndt {\n  font-weight: $dt-font-weight;\n}\n\ndd {\n  margin-bottom: .5rem;\n  margin-left: 0; // Undo browser default\n}\n\nblockquote {\n  margin: 0 0 1rem;\n}\n\nb,\nstrong {\n  font-weight: $font-weight-bolder; // Add the correct font weight in Chrome, Edge, and Safari\n}\n\nsmall {\n  @include font-size(80%); // Add the correct font size in all browsers\n}\n\n//\n// Prevent `sub` and `sup` elements from affecting the line height in\n// all browsers.\n//\n\nsub,\nsup {\n  position: relative;\n  @include font-size(75%);\n  line-height: 0;\n  vertical-align: baseline;\n}\n\nsub { bottom: -.25em; }\nsup { top: -.5em; }\n\n\n//\n// Links\n//\n\na {\n  color: $link-color;\n  text-decoration: $link-decoration;\n  background-color: transparent; // Remove the gray background on active links in IE 10.\n\n  @include hover() {\n    color: $link-hover-color;\n    text-decoration: $link-hover-decoration;\n  }\n}\n\n// And undo these styles for placeholder links/named anchors (without href).\n// It would be more straightforward to just use a[href] in previous block, but that\n// causes specificity issues in many other styles that are too complex to fix.\n// See https://github.com/twbs/bootstrap/issues/19402\n\na:not([href]) {\n  color: inherit;\n  text-decoration: none;\n\n  @include hover() {\n    color: inherit;\n    text-decoration: none;\n  }\n}\n\n\n//\n// Code\n//\n\npre,\ncode,\nkbd,\nsamp {\n  font-family: $font-family-monospace;\n  @include font-size(1em); // Correct the odd `em` font sizing in all browsers.\n}\n\npre {\n  // Remove browser default top margin\n  margin-top: 0;\n  // Reset browser default of `1em` to use `rem`s\n  margin-bottom: 1rem;\n  // Don't allow content to break outside\n  overflow: auto;\n}\n\n\n//\n// Figures\n//\n\nfigure {\n  // Apply a consistent margin strategy (matches our type styles).\n  margin: 0 0 1rem;\n}\n\n\n//\n// Images and content\n//\n\nimg {\n  vertical-align: middle;\n  border-style: none; // Remove the border on images inside links in IE 10-.\n}\n\nsvg {\n  // Workaround for the SVG overflow bug in IE10/11 is still required.\n  // See https://github.com/twbs/bootstrap/issues/26878\n  overflow: hidden;\n  vertical-align: middle;\n}\n\n\n//\n// Tables\n//\n\ntable {\n  border-collapse: collapse; // Prevent double borders\n}\n\ncaption {\n  padding-top: $table-cell-padding;\n  padding-bottom: $table-cell-padding;\n  color: $table-caption-color;\n  text-align: left;\n  caption-side: bottom;\n}\n\nth {\n  // Matches default `<td>` alignment by inheriting from the `<body>`, or the\n  // closest parent with a set `text-align`.\n  text-align: inherit;\n}\n\n\n//\n// Forms\n//\n\nlabel {\n  // Allow labels to use `margin` for spacing.\n  display: inline-block;\n  margin-bottom: $label-margin-bottom;\n}\n\n// Remove the default `border-radius` that macOS Chrome adds.\n//\n// Details at https://github.com/twbs/bootstrap/issues/24093\nbutton {\n  // stylelint-disable-next-line property-blacklist\n  border-radius: 0;\n}\n\n// Work around a Firefox/IE bug where the transparent `button` background\n// results in a loss of the default `button` focus styles.\n//\n// Credit: https://github.com/suitcss/base/\nbutton:focus {\n  outline: 1px dotted;\n  outline: 5px auto -webkit-focus-ring-color;\n}\n\ninput,\nbutton,\nselect,\noptgroup,\ntextarea {\n  margin: 0; // Remove the margin in Firefox and Safari\n  font-family: inherit;\n  @include font-size(inherit);\n  line-height: inherit;\n}\n\nbutton,\ninput {\n  overflow: visible; // Show the overflow in Edge\n}\n\nbutton,\nselect {\n  text-transform: none; // Remove the inheritance of text transform in Firefox\n}\n\n// Remove the inheritance of word-wrap in Safari.\n//\n// Details at https://github.com/twbs/bootstrap/issues/24990\nselect {\n  word-wrap: normal;\n}\n\n\n// 1. Prevent a WebKit bug where (2) destroys native `audio` and `video`\n//    controls in Android 4.\n// 2. Correct the inability to style clickable types in iOS and Safari.\nbutton,\n[type=\"button\"], // 1\n[type=\"reset\"],\n[type=\"submit\"] {\n  -webkit-appearance: button; // 2\n}\n\n// Opinionated: add \"hand\" cursor to non-disabled button elements.\n@if $enable-pointer-cursor-for-buttons {\n  button,\n  [type=\"button\"],\n  [type=\"reset\"],\n  [type=\"submit\"] {\n    &:not(:disabled) {\n      cursor: pointer;\n    }\n  }\n}\n\n// Remove inner border and padding from Firefox, but don't restore the outline like Normalize.\nbutton::-moz-focus-inner,\n[type=\"button\"]::-moz-focus-inner,\n[type=\"reset\"]::-moz-focus-inner,\n[type=\"submit\"]::-moz-focus-inner {\n  padding: 0;\n  border-style: none;\n}\n\ninput[type=\"radio\"],\ninput[type=\"checkbox\"] {\n  box-sizing: border-box; // 1. Add the correct box sizing in IE 10-\n  padding: 0; // 2. Remove the padding in IE 10-\n}\n\n\ninput[type=\"date\"],\ninput[type=\"time\"],\ninput[type=\"datetime-local\"],\ninput[type=\"month\"] {\n  // Remove the default appearance of temporal inputs to avoid a Mobile Safari\n  // bug where setting a custom line-height prevents text from being vertically\n  // centered within the input.\n  // See https://bugs.webkit.org/show_bug.cgi?id=139848\n  // and https://github.com/twbs/bootstrap/issues/11266\n  -webkit-appearance: listbox;\n}\n\ntextarea {\n  overflow: auto; // Remove the default vertical scrollbar in IE.\n  // Textareas should really only resize vertically so they don't break their (horizontal) containers.\n  resize: vertical;\n}\n\nfieldset {\n  // Browsers set a default `min-width: min-content;` on fieldsets,\n  // unlike e.g. `<div>`s, which have `min-width: 0;` by default.\n  // So we reset that to ensure fieldsets behave more like a standard block element.\n  // See https://github.com/twbs/bootstrap/issues/12359\n  // and https://html.spec.whatwg.org/multipage/#the-fieldset-and-legend-elements\n  min-width: 0;\n  // Reset the default outline behavior of fieldsets so they don't affect page layout.\n  padding: 0;\n  margin: 0;\n  border: 0;\n}\n\n// 1. Correct the text wrapping in Edge and IE.\n// 2. Correct the color inheritance from `fieldset` elements in IE.\nlegend {\n  display: block;\n  width: 100%;\n  max-width: 100%; // 1\n  padding: 0;\n  margin-bottom: .5rem;\n  @include font-size(1.5rem);\n  line-height: inherit;\n  color: inherit; // 2\n  white-space: normal; // 1\n}\n\nprogress {\n  vertical-align: baseline; // Add the correct vertical alignment in Chrome, Firefox, and Opera.\n}\n\n// Correct the cursor style of increment and decrement buttons in Chrome.\n[type=\"number\"]::-webkit-inner-spin-button,\n[type=\"number\"]::-webkit-outer-spin-button {\n  height: auto;\n}\n\n[type=\"search\"] {\n  // This overrides the extra rounded corners on search inputs in iOS so that our\n  // `.form-control` class can properly style them. Note that this cannot simply\n  // be added to `.form-control` as it's not specific enough. For details, see\n  // https://github.com/twbs/bootstrap/issues/11586.\n  outline-offset: -2px; // 2. Correct the outline style in Safari.\n  -webkit-appearance: none;\n}\n\n//\n// Remove the inner padding in Chrome and Safari on macOS.\n//\n\n[type=\"search\"]::-webkit-search-decoration {\n  -webkit-appearance: none;\n}\n\n//\n// 1. Correct the inability to style clickable types in iOS and Safari.\n// 2. Change font properties to `inherit` in Safari.\n//\n\n::-webkit-file-upload-button {\n  font: inherit; // 2\n  -webkit-appearance: button; // 1\n}\n\n//\n// Correct element displays\n//\n\noutput {\n  display: inline-block;\n}\n\nsummary {\n  display: list-item; // Add the correct display in all browsers\n  cursor: pointer;\n}\n\ntemplate {\n  display: none; // Add the correct display in IE\n}\n\n// Always hide an element with the `hidden` HTML attribute (from PureCSS).\n// Needed for proper display in IE 10-.\n[hidden] {\n  display: none !important;\n}\n","/*!\n * Bootstrap v4.4.1 (https://getbootstrap.com/)\n * Copyright 2011-2019 The Bootstrap Authors\n * Copyright 2011-2019 Twitter, Inc.\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)\n */\n:root {\n  --blue: #007bff;\n  --indigo: #6610f2;\n  --purple: #6f42c1;\n  --pink: #e83e8c;\n  --red: #dc3545;\n  --orange: #fd7e14;\n  --yellow: #ffc107;\n  --green: #28a745;\n  --teal: #20c997;\n  --cyan: #17a2b8;\n  --white: #fff;\n  --gray: #6c757d;\n  --gray-dark: #343a40;\n  --primary: #007bff;\n  --secondary: #6c757d;\n  --success: #28a745;\n  --info: #17a2b8;\n  --warning: #ffc107;\n  --danger: #dc3545;\n  --light: #f8f9fa;\n  --dark: #343a40;\n  --breakpoint-xs: 0;\n  --breakpoint-sm: 576px;\n  --breakpoint-md: 768px;\n  --breakpoint-lg: 992px;\n  --breakpoint-xl: 1200px;\n  --font-family-sans-serif: -apple-system, BlinkMacSystemFont, \"Segoe UI\", Roboto, \"Helvetica Neue\", Arial, \"Noto Sans\", sans-serif, \"Apple Color Emoji\", \"Segoe UI Emoji\", \"Segoe UI Symbol\", \"Noto Color Emoji\";\n  --font-family-monospace: SFMono-Regular, Menlo, Monaco, Consolas, \"Liberation Mono\", \"Courier New\", monospace;\n}\n\n*,\n*::before,\n*::after {\n  box-sizing: border-box;\n}\n\nhtml {\n  font-family: sans-serif;\n  line-height: 1.15;\n  -webkit-text-size-adjust: 100%;\n  -webkit-tap-highlight-color: rgba(0, 0, 0, 0);\n}\n\narticle, aside, figcaption, figure, footer, header, hgroup, main, nav, section {\n  display: block;\n}\n\nbody {\n  margin: 0;\n  font-family: -apple-system, BlinkMacSystemFont, \"Segoe UI\", Roboto, \"Helvetica Neue\", Arial, \"Noto Sans\", sans-serif, \"Apple Color Emoji\", \"Segoe UI Emoji\", \"Segoe UI Symbol\", \"Noto Color Emoji\";\n  font-size: 1rem;\n  font-weight: 400;\n  line-height: 1.5;\n  color: #212529;\n  text-align: left;\n  background-color: #fff;\n}\n\n[tabindex=\"-1\"]:focus:not(:focus-visible) {\n  outline: 0 !important;\n}\n\nhr {\n  box-sizing: content-box;\n  height: 0;\n  overflow: visible;\n}\n\nh1, h2, h3, h4, h5, h6 {\n  margin-top: 0;\n  margin-bottom: 0.5rem;\n}\n\np {\n  margin-top: 0;\n  margin-bottom: 1rem;\n}\n\nabbr[title],\nabbr[data-original-title] {\n  text-decoration: underline;\n  -webkit-text-decoration: underline dotted;\n  text-decoration: underline dotted;\n  cursor: help;\n  border-bottom: 0;\n  -webkit-text-decoration-skip-ink: none;\n  text-decoration-skip-ink: none;\n}\n\naddress {\n  margin-bottom: 1rem;\n  font-style: normal;\n  line-height: inherit;\n}\n\nol,\nul,\ndl {\n  margin-top: 0;\n  margin-bottom: 1rem;\n}\n\nol ol,\nul ul,\nol ul,\nul ol {\n  margin-bottom: 0;\n}\n\ndt {\n  font-weight: 700;\n}\n\ndd {\n  margin-bottom: .5rem;\n  margin-left: 0;\n}\n\nblockquote {\n  margin: 0 0 1rem;\n}\n\nb,\nstrong {\n  font-weight: bolder;\n}\n\nsmall {\n  font-size: 80%;\n}\n\nsub,\nsup {\n  position: relative;\n  font-size: 75%;\n  line-height: 0;\n  vertical-align: baseline;\n}\n\nsub {\n  bottom: -.25em;\n}\n\nsup {\n  top: -.5em;\n}\n\na {\n  color: #007bff;\n  text-decoration: none;\n  background-color: transparent;\n}\n\na:hover {\n  color: #0056b3;\n  text-decoration: underline;\n}\n\na:not([href]) {\n  color: inherit;\n  text-decoration: none;\n}\n\na:not([href]):hover {\n  color: inherit;\n  text-decoration: none;\n}\n\npre,\ncode,\nkbd,\nsamp {\n  font-family: SFMono-Regular, Menlo, Monaco, Consolas, \"Liberation Mono\", \"Courier New\", monospace;\n  font-size: 1em;\n}\n\npre {\n  margin-top: 0;\n  margin-bottom: 1rem;\n  overflow: auto;\n}\n\nfigure {\n  margin: 0 0 1rem;\n}\n\nimg {\n  vertical-align: middle;\n  border-style: none;\n}\n\nsvg {\n  overflow: hidden;\n  vertical-align: middle;\n}\n\ntable {\n  border-collapse: collapse;\n}\n\ncaption {\n  padding-top: 0.75rem;\n  padding-bottom: 0.75rem;\n  color: #6c757d;\n  text-align: left;\n  caption-side: bottom;\n}\n\nth {\n  text-align: inherit;\n}\n\nlabel {\n  display: inline-block;\n  margin-bottom: 0.5rem;\n}\n\nbutton {\n  border-radius: 0;\n}\n\nbutton:focus {\n  outline: 1px dotted;\n  outline: 5px auto -webkit-focus-ring-color;\n}\n\ninput,\nbutton,\nselect,\noptgroup,\ntextarea {\n  margin: 0;\n  font-family: inherit;\n  font-size: inherit;\n  line-height: inherit;\n}\n\nbutton,\ninput {\n  overflow: visible;\n}\n\nbutton,\nselect {\n  text-transform: none;\n}\n\nselect {\n  word-wrap: normal;\n}\n\nbutton,\n[type=\"button\"],\n[type=\"reset\"],\n[type=\"submit\"] {\n  -webkit-appearance: button;\n}\n\nbutton:not(:disabled),\n[type=\"button\"]:not(:disabled),\n[type=\"reset\"]:not(:disabled),\n[type=\"submit\"]:not(:disabled) {\n  cursor: pointer;\n}\n\nbutton::-moz-focus-inner,\n[type=\"button\"]::-moz-focus-inner,\n[type=\"reset\"]::-moz-focus-inner,\n[type=\"submit\"]::-moz-focus-inner {\n  padding: 0;\n  border-style: none;\n}\n\ninput[type=\"radio\"],\ninput[type=\"checkbox\"] {\n  box-sizing: border-box;\n  padding: 0;\n}\n\ninput[type=\"date\"],\ninput[type=\"time\"],\ninput[type=\"datetime-local\"],\ninput[type=\"month\"] {\n  -webkit-appearance: listbox;\n}\n\ntextarea {\n  overflow: auto;\n  resize: vertical;\n}\n\nfieldset {\n  min-width: 0;\n  padding: 0;\n  margin: 0;\n  border: 0;\n}\n\nlegend {\n  display: block;\n  width: 100%;\n  max-width: 100%;\n  padding: 0;\n  margin-bottom: .5rem;\n  font-size: 1.5rem;\n  line-height: inherit;\n  color: inherit;\n  white-space: normal;\n}\n\nprogress {\n  vertical-align: baseline;\n}\n\n[type=\"number\"]::-webkit-inner-spin-button,\n[type=\"number\"]::-webkit-outer-spin-button {\n  height: auto;\n}\n\n[type=\"search\"] {\n  outline-offset: -2px;\n  -webkit-appearance: none;\n}\n\n[type=\"search\"]::-webkit-search-decoration {\n  -webkit-appearance: none;\n}\n\n::-webkit-file-upload-button {\n  font: inherit;\n  -webkit-appearance: button;\n}\n\noutput {\n  display: inline-block;\n}\n\nsummary {\n  display: list-item;\n  cursor: pointer;\n}\n\ntemplate {\n  display: none;\n}\n\n[hidden] {\n  display: none !important;\n}\n\nh1, h2, h3, h4, h5, h6,\n.h1, .h2, .h3, .h4, .h5, .h6 {\n  margin-bottom: 0.5rem;\n  font-weight: 500;\n  line-height: 1.2;\n}\n\nh1, .h1 {\n  font-size: 2.5rem;\n}\n\nh2, .h2 {\n  font-size: 2rem;\n}\n\nh3, .h3 {\n  font-size: 1.75rem;\n}\n\nh4, .h4 {\n  font-size: 1.5rem;\n}\n\nh5, .h5 {\n  font-size: 1.25rem;\n}\n\nh6, .h6 {\n  font-size: 1rem;\n}\n\n.lead {\n  font-size: 1.25rem;\n  font-weight: 300;\n}\n\n.display-1 {\n  font-size: 6rem;\n  font-weight: 300;\n  line-height: 1.2;\n}\n\n.display-2 {\n  font-size: 5.5rem;\n  font-weight: 300;\n  line-height: 1.2;\n}\n\n.display-3 {\n  font-size: 4.5rem;\n  font-weight: 300;\n  line-height: 1.2;\n}\n\n.display-4 {\n  font-size: 3.5rem;\n  font-weight: 300;\n  line-height: 1.2;\n}\n\nhr {\n  margin-top: 1rem;\n  margin-bottom: 1rem;\n  border: 0;\n  border-top: 1px solid rgba(0, 0, 0, 0.1);\n}\n\nsmall,\n.small {\n  font-size: 80%;\n  font-weight: 400;\n}\n\nmark,\n.mark {\n  padding: 0.2em;\n  background-color: #fcf8e3;\n}\n\n.list-unstyled {\n  padding-left: 0;\n  list-style: none;\n}\n\n.list-inline {\n  padding-left: 0;\n  list-style: none;\n}\n\n.list-inline-item {\n  display: inline-block;\n}\n\n.list-inline-item:not(:last-child) {\n  margin-right: 0.5rem;\n}\n\n.initialism {\n  font-size: 90%;\n  text-transform: uppercase;\n}\n\n.blockquote {\n  margin-bottom: 1rem;\n  font-size: 1.25rem;\n}\n\n.blockquote-footer {\n  display: block;\n  font-size: 80%;\n  color: #6c757d;\n}\n\n.blockquote-footer::before {\n  content: \"\\2014\\00A0\";\n}\n\n.img-fluid {\n  max-width: 100%;\n  height: auto;\n}\n\n.img-thumbnail {\n  padding: 0.25rem;\n  background-color: #fff;\n  border: 1px solid #dee2e6;\n  border-radius: 0.25rem;\n  max-width: 100%;\n  height: auto;\n}\n\n.figure {\n  display: inline-block;\n}\n\n.figure-img {\n  margin-bottom: 0.5rem;\n  line-height: 1;\n}\n\n.figure-caption {\n  font-size: 90%;\n  color: #6c757d;\n}\n\ncode {\n  font-size: 87.5%;\n  color: #e83e8c;\n  word-wrap: break-word;\n}\n\na > code {\n  color: inherit;\n}\n\nkbd {\n  padding: 0.2rem 0.4rem;\n  font-size: 87.5%;\n  color: #fff;\n  background-color: #212529;\n  border-radius: 0.2rem;\n}\n\nkbd kbd {\n  padding: 0;\n  font-size: 100%;\n  font-weight: 700;\n}\n\npre {\n  display: block;\n  font-size: 87.5%;\n  color: #212529;\n}\n\npre code {\n  font-size: inherit;\n  color: inherit;\n  word-break: normal;\n}\n\n.pre-scrollable {\n  max-height: 340px;\n  overflow-y: scroll;\n}\n\n.container {\n  width: 100%;\n  padding-right: 15px;\n  padding-left: 15px;\n  margin-right: auto;\n  margin-left: auto;\n}\n\n@media (min-width: 576px) {\n  .container {\n    max-width: 540px;\n  }\n}\n\n@media (min-width: 768px) {\n  .container {\n    max-width: 720px;\n  }\n}\n\n@media (min-width: 992px) {\n  .container {\n    max-width: 960px;\n  }\n}\n\n@media (min-width: 1200px) {\n  .container {\n    max-width: 1140px;\n  }\n}\n\n.container-fluid, .container-sm, .container-md, .container-lg, .container-xl {\n  width: 100%;\n  padding-right: 15px;\n  padding-left: 15px;\n  margin-right: auto;\n  margin-left: auto;\n}\n\n@media (min-width: 576px) {\n  .container, .container-sm {\n    max-width: 540px;\n  }\n}\n\n@media (min-width: 768px) {\n  .container, .container-sm, .container-md {\n    max-width: 720px;\n  }\n}\n\n@media (min-width: 992px) {\n  .container, .container-sm, .container-md, .container-lg {\n    max-width: 960px;\n  }\n}\n\n@media (min-width: 1200px) {\n  .container, .container-sm, .container-md, .container-lg, .container-xl {\n    max-width: 1140px;\n  }\n}\n\n.row {\n  display: -ms-flexbox;\n  display: flex;\n  -ms-flex-wrap: wrap;\n  flex-wrap: wrap;\n  margin-right: -15px;\n  margin-left: -15px;\n}\n\n.no-gutters {\n  margin-right: 0;\n  margin-left: 0;\n}\n\n.no-gutters > .col,\n.no-gutters > [class*=\"col-\"] {\n  padding-right: 0;\n  padding-left: 0;\n}\n\n.col-1, .col-2, .col-3, .col-4, .col-5, .col-6, .col-7, .col-8, .col-9, .col-10, .col-11, .col-12, .col,\n.col-auto, .col-sm-1, .col-sm-2, .col-sm-3, .col-sm-4, .col-sm-5, .col-sm-6, .col-sm-7, .col-sm-8, .col-sm-9, .col-sm-10, .col-sm-11, .col-sm-12, .col-sm,\n.col-sm-auto, .col-md-1, .col-md-2, .col-md-3, .col-md-4, .col-md-5, .col-md-6, .col-md-7, .col-md-8, .col-md-9, .col-md-10, .col-md-11, .col-md-12, .col-md,\n.col-md-auto, .col-lg-1, .col-lg-2, .col-lg-3, .col-lg-4, .col-lg-5, .col-lg-6, .col-lg-7, .col-lg-8, .col-lg-9, .col-lg-10, .col-lg-11, .col-lg-12, .col-lg,\n.col-lg-auto, .col-xl-1, .col-xl-2, .col-xl-3, .col-xl-4, .col-xl-5, .col-xl-6, .col-xl-7, .col-xl-8, .col-xl-9, .col-xl-10, .col-xl-11, .col-xl-12, .col-xl,\n.col-xl-auto {\n  position: relative;\n  width: 100%;\n  padding-right: 15px;\n  padding-left: 15px;\n}\n\n.col {\n  -ms-flex-preferred-size: 0;\n  flex-basis: 0;\n  -ms-flex-positive: 1;\n  flex-grow: 1;\n  max-width: 100%;\n}\n\n.row-cols-1 > * {\n  -ms-flex: 0 0 100%;\n  flex: 0 0 100%;\n  max-width: 100%;\n}\n\n.row-cols-2 > * {\n  -ms-flex: 0 0 50%;\n  flex: 0 0 50%;\n  max-width: 50%;\n}\n\n.row-cols-3 > * {\n  -ms-flex: 0 0 33.333333%;\n  flex: 0 0 33.333333%;\n  max-width: 33.333333%;\n}\n\n.row-cols-4 > * {\n  -ms-flex: 0 0 25%;\n  flex: 0 0 25%;\n  max-width: 25%;\n}\n\n.row-cols-5 > * {\n  -ms-flex: 0 0 20%;\n  flex: 0 0 20%;\n  max-width: 20%;\n}\n\n.row-cols-6 > * {\n  -ms-flex: 0 0 16.666667%;\n  flex: 0 0 16.666667%;\n  max-width: 16.666667%;\n}\n\n.col-auto {\n  -ms-flex: 0 0 auto;\n  flex: 0 0 auto;\n  width: auto;\n  max-width: 100%;\n}\n\n.col-1 {\n  -ms-flex: 0 0 8.333333%;\n  flex: 0 0 8.333333%;\n  max-width: 8.333333%;\n}\n\n.col-2 {\n  -ms-flex: 0 0 16.666667%;\n  flex: 0 0 16.666667%;\n  max-width: 16.666667%;\n}\n\n.col-3 {\n  -ms-flex: 0 0 25%;\n  flex: 0 0 25%;\n  max-width: 25%;\n}\n\n.col-4 {\n  -ms-flex: 0 0 33.333333%;\n  flex: 0 0 33.333333%;\n  max-width: 33.333333%;\n}\n\n.col-5 {\n  -ms-flex: 0 0 41.666667%;\n  flex: 0 0 41.666667%;\n  max-width: 41.666667%;\n}\n\n.col-6 {\n  -ms-flex: 0 0 50%;\n  flex: 0 0 50%;\n  max-width: 50%;\n}\n\n.col-7 {\n  -ms-flex: 0 0 58.333333%;\n  flex: 0 0 58.333333%;\n  max-width: 58.333333%;\n}\n\n.col-8 {\n  -ms-flex: 0 0 66.666667%;\n  flex: 0 0 66.666667%;\n  max-width: 66.666667%;\n}\n\n.col-9 {\n  -ms-flex: 0 0 75%;\n  flex: 0 0 75%;\n  max-width: 75%;\n}\n\n.col-10 {\n  -ms-flex: 0 0 83.333333%;\n  flex: 0 0 83.333333%;\n  max-width: 83.333333%;\n}\n\n.col-11 {\n  -ms-flex: 0 0 91.666667%;\n  flex: 0 0 91.666667%;\n  max-width: 91.666667%;\n}\n\n.col-12 {\n  -ms-flex: 0 0 100%;\n  flex: 0 0 100%;\n  max-width: 100%;\n}\n\n.order-first {\n  -ms-flex-order: -1;\n  order: -1;\n}\n\n.order-last {\n  -ms-flex-order: 13;\n  order: 13;\n}\n\n.order-0 {\n  -ms-flex-order: 0;\n  order: 0;\n}\n\n.order-1 {\n  -ms-flex-order: 1;\n  order: 1;\n}\n\n.order-2 {\n  -ms-flex-order: 2;\n  order: 2;\n}\n\n.order-3 {\n  -ms-flex-order: 3;\n  order: 3;\n}\n\n.order-4 {\n  -ms-flex-order: 4;\n  order: 4;\n}\n\n.order-5 {\n  -ms-flex-order: 5;\n  order: 5;\n}\n\n.order-6 {\n  -ms-flex-order: 6;\n  order: 6;\n}\n\n.order-7 {\n  -ms-flex-order: 7;\n  order: 7;\n}\n\n.order-8 {\n  -ms-flex-order: 8;\n  order: 8;\n}\n\n.order-9 {\n  -ms-flex-order: 9;\n  order: 9;\n}\n\n.order-10 {\n  -ms-flex-order: 10;\n  order: 10;\n}\n\n.order-11 {\n  -ms-flex-order: 11;\n  order: 11;\n}\n\n.order-12 {\n  -ms-flex-order: 12;\n  order: 12;\n}\n\n.offset-1 {\n  margin-left: 8.333333%;\n}\n\n.offset-2 {\n  margin-left: 16.666667%;\n}\n\n.offset-3 {\n  margin-left: 25%;\n}\n\n.offset-4 {\n  margin-left: 33.333333%;\n}\n\n.offset-5 {\n  margin-left: 41.666667%;\n}\n\n.offset-6 {\n  margin-left: 50%;\n}\n\n.offset-7 {\n  margin-left: 58.333333%;\n}\n\n.offset-8 {\n  margin-left: 66.666667%;\n}\n\n.offset-9 {\n  margin-left: 75%;\n}\n\n.offset-10 {\n  margin-left: 83.333333%;\n}\n\n.offset-11 {\n  margin-left: 91.666667%;\n}\n\n@media (min-width: 576px) {\n  .col-sm {\n    -ms-flex-preferred-size: 0;\n    flex-basis: 0;\n    -ms-flex-positive: 1;\n    flex-grow: 1;\n    max-width: 100%;\n  }\n  .row-cols-sm-1 > * {\n    -ms-flex: 0 0 100%;\n    flex: 0 0 100%;\n    max-width: 100%;\n  }\n  .row-cols-sm-2 > * {\n    -ms-flex: 0 0 50%;\n    flex: 0 0 50%;\n    max-width: 50%;\n  }\n  .row-cols-sm-3 > * {\n    -ms-flex: 0 0 33.333333%;\n    flex: 0 0 33.333333%;\n    max-width: 33.333333%;\n  }\n  .row-cols-sm-4 > * {\n    -ms-flex: 0 0 25%;\n    flex: 0 0 25%;\n    max-width: 25%;\n  }\n  .row-cols-sm-5 > * {\n    -ms-flex: 0 0 20%;\n    flex: 0 0 20%;\n    max-width: 20%;\n  }\n  .row-cols-sm-6 > * {\n    -ms-flex: 0 0 16.666667%;\n    flex: 0 0 16.666667%;\n    max-width: 16.666667%;\n  }\n  .col-sm-auto {\n    -ms-flex: 0 0 auto;\n    flex: 0 0 auto;\n    width: auto;\n    max-width: 100%;\n  }\n  .col-sm-1 {\n    -ms-flex: 0 0 8.333333%;\n    flex: 0 0 8.333333%;\n    max-width: 8.333333%;\n  }\n  .col-sm-2 {\n    -ms-flex: 0 0 16.666667%;\n    flex: 0 0 16.666667%;\n    max-width: 16.666667%;\n  }\n  .col-sm-3 {\n    -ms-flex: 0 0 25%;\n    flex: 0 0 25%;\n    max-width: 25%;\n  }\n  .col-sm-4 {\n    -ms-flex: 0 0 33.333333%;\n    flex: 0 0 33.333333%;\n    max-width: 33.333333%;\n  }\n  .col-sm-5 {\n    -ms-flex: 0 0 41.666667%;\n    flex: 0 0 41.666667%;\n    max-width: 41.666667%;\n  }\n  .col-sm-6 {\n    -ms-flex: 0 0 50%;\n    flex: 0 0 50%;\n    max-width: 50%;\n  }\n  .col-sm-7 {\n    -ms-flex: 0 0 58.333333%;\n    flex: 0 0 58.333333%;\n    max-width: 58.333333%;\n  }\n  .col-sm-8 {\n    -ms-flex: 0 0 66.666667%;\n    flex: 0 0 66.666667%;\n    max-width: 66.666667%;\n  }\n  .col-sm-9 {\n    -ms-flex: 0 0 75%;\n    flex: 0 0 75%;\n    max-width: 75%;\n  }\n  .col-sm-10 {\n    -ms-flex: 0 0 83.333333%;\n    flex: 0 0 83.333333%;\n    max-width: 83.333333%;\n  }\n  .col-sm-11 {\n    -ms-flex: 0 0 91.666667%;\n    flex: 0 0 91.666667%;\n    max-width: 91.666667%;\n  }\n  .col-sm-12 {\n    -ms-flex: 0 0 100%;\n    flex: 0 0 100%;\n    max-width: 100%;\n  }\n  .order-sm-first {\n    -ms-flex-order: -1;\n    order: -1;\n  }\n  .order-sm-last {\n    -ms-flex-order: 13;\n    order: 13;\n  }\n  .order-sm-0 {\n    -ms-flex-order: 0;\n    order: 0;\n  }\n  .order-sm-1 {\n    -ms-flex-order: 1;\n    order: 1;\n  }\n  .order-sm-2 {\n    -ms-flex-order: 2;\n    order: 2;\n  }\n  .order-sm-3 {\n    -ms-flex-order: 3;\n    order: 3;\n  }\n  .order-sm-4 {\n    -ms-flex-order: 4;\n    order: 4;\n  }\n  .order-sm-5 {\n    -ms-flex-order: 5;\n    order: 5;\n  }\n  .order-sm-6 {\n    -ms-flex-order: 6;\n    order: 6;\n  }\n  .order-sm-7 {\n    -ms-flex-order: 7;\n    order: 7;\n  }\n  .order-sm-8 {\n    -ms-flex-order: 8;\n    order: 8;\n  }\n  .order-sm-9 {\n    -ms-flex-order: 9;\n    order: 9;\n  }\n  .order-sm-10 {\n    -ms-flex-order: 10;\n    order: 10;\n  }\n  .order-sm-11 {\n    -ms-flex-order: 11;\n    order: 11;\n  }\n  .order-sm-12 {\n    -ms-flex-order: 12;\n    order: 12;\n  }\n  .offset-sm-0 {\n    margin-left: 0;\n  }\n  .offset-sm-1 {\n    margin-left: 8.333333%;\n  }\n  .offset-sm-2 {\n    margin-left: 16.666667%;\n  }\n  .offset-sm-3 {\n    margin-left: 25%;\n  }\n  .offset-sm-4 {\n    margin-left: 33.333333%;\n  }\n  .offset-sm-5 {\n    margin-left: 41.666667%;\n  }\n  .offset-sm-6 {\n    margin-left: 50%;\n  }\n  .offset-sm-7 {\n    margin-left: 58.333333%;\n  }\n  .offset-sm-8 {\n    margin-left: 66.666667%;\n  }\n  .offset-sm-9 {\n    margin-left: 75%;\n  }\n  .offset-sm-10 {\n    margin-left: 83.333333%;\n  }\n  .offset-sm-11 {\n    margin-left: 91.666667%;\n  }\n}\n\n@media (min-width: 768px) {\n  .col-md {\n    -ms-flex-preferred-size: 0;\n    flex-basis: 0;\n    -ms-flex-positive: 1;\n    flex-grow: 1;\n    max-width: 100%;\n  }\n  .row-cols-md-1 > * {\n    -ms-flex: 0 0 100%;\n    flex: 0 0 100%;\n    max-width: 100%;\n  }\n  .row-cols-md-2 > * {\n    -ms-flex: 0 0 50%;\n    flex: 0 0 50%;\n    max-width: 50%;\n  }\n  .row-cols-md-3 > * {\n    -ms-flex: 0 0 33.333333%;\n    flex: 0 0 33.333333%;\n    max-width: 33.333333%;\n  }\n  .row-cols-md-4 > * {\n    -ms-flex: 0 0 25%;\n    flex: 0 0 25%;\n    max-width: 25%;\n  }\n  .row-cols-md-5 > * {\n    -ms-flex: 0 0 20%;\n    flex: 0 0 20%;\n    max-width: 20%;\n  }\n  .row-cols-md-6 > * {\n    -ms-flex: 0 0 16.666667%;\n    flex: 0 0 16.666667%;\n    max-width: 16.666667%;\n  }\n  .col-md-auto {\n    -ms-flex: 0 0 auto;\n    flex: 0 0 auto;\n    width: auto;\n    max-width: 100%;\n  }\n  .col-md-1 {\n    -ms-flex: 0 0 8.333333%;\n    flex: 0 0 8.333333%;\n    max-width: 8.333333%;\n  }\n  .col-md-2 {\n    -ms-flex: 0 0 16.666667%;\n    flex: 0 0 16.666667%;\n    max-width: 16.666667%;\n  }\n  .col-md-3 {\n    -ms-flex: 0 0 25%;\n    flex: 0 0 25%;\n    max-width: 25%;\n  }\n  .col-md-4 {\n    -ms-flex: 0 0 33.333333%;\n    flex: 0 0 33.333333%;\n    max-width: 33.333333%;\n  }\n  .col-md-5 {\n    -ms-flex: 0 0 41.666667%;\n    flex: 0 0 41.666667%;\n    max-width: 41.666667%;\n  }\n  .col-md-6 {\n    -ms-flex: 0 0 50%;\n    flex: 0 0 50%;\n    max-width: 50%;\n  }\n  .col-md-7 {\n    -ms-flex: 0 0 58.333333%;\n    flex: 0 0 58.333333%;\n    max-width: 58.333333%;\n  }\n  .col-md-8 {\n    -ms-flex: 0 0 66.666667%;\n    flex: 0 0 66.666667%;\n    max-width: 66.666667%;\n  }\n  .col-md-9 {\n    -ms-flex: 0 0 75%;\n    flex: 0 0 75%;\n    max-width: 75%;\n  }\n  .col-md-10 {\n    -ms-flex: 0 0 83.333333%;\n    flex: 0 0 83.333333%;\n    max-width: 83.333333%;\n  }\n  .col-md-11 {\n    -ms-flex: 0 0 91.666667%;\n    flex: 0 0 91.666667%;\n    max-width: 91.666667%;\n  }\n  .col-md-12 {\n    -ms-flex: 0 0 100%;\n    flex: 0 0 100%;\n    max-width: 100%;\n  }\n  .order-md-first {\n    -ms-flex-order: -1;\n    order: -1;\n  }\n  .order-md-last {\n    -ms-flex-order: 13;\n    order: 13;\n  }\n  .order-md-0 {\n    -ms-flex-order: 0;\n    order: 0;\n  }\n  .order-md-1 {\n    -ms-flex-order: 1;\n    order: 1;\n  }\n  .order-md-2 {\n    -ms-flex-order: 2;\n    order: 2;\n  }\n  .order-md-3 {\n    -ms-flex-order: 3;\n    order: 3;\n  }\n  .order-md-4 {\n    -ms-flex-order: 4;\n    order: 4;\n  }\n  .order-md-5 {\n    -ms-flex-order: 5;\n    order: 5;\n  }\n  .order-md-6 {\n    -ms-flex-order: 6;\n    order: 6;\n  }\n  .order-md-7 {\n    -ms-flex-order: 7;\n    order: 7;\n  }\n  .order-md-8 {\n    -ms-flex-order: 8;\n    order: 8;\n  }\n  .order-md-9 {\n    -ms-flex-order: 9;\n    order: 9;\n  }\n  .order-md-10 {\n    -ms-flex-order: 10;\n    order: 10;\n  }\n  .order-md-11 {\n    -ms-flex-order: 11;\n    order: 11;\n  }\n  .order-md-12 {\n    -ms-flex-order: 12;\n    order: 12;\n  }\n  .offset-md-0 {\n    margin-left: 0;\n  }\n  .offset-md-1 {\n    margin-left: 8.333333%;\n  }\n  .offset-md-2 {\n    margin-left: 16.666667%;\n  }\n  .offset-md-3 {\n    margin-left: 25%;\n  }\n  .offset-md-4 {\n    margin-left: 33.333333%;\n  }\n  .offset-md-5 {\n    margin-left: 41.666667%;\n  }\n  .offset-md-6 {\n    margin-left: 50%;\n  }\n  .offset-md-7 {\n    margin-left: 58.333333%;\n  }\n  .offset-md-8 {\n    margin-left: 66.666667%;\n  }\n  .offset-md-9 {\n    margin-left: 75%;\n  }\n  .offset-md-10 {\n    margin-left: 83.333333%;\n  }\n  .offset-md-11 {\n    margin-left: 91.666667%;\n  }\n}\n\n@media (min-width: 992px) {\n  .col-lg {\n    -ms-flex-preferred-size: 0;\n    flex-basis: 0;\n    -ms-flex-positive: 1;\n    flex-grow: 1;\n    max-width: 100%;\n  }\n  .row-cols-lg-1 > * {\n    -ms-flex: 0 0 100%;\n    flex: 0 0 100%;\n    max-width: 100%;\n  }\n  .row-cols-lg-2 > * {\n    -ms-flex: 0 0 50%;\n    flex: 0 0 50%;\n    max-width: 50%;\n  }\n  .row-cols-lg-3 > * {\n    -ms-flex: 0 0 33.333333%;\n    flex: 0 0 33.333333%;\n    max-width: 33.333333%;\n  }\n  .row-cols-lg-4 > * {\n    -ms-flex: 0 0 25%;\n    flex: 0 0 25%;\n    max-width: 25%;\n  }\n  .row-cols-lg-5 > * {\n    -ms-flex: 0 0 20%;\n    flex: 0 0 20%;\n    max-width: 20%;\n  }\n  .row-cols-lg-6 > * {\n    -ms-flex: 0 0 16.666667%;\n    flex: 0 0 16.666667%;\n    max-width: 16.666667%;\n  }\n  .col-lg-auto {\n    -ms-flex: 0 0 auto;\n    flex: 0 0 auto;\n    width: auto;\n    max-width: 100%;\n  }\n  .col-lg-1 {\n    -ms-flex: 0 0 8.333333%;\n    flex: 0 0 8.333333%;\n    max-width: 8.333333%;\n  }\n  .col-lg-2 {\n    -ms-flex: 0 0 16.666667%;\n    flex: 0 0 16.666667%;\n    max-width: 16.666667%;\n  }\n  .col-lg-3 {\n    -ms-flex: 0 0 25%;\n    flex: 0 0 25%;\n    max-width: 25%;\n  }\n  .col-lg-4 {\n    -ms-flex: 0 0 33.333333%;\n    flex: 0 0 33.333333%;\n    max-width: 33.333333%;\n  }\n  .col-lg-5 {\n    -ms-flex: 0 0 41.666667%;\n    flex: 0 0 41.666667%;\n    max-width: 41.666667%;\n  }\n  .col-lg-6 {\n    -ms-flex: 0 0 50%;\n    flex: 0 0 50%;\n    max-width: 50%;\n  }\n  .col-lg-7 {\n    -ms-flex: 0 0 58.333333%;\n    flex: 0 0 58.333333%;\n    max-width: 58.333333%;\n  }\n  .col-lg-8 {\n    -ms-flex: 0 0 66.666667%;\n    flex: 0 0 66.666667%;\n    max-width: 66.666667%;\n  }\n  .col-lg-9 {\n    -ms-flex: 0 0 75%;\n    flex: 0 0 75%;\n    max-width: 75%;\n  }\n  .col-lg-10 {\n    -ms-flex: 0 0 83.333333%;\n    flex: 0 0 83.333333%;\n    max-width: 83.333333%;\n  }\n  .col-lg-11 {\n    -ms-flex: 0 0 91.666667%;\n    flex: 0 0 91.666667%;\n    max-width: 91.666667%;\n  }\n  .col-lg-12 {\n    -ms-flex: 0 0 100%;\n    flex: 0 0 100%;\n    max-width: 100%;\n  }\n  .order-lg-first {\n    -ms-flex-order: -1;\n    order: -1;\n  }\n  .order-lg-last {\n    -ms-flex-order: 13;\n    order: 13;\n  }\n  .order-lg-0 {\n    -ms-flex-order: 0;\n    order: 0;\n  }\n  .order-lg-1 {\n    -ms-flex-order: 1;\n    order: 1;\n  }\n  .order-lg-2 {\n    -ms-flex-order: 2;\n    order: 2;\n  }\n  .order-lg-3 {\n    -ms-flex-order: 3;\n    order: 3;\n  }\n  .order-lg-4 {\n    -ms-flex-order: 4;\n    order: 4;\n  }\n  .order-lg-5 {\n    -ms-flex-order: 5;\n    order: 5;\n  }\n  .order-lg-6 {\n    -ms-flex-order: 6;\n    order: 6;\n  }\n  .order-lg-7 {\n    -ms-flex-order: 7;\n    order: 7;\n  }\n  .order-lg-8 {\n    -ms-flex-order: 8;\n    order: 8;\n  }\n  .order-lg-9 {\n    -ms-flex-order: 9;\n    order: 9;\n  }\n  .order-lg-10 {\n    -ms-flex-order: 10;\n    order: 10;\n  }\n  .order-lg-11 {\n    -ms-flex-order: 11;\n    order: 11;\n  }\n  .order-lg-12 {\n    -ms-flex-order: 12;\n    order: 12;\n  }\n  .offset-lg-0 {\n    margin-left: 0;\n  }\n  .offset-lg-1 {\n    margin-left: 8.333333%;\n  }\n  .offset-lg-2 {\n    margin-left: 16.666667%;\n  }\n  .offset-lg-3 {\n    margin-left: 25%;\n  }\n  .offset-lg-4 {\n    margin-left: 33.333333%;\n  }\n  .offset-lg-5 {\n    margin-left: 41.666667%;\n  }\n  .offset-lg-6 {\n    margin-left: 50%;\n  }\n  .offset-lg-7 {\n    margin-left: 58.333333%;\n  }\n  .offset-lg-8 {\n    margin-left: 66.666667%;\n  }\n  .offset-lg-9 {\n    margin-left: 75%;\n  }\n  .offset-lg-10 {\n    margin-left: 83.333333%;\n  }\n  .offset-lg-11 {\n    margin-left: 91.666667%;\n  }\n}\n\n@media (min-width: 1200px) {\n  .col-xl {\n    -ms-flex-preferred-size: 0;\n    flex-basis: 0;\n    -ms-flex-positive: 1;\n    flex-grow: 1;\n    max-width: 100%;\n  }\n  .row-cols-xl-1 > * {\n    -ms-flex: 0 0 100%;\n    flex: 0 0 100%;\n    max-width: 100%;\n  }\n  .row-cols-xl-2 > * {\n    -ms-flex: 0 0 50%;\n    flex: 0 0 50%;\n    max-width: 50%;\n  }\n  .row-cols-xl-3 > * {\n    -ms-flex: 0 0 33.333333%;\n    flex: 0 0 33.333333%;\n    max-width: 33.333333%;\n  }\n  .row-cols-xl-4 > * {\n    -ms-flex: 0 0 25%;\n    flex: 0 0 25%;\n    max-width: 25%;\n  }\n  .row-cols-xl-5 > * {\n    -ms-flex: 0 0 20%;\n    flex: 0 0 20%;\n    max-width: 20%;\n  }\n  .row-cols-xl-6 > * {\n    -ms-flex: 0 0 16.666667%;\n    flex: 0 0 16.666667%;\n    max-width: 16.666667%;\n  }\n  .col-xl-auto {\n    -ms-flex: 0 0 auto;\n    flex: 0 0 auto;\n    width: auto;\n    max-width: 100%;\n  }\n  .col-xl-1 {\n    -ms-flex: 0 0 8.333333%;\n    flex: 0 0 8.333333%;\n    max-width: 8.333333%;\n  }\n  .col-xl-2 {\n    -ms-flex: 0 0 16.666667%;\n    flex: 0 0 16.666667%;\n    max-width: 16.666667%;\n  }\n  .col-xl-3 {\n    -ms-flex: 0 0 25%;\n    flex: 0 0 25%;\n    max-width: 25%;\n  }\n  .col-xl-4 {\n    -ms-flex: 0 0 33.333333%;\n    flex: 0 0 33.333333%;\n    max-width: 33.333333%;\n  }\n  .col-xl-5 {\n    -ms-flex: 0 0 41.666667%;\n    flex: 0 0 41.666667%;\n    max-width: 41.666667%;\n  }\n  .col-xl-6 {\n    -ms-flex: 0 0 50%;\n    flex: 0 0 50%;\n    max-width: 50%;\n  }\n  .col-xl-7 {\n    -ms-flex: 0 0 58.333333%;\n    flex: 0 0 58.333333%;\n    max-width: 58.333333%;\n  }\n  .col-xl-8 {\n    -ms-flex: 0 0 66.666667%;\n    flex: 0 0 66.666667%;\n    max-width: 66.666667%;\n  }\n  .col-xl-9 {\n    -ms-flex: 0 0 75%;\n    flex: 0 0 75%;\n    max-width: 75%;\n  }\n  .col-xl-10 {\n    -ms-flex: 0 0 83.333333%;\n    flex: 0 0 83.333333%;\n    max-width: 83.333333%;\n  }\n  .col-xl-11 {\n    -ms-flex: 0 0 91.666667%;\n    flex: 0 0 91.666667%;\n    max-width: 91.666667%;\n  }\n  .col-xl-12 {\n    -ms-flex: 0 0 100%;\n    flex: 0 0 100%;\n    max-width: 100%;\n  }\n  .order-xl-first {\n    -ms-flex-order: -1;\n    order: -1;\n  }\n  .order-xl-last {\n    -ms-flex-order: 13;\n    order: 13;\n  }\n  .order-xl-0 {\n    -ms-flex-order: 0;\n    order: 0;\n  }\n  .order-xl-1 {\n    -ms-flex-order: 1;\n    order: 1;\n  }\n  .order-xl-2 {\n    -ms-flex-order: 2;\n    order: 2;\n  }\n  .order-xl-3 {\n    -ms-flex-order: 3;\n    order: 3;\n  }\n  .order-xl-4 {\n    -ms-flex-order: 4;\n    order: 4;\n  }\n  .order-xl-5 {\n    -ms-flex-order: 5;\n    order: 5;\n  }\n  .order-xl-6 {\n    -ms-flex-order: 6;\n    order: 6;\n  }\n  .order-xl-7 {\n    -ms-flex-order: 7;\n    order: 7;\n  }\n  .order-xl-8 {\n    -ms-flex-order: 8;\n    order: 8;\n  }\n  .order-xl-9 {\n    -ms-flex-order: 9;\n    order: 9;\n  }\n  .order-xl-10 {\n    -ms-flex-order: 10;\n    order: 10;\n  }\n  .order-xl-11 {\n    -ms-flex-order: 11;\n    order: 11;\n  }\n  .order-xl-12 {\n    -ms-flex-order: 12;\n    order: 12;\n  }\n  .offset-xl-0 {\n    margin-left: 0;\n  }\n  .offset-xl-1 {\n    margin-left: 8.333333%;\n  }\n  .offset-xl-2 {\n    margin-left: 16.666667%;\n  }\n  .offset-xl-3 {\n    margin-left: 25%;\n  }\n  .offset-xl-4 {\n    margin-left: 33.333333%;\n  }\n  .offset-xl-5 {\n    margin-left: 41.666667%;\n  }\n  .offset-xl-6 {\n    margin-left: 50%;\n  }\n  .offset-xl-7 {\n    margin-left: 58.333333%;\n  }\n  .offset-xl-8 {\n    margin-left: 66.666667%;\n  }\n  .offset-xl-9 {\n    margin-left: 75%;\n  }\n  .offset-xl-10 {\n    margin-left: 83.333333%;\n  }\n  .offset-xl-11 {\n    margin-left: 91.666667%;\n  }\n}\n\n.table {\n  width: 100%;\n  margin-bottom: 1rem;\n  color: #212529;\n}\n\n.table th,\n.table td {\n  padding: 0.75rem;\n  vertical-align: top;\n  border-top: 1px solid #dee2e6;\n}\n\n.table thead th {\n  vertical-align: bottom;\n  border-bottom: 2px solid #dee2e6;\n}\n\n.table tbody + tbody {\n  border-top: 2px solid #dee2e6;\n}\n\n.table-sm th,\n.table-sm td {\n  padding: 0.3rem;\n}\n\n.table-bordered {\n  border: 1px solid #dee2e6;\n}\n\n.table-bordered th,\n.table-bordered td {\n  border: 1px solid #dee2e6;\n}\n\n.table-bordered thead th,\n.table-bordered thead td {\n  border-bottom-width: 2px;\n}\n\n.table-borderless th,\n.table-borderless td,\n.table-borderless thead th,\n.table-borderless tbody + tbody {\n  border: 0;\n}\n\n.table-striped tbody tr:nth-of-type(odd) {\n  background-color: rgba(0, 0, 0, 0.05);\n}\n\n.table-hover tbody tr:hover {\n  color: #212529;\n  background-color: rgba(0, 0, 0, 0.075);\n}\n\n.table-primary,\n.table-primary > th,\n.table-primary > td {\n  background-color: #b8daff;\n}\n\n.table-primary th,\n.table-primary td,\n.table-primary thead th,\n.table-primary tbody + tbody {\n  border-color: #7abaff;\n}\n\n.table-hover .table-primary:hover {\n  background-color: #9fcdff;\n}\n\n.table-hover .table-primary:hover > td,\n.table-hover .table-primary:hover > th {\n  background-color: #9fcdff;\n}\n\n.table-secondary,\n.table-secondary > th,\n.table-secondary > td {\n  background-color: #d6d8db;\n}\n\n.table-secondary th,\n.table-secondary td,\n.table-secondary thead th,\n.table-secondary tbody + tbody {\n  border-color: #b3b7bb;\n}\n\n.table-hover .table-secondary:hover {\n  background-color: #c8cbcf;\n}\n\n.table-hover .table-secondary:hover > td,\n.table-hover .table-secondary:hover > th {\n  background-color: #c8cbcf;\n}\n\n.table-success,\n.table-success > th,\n.table-success > td {\n  background-color: #c3e6cb;\n}\n\n.table-success th,\n.table-success td,\n.table-success thead th,\n.table-success tbody + tbody {\n  border-color: #8fd19e;\n}\n\n.table-hover .table-success:hover {\n  background-color: #b1dfbb;\n}\n\n.table-hover .table-success:hover > td,\n.table-hover .table-success:hover > th {\n  background-color: #b1dfbb;\n}\n\n.table-info,\n.table-info > th,\n.table-info > td {\n  background-color: #bee5eb;\n}\n\n.table-info th,\n.table-info td,\n.table-info thead th,\n.table-info tbody + tbody {\n  border-color: #86cfda;\n}\n\n.table-hover .table-info:hover {\n  background-color: #abdde5;\n}\n\n.table-hover .table-info:hover > td,\n.table-hover .table-info:hover > th {\n  background-color: #abdde5;\n}\n\n.table-warning,\n.table-warning > th,\n.table-warning > td {\n  background-color: #ffeeba;\n}\n\n.table-warning th,\n.table-warning td,\n.table-warning thead th,\n.table-warning tbody + tbody {\n  border-color: #ffdf7e;\n}\n\n.table-hover .table-warning:hover {\n  background-color: #ffe8a1;\n}\n\n.table-hover .table-warning:hover > td,\n.table-hover .table-warning:hover > th {\n  background-color: #ffe8a1;\n}\n\n.table-danger,\n.table-danger > th,\n.table-danger > td {\n  background-color: #f5c6cb;\n}\n\n.table-danger th,\n.table-danger td,\n.table-danger thead th,\n.table-danger tbody + tbody {\n  border-color: #ed969e;\n}\n\n.table-hover .table-danger:hover {\n  background-color: #f1b0b7;\n}\n\n.table-hover .table-danger:hover > td,\n.table-hover .table-danger:hover > th {\n  background-color: #f1b0b7;\n}\n\n.table-light,\n.table-light > th,\n.table-light > td {\n  background-color: #fdfdfe;\n}\n\n.table-light th,\n.table-light td,\n.table-light thead th,\n.table-light tbody + tbody {\n  border-color: #fbfcfc;\n}\n\n.table-hover .table-light:hover {\n  background-color: #ececf6;\n}\n\n.table-hover .table-light:hover > td,\n.table-hover .table-light:hover > th {\n  background-color: #ececf6;\n}\n\n.table-dark,\n.table-dark > th,\n.table-dark > td {\n  background-color: #c6c8ca;\n}\n\n.table-dark th,\n.table-dark td,\n.table-dark thead th,\n.table-dark tbody + tbody {\n  border-color: #95999c;\n}\n\n.table-hover .table-dark:hover {\n  background-color: #b9bbbe;\n}\n\n.table-hover .table-dark:hover > td,\n.table-hover .table-dark:hover > th {\n  background-color: #b9bbbe;\n}\n\n.table-active,\n.table-active > th,\n.table-active > td {\n  background-color: rgba(0, 0, 0, 0.075);\n}\n\n.table-hover .table-active:hover {\n  background-color: rgba(0, 0, 0, 0.075);\n}\n\n.table-hover .table-active:hover > td,\n.table-hover .table-active:hover > th {\n  background-color: rgba(0, 0, 0, 0.075);\n}\n\n.table .thead-dark th {\n  color: #fff;\n  background-color: #343a40;\n  border-color: #454d55;\n}\n\n.table .thead-light th {\n  color: #495057;\n  background-color: #e9ecef;\n  border-color: #dee2e6;\n}\n\n.table-dark {\n  color: #fff;\n  background-color: #343a40;\n}\n\n.table-dark th,\n.table-dark td,\n.table-dark thead th {\n  border-color: #454d55;\n}\n\n.table-dark.table-bordered {\n  border: 0;\n}\n\n.table-dark.table-striped tbody tr:nth-of-type(odd) {\n  background-color: rgba(255, 255, 255, 0.05);\n}\n\n.table-dark.table-hover tbody tr:hover {\n  color: #fff;\n  background-color: rgba(255, 255, 255, 0.075);\n}\n\n@media (max-width: 575.98px) {\n  .table-responsive-sm {\n    display: block;\n    width: 100%;\n    overflow-x: auto;\n    -webkit-overflow-scrolling: touch;\n  }\n  .table-responsive-sm > .table-bordered {\n    border: 0;\n  }\n}\n\n@media (max-width: 767.98px) {\n  .table-responsive-md {\n    display: block;\n    width: 100%;\n    overflow-x: auto;\n    -webkit-overflow-scrolling: touch;\n  }\n  .table-responsive-md > .table-bordered {\n    border: 0;\n  }\n}\n\n@media (max-width: 991.98px) {\n  .table-responsive-lg {\n    display: block;\n    width: 100%;\n    overflow-x: auto;\n    -webkit-overflow-scrolling: touch;\n  }\n  .table-responsive-lg > .table-bordered {\n    border: 0;\n  }\n}\n\n@media (max-width: 1199.98px) {\n  .table-responsive-xl {\n    display: block;\n    width: 100%;\n    overflow-x: auto;\n    -webkit-overflow-scrolling: touch;\n  }\n  .table-responsive-xl > .table-bordered {\n    border: 0;\n  }\n}\n\n.table-responsive {\n  display: block;\n  width: 100%;\n  overflow-x: auto;\n  -webkit-overflow-scrolling: touch;\n}\n\n.table-responsive > .table-bordered {\n  border: 0;\n}\n\n.form-control {\n  display: block;\n  width: 100%;\n  height: calc(1.5em + 0.75rem + 2px);\n  padding: 0.375rem 0.75rem;\n  font-size: 1rem;\n  font-weight: 400;\n  line-height: 1.5;\n  color: #495057;\n  background-color: #fff;\n  background-clip: padding-box;\n  border: 1px solid #ced4da;\n  border-radius: 0.25rem;\n  transition: border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .form-control {\n    transition: none;\n  }\n}\n\n.form-control::-ms-expand {\n  background-color: transparent;\n  border: 0;\n}\n\n.form-control:-moz-focusring {\n  color: transparent;\n  text-shadow: 0 0 0 #495057;\n}\n\n.form-control:focus {\n  color: #495057;\n  background-color: #fff;\n  border-color: #80bdff;\n  outline: 0;\n  box-shadow: 0 0 0 0.2rem rgba(0, 123, 255, 0.25);\n}\n\n.form-control::-webkit-input-placeholder {\n  color: #6c757d;\n  opacity: 1;\n}\n\n.form-control::-moz-placeholder {\n  color: #6c757d;\n  opacity: 1;\n}\n\n.form-control:-ms-input-placeholder {\n  color: #6c757d;\n  opacity: 1;\n}\n\n.form-control::-ms-input-placeholder {\n  color: #6c757d;\n  opacity: 1;\n}\n\n.form-control::placeholder {\n  color: #6c757d;\n  opacity: 1;\n}\n\n.form-control:disabled, .form-control[readonly] {\n  background-color: #e9ecef;\n  opacity: 1;\n}\n\nselect.form-control:focus::-ms-value {\n  color: #495057;\n  background-color: #fff;\n}\n\n.form-control-file,\n.form-control-range {\n  display: block;\n  width: 100%;\n}\n\n.col-form-label {\n  padding-top: calc(0.375rem + 1px);\n  padding-bottom: calc(0.375rem + 1px);\n  margin-bottom: 0;\n  font-size: inherit;\n  line-height: 1.5;\n}\n\n.col-form-label-lg {\n  padding-top: calc(0.5rem + 1px);\n  padding-bottom: calc(0.5rem + 1px);\n  font-size: 1.25rem;\n  line-height: 1.5;\n}\n\n.col-form-label-sm {\n  padding-top: calc(0.25rem + 1px);\n  padding-bottom: calc(0.25rem + 1px);\n  font-size: 0.875rem;\n  line-height: 1.5;\n}\n\n.form-control-plaintext {\n  display: block;\n  width: 100%;\n  padding: 0.375rem 0;\n  margin-bottom: 0;\n  font-size: 1rem;\n  line-height: 1.5;\n  color: #212529;\n  background-color: transparent;\n  border: solid transparent;\n  border-width: 1px 0;\n}\n\n.form-control-plaintext.form-control-sm, .form-control-plaintext.form-control-lg {\n  padding-right: 0;\n  padding-left: 0;\n}\n\n.form-control-sm {\n  height: calc(1.5em + 0.5rem + 2px);\n  padding: 0.25rem 0.5rem;\n  font-size: 0.875rem;\n  line-height: 1.5;\n  border-radius: 0.2rem;\n}\n\n.form-control-lg {\n  height: calc(1.5em + 1rem + 2px);\n  padding: 0.5rem 1rem;\n  font-size: 1.25rem;\n  line-height: 1.5;\n  border-radius: 0.3rem;\n}\n\nselect.form-control[size], select.form-control[multiple] {\n  height: auto;\n}\n\ntextarea.form-control {\n  height: auto;\n}\n\n.form-group {\n  margin-bottom: 1rem;\n}\n\n.form-text {\n  display: block;\n  margin-top: 0.25rem;\n}\n\n.form-row {\n  display: -ms-flexbox;\n  display: flex;\n  -ms-flex-wrap: wrap;\n  flex-wrap: wrap;\n  margin-right: -5px;\n  margin-left: -5px;\n}\n\n.form-row > .col,\n.form-row > [class*=\"col-\"] {\n  padding-right: 5px;\n  padding-left: 5px;\n}\n\n.form-check {\n  position: relative;\n  display: block;\n  padding-left: 1.25rem;\n}\n\n.form-check-input {\n  position: absolute;\n  margin-top: 0.3rem;\n  margin-left: -1.25rem;\n}\n\n.form-check-input[disabled] ~ .form-check-label,\n.form-check-input:disabled ~ .form-check-label {\n  color: #6c757d;\n}\n\n.form-check-label {\n  margin-bottom: 0;\n}\n\n.form-check-inline {\n  display: -ms-inline-flexbox;\n  display: inline-flex;\n  -ms-flex-align: center;\n  align-items: center;\n  padding-left: 0;\n  margin-right: 0.75rem;\n}\n\n.form-check-inline .form-check-input {\n  position: static;\n  margin-top: 0;\n  margin-right: 0.3125rem;\n  margin-left: 0;\n}\n\n.valid-feedback {\n  display: none;\n  width: 100%;\n  margin-top: 0.25rem;\n  font-size: 80%;\n  color: #28a745;\n}\n\n.valid-tooltip {\n  position: absolute;\n  top: 100%;\n  z-index: 5;\n  display: none;\n  max-width: 100%;\n  padding: 0.25rem 0.5rem;\n  margin-top: .1rem;\n  font-size: 0.875rem;\n  line-height: 1.5;\n  color: #fff;\n  background-color: rgba(40, 167, 69, 0.9);\n  border-radius: 0.25rem;\n}\n\n.was-validated :valid ~ .valid-feedback,\n.was-validated :valid ~ .valid-tooltip,\n.is-valid ~ .valid-feedback,\n.is-valid ~ .valid-tooltip {\n  display: block;\n}\n\n.was-validated .form-control:valid, .form-control.is-valid {\n  border-color: #28a745;\n  padding-right: calc(1.5em + 0.75rem);\n  background-image: url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath fill='%2328a745' d='M2.3 6.73L.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e\");\n  background-repeat: no-repeat;\n  background-position: right calc(0.375em + 0.1875rem) center;\n  background-size: calc(0.75em + 0.375rem) calc(0.75em + 0.375rem);\n}\n\n.was-validated .form-control:valid:focus, .form-control.is-valid:focus {\n  border-color: #28a745;\n  box-shadow: 0 0 0 0.2rem rgba(40, 167, 69, 0.25);\n}\n\n.was-validated textarea.form-control:valid, textarea.form-control.is-valid {\n  padding-right: calc(1.5em + 0.75rem);\n  background-position: top calc(0.375em + 0.1875rem) right calc(0.375em + 0.1875rem);\n}\n\n.was-validated .custom-select:valid, .custom-select.is-valid {\n  border-color: #28a745;\n  padding-right: calc(0.75em + 2.3125rem);\n  background: url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='5' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e\") no-repeat right 0.75rem center/8px 10px, url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath fill='%2328a745' d='M2.3 6.73L.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e\") #fff no-repeat center right 1.75rem/calc(0.75em + 0.375rem) calc(0.75em + 0.375rem);\n}\n\n.was-validated .custom-select:valid:focus, .custom-select.is-valid:focus {\n  border-color: #28a745;\n  box-shadow: 0 0 0 0.2rem rgba(40, 167, 69, 0.25);\n}\n\n.was-validated .form-check-input:valid ~ .form-check-label, .form-check-input.is-valid ~ .form-check-label {\n  color: #28a745;\n}\n\n.was-validated .form-check-input:valid ~ .valid-feedback,\n.was-validated .form-check-input:valid ~ .valid-tooltip, .form-check-input.is-valid ~ .valid-feedback,\n.form-check-input.is-valid ~ .valid-tooltip {\n  display: block;\n}\n\n.was-validated .custom-control-input:valid ~ .custom-control-label, .custom-control-input.is-valid ~ .custom-control-label {\n  color: #28a745;\n}\n\n.was-validated .custom-control-input:valid ~ .custom-control-label::before, .custom-control-input.is-valid ~ .custom-control-label::before {\n  border-color: #28a745;\n}\n\n.was-validated .custom-control-input:valid:checked ~ .custom-control-label::before, .custom-control-input.is-valid:checked ~ .custom-control-label::before {\n  border-color: #34ce57;\n  background-color: #34ce57;\n}\n\n.was-validated .custom-control-input:valid:focus ~ .custom-control-label::before, .custom-control-input.is-valid:focus ~ .custom-control-label::before {\n  box-shadow: 0 0 0 0.2rem rgba(40, 167, 69, 0.25);\n}\n\n.was-validated .custom-control-input:valid:focus:not(:checked) ~ .custom-control-label::before, .custom-control-input.is-valid:focus:not(:checked) ~ .custom-control-label::before {\n  border-color: #28a745;\n}\n\n.was-validated .custom-file-input:valid ~ .custom-file-label, .custom-file-input.is-valid ~ .custom-file-label {\n  border-color: #28a745;\n}\n\n.was-validated .custom-file-input:valid:focus ~ .custom-file-label, .custom-file-input.is-valid:focus ~ .custom-file-label {\n  border-color: #28a745;\n  box-shadow: 0 0 0 0.2rem rgba(40, 167, 69, 0.25);\n}\n\n.invalid-feedback {\n  display: none;\n  width: 100%;\n  margin-top: 0.25rem;\n  font-size: 80%;\n  color: #dc3545;\n}\n\n.invalid-tooltip {\n  position: absolute;\n  top: 100%;\n  z-index: 5;\n  display: none;\n  max-width: 100%;\n  padding: 0.25rem 0.5rem;\n  margin-top: .1rem;\n  font-size: 0.875rem;\n  line-height: 1.5;\n  color: #fff;\n  background-color: rgba(220, 53, 69, 0.9);\n  border-radius: 0.25rem;\n}\n\n.was-validated :invalid ~ .invalid-feedback,\n.was-validated :invalid ~ .invalid-tooltip,\n.is-invalid ~ .invalid-feedback,\n.is-invalid ~ .invalid-tooltip {\n  display: block;\n}\n\n.was-validated .form-control:invalid, .form-control.is-invalid {\n  border-color: #dc3545;\n  padding-right: calc(1.5em + 0.75rem);\n  background-image: url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' fill='none' stroke='%23dc3545' viewBox='0 0 12 12'%3e%3ccircle cx='6' cy='6' r='4.5'/%3e%3cpath stroke-linejoin='round' d='M5.8 3.6h.4L6 6.5z'/%3e%3ccircle cx='6' cy='8.2' r='.6' fill='%23dc3545' stroke='none'/%3e%3c/svg%3e\");\n  background-repeat: no-repeat;\n  background-position: right calc(0.375em + 0.1875rem) center;\n  background-size: calc(0.75em + 0.375rem) calc(0.75em + 0.375rem);\n}\n\n.was-validated .form-control:invalid:focus, .form-control.is-invalid:focus {\n  border-color: #dc3545;\n  box-shadow: 0 0 0 0.2rem rgba(220, 53, 69, 0.25);\n}\n\n.was-validated textarea.form-control:invalid, textarea.form-control.is-invalid {\n  padding-right: calc(1.5em + 0.75rem);\n  background-position: top calc(0.375em + 0.1875rem) right calc(0.375em + 0.1875rem);\n}\n\n.was-validated .custom-select:invalid, .custom-select.is-invalid {\n  border-color: #dc3545;\n  padding-right: calc(0.75em + 2.3125rem);\n  background: url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='5' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e\") no-repeat right 0.75rem center/8px 10px, url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' fill='none' stroke='%23dc3545' viewBox='0 0 12 12'%3e%3ccircle cx='6' cy='6' r='4.5'/%3e%3cpath stroke-linejoin='round' d='M5.8 3.6h.4L6 6.5z'/%3e%3ccircle cx='6' cy='8.2' r='.6' fill='%23dc3545' stroke='none'/%3e%3c/svg%3e\") #fff no-repeat center right 1.75rem/calc(0.75em + 0.375rem) calc(0.75em + 0.375rem);\n}\n\n.was-validated .custom-select:invalid:focus, .custom-select.is-invalid:focus {\n  border-color: #dc3545;\n  box-shadow: 0 0 0 0.2rem rgba(220, 53, 69, 0.25);\n}\n\n.was-validated .form-check-input:invalid ~ .form-check-label, .form-check-input.is-invalid ~ .form-check-label {\n  color: #dc3545;\n}\n\n.was-validated .form-check-input:invalid ~ .invalid-feedback,\n.was-validated .form-check-input:invalid ~ .invalid-tooltip, .form-check-input.is-invalid ~ .invalid-feedback,\n.form-check-input.is-invalid ~ .invalid-tooltip {\n  display: block;\n}\n\n.was-validated .custom-control-input:invalid ~ .custom-control-label, .custom-control-input.is-invalid ~ .custom-control-label {\n  color: #dc3545;\n}\n\n.was-validated .custom-control-input:invalid ~ .custom-control-label::before, .custom-control-input.is-invalid ~ .custom-control-label::before {\n  border-color: #dc3545;\n}\n\n.was-validated .custom-control-input:invalid:checked ~ .custom-control-label::before, .custom-control-input.is-invalid:checked ~ .custom-control-label::before {\n  border-color: #e4606d;\n  background-color: #e4606d;\n}\n\n.was-validated .custom-control-input:invalid:focus ~ .custom-control-label::before, .custom-control-input.is-invalid:focus ~ .custom-control-label::before {\n  box-shadow: 0 0 0 0.2rem rgba(220, 53, 69, 0.25);\n}\n\n.was-validated .custom-control-input:invalid:focus:not(:checked) ~ .custom-control-label::before, .custom-control-input.is-invalid:focus:not(:checked) ~ .custom-control-label::before {\n  border-color: #dc3545;\n}\n\n.was-validated .custom-file-input:invalid ~ .custom-file-label, .custom-file-input.is-invalid ~ .custom-file-label {\n  border-color: #dc3545;\n}\n\n.was-validated .custom-file-input:invalid:focus ~ .custom-file-label, .custom-file-input.is-invalid:focus ~ .custom-file-label {\n  border-color: #dc3545;\n  box-shadow: 0 0 0 0.2rem rgba(220, 53, 69, 0.25);\n}\n\n.form-inline {\n  display: -ms-flexbox;\n  display: flex;\n  -ms-flex-flow: row wrap;\n  flex-flow: row wrap;\n  -ms-flex-align: center;\n  align-items: center;\n}\n\n.form-inline .form-check {\n  width: 100%;\n}\n\n@media (min-width: 576px) {\n  .form-inline label {\n    display: -ms-flexbox;\n    display: flex;\n    -ms-flex-align: center;\n    align-items: center;\n    -ms-flex-pack: center;\n    justify-content: center;\n    margin-bottom: 0;\n  }\n  .form-inline .form-group {\n    display: -ms-flexbox;\n    display: flex;\n    -ms-flex: 0 0 auto;\n    flex: 0 0 auto;\n    -ms-flex-flow: row wrap;\n    flex-flow: row wrap;\n    -ms-flex-align: center;\n    align-items: center;\n    margin-bottom: 0;\n  }\n  .form-inline .form-control {\n    display: inline-block;\n    width: auto;\n    vertical-align: middle;\n  }\n  .form-inline .form-control-plaintext {\n    display: inline-block;\n  }\n  .form-inline .input-group,\n  .form-inline .custom-select {\n    width: auto;\n  }\n  .form-inline .form-check {\n    display: -ms-flexbox;\n    display: flex;\n    -ms-flex-align: center;\n    align-items: center;\n    -ms-flex-pack: center;\n    justify-content: center;\n    width: auto;\n    padding-left: 0;\n  }\n  .form-inline .form-check-input {\n    position: relative;\n    -ms-flex-negative: 0;\n    flex-shrink: 0;\n    margin-top: 0;\n    margin-right: 0.25rem;\n    margin-left: 0;\n  }\n  .form-inline .custom-control {\n    -ms-flex-align: center;\n    align-items: center;\n    -ms-flex-pack: center;\n    justify-content: center;\n  }\n  .form-inline .custom-control-label {\n    margin-bottom: 0;\n  }\n}\n\n.btn {\n  display: inline-block;\n  font-weight: 400;\n  color: #212529;\n  text-align: center;\n  vertical-align: middle;\n  cursor: pointer;\n  -webkit-user-select: none;\n  -moz-user-select: none;\n  -ms-user-select: none;\n  user-select: none;\n  background-color: transparent;\n  border: 1px solid transparent;\n  padding: 0.375rem 0.75rem;\n  font-size: 1rem;\n  line-height: 1.5;\n  border-radius: 0.25rem;\n  transition: color 0.15s ease-in-out, background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .btn {\n    transition: none;\n  }\n}\n\n.btn:hover {\n  color: #212529;\n  text-decoration: none;\n}\n\n.btn:focus, .btn.focus {\n  outline: 0;\n  box-shadow: 0 0 0 0.2rem rgba(0, 123, 255, 0.25);\n}\n\n.btn.disabled, .btn:disabled {\n  opacity: 0.65;\n}\n\na.btn.disabled,\nfieldset:disabled a.btn {\n  pointer-events: none;\n}\n\n.btn-primary {\n  color: #fff;\n  background-color: #007bff;\n  border-color: #007bff;\n}\n\n.btn-primary:hover {\n  color: #fff;\n  background-color: #0069d9;\n  border-color: #0062cc;\n}\n\n.btn-primary:focus, .btn-primary.focus {\n  color: #fff;\n  background-color: #0069d9;\n  border-color: #0062cc;\n  box-shadow: 0 0 0 0.2rem rgba(38, 143, 255, 0.5);\n}\n\n.btn-primary.disabled, .btn-primary:disabled {\n  color: #fff;\n  background-color: #007bff;\n  border-color: #007bff;\n}\n\n.btn-primary:not(:disabled):not(.disabled):active, .btn-primary:not(:disabled):not(.disabled).active,\n.show > .btn-primary.dropdown-toggle {\n  color: #fff;\n  background-color: #0062cc;\n  border-color: #005cbf;\n}\n\n.btn-primary:not(:disabled):not(.disabled):active:focus, .btn-primary:not(:disabled):not(.disabled).active:focus,\n.show > .btn-primary.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(38, 143, 255, 0.5);\n}\n\n.btn-secondary {\n  color: #fff;\n  background-color: #6c757d;\n  border-color: #6c757d;\n}\n\n.btn-secondary:hover {\n  color: #fff;\n  background-color: #5a6268;\n  border-color: #545b62;\n}\n\n.btn-secondary:focus, .btn-secondary.focus {\n  color: #fff;\n  background-color: #5a6268;\n  border-color: #545b62;\n  box-shadow: 0 0 0 0.2rem rgba(130, 138, 145, 0.5);\n}\n\n.btn-secondary.disabled, .btn-secondary:disabled {\n  color: #fff;\n  background-color: #6c757d;\n  border-color: #6c757d;\n}\n\n.btn-secondary:not(:disabled):not(.disabled):active, .btn-secondary:not(:disabled):not(.disabled).active,\n.show > .btn-secondary.dropdown-toggle {\n  color: #fff;\n  background-color: #545b62;\n  border-color: #4e555b;\n}\n\n.btn-secondary:not(:disabled):not(.disabled):active:focus, .btn-secondary:not(:disabled):not(.disabled).active:focus,\n.show > .btn-secondary.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(130, 138, 145, 0.5);\n}\n\n.btn-success {\n  color: #fff;\n  background-color: #28a745;\n  border-color: #28a745;\n}\n\n.btn-success:hover {\n  color: #fff;\n  background-color: #218838;\n  border-color: #1e7e34;\n}\n\n.btn-success:focus, .btn-success.focus {\n  color: #fff;\n  background-color: #218838;\n  border-color: #1e7e34;\n  box-shadow: 0 0 0 0.2rem rgba(72, 180, 97, 0.5);\n}\n\n.btn-success.disabled, .btn-success:disabled {\n  color: #fff;\n  background-color: #28a745;\n  border-color: #28a745;\n}\n\n.btn-success:not(:disabled):not(.disabled):active, .btn-success:not(:disabled):not(.disabled).active,\n.show > .btn-success.dropdown-toggle {\n  color: #fff;\n  background-color: #1e7e34;\n  border-color: #1c7430;\n}\n\n.btn-success:not(:disabled):not(.disabled):active:focus, .btn-success:not(:disabled):not(.disabled).active:focus,\n.show > .btn-success.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(72, 180, 97, 0.5);\n}\n\n.btn-info {\n  color: #fff;\n  background-color: #17a2b8;\n  border-color: #17a2b8;\n}\n\n.btn-info:hover {\n  color: #fff;\n  background-color: #138496;\n  border-color: #117a8b;\n}\n\n.btn-info:focus, .btn-info.focus {\n  color: #fff;\n  background-color: #138496;\n  border-color: #117a8b;\n  box-shadow: 0 0 0 0.2rem rgba(58, 176, 195, 0.5);\n}\n\n.btn-info.disabled, .btn-info:disabled {\n  color: #fff;\n  background-color: #17a2b8;\n  border-color: #17a2b8;\n}\n\n.btn-info:not(:disabled):not(.disabled):active, .btn-info:not(:disabled):not(.disabled).active,\n.show > .btn-info.dropdown-toggle {\n  color: #fff;\n  background-color: #117a8b;\n  border-color: #10707f;\n}\n\n.btn-info:not(:disabled):not(.disabled):active:focus, .btn-info:not(:disabled):not(.disabled).active:focus,\n.show > .btn-info.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(58, 176, 195, 0.5);\n}\n\n.btn-warning {\n  color: #212529;\n  background-color: #ffc107;\n  border-color: #ffc107;\n}\n\n.btn-warning:hover {\n  color: #212529;\n  background-color: #e0a800;\n  border-color: #d39e00;\n}\n\n.btn-warning:focus, .btn-warning.focus {\n  color: #212529;\n  background-color: #e0a800;\n  border-color: #d39e00;\n  box-shadow: 0 0 0 0.2rem rgba(222, 170, 12, 0.5);\n}\n\n.btn-warning.disabled, .btn-warning:disabled {\n  color: #212529;\n  background-color: #ffc107;\n  border-color: #ffc107;\n}\n\n.btn-warning:not(:disabled):not(.disabled):active, .btn-warning:not(:disabled):not(.disabled).active,\n.show > .btn-warning.dropdown-toggle {\n  color: #212529;\n  background-color: #d39e00;\n  border-color: #c69500;\n}\n\n.btn-warning:not(:disabled):not(.disabled):active:focus, .btn-warning:not(:disabled):not(.disabled).active:focus,\n.show > .btn-warning.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(222, 170, 12, 0.5);\n}\n\n.btn-danger {\n  color: #fff;\n  background-color: #dc3545;\n  border-color: #dc3545;\n}\n\n.btn-danger:hover {\n  color: #fff;\n  background-color: #c82333;\n  border-color: #bd2130;\n}\n\n.btn-danger:focus, .btn-danger.focus {\n  color: #fff;\n  background-color: #c82333;\n  border-color: #bd2130;\n  box-shadow: 0 0 0 0.2rem rgba(225, 83, 97, 0.5);\n}\n\n.btn-danger.disabled, .btn-danger:disabled {\n  color: #fff;\n  background-color: #dc3545;\n  border-color: #dc3545;\n}\n\n.btn-danger:not(:disabled):not(.disabled):active, .btn-danger:not(:disabled):not(.disabled).active,\n.show > .btn-danger.dropdown-toggle {\n  color: #fff;\n  background-color: #bd2130;\n  border-color: #b21f2d;\n}\n\n.btn-danger:not(:disabled):not(.disabled):active:focus, .btn-danger:not(:disabled):not(.disabled).active:focus,\n.show > .btn-danger.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(225, 83, 97, 0.5);\n}\n\n.btn-light {\n  color: #212529;\n  background-color: #f8f9fa;\n  border-color: #f8f9fa;\n}\n\n.btn-light:hover {\n  color: #212529;\n  background-color: #e2e6ea;\n  border-color: #dae0e5;\n}\n\n.btn-light:focus, .btn-light.focus {\n  color: #212529;\n  background-color: #e2e6ea;\n  border-color: #dae0e5;\n  box-shadow: 0 0 0 0.2rem rgba(216, 217, 219, 0.5);\n}\n\n.btn-light.disabled, .btn-light:disabled {\n  color: #212529;\n  background-color: #f8f9fa;\n  border-color: #f8f9fa;\n}\n\n.btn-light:not(:disabled):not(.disabled):active, .btn-light:not(:disabled):not(.disabled).active,\n.show > .btn-light.dropdown-toggle {\n  color: #212529;\n  background-color: #dae0e5;\n  border-color: #d3d9df;\n}\n\n.btn-light:not(:disabled):not(.disabled):active:focus, .btn-light:not(:disabled):not(.disabled).active:focus,\n.show > .btn-light.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(216, 217, 219, 0.5);\n}\n\n.btn-dark {\n  color: #fff;\n  background-color: #343a40;\n  border-color: #343a40;\n}\n\n.btn-dark:hover {\n  color: #fff;\n  background-color: #23272b;\n  border-color: #1d2124;\n}\n\n.btn-dark:focus, .btn-dark.focus {\n  color: #fff;\n  background-color: #23272b;\n  border-color: #1d2124;\n  box-shadow: 0 0 0 0.2rem rgba(82, 88, 93, 0.5);\n}\n\n.btn-dark.disabled, .btn-dark:disabled {\n  color: #fff;\n  background-color: #343a40;\n  border-color: #343a40;\n}\n\n.btn-dark:not(:disabled):not(.disabled):active, .btn-dark:not(:disabled):not(.disabled).active,\n.show > .btn-dark.dropdown-toggle {\n  color: #fff;\n  background-color: #1d2124;\n  border-color: #171a1d;\n}\n\n.btn-dark:not(:disabled):not(.disabled):active:focus, .btn-dark:not(:disabled):not(.disabled).active:focus,\n.show > .btn-dark.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(82, 88, 93, 0.5);\n}\n\n.btn-outline-primary {\n  color: #007bff;\n  border-color: #007bff;\n}\n\n.btn-outline-primary:hover {\n  color: #fff;\n  background-color: #007bff;\n  border-color: #007bff;\n}\n\n.btn-outline-primary:focus, .btn-outline-primary.focus {\n  box-shadow: 0 0 0 0.2rem rgba(0, 123, 255, 0.5);\n}\n\n.btn-outline-primary.disabled, .btn-outline-primary:disabled {\n  color: #007bff;\n  background-color: transparent;\n}\n\n.btn-outline-primary:not(:disabled):not(.disabled):active, .btn-outline-primary:not(:disabled):not(.disabled).active,\n.show > .btn-outline-primary.dropdown-toggle {\n  color: #fff;\n  background-color: #007bff;\n  border-color: #007bff;\n}\n\n.btn-outline-primary:not(:disabled):not(.disabled):active:focus, .btn-outline-primary:not(:disabled):not(.disabled).active:focus,\n.show > .btn-outline-primary.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(0, 123, 255, 0.5);\n}\n\n.btn-outline-secondary {\n  color: #6c757d;\n  border-color: #6c757d;\n}\n\n.btn-outline-secondary:hover {\n  color: #fff;\n  background-color: #6c757d;\n  border-color: #6c757d;\n}\n\n.btn-outline-secondary:focus, .btn-outline-secondary.focus {\n  box-shadow: 0 0 0 0.2rem rgba(108, 117, 125, 0.5);\n}\n\n.btn-outline-secondary.disabled, .btn-outline-secondary:disabled {\n  color: #6c757d;\n  background-color: transparent;\n}\n\n.btn-outline-secondary:not(:disabled):not(.disabled):active, .btn-outline-secondary:not(:disabled):not(.disabled).active,\n.show > .btn-outline-secondary.dropdown-toggle {\n  color: #fff;\n  background-color: #6c757d;\n  border-color: #6c757d;\n}\n\n.btn-outline-secondary:not(:disabled):not(.disabled):active:focus, .btn-outline-secondary:not(:disabled):not(.disabled).active:focus,\n.show > .btn-outline-secondary.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(108, 117, 125, 0.5);\n}\n\n.btn-outline-success {\n  color: #28a745;\n  border-color: #28a745;\n}\n\n.btn-outline-success:hover {\n  color: #fff;\n  background-color: #28a745;\n  border-color: #28a745;\n}\n\n.btn-outline-success:focus, .btn-outline-success.focus {\n  box-shadow: 0 0 0 0.2rem rgba(40, 167, 69, 0.5);\n}\n\n.btn-outline-success.disabled, .btn-outline-success:disabled {\n  color: #28a745;\n  background-color: transparent;\n}\n\n.btn-outline-success:not(:disabled):not(.disabled):active, .btn-outline-success:not(:disabled):not(.disabled).active,\n.show > .btn-outline-success.dropdown-toggle {\n  color: #fff;\n  background-color: #28a745;\n  border-color: #28a745;\n}\n\n.btn-outline-success:not(:disabled):not(.disabled):active:focus, .btn-outline-success:not(:disabled):not(.disabled).active:focus,\n.show > .btn-outline-success.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(40, 167, 69, 0.5);\n}\n\n.btn-outline-info {\n  color: #17a2b8;\n  border-color: #17a2b8;\n}\n\n.btn-outline-info:hover {\n  color: #fff;\n  background-color: #17a2b8;\n  border-color: #17a2b8;\n}\n\n.btn-outline-info:focus, .btn-outline-info.focus {\n  box-shadow: 0 0 0 0.2rem rgba(23, 162, 184, 0.5);\n}\n\n.btn-outline-info.disabled, .btn-outline-info:disabled {\n  color: #17a2b8;\n  background-color: transparent;\n}\n\n.btn-outline-info:not(:disabled):not(.disabled):active, .btn-outline-info:not(:disabled):not(.disabled).active,\n.show > .btn-outline-info.dropdown-toggle {\n  color: #fff;\n  background-color: #17a2b8;\n  border-color: #17a2b8;\n}\n\n.btn-outline-info:not(:disabled):not(.disabled):active:focus, .btn-outline-info:not(:disabled):not(.disabled).active:focus,\n.show > .btn-outline-info.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(23, 162, 184, 0.5);\n}\n\n.btn-outline-warning {\n  color: #ffc107;\n  border-color: #ffc107;\n}\n\n.btn-outline-warning:hover {\n  color: #212529;\n  background-color: #ffc107;\n  border-color: #ffc107;\n}\n\n.btn-outline-warning:focus, .btn-outline-warning.focus {\n  box-shadow: 0 0 0 0.2rem rgba(255, 193, 7, 0.5);\n}\n\n.btn-outline-warning.disabled, .btn-outline-warning:disabled {\n  color: #ffc107;\n  background-color: transparent;\n}\n\n.btn-outline-warning:not(:disabled):not(.disabled):active, .btn-outline-warning:not(:disabled):not(.disabled).active,\n.show > .btn-outline-warning.dropdown-toggle {\n  color: #212529;\n  background-color: #ffc107;\n  border-color: #ffc107;\n}\n\n.btn-outline-warning:not(:disabled):not(.disabled):active:focus, .btn-outline-warning:not(:disabled):not(.disabled).active:focus,\n.show > .btn-outline-warning.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(255, 193, 7, 0.5);\n}\n\n.btn-outline-danger {\n  color: #dc3545;\n  border-color: #dc3545;\n}\n\n.btn-outline-danger:hover {\n  color: #fff;\n  background-color: #dc3545;\n  border-color: #dc3545;\n}\n\n.btn-outline-danger:focus, .btn-outline-danger.focus {\n  box-shadow: 0 0 0 0.2rem rgba(220, 53, 69, 0.5);\n}\n\n.btn-outline-danger.disabled, .btn-outline-danger:disabled {\n  color: #dc3545;\n  background-color: transparent;\n}\n\n.btn-outline-danger:not(:disabled):not(.disabled):active, .btn-outline-danger:not(:disabled):not(.disabled).active,\n.show > .btn-outline-danger.dropdown-toggle {\n  color: #fff;\n  background-color: #dc3545;\n  border-color: #dc3545;\n}\n\n.btn-outline-danger:not(:disabled):not(.disabled):active:focus, .btn-outline-danger:not(:disabled):not(.disabled).active:focus,\n.show > .btn-outline-danger.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(220, 53, 69, 0.5);\n}\n\n.btn-outline-light {\n  color: #f8f9fa;\n  border-color: #f8f9fa;\n}\n\n.btn-outline-light:hover {\n  color: #212529;\n  background-color: #f8f9fa;\n  border-color: #f8f9fa;\n}\n\n.btn-outline-light:focus, .btn-outline-light.focus {\n  box-shadow: 0 0 0 0.2rem rgba(248, 249, 250, 0.5);\n}\n\n.btn-outline-light.disabled, .btn-outline-light:disabled {\n  color: #f8f9fa;\n  background-color: transparent;\n}\n\n.btn-outline-light:not(:disabled):not(.disabled):active, .btn-outline-light:not(:disabled):not(.disabled).active,\n.show > .btn-outline-light.dropdown-toggle {\n  color: #212529;\n  background-color: #f8f9fa;\n  border-color: #f8f9fa;\n}\n\n.btn-outline-light:not(:disabled):not(.disabled):active:focus, .btn-outline-light:not(:disabled):not(.disabled).active:focus,\n.show > .btn-outline-light.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(248, 249, 250, 0.5);\n}\n\n.btn-outline-dark {\n  color: #343a40;\n  border-color: #343a40;\n}\n\n.btn-outline-dark:hover {\n  color: #fff;\n  background-color: #343a40;\n  border-color: #343a40;\n}\n\n.btn-outline-dark:focus, .btn-outline-dark.focus {\n  box-shadow: 0 0 0 0.2rem rgba(52, 58, 64, 0.5);\n}\n\n.btn-outline-dark.disabled, .btn-outline-dark:disabled {\n  color: #343a40;\n  background-color: transparent;\n}\n\n.btn-outline-dark:not(:disabled):not(.disabled):active, .btn-outline-dark:not(:disabled):not(.disabled).active,\n.show > .btn-outline-dark.dropdown-toggle {\n  color: #fff;\n  background-color: #343a40;\n  border-color: #343a40;\n}\n\n.btn-outline-dark:not(:disabled):not(.disabled):active:focus, .btn-outline-dark:not(:disabled):not(.disabled).active:focus,\n.show > .btn-outline-dark.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(52, 58, 64, 0.5);\n}\n\n.btn-link {\n  font-weight: 400;\n  color: #007bff;\n  text-decoration: none;\n}\n\n.btn-link:hover {\n  color: #0056b3;\n  text-decoration: underline;\n}\n\n.btn-link:focus, .btn-link.focus {\n  text-decoration: underline;\n  box-shadow: none;\n}\n\n.btn-link:disabled, .btn-link.disabled {\n  color: #6c757d;\n  pointer-events: none;\n}\n\n.btn-lg, .btn-group-lg > .btn {\n  padding: 0.5rem 1rem;\n  font-size: 1.25rem;\n  line-height: 1.5;\n  border-radius: 0.3rem;\n}\n\n.btn-sm, .btn-group-sm > .btn {\n  padding: 0.25rem 0.5rem;\n  font-size: 0.875rem;\n  line-height: 1.5;\n  border-radius: 0.2rem;\n}\n\n.btn-block {\n  display: block;\n  width: 100%;\n}\n\n.btn-block + .btn-block {\n  margin-top: 0.5rem;\n}\n\ninput[type=\"submit\"].btn-block,\ninput[type=\"reset\"].btn-block,\ninput[type=\"button\"].btn-block {\n  width: 100%;\n}\n\n.fade {\n  transition: opacity 0.15s linear;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .fade {\n    transition: none;\n  }\n}\n\n.fade:not(.show) {\n  opacity: 0;\n}\n\n.collapse:not(.show) {\n  display: none;\n}\n\n.collapsing {\n  position: relative;\n  height: 0;\n  overflow: hidden;\n  transition: height 0.35s ease;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .collapsing {\n    transition: none;\n  }\n}\n\n.dropup,\n.dropright,\n.dropdown,\n.dropleft {\n  position: relative;\n}\n\n.dropdown-toggle {\n  white-space: nowrap;\n}\n\n.dropdown-toggle::after {\n  display: inline-block;\n  margin-left: 0.255em;\n  vertical-align: 0.255em;\n  content: \"\";\n  border-top: 0.3em solid;\n  border-right: 0.3em solid transparent;\n  border-bottom: 0;\n  border-left: 0.3em solid transparent;\n}\n\n.dropdown-toggle:empty::after {\n  margin-left: 0;\n}\n\n.dropdown-menu {\n  position: absolute;\n  top: 100%;\n  left: 0;\n  z-index: 1000;\n  display: none;\n  float: left;\n  min-width: 10rem;\n  padding: 0.5rem 0;\n  margin: 0.125rem 0 0;\n  font-size: 1rem;\n  color: #212529;\n  text-align: left;\n  list-style: none;\n  background-color: #fff;\n  background-clip: padding-box;\n  border: 1px solid rgba(0, 0, 0, 0.15);\n  border-radius: 0.25rem;\n}\n\n.dropdown-menu-left {\n  right: auto;\n  left: 0;\n}\n\n.dropdown-menu-right {\n  right: 0;\n  left: auto;\n}\n\n@media (min-width: 576px) {\n  .dropdown-menu-sm-left {\n    right: auto;\n    left: 0;\n  }\n  .dropdown-menu-sm-right {\n    right: 0;\n    left: auto;\n  }\n}\n\n@media (min-width: 768px) {\n  .dropdown-menu-md-left {\n    right: auto;\n    left: 0;\n  }\n  .dropdown-menu-md-right {\n    right: 0;\n    left: auto;\n  }\n}\n\n@media (min-width: 992px) {\n  .dropdown-menu-lg-left {\n    right: auto;\n    left: 0;\n  }\n  .dropdown-menu-lg-right {\n    right: 0;\n    left: auto;\n  }\n}\n\n@media (min-width: 1200px) {\n  .dropdown-menu-xl-left {\n    right: auto;\n    left: 0;\n  }\n  .dropdown-menu-xl-right {\n    right: 0;\n    left: auto;\n  }\n}\n\n.dropup .dropdown-menu {\n  top: auto;\n  bottom: 100%;\n  margin-top: 0;\n  margin-bottom: 0.125rem;\n}\n\n.dropup .dropdown-toggle::after {\n  display: inline-block;\n  margin-left: 0.255em;\n  vertical-align: 0.255em;\n  content: \"\";\n  border-top: 0;\n  border-right: 0.3em solid transparent;\n  border-bottom: 0.3em solid;\n  border-left: 0.3em solid transparent;\n}\n\n.dropup .dropdown-toggle:empty::after {\n  margin-left: 0;\n}\n\n.dropright .dropdown-menu {\n  top: 0;\n  right: auto;\n  left: 100%;\n  margin-top: 0;\n  margin-left: 0.125rem;\n}\n\n.dropright .dropdown-toggle::after {\n  display: inline-block;\n  margin-left: 0.255em;\n  vertical-align: 0.255em;\n  content: \"\";\n  border-top: 0.3em solid transparent;\n  border-right: 0;\n  border-bottom: 0.3em solid transparent;\n  border-left: 0.3em solid;\n}\n\n.dropright .dropdown-toggle:empty::after {\n  margin-left: 0;\n}\n\n.dropright .dropdown-toggle::after {\n  vertical-align: 0;\n}\n\n.dropleft .dropdown-menu {\n  top: 0;\n  right: 100%;\n  left: auto;\n  margin-top: 0;\n  margin-right: 0.125rem;\n}\n\n.dropleft .dropdown-toggle::after {\n  display: inline-block;\n  margin-left: 0.255em;\n  vertical-align: 0.255em;\n  content: \"\";\n}\n\n.dropleft .dropdown-toggle::after {\n  display: none;\n}\n\n.dropleft .dropdown-toggle::before {\n  display: inline-block;\n  margin-right: 0.255em;\n  vertical-align: 0.255em;\n  content: \"\";\n  border-top: 0.3em solid transparent;\n  border-right: 0.3em solid;\n  border-bottom: 0.3em solid transparent;\n}\n\n.dropleft .dropdown-toggle:empty::after {\n  margin-left: 0;\n}\n\n.dropleft .dropdown-toggle::before {\n  vertical-align: 0;\n}\n\n.dropdown-menu[x-placement^=\"top\"], .dropdown-menu[x-placement^=\"right\"], .dropdown-menu[x-placement^=\"bottom\"], .dropdown-menu[x-placement^=\"left\"] {\n  right: auto;\n  bottom: auto;\n}\n\n.dropdown-divider {\n  height: 0;\n  margin: 0.5rem 0;\n  overflow: hidden;\n  border-top: 1px solid #e9ecef;\n}\n\n.dropdown-item {\n  display: block;\n  width: 100%;\n  padding: 0.25rem 1.5rem;\n  clear: both;\n  font-weight: 400;\n  color: #212529;\n  text-align: inherit;\n  white-space: nowrap;\n  background-color: transparent;\n  border: 0;\n}\n\n.dropdown-item:hover, .dropdown-item:focus {\n  color: #16181b;\n  text-decoration: none;\n  background-color: #f8f9fa;\n}\n\n.dropdown-item.active, .dropdown-item:active {\n  color: #fff;\n  text-decoration: none;\n  background-color: #007bff;\n}\n\n.dropdown-item.disabled, .dropdown-item:disabled {\n  color: #6c757d;\n  pointer-events: none;\n  background-color: transparent;\n}\n\n.dropdown-menu.show {\n  display: block;\n}\n\n.dropdown-header {\n  display: block;\n  padding: 0.5rem 1.5rem;\n  margin-bottom: 0;\n  font-size: 0.875rem;\n  color: #6c757d;\n  white-space: nowrap;\n}\n\n.dropdown-item-text {\n  display: block;\n  padding: 0.25rem 1.5rem;\n  color: #212529;\n}\n\n.btn-group,\n.btn-group-vertical {\n  position: relative;\n  display: -ms-inline-flexbox;\n  display: inline-flex;\n  vertical-align: middle;\n}\n\n.btn-group > .btn,\n.btn-group-vertical > .btn {\n  position: relative;\n  -ms-flex: 1 1 auto;\n  flex: 1 1 auto;\n}\n\n.btn-group > .btn:hover,\n.btn-group-vertical > .btn:hover {\n  z-index: 1;\n}\n\n.btn-group > .btn:focus, .btn-group > .btn:active, .btn-group > .btn.active,\n.btn-group-vertical > .btn:focus,\n.btn-group-vertical > .btn:active,\n.btn-group-vertical > .btn.active {\n  z-index: 1;\n}\n\n.btn-toolbar {\n  display: -ms-flexbox;\n  display: flex;\n  -ms-flex-wrap: wrap;\n  flex-wrap: wrap;\n  -ms-flex-pack: start;\n  justify-content: flex-start;\n}\n\n.btn-toolbar .input-group {\n  width: auto;\n}\n\n.btn-group > .btn:not(:first-child),\n.btn-group > .btn-group:not(:first-child) {\n  margin-left: -1px;\n}\n\n.btn-group > .btn:not(:last-child):not(.dropdown-toggle),\n.btn-group > .btn-group:not(:last-child) > .btn {\n  border-top-right-radius: 0;\n  border-bottom-right-radius: 0;\n}\n\n.btn-group > .btn:not(:first-child),\n.btn-group > .btn-group:not(:first-child) > .btn {\n  border-top-left-radius: 0;\n  border-bottom-left-radius: 0;\n}\n\n.dropdown-toggle-split {\n  padding-right: 0.5625rem;\n  padding-left: 0.5625rem;\n}\n\n.dropdown-toggle-split::after,\n.dropup .dropdown-toggle-split::after,\n.dropright .dropdown-toggle-split::after {\n  margin-left: 0;\n}\n\n.dropleft .dropdown-toggle-split::before {\n  margin-right: 0;\n}\n\n.btn-sm + .dropdown-toggle-split, .btn-group-sm > .btn + .dropdown-toggle-split {\n  padding-right: 0.375rem;\n  padding-left: 0.375rem;\n}\n\n.btn-lg + .dropdown-toggle-split, .btn-group-lg > .btn + .dropdown-toggle-split {\n  padding-right: 0.75rem;\n  padding-left: 0.75rem;\n}\n\n.btn-group-vertical {\n  -ms-flex-direction: column;\n  flex-direction: column;\n  -ms-flex-align: start;\n  align-items: flex-start;\n  -ms-flex-pack: center;\n  justify-content: center;\n}\n\n.btn-group-vertical > .btn,\n.btn-group-vertical > .btn-group {\n  width: 100%;\n}\n\n.btn-group-vertical > .btn:not(:first-child),\n.btn-group-vertical > .btn-group:not(:first-child) {\n  margin-top: -1px;\n}\n\n.btn-group-vertical > .btn:not(:last-child):not(.dropdown-toggle),\n.btn-group-vertical > .btn-group:not(:last-child) > .btn {\n  border-bottom-right-radius: 0;\n  border-bottom-left-radius: 0;\n}\n\n.btn-group-vertical > .btn:not(:first-child),\n.btn-group-vertical > .btn-group:not(:first-child) > .btn {\n  border-top-left-radius: 0;\n  border-top-right-radius: 0;\n}\n\n.btn-group-toggle > .btn,\n.btn-group-toggle > .btn-group > .btn {\n  margin-bottom: 0;\n}\n\n.btn-group-toggle > .btn input[type=\"radio\"],\n.btn-group-toggle > .btn input[type=\"checkbox\"],\n.btn-group-toggle > .btn-group > .btn input[type=\"radio\"],\n.btn-group-toggle > .btn-group > .btn input[type=\"checkbox\"] {\n  position: absolute;\n  clip: rect(0, 0, 0, 0);\n  pointer-events: none;\n}\n\n.input-group {\n  position: relative;\n  display: -ms-flexbox;\n  display: flex;\n  -ms-flex-wrap: wrap;\n  flex-wrap: wrap;\n  -ms-flex-align: stretch;\n  align-items: stretch;\n  width: 100%;\n}\n\n.input-group > .form-control,\n.input-group > .form-control-plaintext,\n.input-group > .custom-select,\n.input-group > .custom-file {\n  position: relative;\n  -ms-flex: 1 1 0%;\n  flex: 1 1 0%;\n  min-width: 0;\n  margin-bottom: 0;\n}\n\n.input-group > .form-control + .form-control,\n.input-group > .form-control + .custom-select,\n.input-group > .form-control + .custom-file,\n.input-group > .form-control-plaintext + .form-control,\n.input-group > .form-control-plaintext + .custom-select,\n.input-group > .form-control-plaintext + .custom-file,\n.input-group > .custom-select + .form-control,\n.input-group > .custom-select + .custom-select,\n.input-group > .custom-select + .custom-file,\n.input-group > .custom-file + .form-control,\n.input-group > .custom-file + .custom-select,\n.input-group > .custom-file + .custom-file {\n  margin-left: -1px;\n}\n\n.input-group > .form-control:focus,\n.input-group > .custom-select:focus,\n.input-group > .custom-file .custom-file-input:focus ~ .custom-file-label {\n  z-index: 3;\n}\n\n.input-group > .custom-file .custom-file-input:focus {\n  z-index: 4;\n}\n\n.input-group > .form-control:not(:last-child),\n.input-group > .custom-select:not(:last-child) {\n  border-top-right-radius: 0;\n  border-bottom-right-radius: 0;\n}\n\n.input-group > .form-control:not(:first-child),\n.input-group > .custom-select:not(:first-child) {\n  border-top-left-radius: 0;\n  border-bottom-left-radius: 0;\n}\n\n.input-group > .custom-file {\n  display: -ms-flexbox;\n  display: flex;\n  -ms-flex-align: center;\n  align-items: center;\n}\n\n.input-group > .custom-file:not(:last-child) .custom-file-label,\n.input-group > .custom-file:not(:last-child) .custom-file-label::after {\n  border-top-right-radius: 0;\n  border-bottom-right-radius: 0;\n}\n\n.input-group > .custom-file:not(:first-child) .custom-file-label {\n  border-top-left-radius: 0;\n  border-bottom-left-radius: 0;\n}\n\n.input-group-prepend,\n.input-group-append {\n  display: -ms-flexbox;\n  display: flex;\n}\n\n.input-group-prepend .btn,\n.input-group-append .btn {\n  position: relative;\n  z-index: 2;\n}\n\n.input-group-prepend .btn:focus,\n.input-group-append .btn:focus {\n  z-index: 3;\n}\n\n.input-group-prepend .btn + .btn,\n.input-group-prepend .btn + .input-group-text,\n.input-group-prepend .input-group-text + .input-group-text,\n.input-group-prepend .input-group-text + .btn,\n.input-group-append .btn + .btn,\n.input-group-append .btn + .input-group-text,\n.input-group-append .input-group-text + .input-group-text,\n.input-group-append .input-group-text + .btn {\n  margin-left: -1px;\n}\n\n.input-group-prepend {\n  margin-right: -1px;\n}\n\n.input-group-append {\n  margin-left: -1px;\n}\n\n.input-group-text {\n  display: -ms-flexbox;\n  display: flex;\n  -ms-flex-align: center;\n  align-items: center;\n  padding: 0.375rem 0.75rem;\n  margin-bottom: 0;\n  font-size: 1rem;\n  font-weight: 400;\n  line-height: 1.5;\n  color: #495057;\n  text-align: center;\n  white-space: nowrap;\n  background-color: #e9ecef;\n  border: 1px solid #ced4da;\n  border-radius: 0.25rem;\n}\n\n.input-group-text input[type=\"radio\"],\n.input-group-text input[type=\"checkbox\"] {\n  margin-top: 0;\n}\n\n.input-group-lg > .form-control:not(textarea),\n.input-group-lg > .custom-select {\n  height: calc(1.5em + 1rem + 2px);\n}\n\n.input-group-lg > .form-control,\n.input-group-lg > .custom-select,\n.input-group-lg > .input-group-prepend > .input-group-text,\n.input-group-lg > .input-group-append > .input-group-text,\n.input-group-lg > .input-group-prepend > .btn,\n.input-group-lg > .input-group-append > .btn {\n  padding: 0.5rem 1rem;\n  font-size: 1.25rem;\n  line-height: 1.5;\n  border-radius: 0.3rem;\n}\n\n.input-group-sm > .form-control:not(textarea),\n.input-group-sm > .custom-select {\n  height: calc(1.5em + 0.5rem + 2px);\n}\n\n.input-group-sm > .form-control,\n.input-group-sm > .custom-select,\n.input-group-sm > .input-group-prepend > .input-group-text,\n.input-group-sm > .input-group-append > .input-group-text,\n.input-group-sm > .input-group-prepend > .btn,\n.input-group-sm > .input-group-append > .btn {\n  padding: 0.25rem 0.5rem;\n  font-size: 0.875rem;\n  line-height: 1.5;\n  border-radius: 0.2rem;\n}\n\n.input-group-lg > .custom-select,\n.input-group-sm > .custom-select {\n  padding-right: 1.75rem;\n}\n\n.input-group > .input-group-prepend > .btn,\n.input-group > .input-group-prepend > .input-group-text,\n.input-group > .input-group-append:not(:last-child) > .btn,\n.input-group > .input-group-append:not(:last-child) > .input-group-text,\n.input-group > .input-group-append:last-child > .btn:not(:last-child):not(.dropdown-toggle),\n.input-group > .input-group-append:last-child > .input-group-text:not(:last-child) {\n  border-top-right-radius: 0;\n  border-bottom-right-radius: 0;\n}\n\n.input-group > .input-group-append > .btn,\n.input-group > .input-group-append > .input-group-text,\n.input-group > .input-group-prepend:not(:first-child) > .btn,\n.input-group > .input-group-prepend:not(:first-child) > .input-group-text,\n.input-group > .input-group-prepend:first-child > .btn:not(:first-child),\n.input-group > .input-group-prepend:first-child > .input-group-text:not(:first-child) {\n  border-top-left-radius: 0;\n  border-bottom-left-radius: 0;\n}\n\n.custom-control {\n  position: relative;\n  display: block;\n  min-height: 1.5rem;\n  padding-left: 1.5rem;\n}\n\n.custom-control-inline {\n  display: -ms-inline-flexbox;\n  display: inline-flex;\n  margin-right: 1rem;\n}\n\n.custom-control-input {\n  position: absolute;\n  left: 0;\n  z-index: -1;\n  width: 1rem;\n  height: 1.25rem;\n  opacity: 0;\n}\n\n.custom-control-input:checked ~ .custom-control-label::before {\n  color: #fff;\n  border-color: #007bff;\n  background-color: #007bff;\n}\n\n.custom-control-input:focus ~ .custom-control-label::before {\n  box-shadow: 0 0 0 0.2rem rgba(0, 123, 255, 0.25);\n}\n\n.custom-control-input:focus:not(:checked) ~ .custom-control-label::before {\n  border-color: #80bdff;\n}\n\n.custom-control-input:not(:disabled):active ~ .custom-control-label::before {\n  color: #fff;\n  background-color: #b3d7ff;\n  border-color: #b3d7ff;\n}\n\n.custom-control-input[disabled] ~ .custom-control-label, .custom-control-input:disabled ~ .custom-control-label {\n  color: #6c757d;\n}\n\n.custom-control-input[disabled] ~ .custom-control-label::before, .custom-control-input:disabled ~ .custom-control-label::before {\n  background-color: #e9ecef;\n}\n\n.custom-control-label {\n  position: relative;\n  margin-bottom: 0;\n  vertical-align: top;\n}\n\n.custom-control-label::before {\n  position: absolute;\n  top: 0.25rem;\n  left: -1.5rem;\n  display: block;\n  width: 1rem;\n  height: 1rem;\n  pointer-events: none;\n  content: \"\";\n  background-color: #fff;\n  border: #adb5bd solid 1px;\n}\n\n.custom-control-label::after {\n  position: absolute;\n  top: 0.25rem;\n  left: -1.5rem;\n  display: block;\n  width: 1rem;\n  height: 1rem;\n  content: \"\";\n  background: no-repeat 50% / 50% 50%;\n}\n\n.custom-checkbox .custom-control-label::before {\n  border-radius: 0.25rem;\n}\n\n.custom-checkbox .custom-control-input:checked ~ .custom-control-label::after {\n  background-image: url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath fill='%23fff' d='M6.564.75l-3.59 3.612-1.538-1.55L0 4.26l2.974 2.99L8 2.193z'/%3e%3c/svg%3e\");\n}\n\n.custom-checkbox .custom-control-input:indeterminate ~ .custom-control-label::before {\n  border-color: #007bff;\n  background-color: #007bff;\n}\n\n.custom-checkbox .custom-control-input:indeterminate ~ .custom-control-label::after {\n  background-image: url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='4' viewBox='0 0 4 4'%3e%3cpath stroke='%23fff' d='M0 2h4'/%3e%3c/svg%3e\");\n}\n\n.custom-checkbox .custom-control-input:disabled:checked ~ .custom-control-label::before {\n  background-color: rgba(0, 123, 255, 0.5);\n}\n\n.custom-checkbox .custom-control-input:disabled:indeterminate ~ .custom-control-label::before {\n  background-color: rgba(0, 123, 255, 0.5);\n}\n\n.custom-radio .custom-control-label::before {\n  border-radius: 50%;\n}\n\n.custom-radio .custom-control-input:checked ~ .custom-control-label::after {\n  background-image: url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='%23fff'/%3e%3c/svg%3e\");\n}\n\n.custom-radio .custom-control-input:disabled:checked ~ .custom-control-label::before {\n  background-color: rgba(0, 123, 255, 0.5);\n}\n\n.custom-switch {\n  padding-left: 2.25rem;\n}\n\n.custom-switch .custom-control-label::before {\n  left: -2.25rem;\n  width: 1.75rem;\n  pointer-events: all;\n  border-radius: 0.5rem;\n}\n\n.custom-switch .custom-control-label::after {\n  top: calc(0.25rem + 2px);\n  left: calc(-2.25rem + 2px);\n  width: calc(1rem - 4px);\n  height: calc(1rem - 4px);\n  background-color: #adb5bd;\n  border-radius: 0.5rem;\n  transition: background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out, -webkit-transform 0.15s ease-in-out;\n  transition: transform 0.15s ease-in-out, background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;\n  transition: transform 0.15s ease-in-out, background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out, -webkit-transform 0.15s ease-in-out;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .custom-switch .custom-control-label::after {\n    transition: none;\n  }\n}\n\n.custom-switch .custom-control-input:checked ~ .custom-control-label::after {\n  background-color: #fff;\n  -webkit-transform: translateX(0.75rem);\n  transform: translateX(0.75rem);\n}\n\n.custom-switch .custom-control-input:disabled:checked ~ .custom-control-label::before {\n  background-color: rgba(0, 123, 255, 0.5);\n}\n\n.custom-select {\n  display: inline-block;\n  width: 100%;\n  height: calc(1.5em + 0.75rem + 2px);\n  padding: 0.375rem 1.75rem 0.375rem 0.75rem;\n  font-size: 1rem;\n  font-weight: 400;\n  line-height: 1.5;\n  color: #495057;\n  vertical-align: middle;\n  background: #fff url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='5' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e\") no-repeat right 0.75rem center/8px 10px;\n  border: 1px solid #ced4da;\n  border-radius: 0.25rem;\n  -webkit-appearance: none;\n  -moz-appearance: none;\n  appearance: none;\n}\n\n.custom-select:focus {\n  border-color: #80bdff;\n  outline: 0;\n  box-shadow: 0 0 0 0.2rem rgba(0, 123, 255, 0.25);\n}\n\n.custom-select:focus::-ms-value {\n  color: #495057;\n  background-color: #fff;\n}\n\n.custom-select[multiple], .custom-select[size]:not([size=\"1\"]) {\n  height: auto;\n  padding-right: 0.75rem;\n  background-image: none;\n}\n\n.custom-select:disabled {\n  color: #6c757d;\n  background-color: #e9ecef;\n}\n\n.custom-select::-ms-expand {\n  display: none;\n}\n\n.custom-select:-moz-focusring {\n  color: transparent;\n  text-shadow: 0 0 0 #495057;\n}\n\n.custom-select-sm {\n  height: calc(1.5em + 0.5rem + 2px);\n  padding-top: 0.25rem;\n  padding-bottom: 0.25rem;\n  padding-left: 0.5rem;\n  font-size: 0.875rem;\n}\n\n.custom-select-lg {\n  height: calc(1.5em + 1rem + 2px);\n  padding-top: 0.5rem;\n  padding-bottom: 0.5rem;\n  padding-left: 1rem;\n  font-size: 1.25rem;\n}\n\n.custom-file {\n  position: relative;\n  display: inline-block;\n  width: 100%;\n  height: calc(1.5em + 0.75rem + 2px);\n  margin-bottom: 0;\n}\n\n.custom-file-input {\n  position: relative;\n  z-index: 2;\n  width: 100%;\n  height: calc(1.5em + 0.75rem + 2px);\n  margin: 0;\n  opacity: 0;\n}\n\n.custom-file-input:focus ~ .custom-file-label {\n  border-color: #80bdff;\n  box-shadow: 0 0 0 0.2rem rgba(0, 123, 255, 0.25);\n}\n\n.custom-file-input[disabled] ~ .custom-file-label,\n.custom-file-input:disabled ~ .custom-file-label {\n  background-color: #e9ecef;\n}\n\n.custom-file-input:lang(en) ~ .custom-file-label::after {\n  content: \"Browse\";\n}\n\n.custom-file-input ~ .custom-file-label[data-browse]::after {\n  content: attr(data-browse);\n}\n\n.custom-file-label {\n  position: absolute;\n  top: 0;\n  right: 0;\n  left: 0;\n  z-index: 1;\n  height: calc(1.5em + 0.75rem + 2px);\n  padding: 0.375rem 0.75rem;\n  font-weight: 400;\n  line-height: 1.5;\n  color: #495057;\n  background-color: #fff;\n  border: 1px solid #ced4da;\n  border-radius: 0.25rem;\n}\n\n.custom-file-label::after {\n  position: absolute;\n  top: 0;\n  right: 0;\n  bottom: 0;\n  z-index: 3;\n  display: block;\n  height: calc(1.5em + 0.75rem);\n  padding: 0.375rem 0.75rem;\n  line-height: 1.5;\n  color: #495057;\n  content: \"Browse\";\n  background-color: #e9ecef;\n  border-left: inherit;\n  border-radius: 0 0.25rem 0.25rem 0;\n}\n\n.custom-range {\n  width: 100%;\n  height: 1.4rem;\n  padding: 0;\n  background-color: transparent;\n  -webkit-appearance: none;\n  -moz-appearance: none;\n  appearance: none;\n}\n\n.custom-range:focus {\n  outline: none;\n}\n\n.custom-range:focus::-webkit-slider-thumb {\n  box-shadow: 0 0 0 1px #fff, 0 0 0 0.2rem rgba(0, 123, 255, 0.25);\n}\n\n.custom-range:focus::-moz-range-thumb {\n  box-shadow: 0 0 0 1px #fff, 0 0 0 0.2rem rgba(0, 123, 255, 0.25);\n}\n\n.custom-range:focus::-ms-thumb {\n  box-shadow: 0 0 0 1px #fff, 0 0 0 0.2rem rgba(0, 123, 255, 0.25);\n}\n\n.custom-range::-moz-focus-outer {\n  border: 0;\n}\n\n.custom-range::-webkit-slider-thumb {\n  width: 1rem;\n  height: 1rem;\n  margin-top: -0.25rem;\n  background-color: #007bff;\n  border: 0;\n  border-radius: 1rem;\n  -webkit-transition: background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;\n  transition: background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;\n  -webkit-appearance: none;\n  appearance: none;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .custom-range::-webkit-slider-thumb {\n    -webkit-transition: none;\n    transition: none;\n  }\n}\n\n.custom-range::-webkit-slider-thumb:active {\n  background-color: #b3d7ff;\n}\n\n.custom-range::-webkit-slider-runnable-track {\n  width: 100%;\n  height: 0.5rem;\n  color: transparent;\n  cursor: pointer;\n  background-color: #dee2e6;\n  border-color: transparent;\n  border-radius: 1rem;\n}\n\n.custom-range::-moz-range-thumb {\n  width: 1rem;\n  height: 1rem;\n  background-color: #007bff;\n  border: 0;\n  border-radius: 1rem;\n  -moz-transition: background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;\n  transition: background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;\n  -moz-appearance: none;\n  appearance: none;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .custom-range::-moz-range-thumb {\n    -moz-transition: none;\n    transition: none;\n  }\n}\n\n.custom-range::-moz-range-thumb:active {\n  background-color: #b3d7ff;\n}\n\n.custom-range::-moz-range-track {\n  width: 100%;\n  height: 0.5rem;\n  color: transparent;\n  cursor: pointer;\n  background-color: #dee2e6;\n  border-color: transparent;\n  border-radius: 1rem;\n}\n\n.custom-range::-ms-thumb {\n  width: 1rem;\n  height: 1rem;\n  margin-top: 0;\n  margin-right: 0.2rem;\n  margin-left: 0.2rem;\n  background-color: #007bff;\n  border: 0;\n  border-radius: 1rem;\n  -ms-transition: background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;\n  transition: background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;\n  appearance: none;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .custom-range::-ms-thumb {\n    -ms-transition: none;\n    transition: none;\n  }\n}\n\n.custom-range::-ms-thumb:active {\n  background-color: #b3d7ff;\n}\n\n.custom-range::-ms-track {\n  width: 100%;\n  height: 0.5rem;\n  color: transparent;\n  cursor: pointer;\n  background-color: transparent;\n  border-color: transparent;\n  border-width: 0.5rem;\n}\n\n.custom-range::-ms-fill-lower {\n  background-color: #dee2e6;\n  border-radius: 1rem;\n}\n\n.custom-range::-ms-fill-upper {\n  margin-right: 15px;\n  background-color: #dee2e6;\n  border-radius: 1rem;\n}\n\n.custom-range:disabled::-webkit-slider-thumb {\n  background-color: #adb5bd;\n}\n\n.custom-range:disabled::-webkit-slider-runnable-track {\n  cursor: default;\n}\n\n.custom-range:disabled::-moz-range-thumb {\n  background-color: #adb5bd;\n}\n\n.custom-range:disabled::-moz-range-track {\n  cursor: default;\n}\n\n.custom-range:disabled::-ms-thumb {\n  background-color: #adb5bd;\n}\n\n.custom-control-label::before,\n.custom-file-label,\n.custom-select {\n  transition: background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .custom-control-label::before,\n  .custom-file-label,\n  .custom-select {\n    transition: none;\n  }\n}\n\n.nav {\n  display: -ms-flexbox;\n  display: flex;\n  -ms-flex-wrap: wrap;\n  flex-wrap: wrap;\n  padding-left: 0;\n  margin-bottom: 0;\n  list-style: none;\n}\n\n.nav-link {\n  display: block;\n  padding: 0.5rem 1rem;\n}\n\n.nav-link:hover, .nav-link:focus {\n  text-decoration: none;\n}\n\n.nav-link.disabled {\n  color: #6c757d;\n  pointer-events: none;\n  cursor: default;\n}\n\n.nav-tabs {\n  border-bottom: 1px solid #dee2e6;\n}\n\n.nav-tabs .nav-item {\n  margin-bottom: -1px;\n}\n\n.nav-tabs .nav-link {\n  border: 1px solid transparent;\n  border-top-left-radius: 0.25rem;\n  border-top-right-radius: 0.25rem;\n}\n\n.nav-tabs .nav-link:hover, .nav-tabs .nav-link:focus {\n  border-color: #e9ecef #e9ecef #dee2e6;\n}\n\n.nav-tabs .nav-link.disabled {\n  color: #6c757d;\n  background-color: transparent;\n  border-color: transparent;\n}\n\n.nav-tabs .nav-link.active,\n.nav-tabs .nav-item.show .nav-link {\n  color: #495057;\n  background-color: #fff;\n  border-color: #dee2e6 #dee2e6 #fff;\n}\n\n.nav-tabs .dropdown-menu {\n  margin-top: -1px;\n  border-top-left-radius: 0;\n  border-top-right-radius: 0;\n}\n\n.nav-pills .nav-link {\n  border-radius: 0.25rem;\n}\n\n.nav-pills .nav-link.active,\n.nav-pills .show > .nav-link {\n  color: #fff;\n  background-color: #007bff;\n}\n\n.nav-fill .nav-item {\n  -ms-flex: 1 1 auto;\n  flex: 1 1 auto;\n  text-align: center;\n}\n\n.nav-justified .nav-item {\n  -ms-flex-preferred-size: 0;\n  flex-basis: 0;\n  -ms-flex-positive: 1;\n  flex-grow: 1;\n  text-align: center;\n}\n\n.tab-content > .tab-pane {\n  display: none;\n}\n\n.tab-content > .active {\n  display: block;\n}\n\n.navbar {\n  position: relative;\n  display: -ms-flexbox;\n  display: flex;\n  -ms-flex-wrap: wrap;\n  flex-wrap: wrap;\n  -ms-flex-align: center;\n  align-items: center;\n  -ms-flex-pack: justify;\n  justify-content: space-between;\n  padding: 0.5rem 1rem;\n}\n\n.navbar .container,\n.navbar .container-fluid, .navbar .container-sm, .navbar .container-md, .navbar .container-lg, .navbar .container-xl {\n  display: -ms-flexbox;\n  display: flex;\n  -ms-flex-wrap: wrap;\n  flex-wrap: wrap;\n  -ms-flex-align: center;\n  align-items: center;\n  -ms-flex-pack: justify;\n  justify-content: space-between;\n}\n\n.navbar-brand {\n  display: inline-block;\n  padding-top: 0.3125rem;\n  padding-bottom: 0.3125rem;\n  margin-right: 1rem;\n  font-size: 1.25rem;\n  line-height: inherit;\n  white-space: nowrap;\n}\n\n.navbar-brand:hover, .navbar-brand:focus {\n  text-decoration: none;\n}\n\n.navbar-nav {\n  display: -ms-flexbox;\n  display: flex;\n  -ms-flex-direction: column;\n  flex-direction: column;\n  padding-left: 0;\n  margin-bottom: 0;\n  list-style: none;\n}\n\n.navbar-nav .nav-link {\n  padding-right: 0;\n  padding-left: 0;\n}\n\n.navbar-nav .dropdown-menu {\n  position: static;\n  float: none;\n}\n\n.navbar-text {\n  display: inline-block;\n  padding-top: 0.5rem;\n  padding-bottom: 0.5rem;\n}\n\n.navbar-collapse {\n  -ms-flex-preferred-size: 100%;\n  flex-basis: 100%;\n  -ms-flex-positive: 1;\n  flex-grow: 1;\n  -ms-flex-align: center;\n  align-items: center;\n}\n\n.navbar-toggler {\n  padding: 0.25rem 0.75rem;\n  font-size: 1.25rem;\n  line-height: 1;\n  background-color: transparent;\n  border: 1px solid transparent;\n  border-radius: 0.25rem;\n}\n\n.navbar-toggler:hover, .navbar-toggler:focus {\n  text-decoration: none;\n}\n\n.navbar-toggler-icon {\n  display: inline-block;\n  width: 1.5em;\n  height: 1.5em;\n  vertical-align: middle;\n  content: \"\";\n  background: no-repeat center center;\n  background-size: 100% 100%;\n}\n\n@media (max-width: 575.98px) {\n  .navbar-expand-sm > .container,\n  .navbar-expand-sm > .container-fluid, .navbar-expand-sm > .container-sm, .navbar-expand-sm > .container-md, .navbar-expand-sm > .container-lg, .navbar-expand-sm > .container-xl {\n    padding-right: 0;\n    padding-left: 0;\n  }\n}\n\n@media (min-width: 576px) {\n  .navbar-expand-sm {\n    -ms-flex-flow: row nowrap;\n    flex-flow: row nowrap;\n    -ms-flex-pack: start;\n    justify-content: flex-start;\n  }\n  .navbar-expand-sm .navbar-nav {\n    -ms-flex-direction: row;\n    flex-direction: row;\n  }\n  .navbar-expand-sm .navbar-nav .dropdown-menu {\n    position: absolute;\n  }\n  .navbar-expand-sm .navbar-nav .nav-link {\n    padding-right: 0.5rem;\n    padding-left: 0.5rem;\n  }\n  .navbar-expand-sm > .container,\n  .navbar-expand-sm > .container-fluid, .navbar-expand-sm > .container-sm, .navbar-expand-sm > .container-md, .navbar-expand-sm > .container-lg, .navbar-expand-sm > .container-xl {\n    -ms-flex-wrap: nowrap;\n    flex-wrap: nowrap;\n  }\n  .navbar-expand-sm .navbar-collapse {\n    display: -ms-flexbox !important;\n    display: flex !important;\n    -ms-flex-preferred-size: auto;\n    flex-basis: auto;\n  }\n  .navbar-expand-sm .navbar-toggler {\n    display: none;\n  }\n}\n\n@media (max-width: 767.98px) {\n  .navbar-expand-md > .container,\n  .navbar-expand-md > .container-fluid, .navbar-expand-md > .container-sm, .navbar-expand-md > .container-md, .navbar-expand-md > .container-lg, .navbar-expand-md > .container-xl {\n    padding-right: 0;\n    padding-left: 0;\n  }\n}\n\n@media (min-width: 768px) {\n  .navbar-expand-md {\n    -ms-flex-flow: row nowrap;\n    flex-flow: row nowrap;\n    -ms-flex-pack: start;\n    justify-content: flex-start;\n  }\n  .navbar-expand-md .navbar-nav {\n    -ms-flex-direction: row;\n    flex-direction: row;\n  }\n  .navbar-expand-md .navbar-nav .dropdown-menu {\n    position: absolute;\n  }\n  .navbar-expand-md .navbar-nav .nav-link {\n    padding-right: 0.5rem;\n    padding-left: 0.5rem;\n  }\n  .navbar-expand-md > .container,\n  .navbar-expand-md > .container-fluid, .navbar-expand-md > .container-sm, .navbar-expand-md > .container-md, .navbar-expand-md > .container-lg, .navbar-expand-md > .container-xl {\n    -ms-flex-wrap: nowrap;\n    flex-wrap: nowrap;\n  }\n  .navbar-expand-md .navbar-collapse {\n    display: -ms-flexbox !important;\n    display: flex !important;\n    -ms-flex-preferred-size: auto;\n    flex-basis: auto;\n  }\n  .navbar-expand-md .navbar-toggler {\n    display: none;\n  }\n}\n\n@media (max-width: 991.98px) {\n  .navbar-expand-lg > .container,\n  .navbar-expand-lg > .container-fluid, .navbar-expand-lg > .container-sm, .navbar-expand-lg > .container-md, .navbar-expand-lg > .container-lg, .navbar-expand-lg > .container-xl {\n    padding-right: 0;\n    padding-left: 0;\n  }\n}\n\n@media (min-width: 992px) {\n  .navbar-expand-lg {\n    -ms-flex-flow: row nowrap;\n    flex-flow: row nowrap;\n    -ms-flex-pack: start;\n    justify-content: flex-start;\n  }\n  .navbar-expand-lg .navbar-nav {\n    -ms-flex-direction: row;\n    flex-direction: row;\n  }\n  .navbar-expand-lg .navbar-nav .dropdown-menu {\n    position: absolute;\n  }\n  .navbar-expand-lg .navbar-nav .nav-link {\n    padding-right: 0.5rem;\n    padding-left: 0.5rem;\n  }\n  .navbar-expand-lg > .container,\n  .navbar-expand-lg > .container-fluid, .navbar-expand-lg > .container-sm, .navbar-expand-lg > .container-md, .navbar-expand-lg > .container-lg, .navbar-expand-lg > .container-xl {\n    -ms-flex-wrap: nowrap;\n    flex-wrap: nowrap;\n  }\n  .navbar-expand-lg .navbar-collapse {\n    display: -ms-flexbox !important;\n    display: flex !important;\n    -ms-flex-preferred-size: auto;\n    flex-basis: auto;\n  }\n  .navbar-expand-lg .navbar-toggler {\n    display: none;\n  }\n}\n\n@media (max-width: 1199.98px) {\n  .navbar-expand-xl > .container,\n  .navbar-expand-xl > .container-fluid, .navbar-expand-xl > .container-sm, .navbar-expand-xl > .container-md, .navbar-expand-xl > .container-lg, .navbar-expand-xl > .container-xl {\n    padding-right: 0;\n    padding-left: 0;\n  }\n}\n\n@media (min-width: 1200px) {\n  .navbar-expand-xl {\n    -ms-flex-flow: row nowrap;\n    flex-flow: row nowrap;\n    -ms-flex-pack: start;\n    justify-content: flex-start;\n  }\n  .navbar-expand-xl .navbar-nav {\n    -ms-flex-direction: row;\n    flex-direction: row;\n  }\n  .navbar-expand-xl .navbar-nav .dropdown-menu {\n    position: absolute;\n  }\n  .navbar-expand-xl .navbar-nav .nav-link {\n    padding-right: 0.5rem;\n    padding-left: 0.5rem;\n  }\n  .navbar-expand-xl > .container,\n  .navbar-expand-xl > .container-fluid, .navbar-expand-xl > .container-sm, .navbar-expand-xl > .container-md, .navbar-expand-xl > .container-lg, .navbar-expand-xl > .container-xl {\n    -ms-flex-wrap: nowrap;\n    flex-wrap: nowrap;\n  }\n  .navbar-expand-xl .navbar-collapse {\n    display: -ms-flexbox !important;\n    display: flex !important;\n    -ms-flex-preferred-size: auto;\n    flex-basis: auto;\n  }\n  .navbar-expand-xl .navbar-toggler {\n    display: none;\n  }\n}\n\n.navbar-expand {\n  -ms-flex-flow: row nowrap;\n  flex-flow: row nowrap;\n  -ms-flex-pack: start;\n  justify-content: flex-start;\n}\n\n.navbar-expand > .container,\n.navbar-expand > .container-fluid, .navbar-expand > .container-sm, .navbar-expand > .container-md, .navbar-expand > .container-lg, .navbar-expand > .container-xl {\n  padding-right: 0;\n  padding-left: 0;\n}\n\n.navbar-expand .navbar-nav {\n  -ms-flex-direction: row;\n  flex-direction: row;\n}\n\n.navbar-expand .navbar-nav .dropdown-menu {\n  position: absolute;\n}\n\n.navbar-expand .navbar-nav .nav-link {\n  padding-right: 0.5rem;\n  padding-left: 0.5rem;\n}\n\n.navbar-expand > .container,\n.navbar-expand > .container-fluid, .navbar-expand > .container-sm, .navbar-expand > .container-md, .navbar-expand > .container-lg, .navbar-expand > .container-xl {\n  -ms-flex-wrap: nowrap;\n  flex-wrap: nowrap;\n}\n\n.navbar-expand .navbar-collapse {\n  display: -ms-flexbox !important;\n  display: flex !important;\n  -ms-flex-preferred-size: auto;\n  flex-basis: auto;\n}\n\n.navbar-expand .navbar-toggler {\n  display: none;\n}\n\n.navbar-light .navbar-brand {\n  color: rgba(0, 0, 0, 0.9);\n}\n\n.navbar-light .navbar-brand:hover, .navbar-light .navbar-brand:focus {\n  color: rgba(0, 0, 0, 0.9);\n}\n\n.navbar-light .navbar-nav .nav-link {\n  color: rgba(0, 0, 0, 0.5);\n}\n\n.navbar-light .navbar-nav .nav-link:hover, .navbar-light .navbar-nav .nav-link:focus {\n  color: rgba(0, 0, 0, 0.7);\n}\n\n.navbar-light .navbar-nav .nav-link.disabled {\n  color: rgba(0, 0, 0, 0.3);\n}\n\n.navbar-light .navbar-nav .show > .nav-link,\n.navbar-light .navbar-nav .active > .nav-link,\n.navbar-light .navbar-nav .nav-link.show,\n.navbar-light .navbar-nav .nav-link.active {\n  color: rgba(0, 0, 0, 0.9);\n}\n\n.navbar-light .navbar-toggler {\n  color: rgba(0, 0, 0, 0.5);\n  border-color: rgba(0, 0, 0, 0.1);\n}\n\n.navbar-light .navbar-toggler-icon {\n  background-image: url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='30' height='30' viewBox='0 0 30 30'%3e%3cpath stroke='rgba(0, 0, 0, 0.5)' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e\");\n}\n\n.navbar-light .navbar-text {\n  color: rgba(0, 0, 0, 0.5);\n}\n\n.navbar-light .navbar-text a {\n  color: rgba(0, 0, 0, 0.9);\n}\n\n.navbar-light .navbar-text a:hover, .navbar-light .navbar-text a:focus {\n  color: rgba(0, 0, 0, 0.9);\n}\n\n.navbar-dark .navbar-brand {\n  color: #fff;\n}\n\n.navbar-dark .navbar-brand:hover, .navbar-dark .navbar-brand:focus {\n  color: #fff;\n}\n\n.navbar-dark .navbar-nav .nav-link {\n  color: rgba(255, 255, 255, 0.5);\n}\n\n.navbar-dark .navbar-nav .nav-link:hover, .navbar-dark .navbar-nav .nav-link:focus {\n  color: rgba(255, 255, 255, 0.75);\n}\n\n.navbar-dark .navbar-nav .nav-link.disabled {\n  color: rgba(255, 255, 255, 0.25);\n}\n\n.navbar-dark .navbar-nav .show > .nav-link,\n.navbar-dark .navbar-nav .active > .nav-link,\n.navbar-dark .navbar-nav .nav-link.show,\n.navbar-dark .navbar-nav .nav-link.active {\n  color: #fff;\n}\n\n.navbar-dark .navbar-toggler {\n  color: rgba(255, 255, 255, 0.5);\n  border-color: rgba(255, 255, 255, 0.1);\n}\n\n.navbar-dark .navbar-toggler-icon {\n  background-image: url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='30' height='30' viewBox='0 0 30 30'%3e%3cpath stroke='rgba(255, 255, 255, 0.5)' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e\");\n}\n\n.navbar-dark .navbar-text {\n  color: rgba(255, 255, 255, 0.5);\n}\n\n.navbar-dark .navbar-text a {\n  color: #fff;\n}\n\n.navbar-dark .navbar-text a:hover, .navbar-dark .navbar-text a:focus {\n  color: #fff;\n}\n\n.card {\n  position: relative;\n  display: -ms-flexbox;\n  display: flex;\n  -ms-flex-direction: column;\n  flex-direction: column;\n  min-width: 0;\n  word-wrap: break-word;\n  background-color: #fff;\n  background-clip: border-box;\n  border: 1px solid rgba(0, 0, 0, 0.125);\n  border-radius: 0.25rem;\n}\n\n.card > hr {\n  margin-right: 0;\n  margin-left: 0;\n}\n\n.card > .list-group:first-child .list-group-item:first-child {\n  border-top-left-radius: 0.25rem;\n  border-top-right-radius: 0.25rem;\n}\n\n.card > .list-group:last-child .list-group-item:last-child {\n  border-bottom-right-radius: 0.25rem;\n  border-bottom-left-radius: 0.25rem;\n}\n\n.card-body {\n  -ms-flex: 1 1 auto;\n  flex: 1 1 auto;\n  min-height: 1px;\n  padding: 1.25rem;\n}\n\n.card-title {\n  margin-bottom: 0.75rem;\n}\n\n.card-subtitle {\n  margin-top: -0.375rem;\n  margin-bottom: 0;\n}\n\n.card-text:last-child {\n  margin-bottom: 0;\n}\n\n.card-link:hover {\n  text-decoration: none;\n}\n\n.card-link + .card-link {\n  margin-left: 1.25rem;\n}\n\n.card-header {\n  padding: 0.75rem 1.25rem;\n  margin-bottom: 0;\n  background-color: rgba(0, 0, 0, 0.03);\n  border-bottom: 1px solid rgba(0, 0, 0, 0.125);\n}\n\n.card-header:first-child {\n  border-radius: calc(0.25rem - 1px) calc(0.25rem - 1px) 0 0;\n}\n\n.card-header + .list-group .list-group-item:first-child {\n  border-top: 0;\n}\n\n.card-footer {\n  padding: 0.75rem 1.25rem;\n  background-color: rgba(0, 0, 0, 0.03);\n  border-top: 1px solid rgba(0, 0, 0, 0.125);\n}\n\n.card-footer:last-child {\n  border-radius: 0 0 calc(0.25rem - 1px) calc(0.25rem - 1px);\n}\n\n.card-header-tabs {\n  margin-right: -0.625rem;\n  margin-bottom: -0.75rem;\n  margin-left: -0.625rem;\n  border-bottom: 0;\n}\n\n.card-header-pills {\n  margin-right: -0.625rem;\n  margin-left: -0.625rem;\n}\n\n.card-img-overlay {\n  position: absolute;\n  top: 0;\n  right: 0;\n  bottom: 0;\n  left: 0;\n  padding: 1.25rem;\n}\n\n.card-img,\n.card-img-top,\n.card-img-bottom {\n  -ms-flex-negative: 0;\n  flex-shrink: 0;\n  width: 100%;\n}\n\n.card-img,\n.card-img-top {\n  border-top-left-radius: calc(0.25rem - 1px);\n  border-top-right-radius: calc(0.25rem - 1px);\n}\n\n.card-img,\n.card-img-bottom {\n  border-bottom-right-radius: calc(0.25rem - 1px);\n  border-bottom-left-radius: calc(0.25rem - 1px);\n}\n\n.card-deck .card {\n  margin-bottom: 15px;\n}\n\n@media (min-width: 576px) {\n  .card-deck {\n    display: -ms-flexbox;\n    display: flex;\n    -ms-flex-flow: row wrap;\n    flex-flow: row wrap;\n    margin-right: -15px;\n    margin-left: -15px;\n  }\n  .card-deck .card {\n    -ms-flex: 1 0 0%;\n    flex: 1 0 0%;\n    margin-right: 15px;\n    margin-bottom: 0;\n    margin-left: 15px;\n  }\n}\n\n.card-group > .card {\n  margin-bottom: 15px;\n}\n\n@media (min-width: 576px) {\n  .card-group {\n    display: -ms-flexbox;\n    display: flex;\n    -ms-flex-flow: row wrap;\n    flex-flow: row wrap;\n  }\n  .card-group > .card {\n    -ms-flex: 1 0 0%;\n    flex: 1 0 0%;\n    margin-bottom: 0;\n  }\n  .card-group > .card + .card {\n    margin-left: 0;\n    border-left: 0;\n  }\n  .card-group > .card:not(:last-child) {\n    border-top-right-radius: 0;\n    border-bottom-right-radius: 0;\n  }\n  .card-group > .card:not(:last-child) .card-img-top,\n  .card-group > .card:not(:last-child) .card-header {\n    border-top-right-radius: 0;\n  }\n  .card-group > .card:not(:last-child) .card-img-bottom,\n  .card-group > .card:not(:last-child) .card-footer {\n    border-bottom-right-radius: 0;\n  }\n  .card-group > .card:not(:first-child) {\n    border-top-left-radius: 0;\n    border-bottom-left-radius: 0;\n  }\n  .card-group > .card:not(:first-child) .card-img-top,\n  .card-group > .card:not(:first-child) .card-header {\n    border-top-left-radius: 0;\n  }\n  .card-group > .card:not(:first-child) .card-img-bottom,\n  .card-group > .card:not(:first-child) .card-footer {\n    border-bottom-left-radius: 0;\n  }\n}\n\n.card-columns .card {\n  margin-bottom: 0.75rem;\n}\n\n@media (min-width: 576px) {\n  .card-columns {\n    -webkit-column-count: 3;\n    -moz-column-count: 3;\n    column-count: 3;\n    -webkit-column-gap: 1.25rem;\n    -moz-column-gap: 1.25rem;\n    column-gap: 1.25rem;\n    orphans: 1;\n    widows: 1;\n  }\n  .card-columns .card {\n    display: inline-block;\n    width: 100%;\n  }\n}\n\n.accordion > .card {\n  overflow: hidden;\n}\n\n.accordion > .card:not(:last-of-type) {\n  border-bottom: 0;\n  border-bottom-right-radius: 0;\n  border-bottom-left-radius: 0;\n}\n\n.accordion > .card:not(:first-of-type) {\n  border-top-left-radius: 0;\n  border-top-right-radius: 0;\n}\n\n.accordion > .card > .card-header {\n  border-radius: 0;\n  margin-bottom: -1px;\n}\n\n.breadcrumb {\n  display: -ms-flexbox;\n  display: flex;\n  -ms-flex-wrap: wrap;\n  flex-wrap: wrap;\n  padding: 0.75rem 1rem;\n  margin-bottom: 1rem;\n  list-style: none;\n  background-color: #e9ecef;\n  border-radius: 0.25rem;\n}\n\n.breadcrumb-item + .breadcrumb-item {\n  padding-left: 0.5rem;\n}\n\n.breadcrumb-item + .breadcrumb-item::before {\n  display: inline-block;\n  padding-right: 0.5rem;\n  color: #6c757d;\n  content: \"/\";\n}\n\n.breadcrumb-item + .breadcrumb-item:hover::before {\n  text-decoration: underline;\n}\n\n.breadcrumb-item + .breadcrumb-item:hover::before {\n  text-decoration: none;\n}\n\n.breadcrumb-item.active {\n  color: #6c757d;\n}\n\n.pagination {\n  display: -ms-flexbox;\n  display: flex;\n  padding-left: 0;\n  list-style: none;\n  border-radius: 0.25rem;\n}\n\n.page-link {\n  position: relative;\n  display: block;\n  padding: 0.5rem 0.75rem;\n  margin-left: -1px;\n  line-height: 1.25;\n  color: #007bff;\n  background-color: #fff;\n  border: 1px solid #dee2e6;\n}\n\n.page-link:hover {\n  z-index: 2;\n  color: #0056b3;\n  text-decoration: none;\n  background-color: #e9ecef;\n  border-color: #dee2e6;\n}\n\n.page-link:focus {\n  z-index: 3;\n  outline: 0;\n  box-shadow: 0 0 0 0.2rem rgba(0, 123, 255, 0.25);\n}\n\n.page-item:first-child .page-link {\n  margin-left: 0;\n  border-top-left-radius: 0.25rem;\n  border-bottom-left-radius: 0.25rem;\n}\n\n.page-item:last-child .page-link {\n  border-top-right-radius: 0.25rem;\n  border-bottom-right-radius: 0.25rem;\n}\n\n.page-item.active .page-link {\n  z-index: 3;\n  color: #fff;\n  background-color: #007bff;\n  border-color: #007bff;\n}\n\n.page-item.disabled .page-link {\n  color: #6c757d;\n  pointer-events: none;\n  cursor: auto;\n  background-color: #fff;\n  border-color: #dee2e6;\n}\n\n.pagination-lg .page-link {\n  padding: 0.75rem 1.5rem;\n  font-size: 1.25rem;\n  line-height: 1.5;\n}\n\n.pagination-lg .page-item:first-child .page-link {\n  border-top-left-radius: 0.3rem;\n  border-bottom-left-radius: 0.3rem;\n}\n\n.pagination-lg .page-item:last-child .page-link {\n  border-top-right-radius: 0.3rem;\n  border-bottom-right-radius: 0.3rem;\n}\n\n.pagination-sm .page-link {\n  padding: 0.25rem 0.5rem;\n  font-size: 0.875rem;\n  line-height: 1.5;\n}\n\n.pagination-sm .page-item:first-child .page-link {\n  border-top-left-radius: 0.2rem;\n  border-bottom-left-radius: 0.2rem;\n}\n\n.pagination-sm .page-item:last-child .page-link {\n  border-top-right-radius: 0.2rem;\n  border-bottom-right-radius: 0.2rem;\n}\n\n.badge {\n  display: inline-block;\n  padding: 0.25em 0.4em;\n  font-size: 75%;\n  font-weight: 700;\n  line-height: 1;\n  text-align: center;\n  white-space: nowrap;\n  vertical-align: baseline;\n  border-radius: 0.25rem;\n  transition: color 0.15s ease-in-out, background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .badge {\n    transition: none;\n  }\n}\n\na.badge:hover, a.badge:focus {\n  text-decoration: none;\n}\n\n.badge:empty {\n  display: none;\n}\n\n.btn .badge {\n  position: relative;\n  top: -1px;\n}\n\n.badge-pill {\n  padding-right: 0.6em;\n  padding-left: 0.6em;\n  border-radius: 10rem;\n}\n\n.badge-primary {\n  color: #fff;\n  background-color: #007bff;\n}\n\na.badge-primary:hover, a.badge-primary:focus {\n  color: #fff;\n  background-color: #0062cc;\n}\n\na.badge-primary:focus, a.badge-primary.focus {\n  outline: 0;\n  box-shadow: 0 0 0 0.2rem rgba(0, 123, 255, 0.5);\n}\n\n.badge-secondary {\n  color: #fff;\n  background-color: #6c757d;\n}\n\na.badge-secondary:hover, a.badge-secondary:focus {\n  color: #fff;\n  background-color: #545b62;\n}\n\na.badge-secondary:focus, a.badge-secondary.focus {\n  outline: 0;\n  box-shadow: 0 0 0 0.2rem rgba(108, 117, 125, 0.5);\n}\n\n.badge-success {\n  color: #fff;\n  background-color: #28a745;\n}\n\na.badge-success:hover, a.badge-success:focus {\n  color: #fff;\n  background-color: #1e7e34;\n}\n\na.badge-success:focus, a.badge-success.focus {\n  outline: 0;\n  box-shadow: 0 0 0 0.2rem rgba(40, 167, 69, 0.5);\n}\n\n.badge-info {\n  color: #fff;\n  background-color: #17a2b8;\n}\n\na.badge-info:hover, a.badge-info:focus {\n  color: #fff;\n  background-color: #117a8b;\n}\n\na.badge-info:focus, a.badge-info.focus {\n  outline: 0;\n  box-shadow: 0 0 0 0.2rem rgba(23, 162, 184, 0.5);\n}\n\n.badge-warning {\n  color: #212529;\n  background-color: #ffc107;\n}\n\na.badge-warning:hover, a.badge-warning:focus {\n  color: #212529;\n  background-color: #d39e00;\n}\n\na.badge-warning:focus, a.badge-warning.focus {\n  outline: 0;\n  box-shadow: 0 0 0 0.2rem rgba(255, 193, 7, 0.5);\n}\n\n.badge-danger {\n  color: #fff;\n  background-color: #dc3545;\n}\n\na.badge-danger:hover, a.badge-danger:focus {\n  color: #fff;\n  background-color: #bd2130;\n}\n\na.badge-danger:focus, a.badge-danger.focus {\n  outline: 0;\n  box-shadow: 0 0 0 0.2rem rgba(220, 53, 69, 0.5);\n}\n\n.badge-light {\n  color: #212529;\n  background-color: #f8f9fa;\n}\n\na.badge-light:hover, a.badge-light:focus {\n  color: #212529;\n  background-color: #dae0e5;\n}\n\na.badge-light:focus, a.badge-light.focus {\n  outline: 0;\n  box-shadow: 0 0 0 0.2rem rgba(248, 249, 250, 0.5);\n}\n\n.badge-dark {\n  color: #fff;\n  background-color: #343a40;\n}\n\na.badge-dark:hover, a.badge-dark:focus {\n  color: #fff;\n  background-color: #1d2124;\n}\n\na.badge-dark:focus, a.badge-dark.focus {\n  outline: 0;\n  box-shadow: 0 0 0 0.2rem rgba(52, 58, 64, 0.5);\n}\n\n.jumbotron {\n  padding: 2rem 1rem;\n  margin-bottom: 2rem;\n  background-color: #e9ecef;\n  border-radius: 0.3rem;\n}\n\n@media (min-width: 576px) {\n  .jumbotron {\n    padding: 4rem 2rem;\n  }\n}\n\n.jumbotron-fluid {\n  padding-right: 0;\n  padding-left: 0;\n  border-radius: 0;\n}\n\n.alert {\n  position: relative;\n  padding: 0.75rem 1.25rem;\n  margin-bottom: 1rem;\n  border: 1px solid transparent;\n  border-radius: 0.25rem;\n}\n\n.alert-heading {\n  color: inherit;\n}\n\n.alert-link {\n  font-weight: 700;\n}\n\n.alert-dismissible {\n  padding-right: 4rem;\n}\n\n.alert-dismissible .close {\n  position: absolute;\n  top: 0;\n  right: 0;\n  padding: 0.75rem 1.25rem;\n  color: inherit;\n}\n\n.alert-primary {\n  color: #004085;\n  background-color: #cce5ff;\n  border-color: #b8daff;\n}\n\n.alert-primary hr {\n  border-top-color: #9fcdff;\n}\n\n.alert-primary .alert-link {\n  color: #002752;\n}\n\n.alert-secondary {\n  color: #383d41;\n  background-color: #e2e3e5;\n  border-color: #d6d8db;\n}\n\n.alert-secondary hr {\n  border-top-color: #c8cbcf;\n}\n\n.alert-secondary .alert-link {\n  color: #202326;\n}\n\n.alert-success {\n  color: #155724;\n  background-color: #d4edda;\n  border-color: #c3e6cb;\n}\n\n.alert-success hr {\n  border-top-color: #b1dfbb;\n}\n\n.alert-success .alert-link {\n  color: #0b2e13;\n}\n\n.alert-info {\n  color: #0c5460;\n  background-color: #d1ecf1;\n  border-color: #bee5eb;\n}\n\n.alert-info hr {\n  border-top-color: #abdde5;\n}\n\n.alert-info .alert-link {\n  color: #062c33;\n}\n\n.alert-warning {\n  color: #856404;\n  background-color: #fff3cd;\n  border-color: #ffeeba;\n}\n\n.alert-warning hr {\n  border-top-color: #ffe8a1;\n}\n\n.alert-warning .alert-link {\n  color: #533f03;\n}\n\n.alert-danger {\n  color: #721c24;\n  background-color: #f8d7da;\n  border-color: #f5c6cb;\n}\n\n.alert-danger hr {\n  border-top-color: #f1b0b7;\n}\n\n.alert-danger .alert-link {\n  color: #491217;\n}\n\n.alert-light {\n  color: #818182;\n  background-color: #fefefe;\n  border-color: #fdfdfe;\n}\n\n.alert-light hr {\n  border-top-color: #ececf6;\n}\n\n.alert-light .alert-link {\n  color: #686868;\n}\n\n.alert-dark {\n  color: #1b1e21;\n  background-color: #d6d8d9;\n  border-color: #c6c8ca;\n}\n\n.alert-dark hr {\n  border-top-color: #b9bbbe;\n}\n\n.alert-dark .alert-link {\n  color: #040505;\n}\n\n@-webkit-keyframes progress-bar-stripes {\n  from {\n    background-position: 1rem 0;\n  }\n  to {\n    background-position: 0 0;\n  }\n}\n\n@keyframes progress-bar-stripes {\n  from {\n    background-position: 1rem 0;\n  }\n  to {\n    background-position: 0 0;\n  }\n}\n\n.progress {\n  display: -ms-flexbox;\n  display: flex;\n  height: 1rem;\n  overflow: hidden;\n  font-size: 0.75rem;\n  background-color: #e9ecef;\n  border-radius: 0.25rem;\n}\n\n.progress-bar {\n  display: -ms-flexbox;\n  display: flex;\n  -ms-flex-direction: column;\n  flex-direction: column;\n  -ms-flex-pack: center;\n  justify-content: center;\n  overflow: hidden;\n  color: #fff;\n  text-align: center;\n  white-space: nowrap;\n  background-color: #007bff;\n  transition: width 0.6s ease;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .progress-bar {\n    transition: none;\n  }\n}\n\n.progress-bar-striped {\n  background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);\n  background-size: 1rem 1rem;\n}\n\n.progress-bar-animated {\n  -webkit-animation: progress-bar-stripes 1s linear infinite;\n  animation: progress-bar-stripes 1s linear infinite;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .progress-bar-animated {\n    -webkit-animation: none;\n    animation: none;\n  }\n}\n\n.media {\n  display: -ms-flexbox;\n  display: flex;\n  -ms-flex-align: start;\n  align-items: flex-start;\n}\n\n.media-body {\n  -ms-flex: 1;\n  flex: 1;\n}\n\n.list-group {\n  display: -ms-flexbox;\n  display: flex;\n  -ms-flex-direction: column;\n  flex-direction: column;\n  padding-left: 0;\n  margin-bottom: 0;\n}\n\n.list-group-item-action {\n  width: 100%;\n  color: #495057;\n  text-align: inherit;\n}\n\n.list-group-item-action:hover, .list-group-item-action:focus {\n  z-index: 1;\n  color: #495057;\n  text-decoration: none;\n  background-color: #f8f9fa;\n}\n\n.list-group-item-action:active {\n  color: #212529;\n  background-color: #e9ecef;\n}\n\n.list-group-item {\n  position: relative;\n  display: block;\n  padding: 0.75rem 1.25rem;\n  background-color: #fff;\n  border: 1px solid rgba(0, 0, 0, 0.125);\n}\n\n.list-group-item:first-child {\n  border-top-left-radius: 0.25rem;\n  border-top-right-radius: 0.25rem;\n}\n\n.list-group-item:last-child {\n  border-bottom-right-radius: 0.25rem;\n  border-bottom-left-radius: 0.25rem;\n}\n\n.list-group-item.disabled, .list-group-item:disabled {\n  color: #6c757d;\n  pointer-events: none;\n  background-color: #fff;\n}\n\n.list-group-item.active {\n  z-index: 2;\n  color: #fff;\n  background-color: #007bff;\n  border-color: #007bff;\n}\n\n.list-group-item + .list-group-item {\n  border-top-width: 0;\n}\n\n.list-group-item + .list-group-item.active {\n  margin-top: -1px;\n  border-top-width: 1px;\n}\n\n.list-group-horizontal {\n  -ms-flex-direction: row;\n  flex-direction: row;\n}\n\n.list-group-horizontal .list-group-item:first-child {\n  border-bottom-left-radius: 0.25rem;\n  border-top-right-radius: 0;\n}\n\n.list-group-horizontal .list-group-item:last-child {\n  border-top-right-radius: 0.25rem;\n  border-bottom-left-radius: 0;\n}\n\n.list-group-horizontal .list-group-item.active {\n  margin-top: 0;\n}\n\n.list-group-horizontal .list-group-item + .list-group-item {\n  border-top-width: 1px;\n  border-left-width: 0;\n}\n\n.list-group-horizontal .list-group-item + .list-group-item.active {\n  margin-left: -1px;\n  border-left-width: 1px;\n}\n\n@media (min-width: 576px) {\n  .list-group-horizontal-sm {\n    -ms-flex-direction: row;\n    flex-direction: row;\n  }\n  .list-group-horizontal-sm .list-group-item:first-child {\n    border-bottom-left-radius: 0.25rem;\n    border-top-right-radius: 0;\n  }\n  .list-group-horizontal-sm .list-group-item:last-child {\n    border-top-right-radius: 0.25rem;\n    border-bottom-left-radius: 0;\n  }\n  .list-group-horizontal-sm .list-group-item.active {\n    margin-top: 0;\n  }\n  .list-group-horizontal-sm .list-group-item + .list-group-item {\n    border-top-width: 1px;\n    border-left-width: 0;\n  }\n  .list-group-horizontal-sm .list-group-item + .list-group-item.active {\n    margin-left: -1px;\n    border-left-width: 1px;\n  }\n}\n\n@media (min-width: 768px) {\n  .list-group-horizontal-md {\n    -ms-flex-direction: row;\n    flex-direction: row;\n  }\n  .list-group-horizontal-md .list-group-item:first-child {\n    border-bottom-left-radius: 0.25rem;\n    border-top-right-radius: 0;\n  }\n  .list-group-horizontal-md .list-group-item:last-child {\n    border-top-right-radius: 0.25rem;\n    border-bottom-left-radius: 0;\n  }\n  .list-group-horizontal-md .list-group-item.active {\n    margin-top: 0;\n  }\n  .list-group-horizontal-md .list-group-item + .list-group-item {\n    border-top-width: 1px;\n    border-left-width: 0;\n  }\n  .list-group-horizontal-md .list-group-item + .list-group-item.active {\n    margin-left: -1px;\n    border-left-width: 1px;\n  }\n}\n\n@media (min-width: 992px) {\n  .list-group-horizontal-lg {\n    -ms-flex-direction: row;\n    flex-direction: row;\n  }\n  .list-group-horizontal-lg .list-group-item:first-child {\n    border-bottom-left-radius: 0.25rem;\n    border-top-right-radius: 0;\n  }\n  .list-group-horizontal-lg .list-group-item:last-child {\n    border-top-right-radius: 0.25rem;\n    border-bottom-left-radius: 0;\n  }\n  .list-group-horizontal-lg .list-group-item.active {\n    margin-top: 0;\n  }\n  .list-group-horizontal-lg .list-group-item + .list-group-item {\n    border-top-width: 1px;\n    border-left-width: 0;\n  }\n  .list-group-horizontal-lg .list-group-item + .list-group-item.active {\n    margin-left: -1px;\n    border-left-width: 1px;\n  }\n}\n\n@media (min-width: 1200px) {\n  .list-group-horizontal-xl {\n    -ms-flex-direction: row;\n    flex-direction: row;\n  }\n  .list-group-horizontal-xl .list-group-item:first-child {\n    border-bottom-left-radius: 0.25rem;\n    border-top-right-radius: 0;\n  }\n  .list-group-horizontal-xl .list-group-item:last-child {\n    border-top-right-radius: 0.25rem;\n    border-bottom-left-radius: 0;\n  }\n  .list-group-horizontal-xl .list-group-item.active {\n    margin-top: 0;\n  }\n  .list-group-horizontal-xl .list-group-item + .list-group-item {\n    border-top-width: 1px;\n    border-left-width: 0;\n  }\n  .list-group-horizontal-xl .list-group-item + .list-group-item.active {\n    margin-left: -1px;\n    border-left-width: 1px;\n  }\n}\n\n.list-group-flush .list-group-item {\n  border-right-width: 0;\n  border-left-width: 0;\n  border-radius: 0;\n}\n\n.list-group-flush .list-group-item:first-child {\n  border-top-width: 0;\n}\n\n.list-group-flush:last-child .list-group-item:last-child {\n  border-bottom-width: 0;\n}\n\n.list-group-item-primary {\n  color: #004085;\n  background-color: #b8daff;\n}\n\n.list-group-item-primary.list-group-item-action:hover, .list-group-item-primary.list-group-item-action:focus {\n  color: #004085;\n  background-color: #9fcdff;\n}\n\n.list-group-item-primary.list-group-item-action.active {\n  color: #fff;\n  background-color: #004085;\n  border-color: #004085;\n}\n\n.list-group-item-secondary {\n  color: #383d41;\n  background-color: #d6d8db;\n}\n\n.list-group-item-secondary.list-group-item-action:hover, .list-group-item-secondary.list-group-item-action:focus {\n  color: #383d41;\n  background-color: #c8cbcf;\n}\n\n.list-group-item-secondary.list-group-item-action.active {\n  color: #fff;\n  background-color: #383d41;\n  border-color: #383d41;\n}\n\n.list-group-item-success {\n  color: #155724;\n  background-color: #c3e6cb;\n}\n\n.list-group-item-success.list-group-item-action:hover, .list-group-item-success.list-group-item-action:focus {\n  color: #155724;\n  background-color: #b1dfbb;\n}\n\n.list-group-item-success.list-group-item-action.active {\n  color: #fff;\n  background-color: #155724;\n  border-color: #155724;\n}\n\n.list-group-item-info {\n  color: #0c5460;\n  background-color: #bee5eb;\n}\n\n.list-group-item-info.list-group-item-action:hover, .list-group-item-info.list-group-item-action:focus {\n  color: #0c5460;\n  background-color: #abdde5;\n}\n\n.list-group-item-info.list-group-item-action.active {\n  color: #fff;\n  background-color: #0c5460;\n  border-color: #0c5460;\n}\n\n.list-group-item-warning {\n  color: #856404;\n  background-color: #ffeeba;\n}\n\n.list-group-item-warning.list-group-item-action:hover, .list-group-item-warning.list-group-item-action:focus {\n  color: #856404;\n  background-color: #ffe8a1;\n}\n\n.list-group-item-warning.list-group-item-action.active {\n  color: #fff;\n  background-color: #856404;\n  border-color: #856404;\n}\n\n.list-group-item-danger {\n  color: #721c24;\n  background-color: #f5c6cb;\n}\n\n.list-group-item-danger.list-group-item-action:hover, .list-group-item-danger.list-group-item-action:focus {\n  color: #721c24;\n  background-color: #f1b0b7;\n}\n\n.list-group-item-danger.list-group-item-action.active {\n  color: #fff;\n  background-color: #721c24;\n  border-color: #721c24;\n}\n\n.list-group-item-light {\n  color: #818182;\n  background-color: #fdfdfe;\n}\n\n.list-group-item-light.list-group-item-action:hover, .list-group-item-light.list-group-item-action:focus {\n  color: #818182;\n  background-color: #ececf6;\n}\n\n.list-group-item-light.list-group-item-action.active {\n  color: #fff;\n  background-color: #818182;\n  border-color: #818182;\n}\n\n.list-group-item-dark {\n  color: #1b1e21;\n  background-color: #c6c8ca;\n}\n\n.list-group-item-dark.list-group-item-action:hover, .list-group-item-dark.list-group-item-action:focus {\n  color: #1b1e21;\n  background-color: #b9bbbe;\n}\n\n.list-group-item-dark.list-group-item-action.active {\n  color: #fff;\n  background-color: #1b1e21;\n  border-color: #1b1e21;\n}\n\n.close {\n  float: right;\n  font-size: 1.5rem;\n  font-weight: 700;\n  line-height: 1;\n  color: #000;\n  text-shadow: 0 1px 0 #fff;\n  opacity: .5;\n}\n\n.close:hover {\n  color: #000;\n  text-decoration: none;\n}\n\n.close:not(:disabled):not(.disabled):hover, .close:not(:disabled):not(.disabled):focus {\n  opacity: .75;\n}\n\nbutton.close {\n  padding: 0;\n  background-color: transparent;\n  border: 0;\n  -webkit-appearance: none;\n  -moz-appearance: none;\n  appearance: none;\n}\n\na.close.disabled {\n  pointer-events: none;\n}\n\n.toast {\n  max-width: 350px;\n  overflow: hidden;\n  font-size: 0.875rem;\n  background-color: rgba(255, 255, 255, 0.85);\n  background-clip: padding-box;\n  border: 1px solid rgba(0, 0, 0, 0.1);\n  box-shadow: 0 0.25rem 0.75rem rgba(0, 0, 0, 0.1);\n  -webkit-backdrop-filter: blur(10px);\n  backdrop-filter: blur(10px);\n  opacity: 0;\n  border-radius: 0.25rem;\n}\n\n.toast:not(:last-child) {\n  margin-bottom: 0.75rem;\n}\n\n.toast.showing {\n  opacity: 1;\n}\n\n.toast.show {\n  display: block;\n  opacity: 1;\n}\n\n.toast.hide {\n  display: none;\n}\n\n.toast-header {\n  display: -ms-flexbox;\n  display: flex;\n  -ms-flex-align: center;\n  align-items: center;\n  padding: 0.25rem 0.75rem;\n  color: #6c757d;\n  background-color: rgba(255, 255, 255, 0.85);\n  background-clip: padding-box;\n  border-bottom: 1px solid rgba(0, 0, 0, 0.05);\n}\n\n.toast-body {\n  padding: 0.75rem;\n}\n\n.modal-open {\n  overflow: hidden;\n}\n\n.modal-open .modal {\n  overflow-x: hidden;\n  overflow-y: auto;\n}\n\n.modal {\n  position: fixed;\n  top: 0;\n  left: 0;\n  z-index: 1050;\n  display: none;\n  width: 100%;\n  height: 100%;\n  overflow: hidden;\n  outline: 0;\n}\n\n.modal-dialog {\n  position: relative;\n  width: auto;\n  margin: 0.5rem;\n  pointer-events: none;\n}\n\n.modal.fade .modal-dialog {\n  transition: -webkit-transform 0.3s ease-out;\n  transition: transform 0.3s ease-out;\n  transition: transform 0.3s ease-out, -webkit-transform 0.3s ease-out;\n  -webkit-transform: translate(0, -50px);\n  transform: translate(0, -50px);\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .modal.fade .modal-dialog {\n    transition: none;\n  }\n}\n\n.modal.show .modal-dialog {\n  -webkit-transform: none;\n  transform: none;\n}\n\n.modal.modal-static .modal-dialog {\n  -webkit-transform: scale(1.02);\n  transform: scale(1.02);\n}\n\n.modal-dialog-scrollable {\n  display: -ms-flexbox;\n  display: flex;\n  max-height: calc(100% - 1rem);\n}\n\n.modal-dialog-scrollable .modal-content {\n  max-height: calc(100vh - 1rem);\n  overflow: hidden;\n}\n\n.modal-dialog-scrollable .modal-header,\n.modal-dialog-scrollable .modal-footer {\n  -ms-flex-negative: 0;\n  flex-shrink: 0;\n}\n\n.modal-dialog-scrollable .modal-body {\n  overflow-y: auto;\n}\n\n.modal-dialog-centered {\n  display: -ms-flexbox;\n  display: flex;\n  -ms-flex-align: center;\n  align-items: center;\n  min-height: calc(100% - 1rem);\n}\n\n.modal-dialog-centered::before {\n  display: block;\n  height: calc(100vh - 1rem);\n  content: \"\";\n}\n\n.modal-dialog-centered.modal-dialog-scrollable {\n  -ms-flex-direction: column;\n  flex-direction: column;\n  -ms-flex-pack: center;\n  justify-content: center;\n  height: 100%;\n}\n\n.modal-dialog-centered.modal-dialog-scrollable .modal-content {\n  max-height: none;\n}\n\n.modal-dialog-centered.modal-dialog-scrollable::before {\n  content: none;\n}\n\n.modal-content {\n  position: relative;\n  display: -ms-flexbox;\n  display: flex;\n  -ms-flex-direction: column;\n  flex-direction: column;\n  width: 100%;\n  pointer-events: auto;\n  background-color: #fff;\n  background-clip: padding-box;\n  border: 1px solid rgba(0, 0, 0, 0.2);\n  border-radius: 0.3rem;\n  outline: 0;\n}\n\n.modal-backdrop {\n  position: fixed;\n  top: 0;\n  left: 0;\n  z-index: 1040;\n  width: 100vw;\n  height: 100vh;\n  background-color: #000;\n}\n\n.modal-backdrop.fade {\n  opacity: 0;\n}\n\n.modal-backdrop.show {\n  opacity: 0.5;\n}\n\n.modal-header {\n  display: -ms-flexbox;\n  display: flex;\n  -ms-flex-align: start;\n  align-items: flex-start;\n  -ms-flex-pack: justify;\n  justify-content: space-between;\n  padding: 1rem 1rem;\n  border-bottom: 1px solid #dee2e6;\n  border-top-left-radius: calc(0.3rem - 1px);\n  border-top-right-radius: calc(0.3rem - 1px);\n}\n\n.modal-header .close {\n  padding: 1rem 1rem;\n  margin: -1rem -1rem -1rem auto;\n}\n\n.modal-title {\n  margin-bottom: 0;\n  line-height: 1.5;\n}\n\n.modal-body {\n  position: relative;\n  -ms-flex: 1 1 auto;\n  flex: 1 1 auto;\n  padding: 1rem;\n}\n\n.modal-footer {\n  display: -ms-flexbox;\n  display: flex;\n  -ms-flex-wrap: wrap;\n  flex-wrap: wrap;\n  -ms-flex-align: center;\n  align-items: center;\n  -ms-flex-pack: end;\n  justify-content: flex-end;\n  padding: 0.75rem;\n  border-top: 1px solid #dee2e6;\n  border-bottom-right-radius: calc(0.3rem - 1px);\n  border-bottom-left-radius: calc(0.3rem - 1px);\n}\n\n.modal-footer > * {\n  margin: 0.25rem;\n}\n\n.modal-scrollbar-measure {\n  position: absolute;\n  top: -9999px;\n  width: 50px;\n  height: 50px;\n  overflow: scroll;\n}\n\n@media (min-width: 576px) {\n  .modal-dialog {\n    max-width: 500px;\n    margin: 1.75rem auto;\n  }\n  .modal-dialog-scrollable {\n    max-height: calc(100% - 3.5rem);\n  }\n  .modal-dialog-scrollable .modal-content {\n    max-height: calc(100vh - 3.5rem);\n  }\n  .modal-dialog-centered {\n    min-height: calc(100% - 3.5rem);\n  }\n  .modal-dialog-centered::before {\n    height: calc(100vh - 3.5rem);\n  }\n  .modal-sm {\n    max-width: 300px;\n  }\n}\n\n@media (min-width: 992px) {\n  .modal-lg,\n  .modal-xl {\n    max-width: 800px;\n  }\n}\n\n@media (min-width: 1200px) {\n  .modal-xl {\n    max-width: 1140px;\n  }\n}\n\n.tooltip {\n  position: absolute;\n  z-index: 1070;\n  display: block;\n  margin: 0;\n  font-family: -apple-system, BlinkMacSystemFont, \"Segoe UI\", Roboto, \"Helvetica Neue\", Arial, \"Noto Sans\", sans-serif, \"Apple Color Emoji\", \"Segoe UI Emoji\", \"Segoe UI Symbol\", \"Noto Color Emoji\";\n  font-style: normal;\n  font-weight: 400;\n  line-height: 1.5;\n  text-align: left;\n  text-align: start;\n  text-decoration: none;\n  text-shadow: none;\n  text-transform: none;\n  letter-spacing: normal;\n  word-break: normal;\n  word-spacing: normal;\n  white-space: normal;\n  line-break: auto;\n  font-size: 0.875rem;\n  word-wrap: break-word;\n  opacity: 0;\n}\n\n.tooltip.show {\n  opacity: 0.9;\n}\n\n.tooltip .arrow {\n  position: absolute;\n  display: block;\n  width: 0.8rem;\n  height: 0.4rem;\n}\n\n.tooltip .arrow::before {\n  position: absolute;\n  content: \"\";\n  border-color: transparent;\n  border-style: solid;\n}\n\n.bs-tooltip-top, .bs-tooltip-auto[x-placement^=\"top\"] {\n  padding: 0.4rem 0;\n}\n\n.bs-tooltip-top .arrow, .bs-tooltip-auto[x-placement^=\"top\"] .arrow {\n  bottom: 0;\n}\n\n.bs-tooltip-top .arrow::before, .bs-tooltip-auto[x-placement^=\"top\"] .arrow::before {\n  top: 0;\n  border-width: 0.4rem 0.4rem 0;\n  border-top-color: #000;\n}\n\n.bs-tooltip-right, .bs-tooltip-auto[x-placement^=\"right\"] {\n  padding: 0 0.4rem;\n}\n\n.bs-tooltip-right .arrow, .bs-tooltip-auto[x-placement^=\"right\"] .arrow {\n  left: 0;\n  width: 0.4rem;\n  height: 0.8rem;\n}\n\n.bs-tooltip-right .arrow::before, .bs-tooltip-auto[x-placement^=\"right\"] .arrow::before {\n  right: 0;\n  border-width: 0.4rem 0.4rem 0.4rem 0;\n  border-right-color: #000;\n}\n\n.bs-tooltip-bottom, .bs-tooltip-auto[x-placement^=\"bottom\"] {\n  padding: 0.4rem 0;\n}\n\n.bs-tooltip-bottom .arrow, .bs-tooltip-auto[x-placement^=\"bottom\"] .arrow {\n  top: 0;\n}\n\n.bs-tooltip-bottom .arrow::before, .bs-tooltip-auto[x-placement^=\"bottom\"] .arrow::before {\n  bottom: 0;\n  border-width: 0 0.4rem 0.4rem;\n  border-bottom-color: #000;\n}\n\n.bs-tooltip-left, .bs-tooltip-auto[x-placement^=\"left\"] {\n  padding: 0 0.4rem;\n}\n\n.bs-tooltip-left .arrow, .bs-tooltip-auto[x-placement^=\"left\"] .arrow {\n  right: 0;\n  width: 0.4rem;\n  height: 0.8rem;\n}\n\n.bs-tooltip-left .arrow::before, .bs-tooltip-auto[x-placement^=\"left\"] .arrow::before {\n  left: 0;\n  border-width: 0.4rem 0 0.4rem 0.4rem;\n  border-left-color: #000;\n}\n\n.tooltip-inner {\n  max-width: 200px;\n  padding: 0.25rem 0.5rem;\n  color: #fff;\n  text-align: center;\n  background-color: #000;\n  border-radius: 0.25rem;\n}\n\n.popover {\n  position: absolute;\n  top: 0;\n  left: 0;\n  z-index: 1060;\n  display: block;\n  max-width: 276px;\n  font-family: -apple-system, BlinkMacSystemFont, \"Segoe UI\", Roboto, \"Helvetica Neue\", Arial, \"Noto Sans\", sans-serif, \"Apple Color Emoji\", \"Segoe UI Emoji\", \"Segoe UI Symbol\", \"Noto Color Emoji\";\n  font-style: normal;\n  font-weight: 400;\n  line-height: 1.5;\n  text-align: left;\n  text-align: start;\n  text-decoration: none;\n  text-shadow: none;\n  text-transform: none;\n  letter-spacing: normal;\n  word-break: normal;\n  word-spacing: normal;\n  white-space: normal;\n  line-break: auto;\n  font-size: 0.875rem;\n  word-wrap: break-word;\n  background-color: #fff;\n  background-clip: padding-box;\n  border: 1px solid rgba(0, 0, 0, 0.2);\n  border-radius: 0.3rem;\n}\n\n.popover .arrow {\n  position: absolute;\n  display: block;\n  width: 1rem;\n  height: 0.5rem;\n  margin: 0 0.3rem;\n}\n\n.popover .arrow::before, .popover .arrow::after {\n  position: absolute;\n  display: block;\n  content: \"\";\n  border-color: transparent;\n  border-style: solid;\n}\n\n.bs-popover-top, .bs-popover-auto[x-placement^=\"top\"] {\n  margin-bottom: 0.5rem;\n}\n\n.bs-popover-top > .arrow, .bs-popover-auto[x-placement^=\"top\"] > .arrow {\n  bottom: calc(-0.5rem - 1px);\n}\n\n.bs-popover-top > .arrow::before, .bs-popover-auto[x-placement^=\"top\"] > .arrow::before {\n  bottom: 0;\n  border-width: 0.5rem 0.5rem 0;\n  border-top-color: rgba(0, 0, 0, 0.25);\n}\n\n.bs-popover-top > .arrow::after, .bs-popover-auto[x-placement^=\"top\"] > .arrow::after {\n  bottom: 1px;\n  border-width: 0.5rem 0.5rem 0;\n  border-top-color: #fff;\n}\n\n.bs-popover-right, .bs-popover-auto[x-placement^=\"right\"] {\n  margin-left: 0.5rem;\n}\n\n.bs-popover-right > .arrow, .bs-popover-auto[x-placement^=\"right\"] > .arrow {\n  left: calc(-0.5rem - 1px);\n  width: 0.5rem;\n  height: 1rem;\n  margin: 0.3rem 0;\n}\n\n.bs-popover-right > .arrow::before, .bs-popover-auto[x-placement^=\"right\"] > .arrow::before {\n  left: 0;\n  border-width: 0.5rem 0.5rem 0.5rem 0;\n  border-right-color: rgba(0, 0, 0, 0.25);\n}\n\n.bs-popover-right > .arrow::after, .bs-popover-auto[x-placement^=\"right\"] > .arrow::after {\n  left: 1px;\n  border-width: 0.5rem 0.5rem 0.5rem 0;\n  border-right-color: #fff;\n}\n\n.bs-popover-bottom, .bs-popover-auto[x-placement^=\"bottom\"] {\n  margin-top: 0.5rem;\n}\n\n.bs-popover-bottom > .arrow, .bs-popover-auto[x-placement^=\"bottom\"] > .arrow {\n  top: calc(-0.5rem - 1px);\n}\n\n.bs-popover-bottom > .arrow::before, .bs-popover-auto[x-placement^=\"bottom\"] > .arrow::before {\n  top: 0;\n  border-width: 0 0.5rem 0.5rem 0.5rem;\n  border-bottom-color: rgba(0, 0, 0, 0.25);\n}\n\n.bs-popover-bottom > .arrow::after, .bs-popover-auto[x-placement^=\"bottom\"] > .arrow::after {\n  top: 1px;\n  border-width: 0 0.5rem 0.5rem 0.5rem;\n  border-bottom-color: #fff;\n}\n\n.bs-popover-bottom .popover-header::before, .bs-popover-auto[x-placement^=\"bottom\"] .popover-header::before {\n  position: absolute;\n  top: 0;\n  left: 50%;\n  display: block;\n  width: 1rem;\n  margin-left: -0.5rem;\n  content: \"\";\n  border-bottom: 1px solid #f7f7f7;\n}\n\n.bs-popover-left, .bs-popover-auto[x-placement^=\"left\"] {\n  margin-right: 0.5rem;\n}\n\n.bs-popover-left > .arrow, .bs-popover-auto[x-placement^=\"left\"] > .arrow {\n  right: calc(-0.5rem - 1px);\n  width: 0.5rem;\n  height: 1rem;\n  margin: 0.3rem 0;\n}\n\n.bs-popover-left > .arrow::before, .bs-popover-auto[x-placement^=\"left\"] > .arrow::before {\n  right: 0;\n  border-width: 0.5rem 0 0.5rem 0.5rem;\n  border-left-color: rgba(0, 0, 0, 0.25);\n}\n\n.bs-popover-left > .arrow::after, .bs-popover-auto[x-placement^=\"left\"] > .arrow::after {\n  right: 1px;\n  border-width: 0.5rem 0 0.5rem 0.5rem;\n  border-left-color: #fff;\n}\n\n.popover-header {\n  padding: 0.5rem 0.75rem;\n  margin-bottom: 0;\n  font-size: 1rem;\n  background-color: #f7f7f7;\n  border-bottom: 1px solid #ebebeb;\n  border-top-left-radius: calc(0.3rem - 1px);\n  border-top-right-radius: calc(0.3rem - 1px);\n}\n\n.popover-header:empty {\n  display: none;\n}\n\n.popover-body {\n  padding: 0.5rem 0.75rem;\n  color: #212529;\n}\n\n.carousel {\n  position: relative;\n}\n\n.carousel.pointer-event {\n  -ms-touch-action: pan-y;\n  touch-action: pan-y;\n}\n\n.carousel-inner {\n  position: relative;\n  width: 100%;\n  overflow: hidden;\n}\n\n.carousel-inner::after {\n  display: block;\n  clear: both;\n  content: \"\";\n}\n\n.carousel-item {\n  position: relative;\n  display: none;\n  float: left;\n  width: 100%;\n  margin-right: -100%;\n  -webkit-backface-visibility: hidden;\n  backface-visibility: hidden;\n  transition: -webkit-transform 0.6s ease-in-out;\n  transition: transform 0.6s ease-in-out;\n  transition: transform 0.6s ease-in-out, -webkit-transform 0.6s ease-in-out;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .carousel-item {\n    transition: none;\n  }\n}\n\n.carousel-item.active,\n.carousel-item-next,\n.carousel-item-prev {\n  display: block;\n}\n\n.carousel-item-next:not(.carousel-item-left),\n.active.carousel-item-right {\n  -webkit-transform: translateX(100%);\n  transform: translateX(100%);\n}\n\n.carousel-item-prev:not(.carousel-item-right),\n.active.carousel-item-left {\n  -webkit-transform: translateX(-100%);\n  transform: translateX(-100%);\n}\n\n.carousel-fade .carousel-item {\n  opacity: 0;\n  transition-property: opacity;\n  -webkit-transform: none;\n  transform: none;\n}\n\n.carousel-fade .carousel-item.active,\n.carousel-fade .carousel-item-next.carousel-item-left,\n.carousel-fade .carousel-item-prev.carousel-item-right {\n  z-index: 1;\n  opacity: 1;\n}\n\n.carousel-fade .active.carousel-item-left,\n.carousel-fade .active.carousel-item-right {\n  z-index: 0;\n  opacity: 0;\n  transition: opacity 0s 0.6s;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .carousel-fade .active.carousel-item-left,\n  .carousel-fade .active.carousel-item-right {\n    transition: none;\n  }\n}\n\n.carousel-control-prev,\n.carousel-control-next {\n  position: absolute;\n  top: 0;\n  bottom: 0;\n  z-index: 1;\n  display: -ms-flexbox;\n  display: flex;\n  -ms-flex-align: center;\n  align-items: center;\n  -ms-flex-pack: center;\n  justify-content: center;\n  width: 15%;\n  color: #fff;\n  text-align: center;\n  opacity: 0.5;\n  transition: opacity 0.15s ease;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .carousel-control-prev,\n  .carousel-control-next {\n    transition: none;\n  }\n}\n\n.carousel-control-prev:hover, .carousel-control-prev:focus,\n.carousel-control-next:hover,\n.carousel-control-next:focus {\n  color: #fff;\n  text-decoration: none;\n  outline: 0;\n  opacity: 0.9;\n}\n\n.carousel-control-prev {\n  left: 0;\n}\n\n.carousel-control-next {\n  right: 0;\n}\n\n.carousel-control-prev-icon,\n.carousel-control-next-icon {\n  display: inline-block;\n  width: 20px;\n  height: 20px;\n  background: no-repeat 50% / 100% 100%;\n}\n\n.carousel-control-prev-icon {\n  background-image: url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath d='M5.25 0l-4 4 4 4 1.5-1.5L4.25 4l2.5-2.5L5.25 0z'/%3e%3c/svg%3e\");\n}\n\n.carousel-control-next-icon {\n  background-image: url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath d='M2.75 0l-1.5 1.5L3.75 4l-2.5 2.5L2.75 8l4-4-4-4z'/%3e%3c/svg%3e\");\n}\n\n.carousel-indicators {\n  position: absolute;\n  right: 0;\n  bottom: 0;\n  left: 0;\n  z-index: 15;\n  display: -ms-flexbox;\n  display: flex;\n  -ms-flex-pack: center;\n  justify-content: center;\n  padding-left: 0;\n  margin-right: 15%;\n  margin-left: 15%;\n  list-style: none;\n}\n\n.carousel-indicators li {\n  box-sizing: content-box;\n  -ms-flex: 0 1 auto;\n  flex: 0 1 auto;\n  width: 30px;\n  height: 3px;\n  margin-right: 3px;\n  margin-left: 3px;\n  text-indent: -999px;\n  cursor: pointer;\n  background-color: #fff;\n  background-clip: padding-box;\n  border-top: 10px solid transparent;\n  border-bottom: 10px solid transparent;\n  opacity: .5;\n  transition: opacity 0.6s ease;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .carousel-indicators li {\n    transition: none;\n  }\n}\n\n.carousel-indicators .active {\n  opacity: 1;\n}\n\n.carousel-caption {\n  position: absolute;\n  right: 15%;\n  bottom: 20px;\n  left: 15%;\n  z-index: 10;\n  padding-top: 20px;\n  padding-bottom: 20px;\n  color: #fff;\n  text-align: center;\n}\n\n@-webkit-keyframes spinner-border {\n  to {\n    -webkit-transform: rotate(360deg);\n    transform: rotate(360deg);\n  }\n}\n\n@keyframes spinner-border {\n  to {\n    -webkit-transform: rotate(360deg);\n    transform: rotate(360deg);\n  }\n}\n\n.spinner-border {\n  display: inline-block;\n  width: 2rem;\n  height: 2rem;\n  vertical-align: text-bottom;\n  border: 0.25em solid currentColor;\n  border-right-color: transparent;\n  border-radius: 50%;\n  -webkit-animation: spinner-border .75s linear infinite;\n  animation: spinner-border .75s linear infinite;\n}\n\n.spinner-border-sm {\n  width: 1rem;\n  height: 1rem;\n  border-width: 0.2em;\n}\n\n@-webkit-keyframes spinner-grow {\n  0% {\n    -webkit-transform: scale(0);\n    transform: scale(0);\n  }\n  50% {\n    opacity: 1;\n  }\n}\n\n@keyframes spinner-grow {\n  0% {\n    -webkit-transform: scale(0);\n    transform: scale(0);\n  }\n  50% {\n    opacity: 1;\n  }\n}\n\n.spinner-grow {\n  display: inline-block;\n  width: 2rem;\n  height: 2rem;\n  vertical-align: text-bottom;\n  background-color: currentColor;\n  border-radius: 50%;\n  opacity: 0;\n  -webkit-animation: spinner-grow .75s linear infinite;\n  animation: spinner-grow .75s linear infinite;\n}\n\n.spinner-grow-sm {\n  width: 1rem;\n  height: 1rem;\n}\n\n.align-baseline {\n  vertical-align: baseline !important;\n}\n\n.align-top {\n  vertical-align: top !important;\n}\n\n.align-middle {\n  vertical-align: middle !important;\n}\n\n.align-bottom {\n  vertical-align: bottom !important;\n}\n\n.align-text-bottom {\n  vertical-align: text-bottom !important;\n}\n\n.align-text-top {\n  vertical-align: text-top !important;\n}\n\n.bg-primary {\n  background-color: #007bff !important;\n}\n\na.bg-primary:hover, a.bg-primary:focus,\nbutton.bg-primary:hover,\nbutton.bg-primary:focus {\n  background-color: #0062cc !important;\n}\n\n.bg-secondary {\n  background-color: #6c757d !important;\n}\n\na.bg-secondary:hover, a.bg-secondary:focus,\nbutton.bg-secondary:hover,\nbutton.bg-secondary:focus {\n  background-color: #545b62 !important;\n}\n\n.bg-success {\n  background-color: #28a745 !important;\n}\n\na.bg-success:hover, a.bg-success:focus,\nbutton.bg-success:hover,\nbutton.bg-success:focus {\n  background-color: #1e7e34 !important;\n}\n\n.bg-info {\n  background-color: #17a2b8 !important;\n}\n\na.bg-info:hover, a.bg-info:focus,\nbutton.bg-info:hover,\nbutton.bg-info:focus {\n  background-color: #117a8b !important;\n}\n\n.bg-warning {\n  background-color: #ffc107 !important;\n}\n\na.bg-warning:hover, a.bg-warning:focus,\nbutton.bg-warning:hover,\nbutton.bg-warning:focus {\n  background-color: #d39e00 !important;\n}\n\n.bg-danger {\n  background-color: #dc3545 !important;\n}\n\na.bg-danger:hover, a.bg-danger:focus,\nbutton.bg-danger:hover,\nbutton.bg-danger:focus {\n  background-color: #bd2130 !important;\n}\n\n.bg-light {\n  background-color: #f8f9fa !important;\n}\n\na.bg-light:hover, a.bg-light:focus,\nbutton.bg-light:hover,\nbutton.bg-light:focus {\n  background-color: #dae0e5 !important;\n}\n\n.bg-dark {\n  background-color: #343a40 !important;\n}\n\na.bg-dark:hover, a.bg-dark:focus,\nbutton.bg-dark:hover,\nbutton.bg-dark:focus {\n  background-color: #1d2124 !important;\n}\n\n.bg-white {\n  background-color: #fff !important;\n}\n\n.bg-transparent {\n  background-color: transparent !important;\n}\n\n.border {\n  border: 1px solid #dee2e6 !important;\n}\n\n.border-top {\n  border-top: 1px solid #dee2e6 !important;\n}\n\n.border-right {\n  border-right: 1px solid #dee2e6 !important;\n}\n\n.border-bottom {\n  border-bottom: 1px solid #dee2e6 !important;\n}\n\n.border-left {\n  border-left: 1px solid #dee2e6 !important;\n}\n\n.border-0 {\n  border: 0 !important;\n}\n\n.border-top-0 {\n  border-top: 0 !important;\n}\n\n.border-right-0 {\n  border-right: 0 !important;\n}\n\n.border-bottom-0 {\n  border-bottom: 0 !important;\n}\n\n.border-left-0 {\n  border-left: 0 !important;\n}\n\n.border-primary {\n  border-color: #007bff !important;\n}\n\n.border-secondary {\n  border-color: #6c757d !important;\n}\n\n.border-success {\n  border-color: #28a745 !important;\n}\n\n.border-info {\n  border-color: #17a2b8 !important;\n}\n\n.border-warning {\n  border-color: #ffc107 !important;\n}\n\n.border-danger {\n  border-color: #dc3545 !important;\n}\n\n.border-light {\n  border-color: #f8f9fa !important;\n}\n\n.border-dark {\n  border-color: #343a40 !important;\n}\n\n.border-white {\n  border-color: #fff !important;\n}\n\n.rounded-sm {\n  border-radius: 0.2rem !important;\n}\n\n.rounded {\n  border-radius: 0.25rem !important;\n}\n\n.rounded-top {\n  border-top-left-radius: 0.25rem !important;\n  border-top-right-radius: 0.25rem !important;\n}\n\n.rounded-right {\n  border-top-right-radius: 0.25rem !important;\n  border-bottom-right-radius: 0.25rem !important;\n}\n\n.rounded-bottom {\n  border-bottom-right-radius: 0.25rem !important;\n  border-bottom-left-radius: 0.25rem !important;\n}\n\n.rounded-left {\n  border-top-left-radius: 0.25rem !important;\n  border-bottom-left-radius: 0.25rem !important;\n}\n\n.rounded-lg {\n  border-radius: 0.3rem !important;\n}\n\n.rounded-circle {\n  border-radius: 50% !important;\n}\n\n.rounded-pill {\n  border-radius: 50rem !important;\n}\n\n.rounded-0 {\n  border-radius: 0 !important;\n}\n\n.clearfix::after {\n  display: block;\n  clear: both;\n  content: \"\";\n}\n\n.d-none {\n  display: none !important;\n}\n\n.d-inline {\n  display: inline !important;\n}\n\n.d-inline-block {\n  display: inline-block !important;\n}\n\n.d-block {\n  display: block !important;\n}\n\n.d-table {\n  display: table !important;\n}\n\n.d-table-row {\n  display: table-row !important;\n}\n\n.d-table-cell {\n  display: table-cell !important;\n}\n\n.d-flex {\n  display: -ms-flexbox !important;\n  display: flex !important;\n}\n\n.d-inline-flex {\n  display: -ms-inline-flexbox !important;\n  display: inline-flex !important;\n}\n\n@media (min-width: 576px) {\n  .d-sm-none {\n    display: none !important;\n  }\n  .d-sm-inline {\n    display: inline !important;\n  }\n  .d-sm-inline-block {\n    display: inline-block !important;\n  }\n  .d-sm-block {\n    display: block !important;\n  }\n  .d-sm-table {\n    display: table !important;\n  }\n  .d-sm-table-row {\n    display: table-row !important;\n  }\n  .d-sm-table-cell {\n    display: table-cell !important;\n  }\n  .d-sm-flex {\n    display: -ms-flexbox !important;\n    display: flex !important;\n  }\n  .d-sm-inline-flex {\n    display: -ms-inline-flexbox !important;\n    display: inline-flex !important;\n  }\n}\n\n@media (min-width: 768px) {\n  .d-md-none {\n    display: none !important;\n  }\n  .d-md-inline {\n    display: inline !important;\n  }\n  .d-md-inline-block {\n    display: inline-block !important;\n  }\n  .d-md-block {\n    display: block !important;\n  }\n  .d-md-table {\n    display: table !important;\n  }\n  .d-md-table-row {\n    display: table-row !important;\n  }\n  .d-md-table-cell {\n    display: table-cell !important;\n  }\n  .d-md-flex {\n    display: -ms-flexbox !important;\n    display: flex !important;\n  }\n  .d-md-inline-flex {\n    display: -ms-inline-flexbox !important;\n    display: inline-flex !important;\n  }\n}\n\n@media (min-width: 992px) {\n  .d-lg-none {\n    display: none !important;\n  }\n  .d-lg-inline {\n    display: inline !important;\n  }\n  .d-lg-inline-block {\n    display: inline-block !important;\n  }\n  .d-lg-block {\n    display: block !important;\n  }\n  .d-lg-table {\n    display: table !important;\n  }\n  .d-lg-table-row {\n    display: table-row !important;\n  }\n  .d-lg-table-cell {\n    display: table-cell !important;\n  }\n  .d-lg-flex {\n    display: -ms-flexbox !important;\n    display: flex !important;\n  }\n  .d-lg-inline-flex {\n    display: -ms-inline-flexbox !important;\n    display: inline-flex !important;\n  }\n}\n\n@media (min-width: 1200px) {\n  .d-xl-none {\n    display: none !important;\n  }\n  .d-xl-inline {\n    display: inline !important;\n  }\n  .d-xl-inline-block {\n    display: inline-block !important;\n  }\n  .d-xl-block {\n    display: block !important;\n  }\n  .d-xl-table {\n    display: table !important;\n  }\n  .d-xl-table-row {\n    display: table-row !important;\n  }\n  .d-xl-table-cell {\n    display: table-cell !important;\n  }\n  .d-xl-flex {\n    display: -ms-flexbox !important;\n    display: flex !important;\n  }\n  .d-xl-inline-flex {\n    display: -ms-inline-flexbox !important;\n    display: inline-flex !important;\n  }\n}\n\n@media print {\n  .d-print-none {\n    display: none !important;\n  }\n  .d-print-inline {\n    display: inline !important;\n  }\n  .d-print-inline-block {\n    display: inline-block !important;\n  }\n  .d-print-block {\n    display: block !important;\n  }\n  .d-print-table {\n    display: table !important;\n  }\n  .d-print-table-row {\n    display: table-row !important;\n  }\n  .d-print-table-cell {\n    display: table-cell !important;\n  }\n  .d-print-flex {\n    display: -ms-flexbox !important;\n    display: flex !important;\n  }\n  .d-print-inline-flex {\n    display: -ms-inline-flexbox !important;\n    display: inline-flex !important;\n  }\n}\n\n.embed-responsive {\n  position: relative;\n  display: block;\n  width: 100%;\n  padding: 0;\n  overflow: hidden;\n}\n\n.embed-responsive::before {\n  display: block;\n  content: \"\";\n}\n\n.embed-responsive .embed-responsive-item,\n.embed-responsive iframe,\n.embed-responsive embed,\n.embed-responsive object,\n.embed-responsive video {\n  position: absolute;\n  top: 0;\n  bottom: 0;\n  left: 0;\n  width: 100%;\n  height: 100%;\n  border: 0;\n}\n\n.embed-responsive-21by9::before {\n  padding-top: 42.857143%;\n}\n\n.embed-responsive-16by9::before {\n  padding-top: 56.25%;\n}\n\n.embed-responsive-4by3::before {\n  padding-top: 75%;\n}\n\n.embed-responsive-1by1::before {\n  padding-top: 100%;\n}\n\n.flex-row {\n  -ms-flex-direction: row !important;\n  flex-direction: row !important;\n}\n\n.flex-column {\n  -ms-flex-direction: column !important;\n  flex-direction: column !important;\n}\n\n.flex-row-reverse {\n  -ms-flex-direction: row-reverse !important;\n  flex-direction: row-reverse !important;\n}\n\n.flex-column-reverse {\n  -ms-flex-direction: column-reverse !important;\n  flex-direction: column-reverse !important;\n}\n\n.flex-wrap {\n  -ms-flex-wrap: wrap !important;\n  flex-wrap: wrap !important;\n}\n\n.flex-nowrap {\n  -ms-flex-wrap: nowrap !important;\n  flex-wrap: nowrap !important;\n}\n\n.flex-wrap-reverse {\n  -ms-flex-wrap: wrap-reverse !important;\n  flex-wrap: wrap-reverse !important;\n}\n\n.flex-fill {\n  -ms-flex: 1 1 auto !important;\n  flex: 1 1 auto !important;\n}\n\n.flex-grow-0 {\n  -ms-flex-positive: 0 !important;\n  flex-grow: 0 !important;\n}\n\n.flex-grow-1 {\n  -ms-flex-positive: 1 !important;\n  flex-grow: 1 !important;\n}\n\n.flex-shrink-0 {\n  -ms-flex-negative: 0 !important;\n  flex-shrink: 0 !important;\n}\n\n.flex-shrink-1 {\n  -ms-flex-negative: 1 !important;\n  flex-shrink: 1 !important;\n}\n\n.justify-content-start {\n  -ms-flex-pack: start !important;\n  justify-content: flex-start !important;\n}\n\n.justify-content-end {\n  -ms-flex-pack: end !important;\n  justify-content: flex-end !important;\n}\n\n.justify-content-center {\n  -ms-flex-pack: center !important;\n  justify-content: center !important;\n}\n\n.justify-content-between {\n  -ms-flex-pack: justify !important;\n  justify-content: space-between !important;\n}\n\n.justify-content-around {\n  -ms-flex-pack: distribute !important;\n  justify-content: space-around !important;\n}\n\n.align-items-start {\n  -ms-flex-align: start !important;\n  align-items: flex-start !important;\n}\n\n.align-items-end {\n  -ms-flex-align: end !important;\n  align-items: flex-end !important;\n}\n\n.align-items-center {\n  -ms-flex-align: center !important;\n  align-items: center !important;\n}\n\n.align-items-baseline {\n  -ms-flex-align: baseline !important;\n  align-items: baseline !important;\n}\n\n.align-items-stretch {\n  -ms-flex-align: stretch !important;\n  align-items: stretch !important;\n}\n\n.align-content-start {\n  -ms-flex-line-pack: start !important;\n  align-content: flex-start !important;\n}\n\n.align-content-end {\n  -ms-flex-line-pack: end !important;\n  align-content: flex-end !important;\n}\n\n.align-content-center {\n  -ms-flex-line-pack: center !important;\n  align-content: center !important;\n}\n\n.align-content-between {\n  -ms-flex-line-pack: justify !important;\n  align-content: space-between !important;\n}\n\n.align-content-around {\n  -ms-flex-line-pack: distribute !important;\n  align-content: space-around !important;\n}\n\n.align-content-stretch {\n  -ms-flex-line-pack: stretch !important;\n  align-content: stretch !important;\n}\n\n.align-self-auto {\n  -ms-flex-item-align: auto !important;\n  align-self: auto !important;\n}\n\n.align-self-start {\n  -ms-flex-item-align: start !important;\n  align-self: flex-start !important;\n}\n\n.align-self-end {\n  -ms-flex-item-align: end !important;\n  align-self: flex-end !important;\n}\n\n.align-self-center {\n  -ms-flex-item-align: center !important;\n  align-self: center !important;\n}\n\n.align-self-baseline {\n  -ms-flex-item-align: baseline !important;\n  align-self: baseline !important;\n}\n\n.align-self-stretch {\n  -ms-flex-item-align: stretch !important;\n  align-self: stretch !important;\n}\n\n@media (min-width: 576px) {\n  .flex-sm-row {\n    -ms-flex-direction: row !important;\n    flex-direction: row !important;\n  }\n  .flex-sm-column {\n    -ms-flex-direction: column !important;\n    flex-direction: column !important;\n  }\n  .flex-sm-row-reverse {\n    -ms-flex-direction: row-reverse !important;\n    flex-direction: row-reverse !important;\n  }\n  .flex-sm-column-reverse {\n    -ms-flex-direction: column-reverse !important;\n    flex-direction: column-reverse !important;\n  }\n  .flex-sm-wrap {\n    -ms-flex-wrap: wrap !important;\n    flex-wrap: wrap !important;\n  }\n  .flex-sm-nowrap {\n    -ms-flex-wrap: nowrap !important;\n    flex-wrap: nowrap !important;\n  }\n  .flex-sm-wrap-reverse {\n    -ms-flex-wrap: wrap-reverse !important;\n    flex-wrap: wrap-reverse !important;\n  }\n  .flex-sm-fill {\n    -ms-flex: 1 1 auto !important;\n    flex: 1 1 auto !important;\n  }\n  .flex-sm-grow-0 {\n    -ms-flex-positive: 0 !important;\n    flex-grow: 0 !important;\n  }\n  .flex-sm-grow-1 {\n    -ms-flex-positive: 1 !important;\n    flex-grow: 1 !important;\n  }\n  .flex-sm-shrink-0 {\n    -ms-flex-negative: 0 !important;\n    flex-shrink: 0 !important;\n  }\n  .flex-sm-shrink-1 {\n    -ms-flex-negative: 1 !important;\n    flex-shrink: 1 !important;\n  }\n  .justify-content-sm-start {\n    -ms-flex-pack: start !important;\n    justify-content: flex-start !important;\n  }\n  .justify-content-sm-end {\n    -ms-flex-pack: end !important;\n    justify-content: flex-end !important;\n  }\n  .justify-content-sm-center {\n    -ms-flex-pack: center !important;\n    justify-content: center !important;\n  }\n  .justify-content-sm-between {\n    -ms-flex-pack: justify !important;\n    justify-content: space-between !important;\n  }\n  .justify-content-sm-around {\n    -ms-flex-pack: distribute !important;\n    justify-content: space-around !important;\n  }\n  .align-items-sm-start {\n    -ms-flex-align: start !important;\n    align-items: flex-start !important;\n  }\n  .align-items-sm-end {\n    -ms-flex-align: end !important;\n    align-items: flex-end !important;\n  }\n  .align-items-sm-center {\n    -ms-flex-align: center !important;\n    align-items: center !important;\n  }\n  .align-items-sm-baseline {\n    -ms-flex-align: baseline !important;\n    align-items: baseline !important;\n  }\n  .align-items-sm-stretch {\n    -ms-flex-align: stretch !important;\n    align-items: stretch !important;\n  }\n  .align-content-sm-start {\n    -ms-flex-line-pack: start !important;\n    align-content: flex-start !important;\n  }\n  .align-content-sm-end {\n    -ms-flex-line-pack: end !important;\n    align-content: flex-end !important;\n  }\n  .align-content-sm-center {\n    -ms-flex-line-pack: center !important;\n    align-content: center !important;\n  }\n  .align-content-sm-between {\n    -ms-flex-line-pack: justify !important;\n    align-content: space-between !important;\n  }\n  .align-content-sm-around {\n    -ms-flex-line-pack: distribute !important;\n    align-content: space-around !important;\n  }\n  .align-content-sm-stretch {\n    -ms-flex-line-pack: stretch !important;\n    align-content: stretch !important;\n  }\n  .align-self-sm-auto {\n    -ms-flex-item-align: auto !important;\n    align-self: auto !important;\n  }\n  .align-self-sm-start {\n    -ms-flex-item-align: start !important;\n    align-self: flex-start !important;\n  }\n  .align-self-sm-end {\n    -ms-flex-item-align: end !important;\n    align-self: flex-end !important;\n  }\n  .align-self-sm-center {\n    -ms-flex-item-align: center !important;\n    align-self: center !important;\n  }\n  .align-self-sm-baseline {\n    -ms-flex-item-align: baseline !important;\n    align-self: baseline !important;\n  }\n  .align-self-sm-stretch {\n    -ms-flex-item-align: stretch !important;\n    align-self: stretch !important;\n  }\n}\n\n@media (min-width: 768px) {\n  .flex-md-row {\n    -ms-flex-direction: row !important;\n    flex-direction: row !important;\n  }\n  .flex-md-column {\n    -ms-flex-direction: column !important;\n    flex-direction: column !important;\n  }\n  .flex-md-row-reverse {\n    -ms-flex-direction: row-reverse !important;\n    flex-direction: row-reverse !important;\n  }\n  .flex-md-column-reverse {\n    -ms-flex-direction: column-reverse !important;\n    flex-direction: column-reverse !important;\n  }\n  .flex-md-wrap {\n    -ms-flex-wrap: wrap !important;\n    flex-wrap: wrap !important;\n  }\n  .flex-md-nowrap {\n    -ms-flex-wrap: nowrap !important;\n    flex-wrap: nowrap !important;\n  }\n  .flex-md-wrap-reverse {\n    -ms-flex-wrap: wrap-reverse !important;\n    flex-wrap: wrap-reverse !important;\n  }\n  .flex-md-fill {\n    -ms-flex: 1 1 auto !important;\n    flex: 1 1 auto !important;\n  }\n  .flex-md-grow-0 {\n    -ms-flex-positive: 0 !important;\n    flex-grow: 0 !important;\n  }\n  .flex-md-grow-1 {\n    -ms-flex-positive: 1 !important;\n    flex-grow: 1 !important;\n  }\n  .flex-md-shrink-0 {\n    -ms-flex-negative: 0 !important;\n    flex-shrink: 0 !important;\n  }\n  .flex-md-shrink-1 {\n    -ms-flex-negative: 1 !important;\n    flex-shrink: 1 !important;\n  }\n  .justify-content-md-start {\n    -ms-flex-pack: start !important;\n    justify-content: flex-start !important;\n  }\n  .justify-content-md-end {\n    -ms-flex-pack: end !important;\n    justify-content: flex-end !important;\n  }\n  .justify-content-md-center {\n    -ms-flex-pack: center !important;\n    justify-content: center !important;\n  }\n  .justify-content-md-between {\n    -ms-flex-pack: justify !important;\n    justify-content: space-between !important;\n  }\n  .justify-content-md-around {\n    -ms-flex-pack: distribute !important;\n    justify-content: space-around !important;\n  }\n  .align-items-md-start {\n    -ms-flex-align: start !important;\n    align-items: flex-start !important;\n  }\n  .align-items-md-end {\n    -ms-flex-align: end !important;\n    align-items: flex-end !important;\n  }\n  .align-items-md-center {\n    -ms-flex-align: center !important;\n    align-items: center !important;\n  }\n  .align-items-md-baseline {\n    -ms-flex-align: baseline !important;\n    align-items: baseline !important;\n  }\n  .align-items-md-stretch {\n    -ms-flex-align: stretch !important;\n    align-items: stretch !important;\n  }\n  .align-content-md-start {\n    -ms-flex-line-pack: start !important;\n    align-content: flex-start !important;\n  }\n  .align-content-md-end {\n    -ms-flex-line-pack: end !important;\n    align-content: flex-end !important;\n  }\n  .align-content-md-center {\n    -ms-flex-line-pack: center !important;\n    align-content: center !important;\n  }\n  .align-content-md-between {\n    -ms-flex-line-pack: justify !important;\n    align-content: space-between !important;\n  }\n  .align-content-md-around {\n    -ms-flex-line-pack: distribute !important;\n    align-content: space-around !important;\n  }\n  .align-content-md-stretch {\n    -ms-flex-line-pack: stretch !important;\n    align-content: stretch !important;\n  }\n  .align-self-md-auto {\n    -ms-flex-item-align: auto !important;\n    align-self: auto !important;\n  }\n  .align-self-md-start {\n    -ms-flex-item-align: start !important;\n    align-self: flex-start !important;\n  }\n  .align-self-md-end {\n    -ms-flex-item-align: end !important;\n    align-self: flex-end !important;\n  }\n  .align-self-md-center {\n    -ms-flex-item-align: center !important;\n    align-self: center !important;\n  }\n  .align-self-md-baseline {\n    -ms-flex-item-align: baseline !important;\n    align-self: baseline !important;\n  }\n  .align-self-md-stretch {\n    -ms-flex-item-align: stretch !important;\n    align-self: stretch !important;\n  }\n}\n\n@media (min-width: 992px) {\n  .flex-lg-row {\n    -ms-flex-direction: row !important;\n    flex-direction: row !important;\n  }\n  .flex-lg-column {\n    -ms-flex-direction: column !important;\n    flex-direction: column !important;\n  }\n  .flex-lg-row-reverse {\n    -ms-flex-direction: row-reverse !important;\n    flex-direction: row-reverse !important;\n  }\n  .flex-lg-column-reverse {\n    -ms-flex-direction: column-reverse !important;\n    flex-direction: column-reverse !important;\n  }\n  .flex-lg-wrap {\n    -ms-flex-wrap: wrap !important;\n    flex-wrap: wrap !important;\n  }\n  .flex-lg-nowrap {\n    -ms-flex-wrap: nowrap !important;\n    flex-wrap: nowrap !important;\n  }\n  .flex-lg-wrap-reverse {\n    -ms-flex-wrap: wrap-reverse !important;\n    flex-wrap: wrap-reverse !important;\n  }\n  .flex-lg-fill {\n    -ms-flex: 1 1 auto !important;\n    flex: 1 1 auto !important;\n  }\n  .flex-lg-grow-0 {\n    -ms-flex-positive: 0 !important;\n    flex-grow: 0 !important;\n  }\n  .flex-lg-grow-1 {\n    -ms-flex-positive: 1 !important;\n    flex-grow: 1 !important;\n  }\n  .flex-lg-shrink-0 {\n    -ms-flex-negative: 0 !important;\n    flex-shrink: 0 !important;\n  }\n  .flex-lg-shrink-1 {\n    -ms-flex-negative: 1 !important;\n    flex-shrink: 1 !important;\n  }\n  .justify-content-lg-start {\n    -ms-flex-pack: start !important;\n    justify-content: flex-start !important;\n  }\n  .justify-content-lg-end {\n    -ms-flex-pack: end !important;\n    justify-content: flex-end !important;\n  }\n  .justify-content-lg-center {\n    -ms-flex-pack: center !important;\n    justify-content: center !important;\n  }\n  .justify-content-lg-between {\n    -ms-flex-pack: justify !important;\n    justify-content: space-between !important;\n  }\n  .justify-content-lg-around {\n    -ms-flex-pack: distribute !important;\n    justify-content: space-around !important;\n  }\n  .align-items-lg-start {\n    -ms-flex-align: start !important;\n    align-items: flex-start !important;\n  }\n  .align-items-lg-end {\n    -ms-flex-align: end !important;\n    align-items: flex-end !important;\n  }\n  .align-items-lg-center {\n    -ms-flex-align: center !important;\n    align-items: center !important;\n  }\n  .align-items-lg-baseline {\n    -ms-flex-align: baseline !important;\n    align-items: baseline !important;\n  }\n  .align-items-lg-stretch {\n    -ms-flex-align: stretch !important;\n    align-items: stretch !important;\n  }\n  .align-content-lg-start {\n    -ms-flex-line-pack: start !important;\n    align-content: flex-start !important;\n  }\n  .align-content-lg-end {\n    -ms-flex-line-pack: end !important;\n    align-content: flex-end !important;\n  }\n  .align-content-lg-center {\n    -ms-flex-line-pack: center !important;\n    align-content: center !important;\n  }\n  .align-content-lg-between {\n    -ms-flex-line-pack: justify !important;\n    align-content: space-between !important;\n  }\n  .align-content-lg-around {\n    -ms-flex-line-pack: distribute !important;\n    align-content: space-around !important;\n  }\n  .align-content-lg-stretch {\n    -ms-flex-line-pack: stretch !important;\n    align-content: stretch !important;\n  }\n  .align-self-lg-auto {\n    -ms-flex-item-align: auto !important;\n    align-self: auto !important;\n  }\n  .align-self-lg-start {\n    -ms-flex-item-align: start !important;\n    align-self: flex-start !important;\n  }\n  .align-self-lg-end {\n    -ms-flex-item-align: end !important;\n    align-self: flex-end !important;\n  }\n  .align-self-lg-center {\n    -ms-flex-item-align: center !important;\n    align-self: center !important;\n  }\n  .align-self-lg-baseline {\n    -ms-flex-item-align: baseline !important;\n    align-self: baseline !important;\n  }\n  .align-self-lg-stretch {\n    -ms-flex-item-align: stretch !important;\n    align-self: stretch !important;\n  }\n}\n\n@media (min-width: 1200px) {\n  .flex-xl-row {\n    -ms-flex-direction: row !important;\n    flex-direction: row !important;\n  }\n  .flex-xl-column {\n    -ms-flex-direction: column !important;\n    flex-direction: column !important;\n  }\n  .flex-xl-row-reverse {\n    -ms-flex-direction: row-reverse !important;\n    flex-direction: row-reverse !important;\n  }\n  .flex-xl-column-reverse {\n    -ms-flex-direction: column-reverse !important;\n    flex-direction: column-reverse !important;\n  }\n  .flex-xl-wrap {\n    -ms-flex-wrap: wrap !important;\n    flex-wrap: wrap !important;\n  }\n  .flex-xl-nowrap {\n    -ms-flex-wrap: nowrap !important;\n    flex-wrap: nowrap !important;\n  }\n  .flex-xl-wrap-reverse {\n    -ms-flex-wrap: wrap-reverse !important;\n    flex-wrap: wrap-reverse !important;\n  }\n  .flex-xl-fill {\n    -ms-flex: 1 1 auto !important;\n    flex: 1 1 auto !important;\n  }\n  .flex-xl-grow-0 {\n    -ms-flex-positive: 0 !important;\n    flex-grow: 0 !important;\n  }\n  .flex-xl-grow-1 {\n    -ms-flex-positive: 1 !important;\n    flex-grow: 1 !important;\n  }\n  .flex-xl-shrink-0 {\n    -ms-flex-negative: 0 !important;\n    flex-shrink: 0 !important;\n  }\n  .flex-xl-shrink-1 {\n    -ms-flex-negative: 1 !important;\n    flex-shrink: 1 !important;\n  }\n  .justify-content-xl-start {\n    -ms-flex-pack: start !important;\n    justify-content: flex-start !important;\n  }\n  .justify-content-xl-end {\n    -ms-flex-pack: end !important;\n    justify-content: flex-end !important;\n  }\n  .justify-content-xl-center {\n    -ms-flex-pack: center !important;\n    justify-content: center !important;\n  }\n  .justify-content-xl-between {\n    -ms-flex-pack: justify !important;\n    justify-content: space-between !important;\n  }\n  .justify-content-xl-around {\n    -ms-flex-pack: distribute !important;\n    justify-content: space-around !important;\n  }\n  .align-items-xl-start {\n    -ms-flex-align: start !important;\n    align-items: flex-start !important;\n  }\n  .align-items-xl-end {\n    -ms-flex-align: end !important;\n    align-items: flex-end !important;\n  }\n  .align-items-xl-center {\n    -ms-flex-align: center !important;\n    align-items: center !important;\n  }\n  .align-items-xl-baseline {\n    -ms-flex-align: baseline !important;\n    align-items: baseline !important;\n  }\n  .align-items-xl-stretch {\n    -ms-flex-align: stretch !important;\n    align-items: stretch !important;\n  }\n  .align-content-xl-start {\n    -ms-flex-line-pack: start !important;\n    align-content: flex-start !important;\n  }\n  .align-content-xl-end {\n    -ms-flex-line-pack: end !important;\n    align-content: flex-end !important;\n  }\n  .align-content-xl-center {\n    -ms-flex-line-pack: center !important;\n    align-content: center !important;\n  }\n  .align-content-xl-between {\n    -ms-flex-line-pack: justify !important;\n    align-content: space-between !important;\n  }\n  .align-content-xl-around {\n    -ms-flex-line-pack: distribute !important;\n    align-content: space-around !important;\n  }\n  .align-content-xl-stretch {\n    -ms-flex-line-pack: stretch !important;\n    align-content: stretch !important;\n  }\n  .align-self-xl-auto {\n    -ms-flex-item-align: auto !important;\n    align-self: auto !important;\n  }\n  .align-self-xl-start {\n    -ms-flex-item-align: start !important;\n    align-self: flex-start !important;\n  }\n  .align-self-xl-end {\n    -ms-flex-item-align: end !important;\n    align-self: flex-end !important;\n  }\n  .align-self-xl-center {\n    -ms-flex-item-align: center !important;\n    align-self: center !important;\n  }\n  .align-self-xl-baseline {\n    -ms-flex-item-align: baseline !important;\n    align-self: baseline !important;\n  }\n  .align-self-xl-stretch {\n    -ms-flex-item-align: stretch !important;\n    align-self: stretch !important;\n  }\n}\n\n.float-left {\n  float: left !important;\n}\n\n.float-right {\n  float: right !important;\n}\n\n.float-none {\n  float: none !important;\n}\n\n@media (min-width: 576px) {\n  .float-sm-left {\n    float: left !important;\n  }\n  .float-sm-right {\n    float: right !important;\n  }\n  .float-sm-none {\n    float: none !important;\n  }\n}\n\n@media (min-width: 768px) {\n  .float-md-left {\n    float: left !important;\n  }\n  .float-md-right {\n    float: right !important;\n  }\n  .float-md-none {\n    float: none !important;\n  }\n}\n\n@media (min-width: 992px) {\n  .float-lg-left {\n    float: left !important;\n  }\n  .float-lg-right {\n    float: right !important;\n  }\n  .float-lg-none {\n    float: none !important;\n  }\n}\n\n@media (min-width: 1200px) {\n  .float-xl-left {\n    float: left !important;\n  }\n  .float-xl-right {\n    float: right !important;\n  }\n  .float-xl-none {\n    float: none !important;\n  }\n}\n\n.overflow-auto {\n  overflow: auto !important;\n}\n\n.overflow-hidden {\n  overflow: hidden !important;\n}\n\n.position-static {\n  position: static !important;\n}\n\n.position-relative {\n  position: relative !important;\n}\n\n.position-absolute {\n  position: absolute !important;\n}\n\n.position-fixed {\n  position: fixed !important;\n}\n\n.position-sticky {\n  position: -webkit-sticky !important;\n  position: sticky !important;\n}\n\n.fixed-top {\n  position: fixed;\n  top: 0;\n  right: 0;\n  left: 0;\n  z-index: 1030;\n}\n\n.fixed-bottom {\n  position: fixed;\n  right: 0;\n  bottom: 0;\n  left: 0;\n  z-index: 1030;\n}\n\n@supports ((position: -webkit-sticky) or (position: sticky)) {\n  .sticky-top {\n    position: -webkit-sticky;\n    position: sticky;\n    top: 0;\n    z-index: 1020;\n  }\n}\n\n.sr-only {\n  position: absolute;\n  width: 1px;\n  height: 1px;\n  padding: 0;\n  margin: -1px;\n  overflow: hidden;\n  clip: rect(0, 0, 0, 0);\n  white-space: nowrap;\n  border: 0;\n}\n\n.sr-only-focusable:active, .sr-only-focusable:focus {\n  position: static;\n  width: auto;\n  height: auto;\n  overflow: visible;\n  clip: auto;\n  white-space: normal;\n}\n\n.shadow-sm {\n  box-shadow: 0 0.125rem 0.25rem rgba(0, 0, 0, 0.075) !important;\n}\n\n.shadow {\n  box-shadow: 0 0.5rem 1rem rgba(0, 0, 0, 0.15) !important;\n}\n\n.shadow-lg {\n  box-shadow: 0 1rem 3rem rgba(0, 0, 0, 0.175) !important;\n}\n\n.shadow-none {\n  box-shadow: none !important;\n}\n\n.w-25 {\n  width: 25% !important;\n}\n\n.w-50 {\n  width: 50% !important;\n}\n\n.w-75 {\n  width: 75% !important;\n}\n\n.w-100 {\n  width: 100% !important;\n}\n\n.w-auto {\n  width: auto !important;\n}\n\n.h-25 {\n  height: 25% !important;\n}\n\n.h-50 {\n  height: 50% !important;\n}\n\n.h-75 {\n  height: 75% !important;\n}\n\n.h-100 {\n  height: 100% !important;\n}\n\n.h-auto {\n  height: auto !important;\n}\n\n.mw-100 {\n  max-width: 100% !important;\n}\n\n.mh-100 {\n  max-height: 100% !important;\n}\n\n.min-vw-100 {\n  min-width: 100vw !important;\n}\n\n.min-vh-100 {\n  min-height: 100vh !important;\n}\n\n.vw-100 {\n  width: 100vw !important;\n}\n\n.vh-100 {\n  height: 100vh !important;\n}\n\n.stretched-link::after {\n  position: absolute;\n  top: 0;\n  right: 0;\n  bottom: 0;\n  left: 0;\n  z-index: 1;\n  pointer-events: auto;\n  content: \"\";\n  background-color: rgba(0, 0, 0, 0);\n}\n\n.m-0 {\n  margin: 0 !important;\n}\n\n.mt-0,\n.my-0 {\n  margin-top: 0 !important;\n}\n\n.mr-0,\n.mx-0 {\n  margin-right: 0 !important;\n}\n\n.mb-0,\n.my-0 {\n  margin-bottom: 0 !important;\n}\n\n.ml-0,\n.mx-0 {\n  margin-left: 0 !important;\n}\n\n.m-1 {\n  margin: 0.25rem !important;\n}\n\n.mt-1,\n.my-1 {\n  margin-top: 0.25rem !important;\n}\n\n.mr-1,\n.mx-1 {\n  margin-right: 0.25rem !important;\n}\n\n.mb-1,\n.my-1 {\n  margin-bottom: 0.25rem !important;\n}\n\n.ml-1,\n.mx-1 {\n  margin-left: 0.25rem !important;\n}\n\n.m-2 {\n  margin: 0.5rem !important;\n}\n\n.mt-2,\n.my-2 {\n  margin-top: 0.5rem !important;\n}\n\n.mr-2,\n.mx-2 {\n  margin-right: 0.5rem !important;\n}\n\n.mb-2,\n.my-2 {\n  margin-bottom: 0.5rem !important;\n}\n\n.ml-2,\n.mx-2 {\n  margin-left: 0.5rem !important;\n}\n\n.m-3 {\n  margin: 1rem !important;\n}\n\n.mt-3,\n.my-3 {\n  margin-top: 1rem !important;\n}\n\n.mr-3,\n.mx-3 {\n  margin-right: 1rem !important;\n}\n\n.mb-3,\n.my-3 {\n  margin-bottom: 1rem !important;\n}\n\n.ml-3,\n.mx-3 {\n  margin-left: 1rem !important;\n}\n\n.m-4 {\n  margin: 1.5rem !important;\n}\n\n.mt-4,\n.my-4 {\n  margin-top: 1.5rem !important;\n}\n\n.mr-4,\n.mx-4 {\n  margin-right: 1.5rem !important;\n}\n\n.mb-4,\n.my-4 {\n  margin-bottom: 1.5rem !important;\n}\n\n.ml-4,\n.mx-4 {\n  margin-left: 1.5rem !important;\n}\n\n.m-5 {\n  margin: 3rem !important;\n}\n\n.mt-5,\n.my-5 {\n  margin-top: 3rem !important;\n}\n\n.mr-5,\n.mx-5 {\n  margin-right: 3rem !important;\n}\n\n.mb-5,\n.my-5 {\n  margin-bottom: 3rem !important;\n}\n\n.ml-5,\n.mx-5 {\n  margin-left: 3rem !important;\n}\n\n.p-0 {\n  padding: 0 !important;\n}\n\n.pt-0,\n.py-0 {\n  padding-top: 0 !important;\n}\n\n.pr-0,\n.px-0 {\n  padding-right: 0 !important;\n}\n\n.pb-0,\n.py-0 {\n  padding-bottom: 0 !important;\n}\n\n.pl-0,\n.px-0 {\n  padding-left: 0 !important;\n}\n\n.p-1 {\n  padding: 0.25rem !important;\n}\n\n.pt-1,\n.py-1 {\n  padding-top: 0.25rem !important;\n}\n\n.pr-1,\n.px-1 {\n  padding-right: 0.25rem !important;\n}\n\n.pb-1,\n.py-1 {\n  padding-bottom: 0.25rem !important;\n}\n\n.pl-1,\n.px-1 {\n  padding-left: 0.25rem !important;\n}\n\n.p-2 {\n  padding: 0.5rem !important;\n}\n\n.pt-2,\n.py-2 {\n  padding-top: 0.5rem !important;\n}\n\n.pr-2,\n.px-2 {\n  padding-right: 0.5rem !important;\n}\n\n.pb-2,\n.py-2 {\n  padding-bottom: 0.5rem !important;\n}\n\n.pl-2,\n.px-2 {\n  padding-left: 0.5rem !important;\n}\n\n.p-3 {\n  padding: 1rem !important;\n}\n\n.pt-3,\n.py-3 {\n  padding-top: 1rem !important;\n}\n\n.pr-3,\n.px-3 {\n  padding-right: 1rem !important;\n}\n\n.pb-3,\n.py-3 {\n  padding-bottom: 1rem !important;\n}\n\n.pl-3,\n.px-3 {\n  padding-left: 1rem !important;\n}\n\n.p-4 {\n  padding: 1.5rem !important;\n}\n\n.pt-4,\n.py-4 {\n  padding-top: 1.5rem !important;\n}\n\n.pr-4,\n.px-4 {\n  padding-right: 1.5rem !important;\n}\n\n.pb-4,\n.py-4 {\n  padding-bottom: 1.5rem !important;\n}\n\n.pl-4,\n.px-4 {\n  padding-left: 1.5rem !important;\n}\n\n.p-5 {\n  padding: 3rem !important;\n}\n\n.pt-5,\n.py-5 {\n  padding-top: 3rem !important;\n}\n\n.pr-5,\n.px-5 {\n  padding-right: 3rem !important;\n}\n\n.pb-5,\n.py-5 {\n  padding-bottom: 3rem !important;\n}\n\n.pl-5,\n.px-5 {\n  padding-left: 3rem !important;\n}\n\n.m-n1 {\n  margin: -0.25rem !important;\n}\n\n.mt-n1,\n.my-n1 {\n  margin-top: -0.25rem !important;\n}\n\n.mr-n1,\n.mx-n1 {\n  margin-right: -0.25rem !important;\n}\n\n.mb-n1,\n.my-n1 {\n  margin-bottom: -0.25rem !important;\n}\n\n.ml-n1,\n.mx-n1 {\n  margin-left: -0.25rem !important;\n}\n\n.m-n2 {\n  margin: -0.5rem !important;\n}\n\n.mt-n2,\n.my-n2 {\n  margin-top: -0.5rem !important;\n}\n\n.mr-n2,\n.mx-n2 {\n  margin-right: -0.5rem !important;\n}\n\n.mb-n2,\n.my-n2 {\n  margin-bottom: -0.5rem !important;\n}\n\n.ml-n2,\n.mx-n2 {\n  margin-left: -0.5rem !important;\n}\n\n.m-n3 {\n  margin: -1rem !important;\n}\n\n.mt-n3,\n.my-n3 {\n  margin-top: -1rem !important;\n}\n\n.mr-n3,\n.mx-n3 {\n  margin-right: -1rem !important;\n}\n\n.mb-n3,\n.my-n3 {\n  margin-bottom: -1rem !important;\n}\n\n.ml-n3,\n.mx-n3 {\n  margin-left: -1rem !important;\n}\n\n.m-n4 {\n  margin: -1.5rem !important;\n}\n\n.mt-n4,\n.my-n4 {\n  margin-top: -1.5rem !important;\n}\n\n.mr-n4,\n.mx-n4 {\n  margin-right: -1.5rem !important;\n}\n\n.mb-n4,\n.my-n4 {\n  margin-bottom: -1.5rem !important;\n}\n\n.ml-n4,\n.mx-n4 {\n  margin-left: -1.5rem !important;\n}\n\n.m-n5 {\n  margin: -3rem !important;\n}\n\n.mt-n5,\n.my-n5 {\n  margin-top: -3rem !important;\n}\n\n.mr-n5,\n.mx-n5 {\n  margin-right: -3rem !important;\n}\n\n.mb-n5,\n.my-n5 {\n  margin-bottom: -3rem !important;\n}\n\n.ml-n5,\n.mx-n5 {\n  margin-left: -3rem !important;\n}\n\n.m-auto {\n  margin: auto !important;\n}\n\n.mt-auto,\n.my-auto {\n  margin-top: auto !important;\n}\n\n.mr-auto,\n.mx-auto {\n  margin-right: auto !important;\n}\n\n.mb-auto,\n.my-auto {\n  margin-bottom: auto !important;\n}\n\n.ml-auto,\n.mx-auto {\n  margin-left: auto !important;\n}\n\n@media (min-width: 576px) {\n  .m-sm-0 {\n    margin: 0 !important;\n  }\n  .mt-sm-0,\n  .my-sm-0 {\n    margin-top: 0 !important;\n  }\n  .mr-sm-0,\n  .mx-sm-0 {\n    margin-right: 0 !important;\n  }\n  .mb-sm-0,\n  .my-sm-0 {\n    margin-bottom: 0 !important;\n  }\n  .ml-sm-0,\n  .mx-sm-0 {\n    margin-left: 0 !important;\n  }\n  .m-sm-1 {\n    margin: 0.25rem !important;\n  }\n  .mt-sm-1,\n  .my-sm-1 {\n    margin-top: 0.25rem !important;\n  }\n  .mr-sm-1,\n  .mx-sm-1 {\n    margin-right: 0.25rem !important;\n  }\n  .mb-sm-1,\n  .my-sm-1 {\n    margin-bottom: 0.25rem !important;\n  }\n  .ml-sm-1,\n  .mx-sm-1 {\n    margin-left: 0.25rem !important;\n  }\n  .m-sm-2 {\n    margin: 0.5rem !important;\n  }\n  .mt-sm-2,\n  .my-sm-2 {\n    margin-top: 0.5rem !important;\n  }\n  .mr-sm-2,\n  .mx-sm-2 {\n    margin-right: 0.5rem !important;\n  }\n  .mb-sm-2,\n  .my-sm-2 {\n    margin-bottom: 0.5rem !important;\n  }\n  .ml-sm-2,\n  .mx-sm-2 {\n    margin-left: 0.5rem !important;\n  }\n  .m-sm-3 {\n    margin: 1rem !important;\n  }\n  .mt-sm-3,\n  .my-sm-3 {\n    margin-top: 1rem !important;\n  }\n  .mr-sm-3,\n  .mx-sm-3 {\n    margin-right: 1rem !important;\n  }\n  .mb-sm-3,\n  .my-sm-3 {\n    margin-bottom: 1rem !important;\n  }\n  .ml-sm-3,\n  .mx-sm-3 {\n    margin-left: 1rem !important;\n  }\n  .m-sm-4 {\n    margin: 1.5rem !important;\n  }\n  .mt-sm-4,\n  .my-sm-4 {\n    margin-top: 1.5rem !important;\n  }\n  .mr-sm-4,\n  .mx-sm-4 {\n    margin-right: 1.5rem !important;\n  }\n  .mb-sm-4,\n  .my-sm-4 {\n    margin-bottom: 1.5rem !important;\n  }\n  .ml-sm-4,\n  .mx-sm-4 {\n    margin-left: 1.5rem !important;\n  }\n  .m-sm-5 {\n    margin: 3rem !important;\n  }\n  .mt-sm-5,\n  .my-sm-5 {\n    margin-top: 3rem !important;\n  }\n  .mr-sm-5,\n  .mx-sm-5 {\n    margin-right: 3rem !important;\n  }\n  .mb-sm-5,\n  .my-sm-5 {\n    margin-bottom: 3rem !important;\n  }\n  .ml-sm-5,\n  .mx-sm-5 {\n    margin-left: 3rem !important;\n  }\n  .p-sm-0 {\n    padding: 0 !important;\n  }\n  .pt-sm-0,\n  .py-sm-0 {\n    padding-top: 0 !important;\n  }\n  .pr-sm-0,\n  .px-sm-0 {\n    padding-right: 0 !important;\n  }\n  .pb-sm-0,\n  .py-sm-0 {\n    padding-bottom: 0 !important;\n  }\n  .pl-sm-0,\n  .px-sm-0 {\n    padding-left: 0 !important;\n  }\n  .p-sm-1 {\n    padding: 0.25rem !important;\n  }\n  .pt-sm-1,\n  .py-sm-1 {\n    padding-top: 0.25rem !important;\n  }\n  .pr-sm-1,\n  .px-sm-1 {\n    padding-right: 0.25rem !important;\n  }\n  .pb-sm-1,\n  .py-sm-1 {\n    padding-bottom: 0.25rem !important;\n  }\n  .pl-sm-1,\n  .px-sm-1 {\n    padding-left: 0.25rem !important;\n  }\n  .p-sm-2 {\n    padding: 0.5rem !important;\n  }\n  .pt-sm-2,\n  .py-sm-2 {\n    padding-top: 0.5rem !important;\n  }\n  .pr-sm-2,\n  .px-sm-2 {\n    padding-right: 0.5rem !important;\n  }\n  .pb-sm-2,\n  .py-sm-2 {\n    padding-bottom: 0.5rem !important;\n  }\n  .pl-sm-2,\n  .px-sm-2 {\n    padding-left: 0.5rem !important;\n  }\n  .p-sm-3 {\n    padding: 1rem !important;\n  }\n  .pt-sm-3,\n  .py-sm-3 {\n    padding-top: 1rem !important;\n  }\n  .pr-sm-3,\n  .px-sm-3 {\n    padding-right: 1rem !important;\n  }\n  .pb-sm-3,\n  .py-sm-3 {\n    padding-bottom: 1rem !important;\n  }\n  .pl-sm-3,\n  .px-sm-3 {\n    padding-left: 1rem !important;\n  }\n  .p-sm-4 {\n    padding: 1.5rem !important;\n  }\n  .pt-sm-4,\n  .py-sm-4 {\n    padding-top: 1.5rem !important;\n  }\n  .pr-sm-4,\n  .px-sm-4 {\n    padding-right: 1.5rem !important;\n  }\n  .pb-sm-4,\n  .py-sm-4 {\n    padding-bottom: 1.5rem !important;\n  }\n  .pl-sm-4,\n  .px-sm-4 {\n    padding-left: 1.5rem !important;\n  }\n  .p-sm-5 {\n    padding: 3rem !important;\n  }\n  .pt-sm-5,\n  .py-sm-5 {\n    padding-top: 3rem !important;\n  }\n  .pr-sm-5,\n  .px-sm-5 {\n    padding-right: 3rem !important;\n  }\n  .pb-sm-5,\n  .py-sm-5 {\n    padding-bottom: 3rem !important;\n  }\n  .pl-sm-5,\n  .px-sm-5 {\n    padding-left: 3rem !important;\n  }\n  .m-sm-n1 {\n    margin: -0.25rem !important;\n  }\n  .mt-sm-n1,\n  .my-sm-n1 {\n    margin-top: -0.25rem !important;\n  }\n  .mr-sm-n1,\n  .mx-sm-n1 {\n    margin-right: -0.25rem !important;\n  }\n  .mb-sm-n1,\n  .my-sm-n1 {\n    margin-bottom: -0.25rem !important;\n  }\n  .ml-sm-n1,\n  .mx-sm-n1 {\n    margin-left: -0.25rem !important;\n  }\n  .m-sm-n2 {\n    margin: -0.5rem !important;\n  }\n  .mt-sm-n2,\n  .my-sm-n2 {\n    margin-top: -0.5rem !important;\n  }\n  .mr-sm-n2,\n  .mx-sm-n2 {\n    margin-right: -0.5rem !important;\n  }\n  .mb-sm-n2,\n  .my-sm-n2 {\n    margin-bottom: -0.5rem !important;\n  }\n  .ml-sm-n2,\n  .mx-sm-n2 {\n    margin-left: -0.5rem !important;\n  }\n  .m-sm-n3 {\n    margin: -1rem !important;\n  }\n  .mt-sm-n3,\n  .my-sm-n3 {\n    margin-top: -1rem !important;\n  }\n  .mr-sm-n3,\n  .mx-sm-n3 {\n    margin-right: -1rem !important;\n  }\n  .mb-sm-n3,\n  .my-sm-n3 {\n    margin-bottom: -1rem !important;\n  }\n  .ml-sm-n3,\n  .mx-sm-n3 {\n    margin-left: -1rem !important;\n  }\n  .m-sm-n4 {\n    margin: -1.5rem !important;\n  }\n  .mt-sm-n4,\n  .my-sm-n4 {\n    margin-top: -1.5rem !important;\n  }\n  .mr-sm-n4,\n  .mx-sm-n4 {\n    margin-right: -1.5rem !important;\n  }\n  .mb-sm-n4,\n  .my-sm-n4 {\n    margin-bottom: -1.5rem !important;\n  }\n  .ml-sm-n4,\n  .mx-sm-n4 {\n    margin-left: -1.5rem !important;\n  }\n  .m-sm-n5 {\n    margin: -3rem !important;\n  }\n  .mt-sm-n5,\n  .my-sm-n5 {\n    margin-top: -3rem !important;\n  }\n  .mr-sm-n5,\n  .mx-sm-n5 {\n    margin-right: -3rem !important;\n  }\n  .mb-sm-n5,\n  .my-sm-n5 {\n    margin-bottom: -3rem !important;\n  }\n  .ml-sm-n5,\n  .mx-sm-n5 {\n    margin-left: -3rem !important;\n  }\n  .m-sm-auto {\n    margin: auto !important;\n  }\n  .mt-sm-auto,\n  .my-sm-auto {\n    margin-top: auto !important;\n  }\n  .mr-sm-auto,\n  .mx-sm-auto {\n    margin-right: auto !important;\n  }\n  .mb-sm-auto,\n  .my-sm-auto {\n    margin-bottom: auto !important;\n  }\n  .ml-sm-auto,\n  .mx-sm-auto {\n    margin-left: auto !important;\n  }\n}\n\n@media (min-width: 768px) {\n  .m-md-0 {\n    margin: 0 !important;\n  }\n  .mt-md-0,\n  .my-md-0 {\n    margin-top: 0 !important;\n  }\n  .mr-md-0,\n  .mx-md-0 {\n    margin-right: 0 !important;\n  }\n  .mb-md-0,\n  .my-md-0 {\n    margin-bottom: 0 !important;\n  }\n  .ml-md-0,\n  .mx-md-0 {\n    margin-left: 0 !important;\n  }\n  .m-md-1 {\n    margin: 0.25rem !important;\n  }\n  .mt-md-1,\n  .my-md-1 {\n    margin-top: 0.25rem !important;\n  }\n  .mr-md-1,\n  .mx-md-1 {\n    margin-right: 0.25rem !important;\n  }\n  .mb-md-1,\n  .my-md-1 {\n    margin-bottom: 0.25rem !important;\n  }\n  .ml-md-1,\n  .mx-md-1 {\n    margin-left: 0.25rem !important;\n  }\n  .m-md-2 {\n    margin: 0.5rem !important;\n  }\n  .mt-md-2,\n  .my-md-2 {\n    margin-top: 0.5rem !important;\n  }\n  .mr-md-2,\n  .mx-md-2 {\n    margin-right: 0.5rem !important;\n  }\n  .mb-md-2,\n  .my-md-2 {\n    margin-bottom: 0.5rem !important;\n  }\n  .ml-md-2,\n  .mx-md-2 {\n    margin-left: 0.5rem !important;\n  }\n  .m-md-3 {\n    margin: 1rem !important;\n  }\n  .mt-md-3,\n  .my-md-3 {\n    margin-top: 1rem !important;\n  }\n  .mr-md-3,\n  .mx-md-3 {\n    margin-right: 1rem !important;\n  }\n  .mb-md-3,\n  .my-md-3 {\n    margin-bottom: 1rem !important;\n  }\n  .ml-md-3,\n  .mx-md-3 {\n    margin-left: 1rem !important;\n  }\n  .m-md-4 {\n    margin: 1.5rem !important;\n  }\n  .mt-md-4,\n  .my-md-4 {\n    margin-top: 1.5rem !important;\n  }\n  .mr-md-4,\n  .mx-md-4 {\n    margin-right: 1.5rem !important;\n  }\n  .mb-md-4,\n  .my-md-4 {\n    margin-bottom: 1.5rem !important;\n  }\n  .ml-md-4,\n  .mx-md-4 {\n    margin-left: 1.5rem !important;\n  }\n  .m-md-5 {\n    margin: 3rem !important;\n  }\n  .mt-md-5,\n  .my-md-5 {\n    margin-top: 3rem !important;\n  }\n  .mr-md-5,\n  .mx-md-5 {\n    margin-right: 3rem !important;\n  }\n  .mb-md-5,\n  .my-md-5 {\n    margin-bottom: 3rem !important;\n  }\n  .ml-md-5,\n  .mx-md-5 {\n    margin-left: 3rem !important;\n  }\n  .p-md-0 {\n    padding: 0 !important;\n  }\n  .pt-md-0,\n  .py-md-0 {\n    padding-top: 0 !important;\n  }\n  .pr-md-0,\n  .px-md-0 {\n    padding-right: 0 !important;\n  }\n  .pb-md-0,\n  .py-md-0 {\n    padding-bottom: 0 !important;\n  }\n  .pl-md-0,\n  .px-md-0 {\n    padding-left: 0 !important;\n  }\n  .p-md-1 {\n    padding: 0.25rem !important;\n  }\n  .pt-md-1,\n  .py-md-1 {\n    padding-top: 0.25rem !important;\n  }\n  .pr-md-1,\n  .px-md-1 {\n    padding-right: 0.25rem !important;\n  }\n  .pb-md-1,\n  .py-md-1 {\n    padding-bottom: 0.25rem !important;\n  }\n  .pl-md-1,\n  .px-md-1 {\n    padding-left: 0.25rem !important;\n  }\n  .p-md-2 {\n    padding: 0.5rem !important;\n  }\n  .pt-md-2,\n  .py-md-2 {\n    padding-top: 0.5rem !important;\n  }\n  .pr-md-2,\n  .px-md-2 {\n    padding-right: 0.5rem !important;\n  }\n  .pb-md-2,\n  .py-md-2 {\n    padding-bottom: 0.5rem !important;\n  }\n  .pl-md-2,\n  .px-md-2 {\n    padding-left: 0.5rem !important;\n  }\n  .p-md-3 {\n    padding: 1rem !important;\n  }\n  .pt-md-3,\n  .py-md-3 {\n    padding-top: 1rem !important;\n  }\n  .pr-md-3,\n  .px-md-3 {\n    padding-right: 1rem !important;\n  }\n  .pb-md-3,\n  .py-md-3 {\n    padding-bottom: 1rem !important;\n  }\n  .pl-md-3,\n  .px-md-3 {\n    padding-left: 1rem !important;\n  }\n  .p-md-4 {\n    padding: 1.5rem !important;\n  }\n  .pt-md-4,\n  .py-md-4 {\n    padding-top: 1.5rem !important;\n  }\n  .pr-md-4,\n  .px-md-4 {\n    padding-right: 1.5rem !important;\n  }\n  .pb-md-4,\n  .py-md-4 {\n    padding-bottom: 1.5rem !important;\n  }\n  .pl-md-4,\n  .px-md-4 {\n    padding-left: 1.5rem !important;\n  }\n  .p-md-5 {\n    padding: 3rem !important;\n  }\n  .pt-md-5,\n  .py-md-5 {\n    padding-top: 3rem !important;\n  }\n  .pr-md-5,\n  .px-md-5 {\n    padding-right: 3rem !important;\n  }\n  .pb-md-5,\n  .py-md-5 {\n    padding-bottom: 3rem !important;\n  }\n  .pl-md-5,\n  .px-md-5 {\n    padding-left: 3rem !important;\n  }\n  .m-md-n1 {\n    margin: -0.25rem !important;\n  }\n  .mt-md-n1,\n  .my-md-n1 {\n    margin-top: -0.25rem !important;\n  }\n  .mr-md-n1,\n  .mx-md-n1 {\n    margin-right: -0.25rem !important;\n  }\n  .mb-md-n1,\n  .my-md-n1 {\n    margin-bottom: -0.25rem !important;\n  }\n  .ml-md-n1,\n  .mx-md-n1 {\n    margin-left: -0.25rem !important;\n  }\n  .m-md-n2 {\n    margin: -0.5rem !important;\n  }\n  .mt-md-n2,\n  .my-md-n2 {\n    margin-top: -0.5rem !important;\n  }\n  .mr-md-n2,\n  .mx-md-n2 {\n    margin-right: -0.5rem !important;\n  }\n  .mb-md-n2,\n  .my-md-n2 {\n    margin-bottom: -0.5rem !important;\n  }\n  .ml-md-n2,\n  .mx-md-n2 {\n    margin-left: -0.5rem !important;\n  }\n  .m-md-n3 {\n    margin: -1rem !important;\n  }\n  .mt-md-n3,\n  .my-md-n3 {\n    margin-top: -1rem !important;\n  }\n  .mr-md-n3,\n  .mx-md-n3 {\n    margin-right: -1rem !important;\n  }\n  .mb-md-n3,\n  .my-md-n3 {\n    margin-bottom: -1rem !important;\n  }\n  .ml-md-n3,\n  .mx-md-n3 {\n    margin-left: -1rem !important;\n  }\n  .m-md-n4 {\n    margin: -1.5rem !important;\n  }\n  .mt-md-n4,\n  .my-md-n4 {\n    margin-top: -1.5rem !important;\n  }\n  .mr-md-n4,\n  .mx-md-n4 {\n    margin-right: -1.5rem !important;\n  }\n  .mb-md-n4,\n  .my-md-n4 {\n    margin-bottom: -1.5rem !important;\n  }\n  .ml-md-n4,\n  .mx-md-n4 {\n    margin-left: -1.5rem !important;\n  }\n  .m-md-n5 {\n    margin: -3rem !important;\n  }\n  .mt-md-n5,\n  .my-md-n5 {\n    margin-top: -3rem !important;\n  }\n  .mr-md-n5,\n  .mx-md-n5 {\n    margin-right: -3rem !important;\n  }\n  .mb-md-n5,\n  .my-md-n5 {\n    margin-bottom: -3rem !important;\n  }\n  .ml-md-n5,\n  .mx-md-n5 {\n    margin-left: -3rem !important;\n  }\n  .m-md-auto {\n    margin: auto !important;\n  }\n  .mt-md-auto,\n  .my-md-auto {\n    margin-top: auto !important;\n  }\n  .mr-md-auto,\n  .mx-md-auto {\n    margin-right: auto !important;\n  }\n  .mb-md-auto,\n  .my-md-auto {\n    margin-bottom: auto !important;\n  }\n  .ml-md-auto,\n  .mx-md-auto {\n    margin-left: auto !important;\n  }\n}\n\n@media (min-width: 992px) {\n  .m-lg-0 {\n    margin: 0 !important;\n  }\n  .mt-lg-0,\n  .my-lg-0 {\n    margin-top: 0 !important;\n  }\n  .mr-lg-0,\n  .mx-lg-0 {\n    margin-right: 0 !important;\n  }\n  .mb-lg-0,\n  .my-lg-0 {\n    margin-bottom: 0 !important;\n  }\n  .ml-lg-0,\n  .mx-lg-0 {\n    margin-left: 0 !important;\n  }\n  .m-lg-1 {\n    margin: 0.25rem !important;\n  }\n  .mt-lg-1,\n  .my-lg-1 {\n    margin-top: 0.25rem !important;\n  }\n  .mr-lg-1,\n  .mx-lg-1 {\n    margin-right: 0.25rem !important;\n  }\n  .mb-lg-1,\n  .my-lg-1 {\n    margin-bottom: 0.25rem !important;\n  }\n  .ml-lg-1,\n  .mx-lg-1 {\n    margin-left: 0.25rem !important;\n  }\n  .m-lg-2 {\n    margin: 0.5rem !important;\n  }\n  .mt-lg-2,\n  .my-lg-2 {\n    margin-top: 0.5rem !important;\n  }\n  .mr-lg-2,\n  .mx-lg-2 {\n    margin-right: 0.5rem !important;\n  }\n  .mb-lg-2,\n  .my-lg-2 {\n    margin-bottom: 0.5rem !important;\n  }\n  .ml-lg-2,\n  .mx-lg-2 {\n    margin-left: 0.5rem !important;\n  }\n  .m-lg-3 {\n    margin: 1rem !important;\n  }\n  .mt-lg-3,\n  .my-lg-3 {\n    margin-top: 1rem !important;\n  }\n  .mr-lg-3,\n  .mx-lg-3 {\n    margin-right: 1rem !important;\n  }\n  .mb-lg-3,\n  .my-lg-3 {\n    margin-bottom: 1rem !important;\n  }\n  .ml-lg-3,\n  .mx-lg-3 {\n    margin-left: 1rem !important;\n  }\n  .m-lg-4 {\n    margin: 1.5rem !important;\n  }\n  .mt-lg-4,\n  .my-lg-4 {\n    margin-top: 1.5rem !important;\n  }\n  .mr-lg-4,\n  .mx-lg-4 {\n    margin-right: 1.5rem !important;\n  }\n  .mb-lg-4,\n  .my-lg-4 {\n    margin-bottom: 1.5rem !important;\n  }\n  .ml-lg-4,\n  .mx-lg-4 {\n    margin-left: 1.5rem !important;\n  }\n  .m-lg-5 {\n    margin: 3rem !important;\n  }\n  .mt-lg-5,\n  .my-lg-5 {\n    margin-top: 3rem !important;\n  }\n  .mr-lg-5,\n  .mx-lg-5 {\n    margin-right: 3rem !important;\n  }\n  .mb-lg-5,\n  .my-lg-5 {\n    margin-bottom: 3rem !important;\n  }\n  .ml-lg-5,\n  .mx-lg-5 {\n    margin-left: 3rem !important;\n  }\n  .p-lg-0 {\n    padding: 0 !important;\n  }\n  .pt-lg-0,\n  .py-lg-0 {\n    padding-top: 0 !important;\n  }\n  .pr-lg-0,\n  .px-lg-0 {\n    padding-right: 0 !important;\n  }\n  .pb-lg-0,\n  .py-lg-0 {\n    padding-bottom: 0 !important;\n  }\n  .pl-lg-0,\n  .px-lg-0 {\n    padding-left: 0 !important;\n  }\n  .p-lg-1 {\n    padding: 0.25rem !important;\n  }\n  .pt-lg-1,\n  .py-lg-1 {\n    padding-top: 0.25rem !important;\n  }\n  .pr-lg-1,\n  .px-lg-1 {\n    padding-right: 0.25rem !important;\n  }\n  .pb-lg-1,\n  .py-lg-1 {\n    padding-bottom: 0.25rem !important;\n  }\n  .pl-lg-1,\n  .px-lg-1 {\n    padding-left: 0.25rem !important;\n  }\n  .p-lg-2 {\n    padding: 0.5rem !important;\n  }\n  .pt-lg-2,\n  .py-lg-2 {\n    padding-top: 0.5rem !important;\n  }\n  .pr-lg-2,\n  .px-lg-2 {\n    padding-right: 0.5rem !important;\n  }\n  .pb-lg-2,\n  .py-lg-2 {\n    padding-bottom: 0.5rem !important;\n  }\n  .pl-lg-2,\n  .px-lg-2 {\n    padding-left: 0.5rem !important;\n  }\n  .p-lg-3 {\n    padding: 1rem !important;\n  }\n  .pt-lg-3,\n  .py-lg-3 {\n    padding-top: 1rem !important;\n  }\n  .pr-lg-3,\n  .px-lg-3 {\n    padding-right: 1rem !important;\n  }\n  .pb-lg-3,\n  .py-lg-3 {\n    padding-bottom: 1rem !important;\n  }\n  .pl-lg-3,\n  .px-lg-3 {\n    padding-left: 1rem !important;\n  }\n  .p-lg-4 {\n    padding: 1.5rem !important;\n  }\n  .pt-lg-4,\n  .py-lg-4 {\n    padding-top: 1.5rem !important;\n  }\n  .pr-lg-4,\n  .px-lg-4 {\n    padding-right: 1.5rem !important;\n  }\n  .pb-lg-4,\n  .py-lg-4 {\n    padding-bottom: 1.5rem !important;\n  }\n  .pl-lg-4,\n  .px-lg-4 {\n    padding-left: 1.5rem !important;\n  }\n  .p-lg-5 {\n    padding: 3rem !important;\n  }\n  .pt-lg-5,\n  .py-lg-5 {\n    padding-top: 3rem !important;\n  }\n  .pr-lg-5,\n  .px-lg-5 {\n    padding-right: 3rem !important;\n  }\n  .pb-lg-5,\n  .py-lg-5 {\n    padding-bottom: 3rem !important;\n  }\n  .pl-lg-5,\n  .px-lg-5 {\n    padding-left: 3rem !important;\n  }\n  .m-lg-n1 {\n    margin: -0.25rem !important;\n  }\n  .mt-lg-n1,\n  .my-lg-n1 {\n    margin-top: -0.25rem !important;\n  }\n  .mr-lg-n1,\n  .mx-lg-n1 {\n    margin-right: -0.25rem !important;\n  }\n  .mb-lg-n1,\n  .my-lg-n1 {\n    margin-bottom: -0.25rem !important;\n  }\n  .ml-lg-n1,\n  .mx-lg-n1 {\n    margin-left: -0.25rem !important;\n  }\n  .m-lg-n2 {\n    margin: -0.5rem !important;\n  }\n  .mt-lg-n2,\n  .my-lg-n2 {\n    margin-top: -0.5rem !important;\n  }\n  .mr-lg-n2,\n  .mx-lg-n2 {\n    margin-right: -0.5rem !important;\n  }\n  .mb-lg-n2,\n  .my-lg-n2 {\n    margin-bottom: -0.5rem !important;\n  }\n  .ml-lg-n2,\n  .mx-lg-n2 {\n    margin-left: -0.5rem !important;\n  }\n  .m-lg-n3 {\n    margin: -1rem !important;\n  }\n  .mt-lg-n3,\n  .my-lg-n3 {\n    margin-top: -1rem !important;\n  }\n  .mr-lg-n3,\n  .mx-lg-n3 {\n    margin-right: -1rem !important;\n  }\n  .mb-lg-n3,\n  .my-lg-n3 {\n    margin-bottom: -1rem !important;\n  }\n  .ml-lg-n3,\n  .mx-lg-n3 {\n    margin-left: -1rem !important;\n  }\n  .m-lg-n4 {\n    margin: -1.5rem !important;\n  }\n  .mt-lg-n4,\n  .my-lg-n4 {\n    margin-top: -1.5rem !important;\n  }\n  .mr-lg-n4,\n  .mx-lg-n4 {\n    margin-right: -1.5rem !important;\n  }\n  .mb-lg-n4,\n  .my-lg-n4 {\n    margin-bottom: -1.5rem !important;\n  }\n  .ml-lg-n4,\n  .mx-lg-n4 {\n    margin-left: -1.5rem !important;\n  }\n  .m-lg-n5 {\n    margin: -3rem !important;\n  }\n  .mt-lg-n5,\n  .my-lg-n5 {\n    margin-top: -3rem !important;\n  }\n  .mr-lg-n5,\n  .mx-lg-n5 {\n    margin-right: -3rem !important;\n  }\n  .mb-lg-n5,\n  .my-lg-n5 {\n    margin-bottom: -3rem !important;\n  }\n  .ml-lg-n5,\n  .mx-lg-n5 {\n    margin-left: -3rem !important;\n  }\n  .m-lg-auto {\n    margin: auto !important;\n  }\n  .mt-lg-auto,\n  .my-lg-auto {\n    margin-top: auto !important;\n  }\n  .mr-lg-auto,\n  .mx-lg-auto {\n    margin-right: auto !important;\n  }\n  .mb-lg-auto,\n  .my-lg-auto {\n    margin-bottom: auto !important;\n  }\n  .ml-lg-auto,\n  .mx-lg-auto {\n    margin-left: auto !important;\n  }\n}\n\n@media (min-width: 1200px) {\n  .m-xl-0 {\n    margin: 0 !important;\n  }\n  .mt-xl-0,\n  .my-xl-0 {\n    margin-top: 0 !important;\n  }\n  .mr-xl-0,\n  .mx-xl-0 {\n    margin-right: 0 !important;\n  }\n  .mb-xl-0,\n  .my-xl-0 {\n    margin-bottom: 0 !important;\n  }\n  .ml-xl-0,\n  .mx-xl-0 {\n    margin-left: 0 !important;\n  }\n  .m-xl-1 {\n    margin: 0.25rem !important;\n  }\n  .mt-xl-1,\n  .my-xl-1 {\n    margin-top: 0.25rem !important;\n  }\n  .mr-xl-1,\n  .mx-xl-1 {\n    margin-right: 0.25rem !important;\n  }\n  .mb-xl-1,\n  .my-xl-1 {\n    margin-bottom: 0.25rem !important;\n  }\n  .ml-xl-1,\n  .mx-xl-1 {\n    margin-left: 0.25rem !important;\n  }\n  .m-xl-2 {\n    margin: 0.5rem !important;\n  }\n  .mt-xl-2,\n  .my-xl-2 {\n    margin-top: 0.5rem !important;\n  }\n  .mr-xl-2,\n  .mx-xl-2 {\n    margin-right: 0.5rem !important;\n  }\n  .mb-xl-2,\n  .my-xl-2 {\n    margin-bottom: 0.5rem !important;\n  }\n  .ml-xl-2,\n  .mx-xl-2 {\n    margin-left: 0.5rem !important;\n  }\n  .m-xl-3 {\n    margin: 1rem !important;\n  }\n  .mt-xl-3,\n  .my-xl-3 {\n    margin-top: 1rem !important;\n  }\n  .mr-xl-3,\n  .mx-xl-3 {\n    margin-right: 1rem !important;\n  }\n  .mb-xl-3,\n  .my-xl-3 {\n    margin-bottom: 1rem !important;\n  }\n  .ml-xl-3,\n  .mx-xl-3 {\n    margin-left: 1rem !important;\n  }\n  .m-xl-4 {\n    margin: 1.5rem !important;\n  }\n  .mt-xl-4,\n  .my-xl-4 {\n    margin-top: 1.5rem !important;\n  }\n  .mr-xl-4,\n  .mx-xl-4 {\n    margin-right: 1.5rem !important;\n  }\n  .mb-xl-4,\n  .my-xl-4 {\n    margin-bottom: 1.5rem !important;\n  }\n  .ml-xl-4,\n  .mx-xl-4 {\n    margin-left: 1.5rem !important;\n  }\n  .m-xl-5 {\n    margin: 3rem !important;\n  }\n  .mt-xl-5,\n  .my-xl-5 {\n    margin-top: 3rem !important;\n  }\n  .mr-xl-5,\n  .mx-xl-5 {\n    margin-right: 3rem !important;\n  }\n  .mb-xl-5,\n  .my-xl-5 {\n    margin-bottom: 3rem !important;\n  }\n  .ml-xl-5,\n  .mx-xl-5 {\n    margin-left: 3rem !important;\n  }\n  .p-xl-0 {\n    padding: 0 !important;\n  }\n  .pt-xl-0,\n  .py-xl-0 {\n    padding-top: 0 !important;\n  }\n  .pr-xl-0,\n  .px-xl-0 {\n    padding-right: 0 !important;\n  }\n  .pb-xl-0,\n  .py-xl-0 {\n    padding-bottom: 0 !important;\n  }\n  .pl-xl-0,\n  .px-xl-0 {\n    padding-left: 0 !important;\n  }\n  .p-xl-1 {\n    padding: 0.25rem !important;\n  }\n  .pt-xl-1,\n  .py-xl-1 {\n    padding-top: 0.25rem !important;\n  }\n  .pr-xl-1,\n  .px-xl-1 {\n    padding-right: 0.25rem !important;\n  }\n  .pb-xl-1,\n  .py-xl-1 {\n    padding-bottom: 0.25rem !important;\n  }\n  .pl-xl-1,\n  .px-xl-1 {\n    padding-left: 0.25rem !important;\n  }\n  .p-xl-2 {\n    padding: 0.5rem !important;\n  }\n  .pt-xl-2,\n  .py-xl-2 {\n    padding-top: 0.5rem !important;\n  }\n  .pr-xl-2,\n  .px-xl-2 {\n    padding-right: 0.5rem !important;\n  }\n  .pb-xl-2,\n  .py-xl-2 {\n    padding-bottom: 0.5rem !important;\n  }\n  .pl-xl-2,\n  .px-xl-2 {\n    padding-left: 0.5rem !important;\n  }\n  .p-xl-3 {\n    padding: 1rem !important;\n  }\n  .pt-xl-3,\n  .py-xl-3 {\n    padding-top: 1rem !important;\n  }\n  .pr-xl-3,\n  .px-xl-3 {\n    padding-right: 1rem !important;\n  }\n  .pb-xl-3,\n  .py-xl-3 {\n    padding-bottom: 1rem !important;\n  }\n  .pl-xl-3,\n  .px-xl-3 {\n    padding-left: 1rem !important;\n  }\n  .p-xl-4 {\n    padding: 1.5rem !important;\n  }\n  .pt-xl-4,\n  .py-xl-4 {\n    padding-top: 1.5rem !important;\n  }\n  .pr-xl-4,\n  .px-xl-4 {\n    padding-right: 1.5rem !important;\n  }\n  .pb-xl-4,\n  .py-xl-4 {\n    padding-bottom: 1.5rem !important;\n  }\n  .pl-xl-4,\n  .px-xl-4 {\n    padding-left: 1.5rem !important;\n  }\n  .p-xl-5 {\n    padding: 3rem !important;\n  }\n  .pt-xl-5,\n  .py-xl-5 {\n    padding-top: 3rem !important;\n  }\n  .pr-xl-5,\n  .px-xl-5 {\n    padding-right: 3rem !important;\n  }\n  .pb-xl-5,\n  .py-xl-5 {\n    padding-bottom: 3rem !important;\n  }\n  .pl-xl-5,\n  .px-xl-5 {\n    padding-left: 3rem !important;\n  }\n  .m-xl-n1 {\n    margin: -0.25rem !important;\n  }\n  .mt-xl-n1,\n  .my-xl-n1 {\n    margin-top: -0.25rem !important;\n  }\n  .mr-xl-n1,\n  .mx-xl-n1 {\n    margin-right: -0.25rem !important;\n  }\n  .mb-xl-n1,\n  .my-xl-n1 {\n    margin-bottom: -0.25rem !important;\n  }\n  .ml-xl-n1,\n  .mx-xl-n1 {\n    margin-left: -0.25rem !important;\n  }\n  .m-xl-n2 {\n    margin: -0.5rem !important;\n  }\n  .mt-xl-n2,\n  .my-xl-n2 {\n    margin-top: -0.5rem !important;\n  }\n  .mr-xl-n2,\n  .mx-xl-n2 {\n    margin-right: -0.5rem !important;\n  }\n  .mb-xl-n2,\n  .my-xl-n2 {\n    margin-bottom: -0.5rem !important;\n  }\n  .ml-xl-n2,\n  .mx-xl-n2 {\n    margin-left: -0.5rem !important;\n  }\n  .m-xl-n3 {\n    margin: -1rem !important;\n  }\n  .mt-xl-n3,\n  .my-xl-n3 {\n    margin-top: -1rem !important;\n  }\n  .mr-xl-n3,\n  .mx-xl-n3 {\n    margin-right: -1rem !important;\n  }\n  .mb-xl-n3,\n  .my-xl-n3 {\n    margin-bottom: -1rem !important;\n  }\n  .ml-xl-n3,\n  .mx-xl-n3 {\n    margin-left: -1rem !important;\n  }\n  .m-xl-n4 {\n    margin: -1.5rem !important;\n  }\n  .mt-xl-n4,\n  .my-xl-n4 {\n    margin-top: -1.5rem !important;\n  }\n  .mr-xl-n4,\n  .mx-xl-n4 {\n    margin-right: -1.5rem !important;\n  }\n  .mb-xl-n4,\n  .my-xl-n4 {\n    margin-bottom: -1.5rem !important;\n  }\n  .ml-xl-n4,\n  .mx-xl-n4 {\n    margin-left: -1.5rem !important;\n  }\n  .m-xl-n5 {\n    margin: -3rem !important;\n  }\n  .mt-xl-n5,\n  .my-xl-n5 {\n    margin-top: -3rem !important;\n  }\n  .mr-xl-n5,\n  .mx-xl-n5 {\n    margin-right: -3rem !important;\n  }\n  .mb-xl-n5,\n  .my-xl-n5 {\n    margin-bottom: -3rem !important;\n  }\n  .ml-xl-n5,\n  .mx-xl-n5 {\n    margin-left: -3rem !important;\n  }\n  .m-xl-auto {\n    margin: auto !important;\n  }\n  .mt-xl-auto,\n  .my-xl-auto {\n    margin-top: auto !important;\n  }\n  .mr-xl-auto,\n  .mx-xl-auto {\n    margin-right: auto !important;\n  }\n  .mb-xl-auto,\n  .my-xl-auto {\n    margin-bottom: auto !important;\n  }\n  .ml-xl-auto,\n  .mx-xl-auto {\n    margin-left: auto !important;\n  }\n}\n\n.text-monospace {\n  font-family: SFMono-Regular, Menlo, Monaco, Consolas, \"Liberation Mono\", \"Courier New\", monospace !important;\n}\n\n.text-justify {\n  text-align: justify !important;\n}\n\n.text-wrap {\n  white-space: normal !important;\n}\n\n.text-nowrap {\n  white-space: nowrap !important;\n}\n\n.text-truncate {\n  overflow: hidden;\n  text-overflow: ellipsis;\n  white-space: nowrap;\n}\n\n.text-left {\n  text-align: left !important;\n}\n\n.text-right {\n  text-align: right !important;\n}\n\n.text-center {\n  text-align: center !important;\n}\n\n@media (min-width: 576px) {\n  .text-sm-left {\n    text-align: left !important;\n  }\n  .text-sm-right {\n    text-align: right !important;\n  }\n  .text-sm-center {\n    text-align: center !important;\n  }\n}\n\n@media (min-width: 768px) {\n  .text-md-left {\n    text-align: left !important;\n  }\n  .text-md-right {\n    text-align: right !important;\n  }\n  .text-md-center {\n    text-align: center !important;\n  }\n}\n\n@media (min-width: 992px) {\n  .text-lg-left {\n    text-align: left !important;\n  }\n  .text-lg-right {\n    text-align: right !important;\n  }\n  .text-lg-center {\n    text-align: center !important;\n  }\n}\n\n@media (min-width: 1200px) {\n  .text-xl-left {\n    text-align: left !important;\n  }\n  .text-xl-right {\n    text-align: right !important;\n  }\n  .text-xl-center {\n    text-align: center !important;\n  }\n}\n\n.text-lowercase {\n  text-transform: lowercase !important;\n}\n\n.text-uppercase {\n  text-transform: uppercase !important;\n}\n\n.text-capitalize {\n  text-transform: capitalize !important;\n}\n\n.font-weight-light {\n  font-weight: 300 !important;\n}\n\n.font-weight-lighter {\n  font-weight: lighter !important;\n}\n\n.font-weight-normal {\n  font-weight: 400 !important;\n}\n\n.font-weight-bold {\n  font-weight: 700 !important;\n}\n\n.font-weight-bolder {\n  font-weight: bolder !important;\n}\n\n.font-italic {\n  font-style: italic !important;\n}\n\n.text-white {\n  color: #fff !important;\n}\n\n.text-primary {\n  color: #007bff !important;\n}\n\na.text-primary:hover, a.text-primary:focus {\n  color: #0056b3 !important;\n}\n\n.text-secondary {\n  color: #6c757d !important;\n}\n\na.text-secondary:hover, a.text-secondary:focus {\n  color: #494f54 !important;\n}\n\n.text-success {\n  color: #28a745 !important;\n}\n\na.text-success:hover, a.text-success:focus {\n  color: #19692c !important;\n}\n\n.text-info {\n  color: #17a2b8 !important;\n}\n\na.text-info:hover, a.text-info:focus {\n  color: #0f6674 !important;\n}\n\n.text-warning {\n  color: #ffc107 !important;\n}\n\na.text-warning:hover, a.text-warning:focus {\n  color: #ba8b00 !important;\n}\n\n.text-danger {\n  color: #dc3545 !important;\n}\n\na.text-danger:hover, a.text-danger:focus {\n  color: #a71d2a !important;\n}\n\n.text-light {\n  color: #f8f9fa !important;\n}\n\na.text-light:hover, a.text-light:focus {\n  color: #cbd3da !important;\n}\n\n.text-dark {\n  color: #343a40 !important;\n}\n\na.text-dark:hover, a.text-dark:focus {\n  color: #121416 !important;\n}\n\n.text-body {\n  color: #212529 !important;\n}\n\n.text-muted {\n  color: #6c757d !important;\n}\n\n.text-black-50 {\n  color: rgba(0, 0, 0, 0.5) !important;\n}\n\n.text-white-50 {\n  color: rgba(255, 255, 255, 0.5) !important;\n}\n\n.text-hide {\n  font: 0/0 a;\n  color: transparent;\n  text-shadow: none;\n  background-color: transparent;\n  border: 0;\n}\n\n.text-decoration-none {\n  text-decoration: none !important;\n}\n\n.text-break {\n  word-break: break-word !important;\n  overflow-wrap: break-word !important;\n}\n\n.text-reset {\n  color: inherit !important;\n}\n\n.visible {\n  visibility: visible !important;\n}\n\n.invisible {\n  visibility: hidden !important;\n}\n\n@media print {\n  *,\n  *::before,\n  *::after {\n    text-shadow: none !important;\n    box-shadow: none !important;\n  }\n  a:not(.btn) {\n    text-decoration: underline;\n  }\n  abbr[title]::after {\n    content: \" (\" attr(title) \")\";\n  }\n  pre {\n    white-space: pre-wrap !important;\n  }\n  pre,\n  blockquote {\n    border: 1px solid #adb5bd;\n    page-break-inside: avoid;\n  }\n  thead {\n    display: table-header-group;\n  }\n  tr,\n  img {\n    page-break-inside: avoid;\n  }\n  p,\n  h2,\n  h3 {\n    orphans: 3;\n    widows: 3;\n  }\n  h2,\n  h3 {\n    page-break-after: avoid;\n  }\n  @page {\n    size: a3;\n  }\n  body {\n    min-width: 992px !important;\n  }\n  .container {\n    min-width: 992px !important;\n  }\n  .navbar {\n    display: none;\n  }\n  .badge {\n    border: 1px solid #000;\n  }\n  .table {\n    border-collapse: collapse !important;\n  }\n  .table td,\n  .table th {\n    background-color: #fff !important;\n  }\n  .table-bordered th,\n  .table-bordered td {\n    border: 1px solid #dee2e6 !important;\n  }\n  .table-dark {\n    color: inherit;\n  }\n  .table-dark th,\n  .table-dark td,\n  .table-dark thead th,\n  .table-dark tbody + tbody {\n    border-color: #dee2e6;\n  }\n  .table .thead-dark th {\n    color: inherit;\n    border-color: #dee2e6;\n  }\n}\n/*# sourceMappingURL=bootstrap.css.map */","// stylelint-disable property-blacklist, scss/dollar-variable-default\n\n// SCSS RFS mixin\n//\n// Automated font-resizing\n//\n// See https://github.com/twbs/rfs\n\n// Configuration\n\n// Base font size\n$rfs-base-font-size: 1.25rem !default;\n$rfs-font-size-unit: rem !default;\n\n// Breakpoint at where font-size starts decreasing if screen width is smaller\n$rfs-breakpoint: 1200px !default;\n$rfs-breakpoint-unit: px !default;\n\n// Resize font-size based on screen height and width\n$rfs-two-dimensional: false !default;\n\n// Factor of decrease\n$rfs-factor: 10 !default;\n\n@if type-of($rfs-factor) != \"number\" or $rfs-factor <= 1 {\n  @error \"`#{$rfs-factor}` is not a valid  $rfs-factor, it must be greater than 1.\";\n}\n\n// Generate enable or disable classes. Possibilities: false, \"enable\" or \"disable\"\n$rfs-class: false !default;\n\n// 1 rem = $rfs-rem-value px\n$rfs-rem-value: 16 !default;\n\n// Safari iframe resize bug: https://github.com/twbs/rfs/issues/14\n$rfs-safari-iframe-resize-bug-fix: false !default;\n\n// Disable RFS by setting $enable-responsive-font-sizes to false\n$enable-responsive-font-sizes: true !default;\n\n// Cache $rfs-base-font-size unit\n$rfs-base-font-size-unit: unit($rfs-base-font-size);\n\n// Remove px-unit from $rfs-base-font-size for calculations\n@if $rfs-base-font-size-unit == \"px\" {\n  $rfs-base-font-size: $rfs-base-font-size / ($rfs-base-font-size * 0 + 1);\n}\n@else if $rfs-base-font-size-unit == \"rem\" {\n  $rfs-base-font-size: $rfs-base-font-size / ($rfs-base-font-size * 0 + 1 / $rfs-rem-value);\n}\n\n// Cache $rfs-breakpoint unit to prevent multiple calls\n$rfs-breakpoint-unit-cache: unit($rfs-breakpoint);\n\n// Remove unit from $rfs-breakpoint for calculations\n@if $rfs-breakpoint-unit-cache == \"px\" {\n  $rfs-breakpoint: $rfs-breakpoint / ($rfs-breakpoint * 0 + 1);\n}\n@else if $rfs-breakpoint-unit-cache == \"rem\" or $rfs-breakpoint-unit-cache == \"em\" {\n  $rfs-breakpoint: $rfs-breakpoint / ($rfs-breakpoint * 0 + 1 / $rfs-rem-value);\n}\n\n// Responsive font-size mixin\n@mixin rfs($fs, $important: false) {\n  // Cache $fs unit\n  $fs-unit: if(type-of($fs) == \"number\", unit($fs), false);\n\n  // Add !important suffix if needed\n  $rfs-suffix: if($important, \" !important\", \"\");\n\n  // If $fs isn't a number (like inherit) or $fs has a unit (not px or rem, like 1.5em) or $ is 0, just print the value\n  @if not $fs-unit or $fs-unit != \"\" and $fs-unit != \"px\" and $fs-unit != \"rem\" or $fs == 0 {\n    font-size: #{$fs}#{$rfs-suffix};\n  }\n  @else {\n    // Variables for storing static and fluid rescaling\n    $rfs-static: null;\n    $rfs-fluid: null;\n\n    // Remove px-unit from $fs for calculations\n    @if $fs-unit == \"px\" {\n      $fs: $fs / ($fs * 0 + 1);\n    }\n    @else if $fs-unit == \"rem\" {\n      $fs: $fs / ($fs * 0 + 1 / $rfs-rem-value);\n    }\n\n    // Set default font-size\n    @if $rfs-font-size-unit == rem {\n      $rfs-static: #{$fs / $rfs-rem-value}rem#{$rfs-suffix};\n    }\n    @else if $rfs-font-size-unit == px {\n      $rfs-static: #{$fs}px#{$rfs-suffix};\n    }\n    @else {\n      @error \"`#{$rfs-font-size-unit}` is not a valid unit for $rfs-font-size-unit. Use `px` or `rem`.\";\n    }\n\n    // Only add media query if font-size is bigger as the minimum font-size\n    // If $rfs-factor == 1, no rescaling will take place\n    @if $fs > $rfs-base-font-size and $enable-responsive-font-sizes {\n      $min-width: null;\n      $variable-unit: null;\n\n      // Calculate minimum font-size for given font-size\n      $fs-min: $rfs-base-font-size + ($fs - $rfs-base-font-size) / $rfs-factor;\n\n      // Calculate difference between given font-size and minimum font-size for given font-size\n      $fs-diff: $fs - $fs-min;\n\n      // Base font-size formatting\n      // No need to check if the unit is valid, because we did that before\n      $min-width: if($rfs-font-size-unit == rem, #{$fs-min / $rfs-rem-value}rem, #{$fs-min}px);\n\n      // If two-dimensional, use smallest of screen width and height\n      $variable-unit: if($rfs-two-dimensional, vmin, vw);\n\n      // Calculate the variable width between 0 and $rfs-breakpoint\n      $variable-width: #{$fs-diff * 100 / $rfs-breakpoint}#{$variable-unit};\n\n      // Set the calculated font-size.\n      $rfs-fluid: calc(#{$min-width} + #{$variable-width}) #{$rfs-suffix};\n    }\n\n    // Rendering\n    @if $rfs-fluid == null {\n      // Only render static font-size if no fluid font-size is available\n      font-size: $rfs-static;\n    }\n    @else {\n      $mq-value: null;\n\n      // RFS breakpoint formatting\n      @if $rfs-breakpoint-unit == em or $rfs-breakpoint-unit == rem {\n        $mq-value: #{$rfs-breakpoint / $rfs-rem-value}#{$rfs-breakpoint-unit};\n      }\n      @else if $rfs-breakpoint-unit == px {\n        $mq-value: #{$rfs-breakpoint}px;\n      }\n      @else {\n        @error \"`#{$rfs-breakpoint-unit}` is not a valid unit for $rfs-breakpoint-unit. Use `px`, `em` or `rem`.\";\n      }\n\n      @if $rfs-class == \"disable\" {\n        // Adding an extra class increases specificity,\n        // which prevents the media query to override the font size\n        &,\n        .disable-responsive-font-size &,\n        &.disable-responsive-font-size {\n          font-size: $rfs-static;\n        }\n      }\n      @else {\n        font-size: $rfs-static;\n      }\n\n      @if $rfs-two-dimensional {\n        @media (max-width: #{$mq-value}), (max-height: #{$mq-value}) {\n          @if $rfs-class == \"enable\" {\n            .enable-responsive-font-size &,\n            &.enable-responsive-font-size {\n              font-size: $rfs-fluid;\n            }\n          }\n          @else {\n            font-size: $rfs-fluid;\n          }\n\n          @if $rfs-safari-iframe-resize-bug-fix {\n            // stylelint-disable-next-line length-zero-no-unit\n            min-width: 0vw;\n          }\n        }\n      }\n      @else {\n        @media (max-width: #{$mq-value}) {\n          @if $rfs-class == \"enable\" {\n            .enable-responsive-font-size &,\n            &.enable-responsive-font-size {\n              font-size: $rfs-fluid;\n            }\n          }\n          @else {\n            font-size: $rfs-fluid;\n          }\n\n          @if $rfs-safari-iframe-resize-bug-fix {\n            // stylelint-disable-next-line length-zero-no-unit\n            min-width: 0vw;\n          }\n        }\n      }\n    }\n  }\n}\n\n// The font-size & responsive-font-size mixin uses RFS to rescale font sizes\n@mixin font-size($fs, $important: false) {\n  @include rfs($fs, $important);\n}\n\n@mixin responsive-font-size($fs, $important: false) {\n  @include rfs($fs, $important);\n}\n","/*!\n * Bootstrap v4.4.1 (https://getbootstrap.com/)\n * Copyright 2011-2019 The Bootstrap Authors\n * Copyright 2011-2019 Twitter, Inc.\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)\n */\n:root {\n  --blue: #007bff;\n  --indigo: #6610f2;\n  --purple: #6f42c1;\n  --pink: #e83e8c;\n  --red: #dc3545;\n  --orange: #fd7e14;\n  --yellow: #ffc107;\n  --green: #28a745;\n  --teal: #20c997;\n  --cyan: #17a2b8;\n  --white: #fff;\n  --gray: #6c757d;\n  --gray-dark: #343a40;\n  --primary: #007bff;\n  --secondary: #6c757d;\n  --success: #28a745;\n  --info: #17a2b8;\n  --warning: #ffc107;\n  --danger: #dc3545;\n  --light: #f8f9fa;\n  --dark: #343a40;\n  --breakpoint-xs: 0;\n  --breakpoint-sm: 576px;\n  --breakpoint-md: 768px;\n  --breakpoint-lg: 992px;\n  --breakpoint-xl: 1200px;\n  --font-family-sans-serif: -apple-system, BlinkMacSystemFont, \"Segoe UI\", Roboto, \"Helvetica Neue\", Arial, \"Noto Sans\", sans-serif, \"Apple Color Emoji\", \"Segoe UI Emoji\", \"Segoe UI Symbol\", \"Noto Color Emoji\";\n  --font-family-monospace: SFMono-Regular, Menlo, Monaco, Consolas, \"Liberation Mono\", \"Courier New\", monospace;\n}\n\n*,\n*::before,\n*::after {\n  box-sizing: border-box;\n}\n\nhtml {\n  font-family: sans-serif;\n  line-height: 1.15;\n  -webkit-text-size-adjust: 100%;\n  -webkit-tap-highlight-color: rgba(0, 0, 0, 0);\n}\n\narticle, aside, figcaption, figure, footer, header, hgroup, main, nav, section {\n  display: block;\n}\n\nbody {\n  margin: 0;\n  font-family: -apple-system, BlinkMacSystemFont, \"Segoe UI\", Roboto, \"Helvetica Neue\", Arial, \"Noto Sans\", sans-serif, \"Apple Color Emoji\", \"Segoe UI Emoji\", \"Segoe UI Symbol\", \"Noto Color Emoji\";\n  font-size: 1rem;\n  font-weight: 400;\n  line-height: 1.5;\n  color: #212529;\n  text-align: left;\n  background-color: #fff;\n}\n\n[tabindex=\"-1\"]:focus:not(:focus-visible) {\n  outline: 0 !important;\n}\n\nhr {\n  box-sizing: content-box;\n  height: 0;\n  overflow: visible;\n}\n\nh1, h2, h3, h4, h5, h6 {\n  margin-top: 0;\n  margin-bottom: 0.5rem;\n}\n\np {\n  margin-top: 0;\n  margin-bottom: 1rem;\n}\n\nabbr[title],\nabbr[data-original-title] {\n  text-decoration: underline;\n  text-decoration: underline dotted;\n  cursor: help;\n  border-bottom: 0;\n  text-decoration-skip-ink: none;\n}\n\naddress {\n  margin-bottom: 1rem;\n  font-style: normal;\n  line-height: inherit;\n}\n\nol,\nul,\ndl {\n  margin-top: 0;\n  margin-bottom: 1rem;\n}\n\nol ol,\nul ul,\nol ul,\nul ol {\n  margin-bottom: 0;\n}\n\ndt {\n  font-weight: 700;\n}\n\ndd {\n  margin-bottom: .5rem;\n  margin-left: 0;\n}\n\nblockquote {\n  margin: 0 0 1rem;\n}\n\nb,\nstrong {\n  font-weight: bolder;\n}\n\nsmall {\n  font-size: 80%;\n}\n\nsub,\nsup {\n  position: relative;\n  font-size: 75%;\n  line-height: 0;\n  vertical-align: baseline;\n}\n\nsub {\n  bottom: -.25em;\n}\n\nsup {\n  top: -.5em;\n}\n\na {\n  color: #007bff;\n  text-decoration: none;\n  background-color: transparent;\n}\n\na:hover {\n  color: #0056b3;\n  text-decoration: underline;\n}\n\na:not([href]) {\n  color: inherit;\n  text-decoration: none;\n}\n\na:not([href]):hover {\n  color: inherit;\n  text-decoration: none;\n}\n\npre,\ncode,\nkbd,\nsamp {\n  font-family: SFMono-Regular, Menlo, Monaco, Consolas, \"Liberation Mono\", \"Courier New\", monospace;\n  font-size: 1em;\n}\n\npre {\n  margin-top: 0;\n  margin-bottom: 1rem;\n  overflow: auto;\n}\n\nfigure {\n  margin: 0 0 1rem;\n}\n\nimg {\n  vertical-align: middle;\n  border-style: none;\n}\n\nsvg {\n  overflow: hidden;\n  vertical-align: middle;\n}\n\ntable {\n  border-collapse: collapse;\n}\n\ncaption {\n  padding-top: 0.75rem;\n  padding-bottom: 0.75rem;\n  color: #6c757d;\n  text-align: left;\n  caption-side: bottom;\n}\n\nth {\n  text-align: inherit;\n}\n\nlabel {\n  display: inline-block;\n  margin-bottom: 0.5rem;\n}\n\nbutton {\n  border-radius: 0;\n}\n\nbutton:focus {\n  outline: 1px dotted;\n  outline: 5px auto -webkit-focus-ring-color;\n}\n\ninput,\nbutton,\nselect,\noptgroup,\ntextarea {\n  margin: 0;\n  font-family: inherit;\n  font-size: inherit;\n  line-height: inherit;\n}\n\nbutton,\ninput {\n  overflow: visible;\n}\n\nbutton,\nselect {\n  text-transform: none;\n}\n\nselect {\n  word-wrap: normal;\n}\n\nbutton,\n[type=\"button\"],\n[type=\"reset\"],\n[type=\"submit\"] {\n  -webkit-appearance: button;\n}\n\nbutton:not(:disabled),\n[type=\"button\"]:not(:disabled),\n[type=\"reset\"]:not(:disabled),\n[type=\"submit\"]:not(:disabled) {\n  cursor: pointer;\n}\n\nbutton::-moz-focus-inner,\n[type=\"button\"]::-moz-focus-inner,\n[type=\"reset\"]::-moz-focus-inner,\n[type=\"submit\"]::-moz-focus-inner {\n  padding: 0;\n  border-style: none;\n}\n\ninput[type=\"radio\"],\ninput[type=\"checkbox\"] {\n  box-sizing: border-box;\n  padding: 0;\n}\n\ninput[type=\"date\"],\ninput[type=\"time\"],\ninput[type=\"datetime-local\"],\ninput[type=\"month\"] {\n  -webkit-appearance: listbox;\n}\n\ntextarea {\n  overflow: auto;\n  resize: vertical;\n}\n\nfieldset {\n  min-width: 0;\n  padding: 0;\n  margin: 0;\n  border: 0;\n}\n\nlegend {\n  display: block;\n  width: 100%;\n  max-width: 100%;\n  padding: 0;\n  margin-bottom: .5rem;\n  font-size: 1.5rem;\n  line-height: inherit;\n  color: inherit;\n  white-space: normal;\n}\n\nprogress {\n  vertical-align: baseline;\n}\n\n[type=\"number\"]::-webkit-inner-spin-button,\n[type=\"number\"]::-webkit-outer-spin-button {\n  height: auto;\n}\n\n[type=\"search\"] {\n  outline-offset: -2px;\n  -webkit-appearance: none;\n}\n\n[type=\"search\"]::-webkit-search-decoration {\n  -webkit-appearance: none;\n}\n\n::-webkit-file-upload-button {\n  font: inherit;\n  -webkit-appearance: button;\n}\n\noutput {\n  display: inline-block;\n}\n\nsummary {\n  display: list-item;\n  cursor: pointer;\n}\n\ntemplate {\n  display: none;\n}\n\n[hidden] {\n  display: none !important;\n}\n\nh1, h2, h3, h4, h5, h6,\n.h1, .h2, .h3, .h4, .h5, .h6 {\n  margin-bottom: 0.5rem;\n  font-weight: 500;\n  line-height: 1.2;\n}\n\nh1, .h1 {\n  font-size: 2.5rem;\n}\n\nh2, .h2 {\n  font-size: 2rem;\n}\n\nh3, .h3 {\n  font-size: 1.75rem;\n}\n\nh4, .h4 {\n  font-size: 1.5rem;\n}\n\nh5, .h5 {\n  font-size: 1.25rem;\n}\n\nh6, .h6 {\n  font-size: 1rem;\n}\n\n.lead {\n  font-size: 1.25rem;\n  font-weight: 300;\n}\n\n.display-1 {\n  font-size: 6rem;\n  font-weight: 300;\n  line-height: 1.2;\n}\n\n.display-2 {\n  font-size: 5.5rem;\n  font-weight: 300;\n  line-height: 1.2;\n}\n\n.display-3 {\n  font-size: 4.5rem;\n  font-weight: 300;\n  line-height: 1.2;\n}\n\n.display-4 {\n  font-size: 3.5rem;\n  font-weight: 300;\n  line-height: 1.2;\n}\n\nhr {\n  margin-top: 1rem;\n  margin-bottom: 1rem;\n  border: 0;\n  border-top: 1px solid rgba(0, 0, 0, 0.1);\n}\n\nsmall,\n.small {\n  font-size: 80%;\n  font-weight: 400;\n}\n\nmark,\n.mark {\n  padding: 0.2em;\n  background-color: #fcf8e3;\n}\n\n.list-unstyled {\n  padding-left: 0;\n  list-style: none;\n}\n\n.list-inline {\n  padding-left: 0;\n  list-style: none;\n}\n\n.list-inline-item {\n  display: inline-block;\n}\n\n.list-inline-item:not(:last-child) {\n  margin-right: 0.5rem;\n}\n\n.initialism {\n  font-size: 90%;\n  text-transform: uppercase;\n}\n\n.blockquote {\n  margin-bottom: 1rem;\n  font-size: 1.25rem;\n}\n\n.blockquote-footer {\n  display: block;\n  font-size: 80%;\n  color: #6c757d;\n}\n\n.blockquote-footer::before {\n  content: \"\\2014\\00A0\";\n}\n\n.img-fluid {\n  max-width: 100%;\n  height: auto;\n}\n\n.img-thumbnail {\n  padding: 0.25rem;\n  background-color: #fff;\n  border: 1px solid #dee2e6;\n  border-radius: 0.25rem;\n  max-width: 100%;\n  height: auto;\n}\n\n.figure {\n  display: inline-block;\n}\n\n.figure-img {\n  margin-bottom: 0.5rem;\n  line-height: 1;\n}\n\n.figure-caption {\n  font-size: 90%;\n  color: #6c757d;\n}\n\ncode {\n  font-size: 87.5%;\n  color: #e83e8c;\n  word-wrap: break-word;\n}\n\na > code {\n  color: inherit;\n}\n\nkbd {\n  padding: 0.2rem 0.4rem;\n  font-size: 87.5%;\n  color: #fff;\n  background-color: #212529;\n  border-radius: 0.2rem;\n}\n\nkbd kbd {\n  padding: 0;\n  font-size: 100%;\n  font-weight: 700;\n}\n\npre {\n  display: block;\n  font-size: 87.5%;\n  color: #212529;\n}\n\npre code {\n  font-size: inherit;\n  color: inherit;\n  word-break: normal;\n}\n\n.pre-scrollable {\n  max-height: 340px;\n  overflow-y: scroll;\n}\n\n.container {\n  width: 100%;\n  padding-right: 15px;\n  padding-left: 15px;\n  margin-right: auto;\n  margin-left: auto;\n}\n\n@media (min-width: 576px) {\n  .container {\n    max-width: 540px;\n  }\n}\n\n@media (min-width: 768px) {\n  .container {\n    max-width: 720px;\n  }\n}\n\n@media (min-width: 992px) {\n  .container {\n    max-width: 960px;\n  }\n}\n\n@media (min-width: 1200px) {\n  .container {\n    max-width: 1140px;\n  }\n}\n\n.container-fluid, .container-sm, .container-md, .container-lg, .container-xl {\n  width: 100%;\n  padding-right: 15px;\n  padding-left: 15px;\n  margin-right: auto;\n  margin-left: auto;\n}\n\n@media (min-width: 576px) {\n  .container, .container-sm {\n    max-width: 540px;\n  }\n}\n\n@media (min-width: 768px) {\n  .container, .container-sm, .container-md {\n    max-width: 720px;\n  }\n}\n\n@media (min-width: 992px) {\n  .container, .container-sm, .container-md, .container-lg {\n    max-width: 960px;\n  }\n}\n\n@media (min-width: 1200px) {\n  .container, .container-sm, .container-md, .container-lg, .container-xl {\n    max-width: 1140px;\n  }\n}\n\n.row {\n  display: flex;\n  flex-wrap: wrap;\n  margin-right: -15px;\n  margin-left: -15px;\n}\n\n.no-gutters {\n  margin-right: 0;\n  margin-left: 0;\n}\n\n.no-gutters > .col,\n.no-gutters > [class*=\"col-\"] {\n  padding-right: 0;\n  padding-left: 0;\n}\n\n.col-1, .col-2, .col-3, .col-4, .col-5, .col-6, .col-7, .col-8, .col-9, .col-10, .col-11, .col-12, .col,\n.col-auto, .col-sm-1, .col-sm-2, .col-sm-3, .col-sm-4, .col-sm-5, .col-sm-6, .col-sm-7, .col-sm-8, .col-sm-9, .col-sm-10, .col-sm-11, .col-sm-12, .col-sm,\n.col-sm-auto, .col-md-1, .col-md-2, .col-md-3, .col-md-4, .col-md-5, .col-md-6, .col-md-7, .col-md-8, .col-md-9, .col-md-10, .col-md-11, .col-md-12, .col-md,\n.col-md-auto, .col-lg-1, .col-lg-2, .col-lg-3, .col-lg-4, .col-lg-5, .col-lg-6, .col-lg-7, .col-lg-8, .col-lg-9, .col-lg-10, .col-lg-11, .col-lg-12, .col-lg,\n.col-lg-auto, .col-xl-1, .col-xl-2, .col-xl-3, .col-xl-4, .col-xl-5, .col-xl-6, .col-xl-7, .col-xl-8, .col-xl-9, .col-xl-10, .col-xl-11, .col-xl-12, .col-xl,\n.col-xl-auto {\n  position: relative;\n  width: 100%;\n  padding-right: 15px;\n  padding-left: 15px;\n}\n\n.col {\n  flex-basis: 0;\n  flex-grow: 1;\n  max-width: 100%;\n}\n\n.row-cols-1 > * {\n  flex: 0 0 100%;\n  max-width: 100%;\n}\n\n.row-cols-2 > * {\n  flex: 0 0 50%;\n  max-width: 50%;\n}\n\n.row-cols-3 > * {\n  flex: 0 0 33.333333%;\n  max-width: 33.333333%;\n}\n\n.row-cols-4 > * {\n  flex: 0 0 25%;\n  max-width: 25%;\n}\n\n.row-cols-5 > * {\n  flex: 0 0 20%;\n  max-width: 20%;\n}\n\n.row-cols-6 > * {\n  flex: 0 0 16.666667%;\n  max-width: 16.666667%;\n}\n\n.col-auto {\n  flex: 0 0 auto;\n  width: auto;\n  max-width: 100%;\n}\n\n.col-1 {\n  flex: 0 0 8.333333%;\n  max-width: 8.333333%;\n}\n\n.col-2 {\n  flex: 0 0 16.666667%;\n  max-width: 16.666667%;\n}\n\n.col-3 {\n  flex: 0 0 25%;\n  max-width: 25%;\n}\n\n.col-4 {\n  flex: 0 0 33.333333%;\n  max-width: 33.333333%;\n}\n\n.col-5 {\n  flex: 0 0 41.666667%;\n  max-width: 41.666667%;\n}\n\n.col-6 {\n  flex: 0 0 50%;\n  max-width: 50%;\n}\n\n.col-7 {\n  flex: 0 0 58.333333%;\n  max-width: 58.333333%;\n}\n\n.col-8 {\n  flex: 0 0 66.666667%;\n  max-width: 66.666667%;\n}\n\n.col-9 {\n  flex: 0 0 75%;\n  max-width: 75%;\n}\n\n.col-10 {\n  flex: 0 0 83.333333%;\n  max-width: 83.333333%;\n}\n\n.col-11 {\n  flex: 0 0 91.666667%;\n  max-width: 91.666667%;\n}\n\n.col-12 {\n  flex: 0 0 100%;\n  max-width: 100%;\n}\n\n.order-first {\n  order: -1;\n}\n\n.order-last {\n  order: 13;\n}\n\n.order-0 {\n  order: 0;\n}\n\n.order-1 {\n  order: 1;\n}\n\n.order-2 {\n  order: 2;\n}\n\n.order-3 {\n  order: 3;\n}\n\n.order-4 {\n  order: 4;\n}\n\n.order-5 {\n  order: 5;\n}\n\n.order-6 {\n  order: 6;\n}\n\n.order-7 {\n  order: 7;\n}\n\n.order-8 {\n  order: 8;\n}\n\n.order-9 {\n  order: 9;\n}\n\n.order-10 {\n  order: 10;\n}\n\n.order-11 {\n  order: 11;\n}\n\n.order-12 {\n  order: 12;\n}\n\n.offset-1 {\n  margin-left: 8.333333%;\n}\n\n.offset-2 {\n  margin-left: 16.666667%;\n}\n\n.offset-3 {\n  margin-left: 25%;\n}\n\n.offset-4 {\n  margin-left: 33.333333%;\n}\n\n.offset-5 {\n  margin-left: 41.666667%;\n}\n\n.offset-6 {\n  margin-left: 50%;\n}\n\n.offset-7 {\n  margin-left: 58.333333%;\n}\n\n.offset-8 {\n  margin-left: 66.666667%;\n}\n\n.offset-9 {\n  margin-left: 75%;\n}\n\n.offset-10 {\n  margin-left: 83.333333%;\n}\n\n.offset-11 {\n  margin-left: 91.666667%;\n}\n\n@media (min-width: 576px) {\n  .col-sm {\n    flex-basis: 0;\n    flex-grow: 1;\n    max-width: 100%;\n  }\n  .row-cols-sm-1 > * {\n    flex: 0 0 100%;\n    max-width: 100%;\n  }\n  .row-cols-sm-2 > * {\n    flex: 0 0 50%;\n    max-width: 50%;\n  }\n  .row-cols-sm-3 > * {\n    flex: 0 0 33.333333%;\n    max-width: 33.333333%;\n  }\n  .row-cols-sm-4 > * {\n    flex: 0 0 25%;\n    max-width: 25%;\n  }\n  .row-cols-sm-5 > * {\n    flex: 0 0 20%;\n    max-width: 20%;\n  }\n  .row-cols-sm-6 > * {\n    flex: 0 0 16.666667%;\n    max-width: 16.666667%;\n  }\n  .col-sm-auto {\n    flex: 0 0 auto;\n    width: auto;\n    max-width: 100%;\n  }\n  .col-sm-1 {\n    flex: 0 0 8.333333%;\n    max-width: 8.333333%;\n  }\n  .col-sm-2 {\n    flex: 0 0 16.666667%;\n    max-width: 16.666667%;\n  }\n  .col-sm-3 {\n    flex: 0 0 25%;\n    max-width: 25%;\n  }\n  .col-sm-4 {\n    flex: 0 0 33.333333%;\n    max-width: 33.333333%;\n  }\n  .col-sm-5 {\n    flex: 0 0 41.666667%;\n    max-width: 41.666667%;\n  }\n  .col-sm-6 {\n    flex: 0 0 50%;\n    max-width: 50%;\n  }\n  .col-sm-7 {\n    flex: 0 0 58.333333%;\n    max-width: 58.333333%;\n  }\n  .col-sm-8 {\n    flex: 0 0 66.666667%;\n    max-width: 66.666667%;\n  }\n  .col-sm-9 {\n    flex: 0 0 75%;\n    max-width: 75%;\n  }\n  .col-sm-10 {\n    flex: 0 0 83.333333%;\n    max-width: 83.333333%;\n  }\n  .col-sm-11 {\n    flex: 0 0 91.666667%;\n    max-width: 91.666667%;\n  }\n  .col-sm-12 {\n    flex: 0 0 100%;\n    max-width: 100%;\n  }\n  .order-sm-first {\n    order: -1;\n  }\n  .order-sm-last {\n    order: 13;\n  }\n  .order-sm-0 {\n    order: 0;\n  }\n  .order-sm-1 {\n    order: 1;\n  }\n  .order-sm-2 {\n    order: 2;\n  }\n  .order-sm-3 {\n    order: 3;\n  }\n  .order-sm-4 {\n    order: 4;\n  }\n  .order-sm-5 {\n    order: 5;\n  }\n  .order-sm-6 {\n    order: 6;\n  }\n  .order-sm-7 {\n    order: 7;\n  }\n  .order-sm-8 {\n    order: 8;\n  }\n  .order-sm-9 {\n    order: 9;\n  }\n  .order-sm-10 {\n    order: 10;\n  }\n  .order-sm-11 {\n    order: 11;\n  }\n  .order-sm-12 {\n    order: 12;\n  }\n  .offset-sm-0 {\n    margin-left: 0;\n  }\n  .offset-sm-1 {\n    margin-left: 8.333333%;\n  }\n  .offset-sm-2 {\n    margin-left: 16.666667%;\n  }\n  .offset-sm-3 {\n    margin-left: 25%;\n  }\n  .offset-sm-4 {\n    margin-left: 33.333333%;\n  }\n  .offset-sm-5 {\n    margin-left: 41.666667%;\n  }\n  .offset-sm-6 {\n    margin-left: 50%;\n  }\n  .offset-sm-7 {\n    margin-left: 58.333333%;\n  }\n  .offset-sm-8 {\n    margin-left: 66.666667%;\n  }\n  .offset-sm-9 {\n    margin-left: 75%;\n  }\n  .offset-sm-10 {\n    margin-left: 83.333333%;\n  }\n  .offset-sm-11 {\n    margin-left: 91.666667%;\n  }\n}\n\n@media (min-width: 768px) {\n  .col-md {\n    flex-basis: 0;\n    flex-grow: 1;\n    max-width: 100%;\n  }\n  .row-cols-md-1 > * {\n    flex: 0 0 100%;\n    max-width: 100%;\n  }\n  .row-cols-md-2 > * {\n    flex: 0 0 50%;\n    max-width: 50%;\n  }\n  .row-cols-md-3 > * {\n    flex: 0 0 33.333333%;\n    max-width: 33.333333%;\n  }\n  .row-cols-md-4 > * {\n    flex: 0 0 25%;\n    max-width: 25%;\n  }\n  .row-cols-md-5 > * {\n    flex: 0 0 20%;\n    max-width: 20%;\n  }\n  .row-cols-md-6 > * {\n    flex: 0 0 16.666667%;\n    max-width: 16.666667%;\n  }\n  .col-md-auto {\n    flex: 0 0 auto;\n    width: auto;\n    max-width: 100%;\n  }\n  .col-md-1 {\n    flex: 0 0 8.333333%;\n    max-width: 8.333333%;\n  }\n  .col-md-2 {\n    flex: 0 0 16.666667%;\n    max-width: 16.666667%;\n  }\n  .col-md-3 {\n    flex: 0 0 25%;\n    max-width: 25%;\n  }\n  .col-md-4 {\n    flex: 0 0 33.333333%;\n    max-width: 33.333333%;\n  }\n  .col-md-5 {\n    flex: 0 0 41.666667%;\n    max-width: 41.666667%;\n  }\n  .col-md-6 {\n    flex: 0 0 50%;\n    max-width: 50%;\n  }\n  .col-md-7 {\n    flex: 0 0 58.333333%;\n    max-width: 58.333333%;\n  }\n  .col-md-8 {\n    flex: 0 0 66.666667%;\n    max-width: 66.666667%;\n  }\n  .col-md-9 {\n    flex: 0 0 75%;\n    max-width: 75%;\n  }\n  .col-md-10 {\n    flex: 0 0 83.333333%;\n    max-width: 83.333333%;\n  }\n  .col-md-11 {\n    flex: 0 0 91.666667%;\n    max-width: 91.666667%;\n  }\n  .col-md-12 {\n    flex: 0 0 100%;\n    max-width: 100%;\n  }\n  .order-md-first {\n    order: -1;\n  }\n  .order-md-last {\n    order: 13;\n  }\n  .order-md-0 {\n    order: 0;\n  }\n  .order-md-1 {\n    order: 1;\n  }\n  .order-md-2 {\n    order: 2;\n  }\n  .order-md-3 {\n    order: 3;\n  }\n  .order-md-4 {\n    order: 4;\n  }\n  .order-md-5 {\n    order: 5;\n  }\n  .order-md-6 {\n    order: 6;\n  }\n  .order-md-7 {\n    order: 7;\n  }\n  .order-md-8 {\n    order: 8;\n  }\n  .order-md-9 {\n    order: 9;\n  }\n  .order-md-10 {\n    order: 10;\n  }\n  .order-md-11 {\n    order: 11;\n  }\n  .order-md-12 {\n    order: 12;\n  }\n  .offset-md-0 {\n    margin-left: 0;\n  }\n  .offset-md-1 {\n    margin-left: 8.333333%;\n  }\n  .offset-md-2 {\n    margin-left: 16.666667%;\n  }\n  .offset-md-3 {\n    margin-left: 25%;\n  }\n  .offset-md-4 {\n    margin-left: 33.333333%;\n  }\n  .offset-md-5 {\n    margin-left: 41.666667%;\n  }\n  .offset-md-6 {\n    margin-left: 50%;\n  }\n  .offset-md-7 {\n    margin-left: 58.333333%;\n  }\n  .offset-md-8 {\n    margin-left: 66.666667%;\n  }\n  .offset-md-9 {\n    margin-left: 75%;\n  }\n  .offset-md-10 {\n    margin-left: 83.333333%;\n  }\n  .offset-md-11 {\n    margin-left: 91.666667%;\n  }\n}\n\n@media (min-width: 992px) {\n  .col-lg {\n    flex-basis: 0;\n    flex-grow: 1;\n    max-width: 100%;\n  }\n  .row-cols-lg-1 > * {\n    flex: 0 0 100%;\n    max-width: 100%;\n  }\n  .row-cols-lg-2 > * {\n    flex: 0 0 50%;\n    max-width: 50%;\n  }\n  .row-cols-lg-3 > * {\n    flex: 0 0 33.333333%;\n    max-width: 33.333333%;\n  }\n  .row-cols-lg-4 > * {\n    flex: 0 0 25%;\n    max-width: 25%;\n  }\n  .row-cols-lg-5 > * {\n    flex: 0 0 20%;\n    max-width: 20%;\n  }\n  .row-cols-lg-6 > * {\n    flex: 0 0 16.666667%;\n    max-width: 16.666667%;\n  }\n  .col-lg-auto {\n    flex: 0 0 auto;\n    width: auto;\n    max-width: 100%;\n  }\n  .col-lg-1 {\n    flex: 0 0 8.333333%;\n    max-width: 8.333333%;\n  }\n  .col-lg-2 {\n    flex: 0 0 16.666667%;\n    max-width: 16.666667%;\n  }\n  .col-lg-3 {\n    flex: 0 0 25%;\n    max-width: 25%;\n  }\n  .col-lg-4 {\n    flex: 0 0 33.333333%;\n    max-width: 33.333333%;\n  }\n  .col-lg-5 {\n    flex: 0 0 41.666667%;\n    max-width: 41.666667%;\n  }\n  .col-lg-6 {\n    flex: 0 0 50%;\n    max-width: 50%;\n  }\n  .col-lg-7 {\n    flex: 0 0 58.333333%;\n    max-width: 58.333333%;\n  }\n  .col-lg-8 {\n    flex: 0 0 66.666667%;\n    max-width: 66.666667%;\n  }\n  .col-lg-9 {\n    flex: 0 0 75%;\n    max-width: 75%;\n  }\n  .col-lg-10 {\n    flex: 0 0 83.333333%;\n    max-width: 83.333333%;\n  }\n  .col-lg-11 {\n    flex: 0 0 91.666667%;\n    max-width: 91.666667%;\n  }\n  .col-lg-12 {\n    flex: 0 0 100%;\n    max-width: 100%;\n  }\n  .order-lg-first {\n    order: -1;\n  }\n  .order-lg-last {\n    order: 13;\n  }\n  .order-lg-0 {\n    order: 0;\n  }\n  .order-lg-1 {\n    order: 1;\n  }\n  .order-lg-2 {\n    order: 2;\n  }\n  .order-lg-3 {\n    order: 3;\n  }\n  .order-lg-4 {\n    order: 4;\n  }\n  .order-lg-5 {\n    order: 5;\n  }\n  .order-lg-6 {\n    order: 6;\n  }\n  .order-lg-7 {\n    order: 7;\n  }\n  .order-lg-8 {\n    order: 8;\n  }\n  .order-lg-9 {\n    order: 9;\n  }\n  .order-lg-10 {\n    order: 10;\n  }\n  .order-lg-11 {\n    order: 11;\n  }\n  .order-lg-12 {\n    order: 12;\n  }\n  .offset-lg-0 {\n    margin-left: 0;\n  }\n  .offset-lg-1 {\n    margin-left: 8.333333%;\n  }\n  .offset-lg-2 {\n    margin-left: 16.666667%;\n  }\n  .offset-lg-3 {\n    margin-left: 25%;\n  }\n  .offset-lg-4 {\n    margin-left: 33.333333%;\n  }\n  .offset-lg-5 {\n    margin-left: 41.666667%;\n  }\n  .offset-lg-6 {\n    margin-left: 50%;\n  }\n  .offset-lg-7 {\n    margin-left: 58.333333%;\n  }\n  .offset-lg-8 {\n    margin-left: 66.666667%;\n  }\n  .offset-lg-9 {\n    margin-left: 75%;\n  }\n  .offset-lg-10 {\n    margin-left: 83.333333%;\n  }\n  .offset-lg-11 {\n    margin-left: 91.666667%;\n  }\n}\n\n@media (min-width: 1200px) {\n  .col-xl {\n    flex-basis: 0;\n    flex-grow: 1;\n    max-width: 100%;\n  }\n  .row-cols-xl-1 > * {\n    flex: 0 0 100%;\n    max-width: 100%;\n  }\n  .row-cols-xl-2 > * {\n    flex: 0 0 50%;\n    max-width: 50%;\n  }\n  .row-cols-xl-3 > * {\n    flex: 0 0 33.333333%;\n    max-width: 33.333333%;\n  }\n  .row-cols-xl-4 > * {\n    flex: 0 0 25%;\n    max-width: 25%;\n  }\n  .row-cols-xl-5 > * {\n    flex: 0 0 20%;\n    max-width: 20%;\n  }\n  .row-cols-xl-6 > * {\n    flex: 0 0 16.666667%;\n    max-width: 16.666667%;\n  }\n  .col-xl-auto {\n    flex: 0 0 auto;\n    width: auto;\n    max-width: 100%;\n  }\n  .col-xl-1 {\n    flex: 0 0 8.333333%;\n    max-width: 8.333333%;\n  }\n  .col-xl-2 {\n    flex: 0 0 16.666667%;\n    max-width: 16.666667%;\n  }\n  .col-xl-3 {\n    flex: 0 0 25%;\n    max-width: 25%;\n  }\n  .col-xl-4 {\n    flex: 0 0 33.333333%;\n    max-width: 33.333333%;\n  }\n  .col-xl-5 {\n    flex: 0 0 41.666667%;\n    max-width: 41.666667%;\n  }\n  .col-xl-6 {\n    flex: 0 0 50%;\n    max-width: 50%;\n  }\n  .col-xl-7 {\n    flex: 0 0 58.333333%;\n    max-width: 58.333333%;\n  }\n  .col-xl-8 {\n    flex: 0 0 66.666667%;\n    max-width: 66.666667%;\n  }\n  .col-xl-9 {\n    flex: 0 0 75%;\n    max-width: 75%;\n  }\n  .col-xl-10 {\n    flex: 0 0 83.333333%;\n    max-width: 83.333333%;\n  }\n  .col-xl-11 {\n    flex: 0 0 91.666667%;\n    max-width: 91.666667%;\n  }\n  .col-xl-12 {\n    flex: 0 0 100%;\n    max-width: 100%;\n  }\n  .order-xl-first {\n    order: -1;\n  }\n  .order-xl-last {\n    order: 13;\n  }\n  .order-xl-0 {\n    order: 0;\n  }\n  .order-xl-1 {\n    order: 1;\n  }\n  .order-xl-2 {\n    order: 2;\n  }\n  .order-xl-3 {\n    order: 3;\n  }\n  .order-xl-4 {\n    order: 4;\n  }\n  .order-xl-5 {\n    order: 5;\n  }\n  .order-xl-6 {\n    order: 6;\n  }\n  .order-xl-7 {\n    order: 7;\n  }\n  .order-xl-8 {\n    order: 8;\n  }\n  .order-xl-9 {\n    order: 9;\n  }\n  .order-xl-10 {\n    order: 10;\n  }\n  .order-xl-11 {\n    order: 11;\n  }\n  .order-xl-12 {\n    order: 12;\n  }\n  .offset-xl-0 {\n    margin-left: 0;\n  }\n  .offset-xl-1 {\n    margin-left: 8.333333%;\n  }\n  .offset-xl-2 {\n    margin-left: 16.666667%;\n  }\n  .offset-xl-3 {\n    margin-left: 25%;\n  }\n  .offset-xl-4 {\n    margin-left: 33.333333%;\n  }\n  .offset-xl-5 {\n    margin-left: 41.666667%;\n  }\n  .offset-xl-6 {\n    margin-left: 50%;\n  }\n  .offset-xl-7 {\n    margin-left: 58.333333%;\n  }\n  .offset-xl-8 {\n    margin-left: 66.666667%;\n  }\n  .offset-xl-9 {\n    margin-left: 75%;\n  }\n  .offset-xl-10 {\n    margin-left: 83.333333%;\n  }\n  .offset-xl-11 {\n    margin-left: 91.666667%;\n  }\n}\n\n.table {\n  width: 100%;\n  margin-bottom: 1rem;\n  color: #212529;\n}\n\n.table th,\n.table td {\n  padding: 0.75rem;\n  vertical-align: top;\n  border-top: 1px solid #dee2e6;\n}\n\n.table thead th {\n  vertical-align: bottom;\n  border-bottom: 2px solid #dee2e6;\n}\n\n.table tbody + tbody {\n  border-top: 2px solid #dee2e6;\n}\n\n.table-sm th,\n.table-sm td {\n  padding: 0.3rem;\n}\n\n.table-bordered {\n  border: 1px solid #dee2e6;\n}\n\n.table-bordered th,\n.table-bordered td {\n  border: 1px solid #dee2e6;\n}\n\n.table-bordered thead th,\n.table-bordered thead td {\n  border-bottom-width: 2px;\n}\n\n.table-borderless th,\n.table-borderless td,\n.table-borderless thead th,\n.table-borderless tbody + tbody {\n  border: 0;\n}\n\n.table-striped tbody tr:nth-of-type(odd) {\n  background-color: rgba(0, 0, 0, 0.05);\n}\n\n.table-hover tbody tr:hover {\n  color: #212529;\n  background-color: rgba(0, 0, 0, 0.075);\n}\n\n.table-primary,\n.table-primary > th,\n.table-primary > td {\n  background-color: #b8daff;\n}\n\n.table-primary th,\n.table-primary td,\n.table-primary thead th,\n.table-primary tbody + tbody {\n  border-color: #7abaff;\n}\n\n.table-hover .table-primary:hover {\n  background-color: #9fcdff;\n}\n\n.table-hover .table-primary:hover > td,\n.table-hover .table-primary:hover > th {\n  background-color: #9fcdff;\n}\n\n.table-secondary,\n.table-secondary > th,\n.table-secondary > td {\n  background-color: #d6d8db;\n}\n\n.table-secondary th,\n.table-secondary td,\n.table-secondary thead th,\n.table-secondary tbody + tbody {\n  border-color: #b3b7bb;\n}\n\n.table-hover .table-secondary:hover {\n  background-color: #c8cbcf;\n}\n\n.table-hover .table-secondary:hover > td,\n.table-hover .table-secondary:hover > th {\n  background-color: #c8cbcf;\n}\n\n.table-success,\n.table-success > th,\n.table-success > td {\n  background-color: #c3e6cb;\n}\n\n.table-success th,\n.table-success td,\n.table-success thead th,\n.table-success tbody + tbody {\n  border-color: #8fd19e;\n}\n\n.table-hover .table-success:hover {\n  background-color: #b1dfbb;\n}\n\n.table-hover .table-success:hover > td,\n.table-hover .table-success:hover > th {\n  background-color: #b1dfbb;\n}\n\n.table-info,\n.table-info > th,\n.table-info > td {\n  background-color: #bee5eb;\n}\n\n.table-info th,\n.table-info td,\n.table-info thead th,\n.table-info tbody + tbody {\n  border-color: #86cfda;\n}\n\n.table-hover .table-info:hover {\n  background-color: #abdde5;\n}\n\n.table-hover .table-info:hover > td,\n.table-hover .table-info:hover > th {\n  background-color: #abdde5;\n}\n\n.table-warning,\n.table-warning > th,\n.table-warning > td {\n  background-color: #ffeeba;\n}\n\n.table-warning th,\n.table-warning td,\n.table-warning thead th,\n.table-warning tbody + tbody {\n  border-color: #ffdf7e;\n}\n\n.table-hover .table-warning:hover {\n  background-color: #ffe8a1;\n}\n\n.table-hover .table-warning:hover > td,\n.table-hover .table-warning:hover > th {\n  background-color: #ffe8a1;\n}\n\n.table-danger,\n.table-danger > th,\n.table-danger > td {\n  background-color: #f5c6cb;\n}\n\n.table-danger th,\n.table-danger td,\n.table-danger thead th,\n.table-danger tbody + tbody {\n  border-color: #ed969e;\n}\n\n.table-hover .table-danger:hover {\n  background-color: #f1b0b7;\n}\n\n.table-hover .table-danger:hover > td,\n.table-hover .table-danger:hover > th {\n  background-color: #f1b0b7;\n}\n\n.table-light,\n.table-light > th,\n.table-light > td {\n  background-color: #fdfdfe;\n}\n\n.table-light th,\n.table-light td,\n.table-light thead th,\n.table-light tbody + tbody {\n  border-color: #fbfcfc;\n}\n\n.table-hover .table-light:hover {\n  background-color: #ececf6;\n}\n\n.table-hover .table-light:hover > td,\n.table-hover .table-light:hover > th {\n  background-color: #ececf6;\n}\n\n.table-dark,\n.table-dark > th,\n.table-dark > td {\n  background-color: #c6c8ca;\n}\n\n.table-dark th,\n.table-dark td,\n.table-dark thead th,\n.table-dark tbody + tbody {\n  border-color: #95999c;\n}\n\n.table-hover .table-dark:hover {\n  background-color: #b9bbbe;\n}\n\n.table-hover .table-dark:hover > td,\n.table-hover .table-dark:hover > th {\n  background-color: #b9bbbe;\n}\n\n.table-active,\n.table-active > th,\n.table-active > td {\n  background-color: rgba(0, 0, 0, 0.075);\n}\n\n.table-hover .table-active:hover {\n  background-color: rgba(0, 0, 0, 0.075);\n}\n\n.table-hover .table-active:hover > td,\n.table-hover .table-active:hover > th {\n  background-color: rgba(0, 0, 0, 0.075);\n}\n\n.table .thead-dark th {\n  color: #fff;\n  background-color: #343a40;\n  border-color: #454d55;\n}\n\n.table .thead-light th {\n  color: #495057;\n  background-color: #e9ecef;\n  border-color: #dee2e6;\n}\n\n.table-dark {\n  color: #fff;\n  background-color: #343a40;\n}\n\n.table-dark th,\n.table-dark td,\n.table-dark thead th {\n  border-color: #454d55;\n}\n\n.table-dark.table-bordered {\n  border: 0;\n}\n\n.table-dark.table-striped tbody tr:nth-of-type(odd) {\n  background-color: rgba(255, 255, 255, 0.05);\n}\n\n.table-dark.table-hover tbody tr:hover {\n  color: #fff;\n  background-color: rgba(255, 255, 255, 0.075);\n}\n\n@media (max-width: 575.98px) {\n  .table-responsive-sm {\n    display: block;\n    width: 100%;\n    overflow-x: auto;\n    -webkit-overflow-scrolling: touch;\n  }\n  .table-responsive-sm > .table-bordered {\n    border: 0;\n  }\n}\n\n@media (max-width: 767.98px) {\n  .table-responsive-md {\n    display: block;\n    width: 100%;\n    overflow-x: auto;\n    -webkit-overflow-scrolling: touch;\n  }\n  .table-responsive-md > .table-bordered {\n    border: 0;\n  }\n}\n\n@media (max-width: 991.98px) {\n  .table-responsive-lg {\n    display: block;\n    width: 100%;\n    overflow-x: auto;\n    -webkit-overflow-scrolling: touch;\n  }\n  .table-responsive-lg > .table-bordered {\n    border: 0;\n  }\n}\n\n@media (max-width: 1199.98px) {\n  .table-responsive-xl {\n    display: block;\n    width: 100%;\n    overflow-x: auto;\n    -webkit-overflow-scrolling: touch;\n  }\n  .table-responsive-xl > .table-bordered {\n    border: 0;\n  }\n}\n\n.table-responsive {\n  display: block;\n  width: 100%;\n  overflow-x: auto;\n  -webkit-overflow-scrolling: touch;\n}\n\n.table-responsive > .table-bordered {\n  border: 0;\n}\n\n.form-control {\n  display: block;\n  width: 100%;\n  height: calc(1.5em + 0.75rem + 2px);\n  padding: 0.375rem 0.75rem;\n  font-size: 1rem;\n  font-weight: 400;\n  line-height: 1.5;\n  color: #495057;\n  background-color: #fff;\n  background-clip: padding-box;\n  border: 1px solid #ced4da;\n  border-radius: 0.25rem;\n  transition: border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .form-control {\n    transition: none;\n  }\n}\n\n.form-control::-ms-expand {\n  background-color: transparent;\n  border: 0;\n}\n\n.form-control:-moz-focusring {\n  color: transparent;\n  text-shadow: 0 0 0 #495057;\n}\n\n.form-control:focus {\n  color: #495057;\n  background-color: #fff;\n  border-color: #80bdff;\n  outline: 0;\n  box-shadow: 0 0 0 0.2rem rgba(0, 123, 255, 0.25);\n}\n\n.form-control::placeholder {\n  color: #6c757d;\n  opacity: 1;\n}\n\n.form-control:disabled, .form-control[readonly] {\n  background-color: #e9ecef;\n  opacity: 1;\n}\n\nselect.form-control:focus::-ms-value {\n  color: #495057;\n  background-color: #fff;\n}\n\n.form-control-file,\n.form-control-range {\n  display: block;\n  width: 100%;\n}\n\n.col-form-label {\n  padding-top: calc(0.375rem + 1px);\n  padding-bottom: calc(0.375rem + 1px);\n  margin-bottom: 0;\n  font-size: inherit;\n  line-height: 1.5;\n}\n\n.col-form-label-lg {\n  padding-top: calc(0.5rem + 1px);\n  padding-bottom: calc(0.5rem + 1px);\n  font-size: 1.25rem;\n  line-height: 1.5;\n}\n\n.col-form-label-sm {\n  padding-top: calc(0.25rem + 1px);\n  padding-bottom: calc(0.25rem + 1px);\n  font-size: 0.875rem;\n  line-height: 1.5;\n}\n\n.form-control-plaintext {\n  display: block;\n  width: 100%;\n  padding: 0.375rem 0;\n  margin-bottom: 0;\n  font-size: 1rem;\n  line-height: 1.5;\n  color: #212529;\n  background-color: transparent;\n  border: solid transparent;\n  border-width: 1px 0;\n}\n\n.form-control-plaintext.form-control-sm, .form-control-plaintext.form-control-lg {\n  padding-right: 0;\n  padding-left: 0;\n}\n\n.form-control-sm {\n  height: calc(1.5em + 0.5rem + 2px);\n  padding: 0.25rem 0.5rem;\n  font-size: 0.875rem;\n  line-height: 1.5;\n  border-radius: 0.2rem;\n}\n\n.form-control-lg {\n  height: calc(1.5em + 1rem + 2px);\n  padding: 0.5rem 1rem;\n  font-size: 1.25rem;\n  line-height: 1.5;\n  border-radius: 0.3rem;\n}\n\nselect.form-control[size], select.form-control[multiple] {\n  height: auto;\n}\n\ntextarea.form-control {\n  height: auto;\n}\n\n.form-group {\n  margin-bottom: 1rem;\n}\n\n.form-text {\n  display: block;\n  margin-top: 0.25rem;\n}\n\n.form-row {\n  display: flex;\n  flex-wrap: wrap;\n  margin-right: -5px;\n  margin-left: -5px;\n}\n\n.form-row > .col,\n.form-row > [class*=\"col-\"] {\n  padding-right: 5px;\n  padding-left: 5px;\n}\n\n.form-check {\n  position: relative;\n  display: block;\n  padding-left: 1.25rem;\n}\n\n.form-check-input {\n  position: absolute;\n  margin-top: 0.3rem;\n  margin-left: -1.25rem;\n}\n\n.form-check-input[disabled] ~ .form-check-label,\n.form-check-input:disabled ~ .form-check-label {\n  color: #6c757d;\n}\n\n.form-check-label {\n  margin-bottom: 0;\n}\n\n.form-check-inline {\n  display: inline-flex;\n  align-items: center;\n  padding-left: 0;\n  margin-right: 0.75rem;\n}\n\n.form-check-inline .form-check-input {\n  position: static;\n  margin-top: 0;\n  margin-right: 0.3125rem;\n  margin-left: 0;\n}\n\n.valid-feedback {\n  display: none;\n  width: 100%;\n  margin-top: 0.25rem;\n  font-size: 80%;\n  color: #28a745;\n}\n\n.valid-tooltip {\n  position: absolute;\n  top: 100%;\n  z-index: 5;\n  display: none;\n  max-width: 100%;\n  padding: 0.25rem 0.5rem;\n  margin-top: .1rem;\n  font-size: 0.875rem;\n  line-height: 1.5;\n  color: #fff;\n  background-color: rgba(40, 167, 69, 0.9);\n  border-radius: 0.25rem;\n}\n\n.was-validated :valid ~ .valid-feedback,\n.was-validated :valid ~ .valid-tooltip,\n.is-valid ~ .valid-feedback,\n.is-valid ~ .valid-tooltip {\n  display: block;\n}\n\n.was-validated .form-control:valid, .form-control.is-valid {\n  border-color: #28a745;\n  padding-right: calc(1.5em + 0.75rem);\n  background-image: url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath fill='%2328a745' d='M2.3 6.73L.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e\");\n  background-repeat: no-repeat;\n  background-position: right calc(0.375em + 0.1875rem) center;\n  background-size: calc(0.75em + 0.375rem) calc(0.75em + 0.375rem);\n}\n\n.was-validated .form-control:valid:focus, .form-control.is-valid:focus {\n  border-color: #28a745;\n  box-shadow: 0 0 0 0.2rem rgba(40, 167, 69, 0.25);\n}\n\n.was-validated textarea.form-control:valid, textarea.form-control.is-valid {\n  padding-right: calc(1.5em + 0.75rem);\n  background-position: top calc(0.375em + 0.1875rem) right calc(0.375em + 0.1875rem);\n}\n\n.was-validated .custom-select:valid, .custom-select.is-valid {\n  border-color: #28a745;\n  padding-right: calc(0.75em + 2.3125rem);\n  background: url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='5' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e\") no-repeat right 0.75rem center/8px 10px, url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath fill='%2328a745' d='M2.3 6.73L.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e\") #fff no-repeat center right 1.75rem/calc(0.75em + 0.375rem) calc(0.75em + 0.375rem);\n}\n\n.was-validated .custom-select:valid:focus, .custom-select.is-valid:focus {\n  border-color: #28a745;\n  box-shadow: 0 0 0 0.2rem rgba(40, 167, 69, 0.25);\n}\n\n.was-validated .form-check-input:valid ~ .form-check-label, .form-check-input.is-valid ~ .form-check-label {\n  color: #28a745;\n}\n\n.was-validated .form-check-input:valid ~ .valid-feedback,\n.was-validated .form-check-input:valid ~ .valid-tooltip, .form-check-input.is-valid ~ .valid-feedback,\n.form-check-input.is-valid ~ .valid-tooltip {\n  display: block;\n}\n\n.was-validated .custom-control-input:valid ~ .custom-control-label, .custom-control-input.is-valid ~ .custom-control-label {\n  color: #28a745;\n}\n\n.was-validated .custom-control-input:valid ~ .custom-control-label::before, .custom-control-input.is-valid ~ .custom-control-label::before {\n  border-color: #28a745;\n}\n\n.was-validated .custom-control-input:valid:checked ~ .custom-control-label::before, .custom-control-input.is-valid:checked ~ .custom-control-label::before {\n  border-color: #34ce57;\n  background-color: #34ce57;\n}\n\n.was-validated .custom-control-input:valid:focus ~ .custom-control-label::before, .custom-control-input.is-valid:focus ~ .custom-control-label::before {\n  box-shadow: 0 0 0 0.2rem rgba(40, 167, 69, 0.25);\n}\n\n.was-validated .custom-control-input:valid:focus:not(:checked) ~ .custom-control-label::before, .custom-control-input.is-valid:focus:not(:checked) ~ .custom-control-label::before {\n  border-color: #28a745;\n}\n\n.was-validated .custom-file-input:valid ~ .custom-file-label, .custom-file-input.is-valid ~ .custom-file-label {\n  border-color: #28a745;\n}\n\n.was-validated .custom-file-input:valid:focus ~ .custom-file-label, .custom-file-input.is-valid:focus ~ .custom-file-label {\n  border-color: #28a745;\n  box-shadow: 0 0 0 0.2rem rgba(40, 167, 69, 0.25);\n}\n\n.invalid-feedback {\n  display: none;\n  width: 100%;\n  margin-top: 0.25rem;\n  font-size: 80%;\n  color: #dc3545;\n}\n\n.invalid-tooltip {\n  position: absolute;\n  top: 100%;\n  z-index: 5;\n  display: none;\n  max-width: 100%;\n  padding: 0.25rem 0.5rem;\n  margin-top: .1rem;\n  font-size: 0.875rem;\n  line-height: 1.5;\n  color: #fff;\n  background-color: rgba(220, 53, 69, 0.9);\n  border-radius: 0.25rem;\n}\n\n.was-validated :invalid ~ .invalid-feedback,\n.was-validated :invalid ~ .invalid-tooltip,\n.is-invalid ~ .invalid-feedback,\n.is-invalid ~ .invalid-tooltip {\n  display: block;\n}\n\n.was-validated .form-control:invalid, .form-control.is-invalid {\n  border-color: #dc3545;\n  padding-right: calc(1.5em + 0.75rem);\n  background-image: url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' fill='none' stroke='%23dc3545' viewBox='0 0 12 12'%3e%3ccircle cx='6' cy='6' r='4.5'/%3e%3cpath stroke-linejoin='round' d='M5.8 3.6h.4L6 6.5z'/%3e%3ccircle cx='6' cy='8.2' r='.6' fill='%23dc3545' stroke='none'/%3e%3c/svg%3e\");\n  background-repeat: no-repeat;\n  background-position: right calc(0.375em + 0.1875rem) center;\n  background-size: calc(0.75em + 0.375rem) calc(0.75em + 0.375rem);\n}\n\n.was-validated .form-control:invalid:focus, .form-control.is-invalid:focus {\n  border-color: #dc3545;\n  box-shadow: 0 0 0 0.2rem rgba(220, 53, 69, 0.25);\n}\n\n.was-validated textarea.form-control:invalid, textarea.form-control.is-invalid {\n  padding-right: calc(1.5em + 0.75rem);\n  background-position: top calc(0.375em + 0.1875rem) right calc(0.375em + 0.1875rem);\n}\n\n.was-validated .custom-select:invalid, .custom-select.is-invalid {\n  border-color: #dc3545;\n  padding-right: calc(0.75em + 2.3125rem);\n  background: url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='5' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e\") no-repeat right 0.75rem center/8px 10px, url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' fill='none' stroke='%23dc3545' viewBox='0 0 12 12'%3e%3ccircle cx='6' cy='6' r='4.5'/%3e%3cpath stroke-linejoin='round' d='M5.8 3.6h.4L6 6.5z'/%3e%3ccircle cx='6' cy='8.2' r='.6' fill='%23dc3545' stroke='none'/%3e%3c/svg%3e\") #fff no-repeat center right 1.75rem/calc(0.75em + 0.375rem) calc(0.75em + 0.375rem);\n}\n\n.was-validated .custom-select:invalid:focus, .custom-select.is-invalid:focus {\n  border-color: #dc3545;\n  box-shadow: 0 0 0 0.2rem rgba(220, 53, 69, 0.25);\n}\n\n.was-validated .form-check-input:invalid ~ .form-check-label, .form-check-input.is-invalid ~ .form-check-label {\n  color: #dc3545;\n}\n\n.was-validated .form-check-input:invalid ~ .invalid-feedback,\n.was-validated .form-check-input:invalid ~ .invalid-tooltip, .form-check-input.is-invalid ~ .invalid-feedback,\n.form-check-input.is-invalid ~ .invalid-tooltip {\n  display: block;\n}\n\n.was-validated .custom-control-input:invalid ~ .custom-control-label, .custom-control-input.is-invalid ~ .custom-control-label {\n  color: #dc3545;\n}\n\n.was-validated .custom-control-input:invalid ~ .custom-control-label::before, .custom-control-input.is-invalid ~ .custom-control-label::before {\n  border-color: #dc3545;\n}\n\n.was-validated .custom-control-input:invalid:checked ~ .custom-control-label::before, .custom-control-input.is-invalid:checked ~ .custom-control-label::before {\n  border-color: #e4606d;\n  background-color: #e4606d;\n}\n\n.was-validated .custom-control-input:invalid:focus ~ .custom-control-label::before, .custom-control-input.is-invalid:focus ~ .custom-control-label::before {\n  box-shadow: 0 0 0 0.2rem rgba(220, 53, 69, 0.25);\n}\n\n.was-validated .custom-control-input:invalid:focus:not(:checked) ~ .custom-control-label::before, .custom-control-input.is-invalid:focus:not(:checked) ~ .custom-control-label::before {\n  border-color: #dc3545;\n}\n\n.was-validated .custom-file-input:invalid ~ .custom-file-label, .custom-file-input.is-invalid ~ .custom-file-label {\n  border-color: #dc3545;\n}\n\n.was-validated .custom-file-input:invalid:focus ~ .custom-file-label, .custom-file-input.is-invalid:focus ~ .custom-file-label {\n  border-color: #dc3545;\n  box-shadow: 0 0 0 0.2rem rgba(220, 53, 69, 0.25);\n}\n\n.form-inline {\n  display: flex;\n  flex-flow: row wrap;\n  align-items: center;\n}\n\n.form-inline .form-check {\n  width: 100%;\n}\n\n@media (min-width: 576px) {\n  .form-inline label {\n    display: flex;\n    align-items: center;\n    justify-content: center;\n    margin-bottom: 0;\n  }\n  .form-inline .form-group {\n    display: flex;\n    flex: 0 0 auto;\n    flex-flow: row wrap;\n    align-items: center;\n    margin-bottom: 0;\n  }\n  .form-inline .form-control {\n    display: inline-block;\n    width: auto;\n    vertical-align: middle;\n  }\n  .form-inline .form-control-plaintext {\n    display: inline-block;\n  }\n  .form-inline .input-group,\n  .form-inline .custom-select {\n    width: auto;\n  }\n  .form-inline .form-check {\n    display: flex;\n    align-items: center;\n    justify-content: center;\n    width: auto;\n    padding-left: 0;\n  }\n  .form-inline .form-check-input {\n    position: relative;\n    flex-shrink: 0;\n    margin-top: 0;\n    margin-right: 0.25rem;\n    margin-left: 0;\n  }\n  .form-inline .custom-control {\n    align-items: center;\n    justify-content: center;\n  }\n  .form-inline .custom-control-label {\n    margin-bottom: 0;\n  }\n}\n\n.btn {\n  display: inline-block;\n  font-weight: 400;\n  color: #212529;\n  text-align: center;\n  vertical-align: middle;\n  cursor: pointer;\n  user-select: none;\n  background-color: transparent;\n  border: 1px solid transparent;\n  padding: 0.375rem 0.75rem;\n  font-size: 1rem;\n  line-height: 1.5;\n  border-radius: 0.25rem;\n  transition: color 0.15s ease-in-out, background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .btn {\n    transition: none;\n  }\n}\n\n.btn:hover {\n  color: #212529;\n  text-decoration: none;\n}\n\n.btn:focus, .btn.focus {\n  outline: 0;\n  box-shadow: 0 0 0 0.2rem rgba(0, 123, 255, 0.25);\n}\n\n.btn.disabled, .btn:disabled {\n  opacity: 0.65;\n}\n\na.btn.disabled,\nfieldset:disabled a.btn {\n  pointer-events: none;\n}\n\n.btn-primary {\n  color: #fff;\n  background-color: #007bff;\n  border-color: #007bff;\n}\n\n.btn-primary:hover {\n  color: #fff;\n  background-color: #0069d9;\n  border-color: #0062cc;\n}\n\n.btn-primary:focus, .btn-primary.focus {\n  color: #fff;\n  background-color: #0069d9;\n  border-color: #0062cc;\n  box-shadow: 0 0 0 0.2rem rgba(38, 143, 255, 0.5);\n}\n\n.btn-primary.disabled, .btn-primary:disabled {\n  color: #fff;\n  background-color: #007bff;\n  border-color: #007bff;\n}\n\n.btn-primary:not(:disabled):not(.disabled):active, .btn-primary:not(:disabled):not(.disabled).active,\n.show > .btn-primary.dropdown-toggle {\n  color: #fff;\n  background-color: #0062cc;\n  border-color: #005cbf;\n}\n\n.btn-primary:not(:disabled):not(.disabled):active:focus, .btn-primary:not(:disabled):not(.disabled).active:focus,\n.show > .btn-primary.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(38, 143, 255, 0.5);\n}\n\n.btn-secondary {\n  color: #fff;\n  background-color: #6c757d;\n  border-color: #6c757d;\n}\n\n.btn-secondary:hover {\n  color: #fff;\n  background-color: #5a6268;\n  border-color: #545b62;\n}\n\n.btn-secondary:focus, .btn-secondary.focus {\n  color: #fff;\n  background-color: #5a6268;\n  border-color: #545b62;\n  box-shadow: 0 0 0 0.2rem rgba(130, 138, 145, 0.5);\n}\n\n.btn-secondary.disabled, .btn-secondary:disabled {\n  color: #fff;\n  background-color: #6c757d;\n  border-color: #6c757d;\n}\n\n.btn-secondary:not(:disabled):not(.disabled):active, .btn-secondary:not(:disabled):not(.disabled).active,\n.show > .btn-secondary.dropdown-toggle {\n  color: #fff;\n  background-color: #545b62;\n  border-color: #4e555b;\n}\n\n.btn-secondary:not(:disabled):not(.disabled):active:focus, .btn-secondary:not(:disabled):not(.disabled).active:focus,\n.show > .btn-secondary.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(130, 138, 145, 0.5);\n}\n\n.btn-success {\n  color: #fff;\n  background-color: #28a745;\n  border-color: #28a745;\n}\n\n.btn-success:hover {\n  color: #fff;\n  background-color: #218838;\n  border-color: #1e7e34;\n}\n\n.btn-success:focus, .btn-success.focus {\n  color: #fff;\n  background-color: #218838;\n  border-color: #1e7e34;\n  box-shadow: 0 0 0 0.2rem rgba(72, 180, 97, 0.5);\n}\n\n.btn-success.disabled, .btn-success:disabled {\n  color: #fff;\n  background-color: #28a745;\n  border-color: #28a745;\n}\n\n.btn-success:not(:disabled):not(.disabled):active, .btn-success:not(:disabled):not(.disabled).active,\n.show > .btn-success.dropdown-toggle {\n  color: #fff;\n  background-color: #1e7e34;\n  border-color: #1c7430;\n}\n\n.btn-success:not(:disabled):not(.disabled):active:focus, .btn-success:not(:disabled):not(.disabled).active:focus,\n.show > .btn-success.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(72, 180, 97, 0.5);\n}\n\n.btn-info {\n  color: #fff;\n  background-color: #17a2b8;\n  border-color: #17a2b8;\n}\n\n.btn-info:hover {\n  color: #fff;\n  background-color: #138496;\n  border-color: #117a8b;\n}\n\n.btn-info:focus, .btn-info.focus {\n  color: #fff;\n  background-color: #138496;\n  border-color: #117a8b;\n  box-shadow: 0 0 0 0.2rem rgba(58, 176, 195, 0.5);\n}\n\n.btn-info.disabled, .btn-info:disabled {\n  color: #fff;\n  background-color: #17a2b8;\n  border-color: #17a2b8;\n}\n\n.btn-info:not(:disabled):not(.disabled):active, .btn-info:not(:disabled):not(.disabled).active,\n.show > .btn-info.dropdown-toggle {\n  color: #fff;\n  background-color: #117a8b;\n  border-color: #10707f;\n}\n\n.btn-info:not(:disabled):not(.disabled):active:focus, .btn-info:not(:disabled):not(.disabled).active:focus,\n.show > .btn-info.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(58, 176, 195, 0.5);\n}\n\n.btn-warning {\n  color: #212529;\n  background-color: #ffc107;\n  border-color: #ffc107;\n}\n\n.btn-warning:hover {\n  color: #212529;\n  background-color: #e0a800;\n  border-color: #d39e00;\n}\n\n.btn-warning:focus, .btn-warning.focus {\n  color: #212529;\n  background-color: #e0a800;\n  border-color: #d39e00;\n  box-shadow: 0 0 0 0.2rem rgba(222, 170, 12, 0.5);\n}\n\n.btn-warning.disabled, .btn-warning:disabled {\n  color: #212529;\n  background-color: #ffc107;\n  border-color: #ffc107;\n}\n\n.btn-warning:not(:disabled):not(.disabled):active, .btn-warning:not(:disabled):not(.disabled).active,\n.show > .btn-warning.dropdown-toggle {\n  color: #212529;\n  background-color: #d39e00;\n  border-color: #c69500;\n}\n\n.btn-warning:not(:disabled):not(.disabled):active:focus, .btn-warning:not(:disabled):not(.disabled).active:focus,\n.show > .btn-warning.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(222, 170, 12, 0.5);\n}\n\n.btn-danger {\n  color: #fff;\n  background-color: #dc3545;\n  border-color: #dc3545;\n}\n\n.btn-danger:hover {\n  color: #fff;\n  background-color: #c82333;\n  border-color: #bd2130;\n}\n\n.btn-danger:focus, .btn-danger.focus {\n  color: #fff;\n  background-color: #c82333;\n  border-color: #bd2130;\n  box-shadow: 0 0 0 0.2rem rgba(225, 83, 97, 0.5);\n}\n\n.btn-danger.disabled, .btn-danger:disabled {\n  color: #fff;\n  background-color: #dc3545;\n  border-color: #dc3545;\n}\n\n.btn-danger:not(:disabled):not(.disabled):active, .btn-danger:not(:disabled):not(.disabled).active,\n.show > .btn-danger.dropdown-toggle {\n  color: #fff;\n  background-color: #bd2130;\n  border-color: #b21f2d;\n}\n\n.btn-danger:not(:disabled):not(.disabled):active:focus, .btn-danger:not(:disabled):not(.disabled).active:focus,\n.show > .btn-danger.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(225, 83, 97, 0.5);\n}\n\n.btn-light {\n  color: #212529;\n  background-color: #f8f9fa;\n  border-color: #f8f9fa;\n}\n\n.btn-light:hover {\n  color: #212529;\n  background-color: #e2e6ea;\n  border-color: #dae0e5;\n}\n\n.btn-light:focus, .btn-light.focus {\n  color: #212529;\n  background-color: #e2e6ea;\n  border-color: #dae0e5;\n  box-shadow: 0 0 0 0.2rem rgba(216, 217, 219, 0.5);\n}\n\n.btn-light.disabled, .btn-light:disabled {\n  color: #212529;\n  background-color: #f8f9fa;\n  border-color: #f8f9fa;\n}\n\n.btn-light:not(:disabled):not(.disabled):active, .btn-light:not(:disabled):not(.disabled).active,\n.show > .btn-light.dropdown-toggle {\n  color: #212529;\n  background-color: #dae0e5;\n  border-color: #d3d9df;\n}\n\n.btn-light:not(:disabled):not(.disabled):active:focus, .btn-light:not(:disabled):not(.disabled).active:focus,\n.show > .btn-light.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(216, 217, 219, 0.5);\n}\n\n.btn-dark {\n  color: #fff;\n  background-color: #343a40;\n  border-color: #343a40;\n}\n\n.btn-dark:hover {\n  color: #fff;\n  background-color: #23272b;\n  border-color: #1d2124;\n}\n\n.btn-dark:focus, .btn-dark.focus {\n  color: #fff;\n  background-color: #23272b;\n  border-color: #1d2124;\n  box-shadow: 0 0 0 0.2rem rgba(82, 88, 93, 0.5);\n}\n\n.btn-dark.disabled, .btn-dark:disabled {\n  color: #fff;\n  background-color: #343a40;\n  border-color: #343a40;\n}\n\n.btn-dark:not(:disabled):not(.disabled):active, .btn-dark:not(:disabled):not(.disabled).active,\n.show > .btn-dark.dropdown-toggle {\n  color: #fff;\n  background-color: #1d2124;\n  border-color: #171a1d;\n}\n\n.btn-dark:not(:disabled):not(.disabled):active:focus, .btn-dark:not(:disabled):not(.disabled).active:focus,\n.show > .btn-dark.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(82, 88, 93, 0.5);\n}\n\n.btn-outline-primary {\n  color: #007bff;\n  border-color: #007bff;\n}\n\n.btn-outline-primary:hover {\n  color: #fff;\n  background-color: #007bff;\n  border-color: #007bff;\n}\n\n.btn-outline-primary:focus, .btn-outline-primary.focus {\n  box-shadow: 0 0 0 0.2rem rgba(0, 123, 255, 0.5);\n}\n\n.btn-outline-primary.disabled, .btn-outline-primary:disabled {\n  color: #007bff;\n  background-color: transparent;\n}\n\n.btn-outline-primary:not(:disabled):not(.disabled):active, .btn-outline-primary:not(:disabled):not(.disabled).active,\n.show > .btn-outline-primary.dropdown-toggle {\n  color: #fff;\n  background-color: #007bff;\n  border-color: #007bff;\n}\n\n.btn-outline-primary:not(:disabled):not(.disabled):active:focus, .btn-outline-primary:not(:disabled):not(.disabled).active:focus,\n.show > .btn-outline-primary.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(0, 123, 255, 0.5);\n}\n\n.btn-outline-secondary {\n  color: #6c757d;\n  border-color: #6c757d;\n}\n\n.btn-outline-secondary:hover {\n  color: #fff;\n  background-color: #6c757d;\n  border-color: #6c757d;\n}\n\n.btn-outline-secondary:focus, .btn-outline-secondary.focus {\n  box-shadow: 0 0 0 0.2rem rgba(108, 117, 125, 0.5);\n}\n\n.btn-outline-secondary.disabled, .btn-outline-secondary:disabled {\n  color: #6c757d;\n  background-color: transparent;\n}\n\n.btn-outline-secondary:not(:disabled):not(.disabled):active, .btn-outline-secondary:not(:disabled):not(.disabled).active,\n.show > .btn-outline-secondary.dropdown-toggle {\n  color: #fff;\n  background-color: #6c757d;\n  border-color: #6c757d;\n}\n\n.btn-outline-secondary:not(:disabled):not(.disabled):active:focus, .btn-outline-secondary:not(:disabled):not(.disabled).active:focus,\n.show > .btn-outline-secondary.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(108, 117, 125, 0.5);\n}\n\n.btn-outline-success {\n  color: #28a745;\n  border-color: #28a745;\n}\n\n.btn-outline-success:hover {\n  color: #fff;\n  background-color: #28a745;\n  border-color: #28a745;\n}\n\n.btn-outline-success:focus, .btn-outline-success.focus {\n  box-shadow: 0 0 0 0.2rem rgba(40, 167, 69, 0.5);\n}\n\n.btn-outline-success.disabled, .btn-outline-success:disabled {\n  color: #28a745;\n  background-color: transparent;\n}\n\n.btn-outline-success:not(:disabled):not(.disabled):active, .btn-outline-success:not(:disabled):not(.disabled).active,\n.show > .btn-outline-success.dropdown-toggle {\n  color: #fff;\n  background-color: #28a745;\n  border-color: #28a745;\n}\n\n.btn-outline-success:not(:disabled):not(.disabled):active:focus, .btn-outline-success:not(:disabled):not(.disabled).active:focus,\n.show > .btn-outline-success.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(40, 167, 69, 0.5);\n}\n\n.btn-outline-info {\n  color: #17a2b8;\n  border-color: #17a2b8;\n}\n\n.btn-outline-info:hover {\n  color: #fff;\n  background-color: #17a2b8;\n  border-color: #17a2b8;\n}\n\n.btn-outline-info:focus, .btn-outline-info.focus {\n  box-shadow: 0 0 0 0.2rem rgba(23, 162, 184, 0.5);\n}\n\n.btn-outline-info.disabled, .btn-outline-info:disabled {\n  color: #17a2b8;\n  background-color: transparent;\n}\n\n.btn-outline-info:not(:disabled):not(.disabled):active, .btn-outline-info:not(:disabled):not(.disabled).active,\n.show > .btn-outline-info.dropdown-toggle {\n  color: #fff;\n  background-color: #17a2b8;\n  border-color: #17a2b8;\n}\n\n.btn-outline-info:not(:disabled):not(.disabled):active:focus, .btn-outline-info:not(:disabled):not(.disabled).active:focus,\n.show > .btn-outline-info.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(23, 162, 184, 0.5);\n}\n\n.btn-outline-warning {\n  color: #ffc107;\n  border-color: #ffc107;\n}\n\n.btn-outline-warning:hover {\n  color: #212529;\n  background-color: #ffc107;\n  border-color: #ffc107;\n}\n\n.btn-outline-warning:focus, .btn-outline-warning.focus {\n  box-shadow: 0 0 0 0.2rem rgba(255, 193, 7, 0.5);\n}\n\n.btn-outline-warning.disabled, .btn-outline-warning:disabled {\n  color: #ffc107;\n  background-color: transparent;\n}\n\n.btn-outline-warning:not(:disabled):not(.disabled):active, .btn-outline-warning:not(:disabled):not(.disabled).active,\n.show > .btn-outline-warning.dropdown-toggle {\n  color: #212529;\n  background-color: #ffc107;\n  border-color: #ffc107;\n}\n\n.btn-outline-warning:not(:disabled):not(.disabled):active:focus, .btn-outline-warning:not(:disabled):not(.disabled).active:focus,\n.show > .btn-outline-warning.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(255, 193, 7, 0.5);\n}\n\n.btn-outline-danger {\n  color: #dc3545;\n  border-color: #dc3545;\n}\n\n.btn-outline-danger:hover {\n  color: #fff;\n  background-color: #dc3545;\n  border-color: #dc3545;\n}\n\n.btn-outline-danger:focus, .btn-outline-danger.focus {\n  box-shadow: 0 0 0 0.2rem rgba(220, 53, 69, 0.5);\n}\n\n.btn-outline-danger.disabled, .btn-outline-danger:disabled {\n  color: #dc3545;\n  background-color: transparent;\n}\n\n.btn-outline-danger:not(:disabled):not(.disabled):active, .btn-outline-danger:not(:disabled):not(.disabled).active,\n.show > .btn-outline-danger.dropdown-toggle {\n  color: #fff;\n  background-color: #dc3545;\n  border-color: #dc3545;\n}\n\n.btn-outline-danger:not(:disabled):not(.disabled):active:focus, .btn-outline-danger:not(:disabled):not(.disabled).active:focus,\n.show > .btn-outline-danger.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(220, 53, 69, 0.5);\n}\n\n.btn-outline-light {\n  color: #f8f9fa;\n  border-color: #f8f9fa;\n}\n\n.btn-outline-light:hover {\n  color: #212529;\n  background-color: #f8f9fa;\n  border-color: #f8f9fa;\n}\n\n.btn-outline-light:focus, .btn-outline-light.focus {\n  box-shadow: 0 0 0 0.2rem rgba(248, 249, 250, 0.5);\n}\n\n.btn-outline-light.disabled, .btn-outline-light:disabled {\n  color: #f8f9fa;\n  background-color: transparent;\n}\n\n.btn-outline-light:not(:disabled):not(.disabled):active, .btn-outline-light:not(:disabled):not(.disabled).active,\n.show > .btn-outline-light.dropdown-toggle {\n  color: #212529;\n  background-color: #f8f9fa;\n  border-color: #f8f9fa;\n}\n\n.btn-outline-light:not(:disabled):not(.disabled):active:focus, .btn-outline-light:not(:disabled):not(.disabled).active:focus,\n.show > .btn-outline-light.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(248, 249, 250, 0.5);\n}\n\n.btn-outline-dark {\n  color: #343a40;\n  border-color: #343a40;\n}\n\n.btn-outline-dark:hover {\n  color: #fff;\n  background-color: #343a40;\n  border-color: #343a40;\n}\n\n.btn-outline-dark:focus, .btn-outline-dark.focus {\n  box-shadow: 0 0 0 0.2rem rgba(52, 58, 64, 0.5);\n}\n\n.btn-outline-dark.disabled, .btn-outline-dark:disabled {\n  color: #343a40;\n  background-color: transparent;\n}\n\n.btn-outline-dark:not(:disabled):not(.disabled):active, .btn-outline-dark:not(:disabled):not(.disabled).active,\n.show > .btn-outline-dark.dropdown-toggle {\n  color: #fff;\n  background-color: #343a40;\n  border-color: #343a40;\n}\n\n.btn-outline-dark:not(:disabled):not(.disabled):active:focus, .btn-outline-dark:not(:disabled):not(.disabled).active:focus,\n.show > .btn-outline-dark.dropdown-toggle:focus {\n  box-shadow: 0 0 0 0.2rem rgba(52, 58, 64, 0.5);\n}\n\n.btn-link {\n  font-weight: 400;\n  color: #007bff;\n  text-decoration: none;\n}\n\n.btn-link:hover {\n  color: #0056b3;\n  text-decoration: underline;\n}\n\n.btn-link:focus, .btn-link.focus {\n  text-decoration: underline;\n  box-shadow: none;\n}\n\n.btn-link:disabled, .btn-link.disabled {\n  color: #6c757d;\n  pointer-events: none;\n}\n\n.btn-lg, .btn-group-lg > .btn {\n  padding: 0.5rem 1rem;\n  font-size: 1.25rem;\n  line-height: 1.5;\n  border-radius: 0.3rem;\n}\n\n.btn-sm, .btn-group-sm > .btn {\n  padding: 0.25rem 0.5rem;\n  font-size: 0.875rem;\n  line-height: 1.5;\n  border-radius: 0.2rem;\n}\n\n.btn-block {\n  display: block;\n  width: 100%;\n}\n\n.btn-block + .btn-block {\n  margin-top: 0.5rem;\n}\n\ninput[type=\"submit\"].btn-block,\ninput[type=\"reset\"].btn-block,\ninput[type=\"button\"].btn-block {\n  width: 100%;\n}\n\n.fade {\n  transition: opacity 0.15s linear;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .fade {\n    transition: none;\n  }\n}\n\n.fade:not(.show) {\n  opacity: 0;\n}\n\n.collapse:not(.show) {\n  display: none;\n}\n\n.collapsing {\n  position: relative;\n  height: 0;\n  overflow: hidden;\n  transition: height 0.35s ease;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .collapsing {\n    transition: none;\n  }\n}\n\n.dropup,\n.dropright,\n.dropdown,\n.dropleft {\n  position: relative;\n}\n\n.dropdown-toggle {\n  white-space: nowrap;\n}\n\n.dropdown-toggle::after {\n  display: inline-block;\n  margin-left: 0.255em;\n  vertical-align: 0.255em;\n  content: \"\";\n  border-top: 0.3em solid;\n  border-right: 0.3em solid transparent;\n  border-bottom: 0;\n  border-left: 0.3em solid transparent;\n}\n\n.dropdown-toggle:empty::after {\n  margin-left: 0;\n}\n\n.dropdown-menu {\n  position: absolute;\n  top: 100%;\n  left: 0;\n  z-index: 1000;\n  display: none;\n  float: left;\n  min-width: 10rem;\n  padding: 0.5rem 0;\n  margin: 0.125rem 0 0;\n  font-size: 1rem;\n  color: #212529;\n  text-align: left;\n  list-style: none;\n  background-color: #fff;\n  background-clip: padding-box;\n  border: 1px solid rgba(0, 0, 0, 0.15);\n  border-radius: 0.25rem;\n}\n\n.dropdown-menu-left {\n  right: auto;\n  left: 0;\n}\n\n.dropdown-menu-right {\n  right: 0;\n  left: auto;\n}\n\n@media (min-width: 576px) {\n  .dropdown-menu-sm-left {\n    right: auto;\n    left: 0;\n  }\n  .dropdown-menu-sm-right {\n    right: 0;\n    left: auto;\n  }\n}\n\n@media (min-width: 768px) {\n  .dropdown-menu-md-left {\n    right: auto;\n    left: 0;\n  }\n  .dropdown-menu-md-right {\n    right: 0;\n    left: auto;\n  }\n}\n\n@media (min-width: 992px) {\n  .dropdown-menu-lg-left {\n    right: auto;\n    left: 0;\n  }\n  .dropdown-menu-lg-right {\n    right: 0;\n    left: auto;\n  }\n}\n\n@media (min-width: 1200px) {\n  .dropdown-menu-xl-left {\n    right: auto;\n    left: 0;\n  }\n  .dropdown-menu-xl-right {\n    right: 0;\n    left: auto;\n  }\n}\n\n.dropup .dropdown-menu {\n  top: auto;\n  bottom: 100%;\n  margin-top: 0;\n  margin-bottom: 0.125rem;\n}\n\n.dropup .dropdown-toggle::after {\n  display: inline-block;\n  margin-left: 0.255em;\n  vertical-align: 0.255em;\n  content: \"\";\n  border-top: 0;\n  border-right: 0.3em solid transparent;\n  border-bottom: 0.3em solid;\n  border-left: 0.3em solid transparent;\n}\n\n.dropup .dropdown-toggle:empty::after {\n  margin-left: 0;\n}\n\n.dropright .dropdown-menu {\n  top: 0;\n  right: auto;\n  left: 100%;\n  margin-top: 0;\n  margin-left: 0.125rem;\n}\n\n.dropright .dropdown-toggle::after {\n  display: inline-block;\n  margin-left: 0.255em;\n  vertical-align: 0.255em;\n  content: \"\";\n  border-top: 0.3em solid transparent;\n  border-right: 0;\n  border-bottom: 0.3em solid transparent;\n  border-left: 0.3em solid;\n}\n\n.dropright .dropdown-toggle:empty::after {\n  margin-left: 0;\n}\n\n.dropright .dropdown-toggle::after {\n  vertical-align: 0;\n}\n\n.dropleft .dropdown-menu {\n  top: 0;\n  right: 100%;\n  left: auto;\n  margin-top: 0;\n  margin-right: 0.125rem;\n}\n\n.dropleft .dropdown-toggle::after {\n  display: inline-block;\n  margin-left: 0.255em;\n  vertical-align: 0.255em;\n  content: \"\";\n}\n\n.dropleft .dropdown-toggle::after {\n  display: none;\n}\n\n.dropleft .dropdown-toggle::before {\n  display: inline-block;\n  margin-right: 0.255em;\n  vertical-align: 0.255em;\n  content: \"\";\n  border-top: 0.3em solid transparent;\n  border-right: 0.3em solid;\n  border-bottom: 0.3em solid transparent;\n}\n\n.dropleft .dropdown-toggle:empty::after {\n  margin-left: 0;\n}\n\n.dropleft .dropdown-toggle::before {\n  vertical-align: 0;\n}\n\n.dropdown-menu[x-placement^=\"top\"], .dropdown-menu[x-placement^=\"right\"], .dropdown-menu[x-placement^=\"bottom\"], .dropdown-menu[x-placement^=\"left\"] {\n  right: auto;\n  bottom: auto;\n}\n\n.dropdown-divider {\n  height: 0;\n  margin: 0.5rem 0;\n  overflow: hidden;\n  border-top: 1px solid #e9ecef;\n}\n\n.dropdown-item {\n  display: block;\n  width: 100%;\n  padding: 0.25rem 1.5rem;\n  clear: both;\n  font-weight: 400;\n  color: #212529;\n  text-align: inherit;\n  white-space: nowrap;\n  background-color: transparent;\n  border: 0;\n}\n\n.dropdown-item:hover, .dropdown-item:focus {\n  color: #16181b;\n  text-decoration: none;\n  background-color: #f8f9fa;\n}\n\n.dropdown-item.active, .dropdown-item:active {\n  color: #fff;\n  text-decoration: none;\n  background-color: #007bff;\n}\n\n.dropdown-item.disabled, .dropdown-item:disabled {\n  color: #6c757d;\n  pointer-events: none;\n  background-color: transparent;\n}\n\n.dropdown-menu.show {\n  display: block;\n}\n\n.dropdown-header {\n  display: block;\n  padding: 0.5rem 1.5rem;\n  margin-bottom: 0;\n  font-size: 0.875rem;\n  color: #6c757d;\n  white-space: nowrap;\n}\n\n.dropdown-item-text {\n  display: block;\n  padding: 0.25rem 1.5rem;\n  color: #212529;\n}\n\n.btn-group,\n.btn-group-vertical {\n  position: relative;\n  display: inline-flex;\n  vertical-align: middle;\n}\n\n.btn-group > .btn,\n.btn-group-vertical > .btn {\n  position: relative;\n  flex: 1 1 auto;\n}\n\n.btn-group > .btn:hover,\n.btn-group-vertical > .btn:hover {\n  z-index: 1;\n}\n\n.btn-group > .btn:focus, .btn-group > .btn:active, .btn-group > .btn.active,\n.btn-group-vertical > .btn:focus,\n.btn-group-vertical > .btn:active,\n.btn-group-vertical > .btn.active {\n  z-index: 1;\n}\n\n.btn-toolbar {\n  display: flex;\n  flex-wrap: wrap;\n  justify-content: flex-start;\n}\n\n.btn-toolbar .input-group {\n  width: auto;\n}\n\n.btn-group > .btn:not(:first-child),\n.btn-group > .btn-group:not(:first-child) {\n  margin-left: -1px;\n}\n\n.btn-group > .btn:not(:last-child):not(.dropdown-toggle),\n.btn-group > .btn-group:not(:last-child) > .btn {\n  border-top-right-radius: 0;\n  border-bottom-right-radius: 0;\n}\n\n.btn-group > .btn:not(:first-child),\n.btn-group > .btn-group:not(:first-child) > .btn {\n  border-top-left-radius: 0;\n  border-bottom-left-radius: 0;\n}\n\n.dropdown-toggle-split {\n  padding-right: 0.5625rem;\n  padding-left: 0.5625rem;\n}\n\n.dropdown-toggle-split::after,\n.dropup .dropdown-toggle-split::after,\n.dropright .dropdown-toggle-split::after {\n  margin-left: 0;\n}\n\n.dropleft .dropdown-toggle-split::before {\n  margin-right: 0;\n}\n\n.btn-sm + .dropdown-toggle-split, .btn-group-sm > .btn + .dropdown-toggle-split {\n  padding-right: 0.375rem;\n  padding-left: 0.375rem;\n}\n\n.btn-lg + .dropdown-toggle-split, .btn-group-lg > .btn + .dropdown-toggle-split {\n  padding-right: 0.75rem;\n  padding-left: 0.75rem;\n}\n\n.btn-group-vertical {\n  flex-direction: column;\n  align-items: flex-start;\n  justify-content: center;\n}\n\n.btn-group-vertical > .btn,\n.btn-group-vertical > .btn-group {\n  width: 100%;\n}\n\n.btn-group-vertical > .btn:not(:first-child),\n.btn-group-vertical > .btn-group:not(:first-child) {\n  margin-top: -1px;\n}\n\n.btn-group-vertical > .btn:not(:last-child):not(.dropdown-toggle),\n.btn-group-vertical > .btn-group:not(:last-child) > .btn {\n  border-bottom-right-radius: 0;\n  border-bottom-left-radius: 0;\n}\n\n.btn-group-vertical > .btn:not(:first-child),\n.btn-group-vertical > .btn-group:not(:first-child) > .btn {\n  border-top-left-radius: 0;\n  border-top-right-radius: 0;\n}\n\n.btn-group-toggle > .btn,\n.btn-group-toggle > .btn-group > .btn {\n  margin-bottom: 0;\n}\n\n.btn-group-toggle > .btn input[type=\"radio\"],\n.btn-group-toggle > .btn input[type=\"checkbox\"],\n.btn-group-toggle > .btn-group > .btn input[type=\"radio\"],\n.btn-group-toggle > .btn-group > .btn input[type=\"checkbox\"] {\n  position: absolute;\n  clip: rect(0, 0, 0, 0);\n  pointer-events: none;\n}\n\n.input-group {\n  position: relative;\n  display: flex;\n  flex-wrap: wrap;\n  align-items: stretch;\n  width: 100%;\n}\n\n.input-group > .form-control,\n.input-group > .form-control-plaintext,\n.input-group > .custom-select,\n.input-group > .custom-file {\n  position: relative;\n  flex: 1 1 0%;\n  min-width: 0;\n  margin-bottom: 0;\n}\n\n.input-group > .form-control + .form-control,\n.input-group > .form-control + .custom-select,\n.input-group > .form-control + .custom-file,\n.input-group > .form-control-plaintext + .form-control,\n.input-group > .form-control-plaintext + .custom-select,\n.input-group > .form-control-plaintext + .custom-file,\n.input-group > .custom-select + .form-control,\n.input-group > .custom-select + .custom-select,\n.input-group > .custom-select + .custom-file,\n.input-group > .custom-file + .form-control,\n.input-group > .custom-file + .custom-select,\n.input-group > .custom-file + .custom-file {\n  margin-left: -1px;\n}\n\n.input-group > .form-control:focus,\n.input-group > .custom-select:focus,\n.input-group > .custom-file .custom-file-input:focus ~ .custom-file-label {\n  z-index: 3;\n}\n\n.input-group > .custom-file .custom-file-input:focus {\n  z-index: 4;\n}\n\n.input-group > .form-control:not(:last-child),\n.input-group > .custom-select:not(:last-child) {\n  border-top-right-radius: 0;\n  border-bottom-right-radius: 0;\n}\n\n.input-group > .form-control:not(:first-child),\n.input-group > .custom-select:not(:first-child) {\n  border-top-left-radius: 0;\n  border-bottom-left-radius: 0;\n}\n\n.input-group > .custom-file {\n  display: flex;\n  align-items: center;\n}\n\n.input-group > .custom-file:not(:last-child) .custom-file-label,\n.input-group > .custom-file:not(:last-child) .custom-file-label::after {\n  border-top-right-radius: 0;\n  border-bottom-right-radius: 0;\n}\n\n.input-group > .custom-file:not(:first-child) .custom-file-label {\n  border-top-left-radius: 0;\n  border-bottom-left-radius: 0;\n}\n\n.input-group-prepend,\n.input-group-append {\n  display: flex;\n}\n\n.input-group-prepend .btn,\n.input-group-append .btn {\n  position: relative;\n  z-index: 2;\n}\n\n.input-group-prepend .btn:focus,\n.input-group-append .btn:focus {\n  z-index: 3;\n}\n\n.input-group-prepend .btn + .btn,\n.input-group-prepend .btn + .input-group-text,\n.input-group-prepend .input-group-text + .input-group-text,\n.input-group-prepend .input-group-text + .btn,\n.input-group-append .btn + .btn,\n.input-group-append .btn + .input-group-text,\n.input-group-append .input-group-text + .input-group-text,\n.input-group-append .input-group-text + .btn {\n  margin-left: -1px;\n}\n\n.input-group-prepend {\n  margin-right: -1px;\n}\n\n.input-group-append {\n  margin-left: -1px;\n}\n\n.input-group-text {\n  display: flex;\n  align-items: center;\n  padding: 0.375rem 0.75rem;\n  margin-bottom: 0;\n  font-size: 1rem;\n  font-weight: 400;\n  line-height: 1.5;\n  color: #495057;\n  text-align: center;\n  white-space: nowrap;\n  background-color: #e9ecef;\n  border: 1px solid #ced4da;\n  border-radius: 0.25rem;\n}\n\n.input-group-text input[type=\"radio\"],\n.input-group-text input[type=\"checkbox\"] {\n  margin-top: 0;\n}\n\n.input-group-lg > .form-control:not(textarea),\n.input-group-lg > .custom-select {\n  height: calc(1.5em + 1rem + 2px);\n}\n\n.input-group-lg > .form-control,\n.input-group-lg > .custom-select,\n.input-group-lg > .input-group-prepend > .input-group-text,\n.input-group-lg > .input-group-append > .input-group-text,\n.input-group-lg > .input-group-prepend > .btn,\n.input-group-lg > .input-group-append > .btn {\n  padding: 0.5rem 1rem;\n  font-size: 1.25rem;\n  line-height: 1.5;\n  border-radius: 0.3rem;\n}\n\n.input-group-sm > .form-control:not(textarea),\n.input-group-sm > .custom-select {\n  height: calc(1.5em + 0.5rem + 2px);\n}\n\n.input-group-sm > .form-control,\n.input-group-sm > .custom-select,\n.input-group-sm > .input-group-prepend > .input-group-text,\n.input-group-sm > .input-group-append > .input-group-text,\n.input-group-sm > .input-group-prepend > .btn,\n.input-group-sm > .input-group-append > .btn {\n  padding: 0.25rem 0.5rem;\n  font-size: 0.875rem;\n  line-height: 1.5;\n  border-radius: 0.2rem;\n}\n\n.input-group-lg > .custom-select,\n.input-group-sm > .custom-select {\n  padding-right: 1.75rem;\n}\n\n.input-group > .input-group-prepend > .btn,\n.input-group > .input-group-prepend > .input-group-text,\n.input-group > .input-group-append:not(:last-child) > .btn,\n.input-group > .input-group-append:not(:last-child) > .input-group-text,\n.input-group > .input-group-append:last-child > .btn:not(:last-child):not(.dropdown-toggle),\n.input-group > .input-group-append:last-child > .input-group-text:not(:last-child) {\n  border-top-right-radius: 0;\n  border-bottom-right-radius: 0;\n}\n\n.input-group > .input-group-append > .btn,\n.input-group > .input-group-append > .input-group-text,\n.input-group > .input-group-prepend:not(:first-child) > .btn,\n.input-group > .input-group-prepend:not(:first-child) > .input-group-text,\n.input-group > .input-group-prepend:first-child > .btn:not(:first-child),\n.input-group > .input-group-prepend:first-child > .input-group-text:not(:first-child) {\n  border-top-left-radius: 0;\n  border-bottom-left-radius: 0;\n}\n\n.custom-control {\n  position: relative;\n  display: block;\n  min-height: 1.5rem;\n  padding-left: 1.5rem;\n}\n\n.custom-control-inline {\n  display: inline-flex;\n  margin-right: 1rem;\n}\n\n.custom-control-input {\n  position: absolute;\n  left: 0;\n  z-index: -1;\n  width: 1rem;\n  height: 1.25rem;\n  opacity: 0;\n}\n\n.custom-control-input:checked ~ .custom-control-label::before {\n  color: #fff;\n  border-color: #007bff;\n  background-color: #007bff;\n}\n\n.custom-control-input:focus ~ .custom-control-label::before {\n  box-shadow: 0 0 0 0.2rem rgba(0, 123, 255, 0.25);\n}\n\n.custom-control-input:focus:not(:checked) ~ .custom-control-label::before {\n  border-color: #80bdff;\n}\n\n.custom-control-input:not(:disabled):active ~ .custom-control-label::before {\n  color: #fff;\n  background-color: #b3d7ff;\n  border-color: #b3d7ff;\n}\n\n.custom-control-input[disabled] ~ .custom-control-label, .custom-control-input:disabled ~ .custom-control-label {\n  color: #6c757d;\n}\n\n.custom-control-input[disabled] ~ .custom-control-label::before, .custom-control-input:disabled ~ .custom-control-label::before {\n  background-color: #e9ecef;\n}\n\n.custom-control-label {\n  position: relative;\n  margin-bottom: 0;\n  vertical-align: top;\n}\n\n.custom-control-label::before {\n  position: absolute;\n  top: 0.25rem;\n  left: -1.5rem;\n  display: block;\n  width: 1rem;\n  height: 1rem;\n  pointer-events: none;\n  content: \"\";\n  background-color: #fff;\n  border: #adb5bd solid 1px;\n}\n\n.custom-control-label::after {\n  position: absolute;\n  top: 0.25rem;\n  left: -1.5rem;\n  display: block;\n  width: 1rem;\n  height: 1rem;\n  content: \"\";\n  background: no-repeat 50% / 50% 50%;\n}\n\n.custom-checkbox .custom-control-label::before {\n  border-radius: 0.25rem;\n}\n\n.custom-checkbox .custom-control-input:checked ~ .custom-control-label::after {\n  background-image: url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath fill='%23fff' d='M6.564.75l-3.59 3.612-1.538-1.55L0 4.26l2.974 2.99L8 2.193z'/%3e%3c/svg%3e\");\n}\n\n.custom-checkbox .custom-control-input:indeterminate ~ .custom-control-label::before {\n  border-color: #007bff;\n  background-color: #007bff;\n}\n\n.custom-checkbox .custom-control-input:indeterminate ~ .custom-control-label::after {\n  background-image: url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='4' viewBox='0 0 4 4'%3e%3cpath stroke='%23fff' d='M0 2h4'/%3e%3c/svg%3e\");\n}\n\n.custom-checkbox .custom-control-input:disabled:checked ~ .custom-control-label::before {\n  background-color: rgba(0, 123, 255, 0.5);\n}\n\n.custom-checkbox .custom-control-input:disabled:indeterminate ~ .custom-control-label::before {\n  background-color: rgba(0, 123, 255, 0.5);\n}\n\n.custom-radio .custom-control-label::before {\n  border-radius: 50%;\n}\n\n.custom-radio .custom-control-input:checked ~ .custom-control-label::after {\n  background-image: url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='%23fff'/%3e%3c/svg%3e\");\n}\n\n.custom-radio .custom-control-input:disabled:checked ~ .custom-control-label::before {\n  background-color: rgba(0, 123, 255, 0.5);\n}\n\n.custom-switch {\n  padding-left: 2.25rem;\n}\n\n.custom-switch .custom-control-label::before {\n  left: -2.25rem;\n  width: 1.75rem;\n  pointer-events: all;\n  border-radius: 0.5rem;\n}\n\n.custom-switch .custom-control-label::after {\n  top: calc(0.25rem + 2px);\n  left: calc(-2.25rem + 2px);\n  width: calc(1rem - 4px);\n  height: calc(1rem - 4px);\n  background-color: #adb5bd;\n  border-radius: 0.5rem;\n  transition: transform 0.15s ease-in-out, background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .custom-switch .custom-control-label::after {\n    transition: none;\n  }\n}\n\n.custom-switch .custom-control-input:checked ~ .custom-control-label::after {\n  background-color: #fff;\n  transform: translateX(0.75rem);\n}\n\n.custom-switch .custom-control-input:disabled:checked ~ .custom-control-label::before {\n  background-color: rgba(0, 123, 255, 0.5);\n}\n\n.custom-select {\n  display: inline-block;\n  width: 100%;\n  height: calc(1.5em + 0.75rem + 2px);\n  padding: 0.375rem 1.75rem 0.375rem 0.75rem;\n  font-size: 1rem;\n  font-weight: 400;\n  line-height: 1.5;\n  color: #495057;\n  vertical-align: middle;\n  background: #fff url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='5' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e\") no-repeat right 0.75rem center/8px 10px;\n  border: 1px solid #ced4da;\n  border-radius: 0.25rem;\n  appearance: none;\n}\n\n.custom-select:focus {\n  border-color: #80bdff;\n  outline: 0;\n  box-shadow: 0 0 0 0.2rem rgba(0, 123, 255, 0.25);\n}\n\n.custom-select:focus::-ms-value {\n  color: #495057;\n  background-color: #fff;\n}\n\n.custom-select[multiple], .custom-select[size]:not([size=\"1\"]) {\n  height: auto;\n  padding-right: 0.75rem;\n  background-image: none;\n}\n\n.custom-select:disabled {\n  color: #6c757d;\n  background-color: #e9ecef;\n}\n\n.custom-select::-ms-expand {\n  display: none;\n}\n\n.custom-select:-moz-focusring {\n  color: transparent;\n  text-shadow: 0 0 0 #495057;\n}\n\n.custom-select-sm {\n  height: calc(1.5em + 0.5rem + 2px);\n  padding-top: 0.25rem;\n  padding-bottom: 0.25rem;\n  padding-left: 0.5rem;\n  font-size: 0.875rem;\n}\n\n.custom-select-lg {\n  height: calc(1.5em + 1rem + 2px);\n  padding-top: 0.5rem;\n  padding-bottom: 0.5rem;\n  padding-left: 1rem;\n  font-size: 1.25rem;\n}\n\n.custom-file {\n  position: relative;\n  display: inline-block;\n  width: 100%;\n  height: calc(1.5em + 0.75rem + 2px);\n  margin-bottom: 0;\n}\n\n.custom-file-input {\n  position: relative;\n  z-index: 2;\n  width: 100%;\n  height: calc(1.5em + 0.75rem + 2px);\n  margin: 0;\n  opacity: 0;\n}\n\n.custom-file-input:focus ~ .custom-file-label {\n  border-color: #80bdff;\n  box-shadow: 0 0 0 0.2rem rgba(0, 123, 255, 0.25);\n}\n\n.custom-file-input[disabled] ~ .custom-file-label,\n.custom-file-input:disabled ~ .custom-file-label {\n  background-color: #e9ecef;\n}\n\n.custom-file-input:lang(en) ~ .custom-file-label::after {\n  content: \"Browse\";\n}\n\n.custom-file-input ~ .custom-file-label[data-browse]::after {\n  content: attr(data-browse);\n}\n\n.custom-file-label {\n  position: absolute;\n  top: 0;\n  right: 0;\n  left: 0;\n  z-index: 1;\n  height: calc(1.5em + 0.75rem + 2px);\n  padding: 0.375rem 0.75rem;\n  font-weight: 400;\n  line-height: 1.5;\n  color: #495057;\n  background-color: #fff;\n  border: 1px solid #ced4da;\n  border-radius: 0.25rem;\n}\n\n.custom-file-label::after {\n  position: absolute;\n  top: 0;\n  right: 0;\n  bottom: 0;\n  z-index: 3;\n  display: block;\n  height: calc(1.5em + 0.75rem);\n  padding: 0.375rem 0.75rem;\n  line-height: 1.5;\n  color: #495057;\n  content: \"Browse\";\n  background-color: #e9ecef;\n  border-left: inherit;\n  border-radius: 0 0.25rem 0.25rem 0;\n}\n\n.custom-range {\n  width: 100%;\n  height: 1.4rem;\n  padding: 0;\n  background-color: transparent;\n  appearance: none;\n}\n\n.custom-range:focus {\n  outline: none;\n}\n\n.custom-range:focus::-webkit-slider-thumb {\n  box-shadow: 0 0 0 1px #fff, 0 0 0 0.2rem rgba(0, 123, 255, 0.25);\n}\n\n.custom-range:focus::-moz-range-thumb {\n  box-shadow: 0 0 0 1px #fff, 0 0 0 0.2rem rgba(0, 123, 255, 0.25);\n}\n\n.custom-range:focus::-ms-thumb {\n  box-shadow: 0 0 0 1px #fff, 0 0 0 0.2rem rgba(0, 123, 255, 0.25);\n}\n\n.custom-range::-moz-focus-outer {\n  border: 0;\n}\n\n.custom-range::-webkit-slider-thumb {\n  width: 1rem;\n  height: 1rem;\n  margin-top: -0.25rem;\n  background-color: #007bff;\n  border: 0;\n  border-radius: 1rem;\n  transition: background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;\n  appearance: none;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .custom-range::-webkit-slider-thumb {\n    transition: none;\n  }\n}\n\n.custom-range::-webkit-slider-thumb:active {\n  background-color: #b3d7ff;\n}\n\n.custom-range::-webkit-slider-runnable-track {\n  width: 100%;\n  height: 0.5rem;\n  color: transparent;\n  cursor: pointer;\n  background-color: #dee2e6;\n  border-color: transparent;\n  border-radius: 1rem;\n}\n\n.custom-range::-moz-range-thumb {\n  width: 1rem;\n  height: 1rem;\n  background-color: #007bff;\n  border: 0;\n  border-radius: 1rem;\n  transition: background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;\n  appearance: none;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .custom-range::-moz-range-thumb {\n    transition: none;\n  }\n}\n\n.custom-range::-moz-range-thumb:active {\n  background-color: #b3d7ff;\n}\n\n.custom-range::-moz-range-track {\n  width: 100%;\n  height: 0.5rem;\n  color: transparent;\n  cursor: pointer;\n  background-color: #dee2e6;\n  border-color: transparent;\n  border-radius: 1rem;\n}\n\n.custom-range::-ms-thumb {\n  width: 1rem;\n  height: 1rem;\n  margin-top: 0;\n  margin-right: 0.2rem;\n  margin-left: 0.2rem;\n  background-color: #007bff;\n  border: 0;\n  border-radius: 1rem;\n  transition: background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;\n  appearance: none;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .custom-range::-ms-thumb {\n    transition: none;\n  }\n}\n\n.custom-range::-ms-thumb:active {\n  background-color: #b3d7ff;\n}\n\n.custom-range::-ms-track {\n  width: 100%;\n  height: 0.5rem;\n  color: transparent;\n  cursor: pointer;\n  background-color: transparent;\n  border-color: transparent;\n  border-width: 0.5rem;\n}\n\n.custom-range::-ms-fill-lower {\n  background-color: #dee2e6;\n  border-radius: 1rem;\n}\n\n.custom-range::-ms-fill-upper {\n  margin-right: 15px;\n  background-color: #dee2e6;\n  border-radius: 1rem;\n}\n\n.custom-range:disabled::-webkit-slider-thumb {\n  background-color: #adb5bd;\n}\n\n.custom-range:disabled::-webkit-slider-runnable-track {\n  cursor: default;\n}\n\n.custom-range:disabled::-moz-range-thumb {\n  background-color: #adb5bd;\n}\n\n.custom-range:disabled::-moz-range-track {\n  cursor: default;\n}\n\n.custom-range:disabled::-ms-thumb {\n  background-color: #adb5bd;\n}\n\n.custom-control-label::before,\n.custom-file-label,\n.custom-select {\n  transition: background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .custom-control-label::before,\n  .custom-file-label,\n  .custom-select {\n    transition: none;\n  }\n}\n\n.nav {\n  display: flex;\n  flex-wrap: wrap;\n  padding-left: 0;\n  margin-bottom: 0;\n  list-style: none;\n}\n\n.nav-link {\n  display: block;\n  padding: 0.5rem 1rem;\n}\n\n.nav-link:hover, .nav-link:focus {\n  text-decoration: none;\n}\n\n.nav-link.disabled {\n  color: #6c757d;\n  pointer-events: none;\n  cursor: default;\n}\n\n.nav-tabs {\n  border-bottom: 1px solid #dee2e6;\n}\n\n.nav-tabs .nav-item {\n  margin-bottom: -1px;\n}\n\n.nav-tabs .nav-link {\n  border: 1px solid transparent;\n  border-top-left-radius: 0.25rem;\n  border-top-right-radius: 0.25rem;\n}\n\n.nav-tabs .nav-link:hover, .nav-tabs .nav-link:focus {\n  border-color: #e9ecef #e9ecef #dee2e6;\n}\n\n.nav-tabs .nav-link.disabled {\n  color: #6c757d;\n  background-color: transparent;\n  border-color: transparent;\n}\n\n.nav-tabs .nav-link.active,\n.nav-tabs .nav-item.show .nav-link {\n  color: #495057;\n  background-color: #fff;\n  border-color: #dee2e6 #dee2e6 #fff;\n}\n\n.nav-tabs .dropdown-menu {\n  margin-top: -1px;\n  border-top-left-radius: 0;\n  border-top-right-radius: 0;\n}\n\n.nav-pills .nav-link {\n  border-radius: 0.25rem;\n}\n\n.nav-pills .nav-link.active,\n.nav-pills .show > .nav-link {\n  color: #fff;\n  background-color: #007bff;\n}\n\n.nav-fill .nav-item {\n  flex: 1 1 auto;\n  text-align: center;\n}\n\n.nav-justified .nav-item {\n  flex-basis: 0;\n  flex-grow: 1;\n  text-align: center;\n}\n\n.tab-content > .tab-pane {\n  display: none;\n}\n\n.tab-content > .active {\n  display: block;\n}\n\n.navbar {\n  position: relative;\n  display: flex;\n  flex-wrap: wrap;\n  align-items: center;\n  justify-content: space-between;\n  padding: 0.5rem 1rem;\n}\n\n.navbar .container,\n.navbar .container-fluid, .navbar .container-sm, .navbar .container-md, .navbar .container-lg, .navbar .container-xl {\n  display: flex;\n  flex-wrap: wrap;\n  align-items: center;\n  justify-content: space-between;\n}\n\n.navbar-brand {\n  display: inline-block;\n  padding-top: 0.3125rem;\n  padding-bottom: 0.3125rem;\n  margin-right: 1rem;\n  font-size: 1.25rem;\n  line-height: inherit;\n  white-space: nowrap;\n}\n\n.navbar-brand:hover, .navbar-brand:focus {\n  text-decoration: none;\n}\n\n.navbar-nav {\n  display: flex;\n  flex-direction: column;\n  padding-left: 0;\n  margin-bottom: 0;\n  list-style: none;\n}\n\n.navbar-nav .nav-link {\n  padding-right: 0;\n  padding-left: 0;\n}\n\n.navbar-nav .dropdown-menu {\n  position: static;\n  float: none;\n}\n\n.navbar-text {\n  display: inline-block;\n  padding-top: 0.5rem;\n  padding-bottom: 0.5rem;\n}\n\n.navbar-collapse {\n  flex-basis: 100%;\n  flex-grow: 1;\n  align-items: center;\n}\n\n.navbar-toggler {\n  padding: 0.25rem 0.75rem;\n  font-size: 1.25rem;\n  line-height: 1;\n  background-color: transparent;\n  border: 1px solid transparent;\n  border-radius: 0.25rem;\n}\n\n.navbar-toggler:hover, .navbar-toggler:focus {\n  text-decoration: none;\n}\n\n.navbar-toggler-icon {\n  display: inline-block;\n  width: 1.5em;\n  height: 1.5em;\n  vertical-align: middle;\n  content: \"\";\n  background: no-repeat center center;\n  background-size: 100% 100%;\n}\n\n@media (max-width: 575.98px) {\n  .navbar-expand-sm > .container,\n  .navbar-expand-sm > .container-fluid, .navbar-expand-sm > .container-sm, .navbar-expand-sm > .container-md, .navbar-expand-sm > .container-lg, .navbar-expand-sm > .container-xl {\n    padding-right: 0;\n    padding-left: 0;\n  }\n}\n\n@media (min-width: 576px) {\n  .navbar-expand-sm {\n    flex-flow: row nowrap;\n    justify-content: flex-start;\n  }\n  .navbar-expand-sm .navbar-nav {\n    flex-direction: row;\n  }\n  .navbar-expand-sm .navbar-nav .dropdown-menu {\n    position: absolute;\n  }\n  .navbar-expand-sm .navbar-nav .nav-link {\n    padding-right: 0.5rem;\n    padding-left: 0.5rem;\n  }\n  .navbar-expand-sm > .container,\n  .navbar-expand-sm > .container-fluid, .navbar-expand-sm > .container-sm, .navbar-expand-sm > .container-md, .navbar-expand-sm > .container-lg, .navbar-expand-sm > .container-xl {\n    flex-wrap: nowrap;\n  }\n  .navbar-expand-sm .navbar-collapse {\n    display: flex !important;\n    flex-basis: auto;\n  }\n  .navbar-expand-sm .navbar-toggler {\n    display: none;\n  }\n}\n\n@media (max-width: 767.98px) {\n  .navbar-expand-md > .container,\n  .navbar-expand-md > .container-fluid, .navbar-expand-md > .container-sm, .navbar-expand-md > .container-md, .navbar-expand-md > .container-lg, .navbar-expand-md > .container-xl {\n    padding-right: 0;\n    padding-left: 0;\n  }\n}\n\n@media (min-width: 768px) {\n  .navbar-expand-md {\n    flex-flow: row nowrap;\n    justify-content: flex-start;\n  }\n  .navbar-expand-md .navbar-nav {\n    flex-direction: row;\n  }\n  .navbar-expand-md .navbar-nav .dropdown-menu {\n    position: absolute;\n  }\n  .navbar-expand-md .navbar-nav .nav-link {\n    padding-right: 0.5rem;\n    padding-left: 0.5rem;\n  }\n  .navbar-expand-md > .container,\n  .navbar-expand-md > .container-fluid, .navbar-expand-md > .container-sm, .navbar-expand-md > .container-md, .navbar-expand-md > .container-lg, .navbar-expand-md > .container-xl {\n    flex-wrap: nowrap;\n  }\n  .navbar-expand-md .navbar-collapse {\n    display: flex !important;\n    flex-basis: auto;\n  }\n  .navbar-expand-md .navbar-toggler {\n    display: none;\n  }\n}\n\n@media (max-width: 991.98px) {\n  .navbar-expand-lg > .container,\n  .navbar-expand-lg > .container-fluid, .navbar-expand-lg > .container-sm, .navbar-expand-lg > .container-md, .navbar-expand-lg > .container-lg, .navbar-expand-lg > .container-xl {\n    padding-right: 0;\n    padding-left: 0;\n  }\n}\n\n@media (min-width: 992px) {\n  .navbar-expand-lg {\n    flex-flow: row nowrap;\n    justify-content: flex-start;\n  }\n  .navbar-expand-lg .navbar-nav {\n    flex-direction: row;\n  }\n  .navbar-expand-lg .navbar-nav .dropdown-menu {\n    position: absolute;\n  }\n  .navbar-expand-lg .navbar-nav .nav-link {\n    padding-right: 0.5rem;\n    padding-left: 0.5rem;\n  }\n  .navbar-expand-lg > .container,\n  .navbar-expand-lg > .container-fluid, .navbar-expand-lg > .container-sm, .navbar-expand-lg > .container-md, .navbar-expand-lg > .container-lg, .navbar-expand-lg > .container-xl {\n    flex-wrap: nowrap;\n  }\n  .navbar-expand-lg .navbar-collapse {\n    display: flex !important;\n    flex-basis: auto;\n  }\n  .navbar-expand-lg .navbar-toggler {\n    display: none;\n  }\n}\n\n@media (max-width: 1199.98px) {\n  .navbar-expand-xl > .container,\n  .navbar-expand-xl > .container-fluid, .navbar-expand-xl > .container-sm, .navbar-expand-xl > .container-md, .navbar-expand-xl > .container-lg, .navbar-expand-xl > .container-xl {\n    padding-right: 0;\n    padding-left: 0;\n  }\n}\n\n@media (min-width: 1200px) {\n  .navbar-expand-xl {\n    flex-flow: row nowrap;\n    justify-content: flex-start;\n  }\n  .navbar-expand-xl .navbar-nav {\n    flex-direction: row;\n  }\n  .navbar-expand-xl .navbar-nav .dropdown-menu {\n    position: absolute;\n  }\n  .navbar-expand-xl .navbar-nav .nav-link {\n    padding-right: 0.5rem;\n    padding-left: 0.5rem;\n  }\n  .navbar-expand-xl > .container,\n  .navbar-expand-xl > .container-fluid, .navbar-expand-xl > .container-sm, .navbar-expand-xl > .container-md, .navbar-expand-xl > .container-lg, .navbar-expand-xl > .container-xl {\n    flex-wrap: nowrap;\n  }\n  .navbar-expand-xl .navbar-collapse {\n    display: flex !important;\n    flex-basis: auto;\n  }\n  .navbar-expand-xl .navbar-toggler {\n    display: none;\n  }\n}\n\n.navbar-expand {\n  flex-flow: row nowrap;\n  justify-content: flex-start;\n}\n\n.navbar-expand > .container,\n.navbar-expand > .container-fluid, .navbar-expand > .container-sm, .navbar-expand > .container-md, .navbar-expand > .container-lg, .navbar-expand > .container-xl {\n  padding-right: 0;\n  padding-left: 0;\n}\n\n.navbar-expand .navbar-nav {\n  flex-direction: row;\n}\n\n.navbar-expand .navbar-nav .dropdown-menu {\n  position: absolute;\n}\n\n.navbar-expand .navbar-nav .nav-link {\n  padding-right: 0.5rem;\n  padding-left: 0.5rem;\n}\n\n.navbar-expand > .container,\n.navbar-expand > .container-fluid, .navbar-expand > .container-sm, .navbar-expand > .container-md, .navbar-expand > .container-lg, .navbar-expand > .container-xl {\n  flex-wrap: nowrap;\n}\n\n.navbar-expand .navbar-collapse {\n  display: flex !important;\n  flex-basis: auto;\n}\n\n.navbar-expand .navbar-toggler {\n  display: none;\n}\n\n.navbar-light .navbar-brand {\n  color: rgba(0, 0, 0, 0.9);\n}\n\n.navbar-light .navbar-brand:hover, .navbar-light .navbar-brand:focus {\n  color: rgba(0, 0, 0, 0.9);\n}\n\n.navbar-light .navbar-nav .nav-link {\n  color: rgba(0, 0, 0, 0.5);\n}\n\n.navbar-light .navbar-nav .nav-link:hover, .navbar-light .navbar-nav .nav-link:focus {\n  color: rgba(0, 0, 0, 0.7);\n}\n\n.navbar-light .navbar-nav .nav-link.disabled {\n  color: rgba(0, 0, 0, 0.3);\n}\n\n.navbar-light .navbar-nav .show > .nav-link,\n.navbar-light .navbar-nav .active > .nav-link,\n.navbar-light .navbar-nav .nav-link.show,\n.navbar-light .navbar-nav .nav-link.active {\n  color: rgba(0, 0, 0, 0.9);\n}\n\n.navbar-light .navbar-toggler {\n  color: rgba(0, 0, 0, 0.5);\n  border-color: rgba(0, 0, 0, 0.1);\n}\n\n.navbar-light .navbar-toggler-icon {\n  background-image: url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='30' height='30' viewBox='0 0 30 30'%3e%3cpath stroke='rgba(0, 0, 0, 0.5)' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e\");\n}\n\n.navbar-light .navbar-text {\n  color: rgba(0, 0, 0, 0.5);\n}\n\n.navbar-light .navbar-text a {\n  color: rgba(0, 0, 0, 0.9);\n}\n\n.navbar-light .navbar-text a:hover, .navbar-light .navbar-text a:focus {\n  color: rgba(0, 0, 0, 0.9);\n}\n\n.navbar-dark .navbar-brand {\n  color: #fff;\n}\n\n.navbar-dark .navbar-brand:hover, .navbar-dark .navbar-brand:focus {\n  color: #fff;\n}\n\n.navbar-dark .navbar-nav .nav-link {\n  color: rgba(255, 255, 255, 0.5);\n}\n\n.navbar-dark .navbar-nav .nav-link:hover, .navbar-dark .navbar-nav .nav-link:focus {\n  color: rgba(255, 255, 255, 0.75);\n}\n\n.navbar-dark .navbar-nav .nav-link.disabled {\n  color: rgba(255, 255, 255, 0.25);\n}\n\n.navbar-dark .navbar-nav .show > .nav-link,\n.navbar-dark .navbar-nav .active > .nav-link,\n.navbar-dark .navbar-nav .nav-link.show,\n.navbar-dark .navbar-nav .nav-link.active {\n  color: #fff;\n}\n\n.navbar-dark .navbar-toggler {\n  color: rgba(255, 255, 255, 0.5);\n  border-color: rgba(255, 255, 255, 0.1);\n}\n\n.navbar-dark .navbar-toggler-icon {\n  background-image: url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='30' height='30' viewBox='0 0 30 30'%3e%3cpath stroke='rgba(255, 255, 255, 0.5)' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e\");\n}\n\n.navbar-dark .navbar-text {\n  color: rgba(255, 255, 255, 0.5);\n}\n\n.navbar-dark .navbar-text a {\n  color: #fff;\n}\n\n.navbar-dark .navbar-text a:hover, .navbar-dark .navbar-text a:focus {\n  color: #fff;\n}\n\n.card {\n  position: relative;\n  display: flex;\n  flex-direction: column;\n  min-width: 0;\n  word-wrap: break-word;\n  background-color: #fff;\n  background-clip: border-box;\n  border: 1px solid rgba(0, 0, 0, 0.125);\n  border-radius: 0.25rem;\n}\n\n.card > hr {\n  margin-right: 0;\n  margin-left: 0;\n}\n\n.card > .list-group:first-child .list-group-item:first-child {\n  border-top-left-radius: 0.25rem;\n  border-top-right-radius: 0.25rem;\n}\n\n.card > .list-group:last-child .list-group-item:last-child {\n  border-bottom-right-radius: 0.25rem;\n  border-bottom-left-radius: 0.25rem;\n}\n\n.card-body {\n  flex: 1 1 auto;\n  min-height: 1px;\n  padding: 1.25rem;\n}\n\n.card-title {\n  margin-bottom: 0.75rem;\n}\n\n.card-subtitle {\n  margin-top: -0.375rem;\n  margin-bottom: 0;\n}\n\n.card-text:last-child {\n  margin-bottom: 0;\n}\n\n.card-link:hover {\n  text-decoration: none;\n}\n\n.card-link + .card-link {\n  margin-left: 1.25rem;\n}\n\n.card-header {\n  padding: 0.75rem 1.25rem;\n  margin-bottom: 0;\n  background-color: rgba(0, 0, 0, 0.03);\n  border-bottom: 1px solid rgba(0, 0, 0, 0.125);\n}\n\n.card-header:first-child {\n  border-radius: calc(0.25rem - 1px) calc(0.25rem - 1px) 0 0;\n}\n\n.card-header + .list-group .list-group-item:first-child {\n  border-top: 0;\n}\n\n.card-footer {\n  padding: 0.75rem 1.25rem;\n  background-color: rgba(0, 0, 0, 0.03);\n  border-top: 1px solid rgba(0, 0, 0, 0.125);\n}\n\n.card-footer:last-child {\n  border-radius: 0 0 calc(0.25rem - 1px) calc(0.25rem - 1px);\n}\n\n.card-header-tabs {\n  margin-right: -0.625rem;\n  margin-bottom: -0.75rem;\n  margin-left: -0.625rem;\n  border-bottom: 0;\n}\n\n.card-header-pills {\n  margin-right: -0.625rem;\n  margin-left: -0.625rem;\n}\n\n.card-img-overlay {\n  position: absolute;\n  top: 0;\n  right: 0;\n  bottom: 0;\n  left: 0;\n  padding: 1.25rem;\n}\n\n.card-img,\n.card-img-top,\n.card-img-bottom {\n  flex-shrink: 0;\n  width: 100%;\n}\n\n.card-img,\n.card-img-top {\n  border-top-left-radius: calc(0.25rem - 1px);\n  border-top-right-radius: calc(0.25rem - 1px);\n}\n\n.card-img,\n.card-img-bottom {\n  border-bottom-right-radius: calc(0.25rem - 1px);\n  border-bottom-left-radius: calc(0.25rem - 1px);\n}\n\n.card-deck .card {\n  margin-bottom: 15px;\n}\n\n@media (min-width: 576px) {\n  .card-deck {\n    display: flex;\n    flex-flow: row wrap;\n    margin-right: -15px;\n    margin-left: -15px;\n  }\n  .card-deck .card {\n    flex: 1 0 0%;\n    margin-right: 15px;\n    margin-bottom: 0;\n    margin-left: 15px;\n  }\n}\n\n.card-group > .card {\n  margin-bottom: 15px;\n}\n\n@media (min-width: 576px) {\n  .card-group {\n    display: flex;\n    flex-flow: row wrap;\n  }\n  .card-group > .card {\n    flex: 1 0 0%;\n    margin-bottom: 0;\n  }\n  .card-group > .card + .card {\n    margin-left: 0;\n    border-left: 0;\n  }\n  .card-group > .card:not(:last-child) {\n    border-top-right-radius: 0;\n    border-bottom-right-radius: 0;\n  }\n  .card-group > .card:not(:last-child) .card-img-top,\n  .card-group > .card:not(:last-child) .card-header {\n    border-top-right-radius: 0;\n  }\n  .card-group > .card:not(:last-child) .card-img-bottom,\n  .card-group > .card:not(:last-child) .card-footer {\n    border-bottom-right-radius: 0;\n  }\n  .card-group > .card:not(:first-child) {\n    border-top-left-radius: 0;\n    border-bottom-left-radius: 0;\n  }\n  .card-group > .card:not(:first-child) .card-img-top,\n  .card-group > .card:not(:first-child) .card-header {\n    border-top-left-radius: 0;\n  }\n  .card-group > .card:not(:first-child) .card-img-bottom,\n  .card-group > .card:not(:first-child) .card-footer {\n    border-bottom-left-radius: 0;\n  }\n}\n\n.card-columns .card {\n  margin-bottom: 0.75rem;\n}\n\n@media (min-width: 576px) {\n  .card-columns {\n    column-count: 3;\n    column-gap: 1.25rem;\n    orphans: 1;\n    widows: 1;\n  }\n  .card-columns .card {\n    display: inline-block;\n    width: 100%;\n  }\n}\n\n.accordion > .card {\n  overflow: hidden;\n}\n\n.accordion > .card:not(:last-of-type) {\n  border-bottom: 0;\n  border-bottom-right-radius: 0;\n  border-bottom-left-radius: 0;\n}\n\n.accordion > .card:not(:first-of-type) {\n  border-top-left-radius: 0;\n  border-top-right-radius: 0;\n}\n\n.accordion > .card > .card-header {\n  border-radius: 0;\n  margin-bottom: -1px;\n}\n\n.breadcrumb {\n  display: flex;\n  flex-wrap: wrap;\n  padding: 0.75rem 1rem;\n  margin-bottom: 1rem;\n  list-style: none;\n  background-color: #e9ecef;\n  border-radius: 0.25rem;\n}\n\n.breadcrumb-item + .breadcrumb-item {\n  padding-left: 0.5rem;\n}\n\n.breadcrumb-item + .breadcrumb-item::before {\n  display: inline-block;\n  padding-right: 0.5rem;\n  color: #6c757d;\n  content: \"/\";\n}\n\n.breadcrumb-item + .breadcrumb-item:hover::before {\n  text-decoration: underline;\n}\n\n.breadcrumb-item + .breadcrumb-item:hover::before {\n  text-decoration: none;\n}\n\n.breadcrumb-item.active {\n  color: #6c757d;\n}\n\n.pagination {\n  display: flex;\n  padding-left: 0;\n  list-style: none;\n  border-radius: 0.25rem;\n}\n\n.page-link {\n  position: relative;\n  display: block;\n  padding: 0.5rem 0.75rem;\n  margin-left: -1px;\n  line-height: 1.25;\n  color: #007bff;\n  background-color: #fff;\n  border: 1px solid #dee2e6;\n}\n\n.page-link:hover {\n  z-index: 2;\n  color: #0056b3;\n  text-decoration: none;\n  background-color: #e9ecef;\n  border-color: #dee2e6;\n}\n\n.page-link:focus {\n  z-index: 3;\n  outline: 0;\n  box-shadow: 0 0 0 0.2rem rgba(0, 123, 255, 0.25);\n}\n\n.page-item:first-child .page-link {\n  margin-left: 0;\n  border-top-left-radius: 0.25rem;\n  border-bottom-left-radius: 0.25rem;\n}\n\n.page-item:last-child .page-link {\n  border-top-right-radius: 0.25rem;\n  border-bottom-right-radius: 0.25rem;\n}\n\n.page-item.active .page-link {\n  z-index: 3;\n  color: #fff;\n  background-color: #007bff;\n  border-color: #007bff;\n}\n\n.page-item.disabled .page-link {\n  color: #6c757d;\n  pointer-events: none;\n  cursor: auto;\n  background-color: #fff;\n  border-color: #dee2e6;\n}\n\n.pagination-lg .page-link {\n  padding: 0.75rem 1.5rem;\n  font-size: 1.25rem;\n  line-height: 1.5;\n}\n\n.pagination-lg .page-item:first-child .page-link {\n  border-top-left-radius: 0.3rem;\n  border-bottom-left-radius: 0.3rem;\n}\n\n.pagination-lg .page-item:last-child .page-link {\n  border-top-right-radius: 0.3rem;\n  border-bottom-right-radius: 0.3rem;\n}\n\n.pagination-sm .page-link {\n  padding: 0.25rem 0.5rem;\n  font-size: 0.875rem;\n  line-height: 1.5;\n}\n\n.pagination-sm .page-item:first-child .page-link {\n  border-top-left-radius: 0.2rem;\n  border-bottom-left-radius: 0.2rem;\n}\n\n.pagination-sm .page-item:last-child .page-link {\n  border-top-right-radius: 0.2rem;\n  border-bottom-right-radius: 0.2rem;\n}\n\n.badge {\n  display: inline-block;\n  padding: 0.25em 0.4em;\n  font-size: 75%;\n  font-weight: 700;\n  line-height: 1;\n  text-align: center;\n  white-space: nowrap;\n  vertical-align: baseline;\n  border-radius: 0.25rem;\n  transition: color 0.15s ease-in-out, background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .badge {\n    transition: none;\n  }\n}\n\na.badge:hover, a.badge:focus {\n  text-decoration: none;\n}\n\n.badge:empty {\n  display: none;\n}\n\n.btn .badge {\n  position: relative;\n  top: -1px;\n}\n\n.badge-pill {\n  padding-right: 0.6em;\n  padding-left: 0.6em;\n  border-radius: 10rem;\n}\n\n.badge-primary {\n  color: #fff;\n  background-color: #007bff;\n}\n\na.badge-primary:hover, a.badge-primary:focus {\n  color: #fff;\n  background-color: #0062cc;\n}\n\na.badge-primary:focus, a.badge-primary.focus {\n  outline: 0;\n  box-shadow: 0 0 0 0.2rem rgba(0, 123, 255, 0.5);\n}\n\n.badge-secondary {\n  color: #fff;\n  background-color: #6c757d;\n}\n\na.badge-secondary:hover, a.badge-secondary:focus {\n  color: #fff;\n  background-color: #545b62;\n}\n\na.badge-secondary:focus, a.badge-secondary.focus {\n  outline: 0;\n  box-shadow: 0 0 0 0.2rem rgba(108, 117, 125, 0.5);\n}\n\n.badge-success {\n  color: #fff;\n  background-color: #28a745;\n}\n\na.badge-success:hover, a.badge-success:focus {\n  color: #fff;\n  background-color: #1e7e34;\n}\n\na.badge-success:focus, a.badge-success.focus {\n  outline: 0;\n  box-shadow: 0 0 0 0.2rem rgba(40, 167, 69, 0.5);\n}\n\n.badge-info {\n  color: #fff;\n  background-color: #17a2b8;\n}\n\na.badge-info:hover, a.badge-info:focus {\n  color: #fff;\n  background-color: #117a8b;\n}\n\na.badge-info:focus, a.badge-info.focus {\n  outline: 0;\n  box-shadow: 0 0 0 0.2rem rgba(23, 162, 184, 0.5);\n}\n\n.badge-warning {\n  color: #212529;\n  background-color: #ffc107;\n}\n\na.badge-warning:hover, a.badge-warning:focus {\n  color: #212529;\n  background-color: #d39e00;\n}\n\na.badge-warning:focus, a.badge-warning.focus {\n  outline: 0;\n  box-shadow: 0 0 0 0.2rem rgba(255, 193, 7, 0.5);\n}\n\n.badge-danger {\n  color: #fff;\n  background-color: #dc3545;\n}\n\na.badge-danger:hover, a.badge-danger:focus {\n  color: #fff;\n  background-color: #bd2130;\n}\n\na.badge-danger:focus, a.badge-danger.focus {\n  outline: 0;\n  box-shadow: 0 0 0 0.2rem rgba(220, 53, 69, 0.5);\n}\n\n.badge-light {\n  color: #212529;\n  background-color: #f8f9fa;\n}\n\na.badge-light:hover, a.badge-light:focus {\n  color: #212529;\n  background-color: #dae0e5;\n}\n\na.badge-light:focus, a.badge-light.focus {\n  outline: 0;\n  box-shadow: 0 0 0 0.2rem rgba(248, 249, 250, 0.5);\n}\n\n.badge-dark {\n  color: #fff;\n  background-color: #343a40;\n}\n\na.badge-dark:hover, a.badge-dark:focus {\n  color: #fff;\n  background-color: #1d2124;\n}\n\na.badge-dark:focus, a.badge-dark.focus {\n  outline: 0;\n  box-shadow: 0 0 0 0.2rem rgba(52, 58, 64, 0.5);\n}\n\n.jumbotron {\n  padding: 2rem 1rem;\n  margin-bottom: 2rem;\n  background-color: #e9ecef;\n  border-radius: 0.3rem;\n}\n\n@media (min-width: 576px) {\n  .jumbotron {\n    padding: 4rem 2rem;\n  }\n}\n\n.jumbotron-fluid {\n  padding-right: 0;\n  padding-left: 0;\n  border-radius: 0;\n}\n\n.alert {\n  position: relative;\n  padding: 0.75rem 1.25rem;\n  margin-bottom: 1rem;\n  border: 1px solid transparent;\n  border-radius: 0.25rem;\n}\n\n.alert-heading {\n  color: inherit;\n}\n\n.alert-link {\n  font-weight: 700;\n}\n\n.alert-dismissible {\n  padding-right: 4rem;\n}\n\n.alert-dismissible .close {\n  position: absolute;\n  top: 0;\n  right: 0;\n  padding: 0.75rem 1.25rem;\n  color: inherit;\n}\n\n.alert-primary {\n  color: #004085;\n  background-color: #cce5ff;\n  border-color: #b8daff;\n}\n\n.alert-primary hr {\n  border-top-color: #9fcdff;\n}\n\n.alert-primary .alert-link {\n  color: #002752;\n}\n\n.alert-secondary {\n  color: #383d41;\n  background-color: #e2e3e5;\n  border-color: #d6d8db;\n}\n\n.alert-secondary hr {\n  border-top-color: #c8cbcf;\n}\n\n.alert-secondary .alert-link {\n  color: #202326;\n}\n\n.alert-success {\n  color: #155724;\n  background-color: #d4edda;\n  border-color: #c3e6cb;\n}\n\n.alert-success hr {\n  border-top-color: #b1dfbb;\n}\n\n.alert-success .alert-link {\n  color: #0b2e13;\n}\n\n.alert-info {\n  color: #0c5460;\n  background-color: #d1ecf1;\n  border-color: #bee5eb;\n}\n\n.alert-info hr {\n  border-top-color: #abdde5;\n}\n\n.alert-info .alert-link {\n  color: #062c33;\n}\n\n.alert-warning {\n  color: #856404;\n  background-color: #fff3cd;\n  border-color: #ffeeba;\n}\n\n.alert-warning hr {\n  border-top-color: #ffe8a1;\n}\n\n.alert-warning .alert-link {\n  color: #533f03;\n}\n\n.alert-danger {\n  color: #721c24;\n  background-color: #f8d7da;\n  border-color: #f5c6cb;\n}\n\n.alert-danger hr {\n  border-top-color: #f1b0b7;\n}\n\n.alert-danger .alert-link {\n  color: #491217;\n}\n\n.alert-light {\n  color: #818182;\n  background-color: #fefefe;\n  border-color: #fdfdfe;\n}\n\n.alert-light hr {\n  border-top-color: #ececf6;\n}\n\n.alert-light .alert-link {\n  color: #686868;\n}\n\n.alert-dark {\n  color: #1b1e21;\n  background-color: #d6d8d9;\n  border-color: #c6c8ca;\n}\n\n.alert-dark hr {\n  border-top-color: #b9bbbe;\n}\n\n.alert-dark .alert-link {\n  color: #040505;\n}\n\n@keyframes progress-bar-stripes {\n  from {\n    background-position: 1rem 0;\n  }\n  to {\n    background-position: 0 0;\n  }\n}\n\n.progress {\n  display: flex;\n  height: 1rem;\n  overflow: hidden;\n  font-size: 0.75rem;\n  background-color: #e9ecef;\n  border-radius: 0.25rem;\n}\n\n.progress-bar {\n  display: flex;\n  flex-direction: column;\n  justify-content: center;\n  overflow: hidden;\n  color: #fff;\n  text-align: center;\n  white-space: nowrap;\n  background-color: #007bff;\n  transition: width 0.6s ease;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .progress-bar {\n    transition: none;\n  }\n}\n\n.progress-bar-striped {\n  background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);\n  background-size: 1rem 1rem;\n}\n\n.progress-bar-animated {\n  animation: progress-bar-stripes 1s linear infinite;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .progress-bar-animated {\n    animation: none;\n  }\n}\n\n.media {\n  display: flex;\n  align-items: flex-start;\n}\n\n.media-body {\n  flex: 1;\n}\n\n.list-group {\n  display: flex;\n  flex-direction: column;\n  padding-left: 0;\n  margin-bottom: 0;\n}\n\n.list-group-item-action {\n  width: 100%;\n  color: #495057;\n  text-align: inherit;\n}\n\n.list-group-item-action:hover, .list-group-item-action:focus {\n  z-index: 1;\n  color: #495057;\n  text-decoration: none;\n  background-color: #f8f9fa;\n}\n\n.list-group-item-action:active {\n  color: #212529;\n  background-color: #e9ecef;\n}\n\n.list-group-item {\n  position: relative;\n  display: block;\n  padding: 0.75rem 1.25rem;\n  background-color: #fff;\n  border: 1px solid rgba(0, 0, 0, 0.125);\n}\n\n.list-group-item:first-child {\n  border-top-left-radius: 0.25rem;\n  border-top-right-radius: 0.25rem;\n}\n\n.list-group-item:last-child {\n  border-bottom-right-radius: 0.25rem;\n  border-bottom-left-radius: 0.25rem;\n}\n\n.list-group-item.disabled, .list-group-item:disabled {\n  color: #6c757d;\n  pointer-events: none;\n  background-color: #fff;\n}\n\n.list-group-item.active {\n  z-index: 2;\n  color: #fff;\n  background-color: #007bff;\n  border-color: #007bff;\n}\n\n.list-group-item + .list-group-item {\n  border-top-width: 0;\n}\n\n.list-group-item + .list-group-item.active {\n  margin-top: -1px;\n  border-top-width: 1px;\n}\n\n.list-group-horizontal {\n  flex-direction: row;\n}\n\n.list-group-horizontal .list-group-item:first-child {\n  border-bottom-left-radius: 0.25rem;\n  border-top-right-radius: 0;\n}\n\n.list-group-horizontal .list-group-item:last-child {\n  border-top-right-radius: 0.25rem;\n  border-bottom-left-radius: 0;\n}\n\n.list-group-horizontal .list-group-item.active {\n  margin-top: 0;\n}\n\n.list-group-horizontal .list-group-item + .list-group-item {\n  border-top-width: 1px;\n  border-left-width: 0;\n}\n\n.list-group-horizontal .list-group-item + .list-group-item.active {\n  margin-left: -1px;\n  border-left-width: 1px;\n}\n\n@media (min-width: 576px) {\n  .list-group-horizontal-sm {\n    flex-direction: row;\n  }\n  .list-group-horizontal-sm .list-group-item:first-child {\n    border-bottom-left-radius: 0.25rem;\n    border-top-right-radius: 0;\n  }\n  .list-group-horizontal-sm .list-group-item:last-child {\n    border-top-right-radius: 0.25rem;\n    border-bottom-left-radius: 0;\n  }\n  .list-group-horizontal-sm .list-group-item.active {\n    margin-top: 0;\n  }\n  .list-group-horizontal-sm .list-group-item + .list-group-item {\n    border-top-width: 1px;\n    border-left-width: 0;\n  }\n  .list-group-horizontal-sm .list-group-item + .list-group-item.active {\n    margin-left: -1px;\n    border-left-width: 1px;\n  }\n}\n\n@media (min-width: 768px) {\n  .list-group-horizontal-md {\n    flex-direction: row;\n  }\n  .list-group-horizontal-md .list-group-item:first-child {\n    border-bottom-left-radius: 0.25rem;\n    border-top-right-radius: 0;\n  }\n  .list-group-horizontal-md .list-group-item:last-child {\n    border-top-right-radius: 0.25rem;\n    border-bottom-left-radius: 0;\n  }\n  .list-group-horizontal-md .list-group-item.active {\n    margin-top: 0;\n  }\n  .list-group-horizontal-md .list-group-item + .list-group-item {\n    border-top-width: 1px;\n    border-left-width: 0;\n  }\n  .list-group-horizontal-md .list-group-item + .list-group-item.active {\n    margin-left: -1px;\n    border-left-width: 1px;\n  }\n}\n\n@media (min-width: 992px) {\n  .list-group-horizontal-lg {\n    flex-direction: row;\n  }\n  .list-group-horizontal-lg .list-group-item:first-child {\n    border-bottom-left-radius: 0.25rem;\n    border-top-right-radius: 0;\n  }\n  .list-group-horizontal-lg .list-group-item:last-child {\n    border-top-right-radius: 0.25rem;\n    border-bottom-left-radius: 0;\n  }\n  .list-group-horizontal-lg .list-group-item.active {\n    margin-top: 0;\n  }\n  .list-group-horizontal-lg .list-group-item + .list-group-item {\n    border-top-width: 1px;\n    border-left-width: 0;\n  }\n  .list-group-horizontal-lg .list-group-item + .list-group-item.active {\n    margin-left: -1px;\n    border-left-width: 1px;\n  }\n}\n\n@media (min-width: 1200px) {\n  .list-group-horizontal-xl {\n    flex-direction: row;\n  }\n  .list-group-horizontal-xl .list-group-item:first-child {\n    border-bottom-left-radius: 0.25rem;\n    border-top-right-radius: 0;\n  }\n  .list-group-horizontal-xl .list-group-item:last-child {\n    border-top-right-radius: 0.25rem;\n    border-bottom-left-radius: 0;\n  }\n  .list-group-horizontal-xl .list-group-item.active {\n    margin-top: 0;\n  }\n  .list-group-horizontal-xl .list-group-item + .list-group-item {\n    border-top-width: 1px;\n    border-left-width: 0;\n  }\n  .list-group-horizontal-xl .list-group-item + .list-group-item.active {\n    margin-left: -1px;\n    border-left-width: 1px;\n  }\n}\n\n.list-group-flush .list-group-item {\n  border-right-width: 0;\n  border-left-width: 0;\n  border-radius: 0;\n}\n\n.list-group-flush .list-group-item:first-child {\n  border-top-width: 0;\n}\n\n.list-group-flush:last-child .list-group-item:last-child {\n  border-bottom-width: 0;\n}\n\n.list-group-item-primary {\n  color: #004085;\n  background-color: #b8daff;\n}\n\n.list-group-item-primary.list-group-item-action:hover, .list-group-item-primary.list-group-item-action:focus {\n  color: #004085;\n  background-color: #9fcdff;\n}\n\n.list-group-item-primary.list-group-item-action.active {\n  color: #fff;\n  background-color: #004085;\n  border-color: #004085;\n}\n\n.list-group-item-secondary {\n  color: #383d41;\n  background-color: #d6d8db;\n}\n\n.list-group-item-secondary.list-group-item-action:hover, .list-group-item-secondary.list-group-item-action:focus {\n  color: #383d41;\n  background-color: #c8cbcf;\n}\n\n.list-group-item-secondary.list-group-item-action.active {\n  color: #fff;\n  background-color: #383d41;\n  border-color: #383d41;\n}\n\n.list-group-item-success {\n  color: #155724;\n  background-color: #c3e6cb;\n}\n\n.list-group-item-success.list-group-item-action:hover, .list-group-item-success.list-group-item-action:focus {\n  color: #155724;\n  background-color: #b1dfbb;\n}\n\n.list-group-item-success.list-group-item-action.active {\n  color: #fff;\n  background-color: #155724;\n  border-color: #155724;\n}\n\n.list-group-item-info {\n  color: #0c5460;\n  background-color: #bee5eb;\n}\n\n.list-group-item-info.list-group-item-action:hover, .list-group-item-info.list-group-item-action:focus {\n  color: #0c5460;\n  background-color: #abdde5;\n}\n\n.list-group-item-info.list-group-item-action.active {\n  color: #fff;\n  background-color: #0c5460;\n  border-color: #0c5460;\n}\n\n.list-group-item-warning {\n  color: #856404;\n  background-color: #ffeeba;\n}\n\n.list-group-item-warning.list-group-item-action:hover, .list-group-item-warning.list-group-item-action:focus {\n  color: #856404;\n  background-color: #ffe8a1;\n}\n\n.list-group-item-warning.list-group-item-action.active {\n  color: #fff;\n  background-color: #856404;\n  border-color: #856404;\n}\n\n.list-group-item-danger {\n  color: #721c24;\n  background-color: #f5c6cb;\n}\n\n.list-group-item-danger.list-group-item-action:hover, .list-group-item-danger.list-group-item-action:focus {\n  color: #721c24;\n  background-color: #f1b0b7;\n}\n\n.list-group-item-danger.list-group-item-action.active {\n  color: #fff;\n  background-color: #721c24;\n  border-color: #721c24;\n}\n\n.list-group-item-light {\n  color: #818182;\n  background-color: #fdfdfe;\n}\n\n.list-group-item-light.list-group-item-action:hover, .list-group-item-light.list-group-item-action:focus {\n  color: #818182;\n  background-color: #ececf6;\n}\n\n.list-group-item-light.list-group-item-action.active {\n  color: #fff;\n  background-color: #818182;\n  border-color: #818182;\n}\n\n.list-group-item-dark {\n  color: #1b1e21;\n  background-color: #c6c8ca;\n}\n\n.list-group-item-dark.list-group-item-action:hover, .list-group-item-dark.list-group-item-action:focus {\n  color: #1b1e21;\n  background-color: #b9bbbe;\n}\n\n.list-group-item-dark.list-group-item-action.active {\n  color: #fff;\n  background-color: #1b1e21;\n  border-color: #1b1e21;\n}\n\n.close {\n  float: right;\n  font-size: 1.5rem;\n  font-weight: 700;\n  line-height: 1;\n  color: #000;\n  text-shadow: 0 1px 0 #fff;\n  opacity: .5;\n}\n\n.close:hover {\n  color: #000;\n  text-decoration: none;\n}\n\n.close:not(:disabled):not(.disabled):hover, .close:not(:disabled):not(.disabled):focus {\n  opacity: .75;\n}\n\nbutton.close {\n  padding: 0;\n  background-color: transparent;\n  border: 0;\n  appearance: none;\n}\n\na.close.disabled {\n  pointer-events: none;\n}\n\n.toast {\n  max-width: 350px;\n  overflow: hidden;\n  font-size: 0.875rem;\n  background-color: rgba(255, 255, 255, 0.85);\n  background-clip: padding-box;\n  border: 1px solid rgba(0, 0, 0, 0.1);\n  box-shadow: 0 0.25rem 0.75rem rgba(0, 0, 0, 0.1);\n  backdrop-filter: blur(10px);\n  opacity: 0;\n  border-radius: 0.25rem;\n}\n\n.toast:not(:last-child) {\n  margin-bottom: 0.75rem;\n}\n\n.toast.showing {\n  opacity: 1;\n}\n\n.toast.show {\n  display: block;\n  opacity: 1;\n}\n\n.toast.hide {\n  display: none;\n}\n\n.toast-header {\n  display: flex;\n  align-items: center;\n  padding: 0.25rem 0.75rem;\n  color: #6c757d;\n  background-color: rgba(255, 255, 255, 0.85);\n  background-clip: padding-box;\n  border-bottom: 1px solid rgba(0, 0, 0, 0.05);\n}\n\n.toast-body {\n  padding: 0.75rem;\n}\n\n.modal-open {\n  overflow: hidden;\n}\n\n.modal-open .modal {\n  overflow-x: hidden;\n  overflow-y: auto;\n}\n\n.modal {\n  position: fixed;\n  top: 0;\n  left: 0;\n  z-index: 1050;\n  display: none;\n  width: 100%;\n  height: 100%;\n  overflow: hidden;\n  outline: 0;\n}\n\n.modal-dialog {\n  position: relative;\n  width: auto;\n  margin: 0.5rem;\n  pointer-events: none;\n}\n\n.modal.fade .modal-dialog {\n  transition: transform 0.3s ease-out;\n  transform: translate(0, -50px);\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .modal.fade .modal-dialog {\n    transition: none;\n  }\n}\n\n.modal.show .modal-dialog {\n  transform: none;\n}\n\n.modal.modal-static .modal-dialog {\n  transform: scale(1.02);\n}\n\n.modal-dialog-scrollable {\n  display: flex;\n  max-height: calc(100% - 1rem);\n}\n\n.modal-dialog-scrollable .modal-content {\n  max-height: calc(100vh - 1rem);\n  overflow: hidden;\n}\n\n.modal-dialog-scrollable .modal-header,\n.modal-dialog-scrollable .modal-footer {\n  flex-shrink: 0;\n}\n\n.modal-dialog-scrollable .modal-body {\n  overflow-y: auto;\n}\n\n.modal-dialog-centered {\n  display: flex;\n  align-items: center;\n  min-height: calc(100% - 1rem);\n}\n\n.modal-dialog-centered::before {\n  display: block;\n  height: calc(100vh - 1rem);\n  content: \"\";\n}\n\n.modal-dialog-centered.modal-dialog-scrollable {\n  flex-direction: column;\n  justify-content: center;\n  height: 100%;\n}\n\n.modal-dialog-centered.modal-dialog-scrollable .modal-content {\n  max-height: none;\n}\n\n.modal-dialog-centered.modal-dialog-scrollable::before {\n  content: none;\n}\n\n.modal-content {\n  position: relative;\n  display: flex;\n  flex-direction: column;\n  width: 100%;\n  pointer-events: auto;\n  background-color: #fff;\n  background-clip: padding-box;\n  border: 1px solid rgba(0, 0, 0, 0.2);\n  border-radius: 0.3rem;\n  outline: 0;\n}\n\n.modal-backdrop {\n  position: fixed;\n  top: 0;\n  left: 0;\n  z-index: 1040;\n  width: 100vw;\n  height: 100vh;\n  background-color: #000;\n}\n\n.modal-backdrop.fade {\n  opacity: 0;\n}\n\n.modal-backdrop.show {\n  opacity: 0.5;\n}\n\n.modal-header {\n  display: flex;\n  align-items: flex-start;\n  justify-content: space-between;\n  padding: 1rem 1rem;\n  border-bottom: 1px solid #dee2e6;\n  border-top-left-radius: calc(0.3rem - 1px);\n  border-top-right-radius: calc(0.3rem - 1px);\n}\n\n.modal-header .close {\n  padding: 1rem 1rem;\n  margin: -1rem -1rem -1rem auto;\n}\n\n.modal-title {\n  margin-bottom: 0;\n  line-height: 1.5;\n}\n\n.modal-body {\n  position: relative;\n  flex: 1 1 auto;\n  padding: 1rem;\n}\n\n.modal-footer {\n  display: flex;\n  flex-wrap: wrap;\n  align-items: center;\n  justify-content: flex-end;\n  padding: 0.75rem;\n  border-top: 1px solid #dee2e6;\n  border-bottom-right-radius: calc(0.3rem - 1px);\n  border-bottom-left-radius: calc(0.3rem - 1px);\n}\n\n.modal-footer > * {\n  margin: 0.25rem;\n}\n\n.modal-scrollbar-measure {\n  position: absolute;\n  top: -9999px;\n  width: 50px;\n  height: 50px;\n  overflow: scroll;\n}\n\n@media (min-width: 576px) {\n  .modal-dialog {\n    max-width: 500px;\n    margin: 1.75rem auto;\n  }\n  .modal-dialog-scrollable {\n    max-height: calc(100% - 3.5rem);\n  }\n  .modal-dialog-scrollable .modal-content {\n    max-height: calc(100vh - 3.5rem);\n  }\n  .modal-dialog-centered {\n    min-height: calc(100% - 3.5rem);\n  }\n  .modal-dialog-centered::before {\n    height: calc(100vh - 3.5rem);\n  }\n  .modal-sm {\n    max-width: 300px;\n  }\n}\n\n@media (min-width: 992px) {\n  .modal-lg,\n  .modal-xl {\n    max-width: 800px;\n  }\n}\n\n@media (min-width: 1200px) {\n  .modal-xl {\n    max-width: 1140px;\n  }\n}\n\n.tooltip {\n  position: absolute;\n  z-index: 1070;\n  display: block;\n  margin: 0;\n  font-family: -apple-system, BlinkMacSystemFont, \"Segoe UI\", Roboto, \"Helvetica Neue\", Arial, \"Noto Sans\", sans-serif, \"Apple Color Emoji\", \"Segoe UI Emoji\", \"Segoe UI Symbol\", \"Noto Color Emoji\";\n  font-style: normal;\n  font-weight: 400;\n  line-height: 1.5;\n  text-align: left;\n  text-align: start;\n  text-decoration: none;\n  text-shadow: none;\n  text-transform: none;\n  letter-spacing: normal;\n  word-break: normal;\n  word-spacing: normal;\n  white-space: normal;\n  line-break: auto;\n  font-size: 0.875rem;\n  word-wrap: break-word;\n  opacity: 0;\n}\n\n.tooltip.show {\n  opacity: 0.9;\n}\n\n.tooltip .arrow {\n  position: absolute;\n  display: block;\n  width: 0.8rem;\n  height: 0.4rem;\n}\n\n.tooltip .arrow::before {\n  position: absolute;\n  content: \"\";\n  border-color: transparent;\n  border-style: solid;\n}\n\n.bs-tooltip-top, .bs-tooltip-auto[x-placement^=\"top\"] {\n  padding: 0.4rem 0;\n}\n\n.bs-tooltip-top .arrow, .bs-tooltip-auto[x-placement^=\"top\"] .arrow {\n  bottom: 0;\n}\n\n.bs-tooltip-top .arrow::before, .bs-tooltip-auto[x-placement^=\"top\"] .arrow::before {\n  top: 0;\n  border-width: 0.4rem 0.4rem 0;\n  border-top-color: #000;\n}\n\n.bs-tooltip-right, .bs-tooltip-auto[x-placement^=\"right\"] {\n  padding: 0 0.4rem;\n}\n\n.bs-tooltip-right .arrow, .bs-tooltip-auto[x-placement^=\"right\"] .arrow {\n  left: 0;\n  width: 0.4rem;\n  height: 0.8rem;\n}\n\n.bs-tooltip-right .arrow::before, .bs-tooltip-auto[x-placement^=\"right\"] .arrow::before {\n  right: 0;\n  border-width: 0.4rem 0.4rem 0.4rem 0;\n  border-right-color: #000;\n}\n\n.bs-tooltip-bottom, .bs-tooltip-auto[x-placement^=\"bottom\"] {\n  padding: 0.4rem 0;\n}\n\n.bs-tooltip-bottom .arrow, .bs-tooltip-auto[x-placement^=\"bottom\"] .arrow {\n  top: 0;\n}\n\n.bs-tooltip-bottom .arrow::before, .bs-tooltip-auto[x-placement^=\"bottom\"] .arrow::before {\n  bottom: 0;\n  border-width: 0 0.4rem 0.4rem;\n  border-bottom-color: #000;\n}\n\n.bs-tooltip-left, .bs-tooltip-auto[x-placement^=\"left\"] {\n  padding: 0 0.4rem;\n}\n\n.bs-tooltip-left .arrow, .bs-tooltip-auto[x-placement^=\"left\"] .arrow {\n  right: 0;\n  width: 0.4rem;\n  height: 0.8rem;\n}\n\n.bs-tooltip-left .arrow::before, .bs-tooltip-auto[x-placement^=\"left\"] .arrow::before {\n  left: 0;\n  border-width: 0.4rem 0 0.4rem 0.4rem;\n  border-left-color: #000;\n}\n\n.tooltip-inner {\n  max-width: 200px;\n  padding: 0.25rem 0.5rem;\n  color: #fff;\n  text-align: center;\n  background-color: #000;\n  border-radius: 0.25rem;\n}\n\n.popover {\n  position: absolute;\n  top: 0;\n  left: 0;\n  z-index: 1060;\n  display: block;\n  max-width: 276px;\n  font-family: -apple-system, BlinkMacSystemFont, \"Segoe UI\", Roboto, \"Helvetica Neue\", Arial, \"Noto Sans\", sans-serif, \"Apple Color Emoji\", \"Segoe UI Emoji\", \"Segoe UI Symbol\", \"Noto Color Emoji\";\n  font-style: normal;\n  font-weight: 400;\n  line-height: 1.5;\n  text-align: left;\n  text-align: start;\n  text-decoration: none;\n  text-shadow: none;\n  text-transform: none;\n  letter-spacing: normal;\n  word-break: normal;\n  word-spacing: normal;\n  white-space: normal;\n  line-break: auto;\n  font-size: 0.875rem;\n  word-wrap: break-word;\n  background-color: #fff;\n  background-clip: padding-box;\n  border: 1px solid rgba(0, 0, 0, 0.2);\n  border-radius: 0.3rem;\n}\n\n.popover .arrow {\n  position: absolute;\n  display: block;\n  width: 1rem;\n  height: 0.5rem;\n  margin: 0 0.3rem;\n}\n\n.popover .arrow::before, .popover .arrow::after {\n  position: absolute;\n  display: block;\n  content: \"\";\n  border-color: transparent;\n  border-style: solid;\n}\n\n.bs-popover-top, .bs-popover-auto[x-placement^=\"top\"] {\n  margin-bottom: 0.5rem;\n}\n\n.bs-popover-top > .arrow, .bs-popover-auto[x-placement^=\"top\"] > .arrow {\n  bottom: calc(-0.5rem - 1px);\n}\n\n.bs-popover-top > .arrow::before, .bs-popover-auto[x-placement^=\"top\"] > .arrow::before {\n  bottom: 0;\n  border-width: 0.5rem 0.5rem 0;\n  border-top-color: rgba(0, 0, 0, 0.25);\n}\n\n.bs-popover-top > .arrow::after, .bs-popover-auto[x-placement^=\"top\"] > .arrow::after {\n  bottom: 1px;\n  border-width: 0.5rem 0.5rem 0;\n  border-top-color: #fff;\n}\n\n.bs-popover-right, .bs-popover-auto[x-placement^=\"right\"] {\n  margin-left: 0.5rem;\n}\n\n.bs-popover-right > .arrow, .bs-popover-auto[x-placement^=\"right\"] > .arrow {\n  left: calc(-0.5rem - 1px);\n  width: 0.5rem;\n  height: 1rem;\n  margin: 0.3rem 0;\n}\n\n.bs-popover-right > .arrow::before, .bs-popover-auto[x-placement^=\"right\"] > .arrow::before {\n  left: 0;\n  border-width: 0.5rem 0.5rem 0.5rem 0;\n  border-right-color: rgba(0, 0, 0, 0.25);\n}\n\n.bs-popover-right > .arrow::after, .bs-popover-auto[x-placement^=\"right\"] > .arrow::after {\n  left: 1px;\n  border-width: 0.5rem 0.5rem 0.5rem 0;\n  border-right-color: #fff;\n}\n\n.bs-popover-bottom, .bs-popover-auto[x-placement^=\"bottom\"] {\n  margin-top: 0.5rem;\n}\n\n.bs-popover-bottom > .arrow, .bs-popover-auto[x-placement^=\"bottom\"] > .arrow {\n  top: calc(-0.5rem - 1px);\n}\n\n.bs-popover-bottom > .arrow::before, .bs-popover-auto[x-placement^=\"bottom\"] > .arrow::before {\n  top: 0;\n  border-width: 0 0.5rem 0.5rem 0.5rem;\n  border-bottom-color: rgba(0, 0, 0, 0.25);\n}\n\n.bs-popover-bottom > .arrow::after, .bs-popover-auto[x-placement^=\"bottom\"] > .arrow::after {\n  top: 1px;\n  border-width: 0 0.5rem 0.5rem 0.5rem;\n  border-bottom-color: #fff;\n}\n\n.bs-popover-bottom .popover-header::before, .bs-popover-auto[x-placement^=\"bottom\"] .popover-header::before {\n  position: absolute;\n  top: 0;\n  left: 50%;\n  display: block;\n  width: 1rem;\n  margin-left: -0.5rem;\n  content: \"\";\n  border-bottom: 1px solid #f7f7f7;\n}\n\n.bs-popover-left, .bs-popover-auto[x-placement^=\"left\"] {\n  margin-right: 0.5rem;\n}\n\n.bs-popover-left > .arrow, .bs-popover-auto[x-placement^=\"left\"] > .arrow {\n  right: calc(-0.5rem - 1px);\n  width: 0.5rem;\n  height: 1rem;\n  margin: 0.3rem 0;\n}\n\n.bs-popover-left > .arrow::before, .bs-popover-auto[x-placement^=\"left\"] > .arrow::before {\n  right: 0;\n  border-width: 0.5rem 0 0.5rem 0.5rem;\n  border-left-color: rgba(0, 0, 0, 0.25);\n}\n\n.bs-popover-left > .arrow::after, .bs-popover-auto[x-placement^=\"left\"] > .arrow::after {\n  right: 1px;\n  border-width: 0.5rem 0 0.5rem 0.5rem;\n  border-left-color: #fff;\n}\n\n.popover-header {\n  padding: 0.5rem 0.75rem;\n  margin-bottom: 0;\n  font-size: 1rem;\n  background-color: #f7f7f7;\n  border-bottom: 1px solid #ebebeb;\n  border-top-left-radius: calc(0.3rem - 1px);\n  border-top-right-radius: calc(0.3rem - 1px);\n}\n\n.popover-header:empty {\n  display: none;\n}\n\n.popover-body {\n  padding: 0.5rem 0.75rem;\n  color: #212529;\n}\n\n.carousel {\n  position: relative;\n}\n\n.carousel.pointer-event {\n  touch-action: pan-y;\n}\n\n.carousel-inner {\n  position: relative;\n  width: 100%;\n  overflow: hidden;\n}\n\n.carousel-inner::after {\n  display: block;\n  clear: both;\n  content: \"\";\n}\n\n.carousel-item {\n  position: relative;\n  display: none;\n  float: left;\n  width: 100%;\n  margin-right: -100%;\n  backface-visibility: hidden;\n  transition: transform 0.6s ease-in-out;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .carousel-item {\n    transition: none;\n  }\n}\n\n.carousel-item.active,\n.carousel-item-next,\n.carousel-item-prev {\n  display: block;\n}\n\n.carousel-item-next:not(.carousel-item-left),\n.active.carousel-item-right {\n  transform: translateX(100%);\n}\n\n.carousel-item-prev:not(.carousel-item-right),\n.active.carousel-item-left {\n  transform: translateX(-100%);\n}\n\n.carousel-fade .carousel-item {\n  opacity: 0;\n  transition-property: opacity;\n  transform: none;\n}\n\n.carousel-fade .carousel-item.active,\n.carousel-fade .carousel-item-next.carousel-item-left,\n.carousel-fade .carousel-item-prev.carousel-item-right {\n  z-index: 1;\n  opacity: 1;\n}\n\n.carousel-fade .active.carousel-item-left,\n.carousel-fade .active.carousel-item-right {\n  z-index: 0;\n  opacity: 0;\n  transition: opacity 0s 0.6s;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .carousel-fade .active.carousel-item-left,\n  .carousel-fade .active.carousel-item-right {\n    transition: none;\n  }\n}\n\n.carousel-control-prev,\n.carousel-control-next {\n  position: absolute;\n  top: 0;\n  bottom: 0;\n  z-index: 1;\n  display: flex;\n  align-items: center;\n  justify-content: center;\n  width: 15%;\n  color: #fff;\n  text-align: center;\n  opacity: 0.5;\n  transition: opacity 0.15s ease;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .carousel-control-prev,\n  .carousel-control-next {\n    transition: none;\n  }\n}\n\n.carousel-control-prev:hover, .carousel-control-prev:focus,\n.carousel-control-next:hover,\n.carousel-control-next:focus {\n  color: #fff;\n  text-decoration: none;\n  outline: 0;\n  opacity: 0.9;\n}\n\n.carousel-control-prev {\n  left: 0;\n}\n\n.carousel-control-next {\n  right: 0;\n}\n\n.carousel-control-prev-icon,\n.carousel-control-next-icon {\n  display: inline-block;\n  width: 20px;\n  height: 20px;\n  background: no-repeat 50% / 100% 100%;\n}\n\n.carousel-control-prev-icon {\n  background-image: url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath d='M5.25 0l-4 4 4 4 1.5-1.5L4.25 4l2.5-2.5L5.25 0z'/%3e%3c/svg%3e\");\n}\n\n.carousel-control-next-icon {\n  background-image: url(\"data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath d='M2.75 0l-1.5 1.5L3.75 4l-2.5 2.5L2.75 8l4-4-4-4z'/%3e%3c/svg%3e\");\n}\n\n.carousel-indicators {\n  position: absolute;\n  right: 0;\n  bottom: 0;\n  left: 0;\n  z-index: 15;\n  display: flex;\n  justify-content: center;\n  padding-left: 0;\n  margin-right: 15%;\n  margin-left: 15%;\n  list-style: none;\n}\n\n.carousel-indicators li {\n  box-sizing: content-box;\n  flex: 0 1 auto;\n  width: 30px;\n  height: 3px;\n  margin-right: 3px;\n  margin-left: 3px;\n  text-indent: -999px;\n  cursor: pointer;\n  background-color: #fff;\n  background-clip: padding-box;\n  border-top: 10px solid transparent;\n  border-bottom: 10px solid transparent;\n  opacity: .5;\n  transition: opacity 0.6s ease;\n}\n\n@media (prefers-reduced-motion: reduce) {\n  .carousel-indicators li {\n    transition: none;\n  }\n}\n\n.carousel-indicators .active {\n  opacity: 1;\n}\n\n.carousel-caption {\n  position: absolute;\n  right: 15%;\n  bottom: 20px;\n  left: 15%;\n  z-index: 10;\n  padding-top: 20px;\n  padding-bottom: 20px;\n  color: #fff;\n  text-align: center;\n}\n\n@keyframes spinner-border {\n  to {\n    transform: rotate(360deg);\n  }\n}\n\n.spinner-border {\n  display: inline-block;\n  width: 2rem;\n  height: 2rem;\n  vertical-align: text-bottom;\n  border: 0.25em solid currentColor;\n  border-right-color: transparent;\n  border-radius: 50%;\n  animation: spinner-border .75s linear infinite;\n}\n\n.spinner-border-sm {\n  width: 1rem;\n  height: 1rem;\n  border-width: 0.2em;\n}\n\n@keyframes spinner-grow {\n  0% {\n    transform: scale(0);\n  }\n  50% {\n    opacity: 1;\n  }\n}\n\n.spinner-grow {\n  display: inline-block;\n  width: 2rem;\n  height: 2rem;\n  vertical-align: text-bottom;\n  background-color: currentColor;\n  border-radius: 50%;\n  opacity: 0;\n  animation: spinner-grow .75s linear infinite;\n}\n\n.spinner-grow-sm {\n  width: 1rem;\n  height: 1rem;\n}\n\n.align-baseline {\n  vertical-align: baseline !important;\n}\n\n.align-top {\n  vertical-align: top !important;\n}\n\n.align-middle {\n  vertical-align: middle !important;\n}\n\n.align-bottom {\n  vertical-align: bottom !important;\n}\n\n.align-text-bottom {\n  vertical-align: text-bottom !important;\n}\n\n.align-text-top {\n  vertical-align: text-top !important;\n}\n\n.bg-primary {\n  background-color: #007bff !important;\n}\n\na.bg-primary:hover, a.bg-primary:focus,\nbutton.bg-primary:hover,\nbutton.bg-primary:focus {\n  background-color: #0062cc !important;\n}\n\n.bg-secondary {\n  background-color: #6c757d !important;\n}\n\na.bg-secondary:hover, a.bg-secondary:focus,\nbutton.bg-secondary:hover,\nbutton.bg-secondary:focus {\n  background-color: #545b62 !important;\n}\n\n.bg-success {\n  background-color: #28a745 !important;\n}\n\na.bg-success:hover, a.bg-success:focus,\nbutton.bg-success:hover,\nbutton.bg-success:focus {\n  background-color: #1e7e34 !important;\n}\n\n.bg-info {\n  background-color: #17a2b8 !important;\n}\n\na.bg-info:hover, a.bg-info:focus,\nbutton.bg-info:hover,\nbutton.bg-info:focus {\n  background-color: #117a8b !important;\n}\n\n.bg-warning {\n  background-color: #ffc107 !important;\n}\n\na.bg-warning:hover, a.bg-warning:focus,\nbutton.bg-warning:hover,\nbutton.bg-warning:focus {\n  background-color: #d39e00 !important;\n}\n\n.bg-danger {\n  background-color: #dc3545 !important;\n}\n\na.bg-danger:hover, a.bg-danger:focus,\nbutton.bg-danger:hover,\nbutton.bg-danger:focus {\n  background-color: #bd2130 !important;\n}\n\n.bg-light {\n  background-color: #f8f9fa !important;\n}\n\na.bg-light:hover, a.bg-light:focus,\nbutton.bg-light:hover,\nbutton.bg-light:focus {\n  background-color: #dae0e5 !important;\n}\n\n.bg-dark {\n  background-color: #343a40 !important;\n}\n\na.bg-dark:hover, a.bg-dark:focus,\nbutton.bg-dark:hover,\nbutton.bg-dark:focus {\n  background-color: #1d2124 !important;\n}\n\n.bg-white {\n  background-color: #fff !important;\n}\n\n.bg-transparent {\n  background-color: transparent !important;\n}\n\n.border {\n  border: 1px solid #dee2e6 !important;\n}\n\n.border-top {\n  border-top: 1px solid #dee2e6 !important;\n}\n\n.border-right {\n  border-right: 1px solid #dee2e6 !important;\n}\n\n.border-bottom {\n  border-bottom: 1px solid #dee2e6 !important;\n}\n\n.border-left {\n  border-left: 1px solid #dee2e6 !important;\n}\n\n.border-0 {\n  border: 0 !important;\n}\n\n.border-top-0 {\n  border-top: 0 !important;\n}\n\n.border-right-0 {\n  border-right: 0 !important;\n}\n\n.border-bottom-0 {\n  border-bottom: 0 !important;\n}\n\n.border-left-0 {\n  border-left: 0 !important;\n}\n\n.border-primary {\n  border-color: #007bff !important;\n}\n\n.border-secondary {\n  border-color: #6c757d !important;\n}\n\n.border-success {\n  border-color: #28a745 !important;\n}\n\n.border-info {\n  border-color: #17a2b8 !important;\n}\n\n.border-warning {\n  border-color: #ffc107 !important;\n}\n\n.border-danger {\n  border-color: #dc3545 !important;\n}\n\n.border-light {\n  border-color: #f8f9fa !important;\n}\n\n.border-dark {\n  border-color: #343a40 !important;\n}\n\n.border-white {\n  border-color: #fff !important;\n}\n\n.rounded-sm {\n  border-radius: 0.2rem !important;\n}\n\n.rounded {\n  border-radius: 0.25rem !important;\n}\n\n.rounded-top {\n  border-top-left-radius: 0.25rem !important;\n  border-top-right-radius: 0.25rem !important;\n}\n\n.rounded-right {\n  border-top-right-radius: 0.25rem !important;\n  border-bottom-right-radius: 0.25rem !important;\n}\n\n.rounded-bottom {\n  border-bottom-right-radius: 0.25rem !important;\n  border-bottom-left-radius: 0.25rem !important;\n}\n\n.rounded-left {\n  border-top-left-radius: 0.25rem !important;\n  border-bottom-left-radius: 0.25rem !important;\n}\n\n.rounded-lg {\n  border-radius: 0.3rem !important;\n}\n\n.rounded-circle {\n  border-radius: 50% !important;\n}\n\n.rounded-pill {\n  border-radius: 50rem !important;\n}\n\n.rounded-0 {\n  border-radius: 0 !important;\n}\n\n.clearfix::after {\n  display: block;\n  clear: both;\n  content: \"\";\n}\n\n.d-none {\n  display: none !important;\n}\n\n.d-inline {\n  display: inline !important;\n}\n\n.d-inline-block {\n  display: inline-block !important;\n}\n\n.d-block {\n  display: block !important;\n}\n\n.d-table {\n  display: table !important;\n}\n\n.d-table-row {\n  display: table-row !important;\n}\n\n.d-table-cell {\n  display: table-cell !important;\n}\n\n.d-flex {\n  display: flex !important;\n}\n\n.d-inline-flex {\n  display: inline-flex !important;\n}\n\n@media (min-width: 576px) {\n  .d-sm-none {\n    display: none !important;\n  }\n  .d-sm-inline {\n    display: inline !important;\n  }\n  .d-sm-inline-block {\n    display: inline-block !important;\n  }\n  .d-sm-block {\n    display: block !important;\n  }\n  .d-sm-table {\n    display: table !important;\n  }\n  .d-sm-table-row {\n    display: table-row !important;\n  }\n  .d-sm-table-cell {\n    display: table-cell !important;\n  }\n  .d-sm-flex {\n    display: flex !important;\n  }\n  .d-sm-inline-flex {\n    display: inline-flex !important;\n  }\n}\n\n@media (min-width: 768px) {\n  .d-md-none {\n    display: none !important;\n  }\n  .d-md-inline {\n    display: inline !important;\n  }\n  .d-md-inline-block {\n    display: inline-block !important;\n  }\n  .d-md-block {\n    display: block !important;\n  }\n  .d-md-table {\n    display: table !important;\n  }\n  .d-md-table-row {\n    display: table-row !important;\n  }\n  .d-md-table-cell {\n    display: table-cell !important;\n  }\n  .d-md-flex {\n    display: flex !important;\n  }\n  .d-md-inline-flex {\n    display: inline-flex !important;\n  }\n}\n\n@media (min-width: 992px) {\n  .d-lg-none {\n    display: none !important;\n  }\n  .d-lg-inline {\n    display: inline !important;\n  }\n  .d-lg-inline-block {\n    display: inline-block !important;\n  }\n  .d-lg-block {\n    display: block !important;\n  }\n  .d-lg-table {\n    display: table !important;\n  }\n  .d-lg-table-row {\n    display: table-row !important;\n  }\n  .d-lg-table-cell {\n    display: table-cell !important;\n  }\n  .d-lg-flex {\n    display: flex !important;\n  }\n  .d-lg-inline-flex {\n    display: inline-flex !important;\n  }\n}\n\n@media (min-width: 1200px) {\n  .d-xl-none {\n    display: none !important;\n  }\n  .d-xl-inline {\n    display: inline !important;\n  }\n  .d-xl-inline-block {\n    display: inline-block !important;\n  }\n  .d-xl-block {\n    display: block !important;\n  }\n  .d-xl-table {\n    display: table !important;\n  }\n  .d-xl-table-row {\n    display: table-row !important;\n  }\n  .d-xl-table-cell {\n    display: table-cell !important;\n  }\n  .d-xl-flex {\n    display: flex !important;\n  }\n  .d-xl-inline-flex {\n    display: inline-flex !important;\n  }\n}\n\n@media print {\n  .d-print-none {\n    display: none !important;\n  }\n  .d-print-inline {\n    display: inline !important;\n  }\n  .d-print-inline-block {\n    display: inline-block !important;\n  }\n  .d-print-block {\n    display: block !important;\n  }\n  .d-print-table {\n    display: table !important;\n  }\n  .d-print-table-row {\n    display: table-row !important;\n  }\n  .d-print-table-cell {\n    display: table-cell !important;\n  }\n  .d-print-flex {\n    display: flex !important;\n  }\n  .d-print-inline-flex {\n    display: inline-flex !important;\n  }\n}\n\n.embed-responsive {\n  position: relative;\n  display: block;\n  width: 100%;\n  padding: 0;\n  overflow: hidden;\n}\n\n.embed-responsive::before {\n  display: block;\n  content: \"\";\n}\n\n.embed-responsive .embed-responsive-item,\n.embed-responsive iframe,\n.embed-responsive embed,\n.embed-responsive object,\n.embed-responsive video {\n  position: absolute;\n  top: 0;\n  bottom: 0;\n  left: 0;\n  width: 100%;\n  height: 100%;\n  border: 0;\n}\n\n.embed-responsive-21by9::before {\n  padding-top: 42.857143%;\n}\n\n.embed-responsive-16by9::before {\n  padding-top: 56.25%;\n}\n\n.embed-responsive-4by3::before {\n  padding-top: 75%;\n}\n\n.embed-responsive-1by1::before {\n  padding-top: 100%;\n}\n\n.flex-row {\n  flex-direction: row !important;\n}\n\n.flex-column {\n  flex-direction: column !important;\n}\n\n.flex-row-reverse {\n  flex-direction: row-reverse !important;\n}\n\n.flex-column-reverse {\n  flex-direction: column-reverse !important;\n}\n\n.flex-wrap {\n  flex-wrap: wrap !important;\n}\n\n.flex-nowrap {\n  flex-wrap: nowrap !important;\n}\n\n.flex-wrap-reverse {\n  flex-wrap: wrap-reverse !important;\n}\n\n.flex-fill {\n  flex: 1 1 auto !important;\n}\n\n.flex-grow-0 {\n  flex-grow: 0 !important;\n}\n\n.flex-grow-1 {\n  flex-grow: 1 !important;\n}\n\n.flex-shrink-0 {\n  flex-shrink: 0 !important;\n}\n\n.flex-shrink-1 {\n  flex-shrink: 1 !important;\n}\n\n.justify-content-start {\n  justify-content: flex-start !important;\n}\n\n.justify-content-end {\n  justify-content: flex-end !important;\n}\n\n.justify-content-center {\n  justify-content: center !important;\n}\n\n.justify-content-between {\n  justify-content: space-between !important;\n}\n\n.justify-content-around {\n  justify-content: space-around !important;\n}\n\n.align-items-start {\n  align-items: flex-start !important;\n}\n\n.align-items-end {\n  align-items: flex-end !important;\n}\n\n.align-items-center {\n  align-items: center !important;\n}\n\n.align-items-baseline {\n  align-items: baseline !important;\n}\n\n.align-items-stretch {\n  align-items: stretch !important;\n}\n\n.align-content-start {\n  align-content: flex-start !important;\n}\n\n.align-content-end {\n  align-content: flex-end !important;\n}\n\n.align-content-center {\n  align-content: center !important;\n}\n\n.align-content-between {\n  align-content: space-between !important;\n}\n\n.align-content-around {\n  align-content: space-around !important;\n}\n\n.align-content-stretch {\n  align-content: stretch !important;\n}\n\n.align-self-auto {\n  align-self: auto !important;\n}\n\n.align-self-start {\n  align-self: flex-start !important;\n}\n\n.align-self-end {\n  align-self: flex-end !important;\n}\n\n.align-self-center {\n  align-self: center !important;\n}\n\n.align-self-baseline {\n  align-self: baseline !important;\n}\n\n.align-self-stretch {\n  align-self: stretch !important;\n}\n\n@media (min-width: 576px) {\n  .flex-sm-row {\n    flex-direction: row !important;\n  }\n  .flex-sm-column {\n    flex-direction: column !important;\n  }\n  .flex-sm-row-reverse {\n    flex-direction: row-reverse !important;\n  }\n  .flex-sm-column-reverse {\n    flex-direction: column-reverse !important;\n  }\n  .flex-sm-wrap {\n    flex-wrap: wrap !important;\n  }\n  .flex-sm-nowrap {\n    flex-wrap: nowrap !important;\n  }\n  .flex-sm-wrap-reverse {\n    flex-wrap: wrap-reverse !important;\n  }\n  .flex-sm-fill {\n    flex: 1 1 auto !important;\n  }\n  .flex-sm-grow-0 {\n    flex-grow: 0 !important;\n  }\n  .flex-sm-grow-1 {\n    flex-grow: 1 !important;\n  }\n  .flex-sm-shrink-0 {\n    flex-shrink: 0 !important;\n  }\n  .flex-sm-shrink-1 {\n    flex-shrink: 1 !important;\n  }\n  .justify-content-sm-start {\n    justify-content: flex-start !important;\n  }\n  .justify-content-sm-end {\n    justify-content: flex-end !important;\n  }\n  .justify-content-sm-center {\n    justify-content: center !important;\n  }\n  .justify-content-sm-between {\n    justify-content: space-between !important;\n  }\n  .justify-content-sm-around {\n    justify-content: space-around !important;\n  }\n  .align-items-sm-start {\n    align-items: flex-start !important;\n  }\n  .align-items-sm-end {\n    align-items: flex-end !important;\n  }\n  .align-items-sm-center {\n    align-items: center !important;\n  }\n  .align-items-sm-baseline {\n    align-items: baseline !important;\n  }\n  .align-items-sm-stretch {\n    align-items: stretch !important;\n  }\n  .align-content-sm-start {\n    align-content: flex-start !important;\n  }\n  .align-content-sm-end {\n    align-content: flex-end !important;\n  }\n  .align-content-sm-center {\n    align-content: center !important;\n  }\n  .align-content-sm-between {\n    align-content: space-between !important;\n  }\n  .align-content-sm-around {\n    align-content: space-around !important;\n  }\n  .align-content-sm-stretch {\n    align-content: stretch !important;\n  }\n  .align-self-sm-auto {\n    align-self: auto !important;\n  }\n  .align-self-sm-start {\n    align-self: flex-start !important;\n  }\n  .align-self-sm-end {\n    align-self: flex-end !important;\n  }\n  .align-self-sm-center {\n    align-self: center !important;\n  }\n  .align-self-sm-baseline {\n    align-self: baseline !important;\n  }\n  .align-self-sm-stretch {\n    align-self: stretch !important;\n  }\n}\n\n@media (min-width: 768px) {\n  .flex-md-row {\n    flex-direction: row !important;\n  }\n  .flex-md-column {\n    flex-direction: column !important;\n  }\n  .flex-md-row-reverse {\n    flex-direction: row-reverse !important;\n  }\n  .flex-md-column-reverse {\n    flex-direction: column-reverse !important;\n  }\n  .flex-md-wrap {\n    flex-wrap: wrap !important;\n  }\n  .flex-md-nowrap {\n    flex-wrap: nowrap !important;\n  }\n  .flex-md-wrap-reverse {\n    flex-wrap: wrap-reverse !important;\n  }\n  .flex-md-fill {\n    flex: 1 1 auto !important;\n  }\n  .flex-md-grow-0 {\n    flex-grow: 0 !important;\n  }\n  .flex-md-grow-1 {\n    flex-grow: 1 !important;\n  }\n  .flex-md-shrink-0 {\n    flex-shrink: 0 !important;\n  }\n  .flex-md-shrink-1 {\n    flex-shrink: 1 !important;\n  }\n  .justify-content-md-start {\n    justify-content: flex-start !important;\n  }\n  .justify-content-md-end {\n    justify-content: flex-end !important;\n  }\n  .justify-content-md-center {\n    justify-content: center !important;\n  }\n  .justify-content-md-between {\n    justify-content: space-between !important;\n  }\n  .justify-content-md-around {\n    justify-content: space-around !important;\n  }\n  .align-items-md-start {\n    align-items: flex-start !important;\n  }\n  .align-items-md-end {\n    align-items: flex-end !important;\n  }\n  .align-items-md-center {\n    align-items: center !important;\n  }\n  .align-items-md-baseline {\n    align-items: baseline !important;\n  }\n  .align-items-md-stretch {\n    align-items: stretch !important;\n  }\n  .align-content-md-start {\n    align-content: flex-start !important;\n  }\n  .align-content-md-end {\n    align-content: flex-end !important;\n  }\n  .align-content-md-center {\n    align-content: center !important;\n  }\n  .align-content-md-between {\n    align-content: space-between !important;\n  }\n  .align-content-md-around {\n    align-content: space-around !important;\n  }\n  .align-content-md-stretch {\n    align-content: stretch !important;\n  }\n  .align-self-md-auto {\n    align-self: auto !important;\n  }\n  .align-self-md-start {\n    align-self: flex-start !important;\n  }\n  .align-self-md-end {\n    align-self: flex-end !important;\n  }\n  .align-self-md-center {\n    align-self: center !important;\n  }\n  .align-self-md-baseline {\n    align-self: baseline !important;\n  }\n  .align-self-md-stretch {\n    align-self: stretch !important;\n  }\n}\n\n@media (min-width: 992px) {\n  .flex-lg-row {\n    flex-direction: row !important;\n  }\n  .flex-lg-column {\n    flex-direction: column !important;\n  }\n  .flex-lg-row-reverse {\n    flex-direction: row-reverse !important;\n  }\n  .flex-lg-column-reverse {\n    flex-direction: column-reverse !important;\n  }\n  .flex-lg-wrap {\n    flex-wrap: wrap !important;\n  }\n  .flex-lg-nowrap {\n    flex-wrap: nowrap !important;\n  }\n  .flex-lg-wrap-reverse {\n    flex-wrap: wrap-reverse !important;\n  }\n  .flex-lg-fill {\n    flex: 1 1 auto !important;\n  }\n  .flex-lg-grow-0 {\n    flex-grow: 0 !important;\n  }\n  .flex-lg-grow-1 {\n    flex-grow: 1 !important;\n  }\n  .flex-lg-shrink-0 {\n    flex-shrink: 0 !important;\n  }\n  .flex-lg-shrink-1 {\n    flex-shrink: 1 !important;\n  }\n  .justify-content-lg-start {\n    justify-content: flex-start !important;\n  }\n  .justify-content-lg-end {\n    justify-content: flex-end !important;\n  }\n  .justify-content-lg-center {\n    justify-content: center !important;\n  }\n  .justify-content-lg-between {\n    justify-content: space-between !important;\n  }\n  .justify-content-lg-around {\n    justify-content: space-around !important;\n  }\n  .align-items-lg-start {\n    align-items: flex-start !important;\n  }\n  .align-items-lg-end {\n    align-items: flex-end !important;\n  }\n  .align-items-lg-center {\n    align-items: center !important;\n  }\n  .align-items-lg-baseline {\n    align-items: baseline !important;\n  }\n  .align-items-lg-stretch {\n    align-items: stretch !important;\n  }\n  .align-content-lg-start {\n    align-content: flex-start !important;\n  }\n  .align-content-lg-end {\n    align-content: flex-end !important;\n  }\n  .align-content-lg-center {\n    align-content: center !important;\n  }\n  .align-content-lg-between {\n    align-content: space-between !important;\n  }\n  .align-content-lg-around {\n    align-content: space-around !important;\n  }\n  .align-content-lg-stretch {\n    align-content: stretch !important;\n  }\n  .align-self-lg-auto {\n    align-self: auto !important;\n  }\n  .align-self-lg-start {\n    align-self: flex-start !important;\n  }\n  .align-self-lg-end {\n    align-self: flex-end !important;\n  }\n  .align-self-lg-center {\n    align-self: center !important;\n  }\n  .align-self-lg-baseline {\n    align-self: baseline !important;\n  }\n  .align-self-lg-stretch {\n    align-self: stretch !important;\n  }\n}\n\n@media (min-width: 1200px) {\n  .flex-xl-row {\n    flex-direction: row !important;\n  }\n  .flex-xl-column {\n    flex-direction: column !important;\n  }\n  .flex-xl-row-reverse {\n    flex-direction: row-reverse !important;\n  }\n  .flex-xl-column-reverse {\n    flex-direction: column-reverse !important;\n  }\n  .flex-xl-wrap {\n    flex-wrap: wrap !important;\n  }\n  .flex-xl-nowrap {\n    flex-wrap: nowrap !important;\n  }\n  .flex-xl-wrap-reverse {\n    flex-wrap: wrap-reverse !important;\n  }\n  .flex-xl-fill {\n    flex: 1 1 auto !important;\n  }\n  .flex-xl-grow-0 {\n    flex-grow: 0 !important;\n  }\n  .flex-xl-grow-1 {\n    flex-grow: 1 !important;\n  }\n  .flex-xl-shrink-0 {\n    flex-shrink: 0 !important;\n  }\n  .flex-xl-shrink-1 {\n    flex-shrink: 1 !important;\n  }\n  .justify-content-xl-start {\n    justify-content: flex-start !important;\n  }\n  .justify-content-xl-end {\n    justify-content: flex-end !important;\n  }\n  .justify-content-xl-center {\n    justify-content: center !important;\n  }\n  .justify-content-xl-between {\n    justify-content: space-between !important;\n  }\n  .justify-content-xl-around {\n    justify-content: space-around !important;\n  }\n  .align-items-xl-start {\n    align-items: flex-start !important;\n  }\n  .align-items-xl-end {\n    align-items: flex-end !important;\n  }\n  .align-items-xl-center {\n    align-items: center !important;\n  }\n  .align-items-xl-baseline {\n    align-items: baseline !important;\n  }\n  .align-items-xl-stretch {\n    align-items: stretch !important;\n  }\n  .align-content-xl-start {\n    align-content: flex-start !important;\n  }\n  .align-content-xl-end {\n    align-content: flex-end !important;\n  }\n  .align-content-xl-center {\n    align-content: center !important;\n  }\n  .align-content-xl-between {\n    align-content: space-between !important;\n  }\n  .align-content-xl-around {\n    align-content: space-around !important;\n  }\n  .align-content-xl-stretch {\n    align-content: stretch !important;\n  }\n  .align-self-xl-auto {\n    align-self: auto !important;\n  }\n  .align-self-xl-start {\n    align-self: flex-start !important;\n  }\n  .align-self-xl-end {\n    align-self: flex-end !important;\n  }\n  .align-self-xl-center {\n    align-self: center !important;\n  }\n  .align-self-xl-baseline {\n    align-self: baseline !important;\n  }\n  .align-self-xl-stretch {\n    align-self: stretch !important;\n  }\n}\n\n.float-left {\n  float: left !important;\n}\n\n.float-right {\n  float: right !important;\n}\n\n.float-none {\n  float: none !important;\n}\n\n@media (min-width: 576px) {\n  .float-sm-left {\n    float: left !important;\n  }\n  .float-sm-right {\n    float: right !important;\n  }\n  .float-sm-none {\n    float: none !important;\n  }\n}\n\n@media (min-width: 768px) {\n  .float-md-left {\n    float: left !important;\n  }\n  .float-md-right {\n    float: right !important;\n  }\n  .float-md-none {\n    float: none !important;\n  }\n}\n\n@media (min-width: 992px) {\n  .float-lg-left {\n    float: left !important;\n  }\n  .float-lg-right {\n    float: right !important;\n  }\n  .float-lg-none {\n    float: none !important;\n  }\n}\n\n@media (min-width: 1200px) {\n  .float-xl-left {\n    float: left !important;\n  }\n  .float-xl-right {\n    float: right !important;\n  }\n  .float-xl-none {\n    float: none !important;\n  }\n}\n\n.overflow-auto {\n  overflow: auto !important;\n}\n\n.overflow-hidden {\n  overflow: hidden !important;\n}\n\n.position-static {\n  position: static !important;\n}\n\n.position-relative {\n  position: relative !important;\n}\n\n.position-absolute {\n  position: absolute !important;\n}\n\n.position-fixed {\n  position: fixed !important;\n}\n\n.position-sticky {\n  position: sticky !important;\n}\n\n.fixed-top {\n  position: fixed;\n  top: 0;\n  right: 0;\n  left: 0;\n  z-index: 1030;\n}\n\n.fixed-bottom {\n  position: fixed;\n  right: 0;\n  bottom: 0;\n  left: 0;\n  z-index: 1030;\n}\n\n@supports (position: sticky) {\n  .sticky-top {\n    position: sticky;\n    top: 0;\n    z-index: 1020;\n  }\n}\n\n.sr-only {\n  position: absolute;\n  width: 1px;\n  height: 1px;\n  padding: 0;\n  margin: -1px;\n  overflow: hidden;\n  clip: rect(0, 0, 0, 0);\n  white-space: nowrap;\n  border: 0;\n}\n\n.sr-only-focusable:active, .sr-only-focusable:focus {\n  position: static;\n  width: auto;\n  height: auto;\n  overflow: visible;\n  clip: auto;\n  white-space: normal;\n}\n\n.shadow-sm {\n  box-shadow: 0 0.125rem 0.25rem rgba(0, 0, 0, 0.075) !important;\n}\n\n.shadow {\n  box-shadow: 0 0.5rem 1rem rgba(0, 0, 0, 0.15) !important;\n}\n\n.shadow-lg {\n  box-shadow: 0 1rem 3rem rgba(0, 0, 0, 0.175) !important;\n}\n\n.shadow-none {\n  box-shadow: none !important;\n}\n\n.w-25 {\n  width: 25% !important;\n}\n\n.w-50 {\n  width: 50% !important;\n}\n\n.w-75 {\n  width: 75% !important;\n}\n\n.w-100 {\n  width: 100% !important;\n}\n\n.w-auto {\n  width: auto !important;\n}\n\n.h-25 {\n  height: 25% !important;\n}\n\n.h-50 {\n  height: 50% !important;\n}\n\n.h-75 {\n  height: 75% !important;\n}\n\n.h-100 {\n  height: 100% !important;\n}\n\n.h-auto {\n  height: auto !important;\n}\n\n.mw-100 {\n  max-width: 100% !important;\n}\n\n.mh-100 {\n  max-height: 100% !important;\n}\n\n.min-vw-100 {\n  min-width: 100vw !important;\n}\n\n.min-vh-100 {\n  min-height: 100vh !important;\n}\n\n.vw-100 {\n  width: 100vw !important;\n}\n\n.vh-100 {\n  height: 100vh !important;\n}\n\n.stretched-link::after {\n  position: absolute;\n  top: 0;\n  right: 0;\n  bottom: 0;\n  left: 0;\n  z-index: 1;\n  pointer-events: auto;\n  content: \"\";\n  background-color: rgba(0, 0, 0, 0);\n}\n\n.m-0 {\n  margin: 0 !important;\n}\n\n.mt-0,\n.my-0 {\n  margin-top: 0 !important;\n}\n\n.mr-0,\n.mx-0 {\n  margin-right: 0 !important;\n}\n\n.mb-0,\n.my-0 {\n  margin-bottom: 0 !important;\n}\n\n.ml-0,\n.mx-0 {\n  margin-left: 0 !important;\n}\n\n.m-1 {\n  margin: 0.25rem !important;\n}\n\n.mt-1,\n.my-1 {\n  margin-top: 0.25rem !important;\n}\n\n.mr-1,\n.mx-1 {\n  margin-right: 0.25rem !important;\n}\n\n.mb-1,\n.my-1 {\n  margin-bottom: 0.25rem !important;\n}\n\n.ml-1,\n.mx-1 {\n  margin-left: 0.25rem !important;\n}\n\n.m-2 {\n  margin: 0.5rem !important;\n}\n\n.mt-2,\n.my-2 {\n  margin-top: 0.5rem !important;\n}\n\n.mr-2,\n.mx-2 {\n  margin-right: 0.5rem !important;\n}\n\n.mb-2,\n.my-2 {\n  margin-bottom: 0.5rem !important;\n}\n\n.ml-2,\n.mx-2 {\n  margin-left: 0.5rem !important;\n}\n\n.m-3 {\n  margin: 1rem !important;\n}\n\n.mt-3,\n.my-3 {\n  margin-top: 1rem !important;\n}\n\n.mr-3,\n.mx-3 {\n  margin-right: 1rem !important;\n}\n\n.mb-3,\n.my-3 {\n  margin-bottom: 1rem !important;\n}\n\n.ml-3,\n.mx-3 {\n  margin-left: 1rem !important;\n}\n\n.m-4 {\n  margin: 1.5rem !important;\n}\n\n.mt-4,\n.my-4 {\n  margin-top: 1.5rem !important;\n}\n\n.mr-4,\n.mx-4 {\n  margin-right: 1.5rem !important;\n}\n\n.mb-4,\n.my-4 {\n  margin-bottom: 1.5rem !important;\n}\n\n.ml-4,\n.mx-4 {\n  margin-left: 1.5rem !important;\n}\n\n.m-5 {\n  margin: 3rem !important;\n}\n\n.mt-5,\n.my-5 {\n  margin-top: 3rem !important;\n}\n\n.mr-5,\n.mx-5 {\n  margin-right: 3rem !important;\n}\n\n.mb-5,\n.my-5 {\n  margin-bottom: 3rem !important;\n}\n\n.ml-5,\n.mx-5 {\n  margin-left: 3rem !important;\n}\n\n.p-0 {\n  padding: 0 !important;\n}\n\n.pt-0,\n.py-0 {\n  padding-top: 0 !important;\n}\n\n.pr-0,\n.px-0 {\n  padding-right: 0 !important;\n}\n\n.pb-0,\n.py-0 {\n  padding-bottom: 0 !important;\n}\n\n.pl-0,\n.px-0 {\n  padding-left: 0 !important;\n}\n\n.p-1 {\n  padding: 0.25rem !important;\n}\n\n.pt-1,\n.py-1 {\n  padding-top: 0.25rem !important;\n}\n\n.pr-1,\n.px-1 {\n  padding-right: 0.25rem !important;\n}\n\n.pb-1,\n.py-1 {\n  padding-bottom: 0.25rem !important;\n}\n\n.pl-1,\n.px-1 {\n  padding-left: 0.25rem !important;\n}\n\n.p-2 {\n  padding: 0.5rem !important;\n}\n\n.pt-2,\n.py-2 {\n  padding-top: 0.5rem !important;\n}\n\n.pr-2,\n.px-2 {\n  padding-right: 0.5rem !important;\n}\n\n.pb-2,\n.py-2 {\n  padding-bottom: 0.5rem !important;\n}\n\n.pl-2,\n.px-2 {\n  padding-left: 0.5rem !important;\n}\n\n.p-3 {\n  padding: 1rem !important;\n}\n\n.pt-3,\n.py-3 {\n  padding-top: 1rem !important;\n}\n\n.pr-3,\n.px-3 {\n  padding-right: 1rem !important;\n}\n\n.pb-3,\n.py-3 {\n  padding-bottom: 1rem !important;\n}\n\n.pl-3,\n.px-3 {\n  padding-left: 1rem !important;\n}\n\n.p-4 {\n  padding: 1.5rem !important;\n}\n\n.pt-4,\n.py-4 {\n  padding-top: 1.5rem !important;\n}\n\n.pr-4,\n.px-4 {\n  padding-right: 1.5rem !important;\n}\n\n.pb-4,\n.py-4 {\n  padding-bottom: 1.5rem !important;\n}\n\n.pl-4,\n.px-4 {\n  padding-left: 1.5rem !important;\n}\n\n.p-5 {\n  padding: 3rem !important;\n}\n\n.pt-5,\n.py-5 {\n  padding-top: 3rem !important;\n}\n\n.pr-5,\n.px-5 {\n  padding-right: 3rem !important;\n}\n\n.pb-5,\n.py-5 {\n  padding-bottom: 3rem !important;\n}\n\n.pl-5,\n.px-5 {\n  padding-left: 3rem !important;\n}\n\n.m-n1 {\n  margin: -0.25rem !important;\n}\n\n.mt-n1,\n.my-n1 {\n  margin-top: -0.25rem !important;\n}\n\n.mr-n1,\n.mx-n1 {\n  margin-right: -0.25rem !important;\n}\n\n.mb-n1,\n.my-n1 {\n  margin-bottom: -0.25rem !important;\n}\n\n.ml-n1,\n.mx-n1 {\n  margin-left: -0.25rem !important;\n}\n\n.m-n2 {\n  margin: -0.5rem !important;\n}\n\n.mt-n2,\n.my-n2 {\n  margin-top: -0.5rem !important;\n}\n\n.mr-n2,\n.mx-n2 {\n  margin-right: -0.5rem !important;\n}\n\n.mb-n2,\n.my-n2 {\n  margin-bottom: -0.5rem !important;\n}\n\n.ml-n2,\n.mx-n2 {\n  margin-left: -0.5rem !important;\n}\n\n.m-n3 {\n  margin: -1rem !important;\n}\n\n.mt-n3,\n.my-n3 {\n  margin-top: -1rem !important;\n}\n\n.mr-n3,\n.mx-n3 {\n  margin-right: -1rem !important;\n}\n\n.mb-n3,\n.my-n3 {\n  margin-bottom: -1rem !important;\n}\n\n.ml-n3,\n.mx-n3 {\n  margin-left: -1rem !important;\n}\n\n.m-n4 {\n  margin: -1.5rem !important;\n}\n\n.mt-n4,\n.my-n4 {\n  margin-top: -1.5rem !important;\n}\n\n.mr-n4,\n.mx-n4 {\n  margin-right: -1.5rem !important;\n}\n\n.mb-n4,\n.my-n4 {\n  margin-bottom: -1.5rem !important;\n}\n\n.ml-n4,\n.mx-n4 {\n  margin-left: -1.5rem !important;\n}\n\n.m-n5 {\n  margin: -3rem !important;\n}\n\n.mt-n5,\n.my-n5 {\n  margin-top: -3rem !important;\n}\n\n.mr-n5,\n.mx-n5 {\n  margin-right: -3rem !important;\n}\n\n.mb-n5,\n.my-n5 {\n  margin-bottom: -3rem !important;\n}\n\n.ml-n5,\n.mx-n5 {\n  margin-left: -3rem !important;\n}\n\n.m-auto {\n  margin: auto !important;\n}\n\n.mt-auto,\n.my-auto {\n  margin-top: auto !important;\n}\n\n.mr-auto,\n.mx-auto {\n  margin-right: auto !important;\n}\n\n.mb-auto,\n.my-auto {\n  margin-bottom: auto !important;\n}\n\n.ml-auto,\n.mx-auto {\n  margin-left: auto !important;\n}\n\n@media (min-width: 576px) {\n  .m-sm-0 {\n    margin: 0 !important;\n  }\n  .mt-sm-0,\n  .my-sm-0 {\n    margin-top: 0 !important;\n  }\n  .mr-sm-0,\n  .mx-sm-0 {\n    margin-right: 0 !important;\n  }\n  .mb-sm-0,\n  .my-sm-0 {\n    margin-bottom: 0 !important;\n  }\n  .ml-sm-0,\n  .mx-sm-0 {\n    margin-left: 0 !important;\n  }\n  .m-sm-1 {\n    margin: 0.25rem !important;\n  }\n  .mt-sm-1,\n  .my-sm-1 {\n    margin-top: 0.25rem !important;\n  }\n  .mr-sm-1,\n  .mx-sm-1 {\n    margin-right: 0.25rem !important;\n  }\n  .mb-sm-1,\n  .my-sm-1 {\n    margin-bottom: 0.25rem !important;\n  }\n  .ml-sm-1,\n  .mx-sm-1 {\n    margin-left: 0.25rem !important;\n  }\n  .m-sm-2 {\n    margin: 0.5rem !important;\n  }\n  .mt-sm-2,\n  .my-sm-2 {\n    margin-top: 0.5rem !important;\n  }\n  .mr-sm-2,\n  .mx-sm-2 {\n    margin-right: 0.5rem !important;\n  }\n  .mb-sm-2,\n  .my-sm-2 {\n    margin-bottom: 0.5rem !important;\n  }\n  .ml-sm-2,\n  .mx-sm-2 {\n    margin-left: 0.5rem !important;\n  }\n  .m-sm-3 {\n    margin: 1rem !important;\n  }\n  .mt-sm-3,\n  .my-sm-3 {\n    margin-top: 1rem !important;\n  }\n  .mr-sm-3,\n  .mx-sm-3 {\n    margin-right: 1rem !important;\n  }\n  .mb-sm-3,\n  .my-sm-3 {\n    margin-bottom: 1rem !important;\n  }\n  .ml-sm-3,\n  .mx-sm-3 {\n    margin-left: 1rem !important;\n  }\n  .m-sm-4 {\n    margin: 1.5rem !important;\n  }\n  .mt-sm-4,\n  .my-sm-4 {\n    margin-top: 1.5rem !important;\n  }\n  .mr-sm-4,\n  .mx-sm-4 {\n    margin-right: 1.5rem !important;\n  }\n  .mb-sm-4,\n  .my-sm-4 {\n    margin-bottom: 1.5rem !important;\n  }\n  .ml-sm-4,\n  .mx-sm-4 {\n    margin-left: 1.5rem !important;\n  }\n  .m-sm-5 {\n    margin: 3rem !important;\n  }\n  .mt-sm-5,\n  .my-sm-5 {\n    margin-top: 3rem !important;\n  }\n  .mr-sm-5,\n  .mx-sm-5 {\n    margin-right: 3rem !important;\n  }\n  .mb-sm-5,\n  .my-sm-5 {\n    margin-bottom: 3rem !important;\n  }\n  .ml-sm-5,\n  .mx-sm-5 {\n    margin-left: 3rem !important;\n  }\n  .p-sm-0 {\n    padding: 0 !important;\n  }\n  .pt-sm-0,\n  .py-sm-0 {\n    padding-top: 0 !important;\n  }\n  .pr-sm-0,\n  .px-sm-0 {\n    padding-right: 0 !important;\n  }\n  .pb-sm-0,\n  .py-sm-0 {\n    padding-bottom: 0 !important;\n  }\n  .pl-sm-0,\n  .px-sm-0 {\n    padding-left: 0 !important;\n  }\n  .p-sm-1 {\n    padding: 0.25rem !important;\n  }\n  .pt-sm-1,\n  .py-sm-1 {\n    padding-top: 0.25rem !important;\n  }\n  .pr-sm-1,\n  .px-sm-1 {\n    padding-right: 0.25rem !important;\n  }\n  .pb-sm-1,\n  .py-sm-1 {\n    padding-bottom: 0.25rem !important;\n  }\n  .pl-sm-1,\n  .px-sm-1 {\n    padding-left: 0.25rem !important;\n  }\n  .p-sm-2 {\n    padding: 0.5rem !important;\n  }\n  .pt-sm-2,\n  .py-sm-2 {\n    padding-top: 0.5rem !important;\n  }\n  .pr-sm-2,\n  .px-sm-2 {\n    padding-right: 0.5rem !important;\n  }\n  .pb-sm-2,\n  .py-sm-2 {\n    padding-bottom: 0.5rem !important;\n  }\n  .pl-sm-2,\n  .px-sm-2 {\n    padding-left: 0.5rem !important;\n  }\n  .p-sm-3 {\n    padding: 1rem !important;\n  }\n  .pt-sm-3,\n  .py-sm-3 {\n    padding-top: 1rem !important;\n  }\n  .pr-sm-3,\n  .px-sm-3 {\n    padding-right: 1rem !important;\n  }\n  .pb-sm-3,\n  .py-sm-3 {\n    padding-bottom: 1rem !important;\n  }\n  .pl-sm-3,\n  .px-sm-3 {\n    padding-left: 1rem !important;\n  }\n  .p-sm-4 {\n    padding: 1.5rem !important;\n  }\n  .pt-sm-4,\n  .py-sm-4 {\n    padding-top: 1.5rem !important;\n  }\n  .pr-sm-4,\n  .px-sm-4 {\n    padding-right: 1.5rem !important;\n  }\n  .pb-sm-4,\n  .py-sm-4 {\n    padding-bottom: 1.5rem !important;\n  }\n  .pl-sm-4,\n  .px-sm-4 {\n    padding-left: 1.5rem !important;\n  }\n  .p-sm-5 {\n    padding: 3rem !important;\n  }\n  .pt-sm-5,\n  .py-sm-5 {\n    padding-top: 3rem !important;\n  }\n  .pr-sm-5,\n  .px-sm-5 {\n    padding-right: 3rem !important;\n  }\n  .pb-sm-5,\n  .py-sm-5 {\n    padding-bottom: 3rem !important;\n  }\n  .pl-sm-5,\n  .px-sm-5 {\n    padding-left: 3rem !important;\n  }\n  .m-sm-n1 {\n    margin: -0.25rem !important;\n  }\n  .mt-sm-n1,\n  .my-sm-n1 {\n    margin-top: -0.25rem !important;\n  }\n  .mr-sm-n1,\n  .mx-sm-n1 {\n    margin-right: -0.25rem !important;\n  }\n  .mb-sm-n1,\n  .my-sm-n1 {\n    margin-bottom: -0.25rem !important;\n  }\n  .ml-sm-n1,\n  .mx-sm-n1 {\n    margin-left: -0.25rem !important;\n  }\n  .m-sm-n2 {\n    margin: -0.5rem !important;\n  }\n  .mt-sm-n2,\n  .my-sm-n2 {\n    margin-top: -0.5rem !important;\n  }\n  .mr-sm-n2,\n  .mx-sm-n2 {\n    margin-right: -0.5rem !important;\n  }\n  .mb-sm-n2,\n  .my-sm-n2 {\n    margin-bottom: -0.5rem !important;\n  }\n  .ml-sm-n2,\n  .mx-sm-n2 {\n    margin-left: -0.5rem !important;\n  }\n  .m-sm-n3 {\n    margin: -1rem !important;\n  }\n  .mt-sm-n3,\n  .my-sm-n3 {\n    margin-top: -1rem !important;\n  }\n  .mr-sm-n3,\n  .mx-sm-n3 {\n    margin-right: -1rem !important;\n  }\n  .mb-sm-n3,\n  .my-sm-n3 {\n    margin-bottom: -1rem !important;\n  }\n  .ml-sm-n3,\n  .mx-sm-n3 {\n    margin-left: -1rem !important;\n  }\n  .m-sm-n4 {\n    margin: -1.5rem !important;\n  }\n  .mt-sm-n4,\n  .my-sm-n4 {\n    margin-top: -1.5rem !important;\n  }\n  .mr-sm-n4,\n  .mx-sm-n4 {\n    margin-right: -1.5rem !important;\n  }\n  .mb-sm-n4,\n  .my-sm-n4 {\n    margin-bottom: -1.5rem !important;\n  }\n  .ml-sm-n4,\n  .mx-sm-n4 {\n    margin-left: -1.5rem !important;\n  }\n  .m-sm-n5 {\n    margin: -3rem !important;\n  }\n  .mt-sm-n5,\n  .my-sm-n5 {\n    margin-top: -3rem !important;\n  }\n  .mr-sm-n5,\n  .mx-sm-n5 {\n    margin-right: -3rem !important;\n  }\n  .mb-sm-n5,\n  .my-sm-n5 {\n    margin-bottom: -3rem !important;\n  }\n  .ml-sm-n5,\n  .mx-sm-n5 {\n    margin-left: -3rem !important;\n  }\n  .m-sm-auto {\n    margin: auto !important;\n  }\n  .mt-sm-auto,\n  .my-sm-auto {\n    margin-top: auto !important;\n  }\n  .mr-sm-auto,\n  .mx-sm-auto {\n    margin-right: auto !important;\n  }\n  .mb-sm-auto,\n  .my-sm-auto {\n    margin-bottom: auto !important;\n  }\n  .ml-sm-auto,\n  .mx-sm-auto {\n    margin-left: auto !important;\n  }\n}\n\n@media (min-width: 768px) {\n  .m-md-0 {\n    margin: 0 !important;\n  }\n  .mt-md-0,\n  .my-md-0 {\n    margin-top: 0 !important;\n  }\n  .mr-md-0,\n  .mx-md-0 {\n    margin-right: 0 !important;\n  }\n  .mb-md-0,\n  .my-md-0 {\n    margin-bottom: 0 !important;\n  }\n  .ml-md-0,\n  .mx-md-0 {\n    margin-left: 0 !important;\n  }\n  .m-md-1 {\n    margin: 0.25rem !important;\n  }\n  .mt-md-1,\n  .my-md-1 {\n    margin-top: 0.25rem !important;\n  }\n  .mr-md-1,\n  .mx-md-1 {\n    margin-right: 0.25rem !important;\n  }\n  .mb-md-1,\n  .my-md-1 {\n    margin-bottom: 0.25rem !important;\n  }\n  .ml-md-1,\n  .mx-md-1 {\n    margin-left: 0.25rem !important;\n  }\n  .m-md-2 {\n    margin: 0.5rem !important;\n  }\n  .mt-md-2,\n  .my-md-2 {\n    margin-top: 0.5rem !important;\n  }\n  .mr-md-2,\n  .mx-md-2 {\n    margin-right: 0.5rem !important;\n  }\n  .mb-md-2,\n  .my-md-2 {\n    margin-bottom: 0.5rem !important;\n  }\n  .ml-md-2,\n  .mx-md-2 {\n    margin-left: 0.5rem !important;\n  }\n  .m-md-3 {\n    margin: 1rem !important;\n  }\n  .mt-md-3,\n  .my-md-3 {\n    margin-top: 1rem !important;\n  }\n  .mr-md-3,\n  .mx-md-3 {\n    margin-right: 1rem !important;\n  }\n  .mb-md-3,\n  .my-md-3 {\n    margin-bottom: 1rem !important;\n  }\n  .ml-md-3,\n  .mx-md-3 {\n    margin-left: 1rem !important;\n  }\n  .m-md-4 {\n    margin: 1.5rem !important;\n  }\n  .mt-md-4,\n  .my-md-4 {\n    margin-top: 1.5rem !important;\n  }\n  .mr-md-4,\n  .mx-md-4 {\n    margin-right: 1.5rem !important;\n  }\n  .mb-md-4,\n  .my-md-4 {\n    margin-bottom: 1.5rem !important;\n  }\n  .ml-md-4,\n  .mx-md-4 {\n    margin-left: 1.5rem !important;\n  }\n  .m-md-5 {\n    margin: 3rem !important;\n  }\n  .mt-md-5,\n  .my-md-5 {\n    margin-top: 3rem !important;\n  }\n  .mr-md-5,\n  .mx-md-5 {\n    margin-right: 3rem !important;\n  }\n  .mb-md-5,\n  .my-md-5 {\n    margin-bottom: 3rem !important;\n  }\n  .ml-md-5,\n  .mx-md-5 {\n    margin-left: 3rem !important;\n  }\n  .p-md-0 {\n    padding: 0 !important;\n  }\n  .pt-md-0,\n  .py-md-0 {\n    padding-top: 0 !important;\n  }\n  .pr-md-0,\n  .px-md-0 {\n    padding-right: 0 !important;\n  }\n  .pb-md-0,\n  .py-md-0 {\n    padding-bottom: 0 !important;\n  }\n  .pl-md-0,\n  .px-md-0 {\n    padding-left: 0 !important;\n  }\n  .p-md-1 {\n    padding: 0.25rem !important;\n  }\n  .pt-md-1,\n  .py-md-1 {\n    padding-top: 0.25rem !important;\n  }\n  .pr-md-1,\n  .px-md-1 {\n    padding-right: 0.25rem !important;\n  }\n  .pb-md-1,\n  .py-md-1 {\n    padding-bottom: 0.25rem !important;\n  }\n  .pl-md-1,\n  .px-md-1 {\n    padding-left: 0.25rem !important;\n  }\n  .p-md-2 {\n    padding: 0.5rem !important;\n  }\n  .pt-md-2,\n  .py-md-2 {\n    padding-top: 0.5rem !important;\n  }\n  .pr-md-2,\n  .px-md-2 {\n    padding-right: 0.5rem !important;\n  }\n  .pb-md-2,\n  .py-md-2 {\n    padding-bottom: 0.5rem !important;\n  }\n  .pl-md-2,\n  .px-md-2 {\n    padding-left: 0.5rem !important;\n  }\n  .p-md-3 {\n    padding: 1rem !important;\n  }\n  .pt-md-3,\n  .py-md-3 {\n    padding-top: 1rem !important;\n  }\n  .pr-md-3,\n  .px-md-3 {\n    padding-right: 1rem !important;\n  }\n  .pb-md-3,\n  .py-md-3 {\n    padding-bottom: 1rem !important;\n  }\n  .pl-md-3,\n  .px-md-3 {\n    padding-left: 1rem !important;\n  }\n  .p-md-4 {\n    padding: 1.5rem !important;\n  }\n  .pt-md-4,\n  .py-md-4 {\n    padding-top: 1.5rem !important;\n  }\n  .pr-md-4,\n  .px-md-4 {\n    padding-right: 1.5rem !important;\n  }\n  .pb-md-4,\n  .py-md-4 {\n    padding-bottom: 1.5rem !important;\n  }\n  .pl-md-4,\n  .px-md-4 {\n    padding-left: 1.5rem !important;\n  }\n  .p-md-5 {\n    padding: 3rem !important;\n  }\n  .pt-md-5,\n  .py-md-5 {\n    padding-top: 3rem !important;\n  }\n  .pr-md-5,\n  .px-md-5 {\n    padding-right: 3rem !important;\n  }\n  .pb-md-5,\n  .py-md-5 {\n    padding-bottom: 3rem !important;\n  }\n  .pl-md-5,\n  .px-md-5 {\n    padding-left: 3rem !important;\n  }\n  .m-md-n1 {\n    margin: -0.25rem !important;\n  }\n  .mt-md-n1,\n  .my-md-n1 {\n    margin-top: -0.25rem !important;\n  }\n  .mr-md-n1,\n  .mx-md-n1 {\n    margin-right: -0.25rem !important;\n  }\n  .mb-md-n1,\n  .my-md-n1 {\n    margin-bottom: -0.25rem !important;\n  }\n  .ml-md-n1,\n  .mx-md-n1 {\n    margin-left: -0.25rem !important;\n  }\n  .m-md-n2 {\n    margin: -0.5rem !important;\n  }\n  .mt-md-n2,\n  .my-md-n2 {\n    margin-top: -0.5rem !important;\n  }\n  .mr-md-n2,\n  .mx-md-n2 {\n    margin-right: -0.5rem !important;\n  }\n  .mb-md-n2,\n  .my-md-n2 {\n    margin-bottom: -0.5rem !important;\n  }\n  .ml-md-n2,\n  .mx-md-n2 {\n    margin-left: -0.5rem !important;\n  }\n  .m-md-n3 {\n    margin: -1rem !important;\n  }\n  .mt-md-n3,\n  .my-md-n3 {\n    margin-top: -1rem !important;\n  }\n  .mr-md-n3,\n  .mx-md-n3 {\n    margin-right: -1rem !important;\n  }\n  .mb-md-n3,\n  .my-md-n3 {\n    margin-bottom: -1rem !important;\n  }\n  .ml-md-n3,\n  .mx-md-n3 {\n    margin-left: -1rem !important;\n  }\n  .m-md-n4 {\n    margin: -1.5rem !important;\n  }\n  .mt-md-n4,\n  .my-md-n4 {\n    margin-top: -1.5rem !important;\n  }\n  .mr-md-n4,\n  .mx-md-n4 {\n    margin-right: -1.5rem !important;\n  }\n  .mb-md-n4,\n  .my-md-n4 {\n    margin-bottom: -1.5rem !important;\n  }\n  .ml-md-n4,\n  .mx-md-n4 {\n    margin-left: -1.5rem !important;\n  }\n  .m-md-n5 {\n    margin: -3rem !important;\n  }\n  .mt-md-n5,\n  .my-md-n5 {\n    margin-top: -3rem !important;\n  }\n  .mr-md-n5,\n  .mx-md-n5 {\n    margin-right: -3rem !important;\n  }\n  .mb-md-n5,\n  .my-md-n5 {\n    margin-bottom: -3rem !important;\n  }\n  .ml-md-n5,\n  .mx-md-n5 {\n    margin-left: -3rem !important;\n  }\n  .m-md-auto {\n    margin: auto !important;\n  }\n  .mt-md-auto,\n  .my-md-auto {\n    margin-top: auto !important;\n  }\n  .mr-md-auto,\n  .mx-md-auto {\n    margin-right: auto !important;\n  }\n  .mb-md-auto,\n  .my-md-auto {\n    margin-bottom: auto !important;\n  }\n  .ml-md-auto,\n  .mx-md-auto {\n    margin-left: auto !important;\n  }\n}\n\n@media (min-width: 992px) {\n  .m-lg-0 {\n    margin: 0 !important;\n  }\n  .mt-lg-0,\n  .my-lg-0 {\n    margin-top: 0 !important;\n  }\n  .mr-lg-0,\n  .mx-lg-0 {\n    margin-right: 0 !important;\n  }\n  .mb-lg-0,\n  .my-lg-0 {\n    margin-bottom: 0 !important;\n  }\n  .ml-lg-0,\n  .mx-lg-0 {\n    margin-left: 0 !important;\n  }\n  .m-lg-1 {\n    margin: 0.25rem !important;\n  }\n  .mt-lg-1,\n  .my-lg-1 {\n    margin-top: 0.25rem !important;\n  }\n  .mr-lg-1,\n  .mx-lg-1 {\n    margin-right: 0.25rem !important;\n  }\n  .mb-lg-1,\n  .my-lg-1 {\n    margin-bottom: 0.25rem !important;\n  }\n  .ml-lg-1,\n  .mx-lg-1 {\n    margin-left: 0.25rem !important;\n  }\n  .m-lg-2 {\n    margin: 0.5rem !important;\n  }\n  .mt-lg-2,\n  .my-lg-2 {\n    margin-top: 0.5rem !important;\n  }\n  .mr-lg-2,\n  .mx-lg-2 {\n    margin-right: 0.5rem !important;\n  }\n  .mb-lg-2,\n  .my-lg-2 {\n    margin-bottom: 0.5rem !important;\n  }\n  .ml-lg-2,\n  .mx-lg-2 {\n    margin-left: 0.5rem !important;\n  }\n  .m-lg-3 {\n    margin: 1rem !important;\n  }\n  .mt-lg-3,\n  .my-lg-3 {\n    margin-top: 1rem !important;\n  }\n  .mr-lg-3,\n  .mx-lg-3 {\n    margin-right: 1rem !important;\n  }\n  .mb-lg-3,\n  .my-lg-3 {\n    margin-bottom: 1rem !important;\n  }\n  .ml-lg-3,\n  .mx-lg-3 {\n    margin-left: 1rem !important;\n  }\n  .m-lg-4 {\n    margin: 1.5rem !important;\n  }\n  .mt-lg-4,\n  .my-lg-4 {\n    margin-top: 1.5rem !important;\n  }\n  .mr-lg-4,\n  .mx-lg-4 {\n    margin-right: 1.5rem !important;\n  }\n  .mb-lg-4,\n  .my-lg-4 {\n    margin-bottom: 1.5rem !important;\n  }\n  .ml-lg-4,\n  .mx-lg-4 {\n    margin-left: 1.5rem !important;\n  }\n  .m-lg-5 {\n    margin: 3rem !important;\n  }\n  .mt-lg-5,\n  .my-lg-5 {\n    margin-top: 3rem !important;\n  }\n  .mr-lg-5,\n  .mx-lg-5 {\n    margin-right: 3rem !important;\n  }\n  .mb-lg-5,\n  .my-lg-5 {\n    margin-bottom: 3rem !important;\n  }\n  .ml-lg-5,\n  .mx-lg-5 {\n    margin-left: 3rem !important;\n  }\n  .p-lg-0 {\n    padding: 0 !important;\n  }\n  .pt-lg-0,\n  .py-lg-0 {\n    padding-top: 0 !important;\n  }\n  .pr-lg-0,\n  .px-lg-0 {\n    padding-right: 0 !important;\n  }\n  .pb-lg-0,\n  .py-lg-0 {\n    padding-bottom: 0 !important;\n  }\n  .pl-lg-0,\n  .px-lg-0 {\n    padding-left: 0 !important;\n  }\n  .p-lg-1 {\n    padding: 0.25rem !important;\n  }\n  .pt-lg-1,\n  .py-lg-1 {\n    padding-top: 0.25rem !important;\n  }\n  .pr-lg-1,\n  .px-lg-1 {\n    padding-right: 0.25rem !important;\n  }\n  .pb-lg-1,\n  .py-lg-1 {\n    padding-bottom: 0.25rem !important;\n  }\n  .pl-lg-1,\n  .px-lg-1 {\n    padding-left: 0.25rem !important;\n  }\n  .p-lg-2 {\n    padding: 0.5rem !important;\n  }\n  .pt-lg-2,\n  .py-lg-2 {\n    padding-top: 0.5rem !important;\n  }\n  .pr-lg-2,\n  .px-lg-2 {\n    padding-right: 0.5rem !important;\n  }\n  .pb-lg-2,\n  .py-lg-2 {\n    padding-bottom: 0.5rem !important;\n  }\n  .pl-lg-2,\n  .px-lg-2 {\n    padding-left: 0.5rem !important;\n  }\n  .p-lg-3 {\n    padding: 1rem !important;\n  }\n  .pt-lg-3,\n  .py-lg-3 {\n    padding-top: 1rem !important;\n  }\n  .pr-lg-3,\n  .px-lg-3 {\n    padding-right: 1rem !important;\n  }\n  .pb-lg-3,\n  .py-lg-3 {\n    padding-bottom: 1rem !important;\n  }\n  .pl-lg-3,\n  .px-lg-3 {\n    padding-left: 1rem !important;\n  }\n  .p-lg-4 {\n    padding: 1.5rem !important;\n  }\n  .pt-lg-4,\n  .py-lg-4 {\n    padding-top: 1.5rem !important;\n  }\n  .pr-lg-4,\n  .px-lg-4 {\n    padding-right: 1.5rem !important;\n  }\n  .pb-lg-4,\n  .py-lg-4 {\n    padding-bottom: 1.5rem !important;\n  }\n  .pl-lg-4,\n  .px-lg-4 {\n    padding-left: 1.5rem !important;\n  }\n  .p-lg-5 {\n    padding: 3rem !important;\n  }\n  .pt-lg-5,\n  .py-lg-5 {\n    padding-top: 3rem !important;\n  }\n  .pr-lg-5,\n  .px-lg-5 {\n    padding-right: 3rem !important;\n  }\n  .pb-lg-5,\n  .py-lg-5 {\n    padding-bottom: 3rem !important;\n  }\n  .pl-lg-5,\n  .px-lg-5 {\n    padding-left: 3rem !important;\n  }\n  .m-lg-n1 {\n    margin: -0.25rem !important;\n  }\n  .mt-lg-n1,\n  .my-lg-n1 {\n    margin-top: -0.25rem !important;\n  }\n  .mr-lg-n1,\n  .mx-lg-n1 {\n    margin-right: -0.25rem !important;\n  }\n  .mb-lg-n1,\n  .my-lg-n1 {\n    margin-bottom: -0.25rem !important;\n  }\n  .ml-lg-n1,\n  .mx-lg-n1 {\n    margin-left: -0.25rem !important;\n  }\n  .m-lg-n2 {\n    margin: -0.5rem !important;\n  }\n  .mt-lg-n2,\n  .my-lg-n2 {\n    margin-top: -0.5rem !important;\n  }\n  .mr-lg-n2,\n  .mx-lg-n2 {\n    margin-right: -0.5rem !important;\n  }\n  .mb-lg-n2,\n  .my-lg-n2 {\n    margin-bottom: -0.5rem !important;\n  }\n  .ml-lg-n2,\n  .mx-lg-n2 {\n    margin-left: -0.5rem !important;\n  }\n  .m-lg-n3 {\n    margin: -1rem !important;\n  }\n  .mt-lg-n3,\n  .my-lg-n3 {\n    margin-top: -1rem !important;\n  }\n  .mr-lg-n3,\n  .mx-lg-n3 {\n    margin-right: -1rem !important;\n  }\n  .mb-lg-n3,\n  .my-lg-n3 {\n    margin-bottom: -1rem !important;\n  }\n  .ml-lg-n3,\n  .mx-lg-n3 {\n    margin-left: -1rem !important;\n  }\n  .m-lg-n4 {\n    margin: -1.5rem !important;\n  }\n  .mt-lg-n4,\n  .my-lg-n4 {\n    margin-top: -1.5rem !important;\n  }\n  .mr-lg-n4,\n  .mx-lg-n4 {\n    margin-right: -1.5rem !important;\n  }\n  .mb-lg-n4,\n  .my-lg-n4 {\n    margin-bottom: -1.5rem !important;\n  }\n  .ml-lg-n4,\n  .mx-lg-n4 {\n    margin-left: -1.5rem !important;\n  }\n  .m-lg-n5 {\n    margin: -3rem !important;\n  }\n  .mt-lg-n5,\n  .my-lg-n5 {\n    margin-top: -3rem !important;\n  }\n  .mr-lg-n5,\n  .mx-lg-n5 {\n    margin-right: -3rem !important;\n  }\n  .mb-lg-n5,\n  .my-lg-n5 {\n    margin-bottom: -3rem !important;\n  }\n  .ml-lg-n5,\n  .mx-lg-n5 {\n    margin-left: -3rem !important;\n  }\n  .m-lg-auto {\n    margin: auto !important;\n  }\n  .mt-lg-auto,\n  .my-lg-auto {\n    margin-top: auto !important;\n  }\n  .mr-lg-auto,\n  .mx-lg-auto {\n    margin-right: auto !important;\n  }\n  .mb-lg-auto,\n  .my-lg-auto {\n    margin-bottom: auto !important;\n  }\n  .ml-lg-auto,\n  .mx-lg-auto {\n    margin-left: auto !important;\n  }\n}\n\n@media (min-width: 1200px) {\n  .m-xl-0 {\n    margin: 0 !important;\n  }\n  .mt-xl-0,\n  .my-xl-0 {\n    margin-top: 0 !important;\n  }\n  .mr-xl-0,\n  .mx-xl-0 {\n    margin-right: 0 !important;\n  }\n  .mb-xl-0,\n  .my-xl-0 {\n    margin-bottom: 0 !important;\n  }\n  .ml-xl-0,\n  .mx-xl-0 {\n    margin-left: 0 !important;\n  }\n  .m-xl-1 {\n    margin: 0.25rem !important;\n  }\n  .mt-xl-1,\n  .my-xl-1 {\n    margin-top: 0.25rem !important;\n  }\n  .mr-xl-1,\n  .mx-xl-1 {\n    margin-right: 0.25rem !important;\n  }\n  .mb-xl-1,\n  .my-xl-1 {\n    margin-bottom: 0.25rem !important;\n  }\n  .ml-xl-1,\n  .mx-xl-1 {\n    margin-left: 0.25rem !important;\n  }\n  .m-xl-2 {\n    margin: 0.5rem !important;\n  }\n  .mt-xl-2,\n  .my-xl-2 {\n    margin-top: 0.5rem !important;\n  }\n  .mr-xl-2,\n  .mx-xl-2 {\n    margin-right: 0.5rem !important;\n  }\n  .mb-xl-2,\n  .my-xl-2 {\n    margin-bottom: 0.5rem !important;\n  }\n  .ml-xl-2,\n  .mx-xl-2 {\n    margin-left: 0.5rem !important;\n  }\n  .m-xl-3 {\n    margin: 1rem !important;\n  }\n  .mt-xl-3,\n  .my-xl-3 {\n    margin-top: 1rem !important;\n  }\n  .mr-xl-3,\n  .mx-xl-3 {\n    margin-right: 1rem !important;\n  }\n  .mb-xl-3,\n  .my-xl-3 {\n    margin-bottom: 1rem !important;\n  }\n  .ml-xl-3,\n  .mx-xl-3 {\n    margin-left: 1rem !important;\n  }\n  .m-xl-4 {\n    margin: 1.5rem !important;\n  }\n  .mt-xl-4,\n  .my-xl-4 {\n    margin-top: 1.5rem !important;\n  }\n  .mr-xl-4,\n  .mx-xl-4 {\n    margin-right: 1.5rem !important;\n  }\n  .mb-xl-4,\n  .my-xl-4 {\n    margin-bottom: 1.5rem !important;\n  }\n  .ml-xl-4,\n  .mx-xl-4 {\n    margin-left: 1.5rem !important;\n  }\n  .m-xl-5 {\n    margin: 3rem !important;\n  }\n  .mt-xl-5,\n  .my-xl-5 {\n    margin-top: 3rem !important;\n  }\n  .mr-xl-5,\n  .mx-xl-5 {\n    margin-right: 3rem !important;\n  }\n  .mb-xl-5,\n  .my-xl-5 {\n    margin-bottom: 3rem !important;\n  }\n  .ml-xl-5,\n  .mx-xl-5 {\n    margin-left: 3rem !important;\n  }\n  .p-xl-0 {\n    padding: 0 !important;\n  }\n  .pt-xl-0,\n  .py-xl-0 {\n    padding-top: 0 !important;\n  }\n  .pr-xl-0,\n  .px-xl-0 {\n    padding-right: 0 !important;\n  }\n  .pb-xl-0,\n  .py-xl-0 {\n    padding-bottom: 0 !important;\n  }\n  .pl-xl-0,\n  .px-xl-0 {\n    padding-left: 0 !important;\n  }\n  .p-xl-1 {\n    padding: 0.25rem !important;\n  }\n  .pt-xl-1,\n  .py-xl-1 {\n    padding-top: 0.25rem !important;\n  }\n  .pr-xl-1,\n  .px-xl-1 {\n    padding-right: 0.25rem !important;\n  }\n  .pb-xl-1,\n  .py-xl-1 {\n    padding-bottom: 0.25rem !important;\n  }\n  .pl-xl-1,\n  .px-xl-1 {\n    padding-left: 0.25rem !important;\n  }\n  .p-xl-2 {\n    padding: 0.5rem !important;\n  }\n  .pt-xl-2,\n  .py-xl-2 {\n    padding-top: 0.5rem !important;\n  }\n  .pr-xl-2,\n  .px-xl-2 {\n    padding-right: 0.5rem !important;\n  }\n  .pb-xl-2,\n  .py-xl-2 {\n    padding-bottom: 0.5rem !important;\n  }\n  .pl-xl-2,\n  .px-xl-2 {\n    padding-left: 0.5rem !important;\n  }\n  .p-xl-3 {\n    padding: 1rem !important;\n  }\n  .pt-xl-3,\n  .py-xl-3 {\n    padding-top: 1rem !important;\n  }\n  .pr-xl-3,\n  .px-xl-3 {\n    padding-right: 1rem !important;\n  }\n  .pb-xl-3,\n  .py-xl-3 {\n    padding-bottom: 1rem !important;\n  }\n  .pl-xl-3,\n  .px-xl-3 {\n    padding-left: 1rem !important;\n  }\n  .p-xl-4 {\n    padding: 1.5rem !important;\n  }\n  .pt-xl-4,\n  .py-xl-4 {\n    padding-top: 1.5rem !important;\n  }\n  .pr-xl-4,\n  .px-xl-4 {\n    padding-right: 1.5rem !important;\n  }\n  .pb-xl-4,\n  .py-xl-4 {\n    padding-bottom: 1.5rem !important;\n  }\n  .pl-xl-4,\n  .px-xl-4 {\n    padding-left: 1.5rem !important;\n  }\n  .p-xl-5 {\n    padding: 3rem !important;\n  }\n  .pt-xl-5,\n  .py-xl-5 {\n    padding-top: 3rem !important;\n  }\n  .pr-xl-5,\n  .px-xl-5 {\n    padding-right: 3rem !important;\n  }\n  .pb-xl-5,\n  .py-xl-5 {\n    padding-bottom: 3rem !important;\n  }\n  .pl-xl-5,\n  .px-xl-5 {\n    padding-left: 3rem !important;\n  }\n  .m-xl-n1 {\n    margin: -0.25rem !important;\n  }\n  .mt-xl-n1,\n  .my-xl-n1 {\n    margin-top: -0.25rem !important;\n  }\n  .mr-xl-n1,\n  .mx-xl-n1 {\n    margin-right: -0.25rem !important;\n  }\n  .mb-xl-n1,\n  .my-xl-n1 {\n    margin-bottom: -0.25rem !important;\n  }\n  .ml-xl-n1,\n  .mx-xl-n1 {\n    margin-left: -0.25rem !important;\n  }\n  .m-xl-n2 {\n    margin: -0.5rem !important;\n  }\n  .mt-xl-n2,\n  .my-xl-n2 {\n    margin-top: -0.5rem !important;\n  }\n  .mr-xl-n2,\n  .mx-xl-n2 {\n    margin-right: -0.5rem !important;\n  }\n  .mb-xl-n2,\n  .my-xl-n2 {\n    margin-bottom: -0.5rem !important;\n  }\n  .ml-xl-n2,\n  .mx-xl-n2 {\n    margin-left: -0.5rem !important;\n  }\n  .m-xl-n3 {\n    margin: -1rem !important;\n  }\n  .mt-xl-n3,\n  .my-xl-n3 {\n    margin-top: -1rem !important;\n  }\n  .mr-xl-n3,\n  .mx-xl-n3 {\n    margin-right: -1rem !important;\n  }\n  .mb-xl-n3,\n  .my-xl-n3 {\n    margin-bottom: -1rem !important;\n  }\n  .ml-xl-n3,\n  .mx-xl-n3 {\n    margin-left: -1rem !important;\n  }\n  .m-xl-n4 {\n    margin: -1.5rem !important;\n  }\n  .mt-xl-n4,\n  .my-xl-n4 {\n    margin-top: -1.5rem !important;\n  }\n  .mr-xl-n4,\n  .mx-xl-n4 {\n    margin-right: -1.5rem !important;\n  }\n  .mb-xl-n4,\n  .my-xl-n4 {\n    margin-bottom: -1.5rem !important;\n  }\n  .ml-xl-n4,\n  .mx-xl-n4 {\n    margin-left: -1.5rem !important;\n  }\n  .m-xl-n5 {\n    margin: -3rem !important;\n  }\n  .mt-xl-n5,\n  .my-xl-n5 {\n    margin-top: -3rem !important;\n  }\n  .mr-xl-n5,\n  .mx-xl-n5 {\n    margin-right: -3rem !important;\n  }\n  .mb-xl-n5,\n  .my-xl-n5 {\n    margin-bottom: -3rem !important;\n  }\n  .ml-xl-n5,\n  .mx-xl-n5 {\n    margin-left: -3rem !important;\n  }\n  .m-xl-auto {\n    margin: auto !important;\n  }\n  .mt-xl-auto,\n  .my-xl-auto {\n    margin-top: auto !important;\n  }\n  .mr-xl-auto,\n  .mx-xl-auto {\n    margin-right: auto !important;\n  }\n  .mb-xl-auto,\n  .my-xl-auto {\n    margin-bottom: auto !important;\n  }\n  .ml-xl-auto,\n  .mx-xl-auto {\n    margin-left: auto !important;\n  }\n}\n\n.text-monospace {\n  font-family: SFMono-Regular, Menlo, Monaco, Consolas, \"Liberation Mono\", \"Courier New\", monospace !important;\n}\n\n.text-justify {\n  text-align: justify !important;\n}\n\n.text-wrap {\n  white-space: normal !important;\n}\n\n.text-nowrap {\n  white-space: nowrap !important;\n}\n\n.text-truncate {\n  overflow: hidden;\n  text-overflow: ellipsis;\n  white-space: nowrap;\n}\n\n.text-left {\n  text-align: left !important;\n}\n\n.text-right {\n  text-align: right !important;\n}\n\n.text-center {\n  text-align: center !important;\n}\n\n@media (min-width: 576px) {\n  .text-sm-left {\n    text-align: left !important;\n  }\n  .text-sm-right {\n    text-align: right !important;\n  }\n  .text-sm-center {\n    text-align: center !important;\n  }\n}\n\n@media (min-width: 768px) {\n  .text-md-left {\n    text-align: left !important;\n  }\n  .text-md-right {\n    text-align: right !important;\n  }\n  .text-md-center {\n    text-align: center !important;\n  }\n}\n\n@media (min-width: 992px) {\n  .text-lg-left {\n    text-align: left !important;\n  }\n  .text-lg-right {\n    text-align: right !important;\n  }\n  .text-lg-center {\n    text-align: center !important;\n  }\n}\n\n@media (min-width: 1200px) {\n  .text-xl-left {\n    text-align: left !important;\n  }\n  .text-xl-right {\n    text-align: right !important;\n  }\n  .text-xl-center {\n    text-align: center !important;\n  }\n}\n\n.text-lowercase {\n  text-transform: lowercase !important;\n}\n\n.text-uppercase {\n  text-transform: uppercase !important;\n}\n\n.text-capitalize {\n  text-transform: capitalize !important;\n}\n\n.font-weight-light {\n  font-weight: 300 !important;\n}\n\n.font-weight-lighter {\n  font-weight: lighter !important;\n}\n\n.font-weight-normal {\n  font-weight: 400 !important;\n}\n\n.font-weight-bold {\n  font-weight: 700 !important;\n}\n\n.font-weight-bolder {\n  font-weight: bolder !important;\n}\n\n.font-italic {\n  font-style: italic !important;\n}\n\n.text-white {\n  color: #fff !important;\n}\n\n.text-primary {\n  color: #007bff !important;\n}\n\na.text-primary:hover, a.text-primary:focus {\n  color: #0056b3 !important;\n}\n\n.text-secondary {\n  color: #6c757d !important;\n}\n\na.text-secondary:hover, a.text-secondary:focus {\n  color: #494f54 !important;\n}\n\n.text-success {\n  color: #28a745 !important;\n}\n\na.text-success:hover, a.text-success:focus {\n  color: #19692c !important;\n}\n\n.text-info {\n  color: #17a2b8 !important;\n}\n\na.text-info:hover, a.text-info:focus {\n  color: #0f6674 !important;\n}\n\n.text-warning {\n  color: #ffc107 !important;\n}\n\na.text-warning:hover, a.text-warning:focus {\n  color: #ba8b00 !important;\n}\n\n.text-danger {\n  color: #dc3545 !important;\n}\n\na.text-danger:hover, a.text-danger:focus {\n  color: #a71d2a !important;\n}\n\n.text-light {\n  color: #f8f9fa !important;\n}\n\na.text-light:hover, a.text-light:focus {\n  color: #cbd3da !important;\n}\n\n.text-dark {\n  color: #343a40 !important;\n}\n\na.text-dark:hover, a.text-dark:focus {\n  color: #121416 !important;\n}\n\n.text-body {\n  color: #212529 !important;\n}\n\n.text-muted {\n  color: #6c757d !important;\n}\n\n.text-black-50 {\n  color: rgba(0, 0, 0, 0.5) !important;\n}\n\n.text-white-50 {\n  color: rgba(255, 255, 255, 0.5) !important;\n}\n\n.text-hide {\n  font: 0/0 a;\n  color: transparent;\n  text-shadow: none;\n  background-color: transparent;\n  border: 0;\n}\n\n.text-decoration-none {\n  text-decoration: none !important;\n}\n\n.text-break {\n  word-break: break-word !important;\n  overflow-wrap: break-word !important;\n}\n\n.text-reset {\n  color: inherit !important;\n}\n\n.visible {\n  visibility: visible !important;\n}\n\n.invisible {\n  visibility: hidden !important;\n}\n\n@media print {\n  *,\n  *::before,\n  *::after {\n    text-shadow: none !important;\n    box-shadow: none !important;\n  }\n  a:not(.btn) {\n    text-decoration: underline;\n  }\n  abbr[title]::after {\n    content: \" (\" attr(title) \")\";\n  }\n  pre {\n    white-space: pre-wrap !important;\n  }\n  pre,\n  blockquote {\n    border: 1px solid #adb5bd;\n    page-break-inside: avoid;\n  }\n  thead {\n    display: table-header-group;\n  }\n  tr,\n  img {\n    page-break-inside: avoid;\n  }\n  p,\n  h2,\n  h3 {\n    orphans: 3;\n    widows: 3;\n  }\n  h2,\n  h3 {\n    page-break-after: avoid;\n  }\n  @page {\n    size: a3;\n  }\n  body {\n    min-width: 992px !important;\n  }\n  .container {\n    min-width: 992px !important;\n  }\n  .navbar {\n    display: none;\n  }\n  .badge {\n    border: 1px solid #000;\n  }\n  .table {\n    border-collapse: collapse !important;\n  }\n  .table td,\n  .table th {\n    background-color: #fff !important;\n  }\n  .table-bordered th,\n  .table-bordered td {\n    border: 1px solid #dee2e6 !important;\n  }\n  .table-dark {\n    color: inherit;\n  }\n  .table-dark th,\n  .table-dark td,\n  .table-dark thead th,\n  .table-dark tbody + tbody {\n    border-color: #dee2e6;\n  }\n  .table .thead-dark th {\n    color: inherit;\n    border-color: #dee2e6;\n  }\n}\n\n/*# sourceMappingURL=bootstrap.css.map */","// Hover mixin and `$enable-hover-media-query` are deprecated.\n//\n// Originally added during our alphas and maintained during betas, this mixin was\n// designed to prevent `:hover` stickiness on iOS-an issue where hover styles\n// would persist after initial touch.\n//\n// For backward compatibility, we've kept these mixins and updated them to\n// always return their regular pseudo-classes instead of a shimmed media query.\n//\n// Issue: https://github.com/twbs/bootstrap/issues/25195\n\n@mixin hover() {\n  &:hover { @content; }\n}\n\n@mixin hover-focus() {\n  &:hover,\n  &:focus {\n    @content;\n  }\n}\n\n@mixin plain-hover-focus() {\n  &,\n  &:hover,\n  &:focus {\n    @content;\n  }\n}\n\n@mixin hover-focus-active() {\n  &:hover,\n  &:focus,\n  &:active {\n    @content;\n  }\n}\n","// stylelint-disable declaration-no-important, selector-list-comma-newline-after\n\n//\n// Headings\n//\n\nh1, h2, h3, h4, h5, h6,\n.h1, .h2, .h3, .h4, .h5, .h6 {\n  margin-bottom: $headings-margin-bottom;\n  font-family: $headings-font-family;\n  font-weight: $headings-font-weight;\n  line-height: $headings-line-height;\n  color: $headings-color;\n}\n\nh1, .h1 { @include font-size($h1-font-size); }\nh2, .h2 { @include font-size($h2-font-size); }\nh3, .h3 { @include font-size($h3-font-size); }\nh4, .h4 { @include font-size($h4-font-size); }\nh5, .h5 { @include font-size($h5-font-size); }\nh6, .h6 { @include font-size($h6-font-size); }\n\n.lead {\n  @include font-size($lead-font-size);\n  font-weight: $lead-font-weight;\n}\n\n// Type display classes\n.display-1 {\n  @include font-size($display1-size);\n  font-weight: $display1-weight;\n  line-height: $display-line-height;\n}\n.display-2 {\n  @include font-size($display2-size);\n  font-weight: $display2-weight;\n  line-height: $display-line-height;\n}\n.display-3 {\n  @include font-size($display3-size);\n  font-weight: $display3-weight;\n  line-height: $display-line-height;\n}\n.display-4 {\n  @include font-size($display4-size);\n  font-weight: $display4-weight;\n  line-height: $display-line-height;\n}\n\n\n//\n// Horizontal rules\n//\n\nhr {\n  margin-top: $hr-margin-y;\n  margin-bottom: $hr-margin-y;\n  border: 0;\n  border-top: $hr-border-width solid $hr-border-color;\n}\n\n\n//\n// Emphasis\n//\n\nsmall,\n.small {\n  @include font-size($small-font-size);\n  font-weight: $font-weight-normal;\n}\n\nmark,\n.mark {\n  padding: $mark-padding;\n  background-color: $mark-bg;\n}\n\n\n//\n// Lists\n//\n\n.list-unstyled {\n  @include list-unstyled();\n}\n\n// Inline turns list items into inline-block\n.list-inline {\n  @include list-unstyled();\n}\n.list-inline-item {\n  display: inline-block;\n\n  &:not(:last-child) {\n    margin-right: $list-inline-padding;\n  }\n}\n\n\n//\n// Misc\n//\n\n// Builds on `abbr`\n.initialism {\n  @include font-size(90%);\n  text-transform: uppercase;\n}\n\n// Blockquotes\n.blockquote {\n  margin-bottom: $spacer;\n  @include font-size($blockquote-font-size);\n}\n\n.blockquote-footer {\n  display: block;\n  @include font-size($blockquote-small-font-size);\n  color: $blockquote-small-color;\n\n  &::before {\n    content: \"\\2014\\00A0\"; // em dash, nbsp\n  }\n}\n","// Lists\n\n// Unstyled keeps list items block level, just removes default browser padding and list-style\n@mixin list-unstyled() {\n  padding-left: 0;\n  list-style: none;\n}\n","// Responsive images (ensure images don't scale beyond their parents)\n//\n// This is purposefully opt-in via an explicit class rather than being the default for all `<img>`s.\n// We previously tried the \"images are responsive by default\" approach in Bootstrap v2,\n// and abandoned it in Bootstrap v3 because it breaks lots of third-party widgets (including Google Maps)\n// which weren't expecting the images within themselves to be involuntarily resized.\n// See also https://github.com/twbs/bootstrap/issues/18178\n.img-fluid {\n  @include img-fluid();\n}\n\n\n// Image thumbnails\n.img-thumbnail {\n  padding: $thumbnail-padding;\n  background-color: $thumbnail-bg;\n  border: $thumbnail-border-width solid $thumbnail-border-color;\n  @include border-radius($thumbnail-border-radius);\n  @include box-shadow($thumbnail-box-shadow);\n\n  // Keep them at most 100% wide\n  @include img-fluid();\n}\n\n//\n// Figures\n//\n\n.figure {\n  // Ensures the caption's text aligns with the image.\n  display: inline-block;\n}\n\n.figure-img {\n  margin-bottom: $spacer / 2;\n  line-height: 1;\n}\n\n.figure-caption {\n  @include font-size($figure-caption-font-size);\n  color: $figure-caption-color;\n}\n","// Image Mixins\n// - Responsive image\n// - Retina image\n\n\n// Responsive image\n//\n// Keep images from scaling beyond the width of their parents.\n\n@mixin img-fluid() {\n  // Part 1: Set a maximum relative to the parent\n  max-width: 100%;\n  // Part 2: Override the height to auto, otherwise images will be stretched\n  // when setting a width and height attribute on the img element.\n  height: auto;\n}\n\n\n// Retina image\n//\n// Short retina mixin for setting background-image and -size.\n\n@mixin img-retina($file-1x, $file-2x, $width-1x, $height-1x) {\n  background-image: url($file-1x);\n\n  // Autoprefixer takes care of adding -webkit-min-device-pixel-ratio and -o-min-device-pixel-ratio,\n  // but doesn't convert dppx=>dpi.\n  // There's no such thing as unprefixed min-device-pixel-ratio since it's nonstandard.\n  // Compatibility info: https://caniuse.com/#feat=css-media-resolution\n  @media only screen and (min-resolution: 192dpi), // IE9-11 don't support dppx\n    only screen and (min-resolution: 2dppx) { // Standardized\n    background-image: url($file-2x);\n    background-size: $width-1x $height-1x;\n  }\n  @include deprecate(\"`img-retina()`\", \"v4.3.0\", \"v5\");\n}\n","// stylelint-disable property-blacklist\n// Single side border-radius\n\n@mixin border-radius($radius: $border-radius, $fallback-border-radius: false) {\n  @if $enable-rounded {\n    border-radius: $radius;\n  }\n  @else if $fallback-border-radius != false {\n    border-radius: $fallback-border-radius;\n  }\n}\n\n@mixin border-top-radius($radius) {\n  @if $enable-rounded {\n    border-top-left-radius: $radius;\n    border-top-right-radius: $radius;\n  }\n}\n\n@mixin border-right-radius($radius) {\n  @if $enable-rounded {\n    border-top-right-radius: $radius;\n    border-bottom-right-radius: $radius;\n  }\n}\n\n@mixin border-bottom-radius($radius) {\n  @if $enable-rounded {\n    border-bottom-right-radius: $radius;\n    border-bottom-left-radius: $radius;\n  }\n}\n\n@mixin border-left-radius($radius) {\n  @if $enable-rounded {\n    border-top-left-radius: $radius;\n    border-bottom-left-radius: $radius;\n  }\n}\n\n@mixin border-top-left-radius($radius) {\n  @if $enable-rounded {\n    border-top-left-radius: $radius;\n  }\n}\n\n@mixin border-top-right-radius($radius) {\n  @if $enable-rounded {\n    border-top-right-radius: $radius;\n  }\n}\n\n@mixin border-bottom-right-radius($radius) {\n  @if $enable-rounded {\n    border-bottom-right-radius: $radius;\n  }\n}\n\n@mixin border-bottom-left-radius($radius) {\n  @if $enable-rounded {\n    border-bottom-left-radius: $radius;\n  }\n}\n","// Inline code\ncode {\n  @include font-size($code-font-size);\n  color: $code-color;\n  word-wrap: break-word;\n\n  // Streamline the style when inside anchors to avoid broken underline and more\n  a > & {\n    color: inherit;\n  }\n}\n\n// User input typically entered via keyboard\nkbd {\n  padding: $kbd-padding-y $kbd-padding-x;\n  @include font-size($kbd-font-size);\n  color: $kbd-color;\n  background-color: $kbd-bg;\n  @include border-radius($border-radius-sm);\n  @include box-shadow($kbd-box-shadow);\n\n  kbd {\n    padding: 0;\n    @include font-size(100%);\n    font-weight: $nested-kbd-font-weight;\n    @include box-shadow(none);\n  }\n}\n\n// Blocks of code\npre {\n  display: block;\n  @include font-size($code-font-size);\n  color: $pre-color;\n\n  // Account for some code outputs that place code tags in pre tags\n  code {\n    @include font-size(inherit);\n    color: inherit;\n    word-break: normal;\n  }\n}\n\n// Enable scrollable blocks of code\n.pre-scrollable {\n  max-height: $pre-scrollable-max-height;\n  overflow-y: scroll;\n}\n","// Container widths\n//\n// Set the container width, and override it for fixed navbars in media queries.\n\n@if $enable-grid-classes {\n  // Single container class with breakpoint max-widths\n  .container {\n    @include make-container();\n    @include make-container-max-widths();\n  }\n\n  // 100% wide container at all breakpoints\n  .container-fluid {\n    @include make-container();\n  }\n\n  // Responsive containers that are 100% wide until a breakpoint\n  @each $breakpoint, $container-max-width in $container-max-widths {\n    .container-#{$breakpoint} {\n      @extend .container-fluid;\n    }\n\n    @include media-breakpoint-up($breakpoint, $grid-breakpoints) {\n      %responsive-container-#{$breakpoint} {\n        max-width: $container-max-width;\n      }\n\n      @each $name, $width in $grid-breakpoints {\n        @if ($container-max-width > $width or $breakpoint == $name) {\n          .container#{breakpoint-infix($name, $grid-breakpoints)} {\n            @extend %responsive-container-#{$breakpoint};\n          }\n        }\n      }\n    }\n  }\n}\n\n\n// Row\n//\n// Rows contain your columns.\n\n@if $enable-grid-classes {\n  .row {\n    @include make-row();\n  }\n\n  // Remove the negative margin from default .row, then the horizontal padding\n  // from all immediate children columns (to prevent runaway style inheritance).\n  .no-gutters {\n    margin-right: 0;\n    margin-left: 0;\n\n    > .col,\n    > [class*=\"col-\"] {\n      padding-right: 0;\n      padding-left: 0;\n    }\n  }\n}\n\n// Columns\n//\n// Common styles for small and large grid columns\n\n@if $enable-grid-classes {\n  @include make-grid-columns();\n}\n","/// Grid system\n//\n// Generate semantic grid columns with these mixins.\n\n@mixin make-container($gutter: $grid-gutter-width) {\n  width: 100%;\n  padding-right: $gutter / 2;\n  padding-left: $gutter / 2;\n  margin-right: auto;\n  margin-left: auto;\n}\n\n\n// For each breakpoint, define the maximum width of the container in a media query\n@mixin make-container-max-widths($max-widths: $container-max-widths, $breakpoints: $grid-breakpoints) {\n  @each $breakpoint, $container-max-width in $max-widths {\n    @include media-breakpoint-up($breakpoint, $breakpoints) {\n      max-width: $container-max-width;\n    }\n  }\n}\n\n@mixin make-row($gutter: $grid-gutter-width) {\n  display: flex;\n  flex-wrap: wrap;\n  margin-right: -$gutter / 2;\n  margin-left: -$gutter / 2;\n}\n\n@mixin make-col-ready($gutter: $grid-gutter-width) {\n  position: relative;\n  // Prevent columns from becoming too narrow when at smaller grid tiers by\n  // always setting `width: 100%;`. This works because we use `flex` values\n  // later on to override this initial width.\n  width: 100%;\n  padding-right: $gutter / 2;\n  padding-left: $gutter / 2;\n}\n\n@mixin make-col($size, $columns: $grid-columns) {\n  flex: 0 0 percentage($size / $columns);\n  // Add a `max-width` to ensure content within each column does not blow out\n  // the width of the column. Applies to IE10+ and Firefox. Chrome and Safari\n  // do not appear to require this.\n  max-width: percentage($size / $columns);\n}\n\n@mixin make-col-auto() {\n  flex: 0 0 auto;\n  width: auto;\n  max-width: 100%; // Reset earlier grid tiers\n}\n\n@mixin make-col-offset($size, $columns: $grid-columns) {\n  $num: $size / $columns;\n  margin-left: if($num == 0, 0, percentage($num));\n}\n\n// Row columns\n//\n// Specify on a parent element(e.g., .row) to force immediate children into NN\n// numberof columns. Supports wrapping to new lines, but does not do a Masonry\n// style grid.\n@mixin row-cols($count) {\n  & > * {\n    flex: 0 0 100% / $count;\n    max-width: 100% / $count;\n  }\n}\n","// Breakpoint viewport sizes and media queries.\n//\n// Breakpoints are defined as a map of (name: minimum width), order from small to large:\n//\n//    (xs: 0, sm: 576px, md: 768px, lg: 992px, xl: 1200px)\n//\n// The map defined in the `$grid-breakpoints` global variable is used as the `$breakpoints` argument by default.\n\n// Name of the next breakpoint, or null for the last breakpoint.\n//\n//    >> breakpoint-next(sm)\n//    md\n//    >> breakpoint-next(sm, (xs: 0, sm: 576px, md: 768px, lg: 992px, xl: 1200px))\n//    md\n//    >> breakpoint-next(sm, $breakpoint-names: (xs sm md lg xl))\n//    md\n@function breakpoint-next($name, $breakpoints: $grid-breakpoints, $breakpoint-names: map-keys($breakpoints)) {\n  $n: index($breakpoint-names, $name);\n  @return if($n != null and $n < length($breakpoint-names), nth($breakpoint-names, $n + 1), null);\n}\n\n// Minimum breakpoint width. Null for the smallest (first) breakpoint.\n//\n//    >> breakpoint-min(sm, (xs: 0, sm: 576px, md: 768px, lg: 992px, xl: 1200px))\n//    576px\n@function breakpoint-min($name, $breakpoints: $grid-breakpoints) {\n  $min: map-get($breakpoints, $name);\n  @return if($min != 0, $min, null);\n}\n\n// Maximum breakpoint width. Null for the largest (last) breakpoint.\n// The maximum value is calculated as the minimum of the next one less 0.02px\n// to work around the limitations of `min-` and `max-` prefixes and viewports with fractional widths.\n// See https://www.w3.org/TR/mediaqueries-4/#mq-min-max\n// Uses 0.02px rather than 0.01px to work around a current rounding bug in Safari.\n// See https://bugs.webkit.org/show_bug.cgi?id=178261\n//\n//    >> breakpoint-max(sm, (xs: 0, sm: 576px, md: 768px, lg: 992px, xl: 1200px))\n//    767.98px\n@function breakpoint-max($name, $breakpoints: $grid-breakpoints) {\n  $next: breakpoint-next($name, $breakpoints);\n  @return if($next, breakpoint-min($next, $breakpoints) - .02, null);\n}\n\n// Returns a blank string if smallest breakpoint, otherwise returns the name with a dash in front.\n// Useful for making responsive utilities.\n//\n//    >> breakpoint-infix(xs, (xs: 0, sm: 576px, md: 768px, lg: 992px, xl: 1200px))\n//    \"\"  (Returns a blank string)\n//    >> breakpoint-infix(sm, (xs: 0, sm: 576px, md: 768px, lg: 992px, xl: 1200px))\n//    \"-sm\"\n@function breakpoint-infix($name, $breakpoints: $grid-breakpoints) {\n  @return if(breakpoint-min($name, $breakpoints) == null, \"\", \"-#{$name}\");\n}\n\n// Media of at least the minimum breakpoint width. No query for the smallest breakpoint.\n// Makes the @content apply to the given breakpoint and wider.\n@mixin media-breakpoint-up($name, $breakpoints: $grid-breakpoints) {\n  $min: breakpoint-min($name, $breakpoints);\n  @if $min {\n    @media (min-width: $min) {\n      @content;\n    }\n  } @else {\n    @content;\n  }\n}\n\n// Media of at most the maximum breakpoint width. No query for the largest breakpoint.\n// Makes the @content apply to the given breakpoint and narrower.\n@mixin media-breakpoint-down($name, $breakpoints: $grid-breakpoints) {\n  $max: breakpoint-max($name, $breakpoints);\n  @if $max {\n    @media (max-width: $max) {\n      @content;\n    }\n  } @else {\n    @content;\n  }\n}\n\n// Media that spans multiple breakpoint widths.\n// Makes the @content apply between the min and max breakpoints\n@mixin media-breakpoint-between($lower, $upper, $breakpoints: $grid-breakpoints) {\n  $min: breakpoint-min($lower, $breakpoints);\n  $max: breakpoint-max($upper, $breakpoints);\n\n  @if $min != null and $max != null {\n    @media (min-width: $min) and (max-width: $max) {\n      @content;\n    }\n  } @else if $max == null {\n    @include media-breakpoint-up($lower, $breakpoints) {\n      @content;\n    }\n  } @else if $min == null {\n    @include media-breakpoint-down($upper, $breakpoints) {\n      @content;\n    }\n  }\n}\n\n// Media between the breakpoint's minimum and maximum widths.\n// No minimum for the smallest breakpoint, and no maximum for the largest one.\n// Makes the @content apply only to the given breakpoint, not viewports any wider or narrower.\n@mixin media-breakpoint-only($name, $breakpoints: $grid-breakpoints) {\n  $min: breakpoint-min($name, $breakpoints);\n  $max: breakpoint-max($name, $breakpoints);\n\n  @if $min != null and $max != null {\n    @media (min-width: $min) and (max-width: $max) {\n      @content;\n    }\n  } @else if $max == null {\n    @include media-breakpoint-up($name, $breakpoints) {\n      @content;\n    }\n  } @else if $min == null {\n    @include media-breakpoint-down($name, $breakpoints) {\n      @content;\n    }\n  }\n}\n","// Framework grid generation\n//\n// Used only by Bootstrap to generate the correct number of grid classes given\n// any value of `$grid-columns`.\n\n@mixin make-grid-columns($columns: $grid-columns, $gutter: $grid-gutter-width, $breakpoints: $grid-breakpoints) {\n  // Common properties for all breakpoints\n  %grid-column {\n    position: relative;\n    width: 100%;\n    padding-right: $gutter / 2;\n    padding-left: $gutter / 2;\n  }\n\n  @each $breakpoint in map-keys($breakpoints) {\n    $infix: breakpoint-infix($breakpoint, $breakpoints);\n\n    // Allow columns to stretch full width below their breakpoints\n    @for $i from 1 through $columns {\n      .col#{$infix}-#{$i} {\n        @extend %grid-column;\n      }\n    }\n    .col#{$infix},\n    .col#{$infix}-auto {\n      @extend %grid-column;\n    }\n\n    @include media-breakpoint-up($breakpoint, $breakpoints) {\n      // Provide basic `.col-{bp}` classes for equal-width flexbox columns\n      .col#{$infix} {\n        flex-basis: 0;\n        flex-grow: 1;\n        max-width: 100%;\n      }\n\n      @for $i from 1 through $grid-row-columns {\n        .row-cols#{$infix}-#{$i} {\n          @include row-cols($i);\n        }\n      }\n\n      .col#{$infix}-auto {\n        @include make-col-auto();\n      }\n\n      @for $i from 1 through $columns {\n        .col#{$infix}-#{$i} {\n          @include make-col($i, $columns);\n        }\n      }\n\n      .order#{$infix}-first { order: -1; }\n\n      .order#{$infix}-last { order: $columns + 1; }\n\n      @for $i from 0 through $columns {\n        .order#{$infix}-#{$i} { order: $i; }\n      }\n\n      // `$columns - 1` because offsetting by the width of an entire row isn't possible\n      @for $i from 0 through ($columns - 1) {\n        @if not ($infix == \"\" and $i == 0) { // Avoid emitting useless .offset-0\n          .offset#{$infix}-#{$i} {\n            @include make-col-offset($i, $columns);\n          }\n        }\n      }\n    }\n  }\n}\n","//\n// Basic Bootstrap table\n//\n\n.table {\n  width: 100%;\n  margin-bottom: $spacer;\n  color: $table-color;\n  background-color: $table-bg; // Reset for nesting within parents with `background-color`.\n\n  th,\n  td {\n    padding: $table-cell-padding;\n    vertical-align: top;\n    border-top: $table-border-width solid $table-border-color;\n  }\n\n  thead th {\n    vertical-align: bottom;\n    border-bottom: (2 * $table-border-width) solid $table-border-color;\n  }\n\n  tbody + tbody {\n    border-top: (2 * $table-border-width) solid $table-border-color;\n  }\n}\n\n\n//\n// Condensed table w/ half padding\n//\n\n.table-sm {\n  th,\n  td {\n    padding: $table-cell-padding-sm;\n  }\n}\n\n\n// Border versions\n//\n// Add or remove borders all around the table and between all the columns.\n\n.table-bordered {\n  border: $table-border-width solid $table-border-color;\n\n  th,\n  td {\n    border: $table-border-width solid $table-border-color;\n  }\n\n  thead {\n    th,\n    td {\n      border-bottom-width: 2 * $table-border-width;\n    }\n  }\n}\n\n.table-borderless {\n  th,\n  td,\n  thead th,\n  tbody + tbody {\n    border: 0;\n  }\n}\n\n// Zebra-striping\n//\n// Default zebra-stripe styles (alternating gray and transparent backgrounds)\n\n.table-striped {\n  tbody tr:nth-of-type(#{$table-striped-order}) {\n    background-color: $table-accent-bg;\n  }\n}\n\n\n// Hover effect\n//\n// Placed here since it has to come after the potential zebra striping\n\n.table-hover {\n  tbody tr {\n    @include hover() {\n      color: $table-hover-color;\n      background-color: $table-hover-bg;\n    }\n  }\n}\n\n\n// Table backgrounds\n//\n// Exact selectors below required to override `.table-striped` and prevent\n// inheritance to nested tables.\n\n@each $color, $value in $theme-colors {\n  @include table-row-variant($color, theme-color-level($color, $table-bg-level), theme-color-level($color, $table-border-level));\n}\n\n@include table-row-variant(active, $table-active-bg);\n\n\n// Dark styles\n//\n// Same table markup, but inverted color scheme: dark background and light text.\n\n// stylelint-disable-next-line no-duplicate-selectors\n.table {\n  .thead-dark {\n    th {\n      color: $table-dark-color;\n      background-color: $table-dark-bg;\n      border-color: $table-dark-border-color;\n    }\n  }\n\n  .thead-light {\n    th {\n      color: $table-head-color;\n      background-color: $table-head-bg;\n      border-color: $table-border-color;\n    }\n  }\n}\n\n.table-dark {\n  color: $table-dark-color;\n  background-color: $table-dark-bg;\n\n  th,\n  td,\n  thead th {\n    border-color: $table-dark-border-color;\n  }\n\n  &.table-bordered {\n    border: 0;\n  }\n\n  &.table-striped {\n    tbody tr:nth-of-type(#{$table-striped-order}) {\n      background-color: $table-dark-accent-bg;\n    }\n  }\n\n  &.table-hover {\n    tbody tr {\n      @include hover() {\n        color: $table-dark-hover-color;\n        background-color: $table-dark-hover-bg;\n      }\n    }\n  }\n}\n\n\n// Responsive tables\n//\n// Generate series of `.table-responsive-*` classes for configuring the screen\n// size of where your table will overflow.\n\n.table-responsive {\n  @each $breakpoint in map-keys($grid-breakpoints) {\n    $next: breakpoint-next($breakpoint, $grid-breakpoints);\n    $infix: breakpoint-infix($next, $grid-breakpoints);\n\n    &#{$infix} {\n      @include media-breakpoint-down($breakpoint) {\n        display: block;\n        width: 100%;\n        overflow-x: auto;\n        -webkit-overflow-scrolling: touch;\n\n        // Prevent double border on horizontal scroll due to use of `display: block;`\n        > .table-bordered {\n          border: 0;\n        }\n      }\n    }\n  }\n}\n","// Tables\n\n@mixin table-row-variant($state, $background, $border: null) {\n  // Exact selectors below required to override `.table-striped` and prevent\n  // inheritance to nested tables.\n  .table-#{$state} {\n    &,\n    > th,\n    > td {\n      background-color: $background;\n    }\n\n    @if $border != null {\n      th,\n      td,\n      thead th,\n      tbody + tbody {\n        border-color: $border;\n      }\n    }\n  }\n\n  // Hover states for `.table-hover`\n  // Note: this is not available for cells or rows within `thead` or `tfoot`.\n  .table-hover {\n    $hover-background: darken($background, 5%);\n\n    .table-#{$state} {\n      @include hover() {\n        background-color: $hover-background;\n\n        > td,\n        > th {\n          background-color: $hover-background;\n        }\n      }\n    }\n  }\n}\n","// stylelint-disable selector-no-qualifying-type\n\n//\n// Textual form controls\n//\n\n.form-control {\n  display: block;\n  width: 100%;\n  height: $input-height;\n  padding: $input-padding-y $input-padding-x;\n  font-family: $input-font-family;\n  @include font-size($input-font-size);\n  font-weight: $input-font-weight;\n  line-height: $input-line-height;\n  color: $input-color;\n  background-color: $input-bg;\n  background-clip: padding-box;\n  border: $input-border-width solid $input-border-color;\n\n  // Note: This has no effect on <select>s in some browsers, due to the limited stylability of `<select>`s in CSS.\n  @include border-radius($input-border-radius, 0);\n\n  @include box-shadow($input-box-shadow);\n  @include transition($input-transition);\n\n  // Unstyle the caret on `<select>`s in IE10+.\n  &::-ms-expand {\n    background-color: transparent;\n    border: 0;\n  }\n\n  // Remove select outline from select box in FF\n  &:-moz-focusring {\n    color: transparent;\n    text-shadow: 0 0 0 $input-color;\n  }\n\n  // Customize the `:focus` state to imitate native WebKit styles.\n  @include form-control-focus($ignore-warning: true);\n\n  // Placeholder\n  &::placeholder {\n    color: $input-placeholder-color;\n    // Override Firefox's unusual default opacity; see https://github.com/twbs/bootstrap/pull/11526.\n    opacity: 1;\n  }\n\n  // Disabled and read-only inputs\n  //\n  // HTML5 says that controls under a fieldset > legend:first-child won't be\n  // disabled if the fieldset is disabled. Due to implementation difficulty, we\n  // don't honor that edge case; we style them as disabled anyway.\n  &:disabled,\n  &[readonly] {\n    background-color: $input-disabled-bg;\n    // iOS fix for unreadable disabled content; see https://github.com/twbs/bootstrap/issues/11655.\n    opacity: 1;\n  }\n}\n\nselect.form-control {\n  &:focus::-ms-value {\n    // Suppress the nested default white text on blue background highlight given to\n    // the selected option text when the (still closed) <select> receives focus\n    // in IE and (under certain conditions) Edge, as it looks bad and cannot be made to\n    // match the appearance of the native widget.\n    // See https://github.com/twbs/bootstrap/issues/19398.\n    color: $input-color;\n    background-color: $input-bg;\n  }\n}\n\n// Make file inputs better match text inputs by forcing them to new lines.\n.form-control-file,\n.form-control-range {\n  display: block;\n  width: 100%;\n}\n\n\n//\n// Labels\n//\n\n// For use with horizontal and inline forms, when you need the label (or legend)\n// text to align with the form controls.\n.col-form-label {\n  padding-top: add($input-padding-y, $input-border-width);\n  padding-bottom: add($input-padding-y, $input-border-width);\n  margin-bottom: 0; // Override the `<label>/<legend>` default\n  @include font-size(inherit); // Override the `<legend>` default\n  line-height: $input-line-height;\n}\n\n.col-form-label-lg {\n  padding-top: add($input-padding-y-lg, $input-border-width);\n  padding-bottom: add($input-padding-y-lg, $input-border-width);\n  @include font-size($input-font-size-lg);\n  line-height: $input-line-height-lg;\n}\n\n.col-form-label-sm {\n  padding-top: add($input-padding-y-sm, $input-border-width);\n  padding-bottom: add($input-padding-y-sm, $input-border-width);\n  @include font-size($input-font-size-sm);\n  line-height: $input-line-height-sm;\n}\n\n\n// Readonly controls as plain text\n//\n// Apply class to a readonly input to make it appear like regular plain\n// text (without any border, background color, focus indicator)\n\n.form-control-plaintext {\n  display: block;\n  width: 100%;\n  padding: $input-padding-y 0;\n  margin-bottom: 0; // match inputs if this class comes on inputs with default margins\n  @include font-size($input-font-size);\n  line-height: $input-line-height;\n  color: $input-plaintext-color;\n  background-color: transparent;\n  border: solid transparent;\n  border-width: $input-border-width 0;\n\n  &.form-control-sm,\n  &.form-control-lg {\n    padding-right: 0;\n    padding-left: 0;\n  }\n}\n\n\n// Form control sizing\n//\n// Build on `.form-control` with modifier classes to decrease or increase the\n// height and font-size of form controls.\n//\n// Repeated in `_input_group.scss` to avoid Sass extend issues.\n\n.form-control-sm {\n  height: $input-height-sm;\n  padding: $input-padding-y-sm $input-padding-x-sm;\n  @include font-size($input-font-size-sm);\n  line-height: $input-line-height-sm;\n  @include border-radius($input-border-radius-sm);\n}\n\n.form-control-lg {\n  height: $input-height-lg;\n  padding: $input-padding-y-lg $input-padding-x-lg;\n  @include font-size($input-font-size-lg);\n  line-height: $input-line-height-lg;\n  @include border-radius($input-border-radius-lg);\n}\n\n// stylelint-disable-next-line no-duplicate-selectors\nselect.form-control {\n  &[size],\n  &[multiple] {\n    height: auto;\n  }\n}\n\ntextarea.form-control {\n  height: auto;\n}\n\n// Form groups\n//\n// Designed to help with the organization and spacing of vertical forms. For\n// horizontal forms, use the predefined grid classes.\n\n.form-group {\n  margin-bottom: $form-group-margin-bottom;\n}\n\n.form-text {\n  display: block;\n  margin-top: $form-text-margin-top;\n}\n\n\n// Form grid\n//\n// Special replacement for our grid system's `.row` for tighter form layouts.\n\n.form-row {\n  display: flex;\n  flex-wrap: wrap;\n  margin-right: -$form-grid-gutter-width / 2;\n  margin-left: -$form-grid-gutter-width / 2;\n\n  > .col,\n  > [class*=\"col-\"] {\n    padding-right: $form-grid-gutter-width / 2;\n    padding-left: $form-grid-gutter-width / 2;\n  }\n}\n\n\n// Checkboxes and radios\n//\n// Indent the labels to position radios/checkboxes as hanging controls.\n\n.form-check {\n  position: relative;\n  display: block;\n  padding-left: $form-check-input-gutter;\n}\n\n.form-check-input {\n  position: absolute;\n  margin-top: $form-check-input-margin-y;\n  margin-left: -$form-check-input-gutter;\n\n  // Use [disabled] and :disabled for workaround https://github.com/twbs/bootstrap/issues/28247\n  &[disabled] ~ .form-check-label,\n  &:disabled ~ .form-check-label {\n    color: $text-muted;\n  }\n}\n\n.form-check-label {\n  margin-bottom: 0; // Override default `<label>` bottom margin\n}\n\n.form-check-inline {\n  display: inline-flex;\n  align-items: center;\n  padding-left: 0; // Override base .form-check\n  margin-right: $form-check-inline-margin-x;\n\n  // Undo .form-check-input defaults and add some `margin-right`.\n  .form-check-input {\n    position: static;\n    margin-top: 0;\n    margin-right: $form-check-inline-input-margin-x;\n    margin-left: 0;\n  }\n}\n\n\n// Form validation\n//\n// Provide feedback to users when form field values are valid or invalid. Works\n// primarily for client-side validation via scoped `:invalid` and `:valid`\n// pseudo-classes but also includes `.is-invalid` and `.is-valid` classes for\n// server side validation.\n\n@each $state, $data in $form-validation-states {\n  @include form-validation-state($state, map-get($data, color), map-get($data, icon));\n}\n\n// Inline forms\n//\n// Make forms appear inline(-block) by adding the `.form-inline` class. Inline\n// forms begin stacked on extra small (mobile) devices and then go inline when\n// viewports reach <768px.\n//\n// Requires wrapping inputs and labels with `.form-group` for proper display of\n// default HTML form controls and our custom form controls (e.g., input groups).\n\n.form-inline {\n  display: flex;\n  flex-flow: row wrap;\n  align-items: center; // Prevent shorter elements from growing to same height as others (e.g., small buttons growing to normal sized button height)\n\n  // Because we use flex, the initial sizing of checkboxes is collapsed and\n  // doesn't occupy the full-width (which is what we want for xs grid tier),\n  // so we force that here.\n  .form-check {\n    width: 100%;\n  }\n\n  // Kick in the inline\n  @include media-breakpoint-up(sm) {\n    label {\n      display: flex;\n      align-items: center;\n      justify-content: center;\n      margin-bottom: 0;\n    }\n\n    // Inline-block all the things for \"inline\"\n    .form-group {\n      display: flex;\n      flex: 0 0 auto;\n      flex-flow: row wrap;\n      align-items: center;\n      margin-bottom: 0;\n    }\n\n    // Allow folks to *not* use `.form-group`\n    .form-control {\n      display: inline-block;\n      width: auto; // Prevent labels from stacking above inputs in `.form-group`\n      vertical-align: middle;\n    }\n\n    // Make static controls behave like regular ones\n    .form-control-plaintext {\n      display: inline-block;\n    }\n\n    .input-group,\n    .custom-select {\n      width: auto;\n    }\n\n    // Remove default margin on radios/checkboxes that were used for stacking, and\n    // then undo the floating of radios and checkboxes to match.\n    .form-check {\n      display: flex;\n      align-items: center;\n      justify-content: center;\n      width: auto;\n      padding-left: 0;\n    }\n    .form-check-input {\n      position: relative;\n      flex-shrink: 0;\n      margin-top: 0;\n      margin-right: $form-check-input-margin-x;\n      margin-left: 0;\n    }\n\n    .custom-control {\n      align-items: center;\n      justify-content: center;\n    }\n    .custom-control-label {\n      margin-bottom: 0;\n    }\n  }\n}\n","// stylelint-disable property-blacklist\n@mixin transition($transition...) {\n  @if $enable-transitions {\n    @if length($transition) == 0 {\n      transition: $transition-base;\n    } @else {\n      transition: $transition;\n    }\n  }\n\n  @if $enable-prefers-reduced-motion-media-query {\n    @media (prefers-reduced-motion: reduce) {\n      transition: none;\n    }\n  }\n}\n","// Form control focus state\n//\n// Generate a customized focus state and for any input with the specified color,\n// which defaults to the `$input-focus-border-color` variable.\n//\n// We highly encourage you to not customize the default value, but instead use\n// this to tweak colors on an as-needed basis. This aesthetic change is based on\n// WebKit's default styles, but applicable to a wider range of browsers. Its\n// usability and accessibility should be taken into account with any change.\n//\n// Example usage: change the default blue border and shadow to white for better\n// contrast against a dark gray background.\n@mixin form-control-focus($ignore-warning: false) {\n  &:focus {\n    color: $input-focus-color;\n    background-color: $input-focus-bg;\n    border-color: $input-focus-border-color;\n    outline: 0;\n    // Avoid using mixin so we can pass custom focus shadow properly\n    @if $enable-shadows {\n      box-shadow: $input-box-shadow, $input-focus-box-shadow;\n    } @else {\n      box-shadow: $input-focus-box-shadow;\n    }\n  }\n  @include deprecate(\"The `form-control-focus()` mixin\", \"v4.4.0\", \"v5\", $ignore-warning);\n}\n\n// This mixin uses an `if()` technique to be compatible with Dart Sass\n// See https://github.com/sass/sass/issues/1873#issuecomment-152293725 for more details\n@mixin form-validation-state-selector($state) {\n  @if ($state == \"valid\" or $state == \"invalid\") {\n    .was-validated #{if(&, \"&\", \"\")}:#{$state},\n    #{if(&, \"&\", \"\")}.is-#{$state} {\n      @content;\n    }\n  } @else {\n    #{if(&, \"&\", \"\")}.is-#{$state} {\n      @content;\n    }\n  }\n}\n\n@mixin form-validation-state($state, $color, $icon) {\n  .#{$state}-feedback {\n    display: none;\n    width: 100%;\n    margin-top: $form-feedback-margin-top;\n    @include font-size($form-feedback-font-size);\n    color: $color;\n  }\n\n  .#{$state}-tooltip {\n    position: absolute;\n    top: 100%;\n    z-index: 5;\n    display: none;\n    max-width: 100%; // Contain to parent when possible\n    padding: $form-feedback-tooltip-padding-y $form-feedback-tooltip-padding-x;\n    margin-top: .1rem;\n    @include font-size($form-feedback-tooltip-font-size);\n    line-height: $form-feedback-tooltip-line-height;\n    color: color-yiq($color);\n    background-color: rgba($color, $form-feedback-tooltip-opacity);\n    @include border-radius($form-feedback-tooltip-border-radius);\n  }\n\n  @include form-validation-state-selector($state) {\n    ~ .#{$state}-feedback,\n    ~ .#{$state}-tooltip {\n      display: block;\n    }\n  }\n\n  .form-control {\n    @include form-validation-state-selector($state) {\n      border-color: $color;\n\n      @if $enable-validation-icons {\n        padding-right: $input-height-inner;\n        background-image: escape-svg($icon);\n        background-repeat: no-repeat;\n        background-position: right $input-height-inner-quarter center;\n        background-size: $input-height-inner-half $input-height-inner-half;\n      }\n\n      &:focus {\n        border-color: $color;\n        box-shadow: 0 0 0 $input-focus-width rgba($color, .25);\n      }\n    }\n  }\n\n  // stylelint-disable-next-line selector-no-qualifying-type\n  textarea.form-control {\n    @include form-validation-state-selector($state) {\n      @if $enable-validation-icons {\n        padding-right: $input-height-inner;\n        background-position: top $input-height-inner-quarter right $input-height-inner-quarter;\n      }\n    }\n  }\n\n  .custom-select {\n    @include form-validation-state-selector($state) {\n      border-color: $color;\n\n      @if $enable-validation-icons {\n        padding-right: $custom-select-feedback-icon-padding-right;\n        background: $custom-select-background, escape-svg($icon) $custom-select-bg no-repeat $custom-select-feedback-icon-position / $custom-select-feedback-icon-size;\n      }\n\n      &:focus {\n        border-color: $color;\n        box-shadow: 0 0 0 $input-focus-width rgba($color, .25);\n      }\n    }\n  }\n\n  .form-check-input {\n    @include form-validation-state-selector($state) {\n      ~ .form-check-label {\n        color: $color;\n      }\n\n      ~ .#{$state}-feedback,\n      ~ .#{$state}-tooltip {\n        display: block;\n      }\n    }\n  }\n\n  .custom-control-input {\n    @include form-validation-state-selector($state) {\n      ~ .custom-control-label {\n        color: $color;\n\n        &::before {\n          border-color: $color;\n        }\n      }\n\n      &:checked {\n        ~ .custom-control-label::before {\n          border-color: lighten($color, 10%);\n          @include gradient-bg(lighten($color, 10%));\n        }\n      }\n\n      &:focus {\n        ~ .custom-control-label::before {\n          box-shadow: 0 0 0 $input-focus-width rgba($color, .25);\n        }\n\n        &:not(:checked) ~ .custom-control-label::before {\n          border-color: $color;\n        }\n      }\n    }\n  }\n\n  // custom file\n  .custom-file-input {\n    @include form-validation-state-selector($state) {\n      ~ .custom-file-label {\n        border-color: $color;\n      }\n\n      &:focus {\n        ~ .custom-file-label {\n          border-color: $color;\n          box-shadow: 0 0 0 $input-focus-width rgba($color, .25);\n        }\n      }\n    }\n  }\n}\n","// Gradients\n\n@mixin gradient-bg($color) {\n  @if $enable-gradients {\n    background: $color linear-gradient(180deg, mix($body-bg, $color, 15%), $color) repeat-x;\n  } @else {\n    background-color: $color;\n  }\n}\n\n// Horizontal gradient, from left to right\n//\n// Creates two color stops, start and end, by specifying a color and position for each color stop.\n@mixin gradient-x($start-color: $gray-700, $end-color: $gray-800, $start-percent: 0%, $end-percent: 100%) {\n  background-image: linear-gradient(to right, $start-color $start-percent, $end-color $end-percent);\n  background-repeat: repeat-x;\n}\n\n// Vertical gradient, from top to bottom\n//\n// Creates two color stops, start and end, by specifying a color and position for each color stop.\n@mixin gradient-y($start-color: $gray-700, $end-color: $gray-800, $start-percent: 0%, $end-percent: 100%) {\n  background-image: linear-gradient(to bottom, $start-color $start-percent, $end-color $end-percent);\n  background-repeat: repeat-x;\n}\n\n@mixin gradient-directional($start-color: $gray-700, $end-color: $gray-800, $deg: 45deg) {\n  background-image: linear-gradient($deg, $start-color, $end-color);\n  background-repeat: repeat-x;\n}\n@mixin gradient-x-three-colors($start-color: $blue, $mid-color: $purple, $color-stop: 50%, $end-color: $red) {\n  background-image: linear-gradient(to right, $start-color, $mid-color $color-stop, $end-color);\n  background-repeat: no-repeat;\n}\n@mixin gradient-y-three-colors($start-color: $blue, $mid-color: $purple, $color-stop: 50%, $end-color: $red) {\n  background-image: linear-gradient($start-color, $mid-color $color-stop, $end-color);\n  background-repeat: no-repeat;\n}\n@mixin gradient-radial($inner-color: $gray-700, $outer-color: $gray-800) {\n  background-image: radial-gradient(circle, $inner-color, $outer-color);\n  background-repeat: no-repeat;\n}\n@mixin gradient-striped($color: rgba($white, .15), $angle: 45deg) {\n  background-image: linear-gradient($angle, $color 25%, transparent 25%, transparent 50%, $color 50%, $color 75%, transparent 75%, transparent);\n}\n","// stylelint-disable selector-no-qualifying-type\n\n//\n// Base styles\n//\n\n.btn {\n  display: inline-block;\n  font-family: $btn-font-family;\n  font-weight: $btn-font-weight;\n  color: $body-color;\n  text-align: center;\n  white-space: $btn-white-space;\n  vertical-align: middle;\n  cursor: if($enable-pointer-cursor-for-buttons, pointer, null);\n  user-select: none;\n  background-color: transparent;\n  border: $btn-border-width solid transparent;\n  @include button-size($btn-padding-y, $btn-padding-x, $btn-font-size, $btn-line-height, $btn-border-radius);\n  @include transition($btn-transition);\n\n  @include hover() {\n    color: $body-color;\n    text-decoration: none;\n  }\n\n  &:focus,\n  &.focus {\n    outline: 0;\n    box-shadow: $btn-focus-box-shadow;\n  }\n\n  // Disabled comes first so active can properly restyle\n  &.disabled,\n  &:disabled {\n    opacity: $btn-disabled-opacity;\n    @include box-shadow(none);\n  }\n\n  &:not(:disabled):not(.disabled):active,\n  &:not(:disabled):not(.disabled).active {\n    @include box-shadow($btn-active-box-shadow);\n\n    &:focus {\n      @include box-shadow($btn-focus-box-shadow, $btn-active-box-shadow);\n    }\n  }\n}\n\n// Future-proof disabling of clicks on `<a>` elements\na.btn.disabled,\nfieldset:disabled a.btn {\n  pointer-events: none;\n}\n\n\n//\n// Alternate buttons\n//\n\n@each $color, $value in $theme-colors {\n  .btn-#{$color} {\n    @include button-variant($value, $value);\n  }\n}\n\n@each $color, $value in $theme-colors {\n  .btn-outline-#{$color} {\n    @include button-outline-variant($value);\n  }\n}\n\n\n//\n// Link buttons\n//\n\n// Make a button look and behave like a link\n.btn-link {\n  font-weight: $font-weight-normal;\n  color: $link-color;\n  text-decoration: $link-decoration;\n\n  @include hover() {\n    color: $link-hover-color;\n    text-decoration: $link-hover-decoration;\n  }\n\n  &:focus,\n  &.focus {\n    text-decoration: $link-hover-decoration;\n    box-shadow: none;\n  }\n\n  &:disabled,\n  &.disabled {\n    color: $btn-link-disabled-color;\n    pointer-events: none;\n  }\n\n  // No need for an active state here\n}\n\n\n//\n// Button Sizes\n//\n\n.btn-lg {\n  @include button-size($btn-padding-y-lg, $btn-padding-x-lg, $btn-font-size-lg, $btn-line-height-lg, $btn-border-radius-lg);\n}\n\n.btn-sm {\n  @include button-size($btn-padding-y-sm, $btn-padding-x-sm, $btn-font-size-sm, $btn-line-height-sm, $btn-border-radius-sm);\n}\n\n\n//\n// Block button\n//\n\n.btn-block {\n  display: block;\n  width: 100%;\n\n  // Vertically space out multiple block buttons\n  + .btn-block {\n    margin-top: $btn-block-spacing-y;\n  }\n}\n\n// Specificity overrides\ninput[type=\"submit\"],\ninput[type=\"reset\"],\ninput[type=\"button\"] {\n  &.btn-block {\n    width: 100%;\n  }\n}\n","// Button variants\n//\n// Easily pump out default styles, as well as :hover, :focus, :active,\n// and disabled options for all buttons\n\n@mixin button-variant($background, $border, $hover-background: darken($background, 7.5%), $hover-border: darken($border, 10%), $active-background: darken($background, 10%), $active-border: darken($border, 12.5%)) {\n  color: color-yiq($background);\n  @include gradient-bg($background);\n  border-color: $border;\n  @include box-shadow($btn-box-shadow);\n\n  @include hover() {\n    color: color-yiq($hover-background);\n    @include gradient-bg($hover-background);\n    border-color: $hover-border;\n  }\n\n  &:focus,\n  &.focus {\n    color: color-yiq($hover-background);\n    @include gradient-bg($hover-background);\n    border-color: $hover-border;\n    // Avoid using mixin so we can pass custom focus shadow properly\n    @if $enable-shadows {\n      box-shadow: $btn-box-shadow, 0 0 0 $btn-focus-width rgba(mix(color-yiq($background), $border, 15%), .5);\n    } @else {\n      box-shadow: 0 0 0 $btn-focus-width rgba(mix(color-yiq($background), $border, 15%), .5);\n    }\n  }\n\n  // Disabled comes first so active can properly restyle\n  &.disabled,\n  &:disabled {\n    color: color-yiq($background);\n    background-color: $background;\n    border-color: $border;\n    // Remove CSS gradients if they're enabled\n    @if $enable-gradients {\n      background-image: none;\n    }\n  }\n\n  &:not(:disabled):not(.disabled):active,\n  &:not(:disabled):not(.disabled).active,\n  .show > &.dropdown-toggle {\n    color: color-yiq($active-background);\n    background-color: $active-background;\n    @if $enable-gradients {\n      background-image: none; // Remove the gradient for the pressed/active state\n    }\n    border-color: $active-border;\n\n    &:focus {\n      // Avoid using mixin so we can pass custom focus shadow properly\n      @if $enable-shadows and $btn-active-box-shadow != none {\n        box-shadow: $btn-active-box-shadow, 0 0 0 $btn-focus-width rgba(mix(color-yiq($background), $border, 15%), .5);\n      } @else {\n        box-shadow: 0 0 0 $btn-focus-width rgba(mix(color-yiq($background), $border, 15%), .5);\n      }\n    }\n  }\n}\n\n@mixin button-outline-variant($color, $color-hover: color-yiq($color), $active-background: $color, $active-border: $color) {\n  color: $color;\n  border-color: $color;\n\n  @include hover() {\n    color: $color-hover;\n    background-color: $active-background;\n    border-color: $active-border;\n  }\n\n  &:focus,\n  &.focus {\n    box-shadow: 0 0 0 $btn-focus-width rgba($color, .5);\n  }\n\n  &.disabled,\n  &:disabled {\n    color: $color;\n    background-color: transparent;\n  }\n\n  &:not(:disabled):not(.disabled):active,\n  &:not(:disabled):not(.disabled).active,\n  .show > &.dropdown-toggle {\n    color: color-yiq($active-background);\n    background-color: $active-background;\n    border-color: $active-border;\n\n    &:focus {\n      // Avoid using mixin so we can pass custom focus shadow properly\n      @if $enable-shadows and $btn-active-box-shadow != none {\n        box-shadow: $btn-active-box-shadow, 0 0 0 $btn-focus-width rgba($color, .5);\n      } @else {\n        box-shadow: 0 0 0 $btn-focus-width rgba($color, .5);\n      }\n    }\n  }\n}\n\n// Button sizes\n@mixin button-size($padding-y, $padding-x, $font-size, $line-height, $border-radius) {\n  padding: $padding-y $padding-x;\n  @include font-size($font-size);\n  line-height: $line-height;\n  // Manually declare to provide an override to the browser default\n  @include border-radius($border-radius, 0);\n}\n",".fade {\n  @include transition($transition-fade);\n\n  &:not(.show) {\n    opacity: 0;\n  }\n}\n\n.collapse {\n  &:not(.show) {\n    display: none;\n  }\n}\n\n.collapsing {\n  position: relative;\n  height: 0;\n  overflow: hidden;\n  @include transition($transition-collapse);\n}\n","// The dropdown wrapper (`<div>`)\n.dropup,\n.dropright,\n.dropdown,\n.dropleft {\n  position: relative;\n}\n\n.dropdown-toggle {\n  white-space: nowrap;\n\n  // Generate the caret automatically\n  @include caret();\n}\n\n// The dropdown menu\n.dropdown-menu {\n  position: absolute;\n  top: 100%;\n  left: 0;\n  z-index: $zindex-dropdown;\n  display: none; // none by default, but block on \"open\" of the menu\n  float: left;\n  min-width: $dropdown-min-width;\n  padding: $dropdown-padding-y 0;\n  margin: $dropdown-spacer 0 0; // override default ul\n  @include font-size($dropdown-font-size);\n  color: $dropdown-color;\n  text-align: left; // Ensures proper alignment if parent has it changed (e.g., modal footer)\n  list-style: none;\n  background-color: $dropdown-bg;\n  background-clip: padding-box;\n  border: $dropdown-border-width solid $dropdown-border-color;\n  @include border-radius($dropdown-border-radius);\n  @include box-shadow($dropdown-box-shadow);\n}\n\n@each $breakpoint in map-keys($grid-breakpoints) {\n  @include media-breakpoint-up($breakpoint) {\n    $infix: breakpoint-infix($breakpoint, $grid-breakpoints);\n\n    .dropdown-menu#{$infix}-left {\n      right: auto;\n      left: 0;\n    }\n\n    .dropdown-menu#{$infix}-right {\n      right: 0;\n      left: auto;\n    }\n  }\n}\n\n// Allow for dropdowns to go bottom up (aka, dropup-menu)\n// Just add .dropup after the standard .dropdown class and you're set.\n.dropup {\n  .dropdown-menu {\n    top: auto;\n    bottom: 100%;\n    margin-top: 0;\n    margin-bottom: $dropdown-spacer;\n  }\n\n  .dropdown-toggle {\n    @include caret(up);\n  }\n}\n\n.dropright {\n  .dropdown-menu {\n    top: 0;\n    right: auto;\n    left: 100%;\n    margin-top: 0;\n    margin-left: $dropdown-spacer;\n  }\n\n  .dropdown-toggle {\n    @include caret(right);\n    &::after {\n      vertical-align: 0;\n    }\n  }\n}\n\n.dropleft {\n  .dropdown-menu {\n    top: 0;\n    right: 100%;\n    left: auto;\n    margin-top: 0;\n    margin-right: $dropdown-spacer;\n  }\n\n  .dropdown-toggle {\n    @include caret(left);\n    &::before {\n      vertical-align: 0;\n    }\n  }\n}\n\n// When enabled Popper.js, reset basic dropdown position\n// stylelint-disable-next-line no-duplicate-selectors\n.dropdown-menu {\n  &[x-placement^=\"top\"],\n  &[x-placement^=\"right\"],\n  &[x-placement^=\"bottom\"],\n  &[x-placement^=\"left\"] {\n    right: auto;\n    bottom: auto;\n  }\n}\n\n// Dividers (basically an `<hr>`) within the dropdown\n.dropdown-divider {\n  @include nav-divider($dropdown-divider-bg, $dropdown-divider-margin-y, true);\n}\n\n// Links, buttons, and more within the dropdown menu\n//\n// `<button>`-specific styles are denoted with `// For <button>s`\n.dropdown-item {\n  display: block;\n  width: 100%; // For `<button>`s\n  padding: $dropdown-item-padding-y $dropdown-item-padding-x;\n  clear: both;\n  font-weight: $font-weight-normal;\n  color: $dropdown-link-color;\n  text-align: inherit; // For `<button>`s\n  white-space: nowrap; // prevent links from randomly breaking onto new lines\n  background-color: transparent; // For `<button>`s\n  border: 0; // For `<button>`s\n\n  // Prevent dropdown overflow if there's no padding\n  // See https://github.com/twbs/bootstrap/pull/27703\n  @if $dropdown-padding-y == 0 {\n    &:first-child {\n      @include border-top-radius($dropdown-inner-border-radius);\n    }\n\n    &:last-child {\n      @include border-bottom-radius($dropdown-inner-border-radius);\n    }\n  }\n\n  @include hover-focus() {\n    color: $dropdown-link-hover-color;\n    text-decoration: none;\n    @include gradient-bg($dropdown-link-hover-bg);\n  }\n\n  &.active,\n  &:active {\n    color: $dropdown-link-active-color;\n    text-decoration: none;\n    @include gradient-bg($dropdown-link-active-bg);\n  }\n\n  &.disabled,\n  &:disabled {\n    color: $dropdown-link-disabled-color;\n    pointer-events: none;\n    background-color: transparent;\n    // Remove CSS gradients if they're enabled\n    @if $enable-gradients {\n      background-image: none;\n    }\n  }\n}\n\n.dropdown-menu.show {\n  display: block;\n}\n\n// Dropdown section headers\n.dropdown-header {\n  display: block;\n  padding: $dropdown-padding-y $dropdown-item-padding-x;\n  margin-bottom: 0; // for use with heading elements\n  @include font-size($font-size-sm);\n  color: $dropdown-header-color;\n  white-space: nowrap; // as with > li > a\n}\n\n// Dropdown text\n.dropdown-item-text {\n  display: block;\n  padding: $dropdown-item-padding-y $dropdown-item-padding-x;\n  color: $dropdown-link-color;\n}\n","@mixin caret-down() {\n  border-top: $caret-width solid;\n  border-right: $caret-width solid transparent;\n  border-bottom: 0;\n  border-left: $caret-width solid transparent;\n}\n\n@mixin caret-up() {\n  border-top: 0;\n  border-right: $caret-width solid transparent;\n  border-bottom: $caret-width solid;\n  border-left: $caret-width solid transparent;\n}\n\n@mixin caret-right() {\n  border-top: $caret-width solid transparent;\n  border-right: 0;\n  border-bottom: $caret-width solid transparent;\n  border-left: $caret-width solid;\n}\n\n@mixin caret-left() {\n  border-top: $caret-width solid transparent;\n  border-right: $caret-width solid;\n  border-bottom: $caret-width solid transparent;\n}\n\n@mixin caret($direction: down) {\n  @if $enable-caret {\n    &::after {\n      display: inline-block;\n      margin-left: $caret-spacing;\n      vertical-align: $caret-vertical-align;\n      content: \"\";\n      @if $direction == down {\n        @include caret-down();\n      } @else if $direction == up {\n        @include caret-up();\n      } @else if $direction == right {\n        @include caret-right();\n      }\n    }\n\n    @if $direction == left {\n      &::after {\n        display: none;\n      }\n\n      &::before {\n        display: inline-block;\n        margin-right: $caret-spacing;\n        vertical-align: $caret-vertical-align;\n        content: \"\";\n        @include caret-left();\n      }\n    }\n\n    &:empty::after {\n      margin-left: 0;\n    }\n  }\n}\n","// Horizontal dividers\n//\n// Dividers (basically an hr) within dropdowns and nav lists\n\n@mixin nav-divider($color: $nav-divider-color, $margin-y: $nav-divider-margin-y, $ignore-warning: false) {\n  height: 0;\n  margin: $margin-y 0;\n  overflow: hidden;\n  border-top: 1px solid $color;\n  @include deprecate(\"The `nav-divider()` mixin\", \"v4.4.0\", \"v5\", $ignore-warning);\n}\n","// stylelint-disable selector-no-qualifying-type\n\n// Make the div behave like a button\n.btn-group,\n.btn-group-vertical {\n  position: relative;\n  display: inline-flex;\n  vertical-align: middle; // match .btn alignment given font-size hack above\n\n  > .btn {\n    position: relative;\n    flex: 1 1 auto;\n\n    // Bring the hover, focused, and \"active\" buttons to the front to overlay\n    // the borders properly\n    @include hover() {\n      z-index: 1;\n    }\n    &:focus,\n    &:active,\n    &.active {\n      z-index: 1;\n    }\n  }\n}\n\n// Optional: Group multiple button groups together for a toolbar\n.btn-toolbar {\n  display: flex;\n  flex-wrap: wrap;\n  justify-content: flex-start;\n\n  .input-group {\n    width: auto;\n  }\n}\n\n.btn-group {\n  // Prevent double borders when buttons are next to each other\n  > .btn:not(:first-child),\n  > .btn-group:not(:first-child) {\n    margin-left: -$btn-border-width;\n  }\n\n  // Reset rounded corners\n  > .btn:not(:last-child):not(.dropdown-toggle),\n  > .btn-group:not(:last-child) > .btn {\n    @include border-right-radius(0);\n  }\n\n  > .btn:not(:first-child),\n  > .btn-group:not(:first-child) > .btn {\n    @include border-left-radius(0);\n  }\n}\n\n// Sizing\n//\n// Remix the default button sizing classes into new ones for easier manipulation.\n\n.btn-group-sm > .btn { @extend .btn-sm; }\n.btn-group-lg > .btn { @extend .btn-lg; }\n\n\n//\n// Split button dropdowns\n//\n\n.dropdown-toggle-split {\n  padding-right: $btn-padding-x * .75;\n  padding-left: $btn-padding-x * .75;\n\n  &::after,\n  .dropup &::after,\n  .dropright &::after {\n    margin-left: 0;\n  }\n\n  .dropleft &::before {\n    margin-right: 0;\n  }\n}\n\n.btn-sm + .dropdown-toggle-split {\n  padding-right: $btn-padding-x-sm * .75;\n  padding-left: $btn-padding-x-sm * .75;\n}\n\n.btn-lg + .dropdown-toggle-split {\n  padding-right: $btn-padding-x-lg * .75;\n  padding-left: $btn-padding-x-lg * .75;\n}\n\n\n// The clickable button for toggling the menu\n// Set the same inset shadow as the :active state\n.btn-group.show .dropdown-toggle {\n  @include box-shadow($btn-active-box-shadow);\n\n  // Show no shadow for `.btn-link` since it has no other button styles.\n  &.btn-link {\n    @include box-shadow(none);\n  }\n}\n\n\n//\n// Vertical button groups\n//\n\n.btn-group-vertical {\n  flex-direction: column;\n  align-items: flex-start;\n  justify-content: center;\n\n  > .btn,\n  > .btn-group {\n    width: 100%;\n  }\n\n  > .btn:not(:first-child),\n  > .btn-group:not(:first-child) {\n    margin-top: -$btn-border-width;\n  }\n\n  // Reset rounded corners\n  > .btn:not(:last-child):not(.dropdown-toggle),\n  > .btn-group:not(:last-child) > .btn {\n    @include border-bottom-radius(0);\n  }\n\n  > .btn:not(:first-child),\n  > .btn-group:not(:first-child) > .btn {\n    @include border-top-radius(0);\n  }\n}\n\n\n// Checkbox and radio options\n//\n// In order to support the browser's form validation feedback, powered by the\n// `required` attribute, we have to \"hide\" the inputs via `clip`. We cannot use\n// `display: none;` or `visibility: hidden;` as that also hides the popover.\n// Simply visually hiding the inputs via `opacity` would leave them clickable in\n// certain cases which is prevented by using `clip` and `pointer-events`.\n// This way, we ensure a DOM element is visible to position the popover from.\n//\n// See https://github.com/twbs/bootstrap/pull/12794 and\n// https://github.com/twbs/bootstrap/pull/14559 for more information.\n\n.btn-group-toggle {\n  > .btn,\n  > .btn-group > .btn {\n    margin-bottom: 0; // Override default `<label>` value\n\n    input[type=\"radio\"],\n    input[type=\"checkbox\"] {\n      position: absolute;\n      clip: rect(0, 0, 0, 0);\n      pointer-events: none;\n    }\n  }\n}\n","// stylelint-disable selector-no-qualifying-type\n\n//\n// Base styles\n//\n\n.input-group {\n  position: relative;\n  display: flex;\n  flex-wrap: wrap; // For form validation feedback\n  align-items: stretch;\n  width: 100%;\n\n  > .form-control,\n  > .form-control-plaintext,\n  > .custom-select,\n  > .custom-file {\n    position: relative; // For focus state's z-index\n    flex: 1 1 0%;\n    min-width: 0; // https://stackoverflow.com/questions/36247140/why-dont-flex-items-shrink-past-content-size\n    margin-bottom: 0;\n\n    + .form-control,\n    + .custom-select,\n    + .custom-file {\n      margin-left: -$input-border-width;\n    }\n  }\n\n  // Bring the \"active\" form control to the top of surrounding elements\n  > .form-control:focus,\n  > .custom-select:focus,\n  > .custom-file .custom-file-input:focus ~ .custom-file-label {\n    z-index: 3;\n  }\n\n  // Bring the custom file input above the label\n  > .custom-file .custom-file-input:focus {\n    z-index: 4;\n  }\n\n  > .form-control,\n  > .custom-select {\n    &:not(:last-child) { @include border-right-radius(0); }\n    &:not(:first-child) { @include border-left-radius(0); }\n  }\n\n  // Custom file inputs have more complex markup, thus requiring different\n  // border-radius overrides.\n  > .custom-file {\n    display: flex;\n    align-items: center;\n\n    &:not(:last-child) .custom-file-label,\n    &:not(:last-child) .custom-file-label::after { @include border-right-radius(0); }\n    &:not(:first-child) .custom-file-label { @include border-left-radius(0); }\n  }\n}\n\n\n// Prepend and append\n//\n// While it requires one extra layer of HTML for each, dedicated prepend and\n// append elements allow us to 1) be less clever, 2) simplify our selectors, and\n// 3) support HTML5 form validation.\n\n.input-group-prepend,\n.input-group-append {\n  display: flex;\n\n  // Ensure buttons are always above inputs for more visually pleasing borders.\n  // This isn't needed for `.input-group-text` since it shares the same border-color\n  // as our inputs.\n  .btn {\n    position: relative;\n    z-index: 2;\n\n    &:focus {\n      z-index: 3;\n    }\n  }\n\n  .btn + .btn,\n  .btn + .input-group-text,\n  .input-group-text + .input-group-text,\n  .input-group-text + .btn {\n    margin-left: -$input-border-width;\n  }\n}\n\n.input-group-prepend { margin-right: -$input-border-width; }\n.input-group-append { margin-left: -$input-border-width; }\n\n\n// Textual addons\n//\n// Serves as a catch-all element for any text or radio/checkbox input you wish\n// to prepend or append to an input.\n\n.input-group-text {\n  display: flex;\n  align-items: center;\n  padding: $input-padding-y $input-padding-x;\n  margin-bottom: 0; // Allow use of <label> elements by overriding our default margin-bottom\n  @include font-size($input-font-size); // Match inputs\n  font-weight: $font-weight-normal;\n  line-height: $input-line-height;\n  color: $input-group-addon-color;\n  text-align: center;\n  white-space: nowrap;\n  background-color: $input-group-addon-bg;\n  border: $input-border-width solid $input-group-addon-border-color;\n  @include border-radius($input-border-radius);\n\n  // Nuke default margins from checkboxes and radios to vertically center within.\n  input[type=\"radio\"],\n  input[type=\"checkbox\"] {\n    margin-top: 0;\n  }\n}\n\n\n// Sizing\n//\n// Remix the default form control sizing classes into new ones for easier\n// manipulation.\n\n.input-group-lg > .form-control:not(textarea),\n.input-group-lg > .custom-select {\n  height: $input-height-lg;\n}\n\n.input-group-lg > .form-control,\n.input-group-lg > .custom-select,\n.input-group-lg > .input-group-prepend > .input-group-text,\n.input-group-lg > .input-group-append > .input-group-text,\n.input-group-lg > .input-group-prepend > .btn,\n.input-group-lg > .input-group-append > .btn {\n  padding: $input-padding-y-lg $input-padding-x-lg;\n  @include font-size($input-font-size-lg);\n  line-height: $input-line-height-lg;\n  @include border-radius($input-border-radius-lg);\n}\n\n.input-group-sm > .form-control:not(textarea),\n.input-group-sm > .custom-select {\n  height: $input-height-sm;\n}\n\n.input-group-sm > .form-control,\n.input-group-sm > .custom-select,\n.input-group-sm > .input-group-prepend > .input-group-text,\n.input-group-sm > .input-group-append > .input-group-text,\n.input-group-sm > .input-group-prepend > .btn,\n.input-group-sm > .input-group-append > .btn {\n  padding: $input-padding-y-sm $input-padding-x-sm;\n  @include font-size($input-font-size-sm);\n  line-height: $input-line-height-sm;\n  @include border-radius($input-border-radius-sm);\n}\n\n.input-group-lg > .custom-select,\n.input-group-sm > .custom-select {\n  padding-right: $custom-select-padding-x + $custom-select-indicator-padding;\n}\n\n\n// Prepend and append rounded corners\n//\n// These rulesets must come after the sizing ones to properly override sm and lg\n// border-radius values when extending. They're more specific than we'd like\n// with the `.input-group >` part, but without it, we cannot override the sizing.\n\n\n.input-group > .input-group-prepend > .btn,\n.input-group > .input-group-prepend > .input-group-text,\n.input-group > .input-group-append:not(:last-child) > .btn,\n.input-group > .input-group-append:not(:last-child) > .input-group-text,\n.input-group > .input-group-append:last-child > .btn:not(:last-child):not(.dropdown-toggle),\n.input-group > .input-group-append:last-child > .input-group-text:not(:last-child) {\n  @include border-right-radius(0);\n}\n\n.input-group > .input-group-append > .btn,\n.input-group > .input-group-append > .input-group-text,\n.input-group > .input-group-prepend:not(:first-child) > .btn,\n.input-group > .input-group-prepend:not(:first-child) > .input-group-text,\n.input-group > .input-group-prepend:first-child > .btn:not(:first-child),\n.input-group > .input-group-prepend:first-child > .input-group-text:not(:first-child) {\n  @include border-left-radius(0);\n}\n","// Embedded icons from Open Iconic.\n// Released under MIT and copyright 2014 Waybury.\n// https://useiconic.com/open\n\n\n// Checkboxes and radios\n//\n// Base class takes care of all the key behavioral aspects.\n\n.custom-control {\n  position: relative;\n  display: block;\n  min-height: $font-size-base * $line-height-base;\n  padding-left: $custom-control-gutter + $custom-control-indicator-size;\n}\n\n.custom-control-inline {\n  display: inline-flex;\n  margin-right: $custom-control-spacer-x;\n}\n\n.custom-control-input {\n  position: absolute;\n  left: 0;\n  z-index: -1; // Put the input behind the label so it doesn't overlay text\n  width: $custom-control-indicator-size;\n  height: ($font-size-base * $line-height-base + $custom-control-indicator-size) / 2;\n  opacity: 0;\n\n  &:checked ~ .custom-control-label::before {\n    color: $custom-control-indicator-checked-color;\n    border-color: $custom-control-indicator-checked-border-color;\n    @include gradient-bg($custom-control-indicator-checked-bg);\n    @include box-shadow($custom-control-indicator-checked-box-shadow);\n  }\n\n  &:focus ~ .custom-control-label::before {\n    // the mixin is not used here to make sure there is feedback\n    @if $enable-shadows {\n      box-shadow: $input-box-shadow, $input-focus-box-shadow;\n    } @else {\n      box-shadow: $custom-control-indicator-focus-box-shadow;\n    }\n  }\n\n  &:focus:not(:checked) ~ .custom-control-label::before {\n    border-color: $custom-control-indicator-focus-border-color;\n  }\n\n  &:not(:disabled):active ~ .custom-control-label::before {\n    color: $custom-control-indicator-active-color;\n    background-color: $custom-control-indicator-active-bg;\n    border-color: $custom-control-indicator-active-border-color;\n    @include box-shadow($custom-control-indicator-active-box-shadow);\n  }\n\n  // Use [disabled] and :disabled to work around https://github.com/twbs/bootstrap/issues/28247\n  &[disabled],\n  &:disabled {\n    ~ .custom-control-label {\n      color: $custom-control-label-disabled-color;\n\n      &::before {\n        background-color: $custom-control-indicator-disabled-bg;\n      }\n    }\n  }\n}\n\n// Custom control indicators\n//\n// Build the custom controls out of pseudo-elements.\n\n.custom-control-label {\n  position: relative;\n  margin-bottom: 0;\n  color: $custom-control-label-color;\n  vertical-align: top;\n  cursor: $custom-control-cursor;\n\n  // Background-color and (when enabled) gradient\n  &::before {\n    position: absolute;\n    top: ($font-size-base * $line-height-base - $custom-control-indicator-size) / 2;\n    left: -($custom-control-gutter + $custom-control-indicator-size);\n    display: block;\n    width: $custom-control-indicator-size;\n    height: $custom-control-indicator-size;\n    pointer-events: none;\n    content: \"\";\n    background-color: $custom-control-indicator-bg;\n    border: $custom-control-indicator-border-color solid $custom-control-indicator-border-width;\n    @include box-shadow($custom-control-indicator-box-shadow);\n  }\n\n  // Foreground (icon)\n  &::after {\n    position: absolute;\n    top: ($font-size-base * $line-height-base - $custom-control-indicator-size) / 2;\n    left: -($custom-control-gutter + $custom-control-indicator-size);\n    display: block;\n    width: $custom-control-indicator-size;\n    height: $custom-control-indicator-size;\n    content: \"\";\n    background: no-repeat 50% / #{$custom-control-indicator-bg-size};\n  }\n}\n\n\n// Checkboxes\n//\n// Tweak just a few things for checkboxes.\n\n.custom-checkbox {\n  .custom-control-label::before {\n    @include border-radius($custom-checkbox-indicator-border-radius);\n  }\n\n  .custom-control-input:checked ~ .custom-control-label {\n    &::after {\n      background-image: escape-svg($custom-checkbox-indicator-icon-checked);\n    }\n  }\n\n  .custom-control-input:indeterminate ~ .custom-control-label {\n    &::before {\n      border-color: $custom-checkbox-indicator-indeterminate-border-color;\n      @include gradient-bg($custom-checkbox-indicator-indeterminate-bg);\n      @include box-shadow($custom-checkbox-indicator-indeterminate-box-shadow);\n    }\n    &::after {\n      background-image: escape-svg($custom-checkbox-indicator-icon-indeterminate);\n    }\n  }\n\n  .custom-control-input:disabled {\n    &:checked ~ .custom-control-label::before {\n      background-color: $custom-control-indicator-checked-disabled-bg;\n    }\n    &:indeterminate ~ .custom-control-label::before {\n      background-color: $custom-control-indicator-checked-disabled-bg;\n    }\n  }\n}\n\n// Radios\n//\n// Tweak just a few things for radios.\n\n.custom-radio {\n  .custom-control-label::before {\n    // stylelint-disable-next-line property-blacklist\n    border-radius: $custom-radio-indicator-border-radius;\n  }\n\n  .custom-control-input:checked ~ .custom-control-label {\n    &::after {\n      background-image: escape-svg($custom-radio-indicator-icon-checked);\n    }\n  }\n\n  .custom-control-input:disabled {\n    &:checked ~ .custom-control-label::before {\n      background-color: $custom-control-indicator-checked-disabled-bg;\n    }\n  }\n}\n\n\n// switches\n//\n// Tweak a few things for switches\n\n.custom-switch {\n  padding-left: $custom-switch-width + $custom-control-gutter;\n\n  .custom-control-label {\n    &::before {\n      left: -($custom-switch-width + $custom-control-gutter);\n      width: $custom-switch-width;\n      pointer-events: all;\n      // stylelint-disable-next-line property-blacklist\n      border-radius: $custom-switch-indicator-border-radius;\n    }\n\n    &::after {\n      top: add(($font-size-base * $line-height-base - $custom-control-indicator-size) / 2, $custom-control-indicator-border-width * 2);\n      left: add(-($custom-switch-width + $custom-control-gutter), $custom-control-indicator-border-width * 2);\n      width: $custom-switch-indicator-size;\n      height: $custom-switch-indicator-size;\n      background-color: $custom-control-indicator-border-color;\n      // stylelint-disable-next-line property-blacklist\n      border-radius: $custom-switch-indicator-border-radius;\n      @include transition(transform .15s ease-in-out, $custom-forms-transition);\n    }\n  }\n\n  .custom-control-input:checked ~ .custom-control-label {\n    &::after {\n      background-color: $custom-control-indicator-bg;\n      transform: translateX($custom-switch-width - $custom-control-indicator-size);\n    }\n  }\n\n  .custom-control-input:disabled {\n    &:checked ~ .custom-control-label::before {\n      background-color: $custom-control-indicator-checked-disabled-bg;\n    }\n  }\n}\n\n\n// Select\n//\n// Replaces the browser default select with a custom one, mostly pulled from\n// https://primer.github.io/.\n//\n\n.custom-select {\n  display: inline-block;\n  width: 100%;\n  height: $custom-select-height;\n  padding: $custom-select-padding-y ($custom-select-padding-x + $custom-select-indicator-padding) $custom-select-padding-y $custom-select-padding-x;\n  font-family: $custom-select-font-family;\n  @include font-size($custom-select-font-size);\n  font-weight: $custom-select-font-weight;\n  line-height: $custom-select-line-height;\n  color: $custom-select-color;\n  vertical-align: middle;\n  background: $custom-select-bg $custom-select-background;\n  border: $custom-select-border-width solid $custom-select-border-color;\n  @include border-radius($custom-select-border-radius, 0);\n  @include box-shadow($custom-select-box-shadow);\n  appearance: none;\n\n  &:focus {\n    border-color: $custom-select-focus-border-color;\n    outline: 0;\n    @if $enable-shadows {\n      box-shadow: $custom-select-box-shadow, $custom-select-focus-box-shadow;\n    } @else {\n      box-shadow: $custom-select-focus-box-shadow;\n    }\n\n    &::-ms-value {\n      // For visual consistency with other platforms/browsers,\n      // suppress the default white text on blue background highlight given to\n      // the selected option text when the (still closed) <select> receives focus\n      // in IE and (under certain conditions) Edge.\n      // See https://github.com/twbs/bootstrap/issues/19398.\n      color: $input-color;\n      background-color: $input-bg;\n    }\n  }\n\n  &[multiple],\n  &[size]:not([size=\"1\"]) {\n    height: auto;\n    padding-right: $custom-select-padding-x;\n    background-image: none;\n  }\n\n  &:disabled {\n    color: $custom-select-disabled-color;\n    background-color: $custom-select-disabled-bg;\n  }\n\n  // Hides the default caret in IE11\n  &::-ms-expand {\n    display: none;\n  }\n\n  // Remove outline from select box in FF\n  &:-moz-focusring {\n    color: transparent;\n    text-shadow: 0 0 0 $custom-select-color;\n  }\n}\n\n.custom-select-sm {\n  height: $custom-select-height-sm;\n  padding-top: $custom-select-padding-y-sm;\n  padding-bottom: $custom-select-padding-y-sm;\n  padding-left: $custom-select-padding-x-sm;\n  @include font-size($custom-select-font-size-sm);\n}\n\n.custom-select-lg {\n  height: $custom-select-height-lg;\n  padding-top: $custom-select-padding-y-lg;\n  padding-bottom: $custom-select-padding-y-lg;\n  padding-left: $custom-select-padding-x-lg;\n  @include font-size($custom-select-font-size-lg);\n}\n\n\n// File\n//\n// Custom file input.\n\n.custom-file {\n  position: relative;\n  display: inline-block;\n  width: 100%;\n  height: $custom-file-height;\n  margin-bottom: 0;\n}\n\n.custom-file-input {\n  position: relative;\n  z-index: 2;\n  width: 100%;\n  height: $custom-file-height;\n  margin: 0;\n  opacity: 0;\n\n  &:focus ~ .custom-file-label {\n    border-color: $custom-file-focus-border-color;\n    box-shadow: $custom-file-focus-box-shadow;\n  }\n\n  // Use [disabled] and :disabled to work around https://github.com/twbs/bootstrap/issues/28247\n  &[disabled] ~ .custom-file-label,\n  &:disabled ~ .custom-file-label {\n    background-color: $custom-file-disabled-bg;\n  }\n\n  @each $lang, $value in $custom-file-text {\n    &:lang(#{$lang}) ~ .custom-file-label::after {\n      content: $value;\n    }\n  }\n\n  ~ .custom-file-label[data-browse]::after {\n    content: attr(data-browse);\n  }\n}\n\n.custom-file-label {\n  position: absolute;\n  top: 0;\n  right: 0;\n  left: 0;\n  z-index: 1;\n  height: $custom-file-height;\n  padding: $custom-file-padding-y $custom-file-padding-x;\n  font-family: $custom-file-font-family;\n  font-weight: $custom-file-font-weight;\n  line-height: $custom-file-line-height;\n  color: $custom-file-color;\n  background-color: $custom-file-bg;\n  border: $custom-file-border-width solid $custom-file-border-color;\n  @include border-radius($custom-file-border-radius);\n  @include box-shadow($custom-file-box-shadow);\n\n  &::after {\n    position: absolute;\n    top: 0;\n    right: 0;\n    bottom: 0;\n    z-index: 3;\n    display: block;\n    height: $custom-file-height-inner;\n    padding: $custom-file-padding-y $custom-file-padding-x;\n    line-height: $custom-file-line-height;\n    color: $custom-file-button-color;\n    content: \"Browse\";\n    @include gradient-bg($custom-file-button-bg);\n    border-left: inherit;\n    @include border-radius(0 $custom-file-border-radius $custom-file-border-radius 0);\n  }\n}\n\n// Range\n//\n// Style range inputs the same across browsers. Vendor-specific rules for pseudo\n// elements cannot be mixed. As such, there are no shared styles for focus or\n// active states on prefixed selectors.\n\n.custom-range {\n  width: 100%;\n  height: add($custom-range-thumb-height, $custom-range-thumb-focus-box-shadow-width * 2);\n  padding: 0; // Need to reset padding\n  background-color: transparent;\n  appearance: none;\n\n  &:focus {\n    outline: none;\n\n    // Pseudo-elements must be split across multiple rulesets to have an effect.\n    // No box-shadow() mixin for focus accessibility.\n    &::-webkit-slider-thumb { box-shadow: $custom-range-thumb-focus-box-shadow; }\n    &::-moz-range-thumb     { box-shadow: $custom-range-thumb-focus-box-shadow; }\n    &::-ms-thumb            { box-shadow: $custom-range-thumb-focus-box-shadow; }\n  }\n\n  &::-moz-focus-outer {\n    border: 0;\n  }\n\n  &::-webkit-slider-thumb {\n    width: $custom-range-thumb-width;\n    height: $custom-range-thumb-height;\n    margin-top: ($custom-range-track-height - $custom-range-thumb-height) / 2; // Webkit specific\n    @include gradient-bg($custom-range-thumb-bg);\n    border: $custom-range-thumb-border;\n    @include border-radius($custom-range-thumb-border-radius);\n    @include box-shadow($custom-range-thumb-box-shadow);\n    @include transition($custom-forms-transition);\n    appearance: none;\n\n    &:active {\n      @include gradient-bg($custom-range-thumb-active-bg);\n    }\n  }\n\n  &::-webkit-slider-runnable-track {\n    width: $custom-range-track-width;\n    height: $custom-range-track-height;\n    color: transparent; // Why?\n    cursor: $custom-range-track-cursor;\n    background-color: $custom-range-track-bg;\n    border-color: transparent;\n    @include border-radius($custom-range-track-border-radius);\n    @include box-shadow($custom-range-track-box-shadow);\n  }\n\n  &::-moz-range-thumb {\n    width: $custom-range-thumb-width;\n    height: $custom-range-thumb-height;\n    @include gradient-bg($custom-range-thumb-bg);\n    border: $custom-range-thumb-border;\n    @include border-radius($custom-range-thumb-border-radius);\n    @include box-shadow($custom-range-thumb-box-shadow);\n    @include transition($custom-forms-transition);\n    appearance: none;\n\n    &:active {\n      @include gradient-bg($custom-range-thumb-active-bg);\n    }\n  }\n\n  &::-moz-range-track {\n    width: $custom-range-track-width;\n    height: $custom-range-track-height;\n    color: transparent;\n    cursor: $custom-range-track-cursor;\n    background-color: $custom-range-track-bg;\n    border-color: transparent; // Firefox specific?\n    @include border-radius($custom-range-track-border-radius);\n    @include box-shadow($custom-range-track-box-shadow);\n  }\n\n  &::-ms-thumb {\n    width: $custom-range-thumb-width;\n    height: $custom-range-thumb-height;\n    margin-top: 0; // Edge specific\n    margin-right: $custom-range-thumb-focus-box-shadow-width; // Workaround that overflowed box-shadow is hidden.\n    margin-left: $custom-range-thumb-focus-box-shadow-width;  // Workaround that overflowed box-shadow is hidden.\n    @include gradient-bg($custom-range-thumb-bg);\n    border: $custom-range-thumb-border;\n    @include border-radius($custom-range-thumb-border-radius);\n    @include box-shadow($custom-range-thumb-box-shadow);\n    @include transition($custom-forms-transition);\n    appearance: none;\n\n    &:active {\n      @include gradient-bg($custom-range-thumb-active-bg);\n    }\n  }\n\n  &::-ms-track {\n    width: $custom-range-track-width;\n    height: $custom-range-track-height;\n    color: transparent;\n    cursor: $custom-range-track-cursor;\n    background-color: transparent;\n    border-color: transparent;\n    border-width: $custom-range-thumb-height / 2;\n    @include box-shadow($custom-range-track-box-shadow);\n  }\n\n  &::-ms-fill-lower {\n    background-color: $custom-range-track-bg;\n    @include border-radius($custom-range-track-border-radius);\n  }\n\n  &::-ms-fill-upper {\n    margin-right: 15px; // arbitrary?\n    background-color: $custom-range-track-bg;\n    @include border-radius($custom-range-track-border-radius);\n  }\n\n  &:disabled {\n    &::-webkit-slider-thumb {\n      background-color: $custom-range-thumb-disabled-bg;\n    }\n\n    &::-webkit-slider-runnable-track {\n      cursor: default;\n    }\n\n    &::-moz-range-thumb {\n      background-color: $custom-range-thumb-disabled-bg;\n    }\n\n    &::-moz-range-track {\n      cursor: default;\n    }\n\n    &::-ms-thumb {\n      background-color: $custom-range-thumb-disabled-bg;\n    }\n  }\n}\n\n.custom-control-label::before,\n.custom-file-label,\n.custom-select {\n  @include transition($custom-forms-transition);\n}\n","// Base class\n//\n// Kickstart any navigation component with a set of style resets. Works with\n// `<nav>`s, `<ul>`s or `<ol>`s.\n\n.nav {\n  display: flex;\n  flex-wrap: wrap;\n  padding-left: 0;\n  margin-bottom: 0;\n  list-style: none;\n}\n\n.nav-link {\n  display: block;\n  padding: $nav-link-padding-y $nav-link-padding-x;\n\n  @include hover-focus() {\n    text-decoration: none;\n  }\n\n  // Disabled state lightens text\n  &.disabled {\n    color: $nav-link-disabled-color;\n    pointer-events: none;\n    cursor: default;\n  }\n}\n\n//\n// Tabs\n//\n\n.nav-tabs {\n  border-bottom: $nav-tabs-border-width solid $nav-tabs-border-color;\n\n  .nav-item {\n    margin-bottom: -$nav-tabs-border-width;\n  }\n\n  .nav-link {\n    border: $nav-tabs-border-width solid transparent;\n    @include border-top-radius($nav-tabs-border-radius);\n\n    @include hover-focus() {\n      border-color: $nav-tabs-link-hover-border-color;\n    }\n\n    &.disabled {\n      color: $nav-link-disabled-color;\n      background-color: transparent;\n      border-color: transparent;\n    }\n  }\n\n  .nav-link.active,\n  .nav-item.show .nav-link {\n    color: $nav-tabs-link-active-color;\n    background-color: $nav-tabs-link-active-bg;\n    border-color: $nav-tabs-link-active-border-color;\n  }\n\n  .dropdown-menu {\n    // Make dropdown border overlap tab border\n    margin-top: -$nav-tabs-border-width;\n    // Remove the top rounded corners here since there is a hard edge above the menu\n    @include border-top-radius(0);\n  }\n}\n\n\n//\n// Pills\n//\n\n.nav-pills {\n  .nav-link {\n    @include border-radius($nav-pills-border-radius);\n  }\n\n  .nav-link.active,\n  .show > .nav-link {\n    color: $nav-pills-link-active-color;\n    background-color: $nav-pills-link-active-bg;\n  }\n}\n\n\n//\n// Justified variants\n//\n\n.nav-fill {\n  .nav-item {\n    flex: 1 1 auto;\n    text-align: center;\n  }\n}\n\n.nav-justified {\n  .nav-item {\n    flex-basis: 0;\n    flex-grow: 1;\n    text-align: center;\n  }\n}\n\n\n// Tabbable tabs\n//\n// Hide tabbable panes to start, show them when `.active`\n\n.tab-content {\n  > .tab-pane {\n    display: none;\n  }\n  > .active {\n    display: block;\n  }\n}\n","// Contents\n//\n// Navbar\n// Navbar brand\n// Navbar nav\n// Navbar text\n// Navbar divider\n// Responsive navbar\n// Navbar position\n// Navbar themes\n\n\n// Navbar\n//\n// Provide a static navbar from which we expand to create full-width, fixed, and\n// other navbar variations.\n\n.navbar {\n  position: relative;\n  display: flex;\n  flex-wrap: wrap; // allow us to do the line break for collapsing content\n  align-items: center;\n  justify-content: space-between; // space out brand from logo\n  padding: $navbar-padding-y $navbar-padding-x;\n\n  // Because flex properties aren't inherited, we need to redeclare these first\n  // few properties so that content nested within behave properly.\n  %container-flex-properties {\n    display: flex;\n    flex-wrap: wrap;\n    align-items: center;\n    justify-content: space-between;\n  }\n\n  .container,\n  .container-fluid {\n    @extend %container-flex-properties;\n  }\n\n  @each $breakpoint, $container-max-width in $container-max-widths {\n    > .container#{breakpoint-infix($breakpoint, $container-max-widths)} {\n      @extend %container-flex-properties;\n    }\n  }\n}\n\n\n// Navbar brand\n//\n// Used for brand, project, or site names.\n\n.navbar-brand {\n  display: inline-block;\n  padding-top: $navbar-brand-padding-y;\n  padding-bottom: $navbar-brand-padding-y;\n  margin-right: $navbar-padding-x;\n  @include font-size($navbar-brand-font-size);\n  line-height: inherit;\n  white-space: nowrap;\n\n  @include hover-focus() {\n    text-decoration: none;\n  }\n}\n\n\n// Navbar nav\n//\n// Custom navbar navigation (doesn't require `.nav`, but does make use of `.nav-link`).\n\n.navbar-nav {\n  display: flex;\n  flex-direction: column; // cannot use `inherit` to get the `.navbar`s value\n  padding-left: 0;\n  margin-bottom: 0;\n  list-style: none;\n\n  .nav-link {\n    padding-right: 0;\n    padding-left: 0;\n  }\n\n  .dropdown-menu {\n    position: static;\n    float: none;\n  }\n}\n\n\n// Navbar text\n//\n//\n\n.navbar-text {\n  display: inline-block;\n  padding-top: $nav-link-padding-y;\n  padding-bottom: $nav-link-padding-y;\n}\n\n\n// Responsive navbar\n//\n// Custom styles for responsive collapsing and toggling of navbar contents.\n// Powered by the collapse Bootstrap JavaScript plugin.\n\n// When collapsed, prevent the toggleable navbar contents from appearing in\n// the default flexbox row orientation. Requires the use of `flex-wrap: wrap`\n// on the `.navbar` parent.\n.navbar-collapse {\n  flex-basis: 100%;\n  flex-grow: 1;\n  // For always expanded or extra full navbars, ensure content aligns itself\n  // properly vertically. Can be easily overridden with flex utilities.\n  align-items: center;\n}\n\n// Button for toggling the navbar when in its collapsed state\n.navbar-toggler {\n  padding: $navbar-toggler-padding-y $navbar-toggler-padding-x;\n  @include font-size($navbar-toggler-font-size);\n  line-height: 1;\n  background-color: transparent; // remove default button style\n  border: $border-width solid transparent; // remove default button style\n  @include border-radius($navbar-toggler-border-radius);\n\n  @include hover-focus() {\n    text-decoration: none;\n  }\n}\n\n// Keep as a separate element so folks can easily override it with another icon\n// or image file as needed.\n.navbar-toggler-icon {\n  display: inline-block;\n  width: 1.5em;\n  height: 1.5em;\n  vertical-align: middle;\n  content: \"\";\n  background: no-repeat center center;\n  background-size: 100% 100%;\n}\n\n// Generate series of `.navbar-expand-*` responsive classes for configuring\n// where your navbar collapses.\n.navbar-expand {\n  @each $breakpoint in map-keys($grid-breakpoints) {\n    $next: breakpoint-next($breakpoint, $grid-breakpoints);\n    $infix: breakpoint-infix($next, $grid-breakpoints);\n\n    &#{$infix} {\n      @include media-breakpoint-down($breakpoint) {\n        %container-navbar-expand-#{$breakpoint} {\n          padding-right: 0;\n          padding-left: 0;\n        }\n\n        > .container,\n        > .container-fluid {\n          @extend %container-navbar-expand-#{$breakpoint};\n        }\n\n        @each $size, $container-max-width in $container-max-widths {\n          > .container#{breakpoint-infix($size, $container-max-widths)} {\n            @extend %container-navbar-expand-#{$breakpoint};\n          }\n        }\n      }\n\n      @include media-breakpoint-up($next) {\n        flex-flow: row nowrap;\n        justify-content: flex-start;\n\n        .navbar-nav {\n          flex-direction: row;\n\n          .dropdown-menu {\n            position: absolute;\n          }\n\n          .nav-link {\n            padding-right: $navbar-nav-link-padding-x;\n            padding-left: $navbar-nav-link-padding-x;\n          }\n        }\n\n        // For nesting containers, have to redeclare for alignment purposes\n        %container-nesting-#{$breakpoint} {\n          flex-wrap: nowrap;\n        }\n\n        > .container,\n        > .container-fluid {\n          @extend %container-nesting-#{$breakpoint};\n        }\n\n        @each $size, $container-max-width in $container-max-widths {\n          > .container#{breakpoint-infix($size, $container-max-widths)} {\n            @extend %container-nesting-#{$breakpoint};\n          }\n        }\n\n        .navbar-collapse {\n          display: flex !important; // stylelint-disable-line declaration-no-important\n\n          // Changes flex-bases to auto because of an IE10 bug\n          flex-basis: auto;\n        }\n\n        .navbar-toggler {\n          display: none;\n        }\n      }\n    }\n  }\n}\n\n\n// Navbar themes\n//\n// Styles for switching between navbars with light or dark background.\n\n// Dark links against a light background\n.navbar-light {\n  .navbar-brand {\n    color: $navbar-light-brand-color;\n\n    @include hover-focus() {\n      color: $navbar-light-brand-hover-color;\n    }\n  }\n\n  .navbar-nav {\n    .nav-link {\n      color: $navbar-light-color;\n\n      @include hover-focus() {\n        color: $navbar-light-hover-color;\n      }\n\n      &.disabled {\n        color: $navbar-light-disabled-color;\n      }\n    }\n\n    .show > .nav-link,\n    .active > .nav-link,\n    .nav-link.show,\n    .nav-link.active {\n      color: $navbar-light-active-color;\n    }\n  }\n\n  .navbar-toggler {\n    color: $navbar-light-color;\n    border-color: $navbar-light-toggler-border-color;\n  }\n\n  .navbar-toggler-icon {\n    background-image: escape-svg($navbar-light-toggler-icon-bg);\n  }\n\n  .navbar-text {\n    color: $navbar-light-color;\n    a {\n      color: $navbar-light-active-color;\n\n      @include hover-focus() {\n        color: $navbar-light-active-color;\n      }\n    }\n  }\n}\n\n// White links against a dark background\n.navbar-dark {\n  .navbar-brand {\n    color: $navbar-dark-brand-color;\n\n    @include hover-focus() {\n      color: $navbar-dark-brand-hover-color;\n    }\n  }\n\n  .navbar-nav {\n    .nav-link {\n      color: $navbar-dark-color;\n\n      @include hover-focus() {\n        color: $navbar-dark-hover-color;\n      }\n\n      &.disabled {\n        color: $navbar-dark-disabled-color;\n      }\n    }\n\n    .show > .nav-link,\n    .active > .nav-link,\n    .nav-link.show,\n    .nav-link.active {\n      color: $navbar-dark-active-color;\n    }\n  }\n\n  .navbar-toggler {\n    color: $navbar-dark-color;\n    border-color: $navbar-dark-toggler-border-color;\n  }\n\n  .navbar-toggler-icon {\n    background-image: escape-svg($navbar-dark-toggler-icon-bg);\n  }\n\n  .navbar-text {\n    color: $navbar-dark-color;\n    a {\n      color: $navbar-dark-active-color;\n\n      @include hover-focus() {\n        color: $navbar-dark-active-color;\n      }\n    }\n  }\n}\n","//\n// Base styles\n//\n\n.card {\n  position: relative;\n  display: flex;\n  flex-direction: column;\n  min-width: 0; // See https://github.com/twbs/bootstrap/pull/22740#issuecomment-305868106\n  height: $card-height;\n  word-wrap: break-word;\n  background-color: $card-bg;\n  background-clip: border-box;\n  border: $card-border-width solid $card-border-color;\n  @include border-radius($card-border-radius);\n\n  > hr {\n    margin-right: 0;\n    margin-left: 0;\n  }\n\n  > .list-group:first-child {\n    .list-group-item:first-child {\n      @include border-top-radius($card-border-radius);\n    }\n  }\n\n  > .list-group:last-child {\n    .list-group-item:last-child {\n      @include border-bottom-radius($card-border-radius);\n    }\n  }\n}\n\n.card-body {\n  // Enable `flex-grow: 1` for decks and groups so that card blocks take up\n  // as much space as possible, ensuring footers are aligned to the bottom.\n  flex: 1 1 auto;\n  // Workaround for the image size bug in IE\n  // See: https://github.com/twbs/bootstrap/pull/28855\n  min-height: 1px;\n  padding: $card-spacer-x;\n  color: $card-color;\n}\n\n.card-title {\n  margin-bottom: $card-spacer-y;\n}\n\n.card-subtitle {\n  margin-top: -$card-spacer-y / 2;\n  margin-bottom: 0;\n}\n\n.card-text:last-child {\n  margin-bottom: 0;\n}\n\n.card-link {\n  @include hover() {\n    text-decoration: none;\n  }\n\n  + .card-link {\n    margin-left: $card-spacer-x;\n  }\n}\n\n//\n// Optional textual caps\n//\n\n.card-header {\n  padding: $card-spacer-y $card-spacer-x;\n  margin-bottom: 0; // Removes the default margin-bottom of <hN>\n  color: $card-cap-color;\n  background-color: $card-cap-bg;\n  border-bottom: $card-border-width solid $card-border-color;\n\n  &:first-child {\n    @include border-radius($card-inner-border-radius $card-inner-border-radius 0 0);\n  }\n\n  + .list-group {\n    .list-group-item:first-child {\n      border-top: 0;\n    }\n  }\n}\n\n.card-footer {\n  padding: $card-spacer-y $card-spacer-x;\n  background-color: $card-cap-bg;\n  border-top: $card-border-width solid $card-border-color;\n\n  &:last-child {\n    @include border-radius(0 0 $card-inner-border-radius $card-inner-border-radius);\n  }\n}\n\n\n//\n// Header navs\n//\n\n.card-header-tabs {\n  margin-right: -$card-spacer-x / 2;\n  margin-bottom: -$card-spacer-y;\n  margin-left: -$card-spacer-x / 2;\n  border-bottom: 0;\n}\n\n.card-header-pills {\n  margin-right: -$card-spacer-x / 2;\n  margin-left: -$card-spacer-x / 2;\n}\n\n// Card image\n.card-img-overlay {\n  position: absolute;\n  top: 0;\n  right: 0;\n  bottom: 0;\n  left: 0;\n  padding: $card-img-overlay-padding;\n}\n\n.card-img,\n.card-img-top,\n.card-img-bottom {\n  flex-shrink: 0; // For IE: https://github.com/twbs/bootstrap/issues/29396\n  width: 100%; // Required because we use flexbox and this inherently applies align-self: stretch\n}\n\n.card-img,\n.card-img-top {\n  @include border-top-radius($card-inner-border-radius);\n}\n\n.card-img,\n.card-img-bottom {\n  @include border-bottom-radius($card-inner-border-radius);\n}\n\n\n// Card deck\n\n.card-deck {\n  .card {\n    margin-bottom: $card-deck-margin;\n  }\n\n  @include media-breakpoint-up(sm) {\n    display: flex;\n    flex-flow: row wrap;\n    margin-right: -$card-deck-margin;\n    margin-left: -$card-deck-margin;\n\n    .card {\n      // Flexbugs #4: https://github.com/philipwalton/flexbugs#flexbug-4\n      flex: 1 0 0%;\n      margin-right: $card-deck-margin;\n      margin-bottom: 0; // Override the default\n      margin-left: $card-deck-margin;\n    }\n  }\n}\n\n\n//\n// Card groups\n//\n\n.card-group {\n  // The child selector allows nested `.card` within `.card-group`\n  // to display properly.\n  > .card {\n    margin-bottom: $card-group-margin;\n  }\n\n  @include media-breakpoint-up(sm) {\n    display: flex;\n    flex-flow: row wrap;\n    // The child selector allows nested `.card` within `.card-group`\n    // to display properly.\n    > .card {\n      // Flexbugs #4: https://github.com/philipwalton/flexbugs#flexbug-4\n      flex: 1 0 0%;\n      margin-bottom: 0;\n\n      + .card {\n        margin-left: 0;\n        border-left: 0;\n      }\n\n      // Handle rounded corners\n      @if $enable-rounded {\n        &:not(:last-child) {\n          @include border-right-radius(0);\n\n          .card-img-top,\n          .card-header {\n            // stylelint-disable-next-line property-blacklist\n            border-top-right-radius: 0;\n          }\n          .card-img-bottom,\n          .card-footer {\n            // stylelint-disable-next-line property-blacklist\n            border-bottom-right-radius: 0;\n          }\n        }\n\n        &:not(:first-child) {\n          @include border-left-radius(0);\n\n          .card-img-top,\n          .card-header {\n            // stylelint-disable-next-line property-blacklist\n            border-top-left-radius: 0;\n          }\n          .card-img-bottom,\n          .card-footer {\n            // stylelint-disable-next-line property-blacklist\n            border-bottom-left-radius: 0;\n          }\n        }\n      }\n    }\n  }\n}\n\n\n//\n// Columns\n//\n\n.card-columns {\n  .card {\n    margin-bottom: $card-columns-margin;\n  }\n\n  @include media-breakpoint-up(sm) {\n    column-count: $card-columns-count;\n    column-gap: $card-columns-gap;\n    orphans: 1;\n    widows: 1;\n\n    .card {\n      display: inline-block; // Don't let them vertically span multiple columns\n      width: 100%; // Don't let their width change\n    }\n  }\n}\n\n\n//\n// Accordion\n//\n\n.accordion {\n  > .card {\n    overflow: hidden;\n\n    &:not(:last-of-type) {\n      border-bottom: 0;\n      @include border-bottom-radius(0);\n    }\n\n    &:not(:first-of-type) {\n      @include border-top-radius(0);\n    }\n\n    > .card-header {\n      @include border-radius(0);\n      margin-bottom: -$card-border-width;\n    }\n  }\n}\n",".breadcrumb {\n  display: flex;\n  flex-wrap: wrap;\n  padding: $breadcrumb-padding-y $breadcrumb-padding-x;\n  margin-bottom: $breadcrumb-margin-bottom;\n  @include font-size($breadcrumb-font-size);\n  list-style: none;\n  background-color: $breadcrumb-bg;\n  @include border-radius($breadcrumb-border-radius);\n}\n\n.breadcrumb-item {\n  // The separator between breadcrumbs (by default, a forward-slash: \"/\")\n  + .breadcrumb-item {\n    padding-left: $breadcrumb-item-padding;\n\n    &::before {\n      display: inline-block; // Suppress underlining of the separator in modern browsers\n      padding-right: $breadcrumb-item-padding;\n      color: $breadcrumb-divider-color;\n      content: escape-svg($breadcrumb-divider);\n    }\n  }\n\n  // IE9-11 hack to properly handle hyperlink underlines for breadcrumbs built\n  // without `<ul>`s. The `::before` pseudo-element generates an element\n  // *within* the .breadcrumb-item and thereby inherits the `text-decoration`.\n  //\n  // To trick IE into suppressing the underline, we give the pseudo-element an\n  // underline and then immediately remove it.\n  + .breadcrumb-item:hover::before {\n    text-decoration: underline;\n  }\n  // stylelint-disable-next-line no-duplicate-selectors\n  + .breadcrumb-item:hover::before {\n    text-decoration: none;\n  }\n\n  &.active {\n    color: $breadcrumb-active-color;\n  }\n}\n",".pagination {\n  display: flex;\n  @include list-unstyled();\n  @include border-radius();\n}\n\n.page-link {\n  position: relative;\n  display: block;\n  padding: $pagination-padding-y $pagination-padding-x;\n  margin-left: -$pagination-border-width;\n  line-height: $pagination-line-height;\n  color: $pagination-color;\n  background-color: $pagination-bg;\n  border: $pagination-border-width solid $pagination-border-color;\n\n  &:hover {\n    z-index: 2;\n    color: $pagination-hover-color;\n    text-decoration: none;\n    background-color: $pagination-hover-bg;\n    border-color: $pagination-hover-border-color;\n  }\n\n  &:focus {\n    z-index: 3;\n    outline: $pagination-focus-outline;\n    box-shadow: $pagination-focus-box-shadow;\n  }\n}\n\n.page-item {\n  &:first-child {\n    .page-link {\n      margin-left: 0;\n      @include border-left-radius($border-radius);\n    }\n  }\n  &:last-child {\n    .page-link {\n      @include border-right-radius($border-radius);\n    }\n  }\n\n  &.active .page-link {\n    z-index: 3;\n    color: $pagination-active-color;\n    background-color: $pagination-active-bg;\n    border-color: $pagination-active-border-color;\n  }\n\n  &.disabled .page-link {\n    color: $pagination-disabled-color;\n    pointer-events: none;\n    // Opinionated: remove the \"hand\" cursor set previously for .page-link\n    cursor: auto;\n    background-color: $pagination-disabled-bg;\n    border-color: $pagination-disabled-border-color;\n  }\n}\n\n\n//\n// Sizing\n//\n\n.pagination-lg {\n  @include pagination-size($pagination-padding-y-lg, $pagination-padding-x-lg, $font-size-lg, $line-height-lg, $border-radius-lg);\n}\n\n.pagination-sm {\n  @include pagination-size($pagination-padding-y-sm, $pagination-padding-x-sm, $font-size-sm, $line-height-sm, $border-radius-sm);\n}\n","// Pagination\n\n@mixin pagination-size($padding-y, $padding-x, $font-size, $line-height, $border-radius) {\n  .page-link {\n    padding: $padding-y $padding-x;\n    @include font-size($font-size);\n    line-height: $line-height;\n  }\n\n  .page-item {\n    &:first-child {\n      .page-link {\n        @include border-left-radius($border-radius);\n      }\n    }\n    &:last-child {\n      .page-link {\n        @include border-right-radius($border-radius);\n      }\n    }\n  }\n}\n","// Base class\n//\n// Requires one of the contextual, color modifier classes for `color` and\n// `background-color`.\n\n.badge {\n  display: inline-block;\n  padding: $badge-padding-y $badge-padding-x;\n  @include font-size($badge-font-size);\n  font-weight: $badge-font-weight;\n  line-height: 1;\n  text-align: center;\n  white-space: nowrap;\n  vertical-align: baseline;\n  @include border-radius($badge-border-radius);\n  @include transition($badge-transition);\n\n  @at-root a#{&} {\n    @include hover-focus() {\n      text-decoration: none;\n    }\n  }\n\n  // Empty badges collapse automatically\n  &:empty {\n    display: none;\n  }\n}\n\n// Quick fix for badges in buttons\n.btn .badge {\n  position: relative;\n  top: -1px;\n}\n\n// Pill badges\n//\n// Make them extra rounded with a modifier to replace v3's badges.\n\n.badge-pill {\n  padding-right: $badge-pill-padding-x;\n  padding-left: $badge-pill-padding-x;\n  @include border-radius($badge-pill-border-radius);\n}\n\n// Colors\n//\n// Contextual variations (linked badges get darker on :hover).\n\n@each $color, $value in $theme-colors {\n  .badge-#{$color} {\n    @include badge-variant($value);\n  }\n}\n","@mixin badge-variant($bg) {\n  color: color-yiq($bg);\n  background-color: $bg;\n\n  @at-root a#{&} {\n    @include hover-focus() {\n      color: color-yiq($bg);\n      background-color: darken($bg, 10%);\n    }\n\n    &:focus,\n    &.focus {\n      outline: 0;\n      box-shadow: 0 0 0 $badge-focus-width rgba($bg, .5);\n    }\n  }\n}\n",".jumbotron {\n  padding: $jumbotron-padding ($jumbotron-padding / 2);\n  margin-bottom: $jumbotron-padding;\n  color: $jumbotron-color;\n  background-color: $jumbotron-bg;\n  @include border-radius($border-radius-lg);\n\n  @include media-breakpoint-up(sm) {\n    padding: ($jumbotron-padding * 2) $jumbotron-padding;\n  }\n}\n\n.jumbotron-fluid {\n  padding-right: 0;\n  padding-left: 0;\n  @include border-radius(0);\n}\n","//\n// Base styles\n//\n\n.alert {\n  position: relative;\n  padding: $alert-padding-y $alert-padding-x;\n  margin-bottom: $alert-margin-bottom;\n  border: $alert-border-width solid transparent;\n  @include border-radius($alert-border-radius);\n}\n\n// Headings for larger alerts\n.alert-heading {\n  // Specified to prevent conflicts of changing $headings-color\n  color: inherit;\n}\n\n// Provide class for links that match alerts\n.alert-link {\n  font-weight: $alert-link-font-weight;\n}\n\n\n// Dismissible alerts\n//\n// Expand the right padding and account for the close button's positioning.\n\n.alert-dismissible {\n  padding-right: $close-font-size + $alert-padding-x * 2;\n\n  // Adjust close link position\n  .close {\n    position: absolute;\n    top: 0;\n    right: 0;\n    padding: $alert-padding-y $alert-padding-x;\n    color: inherit;\n  }\n}\n\n\n// Alternate styles\n//\n// Generate contextual modifier classes for colorizing the alert.\n\n@each $color, $value in $theme-colors {\n  .alert-#{$color} {\n    @include alert-variant(theme-color-level($color, $alert-bg-level), theme-color-level($color, $alert-border-level), theme-color-level($color, $alert-color-level));\n  }\n}\n","@mixin alert-variant($background, $border, $color) {\n  color: $color;\n  @include gradient-bg($background);\n  border-color: $border;\n\n  hr {\n    border-top-color: darken($border, 5%);\n  }\n\n  .alert-link {\n    color: darken($color, 10%);\n  }\n}\n","// Disable animation if transitions are disabled\n@if $enable-transitions {\n  @keyframes progress-bar-stripes {\n    from { background-position: $progress-height 0; }\n    to { background-position: 0 0; }\n  }\n}\n\n.progress {\n  display: flex;\n  height: $progress-height;\n  overflow: hidden; // force rounded corners by cropping it\n  @include font-size($progress-font-size);\n  background-color: $progress-bg;\n  @include border-radius($progress-border-radius);\n  @include box-shadow($progress-box-shadow);\n}\n\n.progress-bar {\n  display: flex;\n  flex-direction: column;\n  justify-content: center;\n  overflow: hidden;\n  color: $progress-bar-color;\n  text-align: center;\n  white-space: nowrap;\n  background-color: $progress-bar-bg;\n  @include transition($progress-bar-transition);\n}\n\n.progress-bar-striped {\n  @include gradient-striped();\n  background-size: $progress-height $progress-height;\n}\n\n@if $enable-transitions {\n  .progress-bar-animated {\n    animation: progress-bar-stripes $progress-bar-animation-timing;\n\n    @if $enable-prefers-reduced-motion-media-query {\n      @media (prefers-reduced-motion: reduce) {\n        animation: none;\n      }\n    }\n  }\n}\n",".media {\n  display: flex;\n  align-items: flex-start;\n}\n\n.media-body {\n  flex: 1;\n}\n","// Base class\n//\n// Easily usable on <ul>, <ol>, or <div>.\n\n.list-group {\n  display: flex;\n  flex-direction: column;\n\n  // No need to set list-style: none; since .list-group-item is block level\n  padding-left: 0; // reset padding because ul and ol\n  margin-bottom: 0;\n}\n\n\n// Interactive list items\n//\n// Use anchor or button elements instead of `li`s or `div`s to create interactive\n// list items. Includes an extra `.active` modifier class for selected items.\n\n.list-group-item-action {\n  width: 100%; // For `<button>`s (anchors become 100% by default though)\n  color: $list-group-action-color;\n  text-align: inherit; // For `<button>`s (anchors inherit)\n\n  // Hover state\n  @include hover-focus() {\n    z-index: 1; // Place hover/focus items above their siblings for proper border styling\n    color: $list-group-action-hover-color;\n    text-decoration: none;\n    background-color: $list-group-hover-bg;\n  }\n\n  &:active {\n    color: $list-group-action-active-color;\n    background-color: $list-group-action-active-bg;\n  }\n}\n\n\n// Individual list items\n//\n// Use on `li`s or `div`s within the `.list-group` parent.\n\n.list-group-item {\n  position: relative;\n  display: block;\n  padding: $list-group-item-padding-y $list-group-item-padding-x;\n  color: $list-group-color;\n  background-color: $list-group-bg;\n  border: $list-group-border-width solid $list-group-border-color;\n\n  &:first-child {\n    @include border-top-radius($list-group-border-radius);\n  }\n\n  &:last-child {\n    @include border-bottom-radius($list-group-border-radius);\n  }\n\n  &.disabled,\n  &:disabled {\n    color: $list-group-disabled-color;\n    pointer-events: none;\n    background-color: $list-group-disabled-bg;\n  }\n\n  // Include both here for `<a>`s and `<button>`s\n  &.active {\n    z-index: 2; // Place active items above their siblings for proper border styling\n    color: $list-group-active-color;\n    background-color: $list-group-active-bg;\n    border-color: $list-group-active-border-color;\n  }\n\n  & + & {\n    border-top-width: 0;\n\n    &.active {\n      margin-top: -$list-group-border-width;\n      border-top-width: $list-group-border-width;\n    }\n  }\n}\n\n\n// Horizontal\n//\n// Change the layout of list group items from vertical (default) to horizontal.\n\n@each $breakpoint in map-keys($grid-breakpoints) {\n  @include media-breakpoint-up($breakpoint) {\n    $infix: breakpoint-infix($breakpoint, $grid-breakpoints);\n\n    .list-group-horizontal#{$infix} {\n      flex-direction: row;\n\n      .list-group-item {\n        &:first-child {\n          @include border-bottom-left-radius($list-group-border-radius);\n          @include border-top-right-radius(0);\n        }\n\n        &:last-child {\n          @include border-top-right-radius($list-group-border-radius);\n          @include border-bottom-left-radius(0);\n        }\n\n        &.active {\n          margin-top: 0;\n        }\n\n        & + .list-group-item {\n          border-top-width: $list-group-border-width;\n          border-left-width: 0;\n\n          &.active {\n            margin-left: -$list-group-border-width;\n            border-left-width: $list-group-border-width;\n          }\n        }\n      }\n    }\n  }\n}\n\n\n// Flush list items\n//\n// Remove borders and border-radius to keep list group items edge-to-edge. Most\n// useful within other components (e.g., cards).\n\n.list-group-flush {\n  .list-group-item {\n    border-right-width: 0;\n    border-left-width: 0;\n    @include border-radius(0);\n\n    &:first-child {\n      border-top-width: 0;\n    }\n  }\n\n  &:last-child {\n    .list-group-item:last-child {\n      border-bottom-width: 0;\n    }\n  }\n}\n\n\n// Contextual variants\n//\n// Add modifier classes to change text and background color on individual items.\n// Organizationally, this must come after the `:hover` states.\n\n@each $color, $value in $theme-colors {\n  @include list-group-item-variant($color, theme-color-level($color, -9), theme-color-level($color, 6));\n}\n","// List Groups\n\n@mixin list-group-item-variant($state, $background, $color) {\n  .list-group-item-#{$state} {\n    color: $color;\n    background-color: $background;\n\n    &.list-group-item-action {\n      @include hover-focus() {\n        color: $color;\n        background-color: darken($background, 5%);\n      }\n\n      &.active {\n        color: $white;\n        background-color: $color;\n        border-color: $color;\n      }\n    }\n  }\n}\n",".close {\n  float: right;\n  @include font-size($close-font-size);\n  font-weight: $close-font-weight;\n  line-height: 1;\n  color: $close-color;\n  text-shadow: $close-text-shadow;\n  opacity: .5;\n\n  // Override <a>'s hover style\n  @include hover() {\n    color: $close-color;\n    text-decoration: none;\n  }\n\n  &:not(:disabled):not(.disabled) {\n    @include hover-focus() {\n      opacity: .75;\n    }\n  }\n}\n\n// Additional properties for button version\n// iOS requires the button element instead of an anchor tag.\n// If you want the anchor version, it requires `href=\"#\"`.\n// See https://developer.mozilla.org/en-US/docs/Web/Events/click#Safari_Mobile\n\n// stylelint-disable-next-line selector-no-qualifying-type\nbutton.close {\n  padding: 0;\n  background-color: transparent;\n  border: 0;\n  appearance: none;\n}\n\n// Future-proof disabling of clicks on `<a>` elements\n\n// stylelint-disable-next-line selector-no-qualifying-type\na.close.disabled {\n  pointer-events: none;\n}\n",".toast {\n  max-width: $toast-max-width;\n  overflow: hidden; // cheap rounded corners on nested items\n  @include font-size($toast-font-size);\n  color: $toast-color;\n  background-color: $toast-background-color;\n  background-clip: padding-box;\n  border: $toast-border-width solid $toast-border-color;\n  box-shadow: $toast-box-shadow;\n  backdrop-filter: blur(10px);\n  opacity: 0;\n  @include border-radius($toast-border-radius);\n\n  &:not(:last-child) {\n    margin-bottom: $toast-padding-x;\n  }\n\n  &.showing {\n    opacity: 1;\n  }\n\n  &.show {\n    display: block;\n    opacity: 1;\n  }\n\n  &.hide {\n    display: none;\n  }\n}\n\n.toast-header {\n  display: flex;\n  align-items: center;\n  padding: $toast-padding-y $toast-padding-x;\n  color: $toast-header-color;\n  background-color: $toast-header-background-color;\n  background-clip: padding-box;\n  border-bottom: $toast-border-width solid $toast-header-border-color;\n}\n\n.toast-body {\n  padding: $toast-padding-x; // apply to both vertical and horizontal\n}\n","// .modal-open      - body class for killing the scroll\n// .modal           - container to scroll within\n// .modal-dialog    - positioning shell for the actual modal\n// .modal-content   - actual modal w/ bg and corners and stuff\n\n\n.modal-open {\n  // Kill the scroll on the body\n  overflow: hidden;\n\n  .modal {\n    overflow-x: hidden;\n    overflow-y: auto;\n  }\n}\n\n// Container that the modal scrolls within\n.modal {\n  position: fixed;\n  top: 0;\n  left: 0;\n  z-index: $zindex-modal;\n  display: none;\n  width: 100%;\n  height: 100%;\n  overflow: hidden;\n  // Prevent Chrome on Windows from adding a focus outline. For details, see\n  // https://github.com/twbs/bootstrap/pull/10951.\n  outline: 0;\n  // We deliberately don't use `-webkit-overflow-scrolling: touch;` due to a\n  // gnarly iOS Safari bug: https://bugs.webkit.org/show_bug.cgi?id=158342\n  // See also https://github.com/twbs/bootstrap/issues/17695\n}\n\n// Shell div to position the modal with bottom padding\n.modal-dialog {\n  position: relative;\n  width: auto;\n  margin: $modal-dialog-margin;\n  // allow clicks to pass through for custom click handling to close modal\n  pointer-events: none;\n\n  // When fading in the modal, animate it to slide down\n  .modal.fade & {\n    @include transition($modal-transition);\n    transform: $modal-fade-transform;\n  }\n  .modal.show & {\n    transform: $modal-show-transform;\n  }\n\n  // When trying to close, animate focus to scale\n  .modal.modal-static & {\n    transform: $modal-scale-transform;\n  }\n}\n\n.modal-dialog-scrollable {\n  display: flex; // IE10/11\n  max-height: subtract(100%, $modal-dialog-margin * 2);\n\n  .modal-content {\n    max-height: subtract(100vh, $modal-dialog-margin * 2); // IE10/11\n    overflow: hidden;\n  }\n\n  .modal-header,\n  .modal-footer {\n    flex-shrink: 0;\n  }\n\n  .modal-body {\n    overflow-y: auto;\n  }\n}\n\n.modal-dialog-centered {\n  display: flex;\n  align-items: center;\n  min-height: subtract(100%, $modal-dialog-margin * 2);\n\n  // Ensure `modal-dialog-centered` extends the full height of the view (IE10/11)\n  &::before {\n    display: block; // IE10\n    height: subtract(100vh, $modal-dialog-margin * 2);\n    content: \"\";\n  }\n\n  // Ensure `.modal-body` shows scrollbar (IE10/11)\n  &.modal-dialog-scrollable {\n    flex-direction: column;\n    justify-content: center;\n    height: 100%;\n\n    .modal-content {\n      max-height: none;\n    }\n\n    &::before {\n      content: none;\n    }\n  }\n}\n\n// Actual modal\n.modal-content {\n  position: relative;\n  display: flex;\n  flex-direction: column;\n  width: 100%; // Ensure `.modal-content` extends the full width of the parent `.modal-dialog`\n  // counteract the pointer-events: none; in the .modal-dialog\n  color: $modal-content-color;\n  pointer-events: auto;\n  background-color: $modal-content-bg;\n  background-clip: padding-box;\n  border: $modal-content-border-width solid $modal-content-border-color;\n  @include border-radius($modal-content-border-radius);\n  @include box-shadow($modal-content-box-shadow-xs);\n  // Remove focus outline from opened modal\n  outline: 0;\n}\n\n// Modal background\n.modal-backdrop {\n  position: fixed;\n  top: 0;\n  left: 0;\n  z-index: $zindex-modal-backdrop;\n  width: 100vw;\n  height: 100vh;\n  background-color: $modal-backdrop-bg;\n\n  // Fade for backdrop\n  &.fade { opacity: 0; }\n  &.show { opacity: $modal-backdrop-opacity; }\n}\n\n// Modal header\n// Top section of the modal w/ title and dismiss\n.modal-header {\n  display: flex;\n  align-items: flex-start; // so the close btn always stays on the upper right corner\n  justify-content: space-between; // Put modal header elements (title and dismiss) on opposite ends\n  padding: $modal-header-padding;\n  border-bottom: $modal-header-border-width solid $modal-header-border-color;\n  @include border-top-radius($modal-content-inner-border-radius);\n\n  .close {\n    padding: $modal-header-padding;\n    // auto on the left force icon to the right even when there is no .modal-title\n    margin: (-$modal-header-padding-y) (-$modal-header-padding-x) (-$modal-header-padding-y) auto;\n  }\n}\n\n// Title text within header\n.modal-title {\n  margin-bottom: 0;\n  line-height: $modal-title-line-height;\n}\n\n// Modal body\n// Where all modal content resides (sibling of .modal-header and .modal-footer)\n.modal-body {\n  position: relative;\n  // Enable `flex-grow: 1` so that the body take up as much space as possible\n  // when there should be a fixed height on `.modal-dialog`.\n  flex: 1 1 auto;\n  padding: $modal-inner-padding;\n}\n\n// Footer (for actions)\n.modal-footer {\n  display: flex;\n  flex-wrap: wrap;\n  align-items: center; // vertically center\n  justify-content: flex-end; // Right align buttons with flex property because text-align doesn't work on flex items\n  padding: $modal-inner-padding - $modal-footer-margin-between / 2;\n  border-top: $modal-footer-border-width solid $modal-footer-border-color;\n  @include border-bottom-radius($modal-content-inner-border-radius);\n\n  // Place margin between footer elements\n  // This solution is far from ideal because of the universal selector usage,\n  // but is needed to fix https://github.com/twbs/bootstrap/issues/24800\n  // stylelint-disable-next-line selector-max-universal\n  > * {\n    margin: $modal-footer-margin-between / 2;\n  }\n}\n\n// Measure scrollbar width for padding body during modal show/hide\n.modal-scrollbar-measure {\n  position: absolute;\n  top: -9999px;\n  width: 50px;\n  height: 50px;\n  overflow: scroll;\n}\n\n// Scale up the modal\n@include media-breakpoint-up(sm) {\n  // Automatically set modal's width for larger viewports\n  .modal-dialog {\n    max-width: $modal-md;\n    margin: $modal-dialog-margin-y-sm-up auto;\n  }\n\n  .modal-dialog-scrollable {\n    max-height: subtract(100%, $modal-dialog-margin-y-sm-up * 2);\n\n    .modal-content {\n      max-height: subtract(100vh, $modal-dialog-margin-y-sm-up * 2);\n    }\n  }\n\n  .modal-dialog-centered {\n    min-height: subtract(100%, $modal-dialog-margin-y-sm-up * 2);\n\n    &::before {\n      height: subtract(100vh, $modal-dialog-margin-y-sm-up * 2);\n    }\n  }\n\n  .modal-content {\n    @include box-shadow($modal-content-box-shadow-sm-up);\n  }\n\n  .modal-sm { max-width: $modal-sm; }\n}\n\n@include media-breakpoint-up(lg) {\n  .modal-lg,\n  .modal-xl {\n    max-width: $modal-lg;\n  }\n}\n\n@include media-breakpoint-up(xl) {\n  .modal-xl { max-width: $modal-xl; }\n}\n","// Base class\n.tooltip {\n  position: absolute;\n  z-index: $zindex-tooltip;\n  display: block;\n  margin: $tooltip-margin;\n  // Our parent element can be arbitrary since tooltips are by default inserted as a sibling of their target element.\n  // So reset our font and text properties to avoid inheriting weird values.\n  @include reset-text();\n  @include font-size($tooltip-font-size);\n  // Allow breaking very long words so they don't overflow the tooltip's bounds\n  word-wrap: break-word;\n  opacity: 0;\n\n  &.show { opacity: $tooltip-opacity; }\n\n  .arrow {\n    position: absolute;\n    display: block;\n    width: $tooltip-arrow-width;\n    height: $tooltip-arrow-height;\n\n    &::before {\n      position: absolute;\n      content: \"\";\n      border-color: transparent;\n      border-style: solid;\n    }\n  }\n}\n\n.bs-tooltip-top {\n  padding: $tooltip-arrow-height 0;\n\n  .arrow {\n    bottom: 0;\n\n    &::before {\n      top: 0;\n      border-width: $tooltip-arrow-height ($tooltip-arrow-width / 2) 0;\n      border-top-color: $tooltip-arrow-color;\n    }\n  }\n}\n\n.bs-tooltip-right {\n  padding: 0 $tooltip-arrow-height;\n\n  .arrow {\n    left: 0;\n    width: $tooltip-arrow-height;\n    height: $tooltip-arrow-width;\n\n    &::before {\n      right: 0;\n      border-width: ($tooltip-arrow-width / 2) $tooltip-arrow-height ($tooltip-arrow-width / 2) 0;\n      border-right-color: $tooltip-arrow-color;\n    }\n  }\n}\n\n.bs-tooltip-bottom {\n  padding: $tooltip-arrow-height 0;\n\n  .arrow {\n    top: 0;\n\n    &::before {\n      bottom: 0;\n      border-width: 0 ($tooltip-arrow-width / 2) $tooltip-arrow-height;\n      border-bottom-color: $tooltip-arrow-color;\n    }\n  }\n}\n\n.bs-tooltip-left {\n  padding: 0 $tooltip-arrow-height;\n\n  .arrow {\n    right: 0;\n    width: $tooltip-arrow-height;\n    height: $tooltip-arrow-width;\n\n    &::before {\n      left: 0;\n      border-width: ($tooltip-arrow-width / 2) 0 ($tooltip-arrow-width / 2) $tooltip-arrow-height;\n      border-left-color: $tooltip-arrow-color;\n    }\n  }\n}\n\n.bs-tooltip-auto {\n  &[x-placement^=\"top\"] {\n    @extend .bs-tooltip-top;\n  }\n  &[x-placement^=\"right\"] {\n    @extend .bs-tooltip-right;\n  }\n  &[x-placement^=\"bottom\"] {\n    @extend .bs-tooltip-bottom;\n  }\n  &[x-placement^=\"left\"] {\n    @extend .bs-tooltip-left;\n  }\n}\n\n// Wrapper for the tooltip content\n.tooltip-inner {\n  max-width: $tooltip-max-width;\n  padding: $tooltip-padding-y $tooltip-padding-x;\n  color: $tooltip-color;\n  text-align: center;\n  background-color: $tooltip-bg;\n  @include border-radius($tooltip-border-radius);\n}\n","@mixin reset-text() {\n  font-family: $font-family-base;\n  // We deliberately do NOT reset font-size or word-wrap.\n  font-style: normal;\n  font-weight: $font-weight-normal;\n  line-height: $line-height-base;\n  text-align: left; // Fallback for where `start` is not supported\n  text-align: start;\n  text-decoration: none;\n  text-shadow: none;\n  text-transform: none;\n  letter-spacing: normal;\n  word-break: normal;\n  word-spacing: normal;\n  white-space: normal;\n  line-break: auto;\n}\n",".popover {\n  position: absolute;\n  top: 0;\n  left: 0;\n  z-index: $zindex-popover;\n  display: block;\n  max-width: $popover-max-width;\n  // Our parent element can be arbitrary since tooltips are by default inserted as a sibling of their target element.\n  // So reset our font and text properties to avoid inheriting weird values.\n  @include reset-text();\n  @include font-size($popover-font-size);\n  // Allow breaking very long words so they don't overflow the popover's bounds\n  word-wrap: break-word;\n  background-color: $popover-bg;\n  background-clip: padding-box;\n  border: $popover-border-width solid $popover-border-color;\n  @include border-radius($popover-border-radius);\n  @include box-shadow($popover-box-shadow);\n\n  .arrow {\n    position: absolute;\n    display: block;\n    width: $popover-arrow-width;\n    height: $popover-arrow-height;\n    margin: 0 $popover-border-radius;\n\n    &::before,\n    &::after {\n      position: absolute;\n      display: block;\n      content: \"\";\n      border-color: transparent;\n      border-style: solid;\n    }\n  }\n}\n\n.bs-popover-top {\n  margin-bottom: $popover-arrow-height;\n\n  > .arrow {\n    bottom: subtract(-$popover-arrow-height, $popover-border-width);\n\n    &::before {\n      bottom: 0;\n      border-width: $popover-arrow-height ($popover-arrow-width / 2) 0;\n      border-top-color: $popover-arrow-outer-color;\n    }\n\n    &::after {\n      bottom: $popover-border-width;\n      border-width: $popover-arrow-height ($popover-arrow-width / 2) 0;\n      border-top-color: $popover-arrow-color;\n    }\n  }\n}\n\n.bs-popover-right {\n  margin-left: $popover-arrow-height;\n\n  > .arrow {\n    left: subtract(-$popover-arrow-height, $popover-border-width);\n    width: $popover-arrow-height;\n    height: $popover-arrow-width;\n    margin: $popover-border-radius 0; // make sure the arrow does not touch the popover's rounded corners\n\n    &::before {\n      left: 0;\n      border-width: ($popover-arrow-width / 2) $popover-arrow-height ($popover-arrow-width / 2) 0;\n      border-right-color: $popover-arrow-outer-color;\n    }\n\n    &::after {\n      left: $popover-border-width;\n      border-width: ($popover-arrow-width / 2) $popover-arrow-height ($popover-arrow-width / 2) 0;\n      border-right-color: $popover-arrow-color;\n    }\n  }\n}\n\n.bs-popover-bottom {\n  margin-top: $popover-arrow-height;\n\n  > .arrow {\n    top: subtract(-$popover-arrow-height, $popover-border-width);\n\n    &::before {\n      top: 0;\n      border-width: 0 ($popover-arrow-width / 2) $popover-arrow-height ($popover-arrow-width / 2);\n      border-bottom-color: $popover-arrow-outer-color;\n    }\n\n    &::after {\n      top: $popover-border-width;\n      border-width: 0 ($popover-arrow-width / 2) $popover-arrow-height ($popover-arrow-width / 2);\n      border-bottom-color: $popover-arrow-color;\n    }\n  }\n\n  // This will remove the popover-header's border just below the arrow\n  .popover-header::before {\n    position: absolute;\n    top: 0;\n    left: 50%;\n    display: block;\n    width: $popover-arrow-width;\n    margin-left: -$popover-arrow-width / 2;\n    content: \"\";\n    border-bottom: $popover-border-width solid $popover-header-bg;\n  }\n}\n\n.bs-popover-left {\n  margin-right: $popover-arrow-height;\n\n  > .arrow {\n    right: subtract(-$popover-arrow-height, $popover-border-width);\n    width: $popover-arrow-height;\n    height: $popover-arrow-width;\n    margin: $popover-border-radius 0; // make sure the arrow does not touch the popover's rounded corners\n\n    &::before {\n      right: 0;\n      border-width: ($popover-arrow-width / 2) 0 ($popover-arrow-width / 2) $popover-arrow-height;\n      border-left-color: $popover-arrow-outer-color;\n    }\n\n    &::after {\n      right: $popover-border-width;\n      border-width: ($popover-arrow-width / 2) 0 ($popover-arrow-width / 2) $popover-arrow-height;\n      border-left-color: $popover-arrow-color;\n    }\n  }\n}\n\n.bs-popover-auto {\n  &[x-placement^=\"top\"] {\n    @extend .bs-popover-top;\n  }\n  &[x-placement^=\"right\"] {\n    @extend .bs-popover-right;\n  }\n  &[x-placement^=\"bottom\"] {\n    @extend .bs-popover-bottom;\n  }\n  &[x-placement^=\"left\"] {\n    @extend .bs-popover-left;\n  }\n}\n\n\n// Offset the popover to account for the popover arrow\n.popover-header {\n  padding: $popover-header-padding-y $popover-header-padding-x;\n  margin-bottom: 0; // Reset the default from Reboot\n  @include font-size($font-size-base);\n  color: $popover-header-color;\n  background-color: $popover-header-bg;\n  border-bottom: $popover-border-width solid darken($popover-header-bg, 5%);\n  @include border-top-radius($popover-inner-border-radius);\n\n  &:empty {\n    display: none;\n  }\n}\n\n.popover-body {\n  padding: $popover-body-padding-y $popover-body-padding-x;\n  color: $popover-body-color;\n}\n","// Notes on the classes:\n//\n// 1. .carousel.pointer-event should ideally be pan-y (to allow for users to scroll vertically)\n//    even when their scroll action started on a carousel, but for compatibility (with Firefox)\n//    we're preventing all actions instead\n// 2. The .carousel-item-left and .carousel-item-right is used to indicate where\n//    the active slide is heading.\n// 3. .active.carousel-item is the current slide.\n// 4. .active.carousel-item-left and .active.carousel-item-right is the current\n//    slide in its in-transition state. Only one of these occurs at a time.\n// 5. .carousel-item-next.carousel-item-left and .carousel-item-prev.carousel-item-right\n//    is the upcoming slide in transition.\n\n.carousel {\n  position: relative;\n}\n\n.carousel.pointer-event {\n  touch-action: pan-y;\n}\n\n.carousel-inner {\n  position: relative;\n  width: 100%;\n  overflow: hidden;\n  @include clearfix();\n}\n\n.carousel-item {\n  position: relative;\n  display: none;\n  float: left;\n  width: 100%;\n  margin-right: -100%;\n  backface-visibility: hidden;\n  @include transition($carousel-transition);\n}\n\n.carousel-item.active,\n.carousel-item-next,\n.carousel-item-prev {\n  display: block;\n}\n\n.carousel-item-next:not(.carousel-item-left),\n.active.carousel-item-right {\n  transform: translateX(100%);\n}\n\n.carousel-item-prev:not(.carousel-item-right),\n.active.carousel-item-left {\n  transform: translateX(-100%);\n}\n\n\n//\n// Alternate transitions\n//\n\n.carousel-fade {\n  .carousel-item {\n    opacity: 0;\n    transition-property: opacity;\n    transform: none;\n  }\n\n  .carousel-item.active,\n  .carousel-item-next.carousel-item-left,\n  .carousel-item-prev.carousel-item-right {\n    z-index: 1;\n    opacity: 1;\n  }\n\n  .active.carousel-item-left,\n  .active.carousel-item-right {\n    z-index: 0;\n    opacity: 0;\n    @include transition(opacity 0s $carousel-transition-duration);\n  }\n}\n\n\n//\n// Left/right controls for nav\n//\n\n.carousel-control-prev,\n.carousel-control-next {\n  position: absolute;\n  top: 0;\n  bottom: 0;\n  z-index: 1;\n  // Use flex for alignment (1-3)\n  display: flex; // 1. allow flex styles\n  align-items: center; // 2. vertically center contents\n  justify-content: center; // 3. horizontally center contents\n  width: $carousel-control-width;\n  color: $carousel-control-color;\n  text-align: center;\n  opacity: $carousel-control-opacity;\n  @include transition($carousel-control-transition);\n\n  // Hover/focus state\n  @include hover-focus() {\n    color: $carousel-control-color;\n    text-decoration: none;\n    outline: 0;\n    opacity: $carousel-control-hover-opacity;\n  }\n}\n.carousel-control-prev {\n  left: 0;\n  @if $enable-gradients {\n    background-image: linear-gradient(90deg, rgba($black, .25), rgba($black, .001));\n  }\n}\n.carousel-control-next {\n  right: 0;\n  @if $enable-gradients {\n    background-image: linear-gradient(270deg, rgba($black, .25), rgba($black, .001));\n  }\n}\n\n// Icons for within\n.carousel-control-prev-icon,\n.carousel-control-next-icon {\n  display: inline-block;\n  width: $carousel-control-icon-width;\n  height: $carousel-control-icon-width;\n  background: no-repeat 50% / 100% 100%;\n}\n.carousel-control-prev-icon {\n  background-image: escape-svg($carousel-control-prev-icon-bg);\n}\n.carousel-control-next-icon {\n  background-image: escape-svg($carousel-control-next-icon-bg);\n}\n\n\n// Optional indicator pips\n//\n// Add an ordered list with the following class and add a list item for each\n// slide your carousel holds.\n\n.carousel-indicators {\n  position: absolute;\n  right: 0;\n  bottom: 0;\n  left: 0;\n  z-index: 15;\n  display: flex;\n  justify-content: center;\n  padding-left: 0; // override <ol> default\n  // Use the .carousel-control's width as margin so we don't overlay those\n  margin-right: $carousel-control-width;\n  margin-left: $carousel-control-width;\n  list-style: none;\n\n  li {\n    box-sizing: content-box;\n    flex: 0 1 auto;\n    width: $carousel-indicator-width;\n    height: $carousel-indicator-height;\n    margin-right: $carousel-indicator-spacer;\n    margin-left: $carousel-indicator-spacer;\n    text-indent: -999px;\n    cursor: pointer;\n    background-color: $carousel-indicator-active-bg;\n    background-clip: padding-box;\n    // Use transparent borders to increase the hit area by 10px on top and bottom.\n    border-top: $carousel-indicator-hit-area-height solid transparent;\n    border-bottom: $carousel-indicator-hit-area-height solid transparent;\n    opacity: .5;\n    @include transition($carousel-indicator-transition);\n  }\n\n  .active {\n    opacity: 1;\n  }\n}\n\n\n// Optional captions\n//\n//\n\n.carousel-caption {\n  position: absolute;\n  right: (100% - $carousel-caption-width) / 2;\n  bottom: 20px;\n  left: (100% - $carousel-caption-width) / 2;\n  z-index: 10;\n  padding-top: 20px;\n  padding-bottom: 20px;\n  color: $carousel-caption-color;\n  text-align: center;\n}\n","@mixin clearfix() {\n  &::after {\n    display: block;\n    clear: both;\n    content: \"\";\n  }\n}\n","//\n// Rotating border\n//\n\n@keyframes spinner-border {\n  to { transform: rotate(360deg); }\n}\n\n.spinner-border {\n  display: inline-block;\n  width: $spinner-width;\n  height: $spinner-height;\n  vertical-align: text-bottom;\n  border: $spinner-border-width solid currentColor;\n  border-right-color: transparent;\n  // stylelint-disable-next-line property-blacklist\n  border-radius: 50%;\n  animation: spinner-border .75s linear infinite;\n}\n\n.spinner-border-sm {\n  width: $spinner-width-sm;\n  height: $spinner-height-sm;\n  border-width: $spinner-border-width-sm;\n}\n\n//\n// Growing circle\n//\n\n@keyframes spinner-grow {\n  0% {\n    transform: scale(0);\n  }\n  50% {\n    opacity: 1;\n  }\n}\n\n.spinner-grow {\n  display: inline-block;\n  width: $spinner-width;\n  height: $spinner-height;\n  vertical-align: text-bottom;\n  background-color: currentColor;\n  // stylelint-disable-next-line property-blacklist\n  border-radius: 50%;\n  opacity: 0;\n  animation: spinner-grow .75s linear infinite;\n}\n\n.spinner-grow-sm {\n  width: $spinner-width-sm;\n  height: $spinner-height-sm;\n}\n","// stylelint-disable declaration-no-important\n\n.align-baseline    { vertical-align: baseline !important; } // Browser default\n.align-top         { vertical-align: top !important; }\n.align-middle      { vertical-align: middle !important; }\n.align-bottom      { vertical-align: bottom !important; }\n.align-text-bottom { vertical-align: text-bottom !important; }\n.align-text-top    { vertical-align: text-top !important; }\n","// stylelint-disable declaration-no-important\n\n// Contextual backgrounds\n\n@mixin bg-variant($parent, $color, $ignore-warning: false) {\n  #{$parent} {\n    background-color: $color !important;\n  }\n  a#{$parent},\n  button#{$parent} {\n    @include hover-focus() {\n      background-color: darken($color, 10%) !important;\n    }\n  }\n  @include deprecate(\"The `bg-variant` mixin\", \"v4.4.0\", \"v5\", $ignore-warning);\n}\n\n@mixin bg-gradient-variant($parent, $color) {\n  #{$parent} {\n    background: $color linear-gradient(180deg, mix($body-bg, $color, 15%), $color) repeat-x !important;\n  }\n}\n","// stylelint-disable declaration-no-important\n\n@each $color, $value in $theme-colors {\n  @include bg-variant(\".bg-#{$color}\", $value, true);\n}\n\n@if $enable-gradients {\n  @each $color, $value in $theme-colors {\n    @include bg-gradient-variant(\".bg-gradient-#{$color}\", $value);\n  }\n}\n\n.bg-white {\n  background-color: $white !important;\n}\n\n.bg-transparent {\n  background-color: transparent !important;\n}\n","// stylelint-disable property-blacklist, declaration-no-important\n\n//\n// Border\n//\n\n.border         { border: $border-width solid $border-color !important; }\n.border-top     { border-top: $border-width solid $border-color !important; }\n.border-right   { border-right: $border-width solid $border-color !important; }\n.border-bottom  { border-bottom: $border-width solid $border-color !important; }\n.border-left    { border-left: $border-width solid $border-color !important; }\n\n.border-0        { border: 0 !important; }\n.border-top-0    { border-top: 0 !important; }\n.border-right-0  { border-right: 0 !important; }\n.border-bottom-0 { border-bottom: 0 !important; }\n.border-left-0   { border-left: 0 !important; }\n\n@each $color, $value in $theme-colors {\n  .border-#{$color} {\n    border-color: $value !important;\n  }\n}\n\n.border-white {\n  border-color: $white !important;\n}\n\n//\n// Border-radius\n//\n\n.rounded-sm {\n  border-radius: $border-radius-sm !important;\n}\n\n.rounded {\n  border-radius: $border-radius !important;\n}\n\n.rounded-top {\n  border-top-left-radius: $border-radius !important;\n  border-top-right-radius: $border-radius !important;\n}\n\n.rounded-right {\n  border-top-right-radius: $border-radius !important;\n  border-bottom-right-radius: $border-radius !important;\n}\n\n.rounded-bottom {\n  border-bottom-right-radius: $border-radius !important;\n  border-bottom-left-radius: $border-radius !important;\n}\n\n.rounded-left {\n  border-top-left-radius: $border-radius !important;\n  border-bottom-left-radius: $border-radius !important;\n}\n\n.rounded-lg {\n  border-radius: $border-radius-lg !important;\n}\n\n.rounded-circle {\n  border-radius: 50% !important;\n}\n\n.rounded-pill {\n  border-radius: $rounded-pill !important;\n}\n\n.rounded-0 {\n  border-radius: 0 !important;\n}\n","// stylelint-disable declaration-no-important\n\n//\n// Utilities for common `display` values\n//\n\n@each $breakpoint in map-keys($grid-breakpoints) {\n  @include media-breakpoint-up($breakpoint) {\n    $infix: breakpoint-infix($breakpoint, $grid-breakpoints);\n\n    @each $value in $displays {\n      .d#{$infix}-#{$value} { display: $value !important; }\n    }\n  }\n}\n\n\n//\n// Utilities for toggling `display` in print\n//\n\n@media print {\n  @each $value in $displays {\n    .d-print-#{$value} { display: $value !important; }\n  }\n}\n","// Credit: Nicolas Gallagher and SUIT CSS.\n\n.embed-responsive {\n  position: relative;\n  display: block;\n  width: 100%;\n  padding: 0;\n  overflow: hidden;\n\n  &::before {\n    display: block;\n    content: \"\";\n  }\n\n  .embed-responsive-item,\n  iframe,\n  embed,\n  object,\n  video {\n    position: absolute;\n    top: 0;\n    bottom: 0;\n    left: 0;\n    width: 100%;\n    height: 100%;\n    border: 0;\n  }\n}\n\n@each $embed-responsive-aspect-ratio in $embed-responsive-aspect-ratios {\n  $embed-responsive-aspect-ratio-x: nth($embed-responsive-aspect-ratio, 1);\n  $embed-responsive-aspect-ratio-y: nth($embed-responsive-aspect-ratio, 2);\n\n  .embed-responsive-#{$embed-responsive-aspect-ratio-x}by#{$embed-responsive-aspect-ratio-y} {\n    &::before {\n      padding-top: percentage($embed-responsive-aspect-ratio-y / $embed-responsive-aspect-ratio-x);\n    }\n  }\n}\n","// stylelint-disable declaration-no-important\n\n// Flex variation\n//\n// Custom styles for additional flex alignment options.\n\n@each $breakpoint in map-keys($grid-breakpoints) {\n  @include media-breakpoint-up($breakpoint) {\n    $infix: breakpoint-infix($breakpoint, $grid-breakpoints);\n\n    .flex#{$infix}-row            { flex-direction: row !important; }\n    .flex#{$infix}-column         { flex-direction: column !important; }\n    .flex#{$infix}-row-reverse    { flex-direction: row-reverse !important; }\n    .flex#{$infix}-column-reverse { flex-direction: column-reverse !important; }\n\n    .flex#{$infix}-wrap         { flex-wrap: wrap !important; }\n    .flex#{$infix}-nowrap       { flex-wrap: nowrap !important; }\n    .flex#{$infix}-wrap-reverse { flex-wrap: wrap-reverse !important; }\n    .flex#{$infix}-fill         { flex: 1 1 auto !important; }\n    .flex#{$infix}-grow-0       { flex-grow: 0 !important; }\n    .flex#{$infix}-grow-1       { flex-grow: 1 !important; }\n    .flex#{$infix}-shrink-0     { flex-shrink: 0 !important; }\n    .flex#{$infix}-shrink-1     { flex-shrink: 1 !important; }\n\n    .justify-content#{$infix}-start   { justify-content: flex-start !important; }\n    .justify-content#{$infix}-end     { justify-content: flex-end !important; }\n    .justify-content#{$infix}-center  { justify-content: center !important; }\n    .justify-content#{$infix}-between { justify-content: space-between !important; }\n    .justify-content#{$infix}-around  { justify-content: space-around !important; }\n\n    .align-items#{$infix}-start    { align-items: flex-start !important; }\n    .align-items#{$infix}-end      { align-items: flex-end !important; }\n    .align-items#{$infix}-center   { align-items: center !important; }\n    .align-items#{$infix}-baseline { align-items: baseline !important; }\n    .align-items#{$infix}-stretch  { align-items: stretch !important; }\n\n    .align-content#{$infix}-start   { align-content: flex-start !important; }\n    .align-content#{$infix}-end     { align-content: flex-end !important; }\n    .align-content#{$infix}-center  { align-content: center !important; }\n    .align-content#{$infix}-between { align-content: space-between !important; }\n    .align-content#{$infix}-around  { align-content: space-around !important; }\n    .align-content#{$infix}-stretch { align-content: stretch !important; }\n\n    .align-self#{$infix}-auto     { align-self: auto !important; }\n    .align-self#{$infix}-start    { align-self: flex-start !important; }\n    .align-self#{$infix}-end      { align-self: flex-end !important; }\n    .align-self#{$infix}-center   { align-self: center !important; }\n    .align-self#{$infix}-baseline { align-self: baseline !important; }\n    .align-self#{$infix}-stretch  { align-self: stretch !important; }\n  }\n}\n","// stylelint-disable declaration-no-important\n\n@each $breakpoint in map-keys($grid-breakpoints) {\n  @include media-breakpoint-up($breakpoint) {\n    $infix: breakpoint-infix($breakpoint, $grid-breakpoints);\n\n    .float#{$infix}-left  { float: left !important; }\n    .float#{$infix}-right { float: right !important; }\n    .float#{$infix}-none  { float: none !important; }\n  }\n}\n","// stylelint-disable declaration-no-important\n\n@each $value in $overflows {\n  .overflow-#{$value} { overflow: $value !important; }\n}\n","// stylelint-disable declaration-no-important\n\n// Common values\n@each $position in $positions {\n  .position-#{$position} { position: $position !important; }\n}\n\n// Shorthand\n\n.fixed-top {\n  position: fixed;\n  top: 0;\n  right: 0;\n  left: 0;\n  z-index: $zindex-fixed;\n}\n\n.fixed-bottom {\n  position: fixed;\n  right: 0;\n  bottom: 0;\n  left: 0;\n  z-index: $zindex-fixed;\n}\n\n.sticky-top {\n  @supports (position: sticky) {\n    position: sticky;\n    top: 0;\n    z-index: $zindex-sticky;\n  }\n}\n","//\n// Screenreaders\n//\n\n.sr-only {\n  @include sr-only();\n}\n\n.sr-only-focusable {\n  @include sr-only-focusable();\n}\n","// Only display content to screen readers\n//\n// See: https://a11yproject.com/posts/how-to-hide-content/\n// See: https://hugogiraudel.com/2016/10/13/css-hide-and-seek/\n\n@mixin sr-only() {\n  position: absolute;\n  width: 1px;\n  height: 1px;\n  padding: 0;\n  margin: -1px; // Fix for https://github.com/twbs/bootstrap/issues/25686\n  overflow: hidden;\n  clip: rect(0, 0, 0, 0);\n  white-space: nowrap;\n  border: 0;\n}\n\n// Use in conjunction with .sr-only to only display content when it's focused.\n//\n// Useful for \"Skip to main content\" links; see https://www.w3.org/TR/2013/NOTE-WCAG20-TECHS-20130905/G1\n//\n// Credit: HTML5 Boilerplate\n\n@mixin sr-only-focusable() {\n  &:active,\n  &:focus {\n    position: static;\n    width: auto;\n    height: auto;\n    overflow: visible;\n    clip: auto;\n    white-space: normal;\n  }\n}\n","// stylelint-disable declaration-no-important\n\n.shadow-sm { box-shadow: $box-shadow-sm !important; }\n.shadow { box-shadow: $box-shadow !important; }\n.shadow-lg { box-shadow: $box-shadow-lg !important; }\n.shadow-none { box-shadow: none !important; }\n","// stylelint-disable declaration-no-important\n\n// Width and height\n\n@each $prop, $abbrev in (width: w, height: h) {\n  @each $size, $length in $sizes {\n    .#{$abbrev}-#{$size} { #{$prop}: $length !important; }\n  }\n}\n\n.mw-100 { max-width: 100% !important; }\n.mh-100 { max-height: 100% !important; }\n\n// Viewport additional helpers\n\n.min-vw-100 { min-width: 100vw !important; }\n.min-vh-100 { min-height: 100vh !important; }\n\n.vw-100 { width: 100vw !important; }\n.vh-100 { height: 100vh !important; }\n","//\n// Stretched link\n//\n\n.stretched-link {\n  &::after {\n    position: absolute;\n    top: 0;\n    right: 0;\n    bottom: 0;\n    left: 0;\n    z-index: 1;\n    // Just in case `pointer-events: none` is set on a parent\n    pointer-events: auto;\n    content: \"\";\n    // IE10 bugfix, see https://stackoverflow.com/questions/16947967/ie10-hover-pseudo-class-doesnt-work-without-background-color\n    background-color: rgba(0, 0, 0, 0);\n  }\n}\n","// stylelint-disable declaration-no-important\n\n// Margin and Padding\n\n@each $breakpoint in map-keys($grid-breakpoints) {\n  @include media-breakpoint-up($breakpoint) {\n    $infix: breakpoint-infix($breakpoint, $grid-breakpoints);\n\n    @each $prop, $abbrev in (margin: m, padding: p) {\n      @each $size, $length in $spacers {\n        .#{$abbrev}#{$infix}-#{$size} { #{$prop}: $length !important; }\n        .#{$abbrev}t#{$infix}-#{$size},\n        .#{$abbrev}y#{$infix}-#{$size} {\n          #{$prop}-top: $length !important;\n        }\n        .#{$abbrev}r#{$infix}-#{$size},\n        .#{$abbrev}x#{$infix}-#{$size} {\n          #{$prop}-right: $length !important;\n        }\n        .#{$abbrev}b#{$infix}-#{$size},\n        .#{$abbrev}y#{$infix}-#{$size} {\n          #{$prop}-bottom: $length !important;\n        }\n        .#{$abbrev}l#{$infix}-#{$size},\n        .#{$abbrev}x#{$infix}-#{$size} {\n          #{$prop}-left: $length !important;\n        }\n      }\n    }\n\n    // Negative margins (e.g., where `.mb-n1` is negative version of `.mb-1`)\n    @each $size, $length in $spacers {\n      @if $size != 0 {\n        .m#{$infix}-n#{$size} { margin: -$length !important; }\n        .mt#{$infix}-n#{$size},\n        .my#{$infix}-n#{$size} {\n          margin-top: -$length !important;\n        }\n        .mr#{$infix}-n#{$size},\n        .mx#{$infix}-n#{$size} {\n          margin-right: -$length !important;\n        }\n        .mb#{$infix}-n#{$size},\n        .my#{$infix}-n#{$size} {\n          margin-bottom: -$length !important;\n        }\n        .ml#{$infix}-n#{$size},\n        .mx#{$infix}-n#{$size} {\n          margin-left: -$length !important;\n        }\n      }\n    }\n\n    // Some special margin utils\n    .m#{$infix}-auto { margin: auto !important; }\n    .mt#{$infix}-auto,\n    .my#{$infix}-auto {\n      margin-top: auto !important;\n    }\n    .mr#{$infix}-auto,\n    .mx#{$infix}-auto {\n      margin-right: auto !important;\n    }\n    .mb#{$infix}-auto,\n    .my#{$infix}-auto {\n      margin-bottom: auto !important;\n    }\n    .ml#{$infix}-auto,\n    .mx#{$infix}-auto {\n      margin-left: auto !important;\n    }\n  }\n}\n","// stylelint-disable declaration-no-important\n\n//\n// Text\n//\n\n.text-monospace { font-family: $font-family-monospace !important; }\n\n// Alignment\n\n.text-justify  { text-align: justify !important; }\n.text-wrap     { white-space: normal !important; }\n.text-nowrap   { white-space: nowrap !important; }\n.text-truncate { @include text-truncate(); }\n\n// Responsive alignment\n\n@each $breakpoint in map-keys($grid-breakpoints) {\n  @include media-breakpoint-up($breakpoint) {\n    $infix: breakpoint-infix($breakpoint, $grid-breakpoints);\n\n    .text#{$infix}-left   { text-align: left !important; }\n    .text#{$infix}-right  { text-align: right !important; }\n    .text#{$infix}-center { text-align: center !important; }\n  }\n}\n\n// Transformation\n\n.text-lowercase  { text-transform: lowercase !important; }\n.text-uppercase  { text-transform: uppercase !important; }\n.text-capitalize { text-transform: capitalize !important; }\n\n// Weight and italics\n\n.font-weight-light   { font-weight: $font-weight-light !important; }\n.font-weight-lighter { font-weight: $font-weight-lighter !important; }\n.font-weight-normal  { font-weight: $font-weight-normal !important; }\n.font-weight-bold    { font-weight: $font-weight-bold !important; }\n.font-weight-bolder  { font-weight: $font-weight-bolder !important; }\n.font-italic         { font-style: italic !important; }\n\n// Contextual colors\n\n.text-white { color: $white !important; }\n\n@each $color, $value in $theme-colors {\n  @include text-emphasis-variant(\".text-#{$color}\", $value, true);\n}\n\n.text-body { color: $body-color !important; }\n.text-muted { color: $text-muted !important; }\n\n.text-black-50 { color: rgba($black, .5) !important; }\n.text-white-50 { color: rgba($white, .5) !important; }\n\n// Misc\n\n.text-hide {\n  @include text-hide($ignore-warning: true);\n}\n\n.text-decoration-none { text-decoration: none !important; }\n\n.text-break {\n  word-break: break-word !important; // IE & < Edge 18\n  overflow-wrap: break-word !important;\n}\n\n// Reset\n\n.text-reset { color: inherit !important; }\n","// Text truncate\n// Requires inline-block or block for proper styling\n\n@mixin text-truncate() {\n  overflow: hidden;\n  text-overflow: ellipsis;\n  white-space: nowrap;\n}\n","// stylelint-disable declaration-no-important\n\n// Typography\n\n@mixin text-emphasis-variant($parent, $color, $ignore-warning: false) {\n  #{$parent} {\n    color: $color !important;\n  }\n  @if $emphasized-link-hover-darken-percentage != 0 {\n    a#{$parent} {\n      @include hover-focus() {\n        color: darken($color, $emphasized-link-hover-darken-percentage) !important;\n      }\n    }\n  }\n  @include deprecate(\"`text-emphasis-variant()`\", \"v4.4.0\", \"v5\", $ignore-warning);\n}\n","// CSS image replacement\n@mixin text-hide($ignore-warning: false) {\n  // stylelint-disable-next-line font-family-no-missing-generic-family-keyword\n  font: 0/0 a;\n  color: transparent;\n  text-shadow: none;\n  background-color: transparent;\n  border: 0;\n\n  @include deprecate(\"`text-hide()`\", \"v4.1.0\", \"v5\", $ignore-warning);\n}\n","// stylelint-disable declaration-no-important\n\n//\n// Visibility utilities\n//\n\n.visible {\n  visibility: visible !important;\n}\n\n.invisible {\n  visibility: hidden !important;\n}\n","// stylelint-disable declaration-no-important, selector-no-qualifying-type\n\n// Source: https://github.com/h5bp/main.css/blob/master/src/_print.css\n\n// ==========================================================================\n// Print styles.\n// Inlined to avoid the additional HTTP request:\n// https://www.phpied.com/delay-loading-your-print-css/\n// ==========================================================================\n\n@if $enable-print-styles {\n  @media print {\n    *,\n    *::before,\n    *::after {\n      // Bootstrap specific; comment out `color` and `background`\n      //color: $black !important; // Black prints faster\n      text-shadow: none !important;\n      //background: transparent !important;\n      box-shadow: none !important;\n    }\n\n    a {\n      &:not(.btn) {\n        text-decoration: underline;\n      }\n    }\n\n    // Bootstrap specific; comment the following selector out\n    //a[href]::after {\n    //  content: \" (\" attr(href) \")\";\n    //}\n\n    abbr[title]::after {\n      content: \" (\" attr(title) \")\";\n    }\n\n    // Bootstrap specific; comment the following selector out\n    //\n    // Don't show links that are fragment identifiers,\n    // or use the `javascript:` pseudo protocol\n    //\n\n    //a[href^=\"#\"]::after,\n    //a[href^=\"javascript:\"]::after {\n    // content: \"\";\n    //}\n\n    pre {\n      white-space: pre-wrap !important;\n    }\n    pre,\n    blockquote {\n      border: $border-width solid $gray-500; // Bootstrap custom code; using `$border-width` instead of 1px\n      page-break-inside: avoid;\n    }\n\n    //\n    // Printing Tables:\n    // https://web.archive.org/web/20180815150934/http://css-discuss.incutio.com/wiki/Printing_Tables\n    //\n\n    thead {\n      display: table-header-group;\n    }\n\n    tr,\n    img {\n      page-break-inside: avoid;\n    }\n\n    p,\n    h2,\n    h3 {\n      orphans: 3;\n      widows: 3;\n    }\n\n    h2,\n    h3 {\n      page-break-after: avoid;\n    }\n\n    // Bootstrap specific changes start\n\n    // Specify a size and min-width to make printing closer across browsers.\n    // We don't set margin here because it breaks `size` in Chrome. We also\n    // don't use `!important` on `size` as it breaks in Chrome.\n    @page {\n      size: $print-page-size;\n    }\n    body {\n      min-width: $print-body-min-width !important;\n    }\n    .container {\n      min-width: $print-body-min-width !important;\n    }\n\n    // Bootstrap components\n    .navbar {\n      display: none;\n    }\n    .badge {\n      border: $border-width solid $black;\n    }\n\n    .table {\n      border-collapse: collapse !important;\n\n      td,\n      th {\n        background-color: $white !important;\n      }\n    }\n\n    .table-bordered {\n      th,\n      td {\n        border: 1px solid $gray-300 !important;\n      }\n    }\n\n    .table-dark {\n      color: inherit;\n\n      th,\n      td,\n      thead th,\n      tbody + tbody {\n        border-color: $table-border-color;\n      }\n    }\n\n    .table .thead-dark th {\n      color: inherit;\n      border-color: $table-border-color;\n    }\n\n    // Bootstrap specific changes end\n  }\n}\n"]}
\ No newline at end of file
diff --git a/docs/css/docsearch.css b/docs/css/docsearch.css
new file mode 100644
index 0000000000000..54059c0c2e9da
--- /dev/null
+++ b/docs/css/docsearch.css
@@ -0,0 +1,36 @@
+/* Main dropdown wrapper */
+.algolia-autocomplete .ds-dropdown-menu {
+    width: 500px;
+}
+
+/* Main category (eg. Getting Started) */
+.algolia-autocomplete .algolia-docsearch-suggestion--category-header {
+    color: darkgray;
+    border: 1px underline gray;
+}
+
+/* Category (eg. Downloads) */
+.algolia-autocomplete .algolia-docsearch-suggestion--subcategory-column {
+    color: gray;
+}
+
+/* Title (eg. Bootstrap CDN) */
+.algolia-autocomplete .algolia-docsearch-suggestion--title {
+    font-weight: bold;
+    color: black;
+}
+
+/* Description description (eg. Bootstrap currently works...) */
+.algolia-autocomplete .algolia-docsearch-suggestion--text {
+    font-size: 0.8rem;
+    color: gray;
+}
+
+/* Highlighted text */
+.algolia-autocomplete .algolia-docsearch-suggestion--highlight {
+    color: blue;
+}
+
+.searchbox {
+    margin-top: 2%;
+}
diff --git a/docs/css/main.css b/docs/css/main.css
index bb34d6e9bec0d..6710b6e8563c9 100755
--- a/docs/css/main.css
+++ b/docs/css/main.css
@@ -2,6 +2,22 @@
    Author's custom styles
    ========================================================================== */
 
+body {
+  font-size: 14px;
+}
+
+a {
+  color: #08c;
+}
+
+a:hover {
+  color: #05c;
+}
+
+img {
+  max-width: 100%;
+}
+
 table {
   margin: 15px 0;
   padding: 0;
@@ -33,10 +49,25 @@ table tr td {
   padding: 6px 13px;
 }
 
-.navbar .brand {
+.navbar {
+  background-color: #fafafa;
+  background-image: linear-gradient(to bottom, #ffffff, #e2f1f8);
+  background-repeat: repeat-x;
+  box-shadow: 0 1px 10px rgba(0,0,0,.1);
+  border-bottom-color: #d4d4d4;
+  border-bottom-style: solid;
+  border-bottom-width: 1px;
+  font-size: 15px;
+  line-height: 1;
+  margin-bottom: 15px;
+  padding: 0 1rem;
+}
+
+.navbar .navbar-brand {
   height: 50px;
   width: 110px;
   margin-left: 1px;
+  margin-right: 0;
   padding: 0;
 }
 
@@ -50,14 +81,66 @@ table tr td {
   color: #777;
 }
 
-.navbar-inner {
+.navbar .container {
+  padding-left: 0;
   padding-top: 2px;
-  height: 50px;
 }
 
-.navbar-inner .nav {
-  margin-top: 5px;
+.navbar .navbar-nav,
+.nav {
+  margin-left: 0;
+}
+
+.navbar .navbar-nav > .nav-item {
+  line-height: 20px;
+}
+
+.navbar .navbar-nav > .nav-item > .nav-link {
+  color: #555;
+  padding: 10px 15px 10px;
+}
+
+.navbar .dropdown-menu {
   font-size: 15px;
+  padding: 5px 0;
+  margin: 0;
+  border-radius: 6px;
+  box-shadow: 0 5px 10px rgba(0,0,0,0.2);
+}
+
+@media (min-width:768px) {
+  .navbar .dropdown-menu::before {
+    position: absolute;
+    top: -7px;
+    left: 9px;
+    display: inline-block;
+    border-right: 7px solid transparent;
+    border-bottom: 7px solid #ccc;
+    border-left: 7px solid transparent;
+    border-bottom-color: rgba(0,0,0,0.2);
+    content: '';
+  }
+
+  .navbar .dropdown-menu::after {
+    position: absolute;
+    top: -6px;
+    left: 10px;
+    display: inline-block;
+    border-right: 6px solid transparent;
+    border-bottom: 6px solid #fff;
+    border-left: 6px solid transparent;
+    content: '';
+  }
+}
+
+.navbar .dropdown-menu .dropdown-item:hover,
+.navbar .dropdown-menu .dropdown-item:focus,
+.navbar .dropdown-submenu:hover .dropdown-item {
+  color: #fff;
+  text-decoration: none;
+  background-color: #0088cc;
+  background-image: linear-gradient(to bottom,#0088cc,#0087b3);
+  background-repeat: repeat-x;
 }
 
 .navbar .divider-vertical {
@@ -79,12 +162,17 @@ body .container-wrapper {
   margin-right: auto;
   border-radius: 15px;
   position: relative;
+  min-height: 100vh;
 }
 
 .title {
   font-size: 32px;
 }
 
+h1, h2, h3, h4, h5, h6 {
+  font-weight: bold;
+}
+
 h1 {
   font-size: 28px;
   margin-top: 12px;
@@ -100,8 +188,22 @@ h3 {
   margin-top: 10px;
 }
 
+h4 {
+  font-size: 18px;
+  line-height: 20px;
+}
+
 pre {
+  background-color: #f5f5f5;
+  border: 1px solid rgba(0,0,0,0.15);
+  border-radius: 4px;
   font-family: "Menlo", "Lucida Console", monospace;
+  padding: 9.5px;
+}
+
+ul, ol {
+  padding: 0;
+  margin: 0 0 10px 25px;
 }
 
 code {
@@ -112,6 +214,10 @@ code {
   color: #444444;
 }
 
+pre code {
+  background: transparent;
+}
+
 dt code {
   white-space: nowrap;
   max-width: 100%;
@@ -148,7 +254,7 @@ a:hover code {
   position: relative;
   background-color: #FFF;
   max-width: 914px;
-  line-height: 1.6; /* Inspired by Github's wiki style */
+  line-height: 1.6; /* Inspired by GitHub's wiki style */
   padding-left: 15px;
 }
 
@@ -157,8 +263,9 @@ a:hover code {
   position: relative;
   background-color: #FFF;
   max-width: 914px;
-  line-height: 1.6; /* Inspired by Github's wiki style */
+  line-height: 1.6; /* Inspired by GitHub's wiki style */
   padding-left: 30px;
+  min-height: 100vh;
 }
 
 .dropdown-menu {
@@ -169,34 +276,11 @@ a:hover code {
   min-width: 50px;
 }
 
-/**
- * Make dropdown menus in nav bars show on hover instead of click
- * using solution at http://stackoverflow.com/questions/8878033/how-
- * to-make-twitter-bootstrap-menu-dropdown-on-hover-rather-than-click
- **/
-ul.nav li.dropdown:hover ul.dropdown-menu{
-  display: block;
-}
-
-a.menu:after, .dropdown-toggle:after {
-  content: none;
-}
-
-/** Make the submenus open on hover on the parent menu item */
-ul.nav li.dropdown ul.dropdown-menu li.dropdown-submenu:hover ul.dropdown-menu {
-  display: block;
-}
-
-/** Make the submenus be invisible until the parent menu item is hovered upon */
-ul.nav li.dropdown ul.dropdown-menu li.dropdown-submenu ul.dropdown-menu {
-  display: none;
-}
-
 /**
  * Made the navigation bar buttons not grey out when clicked.
  * Essentially making nav bar buttons not react to clicks, only hover events.
  */
-.navbar .nav li.dropdown.open > .dropdown-toggle {
+.navbar .navbar-nav .nav-item.dropdown.open > .dropdown-toggle {
   background-color: transparent;
 }
 
@@ -205,14 +289,18 @@ ul.nav li.dropdown ul.dropdown-menu li.dropdown-submenu ul.dropdown-menu {
  * That looks weird. Changed the colors to active - blue, inactive - black, and
  * no color change on hover.
  */
-.nav-tabs > .active > a, .nav-tabs > .active > a:hover {
+.nav-tabs .nav-link.active, .nav-tabs .nav-link.active:hover {
   color: #08c;
 }
 
-.nav-tabs > li > a, .nav-tabs > li > a:hover {
+.nav-tabs .nav-link, .nav-tabs .nav-link:hover {
   color: #333;
 }
 
+.nav-tabs .nav-link:not(.active):focus, .nav-tabs .nav-link:not(.active):hover {
+  background-color: #e9ecef;
+}
+
 /**
  * MathJax (embedded latex formulas)
  */
@@ -238,17 +326,13 @@ a.anchorjs-link:hover { text-decoration: none; }
   border-left-width: 0px;
   border-bottom-width: 0px;
   margin-top: 0px;
-  width: 210px;
+  width: 220px;
+  height: 80%;
   float: left;
   position: fixed;
   overflow-y: scroll;
 }
 
-.left-menu {
-  padding: 0px;
-  width: 399px;
-}
-
 .left-menu h3 {
   margin-left: 10px;
   line-height: 30px;
@@ -280,6 +364,7 @@ label[for="nav-trigger"] {
   cursor: pointer;
   background-size: contain;
   background-color: #D4F0FF;
+  box-sizing: content-box;
 }
 
 label[for="nav-trigger"]:hover {
diff --git a/docs/graphx-programming-guide.md b/docs/graphx-programming-guide.md
index 50c9366a0999f..a1026669dc4fd 100644
--- a/docs/graphx-programming-guide.md
+++ b/docs/graphx-programming-guide.md
@@ -571,7 +571,7 @@ messages to the source and destination attributes.  Think of `sendMsg` as the <i
 function in map-reduce.
 The user defined `mergeMsg` function takes two messages destined to the same vertex and
 yields a single message.  Think of `mergeMsg` as the <i>reduce</i> function in map-reduce.
-The  [`aggregateMessages`][Graph.aggregateMessages] operator returns a `VertexRDD[Msg]`
+The  [`aggregateMessages`][Graph.aggregateMessages] operator returns an `VertexRDD[Msg]`
 containing the aggregate message (of type `Msg`) destined to each vertex.  Vertices that did not
 receive a message are not included in the returned `VertexRDD`[VertexRDD].
 
@@ -874,7 +874,7 @@ change the `VertexId` thereby enabling the same `HashMap` data structures to be
 `HashMap` and implement the join by linear scan rather than costly point lookups.
 
 The `aggregateUsingIndex` operator is useful for efficient construction of a new `VertexRDD`[VertexRDD] from an
-`RDD[(VertexId, A)]`.  Conceptually, if I have constructed a `VertexRDD[B]` over a set of vertices,
+`RDD[(VertexId, A)]`.  Conceptually, if I have constructed an `VertexRDD[B]` over a set of vertices,
 *which is a super-set* of the vertices in some `RDD[(VertexId, A)]` then I can reuse the index to
 both aggregate and then subsequently index the `RDD[(VertexId, A)]`.  For example:
 
diff --git a/docs/img/pycharm-with-pyspark1.png b/docs/img/pycharm-with-pyspark1.png
new file mode 100644
index 0000000000000..6e2c0bc02d2b7
Binary files /dev/null and b/docs/img/pycharm-with-pyspark1.png differ
diff --git a/docs/img/pycharm-with-pyspark2.png b/docs/img/pycharm-with-pyspark2.png
new file mode 100644
index 0000000000000..8acefc47476c9
Binary files /dev/null and b/docs/img/pycharm-with-pyspark2.png differ
diff --git a/docs/img/pycharm-with-pyspark3.png b/docs/img/pycharm-with-pyspark3.png
new file mode 100644
index 0000000000000..7a4113dd4e658
Binary files /dev/null and b/docs/img/pycharm-with-pyspark3.png differ
diff --git a/docs/img/pyspark-components.png b/docs/img/pyspark-components.png
new file mode 100644
index 0000000000000..a0979d3465a92
Binary files /dev/null and b/docs/img/pyspark-components.png differ
diff --git a/docs/img/pyspark-components.pptx b/docs/img/pyspark-components.pptx
new file mode 100644
index 0000000000000..e0111a44e186e
Binary files /dev/null and b/docs/img/pyspark-components.pptx differ
diff --git a/docs/img/pyspark-remote-debug1.png b/docs/img/pyspark-remote-debug1.png
new file mode 100644
index 0000000000000..3469e1babc2bd
Binary files /dev/null and b/docs/img/pyspark-remote-debug1.png differ
diff --git a/docs/img/pyspark-remote-debug2.png b/docs/img/pyspark-remote-debug2.png
new file mode 100644
index 0000000000000..4790773681cc8
Binary files /dev/null and b/docs/img/pyspark-remote-debug2.png differ
diff --git a/docs/img/spark-logo-reverse.png b/docs/img/spark-logo-reverse.png
new file mode 100644
index 0000000000000..a3e4ed4bb3d08
Binary files /dev/null and b/docs/img/spark-logo-reverse.png differ
diff --git a/docs/img/webui-structured-streaming-detail.png b/docs/img/webui-structured-streaming-detail.png
new file mode 100644
index 0000000000000..f4850523c5c2f
Binary files /dev/null and b/docs/img/webui-structured-streaming-detail.png differ
diff --git a/docs/img/webui-structured-streaming-detail2.png b/docs/img/webui-structured-streaming-detail2.png
new file mode 100644
index 0000000000000..1b9c941549d01
Binary files /dev/null and b/docs/img/webui-structured-streaming-detail2.png differ
diff --git a/docs/index.md b/docs/index.md
index 38f12dd4db77b..f5a4f9cf43f40 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -25,11 +25,6 @@ It provides high-level APIs in Java, Scala, Python and R,
 and an optimized engine that supports general execution graphs.
 It also supports a rich set of higher-level tools including [Spark SQL](sql-programming-guide.html) for SQL and structured data processing, [MLlib](ml-guide.html) for machine learning, [GraphX](graphx-programming-guide.html) for graph processing, and [Structured Streaming](structured-streaming-programming-guide.html) for incremental computation and stream processing.
 
-# Security
-
-Security in Spark is OFF by default. This could mean you are vulnerable to attack by default.
-Please see [Spark Security](security.html) before downloading and running Spark.
-
 # Downloading
 
 Get Spark from the [downloads page](https://spark.apache.org/downloads.html) of the project website. This documentation is for Spark version {{site.SPARK_VERSION}}. Spark uses Hadoop's client libraries for HDFS and YARN. Downloads are pre-packaged for a handful of popular Hadoop versions.
@@ -44,14 +39,13 @@ source, visit [Building Spark](building-spark.html).
 
 Spark runs on both Windows and UNIX-like systems (e.g. Linux, Mac OS), and it should run on any platform that runs a supported version of Java. This should include JVMs on x86_64 and ARM64. It's easy to run locally on one machine --- all you need is to have `java` installed on your system `PATH`, or the `JAVA_HOME` environment variable pointing to a Java installation.
 
-Spark runs on Java 8/11, Scala 2.12, Python 2.7+/3.4+ and R 3.1+.
+Spark runs on Java 8/11, Scala 2.12, Python 3.6+ and R 3.5+.
 Java 8 prior to version 8u92 support is deprecated as of Spark 3.0.0.
-Python 2 and Python 3 prior to version 3.6 support is deprecated as of Spark 3.0.0.
-R prior to version 3.4 support is deprecated as of Spark 3.0.0.
 For the Scala API, Spark {{site.SPARK_VERSION}}
 uses Scala {{site.SCALA_BINARY_VERSION}}. You will need to use a compatible Scala version
 ({{site.SCALA_BINARY_VERSION}}.x).
 
+For Python 3.9, Arrow optimization and pandas UDFs might not work due to the supported Python versions in Apache Arrow. Please refer to the latest [Python Compatibility](https://arrow.apache.org/docs/python/install.html#python-compatibility) page.
 For Java 11, `-Dio.netty.tryReflectionSetAccessible=true` is required additionally for Apache Arrow library. This prevents `java.lang.UnsupportedOperationException: sun.misc.Unsafe or java.nio.DirectByteBuffer.(long, int) not available` when Apache Arrow uses Netty internally.
 
 # Running the Examples and Shell
@@ -100,7 +94,7 @@ Spark can run both by itself, or over several existing cluster managers. It curr
 options for deployment:
 
 * [Standalone Deploy Mode](spark-standalone.html): simplest way to deploy Spark on a private cluster
-* [Apache Mesos](running-on-mesos.html)
+* [Apache Mesos](running-on-mesos.html) (deprecated)
 * [Hadoop YARN](running-on-yarn.html)
 * [Kubernetes](running-on-kubernetes.html)
 
@@ -115,6 +109,8 @@ options for deployment:
 * [Spark Streaming](streaming-programming-guide.html): processing data streams using DStreams (old API)
 * [MLlib](ml-guide.html): applying machine learning algorithms
 * [GraphX](graphx-programming-guide.html): processing graphs 
+* [SparkR](sparkr.html): processing data with Spark in R
+* [PySpark](api/python/getting_started/index.html): processing data with Spark in Python
 
 **API Docs:**
 
diff --git a/docs/job-scheduling.md b/docs/job-scheduling.md
index eaacfa49c657c..51060ddf81328 100644
--- a/docs/job-scheduling.md
+++ b/docs/job-scheduling.md
@@ -79,23 +79,24 @@ are no longer used and request them again later when there is demand. This featu
 useful if multiple applications share resources in your Spark cluster.
 
 This feature is disabled by default and available on all coarse-grained cluster managers, i.e.
-[standalone mode](spark-standalone.html), [YARN mode](running-on-yarn.html), and
-[Mesos coarse-grained mode](running-on-mesos.html#mesos-run-modes).
+[standalone mode](spark-standalone.html), [YARN mode](running-on-yarn.html),
+[Mesos coarse-grained mode](running-on-mesos.html#mesos-run-modes) and [K8s mode](running-on-kubernetes.html).
+
 
 ### Configuration and Setup
 
-There are two requirements for using this feature. First, your application must set
-`spark.dynamicAllocation.enabled` to `true`. Second, you must set up an *external shuffle service*
-on each worker node in the same cluster and set `spark.shuffle.service.enabled` to true in your
-application. The purpose of the external shuffle service is to allow executors to be removed
+There are two ways for using this feature.
+First, your application must set both `spark.dynamicAllocation.enabled` and `spark.dynamicAllocation.shuffleTracking.enabled` to `true`.
+Second, your application must set both `spark.dynamicAllocation.enabled` and `spark.shuffle.service.enabled` to `true`
+after you set up an *external shuffle service* on each worker node in the same cluster.
+The purpose of the shuffle tracking or the external shuffle service is to allow executors to be removed
 without deleting shuffle files written by them (more detail described
-[below](job-scheduling.html#graceful-decommission-of-executors)). The way to set up this service
-varies across cluster managers:
+[below](job-scheduling.html#graceful-decommission-of-executors)). While it is simple to enable shuffle tracking, the way to set up the external shuffle service varies across cluster managers:
 
 In standalone mode, simply start your workers with `spark.shuffle.service.enabled` set to `true`.
 
 In Mesos coarse-grained mode, run `$SPARK_HOME/sbin/start-mesos-shuffle-service.sh` on all
-slave nodes with `spark.shuffle.service.enabled` set to `true`. For instance, you may do so
+worker nodes with `spark.shuffle.service.enabled` set to `true`. For instance, you may do so
 through Marathon.
 
 In YARN mode, follow the instructions [here](running-on-yarn.html#configuring-the-external-shuffle-service).
@@ -251,10 +252,11 @@ properties:
 
 The pool properties can be set by creating an XML file, similar to `conf/fairscheduler.xml.template`,
 and either putting a file named `fairscheduler.xml` on the classpath, or setting `spark.scheduler.allocation.file` property in your
-[SparkConf](configuration.html#spark-properties).
+[SparkConf](configuration.html#spark-properties). The file path can either be a local file path or HDFS file path.
 
 {% highlight scala %}
 conf.set("spark.scheduler.allocation.file", "/path/to/file")
+conf.set("spark.scheduler.allocation.file", "hdfs:///path/to/file")
 {% endhighlight %}
 
 The format of the XML file is simply a `<pool>` element for each pool, with different elements
@@ -297,11 +299,9 @@ via `sc.setJobGroup` in a separate PVM thread, which also disallows to cancel th
 later.
 
 In order to synchronize PVM threads with JVM threads, you should set `PYSPARK_PIN_THREAD` environment variable
-to `true`. This pinned thread mode allows one PVM thread has one corresponding JVM thread.
-
-However, currently it cannot inherit the local properties from the parent thread although it isolates
-each thread with its own local properties. To work around this, you should manually copy and set the
-local properties from the parent thread to the child thread when you create another thread in PVM.
+to `true`. This pinned thread mode allows one PVM thread has one corresponding JVM thread. With this mode,
+`pyspark.InheritableThread` is recommended to use together for a PVM thread to inherit the inheritable attributes
+ such as local properties in a JVM thread.
 
 Note that `PYSPARK_PIN_THREAD` is currently experimental and not recommended for use in production.
 
diff --git a/docs/js/main.js b/docs/js/main.js
index 3ee3dbe8eaed2..968097c8041d6 100755
--- a/docs/js/main.js
+++ b/docs/js/main.js
@@ -29,7 +29,7 @@ function codeTabs() {
     $(this).addClass("tab-content");
 
     // Insert the tab bar
-    var tabBar = $('<ul class="nav nav-tabs" data-tabs="tabs"></ul>');
+    var tabBar = $('<ul class="nav nav-tabs mb-4" data-tabs="tabs" role="tablist"></ul>');
     $(this).before(tabBar);
 
     // Add each code sample to the tab bar:
@@ -50,12 +50,12 @@ function codeTabs() {
         var buttonLabel = ""
       }
       tabBar.append(
-        '<li><a class="tab_' + lang + '" href="#' + id + '">' + buttonLabel + '</a></li>'
+        '<li class="nav-item"><a class="nav-link tab_' + lang + '" href="#' + id + '" data-toggle="tab">' + buttonLabel + '</a></li>'
       );
     });
 
     codeSamples.first().addClass("active");
-    tabBar.children("li").first().addClass("active");
+    tabBar.children("li").first().children("a").first().addClass("active");
     counter++;
   });
   $("ul.nav-tabs a").click(function (e) {
@@ -94,4 +94,32 @@ $(function() {
   // Scroll now too in case we had opened the page on a hash, but wait a bit because some browsers
   // will try to do *their* initial scroll after running the onReady handler.
   $(window).on('load', function() { setTimeout(function() { maybeScrollToHash(); }, 25); });
+
+  // Make dropdown menus in nav bars show on hover instead of click
+  // using solution at https://webdesign.tutsplus.com/tutorials/how-
+  // to-make-the-bootstrap-navbar-dropdown-work-on-hover--cms-33840
+  const $dropdown = $(".dropdown");
+  const $dropdownToggle = $(".dropdown-toggle");
+  const $dropdownMenu = $(".dropdown-menu");
+  const showClass = "show";
+  $(window).on("load resize", function() {
+    if (this.matchMedia("(min-width: 768px)").matches) {
+      $dropdown.hover(
+        function() {
+          const $this = $(this);
+          $this.addClass(showClass);
+          $this.find($dropdownToggle).attr("aria-expanded", "true");
+          $this.find($dropdownMenu).addClass(showClass);
+        },
+        function() {
+          const $this = $(this);
+          $this.removeClass(showClass);
+          $this.find($dropdownToggle).attr("aria-expanded", "false");
+          $this.find($dropdownMenu).removeClass(showClass);
+        }
+      );
+    } else {
+      $dropdown.off("mouseenter mouseleave");
+    }
+  });
 });
diff --git a/docs/js/vendor/bootstrap.bundle.min.js b/docs/js/vendor/bootstrap.bundle.min.js
new file mode 100644
index 0000000000000..78c533b4e7ecb
--- /dev/null
+++ b/docs/js/vendor/bootstrap.bundle.min.js
@@ -0,0 +1,7 @@
+/*!
+  * Bootstrap v4.4.1 (https://getbootstrap.com/)
+  * Copyright 2011-2019 The Bootstrap Authors (https://github.com/twbs/bootstrap/graphs/contributors)
+  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
+  */
+!function(e,t){"object"==typeof exports&&"undefined"!=typeof module?t(exports,require("jquery")):"function"==typeof define&&define.amd?define(["exports","jquery"],t):t((e=e||self).bootstrap={},e.jQuery)}(this,function(e,p){"use strict";function i(e,t){for(var n=0;n<t.length;n++){var i=t[n];i.enumerable=i.enumerable||!1,i.configurable=!0,"value"in i&&(i.writable=!0),Object.defineProperty(e,i.key,i)}}function s(e,t,n){return t&&i(e.prototype,t),n&&i(e,n),e}function t(t,e){var n=Object.keys(t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(t);e&&(i=i.filter(function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable})),n.push.apply(n,i)}return n}function l(o){for(var e=1;e<arguments.length;e++){var r=null!=arguments[e]?arguments[e]:{};e%2?t(Object(r),!0).forEach(function(e){var t,n,i;t=o,i=r[n=e],n in t?Object.defineProperty(t,n,{value:i,enumerable:!0,configurable:!0,writable:!0}):t[n]=i}):Object.getOwnPropertyDescriptors?Object.defineProperties(o,Object.getOwnPropertyDescriptors(r)):t(Object(r)).forEach(function(e){Object.defineProperty(o,e,Object.getOwnPropertyDescriptor(r,e))})}return o}p=p&&p.hasOwnProperty("default")?p.default:p;var n="transitionend";function o(e){var t=this,n=!1;return p(this).one(m.TRANSITION_END,function(){n=!0}),setTimeout(function(){n||m.triggerTransitionEnd(t)},e),this}var m={TRANSITION_END:"bsTransitionEnd",getUID:function(e){for(;e+=~~(1e6*Math.random()),document.getElementById(e););return e},getSelectorFromElement:function(e){var t=e.getAttribute("data-target");if(!t||"#"===t){var n=e.getAttribute("href");t=n&&"#"!==n?n.trim():""}try{return document.querySelector(t)?t:null}catch(e){return null}},getTransitionDurationFromElement:function(e){if(!e)return 0;var t=p(e).css("transition-duration"),n=p(e).css("transition-delay"),i=parseFloat(t),o=parseFloat(n);return i||o?(t=t.split(",")[0],n=n.split(",")[0],1e3*(parseFloat(t)+parseFloat(n))):0},reflow:function(e){return e.offsetHeight},triggerTransitionEnd:function(e){p(e).trigger(n)},supportsTransitionEnd:function(){return Boolean(n)},isElement:function(e){return(e[0]||e).nodeType},typeCheckConfig:function(e,t,n){for(var i in n)if(Object.prototype.hasOwnProperty.call(n,i)){var o=n[i],r=t[i],s=r&&m.isElement(r)?"element":(a=r,{}.toString.call(a).match(/\s([a-z]+)/i)[1].toLowerCase());if(!new RegExp(o).test(s))throw new Error(e.toUpperCase()+': Option "'+i+'" provided type "'+s+'" but expected type "'+o+'".')}var a},findShadowRoot:function(e){if(!document.documentElement.attachShadow)return null;if("function"!=typeof e.getRootNode)return e instanceof ShadowRoot?e:e.parentNode?m.findShadowRoot(e.parentNode):null;var t=e.getRootNode();return t instanceof ShadowRoot?t:null},jQueryDetection:function(){if("undefined"==typeof p)throw new TypeError("Bootstrap's JavaScript requires jQuery. jQuery must be included before Bootstrap's JavaScript.");var e=p.fn.jquery.split(" ")[0].split(".");if(e[0]<2&&e[1]<9||1===e[0]&&9===e[1]&&e[2]<1||4<=e[0])throw new Error("Bootstrap's JavaScript requires at least jQuery v1.9.1 but less than v4.0.0")}};m.jQueryDetection(),p.fn.emulateTransitionEnd=o,p.event.special[m.TRANSITION_END]={bindType:n,delegateType:n,handle:function(e){if(p(e.target).is(this))return e.handleObj.handler.apply(this,arguments)}};var r="alert",a="bs.alert",c="."+a,h=p.fn[r],u={CLOSE:"close"+c,CLOSED:"closed"+c,CLICK_DATA_API:"click"+c+".data-api"},f="alert",d="fade",g="show",_=function(){function i(e){this._element=e}var e=i.prototype;return e.close=function(e){var t=this._element;e&&(t=this._getRootElement(e)),this._triggerCloseEvent(t).isDefaultPrevented()||this._removeElement(t)},e.dispose=function(){p.removeData(this._element,a),this._element=null},e._getRootElement=function(e){var t=m.getSelectorFromElement(e),n=!1;return t&&(n=document.querySelector(t)),n=n||p(e).closest("."+f)[0]},e._triggerCloseEvent=function(e){var t=p.Event(u.CLOSE);return p(e).trigger(t),t},e._removeElement=function(t){var n=this;if(p(t).removeClass(g),p(t).hasClass(d)){var e=m.getTransitionDurationFromElement(t);p(t).one(m.TRANSITION_END,function(e){return n._destroyElement(t,e)}).emulateTransitionEnd(e)}else this._destroyElement(t)},e._destroyElement=function(e){p(e).detach().trigger(u.CLOSED).remove()},i._jQueryInterface=function(n){return this.each(function(){var e=p(this),t=e.data(a);t||(t=new i(this),e.data(a,t)),"close"===n&&t[n](this)})},i._handleDismiss=function(t){return function(e){e&&e.preventDefault(),t.close(this)}},s(i,null,[{key:"VERSION",get:function(){return"4.4.1"}}]),i}();p(document).on(u.CLICK_DATA_API,'[data-dismiss="alert"]',_._handleDismiss(new _)),p.fn[r]=_._jQueryInterface,p.fn[r].Constructor=_,p.fn[r].noConflict=function(){return p.fn[r]=h,_._jQueryInterface};var v="button",y="bs.button",E="."+y,b=".data-api",w=p.fn[v],T="active",C="btn",S="focus",D='[data-toggle^="button"]',I='[data-toggle="buttons"]',A='[data-toggle="button"]',O='[data-toggle="buttons"] .btn',N='input:not([type="hidden"])',k=".active",L=".btn",P={CLICK_DATA_API:"click"+E+b,FOCUS_BLUR_DATA_API:"focus"+E+b+" blur"+E+b,LOAD_DATA_API:"load"+E+b},x=function(){function n(e){this._element=e}var e=n.prototype;return e.toggle=function(){var e=!0,t=!0,n=p(this._element).closest(I)[0];if(n){var i=this._element.querySelector(N);if(i){if("radio"===i.type)if(i.checked&&this._element.classList.contains(T))e=!1;else{var o=n.querySelector(k);o&&p(o).removeClass(T)}else"checkbox"===i.type?"LABEL"===this._element.tagName&&i.checked===this._element.classList.contains(T)&&(e=!1):e=!1;e&&(i.checked=!this._element.classList.contains(T),p(i).trigger("change")),i.focus(),t=!1}}this._element.hasAttribute("disabled")||this._element.classList.contains("disabled")||(t&&this._element.setAttribute("aria-pressed",!this._element.classList.contains(T)),e&&p(this._element).toggleClass(T))},e.dispose=function(){p.removeData(this._element,y),this._element=null},n._jQueryInterface=function(t){return this.each(function(){var e=p(this).data(y);e||(e=new n(this),p(this).data(y,e)),"toggle"===t&&e[t]()})},s(n,null,[{key:"VERSION",get:function(){return"4.4.1"}}]),n}();p(document).on(P.CLICK_DATA_API,D,function(e){var t=e.target;if(p(t).hasClass(C)||(t=p(t).closest(L)[0]),!t||t.hasAttribute("disabled")||t.classList.contains("disabled"))e.preventDefault();else{var n=t.querySelector(N);if(n&&(n.hasAttribute("disabled")||n.classList.contains("disabled")))return void e.preventDefault();x._jQueryInterface.call(p(t),"toggle")}}).on(P.FOCUS_BLUR_DATA_API,D,function(e){var t=p(e.target).closest(L)[0];p(t).toggleClass(S,/^focus(in)?$/.test(e.type))}),p(window).on(P.LOAD_DATA_API,function(){for(var e=[].slice.call(document.querySelectorAll(O)),t=0,n=e.length;t<n;t++){var i=e[t],o=i.querySelector(N);o.checked||o.hasAttribute("checked")?i.classList.add(T):i.classList.remove(T)}for(var r=0,s=(e=[].slice.call(document.querySelectorAll(A))).length;r<s;r++){var a=e[r];"true"===a.getAttribute("aria-pressed")?a.classList.add(T):a.classList.remove(T)}}),p.fn[v]=x._jQueryInterface,p.fn[v].Constructor=x,p.fn[v].noConflict=function(){return p.fn[v]=w,x._jQueryInterface};var j="carousel",H="bs.carousel",R="."+H,F=".data-api",M=p.fn[j],W={interval:5e3,keyboard:!0,slide:!1,pause:"hover",wrap:!0,touch:!0},U={interval:"(number|boolean)",keyboard:"boolean",slide:"(boolean|string)",pause:"(string|boolean)",wrap:"boolean",touch:"boolean"},B="next",q="prev",K="left",Q="right",V={SLIDE:"slide"+R,SLID:"slid"+R,KEYDOWN:"keydown"+R,MOUSEENTER:"mouseenter"+R,MOUSELEAVE:"mouseleave"+R,TOUCHSTART:"touchstart"+R,TOUCHMOVE:"touchmove"+R,TOUCHEND:"touchend"+R,POINTERDOWN:"pointerdown"+R,POINTERUP:"pointerup"+R,DRAG_START:"dragstart"+R,LOAD_DATA_API:"load"+R+F,CLICK_DATA_API:"click"+R+F},Y="carousel",z="active",X="slide",G="carousel-item-right",$="carousel-item-left",J="carousel-item-next",Z="carousel-item-prev",ee="pointer-event",te=".active",ne=".active.carousel-item",ie=".carousel-item",oe=".carousel-item img",re=".carousel-item-next, .carousel-item-prev",se=".carousel-indicators",ae="[data-slide], [data-slide-to]",le='[data-ride="carousel"]',ce={TOUCH:"touch",PEN:"pen"},he=function(){function r(e,t){this._items=null,this._interval=null,this._activeElement=null,this._isPaused=!1,this._isSliding=!1,this.touchTimeout=null,this.touchStartX=0,this.touchDeltaX=0,this._config=this._getConfig(t),this._element=e,this._indicatorsElement=this._element.querySelector(se),this._touchSupported="ontouchstart"in document.documentElement||0<navigator.maxTouchPoints,this._pointerEvent=Boolean(window.PointerEvent||window.MSPointerEvent),this._addEventListeners()}var e=r.prototype;return e.next=function(){this._isSliding||this._slide(B)},e.nextWhenVisible=function(){!document.hidden&&p(this._element).is(":visible")&&"hidden"!==p(this._element).css("visibility")&&this.next()},e.prev=function(){this._isSliding||this._slide(q)},e.pause=function(e){e||(this._isPaused=!0),this._element.querySelector(re)&&(m.triggerTransitionEnd(this._element),this.cycle(!0)),clearInterval(this._interval),this._interval=null},e.cycle=function(e){e||(this._isPaused=!1),this._interval&&(clearInterval(this._interval),this._interval=null),this._config.interval&&!this._isPaused&&(this._interval=setInterval((document.visibilityState?this.nextWhenVisible:this.next).bind(this),this._config.interval))},e.to=function(e){var t=this;this._activeElement=this._element.querySelector(ne);var n=this._getItemIndex(this._activeElement);if(!(e>this._items.length-1||e<0))if(this._isSliding)p(this._element).one(V.SLID,function(){return t.to(e)});else{if(n===e)return this.pause(),void this.cycle();var i=n<e?B:q;this._slide(i,this._items[e])}},e.dispose=function(){p(this._element).off(R),p.removeData(this._element,H),this._items=null,this._config=null,this._element=null,this._interval=null,this._isPaused=null,this._isSliding=null,this._activeElement=null,this._indicatorsElement=null},e._getConfig=function(e){return e=l({},W,{},e),m.typeCheckConfig(j,e,U),e},e._handleSwipe=function(){var e=Math.abs(this.touchDeltaX);if(!(e<=40)){var t=e/this.touchDeltaX;(this.touchDeltaX=0)<t&&this.prev(),t<0&&this.next()}},e._addEventListeners=function(){var t=this;this._config.keyboard&&p(this._element).on(V.KEYDOWN,function(e){return t._keydown(e)}),"hover"===this._config.pause&&p(this._element).on(V.MOUSEENTER,function(e){return t.pause(e)}).on(V.MOUSELEAVE,function(e){return t.cycle(e)}),this._config.touch&&this._addTouchEventListeners()},e._addTouchEventListeners=function(){var t=this;if(this._touchSupported){var n=function(e){t._pointerEvent&&ce[e.originalEvent.pointerType.toUpperCase()]?t.touchStartX=e.originalEvent.clientX:t._pointerEvent||(t.touchStartX=e.originalEvent.touches[0].clientX)},i=function(e){t._pointerEvent&&ce[e.originalEvent.pointerType.toUpperCase()]&&(t.touchDeltaX=e.originalEvent.clientX-t.touchStartX),t._handleSwipe(),"hover"===t._config.pause&&(t.pause(),t.touchTimeout&&clearTimeout(t.touchTimeout),t.touchTimeout=setTimeout(function(e){return t.cycle(e)},500+t._config.interval))};p(this._element.querySelectorAll(oe)).on(V.DRAG_START,function(e){return e.preventDefault()}),this._pointerEvent?(p(this._element).on(V.POINTERDOWN,function(e){return n(e)}),p(this._element).on(V.POINTERUP,function(e){return i(e)}),this._element.classList.add(ee)):(p(this._element).on(V.TOUCHSTART,function(e){return n(e)}),p(this._element).on(V.TOUCHMOVE,function(e){return function(e){e.originalEvent.touches&&1<e.originalEvent.touches.length?t.touchDeltaX=0:t.touchDeltaX=e.originalEvent.touches[0].clientX-t.touchStartX}(e)}),p(this._element).on(V.TOUCHEND,function(e){return i(e)}))}},e._keydown=function(e){if(!/input|textarea/i.test(e.target.tagName))switch(e.which){case 37:e.preventDefault(),this.prev();break;case 39:e.preventDefault(),this.next()}},e._getItemIndex=function(e){return this._items=e&&e.parentNode?[].slice.call(e.parentNode.querySelectorAll(ie)):[],this._items.indexOf(e)},e._getItemByDirection=function(e,t){var n=e===B,i=e===q,o=this._getItemIndex(t),r=this._items.length-1;if((i&&0===o||n&&o===r)&&!this._config.wrap)return t;var s=(o+(e===q?-1:1))%this._items.length;return-1==s?this._items[this._items.length-1]:this._items[s]},e._triggerSlideEvent=function(e,t){var n=this._getItemIndex(e),i=this._getItemIndex(this._element.querySelector(ne)),o=p.Event(V.SLIDE,{relatedTarget:e,direction:t,from:i,to:n});return p(this._element).trigger(o),o},e._setActiveIndicatorElement=function(e){if(this._indicatorsElement){var t=[].slice.call(this._indicatorsElement.querySelectorAll(te));p(t).removeClass(z);var n=this._indicatorsElement.children[this._getItemIndex(e)];n&&p(n).addClass(z)}},e._slide=function(e,t){var n,i,o,r=this,s=this._element.querySelector(ne),a=this._getItemIndex(s),l=t||s&&this._getItemByDirection(e,s),c=this._getItemIndex(l),h=Boolean(this._interval);if(o=e===B?(n=$,i=J,K):(n=G,i=Z,Q),l&&p(l).hasClass(z))this._isSliding=!1;else if(!this._triggerSlideEvent(l,o).isDefaultPrevented()&&s&&l){this._isSliding=!0,h&&this.pause(),this._setActiveIndicatorElement(l);var u=p.Event(V.SLID,{relatedTarget:l,direction:o,from:a,to:c});if(p(this._element).hasClass(X)){p(l).addClass(i),m.reflow(l),p(s).addClass(n),p(l).addClass(n);var f=parseInt(l.getAttribute("data-interval"),10);f?(this._config.defaultInterval=this._config.defaultInterval||this._config.interval,this._config.interval=f):this._config.interval=this._config.defaultInterval||this._config.interval;var d=m.getTransitionDurationFromElement(s);p(s).one(m.TRANSITION_END,function(){p(l).removeClass(n+" "+i).addClass(z),p(s).removeClass(z+" "+i+" "+n),r._isSliding=!1,setTimeout(function(){return p(r._element).trigger(u)},0)}).emulateTransitionEnd(d)}else p(s).removeClass(z),p(l).addClass(z),this._isSliding=!1,p(this._element).trigger(u);h&&this.cycle()}},r._jQueryInterface=function(i){return this.each(function(){var e=p(this).data(H),t=l({},W,{},p(this).data());"object"==typeof i&&(t=l({},t,{},i));var n="string"==typeof i?i:t.slide;if(e||(e=new r(this,t),p(this).data(H,e)),"number"==typeof i)e.to(i);else if("string"==typeof n){if("undefined"==typeof e[n])throw new TypeError('No method named "'+n+'"');e[n]()}else t.interval&&t.ride&&(e.pause(),e.cycle())})},r._dataApiClickHandler=function(e){var t=m.getSelectorFromElement(this);if(t){var n=p(t)[0];if(n&&p(n).hasClass(Y)){var i=l({},p(n).data(),{},p(this).data()),o=this.getAttribute("data-slide-to");o&&(i.interval=!1),r._jQueryInterface.call(p(n),i),o&&p(n).data(H).to(o),e.preventDefault()}}},s(r,null,[{key:"VERSION",get:function(){return"4.4.1"}},{key:"Default",get:function(){return W}}]),r}();p(document).on(V.CLICK_DATA_API,ae,he._dataApiClickHandler),p(window).on(V.LOAD_DATA_API,function(){for(var e=[].slice.call(document.querySelectorAll(le)),t=0,n=e.length;t<n;t++){var i=p(e[t]);he._jQueryInterface.call(i,i.data())}}),p.fn[j]=he._jQueryInterface,p.fn[j].Constructor=he,p.fn[j].noConflict=function(){return p.fn[j]=M,he._jQueryInterface};var ue="collapse",fe="bs.collapse",de="."+fe,pe=p.fn[ue],me={toggle:!0,parent:""},ge={toggle:"boolean",parent:"(string|element)"},_e={SHOW:"show"+de,SHOWN:"shown"+de,HIDE:"hide"+de,HIDDEN:"hidden"+de,CLICK_DATA_API:"click"+de+".data-api"},ve="show",ye="collapse",Ee="collapsing",be="collapsed",we="width",Te="height",Ce=".show, .collapsing",Se='[data-toggle="collapse"]',De=function(){function a(t,e){this._isTransitioning=!1,this._element=t,this._config=this._getConfig(e),this._triggerArray=[].slice.call(document.querySelectorAll('[data-toggle="collapse"][href="#'+t.id+'"],[data-toggle="collapse"][data-target="#'+t.id+'"]'));for(var n=[].slice.call(document.querySelectorAll(Se)),i=0,o=n.length;i<o;i++){var r=n[i],s=m.getSelectorFromElement(r),a=[].slice.call(document.querySelectorAll(s)).filter(function(e){return e===t});null!==s&&0<a.length&&(this._selector=s,this._triggerArray.push(r))}this._parent=this._config.parent?this._getParent():null,this._config.parent||this._addAriaAndCollapsedClass(this._element,this._triggerArray),this._config.toggle&&this.toggle()}var e=a.prototype;return e.toggle=function(){p(this._element).hasClass(ve)?this.hide():this.show()},e.show=function(){var e,t,n=this;if(!this._isTransitioning&&!p(this._element).hasClass(ve)&&(this._parent&&0===(e=[].slice.call(this._parent.querySelectorAll(Ce)).filter(function(e){return"string"==typeof n._config.parent?e.getAttribute("data-parent")===n._config.parent:e.classList.contains(ye)})).length&&(e=null),!(e&&(t=p(e).not(this._selector).data(fe))&&t._isTransitioning))){var i=p.Event(_e.SHOW);if(p(this._element).trigger(i),!i.isDefaultPrevented()){e&&(a._jQueryInterface.call(p(e).not(this._selector),"hide"),t||p(e).data(fe,null));var o=this._getDimension();p(this._element).removeClass(ye).addClass(Ee),this._element.style[o]=0,this._triggerArray.length&&p(this._triggerArray).removeClass(be).attr("aria-expanded",!0),this.setTransitioning(!0);var r="scroll"+(o[0].toUpperCase()+o.slice(1)),s=m.getTransitionDurationFromElement(this._element);p(this._element).one(m.TRANSITION_END,function(){p(n._element).removeClass(Ee).addClass(ye).addClass(ve),n._element.style[o]="",n.setTransitioning(!1),p(n._element).trigger(_e.SHOWN)}).emulateTransitionEnd(s),this._element.style[o]=this._element[r]+"px"}}},e.hide=function(){var e=this;if(!this._isTransitioning&&p(this._element).hasClass(ve)){var t=p.Event(_e.HIDE);if(p(this._element).trigger(t),!t.isDefaultPrevented()){var n=this._getDimension();this._element.style[n]=this._element.getBoundingClientRect()[n]+"px",m.reflow(this._element),p(this._element).addClass(Ee).removeClass(ye).removeClass(ve);var i=this._triggerArray.length;if(0<i)for(var o=0;o<i;o++){var r=this._triggerArray[o],s=m.getSelectorFromElement(r);if(null!==s)p([].slice.call(document.querySelectorAll(s))).hasClass(ve)||p(r).addClass(be).attr("aria-expanded",!1)}this.setTransitioning(!0);this._element.style[n]="";var a=m.getTransitionDurationFromElement(this._element);p(this._element).one(m.TRANSITION_END,function(){e.setTransitioning(!1),p(e._element).removeClass(Ee).addClass(ye).trigger(_e.HIDDEN)}).emulateTransitionEnd(a)}}},e.setTransitioning=function(e){this._isTransitioning=e},e.dispose=function(){p.removeData(this._element,fe),this._config=null,this._parent=null,this._element=null,this._triggerArray=null,this._isTransitioning=null},e._getConfig=function(e){return(e=l({},me,{},e)).toggle=Boolean(e.toggle),m.typeCheckConfig(ue,e,ge),e},e._getDimension=function(){return p(this._element).hasClass(we)?we:Te},e._getParent=function(){var e,n=this;m.isElement(this._config.parent)?(e=this._config.parent,"undefined"!=typeof this._config.parent.jquery&&(e=this._config.parent[0])):e=document.querySelector(this._config.parent);var t='[data-toggle="collapse"][data-parent="'+this._config.parent+'"]',i=[].slice.call(e.querySelectorAll(t));return p(i).each(function(e,t){n._addAriaAndCollapsedClass(a._getTargetFromElement(t),[t])}),e},e._addAriaAndCollapsedClass=function(e,t){var n=p(e).hasClass(ve);t.length&&p(t).toggleClass(be,!n).attr("aria-expanded",n)},a._getTargetFromElement=function(e){var t=m.getSelectorFromElement(e);return t?document.querySelector(t):null},a._jQueryInterface=function(i){return this.each(function(){var e=p(this),t=e.data(fe),n=l({},me,{},e.data(),{},"object"==typeof i&&i?i:{});if(!t&&n.toggle&&/show|hide/.test(i)&&(n.toggle=!1),t||(t=new a(this,n),e.data(fe,t)),"string"==typeof i){if("undefined"==typeof t[i])throw new TypeError('No method named "'+i+'"');t[i]()}})},s(a,null,[{key:"VERSION",get:function(){return"4.4.1"}},{key:"Default",get:function(){return me}}]),a}();p(document).on(_e.CLICK_DATA_API,Se,function(e){"A"===e.currentTarget.tagName&&e.preventDefault();var n=p(this),t=m.getSelectorFromElement(this),i=[].slice.call(document.querySelectorAll(t));p(i).each(function(){var e=p(this),t=e.data(fe)?"toggle":n.data();De._jQueryInterface.call(e,t)})}),p.fn[ue]=De._jQueryInterface,p.fn[ue].Constructor=De,p.fn[ue].noConflict=function(){return p.fn[ue]=pe,De._jQueryInterface};var Ie="undefined"!=typeof window&&"undefined"!=typeof document&&"undefined"!=typeof navigator,Ae=function(){for(var e=["Edge","Trident","Firefox"],t=0;t<e.length;t+=1)if(Ie&&0<=navigator.userAgent.indexOf(e[t]))return 1;return 0}();var Oe=Ie&&window.Promise?function(e){var t=!1;return function(){t||(t=!0,window.Promise.resolve().then(function(){t=!1,e()}))}}:function(e){var t=!1;return function(){t||(t=!0,setTimeout(function(){t=!1,e()},Ae))}};function Ne(e){return e&&"[object Function]"==={}.toString.call(e)}function ke(e,t){if(1!==e.nodeType)return[];var n=e.ownerDocument.defaultView.getComputedStyle(e,null);return t?n[t]:n}function Le(e){return"HTML"===e.nodeName?e:e.parentNode||e.host}function Pe(e){if(!e)return document.body;switch(e.nodeName){case"HTML":case"BODY":return e.ownerDocument.body;case"#document":return e.body}var t=ke(e),n=t.overflow,i=t.overflowX,o=t.overflowY;return/(auto|scroll|overlay)/.test(n+o+i)?e:Pe(Le(e))}function xe(e){return e&&e.referenceNode?e.referenceNode:e}var je=Ie&&!(!window.MSInputMethodContext||!document.documentMode),He=Ie&&/MSIE 10/.test(navigator.userAgent);function Re(e){return 11===e?je:10===e?He:je||He}function Fe(e){if(!e)return document.documentElement;for(var t=Re(10)?document.body:null,n=e.offsetParent||null;n===t&&e.nextElementSibling;)n=(e=e.nextElementSibling).offsetParent;var i=n&&n.nodeName;return i&&"BODY"!==i&&"HTML"!==i?-1!==["TH","TD","TABLE"].indexOf(n.nodeName)&&"static"===ke(n,"position")?Fe(n):n:e?e.ownerDocument.documentElement:document.documentElement}function Me(e){return null!==e.parentNode?Me(e.parentNode):e}function We(e,t){if(!(e&&e.nodeType&&t&&t.nodeType))return document.documentElement;var n=e.compareDocumentPosition(t)&Node.DOCUMENT_POSITION_FOLLOWING,i=n?e:t,o=n?t:e,r=document.createRange();r.setStart(i,0),r.setEnd(o,0);var s=r.commonAncestorContainer;if(e!==s&&t!==s||i.contains(o))return function(e){var t=e.nodeName;return"BODY"!==t&&("HTML"===t||Fe(e.firstElementChild)===e)}(s)?s:Fe(s);var a=Me(e);return a.host?We(a.host,t):We(e,Me(t).host)}function Ue(e,t){var n="top"===(1<arguments.length&&void 0!==t?t:"top")?"scrollTop":"scrollLeft",i=e.nodeName;if("BODY"!==i&&"HTML"!==i)return e[n];var o=e.ownerDocument.documentElement;return(e.ownerDocument.scrollingElement||o)[n]}function Be(e,t){var n="x"===t?"Left":"Top",i="Left"==n?"Right":"Bottom";return parseFloat(e["border"+n+"Width"],10)+parseFloat(e["border"+i+"Width"],10)}function qe(e,t,n,i){return Math.max(t["offset"+e],t["scroll"+e],n["client"+e],n["offset"+e],n["scroll"+e],Re(10)?parseInt(n["offset"+e])+parseInt(i["margin"+("Height"===e?"Top":"Left")])+parseInt(i["margin"+("Height"===e?"Bottom":"Right")]):0)}function Ke(e){var t=e.body,n=e.documentElement,i=Re(10)&&getComputedStyle(n);return{height:qe("Height",t,n,i),width:qe("Width",t,n,i)}}var Qe=function(e,t,n){return t&&Ve(e.prototype,t),n&&Ve(e,n),e};function Ve(e,t){for(var n=0;n<t.length;n++){var i=t[n];i.enumerable=i.enumerable||!1,i.configurable=!0,"value"in i&&(i.writable=!0),Object.defineProperty(e,i.key,i)}}function Ye(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}var ze=Object.assign||function(e){for(var t=1;t<arguments.length;t++){var n=arguments[t];for(var i in n)Object.prototype.hasOwnProperty.call(n,i)&&(e[i]=n[i])}return e};function Xe(e){return ze({},e,{right:e.left+e.width,bottom:e.top+e.height})}function Ge(e){var t={};try{if(Re(10)){t=e.getBoundingClientRect();var n=Ue(e,"top"),i=Ue(e,"left");t.top+=n,t.left+=i,t.bottom+=n,t.right+=i}else t=e.getBoundingClientRect()}catch(e){}var o={left:t.left,top:t.top,width:t.right-t.left,height:t.bottom-t.top},r="HTML"===e.nodeName?Ke(e.ownerDocument):{},s=r.width||e.clientWidth||o.width,a=r.height||e.clientHeight||o.height,l=e.offsetWidth-s,c=e.offsetHeight-a;if(l||c){var h=ke(e);l-=Be(h,"x"),c-=Be(h,"y"),o.width-=l,o.height-=c}return Xe(o)}function $e(e,t,n){var i=2<arguments.length&&void 0!==n&&n,o=Re(10),r="HTML"===t.nodeName,s=Ge(e),a=Ge(t),l=Pe(e),c=ke(t),h=parseFloat(c.borderTopWidth,10),u=parseFloat(c.borderLeftWidth,10);i&&r&&(a.top=Math.max(a.top,0),a.left=Math.max(a.left,0));var f=Xe({top:s.top-a.top-h,left:s.left-a.left-u,width:s.width,height:s.height});if(f.marginTop=0,f.marginLeft=0,!o&&r){var d=parseFloat(c.marginTop,10),p=parseFloat(c.marginLeft,10);f.top-=h-d,f.bottom-=h-d,f.left-=u-p,f.right-=u-p,f.marginTop=d,f.marginLeft=p}return(o&&!i?t.contains(l):t===l&&"BODY"!==l.nodeName)&&(f=function(e,t,n){var i=2<arguments.length&&void 0!==n&&n,o=Ue(t,"top"),r=Ue(t,"left"),s=i?-1:1;return e.top+=o*s,e.bottom+=o*s,e.left+=r*s,e.right+=r*s,e}(f,t)),f}function Je(e){if(!e||!e.parentElement||Re())return document.documentElement;for(var t=e.parentElement;t&&"none"===ke(t,"transform");)t=t.parentElement;return t||document.documentElement}function Ze(e,t,n,i,o){var r=4<arguments.length&&void 0!==o&&o,s={top:0,left:0},a=r?Je(e):We(e,xe(t));if("viewport"===i)s=function(e,t){var n=1<arguments.length&&void 0!==t&&t,i=e.ownerDocument.documentElement,o=$e(e,i),r=Math.max(i.clientWidth,window.innerWidth||0),s=Math.max(i.clientHeight,window.innerHeight||0),a=n?0:Ue(i),l=n?0:Ue(i,"left");return Xe({top:a-o.top+o.marginTop,left:l-o.left+o.marginLeft,width:r,height:s})}(a,r);else{var l=void 0;"scrollParent"===i?"BODY"===(l=Pe(Le(t))).nodeName&&(l=e.ownerDocument.documentElement):l="window"===i?e.ownerDocument.documentElement:i;var c=$e(l,a,r);if("HTML"!==l.nodeName||function e(t){var n=t.nodeName;if("BODY"===n||"HTML"===n)return!1;if("fixed"===ke(t,"position"))return!0;var i=Le(t);return!!i&&e(i)}(a))s=c;else{var h=Ke(e.ownerDocument),u=h.height,f=h.width;s.top+=c.top-c.marginTop,s.bottom=u+c.top,s.left+=c.left-c.marginLeft,s.right=f+c.left}}var d="number"==typeof(n=n||0);return s.left+=d?n:n.left||0,s.top+=d?n:n.top||0,s.right-=d?n:n.right||0,s.bottom-=d?n:n.bottom||0,s}function et(e,t,i,n,o,r){var s=5<arguments.length&&void 0!==r?r:0;if(-1===e.indexOf("auto"))return e;var a=Ze(i,n,s,o),l={top:{width:a.width,height:t.top-a.top},right:{width:a.right-t.right,height:a.height},bottom:{width:a.width,height:a.bottom-t.bottom},left:{width:t.left-a.left,height:a.height}},c=Object.keys(l).map(function(e){return ze({key:e},l[e],{area:function(e){return e.width*e.height}(l[e])})}).sort(function(e,t){return t.area-e.area}),h=c.filter(function(e){var t=e.width,n=e.height;return t>=i.clientWidth&&n>=i.clientHeight}),u=0<h.length?h[0].key:c[0].key,f=e.split("-")[1];return u+(f?"-"+f:"")}function tt(e,t,n,i){var o=3<arguments.length&&void 0!==i?i:null;return $e(n,o?Je(t):We(t,xe(n)),o)}function nt(e){var t=e.ownerDocument.defaultView.getComputedStyle(e),n=parseFloat(t.marginTop||0)+parseFloat(t.marginBottom||0),i=parseFloat(t.marginLeft||0)+parseFloat(t.marginRight||0);return{width:e.offsetWidth+i,height:e.offsetHeight+n}}function it(e){var t={left:"right",right:"left",bottom:"top",top:"bottom"};return e.replace(/left|right|bottom|top/g,function(e){return t[e]})}function ot(e,t,n){n=n.split("-")[0];var i=nt(e),o={width:i.width,height:i.height},r=-1!==["right","left"].indexOf(n),s=r?"top":"left",a=r?"left":"top",l=r?"height":"width",c=r?"width":"height";return o[s]=t[s]+t[l]/2-i[l]/2,o[a]=n===a?t[a]-i[c]:t[it(a)],o}function rt(e,t){return Array.prototype.find?e.find(t):e.filter(t)[0]}function st(e,n,t){return(void 0===t?e:e.slice(0,function(e,t,n){if(Array.prototype.findIndex)return e.findIndex(function(e){return e[t]===n});var i=rt(e,function(e){return e[t]===n});return e.indexOf(i)}(e,"name",t))).forEach(function(e){e.function&&console.warn("`modifier.function` is deprecated, use `modifier.fn`!");var t=e.function||e.fn;e.enabled&&Ne(t)&&(n.offsets.popper=Xe(n.offsets.popper),n.offsets.reference=Xe(n.offsets.reference),n=t(n,e))}),n}function at(e,n){return e.some(function(e){var t=e.name;return e.enabled&&t===n})}function lt(e){for(var t=[!1,"ms","Webkit","Moz","O"],n=e.charAt(0).toUpperCase()+e.slice(1),i=0;i<t.length;i++){var o=t[i],r=o?""+o+n:e;if("undefined"!=typeof document.body.style[r])return r}return null}function ct(e){var t=e.ownerDocument;return t?t.defaultView:window}function ht(e,t,n,i){n.updateBound=i,ct(e).addEventListener("resize",n.updateBound,{passive:!0});var o=Pe(e);return function e(t,n,i,o){var r="BODY"===t.nodeName,s=r?t.ownerDocument.defaultView:t;s.addEventListener(n,i,{passive:!0}),r||e(Pe(s.parentNode),n,i,o),o.push(s)}(o,"scroll",n.updateBound,n.scrollParents),n.scrollElement=o,n.eventsEnabled=!0,n}function ut(){this.state.eventsEnabled&&(cancelAnimationFrame(this.scheduleUpdate),this.state=function(e,t){return ct(e).removeEventListener("resize",t.updateBound),t.scrollParents.forEach(function(e){e.removeEventListener("scroll",t.updateBound)}),t.updateBound=null,t.scrollParents=[],t.scrollElement=null,t.eventsEnabled=!1,t}(this.reference,this.state))}function ft(e){return""!==e&&!isNaN(parseFloat(e))&&isFinite(e)}function dt(n,i){Object.keys(i).forEach(function(e){var t="";-1!==["width","height","top","right","bottom","left"].indexOf(e)&&ft(i[e])&&(t="px"),n.style[e]=i[e]+t})}function pt(e,t){function n(e){return e}var i=e.offsets,o=i.popper,r=i.reference,s=Math.round,a=Math.floor,l=s(r.width),c=s(o.width),h=-1!==["left","right"].indexOf(e.placement),u=-1!==e.placement.indexOf("-"),f=t?h||u||l%2==c%2?s:a:n,d=t?s:n;return{left:f(l%2==1&&c%2==1&&!u&&t?o.left-1:o.left),top:d(o.top),bottom:d(o.bottom),right:f(o.right)}}var mt=Ie&&/Firefox/i.test(navigator.userAgent);function gt(e,t,n){var i=rt(e,function(e){return e.name===t}),o=!!i&&e.some(function(e){return e.name===n&&e.enabled&&e.order<i.order});if(!o){var r="`"+t+"`",s="`"+n+"`";console.warn(s+" modifier is required by "+r+" modifier in order to work, be sure to include it before "+r+"!")}return o}var _t=["auto-start","auto","auto-end","top-start","top","top-end","right-start","right","right-end","bottom-end","bottom","bottom-start","left-end","left","left-start"],vt=_t.slice(3);function yt(e,t){var n=1<arguments.length&&void 0!==t&&t,i=vt.indexOf(e),o=vt.slice(i+1).concat(vt.slice(0,i));return n?o.reverse():o}var Et="flip",bt="clockwise",wt="counterclockwise";function Tt(e,o,r,t){var s=[0,0],a=-1!==["right","left"].indexOf(t),n=e.split(/(\+|\-)/).map(function(e){return e.trim()}),i=n.indexOf(rt(n,function(e){return-1!==e.search(/,|\s/)}));n[i]&&-1===n[i].indexOf(",")&&console.warn("Offsets separated by white space(s) are deprecated, use a comma (,) instead.");var l=/\s*,\s*|\s+/,c=-1!==i?[n.slice(0,i).concat([n[i].split(l)[0]]),[n[i].split(l)[1]].concat(n.slice(i+1))]:[n];return(c=c.map(function(e,t){var n=(1===t?!a:a)?"height":"width",i=!1;return e.reduce(function(e,t){return""===e[e.length-1]&&-1!==["+","-"].indexOf(t)?(e[e.length-1]=t,i=!0,e):i?(e[e.length-1]+=t,i=!1,e):e.concat(t)},[]).map(function(e){return function(e,t,n,i){var o=e.match(/((?:\-|\+)?\d*\.?\d*)(.*)/),r=+o[1],s=o[2];if(!r)return e;if(0!==s.indexOf("%"))return"vh"!==s&&"vw"!==s?r:("vh"===s?Math.max(document.documentElement.clientHeight,window.innerHeight||0):Math.max(document.documentElement.clientWidth,window.innerWidth||0))/100*r;var a=void 0;switch(s){case"%p":a=n;break;case"%":case"%r":default:a=i}return Xe(a)[t]/100*r}(e,n,o,r)})})).forEach(function(n,i){n.forEach(function(e,t){ft(e)&&(s[i]+=e*("-"===n[t-1]?-1:1))})}),s}var Ct={placement:"bottom",positionFixed:!1,eventsEnabled:!0,removeOnDestroy:!1,onCreate:function(){},onUpdate:function(){},modifiers:{shift:{order:100,enabled:!0,fn:function(e){var t=e.placement,n=t.split("-")[0],i=t.split("-")[1];if(i){var o=e.offsets,r=o.reference,s=o.popper,a=-1!==["bottom","top"].indexOf(n),l=a?"left":"top",c=a?"width":"height",h={start:Ye({},l,r[l]),end:Ye({},l,r[l]+r[c]-s[c])};e.offsets.popper=ze({},s,h[i])}return e}},offset:{order:200,enabled:!0,fn:function(e,t){var n=t.offset,i=e.placement,o=e.offsets,r=o.popper,s=o.reference,a=i.split("-")[0],l=void 0;return l=ft(+n)?[+n,0]:Tt(n,r,s,a),"left"===a?(r.top+=l[0],r.left-=l[1]):"right"===a?(r.top+=l[0],r.left+=l[1]):"top"===a?(r.left+=l[0],r.top-=l[1]):"bottom"===a&&(r.left+=l[0],r.top+=l[1]),e.popper=r,e},offset:0},preventOverflow:{order:300,enabled:!0,fn:function(e,i){var t=i.boundariesElement||Fe(e.instance.popper);e.instance.reference===t&&(t=Fe(t));var n=lt("transform"),o=e.instance.popper.style,r=o.top,s=o.left,a=o[n];o.top="",o.left="",o[n]="";var l=Ze(e.instance.popper,e.instance.reference,i.padding,t,e.positionFixed);o.top=r,o.left=s,o[n]=a,i.boundaries=l;var c=i.priority,h=e.offsets.popper,u={primary:function(e){var t=h[e];return h[e]<l[e]&&!i.escapeWithReference&&(t=Math.max(h[e],l[e])),Ye({},e,t)},secondary:function(e){var t="right"===e?"left":"top",n=h[t];return h[e]>l[e]&&!i.escapeWithReference&&(n=Math.min(h[t],l[e]-("right"===e?h.width:h.height))),Ye({},t,n)}};return c.forEach(function(e){var t=-1!==["left","top"].indexOf(e)?"primary":"secondary";h=ze({},h,u[t](e))}),e.offsets.popper=h,e},priority:["left","right","top","bottom"],padding:5,boundariesElement:"scrollParent"},keepTogether:{order:400,enabled:!0,fn:function(e){var t=e.offsets,n=t.popper,i=t.reference,o=e.placement.split("-")[0],r=Math.floor,s=-1!==["top","bottom"].indexOf(o),a=s?"right":"bottom",l=s?"left":"top",c=s?"width":"height";return n[a]<r(i[l])&&(e.offsets.popper[l]=r(i[l])-n[c]),n[l]>r(i[a])&&(e.offsets.popper[l]=r(i[a])),e}},arrow:{order:500,enabled:!0,fn:function(e,t){var n;if(!gt(e.instance.modifiers,"arrow","keepTogether"))return e;var i=t.element;if("string"==typeof i){if(!(i=e.instance.popper.querySelector(i)))return e}else if(!e.instance.popper.contains(i))return console.warn("WARNING: `arrow.element` must be child of its popper element!"),e;var o=e.placement.split("-")[0],r=e.offsets,s=r.popper,a=r.reference,l=-1!==["left","right"].indexOf(o),c=l?"height":"width",h=l?"Top":"Left",u=h.toLowerCase(),f=l?"left":"top",d=l?"bottom":"right",p=nt(i)[c];a[d]-p<s[u]&&(e.offsets.popper[u]-=s[u]-(a[d]-p)),a[u]+p>s[d]&&(e.offsets.popper[u]+=a[u]+p-s[d]),e.offsets.popper=Xe(e.offsets.popper);var m=a[u]+a[c]/2-p/2,g=ke(e.instance.popper),_=parseFloat(g["margin"+h],10),v=parseFloat(g["border"+h+"Width"],10),y=m-e.offsets.popper[u]-_-v;return y=Math.max(Math.min(s[c]-p,y),0),e.arrowElement=i,e.offsets.arrow=(Ye(n={},u,Math.round(y)),Ye(n,f,""),n),e},element:"[x-arrow]"},flip:{order:600,enabled:!0,fn:function(m,g){if(at(m.instance.modifiers,"inner"))return m;if(m.flipped&&m.placement===m.originalPlacement)return m;var _=Ze(m.instance.popper,m.instance.reference,g.padding,g.boundariesElement,m.positionFixed),v=m.placement.split("-")[0],y=it(v),E=m.placement.split("-")[1]||"",b=[];switch(g.behavior){case Et:b=[v,y];break;case bt:b=yt(v);break;case wt:b=yt(v,!0);break;default:b=g.behavior}return b.forEach(function(e,t){if(v!==e||b.length===t+1)return m;v=m.placement.split("-")[0],y=it(v);var n=m.offsets.popper,i=m.offsets.reference,o=Math.floor,r="left"===v&&o(n.right)>o(i.left)||"right"===v&&o(n.left)<o(i.right)||"top"===v&&o(n.bottom)>o(i.top)||"bottom"===v&&o(n.top)<o(i.bottom),s=o(n.left)<o(_.left),a=o(n.right)>o(_.right),l=o(n.top)<o(_.top),c=o(n.bottom)>o(_.bottom),h="left"===v&&s||"right"===v&&a||"top"===v&&l||"bottom"===v&&c,u=-1!==["top","bottom"].indexOf(v),f=!!g.flipVariations&&(u&&"start"===E&&s||u&&"end"===E&&a||!u&&"start"===E&&l||!u&&"end"===E&&c),d=!!g.flipVariationsByContent&&(u&&"start"===E&&a||u&&"end"===E&&s||!u&&"start"===E&&c||!u&&"end"===E&&l),p=f||d;(r||h||p)&&(m.flipped=!0,(r||h)&&(v=b[t+1]),p&&(E=function(e){return"end"===e?"start":"start"===e?"end":e}(E)),m.placement=v+(E?"-"+E:""),m.offsets.popper=ze({},m.offsets.popper,ot(m.instance.popper,m.offsets.reference,m.placement)),m=st(m.instance.modifiers,m,"flip"))}),m},behavior:"flip",padding:5,boundariesElement:"viewport",flipVariations:!1,flipVariationsByContent:!1},inner:{order:700,enabled:!1,fn:function(e){var t=e.placement,n=t.split("-")[0],i=e.offsets,o=i.popper,r=i.reference,s=-1!==["left","right"].indexOf(n),a=-1===["top","left"].indexOf(n);return o[s?"left":"top"]=r[n]-(a?o[s?"width":"height"]:0),e.placement=it(t),e.offsets.popper=Xe(o),e}},hide:{order:800,enabled:!0,fn:function(e){if(!gt(e.instance.modifiers,"hide","preventOverflow"))return e;var t=e.offsets.reference,n=rt(e.instance.modifiers,function(e){return"preventOverflow"===e.name}).boundaries;if(t.bottom<n.top||t.left>n.right||t.top>n.bottom||t.right<n.left){if(!0===e.hide)return e;e.hide=!0,e.attributes["x-out-of-boundaries"]=""}else{if(!1===e.hide)return e;e.hide=!1,e.attributes["x-out-of-boundaries"]=!1}return e}},computeStyle:{order:850,enabled:!0,fn:function(e,t){var n=t.x,i=t.y,o=e.offsets.popper,r=rt(e.instance.modifiers,function(e){return"applyStyle"===e.name}).gpuAcceleration;void 0!==r&&console.warn("WARNING: `gpuAcceleration` option moved to `computeStyle` modifier and will not be supported in future versions of Popper.js!");var s=void 0!==r?r:t.gpuAcceleration,a=Fe(e.instance.popper),l=Ge(a),c={position:o.position},h=pt(e,window.devicePixelRatio<2||!mt),u="bottom"===n?"top":"bottom",f="right"===i?"left":"right",d=lt("transform"),p=void 0,m=void 0;if(m="bottom"==u?"HTML"===a.nodeName?-a.clientHeight+h.bottom:-l.height+h.bottom:h.top,p="right"==f?"HTML"===a.nodeName?-a.clientWidth+h.right:-l.width+h.right:h.left,s&&d)c[d]="translate3d("+p+"px, "+m+"px, 0)",c[u]=0,c[f]=0,c.willChange="transform";else{var g="bottom"==u?-1:1,_="right"==f?-1:1;c[u]=m*g,c[f]=p*_,c.willChange=u+", "+f}var v={"x-placement":e.placement};return e.attributes=ze({},v,e.attributes),e.styles=ze({},c,e.styles),e.arrowStyles=ze({},e.offsets.arrow,e.arrowStyles),e},gpuAcceleration:!0,x:"bottom",y:"right"},applyStyle:{order:900,enabled:!0,fn:function(e){return dt(e.instance.popper,e.styles),function(t,n){Object.keys(n).forEach(function(e){!1!==n[e]?t.setAttribute(e,n[e]):t.removeAttribute(e)})}(e.instance.popper,e.attributes),e.arrowElement&&Object.keys(e.arrowStyles).length&&dt(e.arrowElement,e.arrowStyles),e},onLoad:function(e,t,n,i,o){var r=tt(o,t,e,n.positionFixed),s=et(n.placement,r,t,e,n.modifiers.flip.boundariesElement,n.modifiers.flip.padding);return t.setAttribute("x-placement",s),dt(t,{position:n.positionFixed?"fixed":"absolute"}),n},gpuAcceleration:void 0}}},St=(Qe(Dt,[{key:"update",value:function(){return function(){if(!this.state.isDestroyed){var e={instance:this,styles:{},arrowStyles:{},attributes:{},flipped:!1,offsets:{}};e.offsets.reference=tt(this.state,this.popper,this.reference,this.options.positionFixed),e.placement=et(this.options.placement,e.offsets.reference,this.popper,this.reference,this.options.modifiers.flip.boundariesElement,this.options.modifiers.flip.padding),e.originalPlacement=e.placement,e.positionFixed=this.options.positionFixed,e.offsets.popper=ot(this.popper,e.offsets.reference,e.placement),e.offsets.popper.position=this.options.positionFixed?"fixed":"absolute",e=st(this.modifiers,e),this.state.isCreated?this.options.onUpdate(e):(this.state.isCreated=!0,this.options.onCreate(e))}}.call(this)}},{key:"destroy",value:function(){return function(){return this.state.isDestroyed=!0,at(this.modifiers,"applyStyle")&&(this.popper.removeAttribute("x-placement"),this.popper.style.position="",this.popper.style.top="",this.popper.style.left="",this.popper.style.right="",this.popper.style.bottom="",this.popper.style.willChange="",this.popper.style[lt("transform")]=""),this.disableEventListeners(),this.options.removeOnDestroy&&this.popper.parentNode.removeChild(this.popper),this}.call(this)}},{key:"enableEventListeners",value:function(){return function(){this.state.eventsEnabled||(this.state=ht(this.reference,this.options,this.state,this.scheduleUpdate))}.call(this)}},{key:"disableEventListeners",value:function(){return ut.call(this)}}]),Dt);function Dt(e,t){var n=this,i=2<arguments.length&&void 0!==arguments[2]?arguments[2]:{};!function(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}(this,Dt),this.scheduleUpdate=function(){return requestAnimationFrame(n.update)},this.update=Oe(this.update.bind(this)),this.options=ze({},Dt.Defaults,i),this.state={isDestroyed:!1,isCreated:!1,scrollParents:[]},this.reference=e&&e.jquery?e[0]:e,this.popper=t&&t.jquery?t[0]:t,this.options.modifiers={},Object.keys(ze({},Dt.Defaults.modifiers,i.modifiers)).forEach(function(e){n.options.modifiers[e]=ze({},Dt.Defaults.modifiers[e]||{},i.modifiers?i.modifiers[e]:{})}),this.modifiers=Object.keys(this.options.modifiers).map(function(e){return ze({name:e},n.options.modifiers[e])}).sort(function(e,t){return e.order-t.order}),this.modifiers.forEach(function(e){e.enabled&&Ne(e.onLoad)&&e.onLoad(n.reference,n.popper,n.options,e,n.state)}),this.update();var o=this.options.eventsEnabled;o&&this.enableEventListeners(),this.state.eventsEnabled=o}St.Utils=("undefined"!=typeof window?window:global).PopperUtils,St.placements=_t,St.Defaults=Ct;var It="dropdown",At="bs.dropdown",Ot="."+At,Nt=".data-api",kt=p.fn[It],Lt=new RegExp("38|40|27"),Pt={HIDE:"hide"+Ot,HIDDEN:"hidden"+Ot,SHOW:"show"+Ot,SHOWN:"shown"+Ot,CLICK:"click"+Ot,CLICK_DATA_API:"click"+Ot+Nt,KEYDOWN_DATA_API:"keydown"+Ot+Nt,KEYUP_DATA_API:"keyup"+Ot+Nt},xt="disabled",jt="show",Ht="dropup",Rt="dropright",Ft="dropleft",Mt="dropdown-menu-right",Wt="position-static",Ut='[data-toggle="dropdown"]',Bt=".dropdown form",qt=".dropdown-menu",Kt=".navbar-nav",Qt=".dropdown-menu .dropdown-item:not(.disabled):not(:disabled)",Vt="top-start",Yt="top-end",zt="bottom-start",Xt="bottom-end",Gt="right-start",$t="left-start",Jt={offset:0,flip:!0,boundary:"scrollParent",reference:"toggle",display:"dynamic",popperConfig:null},Zt={offset:"(number|string|function)",flip:"boolean",boundary:"(string|element)",reference:"(string|element)",display:"string",popperConfig:"(null|object)"},en=function(){function c(e,t){this._element=e,this._popper=null,this._config=this._getConfig(t),this._menu=this._getMenuElement(),this._inNavbar=this._detectNavbar(),this._addEventListeners()}var e=c.prototype;return e.toggle=function(){if(!this._element.disabled&&!p(this._element).hasClass(xt)){var e=p(this._menu).hasClass(jt);c._clearMenus(),e||this.show(!0)}},e.show=function(e){if(void 0===e&&(e=!1),!(this._element.disabled||p(this._element).hasClass(xt)||p(this._menu).hasClass(jt))){var t={relatedTarget:this._element},n=p.Event(Pt.SHOW,t),i=c._getParentFromElement(this._element);if(p(i).trigger(n),!n.isDefaultPrevented()){if(!this._inNavbar&&e){if("undefined"==typeof St)throw new TypeError("Bootstrap's dropdowns require Popper.js (https://popper.js.org/)");var o=this._element;"parent"===this._config.reference?o=i:m.isElement(this._config.reference)&&(o=this._config.reference,"undefined"!=typeof this._config.reference.jquery&&(o=this._config.reference[0])),"scrollParent"!==this._config.boundary&&p(i).addClass(Wt),this._popper=new St(o,this._menu,this._getPopperConfig())}"ontouchstart"in document.documentElement&&0===p(i).closest(Kt).length&&p(document.body).children().on("mouseover",null,p.noop),this._element.focus(),this._element.setAttribute("aria-expanded",!0),p(this._menu).toggleClass(jt),p(i).toggleClass(jt).trigger(p.Event(Pt.SHOWN,t))}}},e.hide=function(){if(!this._element.disabled&&!p(this._element).hasClass(xt)&&p(this._menu).hasClass(jt)){var e={relatedTarget:this._element},t=p.Event(Pt.HIDE,e),n=c._getParentFromElement(this._element);p(n).trigger(t),t.isDefaultPrevented()||(this._popper&&this._popper.destroy(),p(this._menu).toggleClass(jt),p(n).toggleClass(jt).trigger(p.Event(Pt.HIDDEN,e)))}},e.dispose=function(){p.removeData(this._element,At),p(this._element).off(Ot),this._element=null,(this._menu=null)!==this._popper&&(this._popper.destroy(),this._popper=null)},e.update=function(){this._inNavbar=this._detectNavbar(),null!==this._popper&&this._popper.scheduleUpdate()},e._addEventListeners=function(){var t=this;p(this._element).on(Pt.CLICK,function(e){e.preventDefault(),e.stopPropagation(),t.toggle()})},e._getConfig=function(e){return e=l({},this.constructor.Default,{},p(this._element).data(),{},e),m.typeCheckConfig(It,e,this.constructor.DefaultType),e},e._getMenuElement=function(){if(!this._menu){var e=c._getParentFromElement(this._element);e&&(this._menu=e.querySelector(qt))}return this._menu},e._getPlacement=function(){var e=p(this._element.parentNode),t=zt;return e.hasClass(Ht)?(t=Vt,p(this._menu).hasClass(Mt)&&(t=Yt)):e.hasClass(Rt)?t=Gt:e.hasClass(Ft)?t=$t:p(this._menu).hasClass(Mt)&&(t=Xt),t},e._detectNavbar=function(){return 0<p(this._element).closest(".navbar").length},e._getOffset=function(){var t=this,e={};return"function"==typeof this._config.offset?e.fn=function(e){return e.offsets=l({},e.offsets,{},t._config.offset(e.offsets,t._element)||{}),e}:e.offset=this._config.offset,e},e._getPopperConfig=function(){var e={placement:this._getPlacement(),modifiers:{offset:this._getOffset(),flip:{enabled:this._config.flip},preventOverflow:{boundariesElement:this._config.boundary}}};return"static"===this._config.display&&(e.modifiers.applyStyle={enabled:!1}),l({},e,{},this._config.popperConfig)},c._jQueryInterface=function(t){return this.each(function(){var e=p(this).data(At);if(e||(e=new c(this,"object"==typeof t?t:null),p(this).data(At,e)),"string"==typeof t){if("undefined"==typeof e[t])throw new TypeError('No method named "'+t+'"');e[t]()}})},c._clearMenus=function(e){if(!e||3!==e.which&&("keyup"!==e.type||9===e.which))for(var t=[].slice.call(document.querySelectorAll(Ut)),n=0,i=t.length;n<i;n++){var o=c._getParentFromElement(t[n]),r=p(t[n]).data(At),s={relatedTarget:t[n]};if(e&&"click"===e.type&&(s.clickEvent=e),r){var a=r._menu;if(p(o).hasClass(jt)&&!(e&&("click"===e.type&&/input|textarea/i.test(e.target.tagName)||"keyup"===e.type&&9===e.which)&&p.contains(o,e.target))){var l=p.Event(Pt.HIDE,s);p(o).trigger(l),l.isDefaultPrevented()||("ontouchstart"in document.documentElement&&p(document.body).children().off("mouseover",null,p.noop),t[n].setAttribute("aria-expanded","false"),r._popper&&r._popper.destroy(),p(a).removeClass(jt),p(o).removeClass(jt).trigger(p.Event(Pt.HIDDEN,s)))}}}},c._getParentFromElement=function(e){var t,n=m.getSelectorFromElement(e);return n&&(t=document.querySelector(n)),t||e.parentNode},c._dataApiKeydownHandler=function(e){if((/input|textarea/i.test(e.target.tagName)?!(32===e.which||27!==e.which&&(40!==e.which&&38!==e.which||p(e.target).closest(qt).length)):Lt.test(e.which))&&(e.preventDefault(),e.stopPropagation(),!this.disabled&&!p(this).hasClass(xt))){var t=c._getParentFromElement(this),n=p(t).hasClass(jt);if(n||27!==e.which)if(n&&(!n||27!==e.which&&32!==e.which)){var i=[].slice.call(t.querySelectorAll(Qt)).filter(function(e){return p(e).is(":visible")});if(0!==i.length){var o=i.indexOf(e.target);38===e.which&&0<o&&o--,40===e.which&&o<i.length-1&&o++,o<0&&(o=0),i[o].focus()}}else{if(27===e.which){var r=t.querySelector(Ut);p(r).trigger("focus")}p(this).trigger("click")}}},s(c,null,[{key:"VERSION",get:function(){return"4.4.1"}},{key:"Default",get:function(){return Jt}},{key:"DefaultType",get:function(){return Zt}}]),c}();p(document).on(Pt.KEYDOWN_DATA_API,Ut,en._dataApiKeydownHandler).on(Pt.KEYDOWN_DATA_API,qt,en._dataApiKeydownHandler).on(Pt.CLICK_DATA_API+" "+Pt.KEYUP_DATA_API,en._clearMenus).on(Pt.CLICK_DATA_API,Ut,function(e){e.preventDefault(),e.stopPropagation(),en._jQueryInterface.call(p(this),"toggle")}).on(Pt.CLICK_DATA_API,Bt,function(e){e.stopPropagation()}),p.fn[It]=en._jQueryInterface,p.fn[It].Constructor=en,p.fn[It].noConflict=function(){return p.fn[It]=kt,en._jQueryInterface};var tn="modal",nn="bs.modal",on="."+nn,rn=p.fn[tn],sn={backdrop:!0,keyboard:!0,focus:!0,show:!0},an={backdrop:"(boolean|string)",keyboard:"boolean",focus:"boolean",show:"boolean"},ln={HIDE:"hide"+on,HIDE_PREVENTED:"hidePrevented"+on,HIDDEN:"hidden"+on,SHOW:"show"+on,SHOWN:"shown"+on,FOCUSIN:"focusin"+on,RESIZE:"resize"+on,CLICK_DISMISS:"click.dismiss"+on,KEYDOWN_DISMISS:"keydown.dismiss"+on,MOUSEUP_DISMISS:"mouseup.dismiss"+on,MOUSEDOWN_DISMISS:"mousedown.dismiss"+on,CLICK_DATA_API:"click"+on+".data-api"},cn="modal-dialog-scrollable",hn="modal-scrollbar-measure",un="modal-backdrop",fn="modal-open",dn="fade",pn="show",mn="modal-static",gn=".modal-dialog",_n=".modal-body",vn='[data-toggle="modal"]',yn='[data-dismiss="modal"]',En=".fixed-top, .fixed-bottom, .is-fixed, .sticky-top",bn=".sticky-top",wn=function(){function o(e,t){this._config=this._getConfig(t),this._element=e,this._dialog=e.querySelector(gn),this._backdrop=null,this._isShown=!1,this._isBodyOverflowing=!1,this._ignoreBackdropClick=!1,this._isTransitioning=!1,this._scrollbarWidth=0}var e=o.prototype;return e.toggle=function(e){return this._isShown?this.hide():this.show(e)},e.show=function(e){var t=this;if(!this._isShown&&!this._isTransitioning){p(this._element).hasClass(dn)&&(this._isTransitioning=!0);var n=p.Event(ln.SHOW,{relatedTarget:e});p(this._element).trigger(n),this._isShown||n.isDefaultPrevented()||(this._isShown=!0,this._checkScrollbar(),this._setScrollbar(),this._adjustDialog(),this._setEscapeEvent(),this._setResizeEvent(),p(this._element).on(ln.CLICK_DISMISS,yn,function(e){return t.hide(e)}),p(this._dialog).on(ln.MOUSEDOWN_DISMISS,function(){p(t._element).one(ln.MOUSEUP_DISMISS,function(e){p(e.target).is(t._element)&&(t._ignoreBackdropClick=!0)})}),this._showBackdrop(function(){return t._showElement(e)}))}},e.hide=function(e){var t=this;if(e&&e.preventDefault(),this._isShown&&!this._isTransitioning){var n=p.Event(ln.HIDE);if(p(this._element).trigger(n),this._isShown&&!n.isDefaultPrevented()){this._isShown=!1;var i=p(this._element).hasClass(dn);if(i&&(this._isTransitioning=!0),this._setEscapeEvent(),this._setResizeEvent(),p(document).off(ln.FOCUSIN),p(this._element).removeClass(pn),p(this._element).off(ln.CLICK_DISMISS),p(this._dialog).off(ln.MOUSEDOWN_DISMISS),i){var o=m.getTransitionDurationFromElement(this._element);p(this._element).one(m.TRANSITION_END,function(e){return t._hideModal(e)}).emulateTransitionEnd(o)}else this._hideModal()}}},e.dispose=function(){[window,this._element,this._dialog].forEach(function(e){return p(e).off(on)}),p(document).off(ln.FOCUSIN),p.removeData(this._element,nn),this._config=null,this._element=null,this._dialog=null,this._backdrop=null,this._isShown=null,this._isBodyOverflowing=null,this._ignoreBackdropClick=null,this._isTransitioning=null,this._scrollbarWidth=null},e.handleUpdate=function(){this._adjustDialog()},e._getConfig=function(e){return e=l({},sn,{},e),m.typeCheckConfig(tn,e,an),e},e._triggerBackdropTransition=function(){var e=this;if("static"===this._config.backdrop){var t=p.Event(ln.HIDE_PREVENTED);if(p(this._element).trigger(t),t.defaultPrevented)return;this._element.classList.add(mn);var n=m.getTransitionDurationFromElement(this._element);p(this._element).one(m.TRANSITION_END,function(){e._element.classList.remove(mn)}).emulateTransitionEnd(n),this._element.focus()}else this.hide()},e._showElement=function(e){var t=this,n=p(this._element).hasClass(dn),i=this._dialog?this._dialog.querySelector(_n):null;this._element.parentNode&&this._element.parentNode.nodeType===Node.ELEMENT_NODE||document.body.appendChild(this._element),this._element.style.display="block",this._element.removeAttribute("aria-hidden"),this._element.setAttribute("aria-modal",!0),p(this._dialog).hasClass(cn)&&i?i.scrollTop=0:this._element.scrollTop=0,n&&m.reflow(this._element),p(this._element).addClass(pn),this._config.focus&&this._enforceFocus();function o(){t._config.focus&&t._element.focus(),t._isTransitioning=!1,p(t._element).trigger(r)}var r=p.Event(ln.SHOWN,{relatedTarget:e});if(n){var s=m.getTransitionDurationFromElement(this._dialog);p(this._dialog).one(m.TRANSITION_END,o).emulateTransitionEnd(s)}else o()},e._enforceFocus=function(){var t=this;p(document).off(ln.FOCUSIN).on(ln.FOCUSIN,function(e){document!==e.target&&t._element!==e.target&&0===p(t._element).has(e.target).length&&t._element.focus()})},e._setEscapeEvent=function(){var t=this;this._isShown&&this._config.keyboard?p(this._element).on(ln.KEYDOWN_DISMISS,function(e){27===e.which&&t._triggerBackdropTransition()}):this._isShown||p(this._element).off(ln.KEYDOWN_DISMISS)},e._setResizeEvent=function(){var t=this;this._isShown?p(window).on(ln.RESIZE,function(e){return t.handleUpdate(e)}):p(window).off(ln.RESIZE)},e._hideModal=function(){var e=this;this._element.style.display="none",this._element.setAttribute("aria-hidden",!0),this._element.removeAttribute("aria-modal"),this._isTransitioning=!1,this._showBackdrop(function(){p(document.body).removeClass(fn),e._resetAdjustments(),e._resetScrollbar(),p(e._element).trigger(ln.HIDDEN)})},e._removeBackdrop=function(){this._backdrop&&(p(this._backdrop).remove(),this._backdrop=null)},e._showBackdrop=function(e){var t=this,n=p(this._element).hasClass(dn)?dn:"";if(this._isShown&&this._config.backdrop){if(this._backdrop=document.createElement("div"),this._backdrop.className=un,n&&this._backdrop.classList.add(n),p(this._backdrop).appendTo(document.body),p(this._element).on(ln.CLICK_DISMISS,function(e){t._ignoreBackdropClick?t._ignoreBackdropClick=!1:e.target===e.currentTarget&&t._triggerBackdropTransition()}),n&&m.reflow(this._backdrop),p(this._backdrop).addClass(pn),!e)return;if(!n)return void e();var i=m.getTransitionDurationFromElement(this._backdrop);p(this._backdrop).one(m.TRANSITION_END,e).emulateTransitionEnd(i)}else if(!this._isShown&&this._backdrop){p(this._backdrop).removeClass(pn);var o=function(){t._removeBackdrop(),e&&e()};if(p(this._element).hasClass(dn)){var r=m.getTransitionDurationFromElement(this._backdrop);p(this._backdrop).one(m.TRANSITION_END,o).emulateTransitionEnd(r)}else o()}else e&&e()},e._adjustDialog=function(){var e=this._element.scrollHeight>document.documentElement.clientHeight;!this._isBodyOverflowing&&e&&(this._element.style.paddingLeft=this._scrollbarWidth+"px"),this._isBodyOverflowing&&!e&&(this._element.style.paddingRight=this._scrollbarWidth+"px")},e._resetAdjustments=function(){this._element.style.paddingLeft="",this._element.style.paddingRight=""},e._checkScrollbar=function(){var e=document.body.getBoundingClientRect();this._isBodyOverflowing=e.left+e.right<window.innerWidth,this._scrollbarWidth=this._getScrollbarWidth()},e._setScrollbar=function(){var o=this;if(this._isBodyOverflowing){var e=[].slice.call(document.querySelectorAll(En)),t=[].slice.call(document.querySelectorAll(bn));p(e).each(function(e,t){var n=t.style.paddingRight,i=p(t).css("padding-right");p(t).data("padding-right",n).css("padding-right",parseFloat(i)+o._scrollbarWidth+"px")}),p(t).each(function(e,t){var n=t.style.marginRight,i=p(t).css("margin-right");p(t).data("margin-right",n).css("margin-right",parseFloat(i)-o._scrollbarWidth+"px")});var n=document.body.style.paddingRight,i=p(document.body).css("padding-right");p(document.body).data("padding-right",n).css("padding-right",parseFloat(i)+this._scrollbarWidth+"px")}p(document.body).addClass(fn)},e._resetScrollbar=function(){var e=[].slice.call(document.querySelectorAll(En));p(e).each(function(e,t){var n=p(t).data("padding-right");p(t).removeData("padding-right"),t.style.paddingRight=n||""});var t=[].slice.call(document.querySelectorAll(""+bn));p(t).each(function(e,t){var n=p(t).data("margin-right");"undefined"!=typeof n&&p(t).css("margin-right",n).removeData("margin-right")});var n=p(document.body).data("padding-right");p(document.body).removeData("padding-right"),document.body.style.paddingRight=n||""},e._getScrollbarWidth=function(){var e=document.createElement("div");e.className=hn,document.body.appendChild(e);var t=e.getBoundingClientRect().width-e.clientWidth;return document.body.removeChild(e),t},o._jQueryInterface=function(n,i){return this.each(function(){var e=p(this).data(nn),t=l({},sn,{},p(this).data(),{},"object"==typeof n&&n?n:{});if(e||(e=new o(this,t),p(this).data(nn,e)),"string"==typeof n){if("undefined"==typeof e[n])throw new TypeError('No method named "'+n+'"');e[n](i)}else t.show&&e.show(i)})},s(o,null,[{key:"VERSION",get:function(){return"4.4.1"}},{key:"Default",get:function(){return sn}}]),o}();p(document).on(ln.CLICK_DATA_API,vn,function(e){var t,n=this,i=m.getSelectorFromElement(this);i&&(t=document.querySelector(i));var o=p(t).data(nn)?"toggle":l({},p(t).data(),{},p(this).data());"A"!==this.tagName&&"AREA"!==this.tagName||e.preventDefault();var r=p(t).one(ln.SHOW,function(e){e.isDefaultPrevented()||r.one(ln.HIDDEN,function(){p(n).is(":visible")&&n.focus()})});wn._jQueryInterface.call(p(t),o,this)}),p.fn[tn]=wn._jQueryInterface,p.fn[tn].Constructor=wn,p.fn[tn].noConflict=function(){return p.fn[tn]=rn,wn._jQueryInterface};var Tn=["background","cite","href","itemtype","longdesc","poster","src","xlink:href"],Cn={"*":["class","dir","id","lang","role",/^aria-[\w-]*$/i],a:["target","href","title","rel"],area:[],b:[],br:[],col:[],code:[],div:[],em:[],hr:[],h1:[],h2:[],h3:[],h4:[],h5:[],h6:[],i:[],img:["src","alt","title","width","height"],li:[],ol:[],p:[],pre:[],s:[],small:[],span:[],sub:[],sup:[],strong:[],u:[],ul:[]},Sn=/^(?:(?:https?|mailto|ftp|tel|file):|[^&:/?#]*(?:[/?#]|$))/gi,Dn=/^data:(?:image\/(?:bmp|gif|jpeg|jpg|png|tiff|webp)|video\/(?:mpeg|mp4|ogg|webm)|audio\/(?:mp3|oga|ogg|opus));base64,[a-z0-9+/]+=*$/i;function In(e,r,t){if(0===e.length)return e;if(t&&"function"==typeof t)return t(e);for(var n=(new window.DOMParser).parseFromString(e,"text/html"),s=Object.keys(r),a=[].slice.call(n.body.querySelectorAll("*")),i=function(e){var t=a[e],n=t.nodeName.toLowerCase();if(-1===s.indexOf(t.nodeName.toLowerCase()))return t.parentNode.removeChild(t),"continue";var i=[].slice.call(t.attributes),o=[].concat(r["*"]||[],r[n]||[]);i.forEach(function(e){!function(e,t){var n=e.nodeName.toLowerCase();if(-1!==t.indexOf(n))return-1===Tn.indexOf(n)||Boolean(e.nodeValue.match(Sn)||e.nodeValue.match(Dn));for(var i=t.filter(function(e){return e instanceof RegExp}),o=0,r=i.length;o<r;o++)if(n.match(i[o]))return!0;return!1}(e,o)&&t.removeAttribute(e.nodeName)})},o=0,l=a.length;o<l;o++)i(o);return n.body.innerHTML}var An="tooltip",On="bs.tooltip",Nn="."+On,kn=p.fn[An],Ln="bs-tooltip",Pn=new RegExp("(^|\\s)"+Ln+"\\S+","g"),xn=["sanitize","whiteList","sanitizeFn"],jn={animation:"boolean",template:"string",title:"(string|element|function)",trigger:"string",delay:"(number|object)",html:"boolean",selector:"(string|boolean)",placement:"(string|function)",offset:"(number|string|function)",container:"(string|element|boolean)",fallbackPlacement:"(string|array)",boundary:"(string|element)",sanitize:"boolean",sanitizeFn:"(null|function)",whiteList:"object",popperConfig:"(null|object)"},Hn={AUTO:"auto",TOP:"top",RIGHT:"right",BOTTOM:"bottom",LEFT:"left"},Rn={animation:!0,template:'<div class="tooltip" role="tooltip"><div class="arrow"></div><div class="tooltip-inner"></div></div>',trigger:"hover focus",title:"",delay:0,html:!1,selector:!1,placement:"top",offset:0,container:!1,fallbackPlacement:"flip",boundary:"scrollParent",sanitize:!0,sanitizeFn:null,whiteList:Cn,popperConfig:null},Fn="show",Mn="out",Wn={HIDE:"hide"+Nn,HIDDEN:"hidden"+Nn,SHOW:"show"+Nn,SHOWN:"shown"+Nn,INSERTED:"inserted"+Nn,CLICK:"click"+Nn,FOCUSIN:"focusin"+Nn,FOCUSOUT:"focusout"+Nn,MOUSEENTER:"mouseenter"+Nn,MOUSELEAVE:"mouseleave"+Nn},Un="fade",Bn="show",qn=".tooltip-inner",Kn=".arrow",Qn="hover",Vn="focus",Yn="click",zn="manual",Xn=function(){function i(e,t){if("undefined"==typeof St)throw new TypeError("Bootstrap's tooltips require Popper.js (https://popper.js.org/)");this._isEnabled=!0,this._timeout=0,this._hoverState="",this._activeTrigger={},this._popper=null,this.element=e,this.config=this._getConfig(t),this.tip=null,this._setListeners()}var e=i.prototype;return e.enable=function(){this._isEnabled=!0},e.disable=function(){this._isEnabled=!1},e.toggleEnabled=function(){this._isEnabled=!this._isEnabled},e.toggle=function(e){if(this._isEnabled)if(e){var t=this.constructor.DATA_KEY,n=p(e.currentTarget).data(t);n||(n=new this.constructor(e.currentTarget,this._getDelegateConfig()),p(e.currentTarget).data(t,n)),n._activeTrigger.click=!n._activeTrigger.click,n._isWithActiveTrigger()?n._enter(null,n):n._leave(null,n)}else{if(p(this.getTipElement()).hasClass(Bn))return void this._leave(null,this);this._enter(null,this)}},e.dispose=function(){clearTimeout(this._timeout),p.removeData(this.element,this.constructor.DATA_KEY),p(this.element).off(this.constructor.EVENT_KEY),p(this.element).closest(".modal").off("hide.bs.modal",this._hideModalHandler),this.tip&&p(this.tip).remove(),this._isEnabled=null,this._timeout=null,this._hoverState=null,this._activeTrigger=null,this._popper&&this._popper.destroy(),this._popper=null,this.element=null,this.config=null,this.tip=null},e.show=function(){var t=this;if("none"===p(this.element).css("display"))throw new Error("Please use show on visible elements");var e=p.Event(this.constructor.Event.SHOW);if(this.isWithContent()&&this._isEnabled){p(this.element).trigger(e);var n=m.findShadowRoot(this.element),i=p.contains(null!==n?n:this.element.ownerDocument.documentElement,this.element);if(e.isDefaultPrevented()||!i)return;var o=this.getTipElement(),r=m.getUID(this.constructor.NAME);o.setAttribute("id",r),this.element.setAttribute("aria-describedby",r),this.setContent(),this.config.animation&&p(o).addClass(Un);var s="function"==typeof this.config.placement?this.config.placement.call(this,o,this.element):this.config.placement,a=this._getAttachment(s);this.addAttachmentClass(a);var l=this._getContainer();p(o).data(this.constructor.DATA_KEY,this),p.contains(this.element.ownerDocument.documentElement,this.tip)||p(o).appendTo(l),p(this.element).trigger(this.constructor.Event.INSERTED),this._popper=new St(this.element,o,this._getPopperConfig(a)),p(o).addClass(Bn),"ontouchstart"in document.documentElement&&p(document.body).children().on("mouseover",null,p.noop);var c=function(){t.config.animation&&t._fixTransition();var e=t._hoverState;t._hoverState=null,p(t.element).trigger(t.constructor.Event.SHOWN),e===Mn&&t._leave(null,t)};if(p(this.tip).hasClass(Un)){var h=m.getTransitionDurationFromElement(this.tip);p(this.tip).one(m.TRANSITION_END,c).emulateTransitionEnd(h)}else c()}},e.hide=function(e){function t(){n._hoverState!==Fn&&i.parentNode&&i.parentNode.removeChild(i),n._cleanTipClass(),n.element.removeAttribute("aria-describedby"),p(n.element).trigger(n.constructor.Event.HIDDEN),null!==n._popper&&n._popper.destroy(),e&&e()}var n=this,i=this.getTipElement(),o=p.Event(this.constructor.Event.HIDE);if(p(this.element).trigger(o),!o.isDefaultPrevented()){if(p(i).removeClass(Bn),"ontouchstart"in document.documentElement&&p(document.body).children().off("mouseover",null,p.noop),this._activeTrigger[Yn]=!1,this._activeTrigger[Vn]=!1,this._activeTrigger[Qn]=!1,p(this.tip).hasClass(Un)){var r=m.getTransitionDurationFromElement(i);p(i).one(m.TRANSITION_END,t).emulateTransitionEnd(r)}else t();this._hoverState=""}},e.update=function(){null!==this._popper&&this._popper.scheduleUpdate()},e.isWithContent=function(){return Boolean(this.getTitle())},e.addAttachmentClass=function(e){p(this.getTipElement()).addClass(Ln+"-"+e)},e.getTipElement=function(){return this.tip=this.tip||p(this.config.template)[0],this.tip},e.setContent=function(){var e=this.getTipElement();this.setElementContent(p(e.querySelectorAll(qn)),this.getTitle()),p(e).removeClass(Un+" "+Bn)},e.setElementContent=function(e,t){"object"!=typeof t||!t.nodeType&&!t.jquery?this.config.html?(this.config.sanitize&&(t=In(t,this.config.whiteList,this.config.sanitizeFn)),e.html(t)):e.text(t):this.config.html?p(t).parent().is(e)||e.empty().append(t):e.text(p(t).text())},e.getTitle=function(){var e=this.element.getAttribute("data-original-title");return e=e||("function"==typeof this.config.title?this.config.title.call(this.element):this.config.title)},e._getPopperConfig=function(e){var t=this;return l({},{placement:e,modifiers:{offset:this._getOffset(),flip:{behavior:this.config.fallbackPlacement},arrow:{element:Kn},preventOverflow:{boundariesElement:this.config.boundary}},onCreate:function(e){e.originalPlacement!==e.placement&&t._handlePopperPlacementChange(e)},onUpdate:function(e){return t._handlePopperPlacementChange(e)}},{},this.config.popperConfig)},e._getOffset=function(){var t=this,e={};return"function"==typeof this.config.offset?e.fn=function(e){return e.offsets=l({},e.offsets,{},t.config.offset(e.offsets,t.element)||{}),e}:e.offset=this.config.offset,e},e._getContainer=function(){return!1===this.config.container?document.body:m.isElement(this.config.container)?p(this.config.container):p(document).find(this.config.container)},e._getAttachment=function(e){return Hn[e.toUpperCase()]},e._setListeners=function(){var i=this;this.config.trigger.split(" ").forEach(function(e){if("click"===e)p(i.element).on(i.constructor.Event.CLICK,i.config.selector,function(e){return i.toggle(e)});else if(e!==zn){var t=e===Qn?i.constructor.Event.MOUSEENTER:i.constructor.Event.FOCUSIN,n=e===Qn?i.constructor.Event.MOUSELEAVE:i.constructor.Event.FOCUSOUT;p(i.element).on(t,i.config.selector,function(e){return i._enter(e)}).on(n,i.config.selector,function(e){return i._leave(e)})}}),this._hideModalHandler=function(){i.element&&i.hide()},p(this.element).closest(".modal").on("hide.bs.modal",this._hideModalHandler),this.config.selector?this.config=l({},this.config,{trigger:"manual",selector:""}):this._fixTitle()},e._fixTitle=function(){var e=typeof this.element.getAttribute("data-original-title");!this.element.getAttribute("title")&&"string"==e||(this.element.setAttribute("data-original-title",this.element.getAttribute("title")||""),this.element.setAttribute("title",""))},e._enter=function(e,t){var n=this.constructor.DATA_KEY;(t=t||p(e.currentTarget).data(n))||(t=new this.constructor(e.currentTarget,this._getDelegateConfig()),p(e.currentTarget).data(n,t)),e&&(t._activeTrigger["focusin"===e.type?Vn:Qn]=!0),p(t.getTipElement()).hasClass(Bn)||t._hoverState===Fn?t._hoverState=Fn:(clearTimeout(t._timeout),t._hoverState=Fn,t.config.delay&&t.config.delay.show?t._timeout=setTimeout(function(){t._hoverState===Fn&&t.show()},t.config.delay.show):t.show())},e._leave=function(e,t){var n=this.constructor.DATA_KEY;(t=t||p(e.currentTarget).data(n))||(t=new this.constructor(e.currentTarget,this._getDelegateConfig()),p(e.currentTarget).data(n,t)),e&&(t._activeTrigger["focusout"===e.type?Vn:Qn]=!1),t._isWithActiveTrigger()||(clearTimeout(t._timeout),t._hoverState=Mn,t.config.delay&&t.config.delay.hide?t._timeout=setTimeout(function(){t._hoverState===Mn&&t.hide()},t.config.delay.hide):t.hide())},e._isWithActiveTrigger=function(){for(var e in this._activeTrigger)if(this._activeTrigger[e])return!0;return!1},e._getConfig=function(e){var t=p(this.element).data();return Object.keys(t).forEach(function(e){-1!==xn.indexOf(e)&&delete t[e]}),"number"==typeof(e=l({},this.constructor.Default,{},t,{},"object"==typeof e&&e?e:{})).delay&&(e.delay={show:e.delay,hide:e.delay}),"number"==typeof e.title&&(e.title=e.title.toString()),"number"==typeof e.content&&(e.content=e.content.toString()),m.typeCheckConfig(An,e,this.constructor.DefaultType),e.sanitize&&(e.template=In(e.template,e.whiteList,e.sanitizeFn)),e},e._getDelegateConfig=function(){var e={};if(this.config)for(var t in this.config)this.constructor.Default[t]!==this.config[t]&&(e[t]=this.config[t]);return e},e._cleanTipClass=function(){var e=p(this.getTipElement()),t=e.attr("class").match(Pn);null!==t&&t.length&&e.removeClass(t.join(""))},e._handlePopperPlacementChange=function(e){var t=e.instance;this.tip=t.popper,this._cleanTipClass(),this.addAttachmentClass(this._getAttachment(e.placement))},e._fixTransition=function(){var e=this.getTipElement(),t=this.config.animation;null===e.getAttribute("x-placement")&&(p(e).removeClass(Un),this.config.animation=!1,this.hide(),this.show(),this.config.animation=t)},i._jQueryInterface=function(n){return this.each(function(){var e=p(this).data(On),t="object"==typeof n&&n;if((e||!/dispose|hide/.test(n))&&(e||(e=new i(this,t),p(this).data(On,e)),"string"==typeof n)){if("undefined"==typeof e[n])throw new TypeError('No method named "'+n+'"');e[n]()}})},s(i,null,[{key:"VERSION",get:function(){return"4.4.1"}},{key:"Default",get:function(){return Rn}},{key:"NAME",get:function(){return An}},{key:"DATA_KEY",get:function(){return On}},{key:"Event",get:function(){return Wn}},{key:"EVENT_KEY",get:function(){return Nn}},{key:"DefaultType",get:function(){return jn}}]),i}();p.fn[An]=Xn._jQueryInterface,p.fn[An].Constructor=Xn,p.fn[An].noConflict=function(){return p.fn[An]=kn,Xn._jQueryInterface};var Gn="popover",$n="bs.popover",Jn="."+$n,Zn=p.fn[Gn],ei="bs-popover",ti=new RegExp("(^|\\s)"+ei+"\\S+","g"),ni=l({},Xn.Default,{placement:"right",trigger:"click",content:"",template:'<div class="popover" role="tooltip"><div class="arrow"></div><h3 class="popover-header"></h3><div class="popover-body"></div></div>'}),ii=l({},Xn.DefaultType,{content:"(string|element|function)"}),oi="fade",ri="show",si=".popover-header",ai=".popover-body",li={HIDE:"hide"+Jn,HIDDEN:"hidden"+Jn,SHOW:"show"+Jn,SHOWN:"shown"+Jn,INSERTED:"inserted"+Jn,CLICK:"click"+Jn,FOCUSIN:"focusin"+Jn,FOCUSOUT:"focusout"+Jn,MOUSEENTER:"mouseenter"+Jn,MOUSELEAVE:"mouseleave"+Jn},ci=function(e){function i(){return e.apply(this,arguments)||this}!function(e,t){e.prototype=Object.create(t.prototype),(e.prototype.constructor=e).__proto__=t}(i,e);var t=i.prototype;return t.isWithContent=function(){return this.getTitle()||this._getContent()},t.addAttachmentClass=function(e){p(this.getTipElement()).addClass(ei+"-"+e)},t.getTipElement=function(){return this.tip=this.tip||p(this.config.template)[0],this.tip},t.setContent=function(){var e=p(this.getTipElement());this.setElementContent(e.find(si),this.getTitle());var t=this._getContent();"function"==typeof t&&(t=t.call(this.element)),this.setElementContent(e.find(ai),t),e.removeClass(oi+" "+ri)},t._getContent=function(){return this.element.getAttribute("data-content")||this.config.content},t._cleanTipClass=function(){var e=p(this.getTipElement()),t=e.attr("class").match(ti);null!==t&&0<t.length&&e.removeClass(t.join(""))},i._jQueryInterface=function(n){return this.each(function(){var e=p(this).data($n),t="object"==typeof n?n:null;if((e||!/dispose|hide/.test(n))&&(e||(e=new i(this,t),p(this).data($n,e)),"string"==typeof n)){if("undefined"==typeof e[n])throw new TypeError('No method named "'+n+'"');e[n]()}})},s(i,null,[{key:"VERSION",get:function(){return"4.4.1"}},{key:"Default",get:function(){return ni}},{key:"NAME",get:function(){return Gn}},{key:"DATA_KEY",get:function(){return $n}},{key:"Event",get:function(){return li}},{key:"EVENT_KEY",get:function(){return Jn}},{key:"DefaultType",get:function(){return ii}}]),i}(Xn);p.fn[Gn]=ci._jQueryInterface,p.fn[Gn].Constructor=ci,p.fn[Gn].noConflict=function(){return p.fn[Gn]=Zn,ci._jQueryInterface};var hi="scrollspy",ui="bs.scrollspy",fi="."+ui,di=p.fn[hi],pi={offset:10,method:"auto",target:""},mi={offset:"number",method:"string",target:"(string|element)"},gi={ACTIVATE:"activate"+fi,SCROLL:"scroll"+fi,LOAD_DATA_API:"load"+fi+".data-api"},_i="dropdown-item",vi="active",yi='[data-spy="scroll"]',Ei=".nav, .list-group",bi=".nav-link",wi=".nav-item",Ti=".list-group-item",Ci=".dropdown",Si=".dropdown-item",Di=".dropdown-toggle",Ii="offset",Ai="position",Oi=function(){function n(e,t){var n=this;this._element=e,this._scrollElement="BODY"===e.tagName?window:e,this._config=this._getConfig(t),this._selector=this._config.target+" "+bi+","+this._config.target+" "+Ti+","+this._config.target+" "+Si,this._offsets=[],this._targets=[],this._activeTarget=null,this._scrollHeight=0,p(this._scrollElement).on(gi.SCROLL,function(e){return n._process(e)}),this.refresh(),this._process()}var e=n.prototype;return e.refresh=function(){var t=this,e=this._scrollElement===this._scrollElement.window?Ii:Ai,o="auto"===this._config.method?e:this._config.method,r=o===Ai?this._getScrollTop():0;this._offsets=[],this._targets=[],this._scrollHeight=this._getScrollHeight(),[].slice.call(document.querySelectorAll(this._selector)).map(function(e){var t,n=m.getSelectorFromElement(e);if(n&&(t=document.querySelector(n)),t){var i=t.getBoundingClientRect();if(i.width||i.height)return[p(t)[o]().top+r,n]}return null}).filter(function(e){return e}).sort(function(e,t){return e[0]-t[0]}).forEach(function(e){t._offsets.push(e[0]),t._targets.push(e[1])})},e.dispose=function(){p.removeData(this._element,ui),p(this._scrollElement).off(fi),this._element=null,this._scrollElement=null,this._config=null,this._selector=null,this._offsets=null,this._targets=null,this._activeTarget=null,this._scrollHeight=null},e._getConfig=function(e){if("string"!=typeof(e=l({},pi,{},"object"==typeof e&&e?e:{})).target){var t=p(e.target).attr("id");t||(t=m.getUID(hi),p(e.target).attr("id",t)),e.target="#"+t}return m.typeCheckConfig(hi,e,mi),e},e._getScrollTop=function(){return this._scrollElement===window?this._scrollElement.pageYOffset:this._scrollElement.scrollTop},e._getScrollHeight=function(){return this._scrollElement.scrollHeight||Math.max(document.body.scrollHeight,document.documentElement.scrollHeight)},e._getOffsetHeight=function(){return this._scrollElement===window?window.innerHeight:this._scrollElement.getBoundingClientRect().height},e._process=function(){var e=this._getScrollTop()+this._config.offset,t=this._getScrollHeight(),n=this._config.offset+t-this._getOffsetHeight();if(this._scrollHeight!==t&&this.refresh(),n<=e){var i=this._targets[this._targets.length-1];this._activeTarget!==i&&this._activate(i)}else{if(this._activeTarget&&e<this._offsets[0]&&0<this._offsets[0])return this._activeTarget=null,void this._clear();for(var o=this._offsets.length;o--;){this._activeTarget!==this._targets[o]&&e>=this._offsets[o]&&("undefined"==typeof this._offsets[o+1]||e<this._offsets[o+1])&&this._activate(this._targets[o])}}},e._activate=function(t){this._activeTarget=t,this._clear();var e=this._selector.split(",").map(function(e){return e+'[data-target="'+t+'"],'+e+'[href="'+t+'"]'}),n=p([].slice.call(document.querySelectorAll(e.join(","))));n.hasClass(_i)?(n.closest(Ci).find(Di).addClass(vi),n.addClass(vi)):(n.addClass(vi),n.parents(Ei).prev(bi+", "+Ti).addClass(vi),n.parents(Ei).prev(wi).children(bi).addClass(vi)),p(this._scrollElement).trigger(gi.ACTIVATE,{relatedTarget:t})},e._clear=function(){[].slice.call(document.querySelectorAll(this._selector)).filter(function(e){return e.classList.contains(vi)}).forEach(function(e){return e.classList.remove(vi)})},n._jQueryInterface=function(t){return this.each(function(){var e=p(this).data(ui);if(e||(e=new n(this,"object"==typeof t&&t),p(this).data(ui,e)),"string"==typeof t){if("undefined"==typeof e[t])throw new TypeError('No method named "'+t+'"');e[t]()}})},s(n,null,[{key:"VERSION",get:function(){return"4.4.1"}},{key:"Default",get:function(){return pi}}]),n}();p(window).on(gi.LOAD_DATA_API,function(){for(var e=[].slice.call(document.querySelectorAll(yi)),t=e.length;t--;){var n=p(e[t]);Oi._jQueryInterface.call(n,n.data())}}),p.fn[hi]=Oi._jQueryInterface,p.fn[hi].Constructor=Oi,p.fn[hi].noConflict=function(){return p.fn[hi]=di,Oi._jQueryInterface};var Ni="bs.tab",ki="."+Ni,Li=p.fn.tab,Pi={HIDE:"hide"+ki,HIDDEN:"hidden"+ki,SHOW:"show"+ki,SHOWN:"shown"+ki,CLICK_DATA_API:"click"+ki+".data-api"},xi="dropdown-menu",ji="active",Hi="disabled",Ri="fade",Fi="show",Mi=".dropdown",Wi=".nav, .list-group",Ui=".active",Bi="> li > .active",qi='[data-toggle="tab"], [data-toggle="pill"], [data-toggle="list"]',Ki=".dropdown-toggle",Qi="> .dropdown-menu .active",Vi=function(){function i(e){this._element=e}var e=i.prototype;return e.show=function(){var n=this;if(!(this._element.parentNode&&this._element.parentNode.nodeType===Node.ELEMENT_NODE&&p(this._element).hasClass(ji)||p(this._element).hasClass(Hi))){var e,i,t=p(this._element).closest(Wi)[0],o=m.getSelectorFromElement(this._element);if(t){var r="UL"===t.nodeName||"OL"===t.nodeName?Bi:Ui;i=(i=p.makeArray(p(t).find(r)))[i.length-1]}var s=p.Event(Pi.HIDE,{relatedTarget:this._element}),a=p.Event(Pi.SHOW,{relatedTarget:i});if(i&&p(i).trigger(s),p(this._element).trigger(a),!a.isDefaultPrevented()&&!s.isDefaultPrevented()){o&&(e=document.querySelector(o)),this._activate(this._element,t);var l=function(){var e=p.Event(Pi.HIDDEN,{relatedTarget:n._element}),t=p.Event(Pi.SHOWN,{relatedTarget:i});p(i).trigger(e),p(n._element).trigger(t)};e?this._activate(e,e.parentNode,l):l()}}},e.dispose=function(){p.removeData(this._element,Ni),this._element=null},e._activate=function(e,t,n){function i(){return o._transitionComplete(e,r,n)}var o=this,r=(!t||"UL"!==t.nodeName&&"OL"!==t.nodeName?p(t).children(Ui):p(t).find(Bi))[0],s=n&&r&&p(r).hasClass(Ri);if(r&&s){var a=m.getTransitionDurationFromElement(r);p(r).removeClass(Fi).one(m.TRANSITION_END,i).emulateTransitionEnd(a)}else i()},e._transitionComplete=function(e,t,n){if(t){p(t).removeClass(ji);var i=p(t.parentNode).find(Qi)[0];i&&p(i).removeClass(ji),"tab"===t.getAttribute("role")&&t.setAttribute("aria-selected",!1)}if(p(e).addClass(ji),"tab"===e.getAttribute("role")&&e.setAttribute("aria-selected",!0),m.reflow(e),e.classList.contains(Ri)&&e.classList.add(Fi),e.parentNode&&p(e.parentNode).hasClass(xi)){var o=p(e).closest(Mi)[0];if(o){var r=[].slice.call(o.querySelectorAll(Ki));p(r).addClass(ji)}e.setAttribute("aria-expanded",!0)}n&&n()},i._jQueryInterface=function(n){return this.each(function(){var e=p(this),t=e.data(Ni);if(t||(t=new i(this),e.data(Ni,t)),"string"==typeof n){if("undefined"==typeof t[n])throw new TypeError('No method named "'+n+'"');t[n]()}})},s(i,null,[{key:"VERSION",get:function(){return"4.4.1"}}]),i}();p(document).on(Pi.CLICK_DATA_API,qi,function(e){e.preventDefault(),Vi._jQueryInterface.call(p(this),"show")}),p.fn.tab=Vi._jQueryInterface,p.fn.tab.Constructor=Vi,p.fn.tab.noConflict=function(){return p.fn.tab=Li,Vi._jQueryInterface};var Yi="toast",zi="bs.toast",Xi="."+zi,Gi=p.fn[Yi],$i={CLICK_DISMISS:"click.dismiss"+Xi,HIDE:"hide"+Xi,HIDDEN:"hidden"+Xi,SHOW:"show"+Xi,SHOWN:"shown"+Xi},Ji="fade",Zi="hide",eo="show",to="showing",no={animation:"boolean",autohide:"boolean",delay:"number"},io={animation:!0,autohide:!0,delay:500},oo='[data-dismiss="toast"]',ro=function(){function i(e,t){this._element=e,this._config=this._getConfig(t),this._timeout=null,this._setListeners()}var e=i.prototype;return e.show=function(){var e=this,t=p.Event($i.SHOW);if(p(this._element).trigger(t),!t.isDefaultPrevented()){this._config.animation&&this._element.classList.add(Ji);var n=function(){e._element.classList.remove(to),e._element.classList.add(eo),p(e._element).trigger($i.SHOWN),e._config.autohide&&(e._timeout=setTimeout(function(){e.hide()},e._config.delay))};if(this._element.classList.remove(Zi),m.reflow(this._element),this._element.classList.add(to),this._config.animation){var i=m.getTransitionDurationFromElement(this._element);p(this._element).one(m.TRANSITION_END,n).emulateTransitionEnd(i)}else n()}},e.hide=function(){if(this._element.classList.contains(eo)){var e=p.Event($i.HIDE);p(this._element).trigger(e),e.isDefaultPrevented()||this._close()}},e.dispose=function(){clearTimeout(this._timeout),this._timeout=null,this._element.classList.contains(eo)&&this._element.classList.remove(eo),p(this._element).off($i.CLICK_DISMISS),p.removeData(this._element,zi),this._element=null,this._config=null},e._getConfig=function(e){return e=l({},io,{},p(this._element).data(),{},"object"==typeof e&&e?e:{}),m.typeCheckConfig(Yi,e,this.constructor.DefaultType),e},e._setListeners=function(){var e=this;p(this._element).on($i.CLICK_DISMISS,oo,function(){return e.hide()})},e._close=function(){function e(){t._element.classList.add(Zi),p(t._element).trigger($i.HIDDEN)}var t=this;if(this._element.classList.remove(eo),this._config.animation){var n=m.getTransitionDurationFromElement(this._element);p(this._element).one(m.TRANSITION_END,e).emulateTransitionEnd(n)}else e()},i._jQueryInterface=function(n){return this.each(function(){var e=p(this),t=e.data(zi);if(t||(t=new i(this,"object"==typeof n&&n),e.data(zi,t)),"string"==typeof n){if("undefined"==typeof t[n])throw new TypeError('No method named "'+n+'"');t[n](this)}})},s(i,null,[{key:"VERSION",get:function(){return"4.4.1"}},{key:"DefaultType",get:function(){return no}},{key:"Default",get:function(){return io}}]),i}();p.fn[Yi]=ro._jQueryInterface,p.fn[Yi].Constructor=ro,p.fn[Yi].noConflict=function(){return p.fn[Yi]=Gi,ro._jQueryInterface},e.Alert=_,e.Button=x,e.Carousel=he,e.Collapse=De,e.Dropdown=en,e.Modal=wn,e.Popover=ci,e.Scrollspy=Oi,e.Tab=Vi,e.Toast=ro,e.Tooltip=Xn,e.Util=m,Object.defineProperty(e,"__esModule",{value:!0})});
+//# sourceMappingURL=bootstrap.bundle.min.js.map
\ No newline at end of file
diff --git a/docs/js/vendor/bootstrap.bundle.min.js.map b/docs/js/vendor/bootstrap.bundle.min.js.map
new file mode 100644
index 0000000000000..54d2495018b0d
--- /dev/null
+++ b/docs/js/vendor/bootstrap.bundle.min.js.map
@@ -0,0 +1 @@
+{"version":3,"sources":["../../js/src/util.js","../../js/src/alert.js","../../js/src/button.js","../../js/src/carousel.js","../../js/src/collapse.js","../../node_modules/popper.js/dist/esm/popper.js","../../js/src/dropdown.js","../../js/src/modal.js","../../js/src/tools/sanitizer.js","../../js/src/tooltip.js","../../js/src/popover.js","../../js/src/scrollspy.js","../../js/src/tab.js","../../js/src/toast.js"],"names":["TRANSITION_END","transitionEndEmulator","duration","_this","this","called","$","one","Util","setTimeout","triggerTransitionEnd","getUID","prefix","Math","random","document","getElementById","getSelectorFromElement","element","selector","getAttribute","hrefAttr","trim","querySelector","err","getTransitionDurationFromElement","transitionDuration","css","transitionDelay","floatTransitionDuration","parseFloat","floatTransitionDelay","split","reflow","offsetHeight","trigger","supportsTransitionEnd","Boolean","isElement","obj","nodeType","typeCheckConfig","componentName","config","configTypes","property","Object","prototype","hasOwnProperty","call","expectedTypes","value","valueType","toString","match","toLowerCase","RegExp","test","Error","toUpperCase","findShadowRoot","documentElement","attachShadow","getRootNode","ShadowRoot","parentNode","root","jQueryDetection","TypeError","version","fn","jquery","emulateTransitionEnd","event","special","bindType","delegateType","handle","target","is","handleObj","handler","apply","arguments","NAME","DATA_KEY","EVENT_KEY","JQUERY_NO_CONFLICT","Event","CLOSE","CLOSED","CLICK_DATA_API","ClassName","Alert","_element","close","rootElement","_getRootElement","_triggerCloseEvent","isDefaultPrevented","_removeElement","dispose","removeData","parent","closest","closeEvent","removeClass","hasClass","_destroyElement","detach","remove","_jQueryInterface","each","$element","data","_handleDismiss","alertInstance","preventDefault","on","Constructor","noConflict","DATA_API_KEY","Selector","FOCUS_BLUR_DATA_API","LOAD_DATA_API","Button","toggle","triggerChangeEvent","addAriaPressed","input","type","checked","classList","contains","activeElement","tagName","focus","hasAttribute","setAttribute","toggleClass","button","inputBtn","window","buttons","slice","querySelectorAll","i","len","length","add","Default","interval","keyboard","slide","pause","wrap","touch","DefaultType","Direction","SLIDE","SLID","KEYDOWN","MOUSEENTER","MOUSELEAVE","TOUCHSTART","TOUCHMOVE","TOUCHEND","POINTERDOWN","POINTERUP","DRAG_START","PointerType","TOUCH","PEN","Carousel","_items","_interval","_activeElement","_isPaused","_isSliding","touchTimeout","touchStartX","touchDeltaX","_config","_getConfig","_indicatorsElement","_touchSupported","navigator","maxTouchPoints","_pointerEvent","PointerEvent","MSPointerEvent","_addEventListeners","next","_slide","nextWhenVisible","hidden","prev","cycle","clearInterval","setInterval","visibilityState","bind","to","index","activeIndex","_getItemIndex","direction","off","_objectSpread2","_handleSwipe","absDeltax","abs","_this2","_keydown","_addTouchEventListeners","_this3","start","originalEvent","pointerType","clientX","touches","end","clearTimeout","e","move","which","indexOf","_getItemByDirection","isNextDirection","isPrevDirection","lastItemIndex","itemIndex","_triggerSlideEvent","relatedTarget","eventDirectionName","targetIndex","fromIndex","slideEvent","from","_setActiveIndicatorElement","indicators","nextIndicator","children","addClass","directionalClassName","orderClassName","_this4","activeElementIndex","nextElement","nextElementIndex","isCycling","slidEvent","nextElementInterval","parseInt","defaultInterval","action","ride","_dataApiClickHandler","slideIndex","carousels","$carousel","SHOW","SHOWN","HIDE","HIDDEN","Dimension","Collapse","_isTransitioning","_triggerArray","id","toggleList","elem","filterElement","filter","foundElem","_selector","push","_parent","_getParent","_addAriaAndCollapsedClass","hide","show","actives","activesData","not","startEvent","dimension","_getDimension","style","attr","setTransitioning","scrollSize","getBoundingClientRect","triggerArrayLength","isTransitioning","_getTargetFromElement","triggerArray","isOpen","$this","currentTarget","$trigger","selectors","$target","isBrowser","timeoutDuration","longerTimeoutBrowsers","userAgent","debounce","Promise","resolve","then","scheduled","isFunction","functionToCheck","getStyleComputedProperty","ownerDocument","defaultView","getComputedStyle","getParentNode","nodeName","host","getScrollParent","body","_getStyleComputedProp","overflow","overflowX","overflowY","getReferenceNode","reference","referenceNode","isIE11","MSInputMethodContext","documentMode","isIE10","isIE","getOffsetParent","noOffsetParent","offsetParent","nextElementSibling","getRoot","node","findCommonOffsetParent","element1","element2","order","compareDocumentPosition","Node","DOCUMENT_POSITION_FOLLOWING","range","createRange","setStart","setEnd","commonAncestorContainer","firstElementChild","isOffsetContainer","element1root","getScroll","upperSide","undefined","html","scrollingElement","getBordersSize","styles","axis","sideA","sideB","getSize","computedStyle","max","getWindowSizes","height","width","createClass","protoProps","staticProps","defineProperties","props","descriptor","enumerable","configurable","writable","defineProperty","key","_extends","assign","source","getClientRect","offsets","right","left","bottom","top","rect","scrollTop","scrollLeft","result","sizes","clientWidth","clientHeight","horizScrollbar","offsetWidth","vertScrollbar","getOffsetRectRelativeToArbitraryNode","fixedPosition","isHTML","childrenRect","parentRect","scrollParent","borderTopWidth","borderLeftWidth","marginTop","marginLeft","subtract","modifier","includeScroll","getFixedPositionOffsetParent","parentElement","el","getBoundaries","popper","padding","boundariesElement","boundaries","excludeScroll","relativeOffset","innerWidth","innerHeight","getViewportOffsetRectRelativeToArtbitraryNode","boundariesNode","isFixed","_getWindowSizes","isPaddingNumber","computeAutoPlacement","placement","refRect","rects","sortedAreas","keys","map","area","_ref","getArea","sort","a","b","filteredAreas","_ref2","computedPlacement","variation","getReferenceOffsets","state","getOuterSizes","x","marginBottom","y","marginRight","getOppositePlacement","hash","replace","matched","getPopperOffsets","referenceOffsets","popperRect","popperOffsets","isHoriz","mainSide","secondarySide","measurement","secondaryMeasurement","find","arr","check","Array","runModifiers","modifiers","ends","prop","findIndex","cur","forEach","console","warn","enabled","isModifierEnabled","modifierName","some","name","getSupportedPropertyName","prefixes","upperProp","charAt","toCheck","getWindow","setupEventListeners","options","updateBound","addEventListener","passive","scrollElement","attachToScrollParents","callback","scrollParents","isBody","eventsEnabled","disableEventListeners","cancelAnimationFrame","scheduleUpdate","removeEventListener","removeEventListeners","isNumeric","n","isNaN","isFinite","setStyles","unit","getRoundedOffsets","shouldRound","noRound","v","_data$offsets","round","floor","referenceWidth","popperWidth","isVertical","isVariation","horizontalToInteger","verticalToInteger","isFirefox","isModifierRequired","requestingName","requestedName","requesting","isRequired","_requesting","requested","placements","validPlacements","clockwise","counter","concat","reverse","BEHAVIORS","parseOffset","offset","basePlacement","useHeight","fragments","frag","divider","search","splitRegex","ops","op","mergeWithPrevious","reduce","str","toValue","index2","Defaults","positionFixed","removeOnDestroy","onCreate","onUpdate","shift","shiftvariation","side","shiftOffsets","preventOverflow","instance","transformProp","popperStyles","transform","priority","primary","escapeWithReference","secondary","min","keepTogether","opSide","arrow","_data$offsets$arrow","arrowElement","sideCapitalized","altSide","arrowElementSize","center","popperMarginSide","popperBorderSide","sideValue","flip","flipped","originalPlacement","placementOpposite","flipOrder","behavior","step","refOffsets","overlapsRef","overflowsLeft","overflowsRight","overflowsTop","overflowsBottom","overflowsBoundaries","flippedVariationByRef","flipVariations","flippedVariationByContent","flipVariationsByContent","flippedVariation","getOppositeVariation","inner","subtractLength","bound","attributes","computeStyle","legacyGpuAccelerationOption","gpuAcceleration","offsetParentRect","position","devicePixelRatio","prefixedProperty","willChange","invertTop","invertLeft","x-placement","arrowStyles","applyStyle","removeAttribute","setAttributes","onLoad","modifierOptions","Popper","isDestroyed","isCreated","removeChild","classCallCheck","requestAnimationFrame","update","enableEventListeners","Utils","global","PopperUtils","REGEXP_KEYDOWN","ARROW_UP_KEYCODE","CLICK","KEYDOWN_DATA_API","KEYUP_DATA_API","AttachmentMap","boundary","display","popperConfig","Dropdown","_popper","_menu","_getMenuElement","_inNavbar","_detectNavbar","disabled","isActive","_clearMenus","usePopper","showEvent","_getParentFromElement","referenceElement","_getPopperConfig","noop","hideEvent","destroy","stopPropagation","constructor","_getPlacement","$parentDropdown","_getOffset","toggles","context","clickEvent","dropdownMenu","_dataApiKeydownHandler","items","item","backdrop","HIDE_PREVENTED","FOCUSIN","RESIZE","CLICK_DISMISS","KEYDOWN_DISMISS","MOUSEUP_DISMISS","MOUSEDOWN_DISMISS","Modal","_dialog","_backdrop","_isShown","_isBodyOverflowing","_ignoreBackdropClick","_scrollbarWidth","_checkScrollbar","_setScrollbar","_adjustDialog","_setEscapeEvent","_setResizeEvent","_showBackdrop","_showElement","transition","_hideModal","htmlElement","handleUpdate","_triggerBackdropTransition","hideEventPrevented","defaultPrevented","modalTransitionDuration","modalBody","ELEMENT_NODE","appendChild","_enforceFocus","transitionComplete","shownEvent","_this5","has","_this6","_this7","_this8","_resetAdjustments","_resetScrollbar","_removeBackdrop","_this9","animate","createElement","className","appendTo","backdropTransitionDuration","callbackRemove","isModalOverflowing","scrollHeight","paddingLeft","paddingRight","_getScrollbarWidth","_this10","fixedContent","stickyContent","actualPadding","calculatedPadding","actualMargin","calculatedMargin","elements","margin","scrollDiv","scrollbarWidth","_this11","uriAttrs","DefaultWhitelist","*","br","col","code","div","em","hr","h1","h2","h3","h4","h5","h6","img","li","ol","p","pre","s","small","span","sub","sup","strong","u","ul","SAFE_URL_PATTERN","DATA_URL_PATTERN","sanitizeHtml","unsafeHtml","whiteList","sanitizeFn","createdDocument","DOMParser","parseFromString","whitelistKeys","_loop","elName","attributeList","whitelistedAttributes","allowedAttributeList","attrName","nodeValue","regExp","attrRegex","l","allowedAttribute","innerHTML","CLASS_PREFIX","BSCLS_PREFIX_REGEX","DISALLOWED_ATTRIBUTES","animation","template","title","delay","container","fallbackPlacement","sanitize","AUTO","TOP","RIGHT","BOTTOM","LEFT","HoverState","INSERTED","FOCUSOUT","Trigger","Tooltip","_isEnabled","_timeout","_hoverState","_activeTrigger","tip","_setListeners","enable","disable","toggleEnabled","dataKey","_getDelegateConfig","click","_isWithActiveTrigger","_enter","_leave","getTipElement","_hideModalHandler","isWithContent","shadowRoot","isInTheDom","tipId","setContent","attachment","_getAttachment","addAttachmentClass","_getContainer","complete","_fixTransition","prevHoverState","_cleanTipClass","getTitle","setElementContent","content","text","empty","append","_handlePopperPlacementChange","eventIn","eventOut","_fixTitle","titleType","dataAttributes","dataAttr","$tip","tabClass","join","popperData","popperInstance","initConfigAnimation","Popover","_getContent","method","ACTIVATE","SCROLL","OffsetMethod","ScrollSpy","_scrollElement","_offsets","_targets","_activeTarget","_scrollHeight","_process","refresh","autoMethod","offsetMethod","offsetBase","_getScrollTop","_getScrollHeight","targetSelector","targetBCR","pageYOffset","_getOffsetHeight","maxScroll","_activate","_clear","queries","$link","parents","scrollSpys","$spy","Tab","previous","listElement","itemSelector","makeArray","hiddenEvent","_transitionComplete","active","dropdownChild","dropdownElement","dropdownToggleList","autohide","Toast","_close"],"mappings":";;;;;ypCAeA,IAAMA,EAAiB,gBAsBvB,SAASC,EAAsBC,GAAU,IAAAC,EAAAC,KACnCC,GAAS,EAYb,OAVAC,EAAEF,MAAMG,IAAIC,EAAKR,eAAgB,WAC/BK,GAAS,IAGXI,WAAW,WACJJ,GACHG,EAAKE,qBAAqBP,IAE3BD,GAEIE,KAcT,IAAMI,EAAO,CAEXR,eAAgB,kBAEhBW,OAJW,SAIJC,GACL,KAEEA,MAvDU,IAuDGC,KAAKC,UACXC,SAASC,eAAeJ,KACjC,OAAOA,GAGTK,uBAZW,SAYYC,GACrB,IAAIC,EAAWD,EAAQE,aAAa,eAEpC,IAAKD,GAAyB,MAAbA,EAAkB,CACjC,IAAME,EAAWH,EAAQE,aAAa,QACtCD,EAAWE,GAAyB,MAAbA,EAAmBA,EAASC,OAAS,GAG9D,IACE,OAAOP,SAASQ,cAAcJ,GAAYA,EAAW,KACrD,MAAOK,GACP,OAAO,OAIXC,iCA3BW,SA2BsBP,GAC/B,IAAKA,EACH,OAAO,EAIT,IAAIQ,EAAqBpB,EAAEY,GAASS,IAAI,uBACpCC,EAAkBtB,EAAEY,GAASS,IAAI,oBAE/BE,EAA0BC,WAAWJ,GACrCK,EAAuBD,WAAWF,GAGxC,OAAKC,GAA4BE,GAKjCL,EAAqBA,EAAmBM,MAAM,KAAK,GACnDJ,EAAkBA,EAAgBI,MAAM,KAAK,GA7FjB,KA+FpBF,WAAWJ,GAAsBI,WAAWF,KAP3C,GAUXK,OAnDW,SAmDJf,GACL,OAAOA,EAAQgB,cAGjBxB,qBAvDW,SAuDUQ,GACnBZ,EAAEY,GAASiB,QAAQnC,IAIrBoC,sBA5DW,WA6DT,OAAOC,QAAQrC,IAGjBsC,UAhEW,SAgEDC,GACR,OAAQA,EAAI,IAAMA,GAAKC,UAGzBC,gBApEW,SAoEKC,EAAeC,EAAQC,GACrC,IAAK,IAAMC,KAAYD,EACrB,GAAIE,OAAOC,UAAUC,eAAeC,KAAKL,EAAaC,GAAW,CAC/D,IAAMK,EAAgBN,EAAYC,GAC5BM,EAAgBR,EAAOE,GACvBO,EAAgBD,GAAS3C,EAAK8B,UAAUa,GAC1C,WAtHIZ,EAsHeY,EArHtB,GAAGE,SAASJ,KAAKV,GAAKe,MAAM,eAAe,GAAGC,eAuH/C,IAAK,IAAIC,OAAON,GAAeO,KAAKL,GAClC,MAAM,IAAIM,MACLhB,EAAciB,cAAjB,aACWd,EADX,oBACuCO,EADvC,wBAEsBF,EAFtB,MA1HZ,IAAgBX,GAkIdqB,eAtFW,SAsFI1C,GACb,IAAKH,SAAS8C,gBAAgBC,aAC5B,OAAO,KAIT,GAAmC,mBAAxB5C,EAAQ6C,YAKnB,OAAI7C,aAAmB8C,WACd9C,EAIJA,EAAQ+C,WAINzD,EAAKoD,eAAe1C,EAAQ+C,YAH1B,KAVP,IAAMC,EAAOhD,EAAQ6C,cACrB,OAAOG,aAAgBF,WAAaE,EAAO,MAe/CC,gBA7GW,WA8GT,GAAiB,oBAAN7D,EACT,MAAM,IAAI8D,UAAU,kGAGtB,IAAMC,EAAU/D,EAAEgE,GAAGC,OAAOvC,MAAM,KAAK,GAAGA,MAAM,KAOhD,GAAIqC,EAAQ,GALI,GAKYA,EAAQ,GAJnB,GAFA,IAMoCA,EAAQ,IAJ5C,IAI+DA,EAAQ,IAAmBA,EAAQ,GAHlG,GACA,GAEmHA,EAAQ,GAC1I,MAAM,IAAIX,MAAM,iFAKtBlD,EAAK2D,kBAzIH7D,EAAEgE,GAAGE,qBAAuBvE,EAC5BK,EAAEmE,MAAMC,QAAQlE,EAAKR,gBA9Bd,CACL2E,SAAU3E,EACV4E,aAAc5E,EACd6E,OAHK,SAGEJ,GACL,GAAInE,EAAEmE,EAAMK,QAAQC,GAAG3E,MACrB,OAAOqE,EAAMO,UAAUC,QAAQC,MAAM9E,KAAM+E,aCdnD,IAAMC,EAAsB,QAEtBC,EAAsB,WACtBC,EAAS,IAAiBD,EAE1BE,EAAsBjF,EAAEgE,GAAGc,GAM3BI,EAAQ,CACZC,MAAK,QAAoBH,EACzBI,OAAM,SAAoBJ,EAC1BK,eAAc,QAAWL,EAVC,aAatBM,EACI,QADJA,EAEI,OAFJA,EAGI,OASJC,aACJ,SAAAA,EAAY3E,GACVd,KAAK0F,SAAW5E,6BAWlB6E,MAAA,SAAM7E,GACJ,IAAI8E,EAAc5F,KAAK0F,SACnB5E,IACF8E,EAAc5F,KAAK6F,gBAAgB/E,IAGjBd,KAAK8F,mBAAmBF,GAE5BG,sBAIhB/F,KAAKgG,eAAeJ,MAGtBK,QAAA,WACE/F,EAAEgG,WAAWlG,KAAK0F,SAAUT,GAC5BjF,KAAK0F,SAAW,QAKlBG,gBAAA,SAAgB/E,GACd,IAAMC,EAAWX,EAAKS,uBAAuBC,GACzCqF,GAAa,EAUjB,OARIpF,IACFoF,EAASxF,SAASQ,cAAcJ,IAIhCoF,EADGA,GACMjG,EAAEY,GAASsF,QAAX,IAAuBZ,GAAmB,MAMvDM,mBAAA,SAAmBhF,GACjB,IAAMuF,EAAanG,EAAEkF,MAAMA,EAAMC,OAGjC,OADAnF,EAAEY,GAASiB,QAAQsE,GACZA,KAGTL,eAAA,SAAelF,GAAS,IAAAf,EAAAC,KAGtB,GAFAE,EAAEY,GAASwF,YAAYd,GAElBtF,EAAEY,GAASyF,SAASf,GAAzB,CAKA,IAAMlE,EAAqBlB,EAAKiB,iCAAiCP,GAEjEZ,EAAEY,GACCX,IAAIC,EAAKR,eAAgB,SAACyE,GAAD,OAAWtE,EAAKyG,gBAAgB1F,EAASuD,KAClED,qBAAqB9C,QARtBtB,KAAKwG,gBAAgB1F,MAWzB0F,gBAAA,SAAgB1F,GACdZ,EAAEY,GACC2F,SACA1E,QAAQqD,EAAME,QACdoB,YAKEC,iBAAP,SAAwBpE,GACtB,OAAOvC,KAAK4G,KAAK,WACf,IAAMC,EAAW3G,EAAEF,MACf8G,EAAaD,EAASC,KAAK7B,GAE1B6B,IACHA,EAAO,IAAIrB,EAAMzF,MACjB6G,EAASC,KAAK7B,EAAU6B,IAGX,UAAXvE,GACFuE,EAAKvE,GAAQvC,WAKZ+G,eAAP,SAAsBC,GACpB,OAAO,SAAU3C,GACXA,GACFA,EAAM4C,iBAGRD,EAAcrB,MAAM3F,gDA/FtB,MApCwB,iBA8I5BE,EAAES,UAAUuG,GACV9B,EAAMG,eAxII,yBA0IVE,EAAMsB,eAAe,IAAItB,IAS3BvF,EAAEgE,GAAGc,GAAoBS,EAAMkB,iBAC/BzG,EAAEgE,GAAGc,GAAMmC,YAAc1B,EACzBvF,EAAEgE,GAAGc,GAAMoC,WAAc,WAEvB,OADAlH,EAAEgE,GAAGc,GAAQG,EACNM,EAAMkB,kBChKf,IAAM3B,EAAsB,SAEtBC,EAAsB,YACtBC,EAAS,IAAiBD,EAC1BoC,EAAsB,YACtBlC,EAAsBjF,EAAEgE,GAAGc,GAE3BQ,EACK,SADLA,EAEK,MAFLA,EAGK,QAGL8B,EACmB,0BADnBA,EAEmB,0BAFnBA,EAGmB,yBAHnBA,EAImB,+BAJnBA,EAKmB,6BALnBA,EAMmB,UANnBA,EAOmB,OAGnBlC,EAAQ,CACZG,eAAc,QAAgBL,EAAYmC,EAC1CE,oBAAsB,QAAQrC,EAAYmC,EAApB,QACSnC,EAAYmC,EAC3CG,cAAa,OAAgBtC,EAAYmC,GASrCI,aACJ,SAAAA,EAAY3G,GACVd,KAAK0F,SAAW5E,6BAWlB4G,OAAA,WACE,IAAIC,GAAqB,EACrBC,GAAiB,EACfhC,EAAc1F,EAAEF,KAAK0F,UAAUU,QACnCkB,GACA,GAEF,GAAI1B,EAAa,CACf,IAAMiC,EAAQ7H,KAAK0F,SAASvE,cAAcmG,GAE1C,GAAIO,EAAO,CACT,GAAmB,UAAfA,EAAMC,KACR,GAAID,EAAME,SACR/H,KAAK0F,SAASsC,UAAUC,SAASzC,GACjCmC,GAAqB,MAChB,CACL,IAAMO,EAAgBtC,EAAYzE,cAAcmG,GAE5CY,GACFhI,EAAEgI,GAAe5B,YAAYd,OAGT,aAAfqC,EAAMC,KACe,UAA1B9H,KAAK0F,SAASyC,SAAuBN,EAAME,UAAY/H,KAAK0F,SAASsC,UAAUC,SAASzC,KAC1FmC,GAAqB,GAIvBA,GAAqB,EAGnBA,IACFE,EAAME,SAAW/H,KAAK0F,SAASsC,UAAUC,SAASzC,GAClDtF,EAAE2H,GAAO9F,QAAQ,WAGnB8F,EAAMO,QACNR,GAAiB,GAIf5H,KAAK0F,SAAS2C,aAAa,aAAerI,KAAK0F,SAASsC,UAAUC,SAAS,cAC3EL,GACF5H,KAAK0F,SAAS4C,aAAa,gBACxBtI,KAAK0F,SAASsC,UAAUC,SAASzC,IAGlCmC,GACFzH,EAAEF,KAAK0F,UAAU6C,YAAY/C,OAKnCS,QAAA,WACE/F,EAAEgG,WAAWlG,KAAK0F,SAAUT,GAC5BjF,KAAK0F,SAAW,QAKXiB,iBAAP,SAAwBpE,GACtB,OAAOvC,KAAK4G,KAAK,WACf,IAAIE,EAAO5G,EAAEF,MAAM8G,KAAK7B,GAEnB6B,IACHA,EAAO,IAAIW,EAAOzH,MAClBE,EAAEF,MAAM8G,KAAK7B,EAAU6B,IAGV,WAAXvE,GACFuE,EAAKvE,gDA3ET,MA3CwB,iBAkI5BrC,EAAES,UACCuG,GAAG9B,EAAMG,eAAgB+B,EAA6B,SAACjD,GACtD,IAAImE,EAASnE,EAAMK,OAMnB,GAJKxE,EAAEsI,GAAQjC,SAASf,KACtBgD,EAAStI,EAAEsI,GAAQpC,QAAQkB,GAAiB,KAGzCkB,GAAUA,EAAOH,aAAa,aAAeG,EAAOR,UAAUC,SAAS,YAC1E5D,EAAM4C,qBACD,CACL,IAAMwB,EAAWD,EAAOrH,cAAcmG,GAEtC,GAAImB,IAAaA,EAASJ,aAAa,aAAeI,EAAST,UAAUC,SAAS,aAEhF,YADA5D,EAAM4C,iBAIRQ,EAAOd,iBAAiB9D,KAAK3C,EAAEsI,GAAS,aAG3CtB,GAAG9B,EAAMmC,oBAAqBD,EAA6B,SAACjD,GAC3D,IAAMmE,EAAStI,EAAEmE,EAAMK,QAAQ0B,QAAQkB,GAAiB,GACxDpH,EAAEsI,GAAQD,YAAY/C,EAAiB,eAAenC,KAAKgB,EAAMyD,SAGrE5H,EAAEwI,QAAQxB,GAAG9B,EAAMoC,cAAe,WAKhC,IADA,IAAImB,EAAU,GAAGC,MAAM/F,KAAKlC,SAASkI,iBAAiBvB,IAC7CwB,EAAI,EAAGC,EAAMJ,EAAQK,OAAQF,EAAIC,EAAKD,IAAK,CAClD,IAAMN,EAASG,EAAQG,GACjBjB,EAAQW,EAAOrH,cAAcmG,GAC/BO,EAAME,SAAWF,EAAMQ,aAAa,WACtCG,EAAOR,UAAUiB,IAAIzD,GAErBgD,EAAOR,UAAUtB,OAAOlB,GAM5B,IAAK,IAAIsD,EAAI,EAAGC,GADhBJ,EAAU,GAAGC,MAAM/F,KAAKlC,SAASkI,iBAAiBvB,KACpB0B,OAAQF,EAAIC,EAAKD,IAAK,CAClD,IAAMN,EAASG,EAAQG,GACqB,SAAxCN,EAAOxH,aAAa,gBACtBwH,EAAOR,UAAUiB,IAAIzD,GAErBgD,EAAOR,UAAUtB,OAAOlB,MAW9BtF,EAAEgE,GAAGc,GAAQyC,EAAOd,iBACpBzG,EAAEgE,GAAGc,GAAMmC,YAAcM,EACzBvH,EAAEgE,GAAGc,GAAMoC,WAAa,WAEtB,OADAlH,EAAEgE,GAAGc,GAAQG,EACNsC,EAAOd,kBCjMhB,IAAM3B,EAAyB,WAEzBC,EAAyB,cACzBC,EAAS,IAAoBD,EAC7BoC,EAAyB,YACzBlC,EAAyBjF,EAAEgE,GAAGc,GAM9BkE,EAAU,CACdC,SAAW,IACXC,UAAW,EACXC,OAAW,EACXC,MAAW,QACXC,MAAW,EACXC,OAAW,GAGPC,EAAc,CAClBN,SAAW,mBACXC,SAAW,UACXC,MAAW,mBACXC,MAAW,mBACXC,KAAW,UACXC,MAAW,WAGPE,EACO,OADPA,EAEO,OAFPA,EAGO,OAHPA,EAIO,QAGPtE,EAAQ,CACZuE,MAAK,QAAoBzE,EACzB0E,KAAI,OAAoB1E,EACxB2E,QAAO,UAAoB3E,EAC3B4E,WAAU,aAAoB5E,EAC9B6E,WAAU,aAAoB7E,EAC9B8E,WAAU,aAAoB9E,EAC9B+E,UAAS,YAAoB/E,EAC7BgF,SAAQ,WAAoBhF,EAC5BiF,YAAW,cAAoBjF,EAC/BkF,UAAS,YAAoBlF,EAC7BmF,WAAU,YAAmBnF,EAC7BsC,cAAa,OAAWtC,EAAYmC,EACpC9B,eAAc,QAAWL,EAAYmC,GAGjC7B,EACY,WADZA,EAEY,SAFZA,EAGY,QAHZA,EAIY,sBAJZA,EAKY,qBALZA,EAMY,qBANZA,EAOY,qBAPZA,GASY,gBAGZ8B,GACU,UADVA,GAEU,wBAFVA,GAGU,iBAHVA,GAIU,qBAJVA,GAKU,2CALVA,GAMU,uBANVA,GAOU,gCAPVA,GAQU,yBAGVgD,GAAc,CAClBC,MAAQ,QACRC,IAAQ,OAQJC,cACJ,SAAAA,EAAY3J,EAASyB,GACnBvC,KAAK0K,OAAiB,KACtB1K,KAAK2K,UAAiB,KACtB3K,KAAK4K,eAAiB,KACtB5K,KAAK6K,WAAiB,EACtB7K,KAAK8K,YAAiB,EACtB9K,KAAK+K,aAAiB,KACtB/K,KAAKgL,YAAiB,EACtBhL,KAAKiL,YAAiB,EAEtBjL,KAAKkL,QAAqBlL,KAAKmL,WAAW5I,GAC1CvC,KAAK0F,SAAqB5E,EAC1Bd,KAAKoL,mBAAqBpL,KAAK0F,SAASvE,cAAcmG,IACtDtH,KAAKqL,gBAAqB,iBAAkB1K,SAAS8C,iBAA8C,EAA3B6H,UAAUC,eAClFvL,KAAKwL,cAAqBvJ,QAAQyG,OAAO+C,cAAgB/C,OAAOgD,gBAEhE1L,KAAK2L,gDAePC,KAAA,WACO5L,KAAK8K,YACR9K,KAAK6L,OAAOnC,MAIhBoC,gBAAA,YAGOnL,SAASoL,QACX7L,EAAEF,KAAK0F,UAAUf,GAAG,aAAsD,WAAvCzE,EAAEF,KAAK0F,UAAUnE,IAAI,eACzDvB,KAAK4L,UAITI,KAAA,WACOhM,KAAK8K,YACR9K,KAAK6L,OAAOnC,MAIhBJ,MAAA,SAAMjF,GACCA,IACHrE,KAAK6K,WAAY,GAGf7K,KAAK0F,SAASvE,cAAcmG,MAC9BlH,EAAKE,qBAAqBN,KAAK0F,UAC/B1F,KAAKiM,OAAM,IAGbC,cAAclM,KAAK2K,WACnB3K,KAAK2K,UAAY,QAGnBsB,MAAA,SAAM5H,GACCA,IACHrE,KAAK6K,WAAY,GAGf7K,KAAK2K,YACPuB,cAAclM,KAAK2K,WACnB3K,KAAK2K,UAAY,MAGf3K,KAAKkL,QAAQ/B,WAAanJ,KAAK6K,YACjC7K,KAAK2K,UAAYwB,aACdxL,SAASyL,gBAAkBpM,KAAK8L,gBAAkB9L,KAAK4L,MAAMS,KAAKrM,MACnEA,KAAKkL,QAAQ/B,cAKnBmD,GAAA,SAAGC,GAAO,IAAAxM,EAAAC,KACRA,KAAK4K,eAAiB5K,KAAK0F,SAASvE,cAAcmG,IAElD,IAAMkF,EAAcxM,KAAKyM,cAAczM,KAAK4K,gBAE5C,KAAI2B,EAAQvM,KAAK0K,OAAO1B,OAAS,GAAKuD,EAAQ,GAI9C,GAAIvM,KAAK8K,WACP5K,EAAEF,KAAK0F,UAAUvF,IAAIiF,EAAMwE,KAAM,WAAA,OAAM7J,EAAKuM,GAAGC,SADjD,CAKA,GAAIC,IAAgBD,EAGlB,OAFAvM,KAAKsJ,aACLtJ,KAAKiM,QAIP,IAAMS,EAAoBF,EAARD,EACd7C,EACAA,EAEJ1J,KAAK6L,OAAOa,EAAW1M,KAAK0K,OAAO6B,QAGrCtG,QAAA,WACE/F,EAAEF,KAAK0F,UAAUiH,IAAIzH,GACrBhF,EAAEgG,WAAWlG,KAAK0F,SAAUT,GAE5BjF,KAAK0K,OAAqB,KAC1B1K,KAAKkL,QAAqB,KAC1BlL,KAAK0F,SAAqB,KAC1B1F,KAAK2K,UAAqB,KAC1B3K,KAAK6K,UAAqB,KAC1B7K,KAAK8K,WAAqB,KAC1B9K,KAAK4K,eAAqB,KAC1B5K,KAAKoL,mBAAqB,QAK5BD,WAAA,SAAW5I,GAMT,OALAA,EAAMqK,EAAA,GACD1D,EADC,GAED3G,GAELnC,EAAKiC,gBAAgB2C,EAAMzC,EAAQkH,GAC5BlH,KAGTsK,aAAA,WACE,IAAMC,EAAYrM,KAAKsM,IAAI/M,KAAKiL,aAEhC,KAAI6B,GAxNuB,IAwN3B,CAIA,IAAMJ,EAAYI,EAAY9M,KAAKiL,aAEnCjL,KAAKiL,YAAc,GAGfyB,GACF1M,KAAKgM,OAIHU,EAAY,GACd1M,KAAK4L,WAITD,mBAAA,WAAqB,IAAAqB,EAAAhN,KACfA,KAAKkL,QAAQ9B,UACflJ,EAAEF,KAAK0F,UACJwB,GAAG9B,EAAMyE,QAAS,SAACxF,GAAD,OAAW2I,EAAKC,SAAS5I,KAGrB,UAAvBrE,KAAKkL,QAAQ5B,OACfpJ,EAAEF,KAAK0F,UACJwB,GAAG9B,EAAM0E,WAAY,SAACzF,GAAD,OAAW2I,EAAK1D,MAAMjF,KAC3C6C,GAAG9B,EAAM2E,WAAY,SAAC1F,GAAD,OAAW2I,EAAKf,MAAM5H,KAG5CrE,KAAKkL,QAAQ1B,OACfxJ,KAAKkN,6BAITA,wBAAA,WAA0B,IAAAC,EAAAnN,KACxB,GAAKA,KAAKqL,gBAAV,CAIA,IAAM+B,EAAQ,SAAC/I,GACT8I,EAAK3B,eAAiBlB,GAAYjG,EAAMgJ,cAAcC,YAAY/J,eACpE4J,EAAKnC,YAAc3G,EAAMgJ,cAAcE,QAC7BJ,EAAK3B,gBACf2B,EAAKnC,YAAc3G,EAAMgJ,cAAcG,QAAQ,GAAGD,UAahDE,EAAM,SAACpJ,GACP8I,EAAK3B,eAAiBlB,GAAYjG,EAAMgJ,cAAcC,YAAY/J,iBACpE4J,EAAKlC,YAAc5G,EAAMgJ,cAAcE,QAAUJ,EAAKnC,aAGxDmC,EAAKN,eACsB,UAAvBM,EAAKjC,QAAQ5B,QASf6D,EAAK7D,QACD6D,EAAKpC,cACP2C,aAAaP,EAAKpC,cAEpBoC,EAAKpC,aAAe1K,WAAW,SAACgE,GAAD,OAAW8I,EAAKlB,MAAM5H,IAtS9B,IAsS+D8I,EAAKjC,QAAQ/B,YAIvGjJ,EAAEF,KAAK0F,SAASmD,iBAAiBvB,KAAoBJ,GAAG9B,EAAMiF,WAAY,SAACsD,GAAD,OAAOA,EAAE1G,mBAC/EjH,KAAKwL,eACPtL,EAAEF,KAAK0F,UAAUwB,GAAG9B,EAAM+E,YAAa,SAAC9F,GAAD,OAAW+I,EAAM/I,KACxDnE,EAAEF,KAAK0F,UAAUwB,GAAG9B,EAAMgF,UAAW,SAAC/F,GAAD,OAAWoJ,EAAIpJ,KAEpDrE,KAAK0F,SAASsC,UAAUiB,IAAIzD,MAE5BtF,EAAEF,KAAK0F,UAAUwB,GAAG9B,EAAM4E,WAAY,SAAC3F,GAAD,OAAW+I,EAAM/I,KACvDnE,EAAEF,KAAK0F,UAAUwB,GAAG9B,EAAM6E,UAAW,SAAC5F,GAAD,OAxC1B,SAACA,GAERA,EAAMgJ,cAAcG,SAAgD,EAArCnJ,EAAMgJ,cAAcG,QAAQxE,OAC7DmE,EAAKlC,YAAc,EAEnBkC,EAAKlC,YAAc5G,EAAMgJ,cAAcG,QAAQ,GAAGD,QAAUJ,EAAKnC,YAmCnB4C,CAAKvJ,KACrDnE,EAAEF,KAAK0F,UAAUwB,GAAG9B,EAAM8E,SAAU,SAAC7F,GAAD,OAAWoJ,EAAIpJ,UAIvD4I,SAAA,SAAS5I,GACP,IAAI,kBAAkBhB,KAAKgB,EAAMK,OAAOyD,SAIxC,OAAQ9D,EAAMwJ,OACZ,KA/TyB,GAgUvBxJ,EAAM4C,iBACNjH,KAAKgM,OACL,MACF,KAlUyB,GAmUvB3H,EAAM4C,iBACNjH,KAAK4L,WAMXa,cAAA,SAAc3L,GAIZ,OAHAd,KAAK0K,OAAS5J,GAAWA,EAAQ+C,WAC7B,GAAG+E,MAAM/F,KAAK/B,EAAQ+C,WAAWgF,iBAAiBvB,KAClD,GACGtH,KAAK0K,OAAOoD,QAAQhN,MAG7BiN,oBAAA,SAAoBrB,EAAWxE,GAC7B,IAAM8F,EAAkBtB,IAAchD,EAChCuE,EAAkBvB,IAAchD,EAChC8C,EAAkBxM,KAAKyM,cAAcvE,GACrCgG,EAAkBlO,KAAK0K,OAAO1B,OAAS,EAI7C,IAHwBiF,GAAmC,IAAhBzB,GACnBwB,GAAmBxB,IAAgB0B,KAErClO,KAAKkL,QAAQ3B,KACjC,OAAOrB,EAGT,IACMiG,GAAa3B,GADDE,IAAchD,GAAkB,EAAI,IACZ1J,KAAK0K,OAAO1B,OAEtD,OAAsB,GAAfmF,EACHnO,KAAK0K,OAAO1K,KAAK0K,OAAO1B,OAAS,GAAKhJ,KAAK0K,OAAOyD,MAGxDC,mBAAA,SAAmBC,EAAeC,GAChC,IAAMC,EAAcvO,KAAKyM,cAAc4B,GACjCG,EAAYxO,KAAKyM,cAAczM,KAAK0F,SAASvE,cAAcmG,KAC3DmH,EAAavO,EAAEkF,MAAMA,EAAMuE,MAAO,CACtC0E,cAAAA,EACA3B,UAAW4B,EACXI,KAAMF,EACNlC,GAAIiC,IAKN,OAFArO,EAAEF,KAAK0F,UAAU3D,QAAQ0M,GAElBA,KAGTE,2BAAA,SAA2B7N,GACzB,GAAId,KAAKoL,mBAAoB,CAC3B,IAAMwD,EAAa,GAAGhG,MAAM/F,KAAK7C,KAAKoL,mBAAmBvC,iBAAiBvB,KAC1EpH,EAAE0O,GACCtI,YAAYd,GAEf,IAAMqJ,EAAgB7O,KAAKoL,mBAAmB0D,SAC5C9O,KAAKyM,cAAc3L,IAGjB+N,GACF3O,EAAE2O,GAAeE,SAASvJ,OAKhCqG,OAAA,SAAOa,EAAW5L,GAAS,IAQrBkO,EACAC,EACAX,EAVqBY,EAAAlP,KACnBkI,EAAgBlI,KAAK0F,SAASvE,cAAcmG,IAC5C6H,EAAqBnP,KAAKyM,cAAcvE,GACxCkH,EAAgBtO,GAAWoH,GAC/BlI,KAAK+N,oBAAoBrB,EAAWxE,GAChCmH,EAAmBrP,KAAKyM,cAAc2C,GACtCE,EAAYrN,QAAQjC,KAAK2K,WAgB/B,GAPE2D,EAHE5B,IAAchD,GAChBsF,EAAuBxJ,EACvByJ,EAAiBzJ,EACIkE,IAErBsF,EAAuBxJ,EACvByJ,EAAiBzJ,EACIkE,GAGnB0F,GAAelP,EAAEkP,GAAa7I,SAASf,GACzCxF,KAAK8K,YAAa,OAKpB,IADmB9K,KAAKoO,mBAAmBgB,EAAad,GACzCvI,sBAIVmC,GAAkBkH,EAAvB,CAKApP,KAAK8K,YAAa,EAEdwE,GACFtP,KAAKsJ,QAGPtJ,KAAK2O,2BAA2BS,GAEhC,IAAMG,EAAYrP,EAAEkF,MAAMA,EAAMwE,KAAM,CACpCyE,cAAee,EACf1C,UAAW4B,EACXI,KAAMS,EACN7C,GAAI+C,IAGN,GAAInP,EAAEF,KAAK0F,UAAUa,SAASf,GAAkB,CAC9CtF,EAAEkP,GAAaL,SAASE,GAExB7O,EAAKyB,OAAOuN,GAEZlP,EAAEgI,GAAe6G,SAASC,GAC1B9O,EAAEkP,GAAaL,SAASC,GAExB,IAAMQ,EAAsBC,SAASL,EAAYpO,aAAa,iBAAkB,IAC5EwO,GACFxP,KAAKkL,QAAQwE,gBAAkB1P,KAAKkL,QAAQwE,iBAAmB1P,KAAKkL,QAAQ/B,SAC5EnJ,KAAKkL,QAAQ/B,SAAWqG,GAExBxP,KAAKkL,QAAQ/B,SAAWnJ,KAAKkL,QAAQwE,iBAAmB1P,KAAKkL,QAAQ/B,SAGvE,IAAM7H,EAAqBlB,EAAKiB,iCAAiC6G,GAEjEhI,EAAEgI,GACC/H,IAAIC,EAAKR,eAAgB,WACxBM,EAAEkP,GACC9I,YAAe0I,EADlB,IAC0CC,GACvCF,SAASvJ,GAEZtF,EAAEgI,GAAe5B,YAAed,EAAhC,IAAoDyJ,EAApD,IAAsED,GAEtEE,EAAKpE,YAAa,EAElBzK,WAAW,WAAA,OAAMH,EAAEgP,EAAKxJ,UAAU3D,QAAQwN,IAAY,KAEvDnL,qBAAqB9C,QAExBpB,EAAEgI,GAAe5B,YAAYd,GAC7BtF,EAAEkP,GAAaL,SAASvJ,GAExBxF,KAAK8K,YAAa,EAClB5K,EAAEF,KAAK0F,UAAU3D,QAAQwN,GAGvBD,GACFtP,KAAKiM,YAMFtF,iBAAP,SAAwBpE,GACtB,OAAOvC,KAAK4G,KAAK,WACf,IAAIE,EAAO5G,EAAEF,MAAM8G,KAAK7B,GACpBiG,EAAO0B,EAAA,GACN1D,EADM,GAENhJ,EAAEF,MAAM8G,QAGS,iBAAXvE,IACT2I,EAAO0B,EAAA,GACF1B,EADE,GAEF3I,IAIP,IAAMoN,EAA2B,iBAAXpN,EAAsBA,EAAS2I,EAAQ7B,MAO7D,GALKvC,IACHA,EAAO,IAAI2D,EAASzK,KAAMkL,GAC1BhL,EAAEF,MAAM8G,KAAK7B,EAAU6B,IAGH,iBAAXvE,EACTuE,EAAKwF,GAAG/J,QACH,GAAsB,iBAAXoN,EAAqB,CACrC,GAA4B,oBAAjB7I,EAAK6I,GACd,MAAM,IAAI3L,UAAJ,oBAAkC2L,EAAlC,KAER7I,EAAK6I,UACIzE,EAAQ/B,UAAY+B,EAAQ0E,OACrC9I,EAAKwC,QACLxC,EAAKmF,cAKJ4D,qBAAP,SAA4BxL,GAC1B,IAAMtD,EAAWX,EAAKS,uBAAuBb,MAE7C,GAAKe,EAAL,CAIA,IAAM2D,EAASxE,EAAEa,GAAU,GAE3B,GAAK2D,GAAWxE,EAAEwE,GAAQ6B,SAASf,GAAnC,CAIA,IAAMjD,EAAMqK,EAAA,GACP1M,EAAEwE,GAAQoC,OADH,GAEP5G,EAAEF,MAAM8G,QAEPgJ,EAAa9P,KAAKgB,aAAa,iBAEjC8O,IACFvN,EAAO4G,UAAW,GAGpBsB,EAAS9D,iBAAiB9D,KAAK3C,EAAEwE,GAASnC,GAEtCuN,GACF5P,EAAEwE,GAAQoC,KAAK7B,GAAUqH,GAAGwD,GAG9BzL,EAAM4C,4DAjcN,MA3G2B,wCA+G3B,OAAOiC,WAucXhJ,EAAES,UACCuG,GAAG9B,EAAMG,eAAgB+B,GAAqBmD,GAASoF,sBAE1D3P,EAAEwI,QAAQxB,GAAG9B,EAAMoC,cAAe,WAEhC,IADA,IAAMuI,EAAY,GAAGnH,MAAM/F,KAAKlC,SAASkI,iBAAiBvB,KACjDwB,EAAI,EAAGC,EAAMgH,EAAU/G,OAAQF,EAAIC,EAAKD,IAAK,CACpD,IAAMkH,EAAY9P,EAAE6P,EAAUjH,IAC9B2B,GAAS9D,iBAAiB9D,KAAKmN,EAAWA,EAAUlJ,WAUxD5G,EAAEgE,GAAGc,GAAQyF,GAAS9D,iBACtBzG,EAAEgE,GAAGc,GAAMmC,YAAcsD,GACzBvK,EAAEgE,GAAGc,GAAMoC,WAAa,WAEtB,OADAlH,EAAEgE,GAAGc,GAAQG,EACNsF,GAAS9D,kBC5kBlB,IAAM3B,GAAsB,WAEtBC,GAAsB,cACtBC,GAAS,IAAiBD,GAE1BE,GAAsBjF,EAAEgE,GAAGc,IAE3BkE,GAAU,CACdxB,QAAS,EACTvB,OAAS,IAGLsD,GAAc,CAClB/B,OAAS,UACTvB,OAAS,oBAGLf,GAAQ,CACZ6K,KAAI,OAAoB/K,GACxBgL,MAAK,QAAoBhL,GACzBiL,KAAI,OAAoBjL,GACxBkL,OAAM,SAAoBlL,GAC1BK,eAAc,QAAWL,GAlBC,aAqBtBM,GACS,OADTA,GAES,WAFTA,GAGS,aAHTA,GAIS,YAGT6K,GACK,QADLA,GAEK,SAGL/I,GACU,qBADVA,GAEU,2BASVgJ,cACJ,SAAAA,EAAYxP,EAASyB,GACnBvC,KAAKuQ,kBAAmB,EACxBvQ,KAAK0F,SAAmB5E,EACxBd,KAAKkL,QAAmBlL,KAAKmL,WAAW5I,GACxCvC,KAAKwQ,cAAmB,GAAG5H,MAAM/F,KAAKlC,SAASkI,iBAC7C,mCAAmC/H,EAAQ2P,GAA3C,6CAC0C3P,EAAQ2P,GADlD,OAKF,IADA,IAAMC,EAAa,GAAG9H,MAAM/F,KAAKlC,SAASkI,iBAAiBvB,KAClDwB,EAAI,EAAGC,EAAM2H,EAAW1H,OAAQF,EAAIC,EAAKD,IAAK,CACrD,IAAM6H,EAAOD,EAAW5H,GAClB/H,EAAWX,EAAKS,uBAAuB8P,GACvCC,EAAgB,GAAGhI,MAAM/F,KAAKlC,SAASkI,iBAAiB9H,IAC3D8P,OAAO,SAACC,GAAD,OAAeA,IAAchQ,IAEtB,OAAbC,GAA4C,EAAvB6P,EAAc5H,SACrChJ,KAAK+Q,UAAYhQ,EACjBf,KAAKwQ,cAAcQ,KAAKL,IAI5B3Q,KAAKiR,QAAUjR,KAAKkL,QAAQ/E,OAASnG,KAAKkR,aAAe,KAEpDlR,KAAKkL,QAAQ/E,QAChBnG,KAAKmR,0BAA0BnR,KAAK0F,SAAU1F,KAAKwQ,eAGjDxQ,KAAKkL,QAAQxD,QACf1H,KAAK0H,oCAgBTA,OAAA,WACMxH,EAAEF,KAAK0F,UAAUa,SAASf,IAC5BxF,KAAKoR,OAELpR,KAAKqR,UAITA,KAAA,WAAO,IAMDC,EACAC,EAPCxR,EAAAC,KACL,IAAIA,KAAKuQ,mBACPrQ,EAAEF,KAAK0F,UAAUa,SAASf,MAOxBxF,KAAKiR,SAUgB,KATvBK,EAAU,GAAG1I,MAAM/F,KAAK7C,KAAKiR,QAAQpI,iBAAiBvB,KACnDuJ,OAAO,SAACF,GACP,MAAmC,iBAAxB5Q,EAAKmL,QAAQ/E,OACfwK,EAAK3P,aAAa,iBAAmBjB,EAAKmL,QAAQ/E,OAGpDwK,EAAK3I,UAAUC,SAASzC,OAGvBwD,SACVsI,EAAU,QAIVA,IACFC,EAAcrR,EAAEoR,GAASE,IAAIxR,KAAK+Q,WAAWjK,KAAK7B,MAC/BsM,EAAYhB,mBAFjC,CAOA,IAAMkB,EAAavR,EAAEkF,MAAMA,GAAM6K,MAEjC,GADA/P,EAAEF,KAAK0F,UAAU3D,QAAQ0P,IACrBA,EAAW1L,qBAAf,CAIIuL,IACFhB,EAAS3J,iBAAiB9D,KAAK3C,EAAEoR,GAASE,IAAIxR,KAAK+Q,WAAY,QAC1DQ,GACHrR,EAAEoR,GAASxK,KAAK7B,GAAU,OAI9B,IAAMyM,EAAY1R,KAAK2R,gBAEvBzR,EAAEF,KAAK0F,UACJY,YAAYd,IACZuJ,SAASvJ,IAEZxF,KAAK0F,SAASkM,MAAMF,GAAa,EAE7B1R,KAAKwQ,cAAcxH,QACrB9I,EAAEF,KAAKwQ,eACJlK,YAAYd,IACZqM,KAAK,iBAAiB,GAG3B7R,KAAK8R,kBAAiB,GAEtB,IAcMC,EAAU,UADaL,EAAU,GAAGnO,cAAgBmO,EAAU9I,MAAM,IAEpEtH,EAAqBlB,EAAKiB,iCAAiCrB,KAAK0F,UAEtExF,EAAEF,KAAK0F,UACJvF,IAAIC,EAAKR,eAlBK,WACfM,EAAEH,EAAK2F,UACJY,YAAYd,IACZuJ,SAASvJ,IACTuJ,SAASvJ,IAEZzF,EAAK2F,SAASkM,MAAMF,GAAa,GAEjC3R,EAAK+R,kBAAiB,GAEtB5R,EAAEH,EAAK2F,UAAU3D,QAAQqD,GAAM8K,SAS9B9L,qBAAqB9C,GAExBtB,KAAK0F,SAASkM,MAAMF,GAAgB1R,KAAK0F,SAASqM,GAAlD,UAGFX,KAAA,WAAO,IAAApE,EAAAhN,KACL,IAAIA,KAAKuQ,kBACNrQ,EAAEF,KAAK0F,UAAUa,SAASf,IAD7B,CAKA,IAAMiM,EAAavR,EAAEkF,MAAMA,GAAM+K,MAEjC,GADAjQ,EAAEF,KAAK0F,UAAU3D,QAAQ0P,IACrBA,EAAW1L,qBAAf,CAIA,IAAM2L,EAAY1R,KAAK2R,gBAEvB3R,KAAK0F,SAASkM,MAAMF,GAAgB1R,KAAK0F,SAASsM,wBAAwBN,GAA1E,KAEAtR,EAAKyB,OAAO7B,KAAK0F,UAEjBxF,EAAEF,KAAK0F,UACJqJ,SAASvJ,IACTc,YAAYd,IACZc,YAAYd,IAEf,IAAMyM,EAAqBjS,KAAKwQ,cAAcxH,OAC9C,GAAyB,EAArBiJ,EACF,IAAK,IAAInJ,EAAI,EAAGA,EAAImJ,EAAoBnJ,IAAK,CAC3C,IAAM/G,EAAU/B,KAAKwQ,cAAc1H,GAC7B/H,EAAWX,EAAKS,uBAAuBkB,GAE7C,GAAiB,OAAbhB,EACYb,EAAE,GAAG0I,MAAM/F,KAAKlC,SAASkI,iBAAiB9H,KAC7CwF,SAASf,KAClBtF,EAAE6B,GAASgN,SAASvJ,IACjBqM,KAAK,iBAAiB,GAMjC7R,KAAK8R,kBAAiB,GAUtB9R,KAAK0F,SAASkM,MAAMF,GAAa,GACjC,IAAMpQ,EAAqBlB,EAAKiB,iCAAiCrB,KAAK0F,UAEtExF,EAAEF,KAAK0F,UACJvF,IAAIC,EAAKR,eAZK,WACfoN,EAAK8E,kBAAiB,GACtB5R,EAAE8M,EAAKtH,UACJY,YAAYd,IACZuJ,SAASvJ,IACTzD,QAAQqD,GAAMgL,UAQhBhM,qBAAqB9C,QAG1BwQ,iBAAA,SAAiBI,GACflS,KAAKuQ,iBAAmB2B,KAG1BjM,QAAA,WACE/F,EAAEgG,WAAWlG,KAAK0F,SAAUT,IAE5BjF,KAAKkL,QAAmB,KACxBlL,KAAKiR,QAAmB,KACxBjR,KAAK0F,SAAmB,KACxB1F,KAAKwQ,cAAmB,KACxBxQ,KAAKuQ,iBAAmB,QAK1BpF,WAAA,SAAW5I,GAOT,OANAA,EAAMqK,EAAA,GACD1D,GADC,GAED3G,IAEEmF,OAASzF,QAAQM,EAAOmF,QAC/BtH,EAAKiC,gBAAgB2C,GAAMzC,EAAQkH,IAC5BlH,KAGToP,cAAA,WAEE,OADiBzR,EAAEF,KAAK0F,UAAUa,SAAS8J,IACzBA,GAAkBA,MAGtCa,WAAA,WAAa,IACP/K,EADOgH,EAAAnN,KAGPI,EAAK8B,UAAUlC,KAAKkL,QAAQ/E,SAC9BA,EAASnG,KAAKkL,QAAQ/E,OAGoB,oBAA/BnG,KAAKkL,QAAQ/E,OAAOhC,SAC7BgC,EAASnG,KAAKkL,QAAQ/E,OAAO,KAG/BA,EAASxF,SAASQ,cAAcnB,KAAKkL,QAAQ/E,QAG/C,IAAMpF,EAAQ,yCAC6Bf,KAAKkL,QAAQ/E,OAD1C,KAGR2I,EAAW,GAAGlG,MAAM/F,KAAKsD,EAAO0C,iBAAiB9H,IAQvD,OAPAb,EAAE4O,GAAUlI,KAAK,SAACkC,EAAGhI,GACnBqM,EAAKgE,0BACHb,EAAS6B,sBAAsBrR,GAC/B,CAACA,MAIEqF,KAGTgL,0BAAA,SAA0BrQ,EAASsR,GACjC,IAAMC,EAASnS,EAAEY,GAASyF,SAASf,IAE/B4M,EAAapJ,QACf9I,EAAEkS,GACC7J,YAAY/C,IAAsB6M,GAClCR,KAAK,gBAAiBQ,MAMtBF,sBAAP,SAA6BrR,GAC3B,IAAMC,EAAWX,EAAKS,uBAAuBC,GAC7C,OAAOC,EAAWJ,SAASQ,cAAcJ,GAAY,QAGhD4F,iBAAP,SAAwBpE,GACtB,OAAOvC,KAAK4G,KAAK,WACf,IAAM0L,EAAUpS,EAAEF,MACd8G,EAAYwL,EAAMxL,KAAK7B,IACrBiG,EAAO0B,EAAA,GACR1D,GADQ,GAERoJ,EAAMxL,OAFE,GAGU,iBAAXvE,GAAuBA,EAASA,EAAS,IAYrD,IATKuE,GAAQoE,EAAQxD,QAAU,YAAYrE,KAAKd,KAC9C2I,EAAQxD,QAAS,GAGdZ,IACHA,EAAO,IAAIwJ,EAAStQ,KAAMkL,GAC1BoH,EAAMxL,KAAK7B,GAAU6B,IAGD,iBAAXvE,EAAqB,CAC9B,GAA4B,oBAAjBuE,EAAKvE,GACd,MAAM,IAAIyB,UAAJ,oBAAkCzB,EAAlC,KAERuE,EAAKvE,iDAjQT,MApFwB,wCAwFxB,OAAO2G,YAyQXhJ,EAAES,UAAUuG,GAAG9B,GAAMG,eAAgB+B,GAAsB,SAAUjD,GAE/B,MAAhCA,EAAMkO,cAAcpK,SACtB9D,EAAM4C,iBAGR,IAAMuL,EAAWtS,EAAEF,MACbe,EAAWX,EAAKS,uBAAuBb,MACvCyS,EAAY,GAAG7J,MAAM/F,KAAKlC,SAASkI,iBAAiB9H,IAE1Db,EAAEuS,GAAW7L,KAAK,WAChB,IAAM8L,EAAUxS,EAAEF,MAEZuC,EADUmQ,EAAQ5L,KAAK7B,IACN,SAAWuN,EAAS1L,OAC3CwJ,GAAS3J,iBAAiB9D,KAAK6P,EAASnQ,OAU5CrC,EAAEgE,GAAGc,IAAQsL,GAAS3J,iBACtBzG,EAAEgE,GAAGc,IAAMmC,YAAcmJ,GACzBpQ,EAAEgE,GAAGc,IAAMoC,WAAa,WAEtB,OADAlH,EAAEgE,GAAGc,IAAQG,GACNmL,GAAS3J,kBCtXlB,IAAIgM,GAA8B,oBAAXjK,QAA8C,oBAAb/H,UAAiD,oBAAd2K,UAEvFsH,GAAkB,WAEpB,IADA,IAAIC,EAAwB,CAAC,OAAQ,UAAW,WACvC/J,EAAI,EAAGA,EAAI+J,EAAsB7J,OAAQF,GAAK,EACrD,GAAI6J,IAAsE,GAAzDrH,UAAUwH,UAAUhF,QAAQ+E,EAAsB/J,IACjE,OAAO,EAGX,OAAO,EAPa,GAqCtB,IAWIiK,GAXqBJ,IAAajK,OAAOsK,QA3B7C,SAA2B9O,GACzB,IAAIjE,GAAS,EACb,OAAO,WACDA,IAGJA,GAAS,EACTyI,OAAOsK,QAAQC,UAAUC,KAAK,WAC5BjT,GAAS,EACTiE,SAKN,SAAsBA,GACpB,IAAIiP,GAAY,EAChB,OAAO,WACAA,IACHA,GAAY,EACZ9S,WAAW,WACT8S,GAAY,EACZjP,KACC0O,OAyBT,SAASQ,GAAWC,GAElB,OAAOA,GAA8D,sBADvD,GACoBpQ,SAASJ,KAAKwQ,GAUlD,SAASC,GAAyBxS,EAAS2B,GACzC,GAAyB,IAArB3B,EAAQsB,SACV,MAAO,GAGT,IACIb,EADST,EAAQyS,cAAcC,YAClBC,iBAAiB3S,EAAS,MAC3C,OAAO2B,EAAWlB,EAAIkB,GAAYlB,EAUpC,SAASmS,GAAc5S,GACrB,MAAyB,SAArBA,EAAQ6S,SACH7S,EAEFA,EAAQ+C,YAAc/C,EAAQ8S,KAUvC,SAASC,GAAgB/S,GAEvB,IAAKA,EACH,OAAOH,SAASmT,KAGlB,OAAQhT,EAAQ6S,UACd,IAAK,OACL,IAAK,OACH,OAAO7S,EAAQyS,cAAcO,KAC/B,IAAK,YACH,OAAOhT,EAAQgT,KAKnB,IAAIC,EAAwBT,GAAyBxS,GACjDkT,EAAWD,EAAsBC,SACjCC,EAAYF,EAAsBE,UAClCC,EAAYH,EAAsBG,UAEtC,MAAI,wBAAwB7Q,KAAK2Q,EAAWE,EAAYD,GAC/CnT,EAGF+S,GAAgBH,GAAc5S,IAUvC,SAASqT,GAAiBC,GACxB,OAAOA,GAAaA,EAAUC,cAAgBD,EAAUC,cAAgBD,EAG1E,IAAIE,GAAS3B,OAAgBjK,OAAO6L,uBAAwB5T,SAAS6T,cACjEC,GAAS9B,IAAa,UAAUtP,KAAKiI,UAAUwH,WASnD,SAAS4B,GAAKzQ,GACZ,OAAgB,KAAZA,EACKqQ,GAEO,KAAZrQ,EACKwQ,GAEFH,IAAUG,GAUnB,SAASE,GAAgB7T,GACvB,IAAKA,EACH,OAAOH,SAAS8C,gBAQlB,IALA,IAAImR,EAAiBF,GAAK,IAAM/T,SAASmT,KAAO,KAG5Ce,EAAe/T,EAAQ+T,cAAgB,KAEpCA,IAAiBD,GAAkB9T,EAAQgU,oBAChDD,GAAgB/T,EAAUA,EAAQgU,oBAAoBD,aAGxD,IAAIlB,EAAWkB,GAAgBA,EAAalB,SAE5C,OAAKA,GAAyB,SAAbA,GAAoC,SAAbA,GAMsB,IAA1D,CAAC,KAAM,KAAM,SAAS7F,QAAQ+G,EAAalB,WAA2E,WAAvDL,GAAyBuB,EAAc,YACjGF,GAAgBE,GAGlBA,EATE/T,EAAUA,EAAQyS,cAAc9P,gBAAkB9C,SAAS8C,gBA4BtE,SAASsR,GAAQC,GACf,OAAwB,OAApBA,EAAKnR,WACAkR,GAAQC,EAAKnR,YAGfmR,EAWT,SAASC,GAAuBC,EAAUC,GAExC,KAAKD,GAAaA,EAAS9S,UAAa+S,GAAaA,EAAS/S,UAC5D,OAAOzB,SAAS8C,gBAIlB,IAAI2R,EAAQF,EAASG,wBAAwBF,GAAYG,KAAKC,4BAC1DnI,EAAQgI,EAAQF,EAAWC,EAC3B1H,EAAM2H,EAAQD,EAAWD,EAGzBM,EAAQ7U,SAAS8U,cACrBD,EAAME,SAAStI,EAAO,GACtBoI,EAAMG,OAAOlI,EAAK,GAClB,IAAImI,EAA0BJ,EAAMI,wBAIpC,GAAIV,IAAaU,GAA2BT,IAAaS,GAA2BxI,EAAMnF,SAASwF,GACjG,OApDJ,SAA2B3M,GACzB,IAAI6S,EAAW7S,EAAQ6S,SAEvB,MAAiB,SAAbA,IAGgB,SAAbA,GAAuBgB,GAAgB7T,EAAQ+U,qBAAuB/U,GA8CvEgV,CAAkBF,GACbA,EAGFjB,GAAgBiB,GAIzB,IAAIG,EAAehB,GAAQG,GAC3B,OAAIa,EAAanC,KACRqB,GAAuBc,EAAanC,KAAMuB,GAE1CF,GAAuBC,EAAUH,GAAQI,GAAUvB,MAY9D,SAASoC,GAAUlV,EAAnB,GACE,IAEImV,EAAqB,SAFK,EAAnBlR,UAAUiE,aAA+BkN,IADtD,EAAA,EACiF,OAE9C,YAAc,aAC3CvC,EAAW7S,EAAQ6S,SAEvB,GAAiB,SAAbA,GAAoC,SAAbA,EAM3B,OAAO7S,EAAQmV,GALb,IAAIE,EAAOrV,EAAQyS,cAAc9P,gBAEjC,OADuB3C,EAAQyS,cAAc6C,kBAAoBD,GACzCF,GAsC5B,SAASI,GAAeC,EAAQC,GAC9B,IAAIC,EAAiB,MAATD,EAAe,OAAS,MAChCE,EAAkB,QAAVD,EAAmB,QAAU,SAEzC,OAAO9U,WAAW4U,EAAO,SAAWE,EAAQ,SAAU,IAAM9U,WAAW4U,EAAO,SAAWG,EAAQ,SAAU,IAG7G,SAASC,GAAQH,EAAMzC,EAAMqC,EAAMQ,GACjC,OAAOlW,KAAKmW,IAAI9C,EAAK,SAAWyC,GAAOzC,EAAK,SAAWyC,GAAOJ,EAAK,SAAWI,GAAOJ,EAAK,SAAWI,GAAOJ,EAAK,SAAWI,GAAO7B,GAAK,IAAMjF,SAAS0G,EAAK,SAAWI,IAAS9G,SAASkH,EAAc,UAAqB,WAATJ,EAAoB,MAAQ,UAAY9G,SAASkH,EAAc,UAAqB,WAATJ,EAAoB,SAAW,WAAa,GAG5U,SAASM,GAAelW,GACtB,IAAImT,EAAOnT,EAASmT,KAChBqC,EAAOxV,EAAS8C,gBAChBkT,EAAgBjC,GAAK,KAAOjB,iBAAiB0C,GAEjD,MAAO,CACLW,OAAQJ,GAAQ,SAAU5C,EAAMqC,EAAMQ,GACtCI,MAAOL,GAAQ,QAAS5C,EAAMqC,EAAMQ,IAIxC,IAMIK,GAWK,SAAU7P,EAAa8P,EAAYC,GAGxC,OAFID,GAAYE,GAAiBhQ,EAAYxE,UAAWsU,GACpDC,GAAaC,GAAiBhQ,EAAa+P,GACxC/P,GAbT,SAASgQ,GAAiBzS,EAAQ0S,GAChC,IAAK,IAAItO,EAAI,EAAGA,EAAIsO,EAAMpO,OAAQF,IAAK,CACrC,IAAIuO,EAAaD,EAAMtO,GACvBuO,EAAWC,WAAaD,EAAWC,aAAc,EACjDD,EAAWE,cAAe,EACtB,UAAWF,IAAYA,EAAWG,UAAW,GACjD9U,OAAO+U,eAAe/S,EAAQ2S,EAAWK,IAAKL,IAe/B,SAAjBI,GAA2BtV,EAAKuV,EAAK3U,GAYvC,OAXI2U,KAAOvV,EACTO,OAAO+U,eAAetV,EAAKuV,EAAK,CAC9B3U,MAAOA,EACPuU,YAAY,EACZC,cAAc,EACdC,UAAU,IAGZrV,EAAIuV,GAAO3U,EAGNZ,EAZT,IAeIwV,GAAWjV,OAAOkV,QAAU,SAAUlT,GACxC,IAAK,IAAIoE,EAAI,EAAGA,EAAI/D,UAAUiE,OAAQF,IAAK,CACzC,IAAI+O,EAAS9S,UAAU+D,GAEvB,IAAK,IAAI4O,KAAOG,EACVnV,OAAOC,UAAUC,eAAeC,KAAKgV,EAAQH,KAC/ChT,EAAOgT,GAAOG,EAAOH,IAK3B,OAAOhT,GAUT,SAASoT,GAAcC,GACrB,OAAOJ,GAAS,GAAII,EAAS,CAC3BC,MAAOD,EAAQE,KAAOF,EAAQhB,MAC9BmB,OAAQH,EAAQI,IAAMJ,EAAQjB,SAWlC,SAAS9E,GAAsBlR,GAC7B,IAAIsX,EAAO,GAKX,IACE,GAAI1D,GAAK,IAAK,CACZ0D,EAAOtX,EAAQkR,wBACf,IAAIqG,EAAYrC,GAAUlV,EAAS,OAC/BwX,EAAatC,GAAUlV,EAAS,QACpCsX,EAAKD,KAAOE,EACZD,EAAKH,MAAQK,EACbF,EAAKF,QAAUG,EACfD,EAAKJ,OAASM,OAEdF,EAAOtX,EAAQkR,wBAEjB,MAAOrE,IAET,IAAI4K,EAAS,CACXN,KAAMG,EAAKH,KACXE,IAAKC,EAAKD,IACVpB,MAAOqB,EAAKJ,MAAQI,EAAKH,KACzBnB,OAAQsB,EAAKF,OAASE,EAAKD,KAIzBK,EAA6B,SAArB1X,EAAQ6S,SAAsBkD,GAAe/V,EAAQyS,eAAiB,GAC9EwD,EAAQyB,EAAMzB,OAASjW,EAAQ2X,aAAeF,EAAOxB,MACrDD,EAAS0B,EAAM1B,QAAUhW,EAAQ4X,cAAgBH,EAAOzB,OAExD6B,EAAiB7X,EAAQ8X,YAAc7B,EACvC8B,EAAgB/X,EAAQgB,aAAegV,EAI3C,GAAI6B,GAAkBE,EAAe,CACnC,IAAIvC,EAAShD,GAAyBxS,GACtC6X,GAAkBtC,GAAeC,EAAQ,KACzCuC,GAAiBxC,GAAeC,EAAQ,KAExCiC,EAAOxB,OAAS4B,EAChBJ,EAAOzB,QAAU+B,EAGnB,OAAOf,GAAcS,GAGvB,SAASO,GAAqChK,EAAU3I,EAAxD,GACE,IAAI4S,EAAmC,EAAnBhU,UAAUiE,aAA+BkN,IAD/D,GAAA,EAGMzB,EAASC,GAAK,IACdsE,EAA6B,SAApB7S,EAAOwN,SAChBsF,EAAejH,GAAsBlD,GACrCoK,EAAalH,GAAsB7L,GACnCgT,EAAetF,GAAgB/E,GAE/BwH,EAAShD,GAAyBnN,GAClCiT,EAAiB1X,WAAW4U,EAAO8C,eAAgB,IACnDC,EAAkB3X,WAAW4U,EAAO+C,gBAAiB,IAGrDN,GAAiBC,IACnBE,EAAWf,IAAM1X,KAAKmW,IAAIsC,EAAWf,IAAK,GAC1Ce,EAAWjB,KAAOxX,KAAKmW,IAAIsC,EAAWjB,KAAM,IAE9C,IAAIF,EAAUD,GAAc,CAC1BK,IAAKc,EAAad,IAAMe,EAAWf,IAAMiB,EACzCnB,KAAMgB,EAAahB,KAAOiB,EAAWjB,KAAOoB,EAC5CtC,MAAOkC,EAAalC,MACpBD,OAAQmC,EAAanC,SASvB,GAPAiB,EAAQuB,UAAY,EACpBvB,EAAQwB,WAAa,GAMhB9E,GAAUuE,EAAQ,CACrB,IAAIM,EAAY5X,WAAW4U,EAAOgD,UAAW,IACzCC,EAAa7X,WAAW4U,EAAOiD,WAAY,IAE/CxB,EAAQI,KAAOiB,EAAiBE,EAChCvB,EAAQG,QAAUkB,EAAiBE,EACnCvB,EAAQE,MAAQoB,EAAkBE,EAClCxB,EAAQC,OAASqB,EAAkBE,EAGnCxB,EAAQuB,UAAYA,EACpBvB,EAAQwB,WAAaA,EAOvB,OAJI9E,IAAWsE,EAAgB5S,EAAO8B,SAASkR,GAAgBhT,IAAWgT,GAA0C,SAA1BA,EAAaxF,YACrGoE,EA1NJ,SAAuBK,EAAMtX,EAA7B,GACE,IAAI0Y,EAA8B,EAAnBzU,UAAUiE,aAA+BkN,IAD1D,GAAA,EAGMmC,EAAYrC,GAAUlV,EAAS,OAC/BwX,EAAatC,GAAUlV,EAAS,QAChC2Y,EAAWD,GAAY,EAAI,EAK/B,OAJApB,EAAKD,KAAOE,EAAYoB,EACxBrB,EAAKF,QAAUG,EAAYoB,EAC3BrB,EAAKH,MAAQK,EAAamB,EAC1BrB,EAAKJ,OAASM,EAAamB,EACpBrB,EAgNKsB,CAAc3B,EAAS5R,IAG5B4R,EAuDT,SAAS4B,GAA6B7Y,GAEpC,IAAKA,IAAYA,EAAQ8Y,eAAiBlF,KACxC,OAAO/T,SAAS8C,gBAGlB,IADA,IAAIoW,EAAK/Y,EAAQ8Y,cACVC,GAAoD,SAA9CvG,GAAyBuG,EAAI,cACxCA,EAAKA,EAAGD,cAEV,OAAOC,GAAMlZ,SAAS8C,gBAcxB,SAASqW,GAAcC,EAAQ3F,EAAW4F,EAASC,EAAnD,GACE,IAAIlB,EAAmC,EAAnBhU,UAAUiE,aAA+BkN,IAD/D,GAAA,EAKMgE,EAAa,CAAE/B,IAAK,EAAGF,KAAM,GAC7BpD,EAAekE,EAAgBY,GAA6BI,GAAU9E,GAAuB8E,EAAQ5F,GAAiBC,IAG1H,GAA0B,aAAtB6F,EACFC,EArFJ,SAAuDpZ,EAAvD,GACE,IAAIqZ,EAAmC,EAAnBpV,UAAUiE,aAA+BkN,IAD/D,GAAA,EAGMC,EAAOrV,EAAQyS,cAAc9P,gBAC7B2W,EAAiBtB,GAAqChY,EAASqV,GAC/DY,EAAQtW,KAAKmW,IAAIT,EAAKsC,YAAa/P,OAAO2R,YAAc,GACxDvD,EAASrW,KAAKmW,IAAIT,EAAKuC,aAAchQ,OAAO4R,aAAe,GAE3DjC,EAAa8B,EAAkC,EAAlBnE,GAAUG,GACvCmC,EAAc6B,EAA0C,EAA1BnE,GAAUG,EAAM,QASlD,OAAO2B,GAPM,CACXK,IAAKE,EAAY+B,EAAejC,IAAMiC,EAAed,UACrDrB,KAAMK,EAAa8B,EAAenC,KAAOmC,EAAeb,WACxDxC,MAAOA,EACPD,OAAQA,IAsEKyD,CAA8C1F,EAAckE,OACpE,CAEL,IAAIyB,OAAiB,EACK,iBAAtBP,EAE8B,UADhCO,EAAiB3G,GAAgBH,GAAcU,KAC5BT,WACjB6G,EAAiBT,EAAOxG,cAAc9P,iBAGxC+W,EAD+B,WAAtBP,EACQF,EAAOxG,cAAc9P,gBAErBwW,EAGnB,IAAIlC,EAAUe,GAAqC0B,EAAgB3F,EAAckE,GAGjF,GAAgC,SAA5ByB,EAAe7G,UA1EvB,SAAS8G,EAAQ3Z,GACf,IAAI6S,EAAW7S,EAAQ6S,SACvB,GAAiB,SAAbA,GAAoC,SAAbA,EACzB,OAAO,EAET,GAAsD,UAAlDL,GAAyBxS,EAAS,YACpC,OAAO,EAET,IAAI+C,EAAa6P,GAAc5S,GAC/B,QAAK+C,GAGE4W,EAAQ5W,GA8D8B4W,CAAQ5F,GAWjDqF,EAAanC,MAXmD,CAChE,IAAI2C,EAAkB7D,GAAekD,EAAOxG,eACxCuD,EAAS4D,EAAgB5D,OACzBC,EAAQ2D,EAAgB3D,MAE5BmD,EAAW/B,KAAOJ,EAAQI,IAAMJ,EAAQuB,UACxCY,EAAWhC,OAASpB,EAASiB,EAAQI,IACrC+B,EAAWjC,MAAQF,EAAQE,KAAOF,EAAQwB,WAC1CW,EAAWlC,MAAQjB,EAAQgB,EAAQE,MASvC,IAAI0C,EAAqC,iBADzCX,EAAUA,GAAW,GAOrB,OALAE,EAAWjC,MAAQ0C,EAAkBX,EAAUA,EAAQ/B,MAAQ,EAC/DiC,EAAW/B,KAAOwC,EAAkBX,EAAUA,EAAQ7B,KAAO,EAC7D+B,EAAWlC,OAAS2C,EAAkBX,EAAUA,EAAQhC,OAAS,EACjEkC,EAAWhC,QAAUyC,EAAkBX,EAAUA,EAAQ9B,QAAU,EAE5DgC,EAmBT,SAASU,GAAqBC,EAAWC,EAASf,EAAQ3F,EAAW6F,EAArE,GACE,IAAID,EAA6B,EAAnBjV,UAAUiE,aAA+BkN,IADzD,EAAA,EACoF,EAElF,IAAmC,IAA/B2E,EAAU/M,QAAQ,QACpB,OAAO+M,EAGT,IAAIX,EAAaJ,GAAcC,EAAQ3F,EAAW4F,EAASC,GAEvDc,EAAQ,CACV5C,IAAK,CACHpB,MAAOmD,EAAWnD,MAClBD,OAAQgE,EAAQ3C,IAAM+B,EAAW/B,KAEnCH,MAAO,CACLjB,MAAOmD,EAAWlC,MAAQ8C,EAAQ9C,MAClClB,OAAQoD,EAAWpD,QAErBoB,OAAQ,CACNnB,MAAOmD,EAAWnD,MAClBD,OAAQoD,EAAWhC,OAAS4C,EAAQ5C,QAEtCD,KAAM,CACJlB,MAAO+D,EAAQ7C,KAAOiC,EAAWjC,KACjCnB,OAAQoD,EAAWpD,SAInBkE,EAActY,OAAOuY,KAAKF,GAAOG,IAAI,SAAUxD,GACjD,OAAOC,GAAS,CACdD,IAAKA,GACJqD,EAAMrD,GAAM,CACbyD,KAhDN,SAAiBC,GAIf,OAHYA,EAAKrE,MACJqE,EAAKtE,OA8CRuE,CAAQN,EAAMrD,QAErB4D,KAAK,SAAUC,EAAGC,GACnB,OAAOA,EAAEL,KAAOI,EAAEJ,OAGhBM,EAAgBT,EAAYnK,OAAO,SAAU6K,GAC/C,IAAI3E,EAAQ2E,EAAM3E,MACdD,EAAS4E,EAAM5E,OACnB,OAAOC,GAASgD,EAAOtB,aAAe3B,GAAUiD,EAAOrB,eAGrDiD,EAA2C,EAAvBF,EAAczS,OAAayS,EAAc,GAAG/D,IAAMsD,EAAY,GAAGtD,IAErFkE,EAAYf,EAAUjZ,MAAM,KAAK,GAErC,OAAO+Z,GAAqBC,EAAY,IAAMA,EAAY,IAa5D,SAASC,GAAoBC,EAAO/B,EAAQ3F,EAA5C,GACE,IAAI2E,EAAmC,EAAnBhU,UAAUiE,aAA+BkN,IAD/D,EAAA,EAC0F,KAGxF,OAAO4C,GAAqC1E,EADnB2E,EAAgBY,GAA6BI,GAAU9E,GAAuB8E,EAAQ5F,GAAiBC,IACrD2E,GAU7E,SAASgD,GAAcjb,GACrB,IACIwV,EADSxV,EAAQyS,cAAcC,YACfC,iBAAiB3S,GACjCkb,EAAIta,WAAW4U,EAAOgD,WAAa,GAAK5X,WAAW4U,EAAO2F,cAAgB,GAC1EC,EAAIxa,WAAW4U,EAAOiD,YAAc,GAAK7X,WAAW4U,EAAO6F,aAAe,GAK9E,MAJa,CACXpF,MAAOjW,EAAQ8X,YAAcsD,EAC7BpF,OAAQhW,EAAQgB,aAAeka,GAYnC,SAASI,GAAqBvB,GAC5B,IAAIwB,EAAO,CAAEpE,KAAM,QAASD,MAAO,OAAQE,OAAQ,MAAOC,IAAK,UAC/D,OAAO0C,EAAUyB,QAAQ,yBAA0B,SAAUC,GAC3D,OAAOF,EAAKE,KAchB,SAASC,GAAiBzC,EAAQ0C,EAAkB5B,GAClDA,EAAYA,EAAUjZ,MAAM,KAAK,GAGjC,IAAI8a,EAAaX,GAAchC,GAG3B4C,EAAgB,CAClB5F,MAAO2F,EAAW3F,MAClBD,OAAQ4F,EAAW5F,QAIjB8F,GAAoD,IAA1C,CAAC,QAAS,QAAQ9O,QAAQ+M,GACpCgC,EAAWD,EAAU,MAAQ,OAC7BE,EAAgBF,EAAU,OAAS,MACnCG,EAAcH,EAAU,SAAW,QACnCI,EAAwBJ,EAAqB,QAAX,SAStC,OAPAD,EAAcE,GAAYJ,EAAiBI,GAAYJ,EAAiBM,GAAe,EAAIL,EAAWK,GAAe,EAEnHJ,EAAcG,GADZjC,IAAciC,EACeL,EAAiBK,GAAiBJ,EAAWM,GAE7CP,EAAiBL,GAAqBU,IAGhEH,EAYT,SAASM,GAAKC,EAAKC,GAEjB,OAAIC,MAAMza,UAAUsa,KACXC,EAAID,KAAKE,GAIXD,EAAIrM,OAAOsM,GAAO,GAqC3B,SAASE,GAAaC,EAAWxW,EAAMyW,GAoBrC,YAnB8BrH,IAATqH,EAAqBD,EAAYA,EAAU1U,MAAM,EA1BxE,SAAmBsU,EAAKM,EAAMza,GAE5B,GAAIqa,MAAMza,UAAU8a,UAClB,OAAOP,EAAIO,UAAU,SAAUC,GAC7B,OAAOA,EAAIF,KAAUza,IAKzB,IAAIG,EAAQ+Z,GAAKC,EAAK,SAAU/a,GAC9B,OAAOA,EAAIqb,KAAUza,IAEvB,OAAOma,EAAIpP,QAAQ5K,GAcsDua,CAAUH,EAAW,OAAQC,KAEvFI,QAAQ,SAAUlE,GAC3BA,EAAmB,UAErBmE,QAAQC,KAAK,yDAEf,IAAI3Z,EAAKuV,EAAmB,UAAKA,EAASvV,GACtCuV,EAASqE,SAAW1K,GAAWlP,KAIjC4C,EAAKiR,QAAQgC,OAASjC,GAAchR,EAAKiR,QAAQgC,QACjDjT,EAAKiR,QAAQ3D,UAAY0D,GAAchR,EAAKiR,QAAQ3D,WAEpDtN,EAAO5C,EAAG4C,EAAM2S,MAIb3S,EA8DT,SAASiX,GAAkBT,EAAWU,GACpC,OAAOV,EAAUW,KAAK,SAAU7C,GAC9B,IAAI8C,EAAO9C,EAAK8C,KAEhB,OADc9C,EAAK0C,SACDI,IAASF,IAW/B,SAASG,GAAyB1b,GAIhC,IAHA,IAAI2b,EAAW,EAAC,EAAO,KAAM,SAAU,MAAO,KAC1CC,EAAY5b,EAAS6b,OAAO,GAAG/a,cAAgBd,EAASmG,MAAM,GAEzDE,EAAI,EAAGA,EAAIsV,EAASpV,OAAQF,IAAK,CACxC,IAAItI,EAAS4d,EAAStV,GAClByV,EAAU/d,EAAS,GAAKA,EAAS6d,EAAY5b,EACjD,GAA4C,oBAAjC9B,SAASmT,KAAKlC,MAAM2M,GAC7B,OAAOA,EAGX,OAAO,KAsCT,SAASC,GAAU1d,GACjB,IAAIyS,EAAgBzS,EAAQyS,cAC5B,OAAOA,EAAgBA,EAAcC,YAAc9K,OAoBrD,SAAS+V,GAAoBrK,EAAWsK,EAAS5C,EAAO6C,GAEtD7C,EAAM6C,YAAcA,EACpBH,GAAUpK,GAAWwK,iBAAiB,SAAU9C,EAAM6C,YAAa,CAAEE,SAAS,IAG9E,IAAIC,EAAgBjL,GAAgBO,GAKpC,OA5BF,SAAS2K,EAAsB5F,EAAc9U,EAAO2a,EAAUC,GAC5D,IAAIC,EAAmC,SAA1B/F,EAAaxF,SACtBjP,EAASwa,EAAS/F,EAAa5F,cAAcC,YAAc2F,EAC/DzU,EAAOka,iBAAiBva,EAAO2a,EAAU,CAAEH,SAAS,IAE/CK,GACHH,EAAsBlL,GAAgBnP,EAAOb,YAAaQ,EAAO2a,EAAUC,GAE7EA,EAAcjO,KAAKtM,GAgBnBqa,CAAsBD,EAAe,SAAUhD,EAAM6C,YAAa7C,EAAMmD,eACxEnD,EAAMgD,cAAgBA,EACtBhD,EAAMqD,eAAgB,EAEfrD,EA6CT,SAASsD,KACHpf,KAAK8b,MAAMqD,gBACbE,qBAAqBrf,KAAKsf,gBAC1Btf,KAAK8b,MA3BT,SAA8B1H,EAAW0H,GAcvC,OAZA0C,GAAUpK,GAAWmL,oBAAoB,SAAUzD,EAAM6C,aAGzD7C,EAAMmD,cAActB,QAAQ,SAAUjZ,GACpCA,EAAO6a,oBAAoB,SAAUzD,EAAM6C,eAI7C7C,EAAM6C,YAAc,KACpB7C,EAAMmD,cAAgB,GACtBnD,EAAMgD,cAAgB,KACtBhD,EAAMqD,eAAgB,EACfrD,EAaQ0D,CAAqBxf,KAAKoU,UAAWpU,KAAK8b,QAW3D,SAAS2D,GAAUC,GACjB,MAAa,KAANA,IAAaC,MAAMje,WAAWge,KAAOE,SAASF,GAWvD,SAASG,GAAU/e,EAASwV,GAC1B5T,OAAOuY,KAAK3E,GAAQqH,QAAQ,SAAUH,GACpC,IAAIsC,EAAO,IAEkE,IAAzE,CAAC,QAAS,SAAU,MAAO,QAAS,SAAU,QAAQhS,QAAQ0P,IAAgBiC,GAAUnJ,EAAOkH,MACjGsC,EAAO,MAEThf,EAAQ8Q,MAAM4L,GAAQlH,EAAOkH,GAAQsC,IAkGzC,SAASC,GAAkBjZ,EAAMkZ,GAOjB,SAAVC,EAA2BC,GAC7B,OAAOA,EAPT,IAAIC,EAAgBrZ,EAAKiR,QACrBgC,EAASoG,EAAcpG,OACvB3F,EAAY+L,EAAc/L,UAC1BgM,EAAQ3f,KAAK2f,MACbC,EAAQ5f,KAAK4f,MAMbC,EAAiBF,EAAMhM,EAAU2C,OACjCwJ,EAAcH,EAAMrG,EAAOhD,OAE3ByJ,GAA4D,IAA/C,CAAC,OAAQ,SAAS1S,QAAQhH,EAAK+T,WAC5C4F,GAA+C,IAAjC3Z,EAAK+T,UAAU/M,QAAQ,KAIrC4S,EAAuBV,EAAwBQ,GAAcC,GAH3CH,EAAiB,GAAMC,EAAc,EAGuCH,EAAQC,EAAjEJ,EACrCU,EAAqBX,EAAwBI,EAAVH,EAEvC,MAAO,CACLhI,KAAMyI,EANWJ,EAAiB,GAAM,GAAKC,EAAc,GAAM,IAMtBE,GAAeT,EAAcjG,EAAO9B,KAAO,EAAI8B,EAAO9B,MACjGE,IAAKwI,EAAkB5G,EAAO5B,KAC9BD,OAAQyI,EAAkB5G,EAAO7B,QACjCF,MAAO0I,EAAoB3G,EAAO/B,QAItC,IAAI4I,GAAYjO,IAAa,WAAWtP,KAAKiI,UAAUwH,WA8GvD,SAAS+N,GAAmBvD,EAAWwD,EAAgBC,GACrD,IAAIC,EAAa/D,GAAKK,EAAW,SAAUlC,GAEzC,OADWA,EAAK8C,OACA4C,IAGdG,IAAeD,GAAc1D,EAAUW,KAAK,SAAUxE,GACxD,OAAOA,EAASyE,OAAS6C,GAAiBtH,EAASqE,SAAWrE,EAASrE,MAAQ4L,EAAW5L,QAG5F,IAAK6L,EAAY,CACf,IAAIC,EAAc,IAAMJ,EAAiB,IACrCK,EAAY,IAAMJ,EAAgB,IACtCnD,QAAQC,KAAKsD,EAAY,4BAA8BD,EAAc,4DAA8DA,EAAc,KAEnJ,OAAOD,EAoIT,IAAIG,GAAa,CAAC,aAAc,OAAQ,WAAY,YAAa,MAAO,UAAW,cAAe,QAAS,YAAa,aAAc,SAAU,eAAgB,WAAY,OAAQ,cAGhLC,GAAkBD,GAAWxY,MAAM,GAYvC,SAAS0Y,GAAUzG,EAAnB,GACE,IAAI0G,EAA6B,EAAnBxc,UAAUiE,aAA+BkN,IADzD,GAAA,EAGM3J,EAAQ8U,GAAgBvT,QAAQ+M,GAChCqC,EAAMmE,GAAgBzY,MAAM2D,EAAQ,GAAGiV,OAAOH,GAAgBzY,MAAM,EAAG2D,IAC3E,OAAOgV,EAAUrE,EAAIuE,UAAYvE,EAGnC,IAAIwE,GACI,OADJA,GAES,YAFTA,GAGgB,mBAiMpB,SAASC,GAAYC,EAAQjF,EAAeF,EAAkBoF,GAC5D,IAAI9J,EAAU,CAAC,EAAG,GAKd+J,GAA0D,IAA9C,CAAC,QAAS,QAAQhU,QAAQ+T,GAItCE,EAAYH,EAAOhgB,MAAM,WAAWsZ,IAAI,SAAU8G,GACpD,OAAOA,EAAK9gB,SAKV+gB,EAAUF,EAAUjU,QAAQmP,GAAK8E,EAAW,SAAUC,GACxD,OAAgC,IAAzBA,EAAKE,OAAO,WAGjBH,EAAUE,KAAiD,IAArCF,EAAUE,GAASnU,QAAQ,MACnD8P,QAAQC,KAAK,gFAKf,IAAIsE,EAAa,cACbC,GAAmB,IAAbH,EAAiB,CAACF,EAAUnZ,MAAM,EAAGqZ,GAAST,OAAO,CAACO,EAAUE,GAASrgB,MAAMugB,GAAY,KAAM,CAACJ,EAAUE,GAASrgB,MAAMugB,GAAY,IAAIX,OAAOO,EAAUnZ,MAAMqZ,EAAU,KAAO,CAACF,GAqC9L,OAlCAK,EAAMA,EAAIlH,IAAI,SAAUmH,EAAI9V,GAE1B,IAAIwQ,GAAyB,IAAVxQ,GAAeuV,EAAYA,GAAa,SAAW,QAClEQ,GAAoB,EACxB,OAAOD,EAGNE,OAAO,SAAUhH,EAAGC,GACnB,MAAwB,KAApBD,EAAEA,EAAEvS,OAAS,KAAwC,IAA3B,CAAC,IAAK,KAAK8E,QAAQ0N,IAC/CD,EAAEA,EAAEvS,OAAS,GAAKwS,EAClB8G,GAAoB,EACb/G,GACE+G,GACT/G,EAAEA,EAAEvS,OAAS,IAAMwS,EACnB8G,GAAoB,EACb/G,GAEAA,EAAEiG,OAAOhG,IAEjB,IAEFN,IAAI,SAAUsH,GACb,OAxGN,SAAiBA,EAAKzF,EAAaJ,EAAeF,GAEhD,IAAI7a,EAAQ4gB,EAAItf,MAAM,6BAClBH,GAASnB,EAAM,GACfke,EAAOle,EAAM,GAGjB,IAAKmB,EACH,OAAOyf,EAGT,GAA0B,IAAtB1C,EAAKhS,QAAQ,KAcV,MAAa,OAATgS,GAA0B,OAATA,EAYnB/c,GATM,OAAT+c,EACKrf,KAAKmW,IAAIjW,SAAS8C,gBAAgBiV,aAAchQ,OAAO4R,aAAe,GAEtE7Z,KAAKmW,IAAIjW,SAAS8C,gBAAgBgV,YAAa/P,OAAO2R,YAAc,IAE/D,IAAMtX,EArBpB,IAAIjC,OAAU,EACd,OAAQgf,GACN,IAAK,KACHhf,EAAU6b,EACV,MACF,IAAK,IACL,IAAK,KACL,QACE7b,EAAU2b,EAId,OADW3E,GAAchX,GACbic,GAAe,IAAMha,EAgFxB0f,CAAQD,EAAKzF,EAAaJ,EAAeF,QAKhDkB,QAAQ,SAAU0E,EAAI9V,GACxB8V,EAAG1E,QAAQ,SAAUqE,EAAMU,GACrBjD,GAAUuC,KACZjK,EAAQxL,IAAUyV,GAA2B,MAAnBK,EAAGK,EAAS,IAAc,EAAI,QAIvD3K,EA2OT,IAkWI4K,GAAW,CAKb9H,UAAW,SAMX+H,eAAe,EAMfzD,eAAe,EAOf0D,iBAAiB,EAQjBC,SAAU,aAUVC,SAAU,aAOVzF,UAnZc,CASd0F,MAAO,CAEL5N,MAAO,IAEP0I,SAAS,EAET5Z,GA9HJ,SAAe4C,GACb,IAAI+T,EAAY/T,EAAK+T,UACjBgH,EAAgBhH,EAAUjZ,MAAM,KAAK,GACrCqhB,EAAiBpI,EAAUjZ,MAAM,KAAK,GAG1C,GAAIqhB,EAAgB,CAClB,IAAI9C,EAAgBrZ,EAAKiR,QACrB3D,EAAY+L,EAAc/L,UAC1B2F,EAASoG,EAAcpG,OAEvByG,GAA2D,IAA9C,CAAC,SAAU,OAAO1S,QAAQ+T,GACvCqB,EAAO1C,EAAa,OAAS,MAC7BzD,EAAcyD,EAAa,QAAU,SAErC2C,EAAe,CACjB/V,MAAOqK,GAAe,GAAIyL,EAAM9O,EAAU8O,IAC1CzV,IAAKgK,GAAe,GAAIyL,EAAM9O,EAAU8O,GAAQ9O,EAAU2I,GAAehD,EAAOgD,KAGlFjW,EAAKiR,QAAQgC,OAASpC,GAAS,GAAIoC,EAAQoJ,EAAaF,IAG1D,OAAOnc,IAgJP8a,OAAQ,CAENxM,MAAO,IAEP0I,SAAS,EAET5Z,GA7RJ,SAAgB4C,EAAMsU,GACpB,IAAIwG,EAASxG,EAAKwG,OACd/G,EAAY/T,EAAK+T,UACjBsF,EAAgBrZ,EAAKiR,QACrBgC,EAASoG,EAAcpG,OACvB3F,EAAY+L,EAAc/L,UAE1ByN,EAAgBhH,EAAUjZ,MAAM,KAAK,GAErCmW,OAAU,EAsBd,OApBEA,EADE0H,IAAWmC,GACH,EAAEA,EAAQ,GAEVD,GAAYC,EAAQ7H,EAAQ3F,EAAWyN,GAG7B,SAAlBA,GACF9H,EAAO5B,KAAOJ,EAAQ,GACtBgC,EAAO9B,MAAQF,EAAQ,IACI,UAAlB8J,GACT9H,EAAO5B,KAAOJ,EAAQ,GACtBgC,EAAO9B,MAAQF,EAAQ,IACI,QAAlB8J,GACT9H,EAAO9B,MAAQF,EAAQ,GACvBgC,EAAO5B,KAAOJ,EAAQ,IACK,WAAlB8J,IACT9H,EAAO9B,MAAQF,EAAQ,GACvBgC,EAAO5B,KAAOJ,EAAQ,IAGxBjR,EAAKiT,OAASA,EACPjT,GAkQL8a,OAAQ,GAoBVwB,gBAAiB,CAEfhO,MAAO,IAEP0I,SAAS,EAET5Z,GAlRJ,SAAyB4C,EAAM4X,GAC7B,IAAIzE,EAAoByE,EAAQzE,mBAAqBtF,GAAgB7N,EAAKuc,SAAStJ,QAK/EjT,EAAKuc,SAASjP,YAAc6F,IAC9BA,EAAoBtF,GAAgBsF,IAMtC,IAAIqJ,EAAgBnF,GAAyB,aACzCoF,EAAezc,EAAKuc,SAAStJ,OAAOnI,MACpCuG,EAAMoL,EAAapL,IACnBF,EAAOsL,EAAatL,KACpBuL,EAAYD,EAAaD,GAE7BC,EAAapL,IAAM,GACnBoL,EAAatL,KAAO,GACpBsL,EAAaD,GAAiB,GAE9B,IAAIpJ,EAAaJ,GAAchT,EAAKuc,SAAStJ,OAAQjT,EAAKuc,SAASjP,UAAWsK,EAAQ1E,QAASC,EAAmBnT,EAAK8b,eAIvHW,EAAapL,IAAMA,EACnBoL,EAAatL,KAAOA,EACpBsL,EAAaD,GAAiBE,EAE9B9E,EAAQxE,WAAaA,EAErB,IAAI9E,EAAQsJ,EAAQ+E,SAChB1J,EAASjT,EAAKiR,QAAQgC,OAEtBoD,EAAQ,CACVuG,QAAS,SAAiB7I,GACxB,IAAI9X,EAAQgX,EAAOc,GAInB,OAHId,EAAOc,GAAaX,EAAWW,KAAe6D,EAAQiF,sBACxD5gB,EAAQtC,KAAKmW,IAAImD,EAAOc,GAAYX,EAAWW,KAE1CpD,GAAe,GAAIoD,EAAW9X,IAEvC6gB,UAAW,SAAmB/I,GAC5B,IAAIgC,EAAyB,UAAdhC,EAAwB,OAAS,MAC5C9X,EAAQgX,EAAO8C,GAInB,OAHI9C,EAAOc,GAAaX,EAAWW,KAAe6D,EAAQiF,sBACxD5gB,EAAQtC,KAAKojB,IAAI9J,EAAO8C,GAAW3C,EAAWW,IAA4B,UAAdA,EAAwBd,EAAOhD,MAAQgD,EAAOjD,UAErGW,GAAe,GAAIoF,EAAU9Z,KAWxC,OAPAqS,EAAMuI,QAAQ,SAAU9C,GACtB,IAAIqI,GAA+C,IAAxC,CAAC,OAAQ,OAAOpV,QAAQ+M,GAAoB,UAAY,YACnEd,EAASpC,GAAS,GAAIoC,EAAQoD,EAAM+F,GAAMrI,MAG5C/T,EAAKiR,QAAQgC,OAASA,EAEfjT,GA2NL2c,SAAU,CAAC,OAAQ,QAAS,MAAO,UAOnCzJ,QAAS,EAMTC,kBAAmB,gBAYrB6J,aAAc,CAEZ1O,MAAO,IAEP0I,SAAS,EAET5Z,GAlgBJ,SAAsB4C,GACpB,IAAIqZ,EAAgBrZ,EAAKiR,QACrBgC,EAASoG,EAAcpG,OACvB3F,EAAY+L,EAAc/L,UAE1ByG,EAAY/T,EAAK+T,UAAUjZ,MAAM,KAAK,GACtCye,EAAQ5f,KAAK4f,MACbG,GAAuD,IAA1C,CAAC,MAAO,UAAU1S,QAAQ+M,GACvCqI,EAAO1C,EAAa,QAAU,SAC9BuD,EAASvD,EAAa,OAAS,MAC/BzD,EAAcyD,EAAa,QAAU,SASzC,OAPIzG,EAAOmJ,GAAQ7C,EAAMjM,EAAU2P,MACjCjd,EAAKiR,QAAQgC,OAAOgK,GAAU1D,EAAMjM,EAAU2P,IAAWhK,EAAOgD,IAE9DhD,EAAOgK,GAAU1D,EAAMjM,EAAU8O,MACnCpc,EAAKiR,QAAQgC,OAAOgK,GAAU1D,EAAMjM,EAAU8O,KAGzCpc,IA4fPkd,MAAO,CAEL5O,MAAO,IAEP0I,SAAS,EAET5Z,GApxBJ,SAAe4C,EAAM4X,GACnB,IAAIuF,EAGJ,IAAKpD,GAAmB/Z,EAAKuc,SAAS/F,UAAW,QAAS,gBACxD,OAAOxW,EAGT,IAAIod,EAAexF,EAAQ5d,QAG3B,GAA4B,iBAAjBojB,GAIT,KAHAA,EAAepd,EAAKuc,SAAStJ,OAAO5Y,cAAc+iB,IAIhD,OAAOpd,OAKT,IAAKA,EAAKuc,SAAStJ,OAAO9R,SAASic,GAEjC,OADAtG,QAAQC,KAAK,iEACN/W,EAIX,IAAI+T,EAAY/T,EAAK+T,UAAUjZ,MAAM,KAAK,GACtCue,EAAgBrZ,EAAKiR,QACrBgC,EAASoG,EAAcpG,OACvB3F,EAAY+L,EAAc/L,UAE1BoM,GAAuD,IAA1C,CAAC,OAAQ,SAAS1S,QAAQ+M,GAEvC9R,EAAMyX,EAAa,SAAW,QAC9B2D,EAAkB3D,EAAa,MAAQ,OACvC0C,EAAOiB,EAAgBhhB,cACvBihB,EAAU5D,EAAa,OAAS,MAChCuD,EAASvD,EAAa,SAAW,QACjC6D,EAAmBtI,GAAcmI,GAAcnb,GAQ/CqL,EAAU2P,GAAUM,EAAmBtK,EAAOmJ,KAChDpc,EAAKiR,QAAQgC,OAAOmJ,IAASnJ,EAAOmJ,IAAS9O,EAAU2P,GAAUM,IAG/DjQ,EAAU8O,GAAQmB,EAAmBtK,EAAOgK,KAC9Cjd,EAAKiR,QAAQgC,OAAOmJ,IAAS9O,EAAU8O,GAAQmB,EAAmBtK,EAAOgK,IAE3Ejd,EAAKiR,QAAQgC,OAASjC,GAAchR,EAAKiR,QAAQgC,QAGjD,IAAIuK,EAASlQ,EAAU8O,GAAQ9O,EAAUrL,GAAO,EAAIsb,EAAmB,EAInE9iB,EAAM+R,GAAyBxM,EAAKuc,SAAStJ,QAC7CwK,EAAmB7iB,WAAWH,EAAI,SAAW4iB,GAAkB,IAC/DK,EAAmB9iB,WAAWH,EAAI,SAAW4iB,EAAkB,SAAU,IACzEM,EAAYH,EAASxd,EAAKiR,QAAQgC,OAAOmJ,GAAQqB,EAAmBC,EAQxE,OALAC,EAAYhkB,KAAKmW,IAAInW,KAAKojB,IAAI9J,EAAOhR,GAAOsb,EAAkBI,GAAY,GAE1E3d,EAAKod,aAAeA,EACpBpd,EAAKiR,QAAQiM,OAAmCvM,GAA1BwM,EAAsB,GAAwCf,EAAMziB,KAAK2f,MAAMqE,IAAahN,GAAewM,EAAqBG,EAAS,IAAKH,GAE7Jnd,GA8sBLhG,QAAS,aAcX4jB,KAAM,CAEJtP,MAAO,IAEP0I,SAAS,EAET5Z,GA5oBJ,SAAc4C,EAAM4X,GAElB,GAAIX,GAAkBjX,EAAKuc,SAAS/F,UAAW,SAC7C,OAAOxW,EAGT,GAAIA,EAAK6d,SAAW7d,EAAK+T,YAAc/T,EAAK8d,kBAE1C,OAAO9d,EAGT,IAAIoT,EAAaJ,GAAchT,EAAKuc,SAAStJ,OAAQjT,EAAKuc,SAASjP,UAAWsK,EAAQ1E,QAAS0E,EAAQzE,kBAAmBnT,EAAK8b,eAE3H/H,EAAY/T,EAAK+T,UAAUjZ,MAAM,KAAK,GACtCijB,EAAoBzI,GAAqBvB,GACzCe,EAAY9U,EAAK+T,UAAUjZ,MAAM,KAAK,IAAM,GAE5CkjB,EAAY,GAEhB,OAAQpG,EAAQqG,UACd,KAAKrD,GACHoD,EAAY,CAACjK,EAAWgK,GACxB,MACF,KAAKnD,GACHoD,EAAYxD,GAAUzG,GACtB,MACF,KAAK6G,GACHoD,EAAYxD,GAAUzG,GAAW,GACjC,MACF,QACEiK,EAAYpG,EAAQqG,SAyDxB,OAtDAD,EAAUnH,QAAQ,SAAUqH,EAAMzY,GAChC,GAAIsO,IAAcmK,GAAQF,EAAU9b,SAAWuD,EAAQ,EACrD,OAAOzF,EAGT+T,EAAY/T,EAAK+T,UAAUjZ,MAAM,KAAK,GACtCijB,EAAoBzI,GAAqBvB,GAEzC,IAAI8B,EAAgB7V,EAAKiR,QAAQgC,OAC7BkL,EAAane,EAAKiR,QAAQ3D,UAG1BiM,EAAQ5f,KAAK4f,MACb6E,EAA4B,SAAdrK,GAAwBwF,EAAM1D,EAAc3E,OAASqI,EAAM4E,EAAWhN,OAAuB,UAAd4C,GAAyBwF,EAAM1D,EAAc1E,MAAQoI,EAAM4E,EAAWjN,QAAwB,QAAd6C,GAAuBwF,EAAM1D,EAAczE,QAAUmI,EAAM4E,EAAW9M,MAAsB,WAAd0C,GAA0BwF,EAAM1D,EAAcxE,KAAOkI,EAAM4E,EAAW/M,QAEjUiN,EAAgB9E,EAAM1D,EAAc1E,MAAQoI,EAAMnG,EAAWjC,MAC7DmN,EAAiB/E,EAAM1D,EAAc3E,OAASqI,EAAMnG,EAAWlC,OAC/DqN,EAAehF,EAAM1D,EAAcxE,KAAOkI,EAAMnG,EAAW/B,KAC3DmN,EAAkBjF,EAAM1D,EAAczE,QAAUmI,EAAMnG,EAAWhC,QAEjEqN,EAAoC,SAAd1K,GAAwBsK,GAA+B,UAAdtK,GAAyBuK,GAAgC,QAAdvK,GAAuBwK,GAA8B,WAAdxK,GAA0ByK,EAG3K9E,GAAuD,IAA1C,CAAC,MAAO,UAAU1S,QAAQ+M,GAGvC2K,IAA0B9G,EAAQ+G,iBAAmBjF,GAA4B,UAAd5E,GAAyBuJ,GAAiB3E,GAA4B,QAAd5E,GAAuBwJ,IAAmB5E,GAA4B,UAAd5E,GAAyByJ,IAAiB7E,GAA4B,QAAd5E,GAAuB0J,GAGlQI,IAA8BhH,EAAQiH,0BAA4BnF,GAA4B,UAAd5E,GAAyBwJ,GAAkB5E,GAA4B,QAAd5E,GAAuBuJ,IAAkB3E,GAA4B,UAAd5E,GAAyB0J,IAAoB9E,GAA4B,QAAd5E,GAAuByJ,GAElRO,EAAmBJ,GAAyBE,GAE5CR,GAAeK,GAAuBK,KAExC9e,EAAK6d,SAAU,GAEXO,GAAeK,KACjB1K,EAAYiK,EAAUvY,EAAQ,IAG5BqZ,IACFhK,EAvJR,SAA8BA,GAC5B,MAAkB,QAAdA,EACK,QACgB,UAAdA,EACF,MAEFA,EAiJWiK,CAAqBjK,IAGnC9U,EAAK+T,UAAYA,GAAae,EAAY,IAAMA,EAAY,IAI5D9U,EAAKiR,QAAQgC,OAASpC,GAAS,GAAI7Q,EAAKiR,QAAQgC,OAAQyC,GAAiB1V,EAAKuc,SAAStJ,OAAQjT,EAAKiR,QAAQ3D,UAAWtN,EAAK+T,YAE5H/T,EAAOuW,GAAavW,EAAKuc,SAAS/F,UAAWxW,EAAM,WAGhDA,GA4jBLie,SAAU,OAKV/K,QAAS,EAOTC,kBAAmB,WAQnBwL,gBAAgB,EAQhBE,yBAAyB,GAU3BG,MAAO,CAEL1Q,MAAO,IAEP0I,SAAS,EAET5Z,GArQJ,SAAe4C,GACb,IAAI+T,EAAY/T,EAAK+T,UACjBgH,EAAgBhH,EAAUjZ,MAAM,KAAK,GACrCue,EAAgBrZ,EAAKiR,QACrBgC,EAASoG,EAAcpG,OACvB3F,EAAY+L,EAAc/L,UAE1BwI,GAAwD,IAA9C,CAAC,OAAQ,SAAS9O,QAAQ+T,GAEpCkE,GAA6D,IAA5C,CAAC,MAAO,QAAQjY,QAAQ+T,GAO7C,OALA9H,EAAO6C,EAAU,OAAS,OAASxI,EAAUyN,IAAkBkE,EAAiBhM,EAAO6C,EAAU,QAAU,UAAY,GAEvH9V,EAAK+T,UAAYuB,GAAqBvB,GACtC/T,EAAKiR,QAAQgC,OAASjC,GAAciC,GAE7BjT,IAkQPsK,KAAM,CAEJgE,MAAO,IAEP0I,SAAS,EAET5Z,GA9TJ,SAAc4C,GACZ,IAAK+Z,GAAmB/Z,EAAKuc,SAAS/F,UAAW,OAAQ,mBACvD,OAAOxW,EAGT,IAAIgU,EAAUhU,EAAKiR,QAAQ3D,UACvB4R,EAAQ/I,GAAKnW,EAAKuc,SAAS/F,UAAW,SAAU7D,GAClD,MAAyB,oBAAlBA,EAASyE,OACfhE,WAEH,GAAIY,EAAQ5C,OAAS8N,EAAM7N,KAAO2C,EAAQ7C,KAAO+N,EAAMhO,OAAS8C,EAAQ3C,IAAM6N,EAAM9N,QAAU4C,EAAQ9C,MAAQgO,EAAM/N,KAAM,CAExH,IAAkB,IAAdnR,EAAKsK,KACP,OAAOtK,EAGTA,EAAKsK,MAAO,EACZtK,EAAKmf,WAAW,uBAAyB,OACpC,CAEL,IAAkB,IAAdnf,EAAKsK,KACP,OAAOtK,EAGTA,EAAKsK,MAAO,EACZtK,EAAKmf,WAAW,wBAAyB,EAG3C,OAAOnf,IAoTPof,aAAc,CAEZ9Q,MAAO,IAEP0I,SAAS,EAET5Z,GAtgCJ,SAAsB4C,EAAM4X,GAC1B,IAAI1C,EAAI0C,EAAQ1C,EACZE,EAAIwC,EAAQxC,EACZnC,EAASjT,EAAKiR,QAAQgC,OAItBoM,EAA8BlJ,GAAKnW,EAAKuc,SAAS/F,UAAW,SAAU7D,GACxE,MAAyB,eAAlBA,EAASyE,OACfkI,qBACiClQ,IAAhCiQ,GACFvI,QAAQC,KAAK,iIAEf,IAAIuI,OAAkDlQ,IAAhCiQ,EAA4CA,EAA8BzH,EAAQ0H,gBAEpGvR,EAAeF,GAAgB7N,EAAKuc,SAAStJ,QAC7CsM,EAAmBrU,GAAsB6C,GAGzCyB,EAAS,CACXgQ,SAAUvM,EAAOuM,UAGfvO,EAAUgI,GAAkBjZ,EAAM4B,OAAO6d,iBAAmB,IAAM3F,IAElEpK,EAAc,WAANwF,EAAiB,MAAQ,SACjCvF,EAAc,UAANyF,EAAgB,OAAS,QAKjCsK,EAAmBrI,GAAyB,aAW5ClG,OAAO,EACPE,OAAM,EAqBV,GAhBIA,EAJU,UAAV3B,EAG4B,SAA1B3B,EAAalB,UACRkB,EAAa6D,aAAeX,EAAQG,QAEpCmO,EAAiBvP,OAASiB,EAAQG,OAGrCH,EAAQI,IAIZF,EAFU,SAAVxB,EAC4B,SAA1B5B,EAAalB,UACPkB,EAAa4D,YAAcV,EAAQC,OAEnCqO,EAAiBtP,MAAQgB,EAAQC,MAGpCD,EAAQE,KAEbmO,GAAmBI,EACrBlQ,EAAOkQ,GAAoB,eAAiBvO,EAAO,OAASE,EAAM,SAClE7B,EAAOE,GAAS,EAChBF,EAAOG,GAAS,EAChBH,EAAOmQ,WAAa,gBACf,CAEL,IAAIC,EAAsB,UAAVlQ,GAAsB,EAAI,EACtCmQ,EAAuB,SAAVlQ,GAAqB,EAAI,EAC1CH,EAAOE,GAAS2B,EAAMuO,EACtBpQ,EAAOG,GAASwB,EAAO0O,EACvBrQ,EAAOmQ,WAAajQ,EAAQ,KAAOC,EAIrC,IAAIwP,EAAa,CACfW,cAAe9f,EAAK+T,WAQtB,OAJA/T,EAAKmf,WAAatO,GAAS,GAAIsO,EAAYnf,EAAKmf,YAChDnf,EAAKwP,OAASqB,GAAS,GAAIrB,EAAQxP,EAAKwP,QACxCxP,EAAK+f,YAAclP,GAAS,GAAI7Q,EAAKiR,QAAQiM,MAAOld,EAAK+f,aAElD/f,GAo7BLsf,iBAAiB,EAMjBpK,EAAG,SAMHE,EAAG,SAkBL4K,WAAY,CAEV1R,MAAO,IAEP0I,SAAS,EAET5Z,GAzpCJ,SAAoB4C,GAgBlB,OAXA+Y,GAAU/Y,EAAKuc,SAAStJ,OAAQjT,EAAKwP,QAzBvC,SAAuBxV,EAASmlB,GAC9BvjB,OAAOuY,KAAKgL,GAAYtI,QAAQ,SAAUH,IAE1B,IADFyI,EAAWzI,GAErB1c,EAAQwH,aAAakV,EAAMyI,EAAWzI,IAEtC1c,EAAQimB,gBAAgBvJ,KAuB5BwJ,CAAclgB,EAAKuc,SAAStJ,OAAQjT,EAAKmf,YAGrCnf,EAAKod,cAAgBxhB,OAAOuY,KAAKnU,EAAK+f,aAAa7d,QACrD6W,GAAU/Y,EAAKod,aAAcpd,EAAK+f,aAG7B/f,GA2oCLmgB,OA9nCJ,SAA0B7S,EAAW2F,EAAQ2E,EAASwI,EAAiBpL,GAErE,IAAIW,EAAmBZ,GAAoBC,EAAO/B,EAAQ3F,EAAWsK,EAAQkE,eAKzE/H,EAAYD,GAAqB8D,EAAQ7D,UAAW4B,EAAkB1C,EAAQ3F,EAAWsK,EAAQpB,UAAUoH,KAAKzK,kBAAmByE,EAAQpB,UAAUoH,KAAK1K,SAQ9J,OANAD,EAAOzR,aAAa,cAAeuS,GAInCgF,GAAU9F,EAAQ,CAAEuM,SAAU5H,EAAQkE,cAAgB,QAAU,aAEzDlE,GAsnCL0H,qBAAiBlQ,KAuGjBiR,IA+EFnQ,GAAYmQ,GAAQ,CAAC,CACnBzP,IAAK,SACL3U,MAAO,WACL,OA9lDN,WAEE,IAAI/C,KAAK8b,MAAMsL,YAAf,CAIA,IAAItgB,EAAO,CACTuc,SAAUrjB,KACVsW,OAAQ,GACRuQ,YAAa,GACbZ,WAAY,GACZtB,SAAS,EACT5M,QAAS,IAIXjR,EAAKiR,QAAQ3D,UAAYyH,GAAoB7b,KAAK8b,MAAO9b,KAAK+Z,OAAQ/Z,KAAKoU,UAAWpU,KAAK0e,QAAQkE,eAKnG9b,EAAK+T,UAAYD,GAAqB5a,KAAK0e,QAAQ7D,UAAW/T,EAAKiR,QAAQ3D,UAAWpU,KAAK+Z,OAAQ/Z,KAAKoU,UAAWpU,KAAK0e,QAAQpB,UAAUoH,KAAKzK,kBAAmBja,KAAK0e,QAAQpB,UAAUoH,KAAK1K,SAG9LlT,EAAK8d,kBAAoB9d,EAAK+T,UAE9B/T,EAAK8b,cAAgB5iB,KAAK0e,QAAQkE,cAGlC9b,EAAKiR,QAAQgC,OAASyC,GAAiBxc,KAAK+Z,OAAQjT,EAAKiR,QAAQ3D,UAAWtN,EAAK+T,WAEjF/T,EAAKiR,QAAQgC,OAAOuM,SAAWtmB,KAAK0e,QAAQkE,cAAgB,QAAU,WAGtE9b,EAAOuW,GAAard,KAAKsd,UAAWxW,GAI/B9G,KAAK8b,MAAMuL,UAIdrnB,KAAK0e,QAAQqE,SAASjc,IAHtB9G,KAAK8b,MAAMuL,WAAY,EACvBrnB,KAAK0e,QAAQoE,SAAShc,MAsjDNjE,KAAK7C,QAEpB,CACD0X,IAAK,UACL3U,MAAO,WACL,OA7gDN,WAsBE,OArBA/C,KAAK8b,MAAMsL,aAAc,EAGrBrJ,GAAkB/d,KAAKsd,UAAW,gBACpCtd,KAAK+Z,OAAOgN,gBAAgB,eAC5B/mB,KAAK+Z,OAAOnI,MAAM0U,SAAW,GAC7BtmB,KAAK+Z,OAAOnI,MAAMuG,IAAM,GACxBnY,KAAK+Z,OAAOnI,MAAMqG,KAAO,GACzBjY,KAAK+Z,OAAOnI,MAAMoG,MAAQ,GAC1BhY,KAAK+Z,OAAOnI,MAAMsG,OAAS,GAC3BlY,KAAK+Z,OAAOnI,MAAM6U,WAAa,GAC/BzmB,KAAK+Z,OAAOnI,MAAMuM,GAAyB,cAAgB,IAG7Dne,KAAKof,wBAIDpf,KAAK0e,QAAQmE,iBACf7iB,KAAK+Z,OAAOlW,WAAWyjB,YAAYtnB,KAAK+Z,QAEnC/Z,MAu/CY6C,KAAK7C,QAErB,CACD0X,IAAK,uBACL3U,MAAO,WACL,OA18CN,WACO/C,KAAK8b,MAAMqD,gBACdnf,KAAK8b,MAAQ2C,GAAoBze,KAAKoU,UAAWpU,KAAK0e,QAAS1e,KAAK8b,MAAO9b,KAAKsf,kBAw8ClDzc,KAAK7C,QAElC,CACD0X,IAAK,wBACL3U,MAAO,WACL,OAAOqc,GAAsBvc,KAAK7C,UA4B/BmnB,IApHP,SAASA,GAAO/S,EAAW2F,GACzB,IAAIha,EAAQC,KAER0e,EAA6B,EAAnB3Z,UAAUiE,aAA+BkN,IAAjBnR,UAAU,GAAmBA,UAAU,GAAK,IA3jEjE,SAAUse,EAAUlc,GACvC,KAAMkc,aAAoBlc,GACxB,MAAM,IAAInD,UAAU,qCA0jEpBujB,CAAevnB,KAAMmnB,IAErBnnB,KAAKsf,eAAiB,WACpB,OAAOkI,sBAAsBznB,EAAM0nB,SAIrCznB,KAAKynB,OAAS1U,GAAS/S,KAAKynB,OAAOpb,KAAKrM,OAGxCA,KAAK0e,QAAU/G,GAAS,GAAIwP,GAAOxE,SAAUjE,GAG7C1e,KAAK8b,MAAQ,CACXsL,aAAa,EACbC,WAAW,EACXpI,cAAe,IAIjBjf,KAAKoU,UAAYA,GAAaA,EAAUjQ,OAASiQ,EAAU,GAAKA,EAChEpU,KAAK+Z,OAASA,GAAUA,EAAO5V,OAAS4V,EAAO,GAAKA,EAGpD/Z,KAAK0e,QAAQpB,UAAY,GACzB5a,OAAOuY,KAAKtD,GAAS,GAAIwP,GAAOxE,SAASrF,UAAWoB,EAAQpB,YAAYK,QAAQ,SAAUO,GACxFne,EAAM2e,QAAQpB,UAAUY,GAAQvG,GAAS,GAAIwP,GAAOxE,SAASrF,UAAUY,IAAS,GAAIQ,EAAQpB,UAAYoB,EAAQpB,UAAUY,GAAQ,MAIpIle,KAAKsd,UAAY5a,OAAOuY,KAAKjb,KAAK0e,QAAQpB,WAAWpC,IAAI,SAAUgD,GACjE,OAAOvG,GAAS,CACduG,KAAMA,GACLne,EAAM2e,QAAQpB,UAAUY,MAG5B5C,KAAK,SAAUC,EAAGC,GACjB,OAAOD,EAAEnG,MAAQoG,EAAEpG,QAOrBpV,KAAKsd,UAAUK,QAAQ,SAAUuJ,GAC3BA,EAAgBpJ,SAAW1K,GAAW8T,EAAgBD,SACxDC,EAAgBD,OAAOlnB,EAAMqU,UAAWrU,EAAMga,OAAQha,EAAM2e,QAASwI,EAAiBnnB,EAAM+b,SAKhG9b,KAAKynB,SAEL,IAAItI,EAAgBnf,KAAK0e,QAAQS,cAC7BA,GAEFnf,KAAK0nB,uBAGP1nB,KAAK8b,MAAMqD,cAAgBA,EA6E/BgI,GAAOQ,OAA2B,oBAAXjf,OAAyBA,OAASkf,QAAQC,YACjEV,GAAO/F,WAAaA,GACpB+F,GAAOxE,SAAWA,GCniFlB,IAAM3d,GAA2B,WAE3BC,GAA2B,cAC3BC,GAAS,IAAsBD,GAC/BoC,GAA2B,YAC3BlC,GAA2BjF,EAAEgE,GAAGc,IAOhC8iB,GAA2B,IAAI1kB,OAAU2kB,YAEzC3iB,GAAQ,CACZ+K,KAAI,OAAsBjL,GAC1BkL,OAAM,SAAsBlL,GAC5B+K,KAAI,OAAsB/K,GAC1BgL,MAAK,QAAsBhL,GAC3B8iB,MAAK,QAAsB9iB,GAC3BK,eAAc,QAAaL,GAAYmC,GACvC4gB,iBAAgB,UAAa/iB,GAAYmC,GACzC6gB,eAAc,QAAahjB,GAAYmC,IAGnC7B,GACc,WADdA,GAEc,OAFdA,GAGc,SAHdA,GAIc,YAJdA,GAKc,WALdA,GAMc,sBANdA,GAQc,kBAGd8B,GACY,2BADZA,GAEY,iBAFZA,GAGY,iBAHZA,GAIY,cAJZA,GAKY,8DAGZ6gB,GACQ,YADRA,GAEQ,UAFRA,GAGQ,eAHRA,GAIQ,aAJRA,GAKQ,cALRA,GAOQ,aAIRjf,GAAU,CACd0Y,OAAe,EACf8C,MAAe,EACf0D,SAAe,eACfhU,UAAe,SACfiU,QAAe,UACfC,aAAe,MAGX7e,GAAc,CAClBmY,OAAe,2BACf8C,KAAe,UACf0D,SAAe,mBACfhU,UAAe,mBACfiU,QAAe,SACfC,aAAe,iBASXC,cACJ,SAAAA,EAAYznB,EAASyB,GACnBvC,KAAK0F,SAAY5E,EACjBd,KAAKwoB,QAAY,KACjBxoB,KAAKkL,QAAYlL,KAAKmL,WAAW5I,GACjCvC,KAAKyoB,MAAYzoB,KAAK0oB,kBACtB1oB,KAAK2oB,UAAY3oB,KAAK4oB,gBAEtB5oB,KAAK2L,gDAmBPjE,OAAA,WACE,IAAI1H,KAAK0F,SAASmjB,WAAY3oB,EAAEF,KAAK0F,UAAUa,SAASf,IAAxD,CAIA,IAAMsjB,EAAW5oB,EAAEF,KAAKyoB,OAAOliB,SAASf,IAExC+iB,EAASQ,cAELD,GAIJ9oB,KAAKqR,MAAK,OAGZA,KAAA,SAAK2X,GACH,QADsB,IAAnBA,IAAAA,GAAY,KACXhpB,KAAK0F,SAASmjB,UAAY3oB,EAAEF,KAAK0F,UAAUa,SAASf,KAAuBtF,EAAEF,KAAKyoB,OAAOliB,SAASf,KAAtG,CAIA,IAAM6I,EAAgB,CACpBA,cAAerO,KAAK0F,UAEhBujB,EAAY/oB,EAAEkF,MAAMA,GAAM6K,KAAM5B,GAChClI,EAASoiB,EAASW,sBAAsBlpB,KAAK0F,UAInD,GAFAxF,EAAEiG,GAAQpE,QAAQknB,IAEdA,EAAUljB,qBAAd,CAKA,IAAK/F,KAAK2oB,WAAaK,EAAW,CAKhC,GAAsB,oBAAX7B,GACT,MAAM,IAAInjB,UAAU,oEAGtB,IAAImlB,EAAmBnpB,KAAK0F,SAEG,WAA3B1F,KAAKkL,QAAQkJ,UACf+U,EAAmBhjB,EACV/F,EAAK8B,UAAUlC,KAAKkL,QAAQkJ,aACrC+U,EAAmBnpB,KAAKkL,QAAQkJ,UAGa,oBAAlCpU,KAAKkL,QAAQkJ,UAAUjQ,SAChCglB,EAAmBnpB,KAAKkL,QAAQkJ,UAAU,KAOhB,iBAA1BpU,KAAKkL,QAAQkd,UACfloB,EAAEiG,GAAQ4I,SAASvJ,IAErBxF,KAAKwoB,QAAU,IAAIrB,GAAOgC,EAAkBnpB,KAAKyoB,MAAOzoB,KAAKopB,oBAO3D,iBAAkBzoB,SAAS8C,iBACuB,IAAlDvD,EAAEiG,GAAQC,QAAQkB,IAAqB0B,QACzC9I,EAAES,SAASmT,MAAMhF,WAAW5H,GAAG,YAAa,KAAMhH,EAAEmpB,MAGtDrpB,KAAK0F,SAAS0C,QACdpI,KAAK0F,SAAS4C,aAAa,iBAAiB,GAE5CpI,EAAEF,KAAKyoB,OAAOlgB,YAAY/C,IAC1BtF,EAAEiG,GACCoC,YAAY/C,IACZzD,QAAQ7B,EAAEkF,MAAMA,GAAM8K,MAAO7B,SAGlC+C,KAAA,WACE,IAAIpR,KAAK0F,SAASmjB,WAAY3oB,EAAEF,KAAK0F,UAAUa,SAASf,KAAwBtF,EAAEF,KAAKyoB,OAAOliB,SAASf,IAAvG,CAIA,IAAM6I,EAAgB,CACpBA,cAAerO,KAAK0F,UAEhB4jB,EAAYppB,EAAEkF,MAAMA,GAAM+K,KAAM9B,GAChClI,EAASoiB,EAASW,sBAAsBlpB,KAAK0F,UAEnDxF,EAAEiG,GAAQpE,QAAQunB,GAEdA,EAAUvjB,uBAIV/F,KAAKwoB,SACPxoB,KAAKwoB,QAAQe,UAGfrpB,EAAEF,KAAKyoB,OAAOlgB,YAAY/C,IAC1BtF,EAAEiG,GACCoC,YAAY/C,IACZzD,QAAQ7B,EAAEkF,MAAMA,GAAMgL,OAAQ/B,SAGnCpI,QAAA,WACE/F,EAAEgG,WAAWlG,KAAK0F,SAAUT,IAC5B/E,EAAEF,KAAK0F,UAAUiH,IAAIzH,IACrBlF,KAAK0F,SAAW,MAChB1F,KAAKyoB,MAAQ,QACTzoB,KAAKwoB,UACPxoB,KAAKwoB,QAAQe,UACbvpB,KAAKwoB,QAAU,SAInBf,OAAA,WACEznB,KAAK2oB,UAAY3oB,KAAK4oB,gBACD,OAAjB5oB,KAAKwoB,SACPxoB,KAAKwoB,QAAQlJ,oBAMjB3T,mBAAA,WAAqB,IAAA5L,EAAAC,KACnBE,EAAEF,KAAK0F,UAAUwB,GAAG9B,GAAM4iB,MAAO,SAAC3jB,GAChCA,EAAM4C,iBACN5C,EAAMmlB,kBACNzpB,EAAK2H,cAITyD,WAAA,SAAW5I,GAaT,OAZAA,EAAMqK,EAAA,GACD5M,KAAKypB,YAAYvgB,QADhB,GAEDhJ,EAAEF,KAAK0F,UAAUoB,OAFhB,GAGDvE,GAGLnC,EAAKiC,gBACH2C,GACAzC,EACAvC,KAAKypB,YAAYhgB,aAGZlH,KAGTmmB,gBAAA,WACE,IAAK1oB,KAAKyoB,MAAO,CACf,IAAMtiB,EAASoiB,EAASW,sBAAsBlpB,KAAK0F,UAE/CS,IACFnG,KAAKyoB,MAAQtiB,EAAOhF,cAAcmG,KAGtC,OAAOtH,KAAKyoB,SAGdiB,cAAA,WACE,IAAMC,EAAkBzpB,EAAEF,KAAK0F,SAAS7B,YACpCgX,EAAYsN,GAehB,OAZIwB,EAAgBpjB,SAASf,KAC3BqV,EAAYsN,GACRjoB,EAAEF,KAAKyoB,OAAOliB,SAASf,MACzBqV,EAAYsN,KAELwB,EAAgBpjB,SAASf,IAClCqV,EAAYsN,GACHwB,EAAgBpjB,SAASf,IAClCqV,EAAYsN,GACHjoB,EAAEF,KAAKyoB,OAAOliB,SAASf,MAChCqV,EAAYsN,IAEPtN,KAGT+N,cAAA,WACE,OAAoD,EAA7C1oB,EAAEF,KAAK0F,UAAUU,QAAQ,WAAW4C,UAG7C4gB,WAAA,WAAa,IAAA5c,EAAAhN,KACL4hB,EAAS,GAef,MAbmC,mBAAxB5hB,KAAKkL,QAAQ0W,OACtBA,EAAO1d,GAAK,SAAC4C,GAMX,OALAA,EAAKiR,QAALnL,EAAA,GACK9F,EAAKiR,QADV,GAEK/K,EAAK9B,QAAQ0W,OAAO9a,EAAKiR,QAAS/K,EAAKtH,WAAa,IAGlDoB,GAGT8a,EAAOA,OAAS5hB,KAAKkL,QAAQ0W,OAGxBA,KAGTwH,iBAAA,WACE,IAAMd,EAAe,CACnBzN,UAAW7a,KAAK0pB,gBAChBpM,UAAW,CACTsE,OAAQ5hB,KAAK4pB,aACblF,KAAM,CACJ5G,QAAS9d,KAAKkL,QAAQwZ,MAExBtB,gBAAiB,CACfnJ,kBAAmBja,KAAKkL,QAAQkd,YAYtC,MAN6B,WAAzBpoB,KAAKkL,QAAQmd,UACfC,EAAahL,UAAUwJ,WAAa,CAClChJ,SAAS,IAIblR,EAAA,GACK0b,EADL,GAEKtoB,KAAKkL,QAAQod,iBAMb3hB,iBAAP,SAAwBpE,GACtB,OAAOvC,KAAK4G,KAAK,WACf,IAAIE,EAAO5G,EAAEF,MAAM8G,KAAK7B,IAQxB,GALK6B,IACHA,EAAO,IAAIyhB,EAASvoB,KAHY,iBAAXuC,EAAsBA,EAAS,MAIpDrC,EAAEF,MAAM8G,KAAK7B,GAAU6B,IAGH,iBAAXvE,EAAqB,CAC9B,GAA4B,oBAAjBuE,EAAKvE,GACd,MAAM,IAAIyB,UAAJ,oBAAkCzB,EAAlC,KAERuE,EAAKvE,WAKJwmB,YAAP,SAAmB1kB,GACjB,IAAIA,GAhWyB,IAgWfA,EAAMwJ,QACH,UAAfxJ,EAAMyD,MApWqB,IAoWDzD,EAAMwJ,OAMlC,IAFA,IAAMgc,EAAU,GAAGjhB,MAAM/F,KAAKlC,SAASkI,iBAAiBvB,KAE/CwB,EAAI,EAAGC,EAAM8gB,EAAQ7gB,OAAQF,EAAIC,EAAKD,IAAK,CAClD,IAAM3C,EAASoiB,EAASW,sBAAsBW,EAAQ/gB,IAChDghB,EAAU5pB,EAAE2pB,EAAQ/gB,IAAIhC,KAAK7B,IAC7BoJ,EAAgB,CACpBA,cAAewb,EAAQ/gB,IAOzB,GAJIzE,GAAwB,UAAfA,EAAMyD,OACjBuG,EAAc0b,WAAa1lB,GAGxBylB,EAAL,CAIA,IAAME,EAAeF,EAAQrB,MAC7B,GAAKvoB,EAAEiG,GAAQI,SAASf,OAIpBnB,IAAyB,UAAfA,EAAMyD,MAChB,kBAAkBzE,KAAKgB,EAAMK,OAAOyD,UAA2B,UAAf9D,EAAMyD,MA/X/B,IA+XmDzD,EAAMwJ,QAChF3N,EAAE+H,SAAS9B,EAAQ9B,EAAMK,SAF7B,CAMA,IAAM4kB,EAAYppB,EAAEkF,MAAMA,GAAM+K,KAAM9B,GACtCnO,EAAEiG,GAAQpE,QAAQunB,GACdA,EAAUvjB,uBAMV,iBAAkBpF,SAAS8C,iBAC7BvD,EAAES,SAASmT,MAAMhF,WAAWnC,IAAI,YAAa,KAAMzM,EAAEmpB,MAGvDQ,EAAQ/gB,GAAGR,aAAa,gBAAiB,SAErCwhB,EAAQtB,SACVsB,EAAQtB,QAAQe,UAGlBrpB,EAAE8pB,GAAc1jB,YAAYd,IAC5BtF,EAAEiG,GACCG,YAAYd,IACZzD,QAAQ7B,EAAEkF,MAAMA,GAAMgL,OAAQ/B,WAI9B6a,sBAAP,SAA6BpoB,GAC3B,IAAIqF,EACEpF,EAAWX,EAAKS,uBAAuBC,GAM7C,OAJIC,IACFoF,EAASxF,SAASQ,cAAcJ,IAG3BoF,GAAUrF,EAAQ+C,cAIpBomB,uBAAP,SAA8B5lB,GAQ5B,IAAI,kBAAkBhB,KAAKgB,EAAMK,OAAOyD,WAlbX,KAmbzB9D,EAAMwJ,OApbmB,KAobQxJ,EAAMwJ,QAhbd,KAib1BxJ,EAAMwJ,OAlboB,KAkbYxJ,EAAMwJ,OAC3C3N,EAAEmE,EAAMK,QAAQ0B,QAAQkB,IAAe0B,SAAW8e,GAAezkB,KAAKgB,EAAMwJ,UAIhFxJ,EAAM4C,iBACN5C,EAAMmlB,mBAEFxpB,KAAK6oB,WAAY3oB,EAAEF,MAAMuG,SAASf,KAAtC,CAIA,IAAMW,EAAWoiB,EAASW,sBAAsBlpB,MAC1C8oB,EAAW5oB,EAAEiG,GAAQI,SAASf,IAEpC,GAAKsjB,GApcwB,KAocZzkB,EAAMwJ,MAIvB,GAAKib,KAAYA,GAxcY,KAwcCzkB,EAAMwJ,OAvcP,KAucmCxJ,EAAMwJ,OAAtE,CAUA,IAAMqc,EAAQ,GAAGthB,MAAM/F,KAAKsD,EAAO0C,iBAAiBvB,KACjDuJ,OAAO,SAACsZ,GAAD,OAAUjqB,EAAEiqB,GAAMxlB,GAAG,cAE/B,GAAqB,IAAjBulB,EAAMlhB,OAAV,CAIA,IAAIuD,EAAQ2d,EAAMpc,QAAQzJ,EAAMK,QAtdH,KAwdzBL,EAAMwJ,OAAsC,EAARtB,GACtCA,IAxd2B,KA2dzBlI,EAAMwJ,OAAgCtB,EAAQ2d,EAAMlhB,OAAS,GAC/DuD,IAGEA,EAAQ,IACVA,EAAQ,GAGV2d,EAAM3d,GAAOnE,aA/Bb,CACE,GAzc2B,KAycvB/D,EAAMwJ,MAA0B,CAClC,IAAMnG,EAASvB,EAAOhF,cAAcmG,IACpCpH,EAAEwH,GAAQ3F,QAAQ,SAGpB7B,EAAEF,MAAM+B,QAAQ,oDAvXlB,MA5F6B,wCAgG7B,OAAOmH,uCAIP,OAAOO,YAkZXvJ,EAAES,UACCuG,GAAG9B,GAAM6iB,iBAAkB3gB,GAAsBihB,GAAS0B,wBAC1D/iB,GAAG9B,GAAM6iB,iBAAkB3gB,GAAeihB,GAAS0B,wBACnD/iB,GAAM9B,GAAMG,eAHf,IAGiCH,GAAM8iB,eAAkBK,GAASQ,aAC/D7hB,GAAG9B,GAAMG,eAAgB+B,GAAsB,SAAUjD,GACxDA,EAAM4C,iBACN5C,EAAMmlB,kBACNjB,GAAS5hB,iBAAiB9D,KAAK3C,EAAEF,MAAO,YAEzCkH,GAAG9B,GAAMG,eAAgB+B,GAAqB,SAACqG,GAC9CA,EAAE6b,oBASNtpB,EAAEgE,GAAGc,IAAQujB,GAAS5hB,iBACtBzG,EAAEgE,GAAGc,IAAMmC,YAAcohB,GACzBroB,EAAEgE,GAAGc,IAAMoC,WAAa,WAEtB,OADAlH,EAAEgE,GAAGc,IAAQG,GACNojB,GAAS5hB,kBC/gBlB,IAAM3B,GAAqB,QAErBC,GAAqB,WACrBC,GAAS,IAAgBD,GAEzBE,GAAqBjF,EAAEgE,GAAGc,IAG1BkE,GAAU,CACdkhB,UAAW,EACXhhB,UAAW,EACXhB,OAAW,EACXiJ,MAAW,GAGP5H,GAAc,CAClB2gB,SAAW,mBACXhhB,SAAW,UACXhB,MAAW,UACXiJ,KAAW,WAGPjM,GAAQ,CACZ+K,KAAI,OAAuBjL,GAC3BmlB,eAAc,gBAAsBnlB,GACpCkL,OAAM,SAAuBlL,GAC7B+K,KAAI,OAAuB/K,GAC3BgL,MAAK,QAAuBhL,GAC5BolB,QAAO,UAAuBplB,GAC9BqlB,OAAM,SAAuBrlB,GAC7BslB,cAAa,gBAAuBtlB,GACpCulB,gBAAe,kBAAuBvlB,GACtCwlB,gBAAe,kBAAuBxlB,GACtCylB,kBAAiB,oBAAuBzlB,GACxCK,eAAc,QAAcL,GA9BH,aAiCrBM,GACiB,0BADjBA,GAEiB,0BAFjBA,GAGiB,iBAHjBA,GAIiB,aAJjBA,GAKiB,OALjBA,GAMiB,OANjBA,GAOiB,eAGjB8B,GACa,gBADbA,GAEa,cAFbA,GAGa,wBAHbA,GAIa,yBAJbA,GAKa,oDALbA,GAMa,cASbsjB,cACJ,SAAAA,EAAY9pB,EAASyB,GACnBvC,KAAKkL,QAAuBlL,KAAKmL,WAAW5I,GAC5CvC,KAAK0F,SAAuB5E,EAC5Bd,KAAK6qB,QAAuB/pB,EAAQK,cAAcmG,IAClDtH,KAAK8qB,UAAuB,KAC5B9qB,KAAK+qB,UAAuB,EAC5B/qB,KAAKgrB,oBAAuB,EAC5BhrB,KAAKirB,sBAAuB,EAC5BjrB,KAAKuQ,kBAAuB,EAC5BvQ,KAAKkrB,gBAAuB,6BAe9BxjB,OAAA,SAAO2G,GACL,OAAOrO,KAAK+qB,SAAW/qB,KAAKoR,OAASpR,KAAKqR,KAAKhD,MAGjDgD,KAAA,SAAKhD,GAAe,IAAAtO,EAAAC,KAClB,IAAIA,KAAK+qB,WAAY/qB,KAAKuQ,iBAA1B,CAIIrQ,EAAEF,KAAK0F,UAAUa,SAASf,MAC5BxF,KAAKuQ,kBAAmB,GAG1B,IAAM0Y,EAAY/oB,EAAEkF,MAAMA,GAAM6K,KAAM,CACpC5B,cAAAA,IAGFnO,EAAEF,KAAK0F,UAAU3D,QAAQknB,GAErBjpB,KAAK+qB,UAAY9B,EAAUljB,uBAI/B/F,KAAK+qB,UAAW,EAEhB/qB,KAAKmrB,kBACLnrB,KAAKorB,gBAELprB,KAAKqrB,gBAELrrB,KAAKsrB,kBACLtrB,KAAKurB,kBAELrrB,EAAEF,KAAK0F,UAAUwB,GACf9B,GAAMolB,cACNljB,GACA,SAACjD,GAAD,OAAWtE,EAAKqR,KAAK/M,KAGvBnE,EAAEF,KAAK6qB,SAAS3jB,GAAG9B,GAAMulB,kBAAmB,WAC1CzqB,EAAEH,EAAK2F,UAAUvF,IAAIiF,GAAMslB,gBAAiB,SAACrmB,GACvCnE,EAAEmE,EAAMK,QAAQC,GAAG5E,EAAK2F,YAC1B3F,EAAKkrB,sBAAuB,OAKlCjrB,KAAKwrB,cAAc,WAAA,OAAMzrB,EAAK0rB,aAAapd,UAG7C+C,KAAA,SAAK/M,GAAO,IAAA2I,EAAAhN,KAKV,GAJIqE,GACFA,EAAM4C,iBAGHjH,KAAK+qB,WAAY/qB,KAAKuQ,iBAA3B,CAIA,IAAM+Y,EAAYppB,EAAEkF,MAAMA,GAAM+K,MAIhC,GAFAjQ,EAAEF,KAAK0F,UAAU3D,QAAQunB,GAEpBtpB,KAAK+qB,WAAYzB,EAAUvjB,qBAAhC,CAIA/F,KAAK+qB,UAAW,EAChB,IAAMW,EAAaxrB,EAAEF,KAAK0F,UAAUa,SAASf,IAiB7C,GAfIkmB,IACF1rB,KAAKuQ,kBAAmB,GAG1BvQ,KAAKsrB,kBACLtrB,KAAKurB,kBAELrrB,EAAES,UAAUgM,IAAIvH,GAAMklB,SAEtBpqB,EAAEF,KAAK0F,UAAUY,YAAYd,IAE7BtF,EAAEF,KAAK0F,UAAUiH,IAAIvH,GAAMolB,eAC3BtqB,EAAEF,KAAK6qB,SAASle,IAAIvH,GAAMulB,mBAGtBe,EAAY,CACd,IAAMpqB,EAAsBlB,EAAKiB,iCAAiCrB,KAAK0F,UAEvExF,EAAEF,KAAK0F,UACJvF,IAAIC,EAAKR,eAAgB,SAACyE,GAAD,OAAW2I,EAAK2e,WAAWtnB,KACpDD,qBAAqB9C,QAExBtB,KAAK2rB,kBAIT1lB,QAAA,WACE,CAACyC,OAAQ1I,KAAK0F,SAAU1F,KAAK6qB,SAC1BlN,QAAQ,SAACiO,GAAD,OAAiB1rB,EAAE0rB,GAAajf,IAAIzH,MAO/ChF,EAAES,UAAUgM,IAAIvH,GAAMklB,SAEtBpqB,EAAEgG,WAAWlG,KAAK0F,SAAUT,IAE5BjF,KAAKkL,QAAuB,KAC5BlL,KAAK0F,SAAuB,KAC5B1F,KAAK6qB,QAAuB,KAC5B7qB,KAAK8qB,UAAuB,KAC5B9qB,KAAK+qB,SAAuB,KAC5B/qB,KAAKgrB,mBAAuB,KAC5BhrB,KAAKirB,qBAAuB,KAC5BjrB,KAAKuQ,iBAAuB,KAC5BvQ,KAAKkrB,gBAAuB,QAG9BW,aAAA,WACE7rB,KAAKqrB,mBAKPlgB,WAAA,SAAW5I,GAMT,OALAA,EAAMqK,EAAA,GACD1D,GADC,GAED3G,GAELnC,EAAKiC,gBAAgB2C,GAAMzC,EAAQkH,IAC5BlH,KAGTupB,2BAAA,WAA6B,IAAA3e,EAAAnN,KAC3B,GAA8B,WAA1BA,KAAKkL,QAAQkf,SAAuB,CACtC,IAAM2B,EAAqB7rB,EAAEkF,MAAMA,GAAMilB,gBAGzC,GADAnqB,EAAEF,KAAK0F,UAAU3D,QAAQgqB,GACrBA,EAAmBC,iBACrB,OAGFhsB,KAAK0F,SAASsC,UAAUiB,IAAIzD,IAE5B,IAAMymB,EAA0B7rB,EAAKiB,iCAAiCrB,KAAK0F,UAE3ExF,EAAEF,KAAK0F,UAAUvF,IAAIC,EAAKR,eAAgB,WACxCuN,EAAKzH,SAASsC,UAAUtB,OAAOlB,MAE9BpB,qBAAqB6nB,GACxBjsB,KAAK0F,SAAS0C,aAEdpI,KAAKoR,UAITqa,aAAA,SAAapd,GAAe,IAAAa,EAAAlP,KACpB0rB,EAAaxrB,EAAEF,KAAK0F,UAAUa,SAASf,IACvC0mB,EAAYlsB,KAAK6qB,QAAU7qB,KAAK6qB,QAAQ1pB,cAAcmG,IAAuB,KAE9EtH,KAAK0F,SAAS7B,YACf7D,KAAK0F,SAAS7B,WAAWzB,WAAakT,KAAK6W,cAE7CxrB,SAASmT,KAAKsY,YAAYpsB,KAAK0F,UAGjC1F,KAAK0F,SAASkM,MAAMyW,QAAU,QAC9BroB,KAAK0F,SAASqhB,gBAAgB,eAC9B/mB,KAAK0F,SAAS4C,aAAa,cAAc,GAErCpI,EAAEF,KAAK6qB,SAAStkB,SAASf,KAAyB0mB,EACpDA,EAAU7T,UAAY,EAEtBrY,KAAK0F,SAAS2S,UAAY,EAGxBqT,GACFtrB,EAAKyB,OAAO7B,KAAK0F,UAGnBxF,EAAEF,KAAK0F,UAAUqJ,SAASvJ,IAEtBxF,KAAKkL,QAAQ9C,OACfpI,KAAKqsB,gBAOoB,SAArBC,IACApd,EAAKhE,QAAQ9C,OACf8G,EAAKxJ,SAAS0C,QAEhB8G,EAAKqB,kBAAmB,EACxBrQ,EAAEgP,EAAKxJ,UAAU3D,QAAQwqB,GAT3B,IAAMA,EAAarsB,EAAEkF,MAAMA,GAAM8K,MAAO,CACtC7B,cAAAA,IAWF,GAAIqd,EAAY,CACd,IAAMpqB,EAAsBlB,EAAKiB,iCAAiCrB,KAAK6qB,SAEvE3qB,EAAEF,KAAK6qB,SACJ1qB,IAAIC,EAAKR,eAAgB0sB,GACzBloB,qBAAqB9C,QAExBgrB,OAIJD,cAAA,WAAgB,IAAAG,EAAAxsB,KACdE,EAAES,UACCgM,IAAIvH,GAAMklB,SACVpjB,GAAG9B,GAAMklB,QAAS,SAACjmB,GACd1D,WAAa0D,EAAMK,QACnB8nB,EAAK9mB,WAAarB,EAAMK,QACsB,IAA9CxE,EAAEssB,EAAK9mB,UAAU+mB,IAAIpoB,EAAMK,QAAQsE,QACrCwjB,EAAK9mB,SAAS0C,aAKtBkjB,gBAAA,WAAkB,IAAAoB,EAAA1sB,KACZA,KAAK+qB,UAAY/qB,KAAKkL,QAAQ9B,SAChClJ,EAAEF,KAAK0F,UAAUwB,GAAG9B,GAAMqlB,gBAAiB,SAACpmB,GAlTvB,KAmTfA,EAAMwJ,OACR6e,EAAKZ,+BAGC9rB,KAAK+qB,UACf7qB,EAAEF,KAAK0F,UAAUiH,IAAIvH,GAAMqlB,oBAI/Bc,gBAAA,WAAkB,IAAAoB,EAAA3sB,KACZA,KAAK+qB,SACP7qB,EAAEwI,QAAQxB,GAAG9B,GAAMmlB,OAAQ,SAAClmB,GAAD,OAAWsoB,EAAKd,aAAaxnB,KAExDnE,EAAEwI,QAAQiE,IAAIvH,GAAMmlB,WAIxBoB,WAAA,WAAa,IAAAiB,EAAA5sB,KACXA,KAAK0F,SAASkM,MAAMyW,QAAU,OAC9BroB,KAAK0F,SAAS4C,aAAa,eAAe,GAC1CtI,KAAK0F,SAASqhB,gBAAgB,cAC9B/mB,KAAKuQ,kBAAmB,EACxBvQ,KAAKwrB,cAAc,WACjBtrB,EAAES,SAASmT,MAAMxN,YAAYd,IAC7BonB,EAAKC,oBACLD,EAAKE,kBACL5sB,EAAE0sB,EAAKlnB,UAAU3D,QAAQqD,GAAMgL,aAInC2c,gBAAA,WACM/sB,KAAK8qB,YACP5qB,EAAEF,KAAK8qB,WAAWpkB,SAClB1G,KAAK8qB,UAAY,SAIrBU,cAAA,SAAcxM,GAAU,IAAAgO,EAAAhtB,KAChBitB,EAAU/sB,EAAEF,KAAK0F,UAAUa,SAASf,IACtCA,GAAiB,GAErB,GAAIxF,KAAK+qB,UAAY/qB,KAAKkL,QAAQkf,SAAU,CA4B1C,GA3BApqB,KAAK8qB,UAAYnqB,SAASusB,cAAc,OACxCltB,KAAK8qB,UAAUqC,UAAY3nB,GAEvBynB,GACFjtB,KAAK8qB,UAAU9iB,UAAUiB,IAAIgkB,GAG/B/sB,EAAEF,KAAK8qB,WAAWsC,SAASzsB,SAASmT,MAEpC5T,EAAEF,KAAK0F,UAAUwB,GAAG9B,GAAMolB,cAAe,SAACnmB,GACpC2oB,EAAK/B,qBACP+B,EAAK/B,sBAAuB,EAG1B5mB,EAAMK,SAAWL,EAAMkO,eAI3Bya,EAAKlB,+BAGHmB,GACF7sB,EAAKyB,OAAO7B,KAAK8qB,WAGnB5qB,EAAEF,KAAK8qB,WAAW/b,SAASvJ,KAEtBwZ,EACH,OAGF,IAAKiO,EAEH,YADAjO,IAIF,IAAMqO,EAA6BjtB,EAAKiB,iCAAiCrB,KAAK8qB,WAE9E5qB,EAAEF,KAAK8qB,WACJ3qB,IAAIC,EAAKR,eAAgBof,GACzB5a,qBAAqBipB,QACnB,IAAKrtB,KAAK+qB,UAAY/qB,KAAK8qB,UAAW,CAC3C5qB,EAAEF,KAAK8qB,WAAWxkB,YAAYd,IAE9B,IAAM8nB,EAAiB,WACrBN,EAAKD,kBACD/N,GACFA,KAIJ,GAAI9e,EAAEF,KAAK0F,UAAUa,SAASf,IAAiB,CAC7C,IAAM6nB,EAA6BjtB,EAAKiB,iCAAiCrB,KAAK8qB,WAE9E5qB,EAAEF,KAAK8qB,WACJ3qB,IAAIC,EAAKR,eAAgB0tB,GACzBlpB,qBAAqBipB,QAExBC,SAEOtO,GACTA,OASJqM,cAAA,WACE,IAAMkC,EACJvtB,KAAK0F,SAAS8nB,aAAe7sB,SAAS8C,gBAAgBiV,cAEnD1Y,KAAKgrB,oBAAsBuC,IAC9BvtB,KAAK0F,SAASkM,MAAM6b,YAAiBztB,KAAKkrB,gBAA1C,MAGElrB,KAAKgrB,qBAAuBuC,IAC9BvtB,KAAK0F,SAASkM,MAAM8b,aAAkB1tB,KAAKkrB,gBAA3C,SAIJ2B,kBAAA,WACE7sB,KAAK0F,SAASkM,MAAM6b,YAAc,GAClCztB,KAAK0F,SAASkM,MAAM8b,aAAe,MAGrCvC,gBAAA,WACE,IAAM/S,EAAOzX,SAASmT,KAAK9B,wBAC3BhS,KAAKgrB,mBAAqB5S,EAAKH,KAAOG,EAAKJ,MAAQtP,OAAO2R,WAC1Dra,KAAKkrB,gBAAkBlrB,KAAK2tB,wBAG9BvC,cAAA,WAAgB,IAAAwC,EAAA5tB,KACd,GAAIA,KAAKgrB,mBAAoB,CAG3B,IAAM6C,EAAe,GAAGjlB,MAAM/F,KAAKlC,SAASkI,iBAAiBvB,KACvDwmB,EAAgB,GAAGllB,MAAM/F,KAAKlC,SAASkI,iBAAiBvB,KAG9DpH,EAAE2tB,GAAcjnB,KAAK,SAAC2F,EAAOzL,GAC3B,IAAMitB,EAAgBjtB,EAAQ8Q,MAAM8b,aAC9BM,EAAoB9tB,EAAEY,GAASS,IAAI,iBACzCrB,EAAEY,GACCgG,KAAK,gBAAiBinB,GACtBxsB,IAAI,gBAAoBG,WAAWssB,GAAqBJ,EAAK1C,gBAFhE,QAMFhrB,EAAE4tB,GAAelnB,KAAK,SAAC2F,EAAOzL,GAC5B,IAAMmtB,EAAentB,EAAQ8Q,MAAMuK,YAC7B+R,EAAmBhuB,EAAEY,GAASS,IAAI,gBACxCrB,EAAEY,GACCgG,KAAK,eAAgBmnB,GACrB1sB,IAAI,eAAmBG,WAAWwsB,GAAoBN,EAAK1C,gBAF9D,QAMF,IAAM6C,EAAgBptB,SAASmT,KAAKlC,MAAM8b,aACpCM,EAAoB9tB,EAAES,SAASmT,MAAMvS,IAAI,iBAC/CrB,EAAES,SAASmT,MACRhN,KAAK,gBAAiBinB,GACtBxsB,IAAI,gBAAoBG,WAAWssB,GAAqBhuB,KAAKkrB,gBAFhE,MAKFhrB,EAAES,SAASmT,MAAM/E,SAASvJ,OAG5BsnB,gBAAA,WAEE,IAAMe,EAAe,GAAGjlB,MAAM/F,KAAKlC,SAASkI,iBAAiBvB,KAC7DpH,EAAE2tB,GAAcjnB,KAAK,SAAC2F,EAAOzL,GAC3B,IAAMkZ,EAAU9Z,EAAEY,GAASgG,KAAK,iBAChC5G,EAAEY,GAASoF,WAAW,iBACtBpF,EAAQ8Q,MAAM8b,aAAe1T,GAAoB,KAInD,IAAMmU,EAAW,GAAGvlB,MAAM/F,KAAKlC,SAASkI,iBAAT,GAA6BvB,KAC5DpH,EAAEiuB,GAAUvnB,KAAK,SAAC2F,EAAOzL,GACvB,IAAMstB,EAASluB,EAAEY,GAASgG,KAAK,gBACT,oBAAXsnB,GACTluB,EAAEY,GAASS,IAAI,eAAgB6sB,GAAQloB,WAAW,kBAKtD,IAAM8T,EAAU9Z,EAAES,SAASmT,MAAMhN,KAAK,iBACtC5G,EAAES,SAASmT,MAAM5N,WAAW,iBAC5BvF,SAASmT,KAAKlC,MAAM8b,aAAe1T,GAAoB,MAGzD2T,mBAAA,WACE,IAAMU,EAAY1tB,SAASusB,cAAc,OACzCmB,EAAUlB,UAAY3nB,GACtB7E,SAASmT,KAAKsY,YAAYiC,GAC1B,IAAMC,EAAiBD,EAAUrc,wBAAwB+E,MAAQsX,EAAU5V,YAE3E,OADA9X,SAASmT,KAAKwT,YAAY+G,GACnBC,KAKF3nB,iBAAP,SAAwBpE,EAAQ8L,GAC9B,OAAOrO,KAAK4G,KAAK,WACf,IAAIE,EAAO5G,EAAEF,MAAM8G,KAAK7B,IAClBiG,EAAO0B,EAAA,GACR1D,GADQ,GAERhJ,EAAEF,MAAM8G,OAFA,GAGU,iBAAXvE,GAAuBA,EAASA,EAAS,IAQrD,GALKuE,IACHA,EAAO,IAAI8jB,EAAM5qB,KAAMkL,GACvBhL,EAAEF,MAAM8G,KAAK7B,GAAU6B,IAGH,iBAAXvE,EAAqB,CAC9B,GAA4B,oBAAjBuE,EAAKvE,GACd,MAAM,IAAIyB,UAAJ,oBAAkCzB,EAAlC,KAERuE,EAAKvE,GAAQ8L,QACJnD,EAAQmG,MACjBvK,EAAKuK,KAAKhD,8CA9cd,MA7EuB,wCAiFvB,OAAOnF,YAsdXhJ,EAAES,UAAUuG,GAAG9B,GAAMG,eAAgB+B,GAAsB,SAAUjD,GAAO,IACtEK,EADsE6pB,EAAAvuB,KAEpEe,EAAWX,EAAKS,uBAAuBb,MAEzCe,IACF2D,EAAS/D,SAASQ,cAAcJ,IAGlC,IAAMwB,EAASrC,EAAEwE,GAAQoC,KAAK7B,IAC1B,SADW2H,EAAA,GAER1M,EAAEwE,GAAQoC,OAFF,GAGR5G,EAAEF,MAAM8G,QAGM,MAAjB9G,KAAKmI,SAAoC,SAAjBnI,KAAKmI,SAC/B9D,EAAM4C,iBAGR,IAAMyL,EAAUxS,EAAEwE,GAAQvE,IAAIiF,GAAM6K,KAAM,SAACgZ,GACrCA,EAAUljB,sBAKd2M,EAAQvS,IAAIiF,GAAMgL,OAAQ,WACpBlQ,EAAEquB,GAAM5pB,GAAG,aACb4pB,EAAKnmB,YAKXwiB,GAAMjkB,iBAAiB9D,KAAK3C,EAAEwE,GAASnC,EAAQvC,QASjDE,EAAEgE,GAAGc,IAAQ4lB,GAAMjkB,iBACnBzG,EAAEgE,GAAGc,IAAMmC,YAAcyjB,GACzB1qB,EAAEgE,GAAGc,IAAMoC,WAAa,WAEtB,OADAlH,EAAEgE,GAAGc,IAAQG,GACNylB,GAAMjkB,kBC7lBf,IAAM6nB,GAAW,CACf,aACA,OACA,OACA,WACA,WACA,SACA,MACA,cAKWC,GAAmB,CAE9BC,IAAK,CAAC,QAAS,MAAO,KAAM,OAAQ,OAJP,kBAK7BnT,EAAG,CAAC,SAAU,OAAQ,QAAS,OAC/BJ,KAAM,GACNK,EAAG,GACHmT,GAAI,GACJC,IAAK,GACLC,KAAM,GACNC,IAAK,GACLC,GAAI,GACJC,GAAI,GACJC,GAAI,GACJC,GAAI,GACJC,GAAI,GACJC,GAAI,GACJC,GAAI,GACJC,GAAI,GACJxmB,EAAG,GACHymB,IAAK,CAAC,MAAO,MAAO,QAAS,QAAS,UACtCC,GAAI,GACJC,GAAI,GACJC,EAAG,GACHC,IAAK,GACLC,EAAG,GACHC,MAAO,GACPC,KAAM,GACNC,IAAK,GACLC,IAAK,GACLC,OAAQ,GACRC,EAAG,GACHC,GAAI,IAQAC,GAAmB,8DAOnBC,GAAmB,sIAyBlB,SAASC,GAAaC,EAAYC,EAAWC,GAClD,GAA0B,IAAtBF,EAAWvnB,OACb,OAAOunB,EAGT,GAAIE,GAAoC,mBAAfA,EACvB,OAAOA,EAAWF,GAQpB,IALA,IACMG,GADY,IAAIhoB,OAAOioB,WACKC,gBAAgBL,EAAY,aACxDM,EAAgBnuB,OAAOuY,KAAKuV,GAC5BrC,EAAW,GAAGvlB,MAAM/F,KAAK6tB,EAAgB5c,KAAKjL,iBAAiB,MAZPioB,EAAA,SAcrDhoB,GACP,IAAM+Q,EAAKsU,EAASrlB,GACdioB,EAASlX,EAAGlG,SAASxQ,cAE3B,IAA0D,IAAtD0tB,EAAc/iB,QAAQ+L,EAAGlG,SAASxQ,eAGpC,OAFA0W,EAAGhW,WAAWyjB,YAAYzN,GAE1B,WAGF,IAAMmX,EAAgB,GAAGpoB,MAAM/F,KAAKgX,EAAGoM,YACjCgL,EAAwB,GAAGzP,OAAOgP,EAAU,MAAQ,GAAIA,EAAUO,IAAW,IAEnFC,EAAcrT,QAAQ,SAAC9L,IAlD3B,SAA0BA,EAAMqf,GAC9B,IAAMC,EAAWtf,EAAK8B,SAASxQ,cAE/B,IAAgD,IAA5C+tB,EAAqBpjB,QAAQqjB,GAC/B,OAAoC,IAAhC3C,GAAS1gB,QAAQqjB,IACZlvB,QAAQ4P,EAAKuf,UAAUluB,MAAMktB,KAAqBve,EAAKuf,UAAUluB,MAAMmtB,KASlF,IAHA,IAAMgB,EAASH,EAAqBrgB,OAAO,SAACygB,GAAD,OAAeA,aAAqBluB,SAGtE0F,EAAI,EAAGyoB,EAAIF,EAAOroB,OAAQF,EAAIyoB,EAAGzoB,IACxC,GAAIqoB,EAASjuB,MAAMmuB,EAAOvoB,IACxB,OAAO,EAIX,OAAO,EA+BE0oB,CAAiB3f,EAAMof,IAC1BpX,EAAGkN,gBAAgBlV,EAAK8B,aAfrB7K,EAAI,EAAGC,EAAMolB,EAASnlB,OAAQF,EAAIC,EAAKD,IAAKgoB,EAA5ChoB,GAoBT,OAAO4nB,EAAgB5c,KAAK2d,UCxG9B,IAAMzsB,GAAwB,UAExBC,GAAwB,aACxBC,GAAS,IAAmBD,GAC5BE,GAAwBjF,EAAEgE,GAAGc,IAC7B0sB,GAAwB,aACxBC,GAAwB,IAAIvuB,OAAJ,UAAqBsuB,GAArB,OAAyC,KACjEE,GAAwB,CAAC,WAAY,YAAa,cAElDnoB,GAAc,CAClBooB,UAAoB,UACpBC,SAAoB,SACpBC,MAAoB,4BACpBhwB,QAAoB,SACpBiwB,MAAoB,kBACpB7b,KAAoB,UACpBpV,SAAoB,mBACpB8Z,UAAoB,oBACpB+G,OAAoB,2BACpBqQ,UAAoB,2BACpBC,kBAAoB,iBACpB9J,SAAoB,mBACpB+J,SAAoB,UACpB1B,WAAoB,kBACpBD,UAAoB,SACpBlI,aAAoB,iBAGhBH,GAAgB,CACpBiK,KAAS,OACTC,IAAS,MACTC,MAAS,QACTC,OAAS,SACTC,KAAS,QAGLtpB,GAAU,CACd2oB,WAAoB,EACpBC,SAAoB,uGAGpB/vB,QAAoB,cACpBgwB,MAAoB,GACpBC,MAAoB,EACpB7b,MAAoB,EACpBpV,UAAoB,EACpB8Z,UAAoB,MACpB+G,OAAoB,EACpBqQ,WAAoB,EACpBC,kBAAoB,OACpB9J,SAAoB,eACpB+J,UAAoB,EACpB1B,WAAoB,KACpBD,UAAoB/B,GACpBnG,aAAoB,MAGhBmK,GACG,OADHA,GAEG,MAGHrtB,GAAQ,CACZ+K,KAAI,OAAgBjL,GACpBkL,OAAM,SAAgBlL,GACtB+K,KAAI,OAAgB/K,GACpBgL,MAAK,QAAgBhL,GACrBwtB,SAAQ,WAAgBxtB,GACxB8iB,MAAK,QAAgB9iB,GACrBolB,QAAO,UAAgBplB,GACvBytB,SAAQ,WAAgBztB,GACxB4E,WAAU,aAAgB5E,GAC1B6E,WAAU,aAAgB7E,IAGtBM,GACG,OADHA,GAEG,OAGH8B,GAEY,iBAFZA,GAGY,SAGZsrB,GACK,QADLA,GAEK,QAFLA,GAGK,QAHLA,GAIK,SAULC,cACJ,SAAAA,EAAY/xB,EAASyB,GACnB,GAAsB,oBAAX4kB,GACT,MAAM,IAAInjB,UAAU,mEAItBhE,KAAK8yB,YAAiB,EACtB9yB,KAAK+yB,SAAiB,EACtB/yB,KAAKgzB,YAAiB,GACtBhzB,KAAKizB,eAAiB,GACtBjzB,KAAKwoB,QAAiB,KAGtBxoB,KAAKc,QAAUA,EACfd,KAAKuC,OAAUvC,KAAKmL,WAAW5I,GAC/BvC,KAAKkzB,IAAU,KAEflzB,KAAKmzB,2CAmCPC,OAAA,WACEpzB,KAAK8yB,YAAa,KAGpBO,QAAA,WACErzB,KAAK8yB,YAAa,KAGpBQ,cAAA,WACEtzB,KAAK8yB,YAAc9yB,KAAK8yB,cAG1BprB,OAAA,SAAOrD,GACL,GAAKrE,KAAK8yB,WAIV,GAAIzuB,EAAO,CACT,IAAMkvB,EAAUvzB,KAAKypB,YAAYxkB,SAC7B6kB,EAAU5pB,EAAEmE,EAAMkO,eAAezL,KAAKysB,GAErCzJ,IACHA,EAAU,IAAI9pB,KAAKypB,YACjBplB,EAAMkO,cACNvS,KAAKwzB,sBAEPtzB,EAAEmE,EAAMkO,eAAezL,KAAKysB,EAASzJ,IAGvCA,EAAQmJ,eAAeQ,OAAS3J,EAAQmJ,eAAeQ,MAEnD3J,EAAQ4J,uBACV5J,EAAQ6J,OAAO,KAAM7J,GAErBA,EAAQ8J,OAAO,KAAM9J,OAElB,CACL,GAAI5pB,EAAEF,KAAK6zB,iBAAiBttB,SAASf,IAEnC,YADAxF,KAAK4zB,OAAO,KAAM5zB,MAIpBA,KAAK2zB,OAAO,KAAM3zB,UAItBiG,QAAA,WACEyH,aAAa1N,KAAK+yB,UAElB7yB,EAAEgG,WAAWlG,KAAKc,QAASd,KAAKypB,YAAYxkB,UAE5C/E,EAAEF,KAAKc,SAAS6L,IAAI3M,KAAKypB,YAAYvkB,WACrChF,EAAEF,KAAKc,SAASsF,QAAQ,UAAUuG,IAAI,gBAAiB3M,KAAK8zB,mBAExD9zB,KAAKkzB,KACPhzB,EAAEF,KAAKkzB,KAAKxsB,SAGd1G,KAAK8yB,WAAiB,KACtB9yB,KAAK+yB,SAAiB,KACtB/yB,KAAKgzB,YAAiB,KACtBhzB,KAAKizB,eAAiB,KAClBjzB,KAAKwoB,SACPxoB,KAAKwoB,QAAQe,UAGfvpB,KAAKwoB,QAAU,KACfxoB,KAAKc,QAAU,KACfd,KAAKuC,OAAU,KACfvC,KAAKkzB,IAAU,QAGjB7hB,KAAA,WAAO,IAAAtR,EAAAC,KACL,GAAuC,SAAnCE,EAAEF,KAAKc,SAASS,IAAI,WACtB,MAAM,IAAI+B,MAAM,uCAGlB,IAAM2lB,EAAY/oB,EAAEkF,MAAMpF,KAAKypB,YAAYrkB,MAAM6K,MACjD,GAAIjQ,KAAK+zB,iBAAmB/zB,KAAK8yB,WAAY,CAC3C5yB,EAAEF,KAAKc,SAASiB,QAAQknB,GAExB,IAAM+K,EAAa5zB,EAAKoD,eAAexD,KAAKc,SACtCmzB,EAAa/zB,EAAE+H,SACJ,OAAf+rB,EAAsBA,EAAah0B,KAAKc,QAAQyS,cAAc9P,gBAC9DzD,KAAKc,SAGP,GAAImoB,EAAUljB,uBAAyBkuB,EACrC,OAGF,IAAMf,EAAQlzB,KAAK6zB,gBACbK,EAAQ9zB,EAAKG,OAAOP,KAAKypB,YAAYzkB,MAE3CkuB,EAAI5qB,aAAa,KAAM4rB,GACvBl0B,KAAKc,QAAQwH,aAAa,mBAAoB4rB,GAE9Cl0B,KAAKm0B,aAEDn0B,KAAKuC,OAAOsvB,WACd3xB,EAAEgzB,GAAKnkB,SAASvJ,IAGlB,IAAMqV,EAA8C,mBAA1B7a,KAAKuC,OAAOsY,UAClC7a,KAAKuC,OAAOsY,UAAUhY,KAAK7C,KAAMkzB,EAAKlzB,KAAKc,SAC3Cd,KAAKuC,OAAOsY,UAEVuZ,EAAap0B,KAAKq0B,eAAexZ,GACvC7a,KAAKs0B,mBAAmBF,GAExB,IAAMnC,EAAYjyB,KAAKu0B,gBACvBr0B,EAAEgzB,GAAKpsB,KAAK9G,KAAKypB,YAAYxkB,SAAUjF,MAElCE,EAAE+H,SAASjI,KAAKc,QAAQyS,cAAc9P,gBAAiBzD,KAAKkzB,MAC/DhzB,EAAEgzB,GAAK9F,SAAS6E,GAGlB/xB,EAAEF,KAAKc,SAASiB,QAAQ/B,KAAKypB,YAAYrkB,MAAMstB,UAE/C1yB,KAAKwoB,QAAU,IAAIrB,GAAOnnB,KAAKc,QAASoyB,EAAKlzB,KAAKopB,iBAAiBgL,IAEnEl0B,EAAEgzB,GAAKnkB,SAASvJ,IAMZ,iBAAkB7E,SAAS8C,iBAC7BvD,EAAES,SAASmT,MAAMhF,WAAW5H,GAAG,YAAa,KAAMhH,EAAEmpB,MAGtD,IAAMmL,EAAW,WACXz0B,EAAKwC,OAAOsvB,WACd9xB,EAAK00B,iBAEP,IAAMC,EAAiB30B,EAAKizB,YAC5BjzB,EAAKizB,YAAkB,KAEvB9yB,EAAEH,EAAKe,SAASiB,QAAQhC,EAAK0pB,YAAYrkB,MAAM8K,OAE3CwkB,IAAmBjC,IACrB1yB,EAAK6zB,OAAO,KAAM7zB,IAItB,GAAIG,EAAEF,KAAKkzB,KAAK3sB,SAASf,IAAiB,CACxC,IAAMlE,EAAqBlB,EAAKiB,iCAAiCrB,KAAKkzB,KAEtEhzB,EAAEF,KAAKkzB,KACJ/yB,IAAIC,EAAKR,eAAgB40B,GACzBpwB,qBAAqB9C,QAExBkzB,QAKNpjB,KAAA,SAAK4N,GAGc,SAAXwV,IACAxnB,EAAKgmB,cAAgBP,IAAmBS,EAAIrvB,YAC9CqvB,EAAIrvB,WAAWyjB,YAAY4L,GAG7BlmB,EAAK2nB,iBACL3nB,EAAKlM,QAAQimB,gBAAgB,oBAC7B7mB,EAAE8M,EAAKlM,SAASiB,QAAQiL,EAAKyc,YAAYrkB,MAAMgL,QAC1B,OAAjBpD,EAAKwb,SACPxb,EAAKwb,QAAQe,UAGXvK,GACFA,IAhBS,IAAAhS,EAAAhN,KACPkzB,EAAYlzB,KAAK6zB,gBACjBvK,EAAYppB,EAAEkF,MAAMpF,KAAKypB,YAAYrkB,MAAM+K,MAoBjD,GAFAjQ,EAAEF,KAAKc,SAASiB,QAAQunB,IAEpBA,EAAUvjB,qBAAd,CAgBA,GAZA7F,EAAEgzB,GAAK5sB,YAAYd,IAIf,iBAAkB7E,SAAS8C,iBAC7BvD,EAAES,SAASmT,MAAMhF,WAAWnC,IAAI,YAAa,KAAMzM,EAAEmpB,MAGvDrpB,KAAKizB,eAAeL,KAAiB,EACrC5yB,KAAKizB,eAAeL,KAAiB,EACrC5yB,KAAKizB,eAAeL,KAAiB,EAEjC1yB,EAAEF,KAAKkzB,KAAK3sB,SAASf,IAAiB,CACxC,IAAMlE,EAAqBlB,EAAKiB,iCAAiC6xB,GAEjEhzB,EAAEgzB,GACC/yB,IAAIC,EAAKR,eAAgB40B,GACzBpwB,qBAAqB9C,QAExBkzB,IAGFx0B,KAAKgzB,YAAc,OAGrBvL,OAAA,WACuB,OAAjBznB,KAAKwoB,SACPxoB,KAAKwoB,QAAQlJ,oBAMjByU,cAAA,WACE,OAAO9xB,QAAQjC,KAAK40B,eAGtBN,mBAAA,SAAmBF,GACjBl0B,EAAEF,KAAK6zB,iBAAiB9kB,SAAY2iB,GAApC,IAAoD0C,MAGtDP,cAAA,WAEE,OADA7zB,KAAKkzB,IAAMlzB,KAAKkzB,KAAOhzB,EAAEF,KAAKuC,OAAOuvB,UAAU,GACxC9xB,KAAKkzB,OAGdiB,WAAA,WACE,IAAMjB,EAAMlzB,KAAK6zB,gBACjB7zB,KAAK60B,kBAAkB30B,EAAEgzB,EAAIrqB,iBAAiBvB,KAA0BtH,KAAK40B,YAC7E10B,EAAEgzB,GAAK5sB,YAAed,GAAtB,IAAwCA,OAG1CqvB,kBAAA,SAAkBhuB,EAAUiuB,GACH,iBAAZA,IAAyBA,EAAQ1yB,WAAY0yB,EAAQ3wB,OAa5DnE,KAAKuC,OAAO4T,MACVnW,KAAKuC,OAAO4vB,WACd2C,EAAUxE,GAAawE,EAAS90B,KAAKuC,OAAOiuB,UAAWxwB,KAAKuC,OAAOkuB,aAGrE5pB,EAASsP,KAAK2e,IAEdjuB,EAASkuB,KAAKD,GAlBV90B,KAAKuC,OAAO4T,KACTjW,EAAE40B,GAAS3uB,SAASxB,GAAGkC,IAC1BA,EAASmuB,QAAQC,OAAOH,GAG1BjuB,EAASkuB,KAAK70B,EAAE40B,GAASC,WAiB/BH,SAAA,WACE,IAAI7C,EAAQ/xB,KAAKc,QAAQE,aAAa,uBAQtC,OALE+wB,EADGA,IACkC,mBAAtB/xB,KAAKuC,OAAOwvB,MACvB/xB,KAAKuC,OAAOwvB,MAAMlvB,KAAK7C,KAAKc,SAC5Bd,KAAKuC,OAAOwvB,UAQpB3I,iBAAA,SAAiBgL,GAAY,IAAAjnB,EAAAnN,KAuB3B,OAAA4M,EAAA,GAtBwB,CACtBiO,UAAWuZ,EACX9W,UAAW,CACTsE,OAAQ5hB,KAAK4pB,aACblF,KAAM,CACJK,SAAU/kB,KAAKuC,OAAO2vB,mBAExBlO,MAAO,CACLljB,QAASwG,IAEX8b,gBAAiB,CACfnJ,kBAAmBja,KAAKuC,OAAO6lB,WAGnCtF,SAAU,SAAChc,GACLA,EAAK8d,oBAAsB9d,EAAK+T,WAClC1N,EAAK+nB,6BAA6BpuB,IAGtCic,SAAU,SAACjc,GAAD,OAAUqG,EAAK+nB,6BAA6BpuB,KAGxD,GAEK9G,KAAKuC,OAAO+lB,iBAInBsB,WAAA,WAAa,IAAA1a,EAAAlP,KACL4hB,EAAS,GAef,MAbkC,mBAAvB5hB,KAAKuC,OAAOqf,OACrBA,EAAO1d,GAAK,SAAC4C,GAMX,OALAA,EAAKiR,QAALnL,EAAA,GACK9F,EAAKiR,QADV,GAEK7I,EAAK3M,OAAOqf,OAAO9a,EAAKiR,QAAS7I,EAAKpO,UAAY,IAGhDgG,GAGT8a,EAAOA,OAAS5hB,KAAKuC,OAAOqf,OAGvBA,KAGT2S,cAAA,WACE,OAA8B,IAA1Bv0B,KAAKuC,OAAO0vB,UACPtxB,SAASmT,KAGd1T,EAAK8B,UAAUlC,KAAKuC,OAAO0vB,WACtB/xB,EAAEF,KAAKuC,OAAO0vB,WAGhB/xB,EAAES,UAAUsc,KAAKjd,KAAKuC,OAAO0vB,cAGtCoC,eAAA,SAAexZ,GACb,OAAOsN,GAActN,EAAUtX,kBAGjC4vB,cAAA,WAAgB,IAAA3G,EAAAxsB,KACGA,KAAKuC,OAAOR,QAAQH,MAAM,KAElC+b,QAAQ,SAAC5b,GAChB,GAAgB,UAAZA,EACF7B,EAAEssB,EAAK1rB,SAASoG,GACdslB,EAAK/C,YAAYrkB,MAAM4iB,MACvBwE,EAAKjqB,OAAOxB,SACZ,SAACsD,GAAD,OAAWmoB,EAAK9kB,OAAOrD,UAEpB,GAAItC,IAAY6wB,GAAgB,CACrC,IAAMuC,EAAUpzB,IAAY6wB,GACxBpG,EAAK/C,YAAYrkB,MAAM0E,WACvB0iB,EAAK/C,YAAYrkB,MAAMklB,QACrB8K,EAAWrzB,IAAY6wB,GACzBpG,EAAK/C,YAAYrkB,MAAM2E,WACvByiB,EAAK/C,YAAYrkB,MAAMutB,SAE3BzyB,EAAEssB,EAAK1rB,SACJoG,GACCiuB,EACA3I,EAAKjqB,OAAOxB,SACZ,SAACsD,GAAD,OAAWmoB,EAAKmH,OAAOtvB,KAExB6C,GACCkuB,EACA5I,EAAKjqB,OAAOxB,SACZ,SAACsD,GAAD,OAAWmoB,EAAKoH,OAAOvvB,QAK/BrE,KAAK8zB,kBAAoB,WACnBtH,EAAK1rB,SACP0rB,EAAKpb,QAITlR,EAAEF,KAAKc,SAASsF,QAAQ,UAAUc,GAChC,gBACAlH,KAAK8zB,mBAGH9zB,KAAKuC,OAAOxB,SACdf,KAAKuC,OAALqK,EAAA,GACK5M,KAAKuC,OADV,CAEER,QAAS,SACThB,SAAU,KAGZf,KAAKq1B,eAITA,UAAA,WACE,IAAMC,SAAmBt1B,KAAKc,QAAQE,aAAa,wBAE/ChB,KAAKc,QAAQE,aAAa,UAA0B,UAAds0B,IACxCt1B,KAAKc,QAAQwH,aACX,sBACAtI,KAAKc,QAAQE,aAAa,UAAY,IAGxChB,KAAKc,QAAQwH,aAAa,QAAS,QAIvCqrB,OAAA,SAAOtvB,EAAOylB,GACZ,IAAMyJ,EAAUvzB,KAAKypB,YAAYxkB,UACjC6kB,EAAUA,GAAW5pB,EAAEmE,EAAMkO,eAAezL,KAAKysB,MAG/CzJ,EAAU,IAAI9pB,KAAKypB,YACjBplB,EAAMkO,cACNvS,KAAKwzB,sBAEPtzB,EAAEmE,EAAMkO,eAAezL,KAAKysB,EAASzJ,IAGnCzlB,IACFylB,EAAQmJ,eACS,YAAf5uB,EAAMyD,KAAqB8qB,GAAgBA,KACzC,GAGF1yB,EAAE4pB,EAAQ+J,iBAAiBttB,SAASf,KAAmBskB,EAAQkJ,cAAgBP,GACjF3I,EAAQkJ,YAAcP,IAIxB/kB,aAAaoc,EAAQiJ,UAErBjJ,EAAQkJ,YAAcP,GAEjB3I,EAAQvnB,OAAOyvB,OAAUlI,EAAQvnB,OAAOyvB,MAAM3gB,KAKnDyY,EAAQiJ,SAAW1yB,WAAW,WACxBypB,EAAQkJ,cAAgBP,IAC1B3I,EAAQzY,QAETyY,EAAQvnB,OAAOyvB,MAAM3gB,MARtByY,EAAQzY,WAWZuiB,OAAA,SAAOvvB,EAAOylB,GACZ,IAAMyJ,EAAUvzB,KAAKypB,YAAYxkB,UACjC6kB,EAAUA,GAAW5pB,EAAEmE,EAAMkO,eAAezL,KAAKysB,MAG/CzJ,EAAU,IAAI9pB,KAAKypB,YACjBplB,EAAMkO,cACNvS,KAAKwzB,sBAEPtzB,EAAEmE,EAAMkO,eAAezL,KAAKysB,EAASzJ,IAGnCzlB,IACFylB,EAAQmJ,eACS,aAAf5uB,EAAMyD,KAAsB8qB,GAAgBA,KAC1C,GAGF9I,EAAQ4J,yBAIZhmB,aAAaoc,EAAQiJ,UAErBjJ,EAAQkJ,YAAcP,GAEjB3I,EAAQvnB,OAAOyvB,OAAUlI,EAAQvnB,OAAOyvB,MAAM5gB,KAKnD0Y,EAAQiJ,SAAW1yB,WAAW,WACxBypB,EAAQkJ,cAAgBP,IAC1B3I,EAAQ1Y,QAET0Y,EAAQvnB,OAAOyvB,MAAM5gB,MARtB0Y,EAAQ1Y,WAWZsiB,qBAAA,WACE,IAAK,IAAM3xB,KAAW/B,KAAKizB,eACzB,GAAIjzB,KAAKizB,eAAelxB,GACtB,OAAO,EAIX,OAAO,KAGToJ,WAAA,SAAW5I,GACT,IAAMgzB,EAAiBr1B,EAAEF,KAAKc,SAASgG,OAwCvC,OAtCApE,OAAOuY,KAAKsa,GACT5X,QAAQ,SAAC6X,IACyC,IAA7C5D,GAAsB9jB,QAAQ0nB,WACzBD,EAAeC,KAUA,iBAN5BjzB,EAAMqK,EAAA,GACD5M,KAAKypB,YAAYvgB,QADhB,GAEDqsB,EAFC,GAGiB,iBAAXhzB,GAAuBA,EAASA,EAAS,KAGnCyvB,QAChBzvB,EAAOyvB,MAAQ,CACb3gB,KAAM9O,EAAOyvB,MACb5gB,KAAM7O,EAAOyvB,QAIW,iBAAjBzvB,EAAOwvB,QAChBxvB,EAAOwvB,MAAQxvB,EAAOwvB,MAAM9uB,YAGA,iBAAnBV,EAAOuyB,UAChBvyB,EAAOuyB,QAAUvyB,EAAOuyB,QAAQ7xB,YAGlC7C,EAAKiC,gBACH2C,GACAzC,EACAvC,KAAKypB,YAAYhgB,aAGflH,EAAO4vB,WACT5vB,EAAOuvB,SAAWxB,GAAa/tB,EAAOuvB,SAAUvvB,EAAOiuB,UAAWjuB,EAAOkuB,aAGpEluB,KAGTixB,mBAAA,WACE,IAAMjxB,EAAS,GAEf,GAAIvC,KAAKuC,OACP,IAAK,IAAMmV,KAAO1X,KAAKuC,OACjBvC,KAAKypB,YAAYvgB,QAAQwO,KAAS1X,KAAKuC,OAAOmV,KAChDnV,EAAOmV,GAAO1X,KAAKuC,OAAOmV,IAKhC,OAAOnV,KAGToyB,eAAA,WACE,IAAMc,EAAOv1B,EAAEF,KAAK6zB,iBACd6B,EAAWD,EAAK5jB,KAAK,SAAS3O,MAAMyuB,IACzB,OAAb+D,GAAqBA,EAAS1sB,QAChCysB,EAAKnvB,YAAYovB,EAASC,KAAK,QAInCT,6BAAA,SAA6BU,GAC3B,IAAMC,EAAiBD,EAAWvS,SAClCrjB,KAAKkzB,IAAM2C,EAAe9b,OAC1B/Z,KAAK20B,iBACL30B,KAAKs0B,mBAAmBt0B,KAAKq0B,eAAeuB,EAAW/a,eAGzD4Z,eAAA,WACE,IAAMvB,EAAMlzB,KAAK6zB,gBACXiC,EAAsB91B,KAAKuC,OAAOsvB,UAEA,OAApCqB,EAAIlyB,aAAa,iBAIrBd,EAAEgzB,GAAK5sB,YAAYd,IACnBxF,KAAKuC,OAAOsvB,WAAY,EACxB7xB,KAAKoR,OACLpR,KAAKqR,OACLrR,KAAKuC,OAAOsvB,UAAYiE,MAKnBnvB,iBAAP,SAAwBpE,GACtB,OAAOvC,KAAK4G,KAAK,WACf,IAAIE,EAAO5G,EAAEF,MAAM8G,KAAK7B,IAClBiG,EAA4B,iBAAX3I,GAAuBA,EAE9C,IAAKuE,IAAQ,eAAezD,KAAKd,MAI5BuE,IACHA,EAAO,IAAI+rB,EAAQ7yB,KAAMkL,GACzBhL,EAAEF,MAAM8G,KAAK7B,GAAU6B,IAGH,iBAAXvE,GAAqB,CAC9B,GAA4B,oBAAjBuE,EAAKvE,GACd,MAAM,IAAIyB,UAAJ,oBAAkCzB,EAAlC,KAERuE,EAAKvE,iDArnBT,MA3H0B,wCA+H1B,OAAO2G,gCAIP,OAAOlE,oCAIP,OAAOC,iCAIP,OAAOG,qCAIP,OAAOF,uCAIP,OAAOuE,YAymBXvJ,EAAEgE,GAAGc,IAAQ6tB,GAAQlsB,iBACrBzG,EAAEgE,GAAGc,IAAMmC,YAAc0rB,GACzB3yB,EAAEgE,GAAGc,IAAMoC,WAAa,WAEtB,OADAlH,EAAEgE,GAAGc,IAAQG,GACN0tB,GAAQlsB,kBCtwBjB,IAAM3B,GAAsB,UAEtBC,GAAsB,aACtBC,GAAS,IAAiBD,GAC1BE,GAAsBjF,EAAEgE,GAAGc,IAC3B0sB,GAAsB,aACtBC,GAAsB,IAAIvuB,OAAJ,UAAqBsuB,GAArB,OAAyC,KAE/DxoB,GAAO0D,EAAA,GACRimB,GAAQ3pB,QADA,CAEX2R,UAAY,QACZ9Y,QAAY,QACZ+yB,QAAY,GACZhD,SAAY,wIAMRroB,GAAWmD,EAAA,GACZimB,GAAQppB,YADI,CAEfqrB,QAAU,8BAGNtvB,GACG,OADHA,GAEG,OAGH8B,GACM,kBADNA,GAEM,gBAGNlC,GAAQ,CACZ+K,KAAI,OAAgBjL,GACpBkL,OAAM,SAAgBlL,GACtB+K,KAAI,OAAgB/K,GACpBgL,MAAK,QAAgBhL,GACrBwtB,SAAQ,WAAgBxtB,GACxB8iB,MAAK,QAAgB9iB,GACrBolB,QAAO,UAAgBplB,GACvBytB,SAAQ,WAAgBztB,GACxB4E,WAAU,aAAgB5E,GAC1B6E,WAAU,aAAgB7E,IAStB6wB,gMAiCJhC,cAAA,WACE,OAAO/zB,KAAK40B,YAAc50B,KAAKg2B,iBAGjC1B,mBAAA,SAAmBF,GACjBl0B,EAAEF,KAAK6zB,iBAAiB9kB,SAAY2iB,GAApC,IAAoD0C,MAGtDP,cAAA,WAEE,OADA7zB,KAAKkzB,IAAMlzB,KAAKkzB,KAAOhzB,EAAEF,KAAKuC,OAAOuvB,UAAU,GACxC9xB,KAAKkzB,OAGdiB,WAAA,WACE,IAAMsB,EAAOv1B,EAAEF,KAAK6zB,iBAGpB7zB,KAAK60B,kBAAkBY,EAAKxY,KAAK3V,IAAiBtH,KAAK40B,YACvD,IAAIE,EAAU90B,KAAKg2B,cACI,mBAAZlB,IACTA,EAAUA,EAAQjyB,KAAK7C,KAAKc,UAE9Bd,KAAK60B,kBAAkBY,EAAKxY,KAAK3V,IAAmBwtB,GAEpDW,EAAKnvB,YAAed,GAApB,IAAsCA,OAKxCwwB,YAAA,WACE,OAAOh2B,KAAKc,QAAQE,aAAa,iBAC/BhB,KAAKuC,OAAOuyB,WAGhBH,eAAA,WACE,IAAMc,EAAOv1B,EAAEF,KAAK6zB,iBACd6B,EAAWD,EAAK5jB,KAAK,SAAS3O,MAAMyuB,IACzB,OAAb+D,GAAuC,EAAlBA,EAAS1sB,QAChCysB,EAAKnvB,YAAYovB,EAASC,KAAK,QAM5BhvB,iBAAP,SAAwBpE,GACtB,OAAOvC,KAAK4G,KAAK,WACf,IAAIE,EAAO5G,EAAEF,MAAM8G,KAAK7B,IAClBiG,EAA4B,iBAAX3I,EAAsBA,EAAS,KAEtD,IAAKuE,IAAQ,eAAezD,KAAKd,MAI5BuE,IACHA,EAAO,IAAIivB,EAAQ/1B,KAAMkL,GACzBhL,EAAEF,MAAM8G,KAAK7B,GAAU6B,IAGH,iBAAXvE,GAAqB,CAC9B,GAA4B,oBAAjBuE,EAAKvE,GACd,MAAM,IAAIyB,UAAJ,oBAAkCzB,EAAlC,KAERuE,EAAKvE,iDA3FT,MAxDwB,wCA4DxB,OAAO2G,gCAIP,OAAOlE,oCAIP,OAAOC,iCAIP,OAAOG,qCAIP,OAAOF,uCAIP,OAAOuE,UA5BWopB,IA2GtB3yB,EAAEgE,GAAGc,IAAQ+wB,GAAQpvB,iBACrBzG,EAAEgE,GAAGc,IAAMmC,YAAc4uB,GACzB71B,EAAEgE,GAAGc,IAAMoC,WAAa,WAEtB,OADAlH,EAAEgE,GAAGc,IAAQG,GACN4wB,GAAQpvB,kBCpKjB,IAAM3B,GAAqB,YAErBC,GAAqB,eACrBC,GAAS,IAAgBD,GAEzBE,GAAqBjF,EAAEgE,GAAGc,IAE1BkE,GAAU,CACd0Y,OAAS,GACTqU,OAAS,OACTvxB,OAAS,IAGL+E,GAAc,CAClBmY,OAAS,SACTqU,OAAS,SACTvxB,OAAS,oBAGLU,GAAQ,CACZ8wB,SAAQ,WAAmBhxB,GAC3BixB,OAAM,SAAmBjxB,GACzBsC,cAAa,OAAUtC,GAlBE,aAqBrBM,GACY,gBADZA,GAGY,SAGZ8B,GACc,sBADdA,GAGc,oBAHdA,GAIc,YAJdA,GAKc,YALdA,GAMc,mBANdA,GAOc,YAPdA,GAQc,iBARdA,GASc,mBAGd8uB,GACO,SADPA,GAEO,WASPC,cACJ,SAAAA,EAAYv1B,EAASyB,GAAQ,IAAAxC,EAAAC,KAC3BA,KAAK0F,SAAiB5E,EACtBd,KAAKs2B,eAAqC,SAApBx1B,EAAQqH,QAAqBO,OAAS5H,EAC5Dd,KAAKkL,QAAiBlL,KAAKmL,WAAW5I,GACtCvC,KAAK+Q,UAAoB/Q,KAAKkL,QAAQxG,OAAhB,IAA0B4C,GAA1B,IACGtH,KAAKkL,QAAQxG,OADhB,IAC0B4C,GAD1B,IAEGtH,KAAKkL,QAAQxG,OAFhB,IAE0B4C,GAChDtH,KAAKu2B,SAAiB,GACtBv2B,KAAKw2B,SAAiB,GACtBx2B,KAAKy2B,cAAiB,KACtBz2B,KAAK02B,cAAiB,EAEtBx2B,EAAEF,KAAKs2B,gBAAgBpvB,GAAG9B,GAAM+wB,OAAQ,SAAC9xB,GAAD,OAAWtE,EAAK42B,SAAStyB,KAEjErE,KAAK42B,UACL52B,KAAK22B,sCAePC,QAAA,WAAU,IAAA5pB,EAAAhN,KACF62B,EAAa72B,KAAKs2B,iBAAmBt2B,KAAKs2B,eAAe5tB,OAC3D0tB,GAAsBA,GAEpBU,EAAuC,SAAxB92B,KAAKkL,QAAQ+qB,OAC9BY,EAAa72B,KAAKkL,QAAQ+qB,OAExBc,EAAaD,IAAiBV,GAChCp2B,KAAKg3B,gBAAkB,EAE3Bh3B,KAAKu2B,SAAW,GAChBv2B,KAAKw2B,SAAW,GAEhBx2B,KAAK02B,cAAgB12B,KAAKi3B,mBAEV,GAAGruB,MAAM/F,KAAKlC,SAASkI,iBAAiB7I,KAAK+Q,YAG1DmK,IAAI,SAACpa,GACJ,IAAI4D,EACEwyB,EAAiB92B,EAAKS,uBAAuBC,GAMnD,GAJIo2B,IACFxyB,EAAS/D,SAASQ,cAAc+1B,IAG9BxyB,EAAQ,CACV,IAAMyyB,EAAYzyB,EAAOsN,wBACzB,GAAImlB,EAAUpgB,OAASogB,EAAUrgB,OAE/B,MAAO,CACL5W,EAAEwE,GAAQoyB,KAAgB3e,IAAM4e,EAChCG,GAIN,OAAO,OAERrmB,OAAO,SAACsZ,GAAD,OAAUA,IACjB7O,KAAK,SAACC,EAAGC,GAAJ,OAAUD,EAAE,GAAKC,EAAE,KACxBmC,QAAQ,SAACwM,GACRnd,EAAKupB,SAASvlB,KAAKmZ,EAAK,IACxBnd,EAAKwpB,SAASxlB,KAAKmZ,EAAK,SAI9BlkB,QAAA,WACE/F,EAAEgG,WAAWlG,KAAK0F,SAAUT,IAC5B/E,EAAEF,KAAKs2B,gBAAgB3pB,IAAIzH,IAE3BlF,KAAK0F,SAAiB,KACtB1F,KAAKs2B,eAAiB,KACtBt2B,KAAKkL,QAAiB,KACtBlL,KAAK+Q,UAAiB,KACtB/Q,KAAKu2B,SAAiB,KACtBv2B,KAAKw2B,SAAiB,KACtBx2B,KAAKy2B,cAAiB,KACtBz2B,KAAK02B,cAAiB,QAKxBvrB,WAAA,SAAW5I,GAMT,GAA6B,iBAL7BA,EAAMqK,EAAA,GACD1D,GADC,GAEiB,iBAAX3G,GAAuBA,EAASA,EAAS,KAGnCmC,OAAqB,CACrC,IAAI+L,EAAKvQ,EAAEqC,EAAOmC,QAAQmN,KAAK,MAC1BpB,IACHA,EAAKrQ,EAAKG,OAAOyE,IACjB9E,EAAEqC,EAAOmC,QAAQmN,KAAK,KAAMpB,IAE9BlO,EAAOmC,OAAP,IAAoB+L,EAKtB,OAFArQ,EAAKiC,gBAAgB2C,GAAMzC,EAAQkH,IAE5BlH,KAGTy0B,cAAA,WACE,OAAOh3B,KAAKs2B,iBAAmB5tB,OAC3B1I,KAAKs2B,eAAec,YAAcp3B,KAAKs2B,eAAeje,aAG5D4e,iBAAA,WACE,OAAOj3B,KAAKs2B,eAAe9I,cAAgB/sB,KAAKmW,IAC9CjW,SAASmT,KAAK0Z,aACd7sB,SAAS8C,gBAAgB+pB,iBAI7B6J,iBAAA,WACE,OAAOr3B,KAAKs2B,iBAAmB5tB,OAC3BA,OAAO4R,YAActa,KAAKs2B,eAAetkB,wBAAwB8E,UAGvE6f,SAAA,WACE,IAAMte,EAAerY,KAAKg3B,gBAAkBh3B,KAAKkL,QAAQ0W,OACnD4L,EAAextB,KAAKi3B,mBACpBK,EAAet3B,KAAKkL,QAAQ0W,OAChC4L,EACAxtB,KAAKq3B,mBAMP,GAJIr3B,KAAK02B,gBAAkBlJ,GACzBxtB,KAAK42B,UAGUU,GAAbjf,EAAJ,CACE,IAAM3T,EAAS1E,KAAKw2B,SAASx2B,KAAKw2B,SAASxtB,OAAS,GAEhDhJ,KAAKy2B,gBAAkB/xB,GACzB1E,KAAKu3B,UAAU7yB,OAJnB,CASA,GAAI1E,KAAKy2B,eAAiBpe,EAAYrY,KAAKu2B,SAAS,IAAyB,EAAnBv2B,KAAKu2B,SAAS,GAGtE,OAFAv2B,KAAKy2B,cAAgB,UACrBz2B,KAAKw3B,SAKP,IADA,IACS1uB,EADY9I,KAAKu2B,SAASvtB,OACRF,KAAM,CACR9I,KAAKy2B,gBAAkBz2B,KAAKw2B,SAAS1tB,IACxDuP,GAAarY,KAAKu2B,SAASztB,KACM,oBAAzB9I,KAAKu2B,SAASztB,EAAI,IACtBuP,EAAYrY,KAAKu2B,SAASztB,EAAI,KAGpC9I,KAAKu3B,UAAUv3B,KAAKw2B,SAAS1tB,SAKnCyuB,UAAA,SAAU7yB,GACR1E,KAAKy2B,cAAgB/xB,EAErB1E,KAAKw3B,SAEL,IAAMC,EAAUz3B,KAAK+Q,UAClBnP,MAAM,KACNsZ,IAAI,SAACna,GAAD,OAAiBA,EAAjB,iBAA0C2D,EAA1C,MAAsD3D,EAAtD,UAAwE2D,EAAxE,OAEDgzB,EAAQx3B,EAAE,GAAG0I,MAAM/F,KAAKlC,SAASkI,iBAAiB4uB,EAAQ9B,KAAK,QAEjE+B,EAAMnxB,SAASf,KACjBkyB,EAAMtxB,QAAQkB,IAAmB2V,KAAK3V,IAA0ByH,SAASvJ,IACzEkyB,EAAM3oB,SAASvJ,MAGfkyB,EAAM3oB,SAASvJ,IAGfkyB,EAAMC,QAAQrwB,IAAyB0E,KAAQ1E,GAA/C,KAAsEA,IAAuByH,SAASvJ,IAEtGkyB,EAAMC,QAAQrwB,IAAyB0E,KAAK1E,IAAoBwH,SAASxH,IAAoByH,SAASvJ,KAGxGtF,EAAEF,KAAKs2B,gBAAgBv0B,QAAQqD,GAAM8wB,SAAU,CAC7C7nB,cAAe3J,OAInB8yB,OAAA,WACE,GAAG5uB,MAAM/F,KAAKlC,SAASkI,iBAAiB7I,KAAK+Q,YAC1CF,OAAO,SAACmE,GAAD,OAAUA,EAAKhN,UAAUC,SAASzC,MACzCmY,QAAQ,SAAC3I,GAAD,OAAUA,EAAKhN,UAAUtB,OAAOlB,SAKtCmB,iBAAP,SAAwBpE,GACtB,OAAOvC,KAAK4G,KAAK,WACf,IAAIE,EAAO5G,EAAEF,MAAM8G,KAAK7B,IAQxB,GALK6B,IACHA,EAAO,IAAIuvB,EAAUr2B,KAHW,iBAAXuC,GAAuBA,GAI5CrC,EAAEF,MAAM8G,KAAK7B,GAAU6B,IAGH,iBAAXvE,EAAqB,CAC9B,GAA4B,oBAAjBuE,EAAKvE,GACd,MAAM,IAAIyB,UAAJ,oBAAkCzB,EAAlC,KAERuE,EAAKvE,iDAtMT,MA3EuB,wCA+EvB,OAAO2G,YA8MXhJ,EAAEwI,QAAQxB,GAAG9B,GAAMoC,cAAe,WAIhC,IAHA,IAAMowB,EAAa,GAAGhvB,MAAM/F,KAAKlC,SAASkI,iBAAiBvB,KAGlDwB,EAFgB8uB,EAAW5uB,OAELF,KAAM,CACnC,IAAM+uB,EAAO33B,EAAE03B,EAAW9uB,IAC1ButB,GAAU1vB,iBAAiB9D,KAAKg1B,EAAMA,EAAK/wB,WAU/C5G,EAAEgE,GAAGc,IAAQqxB,GAAU1vB,iBACvBzG,EAAEgE,GAAGc,IAAMmC,YAAckvB,GACzBn2B,EAAEgE,GAAGc,IAAMoC,WAAa,WAEtB,OADAlH,EAAEgE,GAAGc,IAAQG,GACNkxB,GAAU1vB,kBClTnB,IAEM1B,GAAqB,SACrBC,GAAS,IAAgBD,GAEzBE,GAAqBjF,EAAEgE,GAAF,IAErBkB,GAAQ,CACZ+K,KAAI,OAAoBjL,GACxBkL,OAAM,SAAoBlL,GAC1B+K,KAAI,OAAoB/K,GACxBgL,MAAK,QAAoBhL,GACzBK,eAAc,QAAWL,GARA,aAWrBM,GACY,gBADZA,GAEY,SAFZA,GAGY,WAHZA,GAIY,OAJZA,GAKY,OAGZ8B,GACoB,YADpBA,GAEoB,oBAFpBA,GAGoB,UAHpBA,GAIoB,iBAJpBA,GAKoB,kEALpBA,GAMoB,mBANpBA,GAOoB,2BASpBwwB,cACJ,SAAAA,EAAYh3B,GACVd,KAAK0F,SAAW5E,6BAWlBuQ,KAAA,WAAO,IAAAtR,EAAAC,KACL,KAAIA,KAAK0F,SAAS7B,YACd7D,KAAK0F,SAAS7B,WAAWzB,WAAakT,KAAK6W,cAC3CjsB,EAAEF,KAAK0F,UAAUa,SAASf,KAC1BtF,EAAEF,KAAK0F,UAAUa,SAASf,KAH9B,CAOA,IAAId,EACAqzB,EACEC,EAAc93B,EAAEF,KAAK0F,UAAUU,QAAQkB,IAAyB,GAChEvG,EAAWX,EAAKS,uBAAuBb,KAAK0F,UAElD,GAAIsyB,EAAa,CACf,IAAMC,EAAwC,OAAzBD,EAAYrkB,UAA8C,OAAzBqkB,EAAYrkB,SAAoBrM,GAAqBA,GAE3GywB,GADAA,EAAW73B,EAAEg4B,UAAUh4B,EAAE83B,GAAa/a,KAAKgb,KACvBF,EAAS/uB,OAAS,GAGxC,IAAMsgB,EAAYppB,EAAEkF,MAAMA,GAAM+K,KAAM,CACpC9B,cAAerO,KAAK0F,WAGhBujB,EAAY/oB,EAAEkF,MAAMA,GAAM6K,KAAM,CACpC5B,cAAe0pB,IASjB,GANIA,GACF73B,EAAE63B,GAAUh2B,QAAQunB,GAGtBppB,EAAEF,KAAK0F,UAAU3D,QAAQknB,IAErBA,EAAUljB,uBACVujB,EAAUvjB,qBADd,CAKIhF,IACF2D,EAAS/D,SAASQ,cAAcJ,IAGlCf,KAAKu3B,UACHv3B,KAAK0F,SACLsyB,GAGF,IAAMxD,EAAW,WACf,IAAM2D,EAAcj4B,EAAEkF,MAAMA,GAAMgL,OAAQ,CACxC/B,cAAetO,EAAK2F,WAGhB6mB,EAAarsB,EAAEkF,MAAMA,GAAM8K,MAAO,CACtC7B,cAAe0pB,IAGjB73B,EAAE63B,GAAUh2B,QAAQo2B,GACpBj4B,EAAEH,EAAK2F,UAAU3D,QAAQwqB,IAGvB7nB,EACF1E,KAAKu3B,UAAU7yB,EAAQA,EAAOb,WAAY2wB,GAE1CA,SAIJvuB,QAAA,WACE/F,EAAEgG,WAAWlG,KAAK0F,SAAUT,IAC5BjF,KAAK0F,SAAW,QAKlB6xB,UAAA,SAAUz2B,EAASmxB,EAAWjT,GAOX,SAAXwV,IAAW,OAAMxnB,EAAKorB,oBAC1Bt3B,EACAu3B,EACArZ,GAVoC,IAAAhS,EAAAhN,KAKhCq4B,IAJiBpG,GAAqC,OAAvBA,EAAUte,UAA4C,OAAvBse,EAAUte,SAE1EzT,EAAE+xB,GAAWnjB,SAASxH,IADtBpH,EAAE+xB,GAAWhV,KAAK3V,KAGQ,GACxB4K,EAAkB8M,GAAaqZ,GAAUn4B,EAAEm4B,GAAQ9xB,SAASf,IAOlE,GAAI6yB,GAAUnmB,EAAiB,CAC7B,IAAM5Q,EAAqBlB,EAAKiB,iCAAiCg3B,GAEjEn4B,EAAEm4B,GACC/xB,YAAYd,IACZrF,IAAIC,EAAKR,eAAgB40B,GACzBpwB,qBAAqB9C,QAExBkzB,OAIJ4D,oBAAA,SAAoBt3B,EAASu3B,EAAQrZ,GACnC,GAAIqZ,EAAQ,CACVn4B,EAAEm4B,GAAQ/xB,YAAYd,IAEtB,IAAM8yB,EAAgBp4B,EAAEm4B,EAAOx0B,YAAYoZ,KACzC3V,IACA,GAEEgxB,GACFp4B,EAAEo4B,GAAehyB,YAAYd,IAGK,QAAhC6yB,EAAOr3B,aAAa,SACtBq3B,EAAO/vB,aAAa,iBAAiB,GAezC,GAXApI,EAAEY,GAASiO,SAASvJ,IACiB,QAAjC1E,EAAQE,aAAa,SACvBF,EAAQwH,aAAa,iBAAiB,GAGxClI,EAAKyB,OAAOf,GAERA,EAAQkH,UAAUC,SAASzC,KAC7B1E,EAAQkH,UAAUiB,IAAIzD,IAGpB1E,EAAQ+C,YAAc3D,EAAEY,EAAQ+C,YAAY0C,SAASf,IAA0B,CACjF,IAAM+yB,EAAkBr4B,EAAEY,GAASsF,QAAQkB,IAAmB,GAE9D,GAAIixB,EAAiB,CACnB,IAAMC,EAAqB,GAAG5vB,MAAM/F,KAAK01B,EAAgB1vB,iBAAiBvB,KAE1EpH,EAAEs4B,GAAoBzpB,SAASvJ,IAGjC1E,EAAQwH,aAAa,iBAAiB,GAGpC0W,GACFA,OAMGrY,iBAAP,SAAwBpE,GACtB,OAAOvC,KAAK4G,KAAK,WACf,IAAM0L,EAAQpS,EAAEF,MACZ8G,EAAOwL,EAAMxL,KAAK7B,IAOtB,GALK6B,IACHA,EAAO,IAAIgxB,EAAI93B,MACfsS,EAAMxL,KAAK7B,GAAU6B,IAGD,iBAAXvE,EAAqB,CAC9B,GAA4B,oBAAjBuE,EAAKvE,GACd,MAAM,IAAIyB,UAAJ,oBAAkCzB,EAAlC,KAERuE,EAAKvE,iDArKT,MA9CuB,iBA+N3BrC,EAAES,UACCuG,GAAG9B,GAAMG,eAAgB+B,GAAsB,SAAUjD,GACxDA,EAAM4C,iBACN6wB,GAAInxB,iBAAiB9D,KAAK3C,EAAEF,MAAO,UASvCE,EAAEgE,GAAF,IAAa4zB,GAAInxB,iBACjBzG,EAAEgE,GAAF,IAAWiD,YAAc2wB,GACzB53B,EAAEgE,GAAF,IAAWkD,WAAa,WAEtB,OADAlH,EAAEgE,GAAF,IAAaiB,GACN2yB,GAAInxB,kBChPb,IAAM3B,GAAqB,QAErBC,GAAqB,WACrBC,GAAS,IAAgBD,GACzBE,GAAqBjF,EAAEgE,GAAGc,IAE1BI,GAAQ,CACZolB,cAAa,gBAAmBtlB,GAChCiL,KAAI,OAAmBjL,GACvBkL,OAAM,SAAmBlL,GACzB+K,KAAI,OAAmB/K,GACvBgL,MAAK,QAAmBhL,IAGpBM,GACM,OADNA,GAEM,OAFNA,GAGM,OAHNA,GAIM,UAGNiE,GAAc,CAClBooB,UAAY,UACZ4G,SAAY,UACZzG,MAAY,UAGR9oB,GAAU,CACd2oB,WAAY,EACZ4G,UAAY,EACZzG,MAAY,KAGR1qB,GACW,yBASXoxB,cACJ,SAAAA,EAAY53B,EAASyB,GACnBvC,KAAK0F,SAAW5E,EAChBd,KAAKkL,QAAWlL,KAAKmL,WAAW5I,GAChCvC,KAAK+yB,SAAW,KAChB/yB,KAAKmzB,2CAmBP9hB,KAAA,WAAO,IAAAtR,EAAAC,KACCipB,EAAY/oB,EAAEkF,MAAMA,GAAM6K,MAGhC,GADA/P,EAAEF,KAAK0F,UAAU3D,QAAQknB,IACrBA,EAAUljB,qBAAd,CAII/F,KAAKkL,QAAQ2mB,WACf7xB,KAAK0F,SAASsC,UAAUiB,IAAIzD,IAG9B,IAAMgvB,EAAW,WACfz0B,EAAK2F,SAASsC,UAAUtB,OAAOlB,IAC/BzF,EAAK2F,SAASsC,UAAUiB,IAAIzD,IAE5BtF,EAAEH,EAAK2F,UAAU3D,QAAQqD,GAAM8K,OAE3BnQ,EAAKmL,QAAQutB,WACf14B,EAAKgzB,SAAW1yB,WAAW,WACzBN,EAAKqR,QACJrR,EAAKmL,QAAQ8mB,SAOpB,GAHAhyB,KAAK0F,SAASsC,UAAUtB,OAAOlB,IAC/BpF,EAAKyB,OAAO7B,KAAK0F,UACjB1F,KAAK0F,SAASsC,UAAUiB,IAAIzD,IACxBxF,KAAKkL,QAAQ2mB,UAAW,CAC1B,IAAMvwB,EAAqBlB,EAAKiB,iCAAiCrB,KAAK0F,UAEtExF,EAAEF,KAAK0F,UACJvF,IAAIC,EAAKR,eAAgB40B,GACzBpwB,qBAAqB9C,QAExBkzB,QAIJpjB,KAAA,WACE,GAAKpR,KAAK0F,SAASsC,UAAUC,SAASzC,IAAtC,CAIA,IAAM8jB,EAAYppB,EAAEkF,MAAMA,GAAM+K,MAEhCjQ,EAAEF,KAAK0F,UAAU3D,QAAQunB,GACrBA,EAAUvjB,sBAId/F,KAAK24B,aAGP1yB,QAAA,WACEyH,aAAa1N,KAAK+yB,UAClB/yB,KAAK+yB,SAAW,KAEZ/yB,KAAK0F,SAASsC,UAAUC,SAASzC,KACnCxF,KAAK0F,SAASsC,UAAUtB,OAAOlB,IAGjCtF,EAAEF,KAAK0F,UAAUiH,IAAIvH,GAAMolB,eAE3BtqB,EAAEgG,WAAWlG,KAAK0F,SAAUT,IAC5BjF,KAAK0F,SAAW,KAChB1F,KAAKkL,QAAW,QAKlBC,WAAA,SAAW5I,GAaT,OAZAA,EAAMqK,EAAA,GACD1D,GADC,GAEDhJ,EAAEF,KAAK0F,UAAUoB,OAFhB,GAGiB,iBAAXvE,GAAuBA,EAASA,EAAS,IAGrDnC,EAAKiC,gBACH2C,GACAzC,EACAvC,KAAKypB,YAAYhgB,aAGZlH,KAGT4wB,cAAA,WAAgB,IAAAnmB,EAAAhN,KACdE,EAAEF,KAAK0F,UAAUwB,GACf9B,GAAMolB,cACNljB,GACA,WAAA,OAAM0F,EAAKoE,YAIfunB,OAAA,WACmB,SAAXnE,IACJrnB,EAAKzH,SAASsC,UAAUiB,IAAIzD,IAC5BtF,EAAEiN,EAAKzH,UAAU3D,QAAQqD,GAAMgL,QAH1B,IAAAjD,EAAAnN,KAOP,GADAA,KAAK0F,SAASsC,UAAUtB,OAAOlB,IAC3BxF,KAAKkL,QAAQ2mB,UAAW,CAC1B,IAAMvwB,EAAqBlB,EAAKiB,iCAAiCrB,KAAK0F,UAEtExF,EAAEF,KAAK0F,UACJvF,IAAIC,EAAKR,eAAgB40B,GACzBpwB,qBAAqB9C,QAExBkzB,OAMG7tB,iBAAP,SAAwBpE,GACtB,OAAOvC,KAAK4G,KAAK,WACf,IAAMC,EAAW3G,EAAEF,MACf8G,EAAaD,EAASC,KAAK7B,IAQ/B,GALK6B,IACHA,EAAO,IAAI4xB,EAAM14B,KAHgB,iBAAXuC,GAAuBA,GAI7CsE,EAASC,KAAK7B,GAAU6B,IAGJ,iBAAXvE,EAAqB,CAC9B,GAA4B,oBAAjBuE,EAAKvE,GACd,MAAM,IAAIyB,UAAJ,oBAAkCzB,EAAlC,KAGRuE,EAAKvE,GAAQvC,kDAhJjB,MArDuB,4CAyDvB,OAAOyJ,mCAIP,OAAOP,YAoJXhJ,EAAEgE,GAAGc,IAAoB0zB,GAAM/xB,iBAC/BzG,EAAEgE,GAAGc,IAAMmC,YAAcuxB,GACzBx4B,EAAEgE,GAAGc,IAAMoC,WAAc,WAEvB,OADAlH,EAAEgE,GAAGc,IAAQG,GACNuzB,GAAM/xB","sourcesContent":["/**\n * --------------------------------------------------------------------------\n * Bootstrap (v4.4.1): util.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport $ from 'jquery'\n\n/**\n * ------------------------------------------------------------------------\n * Private TransitionEnd Helpers\n * ------------------------------------------------------------------------\n */\n\nconst TRANSITION_END = 'transitionend'\nconst MAX_UID = 1000000\nconst MILLISECONDS_MULTIPLIER = 1000\n\n// Shoutout AngusCroll (https://goo.gl/pxwQGp)\nfunction toType(obj) {\n  return {}.toString.call(obj).match(/\\s([a-z]+)/i)[1].toLowerCase()\n}\n\nfunction getSpecialTransitionEndEvent() {\n  return {\n    bindType: TRANSITION_END,\n    delegateType: TRANSITION_END,\n    handle(event) {\n      if ($(event.target).is(this)) {\n        return event.handleObj.handler.apply(this, arguments) // eslint-disable-line prefer-rest-params\n      }\n      return undefined // eslint-disable-line no-undefined\n    }\n  }\n}\n\nfunction transitionEndEmulator(duration) {\n  let called = false\n\n  $(this).one(Util.TRANSITION_END, () => {\n    called = true\n  })\n\n  setTimeout(() => {\n    if (!called) {\n      Util.triggerTransitionEnd(this)\n    }\n  }, duration)\n\n  return this\n}\n\nfunction setTransitionEndSupport() {\n  $.fn.emulateTransitionEnd = transitionEndEmulator\n  $.event.special[Util.TRANSITION_END] = getSpecialTransitionEndEvent()\n}\n\n/**\n * --------------------------------------------------------------------------\n * Public Util Api\n * --------------------------------------------------------------------------\n */\n\nconst Util = {\n\n  TRANSITION_END: 'bsTransitionEnd',\n\n  getUID(prefix) {\n    do {\n      // eslint-disable-next-line no-bitwise\n      prefix += ~~(Math.random() * MAX_UID) // \"~~\" acts like a faster Math.floor() here\n    } while (document.getElementById(prefix))\n    return prefix\n  },\n\n  getSelectorFromElement(element) {\n    let selector = element.getAttribute('data-target')\n\n    if (!selector || selector === '#') {\n      const hrefAttr = element.getAttribute('href')\n      selector = hrefAttr && hrefAttr !== '#' ? hrefAttr.trim() : ''\n    }\n\n    try {\n      return document.querySelector(selector) ? selector : null\n    } catch (err) {\n      return null\n    }\n  },\n\n  getTransitionDurationFromElement(element) {\n    if (!element) {\n      return 0\n    }\n\n    // Get transition-duration of the element\n    let transitionDuration = $(element).css('transition-duration')\n    let transitionDelay = $(element).css('transition-delay')\n\n    const floatTransitionDuration = parseFloat(transitionDuration)\n    const floatTransitionDelay = parseFloat(transitionDelay)\n\n    // Return 0 if element or transition duration is not found\n    if (!floatTransitionDuration && !floatTransitionDelay) {\n      return 0\n    }\n\n    // If multiple durations are defined, take the first\n    transitionDuration = transitionDuration.split(',')[0]\n    transitionDelay = transitionDelay.split(',')[0]\n\n    return (parseFloat(transitionDuration) + parseFloat(transitionDelay)) * MILLISECONDS_MULTIPLIER\n  },\n\n  reflow(element) {\n    return element.offsetHeight\n  },\n\n  triggerTransitionEnd(element) {\n    $(element).trigger(TRANSITION_END)\n  },\n\n  // TODO: Remove in v5\n  supportsTransitionEnd() {\n    return Boolean(TRANSITION_END)\n  },\n\n  isElement(obj) {\n    return (obj[0] || obj).nodeType\n  },\n\n  typeCheckConfig(componentName, config, configTypes) {\n    for (const property in configTypes) {\n      if (Object.prototype.hasOwnProperty.call(configTypes, property)) {\n        const expectedTypes = configTypes[property]\n        const value         = config[property]\n        const valueType     = value && Util.isElement(value)\n          ? 'element' : toType(value)\n\n        if (!new RegExp(expectedTypes).test(valueType)) {\n          throw new Error(\n            `${componentName.toUpperCase()}: ` +\n            `Option \"${property}\" provided type \"${valueType}\" ` +\n            `but expected type \"${expectedTypes}\".`)\n        }\n      }\n    }\n  },\n\n  findShadowRoot(element) {\n    if (!document.documentElement.attachShadow) {\n      return null\n    }\n\n    // Can find the shadow root otherwise it'll return the document\n    if (typeof element.getRootNode === 'function') {\n      const root = element.getRootNode()\n      return root instanceof ShadowRoot ? root : null\n    }\n\n    if (element instanceof ShadowRoot) {\n      return element\n    }\n\n    // when we don't find a shadow root\n    if (!element.parentNode) {\n      return null\n    }\n\n    return Util.findShadowRoot(element.parentNode)\n  },\n\n  jQueryDetection() {\n    if (typeof $ === 'undefined') {\n      throw new TypeError('Bootstrap\\'s JavaScript requires jQuery. jQuery must be included before Bootstrap\\'s JavaScript.')\n    }\n\n    const version = $.fn.jquery.split(' ')[0].split('.')\n    const minMajor = 1\n    const ltMajor = 2\n    const minMinor = 9\n    const minPatch = 1\n    const maxMajor = 4\n\n    if (version[0] < ltMajor && version[1] < minMinor || version[0] === minMajor && version[1] === minMinor && version[2] < minPatch || version[0] >= maxMajor) {\n      throw new Error('Bootstrap\\'s JavaScript requires at least jQuery v1.9.1 but less than v4.0.0')\n    }\n  }\n}\n\nUtil.jQueryDetection()\nsetTransitionEndSupport()\n\nexport default Util\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap (v4.4.1): alert.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport $ from 'jquery'\nimport Util from './util'\n\n/**\n * ------------------------------------------------------------------------\n * Constants\n * ------------------------------------------------------------------------\n */\n\nconst NAME                = 'alert'\nconst VERSION             = '4.4.1'\nconst DATA_KEY            = 'bs.alert'\nconst EVENT_KEY           = `.${DATA_KEY}`\nconst DATA_API_KEY        = '.data-api'\nconst JQUERY_NO_CONFLICT  = $.fn[NAME]\n\nconst Selector = {\n  DISMISS : '[data-dismiss=\"alert\"]'\n}\n\nconst Event = {\n  CLOSE          : `close${EVENT_KEY}`,\n  CLOSED         : `closed${EVENT_KEY}`,\n  CLICK_DATA_API : `click${EVENT_KEY}${DATA_API_KEY}`\n}\n\nconst ClassName = {\n  ALERT : 'alert',\n  FADE  : 'fade',\n  SHOW  : 'show'\n}\n\n/**\n * ------------------------------------------------------------------------\n * Class Definition\n * ------------------------------------------------------------------------\n */\n\nclass Alert {\n  constructor(element) {\n    this._element = element\n  }\n\n  // Getters\n\n  static get VERSION() {\n    return VERSION\n  }\n\n  // Public\n\n  close(element) {\n    let rootElement = this._element\n    if (element) {\n      rootElement = this._getRootElement(element)\n    }\n\n    const customEvent = this._triggerCloseEvent(rootElement)\n\n    if (customEvent.isDefaultPrevented()) {\n      return\n    }\n\n    this._removeElement(rootElement)\n  }\n\n  dispose() {\n    $.removeData(this._element, DATA_KEY)\n    this._element = null\n  }\n\n  // Private\n\n  _getRootElement(element) {\n    const selector = Util.getSelectorFromElement(element)\n    let parent     = false\n\n    if (selector) {\n      parent = document.querySelector(selector)\n    }\n\n    if (!parent) {\n      parent = $(element).closest(`.${ClassName.ALERT}`)[0]\n    }\n\n    return parent\n  }\n\n  _triggerCloseEvent(element) {\n    const closeEvent = $.Event(Event.CLOSE)\n\n    $(element).trigger(closeEvent)\n    return closeEvent\n  }\n\n  _removeElement(element) {\n    $(element).removeClass(ClassName.SHOW)\n\n    if (!$(element).hasClass(ClassName.FADE)) {\n      this._destroyElement(element)\n      return\n    }\n\n    const transitionDuration = Util.getTransitionDurationFromElement(element)\n\n    $(element)\n      .one(Util.TRANSITION_END, (event) => this._destroyElement(element, event))\n      .emulateTransitionEnd(transitionDuration)\n  }\n\n  _destroyElement(element) {\n    $(element)\n      .detach()\n      .trigger(Event.CLOSED)\n      .remove()\n  }\n\n  // Static\n\n  static _jQueryInterface(config) {\n    return this.each(function () {\n      const $element = $(this)\n      let data       = $element.data(DATA_KEY)\n\n      if (!data) {\n        data = new Alert(this)\n        $element.data(DATA_KEY, data)\n      }\n\n      if (config === 'close') {\n        data[config](this)\n      }\n    })\n  }\n\n  static _handleDismiss(alertInstance) {\n    return function (event) {\n      if (event) {\n        event.preventDefault()\n      }\n\n      alertInstance.close(this)\n    }\n  }\n}\n\n/**\n * ------------------------------------------------------------------------\n * Data Api implementation\n * ------------------------------------------------------------------------\n */\n\n$(document).on(\n  Event.CLICK_DATA_API,\n  Selector.DISMISS,\n  Alert._handleDismiss(new Alert())\n)\n\n/**\n * ------------------------------------------------------------------------\n * jQuery\n * ------------------------------------------------------------------------\n */\n\n$.fn[NAME]             = Alert._jQueryInterface\n$.fn[NAME].Constructor = Alert\n$.fn[NAME].noConflict  = () => {\n  $.fn[NAME] = JQUERY_NO_CONFLICT\n  return Alert._jQueryInterface\n}\n\nexport default Alert\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap (v4.4.1): button.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport $ from 'jquery'\n\n/**\n * ------------------------------------------------------------------------\n * Constants\n * ------------------------------------------------------------------------\n */\n\nconst NAME                = 'button'\nconst VERSION             = '4.4.1'\nconst DATA_KEY            = 'bs.button'\nconst EVENT_KEY           = `.${DATA_KEY}`\nconst DATA_API_KEY        = '.data-api'\nconst JQUERY_NO_CONFLICT  = $.fn[NAME]\n\nconst ClassName = {\n  ACTIVE : 'active',\n  BUTTON : 'btn',\n  FOCUS  : 'focus'\n}\n\nconst Selector = {\n  DATA_TOGGLE_CARROT   : '[data-toggle^=\"button\"]',\n  DATA_TOGGLES         : '[data-toggle=\"buttons\"]',\n  DATA_TOGGLE          : '[data-toggle=\"button\"]',\n  DATA_TOGGLES_BUTTONS : '[data-toggle=\"buttons\"] .btn',\n  INPUT                : 'input:not([type=\"hidden\"])',\n  ACTIVE               : '.active',\n  BUTTON               : '.btn'\n}\n\nconst Event = {\n  CLICK_DATA_API      : `click${EVENT_KEY}${DATA_API_KEY}`,\n  FOCUS_BLUR_DATA_API : `focus${EVENT_KEY}${DATA_API_KEY} ` +\n                          `blur${EVENT_KEY}${DATA_API_KEY}`,\n  LOAD_DATA_API       : `load${EVENT_KEY}${DATA_API_KEY}`\n}\n\n/**\n * ------------------------------------------------------------------------\n * Class Definition\n * ------------------------------------------------------------------------\n */\n\nclass Button {\n  constructor(element) {\n    this._element = element\n  }\n\n  // Getters\n\n  static get VERSION() {\n    return VERSION\n  }\n\n  // Public\n\n  toggle() {\n    let triggerChangeEvent = true\n    let addAriaPressed = true\n    const rootElement = $(this._element).closest(\n      Selector.DATA_TOGGLES\n    )[0]\n\n    if (rootElement) {\n      const input = this._element.querySelector(Selector.INPUT)\n\n      if (input) {\n        if (input.type === 'radio') {\n          if (input.checked &&\n            this._element.classList.contains(ClassName.ACTIVE)) {\n            triggerChangeEvent = false\n          } else {\n            const activeElement = rootElement.querySelector(Selector.ACTIVE)\n\n            if (activeElement) {\n              $(activeElement).removeClass(ClassName.ACTIVE)\n            }\n          }\n        } else if (input.type === 'checkbox') {\n          if (this._element.tagName === 'LABEL' && input.checked === this._element.classList.contains(ClassName.ACTIVE)) {\n            triggerChangeEvent = false\n          }\n        } else {\n          // if it's not a radio button or checkbox don't add a pointless/invalid checked property to the input\n          triggerChangeEvent = false\n        }\n\n        if (triggerChangeEvent) {\n          input.checked = !this._element.classList.contains(ClassName.ACTIVE)\n          $(input).trigger('change')\n        }\n\n        input.focus()\n        addAriaPressed = false\n      }\n    }\n\n    if (!(this._element.hasAttribute('disabled') || this._element.classList.contains('disabled'))) {\n      if (addAriaPressed) {\n        this._element.setAttribute('aria-pressed',\n          !this._element.classList.contains(ClassName.ACTIVE))\n      }\n\n      if (triggerChangeEvent) {\n        $(this._element).toggleClass(ClassName.ACTIVE)\n      }\n    }\n  }\n\n  dispose() {\n    $.removeData(this._element, DATA_KEY)\n    this._element = null\n  }\n\n  // Static\n\n  static _jQueryInterface(config) {\n    return this.each(function () {\n      let data = $(this).data(DATA_KEY)\n\n      if (!data) {\n        data = new Button(this)\n        $(this).data(DATA_KEY, data)\n      }\n\n      if (config === 'toggle') {\n        data[config]()\n      }\n    })\n  }\n}\n\n/**\n * ------------------------------------------------------------------------\n * Data Api implementation\n * ------------------------------------------------------------------------\n */\n\n$(document)\n  .on(Event.CLICK_DATA_API, Selector.DATA_TOGGLE_CARROT, (event) => {\n    let button = event.target\n\n    if (!$(button).hasClass(ClassName.BUTTON)) {\n      button = $(button).closest(Selector.BUTTON)[0]\n    }\n\n    if (!button || button.hasAttribute('disabled') || button.classList.contains('disabled')) {\n      event.preventDefault() // work around Firefox bug #1540995\n    } else {\n      const inputBtn = button.querySelector(Selector.INPUT)\n\n      if (inputBtn && (inputBtn.hasAttribute('disabled') || inputBtn.classList.contains('disabled'))) {\n        event.preventDefault() // work around Firefox bug #1540995\n        return\n      }\n\n      Button._jQueryInterface.call($(button), 'toggle')\n    }\n  })\n  .on(Event.FOCUS_BLUR_DATA_API, Selector.DATA_TOGGLE_CARROT, (event) => {\n    const button = $(event.target).closest(Selector.BUTTON)[0]\n    $(button).toggleClass(ClassName.FOCUS, /^focus(in)?$/.test(event.type))\n  })\n\n$(window).on(Event.LOAD_DATA_API, () => {\n  // ensure correct active class is set to match the controls' actual values/states\n\n  // find all checkboxes/readio buttons inside data-toggle groups\n  let buttons = [].slice.call(document.querySelectorAll(Selector.DATA_TOGGLES_BUTTONS))\n  for (let i = 0, len = buttons.length; i < len; i++) {\n    const button = buttons[i]\n    const input = button.querySelector(Selector.INPUT)\n    if (input.checked || input.hasAttribute('checked')) {\n      button.classList.add(ClassName.ACTIVE)\n    } else {\n      button.classList.remove(ClassName.ACTIVE)\n    }\n  }\n\n  // find all button toggles\n  buttons = [].slice.call(document.querySelectorAll(Selector.DATA_TOGGLE))\n  for (let i = 0, len = buttons.length; i < len; i++) {\n    const button = buttons[i]\n    if (button.getAttribute('aria-pressed') === 'true') {\n      button.classList.add(ClassName.ACTIVE)\n    } else {\n      button.classList.remove(ClassName.ACTIVE)\n    }\n  }\n})\n\n/**\n * ------------------------------------------------------------------------\n * jQuery\n * ------------------------------------------------------------------------\n */\n\n$.fn[NAME] = Button._jQueryInterface\n$.fn[NAME].Constructor = Button\n$.fn[NAME].noConflict = () => {\n  $.fn[NAME] = JQUERY_NO_CONFLICT\n  return Button._jQueryInterface\n}\n\nexport default Button\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap (v4.4.1): carousel.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport $ from 'jquery'\nimport Util from './util'\n\n/**\n * ------------------------------------------------------------------------\n * Constants\n * ------------------------------------------------------------------------\n */\n\nconst NAME                   = 'carousel'\nconst VERSION                = '4.4.1'\nconst DATA_KEY               = 'bs.carousel'\nconst EVENT_KEY              = `.${DATA_KEY}`\nconst DATA_API_KEY           = '.data-api'\nconst JQUERY_NO_CONFLICT     = $.fn[NAME]\nconst ARROW_LEFT_KEYCODE     = 37 // KeyboardEvent.which value for left arrow key\nconst ARROW_RIGHT_KEYCODE    = 39 // KeyboardEvent.which value for right arrow key\nconst TOUCHEVENT_COMPAT_WAIT = 500 // Time for mouse compat events to fire after touch\nconst SWIPE_THRESHOLD        = 40\n\nconst Default = {\n  interval : 5000,\n  keyboard : true,\n  slide    : false,\n  pause    : 'hover',\n  wrap     : true,\n  touch    : true\n}\n\nconst DefaultType = {\n  interval : '(number|boolean)',\n  keyboard : 'boolean',\n  slide    : '(boolean|string)',\n  pause    : '(string|boolean)',\n  wrap     : 'boolean',\n  touch    : 'boolean'\n}\n\nconst Direction = {\n  NEXT     : 'next',\n  PREV     : 'prev',\n  LEFT     : 'left',\n  RIGHT    : 'right'\n}\n\nconst Event = {\n  SLIDE          : `slide${EVENT_KEY}`,\n  SLID           : `slid${EVENT_KEY}`,\n  KEYDOWN        : `keydown${EVENT_KEY}`,\n  MOUSEENTER     : `mouseenter${EVENT_KEY}`,\n  MOUSELEAVE     : `mouseleave${EVENT_KEY}`,\n  TOUCHSTART     : `touchstart${EVENT_KEY}`,\n  TOUCHMOVE      : `touchmove${EVENT_KEY}`,\n  TOUCHEND       : `touchend${EVENT_KEY}`,\n  POINTERDOWN    : `pointerdown${EVENT_KEY}`,\n  POINTERUP      : `pointerup${EVENT_KEY}`,\n  DRAG_START     : `dragstart${EVENT_KEY}`,\n  LOAD_DATA_API  : `load${EVENT_KEY}${DATA_API_KEY}`,\n  CLICK_DATA_API : `click${EVENT_KEY}${DATA_API_KEY}`\n}\n\nconst ClassName = {\n  CAROUSEL      : 'carousel',\n  ACTIVE        : 'active',\n  SLIDE         : 'slide',\n  RIGHT         : 'carousel-item-right',\n  LEFT          : 'carousel-item-left',\n  NEXT          : 'carousel-item-next',\n  PREV          : 'carousel-item-prev',\n  ITEM          : 'carousel-item',\n  POINTER_EVENT : 'pointer-event'\n}\n\nconst Selector = {\n  ACTIVE      : '.active',\n  ACTIVE_ITEM : '.active.carousel-item',\n  ITEM        : '.carousel-item',\n  ITEM_IMG    : '.carousel-item img',\n  NEXT_PREV   : '.carousel-item-next, .carousel-item-prev',\n  INDICATORS  : '.carousel-indicators',\n  DATA_SLIDE  : '[data-slide], [data-slide-to]',\n  DATA_RIDE   : '[data-ride=\"carousel\"]'\n}\n\nconst PointerType = {\n  TOUCH : 'touch',\n  PEN   : 'pen'\n}\n\n/**\n * ------------------------------------------------------------------------\n * Class Definition\n * ------------------------------------------------------------------------\n */\nclass Carousel {\n  constructor(element, config) {\n    this._items         = null\n    this._interval      = null\n    this._activeElement = null\n    this._isPaused      = false\n    this._isSliding     = false\n    this.touchTimeout   = null\n    this.touchStartX    = 0\n    this.touchDeltaX    = 0\n\n    this._config            = this._getConfig(config)\n    this._element           = element\n    this._indicatorsElement = this._element.querySelector(Selector.INDICATORS)\n    this._touchSupported    = 'ontouchstart' in document.documentElement || navigator.maxTouchPoints > 0\n    this._pointerEvent      = Boolean(window.PointerEvent || window.MSPointerEvent)\n\n    this._addEventListeners()\n  }\n\n  // Getters\n\n  static get VERSION() {\n    return VERSION\n  }\n\n  static get Default() {\n    return Default\n  }\n\n  // Public\n\n  next() {\n    if (!this._isSliding) {\n      this._slide(Direction.NEXT)\n    }\n  }\n\n  nextWhenVisible() {\n    // Don't call next when the page isn't visible\n    // or the carousel or its parent isn't visible\n    if (!document.hidden &&\n      ($(this._element).is(':visible') && $(this._element).css('visibility') !== 'hidden')) {\n      this.next()\n    }\n  }\n\n  prev() {\n    if (!this._isSliding) {\n      this._slide(Direction.PREV)\n    }\n  }\n\n  pause(event) {\n    if (!event) {\n      this._isPaused = true\n    }\n\n    if (this._element.querySelector(Selector.NEXT_PREV)) {\n      Util.triggerTransitionEnd(this._element)\n      this.cycle(true)\n    }\n\n    clearInterval(this._interval)\n    this._interval = null\n  }\n\n  cycle(event) {\n    if (!event) {\n      this._isPaused = false\n    }\n\n    if (this._interval) {\n      clearInterval(this._interval)\n      this._interval = null\n    }\n\n    if (this._config.interval && !this._isPaused) {\n      this._interval = setInterval(\n        (document.visibilityState ? this.nextWhenVisible : this.next).bind(this),\n        this._config.interval\n      )\n    }\n  }\n\n  to(index) {\n    this._activeElement = this._element.querySelector(Selector.ACTIVE_ITEM)\n\n    const activeIndex = this._getItemIndex(this._activeElement)\n\n    if (index > this._items.length - 1 || index < 0) {\n      return\n    }\n\n    if (this._isSliding) {\n      $(this._element).one(Event.SLID, () => this.to(index))\n      return\n    }\n\n    if (activeIndex === index) {\n      this.pause()\n      this.cycle()\n      return\n    }\n\n    const direction = index > activeIndex\n      ? Direction.NEXT\n      : Direction.PREV\n\n    this._slide(direction, this._items[index])\n  }\n\n  dispose() {\n    $(this._element).off(EVENT_KEY)\n    $.removeData(this._element, DATA_KEY)\n\n    this._items             = null\n    this._config            = null\n    this._element           = null\n    this._interval          = null\n    this._isPaused          = null\n    this._isSliding         = null\n    this._activeElement     = null\n    this._indicatorsElement = null\n  }\n\n  // Private\n\n  _getConfig(config) {\n    config = {\n      ...Default,\n      ...config\n    }\n    Util.typeCheckConfig(NAME, config, DefaultType)\n    return config\n  }\n\n  _handleSwipe() {\n    const absDeltax = Math.abs(this.touchDeltaX)\n\n    if (absDeltax <= SWIPE_THRESHOLD) {\n      return\n    }\n\n    const direction = absDeltax / this.touchDeltaX\n\n    this.touchDeltaX = 0\n\n    // swipe left\n    if (direction > 0) {\n      this.prev()\n    }\n\n    // swipe right\n    if (direction < 0) {\n      this.next()\n    }\n  }\n\n  _addEventListeners() {\n    if (this._config.keyboard) {\n      $(this._element)\n        .on(Event.KEYDOWN, (event) => this._keydown(event))\n    }\n\n    if (this._config.pause === 'hover') {\n      $(this._element)\n        .on(Event.MOUSEENTER, (event) => this.pause(event))\n        .on(Event.MOUSELEAVE, (event) => this.cycle(event))\n    }\n\n    if (this._config.touch) {\n      this._addTouchEventListeners()\n    }\n  }\n\n  _addTouchEventListeners() {\n    if (!this._touchSupported) {\n      return\n    }\n\n    const start = (event) => {\n      if (this._pointerEvent && PointerType[event.originalEvent.pointerType.toUpperCase()]) {\n        this.touchStartX = event.originalEvent.clientX\n      } else if (!this._pointerEvent) {\n        this.touchStartX = event.originalEvent.touches[0].clientX\n      }\n    }\n\n    const move = (event) => {\n      // ensure swiping with one touch and not pinching\n      if (event.originalEvent.touches && event.originalEvent.touches.length > 1) {\n        this.touchDeltaX = 0\n      } else {\n        this.touchDeltaX = event.originalEvent.touches[0].clientX - this.touchStartX\n      }\n    }\n\n    const end = (event) => {\n      if (this._pointerEvent && PointerType[event.originalEvent.pointerType.toUpperCase()]) {\n        this.touchDeltaX = event.originalEvent.clientX - this.touchStartX\n      }\n\n      this._handleSwipe()\n      if (this._config.pause === 'hover') {\n        // If it's a touch-enabled device, mouseenter/leave are fired as\n        // part of the mouse compatibility events on first tap - the carousel\n        // would stop cycling until user tapped out of it;\n        // here, we listen for touchend, explicitly pause the carousel\n        // (as if it's the second time we tap on it, mouseenter compat event\n        // is NOT fired) and after a timeout (to allow for mouse compatibility\n        // events to fire) we explicitly restart cycling\n\n        this.pause()\n        if (this.touchTimeout) {\n          clearTimeout(this.touchTimeout)\n        }\n        this.touchTimeout = setTimeout((event) => this.cycle(event), TOUCHEVENT_COMPAT_WAIT + this._config.interval)\n      }\n    }\n\n    $(this._element.querySelectorAll(Selector.ITEM_IMG)).on(Event.DRAG_START, (e) => e.preventDefault())\n    if (this._pointerEvent) {\n      $(this._element).on(Event.POINTERDOWN, (event) => start(event))\n      $(this._element).on(Event.POINTERUP, (event) => end(event))\n\n      this._element.classList.add(ClassName.POINTER_EVENT)\n    } else {\n      $(this._element).on(Event.TOUCHSTART, (event) => start(event))\n      $(this._element).on(Event.TOUCHMOVE, (event) => move(event))\n      $(this._element).on(Event.TOUCHEND, (event) => end(event))\n    }\n  }\n\n  _keydown(event) {\n    if (/input|textarea/i.test(event.target.tagName)) {\n      return\n    }\n\n    switch (event.which) {\n      case ARROW_LEFT_KEYCODE:\n        event.preventDefault()\n        this.prev()\n        break\n      case ARROW_RIGHT_KEYCODE:\n        event.preventDefault()\n        this.next()\n        break\n      default:\n    }\n  }\n\n  _getItemIndex(element) {\n    this._items = element && element.parentNode\n      ? [].slice.call(element.parentNode.querySelectorAll(Selector.ITEM))\n      : []\n    return this._items.indexOf(element)\n  }\n\n  _getItemByDirection(direction, activeElement) {\n    const isNextDirection = direction === Direction.NEXT\n    const isPrevDirection = direction === Direction.PREV\n    const activeIndex     = this._getItemIndex(activeElement)\n    const lastItemIndex   = this._items.length - 1\n    const isGoingToWrap   = isPrevDirection && activeIndex === 0 ||\n                            isNextDirection && activeIndex === lastItemIndex\n\n    if (isGoingToWrap && !this._config.wrap) {\n      return activeElement\n    }\n\n    const delta     = direction === Direction.PREV ? -1 : 1\n    const itemIndex = (activeIndex + delta) % this._items.length\n\n    return itemIndex === -1\n      ? this._items[this._items.length - 1] : this._items[itemIndex]\n  }\n\n  _triggerSlideEvent(relatedTarget, eventDirectionName) {\n    const targetIndex = this._getItemIndex(relatedTarget)\n    const fromIndex = this._getItemIndex(this._element.querySelector(Selector.ACTIVE_ITEM))\n    const slideEvent = $.Event(Event.SLIDE, {\n      relatedTarget,\n      direction: eventDirectionName,\n      from: fromIndex,\n      to: targetIndex\n    })\n\n    $(this._element).trigger(slideEvent)\n\n    return slideEvent\n  }\n\n  _setActiveIndicatorElement(element) {\n    if (this._indicatorsElement) {\n      const indicators = [].slice.call(this._indicatorsElement.querySelectorAll(Selector.ACTIVE))\n      $(indicators)\n        .removeClass(ClassName.ACTIVE)\n\n      const nextIndicator = this._indicatorsElement.children[\n        this._getItemIndex(element)\n      ]\n\n      if (nextIndicator) {\n        $(nextIndicator).addClass(ClassName.ACTIVE)\n      }\n    }\n  }\n\n  _slide(direction, element) {\n    const activeElement = this._element.querySelector(Selector.ACTIVE_ITEM)\n    const activeElementIndex = this._getItemIndex(activeElement)\n    const nextElement   = element || activeElement &&\n      this._getItemByDirection(direction, activeElement)\n    const nextElementIndex = this._getItemIndex(nextElement)\n    const isCycling = Boolean(this._interval)\n\n    let directionalClassName\n    let orderClassName\n    let eventDirectionName\n\n    if (direction === Direction.NEXT) {\n      directionalClassName = ClassName.LEFT\n      orderClassName = ClassName.NEXT\n      eventDirectionName = Direction.LEFT\n    } else {\n      directionalClassName = ClassName.RIGHT\n      orderClassName = ClassName.PREV\n      eventDirectionName = Direction.RIGHT\n    }\n\n    if (nextElement && $(nextElement).hasClass(ClassName.ACTIVE)) {\n      this._isSliding = false\n      return\n    }\n\n    const slideEvent = this._triggerSlideEvent(nextElement, eventDirectionName)\n    if (slideEvent.isDefaultPrevented()) {\n      return\n    }\n\n    if (!activeElement || !nextElement) {\n      // Some weirdness is happening, so we bail\n      return\n    }\n\n    this._isSliding = true\n\n    if (isCycling) {\n      this.pause()\n    }\n\n    this._setActiveIndicatorElement(nextElement)\n\n    const slidEvent = $.Event(Event.SLID, {\n      relatedTarget: nextElement,\n      direction: eventDirectionName,\n      from: activeElementIndex,\n      to: nextElementIndex\n    })\n\n    if ($(this._element).hasClass(ClassName.SLIDE)) {\n      $(nextElement).addClass(orderClassName)\n\n      Util.reflow(nextElement)\n\n      $(activeElement).addClass(directionalClassName)\n      $(nextElement).addClass(directionalClassName)\n\n      const nextElementInterval = parseInt(nextElement.getAttribute('data-interval'), 10)\n      if (nextElementInterval) {\n        this._config.defaultInterval = this._config.defaultInterval || this._config.interval\n        this._config.interval = nextElementInterval\n      } else {\n        this._config.interval = this._config.defaultInterval || this._config.interval\n      }\n\n      const transitionDuration = Util.getTransitionDurationFromElement(activeElement)\n\n      $(activeElement)\n        .one(Util.TRANSITION_END, () => {\n          $(nextElement)\n            .removeClass(`${directionalClassName} ${orderClassName}`)\n            .addClass(ClassName.ACTIVE)\n\n          $(activeElement).removeClass(`${ClassName.ACTIVE} ${orderClassName} ${directionalClassName}`)\n\n          this._isSliding = false\n\n          setTimeout(() => $(this._element).trigger(slidEvent), 0)\n        })\n        .emulateTransitionEnd(transitionDuration)\n    } else {\n      $(activeElement).removeClass(ClassName.ACTIVE)\n      $(nextElement).addClass(ClassName.ACTIVE)\n\n      this._isSliding = false\n      $(this._element).trigger(slidEvent)\n    }\n\n    if (isCycling) {\n      this.cycle()\n    }\n  }\n\n  // Static\n\n  static _jQueryInterface(config) {\n    return this.each(function () {\n      let data = $(this).data(DATA_KEY)\n      let _config = {\n        ...Default,\n        ...$(this).data()\n      }\n\n      if (typeof config === 'object') {\n        _config = {\n          ..._config,\n          ...config\n        }\n      }\n\n      const action = typeof config === 'string' ? config : _config.slide\n\n      if (!data) {\n        data = new Carousel(this, _config)\n        $(this).data(DATA_KEY, data)\n      }\n\n      if (typeof config === 'number') {\n        data.to(config)\n      } else if (typeof action === 'string') {\n        if (typeof data[action] === 'undefined') {\n          throw new TypeError(`No method named \"${action}\"`)\n        }\n        data[action]()\n      } else if (_config.interval && _config.ride) {\n        data.pause()\n        data.cycle()\n      }\n    })\n  }\n\n  static _dataApiClickHandler(event) {\n    const selector = Util.getSelectorFromElement(this)\n\n    if (!selector) {\n      return\n    }\n\n    const target = $(selector)[0]\n\n    if (!target || !$(target).hasClass(ClassName.CAROUSEL)) {\n      return\n    }\n\n    const config = {\n      ...$(target).data(),\n      ...$(this).data()\n    }\n    const slideIndex = this.getAttribute('data-slide-to')\n\n    if (slideIndex) {\n      config.interval = false\n    }\n\n    Carousel._jQueryInterface.call($(target), config)\n\n    if (slideIndex) {\n      $(target).data(DATA_KEY).to(slideIndex)\n    }\n\n    event.preventDefault()\n  }\n}\n\n/**\n * ------------------------------------------------------------------------\n * Data Api implementation\n * ------------------------------------------------------------------------\n */\n\n$(document)\n  .on(Event.CLICK_DATA_API, Selector.DATA_SLIDE, Carousel._dataApiClickHandler)\n\n$(window).on(Event.LOAD_DATA_API, () => {\n  const carousels = [].slice.call(document.querySelectorAll(Selector.DATA_RIDE))\n  for (let i = 0, len = carousels.length; i < len; i++) {\n    const $carousel = $(carousels[i])\n    Carousel._jQueryInterface.call($carousel, $carousel.data())\n  }\n})\n\n/**\n * ------------------------------------------------------------------------\n * jQuery\n * ------------------------------------------------------------------------\n */\n\n$.fn[NAME] = Carousel._jQueryInterface\n$.fn[NAME].Constructor = Carousel\n$.fn[NAME].noConflict = () => {\n  $.fn[NAME] = JQUERY_NO_CONFLICT\n  return Carousel._jQueryInterface\n}\n\nexport default Carousel\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap (v4.4.1): collapse.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport $ from 'jquery'\nimport Util from './util'\n\n/**\n * ------------------------------------------------------------------------\n * Constants\n * ------------------------------------------------------------------------\n */\n\nconst NAME                = 'collapse'\nconst VERSION             = '4.4.1'\nconst DATA_KEY            = 'bs.collapse'\nconst EVENT_KEY           = `.${DATA_KEY}`\nconst DATA_API_KEY        = '.data-api'\nconst JQUERY_NO_CONFLICT  = $.fn[NAME]\n\nconst Default = {\n  toggle : true,\n  parent : ''\n}\n\nconst DefaultType = {\n  toggle : 'boolean',\n  parent : '(string|element)'\n}\n\nconst Event = {\n  SHOW           : `show${EVENT_KEY}`,\n  SHOWN          : `shown${EVENT_KEY}`,\n  HIDE           : `hide${EVENT_KEY}`,\n  HIDDEN         : `hidden${EVENT_KEY}`,\n  CLICK_DATA_API : `click${EVENT_KEY}${DATA_API_KEY}`\n}\n\nconst ClassName = {\n  SHOW       : 'show',\n  COLLAPSE   : 'collapse',\n  COLLAPSING : 'collapsing',\n  COLLAPSED  : 'collapsed'\n}\n\nconst Dimension = {\n  WIDTH  : 'width',\n  HEIGHT : 'height'\n}\n\nconst Selector = {\n  ACTIVES     : '.show, .collapsing',\n  DATA_TOGGLE : '[data-toggle=\"collapse\"]'\n}\n\n/**\n * ------------------------------------------------------------------------\n * Class Definition\n * ------------------------------------------------------------------------\n */\n\nclass Collapse {\n  constructor(element, config) {\n    this._isTransitioning = false\n    this._element         = element\n    this._config          = this._getConfig(config)\n    this._triggerArray    = [].slice.call(document.querySelectorAll(\n      `[data-toggle=\"collapse\"][href=\"#${element.id}\"],` +\n      `[data-toggle=\"collapse\"][data-target=\"#${element.id}\"]`\n    ))\n\n    const toggleList = [].slice.call(document.querySelectorAll(Selector.DATA_TOGGLE))\n    for (let i = 0, len = toggleList.length; i < len; i++) {\n      const elem = toggleList[i]\n      const selector = Util.getSelectorFromElement(elem)\n      const filterElement = [].slice.call(document.querySelectorAll(selector))\n        .filter((foundElem) => foundElem === element)\n\n      if (selector !== null && filterElement.length > 0) {\n        this._selector = selector\n        this._triggerArray.push(elem)\n      }\n    }\n\n    this._parent = this._config.parent ? this._getParent() : null\n\n    if (!this._config.parent) {\n      this._addAriaAndCollapsedClass(this._element, this._triggerArray)\n    }\n\n    if (this._config.toggle) {\n      this.toggle()\n    }\n  }\n\n  // Getters\n\n  static get VERSION() {\n    return VERSION\n  }\n\n  static get Default() {\n    return Default\n  }\n\n  // Public\n\n  toggle() {\n    if ($(this._element).hasClass(ClassName.SHOW)) {\n      this.hide()\n    } else {\n      this.show()\n    }\n  }\n\n  show() {\n    if (this._isTransitioning ||\n      $(this._element).hasClass(ClassName.SHOW)) {\n      return\n    }\n\n    let actives\n    let activesData\n\n    if (this._parent) {\n      actives = [].slice.call(this._parent.querySelectorAll(Selector.ACTIVES))\n        .filter((elem) => {\n          if (typeof this._config.parent === 'string') {\n            return elem.getAttribute('data-parent') === this._config.parent\n          }\n\n          return elem.classList.contains(ClassName.COLLAPSE)\n        })\n\n      if (actives.length === 0) {\n        actives = null\n      }\n    }\n\n    if (actives) {\n      activesData = $(actives).not(this._selector).data(DATA_KEY)\n      if (activesData && activesData._isTransitioning) {\n        return\n      }\n    }\n\n    const startEvent = $.Event(Event.SHOW)\n    $(this._element).trigger(startEvent)\n    if (startEvent.isDefaultPrevented()) {\n      return\n    }\n\n    if (actives) {\n      Collapse._jQueryInterface.call($(actives).not(this._selector), 'hide')\n      if (!activesData) {\n        $(actives).data(DATA_KEY, null)\n      }\n    }\n\n    const dimension = this._getDimension()\n\n    $(this._element)\n      .removeClass(ClassName.COLLAPSE)\n      .addClass(ClassName.COLLAPSING)\n\n    this._element.style[dimension] = 0\n\n    if (this._triggerArray.length) {\n      $(this._triggerArray)\n        .removeClass(ClassName.COLLAPSED)\n        .attr('aria-expanded', true)\n    }\n\n    this.setTransitioning(true)\n\n    const complete = () => {\n      $(this._element)\n        .removeClass(ClassName.COLLAPSING)\n        .addClass(ClassName.COLLAPSE)\n        .addClass(ClassName.SHOW)\n\n      this._element.style[dimension] = ''\n\n      this.setTransitioning(false)\n\n      $(this._element).trigger(Event.SHOWN)\n    }\n\n    const capitalizedDimension = dimension[0].toUpperCase() + dimension.slice(1)\n    const scrollSize = `scroll${capitalizedDimension}`\n    const transitionDuration = Util.getTransitionDurationFromElement(this._element)\n\n    $(this._element)\n      .one(Util.TRANSITION_END, complete)\n      .emulateTransitionEnd(transitionDuration)\n\n    this._element.style[dimension] = `${this._element[scrollSize]}px`\n  }\n\n  hide() {\n    if (this._isTransitioning ||\n      !$(this._element).hasClass(ClassName.SHOW)) {\n      return\n    }\n\n    const startEvent = $.Event(Event.HIDE)\n    $(this._element).trigger(startEvent)\n    if (startEvent.isDefaultPrevented()) {\n      return\n    }\n\n    const dimension = this._getDimension()\n\n    this._element.style[dimension] = `${this._element.getBoundingClientRect()[dimension]}px`\n\n    Util.reflow(this._element)\n\n    $(this._element)\n      .addClass(ClassName.COLLAPSING)\n      .removeClass(ClassName.COLLAPSE)\n      .removeClass(ClassName.SHOW)\n\n    const triggerArrayLength = this._triggerArray.length\n    if (triggerArrayLength > 0) {\n      for (let i = 0; i < triggerArrayLength; i++) {\n        const trigger = this._triggerArray[i]\n        const selector = Util.getSelectorFromElement(trigger)\n\n        if (selector !== null) {\n          const $elem = $([].slice.call(document.querySelectorAll(selector)))\n          if (!$elem.hasClass(ClassName.SHOW)) {\n            $(trigger).addClass(ClassName.COLLAPSED)\n              .attr('aria-expanded', false)\n          }\n        }\n      }\n    }\n\n    this.setTransitioning(true)\n\n    const complete = () => {\n      this.setTransitioning(false)\n      $(this._element)\n        .removeClass(ClassName.COLLAPSING)\n        .addClass(ClassName.COLLAPSE)\n        .trigger(Event.HIDDEN)\n    }\n\n    this._element.style[dimension] = ''\n    const transitionDuration = Util.getTransitionDurationFromElement(this._element)\n\n    $(this._element)\n      .one(Util.TRANSITION_END, complete)\n      .emulateTransitionEnd(transitionDuration)\n  }\n\n  setTransitioning(isTransitioning) {\n    this._isTransitioning = isTransitioning\n  }\n\n  dispose() {\n    $.removeData(this._element, DATA_KEY)\n\n    this._config          = null\n    this._parent          = null\n    this._element         = null\n    this._triggerArray    = null\n    this._isTransitioning = null\n  }\n\n  // Private\n\n  _getConfig(config) {\n    config = {\n      ...Default,\n      ...config\n    }\n    config.toggle = Boolean(config.toggle) // Coerce string values\n    Util.typeCheckConfig(NAME, config, DefaultType)\n    return config\n  }\n\n  _getDimension() {\n    const hasWidth = $(this._element).hasClass(Dimension.WIDTH)\n    return hasWidth ? Dimension.WIDTH : Dimension.HEIGHT\n  }\n\n  _getParent() {\n    let parent\n\n    if (Util.isElement(this._config.parent)) {\n      parent = this._config.parent\n\n      // It's a jQuery object\n      if (typeof this._config.parent.jquery !== 'undefined') {\n        parent = this._config.parent[0]\n      }\n    } else {\n      parent = document.querySelector(this._config.parent)\n    }\n\n    const selector =\n      `[data-toggle=\"collapse\"][data-parent=\"${this._config.parent}\"]`\n\n    const children = [].slice.call(parent.querySelectorAll(selector))\n    $(children).each((i, element) => {\n      this._addAriaAndCollapsedClass(\n        Collapse._getTargetFromElement(element),\n        [element]\n      )\n    })\n\n    return parent\n  }\n\n  _addAriaAndCollapsedClass(element, triggerArray) {\n    const isOpen = $(element).hasClass(ClassName.SHOW)\n\n    if (triggerArray.length) {\n      $(triggerArray)\n        .toggleClass(ClassName.COLLAPSED, !isOpen)\n        .attr('aria-expanded', isOpen)\n    }\n  }\n\n  // Static\n\n  static _getTargetFromElement(element) {\n    const selector = Util.getSelectorFromElement(element)\n    return selector ? document.querySelector(selector) : null\n  }\n\n  static _jQueryInterface(config) {\n    return this.each(function () {\n      const $this   = $(this)\n      let data      = $this.data(DATA_KEY)\n      const _config = {\n        ...Default,\n        ...$this.data(),\n        ...typeof config === 'object' && config ? config : {}\n      }\n\n      if (!data && _config.toggle && /show|hide/.test(config)) {\n        _config.toggle = false\n      }\n\n      if (!data) {\n        data = new Collapse(this, _config)\n        $this.data(DATA_KEY, data)\n      }\n\n      if (typeof config === 'string') {\n        if (typeof data[config] === 'undefined') {\n          throw new TypeError(`No method named \"${config}\"`)\n        }\n        data[config]()\n      }\n    })\n  }\n}\n\n/**\n * ------------------------------------------------------------------------\n * Data Api implementation\n * ------------------------------------------------------------------------\n */\n\n$(document).on(Event.CLICK_DATA_API, Selector.DATA_TOGGLE, function (event) {\n  // preventDefault only for <a> elements (which change the URL) not inside the collapsible element\n  if (event.currentTarget.tagName === 'A') {\n    event.preventDefault()\n  }\n\n  const $trigger = $(this)\n  const selector = Util.getSelectorFromElement(this)\n  const selectors = [].slice.call(document.querySelectorAll(selector))\n\n  $(selectors).each(function () {\n    const $target = $(this)\n    const data    = $target.data(DATA_KEY)\n    const config  = data ? 'toggle' : $trigger.data()\n    Collapse._jQueryInterface.call($target, config)\n  })\n})\n\n/**\n * ------------------------------------------------------------------------\n * jQuery\n * ------------------------------------------------------------------------\n */\n\n$.fn[NAME] = Collapse._jQueryInterface\n$.fn[NAME].Constructor = Collapse\n$.fn[NAME].noConflict = () => {\n  $.fn[NAME] = JQUERY_NO_CONFLICT\n  return Collapse._jQueryInterface\n}\n\nexport default Collapse\n","/**!\n * @fileOverview Kickass library to create and place poppers near their reference elements.\n * @version 1.16.0\n * @license\n * Copyright (c) 2016 Federico Zivolo and contributors\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all\n * copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n * SOFTWARE.\n */\nvar isBrowser = typeof window !== 'undefined' && typeof document !== 'undefined' && typeof navigator !== 'undefined';\n\nvar timeoutDuration = function () {\n  var longerTimeoutBrowsers = ['Edge', 'Trident', 'Firefox'];\n  for (var i = 0; i < longerTimeoutBrowsers.length; i += 1) {\n    if (isBrowser && navigator.userAgent.indexOf(longerTimeoutBrowsers[i]) >= 0) {\n      return 1;\n    }\n  }\n  return 0;\n}();\n\nfunction microtaskDebounce(fn) {\n  var called = false;\n  return function () {\n    if (called) {\n      return;\n    }\n    called = true;\n    window.Promise.resolve().then(function () {\n      called = false;\n      fn();\n    });\n  };\n}\n\nfunction taskDebounce(fn) {\n  var scheduled = false;\n  return function () {\n    if (!scheduled) {\n      scheduled = true;\n      setTimeout(function () {\n        scheduled = false;\n        fn();\n      }, timeoutDuration);\n    }\n  };\n}\n\nvar supportsMicroTasks = isBrowser && window.Promise;\n\n/**\n* Create a debounced version of a method, that's asynchronously deferred\n* but called in the minimum time possible.\n*\n* @method\n* @memberof Popper.Utils\n* @argument {Function} fn\n* @returns {Function}\n*/\nvar debounce = supportsMicroTasks ? microtaskDebounce : taskDebounce;\n\n/**\n * Check if the given variable is a function\n * @method\n * @memberof Popper.Utils\n * @argument {Any} functionToCheck - variable to check\n * @returns {Boolean} answer to: is a function?\n */\nfunction isFunction(functionToCheck) {\n  var getType = {};\n  return functionToCheck && getType.toString.call(functionToCheck) === '[object Function]';\n}\n\n/**\n * Get CSS computed property of the given element\n * @method\n * @memberof Popper.Utils\n * @argument {Eement} element\n * @argument {String} property\n */\nfunction getStyleComputedProperty(element, property) {\n  if (element.nodeType !== 1) {\n    return [];\n  }\n  // NOTE: 1 DOM access here\n  var window = element.ownerDocument.defaultView;\n  var css = window.getComputedStyle(element, null);\n  return property ? css[property] : css;\n}\n\n/**\n * Returns the parentNode or the host of the element\n * @method\n * @memberof Popper.Utils\n * @argument {Element} element\n * @returns {Element} parent\n */\nfunction getParentNode(element) {\n  if (element.nodeName === 'HTML') {\n    return element;\n  }\n  return element.parentNode || element.host;\n}\n\n/**\n * Returns the scrolling parent of the given element\n * @method\n * @memberof Popper.Utils\n * @argument {Element} element\n * @returns {Element} scroll parent\n */\nfunction getScrollParent(element) {\n  // Return body, `getScroll` will take care to get the correct `scrollTop` from it\n  if (!element) {\n    return document.body;\n  }\n\n  switch (element.nodeName) {\n    case 'HTML':\n    case 'BODY':\n      return element.ownerDocument.body;\n    case '#document':\n      return element.body;\n  }\n\n  // Firefox want us to check `-x` and `-y` variations as well\n\n  var _getStyleComputedProp = getStyleComputedProperty(element),\n      overflow = _getStyleComputedProp.overflow,\n      overflowX = _getStyleComputedProp.overflowX,\n      overflowY = _getStyleComputedProp.overflowY;\n\n  if (/(auto|scroll|overlay)/.test(overflow + overflowY + overflowX)) {\n    return element;\n  }\n\n  return getScrollParent(getParentNode(element));\n}\n\n/**\n * Returns the reference node of the reference object, or the reference object itself.\n * @method\n * @memberof Popper.Utils\n * @param {Element|Object} reference - the reference element (the popper will be relative to this)\n * @returns {Element} parent\n */\nfunction getReferenceNode(reference) {\n  return reference && reference.referenceNode ? reference.referenceNode : reference;\n}\n\nvar isIE11 = isBrowser && !!(window.MSInputMethodContext && document.documentMode);\nvar isIE10 = isBrowser && /MSIE 10/.test(navigator.userAgent);\n\n/**\n * Determines if the browser is Internet Explorer\n * @method\n * @memberof Popper.Utils\n * @param {Number} version to check\n * @returns {Boolean} isIE\n */\nfunction isIE(version) {\n  if (version === 11) {\n    return isIE11;\n  }\n  if (version === 10) {\n    return isIE10;\n  }\n  return isIE11 || isIE10;\n}\n\n/**\n * Returns the offset parent of the given element\n * @method\n * @memberof Popper.Utils\n * @argument {Element} element\n * @returns {Element} offset parent\n */\nfunction getOffsetParent(element) {\n  if (!element) {\n    return document.documentElement;\n  }\n\n  var noOffsetParent = isIE(10) ? document.body : null;\n\n  // NOTE: 1 DOM access here\n  var offsetParent = element.offsetParent || null;\n  // Skip hidden elements which don't have an offsetParent\n  while (offsetParent === noOffsetParent && element.nextElementSibling) {\n    offsetParent = (element = element.nextElementSibling).offsetParent;\n  }\n\n  var nodeName = offsetParent && offsetParent.nodeName;\n\n  if (!nodeName || nodeName === 'BODY' || nodeName === 'HTML') {\n    return element ? element.ownerDocument.documentElement : document.documentElement;\n  }\n\n  // .offsetParent will return the closest TH, TD or TABLE in case\n  // no offsetParent is present, I hate this job...\n  if (['TH', 'TD', 'TABLE'].indexOf(offsetParent.nodeName) !== -1 && getStyleComputedProperty(offsetParent, 'position') === 'static') {\n    return getOffsetParent(offsetParent);\n  }\n\n  return offsetParent;\n}\n\nfunction isOffsetContainer(element) {\n  var nodeName = element.nodeName;\n\n  if (nodeName === 'BODY') {\n    return false;\n  }\n  return nodeName === 'HTML' || getOffsetParent(element.firstElementChild) === element;\n}\n\n/**\n * Finds the root node (document, shadowDOM root) of the given element\n * @method\n * @memberof Popper.Utils\n * @argument {Element} node\n * @returns {Element} root node\n */\nfunction getRoot(node) {\n  if (node.parentNode !== null) {\n    return getRoot(node.parentNode);\n  }\n\n  return node;\n}\n\n/**\n * Finds the offset parent common to the two provided nodes\n * @method\n * @memberof Popper.Utils\n * @argument {Element} element1\n * @argument {Element} element2\n * @returns {Element} common offset parent\n */\nfunction findCommonOffsetParent(element1, element2) {\n  // This check is needed to avoid errors in case one of the elements isn't defined for any reason\n  if (!element1 || !element1.nodeType || !element2 || !element2.nodeType) {\n    return document.documentElement;\n  }\n\n  // Here we make sure to give as \"start\" the element that comes first in the DOM\n  var order = element1.compareDocumentPosition(element2) & Node.DOCUMENT_POSITION_FOLLOWING;\n  var start = order ? element1 : element2;\n  var end = order ? element2 : element1;\n\n  // Get common ancestor container\n  var range = document.createRange();\n  range.setStart(start, 0);\n  range.setEnd(end, 0);\n  var commonAncestorContainer = range.commonAncestorContainer;\n\n  // Both nodes are inside #document\n\n  if (element1 !== commonAncestorContainer && element2 !== commonAncestorContainer || start.contains(end)) {\n    if (isOffsetContainer(commonAncestorContainer)) {\n      return commonAncestorContainer;\n    }\n\n    return getOffsetParent(commonAncestorContainer);\n  }\n\n  // one of the nodes is inside shadowDOM, find which one\n  var element1root = getRoot(element1);\n  if (element1root.host) {\n    return findCommonOffsetParent(element1root.host, element2);\n  } else {\n    return findCommonOffsetParent(element1, getRoot(element2).host);\n  }\n}\n\n/**\n * Gets the scroll value of the given element in the given side (top and left)\n * @method\n * @memberof Popper.Utils\n * @argument {Element} element\n * @argument {String} side `top` or `left`\n * @returns {number} amount of scrolled pixels\n */\nfunction getScroll(element) {\n  var side = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 'top';\n\n  var upperSide = side === 'top' ? 'scrollTop' : 'scrollLeft';\n  var nodeName = element.nodeName;\n\n  if (nodeName === 'BODY' || nodeName === 'HTML') {\n    var html = element.ownerDocument.documentElement;\n    var scrollingElement = element.ownerDocument.scrollingElement || html;\n    return scrollingElement[upperSide];\n  }\n\n  return element[upperSide];\n}\n\n/*\n * Sum or subtract the element scroll values (left and top) from a given rect object\n * @method\n * @memberof Popper.Utils\n * @param {Object} rect - Rect object you want to change\n * @param {HTMLElement} element - The element from the function reads the scroll values\n * @param {Boolean} subtract - set to true if you want to subtract the scroll values\n * @return {Object} rect - The modifier rect object\n */\nfunction includeScroll(rect, element) {\n  var subtract = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : false;\n\n  var scrollTop = getScroll(element, 'top');\n  var scrollLeft = getScroll(element, 'left');\n  var modifier = subtract ? -1 : 1;\n  rect.top += scrollTop * modifier;\n  rect.bottom += scrollTop * modifier;\n  rect.left += scrollLeft * modifier;\n  rect.right += scrollLeft * modifier;\n  return rect;\n}\n\n/*\n * Helper to detect borders of a given element\n * @method\n * @memberof Popper.Utils\n * @param {CSSStyleDeclaration} styles\n * Result of `getStyleComputedProperty` on the given element\n * @param {String} axis - `x` or `y`\n * @return {number} borders - The borders size of the given axis\n */\n\nfunction getBordersSize(styles, axis) {\n  var sideA = axis === 'x' ? 'Left' : 'Top';\n  var sideB = sideA === 'Left' ? 'Right' : 'Bottom';\n\n  return parseFloat(styles['border' + sideA + 'Width'], 10) + parseFloat(styles['border' + sideB + 'Width'], 10);\n}\n\nfunction getSize(axis, body, html, computedStyle) {\n  return Math.max(body['offset' + axis], body['scroll' + axis], html['client' + axis], html['offset' + axis], html['scroll' + axis], isIE(10) ? parseInt(html['offset' + axis]) + parseInt(computedStyle['margin' + (axis === 'Height' ? 'Top' : 'Left')]) + parseInt(computedStyle['margin' + (axis === 'Height' ? 'Bottom' : 'Right')]) : 0);\n}\n\nfunction getWindowSizes(document) {\n  var body = document.body;\n  var html = document.documentElement;\n  var computedStyle = isIE(10) && getComputedStyle(html);\n\n  return {\n    height: getSize('Height', body, html, computedStyle),\n    width: getSize('Width', body, html, computedStyle)\n  };\n}\n\nvar classCallCheck = function (instance, Constructor) {\n  if (!(instance instanceof Constructor)) {\n    throw new TypeError(\"Cannot call a class as a function\");\n  }\n};\n\nvar createClass = function () {\n  function defineProperties(target, props) {\n    for (var i = 0; i < props.length; i++) {\n      var descriptor = props[i];\n      descriptor.enumerable = descriptor.enumerable || false;\n      descriptor.configurable = true;\n      if (\"value\" in descriptor) descriptor.writable = true;\n      Object.defineProperty(target, descriptor.key, descriptor);\n    }\n  }\n\n  return function (Constructor, protoProps, staticProps) {\n    if (protoProps) defineProperties(Constructor.prototype, protoProps);\n    if (staticProps) defineProperties(Constructor, staticProps);\n    return Constructor;\n  };\n}();\n\n\n\n\n\nvar defineProperty = function (obj, key, value) {\n  if (key in obj) {\n    Object.defineProperty(obj, key, {\n      value: value,\n      enumerable: true,\n      configurable: true,\n      writable: true\n    });\n  } else {\n    obj[key] = value;\n  }\n\n  return obj;\n};\n\nvar _extends = Object.assign || function (target) {\n  for (var i = 1; i < arguments.length; i++) {\n    var source = arguments[i];\n\n    for (var key in source) {\n      if (Object.prototype.hasOwnProperty.call(source, key)) {\n        target[key] = source[key];\n      }\n    }\n  }\n\n  return target;\n};\n\n/**\n * Given element offsets, generate an output similar to getBoundingClientRect\n * @method\n * @memberof Popper.Utils\n * @argument {Object} offsets\n * @returns {Object} ClientRect like output\n */\nfunction getClientRect(offsets) {\n  return _extends({}, offsets, {\n    right: offsets.left + offsets.width,\n    bottom: offsets.top + offsets.height\n  });\n}\n\n/**\n * Get bounding client rect of given element\n * @method\n * @memberof Popper.Utils\n * @param {HTMLElement} element\n * @return {Object} client rect\n */\nfunction getBoundingClientRect(element) {\n  var rect = {};\n\n  // IE10 10 FIX: Please, don't ask, the element isn't\n  // considered in DOM in some circumstances...\n  // This isn't reproducible in IE10 compatibility mode of IE11\n  try {\n    if (isIE(10)) {\n      rect = element.getBoundingClientRect();\n      var scrollTop = getScroll(element, 'top');\n      var scrollLeft = getScroll(element, 'left');\n      rect.top += scrollTop;\n      rect.left += scrollLeft;\n      rect.bottom += scrollTop;\n      rect.right += scrollLeft;\n    } else {\n      rect = element.getBoundingClientRect();\n    }\n  } catch (e) {}\n\n  var result = {\n    left: rect.left,\n    top: rect.top,\n    width: rect.right - rect.left,\n    height: rect.bottom - rect.top\n  };\n\n  // subtract scrollbar size from sizes\n  var sizes = element.nodeName === 'HTML' ? getWindowSizes(element.ownerDocument) : {};\n  var width = sizes.width || element.clientWidth || result.width;\n  var height = sizes.height || element.clientHeight || result.height;\n\n  var horizScrollbar = element.offsetWidth - width;\n  var vertScrollbar = element.offsetHeight - height;\n\n  // if an hypothetical scrollbar is detected, we must be sure it's not a `border`\n  // we make this check conditional for performance reasons\n  if (horizScrollbar || vertScrollbar) {\n    var styles = getStyleComputedProperty(element);\n    horizScrollbar -= getBordersSize(styles, 'x');\n    vertScrollbar -= getBordersSize(styles, 'y');\n\n    result.width -= horizScrollbar;\n    result.height -= vertScrollbar;\n  }\n\n  return getClientRect(result);\n}\n\nfunction getOffsetRectRelativeToArbitraryNode(children, parent) {\n  var fixedPosition = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : false;\n\n  var isIE10 = isIE(10);\n  var isHTML = parent.nodeName === 'HTML';\n  var childrenRect = getBoundingClientRect(children);\n  var parentRect = getBoundingClientRect(parent);\n  var scrollParent = getScrollParent(children);\n\n  var styles = getStyleComputedProperty(parent);\n  var borderTopWidth = parseFloat(styles.borderTopWidth, 10);\n  var borderLeftWidth = parseFloat(styles.borderLeftWidth, 10);\n\n  // In cases where the parent is fixed, we must ignore negative scroll in offset calc\n  if (fixedPosition && isHTML) {\n    parentRect.top = Math.max(parentRect.top, 0);\n    parentRect.left = Math.max(parentRect.left, 0);\n  }\n  var offsets = getClientRect({\n    top: childrenRect.top - parentRect.top - borderTopWidth,\n    left: childrenRect.left - parentRect.left - borderLeftWidth,\n    width: childrenRect.width,\n    height: childrenRect.height\n  });\n  offsets.marginTop = 0;\n  offsets.marginLeft = 0;\n\n  // Subtract margins of documentElement in case it's being used as parent\n  // we do this only on HTML because it's the only element that behaves\n  // differently when margins are applied to it. The margins are included in\n  // the box of the documentElement, in the other cases not.\n  if (!isIE10 && isHTML) {\n    var marginTop = parseFloat(styles.marginTop, 10);\n    var marginLeft = parseFloat(styles.marginLeft, 10);\n\n    offsets.top -= borderTopWidth - marginTop;\n    offsets.bottom -= borderTopWidth - marginTop;\n    offsets.left -= borderLeftWidth - marginLeft;\n    offsets.right -= borderLeftWidth - marginLeft;\n\n    // Attach marginTop and marginLeft because in some circumstances we may need them\n    offsets.marginTop = marginTop;\n    offsets.marginLeft = marginLeft;\n  }\n\n  if (isIE10 && !fixedPosition ? parent.contains(scrollParent) : parent === scrollParent && scrollParent.nodeName !== 'BODY') {\n    offsets = includeScroll(offsets, parent);\n  }\n\n  return offsets;\n}\n\nfunction getViewportOffsetRectRelativeToArtbitraryNode(element) {\n  var excludeScroll = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : false;\n\n  var html = element.ownerDocument.documentElement;\n  var relativeOffset = getOffsetRectRelativeToArbitraryNode(element, html);\n  var width = Math.max(html.clientWidth, window.innerWidth || 0);\n  var height = Math.max(html.clientHeight, window.innerHeight || 0);\n\n  var scrollTop = !excludeScroll ? getScroll(html) : 0;\n  var scrollLeft = !excludeScroll ? getScroll(html, 'left') : 0;\n\n  var offset = {\n    top: scrollTop - relativeOffset.top + relativeOffset.marginTop,\n    left: scrollLeft - relativeOffset.left + relativeOffset.marginLeft,\n    width: width,\n    height: height\n  };\n\n  return getClientRect(offset);\n}\n\n/**\n * Check if the given element is fixed or is inside a fixed parent\n * @method\n * @memberof Popper.Utils\n * @argument {Element} element\n * @argument {Element} customContainer\n * @returns {Boolean} answer to \"isFixed?\"\n */\nfunction isFixed(element) {\n  var nodeName = element.nodeName;\n  if (nodeName === 'BODY' || nodeName === 'HTML') {\n    return false;\n  }\n  if (getStyleComputedProperty(element, 'position') === 'fixed') {\n    return true;\n  }\n  var parentNode = getParentNode(element);\n  if (!parentNode) {\n    return false;\n  }\n  return isFixed(parentNode);\n}\n\n/**\n * Finds the first parent of an element that has a transformed property defined\n * @method\n * @memberof Popper.Utils\n * @argument {Element} element\n * @returns {Element} first transformed parent or documentElement\n */\n\nfunction getFixedPositionOffsetParent(element) {\n  // This check is needed to avoid errors in case one of the elements isn't defined for any reason\n  if (!element || !element.parentElement || isIE()) {\n    return document.documentElement;\n  }\n  var el = element.parentElement;\n  while (el && getStyleComputedProperty(el, 'transform') === 'none') {\n    el = el.parentElement;\n  }\n  return el || document.documentElement;\n}\n\n/**\n * Computed the boundaries limits and return them\n * @method\n * @memberof Popper.Utils\n * @param {HTMLElement} popper\n * @param {HTMLElement} reference\n * @param {number} padding\n * @param {HTMLElement} boundariesElement - Element used to define the boundaries\n * @param {Boolean} fixedPosition - Is in fixed position mode\n * @returns {Object} Coordinates of the boundaries\n */\nfunction getBoundaries(popper, reference, padding, boundariesElement) {\n  var fixedPosition = arguments.length > 4 && arguments[4] !== undefined ? arguments[4] : false;\n\n  // NOTE: 1 DOM access here\n\n  var boundaries = { top: 0, left: 0 };\n  var offsetParent = fixedPosition ? getFixedPositionOffsetParent(popper) : findCommonOffsetParent(popper, getReferenceNode(reference));\n\n  // Handle viewport case\n  if (boundariesElement === 'viewport') {\n    boundaries = getViewportOffsetRectRelativeToArtbitraryNode(offsetParent, fixedPosition);\n  } else {\n    // Handle other cases based on DOM element used as boundaries\n    var boundariesNode = void 0;\n    if (boundariesElement === 'scrollParent') {\n      boundariesNode = getScrollParent(getParentNode(reference));\n      if (boundariesNode.nodeName === 'BODY') {\n        boundariesNode = popper.ownerDocument.documentElement;\n      }\n    } else if (boundariesElement === 'window') {\n      boundariesNode = popper.ownerDocument.documentElement;\n    } else {\n      boundariesNode = boundariesElement;\n    }\n\n    var offsets = getOffsetRectRelativeToArbitraryNode(boundariesNode, offsetParent, fixedPosition);\n\n    // In case of HTML, we need a different computation\n    if (boundariesNode.nodeName === 'HTML' && !isFixed(offsetParent)) {\n      var _getWindowSizes = getWindowSizes(popper.ownerDocument),\n          height = _getWindowSizes.height,\n          width = _getWindowSizes.width;\n\n      boundaries.top += offsets.top - offsets.marginTop;\n      boundaries.bottom = height + offsets.top;\n      boundaries.left += offsets.left - offsets.marginLeft;\n      boundaries.right = width + offsets.left;\n    } else {\n      // for all the other DOM elements, this one is good\n      boundaries = offsets;\n    }\n  }\n\n  // Add paddings\n  padding = padding || 0;\n  var isPaddingNumber = typeof padding === 'number';\n  boundaries.left += isPaddingNumber ? padding : padding.left || 0;\n  boundaries.top += isPaddingNumber ? padding : padding.top || 0;\n  boundaries.right -= isPaddingNumber ? padding : padding.right || 0;\n  boundaries.bottom -= isPaddingNumber ? padding : padding.bottom || 0;\n\n  return boundaries;\n}\n\nfunction getArea(_ref) {\n  var width = _ref.width,\n      height = _ref.height;\n\n  return width * height;\n}\n\n/**\n * Utility used to transform the `auto` placement to the placement with more\n * available space.\n * @method\n * @memberof Popper.Utils\n * @argument {Object} data - The data object generated by update method\n * @argument {Object} options - Modifiers configuration and options\n * @returns {Object} The data object, properly modified\n */\nfunction computeAutoPlacement(placement, refRect, popper, reference, boundariesElement) {\n  var padding = arguments.length > 5 && arguments[5] !== undefined ? arguments[5] : 0;\n\n  if (placement.indexOf('auto') === -1) {\n    return placement;\n  }\n\n  var boundaries = getBoundaries(popper, reference, padding, boundariesElement);\n\n  var rects = {\n    top: {\n      width: boundaries.width,\n      height: refRect.top - boundaries.top\n    },\n    right: {\n      width: boundaries.right - refRect.right,\n      height: boundaries.height\n    },\n    bottom: {\n      width: boundaries.width,\n      height: boundaries.bottom - refRect.bottom\n    },\n    left: {\n      width: refRect.left - boundaries.left,\n      height: boundaries.height\n    }\n  };\n\n  var sortedAreas = Object.keys(rects).map(function (key) {\n    return _extends({\n      key: key\n    }, rects[key], {\n      area: getArea(rects[key])\n    });\n  }).sort(function (a, b) {\n    return b.area - a.area;\n  });\n\n  var filteredAreas = sortedAreas.filter(function (_ref2) {\n    var width = _ref2.width,\n        height = _ref2.height;\n    return width >= popper.clientWidth && height >= popper.clientHeight;\n  });\n\n  var computedPlacement = filteredAreas.length > 0 ? filteredAreas[0].key : sortedAreas[0].key;\n\n  var variation = placement.split('-')[1];\n\n  return computedPlacement + (variation ? '-' + variation : '');\n}\n\n/**\n * Get offsets to the reference element\n * @method\n * @memberof Popper.Utils\n * @param {Object} state\n * @param {Element} popper - the popper element\n * @param {Element} reference - the reference element (the popper will be relative to this)\n * @param {Element} fixedPosition - is in fixed position mode\n * @returns {Object} An object containing the offsets which will be applied to the popper\n */\nfunction getReferenceOffsets(state, popper, reference) {\n  var fixedPosition = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : null;\n\n  var commonOffsetParent = fixedPosition ? getFixedPositionOffsetParent(popper) : findCommonOffsetParent(popper, getReferenceNode(reference));\n  return getOffsetRectRelativeToArbitraryNode(reference, commonOffsetParent, fixedPosition);\n}\n\n/**\n * Get the outer sizes of the given element (offset size + margins)\n * @method\n * @memberof Popper.Utils\n * @argument {Element} element\n * @returns {Object} object containing width and height properties\n */\nfunction getOuterSizes(element) {\n  var window = element.ownerDocument.defaultView;\n  var styles = window.getComputedStyle(element);\n  var x = parseFloat(styles.marginTop || 0) + parseFloat(styles.marginBottom || 0);\n  var y = parseFloat(styles.marginLeft || 0) + parseFloat(styles.marginRight || 0);\n  var result = {\n    width: element.offsetWidth + y,\n    height: element.offsetHeight + x\n  };\n  return result;\n}\n\n/**\n * Get the opposite placement of the given one\n * @method\n * @memberof Popper.Utils\n * @argument {String} placement\n * @returns {String} flipped placement\n */\nfunction getOppositePlacement(placement) {\n  var hash = { left: 'right', right: 'left', bottom: 'top', top: 'bottom' };\n  return placement.replace(/left|right|bottom|top/g, function (matched) {\n    return hash[matched];\n  });\n}\n\n/**\n * Get offsets to the popper\n * @method\n * @memberof Popper.Utils\n * @param {Object} position - CSS position the Popper will get applied\n * @param {HTMLElement} popper - the popper element\n * @param {Object} referenceOffsets - the reference offsets (the popper will be relative to this)\n * @param {String} placement - one of the valid placement options\n * @returns {Object} popperOffsets - An object containing the offsets which will be applied to the popper\n */\nfunction getPopperOffsets(popper, referenceOffsets, placement) {\n  placement = placement.split('-')[0];\n\n  // Get popper node sizes\n  var popperRect = getOuterSizes(popper);\n\n  // Add position, width and height to our offsets object\n  var popperOffsets = {\n    width: popperRect.width,\n    height: popperRect.height\n  };\n\n  // depending by the popper placement we have to compute its offsets slightly differently\n  var isHoriz = ['right', 'left'].indexOf(placement) !== -1;\n  var mainSide = isHoriz ? 'top' : 'left';\n  var secondarySide = isHoriz ? 'left' : 'top';\n  var measurement = isHoriz ? 'height' : 'width';\n  var secondaryMeasurement = !isHoriz ? 'height' : 'width';\n\n  popperOffsets[mainSide] = referenceOffsets[mainSide] + referenceOffsets[measurement] / 2 - popperRect[measurement] / 2;\n  if (placement === secondarySide) {\n    popperOffsets[secondarySide] = referenceOffsets[secondarySide] - popperRect[secondaryMeasurement];\n  } else {\n    popperOffsets[secondarySide] = referenceOffsets[getOppositePlacement(secondarySide)];\n  }\n\n  return popperOffsets;\n}\n\n/**\n * Mimics the `find` method of Array\n * @method\n * @memberof Popper.Utils\n * @argument {Array} arr\n * @argument prop\n * @argument value\n * @returns index or -1\n */\nfunction find(arr, check) {\n  // use native find if supported\n  if (Array.prototype.find) {\n    return arr.find(check);\n  }\n\n  // use `filter` to obtain the same behavior of `find`\n  return arr.filter(check)[0];\n}\n\n/**\n * Return the index of the matching object\n * @method\n * @memberof Popper.Utils\n * @argument {Array} arr\n * @argument prop\n * @argument value\n * @returns index or -1\n */\nfunction findIndex(arr, prop, value) {\n  // use native findIndex if supported\n  if (Array.prototype.findIndex) {\n    return arr.findIndex(function (cur) {\n      return cur[prop] === value;\n    });\n  }\n\n  // use `find` + `indexOf` if `findIndex` isn't supported\n  var match = find(arr, function (obj) {\n    return obj[prop] === value;\n  });\n  return arr.indexOf(match);\n}\n\n/**\n * Loop trough the list of modifiers and run them in order,\n * each of them will then edit the data object.\n * @method\n * @memberof Popper.Utils\n * @param {dataObject} data\n * @param {Array} modifiers\n * @param {String} ends - Optional modifier name used as stopper\n * @returns {dataObject}\n */\nfunction runModifiers(modifiers, data, ends) {\n  var modifiersToRun = ends === undefined ? modifiers : modifiers.slice(0, findIndex(modifiers, 'name', ends));\n\n  modifiersToRun.forEach(function (modifier) {\n    if (modifier['function']) {\n      // eslint-disable-line dot-notation\n      console.warn('`modifier.function` is deprecated, use `modifier.fn`!');\n    }\n    var fn = modifier['function'] || modifier.fn; // eslint-disable-line dot-notation\n    if (modifier.enabled && isFunction(fn)) {\n      // Add properties to offsets to make them a complete clientRect object\n      // we do this before each modifier to make sure the previous one doesn't\n      // mess with these values\n      data.offsets.popper = getClientRect(data.offsets.popper);\n      data.offsets.reference = getClientRect(data.offsets.reference);\n\n      data = fn(data, modifier);\n    }\n  });\n\n  return data;\n}\n\n/**\n * Updates the position of the popper, computing the new offsets and applying\n * the new style.<br />\n * Prefer `scheduleUpdate` over `update` because of performance reasons.\n * @method\n * @memberof Popper\n */\nfunction update() {\n  // if popper is destroyed, don't perform any further update\n  if (this.state.isDestroyed) {\n    return;\n  }\n\n  var data = {\n    instance: this,\n    styles: {},\n    arrowStyles: {},\n    attributes: {},\n    flipped: false,\n    offsets: {}\n  };\n\n  // compute reference element offsets\n  data.offsets.reference = getReferenceOffsets(this.state, this.popper, this.reference, this.options.positionFixed);\n\n  // compute auto placement, store placement inside the data object,\n  // modifiers will be able to edit `placement` if needed\n  // and refer to originalPlacement to know the original value\n  data.placement = computeAutoPlacement(this.options.placement, data.offsets.reference, this.popper, this.reference, this.options.modifiers.flip.boundariesElement, this.options.modifiers.flip.padding);\n\n  // store the computed placement inside `originalPlacement`\n  data.originalPlacement = data.placement;\n\n  data.positionFixed = this.options.positionFixed;\n\n  // compute the popper offsets\n  data.offsets.popper = getPopperOffsets(this.popper, data.offsets.reference, data.placement);\n\n  data.offsets.popper.position = this.options.positionFixed ? 'fixed' : 'absolute';\n\n  // run the modifiers\n  data = runModifiers(this.modifiers, data);\n\n  // the first `update` will call `onCreate` callback\n  // the other ones will call `onUpdate` callback\n  if (!this.state.isCreated) {\n    this.state.isCreated = true;\n    this.options.onCreate(data);\n  } else {\n    this.options.onUpdate(data);\n  }\n}\n\n/**\n * Helper used to know if the given modifier is enabled.\n * @method\n * @memberof Popper.Utils\n * @returns {Boolean}\n */\nfunction isModifierEnabled(modifiers, modifierName) {\n  return modifiers.some(function (_ref) {\n    var name = _ref.name,\n        enabled = _ref.enabled;\n    return enabled && name === modifierName;\n  });\n}\n\n/**\n * Get the prefixed supported property name\n * @method\n * @memberof Popper.Utils\n * @argument {String} property (camelCase)\n * @returns {String} prefixed property (camelCase or PascalCase, depending on the vendor prefix)\n */\nfunction getSupportedPropertyName(property) {\n  var prefixes = [false, 'ms', 'Webkit', 'Moz', 'O'];\n  var upperProp = property.charAt(0).toUpperCase() + property.slice(1);\n\n  for (var i = 0; i < prefixes.length; i++) {\n    var prefix = prefixes[i];\n    var toCheck = prefix ? '' + prefix + upperProp : property;\n    if (typeof document.body.style[toCheck] !== 'undefined') {\n      return toCheck;\n    }\n  }\n  return null;\n}\n\n/**\n * Destroys the popper.\n * @method\n * @memberof Popper\n */\nfunction destroy() {\n  this.state.isDestroyed = true;\n\n  // touch DOM only if `applyStyle` modifier is enabled\n  if (isModifierEnabled(this.modifiers, 'applyStyle')) {\n    this.popper.removeAttribute('x-placement');\n    this.popper.style.position = '';\n    this.popper.style.top = '';\n    this.popper.style.left = '';\n    this.popper.style.right = '';\n    this.popper.style.bottom = '';\n    this.popper.style.willChange = '';\n    this.popper.style[getSupportedPropertyName('transform')] = '';\n  }\n\n  this.disableEventListeners();\n\n  // remove the popper if user explicitly asked for the deletion on destroy\n  // do not use `remove` because IE11 doesn't support it\n  if (this.options.removeOnDestroy) {\n    this.popper.parentNode.removeChild(this.popper);\n  }\n  return this;\n}\n\n/**\n * Get the window associated with the element\n * @argument {Element} element\n * @returns {Window}\n */\nfunction getWindow(element) {\n  var ownerDocument = element.ownerDocument;\n  return ownerDocument ? ownerDocument.defaultView : window;\n}\n\nfunction attachToScrollParents(scrollParent, event, callback, scrollParents) {\n  var isBody = scrollParent.nodeName === 'BODY';\n  var target = isBody ? scrollParent.ownerDocument.defaultView : scrollParent;\n  target.addEventListener(event, callback, { passive: true });\n\n  if (!isBody) {\n    attachToScrollParents(getScrollParent(target.parentNode), event, callback, scrollParents);\n  }\n  scrollParents.push(target);\n}\n\n/**\n * Setup needed event listeners used to update the popper position\n * @method\n * @memberof Popper.Utils\n * @private\n */\nfunction setupEventListeners(reference, options, state, updateBound) {\n  // Resize event listener on window\n  state.updateBound = updateBound;\n  getWindow(reference).addEventListener('resize', state.updateBound, { passive: true });\n\n  // Scroll event listener on scroll parents\n  var scrollElement = getScrollParent(reference);\n  attachToScrollParents(scrollElement, 'scroll', state.updateBound, state.scrollParents);\n  state.scrollElement = scrollElement;\n  state.eventsEnabled = true;\n\n  return state;\n}\n\n/**\n * It will add resize/scroll events and start recalculating\n * position of the popper element when they are triggered.\n * @method\n * @memberof Popper\n */\nfunction enableEventListeners() {\n  if (!this.state.eventsEnabled) {\n    this.state = setupEventListeners(this.reference, this.options, this.state, this.scheduleUpdate);\n  }\n}\n\n/**\n * Remove event listeners used to update the popper position\n * @method\n * @memberof Popper.Utils\n * @private\n */\nfunction removeEventListeners(reference, state) {\n  // Remove resize event listener on window\n  getWindow(reference).removeEventListener('resize', state.updateBound);\n\n  // Remove scroll event listener on scroll parents\n  state.scrollParents.forEach(function (target) {\n    target.removeEventListener('scroll', state.updateBound);\n  });\n\n  // Reset state\n  state.updateBound = null;\n  state.scrollParents = [];\n  state.scrollElement = null;\n  state.eventsEnabled = false;\n  return state;\n}\n\n/**\n * It will remove resize/scroll events and won't recalculate popper position\n * when they are triggered. It also won't trigger `onUpdate` callback anymore,\n * unless you call `update` method manually.\n * @method\n * @memberof Popper\n */\nfunction disableEventListeners() {\n  if (this.state.eventsEnabled) {\n    cancelAnimationFrame(this.scheduleUpdate);\n    this.state = removeEventListeners(this.reference, this.state);\n  }\n}\n\n/**\n * Tells if a given input is a number\n * @method\n * @memberof Popper.Utils\n * @param {*} input to check\n * @return {Boolean}\n */\nfunction isNumeric(n) {\n  return n !== '' && !isNaN(parseFloat(n)) && isFinite(n);\n}\n\n/**\n * Set the style to the given popper\n * @method\n * @memberof Popper.Utils\n * @argument {Element} element - Element to apply the style to\n * @argument {Object} styles\n * Object with a list of properties and values which will be applied to the element\n */\nfunction setStyles(element, styles) {\n  Object.keys(styles).forEach(function (prop) {\n    var unit = '';\n    // add unit if the value is numeric and is one of the following\n    if (['width', 'height', 'top', 'right', 'bottom', 'left'].indexOf(prop) !== -1 && isNumeric(styles[prop])) {\n      unit = 'px';\n    }\n    element.style[prop] = styles[prop] + unit;\n  });\n}\n\n/**\n * Set the attributes to the given popper\n * @method\n * @memberof Popper.Utils\n * @argument {Element} element - Element to apply the attributes to\n * @argument {Object} styles\n * Object with a list of properties and values which will be applied to the element\n */\nfunction setAttributes(element, attributes) {\n  Object.keys(attributes).forEach(function (prop) {\n    var value = attributes[prop];\n    if (value !== false) {\n      element.setAttribute(prop, attributes[prop]);\n    } else {\n      element.removeAttribute(prop);\n    }\n  });\n}\n\n/**\n * @function\n * @memberof Modifiers\n * @argument {Object} data - The data object generated by `update` method\n * @argument {Object} data.styles - List of style properties - values to apply to popper element\n * @argument {Object} data.attributes - List of attribute properties - values to apply to popper element\n * @argument {Object} options - Modifiers configuration and options\n * @returns {Object} The same data object\n */\nfunction applyStyle(data) {\n  // any property present in `data.styles` will be applied to the popper,\n  // in this way we can make the 3rd party modifiers add custom styles to it\n  // Be aware, modifiers could override the properties defined in the previous\n  // lines of this modifier!\n  setStyles(data.instance.popper, data.styles);\n\n  // any property present in `data.attributes` will be applied to the popper,\n  // they will be set as HTML attributes of the element\n  setAttributes(data.instance.popper, data.attributes);\n\n  // if arrowElement is defined and arrowStyles has some properties\n  if (data.arrowElement && Object.keys(data.arrowStyles).length) {\n    setStyles(data.arrowElement, data.arrowStyles);\n  }\n\n  return data;\n}\n\n/**\n * Set the x-placement attribute before everything else because it could be used\n * to add margins to the popper margins needs to be calculated to get the\n * correct popper offsets.\n * @method\n * @memberof Popper.modifiers\n * @param {HTMLElement} reference - The reference element used to position the popper\n * @param {HTMLElement} popper - The HTML element used as popper\n * @param {Object} options - Popper.js options\n */\nfunction applyStyleOnLoad(reference, popper, options, modifierOptions, state) {\n  // compute reference element offsets\n  var referenceOffsets = getReferenceOffsets(state, popper, reference, options.positionFixed);\n\n  // compute auto placement, store placement inside the data object,\n  // modifiers will be able to edit `placement` if needed\n  // and refer to originalPlacement to know the original value\n  var placement = computeAutoPlacement(options.placement, referenceOffsets, popper, reference, options.modifiers.flip.boundariesElement, options.modifiers.flip.padding);\n\n  popper.setAttribute('x-placement', placement);\n\n  // Apply `position` to popper before anything else because\n  // without the position applied we can't guarantee correct computations\n  setStyles(popper, { position: options.positionFixed ? 'fixed' : 'absolute' });\n\n  return options;\n}\n\n/**\n * @function\n * @memberof Popper.Utils\n * @argument {Object} data - The data object generated by `update` method\n * @argument {Boolean} shouldRound - If the offsets should be rounded at all\n * @returns {Object} The popper's position offsets rounded\n *\n * The tale of pixel-perfect positioning. It's still not 100% perfect, but as\n * good as it can be within reason.\n * Discussion here: https://github.com/FezVrasta/popper.js/pull/715\n *\n * Low DPI screens cause a popper to be blurry if not using full pixels (Safari\n * as well on High DPI screens).\n *\n * Firefox prefers no rounding for positioning and does not have blurriness on\n * high DPI screens.\n *\n * Only horizontal placement and left/right values need to be considered.\n */\nfunction getRoundedOffsets(data, shouldRound) {\n  var _data$offsets = data.offsets,\n      popper = _data$offsets.popper,\n      reference = _data$offsets.reference;\n  var round = Math.round,\n      floor = Math.floor;\n\n  var noRound = function noRound(v) {\n    return v;\n  };\n\n  var referenceWidth = round(reference.width);\n  var popperWidth = round(popper.width);\n\n  var isVertical = ['left', 'right'].indexOf(data.placement) !== -1;\n  var isVariation = data.placement.indexOf('-') !== -1;\n  var sameWidthParity = referenceWidth % 2 === popperWidth % 2;\n  var bothOddWidth = referenceWidth % 2 === 1 && popperWidth % 2 === 1;\n\n  var horizontalToInteger = !shouldRound ? noRound : isVertical || isVariation || sameWidthParity ? round : floor;\n  var verticalToInteger = !shouldRound ? noRound : round;\n\n  return {\n    left: horizontalToInteger(bothOddWidth && !isVariation && shouldRound ? popper.left - 1 : popper.left),\n    top: verticalToInteger(popper.top),\n    bottom: verticalToInteger(popper.bottom),\n    right: horizontalToInteger(popper.right)\n  };\n}\n\nvar isFirefox = isBrowser && /Firefox/i.test(navigator.userAgent);\n\n/**\n * @function\n * @memberof Modifiers\n * @argument {Object} data - The data object generated by `update` method\n * @argument {Object} options - Modifiers configuration and options\n * @returns {Object} The data object, properly modified\n */\nfunction computeStyle(data, options) {\n  var x = options.x,\n      y = options.y;\n  var popper = data.offsets.popper;\n\n  // Remove this legacy support in Popper.js v2\n\n  var legacyGpuAccelerationOption = find(data.instance.modifiers, function (modifier) {\n    return modifier.name === 'applyStyle';\n  }).gpuAcceleration;\n  if (legacyGpuAccelerationOption !== undefined) {\n    console.warn('WARNING: `gpuAcceleration` option moved to `computeStyle` modifier and will not be supported in future versions of Popper.js!');\n  }\n  var gpuAcceleration = legacyGpuAccelerationOption !== undefined ? legacyGpuAccelerationOption : options.gpuAcceleration;\n\n  var offsetParent = getOffsetParent(data.instance.popper);\n  var offsetParentRect = getBoundingClientRect(offsetParent);\n\n  // Styles\n  var styles = {\n    position: popper.position\n  };\n\n  var offsets = getRoundedOffsets(data, window.devicePixelRatio < 2 || !isFirefox);\n\n  var sideA = x === 'bottom' ? 'top' : 'bottom';\n  var sideB = y === 'right' ? 'left' : 'right';\n\n  // if gpuAcceleration is set to `true` and transform is supported,\n  //  we use `translate3d` to apply the position to the popper we\n  // automatically use the supported prefixed version if needed\n  var prefixedProperty = getSupportedPropertyName('transform');\n\n  // now, let's make a step back and look at this code closely (wtf?)\n  // If the content of the popper grows once it's been positioned, it\n  // may happen that the popper gets misplaced because of the new content\n  // overflowing its reference element\n  // To avoid this problem, we provide two options (x and y), which allow\n  // the consumer to define the offset origin.\n  // If we position a popper on top of a reference element, we can set\n  // `x` to `top` to make the popper grow towards its top instead of\n  // its bottom.\n  var left = void 0,\n      top = void 0;\n  if (sideA === 'bottom') {\n    // when offsetParent is <html> the positioning is relative to the bottom of the screen (excluding the scrollbar)\n    // and not the bottom of the html element\n    if (offsetParent.nodeName === 'HTML') {\n      top = -offsetParent.clientHeight + offsets.bottom;\n    } else {\n      top = -offsetParentRect.height + offsets.bottom;\n    }\n  } else {\n    top = offsets.top;\n  }\n  if (sideB === 'right') {\n    if (offsetParent.nodeName === 'HTML') {\n      left = -offsetParent.clientWidth + offsets.right;\n    } else {\n      left = -offsetParentRect.width + offsets.right;\n    }\n  } else {\n    left = offsets.left;\n  }\n  if (gpuAcceleration && prefixedProperty) {\n    styles[prefixedProperty] = 'translate3d(' + left + 'px, ' + top + 'px, 0)';\n    styles[sideA] = 0;\n    styles[sideB] = 0;\n    styles.willChange = 'transform';\n  } else {\n    // othwerise, we use the standard `top`, `left`, `bottom` and `right` properties\n    var invertTop = sideA === 'bottom' ? -1 : 1;\n    var invertLeft = sideB === 'right' ? -1 : 1;\n    styles[sideA] = top * invertTop;\n    styles[sideB] = left * invertLeft;\n    styles.willChange = sideA + ', ' + sideB;\n  }\n\n  // Attributes\n  var attributes = {\n    'x-placement': data.placement\n  };\n\n  // Update `data` attributes, styles and arrowStyles\n  data.attributes = _extends({}, attributes, data.attributes);\n  data.styles = _extends({}, styles, data.styles);\n  data.arrowStyles = _extends({}, data.offsets.arrow, data.arrowStyles);\n\n  return data;\n}\n\n/**\n * Helper used to know if the given modifier depends from another one.<br />\n * It checks if the needed modifier is listed and enabled.\n * @method\n * @memberof Popper.Utils\n * @param {Array} modifiers - list of modifiers\n * @param {String} requestingName - name of requesting modifier\n * @param {String} requestedName - name of requested modifier\n * @returns {Boolean}\n */\nfunction isModifierRequired(modifiers, requestingName, requestedName) {\n  var requesting = find(modifiers, function (_ref) {\n    var name = _ref.name;\n    return name === requestingName;\n  });\n\n  var isRequired = !!requesting && modifiers.some(function (modifier) {\n    return modifier.name === requestedName && modifier.enabled && modifier.order < requesting.order;\n  });\n\n  if (!isRequired) {\n    var _requesting = '`' + requestingName + '`';\n    var requested = '`' + requestedName + '`';\n    console.warn(requested + ' modifier is required by ' + _requesting + ' modifier in order to work, be sure to include it before ' + _requesting + '!');\n  }\n  return isRequired;\n}\n\n/**\n * @function\n * @memberof Modifiers\n * @argument {Object} data - The data object generated by update method\n * @argument {Object} options - Modifiers configuration and options\n * @returns {Object} The data object, properly modified\n */\nfunction arrow(data, options) {\n  var _data$offsets$arrow;\n\n  // arrow depends on keepTogether in order to work\n  if (!isModifierRequired(data.instance.modifiers, 'arrow', 'keepTogether')) {\n    return data;\n  }\n\n  var arrowElement = options.element;\n\n  // if arrowElement is a string, suppose it's a CSS selector\n  if (typeof arrowElement === 'string') {\n    arrowElement = data.instance.popper.querySelector(arrowElement);\n\n    // if arrowElement is not found, don't run the modifier\n    if (!arrowElement) {\n      return data;\n    }\n  } else {\n    // if the arrowElement isn't a query selector we must check that the\n    // provided DOM node is child of its popper node\n    if (!data.instance.popper.contains(arrowElement)) {\n      console.warn('WARNING: `arrow.element` must be child of its popper element!');\n      return data;\n    }\n  }\n\n  var placement = data.placement.split('-')[0];\n  var _data$offsets = data.offsets,\n      popper = _data$offsets.popper,\n      reference = _data$offsets.reference;\n\n  var isVertical = ['left', 'right'].indexOf(placement) !== -1;\n\n  var len = isVertical ? 'height' : 'width';\n  var sideCapitalized = isVertical ? 'Top' : 'Left';\n  var side = sideCapitalized.toLowerCase();\n  var altSide = isVertical ? 'left' : 'top';\n  var opSide = isVertical ? 'bottom' : 'right';\n  var arrowElementSize = getOuterSizes(arrowElement)[len];\n\n  //\n  // extends keepTogether behavior making sure the popper and its\n  // reference have enough pixels in conjunction\n  //\n\n  // top/left side\n  if (reference[opSide] - arrowElementSize < popper[side]) {\n    data.offsets.popper[side] -= popper[side] - (reference[opSide] - arrowElementSize);\n  }\n  // bottom/right side\n  if (reference[side] + arrowElementSize > popper[opSide]) {\n    data.offsets.popper[side] += reference[side] + arrowElementSize - popper[opSide];\n  }\n  data.offsets.popper = getClientRect(data.offsets.popper);\n\n  // compute center of the popper\n  var center = reference[side] + reference[len] / 2 - arrowElementSize / 2;\n\n  // Compute the sideValue using the updated popper offsets\n  // take popper margin in account because we don't have this info available\n  var css = getStyleComputedProperty(data.instance.popper);\n  var popperMarginSide = parseFloat(css['margin' + sideCapitalized], 10);\n  var popperBorderSide = parseFloat(css['border' + sideCapitalized + 'Width'], 10);\n  var sideValue = center - data.offsets.popper[side] - popperMarginSide - popperBorderSide;\n\n  // prevent arrowElement from being placed not contiguously to its popper\n  sideValue = Math.max(Math.min(popper[len] - arrowElementSize, sideValue), 0);\n\n  data.arrowElement = arrowElement;\n  data.offsets.arrow = (_data$offsets$arrow = {}, defineProperty(_data$offsets$arrow, side, Math.round(sideValue)), defineProperty(_data$offsets$arrow, altSide, ''), _data$offsets$arrow);\n\n  return data;\n}\n\n/**\n * Get the opposite placement variation of the given one\n * @method\n * @memberof Popper.Utils\n * @argument {String} placement variation\n * @returns {String} flipped placement variation\n */\nfunction getOppositeVariation(variation) {\n  if (variation === 'end') {\n    return 'start';\n  } else if (variation === 'start') {\n    return 'end';\n  }\n  return variation;\n}\n\n/**\n * List of accepted placements to use as values of the `placement` option.<br />\n * Valid placements are:\n * - `auto`\n * - `top`\n * - `right`\n * - `bottom`\n * - `left`\n *\n * Each placement can have a variation from this list:\n * - `-start`\n * - `-end`\n *\n * Variations are interpreted easily if you think of them as the left to right\n * written languages. Horizontally (`top` and `bottom`), `start` is left and `end`\n * is right.<br />\n * Vertically (`left` and `right`), `start` is top and `end` is bottom.\n *\n * Some valid examples are:\n * - `top-end` (on top of reference, right aligned)\n * - `right-start` (on right of reference, top aligned)\n * - `bottom` (on bottom, centered)\n * - `auto-end` (on the side with more space available, alignment depends by placement)\n *\n * @static\n * @type {Array}\n * @enum {String}\n * @readonly\n * @method placements\n * @memberof Popper\n */\nvar placements = ['auto-start', 'auto', 'auto-end', 'top-start', 'top', 'top-end', 'right-start', 'right', 'right-end', 'bottom-end', 'bottom', 'bottom-start', 'left-end', 'left', 'left-start'];\n\n// Get rid of `auto` `auto-start` and `auto-end`\nvar validPlacements = placements.slice(3);\n\n/**\n * Given an initial placement, returns all the subsequent placements\n * clockwise (or counter-clockwise).\n *\n * @method\n * @memberof Popper.Utils\n * @argument {String} placement - A valid placement (it accepts variations)\n * @argument {Boolean} counter - Set to true to walk the placements counterclockwise\n * @returns {Array} placements including their variations\n */\nfunction clockwise(placement) {\n  var counter = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : false;\n\n  var index = validPlacements.indexOf(placement);\n  var arr = validPlacements.slice(index + 1).concat(validPlacements.slice(0, index));\n  return counter ? arr.reverse() : arr;\n}\n\nvar BEHAVIORS = {\n  FLIP: 'flip',\n  CLOCKWISE: 'clockwise',\n  COUNTERCLOCKWISE: 'counterclockwise'\n};\n\n/**\n * @function\n * @memberof Modifiers\n * @argument {Object} data - The data object generated by update method\n * @argument {Object} options - Modifiers configuration and options\n * @returns {Object} The data object, properly modified\n */\nfunction flip(data, options) {\n  // if `inner` modifier is enabled, we can't use the `flip` modifier\n  if (isModifierEnabled(data.instance.modifiers, 'inner')) {\n    return data;\n  }\n\n  if (data.flipped && data.placement === data.originalPlacement) {\n    // seems like flip is trying to loop, probably there's not enough space on any of the flippable sides\n    return data;\n  }\n\n  var boundaries = getBoundaries(data.instance.popper, data.instance.reference, options.padding, options.boundariesElement, data.positionFixed);\n\n  var placement = data.placement.split('-')[0];\n  var placementOpposite = getOppositePlacement(placement);\n  var variation = data.placement.split('-')[1] || '';\n\n  var flipOrder = [];\n\n  switch (options.behavior) {\n    case BEHAVIORS.FLIP:\n      flipOrder = [placement, placementOpposite];\n      break;\n    case BEHAVIORS.CLOCKWISE:\n      flipOrder = clockwise(placement);\n      break;\n    case BEHAVIORS.COUNTERCLOCKWISE:\n      flipOrder = clockwise(placement, true);\n      break;\n    default:\n      flipOrder = options.behavior;\n  }\n\n  flipOrder.forEach(function (step, index) {\n    if (placement !== step || flipOrder.length === index + 1) {\n      return data;\n    }\n\n    placement = data.placement.split('-')[0];\n    placementOpposite = getOppositePlacement(placement);\n\n    var popperOffsets = data.offsets.popper;\n    var refOffsets = data.offsets.reference;\n\n    // using floor because the reference offsets may contain decimals we are not going to consider here\n    var floor = Math.floor;\n    var overlapsRef = placement === 'left' && floor(popperOffsets.right) > floor(refOffsets.left) || placement === 'right' && floor(popperOffsets.left) < floor(refOffsets.right) || placement === 'top' && floor(popperOffsets.bottom) > floor(refOffsets.top) || placement === 'bottom' && floor(popperOffsets.top) < floor(refOffsets.bottom);\n\n    var overflowsLeft = floor(popperOffsets.left) < floor(boundaries.left);\n    var overflowsRight = floor(popperOffsets.right) > floor(boundaries.right);\n    var overflowsTop = floor(popperOffsets.top) < floor(boundaries.top);\n    var overflowsBottom = floor(popperOffsets.bottom) > floor(boundaries.bottom);\n\n    var overflowsBoundaries = placement === 'left' && overflowsLeft || placement === 'right' && overflowsRight || placement === 'top' && overflowsTop || placement === 'bottom' && overflowsBottom;\n\n    // flip the variation if required\n    var isVertical = ['top', 'bottom'].indexOf(placement) !== -1;\n\n    // flips variation if reference element overflows boundaries\n    var flippedVariationByRef = !!options.flipVariations && (isVertical && variation === 'start' && overflowsLeft || isVertical && variation === 'end' && overflowsRight || !isVertical && variation === 'start' && overflowsTop || !isVertical && variation === 'end' && overflowsBottom);\n\n    // flips variation if popper content overflows boundaries\n    var flippedVariationByContent = !!options.flipVariationsByContent && (isVertical && variation === 'start' && overflowsRight || isVertical && variation === 'end' && overflowsLeft || !isVertical && variation === 'start' && overflowsBottom || !isVertical && variation === 'end' && overflowsTop);\n\n    var flippedVariation = flippedVariationByRef || flippedVariationByContent;\n\n    if (overlapsRef || overflowsBoundaries || flippedVariation) {\n      // this boolean to detect any flip loop\n      data.flipped = true;\n\n      if (overlapsRef || overflowsBoundaries) {\n        placement = flipOrder[index + 1];\n      }\n\n      if (flippedVariation) {\n        variation = getOppositeVariation(variation);\n      }\n\n      data.placement = placement + (variation ? '-' + variation : '');\n\n      // this object contains `position`, we want to preserve it along with\n      // any additional property we may add in the future\n      data.offsets.popper = _extends({}, data.offsets.popper, getPopperOffsets(data.instance.popper, data.offsets.reference, data.placement));\n\n      data = runModifiers(data.instance.modifiers, data, 'flip');\n    }\n  });\n  return data;\n}\n\n/**\n * @function\n * @memberof Modifiers\n * @argument {Object} data - The data object generated by update method\n * @argument {Object} options - Modifiers configuration and options\n * @returns {Object} The data object, properly modified\n */\nfunction keepTogether(data) {\n  var _data$offsets = data.offsets,\n      popper = _data$offsets.popper,\n      reference = _data$offsets.reference;\n\n  var placement = data.placement.split('-')[0];\n  var floor = Math.floor;\n  var isVertical = ['top', 'bottom'].indexOf(placement) !== -1;\n  var side = isVertical ? 'right' : 'bottom';\n  var opSide = isVertical ? 'left' : 'top';\n  var measurement = isVertical ? 'width' : 'height';\n\n  if (popper[side] < floor(reference[opSide])) {\n    data.offsets.popper[opSide] = floor(reference[opSide]) - popper[measurement];\n  }\n  if (popper[opSide] > floor(reference[side])) {\n    data.offsets.popper[opSide] = floor(reference[side]);\n  }\n\n  return data;\n}\n\n/**\n * Converts a string containing value + unit into a px value number\n * @function\n * @memberof {modifiers~offset}\n * @private\n * @argument {String} str - Value + unit string\n * @argument {String} measurement - `height` or `width`\n * @argument {Object} popperOffsets\n * @argument {Object} referenceOffsets\n * @returns {Number|String}\n * Value in pixels, or original string if no values were extracted\n */\nfunction toValue(str, measurement, popperOffsets, referenceOffsets) {\n  // separate value from unit\n  var split = str.match(/((?:\\-|\\+)?\\d*\\.?\\d*)(.*)/);\n  var value = +split[1];\n  var unit = split[2];\n\n  // If it's not a number it's an operator, I guess\n  if (!value) {\n    return str;\n  }\n\n  if (unit.indexOf('%') === 0) {\n    var element = void 0;\n    switch (unit) {\n      case '%p':\n        element = popperOffsets;\n        break;\n      case '%':\n      case '%r':\n      default:\n        element = referenceOffsets;\n    }\n\n    var rect = getClientRect(element);\n    return rect[measurement] / 100 * value;\n  } else if (unit === 'vh' || unit === 'vw') {\n    // if is a vh or vw, we calculate the size based on the viewport\n    var size = void 0;\n    if (unit === 'vh') {\n      size = Math.max(document.documentElement.clientHeight, window.innerHeight || 0);\n    } else {\n      size = Math.max(document.documentElement.clientWidth, window.innerWidth || 0);\n    }\n    return size / 100 * value;\n  } else {\n    // if is an explicit pixel unit, we get rid of the unit and keep the value\n    // if is an implicit unit, it's px, and we return just the value\n    return value;\n  }\n}\n\n/**\n * Parse an `offset` string to extrapolate `x` and `y` numeric offsets.\n * @function\n * @memberof {modifiers~offset}\n * @private\n * @argument {String} offset\n * @argument {Object} popperOffsets\n * @argument {Object} referenceOffsets\n * @argument {String} basePlacement\n * @returns {Array} a two cells array with x and y offsets in numbers\n */\nfunction parseOffset(offset, popperOffsets, referenceOffsets, basePlacement) {\n  var offsets = [0, 0];\n\n  // Use height if placement is left or right and index is 0 otherwise use width\n  // in this way the first offset will use an axis and the second one\n  // will use the other one\n  var useHeight = ['right', 'left'].indexOf(basePlacement) !== -1;\n\n  // Split the offset string to obtain a list of values and operands\n  // The regex addresses values with the plus or minus sign in front (+10, -20, etc)\n  var fragments = offset.split(/(\\+|\\-)/).map(function (frag) {\n    return frag.trim();\n  });\n\n  // Detect if the offset string contains a pair of values or a single one\n  // they could be separated by comma or space\n  var divider = fragments.indexOf(find(fragments, function (frag) {\n    return frag.search(/,|\\s/) !== -1;\n  }));\n\n  if (fragments[divider] && fragments[divider].indexOf(',') === -1) {\n    console.warn('Offsets separated by white space(s) are deprecated, use a comma (,) instead.');\n  }\n\n  // If divider is found, we divide the list of values and operands to divide\n  // them by ofset X and Y.\n  var splitRegex = /\\s*,\\s*|\\s+/;\n  var ops = divider !== -1 ? [fragments.slice(0, divider).concat([fragments[divider].split(splitRegex)[0]]), [fragments[divider].split(splitRegex)[1]].concat(fragments.slice(divider + 1))] : [fragments];\n\n  // Convert the values with units to absolute pixels to allow our computations\n  ops = ops.map(function (op, index) {\n    // Most of the units rely on the orientation of the popper\n    var measurement = (index === 1 ? !useHeight : useHeight) ? 'height' : 'width';\n    var mergeWithPrevious = false;\n    return op\n    // This aggregates any `+` or `-` sign that aren't considered operators\n    // e.g.: 10 + +5 => [10, +, +5]\n    .reduce(function (a, b) {\n      if (a[a.length - 1] === '' && ['+', '-'].indexOf(b) !== -1) {\n        a[a.length - 1] = b;\n        mergeWithPrevious = true;\n        return a;\n      } else if (mergeWithPrevious) {\n        a[a.length - 1] += b;\n        mergeWithPrevious = false;\n        return a;\n      } else {\n        return a.concat(b);\n      }\n    }, [])\n    // Here we convert the string values into number values (in px)\n    .map(function (str) {\n      return toValue(str, measurement, popperOffsets, referenceOffsets);\n    });\n  });\n\n  // Loop trough the offsets arrays and execute the operations\n  ops.forEach(function (op, index) {\n    op.forEach(function (frag, index2) {\n      if (isNumeric(frag)) {\n        offsets[index] += frag * (op[index2 - 1] === '-' ? -1 : 1);\n      }\n    });\n  });\n  return offsets;\n}\n\n/**\n * @function\n * @memberof Modifiers\n * @argument {Object} data - The data object generated by update method\n * @argument {Object} options - Modifiers configuration and options\n * @argument {Number|String} options.offset=0\n * The offset value as described in the modifier description\n * @returns {Object} The data object, properly modified\n */\nfunction offset(data, _ref) {\n  var offset = _ref.offset;\n  var placement = data.placement,\n      _data$offsets = data.offsets,\n      popper = _data$offsets.popper,\n      reference = _data$offsets.reference;\n\n  var basePlacement = placement.split('-')[0];\n\n  var offsets = void 0;\n  if (isNumeric(+offset)) {\n    offsets = [+offset, 0];\n  } else {\n    offsets = parseOffset(offset, popper, reference, basePlacement);\n  }\n\n  if (basePlacement === 'left') {\n    popper.top += offsets[0];\n    popper.left -= offsets[1];\n  } else if (basePlacement === 'right') {\n    popper.top += offsets[0];\n    popper.left += offsets[1];\n  } else if (basePlacement === 'top') {\n    popper.left += offsets[0];\n    popper.top -= offsets[1];\n  } else if (basePlacement === 'bottom') {\n    popper.left += offsets[0];\n    popper.top += offsets[1];\n  }\n\n  data.popper = popper;\n  return data;\n}\n\n/**\n * @function\n * @memberof Modifiers\n * @argument {Object} data - The data object generated by `update` method\n * @argument {Object} options - Modifiers configuration and options\n * @returns {Object} The data object, properly modified\n */\nfunction preventOverflow(data, options) {\n  var boundariesElement = options.boundariesElement || getOffsetParent(data.instance.popper);\n\n  // If offsetParent is the reference element, we really want to\n  // go one step up and use the next offsetParent as reference to\n  // avoid to make this modifier completely useless and look like broken\n  if (data.instance.reference === boundariesElement) {\n    boundariesElement = getOffsetParent(boundariesElement);\n  }\n\n  // NOTE: DOM access here\n  // resets the popper's position so that the document size can be calculated excluding\n  // the size of the popper element itself\n  var transformProp = getSupportedPropertyName('transform');\n  var popperStyles = data.instance.popper.style; // assignment to help minification\n  var top = popperStyles.top,\n      left = popperStyles.left,\n      transform = popperStyles[transformProp];\n\n  popperStyles.top = '';\n  popperStyles.left = '';\n  popperStyles[transformProp] = '';\n\n  var boundaries = getBoundaries(data.instance.popper, data.instance.reference, options.padding, boundariesElement, data.positionFixed);\n\n  // NOTE: DOM access here\n  // restores the original style properties after the offsets have been computed\n  popperStyles.top = top;\n  popperStyles.left = left;\n  popperStyles[transformProp] = transform;\n\n  options.boundaries = boundaries;\n\n  var order = options.priority;\n  var popper = data.offsets.popper;\n\n  var check = {\n    primary: function primary(placement) {\n      var value = popper[placement];\n      if (popper[placement] < boundaries[placement] && !options.escapeWithReference) {\n        value = Math.max(popper[placement], boundaries[placement]);\n      }\n      return defineProperty({}, placement, value);\n    },\n    secondary: function secondary(placement) {\n      var mainSide = placement === 'right' ? 'left' : 'top';\n      var value = popper[mainSide];\n      if (popper[placement] > boundaries[placement] && !options.escapeWithReference) {\n        value = Math.min(popper[mainSide], boundaries[placement] - (placement === 'right' ? popper.width : popper.height));\n      }\n      return defineProperty({}, mainSide, value);\n    }\n  };\n\n  order.forEach(function (placement) {\n    var side = ['left', 'top'].indexOf(placement) !== -1 ? 'primary' : 'secondary';\n    popper = _extends({}, popper, check[side](placement));\n  });\n\n  data.offsets.popper = popper;\n\n  return data;\n}\n\n/**\n * @function\n * @memberof Modifiers\n * @argument {Object} data - The data object generated by `update` method\n * @argument {Object} options - Modifiers configuration and options\n * @returns {Object} The data object, properly modified\n */\nfunction shift(data) {\n  var placement = data.placement;\n  var basePlacement = placement.split('-')[0];\n  var shiftvariation = placement.split('-')[1];\n\n  // if shift shiftvariation is specified, run the modifier\n  if (shiftvariation) {\n    var _data$offsets = data.offsets,\n        reference = _data$offsets.reference,\n        popper = _data$offsets.popper;\n\n    var isVertical = ['bottom', 'top'].indexOf(basePlacement) !== -1;\n    var side = isVertical ? 'left' : 'top';\n    var measurement = isVertical ? 'width' : 'height';\n\n    var shiftOffsets = {\n      start: defineProperty({}, side, reference[side]),\n      end: defineProperty({}, side, reference[side] + reference[measurement] - popper[measurement])\n    };\n\n    data.offsets.popper = _extends({}, popper, shiftOffsets[shiftvariation]);\n  }\n\n  return data;\n}\n\n/**\n * @function\n * @memberof Modifiers\n * @argument {Object} data - The data object generated by update method\n * @argument {Object} options - Modifiers configuration and options\n * @returns {Object} The data object, properly modified\n */\nfunction hide(data) {\n  if (!isModifierRequired(data.instance.modifiers, 'hide', 'preventOverflow')) {\n    return data;\n  }\n\n  var refRect = data.offsets.reference;\n  var bound = find(data.instance.modifiers, function (modifier) {\n    return modifier.name === 'preventOverflow';\n  }).boundaries;\n\n  if (refRect.bottom < bound.top || refRect.left > bound.right || refRect.top > bound.bottom || refRect.right < bound.left) {\n    // Avoid unnecessary DOM access if visibility hasn't changed\n    if (data.hide === true) {\n      return data;\n    }\n\n    data.hide = true;\n    data.attributes['x-out-of-boundaries'] = '';\n  } else {\n    // Avoid unnecessary DOM access if visibility hasn't changed\n    if (data.hide === false) {\n      return data;\n    }\n\n    data.hide = false;\n    data.attributes['x-out-of-boundaries'] = false;\n  }\n\n  return data;\n}\n\n/**\n * @function\n * @memberof Modifiers\n * @argument {Object} data - The data object generated by `update` method\n * @argument {Object} options - Modifiers configuration and options\n * @returns {Object} The data object, properly modified\n */\nfunction inner(data) {\n  var placement = data.placement;\n  var basePlacement = placement.split('-')[0];\n  var _data$offsets = data.offsets,\n      popper = _data$offsets.popper,\n      reference = _data$offsets.reference;\n\n  var isHoriz = ['left', 'right'].indexOf(basePlacement) !== -1;\n\n  var subtractLength = ['top', 'left'].indexOf(basePlacement) === -1;\n\n  popper[isHoriz ? 'left' : 'top'] = reference[basePlacement] - (subtractLength ? popper[isHoriz ? 'width' : 'height'] : 0);\n\n  data.placement = getOppositePlacement(placement);\n  data.offsets.popper = getClientRect(popper);\n\n  return data;\n}\n\n/**\n * Modifier function, each modifier can have a function of this type assigned\n * to its `fn` property.<br />\n * These functions will be called on each update, this means that you must\n * make sure they are performant enough to avoid performance bottlenecks.\n *\n * @function ModifierFn\n * @argument {dataObject} data - The data object generated by `update` method\n * @argument {Object} options - Modifiers configuration and options\n * @returns {dataObject} The data object, properly modified\n */\n\n/**\n * Modifiers are plugins used to alter the behavior of your poppers.<br />\n * Popper.js uses a set of 9 modifiers to provide all the basic functionalities\n * needed by the library.\n *\n * Usually you don't want to override the `order`, `fn` and `onLoad` props.\n * All the other properties are configurations that could be tweaked.\n * @namespace modifiers\n */\nvar modifiers = {\n  /**\n   * Modifier used to shift the popper on the start or end of its reference\n   * element.<br />\n   * It will read the variation of the `placement` property.<br />\n   * It can be one either `-end` or `-start`.\n   * @memberof modifiers\n   * @inner\n   */\n  shift: {\n    /** @prop {number} order=100 - Index used to define the order of execution */\n    order: 100,\n    /** @prop {Boolean} enabled=true - Whether the modifier is enabled or not */\n    enabled: true,\n    /** @prop {ModifierFn} */\n    fn: shift\n  },\n\n  /**\n   * The `offset` modifier can shift your popper on both its axis.\n   *\n   * It accepts the following units:\n   * - `px` or unit-less, interpreted as pixels\n   * - `%` or `%r`, percentage relative to the length of the reference element\n   * - `%p`, percentage relative to the length of the popper element\n   * - `vw`, CSS viewport width unit\n   * - `vh`, CSS viewport height unit\n   *\n   * For length is intended the main axis relative to the placement of the popper.<br />\n   * This means that if the placement is `top` or `bottom`, the length will be the\n   * `width`. In case of `left` or `right`, it will be the `height`.\n   *\n   * You can provide a single value (as `Number` or `String`), or a pair of values\n   * as `String` divided by a comma or one (or more) white spaces.<br />\n   * The latter is a deprecated method because it leads to confusion and will be\n   * removed in v2.<br />\n   * Additionally, it accepts additions and subtractions between different units.\n   * Note that multiplications and divisions aren't supported.\n   *\n   * Valid examples are:\n   * ```\n   * 10\n   * '10%'\n   * '10, 10'\n   * '10%, 10'\n   * '10 + 10%'\n   * '10 - 5vh + 3%'\n   * '-10px + 5vh, 5px - 6%'\n   * ```\n   * > **NB**: If you desire to apply offsets to your poppers in a way that may make them overlap\n   * > with their reference element, unfortunately, you will have to disable the `flip` modifier.\n   * > You can read more on this at this [issue](https://github.com/FezVrasta/popper.js/issues/373).\n   *\n   * @memberof modifiers\n   * @inner\n   */\n  offset: {\n    /** @prop {number} order=200 - Index used to define the order of execution */\n    order: 200,\n    /** @prop {Boolean} enabled=true - Whether the modifier is enabled or not */\n    enabled: true,\n    /** @prop {ModifierFn} */\n    fn: offset,\n    /** @prop {Number|String} offset=0\n     * The offset value as described in the modifier description\n     */\n    offset: 0\n  },\n\n  /**\n   * Modifier used to prevent the popper from being positioned outside the boundary.\n   *\n   * A scenario exists where the reference itself is not within the boundaries.<br />\n   * We can say it has \"escaped the boundaries\" — or just \"escaped\".<br />\n   * In this case we need to decide whether the popper should either:\n   *\n   * - detach from the reference and remain \"trapped\" in the boundaries, or\n   * - if it should ignore the boundary and \"escape with its reference\"\n   *\n   * When `escapeWithReference` is set to`true` and reference is completely\n   * outside its boundaries, the popper will overflow (or completely leave)\n   * the boundaries in order to remain attached to the edge of the reference.\n   *\n   * @memberof modifiers\n   * @inner\n   */\n  preventOverflow: {\n    /** @prop {number} order=300 - Index used to define the order of execution */\n    order: 300,\n    /** @prop {Boolean} enabled=true - Whether the modifier is enabled or not */\n    enabled: true,\n    /** @prop {ModifierFn} */\n    fn: preventOverflow,\n    /**\n     * @prop {Array} [priority=['left','right','top','bottom']]\n     * Popper will try to prevent overflow following these priorities by default,\n     * then, it could overflow on the left and on top of the `boundariesElement`\n     */\n    priority: ['left', 'right', 'top', 'bottom'],\n    /**\n     * @prop {number} padding=5\n     * Amount of pixel used to define a minimum distance between the boundaries\n     * and the popper. This makes sure the popper always has a little padding\n     * between the edges of its container\n     */\n    padding: 5,\n    /**\n     * @prop {String|HTMLElement} boundariesElement='scrollParent'\n     * Boundaries used by the modifier. Can be `scrollParent`, `window`,\n     * `viewport` or any DOM element.\n     */\n    boundariesElement: 'scrollParent'\n  },\n\n  /**\n   * Modifier used to make sure the reference and its popper stay near each other\n   * without leaving any gap between the two. Especially useful when the arrow is\n   * enabled and you want to ensure that it points to its reference element.\n   * It cares only about the first axis. You can still have poppers with margin\n   * between the popper and its reference element.\n   * @memberof modifiers\n   * @inner\n   */\n  keepTogether: {\n    /** @prop {number} order=400 - Index used to define the order of execution */\n    order: 400,\n    /** @prop {Boolean} enabled=true - Whether the modifier is enabled or not */\n    enabled: true,\n    /** @prop {ModifierFn} */\n    fn: keepTogether\n  },\n\n  /**\n   * This modifier is used to move the `arrowElement` of the popper to make\n   * sure it is positioned between the reference element and its popper element.\n   * It will read the outer size of the `arrowElement` node to detect how many\n   * pixels of conjunction are needed.\n   *\n   * It has no effect if no `arrowElement` is provided.\n   * @memberof modifiers\n   * @inner\n   */\n  arrow: {\n    /** @prop {number} order=500 - Index used to define the order of execution */\n    order: 500,\n    /** @prop {Boolean} enabled=true - Whether the modifier is enabled or not */\n    enabled: true,\n    /** @prop {ModifierFn} */\n    fn: arrow,\n    /** @prop {String|HTMLElement} element='[x-arrow]' - Selector or node used as arrow */\n    element: '[x-arrow]'\n  },\n\n  /**\n   * Modifier used to flip the popper's placement when it starts to overlap its\n   * reference element.\n   *\n   * Requires the `preventOverflow` modifier before it in order to work.\n   *\n   * **NOTE:** this modifier will interrupt the current update cycle and will\n   * restart it if it detects the need to flip the placement.\n   * @memberof modifiers\n   * @inner\n   */\n  flip: {\n    /** @prop {number} order=600 - Index used to define the order of execution */\n    order: 600,\n    /** @prop {Boolean} enabled=true - Whether the modifier is enabled or not */\n    enabled: true,\n    /** @prop {ModifierFn} */\n    fn: flip,\n    /**\n     * @prop {String|Array} behavior='flip'\n     * The behavior used to change the popper's placement. It can be one of\n     * `flip`, `clockwise`, `counterclockwise` or an array with a list of valid\n     * placements (with optional variations)\n     */\n    behavior: 'flip',\n    /**\n     * @prop {number} padding=5\n     * The popper will flip if it hits the edges of the `boundariesElement`\n     */\n    padding: 5,\n    /**\n     * @prop {String|HTMLElement} boundariesElement='viewport'\n     * The element which will define the boundaries of the popper position.\n     * The popper will never be placed outside of the defined boundaries\n     * (except if `keepTogether` is enabled)\n     */\n    boundariesElement: 'viewport',\n    /**\n     * @prop {Boolean} flipVariations=false\n     * The popper will switch placement variation between `-start` and `-end` when\n     * the reference element overlaps its boundaries.\n     *\n     * The original placement should have a set variation.\n     */\n    flipVariations: false,\n    /**\n     * @prop {Boolean} flipVariationsByContent=false\n     * The popper will switch placement variation between `-start` and `-end` when\n     * the popper element overlaps its reference boundaries.\n     *\n     * The original placement should have a set variation.\n     */\n    flipVariationsByContent: false\n  },\n\n  /**\n   * Modifier used to make the popper flow toward the inner of the reference element.\n   * By default, when this modifier is disabled, the popper will be placed outside\n   * the reference element.\n   * @memberof modifiers\n   * @inner\n   */\n  inner: {\n    /** @prop {number} order=700 - Index used to define the order of execution */\n    order: 700,\n    /** @prop {Boolean} enabled=false - Whether the modifier is enabled or not */\n    enabled: false,\n    /** @prop {ModifierFn} */\n    fn: inner\n  },\n\n  /**\n   * Modifier used to hide the popper when its reference element is outside of the\n   * popper boundaries. It will set a `x-out-of-boundaries` attribute which can\n   * be used to hide with a CSS selector the popper when its reference is\n   * out of boundaries.\n   *\n   * Requires the `preventOverflow` modifier before it in order to work.\n   * @memberof modifiers\n   * @inner\n   */\n  hide: {\n    /** @prop {number} order=800 - Index used to define the order of execution */\n    order: 800,\n    /** @prop {Boolean} enabled=true - Whether the modifier is enabled or not */\n    enabled: true,\n    /** @prop {ModifierFn} */\n    fn: hide\n  },\n\n  /**\n   * Computes the style that will be applied to the popper element to gets\n   * properly positioned.\n   *\n   * Note that this modifier will not touch the DOM, it just prepares the styles\n   * so that `applyStyle` modifier can apply it. This separation is useful\n   * in case you need to replace `applyStyle` with a custom implementation.\n   *\n   * This modifier has `850` as `order` value to maintain backward compatibility\n   * with previous versions of Popper.js. Expect the modifiers ordering method\n   * to change in future major versions of the library.\n   *\n   * @memberof modifiers\n   * @inner\n   */\n  computeStyle: {\n    /** @prop {number} order=850 - Index used to define the order of execution */\n    order: 850,\n    /** @prop {Boolean} enabled=true - Whether the modifier is enabled or not */\n    enabled: true,\n    /** @prop {ModifierFn} */\n    fn: computeStyle,\n    /**\n     * @prop {Boolean} gpuAcceleration=true\n     * If true, it uses the CSS 3D transformation to position the popper.\n     * Otherwise, it will use the `top` and `left` properties\n     */\n    gpuAcceleration: true,\n    /**\n     * @prop {string} [x='bottom']\n     * Where to anchor the X axis (`bottom` or `top`). AKA X offset origin.\n     * Change this if your popper should grow in a direction different from `bottom`\n     */\n    x: 'bottom',\n    /**\n     * @prop {string} [x='left']\n     * Where to anchor the Y axis (`left` or `right`). AKA Y offset origin.\n     * Change this if your popper should grow in a direction different from `right`\n     */\n    y: 'right'\n  },\n\n  /**\n   * Applies the computed styles to the popper element.\n   *\n   * All the DOM manipulations are limited to this modifier. This is useful in case\n   * you want to integrate Popper.js inside a framework or view library and you\n   * want to delegate all the DOM manipulations to it.\n   *\n   * Note that if you disable this modifier, you must make sure the popper element\n   * has its position set to `absolute` before Popper.js can do its work!\n   *\n   * Just disable this modifier and define your own to achieve the desired effect.\n   *\n   * @memberof modifiers\n   * @inner\n   */\n  applyStyle: {\n    /** @prop {number} order=900 - Index used to define the order of execution */\n    order: 900,\n    /** @prop {Boolean} enabled=true - Whether the modifier is enabled or not */\n    enabled: true,\n    /** @prop {ModifierFn} */\n    fn: applyStyle,\n    /** @prop {Function} */\n    onLoad: applyStyleOnLoad,\n    /**\n     * @deprecated since version 1.10.0, the property moved to `computeStyle` modifier\n     * @prop {Boolean} gpuAcceleration=true\n     * If true, it uses the CSS 3D transformation to position the popper.\n     * Otherwise, it will use the `top` and `left` properties\n     */\n    gpuAcceleration: undefined\n  }\n};\n\n/**\n * The `dataObject` is an object containing all the information used by Popper.js.\n * This object is passed to modifiers and to the `onCreate` and `onUpdate` callbacks.\n * @name dataObject\n * @property {Object} data.instance The Popper.js instance\n * @property {String} data.placement Placement applied to popper\n * @property {String} data.originalPlacement Placement originally defined on init\n * @property {Boolean} data.flipped True if popper has been flipped by flip modifier\n * @property {Boolean} data.hide True if the reference element is out of boundaries, useful to know when to hide the popper\n * @property {HTMLElement} data.arrowElement Node used as arrow by arrow modifier\n * @property {Object} data.styles Any CSS property defined here will be applied to the popper. It expects the JavaScript nomenclature (eg. `marginBottom`)\n * @property {Object} data.arrowStyles Any CSS property defined here will be applied to the popper arrow. It expects the JavaScript nomenclature (eg. `marginBottom`)\n * @property {Object} data.boundaries Offsets of the popper boundaries\n * @property {Object} data.offsets The measurements of popper, reference and arrow elements\n * @property {Object} data.offsets.popper `top`, `left`, `width`, `height` values\n * @property {Object} data.offsets.reference `top`, `left`, `width`, `height` values\n * @property {Object} data.offsets.arrow] `top` and `left` offsets, only one of them will be different from 0\n */\n\n/**\n * Default options provided to Popper.js constructor.<br />\n * These can be overridden using the `options` argument of Popper.js.<br />\n * To override an option, simply pass an object with the same\n * structure of the `options` object, as the 3rd argument. For example:\n * ```\n * new Popper(ref, pop, {\n *   modifiers: {\n *     preventOverflow: { enabled: false }\n *   }\n * })\n * ```\n * @type {Object}\n * @static\n * @memberof Popper\n */\nvar Defaults = {\n  /**\n   * Popper's placement.\n   * @prop {Popper.placements} placement='bottom'\n   */\n  placement: 'bottom',\n\n  /**\n   * Set this to true if you want popper to position it self in 'fixed' mode\n   * @prop {Boolean} positionFixed=false\n   */\n  positionFixed: false,\n\n  /**\n   * Whether events (resize, scroll) are initially enabled.\n   * @prop {Boolean} eventsEnabled=true\n   */\n  eventsEnabled: true,\n\n  /**\n   * Set to true if you want to automatically remove the popper when\n   * you call the `destroy` method.\n   * @prop {Boolean} removeOnDestroy=false\n   */\n  removeOnDestroy: false,\n\n  /**\n   * Callback called when the popper is created.<br />\n   * By default, it is set to no-op.<br />\n   * Access Popper.js instance with `data.instance`.\n   * @prop {onCreate}\n   */\n  onCreate: function onCreate() {},\n\n  /**\n   * Callback called when the popper is updated. This callback is not called\n   * on the initialization/creation of the popper, but only on subsequent\n   * updates.<br />\n   * By default, it is set to no-op.<br />\n   * Access Popper.js instance with `data.instance`.\n   * @prop {onUpdate}\n   */\n  onUpdate: function onUpdate() {},\n\n  /**\n   * List of modifiers used to modify the offsets before they are applied to the popper.\n   * They provide most of the functionalities of Popper.js.\n   * @prop {modifiers}\n   */\n  modifiers: modifiers\n};\n\n/**\n * @callback onCreate\n * @param {dataObject} data\n */\n\n/**\n * @callback onUpdate\n * @param {dataObject} data\n */\n\n// Utils\n// Methods\nvar Popper = function () {\n  /**\n   * Creates a new Popper.js instance.\n   * @class Popper\n   * @param {Element|referenceObject} reference - The reference element used to position the popper\n   * @param {Element} popper - The HTML / XML element used as the popper\n   * @param {Object} options - Your custom options to override the ones defined in [Defaults](#defaults)\n   * @return {Object} instance - The generated Popper.js instance\n   */\n  function Popper(reference, popper) {\n    var _this = this;\n\n    var options = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : {};\n    classCallCheck(this, Popper);\n\n    this.scheduleUpdate = function () {\n      return requestAnimationFrame(_this.update);\n    };\n\n    // make update() debounced, so that it only runs at most once-per-tick\n    this.update = debounce(this.update.bind(this));\n\n    // with {} we create a new object with the options inside it\n    this.options = _extends({}, Popper.Defaults, options);\n\n    // init state\n    this.state = {\n      isDestroyed: false,\n      isCreated: false,\n      scrollParents: []\n    };\n\n    // get reference and popper elements (allow jQuery wrappers)\n    this.reference = reference && reference.jquery ? reference[0] : reference;\n    this.popper = popper && popper.jquery ? popper[0] : popper;\n\n    // Deep merge modifiers options\n    this.options.modifiers = {};\n    Object.keys(_extends({}, Popper.Defaults.modifiers, options.modifiers)).forEach(function (name) {\n      _this.options.modifiers[name] = _extends({}, Popper.Defaults.modifiers[name] || {}, options.modifiers ? options.modifiers[name] : {});\n    });\n\n    // Refactoring modifiers' list (Object => Array)\n    this.modifiers = Object.keys(this.options.modifiers).map(function (name) {\n      return _extends({\n        name: name\n      }, _this.options.modifiers[name]);\n    })\n    // sort the modifiers by order\n    .sort(function (a, b) {\n      return a.order - b.order;\n    });\n\n    // modifiers have the ability to execute arbitrary code when Popper.js get inited\n    // such code is executed in the same order of its modifier\n    // they could add new properties to their options configuration\n    // BE AWARE: don't add options to `options.modifiers.name` but to `modifierOptions`!\n    this.modifiers.forEach(function (modifierOptions) {\n      if (modifierOptions.enabled && isFunction(modifierOptions.onLoad)) {\n        modifierOptions.onLoad(_this.reference, _this.popper, _this.options, modifierOptions, _this.state);\n      }\n    });\n\n    // fire the first update to position the popper in the right place\n    this.update();\n\n    var eventsEnabled = this.options.eventsEnabled;\n    if (eventsEnabled) {\n      // setup event listeners, they will take care of update the position in specific situations\n      this.enableEventListeners();\n    }\n\n    this.state.eventsEnabled = eventsEnabled;\n  }\n\n  // We can't use class properties because they don't get listed in the\n  // class prototype and break stuff like Sinon stubs\n\n\n  createClass(Popper, [{\n    key: 'update',\n    value: function update$$1() {\n      return update.call(this);\n    }\n  }, {\n    key: 'destroy',\n    value: function destroy$$1() {\n      return destroy.call(this);\n    }\n  }, {\n    key: 'enableEventListeners',\n    value: function enableEventListeners$$1() {\n      return enableEventListeners.call(this);\n    }\n  }, {\n    key: 'disableEventListeners',\n    value: function disableEventListeners$$1() {\n      return disableEventListeners.call(this);\n    }\n\n    /**\n     * Schedules an update. It will run on the next UI update available.\n     * @method scheduleUpdate\n     * @memberof Popper\n     */\n\n\n    /**\n     * Collection of utilities useful when writing custom modifiers.\n     * Starting from version 1.7, this method is available only if you\n     * include `popper-utils.js` before `popper.js`.\n     *\n     * **DEPRECATION**: This way to access PopperUtils is deprecated\n     * and will be removed in v2! Use the PopperUtils module directly instead.\n     * Due to the high instability of the methods contained in Utils, we can't\n     * guarantee them to follow semver. Use them at your own risk!\n     * @static\n     * @private\n     * @type {Object}\n     * @deprecated since version 1.8\n     * @member Utils\n     * @memberof Popper\n     */\n\n  }]);\n  return Popper;\n}();\n\n/**\n * The `referenceObject` is an object that provides an interface compatible with Popper.js\n * and lets you use it as replacement of a real DOM node.<br />\n * You can use this method to position a popper relatively to a set of coordinates\n * in case you don't have a DOM node to use as reference.\n *\n * ```\n * new Popper(referenceObject, popperNode);\n * ```\n *\n * NB: This feature isn't supported in Internet Explorer 10.\n * @name referenceObject\n * @property {Function} data.getBoundingClientRect\n * A function that returns a set of coordinates compatible with the native `getBoundingClientRect` method.\n * @property {number} data.clientWidth\n * An ES6 getter that will return the width of the virtual reference element.\n * @property {number} data.clientHeight\n * An ES6 getter that will return the height of the virtual reference element.\n */\n\n\nPopper.Utils = (typeof window !== 'undefined' ? window : global).PopperUtils;\nPopper.placements = placements;\nPopper.Defaults = Defaults;\n\nexport default Popper;\n//# sourceMappingURL=popper.js.map\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap (v4.4.1): dropdown.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport $ from 'jquery'\nimport Popper from 'popper.js'\nimport Util from './util'\n\n/**\n * ------------------------------------------------------------------------\n * Constants\n * ------------------------------------------------------------------------\n */\n\nconst NAME                     = 'dropdown'\nconst VERSION                  = '4.4.1'\nconst DATA_KEY                 = 'bs.dropdown'\nconst EVENT_KEY                = `.${DATA_KEY}`\nconst DATA_API_KEY             = '.data-api'\nconst JQUERY_NO_CONFLICT       = $.fn[NAME]\nconst ESCAPE_KEYCODE           = 27 // KeyboardEvent.which value for Escape (Esc) key\nconst SPACE_KEYCODE            = 32 // KeyboardEvent.which value for space key\nconst TAB_KEYCODE              = 9 // KeyboardEvent.which value for tab key\nconst ARROW_UP_KEYCODE         = 38 // KeyboardEvent.which value for up arrow key\nconst ARROW_DOWN_KEYCODE       = 40 // KeyboardEvent.which value for down arrow key\nconst RIGHT_MOUSE_BUTTON_WHICH = 3 // MouseEvent.which value for the right button (assuming a right-handed mouse)\nconst REGEXP_KEYDOWN           = new RegExp(`${ARROW_UP_KEYCODE}|${ARROW_DOWN_KEYCODE}|${ESCAPE_KEYCODE}`)\n\nconst Event = {\n  HIDE             : `hide${EVENT_KEY}`,\n  HIDDEN           : `hidden${EVENT_KEY}`,\n  SHOW             : `show${EVENT_KEY}`,\n  SHOWN            : `shown${EVENT_KEY}`,\n  CLICK            : `click${EVENT_KEY}`,\n  CLICK_DATA_API   : `click${EVENT_KEY}${DATA_API_KEY}`,\n  KEYDOWN_DATA_API : `keydown${EVENT_KEY}${DATA_API_KEY}`,\n  KEYUP_DATA_API   : `keyup${EVENT_KEY}${DATA_API_KEY}`\n}\n\nconst ClassName = {\n  DISABLED        : 'disabled',\n  SHOW            : 'show',\n  DROPUP          : 'dropup',\n  DROPRIGHT       : 'dropright',\n  DROPLEFT        : 'dropleft',\n  MENURIGHT       : 'dropdown-menu-right',\n  MENULEFT        : 'dropdown-menu-left',\n  POSITION_STATIC : 'position-static'\n}\n\nconst Selector = {\n  DATA_TOGGLE   : '[data-toggle=\"dropdown\"]',\n  FORM_CHILD    : '.dropdown form',\n  MENU          : '.dropdown-menu',\n  NAVBAR_NAV    : '.navbar-nav',\n  VISIBLE_ITEMS : '.dropdown-menu .dropdown-item:not(.disabled):not(:disabled)'\n}\n\nconst AttachmentMap = {\n  TOP       : 'top-start',\n  TOPEND    : 'top-end',\n  BOTTOM    : 'bottom-start',\n  BOTTOMEND : 'bottom-end',\n  RIGHT     : 'right-start',\n  RIGHTEND  : 'right-end',\n  LEFT      : 'left-start',\n  LEFTEND   : 'left-end'\n}\n\nconst Default = {\n  offset       : 0,\n  flip         : true,\n  boundary     : 'scrollParent',\n  reference    : 'toggle',\n  display      : 'dynamic',\n  popperConfig : null\n}\n\nconst DefaultType = {\n  offset       : '(number|string|function)',\n  flip         : 'boolean',\n  boundary     : '(string|element)',\n  reference    : '(string|element)',\n  display      : 'string',\n  popperConfig : '(null|object)'\n}\n\n/**\n * ------------------------------------------------------------------------\n * Class Definition\n * ------------------------------------------------------------------------\n */\n\nclass Dropdown {\n  constructor(element, config) {\n    this._element  = element\n    this._popper   = null\n    this._config   = this._getConfig(config)\n    this._menu     = this._getMenuElement()\n    this._inNavbar = this._detectNavbar()\n\n    this._addEventListeners()\n  }\n\n  // Getters\n\n  static get VERSION() {\n    return VERSION\n  }\n\n  static get Default() {\n    return Default\n  }\n\n  static get DefaultType() {\n    return DefaultType\n  }\n\n  // Public\n\n  toggle() {\n    if (this._element.disabled || $(this._element).hasClass(ClassName.DISABLED)) {\n      return\n    }\n\n    const isActive = $(this._menu).hasClass(ClassName.SHOW)\n\n    Dropdown._clearMenus()\n\n    if (isActive) {\n      return\n    }\n\n    this.show(true)\n  }\n\n  show(usePopper = false) {\n    if (this._element.disabled || $(this._element).hasClass(ClassName.DISABLED) || $(this._menu).hasClass(ClassName.SHOW)) {\n      return\n    }\n\n    const relatedTarget = {\n      relatedTarget: this._element\n    }\n    const showEvent = $.Event(Event.SHOW, relatedTarget)\n    const parent = Dropdown._getParentFromElement(this._element)\n\n    $(parent).trigger(showEvent)\n\n    if (showEvent.isDefaultPrevented()) {\n      return\n    }\n\n    // Disable totally Popper.js for Dropdown in Navbar\n    if (!this._inNavbar && usePopper) {\n      /**\n       * Check for Popper dependency\n       * Popper - https://popper.js.org\n       */\n      if (typeof Popper === 'undefined') {\n        throw new TypeError('Bootstrap\\'s dropdowns require Popper.js (https://popper.js.org/)')\n      }\n\n      let referenceElement = this._element\n\n      if (this._config.reference === 'parent') {\n        referenceElement = parent\n      } else if (Util.isElement(this._config.reference)) {\n        referenceElement = this._config.reference\n\n        // Check if it's jQuery element\n        if (typeof this._config.reference.jquery !== 'undefined') {\n          referenceElement = this._config.reference[0]\n        }\n      }\n\n      // If boundary is not `scrollParent`, then set position to `static`\n      // to allow the menu to \"escape\" the scroll parent's boundaries\n      // https://github.com/twbs/bootstrap/issues/24251\n      if (this._config.boundary !== 'scrollParent') {\n        $(parent).addClass(ClassName.POSITION_STATIC)\n      }\n      this._popper = new Popper(referenceElement, this._menu, this._getPopperConfig())\n    }\n\n    // If this is a touch-enabled device we add extra\n    // empty mouseover listeners to the body's immediate children;\n    // only needed because of broken event delegation on iOS\n    // https://www.quirksmode.org/blog/archives/2014/02/mouse_event_bub.html\n    if ('ontouchstart' in document.documentElement &&\n        $(parent).closest(Selector.NAVBAR_NAV).length === 0) {\n      $(document.body).children().on('mouseover', null, $.noop)\n    }\n\n    this._element.focus()\n    this._element.setAttribute('aria-expanded', true)\n\n    $(this._menu).toggleClass(ClassName.SHOW)\n    $(parent)\n      .toggleClass(ClassName.SHOW)\n      .trigger($.Event(Event.SHOWN, relatedTarget))\n  }\n\n  hide() {\n    if (this._element.disabled || $(this._element).hasClass(ClassName.DISABLED) || !$(this._menu).hasClass(ClassName.SHOW)) {\n      return\n    }\n\n    const relatedTarget = {\n      relatedTarget: this._element\n    }\n    const hideEvent = $.Event(Event.HIDE, relatedTarget)\n    const parent = Dropdown._getParentFromElement(this._element)\n\n    $(parent).trigger(hideEvent)\n\n    if (hideEvent.isDefaultPrevented()) {\n      return\n    }\n\n    if (this._popper) {\n      this._popper.destroy()\n    }\n\n    $(this._menu).toggleClass(ClassName.SHOW)\n    $(parent)\n      .toggleClass(ClassName.SHOW)\n      .trigger($.Event(Event.HIDDEN, relatedTarget))\n  }\n\n  dispose() {\n    $.removeData(this._element, DATA_KEY)\n    $(this._element).off(EVENT_KEY)\n    this._element = null\n    this._menu = null\n    if (this._popper !== null) {\n      this._popper.destroy()\n      this._popper = null\n    }\n  }\n\n  update() {\n    this._inNavbar = this._detectNavbar()\n    if (this._popper !== null) {\n      this._popper.scheduleUpdate()\n    }\n  }\n\n  // Private\n\n  _addEventListeners() {\n    $(this._element).on(Event.CLICK, (event) => {\n      event.preventDefault()\n      event.stopPropagation()\n      this.toggle()\n    })\n  }\n\n  _getConfig(config) {\n    config = {\n      ...this.constructor.Default,\n      ...$(this._element).data(),\n      ...config\n    }\n\n    Util.typeCheckConfig(\n      NAME,\n      config,\n      this.constructor.DefaultType\n    )\n\n    return config\n  }\n\n  _getMenuElement() {\n    if (!this._menu) {\n      const parent = Dropdown._getParentFromElement(this._element)\n\n      if (parent) {\n        this._menu = parent.querySelector(Selector.MENU)\n      }\n    }\n    return this._menu\n  }\n\n  _getPlacement() {\n    const $parentDropdown = $(this._element.parentNode)\n    let placement = AttachmentMap.BOTTOM\n\n    // Handle dropup\n    if ($parentDropdown.hasClass(ClassName.DROPUP)) {\n      placement = AttachmentMap.TOP\n      if ($(this._menu).hasClass(ClassName.MENURIGHT)) {\n        placement = AttachmentMap.TOPEND\n      }\n    } else if ($parentDropdown.hasClass(ClassName.DROPRIGHT)) {\n      placement = AttachmentMap.RIGHT\n    } else if ($parentDropdown.hasClass(ClassName.DROPLEFT)) {\n      placement = AttachmentMap.LEFT\n    } else if ($(this._menu).hasClass(ClassName.MENURIGHT)) {\n      placement = AttachmentMap.BOTTOMEND\n    }\n    return placement\n  }\n\n  _detectNavbar() {\n    return $(this._element).closest('.navbar').length > 0\n  }\n\n  _getOffset() {\n    const offset = {}\n\n    if (typeof this._config.offset === 'function') {\n      offset.fn = (data) => {\n        data.offsets = {\n          ...data.offsets,\n          ...this._config.offset(data.offsets, this._element) || {}\n        }\n\n        return data\n      }\n    } else {\n      offset.offset = this._config.offset\n    }\n\n    return offset\n  }\n\n  _getPopperConfig() {\n    const popperConfig = {\n      placement: this._getPlacement(),\n      modifiers: {\n        offset: this._getOffset(),\n        flip: {\n          enabled: this._config.flip\n        },\n        preventOverflow: {\n          boundariesElement: this._config.boundary\n        }\n      }\n    }\n\n    // Disable Popper.js if we have a static display\n    if (this._config.display === 'static') {\n      popperConfig.modifiers.applyStyle = {\n        enabled: false\n      }\n    }\n\n    return {\n      ...popperConfig,\n      ...this._config.popperConfig\n    }\n  }\n\n  // Static\n\n  static _jQueryInterface(config) {\n    return this.each(function () {\n      let data = $(this).data(DATA_KEY)\n      const _config = typeof config === 'object' ? config : null\n\n      if (!data) {\n        data = new Dropdown(this, _config)\n        $(this).data(DATA_KEY, data)\n      }\n\n      if (typeof config === 'string') {\n        if (typeof data[config] === 'undefined') {\n          throw new TypeError(`No method named \"${config}\"`)\n        }\n        data[config]()\n      }\n    })\n  }\n\n  static _clearMenus(event) {\n    if (event && (event.which === RIGHT_MOUSE_BUTTON_WHICH ||\n      event.type === 'keyup' && event.which !== TAB_KEYCODE)) {\n      return\n    }\n\n    const toggles = [].slice.call(document.querySelectorAll(Selector.DATA_TOGGLE))\n\n    for (let i = 0, len = toggles.length; i < len; i++) {\n      const parent = Dropdown._getParentFromElement(toggles[i])\n      const context = $(toggles[i]).data(DATA_KEY)\n      const relatedTarget = {\n        relatedTarget: toggles[i]\n      }\n\n      if (event && event.type === 'click') {\n        relatedTarget.clickEvent = event\n      }\n\n      if (!context) {\n        continue\n      }\n\n      const dropdownMenu = context._menu\n      if (!$(parent).hasClass(ClassName.SHOW)) {\n        continue\n      }\n\n      if (event && (event.type === 'click' &&\n          /input|textarea/i.test(event.target.tagName) || event.type === 'keyup' && event.which === TAB_KEYCODE) &&\n          $.contains(parent, event.target)) {\n        continue\n      }\n\n      const hideEvent = $.Event(Event.HIDE, relatedTarget)\n      $(parent).trigger(hideEvent)\n      if (hideEvent.isDefaultPrevented()) {\n        continue\n      }\n\n      // If this is a touch-enabled device we remove the extra\n      // empty mouseover listeners we added for iOS support\n      if ('ontouchstart' in document.documentElement) {\n        $(document.body).children().off('mouseover', null, $.noop)\n      }\n\n      toggles[i].setAttribute('aria-expanded', 'false')\n\n      if (context._popper) {\n        context._popper.destroy()\n      }\n\n      $(dropdownMenu).removeClass(ClassName.SHOW)\n      $(parent)\n        .removeClass(ClassName.SHOW)\n        .trigger($.Event(Event.HIDDEN, relatedTarget))\n    }\n  }\n\n  static _getParentFromElement(element) {\n    let parent\n    const selector = Util.getSelectorFromElement(element)\n\n    if (selector) {\n      parent = document.querySelector(selector)\n    }\n\n    return parent || element.parentNode\n  }\n\n  // eslint-disable-next-line complexity\n  static _dataApiKeydownHandler(event) {\n    // If not input/textarea:\n    //  - And not a key in REGEXP_KEYDOWN => not a dropdown command\n    // If input/textarea:\n    //  - If space key => not a dropdown command\n    //  - If key is other than escape\n    //    - If key is not up or down => not a dropdown command\n    //    - If trigger inside the menu => not a dropdown command\n    if (/input|textarea/i.test(event.target.tagName)\n      ? event.which === SPACE_KEYCODE || event.which !== ESCAPE_KEYCODE &&\n      (event.which !== ARROW_DOWN_KEYCODE && event.which !== ARROW_UP_KEYCODE ||\n        $(event.target).closest(Selector.MENU).length) : !REGEXP_KEYDOWN.test(event.which)) {\n      return\n    }\n\n    event.preventDefault()\n    event.stopPropagation()\n\n    if (this.disabled || $(this).hasClass(ClassName.DISABLED)) {\n      return\n    }\n\n    const parent   = Dropdown._getParentFromElement(this)\n    const isActive = $(parent).hasClass(ClassName.SHOW)\n\n    if (!isActive && event.which === ESCAPE_KEYCODE) {\n      return\n    }\n\n    if (!isActive || isActive && (event.which === ESCAPE_KEYCODE || event.which === SPACE_KEYCODE)) {\n      if (event.which === ESCAPE_KEYCODE) {\n        const toggle = parent.querySelector(Selector.DATA_TOGGLE)\n        $(toggle).trigger('focus')\n      }\n\n      $(this).trigger('click')\n      return\n    }\n\n    const items = [].slice.call(parent.querySelectorAll(Selector.VISIBLE_ITEMS))\n      .filter((item) => $(item).is(':visible'))\n\n    if (items.length === 0) {\n      return\n    }\n\n    let index = items.indexOf(event.target)\n\n    if (event.which === ARROW_UP_KEYCODE && index > 0) { // Up\n      index--\n    }\n\n    if (event.which === ARROW_DOWN_KEYCODE && index < items.length - 1) { // Down\n      index++\n    }\n\n    if (index < 0) {\n      index = 0\n    }\n\n    items[index].focus()\n  }\n}\n\n/**\n * ------------------------------------------------------------------------\n * Data Api implementation\n * ------------------------------------------------------------------------\n */\n\n$(document)\n  .on(Event.KEYDOWN_DATA_API, Selector.DATA_TOGGLE, Dropdown._dataApiKeydownHandler)\n  .on(Event.KEYDOWN_DATA_API, Selector.MENU, Dropdown._dataApiKeydownHandler)\n  .on(`${Event.CLICK_DATA_API} ${Event.KEYUP_DATA_API}`, Dropdown._clearMenus)\n  .on(Event.CLICK_DATA_API, Selector.DATA_TOGGLE, function (event) {\n    event.preventDefault()\n    event.stopPropagation()\n    Dropdown._jQueryInterface.call($(this), 'toggle')\n  })\n  .on(Event.CLICK_DATA_API, Selector.FORM_CHILD, (e) => {\n    e.stopPropagation()\n  })\n\n/**\n * ------------------------------------------------------------------------\n * jQuery\n * ------------------------------------------------------------------------\n */\n\n$.fn[NAME] = Dropdown._jQueryInterface\n$.fn[NAME].Constructor = Dropdown\n$.fn[NAME].noConflict = () => {\n  $.fn[NAME] = JQUERY_NO_CONFLICT\n  return Dropdown._jQueryInterface\n}\n\n\nexport default Dropdown\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap (v4.4.1): modal.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport $ from 'jquery'\nimport Util from './util'\n\n/**\n * ------------------------------------------------------------------------\n * Constants\n * ------------------------------------------------------------------------\n */\n\nconst NAME               = 'modal'\nconst VERSION            = '4.4.1'\nconst DATA_KEY           = 'bs.modal'\nconst EVENT_KEY          = `.${DATA_KEY}`\nconst DATA_API_KEY       = '.data-api'\nconst JQUERY_NO_CONFLICT = $.fn[NAME]\nconst ESCAPE_KEYCODE     = 27 // KeyboardEvent.which value for Escape (Esc) key\n\nconst Default = {\n  backdrop : true,\n  keyboard : true,\n  focus    : true,\n  show     : true\n}\n\nconst DefaultType = {\n  backdrop : '(boolean|string)',\n  keyboard : 'boolean',\n  focus    : 'boolean',\n  show     : 'boolean'\n}\n\nconst Event = {\n  HIDE              : `hide${EVENT_KEY}`,\n  HIDE_PREVENTED    : `hidePrevented${EVENT_KEY}`,\n  HIDDEN            : `hidden${EVENT_KEY}`,\n  SHOW              : `show${EVENT_KEY}`,\n  SHOWN             : `shown${EVENT_KEY}`,\n  FOCUSIN           : `focusin${EVENT_KEY}`,\n  RESIZE            : `resize${EVENT_KEY}`,\n  CLICK_DISMISS     : `click.dismiss${EVENT_KEY}`,\n  KEYDOWN_DISMISS   : `keydown.dismiss${EVENT_KEY}`,\n  MOUSEUP_DISMISS   : `mouseup.dismiss${EVENT_KEY}`,\n  MOUSEDOWN_DISMISS : `mousedown.dismiss${EVENT_KEY}`,\n  CLICK_DATA_API    : `click${EVENT_KEY}${DATA_API_KEY}`\n}\n\nconst ClassName = {\n  SCROLLABLE         : 'modal-dialog-scrollable',\n  SCROLLBAR_MEASURER : 'modal-scrollbar-measure',\n  BACKDROP           : 'modal-backdrop',\n  OPEN               : 'modal-open',\n  FADE               : 'fade',\n  SHOW               : 'show',\n  STATIC             : 'modal-static'\n}\n\nconst Selector = {\n  DIALOG         : '.modal-dialog',\n  MODAL_BODY     : '.modal-body',\n  DATA_TOGGLE    : '[data-toggle=\"modal\"]',\n  DATA_DISMISS   : '[data-dismiss=\"modal\"]',\n  FIXED_CONTENT  : '.fixed-top, .fixed-bottom, .is-fixed, .sticky-top',\n  STICKY_CONTENT : '.sticky-top'\n}\n\n/**\n * ------------------------------------------------------------------------\n * Class Definition\n * ------------------------------------------------------------------------\n */\n\nclass Modal {\n  constructor(element, config) {\n    this._config              = this._getConfig(config)\n    this._element             = element\n    this._dialog              = element.querySelector(Selector.DIALOG)\n    this._backdrop            = null\n    this._isShown             = false\n    this._isBodyOverflowing   = false\n    this._ignoreBackdropClick = false\n    this._isTransitioning     = false\n    this._scrollbarWidth      = 0\n  }\n\n  // Getters\n\n  static get VERSION() {\n    return VERSION\n  }\n\n  static get Default() {\n    return Default\n  }\n\n  // Public\n\n  toggle(relatedTarget) {\n    return this._isShown ? this.hide() : this.show(relatedTarget)\n  }\n\n  show(relatedTarget) {\n    if (this._isShown || this._isTransitioning) {\n      return\n    }\n\n    if ($(this._element).hasClass(ClassName.FADE)) {\n      this._isTransitioning = true\n    }\n\n    const showEvent = $.Event(Event.SHOW, {\n      relatedTarget\n    })\n\n    $(this._element).trigger(showEvent)\n\n    if (this._isShown || showEvent.isDefaultPrevented()) {\n      return\n    }\n\n    this._isShown = true\n\n    this._checkScrollbar()\n    this._setScrollbar()\n\n    this._adjustDialog()\n\n    this._setEscapeEvent()\n    this._setResizeEvent()\n\n    $(this._element).on(\n      Event.CLICK_DISMISS,\n      Selector.DATA_DISMISS,\n      (event) => this.hide(event)\n    )\n\n    $(this._dialog).on(Event.MOUSEDOWN_DISMISS, () => {\n      $(this._element).one(Event.MOUSEUP_DISMISS, (event) => {\n        if ($(event.target).is(this._element)) {\n          this._ignoreBackdropClick = true\n        }\n      })\n    })\n\n    this._showBackdrop(() => this._showElement(relatedTarget))\n  }\n\n  hide(event) {\n    if (event) {\n      event.preventDefault()\n    }\n\n    if (!this._isShown || this._isTransitioning) {\n      return\n    }\n\n    const hideEvent = $.Event(Event.HIDE)\n\n    $(this._element).trigger(hideEvent)\n\n    if (!this._isShown || hideEvent.isDefaultPrevented()) {\n      return\n    }\n\n    this._isShown = false\n    const transition = $(this._element).hasClass(ClassName.FADE)\n\n    if (transition) {\n      this._isTransitioning = true\n    }\n\n    this._setEscapeEvent()\n    this._setResizeEvent()\n\n    $(document).off(Event.FOCUSIN)\n\n    $(this._element).removeClass(ClassName.SHOW)\n\n    $(this._element).off(Event.CLICK_DISMISS)\n    $(this._dialog).off(Event.MOUSEDOWN_DISMISS)\n\n\n    if (transition) {\n      const transitionDuration  = Util.getTransitionDurationFromElement(this._element)\n\n      $(this._element)\n        .one(Util.TRANSITION_END, (event) => this._hideModal(event))\n        .emulateTransitionEnd(transitionDuration)\n    } else {\n      this._hideModal()\n    }\n  }\n\n  dispose() {\n    [window, this._element, this._dialog]\n      .forEach((htmlElement) => $(htmlElement).off(EVENT_KEY))\n\n    /**\n     * `document` has 2 events `Event.FOCUSIN` and `Event.CLICK_DATA_API`\n     * Do not move `document` in `htmlElements` array\n     * It will remove `Event.CLICK_DATA_API` event that should remain\n     */\n    $(document).off(Event.FOCUSIN)\n\n    $.removeData(this._element, DATA_KEY)\n\n    this._config              = null\n    this._element             = null\n    this._dialog              = null\n    this._backdrop            = null\n    this._isShown             = null\n    this._isBodyOverflowing   = null\n    this._ignoreBackdropClick = null\n    this._isTransitioning     = null\n    this._scrollbarWidth      = null\n  }\n\n  handleUpdate() {\n    this._adjustDialog()\n  }\n\n  // Private\n\n  _getConfig(config) {\n    config = {\n      ...Default,\n      ...config\n    }\n    Util.typeCheckConfig(NAME, config, DefaultType)\n    return config\n  }\n\n  _triggerBackdropTransition() {\n    if (this._config.backdrop === 'static') {\n      const hideEventPrevented = $.Event(Event.HIDE_PREVENTED)\n\n      $(this._element).trigger(hideEventPrevented)\n      if (hideEventPrevented.defaultPrevented) {\n        return\n      }\n\n      this._element.classList.add(ClassName.STATIC)\n\n      const modalTransitionDuration = Util.getTransitionDurationFromElement(this._element)\n\n      $(this._element).one(Util.TRANSITION_END, () => {\n        this._element.classList.remove(ClassName.STATIC)\n      })\n        .emulateTransitionEnd(modalTransitionDuration)\n      this._element.focus()\n    } else {\n      this.hide()\n    }\n  }\n\n  _showElement(relatedTarget) {\n    const transition = $(this._element).hasClass(ClassName.FADE)\n    const modalBody = this._dialog ? this._dialog.querySelector(Selector.MODAL_BODY) : null\n\n    if (!this._element.parentNode ||\n        this._element.parentNode.nodeType !== Node.ELEMENT_NODE) {\n      // Don't move modal's DOM position\n      document.body.appendChild(this._element)\n    }\n\n    this._element.style.display = 'block'\n    this._element.removeAttribute('aria-hidden')\n    this._element.setAttribute('aria-modal', true)\n\n    if ($(this._dialog).hasClass(ClassName.SCROLLABLE) && modalBody) {\n      modalBody.scrollTop = 0\n    } else {\n      this._element.scrollTop = 0\n    }\n\n    if (transition) {\n      Util.reflow(this._element)\n    }\n\n    $(this._element).addClass(ClassName.SHOW)\n\n    if (this._config.focus) {\n      this._enforceFocus()\n    }\n\n    const shownEvent = $.Event(Event.SHOWN, {\n      relatedTarget\n    })\n\n    const transitionComplete = () => {\n      if (this._config.focus) {\n        this._element.focus()\n      }\n      this._isTransitioning = false\n      $(this._element).trigger(shownEvent)\n    }\n\n    if (transition) {\n      const transitionDuration  = Util.getTransitionDurationFromElement(this._dialog)\n\n      $(this._dialog)\n        .one(Util.TRANSITION_END, transitionComplete)\n        .emulateTransitionEnd(transitionDuration)\n    } else {\n      transitionComplete()\n    }\n  }\n\n  _enforceFocus() {\n    $(document)\n      .off(Event.FOCUSIN) // Guard against infinite focus loop\n      .on(Event.FOCUSIN, (event) => {\n        if (document !== event.target &&\n            this._element !== event.target &&\n            $(this._element).has(event.target).length === 0) {\n          this._element.focus()\n        }\n      })\n  }\n\n  _setEscapeEvent() {\n    if (this._isShown && this._config.keyboard) {\n      $(this._element).on(Event.KEYDOWN_DISMISS, (event) => {\n        if (event.which === ESCAPE_KEYCODE) {\n          this._triggerBackdropTransition()\n        }\n      })\n    } else if (!this._isShown) {\n      $(this._element).off(Event.KEYDOWN_DISMISS)\n    }\n  }\n\n  _setResizeEvent() {\n    if (this._isShown) {\n      $(window).on(Event.RESIZE, (event) => this.handleUpdate(event))\n    } else {\n      $(window).off(Event.RESIZE)\n    }\n  }\n\n  _hideModal() {\n    this._element.style.display = 'none'\n    this._element.setAttribute('aria-hidden', true)\n    this._element.removeAttribute('aria-modal')\n    this._isTransitioning = false\n    this._showBackdrop(() => {\n      $(document.body).removeClass(ClassName.OPEN)\n      this._resetAdjustments()\n      this._resetScrollbar()\n      $(this._element).trigger(Event.HIDDEN)\n    })\n  }\n\n  _removeBackdrop() {\n    if (this._backdrop) {\n      $(this._backdrop).remove()\n      this._backdrop = null\n    }\n  }\n\n  _showBackdrop(callback) {\n    const animate = $(this._element).hasClass(ClassName.FADE)\n      ? ClassName.FADE : ''\n\n    if (this._isShown && this._config.backdrop) {\n      this._backdrop = document.createElement('div')\n      this._backdrop.className = ClassName.BACKDROP\n\n      if (animate) {\n        this._backdrop.classList.add(animate)\n      }\n\n      $(this._backdrop).appendTo(document.body)\n\n      $(this._element).on(Event.CLICK_DISMISS, (event) => {\n        if (this._ignoreBackdropClick) {\n          this._ignoreBackdropClick = false\n          return\n        }\n        if (event.target !== event.currentTarget) {\n          return\n        }\n\n        this._triggerBackdropTransition()\n      })\n\n      if (animate) {\n        Util.reflow(this._backdrop)\n      }\n\n      $(this._backdrop).addClass(ClassName.SHOW)\n\n      if (!callback) {\n        return\n      }\n\n      if (!animate) {\n        callback()\n        return\n      }\n\n      const backdropTransitionDuration = Util.getTransitionDurationFromElement(this._backdrop)\n\n      $(this._backdrop)\n        .one(Util.TRANSITION_END, callback)\n        .emulateTransitionEnd(backdropTransitionDuration)\n    } else if (!this._isShown && this._backdrop) {\n      $(this._backdrop).removeClass(ClassName.SHOW)\n\n      const callbackRemove = () => {\n        this._removeBackdrop()\n        if (callback) {\n          callback()\n        }\n      }\n\n      if ($(this._element).hasClass(ClassName.FADE)) {\n        const backdropTransitionDuration = Util.getTransitionDurationFromElement(this._backdrop)\n\n        $(this._backdrop)\n          .one(Util.TRANSITION_END, callbackRemove)\n          .emulateTransitionEnd(backdropTransitionDuration)\n      } else {\n        callbackRemove()\n      }\n    } else if (callback) {\n      callback()\n    }\n  }\n\n  // ----------------------------------------------------------------------\n  // the following methods are used to handle overflowing modals\n  // todo (fat): these should probably be refactored out of modal.js\n  // ----------------------------------------------------------------------\n\n  _adjustDialog() {\n    const isModalOverflowing =\n      this._element.scrollHeight > document.documentElement.clientHeight\n\n    if (!this._isBodyOverflowing && isModalOverflowing) {\n      this._element.style.paddingLeft = `${this._scrollbarWidth}px`\n    }\n\n    if (this._isBodyOverflowing && !isModalOverflowing) {\n      this._element.style.paddingRight = `${this._scrollbarWidth}px`\n    }\n  }\n\n  _resetAdjustments() {\n    this._element.style.paddingLeft = ''\n    this._element.style.paddingRight = ''\n  }\n\n  _checkScrollbar() {\n    const rect = document.body.getBoundingClientRect()\n    this._isBodyOverflowing = rect.left + rect.right < window.innerWidth\n    this._scrollbarWidth = this._getScrollbarWidth()\n  }\n\n  _setScrollbar() {\n    if (this._isBodyOverflowing) {\n      // Note: DOMNode.style.paddingRight returns the actual value or '' if not set\n      //   while $(DOMNode).css('padding-right') returns the calculated value or 0 if not set\n      const fixedContent = [].slice.call(document.querySelectorAll(Selector.FIXED_CONTENT))\n      const stickyContent = [].slice.call(document.querySelectorAll(Selector.STICKY_CONTENT))\n\n      // Adjust fixed content padding\n      $(fixedContent).each((index, element) => {\n        const actualPadding = element.style.paddingRight\n        const calculatedPadding = $(element).css('padding-right')\n        $(element)\n          .data('padding-right', actualPadding)\n          .css('padding-right', `${parseFloat(calculatedPadding) + this._scrollbarWidth}px`)\n      })\n\n      // Adjust sticky content margin\n      $(stickyContent).each((index, element) => {\n        const actualMargin = element.style.marginRight\n        const calculatedMargin = $(element).css('margin-right')\n        $(element)\n          .data('margin-right', actualMargin)\n          .css('margin-right', `${parseFloat(calculatedMargin) - this._scrollbarWidth}px`)\n      })\n\n      // Adjust body padding\n      const actualPadding = document.body.style.paddingRight\n      const calculatedPadding = $(document.body).css('padding-right')\n      $(document.body)\n        .data('padding-right', actualPadding)\n        .css('padding-right', `${parseFloat(calculatedPadding) + this._scrollbarWidth}px`)\n    }\n\n    $(document.body).addClass(ClassName.OPEN)\n  }\n\n  _resetScrollbar() {\n    // Restore fixed content padding\n    const fixedContent = [].slice.call(document.querySelectorAll(Selector.FIXED_CONTENT))\n    $(fixedContent).each((index, element) => {\n      const padding = $(element).data('padding-right')\n      $(element).removeData('padding-right')\n      element.style.paddingRight = padding ? padding : ''\n    })\n\n    // Restore sticky content\n    const elements = [].slice.call(document.querySelectorAll(`${Selector.STICKY_CONTENT}`))\n    $(elements).each((index, element) => {\n      const margin = $(element).data('margin-right')\n      if (typeof margin !== 'undefined') {\n        $(element).css('margin-right', margin).removeData('margin-right')\n      }\n    })\n\n    // Restore body padding\n    const padding = $(document.body).data('padding-right')\n    $(document.body).removeData('padding-right')\n    document.body.style.paddingRight = padding ? padding : ''\n  }\n\n  _getScrollbarWidth() { // thx d.walsh\n    const scrollDiv = document.createElement('div')\n    scrollDiv.className = ClassName.SCROLLBAR_MEASURER\n    document.body.appendChild(scrollDiv)\n    const scrollbarWidth = scrollDiv.getBoundingClientRect().width - scrollDiv.clientWidth\n    document.body.removeChild(scrollDiv)\n    return scrollbarWidth\n  }\n\n  // Static\n\n  static _jQueryInterface(config, relatedTarget) {\n    return this.each(function () {\n      let data = $(this).data(DATA_KEY)\n      const _config = {\n        ...Default,\n        ...$(this).data(),\n        ...typeof config === 'object' && config ? config : {}\n      }\n\n      if (!data) {\n        data = new Modal(this, _config)\n        $(this).data(DATA_KEY, data)\n      }\n\n      if (typeof config === 'string') {\n        if (typeof data[config] === 'undefined') {\n          throw new TypeError(`No method named \"${config}\"`)\n        }\n        data[config](relatedTarget)\n      } else if (_config.show) {\n        data.show(relatedTarget)\n      }\n    })\n  }\n}\n\n/**\n * ------------------------------------------------------------------------\n * Data Api implementation\n * ------------------------------------------------------------------------\n */\n\n$(document).on(Event.CLICK_DATA_API, Selector.DATA_TOGGLE, function (event) {\n  let target\n  const selector = Util.getSelectorFromElement(this)\n\n  if (selector) {\n    target = document.querySelector(selector)\n  }\n\n  const config = $(target).data(DATA_KEY)\n    ? 'toggle' : {\n      ...$(target).data(),\n      ...$(this).data()\n    }\n\n  if (this.tagName === 'A' || this.tagName === 'AREA') {\n    event.preventDefault()\n  }\n\n  const $target = $(target).one(Event.SHOW, (showEvent) => {\n    if (showEvent.isDefaultPrevented()) {\n      // Only register focus restorer if modal will actually get shown\n      return\n    }\n\n    $target.one(Event.HIDDEN, () => {\n      if ($(this).is(':visible')) {\n        this.focus()\n      }\n    })\n  })\n\n  Modal._jQueryInterface.call($(target), config, this)\n})\n\n/**\n * ------------------------------------------------------------------------\n * jQuery\n * ------------------------------------------------------------------------\n */\n\n$.fn[NAME] = Modal._jQueryInterface\n$.fn[NAME].Constructor = Modal\n$.fn[NAME].noConflict = () => {\n  $.fn[NAME] = JQUERY_NO_CONFLICT\n  return Modal._jQueryInterface\n}\n\nexport default Modal\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap (v4.4.1): tools/sanitizer.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nconst uriAttrs = [\n  'background',\n  'cite',\n  'href',\n  'itemtype',\n  'longdesc',\n  'poster',\n  'src',\n  'xlink:href'\n]\n\nconst ARIA_ATTRIBUTE_PATTERN = /^aria-[\\w-]*$/i\n\nexport const DefaultWhitelist = {\n  // Global attributes allowed on any supplied element below.\n  '*': ['class', 'dir', 'id', 'lang', 'role', ARIA_ATTRIBUTE_PATTERN],\n  a: ['target', 'href', 'title', 'rel'],\n  area: [],\n  b: [],\n  br: [],\n  col: [],\n  code: [],\n  div: [],\n  em: [],\n  hr: [],\n  h1: [],\n  h2: [],\n  h3: [],\n  h4: [],\n  h5: [],\n  h6: [],\n  i: [],\n  img: ['src', 'alt', 'title', 'width', 'height'],\n  li: [],\n  ol: [],\n  p: [],\n  pre: [],\n  s: [],\n  small: [],\n  span: [],\n  sub: [],\n  sup: [],\n  strong: [],\n  u: [],\n  ul: []\n}\n\n/**\n * A pattern that recognizes a commonly useful subset of URLs that are safe.\n *\n * Shoutout to Angular 7 https://github.com/angular/angular/blob/7.2.4/packages/core/src/sanitization/url_sanitizer.ts\n */\nconst SAFE_URL_PATTERN = /^(?:(?:https?|mailto|ftp|tel|file):|[^&:/?#]*(?:[/?#]|$))/gi\n\n/**\n * A pattern that matches safe data URLs. Only matches image, video and audio types.\n *\n * Shoutout to Angular 7 https://github.com/angular/angular/blob/7.2.4/packages/core/src/sanitization/url_sanitizer.ts\n */\nconst DATA_URL_PATTERN = /^data:(?:image\\/(?:bmp|gif|jpeg|jpg|png|tiff|webp)|video\\/(?:mpeg|mp4|ogg|webm)|audio\\/(?:mp3|oga|ogg|opus));base64,[a-z0-9+/]+=*$/i\n\nfunction allowedAttribute(attr, allowedAttributeList) {\n  const attrName = attr.nodeName.toLowerCase()\n\n  if (allowedAttributeList.indexOf(attrName) !== -1) {\n    if (uriAttrs.indexOf(attrName) !== -1) {\n      return Boolean(attr.nodeValue.match(SAFE_URL_PATTERN) || attr.nodeValue.match(DATA_URL_PATTERN))\n    }\n\n    return true\n  }\n\n  const regExp = allowedAttributeList.filter((attrRegex) => attrRegex instanceof RegExp)\n\n  // Check if a regular expression validates the attribute.\n  for (let i = 0, l = regExp.length; i < l; i++) {\n    if (attrName.match(regExp[i])) {\n      return true\n    }\n  }\n\n  return false\n}\n\nexport function sanitizeHtml(unsafeHtml, whiteList, sanitizeFn) {\n  if (unsafeHtml.length === 0) {\n    return unsafeHtml\n  }\n\n  if (sanitizeFn && typeof sanitizeFn === 'function') {\n    return sanitizeFn(unsafeHtml)\n  }\n\n  const domParser = new window.DOMParser()\n  const createdDocument = domParser.parseFromString(unsafeHtml, 'text/html')\n  const whitelistKeys = Object.keys(whiteList)\n  const elements = [].slice.call(createdDocument.body.querySelectorAll('*'))\n\n  for (let i = 0, len = elements.length; i < len; i++) {\n    const el = elements[i]\n    const elName = el.nodeName.toLowerCase()\n\n    if (whitelistKeys.indexOf(el.nodeName.toLowerCase()) === -1) {\n      el.parentNode.removeChild(el)\n\n      continue\n    }\n\n    const attributeList = [].slice.call(el.attributes)\n    const whitelistedAttributes = [].concat(whiteList['*'] || [], whiteList[elName] || [])\n\n    attributeList.forEach((attr) => {\n      if (!allowedAttribute(attr, whitelistedAttributes)) {\n        el.removeAttribute(attr.nodeName)\n      }\n    })\n  }\n\n  return createdDocument.body.innerHTML\n}\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap (v4.4.1): tooltip.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport {\n  DefaultWhitelist,\n  sanitizeHtml\n} from './tools/sanitizer'\nimport $ from 'jquery'\nimport Popper from 'popper.js'\nimport Util from './util'\n\n/**\n * ------------------------------------------------------------------------\n * Constants\n * ------------------------------------------------------------------------\n */\n\nconst NAME                  = 'tooltip'\nconst VERSION               = '4.4.1'\nconst DATA_KEY              = 'bs.tooltip'\nconst EVENT_KEY             = `.${DATA_KEY}`\nconst JQUERY_NO_CONFLICT    = $.fn[NAME]\nconst CLASS_PREFIX          = 'bs-tooltip'\nconst BSCLS_PREFIX_REGEX    = new RegExp(`(^|\\\\s)${CLASS_PREFIX}\\\\S+`, 'g')\nconst DISALLOWED_ATTRIBUTES = ['sanitize', 'whiteList', 'sanitizeFn']\n\nconst DefaultType = {\n  animation         : 'boolean',\n  template          : 'string',\n  title             : '(string|element|function)',\n  trigger           : 'string',\n  delay             : '(number|object)',\n  html              : 'boolean',\n  selector          : '(string|boolean)',\n  placement         : '(string|function)',\n  offset            : '(number|string|function)',\n  container         : '(string|element|boolean)',\n  fallbackPlacement : '(string|array)',\n  boundary          : '(string|element)',\n  sanitize          : 'boolean',\n  sanitizeFn        : '(null|function)',\n  whiteList         : 'object',\n  popperConfig      : '(null|object)'\n}\n\nconst AttachmentMap = {\n  AUTO   : 'auto',\n  TOP    : 'top',\n  RIGHT  : 'right',\n  BOTTOM : 'bottom',\n  LEFT   : 'left'\n}\n\nconst Default = {\n  animation         : true,\n  template          : '<div class=\"tooltip\" role=\"tooltip\">' +\n                    '<div class=\"arrow\"></div>' +\n                    '<div class=\"tooltip-inner\"></div></div>',\n  trigger           : 'hover focus',\n  title             : '',\n  delay             : 0,\n  html              : false,\n  selector          : false,\n  placement         : 'top',\n  offset            : 0,\n  container         : false,\n  fallbackPlacement : 'flip',\n  boundary          : 'scrollParent',\n  sanitize          : true,\n  sanitizeFn        : null,\n  whiteList         : DefaultWhitelist,\n  popperConfig      : null\n}\n\nconst HoverState = {\n  SHOW : 'show',\n  OUT  : 'out'\n}\n\nconst Event = {\n  HIDE       : `hide${EVENT_KEY}`,\n  HIDDEN     : `hidden${EVENT_KEY}`,\n  SHOW       : `show${EVENT_KEY}`,\n  SHOWN      : `shown${EVENT_KEY}`,\n  INSERTED   : `inserted${EVENT_KEY}`,\n  CLICK      : `click${EVENT_KEY}`,\n  FOCUSIN    : `focusin${EVENT_KEY}`,\n  FOCUSOUT   : `focusout${EVENT_KEY}`,\n  MOUSEENTER : `mouseenter${EVENT_KEY}`,\n  MOUSELEAVE : `mouseleave${EVENT_KEY}`\n}\n\nconst ClassName = {\n  FADE : 'fade',\n  SHOW : 'show'\n}\n\nconst Selector = {\n  TOOLTIP       : '.tooltip',\n  TOOLTIP_INNER : '.tooltip-inner',\n  ARROW         : '.arrow'\n}\n\nconst Trigger = {\n  HOVER  : 'hover',\n  FOCUS  : 'focus',\n  CLICK  : 'click',\n  MANUAL : 'manual'\n}\n\n\n/**\n * ------------------------------------------------------------------------\n * Class Definition\n * ------------------------------------------------------------------------\n */\n\nclass Tooltip {\n  constructor(element, config) {\n    if (typeof Popper === 'undefined') {\n      throw new TypeError('Bootstrap\\'s tooltips require Popper.js (https://popper.js.org/)')\n    }\n\n    // private\n    this._isEnabled     = true\n    this._timeout       = 0\n    this._hoverState    = ''\n    this._activeTrigger = {}\n    this._popper        = null\n\n    // Protected\n    this.element = element\n    this.config  = this._getConfig(config)\n    this.tip     = null\n\n    this._setListeners()\n  }\n\n  // Getters\n\n  static get VERSION() {\n    return VERSION\n  }\n\n  static get Default() {\n    return Default\n  }\n\n  static get NAME() {\n    return NAME\n  }\n\n  static get DATA_KEY() {\n    return DATA_KEY\n  }\n\n  static get Event() {\n    return Event\n  }\n\n  static get EVENT_KEY() {\n    return EVENT_KEY\n  }\n\n  static get DefaultType() {\n    return DefaultType\n  }\n\n  // Public\n\n  enable() {\n    this._isEnabled = true\n  }\n\n  disable() {\n    this._isEnabled = false\n  }\n\n  toggleEnabled() {\n    this._isEnabled = !this._isEnabled\n  }\n\n  toggle(event) {\n    if (!this._isEnabled) {\n      return\n    }\n\n    if (event) {\n      const dataKey = this.constructor.DATA_KEY\n      let context = $(event.currentTarget).data(dataKey)\n\n      if (!context) {\n        context = new this.constructor(\n          event.currentTarget,\n          this._getDelegateConfig()\n        )\n        $(event.currentTarget).data(dataKey, context)\n      }\n\n      context._activeTrigger.click = !context._activeTrigger.click\n\n      if (context._isWithActiveTrigger()) {\n        context._enter(null, context)\n      } else {\n        context._leave(null, context)\n      }\n    } else {\n      if ($(this.getTipElement()).hasClass(ClassName.SHOW)) {\n        this._leave(null, this)\n        return\n      }\n\n      this._enter(null, this)\n    }\n  }\n\n  dispose() {\n    clearTimeout(this._timeout)\n\n    $.removeData(this.element, this.constructor.DATA_KEY)\n\n    $(this.element).off(this.constructor.EVENT_KEY)\n    $(this.element).closest('.modal').off('hide.bs.modal', this._hideModalHandler)\n\n    if (this.tip) {\n      $(this.tip).remove()\n    }\n\n    this._isEnabled     = null\n    this._timeout       = null\n    this._hoverState    = null\n    this._activeTrigger = null\n    if (this._popper) {\n      this._popper.destroy()\n    }\n\n    this._popper = null\n    this.element = null\n    this.config  = null\n    this.tip     = null\n  }\n\n  show() {\n    if ($(this.element).css('display') === 'none') {\n      throw new Error('Please use show on visible elements')\n    }\n\n    const showEvent = $.Event(this.constructor.Event.SHOW)\n    if (this.isWithContent() && this._isEnabled) {\n      $(this.element).trigger(showEvent)\n\n      const shadowRoot = Util.findShadowRoot(this.element)\n      const isInTheDom = $.contains(\n        shadowRoot !== null ? shadowRoot : this.element.ownerDocument.documentElement,\n        this.element\n      )\n\n      if (showEvent.isDefaultPrevented() || !isInTheDom) {\n        return\n      }\n\n      const tip   = this.getTipElement()\n      const tipId = Util.getUID(this.constructor.NAME)\n\n      tip.setAttribute('id', tipId)\n      this.element.setAttribute('aria-describedby', tipId)\n\n      this.setContent()\n\n      if (this.config.animation) {\n        $(tip).addClass(ClassName.FADE)\n      }\n\n      const placement  = typeof this.config.placement === 'function'\n        ? this.config.placement.call(this, tip, this.element)\n        : this.config.placement\n\n      const attachment = this._getAttachment(placement)\n      this.addAttachmentClass(attachment)\n\n      const container = this._getContainer()\n      $(tip).data(this.constructor.DATA_KEY, this)\n\n      if (!$.contains(this.element.ownerDocument.documentElement, this.tip)) {\n        $(tip).appendTo(container)\n      }\n\n      $(this.element).trigger(this.constructor.Event.INSERTED)\n\n      this._popper = new Popper(this.element, tip, this._getPopperConfig(attachment))\n\n      $(tip).addClass(ClassName.SHOW)\n\n      // If this is a touch-enabled device we add extra\n      // empty mouseover listeners to the body's immediate children;\n      // only needed because of broken event delegation on iOS\n      // https://www.quirksmode.org/blog/archives/2014/02/mouse_event_bub.html\n      if ('ontouchstart' in document.documentElement) {\n        $(document.body).children().on('mouseover', null, $.noop)\n      }\n\n      const complete = () => {\n        if (this.config.animation) {\n          this._fixTransition()\n        }\n        const prevHoverState = this._hoverState\n        this._hoverState     = null\n\n        $(this.element).trigger(this.constructor.Event.SHOWN)\n\n        if (prevHoverState === HoverState.OUT) {\n          this._leave(null, this)\n        }\n      }\n\n      if ($(this.tip).hasClass(ClassName.FADE)) {\n        const transitionDuration = Util.getTransitionDurationFromElement(this.tip)\n\n        $(this.tip)\n          .one(Util.TRANSITION_END, complete)\n          .emulateTransitionEnd(transitionDuration)\n      } else {\n        complete()\n      }\n    }\n  }\n\n  hide(callback) {\n    const tip       = this.getTipElement()\n    const hideEvent = $.Event(this.constructor.Event.HIDE)\n    const complete = () => {\n      if (this._hoverState !== HoverState.SHOW && tip.parentNode) {\n        tip.parentNode.removeChild(tip)\n      }\n\n      this._cleanTipClass()\n      this.element.removeAttribute('aria-describedby')\n      $(this.element).trigger(this.constructor.Event.HIDDEN)\n      if (this._popper !== null) {\n        this._popper.destroy()\n      }\n\n      if (callback) {\n        callback()\n      }\n    }\n\n    $(this.element).trigger(hideEvent)\n\n    if (hideEvent.isDefaultPrevented()) {\n      return\n    }\n\n    $(tip).removeClass(ClassName.SHOW)\n\n    // If this is a touch-enabled device we remove the extra\n    // empty mouseover listeners we added for iOS support\n    if ('ontouchstart' in document.documentElement) {\n      $(document.body).children().off('mouseover', null, $.noop)\n    }\n\n    this._activeTrigger[Trigger.CLICK] = false\n    this._activeTrigger[Trigger.FOCUS] = false\n    this._activeTrigger[Trigger.HOVER] = false\n\n    if ($(this.tip).hasClass(ClassName.FADE)) {\n      const transitionDuration = Util.getTransitionDurationFromElement(tip)\n\n      $(tip)\n        .one(Util.TRANSITION_END, complete)\n        .emulateTransitionEnd(transitionDuration)\n    } else {\n      complete()\n    }\n\n    this._hoverState = ''\n  }\n\n  update() {\n    if (this._popper !== null) {\n      this._popper.scheduleUpdate()\n    }\n  }\n\n  // Protected\n\n  isWithContent() {\n    return Boolean(this.getTitle())\n  }\n\n  addAttachmentClass(attachment) {\n    $(this.getTipElement()).addClass(`${CLASS_PREFIX}-${attachment}`)\n  }\n\n  getTipElement() {\n    this.tip = this.tip || $(this.config.template)[0]\n    return this.tip\n  }\n\n  setContent() {\n    const tip = this.getTipElement()\n    this.setElementContent($(tip.querySelectorAll(Selector.TOOLTIP_INNER)), this.getTitle())\n    $(tip).removeClass(`${ClassName.FADE} ${ClassName.SHOW}`)\n  }\n\n  setElementContent($element, content) {\n    if (typeof content === 'object' && (content.nodeType || content.jquery)) {\n      // Content is a DOM node or a jQuery\n      if (this.config.html) {\n        if (!$(content).parent().is($element)) {\n          $element.empty().append(content)\n        }\n      } else {\n        $element.text($(content).text())\n      }\n\n      return\n    }\n\n    if (this.config.html) {\n      if (this.config.sanitize) {\n        content = sanitizeHtml(content, this.config.whiteList, this.config.sanitizeFn)\n      }\n\n      $element.html(content)\n    } else {\n      $element.text(content)\n    }\n  }\n\n  getTitle() {\n    let title = this.element.getAttribute('data-original-title')\n\n    if (!title) {\n      title = typeof this.config.title === 'function'\n        ? this.config.title.call(this.element)\n        : this.config.title\n    }\n\n    return title\n  }\n\n  // Private\n\n  _getPopperConfig(attachment) {\n    const defaultBsConfig = {\n      placement: attachment,\n      modifiers: {\n        offset: this._getOffset(),\n        flip: {\n          behavior: this.config.fallbackPlacement\n        },\n        arrow: {\n          element: Selector.ARROW\n        },\n        preventOverflow: {\n          boundariesElement: this.config.boundary\n        }\n      },\n      onCreate: (data) => {\n        if (data.originalPlacement !== data.placement) {\n          this._handlePopperPlacementChange(data)\n        }\n      },\n      onUpdate: (data) => this._handlePopperPlacementChange(data)\n    }\n\n    return {\n      ...defaultBsConfig,\n      ...this.config.popperConfig\n    }\n  }\n\n  _getOffset() {\n    const offset = {}\n\n    if (typeof this.config.offset === 'function') {\n      offset.fn = (data) => {\n        data.offsets = {\n          ...data.offsets,\n          ...this.config.offset(data.offsets, this.element) || {}\n        }\n\n        return data\n      }\n    } else {\n      offset.offset = this.config.offset\n    }\n\n    return offset\n  }\n\n  _getContainer() {\n    if (this.config.container === false) {\n      return document.body\n    }\n\n    if (Util.isElement(this.config.container)) {\n      return $(this.config.container)\n    }\n\n    return $(document).find(this.config.container)\n  }\n\n  _getAttachment(placement) {\n    return AttachmentMap[placement.toUpperCase()]\n  }\n\n  _setListeners() {\n    const triggers = this.config.trigger.split(' ')\n\n    triggers.forEach((trigger) => {\n      if (trigger === 'click') {\n        $(this.element).on(\n          this.constructor.Event.CLICK,\n          this.config.selector,\n          (event) => this.toggle(event)\n        )\n      } else if (trigger !== Trigger.MANUAL) {\n        const eventIn = trigger === Trigger.HOVER\n          ? this.constructor.Event.MOUSEENTER\n          : this.constructor.Event.FOCUSIN\n        const eventOut = trigger === Trigger.HOVER\n          ? this.constructor.Event.MOUSELEAVE\n          : this.constructor.Event.FOCUSOUT\n\n        $(this.element)\n          .on(\n            eventIn,\n            this.config.selector,\n            (event) => this._enter(event)\n          )\n          .on(\n            eventOut,\n            this.config.selector,\n            (event) => this._leave(event)\n          )\n      }\n    })\n\n    this._hideModalHandler = () => {\n      if (this.element) {\n        this.hide()\n      }\n    }\n\n    $(this.element).closest('.modal').on(\n      'hide.bs.modal',\n      this._hideModalHandler\n    )\n\n    if (this.config.selector) {\n      this.config = {\n        ...this.config,\n        trigger: 'manual',\n        selector: ''\n      }\n    } else {\n      this._fixTitle()\n    }\n  }\n\n  _fixTitle() {\n    const titleType = typeof this.element.getAttribute('data-original-title')\n\n    if (this.element.getAttribute('title') || titleType !== 'string') {\n      this.element.setAttribute(\n        'data-original-title',\n        this.element.getAttribute('title') || ''\n      )\n\n      this.element.setAttribute('title', '')\n    }\n  }\n\n  _enter(event, context) {\n    const dataKey = this.constructor.DATA_KEY\n    context = context || $(event.currentTarget).data(dataKey)\n\n    if (!context) {\n      context = new this.constructor(\n        event.currentTarget,\n        this._getDelegateConfig()\n      )\n      $(event.currentTarget).data(dataKey, context)\n    }\n\n    if (event) {\n      context._activeTrigger[\n        event.type === 'focusin' ? Trigger.FOCUS : Trigger.HOVER\n      ] = true\n    }\n\n    if ($(context.getTipElement()).hasClass(ClassName.SHOW) || context._hoverState === HoverState.SHOW) {\n      context._hoverState = HoverState.SHOW\n      return\n    }\n\n    clearTimeout(context._timeout)\n\n    context._hoverState = HoverState.SHOW\n\n    if (!context.config.delay || !context.config.delay.show) {\n      context.show()\n      return\n    }\n\n    context._timeout = setTimeout(() => {\n      if (context._hoverState === HoverState.SHOW) {\n        context.show()\n      }\n    }, context.config.delay.show)\n  }\n\n  _leave(event, context) {\n    const dataKey = this.constructor.DATA_KEY\n    context = context || $(event.currentTarget).data(dataKey)\n\n    if (!context) {\n      context = new this.constructor(\n        event.currentTarget,\n        this._getDelegateConfig()\n      )\n      $(event.currentTarget).data(dataKey, context)\n    }\n\n    if (event) {\n      context._activeTrigger[\n        event.type === 'focusout' ? Trigger.FOCUS : Trigger.HOVER\n      ] = false\n    }\n\n    if (context._isWithActiveTrigger()) {\n      return\n    }\n\n    clearTimeout(context._timeout)\n\n    context._hoverState = HoverState.OUT\n\n    if (!context.config.delay || !context.config.delay.hide) {\n      context.hide()\n      return\n    }\n\n    context._timeout = setTimeout(() => {\n      if (context._hoverState === HoverState.OUT) {\n        context.hide()\n      }\n    }, context.config.delay.hide)\n  }\n\n  _isWithActiveTrigger() {\n    for (const trigger in this._activeTrigger) {\n      if (this._activeTrigger[trigger]) {\n        return true\n      }\n    }\n\n    return false\n  }\n\n  _getConfig(config) {\n    const dataAttributes = $(this.element).data()\n\n    Object.keys(dataAttributes)\n      .forEach((dataAttr) => {\n        if (DISALLOWED_ATTRIBUTES.indexOf(dataAttr) !== -1) {\n          delete dataAttributes[dataAttr]\n        }\n      })\n\n    config = {\n      ...this.constructor.Default,\n      ...dataAttributes,\n      ...typeof config === 'object' && config ? config : {}\n    }\n\n    if (typeof config.delay === 'number') {\n      config.delay = {\n        show: config.delay,\n        hide: config.delay\n      }\n    }\n\n    if (typeof config.title === 'number') {\n      config.title = config.title.toString()\n    }\n\n    if (typeof config.content === 'number') {\n      config.content = config.content.toString()\n    }\n\n    Util.typeCheckConfig(\n      NAME,\n      config,\n      this.constructor.DefaultType\n    )\n\n    if (config.sanitize) {\n      config.template = sanitizeHtml(config.template, config.whiteList, config.sanitizeFn)\n    }\n\n    return config\n  }\n\n  _getDelegateConfig() {\n    const config = {}\n\n    if (this.config) {\n      for (const key in this.config) {\n        if (this.constructor.Default[key] !== this.config[key]) {\n          config[key] = this.config[key]\n        }\n      }\n    }\n\n    return config\n  }\n\n  _cleanTipClass() {\n    const $tip = $(this.getTipElement())\n    const tabClass = $tip.attr('class').match(BSCLS_PREFIX_REGEX)\n    if (tabClass !== null && tabClass.length) {\n      $tip.removeClass(tabClass.join(''))\n    }\n  }\n\n  _handlePopperPlacementChange(popperData) {\n    const popperInstance = popperData.instance\n    this.tip = popperInstance.popper\n    this._cleanTipClass()\n    this.addAttachmentClass(this._getAttachment(popperData.placement))\n  }\n\n  _fixTransition() {\n    const tip = this.getTipElement()\n    const initConfigAnimation = this.config.animation\n\n    if (tip.getAttribute('x-placement') !== null) {\n      return\n    }\n\n    $(tip).removeClass(ClassName.FADE)\n    this.config.animation = false\n    this.hide()\n    this.show()\n    this.config.animation = initConfigAnimation\n  }\n\n  // Static\n\n  static _jQueryInterface(config) {\n    return this.each(function () {\n      let data = $(this).data(DATA_KEY)\n      const _config = typeof config === 'object' && config\n\n      if (!data && /dispose|hide/.test(config)) {\n        return\n      }\n\n      if (!data) {\n        data = new Tooltip(this, _config)\n        $(this).data(DATA_KEY, data)\n      }\n\n      if (typeof config === 'string') {\n        if (typeof data[config] === 'undefined') {\n          throw new TypeError(`No method named \"${config}\"`)\n        }\n        data[config]()\n      }\n    })\n  }\n}\n\n/**\n * ------------------------------------------------------------------------\n * jQuery\n * ------------------------------------------------------------------------\n */\n\n$.fn[NAME] = Tooltip._jQueryInterface\n$.fn[NAME].Constructor = Tooltip\n$.fn[NAME].noConflict = () => {\n  $.fn[NAME] = JQUERY_NO_CONFLICT\n  return Tooltip._jQueryInterface\n}\n\nexport default Tooltip\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap (v4.4.1): popover.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport $ from 'jquery'\nimport Tooltip from './tooltip'\n\n/**\n * ------------------------------------------------------------------------\n * Constants\n * ------------------------------------------------------------------------\n */\n\nconst NAME                = 'popover'\nconst VERSION             = '4.4.1'\nconst DATA_KEY            = 'bs.popover'\nconst EVENT_KEY           = `.${DATA_KEY}`\nconst JQUERY_NO_CONFLICT  = $.fn[NAME]\nconst CLASS_PREFIX        = 'bs-popover'\nconst BSCLS_PREFIX_REGEX  = new RegExp(`(^|\\\\s)${CLASS_PREFIX}\\\\S+`, 'g')\n\nconst Default = {\n  ...Tooltip.Default,\n  placement : 'right',\n  trigger   : 'click',\n  content   : '',\n  template  : '<div class=\"popover\" role=\"tooltip\">' +\n              '<div class=\"arrow\"></div>' +\n              '<h3 class=\"popover-header\"></h3>' +\n              '<div class=\"popover-body\"></div></div>'\n}\n\nconst DefaultType = {\n  ...Tooltip.DefaultType,\n  content : '(string|element|function)'\n}\n\nconst ClassName = {\n  FADE : 'fade',\n  SHOW : 'show'\n}\n\nconst Selector = {\n  TITLE   : '.popover-header',\n  CONTENT : '.popover-body'\n}\n\nconst Event = {\n  HIDE       : `hide${EVENT_KEY}`,\n  HIDDEN     : `hidden${EVENT_KEY}`,\n  SHOW       : `show${EVENT_KEY}`,\n  SHOWN      : `shown${EVENT_KEY}`,\n  INSERTED   : `inserted${EVENT_KEY}`,\n  CLICK      : `click${EVENT_KEY}`,\n  FOCUSIN    : `focusin${EVENT_KEY}`,\n  FOCUSOUT   : `focusout${EVENT_KEY}`,\n  MOUSEENTER : `mouseenter${EVENT_KEY}`,\n  MOUSELEAVE : `mouseleave${EVENT_KEY}`\n}\n\n/**\n * ------------------------------------------------------------------------\n * Class Definition\n * ------------------------------------------------------------------------\n */\n\nclass Popover extends Tooltip {\n  // Getters\n\n  static get VERSION() {\n    return VERSION\n  }\n\n  static get Default() {\n    return Default\n  }\n\n  static get NAME() {\n    return NAME\n  }\n\n  static get DATA_KEY() {\n    return DATA_KEY\n  }\n\n  static get Event() {\n    return Event\n  }\n\n  static get EVENT_KEY() {\n    return EVENT_KEY\n  }\n\n  static get DefaultType() {\n    return DefaultType\n  }\n\n  // Overrides\n\n  isWithContent() {\n    return this.getTitle() || this._getContent()\n  }\n\n  addAttachmentClass(attachment) {\n    $(this.getTipElement()).addClass(`${CLASS_PREFIX}-${attachment}`)\n  }\n\n  getTipElement() {\n    this.tip = this.tip || $(this.config.template)[0]\n    return this.tip\n  }\n\n  setContent() {\n    const $tip = $(this.getTipElement())\n\n    // We use append for html objects to maintain js events\n    this.setElementContent($tip.find(Selector.TITLE), this.getTitle())\n    let content = this._getContent()\n    if (typeof content === 'function') {\n      content = content.call(this.element)\n    }\n    this.setElementContent($tip.find(Selector.CONTENT), content)\n\n    $tip.removeClass(`${ClassName.FADE} ${ClassName.SHOW}`)\n  }\n\n  // Private\n\n  _getContent() {\n    return this.element.getAttribute('data-content') ||\n      this.config.content\n  }\n\n  _cleanTipClass() {\n    const $tip = $(this.getTipElement())\n    const tabClass = $tip.attr('class').match(BSCLS_PREFIX_REGEX)\n    if (tabClass !== null && tabClass.length > 0) {\n      $tip.removeClass(tabClass.join(''))\n    }\n  }\n\n  // Static\n\n  static _jQueryInterface(config) {\n    return this.each(function () {\n      let data = $(this).data(DATA_KEY)\n      const _config = typeof config === 'object' ? config : null\n\n      if (!data && /dispose|hide/.test(config)) {\n        return\n      }\n\n      if (!data) {\n        data = new Popover(this, _config)\n        $(this).data(DATA_KEY, data)\n      }\n\n      if (typeof config === 'string') {\n        if (typeof data[config] === 'undefined') {\n          throw new TypeError(`No method named \"${config}\"`)\n        }\n        data[config]()\n      }\n    })\n  }\n}\n\n/**\n * ------------------------------------------------------------------------\n * jQuery\n * ------------------------------------------------------------------------\n */\n\n$.fn[NAME] = Popover._jQueryInterface\n$.fn[NAME].Constructor = Popover\n$.fn[NAME].noConflict = () => {\n  $.fn[NAME] = JQUERY_NO_CONFLICT\n  return Popover._jQueryInterface\n}\n\nexport default Popover\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap (v4.4.1): scrollspy.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport $ from 'jquery'\nimport Util from './util'\n\n/**\n * ------------------------------------------------------------------------\n * Constants\n * ------------------------------------------------------------------------\n */\n\nconst NAME               = 'scrollspy'\nconst VERSION            = '4.4.1'\nconst DATA_KEY           = 'bs.scrollspy'\nconst EVENT_KEY          = `.${DATA_KEY}`\nconst DATA_API_KEY       = '.data-api'\nconst JQUERY_NO_CONFLICT = $.fn[NAME]\n\nconst Default = {\n  offset : 10,\n  method : 'auto',\n  target : ''\n}\n\nconst DefaultType = {\n  offset : 'number',\n  method : 'string',\n  target : '(string|element)'\n}\n\nconst Event = {\n  ACTIVATE      : `activate${EVENT_KEY}`,\n  SCROLL        : `scroll${EVENT_KEY}`,\n  LOAD_DATA_API : `load${EVENT_KEY}${DATA_API_KEY}`\n}\n\nconst ClassName = {\n  DROPDOWN_ITEM : 'dropdown-item',\n  DROPDOWN_MENU : 'dropdown-menu',\n  ACTIVE        : 'active'\n}\n\nconst Selector = {\n  DATA_SPY        : '[data-spy=\"scroll\"]',\n  ACTIVE          : '.active',\n  NAV_LIST_GROUP  : '.nav, .list-group',\n  NAV_LINKS       : '.nav-link',\n  NAV_ITEMS       : '.nav-item',\n  LIST_ITEMS      : '.list-group-item',\n  DROPDOWN        : '.dropdown',\n  DROPDOWN_ITEMS  : '.dropdown-item',\n  DROPDOWN_TOGGLE : '.dropdown-toggle'\n}\n\nconst OffsetMethod = {\n  OFFSET   : 'offset',\n  POSITION : 'position'\n}\n\n/**\n * ------------------------------------------------------------------------\n * Class Definition\n * ------------------------------------------------------------------------\n */\n\nclass ScrollSpy {\n  constructor(element, config) {\n    this._element       = element\n    this._scrollElement = element.tagName === 'BODY' ? window : element\n    this._config        = this._getConfig(config)\n    this._selector      = `${this._config.target} ${Selector.NAV_LINKS},` +\n                          `${this._config.target} ${Selector.LIST_ITEMS},` +\n                          `${this._config.target} ${Selector.DROPDOWN_ITEMS}`\n    this._offsets       = []\n    this._targets       = []\n    this._activeTarget  = null\n    this._scrollHeight  = 0\n\n    $(this._scrollElement).on(Event.SCROLL, (event) => this._process(event))\n\n    this.refresh()\n    this._process()\n  }\n\n  // Getters\n\n  static get VERSION() {\n    return VERSION\n  }\n\n  static get Default() {\n    return Default\n  }\n\n  // Public\n\n  refresh() {\n    const autoMethod = this._scrollElement === this._scrollElement.window\n      ? OffsetMethod.OFFSET : OffsetMethod.POSITION\n\n    const offsetMethod = this._config.method === 'auto'\n      ? autoMethod : this._config.method\n\n    const offsetBase = offsetMethod === OffsetMethod.POSITION\n      ? this._getScrollTop() : 0\n\n    this._offsets = []\n    this._targets = []\n\n    this._scrollHeight = this._getScrollHeight()\n\n    const targets = [].slice.call(document.querySelectorAll(this._selector))\n\n    targets\n      .map((element) => {\n        let target\n        const targetSelector = Util.getSelectorFromElement(element)\n\n        if (targetSelector) {\n          target = document.querySelector(targetSelector)\n        }\n\n        if (target) {\n          const targetBCR = target.getBoundingClientRect()\n          if (targetBCR.width || targetBCR.height) {\n            // TODO (fat): remove sketch reliance on jQuery position/offset\n            return [\n              $(target)[offsetMethod]().top + offsetBase,\n              targetSelector\n            ]\n          }\n        }\n        return null\n      })\n      .filter((item) => item)\n      .sort((a, b) => a[0] - b[0])\n      .forEach((item) => {\n        this._offsets.push(item[0])\n        this._targets.push(item[1])\n      })\n  }\n\n  dispose() {\n    $.removeData(this._element, DATA_KEY)\n    $(this._scrollElement).off(EVENT_KEY)\n\n    this._element       = null\n    this._scrollElement = null\n    this._config        = null\n    this._selector      = null\n    this._offsets       = null\n    this._targets       = null\n    this._activeTarget  = null\n    this._scrollHeight  = null\n  }\n\n  // Private\n\n  _getConfig(config) {\n    config = {\n      ...Default,\n      ...typeof config === 'object' && config ? config : {}\n    }\n\n    if (typeof config.target !== 'string') {\n      let id = $(config.target).attr('id')\n      if (!id) {\n        id = Util.getUID(NAME)\n        $(config.target).attr('id', id)\n      }\n      config.target = `#${id}`\n    }\n\n    Util.typeCheckConfig(NAME, config, DefaultType)\n\n    return config\n  }\n\n  _getScrollTop() {\n    return this._scrollElement === window\n      ? this._scrollElement.pageYOffset : this._scrollElement.scrollTop\n  }\n\n  _getScrollHeight() {\n    return this._scrollElement.scrollHeight || Math.max(\n      document.body.scrollHeight,\n      document.documentElement.scrollHeight\n    )\n  }\n\n  _getOffsetHeight() {\n    return this._scrollElement === window\n      ? window.innerHeight : this._scrollElement.getBoundingClientRect().height\n  }\n\n  _process() {\n    const scrollTop    = this._getScrollTop() + this._config.offset\n    const scrollHeight = this._getScrollHeight()\n    const maxScroll    = this._config.offset +\n      scrollHeight -\n      this._getOffsetHeight()\n\n    if (this._scrollHeight !== scrollHeight) {\n      this.refresh()\n    }\n\n    if (scrollTop >= maxScroll) {\n      const target = this._targets[this._targets.length - 1]\n\n      if (this._activeTarget !== target) {\n        this._activate(target)\n      }\n      return\n    }\n\n    if (this._activeTarget && scrollTop < this._offsets[0] && this._offsets[0] > 0) {\n      this._activeTarget = null\n      this._clear()\n      return\n    }\n\n    const offsetLength = this._offsets.length\n    for (let i = offsetLength; i--;) {\n      const isActiveTarget = this._activeTarget !== this._targets[i] &&\n          scrollTop >= this._offsets[i] &&\n          (typeof this._offsets[i + 1] === 'undefined' ||\n              scrollTop < this._offsets[i + 1])\n\n      if (isActiveTarget) {\n        this._activate(this._targets[i])\n      }\n    }\n  }\n\n  _activate(target) {\n    this._activeTarget = target\n\n    this._clear()\n\n    const queries = this._selector\n      .split(',')\n      .map((selector) => `${selector}[data-target=\"${target}\"],${selector}[href=\"${target}\"]`)\n\n    const $link = $([].slice.call(document.querySelectorAll(queries.join(','))))\n\n    if ($link.hasClass(ClassName.DROPDOWN_ITEM)) {\n      $link.closest(Selector.DROPDOWN).find(Selector.DROPDOWN_TOGGLE).addClass(ClassName.ACTIVE)\n      $link.addClass(ClassName.ACTIVE)\n    } else {\n      // Set triggered link as active\n      $link.addClass(ClassName.ACTIVE)\n      // Set triggered links parents as active\n      // With both <ul> and <nav> markup a parent is the previous sibling of any nav ancestor\n      $link.parents(Selector.NAV_LIST_GROUP).prev(`${Selector.NAV_LINKS}, ${Selector.LIST_ITEMS}`).addClass(ClassName.ACTIVE)\n      // Handle special case when .nav-link is inside .nav-item\n      $link.parents(Selector.NAV_LIST_GROUP).prev(Selector.NAV_ITEMS).children(Selector.NAV_LINKS).addClass(ClassName.ACTIVE)\n    }\n\n    $(this._scrollElement).trigger(Event.ACTIVATE, {\n      relatedTarget: target\n    })\n  }\n\n  _clear() {\n    [].slice.call(document.querySelectorAll(this._selector))\n      .filter((node) => node.classList.contains(ClassName.ACTIVE))\n      .forEach((node) => node.classList.remove(ClassName.ACTIVE))\n  }\n\n  // Static\n\n  static _jQueryInterface(config) {\n    return this.each(function () {\n      let data = $(this).data(DATA_KEY)\n      const _config = typeof config === 'object' && config\n\n      if (!data) {\n        data = new ScrollSpy(this, _config)\n        $(this).data(DATA_KEY, data)\n      }\n\n      if (typeof config === 'string') {\n        if (typeof data[config] === 'undefined') {\n          throw new TypeError(`No method named \"${config}\"`)\n        }\n        data[config]()\n      }\n    })\n  }\n}\n\n/**\n * ------------------------------------------------------------------------\n * Data Api implementation\n * ------------------------------------------------------------------------\n */\n\n$(window).on(Event.LOAD_DATA_API, () => {\n  const scrollSpys = [].slice.call(document.querySelectorAll(Selector.DATA_SPY))\n  const scrollSpysLength = scrollSpys.length\n\n  for (let i = scrollSpysLength; i--;) {\n    const $spy = $(scrollSpys[i])\n    ScrollSpy._jQueryInterface.call($spy, $spy.data())\n  }\n})\n\n/**\n * ------------------------------------------------------------------------\n * jQuery\n * ------------------------------------------------------------------------\n */\n\n$.fn[NAME] = ScrollSpy._jQueryInterface\n$.fn[NAME].Constructor = ScrollSpy\n$.fn[NAME].noConflict = () => {\n  $.fn[NAME] = JQUERY_NO_CONFLICT\n  return ScrollSpy._jQueryInterface\n}\n\nexport default ScrollSpy\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap (v4.4.1): tab.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport $ from 'jquery'\nimport Util from './util'\n\n/**\n * ------------------------------------------------------------------------\n * Constants\n * ------------------------------------------------------------------------\n */\n\nconst NAME               = 'tab'\nconst VERSION            = '4.4.1'\nconst DATA_KEY           = 'bs.tab'\nconst EVENT_KEY          = `.${DATA_KEY}`\nconst DATA_API_KEY       = '.data-api'\nconst JQUERY_NO_CONFLICT = $.fn[NAME]\n\nconst Event = {\n  HIDE           : `hide${EVENT_KEY}`,\n  HIDDEN         : `hidden${EVENT_KEY}`,\n  SHOW           : `show${EVENT_KEY}`,\n  SHOWN          : `shown${EVENT_KEY}`,\n  CLICK_DATA_API : `click${EVENT_KEY}${DATA_API_KEY}`\n}\n\nconst ClassName = {\n  DROPDOWN_MENU : 'dropdown-menu',\n  ACTIVE        : 'active',\n  DISABLED      : 'disabled',\n  FADE          : 'fade',\n  SHOW          : 'show'\n}\n\nconst Selector = {\n  DROPDOWN              : '.dropdown',\n  NAV_LIST_GROUP        : '.nav, .list-group',\n  ACTIVE                : '.active',\n  ACTIVE_UL             : '> li > .active',\n  DATA_TOGGLE           : '[data-toggle=\"tab\"], [data-toggle=\"pill\"], [data-toggle=\"list\"]',\n  DROPDOWN_TOGGLE       : '.dropdown-toggle',\n  DROPDOWN_ACTIVE_CHILD : '> .dropdown-menu .active'\n}\n\n/**\n * ------------------------------------------------------------------------\n * Class Definition\n * ------------------------------------------------------------------------\n */\n\nclass Tab {\n  constructor(element) {\n    this._element = element\n  }\n\n  // Getters\n\n  static get VERSION() {\n    return VERSION\n  }\n\n  // Public\n\n  show() {\n    if (this._element.parentNode &&\n        this._element.parentNode.nodeType === Node.ELEMENT_NODE &&\n        $(this._element).hasClass(ClassName.ACTIVE) ||\n        $(this._element).hasClass(ClassName.DISABLED)) {\n      return\n    }\n\n    let target\n    let previous\n    const listElement = $(this._element).closest(Selector.NAV_LIST_GROUP)[0]\n    const selector = Util.getSelectorFromElement(this._element)\n\n    if (listElement) {\n      const itemSelector = listElement.nodeName === 'UL' || listElement.nodeName === 'OL' ? Selector.ACTIVE_UL : Selector.ACTIVE\n      previous = $.makeArray($(listElement).find(itemSelector))\n      previous = previous[previous.length - 1]\n    }\n\n    const hideEvent = $.Event(Event.HIDE, {\n      relatedTarget: this._element\n    })\n\n    const showEvent = $.Event(Event.SHOW, {\n      relatedTarget: previous\n    })\n\n    if (previous) {\n      $(previous).trigger(hideEvent)\n    }\n\n    $(this._element).trigger(showEvent)\n\n    if (showEvent.isDefaultPrevented() ||\n        hideEvent.isDefaultPrevented()) {\n      return\n    }\n\n    if (selector) {\n      target = document.querySelector(selector)\n    }\n\n    this._activate(\n      this._element,\n      listElement\n    )\n\n    const complete = () => {\n      const hiddenEvent = $.Event(Event.HIDDEN, {\n        relatedTarget: this._element\n      })\n\n      const shownEvent = $.Event(Event.SHOWN, {\n        relatedTarget: previous\n      })\n\n      $(previous).trigger(hiddenEvent)\n      $(this._element).trigger(shownEvent)\n    }\n\n    if (target) {\n      this._activate(target, target.parentNode, complete)\n    } else {\n      complete()\n    }\n  }\n\n  dispose() {\n    $.removeData(this._element, DATA_KEY)\n    this._element = null\n  }\n\n  // Private\n\n  _activate(element, container, callback) {\n    const activeElements = container && (container.nodeName === 'UL' || container.nodeName === 'OL')\n      ? $(container).find(Selector.ACTIVE_UL)\n      : $(container).children(Selector.ACTIVE)\n\n    const active = activeElements[0]\n    const isTransitioning = callback && (active && $(active).hasClass(ClassName.FADE))\n    const complete = () => this._transitionComplete(\n      element,\n      active,\n      callback\n    )\n\n    if (active && isTransitioning) {\n      const transitionDuration = Util.getTransitionDurationFromElement(active)\n\n      $(active)\n        .removeClass(ClassName.SHOW)\n        .one(Util.TRANSITION_END, complete)\n        .emulateTransitionEnd(transitionDuration)\n    } else {\n      complete()\n    }\n  }\n\n  _transitionComplete(element, active, callback) {\n    if (active) {\n      $(active).removeClass(ClassName.ACTIVE)\n\n      const dropdownChild = $(active.parentNode).find(\n        Selector.DROPDOWN_ACTIVE_CHILD\n      )[0]\n\n      if (dropdownChild) {\n        $(dropdownChild).removeClass(ClassName.ACTIVE)\n      }\n\n      if (active.getAttribute('role') === 'tab') {\n        active.setAttribute('aria-selected', false)\n      }\n    }\n\n    $(element).addClass(ClassName.ACTIVE)\n    if (element.getAttribute('role') === 'tab') {\n      element.setAttribute('aria-selected', true)\n    }\n\n    Util.reflow(element)\n\n    if (element.classList.contains(ClassName.FADE)) {\n      element.classList.add(ClassName.SHOW)\n    }\n\n    if (element.parentNode && $(element.parentNode).hasClass(ClassName.DROPDOWN_MENU)) {\n      const dropdownElement = $(element).closest(Selector.DROPDOWN)[0]\n\n      if (dropdownElement) {\n        const dropdownToggleList = [].slice.call(dropdownElement.querySelectorAll(Selector.DROPDOWN_TOGGLE))\n\n        $(dropdownToggleList).addClass(ClassName.ACTIVE)\n      }\n\n      element.setAttribute('aria-expanded', true)\n    }\n\n    if (callback) {\n      callback()\n    }\n  }\n\n  // Static\n\n  static _jQueryInterface(config) {\n    return this.each(function () {\n      const $this = $(this)\n      let data = $this.data(DATA_KEY)\n\n      if (!data) {\n        data = new Tab(this)\n        $this.data(DATA_KEY, data)\n      }\n\n      if (typeof config === 'string') {\n        if (typeof data[config] === 'undefined') {\n          throw new TypeError(`No method named \"${config}\"`)\n        }\n        data[config]()\n      }\n    })\n  }\n}\n\n/**\n * ------------------------------------------------------------------------\n * Data Api implementation\n * ------------------------------------------------------------------------\n */\n\n$(document)\n  .on(Event.CLICK_DATA_API, Selector.DATA_TOGGLE, function (event) {\n    event.preventDefault()\n    Tab._jQueryInterface.call($(this), 'show')\n  })\n\n/**\n * ------------------------------------------------------------------------\n * jQuery\n * ------------------------------------------------------------------------\n */\n\n$.fn[NAME] = Tab._jQueryInterface\n$.fn[NAME].Constructor = Tab\n$.fn[NAME].noConflict = () => {\n  $.fn[NAME] = JQUERY_NO_CONFLICT\n  return Tab._jQueryInterface\n}\n\nexport default Tab\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap (v4.4.1): toast.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport $ from 'jquery'\nimport Util from './util'\n\n/**\n * ------------------------------------------------------------------------\n * Constants\n * ------------------------------------------------------------------------\n */\n\nconst NAME               = 'toast'\nconst VERSION            = '4.4.1'\nconst DATA_KEY           = 'bs.toast'\nconst EVENT_KEY          = `.${DATA_KEY}`\nconst JQUERY_NO_CONFLICT = $.fn[NAME]\n\nconst Event = {\n  CLICK_DISMISS : `click.dismiss${EVENT_KEY}`,\n  HIDE          : `hide${EVENT_KEY}`,\n  HIDDEN        : `hidden${EVENT_KEY}`,\n  SHOW          : `show${EVENT_KEY}`,\n  SHOWN         : `shown${EVENT_KEY}`\n}\n\nconst ClassName = {\n  FADE    : 'fade',\n  HIDE    : 'hide',\n  SHOW    : 'show',\n  SHOWING : 'showing'\n}\n\nconst DefaultType = {\n  animation : 'boolean',\n  autohide  : 'boolean',\n  delay     : 'number'\n}\n\nconst Default = {\n  animation : true,\n  autohide  : true,\n  delay     : 500\n}\n\nconst Selector = {\n  DATA_DISMISS : '[data-dismiss=\"toast\"]'\n}\n\n/**\n * ------------------------------------------------------------------------\n * Class Definition\n * ------------------------------------------------------------------------\n */\n\nclass Toast {\n  constructor(element, config) {\n    this._element = element\n    this._config  = this._getConfig(config)\n    this._timeout = null\n    this._setListeners()\n  }\n\n  // Getters\n\n  static get VERSION() {\n    return VERSION\n  }\n\n  static get DefaultType() {\n    return DefaultType\n  }\n\n  static get Default() {\n    return Default\n  }\n\n  // Public\n\n  show() {\n    const showEvent = $.Event(Event.SHOW)\n\n    $(this._element).trigger(showEvent)\n    if (showEvent.isDefaultPrevented()) {\n      return\n    }\n\n    if (this._config.animation) {\n      this._element.classList.add(ClassName.FADE)\n    }\n\n    const complete = () => {\n      this._element.classList.remove(ClassName.SHOWING)\n      this._element.classList.add(ClassName.SHOW)\n\n      $(this._element).trigger(Event.SHOWN)\n\n      if (this._config.autohide) {\n        this._timeout = setTimeout(() => {\n          this.hide()\n        }, this._config.delay)\n      }\n    }\n\n    this._element.classList.remove(ClassName.HIDE)\n    Util.reflow(this._element)\n    this._element.classList.add(ClassName.SHOWING)\n    if (this._config.animation) {\n      const transitionDuration = Util.getTransitionDurationFromElement(this._element)\n\n      $(this._element)\n        .one(Util.TRANSITION_END, complete)\n        .emulateTransitionEnd(transitionDuration)\n    } else {\n      complete()\n    }\n  }\n\n  hide() {\n    if (!this._element.classList.contains(ClassName.SHOW)) {\n      return\n    }\n\n    const hideEvent = $.Event(Event.HIDE)\n\n    $(this._element).trigger(hideEvent)\n    if (hideEvent.isDefaultPrevented()) {\n      return\n    }\n\n    this._close()\n  }\n\n  dispose() {\n    clearTimeout(this._timeout)\n    this._timeout = null\n\n    if (this._element.classList.contains(ClassName.SHOW)) {\n      this._element.classList.remove(ClassName.SHOW)\n    }\n\n    $(this._element).off(Event.CLICK_DISMISS)\n\n    $.removeData(this._element, DATA_KEY)\n    this._element = null\n    this._config  = null\n  }\n\n  // Private\n\n  _getConfig(config) {\n    config = {\n      ...Default,\n      ...$(this._element).data(),\n      ...typeof config === 'object' && config ? config : {}\n    }\n\n    Util.typeCheckConfig(\n      NAME,\n      config,\n      this.constructor.DefaultType\n    )\n\n    return config\n  }\n\n  _setListeners() {\n    $(this._element).on(\n      Event.CLICK_DISMISS,\n      Selector.DATA_DISMISS,\n      () => this.hide()\n    )\n  }\n\n  _close() {\n    const complete = () => {\n      this._element.classList.add(ClassName.HIDE)\n      $(this._element).trigger(Event.HIDDEN)\n    }\n\n    this._element.classList.remove(ClassName.SHOW)\n    if (this._config.animation) {\n      const transitionDuration = Util.getTransitionDurationFromElement(this._element)\n\n      $(this._element)\n        .one(Util.TRANSITION_END, complete)\n        .emulateTransitionEnd(transitionDuration)\n    } else {\n      complete()\n    }\n  }\n\n  // Static\n\n  static _jQueryInterface(config) {\n    return this.each(function () {\n      const $element = $(this)\n      let data       = $element.data(DATA_KEY)\n      const _config  = typeof config === 'object' && config\n\n      if (!data) {\n        data = new Toast(this, _config)\n        $element.data(DATA_KEY, data)\n      }\n\n      if (typeof config === 'string') {\n        if (typeof data[config] === 'undefined') {\n          throw new TypeError(`No method named \"${config}\"`)\n        }\n\n        data[config](this)\n      }\n    })\n  }\n}\n\n/**\n * ------------------------------------------------------------------------\n * jQuery\n * ------------------------------------------------------------------------\n */\n\n$.fn[NAME]             = Toast._jQueryInterface\n$.fn[NAME].Constructor = Toast\n$.fn[NAME].noConflict  = () => {\n  $.fn[NAME] = JQUERY_NO_CONFLICT\n  return Toast._jQueryInterface\n}\n\nexport default Toast\n"]}
\ No newline at end of file
diff --git a/docs/js/vendor/bootstrap.js b/docs/js/vendor/bootstrap.js
deleted file mode 100755
index 7f303eb88a106..0000000000000
--- a/docs/js/vendor/bootstrap.js
+++ /dev/null
@@ -1,2027 +0,0 @@
-/* ===================================================
- * bootstrap-transition.js v2.1.0
- * http://twitter.github.com/bootstrap/javascript.html#transitions
- * ===================================================
- * Copyright 2012 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ========================================================== */
-
-
-!function ($) {
-
-  $(function () {
-
-    "use strict"; // jshint ;_;
-
-
-    /* CSS TRANSITION SUPPORT (http://www.modernizr.com/)
-     * ======================================================= */
-
-    $.support.transition = (function () {
-
-      var transitionEnd = (function () {
-
-        var el = document.createElement('bootstrap')
-          , transEndEventNames = {
-               'WebkitTransition' : 'webkitTransitionEnd'
-            ,  'MozTransition'    : 'transitionend'
-            ,  'OTransition'      : 'oTransitionEnd otransitionend'
-            ,  'transition'       : 'transitionend'
-            }
-          , name
-
-        for (name in transEndEventNames){
-          if (el.style[name] !== undefined) {
-            return transEndEventNames[name]
-          }
-        }
-
-      }())
-
-      return transitionEnd && {
-        end: transitionEnd
-      }
-
-    })()
-
-  })
-
-}(window.jQuery);/* ==========================================================
- * bootstrap-alert.js v2.1.0
- * http://twitter.github.com/bootstrap/javascript.html#alerts
- * ==========================================================
- * Copyright 2012 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ========================================================== */
-
-
-!function ($) {
-
-  "use strict"; // jshint ;_;
-
-
- /* ALERT CLASS DEFINITION
-  * ====================== */
-
-  var dismiss = '[data-dismiss="alert"]'
-    , Alert = function (el) {
-        $(el).on('click', dismiss, this.close)
-      }
-
-  Alert.prototype.close = function (e) {
-    var $this = $(this)
-      , selector = $this.attr('data-target')
-      , $parent
-
-    if (!selector) {
-      selector = $this.attr('href')
-      selector = selector && selector.replace(/.*(?=#[^\s]*$)/, '') //strip for ie7
-    }
-
-    $parent = $(selector)
-
-    e && e.preventDefault()
-
-    $parent.length || ($parent = $this.hasClass('alert') ? $this : $this.parent())
-
-    $parent.trigger(e = $.Event('close'))
-
-    if (e.isDefaultPrevented()) return
-
-    $parent.removeClass('in')
-
-    function removeElement() {
-      $parent
-        .trigger('closed')
-        .remove()
-    }
-
-    $.support.transition && $parent.hasClass('fade') ?
-      $parent.on($.support.transition.end, removeElement) :
-      removeElement()
-  }
-
-
- /* ALERT PLUGIN DEFINITION
-  * ======================= */
-
-  $.fn.alert = function (option) {
-    return this.each(function () {
-      var $this = $(this)
-        , data = $this.data('alert')
-      if (!data) $this.data('alert', (data = new Alert(this)))
-      if (typeof option == 'string') data[option].call($this)
-    })
-  }
-
-  $.fn.alert.Constructor = Alert
-
-
- /* ALERT DATA-API
-  * ============== */
-
-  $(function () {
-    $('body').on('click.alert.data-api', dismiss, Alert.prototype.close)
-  })
-
-}(window.jQuery);/* ============================================================
- * bootstrap-button.js v2.1.0
- * http://twitter.github.com/bootstrap/javascript.html#buttons
- * ============================================================
- * Copyright 2012 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ============================================================ */
-
-
-!function ($) {
-
-  "use strict"; // jshint ;_;
-
-
- /* BUTTON PUBLIC CLASS DEFINITION
-  * ============================== */
-
-  var Button = function (element, options) {
-    this.$element = $(element)
-    this.options = $.extend({}, $.fn.button.defaults, options)
-  }
-
-  Button.prototype.setState = function (state) {
-    var d = 'disabled'
-      , $el = this.$element
-      , data = $el.data()
-      , val = $el.is('input') ? 'val' : 'html'
-
-    state = state + 'Text'
-    data.resetText || $el.data('resetText', $el[val]())
-
-    $el[val](data[state] || this.options[state])
-
-    // push to event loop to allow forms to submit
-    setTimeout(function () {
-      state == 'loadingText' ?
-        $el.addClass(d).attr(d, d) :
-        $el.removeClass(d).removeAttr(d)
-    }, 0)
-  }
-
-  Button.prototype.toggle = function () {
-    var $parent = this.$element.parent('[data-toggle="buttons-radio"]')
-
-    $parent && $parent
-      .find('.active')
-      .removeClass('active')
-
-    this.$element.toggleClass('active')
-  }
-
-
- /* BUTTON PLUGIN DEFINITION
-  * ======================== */
-
-  $.fn.button = function (option) {
-    return this.each(function () {
-      var $this = $(this)
-        , data = $this.data('button')
-        , options = typeof option == 'object' && option
-      if (!data) $this.data('button', (data = new Button(this, options)))
-      if (option == 'toggle') data.toggle()
-      else if (option) data.setState(option)
-    })
-  }
-
-  $.fn.button.defaults = {
-    loadingText: 'loading...'
-  }
-
-  $.fn.button.Constructor = Button
-
-
- /* BUTTON DATA-API
-  * =============== */
-
-  $(function () {
-    $('body').on('click.button.data-api', '[data-toggle^=button]', function ( e ) {
-      var $btn = $(e.target)
-      if (!$btn.hasClass('btn')) $btn = $btn.closest('.btn')
-      $btn.button('toggle')
-    })
-  })
-
-}(window.jQuery);/* ==========================================================
- * bootstrap-carousel.js v2.1.0
- * http://twitter.github.com/bootstrap/javascript.html#carousel
- * ==========================================================
- * Copyright 2012 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ========================================================== */
-
-
-!function ($) {
-
-  "use strict"; // jshint ;_;
-
-
- /* CAROUSEL CLASS DEFINITION
-  * ========================= */
-
-  var Carousel = function (element, options) {
-    this.$element = $(element)
-    this.options = options
-    this.options.slide && this.slide(this.options.slide)
-    this.options.pause == 'hover' && this.$element
-      .on('mouseenter', $.proxy(this.pause, this))
-      .on('mouseleave', $.proxy(this.cycle, this))
-  }
-
-  Carousel.prototype = {
-
-    cycle: function (e) {
-      if (!e) this.paused = false
-      this.options.interval
-        && !this.paused
-        && (this.interval = setInterval($.proxy(this.next, this), this.options.interval))
-      return this
-    }
-
-  , to: function (pos) {
-      var $active = this.$element.find('.item.active')
-        , children = $active.parent().children()
-        , activePos = children.index($active)
-        , that = this
-
-      if (pos > (children.length - 1) || pos < 0) return
-
-      if (this.sliding) {
-        return this.$element.one('slid', function () {
-          that.to(pos)
-        })
-      }
-
-      if (activePos == pos) {
-        return this.pause().cycle()
-      }
-
-      return this.slide(pos > activePos ? 'next' : 'prev', $(children[pos]))
-    }
-
-  , pause: function (e) {
-      if (!e) this.paused = true
-      if (this.$element.find('.next, .prev').length && $.support.transition.end) {
-        this.$element.trigger($.support.transition.end)
-        this.cycle()
-      }
-      clearInterval(this.interval)
-      this.interval = null
-      return this
-    }
-
-  , next: function () {
-      if (this.sliding) return
-      return this.slide('next')
-    }
-
-  , prev: function () {
-      if (this.sliding) return
-      return this.slide('prev')
-    }
-
-  , slide: function (type, next) {
-      var $active = this.$element.find('.item.active')
-        , $next = next || $active[type]()
-        , isCycling = this.interval
-        , direction = type == 'next' ? 'left' : 'right'
-        , fallback  = type == 'next' ? 'first' : 'last'
-        , that = this
-        , e = $.Event('slide', {
-            relatedTarget: $next[0]
-          })
-
-      this.sliding = true
-
-      isCycling && this.pause()
-
-      $next = $next.length ? $next : this.$element.find('.item')[fallback]()
-
-      if ($next.hasClass('active')) return
-
-      if ($.support.transition && this.$element.hasClass('slide')) {
-        this.$element.trigger(e)
-        if (e.isDefaultPrevented()) return
-        $next.addClass(type)
-        $next[0].offsetWidth // force reflow
-        $active.addClass(direction)
-        $next.addClass(direction)
-        this.$element.one($.support.transition.end, function () {
-          $next.removeClass([type, direction].join(' ')).addClass('active')
-          $active.removeClass(['active', direction].join(' '))
-          that.sliding = false
-          setTimeout(function () { that.$element.trigger('slid') }, 0)
-        })
-      } else {
-        this.$element.trigger(e)
-        if (e.isDefaultPrevented()) return
-        $active.removeClass('active')
-        $next.addClass('active')
-        this.sliding = false
-        this.$element.trigger('slid')
-      }
-
-      isCycling && this.cycle()
-
-      return this
-    }
-
-  }
-
-
- /* CAROUSEL PLUGIN DEFINITION
-  * ========================== */
-
-  $.fn.carousel = function (option) {
-    return this.each(function () {
-      var $this = $(this)
-        , data = $this.data('carousel')
-        , options = $.extend({}, $.fn.carousel.defaults, typeof option == 'object' && option)
-        , action = typeof option == 'string' ? option : options.slide
-      if (!data) $this.data('carousel', (data = new Carousel(this, options)))
-      if (typeof option == 'number') data.to(option)
-      else if (action) data[action]()
-      else if (options.interval) data.cycle()
-    })
-  }
-
-  $.fn.carousel.defaults = {
-    interval: 5000
-  , pause: 'hover'
-  }
-
-  $.fn.carousel.Constructor = Carousel
-
-
- /* CAROUSEL DATA-API
-  * ================= */
-
-  $(function () {
-    $('body').on('click.carousel.data-api', '[data-slide]', function ( e ) {
-      var $this = $(this), href
-        , $target = $($this.attr('data-target') || (href = $this.attr('href')) && href.replace(/.*(?=#[^\s]+$)/, '')) //strip for ie7
-        , options = !$target.data('modal') && $.extend({}, $target.data(), $this.data())
-      $target.carousel(options)
-      e.preventDefault()
-    })
-  })
-
-}(window.jQuery);/* =============================================================
- * bootstrap-collapse.js v2.1.0
- * http://twitter.github.com/bootstrap/javascript.html#collapse
- * =============================================================
- * Copyright 2012 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ============================================================ */
-
-
-!function ($) {
-
-  "use strict"; // jshint ;_;
-
-
- /* COLLAPSE PUBLIC CLASS DEFINITION
-  * ================================ */
-
-  var Collapse = function (element, options) {
-    this.$element = $(element)
-    this.options = $.extend({}, $.fn.collapse.defaults, options)
-
-    if (this.options.parent) {
-      this.$parent = $(this.options.parent)
-    }
-
-    this.options.toggle && this.toggle()
-  }
-
-  Collapse.prototype = {
-
-    constructor: Collapse
-
-  , dimension: function () {
-      var hasWidth = this.$element.hasClass('width')
-      return hasWidth ? 'width' : 'height'
-    }
-
-  , show: function () {
-      var dimension
-        , scroll
-        , actives
-        , hasData
-
-      if (this.transitioning) return
-
-      dimension = this.dimension()
-      scroll = $.camelCase(['scroll', dimension].join('-'))
-      actives = this.$parent && this.$parent.find('> .accordion-group > .in')
-
-      if (actives && actives.length) {
-        hasData = actives.data('collapse')
-        if (hasData && hasData.transitioning) return
-        actives.collapse('hide')
-        hasData || actives.data('collapse', null)
-      }
-
-      this.$element[dimension](0)
-      this.transition('addClass', $.Event('show'), 'shown')
-      $.support.transition && this.$element[dimension](this.$element[0][scroll])
-    }
-
-  , hide: function () {
-      var dimension
-      if (this.transitioning) return
-      dimension = this.dimension()
-      this.reset(this.$element[dimension]())
-      this.transition('removeClass', $.Event('hide'), 'hidden')
-      this.$element[dimension](0)
-    }
-
-  , reset: function (size) {
-      var dimension = this.dimension()
-
-      this.$element
-        .removeClass('collapse')
-        [dimension](size || 'auto')
-        [0].offsetWidth
-
-      this.$element[size !== null ? 'addClass' : 'removeClass']('collapse')
-
-      return this
-    }
-
-  , transition: function (method, startEvent, completeEvent) {
-      var that = this
-        , complete = function () {
-            if (startEvent.type == 'show') that.reset()
-            that.transitioning = 0
-            that.$element.trigger(completeEvent)
-          }
-
-      this.$element.trigger(startEvent)
-
-      if (startEvent.isDefaultPrevented()) return
-
-      this.transitioning = 1
-
-      this.$element[method]('in')
-
-      $.support.transition && this.$element.hasClass('collapse') ?
-        this.$element.one($.support.transition.end, complete) :
-        complete()
-    }
-
-  , toggle: function () {
-      this[this.$element.hasClass('in') ? 'hide' : 'show']()
-    }
-
-  }
-
-
- /* COLLAPSIBLE PLUGIN DEFINITION
-  * ============================== */
-
-  $.fn.collapse = function (option) {
-    return this.each(function () {
-      var $this = $(this)
-        , data = $this.data('collapse')
-        , options = typeof option == 'object' && option
-      if (!data) $this.data('collapse', (data = new Collapse(this, options)))
-      if (typeof option == 'string') data[option]()
-    })
-  }
-
-  $.fn.collapse.defaults = {
-    toggle: true
-  }
-
-  $.fn.collapse.Constructor = Collapse
-
-
- /* COLLAPSIBLE DATA-API
-  * ==================== */
-
-  $(function () {
-    $('body').on('click.collapse.data-api', '[data-toggle=collapse]', function (e) {
-      var $this = $(this), href
-        , target = $this.attr('data-target')
-          || e.preventDefault()
-          || (href = $this.attr('href')) && href.replace(/.*(?=#[^\s]+$)/, '') //strip for ie7
-        , option = $(target).data('collapse') ? 'toggle' : $this.data()
-      $this[$(target).hasClass('in') ? 'addClass' : 'removeClass']('collapsed')
-      $(target).collapse(option)
-    })
-  })
-
-}(window.jQuery);/* ============================================================
- * bootstrap-dropdown.js v2.1.0
- * http://twitter.github.com/bootstrap/javascript.html#dropdowns
- * ============================================================
- * Copyright 2012 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ============================================================ */
-
-
-!function ($) {
-
-  "use strict"; // jshint ;_;
-
-
- /* DROPDOWN CLASS DEFINITION
-  * ========================= */
-
-  var toggle = '[data-toggle=dropdown]'
-    , Dropdown = function (element) {
-        var $el = $(element).on('click.dropdown.data-api', this.toggle)
-        $('html').on('click.dropdown.data-api', function () {
-          $el.parent().removeClass('open')
-        })
-      }
-
-  Dropdown.prototype = {
-
-    constructor: Dropdown
-
-  , toggle: function (e) {
-      var $this = $(this)
-        , $parent
-        , isActive
-
-      if ($this.is('.disabled, :disabled')) return
-
-      $parent = getParent($this)
-
-      isActive = $parent.hasClass('open')
-
-      clearMenus()
-
-      if (!isActive) {
-        $parent.toggleClass('open')
-        $this.focus()
-      }
-
-      return false
-    }
-
-  , keydown: function (e) {
-      var $this
-        , $items
-        , $active
-        , $parent
-        , isActive
-        , index
-
-      if (!/(38|40|27)/.test(e.keyCode)) return
-
-      $this = $(this)
-
-      e.preventDefault()
-      e.stopPropagation()
-
-      if ($this.is('.disabled, :disabled')) return
-
-      $parent = getParent($this)
-
-      isActive = $parent.hasClass('open')
-
-      if (!isActive || (isActive && e.keyCode == 27)) return $this.click()
-
-      $items = $('[role=menu] li:not(.divider) a', $parent)
-
-      if (!$items.length) return
-
-      index = $items.index($items.filter(':focus'))
-
-      if (e.keyCode == 38 && index > 0) index--                                        // up
-      if (e.keyCode == 40 && index < $items.length - 1) index++                        // down
-      if (!~index) index = 0
-
-      $items
-        .eq(index)
-        .focus()
-    }
-
-  }
-
-  function clearMenus() {
-    getParent($(toggle))
-      .removeClass('open')
-  }
-
-  function getParent($this) {
-    var selector = $this.attr('data-target')
-      , $parent
-
-    if (!selector) {
-      selector = $this.attr('href')
-      selector = selector && selector.replace(/.*(?=#[^\s]*$)/, '') //strip for ie7
-    }
-
-    $parent = $(selector)
-    $parent.length || ($parent = $this.parent())
-
-    return $parent
-  }
-
-
-  /* DROPDOWN PLUGIN DEFINITION
-   * ========================== */
-
-  $.fn.dropdown = function (option) {
-    return this.each(function () {
-      var $this = $(this)
-        , data = $this.data('dropdown')
-      if (!data) $this.data('dropdown', (data = new Dropdown(this)))
-      if (typeof option == 'string') data[option].call($this)
-    })
-  }
-
-  $.fn.dropdown.Constructor = Dropdown
-
-
-  /* APPLY TO STANDARD DROPDOWN ELEMENTS
-   * =================================== */
-
-  $(function () {
-    $('html')
-      .on('click.dropdown.data-api touchstart.dropdown.data-api', clearMenus)
-    $('body')
-      .on('click.dropdown touchstart.dropdown.data-api', '.dropdown', function (e) { e.stopPropagation() })
-      .on('click.dropdown.data-api touchstart.dropdown.data-api'  , toggle, Dropdown.prototype.toggle)
-      .on('keydown.dropdown.data-api touchstart.dropdown.data-api', toggle + ', [role=menu]' , Dropdown.prototype.keydown)
-  })
-
-}(window.jQuery);/* =========================================================
- * bootstrap-modal.js v2.1.0
- * http://twitter.github.com/bootstrap/javascript.html#modals
- * =========================================================
- * Copyright 2012 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ========================================================= */
-
-
-!function ($) {
-
-  "use strict"; // jshint ;_;
-
-
- /* MODAL CLASS DEFINITION
-  * ====================== */
-
-  var Modal = function (element, options) {
-    this.options = options
-    this.$element = $(element)
-      .delegate('[data-dismiss="modal"]', 'click.dismiss.modal', $.proxy(this.hide, this))
-    this.options.remote && this.$element.find('.modal-body').load(this.options.remote)
-  }
-
-  Modal.prototype = {
-
-      constructor: Modal
-
-    , toggle: function () {
-        return this[!this.isShown ? 'show' : 'hide']()
-      }
-
-    , show: function () {
-        var that = this
-          , e = $.Event('show')
-
-        this.$element.trigger(e)
-
-        if (this.isShown || e.isDefaultPrevented()) return
-
-        $('body').addClass('modal-open')
-
-        this.isShown = true
-
-        this.escape()
-
-        this.backdrop(function () {
-          var transition = $.support.transition && that.$element.hasClass('fade')
-
-          if (!that.$element.parent().length) {
-            that.$element.appendTo(document.body) //don't move modals dom position
-          }
-
-          that.$element
-            .show()
-
-          if (transition) {
-            that.$element[0].offsetWidth // force reflow
-          }
-
-          that.$element
-            .addClass('in')
-            .attr('aria-hidden', false)
-            .focus()
-
-          that.enforceFocus()
-
-          transition ?
-            that.$element.one($.support.transition.end, function () { that.$element.trigger('shown') }) :
-            that.$element.trigger('shown')
-
-        })
-      }
-
-    , hide: function (e) {
-        e && e.preventDefault()
-
-        var that = this
-
-        e = $.Event('hide')
-
-        this.$element.trigger(e)
-
-        if (!this.isShown || e.isDefaultPrevented()) return
-
-        this.isShown = false
-
-        $('body').removeClass('modal-open')
-
-        this.escape()
-
-        $(document).off('focusin.modal')
-
-        this.$element
-          .removeClass('in')
-          .attr('aria-hidden', true)
-
-        $.support.transition && this.$element.hasClass('fade') ?
-          this.hideWithTransition() :
-          this.hideModal()
-      }
-
-    , enforceFocus: function () {
-        var that = this
-        $(document).on('focusin.modal', function (e) {
-          if (that.$element[0] !== e.target && !that.$element.has(e.target).length) {
-            that.$element.focus()
-          }
-        })
-      }
-
-    , escape: function () {
-        var that = this
-        if (this.isShown && this.options.keyboard) {
-          this.$element.on('keyup.dismiss.modal', function ( e ) {
-            e.which == 27 && that.hide()
-          })
-        } else if (!this.isShown) {
-          this.$element.off('keyup.dismiss.modal')
-        }
-      }
-
-    , hideWithTransition: function () {
-        var that = this
-          , timeout = setTimeout(function () {
-              that.$element.off($.support.transition.end)
-              that.hideModal()
-            }, 500)
-
-        this.$element.one($.support.transition.end, function () {
-          clearTimeout(timeout)
-          that.hideModal()
-        })
-      }
-
-    , hideModal: function (that) {
-        this.$element
-          .hide()
-          .trigger('hidden')
-
-        this.backdrop()
-      }
-
-    , removeBackdrop: function () {
-        this.$backdrop.remove()
-        this.$backdrop = null
-      }
-
-    , backdrop: function (callback) {
-        var that = this
-          , animate = this.$element.hasClass('fade') ? 'fade' : ''
-
-        if (this.isShown && this.options.backdrop) {
-          var doAnimate = $.support.transition && animate
-
-          this.$backdrop = $('<div class="modal-backdrop ' + animate + '" />')
-            .appendTo(document.body)
-
-          if (this.options.backdrop != 'static') {
-            this.$backdrop.click($.proxy(this.hide, this))
-          }
-
-          if (doAnimate) this.$backdrop[0].offsetWidth // force reflow
-
-          this.$backdrop.addClass('in')
-
-          doAnimate ?
-            this.$backdrop.one($.support.transition.end, callback) :
-            callback()
-
-        } else if (!this.isShown && this.$backdrop) {
-          this.$backdrop.removeClass('in')
-
-          $.support.transition && this.$element.hasClass('fade')?
-            this.$backdrop.one($.support.transition.end, $.proxy(this.removeBackdrop, this)) :
-            this.removeBackdrop()
-
-        } else if (callback) {
-          callback()
-        }
-      }
-  }
-
-
- /* MODAL PLUGIN DEFINITION
-  * ======================= */
-
-  $.fn.modal = function (option) {
-    return this.each(function () {
-      var $this = $(this)
-        , data = $this.data('modal')
-        , options = $.extend({}, $.fn.modal.defaults, $this.data(), typeof option == 'object' && option)
-      if (!data) $this.data('modal', (data = new Modal(this, options)))
-      if (typeof option == 'string') data[option]()
-      else if (options.show) data.show()
-    })
-  }
-
-  $.fn.modal.defaults = {
-      backdrop: true
-    , keyboard: true
-    , show: true
-  }
-
-  $.fn.modal.Constructor = Modal
-
-
- /* MODAL DATA-API
-  * ============== */
-
-  $(function () {
-    $('body').on('click.modal.data-api', '[data-toggle="modal"]', function ( e ) {
-      var $this = $(this)
-        , href = $this.attr('href')
-        , $target = $($this.attr('data-target') || (href && href.replace(/.*(?=#[^\s]+$)/, ''))) //strip for ie7
-        , option = $target.data('modal') ? 'toggle' : $.extend({ remote: !/#/.test(href) && href }, $target.data(), $this.data())
-
-      e.preventDefault()
-
-      $target
-        .modal(option)
-        .one('hide', function () {
-          $this.focus()
-        })
-    })
-  })
-
-}(window.jQuery);/* ===========================================================
- * bootstrap-tooltip.js v2.1.0
- * http://twitter.github.com/bootstrap/javascript.html#tooltips
- * Inspired by the original jQuery.tipsy by Jason Frame
- * ===========================================================
- * Copyright 2012 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ========================================================== */
-
-
-!function ($) {
-
-  "use strict"; // jshint ;_;
-
-
- /* TOOLTIP PUBLIC CLASS DEFINITION
-  * =============================== */
-
-  var Tooltip = function (element, options) {
-    this.init('tooltip', element, options)
-  }
-
-  Tooltip.prototype = {
-
-    constructor: Tooltip
-
-  , init: function (type, element, options) {
-      var eventIn
-        , eventOut
-
-      this.type = type
-      this.$element = $(element)
-      this.options = this.getOptions(options)
-      this.enabled = true
-
-      if (this.options.trigger == 'click') {
-        this.$element.on('click.' + this.type, this.options.selector, $.proxy(this.toggle, this))
-      } else if (this.options.trigger != 'manual') {
-        eventIn = this.options.trigger == 'hover' ? 'mouseenter' : 'focus'
-        eventOut = this.options.trigger == 'hover' ? 'mouseleave' : 'blur'
-        this.$element.on(eventIn + '.' + this.type, this.options.selector, $.proxy(this.enter, this))
-        this.$element.on(eventOut + '.' + this.type, this.options.selector, $.proxy(this.leave, this))
-      }
-
-      this.options.selector ?
-        (this._options = $.extend({}, this.options, { trigger: 'manual', selector: '' })) :
-        this.fixTitle()
-    }
-
-  , getOptions: function (options) {
-      options = $.extend({}, $.fn[this.type].defaults, options, this.$element.data())
-
-      if (options.delay && typeof options.delay == 'number') {
-        options.delay = {
-          show: options.delay
-        , hide: options.delay
-        }
-      }
-
-      return options
-    }
-
-  , enter: function (e) {
-      var self = $(e.currentTarget)[this.type](this._options).data(this.type)
-
-      if (!self.options.delay || !self.options.delay.show) return self.show()
-
-      clearTimeout(this.timeout)
-      self.hoverState = 'in'
-      this.timeout = setTimeout(function() {
-        if (self.hoverState == 'in') self.show()
-      }, self.options.delay.show)
-    }
-
-  , leave: function (e) {
-      var self = $(e.currentTarget)[this.type](this._options).data(this.type)
-
-      if (this.timeout) clearTimeout(this.timeout)
-      if (!self.options.delay || !self.options.delay.hide) return self.hide()
-
-      self.hoverState = 'out'
-      this.timeout = setTimeout(function() {
-        if (self.hoverState == 'out') self.hide()
-      }, self.options.delay.hide)
-    }
-
-  , show: function () {
-      var $tip
-        , inside
-        , pos
-        , actualWidth
-        , actualHeight
-        , placement
-        , tp
-
-      if (this.hasContent() && this.enabled) {
-        $tip = this.tip()
-        this.setContent()
-
-        if (this.options.animation) {
-          $tip.addClass('fade')
-        }
-
-        placement = typeof this.options.placement == 'function' ?
-          this.options.placement.call(this, $tip[0], this.$element[0]) :
-          this.options.placement
-
-        inside = /in/.test(placement)
-
-        $tip
-          .remove()
-          .css({ top: 0, left: 0, display: 'block' })
-          .appendTo(inside ? this.$element : document.body)
-
-        pos = this.getPosition(inside)
-
-        actualWidth = $tip[0].offsetWidth
-        actualHeight = $tip[0].offsetHeight
-
-        switch (inside ? placement.split(' ')[1] : placement) {
-          case 'bottom':
-            tp = {top: pos.top + pos.height, left: pos.left + pos.width / 2 - actualWidth / 2}
-            break
-          case 'top':
-            tp = {top: pos.top - actualHeight, left: pos.left + pos.width / 2 - actualWidth / 2}
-            break
-          case 'left':
-            tp = {top: pos.top + pos.height / 2 - actualHeight / 2, left: pos.left - actualWidth}
-            break
-          case 'right':
-            tp = {top: pos.top + pos.height / 2 - actualHeight / 2, left: pos.left + pos.width}
-            break
-        }
-
-        $tip
-          .css(tp)
-          .addClass(placement)
-          .addClass('in')
-      }
-    }
-
-  , setContent: function () {
-      var $tip = this.tip()
-        , title = this.getTitle()
-
-      $tip.find('.tooltip-inner')[this.options.html ? 'html' : 'text'](title)
-      $tip.removeClass('fade in top bottom left right')
-    }
-
-  , hide: function () {
-      var that = this
-        , $tip = this.tip()
-
-      $tip.removeClass('in')
-
-      function removeWithAnimation() {
-        var timeout = setTimeout(function () {
-          $tip.off($.support.transition.end).remove()
-        }, 500)
-
-        $tip.one($.support.transition.end, function () {
-          clearTimeout(timeout)
-          $tip.remove()
-        })
-      }
-
-      $.support.transition && this.$tip.hasClass('fade') ?
-        removeWithAnimation() :
-        $tip.remove()
-
-      return this
-    }
-
-  , fixTitle: function () {
-      var $e = this.$element
-      if ($e.attr('title') || typeof($e.attr('data-original-title')) != 'string') {
-        $e.attr('data-original-title', $e.attr('title') || '').removeAttr('title')
-      }
-    }
-
-  , hasContent: function () {
-      return this.getTitle()
-    }
-
-  , getPosition: function (inside) {
-      return $.extend({}, (inside ? {top: 0, left: 0} : this.$element.offset()), {
-        width: this.$element[0].offsetWidth
-      , height: this.$element[0].offsetHeight
-      })
-    }
-
-  , getTitle: function () {
-      var title
-        , $e = this.$element
-        , o = this.options
-
-      title = $e.attr('data-original-title')
-        || (typeof o.title == 'function' ? o.title.call($e[0]) :  o.title)
-
-      return title
-    }
-
-  , tip: function () {
-      return this.$tip = this.$tip || $(this.options.template)
-    }
-
-  , validate: function () {
-      if (!this.$element[0].parentNode) {
-        this.hide()
-        this.$element = null
-        this.options = null
-      }
-    }
-
-  , enable: function () {
-      this.enabled = true
-    }
-
-  , disable: function () {
-      this.enabled = false
-    }
-
-  , toggleEnabled: function () {
-      this.enabled = !this.enabled
-    }
-
-  , toggle: function () {
-      this[this.tip().hasClass('in') ? 'hide' : 'show']()
-    }
-
-  , destroy: function () {
-      this.hide().$element.off('.' + this.type).removeData(this.type)
-    }
-
-  }
-
-
- /* TOOLTIP PLUGIN DEFINITION
-  * ========================= */
-
-  $.fn.tooltip = function ( option ) {
-    return this.each(function () {
-      var $this = $(this)
-        , data = $this.data('tooltip')
-        , options = typeof option == 'object' && option
-      if (!data) $this.data('tooltip', (data = new Tooltip(this, options)))
-      if (typeof option == 'string') data[option]()
-    })
-  }
-
-  $.fn.tooltip.Constructor = Tooltip
-
-  $.fn.tooltip.defaults = {
-    animation: true
-  , placement: 'top'
-  , selector: false
-  , template: '<div class="tooltip"><div class="tooltip-arrow"></div><div class="tooltip-inner"></div></div>'
-  , trigger: 'hover'
-  , title: ''
-  , delay: 0
-  , html: true
-  }
-
-}(window.jQuery);
-/* ===========================================================
- * bootstrap-popover.js v2.1.0
- * http://twitter.github.com/bootstrap/javascript.html#popovers
- * ===========================================================
- * Copyright 2012 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * =========================================================== */
-
-
-!function ($) {
-
-  "use strict"; // jshint ;_;
-
-
- /* POPOVER PUBLIC CLASS DEFINITION
-  * =============================== */
-
-  var Popover = function (element, options) {
-    this.init('popover', element, options)
-  }
-
-
-  /* NOTE: POPOVER EXTENDS BOOTSTRAP-TOOLTIP.js
-     ========================================== */
-
-  Popover.prototype = $.extend({}, $.fn.tooltip.Constructor.prototype, {
-
-    constructor: Popover
-
-  , setContent: function () {
-      var $tip = this.tip()
-        , title = this.getTitle()
-        , content = this.getContent()
-
-      $tip.find('.popover-title')[this.options.html ? 'html' : 'text'](title)
-      $tip.find('.popover-content > *')[this.options.html ? 'html' : 'text'](content)
-
-      $tip.removeClass('fade top bottom left right in')
-    }
-
-  , hasContent: function () {
-      return this.getTitle() || this.getContent()
-    }
-
-  , getContent: function () {
-      var content
-        , $e = this.$element
-        , o = this.options
-
-      content = $e.attr('data-content')
-        || (typeof o.content == 'function' ? o.content.call($e[0]) :  o.content)
-
-      return content
-    }
-
-  , tip: function () {
-      if (!this.$tip) {
-        this.$tip = $(this.options.template)
-      }
-      return this.$tip
-    }
-
-  , destroy: function () {
-      this.hide().$element.off('.' + this.type).removeData(this.type)
-    }
-
-  })
-
-
- /* POPOVER PLUGIN DEFINITION
-  * ======================= */
-
-  $.fn.popover = function (option) {
-    return this.each(function () {
-      var $this = $(this)
-        , data = $this.data('popover')
-        , options = typeof option == 'object' && option
-      if (!data) $this.data('popover', (data = new Popover(this, options)))
-      if (typeof option == 'string') data[option]()
-    })
-  }
-
-  $.fn.popover.Constructor = Popover
-
-  $.fn.popover.defaults = $.extend({} , $.fn.tooltip.defaults, {
-    placement: 'right'
-  , trigger: 'click'
-  , content: ''
-  , template: '<div class="popover"><div class="arrow"></div><div class="popover-inner"><h3 class="popover-title"></h3><div class="popover-content"><p></p></div></div></div>'
-  })
-
-}(window.jQuery);/* =============================================================
- * bootstrap-scrollspy.js v2.1.0
- * http://twitter.github.com/bootstrap/javascript.html#scrollspy
- * =============================================================
- * Copyright 2012 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ============================================================== */
-
-
-!function ($) {
-
-  "use strict"; // jshint ;_;
-
-
- /* SCROLLSPY CLASS DEFINITION
-  * ========================== */
-
-  function ScrollSpy(element, options) {
-    var process = $.proxy(this.process, this)
-      , $element = $(element).is('body') ? $(window) : $(element)
-      , href
-    this.options = $.extend({}, $.fn.scrollspy.defaults, options)
-    this.$scrollElement = $element.on('scroll.scroll-spy.data-api', process)
-    this.selector = (this.options.target
-      || ((href = $(element).attr('href')) && href.replace(/.*(?=#[^\s]+$)/, '')) //strip for ie7
-      || '') + ' .nav li > a'
-    this.$body = $('body')
-    this.refresh()
-    this.process()
-  }
-
-  ScrollSpy.prototype = {
-
-      constructor: ScrollSpy
-
-    , refresh: function () {
-        var self = this
-          , $targets
-
-        this.offsets = $([])
-        this.targets = $([])
-
-        $targets = this.$body
-          .find(this.selector)
-          .map(function () {
-            var $el = $(this)
-              , href = $el.data('target') || $el.attr('href')
-              , $href = /^#\w/.test(href) && $(href)
-            return ( $href
-              && $href.length
-              && [[ $href.position().top, href ]] ) || null
-          })
-          .sort(function (a, b) { return a[0] - b[0] })
-          .each(function () {
-            self.offsets.push(this[0])
-            self.targets.push(this[1])
-          })
-      }
-
-    , process: function () {
-        var scrollTop = this.$scrollElement.scrollTop() + this.options.offset
-          , scrollHeight = this.$scrollElement[0].scrollHeight || this.$body[0].scrollHeight
-          , maxScroll = scrollHeight - this.$scrollElement.height()
-          , offsets = this.offsets
-          , targets = this.targets
-          , activeTarget = this.activeTarget
-          , i
-
-        if (scrollTop >= maxScroll) {
-          return activeTarget != (i = targets.last()[0])
-            && this.activate ( i )
-        }
-
-        for (i = offsets.length; i--;) {
-          activeTarget != targets[i]
-            && scrollTop >= offsets[i]
-            && (!offsets[i + 1] || scrollTop <= offsets[i + 1])
-            && this.activate( targets[i] )
-        }
-      }
-
-    , activate: function (target) {
-        var active
-          , selector
-
-        this.activeTarget = target
-
-        $(this.selector)
-          .parent('.active')
-          .removeClass('active')
-
-        selector = this.selector
-          + '[data-target="' + target + '"],'
-          + this.selector + '[href="' + target + '"]'
-
-        active = $(selector)
-          .parent('li')
-          .addClass('active')
-
-        if (active.parent('.dropdown-menu').length)  {
-          active = active.closest('li.dropdown').addClass('active')
-        }
-
-        active.trigger('activate')
-      }
-
-  }
-
-
- /* SCROLLSPY PLUGIN DEFINITION
-  * =========================== */
-
-  $.fn.scrollspy = function (option) {
-    return this.each(function () {
-      var $this = $(this)
-        , data = $this.data('scrollspy')
-        , options = typeof option == 'object' && option
-      if (!data) $this.data('scrollspy', (data = new ScrollSpy(this, options)))
-      if (typeof option == 'string') data[option]()
-    })
-  }
-
-  $.fn.scrollspy.Constructor = ScrollSpy
-
-  $.fn.scrollspy.defaults = {
-    offset: 10
-  }
-
-
- /* SCROLLSPY DATA-API
-  * ================== */
-
-  $(window).on('load', function () {
-    $('[data-spy="scroll"]').each(function () {
-      var $spy = $(this)
-      $spy.scrollspy($spy.data())
-    })
-  })
-
-}(window.jQuery);/* ========================================================
- * bootstrap-tab.js v2.1.0
- * http://twitter.github.com/bootstrap/javascript.html#tabs
- * ========================================================
- * Copyright 2012 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ======================================================== */
-
-
-!function ($) {
-
-  "use strict"; // jshint ;_;
-
-
- /* TAB CLASS DEFINITION
-  * ==================== */
-
-  var Tab = function (element) {
-    this.element = $(element)
-  }
-
-  Tab.prototype = {
-
-    constructor: Tab
-
-  , show: function () {
-      var $this = this.element
-        , $ul = $this.closest('ul:not(.dropdown-menu)')
-        , selector = $this.attr('data-target')
-        , previous
-        , $target
-        , e
-
-      if (!selector) {
-        selector = $this.attr('href')
-        selector = selector && selector.replace(/.*(?=#[^\s]*$)/, '') //strip for ie7
-      }
-
-      if ( $this.parent('li').hasClass('active') ) return
-
-      previous = $ul.find('.active a').last()[0]
-
-      e = $.Event('show', {
-        relatedTarget: previous
-      })
-
-      $this.trigger(e)
-
-      if (e.isDefaultPrevented()) return
-
-      $target = $(selector)
-
-      this.activate($this.parent('li'), $ul)
-      this.activate($target, $target.parent(), function () {
-        $this.trigger({
-          type: 'shown'
-        , relatedTarget: previous
-        })
-      })
-    }
-
-  , activate: function ( element, container, callback) {
-      var $active = container.find('> .active')
-        , transition = callback
-            && $.support.transition
-            && $active.hasClass('fade')
-
-      function next() {
-        $active
-          .removeClass('active')
-          .find('> .dropdown-menu > .active')
-          .removeClass('active')
-
-        element.addClass('active')
-
-        if (transition) {
-          element[0].offsetWidth // reflow for transition
-          element.addClass('in')
-        } else {
-          element.removeClass('fade')
-        }
-
-        if ( element.parent('.dropdown-menu') ) {
-          element.closest('li.dropdown').addClass('active')
-        }
-
-        callback && callback()
-      }
-
-      transition ?
-        $active.one($.support.transition.end, next) :
-        next()
-
-      $active.removeClass('in')
-    }
-  }
-
-
- /* TAB PLUGIN DEFINITION
-  * ===================== */
-
-  $.fn.tab = function ( option ) {
-    return this.each(function () {
-      var $this = $(this)
-        , data = $this.data('tab')
-      if (!data) $this.data('tab', (data = new Tab(this)))
-      if (typeof option == 'string') data[option]()
-    })
-  }
-
-  $.fn.tab.Constructor = Tab
-
-
- /* TAB DATA-API
-  * ============ */
-
-  $(function () {
-    $('body').on('click.tab.data-api', '[data-toggle="tab"], [data-toggle="pill"]', function (e) {
-      e.preventDefault()
-      $(this).tab('show')
-    })
-  })
-
-}(window.jQuery);/* =============================================================
- * bootstrap-typeahead.js v2.1.0
- * http://twitter.github.com/bootstrap/javascript.html#typeahead
- * =============================================================
- * Copyright 2012 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ============================================================ */
-
-
-!function($){
-
-  "use strict"; // jshint ;_;
-
-
- /* TYPEAHEAD PUBLIC CLASS DEFINITION
-  * ================================= */
-
-  var Typeahead = function (element, options) {
-    this.$element = $(element)
-    this.options = $.extend({}, $.fn.typeahead.defaults, options)
-    this.matcher = this.options.matcher || this.matcher
-    this.sorter = this.options.sorter || this.sorter
-    this.highlighter = this.options.highlighter || this.highlighter
-    this.updater = this.options.updater || this.updater
-    this.$menu = $(this.options.menu).appendTo('body')
-    this.source = this.options.source
-    this.shown = false
-    this.listen()
-  }
-
-  Typeahead.prototype = {
-
-    constructor: Typeahead
-
-  , select: function () {
-      var val = this.$menu.find('.active').attr('data-value')
-      this.$element
-        .val(this.updater(val))
-        .change()
-      return this.hide()
-    }
-
-  , updater: function (item) {
-      return item
-    }
-
-  , show: function () {
-      var pos = $.extend({}, this.$element.offset(), {
-        height: this.$element[0].offsetHeight
-      })
-
-      this.$menu.css({
-        top: pos.top + pos.height
-      , left: pos.left
-      })
-
-      this.$menu.show()
-      this.shown = true
-      return this
-    }
-
-  , hide: function () {
-      this.$menu.hide()
-      this.shown = false
-      return this
-    }
-
-  , lookup: function (event) {
-      var items
-
-      this.query = this.$element.val()
-
-      if (!this.query || this.query.length < this.options.minLength) {
-        return this.shown ? this.hide() : this
-      }
-
-      items = $.isFunction(this.source) ? this.source(this.query, $.proxy(this.process, this)) : this.source
-
-      return items ? this.process(items) : this
-    }
-
-  , process: function (items) {
-      var that = this
-
-      items = $.grep(items, function (item) {
-        return that.matcher(item)
-      })
-
-      items = this.sorter(items)
-
-      if (!items.length) {
-        return this.shown ? this.hide() : this
-      }
-
-      return this.render(items.slice(0, this.options.items)).show()
-    }
-
-  , matcher: function (item) {
-      return ~item.toLowerCase().indexOf(this.query.toLowerCase())
-    }
-
-  , sorter: function (items) {
-      var beginswith = []
-        , caseSensitive = []
-        , caseInsensitive = []
-        , item
-
-      while (item = items.shift()) {
-        if (!item.toLowerCase().indexOf(this.query.toLowerCase())) beginswith.push(item)
-        else if (~item.indexOf(this.query)) caseSensitive.push(item)
-        else caseInsensitive.push(item)
-      }
-
-      return beginswith.concat(caseSensitive, caseInsensitive)
-    }
-
-  , highlighter: function (item) {
-      var query = this.query.replace(/[\-\[\]{}()*+?.,\\\^$|#\s]/g, '\\$&')
-      return item.replace(new RegExp('(' + query + ')', 'ig'), function ($1, match) {
-        return '<strong>' + match + '</strong>'
-      })
-    }
-
-  , render: function (items) {
-      var that = this
-
-      items = $(items).map(function (i, item) {
-        i = $(that.options.item).attr('data-value', item)
-        i.find('a').html(that.highlighter(item))
-        return i[0]
-      })
-
-      items.first().addClass('active')
-      this.$menu.html(items)
-      return this
-    }
-
-  , next: function (event) {
-      var active = this.$menu.find('.active').removeClass('active')
-        , next = active.next()
-
-      if (!next.length) {
-        next = $(this.$menu.find('li')[0])
-      }
-
-      next.addClass('active')
-    }
-
-  , prev: function (event) {
-      var active = this.$menu.find('.active').removeClass('active')
-        , prev = active.prev()
-
-      if (!prev.length) {
-        prev = this.$menu.find('li').last()
-      }
-
-      prev.addClass('active')
-    }
-
-  , listen: function () {
-      this.$element
-        .on('blur',     $.proxy(this.blur, this))
-        .on('keypress', $.proxy(this.keypress, this))
-        .on('keyup',    $.proxy(this.keyup, this))
-
-      if ($.browser.webkit || $.browser.msie) {
-        this.$element.on('keydown', $.proxy(this.keydown, this))
-      }
-
-      this.$menu
-        .on('click', $.proxy(this.click, this))
-        .on('mouseenter', 'li', $.proxy(this.mouseenter, this))
-    }
-
-  , move: function (e) {
-      if (!this.shown) return
-
-      switch(e.keyCode) {
-        case 9: // tab
-        case 13: // enter
-        case 27: // escape
-          e.preventDefault()
-          break
-
-        case 38: // up arrow
-          e.preventDefault()
-          this.prev()
-          break
-
-        case 40: // down arrow
-          e.preventDefault()
-          this.next()
-          break
-      }
-
-      e.stopPropagation()
-    }
-
-  , keydown: function (e) {
-      this.suppressKeyPressRepeat = !~$.inArray(e.keyCode, [40,38,9,13,27])
-      this.move(e)
-    }
-
-  , keypress: function (e) {
-      if (this.suppressKeyPressRepeat) return
-      this.move(e)
-    }
-
-  , keyup: function (e) {
-      switch(e.keyCode) {
-        case 40: // down arrow
-        case 38: // up arrow
-          break
-
-        case 9: // tab
-        case 13: // enter
-          if (!this.shown) return
-          this.select()
-          break
-
-        case 27: // escape
-          if (!this.shown) return
-          this.hide()
-          break
-
-        default:
-          this.lookup()
-      }
-
-      e.stopPropagation()
-      e.preventDefault()
-  }
-
-  , blur: function (e) {
-      var that = this
-      setTimeout(function () { that.hide() }, 150)
-    }
-
-  , click: function (e) {
-      e.stopPropagation()
-      e.preventDefault()
-      this.select()
-    }
-
-  , mouseenter: function (e) {
-      this.$menu.find('.active').removeClass('active')
-      $(e.currentTarget).addClass('active')
-    }
-
-  }
-
-
-  /* TYPEAHEAD PLUGIN DEFINITION
-   * =========================== */
-
-  $.fn.typeahead = function (option) {
-    return this.each(function () {
-      var $this = $(this)
-        , data = $this.data('typeahead')
-        , options = typeof option == 'object' && option
-      if (!data) $this.data('typeahead', (data = new Typeahead(this, options)))
-      if (typeof option == 'string') data[option]()
-    })
-  }
-
-  $.fn.typeahead.defaults = {
-    source: []
-  , items: 8
-  , menu: '<ul class="typeahead dropdown-menu"></ul>'
-  , item: '<li><a href="#"></a></li>'
-  , minLength: 1
-  }
-
-  $.fn.typeahead.Constructor = Typeahead
-
-
- /*   TYPEAHEAD DATA-API
-  * ================== */
-
-  $(function () {
-    $('body').on('focus.typeahead.data-api', '[data-provide="typeahead"]', function (e) {
-      var $this = $(this)
-      if ($this.data('typeahead')) return
-      e.preventDefault()
-      $this.typeahead($this.data())
-    })
-  })
-
-}(window.jQuery);
-/* ==========================================================
- * bootstrap-affix.js v2.1.0
- * http://twitter.github.com/bootstrap/javascript.html#affix
- * ==========================================================
- * Copyright 2012 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ========================================================== */
-
-
-!function ($) {
-
-  "use strict"; // jshint ;_;
-
-
- /* AFFIX CLASS DEFINITION
-  * ====================== */
-
-  var Affix = function (element, options) {
-    this.options = $.extend({}, $.fn.affix.defaults, options)
-    this.$window = $(window).on('scroll.affix.data-api', $.proxy(this.checkPosition, this))
-    this.$element = $(element)
-    this.checkPosition()
-  }
-
-  Affix.prototype.checkPosition = function () {
-    if (!this.$element.is(':visible')) return
-
-    var scrollHeight = $(document).height()
-      , scrollTop = this.$window.scrollTop()
-      , position = this.$element.offset()
-      , offset = this.options.offset
-      , offsetBottom = offset.bottom
-      , offsetTop = offset.top
-      , reset = 'affix affix-top affix-bottom'
-      , affix
-
-    if (typeof offset != 'object') offsetBottom = offsetTop = offset
-    if (typeof offsetTop == 'function') offsetTop = offset.top()
-    if (typeof offsetBottom == 'function') offsetBottom = offset.bottom()
-
-    affix = this.unpin != null && (scrollTop + this.unpin <= position.top) ?
-      false    : offsetBottom != null && (position.top + this.$element.height() >= scrollHeight - offsetBottom) ?
-      'bottom' : offsetTop != null && scrollTop <= offsetTop ?
-      'top'    : false
-
-    if (this.affixed === affix) return
-
-    this.affixed = affix
-    this.unpin = affix == 'bottom' ? position.top - scrollTop : null
-
-    this.$element.removeClass(reset).addClass('affix' + (affix ? '-' + affix : ''))
-  }
-
-
- /* AFFIX PLUGIN DEFINITION
-  * ======================= */
-
-  $.fn.affix = function (option) {
-    return this.each(function () {
-      var $this = $(this)
-        , data = $this.data('affix')
-        , options = typeof option == 'object' && option
-      if (!data) $this.data('affix', (data = new Affix(this, options)))
-      if (typeof option == 'string') data[option]()
-    })
-  }
-
-  $.fn.affix.Constructor = Affix
-
-  $.fn.affix.defaults = {
-    offset: 0
-  }
-
-
- /* AFFIX DATA-API
-  * ============== */
-
-  $(window).on('load', function () {
-    $('[data-spy="affix"]').each(function () {
-      var $spy = $(this)
-        , data = $spy.data()
-
-      data.offset = data.offset || {}
-
-      data.offsetBottom && (data.offset.bottom = data.offsetBottom)
-      data.offsetTop && (data.offset.top = data.offsetTop)
-
-      $spy.affix(data)
-    })
-  })
-
-
-}(window.jQuery);
\ No newline at end of file
diff --git a/docs/js/vendor/bootstrap.min.js b/docs/js/vendor/bootstrap.min.js
deleted file mode 100755
index 66e887bb3fde7..0000000000000
--- a/docs/js/vendor/bootstrap.min.js
+++ /dev/null
@@ -1,6 +0,0 @@
-/*!
-* Bootstrap.js by @fat & @mdo
-* Copyright 2012 Twitter, Inc.
-* http://www.apache.org/licenses/LICENSE-2.0.txt
-*/
-!function(e){e(function(){"use strict";e.support.transition=function(){var e=function(){var e=document.createElement("bootstrap"),t={WebkitTransition:"webkitTransitionEnd",MozTransition:"transitionend",OTransition:"oTransitionEnd otransitionend",transition:"transitionend"},n;for(n in t)if(e.style[n]!==undefined)return t[n]}();return e&&{end:e}}()})}(window.jQuery),!function(e){"use strict";var t='[data-dismiss="alert"]',n=function(n){e(n).on("click",t,this.close)};n.prototype.close=function(t){function s(){i.trigger("closed").remove()}var n=e(this),r=n.attr("data-target"),i;r||(r=n.attr("href"),r=r&&r.replace(/.*(?=#[^\s]*$)/,"")),i=e(r),t&&t.preventDefault(),i.length||(i=n.hasClass("alert")?n:n.parent()),i.trigger(t=e.Event("close"));if(t.isDefaultPrevented())return;i.removeClass("in"),e.support.transition&&i.hasClass("fade")?i.on(e.support.transition.end,s):s()},e.fn.alert=function(t){return this.each(function(){var r=e(this),i=r.data("alert");i||r.data("alert",i=new n(this)),typeof t=="string"&&i[t].call(r)})},e.fn.alert.Constructor=n,e(function(){e("body").on("click.alert.data-api",t,n.prototype.close)})}(window.jQuery),!function(e){"use strict";var t=function(t,n){this.$element=e(t),this.options=e.extend({},e.fn.button.defaults,n)};t.prototype.setState=function(e){var t="disabled",n=this.$element,r=n.data(),i=n.is("input")?"val":"html";e+="Text",r.resetText||n.data("resetText",n[i]()),n[i](r[e]||this.options[e]),setTimeout(function(){e=="loadingText"?n.addClass(t).attr(t,t):n.removeClass(t).removeAttr(t)},0)},t.prototype.toggle=function(){var e=this.$element.parent('[data-toggle="buttons-radio"]');e&&e.find(".active").removeClass("active"),this.$element.toggleClass("active")},e.fn.button=function(n){return this.each(function(){var r=e(this),i=r.data("button"),s=typeof n=="object"&&n;i||r.data("button",i=new t(this,s)),n=="toggle"?i.toggle():n&&i.setState(n)})},e.fn.button.defaults={loadingText:"loading..."},e.fn.button.Constructor=t,e(function(){e("body").on("click.button.data-api","[data-toggle^=button]",function(t){var n=e(t.target);n.hasClass("btn")||(n=n.closest(".btn")),n.button("toggle")})})}(window.jQuery),!function(e){"use strict";var t=function(t,n){this.$element=e(t),this.options=n,this.options.slide&&this.slide(this.options.slide),this.options.pause=="hover"&&this.$element.on("mouseenter",e.proxy(this.pause,this)).on("mouseleave",e.proxy(this.cycle,this))};t.prototype={cycle:function(t){return t||(this.paused=!1),this.options.interval&&!this.paused&&(this.interval=setInterval(e.proxy(this.next,this),this.options.interval)),this},to:function(t){var n=this.$element.find(".item.active"),r=n.parent().children(),i=r.index(n),s=this;if(t>r.length-1||t<0)return;return this.sliding?this.$element.one("slid",function(){s.to(t)}):i==t?this.pause().cycle():this.slide(t>i?"next":"prev",e(r[t]))},pause:function(t){return t||(this.paused=!0),this.$element.find(".next, .prev").length&&e.support.transition.end&&(this.$element.trigger(e.support.transition.end),this.cycle()),clearInterval(this.interval),this.interval=null,this},next:function(){if(this.sliding)return;return this.slide("next")},prev:function(){if(this.sliding)return;return this.slide("prev")},slide:function(t,n){var r=this.$element.find(".item.active"),i=n||r[t](),s=this.interval,o=t=="next"?"left":"right",u=t=="next"?"first":"last",a=this,f=e.Event("slide",{relatedTarget:i[0]});this.sliding=!0,s&&this.pause(),i=i.length?i:this.$element.find(".item")[u]();if(i.hasClass("active"))return;if(e.support.transition&&this.$element.hasClass("slide")){this.$element.trigger(f);if(f.isDefaultPrevented())return;i.addClass(t),i[0].offsetWidth,r.addClass(o),i.addClass(o),this.$element.one(e.support.transition.end,function(){i.removeClass([t,o].join(" ")).addClass("active"),r.removeClass(["active",o].join(" ")),a.sliding=!1,setTimeout(function(){a.$element.trigger("slid")},0)})}else{this.$element.trigger(f);if(f.isDefaultPrevented())return;r.removeClass("active"),i.addClass("active"),this.sliding=!1,this.$element.trigger("slid")}return s&&this.cycle(),this}},e.fn.carousel=function(n){return this.each(function(){var r=e(this),i=r.data("carousel"),s=e.extend({},e.fn.carousel.defaults,typeof n=="object"&&n),o=typeof n=="string"?n:s.slide;i||r.data("carousel",i=new t(this,s)),typeof n=="number"?i.to(n):o?i[o]():s.interval&&i.cycle()})},e.fn.carousel.defaults={interval:5e3,pause:"hover"},e.fn.carousel.Constructor=t,e(function(){e("body").on("click.carousel.data-api","[data-slide]",function(t){var n=e(this),r,i=e(n.attr("data-target")||(r=n.attr("href"))&&r.replace(/.*(?=#[^\s]+$)/,"")),s=!i.data("modal")&&e.extend({},i.data(),n.data());i.carousel(s),t.preventDefault()})})}(window.jQuery),!function(e){"use strict";var t=function(t,n){this.$element=e(t),this.options=e.extend({},e.fn.collapse.defaults,n),this.options.parent&&(this.$parent=e(this.options.parent)),this.options.toggle&&this.toggle()};t.prototype={constructor:t,dimension:function(){var e=this.$element.hasClass("width");return e?"width":"height"},show:function(){var t,n,r,i;if(this.transitioning)return;t=this.dimension(),n=e.camelCase(["scroll",t].join("-")),r=this.$parent&&this.$parent.find("> .accordion-group > .in");if(r&&r.length){i=r.data("collapse");if(i&&i.transitioning)return;r.collapse("hide"),i||r.data("collapse",null)}this.$element[t](0),this.transition("addClass",e.Event("show"),"shown"),e.support.transition&&this.$element[t](this.$element[0][n])},hide:function(){var t;if(this.transitioning)return;t=this.dimension(),this.reset(this.$element[t]()),this.transition("removeClass",e.Event("hide"),"hidden"),this.$element[t](0)},reset:function(e){var t=this.dimension();return this.$element.removeClass("collapse")[t](e||"auto")[0].offsetWidth,this.$element[e!==null?"addClass":"removeClass"]("collapse"),this},transition:function(t,n,r){var i=this,s=function(){n.type=="show"&&i.reset(),i.transitioning=0,i.$element.trigger(r)};this.$element.trigger(n);if(n.isDefaultPrevented())return;this.transitioning=1,this.$element[t]("in"),e.support.transition&&this.$element.hasClass("collapse")?this.$element.one(e.support.transition.end,s):s()},toggle:function(){this[this.$element.hasClass("in")?"hide":"show"]()}},e.fn.collapse=function(n){return this.each(function(){var r=e(this),i=r.data("collapse"),s=typeof n=="object"&&n;i||r.data("collapse",i=new t(this,s)),typeof n=="string"&&i[n]()})},e.fn.collapse.defaults={toggle:!0},e.fn.collapse.Constructor=t,e(function(){e("body").on("click.collapse.data-api","[data-toggle=collapse]",function(t){var n=e(this),r,i=n.attr("data-target")||t.preventDefault()||(r=n.attr("href"))&&r.replace(/.*(?=#[^\s]+$)/,""),s=e(i).data("collapse")?"toggle":n.data();n[e(i).hasClass("in")?"addClass":"removeClass"]("collapsed"),e(i).collapse(s)})})}(window.jQuery),!function(e){"use strict";function r(){i(e(t)).removeClass("open")}function i(t){var n=t.attr("data-target"),r;return n||(n=t.attr("href"),n=n&&n.replace(/.*(?=#[^\s]*$)/,"")),r=e(n),r.length||(r=t.parent()),r}var t="[data-toggle=dropdown]",n=function(t){var n=e(t).on("click.dropdown.data-api",this.toggle);e("html").on("click.dropdown.data-api",function(){n.parent().removeClass("open")})};n.prototype={constructor:n,toggle:function(t){var n=e(this),s,o;if(n.is(".disabled, :disabled"))return;return s=i(n),o=s.hasClass("open"),r(),o||(s.toggleClass("open"),n.focus()),!1},keydown:function(t){var n,r,s,o,u,a;if(!/(38|40|27)/.test(t.keyCode))return;n=e(this),t.preventDefault(),t.stopPropagation();if(n.is(".disabled, :disabled"))return;o=i(n),u=o.hasClass("open");if(!u||u&&t.keyCode==27)return n.click();r=e("[role=menu] li:not(.divider) a",o);if(!r.length)return;a=r.index(r.filter(":focus")),t.keyCode==38&&a>0&&a--,t.keyCode==40&&a<r.length-1&&a++,~a||(a=0),r.eq(a).focus()}},e.fn.dropdown=function(t){return this.each(function(){var r=e(this),i=r.data("dropdown");i||r.data("dropdown",i=new n(this)),typeof t=="string"&&i[t].call(r)})},e.fn.dropdown.Constructor=n,e(function(){e("html").on("click.dropdown.data-api touchstart.dropdown.data-api",r),e("body").on("click.dropdown touchstart.dropdown.data-api",".dropdown",function(e){e.stopPropagation()}).on("click.dropdown.data-api touchstart.dropdown.data-api",t,n.prototype.toggle).on("keydown.dropdown.data-api touchstart.dropdown.data-api",t+", [role=menu]",n.prototype.keydown)})}(window.jQuery),!function(e){"use strict";var t=function(t,n){this.options=n,this.$element=e(t).delegate('[data-dismiss="modal"]',"click.dismiss.modal",e.proxy(this.hide,this)),this.options.remote&&this.$element.find(".modal-body").load(this.options.remote)};t.prototype={constructor:t,toggle:function(){return this[this.isShown?"hide":"show"]()},show:function(){var t=this,n=e.Event("show");this.$element.trigger(n);if(this.isShown||n.isDefaultPrevented())return;e("body").addClass("modal-open"),this.isShown=!0,this.escape(),this.backdrop(function(){var n=e.support.transition&&t.$element.hasClass("fade");t.$element.parent().length||t.$element.appendTo(document.body),t.$element.show(),n&&t.$element[0].offsetWidth,t.$element.addClass("in").attr("aria-hidden",!1).focus(),t.enforceFocus(),n?t.$element.one(e.support.transition.end,function(){t.$element.trigger("shown")}):t.$element.trigger("shown")})},hide:function(t){t&&t.preventDefault();var n=this;t=e.Event("hide"),this.$element.trigger(t);if(!this.isShown||t.isDefaultPrevented())return;this.isShown=!1,e("body").removeClass("modal-open"),this.escape(),e(document).off("focusin.modal"),this.$element.removeClass("in").attr("aria-hidden",!0),e.support.transition&&this.$element.hasClass("fade")?this.hideWithTransition():this.hideModal()},enforceFocus:function(){var t=this;e(document).on("focusin.modal",function(e){t.$element[0]!==e.target&&!t.$element.has(e.target).length&&t.$element.focus()})},escape:function(){var e=this;this.isShown&&this.options.keyboard?this.$element.on("keyup.dismiss.modal",function(t){t.which==27&&e.hide()}):this.isShown||this.$element.off("keyup.dismiss.modal")},hideWithTransition:function(){var t=this,n=setTimeout(function(){t.$element.off(e.support.transition.end),t.hideModal()},500);this.$element.one(e.support.transition.end,function(){clearTimeout(n),t.hideModal()})},hideModal:function(e){this.$element.hide().trigger("hidden"),this.backdrop()},removeBackdrop:function(){this.$backdrop.remove(),this.$backdrop=null},backdrop:function(t){var n=this,r=this.$element.hasClass("fade")?"fade":"";if(this.isShown&&this.options.backdrop){var i=e.support.transition&&r;this.$backdrop=e('<div class="modal-backdrop '+r+'" />').appendTo(document.body),this.options.backdrop!="static"&&this.$backdrop.click(e.proxy(this.hide,this)),i&&this.$backdrop[0].offsetWidth,this.$backdrop.addClass("in"),i?this.$backdrop.one(e.support.transition.end,t):t()}else!this.isShown&&this.$backdrop?(this.$backdrop.removeClass("in"),e.support.transition&&this.$element.hasClass("fade")?this.$backdrop.one(e.support.transition.end,e.proxy(this.removeBackdrop,this)):this.removeBackdrop()):t&&t()}},e.fn.modal=function(n){return this.each(function(){var r=e(this),i=r.data("modal"),s=e.extend({},e.fn.modal.defaults,r.data(),typeof n=="object"&&n);i||r.data("modal",i=new t(this,s)),typeof n=="string"?i[n]():s.show&&i.show()})},e.fn.modal.defaults={backdrop:!0,keyboard:!0,show:!0},e.fn.modal.Constructor=t,e(function(){e("body").on("click.modal.data-api",'[data-toggle="modal"]',function(t){var n=e(this),r=n.attr("href"),i=e(n.attr("data-target")||r&&r.replace(/.*(?=#[^\s]+$)/,"")),s=i.data("modal")?"toggle":e.extend({remote:!/#/.test(r)&&r},i.data(),n.data());t.preventDefault(),i.modal(s).one("hide",function(){n.focus()})})})}(window.jQuery),!function(e){"use strict";var t=function(e,t){this.init("tooltip",e,t)};t.prototype={constructor:t,init:function(t,n,r){var i,s;this.type=t,this.$element=e(n),this.options=this.getOptions(r),this.enabled=!0,this.options.trigger=="click"?this.$element.on("click."+this.type,this.options.selector,e.proxy(this.toggle,this)):this.options.trigger!="manual"&&(i=this.options.trigger=="hover"?"mouseenter":"focus",s=this.options.trigger=="hover"?"mouseleave":"blur",this.$element.on(i+"."+this.type,this.options.selector,e.proxy(this.enter,this)),this.$element.on(s+"."+this.type,this.options.selector,e.proxy(this.leave,this))),this.options.selector?this._options=e.extend({},this.options,{trigger:"manual",selector:""}):this.fixTitle()},getOptions:function(t){return t=e.extend({},e.fn[this.type].defaults,t,this.$element.data()),t.delay&&typeof t.delay=="number"&&(t.delay={show:t.delay,hide:t.delay}),t},enter:function(t){var n=e(t.currentTarget)[this.type](this._options).data(this.type);if(!n.options.delay||!n.options.delay.show)return n.show();clearTimeout(this.timeout),n.hoverState="in",this.timeout=setTimeout(function(){n.hoverState=="in"&&n.show()},n.options.delay.show)},leave:function(t){var n=e(t.currentTarget)[this.type](this._options).data(this.type);this.timeout&&clearTimeout(this.timeout);if(!n.options.delay||!n.options.delay.hide)return n.hide();n.hoverState="out",this.timeout=setTimeout(function(){n.hoverState=="out"&&n.hide()},n.options.delay.hide)},show:function(){var e,t,n,r,i,s,o;if(this.hasContent()&&this.enabled){e=this.tip(),this.setContent(),this.options.animation&&e.addClass("fade"),s=typeof this.options.placement=="function"?this.options.placement.call(this,e[0],this.$element[0]):this.options.placement,t=/in/.test(s),e.remove().css({top:0,left:0,display:"block"}).appendTo(t?this.$element:document.body),n=this.getPosition(t),r=e[0].offsetWidth,i=e[0].offsetHeight;switch(t?s.split(" ")[1]:s){case"bottom":o={top:n.top+n.height,left:n.left+n.width/2-r/2};break;case"top":o={top:n.top-i,left:n.left+n.width/2-r/2};break;case"left":o={top:n.top+n.height/2-i/2,left:n.left-r};break;case"right":o={top:n.top+n.height/2-i/2,left:n.left+n.width}}e.css(o).addClass(s).addClass("in")}},setContent:function(){var e=this.tip(),t=this.getTitle();e.find(".tooltip-inner")[this.options.html?"html":"text"](t),e.removeClass("fade in top bottom left right")},hide:function(){function r(){var t=setTimeout(function(){n.off(e.support.transition.end).remove()},500);n.one(e.support.transition.end,function(){clearTimeout(t),n.remove()})}var t=this,n=this.tip();return n.removeClass("in"),e.support.transition&&this.$tip.hasClass("fade")?r():n.remove(),this},fixTitle:function(){var e=this.$element;(e.attr("title")||typeof e.attr("data-original-title")!="string")&&e.attr("data-original-title",e.attr("title")||"").removeAttr("title")},hasContent:function(){return this.getTitle()},getPosition:function(t){return e.extend({},t?{top:0,left:0}:this.$element.offset(),{width:this.$element[0].offsetWidth,height:this.$element[0].offsetHeight})},getTitle:function(){var e,t=this.$element,n=this.options;return e=t.attr("data-original-title")||(typeof n.title=="function"?n.title.call(t[0]):n.title),e},tip:function(){return this.$tip=this.$tip||e(this.options.template)},validate:function(){this.$element[0].parentNode||(this.hide(),this.$element=null,this.options=null)},enable:function(){this.enabled=!0},disable:function(){this.enabled=!1},toggleEnabled:function(){this.enabled=!this.enabled},toggle:function(){this[this.tip().hasClass("in")?"hide":"show"]()},destroy:function(){this.hide().$element.off("."+this.type).removeData(this.type)}},e.fn.tooltip=function(n){return this.each(function(){var r=e(this),i=r.data("tooltip"),s=typeof n=="object"&&n;i||r.data("tooltip",i=new t(this,s)),typeof n=="string"&&i[n]()})},e.fn.tooltip.Constructor=t,e.fn.tooltip.defaults={animation:!0,placement:"top",selector:!1,template:'<div class="tooltip"><div class="tooltip-arrow"></div><div class="tooltip-inner"></div></div>',trigger:"hover",title:"",delay:0,html:!0}}(window.jQuery),!function(e){"use strict";var t=function(e,t){this.init("popover",e,t)};t.prototype=e.extend({},e.fn.tooltip.Constructor.prototype,{constructor:t,setContent:function(){var e=this.tip(),t=this.getTitle(),n=this.getContent();e.find(".popover-title")[this.options.html?"html":"text"](t),e.find(".popover-content > *")[this.options.html?"html":"text"](n),e.removeClass("fade top bottom left right in")},hasContent:function(){return this.getTitle()||this.getContent()},getContent:function(){var e,t=this.$element,n=this.options;return e=t.attr("data-content")||(typeof n.content=="function"?n.content.call(t[0]):n.content),e},tip:function(){return this.$tip||(this.$tip=e(this.options.template)),this.$tip},destroy:function(){this.hide().$element.off("."+this.type).removeData(this.type)}}),e.fn.popover=function(n){return this.each(function(){var r=e(this),i=r.data("popover"),s=typeof n=="object"&&n;i||r.data("popover",i=new t(this,s)),typeof n=="string"&&i[n]()})},e.fn.popover.Constructor=t,e.fn.popover.defaults=e.extend({},e.fn.tooltip.defaults,{placement:"right",trigger:"click",content:"",template:'<div class="popover"><div class="arrow"></div><div class="popover-inner"><h3 class="popover-title"></h3><div class="popover-content"><p></p></div></div></div>'})}(window.jQuery),!function(e){"use strict";function t(t,n){var r=e.proxy(this.process,this),i=e(t).is("body")?e(window):e(t),s;this.options=e.extend({},e.fn.scrollspy.defaults,n),this.$scrollElement=i.on("scroll.scroll-spy.data-api",r),this.selector=(this.options.target||(s=e(t).attr("href"))&&s.replace(/.*(?=#[^\s]+$)/,"")||"")+" .nav li > a",this.$body=e("body"),this.refresh(),this.process()}t.prototype={constructor:t,refresh:function(){var t=this,n;this.offsets=e([]),this.targets=e([]),n=this.$body.find(this.selector).map(function(){var t=e(this),n=t.data("target")||t.attr("href"),r=/^#\w/.test(n)&&e(n);return r&&r.length&&[[r.position().top,n]]||null}).sort(function(e,t){return e[0]-t[0]}).each(function(){t.offsets.push(this[0]),t.targets.push(this[1])})},process:function(){var e=this.$scrollElement.scrollTop()+this.options.offset,t=this.$scrollElement[0].scrollHeight||this.$body[0].scrollHeight,n=t-this.$scrollElement.height(),r=this.offsets,i=this.targets,s=this.activeTarget,o;if(e>=n)return s!=(o=i.last()[0])&&this.activate(o);for(o=r.length;o--;)s!=i[o]&&e>=r[o]&&(!r[o+1]||e<=r[o+1])&&this.activate(i[o])},activate:function(t){var n,r;this.activeTarget=t,e(this.selector).parent(".active").removeClass("active"),r=this.selector+'[data-target="'+t+'"],'+this.selector+'[href="'+t+'"]',n=e(r).parent("li").addClass("active"),n.parent(".dropdown-menu").length&&(n=n.closest("li.dropdown").addClass("active")),n.trigger("activate")}},e.fn.scrollspy=function(n){return this.each(function(){var r=e(this),i=r.data("scrollspy"),s=typeof n=="object"&&n;i||r.data("scrollspy",i=new t(this,s)),typeof n=="string"&&i[n]()})},e.fn.scrollspy.Constructor=t,e.fn.scrollspy.defaults={offset:10},e(window).on("load",function(){e('[data-spy="scroll"]').each(function(){var t=e(this);t.scrollspy(t.data())})})}(window.jQuery),!function(e){"use strict";var t=function(t){this.element=e(t)};t.prototype={constructor:t,show:function(){var t=this.element,n=t.closest("ul:not(.dropdown-menu)"),r=t.attr("data-target"),i,s,o;r||(r=t.attr("href"),r=r&&r.replace(/.*(?=#[^\s]*$)/,""));if(t.parent("li").hasClass("active"))return;i=n.find(".active a").last()[0],o=e.Event("show",{relatedTarget:i}),t.trigger(o);if(o.isDefaultPrevented())return;s=e(r),this.activate(t.parent("li"),n),this.activate(s,s.parent(),function(){t.trigger({type:"shown",relatedTarget:i})})},activate:function(t,n,r){function o(){i.removeClass("active").find("> .dropdown-menu > .active").removeClass("active"),t.addClass("active"),s?(t[0].offsetWidth,t.addClass("in")):t.removeClass("fade"),t.parent(".dropdown-menu")&&t.closest("li.dropdown").addClass("active"),r&&r()}var i=n.find("> .active"),s=r&&e.support.transition&&i.hasClass("fade");s?i.one(e.support.transition.end,o):o(),i.removeClass("in")}},e.fn.tab=function(n){return this.each(function(){var r=e(this),i=r.data("tab");i||r.data("tab",i=new t(this)),typeof n=="string"&&i[n]()})},e.fn.tab.Constructor=t,e(function(){e("body").on("click.tab.data-api",'[data-toggle="tab"], [data-toggle="pill"]',function(t){t.preventDefault(),e(this).tab("show")})})}(window.jQuery),!function(e){"use strict";var t=function(t,n){this.$element=e(t),this.options=e.extend({},e.fn.typeahead.defaults,n),this.matcher=this.options.matcher||this.matcher,this.sorter=this.options.sorter||this.sorter,this.highlighter=this.options.highlighter||this.highlighter,this.updater=this.options.updater||this.updater,this.$menu=e(this.options.menu).appendTo("body"),this.source=this.options.source,this.shown=!1,this.listen()};t.prototype={constructor:t,select:function(){var e=this.$menu.find(".active").attr("data-value");return this.$element.val(this.updater(e)).change(),this.hide()},updater:function(e){return e},show:function(){var t=e.extend({},this.$element.offset(),{height:this.$element[0].offsetHeight});return this.$menu.css({top:t.top+t.height,left:t.left}),this.$menu.show(),this.shown=!0,this},hide:function(){return this.$menu.hide(),this.shown=!1,this},lookup:function(t){var n;return this.query=this.$element.val(),!this.query||this.query.length<this.options.minLength?this.shown?this.hide():this:(n=e.isFunction(this.source)?this.source(this.query,e.proxy(this.process,this)):this.source,n?this.process(n):this)},process:function(t){var n=this;return t=e.grep(t,function(e){return n.matcher(e)}),t=this.sorter(t),t.length?this.render(t.slice(0,this.options.items)).show():this.shown?this.hide():this},matcher:function(e){return~e.toLowerCase().indexOf(this.query.toLowerCase())},sorter:function(e){var t=[],n=[],r=[],i;while(i=e.shift())i.toLowerCase().indexOf(this.query.toLowerCase())?~i.indexOf(this.query)?n.push(i):r.push(i):t.push(i);return t.concat(n,r)},highlighter:function(e){var t=this.query.replace(/[\-\[\]{}()*+?.,\\\^$|#\s]/g,"\\$&");return e.replace(new RegExp("("+t+")","ig"),function(e,t){return"<strong>"+t+"</strong>"})},render:function(t){var n=this;return t=e(t).map(function(t,r){return t=e(n.options.item).attr("data-value",r),t.find("a").html(n.highlighter(r)),t[0]}),t.first().addClass("active"),this.$menu.html(t),this},next:function(t){var n=this.$menu.find(".active").removeClass("active"),r=n.next();r.length||(r=e(this.$menu.find("li")[0])),r.addClass("active")},prev:function(e){var t=this.$menu.find(".active").removeClass("active"),n=t.prev();n.length||(n=this.$menu.find("li").last()),n.addClass("active")},listen:function(){this.$element.on("blur",e.proxy(this.blur,this)).on("keypress",e.proxy(this.keypress,this)).on("keyup",e.proxy(this.keyup,this)),(e.browser.webkit||e.browser.msie)&&this.$element.on("keydown",e.proxy(this.keydown,this)),this.$menu.on("click",e.proxy(this.click,this)).on("mouseenter","li",e.proxy(this.mouseenter,this))},move:function(e){if(!this.shown)return;switch(e.keyCode){case 9:case 13:case 27:e.preventDefault();break;case 38:e.preventDefault(),this.prev();break;case 40:e.preventDefault(),this.next()}e.stopPropagation()},keydown:function(t){this.suppressKeyPressRepeat=!~e.inArray(t.keyCode,[40,38,9,13,27]),this.move(t)},keypress:function(e){if(this.suppressKeyPressRepeat)return;this.move(e)},keyup:function(e){switch(e.keyCode){case 40:case 38:break;case 9:case 13:if(!this.shown)return;this.select();break;case 27:if(!this.shown)return;this.hide();break;default:this.lookup()}e.stopPropagation(),e.preventDefault()},blur:function(e){var t=this;setTimeout(function(){t.hide()},150)},click:function(e){e.stopPropagation(),e.preventDefault(),this.select()},mouseenter:function(t){this.$menu.find(".active").removeClass("active"),e(t.currentTarget).addClass("active")}},e.fn.typeahead=function(n){return this.each(function(){var r=e(this),i=r.data("typeahead"),s=typeof n=="object"&&n;i||r.data("typeahead",i=new t(this,s)),typeof n=="string"&&i[n]()})},e.fn.typeahead.defaults={source:[],items:8,menu:'<ul class="typeahead dropdown-menu"></ul>',item:'<li><a href="#"></a></li>',minLength:1},e.fn.typeahead.Constructor=t,e(function(){e("body").on("focus.typeahead.data-api",'[data-provide="typeahead"]',function(t){var n=e(this);if(n.data("typeahead"))return;t.preventDefault(),n.typeahead(n.data())})})}(window.jQuery),!function(e){"use strict";var t=function(t,n){this.options=e.extend({},e.fn.affix.defaults,n),this.$window=e(window).on("scroll.affix.data-api",e.proxy(this.checkPosition,this)),this.$element=e(t),this.checkPosition()};t.prototype.checkPosition=function(){if(!this.$element.is(":visible"))return;var t=e(document).height(),n=this.$window.scrollTop(),r=this.$element.offset(),i=this.options.offset,s=i.bottom,o=i.top,u="affix affix-top affix-bottom",a;typeof i!="object"&&(s=o=i),typeof o=="function"&&(o=i.top()),typeof s=="function"&&(s=i.bottom()),a=this.unpin!=null&&n+this.unpin<=r.top?!1:s!=null&&r.top+this.$element.height()>=t-s?"bottom":o!=null&&n<=o?"top":!1;if(this.affixed===a)return;this.affixed=a,this.unpin=a=="bottom"?r.top-n:null,this.$element.removeClass(u).addClass("affix"+(a?"-"+a:""))},e.fn.affix=function(n){return this.each(function(){var r=e(this),i=r.data("affix"),s=typeof n=="object"&&n;i||r.data("affix",i=new t(this,s)),typeof n=="string"&&i[n]()})},e.fn.affix.Constructor=t,e.fn.affix.defaults={offset:0},e(window).on("load",function(){e('[data-spy="affix"]').each(function(){var t=e(this),n=t.data();n.offset=n.offset||{},n.offsetBottom&&(n.offset.bottom=n.offsetBottom),n.offsetTop&&(n.offset.top=n.offsetTop),t.affix(n)})})}(window.jQuery);
\ No newline at end of file
diff --git a/docs/js/vendor/jquery-3.4.1.min.js b/docs/js/vendor/jquery-3.4.1.min.js
deleted file mode 100644
index 07c00cd227da0..0000000000000
--- a/docs/js/vendor/jquery-3.4.1.min.js
+++ /dev/null
@@ -1,2 +0,0 @@
-/*! jQuery v3.4.1 | (c) JS Foundation and other contributors | jquery.org/license */
-!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(C,e){"use strict";var t=[],E=C.document,r=Object.getPrototypeOf,s=t.slice,g=t.concat,u=t.push,i=t.indexOf,n={},o=n.toString,v=n.hasOwnProperty,a=v.toString,l=a.call(Object),y={},m=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType},x=function(e){return null!=e&&e===e.window},c={type:!0,src:!0,nonce:!0,noModule:!0};function b(e,t,n){var r,i,o=(n=n||E).createElement("script");if(o.text=e,t)for(r in c)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function w(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[o.call(e)]||"object":typeof e}var f="3.4.1",k=function(e,t){return new k.fn.init(e,t)},p=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g;function d(e){var t=!!e&&"length"in e&&e.length,n=w(e);return!m(e)&&!x(e)&&("array"===n||0===t||"number"==typeof t&&0<t&&t-1 in e)}k.fn=k.prototype={jquery:f,constructor:k,length:0,toArray:function(){return s.call(this)},get:function(e){return null==e?s.call(this):e<0?this[e+this.length]:this[e]},pushStack:function(e){var t=k.merge(this.constructor(),e);return t.prevObject=this,t},each:function(e){return k.each(this,e)},map:function(n){return this.pushStack(k.map(this,function(e,t){return n.call(e,t,e)}))},slice:function(){return this.pushStack(s.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(e){var t=this.length,n=+e+(e<0?t:0);return this.pushStack(0<=n&&n<t?[this[n]]:[])},end:function(){return this.prevObject||this.constructor()},push:u,sort:t.sort,splice:t.splice},k.extend=k.fn.extend=function(){var e,t,n,r,i,o,a=arguments[0]||{},s=1,u=arguments.length,l=!1;for("boolean"==typeof a&&(l=a,a=arguments[s]||{},s++),"object"==typeof a||m(a)||(a={}),s===u&&(a=this,s--);s<u;s++)if(null!=(e=arguments[s]))for(t in e)r=e[t],"__proto__"!==t&&a!==r&&(l&&r&&(k.isPlainObject(r)||(i=Array.isArray(r)))?(n=a[t],o=i&&!Array.isArray(n)?[]:i||k.isPlainObject(n)?n:{},i=!1,a[t]=k.extend(l,o,r)):void 0!==r&&(a[t]=r));return a},k.extend({expando:"jQuery"+(f+Math.random()).replace(/\D/g,""),isReady:!0,error:function(e){throw new Error(e)},noop:function(){},isPlainObject:function(e){var t,n;return!(!e||"[object Object]"!==o.call(e))&&(!(t=r(e))||"function"==typeof(n=v.call(t,"constructor")&&t.constructor)&&a.call(n)===l)},isEmptyObject:function(e){var t;for(t in e)return!1;return!0},globalEval:function(e,t){b(e,{nonce:t&&t.nonce})},each:function(e,t){var n,r=0;if(d(e)){for(n=e.length;r<n;r++)if(!1===t.call(e[r],r,e[r]))break}else for(r in e)if(!1===t.call(e[r],r,e[r]))break;return e},trim:function(e){return null==e?"":(e+"").replace(p,"")},makeArray:function(e,t){var n=t||[];return null!=e&&(d(Object(e))?k.merge(n,"string"==typeof e?[e]:e):u.call(n,e)),n},inArray:function(e,t,n){return null==t?-1:i.call(t,e,n)},merge:function(e,t){for(var n=+t.length,r=0,i=e.length;r<n;r++)e[i++]=t[r];return e.length=i,e},grep:function(e,t,n){for(var r=[],i=0,o=e.length,a=!n;i<o;i++)!t(e[i],i)!==a&&r.push(e[i]);return r},map:function(e,t,n){var r,i,o=0,a=[];if(d(e))for(r=e.length;o<r;o++)null!=(i=t(e[o],o,n))&&a.push(i);else for(o in e)null!=(i=t(e[o],o,n))&&a.push(i);return g.apply([],a)},guid:1,support:y}),"function"==typeof Symbol&&(k.fn[Symbol.iterator]=t[Symbol.iterator]),k.each("Boolean Number String Function Array Date RegExp Object Error Symbol".split(" "),function(e,t){n["[object "+t+"]"]=t.toLowerCase()});var h=function(n){var e,d,b,o,i,h,f,g,w,u,l,T,C,a,E,v,s,c,y,k="sizzle"+1*new Date,m=n.document,S=0,r=0,p=ue(),x=ue(),N=ue(),A=ue(),D=function(e,t){return e===t&&(l=!0),0},j={}.hasOwnProperty,t=[],q=t.pop,L=t.push,H=t.push,O=t.slice,P=function(e,t){for(var n=0,r=e.length;n<r;n++)if(e[n]===t)return n;return-1},R="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",M="[\\x20\\t\\r\\n\\f]",I="(?:\\\\.|[\\w-]|[^\0-\\xa0])+",W="\\["+M+"*("+I+")(?:"+M+"*([*^$|!~]?=)"+M+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+I+"))|)"+M+"*\\]",$=":("+I+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+W+")*)|.*)\\)|)",F=new RegExp(M+"+","g"),B=new RegExp("^"+M+"+|((?:^|[^\\\\])(?:\\\\.)*)"+M+"+$","g"),_=new RegExp("^"+M+"*,"+M+"*"),z=new RegExp("^"+M+"*([>+~]|"+M+")"+M+"*"),U=new RegExp(M+"|>"),X=new RegExp($),V=new RegExp("^"+I+"$"),G={ID:new RegExp("^#("+I+")"),CLASS:new RegExp("^\\.("+I+")"),TAG:new RegExp("^("+I+"|[*])"),ATTR:new RegExp("^"+W),PSEUDO:new RegExp("^"+$),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+R+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+M+"*((?:-\\d)?\\d*)"+M+"*\\)|)(?=[^-]|$)","i")},Y=/HTML$/i,Q=/^(?:input|select|textarea|button)$/i,J=/^h\d$/i,K=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ee=/[+~]/,te=new RegExp("\\\\([\\da-f]{1,6}"+M+"?|("+M+")|.)","ig"),ne=function(e,t,n){var r="0x"+t-65536;return r!=r||n?t:r<0?String.fromCharCode(r+65536):String.fromCharCode(r>>10|55296,1023&r|56320)},re=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ie=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},oe=function(){T()},ae=be(function(e){return!0===e.disabled&&"fieldset"===e.nodeName.toLowerCase()},{dir:"parentNode",next:"legend"});try{H.apply(t=O.call(m.childNodes),m.childNodes),t[m.childNodes.length].nodeType}catch(e){H={apply:t.length?function(e,t){L.apply(e,O.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function se(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,p=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==p&&9!==p&&11!==p)return n;if(!r&&((e?e.ownerDocument||e:m)!==C&&T(e),e=e||C,E)){if(11!==p&&(u=Z.exec(t)))if(i=u[1]){if(9===p){if(!(a=e.getElementById(i)))return n;if(a.id===i)return n.push(a),n}else if(f&&(a=f.getElementById(i))&&y(e,a)&&a.id===i)return n.push(a),n}else{if(u[2])return H.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&d.getElementsByClassName&&e.getElementsByClassName)return H.apply(n,e.getElementsByClassName(i)),n}if(d.qsa&&!A[t+" "]&&(!v||!v.test(t))&&(1!==p||"object"!==e.nodeName.toLowerCase())){if(c=t,f=e,1===p&&U.test(t)){(s=e.getAttribute("id"))?s=s.replace(re,ie):e.setAttribute("id",s=k),o=(l=h(t)).length;while(o--)l[o]="#"+s+" "+xe(l[o]);c=l.join(","),f=ee.test(t)&&ye(e.parentNode)||e}try{return H.apply(n,f.querySelectorAll(c)),n}catch(e){A(t,!0)}finally{s===k&&e.removeAttribute("id")}}}return g(t.replace(B,"$1"),e,n,r)}function ue(){var r=[];return function e(t,n){return r.push(t+" ")>b.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function le(e){return e[k]=!0,e}function ce(e){var t=C.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){var n=e.split("|"),r=n.length;while(r--)b.attrHandle[n[r]]=t}function pe(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function de(t){return function(e){return"input"===e.nodeName.toLowerCase()&&e.type===t}}function he(n){return function(e){var t=e.nodeName.toLowerCase();return("input"===t||"button"===t)&&e.type===n}}function ge(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&ae(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function ve(a){return le(function(o){return o=+o,le(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function ye(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}for(e in d=se.support={},i=se.isXML=function(e){var t=e.namespaceURI,n=(e.ownerDocument||e).documentElement;return!Y.test(t||n&&n.nodeName||"HTML")},T=se.setDocument=function(e){var t,n,r=e?e.ownerDocument||e:m;return r!==C&&9===r.nodeType&&r.documentElement&&(a=(C=r).documentElement,E=!i(C),m!==C&&(n=C.defaultView)&&n.top!==n&&(n.addEventListener?n.addEventListener("unload",oe,!1):n.attachEvent&&n.attachEvent("onunload",oe)),d.attributes=ce(function(e){return e.className="i",!e.getAttribute("className")}),d.getElementsByTagName=ce(function(e){return e.appendChild(C.createComment("")),!e.getElementsByTagName("*").length}),d.getElementsByClassName=K.test(C.getElementsByClassName),d.getById=ce(function(e){return a.appendChild(e).id=k,!C.getElementsByName||!C.getElementsByName(k).length}),d.getById?(b.filter.ID=function(e){var t=e.replace(te,ne);return function(e){return e.getAttribute("id")===t}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n=t.getElementById(e);return n?[n]:[]}}):(b.filter.ID=function(e){var n=e.replace(te,ne);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),b.find.TAG=d.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):d.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},b.find.CLASS=d.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&E)return t.getElementsByClassName(e)},s=[],v=[],(d.qsa=K.test(C.querySelectorAll))&&(ce(function(e){a.appendChild(e).innerHTML="<a id='"+k+"'></a><select id='"+k+"-\r\\' msallowcapture=''><option selected=''></option></select>",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+M+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+M+"*(?:value|"+R+")"),e.querySelectorAll("[id~="+k+"-]").length||v.push("~="),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+k+"+*").length||v.push(".#.+[+~]")}),ce(function(e){e.innerHTML="<a href='' disabled='disabled'></a><select disabled='disabled'><option/></select>";var t=C.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+M+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")})),(d.matchesSelector=K.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce(function(e){d.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",$)}),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=K.test(a.compareDocumentPosition),y=t||K.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},D=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)===(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!d.sortDetached&&t.compareDocumentPosition(e)===n?e===C||e.ownerDocument===m&&y(m,e)?-1:t===C||t.ownerDocument===m&&y(m,t)?1:u?P(u,e)-P(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e===C?-1:t===C?1:i?-1:o?1:u?P(u,e)-P(u,t):0;if(i===o)return pe(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?pe(a[r],s[r]):a[r]===m?-1:s[r]===m?1:0}),C},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if((e.ownerDocument||e)!==C&&T(e),d.matchesSelector&&E&&!A[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||d.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){A(t,!0)}return 0<se(t,C,null,[e]).length},se.contains=function(e,t){return(e.ownerDocument||e)!==C&&T(e),y(e,t)},se.attr=function(e,t){(e.ownerDocument||e)!==C&&T(e);var n=b.attrHandle[t.toLowerCase()],r=n&&j.call(b.attrHandle,t.toLowerCase())?n(e,t,!E):void 0;return void 0!==r?r:d.attributes||!E?e.getAttribute(t):(r=e.getAttributeNode(t))&&r.specified?r.value:null},se.escape=function(e){return(e+"").replace(re,ie)},se.error=function(e){throw new Error("Syntax error, unrecognized expression: "+e)},se.uniqueSort=function(e){var t,n=[],r=0,i=0;if(l=!d.detectDuplicates,u=!d.sortStable&&e.slice(0),e.sort(D),l){while(t=e[i++])t===e[i]&&(r=n.push(i));while(r--)e.splice(n[r],1)}return u=null,e},o=se.getText=function(e){var t,n="",r=0,i=e.nodeType;if(i){if(1===i||9===i||11===i){if("string"==typeof e.textContent)return e.textContent;for(e=e.firstChild;e;e=e.nextSibling)n+=o(e)}else if(3===i||4===i)return e.nodeValue}else while(t=e[r++])n+=o(t);return n},(b=se.selectors={cacheLength:50,createPseudo:le,match:G,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||"").replace(te,ne),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return G.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&X.test(n)&&(t=h(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=p[e+" "];return t||(t=new RegExp("(^|"+M+")"+e+"("+M+"|$)"))&&p(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=se.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1<t.indexOf(i):"$="===r?i&&t.slice(-i.length)===i:"~="===r?-1<(" "+t.replace(F," ")+" ").indexOf(i):"|="===r&&(t===i||t.slice(0,i.length+1)===i+"-"))}},CHILD:function(h,e,t,g,v){var y="nth"!==h.slice(0,3),m="last"!==h.slice(-4),x="of-type"===e;return 1===g&&0===v?function(e){return!!e.parentNode}:function(e,t,n){var r,i,o,a,s,u,l=y!==m?"nextSibling":"previousSibling",c=e.parentNode,f=x&&e.nodeName.toLowerCase(),p=!n&&!x,d=!1;if(c){if(y){while(l){a=e;while(a=a[l])if(x?a.nodeName.toLowerCase()===f:1===a.nodeType)return!1;u=l="only"===h&&!u&&"nextSibling"}return!0}if(u=[m?c.firstChild:c.lastChild],m&&p){d=(s=(r=(i=(o=(a=c)[k]||(a[k]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]||[])[0]===S&&r[1])&&r[2],a=s&&c.childNodes[s];while(a=++s&&a&&a[l]||(d=s=0)||u.pop())if(1===a.nodeType&&++d&&a===e){i[h]=[S,s,d];break}}else if(p&&(d=s=(r=(i=(o=(a=e)[k]||(a[k]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]||[])[0]===S&&r[1]),!1===d)while(a=++s&&a&&a[l]||(d=s=0)||u.pop())if((x?a.nodeName.toLowerCase()===f:1===a.nodeType)&&++d&&(p&&((i=(o=a[k]||(a[k]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]=[S,d]),a===e))break;return(d-=v)===g||d%g==0&&0<=d/g}}},PSEUDO:function(e,o){var t,a=b.pseudos[e]||b.setFilters[e.toLowerCase()]||se.error("unsupported pseudo: "+e);return a[k]?a(o):1<a.length?(t=[e,e,"",o],b.setFilters.hasOwnProperty(e.toLowerCase())?le(function(e,t){var n,r=a(e,o),i=r.length;while(i--)e[n=P(e,r[i])]=!(t[n]=r[i])}):function(e){return a(e,0,t)}):a}},pseudos:{not:le(function(e){var r=[],i=[],s=f(e.replace(B,"$1"));return s[k]?le(function(e,t,n,r){var i,o=s(e,null,r,[]),a=e.length;while(a--)(i=o[a])&&(e[a]=!(t[a]=i))}):function(e,t,n){return r[0]=e,s(r,null,n,i),r[0]=null,!i.pop()}}),has:le(function(t){return function(e){return 0<se(t,e).length}}),contains:le(function(t){return t=t.replace(te,ne),function(e){return-1<(e.textContent||o(e)).indexOf(t)}}),lang:le(function(n){return V.test(n||"")||se.error("unsupported lang: "+n),n=n.replace(te,ne).toLowerCase(),function(e){var t;do{if(t=E?e.lang:e.getAttribute("xml:lang")||e.getAttribute("lang"))return(t=t.toLowerCase())===n||0===t.indexOf(n+"-")}while((e=e.parentNode)&&1===e.nodeType);return!1}}),target:function(e){var t=n.location&&n.location.hash;return t&&t.slice(1)===e.id},root:function(e){return e===a},focus:function(e){return e===C.activeElement&&(!C.hasFocus||C.hasFocus())&&!!(e.type||e.href||~e.tabIndex)},enabled:ge(!1),disabled:ge(!0),checked:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&!!e.checked||"option"===t&&!!e.selected},selected:function(e){return e.parentNode&&e.parentNode.selectedIndex,!0===e.selected},empty:function(e){for(e=e.firstChild;e;e=e.nextSibling)if(e.nodeType<6)return!1;return!0},parent:function(e){return!b.pseudos.empty(e)},header:function(e){return J.test(e.nodeName)},input:function(e){return Q.test(e.nodeName)},button:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&"button"===e.type||"button"===t},text:function(e){var t;return"input"===e.nodeName.toLowerCase()&&"text"===e.type&&(null==(t=e.getAttribute("type"))||"text"===t.toLowerCase())},first:ve(function(){return[0]}),last:ve(function(e,t){return[t-1]}),eq:ve(function(e,t,n){return[n<0?n+t:n]}),even:ve(function(e,t){for(var n=0;n<t;n+=2)e.push(n);return e}),odd:ve(function(e,t){for(var n=1;n<t;n+=2)e.push(n);return e}),lt:ve(function(e,t,n){for(var r=n<0?n+t:t<n?t:n;0<=--r;)e.push(r);return e}),gt:ve(function(e,t,n){for(var r=n<0?n+t:n;++r<t;)e.push(r);return e})}}).pseudos.nth=b.pseudos.eq,{radio:!0,checkbox:!0,file:!0,password:!0,image:!0})b.pseudos[e]=de(e);for(e in{submit:!0,reset:!0})b.pseudos[e]=he(e);function me(){}function xe(e){for(var t=0,n=e.length,r="";t<n;t++)r+=e[t].value;return r}function be(s,e,t){var u=e.dir,l=e.next,c=l||u,f=t&&"parentNode"===c,p=r++;return e.first?function(e,t,n){while(e=e[u])if(1===e.nodeType||f)return s(e,t,n);return!1}:function(e,t,n){var r,i,o,a=[S,p];if(n){while(e=e[u])if((1===e.nodeType||f)&&s(e,t,n))return!0}else while(e=e[u])if(1===e.nodeType||f)if(i=(o=e[k]||(e[k]={}))[e.uniqueID]||(o[e.uniqueID]={}),l&&l===e.nodeName.toLowerCase())e=e[u]||e;else{if((r=i[c])&&r[0]===S&&r[1]===p)return a[2]=r[2];if((i[c]=a)[2]=s(e,t,n))return!0}return!1}}function we(i){return 1<i.length?function(e,t,n){var r=i.length;while(r--)if(!i[r](e,t,n))return!1;return!0}:i[0]}function Te(e,t,n,r,i){for(var o,a=[],s=0,u=e.length,l=null!=t;s<u;s++)(o=e[s])&&(n&&!n(o,r,i)||(a.push(o),l&&t.push(s)));return a}function Ce(d,h,g,v,y,e){return v&&!v[k]&&(v=Ce(v)),y&&!y[k]&&(y=Ce(y,e)),le(function(e,t,n,r){var i,o,a,s=[],u=[],l=t.length,c=e||function(e,t,n){for(var r=0,i=t.length;r<i;r++)se(e,t[r],n);return n}(h||"*",n.nodeType?[n]:n,[]),f=!d||!e&&h?c:Te(c,s,d,n,r),p=g?y||(e?d:l||v)?[]:t:f;if(g&&g(f,p,n,r),v){i=Te(p,u),v(i,[],n,r),o=i.length;while(o--)(a=i[o])&&(p[u[o]]=!(f[u[o]]=a))}if(e){if(y||d){if(y){i=[],o=p.length;while(o--)(a=p[o])&&i.push(f[o]=a);y(null,p=[],i,r)}o=p.length;while(o--)(a=p[o])&&-1<(i=y?P(e,a):s[o])&&(e[i]=!(t[i]=a))}}else p=Te(p===t?p.splice(l,p.length):p),y?y(null,t,p,r):H.apply(t,p)})}function Ee(e){for(var i,t,n,r=e.length,o=b.relative[e[0].type],a=o||b.relative[" "],s=o?1:0,u=be(function(e){return e===i},a,!0),l=be(function(e){return-1<P(i,e)},a,!0),c=[function(e,t,n){var r=!o&&(n||t!==w)||((i=t).nodeType?u(e,t,n):l(e,t,n));return i=null,r}];s<r;s++)if(t=b.relative[e[s].type])c=[be(we(c),t)];else{if((t=b.filter[e[s].type].apply(null,e[s].matches))[k]){for(n=++s;n<r;n++)if(b.relative[e[n].type])break;return Ce(1<s&&we(c),1<s&&xe(e.slice(0,s-1).concat({value:" "===e[s-2].type?"*":""})).replace(B,"$1"),t,s<n&&Ee(e.slice(s,n)),n<r&&Ee(e=e.slice(n)),n<r&&xe(e))}c.push(t)}return we(c)}return me.prototype=b.filters=b.pseudos,b.setFilters=new me,h=se.tokenize=function(e,t){var n,r,i,o,a,s,u,l=x[e+" "];if(l)return t?0:l.slice(0);a=e,s=[],u=b.preFilter;while(a){for(o in n&&!(r=_.exec(a))||(r&&(a=a.slice(r[0].length)||a),s.push(i=[])),n=!1,(r=z.exec(a))&&(n=r.shift(),i.push({value:n,type:r[0].replace(B," ")}),a=a.slice(n.length)),b.filter)!(r=G[o].exec(a))||u[o]&&!(r=u[o](r))||(n=r.shift(),i.push({value:n,type:o,matches:r}),a=a.slice(n.length));if(!n)break}return t?a.length:a?se.error(e):x(e,s).slice(0)},f=se.compile=function(e,t){var n,v,y,m,x,r,i=[],o=[],a=N[e+" "];if(!a){t||(t=h(e)),n=t.length;while(n--)(a=Ee(t[n]))[k]?i.push(a):o.push(a);(a=N(e,(v=o,m=0<(y=i).length,x=0<v.length,r=function(e,t,n,r,i){var o,a,s,u=0,l="0",c=e&&[],f=[],p=w,d=e||x&&b.find.TAG("*",i),h=S+=null==p?1:Math.random()||.1,g=d.length;for(i&&(w=t===C||t||i);l!==g&&null!=(o=d[l]);l++){if(x&&o){a=0,t||o.ownerDocument===C||(T(o),n=!E);while(s=v[a++])if(s(o,t||C,n)){r.push(o);break}i&&(S=h)}m&&((o=!s&&o)&&u--,e&&c.push(o))}if(u+=l,m&&l!==u){a=0;while(s=y[a++])s(c,f,t,n);if(e){if(0<u)while(l--)c[l]||f[l]||(f[l]=q.call(r));f=Te(f)}H.apply(r,f),i&&!e&&0<f.length&&1<u+y.length&&se.uniqueSort(r)}return i&&(S=h,w=p),c},m?le(r):r))).selector=e}return a},g=se.select=function(e,t,n,r){var i,o,a,s,u,l="function"==typeof e&&e,c=!r&&h(e=l.selector||e);if(n=n||[],1===c.length){if(2<(o=c[0]=c[0].slice(0)).length&&"ID"===(a=o[0]).type&&9===t.nodeType&&E&&b.relative[o[1].type]){if(!(t=(b.find.ID(a.matches[0].replace(te,ne),t)||[])[0]))return n;l&&(t=t.parentNode),e=e.slice(o.shift().value.length)}i=G.needsContext.test(e)?0:o.length;while(i--){if(a=o[i],b.relative[s=a.type])break;if((u=b.find[s])&&(r=u(a.matches[0].replace(te,ne),ee.test(o[0].type)&&ye(t.parentNode)||t))){if(o.splice(i,1),!(e=r.length&&xe(o)))return H.apply(n,r),n;break}}}return(l||f(e,c))(r,t,!E,n,!t||ee.test(e)&&ye(t.parentNode)||t),n},d.sortStable=k.split("").sort(D).join("")===k,d.detectDuplicates=!!l,T(),d.sortDetached=ce(function(e){return 1&e.compareDocumentPosition(C.createElement("fieldset"))}),ce(function(e){return e.innerHTML="<a href='#'></a>","#"===e.firstChild.getAttribute("href")})||fe("type|href|height|width",function(e,t,n){if(!n)return e.getAttribute(t,"type"===t.toLowerCase()?1:2)}),d.attributes&&ce(function(e){return e.innerHTML="<input/>",e.firstChild.setAttribute("value",""),""===e.firstChild.getAttribute("value")})||fe("value",function(e,t,n){if(!n&&"input"===e.nodeName.toLowerCase())return e.defaultValue}),ce(function(e){return null==e.getAttribute("disabled")})||fe(R,function(e,t,n){var r;if(!n)return!0===e[t]?t.toLowerCase():(r=e.getAttributeNode(t))&&r.specified?r.value:null}),se}(C);k.find=h,k.expr=h.selectors,k.expr[":"]=k.expr.pseudos,k.uniqueSort=k.unique=h.uniqueSort,k.text=h.getText,k.isXMLDoc=h.isXML,k.contains=h.contains,k.escapeSelector=h.escape;var T=function(e,t,n){var r=[],i=void 0!==n;while((e=e[t])&&9!==e.nodeType)if(1===e.nodeType){if(i&&k(e).is(n))break;r.push(e)}return r},S=function(e,t){for(var n=[];e;e=e.nextSibling)1===e.nodeType&&e!==t&&n.push(e);return n},N=k.expr.match.needsContext;function A(e,t){return e.nodeName&&e.nodeName.toLowerCase()===t.toLowerCase()}var D=/^<([a-z][^\/\0>:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function j(e,n,r){return m(n)?k.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?k.grep(e,function(e){return e===n!==r}):"string"!=typeof n?k.grep(e,function(e){return-1<i.call(n,e)!==r}):k.filter(n,e,r)}k.filter=function(e,t,n){var r=t[0];return n&&(e=":not("+e+")"),1===t.length&&1===r.nodeType?k.find.matchesSelector(r,e)?[r]:[]:k.find.matches(e,k.grep(t,function(e){return 1===e.nodeType}))},k.fn.extend({find:function(e){var t,n,r=this.length,i=this;if("string"!=typeof e)return this.pushStack(k(e).filter(function(){for(t=0;t<r;t++)if(k.contains(i[t],this))return!0}));for(n=this.pushStack([]),t=0;t<r;t++)k.find(e,i[t],n);return 1<r?k.uniqueSort(n):n},filter:function(e){return this.pushStack(j(this,e||[],!1))},not:function(e){return this.pushStack(j(this,e||[],!0))},is:function(e){return!!j(this,"string"==typeof e&&N.test(e)?k(e):e||[],!1).length}});var q,L=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]+))$/;(k.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||q,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:L.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof k?t[0]:t,k.merge(this,k.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:E,!0)),D.test(r[1])&&k.isPlainObject(t))for(r in t)m(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=E.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):m(e)?void 0!==n.ready?n.ready(e):e(k):k.makeArray(e,this)}).prototype=k.fn,q=k(E);var H=/^(?:parents|prev(?:Until|All))/,O={children:!0,contents:!0,next:!0,prev:!0};function P(e,t){while((e=e[t])&&1!==e.nodeType);return e}k.fn.extend({has:function(e){var t=k(e,this),n=t.length;return this.filter(function(){for(var e=0;e<n;e++)if(k.contains(this,t[e]))return!0})},closest:function(e,t){var n,r=0,i=this.length,o=[],a="string"!=typeof e&&k(e);if(!N.test(e))for(;r<i;r++)for(n=this[r];n&&n!==t;n=n.parentNode)if(n.nodeType<11&&(a?-1<a.index(n):1===n.nodeType&&k.find.matchesSelector(n,e))){o.push(n);break}return this.pushStack(1<o.length?k.uniqueSort(o):o)},index:function(e){return e?"string"==typeof e?i.call(k(e),this[0]):i.call(this,e.jquery?e[0]:e):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(e,t){return this.pushStack(k.uniqueSort(k.merge(this.get(),k(e,t))))},addBack:function(e){return this.add(null==e?this.prevObject:this.prevObject.filter(e))}}),k.each({parent:function(e){var t=e.parentNode;return t&&11!==t.nodeType?t:null},parents:function(e){return T(e,"parentNode")},parentsUntil:function(e,t,n){return T(e,"parentNode",n)},next:function(e){return P(e,"nextSibling")},prev:function(e){return P(e,"previousSibling")},nextAll:function(e){return T(e,"nextSibling")},prevAll:function(e){return T(e,"previousSibling")},nextUntil:function(e,t,n){return T(e,"nextSibling",n)},prevUntil:function(e,t,n){return T(e,"previousSibling",n)},siblings:function(e){return S((e.parentNode||{}).firstChild,e)},children:function(e){return S(e.firstChild)},contents:function(e){return"undefined"!=typeof e.contentDocument?e.contentDocument:(A(e,"template")&&(e=e.content||e),k.merge([],e.childNodes))}},function(r,i){k.fn[r]=function(e,t){var n=k.map(this,i,e);return"Until"!==r.slice(-5)&&(t=e),t&&"string"==typeof t&&(n=k.filter(t,n)),1<this.length&&(O[r]||k.uniqueSort(n),H.test(r)&&n.reverse()),this.pushStack(n)}});var R=/[^\x20\t\r\n\f]+/g;function M(e){return e}function I(e){throw e}function W(e,t,n,r){var i;try{e&&m(i=e.promise)?i.call(e).done(t).fail(n):e&&m(i=e.then)?i.call(e,t,n):t.apply(void 0,[e].slice(r))}catch(e){n.apply(void 0,[e])}}k.Callbacks=function(r){var e,n;r="string"==typeof r?(e=r,n={},k.each(e.match(R)||[],function(e,t){n[t]=!0}),n):k.extend({},r);var i,t,o,a,s=[],u=[],l=-1,c=function(){for(a=a||r.once,o=i=!0;u.length;l=-1){t=u.shift();while(++l<s.length)!1===s[l].apply(t[0],t[1])&&r.stopOnFalse&&(l=s.length,t=!1)}r.memory||(t=!1),i=!1,a&&(s=t?[]:"")},f={add:function(){return s&&(t&&!i&&(l=s.length-1,u.push(t)),function n(e){k.each(e,function(e,t){m(t)?r.unique&&f.has(t)||s.push(t):t&&t.length&&"string"!==w(t)&&n(t)})}(arguments),t&&!i&&c()),this},remove:function(){return k.each(arguments,function(e,t){var n;while(-1<(n=k.inArray(t,s,n)))s.splice(n,1),n<=l&&l--}),this},has:function(e){return e?-1<k.inArray(e,s):0<s.length},empty:function(){return s&&(s=[]),this},disable:function(){return a=u=[],s=t="",this},disabled:function(){return!s},lock:function(){return a=u=[],t||i||(s=t=""),this},locked:function(){return!!a},fireWith:function(e,t){return a||(t=[e,(t=t||[]).slice?t.slice():t],u.push(t),i||c()),this},fire:function(){return f.fireWith(this,arguments),this},fired:function(){return!!o}};return f},k.extend({Deferred:function(e){var o=[["notify","progress",k.Callbacks("memory"),k.Callbacks("memory"),2],["resolve","done",k.Callbacks("once memory"),k.Callbacks("once memory"),0,"resolved"],["reject","fail",k.Callbacks("once memory"),k.Callbacks("once memory"),1,"rejected"]],i="pending",a={state:function(){return i},always:function(){return s.done(arguments).fail(arguments),this},"catch":function(e){return a.then(null,e)},pipe:function(){var i=arguments;return k.Deferred(function(r){k.each(o,function(e,t){var n=m(i[t[4]])&&i[t[4]];s[t[1]](function(){var e=n&&n.apply(this,arguments);e&&m(e.promise)?e.promise().progress(r.notify).done(r.resolve).fail(r.reject):r[t[0]+"With"](this,n?[e]:arguments)})}),i=null}).promise()},then:function(t,n,r){var u=0;function l(i,o,a,s){return function(){var n=this,r=arguments,e=function(){var e,t;if(!(i<u)){if((e=a.apply(n,r))===o.promise())throw new TypeError("Thenable self-resolution");t=e&&("object"==typeof e||"function"==typeof e)&&e.then,m(t)?s?t.call(e,l(u,o,M,s),l(u,o,I,s)):(u++,t.call(e,l(u,o,M,s),l(u,o,I,s),l(u,o,M,o.notifyWith))):(a!==M&&(n=void 0,r=[e]),(s||o.resolveWith)(n,r))}},t=s?e:function(){try{e()}catch(e){k.Deferred.exceptionHook&&k.Deferred.exceptionHook(e,t.stackTrace),u<=i+1&&(a!==I&&(n=void 0,r=[e]),o.rejectWith(n,r))}};i?t():(k.Deferred.getStackHook&&(t.stackTrace=k.Deferred.getStackHook()),C.setTimeout(t))}}return k.Deferred(function(e){o[0][3].add(l(0,e,m(r)?r:M,e.notifyWith)),o[1][3].add(l(0,e,m(t)?t:M)),o[2][3].add(l(0,e,m(n)?n:I))}).promise()},promise:function(e){return null!=e?k.extend(e,a):a}},s={};return k.each(o,function(e,t){var n=t[2],r=t[5];a[t[1]]=n.add,r&&n.add(function(){i=r},o[3-e][2].disable,o[3-e][3].disable,o[0][2].lock,o[0][3].lock),n.add(t[3].fire),s[t[0]]=function(){return s[t[0]+"With"](this===s?void 0:this,arguments),this},s[t[0]+"With"]=n.fireWith}),a.promise(s),e&&e.call(s,s),s},when:function(e){var n=arguments.length,t=n,r=Array(t),i=s.call(arguments),o=k.Deferred(),a=function(t){return function(e){r[t]=this,i[t]=1<arguments.length?s.call(arguments):e,--n||o.resolveWith(r,i)}};if(n<=1&&(W(e,o.done(a(t)).resolve,o.reject,!n),"pending"===o.state()||m(i[t]&&i[t].then)))return o.then();while(t--)W(i[t],a(t),o.reject);return o.promise()}});var $=/^(Eval|Internal|Range|Reference|Syntax|Type|URI)Error$/;k.Deferred.exceptionHook=function(e,t){C.console&&C.console.warn&&e&&$.test(e.name)&&C.console.warn("jQuery.Deferred exception: "+e.message,e.stack,t)},k.readyException=function(e){C.setTimeout(function(){throw e})};var F=k.Deferred();function B(){E.removeEventListener("DOMContentLoaded",B),C.removeEventListener("load",B),k.ready()}k.fn.ready=function(e){return F.then(e)["catch"](function(e){k.readyException(e)}),this},k.extend({isReady:!1,readyWait:1,ready:function(e){(!0===e?--k.readyWait:k.isReady)||(k.isReady=!0)!==e&&0<--k.readyWait||F.resolveWith(E,[k])}}),k.ready.then=F.then,"complete"===E.readyState||"loading"!==E.readyState&&!E.documentElement.doScroll?C.setTimeout(k.ready):(E.addEventListener("DOMContentLoaded",B),C.addEventListener("load",B));var _=function(e,t,n,r,i,o,a){var s=0,u=e.length,l=null==n;if("object"===w(n))for(s in i=!0,n)_(e,t,s,n[s],!0,o,a);else if(void 0!==r&&(i=!0,m(r)||(a=!0),l&&(a?(t.call(e,r),t=null):(l=t,t=function(e,t,n){return l.call(k(e),n)})),t))for(;s<u;s++)t(e[s],n,a?r:r.call(e[s],s,t(e[s],n)));return i?e:l?t.call(e):u?t(e[0],n):o},z=/^-ms-/,U=/-([a-z])/g;function X(e,t){return t.toUpperCase()}function V(e){return e.replace(z,"ms-").replace(U,X)}var G=function(e){return 1===e.nodeType||9===e.nodeType||!+e.nodeType};function Y(){this.expando=k.expando+Y.uid++}Y.uid=1,Y.prototype={cache:function(e){var t=e[this.expando];return t||(t={},G(e)&&(e.nodeType?e[this.expando]=t:Object.defineProperty(e,this.expando,{value:t,configurable:!0}))),t},set:function(e,t,n){var r,i=this.cache(e);if("string"==typeof t)i[V(t)]=n;else for(r in t)i[V(r)]=t[r];return i},get:function(e,t){return void 0===t?this.cache(e):e[this.expando]&&e[this.expando][V(t)]},access:function(e,t,n){return void 0===t||t&&"string"==typeof t&&void 0===n?this.get(e,t):(this.set(e,t,n),void 0!==n?n:t)},remove:function(e,t){var n,r=e[this.expando];if(void 0!==r){if(void 0!==t){n=(t=Array.isArray(t)?t.map(V):(t=V(t))in r?[t]:t.match(R)||[]).length;while(n--)delete r[t[n]]}(void 0===t||k.isEmptyObject(r))&&(e.nodeType?e[this.expando]=void 0:delete e[this.expando])}},hasData:function(e){var t=e[this.expando];return void 0!==t&&!k.isEmptyObject(t)}};var Q=new Y,J=new Y,K=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,Z=/[A-Z]/g;function ee(e,t,n){var r,i;if(void 0===n&&1===e.nodeType)if(r="data-"+t.replace(Z,"-$&").toLowerCase(),"string"==typeof(n=e.getAttribute(r))){try{n="true"===(i=n)||"false"!==i&&("null"===i?null:i===+i+""?+i:K.test(i)?JSON.parse(i):i)}catch(e){}J.set(e,t,n)}else n=void 0;return n}k.extend({hasData:function(e){return J.hasData(e)||Q.hasData(e)},data:function(e,t,n){return J.access(e,t,n)},removeData:function(e,t){J.remove(e,t)},_data:function(e,t,n){return Q.access(e,t,n)},_removeData:function(e,t){Q.remove(e,t)}}),k.fn.extend({data:function(n,e){var t,r,i,o=this[0],a=o&&o.attributes;if(void 0===n){if(this.length&&(i=J.get(o),1===o.nodeType&&!Q.get(o,"hasDataAttrs"))){t=a.length;while(t--)a[t]&&0===(r=a[t].name).indexOf("data-")&&(r=V(r.slice(5)),ee(o,r,i[r]));Q.set(o,"hasDataAttrs",!0)}return i}return"object"==typeof n?this.each(function(){J.set(this,n)}):_(this,function(e){var t;if(o&&void 0===e)return void 0!==(t=J.get(o,n))?t:void 0!==(t=ee(o,n))?t:void 0;this.each(function(){J.set(this,n,e)})},null,e,1<arguments.length,null,!0)},removeData:function(e){return this.each(function(){J.remove(this,e)})}}),k.extend({queue:function(e,t,n){var r;if(e)return t=(t||"fx")+"queue",r=Q.get(e,t),n&&(!r||Array.isArray(n)?r=Q.access(e,t,k.makeArray(n)):r.push(n)),r||[]},dequeue:function(e,t){t=t||"fx";var n=k.queue(e,t),r=n.length,i=n.shift(),o=k._queueHooks(e,t);"inprogress"===i&&(i=n.shift(),r--),i&&("fx"===t&&n.unshift("inprogress"),delete o.stop,i.call(e,function(){k.dequeue(e,t)},o)),!r&&o&&o.empty.fire()},_queueHooks:function(e,t){var n=t+"queueHooks";return Q.get(e,n)||Q.access(e,n,{empty:k.Callbacks("once memory").add(function(){Q.remove(e,[t+"queue",n])})})}}),k.fn.extend({queue:function(t,n){var e=2;return"string"!=typeof t&&(n=t,t="fx",e--),arguments.length<e?k.queue(this[0],t):void 0===n?this:this.each(function(){var e=k.queue(this,t,n);k._queueHooks(this,t),"fx"===t&&"inprogress"!==e[0]&&k.dequeue(this,t)})},dequeue:function(e){return this.each(function(){k.dequeue(this,e)})},clearQueue:function(e){return this.queue(e||"fx",[])},promise:function(e,t){var n,r=1,i=k.Deferred(),o=this,a=this.length,s=function(){--r||i.resolveWith(o,[o])};"string"!=typeof e&&(t=e,e=void 0),e=e||"fx";while(a--)(n=Q.get(o[a],e+"queueHooks"))&&n.empty&&(r++,n.empty.add(s));return s(),i.promise(t)}});var te=/[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/.source,ne=new RegExp("^(?:([+-])=|)("+te+")([a-z%]*)$","i"),re=["Top","Right","Bottom","Left"],ie=E.documentElement,oe=function(e){return k.contains(e.ownerDocument,e)},ae={composed:!0};ie.getRootNode&&(oe=function(e){return k.contains(e.ownerDocument,e)||e.getRootNode(ae)===e.ownerDocument});var se=function(e,t){return"none"===(e=t||e).style.display||""===e.style.display&&oe(e)&&"none"===k.css(e,"display")},ue=function(e,t,n,r){var i,o,a={};for(o in t)a[o]=e.style[o],e.style[o]=t[o];for(o in i=n.apply(e,r||[]),t)e.style[o]=a[o];return i};function le(e,t,n,r){var i,o,a=20,s=r?function(){return r.cur()}:function(){return k.css(e,t,"")},u=s(),l=n&&n[3]||(k.cssNumber[t]?"":"px"),c=e.nodeType&&(k.cssNumber[t]||"px"!==l&&+u)&&ne.exec(k.css(e,t));if(c&&c[3]!==l){u/=2,l=l||c[3],c=+u||1;while(a--)k.style(e,t,c+l),(1-o)*(1-(o=s()/u||.5))<=0&&(a=0),c/=o;c*=2,k.style(e,t,c+l),n=n||[]}return n&&(c=+c||+u||0,i=n[1]?c+(n[1]+1)*n[2]:+n[2],r&&(r.unit=l,r.start=c,r.end=i)),i}var ce={};function fe(e,t){for(var n,r,i,o,a,s,u,l=[],c=0,f=e.length;c<f;c++)(r=e[c]).style&&(n=r.style.display,t?("none"===n&&(l[c]=Q.get(r,"display")||null,l[c]||(r.style.display="")),""===r.style.display&&se(r)&&(l[c]=(u=a=o=void 0,a=(i=r).ownerDocument,s=i.nodeName,(u=ce[s])||(o=a.body.appendChild(a.createElement(s)),u=k.css(o,"display"),o.parentNode.removeChild(o),"none"===u&&(u="block"),ce[s]=u)))):"none"!==n&&(l[c]="none",Q.set(r,"display",n)));for(c=0;c<f;c++)null!=l[c]&&(e[c].style.display=l[c]);return e}k.fn.extend({show:function(){return fe(this,!0)},hide:function(){return fe(this)},toggle:function(e){return"boolean"==typeof e?e?this.show():this.hide():this.each(function(){se(this)?k(this).show():k(this).hide()})}});var pe=/^(?:checkbox|radio)$/i,de=/<([a-z][^\/\0>\x20\t\r\n\f]*)/i,he=/^$|^module$|\/(?:java|ecma)script/i,ge={option:[1,"<select multiple='multiple'>","</select>"],thead:[1,"<table>","</table>"],col:[2,"<table><colgroup>","</colgroup></table>"],tr:[2,"<table><tbody>","</tbody></table>"],td:[3,"<table><tbody><tr>","</tr></tbody></table>"],_default:[0,"",""]};function ve(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&A(e,t)?k.merge([e],n):n}function ye(e,t){for(var n=0,r=e.length;n<r;n++)Q.set(e[n],"globalEval",!t||Q.get(t[n],"globalEval"))}ge.optgroup=ge.option,ge.tbody=ge.tfoot=ge.colgroup=ge.caption=ge.thead,ge.th=ge.td;var me,xe,be=/<|&#?\w+;/;function we(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),p=[],d=0,h=e.length;d<h;d++)if((o=e[d])||0===o)if("object"===w(o))k.merge(p,o.nodeType?[o]:o);else if(be.test(o)){a=a||f.appendChild(t.createElement("div")),s=(de.exec(o)||["",""])[1].toLowerCase(),u=ge[s]||ge._default,a.innerHTML=u[1]+k.htmlPrefilter(o)+u[2],c=u[0];while(c--)a=a.lastChild;k.merge(p,a.childNodes),(a=f.firstChild).textContent=""}else p.push(t.createTextNode(o));f.textContent="",d=0;while(o=p[d++])if(r&&-1<k.inArray(o,r))i&&i.push(o);else if(l=oe(o),a=ve(f.appendChild(o),"script"),l&&ye(a),n){c=0;while(o=a[c++])he.test(o.type||"")&&n.push(o)}return f}me=E.createDocumentFragment().appendChild(E.createElement("div")),(xe=E.createElement("input")).setAttribute("type","radio"),xe.setAttribute("checked","checked"),xe.setAttribute("name","t"),me.appendChild(xe),y.checkClone=me.cloneNode(!0).cloneNode(!0).lastChild.checked,me.innerHTML="<textarea>x</textarea>",y.noCloneChecked=!!me.cloneNode(!0).lastChild.defaultValue;var Te=/^key/,Ce=/^(?:mouse|pointer|contextmenu|drag|drop)|click/,Ee=/^([^.]*)(?:\.(.+)|)/;function ke(){return!0}function Se(){return!1}function Ne(e,t){return e===function(){try{return E.activeElement}catch(e){}}()==("focus"===t)}function Ae(e,t,n,r,i,o){var a,s;if("object"==typeof t){for(s in"string"!=typeof n&&(r=r||n,n=void 0),t)Ae(e,s,n,r,t[s],o);return e}if(null==r&&null==i?(i=n,r=n=void 0):null==i&&("string"==typeof n?(i=r,r=void 0):(i=r,r=n,n=void 0)),!1===i)i=Se;else if(!i)return e;return 1===o&&(a=i,(i=function(e){return k().off(e),a.apply(this,arguments)}).guid=a.guid||(a.guid=k.guid++)),e.each(function(){k.event.add(this,t,i,r,n)})}function De(e,i,o){o?(Q.set(e,i,!1),k.event.add(e,i,{namespace:!1,handler:function(e){var t,n,r=Q.get(this,i);if(1&e.isTrigger&&this[i]){if(r.length)(k.event.special[i]||{}).delegateType&&e.stopPropagation();else if(r=s.call(arguments),Q.set(this,i,r),t=o(this,i),this[i](),r!==(n=Q.get(this,i))||t?Q.set(this,i,!1):n={},r!==n)return e.stopImmediatePropagation(),e.preventDefault(),n.value}else r.length&&(Q.set(this,i,{value:k.event.trigger(k.extend(r[0],k.Event.prototype),r.slice(1),this)}),e.stopImmediatePropagation())}})):void 0===Q.get(e,i)&&k.event.add(e,i,ke)}k.event={global:{},add:function(t,e,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Q.get(t);if(v){n.handler&&(n=(o=n).handler,i=o.selector),i&&k.find.matchesSelector(ie,i),n.guid||(n.guid=k.guid++),(u=v.events)||(u=v.events={}),(a=v.handle)||(a=v.handle=function(e){return"undefined"!=typeof k&&k.event.triggered!==e.type?k.event.dispatch.apply(t,arguments):void 0}),l=(e=(e||"").match(R)||[""]).length;while(l--)d=g=(s=Ee.exec(e[l])||[])[1],h=(s[2]||"").split(".").sort(),d&&(f=k.event.special[d]||{},d=(i?f.delegateType:f.bindType)||d,f=k.event.special[d]||{},c=k.extend({type:d,origType:g,data:r,handler:n,guid:n.guid,selector:i,needsContext:i&&k.expr.match.needsContext.test(i),namespace:h.join(".")},o),(p=u[d])||((p=u[d]=[]).delegateCount=0,f.setup&&!1!==f.setup.call(t,r,h,a)||t.addEventListener&&t.addEventListener(d,a)),f.add&&(f.add.call(t,c),c.handler.guid||(c.handler.guid=n.guid)),i?p.splice(p.delegateCount++,0,c):p.push(c),k.event.global[d]=!0)}},remove:function(e,t,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Q.hasData(e)&&Q.get(e);if(v&&(u=v.events)){l=(t=(t||"").match(R)||[""]).length;while(l--)if(d=g=(s=Ee.exec(t[l])||[])[1],h=(s[2]||"").split(".").sort(),d){f=k.event.special[d]||{},p=u[d=(r?f.delegateType:f.bindType)||d]||[],s=s[2]&&new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"),a=o=p.length;while(o--)c=p[o],!i&&g!==c.origType||n&&n.guid!==c.guid||s&&!s.test(c.namespace)||r&&r!==c.selector&&("**"!==r||!c.selector)||(p.splice(o,1),c.selector&&p.delegateCount--,f.remove&&f.remove.call(e,c));a&&!p.length&&(f.teardown&&!1!==f.teardown.call(e,h,v.handle)||k.removeEvent(e,d,v.handle),delete u[d])}else for(d in u)k.event.remove(e,d+t[l],n,r,!0);k.isEmptyObject(u)&&Q.remove(e,"handle events")}},dispatch:function(e){var t,n,r,i,o,a,s=k.event.fix(e),u=new Array(arguments.length),l=(Q.get(this,"events")||{})[s.type]||[],c=k.event.special[s.type]||{};for(u[0]=s,t=1;t<arguments.length;t++)u[t]=arguments[t];if(s.delegateTarget=this,!c.preDispatch||!1!==c.preDispatch.call(this,s)){a=k.event.handlers.call(this,s,l),t=0;while((i=a[t++])&&!s.isPropagationStopped()){s.currentTarget=i.elem,n=0;while((o=i.handlers[n++])&&!s.isImmediatePropagationStopped())s.rnamespace&&!1!==o.namespace&&!s.rnamespace.test(o.namespace)||(s.handleObj=o,s.data=o.data,void 0!==(r=((k.event.special[o.origType]||{}).handle||o.handler).apply(i.elem,u))&&!1===(s.result=r)&&(s.preventDefault(),s.stopPropagation()))}return c.postDispatch&&c.postDispatch.call(this,s),s.result}},handlers:function(e,t){var n,r,i,o,a,s=[],u=t.delegateCount,l=e.target;if(u&&l.nodeType&&!("click"===e.type&&1<=e.button))for(;l!==this;l=l.parentNode||this)if(1===l.nodeType&&("click"!==e.type||!0!==l.disabled)){for(o=[],a={},n=0;n<u;n++)void 0===a[i=(r=t[n]).selector+" "]&&(a[i]=r.needsContext?-1<k(i,this).index(l):k.find(i,this,null,[l]).length),a[i]&&o.push(r);o.length&&s.push({elem:l,handlers:o})}return l=this,u<t.length&&s.push({elem:l,handlers:t.slice(u)}),s},addProp:function(t,e){Object.defineProperty(k.Event.prototype,t,{enumerable:!0,configurable:!0,get:m(e)?function(){if(this.originalEvent)return e(this.originalEvent)}:function(){if(this.originalEvent)return this.originalEvent[t]},set:function(e){Object.defineProperty(this,t,{enumerable:!0,configurable:!0,writable:!0,value:e})}})},fix:function(e){return e[k.expando]?e:new k.Event(e)},special:{load:{noBubble:!0},click:{setup:function(e){var t=this||e;return pe.test(t.type)&&t.click&&A(t,"input")&&De(t,"click",ke),!1},trigger:function(e){var t=this||e;return pe.test(t.type)&&t.click&&A(t,"input")&&De(t,"click"),!0},_default:function(e){var t=e.target;return pe.test(t.type)&&t.click&&A(t,"input")&&Q.get(t,"click")||A(t,"a")}},beforeunload:{postDispatch:function(e){void 0!==e.result&&e.originalEvent&&(e.originalEvent.returnValue=e.result)}}}},k.removeEvent=function(e,t,n){e.removeEventListener&&e.removeEventListener(t,n)},k.Event=function(e,t){if(!(this instanceof k.Event))return new k.Event(e,t);e&&e.type?(this.originalEvent=e,this.type=e.type,this.isDefaultPrevented=e.defaultPrevented||void 0===e.defaultPrevented&&!1===e.returnValue?ke:Se,this.target=e.target&&3===e.target.nodeType?e.target.parentNode:e.target,this.currentTarget=e.currentTarget,this.relatedTarget=e.relatedTarget):this.type=e,t&&k.extend(this,t),this.timeStamp=e&&e.timeStamp||Date.now(),this[k.expando]=!0},k.Event.prototype={constructor:k.Event,isDefaultPrevented:Se,isPropagationStopped:Se,isImmediatePropagationStopped:Se,isSimulated:!1,preventDefault:function(){var e=this.originalEvent;this.isDefaultPrevented=ke,e&&!this.isSimulated&&e.preventDefault()},stopPropagation:function(){var e=this.originalEvent;this.isPropagationStopped=ke,e&&!this.isSimulated&&e.stopPropagation()},stopImmediatePropagation:function(){var e=this.originalEvent;this.isImmediatePropagationStopped=ke,e&&!this.isSimulated&&e.stopImmediatePropagation(),this.stopPropagation()}},k.each({altKey:!0,bubbles:!0,cancelable:!0,changedTouches:!0,ctrlKey:!0,detail:!0,eventPhase:!0,metaKey:!0,pageX:!0,pageY:!0,shiftKey:!0,view:!0,"char":!0,code:!0,charCode:!0,key:!0,keyCode:!0,button:!0,buttons:!0,clientX:!0,clientY:!0,offsetX:!0,offsetY:!0,pointerId:!0,pointerType:!0,screenX:!0,screenY:!0,targetTouches:!0,toElement:!0,touches:!0,which:function(e){var t=e.button;return null==e.which&&Te.test(e.type)?null!=e.charCode?e.charCode:e.keyCode:!e.which&&void 0!==t&&Ce.test(e.type)?1&t?1:2&t?3:4&t?2:0:e.which}},k.event.addProp),k.each({focus:"focusin",blur:"focusout"},function(e,t){k.event.special[e]={setup:function(){return De(this,e,Ne),!1},trigger:function(){return De(this,e),!0},delegateType:t}}),k.each({mouseenter:"mouseover",mouseleave:"mouseout",pointerenter:"pointerover",pointerleave:"pointerout"},function(e,i){k.event.special[e]={delegateType:i,bindType:i,handle:function(e){var t,n=e.relatedTarget,r=e.handleObj;return n&&(n===this||k.contains(this,n))||(e.type=r.origType,t=r.handler.apply(this,arguments),e.type=i),t}}}),k.fn.extend({on:function(e,t,n,r){return Ae(this,e,t,n,r)},one:function(e,t,n,r){return Ae(this,e,t,n,r,1)},off:function(e,t,n){var r,i;if(e&&e.preventDefault&&e.handleObj)return r=e.handleObj,k(e.delegateTarget).off(r.namespace?r.origType+"."+r.namespace:r.origType,r.selector,r.handler),this;if("object"==typeof e){for(i in e)this.off(i,t,e[i]);return this}return!1!==t&&"function"!=typeof t||(n=t,t=void 0),!1===n&&(n=Se),this.each(function(){k.event.remove(this,e,n,t)})}});var je=/<(?!area|br|col|embed|hr|img|input|link|meta|param)(([a-z][^\/\0>\x20\t\r\n\f]*)[^>]*)\/>/gi,qe=/<script|<style|<link/i,Le=/checked\s*(?:[^=]|=\s*.checked.)/i,He=/^\s*<!(?:\[CDATA\[|--)|(?:\]\]|--)>\s*$/g;function Oe(e,t){return A(e,"table")&&A(11!==t.nodeType?t:t.firstChild,"tr")&&k(e).children("tbody")[0]||e}function Pe(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function Re(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Me(e,t){var n,r,i,o,a,s,u,l;if(1===t.nodeType){if(Q.hasData(e)&&(o=Q.access(e),a=Q.set(t,o),l=o.events))for(i in delete a.handle,a.events={},l)for(n=0,r=l[i].length;n<r;n++)k.event.add(t,i,l[i][n]);J.hasData(e)&&(s=J.access(e),u=k.extend({},s),J.set(t,u))}}function Ie(n,r,i,o){r=g.apply([],r);var e,t,a,s,u,l,c=0,f=n.length,p=f-1,d=r[0],h=m(d);if(h||1<f&&"string"==typeof d&&!y.checkClone&&Le.test(d))return n.each(function(e){var t=n.eq(e);h&&(r[0]=d.call(this,e,t.html())),Ie(t,r,i,o)});if(f&&(t=(e=we(r,n[0].ownerDocument,!1,n,o)).firstChild,1===e.childNodes.length&&(e=t),t||o)){for(s=(a=k.map(ve(e,"script"),Pe)).length;c<f;c++)u=e,c!==p&&(u=k.clone(u,!0,!0),s&&k.merge(a,ve(u,"script"))),i.call(n[c],u,c);if(s)for(l=a[a.length-1].ownerDocument,k.map(a,Re),c=0;c<s;c++)u=a[c],he.test(u.type||"")&&!Q.access(u,"globalEval")&&k.contains(l,u)&&(u.src&&"module"!==(u.type||"").toLowerCase()?k._evalUrl&&!u.noModule&&k._evalUrl(u.src,{nonce:u.nonce||u.getAttribute("nonce")}):b(u.textContent.replace(He,""),u,l))}return n}function We(e,t,n){for(var r,i=t?k.filter(t,e):e,o=0;null!=(r=i[o]);o++)n||1!==r.nodeType||k.cleanData(ve(r)),r.parentNode&&(n&&oe(r)&&ye(ve(r,"script")),r.parentNode.removeChild(r));return e}k.extend({htmlPrefilter:function(e){return e.replace(je,"<$1></$2>")},clone:function(e,t,n){var r,i,o,a,s,u,l,c=e.cloneNode(!0),f=oe(e);if(!(y.noCloneChecked||1!==e.nodeType&&11!==e.nodeType||k.isXMLDoc(e)))for(a=ve(c),r=0,i=(o=ve(e)).length;r<i;r++)s=o[r],u=a[r],void 0,"input"===(l=u.nodeName.toLowerCase())&&pe.test(s.type)?u.checked=s.checked:"input"!==l&&"textarea"!==l||(u.defaultValue=s.defaultValue);if(t)if(n)for(o=o||ve(e),a=a||ve(c),r=0,i=o.length;r<i;r++)Me(o[r],a[r]);else Me(e,c);return 0<(a=ve(c,"script")).length&&ye(a,!f&&ve(e,"script")),c},cleanData:function(e){for(var t,n,r,i=k.event.special,o=0;void 0!==(n=e[o]);o++)if(G(n)){if(t=n[Q.expando]){if(t.events)for(r in t.events)i[r]?k.event.remove(n,r):k.removeEvent(n,r,t.handle);n[Q.expando]=void 0}n[J.expando]&&(n[J.expando]=void 0)}}}),k.fn.extend({detach:function(e){return We(this,e,!0)},remove:function(e){return We(this,e)},text:function(e){return _(this,function(e){return void 0===e?k.text(this):this.empty().each(function(){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||(this.textContent=e)})},null,e,arguments.length)},append:function(){return Ie(this,arguments,function(e){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||Oe(this,e).appendChild(e)})},prepend:function(){return Ie(this,arguments,function(e){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var t=Oe(this,e);t.insertBefore(e,t.firstChild)}})},before:function(){return Ie(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this)})},after:function(){return Ie(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this.nextSibling)})},empty:function(){for(var e,t=0;null!=(e=this[t]);t++)1===e.nodeType&&(k.cleanData(ve(e,!1)),e.textContent="");return this},clone:function(e,t){return e=null!=e&&e,t=null==t?e:t,this.map(function(){return k.clone(this,e,t)})},html:function(e){return _(this,function(e){var t=this[0]||{},n=0,r=this.length;if(void 0===e&&1===t.nodeType)return t.innerHTML;if("string"==typeof e&&!qe.test(e)&&!ge[(de.exec(e)||["",""])[1].toLowerCase()]){e=k.htmlPrefilter(e);try{for(;n<r;n++)1===(t=this[n]||{}).nodeType&&(k.cleanData(ve(t,!1)),t.innerHTML=e);t=0}catch(e){}}t&&this.empty().append(e)},null,e,arguments.length)},replaceWith:function(){var n=[];return Ie(this,arguments,function(e){var t=this.parentNode;k.inArray(this,n)<0&&(k.cleanData(ve(this)),t&&t.replaceChild(e,this))},n)}}),k.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(e,a){k.fn[e]=function(e){for(var t,n=[],r=k(e),i=r.length-1,o=0;o<=i;o++)t=o===i?this:this.clone(!0),k(r[o])[a](t),u.apply(n,t.get());return this.pushStack(n)}});var $e=new RegExp("^("+te+")(?!px)[a-z%]+$","i"),Fe=function(e){var t=e.ownerDocument.defaultView;return t&&t.opener||(t=C),t.getComputedStyle(e)},Be=new RegExp(re.join("|"),"i");function _e(e,t,n){var r,i,o,a,s=e.style;return(n=n||Fe(e))&&(""!==(a=n.getPropertyValue(t)||n[t])||oe(e)||(a=k.style(e,t)),!y.pixelBoxStyles()&&$e.test(a)&&Be.test(t)&&(r=s.width,i=s.minWidth,o=s.maxWidth,s.minWidth=s.maxWidth=s.width=a,a=n.width,s.width=r,s.minWidth=i,s.maxWidth=o)),void 0!==a?a+"":a}function ze(e,t){return{get:function(){if(!e())return(this.get=t).apply(this,arguments);delete this.get}}}!function(){function e(){if(u){s.style.cssText="position:absolute;left:-11111px;width:60px;margin-top:1px;padding:0;border:0",u.style.cssText="position:relative;display:block;box-sizing:border-box;overflow:scroll;margin:auto;border:1px;padding:1px;width:60%;top:1%",ie.appendChild(s).appendChild(u);var e=C.getComputedStyle(u);n="1%"!==e.top,a=12===t(e.marginLeft),u.style.right="60%",o=36===t(e.right),r=36===t(e.width),u.style.position="absolute",i=12===t(u.offsetWidth/3),ie.removeChild(s),u=null}}function t(e){return Math.round(parseFloat(e))}var n,r,i,o,a,s=E.createElement("div"),u=E.createElement("div");u.style&&(u.style.backgroundClip="content-box",u.cloneNode(!0).style.backgroundClip="",y.clearCloneStyle="content-box"===u.style.backgroundClip,k.extend(y,{boxSizingReliable:function(){return e(),r},pixelBoxStyles:function(){return e(),o},pixelPosition:function(){return e(),n},reliableMarginLeft:function(){return e(),a},scrollboxSize:function(){return e(),i}}))}();var Ue=["Webkit","Moz","ms"],Xe=E.createElement("div").style,Ve={};function Ge(e){var t=k.cssProps[e]||Ve[e];return t||(e in Xe?e:Ve[e]=function(e){var t=e[0].toUpperCase()+e.slice(1),n=Ue.length;while(n--)if((e=Ue[n]+t)in Xe)return e}(e)||e)}var Ye=/^(none|table(?!-c[ea]).+)/,Qe=/^--/,Je={position:"absolute",visibility:"hidden",display:"block"},Ke={letterSpacing:"0",fontWeight:"400"};function Ze(e,t,n){var r=ne.exec(t);return r?Math.max(0,r[2]-(n||0))+(r[3]||"px"):t}function et(e,t,n,r,i,o){var a="width"===t?1:0,s=0,u=0;if(n===(r?"border":"content"))return 0;for(;a<4;a+=2)"margin"===n&&(u+=k.css(e,n+re[a],!0,i)),r?("content"===n&&(u-=k.css(e,"padding"+re[a],!0,i)),"margin"!==n&&(u-=k.css(e,"border"+re[a]+"Width",!0,i))):(u+=k.css(e,"padding"+re[a],!0,i),"padding"!==n?u+=k.css(e,"border"+re[a]+"Width",!0,i):s+=k.css(e,"border"+re[a]+"Width",!0,i));return!r&&0<=o&&(u+=Math.max(0,Math.ceil(e["offset"+t[0].toUpperCase()+t.slice(1)]-o-u-s-.5))||0),u}function tt(e,t,n){var r=Fe(e),i=(!y.boxSizingReliable()||n)&&"border-box"===k.css(e,"boxSizing",!1,r),o=i,a=_e(e,t,r),s="offset"+t[0].toUpperCase()+t.slice(1);if($e.test(a)){if(!n)return a;a="auto"}return(!y.boxSizingReliable()&&i||"auto"===a||!parseFloat(a)&&"inline"===k.css(e,"display",!1,r))&&e.getClientRects().length&&(i="border-box"===k.css(e,"boxSizing",!1,r),(o=s in e)&&(a=e[s])),(a=parseFloat(a)||0)+et(e,t,n||(i?"border":"content"),o,r,a)+"px"}function nt(e,t,n,r,i){return new nt.prototype.init(e,t,n,r,i)}k.extend({cssHooks:{opacity:{get:function(e,t){if(t){var n=_e(e,"opacity");return""===n?"1":n}}}},cssNumber:{animationIterationCount:!0,columnCount:!0,fillOpacity:!0,flexGrow:!0,flexShrink:!0,fontWeight:!0,gridArea:!0,gridColumn:!0,gridColumnEnd:!0,gridColumnStart:!0,gridRow:!0,gridRowEnd:!0,gridRowStart:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{},style:function(e,t,n,r){if(e&&3!==e.nodeType&&8!==e.nodeType&&e.style){var i,o,a,s=V(t),u=Qe.test(t),l=e.style;if(u||(t=Ge(s)),a=k.cssHooks[t]||k.cssHooks[s],void 0===n)return a&&"get"in a&&void 0!==(i=a.get(e,!1,r))?i:l[t];"string"===(o=typeof n)&&(i=ne.exec(n))&&i[1]&&(n=le(e,t,i),o="number"),null!=n&&n==n&&("number"!==o||u||(n+=i&&i[3]||(k.cssNumber[s]?"":"px")),y.clearCloneStyle||""!==n||0!==t.indexOf("background")||(l[t]="inherit"),a&&"set"in a&&void 0===(n=a.set(e,n,r))||(u?l.setProperty(t,n):l[t]=n))}},css:function(e,t,n,r){var i,o,a,s=V(t);return Qe.test(t)||(t=Ge(s)),(a=k.cssHooks[t]||k.cssHooks[s])&&"get"in a&&(i=a.get(e,!0,n)),void 0===i&&(i=_e(e,t,r)),"normal"===i&&t in Ke&&(i=Ke[t]),""===n||n?(o=parseFloat(i),!0===n||isFinite(o)?o||0:i):i}}),k.each(["height","width"],function(e,u){k.cssHooks[u]={get:function(e,t,n){if(t)return!Ye.test(k.css(e,"display"))||e.getClientRects().length&&e.getBoundingClientRect().width?tt(e,u,n):ue(e,Je,function(){return tt(e,u,n)})},set:function(e,t,n){var r,i=Fe(e),o=!y.scrollboxSize()&&"absolute"===i.position,a=(o||n)&&"border-box"===k.css(e,"boxSizing",!1,i),s=n?et(e,u,n,a,i):0;return a&&o&&(s-=Math.ceil(e["offset"+u[0].toUpperCase()+u.slice(1)]-parseFloat(i[u])-et(e,u,"border",!1,i)-.5)),s&&(r=ne.exec(t))&&"px"!==(r[3]||"px")&&(e.style[u]=t,t=k.css(e,u)),Ze(0,t,s)}}}),k.cssHooks.marginLeft=ze(y.reliableMarginLeft,function(e,t){if(t)return(parseFloat(_e(e,"marginLeft"))||e.getBoundingClientRect().left-ue(e,{marginLeft:0},function(){return e.getBoundingClientRect().left}))+"px"}),k.each({margin:"",padding:"",border:"Width"},function(i,o){k.cssHooks[i+o]={expand:function(e){for(var t=0,n={},r="string"==typeof e?e.split(" "):[e];t<4;t++)n[i+re[t]+o]=r[t]||r[t-2]||r[0];return n}},"margin"!==i&&(k.cssHooks[i+o].set=Ze)}),k.fn.extend({css:function(e,t){return _(this,function(e,t,n){var r,i,o={},a=0;if(Array.isArray(t)){for(r=Fe(e),i=t.length;a<i;a++)o[t[a]]=k.css(e,t[a],!1,r);return o}return void 0!==n?k.style(e,t,n):k.css(e,t)},e,t,1<arguments.length)}}),((k.Tween=nt).prototype={constructor:nt,init:function(e,t,n,r,i,o){this.elem=e,this.prop=n,this.easing=i||k.easing._default,this.options=t,this.start=this.now=this.cur(),this.end=r,this.unit=o||(k.cssNumber[n]?"":"px")},cur:function(){var e=nt.propHooks[this.prop];return e&&e.get?e.get(this):nt.propHooks._default.get(this)},run:function(e){var t,n=nt.propHooks[this.prop];return this.options.duration?this.pos=t=k.easing[this.easing](e,this.options.duration*e,0,1,this.options.duration):this.pos=t=e,this.now=(this.end-this.start)*t+this.start,this.options.step&&this.options.step.call(this.elem,this.now,this),n&&n.set?n.set(this):nt.propHooks._default.set(this),this}}).init.prototype=nt.prototype,(nt.propHooks={_default:{get:function(e){var t;return 1!==e.elem.nodeType||null!=e.elem[e.prop]&&null==e.elem.style[e.prop]?e.elem[e.prop]:(t=k.css(e.elem,e.prop,""))&&"auto"!==t?t:0},set:function(e){k.fx.step[e.prop]?k.fx.step[e.prop](e):1!==e.elem.nodeType||!k.cssHooks[e.prop]&&null==e.elem.style[Ge(e.prop)]?e.elem[e.prop]=e.now:k.style(e.elem,e.prop,e.now+e.unit)}}}).scrollTop=nt.propHooks.scrollLeft={set:function(e){e.elem.nodeType&&e.elem.parentNode&&(e.elem[e.prop]=e.now)}},k.easing={linear:function(e){return e},swing:function(e){return.5-Math.cos(e*Math.PI)/2},_default:"swing"},k.fx=nt.prototype.init,k.fx.step={};var rt,it,ot,at,st=/^(?:toggle|show|hide)$/,ut=/queueHooks$/;function lt(){it&&(!1===E.hidden&&C.requestAnimationFrame?C.requestAnimationFrame(lt):C.setTimeout(lt,k.fx.interval),k.fx.tick())}function ct(){return C.setTimeout(function(){rt=void 0}),rt=Date.now()}function ft(e,t){var n,r=0,i={height:e};for(t=t?1:0;r<4;r+=2-t)i["margin"+(n=re[r])]=i["padding"+n]=e;return t&&(i.opacity=i.width=e),i}function pt(e,t,n){for(var r,i=(dt.tweeners[t]||[]).concat(dt.tweeners["*"]),o=0,a=i.length;o<a;o++)if(r=i[o].call(n,t,e))return r}function dt(o,e,t){var n,a,r=0,i=dt.prefilters.length,s=k.Deferred().always(function(){delete u.elem}),u=function(){if(a)return!1;for(var e=rt||ct(),t=Math.max(0,l.startTime+l.duration-e),n=1-(t/l.duration||0),r=0,i=l.tweens.length;r<i;r++)l.tweens[r].run(n);return s.notifyWith(o,[l,n,t]),n<1&&i?t:(i||s.notifyWith(o,[l,1,0]),s.resolveWith(o,[l]),!1)},l=s.promise({elem:o,props:k.extend({},e),opts:k.extend(!0,{specialEasing:{},easing:k.easing._default},t),originalProperties:e,originalOptions:t,startTime:rt||ct(),duration:t.duration,tweens:[],createTween:function(e,t){var n=k.Tween(o,l.opts,e,t,l.opts.specialEasing[e]||l.opts.easing);return l.tweens.push(n),n},stop:function(e){var t=0,n=e?l.tweens.length:0;if(a)return this;for(a=!0;t<n;t++)l.tweens[t].run(1);return e?(s.notifyWith(o,[l,1,0]),s.resolveWith(o,[l,e])):s.rejectWith(o,[l,e]),this}}),c=l.props;for(!function(e,t){var n,r,i,o,a;for(n in e)if(i=t[r=V(n)],o=e[n],Array.isArray(o)&&(i=o[1],o=e[n]=o[0]),n!==r&&(e[r]=o,delete e[n]),(a=k.cssHooks[r])&&"expand"in a)for(n in o=a.expand(o),delete e[r],o)n in e||(e[n]=o[n],t[n]=i);else t[r]=i}(c,l.opts.specialEasing);r<i;r++)if(n=dt.prefilters[r].call(l,o,c,l.opts))return m(n.stop)&&(k._queueHooks(l.elem,l.opts.queue).stop=n.stop.bind(n)),n;return k.map(c,pt,l),m(l.opts.start)&&l.opts.start.call(o,l),l.progress(l.opts.progress).done(l.opts.done,l.opts.complete).fail(l.opts.fail).always(l.opts.always),k.fx.timer(k.extend(u,{elem:o,anim:l,queue:l.opts.queue})),l}k.Animation=k.extend(dt,{tweeners:{"*":[function(e,t){var n=this.createTween(e,t);return le(n.elem,e,ne.exec(t),n),n}]},tweener:function(e,t){m(e)?(t=e,e=["*"]):e=e.match(R);for(var n,r=0,i=e.length;r<i;r++)n=e[r],dt.tweeners[n]=dt.tweeners[n]||[],dt.tweeners[n].unshift(t)},prefilters:[function(e,t,n){var r,i,o,a,s,u,l,c,f="width"in t||"height"in t,p=this,d={},h=e.style,g=e.nodeType&&se(e),v=Q.get(e,"fxshow");for(r in n.queue||(null==(a=k._queueHooks(e,"fx")).unqueued&&(a.unqueued=0,s=a.empty.fire,a.empty.fire=function(){a.unqueued||s()}),a.unqueued++,p.always(function(){p.always(function(){a.unqueued--,k.queue(e,"fx").length||a.empty.fire()})})),t)if(i=t[r],st.test(i)){if(delete t[r],o=o||"toggle"===i,i===(g?"hide":"show")){if("show"!==i||!v||void 0===v[r])continue;g=!0}d[r]=v&&v[r]||k.style(e,r)}if((u=!k.isEmptyObject(t))||!k.isEmptyObject(d))for(r in f&&1===e.nodeType&&(n.overflow=[h.overflow,h.overflowX,h.overflowY],null==(l=v&&v.display)&&(l=Q.get(e,"display")),"none"===(c=k.css(e,"display"))&&(l?c=l:(fe([e],!0),l=e.style.display||l,c=k.css(e,"display"),fe([e]))),("inline"===c||"inline-block"===c&&null!=l)&&"none"===k.css(e,"float")&&(u||(p.done(function(){h.display=l}),null==l&&(c=h.display,l="none"===c?"":c)),h.display="inline-block")),n.overflow&&(h.overflow="hidden",p.always(function(){h.overflow=n.overflow[0],h.overflowX=n.overflow[1],h.overflowY=n.overflow[2]})),u=!1,d)u||(v?"hidden"in v&&(g=v.hidden):v=Q.access(e,"fxshow",{display:l}),o&&(v.hidden=!g),g&&fe([e],!0),p.done(function(){for(r in g||fe([e]),Q.remove(e,"fxshow"),d)k.style(e,r,d[r])})),u=pt(g?v[r]:0,r,p),r in v||(v[r]=u.start,g&&(u.end=u.start,u.start=0))}],prefilter:function(e,t){t?dt.prefilters.unshift(e):dt.prefilters.push(e)}}),k.speed=function(e,t,n){var r=e&&"object"==typeof e?k.extend({},e):{complete:n||!n&&t||m(e)&&e,duration:e,easing:n&&t||t&&!m(t)&&t};return k.fx.off?r.duration=0:"number"!=typeof r.duration&&(r.duration in k.fx.speeds?r.duration=k.fx.speeds[r.duration]:r.duration=k.fx.speeds._default),null!=r.queue&&!0!==r.queue||(r.queue="fx"),r.old=r.complete,r.complete=function(){m(r.old)&&r.old.call(this),r.queue&&k.dequeue(this,r.queue)},r},k.fn.extend({fadeTo:function(e,t,n,r){return this.filter(se).css("opacity",0).show().end().animate({opacity:t},e,n,r)},animate:function(t,e,n,r){var i=k.isEmptyObject(t),o=k.speed(e,n,r),a=function(){var e=dt(this,k.extend({},t),o);(i||Q.get(this,"finish"))&&e.stop(!0)};return a.finish=a,i||!1===o.queue?this.each(a):this.queue(o.queue,a)},stop:function(i,e,o){var a=function(e){var t=e.stop;delete e.stop,t(o)};return"string"!=typeof i&&(o=e,e=i,i=void 0),e&&!1!==i&&this.queue(i||"fx",[]),this.each(function(){var e=!0,t=null!=i&&i+"queueHooks",n=k.timers,r=Q.get(this);if(t)r[t]&&r[t].stop&&a(r[t]);else for(t in r)r[t]&&r[t].stop&&ut.test(t)&&a(r[t]);for(t=n.length;t--;)n[t].elem!==this||null!=i&&n[t].queue!==i||(n[t].anim.stop(o),e=!1,n.splice(t,1));!e&&o||k.dequeue(this,i)})},finish:function(a){return!1!==a&&(a=a||"fx"),this.each(function(){var e,t=Q.get(this),n=t[a+"queue"],r=t[a+"queueHooks"],i=k.timers,o=n?n.length:0;for(t.finish=!0,k.queue(this,a,[]),r&&r.stop&&r.stop.call(this,!0),e=i.length;e--;)i[e].elem===this&&i[e].queue===a&&(i[e].anim.stop(!0),i.splice(e,1));for(e=0;e<o;e++)n[e]&&n[e].finish&&n[e].finish.call(this);delete t.finish})}}),k.each(["toggle","show","hide"],function(e,r){var i=k.fn[r];k.fn[r]=function(e,t,n){return null==e||"boolean"==typeof e?i.apply(this,arguments):this.animate(ft(r,!0),e,t,n)}}),k.each({slideDown:ft("show"),slideUp:ft("hide"),slideToggle:ft("toggle"),fadeIn:{opacity:"show"},fadeOut:{opacity:"hide"},fadeToggle:{opacity:"toggle"}},function(e,r){k.fn[e]=function(e,t,n){return this.animate(r,e,t,n)}}),k.timers=[],k.fx.tick=function(){var e,t=0,n=k.timers;for(rt=Date.now();t<n.length;t++)(e=n[t])()||n[t]!==e||n.splice(t--,1);n.length||k.fx.stop(),rt=void 0},k.fx.timer=function(e){k.timers.push(e),k.fx.start()},k.fx.interval=13,k.fx.start=function(){it||(it=!0,lt())},k.fx.stop=function(){it=null},k.fx.speeds={slow:600,fast:200,_default:400},k.fn.delay=function(r,e){return r=k.fx&&k.fx.speeds[r]||r,e=e||"fx",this.queue(e,function(e,t){var n=C.setTimeout(e,r);t.stop=function(){C.clearTimeout(n)}})},ot=E.createElement("input"),at=E.createElement("select").appendChild(E.createElement("option")),ot.type="checkbox",y.checkOn=""!==ot.value,y.optSelected=at.selected,(ot=E.createElement("input")).value="t",ot.type="radio",y.radioValue="t"===ot.value;var ht,gt=k.expr.attrHandle;k.fn.extend({attr:function(e,t){return _(this,k.attr,e,t,1<arguments.length)},removeAttr:function(e){return this.each(function(){k.removeAttr(this,e)})}}),k.extend({attr:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return"undefined"==typeof e.getAttribute?k.prop(e,t,n):(1===o&&k.isXMLDoc(e)||(i=k.attrHooks[t.toLowerCase()]||(k.expr.match.bool.test(t)?ht:void 0)),void 0!==n?null===n?void k.removeAttr(e,t):i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:(e.setAttribute(t,n+""),n):i&&"get"in i&&null!==(r=i.get(e,t))?r:null==(r=k.find.attr(e,t))?void 0:r)},attrHooks:{type:{set:function(e,t){if(!y.radioValue&&"radio"===t&&A(e,"input")){var n=e.value;return e.setAttribute("type",t),n&&(e.value=n),t}}}},removeAttr:function(e,t){var n,r=0,i=t&&t.match(R);if(i&&1===e.nodeType)while(n=i[r++])e.removeAttribute(n)}}),ht={set:function(e,t,n){return!1===t?k.removeAttr(e,n):e.setAttribute(n,n),n}},k.each(k.expr.match.bool.source.match(/\w+/g),function(e,t){var a=gt[t]||k.find.attr;gt[t]=function(e,t,n){var r,i,o=t.toLowerCase();return n||(i=gt[o],gt[o]=r,r=null!=a(e,t,n)?o:null,gt[o]=i),r}});var vt=/^(?:input|select|textarea|button)$/i,yt=/^(?:a|area)$/i;function mt(e){return(e.match(R)||[]).join(" ")}function xt(e){return e.getAttribute&&e.getAttribute("class")||""}function bt(e){return Array.isArray(e)?e:"string"==typeof e&&e.match(R)||[]}k.fn.extend({prop:function(e,t){return _(this,k.prop,e,t,1<arguments.length)},removeProp:function(e){return this.each(function(){delete this[k.propFix[e]||e]})}}),k.extend({prop:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return 1===o&&k.isXMLDoc(e)||(t=k.propFix[t]||t,i=k.propHooks[t]),void 0!==n?i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:e[t]=n:i&&"get"in i&&null!==(r=i.get(e,t))?r:e[t]},propHooks:{tabIndex:{get:function(e){var t=k.find.attr(e,"tabindex");return t?parseInt(t,10):vt.test(e.nodeName)||yt.test(e.nodeName)&&e.href?0:-1}}},propFix:{"for":"htmlFor","class":"className"}}),y.optSelected||(k.propHooks.selected={get:function(e){var t=e.parentNode;return t&&t.parentNode&&t.parentNode.selectedIndex,null},set:function(e){var t=e.parentNode;t&&(t.selectedIndex,t.parentNode&&t.parentNode.selectedIndex)}}),k.each(["tabIndex","readOnly","maxLength","cellSpacing","cellPadding","rowSpan","colSpan","useMap","frameBorder","contentEditable"],function(){k.propFix[this.toLowerCase()]=this}),k.fn.extend({addClass:function(t){var e,n,r,i,o,a,s,u=0;if(m(t))return this.each(function(e){k(this).addClass(t.call(this,e,xt(this)))});if((e=bt(t)).length)while(n=this[u++])if(i=xt(n),r=1===n.nodeType&&" "+mt(i)+" "){a=0;while(o=e[a++])r.indexOf(" "+o+" ")<0&&(r+=o+" ");i!==(s=mt(r))&&n.setAttribute("class",s)}return this},removeClass:function(t){var e,n,r,i,o,a,s,u=0;if(m(t))return this.each(function(e){k(this).removeClass(t.call(this,e,xt(this)))});if(!arguments.length)return this.attr("class","");if((e=bt(t)).length)while(n=this[u++])if(i=xt(n),r=1===n.nodeType&&" "+mt(i)+" "){a=0;while(o=e[a++])while(-1<r.indexOf(" "+o+" "))r=r.replace(" "+o+" "," ");i!==(s=mt(r))&&n.setAttribute("class",s)}return this},toggleClass:function(i,t){var o=typeof i,a="string"===o||Array.isArray(i);return"boolean"==typeof t&&a?t?this.addClass(i):this.removeClass(i):m(i)?this.each(function(e){k(this).toggleClass(i.call(this,e,xt(this),t),t)}):this.each(function(){var e,t,n,r;if(a){t=0,n=k(this),r=bt(i);while(e=r[t++])n.hasClass(e)?n.removeClass(e):n.addClass(e)}else void 0!==i&&"boolean"!==o||((e=xt(this))&&Q.set(this,"__className__",e),this.setAttribute&&this.setAttribute("class",e||!1===i?"":Q.get(this,"__className__")||""))})},hasClass:function(e){var t,n,r=0;t=" "+e+" ";while(n=this[r++])if(1===n.nodeType&&-1<(" "+mt(xt(n))+" ").indexOf(t))return!0;return!1}});var wt=/\r/g;k.fn.extend({val:function(n){var r,e,i,t=this[0];return arguments.length?(i=m(n),this.each(function(e){var t;1===this.nodeType&&(null==(t=i?n.call(this,e,k(this).val()):n)?t="":"number"==typeof t?t+="":Array.isArray(t)&&(t=k.map(t,function(e){return null==e?"":e+""})),(r=k.valHooks[this.type]||k.valHooks[this.nodeName.toLowerCase()])&&"set"in r&&void 0!==r.set(this,t,"value")||(this.value=t))})):t?(r=k.valHooks[t.type]||k.valHooks[t.nodeName.toLowerCase()])&&"get"in r&&void 0!==(e=r.get(t,"value"))?e:"string"==typeof(e=t.value)?e.replace(wt,""):null==e?"":e:void 0}}),k.extend({valHooks:{option:{get:function(e){var t=k.find.attr(e,"value");return null!=t?t:mt(k.text(e))}},select:{get:function(e){var t,n,r,i=e.options,o=e.selectedIndex,a="select-one"===e.type,s=a?null:[],u=a?o+1:i.length;for(r=o<0?u:a?o:0;r<u;r++)if(((n=i[r]).selected||r===o)&&!n.disabled&&(!n.parentNode.disabled||!A(n.parentNode,"optgroup"))){if(t=k(n).val(),a)return t;s.push(t)}return s},set:function(e,t){var n,r,i=e.options,o=k.makeArray(t),a=i.length;while(a--)((r=i[a]).selected=-1<k.inArray(k.valHooks.option.get(r),o))&&(n=!0);return n||(e.selectedIndex=-1),o}}}}),k.each(["radio","checkbox"],function(){k.valHooks[this]={set:function(e,t){if(Array.isArray(t))return e.checked=-1<k.inArray(k(e).val(),t)}},y.checkOn||(k.valHooks[this].get=function(e){return null===e.getAttribute("value")?"on":e.value})}),y.focusin="onfocusin"in C;var Tt=/^(?:focusinfocus|focusoutblur)$/,Ct=function(e){e.stopPropagation()};k.extend(k.event,{trigger:function(e,t,n,r){var i,o,a,s,u,l,c,f,p=[n||E],d=v.call(e,"type")?e.type:e,h=v.call(e,"namespace")?e.namespace.split("."):[];if(o=f=a=n=n||E,3!==n.nodeType&&8!==n.nodeType&&!Tt.test(d+k.event.triggered)&&(-1<d.indexOf(".")&&(d=(h=d.split(".")).shift(),h.sort()),u=d.indexOf(":")<0&&"on"+d,(e=e[k.expando]?e:new k.Event(d,"object"==typeof e&&e)).isTrigger=r?2:3,e.namespace=h.join("."),e.rnamespace=e.namespace?new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,e.result=void 0,e.target||(e.target=n),t=null==t?[e]:k.makeArray(t,[e]),c=k.event.special[d]||{},r||!c.trigger||!1!==c.trigger.apply(n,t))){if(!r&&!c.noBubble&&!x(n)){for(s=c.delegateType||d,Tt.test(s+d)||(o=o.parentNode);o;o=o.parentNode)p.push(o),a=o;a===(n.ownerDocument||E)&&p.push(a.defaultView||a.parentWindow||C)}i=0;while((o=p[i++])&&!e.isPropagationStopped())f=o,e.type=1<i?s:c.bindType||d,(l=(Q.get(o,"events")||{})[e.type]&&Q.get(o,"handle"))&&l.apply(o,t),(l=u&&o[u])&&l.apply&&G(o)&&(e.result=l.apply(o,t),!1===e.result&&e.preventDefault());return e.type=d,r||e.isDefaultPrevented()||c._default&&!1!==c._default.apply(p.pop(),t)||!G(n)||u&&m(n[d])&&!x(n)&&((a=n[u])&&(n[u]=null),k.event.triggered=d,e.isPropagationStopped()&&f.addEventListener(d,Ct),n[d](),e.isPropagationStopped()&&f.removeEventListener(d,Ct),k.event.triggered=void 0,a&&(n[u]=a)),e.result}},simulate:function(e,t,n){var r=k.extend(new k.Event,n,{type:e,isSimulated:!0});k.event.trigger(r,null,t)}}),k.fn.extend({trigger:function(e,t){return this.each(function(){k.event.trigger(e,t,this)})},triggerHandler:function(e,t){var n=this[0];if(n)return k.event.trigger(e,t,n,!0)}}),y.focusin||k.each({focus:"focusin",blur:"focusout"},function(n,r){var i=function(e){k.event.simulate(r,e.target,k.event.fix(e))};k.event.special[r]={setup:function(){var e=this.ownerDocument||this,t=Q.access(e,r);t||e.addEventListener(n,i,!0),Q.access(e,r,(t||0)+1)},teardown:function(){var e=this.ownerDocument||this,t=Q.access(e,r)-1;t?Q.access(e,r,t):(e.removeEventListener(n,i,!0),Q.remove(e,r))}}});var Et=C.location,kt=Date.now(),St=/\?/;k.parseXML=function(e){var t;if(!e||"string"!=typeof e)return null;try{t=(new C.DOMParser).parseFromString(e,"text/xml")}catch(e){t=void 0}return t&&!t.getElementsByTagName("parsererror").length||k.error("Invalid XML: "+e),t};var Nt=/\[\]$/,At=/\r?\n/g,Dt=/^(?:submit|button|image|reset|file)$/i,jt=/^(?:input|select|textarea|keygen)/i;function qt(n,e,r,i){var t;if(Array.isArray(e))k.each(e,function(e,t){r||Nt.test(n)?i(n,t):qt(n+"["+("object"==typeof t&&null!=t?e:"")+"]",t,r,i)});else if(r||"object"!==w(e))i(n,e);else for(t in e)qt(n+"["+t+"]",e[t],r,i)}k.param=function(e,t){var n,r=[],i=function(e,t){var n=m(t)?t():t;r[r.length]=encodeURIComponent(e)+"="+encodeURIComponent(null==n?"":n)};if(null==e)return"";if(Array.isArray(e)||e.jquery&&!k.isPlainObject(e))k.each(e,function(){i(this.name,this.value)});else for(n in e)qt(n,e[n],t,i);return r.join("&")},k.fn.extend({serialize:function(){return k.param(this.serializeArray())},serializeArray:function(){return this.map(function(){var e=k.prop(this,"elements");return e?k.makeArray(e):this}).filter(function(){var e=this.type;return this.name&&!k(this).is(":disabled")&&jt.test(this.nodeName)&&!Dt.test(e)&&(this.checked||!pe.test(e))}).map(function(e,t){var n=k(this).val();return null==n?null:Array.isArray(n)?k.map(n,function(e){return{name:t.name,value:e.replace(At,"\r\n")}}):{name:t.name,value:n.replace(At,"\r\n")}}).get()}});var Lt=/%20/g,Ht=/#.*$/,Ot=/([?&])_=[^&]*/,Pt=/^(.*?):[ \t]*([^\r\n]*)$/gm,Rt=/^(?:GET|HEAD)$/,Mt=/^\/\//,It={},Wt={},$t="*/".concat("*"),Ft=E.createElement("a");function Bt(o){return function(e,t){"string"!=typeof e&&(t=e,e="*");var n,r=0,i=e.toLowerCase().match(R)||[];if(m(t))while(n=i[r++])"+"===n[0]?(n=n.slice(1)||"*",(o[n]=o[n]||[]).unshift(t)):(o[n]=o[n]||[]).push(t)}}function _t(t,i,o,a){var s={},u=t===Wt;function l(e){var r;return s[e]=!0,k.each(t[e]||[],function(e,t){var n=t(i,o,a);return"string"!=typeof n||u||s[n]?u?!(r=n):void 0:(i.dataTypes.unshift(n),l(n),!1)}),r}return l(i.dataTypes[0])||!s["*"]&&l("*")}function zt(e,t){var n,r,i=k.ajaxSettings.flatOptions||{};for(n in t)void 0!==t[n]&&((i[n]?e:r||(r={}))[n]=t[n]);return r&&k.extend(!0,e,r),e}Ft.href=Et.href,k.extend({active:0,lastModified:{},etag:{},ajaxSettings:{url:Et.href,type:"GET",isLocal:/^(?:about|app|app-storage|.+-extension|file|res|widget):$/.test(Et.protocol),global:!0,processData:!0,async:!0,contentType:"application/x-www-form-urlencoded; charset=UTF-8",accepts:{"*":$t,text:"text/plain",html:"text/html",xml:"application/xml, text/xml",json:"application/json, text/javascript"},contents:{xml:/\bxml\b/,html:/\bhtml/,json:/\bjson\b/},responseFields:{xml:"responseXML",text:"responseText",json:"responseJSON"},converters:{"* text":String,"text html":!0,"text json":JSON.parse,"text xml":k.parseXML},flatOptions:{url:!0,context:!0}},ajaxSetup:function(e,t){return t?zt(zt(e,k.ajaxSettings),t):zt(k.ajaxSettings,e)},ajaxPrefilter:Bt(It),ajaxTransport:Bt(Wt),ajax:function(e,t){"object"==typeof e&&(t=e,e=void 0),t=t||{};var c,f,p,n,d,r,h,g,i,o,v=k.ajaxSetup({},t),y=v.context||v,m=v.context&&(y.nodeType||y.jquery)?k(y):k.event,x=k.Deferred(),b=k.Callbacks("once memory"),w=v.statusCode||{},a={},s={},u="canceled",T={readyState:0,getResponseHeader:function(e){var t;if(h){if(!n){n={};while(t=Pt.exec(p))n[t[1].toLowerCase()+" "]=(n[t[1].toLowerCase()+" "]||[]).concat(t[2])}t=n[e.toLowerCase()+" "]}return null==t?null:t.join(", ")},getAllResponseHeaders:function(){return h?p:null},setRequestHeader:function(e,t){return null==h&&(e=s[e.toLowerCase()]=s[e.toLowerCase()]||e,a[e]=t),this},overrideMimeType:function(e){return null==h&&(v.mimeType=e),this},statusCode:function(e){var t;if(e)if(h)T.always(e[T.status]);else for(t in e)w[t]=[w[t],e[t]];return this},abort:function(e){var t=e||u;return c&&c.abort(t),l(0,t),this}};if(x.promise(T),v.url=((e||v.url||Et.href)+"").replace(Mt,Et.protocol+"//"),v.type=t.method||t.type||v.method||v.type,v.dataTypes=(v.dataType||"*").toLowerCase().match(R)||[""],null==v.crossDomain){r=E.createElement("a");try{r.href=v.url,r.href=r.href,v.crossDomain=Ft.protocol+"//"+Ft.host!=r.protocol+"//"+r.host}catch(e){v.crossDomain=!0}}if(v.data&&v.processData&&"string"!=typeof v.data&&(v.data=k.param(v.data,v.traditional)),_t(It,v,t,T),h)return T;for(i in(g=k.event&&v.global)&&0==k.active++&&k.event.trigger("ajaxStart"),v.type=v.type.toUpperCase(),v.hasContent=!Rt.test(v.type),f=v.url.replace(Ht,""),v.hasContent?v.data&&v.processData&&0===(v.contentType||"").indexOf("application/x-www-form-urlencoded")&&(v.data=v.data.replace(Lt,"+")):(o=v.url.slice(f.length),v.data&&(v.processData||"string"==typeof v.data)&&(f+=(St.test(f)?"&":"?")+v.data,delete v.data),!1===v.cache&&(f=f.replace(Ot,"$1"),o=(St.test(f)?"&":"?")+"_="+kt+++o),v.url=f+o),v.ifModified&&(k.lastModified[f]&&T.setRequestHeader("If-Modified-Since",k.lastModified[f]),k.etag[f]&&T.setRequestHeader("If-None-Match",k.etag[f])),(v.data&&v.hasContent&&!1!==v.contentType||t.contentType)&&T.setRequestHeader("Content-Type",v.contentType),T.setRequestHeader("Accept",v.dataTypes[0]&&v.accepts[v.dataTypes[0]]?v.accepts[v.dataTypes[0]]+("*"!==v.dataTypes[0]?", "+$t+"; q=0.01":""):v.accepts["*"]),v.headers)T.setRequestHeader(i,v.headers[i]);if(v.beforeSend&&(!1===v.beforeSend.call(y,T,v)||h))return T.abort();if(u="abort",b.add(v.complete),T.done(v.success),T.fail(v.error),c=_t(Wt,v,t,T)){if(T.readyState=1,g&&m.trigger("ajaxSend",[T,v]),h)return T;v.async&&0<v.timeout&&(d=C.setTimeout(function(){T.abort("timeout")},v.timeout));try{h=!1,c.send(a,l)}catch(e){if(h)throw e;l(-1,e)}}else l(-1,"No Transport");function l(e,t,n,r){var i,o,a,s,u,l=t;h||(h=!0,d&&C.clearTimeout(d),c=void 0,p=r||"",T.readyState=0<e?4:0,i=200<=e&&e<300||304===e,n&&(s=function(e,t,n){var r,i,o,a,s=e.contents,u=e.dataTypes;while("*"===u[0])u.shift(),void 0===r&&(r=e.mimeType||t.getResponseHeader("Content-Type"));if(r)for(i in s)if(s[i]&&s[i].test(r)){u.unshift(i);break}if(u[0]in n)o=u[0];else{for(i in n){if(!u[0]||e.converters[i+" "+u[0]]){o=i;break}a||(a=i)}o=o||a}if(o)return o!==u[0]&&u.unshift(o),n[o]}(v,T,n)),s=function(e,t,n,r){var i,o,a,s,u,l={},c=e.dataTypes.slice();if(c[1])for(a in e.converters)l[a.toLowerCase()]=e.converters[a];o=c.shift();while(o)if(e.responseFields[o]&&(n[e.responseFields[o]]=t),!u&&r&&e.dataFilter&&(t=e.dataFilter(t,e.dataType)),u=o,o=c.shift())if("*"===o)o=u;else if("*"!==u&&u!==o){if(!(a=l[u+" "+o]||l["* "+o]))for(i in l)if((s=i.split(" "))[1]===o&&(a=l[u+" "+s[0]]||l["* "+s[0]])){!0===a?a=l[i]:!0!==l[i]&&(o=s[0],c.unshift(s[1]));break}if(!0!==a)if(a&&e["throws"])t=a(t);else try{t=a(t)}catch(e){return{state:"parsererror",error:a?e:"No conversion from "+u+" to "+o}}}return{state:"success",data:t}}(v,s,T,i),i?(v.ifModified&&((u=T.getResponseHeader("Last-Modified"))&&(k.lastModified[f]=u),(u=T.getResponseHeader("etag"))&&(k.etag[f]=u)),204===e||"HEAD"===v.type?l="nocontent":304===e?l="notmodified":(l=s.state,o=s.data,i=!(a=s.error))):(a=l,!e&&l||(l="error",e<0&&(e=0))),T.status=e,T.statusText=(t||l)+"",i?x.resolveWith(y,[o,l,T]):x.rejectWith(y,[T,l,a]),T.statusCode(w),w=void 0,g&&m.trigger(i?"ajaxSuccess":"ajaxError",[T,v,i?o:a]),b.fireWith(y,[T,l]),g&&(m.trigger("ajaxComplete",[T,v]),--k.active||k.event.trigger("ajaxStop")))}return T},getJSON:function(e,t,n){return k.get(e,t,n,"json")},getScript:function(e,t){return k.get(e,void 0,t,"script")}}),k.each(["get","post"],function(e,i){k[i]=function(e,t,n,r){return m(t)&&(r=r||n,n=t,t=void 0),k.ajax(k.extend({url:e,type:i,dataType:r,data:t,success:n},k.isPlainObject(e)&&e))}}),k._evalUrl=function(e,t){return k.ajax({url:e,type:"GET",dataType:"script",cache:!0,async:!1,global:!1,converters:{"text script":function(){}},dataFilter:function(e){k.globalEval(e,t)}})},k.fn.extend({wrapAll:function(e){var t;return this[0]&&(m(e)&&(e=e.call(this[0])),t=k(e,this[0].ownerDocument).eq(0).clone(!0),this[0].parentNode&&t.insertBefore(this[0]),t.map(function(){var e=this;while(e.firstElementChild)e=e.firstElementChild;return e}).append(this)),this},wrapInner:function(n){return m(n)?this.each(function(e){k(this).wrapInner(n.call(this,e))}):this.each(function(){var e=k(this),t=e.contents();t.length?t.wrapAll(n):e.append(n)})},wrap:function(t){var n=m(t);return this.each(function(e){k(this).wrapAll(n?t.call(this,e):t)})},unwrap:function(e){return this.parent(e).not("body").each(function(){k(this).replaceWith(this.childNodes)}),this}}),k.expr.pseudos.hidden=function(e){return!k.expr.pseudos.visible(e)},k.expr.pseudos.visible=function(e){return!!(e.offsetWidth||e.offsetHeight||e.getClientRects().length)},k.ajaxSettings.xhr=function(){try{return new C.XMLHttpRequest}catch(e){}};var Ut={0:200,1223:204},Xt=k.ajaxSettings.xhr();y.cors=!!Xt&&"withCredentials"in Xt,y.ajax=Xt=!!Xt,k.ajaxTransport(function(i){var o,a;if(y.cors||Xt&&!i.crossDomain)return{send:function(e,t){var n,r=i.xhr();if(r.open(i.type,i.url,i.async,i.username,i.password),i.xhrFields)for(n in i.xhrFields)r[n]=i.xhrFields[n];for(n in i.mimeType&&r.overrideMimeType&&r.overrideMimeType(i.mimeType),i.crossDomain||e["X-Requested-With"]||(e["X-Requested-With"]="XMLHttpRequest"),e)r.setRequestHeader(n,e[n]);o=function(e){return function(){o&&(o=a=r.onload=r.onerror=r.onabort=r.ontimeout=r.onreadystatechange=null,"abort"===e?r.abort():"error"===e?"number"!=typeof r.status?t(0,"error"):t(r.status,r.statusText):t(Ut[r.status]||r.status,r.statusText,"text"!==(r.responseType||"text")||"string"!=typeof r.responseText?{binary:r.response}:{text:r.responseText},r.getAllResponseHeaders()))}},r.onload=o(),a=r.onerror=r.ontimeout=o("error"),void 0!==r.onabort?r.onabort=a:r.onreadystatechange=function(){4===r.readyState&&C.setTimeout(function(){o&&a()})},o=o("abort");try{r.send(i.hasContent&&i.data||null)}catch(e){if(o)throw e}},abort:function(){o&&o()}}}),k.ajaxPrefilter(function(e){e.crossDomain&&(e.contents.script=!1)}),k.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},contents:{script:/\b(?:java|ecma)script\b/},converters:{"text script":function(e){return k.globalEval(e),e}}}),k.ajaxPrefilter("script",function(e){void 0===e.cache&&(e.cache=!1),e.crossDomain&&(e.type="GET")}),k.ajaxTransport("script",function(n){var r,i;if(n.crossDomain||n.scriptAttrs)return{send:function(e,t){r=k("<script>").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),E.head.appendChild(r[0])},abort:function(){i&&i()}}});var Vt,Gt=[],Yt=/(=)\?(?=&|$)|\?\?/;k.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=Gt.pop()||k.expando+"_"+kt++;return this[e]=!0,e}}),k.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Yt.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Yt.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=m(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Yt,"$1"+r):!1!==e.jsonp&&(e.url+=(St.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||k.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=C[r],C[r]=function(){o=arguments},n.always(function(){void 0===i?k(C).removeProp(r):C[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,Gt.push(r)),o&&m(i)&&i(o[0]),o=i=void 0}),"script"}),y.createHTMLDocument=((Vt=E.implementation.createHTMLDocument("").body).innerHTML="<form></form><form></form>",2===Vt.childNodes.length),k.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(y.createHTMLDocument?((r=(t=E.implementation.createHTMLDocument("")).createElement("base")).href=E.location.href,t.head.appendChild(r)):t=E),o=!n&&[],(i=D.exec(e))?[t.createElement(i[1])]:(i=we([e],t,o),o&&o.length&&k(o).remove(),k.merge([],i.childNodes)));var r,i,o},k.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1<s&&(r=mt(e.slice(s)),e=e.slice(0,s)),m(t)?(n=t,t=void 0):t&&"object"==typeof t&&(i="POST"),0<a.length&&k.ajax({url:e,type:i||"GET",dataType:"html",data:t}).done(function(e){o=arguments,a.html(r?k("<div>").append(k.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},k.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){k.fn[t]=function(e){return this.on(t,e)}}),k.expr.pseudos.animated=function(t){return k.grep(k.timers,function(e){return t===e.elem}).length},k.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=k.css(e,"position"),c=k(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=k.css(e,"top"),u=k.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),m(t)&&(t=t.call(e,n,k.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):c.css(f)}},k.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){k.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===k.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===k.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=k(e).offset()).top+=k.css(e,"borderTopWidth",!0),i.left+=k.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-k.css(r,"marginTop",!0),left:t.left-i.left-k.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===k.css(e,"position"))e=e.offsetParent;return e||ie})}}),k.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;k.fn[t]=function(e){return _(this,function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),k.each(["top","left"],function(e,n){k.cssHooks[n]=ze(y.pixelPosition,function(e,t){if(t)return t=_e(e,n),$e.test(t)?k(e).position()[n]+"px":t})}),k.each({Height:"height",Width:"width"},function(a,s){k.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){k.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return _(this,function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?k.css(e,t,i):k.style(e,t,n,i)},s,n?e:void 0,n)}})}),k.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){k.fn[n]=function(e,t){return 0<arguments.length?this.on(n,null,e,t):this.trigger(n)}}),k.fn.extend({hover:function(e,t){return this.mouseenter(e).mouseleave(t||e)}}),k.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)}}),k.proxy=function(e,t){var n,r,i;if("string"==typeof t&&(n=e[t],t=e,e=n),m(e))return r=s.call(arguments,2),(i=function(){return e.apply(t||this,r.concat(s.call(arguments)))}).guid=e.guid=e.guid||k.guid++,i},k.holdReady=function(e){e?k.readyWait++:k.ready(!0)},k.isArray=Array.isArray,k.parseJSON=JSON.parse,k.nodeName=A,k.isFunction=m,k.isWindow=x,k.camelCase=V,k.type=w,k.now=Date.now,k.isNumeric=function(e){var t=k.type(e);return("number"===t||"string"===t)&&!isNaN(e-parseFloat(e))},"function"==typeof define&&define.amd&&define("jquery",[],function(){return k});var Qt=C.jQuery,Jt=C.$;return k.noConflict=function(e){return C.$===k&&(C.$=Jt),e&&C.jQuery===k&&(C.jQuery=Qt),k},e||(C.jQuery=C.$=k),k});
\ No newline at end of file
diff --git a/docs/js/vendor/jquery-3.5.1.min.js b/docs/js/vendor/jquery-3.5.1.min.js
new file mode 100644
index 0000000000000..b0614034ad3a9
--- /dev/null
+++ b/docs/js/vendor/jquery-3.5.1.min.js
@@ -0,0 +1,2 @@
+/*! jQuery v3.5.1 | (c) JS Foundation and other contributors | jquery.org/license */
+!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(C,e){"use strict";var t=[],r=Object.getPrototypeOf,s=t.slice,g=t.flat?function(e){return t.flat.call(e)}:function(e){return t.concat.apply([],e)},u=t.push,i=t.indexOf,n={},o=n.toString,v=n.hasOwnProperty,a=v.toString,l=a.call(Object),y={},m=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType},x=function(e){return null!=e&&e===e.window},E=C.document,c={type:!0,src:!0,nonce:!0,noModule:!0};function b(e,t,n){var r,i,o=(n=n||E).createElement("script");if(o.text=e,t)for(r in c)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function w(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[o.call(e)]||"object":typeof e}var f="3.5.1",S=function(e,t){return new S.fn.init(e,t)};function p(e){var t=!!e&&"length"in e&&e.length,n=w(e);return!m(e)&&!x(e)&&("array"===n||0===t||"number"==typeof t&&0<t&&t-1 in e)}S.fn=S.prototype={jquery:f,constructor:S,length:0,toArray:function(){return s.call(this)},get:function(e){return null==e?s.call(this):e<0?this[e+this.length]:this[e]},pushStack:function(e){var t=S.merge(this.constructor(),e);return t.prevObject=this,t},each:function(e){return S.each(this,e)},map:function(n){return this.pushStack(S.map(this,function(e,t){return n.call(e,t,e)}))},slice:function(){return this.pushStack(s.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},even:function(){return this.pushStack(S.grep(this,function(e,t){return(t+1)%2}))},odd:function(){return this.pushStack(S.grep(this,function(e,t){return t%2}))},eq:function(e){var t=this.length,n=+e+(e<0?t:0);return this.pushStack(0<=n&&n<t?[this[n]]:[])},end:function(){return this.prevObject||this.constructor()},push:u,sort:t.sort,splice:t.splice},S.extend=S.fn.extend=function(){var e,t,n,r,i,o,a=arguments[0]||{},s=1,u=arguments.length,l=!1;for("boolean"==typeof a&&(l=a,a=arguments[s]||{},s++),"object"==typeof a||m(a)||(a={}),s===u&&(a=this,s--);s<u;s++)if(null!=(e=arguments[s]))for(t in e)r=e[t],"__proto__"!==t&&a!==r&&(l&&r&&(S.isPlainObject(r)||(i=Array.isArray(r)))?(n=a[t],o=i&&!Array.isArray(n)?[]:i||S.isPlainObject(n)?n:{},i=!1,a[t]=S.extend(l,o,r)):void 0!==r&&(a[t]=r));return a},S.extend({expando:"jQuery"+(f+Math.random()).replace(/\D/g,""),isReady:!0,error:function(e){throw new Error(e)},noop:function(){},isPlainObject:function(e){var t,n;return!(!e||"[object Object]"!==o.call(e))&&(!(t=r(e))||"function"==typeof(n=v.call(t,"constructor")&&t.constructor)&&a.call(n)===l)},isEmptyObject:function(e){var t;for(t in e)return!1;return!0},globalEval:function(e,t,n){b(e,{nonce:t&&t.nonce},n)},each:function(e,t){var n,r=0;if(p(e)){for(n=e.length;r<n;r++)if(!1===t.call(e[r],r,e[r]))break}else for(r in e)if(!1===t.call(e[r],r,e[r]))break;return e},makeArray:function(e,t){var n=t||[];return null!=e&&(p(Object(e))?S.merge(n,"string"==typeof e?[e]:e):u.call(n,e)),n},inArray:function(e,t,n){return null==t?-1:i.call(t,e,n)},merge:function(e,t){for(var n=+t.length,r=0,i=e.length;r<n;r++)e[i++]=t[r];return e.length=i,e},grep:function(e,t,n){for(var r=[],i=0,o=e.length,a=!n;i<o;i++)!t(e[i],i)!==a&&r.push(e[i]);return r},map:function(e,t,n){var r,i,o=0,a=[];if(p(e))for(r=e.length;o<r;o++)null!=(i=t(e[o],o,n))&&a.push(i);else for(o in e)null!=(i=t(e[o],o,n))&&a.push(i);return g(a)},guid:1,support:y}),"function"==typeof Symbol&&(S.fn[Symbol.iterator]=t[Symbol.iterator]),S.each("Boolean Number String Function Array Date RegExp Object Error Symbol".split(" "),function(e,t){n["[object "+t+"]"]=t.toLowerCase()});var d=function(n){var e,d,b,o,i,h,f,g,w,u,l,T,C,a,E,v,s,c,y,S="sizzle"+1*new Date,p=n.document,k=0,r=0,m=ue(),x=ue(),A=ue(),N=ue(),D=function(e,t){return e===t&&(l=!0),0},j={}.hasOwnProperty,t=[],q=t.pop,L=t.push,H=t.push,O=t.slice,P=function(e,t){for(var n=0,r=e.length;n<r;n++)if(e[n]===t)return n;return-1},R="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",M="[\\x20\\t\\r\\n\\f]",I="(?:\\\\[\\da-fA-F]{1,6}"+M+"?|\\\\[^\\r\\n\\f]|[\\w-]|[^\0-\\x7f])+",W="\\["+M+"*("+I+")(?:"+M+"*([*^$|!~]?=)"+M+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+I+"))|)"+M+"*\\]",F=":("+I+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+W+")*)|.*)\\)|)",B=new RegExp(M+"+","g"),$=new RegExp("^"+M+"+|((?:^|[^\\\\])(?:\\\\.)*)"+M+"+$","g"),_=new RegExp("^"+M+"*,"+M+"*"),z=new RegExp("^"+M+"*([>+~]|"+M+")"+M+"*"),U=new RegExp(M+"|>"),X=new RegExp(F),V=new RegExp("^"+I+"$"),G={ID:new RegExp("^#("+I+")"),CLASS:new RegExp("^\\.("+I+")"),TAG:new RegExp("^("+I+"|[*])"),ATTR:new RegExp("^"+W),PSEUDO:new RegExp("^"+F),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+R+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+M+"*((?:-\\d)?\\d*)"+M+"*\\)|)(?=[^-]|$)","i")},Y=/HTML$/i,Q=/^(?:input|select|textarea|button)$/i,J=/^h\d$/i,K=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ee=/[+~]/,te=new RegExp("\\\\[\\da-fA-F]{1,6}"+M+"?|\\\\([^\\r\\n\\f])","g"),ne=function(e,t){var n="0x"+e.slice(1)-65536;return t||(n<0?String.fromCharCode(n+65536):String.fromCharCode(n>>10|55296,1023&n|56320))},re=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ie=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},oe=function(){T()},ae=be(function(e){return!0===e.disabled&&"fieldset"===e.nodeName.toLowerCase()},{dir:"parentNode",next:"legend"});try{H.apply(t=O.call(p.childNodes),p.childNodes),t[p.childNodes.length].nodeType}catch(e){H={apply:t.length?function(e,t){L.apply(e,O.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function se(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,p=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==p&&9!==p&&11!==p)return n;if(!r&&(T(e),e=e||C,E)){if(11!==p&&(u=Z.exec(t)))if(i=u[1]){if(9===p){if(!(a=e.getElementById(i)))return n;if(a.id===i)return n.push(a),n}else if(f&&(a=f.getElementById(i))&&y(e,a)&&a.id===i)return n.push(a),n}else{if(u[2])return H.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&d.getElementsByClassName&&e.getElementsByClassName)return H.apply(n,e.getElementsByClassName(i)),n}if(d.qsa&&!N[t+" "]&&(!v||!v.test(t))&&(1!==p||"object"!==e.nodeName.toLowerCase())){if(c=t,f=e,1===p&&(U.test(t)||z.test(t))){(f=ee.test(t)&&ye(e.parentNode)||e)===e&&d.scope||((s=e.getAttribute("id"))?s=s.replace(re,ie):e.setAttribute("id",s=S)),o=(l=h(t)).length;while(o--)l[o]=(s?"#"+s:":scope")+" "+xe(l[o]);c=l.join(",")}try{return H.apply(n,f.querySelectorAll(c)),n}catch(e){N(t,!0)}finally{s===S&&e.removeAttribute("id")}}}return g(t.replace($,"$1"),e,n,r)}function ue(){var r=[];return function e(t,n){return r.push(t+" ")>b.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function le(e){return e[S]=!0,e}function ce(e){var t=C.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){var n=e.split("|"),r=n.length;while(r--)b.attrHandle[n[r]]=t}function pe(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function de(t){return function(e){return"input"===e.nodeName.toLowerCase()&&e.type===t}}function he(n){return function(e){var t=e.nodeName.toLowerCase();return("input"===t||"button"===t)&&e.type===n}}function ge(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&ae(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function ve(a){return le(function(o){return o=+o,le(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function ye(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}for(e in d=se.support={},i=se.isXML=function(e){var t=e.namespaceURI,n=(e.ownerDocument||e).documentElement;return!Y.test(t||n&&n.nodeName||"HTML")},T=se.setDocument=function(e){var t,n,r=e?e.ownerDocument||e:p;return r!=C&&9===r.nodeType&&r.documentElement&&(a=(C=r).documentElement,E=!i(C),p!=C&&(n=C.defaultView)&&n.top!==n&&(n.addEventListener?n.addEventListener("unload",oe,!1):n.attachEvent&&n.attachEvent("onunload",oe)),d.scope=ce(function(e){return a.appendChild(e).appendChild(C.createElement("div")),"undefined"!=typeof e.querySelectorAll&&!e.querySelectorAll(":scope fieldset div").length}),d.attributes=ce(function(e){return e.className="i",!e.getAttribute("className")}),d.getElementsByTagName=ce(function(e){return e.appendChild(C.createComment("")),!e.getElementsByTagName("*").length}),d.getElementsByClassName=K.test(C.getElementsByClassName),d.getById=ce(function(e){return a.appendChild(e).id=S,!C.getElementsByName||!C.getElementsByName(S).length}),d.getById?(b.filter.ID=function(e){var t=e.replace(te,ne);return function(e){return e.getAttribute("id")===t}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n=t.getElementById(e);return n?[n]:[]}}):(b.filter.ID=function(e){var n=e.replace(te,ne);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),b.find.TAG=d.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):d.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},b.find.CLASS=d.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&E)return t.getElementsByClassName(e)},s=[],v=[],(d.qsa=K.test(C.querySelectorAll))&&(ce(function(e){var t;a.appendChild(e).innerHTML="<a id='"+S+"'></a><select id='"+S+"-\r\\' msallowcapture=''><option selected=''></option></select>",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+M+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+M+"*(?:value|"+R+")"),e.querySelectorAll("[id~="+S+"-]").length||v.push("~="),(t=C.createElement("input")).setAttribute("name",""),e.appendChild(t),e.querySelectorAll("[name='']").length||v.push("\\["+M+"*name"+M+"*="+M+"*(?:''|\"\")"),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+S+"+*").length||v.push(".#.+[+~]"),e.querySelectorAll("\\\f"),v.push("[\\r\\n\\f]")}),ce(function(e){e.innerHTML="<a href='' disabled='disabled'></a><select disabled='disabled'><option/></select>";var t=C.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+M+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")})),(d.matchesSelector=K.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce(function(e){d.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",F)}),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=K.test(a.compareDocumentPosition),y=t||K.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},D=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)==(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!d.sortDetached&&t.compareDocumentPosition(e)===n?e==C||e.ownerDocument==p&&y(p,e)?-1:t==C||t.ownerDocument==p&&y(p,t)?1:u?P(u,e)-P(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e==C?-1:t==C?1:i?-1:o?1:u?P(u,e)-P(u,t):0;if(i===o)return pe(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?pe(a[r],s[r]):a[r]==p?-1:s[r]==p?1:0}),C},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if(T(e),d.matchesSelector&&E&&!N[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||d.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){N(t,!0)}return 0<se(t,C,null,[e]).length},se.contains=function(e,t){return(e.ownerDocument||e)!=C&&T(e),y(e,t)},se.attr=function(e,t){(e.ownerDocument||e)!=C&&T(e);var n=b.attrHandle[t.toLowerCase()],r=n&&j.call(b.attrHandle,t.toLowerCase())?n(e,t,!E):void 0;return void 0!==r?r:d.attributes||!E?e.getAttribute(t):(r=e.getAttributeNode(t))&&r.specified?r.value:null},se.escape=function(e){return(e+"").replace(re,ie)},se.error=function(e){throw new Error("Syntax error, unrecognized expression: "+e)},se.uniqueSort=function(e){var t,n=[],r=0,i=0;if(l=!d.detectDuplicates,u=!d.sortStable&&e.slice(0),e.sort(D),l){while(t=e[i++])t===e[i]&&(r=n.push(i));while(r--)e.splice(n[r],1)}return u=null,e},o=se.getText=function(e){var t,n="",r=0,i=e.nodeType;if(i){if(1===i||9===i||11===i){if("string"==typeof e.textContent)return e.textContent;for(e=e.firstChild;e;e=e.nextSibling)n+=o(e)}else if(3===i||4===i)return e.nodeValue}else while(t=e[r++])n+=o(t);return n},(b=se.selectors={cacheLength:50,createPseudo:le,match:G,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||"").replace(te,ne),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return G.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&X.test(n)&&(t=h(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=m[e+" "];return t||(t=new RegExp("(^|"+M+")"+e+"("+M+"|$)"))&&m(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=se.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1<t.indexOf(i):"$="===r?i&&t.slice(-i.length)===i:"~="===r?-1<(" "+t.replace(B," ")+" ").indexOf(i):"|="===r&&(t===i||t.slice(0,i.length+1)===i+"-"))}},CHILD:function(h,e,t,g,v){var y="nth"!==h.slice(0,3),m="last"!==h.slice(-4),x="of-type"===e;return 1===g&&0===v?function(e){return!!e.parentNode}:function(e,t,n){var r,i,o,a,s,u,l=y!==m?"nextSibling":"previousSibling",c=e.parentNode,f=x&&e.nodeName.toLowerCase(),p=!n&&!x,d=!1;if(c){if(y){while(l){a=e;while(a=a[l])if(x?a.nodeName.toLowerCase()===f:1===a.nodeType)return!1;u=l="only"===h&&!u&&"nextSibling"}return!0}if(u=[m?c.firstChild:c.lastChild],m&&p){d=(s=(r=(i=(o=(a=c)[S]||(a[S]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]||[])[0]===k&&r[1])&&r[2],a=s&&c.childNodes[s];while(a=++s&&a&&a[l]||(d=s=0)||u.pop())if(1===a.nodeType&&++d&&a===e){i[h]=[k,s,d];break}}else if(p&&(d=s=(r=(i=(o=(a=e)[S]||(a[S]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]||[])[0]===k&&r[1]),!1===d)while(a=++s&&a&&a[l]||(d=s=0)||u.pop())if((x?a.nodeName.toLowerCase()===f:1===a.nodeType)&&++d&&(p&&((i=(o=a[S]||(a[S]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]=[k,d]),a===e))break;return(d-=v)===g||d%g==0&&0<=d/g}}},PSEUDO:function(e,o){var t,a=b.pseudos[e]||b.setFilters[e.toLowerCase()]||se.error("unsupported pseudo: "+e);return a[S]?a(o):1<a.length?(t=[e,e,"",o],b.setFilters.hasOwnProperty(e.toLowerCase())?le(function(e,t){var n,r=a(e,o),i=r.length;while(i--)e[n=P(e,r[i])]=!(t[n]=r[i])}):function(e){return a(e,0,t)}):a}},pseudos:{not:le(function(e){var r=[],i=[],s=f(e.replace($,"$1"));return s[S]?le(function(e,t,n,r){var i,o=s(e,null,r,[]),a=e.length;while(a--)(i=o[a])&&(e[a]=!(t[a]=i))}):function(e,t,n){return r[0]=e,s(r,null,n,i),r[0]=null,!i.pop()}}),has:le(function(t){return function(e){return 0<se(t,e).length}}),contains:le(function(t){return t=t.replace(te,ne),function(e){return-1<(e.textContent||o(e)).indexOf(t)}}),lang:le(function(n){return V.test(n||"")||se.error("unsupported lang: "+n),n=n.replace(te,ne).toLowerCase(),function(e){var t;do{if(t=E?e.lang:e.getAttribute("xml:lang")||e.getAttribute("lang"))return(t=t.toLowerCase())===n||0===t.indexOf(n+"-")}while((e=e.parentNode)&&1===e.nodeType);return!1}}),target:function(e){var t=n.location&&n.location.hash;return t&&t.slice(1)===e.id},root:function(e){return e===a},focus:function(e){return e===C.activeElement&&(!C.hasFocus||C.hasFocus())&&!!(e.type||e.href||~e.tabIndex)},enabled:ge(!1),disabled:ge(!0),checked:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&!!e.checked||"option"===t&&!!e.selected},selected:function(e){return e.parentNode&&e.parentNode.selectedIndex,!0===e.selected},empty:function(e){for(e=e.firstChild;e;e=e.nextSibling)if(e.nodeType<6)return!1;return!0},parent:function(e){return!b.pseudos.empty(e)},header:function(e){return J.test(e.nodeName)},input:function(e){return Q.test(e.nodeName)},button:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&"button"===e.type||"button"===t},text:function(e){var t;return"input"===e.nodeName.toLowerCase()&&"text"===e.type&&(null==(t=e.getAttribute("type"))||"text"===t.toLowerCase())},first:ve(function(){return[0]}),last:ve(function(e,t){return[t-1]}),eq:ve(function(e,t,n){return[n<0?n+t:n]}),even:ve(function(e,t){for(var n=0;n<t;n+=2)e.push(n);return e}),odd:ve(function(e,t){for(var n=1;n<t;n+=2)e.push(n);return e}),lt:ve(function(e,t,n){for(var r=n<0?n+t:t<n?t:n;0<=--r;)e.push(r);return e}),gt:ve(function(e,t,n){for(var r=n<0?n+t:n;++r<t;)e.push(r);return e})}}).pseudos.nth=b.pseudos.eq,{radio:!0,checkbox:!0,file:!0,password:!0,image:!0})b.pseudos[e]=de(e);for(e in{submit:!0,reset:!0})b.pseudos[e]=he(e);function me(){}function xe(e){for(var t=0,n=e.length,r="";t<n;t++)r+=e[t].value;return r}function be(s,e,t){var u=e.dir,l=e.next,c=l||u,f=t&&"parentNode"===c,p=r++;return e.first?function(e,t,n){while(e=e[u])if(1===e.nodeType||f)return s(e,t,n);return!1}:function(e,t,n){var r,i,o,a=[k,p];if(n){while(e=e[u])if((1===e.nodeType||f)&&s(e,t,n))return!0}else while(e=e[u])if(1===e.nodeType||f)if(i=(o=e[S]||(e[S]={}))[e.uniqueID]||(o[e.uniqueID]={}),l&&l===e.nodeName.toLowerCase())e=e[u]||e;else{if((r=i[c])&&r[0]===k&&r[1]===p)return a[2]=r[2];if((i[c]=a)[2]=s(e,t,n))return!0}return!1}}function we(i){return 1<i.length?function(e,t,n){var r=i.length;while(r--)if(!i[r](e,t,n))return!1;return!0}:i[0]}function Te(e,t,n,r,i){for(var o,a=[],s=0,u=e.length,l=null!=t;s<u;s++)(o=e[s])&&(n&&!n(o,r,i)||(a.push(o),l&&t.push(s)));return a}function Ce(d,h,g,v,y,e){return v&&!v[S]&&(v=Ce(v)),y&&!y[S]&&(y=Ce(y,e)),le(function(e,t,n,r){var i,o,a,s=[],u=[],l=t.length,c=e||function(e,t,n){for(var r=0,i=t.length;r<i;r++)se(e,t[r],n);return n}(h||"*",n.nodeType?[n]:n,[]),f=!d||!e&&h?c:Te(c,s,d,n,r),p=g?y||(e?d:l||v)?[]:t:f;if(g&&g(f,p,n,r),v){i=Te(p,u),v(i,[],n,r),o=i.length;while(o--)(a=i[o])&&(p[u[o]]=!(f[u[o]]=a))}if(e){if(y||d){if(y){i=[],o=p.length;while(o--)(a=p[o])&&i.push(f[o]=a);y(null,p=[],i,r)}o=p.length;while(o--)(a=p[o])&&-1<(i=y?P(e,a):s[o])&&(e[i]=!(t[i]=a))}}else p=Te(p===t?p.splice(l,p.length):p),y?y(null,t,p,r):H.apply(t,p)})}function Ee(e){for(var i,t,n,r=e.length,o=b.relative[e[0].type],a=o||b.relative[" "],s=o?1:0,u=be(function(e){return e===i},a,!0),l=be(function(e){return-1<P(i,e)},a,!0),c=[function(e,t,n){var r=!o&&(n||t!==w)||((i=t).nodeType?u(e,t,n):l(e,t,n));return i=null,r}];s<r;s++)if(t=b.relative[e[s].type])c=[be(we(c),t)];else{if((t=b.filter[e[s].type].apply(null,e[s].matches))[S]){for(n=++s;n<r;n++)if(b.relative[e[n].type])break;return Ce(1<s&&we(c),1<s&&xe(e.slice(0,s-1).concat({value:" "===e[s-2].type?"*":""})).replace($,"$1"),t,s<n&&Ee(e.slice(s,n)),n<r&&Ee(e=e.slice(n)),n<r&&xe(e))}c.push(t)}return we(c)}return me.prototype=b.filters=b.pseudos,b.setFilters=new me,h=se.tokenize=function(e,t){var n,r,i,o,a,s,u,l=x[e+" "];if(l)return t?0:l.slice(0);a=e,s=[],u=b.preFilter;while(a){for(o in n&&!(r=_.exec(a))||(r&&(a=a.slice(r[0].length)||a),s.push(i=[])),n=!1,(r=z.exec(a))&&(n=r.shift(),i.push({value:n,type:r[0].replace($," ")}),a=a.slice(n.length)),b.filter)!(r=G[o].exec(a))||u[o]&&!(r=u[o](r))||(n=r.shift(),i.push({value:n,type:o,matches:r}),a=a.slice(n.length));if(!n)break}return t?a.length:a?se.error(e):x(e,s).slice(0)},f=se.compile=function(e,t){var n,v,y,m,x,r,i=[],o=[],a=A[e+" "];if(!a){t||(t=h(e)),n=t.length;while(n--)(a=Ee(t[n]))[S]?i.push(a):o.push(a);(a=A(e,(v=o,m=0<(y=i).length,x=0<v.length,r=function(e,t,n,r,i){var o,a,s,u=0,l="0",c=e&&[],f=[],p=w,d=e||x&&b.find.TAG("*",i),h=k+=null==p?1:Math.random()||.1,g=d.length;for(i&&(w=t==C||t||i);l!==g&&null!=(o=d[l]);l++){if(x&&o){a=0,t||o.ownerDocument==C||(T(o),n=!E);while(s=v[a++])if(s(o,t||C,n)){r.push(o);break}i&&(k=h)}m&&((o=!s&&o)&&u--,e&&c.push(o))}if(u+=l,m&&l!==u){a=0;while(s=y[a++])s(c,f,t,n);if(e){if(0<u)while(l--)c[l]||f[l]||(f[l]=q.call(r));f=Te(f)}H.apply(r,f),i&&!e&&0<f.length&&1<u+y.length&&se.uniqueSort(r)}return i&&(k=h,w=p),c},m?le(r):r))).selector=e}return a},g=se.select=function(e,t,n,r){var i,o,a,s,u,l="function"==typeof e&&e,c=!r&&h(e=l.selector||e);if(n=n||[],1===c.length){if(2<(o=c[0]=c[0].slice(0)).length&&"ID"===(a=o[0]).type&&9===t.nodeType&&E&&b.relative[o[1].type]){if(!(t=(b.find.ID(a.matches[0].replace(te,ne),t)||[])[0]))return n;l&&(t=t.parentNode),e=e.slice(o.shift().value.length)}i=G.needsContext.test(e)?0:o.length;while(i--){if(a=o[i],b.relative[s=a.type])break;if((u=b.find[s])&&(r=u(a.matches[0].replace(te,ne),ee.test(o[0].type)&&ye(t.parentNode)||t))){if(o.splice(i,1),!(e=r.length&&xe(o)))return H.apply(n,r),n;break}}}return(l||f(e,c))(r,t,!E,n,!t||ee.test(e)&&ye(t.parentNode)||t),n},d.sortStable=S.split("").sort(D).join("")===S,d.detectDuplicates=!!l,T(),d.sortDetached=ce(function(e){return 1&e.compareDocumentPosition(C.createElement("fieldset"))}),ce(function(e){return e.innerHTML="<a href='#'></a>","#"===e.firstChild.getAttribute("href")})||fe("type|href|height|width",function(e,t,n){if(!n)return e.getAttribute(t,"type"===t.toLowerCase()?1:2)}),d.attributes&&ce(function(e){return e.innerHTML="<input/>",e.firstChild.setAttribute("value",""),""===e.firstChild.getAttribute("value")})||fe("value",function(e,t,n){if(!n&&"input"===e.nodeName.toLowerCase())return e.defaultValue}),ce(function(e){return null==e.getAttribute("disabled")})||fe(R,function(e,t,n){var r;if(!n)return!0===e[t]?t.toLowerCase():(r=e.getAttributeNode(t))&&r.specified?r.value:null}),se}(C);S.find=d,S.expr=d.selectors,S.expr[":"]=S.expr.pseudos,S.uniqueSort=S.unique=d.uniqueSort,S.text=d.getText,S.isXMLDoc=d.isXML,S.contains=d.contains,S.escapeSelector=d.escape;var h=function(e,t,n){var r=[],i=void 0!==n;while((e=e[t])&&9!==e.nodeType)if(1===e.nodeType){if(i&&S(e).is(n))break;r.push(e)}return r},T=function(e,t){for(var n=[];e;e=e.nextSibling)1===e.nodeType&&e!==t&&n.push(e);return n},k=S.expr.match.needsContext;function A(e,t){return e.nodeName&&e.nodeName.toLowerCase()===t.toLowerCase()}var N=/^<([a-z][^\/\0>:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function D(e,n,r){return m(n)?S.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?S.grep(e,function(e){return e===n!==r}):"string"!=typeof n?S.grep(e,function(e){return-1<i.call(n,e)!==r}):S.filter(n,e,r)}S.filter=function(e,t,n){var r=t[0];return n&&(e=":not("+e+")"),1===t.length&&1===r.nodeType?S.find.matchesSelector(r,e)?[r]:[]:S.find.matches(e,S.grep(t,function(e){return 1===e.nodeType}))},S.fn.extend({find:function(e){var t,n,r=this.length,i=this;if("string"!=typeof e)return this.pushStack(S(e).filter(function(){for(t=0;t<r;t++)if(S.contains(i[t],this))return!0}));for(n=this.pushStack([]),t=0;t<r;t++)S.find(e,i[t],n);return 1<r?S.uniqueSort(n):n},filter:function(e){return this.pushStack(D(this,e||[],!1))},not:function(e){return this.pushStack(D(this,e||[],!0))},is:function(e){return!!D(this,"string"==typeof e&&k.test(e)?S(e):e||[],!1).length}});var j,q=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]+))$/;(S.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||j,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:q.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof S?t[0]:t,S.merge(this,S.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:E,!0)),N.test(r[1])&&S.isPlainObject(t))for(r in t)m(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=E.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):m(e)?void 0!==n.ready?n.ready(e):e(S):S.makeArray(e,this)}).prototype=S.fn,j=S(E);var L=/^(?:parents|prev(?:Until|All))/,H={children:!0,contents:!0,next:!0,prev:!0};function O(e,t){while((e=e[t])&&1!==e.nodeType);return e}S.fn.extend({has:function(e){var t=S(e,this),n=t.length;return this.filter(function(){for(var e=0;e<n;e++)if(S.contains(this,t[e]))return!0})},closest:function(e,t){var n,r=0,i=this.length,o=[],a="string"!=typeof e&&S(e);if(!k.test(e))for(;r<i;r++)for(n=this[r];n&&n!==t;n=n.parentNode)if(n.nodeType<11&&(a?-1<a.index(n):1===n.nodeType&&S.find.matchesSelector(n,e))){o.push(n);break}return this.pushStack(1<o.length?S.uniqueSort(o):o)},index:function(e){return e?"string"==typeof e?i.call(S(e),this[0]):i.call(this,e.jquery?e[0]:e):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(e,t){return this.pushStack(S.uniqueSort(S.merge(this.get(),S(e,t))))},addBack:function(e){return this.add(null==e?this.prevObject:this.prevObject.filter(e))}}),S.each({parent:function(e){var t=e.parentNode;return t&&11!==t.nodeType?t:null},parents:function(e){return h(e,"parentNode")},parentsUntil:function(e,t,n){return h(e,"parentNode",n)},next:function(e){return O(e,"nextSibling")},prev:function(e){return O(e,"previousSibling")},nextAll:function(e){return h(e,"nextSibling")},prevAll:function(e){return h(e,"previousSibling")},nextUntil:function(e,t,n){return h(e,"nextSibling",n)},prevUntil:function(e,t,n){return h(e,"previousSibling",n)},siblings:function(e){return T((e.parentNode||{}).firstChild,e)},children:function(e){return T(e.firstChild)},contents:function(e){return null!=e.contentDocument&&r(e.contentDocument)?e.contentDocument:(A(e,"template")&&(e=e.content||e),S.merge([],e.childNodes))}},function(r,i){S.fn[r]=function(e,t){var n=S.map(this,i,e);return"Until"!==r.slice(-5)&&(t=e),t&&"string"==typeof t&&(n=S.filter(t,n)),1<this.length&&(H[r]||S.uniqueSort(n),L.test(r)&&n.reverse()),this.pushStack(n)}});var P=/[^\x20\t\r\n\f]+/g;function R(e){return e}function M(e){throw e}function I(e,t,n,r){var i;try{e&&m(i=e.promise)?i.call(e).done(t).fail(n):e&&m(i=e.then)?i.call(e,t,n):t.apply(void 0,[e].slice(r))}catch(e){n.apply(void 0,[e])}}S.Callbacks=function(r){var e,n;r="string"==typeof r?(e=r,n={},S.each(e.match(P)||[],function(e,t){n[t]=!0}),n):S.extend({},r);var i,t,o,a,s=[],u=[],l=-1,c=function(){for(a=a||r.once,o=i=!0;u.length;l=-1){t=u.shift();while(++l<s.length)!1===s[l].apply(t[0],t[1])&&r.stopOnFalse&&(l=s.length,t=!1)}r.memory||(t=!1),i=!1,a&&(s=t?[]:"")},f={add:function(){return s&&(t&&!i&&(l=s.length-1,u.push(t)),function n(e){S.each(e,function(e,t){m(t)?r.unique&&f.has(t)||s.push(t):t&&t.length&&"string"!==w(t)&&n(t)})}(arguments),t&&!i&&c()),this},remove:function(){return S.each(arguments,function(e,t){var n;while(-1<(n=S.inArray(t,s,n)))s.splice(n,1),n<=l&&l--}),this},has:function(e){return e?-1<S.inArray(e,s):0<s.length},empty:function(){return s&&(s=[]),this},disable:function(){return a=u=[],s=t="",this},disabled:function(){return!s},lock:function(){return a=u=[],t||i||(s=t=""),this},locked:function(){return!!a},fireWith:function(e,t){return a||(t=[e,(t=t||[]).slice?t.slice():t],u.push(t),i||c()),this},fire:function(){return f.fireWith(this,arguments),this},fired:function(){return!!o}};return f},S.extend({Deferred:function(e){var o=[["notify","progress",S.Callbacks("memory"),S.Callbacks("memory"),2],["resolve","done",S.Callbacks("once memory"),S.Callbacks("once memory"),0,"resolved"],["reject","fail",S.Callbacks("once memory"),S.Callbacks("once memory"),1,"rejected"]],i="pending",a={state:function(){return i},always:function(){return s.done(arguments).fail(arguments),this},"catch":function(e){return a.then(null,e)},pipe:function(){var i=arguments;return S.Deferred(function(r){S.each(o,function(e,t){var n=m(i[t[4]])&&i[t[4]];s[t[1]](function(){var e=n&&n.apply(this,arguments);e&&m(e.promise)?e.promise().progress(r.notify).done(r.resolve).fail(r.reject):r[t[0]+"With"](this,n?[e]:arguments)})}),i=null}).promise()},then:function(t,n,r){var u=0;function l(i,o,a,s){return function(){var n=this,r=arguments,e=function(){var e,t;if(!(i<u)){if((e=a.apply(n,r))===o.promise())throw new TypeError("Thenable self-resolution");t=e&&("object"==typeof e||"function"==typeof e)&&e.then,m(t)?s?t.call(e,l(u,o,R,s),l(u,o,M,s)):(u++,t.call(e,l(u,o,R,s),l(u,o,M,s),l(u,o,R,o.notifyWith))):(a!==R&&(n=void 0,r=[e]),(s||o.resolveWith)(n,r))}},t=s?e:function(){try{e()}catch(e){S.Deferred.exceptionHook&&S.Deferred.exceptionHook(e,t.stackTrace),u<=i+1&&(a!==M&&(n=void 0,r=[e]),o.rejectWith(n,r))}};i?t():(S.Deferred.getStackHook&&(t.stackTrace=S.Deferred.getStackHook()),C.setTimeout(t))}}return S.Deferred(function(e){o[0][3].add(l(0,e,m(r)?r:R,e.notifyWith)),o[1][3].add(l(0,e,m(t)?t:R)),o[2][3].add(l(0,e,m(n)?n:M))}).promise()},promise:function(e){return null!=e?S.extend(e,a):a}},s={};return S.each(o,function(e,t){var n=t[2],r=t[5];a[t[1]]=n.add,r&&n.add(function(){i=r},o[3-e][2].disable,o[3-e][3].disable,o[0][2].lock,o[0][3].lock),n.add(t[3].fire),s[t[0]]=function(){return s[t[0]+"With"](this===s?void 0:this,arguments),this},s[t[0]+"With"]=n.fireWith}),a.promise(s),e&&e.call(s,s),s},when:function(e){var n=arguments.length,t=n,r=Array(t),i=s.call(arguments),o=S.Deferred(),a=function(t){return function(e){r[t]=this,i[t]=1<arguments.length?s.call(arguments):e,--n||o.resolveWith(r,i)}};if(n<=1&&(I(e,o.done(a(t)).resolve,o.reject,!n),"pending"===o.state()||m(i[t]&&i[t].then)))return o.then();while(t--)I(i[t],a(t),o.reject);return o.promise()}});var W=/^(Eval|Internal|Range|Reference|Syntax|Type|URI)Error$/;S.Deferred.exceptionHook=function(e,t){C.console&&C.console.warn&&e&&W.test(e.name)&&C.console.warn("jQuery.Deferred exception: "+e.message,e.stack,t)},S.readyException=function(e){C.setTimeout(function(){throw e})};var F=S.Deferred();function B(){E.removeEventListener("DOMContentLoaded",B),C.removeEventListener("load",B),S.ready()}S.fn.ready=function(e){return F.then(e)["catch"](function(e){S.readyException(e)}),this},S.extend({isReady:!1,readyWait:1,ready:function(e){(!0===e?--S.readyWait:S.isReady)||(S.isReady=!0)!==e&&0<--S.readyWait||F.resolveWith(E,[S])}}),S.ready.then=F.then,"complete"===E.readyState||"loading"!==E.readyState&&!E.documentElement.doScroll?C.setTimeout(S.ready):(E.addEventListener("DOMContentLoaded",B),C.addEventListener("load",B));var $=function(e,t,n,r,i,o,a){var s=0,u=e.length,l=null==n;if("object"===w(n))for(s in i=!0,n)$(e,t,s,n[s],!0,o,a);else if(void 0!==r&&(i=!0,m(r)||(a=!0),l&&(a?(t.call(e,r),t=null):(l=t,t=function(e,t,n){return l.call(S(e),n)})),t))for(;s<u;s++)t(e[s],n,a?r:r.call(e[s],s,t(e[s],n)));return i?e:l?t.call(e):u?t(e[0],n):o},_=/^-ms-/,z=/-([a-z])/g;function U(e,t){return t.toUpperCase()}function X(e){return e.replace(_,"ms-").replace(z,U)}var V=function(e){return 1===e.nodeType||9===e.nodeType||!+e.nodeType};function G(){this.expando=S.expando+G.uid++}G.uid=1,G.prototype={cache:function(e){var t=e[this.expando];return t||(t={},V(e)&&(e.nodeType?e[this.expando]=t:Object.defineProperty(e,this.expando,{value:t,configurable:!0}))),t},set:function(e,t,n){var r,i=this.cache(e);if("string"==typeof t)i[X(t)]=n;else for(r in t)i[X(r)]=t[r];return i},get:function(e,t){return void 0===t?this.cache(e):e[this.expando]&&e[this.expando][X(t)]},access:function(e,t,n){return void 0===t||t&&"string"==typeof t&&void 0===n?this.get(e,t):(this.set(e,t,n),void 0!==n?n:t)},remove:function(e,t){var n,r=e[this.expando];if(void 0!==r){if(void 0!==t){n=(t=Array.isArray(t)?t.map(X):(t=X(t))in r?[t]:t.match(P)||[]).length;while(n--)delete r[t[n]]}(void 0===t||S.isEmptyObject(r))&&(e.nodeType?e[this.expando]=void 0:delete e[this.expando])}},hasData:function(e){var t=e[this.expando];return void 0!==t&&!S.isEmptyObject(t)}};var Y=new G,Q=new G,J=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,K=/[A-Z]/g;function Z(e,t,n){var r,i;if(void 0===n&&1===e.nodeType)if(r="data-"+t.replace(K,"-$&").toLowerCase(),"string"==typeof(n=e.getAttribute(r))){try{n="true"===(i=n)||"false"!==i&&("null"===i?null:i===+i+""?+i:J.test(i)?JSON.parse(i):i)}catch(e){}Q.set(e,t,n)}else n=void 0;return n}S.extend({hasData:function(e){return Q.hasData(e)||Y.hasData(e)},data:function(e,t,n){return Q.access(e,t,n)},removeData:function(e,t){Q.remove(e,t)},_data:function(e,t,n){return Y.access(e,t,n)},_removeData:function(e,t){Y.remove(e,t)}}),S.fn.extend({data:function(n,e){var t,r,i,o=this[0],a=o&&o.attributes;if(void 0===n){if(this.length&&(i=Q.get(o),1===o.nodeType&&!Y.get(o,"hasDataAttrs"))){t=a.length;while(t--)a[t]&&0===(r=a[t].name).indexOf("data-")&&(r=X(r.slice(5)),Z(o,r,i[r]));Y.set(o,"hasDataAttrs",!0)}return i}return"object"==typeof n?this.each(function(){Q.set(this,n)}):$(this,function(e){var t;if(o&&void 0===e)return void 0!==(t=Q.get(o,n))?t:void 0!==(t=Z(o,n))?t:void 0;this.each(function(){Q.set(this,n,e)})},null,e,1<arguments.length,null,!0)},removeData:function(e){return this.each(function(){Q.remove(this,e)})}}),S.extend({queue:function(e,t,n){var r;if(e)return t=(t||"fx")+"queue",r=Y.get(e,t),n&&(!r||Array.isArray(n)?r=Y.access(e,t,S.makeArray(n)):r.push(n)),r||[]},dequeue:function(e,t){t=t||"fx";var n=S.queue(e,t),r=n.length,i=n.shift(),o=S._queueHooks(e,t);"inprogress"===i&&(i=n.shift(),r--),i&&("fx"===t&&n.unshift("inprogress"),delete o.stop,i.call(e,function(){S.dequeue(e,t)},o)),!r&&o&&o.empty.fire()},_queueHooks:function(e,t){var n=t+"queueHooks";return Y.get(e,n)||Y.access(e,n,{empty:S.Callbacks("once memory").add(function(){Y.remove(e,[t+"queue",n])})})}}),S.fn.extend({queue:function(t,n){var e=2;return"string"!=typeof t&&(n=t,t="fx",e--),arguments.length<e?S.queue(this[0],t):void 0===n?this:this.each(function(){var e=S.queue(this,t,n);S._queueHooks(this,t),"fx"===t&&"inprogress"!==e[0]&&S.dequeue(this,t)})},dequeue:function(e){return this.each(function(){S.dequeue(this,e)})},clearQueue:function(e){return this.queue(e||"fx",[])},promise:function(e,t){var n,r=1,i=S.Deferred(),o=this,a=this.length,s=function(){--r||i.resolveWith(o,[o])};"string"!=typeof e&&(t=e,e=void 0),e=e||"fx";while(a--)(n=Y.get(o[a],e+"queueHooks"))&&n.empty&&(r++,n.empty.add(s));return s(),i.promise(t)}});var ee=/[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/.source,te=new RegExp("^(?:([+-])=|)("+ee+")([a-z%]*)$","i"),ne=["Top","Right","Bottom","Left"],re=E.documentElement,ie=function(e){return S.contains(e.ownerDocument,e)},oe={composed:!0};re.getRootNode&&(ie=function(e){return S.contains(e.ownerDocument,e)||e.getRootNode(oe)===e.ownerDocument});var ae=function(e,t){return"none"===(e=t||e).style.display||""===e.style.display&&ie(e)&&"none"===S.css(e,"display")};function se(e,t,n,r){var i,o,a=20,s=r?function(){return r.cur()}:function(){return S.css(e,t,"")},u=s(),l=n&&n[3]||(S.cssNumber[t]?"":"px"),c=e.nodeType&&(S.cssNumber[t]||"px"!==l&&+u)&&te.exec(S.css(e,t));if(c&&c[3]!==l){u/=2,l=l||c[3],c=+u||1;while(a--)S.style(e,t,c+l),(1-o)*(1-(o=s()/u||.5))<=0&&(a=0),c/=o;c*=2,S.style(e,t,c+l),n=n||[]}return n&&(c=+c||+u||0,i=n[1]?c+(n[1]+1)*n[2]:+n[2],r&&(r.unit=l,r.start=c,r.end=i)),i}var ue={};function le(e,t){for(var n,r,i,o,a,s,u,l=[],c=0,f=e.length;c<f;c++)(r=e[c]).style&&(n=r.style.display,t?("none"===n&&(l[c]=Y.get(r,"display")||null,l[c]||(r.style.display="")),""===r.style.display&&ae(r)&&(l[c]=(u=a=o=void 0,a=(i=r).ownerDocument,s=i.nodeName,(u=ue[s])||(o=a.body.appendChild(a.createElement(s)),u=S.css(o,"display"),o.parentNode.removeChild(o),"none"===u&&(u="block"),ue[s]=u)))):"none"!==n&&(l[c]="none",Y.set(r,"display",n)));for(c=0;c<f;c++)null!=l[c]&&(e[c].style.display=l[c]);return e}S.fn.extend({show:function(){return le(this,!0)},hide:function(){return le(this)},toggle:function(e){return"boolean"==typeof e?e?this.show():this.hide():this.each(function(){ae(this)?S(this).show():S(this).hide()})}});var ce,fe,pe=/^(?:checkbox|radio)$/i,de=/<([a-z][^\/\0>\x20\t\r\n\f]*)/i,he=/^$|^module$|\/(?:java|ecma)script/i;ce=E.createDocumentFragment().appendChild(E.createElement("div")),(fe=E.createElement("input")).setAttribute("type","radio"),fe.setAttribute("checked","checked"),fe.setAttribute("name","t"),ce.appendChild(fe),y.checkClone=ce.cloneNode(!0).cloneNode(!0).lastChild.checked,ce.innerHTML="<textarea>x</textarea>",y.noCloneChecked=!!ce.cloneNode(!0).lastChild.defaultValue,ce.innerHTML="<option></option>",y.option=!!ce.lastChild;var ge={thead:[1,"<table>","</table>"],col:[2,"<table><colgroup>","</colgroup></table>"],tr:[2,"<table><tbody>","</tbody></table>"],td:[3,"<table><tbody><tr>","</tr></tbody></table>"],_default:[0,"",""]};function ve(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&A(e,t)?S.merge([e],n):n}function ye(e,t){for(var n=0,r=e.length;n<r;n++)Y.set(e[n],"globalEval",!t||Y.get(t[n],"globalEval"))}ge.tbody=ge.tfoot=ge.colgroup=ge.caption=ge.thead,ge.th=ge.td,y.option||(ge.optgroup=ge.option=[1,"<select multiple='multiple'>","</select>"]);var me=/<|&#?\w+;/;function xe(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),p=[],d=0,h=e.length;d<h;d++)if((o=e[d])||0===o)if("object"===w(o))S.merge(p,o.nodeType?[o]:o);else if(me.test(o)){a=a||f.appendChild(t.createElement("div")),s=(de.exec(o)||["",""])[1].toLowerCase(),u=ge[s]||ge._default,a.innerHTML=u[1]+S.htmlPrefilter(o)+u[2],c=u[0];while(c--)a=a.lastChild;S.merge(p,a.childNodes),(a=f.firstChild).textContent=""}else p.push(t.createTextNode(o));f.textContent="",d=0;while(o=p[d++])if(r&&-1<S.inArray(o,r))i&&i.push(o);else if(l=ie(o),a=ve(f.appendChild(o),"script"),l&&ye(a),n){c=0;while(o=a[c++])he.test(o.type||"")&&n.push(o)}return f}var be=/^key/,we=/^(?:mouse|pointer|contextmenu|drag|drop)|click/,Te=/^([^.]*)(?:\.(.+)|)/;function Ce(){return!0}function Ee(){return!1}function Se(e,t){return e===function(){try{return E.activeElement}catch(e){}}()==("focus"===t)}function ke(e,t,n,r,i,o){var a,s;if("object"==typeof t){for(s in"string"!=typeof n&&(r=r||n,n=void 0),t)ke(e,s,n,r,t[s],o);return e}if(null==r&&null==i?(i=n,r=n=void 0):null==i&&("string"==typeof n?(i=r,r=void 0):(i=r,r=n,n=void 0)),!1===i)i=Ee;else if(!i)return e;return 1===o&&(a=i,(i=function(e){return S().off(e),a.apply(this,arguments)}).guid=a.guid||(a.guid=S.guid++)),e.each(function(){S.event.add(this,t,i,r,n)})}function Ae(e,i,o){o?(Y.set(e,i,!1),S.event.add(e,i,{namespace:!1,handler:function(e){var t,n,r=Y.get(this,i);if(1&e.isTrigger&&this[i]){if(r.length)(S.event.special[i]||{}).delegateType&&e.stopPropagation();else if(r=s.call(arguments),Y.set(this,i,r),t=o(this,i),this[i](),r!==(n=Y.get(this,i))||t?Y.set(this,i,!1):n={},r!==n)return e.stopImmediatePropagation(),e.preventDefault(),n.value}else r.length&&(Y.set(this,i,{value:S.event.trigger(S.extend(r[0],S.Event.prototype),r.slice(1),this)}),e.stopImmediatePropagation())}})):void 0===Y.get(e,i)&&S.event.add(e,i,Ce)}S.event={global:{},add:function(t,e,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Y.get(t);if(V(t)){n.handler&&(n=(o=n).handler,i=o.selector),i&&S.find.matchesSelector(re,i),n.guid||(n.guid=S.guid++),(u=v.events)||(u=v.events=Object.create(null)),(a=v.handle)||(a=v.handle=function(e){return"undefined"!=typeof S&&S.event.triggered!==e.type?S.event.dispatch.apply(t,arguments):void 0}),l=(e=(e||"").match(P)||[""]).length;while(l--)d=g=(s=Te.exec(e[l])||[])[1],h=(s[2]||"").split(".").sort(),d&&(f=S.event.special[d]||{},d=(i?f.delegateType:f.bindType)||d,f=S.event.special[d]||{},c=S.extend({type:d,origType:g,data:r,handler:n,guid:n.guid,selector:i,needsContext:i&&S.expr.match.needsContext.test(i),namespace:h.join(".")},o),(p=u[d])||((p=u[d]=[]).delegateCount=0,f.setup&&!1!==f.setup.call(t,r,h,a)||t.addEventListener&&t.addEventListener(d,a)),f.add&&(f.add.call(t,c),c.handler.guid||(c.handler.guid=n.guid)),i?p.splice(p.delegateCount++,0,c):p.push(c),S.event.global[d]=!0)}},remove:function(e,t,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Y.hasData(e)&&Y.get(e);if(v&&(u=v.events)){l=(t=(t||"").match(P)||[""]).length;while(l--)if(d=g=(s=Te.exec(t[l])||[])[1],h=(s[2]||"").split(".").sort(),d){f=S.event.special[d]||{},p=u[d=(r?f.delegateType:f.bindType)||d]||[],s=s[2]&&new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"),a=o=p.length;while(o--)c=p[o],!i&&g!==c.origType||n&&n.guid!==c.guid||s&&!s.test(c.namespace)||r&&r!==c.selector&&("**"!==r||!c.selector)||(p.splice(o,1),c.selector&&p.delegateCount--,f.remove&&f.remove.call(e,c));a&&!p.length&&(f.teardown&&!1!==f.teardown.call(e,h,v.handle)||S.removeEvent(e,d,v.handle),delete u[d])}else for(d in u)S.event.remove(e,d+t[l],n,r,!0);S.isEmptyObject(u)&&Y.remove(e,"handle events")}},dispatch:function(e){var t,n,r,i,o,a,s=new Array(arguments.length),u=S.event.fix(e),l=(Y.get(this,"events")||Object.create(null))[u.type]||[],c=S.event.special[u.type]||{};for(s[0]=u,t=1;t<arguments.length;t++)s[t]=arguments[t];if(u.delegateTarget=this,!c.preDispatch||!1!==c.preDispatch.call(this,u)){a=S.event.handlers.call(this,u,l),t=0;while((i=a[t++])&&!u.isPropagationStopped()){u.currentTarget=i.elem,n=0;while((o=i.handlers[n++])&&!u.isImmediatePropagationStopped())u.rnamespace&&!1!==o.namespace&&!u.rnamespace.test(o.namespace)||(u.handleObj=o,u.data=o.data,void 0!==(r=((S.event.special[o.origType]||{}).handle||o.handler).apply(i.elem,s))&&!1===(u.result=r)&&(u.preventDefault(),u.stopPropagation()))}return c.postDispatch&&c.postDispatch.call(this,u),u.result}},handlers:function(e,t){var n,r,i,o,a,s=[],u=t.delegateCount,l=e.target;if(u&&l.nodeType&&!("click"===e.type&&1<=e.button))for(;l!==this;l=l.parentNode||this)if(1===l.nodeType&&("click"!==e.type||!0!==l.disabled)){for(o=[],a={},n=0;n<u;n++)void 0===a[i=(r=t[n]).selector+" "]&&(a[i]=r.needsContext?-1<S(i,this).index(l):S.find(i,this,null,[l]).length),a[i]&&o.push(r);o.length&&s.push({elem:l,handlers:o})}return l=this,u<t.length&&s.push({elem:l,handlers:t.slice(u)}),s},addProp:function(t,e){Object.defineProperty(S.Event.prototype,t,{enumerable:!0,configurable:!0,get:m(e)?function(){if(this.originalEvent)return e(this.originalEvent)}:function(){if(this.originalEvent)return this.originalEvent[t]},set:function(e){Object.defineProperty(this,t,{enumerable:!0,configurable:!0,writable:!0,value:e})}})},fix:function(e){return e[S.expando]?e:new S.Event(e)},special:{load:{noBubble:!0},click:{setup:function(e){var t=this||e;return pe.test(t.type)&&t.click&&A(t,"input")&&Ae(t,"click",Ce),!1},trigger:function(e){var t=this||e;return pe.test(t.type)&&t.click&&A(t,"input")&&Ae(t,"click"),!0},_default:function(e){var t=e.target;return pe.test(t.type)&&t.click&&A(t,"input")&&Y.get(t,"click")||A(t,"a")}},beforeunload:{postDispatch:function(e){void 0!==e.result&&e.originalEvent&&(e.originalEvent.returnValue=e.result)}}}},S.removeEvent=function(e,t,n){e.removeEventListener&&e.removeEventListener(t,n)},S.Event=function(e,t){if(!(this instanceof S.Event))return new S.Event(e,t);e&&e.type?(this.originalEvent=e,this.type=e.type,this.isDefaultPrevented=e.defaultPrevented||void 0===e.defaultPrevented&&!1===e.returnValue?Ce:Ee,this.target=e.target&&3===e.target.nodeType?e.target.parentNode:e.target,this.currentTarget=e.currentTarget,this.relatedTarget=e.relatedTarget):this.type=e,t&&S.extend(this,t),this.timeStamp=e&&e.timeStamp||Date.now(),this[S.expando]=!0},S.Event.prototype={constructor:S.Event,isDefaultPrevented:Ee,isPropagationStopped:Ee,isImmediatePropagationStopped:Ee,isSimulated:!1,preventDefault:function(){var e=this.originalEvent;this.isDefaultPrevented=Ce,e&&!this.isSimulated&&e.preventDefault()},stopPropagation:function(){var e=this.originalEvent;this.isPropagationStopped=Ce,e&&!this.isSimulated&&e.stopPropagation()},stopImmediatePropagation:function(){var e=this.originalEvent;this.isImmediatePropagationStopped=Ce,e&&!this.isSimulated&&e.stopImmediatePropagation(),this.stopPropagation()}},S.each({altKey:!0,bubbles:!0,cancelable:!0,changedTouches:!0,ctrlKey:!0,detail:!0,eventPhase:!0,metaKey:!0,pageX:!0,pageY:!0,shiftKey:!0,view:!0,"char":!0,code:!0,charCode:!0,key:!0,keyCode:!0,button:!0,buttons:!0,clientX:!0,clientY:!0,offsetX:!0,offsetY:!0,pointerId:!0,pointerType:!0,screenX:!0,screenY:!0,targetTouches:!0,toElement:!0,touches:!0,which:function(e){var t=e.button;return null==e.which&&be.test(e.type)?null!=e.charCode?e.charCode:e.keyCode:!e.which&&void 0!==t&&we.test(e.type)?1&t?1:2&t?3:4&t?2:0:e.which}},S.event.addProp),S.each({focus:"focusin",blur:"focusout"},function(e,t){S.event.special[e]={setup:function(){return Ae(this,e,Se),!1},trigger:function(){return Ae(this,e),!0},delegateType:t}}),S.each({mouseenter:"mouseover",mouseleave:"mouseout",pointerenter:"pointerover",pointerleave:"pointerout"},function(e,i){S.event.special[e]={delegateType:i,bindType:i,handle:function(e){var t,n=e.relatedTarget,r=e.handleObj;return n&&(n===this||S.contains(this,n))||(e.type=r.origType,t=r.handler.apply(this,arguments),e.type=i),t}}}),S.fn.extend({on:function(e,t,n,r){return ke(this,e,t,n,r)},one:function(e,t,n,r){return ke(this,e,t,n,r,1)},off:function(e,t,n){var r,i;if(e&&e.preventDefault&&e.handleObj)return r=e.handleObj,S(e.delegateTarget).off(r.namespace?r.origType+"."+r.namespace:r.origType,r.selector,r.handler),this;if("object"==typeof e){for(i in e)this.off(i,t,e[i]);return this}return!1!==t&&"function"!=typeof t||(n=t,t=void 0),!1===n&&(n=Ee),this.each(function(){S.event.remove(this,e,n,t)})}});var Ne=/<script|<style|<link/i,De=/checked\s*(?:[^=]|=\s*.checked.)/i,je=/^\s*<!(?:\[CDATA\[|--)|(?:\]\]|--)>\s*$/g;function qe(e,t){return A(e,"table")&&A(11!==t.nodeType?t:t.firstChild,"tr")&&S(e).children("tbody")[0]||e}function Le(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function He(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Oe(e,t){var n,r,i,o,a,s;if(1===t.nodeType){if(Y.hasData(e)&&(s=Y.get(e).events))for(i in Y.remove(t,"handle events"),s)for(n=0,r=s[i].length;n<r;n++)S.event.add(t,i,s[i][n]);Q.hasData(e)&&(o=Q.access(e),a=S.extend({},o),Q.set(t,a))}}function Pe(n,r,i,o){r=g(r);var e,t,a,s,u,l,c=0,f=n.length,p=f-1,d=r[0],h=m(d);if(h||1<f&&"string"==typeof d&&!y.checkClone&&De.test(d))return n.each(function(e){var t=n.eq(e);h&&(r[0]=d.call(this,e,t.html())),Pe(t,r,i,o)});if(f&&(t=(e=xe(r,n[0].ownerDocument,!1,n,o)).firstChild,1===e.childNodes.length&&(e=t),t||o)){for(s=(a=S.map(ve(e,"script"),Le)).length;c<f;c++)u=e,c!==p&&(u=S.clone(u,!0,!0),s&&S.merge(a,ve(u,"script"))),i.call(n[c],u,c);if(s)for(l=a[a.length-1].ownerDocument,S.map(a,He),c=0;c<s;c++)u=a[c],he.test(u.type||"")&&!Y.access(u,"globalEval")&&S.contains(l,u)&&(u.src&&"module"!==(u.type||"").toLowerCase()?S._evalUrl&&!u.noModule&&S._evalUrl(u.src,{nonce:u.nonce||u.getAttribute("nonce")},l):b(u.textContent.replace(je,""),u,l))}return n}function Re(e,t,n){for(var r,i=t?S.filter(t,e):e,o=0;null!=(r=i[o]);o++)n||1!==r.nodeType||S.cleanData(ve(r)),r.parentNode&&(n&&ie(r)&&ye(ve(r,"script")),r.parentNode.removeChild(r));return e}S.extend({htmlPrefilter:function(e){return e},clone:function(e,t,n){var r,i,o,a,s,u,l,c=e.cloneNode(!0),f=ie(e);if(!(y.noCloneChecked||1!==e.nodeType&&11!==e.nodeType||S.isXMLDoc(e)))for(a=ve(c),r=0,i=(o=ve(e)).length;r<i;r++)s=o[r],u=a[r],void 0,"input"===(l=u.nodeName.toLowerCase())&&pe.test(s.type)?u.checked=s.checked:"input"!==l&&"textarea"!==l||(u.defaultValue=s.defaultValue);if(t)if(n)for(o=o||ve(e),a=a||ve(c),r=0,i=o.length;r<i;r++)Oe(o[r],a[r]);else Oe(e,c);return 0<(a=ve(c,"script")).length&&ye(a,!f&&ve(e,"script")),c},cleanData:function(e){for(var t,n,r,i=S.event.special,o=0;void 0!==(n=e[o]);o++)if(V(n)){if(t=n[Y.expando]){if(t.events)for(r in t.events)i[r]?S.event.remove(n,r):S.removeEvent(n,r,t.handle);n[Y.expando]=void 0}n[Q.expando]&&(n[Q.expando]=void 0)}}}),S.fn.extend({detach:function(e){return Re(this,e,!0)},remove:function(e){return Re(this,e)},text:function(e){return $(this,function(e){return void 0===e?S.text(this):this.empty().each(function(){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||(this.textContent=e)})},null,e,arguments.length)},append:function(){return Pe(this,arguments,function(e){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||qe(this,e).appendChild(e)})},prepend:function(){return Pe(this,arguments,function(e){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var t=qe(this,e);t.insertBefore(e,t.firstChild)}})},before:function(){return Pe(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this)})},after:function(){return Pe(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this.nextSibling)})},empty:function(){for(var e,t=0;null!=(e=this[t]);t++)1===e.nodeType&&(S.cleanData(ve(e,!1)),e.textContent="");return this},clone:function(e,t){return e=null!=e&&e,t=null==t?e:t,this.map(function(){return S.clone(this,e,t)})},html:function(e){return $(this,function(e){var t=this[0]||{},n=0,r=this.length;if(void 0===e&&1===t.nodeType)return t.innerHTML;if("string"==typeof e&&!Ne.test(e)&&!ge[(de.exec(e)||["",""])[1].toLowerCase()]){e=S.htmlPrefilter(e);try{for(;n<r;n++)1===(t=this[n]||{}).nodeType&&(S.cleanData(ve(t,!1)),t.innerHTML=e);t=0}catch(e){}}t&&this.empty().append(e)},null,e,arguments.length)},replaceWith:function(){var n=[];return Pe(this,arguments,function(e){var t=this.parentNode;S.inArray(this,n)<0&&(S.cleanData(ve(this)),t&&t.replaceChild(e,this))},n)}}),S.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(e,a){S.fn[e]=function(e){for(var t,n=[],r=S(e),i=r.length-1,o=0;o<=i;o++)t=o===i?this:this.clone(!0),S(r[o])[a](t),u.apply(n,t.get());return this.pushStack(n)}});var Me=new RegExp("^("+ee+")(?!px)[a-z%]+$","i"),Ie=function(e){var t=e.ownerDocument.defaultView;return t&&t.opener||(t=C),t.getComputedStyle(e)},We=function(e,t,n){var r,i,o={};for(i in t)o[i]=e.style[i],e.style[i]=t[i];for(i in r=n.call(e),t)e.style[i]=o[i];return r},Fe=new RegExp(ne.join("|"),"i");function Be(e,t,n){var r,i,o,a,s=e.style;return(n=n||Ie(e))&&(""!==(a=n.getPropertyValue(t)||n[t])||ie(e)||(a=S.style(e,t)),!y.pixelBoxStyles()&&Me.test(a)&&Fe.test(t)&&(r=s.width,i=s.minWidth,o=s.maxWidth,s.minWidth=s.maxWidth=s.width=a,a=n.width,s.width=r,s.minWidth=i,s.maxWidth=o)),void 0!==a?a+"":a}function $e(e,t){return{get:function(){if(!e())return(this.get=t).apply(this,arguments);delete this.get}}}!function(){function e(){if(l){u.style.cssText="position:absolute;left:-11111px;width:60px;margin-top:1px;padding:0;border:0",l.style.cssText="position:relative;display:block;box-sizing:border-box;overflow:scroll;margin:auto;border:1px;padding:1px;width:60%;top:1%",re.appendChild(u).appendChild(l);var e=C.getComputedStyle(l);n="1%"!==e.top,s=12===t(e.marginLeft),l.style.right="60%",o=36===t(e.right),r=36===t(e.width),l.style.position="absolute",i=12===t(l.offsetWidth/3),re.removeChild(u),l=null}}function t(e){return Math.round(parseFloat(e))}var n,r,i,o,a,s,u=E.createElement("div"),l=E.createElement("div");l.style&&(l.style.backgroundClip="content-box",l.cloneNode(!0).style.backgroundClip="",y.clearCloneStyle="content-box"===l.style.backgroundClip,S.extend(y,{boxSizingReliable:function(){return e(),r},pixelBoxStyles:function(){return e(),o},pixelPosition:function(){return e(),n},reliableMarginLeft:function(){return e(),s},scrollboxSize:function(){return e(),i},reliableTrDimensions:function(){var e,t,n,r;return null==a&&(e=E.createElement("table"),t=E.createElement("tr"),n=E.createElement("div"),e.style.cssText="position:absolute;left:-11111px",t.style.height="1px",n.style.height="9px",re.appendChild(e).appendChild(t).appendChild(n),r=C.getComputedStyle(t),a=3<parseInt(r.height),re.removeChild(e)),a}}))}();var _e=["Webkit","Moz","ms"],ze=E.createElement("div").style,Ue={};function Xe(e){var t=S.cssProps[e]||Ue[e];return t||(e in ze?e:Ue[e]=function(e){var t=e[0].toUpperCase()+e.slice(1),n=_e.length;while(n--)if((e=_e[n]+t)in ze)return e}(e)||e)}var Ve=/^(none|table(?!-c[ea]).+)/,Ge=/^--/,Ye={position:"absolute",visibility:"hidden",display:"block"},Qe={letterSpacing:"0",fontWeight:"400"};function Je(e,t,n){var r=te.exec(t);return r?Math.max(0,r[2]-(n||0))+(r[3]||"px"):t}function Ke(e,t,n,r,i,o){var a="width"===t?1:0,s=0,u=0;if(n===(r?"border":"content"))return 0;for(;a<4;a+=2)"margin"===n&&(u+=S.css(e,n+ne[a],!0,i)),r?("content"===n&&(u-=S.css(e,"padding"+ne[a],!0,i)),"margin"!==n&&(u-=S.css(e,"border"+ne[a]+"Width",!0,i))):(u+=S.css(e,"padding"+ne[a],!0,i),"padding"!==n?u+=S.css(e,"border"+ne[a]+"Width",!0,i):s+=S.css(e,"border"+ne[a]+"Width",!0,i));return!r&&0<=o&&(u+=Math.max(0,Math.ceil(e["offset"+t[0].toUpperCase()+t.slice(1)]-o-u-s-.5))||0),u}function Ze(e,t,n){var r=Ie(e),i=(!y.boxSizingReliable()||n)&&"border-box"===S.css(e,"boxSizing",!1,r),o=i,a=Be(e,t,r),s="offset"+t[0].toUpperCase()+t.slice(1);if(Me.test(a)){if(!n)return a;a="auto"}return(!y.boxSizingReliable()&&i||!y.reliableTrDimensions()&&A(e,"tr")||"auto"===a||!parseFloat(a)&&"inline"===S.css(e,"display",!1,r))&&e.getClientRects().length&&(i="border-box"===S.css(e,"boxSizing",!1,r),(o=s in e)&&(a=e[s])),(a=parseFloat(a)||0)+Ke(e,t,n||(i?"border":"content"),o,r,a)+"px"}function et(e,t,n,r,i){return new et.prototype.init(e,t,n,r,i)}S.extend({cssHooks:{opacity:{get:function(e,t){if(t){var n=Be(e,"opacity");return""===n?"1":n}}}},cssNumber:{animationIterationCount:!0,columnCount:!0,fillOpacity:!0,flexGrow:!0,flexShrink:!0,fontWeight:!0,gridArea:!0,gridColumn:!0,gridColumnEnd:!0,gridColumnStart:!0,gridRow:!0,gridRowEnd:!0,gridRowStart:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{},style:function(e,t,n,r){if(e&&3!==e.nodeType&&8!==e.nodeType&&e.style){var i,o,a,s=X(t),u=Ge.test(t),l=e.style;if(u||(t=Xe(s)),a=S.cssHooks[t]||S.cssHooks[s],void 0===n)return a&&"get"in a&&void 0!==(i=a.get(e,!1,r))?i:l[t];"string"===(o=typeof n)&&(i=te.exec(n))&&i[1]&&(n=se(e,t,i),o="number"),null!=n&&n==n&&("number"!==o||u||(n+=i&&i[3]||(S.cssNumber[s]?"":"px")),y.clearCloneStyle||""!==n||0!==t.indexOf("background")||(l[t]="inherit"),a&&"set"in a&&void 0===(n=a.set(e,n,r))||(u?l.setProperty(t,n):l[t]=n))}},css:function(e,t,n,r){var i,o,a,s=X(t);return Ge.test(t)||(t=Xe(s)),(a=S.cssHooks[t]||S.cssHooks[s])&&"get"in a&&(i=a.get(e,!0,n)),void 0===i&&(i=Be(e,t,r)),"normal"===i&&t in Qe&&(i=Qe[t]),""===n||n?(o=parseFloat(i),!0===n||isFinite(o)?o||0:i):i}}),S.each(["height","width"],function(e,u){S.cssHooks[u]={get:function(e,t,n){if(t)return!Ve.test(S.css(e,"display"))||e.getClientRects().length&&e.getBoundingClientRect().width?Ze(e,u,n):We(e,Ye,function(){return Ze(e,u,n)})},set:function(e,t,n){var r,i=Ie(e),o=!y.scrollboxSize()&&"absolute"===i.position,a=(o||n)&&"border-box"===S.css(e,"boxSizing",!1,i),s=n?Ke(e,u,n,a,i):0;return a&&o&&(s-=Math.ceil(e["offset"+u[0].toUpperCase()+u.slice(1)]-parseFloat(i[u])-Ke(e,u,"border",!1,i)-.5)),s&&(r=te.exec(t))&&"px"!==(r[3]||"px")&&(e.style[u]=t,t=S.css(e,u)),Je(0,t,s)}}}),S.cssHooks.marginLeft=$e(y.reliableMarginLeft,function(e,t){if(t)return(parseFloat(Be(e,"marginLeft"))||e.getBoundingClientRect().left-We(e,{marginLeft:0},function(){return e.getBoundingClientRect().left}))+"px"}),S.each({margin:"",padding:"",border:"Width"},function(i,o){S.cssHooks[i+o]={expand:function(e){for(var t=0,n={},r="string"==typeof e?e.split(" "):[e];t<4;t++)n[i+ne[t]+o]=r[t]||r[t-2]||r[0];return n}},"margin"!==i&&(S.cssHooks[i+o].set=Je)}),S.fn.extend({css:function(e,t){return $(this,function(e,t,n){var r,i,o={},a=0;if(Array.isArray(t)){for(r=Ie(e),i=t.length;a<i;a++)o[t[a]]=S.css(e,t[a],!1,r);return o}return void 0!==n?S.style(e,t,n):S.css(e,t)},e,t,1<arguments.length)}}),((S.Tween=et).prototype={constructor:et,init:function(e,t,n,r,i,o){this.elem=e,this.prop=n,this.easing=i||S.easing._default,this.options=t,this.start=this.now=this.cur(),this.end=r,this.unit=o||(S.cssNumber[n]?"":"px")},cur:function(){var e=et.propHooks[this.prop];return e&&e.get?e.get(this):et.propHooks._default.get(this)},run:function(e){var t,n=et.propHooks[this.prop];return this.options.duration?this.pos=t=S.easing[this.easing](e,this.options.duration*e,0,1,this.options.duration):this.pos=t=e,this.now=(this.end-this.start)*t+this.start,this.options.step&&this.options.step.call(this.elem,this.now,this),n&&n.set?n.set(this):et.propHooks._default.set(this),this}}).init.prototype=et.prototype,(et.propHooks={_default:{get:function(e){var t;return 1!==e.elem.nodeType||null!=e.elem[e.prop]&&null==e.elem.style[e.prop]?e.elem[e.prop]:(t=S.css(e.elem,e.prop,""))&&"auto"!==t?t:0},set:function(e){S.fx.step[e.prop]?S.fx.step[e.prop](e):1!==e.elem.nodeType||!S.cssHooks[e.prop]&&null==e.elem.style[Xe(e.prop)]?e.elem[e.prop]=e.now:S.style(e.elem,e.prop,e.now+e.unit)}}}).scrollTop=et.propHooks.scrollLeft={set:function(e){e.elem.nodeType&&e.elem.parentNode&&(e.elem[e.prop]=e.now)}},S.easing={linear:function(e){return e},swing:function(e){return.5-Math.cos(e*Math.PI)/2},_default:"swing"},S.fx=et.prototype.init,S.fx.step={};var tt,nt,rt,it,ot=/^(?:toggle|show|hide)$/,at=/queueHooks$/;function st(){nt&&(!1===E.hidden&&C.requestAnimationFrame?C.requestAnimationFrame(st):C.setTimeout(st,S.fx.interval),S.fx.tick())}function ut(){return C.setTimeout(function(){tt=void 0}),tt=Date.now()}function lt(e,t){var n,r=0,i={height:e};for(t=t?1:0;r<4;r+=2-t)i["margin"+(n=ne[r])]=i["padding"+n]=e;return t&&(i.opacity=i.width=e),i}function ct(e,t,n){for(var r,i=(ft.tweeners[t]||[]).concat(ft.tweeners["*"]),o=0,a=i.length;o<a;o++)if(r=i[o].call(n,t,e))return r}function ft(o,e,t){var n,a,r=0,i=ft.prefilters.length,s=S.Deferred().always(function(){delete u.elem}),u=function(){if(a)return!1;for(var e=tt||ut(),t=Math.max(0,l.startTime+l.duration-e),n=1-(t/l.duration||0),r=0,i=l.tweens.length;r<i;r++)l.tweens[r].run(n);return s.notifyWith(o,[l,n,t]),n<1&&i?t:(i||s.notifyWith(o,[l,1,0]),s.resolveWith(o,[l]),!1)},l=s.promise({elem:o,props:S.extend({},e),opts:S.extend(!0,{specialEasing:{},easing:S.easing._default},t),originalProperties:e,originalOptions:t,startTime:tt||ut(),duration:t.duration,tweens:[],createTween:function(e,t){var n=S.Tween(o,l.opts,e,t,l.opts.specialEasing[e]||l.opts.easing);return l.tweens.push(n),n},stop:function(e){var t=0,n=e?l.tweens.length:0;if(a)return this;for(a=!0;t<n;t++)l.tweens[t].run(1);return e?(s.notifyWith(o,[l,1,0]),s.resolveWith(o,[l,e])):s.rejectWith(o,[l,e]),this}}),c=l.props;for(!function(e,t){var n,r,i,o,a;for(n in e)if(i=t[r=X(n)],o=e[n],Array.isArray(o)&&(i=o[1],o=e[n]=o[0]),n!==r&&(e[r]=o,delete e[n]),(a=S.cssHooks[r])&&"expand"in a)for(n in o=a.expand(o),delete e[r],o)n in e||(e[n]=o[n],t[n]=i);else t[r]=i}(c,l.opts.specialEasing);r<i;r++)if(n=ft.prefilters[r].call(l,o,c,l.opts))return m(n.stop)&&(S._queueHooks(l.elem,l.opts.queue).stop=n.stop.bind(n)),n;return S.map(c,ct,l),m(l.opts.start)&&l.opts.start.call(o,l),l.progress(l.opts.progress).done(l.opts.done,l.opts.complete).fail(l.opts.fail).always(l.opts.always),S.fx.timer(S.extend(u,{elem:o,anim:l,queue:l.opts.queue})),l}S.Animation=S.extend(ft,{tweeners:{"*":[function(e,t){var n=this.createTween(e,t);return se(n.elem,e,te.exec(t),n),n}]},tweener:function(e,t){m(e)?(t=e,e=["*"]):e=e.match(P);for(var n,r=0,i=e.length;r<i;r++)n=e[r],ft.tweeners[n]=ft.tweeners[n]||[],ft.tweeners[n].unshift(t)},prefilters:[function(e,t,n){var r,i,o,a,s,u,l,c,f="width"in t||"height"in t,p=this,d={},h=e.style,g=e.nodeType&&ae(e),v=Y.get(e,"fxshow");for(r in n.queue||(null==(a=S._queueHooks(e,"fx")).unqueued&&(a.unqueued=0,s=a.empty.fire,a.empty.fire=function(){a.unqueued||s()}),a.unqueued++,p.always(function(){p.always(function(){a.unqueued--,S.queue(e,"fx").length||a.empty.fire()})})),t)if(i=t[r],ot.test(i)){if(delete t[r],o=o||"toggle"===i,i===(g?"hide":"show")){if("show"!==i||!v||void 0===v[r])continue;g=!0}d[r]=v&&v[r]||S.style(e,r)}if((u=!S.isEmptyObject(t))||!S.isEmptyObject(d))for(r in f&&1===e.nodeType&&(n.overflow=[h.overflow,h.overflowX,h.overflowY],null==(l=v&&v.display)&&(l=Y.get(e,"display")),"none"===(c=S.css(e,"display"))&&(l?c=l:(le([e],!0),l=e.style.display||l,c=S.css(e,"display"),le([e]))),("inline"===c||"inline-block"===c&&null!=l)&&"none"===S.css(e,"float")&&(u||(p.done(function(){h.display=l}),null==l&&(c=h.display,l="none"===c?"":c)),h.display="inline-block")),n.overflow&&(h.overflow="hidden",p.always(function(){h.overflow=n.overflow[0],h.overflowX=n.overflow[1],h.overflowY=n.overflow[2]})),u=!1,d)u||(v?"hidden"in v&&(g=v.hidden):v=Y.access(e,"fxshow",{display:l}),o&&(v.hidden=!g),g&&le([e],!0),p.done(function(){for(r in g||le([e]),Y.remove(e,"fxshow"),d)S.style(e,r,d[r])})),u=ct(g?v[r]:0,r,p),r in v||(v[r]=u.start,g&&(u.end=u.start,u.start=0))}],prefilter:function(e,t){t?ft.prefilters.unshift(e):ft.prefilters.push(e)}}),S.speed=function(e,t,n){var r=e&&"object"==typeof e?S.extend({},e):{complete:n||!n&&t||m(e)&&e,duration:e,easing:n&&t||t&&!m(t)&&t};return S.fx.off?r.duration=0:"number"!=typeof r.duration&&(r.duration in S.fx.speeds?r.duration=S.fx.speeds[r.duration]:r.duration=S.fx.speeds._default),null!=r.queue&&!0!==r.queue||(r.queue="fx"),r.old=r.complete,r.complete=function(){m(r.old)&&r.old.call(this),r.queue&&S.dequeue(this,r.queue)},r},S.fn.extend({fadeTo:function(e,t,n,r){return this.filter(ae).css("opacity",0).show().end().animate({opacity:t},e,n,r)},animate:function(t,e,n,r){var i=S.isEmptyObject(t),o=S.speed(e,n,r),a=function(){var e=ft(this,S.extend({},t),o);(i||Y.get(this,"finish"))&&e.stop(!0)};return a.finish=a,i||!1===o.queue?this.each(a):this.queue(o.queue,a)},stop:function(i,e,o){var a=function(e){var t=e.stop;delete e.stop,t(o)};return"string"!=typeof i&&(o=e,e=i,i=void 0),e&&this.queue(i||"fx",[]),this.each(function(){var e=!0,t=null!=i&&i+"queueHooks",n=S.timers,r=Y.get(this);if(t)r[t]&&r[t].stop&&a(r[t]);else for(t in r)r[t]&&r[t].stop&&at.test(t)&&a(r[t]);for(t=n.length;t--;)n[t].elem!==this||null!=i&&n[t].queue!==i||(n[t].anim.stop(o),e=!1,n.splice(t,1));!e&&o||S.dequeue(this,i)})},finish:function(a){return!1!==a&&(a=a||"fx"),this.each(function(){var e,t=Y.get(this),n=t[a+"queue"],r=t[a+"queueHooks"],i=S.timers,o=n?n.length:0;for(t.finish=!0,S.queue(this,a,[]),r&&r.stop&&r.stop.call(this,!0),e=i.length;e--;)i[e].elem===this&&i[e].queue===a&&(i[e].anim.stop(!0),i.splice(e,1));for(e=0;e<o;e++)n[e]&&n[e].finish&&n[e].finish.call(this);delete t.finish})}}),S.each(["toggle","show","hide"],function(e,r){var i=S.fn[r];S.fn[r]=function(e,t,n){return null==e||"boolean"==typeof e?i.apply(this,arguments):this.animate(lt(r,!0),e,t,n)}}),S.each({slideDown:lt("show"),slideUp:lt("hide"),slideToggle:lt("toggle"),fadeIn:{opacity:"show"},fadeOut:{opacity:"hide"},fadeToggle:{opacity:"toggle"}},function(e,r){S.fn[e]=function(e,t,n){return this.animate(r,e,t,n)}}),S.timers=[],S.fx.tick=function(){var e,t=0,n=S.timers;for(tt=Date.now();t<n.length;t++)(e=n[t])()||n[t]!==e||n.splice(t--,1);n.length||S.fx.stop(),tt=void 0},S.fx.timer=function(e){S.timers.push(e),S.fx.start()},S.fx.interval=13,S.fx.start=function(){nt||(nt=!0,st())},S.fx.stop=function(){nt=null},S.fx.speeds={slow:600,fast:200,_default:400},S.fn.delay=function(r,e){return r=S.fx&&S.fx.speeds[r]||r,e=e||"fx",this.queue(e,function(e,t){var n=C.setTimeout(e,r);t.stop=function(){C.clearTimeout(n)}})},rt=E.createElement("input"),it=E.createElement("select").appendChild(E.createElement("option")),rt.type="checkbox",y.checkOn=""!==rt.value,y.optSelected=it.selected,(rt=E.createElement("input")).value="t",rt.type="radio",y.radioValue="t"===rt.value;var pt,dt=S.expr.attrHandle;S.fn.extend({attr:function(e,t){return $(this,S.attr,e,t,1<arguments.length)},removeAttr:function(e){return this.each(function(){S.removeAttr(this,e)})}}),S.extend({attr:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return"undefined"==typeof e.getAttribute?S.prop(e,t,n):(1===o&&S.isXMLDoc(e)||(i=S.attrHooks[t.toLowerCase()]||(S.expr.match.bool.test(t)?pt:void 0)),void 0!==n?null===n?void S.removeAttr(e,t):i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:(e.setAttribute(t,n+""),n):i&&"get"in i&&null!==(r=i.get(e,t))?r:null==(r=S.find.attr(e,t))?void 0:r)},attrHooks:{type:{set:function(e,t){if(!y.radioValue&&"radio"===t&&A(e,"input")){var n=e.value;return e.setAttribute("type",t),n&&(e.value=n),t}}}},removeAttr:function(e,t){var n,r=0,i=t&&t.match(P);if(i&&1===e.nodeType)while(n=i[r++])e.removeAttribute(n)}}),pt={set:function(e,t,n){return!1===t?S.removeAttr(e,n):e.setAttribute(n,n),n}},S.each(S.expr.match.bool.source.match(/\w+/g),function(e,t){var a=dt[t]||S.find.attr;dt[t]=function(e,t,n){var r,i,o=t.toLowerCase();return n||(i=dt[o],dt[o]=r,r=null!=a(e,t,n)?o:null,dt[o]=i),r}});var ht=/^(?:input|select|textarea|button)$/i,gt=/^(?:a|area)$/i;function vt(e){return(e.match(P)||[]).join(" ")}function yt(e){return e.getAttribute&&e.getAttribute("class")||""}function mt(e){return Array.isArray(e)?e:"string"==typeof e&&e.match(P)||[]}S.fn.extend({prop:function(e,t){return $(this,S.prop,e,t,1<arguments.length)},removeProp:function(e){return this.each(function(){delete this[S.propFix[e]||e]})}}),S.extend({prop:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return 1===o&&S.isXMLDoc(e)||(t=S.propFix[t]||t,i=S.propHooks[t]),void 0!==n?i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:e[t]=n:i&&"get"in i&&null!==(r=i.get(e,t))?r:e[t]},propHooks:{tabIndex:{get:function(e){var t=S.find.attr(e,"tabindex");return t?parseInt(t,10):ht.test(e.nodeName)||gt.test(e.nodeName)&&e.href?0:-1}}},propFix:{"for":"htmlFor","class":"className"}}),y.optSelected||(S.propHooks.selected={get:function(e){var t=e.parentNode;return t&&t.parentNode&&t.parentNode.selectedIndex,null},set:function(e){var t=e.parentNode;t&&(t.selectedIndex,t.parentNode&&t.parentNode.selectedIndex)}}),S.each(["tabIndex","readOnly","maxLength","cellSpacing","cellPadding","rowSpan","colSpan","useMap","frameBorder","contentEditable"],function(){S.propFix[this.toLowerCase()]=this}),S.fn.extend({addClass:function(t){var e,n,r,i,o,a,s,u=0;if(m(t))return this.each(function(e){S(this).addClass(t.call(this,e,yt(this)))});if((e=mt(t)).length)while(n=this[u++])if(i=yt(n),r=1===n.nodeType&&" "+vt(i)+" "){a=0;while(o=e[a++])r.indexOf(" "+o+" ")<0&&(r+=o+" ");i!==(s=vt(r))&&n.setAttribute("class",s)}return this},removeClass:function(t){var e,n,r,i,o,a,s,u=0;if(m(t))return this.each(function(e){S(this).removeClass(t.call(this,e,yt(this)))});if(!arguments.length)return this.attr("class","");if((e=mt(t)).length)while(n=this[u++])if(i=yt(n),r=1===n.nodeType&&" "+vt(i)+" "){a=0;while(o=e[a++])while(-1<r.indexOf(" "+o+" "))r=r.replace(" "+o+" "," ");i!==(s=vt(r))&&n.setAttribute("class",s)}return this},toggleClass:function(i,t){var o=typeof i,a="string"===o||Array.isArray(i);return"boolean"==typeof t&&a?t?this.addClass(i):this.removeClass(i):m(i)?this.each(function(e){S(this).toggleClass(i.call(this,e,yt(this),t),t)}):this.each(function(){var e,t,n,r;if(a){t=0,n=S(this),r=mt(i);while(e=r[t++])n.hasClass(e)?n.removeClass(e):n.addClass(e)}else void 0!==i&&"boolean"!==o||((e=yt(this))&&Y.set(this,"__className__",e),this.setAttribute&&this.setAttribute("class",e||!1===i?"":Y.get(this,"__className__")||""))})},hasClass:function(e){var t,n,r=0;t=" "+e+" ";while(n=this[r++])if(1===n.nodeType&&-1<(" "+vt(yt(n))+" ").indexOf(t))return!0;return!1}});var xt=/\r/g;S.fn.extend({val:function(n){var r,e,i,t=this[0];return arguments.length?(i=m(n),this.each(function(e){var t;1===this.nodeType&&(null==(t=i?n.call(this,e,S(this).val()):n)?t="":"number"==typeof t?t+="":Array.isArray(t)&&(t=S.map(t,function(e){return null==e?"":e+""})),(r=S.valHooks[this.type]||S.valHooks[this.nodeName.toLowerCase()])&&"set"in r&&void 0!==r.set(this,t,"value")||(this.value=t))})):t?(r=S.valHooks[t.type]||S.valHooks[t.nodeName.toLowerCase()])&&"get"in r&&void 0!==(e=r.get(t,"value"))?e:"string"==typeof(e=t.value)?e.replace(xt,""):null==e?"":e:void 0}}),S.extend({valHooks:{option:{get:function(e){var t=S.find.attr(e,"value");return null!=t?t:vt(S.text(e))}},select:{get:function(e){var t,n,r,i=e.options,o=e.selectedIndex,a="select-one"===e.type,s=a?null:[],u=a?o+1:i.length;for(r=o<0?u:a?o:0;r<u;r++)if(((n=i[r]).selected||r===o)&&!n.disabled&&(!n.parentNode.disabled||!A(n.parentNode,"optgroup"))){if(t=S(n).val(),a)return t;s.push(t)}return s},set:function(e,t){var n,r,i=e.options,o=S.makeArray(t),a=i.length;while(a--)((r=i[a]).selected=-1<S.inArray(S.valHooks.option.get(r),o))&&(n=!0);return n||(e.selectedIndex=-1),o}}}}),S.each(["radio","checkbox"],function(){S.valHooks[this]={set:function(e,t){if(Array.isArray(t))return e.checked=-1<S.inArray(S(e).val(),t)}},y.checkOn||(S.valHooks[this].get=function(e){return null===e.getAttribute("value")?"on":e.value})}),y.focusin="onfocusin"in C;var bt=/^(?:focusinfocus|focusoutblur)$/,wt=function(e){e.stopPropagation()};S.extend(S.event,{trigger:function(e,t,n,r){var i,o,a,s,u,l,c,f,p=[n||E],d=v.call(e,"type")?e.type:e,h=v.call(e,"namespace")?e.namespace.split("."):[];if(o=f=a=n=n||E,3!==n.nodeType&&8!==n.nodeType&&!bt.test(d+S.event.triggered)&&(-1<d.indexOf(".")&&(d=(h=d.split(".")).shift(),h.sort()),u=d.indexOf(":")<0&&"on"+d,(e=e[S.expando]?e:new S.Event(d,"object"==typeof e&&e)).isTrigger=r?2:3,e.namespace=h.join("."),e.rnamespace=e.namespace?new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,e.result=void 0,e.target||(e.target=n),t=null==t?[e]:S.makeArray(t,[e]),c=S.event.special[d]||{},r||!c.trigger||!1!==c.trigger.apply(n,t))){if(!r&&!c.noBubble&&!x(n)){for(s=c.delegateType||d,bt.test(s+d)||(o=o.parentNode);o;o=o.parentNode)p.push(o),a=o;a===(n.ownerDocument||E)&&p.push(a.defaultView||a.parentWindow||C)}i=0;while((o=p[i++])&&!e.isPropagationStopped())f=o,e.type=1<i?s:c.bindType||d,(l=(Y.get(o,"events")||Object.create(null))[e.type]&&Y.get(o,"handle"))&&l.apply(o,t),(l=u&&o[u])&&l.apply&&V(o)&&(e.result=l.apply(o,t),!1===e.result&&e.preventDefault());return e.type=d,r||e.isDefaultPrevented()||c._default&&!1!==c._default.apply(p.pop(),t)||!V(n)||u&&m(n[d])&&!x(n)&&((a=n[u])&&(n[u]=null),S.event.triggered=d,e.isPropagationStopped()&&f.addEventListener(d,wt),n[d](),e.isPropagationStopped()&&f.removeEventListener(d,wt),S.event.triggered=void 0,a&&(n[u]=a)),e.result}},simulate:function(e,t,n){var r=S.extend(new S.Event,n,{type:e,isSimulated:!0});S.event.trigger(r,null,t)}}),S.fn.extend({trigger:function(e,t){return this.each(function(){S.event.trigger(e,t,this)})},triggerHandler:function(e,t){var n=this[0];if(n)return S.event.trigger(e,t,n,!0)}}),y.focusin||S.each({focus:"focusin",blur:"focusout"},function(n,r){var i=function(e){S.event.simulate(r,e.target,S.event.fix(e))};S.event.special[r]={setup:function(){var e=this.ownerDocument||this.document||this,t=Y.access(e,r);t||e.addEventListener(n,i,!0),Y.access(e,r,(t||0)+1)},teardown:function(){var e=this.ownerDocument||this.document||this,t=Y.access(e,r)-1;t?Y.access(e,r,t):(e.removeEventListener(n,i,!0),Y.remove(e,r))}}});var Tt=C.location,Ct={guid:Date.now()},Et=/\?/;S.parseXML=function(e){var t;if(!e||"string"!=typeof e)return null;try{t=(new C.DOMParser).parseFromString(e,"text/xml")}catch(e){t=void 0}return t&&!t.getElementsByTagName("parsererror").length||S.error("Invalid XML: "+e),t};var St=/\[\]$/,kt=/\r?\n/g,At=/^(?:submit|button|image|reset|file)$/i,Nt=/^(?:input|select|textarea|keygen)/i;function Dt(n,e,r,i){var t;if(Array.isArray(e))S.each(e,function(e,t){r||St.test(n)?i(n,t):Dt(n+"["+("object"==typeof t&&null!=t?e:"")+"]",t,r,i)});else if(r||"object"!==w(e))i(n,e);else for(t in e)Dt(n+"["+t+"]",e[t],r,i)}S.param=function(e,t){var n,r=[],i=function(e,t){var n=m(t)?t():t;r[r.length]=encodeURIComponent(e)+"="+encodeURIComponent(null==n?"":n)};if(null==e)return"";if(Array.isArray(e)||e.jquery&&!S.isPlainObject(e))S.each(e,function(){i(this.name,this.value)});else for(n in e)Dt(n,e[n],t,i);return r.join("&")},S.fn.extend({serialize:function(){return S.param(this.serializeArray())},serializeArray:function(){return this.map(function(){var e=S.prop(this,"elements");return e?S.makeArray(e):this}).filter(function(){var e=this.type;return this.name&&!S(this).is(":disabled")&&Nt.test(this.nodeName)&&!At.test(e)&&(this.checked||!pe.test(e))}).map(function(e,t){var n=S(this).val();return null==n?null:Array.isArray(n)?S.map(n,function(e){return{name:t.name,value:e.replace(kt,"\r\n")}}):{name:t.name,value:n.replace(kt,"\r\n")}}).get()}});var jt=/%20/g,qt=/#.*$/,Lt=/([?&])_=[^&]*/,Ht=/^(.*?):[ \t]*([^\r\n]*)$/gm,Ot=/^(?:GET|HEAD)$/,Pt=/^\/\//,Rt={},Mt={},It="*/".concat("*"),Wt=E.createElement("a");function Ft(o){return function(e,t){"string"!=typeof e&&(t=e,e="*");var n,r=0,i=e.toLowerCase().match(P)||[];if(m(t))while(n=i[r++])"+"===n[0]?(n=n.slice(1)||"*",(o[n]=o[n]||[]).unshift(t)):(o[n]=o[n]||[]).push(t)}}function Bt(t,i,o,a){var s={},u=t===Mt;function l(e){var r;return s[e]=!0,S.each(t[e]||[],function(e,t){var n=t(i,o,a);return"string"!=typeof n||u||s[n]?u?!(r=n):void 0:(i.dataTypes.unshift(n),l(n),!1)}),r}return l(i.dataTypes[0])||!s["*"]&&l("*")}function $t(e,t){var n,r,i=S.ajaxSettings.flatOptions||{};for(n in t)void 0!==t[n]&&((i[n]?e:r||(r={}))[n]=t[n]);return r&&S.extend(!0,e,r),e}Wt.href=Tt.href,S.extend({active:0,lastModified:{},etag:{},ajaxSettings:{url:Tt.href,type:"GET",isLocal:/^(?:about|app|app-storage|.+-extension|file|res|widget):$/.test(Tt.protocol),global:!0,processData:!0,async:!0,contentType:"application/x-www-form-urlencoded; charset=UTF-8",accepts:{"*":It,text:"text/plain",html:"text/html",xml:"application/xml, text/xml",json:"application/json, text/javascript"},contents:{xml:/\bxml\b/,html:/\bhtml/,json:/\bjson\b/},responseFields:{xml:"responseXML",text:"responseText",json:"responseJSON"},converters:{"* text":String,"text html":!0,"text json":JSON.parse,"text xml":S.parseXML},flatOptions:{url:!0,context:!0}},ajaxSetup:function(e,t){return t?$t($t(e,S.ajaxSettings),t):$t(S.ajaxSettings,e)},ajaxPrefilter:Ft(Rt),ajaxTransport:Ft(Mt),ajax:function(e,t){"object"==typeof e&&(t=e,e=void 0),t=t||{};var c,f,p,n,d,r,h,g,i,o,v=S.ajaxSetup({},t),y=v.context||v,m=v.context&&(y.nodeType||y.jquery)?S(y):S.event,x=S.Deferred(),b=S.Callbacks("once memory"),w=v.statusCode||{},a={},s={},u="canceled",T={readyState:0,getResponseHeader:function(e){var t;if(h){if(!n){n={};while(t=Ht.exec(p))n[t[1].toLowerCase()+" "]=(n[t[1].toLowerCase()+" "]||[]).concat(t[2])}t=n[e.toLowerCase()+" "]}return null==t?null:t.join(", ")},getAllResponseHeaders:function(){return h?p:null},setRequestHeader:function(e,t){return null==h&&(e=s[e.toLowerCase()]=s[e.toLowerCase()]||e,a[e]=t),this},overrideMimeType:function(e){return null==h&&(v.mimeType=e),this},statusCode:function(e){var t;if(e)if(h)T.always(e[T.status]);else for(t in e)w[t]=[w[t],e[t]];return this},abort:function(e){var t=e||u;return c&&c.abort(t),l(0,t),this}};if(x.promise(T),v.url=((e||v.url||Tt.href)+"").replace(Pt,Tt.protocol+"//"),v.type=t.method||t.type||v.method||v.type,v.dataTypes=(v.dataType||"*").toLowerCase().match(P)||[""],null==v.crossDomain){r=E.createElement("a");try{r.href=v.url,r.href=r.href,v.crossDomain=Wt.protocol+"//"+Wt.host!=r.protocol+"//"+r.host}catch(e){v.crossDomain=!0}}if(v.data&&v.processData&&"string"!=typeof v.data&&(v.data=S.param(v.data,v.traditional)),Bt(Rt,v,t,T),h)return T;for(i in(g=S.event&&v.global)&&0==S.active++&&S.event.trigger("ajaxStart"),v.type=v.type.toUpperCase(),v.hasContent=!Ot.test(v.type),f=v.url.replace(qt,""),v.hasContent?v.data&&v.processData&&0===(v.contentType||"").indexOf("application/x-www-form-urlencoded")&&(v.data=v.data.replace(jt,"+")):(o=v.url.slice(f.length),v.data&&(v.processData||"string"==typeof v.data)&&(f+=(Et.test(f)?"&":"?")+v.data,delete v.data),!1===v.cache&&(f=f.replace(Lt,"$1"),o=(Et.test(f)?"&":"?")+"_="+Ct.guid+++o),v.url=f+o),v.ifModified&&(S.lastModified[f]&&T.setRequestHeader("If-Modified-Since",S.lastModified[f]),S.etag[f]&&T.setRequestHeader("If-None-Match",S.etag[f])),(v.data&&v.hasContent&&!1!==v.contentType||t.contentType)&&T.setRequestHeader("Content-Type",v.contentType),T.setRequestHeader("Accept",v.dataTypes[0]&&v.accepts[v.dataTypes[0]]?v.accepts[v.dataTypes[0]]+("*"!==v.dataTypes[0]?", "+It+"; q=0.01":""):v.accepts["*"]),v.headers)T.setRequestHeader(i,v.headers[i]);if(v.beforeSend&&(!1===v.beforeSend.call(y,T,v)||h))return T.abort();if(u="abort",b.add(v.complete),T.done(v.success),T.fail(v.error),c=Bt(Mt,v,t,T)){if(T.readyState=1,g&&m.trigger("ajaxSend",[T,v]),h)return T;v.async&&0<v.timeout&&(d=C.setTimeout(function(){T.abort("timeout")},v.timeout));try{h=!1,c.send(a,l)}catch(e){if(h)throw e;l(-1,e)}}else l(-1,"No Transport");function l(e,t,n,r){var i,o,a,s,u,l=t;h||(h=!0,d&&C.clearTimeout(d),c=void 0,p=r||"",T.readyState=0<e?4:0,i=200<=e&&e<300||304===e,n&&(s=function(e,t,n){var r,i,o,a,s=e.contents,u=e.dataTypes;while("*"===u[0])u.shift(),void 0===r&&(r=e.mimeType||t.getResponseHeader("Content-Type"));if(r)for(i in s)if(s[i]&&s[i].test(r)){u.unshift(i);break}if(u[0]in n)o=u[0];else{for(i in n){if(!u[0]||e.converters[i+" "+u[0]]){o=i;break}a||(a=i)}o=o||a}if(o)return o!==u[0]&&u.unshift(o),n[o]}(v,T,n)),!i&&-1<S.inArray("script",v.dataTypes)&&(v.converters["text script"]=function(){}),s=function(e,t,n,r){var i,o,a,s,u,l={},c=e.dataTypes.slice();if(c[1])for(a in e.converters)l[a.toLowerCase()]=e.converters[a];o=c.shift();while(o)if(e.responseFields[o]&&(n[e.responseFields[o]]=t),!u&&r&&e.dataFilter&&(t=e.dataFilter(t,e.dataType)),u=o,o=c.shift())if("*"===o)o=u;else if("*"!==u&&u!==o){if(!(a=l[u+" "+o]||l["* "+o]))for(i in l)if((s=i.split(" "))[1]===o&&(a=l[u+" "+s[0]]||l["* "+s[0]])){!0===a?a=l[i]:!0!==l[i]&&(o=s[0],c.unshift(s[1]));break}if(!0!==a)if(a&&e["throws"])t=a(t);else try{t=a(t)}catch(e){return{state:"parsererror",error:a?e:"No conversion from "+u+" to "+o}}}return{state:"success",data:t}}(v,s,T,i),i?(v.ifModified&&((u=T.getResponseHeader("Last-Modified"))&&(S.lastModified[f]=u),(u=T.getResponseHeader("etag"))&&(S.etag[f]=u)),204===e||"HEAD"===v.type?l="nocontent":304===e?l="notmodified":(l=s.state,o=s.data,i=!(a=s.error))):(a=l,!e&&l||(l="error",e<0&&(e=0))),T.status=e,T.statusText=(t||l)+"",i?x.resolveWith(y,[o,l,T]):x.rejectWith(y,[T,l,a]),T.statusCode(w),w=void 0,g&&m.trigger(i?"ajaxSuccess":"ajaxError",[T,v,i?o:a]),b.fireWith(y,[T,l]),g&&(m.trigger("ajaxComplete",[T,v]),--S.active||S.event.trigger("ajaxStop")))}return T},getJSON:function(e,t,n){return S.get(e,t,n,"json")},getScript:function(e,t){return S.get(e,void 0,t,"script")}}),S.each(["get","post"],function(e,i){S[i]=function(e,t,n,r){return m(t)&&(r=r||n,n=t,t=void 0),S.ajax(S.extend({url:e,type:i,dataType:r,data:t,success:n},S.isPlainObject(e)&&e))}}),S.ajaxPrefilter(function(e){var t;for(t in e.headers)"content-type"===t.toLowerCase()&&(e.contentType=e.headers[t]||"")}),S._evalUrl=function(e,t,n){return S.ajax({url:e,type:"GET",dataType:"script",cache:!0,async:!1,global:!1,converters:{"text script":function(){}},dataFilter:function(e){S.globalEval(e,t,n)}})},S.fn.extend({wrapAll:function(e){var t;return this[0]&&(m(e)&&(e=e.call(this[0])),t=S(e,this[0].ownerDocument).eq(0).clone(!0),this[0].parentNode&&t.insertBefore(this[0]),t.map(function(){var e=this;while(e.firstElementChild)e=e.firstElementChild;return e}).append(this)),this},wrapInner:function(n){return m(n)?this.each(function(e){S(this).wrapInner(n.call(this,e))}):this.each(function(){var e=S(this),t=e.contents();t.length?t.wrapAll(n):e.append(n)})},wrap:function(t){var n=m(t);return this.each(function(e){S(this).wrapAll(n?t.call(this,e):t)})},unwrap:function(e){return this.parent(e).not("body").each(function(){S(this).replaceWith(this.childNodes)}),this}}),S.expr.pseudos.hidden=function(e){return!S.expr.pseudos.visible(e)},S.expr.pseudos.visible=function(e){return!!(e.offsetWidth||e.offsetHeight||e.getClientRects().length)},S.ajaxSettings.xhr=function(){try{return new C.XMLHttpRequest}catch(e){}};var _t={0:200,1223:204},zt=S.ajaxSettings.xhr();y.cors=!!zt&&"withCredentials"in zt,y.ajax=zt=!!zt,S.ajaxTransport(function(i){var o,a;if(y.cors||zt&&!i.crossDomain)return{send:function(e,t){var n,r=i.xhr();if(r.open(i.type,i.url,i.async,i.username,i.password),i.xhrFields)for(n in i.xhrFields)r[n]=i.xhrFields[n];for(n in i.mimeType&&r.overrideMimeType&&r.overrideMimeType(i.mimeType),i.crossDomain||e["X-Requested-With"]||(e["X-Requested-With"]="XMLHttpRequest"),e)r.setRequestHeader(n,e[n]);o=function(e){return function(){o&&(o=a=r.onload=r.onerror=r.onabort=r.ontimeout=r.onreadystatechange=null,"abort"===e?r.abort():"error"===e?"number"!=typeof r.status?t(0,"error"):t(r.status,r.statusText):t(_t[r.status]||r.status,r.statusText,"text"!==(r.responseType||"text")||"string"!=typeof r.responseText?{binary:r.response}:{text:r.responseText},r.getAllResponseHeaders()))}},r.onload=o(),a=r.onerror=r.ontimeout=o("error"),void 0!==r.onabort?r.onabort=a:r.onreadystatechange=function(){4===r.readyState&&C.setTimeout(function(){o&&a()})},o=o("abort");try{r.send(i.hasContent&&i.data||null)}catch(e){if(o)throw e}},abort:function(){o&&o()}}}),S.ajaxPrefilter(function(e){e.crossDomain&&(e.contents.script=!1)}),S.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},contents:{script:/\b(?:java|ecma)script\b/},converters:{"text script":function(e){return S.globalEval(e),e}}}),S.ajaxPrefilter("script",function(e){void 0===e.cache&&(e.cache=!1),e.crossDomain&&(e.type="GET")}),S.ajaxTransport("script",function(n){var r,i;if(n.crossDomain||n.scriptAttrs)return{send:function(e,t){r=S("<script>").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),E.head.appendChild(r[0])},abort:function(){i&&i()}}});var Ut,Xt=[],Vt=/(=)\?(?=&|$)|\?\?/;S.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=Xt.pop()||S.expando+"_"+Ct.guid++;return this[e]=!0,e}}),S.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Vt.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Vt.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=m(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Vt,"$1"+r):!1!==e.jsonp&&(e.url+=(Et.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||S.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=C[r],C[r]=function(){o=arguments},n.always(function(){void 0===i?S(C).removeProp(r):C[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,Xt.push(r)),o&&m(i)&&i(o[0]),o=i=void 0}),"script"}),y.createHTMLDocument=((Ut=E.implementation.createHTMLDocument("").body).innerHTML="<form></form><form></form>",2===Ut.childNodes.length),S.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(y.createHTMLDocument?((r=(t=E.implementation.createHTMLDocument("")).createElement("base")).href=E.location.href,t.head.appendChild(r)):t=E),o=!n&&[],(i=N.exec(e))?[t.createElement(i[1])]:(i=xe([e],t,o),o&&o.length&&S(o).remove(),S.merge([],i.childNodes)));var r,i,o},S.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1<s&&(r=vt(e.slice(s)),e=e.slice(0,s)),m(t)?(n=t,t=void 0):t&&"object"==typeof t&&(i="POST"),0<a.length&&S.ajax({url:e,type:i||"GET",dataType:"html",data:t}).done(function(e){o=arguments,a.html(r?S("<div>").append(S.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},S.expr.pseudos.animated=function(t){return S.grep(S.timers,function(e){return t===e.elem}).length},S.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=S.css(e,"position"),c=S(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=S.css(e,"top"),u=S.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),m(t)&&(t=t.call(e,n,S.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):("number"==typeof f.top&&(f.top+="px"),"number"==typeof f.left&&(f.left+="px"),c.css(f))}},S.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){S.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===S.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===S.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=S(e).offset()).top+=S.css(e,"borderTopWidth",!0),i.left+=S.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-S.css(r,"marginTop",!0),left:t.left-i.left-S.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===S.css(e,"position"))e=e.offsetParent;return e||re})}}),S.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;S.fn[t]=function(e){return $(this,function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),S.each(["top","left"],function(e,n){S.cssHooks[n]=$e(y.pixelPosition,function(e,t){if(t)return t=Be(e,n),Me.test(t)?S(e).position()[n]+"px":t})}),S.each({Height:"height",Width:"width"},function(a,s){S.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){S.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return $(this,function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?S.css(e,t,i):S.style(e,t,n,i)},s,n?e:void 0,n)}})}),S.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){S.fn[t]=function(e){return this.on(t,e)}}),S.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.mouseenter(e).mouseleave(t||e)}}),S.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){S.fn[n]=function(e,t){return 0<arguments.length?this.on(n,null,e,t):this.trigger(n)}});var Gt=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g;S.proxy=function(e,t){var n,r,i;if("string"==typeof t&&(n=e[t],t=e,e=n),m(e))return r=s.call(arguments,2),(i=function(){return e.apply(t||this,r.concat(s.call(arguments)))}).guid=e.guid=e.guid||S.guid++,i},S.holdReady=function(e){e?S.readyWait++:S.ready(!0)},S.isArray=Array.isArray,S.parseJSON=JSON.parse,S.nodeName=A,S.isFunction=m,S.isWindow=x,S.camelCase=X,S.type=w,S.now=Date.now,S.isNumeric=function(e){var t=S.type(e);return("number"===t||"string"===t)&&!isNaN(e-parseFloat(e))},S.trim=function(e){return null==e?"":(e+"").replace(Gt,"")},"function"==typeof define&&define.amd&&define("jquery",[],function(){return S});var Yt=C.jQuery,Qt=C.$;return S.noConflict=function(e){return C.$===S&&(C.$=Qt),e&&C.jQuery===S&&(C.jQuery=Yt),S},"undefined"==typeof e&&(C.jQuery=C.$=S),S});
diff --git a/docs/ml-classification-regression.md b/docs/ml-classification-regression.md
index 247989d16bcd7..bad74cbcf6cc9 100644
--- a/docs/ml-classification-regression.md
+++ b/docs/ml-classification-regression.md
@@ -85,7 +85,7 @@ More details on parameters can be found in the [Java API documentation](api/java
 
 <div data-lang="python" markdown="1">
 
-More details on parameters can be found in the [Python API documentation](api/python/pyspark.ml.html#pyspark.ml.classification.LogisticRegression).
+More details on parameters can be found in the [Python API documentation](api/python/reference/api/pyspark.ml.classification.LogisticRegression.html).
 
 {% include_example python/ml/logistic_regression_with_elastic_net.py %}
 </div>
@@ -135,11 +135,11 @@ Continuing the earlier example:
 </div>
 
 <div data-lang="python" markdown="1">
-[`LogisticRegressionTrainingSummary`](api/python/pyspark.ml.html#pyspark.ml.classification.LogisticRegressionSummary)
+[`LogisticRegressionTrainingSummary`](api/python/reference/api/pyspark.ml.classification.LogisticRegressionSummary.html)
 provides a summary for a
-[`LogisticRegressionModel`](api/python/pyspark.ml.html#pyspark.ml.classification.LogisticRegressionModel).
+[`LogisticRegressionModel`](api/python/reference/api/pyspark.ml.classification.LogisticRegressionModel.html).
 In the case of binary classification, certain additional metrics are
-available, e.g. ROC curve. See [`BinaryLogisticRegressionTrainingSummary`](api/python/pyspark.ml.html#pyspark.ml.classification.BinaryLogisticRegressionTrainingSummary).
+available, e.g. ROC curve. See [`BinaryLogisticRegressionTrainingSummary`](api/python/reference/api/pyspark.ml.classification.BinaryLogisticRegressionTrainingSummary.html).
 
 Continuing the earlier example:
 
@@ -232,7 +232,7 @@ More details on parameters can be found in the [Java API documentation](api/java
 
 <div data-lang="python" markdown="1">
 
-More details on parameters can be found in the [Python API documentation](api/python/pyspark.ml.html#pyspark.ml.classification.DecisionTreeClassifier).
+More details on parameters can be found in the [Python API documentation](api/python/reference/api/pyspark.ml.classification.DecisionTreeClassifier.html).
 
 {% include_example python/ml/decision_tree_classification_example.py %}
 
@@ -275,7 +275,7 @@ Refer to the [Java API docs](api/java/org/apache/spark/ml/classification/RandomF
 
 <div data-lang="python" markdown="1">
 
-Refer to the [Python API docs](api/python/pyspark.ml.html#pyspark.ml.classification.RandomForestClassifier) for more details.
+Refer to the [Python API docs](api/python/reference/api/pyspark.ml.classification.RandomForestClassifier.html) for more details.
 
 {% include_example python/ml/random_forest_classifier_example.py %}
 </div>
@@ -316,7 +316,7 @@ Refer to the [Java API docs](api/java/org/apache/spark/ml/classification/GBTClas
 
 <div data-lang="python" markdown="1">
 
-Refer to the [Python API docs](api/python/pyspark.ml.html#pyspark.ml.classification.GBTClassifier) for more details.
+Refer to the [Python API docs](api/python/reference/api/pyspark.ml.classification.GBTClassifier.html) for more details.
 
 {% include_example python/ml/gradient_boosted_tree_classifier_example.py %}
 </div>
@@ -372,7 +372,7 @@ Refer to the [Java API docs](api/java/org/apache/spark/ml/classification/Multila
 
 <div data-lang="python" markdown="1">
 
-Refer to the [Python API docs](api/python/pyspark.ml.html#pyspark.ml.classification.MultilayerPerceptronClassifier) for more details.
+Refer to the [Python API docs](api/python/reference/api/pyspark.ml.classification.MultilayerPerceptronClassifier.html) for more details.
 
 {% include_example python/ml/multilayer_perceptron_classification.py %}
 </div>
@@ -417,7 +417,7 @@ Refer to the [Java API docs](api/java/org/apache/spark/ml/classification/LinearS
 
 <div data-lang="python" markdown="1">
 
-Refer to the [Python API docs](api/python/pyspark.ml.html#pyspark.ml.classification.LinearSVC) for more details.
+Refer to the [Python API docs](api/python/reference/api/pyspark.ml.classification.LinearSVC.html) for more details.
 
 {% include_example python/ml/linearsvc.py %}
 </div>
@@ -461,7 +461,7 @@ Refer to the [Java API docs](api/java/org/apache/spark/ml/classification/OneVsRe
 
 <div data-lang="python" markdown="1">
 
-Refer to the [Python API docs](api/python/pyspark.ml.html#pyspark.ml.classification.OneVsRest) for more details.
+Refer to the [Python API docs](api/python/reference/api/pyspark.ml.classification.OneVsRest.html) for more details.
 
 {% include_example python/ml/one_vs_rest_example.py %}
 </div>
@@ -515,7 +515,7 @@ Refer to the [Java API docs](api/java/org/apache/spark/ml/classification/NaiveBa
 
 <div data-lang="python" markdown="1">
 
-Refer to the [Python API docs](api/python/pyspark.ml.html#pyspark.ml.classification.NaiveBayes) for more details.
+Refer to the [Python API docs](api/python/reference/api/pyspark.ml.classification.NaiveBayes.html) for more details.
 
 {% include_example python/ml/naive_bayes_example.py %}
 </div>
@@ -558,7 +558,7 @@ Refer to the [Java API docs](api/java/org/apache/spark/ml/classification/FMClass
 
 <div data-lang="python" markdown="1">
 
-Refer to the [Python API docs](api/python/pyspark.ml.html#pyspark.ml.classification.FMClassifier) for more details.
+Refer to the [Python API docs](api/python/reference/api/pyspark.ml.classification.FMClassifier.html) for more details.
 
 {% include_example python/ml/fm_classifier_example.py %}
 </div>
@@ -609,7 +609,7 @@ More details on parameters can be found in the [Java API documentation](api/java
 <div data-lang="python" markdown="1">
 <!--- TODO: Add python model summaries once implemented -->
 
-More details on parameters can be found in the [Python API documentation](api/python/pyspark.ml.html#pyspark.ml.regression.LinearRegression).
+More details on parameters can be found in the [Python API documentation](api/python/reference/api/pyspark.ml.regression.LinearRegression.html#pyspark.ml.regression.LinearRegression).
 
 {% include_example python/ml/linear_regression_with_elastic_net.py %}
 </div>
@@ -756,7 +756,7 @@ Refer to the [Java API docs](api/java/org/apache/spark/ml/regression/Generalized
 
 <div data-lang="python" markdown="1">
 
-Refer to the [Python API docs](api/python/pyspark.ml.html#pyspark.ml.regression.GeneralizedLinearRegression) for more details.
+Refer to the [Python API docs](api/python/reference/api/pyspark.ml.regression.GeneralizedLinearRegression.html#pyspark.ml.regression.GeneralizedLinearRegression) for more details.
 
 {% include_example python/ml/generalized_linear_regression_example.py %}
 </div>
@@ -798,7 +798,7 @@ More details on parameters can be found in the [Java API documentation](api/java
 
 <div data-lang="python" markdown="1">
 
-More details on parameters can be found in the [Python API documentation](api/python/pyspark.ml.html#pyspark.ml.regression.DecisionTreeRegressor).
+More details on parameters can be found in the [Python API documentation](api/python/reference/api/pyspark.ml.regression.DecisionTreeRegressor.html#pyspark.ml.regression.DecisionTreeRegressor).
 
 {% include_example python/ml/decision_tree_regression_example.py %}
 </div>
@@ -840,7 +840,7 @@ Refer to the [Java API docs](api/java/org/apache/spark/ml/regression/RandomFores
 
 <div data-lang="python" markdown="1">
 
-Refer to the [Python API docs](api/python/pyspark.ml.html#pyspark.ml.regression.RandomForestRegressor) for more details.
+Refer to the [Python API docs](api/python/reference/api/pyspark.ml.regression.RandomForestRegressor.html#pyspark.ml.regression.RandomForestRegressor) for more details.
 
 {% include_example python/ml/random_forest_regressor_example.py %}
 </div>
@@ -881,7 +881,7 @@ Refer to the [Java API docs](api/java/org/apache/spark/ml/regression/GBTRegresso
 
 <div data-lang="python" markdown="1">
 
-Refer to the [Python API docs](api/python/pyspark.ml.html#pyspark.ml.regression.GBTRegressor) for more details.
+Refer to the [Python API docs](api/python/reference/api/pyspark.ml.regression.GBTRegressor.html#pyspark.ml.regression.GBTRegressor) for more details.
 
 {% include_example python/ml/gradient_boosted_tree_regressor_example.py %}
 </div>
@@ -975,7 +975,7 @@ Refer to the [Java API docs](api/java/org/apache/spark/ml/regression/AFTSurvival
 
 <div data-lang="python" markdown="1">
 
-Refer to the [Python API docs](api/python/pyspark.ml.html#pyspark.ml.regression.AFTSurvivalRegression) for more details.
+Refer to the [Python API docs](api/python/reference/api/pyspark.ml.regression.AFTSurvivalRegression.html#pyspark.ml.regression.AFTSurvivalRegression) for more details.
 
 {% include_example python/ml/aft_survival_regression.py %}
 </div>
@@ -1053,7 +1053,7 @@ Refer to the [`IsotonicRegression` Java docs](api/java/org/apache/spark/ml/regre
 </div>
 <div data-lang="python" markdown="1">
 
-Refer to the [`IsotonicRegression` Python docs](api/python/pyspark.ml.html#pyspark.ml.regression.IsotonicRegression) for more details on the API.
+Refer to the [`IsotonicRegression` Python docs](api/python/reference/api/pyspark.ml.regression.IsotonicRegression.html#pyspark.ml.regression.IsotonicRegression) for more details on the API.
 
 {% include_example python/ml/isotonic_regression_example.py %}
 </div>
@@ -1096,7 +1096,7 @@ Refer to the [Java API docs](api/java/org/apache/spark/ml/regression/FMRegressor
 
 <div data-lang="python" markdown="1">
 
-Refer to the [Python API docs](api/python/pyspark.ml.html#pyspark.ml.regression.FMRegressor) for more details.
+Refer to the [Python API docs](api/python/reference/api/pyspark.ml.regression.FMRegressor.html#pyspark.ml.regression.FMRegressor) for more details.
 
 {% include_example python/ml/fm_regressor_example.py %}
 </div>
diff --git a/docs/ml-clustering.md b/docs/ml-clustering.md
index 4574567fa9d50..f478776196d42 100644
--- a/docs/ml-clustering.md
+++ b/docs/ml-clustering.md
@@ -97,7 +97,7 @@ Refer to the [Java API docs](api/java/org/apache/spark/ml/clustering/KMeans.html
 </div>
 
 <div data-lang="python" markdown="1">
-Refer to the [Python API docs](api/python/pyspark.ml.html#pyspark.ml.clustering.KMeans) for more details.
+Refer to the [Python API docs](api/python/reference/api/pyspark.ml.clustering.KMeans.html) for more details.
 
 {% include_example python/ml/kmeans_example.py %}
 </div>
@@ -137,7 +137,7 @@ Refer to the [Java API docs](api/java/org/apache/spark/ml/clustering/LDA.html) f
 
 <div data-lang="python" markdown="1">
 
-Refer to the [Python API docs](api/python/pyspark.ml.html#pyspark.ml.clustering.LDA) for more details.
+Refer to the [Python API docs](api/python/reference/api/pyspark.ml.clustering.LDA.html) for more details.
 
 {% include_example python/ml/lda_example.py %}
 </div>
@@ -178,7 +178,7 @@ Refer to the [Java API docs](api/java/org/apache/spark/ml/clustering/BisectingKM
 </div>
 
 <div data-lang="python" markdown="1">
-Refer to the [Python API docs](api/python/pyspark.ml.html#pyspark.ml.clustering.BisectingKMeans) for more details.
+Refer to the [Python API docs](api/python/reference/api/pyspark.ml.clustering.BisectingKMeans.html) for more details.
 
 {% include_example python/ml/bisecting_k_means_example.py %}
 </div>
@@ -267,7 +267,7 @@ Refer to the [Java API docs](api/java/org/apache/spark/ml/clustering/GaussianMix
 </div>
 
 <div data-lang="python" markdown="1">
-Refer to the [Python API docs](api/python/pyspark.ml.html#pyspark.ml.clustering.GaussianMixture) for more details.
+Refer to the [Python API docs](api/python/reference/api/pyspark.ml.clustering.GaussianMixture.html) for more details.
 
 {% include_example python/ml/gaussian_mixture_example.py %}
 </div>
@@ -314,7 +314,7 @@ Refer to the [Java API docs](api/java/org/apache/spark/ml/clustering/PowerIterat
 </div>
 
 <div data-lang="python" markdown="1">
-Refer to the [Python API docs](api/python/pyspark.ml.html#pyspark.ml.clustering.PowerIterationClustering) for more details.
+Refer to the [Python API docs](api/python/reference/api/pyspark.ml.clustering.PowerIterationClustering.html) for more details.
 
 {% include_example python/ml/power_iteration_clustering_example.py %}
 </div>
diff --git a/docs/ml-collaborative-filtering.md b/docs/ml-collaborative-filtering.md
index 6c41efd5cc306..ddc90406648a4 100644
--- a/docs/ml-collaborative-filtering.md
+++ b/docs/ml-collaborative-filtering.md
@@ -177,7 +177,7 @@ explicit (`implicitPrefs` is `False`).
 We evaluate the recommendation model by measuring the root-mean-square error of
 rating prediction.
 
-Refer to the [`ALS` Python docs](api/python/pyspark.ml.html#pyspark.ml.recommendation.ALS)
+Refer to the [`ALS` Python docs](api/python/reference/api/pyspark.ml.recommendation.ALS.html)
 for more details on the API.
 
 {% include_example python/ml/als_example.py %}
diff --git a/docs/ml-datasource.md b/docs/ml-datasource.md
index 0f2f5f482ec50..8e9c947b75f38 100644
--- a/docs/ml-datasource.md
+++ b/docs/ml-datasource.md
@@ -86,7 +86,7 @@ Will output:
 In PySpark we provide Spark SQL data source API for loading image data as a DataFrame.
 
 {% highlight python %}
->>> df = spark.read.format("image").option("dropInvalid", true).load("data/mllib/images/origin/kittens")
+>>> df = spark.read.format("image").option("dropInvalid", True).load("data/mllib/images/origin/kittens")
 >>> df.select("image.origin", "image.width", "image.height").show(truncate=False)
 +-----------------------------------------------------------------------+-----+------+
 |origin                                                                 |width|height|
diff --git a/docs/ml-features.md b/docs/ml-features.md
index 660c27250ebfb..e01acfd0b979d 100644
--- a/docs/ml-features.md
+++ b/docs/ml-features.md
@@ -112,8 +112,8 @@ Refer to the [HashingTF Java docs](api/java/org/apache/spark/ml/feature/HashingT
 
 <div data-lang="python" markdown="1">
 
-Refer to the [HashingTF Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.HashingTF) and
-the [IDF Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.IDF) for more details on the API.
+Refer to the [HashingTF Python docs](api/python/reference/api/pyspark.ml.feature.HashingTF.html) and
+the [IDF Python docs](api/python/reference/api/pyspark.ml.feature.IDF.html) for more details on the API.
 
 {% include_example python/ml/tf_idf_example.py %}
 </div>
@@ -151,7 +151,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [Word2Vec Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.Word2Vec)
+Refer to the [Word2Vec Python docs](api/python/reference/api/pyspark.ml.feature.Word2Vec.html)
 for more details on the API.
 
 {% include_example python/ml/word2vec_example.py %}
@@ -218,8 +218,8 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [CountVectorizer Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.CountVectorizer)
-and the [CountVectorizerModel Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.CountVectorizerModel)
+Refer to the [CountVectorizer Python docs](api/python/reference/api/pyspark.ml.feature.CountVectorizer.html)
+and the [CountVectorizerModel Python docs](api/python/reference/api/pyspark.ml.feature.CountVectorizerModel.html)
 for more details on the API.
 
 {% include_example python/ml/count_vectorizer_example.py %}
@@ -302,7 +302,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [FeatureHasher Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.FeatureHasher)
+Refer to the [FeatureHasher Python docs](api/python/reference/api/pyspark.ml.feature.FeatureHasher.html)
 for more details on the API.
 
 {% include_example python/ml/feature_hasher_example.py %}
@@ -344,8 +344,8 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [Tokenizer Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.Tokenizer) and
-the [RegexTokenizer Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.RegexTokenizer)
+Refer to the [Tokenizer Python docs](api/python/reference/api/pyspark.ml.feature.Tokenizer.html) and
+the [RegexTokenizer Python docs](api/python/reference/api/pyspark.ml.feature.RegexTokenizer.html)
 for more details on the API.
 
 {% include_example python/ml/tokenizer_example.py %}
@@ -411,7 +411,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [StopWordsRemover Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.StopWordsRemover)
+Refer to the [StopWordsRemover Python docs](api/python/reference/api/pyspark.ml.feature.StopWordsRemover.html)
 for more details on the API.
 
 {% include_example python/ml/stopwords_remover_example.py %}
@@ -446,7 +446,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [NGram Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.NGram)
+Refer to the [NGram Python docs](api/python/reference/api/pyspark.ml.feature.NGram.html)
 for more details on the API.
 
 {% include_example python/ml/n_gram_example.py %}
@@ -484,7 +484,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [Binarizer Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.Binarizer)
+Refer to the [Binarizer Python docs](api/python/reference/api/pyspark.ml.feature.Binarizer.html)
 for more details on the API.
 
 {% include_example python/ml/binarizer_example.py %}
@@ -516,7 +516,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [PCA Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.PCA)
+Refer to the [PCA Python docs](api/python/reference/api/pyspark.ml.feature.PCA.html)
 for more details on the API.
 
 {% include_example python/ml/pca_example.py %}
@@ -548,7 +548,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [PolynomialExpansion Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.PolynomialExpansion)
+Refer to the [PolynomialExpansion Python docs](api/python/reference/api/pyspark.ml.feature.PolynomialExpansion.html)
 for more details on the API.
 
 {% include_example python/ml/polynomial_expansion_example.py %}
@@ -590,7 +590,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [DCT Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.DCT)
+Refer to the [DCT Python docs](api/python/reference/api/pyspark.ml.feature.DCT.html)
 for more details on the API.
 
 {% include_example python/ml/dct_example.py %}
@@ -720,7 +720,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [StringIndexer Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.StringIndexer)
+Refer to the [StringIndexer Python docs](api/python/reference/api/pyspark.ml.feature.StringIndexer.html)
 for more details on the API.
 
 {% include_example python/ml/string_indexer_example.py %}
@@ -788,7 +788,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [IndexToString Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.IndexToString)
+Refer to the [IndexToString Python docs](api/python/reference/api/pyspark.ml.feature.IndexToString.html)
 for more details on the API.
 
 {% include_example python/ml/index_to_string_example.py %}
@@ -824,7 +824,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [OneHotEncoder Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.OneHotEncoder) for more details on the API.
+Refer to the [OneHotEncoder Python docs](api/python/reference/api/pyspark.ml.feature.OneHotEncoder.html) for more details on the API.
 
 {% include_example python/ml/onehot_encoder_example.py %}
 </div>
@@ -865,7 +865,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [VectorIndexer Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.VectorIndexer)
+Refer to the [VectorIndexer Python docs](api/python/reference/api/pyspark.ml.feature.VectorIndexer.html)
 for more details on the API.
 
 {% include_example python/ml/vector_indexer_example.py %}
@@ -926,7 +926,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [Interaction Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.Interaction)
+Refer to the [Interaction Python docs](api/python/reference/api/pyspark.ml.feature.Interaction.html)
 for more details on the API.
 
 {% include_example python/ml/interaction_example.py %}
@@ -960,7 +960,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [Normalizer Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.Normalizer)
+Refer to the [Normalizer Python docs](api/python/reference/api/pyspark.ml.feature.Normalizer.html)
 for more details on the API.
 
 {% include_example python/ml/normalizer_example.py %}
@@ -1002,7 +1002,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [StandardScaler Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.StandardScaler)
+Refer to the [StandardScaler Python docs](api/python/reference/api/pyspark.ml.feature.StandardScaler.html)
 for more details on the API.
 
 {% include_example python/ml/standard_scaler_example.py %}
@@ -1046,7 +1046,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [RobustScaler Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.RobustScaler)
+Refer to the [RobustScaler Python docs](api/python/reference/api/pyspark.ml.feature.RobustScaler.html)
 for more details on the API.
 
 {% include_example python/ml/robust_scaler_example.py %}
@@ -1096,8 +1096,8 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [MinMaxScaler Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.MinMaxScaler)
-and the [MinMaxScalerModel Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.MinMaxScalerModel)
+Refer to the [MinMaxScaler Python docs](api/python/reference/api/pyspark.ml.feature.MinMaxScaler.html)
+and the [MinMaxScalerModel Python docs](api/python/reference/api/pyspark.ml.feature.MinMaxScalerModel.html)
 for more details on the API.
 
 {% include_example python/ml/min_max_scaler_example.py %}
@@ -1139,8 +1139,8 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [MaxAbsScaler Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.MaxAbsScaler)
-and the [MaxAbsScalerModel Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.MaxAbsScalerModel)
+Refer to the [MaxAbsScaler Python docs](api/python/reference/api/pyspark.ml.feature.MaxAbsScaler.html)
+and the [MaxAbsScalerModel Python docs](api/python/reference/api/pyspark.ml.feature.MaxAbsScalerModel.html)
 for more details on the API.
 
 {% include_example python/ml/max_abs_scaler_example.py %}
@@ -1182,7 +1182,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [Bucketizer Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.Bucketizer)
+Refer to the [Bucketizer Python docs](api/python/reference/api/pyspark.ml.feature.Bucketizer.html)
 for more details on the API.
 
 {% include_example python/ml/bucketizer_example.py %}
@@ -1232,7 +1232,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [ElementwiseProduct Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.ElementwiseProduct)
+Refer to the [ElementwiseProduct Python docs](api/python/reference/api/pyspark.ml.feature.ElementwiseProduct.html)
 for more details on the API.
 
 {% include_example python/ml/elementwise_product_example.py %}
@@ -1292,7 +1292,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [SQLTransformer Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.SQLTransformer) for more details on the API.
+Refer to the [SQLTransformer Python docs](api/python/reference/api/pyspark.ml.feature.SQLTransformer.html) for more details on the API.
 
 {% include_example python/ml/sql_transformer.py %}
 </div>
@@ -1352,7 +1352,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [VectorAssembler Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.VectorAssembler)
+Refer to the [VectorAssembler Python docs](api/python/reference/api/pyspark.ml.feature.VectorAssembler.html)
 for more details on the API.
 
 {% include_example python/ml/vector_assembler_example.py %}
@@ -1403,7 +1403,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [VectorSizeHint Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.VectorSizeHint)
+Refer to the [VectorSizeHint Python docs](api/python/reference/api/pyspark.ml.feature.VectorSizeHint.html)
 for more details on the API.
 
 {% include_example python/ml/vector_size_hint_example.py %}
@@ -1486,7 +1486,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [QuantileDiscretizer Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.QuantileDiscretizer)
+Refer to the [QuantileDiscretizer Python docs](api/python/reference/api/pyspark.ml.feature.QuantileDiscretizer.html)
 for more details on the API.
 
 {% include_example python/ml/quantile_discretizer_example.py %}
@@ -1497,8 +1497,8 @@ for more details on the API.
 
 ## Imputer
 
-The `Imputer` estimator completes missing values in a dataset, either using the mean or the 
-median of the columns in which the missing values are located. The input columns should be of
+The `Imputer` estimator completes missing values in a dataset, using the mean, median or mode
+of the columns in which the missing values are located. The input columns should be of
 numeric type. Currently `Imputer` does not support categorical features and possibly
 creates incorrect values for columns containing categorical features. Imputer can impute custom values 
 other than 'NaN' by `.setMissingValue(custom_value)`. For example, `.setMissingValue(0)` will impute 
@@ -1555,7 +1555,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [Imputer Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.Imputer)
+Refer to the [Imputer Python docs](api/python/reference/api/pyspark.ml.feature.Imputer.html)
 for more details on the API.
 
 {% include_example python/ml/imputer_example.py %}
@@ -1636,7 +1636,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [VectorSlicer Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.VectorSlicer)
+Refer to the [VectorSlicer Python docs](api/python/reference/api/pyspark.ml.feature.VectorSlicer.html)
 for more details on the API.
 
 {% include_example python/ml/vector_slicer_example.py %}
@@ -1722,7 +1722,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [RFormula Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.RFormula)
+Refer to the [RFormula Python docs](api/python/reference/api/pyspark.ml.feature.RFormula.html)
 for more details on the API.
 
 {% include_example python/ml/rformula_example.py %}
@@ -1786,26 +1786,35 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [ChiSqSelector Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.ChiSqSelector)
+Refer to the [ChiSqSelector Python docs](api/python/reference/api/pyspark.ml.feature.ChiSqSelector.html)
 for more details on the API.
 
 {% include_example python/ml/chisq_selector_example.py %}
 </div>
 </div>
 
-## ANOVASelector
+## UnivariateFeatureSelector
 
-`ANOVASelector` operates on categorical labels with continuous features. It uses the
-[one-way ANOVA F-test](https://en.wikipedia.org/wiki/F-test#Multiple-comparison_ANOVA_problems) to decide which
-features to choose.
-It supports five selection methods: `numTopFeatures`, `percentile`, `fpr`, `fdr`, `fwe`:
-* `numTopFeatures` chooses a fixed number of top features according to ANOVA F-test.
+`UnivariateFeatureSelector` operates on categorical/continuous labels with categorical/continuous features. 
+User can set `featureType` and `labelType`, and Spark will pick the score function to use based on the specified 
+`featureType` and `labelType`. 
+
+~~~
+featureType |  labelType |score function
+------------|------------|--------------
+categorical |categorical | chi-squared (chi2)
+continuous  |categorical | ANOVATest (f_classif)
+continuous  |continuous  | F-value (f_regression)
+~~~
+
+It supports five selection modes: `numTopFeatures`, `percentile`, `fpr`, `fdr`, `fwe`:
+* `numTopFeatures` chooses a fixed number of top features.
 * `percentile` is similar to `numTopFeatures` but chooses a fraction of all features instead of a fixed number.
 * `fpr` chooses all features whose p-values are below a threshold, thus controlling the false positive rate of selection.
 * `fdr` uses the [Benjamini-Hochberg procedure](https://en.wikipedia.org/wiki/False_discovery_rate#Benjamini.E2.80.93Hochberg_procedure) to choose all features whose false discovery rate is below a threshold.
 * `fwe` chooses all features whose p-values are below a threshold. The threshold is scaled by 1/numFeatures, thus controlling the family-wise error rate of selection.
-By default, the selection method is `numTopFeatures`, with the default number of top features set to 50.
-The user can choose a selection method using `setSelectorType`.
+
+By default, the selection mode is `numTopFeatures`, with the default selectionThreshold sets to 50.
 
 **Examples**
 
@@ -1823,7 +1832,7 @@ id | features                       | label
  6 | [7.9, 8.5, 9.2, 4.0, 9.4, 2.1] | 4.0
 ~~~
 
-If we use `ANOVASelector` with `numTopFeatures = 1`, the
+If we set `featureType` to `continuous` and `labelType` to `categorical` with `numTopFeatures = 1`, the
 last column in our `features` is chosen as the most useful feature:
 
 ~~~
@@ -1840,96 +1849,26 @@ id | features                       | label   | selectedFeatures
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [ANOVASelector Scala docs](api/scala/org/apache/spark/ml/feature/ANOVASelector.html)
-for more details on the API.
-
-{% include_example scala/org/apache/spark/examples/ml/ANOVASelectorExample.scala %}
-</div>
-
-<div data-lang="java" markdown="1">
-
-Refer to the [ANOVASelector Java docs](api/java/org/apache/spark/ml/feature/ANOVASelector.html)
-for more details on the API.
-
-{% include_example java/org/apache/spark/examples/ml/JavaANOVASelectorExample.java %}
-</div>
-
-<div data-lang="python" markdown="1">
-
-Refer to the [ANOVASelector Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.ANOVASelector)
-for more details on the API.
-
-{% include_example python/ml/anova_selector_example.py %}
-</div>
-</div>
-
-## FValueSelector
-
-`FValueSelector` operates on categorical labels with continuous features. It uses the
-[F-test for regression](https://en.wikipedia.org/wiki/F-test#Regression_problems) to decide which
-features to choose.
-It supports five selection methods: `numTopFeatures`, `percentile`, `fpr`, `fdr`, `fwe`:
-* `numTopFeatures` chooses a fixed number of top features according to a F-test for regression.
-* `percentile` is similar to `numTopFeatures` but chooses a fraction of all features instead of a fixed number.
-* `fpr` chooses all features whose p-values are below a threshold, thus controlling the false positive rate of selection.
-* `fdr` uses the [Benjamini-Hochberg procedure](https://en.wikipedia.org/wiki/False_discovery_rate#Benjamini.E2.80.93Hochberg_procedure) to choose all features whose false discovery rate is below a threshold.
-* `fwe` chooses all features whose p-values are below a threshold. The threshold is scaled by 1/numFeatures, thus controlling the family-wise error rate of selection.
-By default, the selection method is `numTopFeatures`, with the default number of top features set to 50.
-The user can choose a selection method using `setSelectorType`.
-
-**Examples**
-
-Assume that we have a DataFrame with the columns `id`, `features`, and `label`, which is used as
-our target to be predicted:
-
-~~~
-id | features                       | label
----|--------------------------------|---------
- 1 | [6.0, 7.0, 0.0, 7.0, 6.0, 0.0] | 4.6
- 2 | [0.0, 9.0, 6.0, 0.0, 5.0, 9.0] | 6.6
- 3 | [0.0, 9.0, 3.0, 0.0, 5.0, 5.0] | 5.1
- 4 | [0.0, 9.0, 8.0, 5.0, 6.0, 4.0] | 7.6
- 5 | [8.0, 9.0, 6.0, 5.0, 4.0, 4.0] | 9.0
- 6 | [8.0, 9.0, 6.0, 4.0, 0.0, 0.0] | 9.0
-~~~
-
-If we use `FValueSelector` with `numTopFeatures = 1`, the
-3rd column in our `features` is chosen as the most useful feature:
-
-~~~
-id | features                       | label   | selectedFeatures
----|--------------------------------|---------|------------------
- 1 | [6.0, 7.0, 0.0, 7.0, 6.0, 0.0] | 4.6     | [0.0]
- 2 | [0.0, 9.0, 6.0, 0.0, 5.0, 9.0] | 6.6     | [6.0]
- 3 | [0.0, 9.0, 3.0, 0.0, 5.0, 5.0] | 5.1     | [3.0]
- 4 | [0.0, 9.0, 8.0, 5.0, 6.0, 4.0] | 7.6     | [8.0]
- 5 | [8.0, 9.0, 6.0, 5.0, 4.0, 4.0] | 9.0     | [6.0]
- 6 | [8.0, 9.0, 6.0, 4.0, 0.0, 0.0] | 9.0     | [6.0]
-~~~
-
-<div class="codetabs">
-<div data-lang="scala" markdown="1">
-
-Refer to the [FValueSelector Scala docs](api/scala/org/apache/spark/ml/feature/FValueSelector.html)
+Refer to the [UnivariateFeatureSelector Scala docs](api/scala/org/apache/spark/ml/feature/UnivariateFeatureSelector.html)
 for more details on the API.
 
-{% include_example scala/org/apache/spark/examples/ml/FValueSelectorExample.scala %}
+{% include_example scala/org/apache/spark/examples/ml/UnivariateFeatureSelectorExample.scala %}
 </div>
 
 <div data-lang="java" markdown="1">
 
-Refer to the [FValueSelector Java docs](api/java/org/apache/spark/ml/feature/FValueSelector.html)
+Refer to the [UnivariateFeatureSelector Java docs](api/java/org/apache/spark/ml/feature/UnivariateFeatureSelector.html)
 for more details on the API.
 
-{% include_example java/org/apache/spark/examples/ml/JavaFValueSelectorExample.java %}
+{% include_example java/org/apache/spark/examples/ml/JavaUnivariateFeatureSelectorExample.java %}
 </div>
 
 <div data-lang="python" markdown="1">
 
-Refer to the [FValueSelector Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.FValueSelector)
+Refer to the [UnivariateFeatureSelector Python docs](api/python/reference/api/pyspark.ml.feature.UnivariateFeatureSelector.html)
 for more details on the API.
 
-{% include_example python/ml/anova_selector_example.py %}
+{% include_example python/ml/univariate_feature_selector_example.py %}
 </div>
 </div>
 
@@ -1974,7 +1913,7 @@ id | features                       | selectedFeatures
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [VarianceThresholdSelector Scala docs]((api/python/pyspark.ml.html#pyspark.ml.feature.ChiSqSelector))
+Refer to the [VarianceThresholdSelector Scala docs](api/scala/org/apache/spark/ml/feature/VarianceThresholdSelector.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/VarianceThresholdSelectorExample.scala %}
@@ -1990,7 +1929,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [VarianceThresholdSelector Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.VarianceThresholdSelector)
+Refer to the [VarianceThresholdSelector Python docs](api/python/reference/api/pyspark.ml.feature.VarianceThresholdSelector.html)
 for more details on the API.
 
 {% include_example python/ml/variance_threshold_selector_example.py %}
@@ -2076,7 +2015,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [BucketedRandomProjectionLSH Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.BucketedRandomProjectionLSH)
+Refer to the [BucketedRandomProjectionLSH Python docs](api/python/reference/api/pyspark.ml.feature.BucketedRandomProjectionLSH.html)
 for more details on the API.
 
 {% include_example python/ml/bucketed_random_projection_lsh_example.py %}
@@ -2117,7 +2056,7 @@ for more details on the API.
 
 <div data-lang="python" markdown="1">
 
-Refer to the [MinHashLSH Python docs](api/python/pyspark.ml.html#pyspark.ml.feature.MinHashLSH)
+Refer to the [MinHashLSH Python docs](api/python/reference/api/pyspark.ml.feature.MinHashLSH.html)
 for more details on the API.
 
 {% include_example python/ml/min_hash_lsh_example.py %}
diff --git a/docs/ml-frequent-pattern-mining.md b/docs/ml-frequent-pattern-mining.md
index 42d7e50357391..80c9580c28954 100644
--- a/docs/ml-frequent-pattern-mining.md
+++ b/docs/ml-frequent-pattern-mining.md
@@ -87,7 +87,7 @@ Refer to the [Java API docs](api/java/org/apache/spark/ml/fpm/FPGrowth.html) for
 </div>
 
 <div data-lang="python" markdown="1">
-Refer to the [Python API docs](api/python/pyspark.ml.html#pyspark.ml.fpm.FPGrowth) for more details.
+Refer to the [Python API docs](api/python/reference/api/pyspark.ml.fpm.FPGrowth.html) for more details.
 
 {% include_example python/ml/fpgrowth_example.py %}
 </div>
@@ -140,7 +140,7 @@ Refer to the [Java API docs](api/java/org/apache/spark/ml/fpm/PrefixSpan.html) f
 </div>
 
 <div data-lang="python" markdown="1">
-Refer to the [Python API docs](api/python/pyspark.ml.html#pyspark.ml.fpm.PrefixSpan) for more details.
+Refer to the [Python API docs](api/python/reference/api/pyspark.ml.fpm.PrefixSpan) for more details.
 
 {% include_example python/ml/prefixspan_example.py %}
 </div>
diff --git a/docs/ml-guide.md b/docs/ml-guide.md
index ddce98b32f941..3202647240b92 100644
--- a/docs/ml-guide.md
+++ b/docs/ml-guide.md
@@ -62,23 +62,12 @@ The primary Machine Learning API for Spark is now the [DataFrame](sql-programmin
 
 # Dependencies
 
-MLlib uses the linear algebra package [Breeze](http://www.scalanlp.org/), which depends on
-[netlib-java](https://github.com/fommil/netlib-java) for optimised numerical processing.
-If native libraries[^1] are not available at runtime, you will see a warning message and a pure JVM
-implementation will be used instead.
+MLlib uses linear algebra packages [Breeze](http://www.scalanlp.org/), [dev.ludovic.netlib](https://github.com/luhenry/netlib), and [netlib-java](https://github.com/fommil/netlib-java) for optimised numerical processing[^1]. Those packages may call native acceleration libraries such as [Intel MKL](https://software.intel.com/content/www/us/en/develop/tools/math-kernel-library.html) or [OpenBLAS](http://www.openblas.net) if they are available as system libraries or in runtime library paths.
 
-Due to licensing issues with runtime proprietary binaries, we do not include `netlib-java`'s native
-proxies by default.
-To configure `netlib-java` / Breeze to use system optimised binaries, include
-`com.github.fommil.netlib:all:1.1.2` (or build Spark with `-Pnetlib-lgpl`) as a dependency of your
-project and read the [netlib-java](https://github.com/fommil/netlib-java) documentation for your
-platform's additional installation instructions.
-
-The most popular native BLAS such as [Intel MKL](https://software.intel.com/en-us/mkl), [OpenBLAS](http://www.openblas.net), can use multiple threads in a single operation, which can conflict with Spark's execution model.
-
-Configuring these BLAS implementations to use a single thread for operations may actually improve performance (see [SPARK-21305](https://issues.apache.org/jira/browse/SPARK-21305)). It is usually optimal to match this to the number of cores each Spark task is configured to use, which is 1 by default and typically left at 1.
-
-Please refer to resources like the following to understand how to configure the number of threads these BLAS implementations use: [Intel MKL](https://software.intel.com/en-us/articles/recommended-settings-for-calling-intel-mkl-routines-from-multi-threaded-applications) or [Intel oneMKL](https://software.intel.com/en-us/onemkl-linux-developer-guide-improving-performance-with-threading) and [OpenBLAS](https://github.com/xianyi/OpenBLAS/wiki/faq#multi-threaded). Note that if nativeBLAS is not properly configured in system, java implementation(f2jBLAS) will be used as fallback option.
+However, native acceleration libraries can't be distributed with Spark. See [MLlib Linear Algebra Acceleration Guide](ml-linalg-guide.html) for how to enable accelerated linear algebra processing. If accelerated native libraries are not enabled, you will see a warning message like below and a pure JVM implementation will be used instead:
+```
+WARN BLAS: Failed to load implementation from:dev.ludovic.netlib.blas.JNIBLAS
+```
 
 To use MLlib in Python, you will need [NumPy](http://www.numpy.org) version 1.4 or newer.
 
diff --git a/docs/ml-linalg-guide.md b/docs/ml-linalg-guide.md
new file mode 100644
index 0000000000000..1e3d9ccbc82ea
--- /dev/null
+++ b/docs/ml-linalg-guide.md
@@ -0,0 +1,93 @@
+---
+layout: global
+title: MLlib Linear Algebra Acceleration Guide
+displayTitle: MLlib Linear Algebra Acceleration Guide
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+     http://www.apache.org/licenses/LICENSE-2.0
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+## Introduction
+
+This guide provides necessary information to enable accelerated linear algebra processing for Spark MLlib.
+
+Spark MLlib defines Vector and Matrix as basic data types for machine learning algorithms. On top of them, [BLAS](https://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms) and [LAPACK](https://en.wikipedia.org/wiki/LAPACK) operations are implemented and supported by [dev.ludovic.netlib](https://github.com/luhenry/netlib) (the algorithms may also call [Breeze](https://github.com/scalanlp/breeze)). `dev.ludovic.netlib` can use optimized native linear algebra libraries (refered to as "native libraries" or "BLAS libraries" hereafter) for faster numerical processing. [Intel MKL](https://software.intel.com/content/www/us/en/develop/tools/math-kernel-library.html) and [OpenBLAS](http://www.openblas.net) are two popular ones.
+
+The official released Spark binaries don't contain these native libraries.
+
+The following sections describe how to install native libraries, configure them properly, and how to point `dev.ludovic.netlib` to these native libraries.
+
+## Install native linear algebra libraries
+
+Intel MKL and OpenBLAS are two popular native linear algebra libraries. You can choose one of them based on your preference. We provide basic instructions as below.
+
+### Intel MKL
+
+- Download and install Intel MKL. The installation should be done on all nodes of the cluster. We assume the installation location is $MKLROOT (e.g. /opt/intel/mkl).
+- Create soft links to `libmkl_rt.so` with specific names in system library search paths. For instance, make sure `/usr/local/lib` is in system library search paths and run the following commands:
+```
+$ ln -sf $MKLROOT/lib/intel64/libmkl_rt.so /usr/local/lib/libblas.so.3
+$ ln -sf $MKLROOT/lib/intel64/libmkl_rt.so /usr/local/lib/liblapack.so.3
+```
+
+### OpenBLAS
+
+The installation should be done on all nodes of the cluster. Generic version of OpenBLAS are available with most distributions. You can install it with a distribution package manager like `apt` or `yum`.
+
+For Debian / Ubuntu:
+```
+sudo apt-get install libopenblas-base
+sudo update-alternatives --config libblas.so.3
+```
+For CentOS / RHEL:
+```
+sudo yum install openblas
+```
+
+## Check if native libraries are enabled for MLlib
+
+To verify native libraries are properly loaded, start `spark-shell` and run the following code:
+```
+scala> import dev.ludovic.netlib.NativeBLAS
+scala> NativeBLAS.getInstance()
+```
+
+If they are correctly loaded, it should print `dev.ludovic.netlib.NativeBLAS = dev.ludovic.netlib.blas.JNIBLAS@...`. Otherwise the warnings should be printed:
+```
+WARN NativeBLAS: Failed to load implementation from:dev.ludovic.netlib.blas.JNIBLAS
+java.lang.RuntimeException: Unable to load native implementation
+  at dev.ludovic.netlib.NativeBLAS.getInstance(NativeBLAS.java:44)
+  ...
+```
+
+You can also point `dev.ludovic.netlib` to specific libraries names and paths. For example, `-Ddev.ludovic.netlib.blas.nativeLib=libmkl_rt.so` or `-Ddev.ludovic.netlib.blas.nativeLibPath=$MKLROOT/lib/intel64/libmkl_rt.so` for Intel MKL. You have similar parameters for LAPACK and ARPACK: `-Ddev.ludovic.netlib.lapack.nativeLib=...`, `-Ddev.ludovic.netlib.lapack.nativeLibPath=...`, `-Ddev.ludovic.netlib.arpack.nativeLib=...`, and `-Ddev.ludovic.netlib.arpack.nativeLibPath=...`.
+
+If native libraries are not properly configured in the system, the Java implementation (javaBLAS) will be used as fallback option.
+
+## Spark Configuration
+
+The default behavior of multi-threading in either Intel MKL or OpenBLAS may not be optimal with Spark's execution model [^1].
+
+Therefore configuring these native libraries to use a single thread for operations may actually improve performance (see [SPARK-21305](https://issues.apache.org/jira/browse/SPARK-21305)). It is usually optimal to match this to the number of `spark.task.cpus`, which is `1` by default and typically left at `1`.
+
+You can use the options in `config/spark-env.sh` to set thread number for Intel MKL or OpenBLAS:
+* For Intel MKL:
+```
+MKL_NUM_THREADS=1
+```
+* For OpenBLAS:
+```
+OPENBLAS_NUM_THREADS=1
+```
+
+[^1]: Please refer to the following resources to understand how to configure the number of threads for these BLAS implementations: [Intel MKL](https://software.intel.com/en-us/articles/recommended-settings-for-calling-intel-mkl-routines-from-multi-threaded-applications) or [Intel oneMKL](https://software.intel.com/en-us/onemkl-linux-developer-guide-improving-performance-with-threading) and [OpenBLAS](https://github.com/xianyi/OpenBLAS/wiki/faq#multi-threaded).
diff --git a/docs/ml-migration-guide.md b/docs/ml-migration-guide.md
index 4e6d68f5a8cf4..43b8de83a9d8c 100644
--- a/docs/ml-migration-guide.md
+++ b/docs/ml-migration-guide.md
@@ -281,7 +281,7 @@ Several deprecated methods were removed in the `spark.mllib` and `spark.ml` pack
 * `weights` in `LinearRegression` and `LogisticRegression` in `spark.ml`
 * `setMaxNumIterations` in `mllib.optimization.LBFGS` (marked as `DeveloperApi`)
 * `treeReduce` and `treeAggregate` in `mllib.rdd.RDDFunctions` (these functions are available on `RDD`s directly, and were marked as `DeveloperApi`)
-* `defaultStategy` in `mllib.tree.configuration.Strategy`
+* `defaultStrategy` in `mllib.tree.configuration.Strategy`
 * `build` in `mllib.tree.Node`
 * libsvm loaders for multiclass and load/save labeledData methods in `mllib.util.MLUtils`
 
diff --git a/docs/ml-pipeline.md b/docs/ml-pipeline.md
index 0b581e1a09c97..8a9599ede20b8 100644
--- a/docs/ml-pipeline.md
+++ b/docs/ml-pipeline.md
@@ -268,9 +268,9 @@ the [`Params` Java docs](api/java/org/apache/spark/ml/param/Params.html) for det
 
 <div data-lang="python" markdown="1">
 
-Refer to the [`Estimator` Python docs](api/python/pyspark.ml.html#pyspark.ml.Estimator),
-the [`Transformer` Python docs](api/python/pyspark.ml.html#pyspark.ml.Transformer) and
-the [`Params` Python docs](api/python/pyspark.ml.html#pyspark.ml.param.Params) for more details on the API.
+Refer to the [`Estimator` Python docs](api/python/reference/api/pyspark.ml.Estimator.html),
+the [`Transformer` Python docs](api/python/reference/api/pyspark.ml.Transformer.html) and
+the [`Params` Python docs](api/python/reference/api/pyspark.ml.param.Params.html) for more details on the API.
 
 {% include_example python/ml/estimator_transformer_param_example.py %}
 </div>
@@ -300,7 +300,7 @@ Refer to the [`Pipeline` Java docs](api/java/org/apache/spark/ml/Pipeline.html)
 
 <div data-lang="python" markdown="1">
 
-Refer to the [`Pipeline` Python docs](api/python/pyspark.ml.html#pyspark.ml.Pipeline) for more details on the API.
+Refer to the [`Pipeline` Python docs](api/python/reference/api/pyspark.ml.Pipeline.html) for more details on the API.
 
 {% include_example python/ml/pipeline_example.py %}
 </div>
diff --git a/docs/ml-statistics.md b/docs/ml-statistics.md
index 637cdd6c78f10..ed8431805df9d 100644
--- a/docs/ml-statistics.md
+++ b/docs/ml-statistics.md
@@ -66,7 +66,7 @@ The output will be a DataFrame that contains the correlation matrix of the colum
 </div>
 
 <div data-lang="python" markdown="1">
-[`Correlation`](api/python/pyspark.ml.html#pyspark.ml.stat.Correlation$)
+[`Correlation`](api/python/reference/api/pyspark.ml.stat.Correlation.html)
 computes the correlation matrix for the input Dataset of Vectors using the specified method.
 The output will be a DataFrame that contains the correlation matrix of the column of vectors.
 
@@ -79,33 +79,7 @@ The output will be a DataFrame that contains the correlation matrix of the colum
 
 Hypothesis testing is a powerful tool in statistics to determine whether a result is statistically
 significant, whether this result occurred by chance or not. `spark.ml` currently supports Pearson's
-Chi-squared ( $\chi^2$) tests for independence, as well as ANOVA test for classification tasks and
-F-value test for regression tasks.
-
-### ANOVATest
-
-`ANOVATest` computes ANOVA F-values between labels and features for classification tasks. The labels should be categorical
-and features should be continuous.
-
-<div class="codetabs">
-<div data-lang="scala" markdown="1">
-Refer to the [`ANOVATest` Scala docs](api/scala/org/apache/spark/ml/stat/ANOVATest$.html) for details on the API.
-
-{% include_example scala/org/apache/spark/examples/ml/ANOVATestExample.scala %}
-</div>
-
-<div data-lang="java" markdown="1">
-Refer to the [`ANOVATest` Java docs](api/java/org/apache/spark/ml/stat/ANOVATest.html) for details on the API.
-
-{% include_example java/org/apache/spark/examples/ml/JavaANOVATestExample.java %}
-</div>
-
-<div data-lang="python" markdown="1">
-Refer to the [`ANOVATest` Python docs](api/python/index.html#pyspark.ml.stat.ANOVATest$) for details on the API.
-
-{% include_example python/ml/anova_test_example.py %}
-</div>
-</div>
+Chi-squared ( $\chi^2$) tests for independence.
 
 ### ChiSquareTest
 
@@ -127,39 +101,13 @@ Refer to the [`ChiSquareTest` Java docs](api/java/org/apache/spark/ml/stat/ChiSq
 </div>
 
 <div data-lang="python" markdown="1">
-Refer to the [`ChiSquareTest` Python docs](api/python/index.html#pyspark.ml.stat.ChiSquareTest$) for details on the API.
+Refer to the [`ChiSquareTest` Python docs](api/python/reference/api/pyspark.ml.stat.ChiSquareTest.html) for details on the API.
 
 {% include_example python/ml/chi_square_test_example.py %}
 </div>
 
 </div>
 
-### FValueTest
-
-`FValueTest` computes F-values between labels and features for regression tasks. Both the labels
- and features should be continuous.
-
- <div class="codetabs">
- <div data-lang="scala" markdown="1">
- Refer to the [`FValueTest` Scala docs](api/scala/org/apache/spark/ml/stat/FValueTest$.html) for details on the API.
-
- {% include_example scala/org/apache/spark/examples/ml/FValueTestExample.scala %}
- </div>
-
- <div data-lang="java" markdown="1">
- Refer to the [`FValueTest` Java docs](api/java/org/apache/spark/ml/stat/FValueTest.html) for details on the API.
-
- {% include_example java/org/apache/spark/examples/ml/JavaFValueTestExample.java %}
- </div>
-
- <div data-lang="python" markdown="1">
- Refer to the [`FValueTest` Python docs](api/python/index.html#pyspark.ml.stat.FValueTest$) for details on the API.
-
- {% include_example python/ml/fvalue_test_example.py %}
- </div>
-
- </div>
-
 ## Summarizer
 
 We provide vector column summary statistics for `Dataframe` through `Summarizer`.
@@ -182,7 +130,7 @@ to compute the mean and variance for a vector column of the input dataframe, wit
 </div>
 
 <div data-lang="python" markdown="1">
-Refer to the [`Summarizer` Python docs](api/python/index.html#pyspark.ml.stat.Summarizer$) for details on the API.
+Refer to the [`Summarizer` Python docs](api/python/reference/api/pyspark.ml.stat.Summarizer.html) for details on the API.
 
 {% include_example python/ml/summarizer_example.py %}
 </div>
diff --git a/docs/ml-tuning.md b/docs/ml-tuning.md
index 274f1951011bd..e7940a3493685 100644
--- a/docs/ml-tuning.md
+++ b/docs/ml-tuning.md
@@ -71,10 +71,44 @@ for multiclass problems, a [`MultilabelClassificationEvaluator`](api/scala/org/a
 [`RankingEvaluator`](api/scala/org/apache/spark/ml/evaluation/RankingEvaluator.html) for ranking problems. The default metric used to
 choose the best `ParamMap` can be overridden by the `setMetricName` method in each of these evaluators.
 
-To help construct the parameter grid, users can use the [`ParamGridBuilder`](api/scala/org/apache/spark/ml/tuning/ParamGridBuilder.html) utility.
+To help construct the parameter grid, users can use the [`ParamGridBuilder`](api/scala/org/apache/spark/ml/tuning/ParamGridBuilder.html) utility (see the *Cross-Validation* section below for an example).
 By default, sets of parameters from the parameter grid are evaluated in serial. Parameter evaluation can be done in parallel by setting `parallelism` with a value of 2 or more (a value of 1 will be serial) before running model selection with `CrossValidator` or `TrainValidationSplit`.
 The value of `parallelism` should be chosen carefully to maximize parallelism without exceeding cluster resources, and larger values may not always lead to improved performance.  Generally speaking, a value up to 10 should be sufficient for most clusters.
 
+Alternatively, users can use the [`ParamRandomBuilder`](api/scala/org/apache/spark/ml/tuning/ParamRandomBuilder.html) utility.
+This has the same properties of `ParamGridBuilder` mentioned above, but hyperparameters are chosen at random within a user-defined range.
+The mathematical principle behind this is that given enough samples, the probability of at least one sample *not* being near the optimum within a range tends to zero.
+Irrespective of machine learning model, the expected number of samples needed to have at least one within 5% of the optimum is about 60. 
+If this 5% volume lies between the parameters defined in a grid search, it will *never* be found by `ParamGridBuilder`.  
+
+<div class="codetabs">
+
+<div data-lang="scala" markdown="1">
+
+Refer to the [`ParamRandomBuilder` Scala docs](api/scala/org/apache/spark/ml/tuning/ParamRandomBuilder.html) for details on the API.
+
+{% include_example scala/org/apache/spark/examples/ml/ModelSelectionViaRandomHyperparametersExample.scala %}
+</div>
+
+<div data-lang="java" markdown="1">
+
+Refer to the [`ParamRandomBuilder` Java docs](api/java/org/apache/spark/ml/tuning/ParamRandomBuilder.html) for details on the API.
+
+{% include_example java/org/apache/spark/examples/ml/JavaModelSelectionViaRandomHyperparametersExample.java %}
+</div>
+
+<div data-lang="python" markdown="1">
+
+Python users are recommended to look at Python libraries that are specifically for hyperparameter tuning such as Hyperopt.  
+
+Refer to the [`ParamRandomBuilder` Java docs](api/python/reference/api/pyspark.ml.tuning.ParamRandomBuilder.html) for details on the API.
+
+{% include_example python/ml/model_selection_random_hyperparameters_example.py %}
+
+</div>
+
+</div>
+
 # Cross-Validation
 
 `CrossValidator` begins by splitting the dataset into a set of *folds* which are used as separate training and test datasets. E.g., with `$k=3$` folds, `CrossValidator` will generate 3 (training, test) dataset pairs, each of which uses 2/3 of the data for training and 1/3 for testing.  To evaluate a particular `ParamMap`, `CrossValidator` computes the average evaluation metric for the 3 `Model`s produced by fitting the `Estimator` on the 3 different (training, test) dataset pairs.
@@ -109,7 +143,7 @@ Refer to the [`CrossValidator` Java docs](api/java/org/apache/spark/ml/tuning/Cr
 
 <div data-lang="python" markdown="1">
 
-Refer to the [`CrossValidator` Python docs](api/python/pyspark.ml.html#pyspark.ml.tuning.CrossValidator) for more details on the API.
+Refer to the [`CrossValidator` Python docs](api/python/reference/api/pyspark.ml.tuning.CrossValidator.html) for more details on the API.
 
 {% include_example python/ml/cross_validator.py %}
 </div>
@@ -149,7 +183,7 @@ Refer to the [`TrainValidationSplit` Java docs](api/java/org/apache/spark/ml/tun
 
 <div data-lang="python" markdown="1">
 
-Refer to the [`TrainValidationSplit` Python docs](api/python/pyspark.ml.html#pyspark.ml.tuning.TrainValidationSplit) for more details on the API.
+Refer to the [`TrainValidationSplit` Python docs](api/python/reference/api/pyspark.ml.tuning.TrainValidationSplit.html) for more details on the API.
 
 {% include_example python/ml/train_validation_split.py %}
 </div>
diff --git a/docs/mllib-clustering.md b/docs/mllib-clustering.md
index 4cb2e259ccfbc..00db04452b50e 100644
--- a/docs/mllib-clustering.md
+++ b/docs/mllib-clustering.md
@@ -85,7 +85,7 @@ data into two clusters. The number of desired clusters is passed to the algorith
 Within Set Sum of Squared Error (WSSSE). You can reduce this error measure by increasing *k*. In
 fact the optimal *k* is usually one where there is an "elbow" in the WSSSE graph.
 
-Refer to the [`KMeans` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.clustering.KMeans) and [`KMeansModel` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.clustering.KMeansModel) for more details on the API.
+Refer to the [`KMeans` Python docs](api/python/reference/api/pyspark.mllib.clustering.KMeans.html) and [`KMeansModel` Python docs](api/python/reference/api/pyspark.mllib.clustering.KMeansModel.html) for more details on the API.
 
 {% include_example python/mllib/k_means_example.py %}
 </div>
@@ -134,11 +134,11 @@ Refer to the [`GaussianMixture` Java docs](api/java/org/apache/spark/mllib/clust
 
 <div data-lang="python" markdown="1">
 In the following example after loading and parsing data, we use a
-[GaussianMixture](api/python/pyspark.mllib.html#pyspark.mllib.clustering.GaussianMixture)
+[GaussianMixture](api/python/reference/api/pyspark.mllib.clustering.GaussianMixture.html)
 object to cluster the data into two clusters. The number of desired clusters is passed
 to the algorithm. We then output the parameters of the mixture model.
 
-Refer to the [`GaussianMixture` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.clustering.GaussianMixture) and [`GaussianMixtureModel` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.clustering.GaussianMixtureModel) for more details on the API.
+Refer to the [`GaussianMixture` Python docs](api/python/reference/api/pyspark.mllib.clustering.GaussianMixture.html) and [`GaussianMixtureModel` Python docs](api/python/reference/api/pyspark.mllib.clustering.GaussianMixtureModel.html) for more details on the API.
 
 {% include_example python/mllib/gaussian_mixture_example.py %}
 </div>
@@ -189,7 +189,7 @@ Refer to the [`PowerIterationClustering` Scala docs](api/scala/org/apache/spark/
 
 [`PowerIterationClustering`](api/java/org/apache/spark/mllib/clustering/PowerIterationClustering.html)
 implements the PIC algorithm.
-It takes an `JavaRDD` of `(srcId: Long, dstId: Long, similarity: Double)` tuples representing the
+It takes a `JavaRDD` of `(srcId: Long, dstId: Long, similarity: Double)` tuples representing the
 affinity matrix.
 Calling `PowerIterationClustering.run` returns a
 [`PowerIterationClusteringModel`](api/java/org/apache/spark/mllib/clustering/PowerIterationClusteringModel.html)
@@ -202,15 +202,15 @@ Refer to the [`PowerIterationClustering` Java docs](api/java/org/apache/spark/ml
 
 <div data-lang="python" markdown="1">
 
-[`PowerIterationClustering`](api/python/pyspark.mllib.html#pyspark.mllib.clustering.PowerIterationClustering)
+[`PowerIterationClustering`](api/python/reference/api/pyspark.mllib.clustering.PowerIterationClustering.html)
 implements the PIC algorithm.
 It takes an `RDD` of `(srcId: Long, dstId: Long, similarity: Double)` tuples representing the
 affinity matrix.
 Calling `PowerIterationClustering.run` returns a
-[`PowerIterationClusteringModel`](api/python/pyspark.mllib.html#pyspark.mllib.clustering.PowerIterationClustering),
+[`PowerIterationClusteringModel`](api/python/reference/api/pyspark.mllib.clustering.PowerIterationClustering.html),
 which contains the computed clustering assignments.
 
-Refer to the [`PowerIterationClustering` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.clustering.PowerIterationClustering) and [`PowerIterationClusteringModel` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.clustering.PowerIterationClusteringModel) for more details on the API.
+Refer to the [`PowerIterationClustering` Python docs](api/python/reference/api/pyspark.mllib.clustering.PowerIterationClustering.html) and [`PowerIterationClusteringModel` Python docs](api/python/reference/api/pyspark.mllib.clustering.PowerIterationClusteringModel.html) for more details on the API.
 
 {% include_example python/mllib/power_iteration_clustering_example.py %}
 </div>
@@ -368,7 +368,7 @@ Refer to the [`LDA` Java docs](api/java/org/apache/spark/mllib/clustering/LDA.ht
 </div>
 
 <div data-lang="python" markdown="1">
-Refer to the [`LDA` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.clustering.LDA) and [`LDAModel` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.clustering.LDAModel) for more details on the API.
+Refer to the [`LDA` Python docs](api/python/reference/api/pyspark.mllib.clustering.LDA.html) and [`LDAModel` Python docs](api/python/reference/api/pyspark.mllib.clustering.LDAModel.html) for more details on the API.
 
 {% include_example python/mllib/latent_dirichlet_allocation_example.py %}
 </div>
@@ -410,7 +410,7 @@ Refer to the [`BisectingKMeans` Java docs](api/java/org/apache/spark/mllib/clust
 </div>
 
 <div data-lang="python" markdown="1">
-Refer to the [`BisectingKMeans` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.clustering.BisectingKMeans) and [`BisectingKMeansModel` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.clustering.BisectingKMeansModel) for more details on the API.
+Refer to the [`BisectingKMeans` Python docs](api/python/reference/api/pyspark.mllib.clustering.BisectingKMeans.html) and [`BisectingKMeansModel` Python docs](api/python/reference/api/pyspark.mllib.clustering.BisectingKMeansModel.html) for more details on the API.
 
 {% include_example python/mllib/bisecting_k_means_example.py %}
 </div>
@@ -458,7 +458,7 @@ And Refer to [Spark Streaming Programming Guide](streaming-programming-guide.htm
 </div>
 
 <div data-lang="python" markdown="1">
-Refer to the [`StreamingKMeans` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.clustering.StreamingKMeans) for more details on the API.
+Refer to the [`StreamingKMeans` Python docs](api/python/reference/api/pyspark.mllib.clustering.StreamingKMeans.html) for more details on the API.
 And Refer to [Spark Streaming Programming Guide](streaming-programming-guide.html#initializing) for details on StreamingContext.
 
 {% include_example python/mllib/streaming_k_means_example.py %}
diff --git a/docs/mllib-collaborative-filtering.md b/docs/mllib-collaborative-filtering.md
index aaefa59c4a9c3..87b8b9369bfab 100644
--- a/docs/mllib-collaborative-filtering.md
+++ b/docs/mllib-collaborative-filtering.md
@@ -111,7 +111,7 @@ In the following example we load rating data. Each row consists of a user, a pro
 We use the default ALS.train() method which assumes ratings are explicit. We evaluate the
 recommendation by measuring the Mean Squared Error of rating prediction.
 
-Refer to the [`ALS` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.recommendation.ALS) for more details on the API.
+Refer to the [`ALS` Python docs](api/python/reference/api/pyspark.mllib.recommendation.ALS.html) for more details on the API.
 
 {% include_example python/mllib/recommendation_example.py %}
 
diff --git a/docs/mllib-data-types.md b/docs/mllib-data-types.md
index 6d3b1a599d48b..2b3359ffa798a 100644
--- a/docs/mllib-data-types.md
+++ b/docs/mllib-data-types.md
@@ -97,15 +97,15 @@ MLlib recognizes the following types as dense vectors:
 
 and the following as sparse vectors:
 
-* MLlib's [`SparseVector`](api/python/pyspark.mllib.html#pyspark.mllib.linalg.SparseVector).
+* MLlib's [`SparseVector`](api/python/reference/api/pyspark.mllib.linalg.SparseVector.html).
 * SciPy's
   [`csc_matrix`](http://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.csc_matrix.html#scipy.sparse.csc_matrix)
   with a single column
 
 We recommend using NumPy arrays over lists for efficiency, and using the factory methods implemented
-in [`Vectors`](api/python/pyspark.mllib.html#pyspark.mllib.linalg.Vectors) to create sparse vectors.
+in [`Vectors`](api/python/reference/api/pyspark.mllib.linalg.Vectors.html) to create sparse vectors.
 
-Refer to the [`Vectors` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.linalg.Vectors) for more details on the API.
+Refer to the [`Vectors` Python docs](api/python/reference/api/pyspark.mllib.linalg.Vectors.html) for more details on the API.
 
 {% highlight python %}
 import numpy as np
@@ -176,9 +176,9 @@ LabeledPoint neg = new LabeledPoint(0.0, Vectors.sparse(3, new int[] {0, 2}, new
 <div data-lang="python" markdown="1">
 
 A labeled point is represented by
-[`LabeledPoint`](api/python/pyspark.mllib.html#pyspark.mllib.regression.LabeledPoint).
+[`LabeledPoint`](api/python/reference/api/pyspark.mllib.regression.LabeledPoint.html).
 
-Refer to the [`LabeledPoint` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.regression.LabeledPoint) for more details on the API.
+Refer to the [`LabeledPoint` Python docs](api/python/reference/api/pyspark.mllib.regression.LabeledPoint.html) for more details on the API.
 
 {% highlight python %}
 from pyspark.mllib.linalg import SparseVector
@@ -242,10 +242,10 @@ JavaRDD<LabeledPoint> examples =
 </div>
 
 <div data-lang="python" markdown="1">
-[`MLUtils.loadLibSVMFile`](api/python/pyspark.mllib.html#pyspark.mllib.util.MLUtils) reads training
+[`MLUtils.loadLibSVMFile`](api/python/reference/api/pyspark.mllib.util.MLUtils.html) reads training
 examples stored in LIBSVM format.
 
-Refer to the [`MLUtils` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.util.MLUtils) for more details on the API.
+Refer to the [`MLUtils` Python docs](api/python/reference/api/pyspark.mllib.util.MLUtils.html) for more details on the API.
 
 {% highlight python %}
 from pyspark.mllib.util import MLUtils
@@ -319,14 +319,14 @@ Matrix sm = Matrices.sparse(3, 2, new int[] {0, 1, 3}, new int[] {0, 2, 1}, new
 <div data-lang="python" markdown="1">
 
 The base class of local matrices is
-[`Matrix`](api/python/pyspark.mllib.html#pyspark.mllib.linalg.Matrix), and we provide two
-implementations: [`DenseMatrix`](api/python/pyspark.mllib.html#pyspark.mllib.linalg.DenseMatrix),
-and [`SparseMatrix`](api/python/pyspark.mllib.html#pyspark.mllib.linalg.SparseMatrix).
+[`Matrix`](api/python/reference/api/pyspark.mllib.linalg.Matrix.html), and we provide two
+implementations: [`DenseMatrix`](api/python/reference/api/pyspark.mllib.linalg.DenseMatrix.html),
+and [`SparseMatrix`](api/python/reference/api/pyspark.mllib.linalg.SparseMatrix.html).
 We recommend using the factory methods implemented
-in [`Matrices`](api/python/pyspark.mllib.html#pyspark.mllib.linalg.Matrices) to create local
+in [`Matrices`](api/python/reference/api/pyspark.mllib.linalg.Matrices.html) to create local
 matrices. Remember, local matrices in MLlib are stored in column-major order.
 
-Refer to the [`Matrix` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.linalg.Matrix) and [`Matrices` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.linalg.Matrices) for more details on the API.
+Refer to the [`Matrix` Python docs](api/python/reference/api/pyspark.mllib.linalg.Matrix.html) and [`Matrices` Python docs](api/python/reference/api/pyspark.mllib.linalg.Matrices.html) for more details on the API.
 
 {% highlight python %}
 from pyspark.mllib.linalg import Matrix, Matrices
@@ -428,10 +428,10 @@ QRDecomposition<RowMatrix, Matrix> result = mat.tallSkinnyQR(true);
 
 <div data-lang="python" markdown="1">
 
-A [`RowMatrix`](api/python/pyspark.mllib.html#pyspark.mllib.linalg.distributed.RowMatrix) can be 
+A [`RowMatrix`](api/python/reference/api/pyspark.mllib.linalg.distributed.RowMatrix.html) can be 
 created from an `RDD` of vectors.
 
-Refer to the [`RowMatrix` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.linalg.distributed.RowMatrix) for more details on the API.
+Refer to the [`RowMatrix` Python docs](api/python/reference/api/pyspark.mllib.linalg.distributed.RowMatrix.html) for more details on the API.
 
 {% highlight python %}
 from pyspark.mllib.linalg.distributed import RowMatrix
@@ -519,13 +519,13 @@ RowMatrix rowMat = mat.toRowMatrix();
 
 <div data-lang="python" markdown="1">
 
-An [`IndexedRowMatrix`](api/python/pyspark.mllib.html#pyspark.mllib.linalg.distributed.IndexedRowMatrix)
+An [`IndexedRowMatrix`](api/python/reference/api/pyspark.mllib.linalg.distributed.IndexedRowMatrix.html)
 can be created from an `RDD` of `IndexedRow`s, where 
-[`IndexedRow`](api/python/pyspark.mllib.html#pyspark.mllib.linalg.distributed.IndexedRow) is a 
+[`IndexedRow`](api/python/reference/api/pyspark.mllib.linalg.distributed.IndexedRow.html) is a 
 wrapper over `(long, vector)`.  An `IndexedRowMatrix` can be converted to a `RowMatrix` by dropping
 its row indices.
 
-Refer to the [`IndexedRowMatrix` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.linalg.distributed.IndexedRowMatrix) for more details on the API.
+Refer to the [`IndexedRowMatrix` Python docs](api/python/reference/api/pyspark.mllib.linalg.distributed.IndexedRowMatrix.html) for more details on the API.
 
 {% highlight python %}
 from pyspark.mllib.linalg.distributed import IndexedRow, IndexedRowMatrix
@@ -626,13 +626,13 @@ IndexedRowMatrix indexedRowMatrix = mat.toIndexedRowMatrix();
 
 <div data-lang="python" markdown="1">
 
-A [`CoordinateMatrix`](api/python/pyspark.mllib.html#pyspark.mllib.linalg.distributed.CoordinateMatrix)
+A [`CoordinateMatrix`](api/python/reference/api/pyspark.mllib.linalg.distributed.CoordinateMatrix.html)
 can be created from an `RDD` of `MatrixEntry` entries, where 
-[`MatrixEntry`](api/python/pyspark.mllib.html#pyspark.mllib.linalg.distributed.MatrixEntry) is a 
+[`MatrixEntry`](api/python/reference/api/pyspark.mllib.linalg.distributed.MatrixEntry.html) is a 
 wrapper over `(long, long, float)`.  A `CoordinateMatrix` can be converted to a `RowMatrix` by 
 calling `toRowMatrix`, or to an `IndexedRowMatrix` with sparse rows by calling `toIndexedRowMatrix`.
 
-Refer to the [`CoordinateMatrix` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.linalg.distributed.CoordinateMatrix) for more details on the API.
+Refer to the [`CoordinateMatrix` Python docs](api/python/reference/api/pyspark.mllib.linalg.distributed.CoordinateMatrix.html) for more details on the API.
 
 {% highlight python %}
 from pyspark.mllib.linalg.distributed import CoordinateMatrix, MatrixEntry
@@ -643,7 +643,7 @@ entries = sc.parallelize([MatrixEntry(0, 0, 1.2), MatrixEntry(1, 0, 2.1), Matrix
 #   - or using (long, long, float) tuples:
 entries = sc.parallelize([(0, 0, 1.2), (1, 0, 2.1), (2, 1, 3.7)])
 
-# Create an CoordinateMatrix from an RDD of MatrixEntries.
+# Create a CoordinateMatrix from an RDD of MatrixEntries.
 mat = CoordinateMatrix(entries)
 
 # Get its size.
@@ -735,11 +735,11 @@ BlockMatrix ata = matA.transpose().multiply(matA);
 
 <div data-lang="python" markdown="1">
 
-A [`BlockMatrix`](api/python/pyspark.mllib.html#pyspark.mllib.linalg.distributed.BlockMatrix) 
+A [`BlockMatrix`](api/python/reference/api/pyspark.mllib.linalg.distributed.BlockMatrix.html) 
 can be created from an `RDD` of sub-matrix blocks, where a sub-matrix block is a 
 `((blockRowIndex, blockColIndex), sub-matrix)` tuple.
 
-Refer to the [`BlockMatrix` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.linalg.distributed.BlockMatrix) for more details on the API.
+Refer to the [`BlockMatrix` Python docs](api/python/reference/api/pyspark.mllib.linalg.distributed.BlockMatrix.html) for more details on the API.
 
 {% highlight python %}
 from pyspark.mllib.linalg import Matrices
diff --git a/docs/mllib-decision-tree.md b/docs/mllib-decision-tree.md
index 455649c8e686e..4b6538d9d50e2 100644
--- a/docs/mllib-decision-tree.md
+++ b/docs/mllib-decision-tree.md
@@ -219,7 +219,7 @@ Refer to the [`DecisionTree` Java docs](api/java/org/apache/spark/mllib/tree/Dec
 </div>
 
 <div data-lang="python" markdown="1">
-Refer to the [`DecisionTree` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.tree.DecisionTree) and [`DecisionTreeModel` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.tree.DecisionTreeModel) for more details on the API.
+Refer to the [`DecisionTree` Python docs](api/python/reference/api/pyspark.mllib.tree.DecisionTree.html) and [`DecisionTreeModel` Python docs](api/python/reference/api/pyspark.mllib.tree.DecisionTreeModel.html) for more details on the API.
 
 {% include_example python/mllib/decision_tree_classification_example.py %}
 </div>
@@ -250,7 +250,7 @@ Refer to the [`DecisionTree` Java docs](api/java/org/apache/spark/mllib/tree/Dec
 </div>
 
 <div data-lang="python" markdown="1">
-Refer to the [`DecisionTree` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.tree.DecisionTree) and [`DecisionTreeModel` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.tree.DecisionTreeModel) for more details on the API.
+Refer to the [`DecisionTree` Python docs](api/python/reference/api/pyspark.mllib.tree.DecisionTree.html) and [`DecisionTreeModel` Python docs](api/python/reference/api/pyspark.mllib.tree.DecisionTreeModel.html) for more details on the API.
 
 {% include_example python/mllib/decision_tree_regression_example.py %}
 </div>
diff --git a/docs/mllib-dimensionality-reduction.md b/docs/mllib-dimensionality-reduction.md
index 8818e403aabbc..e5af742461ba7 100644
--- a/docs/mllib-dimensionality-reduction.md
+++ b/docs/mllib-dimensionality-reduction.md
@@ -93,7 +93,7 @@ The same code applies to `IndexedRowMatrix` if `U` is defined as an
 `IndexedRowMatrix`.
 </div>
 <div data-lang="python" markdown="1">
-Refer to the [`SingularValueDecomposition` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.linalg.distributed.SingularValueDecomposition) for details on the API.
+Refer to the [`SingularValueDecomposition` Python docs](api/python/reference/api/pyspark.mllib.linalg.distributed.SingularValueDecomposition.html) for details on the API.
 
 {% include_example python/mllib/svd_example.py %}
 
@@ -146,7 +146,7 @@ Refer to the [`RowMatrix` Java docs](api/java/org/apache/spark/mllib/linalg/dist
 The following code demonstrates how to compute principal components on a `RowMatrix`
 and use them to project the vectors into a low-dimensional space.
 
-Refer to the [`RowMatrix` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.linalg.distributed.RowMatrix) for details on the API.
+Refer to the [`RowMatrix` Python docs](api/python/reference/api/pyspark.mllib.linalg.distributed.RowMatrix.html) for details on the API.
 
 {% include_example python/mllib/pca_rowmatrix_example.py %}
 
diff --git a/docs/mllib-ensembles.md b/docs/mllib-ensembles.md
index 27a9fe67f3f6d..8821ccd341dc0 100644
--- a/docs/mllib-ensembles.md
+++ b/docs/mllib-ensembles.md
@@ -123,7 +123,7 @@ Refer to the [`RandomForest` Java docs](api/java/org/apache/spark/mllib/tree/Ran
 </div>
 
 <div data-lang="python" markdown="1">
-Refer to the [`RandomForest` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.tree.RandomForest) and [`RandomForest` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.tree.RandomForestModel) for more details on the API.
+Refer to the [`RandomForest` Python docs](api/python/reference/api/pyspark.mllib.tree.RandomForest.html) and [`RandomForest` Python docs](api/python/reference/api/pyspark.mllib.tree.RandomForestModel.html) for more details on the API.
 
 {% include_example python/mllib/random_forest_classification_example.py %}
 </div>
@@ -154,7 +154,7 @@ Refer to the [`RandomForest` Java docs](api/java/org/apache/spark/mllib/tree/Ran
 </div>
 
 <div data-lang="python" markdown="1">
-Refer to the [`RandomForest` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.tree.RandomForest) and [`RandomForest` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.tree.RandomForestModel) for more details on the API.
+Refer to the [`RandomForest` Python docs](api/python/reference/api/pyspark.mllib.tree.RandomForest.html) and [`RandomForest` Python docs](api/python/reference/api/pyspark.mllib.tree.RandomForestModel.html) for more details on the API.
 
 {% include_example python/mllib/random_forest_regression_example.py %}
 </div>
@@ -264,7 +264,7 @@ Refer to the [`GradientBoostedTrees` Java docs](api/java/org/apache/spark/mllib/
 </div>
 
 <div data-lang="python" markdown="1">
-Refer to the [`GradientBoostedTrees` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.tree.GradientBoostedTrees) and [`GradientBoostedTreesModel` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.tree.GradientBoostedTreesModel) for more details on the API.
+Refer to the [`GradientBoostedTrees` Python docs](api/python/reference/api/pyspark.mllib.tree.GradientBoostedTrees.html) and [`GradientBoostedTreesModel` Python docs](api/python/reference/api/pyspark.mllib.tree.GradientBoostedTreesModel.html) for more details on the API.
 
 {% include_example python/mllib/gradient_boosting_classification_example.py %}
 </div>
@@ -295,7 +295,7 @@ Refer to the [`GradientBoostedTrees` Java docs](api/java/org/apache/spark/mllib/
 </div>
 
 <div data-lang="python" markdown="1">
-Refer to the [`GradientBoostedTrees` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.tree.GradientBoostedTrees) and [`GradientBoostedTreesModel` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.tree.GradientBoostedTreesModel) for more details on the API.
+Refer to the [`GradientBoostedTrees` Python docs](api/python/reference/api/pyspark.mllib.tree.GradientBoostedTrees.html) and [`GradientBoostedTreesModel` Python docs](api/python/reference/api/pyspark.mllib.tree.GradientBoostedTreesModel.html) for more details on the API.
 
 {% include_example python/mllib/gradient_boosting_regression_example.py %}
 </div>
diff --git a/docs/mllib-evaluation-metrics.md b/docs/mllib-evaluation-metrics.md
index f9efa769fc140..2ef1c88aa1dc2 100644
--- a/docs/mllib-evaluation-metrics.md
+++ b/docs/mllib-evaluation-metrics.md
@@ -131,7 +131,7 @@ Refer to the [`LogisticRegressionModel` Java docs](api/java/org/apache/spark/mll
 </div>
 
 <div data-lang="python" markdown="1">
-Refer to the [`BinaryClassificationMetrics` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.evaluation.BinaryClassificationMetrics) and [`LogisticRegressionWithLBFGS` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.classification.LogisticRegressionWithLBFGS) for more details on the API.
+Refer to the [`BinaryClassificationMetrics` Python docs](api/python/reference/api/pyspark.mllib.evaluation.BinaryClassificationMetrics.html) and [`LogisticRegressionWithLBFGS` Python docs](api/python/reference/api/pyspark.mllib.classification.LogisticRegressionWithLBFGS.html) for more details on the API.
 
 {% include_example python/mllib/binary_classification_metrics_example.py %}
 </div>
@@ -257,7 +257,7 @@ Refer to the [`MulticlassMetrics` Java docs](api/java/org/apache/spark/mllib/eva
 </div>
 
 <div data-lang="python" markdown="1">
-Refer to the [`MulticlassMetrics` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.evaluation.MulticlassMetrics) for more details on the API.
+Refer to the [`MulticlassMetrics` Python docs](api/python/reference/api/pyspark.mllib.evaluation.MulticlassMetrics.html) for more details on the API.
 
 {% include_example python/mllib/multi_class_metrics_example.py %}
 
@@ -407,7 +407,7 @@ Refer to the [`MultilabelMetrics` Java docs](api/java/org/apache/spark/mllib/eva
 </div>
 
 <div data-lang="python" markdown="1">
-Refer to the [`MultilabelMetrics` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.evaluation.MultilabelMetrics) for more details on the API.
+Refer to the [`MultilabelMetrics` Python docs](api/python/reference/api/pyspark.mllib.evaluation.MultilabelMetrics.html) for more details on the API.
 
 {% include_example python/mllib/multi_label_metrics_example.py %}
 
@@ -535,7 +535,7 @@ Refer to the [`RegressionMetrics` Java docs](api/java/org/apache/spark/mllib/eva
 </div>
 
 <div data-lang="python" markdown="1">
-Refer to the [`RegressionMetrics` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.evaluation.RegressionMetrics) and [`RankingMetrics` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.evaluation.RankingMetrics) for more details on the API.
+Refer to the [`RegressionMetrics` Python docs](api/python/reference/api/pyspark.mllib.evaluation.RegressionMetrics.html) and [`RankingMetrics` Python docs](api/python/reference/api/pyspark.mllib.evaluation.RankingMetrics.html) for more details on the API.
 
 {% include_example python/mllib/ranking_metrics_example.py %}
 
diff --git a/docs/mllib-feature-extraction.md b/docs/mllib-feature-extraction.md
index 8df9699150f6a..98c966d43cfdf 100644
--- a/docs/mllib-feature-extraction.md
+++ b/docs/mllib-feature-extraction.md
@@ -80,13 +80,13 @@ Refer to the [`HashingTF` Scala docs](api/scala/org/apache/spark/mllib/feature/H
 </div>
 <div data-lang="python" markdown="1">
 
-TF and IDF are implemented in [HashingTF](api/python/pyspark.mllib.html#pyspark.mllib.feature.HashingTF)
-and [IDF](api/python/pyspark.mllib.html#pyspark.mllib.feature.IDF).
+TF and IDF are implemented in [HashingTF](api/python/reference/api/pyspark.mllib.feature.HashingTF.html)
+and [IDF](api/python/reference/api/pyspark.mllib.feature.IDF.html).
 `HashingTF` takes an RDD of list as the input.
 Each record could be an iterable of strings or other types.
 
 
-Refer to the [`HashingTF` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.feature.HashingTF) for details on the API.
+Refer to the [`HashingTF` Python docs](api/python/reference/api/pyspark.mllib.feature.HashingTF.html) for details on the API.
 
 {% include_example python/mllib/tf_idf_example.py %}
 </div>
@@ -140,7 +140,7 @@ Refer to the [`Word2Vec` Scala docs](api/scala/org/apache/spark/mllib/feature/Wo
 {% include_example scala/org/apache/spark/examples/mllib/Word2VecExample.scala %}
 </div>
 <div data-lang="python" markdown="1">
-Refer to the [`Word2Vec` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.feature.Word2Vec) for more details on the API.
+Refer to the [`Word2Vec` Python docs](api/python/reference/api/pyspark.mllib.feature.Word2Vec.html) for more details on the API.
 
 {% include_example python/mllib/word2vec_example.py %}
 </div>
@@ -191,7 +191,7 @@ Refer to the [`StandardScaler` Scala docs](api/scala/org/apache/spark/mllib/feat
 </div>
 
 <div data-lang="python" markdown="1">
-Refer to the [`StandardScaler` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.feature.StandardScaler) for more details on the API.
+Refer to the [`StandardScaler` Python docs](api/python/reference/api/pyspark.mllib.feature.StandardScaler.html) for more details on the API.
 
 {% include_example python/mllib/standard_scaler_example.py %}
 </div>
@@ -227,7 +227,7 @@ Refer to the [`Normalizer` Scala docs](api/scala/org/apache/spark/mllib/feature/
 </div>
 
 <div data-lang="python" markdown="1">
-Refer to the [`Normalizer` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.feature.Normalizer) for more details on the API.
+Refer to the [`Normalizer` Python docs](api/python/reference/api/pyspark.mllib.feature.Normalizer.html) for more details on the API.
 
 {% include_example python/mllib/normalizer_example.py %}
 </div>
@@ -337,7 +337,7 @@ Refer to the [`ElementwiseProduct` Java docs](api/java/org/apache/spark/mllib/fe
 </div>
 
 <div data-lang="python" markdown="1">
-Refer to the [`ElementwiseProduct` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.feature.ElementwiseProduct) for more details on the API.
+Refer to the [`ElementwiseProduct` Python docs](api/python/reference/api/pyspark.mllib.feature.ElementwiseProduct.html) for more details on the API.
 
 {% include_example python/mllib/elementwise_product_example.py %}
 </div>
diff --git a/docs/mllib-frequent-pattern-mining.md b/docs/mllib-frequent-pattern-mining.md
index 709acde062d7e..9f782514d4b16 100644
--- a/docs/mllib-frequent-pattern-mining.md
+++ b/docs/mllib-frequent-pattern-mining.md
@@ -92,14 +92,14 @@ Refer to the [`FPGrowth` Java docs](api/java/org/apache/spark/mllib/fpm/FPGrowth
 
 <div data-lang="python" markdown="1">
 
-[`FPGrowth`](api/python/pyspark.mllib.html#pyspark.mllib.fpm.FPGrowth) implements the
+[`FPGrowth`](api/python/reference/api/pyspark.mllib.fpm.FPGrowth.html) implements the
 FP-growth algorithm.
 It takes an `RDD` of transactions, where each transaction is an `List` of items of a generic type.
 Calling `FPGrowth.train` with transactions returns an
-[`FPGrowthModel`](api/python/pyspark.mllib.html#pyspark.mllib.fpm.FPGrowthModel)
+[`FPGrowthModel`](api/python/reference/api/pyspark.mllib.fpm.FPGrowthModel.html)
 that stores the frequent itemsets with their frequencies.
 
-Refer to the [`FPGrowth` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.fpm.FPGrowth) for more details on the API.
+Refer to the [`FPGrowth` Python docs](api/python/reference/api/pyspark.mllib.fpm.FPGrowth.html) for more details on the API.
 
 {% include_example python/mllib/fpgrowth_example.py %}
 
diff --git a/docs/mllib-isotonic-regression.md b/docs/mllib-isotonic-regression.md
index 94ffadaf65c55..95be32a819e8a 100644
--- a/docs/mllib-isotonic-regression.md
+++ b/docs/mllib-isotonic-regression.md
@@ -94,7 +94,7 @@ i.e. 4710.28,500.00. The data are split to training and testing set.
 Model is created using the training set and a mean squared error is calculated from the predicted
 labels and real labels in the test set.
 
-Refer to the [`IsotonicRegression` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.regression.IsotonicRegression) and [`IsotonicRegressionModel` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.regression.IsotonicRegressionModel) for more details on the API.
+Refer to the [`IsotonicRegression` Python docs](api/python/reference/api/pyspark.mllib.regression.IsotonicRegression.html) and [`IsotonicRegressionModel` Python docs](api/python/reference/api/pyspark.mllib.regression.IsotonicRegressionModel.html) for more details on the API.
 
 {% include_example python/mllib/isotonic_regression_example.py %}
 </div>
diff --git a/docs/mllib-linear-methods.md b/docs/mllib-linear-methods.md
index e7726271ccb72..facf5e07fae11 100644
--- a/docs/mllib-linear-methods.md
+++ b/docs/mllib-linear-methods.md
@@ -251,7 +251,7 @@ a dependency.
 The following example shows how to load a sample dataset, build SVM model,
 and make predictions with the resulting model to compute the training error.
 
-Refer to the [`SVMWithSGD` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.classification.SVMWithSGD) and [`SVMModel` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.classification.SVMModel) for more details on the API.
+Refer to the [`SVMWithSGD` Python docs](api/python/reference/api/pyspark.mllib.classification.SVMWithSGD.html) and [`SVMModel` Python docs](api/python/reference/api/pyspark.mllib.classification.SVMModel.html) for more details on the API.
 
 {% include_example python/mllib/svm_with_sgd_example.py %}
 </div>
@@ -334,7 +334,7 @@ and make predictions with the resulting model to compute the training error.
 Note that the Python API does not yet support multiclass classification and model save/load but
 will in the future.
 
-Refer to the [`LogisticRegressionWithLBFGS` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.classification.LogisticRegressionWithLBFGS) and [`LogisticRegressionModel` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.classification.LogisticRegressionModel) for more details on the API.
+Refer to the [`LogisticRegressionWithLBFGS` Python docs](api/python/reference/api/pyspark.mllib.classification.LogisticRegressionWithLBFGS.html) and [`LogisticRegressionModel` Python docs](api/python/reference/api/pyspark.mllib.classification.LogisticRegressionModel.html) for more details on the API.
 
 {% include_example python/mllib/logistic_regression_with_lbfgs_example.py %}
 </div>
diff --git a/docs/mllib-naive-bayes.md b/docs/mllib-naive-bayes.md
index a3602667b5835..496720de30152 100644
--- a/docs/mllib-naive-bayes.md
+++ b/docs/mllib-naive-bayes.md
@@ -72,16 +72,16 @@ Refer to the [`NaiveBayes` Java docs](api/java/org/apache/spark/mllib/classifica
 </div>
 <div data-lang="python" markdown="1">
 
-[NaiveBayes](api/python/pyspark.mllib.html#pyspark.mllib.classification.NaiveBayes) implements multinomial
+[NaiveBayes](api/python/reference/api/pyspark.mllib.classification.NaiveBayes.html) implements multinomial
 naive Bayes. It takes an RDD of
-[LabeledPoint](api/python/pyspark.mllib.html#pyspark.mllib.regression.LabeledPoint) and an optionally
+[LabeledPoint](api/python/reference/api/pyspark.mllib.regression.LabeledPoint.html) and an optionally
 smoothing parameter `lambda` as input, and output a
-[NaiveBayesModel](api/python/pyspark.mllib.html#pyspark.mllib.classification.NaiveBayesModel), which can be
+[NaiveBayesModel](api/python/reference/api/pyspark.mllib.classification.NaiveBayesModel.html), which can be
 used for evaluation and prediction.
 
 Note that the Python API does not yet support model save/load but will in the future.
 
-Refer to the [`NaiveBayes` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.classification.NaiveBayes) and [`NaiveBayesModel` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.classification.NaiveBayesModel) for more details on the API.
+Refer to the [`NaiveBayes` Python docs](api/python/reference/api/pyspark.mllib.classification.NaiveBayes.html) and [`NaiveBayesModel` Python docs](api/python/reference/api/pyspark.mllib.classification.NaiveBayesModel.html) for more details on the API.
 
 {% include_example python/mllib/naive_bayes_example.py %}
 </div>
diff --git a/docs/mllib-statistics.md b/docs/mllib-statistics.md
index 7de214bb6b6a6..bf0828cd1524d 100644
--- a/docs/mllib-statistics.md
+++ b/docs/mllib-statistics.md
@@ -71,12 +71,12 @@ Refer to the [`MultivariateStatisticalSummary` Java docs](api/java/org/apache/sp
 </div>
 
 <div data-lang="python" markdown="1">
-[`colStats()`](api/python/pyspark.mllib.html#pyspark.mllib.stat.Statistics.colStats) returns an instance of
-[`MultivariateStatisticalSummary`](api/python/pyspark.mllib.html#pyspark.mllib.stat.MultivariateStatisticalSummary),
+[`colStats()`](api/python/reference/api/pyspark.mllib.stat.Statistics.html#pyspark.mllib.stat.Statistics.colStats) returns an instance of
+[`MultivariateStatisticalSummary`](api/python/reference/api/pyspark.mllib.stat.MultivariateStatisticalSummary.html),
 which contains the column-wise max, min, mean, variance, and number of nonzeros, as well as the
 total count.
 
-Refer to the [`MultivariateStatisticalSummary` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.stat.MultivariateStatisticalSummary) for more details on the API.
+Refer to the [`MultivariateStatisticalSummary` Python docs](api/python/reference/api/pyspark.mllib.stat.MultivariateStatisticalSummary.html) for more details on the API.
 
 {% include_example python/mllib/summary_statistics_example.py %}
 </div>
@@ -111,11 +111,11 @@ Refer to the [`Statistics` Java docs](api/java/org/apache/spark/mllib/stat/Stati
 </div>
 
 <div data-lang="python" markdown="1">
-[`Statistics`](api/python/pyspark.mllib.html#pyspark.mllib.stat.Statistics) provides methods to
+[`Statistics`](api/python/reference/api/pyspark.mllib.stat.Statistics.html) provides methods to
 calculate correlations between series. Depending on the type of input, two `RDD[Double]`s or
 an `RDD[Vector]`, the output will be a `Double` or the correlation `Matrix` respectively.
 
-Refer to the [`Statistics` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.stat.Statistics) for more details on the API.
+Refer to the [`Statistics` Python docs](api/python/reference/api/pyspark.mllib.stat.Statistics.html) for more details on the API.
 
 {% include_example python/mllib/correlations_example.py %}
 </div>
@@ -156,7 +156,7 @@ size, whereas sampling with replacement requires two additional passes.
 {% include_example java/org/apache/spark/examples/mllib/JavaStratifiedSamplingExample.java %}
 </div>
 <div data-lang="python" markdown="1">
-[`sampleByKey()`](api/python/pyspark.html#pyspark.RDD.sampleByKey) allows users to
+[`sampleByKey()`](api/python/reference/api/pyspark.RDD.sampleByKey.html#pyspark.RDD.sampleByKey) allows users to
 sample approximately $\lceil f_k \cdot n_k \rceil \, \forall k \in K$ items, where $f_k$ is the
 desired fraction for key $k$, $n_k$ is the number of key-value pairs for key $k$, and $K$ is the
 set of keys.
@@ -199,11 +199,11 @@ Refer to the [`ChiSqTestResult` Java docs](api/java/org/apache/spark/mllib/stat/
 </div>
 
 <div data-lang="python" markdown="1">
-[`Statistics`](api/python/index.html#pyspark.mllib.stat.Statistics$) provides methods to
+[`Statistics`](api/python/reference/api/pyspark.mllib.stat.Statistics.html) provides methods to
 run Pearson's chi-squared tests. The following example demonstrates how to run and interpret
 hypothesis tests.
 
-Refer to the [`Statistics` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.stat.Statistics) for more details on the API.
+Refer to the [`Statistics` Python docs](api/python/reference/api/pyspark.mllib.stat.Statistics.html) for more details on the API.
 
 {% include_example python/mllib/hypothesis_testing_example.py %}
 </div>
@@ -241,11 +241,11 @@ Refer to the [`Statistics` Java docs](api/java/org/apache/spark/mllib/stat/Stati
 </div>
 
 <div data-lang="python" markdown="1">
-[`Statistics`](api/python/pyspark.mllib.html#pyspark.mllib.stat.Statistics) provides methods to
+[`Statistics`](api/python/reference/api/pyspark.mllib.stat.Statistics.html) provides methods to
 run a 1-sample, 2-sided Kolmogorov-Smirnov test. The following example demonstrates how to run
 and interpret the hypothesis tests.
 
-Refer to the [`Statistics` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.stat.Statistics) for more details on the API.
+Refer to the [`Statistics` Python docs](api/python/reference/api/pyspark.mllib.stat.Statistics.html) for more details on the API.
 
 {% include_example python/mllib/hypothesis_testing_kolmogorov_smirnov_test_example.py %}
 </div>
@@ -337,12 +337,12 @@ JavaDoubleRDD v = u.mapToDouble(x -> 1.0 + 2.0 * x);
 </div>
 
 <div data-lang="python" markdown="1">
-[`RandomRDDs`](api/python/pyspark.mllib.html#pyspark.mllib.random.RandomRDDs) provides factory
+[`RandomRDDs`](api/python/reference/api/pyspark.mllib.random.RandomRDDs.html) provides factory
 methods to generate random double RDDs or vector RDDs.
 The following example generates a random double RDD, whose values follows the standard normal
 distribution `N(0, 1)`, and then map it to `N(1, 4)`.
 
-Refer to the [`RandomRDDs` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.random.RandomRDDs) for more details on the API.
+Refer to the [`RandomRDDs` Python docs](api/python/reference/api/pyspark.mllib.random.RandomRDDs.html) for more details on the API.
 
 {% highlight python %}
 from pyspark.mllib.random import RandomRDDs
@@ -390,11 +390,11 @@ Refer to the [`KernelDensity` Java docs](api/java/org/apache/spark/mllib/stat/Ke
 </div>
 
 <div data-lang="python" markdown="1">
-[`KernelDensity`](api/python/pyspark.mllib.html#pyspark.mllib.stat.KernelDensity) provides methods
+[`KernelDensity`](api/python/reference/api/pyspark.mllib.stat.KernelDensity.html) provides methods
 to compute kernel density estimates from an RDD of samples. The following example demonstrates how
 to do so.
 
-Refer to the [`KernelDensity` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.stat.KernelDensity) for more details on the API.
+Refer to the [`KernelDensity` Python docs](api/python/reference/api/pyspark.mllib.stat.KernelDensity.html) for more details on the API.
 
 {% include_example python/mllib/kernel_density_estimation_example.py %}
 </div>
diff --git a/docs/monitoring.md b/docs/monitoring.md
index 4da0f8e9d71e9..a9f25bac52d64 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -19,6 +19,9 @@ license: |
   limitations under the License.
 ---
 
+* This will become a table of contents (this text will be scraped).
+{:toc}
+
 There are several ways to monitor Spark applications: web UIs, metrics, and external instrumentation.
 
 # Web Interfaces
@@ -373,6 +376,25 @@ Security options for the Spark History Server are covered more detail in the
     </td>
     <td>3.0.0</td>
   </tr>
+  <tr>
+    <td>spark.history.store.hybridStore.enabled</td>
+    <td>false</td>
+    <td>
+      Whether to use HybridStore as the store when parsing event logs. HybridStore will first write data
+      to an in-memory store and having a background thread that dumps data to a disk store after the writing
+      to in-memory store is completed.
+    </td>
+    <td>3.1.0</td>
+  </tr>
+  <tr>
+    <td>spark.history.store.hybridStore.maxMemoryUsage</td>
+    <td>2g</td>
+    <td>
+      Maximum memory space that can be used to create HybridStore. The HybridStore co-uses the heap memory,
+      so the heap memory should be increased through the memory option for SHS if the HybridStore is enabled.
+    </td>
+    <td>3.1.0</td>
+  </tr>
 </table>
 
 Note that in all of these UIs, the tables are sortable by clicking their headers,
@@ -402,7 +424,7 @@ to handle the Spark Context setup and tear down.
 
 In addition to viewing the metrics in the UI, they are also available as JSON.  This gives developers
 an easy way to create new visualizations and monitoring tools for Spark.  The JSON is available for
-both running applications, and in the history server.  The endpoints are mounted at `/api/v1`.  Eg.,
+both running applications, and in the history server.  The endpoints are mounted at `/api/v1`.  For example,
 for the history server, they would typically be accessible at `http://<server-url>:18080/api/v1`, and
 for a running application, at `http://localhost:4040/api/v1`.
 
@@ -453,18 +475,42 @@ can be identified by their `[attempt-id]`. In the API listed below, when running
     <td><code>/applications/[app-id]/stages</code></td>
     <td>
       A list of all stages for a given application.
-      <br><code>?status=[active|complete|pending|failed]</code> list only stages in the state.
+        <br><code>?status=[active|complete|pending|failed]</code> list only stages in the given state.
+        <br><code>?details=true</code> lists all stages with the task data.
+        <br><code>?taskStatus=[RUNNING|SUCCESS|FAILED|KILLED|PENDING]</code> lists stages only those tasks with the specified task status. Query parameter taskStatus takes effect only when <code>details=true</code>.
+        <br><code>?withSummaries=true</code> lists stages with task metrics distribution and executor metrics distribution.
+        <br><code>?quantiles=0.0,0.25,0.5,0.75,1.0</code> summarize the metrics with the given quantiles. Query parameter quantiles takes effect only when <code>withSummaries=true</code>. Default value is <code>0.0,0.25,0.5,0.75,1.0</code>. 
     </td>
   </tr>
   <tr>
     <td><code>/applications/[app-id]/stages/[stage-id]</code></td>
     <td>
       A list of all attempts for the given stage.
+        <br><code>?details=true</code> lists all attempts with the task data for the given stage.
+        <br><code>?taskStatus=[RUNNING|SUCCESS|FAILED|KILLED|PENDING]</code> lists only those tasks with the specified task status. Query parameter taskStatus takes effect only when <code>details=true</code>.
+        <br><code>?withSummaries=true</code> lists task metrics distribution and executor metrics distribution of each attempt.
+        <br><code>?quantiles=0.0,0.25,0.5,0.75,1.0</code> summarize the metrics with the given quantiles. Query parameter quantiles takes effect only when <code>withSummaries=true</code>. Default value is <code>0.0,0.25,0.5,0.75,1.0</code>. 
+      <br>Example:
+        <br><code>?details=true</code>
+        <br><code>?details=true&taskStatus=RUNNING</code>
+        <br><code>?withSummaries=true</code>
+        <br><code>?details=true&withSummaries=true&quantiles=0.01,0.5,0.99</code>
     </td>
   </tr>
   <tr>
     <td><code>/applications/[app-id]/stages/[stage-id]/[stage-attempt-id]</code></td>
-    <td>Details for the given stage attempt.</td>
+    <td>
+      Details for the given stage attempt.
+        <br><code>?details=true</code> lists all task data for the given stage attempt.
+        <br><code>?taskStatus=[RUNNING|SUCCESS|FAILED|KILLED|PENDING]</code> lists only those tasks with the specified task status. Query parameter taskStatus takes effect only when <code>details=true</code>.
+        <br><code>?withSummaries=true</code> lists task metrics distribution and executor metrics distribution for the given stage attempt.
+        <br><code>?quantiles=0.0,0.25,0.5,0.75,1.0</code> summarize the metrics with the given quantiles. Query parameter quantiles takes effect only when <code>withSummaries=true</code>. Default value is <code>0.0,0.25,0.5,0.75,1.0</code>. 
+      <br>Example:
+        <br><code>?details=true</code>
+        <br><code>?details=true&taskStatus=RUNNING</code>
+        <br><code>?withSummaries=true</code>
+        <br><code>?details=true&withSummaries=true&quantiles=0.01,0.5,0.99</code>
+    </td>
   </tr>
   <tr>
     <td><code>/applications/[app-id]/stages/[stage-id]/[stage-attempt-id]/taskSummary</code></td>
@@ -480,7 +526,8 @@ can be identified by their `[attempt-id]`. In the API listed below, when running
        A list of all tasks for the given stage attempt.
       <br><code>?offset=[offset]&amp;length=[len]</code> list tasks in the given range.
       <br><code>?sortBy=[runtime|-runtime]</code> sort the tasks.
-      <br>Example: <code>?offset=10&amp;length=50&amp;sortBy=runtime</code>
+      <br><code>?status=[running|success|killed|failed|unknown]</code> list only tasks in the state.
+      <br>Example: <code>?offset=10&amp;length=50&amp;sortBy=runtime&amp;status=running</code>
     </td>
   </tr>
   <tr>
@@ -544,6 +591,26 @@ can be identified by their `[attempt-id]`. In the API listed below, when running
     <td><code>/applications/[app-id]/streaming/batches/[batch-id]/operations/[outputOp-id]</code></td>
     <td>Details of the given operation and given batch.</td>
   </tr>
+  <tr>
+    <td><code>/applications/[app-id]/sql</code></td>
+    <td>A list of all queries for a given application.
+    <br>
+    <code>?details=[true (default) | false]</code> lists/hides details of Spark plan nodes.
+    <br>
+    <code>?planDescription=[true (default) | false]</code> enables/disables Physical <code>planDescription</code> on demand when Physical Plan size is high.
+    <br>
+    <code>?offset=[offset]&length=[len]</code> lists queries in the given range.
+    </td>
+  </tr>
+  <tr>
+    <td><code>/applications/[app-id]/sql/[execution-id]</code></td>
+    <td>Details for the given query.
+    <br>
+    <code>?details=[true (default) | false]</code> lists/hides metric details in addition to given query details.
+    <br>
+    <code>?planDescription=[true (default) | false]</code> enables/disables Physical <code>planDescription</code> on demand for the given query when Physical Plan size is high.
+    </td>
+  </tr>
   <tr>
     <td><code>/applications/[app-id]/environment</code></td>
     <td>Environment details of the given application.</td>
@@ -715,10 +782,10 @@ A list of the available metrics, with a short description:
 Executor-level metrics are sent from each executor to the driver as part of the Heartbeat to describe the performance metrics of Executor itself like JVM heap memory, GC information.
 Executor metric values and their measured memory peak values per executor are exposed via the REST API in JSON format and in Prometheus format.
 The JSON end point is exposed at: `/applications/[app-id]/executors`, and the Prometheus endpoint at: `/metrics/executors/prometheus`.
-The Prometheus endpoint is experimental and conditional to a configuration parameter: `spark.ui.prometheus.enabled=true` (the default is `false`).
+The Prometheus endpoint is conditional to a configuration parameter: `spark.ui.prometheus.enabled=true` (the default is `false`).
 In addition, aggregated per-stage peak values of the executor memory metrics are written to the event log if
 `spark.eventLog.logStageExecutorMetrics` is true.  
-Executor memory metrics are also exposed via the Spark metrics system based on the Dropwizard metrics library.
+Executor memory metrics are also exposed via the Spark metrics system based on the [Dropwizard metrics library](http://metrics.dropwizard.io/4.1.1).
 A list of the available metrics, with a short description:
 
 <table class="table">
@@ -911,18 +978,18 @@ These endpoints have been strongly versioned to make it easier to develop applic
 * Individual fields will never be removed for any given endpoint
 * New endpoints may be added
 * New fields may be added to existing endpoints
-* New versions of the api may be added in the future as a separate endpoint (eg., `api/v2`).  New versions are *not* required to be backwards compatible.
+* New versions of the api may be added in the future as a separate endpoint (e.g., `api/v2`).  New versions are *not* required to be backwards compatible.
 * Api versions may be dropped, but only after at least one minor release of co-existing with a new api version.
 
 Note that even when examining the UI of running applications, the `applications/[app-id]` portion is
-still required, though there is only one application available.  Eg. to see the list of jobs for the
+still required, though there is only one application available.  E.g. to see the list of jobs for the
 running app, you would go to `http://localhost:4040/api/v1/applications/[app-id]/jobs`.  This is to
 keep the paths consistent in both modes.
 
 # Metrics
 
 Spark has a configurable metrics system based on the
-[Dropwizard Metrics Library](http://metrics.dropwizard.io/).
+[Dropwizard Metrics Library](http://metrics.dropwizard.io/4.1.1).
 This allows users to report Spark metrics to a variety of sinks including HTTP, JMX, and CSV
 files. The metrics are generated by sources embedded in the Spark code base. They
 provide instrumentation for specific activities and Spark components.
@@ -1016,7 +1083,7 @@ activates the JVM source:
 ## List of available metrics providers 
 
 Metrics used by Spark are of multiple types: gauge, counter, histogram, meter and timer, 
-see [Dropwizard library documentation for details](https://metrics.dropwizard.io/3.1.0/getting-started/).
+see [Dropwizard library documentation for details](https://metrics.dropwizard.io/4.1.1/getting-started.html).
 The following list of components and metrics reports the name and some details about the available metrics,
 grouped per component instance and source namespace.
 The most common time of metrics used in Spark instrumentation are gauges and counters. 
@@ -1042,7 +1109,7 @@ This is the component with the largest amount of instrumented metrics
   - memory.remainingOnHeapMem_MB
 
 - namespace=HiveExternalCatalog
-  - **note:**: these metrics are conditional to a configuration parameter:
+  - **note:** these metrics are conditional to a configuration parameter:
     `spark.metrics.staticSources.enabled` (default is true) 
   - fileCacheHits.count
   - filesDiscovered.count
@@ -1051,7 +1118,7 @@ This is the component with the largest amount of instrumented metrics
   - partitionsFetched.count
 
 - namespace=CodeGenerator
-  - **note:**: these metrics are conditional to a configuration parameter:
+  - **note:** these metrics are conditional to a configuration parameter:
     `spark.metrics.staticSources.enabled` (default is true) 
   - compilationTime (histogram)
   - generatedClassSize (histogram)
@@ -1085,12 +1152,14 @@ This is the component with the largest amount of instrumented metrics
   - stages.failedStages.count
   - stages.skippedStages.count
   - stages.completedStages.count
-  - tasks.blackListedExecutors.count
+  - tasks.blackListedExecutors.count // deprecated use excludedExecutors instead
+  - tasks.excludedExecutors.count
   - tasks.completedTasks.count
   - tasks.failedTasks.count
   - tasks.killedTasks.count
   - tasks.skippedTasks.count
-  - tasks.unblackListedExecutors.count
+  - tasks.unblackListedExecutors.count // deprecated use unexcludedExecutors instead
+  - tasks.unexcludedExecutors.count
   - jobs.succeededJobs
   - jobs.failedJobs
   - jobDuration
@@ -1113,22 +1182,41 @@ This is the component with the largest amount of instrumented metrics
 - namespace=JVMCPU
   - jvmCpuTime
 
+- namespace=executor
+  - **note:** These metrics are available in the driver in local mode only.
+  - A full list of available metrics in this 
+    namespace can be found in the corresponding entry for the Executor component instance.
+    
 - namespace=ExecutorMetrics
   - **note:** these metrics are conditional to a configuration parameter:
     `spark.metrics.executorMetricsSource.enabled` (default is true) 
   - This source contains memory-related metrics. A full list of available metrics in this 
     namespace can be found in the corresponding entry for the Executor component instance.
- 
+
+- namespace=ExecutorAllocationManager
+  - **note:** these metrics are only emitted when using dynamic allocation. Conditional to a configuration
+    parameter `spark.dynamicAllocation.enabled` (default is false)
+  - executors.numberExecutorsToAdd  
+  - executors.numberExecutorsPendingToRemove
+  - executors.numberAllExecutors
+  - executors.numberTargetExecutors
+  - executors.numberMaxNeededExecutors
+  - executors.numberExecutorsGracefullyDecommissioned.count
+  - executors.numberExecutorsDecommissionUnfinished.count
+  - executors.numberExecutorsExitedUnexpectedly.count
+  - executors.numberExecutorsKilledByDriver.count
+
 - namespace=plugin.\<Plugin Class Name>
   - Optional namespace(s). Metrics in this namespace are defined by user-supplied code, and
   configured using the Spark plugin API. See "Advanced Instrumentation" below for how to load
   custom plugins into Spark.
 
 ### Component instance = Executor
-These metrics are exposed by Spark executors. Note, currently they are not available
-when running in local mode.
+These metrics are exposed by Spark executors. 
  
 - namespace=executor (metrics are of type counter or gauge)
+  - **notes:**
+    - `spark.executor.metrics.fileSystemSchemes` (default: `file,hdfs`) determines the exposed file system metrics.
   - bytesRead.count
   - bytesWritten.count
   - cpuTime.count
@@ -1214,7 +1302,7 @@ when running in local mode.
   - shuffle-server.usedHeapMemory
 
 - namespace=HiveExternalCatalog
-  - **note:**: these metrics are conditional to a configuration parameter:
+  - **note:** these metrics are conditional to a configuration parameter:
     `spark.metrics.staticSources.enabled` (default is true) 
   - fileCacheHits.count
   - filesDiscovered.count
@@ -1223,12 +1311,11 @@ when running in local mode.
   - partitionsFetched.count
 
 - namespace=CodeGenerator
-  - **note:**: these metrics are conditional to a configuration parameter:
+  - **note:** these metrics are conditional to a configuration parameter:
     `spark.metrics.staticSources.enabled` (default is true) 
   - compilationTime (histogram)
   - generatedClassSize (histogram)
   - generatedMethodSize (histogram)
-  - hiveClientCalls.count
   - sourceCodeSize (histogram)
 
 - namespace=plugin.\<Plugin Class Name>
@@ -1240,9 +1327,11 @@ when running in local mode.
 Notes: 
   - Activate this source by setting the relevant `metrics.properties` file entry or the 
   configuration parameter:`spark.metrics.conf.*.source.jvm.class=org.apache.spark.metrics.source.JvmSource`  
+  - These metrics are conditional to a configuration parameter:
+    `spark.metrics.staticSources.enabled` (default is true)
   - This source is available for driver and executor instances and is also available for other instances.  
   - This source provides information on JVM metrics using the 
-  [Dropwizard/Codahale Metric Sets for JVM instrumentation](https://metrics.dropwizard.io/3.1.0/manual/jvm/)
+  [Dropwizard/Codahale Metric Sets for JVM instrumentation](https://metrics.dropwizard.io/4.1.1/manual/jvm.html)
    and in particular the metric sets BufferPoolMetricSet, GarbageCollectorMetricSet and MemoryUsageGaugeSet. 
 
 ### Component instance = applicationMaster
@@ -1324,9 +1413,3 @@ Both take a comma-separated list of class names that implement the
 possible for one list to be placed in the Spark default config file, allowing users to
 easily add other plugins from the command line without overwriting the config file's list. Duplicate
 plugins are ignored.
-
-Distribution of the jar files containing the plugin code is currently not done by Spark. The user
-or admin should make sure that the jar files are available to Spark applications, for example, by
-including the plugin jar with the Spark distribution. The exception to this rule is the YARN
-backend, where the <code>--jars</code> command line option (or equivalent config entry) can be
-used to make the plugin code available to both executors and cluster-mode drivers.
diff --git a/docs/pyspark-migration-guide.md b/docs/pyspark-migration-guide.md
index 6f0fbbfb78de8..5a1ca9be94d8d 100644
--- a/docs/pyspark-migration-guide.md
+++ b/docs/pyspark-migration-guide.md
@@ -19,63 +19,4 @@ license: |
   limitations under the License.
 ---
 
-* Table of contents
-{:toc}
-
-Note that this migration guide describes the items specific to PySpark.
-Many items of SQL migration can be applied when migrating PySpark to higher versions.
-Please refer [Migration Guide: SQL, Datasets and DataFrame](sql-migration-guide.html).
-
-## Upgrading from PySpark 2.4 to 3.0
-- In Spark 3.0, PySpark requires a pandas version of 0.23.2 or higher to use pandas related functionality, such as `toPandas`, `createDataFrame` from pandas DataFrame, and so on.
-
-- In Spark 3.0, PySpark requires a PyArrow version of 0.12.1 or higher to use PyArrow related functionality, such as `pandas_udf`, `toPandas` and `createDataFrame` with "spark.sql.execution.arrow.enabled=true", etc.
-
-- In PySpark, when creating a `SparkSession` with `SparkSession.builder.getOrCreate()`, if there is an existing `SparkContext`, the builder was trying to update the `SparkConf` of the existing `SparkContext` with configurations specified to the builder, but the `SparkContext` is shared by all `SparkSession`s, so we should not update them. In 3.0, the builder comes to not update the configurations. This is the same behavior as Java/Scala API in 2.3 and above. If you want to update them, you need to update them prior to creating a `SparkSession`.
-
-- In PySpark, when Arrow optimization is enabled, if Arrow version is higher than 0.11.0, Arrow can perform safe type conversion when converting `pandas.Series` to an Arrow array during serialization. Arrow raises errors when detecting unsafe type conversions like overflow. You enable it by setting `spark.sql.execution.pandas.convertToArrowArraySafely` to `true`. The default setting is `false`. PySpark behavior for Arrow versions is illustrated in the following table:
-
-  | PyArrow version  | Integer overflow | Floating point truncation |
-  | ---------------- | ---------------- | ------------------------- |
-  | 0.11.0 and below | Raise error      | Silently allows           |
-  | \> 0.11.0, arrowSafeTypeConversion=false | Silent overflow  | Silently allows |
-  | \> 0.11.0, arrowSafeTypeConversion=true  | Raise error      | Raise error |
-   
-- In Spark 3.0, `createDataFrame(..., verifySchema=True)` validates `LongType` as well in PySpark. Previously, `LongType` was not verified and resulted in `None` in case the value overflows. To restore this behavior, `verifySchema` can be set to `False` to disable the validation.
-
-- As of Spark 3.0, `Row` field names are no longer sorted alphabetically when constructing with named arguments for Python versions 3.6 and above, and the order of fields will match that as entered. To enable sorted fields by default, as in Spark 2.4, set the environment variable `PYSPARK_ROW_FIELD_SORTING_ENABLED` to `true` for both executors and driver - this environment variable must be consistent on all executors and driver; otherwise, it may cause failures or incorrect answers. For Python versions less than 3.6, the field names will be sorted alphabetically as the only option.
-
-## Upgrading from PySpark 2.3 to 2.4
-
-  - In PySpark, when Arrow optimization is enabled, previously `toPandas` just failed when Arrow optimization is unable to be used whereas `createDataFrame` from Pandas DataFrame allowed the fallback to non-optimization. Now, both `toPandas` and `createDataFrame` from Pandas DataFrame allow the fallback by default, which can be switched off by `spark.sql.execution.arrow.fallback.enabled`.
-
-## Upgrading from PySpark 2.3.0 to 2.3.1 and above
-
-  - As of version 2.3.1 Arrow functionality, including `pandas_udf` and `toPandas()`/`createDataFrame()` with `spark.sql.execution.arrow.enabled` set to `True`, has been marked as experimental. These are still evolving and not currently recommended for use in production.
-
-## Upgrading from PySpark 2.2 to 2.3
-
-  - In PySpark, now we need Pandas 0.19.2 or upper if you want to use Pandas related functionalities, such as `toPandas`, `createDataFrame` from Pandas DataFrame, etc.
-
-  - In PySpark, the behavior of timestamp values for Pandas related functionalities was changed to respect session timezone. If you want to use the old behavior, you need to set a configuration `spark.sql.execution.pandas.respectSessionTimeZone` to `False`. See [SPARK-22395](https://issues.apache.org/jira/browse/SPARK-22395) for details.
-
-  - In PySpark, `na.fill()` or `fillna` also accepts boolean and replaces nulls with booleans. In prior Spark versions, PySpark just ignores it and returns the original Dataset/DataFrame.
-
-  - In PySpark, `df.replace` does not allow to omit `value` when `to_replace` is not a dictionary. Previously, `value` could be omitted in the other cases and had `None` by default, which is counterintuitive and error-prone.
-
-## Upgrading from PySpark 1.4 to 1.5
-
- - Resolution of strings to columns in Python now supports using dots (`.`) to qualify the column or
-   access nested values. For example `df['table.column.nestedField']`. However, this means that if
-   your column name contains any dots you must now escape them using backticks (e.g., ``table.`column.with.dots`.nested``).
-
- - DataFrame.withColumn method in PySpark supports adding a new column or replacing existing columns of the same name.
-
-
-## Upgrading from PySpark 1.0-1.2 to 1.3
-
-#### Python DataTypes No Longer Singletons
-{:.no_toc}
-
-When using DataTypes in Python you will need to construct them (i.e. `StringType()`) instead of
-referencing a singleton.
+The migration guide is now archived on [this page](api/python/migration_guide/index.html).
diff --git a/docs/quick-start.md b/docs/quick-start.md
index e7a16a3461653..958e1ba9209cf 100644
--- a/docs/quick-start.md
+++ b/docs/quick-start.md
@@ -32,11 +32,6 @@ you can download a package for any version of Hadoop.
 
 Note that, before Spark 2.0, the main programming interface of Spark was the Resilient Distributed Dataset (RDD). After Spark 2.0, RDDs are replaced by Dataset, which is strongly-typed like an RDD, but with richer optimizations under the hood. The RDD interface is still supported, and you can get a more detailed reference at the [RDD programming guide](rdd-programming-guide.html). However, we highly recommend you to switch to use Dataset, which has better performance than RDD. See the [SQL programming guide](sql-programming-guide.html) to get more information about Dataset.
 
-# Security
-
-Security in Spark is OFF by default. This could mean you are vulnerable to attack by default.
-Please see [Spark Security](security.html) before running Spark.
-
 # Interactive Analysis with the Spark Shell
 
 ## Basics
@@ -399,7 +394,7 @@ If you are building a packaged PySpark application or library you can add it to
 
 {% highlight python %}
     install_requires=[
-        'pyspark=={site.SPARK_VERSION}'
+        'pyspark=={{site.SPARK_VERSION}}'
     ]
 {% endhighlight %}
 
diff --git a/docs/rdd-programming-guide.md b/docs/rdd-programming-guide.md
index 70bfefce475a1..acc682b27681b 100644
--- a/docs/rdd-programming-guide.md
+++ b/docs/rdd-programming-guide.md
@@ -101,10 +101,10 @@ import org.apache.spark.SparkConf;
 
 <div data-lang="python"  markdown="1">
 
-Spark {{site.SPARK_VERSION}} works with Python 2.7+ or Python 3.4+. It can use the standard CPython interpreter,
+Spark {{site.SPARK_VERSION}} works with Python 3.6+. It can use the standard CPython interpreter,
 so C libraries like NumPy can be used. It also works with PyPy 2.3+.
 
-Note that Python 2 support is deprecated as of Spark 3.0.0.
+Python 2, 3.4 and 3.5 supports were removed in Spark 3.1.0.
 
 Spark applications in Python can either be run with the `bin/spark-submit` script which includes Spark at runtime, or by including it in your setup.py as:
 
@@ -134,8 +134,8 @@ PySpark requires the same minor version of Python in both driver and workers. It
 you can specify which version of Python you want to use by `PYSPARK_PYTHON`, for example:
 
 {% highlight bash %}
-$ PYSPARK_PYTHON=python3.4 bin/pyspark
-$ PYSPARK_PYTHON=/opt/pypy-2.5/bin/pypy bin/spark-submit examples/src/main/python/pi.py
+$ PYSPARK_PYTHON=python3.8 bin/pyspark
+$ PYSPARK_PYTHON=/path-to-your-pypy/pypy bin/spark-submit examples/src/main/python/pi.py
 {% endhighlight %}
 
 </div>
@@ -276,7 +276,7 @@ $ PYSPARK_DRIVER_PYTHON=jupyter PYSPARK_DRIVER_PYTHON_OPTS=notebook ./bin/pyspar
 
 You can customize the `ipython` or `jupyter` commands by setting `PYSPARK_DRIVER_PYTHON_OPTS`.
 
-After the Jupyter Notebook server is launched, you can create a new "Python 2" notebook from
+After the Jupyter Notebook server is launched, you can create a new notebook from
 the "Files" tab. Inside the notebook, you can input the command `%pylab inline` as part of
 your notebook before you start to try Spark from the Jupyter notebook.
 
@@ -447,7 +447,7 @@ Writables are automatically converted:
 
 <table class="table">
 <tr><th>Writable Type</th><th>Python Type</th></tr>
-<tr><td>Text</td><td>unicode str</td></tr>
+<tr><td>Text</td><td>str</td></tr>
 <tr><td>IntWritable</td><td>int</td></tr>
 <tr><td>FloatWritable</td><td>float</td></tr>
 <tr><td>DoubleWritable</td><td>float</td></tr>
@@ -1254,9 +1254,9 @@ storage levels is:
 </tr>
 </table>
 
-**Note:** *In Python, stored objects will always be serialized with the [Pickle](https://docs.python.org/2/library/pickle.html) library,
+**Note:** *In Python, stored objects will always be serialized with the [Pickle](https://docs.python.org/3/library/pickle.html) library,
 so it does not matter whether you choose a serialized level. The available storage levels in Python include `MEMORY_ONLY`, `MEMORY_ONLY_2`,
-`MEMORY_AND_DISK`, `MEMORY_AND_DISK_2`, `DISK_ONLY`, and `DISK_ONLY_2`.*
+`MEMORY_AND_DISK`, `MEMORY_AND_DISK_2`, `DISK_ONLY`, `DISK_ONLY_2`, and `DISK_ONLY_3`.*
 
 Spark also automatically persists some intermediate data in shuffle operations (e.g. `reduceByKey`), even without users calling `persist`. This is done to avoid recomputing the entire input if a node fails during the shuffle. We still recommend users call `persist` on the resulting RDD if they plan to reuse it.
 
diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md
index ba735cacd4c38..530951e839543 100644
--- a/docs/running-on-kubernetes.md
+++ b/docs/running-on-kubernetes.md
@@ -23,14 +23,12 @@ license: |
 Spark can run on clusters managed by [Kubernetes](https://kubernetes.io). This feature makes use of native
 Kubernetes scheduler that has been added to Spark.
 
-**The Kubernetes scheduler is currently experimental.
-In future versions, there may be behavioral changes around configuration,
-container images and entrypoints.**
-
 # Security
 
-Security in Spark is OFF by default. This could mean you are vulnerable to attack by default.
-Please see [Spark Security](security.html) and the specific advice below before running Spark.
+Security features like authentication are not enabled by default. When deploying a cluster that is open to the internet
+or an untrusted network, it's important to secure access to the cluster to prevent unauthorized applications
+from running on the cluster.
+Please see [Spark Security](security.html) and the specific security sections in this doc before running Spark.
 
 ## User Identity
 
@@ -197,6 +195,9 @@ for any reason, these pods will remain in the cluster. The executor processes sh
 driver, so the executor pods should not consume compute resources (cpu and memory) in the cluster after your application
 exits.
 
+You may use `spark.kubernetes.executor.podNamePrefix` to fully control the executor pod names.
+When this property is set, it's highly recommended to make it unique across all jobs in the same namespace.
+
 ### Authentication Parameters
 
 Use the exact prefix `spark.kubernetes.authenticate` for Kubernetes authentication parameters in client mode.
@@ -213,7 +214,7 @@ A typical example of this using S3 is via passing the following options:
 
 ```
 ...
---packages com.amazonaws:aws-java-sdk:1.7.4,org.apache.hadoop:hadoop-aws:2.7.6
+--packages org.apache.hadoop:hadoop-aws:3.2.2
 --conf spark.kubernetes.file.upload.path=s3a://<s3-bucket>/path
 --conf spark.hadoop.fs.s3a.access.key=...
 --conf spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem
@@ -226,7 +227,7 @@ The app jar file will be uploaded to the S3 and then when the driver is launched
 to the driver pod and will be added to its classpath. Spark will generate a subdir under the upload path with a random name
 to avoid conflicts with spark apps running in parallel. User could manage the subdirs created according to his needs.
 
-The client scheme is supported for the application jar, and dependencies specified by properties `spark.jars` and `spark.files`.
+The client scheme is supported for the application jar, and dependencies specified by properties `spark.jars`, `spark.files` and `spark.archives`.
 
 Important: all client-side dependencies will be uploaded to the given path with a flat directory structure so
 file names must be unique otherwise files will be overwritten. Also make sure in the derived k8s image default ivy dir
@@ -257,7 +258,14 @@ To use a secret through an environment variable use the following options to the
 Kubernetes allows defining pods from [template files](https://kubernetes.io/docs/concepts/workloads/pods/pod-overview/#pod-templates).
 Spark users can similarly use template files to define the driver or executor pod configurations that Spark configurations do not support.
 To do so, specify the spark properties `spark.kubernetes.driver.podTemplateFile` and `spark.kubernetes.executor.podTemplateFile`
-to point to local files accessible to the `spark-submit` process. To allow the driver pod access the executor pod template
+to point to files accessible to the `spark-submit` process.
+
+```
+--conf spark.kubernetes.driver.podTemplateFile=s3a://bucket/driver.yml
+--conf spark.kubernetes.executor.podTemplateFile=s3a://bucket/executor.yml
+```
+
+To allow the driver pod access the executor pod template
 file, the file will be automatically mounted onto a volume in the driver pod when it's created.
 Spark does not do any validation after unmarshalling these template files and relies on the Kubernetes API server for validation.
 
@@ -311,7 +319,18 @@ And, the claim name of a `persistentVolumeClaim` with volume name `checkpointpvc
 spark.kubernetes.driver.volumes.persistentVolumeClaim.checkpointpvc.options.claimName=check-point-pvc-claim
 ```
 
-The configuration properties for mounting volumes into the executor pods use prefix `spark.kubernetes.executor.` instead of `spark.kubernetes.driver.`. For a complete list of available options for each supported type of volumes, please refer to the [Spark Properties](#spark-properties) section below.
+The configuration properties for mounting volumes into the executor pods use prefix `spark.kubernetes.executor.` instead of `spark.kubernetes.driver.`.
+
+For example, you can mount a dynamically-created persistent volume claim per executor by using `OnDemand` as a claim name and `storageClass` and `sizeLimit` options like the following. This is useful in case of [Dynamic Allocation](configuration.html#dynamic-allocation).
+```
+spark.kubernetes.executor.volumes.persistentVolumeClaim.data.options.claimName=OnDemand
+spark.kubernetes.executor.volumes.persistentVolumeClaim.data.options.storageClass=gp
+spark.kubernetes.executor.volumes.persistentVolumeClaim.data.options.sizeLimit=500Gi
+spark.kubernetes.executor.volumes.persistentVolumeClaim.data.mount.path=/data
+spark.kubernetes.executor.volumes.persistentVolumeClaim.data.mount.readOnly=false
+```
+
+For a complete list of available options for each supported type of volumes, please refer to the [Spark Properties](#spark-properties) section below.
 
 ## Local Storage
 
@@ -322,6 +341,15 @@ Spark supports using volumes to spill data during shuffles and other operations.
 --conf spark.kubernetes.driver.volumes.[VolumeType].spark-local-dir-[VolumeName].mount.readOnly=false
 ```
 
+Specifically, you can use persistent volume claims if the jobs require large shuffle and sorting operations in executors.
+
+```
+spark.kubernetes.executor.volumes.persistentVolumeClaim.spark-local-dir-1.options.claimName=OnDemand
+spark.kubernetes.executor.volumes.persistentVolumeClaim.spark-local-dir-1.options.storageClass=gp
+spark.kubernetes.executor.volumes.persistentVolumeClaim.spark-local-dir-1.options.sizeLimit=500Gi
+spark.kubernetes.executor.volumes.persistentVolumeClaim.spark-local-dir-1.mount.path=/data
+spark.kubernetes.executor.volumes.persistentVolumeClaim.spark-local-dir-1.mount.readOnly=false
+```
 
 If no volume is set as local storage, Spark uses temporary scratch space to spill data to disk during shuffles and other operations. When using Kubernetes as the resource manager the pods will be created with an [emptyDir](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir) volume mounted for each directory listed in `spark.local.dir` or the environment variable `SPARK_LOCAL_DIRS` .  If no directories are explicitly specified then a default directory is created and configured appropriately.
 
@@ -857,6 +885,14 @@ See the [configuration page](configuration.html) for information on Spark config
   </td>
   <td>2.3.0</td>
 </tr>
+<tr>
+  <td><code>spark.kubernetes.executor.podNamePrefix</code></td>
+  <td>(none)</td>
+  <td>
+    Prefix to use in front of the executor pod names.
+  </td>
+  <td>2.3.0</td>
+</tr>
 <tr>
   <td><code>spark.kubernetes.executor.lostCheck.maxAttempts</code></td>
   <td><code>10</code></td>
@@ -1063,7 +1099,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td><code>0.1</code></td>
   <td>
     This sets the Memory Overhead Factor that will allocate memory to non-JVM memory, which includes off-heap memory allocations, non-JVM tasks, and various systems processes. For JVM-based jobs this value will default to 0.10 and 0.40 for non-JVM jobs.
-    This is done as non-JVM tasks need more non-JVM heap space and such tasks commonly fail with "Memory Overhead Exceeded" errors. This prempts this error with a higher default.
+    This is done as non-JVM tasks need more non-JVM heap space and such tasks commonly fail with "Memory Overhead Exceeded" errors. This preempts this error with a higher default.
   </td>
   <td>2.4.0</td>
 </tr>
@@ -1071,7 +1107,10 @@ See the [configuration page](configuration.html) for information on Spark config
   <td><code>spark.kubernetes.pyspark.pythonVersion</code></td>
   <td><code>"3"</code></td>
   <td>
-   This sets the major Python version of the docker image used to run the driver and executor containers. Can either be 2 or 3.
+   This sets the major Python version of the docker image used to run the driver and executor containers.
+   It can be only "3". This configuration was deprecated from Spark 3.1.0, and is effectively no-op.
+   Users should set 'spark.pyspark.python' and 'spark.pyspark.driver.python' configurations or
+   'PYSPARK_PYTHON' and 'PYSPARK_DRIVER_PYTHON' environment variables.
   </td>
   <td>2.4.0</td>
 </tr>
@@ -1165,6 +1204,14 @@ See the [configuration page](configuration.html) for information on Spark config
   </td>
   <td>3.0.0</td>
 </tr>
+<tr>
+  <td><code>spark.kubernetes.executor.checkAllContainers</code></td>
+  <td>false</td>
+  <td>
+  Specify whether executor pods should be check all containers (including sidecars) or only the executor container when determining the pod status.
+  </td>
+  <td>3.1.0</td>
+</tr>
 <tr>
   <td><code>spark.kubernetes.submission.connectionTimeout</code></td>
   <td>10000</td>
@@ -1341,7 +1388,7 @@ The following affect the driver and executor containers. All other containers in
   <td>See description</td>
   <td>
     The container name will be assigned by spark ("spark-kubernetes-driver" for the driver container, and
-    "executor" for each executor container) if not defined by the pod template. If the container is defined by the
+    "spark-kubernetes-executor" for each executor container) if not defined by the pod template. If the container is defined by the
     template, the template's name will be used.
   </td>
 </tr>
@@ -1375,3 +1422,7 @@ Spark automatically handles translating the Spark configs <code>spark.{driver/ex
 
 Kubernetes does not tell Spark the addresses of the resources allocated to each container. For that reason, the user must specify a discovery script that gets run by the executor on startup to discover what resources are available to that executor. You can find an example scripts in `examples/src/main/scripts/getGpusResources.sh`. The script must have execute permissions set and the user should setup permissions to not allow malicious users to modify it. The script should write to STDOUT a JSON string in the format of the ResourceInformation class. This has the resource name and an array of resource addresses available to just that executor.
 
+### Stage Level Scheduling Overview
+
+Stage level scheduling is supported on Kubernetes when dynamic allocation is enabled. This also requires <code>spark.dynamicAllocation.shuffleTracking.enabled</code> to be enabled since Kubernetes doesn't support an external shuffle service at this time. The order in which containers for different profiles is requested from Kubernetes is not guaranteed. Note that since dynamic allocation on Kubernetes requires the shuffle tracking feature, this means that executors from previous stages that used a different ResourceProfile may not idle timeout due to having shuffle data on them. This could result in using more cluster resources and in the worst case if there are no remaining resources on the Kubernetes cluster then Spark could potentially hang. You may consider looking at config <code>spark.dynamicAllocation.shuffleTracking.timeout</code> to set a timeout, but that could result in data having to be recomputed if the shuffle data is really needed.
+Note, there is a difference in the way pod template resources are handled between the base default profile and custom ResourceProfiles. Any resources specified in the pod template file will only be used with the base default profile. If you create custom ResourceProfiles be sure to include all necessary resources there since the resources from the template file will not be propagated to custom ResourceProfiles.
diff --git a/docs/running-on-mesos.md b/docs/running-on-mesos.md
index 6f6ae1c0ff264..52325f370db1d 100644
--- a/docs/running-on-mesos.md
+++ b/docs/running-on-mesos.md
@@ -19,6 +19,8 @@ license: |
 ---
 * This will become a table of contents (this text will be scraped).
 {:toc}
+  
+*Note*: Apache Mesos support is deprecated as of Apache Spark 3.2.0. It will be removed in a future version.
 
 Spark can run on hardware clusters managed by [Apache Mesos](http://mesos.apache.org/).
 
@@ -30,7 +32,9 @@ The advantages of deploying Spark with Mesos include:
 
 # Security
 
-Security in Spark is OFF by default. This could mean you are vulnerable to attack by default.
+Security features like authentication are not enabled by default. When deploying a cluster that is open to the internet
+or an untrusted network, it's important to secure access to the cluster to prevent unauthorized applications
+from running on the cluster.
 Please see [Spark Security](security.html) and the specific security sections in this doc before running Spark.
 
 # How it Works
@@ -91,7 +95,7 @@ but Mesos can be run without ZooKeeper using a single master as well.
 ## Verification
 
 To verify that the Mesos cluster is ready for Spark, navigate to the Mesos master webui at port
-`:5050`  Confirm that all expected machines are present in the slaves tab.
+`:5050`  Confirm that all expected machines are present in the agents tab.
 
 
 # Connecting Spark to Mesos
@@ -99,7 +103,7 @@ To verify that the Mesos cluster is ready for Spark, navigate to the Mesos maste
 To use Mesos from Spark, you need a Spark binary package available in a place accessible by Mesos, and
 a Spark driver program configured to connect to Mesos.
 
-Alternatively, you can also install Spark in the same location in all the Mesos slaves, and configure
+Alternatively, you can also install Spark in the same location in all the Mesos agents, and configure
 `spark.mesos.executor.home` (defaults to SPARK_HOME) to point to that location.
 
 ## Authenticating to Mesos
@@ -138,7 +142,7 @@ Then submit happens as described in Client mode or Cluster mode below
 
 ## Uploading Spark Package
 
-When Mesos runs a task on a Mesos slave for the first time, that slave must have a Spark binary
+When Mesos runs a task on a Mesos agent for the first time, that agent must have a Spark binary
 package for running the Spark Mesos executor backend.
 The Spark package can be hosted at any Hadoop-accessible URI, including HTTP via `http://`,
 [Amazon Simple Storage Service](http://aws.amazon.com/s3) via `s3n://`, or HDFS via `hdfs://`.
@@ -237,7 +241,7 @@ For example:
 {% endhighlight %}
 
 
-Note that jars or python files that are passed to spark-submit should be URIs reachable by Mesos slaves, as the Spark driver doesn't automatically upload local jars.
+Note that jars or python files that are passed to spark-submit should be URIs reachable by Mesos agents, as the Spark driver doesn't automatically upload local jars.
 
 # Mesos Run Modes
 
@@ -360,7 +364,7 @@ see [Dynamic Resource Allocation](job-scheduling.html#dynamic-resource-allocatio
 
 The External Shuffle Service to use is the Mesos Shuffle Service. It provides shuffle data cleanup functionality
 on top of the Shuffle Service since Mesos doesn't yet support notifying another framework's
-termination. To launch it, run `$SPARK_HOME/sbin/start-mesos-shuffle-service.sh` on all slave nodes, with `spark.shuffle.service.enabled` set to `true`.
+termination. To launch it, run `$SPARK_HOME/sbin/start-mesos-shuffle-service.sh` on all agent nodes, with `spark.shuffle.service.enabled` set to `true`.
 
 This can also be achieved through Marathon, using a unique host constraint, and the following command: `./bin/spark-class org.apache.spark.deploy.mesos.MesosExternalShuffleService`.
 
@@ -480,6 +484,15 @@ See the [configuration page](configuration.html) for information on Spark config
   </td>
   <td>1.1.1</td>
 </tr>
+<tr>
+  <td><code>spark.mesos.driver.memoryOverhead</code></td>
+  <td>driver memory * 0.10, with minimum of 384</td>
+  <td>
+    The amount of additional memory, specified in MB, to be allocated to the driver. By default,
+    the overhead will be larger of either 384 or 10% of <code>spark.driver.memory</code>. If set,
+    the final overhead will be this value. Only applies to cluster mode.
+  </td>
+</tr>
 <tr>
   <td><code>spark.mesos.uris</code></td>
   <td>(none)</td>
@@ -734,6 +747,38 @@ See the [configuration page](configuration.html) for information on Spark config
   </td>
   <td>2.1.0</td>
 </tr>
+<tr>
+  <td><code>spark.mesos.dispatcher.queue</code></td>
+  <td><code>(none)</code></td>
+  <td>
+    Set the name of the dispatcher queue to which the application is submitted.
+    The specified queue must be added to the dispatcher with <code>spark.mesos.dispatcher.queue.[QueueName]</code>.
+    If no queue is specified, then the application is submitted to the "default" queue with 0.0 priority.
+  </td>
+  <td>3.1.0</td>
+</tr>
+<tr>
+  <td><code>spark.mesos.dispatcher.queue.[QueueName]</code></td>
+  <td><code>0.0</code></td>
+  <td>
+    Add a new queue for submitted drivers with the specified priority.
+    Higher numbers indicate higher priority.
+    The user can specify multiple queues to define a workload management policy for queued drivers in the dispatcher.
+    A driver can then be submitted to a specific queue with <code>spark.mesos.dispatcher.queue</code>.
+    By default, the dispatcher has a single queue with 0.0 priority (cannot be overridden).
+    It is possible to implement a consistent and overall workload management policy throughout the lifecycle of drivers
+    by mapping priority queues to weighted Mesos roles, and by specifying a
+    <code>spark.mesos.role</code> along with a <code>spark.mesos.dispatcher.queue</code> when submitting an application.
+    For example, with the URGENT Mesos role:
+    <pre>
+    spark.mesos.dispatcher.queue.URGENT=1.0
+
+    spark.mesos.dispatcher.queue=URGENT
+    spark.mesos.role=URGENT
+    </pre>
+  </td>
+  <td>3.1.0</td>
+</tr>
 <tr>
   <td><code>spark.mesos.gpus.max</code></td>
   <td><code>0</code></td>
@@ -825,7 +870,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td><code>host</code></td>
   <td>
     Provides support for the `local:///` scheme to reference the app jar resource in cluster mode.
-    If user uses a local resource (`local:///path/to/jar`) and the config option is not used it defaults to `host` eg.
+    If user uses a local resource (`local:///path/to/jar`) and the config option is not used it defaults to `host` e.g.
     the mesos fetcher tries to get the resource from the host's file system.
     If the value is unknown it prints a warning msg in the dispatcher logs and defaults to `host`.
     If the value is `container` then spark submit in the container will use the jar in the container's path:
@@ -840,17 +885,17 @@ See the [configuration page](configuration.html) for information on Spark config
 A few places to look during debugging:
 
 - Mesos master on port `:5050`
-  - Slaves should appear in the slaves tab
+  - Agents should appear in the agents tab
   - Spark applications should appear in the frameworks tab
   - Tasks should appear in the details of a framework
   - Check the stdout and stderr of the sandbox of failed tasks
 - Mesos logs
-  - Master and slave logs are both in `/var/log/mesos` by default
+  - Master and agent logs are both in `/var/log/mesos` by default
 
 And common pitfalls:
 
 - Spark assembly not reachable/accessible
-  - Slaves must be able to download the Spark binary package from the `http://`, `hdfs://` or `s3n://` URL you gave
+  - Agents must be able to download the Spark binary package from the `http://`, `hdfs://` or `s3n://` URL you gave
 - Firewall blocking communications
   - Check for messages about failed connections
   - Temporarily disable firewalls for debugging and then poke appropriate holes
diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md
index b58cd24ab051f..5969ed33f5a8e 100644
--- a/docs/running-on-yarn.md
+++ b/docs/running-on-yarn.md
@@ -26,7 +26,9 @@ was added to Spark in version 0.6.0, and improved in subsequent releases.
 
 # Security
 
-Security in Spark is OFF by default. This could mean you are vulnerable to attack by default.
+Security features like authentication are not enabled by default. When deploying a cluster that is open to the internet
+or an untrusted network, it's important to secure access to the cluster to prevent unauthorized applications
+from running on the cluster.
 Please see [Spark Security](security.html) and the specific security sections in this doc before running Spark.
 
 # Launching Spark on YARN
@@ -60,7 +62,7 @@ For example:
         examples/jars/spark-examples*.jar \
         10
 
-The above starts a YARN client program which starts the default Application Master. Then SparkPi will be run as a child thread of Application Master. The client will periodically poll the Application Master for status updates and display them in the console. The client will exit once your application has finished running.  Refer to the "Debugging your Application" section below for how to see driver and executor logs.
+The above starts a YARN client program which starts the default Application Master. Then SparkPi will be run as a child thread of Application Master. The client will periodically poll the Application Master for status updates and display them in the console. The client will exit once your application has finished running.  Refer to the [Debugging your Application](#debugging-your-application) section below for how to see driver and executor logs.
 
 To launch a Spark application in `client` mode, do the same, but replace `cluster` with `client`. The following shows how you can run `spark-shell` in `client` mode:
 
@@ -82,9 +84,21 @@ In `cluster` mode, the driver runs on a different machine than the client, so `S
 
 Running Spark on YARN requires a binary distribution of Spark which is built with YARN support.
 Binary distributions can be downloaded from the [downloads page](https://spark.apache.org/downloads.html) of the project website.
+There are two variants of Spark binary distributions you can download. One is pre-built with a certain
+version of Apache Hadoop; this Spark distribution contains built-in Hadoop runtime, so we call it `with-hadoop` Spark
+distribution. The other one is pre-built with user-provided Hadoop; since this Spark distribution
+doesn't contain a built-in Hadoop runtime, it's smaller, but users have to provide a Hadoop installation separately.
+We call this variant `no-hadoop` Spark distribution. For `with-hadoop` Spark distribution, since
+it contains a built-in Hadoop runtime already, by default, when a job is submitted to Hadoop Yarn cluster, to prevent jar conflict, it will not
+populate Yarn's classpath into Spark. To override this behavior, you can set <code>spark.yarn.populateHadoopClasspath=true</code>.
+For `no-hadoop` Spark distribution, Spark will populate Yarn's classpath by default in order to get Hadoop runtime. For `with-hadoop` Spark distribution,
+if your application depends on certain library that is only available in the cluster, you can try to populate the Yarn classpath by setting
+the property mentioned above. If you run into jar conflict issue by doing so, you will need to turn it off and include this library
+in your application jar.
+
 To build Spark yourself, refer to [Building Spark](building-spark.html).
 
-To make Spark runtime jars accessible from YARN side, you can specify `spark.yarn.archive` or `spark.yarn.jars`. For details please refer to [Spark Properties](running-on-yarn.html#spark-properties). If neither `spark.yarn.archive` nor `spark.yarn.jars` is specified, Spark will create a zip file with all jars under `$SPARK_HOME/jars` and upload it to the distributed cache.
+To make Spark runtime jars accessible from YARN side, you can specify `spark.yarn.archive` or `spark.yarn.jars`. For details please refer to [Spark Properties](#spark-properties). If neither `spark.yarn.archive` nor `spark.yarn.jars` is specified, Spark will create a zip file with all jars under `$SPARK_HOME/jars` and upload it to the distributed cache.
 
 # Configuration
 
@@ -396,7 +410,10 @@ To use a custom metrics.properties for the application master and executors, upd
 </tr>
 <tr>
   <td><code>spark.yarn.populateHadoopClasspath</code></td>
-  <td>true</td>
+  <td>
+    For <code>with-hadoop</code> Spark distribution, this is set to false; 
+    for <code>no-hadoop</code> distribution, this is set to true.
+  </td>
   <td>
     Whether to populate Hadoop classpath from <code>yarn.application.classpath</code> and
     <code>mapreduce.application.classpath</code> Note that if this is set to <code>false</code>, 
@@ -536,12 +553,12 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>2.0.0</td>
 </tr>
 <tr>
-  <td><code>spark.yarn.blacklist.executor.launch.blacklisting.enabled</code></td>
+  <td><code>spark.yarn.executor.launch.excludeOnFailure.enabled</code></td>
   <td>false</td>
   <td>
-  Flag to enable blacklisting of nodes having YARN resource allocation problems.
-  The error limit for blacklisting can be configured by
-  <code>spark.blacklist.application.maxFailedExecutorsPerNode</code>.
+  Flag to enable exclusion of nodes having YARN resource allocation problems.
+  The error limit for excluding can be configured by
+  <code>spark.excludeOnFailure.application.maxFailedExecutorsPerNode</code>.
   </td>
   <td>2.4.0</td>
 </tr>
@@ -569,48 +586,48 @@ To use a custom metrics.properties for the application master and executors, upd
 <table class="table">
     <tr><th>Pattern</th><th>Meaning</th></tr>
     <tr>
-      <td>{{HTTP_SCHEME}}</td>
-      <td>`http://` or `https://` according to YARN HTTP policy. (Configured via `yarn.http.policy`)</td>
+      <td>&#123;&#123;HTTP_SCHEME&#125;&#125;</td>
+      <td><code>http://</code> or <code>https://</code> according to YARN HTTP policy. (Configured via <code>yarn.http.policy</code>)</td>
     </tr>
     <tr>
-      <td>{{NM_HOST}}</td>
+      <td>&#123;&#123;NM_HOST&#125;&#125;</td>
       <td>The "host" of node where container was run.</td>
     </tr>
     <tr>
-      <td>{{NM_PORT}}</td>
+      <td>&#123;&#123;NM_PORT&#125;&#125;</td>
       <td>The "port" of node manager where container was run.</td>
     </tr>
     <tr>
-      <td>{{NM_HTTP_PORT}}</td>
+      <td>&#123;&#123;NM_HTTP_PORT&#125;&#125;</td>
       <td>The "port" of node manager's http server where container was run.</td>
     </tr>
     <tr>
-      <td>{{NM_HTTP_ADDRESS}}</td>
+      <td>&#123;&#123;NM_HTTP_ADDRESS&#125;&#125;</td>
       <td>Http URI of the node on which the container is allocated.</td>
     </tr>
     <tr>
-      <td>{{CLUSTER_ID}}</td>
-      <td>The cluster ID of Resource Manager. (Configured via `yarn.resourcemanager.cluster-id`)</td>
+      <td>&#123;&#123;CLUSTER_ID&#125;&#125;</td>
+      <td>The cluster ID of Resource Manager. (Configured via <code>yarn.resourcemanager.cluster-id</code>)</td>
     </tr>
     <tr>
-      <td>{{CONTAINER_ID}}</td>
+      <td>&#123;&#123;CONTAINER_ID&#125;&#125;</td>
       <td>The ID of container.</td>
     </tr>
     <tr>
-      <td>{{USER}}</td>
-      <td>'SPARK_USER' on system environment.</td>
+      <td>&#123;&#123;USER&#125;&#125;</td>
+      <td><code>SPARK_USER</code> on system environment.</td>
     </tr>
     <tr>
-      <td>{{FILE_NAME}}</td>
-      <td>`stdout`, `stderr`.</td>
+      <td>&#123;&#123;FILE_NAME&#125;&#125;</td>
+      <td><code>stdout</code>, <code>stderr</code>.</td>
     </tr>
 </table>
 
 For example, suppose you would like to point log url link to Job History Server directly instead of let NodeManager http server redirects it, you can configure `spark.history.custom.executor.log.url` as below:
 
- `{{HTTP_SCHEME}}<JHS_HOST>:<JHS_PORT>/jobhistory/logs/{{NM_HOST}}:{{NM_PORT}}/{{CONTAINER_ID}}/{{CONTAINER_ID}}/{{USER}}/{{FILE_NAME}}?start=-4096`
+<code>&#123;&#123;HTTP_SCHEME&#125;&#125;&lt;JHS_HOST&gt;:&lt;JHS_PORT&gt;/jobhistory/logs/&#123;&#123;NM_HOST&#125;&#125;:&#123;&#123;NM_PORT&#125;&#125;/&#123;&#123;CONTAINER_ID&#125;&#125;/&#123;&#123;CONTAINER_ID&#125;&#125;/&#123;&#123;USER&#125;&#125;/&#123;&#123;FILE_NAME&#125;&#125;?start=-4096</code>
 
- NOTE: you need to replace `<JHS_POST>` and `<JHS_PORT>` with actual value.
+NOTE: you need to replace `<JHS_POST>` and `<JHS_PORT>` with actual value.
 
 # Resource Allocation and Configuration Overview
 
@@ -626,6 +643,11 @@ If the user has a user defined YARN resource, lets call it `acceleratorX` then t
 
 YARN does not tell Spark the addresses of the resources allocated to each container. For that reason, the user must specify a discovery script that gets run by the executor on startup to discover what resources are available to that executor. You can find an example scripts in `examples/src/main/scripts/getGpusResources.sh`. The script must have execute permissions set and the user should setup permissions to not allow malicious users to modify it. The script should write to STDOUT a JSON string in the format of the ResourceInformation class. This has the resource name and an array of resource addresses available to just that executor.
 
+# Stage Level Scheduling Overview
+
+Stage level scheduling is supported on YARN when dynamic allocation is enabled. One thing to note that is YARN specific is that each ResourceProfile requires a different container priority on YARN. The mapping is simply the ResourceProfile id becomes the priority, on YARN lower numbers are higher priority. This means that profiles created earlier will have a higher priority in YARN. Normally this won't matter as Spark finishes one stage before starting another one, the only case this might have an affect is in a job server type scenario, so its something to keep in mind.
+Note there is a difference in the way custom resources are handled between the base default profile and custom ResourceProfiles. To allow for the user to request YARN containers with extra resources without Spark scheduling on them, the user can specify resources via the <code>spark.yarn.executor.resource.</code> config. Those configs are only used in the base default profile though and do not get propagated into any other custom ResourceProfiles. This is because there would be no way to remove them if you wanted a stage to not have them. This results in your default profile getting custom resources defined in <code>spark.yarn.executor.resource.</code> plus spark defined resources of GPU or FPGA. Spark converts GPU and FPGA resources into the YARN built in types <code>yarn.io/gpu</code>) and <code>yarn.io/fpga</code>, but does not know the mapping of any other resources. Any other Spark custom resources are not propagated to YARN for the default profile. So if you want Spark to schedule based off a custom resource and have it requested from YARN, you must specify it in both YARN (<code>spark.yarn.{driver/executor}.resource.</code>) and Spark (<code>spark.{driver/executor}.resource.</code>) configs. Leave the Spark config off if you only want YARN containers with the extra resources but Spark not to schedule using them. Now for custom ResourceProfiles, it doesn't currently have a way to only specify YARN resources without Spark scheduling off of them. This means for custom ResourceProfiles we propagate all the resources defined in the ResourceProfile to YARN. We still convert GPU and FPGA to the YARN build in types as well. This requires that the name of any custom resources you specify match what they are defined as in YARN.
+
 # Important notes
 
 - Whether core requests are honored in scheduling decisions depends on which scheduler is in use and how it is configured.
@@ -679,6 +701,18 @@ staging directory of the Spark application.
   </td>
   <td>2.3.0</td>
 </tr>
+<tr>
+  <td><code>spark.yarn.kerberos.renewal.excludeHadoopFileSystems</code></td>
+  <td>(none)</td>
+  <td>
+    A comma-separated list of Hadoop filesystems for whose hosts will be excluded from from delegation
+    token renewal at resource scheduler. For example, <code>spark.yarn.kerberos.renewal.excludeHadoopFileSystems=hdfs://nn1.com:8032,
+    hdfs://nn2.com:8032</code>. This is known to work under YARN for now, so YARN Resource Manager won't renew tokens for the application.
+    Note that as resource scheduler does not renew token, so any application running longer than the original token expiration that tries
+    to use that token will likely fail.
+  </td>
+  <td>3.2.0</td>
+</tr>
 </table>
 
 ## Troubleshooting Kerberos
@@ -741,8 +775,28 @@ The following extra configuration options are available when the shuffle service
     NodeManagers where the Spark Shuffle Service is not running.
   </td>
 </tr>
+<tr>
+  <td><code>spark.yarn.shuffle.service.metrics.namespace</code></td>
+  <td><code>sparkShuffleService</code></td>
+  <td>
+    The namespace to use when emitting shuffle service metrics into Hadoop metrics2 system of the
+    NodeManager.
+  </td>
+</tr>
 </table>
 
+Please note that the instructions above assume that the default shuffle service name,
+`spark_shuffle`, has been used. It is possible to use any name here, but the values used in the
+YARN NodeManager configurations must match the value of `spark.shuffle.service.name` in the
+Spark application.
+
+The shuffle service will, by default, take all of its configurations from the Hadoop Configuration
+used by the NodeManager (e.g. `yarn-site.xml`). However, it is also possible to configure the
+shuffle service independently using a file named `spark-shuffle-site.xml` which should be placed
+onto the classpath of the shuffle service (which is, by default, shared with the classpath of the
+NodeManager). The shuffle service will treat this as a standard Hadoop Configuration resource and
+overlay it on top of the NodeManager's configuration.
+
 # Launching your application with Apache Oozie
 
 Apache Oozie can launch Spark applications as part of a workflow.
@@ -791,3 +845,54 @@ do the following:
   to the list of filters in the <code>spark.ui.filters</code> configuration.
 
 Be aware that the history server information may not be up-to-date with the application's state.
+
+# Running multiple versions of the Spark Shuffle Service
+
+Please note that this section only applies when running on YARN versions >= 2.9.0.
+
+In some cases it may be desirable to run multiple instances of the Spark Shuffle Service which are
+using different versions of Spark. This can be helpful, for example, when running a YARN cluster
+with a mixed workload of applications running multiple Spark versions, since a given version of
+the shuffle service is not always compatible with other versions of Spark. YARN versions since 2.9.0
+support the ability to run shuffle services within an isolated classloader
+(see [YARN-4577](https://issues.apache.org/jira/browse/YARN-4577)), meaning multiple Spark versions
+can coexist within a single NodeManager. The
+`yarn.nodemanager.aux-services.<service-name>.classpath` and, starting from YARN 2.10.2/3.1.1/3.2.0,
+`yarn.nodemanager.aux-services.<service-name>.remote-classpath` options can be used to configure
+this. In addition to setting up separate classpaths, it's necessary to ensure the two versions
+advertise to different ports. This can be achieved using the `spark-shuffle-site.xml` file described
+above. For example, you may have configuration like:
+
+```properties
+  yarn.nodemanager.aux-services = spark_shuffle_x,spark_shuffle_y
+  yarn.nodemanager.aux-services.spark_shuffle_x.classpath = /path/to/spark-x-yarn-shuffle.jar,/path/to/spark-x-config
+  yarn.nodemanager.aux-services.spark_shuffle_y.classpath = /path/to/spark-y-yarn-shuffle.jar,/path/to/spark-y-config
+```
+
+The two `spark-*-config` directories each contain one file, `spark-shuffle-site.xml`. These are XML
+files in the [Hadoop Configuration format](https://hadoop.apache.org/docs/r3.2.2/api/org/apache/hadoop/conf/Configuration.html)
+which each contain a few configurations to adjust the port number and metrics name prefix used:
+```xml
+<configuration>
+  <property>
+    <name>spark.shuffle.service.port</name>
+    <value>7001</value>
+  </property>
+  <property>
+    <name>spark.yarn.shuffle.service.metrics.namespace</name>
+    <value>sparkShuffleServiceX</value>
+  </property>
+</configuration>
+```
+The values should both be different for the two different services.
+
+Then, in the configuration of the Spark applications, one should be configured with:
+```properties
+  spark.shuffle.service.name = spark_shuffle_x
+  spark.shuffle.service.port = 7001
+```
+and one should be configured with:
+```properties
+  spark.shuffle.service.name = spark_shuffle_y
+  spark.shuffle.service.port = <other value>
+```
diff --git a/docs/security.md b/docs/security.md
index aef6e69382a80..a75ca82e3233b 100644
--- a/docs/security.md
+++ b/docs/security.md
@@ -23,7 +23,10 @@ license: |
 
 # Spark Security: Things You Need To Know
 
-Security in Spark is OFF by default. This could mean you are vulnerable to attack by default.
+Security features like authentication are not enabled by default. When deploying a cluster that is open to the internet
+or an untrusted network, it's important to secure access to the cluster to prevent unauthorized applications
+from running on the cluster.
+
 Spark supports multiple deployments types and each one supports different levels of security. Not
 all deployment types will be secure in all environments and none are secure by default. Be
 sure to evaluate your environment, what Spark supports, and take the appropriate measure to secure
@@ -840,6 +843,9 @@ The following options provides finer-grained control for this feature:
 </tr>
 </table>
 
+Users can exclude Kerberos delegation token renewal at resource scheduler. Currently it is only supported
+on YARN. The configuration is covered in the [Running Spark on YARN](running-on-yarn.html#yarn-specific-kerberos-configuration) page.
+
 ## Long-Running Applications
 
 Long-running applications may run into issues if their run time exceeds the maximum delegation
diff --git a/docs/spark-standalone.md b/docs/spark-standalone.md
index 1e6f8c586d546..1991d64fe48a4 100644
--- a/docs/spark-standalone.md
+++ b/docs/spark-standalone.md
@@ -25,7 +25,9 @@ In addition to running on the Mesos or YARN cluster managers, Spark also provide
 
 # Security
 
-Security in Spark is OFF by default. This could mean you are vulnerable to attack by default.
+Security features like authentication are not enabled by default. When deploying a cluster that is open to the internet
+or an untrusted network, it's important to secure access to the cluster to prevent unauthorized applications
+from running on the cluster.
 Please see [Spark Security](security.html) and the specific security sections in this doc before running Spark.
 
 # Installing Spark Standalone to a Cluster
@@ -44,7 +46,7 @@ the master's web UI, which is [http://localhost:8080](http://localhost:8080) by
 
 Similarly, you can start one or more workers and connect them to the master via:
 
-    ./sbin/start-slave.sh <master-spark-URL>
+    ./sbin/start-worker.sh <master-spark-URL>
 
 Once you have started a worker, look at the master's web UI ([http://localhost:8080](http://localhost:8080) by default).
 You should see the new node listed there, along with its number of CPUs and memory (minus one gigabyte left for the OS).
@@ -90,9 +92,9 @@ Finally, the following configuration options can be passed to the master and wor
 
 # Cluster Launch Scripts
 
-To launch a Spark standalone cluster with the launch scripts, you should create a file called conf/slaves in your Spark directory,
+To launch a Spark standalone cluster with the launch scripts, you should create a file called conf/workers in your Spark directory,
 which must contain the hostnames of all the machines where you intend to start Spark workers, one per line.
-If conf/slaves does not exist, the launch scripts defaults to a single machine (localhost), which is useful for testing.
+If conf/workers does not exist, the launch scripts defaults to a single machine (localhost), which is useful for testing.
 Note, the master machine accesses each of the worker machines via ssh. By default, ssh is run in parallel and requires password-less (using a private key) access to be setup.
 If you do not have a password-less setup, you can set the environment variable SPARK_SSH_FOREGROUND and serially provide a password for each worker.
 
@@ -100,12 +102,12 @@ If you do not have a password-less setup, you can set the environment variable S
 Once you've set up this file, you can launch or stop your cluster with the following shell scripts, based on Hadoop's deploy scripts, and available in `SPARK_HOME/sbin`:
 
 - `sbin/start-master.sh` - Starts a master instance on the machine the script is executed on.
-- `sbin/start-slaves.sh` - Starts a worker instance on each machine specified in the `conf/slaves` file.
-- `sbin/start-slave.sh` - Starts a worker instance on the machine the script is executed on.
+- `sbin/start-workers.sh` - Starts a worker instance on each machine specified in the `conf/workers` file.
+- `sbin/start-worker.sh` - Starts a worker instance on the machine the script is executed on.
 - `sbin/start-all.sh` - Starts both a master and a number of workers as described above.
 - `sbin/stop-master.sh` - Stops the master that was started via the `sbin/start-master.sh` script.
-- `sbin/stop-slave.sh` - Stops all worker instances on the machine the script is executed on.
-- `sbin/stop-slaves.sh` - Stops all worker instances on the machines specified in the `conf/slaves` file.
+- `sbin/stop-worker.sh` - Stops all worker instances on the machine the script is executed on.
+- `sbin/stop-workers.sh` - Stops all worker instances on the machines specified in the `conf/workers` file.
 - `sbin/stop-all.sh` - Stops both the master and the workers as described above.
 
 Note that these scripts must be executed on the machine you want to run the Spark master on, not your local machine.
@@ -359,7 +361,7 @@ Spark Standalone has 2 parts, the first is configuring the resources for the Wor
 
 The user must configure the Workers to have a set of resources available so that it can assign them out to Executors. The <code>spark.worker.resource.{resourceName}.amount</code> is used to control the amount of each resource the worker has allocated. The user must also specify either <code>spark.worker.resourcesFile</code> or <code>spark.worker.resource.{resourceName}.discoveryScript</code> to specify how the Worker discovers the resources its assigned. See the descriptions above for each of those to see which method works best for your setup.
 
-The second part is running an application on Spark Standalone. The only special case from the standard Spark resource configs is when you are running the Driver in client mode. For a Driver in client mode, the user can specify the resources it uses via <code>spark.driver.resourcesfile</code> or <code>spark.driver.resource.{resourceName}.discoveryScript</code>. If the Driver is running on the same host as other Drivers, please make sure the resources file or discovery script only returns resources that do not conflict with other Drivers running on the same node.
+The second part is running an application on Spark Standalone. The only special case from the standard Spark resource configs is when you are running the Driver in client mode. For a Driver in client mode, the user can specify the resources it uses via <code>spark.driver.resourcesFile</code> or <code>spark.driver.resource.{resourceName}.discoveryScript</code>. If the Driver is running on the same host as other Drivers, please make sure the resources file or discovery script only returns resources that do not conflict with other Drivers running on the same node.
 
 Note, the user does not need to specify a discovery script when submitting an application as the Worker will start each Executor with the resources it allocates to it.
 
@@ -374,6 +376,25 @@ To run an interactive Spark shell against the cluster, run the following command
 
 You can also pass an option `--total-executor-cores <numCores>` to control the number of cores that spark-shell uses on the cluster.
 
+# Client Properties
+
+Spark applications supports the following configuration properties specific to standalone mode: 
+
+<table class="table">
+  <tr><th style="width:21%">Property Name</th><th>Default Value</th><th>Meaning</th><th>Since Version</th></tr>
+  <tr>
+  <td><code>spark.standalone.submit.waitAppCompletion</code></td>
+  <td><code>false</code></td>
+  <td>
+  In standalone cluster mode, controls whether the client waits to exit until the application completes.
+  If set to <code>true</code>, the client process will stay alive polling the driver's status.
+  Otherwise, the client process will exit after submission.
+  </td>
+  <td>3.1.0</td>
+  </tr>
+</table>
+
+
 # Launching Spark Applications
 
 The [`spark-submit` script](submitting-applications.html) provides the most straightforward way to
@@ -438,7 +459,7 @@ worker during one single schedule iteration.
 
 Spark's standalone mode offers a web-based user interface to monitor the cluster. The master and each worker has its own web UI that shows cluster and job statistics. By default, you can access the web UI for the master at port 8080. The port can be changed either in the configuration file or via command-line options.
 
-In addition, detailed log output for each job is also written to the work directory of each slave node (`SPARK_HOME/work` by default). You will see two files for each job, `stdout` and `stderr`, with all output it wrote to its console.
+In addition, detailed log output for each job is also written to the work directory of each worker node (`SPARK_HOME/work` by default). You will see two files for each job, `stdout` and `stderr`, with all output it wrote to its console.
 
 
 # Running Alongside Hadoop
diff --git a/docs/sparkr.md b/docs/sparkr.md
index d86fa86c89853..002da5a56fa9e 100644
--- a/docs/sparkr.md
+++ b/docs/sparkr.md
@@ -671,22 +671,22 @@ Arrow R library is available on CRAN and it can be installed as below.
 ```bash
 Rscript -e 'install.packages("arrow", repos="https://cloud.r-project.org/")'
 ```
-Please refer [the official documentation of Apache Arrow](https://arrow.apache.org/docs/r/) for more detials.
+Please refer [the official documentation of Apache Arrow](https://arrow.apache.org/docs/r/) for more details.
 
 Note that you must ensure that Arrow R package is installed and available on all cluster nodes.
-The current supported minimum version is 0.15.1; however, this might change between the minor releases since Arrow optimization in SparkR is experimental.
+The current supported minimum version is 1.0.0; however, this might change between the minor releases since Arrow optimization in SparkR is experimental.
 
 ## Enabling for Conversion to/from R DataFrame, `dapply` and `gapply`
 
 Arrow optimization is available when converting a Spark DataFrame to an R DataFrame using the call `collect(spark_df)`,
 when creating a Spark DataFrame from an R DataFrame with `createDataFrame(r_df)`, when applying an R native function to each partition
 via `dapply(...)` and when applying an R native function to grouped data via `gapply(...)`.
-To use Arrow when executing these calls, users need to first set the Spark configuration ‘spark.sql.execution.arrow.sparkr.enabled’
-to ‘true’. This is disabled by default.
+To use Arrow when executing these, users need to set the Spark configuration ‘spark.sql.execution.arrow.sparkr.enabled’
+to ‘true’ first. This is disabled by default.
 
-In addition, optimizations enabled by ‘spark.sql.execution.arrow.sparkr.enabled’ could fallback automatically to non-Arrow optimization
-implementation if an error occurs before the actual computation within Spark during converting a Spark DataFrame to/from an R
-DataFrame.
+Whether the optimization is enabled or not, SparkR produces the same results. In addition, the conversion
+between Spark DataFrame and R DataFrame falls back automatically to non-Arrow optimization implementation
+when the optimization fails for any reasons before the actual computation.
 
 <div data-lang="r" markdown="1">
 {% highlight r %}
@@ -713,9 +713,9 @@ collect(gapply(spark_df,
 {% endhighlight %}
 </div>
 
-Using the above optimizations with Arrow will produce the same results as when Arrow is not enabled. Note that even with Arrow,
-`collect(spark_df)` results in the collection of all records in the DataFrame to the driver program and should be done on a
-small subset of the data.
+Note that even with Arrow, `collect(spark_df)` results in the collection of all records in the DataFrame to
+the driver program and should be done on a small subset of the data. In addition, the specified output schema
+in `gapply(...)` and `dapply(...)` should be matched to the R DataFrame's returned by the given function.
 
 ## Supported SQL Types
 
diff --git a/docs/sql-data-sources-avro.md b/docs/sql-data-sources-avro.md
index d926ae7703268..ab1163a8d1bfa 100644
--- a/docs/sql-data-sources-avro.md
+++ b/docs/sql-data-sources-avro.md
@@ -88,8 +88,6 @@ Kafka key-value record will be augmented with some metadata, such as the ingesti
 * If the "value" field that contains your data is in Avro, you could use `from_avro()` to extract your data, enrich it, clean it, and then push it downstream to Kafka again or write it out to a file.
 * `to_avro()` can be used to turn structs into Avro records. This method is particularly useful when you would like to re-encode multiple columns into a single one when writing data out to Kafka.
 
-Both functions are currently only available in Scala and Java.
-
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 {% highlight scala %}
@@ -109,9 +107,9 @@ val df = spark
 // 2. Filter by column `favorite_color`;
 // 3. Encode the column `name` in Avro format.
 val output = df
-  .select(from_avro('value, jsonFormatSchema) as 'user)
+  .select(from_avro($"value", jsonFormatSchema) as $"user")
   .where("user.favorite_color == \"red\"")
-  .select(to_avro($"user.name") as 'value)
+  .select(to_avro($"user.name") as $"value")
 
 val query = output
   .writeStream
@@ -183,6 +181,38 @@ query = output\
   .option("topic", "topic2")\
   .start()
 
+{% endhighlight %}
+</div>
+<div data-lang="r" markdown="1">
+{% highlight r %}
+
+# `from_avro` requires Avro schema in JSON string format.
+jsonFormatSchema <- paste0(readLines("examples/src/main/resources/user.avsc"), collapse=" ")
+
+df <- read.stream(
+  "kafka",
+  kafka.bootstrap.servers = "host1:port1,host2:port2",
+  subscribe = "topic1"
+)
+
+# 1. Decode the Avro data into a struct;
+# 2. Filter by column `favorite_color`;
+# 3. Encode the column `name` in Avro format.
+
+output <- select(
+  filter(
+    select(df, alias(from_avro("value", jsonFormatSchema), "user")),
+    column("user.favorite_color") == "red"
+  ),
+  alias(to_avro("user.name"), "value")
+)
+
+write.stream(
+  output,
+  "kafka",
+  kafka.bootstrap.servers = "host1:port1,host2:port2",
+  topic = "topic2"
+)
 {% endhighlight %}
 </div>
 </div>
@@ -253,6 +283,19 @@ Data source options of Avro can be set via:
     </td>
     <td>function <code>from_avro</code></td>
   </tr>
+  <tr>
+    <td><code>datetimeRebaseMode</code></td>
+    <td>The SQL config <code>spark.sql.avro</code> <code>.datetimeRebaseModeInRead</code> which is <code>EXCEPTION</code> by default</td>
+    <td>The <code>datetimeRebaseMode</code> option allows to specify the rebasing mode for the values of the <code>date</code>, <code>timestamp-micros</code>, <code>timestamp-millis</code> logical types from the Julian to Proleptic Gregorian calendar.<br>
+      Currently supported modes are:
+      <ul>
+        <li><code>EXCEPTION</code>: fails in reads of ancient dates/timestamps that are ambiguous between the two calendars.</li>
+        <li><code>CORRECTED</code>: loads dates/timestamps without rebasing.</li>
+        <li><code>LEGACY</code>: performs rebasing of ancient dates/timestamps from the Julian to Proleptic Gregorian calendar.</li>
+      </ul>
+    </td>
+    <td>read and function <code>from_avro</code></td>
+  </tr>
 </table>
 
 ## Configuration
@@ -265,6 +308,7 @@ Configuration of Avro can be done using the `setConf` method on SparkSession or
     <td>
       If it is set to true, the data source provider <code>com.databricks.spark.avro</code> is mapped
       to the built-in but external Avro data source module for backward compatibility.
+      <br><b>Note:</b> the SQL config has been deprecated in Spark 3.2 and might be removed in the future.
     </td>
     <td>2.4.0</td>
   </tr>
@@ -287,6 +331,31 @@ Configuration of Avro can be done using the `setConf` method on SparkSession or
     </td>
     <td>2.4.0</td>
   </tr>
+  <tr>
+    <td>spark.sql.avro.datetimeRebaseModeInRead</td>
+    <td><code>EXCEPTION</code></td>
+    <td>The rebasing mode for the values of the <code>date</code>, <code>timestamp-micros</code>, <code>timestamp-millis</code> logical types from the Julian to Proleptic Gregorian calendar:<br>
+      <ul>
+        <li><code>EXCEPTION</code>: Spark will fail the reading if it sees ancient dates/timestamps that are ambiguous between the two calendars.</li>
+        <li><code>CORRECTED</code>: Spark will not do rebase and read the dates/timestamps as it is.</li>
+        <li><code>LEGACY</code>: Spark will rebase dates/timestamps from the legacy hybrid (Julian + Gregorian) calendar to Proleptic Gregorian calendar when reading Avro files.</li>
+      </ul>
+      This config is only effective if the writer info (like Spark, Hive) of the Avro files is unknown.
+    </td>
+    <td>3.0.0</td>
+  </tr>
+  <tr>
+    <td>spark.sql.avro.datetimeRebaseModeInWrite</td>
+    <td><code>EXCEPTION</code></td>
+    <td>The rebasing mode for the values of the <code>date</code>, <code>timestamp-micros</code>, <code>timestamp-millis</code> logical types from the Proleptic Gregorian to Julian calendar:<br>
+      <ul>
+        <li><code>EXCEPTION</code>: Spark will fail the writing if it sees ancient dates/timestamps that are ambiguous between the two calendars.</li>
+        <li><code>CORRECTED</code>: Spark will not do rebase and write the dates/timestamps as it is.</li>
+        <li><code>LEGACY</code>: Spark will rebase dates/timestamps from Proleptic Gregorian calendar to the legacy hybrid (Julian + Gregorian) calendar when writing Avro files.</li>
+      </ul>
+    </td>
+    <td>3.0.0</td>
+  </tr>
 </table>
 
 ## Compatibility with Databricks spark-avro
@@ -309,7 +378,7 @@ applications. Read the [Advanced Dependency Management](https://spark.apache
 Submission Guide for more details. 
 
 ## Supported types for Avro -> Spark SQL conversion
-Currently Spark supports reading all [primitive types](https://avro.apache.org/docs/1.8.2/spec.html#schema_primitive) and [complex types](https://avro.apache.org/docs/1.8.2/spec.html#schema_complex) under records of Avro.
+Currently Spark supports reading all [primitive types](https://avro.apache.org/docs/1.10.2/spec.html#schema_primitive) and [complex types](https://avro.apache.org/docs/1.10.2/spec.html#schema_complex) under records of Avro.
 <table class="table">
   <tr><th><b>Avro type</b></th><th><b>Spark SQL type</b></th></tr>
   <tr>
@@ -373,7 +442,7 @@ In addition to the types listed above, it supports reading `union` types. The fo
 3. `union(something, null)`, where something is any supported Avro type. This will be mapped to the same Spark SQL type as that of something, with nullable set to true.
 All other union types are considered complex. They will be mapped to StructType where field names are member0, member1, etc., in accordance with members of the union. This is consistent with the behavior when converting between Avro and Parquet.
 
-It also supports reading the following Avro [logical types](https://avro.apache.org/docs/1.8.2/spec.html#Logical+Types):
+It also supports reading the following Avro [logical types](https://avro.apache.org/docs/1.10.2/spec.html#Logical+Types):
 
 <table class="table">
   <tr><th><b>Avro logical type</b></th><th><b>Avro type</b></th><th><b>Spark SQL type</b></th></tr>
diff --git a/docs/sql-data-sources-csv.md b/docs/sql-data-sources-csv.md
new file mode 100644
index 0000000000000..d5390e535eff1
--- /dev/null
+++ b/docs/sql-data-sources-csv.md
@@ -0,0 +1,40 @@
+---
+layout: global
+title: CSV Files
+displayTitle: CSV Files
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+Spark SQL provides `spark.read().csv("file_name")` to read a file or directory of files in CSV format into Spark DataFrame, and `dataframe.write().csv("path")` to write to a CSV file. Function `option()` can be used to customize the behavior of reading or writing, such as controlling behavior of the header, delimiter character, character set, and so on. 
+
+<!--TODO: add `option()` document reference--> 
+
+<div class="codetabs">
+
+<div data-lang="scala"  markdown="1">
+{% include_example csv_dataset scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
+</div>
+
+<div data-lang="java"  markdown="1">
+{% include_example csv_dataset java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
+</div>
+
+<div data-lang="python"  markdown="1">
+{% include_example csv_dataset python/sql/datasource.py %}
+</div>
+
+</div>
diff --git a/docs/sql-data-sources-generic-options.md b/docs/sql-data-sources-generic-options.md
index 6bcf48235bced..2e4fc879a435f 100644
--- a/docs/sql-data-sources-generic-options.md
+++ b/docs/sql-data-sources-generic-options.md
@@ -119,3 +119,40 @@ To load all files recursively, you can use:
 {% include_example recursive_file_lookup r/RSparkSQLExample.R %}
 </div>
 </div>
+
+### Modification Time Path Filters
+
+`modifiedBefore` and `modifiedAfter` are options that can be 
+applied together or separately in order to achieve greater
+granularity over which files may load during a Spark batch query.
+(Note that Structured Streaming file sources don't support these options.)
+
+* `modifiedBefore`: an optional timestamp to only include files with
+modification times occurring before the specified time. The provided timestamp
+must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)
+* `modifiedAfter`: an optional timestamp to only include files with
+modification times occurring after the specified time. The provided timestamp
+must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)
+
+When a timezone option is not provided, the timestamps will be interpreted according
+to the Spark session timezone (`spark.sql.session.timeZone`).
+
+To load files with paths matching a given modified time range, you can use:
+
+<div class="codetabs">
+<div data-lang="scala"  markdown="1">
+{% include_example load_with_modified_time_filter scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
+</div>
+
+<div data-lang="java"  markdown="1">
+{% include_example load_with_modified_time_filter  java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
+</div>
+
+<div data-lang="python"  markdown="1">
+{% include_example load_with_modified_time_filter  python/sql/datasource.py %}
+</div>
+
+<div data-lang="r"  markdown="1">
+{% include_example load_with_modified_time_filter  r/RSparkSQLExample.R %}
+</div>
+</div>
\ No newline at end of file
diff --git a/docs/sql-data-sources-hive-tables.md b/docs/sql-data-sources-hive-tables.md
index ae3572c474e1a..14dd750709942 100644
--- a/docs/sql-data-sources-hive-tables.md
+++ b/docs/sql-data-sources-hive-tables.md
@@ -127,10 +127,10 @@ The following options can be used to configure the version of Hive that is used
   <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
   <tr>
     <td><code>spark.sql.hive.metastore.version</code></td>
-    <td><code>2.3.7</code></td>
+    <td><code>2.3.8</code></td>
     <td>
       Version of the Hive metastore. Available
-      options are <code>0.12.0</code> through <code>2.3.7</code> and <code>3.0.0</code> through <code>3.1.2</code>.
+      options are <code>0.12.0</code> through <code>2.3.8</code> and <code>3.0.0</code> through <code>3.1.2</code>.
     </td>
     <td>1.4.0</td>
   </tr>
@@ -139,23 +139,50 @@ The following options can be used to configure the version of Hive that is used
     <td><code>builtin</code></td>
     <td>
       Location of the jars that should be used to instantiate the HiveMetastoreClient. This
-      property can be one of three options:
+      property can be one of four options:
       <ol>
         <li><code>builtin</code></li>
-        Use Hive 2.3.7, which is bundled with the Spark assembly when <code>-Phive</code> is
+        Use Hive 2.3.8, which is bundled with the Spark assembly when <code>-Phive</code> is
         enabled. When this option is chosen, <code>spark.sql.hive.metastore.version</code> must be
-        either <code>2.3.7</code> or not defined.
+        either <code>2.3.8</code> or not defined.
         <li><code>maven</code></li>
         Use Hive jars of specified version downloaded from Maven repositories. This configuration
         is not generally recommended for production deployments.
+        <li><code>path</code></li>
+        Use Hive jars configured by <code>spark.sql.hive.metastore.jars.path</code>
+        in comma separated format. Support both local or remote paths. The provided jars should be
+        the same version as <code>spark.sql.hive.metastore.version</code>.
         <li>A classpath in the standard format for the JVM. This classpath must include all of Hive
-        and its dependencies, including the correct version of Hadoop. These jars only need to be
-        present on the driver, but if you are running in yarn cluster mode then you must ensure
-        they are packaged with your application.</li>
+        and its dependencies, including the correct version of Hadoop. The provided jars should be
+        the same version as <code>spark.sql.hive.metastore.version</code>. These jars only need to be present on the
+        driver, but if you are running in yarn cluster mode then you must ensure they are packaged
+        with your application.</li>
       </ol>
     </td>
     <td>1.4.0</td>
   </tr>
+  <tr>
+    <td><code>spark.sql.hive.metastore.jars.path</code></td>
+    <td><code>(empty)</code></td>
+    <td>
+      Comma-separated paths of the jars that used to instantiate the HiveMetastoreClient.
+      This configuration is useful only when <code>spark.sql.hive.metastore.jars</code> is set as <code>path</code>. 
+      <br/>
+      The paths can be any of the following format:
+      <ol>
+        <li><code>file://path/to/jar/foo.jar</code></li>
+        <li><code>hdfs://nameservice/path/to/jar/foo.jar</code></li>
+        <li><code>/path/to/jar/</code>(path without URI scheme follow conf <code>fs.defaultFS</code>'s URI schema)</li>
+        <li><code>[http/https/ftp]://path/to/jar/foo.jar</code></li>
+      </ol>
+      Note that 1, 2, and 3 support wildcard. For example:
+      <ol>
+        <li><code>file://path/to/jar/*,file://path2/to/jar/*/*.jar</code></li>
+        <li><code>hdfs://nameservice/path/to/jar/*,hdfs://nameservice2/path/to/jar/*/*.jar</code></li>
+      </ol>
+    </td>
+    <td>3.1.0</td>
+  </tr>
   <tr>
     <td><code>spark.sql.hive.metastore.sharedPrefixes</code></td>
     <td><code>com.mysql.jdbc,<br/>org.postgresql,<br/>com.microsoft.sqlserver,<br/>oracle.jdbc</code></td>
diff --git a/docs/sql-data-sources-jdbc.md b/docs/sql-data-sources-jdbc.md
index b95be0974585e..90353ef998e05 100644
--- a/docs/sql-data-sources-jdbc.md
+++ b/docs/sql-data-sources-jdbc.md
@@ -131,7 +131,7 @@ the following case-insensitive options:
   <tr>
     <td><code>fetchsize</code></td>
     <td>
-      The JDBC fetch size, which determines how many rows to fetch per round trip. This can help performance on JDBC drivers which default to low fetch size (eg. Oracle with 10 rows). This option applies only to reading.
+      The JDBC fetch size, which determines how many rows to fetch per round trip. This can help performance on JDBC drivers which default to low fetch size (e.g. Oracle with 10 rows). This option applies only to reading.
     </td>
   </tr>
 
@@ -211,8 +211,41 @@ the following case-insensitive options:
      Specifies kerberos principal name for the JDBC client. If both <code>keytab</code> and <code>principal</code> are defined then Spark tries to do kerberos authentication.
     </td>
   </tr>
+
+  <tr>
+    <td><code>refreshKrb5Config</code></td>
+    <td>
+      This option controls whether the kerberos configuration is to be refreshed or not for the JDBC client before
+      establishing a new connection. Set to true if you want to refresh the configuration, otherwise set to false.
+      The default value is false. Note that if you set this option to true and try to establish multiple connections,
+      a race condition can occur. One possble situation would be like as follows.
+      <ol>
+        <li>refreshKrb5Config flag is set with security context 1</li>
+        <li>A JDBC connection provider is used for the corresponding DBMS</li>
+        <li>The krb5.conf is modified but the JVM not yet realized that it must be reloaded</li>
+        <li>Spark authenticates successfully for security context 1</li>
+        <li>The JVM loads security context 2 from the modified krb5.conf</li>
+        <li>Spark restores the previously saved security context 1</li>
+        <li>The modified krb5.conf content just gone</li>
+      </ol>
+    </td>
+  </tr>  
 </table>
 
+Note that kerberos authentication with keytab is not always supported by the JDBC driver.<br>
+Before using <code>keytab</code> and <code>principal</code> configuration options, please make sure the following requirements are met:
+* The included JDBC driver version supports kerberos authentication with keytab. 
+* There is a built-in connection provider which supports the used database.
+
+There is a built-in connection providers for the following databases:
+* DB2
+* MariaDB
+* MS Sql
+* Oracle
+* PostgreSQL
+
+If the requirements are not met, please consider using the <code>JdbcConnectionProvider</code> developer API to handle custom authentication.
+
 <div class="codetabs">
 
 <div data-lang="scala"  markdown="1">
diff --git a/docs/sql-data-sources-load-save-functions.md b/docs/sql-data-sources-load-save-functions.md
index 0866f37fbdc71..25df34ef5b008 100644
--- a/docs/sql-data-sources-load-save-functions.md
+++ b/docs/sql-data-sources-load-save-functions.md
@@ -105,9 +105,11 @@ To load a CSV file you can use:
 The extra options are also used during write operation.
 For example, you can control bloom filters and dictionary encodings for ORC data sources.
 The following ORC example will create bloom filter and use dictionary encoding only for `favorite_color`.
-For Parquet, there exists `parquet.enable.dictionary`, too.
+For Parquet, there exists `parquet.bloom.filter.enabled` and `parquet.enable.dictionary`, too.
 To find more detailed information about the extra ORC/Parquet options,
-visit the official Apache ORC/Parquet websites.
+visit the official Apache [ORC](https://orc.apache.org/docs/spark-config.html) / [Parquet](https://github.com/apache/parquet-mr/tree/master/parquet-hadoop) websites.
+
+ORC data source:
 
 <div class="codetabs">
 
@@ -146,6 +148,46 @@ OPTIONS (
 
 </div>
 
+Parquet data source:
+
+<div class="codetabs">
+
+<div data-lang="scala"  markdown="1">
+{% include_example manual_save_options_parquet scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
+</div>
+
+<div data-lang="java"  markdown="1">
+{% include_example manual_save_options_parquet java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
+</div>
+
+<div data-lang="python"  markdown="1">
+{% include_example manual_save_options_parquet python/sql/datasource.py %}
+</div>
+
+<div data-lang="r"  markdown="1">
+{% include_example manual_save_options_parquet r/RSparkSQLExample.R %}
+</div>
+
+<div data-lang="SQL"  markdown="1">
+
+{% highlight sql %}
+CREATE TABLE users_with_options (
+  name STRING,
+  favorite_color STRING,
+  favorite_numbers array<integer>
+) USING parquet
+OPTIONS (
+  `parquet.bloom.filter.enabled#favorite_color` true,
+  `parquet.bloom.filter.expected.ndv#favorite_color` 1000000,
+  parquet.enable.dictionary true,
+  parquet.page.write-checksum.enabled true
+)
+{% endhighlight %}
+
+</div>
+
+</div>
+
 ### Run SQL on files directly
 
 Instead of using read API to load a file into DataFrame and query it, you can also query that
diff --git a/docs/sql-data-sources-orc.md b/docs/sql-data-sources-orc.md
index 4c4b3b1eee8c2..f5c9677c343dc 100644
--- a/docs/sql-data-sources-orc.md
+++ b/docs/sql-data-sources-orc.md
@@ -19,12 +19,115 @@ license: |
   limitations under the License.
 ---
 
-Since Spark 2.3, Spark supports a vectorized ORC reader with a new ORC file format for ORC files.
-To do that, the following configurations are newly added. The vectorized reader is used for the
-native ORC tables (e.g., the ones created using the clause `USING ORC`) when `spark.sql.orc.impl`
-is set to `native` and `spark.sql.orc.enableVectorizedReader` is set to `true`. For the Hive ORC
-serde tables (e.g., the ones created using the clause `USING HIVE OPTIONS (fileFormat 'ORC')`),
-the vectorized reader is used when `spark.sql.hive.convertMetastoreOrc` is also set to `true`.
+* Table of contents
+{:toc}
+
+[Apache ORC](https://orc.apache.org) is a columnar format which has more advanced features like native zstd compression, bloom filter and columnar encryption.
+
+### ORC Implementation
+
+Spark supports two ORC implementations (`native` and `hive`) which is controlled by `spark.sql.orc.impl`.
+Two implementations share most functionalities with different design goals.
+- `native` implementation is designed to follow Spark's data source behavior like `Parquet`.
+- `hive` implementation is designed to follow Hive's behavior and uses Hive SerDe.
+
+For example, historically, `native` implementation handles `CHAR/VARCHAR` with Spark's native `String` while `hive` implementation handles it via Hive `CHAR/VARCHAR`. The query results are different. Since Spark 3.1.0, [SPARK-33480](https://issues.apache.org/jira/browse/SPARK-33480) removes this difference by supporting `CHAR/VARCHAR` from Spark-side.
+
+### Vectorized Reader
+
+`native` implementation supports a vectorized ORC reader and has been the default ORC implementaion since Spark 2.3.
+The vectorized reader is used for the native ORC tables (e.g., the ones created using the clause `USING ORC`) when `spark.sql.orc.impl` is set to `native` and `spark.sql.orc.enableVectorizedReader` is set to `true`.
+For the Hive ORC serde tables (e.g., the ones created using the clause `USING HIVE OPTIONS (fileFormat 'ORC')`),
+the vectorized reader is used when `spark.sql.hive.convertMetastoreOrc` is also set to `true`, and is turned on by default.
+
+### Schema Merging
+
+Like Protocol Buffer, Avro, and Thrift, ORC also supports schema evolution. Users can start with
+a simple schema, and gradually add more columns to the schema as needed. In this way, users may end
+up with multiple ORC files with different but mutually compatible schemas. The ORC data
+source is now able to automatically detect this case and merge schemas of all these files.
+
+Since schema merging is a relatively expensive operation, and is not a necessity in most cases, we
+turned it off by default . You may enable it by
+
+1. setting data source option `mergeSchema` to `true` when reading ORC files, or
+2. setting the global SQL option `spark.sql.orc.mergeSchema` to `true`.
+
+### Zstandard
+
+Spark supports both Hadoop 2 and 3. Since Spark 3.2, you can take advantage
+of Zstandard compression in ORC files on both Hadoop versions.
+Please see [Zstandard](https://facebook.github.io/zstd/) for the benefits.
+
+<div class="codetabs">
+<div data-lang="SQL"  markdown="1">
+
+{% highlight sql %}
+CREATE TABLE compressed (
+  key STRING,
+  value STRING
+)
+USING ORC
+OPTIONS (
+  compression 'zstd'
+)
+{% endhighlight %}
+</div>
+</div>
+
+### Bloom Filters
+
+You can control bloom filters and dictionary encodings for ORC data sources. The following ORC example will create bloom filter and use dictionary encoding only for `favorite_color`. To find more detailed information about the extra ORC options, visit the official Apache ORC websites.
+
+<div class="codetabs">
+<div data-lang="SQL"  markdown="1">
+
+{% highlight sql %}
+CREATE TABLE users_with_options (
+  name STRING,
+  favorite_color STRING,
+  favorite_numbers array<integer>
+)
+USING ORC
+OPTIONS (
+  orc.bloom.filter.columns 'favorite_color',
+  orc.dictionary.key.threshold '1.0',
+  orc.column.encoding.direct 'name'
+)
+{% endhighlight %}
+</div>
+</div>
+
+### Columnar Encryption
+
+Since Spark 3.2, columnar encryption is supported for ORC tables with Apache ORC 1.6.
+The following example is using Hadoop KMS as a key provider with the given location.
+Please visit [Apache Hadoop KMS](https://hadoop.apache.org/docs/current/hadoop-kms/index.html) for the detail.
+
+<div class="codetabs">
+<div data-lang="SQL"  markdown="1">
+{% highlight sql %}
+CREATE TABLE encrypted (
+  ssn STRING,
+  email STRING,
+  name STRING
+)
+USING ORC
+OPTIONS (
+  hadoop.security.key.provider.path "kms://http@localhost:9600/kms",
+  orc.key.provider "hadoop",
+  orc.encrypt "pii:ssn,email",
+  orc.mask "nullify:ssn;sha256:email"
+)
+{% endhighlight %}
+</div>
+</div>
+
+### Hive metastore ORC table conversion
+
+When reading from Hive metastore ORC tables and inserting to Hive metastore ORC tables, Spark SQL will try to use its own ORC support instead of Hive SerDe for better performance. For CTAS statement, only non-partitioned Hive metastore ORC tables are converted. This behavior is controlled by the `spark.sql.hive.convertMetastoreOrc` configuration, and is turned on by default.
+
+### Configuration
 
 <table class="table">
   <tr><th><b>Property Name</b></th><th><b>Default</b></th><th><b>Meaning</b></th><th><b>Since Version</b></th></tr>
@@ -48,4 +151,24 @@ the vectorized reader is used when `spark.sql.hive.convertMetastoreOrc` is also
     </td>
     <td>2.3.0</td>
   </tr>
+  <tr>
+  <td><code>spark.sql.orc.mergeSchema</code></td>
+  <td>false</td>
+  <td>
+    <p>
+      When true, the ORC data source merges schemas collected from all data files,
+      otherwise the schema is picked from a random data file.
+    </p>
+  </td>
+  <td>3.0.0</td>
+  </tr>
+  <tr>
+  <td><code>spark.sql.hive.convertMetastoreOrc</code></td>
+  <td>true</td>
+  <td>
+    When set to false, Spark SQL will use the Hive SerDe for ORC tables instead of the built in
+    support.
+  </td>
+  <td>2.0.0</td>
+  </tr>
 </table>
diff --git a/docs/sql-data-sources-parquet.md b/docs/sql-data-sources-parquet.md
index 7875b10b0f697..119eae56ebf74 100644
--- a/docs/sql-data-sources-parquet.md
+++ b/docs/sql-data-sources-parquet.md
@@ -22,7 +22,7 @@ license: |
 * Table of contents
 {:toc}
 
-[Parquet](http://parquet.io) is a columnar format that is supported by many other data processing systems.
+[Parquet](https://parquet.apache.org) is a columnar format that is supported by many other data processing systems.
 Spark SQL provides support for both reading and writing Parquet files that automatically preserves the schema
 of the original data. When reading Parquet files, all columns are automatically converted to be nullable for
 compatibility reasons.
@@ -252,6 +252,42 @@ REFRESH TABLE my_table;
 
 </div>
 
+## Data Source Option
+
+Data source options of Parquet can be set via:
+* the `.option`/`.options` methods of `DataFrameReader` or `DataFrameWriter`
+* the `.option`/`.options` methods of `DataStreamReader` or `DataStreamWriter`
+
+<table class="table">
+  <tr><th><b>Property Name</b></th><th><b>Default</b></th><th><b>Meaning</b></th><th><b>Scope</b></th></tr>
+  <tr>
+    <td><code>datetimeRebaseMode</code></td>
+    <td>The SQL config <code>spark.sql.parquet</code> <code>.datetimeRebaseModeInRead</code> which is <code>EXCEPTION</code> by default</td>
+    <td>The <code>datetimeRebaseMode</code> option allows to specify the rebasing mode for the values of the <code>DATE</code>, <code>TIMESTAMP_MILLIS</code>, <code>TIMESTAMP_MICROS</code> logical types from the Julian to Proleptic Gregorian calendar.<br>
+      Currently supported modes are:
+      <ul>
+        <li><code>EXCEPTION</code>: fails in reads of ancient dates/timestamps that are ambiguous between the two calendars.</li>
+        <li><code>CORRECTED</code>: loads dates/timestamps without rebasing.</li>
+        <li><code>LEGACY</code>: performs rebasing of ancient dates/timestamps from the Julian to Proleptic Gregorian calendar.</li>
+      </ul>
+    </td>
+    <td>read</td>
+  </tr>
+  <tr>
+    <td><code>int96RebaseMode</code></td>
+    <td>The SQL config <code>spark.sql.parquet</code> <code>.int96RebaseModeInRead</code> which is <code>EXCEPTION</code> by default</td>
+    <td>The <code>int96RebaseMode</code> option allows to specify the rebasing mode for INT96 timestamps from the Julian to Proleptic Gregorian calendar.<br>
+      Currently supported modes are:
+      <ul>
+        <li><code>EXCEPTION</code>: fails in reads of ancient INT96 timestamps that are ambiguous between the two calendars.</li>
+        <li><code>CORRECTED</code>: loads INT96 timestamps without rebasing.</li>
+        <li><code>LEGACY</code>: performs rebasing of ancient timestamps from the Julian to Proleptic Gregorian calendar.</li>
+      </ul>
+    </td>
+    <td>read</td>
+  </tr>
+</table>
+
 ### Configuration
 
 Configuration of Parquet can be done using the `setConf` method on `SparkSession` or by running
@@ -329,4 +365,54 @@ Configuration of Parquet can be done using the `setConf` method on `SparkSession
   </td>
   <td>1.6.0</td>
 </tr>
+<tr>
+<td>spark.sql.parquet.datetimeRebaseModeInRead</td>
+  <td><code>EXCEPTION</code></td>
+  <td>The rebasing mode for the values of the <code>DATE</code>, <code>TIMESTAMP_MILLIS</code>, <code>TIMESTAMP_MICROS</code> logical types from the Julian to Proleptic Gregorian calendar:<br>
+    <ul>
+      <li><code>EXCEPTION</code>: Spark will fail the reading if it sees ancient dates/timestamps that are ambiguous between the two calendars.</li>
+      <li><code>CORRECTED</code>: Spark will not do rebase and read the dates/timestamps as it is.</li>
+      <li><code>LEGACY</code>: Spark will rebase dates/timestamps from the legacy hybrid (Julian + Gregorian) calendar to Proleptic Gregorian calendar when reading Parquet files.</li>
+    </ul>
+    This config is only effective if the writer info (like Spark, Hive) of the Parquet files is unknown.
+  </td>
+  <td>3.0.0</td>
+</tr>
+<tr>
+  <td>spark.sql.parquet.datetimeRebaseModeInWrite</td>
+  <td><code>EXCEPTION</code></td>
+  <td>The rebasing mode for the values of the <code>DATE</code>, <code>TIMESTAMP_MILLIS</code>, <code>TIMESTAMP_MICROS</code> logical types from the Proleptic Gregorian to Julian calendar:<br>
+    <ul>
+      <li><code>EXCEPTION</code>: Spark will fail the writing if it sees ancient dates/timestamps that are ambiguous between the two calendars.</li>
+      <li><code>CORRECTED</code>: Spark will not do rebase and write the dates/timestamps as it is.</li>
+      <li><code>LEGACY</code>: Spark will rebase dates/timestamps from Proleptic Gregorian calendar to the legacy hybrid (Julian + Gregorian) calendar when writing Parquet files.</li>
+    </ul>
+  </td>
+  <td>3.0.0</td>
+</tr>
+<tr>
+  <td>spark.sql.parquet.int96RebaseModeInRead</td>
+  <td><code>EXCEPTION</code></td>
+  <td>The rebasing mode for the values of the <code>INT96</code> timestamp type from the Julian to Proleptic Gregorian calendar:<br>
+    <ul>
+      <li><code>EXCEPTION</code>: Spark will fail the reading if it sees ancient INT96 timestamps that are ambiguous between the two calendars.</li>
+      <li><code>CORRECTED</code>: Spark will not do rebase and read the dates/timestamps as it is.</li>
+      <li><code>LEGACY</code>: Spark will rebase INT96 timestamps from the legacy hybrid (Julian + Gregorian) calendar to Proleptic Gregorian calendar when reading Parquet files.</li>
+    </ul>
+    This config is only effective if the writer info (like Spark, Hive) of the Parquet files is unknown.
+  </td>
+  <td>3.1.0</td>
+</tr>
+<tr>
+  <td>spark.sql.parquet.int96RebaseModeInWrite</td>
+  <td><code>EXCEPTION</code></td>
+  <td>The rebasing mode for the values of the <code>INT96</code> timestamp type from the Proleptic Gregorian to Julian calendar:<br>
+    <ul>
+      <li><code>EXCEPTION</code>: Spark will fail the writing if it sees ancient timestamps that are ambiguous between the two calendars.</li>
+      <li><code>CORRECTED</code>: Spark will not do rebase and write the dates/timestamps as it is.</li>
+      <li><code>LEGACY</code>: Spark will rebase INT96 timestamps from Proleptic Gregorian calendar to the legacy hybrid (Julian + Gregorian) calendar when writing Parquet files.</li>
+    </ul>
+  </td>
+  <td>3.1.0</td>
+</tr>
 </table>
diff --git a/docs/sql-data-sources-text.md b/docs/sql-data-sources-text.md
new file mode 100644
index 0000000000000..c32395f8ebb1c
--- /dev/null
+++ b/docs/sql-data-sources-text.md
@@ -0,0 +1,40 @@
+---
+layout: global
+title: Text Files
+displayTitle: Text Files
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+Spark SQL provides `spark.read().text("file_name")` to read a file or directory of text files into a Spark DataFrame, and `dataframe.write().text("path")` to write to a text file. When reading a text file, each line becomes each row that has string "value" column by default. The line separator can be changed as shown in the example below. The `option()` function can be used to customize the behavior of reading or writing, such as controlling behavior of the line separator, compression, and so on.
+
+<!--TODO(SPARK-34491): add `option()` document reference-->
+
+<div class="codetabs">
+
+<div data-lang="scala"  markdown="1">
+{% include_example text_dataset scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
+</div>
+
+<div data-lang="java"  markdown="1">
+{% include_example text_dataset java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
+</div>
+
+<div data-lang="python"  markdown="1">
+{% include_example text_dataset python/sql/datasource.py %}
+</div>
+
+</div>
diff --git a/docs/sql-data-sources.md b/docs/sql-data-sources.md
index 9396846041709..72a9f90766675 100644
--- a/docs/sql-data-sources.md
+++ b/docs/sql-data-sources.md
@@ -46,6 +46,8 @@ goes into specific options that are available for the built-in data sources.
   * [Configuration](sql-data-sources-parquet.html#configuration)
 * [ORC Files](sql-data-sources-orc.html)
 * [JSON Files](sql-data-sources-json.html)
+* [CSV Files](sql-data-sources-csv.html)
+* [Text Files](sql-data-sources-text.html)
 * [Hive Tables](sql-data-sources-hive-tables.html)
   * [Specifying storage format for Hive tables](sql-data-sources-hive-tables.html#specifying-storage-format-for-hive-tables)
   * [Interacting with Different Versions of Hive Metastore](sql-data-sources-hive-tables.html#interacting-with-different-versions-of-hive-metastore)
diff --git a/docs/sql-getting-started.md b/docs/sql-getting-started.md
index 5a6f18263ac6b..5486e737f8e37 100644
--- a/docs/sql-getting-started.md
+++ b/docs/sql-getting-started.md
@@ -352,7 +352,7 @@ Scalar functions are functions that return a single value per row, as opposed to
 
 ## Aggregate Functions
 
-Aggregate functions are functions that return a single value on a group of rows. The [Built-in Aggregation Functions](sql-ref-functions-builtin.html#aggregate-functions) provide common aggregations such as `count()`, `countDistinct()`, `avg()`, `max()`, `min()`, etc.
+Aggregate functions are functions that return a single value on a group of rows. The [Built-in Aggregation Functions](sql-ref-functions-builtin.html#aggregate-functions) provide common aggregations such as `count()`, `count_distinct()`, `avg()`, `max()`, `min()`, etc.
 Users are not limited to the predefined aggregate functions and can create their own. For more details
 about user defined aggregate functions, please refer to the documentation of
 [User Defined Aggregate Functions](sql-ref-functions-udf-aggregate.html).
diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 96f2c5dcf9735..ff2ad0464e75b 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -22,11 +22,128 @@ license: |
 * Table of contents
 {:toc}
 
+## Upgrading from Spark SQL 3.1 to 3.2
+
+  - Since Spark 3.2, ADD FILE/JAR/ARCHIVE commands require each path to be enclosed by `"` or `'` if the path contains whitespaces.
+
+  - Since Spark 3.2, all the supported JDBC dialects use StringType for ROWID. In Spark 3.1 or earlier, Oracle dialect uses StringType and the other dialects use LongType.
+
+  - In Spark 3.2, PostgreSQL JDBC dialect uses StringType for MONEY and MONEY[] is not supported due to the JDBC driver for PostgreSQL can't handle those types properly. In Spark 3.1 or earlier, DoubleType and ArrayType of DoubleType are used respectively.
+
+  - In Spark 3.2, `spark.sql.adaptive.enabled` is enabled by default. To restore the behavior before Spark 3.2, you can set `spark.sql.adaptive.enabled` to `false`.
+
+  - In Spark 3.2, the following meta-characters are escaped in the `show()` action. In Spark 3.1 or earlier, the following metacharacters are output as it is.
+    * `\n` (new line)
+    * `\r` (carrige ret)
+    * `\t` (horizontal tab)
+    * `\f` (form feed)
+    * `\b` (backspace)
+    * `\u000B` (vertical tab)
+    * `\u0007` (bell)
+
+  - In Spark 3.2, `ALTER TABLE .. RENAME TO PARTITION` throws `PartitionAlreadyExistsException` instead of `AnalysisException` for tables from Hive external when the target partition already exists.
+
+  - In Spark 3.2, script transform default FIELD DELIMIT is `\u0001` for no serde mode, serde property `field.delim` is `\t` for Hive serde mode when user specifies serde. In Spark 3.1 or earlier, the default FIELD DELIMIT is `\t`, serde property `field.delim` is `\u0001` for Hive serde mode when user specifies serde.
+
+  - In Spark 3.2, the auto-generated `Cast` (such as those added by type coercion rules) will be stripped when generating column alias names. E.g., `sql("SELECT floor(1)").columns` will be `FLOOR(1)` instead of `FLOOR(CAST(1 AS DOUBLE))`.
+  
+  - In Spark 3.2, the output schema of `SHOW TABLES` becomes `namespace: string, tableName: string, isTemporary: boolean`. In Spark 3.1 or earlier, the `namespace` field was named `database` for the builtin catalog, and there is no `isTemporary` field for v2 catalogs. To restore the old schema with the builtin catalog, you can set `spark.sql.legacy.keepCommandOutputSchema` to `true`.
+  
+  - In Spark 3.2, the output schema of `SHOW TABLE EXTENDED` becomes `namespace: string, tableName: string, isTemporary: boolean, information: string`. In Spark 3.1 or earlier, the `namespace` field was named `database` for the builtin catalog, and no change for the v2 catalogs. To restore the old schema with the builtin catalog, you can set `spark.sql.legacy.keepCommandOutputSchema` to `true`.
+
+  - In Spark 3.2, the output schema of `SHOW TBLPROPERTIES` becomes `key: string, value: string` whether you specify the table property key or not. In Spark 3.1 and earlier, the output schema of `SHOW TBLPROPERTIES` is `value: string` when you specify the table property key. To restore the old schema with the builtin catalog, you can set `spark.sql.legacy.keepCommandOutputSchema` to `true`.
+
+  - In Spark 3.2, the output schema of `DESCRIBE NAMESPACE` becomes `info_name: string, info_value: string`. In Spark 3.1 or earlier, the `info_name` field was named `database_description_item` and the `info_value` field was named `database_description_value` for the builtin catalog. To restore the old schema with the builtin catalog, you can set `spark.sql.legacy.keepCommandOutputSchema` to `true`.
+
+  - In Spark 3.2, table refreshing clears cached data of the table as well as of all its dependents such as views while keeping the dependents cached. The following commands perform table refreshing:
+    * `ALTER TABLE .. ADD PARTITION`
+    * `ALTER TABLE .. RENAME PARTITION`
+    * `ALTER TABLE .. DROP PARTITION`
+    * `ALTER TABLE .. RECOVER PARTITIONS`
+    * `MSCK REPAIR TABLE`
+    * `LOAD DATA`
+    * `REFRESH TABLE`
+    * `TRUNCATE TABLE`
+    * and the method `spark.catalog.refreshTable`
+  In Spark 3.1 and earlier, table refreshing leaves dependents uncached.
+
+  - In Spark 3.2, the usage of `count(tblName.*)` is blocked to avoid producing ambiguous results. Because `count(*)` and `count(tblName.*)` will output differently if there is any null values. To restore the behavior before Spark 3.2, you can set `spark.sql.legacy.allowStarWithSingleTableIdentifierInCount` to `true`.
+
+  - In Spark 3.2, we support typed literals in the partition spec of INSERT and ADD/DROP/RENAME PARTITION. For example, `ADD PARTITION(dt = date'2020-01-01')` adds a partition with date value `2020-01-01`. In Spark 3.1 and earlier, the partition value will be parsed as string value `date '2020-01-01'`, which is an illegal date value, and we add a partition with null value at the end.
+      
+  - In Spark 3.2, `DataFrameNaFunctions.replace()` no longer uses exact string match for the input column names, to match the SQL syntax and support qualified column names. Input column name having a dot in the name (not nested) needs to be escaped with backtick \`. Now, it throws `AnalysisException` if the column is not found in the data frame schema. It also throws `IllegalArgumentException` if the input column name is a nested column. In Spark 3.1 and earlier, it used to ignore invalid input column name and nested column name.
+
+  - In Spark 3.2, the dates subtraction expression such as `date1 - date2` returns values of `DayTimeIntervalType`. In Spark 3.1 and earlier, the returned type is `CalendarIntervalType`. To restore the behavior before Spark 3.2, you can set `spark.sql.legacy.interval.enabled` to `true`.
+
+  - In Spark 3.2, the timestamps subtraction expression such as `timestamp '2021-03-31 23:48:00' - timestamp '2021-01-01 00:00:00'` returns values of `DayTimeIntervalType`. In Spark 3.1 and earlier, the type of the same expression is `CalendarIntervalType`. To restore the behavior before Spark 3.2, you can set `spark.sql.legacy.interval.enabled` to `true`.
+
+  - In Spark 3.2, `CREATE TABLE .. LIKE ..` command can not use reserved properties. You need their specific clauses to specify them, for example, `CREATE TABLE test1 LIKE test LOCATION 'some path'`. You can set `spark.sql.legacy.notReserveProperties` to `true` to ignore the `ParseException`, in this case, these properties will be silently removed, for example: `TBLPROPERTIES('owner'='yao')` will have no effect. In Spark version 3.1 and below, the reserved properties can be used in `CREATE TABLE .. LIKE ..` command but have no side effects, for example, `TBLPROPERTIES('location'='/tmp')` does not change the location of the table but only create a headless property just like `'a'='b'`.
+
+  - In Spark 3.2, `TRANSFORM` operator can't support alias in inputs. In Spark 3.1 and earlier, we can write script transform like `SELECT TRANSFORM(a AS c1, b AS c2) USING 'cat' FROM TBL`.
+
+  - In Spark 3.2, `TRANSFORM` operator can support `ArrayType/MapType/StructType` without Hive SerDe, in this mode, we use `StructsToJosn` to convert `ArrayType/MapType/StructType` column to `STRING` and use `JsonToStructs` to parse `STRING` to `ArrayType/MapType/StructType`. In Spark 3.1, Spark just support case `ArrayType/MapType/StructType` column as `STRING` but can't support parse `STRING` to `ArrayType/MapType/StructType` output columns.
+
+  - In Spark 3.2, the unit-to-unit interval literals like `INTERVAL '1-1' YEAR TO MONTH` are converted to ANSI interval types: `YearMonthIntervalType` or `DayTimeIntervalType`. In Spark 3.1 and earlier, such interval literals are converted to `CalendarIntervalType`. To restore the behavior before Spark 3.2, you can set `spark.sql.legacy.interval.enabled` to `true`.
+
+  - In Spark 3.2, Spark supports `DayTimeIntervalType` and `YearMonthIntervalType` as inputs and outputs of `TRANSFORM` clause in Hive `SERDE` mode, the behavior is different between Hive `SERDE` mode and `ROW FORMAT DELIMITED` mode when these two types are used as inputs. In Hive `SERDE` mode, `DayTimeIntervalType` column is converted to `HiveIntervalDayTime`, its string format is `[-]?d h:m:s.n`, but in `ROW FORMAT DELIMITED` mode the format is `INTERVAL '[-]?d h:m:s.n' DAY TO TIME`. In Hive `SERDE` mode, `YearMonthIntervalType` column is converted to `HiveIntervalYearMonth`, its string format is `[-]?y-m`, but in `ROW FORMAT DELIMITED` mode the format is `INTERVAL '[-]?y-m' YEAR TO MONTH`.
+
+  - In Spark 3.2, `hash(0) == hash(-0)` for floating point types. Previously, different values were generated.
+
 ## Upgrading from Spark SQL 3.0 to 3.1
 
-  - In Spark 3.1, grouping_id() returns long values. In Spark version 3.0 and earlier, this function returns int values. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.integerGroupingId` to `true`.
+  - In Spark 3.1, statistical aggregation function includes `std`, `stddev`, `stddev_samp`, `variance`, `var_samp`, `skewness`, `kurtosis`, `covar_samp`, `corr` will return `NULL` instead of `Double.NaN` when `DivideByZero` occurs during expression evaluation, for example, when `stddev_samp` applied on a single element set. In Spark version 3.0 and earlier, it will return `Double.NaN` in such case. To restore the behavior before Spark 3.1, you can set `spark.sql.legacy.statisticalAggregate` to `true`.
+
+  - In Spark 3.1, grouping_id() returns long values. In Spark version 3.0 and earlier, this function returns int values. To restore the behavior before Spark 3.1, you can set `spark.sql.legacy.integerGroupingId` to `true`.
+
+  - In Spark 3.1, SQL UI data adopts the `formatted` mode for the query plan explain results. To restore the behavior before Spark 3.1, you can set `spark.sql.ui.explainMode` to `extended`.
+  
+  - In Spark 3.1, `from_unixtime`, `unix_timestamp`,`to_unix_timestamp`, `to_timestamp` and `to_date` will fail if the specified datetime pattern is invalid. In Spark 3.0 or earlier, they result `NULL`.
+  
+  - In Spark 3.1, the Parquet, ORC, Avro and JSON datasources throw the exception `org.apache.spark.sql.AnalysisException: Found duplicate column(s) in the data schema` in read if they detect duplicate names in top-level columns as well in nested structures. The datasources take into account the SQL config `spark.sql.caseSensitive` while detecting column name duplicates.
+
+  - In Spark 3.1, structs and maps are wrapped by the `{}` brackets in casting them to strings. For instance, the `show()` action and the `CAST` expression use such brackets. In Spark 3.0 and earlier, the `[]` brackets are used for the same purpose. To restore the behavior before Spark 3.1, you can set `spark.sql.legacy.castComplexTypesToString.enabled` to `true`.
+
+  - In Spark 3.1, NULL elements of structures, arrays and maps are converted to "null" in casting them to strings. In Spark 3.0 or earlier, NULL elements are converted to empty strings. To restore the behavior before Spark 3.1, you can set `spark.sql.legacy.castComplexTypesToString.enabled` to `true`.
+
+  - In Spark 3.1, when `spark.sql.ansi.enabled` is false, Spark always returns null if the sum of decimal type column overflows. In Spark 3.0 or earlier, in the case, the sum of decimal type column may return null or incorrect result, or even fails at runtime (depending on the actual query plan execution).
+
+  - In Spark 3.1, `path` option cannot coexist when the following methods are called with path parameter(s): `DataFrameReader.load()`, `DataFrameWriter.save()`, `DataStreamReader.load()`, or `DataStreamWriter.start()`. In addition, `paths` option cannot coexist for `DataFrameReader.load()`. For example, `spark.read.format("csv").option("path", "/tmp").load("/tmp2")` or `spark.read.option("path", "/tmp").csv("/tmp2")` will throw `org.apache.spark.sql.AnalysisException`. In Spark version 3.0 and below, `path` option is overwritten if one path parameter is passed to above methods; `path` option is added to the overall paths if multiple path parameters are passed to `DataFrameReader.load()`. To restore the behavior before Spark 3.1, you can set `spark.sql.legacy.pathOptionBehavior.enabled` to `true`.
+
+  - In Spark 3.1, `IllegalArgumentException` is returned for the incomplete interval literals, e.g. `INTERVAL '1'`, `INTERVAL '1 DAY 2'`, which are invalid. In Spark 3.0, these literals result in `NULL`s.
+
+  - In Spark 3.1, we remove the built-in Hive 1.2. You need to migrate your custom SerDes to Hive 2.3. See [HIVE-15167](https://issues.apache.org/jira/browse/HIVE-15167) for more details.
+  
+  - In Spark 3.1, loading and saving of timestamps from/to parquet files fails if the timestamps are before 1900-01-01 00:00:00Z, and loaded (saved) as the INT96 type. In Spark 3.0, the actions don't fail but might lead to shifting of the input timestamps due to rebasing from/to Julian to/from Proleptic Gregorian calendar. To restore the behavior before Spark 3.1, you can set `spark.sql.legacy.parquet.int96RebaseModeInRead` or/and `spark.sql.legacy.parquet.int96RebaseModeInWrite` to `LEGACY`.
+  
+  - In Spark 3.1, the `schema_of_json` and `schema_of_csv` functions return the schema in the SQL format in which field names are quoted. In Spark 3.0, the function returns a catalog string without field quoting and in lower case. 
+
+  - In Spark 3.1, refreshing a table will trigger an uncache operation for all other caches that reference the table, even if the table itself is not cached. In Spark 3.0 the operation will only be triggered if the table itself is cached.
+  
+  - In Spark 3.1, creating or altering a permanent view will capture runtime SQL configs and store them as view properties. These configs will be applied during the parsing and analysis phases of the view resolution. To restore the behavior before Spark 3.1, you can set `spark.sql.legacy.useCurrentConfigsForView` to `true`.
+
+  - In Spark 3.1, the temporary view will have same behaviors with the permanent view, i.e. capture and store runtime SQL configs, SQL text, catalog and namespace. The capatured view properties will be applied during the parsing and analysis phases of the view resolution. To restore the behavior before Spark 3.1, you can set `spark.sql.legacy.storeAnalyzedPlanForView` to `true`.
 
-  - In Spark 3.1, SQL UI data adopts the `formatted` mode for the query plan explain results. To restore the behavior before Spark 3.0, you can set `spark.sql.ui.explainMode` to `extended`.
+  - In Spark 3.1, temporary view created via `CACHE TABLE ... AS SELECT` will also have the same behavior with permanent view. In particular, when the temporary view is dropped, Spark will invalidate all its cache dependents, as well as the cache for the temporary view itself. This is different from Spark 3.0 and below, which only does the latter. To restore the previous behavior, you can set `spark.sql.legacy.storeAnalyzedPlanForView` to `true`.
+
+  - Since Spark 3.1, CHAR/CHARACTER and VARCHAR types are supported in the table schema. Table scan/insertion will respect the char/varchar semantic. If char/varchar is used in places other than table schema, an exception will be thrown (CAST is an exception that simply treats char/varchar as string like before). To restore the behavior before Spark 3.1, which treats them as STRING types and ignores a length parameter, e.g. `CHAR(4)`, you can set `spark.sql.legacy.charVarcharAsString` to `true`.
+
+  - In Spark 3.1, `AnalysisException` is replaced by its sub-classes that are thrown for tables from Hive external catalog in the following situations:
+    * `ALTER TABLE .. ADD PARTITION` throws `PartitionsAlreadyExistException` if new partition exists already
+    * `ALTER TABLE .. DROP PARTITION` throws `NoSuchPartitionsException` for not existing partitions
+
+## Upgrading from Spark SQL 3.0.1 to 3.0.2
+
+  - In Spark 3.0.2, `AnalysisException` is replaced by its sub-classes that are thrown for tables from Hive external catalog in the following situations:
+    * `ALTER TABLE .. ADD PARTITION` throws `PartitionsAlreadyExistException` if new partition exists already
+    * `ALTER TABLE .. DROP PARTITION` throws `NoSuchPartitionsException` for not existing partitions
+
+  - In Spark 3.0.2, `PARTITION(col=null)` is always parsed as a null literal in the partition spec. In Spark 3.0.1 or earlier, it is parsed as a string literal of its text representation, e.g., string "null", if the partition column is string type. To restore the legacy behavior, you can set `spark.sql.legacy.parseNullPartitionSpecAsStringLiteral` as true.
+
+  - In Spark 3.0.0, the output schema of `SHOW DATABASES` becomes `namespace: string`. In Spark version 2.4 and earlier, the schema was `databaseName: string`. Since Spark 3.0.2, you can restore the old schema by setting `spark.sql.legacy.keepCommandOutputSchema` to `true`.
+
+## Upgrading from Spark SQL 3.0 to 3.0.1
+
+- In Spark 3.0, JSON datasource and JSON function `schema_of_json` infer TimestampType from string values if they match to the pattern defined by the JSON option `timestampFormat`. Since version 3.0.1, the timestamp type inference is disabled by default. Set the JSON option `inferTimestamp` to `true` to enable such type inference.
 
 ## Upgrading from Spark SQL 2.4 to 3.0
 
@@ -61,7 +178,7 @@ license: |
 
   - In Spark 3.0, `SHOW TBLPROPERTIES` throws `AnalysisException` if the table does not exist. In Spark version 2.4 and below, this scenario caused `NoSuchTableException`.
 
-  - In Spark 3.0, `SHOW CREATE TABLE` always returns Spark DDL, even when the given table is a Hive SerDe table. For generating Hive DDL, use `SHOW CREATE TABLE AS SERDE` command instead.
+  - In Spark 3.0, `SHOW CREATE TABLE table_identifier` always returns Spark DDL, even when the given table is a Hive SerDe table. For generating Hive DDL, use `SHOW CREATE TABLE table_identifier AS SERDE` command instead.
 
   - In Spark 3.0, column of CHAR type is not allowed in non-Hive-Serde tables, and CREATE/ALTER TABLE commands will fail if CHAR type is detected. Please use STRING type instead. In Spark version 2.4 and below, CHAR type is treated as STRING type and the length parameter is simply ignored.
 
@@ -83,7 +200,7 @@ license: |
 
   - In Spark 3.0, using `org.apache.spark.sql.functions.udf(AnyRef, DataType)` is not allowed by default. Remove the return type parameter to automatically switch to typed Scala udf is recommended, or set `spark.sql.legacy.allowUntypedScalaUDF` to true to keep using it. In Spark version 2.4 and below, if `org.apache.spark.sql.functions.udf(AnyRef, DataType)` gets a Scala closure with primitive-type argument, the returned UDF returns null if the input values is null. However, in Spark 3.0, the UDF returns the default value of the Java type if the input value is null. For example, `val f = udf((x: Int) => x, IntegerType)`, `f($"x")` returns null in Spark 2.4 and below if column `x` is null, and return 0 in Spark 3.0. This behavior change is introduced because Spark 3.0 is built with Scala 2.12 by default.
 
-  - In Spark 3.0, a higher-order function `exists` follows the three-valued boolean logic, that is, if the `predicate` returns any `null`s and no `true` is obtained, then `exists` returns `null` instead of `false`. For example, `exists(array(1, null, 3), x -> x % 2 == 0)` is `null`. The previous behaviorcan be restored by setting `spark.sql.legacy.followThreeValuedLogicInArrayExists` to `false`.
+  - In Spark 3.0, a higher-order function `exists` follows the three-valued boolean logic, that is, if the `predicate` returns any `null`s and no `true` is obtained, then `exists` returns `null` instead of `false`. For example, `exists(array(1, null, 3), x -> x % 2 == 0)` is `null`. The previous behavior can be restored by setting `spark.sql.legacy.followThreeValuedLogicInArrayExists` to `false`.
 
   - In Spark 3.0, the `add_months` function does not adjust the resulting date to a last day of month if the original date is a last day of months. For example, `select add_months(DATE'2019-02-28', 1)` results `2019-03-28`. In Spark version 2.4 and below, the resulting date is adjusted when the original date is a last day of months. For example, adding a month to `2019-02-28` results in `2019-03-31`.
 
@@ -93,7 +210,7 @@ license: |
 
   - The result of `java.lang.Math`'s `log`, `log1p`, `exp`, `expm1`, and `pow` may vary across platforms. In Spark 3.0, the result of the equivalent SQL functions (including related SQL functions like `LOG10`) return values consistent with `java.lang.StrictMath`. In virtually all cases this makes no difference in the return value, and the difference is very small, but may not exactly match `java.lang.Math` on x86 platforms in cases like, for example, `log(3.0)`, whose value varies between `Math.log()` and `StrictMath.log()`.
 
-  - In Spark 3.0, the `Cast` function processes string literals such as 'Infinity', '+Infinity', '-Infinity', 'NaN', 'Inf', '+Inf', '-Inf' in a case-insensitive manner when casting the literals to `Double` or `Float` type to ensure greater compatibility with other database systems. This behavior change is illustrated in the table below:
+  - In Spark 3.0, the `cast` function processes string literals such as 'Infinity', '+Infinity', '-Infinity', 'NaN', 'Inf', '+Inf', '-Inf' in a case-insensitive manner when casting the literals to `Double` or `Float` type to ensure greater compatibility with other database systems. This behavior change is illustrated in the table below:
 
     | Operation | Result before Spark 3.0 | Result in Spark 3.0 |
     | --------- | ----------------------- | ------------------- |
@@ -142,7 +259,8 @@ license: |
 
   - In Spark 3.0, Proleptic Gregorian calendar is used in parsing, formatting, and converting dates and timestamps as well as in extracting sub-components like years, days and so on. Spark 3.0 uses Java 8 API classes from the `java.time` packages that are based on [ISO chronology](https://docs.oracle.com/javase/8/docs/api/java/time/chrono/IsoChronology.html). In Spark version 2.4 and below, those operations are performed using the hybrid calendar ([Julian + Gregorian](https://docs.oracle.com/javase/7/docs/api/java/util/GregorianCalendar.html). The changes impact on the results for dates before October 15, 1582 (Gregorian) and affect on the following Spark 3.0 API:
 
-    * Parsing/formatting of timestamp/date strings. This effects on CSV/JSON datasources and on the `unix_timestamp`, `date_format`, `to_unix_timestamp`, `from_unixtime`, `to_date`, `to_timestamp` functions when patterns specified by users is used for parsing and formatting. In Spark 3.0, we define our own pattern strings in `sql-ref-datetime-pattern.md`, which is implemented via `java.time.format.DateTimeFormatter` under the hood. New implementation performs strict checking of its input. For example, the `2015-07-22 10:00:00` timestamp cannot be parse if pattern is `yyyy-MM-dd` because the parser does not consume whole input. Another example is the `31/01/2015 00:00` input cannot be parsed by the `dd/MM/yyyy hh:mm` pattern because `hh` supposes hours in the range `1-12`. In Spark version 2.4 and below, `java.text.SimpleDateFormat` is used for timestamp/date string conversions, and the supported patterns are described in [simpleDateFormat](https://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html). The old behavior can be restored by setting `spark.sql.legacy.timeParserPolicy` to `LEGACY`.
+    * Parsing/formatting of timestamp/date strings. This effects on CSV/JSON datasources and on the `unix_timestamp`, `date_format`, `to_unix_timestamp`, `from_unixtime`, `to_date`, `to_timestamp` functions when patterns specified by users is used for parsing and formatting. In Spark 3.0, we define our own pattern strings in [Datetime Patterns for Formatting and Parsing](sql-ref-datetime-pattern.html),
+     which is implemented via [DateTimeFormatter](https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html) under the hood. New implementation performs strict checking of its input. For example, the `2015-07-22 10:00:00` timestamp cannot be parse if pattern is `yyyy-MM-dd` because the parser does not consume whole input. Another example is the `31/01/2015 00:00` input cannot be parsed by the `dd/MM/yyyy hh:mm` pattern because `hh` supposes hours in the range `1-12`. In Spark version 2.4 and below, `java.text.SimpleDateFormat` is used for timestamp/date string conversions, and the supported patterns are described in [SimpleDateFormat](https://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html). The old behavior can be restored by setting `spark.sql.legacy.timeParserPolicy` to `LEGACY`.
 
     * The `weekofyear`, `weekday`, `dayofweek`, `date_trunc`, `from_utc_timestamp`, `to_utc_timestamp`, and `unix_timestamp` functions use java.time API for calculation week number of year, day number of week as well for conversion from/to TimestampType values in UTC time zone.
 
@@ -174,6 +292,8 @@ license: |
 
   - Since Spark 3.0, when using `EXTRACT` expression to extract the second field from date/timestamp values, the result will be a `DecimalType(8, 6)` value with 2 digits for second part, and 6 digits for the fractional part with microsecond precision. e.g. `extract(second from to_timestamp('2019-09-20 10:10:10.1'))` results `10.100000`.  In Spark version 2.4 and earlier, it returns an `IntegerType` value and the result for the former example is `10`.
 
+  - In Spark 3.0, datetime pattern letter `F` is **aligned day of week in month** that represents the concept of the count of days within the period of a week where the weeks are aligned to the start of the month. In Spark version 2.4 and earlier, it is **week of month** that represents the concept of the count of weeks within the month where weeks start on a fixed day-of-week, e.g. `2020-07-30` is 30 days (4 weeks and 2 days) after the first day of the month, so `date_format(date '2020-07-30', 'F')` returns 2 in Spark 3.0, but as a week count in Spark 2.x, it returns 5 because it locates in the 5th week of July 2020, where week one is 2020-07-01 to 07-04.
+
 ### Data Sources
 
   - In Spark version 2.4 and below, when reading a Hive SerDe table with Spark native data sources(parquet/orc), Spark infers the actual file schema and update the table schema in metastore. In Spark 3.0, Spark doesn't infer the schema anymore. This should not cause any problems to end users, but if it does, set `spark.sql.hive.caseSensitiveInferenceMode` to `INFER_AND_SAVE`.
@@ -214,6 +334,12 @@ license: |
 
     * The decimal string representation can be different between Hive 1.2 and Hive 2.3 when using `TRANSFORM` operator in SQL for script transformation, which depends on hive's behavior. In Hive 1.2, the string representation omits trailing zeroes. But in Hive 2.3, it is always padded to 18 digits with trailing zeroes if necessary.
 
+## Upgrading from Spark SQL 2.4.7 to 2.4.8
+
+  - In Spark 2.4.8, `AnalysisException` is replaced by its sub-classes that are thrown for tables from Hive external catalog in the following situations:
+    * `ALTER TABLE .. ADD PARTITION` throws `PartitionsAlreadyExistException` if new partition exists already
+    * `ALTER TABLE .. DROP PARTITION` throws `NoSuchPartitionsException` for not existing partitions
+    
 ## Upgrading from Spark SQL 2.4.5 to 2.4.6
 
   - In Spark 2.4.6, the `RESET` command does not reset the static SQL configuration values to the default. It only clears the runtime SQL configuration values.
@@ -300,7 +426,7 @@ license: |
         </tr>
     </table>
 
-  - Since Spark 2.4, when there is a struct field in front of the IN operator before a subquery, the inner query must contain a struct field as well. In previous versions, instead, the fields of the struct were compared to the output of the inner query. Eg. if `a` is a `struct(a string, b int)`, in Spark 2.4 `a in (select (1 as a, 'a' as b) from range(1))` is a valid query, while `a in (select 1, 'a' from range(1))` is not. In previous version it was the opposite.
+  - Since Spark 2.4, when there is a struct field in front of the IN operator before a subquery, the inner query must contain a struct field as well. In previous versions, instead, the fields of the struct were compared to the output of the inner query. For example, if `a` is a `struct(a string, b int)`, in Spark 2.4 `a in (select (1 as a, 'a' as b) from range(1))` is a valid query, while `a in (select 1, 'a' from range(1))` is not. In previous version it was the opposite.
 
   - In versions 2.2.1+ and 2.3, if `spark.sql.caseSensitive` is set to true, then the `CURRENT_DATE` and `CURRENT_TIMESTAMP` functions incorrectly became case-sensitive and would resolve to columns (unless typed in lower case). In Spark 2.4 this has been fixed and the functions are no longer case-sensitive.
 
@@ -499,11 +625,11 @@ license: |
 
  - Since Spark 2.3, by default arithmetic operations between decimals return a rounded value if an exact representation is not possible (instead of returning NULL). This is compliant with SQL ANSI 2011 specification and Hive's new behavior introduced in Hive 2.2 (HIVE-15331). This involves the following changes
 
-    - The rules to determine the result type of an arithmetic operation have been updated. In particular, if the precision / scale needed are out of the range of available values, the scale is reduced up to 6, in order to prevent the truncation of the integer part of the decimals. All the arithmetic operations are affected by the change, ie. addition (`+`), subtraction (`-`), multiplication (`*`), division (`/`), remainder (`%`) and positive module (`pmod`).
+    - The rules to determine the result type of an arithmetic operation have been updated. In particular, if the precision / scale needed are out of the range of available values, the scale is reduced up to 6, in order to prevent the truncation of the integer part of the decimals. All the arithmetic operations are affected by the change, i.e. addition (`+`), subtraction (`-`), multiplication (`*`), division (`/`), remainder (`%`) and positive modulus (`pmod`).
 
     - Literal values used in SQL operations are converted to DECIMAL with the exact precision and scale needed by them.
 
-    - The configuration `spark.sql.decimalOperations.allowPrecisionLoss` has been introduced. It defaults to `true`, which means the new behavior described here; if set to `false`, Spark uses previous rules, ie. it doesn't adjust the needed scale to represent the values and it returns NULL if an exact representation of the value is not possible.
+    - The configuration `spark.sql.decimalOperations.allowPrecisionLoss` has been introduced. It defaults to `true`, which means the new behavior described here; if set to `false`, Spark uses previous rules, i.e. it doesn't adjust the needed scale to represent the values and it returns NULL if an exact representation of the value is not possible.
 
   - Un-aliased subquery's semantic has not been well defined with confusing behaviors. Since Spark 2.3, we invalidate such confusing cases, for example: `SELECT v.i from (SELECT i FROM v)`, Spark will throw an analysis exception in this case because users should not be able to use the qualifier inside a subquery. See [SPARK-20690](https://issues.apache.org/jira/browse/SPARK-20690) and [SPARK-21335](https://issues.apache.org/jira/browse/SPARK-21335) for more details.
 
@@ -791,7 +917,7 @@ Python UDF registration is unchanged.
 Spark SQL is designed to be compatible with the Hive Metastore, SerDes and UDFs.
 Currently, Hive SerDes and UDFs are based on built-in Hive,
 and Spark SQL can be connected to different versions of Hive Metastore
-(from 0.12.0 to 2.3.7 and 3.0.0 to 3.1.2. Also see [Interacting with Different Versions of Hive Metastore](sql-data-sources-hive-tables.html#interacting-with-different-versions-of-hive-metastore)).
+(from 0.12.0 to 2.3.8 and 3.0.0 to 3.1.2. Also see [Interacting with Different Versions of Hive Metastore](sql-data-sources-hive-tables.html#interacting-with-different-versions-of-hive-metastore)).
 
 #### Deploying in Existing Hive Warehouses
 {:.no_toc}
@@ -815,7 +941,7 @@ Spark SQL supports the vast majority of Hive features, such as:
 * All Hive operators, including:
   * Relational operators (`=`, `<=>`, `==`, `<>`, `<`, `>`, `>=`, `<=`, etc)
   * Arithmetic operators (`+`, `-`, `*`, `/`, `%`, etc)
-  * Logical operators (`AND`, `&&`, `OR`, `||`, etc)
+  * Logical operators (`AND`, `OR`, etc)
   * Complex type constructors
   * Mathematical functions (`sign`, `ln`, `cos`, etc)
   * String functions (`instr`, `length`, `printf`, etc)
@@ -961,3 +1087,4 @@ Below are the scenarios in which Hive and Spark generate different results:
 * `SQRT(n)` If n < 0, Hive returns null, Spark SQL returns NaN.
 * `ACOS(n)` If n < -1 or n > 1, Hive returns null, Spark SQL returns NaN.
 * `ASIN(n)` If n < -1 or n > 1, Hive returns null, Spark SQL returns NaN.
+* `CAST(n AS TIMESTAMP)` If n is integral numbers, Hive treats n as milliseconds, Spark SQL treats n as seconds.
diff --git a/docs/sql-performance-tuning.md b/docs/sql-performance-tuning.md
index 7cd85b6a9ab4c..bdfe6cde8a728 100644
--- a/docs/sql-performance-tuning.md
+++ b/docs/sql-performance-tuning.md
@@ -85,6 +85,16 @@ that these options will be deprecated in future release as more optimizations ar
     </td>
     <td>2.0.0</td>
   </tr>
+  <tr>
+    <td><code>spark.sql.files.minPartitionNum</code></td>
+    <td>Default Parallelism</td>
+    <td>
+      The suggested (not guaranteed) minimum number of split file partitions. If not set, the default
+      value is `spark.default.parallelism`. This configuration is effective only when using file-based
+      sources such as Parquet, JSON and ORC.
+    </td>
+    <td>3.1.0</td>
+  </tr>
   <tr>
     <td><code>spark.sql.broadcastTimeout</code></td>
     <td>300</td>
@@ -114,6 +124,28 @@ that these options will be deprecated in future release as more optimizations ar
     </td>
     <td>1.1.0</td>
   </tr>
+  <tr>
+    <td><code>spark.sql.sources.parallelPartitionDiscovery.threshold</code></td>
+    <td>32</td>
+    <td>
+      Configures the threshold to enable parallel listing for job input paths. If the number of
+      input paths is larger than this threshold, Spark will list the files by using Spark distributed job.
+      Otherwise, it will fallback to sequential listing. This configuration is only effective when
+      using file-based data sources such as Parquet, ORC and JSON.
+    </td>
+    <td>1.5.0</td>
+  </tr>
+  <tr>
+    <td><code>spark.sql.sources.parallelPartitionDiscovery.parallelism</code></td>
+    <td>10000</td>
+    <td>
+      Configures the maximum listing parallelism for job input paths. In case the number of input
+      paths is larger than this value, it will be throttled down to use this value. Same as above,
+      this configuration is only effective when using file-based data sources such as Parquet, ORC
+      and JSON.
+    </td>
+    <td>2.1.1</td>
+  </tr>
 </table>
 
 ## Join Strategy Hints for SQL Queries
@@ -179,7 +211,7 @@ SELECT /*+ BROADCAST(r) */ * FROM records r JOIN src s ON r.key = s.key
 </div>
 </div>
 
-For more details please refer to the documentation of [Join Hints](sql-ref-syntax-qry-select-hints.html).
+For more details please refer to the documentation of [Join Hints](sql-ref-syntax-qry-select-hints.html#join-hints).
 
 ## Coalesce Hints for SQL Queries
 
@@ -196,6 +228,8 @@ The "REPARTITION_BY_RANGE" hint must have column names and a partition number is
     SELECT /*+ REPARTITION_BY_RANGE(c) */ * FROM t
     SELECT /*+ REPARTITION_BY_RANGE(3, c) */ * FROM t
 
+For more details please refer to the documentation of [Partitioning Hints](sql-ref-syntax-qry-select-hints.html#partitioning-hints).
+
 ## Adaptive Query Execution
 Adaptive Query Execution (AQE) is an optimization technique in Spark SQL that makes use of the runtime statistics to choose the most efficient query execution plan. AQE is disabled by default. Spark SQL can use the umbrella configuration of `spark.sql.adaptive.enabled` to control whether turn it on/off. As of Spark 3.0, there are three major features in AQE, including coalescing post-shuffle partitions, converting sort-merge join to broadcast join, and skew join optimization.
 
@@ -221,9 +255,9 @@ This feature coalesces the post shuffle partitions based on the map output stati
    </tr>
    <tr>
      <td><code>spark.sql.adaptive.coalescePartitions.initialPartitionNum</code></td>
-     <td>200</td>
+     <td>(none)</td>
      <td>
-       The initial number of shuffle partitions before coalescing. By default it equals to <code>spark.sql.shuffle.partitions</code>. This configuration only has an effect when <code>spark.sql.adaptive.enabled</code> and <code>spark.sql.adaptive.coalescePartitions.enabled</code> are both enabled.
+       The initial number of shuffle partitions before coalescing. If not set, it equals to <code>spark.sql.shuffle.partitions</code>. This configuration only has an effect when <code>spark.sql.adaptive.enabled</code> and <code>spark.sql.adaptive.coalescePartitions.enabled</code> are both enabled.
      </td>
      <td>3.0.0</td>
    </tr>
@@ -254,9 +288,9 @@ Data skew can severely downgrade the performance of join queries. This feature d
      </tr>
      <tr>
        <td><code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code></td>
-       <td>10</td>
+       <td>5</td>
        <td>
-         A partition is considered as skewed if its size is larger than this factor multiplying the median partition size and also larger than <code>spark.sql.adaptive.skewedPartitionThresholdInBytes</code>.
+         A partition is considered as skewed if its size is larger than this factor multiplying the median partition size and also larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes</code>.
        </td>
        <td>3.0.0</td>
      </tr>
diff --git a/docs/sql-pyspark-pandas-with-arrow.md b/docs/sql-pyspark-pandas-with-arrow.md
index a11162963b380..6895376dfb3b8 100644
--- a/docs/sql-pyspark-pandas-with-arrow.md
+++ b/docs/sql-pyspark-pandas-with-arrow.md
@@ -19,351 +19,4 @@ license: |
   limitations under the License.
 ---
 
-* Table of contents
-{:toc}
-
-## Apache Arrow in PySpark
-
-Apache Arrow is an in-memory columnar data format that is used in Spark to efficiently transfer
-data between JVM and Python processes. This currently is most beneficial to Python users that
-work with Pandas/NumPy data. Its usage is not automatic and might require some minor
-changes to configuration or code to take full advantage and ensure compatibility. This guide will
-give a high-level description of how to use Arrow in Spark and highlight any differences when
-working with Arrow-enabled data.
-
-### Ensure PyArrow Installed
-
-To use Apache Arrow in PySpark, [the recommended version of PyArrow](#recommended-pandas-and-pyarrow-versions)
-should be installed.
-If you install PySpark using pip, then PyArrow can be brought in as an extra dependency of the
-SQL module with the command `pip install pyspark[sql]`. Otherwise, you must ensure that PyArrow
-is installed and available on all cluster nodes.
-You can install using pip or conda from the conda-forge channel. See PyArrow
-[installation](https://arrow.apache.org/docs/python/install.html) for details.
-
-## Enabling for Conversion to/from Pandas
-
-Arrow is available as an optimization when converting a Spark DataFrame to a Pandas DataFrame
-using the call `toPandas()` and when creating a Spark DataFrame from a Pandas DataFrame with
-`createDataFrame(pandas_df)`. To use Arrow when executing these calls, users need to first set
-the Spark configuration `spark.sql.execution.arrow.pyspark.enabled` to `true`. This is disabled by default.
-
-In addition, optimizations enabled by `spark.sql.execution.arrow.pyspark.enabled` could fallback automatically
-to non-Arrow optimization implementation if an error occurs before the actual computation within Spark.
-This can be controlled by `spark.sql.execution.arrow.pyspark.fallback.enabled`.
-
-<div class="codetabs">
-<div data-lang="python" markdown="1">
-{% include_example dataframe_with_arrow python/sql/arrow.py %}
-</div>
-</div>
-
-Using the above optimizations with Arrow will produce the same results as when Arrow is not
-enabled. Note that even with Arrow, `toPandas()` results in the collection of all records in the
-DataFrame to the driver program and should be done on a small subset of the data. Not all Spark
-data types are currently supported and an error can be raised if a column has an unsupported type,
-see [Supported SQL Types](#supported-sql-types). If an error occurs during `createDataFrame()`,
-Spark will fall back to create the DataFrame without Arrow.
-
-## Pandas UDFs (a.k.a. Vectorized UDFs)
-
-Pandas UDFs are user defined functions that are executed by Spark using
-Arrow to transfer data and Pandas to work with the data, which allows vectorized operations. A Pandas
-UDF is defined using the `pandas_udf` as a decorator or to wrap the function, and no additional
-configuration is required. A Pandas UDF behaves as a regular PySpark function API in general.
-
-Before Spark 3.0, Pandas UDFs used to be defined with `PandasUDFType`. From Spark 3.0
-with Python 3.6+, you can also use [Python type hints](https://www.python.org/dev/peps/pep-0484).
-Using Python type hints are preferred and using `PandasUDFType` will be deprecated in
-the future release.
-
-Note that the type hint should use `pandas.Series` in all cases but there is one variant
-that `pandas.DataFrame` should be used for its input or output type hint instead when the input
-or output column is of `StructType`. The following example shows a Pandas UDF which takes long
-column, string column and struct column, and outputs a struct column. It requires the function to
-specify the type hints of `pandas.Series` and `pandas.DataFrame` as below:
-
-<p>
-<div class="codetabs">
-<div data-lang="python" markdown="1">
-{% include_example ser_to_frame_pandas_udf python/sql/arrow.py %}
-</div>
-</div>
-</p>
-
-In the following sections, it describes the combinations of the supported type hints. For simplicity,
-`pandas.DataFrame` variant is omitted.
-
-### Series to Series
-
-The type hint can be expressed as `pandas.Series`, ... -> `pandas.Series`.
-
-By using `pandas_udf` with the function having such type hints above, it creates a Pandas UDF where the given
-function takes one or more `pandas.Series` and outputs one `pandas.Series`. The output of the function should
-always be of the same length as the input. Internally, PySpark will execute a Pandas UDF by splitting
-columns into batches and calling the function for each batch as a subset of the data, then concatenating
-the results together.
-
-The following example shows how to create this Pandas UDF that computes the product of 2 columns.
-
-<div class="codetabs">
-<div data-lang="python" markdown="1">
-{% include_example ser_to_ser_pandas_udf python/sql/arrow.py %}
-</div>
-</div>
-
-For detailed usage, please see [`pyspark.sql.functions.pandas_udf`](api/python/pyspark.sql.html#pyspark.sql.functions.pandas_udf)
-
-### Iterator of Series to Iterator of Series
-
-The type hint can be expressed as `Iterator[pandas.Series]` -> `Iterator[pandas.Series]`.
-
-By using `pandas_udf` with the function having such type hints above, it creates a Pandas UDF where the given
-function takes an iterator of `pandas.Series` and outputs an iterator of `pandas.Series`. The
-length of the entire output from the function should be the same length of the entire input; therefore, it can
-prefetch the data from the input iterator as long as the lengths are the same.
-In this case, the created Pandas UDF requires one input column when the Pandas UDF is called. To use
-multiple input columns, a different type hint is required. See Iterator of Multiple Series to Iterator
-of Series.
-
-It is also useful when the UDF execution requires initializing some states although internally it works
-identically as Series to Series case. The pseudocode below illustrates the example.
-
-{% highlight python %}
-@pandas_udf("long")
-def calculate(iterator: Iterator[pd.Series]) -> Iterator[pd.Series]:
-    # Do some expensive initialization with a state
-    state = very_expensive_initialization()
-    for x in iterator:
-        # Use that state for whole iterator.
-        yield calculate_with_state(x, state)
-
-df.select(calculate("value")).show()
-{% endhighlight %}
-
-The following example shows how to create this Pandas UDF:
-
-<div class="codetabs">
-<div data-lang="python" markdown="1">
-{% include_example iter_ser_to_iter_ser_pandas_udf python/sql/arrow.py %}
-</div>
-</div>
-
-For detailed usage, please see [`pyspark.sql.functions.pandas_udf`](api/python/pyspark.sql.html#pyspark.sql.functions.pandas_udf)
-
-### Iterator of Multiple Series to Iterator of Series
-
-The type hint can be expressed as `Iterator[Tuple[pandas.Series, ...]]` -> `Iterator[pandas.Series]`.
-
-By using `pandas_udf` with the function having such type hints above, it creates a Pandas UDF where the
-given function takes an iterator of a tuple of multiple `pandas.Series` and outputs an iterator of `pandas.Series`.
-In this case, the created pandas UDF requires multiple input columns as many as the series in the tuple
-when the Pandas UDF is called. Otherwise, it has the same characteristics and restrictions as Iterator of Series
-to Iterator of Series case.
-
-The following example shows how to create this Pandas UDF:
-
-<div class="codetabs">
-<div data-lang="python" markdown="1">
-{% include_example iter_sers_to_iter_ser_pandas_udf python/sql/arrow.py %}
-</div>
-</div>
-
-For detailed usage, please see [`pyspark.sql.functions.pandas_udf`](api/python/pyspark.sql.html#pyspark.sql.functions.pandas_udf)
-
-### Series to Scalar
-
-The type hint can be expressed as `pandas.Series`, ... -> `Any`.
-
-By using `pandas_udf` with the function having such type hints above, it creates a Pandas UDF similar
-to PySpark's aggregate functions. The given function takes `pandas.Series` and returns a scalar value.
-The return type should be a primitive data type, and the returned scalar can be either a python
-primitive type, e.g., `int` or `float` or a numpy data type, e.g., `numpy.int64` or `numpy.float64`.
-`Any` should ideally be a specific scalar type accordingly.
-
-This UDF can be also used with `groupBy().agg()` and [`pyspark.sql.Window`](api/python/pyspark.sql.html#pyspark.sql.Window).
-It defines an aggregation from one or more `pandas.Series` to a scalar value, where each `pandas.Series`
-represents a column within the group or window.
-
-Note that this type of UDF does not support partial aggregation and all data for a group or window
-will be loaded into memory. Also, only unbounded window is supported with Grouped aggregate Pandas
-UDFs currently. The following example shows how to use this type of UDF to compute mean with a group-by
-and window operations:
-
-<div class="codetabs">
-<div data-lang="python" markdown="1">
-{% include_example ser_to_scalar_pandas_udf python/sql/arrow.py %}
-</div>
-</div>
-
-For detailed usage, please see [`pyspark.sql.functions.pandas_udf`](api/python/pyspark.sql.html#pyspark.sql.functions.pandas_udf)
-
-
-## Pandas Function APIs
-
-Pandas Function APIs can directly apply a Python native function against the whole `DataFrame` by
-using Pandas instances. Internally it works similarly with Pandas UDFs by using Arrow to transfer
-data and Pandas to work with the data, which allows vectorized operations. However, A Pandas Function
-API behaves as a regular API under PySpark `DataFrame` instead of `Column`, and Python type hints in Pandas
-Functions APIs are optional and do not affect how it works internally at this moment although they
-might be required in the future.
-
-From Spark 3.0, grouped map pandas UDF is now categorized as a separate Pandas Function API,
-`DataFrame.groupby().applyInPandas()`. It is still possible to use it with `PandasUDFType`
-and `DataFrame.groupby().apply()` as it was; however, it is preferred to use
-`DataFrame.groupby().applyInPandas()` directly. Using `PandasUDFType` will be deprecated
-in the future.
-
-### Grouped Map
-
-Grouped map operations with Pandas instances are supported by `DataFrame.groupby().applyInPandas()`
-which requires a Python function that takes a `pandas.DataFrame` and return another `pandas.DataFrame`.
-It maps each group to each `pandas.DataFrame` in the Python function.
-
-This API implements the "split-apply-combine" pattern which consists of three steps:
-* Split the data into groups by using `DataFrame.groupBy`.
-* Apply a function on each group. The input and output of the function are both `pandas.DataFrame`. The
-  input data contains all the rows and columns for each group.
-* Combine the results into a new PySpark `DataFrame`.
-
-To use `groupBy().applyInPandas()`, the user needs to define the following:
-* A Python function that defines the computation for each group.
-* A `StructType` object or a string that defines the schema of the output PySpark `DataFrame`.
-
-The column labels of the returned `pandas.DataFrame` must either match the field names in the
-defined output schema if specified as strings, or match the field data types by position if not
-strings, e.g. integer indices. See [pandas.DataFrame](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html#pandas.DataFrame)
-on how to label columns when constructing a `pandas.DataFrame`.
-
-Note that all data for a group will be loaded into memory before the function is applied. This can
-lead to out of memory exceptions, especially if the group sizes are skewed. The configuration for
-[maxRecordsPerBatch](#setting-arrow-batch-size) is not applied on groups and it is up to the user
-to ensure that the grouped data will fit into the available memory.
-
-The following example shows how to use `groupby().applyInPandas()` to subtract the mean from each value
-in the group.
-
-<div class="codetabs">
-<div data-lang="python" markdown="1">
-{% include_example grouped_apply_in_pandas python/sql/arrow.py %}
-</div>
-</div>
-
-For detailed usage, please see [`pyspark.sql.GroupedData.applyInPandas`](api/python/pyspark.sql.html#pyspark.sql.GroupedData.applyInPandas).
-
-### Map
-
-Map operations with Pandas instances are supported by `DataFrame.mapInPandas()` which maps an iterator
-of `pandas.DataFrame`s to another iterator of `pandas.DataFrame`s that represents the current
-PySpark `DataFrame` and returns the result as a PySpark `DataFrame`. The functions takes and outputs
-an iterator of `pandas.DataFrame`. It can return the output of arbitrary length in contrast to some
-Pandas UDFs although internally it works similarly with Series to Series Pandas UDF.
-
-The following example shows how to use `mapInPandas()`:
-
-<div class="codetabs">
-<div data-lang="python" markdown="1">
-{% include_example map_in_pandas python/sql/arrow.py %}
-</div>
-</div>
-
-For detailed usage, please see [`pyspark.sql.DataFrame.mapsInPandas`](api/python/pyspark.sql.html#pyspark.sql.DataFrame.mapInPandas).
-
-### Co-grouped Map
-
-Co-grouped map operations with Pandas instances are supported by `DataFrame.groupby().cogroup().applyInPandas()` which
-allows two PySpark `DataFrame`s to be cogrouped by a common key and then a Python function applied to each
-cogroup. It consists of the following steps:
-* Shuffle the data such that the groups of each dataframe which share a key are cogrouped together.
-* Apply a function to each cogroup. The input of the function is two `pandas.DataFrame` (with an optional tuple
-representing the key). The output of the function is a `pandas.DataFrame`.
-* Combine the `pandas.DataFrame`s from all groups into a new PySpark `DataFrame`. 
-
-To use `groupBy().cogroup().applyInPandas()`, the user needs to define the following:
-* A Python function that defines the computation for each cogroup.
-* A `StructType` object or a string that defines the schema of the output PySpark `DataFrame`.
-
-The column labels of the returned `pandas.DataFrame` must either match the field names in the
-defined output schema if specified as strings, or match the field data types by position if not
-strings, e.g. integer indices. See [pandas.DataFrame](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html#pandas.DataFrame)
-on how to label columns when constructing a `pandas.DataFrame`.
-
-Note that all data for a cogroup will be loaded into memory before the function is applied. This can lead to out of
-memory exceptions, especially if the group sizes are skewed. The configuration for [maxRecordsPerBatch](#setting-arrow-batch-size)
-is not applied and it is up to the user to ensure that the cogrouped data will fit into the available memory.
-
-The following example shows how to use `groupby().cogroup().applyInPandas()` to perform an asof join between two datasets.
-
-<div class="codetabs">
-<div data-lang="python" markdown="1">
-{% include_example cogrouped_apply_in_pandas python/sql/arrow.py %}
-</div>
-</div>
-
-For detailed usage, please see [`pyspark.sql.PandasCogroupedOps.applyInPandas()`](api/python/pyspark.sql.html#pyspark.sql.PandasCogroupedOps.applyInPandas).
-
-
-## Usage Notes
-
-### Supported SQL Types
-
-Currently, all Spark SQL data types are supported by Arrow-based conversion except `MapType`,
-`ArrayType` of `TimestampType`, and nested `StructType`.
-
-### Setting Arrow Batch Size
-
-Data partitions in Spark are converted into Arrow record batches, which can temporarily lead to
-high memory usage in the JVM. To avoid possible out of memory exceptions, the size of the Arrow
-record batches can be adjusted by setting the conf "spark.sql.execution.arrow.maxRecordsPerBatch"
-to an integer that will determine the maximum number of rows for each batch. The default value is
-10,000 records per batch. If the number of columns is large, the value should be adjusted
-accordingly. Using this limit, each data partition will be made into 1 or more record batches for
-processing.
-
-### Timestamp with Time Zone Semantics
-
-Spark internally stores timestamps as UTC values, and timestamp data that is brought in without
-a specified time zone is converted as local time to UTC with microsecond resolution. When timestamp
-data is exported or displayed in Spark, the session time zone is used to localize the timestamp
-values. The session time zone is set with the configuration 'spark.sql.session.timeZone' and will
-default to the JVM system local time zone if not set. Pandas uses a `datetime64` type with nanosecond
-resolution, `datetime64[ns]`, with optional time zone on a per-column basis.
-
-When timestamp data is transferred from Spark to Pandas it will be converted to nanoseconds
-and each column will be converted to the Spark session time zone then localized to that time
-zone, which removes the time zone and displays values as local time. This will occur
-when calling `toPandas()` or `pandas_udf` with timestamp columns.
-
-When timestamp data is transferred from Pandas to Spark, it will be converted to UTC microseconds. This
-occurs when calling `createDataFrame` with a Pandas DataFrame or when returning a timestamp from a
-`pandas_udf`. These conversions are done automatically to ensure Spark will have data in the
-expected format, so it is not necessary to do any of these conversions yourself. Any nanosecond
-values will be truncated.
-
-Note that a standard UDF (non-Pandas) will load timestamp data as Python datetime objects, which is
-different than a Pandas timestamp. It is recommended to use Pandas time series functionality when
-working with timestamps in `pandas_udf`s to get the best performance, see
-[here](https://pandas.pydata.org/pandas-docs/stable/timeseries.html) for details.
-
-### Recommended Pandas and PyArrow Versions
-
-For usage with pyspark.sql, the supported versions of Pandas is 0.24.2 and PyArrow is 0.15.1. Higher
-versions may be used, however, compatibility and data correctness can not be guaranteed and should
-be verified by the user.
-
-### Compatibility Setting for PyArrow >= 0.15.0 and Spark 2.3.x, 2.4.x
-
-Since Arrow 0.15.0, a change in the binary IPC format requires an environment variable to be
-compatible with previous versions of Arrow <= 0.14.1. This is only necessary to do for PySpark
-users with versions 2.3.x and 2.4.x that have manually upgraded PyArrow to 0.15.0. The following
-can be added to `conf/spark-env.sh` to use the legacy Arrow IPC format:
-
-```
-ARROW_PRE_0_15_IPC_FORMAT=1
-```
-
-This will instruct PyArrow >= 0.15.0 to use the legacy IPC format with the older Arrow Java that
-is in Spark 2.3.x and 2.4.x. Not setting this environment variable will lead to a similar error as
-described in [SPARK-29367](https://issues.apache.org/jira/browse/SPARK-29367) when running
-`pandas_udf`s or `toPandas()` with Arrow enabled. More information about the Arrow IPC change can
-be read on the Arrow 0.15.0 release [blog](http://arrow.apache.org/blog/2019/10/06/0.15.0-release/#columnar-streaming-protocol-change-since-0140).
+The Arrow usage guide is now archived on [this page](https://spark.apache.org/docs/latest/api/python/user_guide/arrow_pandas.html).
diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md
index b62834ebe9067..4e88ce0b6baaa 100644
--- a/docs/sql-ref-ansi-compliance.md
+++ b/docs/sql-ref-ansi-compliance.md
@@ -21,7 +21,8 @@ license: |
 
 Since Spark 3.0, Spark SQL introduces two experimental options to comply with the SQL standard: `spark.sql.ansi.enabled` and `spark.sql.storeAssignmentPolicy` (See a table below for details).
 
-When `spark.sql.ansi.enabled` is set to `true`, Spark SQL follows the standard in basic behaviours (e.g., arithmetic operations, type conversion, SQL functions and SQL parsing).
+When `spark.sql.ansi.enabled` is set to `true`, Spark SQL uses an ANSI compliant dialect instead of being Hive compliant. For example, Spark will throw an exception at runtime instead of returning null results if the inputs to a SQL operator/function are invalid. Some ANSI dialect features may be not from the ANSI SQL standard directly, but their behaviors align with ANSI SQL's style.
+
 Moreover, Spark SQL has an independent option to control implicit casting behaviours when inserting rows in a table.
 The casting behaviours are defined as store assignment rules in the standard.
 
@@ -46,6 +47,9 @@ When `spark.sql.ansi.enabled` is set to `true` and an overflow occurs in numeric
 SELECT 2147483647 + 1;
 java.lang.ArithmeticException: integer overflow
 
+SELECT abs(-2147483648);
+java.lang.ArithmeticException: integer overflow
+
 -- `spark.sql.ansi.enabled=false`
 SELECT 2147483647 + 1;
 +----------------+
@@ -53,6 +57,13 @@ SELECT 2147483647 + 1;
 +----------------+
 |     -2147483648|
 +----------------+
+
+SELECT abs(-2147483648);
++----------------+
+|abs(-2147483648)|
++----------------+
+|     -2147483648|
++----------------+
 ```
 
 ### Type Conversion
@@ -61,6 +72,37 @@ Spark SQL has three kinds of type conversions: explicit casting, type coercion,
 When `spark.sql.ansi.enabled` is set to `true`, explicit casting by `CAST` syntax throws a runtime exception for illegal cast patterns defined in the standard, e.g. casts from a string to an integer.
 On the other hand, `INSERT INTO` syntax throws an analysis exception when the ANSI mode enabled via `spark.sql.storeAssignmentPolicy=ANSI`.
 
+The type conversion of Spark ANSI mode follows the syntax rules of section 6.13 "cast specification" in [ISO/IEC 9075-2:2011 Information technology — Database languages - SQL — Part 2: Foundation (SQL/Foundation)](https://www.iso.org/standard/53682.html), except it specially allows the following
+ straightforward type conversions which are disallowed as per the ANSI standard:
+* NumericType <=> BooleanType
+* StringType <=> BinaryType
+* ArrayType => String
+* MapType => String
+* StructType => String
+
+ The valid combinations of target data type and source data type in a `CAST` expression are given by the following table.
+“Y” indicates that the combination is syntactically valid without restriction and “N” indicates that the combination is not valid.
+
+| Source\Target | Numeric | String | Date | Timestamp | Interval | Boolean | Binary | Array | Map | Struct |
+|-----------|---------|--------|------|-----------|----------|---------|--------|-------|-----|--------|
+| Numeric   | <span style="color:red">**Y**</span> | Y      | N    | N         | N        | Y       | N      | N     | N   | N      |
+| String    | <span style="color:red">**Y**</span> | Y | <span style="color:red">**Y**</span> | <span style="color:red">**Y**</span> | <span style="color:red">**Y**</span> | <span style="color:red">**Y**</span> | Y | N     | N   | N      |
+| Date      | N       | Y      | Y    | Y         | N        | N       | N      | N     | N   | N      |
+| Timestamp | N       | Y      | Y    | Y         | N        | N       | N      | N     | N   | N      |
+| Interval  | N       | Y      | N    | N         | Y        | N       | N      | N     | N   | N      |
+| Boolean   | Y       | Y      | N    | N         | N        | Y       | N      | N     | N   | N      |
+| Binary    | N       | Y      | N    | N         | N        | N       | Y      | N     | N   | N      |
+| Array     | N       | Y      | N    | N         | N        | N       | N      | <span style="color:red">**Y**</span> | N   | N      |
+| Map       | N       | Y      | N    | N         | N        | N       | N      | N     | <span style="color:red">**Y**</span> | N      |
+| Struct    | N       | Y      | N    | N         | N        | N       | N      | N     | N   | <span style="color:red">**Y**</span> |
+
+In the table above, all the `CAST`s that can cause runtime exceptions are marked as red <span style="color:red">**Y**</span>:
+* CAST(Numeric AS Numeric): raise an overflow exception if the value is out of the target data type's range.
+* CAST(String AS (Numeric/Date/Timestamp/Interval/Boolean)): raise a runtime exception if the value can't be parsed as the target data type.
+* CAST(Array AS Array): raise an exception if there is any on the conversion of the elements.
+* CAST(Map AS Map): raise an exception if there is any on the conversion of the keys and the values.
+* CAST(Struct AS Struct): raise an exception if there is any on the conversion of the struct fields.
+
 Currently, the ANSI mode affects explicit casting and assignment casting only.
 In future releases, the behaviour of type coercion might change along with the other two type conversion rules.
 
@@ -74,6 +116,10 @@ java.lang.NumberFormatException: invalid input syntax for type numeric: a
 SELECT CAST(2147483648L AS INT);
 java.lang.ArithmeticException: Casting 2147483648 to int causes overflow
 
+SELECT CAST(DATE'2020-01-01' AS INT)
+org.apache.spark.sql.AnalysisException: cannot resolve 'CAST(DATE '2020-01-01' AS INT)' due to data type mismatch: cannot cast date to int.
+To convert values from date to int, you can use function UNIX_DATE instead.
+
 -- `spark.sql.ansi.enabled=false` (This is a default behaviour)
 SELECT CAST('a' AS INT);
 +--------------+
@@ -89,13 +135,20 @@ SELECT CAST(2147483648L AS INT);
 |            -2147483648|
 +-----------------------+
 
+SELECT CAST(DATE'2020-01-01' AS INT)
++------------------------------+
+|CAST(DATE '2020-01-01' AS INT)|
++------------------------------+
+|                          null|
++------------------------------+
+
 -- Examples of store assignment rules
 CREATE TABLE t (v INT);
 
 -- `spark.sql.storeAssignmentPolicy=ANSI`
 INSERT INTO t VALUES ('1');
 org.apache.spark.sql.AnalysisException: Cannot write incompatible data to table '`default`.`t`':
-- Cannot safely cast 'v': StringType to IntegerType;
+- Cannot safely cast 'v': string to int;
 
 -- `spark.sql.storeAssignmentPolicy=LEGACY` (This is a legacy behaviour until Spark 2.x)
 INSERT INTO t VALUES ('1');
@@ -110,7 +163,27 @@ SELECT * FROM t;
 ### SQL Functions
 
 The behavior of some SQL functions can be different under ANSI mode (`spark.sql.ansi.enabled=true`).
-  - `size`: This function returns null for null input under ANSI mode.
+  - `size`: This function returns null for null input.
+  - `element_at`:
+    - This function throws `ArrayIndexOutOfBoundsException` if using invalid indices.
+    - This function throws `NoSuchElementException` if key does not exist in map.
+  - `elt`: This function throws `ArrayIndexOutOfBoundsException` if using invalid indices.
+  - `parse_url`: This function throws `IllegalArgumentException` if an input string is not a valid url.
+  - `to_date`: This function should fail with an exception if the input string can't be parsed, or the pattern string is invalid.
+  - `to_timestamp`: This function should fail with an exception if the input string can't be parsed, or the pattern string is invalid.
+  - `unix_timestamp`: This function should fail with an exception if the input string can't be parsed, or the pattern string is invalid.
+  - `to_unix_timestamp`: This function should fail with an exception if the input string can't be parsed, or the pattern string is invalid.
+  - `make_date`: This function should fail with an exception if the result date is invalid.
+  - `make_timestamp`: This function should fail with an exception if the result timestamp is invalid.
+  - `make_interval`:  This function should fail with an exception if the result interval is invalid.
+  - `next_day`: This function throws `IllegalArgumentException` if input is not a valid day of week.
+
+### SQL Operators
+
+The behavior of some SQL operators can be different under ANSI mode (`spark.sql.ansi.enabled=true`).
+  - `array_col[index]`: This operator throws `ArrayIndexOutOfBoundsException` if using invalid indices.
+  - `map_col[key]`: This operator throws `NoSuchElementException` if key does not exist in map.
+  - `GROUP BY`: aliases in a select list can not be used in GROUP BY clauses. Each column referenced in a GROUP BY clause shall unambiguously reference a column of the table resulting from the FROM clause.
 
 ### SQL Keywords
 
@@ -127,7 +200,7 @@ By default `spark.sql.ansi.enabled` is false.
 
 Below is a list of all the keywords in Spark SQL.
 
-|Keyword|Spark SQL<br/>ANSI Mode|Spark SQL<br/>Default Mode|SQL-2011|
+|Keyword|Spark SQL<br/>ANSI Mode|Spark SQL<br/>Default Mode|SQL-2016|
 |-------|----------------------|-------------------------|--------|
 |ADD|non-reserved|non-reserved|non-reserved|
 |AFTER|non-reserved|non-reserved|non-reserved|
@@ -135,7 +208,7 @@ Below is a list of all the keywords in Spark SQL.
 |ALTER|non-reserved|non-reserved|reserved|
 |ANALYZE|non-reserved|non-reserved|non-reserved|
 |AND|reserved|non-reserved|reserved|
-|ANTI|reserved|strict-non-reserved|non-reserved|
+|ANTI|non-reserved|strict-non-reserved|non-reserved|
 |ANY|reserved|non-reserved|reserved|
 |ARCHIVE|non-reserved|non-reserved|non-reserved|
 |ARRAY|non-reserved|non-reserved|reserved|
@@ -149,7 +222,7 @@ Below is a list of all the keywords in Spark SQL.
 |BUCKETS|non-reserved|non-reserved|non-reserved|
 |BY|non-reserved|non-reserved|reserved|
 |CACHE|non-reserved|non-reserved|non-reserved|
-|CASCADE|non-reserved|non-reserved|reserved|
+|CASCADE|non-reserved|non-reserved|non-reserved|
 |CASE|reserved|non-reserved|reserved|
 |CAST|reserved|non-reserved|reserved|
 |CHANGE|non-reserved|non-reserved|non-reserved|
@@ -181,7 +254,7 @@ Below is a list of all the keywords in Spark SQL.
 |DATA|non-reserved|non-reserved|non-reserved|
 |DATABASE|non-reserved|non-reserved|non-reserved|
 |DATABASES|non-reserved|non-reserved|non-reserved|
-|DAY|reserved|non-reserved|reserved|
+|DAY|non-reserved|non-reserved|non-reserved|
 |DBPROPERTIES|non-reserved|non-reserved|non-reserved|
 |DEFINED|non-reserved|non-reserved|non-reserved|
 |DELETE|non-reserved|non-reserved|reserved|
@@ -193,7 +266,7 @@ Below is a list of all the keywords in Spark SQL.
 |DIRECTORY|non-reserved|non-reserved|non-reserved|
 |DISTINCT|reserved|non-reserved|reserved|
 |DISTRIBUTE|non-reserved|non-reserved|non-reserved|
-|DIV|non-reserved|non-reserved|non-reserved|
+|DIV|non-reserved|non-reserved|not a keyword|
 |DROP|non-reserved|non-reserved|reserved|
 |ELSE|reserved|non-reserved|reserved|
 |END|reserved|non-reserved|reserved|
@@ -227,8 +300,7 @@ Below is a list of all the keywords in Spark SQL.
 |GROUP|reserved|non-reserved|reserved|
 |GROUPING|non-reserved|non-reserved|reserved|
 |HAVING|reserved|non-reserved|reserved|
-|HOUR|reserved|non-reserved|reserved|
-|IF|non-reserved|non-reserved|reserved|
+|IF|non-reserved|non-reserved|not a keyword|
 |IGNORE|non-reserved|non-reserved|non-reserved|
 |IMPORT|non-reserved|non-reserved|non-reserved|
 |IN|reserved|non-reserved|reserved|
@@ -264,9 +336,8 @@ Below is a list of all the keywords in Spark SQL.
 |MAP|non-reserved|non-reserved|non-reserved|
 |MATCHED|non-reserved|non-reserved|non-reserved|
 |MERGE|non-reserved|non-reserved|non-reserved|
-|MINUS|reserved|strict-non-reserved|non-reserved|
-|MINUTE|reserved|non-reserved|reserved|
-|MONTH|reserved|non-reserved|reserved|
+|MINUS|non-reserved|strict-non-reserved|non-reserved|
+|MONTH|non-reserved|non-reserved|non-reserved|
 |MSCK|non-reserved|non-reserved|non-reserved|
 |NAMESPACE|non-reserved|non-reserved|non-reserved|
 |NAMESPACES|non-reserved|non-reserved|non-reserved|
@@ -302,16 +373,19 @@ Below is a list of all the keywords in Spark SQL.
 |PROPERTIES|non-reserved|non-reserved|non-reserved|
 |PURGE|non-reserved|non-reserved|non-reserved|
 |QUERY|non-reserved|non-reserved|non-reserved|
+|RANGE|non-reserved|non-reserved|reserved|
 |RECORDREADER|non-reserved|non-reserved|non-reserved|
 |RECORDWRITER|non-reserved|non-reserved|non-reserved|
 |RECOVER|non-reserved|non-reserved|non-reserved|
 |REDUCE|non-reserved|non-reserved|non-reserved|
 |REFERENCES|reserved|non-reserved|reserved|
 |REFRESH|non-reserved|non-reserved|non-reserved|
+|REGEXP|non-reserved|non-reserved|not a keyword|
 |RENAME|non-reserved|non-reserved|non-reserved|
 |REPAIR|non-reserved|non-reserved|non-reserved|
 |REPLACE|non-reserved|non-reserved|non-reserved|
 |RESET|non-reserved|non-reserved|non-reserved|
+|RESPECT|non-reserved|non-reserved|non-reserved|
 |RESTRICT|non-reserved|non-reserved|non-reserved|
 |REVOKE|non-reserved|non-reserved|reserved|
 |RIGHT|reserved|strict-non-reserved|reserved|
@@ -323,9 +397,10 @@ Below is a list of all the keywords in Spark SQL.
 |ROW|non-reserved|non-reserved|reserved|
 |ROWS|non-reserved|non-reserved|reserved|
 |SCHEMA|non-reserved|non-reserved|non-reserved|
-|SECOND|reserved|non-reserved|reserved|
+|SCHEMAS|non-reserved|non-reserved|not a keyword|
+|SECOND|non-reserved|non-reserved|non-reserved|
 |SELECT|reserved|non-reserved|reserved|
-|SEMI|reserved|strict-non-reserved|non-reserved|
+|SEMI|non-reserved|strict-non-reserved|non-reserved|
 |SEPARATED|non-reserved|non-reserved|non-reserved|
 |SERDE|non-reserved|non-reserved|non-reserved|
 |SERDEPROPERTIES|non-reserved|non-reserved|non-reserved|
@@ -344,13 +419,16 @@ Below is a list of all the keywords in Spark SQL.
 |STRUCT|non-reserved|non-reserved|non-reserved|
 |SUBSTR|non-reserved|non-reserved|non-reserved|
 |SUBSTRING|non-reserved|non-reserved|non-reserved|
+|SYNC|non-reserved|non-reserved|non-reserved|
 |TABLE|reserved|non-reserved|reserved|
 |TABLES|non-reserved|non-reserved|non-reserved|
 |TABLESAMPLE|non-reserved|non-reserved|reserved|
 |TBLPROPERTIES|non-reserved|non-reserved|non-reserved|
+|TEMP|non-reserved|non-reserved|not a keyword|
 |TEMPORARY|non-reserved|non-reserved|non-reserved|
 |TERMINATED|non-reserved|non-reserved|non-reserved|
 |THEN|reserved|non-reserved|reserved|
+|TIME|reserved|non-reserved|reserved|
 |TO|reserved|non-reserved|reserved|
 |TOUCH|non-reserved|non-reserved|non-reserved|
 |TRAILING|reserved|non-reserved|reserved|
@@ -360,6 +438,8 @@ Below is a list of all the keywords in Spark SQL.
 |TRIM|non-reserved|non-reserved|non-reserved|
 |TRUE|non-reserved|non-reserved|reserved|
 |TRUNCATE|non-reserved|non-reserved|reserved|
+|TRY_CAST|non-reserved|non-reserved|non-reserved|
+|TYPE|non-reserved|non-reserved|non-reserved|
 |UNARCHIVE|non-reserved|non-reserved|non-reserved|
 |UNBOUNDED|non-reserved|non-reserved|non-reserved|
 |UNCACHE|non-reserved|non-reserved|non-reserved|
@@ -379,4 +459,5 @@ Below is a list of all the keywords in Spark SQL.
 |WHERE|reserved|non-reserved|reserved|
 |WINDOW|non-reserved|non-reserved|reserved|
 |WITH|reserved|non-reserved|reserved|
-|YEAR|reserved|non-reserved|reserved|
+|YEAR|non-reserved|non-reserved|non-reserved|
+|ZONE|non-reserved|non-reserved|non-reserved|
diff --git a/docs/sql-ref-datatypes.md b/docs/sql-ref-datatypes.md
index f27f1a0ca967f..fe1090e43d459 100644
--- a/docs/sql-ref-datatypes.md
+++ b/docs/sql-ref-datatypes.md
@@ -37,6 +37,8 @@ Spark SQL and DataFrames support the following data types:
   - `DecimalType`: Represents arbitrary-precision signed decimal numbers. Backed internally by `java.math.BigDecimal`. A `BigDecimal` consists of an arbitrary precision integer unscaled value and a 32-bit integer scale.
 * String type
   - `StringType`: Represents character string values.
+  - `VarcharType(length)`: A variant of `StringType` which has a length limitation. Data writing will fail if the input string exceeds the length limitation. Note: this type can only be used in table schema, not functions/operators.
+  - `CharType(length)`: A variant of `VarcharType(length)` which is fixed length. Reading column of type `CharType(n)` always returns string values of length `n`. Char type column comparison will pad the short one to the longer length.
 * Binary type
   - `BinaryType`: Represents byte sequence values.
 * Boolean type
@@ -191,7 +193,7 @@ The following table shows the type names as well as aliases used in Spark SQL pa
 |**BinaryType**|BINARY|
 |**DecimalType**|DECIMAL, DEC, NUMERIC|
 |**CalendarIntervalType**|INTERVAL|
-|**ArrayType**|ARRAY<element_type>|
+|**ArrayType**|ARRAY\<element_type>|
 |**StructType**|STRUCT<field1_name: field1_type, field2_name: field2_type, ...>|
 |**MapType**|MAP<key_type, value_type>|
 
diff --git a/docs/sql-ref-datetime-pattern.md b/docs/sql-ref-datetime-pattern.md
index df19b9ce4c082..4b02cdad36109 100644
--- a/docs/sql-ref-datetime-pattern.md
+++ b/docs/sql-ref-datetime-pattern.md
@@ -30,25 +30,21 @@ Spark uses pattern letters in the following table for date and timestamp parsing
 
 |Symbol|Meaning|Presentation|Examples|
 |------|-------|------------|--------|
-|**G**|era|text|AD; Anno Domini; A|
+|**G**|era|text|AD; Anno Domini|
 |**y**|year|year|2020; 20|
-|**D**|day-of-year|number|189|
-|**M/L**|month-of-year|number/text|7; 07; Jul; July; J|
-|**d**|day-of-month|number|28|
+|**D**|day-of-year|number(3)|189|
+|**M/L**|month-of-year|month|7; 07; Jul; July|
+|**d**|day-of-month|number(3)|28|
 |**Q/q**|quarter-of-year|number/text|3; 03; Q3; 3rd quarter|
-|**Y**|week-based-year|year|1996; 96|
-|**w**|week-of-week-based-year|number|27|
-|**W**|week-of-month|number|4|
-|**E**|day-of-week|text|Tue; Tuesday; T|
-|**u**|localized day-of-week|number/text|2; 02; Tue; Tuesday; T|
-|**F**|week-of-month|number|3|
-|**a**|am-pm-of-day|text|PM|
-|**h**|clock-hour-of-am-pm (1-12)|number|12|
-|**K**|hour-of-am-pm (0-11)|number|0|
-|**k**|clock-hour-of-day (1-24)|number|0|
-|**H**|hour-of-day (0-23)|number|0|
-|**m**|minute-of-hour|number|30|
-|**s**|second-of-minute|number|55|
+|**E**|day-of-week|text|Tue; Tuesday|
+|**F**|aligned day of week in month|number(1)|3|
+|**a**|am-pm-of-day|am-pm|PM|
+|**h**|clock-hour-of-am-pm (1-12)|number(2)|12|
+|**K**|hour-of-am-pm (0-11)|number(2)|0|
+|**k**|clock-hour-of-day (1-24)|number(2)|0|
+|**H**|hour-of-day (0-23)|number(2)|0|
+|**m**|minute-of-hour|number(2)|30|
+|**s**|second-of-minute|number(2)|55|
 |**S**|fraction-of-second|fraction|978|
 |**V**|time-zone ID|zone-id|America/Los_Angeles; Z; -08:30|
 |**z**|time-zone name|zone-name|Pacific Standard Time; PST|
@@ -63,9 +59,9 @@ Spark uses pattern letters in the following table for date and timestamp parsing
 
 The count of pattern letters determines the format.
 
-- Text: The text style is determined based on the number of pattern letters used. Less than 4 pattern letters will use the short form. Exactly 4 pattern letters will use the full form. Exactly 5 pattern letters will use the narrow form. Six or more letters will fail.
+- Text: The text style is determined based on the number of pattern letters used. Less than 4 pattern letters will use the short text form, typically an abbreviation, e.g. day-of-week Monday might output "Mon". Exactly 4 pattern letters will use the full text form, typically the full description, e.g, day-of-week Monday might output "Monday". 5 or more letters will fail.
 
-- Number: If the count of letters is one, then the value is output using the minimum number of digits and without padding. Otherwise, the count of digits is used as the width of the output field, with the value zero-padded as necessary. The following pattern letters have constraints on the count of letters. Only one letter 'F' can be specified. Up to two letters of 'd', 'H', 'h', 'K', 'k', 'm', and 's' can be specified. Up to three letters of 'D' can be specified.
+- Number(n): The n here represents the maximum count of letters this type of datetime pattern can be used. If the count of letters is one, then the value is output using the minimum number of digits and without padding. Otherwise, the count of digits is used as the width of the output field, with the value zero-padded as necessary.
 
 - Number/Text: If the count of pattern letters is 3 or greater, use the Text rules above. Otherwise use the Number rules above.
 
@@ -74,9 +70,9 @@ The count of pattern letters determines the format.
   For formatting, the fraction length would be padded to the number of contiguous 'S' with zeros.
   Spark supports datetime of micro-of-second precision, which has up to 6 significant digits, but can parse nano-of-second with exceeded part truncated.
 
-- Year: The count of letters determines the minimum field width below which padding is used. If the count of letters is two, then a reduced two digit form is used. For printing, this outputs the rightmost two digits. For parsing, this will parse using the base value of 2000, resulting in a year within the range 2000 to 2099 inclusive. If the count of letters is less than four (but not two), then the sign is only output for negative years. Otherwise, the sign is output if the pad width is exceeded when 'G' is not present.
+- Year: The count of letters determines the minimum field width below which padding is used. If the count of letters is two, then a reduced two digit form is used. For printing, this outputs the rightmost two digits. For parsing, this will parse using the base value of 2000, resulting in a year within the range 2000 to 2099 inclusive. If the count of letters is less than four (but not two), then the sign is only output for negative years. Otherwise, the sign is output if the pad width is exceeded when 'G' is not present. 7 or more letters will fail.
 
-- Month: If the number of pattern letters is 3 or more, the month is interpreted as text; otherwise, it is interpreted as a number. The text form is depend on letters - 'M' denotes the 'standard' form, and 'L' is for 'stand-alone' form. The difference between the 'standard' and 'stand-alone' forms is trickier to describe as there is no difference in English. However, in other languages there is a difference in the word used when the text is used alone, as opposed to in a complete date. For example, the word used for a month when used alone in a date picker is different to the word used for month in association with a day and year in a date. In Russian, 'Июль' is the stand-alone form of July, and 'Июля' is the standard form. Here are examples for all supported pattern letters (more than 5 letters is invalid):
+- Month: It follows the rule of Number/Text. The text form is depend on letters - 'M' denotes the 'standard' form, and 'L' is for 'stand-alone' form. These two forms are different only in some certain languages. For example, in Russian, 'Июль' is the stand-alone form of July, and 'Июля' is the standard form. Here are examples for all supported pattern letters:
   - `'M'` or `'L'`: Month number in a year starting from 1. There is no difference between 'M' and 'L'. Month from 1 to 9 are printed without padding.
     ```sql
     spark-sql> select date_format(date '1970-01-01', "M");
@@ -107,8 +103,8 @@ The count of pattern letters determines the format.
     ```
   - `'MMMM'`: full textual month representation in the standard form. It is used for parsing/formatting months as a part of dates/timestamps.
     ```sql
-    spark-sql> select date_format(date '1970-01-01', "MMMM yyyy");
-    January 1970
+    spark-sql> select date_format(date '1970-01-01', "d MMMM");
+    1 January
     spark-sql> select to_csv(named_struct('date', date '1970-01-01'), map('dateFormat', 'd MMMM', 'locale', 'RU'));
     1 января
     ```
@@ -119,13 +115,8 @@ The count of pattern letters determines the format.
     spark-sql> select to_csv(named_struct('date', date '1970-01-01'), map('dateFormat', 'LLLL', 'locale', 'RU'));
     январь
     ```
-  - `'LLLLL'` or `'MMMMM'`: Narrow textual representation of standard or stand-alone forms. Typically it is a single letter.
-    ```sql
-    spark-sql> select date_format(date '1970-07-01', "LLLLL");
-    J
-    spark-sql> select date_format(date '1970-01-01', "MMMMM");
-    J
-    ```
+
+- am-pm: This outputs the am-pm-of-day. Pattern letter count must be 1.
 
 - Zone ID(V): This outputs the display the time-zone ID. Pattern letter count must be 2.
 
@@ -141,11 +132,5 @@ The count of pattern letters determines the format.
   During formatting, all valid data will be output even it is in the optional section.
   During parsing, the whole section may be missing from the parsed string.
   An optional section is started by `[` and ended using `]` (or at the end of the pattern).
-
-More details for the text style:
-
-- Short Form: Short text, typically an abbreviation. For example, day-of-week Monday might output "Mon".
-
-- Full Form: Full text, typically the full description. For example, day-of-week Monday might output "Monday".
-
-- Narrow Form: Narrow text, typically a single letter. For example, day-of-week Monday might output "M".
+  
+- Symbols of 'E', 'F', 'q' and 'Q' can only be used for datetime formatting, e.g. `date_format`. They are not allowed used for datetime parsing, e.g. `to_timestamp`.
diff --git a/docs/sql-ref-literals.md b/docs/sql-ref-literals.md
index b83f7f0a97c24..3dbed846d40b8 100644
--- a/docs/sql-ref-literals.md
+++ b/docs/sql-ref-literals.md
@@ -219,6 +219,11 @@ double literals:
 decimal_digits  { D | exponent [ D ] }  | digit [ ... ] { exponent [ D ] | [ exponent ] D }
 ```
 
+float literals:
+```sql
+decimal_digits  { F | exponent [ F ] }  | digit [ ... ] { exponent [ F ] | [ exponent ] F }
+```
+
 While decimal_digits is defined as
 ```sql
 [ + | - ] { digit [ ... ] . [ digit [ ... ] ] | . digit [ ... ] }
@@ -239,6 +244,10 @@ E [ + | - ] digit [ ... ]
 
     Case insensitive, indicates `DOUBLE`, which is an 8-byte double-precision floating point number.
 
+* **F**
+
+    Case insensitive, indicates `FLOAT`, which is a 4-byte single-precision floating point number.
+
 * **BD**
 
     Case insensitive, indicates `DECIMAL`, with the total number of digits as precision and the number of digits to right of decimal point as scale.
diff --git a/docs/sql-ref-null-semantics.md b/docs/sql-ref-null-semantics.md
index fb5d2a312d0e1..3c12e7a28b64e 100644
--- a/docs/sql-ref-null-semantics.md
+++ b/docs/sql-ref-null-semantics.md
@@ -125,7 +125,7 @@ The following tables illustrate the behavior of logical operators when one or bo
 |True|NULL|True|NULL|
 |False|NULL|NULL|False|
 |NULL|True|True|NULL|
-|NULL|False|NULL|NULL|
+|NULL|False|NULL|False|
 |NULL|NULL|NULL|NULL|
 
 |operand|NOT|
diff --git a/docs/sql-ref-syntax-aux-analyze-table.md b/docs/sql-ref-syntax-aux-analyze-table.md
index 8f43d7388d7db..da5338564cda9 100644
--- a/docs/sql-ref-syntax-aux-analyze-table.md
+++ b/docs/sql-ref-syntax-aux-analyze-table.md
@@ -50,7 +50,7 @@ ANALYZE TABLE table_identifier [ partition_spec ]
      * If no analyze option is specified, `ANALYZE TABLE` collects the table's number of rows and size in bytes.
      * **NOSCAN**
 
-       Collects only the table's size in bytes ( which does not require scanning the entire table ).
+       Collects only the table's size in bytes (which does not require scanning the entire table).
      * **FOR COLUMNS col [ , ... ] `|` FOR ALL COLUMNS**
 
        Collects column statistics for each column specified, or alternatively for every column, as well as table statistics.
@@ -122,3 +122,7 @@ DESC EXTENDED students name;
 |     histogram|      NULL|
 +--------------+----------+
 ```
+
+### Related Statements
+
+* [ANALYZE TABLES](sql-ref-syntax-aux-analyze-tables.html)
diff --git a/docs/sql-ref-syntax-aux-analyze-tables.md b/docs/sql-ref-syntax-aux-analyze-tables.md
new file mode 100644
index 0000000000000..f70cfa4d7de2e
--- /dev/null
+++ b/docs/sql-ref-syntax-aux-analyze-tables.md
@@ -0,0 +1,110 @@
+---
+layout: global
+title: ANALYZE TABLES
+displayTitle: ANALYZE TABLES
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+The `ANALYZE TABLES` statement collects statistics about all the tables in a specified database to be used by the query optimizer to find a better query execution plan.
+
+### Syntax
+
+```sql
+ANALYZE TABLES [ { FROM | IN } database_name ] COMPUTE STATISTICS [ NOSCAN ]
+```
+
+### Parameters
+
+* **{ FROM `|` IN } database_name**
+
+    Specifies the name of the database to be analyzed. Without a database name, `ANALYZE` collects all tables in the current database that the current user has permission to analyze.
+
+* **[ NOSCAN ]**
+
+    Collects only the table's size in bytes (which does not require scanning the entire table).
+
+### Examples
+
+```sql
+CREATE DATABASE school_db;
+USE school_db;
+
+CREATE TABLE teachers (name STRING, teacher_id INT);
+INSERT INTO teachers VALUES ('Tom', 1), ('Jerry', 2);
+
+CREATE TABLE students (name STRING, student_id INT, age SHORT);
+INSERT INTO students VALUES ('Mark', 111111, 10), ('John', 222222, 11);
+
+ANALYZE TABLES IN school_db COMPUTE STATISTICS NOSCAN;
+
+DESC EXTENDED teachers;
++--------------------+--------------------+-------+
+|            col_name|           data_type|comment|
++--------------------+--------------------+-------+
+|                name|              string|   null|
+|          teacher_id|                 int|   null|
+|                 ...|                 ...|    ...|
+|            Provider|             parquet|       |
+|          Statistics|          1382 bytes|       |
+|                 ...|                 ...|    ...|
++--------------------+--------------------+-------+
+
+DESC EXTENDED students;
++--------------------+--------------------+-------+
+|            col_name|           data_type|comment|
++--------------------+--------------------+-------+
+|                name|              string|   null|
+|          student_id|                 int|   null|
+|                 age|            smallint|   null|
+|                 ...|                 ...|    ...|
+|          Statistics|          1828 bytes|       |
+|                 ...|                 ...|    ...|
++--------------------+--------------------+-------+
+
+ANALYZE TABLES COMPUTE STATISTICS;
+
+DESC EXTENDED teachers;
++--------------------+--------------------+-------+
+|            col_name|           data_type|comment|
++--------------------+--------------------+-------+
+|                name|              string|   null|
+|          teacher_id|                 int|   null|
+|                 ...|                 ...|    ...|
+|            Provider|             parquet|       |
+|          Statistics|  1382 bytes, 2 rows|       |
+|                 ...|                 ...|    ...|
++--------------------+--------------------+-------+
+
+DESC EXTENDED students;
++--------------------+--------------------+-------+
+|            col_name|           data_type|comment|
++--------------------+--------------------+-------+
+|                name|              string|   null|
+|          student_id|                 int|   null|
+|                 age|            smallint|   null|
+|                 ...|                 ...|    ...|
+|            Provider|             parquet|       |
+|          Statistics|  1828 bytes, 2 rows|       |
+|                 ...|                 ...|    ...|
++--------------------+--------------------+-------+
+```
+
+### Related Statements
+
+* [ANALYZE TABLE](sql-ref-syntax-aux-analyze-table.html)
diff --git a/docs/sql-ref-syntax-aux-analyze.md b/docs/sql-ref-syntax-aux-analyze.md
index 4c68e6b9ec974..7808966ffe145 100644
--- a/docs/sql-ref-syntax-aux-analyze.md
+++ b/docs/sql-ref-syntax-aux-analyze.md
@@ -20,3 +20,4 @@ license: |
 ---
 
  * [ANALYZE TABLE statement](sql-ref-syntax-aux-analyze-table.html)
+ * [ANALYZE TABLES statement](sql-ref-syntax-aux-analyze-tables.html)
diff --git a/docs/sql-ref-syntax-aux-cache-cache-table.md b/docs/sql-ref-syntax-aux-cache-cache-table.md
index 193e209d792b3..9a1e61abbabb2 100644
--- a/docs/sql-ref-syntax-aux-cache-cache-table.md
+++ b/docs/sql-ref-syntax-aux-cache-cache-table.md
@@ -49,6 +49,7 @@ CACHE [ LAZY ] TABLE table_identifier
      * `NONE`
      * `DISK_ONLY`
      * `DISK_ONLY_2`
+     * `DISK_ONLY_3`
      * `MEMORY_ONLY`
      * `MEMORY_ONLY_2`
      * `MEMORY_ONLY_SER`
@@ -78,5 +79,6 @@ CACHE TABLE testCache OPTIONS ('storageLevel' 'DISK_ONLY') SELECT * FROM testDat
 
 * [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
 * [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
-* [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
+* [REFRESH TABLE](sql-ref-syntax-aux-cache-refresh-table.html)
 * [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
+* [REFRESH FUNCTION](sql-ref-syntax-aux-cache-refresh-function.html)
\ No newline at end of file
diff --git a/docs/sql-ref-syntax-aux-cache-clear-cache.md b/docs/sql-ref-syntax-aux-cache-clear-cache.md
index ee33e6a98296d..aae4e39600375 100644
--- a/docs/sql-ref-syntax-aux-cache-clear-cache.md
+++ b/docs/sql-ref-syntax-aux-cache-clear-cache.md
@@ -39,5 +39,6 @@ CLEAR CACHE;
 
 * [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
 * [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
-* [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
+* [REFRESH TABLE](sql-ref-syntax-aux-cache-refresh-table.html)
 * [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
+* [REFRESH FUNCTION](sql-ref-syntax-aux-cache-refresh-function.html)
\ No newline at end of file
diff --git a/docs/sql-ref-syntax-aux-cache-refresh-function.md b/docs/sql-ref-syntax-aux-cache-refresh-function.md
new file mode 100644
index 0000000000000..d91fc062eb714
--- /dev/null
+++ b/docs/sql-ref-syntax-aux-cache-refresh-function.md
@@ -0,0 +1,60 @@
+---
+layout: global
+title: REFRESH FUNCTION
+displayTitle: REFRESH FUNCTION
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+`REFRESH FUNCTION` statement invalidates the cached function entry, which includes a class name
+and resource location of the given function. The invalidated cache is populated right away.
+Note that `REFRESH FUNCTION` only works for permanent functions. Refreshing native functions or temporary functions will cause an exception.
+
+### Syntax
+
+```sql
+REFRESH FUNCTION function_identifier
+```
+
+### Parameters
+
+* **function_identifier**
+
+    Specifies a function name, which is either a qualified or unqualified name. If no database identifier is provided, uses the current database.
+
+    **Syntax:** `[ database_name. ] function_name`
+
+### Examples
+
+```sql
+-- The cached entry of the function will be refreshed
+-- The function is resolved from the current database as the function name is unqualified.
+REFRESH FUNCTION func1;
+
+-- The cached entry of the function will be refreshed
+-- The function is resolved from tempDB database as the function name is qualified.
+REFRESH FUNCTION db1.func1;   
+```
+
+### Related Statements
+
+* [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
+* [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
+* [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
+* [REFRESH TABLE](sql-ref-syntax-aux-cache-refresh-table.html)
+* [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
diff --git a/docs/sql-ref-syntax-aux-cache-refresh-table.md b/docs/sql-ref-syntax-aux-cache-refresh-table.md
new file mode 100644
index 0000000000000..cc35c0451d5cb
--- /dev/null
+++ b/docs/sql-ref-syntax-aux-cache-refresh-table.md
@@ -0,0 +1,60 @@
+---
+layout: global
+title: REFRESH TABLE
+displayTitle: REFRESH TABLE
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+`REFRESH TABLE` statement invalidates the cached entries, which include data
+and metadata of the given table or view. The invalidated cache is populated in
+lazy manner when the cached table or the query associated with it is executed again.
+
+### Syntax
+
+```sql
+REFRESH [TABLE] table_identifier
+```
+
+### Parameters
+
+* **table_identifier**
+
+    Specifies a table name, which is either a qualified or unqualified name that designates a table/view. If no database identifier is provided, it refers to a temporary view or a table/view in the current database.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+### Examples
+
+```sql
+-- The cached entries of the table will be refreshed  
+-- The table is resolved from the current database as the table name is unqualified.
+REFRESH TABLE tbl1;
+
+-- The cached entries of the view will be refreshed or invalidated
+-- The view is resolved from tempDB database, as the view name is qualified.
+REFRESH TABLE tempDB.view1;   
+```
+
+### Related Statements
+
+* [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
+* [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
+* [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
+* [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
+* [REFRESH FUNCTION](sql-ref-syntax-aux-cache-refresh-function.html)
\ No newline at end of file
diff --git a/docs/sql-ref-syntax-aux-cache-refresh.md b/docs/sql-ref-syntax-aux-cache-refresh.md
index 82bc12da5d1ac..715bdcac3b6f1 100644
--- a/docs/sql-ref-syntax-aux-cache-refresh.md
+++ b/docs/sql-ref-syntax-aux-cache-refresh.md
@@ -53,4 +53,5 @@ REFRESH "hdfs://path/to/table";
 * [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
 * [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
 * [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
-* [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
+* [REFRESH TABLE](sql-ref-syntax-aux-cache-refresh-table.html)
+* [REFRESH FUNCTION](sql-ref-syntax-aux-cache-refresh-function.html)
diff --git a/docs/sql-ref-syntax-aux-cache-uncache-table.md b/docs/sql-ref-syntax-aux-cache-uncache-table.md
index c5a8fbbe08281..4456378cdee15 100644
--- a/docs/sql-ref-syntax-aux-cache-uncache-table.md
+++ b/docs/sql-ref-syntax-aux-cache-uncache-table.md
@@ -48,5 +48,6 @@ UNCACHE TABLE t1;
 
 * [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
 * [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
-* [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
+* [REFRESH TABLE](sql-ref-syntax-aux-cache-refresh-table.html)
 * [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
+* [REFRESH FUNCTION](sql-ref-syntax-aux-cache-refresh-function.html)
\ No newline at end of file
diff --git a/docs/sql-ref-syntax-aux-cache.md b/docs/sql-ref-syntax-aux-cache.md
index 418b8cc3403b5..17a13e67e532e 100644
--- a/docs/sql-ref-syntax-aux-cache.md
+++ b/docs/sql-ref-syntax-aux-cache.md
@@ -22,5 +22,6 @@ license: |
  * [CACHE TABLE statement](sql-ref-syntax-aux-cache-cache-table.html)
  * [UNCACHE TABLE statement](sql-ref-syntax-aux-cache-uncache-table.html)
  * [CLEAR CACHE statement](sql-ref-syntax-aux-cache-clear-cache.html)
- * [REFRESH TABLE statement](sql-ref-syntax-aux-refresh-table.html)
- * [REFRESH statement](sql-ref-syntax-aux-cache-refresh.html)
\ No newline at end of file
+ * [REFRESH TABLE statement](sql-ref-syntax-aux-cache-refresh-table.html)
+ * [REFRESH statement](sql-ref-syntax-aux-cache-refresh.html)
+ * [REFRESH FUNCTION statement](sql-ref-syntax-aux-cache-refresh-function.html)
diff --git a/docs/sql-ref-syntax-aux-conf-mgmt-reset.md b/docs/sql-ref-syntax-aux-conf-mgmt-reset.md
index 4caf57a232f89..68df0913b29d4 100644
--- a/docs/sql-ref-syntax-aux-conf-mgmt-reset.md
+++ b/docs/sql-ref-syntax-aux-conf-mgmt-reset.md
@@ -21,19 +21,34 @@ license: |
 
 ### Description
 
-Reset any runtime configurations specific to the current session which were set via the [SET](sql-ref-syntax-aux-conf-mgmt-set.html) command to their default values.
+The RESET command resets runtime configurations specific to the current session which were set via the [SET](sql-ref-syntax-aux-conf-mgmt-set.html) command to their default values.
 
 ### Syntax
 
 ```sql
-RESET
+RESET;
+
+RESET configuration_key;
 ```
 
+### Parameters
+
+* **(none)**
+
+    Reset any runtime configurations specific to the current session which were set via the [SET](sql-ref-syntax-aux-conf-mgmt-set.html) command to their default values.
+
+* **configuration_key**
+
+    Restore the value of the `configuration_key` to the default value. If the default value is undefined, the `configuration_key` will be removed.
+
 ### Examples
 
 ```sql
 -- Reset any runtime configurations specific to the current session which were set via the SET command to their default values.
 RESET;
+
+-- If you start your application with --conf spark.foo=bar and set spark.foo=foobar in runtime, the example below will restore it to 'bar'. If spark.foo is not specified during starting, the example bellow will remove this config from the SQLConf. It will ignore nonexistent keys.
+RESET spark.abc;
 ```
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-aux-conf-mgmt-set-timezone.md b/docs/sql-ref-syntax-aux-conf-mgmt-set-timezone.md
new file mode 100644
index 0000000000000..ada86d8dd3913
--- /dev/null
+++ b/docs/sql-ref-syntax-aux-conf-mgmt-set-timezone.md
@@ -0,0 +1,67 @@
+---
+layout: global
+title: SET TIME ZONE
+displayTitle: SET TIME ZONE
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+The SET TIME ZONE command sets the time zone of the current session.
+
+### Syntax
+
+```sql
+SET TIME ZONE LOCAL
+SET TIME ZONE 'timezone_value'
+SET TIME ZONE INTERVAL interval_literal
+```
+
+### Parameters
+
+* **LOCAL**
+
+    Set the time zone to the one specified in the java `user.timezone` property, or to the environment variable `TZ` if `user.timezone` is undefined, or to the system time zone if both of them are undefined.
+
+* **timezone_value**
+
+    The ID of session local timezone in the format of either region-based zone IDs or zone offsets. Region IDs must have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format '`(+|-)HH`', '`(+|-)HH:mm`' or '`(+|-)HH:mm:ss`', e.g '-08', '+01:00' or '-13:33:33'. Also, 'UTC' and 'Z' are supported as aliases of '+00:00'. Other short names are not recommended to use because they can be ambiguous.
+
+* **interval_literal**
+
+    The [interval literal](sql-ref-literals.html#interval-literal) represents the difference between the session time zone to the 'UTC'. It must be in the range of [-18, 18] hours and max to second precision, e.g. `INTERVAL 2 HOURS 30 MINUTES` or `INTERVAL '15:40:32' HOUR TO SECOND`.
+
+### Examples
+
+```sql
+-- Set time zone to the system default.
+SET TIME ZONE LOCAL;
+
+-- Set time zone to the region-based zone ID.
+SET TIME ZONE 'America/Los_Angeles';
+
+-- Set time zone to the Zone offset.
+SET TIME ZONE '+08:00';
+
+-- Set time zone with intervals.
+SET TIME ZONE INTERVAL 1 HOUR 30 MINUTES;
+SET TIME ZONE INTERVAL '08:30:00' HOUR TO SECOND;
+```
+
+### Related Statements
+
+* [SET](sql-ref-syntax-aux-conf-mgmt-set.html)
diff --git a/docs/sql-ref-syntax-aux-conf-mgmt.md b/docs/sql-ref-syntax-aux-conf-mgmt.md
index f5e48ef2fee30..3312bcb503500 100644
--- a/docs/sql-ref-syntax-aux-conf-mgmt.md
+++ b/docs/sql-ref-syntax-aux-conf-mgmt.md
@@ -20,4 +20,5 @@ license: |
 ---
 
  * [SET](sql-ref-syntax-aux-conf-mgmt-set.html)
- * [UNSET](sql-ref-syntax-aux-conf-mgmt-reset.html)
+ * [RESET](sql-ref-syntax-aux-conf-mgmt-reset.html)
+ * [SET TIME ZONE](sql-ref-syntax-aux-conf-mgmt-set-timezone.html)
diff --git a/docs/sql-ref-syntax-aux-refresh-table.md b/docs/sql-ref-syntax-aux-refresh-table.md
deleted file mode 100644
index 8d4a804f88671..0000000000000
--- a/docs/sql-ref-syntax-aux-refresh-table.md
+++ /dev/null
@@ -1,59 +0,0 @@
----
-layout: global
-title: REFRESH TABLE
-displayTitle: REFRESH TABLE
-license: |
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
- 
-     http://www.apache.org/licenses/LICENSE-2.0
- 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
----
-
-### Description
-
-`REFRESH TABLE` statement invalidates the cached entries, which include data
-and metadata of the given table or view. The invalidated cache is populated in
-lazy manner when the cached table or the query associated with it is executed again.
-
-### Syntax
-
-```sql
-REFRESH [TABLE] table_identifier
-```
-
-### Parameters
-
-* **table_identifier**
-
-    Specifies a table name, which is either a qualified or unqualified name that designates a table/view. If no database identifier is provided, it refers to a temporary view or a table/view in the current database.
-
-    **Syntax:** `[ database_name. ] table_name`
-
-### Examples
-
-```sql
--- The cached entries of the table will be refreshed  
--- The table is resolved from the current database as the table name is unqualified.
-REFRESH TABLE tbl1;
-
--- The cached entries of the view will be refreshed or invalidated
--- The view is resolved from tempDB database, as the view name is qualified.
-REFRESH TABLE tempDB.view1;   
-```
-
-### Related Statements
-
-* [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
-* [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
-* [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
-* [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
diff --git a/docs/sql-ref-syntax-aux-resource-mgmt-add-archive.md b/docs/sql-ref-syntax-aux-resource-mgmt-add-archive.md
new file mode 100644
index 0000000000000..0ae444a96b980
--- /dev/null
+++ b/docs/sql-ref-syntax-aux-resource-mgmt-add-archive.md
@@ -0,0 +1,54 @@
+---
+layout: global
+title: ADD ARCHIVE
+displayTitle: ADD ARCHIVE
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+`ADD ARCHIVE` can be used to add an archive file to the list of resources. The given archive file should be one of .zip, .tar, .tar.gz, .tgz and .jar. The added archive file can be listed using [LIST ARCHIVE](sql-ref-syntax-aux-resource-mgmt-list-archive.html).
+
+### Syntax
+
+```sql
+ADD { ARCHIVE | ARCHIVES } file_name [ ... ]
+```
+
+### Parameters
+
+* **file_name**
+
+    The name of the archive file to be added. It could be either on a local file system or a distributed file system.
+
+### Examples
+
+```sql
+ADD ARCHIVE /tmp/test.tar.gz;
+ADD ARCHIVE "/path/to/some.zip";
+ADD ARCHIVE '/some/other.tgz';
+ADD ARCHIVE "/path with space/abc.tar";
+ADD ARCHIVES "/path with space/def.tgz" '/path with space/ghi.zip';
+```
+
+### Related Statements
+
+* [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
+* [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
+* [LIST ARCHIVE](sql-ref-syntax-aux-resource-mgmt-list-archive.html)
+* [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
+* [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
diff --git a/docs/sql-ref-syntax-aux-resource-mgmt-add-file.md b/docs/sql-ref-syntax-aux-resource-mgmt-add-file.md
index 9203293d0c981..c1f55b5598a0e 100644
--- a/docs/sql-ref-syntax-aux-resource-mgmt-add-file.md
+++ b/docs/sql-ref-syntax-aux-resource-mgmt-add-file.md
@@ -26,7 +26,7 @@ license: |
 ### Syntax
 
 ```sql
-ADD FILE resource_name
+ADD { FILE | FILES } resource_name [ ... ]
 ```
 
 ### Parameters
@@ -43,11 +43,13 @@ ADD FILE "/path/to/file/abc.txt";
 ADD FILE '/another/test.txt';
 ADD FILE "/path with space/abc.txt";
 ADD FILE "/path/to/some/directory";
+ADD FILES "/path with space/cde.txt" '/path with space/fgh.txt';
 ```
 
 ### Related Statements
 
 * [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
 * [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
+* [LIST ARCHIVE](sql-ref-syntax-aux-resource-mgmt-list-archive.html)
 * [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
-
+* [ADD ARCHIVE](sql-ref-syntax-aux-resource-mgmt-add-archive.html)
diff --git a/docs/sql-ref-syntax-aux-resource-mgmt-add-jar.md b/docs/sql-ref-syntax-aux-resource-mgmt-add-jar.md
index 4694bff99daf5..7ad56bf0657b2 100644
--- a/docs/sql-ref-syntax-aux-resource-mgmt-add-jar.md
+++ b/docs/sql-ref-syntax-aux-resource-mgmt-add-jar.md
@@ -26,15 +26,25 @@ license: |
 ### Syntax
 
 ```sql
-ADD JAR file_name
+ADD { JAR | JARS } file_name [ ... ]
 ```
 
 ### Parameters
 
 * **file_name**
 
-    The name of the JAR file to be added. It could be either on a local file system or a distributed file system.
+    The name of the JAR file to be added. It could be either on a local file system or a distributed file system or an Ivy URI.
+    Apache Ivy is a popular dependency manager focusing on flexibility and simplicity. Now we support two parameter in URI query string:
 
+     * transitive: whether to download dependent jars related to your ivy URL. The parameter name is case-sensitive, and the parameter value is case-insensitive. If multiple transitive parameters are specified, the last one wins.
+     * exclude: exclusion list during downloading Ivy URI jar and dependent jars.
+
+    User can write Ivy URI such as:
+
+      ivy://group:module:version
+      ivy://group:module:version?transitive=[true|false]
+      ivy://group:module:version?transitive=[true|false]&exclude=group:module,group:module
+        
 ### Examples
 
 ```sql
@@ -42,6 +52,11 @@ ADD JAR /tmp/test.jar;
 ADD JAR "/path/to/some.jar";
 ADD JAR '/some/other.jar';
 ADD JAR "/path with space/abc.jar";
+ADD JARS "/path with space/def.jar" '/path with space/ghi.jar';
+ADD JAR "ivy://group:module:version";
+ADD JAR "ivy://group:module:version?transitive=false"
+ADD JAR "ivy://group:module:version?transitive=true"
+ADD JAR "ivy://group:module:version?exclude=group:module&transitive=true"
 ```
 
 ### Related Statements
@@ -49,3 +64,5 @@ ADD JAR "/path with space/abc.jar";
 * [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
 * [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
 * [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
+* [ADD ARCHIVE](sql-ref-syntax-aux-resource-mgmt-add-archive.html)
+* [LIST ARCHIVE](sql-ref-syntax-aux-resource-mgmt-list-archive.html)
diff --git a/docs/sql-ref-syntax-aux-resource-mgmt-list-archive.md b/docs/sql-ref-syntax-aux-resource-mgmt-list-archive.md
new file mode 100644
index 0000000000000..ae5051ab116dc
--- /dev/null
+++ b/docs/sql-ref-syntax-aux-resource-mgmt-list-archive.md
@@ -0,0 +1,53 @@
+---
+layout: global
+title: LIST ARCHIVE
+displayTitle: LIST ARCHIVE
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+`LIST ARCHIVE` lists the archives added by [ADD ARCHIVE](sql-ref-syntax-aux-resource-mgmt-add-archive.html).
+
+### Syntax
+
+```sql
+LIST { ARCHIVE | ARCHIVES } file_name [ ... ]
+```
+
+### Examples
+
+```sql
+ADD ARCHIVE /tmp/test.zip;
+ADD ARCHIVE /tmp/test_2.tar.gz;
+LIST ARCHIVE;
+-- output for LIST ARCHIVE
+file:/tmp/test.zip
+file:/tmp/test_2.tar.gz
+
+LIST ARCHIVE /tmp/test.zip /some/random.tgz /another/random.tar;
+-- output
+file:/tmp/test.zip
+```
+
+### Related Statements
+
+* [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
+* [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
+* [ADD ARCHIVE](sql-ref-syntax-aux-resource-mgmt-add-archive.html)
+* [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
+* [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
diff --git a/docs/sql-ref-syntax-aux-resource-mgmt-list-file.md b/docs/sql-ref-syntax-aux-resource-mgmt-list-file.md
index 9b9a7df7f612f..d92f5fe162794 100644
--- a/docs/sql-ref-syntax-aux-resource-mgmt-list-file.md
+++ b/docs/sql-ref-syntax-aux-resource-mgmt-list-file.md
@@ -26,7 +26,7 @@ license: |
 ### Syntax
 
 ```sql
-LIST FILE
+LIST { FILE | FILES } file_name [ ... ]
 ```
 
 ### Examples
@@ -48,5 +48,6 @@ file:/private/tmp/test
 
 * [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
 * [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
+* [ADD ARCHIVE](sql-ref-syntax-aux-resource-mgmt-add-archive.html)
 * [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
-
+* [LIST ARCHIVE](sql-ref-syntax-aux-resource-mgmt-list-archive.html)
diff --git a/docs/sql-ref-syntax-aux-resource-mgmt-list-jar.md b/docs/sql-ref-syntax-aux-resource-mgmt-list-jar.md
index 04aa52c2ad8af..6c705aa8b1dc7 100644
--- a/docs/sql-ref-syntax-aux-resource-mgmt-list-jar.md
+++ b/docs/sql-ref-syntax-aux-resource-mgmt-list-jar.md
@@ -26,7 +26,8 @@ license: |
 ### Syntax
 
 ```sql
-LIST JAR
+LIST { JAR | JARS } file_name [ ... ]
+
 ```
 
 ### Examples
@@ -48,5 +49,6 @@ spark://192.168.1.112:62859/jars/test.jar
 
 * [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
 * [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
+* [ADD ARCHIVE](sql-ref-syntax-aux-resource-mgmt-add-archive.html)
 * [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
-
+* [LIST ARCHIVE](sql-ref-syntax-aux-resource-mgmt-list-archive.html)
diff --git a/docs/sql-ref-syntax-aux-resource-mgmt.md b/docs/sql-ref-syntax-aux-resource-mgmt.md
index 50c12ef7c2beb..4a262824b7ae6 100644
--- a/docs/sql-ref-syntax-aux-resource-mgmt.md
+++ b/docs/sql-ref-syntax-aux-resource-mgmt.md
@@ -21,5 +21,7 @@ license: |
 
  * [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
  * [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
+ * [ADD ARCHIVE](sql-ref-syntax-aux-resource-mgmt-add-archive.html)
  * [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
  * [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
+ * [LIST ARCHIVE](sql-ref-syntax-aux-resource-mgmt-list-archive.html)
\ No newline at end of file
diff --git a/docs/sql-ref-syntax-aux-show-table.md b/docs/sql-ref-syntax-aux-show-table.md
index 0ce0a3eefa538..3314402ea3e2b 100644
--- a/docs/sql-ref-syntax-aux-show-table.md
+++ b/docs/sql-ref-syntax-aux-show-table.md
@@ -97,7 +97,7 @@ SHOW TABLE EXTENDED LIKE 'employee';
 +--------+---------+-----------+--------------------------------------------------------------+
 
 -- showing the multiple table details with pattern matching
-SHOW TABLE EXTENDED  LIKE `employe*`;
+SHOW TABLE EXTENDED  LIKE 'employe*';
 +--------+---------+-----------+--------------------------------------------------------------+
 |database|tableName|isTemporary|                         information                          |
 +--------+---------+-----------+--------------------------------------------------------------+
@@ -146,7 +146,7 @@ SHOW TABLE EXTENDED  LIKE `employe*`;
 +--------+---------+----------+---------------------------------------------------------------+
   
 -- show partition file system details
-SHOW TABLE EXTENDED  IN default LIKE `employee` PARTITION (`grade=1`);
+SHOW TABLE EXTENDED  IN default LIKE 'employee' PARTITION (grade=1);
 +--------+---------+-----------+--------------------------------------------------------------+
 |database|tableName|isTemporary|                         information                          |
 +--------+---------+-----------+--------------------------------------------------------------+
@@ -169,7 +169,7 @@ SHOW TABLE EXTENDED  IN default LIKE `employee` PARTITION (`grade=1`);
 +--------+---------+-----------+--------------------------------------------------------------+
 
 -- show partition file system details with regex fails as shown below
-SHOW TABLE EXTENDED  IN default LIKE `empl*` PARTITION (`grade=1`);
+SHOW TABLE EXTENDED  IN default LIKE 'empl*' PARTITION (grade=1);
 Error: Error running query: org.apache.spark.sql.catalyst.analysis.NoSuchTableException:
  Table or view 'emplo*' not found in database 'default'; (state=,code=0)
 ```
diff --git a/docs/sql-ref-syntax-ddl-alter-table.md b/docs/sql-ref-syntax-ddl-alter-table.md
index eb0e9a9f9cf73..2d42eb478001c 100644
--- a/docs/sql-ref-syntax-ddl-alter-table.md
+++ b/docs/sql-ref-syntax-ddl-alter-table.md
@@ -25,7 +25,11 @@ license: |
 
 ### RENAME 
 
-`ALTER TABLE RENAME TO` statement changes the table name of an existing table in the database.
+`ALTER TABLE RENAME TO` statement changes the table name of an existing table in the database. The table rename command cannot be used to move a table between databases, only to rename a table within the same database.
+
+If the table is cached, the commands clear cached data of the table. The cache will be lazily filled when the next time the table is accessed. Additionally:
+  * the table rename command uncaches all table's dependents such as views that refer to the table. The dependents should be cached again explicitly.
+  * the partition rename command clears caches of all table dependents while keeping them as cached. So, their caches will be lazily filled when the next time they are accessed.
 
 #### Syntax
 
@@ -45,7 +49,7 @@ ALTER TABLE table_identifier partition_spec RENAME TO partition_spec
 
 * **partition_spec**
 
-    Partition to be renamed.
+    Partition to be renamed. Note that one can use a typed literal (e.g., date'2019-01-02') in the partition spec.
 
     **Syntax:** `PARTITION ( partition_col_name  = partition_col_val [ , ... ] )`
 
@@ -103,6 +107,8 @@ ALTER TABLE table_identifier { ALTER | CHANGE } [ COLUMN ] col_spec alterColumnA
 
 `ALTER TABLE ADD` statement adds partition to the partitioned table.
 
+If the table is cached, the command clears cached data of the table and all its dependents that refer to it. The cache will be lazily filled when the next time the table or the dependents are accessed.
+
 ##### Syntax
 
 ```sql
@@ -120,7 +126,7 @@ ALTER TABLE table_identifier ADD [IF NOT EXISTS]
 
 * **partition_spec**
 
-    Partition to be added.
+    Partition to be added. Note that one can use a typed literal (e.g., date'2019-01-02') in the partition spec.
 
     **Syntax:** `PARTITION ( partition_col_name  = partition_col_val [ , ... ] )`
 
@@ -128,6 +134,8 @@ ALTER TABLE table_identifier ADD [IF NOT EXISTS]
 
 `ALTER TABLE DROP` statement drops the partition of the table.
 
+If the table is cached, the command clears cached data of the table and all its dependents that refer to it. The cache will be lazily filled when the next time the table or the dependents are accessed.
+
 ##### Syntax
 
 ```sql
@@ -144,7 +152,7 @@ ALTER TABLE table_identifier DROP [ IF EXISTS ] partition_spec [PURGE]
 
 * **partition_spec**
 
-    Partition to be dropped.
+    Partition to be dropped. Note that one can use a typed literal (e.g., date'2019-01-02') in the partition spec.
 
     **Syntax:** `PARTITION ( partition_col_name  = partition_col_val [ , ... ] )`
      
@@ -187,6 +195,8 @@ ALTER TABLE table_identifier [ partition_spec ] SET SERDE serde_class_name
 `ALTER TABLE SET` command can also be used for changing the file location and file format for 
 existing tables. 
 
+If the table is cached, the `ALTER TABLE .. SET LOCATION` command clears cached data of the table and all its dependents that refer to it. The cache will be lazily filled when the next time the table or the dependents are accessed.
+
 ##### Syntax
 
 ```sql
@@ -207,7 +217,7 @@ ALTER TABLE table_identifier [ partition_spec ] SET LOCATION 'new_location'
 
 * **partition_spec**
 
-    Specifies the partition on which the property has to be set.
+    Specifies the partition on which the property has to be set. Note that one can use a typed literal (e.g., date'2019-01-02') in the partition spec.
 
     **Syntax:** `PARTITION ( partition_col_name  = partition_col_val [ , ... ] )`
 
diff --git a/docs/sql-ref-syntax-ddl-alter-view.md b/docs/sql-ref-syntax-ddl-alter-view.md
index a34e77decf593..d69f24677116d 100644
--- a/docs/sql-ref-syntax-ddl-alter-view.md
+++ b/docs/sql-ref-syntax-ddl-alter-view.md
@@ -28,6 +28,8 @@ the name of a view to a different name, set and unset the metadata of the view b
 Renames the existing view. If the new view name already exists in the source database, a `TableAlreadyExistsException` is thrown. This operation
 does not support moving the views across databases.
 
+If the view is cached, the command clears cached data of the view and all its dependents that refer to it. View's cache will be lazily filled when the next time the view is accessed. The command leaves view's dependents as uncached.
+
 #### Syntax
 ```sql
 ALTER VIEW view_identifier RENAME TO view_identifier
diff --git a/docs/sql-ref-syntax-ddl-create-function.md b/docs/sql-ref-syntax-ddl-create-function.md
index aa6c1fad7b56b..dfa4f4f8123d8 100644
--- a/docs/sql-ref-syntax-ddl-create-function.md
+++ b/docs/sql-ref-syntax-ddl-create-function.md
@@ -112,7 +112,7 @@ SHOW USER FUNCTIONS;
 +------------------+
 
 -- Invoke the function. Every selected value should be incremented by 10.
-SELECT simple_udf(c1) AS function_return_value FROM t1;
+SELECT simple_udf(c1) AS function_return_value FROM test;
 +---------------------+
 |function_return_value|
 +---------------------+
@@ -150,7 +150,7 @@ CREATE OR REPLACE FUNCTION simple_udf AS 'SimpleUdfR'
     USING JAR '/tmp/SimpleUdfR.jar';
 
 -- Invoke the function. Every selected value should be incremented by 20.
-SELECT simple_udf(c1) AS function_return_value FROM t1;
+SELECT simple_udf(c1) AS function_return_value FROM test;
 +---------------------+
 |function_return_value|
 +---------------------+
diff --git a/docs/sql-ref-syntax-ddl-create-table-datasource.md b/docs/sql-ref-syntax-ddl-create-table-datasource.md
index d334447a91011..ba0516afbbfad 100644
--- a/docs/sql-ref-syntax-ddl-create-table-datasource.md
+++ b/docs/sql-ref-syntax-ddl-create-table-datasource.md
@@ -67,7 +67,12 @@ as any order. For example, you can write COMMENT table_comment after TBLPROPERTI
 
 * **SORTED BY**
 
-    Determines the order in which the data is stored in buckets. Default is Ascending order.
+    Specifies an ordering of bucket columns. Optionally, one can use ASC for an ascending order or DESC for a descending order after any column names in the SORTED BY clause.
+    If not specified, ASC is assumed by default.
+   
+* **INTO num_buckets BUCKETS**
+
+    Specifies buckets numbers, which is used in `CLUSTERED BY` clause.
 
 * **LOCATION**
 
diff --git a/docs/sql-ref-syntax-ddl-create-table-hiveformat.md b/docs/sql-ref-syntax-ddl-create-table-hiveformat.md
index 38f8856a24e3d..b2f5957416a80 100644
--- a/docs/sql-ref-syntax-ddl-create-table-hiveformat.md
+++ b/docs/sql-ref-syntax-ddl-create-table-hiveformat.md
@@ -31,6 +31,9 @@ CREATE [ EXTERNAL ] TABLE [ IF NOT EXISTS ] table_identifier
     [ COMMENT table_comment ]
     [ PARTITIONED BY ( col_name2[:] col_type2 [ COMMENT col_comment2 ], ... ) 
         | ( col_name1, col_name2, ... ) ]
+    [ CLUSTERED BY ( col_name1, col_name2, ...) 
+        [ SORTED BY ( col_name1 [ ASC | DESC ], col_name2 [ ASC | DESC ], ... ) ] 
+        INTO num_buckets BUCKETS ]
     [ ROW FORMAT row_format ]
     [ STORED AS file_format ]
     [ LOCATION path ]
@@ -51,15 +54,30 @@ as any order. For example, you can write COMMENT table_comment after TBLPROPERTI
 
 * **EXTERNAL**
 
-    Table is defined using the path provided as LOCATION, does not use default location for this table.
+    Table is defined using the path provided as `LOCATION`, does not use default location for this table.
 
 * **PARTITIONED BY**
 
     Partitions are created on the table, based on the columns specified.
+    
+* **CLUSTERED BY**
 
-* **ROW FORMAT**
+    Partitions created on the table will be bucketed into fixed buckets based on the column specified for bucketing.
 
-    SERDE is used to specify a custom SerDe or the DELIMITED clause in order to use the native SerDe.
+    **NOTE:** Bucketing is an optimization technique that uses buckets (and bucketing columns) to determine data partitioning and avoid data shuffle.
+
+* **SORTED BY**
+
+    Specifies an ordering of bucket columns. Optionally, one can use ASC for an ascending order or DESC for a descending order after any column names in the SORTED BY clause.
+    If not specified, ASC is assumed by default.
+
+* **INTO num_buckets BUCKETS**
+
+    Specifies buckets numbers, which is used in `CLUSTERED BY` clause.
+
+* **row_format** 
+
+    Specifies the row format for input and output. See [HIVE FORMAT](sql-ref-syntax-hive-format.html) for more syntax details.
 
 * **STORED AS**
 
@@ -114,9 +132,61 @@ CREATE TABLE student (id INT, name STRING)
     PARTITIONED BY (age INT);
 
 --Use Row Format and file format
-CREATE TABLE student (id INT,name STRING)
+CREATE TABLE student (id INT, name STRING)
     ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
     STORED AS TEXTFILE;
+
+--Use complex datatype
+CREATE EXTERNAL TABLE family(
+        name STRING,
+        friends ARRAY<STRING>,
+        children MAP<STRING, INT>,
+        address STRUCT<street: STRING, city: STRING>
+    )
+    ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' ESCAPED BY '\\'
+    COLLECTION ITEMS TERMINATED BY '_'
+    MAP KEYS TERMINATED BY ':'
+    LINES TERMINATED BY '\n'
+    NULL DEFINED AS 'foonull'
+    STORED AS TEXTFILE
+    LOCATION '/tmp/family/';
+
+--Use predefined custom SerDe
+CREATE TABLE avroExample
+    ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
+    STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
+        OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
+    TBLPROPERTIES ('avro.schema.literal'='{ "namespace": "org.apache.hive",
+        "name": "first_schema",
+        "type": "record",
+        "fields": [
+                { "name":"string1", "type":"string" },
+                { "name":"string2", "type":"string" }
+            ] }');
+
+--Use personalized custom SerDe(we may need to `ADD JAR xxx.jar` first to ensure we can find the serde_class,
+--or you may run into `CLASSNOTFOUND` exception)
+ADD JAR /tmp/hive_serde_example.jar;
+
+CREATE EXTERNAL TABLE family (id INT, name STRING)
+    ROW FORMAT SERDE 'com.ly.spark.serde.SerDeExample'
+    STORED AS INPUTFORMAT 'com.ly.spark.example.serde.io.SerDeExampleInputFormat'
+        OUTPUTFORMAT 'com.ly.spark.example.serde.io.SerDeExampleOutputFormat'
+    LOCATION '/tmp/family/';
+
+--Use `CLUSTERED BY` clause to create bucket table without `SORTED BY`
+CREATE TABLE clustered_by_test1 (ID INT, AGE STRING)
+    CLUSTERED BY (ID)
+    INTO 4 BUCKETS
+    STORED AS ORC
+
+--Use `CLUSTERED BY` clause to create bucket table with `SORTED BY`
+CREATE TABLE clustered_by_test2 (ID INT, NAME STRING)
+    PARTITIONED BY (YEAR STRING)
+    CLUSTERED BY (ID, NAME)
+    SORTED BY (ID ASC)
+    INTO 3 BUCKETS
+    STORED AS PARQUET
 ```
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-ddl-drop-table.md b/docs/sql-ref-syntax-ddl-drop-table.md
index a15a9928f437d..6c115fd5f83aa 100644
--- a/docs/sql-ref-syntax-ddl-drop-table.md
+++ b/docs/sql-ref-syntax-ddl-drop-table.md
@@ -26,6 +26,8 @@ if the table is not `EXTERNAL` table. If the table is not present it throws an e
 
 In case of an external table, only the associated metadata information is removed from the metastore database.
 
+If the table is cached, the command uncaches the table and all its dependents.
+
 ### Syntax
 
 ```sql
diff --git a/docs/sql-ref-syntax-ddl-repair-table.md b/docs/sql-ref-syntax-ddl-repair-table.md
index c2ef0a7b7fbe9..2e3711c260282 100644
--- a/docs/sql-ref-syntax-ddl-repair-table.md
+++ b/docs/sql-ref-syntax-ddl-repair-table.md
@@ -23,10 +23,12 @@ license: |
 
 `MSCK REPAIR TABLE` recovers all the partitions in the directory of a table and updates the Hive metastore. When creating a table using `PARTITIONED BY` clause, partitions are generated and registered in the Hive metastore. However, if the partitioned table is created from existing data, partitions are not registered automatically in the Hive metastore. User needs to run `MSCK REPAIR TABLE` to register the partitions. `MSCK REPAIR TABLE` on a non-existent table or a table without partitions throws an exception. Another way to recover partitions is to use `ALTER TABLE RECOVER PARTITIONS`.
 
+If the table is cached, the command clears cached data of the table and all its dependents that refer to it. The cache will be lazily filled when the next time the table or the dependents are accessed.
+
 ### Syntax
 
 ```sql
-MSCK REPAIR TABLE table_identifier
+MSCK REPAIR TABLE table_identifier [{ADD|DROP|SYNC} PARTITIONS]
 ```
 
 ### Parameters
@@ -37,6 +39,14 @@ MSCK REPAIR TABLE table_identifier
 
     **Syntax:** `[ database_name. ] table_name`
 
+* **`{ADD|DROP|SYNC} PARTITIONS`**
+
+    Specifies how to recover partitions. If not specified, **ADD** is the default.
+
+    * **ADD**, the command adds new partitions to the session catalog for all sub-folder in the base table folder that don't belong to any table partitions.
+    * **DROP**, the command drops all partitions from the session catalog that have non-existing locations in the file system.
+    * **SYNC** is the combination of **DROP** and **ADD**. 
+
 ### Examples
 
 ```sql
diff --git a/docs/sql-ref-syntax-ddl-truncate-table.md b/docs/sql-ref-syntax-ddl-truncate-table.md
index 6139814a3259a..3bc4d7a93e520 100644
--- a/docs/sql-ref-syntax-ddl-truncate-table.md
+++ b/docs/sql-ref-syntax-ddl-truncate-table.md
@@ -25,6 +25,8 @@ The `TRUNCATE TABLE` statement removes all the rows from a table or partition(s)
 or an external/temporary table. In order to truncate multiple partitions at once, the user can specify the partitions 
 in `partition_spec`. If no `partition_spec` is specified it will remove all partitions in the table.
 
+If the table is cached, the command clears cached data of the table and all its dependents that refer to it. The cache will be lazily filled when the next time the table or the dependents are accessed.
+
 ### Syntax
 
 ```sql
diff --git a/docs/sql-ref-syntax-qry-select-usedb.md b/docs/sql-ref-syntax-ddl-usedb.md
similarity index 100%
rename from docs/sql-ref-syntax-qry-select-usedb.md
rename to docs/sql-ref-syntax-ddl-usedb.md
diff --git a/docs/sql-ref-syntax-ddl.md b/docs/sql-ref-syntax-ddl.md
index 82fbf0498a20f..cb3e04c0ec910 100644
--- a/docs/sql-ref-syntax-ddl.md
+++ b/docs/sql-ref-syntax-ddl.md
@@ -34,4 +34,4 @@ Data Definition Statements are used to create or modify the structure of databas
  * [DROP VIEW](sql-ref-syntax-ddl-drop-view.html)
  * [TRUNCATE TABLE](sql-ref-syntax-ddl-truncate-table.html)
  * [REPAIR TABLE](sql-ref-syntax-ddl-repair-table.html)
- * [USE DATABASE](sql-ref-syntax-qry-select-usedb.html)
+ * [USE DATABASE](sql-ref-syntax-ddl-usedb.html)
diff --git a/docs/sql-ref-syntax-dml-insert-into.md b/docs/sql-ref-syntax-dml-insert-into.md
index ed5da2b2d28df..1120c6e0a89df 100644
--- a/docs/sql-ref-syntax-dml-insert-into.md
+++ b/docs/sql-ref-syntax-dml-insert-into.md
@@ -26,7 +26,7 @@ The `INSERT INTO` statement inserts new rows into a table. The inserted rows can
 ### Syntax
 
 ```sql
-INSERT INTO [ TABLE ] table_identifier [ partition_spec ]
+INSERT INTO [ TABLE ] table_identifier [ partition_spec ] [ ( column_list ) ]
     { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] | query }
 ```
 
@@ -40,11 +40,19 @@ INSERT INTO [ TABLE ] table_identifier [ partition_spec ]
 
 * **partition_spec**
 
-    An optional parameter that specifies a comma separated list of key and value pairs
-    for partitions.
+    An optional parameter that specifies a comma-separated list of key and value pairs
+    for partitions. Note that one can use a typed literal (e.g., date'2019-01-02') in the partition spec.
 
     **Syntax:** `PARTITION ( partition_col_name  = partition_col_val [ , ... ] )`
 
+* **column_list**
+
+    An optional parameter that specifies a comma-separated list of columns belonging to the `table_identifier` table. Spark will reorder the columns of the input query to match the table schema according to the specified column list.
+
+    **Note:**The current behaviour has some limitations:
+    - All specified columns should exist in the table and not be duplicated from each other. It includes all columns except the static partition columns.
+    - The size of the column list should be exactly the size of the data from `VALUES` clause or query.
+
 * **VALUES ( { value `|` NULL } [ , ... ] ) [ , ( ... ) ]**
 
     Specifies the values to be inserted. Either an explicitly specified value or a NULL can be inserted.
@@ -62,18 +70,18 @@ INSERT INTO [ TABLE ] table_identifier [ partition_spec ]
 #### Single Row Insert Using a VALUES Clause
 
 ```sql
-CREATE TABLE students (name VARCHAR(64), address VARCHAR(64), student_id INT)
-    USING PARQUET PARTITIONED BY (student_id);
+CREATE TABLE students (name VARCHAR(64), address VARCHAR(64))
+    USING PARQUET PARTITIONED BY (student_id INT);
 
 INSERT INTO students VALUES
     ('Amy Smith', '123 Park Ave, San Jose', 111111);
 
 SELECT * FROM students;
-+---------+---------------------+----------+
-|     name|              address|student_id|
-+---------+---------------------+----------+
-|Amy Smith|123 Park Ave,San Jose|    111111|
-+---------+---------------------+----------+
++---------+----------------------+----------+
+|     name|               address|student_id|
++---------+----------------------+----------+
+|Amy Smith|123 Park Ave, San Jose|    111111|
++---------+----------------------+----------+
 ```
 
 #### Multi-Row Insert Using a VALUES Clause
@@ -100,29 +108,29 @@ SELECT * FROM students;
 ```sql
 -- Assuming the persons table has already been created and populated.
 SELECT * FROM persons;
-+-------------+-------------------------+---------+
-|         name|                  address|      ssn|
-+-------------+-------------------------+---------+
-|Dora Williams|134 Forest Ave, Melo Park|123456789|
-+-------------+-------------------------+---------+
-|  Eddie Davis|  245 Market St, Milpitas|345678901|
-+-------------+-------------------------+---------+
++-------------+--------------------------+---------+
+|         name|                   address|      ssn|
++-------------+--------------------------+---------+
+|Dora Williams|134 Forest Ave, Menlo Park|123456789|
++-------------+--------------------------+---------+
+|  Eddie Davis|   245 Market St, Milpitas|345678901|
++-------------+--------------------------+---------+
 
 INSERT INTO students PARTITION (student_id = 444444)
     SELECT name, address FROM persons WHERE name = "Dora Williams";
 
 SELECT * FROM students;
-+-------------+-------------------------+----------+
-|         name|                  address|student_id|
-+-------------+-------------------------+----------+
-|    Amy Smith|   123 Park Ave, San Jose|    111111|
-+-------------+-------------------------+----------+
-|    Bob Brown| 456 Taylor St, Cupertino|    222222|
-+-------------+-------------------------+----------+
-|Cathy Johnson|  789 Race Ave, Palo Alto|    333333|
-+-------------+-------------------------+----------+
-|Dora Williams|134 Forest Ave, Melo Park|    444444|
-+-------------+-------------------------+----------+
++-------------+--------------------------+----------+
+|         name|                   address|student_id|
++-------------+--------------------------+----------+
+|    Amy Smith|    123 Park Ave, San Jose|    111111|
++-------------+--------------------------+----------+
+|    Bob Brown|  456 Taylor St, Cupertino|    222222|
++-------------+--------------------------+----------+
+|Cathy Johnson|   789 Race Ave, Palo Alto|    333333|
++-------------+--------------------------+----------+
+|Dora Williams|134 Forest Ave, Menlo Park|    444444|
++-------------+--------------------------+----------+
 ```
 
 #### Insert Using a TABLE Statement
@@ -141,21 +149,21 @@ SELECT * FROM visiting_students;
 INSERT INTO students TABLE visiting_students;
 
 SELECT * FROM students;
-+-------------+-------------------------+----------+
-|         name|                  address|student_id|
-+-------------+-------------------------+----------+
-|    Amy Smith|    123 Park Ave,San Jose|    111111|
-+-------------+-------------------------+----------+
-|    Bob Brown| 456 Taylor St, Cupertino|    222222|
-+-------------+-------------------------+----------+
-|Cathy Johnson|  789 Race Ave, Palo Alto|    333333|
-+-------------+-------------------------+----------+
-|Dora Williams|134 Forest Ave, Melo Park|    444444|
-+-------------+-------------------------+----------+
-|Fleur Laurent|    345 Copper St, London|    777777|
-+-------------+-------------------------+----------+
-|Gordon Martin|     779 Lake Ave, Oxford|    888888|
-+-------------+-------------------------+----------+
++-------------+--------------------------+----------+
+|         name|                   address|student_id|
++-------------+--------------------------+----------+
+|    Amy Smith|    123 Park Ave, San Jose|    111111|
++-------------+--------------------------+----------+
+|    Bob Brown|  456 Taylor St, Cupertino|    222222|
++-------------+--------------------------+----------+
+|Cathy Johnson|   789 Race Ave, Palo Alto|    333333|
++-------------+--------------------------+----------+
+|Dora Williams|134 Forest Ave, Menlo Park|    444444|
++-------------+--------------------------+----------+
+|Fleur Laurent|     345 Copper St, London|    777777|
++-------------+--------------------------+----------+
+|Gordon Martin|      779 Lake Ave, Oxford|    888888|
++-------------+--------------------------+----------+
 ```
 
 #### Insert Using a FROM Statement
@@ -177,25 +185,66 @@ INSERT INTO students
      FROM applicants SELECT name, address, id applicants WHERE qualified = true;
 
 SELECT * FROM students;
-+-------------+-------------------------+----------+
-|         name|                  address|student_id|
-+-------------+-------------------------+----------+
-|    Amy Smith|   123 Park Ave, San Jose|    111111|
-+-------------+-------------------------+----------+
-|    Bob Brown| 456 Taylor St, Cupertino|    222222|
-+-------------+-------------------------+----------+
-|Cathy Johnson|  789 Race Ave, Palo Alto|    333333|
-+-------------+-------------------------+----------+
-|Dora Williams|134 Forest Ave, Melo Park|    444444|
-+-------------+-------------------------+----------+
-|Fleur Laurent|    345 Copper St, London|    777777|
-+-------------+-------------------------+----------+
-|Gordon Martin|     779 Lake Ave, Oxford|    888888|
-+-------------+-------------------------+----------+
-|  Helen Davis|469 Mission St, San Diego|    999999|
-+-------------+-------------------------+----------+
-|   Jason Wang|    908 Bird St, Saratoga|    121212|
-+-------------+-------------------------+----------+
++-------------+--------------------------+----------+
+|         name|                   address|student_id|
++-------------+--------------------------+----------+
+|    Amy Smith|    123 Park Ave, San Jose|    111111|
++-------------+--------------------------+----------+
+|    Bob Brown|  456 Taylor St, Cupertino|    222222|
++-------------+--------------------------+----------+
+|Cathy Johnson|   789 Race Ave, Palo Alto|    333333|
++-------------+--------------------------+----------+
+|Dora Williams|134 Forest Ave, Menlo Park|    444444|
++-------------+--------------------------+----------+
+|Fleur Laurent|     345 Copper St, London|    777777|
++-------------+--------------------------+----------+
+|Gordon Martin|      779 Lake Ave, Oxford|    888888|
++-------------+--------------------------+----------+
+|  Helen Davis| 469 Mission St, San Diego|    999999|
++-------------+--------------------------+----------+
+|   Jason Wang|     908 Bird St, Saratoga|    121212|
++-------------+--------------------------+----------+
+```
+
+#### Insert Using a Typed Date Literal for a Partition Column Value
+```sql
+CREATE TABLE students (name STRING, address  STRING) PARTITIONED BY (birthday DATE);
+
+INSERT INTO students PARTITION (birthday = date'2019-01-02')
+    VALUES ('Amy Smith', '123 Park Ave, San Jose');
+
+SELECT * FROM students;
++-------------+-------------------------+-----------+
+|         name|                  address|   birthday|
++-------------+-------------------------+-----------+
+|    Amy Smith|   123 Park Ave, San Jose| 2019-01-02|
++-------------+-------------------------+-----------+
+#### Insert with a column list
+
+```sql
+INSERT INTO students (address, name, student_id) VALUES
+    ('Hangzhou, China', 'Kent Yao', 11215016);
+
+SELECT * FROM students WHERE name = 'Kent Yao';
++---------+----------------------+----------+
+|     name|               address|student_id|
++---------+----------------------+----------+
+|Kent Yao |       Hangzhou, China|  11215016|
++---------+----------------------+----------+
+```
+
+#### Insert with both a partition spec and a column list
+
+```sql
+INSERT INTO students PARTITION (student_id = 11215017) (address, name) VALUES
+    ('Hangzhou, China', 'Kent Yao Jr.');
+
+SELECT * FROM students WHERE student_id = 11215017;
++------------+----------------------+----------+
+|        name|               address|student_id|
++------------+----------------------+----------+
+|Kent Yao Jr.|       Hangzhou, China|  11215017|
++------------+----------------------+----------+
 ```
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-dml-insert-overwrite-table.md b/docs/sql-ref-syntax-dml-insert-overwrite-table.md
index ecfd060dfd5ee..04e2ce5caf843 100644
--- a/docs/sql-ref-syntax-dml-insert-overwrite-table.md
+++ b/docs/sql-ref-syntax-dml-insert-overwrite-table.md
@@ -26,7 +26,7 @@ The `INSERT OVERWRITE` statement overwrites the existing data in the table using
 ### Syntax
 
 ```sql
-INSERT OVERWRITE [ TABLE ] table_identifier [ partition_spec [ IF NOT EXISTS ] ]
+INSERT OVERWRITE [ TABLE ] table_identifier [ partition_spec [ IF NOT EXISTS ] ] [ ( column_list ) ]
     { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] | query }
 ```
 
@@ -40,11 +40,21 @@ INSERT OVERWRITE [ TABLE ] table_identifier [ partition_spec [ IF NOT EXISTS ] ]
 
 * **partition_spec**
 
-    An optional parameter that specifies a comma separated list of key and value pairs
-    for partitions.
+    An optional parameter that specifies a comma-separated list of key and value pairs
+    for partitions. Note that one can use a typed literal (e.g., date'2019-01-02') in the partition spec.
 
     **Syntax:** `PARTITION ( partition_col_name [ = partition_col_val ] [ , ... ] )`
 
+* **column_list**
+
+    An optional parameter that specifies a comma-separated list of columns belonging to the `table_identifier` table. Spark will reorder the columns of the input query to match the table schema according to the specified column list.
+
+    **Note**
+
+    The current behaviour has some limitations:
+    - All specified columns should exist in the table and not be duplicated from each other. It includes all columns except the static partition columns.
+    - The size of the column list should be exactly the size of the data from `VALUES` clause or query.
+
 * **VALUES ( { value `|` NULL } [ , ... ] ) [ , ( ... ) ]**
 
     Specifies the values to be inserted. Either an explicitly specified value or a NULL can be inserted.
@@ -64,18 +74,18 @@ INSERT OVERWRITE [ TABLE ] table_identifier [ partition_spec [ IF NOT EXISTS ] ]
 ```sql
 -- Assuming the students table has already been created and populated.
 SELECT * FROM students;
-+-------------+-------------------------+----------+
-|         name|                  address|student_id|
-+-------------+-------------------------+----------+
-|    Amy Smith|   123 Park Ave, San Jose|    111111|
-|    Bob Brown| 456 Taylor St, Cupertino|    222222|
-|Cathy Johnson|  789 Race Ave, Palo Alto|    333333|
-|Dora Williams|134 Forest Ave, Melo Park|    444444|
-|Fleur Laurent|    345 Copper St, London|    777777|
-|Gordon Martin|     779 Lake Ave, Oxford|    888888|
-|  Helen Davis|469 Mission St, San Diego|    999999|
-|   Jason Wang|    908 Bird St, Saratoga|    121212|
-+-------------+-------------------------+----------+
++-------------+--------------------------+----------+
+|         name|                   address|student_id|
++-------------+--------------------------+----------+
+|    Amy Smith|    123 Park Ave, San Jose|    111111|
+|    Bob Brown|  456 Taylor St, Cupertino|    222222|
+|Cathy Johnson|   789 Race Ave, Palo Alto|    333333|
+|Dora Williams|134 Forest Ave, Menlo Park|    444444|
+|Fleur Laurent|     345 Copper St, London|    777777|
+|Gordon Martin|      779 Lake Ave, Oxford|    888888|
+|  Helen Davis| 469 Mission St, San Diego|    999999|
+|   Jason Wang|     908 Bird St, Saratoga|    121212|
++-------------+--------------------------+----------+
 
 INSERT OVERWRITE students VALUES
     ('Ashua Hill', '456 Erica Ct, Cupertino', 111111),
@@ -95,25 +105,25 @@ SELECT * FROM students;
 ```sql
 -- Assuming the persons table has already been created and populated.
 SELECT * FROM persons;
-+-------------+-------------------------+---------+
-|         name|                  address|      ssn|
-+-------------+-------------------------+---------+
-|Dora Williams|134 Forest Ave, Melo Park|123456789|
-+-------------+-------------------------+---------+
-|  Eddie Davis|   245 Market St,Milpitas|345678901|
-+-------------+-------------------------+---------+
++-------------+--------------------------+---------+
+|         name|                   address|      ssn|
++-------------+--------------------------+---------+
+|Dora Williams|134 Forest Ave, Menlo Park|123456789|
++-------------+--------------------------+---------+
+|  Eddie Davis|   245 Market St, Milpitas|345678901|
++-------------+--------------------------+---------+
 
 INSERT OVERWRITE students PARTITION (student_id = 222222)
     SELECT name, address FROM persons WHERE name = "Dora Williams";
 
 SELECT * FROM students;
-+-------------+-------------------------+----------+
-|         name|                  address|student_id|
-+-------------+-------------------------+----------+
-|   Ashua Hill|  456 Erica Ct, Cupertino|    111111|
-+-------------+-------------------------+----------+
-|Dora Williams|134 Forest Ave, Melo Park|    222222|
-+-------------+-------------------------+----------+
++-------------+--------------------------+----------+
+|         name|                   address|student_id|
++-------------+--------------------------+----------+
+|   Ashua Hill|   456 Erica Ct, Cupertino|    111111|
++-------------+--------------------------+----------+
+|Dora Williams|134 Forest Ave, Menlo Park|    222222|
++-------------+--------------------------+----------+
 ```
 
 #### Insert Using a TABLE Statement
@@ -169,6 +179,57 @@ SELECT * FROM students;
 +-----------+-------------------------+----------+
 ```
 
+#### Insert Using a Typed Date Literal for a Partition Column Value
+```sql
+CREATE TABLE students (name STRING, address  STRING) PARTITIONED BY (birthday DATE);
+
+INSERT INTO students PARTITION (birthday = date'2019-01-02')
+    VALUES ('Amy Smith', '123 Park Ave, San Jose');
+
+SELECT * FROM students;
++-------------+-------------------------+-----------+
+|         name|                  address|   birthday|
++-------------+-------------------------+-----------+
+|    Amy Smith|   123 Park Ave, San Jose| 2019-01-02|
++-------------+-------------------------+-----------+
+
+INSERT INTO students PARTITION (birthday = date'2019-01-02')
+    VALUES('Jason Wang', '908 Bird St, Saratoga');
+
+SELECT * FROM students;
++-----------+-------------------------+-----------+
+|       name|                  address|   birthday|
++-----------+-------------------------+-----------+
+| Jason Wang|    908 Bird St, Saratoga| 2019-01-02|
++-----------+-------------------------+-----------+
+#### Insert with a column list
+
+```sql
+INSERT OVERWRITE students (address, name, student_id) VALUES
+    ('Hangzhou, China', 'Kent Yao', 11215016);
+
+SELECT * FROM students WHERE name = 'Kent Yao';
++---------+----------------------+----------+
+|     name|               address|student_id|
++---------+----------------------+----------+
+|Kent Yao |       Hangzhou, China|  11215016|
++---------+----------------------+----------+
+```
+
+#### Insert with both a partition spec and a column list
+
+```sql
+INSERT OVERWRITE students PARTITION (student_id = 11215016) (address, name) VALUES
+    ('Hangzhou, China', 'Kent Yao Jr.');
+
+SELECT * FROM students WHERE student_id = 11215016;
++------------+----------------------+----------+
+|        name|               address|student_id|
++------------+----------------------+----------+
+|Kent Yao Jr.|       Hangzhou, China|  11215016|
++------------+----------------------+----------+
+```
+
 ### Related Statements
 
 * [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
diff --git a/docs/sql-ref-syntax-dml-load.md b/docs/sql-ref-syntax-dml-load.md
index 9381b4267fb24..08922b869da32 100644
--- a/docs/sql-ref-syntax-dml-load.md
+++ b/docs/sql-ref-syntax-dml-load.md
@@ -23,6 +23,8 @@ license: |
 
 `LOAD DATA` statement loads the data into a Hive serde table from the user specified directory or file. If a directory is specified then all the files from the directory are loaded. If a file is specified then only the single file is loaded. Additionally the `LOAD DATA` statement takes an optional partition specification. When a partition is specified, the data files (when input source is a directory) or the single file (when input source is a file) are loaded into the partition of the target table.
 
+If the table is cached, the command clears cached data of the table and all its dependents that refer to it. The cache will be lazily filled when the next time the table or the dependents are accessed.
+
 ### Syntax
 
 ```sql
diff --git a/docs/sql-ref-syntax-hive-format.md b/docs/sql-ref-syntax-hive-format.md
new file mode 100644
index 0000000000000..8092e582d97ad
--- /dev/null
+++ b/docs/sql-ref-syntax-hive-format.md
@@ -0,0 +1,73 @@
+---
+layout: global
+title: Hive Row Format
+displayTitle: Hive Row Format
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+Spark supports a Hive row format in `CREATE TABLE` and `TRANSFORM` clause to specify serde or text delimiter.
+There are two ways to define a row format in `row_format` of `CREATE TABLE` and `TRANSFORM` clauses.
+  1. `SERDE` clause to specify a custom SerDe class.
+  2. `DELIMITED` clause to specify a delimiter, an escape character, a null character, and so on for the native SerDe.
+
+### Syntax
+
+```sql
+row_format:    
+    SERDE serde_class [ WITH SERDEPROPERTIES (k1=v1, k2=v2, ... ) ]
+    | DELIMITED [ FIELDS TERMINATED BY fields_terminated_char [ ESCAPED BY escaped_char ] ] 
+        [ COLLECTION ITEMS TERMINATED BY collection_items_terminated_char ] 
+        [ MAP KEYS TERMINATED BY map_key_terminated_char ]
+        [ LINES TERMINATED BY row_terminated_char ]
+        [ NULL DEFINED AS null_char ]
+```
+
+### Parameters
+   
+* **SERDE serde_class**
+
+    Specifies a fully-qualified class name of custom SerDe.
+
+* **SERDEPROPERTIES**
+
+    A list of key-value pairs that is used to tag the SerDe definition.
+
+* **FIELDS TERMINATED BY**
+
+    Used to define a column separator.
+    
+* **COLLECTION ITEMS TERMINATED BY**
+
+    Used to define a collection item separator.
+   
+* **MAP KEYS TERMINATED BY**
+
+    Used to define a map key separator.
+    
+* **LINES TERMINATED BY**
+
+    Used to define a row separator.
+    
+* **NULL DEFINED AS**
+
+    Used to define the specific value for NULL.
+    
+* **ESCAPED BY**
+
+    Used for escape mechanism.
diff --git a/docs/sql-ref-syntax-qry-select-case.md b/docs/sql-ref-syntax-qry-select-case.md
new file mode 100644
index 0000000000000..6136b161d234b
--- /dev/null
+++ b/docs/sql-ref-syntax-qry-select-case.md
@@ -0,0 +1,109 @@
+---
+layout: global
+title: CASE Clause
+displayTitle: CASE Clause
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+`CASE` clause uses a rule to return a specific result based on the specified condition, similar to if/else statements in other programming languages.
+
+### Syntax
+
+```sql
+CASE [ expression ] { WHEN boolean_expression THEN then_expression } [ ... ]
+    [ ELSE else_expression ]
+END
+```
+
+### Parameters
+    
+* **boolean_expression**
+
+    Specifies any expression that evaluates to a result type `boolean`. Two or
+    more expressions may be combined together using the logical
+    operators ( `AND`, `OR` ).
+
+* **then_expression**
+
+    Specifies the then expression based on the `boolean_expression` condition; `then_expression` and `else_expression` should all be same type or coercible to a common type.
+    
+* **else_expression**
+
+    Specifies the default expression; `then_expression` and `else_expression` should all be same type or coercible to a common type.
+    
+### Examples
+
+```sql
+CREATE TABLE person (id INT, name STRING, age INT);
+INSERT INTO person VALUES
+    (100, 'John', 30),
+    (200, 'Mary', NULL),
+    (300, 'Mike', 80),
+    (400, 'Dan', 50);
+
+SELECT id, CASE WHEN id > 200 THEN 'bigger' ELSE 'small' END FROM person;
++------+--------------------------------------------------+
+|  id  | CASE WHEN (id > 200) THEN bigger ELSE small END  |
++------+--------------------------------------------------+
+| 100  | small                                            |
+| 200  | small                                            |
+| 300  | bigger                                           |
+| 400  | bigger                                           |
++------+--------------------------------------------------+
+
+SELECT id, CASE id WHEN 100 then 'bigger' WHEN  id > 300 THEN '300' ELSE 'small' END FROM person;
++------+-----------------------------------------------------------------------------------------------+
+|  id  | CASE WHEN (id = 100) THEN bigger WHEN (id = CAST((id > 300) AS INT)) THEN 300 ELSE small END  |
++------+-----------------------------------------------------------------------------------------------+
+| 100  | bigger                                                                                        |
+| 200  | small                                                                                         |
+| 300  | small                                                                                         |
+| 400  | small                                                                                         |
++------+-----------------------------------------------------------------------------------------------+
+
+SELECT * FROM person
+    WHERE 
+        CASE 1 = 1 
+            WHEN 100 THEN 'big' 
+            WHEN 200 THEN 'bigger'
+            WHEN 300 THEN 'biggest' 
+            ELSE 'small'
+        END = 'small';
++------+-------+-------+
+|  id  | name  |  age  |
++------+-------+-------+
+| 100  | John  | 30    |
+| 200  | Mary  | NULL  |
+| 300  | Mike  | 80    |
+| 400  | Dan   | 50    |
++------+-------+-------+
+```
+
+### Related Statements
+
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-clusterby.md b/docs/sql-ref-syntax-qry-select-clusterby.md
index e3bd2ed926ecc..9bcfac5b3b5c6 100644
--- a/docs/sql-ref-syntax-qry-select-clusterby.md
+++ b/docs/sql-ref-syntax-qry-select-clusterby.md
@@ -99,3 +99,6 @@ SELECT age, name FROM person CLUSTER BY age;
 * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
 * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
 * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-distribute-by.md b/docs/sql-ref-syntax-qry-select-distribute-by.md
index 1fdfb91dad286..fbf662d3359f4 100644
--- a/docs/sql-ref-syntax-qry-select-distribute-by.md
+++ b/docs/sql-ref-syntax-qry-select-distribute-by.md
@@ -94,3 +94,6 @@ SELECT age, name FROM person DISTRIBUTE BY age;
 * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
 * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
 * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
\ No newline at end of file
diff --git a/docs/sql-ref-syntax-qry-select-file.md b/docs/sql-ref-syntax-qry-select-file.md
new file mode 100644
index 0000000000000..c3dc406ee79e6
--- /dev/null
+++ b/docs/sql-ref-syntax-qry-select-file.md
@@ -0,0 +1,76 @@
+---
+layout: global
+title: File
+displayTitle: File
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+You can query a file with a specified format directly with SQL.
+
+### Syntax
+
+```sql
+file_format.`file_path`
+```
+
+### Parameters
+
+* **file_format**
+
+    Specifies a file format for a given file path, could be TEXTFILE, ORC, PARQUET, etc.
+
+* **file_path**
+
+    Specifies a file path with a given format.
+
+### Examples
+
+```sql
+-- PARQUET file
+SELECT * FROM parquet.`examples/src/main/resources/users.parquet`;
++------+--------------+----------------+
+|  name|favorite_color|favorite_numbers|
++------+--------------+----------------+
+|Alyssa|          null|  [3, 9, 15, 20]|
+|   Ben|           red|              []|
++------+--------------+----------------+
+
+-- ORC file
+SELECT * FROM orc.`examples/src/main/resources/users.orc`;
++------+--------------+----------------+
+|  name|favorite_color|favorite_numbers|
++------+--------------+----------------+
+|Alyssa|          null|  [3, 9, 15, 20]|
+|   Ben|           red|              []|
++------+--------------+----------------+
+
+-- JSON file
+SELECT * FROM json.`examples/src/main/resources/people.json`;
++----+-------+
+| age|   name|
++----+-------+
+|null|Michael|
+|  30|   Andy|
+|  19| Justin|
++----+-------+
+```
+
+### Related Statements
+
+* [SELECT](sql-ref-syntax-qry-select.html)
diff --git a/docs/sql-ref-syntax-qry-select-groupby.md b/docs/sql-ref-syntax-qry-select-groupby.md
index bd9377ef78df6..d7827f88801d4 100644
--- a/docs/sql-ref-syntax-qry-select-groupby.md
+++ b/docs/sql-ref-syntax-qry-select-groupby.md
@@ -24,15 +24,16 @@ license: |
 The `GROUP BY` clause is used to group the rows based on a set of specified grouping expressions and compute aggregations on
 the group of rows based on one or more specified aggregate functions. Spark also supports advanced aggregations to do multiple
 aggregations for the same input record set via `GROUPING SETS`, `CUBE`, `ROLLUP` clauses.
-When a FILTER clause is attached to an aggregate function, only the matching rows are passed to that function.
+The grouping expressions and advanced aggregations can be mixed in the `GROUP BY` clause and nested in a `GROUPING SETS` clause.
+See more details in the `Mixed/Nested Grouping Analytics` section. When a FILTER clause is attached to
+an aggregate function, only the matching rows are passed to that function.
 
 ### Syntax
 
 ```sql
-GROUP BY group_expression [ , group_expression [ , ... ] ]
-    [ { WITH ROLLUP | WITH CUBE | GROUPING SETS (grouping_set [ , ...]) } ]
+GROUP BY group_expression [ , group_expression [ , ... ] ] [ WITH { ROLLUP | CUBE } ]
 
-GROUP BY GROUPING SETS (grouping_set [ , ...])
+GROUP BY { group_expression | { ROLLUP | CUBE | GROUPING SETS } (grouping_set [ , ...]) } [ , ... ]
 ```
 
 While aggregate functions are defined as
@@ -42,42 +43,84 @@ aggregate_name ( [ DISTINCT ] expression [ , ... ] ) [ FILTER ( WHERE boolean_ex
 
 ### Parameters
 
-* **GROUPING SETS**
+* **group_expression**
 
-    Groups the rows for each subset of the expressions specified in the grouping sets. For example,
-    `GROUP BY GROUPING SETS (warehouse, product)` is semantically equivalent
-    to union of results of `GROUP BY warehouse` and `GROUP BY product`. This clause
-    is a shorthand for a `UNION ALL` where each leg of the `UNION ALL`
-    operator performs aggregation of subset of the columns specified in the `GROUPING SETS` clause.
+    Specifies the criteria based on which the rows are grouped together. The grouping of rows is performed based on
+    result values of the grouping expressions. A grouping expression may be a column name like `GROUP BY a`, a column position like
+    `GROUP BY 0`, or an expression like `GROUP BY a + b`.
 
 * **grouping_set**
 
-    A grouping set is specified by zero or more comma-separated expressions in parentheses.
+    A grouping set is specified by zero or more comma-separated expressions in parentheses. When the
+    grouping set has only one element, parentheses can be omitted. For example, `GROUPING SETS ((a), (b))`
+    is the same as `GROUPING SETS (a, b)`.
 
-    **Syntax:** `( [ expression [ , ... ] ] )`
+    **Syntax:** `{ ( [ expression [ , ... ] ] ) | expression }`
 
-* **grouping_expression**
+* **GROUPING SETS**
 
-    Specifies the critieria based on which the rows are grouped together. The grouping of rows is performed based on
-    result values of the grouping expressions. A grouping expression may be a column alias, a column position
-    or an expression.
+    Groups the rows for each grouping set specified after GROUPING SETS. For example,
+    `GROUP BY GROUPING SETS ((warehouse), (product))` is semantically equivalent
+    to union of results of `GROUP BY warehouse` and `GROUP BY product`. This clause
+    is a shorthand for a `UNION ALL` where each leg of the `UNION ALL`
+    operator performs aggregation of each grouping set specified in the `GROUPING SETS` clause.
+    Similarly, `GROUP BY GROUPING SETS ((warehouse, product), (product), ())` is semantically
+    equivalent to the union of results of `GROUP BY warehouse, product`, `GROUP BY product`
+    and global aggregate.
+    
+    **Note:** For Hive compatibility Spark allows `GROUP BY ... GROUPING SETS (...)`. The GROUP BY
+    expressions are usually ignored, but if it contains extra expressions than the GROUPING SETS
+    expressions, the extra expressions will be included in the grouping expressions and the value
+    is always null. For example, `SELECT a, b, c FROM ... GROUP BY a, b, c GROUPING SETS (a, b)`,
+    the output of column `c` is always null.
 
 * **ROLLUP**
 
     Specifies multiple levels of aggregations in a single statement. This clause is used to compute aggregations
     based on multiple grouping sets. `ROLLUP` is a shorthand for `GROUPING SETS`. For example,
-    `GROUP BY warehouse, product WITH ROLLUP` is equivalent to `GROUP BY GROUPING SETS
-    ((warehouse, product), (warehouse), ())`.
+    `GROUP BY warehouse, product WITH ROLLUP` or `GROUP BY ROLLUP(warehouse, product)` is equivalent to
+    `GROUP BY GROUPING SETS((warehouse, product), (warehouse), ())`.
+    `GROUP BY ROLLUP(warehouse, product, (warehouse, location))` is equivalent to
+    `GROUP BY GROUPING SETS((warehouse, product, location), (warehouse, product), (warehouse), ())`.
     The N elements of a `ROLLUP` specification results in N+1 `GROUPING SETS`.
 
 * **CUBE**
 
     `CUBE` clause is used to perform aggregations based on combination of grouping columns specified in the
     `GROUP BY` clause. `CUBE` is a shorthand for `GROUPING SETS`. For example,
-    `GROUP BY warehouse, product WITH CUBE` is equivalent to `GROUP BY GROUPING SETS
-    ((warehouse, product), (warehouse), (product), ())`.
+    `GROUP BY warehouse, product WITH CUBE` or `GROUP BY CUBE(warehouse, product)` is equivalent to 
+    `GROUP BY GROUPING SETS((warehouse, product), (warehouse), (product), ())`.
+    `GROUP BY CUBE(warehouse, product, (warehouse, location))` is equivalent to
+    `GROUP BY GROUPING SETS((warehouse, product, location), (warehouse, product), (warehouse, location),
+     (product, warehouse, location), (warehouse), (product), (warehouse, product), ())`.
     The N elements of a `CUBE` specification results in 2^N `GROUPING SETS`.
 
+* **Mixed/Nested Grouping Analytics**
+
+    A GROUP BY clause can include multiple `group_expression`s and multiple `CUBE|ROLLUP|GROUPING SETS`s.
+    `GROUPING SETS` can also have nested `CUBE|ROLLUP|GROUPING SETS` clauses, e.g.
+    `GROUPING SETS(ROLLUP(warehouse, location), CUBE(warehouse, location))`,
+    `GROUPING SETS(warehouse, GROUPING SETS(location, GROUPING SETS(ROLLUP(warehouse, location), CUBE(warehouse, location))))`.
+    `CUBE|ROLLUP` is just a syntax sugar for `GROUPING SETS`, please refer to the sections above for
+    how to translate `CUBE|ROLLUP` to `GROUPING SETS`. `group_expression` can be treated as a single-group
+    `GROUPING SETS` under this context. For multiple `GROUPING SETS` in the `GROUP BY` clause, we generate
+    a single `GROUPING SETS` by doing a cross-product of the original `GROUPING SETS`s. For nested `GROUPING SETS` in the `GROUPING SETS` clause,
+    we simply take its grouping sets and strip it. For example,
+    `GROUP BY warehouse, GROUPING SETS((product), ()), GROUPING SETS((location, size), (location), (size), ())`
+    and `GROUP BY warehouse, ROLLUP(product), CUBE(location, size)` is equivalent to 
+    `GROUP BY GROUPING SETS(
+        (warehouse, product, location, size), 
+        (warehouse, product, location),
+        (warehouse, product, size), 
+        (warehouse, product),
+        (warehouse, location, size),
+        (warehouse, location),
+        (warehouse, size),
+        (warehouse))`.
+    
+    `GROUP BY GROUPING SETS(GROUPING SETS(warehouse), GROUPING SETS((warehouse, product)))` is equivalent to 
+    `GROUP BY GROUPING SETS((warehouse), (warehouse, product))`.
+
 * **aggregate_name**
 
     Specifies an aggregate function name (MIN, MAX, COUNT, SUM, AVG, etc.).
@@ -189,31 +232,6 @@ SELECT city, car_model, sum(quantity) AS sum FROM dealer
 | San Jose|  HondaCivic|  5|
 +---------+------------+---+
 
--- Alternate syntax for `GROUPING SETS` in which both `GROUP BY` and `GROUPING SETS`
--- specifications are present.
-SELECT city, car_model, sum(quantity) AS sum FROM dealer
-    GROUP BY city, car_model GROUPING SETS ((city, car_model), (city), (car_model), ())
-    ORDER BY city, car_model;
-+---------+------------+---+
-|     city|   car_model|sum|
-+---------+------------+---+
-|     null|        null| 78|
-|     null| HondaAccord| 33|
-|     null|    HondaCRV| 10|
-|     null|  HondaCivic| 35|
-|   Dublin|        null| 33|
-|   Dublin| HondaAccord| 10|
-|   Dublin|    HondaCRV|  3|
-|   Dublin|  HondaCivic| 20|
-|  Fremont|        null| 32|
-|  Fremont| HondaAccord| 15|
-|  Fremont|    HondaCRV|  7|
-|  Fremont|  HondaCivic| 10|
-| San Jose|        null| 13|
-| San Jose| HondaAccord|  8|
-| San Jose|  HondaCivic|  5|
-+---------+------------+---+
-
 -- Group by processing with `ROLLUP` clause.
 -- Equivalent GROUP BY GROUPING SETS ((city, car_model), (city), ())
 SELECT city, car_model, sum(quantity) AS sum FROM dealer
@@ -260,6 +278,30 @@ SELECT city, car_model, sum(quantity) AS sum FROM dealer
 | San Jose| HondaAccord|  8|
 | San Jose|  HondaCivic|  5|
 +---------+------------+---+
+
+--Prepare data for ignore nulls example
+CREATE TABLE person (id INT, name STRING, age INT);
+INSERT INTO person VALUES
+    (100, 'Mary', NULL),
+    (200, 'John', 30),
+    (300, 'Mike', 80),
+    (400, 'Dan', 50);
+
+--Select the first row in column age
+SELECT FIRST(age) FROM person;
++--------------------+
+| first(age, false)  |
++--------------------+
+| NULL               |
++--------------------+
+
+--Get the first row in column `age` ignore nulls,last row in column `id` and sum of column `id`.
+SELECT FIRST(age IGNORE NULLS), LAST(id), SUM(id) FROM person;
++-------------------+------------------+----------+
+| first(age, true)  | last(id, false)  | sum(id)  |
++-------------------+------------------+----------+
+| 30                | 400              | 1000     |
++-------------------+------------------+----------+
 ```
 
 ### Related Statements
@@ -272,3 +314,6 @@ SELECT city, car_model, sum(quantity) AS sum FROM dealer
 * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
 * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
 * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-having.md b/docs/sql-ref-syntax-qry-select-having.md
index 935782c551e1f..59a8c680945e6 100644
--- a/docs/sql-ref-syntax-qry-select-having.md
+++ b/docs/sql-ref-syntax-qry-select-having.md
@@ -125,3 +125,6 @@ SELECT sum(quantity) AS sum FROM dealer HAVING sum(quantity) > 10;
 * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
 * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
 * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-hints.md b/docs/sql-ref-syntax-qry-select-hints.md
index 4bb48b08d5e3b..5f1cb4c5bed0b 100644
--- a/docs/sql-ref-syntax-qry-select-hints.md
+++ b/docs/sql-ref-syntax-qry-select-hints.md
@@ -1,7 +1,7 @@
 ---
 layout: global
-title: Join Hints
-displayTitle: Join Hints
+title: Hints
+displayTitle: Hints
 license: |
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
@@ -21,15 +21,86 @@ license: |
 
 ### Description
 
-Join Hints allow users to suggest the join strategy that Spark should use. Prior to Spark 3.0, only the `BROADCAST` Join Hint was supported. `MERGE`, `SHUFFLE_HASH` and `SHUFFLE_REPLICATE_NL` Joint Hints support was added in 3.0. When different join strategy hints are specified on both sides of a join, Spark prioritizes hints in the following order: `BROADCAST` over `MERGE` over `SHUFFLE_HASH` over `SHUFFLE_REPLICATE_NL`. When both sides are specified with the `BROADCAST` hint or the `SHUFFLE_HASH` hint, Spark will pick the build side based on the join type and the sizes of the relations. Since a given strategy may not support all join types, Spark is not guaranteed to use the join strategy suggested by the hint.
+Hints give users a way to suggest how Spark SQL to use specific approaches to generate its execution plan.
 
 ### Syntax
 
 ```sql
-/*+ join_hint [ , ... ] */
+/*+ hint [ , ... ] */
 ```
 
-### Join Hints Types
+### Partitioning Hints
+
+Partitioning hints allow users to suggest a partitioning strategy that Spark should follow. `COALESCE`, `REPARTITION`,
+and `REPARTITION_BY_RANGE` hints are supported and are equivalent to `coalesce`, `repartition`, and
+`repartitionByRange` [Dataset APIs](api/scala/org/apache/spark/sql/Dataset.html), respectively. These hints give users
+a way to tune performance and control the number of output files in Spark SQL. When multiple partitioning hints are
+specified, multiple nodes are inserted into the logical plan, but the leftmost hint is picked by the optimizer.
+
+#### Partitioning Hints Types
+
+* **COALESCE**
+
+  The `COALESCE` hint can be used to reduce the number of partitions to the specified number of partitions. It takes a partition number as a parameter.
+
+* **REPARTITION**
+
+  The `REPARTITION` hint can be used to repartition to the specified number of partitions using the specified partitioning expressions. It takes a partition number, column names, or both as parameters.
+
+* **REPARTITION_BY_RANGE**
+
+  The `REPARTITION_BY_RANGE` hint can be used to repartition to the specified number of partitions using the specified partitioning expressions. It takes column names and an optional partition number as parameters.
+
+#### Examples
+
+```sql
+SELECT /*+ COALESCE(3) */ * FROM t;
+
+SELECT /*+ REPARTITION(3) */ * FROM t;
+
+SELECT /*+ REPARTITION(c) */ * FROM t;
+
+SELECT /*+ REPARTITION(3, c) */ * FROM t;
+
+SELECT /*+ REPARTITION_BY_RANGE(c) */ * FROM t;
+
+SELECT /*+ REPARTITION_BY_RANGE(3, c) */ * FROM t;
+
+-- multiple partitioning hints
+EXPLAIN EXTENDED SELECT /*+ REPARTITION(100), COALESCE(500), REPARTITION_BY_RANGE(3, c) */ * FROM t;
+== Parsed Logical Plan ==
+'UnresolvedHint REPARTITION, [100]
++- 'UnresolvedHint COALESCE, [500]
+   +- 'UnresolvedHint REPARTITION_BY_RANGE, [3, 'c]
+      +- 'Project [*]
+         +- 'UnresolvedRelation [t]
+
+== Analyzed Logical Plan ==
+name: string, c: int
+Repartition 100, true
++- Repartition 500, false
+   +- RepartitionByExpression [c#30 ASC NULLS FIRST], 3
+      +- Project [name#29, c#30]
+         +- SubqueryAlias spark_catalog.default.t
+            +- Relation[name#29,c#30] parquet
+
+== Optimized Logical Plan ==
+Repartition 100, true
++- Relation[name#29,c#30] parquet
+
+== Physical Plan ==
+Exchange RoundRobinPartitioning(100), false, [id=#121]
++- *(1) ColumnarToRow
+   +- FileScan parquet default.t[name#29,c#30] Batched: true, DataFilters: [], Format: Parquet,
+      Location: CatalogFileIndex[file:/spark/spark-warehouse/t], PartitionFilters: [],
+      PushedFilters: [], ReadSchema: struct<name:string>
+```
+
+### Join Hints
+
+Join hints allow users to suggest the join strategy that Spark should use. Prior to Spark 3.0, only the `BROADCAST` Join Hint was supported. `MERGE`, `SHUFFLE_HASH` and `SHUFFLE_REPLICATE_NL` Joint Hints support was added in 3.0. When different join strategy hints are specified on both sides of a join, Spark prioritizes hints in the following order: `BROADCAST` over `MERGE` over `SHUFFLE_HASH` over `SHUFFLE_REPLICATE_NL`. When both sides are specified with the `BROADCAST` hint or the `SHUFFLE_HASH` hint, Spark will pick the build side based on the join type and the sizes of the relations. Since a given strategy may not support all join types, Spark is not guaranteed to use the join strategy suggested by the hint.
+
+#### Join Hints Types
 
 * **BROADCAST**
 
@@ -47,7 +118,7 @@ Join Hints allow users to suggest the join strategy that Spark should use. Prior
 
     Suggests that Spark use shuffle-and-replicate nested loop join.
 
-### Examples
+#### Examples
 
 ```sql
 -- Join Hints for broadcast join
diff --git a/docs/sql-ref-syntax-qry-select-join.md b/docs/sql-ref-syntax-qry-select-join.md
index 28b21f5e3f0ff..09b0efd7b5751 100644
--- a/docs/sql-ref-syntax-qry-select-join.md
+++ b/docs/sql-ref-syntax-qry-select-join.md
@@ -235,4 +235,4 @@ SELECT * FROM employee ANTI JOIN department ON employee.deptno = department.dept
 ### Related Statements
 
 * [SELECT](sql-ref-syntax-qry-select.html)
-* [Join Hints](sql-ref-syntax-qry-select-hints.html)
+* [Hints](sql-ref-syntax-qry-select-hints.html)
diff --git a/docs/sql-ref-syntax-qry-select-lateral-view.md b/docs/sql-ref-syntax-qry-select-lateral-view.md
new file mode 100644
index 0000000000000..c854625a1a959
--- /dev/null
+++ b/docs/sql-ref-syntax-qry-select-lateral-view.md
@@ -0,0 +1,125 @@
+---
+layout: global
+title: LATERAL VIEW Clause
+displayTitle: LATERAL VIEW Clause
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+The `LATERAL VIEW` clause is used in conjunction with generator functions such as `EXPLODE`, which will generate a virtual table containing one or more rows. `LATERAL VIEW` will apply the rows to each original output row.
+
+### Syntax
+
+```sql
+LATERAL VIEW [ OUTER ] generator_function ( expression [ , ... ] ) [ table_alias ] AS column_alias [ , ... ]
+```
+
+### Parameters
+
+* **OUTER**
+
+    If `OUTER` specified, returns null if an input array/map is empty or null. 
+    
+* **generator_function**
+
+    Specifies a generator function (EXPLODE, INLINE, etc.).
+    
+* **table_alias**
+
+    The alias for `generator_function`, which is optional.
+     
+* **column_alias**
+
+    Lists the column aliases of `generator_function`, which may be used in output rows. We may have multiple aliases if `generator_function` have multiple output columns.
+         
+### Examples
+
+```sql
+CREATE TABLE person (id INT, name STRING, age INT, class INT, address STRING);
+INSERT INTO person VALUES
+    (100, 'John', 30, 1, 'Street 1'),
+    (200, 'Mary', NULL, 1, 'Street 2'),
+    (300, 'Mike', 80, 3, 'Street 3'),
+    (400, 'Dan', 50, 4, 'Street 4');
+
+SELECT * FROM person
+    LATERAL VIEW EXPLODE(ARRAY(30, 60)) tableName AS c_age
+    LATERAL VIEW EXPLODE(ARRAY(40, 80)) AS d_age;
++------+-------+-------+--------+-----------+--------+--------+
+|  id  | name  |  age  | class  |  address  | c_age  | d_age  |
++------+-------+-------+--------+-----------+--------+--------+
+| 100  | John  | 30    | 1      | Street 1  | 30     | 40     |
+| 100  | John  | 30    | 1      | Street 1  | 30     | 80     |
+| 100  | John  | 30    | 1      | Street 1  | 60     | 40     |
+| 100  | John  | 30    | 1      | Street 1  | 60     | 80     |
+| 200  | Mary  | NULL  | 1      | Street 2  | 30     | 40     |
+| 200  | Mary  | NULL  | 1      | Street 2  | 30     | 80     |
+| 200  | Mary  | NULL  | 1      | Street 2  | 60     | 40     |
+| 200  | Mary  | NULL  | 1      | Street 2  | 60     | 80     |
+| 300  | Mike  | 80    | 3      | Street 3  | 30     | 40     |
+| 300  | Mike  | 80    | 3      | Street 3  | 30     | 80     |
+| 300  | Mike  | 80    | 3      | Street 3  | 60     | 40     |
+| 300  | Mike  | 80    | 3      | Street 3  | 60     | 80     |
+| 400  | Dan   | 50    | 4      | Street 4  | 30     | 40     |
+| 400  | Dan   | 50    | 4      | Street 4  | 30     | 80     |
+| 400  | Dan   | 50    | 4      | Street 4  | 60     | 40     |
+| 400  | Dan   | 50    | 4      | Street 4  | 60     | 80     |
++------+-------+-------+--------+-----------+--------+--------+
+
+SELECT c_age, COUNT(1) FROM person
+    LATERAL VIEW EXPLODE(ARRAY(30, 60)) AS c_age
+    LATERAL VIEW EXPLODE(ARRAY(40, 80)) AS d_age 
+GROUP BY c_age;
++--------+-----------+
+| c_age  | count(1)  |
++--------+-----------+
+| 60     | 8         |
+| 30     | 8         |
++--------+-----------+
+
+SELECT * FROM person
+    LATERAL VIEW EXPLODE(ARRAY()) tableName AS c_age;
++-----+-------+------+--------+----------+--------+
+| id  | name  | age  | class  | address  | c_age  |
++-----+-------+------+--------+----------+--------+
++-----+-------+------+--------+----------+--------+
+
+SELECT * FROM person
+    LATERAL VIEW OUTER EXPLODE(ARRAY()) tableName AS c_age;
++------+-------+-------+--------+-----------+--------+
+|  id  | name  |  age  | class  |  address  | c_age  |
++------+-------+-------+--------+-----------+--------+
+| 100  | John  | 30    | 1      | Street 1  | NULL   |
+| 200  | Mary  | NULL  | 1      | Street 2  | NULL   |
+| 300  | Mike  | 80    | 3      | Street 3  | NULL   |
+| 400  | Dan   | 50    | 4      | Street 4  | NULL   |
++------+-------+-------+--------+-----------+--------+
+```
+
+### Related Statements
+
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
diff --git a/docs/sql-ref-syntax-qry-select-like.md b/docs/sql-ref-syntax-qry-select-like.md
index feb5eb7b3c80d..3604a9ba1ea02 100644
--- a/docs/sql-ref-syntax-qry-select-like.md
+++ b/docs/sql-ref-syntax-qry-select-like.md
@@ -21,12 +21,14 @@ license: |
 
 ### Description
 
-A LIKE predicate is used to search for a specific pattern.
+A LIKE predicate is used to search for a specific pattern. This predicate also supports multiple patterns with quantifiers include `ANY`, `SOME` and `ALL`.
 
 ### Syntax
 
 ```sql
-[ NOT ] { LIKE search_pattern [ ESCAPE esc_char ] | RLIKE regex_pattern }
+[ NOT ] { LIKE search_pattern [ ESCAPE esc_char ] | [ RLIKE | REGEXP ] regex_pattern }
+
+[ NOT ] { LIKE quantifiers ( search_pattern [ , ... ]) }
 ```
 
 ### Parameters
@@ -44,7 +46,11 @@ A LIKE predicate is used to search for a specific pattern.
 
 * **regex_pattern**
 
-    Specifies a regular expression search pattern to be searched by the `RLIKE` clause.
+    Specifies a regular expression search pattern to be searched by the `RLIKE` or `REGEXP` clause.
+    
+* **quantifiers**
+
+    Specifies the predicate quantifiers include `ANY`, `SOME` and `ALL`. `ANY` or `SOME` means if one of the patterns matches the input, then return true; `ALL` means if all the patterns matches the input, then return true.
 
 ### Examples
 
@@ -90,6 +96,14 @@ SELECT * FROM person WHERE name RLIKE 'M+';
 |200|Mary|null|
 +---+----+----+
 
+SELECT * FROM person WHERE name REGEXP 'M+';
++---+----+----+
+| id|name| age|
++---+----+----+
+|300|Mike|  80|
+|200|Mary|null|
++---+----+----+
+
 SELECT * FROM person WHERE name LIKE '%\_%';
 +---+------+---+
 | id|  name|age|
@@ -103,6 +117,58 @@ SELECT * FROM person WHERE name LIKE '%$_%' ESCAPE '$';
 +---+------+---+
 |500|Evan_W| 16|
 +---+------+---+
+
+SELECT * FROM person WHERE name LIKE ALL ('%an%', '%an');
++---+----+----+
+| id|name| age|
++---+----+----+
+|400| Dan|  50|
++---+----+----+
+
+SELECT * FROM person WHERE name LIKE ANY ('%an%', '%an');
++---+------+---+
+| id|  name|age|
++---+------+---+
+|400|   Dan| 50|
+|500|Evan_W| 16|
++---+------+---+
+
+SELECT * FROM person WHERE name LIKE SOME ('%an%', '%an');
++---+------+---+
+| id|  name|age|
++---+------+---+
+|400|   Dan| 50|
+|500|Evan_W| 16|
++---+------+---+
+
+SELECT * FROM person WHERE name NOT LIKE ALL ('%an%', '%an');
++---+----+----+
+| id|name| age|
++---+----+----+
+|100|John|  30|
+|200|Mary|null|
+|300|Mike|  80|
++---+----+----+
+
+SELECT * FROM person WHERE name NOT LIKE ANY ('%an%', '%an');
++---+------+----+
+| id|  name| age|
++---+------+----+
+|100|  John|  30|
+|200|  Mary|null|
+|300|  Mike|  80|
+|500|Evan_W|  16|
++---+------+----+
+
+SELECT * FROM person WHERE name NOT LIKE SOME ('%an%', '%an');
++---+------+----+
+| id|  name| age|
++---+------+----+
+|100|  John|  30|
+|200|  Mary|null|
+|300|  Mike|  80|
+|500|Evan_W|  16|
++---+------+----+
 ```
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-qry-select-limit.md b/docs/sql-ref-syntax-qry-select-limit.md
index 03c4df3cbc442..bd64ba890901b 100644
--- a/docs/sql-ref-syntax-qry-select-limit.md
+++ b/docs/sql-ref-syntax-qry-select-limit.md
@@ -104,3 +104,6 @@ org.apache.spark.sql.AnalysisException: The limit expression must evaluate to a
 * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
 * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
 * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-orderby.md b/docs/sql-ref-syntax-qry-select-orderby.md
index 85bbe514cdc95..552ee9be66d1e 100644
--- a/docs/sql-ref-syntax-qry-select-orderby.md
+++ b/docs/sql-ref-syntax-qry-select-orderby.md
@@ -28,7 +28,7 @@ clause, this clause guarantees a total order in the output.
 ### Syntax
 
 ```sql
-ORDER BY { expression [ sort_direction | nulls_sort_oder ] [ , ... ] }
+ORDER BY { expression [ sort_direction | nulls_sort_order ] [ , ... ] }
 ```
 
 ### Parameters
@@ -143,3 +143,6 @@ SELECT * FROM person ORDER BY name ASC, age DESC;
 * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
 * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
 * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-pivot.md b/docs/sql-ref-syntax-qry-select-pivot.md
new file mode 100644
index 0000000000000..649c2518b28eb
--- /dev/null
+++ b/docs/sql-ref-syntax-qry-select-pivot.md
@@ -0,0 +1,101 @@
+---
+layout: global
+title: PIVOT Clause
+displayTitle: PIVOT Clause
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+The `PIVOT` clause is used for data perspective. We can get the aggregated values based on specific column values, which will be turned to multiple columns used in `SELECT` clause. The `PIVOT` clause can be specified after the table name or subquery.
+
+### Syntax
+
+```sql
+PIVOT ( { aggregate_expression [ AS aggregate_expression_alias ] } [ , ... ]
+    FOR column_list IN ( expression_list ) )
+```
+
+### Parameters
+    
+* **aggregate_expression**
+
+    Specifies an aggregate expression (SUM(a), COUNT(DISTINCT b), etc.).
+    
+* **aggregate_expression_alias**
+
+    Specifies an alias for the aggregate expression.
+     
+* **column_list**
+
+    Contains columns in the `FROM` clause, which specifies the columns we want to replace with new columns. We can use brackets to surround the columns, such as `(c1, c2)`.
+      
+* **expression_list**
+
+    Specifies new columns, which are used to match values in `column_list` as the aggregating condition. We can also add aliases for them.
+    
+### Examples
+
+```sql
+CREATE TABLE person (id INT, name STRING, age INT, class INT, address STRING);
+INSERT INTO person VALUES
+    (100, 'John', 30, 1, 'Street 1'),
+    (200, 'Mary', NULL, 1, 'Street 2'),
+    (300, 'Mike', 80, 3, 'Street 3'),
+    (400, 'Dan', 50, 4, 'Street 4');
+
+SELECT * FROM person
+    PIVOT (
+        SUM(age) AS a, AVG(class) AS c
+        FOR name IN ('John' AS john, 'Mike' AS mike)
+    );
++------+-----------+---------+---------+---------+---------+
+|  id  |  address  | john_a  | john_c  | mike_a  | mike_c  |
++------+-----------+---------+---------+---------+---------+
+| 200  | Street 2  | NULL    | NULL    | NULL    | NULL    |
+| 100  | Street 1  | 30      | 1.0     | NULL    | NULL    |
+| 300  | Street 3  | NULL    | NULL    | 80      | 3.0     |
+| 400  | Street 4  | NULL    | NULL    | NULL    | NULL    |
++------+-----------+---------+---------+---------+---------+
+
+SELECT * FROM person
+    PIVOT (
+        SUM(age) AS a, AVG(class) AS c
+        FOR (name, age) IN (('John', 30) AS c1, ('Mike', 40) AS c2)
+    );
++------+-----------+-------+-------+-------+-------+
+|  id  |  address  | c1_a  | c1_c  | c2_a  | c2_c  |
++------+-----------+-------+-------+-------+-------+
+| 200  | Street 2  | NULL  | NULL  | NULL  | NULL  |
+| 100  | Street 1  | 30    | 1.0   | NULL  | NULL  |
+| 300  | Street 3  | NULL  | NULL  | NULL  | NULL  |
+| 400  | Street 4  | NULL  | NULL  | NULL  | NULL  |
++------+-----------+-------+-------+-------+-------+
+```
+
+### Related Statements
+
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-sampling.md b/docs/sql-ref-syntax-qry-select-sampling.md
similarity index 100%
rename from docs/sql-ref-syntax-qry-sampling.md
rename to docs/sql-ref-syntax-qry-select-sampling.md
diff --git a/docs/sql-ref-syntax-qry-select-sortby.md b/docs/sql-ref-syntax-qry-select-sortby.md
index 554bdb569d005..09e559adcd6ab 100644
--- a/docs/sql-ref-syntax-qry-select-sortby.md
+++ b/docs/sql-ref-syntax-qry-select-sortby.md
@@ -176,3 +176,6 @@ SELECT /*+ REPARTITION(zip_code) */ name, age, zip_code FROM person
 * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
 * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
 * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-transform.md b/docs/sql-ref-syntax-qry-select-transform.md
new file mode 100644
index 0000000000000..21966f2e1cc34
--- /dev/null
+++ b/docs/sql-ref-syntax-qry-select-transform.md
@@ -0,0 +1,191 @@
+---
+layout: global
+title: TRANSFORM
+displayTitle: TRANSFORM
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+The `TRANSFORM` clause is used to specify a Hive-style transform query specification 
+to transform the inputs by running a user-specified command or script.
+
+### Syntax
+
+```sql
+SELECT TRANSFORM ( expression [ , ... ] )
+    [ ROW FORMAT row_format ]
+    [ RECORDWRITER record_writer_class ]
+    USING command_or_script [ AS ( [ col_name [ col_type ] ] [ , ... ] ) ]
+    [ ROW FORMAT row_format ]
+    [ RECORDREADER record_reader_class ]
+```
+
+### Parameters
+
+* **expression**
+    
+    Specifies a combination of one or more values, operators and SQL functions that results in a value.
+
+* **row_format** 
+
+    Specifies the row format for input and output. See [HIVE FORMAT](sql-ref-syntax-hive-format.html) for more syntax details.
+
+* **RECORDWRITER**
+
+    Specifies a fully-qualified class name of a custom RecordWriter. The default value is `org.apache.hadoop.hive.ql.exec.TextRecordWriter`.
+
+* **RECORDREADER**
+
+    Specifies a fully-qualified class name of a custom RecordReader. The default value is `org.apache.hadoop.hive.ql.exec.TextRecordReader`.
+
+* **command_or_script**
+
+    Specifies a command or a path to script to process data.
+
+### SerDe behavior
+
+Spark uses the Hive SerDe `org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe` by default, so columns will be casted
+to `STRING` and combined by tabs before feeding to the user script. All `NULL` values will be converted
+to the literal string `"\N"` in order to differentiate `NULL` values from empty strings. The standard output of the
+user script will be treated as tab-separated `STRING` columns, any cell containing only `"\N"` will be re-interpreted
+as a `NULL` value, and then the resulting STRING column will be cast to the data type specified in `col_type`. If the actual
+number of output columns is less than the number of specified output columns, insufficient output columns will be
+supplemented with `NULL`. If the actual number of output columns is more than the number of specified output columns,
+the output columns will only select the corresponding columns and the remaining part will be discarded.
+If there is no `AS` clause after `USING my_script`, an output schema will be `key: STRING, value: STRING`.
+The `key` column contains all the characters before the first tab and the `value` column contains the remaining characters after the first tab.
+If there is no enough tab, Spark will return `NULL` value. These defaults can be overridden with `ROW FORMAT SERDE` or `ROW FORMAT DELIMITED`. 
+
+### Examples
+
+```sql
+CREATE TABLE person (zip_code INT, name STRING, age INT);
+INSERT INTO person VALUES
+    (94588, 'Zen Hui', 50),
+    (94588, 'Dan Li', 18),
+    (94588, 'Anil K', 27),
+    (94588, 'John V', NULL),
+    (94511, 'David K', 42),
+    (94511, 'Aryan B.', 18),
+    (94511, 'Lalit B.', NULL);
+
+-- With specified output without data type
+SELECT TRANSFORM(zip_code, name, age)
+   USING 'cat' AS (a, b, c)
+FROM person
+WHERE zip_code > 94511;
++-------+---------+-----+
+|    a  |        b|    c|
++-------+---------+-----+
+|  94588|   Anil K|   27|
+|  94588|   John V| NULL|
+|  94588|  Zen Hui|   50|
+|  94588|   Dan Li|   18|
++-------+---------+-----+
+
+-- With specified output with data type
+SELECT TRANSFORM(zip_code, name, age)
+   USING 'cat' AS (a STRING, b STRING, c STRING)
+FROM person
+WHERE zip_code > 94511;
++-------+---------+-----+
+|    a  |        b|    c|
++-------+---------+-----+
+|  94588|   Anil K|   27|
+|  94588|   John V| NULL|
+|  94588|  Zen Hui|   50|
+|  94588|   Dan Li|   18|
++-------+---------+-----+
+
+-- Using ROW FORMAT DELIMITED
+SELECT TRANSFORM(name, age)
+    ROW FORMAT DELIMITED
+    FIELDS TERMINATED BY ','
+    LINES TERMINATED BY '\n'
+    NULL DEFINED AS 'NULL'
+    USING 'cat' AS (name_age string)
+    ROW FORMAT DELIMITED
+    FIELDS TERMINATED BY '@'
+    LINES TERMINATED BY '\n'
+    NULL DEFINED AS 'NULL'
+FROM person;
++---------------+
+|       name_age|
++---------------+
+|      Anil K,27|
+|    John V,null|
+|     ryan B.,18|
+|     David K,42|
+|     Zen Hui,50|
+|      Dan Li,18|
+|  Lalit B.,null|
++---------------+
+
+-- Using Hive Serde
+SELECT TRANSFORM(zip_code, name, age)
+    ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+    WITH SERDEPROPERTIES (
+      'field.delim' = '\t'
+    )
+    USING 'cat' AS (a STRING, b STRING, c STRING)
+    ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+    WITH SERDEPROPERTIES (
+      'field.delim' = '\t'
+    )
+FROM person
+WHERE zip_code > 94511;
++-------+---------+-----+
+|    a  |        b|    c|
++-------+---------+-----+
+|  94588|   Anil K|   27|
+|  94588|   John V| NULL|
+|  94588|  Zen Hui|   50|
+|  94588|   Dan Li|   18|
++-------+---------+-----+
+
+-- Schema-less mode
+SELECT TRANSFORM(zip_code, name, age)
+    USING 'cat'
+FROM person
+WHERE zip_code > 94500;
++-------+---------------------+
+|    key|                value|
++-------+---------------------+
+|  94588|	  Anil K    27|
+|  94588|	  John V    \N|
+|  94511|	Aryan B.    18|
+|  94511|	 David K    42|
+|  94588|	 Zen Hui    50|
+|  94588|	  Dan Li    18|
+|  94511|	Lalit B.    \N|
++-------+---------------------+
+```
+
+### Related Statements
+
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-tvf.md b/docs/sql-ref-syntax-qry-select-tvf.md
index cc8d7c34645fb..b04e2f5c3fbd7 100644
--- a/docs/sql-ref-syntax-qry-select-tvf.md
+++ b/docs/sql-ref-syntax-qry-select-tvf.md
@@ -21,28 +21,14 @@ license: |
 
 ### Description
 
-A table-valued function (TVF) is a function that returns a relation or a set of rows.
-
-### Syntax
-
-```sql
-function_name ( expression [ , ... ] ) [ table_alias ]
-```
-
-### Parameters
-
-* **expression**
-
-    Specifies a combination of one or more values, operators and SQL functions that results in a value.
-
-* **table_alias**
-
-    Specifies a temporary name with an optional column name list.
-
-    **Syntax:** `[ AS ] table_name [ ( column_name [ , ... ] ) ]`
+A table-valued function (TVF) is a function that returns a relation or a set of rows. There are two types of TVFs in Spark SQL:
+1. a TVF that can be specified in a FROM clause, e.g. range;
+2. a TVF that can be specified in SELECT/LATERAL VIEW clauses, e.g. explode.
 
 ### Supported Table-valued Functions
 
+#### TVFs that can be specified in a FROM clause:
+
 |Function|Argument Type(s)|Description|
 |--------|----------------|-----------|
 |**range** ( *end* )|Long|Creates a table with a single *LongType* column named *id*, <br/> containing rows in a range from 0 to *end* (exclusive) with step value 1.|
@@ -50,6 +36,20 @@ function_name ( expression [ , ... ] ) [ table_alias ]
 |**range** ( *start, end, step* )|Long, Long, Long|Creates a table with a single *LongType* column named *id*, <br/> containing rows in a range from *start* to *end* (exclusive) with *step* value.|
 |**range** ( *start, end, step, numPartitions* )|Long, Long, Long, Int|Creates a table with a single *LongType* column named *id*, <br/> containing rows in a range from *start* to *end* (exclusive) with *step* value, with partition number *numPartitions* specified.|
 
+#### TVFs that can be specified in SELECT/LATERAL VIEW clauses:
+
+|Function|Argument Type(s)|Description|
+|--------|----------------|-----------|
+|**explode** ( *expr* )|Array/Map|Separates the elements of array *expr* into multiple rows, or the elements of map *expr* into multiple rows and columns. Unless specified otherwise, uses the default column name col for elements of the array or key and value for the elements of the map.|
+|**explode_outer** <br> ( *expr* )|Array/Map|Separates the elements of array *expr* into multiple rows, or the elements of map *expr* into multiple rows and columns. Unless specified otherwise, uses the default column name col for elements of the array or key and value for the elements of the map.|
+|**inline** ( *expr* )|Expression|Explodes an array of structs into a table. Uses column names col1, col2, etc. by default unless specified otherwise.|
+|**inline_outer** <br> ( *expr* )|Expression|Explodes an array of structs into a table. Uses column names col1, col2, etc. by default unless specified otherwise.|
+|**posexplode** <br> ( *expr* )|Array/Map|Separates the elements of array *expr* into multiple rows with positions, or the elements of map *expr* into multiple rows and columns with positions. Unless specified otherwise, uses the column name pos for position, col for elements of the array or key and value for elements of the map.|
+|**posexplode_outer** ( *expr* )|Array/Map|Separates the elements of array *expr* into multiple rows with positions, or the elements of map *expr* into multiple rows and columns with positions. Unless specified otherwise, uses the column name pos for position, col for elements of the array or key and value for elements of the map.|
+|**stack** ( *n, expr1, ..., exprk* )|Seq[Expression]|Separates *expr1, ..., exprk* into n rows. Uses column names col0, col1, etc. by default unless specified otherwise.|
+|**json_tuple** <br> ( *jsonStr, p1, p2, ..., pn* )|Seq[Expression]|Returns a tuple like the function *get_json_object*, but it takes multiple names. All the input parameters and output column types are string.|
+|**parse_url** <br> ( *url, partToExtract[, key]* )|Seq[Expression]|Extracts a part from a URL.|
+
 ### Examples
 
 ```sql
@@ -98,8 +98,69 @@ SELECT * FROM range(5, 8) AS test;
 |  6|
 |  7|
 +---+
+
+SELECT explode(array(10, 20));
++---+
+|col|
++---+
+| 10|
+| 20|
++---+
+
+SELECT inline(array(struct(1, 'a'), struct(2, 'b')));
++----+----+
+|col1|col2|
++----+----+
+|   1|   a|
+|   2|   b|
++----+----+
+
+SELECT posexplode(array(10,20));
++---+---+
+|pos|col|
++---+---+
+|  0| 10|
+|  1| 20|
++---+---+
+
+SELECT stack(2, 1, 2, 3);
++----+----+
+|col0|col1|
++----+----+
+|   1|   2|
+|   3|null|
++----+----+
+
+SELECT json_tuple('{"a":1, "b":2}', 'a', 'b');
++---+---+
+| c0| c1|
++---+---+
+|  1|  2|
++---+---+
+
+SELECT parse_url('http://spark.apache.org/path?query=1', 'HOST');
++-----------------------------------------------------+
+|parse_url(http://spark.apache.org/path?query=1, HOST)|
++-----------------------------------------------------+
+|                                     spark.apache.org|
++-----------------------------------------------------+
+
+-- Use explode in a LATERAL VIEW clause
+CREATE TABLE test (c1 INT);
+INSERT INTO test VALUES (1);
+INSERT INTO test VALUES (2);
+SELECT * FROM test LATERAL VIEW explode (ARRAY(3,4)) AS c2;
++--+--+
+|c1|c2|
++--+--+
+| 1| 3|
+| 1| 4|
+| 2| 3|
+| 2| 4|
++--+--+
 ```
 
 ### Related Statements
 
 * [SELECT](sql-ref-syntax-qry-select.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-where.md b/docs/sql-ref-syntax-qry-select-where.md
index ca3f5ec7866c6..9ff7993d40c58 100644
--- a/docs/sql-ref-syntax-qry-select-where.md
+++ b/docs/sql-ref-syntax-qry-select-where.md
@@ -125,3 +125,6 @@ SELECT * FROM person AS parent
 * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
 * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
 * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
\ No newline at end of file
diff --git a/docs/sql-ref-syntax-qry-window.md b/docs/sql-ref-syntax-qry-select-window.md
similarity index 100%
rename from docs/sql-ref-syntax-qry-window.md
rename to docs/sql-ref-syntax-qry-select-window.md
diff --git a/docs/sql-ref-syntax-qry-select.md b/docs/sql-ref-syntax-qry-select.md
index 1aeecdb982c4c..500eda162bf75 100644
--- a/docs/sql-ref-syntax-qry-select.md
+++ b/docs/sql-ref-syntax-qry-select.md
@@ -31,20 +31,22 @@ of a query along with examples.
 ```sql
 [ WITH with_query [ , ... ] ]
 select_statement [ { UNION | INTERSECT | EXCEPT } [ ALL | DISTINCT ] select_statement, ... ]
-    [ ORDER BY { expression [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [ , ...] } ]
-    [ SORT BY { expression [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [ , ...] } ]
-    [ CLUSTER BY { expression [ , ...] } ]
-    [ DISTRIBUTE BY { expression [, ...] } ]
+    [ ORDER BY { expression [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [ , ... ] } ]
+    [ SORT BY { expression [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [ , ... ] } ]
+    [ CLUSTER BY { expression [ , ... ] } ]
+    [ DISTRIBUTE BY { expression [, ... ] } ]
     [ WINDOW { named_window [ , WINDOW named_window, ... ] } ]
     [ LIMIT { ALL | expression } ]
 ```
 
 While `select_statement` is defined as
 ```sql
-SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
-    FROM { from_item [ , ...] }
+SELECT [ hints , ... ] [ ALL | DISTINCT ] { [ [ named_expression | regex_column_names ] [ , ... ] | TRANSFORM (...) ] }
+    FROM { from_item [ , ... ] }
+    [ PIVOT clause ]
+    [ LATERAL VIEW clause ] [ ... ] 
     [ WHERE boolean_expression ]
-    [ GROUP BY expression [ , ...] ]
+    [ GROUP BY expression [ , ... ] ]
     [ HAVING boolean_expression ]
 ```
 
@@ -75,7 +77,7 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
 
     **Syntax:** `expression [AS] [alias]`
 
- * **from_item**
+* **from_item**
 
      Specifies a source of input for the query. It can be one of the following:
      * Table relation
@@ -83,62 +85,88 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
      * [Table-value function](sql-ref-syntax-qry-select-tvf.html)
      * [Inline table](sql-ref-syntax-qry-select-inline-table.html)
      * Subquery
+     * [File](sql-ref-syntax-qry-select-file.html)
+     
+* **PIVOT**
 
+     The `PIVOT` clause is used for data perspective; We can get the aggregated values based on specific column value.
 
- * **WHERE**
+* **LATERAL VIEW**
+     
+     The `LATERAL VIEW` clause is used in conjunction with generator functions such as `EXPLODE`, which will generate a virtual table containing one or more rows. `LATERAL VIEW` will apply the rows to each original output row.
+ 
+* **WHERE**
 
      Filters the result of the FROM clause based on the supplied predicates.
 
- * **GROUP BY**
+* **GROUP BY**
 
      Specifies the expressions that are used to group the rows. This is used in conjunction with aggregate functions
      (MIN, MAX, COUNT, SUM, AVG, etc.) to group rows based on the grouping expressions and aggregate values in each group.
      When a FILTER clause is attached to an aggregate function, only the matching rows are passed to that function.
 
- * **HAVING**
+* **HAVING**
 
      Specifies the predicates by which the rows produced by GROUP BY are filtered. The HAVING clause is used to
      filter rows after the grouping is performed. If HAVING is specified without GROUP BY, it indicates a GROUP BY
      without grouping expressions (global aggregate).
 
- * **ORDER BY**
+* **ORDER BY**
 
      Specifies an ordering of the rows of the complete result set of the query. The output rows are ordered
      across the partitions. This parameter is mutually exclusive with `SORT BY`,
      `CLUSTER BY` and `DISTRIBUTE BY` and can not be specified together.
 
- * **SORT BY**
+* **SORT BY**
 
      Specifies an ordering by which the rows are ordered within each partition. This parameter is mutually
      exclusive with `ORDER BY` and `CLUSTER BY` and can not be specified together.
 
- * **CLUSTER BY**
+* **CLUSTER BY**
 
      Specifies a set of expressions that is used to repartition and sort the rows. Using this clause has
      the same effect of using `DISTRIBUTE BY` and `SORT BY` together.
 
- * **DISTRIBUTE BY**
+* **DISTRIBUTE BY**
 
      Specifies a set of expressions by which the result rows are repartitioned. This parameter is mutually
      exclusive with `ORDER BY` and `CLUSTER BY` and can not be specified together.
 
- * **LIMIT**
+* **LIMIT**
 
      Specifies the maximum number of rows that can be returned by a statement or subquery. This clause
      is mostly used in the conjunction with `ORDER BY` to produce a deterministic result.
 
- * **boolean_expression**
+* **boolean_expression**
 
-     Specifies an expression with a return type of boolean.
+     Specifies any expression that evaluates to a result type `boolean`. Two or
+     more expressions may be combined together using the logical
+     operators ( `AND`, `OR` ).
 
- * **expression**
+* **expression**
 
      Specifies a combination of one or more values, operators, and SQL functions that evaluates to a value.
 
- * **named_window**
+* **named_window**
 
-      Specifies aliases for one or more source window specifications. The source window specifications can
-      be referenced in the widow definitions in the query.
+     Specifies aliases for one or more source window specifications. The source window specifications can
+     be referenced in the widow definitions in the query.
+     
+* **regex_column_names**
+
+     When `spark.sql.parser.quotedRegexColumnNames` is true, quoted identifiers (using backticks) in `SELECT`
+     statement are interpreted as regular expressions and `SELECT` statement can take regex-based column specification.
+     For example, below SQL will only take column `c`:
+
+     ```sql
+     SELECT `(a|b)?+.+` FROM (
+       SELECT 1 as a, 2 as b, 3 as c
+     )
+     ```
+
+* **TRANSFORM**
+
+     Specifies a hive-style transform query specification to transform the input by forking and running user-specified command or script.
 
 ### Related Statements
 
@@ -151,11 +179,16 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
 * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
 * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
 * [Common Table Expression](sql-ref-syntax-qry-select-cte.html)
+* [Hints](sql-ref-syntax-qry-select-hints.html)
 * [Inline Table](sql-ref-syntax-qry-select-inline-table.html)
+* [File](sql-ref-syntax-qry-select-file.html)
 * [JOIN](sql-ref-syntax-qry-select-join.html)
-* [Join Hints](sql-ref-syntax-qry-select-hints.html)
 * [LIKE Predicate](sql-ref-syntax-qry-select-like.html)
 * [Set Operators](sql-ref-syntax-qry-select-setops.html)
-* [TABLESAMPLE](sql-ref-syntax-qry-sampling.html)
+* [TABLESAMPLE](sql-ref-syntax-qry-select-sampling.html)
 * [Table-valued Function](sql-ref-syntax-qry-select-tvf.html)
-* [Window Function](sql-ref-syntax-qry-window.html)
+* [Window Function](sql-ref-syntax-qry-select-window.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
+* [TRANSFORM Clause](sql-ref-syntax-qry-select-transform.html)
diff --git a/docs/sql-ref-syntax-qry.md b/docs/sql-ref-syntax-qry.md
index 1171fead55e30..9fb62dfd548ed 100644
--- a/docs/sql-ref-syntax-qry.md
+++ b/docs/sql-ref-syntax-qry.md
@@ -27,21 +27,27 @@ to SELECT are also included in this section. Spark also provides the
 ability to generate logical and physical plan for a given query using
 [EXPLAIN](sql-ref-syntax-qry-explain.html) statement.
 
-* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
-* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
-* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
-* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
-* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
-* [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
-* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
-* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
-* [Common Table Expression](sql-ref-syntax-qry-select-cte.html)
-* [Inline Table](sql-ref-syntax-qry-select-inline-table.html)
-* [JOIN](sql-ref-syntax-qry-select-join.html)
-* [Join Hints](sql-ref-syntax-qry-select-hints.html)
-* [LIKE Predicate](sql-ref-syntax-qry-select-like.html)
-* [Set Operators](sql-ref-syntax-qry-select-setops.html)
-* [TABLESAMPLE](sql-ref-syntax-qry-sampling.html)
-* [Table-valued Function](sql-ref-syntax-qry-select-tvf.html)
-* [Window Function](sql-ref-syntax-qry-window.html)
+* [SELECT Statement](sql-ref-syntax-qry-select.html)
+  * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+  * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+  * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+  * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+  * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+  * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+  * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+  * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+  * [Common Table Expression](sql-ref-syntax-qry-select-cte.html)
+  * [Hints](sql-ref-syntax-qry-select-hints.html)
+  * [Inline Table](sql-ref-syntax-qry-select-inline-table.html)
+  * [File](sql-ref-syntax-qry-select-file.html)
+  * [JOIN](sql-ref-syntax-qry-select-join.html)
+  * [LIKE Predicate](sql-ref-syntax-qry-select-like.html)
+  * [Set Operators](sql-ref-syntax-qry-select-setops.html)
+  * [TABLESAMPLE](sql-ref-syntax-qry-select-sampling.html)
+  * [Table-valued Function](sql-ref-syntax-qry-select-tvf.html)
+  * [Window Function](sql-ref-syntax-qry-select-window.html)
+  * [CASE Clause](sql-ref-syntax-qry-select-case.html)
+  * [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+  * [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
+  * [TRANSFORM Clause](sql-ref-syntax-qry-select-transform.html)
 * [EXPLAIN Statement](sql-ref-syntax-qry-explain.html)
diff --git a/docs/sql-ref-syntax.md b/docs/sql-ref-syntax.md
index 94bd476ffb7b1..f02b1ac08075a 100644
--- a/docs/sql-ref-syntax.md
+++ b/docs/sql-ref-syntax.md
@@ -36,7 +36,7 @@ Spark SQL is Apache Spark's module for working with structured data. The SQL Syn
  * [DROP VIEW](sql-ref-syntax-ddl-drop-view.html)
  * [REPAIR TABLE](sql-ref-syntax-ddl-repair-table.html)
  * [TRUNCATE TABLE](sql-ref-syntax-ddl-truncate-table.html)
- * [USE DATABASE](sql-ref-syntax-qry-select-usedb.html)
+ * [USE DATABASE](sql-ref-syntax-ddl-usedb.html)
 
 ### DML Statements
 
@@ -48,21 +48,37 @@ Spark SQL is Apache Spark's module for working with structured data. The SQL Syn
 
 ### Data Retrieval Statements
 
- * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
- * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+ * [SELECT Statement](sql-ref-syntax-qry-select.html)
+   * [Common Table Expression](sql-ref-syntax-qry-select-cte.html)
+   * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+   * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+   * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+   * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+   * [Hints](sql-ref-syntax-qry-select-hints.html)
+   * [Inline Table](sql-ref-syntax-qry-select-inline-table.html)
+   * [File](sql-ref-syntax-qry-select-file.html)
+   * [JOIN](sql-ref-syntax-qry-select-join.html)
+   * [LIKE Predicate](sql-ref-syntax-qry-select-like.html)
+   * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+   * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+   * [Set Operators](sql-ref-syntax-qry-select-setops.html)
+   * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+   * [TABLESAMPLE](sql-ref-syntax-qry-select-sampling.html)
+   * [Table-valued Function](sql-ref-syntax-qry-select-tvf.html)
+   * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+   * [Window Function](sql-ref-syntax-qry-select-window.html)
+   * [CASE Clause](sql-ref-syntax-qry-select-case.html)
+   * [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+   * [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
+   * [TRANSFORM Clause](sql-ref-syntax-qry-select-transform.html)
  * [EXPLAIN](sql-ref-syntax-qry-explain.html)
- * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
- * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
- * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
- * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
- * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
- * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
 
 ### Auxiliary Statements
 
  * [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
  * [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
  * [ANALYZE TABLE](sql-ref-syntax-aux-analyze-table.html)
+ * [ANALYZE TABLES](sql-ref-syntax-aux-analyze-tables.html)
  * [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
  * [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
  * [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
@@ -72,7 +88,9 @@ Spark SQL is Apache Spark's module for working with structured data. The SQL Syn
  * [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
  * [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
  * [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
- * [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
+ * [REFRESH TABLE](sql-ref-syntax-aux-cache-refresh-table.html)
+ * [REFRESH FUNCTION](sql-ref-syntax-aux-cache-refresh-function.html)
+ * [RESET](sql-ref-syntax-aux-conf-mgmt-reset.html)
  * [SET](sql-ref-syntax-aux-conf-mgmt-set.html)
  * [SHOW COLUMNS](sql-ref-syntax-aux-show-columns.html)
  * [SHOW CREATE TABLE](sql-ref-syntax-aux-show-create-table.html)
@@ -84,4 +102,3 @@ Spark SQL is Apache Spark's module for working with structured data. The SQL Syn
  * [SHOW TBLPROPERTIES](sql-ref-syntax-aux-show-tblproperties.html)
  * [SHOW VIEWS](sql-ref-syntax-aux-show-views.html)
  * [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
- * [UNSET](sql-ref-syntax-aux-conf-mgmt-reset.html)
diff --git a/docs/sql-ref.md b/docs/sql-ref.md
index db51fe1978eec..6a87166f7133d 100644
--- a/docs/sql-ref.md
+++ b/docs/sql-ref.md
@@ -1,6 +1,7 @@
 ---
 layout: global
 title: SQL Reference
+redirect_from: /sql-reference
 displayTitle: SQL Reference
 license: |
   Licensed to the Apache Software Foundation (ASF) under one or more
@@ -21,19 +22,19 @@ license: |
 
 Spark SQL is Apache Spark's module for working with structured data. This guide is a reference for Structured Query Language (SQL) and includes syntax, semantics, keywords, and examples for common SQL usage. It contains information for the following topics:
 
+ * [ANSI Compliance](sql-ref-ansi-compliance.html)
  * [Data Types](sql-ref-datatypes.html)
+ * [Datetime Pattern](sql-ref-datetime-pattern.html)
+ * [Functions](sql-ref-functions.html)
+   * [Built-in Functions](sql-ref-functions-builtin.html)
+   * [Scalar User-Defined Functions (UDFs)](sql-ref-functions-udf-scalar.html)
+   * [User-Defined Aggregate Functions (UDAFs)](sql-ref-functions-udf-aggregate.html)
+   * [Integration with Hive UDFs/UDAFs/UDTFs](sql-ref-functions-udf-hive.html)
  * [Identifiers](sql-ref-identifier.html)
  * [Literals](sql-ref-literals.html)
- * [Null Semanitics](sql-ref-null-semantics.html)
- * [ANSI Compliance](sql-ref-ansi-compliance.html)
+ * [Null Semantics](sql-ref-null-semantics.html)
  * [SQL Syntax](sql-ref-syntax.html)
    * [DDL Statements](sql-ref-syntax-ddl.html)
-   * [DML Statements](sql-ref-syntax-ddl.html)
+   * [DML Statements](sql-ref-syntax-dml.html)
    * [Data Retrieval Statements](sql-ref-syntax-qry.html)
    * [Auxiliary Statements](sql-ref-syntax-aux.html)
- * [Functions](sql-ref-functions.html)
-   * [Built-in Functions](sql-ref-functions-builtin.html)
-   * [Scalar User-Defined Functions (UDFs)](sql-ref-functions-udf-scalar.html)
-   * [User-Defined Aggregate Functions (UDAFs)](sql-ref-functions-udf-aggregate.html)
-   * [Integration with Hive UDFs/UDAFs/UDTFs](sql-ref-functions-udf-hive.html)
- * [Datetime Pattern](sql-ref-datetime-pattern.html)
diff --git a/docs/ss-migration-guide.md b/docs/ss-migration-guide.md
index 963ef07af7ace..480e5e2695a16 100644
--- a/docs/ss-migration-guide.md
+++ b/docs/ss-migration-guide.md
@@ -26,8 +26,19 @@ Note that this migration guide describes the items specific to Structured Stream
 Many items of SQL migration can be applied when migrating Structured Streaming to higher versions.
 Please refer [Migration Guide: SQL, Datasets and DataFrame](sql-migration-guide.html).
 
+## Upgrading from Structured Streaming 3.0 to 3.1
+
+- In Spark 3.0 and before, for the queries that have stateful operation which can emit rows older than the current watermark plus allowed late record delay, which are "late rows" in downstream stateful operations and these rows can be discarded, Spark only prints a warning message. Since Spark 3.1, Spark will check for such queries with possible correctness issue and throw AnalysisException for it by default. For the users who understand the possible risk of correctness issue and still decide to run the query, please disable this check by setting the config `spark.sql.streaming.statefulOperator.checkCorrectness.enabled` to false.
+
+- In Spark 3.0 and before Spark uses `KafkaConsumer` for offset fetching which could cause infinite wait in the driver.
+  In Spark 3.1 a new configuration option added `spark.sql.streaming.kafka.useDeprecatedOffsetFetching` (default: `true`)
+  which could be set to `false` allowing Spark to use new offset fetching mechanism using `AdminClient`.
+  For further details please see [Structured Streaming Kafka Integration](structured-streaming-kafka-integration.html#offset-fetching).
+
 ## Upgrading from Structured Streaming 2.4 to 3.0
 
 - In Spark 3.0, Structured Streaming forces the source schema into nullable when file-based datasources such as text, json, csv, parquet and orc are used via `spark.readStream(...)`. Previously, it respected the nullability in source schema; however, it caused issues tricky to debug with NPE. To restore the previous behavior, set `spark.sql.streaming.fileSource.schema.forceNullable` to `false`.
 
 - Spark 3.0 fixes the correctness issue on Stream-stream outer join, which changes the schema of state. (See [SPARK-26154](https://issues.apache.org/jira/browse/SPARK-26154) for more details). If you start your query from checkpoint constructed from Spark 2.x which uses stream-stream outer join, Spark 3.0 fails the query. To recalculate outputs, discard the checkpoint and replay previous inputs.
+
+- In Spark 3.0, the deprecated class `org.apache.spark.sql.streaming.ProcessingTime` has been removed. Use `org.apache.spark.sql.streaming.Trigger.ProcessingTime` instead. Likewise, `org.apache.spark.sql.execution.streaming.continuous.ContinuousTrigger` has been removed in favor of `Trigger.Continuous`, and `org.apache.spark.sql.execution.streaming.OneTimeTrigger` has been hidden in favor of `Trigger.Once`.
diff --git a/docs/streaming-kinesis-integration.md b/docs/streaming-kinesis-integration.md
index db813c46949c2..c7959d4201151 100644
--- a/docs/streaming-kinesis-integration.md
+++ b/docs/streaming-kinesis-integration.md
@@ -229,7 +229,7 @@ To run the example,
 
 - Set up Kinesis stream (see earlier section) within AWS. Note the name of the Kinesis stream and the endpoint URL corresponding to the region where the stream was created.
 
-- Set up the environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_KEY` with your AWS credentials.
+- Set up the environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` with your AWS credentials.
 
 - In the Spark root directory, run the example as
 
diff --git a/docs/streaming-programming-guide.md b/docs/streaming-programming-guide.md
index ac4aa9255ae68..56a455a1b8d21 100644
--- a/docs/streaming-programming-guide.md
+++ b/docs/streaming-programming-guide.md
@@ -1822,7 +1822,7 @@ This is shown in the following example.
 <div data-lang="scala" markdown="1">
 {% highlight scala %}
 
-object WordBlacklist {
+object WordExcludeList {
 
   @volatile private var instance: Broadcast[Seq[String]] = null
 
@@ -1830,8 +1830,8 @@ object WordBlacklist {
     if (instance == null) {
       synchronized {
         if (instance == null) {
-          val wordBlacklist = Seq("a", "b", "c")
-          instance = sc.broadcast(wordBlacklist)
+          val wordExcludeList = Seq("a", "b", "c")
+          instance = sc.broadcast(wordExcludeList)
         }
       }
     }
@@ -1847,7 +1847,7 @@ object DroppedWordsCounter {
     if (instance == null) {
       synchronized {
         if (instance == null) {
-          instance = sc.longAccumulator("WordsInBlacklistCounter")
+          instance = sc.longAccumulator("DroppedWordsCounter")
         }
       }
     }
@@ -1856,13 +1856,13 @@ object DroppedWordsCounter {
 }
 
 wordCounts.foreachRDD { (rdd: RDD[(String, Int)], time: Time) =>
-  // Get or register the blacklist Broadcast
-  val blacklist = WordBlacklist.getInstance(rdd.sparkContext)
+  // Get or register the excludeList Broadcast
+  val excludeList = WordExcludeList.getInstance(rdd.sparkContext)
   // Get or register the droppedWordsCounter Accumulator
   val droppedWordsCounter = DroppedWordsCounter.getInstance(rdd.sparkContext)
-  // Use blacklist to drop words and use droppedWordsCounter to count them
+  // Use excludeList to drop words and use droppedWordsCounter to count them
   val counts = rdd.filter { case (word, count) =>
-    if (blacklist.value.contains(word)) {
+    if (excludeList.value.contains(word)) {
       droppedWordsCounter.add(count)
       false
     } else {
@@ -1879,16 +1879,16 @@ See the full [source code]({{site.SPARK_GITHUB_URL}}/blob/v{{site.SPARK_VERSION_
 <div data-lang="java" markdown="1">
 {% highlight java %}
 
-class JavaWordBlacklist {
+class JavaWordExcludeList {
 
   private static volatile Broadcast<List<String>> instance = null;
 
   public static Broadcast<List<String>> getInstance(JavaSparkContext jsc) {
     if (instance == null) {
-      synchronized (JavaWordBlacklist.class) {
+      synchronized (JavaWordExcludeList.class) {
         if (instance == null) {
-          List<String> wordBlacklist = Arrays.asList("a", "b", "c");
-          instance = jsc.broadcast(wordBlacklist);
+          List<String> wordExcludeList = Arrays.asList("a", "b", "c");
+          instance = jsc.broadcast(wordExcludeList);
         }
       }
     }
@@ -1904,7 +1904,7 @@ class JavaDroppedWordsCounter {
     if (instance == null) {
       synchronized (JavaDroppedWordsCounter.class) {
         if (instance == null) {
-          instance = jsc.sc().longAccumulator("WordsInBlacklistCounter");
+          instance = jsc.sc().longAccumulator("DroppedWordsCounter");
         }
       }
     }
@@ -1913,13 +1913,13 @@ class JavaDroppedWordsCounter {
 }
 
 wordCounts.foreachRDD((rdd, time) -> {
-  // Get or register the blacklist Broadcast
-  Broadcast<List<String>> blacklist = JavaWordBlacklist.getInstance(new JavaSparkContext(rdd.context()));
+  // Get or register the excludeList Broadcast
+  Broadcast<List<String>> excludeList = JavaWordExcludeList.getInstance(new JavaSparkContext(rdd.context()));
   // Get or register the droppedWordsCounter Accumulator
   LongAccumulator droppedWordsCounter = JavaDroppedWordsCounter.getInstance(new JavaSparkContext(rdd.context()));
-  // Use blacklist to drop words and use droppedWordsCounter to count them
+  // Use excludeList to drop words and use droppedWordsCounter to count them
   String counts = rdd.filter(wordCount -> {
-    if (blacklist.value().contains(wordCount._1())) {
+    if (excludeList.value().contains(wordCount._1())) {
       droppedWordsCounter.add(wordCount._2());
       return false;
     } else {
@@ -1935,10 +1935,10 @@ See the full [source code]({{site.SPARK_GITHUB_URL}}/blob/v{{site.SPARK_VERSION_
 </div>
 <div data-lang="python" markdown="1">
 {% highlight python %}
-def getWordBlacklist(sparkContext):
-    if ("wordBlacklist" not in globals()):
-        globals()["wordBlacklist"] = sparkContext.broadcast(["a", "b", "c"])
-    return globals()["wordBlacklist"]
+def getWordExcludeList(sparkContext):
+    if ("wordExcludeList" not in globals()):
+        globals()["wordExcludeList"] = sparkContext.broadcast(["a", "b", "c"])
+    return globals()["wordExcludeList"]
 
 def getDroppedWordsCounter(sparkContext):
     if ("droppedWordsCounter" not in globals()):
@@ -1946,14 +1946,14 @@ def getDroppedWordsCounter(sparkContext):
     return globals()["droppedWordsCounter"]
 
 def echo(time, rdd):
-    # Get or register the blacklist Broadcast
-    blacklist = getWordBlacklist(rdd.context)
+    # Get or register the excludeList Broadcast
+    excludeList = getWordExcludeList(rdd.context)
     # Get or register the droppedWordsCounter Accumulator
     droppedWordsCounter = getDroppedWordsCounter(rdd.context)
 
-    # Use blacklist to drop words and use droppedWordsCounter to count them
+    # Use excludeList to drop words and use droppedWordsCounter to count them
     def filterFunc(wordCount):
-        if wordCount[0] in blacklist.value:
+        if wordCount[0] in excludeList.value:
             droppedWordsCounter.add(wordCount[1])
             False
         else:
@@ -2216,7 +2216,7 @@ In specific cases where the amount of data that needs to be retained for the str
 ### Task Launching Overheads
 {:.no_toc}
 If the number of tasks launched per second is high (say, 50 or more per second), then the overhead
-of sending out tasks to the slaves may be significant and will make it hard to achieve sub-second
+of sending out tasks to the executors may be significant and will make it hard to achieve sub-second
 latencies. The overhead can be reduced by the following changes:
 
 * **Execution mode**: Running Spark in Standalone mode or coarse-grained Mesos mode leads to
diff --git a/docs/structured-streaming-kafka-integration.md b/docs/structured-streaming-kafka-integration.md
index 016faa735acd6..0411b3076b556 100644
--- a/docs/structured-streaming-kafka-integration.md
+++ b/docs/structured-streaming-kafka-integration.md
@@ -61,7 +61,7 @@ val df = spark
   .option("includeHeaders", "true")
   .load()
 df.selectExpr("CAST(key AS STRING)", "CAST(value AS STRING)", "headers")
-  .as[(String, String, Map)]
+  .as[(String, String, Array[(String, Array[Byte])])]
 
 // Subscribe to multiple topics
 val df = spark
@@ -140,7 +140,7 @@ df = spark \
 df.selectExpr("CAST(key AS STRING)", "CAST(value AS STRING)")
 
 # Subscribe to 1 topic, with headers
-val df = spark \
+df = spark \
   .readStream \
   .format("kafka") \
   .option("kafka.bootstrap.servers", "host1:port1,host2:port2") \
@@ -440,9 +440,10 @@ The following configurations are optional:
 <tr>
   <td>kafkaConsumer.pollTimeoutMs</td>
   <td>long</td>
-  <td>512</td>
+  <td>120000</td>
   <td>streaming and batch</td>
-  <td>The timeout in milliseconds to poll data from Kafka in executors.</td>
+  <td>The timeout in milliseconds to poll data from Kafka in executors. When not defined it falls
+  back to <code>spark.network.timeout</code>.</td>
 </tr>
 <tr>
   <td>fetchOffset.numRetries</td>
@@ -511,6 +512,26 @@ The following configurations are optional:
 </tr>
 </table>
 
+### Offset fetching
+
+In Spark 3.0 and before Spark uses <code>KafkaConsumer</code> for offset fetching which could cause infinite wait in the driver.
+In Spark 3.1 a new configuration option added <code>spark.sql.streaming.kafka.useDeprecatedOffsetFetching</code> (default: <code>true</code>)
+which could be set to `false` allowing Spark to use new offset fetching mechanism using <code>AdminClient</code>.
+When the new mechanism used the following applies.
+
+First of all the new approach supports Kafka brokers `0.11.0.0+`.
+
+In Spark 3.0 and below, secure Kafka processing needed the following ACLs from driver perspective:
+* Topic resource describe operation
+* Topic resource read operation
+* Group resource read operation
+
+Since Spark 3.1, offsets can be obtained with <code>AdminClient</code> instead of <code>KafkaConsumer</code> and for that the following ACLs needed from driver perspective:
+* Topic resource describe operation
+
+Since <code>AdminClient</code> in driver is not connecting to consumer group, <code>group.id</code> based authorization will not work anymore (executors never done group based authorization).
+Worth to mention executor side is behaving the exact same way like before (group prefix and override works).
+
 ### Consumer Caching
 
 It's time-consuming to initialize Kafka consumers, especially in streaming scenarios where processing time is a key factor.
@@ -528,28 +549,28 @@ The following properties are available to configure the consumer pool:
 <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td>spark.kafka.consumer.cache.capacity</td>
-  <td>The maximum number of consumers cached. Please note that it's a soft limit.</td>
   <td>64</td>
+  <td>The maximum number of consumers cached. Please note that it's a soft limit.</td>
   <td>3.0.0</td>
 </tr>
 <tr>
   <td>spark.kafka.consumer.cache.timeout</td>
-  <td>The minimum amount of time a consumer may sit idle in the pool before it is eligible for eviction by the evictor.</td>
   <td>5m (5 minutes)</td>
+  <td>The minimum amount of time a consumer may sit idle in the pool before it is eligible for eviction by the evictor.</td>
   <td>3.0.0</td>
 </tr>
 <tr>
   <td>spark.kafka.consumer.cache.evictorThreadRunInterval</td>
-  <td>The interval of time between runs of the idle evictor thread for consumer pool. When non-positive, no idle evictor thread will be run.</td>
   <td>1m (1 minute)</td>
+  <td>The interval of time between runs of the idle evictor thread for consumer pool. When non-positive, no idle evictor thread will be run.</td>
   <td>3.0.0</td>
 </tr>
 <tr>
   <td>spark.kafka.consumer.cache.jmx.enable</td>
+  <td>false</td>
   <td>Enable or disable JMX for pools created with this configuration instance. Statistics of the pool are available via JMX instance.
   The prefix of JMX name is set to "kafka010-cached-simple-kafka-consumer-pool".
   </td>
-  <td>false</td>
   <td>3.0.0</td>
 </tr>
 </table>
@@ -578,14 +599,14 @@ The following properties are available to configure the fetched data pool:
 <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td>spark.kafka.consumer.fetchedData.cache.timeout</td>
-  <td>The minimum amount of time a fetched data may sit idle in the pool before it is eligible for eviction by the evictor.</td>
   <td>5m (5 minutes)</td>
+  <td>The minimum amount of time a fetched data may sit idle in the pool before it is eligible for eviction by the evictor.</td>
   <td>3.0.0</td>
 </tr>
 <tr>
   <td>spark.kafka.consumer.fetchedData.cache.evictorThreadRunInterval</td>
-  <td>The interval of time between runs of the idle evictor thread for fetched data pool. When non-positive, no idle evictor thread will be run.</td>
   <td>1m (1 minute)</td>
+  <td>The interval of time between runs of the idle evictor thread for fetched data pool. When non-positive, no idle evictor thread will be run.</td>
   <td>3.0.0</td>
 </tr>
 </table>
@@ -825,14 +846,14 @@ The following properties are available to configure the producer pool:
 <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td>spark.kafka.producer.cache.timeout</td>
-  <td>The minimum amount of time a producer may sit idle in the pool before it is eligible for eviction by the evictor.</td>
   <td>10m (10 minutes)</td>
+  <td>The minimum amount of time a producer may sit idle in the pool before it is eligible for eviction by the evictor.</td>
   <td>2.2.1</td>
 </tr>
 <tr>
   <td>spark.kafka.producer.cache.evictorThreadRunInterval</td>
-  <td>The interval of time between runs of the idle evictor thread for producer pool. When non-positive, no idle evictor thread will be run.</td>
   <td>1m (1 minute)</td>
+  <td>The interval of time between runs of the idle evictor thread for producer pool. When non-positive, no idle evictor thread will be run.</td>
   <td>3.0.0</td>
 </tr>
 </table>
@@ -857,7 +878,13 @@ group id, however, please read warnings for this option and use it with caution.
  where to start instead. Structured Streaming manages which offsets are consumed internally, rather
  than rely on the kafka Consumer to do it. This will ensure that no data is missed when new
  topics/partitions are dynamically subscribed. Note that `startingOffsets` only applies when a new
- streaming query is started, and that resuming will always pick up from where the query left off.
+ streaming query is started, and that resuming will always pick up from where the query left off. Note
+ that when the offsets consumed by a streaming application no longer exist in Kafka (e.g., topics are deleted,
+ offsets are out of range, or offsets are removed after retention period), the offsets will not be reset
+ and the streaming application will see data loss. In extreme cases, for example the throughput of the
+ streaming application cannot catch up the retention speed of Kafka, the input rows of a batch might be
+ gradually reduced until zero when the offset ranges of the batch are completely not in Kafka. Enabling
+ `failOnDataLoss` option can ask Structured Streaming to fail the query for such cases.
 - **key.deserializer**: Keys are always deserialized as byte arrays with ByteArrayDeserializer. Use
  DataFrame operations to explicitly deserialize the keys.
 - **value.deserializer**: Values are always deserialized as byte arrays with ByteArrayDeserializer.
@@ -983,6 +1010,14 @@ Delegation tokens can be obtained from multiple clusters and <code>${cluster}</c
     </td>
     <td>3.0.0</td>
   </tr>
+  <tr>
+    <td><code>spark.kafka.clusters.${cluster}.ssl.truststore.type</code></td>
+    <td>None</td>
+    <td>
+      The file format of the trust store file. For further details please see Kafka documentation. Only used to obtain delegation token.
+    </td>
+    <td>3.2.0</td>
+  </tr>
   <tr>
     <td><code>spark.kafka.clusters.${cluster}.ssl.truststore.location</code></td>
     <td>None</td>
@@ -1000,6 +1035,15 @@ Delegation tokens can be obtained from multiple clusters and <code>${cluster}</c
     </td>
     <td>3.0.0</td>
   </tr>
+  <tr>
+    <td><code>spark.kafka.clusters.${cluster}.ssl.keystore.type</code></td>
+    <td>None</td>
+    <td>
+      The file format of the key store file. This is optional for client.
+      For further details please see Kafka documentation. Only used to obtain delegation token.
+    </td>
+    <td>3.2.0</td>
+  </tr>
   <tr>
     <td><code>spark.kafka.clusters.${cluster}.ssl.keystore.location</code></td>
     <td>None</td>
diff --git a/docs/structured-streaming-programming-guide.md b/docs/structured-streaming-programming-guide.md
index 1776d23607f7f..1eabcdd2e035e 100644
--- a/docs/structured-streaming-programming-guide.md
+++ b/docs/structured-streaming-programming-guide.md
@@ -540,12 +540,13 @@ Here are the details of all the sources in Spark.
         <br/>
         <code>fileNameOnly</code>: whether to check new files based on only the filename instead of on the full path (default: false). With this set to `true`, the following files would be considered as the same file, because their filenames, "dataset.txt", are the same:
         <br/>
-        <code>maxFileAge</code>: Maximum age of a file that can be found in this directory, before it is ignored. For the first batch all files will be considered valid. If <code>latestFirst</code> is set to `true` and <code>maxFilesPerTrigger</code> is set, then this parameter will be ignored, because old files that are valid, and should be processed, may be ignored. The max age is specified with respect to the timestamp of the latest file, and not the timestamp of the current system.(default: 1 week)
-        <br/>
         "file:///dataset.txt"<br/>
         "s3://a/dataset.txt"<br/>
         "s3n://a/b/dataset.txt"<br/>
-        "s3a://a/b/c/dataset.txt"<br/>
+        "s3a://a/b/c/dataset.txt"
+        <br/>
+        <code>maxFileAge</code>: Maximum age of a file that can be found in this directory, before it is ignored. For the first batch all files will be considered valid. If <code>latestFirst</code> is set to `true` and <code>maxFilesPerTrigger</code> is set, then this parameter will be ignored, because old files that are valid, and should be processed, may be ignored. The max age is specified with respect to the timestamp of the latest file, and not the timestamp of the current system.(default: 1 week)
+        <br/>
         <code>cleanSource</code>: option to clean up completed files after processing.<br/>
         Available options are "archive", "delete", "off". If the option is not provided, the default value is "off".<br/>
         When "archive" is provided, additional option <code>sourceArchiveDir</code> must be provided as well. The value of "sourceArchiveDir" must not match with source pattern in depth (the number of directories from the root directory), where the depth is minimum of depth on both paths. This will ensure archived files are never included as new source files.<br/>
@@ -711,6 +712,8 @@ csvDF <- read.stream("csv", path = "/path/to/directory", schema = schema, sep =
 
 These examples generate streaming DataFrames that are untyped, meaning that the schema of the DataFrame is not checked at compile time, only checked at runtime when the query is submitted. Some operations like `map`, `flatMap`, etc. need the type to be known at compile time. To do those, you can convert these untyped streaming DataFrames to typed streaming Datasets using the same methods as static DataFrame. See the [SQL Programming Guide](sql-programming-guide.html) for more details. Additionally, more details on the supported streaming sources are discussed later in the document.
 
+Since Spark 3.1, you can also create streaming DataFrames from tables with `DataStreamReader.table()`. See [Streaming Table APIs](#streaming-table-apis) for more details.
+
 ### Schema inference and partition of streaming DataFrames/Datasets
 
 By default, Structured Streaming from file based sources requires you to specify the schema, rather than rely on Spark to infer it automatically. This restriction ensures a consistent schema will be used for the streaming query, even in the case of failures. For ad-hoc use cases, you can reenable schema inference by setting `spark.sql.streaming.schemaInference` to `true`.
@@ -860,6 +863,8 @@ isStreaming(df)
 </div>
 </div>
 
+You may want to check the query plan of the query, as Spark could inject stateful operations during interpret of SQL statement against streaming dataset. Once stateful operations are injected in the query plan, you may need to check your query with considerations in stateful operations. (e.g. output mode, watermark, state store size maintenance, etc.)
+
 ### Window Operations on Event Time
 Aggregations over a sliding event-time window are straightforward with Structured Streaming and are very similar to grouped aggregations. In a grouped aggregation, aggregate values (e.g. counts) are maintained for each unique value in the user-specified grouping column. In case of window-based aggregations, aggregate values are maintained for each window the event-time of a row falls into. Let's understand this with an illustration. 
 
@@ -1095,7 +1100,7 @@ likely is the engine going to process it.
 Structured Streaming supports joining a streaming Dataset/DataFrame with a static Dataset/DataFrame
 as well as another streaming Dataset/DataFrame. The result of the streaming join is generated
 incrementally, similar to the results of streaming aggregations in the previous section. In this
-section we will explore what type of joins (i.e. inner, outer, etc.) are supported in the above
+section we will explore what type of joins (i.e. inner, outer, semi, etc.) are supported in the above
 cases. Note that in all the supported join types, the result of the join with a streaming
 Dataset/DataFrame will be the exactly the same as if it was with a static Dataset/DataFrame
 containing the same data in the stream.
@@ -1114,7 +1119,7 @@ val staticDf = spark.read. ...
 val streamingDf = spark.readStream. ...
 
 streamingDf.join(staticDf, "type")          // inner equi-join with a static DF
-streamingDf.join(staticDf, "type", "right_join")  // right outer join with a static DF  
+streamingDf.join(staticDf, "type", "left_outer")  // left outer join with a static DF
 
 {% endhighlight %}
 
@@ -1125,7 +1130,7 @@ streamingDf.join(staticDf, "type", "right_join")  // right outer join with a sta
 Dataset<Row> staticDf = spark.read(). ...;
 Dataset<Row> streamingDf = spark.readStream(). ...;
 streamingDf.join(staticDf, "type");         // inner equi-join with a static DF
-streamingDf.join(staticDf, "type", "right_join");  // right outer join with a static DF
+streamingDf.join(staticDf, "type", "left_outer");  // left outer join with a static DF
 {% endhighlight %}
 
 
@@ -1136,7 +1141,7 @@ streamingDf.join(staticDf, "type", "right_join");  // right outer join with a st
 staticDf = spark.read. ...
 streamingDf = spark.readStream. ...
 streamingDf.join(staticDf, "type")  # inner equi-join with a static DF
-streamingDf.join(staticDf, "type", "right_join")  # right outer join with a static DF
+streamingDf.join(staticDf, "type", "left_outer")  # left outer join with a static DF
 {% endhighlight %}
 
 </div>
@@ -1148,10 +1153,10 @@ staticDf <- read.df(...)
 streamingDf <- read.stream(...)
 joined <- merge(streamingDf, staticDf, sort = FALSE)  # inner equi-join with a static DF
 joined <- join(
+            streamingDf,
             staticDf,
-            streamingDf, 
             streamingDf$value == staticDf$value,
-            "right_outer")  # right outer join with a static DF
+            "left_outer")  # left outer join with a static DF
 {% endhighlight %}
 
 </div>
@@ -1315,8 +1320,8 @@ A watermark delay of "2 hours" guarantees that the engine will never drop any da
  2 hours delayed. But data delayed by more than 2 hours may or may not get processed.
 
 ##### Outer Joins with Watermarking
-While the watermark + event-time constraints is optional for inner joins, for left and right outer
-joins they must be specified. This is because for generating the NULL results in outer join, the
+While the watermark + event-time constraints is optional for inner joins, for outer joins
+they must be specified. This is because for generating the NULL results in outer join, the
 engine must know when an input row is not going to match with anything in future. Hence, the
 watermark + event-time constraints must be specified for generating correct results. Therefore,
 a query with outer-join will look quite like the ad-monetization example earlier, except that
@@ -1334,7 +1339,7 @@ impressionsWithWatermark.join(
     clickTime >= impressionTime AND
     clickTime <= impressionTime + interval 1 hour
     """),
-  joinType = "leftOuter"      // can be "inner", "leftOuter", "rightOuter"
+  joinType = "leftOuter"      // can be "inner", "leftOuter", "rightOuter", "fullOuter", "leftSemi"
  )
 
 {% endhighlight %}
@@ -1349,7 +1354,7 @@ impressionsWithWatermark.join(
     "clickAdId = impressionAdId AND " +
     "clickTime >= impressionTime AND " +
     "clickTime <= impressionTime + interval 1 hour "),
-  "leftOuter"                 // can be "inner", "leftOuter", "rightOuter"
+  "leftOuter"                 // can be "inner", "leftOuter", "rightOuter", "fullOuter", "leftSemi"
 );
 
 {% endhighlight %}
@@ -1366,7 +1371,7 @@ impressionsWithWatermark.join(
     clickTime >= impressionTime AND
     clickTime <= impressionTime + interval 1 hour
     """),
-  "leftOuter"                 # can be "inner", "leftOuter", "rightOuter"
+  "leftOuter"                 # can be "inner", "leftOuter", "rightOuter", "fullOuter", "leftSemi"
 )
 
 {% endhighlight %}
@@ -1383,7 +1388,7 @@ joined <- join(
       "clickAdId = impressionAdId AND",
       "clickTime >= impressionTime AND",
       "clickTime <= impressionTime + interval 1 hour"),
-  "left_outer"                 # can be "inner", "left_outer", "right_outer"
+  "left_outer"                 # can be "inner", "left_outer", "right_outer", "full_outer", "left_semi"
 ))
 
 {% endhighlight %}
@@ -1412,6 +1417,18 @@ generation of the outer result may get delayed if there no new data being receiv
 *In short, if any of the two input streams being joined does not receive data for a while, the
 outer (both cases, left or right) output may get delayed.*
 
+##### Semi Joins with Watermarking
+A semi join returns values from the left side of the relation that has a match with the right.
+It is also referred to as a left semi join. Similar to outer joins, watermark + event-time
+constraints must be specified for semi join. This is to evict unmatched input rows on left side,
+the engine must know when an input row on left side is not going to match with anything on right
+side in future.
+
+###### Semantic Guarantees of Stream-stream Semi Joins with Watermarking
+{:.no_toc}
+Semi joins have the same guarantees as [inner joins](#semantic-guarantees-of-stream-stream-inner-joins-with-watermarking)
+regarding watermark delays and whether data will be dropped or not.
+
 ##### Support matrix for joins in streaming queries
 
 <table class ="table">
@@ -1431,8 +1448,8 @@ outer (both cases, left or right) output may get delayed.*
       </td>
   </tr>
   <tr>
-    <td rowspan="4" style="vertical-align: middle;">Stream</td>
-    <td rowspan="4" style="vertical-align: middle;">Static</td>
+    <td rowspan="5" style="vertical-align: middle;">Stream</td>
+    <td rowspan="5" style="vertical-align: middle;">Static</td>
     <td style="vertical-align: middle;">Inner</td>
     <td style="vertical-align: middle;">Supported, not stateful</td>
   </tr>
@@ -1449,8 +1466,12 @@ outer (both cases, left or right) output may get delayed.*
     <td style="vertical-align: middle;">Not supported</td>
   </tr>
   <tr>
-    <td rowspan="4" style="vertical-align: middle;">Static</td>
-    <td rowspan="4" style="vertical-align: middle;">Stream</td>
+    <td style="vertical-align: middle;">Left Semi</td>
+    <td style="vertical-align: middle;">Supported, not stateful</td>
+  </tr>
+  <tr>
+    <td rowspan="5" style="vertical-align: middle;">Static</td>
+    <td rowspan="5" style="vertical-align: middle;">Stream</td>
     <td style="vertical-align: middle;">Inner</td>
     <td style="vertical-align: middle;">Supported, not stateful</td>
   </tr>
@@ -1467,8 +1488,12 @@ outer (both cases, left or right) output may get delayed.*
     <td style="vertical-align: middle;">Not supported</td>
   </tr>
   <tr>
-    <td rowspan="4" style="vertical-align: middle;">Stream</td>
-    <td rowspan="4" style="vertical-align: middle;">Stream</td>
+    <td style="vertical-align: middle;">Left Semi</td>
+    <td style="vertical-align: middle;">Not supported</td>
+  </tr>
+  <tr>
+    <td rowspan="5" style="vertical-align: middle;">Stream</td>
+    <td rowspan="5" style="vertical-align: middle;">Stream</td>
     <td style="vertical-align: middle;">Inner</td>
     <td style="vertical-align: middle;">
       Supported, optionally specify watermark on both sides +
@@ -1491,9 +1516,19 @@ outer (both cases, left or right) output may get delayed.*
   </tr>
   <tr>
     <td style="vertical-align: middle;">Full Outer</td>
-    <td style="vertical-align: middle;">Not supported</td>
+    <td style="vertical-align: middle;">
+      Conditionally supported, must specify watermark on one side + time constraints for correct
+      results, optionally specify watermark on the other side for all state cleanup
+    </td>
+  </tr>
+  <tr>
+    <td style="vertical-align: middle;">Left Semi</td>
+    <td style="vertical-align: middle;">
+      Conditionally supported, must specify watermark on right + time constraints for correct
+      results, optionally specify watermark on left for all state cleanup
+    </td>
   </tr>
- <tr>
+  <tr>
     <td></td>
     <td></td>
     <td></td>
@@ -1674,9 +1709,40 @@ Any of the stateful operation(s) after any of below stateful operations can have
 As Spark cannot check the state function of `mapGroupsWithState`/`flatMapGroupsWithState`, Spark assumes that the state function
 emits late rows if the operator uses Append mode.
 
+Spark provides two ways to check the number of late rows on stateful operators which would help you identify the issue:
+
+1. On Spark UI: check the metrics in stateful operator nodes in query execution details page in SQL tab
+2. On Streaming Query Listener: check "numRowsDroppedByWatermark" in "stateOperators" in QueryProcessEvent.
+
+Please note that "numRowsDroppedByWatermark" represents the number of "dropped" rows by watermark, which is not always same as the count of "late input rows" for the operator.
+It depends on the implementation of the operator - e.g. streaming aggregation does pre-aggregate input rows and checks the late inputs against pre-aggregated inputs,
+hence the number is not same as the number of original input rows. You'd like to just check the fact whether the value is zero or non-zero.
+
 There's a known workaround: split your streaming query into multiple queries per stateful operator, and ensure
 end-to-end exactly once per query. Ensuring end-to-end exactly once for the last query is optional.
 
+### State Store and task locality
+
+The stateful operations store states for events in state stores of executors. State stores occupy resources such as memory and disk space to store the states.
+So it is more efficient to keep a state store provider running in the same executor across different streaming batches.
+Changing the location of a state store provider requires the extra overhead of loading checkpointed states. The overhead of loading state from checkpoint depends
+on the external storage and the size of the state, which tends to hurt the latency of micro-batch run. For some use cases such as processing very large state data,
+loading new state store providers from checkpointed states can be very time-consuming and inefficient.
+
+The stateful operations in Structured Streaming queries rely on the preferred location feature of Spark's RDD to run the state store provider on the same executor.
+If in the next batch the corresponding state store provider is scheduled on this executor again, it could reuse the previous states and save the time of loading checkpointed states.
+
+However, generally the preferred location is not a hard requirement and it is still possible that Spark schedules tasks to the executors other than the preferred ones.
+In this case, Spark will load state store providers from checkpointed states on new executors. The state store providers run in the previous batch will not be unloaded immediately.
+Spark runs a maintenance task which checks and unloads the state store providers that are inactive on the executors.
+
+By changing the Spark configurations related to task scheduling, for example `spark.locality.wait`, users can configure Spark how long to wait to launch a data-local task.
+For stateful operations in Structured Streaming, it can be used to let state store providers running on the same executors across batches.
+
+Specifically for built-in HDFS state store provider, users can check the state store metrics such as `loadedMapCacheHitCount` and `loadedMapCacheMissCount`. Ideally,
+it is best if cache missing count is minimized that means Spark won't waste too much time on loading checkpointed state.
+User can increase Spark locality waiting configurations to avoid loading state store providers in different executors across batches.
+
 ## Starting Streaming Queries
 Once you have defined the final result DataFrame/Dataset, all that is left is for you to start the streaming computation. To do that, you have to use the `DataStreamWriter`
 ([Scala](api/scala/org/apache/spark/sql/streaming/DataStreamWriter.html)/[Java](api/java/org/apache/spark/sql/streaming/DataStreamWriter.html)/[Python](api/python/pyspark.sql.html#pyspark.sql.streaming.DataStreamWriter) docs)
@@ -1751,7 +1817,9 @@ Here is the compatibility matrix.
   <tr>
     <td colspan="2" style="vertical-align: middle;">Queries with <code>mapGroupsWithState</code></td>
     <td style="vertical-align: middle;">Update</td>
-    <td style="vertical-align: middle;"></td>
+    <td style="vertical-align: middle;">
+      Aggregations not allowed in a query with <code>mapGroupsWithState</code>.
+    </td>
   </tr>
   <tr>
     <td rowspan="2" style="vertical-align: middle;">Queries with <code>flatMapGroupsWithState</code></td>
@@ -1765,7 +1833,7 @@ Here is the compatibility matrix.
     <td style="vertical-align: middle;">Update operation mode</td>
     <td style="vertical-align: middle;">Update</td>
     <td style="vertical-align: middle;">
-      Aggregations not allowed after <code>flatMapGroupsWithState</code>.
+      Aggregations not allowed in a query with <code>flatMapGroupsWithState</code>.
     </td>
   </tr>
   <tr>
@@ -1860,7 +1928,11 @@ Here are the details of all the sinks in Spark.
     <td><b>File Sink</b></td>
     <td>Append</td>
     <td>
-        <code>path</code>: path to the output directory, must be specified.
+        <code>path</code>: path to the output directory, must be specified.<br/>
+        <code>retention</code>: time to live (TTL) for output files. Output files which batches were
+        committed older than TTL will be eventually excluded in metadata log. This means reader queries which read
+        the sink's output directory may not process them. You can provide the value as string format of the time. (like "12h", "7d", etc.)
+        By default it's disabled.
         <br/><br/>
         For file-format-specific options, see the related methods in DataFrameWriter
         (<a href="api/scala/org/apache/spark/sql/DataFrameWriter.html">Scala</a>/<a href="api/java/org/apache/spark/sql/DataFrameWriter.html">Java</a>/<a href="api/python/pyspark.sql.html#pyspark.sql.DataFrameWriter">Python</a>/<a
@@ -2295,6 +2367,119 @@ When the streaming query is started, Spark calls the function or the object’s
   See [SPARK-28650](https://issues.apache.org/jira/browse/SPARK-28650) for more details.
   If you need deduplication on output, try out `foreachBatch` instead.
 
+#### Streaming Table APIs
+Since Spark 3.1, you can also use `DataStreamReader.table()` to read tables as streaming DataFrames and use `DataStreamWriter.toTable()` to write streaming DataFrames as tables:
+
+<div class="codetabs">
+<div data-lang="scala"  markdown="1">
+
+{% highlight scala %}
+val spark: SparkSession = ...
+
+// Create a streaming DataFrame
+val df = spark.readStream
+  .format("rate")
+  .option("rowsPerSecond", 10)
+  .load()
+
+// Write the streaming DataFrame to a table
+df.writeStream
+  .option("checkpointLocation", "path/to/checkpoint/dir")
+  .toTable("myTable")
+
+// Check the table result
+spark.read.table("myTable").show()
+
+// Transform the source dataset and write to a new table
+spark.readStream
+  .table("myTable")
+  .select("value")
+  .writeStream
+  .option("checkpointLocation", "path/to/checkpoint/dir")
+  .format("parquet")
+  .toTable("newTable")
+
+// Check the new table result
+spark.read.table("newTable").show()
+{% endhighlight %}
+
+</div>
+
+<div data-lang="java"  markdown="1">
+
+{% highlight java %}
+SparkSession spark = ...
+
+// Create a streaming DataFrame
+Dataset<Row> df = spark.readStream()
+  .format("rate")
+  .option("rowsPerSecond", 10)
+  .load();
+
+// Write the streaming DataFrame to a table
+df.writeStream()
+  .option("checkpointLocation", "path/to/checkpoint/dir")
+  .toTable("myTable");
+
+// Check the table result
+spark.read().table("myTable").show();
+
+// Transform the source dataset and write to a new table
+spark.readStream()
+  .table("myTable")
+  .select("value")
+  .writeStream()
+  .option("checkpointLocation", "path/to/checkpoint/dir")
+  .format("parquet")
+  .toTable("newTable");
+
+// Check the new table result
+spark.read().table("newTable").show();
+{% endhighlight %}
+
+</div>
+
+<div data-lang="python"  markdown="1">
+
+{% highlight python %}
+spark = ...  # spark session
+
+# Create a streaming DataFrame
+df = spark.readStream \
+    .format("rate") \
+    .option("rowsPerSecond", 10) \
+    .load()
+
+# Write the streaming DataFrame to a table
+df.writeStream \
+    .option("checkpointLocation", "path/to/checkpoint/dir") \
+    .toTable("myTable")
+
+# Check the table result
+spark.read.table("myTable").show()
+
+# Transform the source dataset and write to a new table
+spark.readStream \
+    .table("myTable") \
+    .select("value") \
+    .writeStream \
+    .option("checkpointLocation", "path/to/checkpoint/dir") \
+    .format("parquet") \
+    .toTable("newTable")
+
+# Check the new table result
+spark.read.table("newTable").show()
+{% endhighlight %}
+
+</div>
+
+<div data-lang="r"  markdown="1">
+Not available in R.
+</div>
+</div>
+
+For more details, please check the docs for DataStreamReader ([Scala](api/scala/org/apache/spark/sql/streaming/DataStreamReader.html)/[Java](api/java/org/apache/spark/sql/streaming/DataStreamReader.html)/[Python](api/python/pyspark.sql.html#pyspark.sql.streaming.DataStreamReader) docs) and DataStreamWriter ([Scala](api/scala/org/apache/spark/sql/streaming/DataStreamWriter.html)/[Java](api/java/org/apache/spark/sql/streaming/DataStreamWriter.html)/[Python](api/python/pyspark.sql.html#pyspark.sql.streaming.DataStreamWriter) docs).
+
 #### Triggers
 The trigger settings of a streaming query define the timing of streaming data processing, whether
 the query is going to be executed as micro-batch query with a fixed batch interval or as a continuous processing query.
@@ -3199,4 +3384,3 @@ See [Input Sources](#input-sources) and [Output Sinks](#output-sinks) sections f
 - Spark Summit 2016
   - A Deep Dive into Structured Streaming - [slides/video](https://spark-summit.org/2016/events/a-deep-dive-into-structured-streaming/)
 
-
diff --git a/docs/submitting-applications.md b/docs/submitting-applications.md
index b27cf36b863ee..831b4f8523828 100644
--- a/docs/submitting-applications.md
+++ b/docs/submitting-applications.md
@@ -76,7 +76,7 @@ locally on your laptop), it is common to use `cluster` mode to minimize network
 the drivers and the executors. Currently, the standalone mode does not support cluster mode for Python
 applications.
 
-For Python applications, simply pass a `.py` file in the place of `<application-jar>` instead of a JAR,
+For Python applications, simply pass a `.py` file in the place of `<application-jar>`,
 and add Python `.zip`, `.egg` or `.py` files to the search path with `--py-files`.
 
 There are a few options available that are specific to the
@@ -181,7 +181,7 @@ The master URL passed to Spark can be in one of the following formats:
         The cluster location will be found based on the <code>HADOOP_CONF_DIR</code> or <code>YARN_CONF_DIR</code> variable.
 </td></tr>
 <tr><td> <code>k8s://HOST:PORT</code> </td><td> Connect to a <a href="running-on-kubernetes.html">Kubernetes</a> cluster in
-        <code>cluster</code> mode. Client mode is currently unsupported and will be supported in future releases.
+        <code>client</code> or <code>cluster</code> mode depending on the value of <code>--deploy-mode</code>.
         The <code>HOST</code> and <code>PORT</code> refer to the <a href="https://kubernetes.io/docs/reference/generated/kube-apiserver/">Kubernetes API Server</a>.
         It connects using TLS by default. In order to force it to use an unsecured connection, you can use
         <code>k8s://http://HOST:PORT</code>.
diff --git a/docs/tuning.md b/docs/tuning.md
index 8e29e5d2e9e74..18d4a6205f4ff 100644
--- a/docs/tuning.md
+++ b/docs/tuning.md
@@ -264,6 +264,17 @@ parent RDD's number of partitions. You can pass the level of parallelism as a se
 or set the config property `spark.default.parallelism` to change the default.
 In general, we recommend 2-3 tasks per CPU core in your cluster.
 
+## Parallel Listing on Input Paths
+
+Sometimes you may also need to increase directory listing parallelism when job input has large number of directories,
+otherwise the process could take a very long time, especially when against object store like S3.
+If your job works on RDD with Hadoop input formats (e.g., via `SparkContext.sequenceFile`), the parallelism is
+controlled via [`spark.hadoop.mapreduce.input.fileinputformat.list-status.num-threads`](https://hadoop.apache.org/docs/current/hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml) (currently default is 1).
+
+For Spark SQL with file-based data sources, you can tune `spark.sql.sources.parallelPartitionDiscovery.threshold` and
+`spark.sql.sources.parallelPartitionDiscovery.parallelism` to improve listing parallelism. Please
+refer to [Spark SQL performance tuning guide](sql-performance-tuning.html) for more details.
+
 ## Memory Usage of Reduce Tasks
 
 Sometimes, you will get an OutOfMemoryError not because your RDDs don't fit in memory, but because the
diff --git a/docs/web-ui.md b/docs/web-ui.md
index 3c35dbeec86a2..deaf50fe5797d 100644
--- a/docs/web-ui.md
+++ b/docs/web-ui.md
@@ -407,10 +407,44 @@ Here is the list of SQL metrics:
 
 </table>
 
-## Streaming Tab
-The web UI includes a Streaming tab if the application uses Spark streaming. This tab displays
-scheduling delay and processing time for each micro-batch in the data stream, which can be useful
-for troubleshooting the streaming application.
+## Structured Streaming Tab
+When running Structured Streaming jobs in micro-batch mode, a Structured Streaming tab will be 
+available on the Web UI. The overview page displays some brief statistics for running and completed
+queries. Also, you can check the latest exception of a failed query. For detailed statistics, please
+click a "run id" in the tables.
+
+<p style="text-align: center;">
+  <img src="img/webui-structured-streaming-detail.png" title="Structured Streaming Query Statistics" alt="Structured Streaming Query Statistics">
+  <img src="img/webui-structured-streaming-detail2.png">
+</p>
+
+The statistics page displays some useful metrics for insight into the status of your streaming 
+queries. Currently, it contains the following metrics.
+
+* **Input Rate.** The aggregate (across all sources) rate of data arriving.
+* **Process Rate.** The aggregate (across all sources) rate at which Spark is processing data.
+* **Input Rows.** The aggregate (across all sources) number of records processed in a trigger.
+* **Batch Duration.** The process duration of each batch. 
+* **Operation Duration.** The amount of time taken to perform various operations in milliseconds.
+The tracked operations are listed as follows.
+    * addBatch: Time taken to read the micro-batch's input data from the sources, process it, and write the batch's output to the sink. This should take the bulk of the micro-batch's time.
+    * getBatch: Time taken to prepare the logical query to read the input of the current micro-batch from the sources.
+    * latestOffset & getOffset: Time taken to query the maximum available offset for this source.
+    * queryPlanning: Time taken to generates the execution plan.
+    * walCommit: Time taken to write the offsets to the metadata log.
+* **Global Watermark Gap.** The gap between batch timestamp and global watermark for the batch.
+* **Aggregated Number Of Total State Rows.** The aggregated number of total state rows.
+* **Aggregated Number Of Updated State Rows.** The aggregated number of updated state rows.
+* **Aggregated State Memory Used In Bytes.** The aggregated state memory used in bytes.
+* **Aggregated Number Of State Rows Dropped By Watermark.** The aggregated number of state rows dropped by watermark.
+    
+As an early-release version, the statistics page is still under development and will be improved in
+future releases.
+
+## Streaming (DStreams) Tab
+The web UI includes a Streaming tab if the application uses Spark Streaming with DStream API.
+This tab displays scheduling delay and processing time for each micro-batch in the data stream,
+which can be useful for troubleshooting the streaming application.
 
 ## JDBC/ODBC Server Tab
 We can see this tab when Spark is running as a [distributed SQL engine](sql-distributed-sql-engine.html). It shows information about sessions and submitted SQL operations.
diff --git a/examples/pom.xml b/examples/pom.xml
index c5bf2409964bb..3d7713f10402f 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
@@ -109,11 +109,6 @@
       <artifactId>scopt_${scala.binary.version}</artifactId>
       <version>3.7.1</version>
     </dependency>
-    <dependency>
-      <groupId>${hive.parquet.group}</groupId>
-      <artifactId>parquet-hadoop-bundle</artifactId>
-      <scope>provided</scope>
-    </dependency>
   </dependencies>
 
   <build>
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaANOVASelectorExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaANOVASelectorExample.java
deleted file mode 100644
index 6f24b4571b5e7..0000000000000
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaANOVASelectorExample.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.examples.ml;
-
-import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.SparkSession;
-
-// $example on$
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.spark.ml.feature.ANOVASelector;
-import org.apache.spark.ml.linalg.VectorUDT;
-import org.apache.spark.ml.linalg.Vectors;
-import org.apache.spark.sql.Row;
-import org.apache.spark.sql.RowFactory;
-import org.apache.spark.sql.types.*;
-// $example off$
-
-/**
- * An example for ANOVASelector.
- * Run with
- * <pre>
- * bin/run-example ml.JavaANOVASelectorExample
- * </pre>
- */
-public class JavaANOVASelectorExample {
-  public static void main(String[] args) {
-    SparkSession spark = SparkSession
-      .builder()
-      .appName("JavaANOVASelectorExample")
-      .getOrCreate();
-
-    // $example on$
-    List<Row> data = Arrays.asList(
-      RowFactory.create(1, Vectors.dense(1.7, 4.4, 7.6, 5.8, 9.6, 2.3), 3.0),
-      RowFactory.create(2, Vectors.dense(8.8, 7.3, 5.7, 7.3, 2.2, 4.1), 2.0),
-      RowFactory.create(3, Vectors.dense(1.2, 9.5, 2.5, 3.1, 8.7, 2.5), 3.0),
-      RowFactory.create(4, Vectors.dense(3.7, 9.2, 6.1, 4.1, 7.5, 3.8), 2.0),
-      RowFactory.create(5, Vectors.dense(8.9, 5.2, 7.8, 8.3, 5.2, 3.0), 4.0),
-      RowFactory.create(6, Vectors.dense(7.9, 8.5, 9.2, 4.0, 9.4, 2.1), 4.0)
-    );
-    StructType schema = new StructType(new StructField[]{
-      new StructField("id", DataTypes.IntegerType, false, Metadata.empty()),
-      new StructField("features", new VectorUDT(), false, Metadata.empty()),
-      new StructField("label", DataTypes.DoubleType, false, Metadata.empty())
-    });
-
-    Dataset<Row> df = spark.createDataFrame(data, schema);
-
-    ANOVASelector selector = new ANOVASelector()
-      .setNumTopFeatures(1)
-      .setFeaturesCol("features")
-      .setLabelCol("label")
-      .setOutputCol("selectedFeatures");
-
-    Dataset<Row> result = selector.fit(df).transform(df);
-
-    System.out.println("ANOVASelector output with top " + selector.getNumTopFeatures()
-        + " features selected");
-    result.show();
-
-    // $example off$
-    spark.stop();
-  }
-}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaANOVATestExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaANOVATestExample.java
deleted file mode 100644
index 4785dbd34f5d4..0000000000000
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaANOVATestExample.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.examples.ml;
-
-import org.apache.spark.sql.SparkSession;
-
-// $example on$
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.spark.ml.linalg.Vectors;
-import org.apache.spark.ml.linalg.VectorUDT;
-import org.apache.spark.ml.stat.ANOVATest;
-import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.Row;
-import org.apache.spark.sql.RowFactory;
-import org.apache.spark.sql.types.*;
-// $example off$
-
-/**
- * An example for ANOVA testing.
- * Run with
- * <pre>
- * bin/run-example ml.JavaANOVATestExample
- * </pre>
- */
-public class JavaANOVATestExample {
-
-  public static void main(String[] args) {
-    SparkSession spark = SparkSession
-      .builder()
-      .appName("JavaANOVATestExample")
-      .getOrCreate();
-
-    // $example on$
-    List<Row> data = Arrays.asList(
-      RowFactory.create(3.0, Vectors.dense(1.7, 4.4, 7.6, 5.8, 9.6, 2.3)),
-      RowFactory.create(2.0, Vectors.dense(8.8, 7.3, 5.7, 7.3, 2.2, 4.1)),
-      RowFactory.create(3.0, Vectors.dense(1.2, 9.5, 2.5, 3.1, 8.7, 2.5)),
-      RowFactory.create(2.0, Vectors.dense(3.7, 9.2, 6.1, 4.1, 7.5, 3.8)),
-      RowFactory.create(4.0, Vectors.dense(8.9, 5.2, 7.8, 8.3, 5.2, 3.0)),
-      RowFactory.create(4.0, Vectors.dense(7.9, 8.5, 9.2, 4.0, 9.4, 2.1))
-    );
-
-    StructType schema = new StructType(new StructField[]{
-      new StructField("label", DataTypes.DoubleType, false, Metadata.empty()),
-      new StructField("features", new VectorUDT(), false, Metadata.empty()),
-    });
-
-    Dataset<Row> df = spark.createDataFrame(data, schema);
-    Row r = ANOVATest.test(df, "features", "label").head();
-    System.out.println("pValues: " + r.get(0).toString());
-    System.out.println("degreesOfFreedom: " + r.getList(1).toString());
-    System.out.println("fValues: " + r.get(2).toString());
-
-    // $example off$
-
-    spark.stop();
-  }
-}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaFValueSelectorExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaFValueSelectorExample.java
deleted file mode 100644
index e8253ff0836cf..0000000000000
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaFValueSelectorExample.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.examples.ml;
-
-import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.SparkSession;
-
-// $example on$
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.spark.ml.feature.FValueSelector;
-import org.apache.spark.ml.linalg.VectorUDT;
-import org.apache.spark.ml.linalg.Vectors;
-import org.apache.spark.sql.Row;
-import org.apache.spark.sql.RowFactory;
-import org.apache.spark.sql.types.*;
-// $example off$
-
-/**
- * An example demonstrating FValueSelector.
- * Run with
- * <pre>
- * bin/run-example ml.JavaFValueSelectorExample
- * </pre>
- */
-public class JavaFValueSelectorExample {
-  public static void main(String[] args) {
-    SparkSession spark = SparkSession
-      .builder()
-      .appName("JavaFValueSelectorExample")
-      .getOrCreate();
-
-    // $example on$
-    List<Row> data = Arrays.asList(
-      RowFactory.create(1, Vectors.dense(6.0, 7.0, 0.0, 7.0, 6.0, 0.0), 4.6),
-      RowFactory.create(2, Vectors.dense(0.0, 9.0, 6.0, 0.0, 5.0, 9.0), 6.6),
-      RowFactory.create(3, Vectors.dense(0.0, 9.0, 3.0, 0.0, 5.0, 5.0), 5.1),
-      RowFactory.create(4, Vectors.dense(0.0, 9.0, 8.0, 5.0, 6.0, 4.0), 7.6),
-      RowFactory.create(5, Vectors.dense(8.0, 9.0, 6.0, 5.0, 4.0, 4.0), 9.0),
-      RowFactory.create(6, Vectors.dense(8.0, 9.0, 6.0, 4.0, 0.0, 0.0), 9.0)
-    );
-    StructType schema = new StructType(new StructField[]{
-      new StructField("id", DataTypes.IntegerType, false, Metadata.empty()),
-      new StructField("features", new VectorUDT(), false, Metadata.empty()),
-      new StructField("label", DataTypes.DoubleType, false, Metadata.empty())
-    });
-
-    Dataset<Row> df = spark.createDataFrame(data, schema);
-
-    FValueSelector selector = new FValueSelector()
-      .setNumTopFeatures(1)
-      .setFeaturesCol("features")
-      .setLabelCol("label")
-      .setOutputCol("selectedFeatures");
-
-    Dataset<Row> result = selector.fit(df).transform(df);
-
-    System.out.println("FValueSelector output with top " + selector.getNumTopFeatures()
-        + " features selected");
-    result.show();
-
-    // $example off$
-    spark.stop();
-  }
-}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaFValueTestExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaFValueTestExample.java
deleted file mode 100644
index cda28dbc7e966..0000000000000
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaFValueTestExample.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.examples.ml;
-
-import org.apache.spark.sql.SparkSession;
-
-// $example on$
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.spark.ml.linalg.Vectors;
-import org.apache.spark.ml.linalg.VectorUDT;
-import org.apache.spark.ml.stat.FValueTest;
-import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.Row;
-import org.apache.spark.sql.RowFactory;
-import org.apache.spark.sql.types.*;
-// $example off$
-
-/**
- * An example for FValue testing.
- * Run with
- * <pre>
- * bin/run-example ml.JavaFValueTestExample
- * </pre>
- */
-public class JavaFValueTestExample {
-
-  public static void main(String[] args) {
-    SparkSession spark = SparkSession
-      .builder()
-      .appName("JavaFValueTestExample")
-      .getOrCreate();
-
-    // $example on$
-    List<Row> data = Arrays.asList(
-      RowFactory.create(4.6, Vectors.dense(6.0, 7.0, 0.0, 7.0, 6.0, 0.0)),
-      RowFactory.create(6.6, Vectors.dense(0.0, 9.0, 6.0, 0.0, 5.0, 9.0)),
-      RowFactory.create(5.1, Vectors.dense(0.0, 9.0, 3.0, 0.0, 5.0, 5.0)),
-      RowFactory.create(7.6, Vectors.dense(0.0, 9.0, 8.0, 5.0, 6.0, 4.0)),
-      RowFactory.create(9.0, Vectors.dense(8.0, 9.0, 6.0, 5.0, 4.0, 4.0)),
-      RowFactory.create(9.0, Vectors.dense(8.0, 9.0, 6.0, 4.0, 0.0, 0.0))
-    );
-
-    StructType schema = new StructType(new StructField[]{
-      new StructField("label", DataTypes.DoubleType, false, Metadata.empty()),
-      new StructField("features", new VectorUDT(), false, Metadata.empty()),
-    });
-
-    Dataset<Row> df = spark.createDataFrame(data, schema);
-    Row r = FValueTest.test(df, "features", "label").head();
-    System.out.println("pValues: " + r.get(0).toString());
-    System.out.println("degreesOfFreedom: " + r.getList(1).toString());
-    System.out.println("fvalues: " + r.get(2).toString());
-
-    // $example off$
-
-    spark.stop();
-  }
-}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaModelSelectionViaRandomHyperparametersExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaModelSelectionViaRandomHyperparametersExample.java
new file mode 100644
index 0000000000000..086920f775362
--- /dev/null
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaModelSelectionViaRandomHyperparametersExample.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.examples.ml;
+
+// $example on$
+import org.apache.spark.ml.evaluation.RegressionEvaluator;
+import org.apache.spark.ml.param.ParamMap;
+import org.apache.spark.ml.regression.LinearRegression;
+import org.apache.spark.ml.tuning.*;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+// $example off$
+
+/**
+ * A simple example demonstrating model selection using ParamRandomBuilder.
+ *
+ * Run with
+ * {{{
+ * bin/run-example ml.JavaModelSelectionViaRandomHyperparametersExample
+ * }}}
+ */
+public class JavaModelSelectionViaRandomHyperparametersExample {
+
+    public static void main(String[] args) {
+        SparkSession spark = SparkSession
+                .builder()
+                .appName("JavaModelSelectionViaTrainValidationSplitExample")
+                .getOrCreate();
+
+        // $example on$
+        Dataset<Row> data = spark.read().format("libsvm")
+                .load("data/mllib/sample_linear_regression_data.txt");
+
+        LinearRegression lr = new LinearRegression();
+
+        // We sample the regularization parameter logarithmically over the range [0.01, 1.0].
+        // This means that values around 0.01, 0.1 and 1.0 are roughly equally likely.
+        // Note that both parameters must be greater than zero as otherwise we'll get an infinity.
+        // We sample the the ElasticNet mixing parameter uniformly over the range [0, 1]
+        // Note that in real life, you'd choose more than the 5 samples we see below.
+        ParamMap[] hyperparameters = new ParamRandomBuilder()
+                .addLog10Random(lr.regParam(), 0.01, 1.0, 5)
+                .addRandom(lr.elasticNetParam(), 0.0, 1.0, 5)
+                .addGrid(lr.fitIntercept())
+                .build();
+
+        System.out.println("hyperparameters:");
+        for (ParamMap param : hyperparameters) {
+            System.out.println(param);
+        }
+
+        CrossValidator cv = new CrossValidator()
+                .setEstimator(lr)
+                .setEstimatorParamMaps(hyperparameters)
+                .setEvaluator(new RegressionEvaluator())
+                .setNumFolds(3);
+        CrossValidatorModel cvModel = cv.fit(data);
+        LinearRegression parent = (LinearRegression)cvModel.bestModel().parent();
+
+        System.out.println("Optimal model has\n" + lr.regParam() + " = " + parent.getRegParam()
+                + "\n" + lr.elasticNetParam() + " = "+ parent.getElasticNetParam()
+                + "\n" + lr.fitIntercept() + " = " + parent.getFitIntercept());
+        // $example off$
+
+        spark.stop();
+    }
+}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaTokenizerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaTokenizerExample.java
index a0979aa2d24e4..091f7d111ba78 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaTokenizerExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaTokenizerExample.java
@@ -23,7 +23,7 @@
 import java.util.Arrays;
 import java.util.List;
 
-import scala.collection.mutable.WrappedArray;
+import scala.collection.mutable.Seq;
 
 import org.apache.spark.ml.feature.RegexTokenizer;
 import org.apache.spark.ml.feature.Tokenizer;
@@ -36,7 +36,7 @@
 import org.apache.spark.sql.types.StructType;
 
 // col("...") is preferable to df.col("...")
-import static org.apache.spark.sql.functions.callUDF;
+import static org.apache.spark.sql.functions.call_udf;
 import static org.apache.spark.sql.functions.col;
 // $example off$
 
@@ -69,16 +69,16 @@ public static void main(String[] args) {
         .setPattern("\\W");  // alternatively .setPattern("\\w+").setGaps(false);
 
     spark.udf().register(
-      "countTokens", (WrappedArray<?> words) -> words.size(), DataTypes.IntegerType);
+      "countTokens", (Seq<?> words) -> words.size(), DataTypes.IntegerType);
 
     Dataset<Row> tokenized = tokenizer.transform(sentenceDataFrame);
     tokenized.select("sentence", "words")
-        .withColumn("tokens", callUDF("countTokens", col("words")))
+        .withColumn("tokens", call_udf("countTokens", col("words")))
         .show(false);
 
     Dataset<Row> regexTokenized = regexTokenizer.transform(sentenceDataFrame);
     regexTokenized.select("sentence", "words")
-        .withColumn("tokens", callUDF("countTokens", col("words")))
+        .withColumn("tokens", call_udf("countTokens", col("words")))
         .show(false);
     // $example off$
 
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaUnivariateFeatureSelectorExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaUnivariateFeatureSelectorExample.java
new file mode 100644
index 0000000000000..748262f49e9ce
--- /dev/null
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaUnivariateFeatureSelectorExample.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.examples.ml;
+
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.SparkSession;
+
+// $example on$
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.spark.ml.feature.UnivariateFeatureSelector;
+import org.apache.spark.ml.linalg.VectorUDT;
+import org.apache.spark.ml.linalg.Vectors;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.RowFactory;
+import org.apache.spark.sql.types.*;
+// $example off$
+
+/**
+ * An example for UnivariateFeatureSelector.
+ * Run with
+ * <pre>
+ * bin/run-example ml.JavaUnivariateFeatureSelectorExample
+ * </pre>
+ */
+public class JavaUnivariateFeatureSelectorExample {
+  public static void main(String[] args) {
+    SparkSession spark = SparkSession
+      .builder()
+      .appName("JavaUnivariateFeatureSelectorExample")
+      .getOrCreate();
+
+    // $example on$
+    List<Row> data = Arrays.asList(
+      RowFactory.create(1, Vectors.dense(1.7, 4.4, 7.6, 5.8, 9.6, 2.3), 3.0),
+      RowFactory.create(2, Vectors.dense(8.8, 7.3, 5.7, 7.3, 2.2, 4.1), 2.0),
+      RowFactory.create(3, Vectors.dense(1.2, 9.5, 2.5, 3.1, 8.7, 2.5), 3.0),
+      RowFactory.create(4, Vectors.dense(3.7, 9.2, 6.1, 4.1, 7.5, 3.8), 2.0),
+      RowFactory.create(5, Vectors.dense(8.9, 5.2, 7.8, 8.3, 5.2, 3.0), 4.0),
+      RowFactory.create(6, Vectors.dense(7.9, 8.5, 9.2, 4.0, 9.4, 2.1), 4.0)
+    );
+    StructType schema = new StructType(new StructField[]{
+      new StructField("id", DataTypes.IntegerType, false, Metadata.empty()),
+      new StructField("features", new VectorUDT(), false, Metadata.empty()),
+      new StructField("label", DataTypes.DoubleType, false, Metadata.empty())
+    });
+
+    Dataset<Row> df = spark.createDataFrame(data, schema);
+
+    UnivariateFeatureSelector selector = new UnivariateFeatureSelector()
+      .setFeatureType("continuous")
+      .setLabelType("categorical")
+      .setSelectionMode("numTopFeatures")
+      .setSelectionThreshold(1)
+      .setFeaturesCol("features")
+      .setLabelCol("label")
+      .setOutputCol("selectedFeatures");
+
+    Dataset<Row> result = selector.fit(df).transform(df);
+
+    System.out.println("UnivariateFeatureSelector output with top "
+        + selector.getSelectionThreshold() + " features selected using f_classif");
+    result.show();
+
+    // $example off$
+    spark.stop();
+  }
+}
diff --git a/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java b/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java
index 2295225387a33..5dcf321a4c830 100644
--- a/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java
@@ -29,8 +29,12 @@
 import org.apache.spark.sql.Encoders;
 // $example on:schema_merging$
 // $example on:json_dataset$
+// $example on:csv_dataset$
+// $example on:text_dataset$
 import org.apache.spark.sql.Dataset;
 import org.apache.spark.sql.Row;
+// $example off:text_dataset$
+// $example off:csv_dataset$
 // $example off:json_dataset$
 // $example off:schema_merging$
 // $example off:basic_parquet_example$
@@ -102,6 +106,8 @@ public static void main(String[] args) {
     runBasicParquetExample(spark);
     runParquetSchemaMergingExample(spark);
     runJsonDatasetExample(spark);
+    runCsvDatasetExample(spark);
+    runTextDatasetExample(spark);
     runJdbcDatasetExample(spark);
 
     spark.stop();
@@ -147,6 +153,22 @@ private static void runGenericFileSourceOptionsExample(SparkSession spark) {
     // |file1.parquet|
     // +-------------+
     // $example off:load_with_path_glob_filter$
+    // $example on:load_with_modified_time_filter$
+    Dataset<Row> beforeFilterDF = spark.read().format("parquet")
+            // Only load files modified before 7/1/2020 at 05:30
+            .option("modifiedBefore", "2020-07-01T05:30:00")
+            // Only load files modified after 6/1/2020 at 05:30
+            .option("modifiedAfter", "2020-06-01T05:30:00")
+            // Interpret both times above relative to CST timezone
+            .option("timeZone", "CST")
+            .load("examples/src/main/resources/dir1");
+    beforeFilterDF.show();
+    // +-------------+
+    // |         file|
+    // +-------------+
+    // |file1.parquet|
+    // +-------------+
+    // $example off:load_with_modified_time_filter$
   }
 
   private static void runBasicDataSourceExample(SparkSession spark) {
@@ -173,6 +195,14 @@ private static void runBasicDataSourceExample(SparkSession spark) {
       .option("orc.column.encoding.direct", "name")
       .save("users_with_options.orc");
     // $example off:manual_save_options_orc$
+    // $example on:manual_save_options_parquet$
+    usersDF.write().format("parquet")
+        .option("parquet.bloom.filter.enabled#favorite_color", "true")
+        .option("parquet.bloom.filter.expected.ndv#favorite_color", "1000000")
+        .option("parquet.enable.dictionary", "true")
+        .option("parquet.page.write-checksum.enabled", "false")
+        .save("users_with_options.parquet");
+    // $example off:manual_save_options_parquet$
     // $example on:direct_sql$
     Dataset<Row> sqlDF =
       spark.sql("SELECT * FROM parquet.`examples/src/main/resources/users.parquet`");
@@ -188,15 +218,15 @@ private static void runBasicDataSourceExample(SparkSession spark) {
       .save("namesPartByColor.parquet");
     // $example off:write_partitioning$
     // $example on:write_partition_and_bucket$
-    peopleDF
+    usersDF
       .write()
       .partitionBy("favorite_color")
       .bucketBy(42, "name")
-      .saveAsTable("people_partitioned_bucketed");
+      .saveAsTable("users_partitioned_bucketed");
     // $example off:write_partition_and_bucket$
 
     spark.sql("DROP TABLE IF EXISTS people_bucketed");
-    spark.sql("DROP TABLE IF EXISTS people_partitioned_bucketed");
+    spark.sql("DROP TABLE IF EXISTS users_partitioned_bucketed");
   }
 
   private static void runBasicParquetExample(SparkSession spark) {
@@ -306,6 +336,117 @@ private static void runJsonDatasetExample(SparkSession spark) {
     // $example off:json_dataset$
   }
 
+  private static void runCsvDatasetExample(SparkSession spark) {
+    // $example on:csv_dataset$
+    // A CSV dataset is pointed to by path.
+    // The path can be either a single CSV file or a directory of CSV files
+    String path = "examples/src/main/resources/people.csv";
+
+    Dataset<Row> df = spark.read().csv(path);
+    df.show();
+    // +------------------+
+    // |               _c0|
+    // +------------------+
+    // |      name;age;job|
+    // |Jorge;30;Developer|
+    // |  Bob;32;Developer|
+    // +------------------+
+
+    // Read a csv with delimiter, the default delimiter is ","
+    Dataset<Row> df2 = spark.read().option("delimiter", ";").csv(path);
+    df2.show();
+    // +-----+---+---------+
+    // |  _c0|_c1|      _c2|
+    // +-----+---+---------+
+    // | name|age|      job|
+    // |Jorge| 30|Developer|
+    // |  Bob| 32|Developer|
+    // +-----+---+---------+
+
+    // Read a csv with delimiter and a header
+    Dataset<Row> df3 = spark.read().option("delimiter", ";").option("header", "true").csv(path);
+    df3.show();
+    // +-----+---+---------+
+    // | name|age|      job|
+    // +-----+---+---------+
+    // |Jorge| 30|Developer|
+    // |  Bob| 32|Developer|
+    // +-----+---+---------+
+
+    // You can also use options() to use multiple options
+    java.util.Map<String, String> optionsMap = new java.util.HashMap<String, String>();
+    optionsMap.put("delimiter",";");
+    optionsMap.put("header","true");
+    Dataset<Row> df4 = spark.read().options(optionsMap).csv(path);
+
+    // "output" is a folder which contains multiple csv files and a _SUCCESS file.
+    df3.write().csv("output");
+
+    // Read all files in a folder, please make sure only CSV files should present in the folder.
+    String folderPath = "examples/src/main/resources";
+    Dataset<Row> df5 = spark.read().csv(folderPath);
+    df5.show();
+    // Wrong schema because non-CSV files are read
+    // +-----------+
+    // |        _c0|
+    // +-----------+
+    // |238val_238|
+    // |  86val_86|
+    // |311val_311|
+    // |  27val_27|
+    // |165val_165|
+    // +-----------+
+
+    // $example off:csv_dataset$
+  }
+
+  private static void runTextDatasetExample(SparkSession spark) {
+    // $example on:text_dataset$
+    // A text dataset is pointed to by path.
+    // The path can be either a single text file or a directory of text files
+    String path = "examples/src/main/resources/people.txt";
+
+    Dataset<Row> df1 = spark.read().text(path);
+    df1.show();
+    // +-----------+
+    // |      value|
+    // +-----------+
+    // |Michael, 29|
+    // |   Andy, 30|
+    // | Justin, 19|
+    // +-----------+
+
+    // You can use 'lineSep' option to define the line separator.
+    // The line separator handles all `\r`, `\r\n` and `\n` by default.
+    Dataset<Row> df2 = spark.read().option("lineSep", ",").text(path);
+    df2.show();
+    // +-----------+
+    // |      value|
+    // +-----------+
+    // |    Michael|
+    // |   29\nAndy|
+    // | 30\nJustin|
+    // |       19\n|
+    // +-----------+
+
+    // You can also use 'wholetext' option to read each input file as a single row.
+    Dataset<Row> df3 = spark.read().option("wholetext", "true").text(path);
+    df3.show();
+    //  +--------------------+
+    //  |               value|
+    //  +--------------------+
+    //  |Michael, 29\nAndy...|
+    //  +--------------------+
+
+    // "output" is a folder which contains multiple text files and a _SUCCESS file.
+    df1.write().text("output");
+
+    // You can specify the compression format using the 'compression' option.
+    df1.write().option("compression", "gzip").text("output_compressed");
+
+    // $example off:text_dataset$
+  }
+
   private static void runJdbcDatasetExample(SparkSession spark) {
     // $example on:jdbc_dataset$
     // Note: JDBC loading and saving can be achieved via either the load/save or jdbc methods
diff --git a/examples/src/main/java/org/apache/spark/examples/sql/JavaSparkSQLExample.java b/examples/src/main/java/org/apache/spark/examples/sql/JavaSparkSQLExample.java
index 8605852d0881c..86a9045d8a2c4 100644
--- a/examples/src/main/java/org/apache/spark/examples/sql/JavaSparkSQLExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/sql/JavaSparkSQLExample.java
@@ -65,7 +65,7 @@ public class JavaSparkSQLExample {
   // $example on:create_ds$
   public static class Person implements Serializable {
     private String name;
-    private int age;
+    private long age;
 
     public String getName() {
       return name;
@@ -75,11 +75,11 @@ public void setName(String name) {
       this.name = name;
     }
 
-    public int getAge() {
+    public long getAge() {
       return age;
     }
 
-    public void setAge(int age) {
+    public void setAge(long age) {
       this.age = age;
     }
   }
@@ -225,11 +225,11 @@ private static void runDatasetCreationExample(SparkSession spark) {
     // +---+----+
 
     // Encoders for most common types are provided in class Encoders
-    Encoder<Integer> integerEncoder = Encoders.INT();
-    Dataset<Integer> primitiveDS = spark.createDataset(Arrays.asList(1, 2, 3), integerEncoder);
-    Dataset<Integer> transformedDS = primitiveDS.map(
-        (MapFunction<Integer, Integer>) value -> value + 1,
-        integerEncoder);
+    Encoder<Long> longEncoder = Encoders.LONG();
+    Dataset<Long> primitiveDS = spark.createDataset(Arrays.asList(1L, 2L, 3L), longEncoder);
+    Dataset<Long> transformedDS = primitiveDS.map(
+        (MapFunction<Long, Long>) value -> value + 1L,
+        longEncoder);
     transformedDS.collect(); // Returns [2, 3, 4]
 
     // DataFrames can be converted to a Dataset by providing a class. Mapping based on name
diff --git a/examples/src/main/java/org/apache/spark/examples/streaming/JavaCustomReceiver.java b/examples/src/main/java/org/apache/spark/examples/streaming/JavaCustomReceiver.java
index 47692ec982890..f84a1978de1ad 100644
--- a/examples/src/main/java/org/apache/spark/examples/streaming/JavaCustomReceiver.java
+++ b/examples/src/main/java/org/apache/spark/examples/streaming/JavaCustomReceiver.java
@@ -67,7 +67,7 @@ public static void main(String[] args) throws Exception {
     JavaStreamingContext ssc = new JavaStreamingContext(sparkConf, new Duration(1000));
 
     // Create an input stream with the custom receiver on target ip:port and count the
-    // words in input stream of \n delimited text (eg. generated by 'nc')
+    // words in input stream of \n delimited text (e.g. generated by 'nc')
     JavaReceiverInputDStream<String> lines = ssc.receiverStream(
       new JavaCustomReceiver(args[0], Integer.parseInt(args[1])));
     JavaDStream<String> words = lines.flatMap(x -> Arrays.asList(SPACE.split(x)).iterator());
diff --git a/examples/src/main/java/org/apache/spark/examples/streaming/JavaNetworkWordCount.java b/examples/src/main/java/org/apache/spark/examples/streaming/JavaNetworkWordCount.java
index b217672def88e..d56134bd99e36 100644
--- a/examples/src/main/java/org/apache/spark/examples/streaming/JavaNetworkWordCount.java
+++ b/examples/src/main/java/org/apache/spark/examples/streaming/JavaNetworkWordCount.java
@@ -57,7 +57,7 @@ public static void main(String[] args) throws Exception {
     JavaStreamingContext ssc = new JavaStreamingContext(sparkConf, Durations.seconds(1));
 
     // Create a JavaReceiverInputDStream on target ip:port and count the
-    // words in input stream of \n delimited text (eg. generated by 'nc')
+    // words in input stream of \n delimited text (e.g. generated by 'nc')
     // Note that no duplication in storage level only for running locally.
     // Replication necessary in distributed scenario for fault tolerance.
     JavaReceiverInputDStream<String> lines = ssc.socketTextStream(
diff --git a/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecoverableNetworkWordCount.java b/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecoverableNetworkWordCount.java
index 45a876decff8b..0c11c40cfe7ed 100644
--- a/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecoverableNetworkWordCount.java
+++ b/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecoverableNetworkWordCount.java
@@ -41,16 +41,16 @@
 /**
  * Use this singleton to get or register a Broadcast variable.
  */
-class JavaWordBlacklist {
+class JavaWordExcludeList {
 
   private static volatile Broadcast<List<String>> instance = null;
 
   public static Broadcast<List<String>> getInstance(JavaSparkContext jsc) {
     if (instance == null) {
-      synchronized (JavaWordBlacklist.class) {
+      synchronized (JavaWordExcludeList.class) {
         if (instance == null) {
-          List<String> wordBlacklist = Arrays.asList("a", "b", "c");
-          instance = jsc.broadcast(wordBlacklist);
+          List<String> wordExcludeList = Arrays.asList("a", "b", "c");
+          instance = jsc.broadcast(wordExcludeList);
         }
       }
     }
@@ -69,7 +69,7 @@ public static LongAccumulator getInstance(JavaSparkContext jsc) {
     if (instance == null) {
       synchronized (JavaDroppedWordsCounter.class) {
         if (instance == null) {
-          instance = jsc.sc().longAccumulator("WordsInBlacklistCounter");
+          instance = jsc.sc().longAccumulator("DroppedWordsCounter");
         }
       }
     }
@@ -126,22 +126,22 @@ private static JavaStreamingContext createContext(String ip,
     ssc.checkpoint(checkpointDirectory);
 
     // Create a socket stream on target ip:port and count the
-    // words in input stream of \n delimited text (eg. generated by 'nc')
+    // words in input stream of \n delimited text (e.g. generated by 'nc')
     JavaReceiverInputDStream<String> lines = ssc.socketTextStream(ip, port);
     JavaDStream<String> words = lines.flatMap(x -> Arrays.asList(SPACE.split(x)).iterator());
     JavaPairDStream<String, Integer> wordCounts = words.mapToPair(s -> new Tuple2<>(s, 1))
         .reduceByKey((i1, i2) -> i1 + i2);
 
     wordCounts.foreachRDD((rdd, time) -> {
-      // Get or register the blacklist Broadcast
-      Broadcast<List<String>> blacklist =
-          JavaWordBlacklist.getInstance(new JavaSparkContext(rdd.context()));
+      // Get or register the excludeList Broadcast
+      Broadcast<List<String>> excludeList =
+          JavaWordExcludeList.getInstance(new JavaSparkContext(rdd.context()));
       // Get or register the droppedWordsCounter Accumulator
       LongAccumulator droppedWordsCounter =
           JavaDroppedWordsCounter.getInstance(new JavaSparkContext(rdd.context()));
-      // Use blacklist to drop words and use droppedWordsCounter to count them
+      // Use excludeList to drop words and use droppedWordsCounter to count them
       String counts = rdd.filter(wordCount -> {
-        if (blacklist.value().contains(wordCount._1())) {
+        if (excludeList.value().contains(wordCount._1())) {
           droppedWordsCounter.add(wordCount._2());
           return false;
         } else {
diff --git a/examples/src/main/java/org/apache/spark/examples/streaming/JavaSqlNetworkWordCount.java b/examples/src/main/java/org/apache/spark/examples/streaming/JavaSqlNetworkWordCount.java
index 948d1a2111780..5d30698c93372 100644
--- a/examples/src/main/java/org/apache/spark/examples/streaming/JavaSqlNetworkWordCount.java
+++ b/examples/src/main/java/org/apache/spark/examples/streaming/JavaSqlNetworkWordCount.java
@@ -59,7 +59,7 @@ public static void main(String[] args) throws Exception {
     JavaStreamingContext ssc = new JavaStreamingContext(sparkConf, Durations.seconds(1));
 
     // Create a JavaReceiverInputDStream on target ip:port and count the
-    // words in input stream of \n delimited text (eg. generated by 'nc')
+    // words in input stream of \n delimited text (e.g. generated by 'nc')
     // Note that no duplication in storage level only for running locally.
     // Replication necessary in distributed scenario for fault tolerance.
     JavaReceiverInputDStream<String> lines = ssc.socketTextStream(
diff --git a/examples/src/main/python/als.py b/examples/src/main/python/als.py
index 6d3241876ad51..511634fd8f6c2 100755
--- a/examples/src/main/python/als.py
+++ b/examples/src/main/python/als.py
@@ -21,8 +21,6 @@
 
 This example requires numpy (http://www.numpy.org/)
 """
-from __future__ import print_function
-
 import sys
 
 import numpy as np
diff --git a/examples/src/main/python/avro_inputformat.py b/examples/src/main/python/avro_inputformat.py
index a18722c687f8b..49ab37e7b3286 100644
--- a/examples/src/main/python/avro_inputformat.py
+++ b/examples/src/main/python/avro_inputformat.py
@@ -43,8 +43,6 @@
 {u'favorite_color': None, u'name': u'Alyssa'}
 {u'favorite_color': u'red', u'name': u'Ben'}
 """
-from __future__ import print_function
-
 import sys
 
 from functools import reduce
diff --git a/examples/src/main/python/kmeans.py b/examples/src/main/python/kmeans.py
index a42d711fc505f..022378619c97f 100755
--- a/examples/src/main/python/kmeans.py
+++ b/examples/src/main/python/kmeans.py
@@ -22,8 +22,6 @@
 
 This example requires NumPy (http://www.numpy.org/).
 """
-from __future__ import print_function
-
 import sys
 
 import numpy as np
diff --git a/examples/src/main/python/logistic_regression.py b/examples/src/main/python/logistic_regression.py
index bcc4e0f4e8eae..4b83740152ca4 100755
--- a/examples/src/main/python/logistic_regression.py
+++ b/examples/src/main/python/logistic_regression.py
@@ -22,8 +22,6 @@
 In practice, one may prefer to use the LogisticRegression algorithm in
 ML, as shown in examples/src/main/python/ml/logistic_regression_with_elastic_net.py.
 """
-from __future__ import print_function
-
 import sys
 
 import numpy as np
diff --git a/examples/src/main/python/ml/aft_survival_regression.py b/examples/src/main/python/ml/aft_survival_regression.py
index 0a71f76418ea6..2040a7876c7fa 100644
--- a/examples/src/main/python/ml/aft_survival_regression.py
+++ b/examples/src/main/python/ml/aft_survival_regression.py
@@ -20,8 +20,6 @@
 Run with:
   bin/spark-submit examples/src/main/python/ml/aft_survival_regression.py
 """
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.regression import AFTSurvivalRegression
 from pyspark.ml.linalg import Vectors
diff --git a/examples/src/main/python/ml/als_example.py b/examples/src/main/python/ml/als_example.py
index 8b7ec9c439f9f..b39263978402b 100644
--- a/examples/src/main/python/ml/als_example.py
+++ b/examples/src/main/python/ml/als_example.py
@@ -15,12 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
-import sys
-if sys.version >= '3':
-    long = int
-
 from pyspark.sql import SparkSession
 
 # $example on$
@@ -39,7 +33,7 @@
     lines = spark.read.text("data/mllib/als/sample_movielens_ratings.txt").rdd
     parts = lines.map(lambda row: row.value.split("::"))
     ratingsRDD = parts.map(lambda p: Row(userId=int(p[0]), movieId=int(p[1]),
-                                         rating=float(p[2]), timestamp=long(p[3])))
+                                         rating=float(p[2]), timestamp=int(p[3])))
     ratings = spark.createDataFrame(ratingsRDD)
     (training, test) = ratings.randomSplit([0.8, 0.2])
 
diff --git a/examples/src/main/python/ml/anova_selector_example.py b/examples/src/main/python/ml/anova_selector_example.py
deleted file mode 100644
index f8458f5d6e487..0000000000000
--- a/examples/src/main/python/ml/anova_selector_example.py
+++ /dev/null
@@ -1,55 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""
-An example for ANOVASelector.
-Run with:
-  bin/spark-submit examples/src/main/python/ml/anova_selector_example.py
-"""
-from __future__ import print_function
-
-from pyspark.sql import SparkSession
-# $example on$
-from pyspark.ml.feature import ANOVASelector
-from pyspark.ml.linalg import Vectors
-# $example off$
-
-if __name__ == "__main__":
-    spark = SparkSession\
-        .builder\
-        .appName("ANOVASelectorExample")\
-        .getOrCreate()
-
-    # $example on$
-    df = spark.createDataFrame([
-        (1, Vectors.dense([1.7, 4.4, 7.6, 5.8, 9.6, 2.3]), 3.0,),
-        (2, Vectors.dense([8.8, 7.3, 5.7, 7.3, 2.2, 4.1]), 2.0,),
-        (3, Vectors.dense([1.2, 9.5, 2.5, 3.1, 8.7, 2.5]), 3.0,),
-        (4, Vectors.dense([3.7, 9.2, 6.1, 4.1, 7.5, 3.8]), 2.0,),
-        (5, Vectors.dense([8.9, 5.2, 7.8, 8.3, 5.2, 3.0]), 4.0,),
-        (6, Vectors.dense([7.9, 8.5, 9.2, 4.0, 9.4, 2.1]), 4.0,)], ["id", "features", "label"])
-
-    selector = ANOVASelector(numTopFeatures=1, featuresCol="features",
-                             outputCol="selectedFeatures", labelCol="label")
-
-    result = selector.fit(df).transform(df)
-
-    print("ANOVASelector output with top %d features selected" % selector.getNumTopFeatures())
-    result.show()
-    # $example off$
-
-    spark.stop()
diff --git a/examples/src/main/python/ml/anova_test_example.py b/examples/src/main/python/ml/anova_test_example.py
deleted file mode 100644
index 4119441cdeab6..0000000000000
--- a/examples/src/main/python/ml/anova_test_example.py
+++ /dev/null
@@ -1,52 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""
-An example for ANOVA testing.
-Run with:
-  bin/spark-submit examples/src/main/python/ml/anova_test_example.py
-"""
-from __future__ import print_function
-
-from pyspark.sql import SparkSession
-# $example on$
-from pyspark.ml.linalg import Vectors
-from pyspark.ml.stat import ANOVATest
-# $example off$
-
-if __name__ == "__main__":
-    spark = SparkSession\
-        .builder\
-        .appName("ANOVATestExample")\
-        .getOrCreate()
-
-    # $example on$
-    data = [(3.0, Vectors.dense([1.7, 4.4, 7.6, 5.8, 9.6, 2.3])),
-            (2.0, Vectors.dense([8.8, 7.3, 5.7, 7.3, 2.2, 4.1])),
-            (3.0, Vectors.dense([1.2, 9.5, 2.5, 3.1, 8.7, 2.5])),
-            (2.0, Vectors.dense([3.7, 9.2, 6.1, 4.1, 7.5, 3.8])),
-            (4.0, Vectors.dense([8.9, 5.2, 7.8, 8.3, 5.2, 3.0])),
-            (4.0, Vectors.dense([7.9, 8.5, 9.2, 4.0, 9.4, 2.1]))]
-    df = spark.createDataFrame(data, ["label", "features"])
-
-    r = ANOVATest.test(df, "features", "label").head()
-    print("pValues: " + str(r.pValues))
-    print("degreesOfFreedom: " + str(r.degreesOfFreedom))
-    print("fValues: " + str(r.fValues))
-    # $example off$
-
-    spark.stop()
diff --git a/examples/src/main/python/ml/binarizer_example.py b/examples/src/main/python/ml/binarizer_example.py
index 669bb2aeabecd..5d5ae4122e1d4 100644
--- a/examples/src/main/python/ml/binarizer_example.py
+++ b/examples/src/main/python/ml/binarizer_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 from pyspark.sql import SparkSession
 # $example on$
 from pyspark.ml.feature import Binarizer
diff --git a/examples/src/main/python/ml/bisecting_k_means_example.py b/examples/src/main/python/ml/bisecting_k_means_example.py
index 82adb338b5d91..513f80a09ef05 100644
--- a/examples/src/main/python/ml/bisecting_k_means_example.py
+++ b/examples/src/main/python/ml/bisecting_k_means_example.py
@@ -20,8 +20,6 @@
 Run with:
   bin/spark-submit examples/src/main/python/ml/bisecting_k_means_example.py
 """
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.clustering import BisectingKMeans
 from pyspark.ml.evaluation import ClusteringEvaluator
diff --git a/examples/src/main/python/ml/bucketed_random_projection_lsh_example.py b/examples/src/main/python/ml/bucketed_random_projection_lsh_example.py
index 610176ea596ca..f5836091f35ba 100644
--- a/examples/src/main/python/ml/bucketed_random_projection_lsh_example.py
+++ b/examples/src/main/python/ml/bucketed_random_projection_lsh_example.py
@@ -20,8 +20,6 @@
 Run with:
   bin/spark-submit examples/src/main/python/ml/bucketed_random_projection_lsh_example.py
 """
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.feature import BucketedRandomProjectionLSH
 from pyspark.ml.linalg import Vectors
diff --git a/examples/src/main/python/ml/bucketizer_example.py b/examples/src/main/python/ml/bucketizer_example.py
index 742f35093b9d2..5de67f7126b5e 100644
--- a/examples/src/main/python/ml/bucketizer_example.py
+++ b/examples/src/main/python/ml/bucketizer_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 from pyspark.sql import SparkSession
 # $example on$
 from pyspark.ml.feature import Bucketizer
diff --git a/examples/src/main/python/ml/chi_square_test_example.py b/examples/src/main/python/ml/chi_square_test_example.py
index 2af7e683cdb72..bf15a03d9cb4c 100644
--- a/examples/src/main/python/ml/chi_square_test_example.py
+++ b/examples/src/main/python/ml/chi_square_test_example.py
@@ -20,8 +20,6 @@
 Run with:
   bin/spark-submit examples/src/main/python/ml/chi_square_test_example.py
 """
-from __future__ import print_function
-
 from pyspark.sql import SparkSession
 # $example on$
 from pyspark.ml.linalg import Vectors
diff --git a/examples/src/main/python/ml/chisq_selector_example.py b/examples/src/main/python/ml/chisq_selector_example.py
index 028a9ea9d67b1..c83a8c1bc7b27 100644
--- a/examples/src/main/python/ml/chisq_selector_example.py
+++ b/examples/src/main/python/ml/chisq_selector_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 from pyspark.sql import SparkSession
 # $example on$
 from pyspark.ml.feature import ChiSqSelector
diff --git a/examples/src/main/python/ml/correlation_example.py b/examples/src/main/python/ml/correlation_example.py
index 1f4e402ac1a51..9006d541491fb 100644
--- a/examples/src/main/python/ml/correlation_example.py
+++ b/examples/src/main/python/ml/correlation_example.py
@@ -20,8 +20,6 @@
 Run with:
   bin/spark-submit examples/src/main/python/ml/correlation_example.py
 """
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.linalg import Vectors
 from pyspark.ml.stat import Correlation
diff --git a/examples/src/main/python/ml/count_vectorizer_example.py b/examples/src/main/python/ml/count_vectorizer_example.py
index f2e41db77d898..b3ddfb128c3d0 100644
--- a/examples/src/main/python/ml/count_vectorizer_example.py
+++ b/examples/src/main/python/ml/count_vectorizer_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 from pyspark.sql import SparkSession
 # $example on$
 from pyspark.ml.feature import CountVectorizer
diff --git a/examples/src/main/python/ml/cross_validator.py b/examples/src/main/python/ml/cross_validator.py
index 6256d11504afb..0ad0865486959 100644
--- a/examples/src/main/python/ml/cross_validator.py
+++ b/examples/src/main/python/ml/cross_validator.py
@@ -22,8 +22,6 @@
 
   bin/spark-submit examples/src/main/python/ml/cross_validator.py
 """
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml import Pipeline
 from pyspark.ml.classification import LogisticRegression
diff --git a/examples/src/main/python/ml/dataframe_example.py b/examples/src/main/python/ml/dataframe_example.py
index cabc3de68f2f4..d2bf93744113b 100644
--- a/examples/src/main/python/ml/dataframe_example.py
+++ b/examples/src/main/python/ml/dataframe_example.py
@@ -19,8 +19,6 @@
 An example of how to use DataFrame for ML. Run with::
     bin/spark-submit examples/src/main/python/ml/dataframe_example.py <input_path>
 """
-from __future__ import print_function
-
 import os
 import sys
 import tempfile
diff --git a/examples/src/main/python/ml/dct_example.py b/examples/src/main/python/ml/dct_example.py
index c0457f8d0f43b..37da4f5e8f1cb 100644
--- a/examples/src/main/python/ml/dct_example.py
+++ b/examples/src/main/python/ml/dct_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.feature import DCT
 from pyspark.ml.linalg import Vectors
diff --git a/examples/src/main/python/ml/decision_tree_classification_example.py b/examples/src/main/python/ml/decision_tree_classification_example.py
index d6e2977de0082..eb7177b845357 100644
--- a/examples/src/main/python/ml/decision_tree_classification_example.py
+++ b/examples/src/main/python/ml/decision_tree_classification_example.py
@@ -18,8 +18,6 @@
 """
 Decision Tree Classification Example.
 """
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml import Pipeline
 from pyspark.ml.classification import DecisionTreeClassifier
diff --git a/examples/src/main/python/ml/decision_tree_regression_example.py b/examples/src/main/python/ml/decision_tree_regression_example.py
index 58d7ad921d8e0..1ed1636a3d962 100644
--- a/examples/src/main/python/ml/decision_tree_regression_example.py
+++ b/examples/src/main/python/ml/decision_tree_regression_example.py
@@ -18,8 +18,6 @@
 """
 Decision Tree Regression Example.
 """
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml import Pipeline
 from pyspark.ml.regression import DecisionTreeRegressor
diff --git a/examples/src/main/python/ml/elementwise_product_example.py b/examples/src/main/python/ml/elementwise_product_example.py
index 590053998bccc..71eec8d432998 100644
--- a/examples/src/main/python/ml/elementwise_product_example.py
+++ b/examples/src/main/python/ml/elementwise_product_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.feature import ElementwiseProduct
 from pyspark.ml.linalg import Vectors
diff --git a/examples/src/main/python/ml/estimator_transformer_param_example.py b/examples/src/main/python/ml/estimator_transformer_param_example.py
index eb21051435393..2cf9432646b5e 100644
--- a/examples/src/main/python/ml/estimator_transformer_param_example.py
+++ b/examples/src/main/python/ml/estimator_transformer_param_example.py
@@ -18,8 +18,6 @@
 """
 Estimator Transformer Param Example.
 """
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.linalg import Vectors
 from pyspark.ml.classification import LogisticRegression
@@ -58,12 +56,14 @@
     # We may alternatively specify parameters using a Python dictionary as a paramMap
     paramMap = {lr.maxIter: 20}
     paramMap[lr.maxIter] = 30  # Specify 1 Param, overwriting the original maxIter.
-    paramMap.update({lr.regParam: 0.1, lr.threshold: 0.55})  # Specify multiple Params.
+    # Specify multiple Params.
+    paramMap.update({lr.regParam: 0.1, lr.threshold: 0.55})  # type: ignore
 
     # You can combine paramMaps, which are python dictionaries.
-    paramMap2 = {lr.probabilityCol: "myProbability"}  # Change output column name
+    # Change output column name
+    paramMap2 = {lr.probabilityCol: "myProbability"}  # type: ignore
     paramMapCombined = paramMap.copy()
-    paramMapCombined.update(paramMap2)
+    paramMapCombined.update(paramMap2)  # type: ignore
 
     # Now learn a new model using the paramMapCombined parameters.
     # paramMapCombined overrides all parameters set earlier via lr.set* methods.
diff --git a/examples/src/main/python/ml/feature_hasher_example.py b/examples/src/main/python/ml/feature_hasher_example.py
index 6cf9ecc396400..4fe573d19dfbc 100644
--- a/examples/src/main/python/ml/feature_hasher_example.py
+++ b/examples/src/main/python/ml/feature_hasher_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 from pyspark.sql import SparkSession
 # $example on$
 from pyspark.ml.feature import FeatureHasher
diff --git a/examples/src/main/python/ml/fm_classifier_example.py b/examples/src/main/python/ml/fm_classifier_example.py
index 6e7c2ccf021ed..da49e5fc2baa9 100644
--- a/examples/src/main/python/ml/fm_classifier_example.py
+++ b/examples/src/main/python/ml/fm_classifier_example.py
@@ -18,8 +18,6 @@
 """
 FMClassifier Example.
 """
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml import Pipeline
 from pyspark.ml.classification import FMClassifier
@@ -69,9 +67,9 @@
     print("Test set accuracy = %g" % accuracy)
 
     fmModel = model.stages[2]
-    print("Factors: " + str(fmModel.factors))
-    print("Linear: " + str(fmModel.linear))
-    print("Intercept: " + str(fmModel.intercept))
+    print("Factors: " + str(fmModel.factors))  # type: ignore
+    print("Linear: " + str(fmModel.linear))  # type: ignore
+    print("Intercept: " + str(fmModel.intercept))  # type: ignore
     # $example off$
 
     spark.stop()
diff --git a/examples/src/main/python/ml/fm_regressor_example.py b/examples/src/main/python/ml/fm_regressor_example.py
index afd76396800b7..47544b6324203 100644
--- a/examples/src/main/python/ml/fm_regressor_example.py
+++ b/examples/src/main/python/ml/fm_regressor_example.py
@@ -18,8 +18,6 @@
 """
 FMRegressor Example.
 """
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml import Pipeline
 from pyspark.ml.regression import FMRegressor
@@ -66,9 +64,9 @@
     print("Root Mean Squared Error (RMSE) on test data = %g" % rmse)
 
     fmModel = model.stages[1]
-    print("Factors: " + str(fmModel.factors))
-    print("Linear: " + str(fmModel.linear))
-    print("Intercept: " + str(fmModel.intercept))
+    print("Factors: " + str(fmModel.factors))  # type: ignore
+    print("Linear: " + str(fmModel.linear))  # type: ignore
+    print("Intercept: " + str(fmModel.intercept))  # type: ignore
     # $example off$
 
     spark.stop()
diff --git a/examples/src/main/python/ml/fvalue_selector_example.py b/examples/src/main/python/ml/fvalue_selector_example.py
deleted file mode 100644
index 3158953a5dfc4..0000000000000
--- a/examples/src/main/python/ml/fvalue_selector_example.py
+++ /dev/null
@@ -1,55 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""
-An example for FValueSelector.
-Run with:
-  bin/spark-submit examples/src/main/python/ml/fvalue_selector_example.py
-"""
-from __future__ import print_function
-
-from pyspark.sql import SparkSession
-# $example on$
-from pyspark.ml.feature import FValueSelector
-from pyspark.ml.linalg import Vectors
-# $example off$
-
-if __name__ == "__main__":
-    spark = SparkSession\
-        .builder\
-        .appName("FValueSelectorExample")\
-        .getOrCreate()
-
-    # $example on$
-    df = spark.createDataFrame([
-        (1, Vectors.dense([6.0, 7.0, 0.0, 7.0, 6.0, 0.0]), 4.6,),
-        (2, Vectors.dense([0.0, 9.0, 6.0, 0.0, 5.0, 9.0]), 6.6,),
-        (3, Vectors.dense([0.0, 9.0, 3.0, 0.0, 5.0, 5.0]), 5.1,),
-        (4, Vectors.dense([0.0, 9.0, 8.0, 5.0, 6.0, 4.0]), 7.6,),
-        (5, Vectors.dense([8.0, 9.0, 6.0, 5.0, 4.0, 4.0]), 9.0,),
-        (6, Vectors.dense([8.0, 9.0, 6.0, 4.0, 0.0, 0.0]), 9.0,)], ["id", "features", "label"])
-
-    selector = FValueSelector(numTopFeatures=1, featuresCol="features",
-                              outputCol="selectedFeatures", labelCol="label")
-
-    result = selector.fit(df).transform(df)
-
-    print("FValueSelector output with top %d features selected" % selector.getNumTopFeatures())
-    result.show()
-    # $example off$
-
-    spark.stop()
diff --git a/examples/src/main/python/ml/fvalue_test_example.py b/examples/src/main/python/ml/fvalue_test_example.py
deleted file mode 100644
index 410b39e4493f8..0000000000000
--- a/examples/src/main/python/ml/fvalue_test_example.py
+++ /dev/null
@@ -1,52 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""
-An example for FValue testing.
-Run with:
-  bin/spark-submit examples/src/main/python/ml/fvalue_test_example.py
-"""
-from __future__ import print_function
-
-from pyspark.sql import SparkSession
-# $example on$
-from pyspark.ml.linalg import Vectors
-from pyspark.ml.stat import FValueTest
-# $example off$
-
-if __name__ == "__main__":
-    spark = SparkSession \
-        .builder \
-        .appName("FValueTestExample") \
-        .getOrCreate()
-
-    # $example on$
-    data = [(4.6, Vectors.dense(6.0, 7.0, 0.0, 7.0, 6.0, 0.0)),
-            (6.6, Vectors.dense(0.0, 9.0, 6.0, 0.0, 5.0, 9.0)),
-            (5.1, Vectors.dense(0.0, 9.0, 3.0, 0.0, 5.0, 5.0)),
-            (7.6, Vectors.dense(0.0, 9.0, 8.0, 5.0, 6.0, 4.0)),
-            (9.0, Vectors.dense(8.0, 9.0, 6.0, 5.0, 4.0, 4.0)),
-            (9.0, Vectors.dense(8.0, 9.0, 6.0, 4.0, 0.0, 0.0))]
-    df = spark.createDataFrame(data, ["label", "features"])
-
-    ftest = FValueTest.test(df, "features", "label").head()
-    print("pValues: " + str(ftest.pValues))
-    print("degreesOfFreedom: " + str(ftest.degreesOfFreedom))
-    print("fvalues: " + str(ftest.fValues))
-    # $example off$
-
-    spark.stop()
diff --git a/examples/src/main/python/ml/gaussian_mixture_example.py b/examples/src/main/python/ml/gaussian_mixture_example.py
index 4938a904189f9..1441faa792983 100644
--- a/examples/src/main/python/ml/gaussian_mixture_example.py
+++ b/examples/src/main/python/ml/gaussian_mixture_example.py
@@ -20,8 +20,6 @@
 Run with:
   bin/spark-submit examples/src/main/python/ml/gaussian_mixture_example.py
 """
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.clustering import GaussianMixture
 # $example off$
diff --git a/examples/src/main/python/ml/generalized_linear_regression_example.py b/examples/src/main/python/ml/generalized_linear_regression_example.py
index a52f4650c1c6f..06a8a5a2e9428 100644
--- a/examples/src/main/python/ml/generalized_linear_regression_example.py
+++ b/examples/src/main/python/ml/generalized_linear_regression_example.py
@@ -20,8 +20,6 @@
 Run with:
   bin/spark-submit examples/src/main/python/ml/generalized_linear_regression_example.py
 """
-from __future__ import print_function
-
 from pyspark.sql import SparkSession
 # $example on$
 from pyspark.ml.regression import GeneralizedLinearRegression
diff --git a/examples/src/main/python/ml/gradient_boosted_tree_classifier_example.py b/examples/src/main/python/ml/gradient_boosted_tree_classifier_example.py
index c2042fd7b7b07..a7efa2170a069 100644
--- a/examples/src/main/python/ml/gradient_boosted_tree_classifier_example.py
+++ b/examples/src/main/python/ml/gradient_boosted_tree_classifier_example.py
@@ -18,8 +18,6 @@
 """
 Gradient Boosted Tree Classifier Example.
 """
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml import Pipeline
 from pyspark.ml.classification import GBTClassifier
diff --git a/examples/src/main/python/ml/gradient_boosted_tree_regressor_example.py b/examples/src/main/python/ml/gradient_boosted_tree_regressor_example.py
index cc96c973e4b23..5e09b96c1ea3a 100644
--- a/examples/src/main/python/ml/gradient_boosted_tree_regressor_example.py
+++ b/examples/src/main/python/ml/gradient_boosted_tree_regressor_example.py
@@ -18,8 +18,6 @@
 """
 Gradient Boosted Tree Regressor Example.
 """
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml import Pipeline
 from pyspark.ml.regression import GBTRegressor
diff --git a/examples/src/main/python/ml/index_to_string_example.py b/examples/src/main/python/ml/index_to_string_example.py
index 33d104e8e3f41..98bdb89ce3039 100644
--- a/examples/src/main/python/ml/index_to_string_example.py
+++ b/examples/src/main/python/ml/index_to_string_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.feature import IndexToString, StringIndexer
 # $example off$
diff --git a/examples/src/main/python/ml/interaction_example.py b/examples/src/main/python/ml/interaction_example.py
index 4b632271916f5..ac365179b0c20 100644
--- a/examples/src/main/python/ml/interaction_example.py
+++ b/examples/src/main/python/ml/interaction_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.feature import Interaction, VectorAssembler
 # $example off$
diff --git a/examples/src/main/python/ml/isotonic_regression_example.py b/examples/src/main/python/ml/isotonic_regression_example.py
index 89cba9dfc7e8f..d7b893894fc71 100644
--- a/examples/src/main/python/ml/isotonic_regression_example.py
+++ b/examples/src/main/python/ml/isotonic_regression_example.py
@@ -21,8 +21,6 @@
 Run with:
   bin/spark-submit examples/src/main/python/ml/isotonic_regression_example.py
 """
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.regression import IsotonicRegression
 # $example off$
diff --git a/examples/src/main/python/ml/kmeans_example.py b/examples/src/main/python/ml/kmeans_example.py
index 80a878af679f4..47223fd953d17 100644
--- a/examples/src/main/python/ml/kmeans_example.py
+++ b/examples/src/main/python/ml/kmeans_example.py
@@ -22,8 +22,6 @@
 
 This example requires NumPy (http://www.numpy.org/).
 """
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.clustering import KMeans
 from pyspark.ml.evaluation import ClusteringEvaluator
diff --git a/examples/src/main/python/ml/lda_example.py b/examples/src/main/python/ml/lda_example.py
index 97d1a042d1479..a47dfa383c895 100644
--- a/examples/src/main/python/ml/lda_example.py
+++ b/examples/src/main/python/ml/lda_example.py
@@ -20,8 +20,6 @@
 Run with:
   bin/spark-submit examples/src/main/python/ml/lda_example.py
 """
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.clustering import LDA
 # $example off$
diff --git a/examples/src/main/python/ml/linear_regression_with_elastic_net.py b/examples/src/main/python/ml/linear_regression_with_elastic_net.py
index 6639e9160ab71..864fc76cff132 100644
--- a/examples/src/main/python/ml/linear_regression_with_elastic_net.py
+++ b/examples/src/main/python/ml/linear_regression_with_elastic_net.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.regression import LinearRegression
 # $example off$
diff --git a/examples/src/main/python/ml/linearsvc.py b/examples/src/main/python/ml/linearsvc.py
index 9b79abbf96f88..61d726cf3f1ae 100644
--- a/examples/src/main/python/ml/linearsvc.py
+++ b/examples/src/main/python/ml/linearsvc.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.classification import LinearSVC
 # $example off$
diff --git a/examples/src/main/python/ml/logistic_regression_summary_example.py b/examples/src/main/python/ml/logistic_regression_summary_example.py
index 2274ff707b2a3..6d045108da0aa 100644
--- a/examples/src/main/python/ml/logistic_regression_summary_example.py
+++ b/examples/src/main/python/ml/logistic_regression_summary_example.py
@@ -20,8 +20,6 @@
 Run with:
   bin/spark-submit examples/src/main/python/ml/logistic_regression_summary_example.py
 """
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.classification import LogisticRegression
 # $example off$
diff --git a/examples/src/main/python/ml/logistic_regression_with_elastic_net.py b/examples/src/main/python/ml/logistic_regression_with_elastic_net.py
index d095fbd373408..916fdade27623 100644
--- a/examples/src/main/python/ml/logistic_regression_with_elastic_net.py
+++ b/examples/src/main/python/ml/logistic_regression_with_elastic_net.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.classification import LogisticRegression
 # $example off$
diff --git a/examples/src/main/python/ml/max_abs_scaler_example.py b/examples/src/main/python/ml/max_abs_scaler_example.py
index 45eda3cdadde3..d7ff3561ce429 100644
--- a/examples/src/main/python/ml/max_abs_scaler_example.py
+++ b/examples/src/main/python/ml/max_abs_scaler_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.feature import MaxAbsScaler
 from pyspark.ml.linalg import Vectors
diff --git a/examples/src/main/python/ml/min_hash_lsh_example.py b/examples/src/main/python/ml/min_hash_lsh_example.py
index 93136e6ae3cae..683f97a055ede 100644
--- a/examples/src/main/python/ml/min_hash_lsh_example.py
+++ b/examples/src/main/python/ml/min_hash_lsh_example.py
@@ -20,8 +20,6 @@
 Run with:
   bin/spark-submit examples/src/main/python/ml/min_hash_lsh_example.py
 """
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.feature import MinHashLSH
 from pyspark.ml.linalg import Vectors
diff --git a/examples/src/main/python/ml/min_max_scaler_example.py b/examples/src/main/python/ml/min_max_scaler_example.py
index b5f272e59bc30..cd74243699894 100644
--- a/examples/src/main/python/ml/min_max_scaler_example.py
+++ b/examples/src/main/python/ml/min_max_scaler_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.feature import MinMaxScaler
 from pyspark.ml.linalg import Vectors
diff --git a/examples/src/main/python/ml/model_selection_random_hyperparameters_example.py b/examples/src/main/python/ml/model_selection_random_hyperparameters_example.py
new file mode 100644
index 0000000000000..b436341b19665
--- /dev/null
+++ b/examples/src/main/python/ml/model_selection_random_hyperparameters_example.py
@@ -0,0 +1,66 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+This example uses random hyperparameters to perform model selection.
+Run with:
+
+  bin/spark-submit examples/src/main/python/ml/model_selection_random_hyperparameters_example.py
+"""
+# $example on$
+from pyspark.ml.evaluation import RegressionEvaluator
+from pyspark.ml.regression import LinearRegression
+from pyspark.ml.tuning import ParamRandomBuilder, CrossValidator
+# $example off$
+from pyspark.sql import SparkSession
+
+if __name__ == "__main__":
+    spark = SparkSession \
+        .builder \
+        .appName("TrainValidationSplit") \
+        .getOrCreate()
+
+    # $example on$
+    data = spark.read.format("libsvm") \
+        .load("data/mllib/sample_linear_regression_data.txt")
+
+    lr = LinearRegression(maxIter=10)
+
+    # We sample the regularization parameter logarithmically over the range [0.01, 1.0].
+    # This means that values around 0.01, 0.1 and 1.0 are roughly equally likely.
+    # Note that both parameters must be greater than zero as otherwise we'll get an infinity.
+    # We sample the the ElasticNet mixing parameter uniformly over the range [0, 1]
+    # Note that in real life, you'd choose more than the 5 samples we see below.
+    hyperparameters = ParamRandomBuilder() \
+        .addLog10Random(lr.regParam, 0.01, 1.0, 5) \
+        .addRandom(lr.elasticNetParam, 0.0, 1.0, 5) \
+        .addGrid(lr.fitIntercept, [False, True]) \
+        .build()
+
+    cv = CrossValidator(estimator=lr,
+                        estimatorParamMaps=hyperparameters,
+                        evaluator=RegressionEvaluator(),
+                        numFolds=2)
+
+    model = cv.fit(data)
+    bestModel = model.bestModel
+    print("Optimal model has regParam = {}, elasticNetParam = {}, fitIntercept = {}"
+          .format(bestModel.getRegParam(), bestModel.getElasticNetParam(),
+                  bestModel.getFitIntercept()))
+
+    # $example off$
+    spark.stop()
diff --git a/examples/src/main/python/ml/multiclass_logistic_regression_with_elastic_net.py b/examples/src/main/python/ml/multiclass_logistic_regression_with_elastic_net.py
index bec9860c79a2d..3bb4a72864101 100644
--- a/examples/src/main/python/ml/multiclass_logistic_regression_with_elastic_net.py
+++ b/examples/src/main/python/ml/multiclass_logistic_regression_with_elastic_net.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.classification import LogisticRegression
 # $example off$
diff --git a/examples/src/main/python/ml/multilayer_perceptron_classification.py b/examples/src/main/python/ml/multilayer_perceptron_classification.py
index 88fc69f753953..74f532193573d 100644
--- a/examples/src/main/python/ml/multilayer_perceptron_classification.py
+++ b/examples/src/main/python/ml/multilayer_perceptron_classification.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.classification import MultilayerPerceptronClassifier
 from pyspark.ml.evaluation import MulticlassClassificationEvaluator
diff --git a/examples/src/main/python/ml/n_gram_example.py b/examples/src/main/python/ml/n_gram_example.py
index 31676e076a11b..8c8031b939458 100644
--- a/examples/src/main/python/ml/n_gram_example.py
+++ b/examples/src/main/python/ml/n_gram_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.feature import NGram
 # $example off$
diff --git a/examples/src/main/python/ml/naive_bayes_example.py b/examples/src/main/python/ml/naive_bayes_example.py
index 7290ab81cd0ec..8d1777c6f9e39 100644
--- a/examples/src/main/python/ml/naive_bayes_example.py
+++ b/examples/src/main/python/ml/naive_bayes_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.classification import NaiveBayes
 from pyspark.ml.evaluation import MulticlassClassificationEvaluator
diff --git a/examples/src/main/python/ml/normalizer_example.py b/examples/src/main/python/ml/normalizer_example.py
index 510bd825fd286..2aa012961a2ee 100644
--- a/examples/src/main/python/ml/normalizer_example.py
+++ b/examples/src/main/python/ml/normalizer_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.feature import Normalizer
 from pyspark.ml.linalg import Vectors
diff --git a/examples/src/main/python/ml/one_vs_rest_example.py b/examples/src/main/python/ml/one_vs_rest_example.py
index 956e94ae4ab62..4cae1a99808e8 100644
--- a/examples/src/main/python/ml/one_vs_rest_example.py
+++ b/examples/src/main/python/ml/one_vs_rest_example.py
@@ -21,8 +21,6 @@
 Run with:
   bin/spark-submit examples/src/main/python/ml/one_vs_rest_example.py
 """
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.classification import LogisticRegression, OneVsRest
 from pyspark.ml.evaluation import MulticlassClassificationEvaluator
diff --git a/examples/src/main/python/ml/onehot_encoder_example.py b/examples/src/main/python/ml/onehot_encoder_example.py
index 73775b79e36cb..6deb84ed785ca 100644
--- a/examples/src/main/python/ml/onehot_encoder_example.py
+++ b/examples/src/main/python/ml/onehot_encoder_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.feature import OneHotEncoder
 # $example off$
diff --git a/examples/src/main/python/ml/pca_example.py b/examples/src/main/python/ml/pca_example.py
index 38746aced096a..03fb709c8e91d 100644
--- a/examples/src/main/python/ml/pca_example.py
+++ b/examples/src/main/python/ml/pca_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.feature import PCA
 from pyspark.ml.linalg import Vectors
diff --git a/examples/src/main/python/ml/pipeline_example.py b/examples/src/main/python/ml/pipeline_example.py
index e1fab7cbe6d80..66fdd73632a70 100644
--- a/examples/src/main/python/ml/pipeline_example.py
+++ b/examples/src/main/python/ml/pipeline_example.py
@@ -62,8 +62,12 @@
     prediction = model.transform(test)
     selected = prediction.select("id", "text", "probability", "prediction")
     for row in selected.collect():
-        rid, text, prob, prediction = row
-        print("(%d, %s) --> prob=%s, prediction=%f" % (rid, text, str(prob), prediction))
+        rid, text, prob, prediction = row  # type: ignore
+        print(
+            "(%d, %s) --> prob=%s, prediction=%f" % (
+                rid, text, str(prob), prediction   # type: ignore
+            )
+        )
     # $example off$
 
     spark.stop()
diff --git a/examples/src/main/python/ml/polynomial_expansion_example.py b/examples/src/main/python/ml/polynomial_expansion_example.py
index 40bcb7b13a3de..75f436e768dc5 100644
--- a/examples/src/main/python/ml/polynomial_expansion_example.py
+++ b/examples/src/main/python/ml/polynomial_expansion_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.feature import PolynomialExpansion
 from pyspark.ml.linalg import Vectors
diff --git a/examples/src/main/python/ml/quantile_discretizer_example.py b/examples/src/main/python/ml/quantile_discretizer_example.py
index 0fc1d1949a77d..82be3936d2598 100644
--- a/examples/src/main/python/ml/quantile_discretizer_example.py
+++ b/examples/src/main/python/ml/quantile_discretizer_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.feature import QuantileDiscretizer
 # $example off$
diff --git a/examples/src/main/python/ml/random_forest_classifier_example.py b/examples/src/main/python/ml/random_forest_classifier_example.py
index 4eaa94dd7f489..8983d1f2e979b 100644
--- a/examples/src/main/python/ml/random_forest_classifier_example.py
+++ b/examples/src/main/python/ml/random_forest_classifier_example.py
@@ -18,8 +18,6 @@
 """
 Random Forest Classifier Example.
 """
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml import Pipeline
 from pyspark.ml.classification import RandomForestClassifier
diff --git a/examples/src/main/python/ml/random_forest_regressor_example.py b/examples/src/main/python/ml/random_forest_regressor_example.py
index a34edff2ecaa2..b9306ddf2f82c 100644
--- a/examples/src/main/python/ml/random_forest_regressor_example.py
+++ b/examples/src/main/python/ml/random_forest_regressor_example.py
@@ -18,8 +18,6 @@
 """
 Random Forest Regressor Example.
 """
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml import Pipeline
 from pyspark.ml.regression import RandomForestRegressor
diff --git a/examples/src/main/python/ml/rformula_example.py b/examples/src/main/python/ml/rformula_example.py
index 6629239db29ec..25bb6dac56e81 100644
--- a/examples/src/main/python/ml/rformula_example.py
+++ b/examples/src/main/python/ml/rformula_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.feature import RFormula
 # $example off$
diff --git a/examples/src/main/python/ml/robust_scaler_example.py b/examples/src/main/python/ml/robust_scaler_example.py
index 435e9ccb806c6..9f7c6d6507c78 100644
--- a/examples/src/main/python/ml/robust_scaler_example.py
+++ b/examples/src/main/python/ml/robust_scaler_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.feature import RobustScaler
 # $example off$
diff --git a/examples/src/main/python/ml/sql_transformer.py b/examples/src/main/python/ml/sql_transformer.py
index 0bf8f35720c95..c8ac5c46aa5e9 100644
--- a/examples/src/main/python/ml/sql_transformer.py
+++ b/examples/src/main/python/ml/sql_transformer.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.feature import SQLTransformer
 # $example off$
diff --git a/examples/src/main/python/ml/standard_scaler_example.py b/examples/src/main/python/ml/standard_scaler_example.py
index c0027480e69b3..9021c10075d81 100644
--- a/examples/src/main/python/ml/standard_scaler_example.py
+++ b/examples/src/main/python/ml/standard_scaler_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.feature import StandardScaler
 # $example off$
diff --git a/examples/src/main/python/ml/stopwords_remover_example.py b/examples/src/main/python/ml/stopwords_remover_example.py
index 3b8e7855e3e79..832a7c7d0ad88 100644
--- a/examples/src/main/python/ml/stopwords_remover_example.py
+++ b/examples/src/main/python/ml/stopwords_remover_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.feature import StopWordsRemover
 # $example off$
diff --git a/examples/src/main/python/ml/string_indexer_example.py b/examples/src/main/python/ml/string_indexer_example.py
index 2255bfb9c1a60..f2ac63eabd71c 100644
--- a/examples/src/main/python/ml/string_indexer_example.py
+++ b/examples/src/main/python/ml/string_indexer_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.feature import StringIndexer
 # $example off$
diff --git a/examples/src/main/python/ml/summarizer_example.py b/examples/src/main/python/ml/summarizer_example.py
index 8835f189a1ad4..4982746450132 100644
--- a/examples/src/main/python/ml/summarizer_example.py
+++ b/examples/src/main/python/ml/summarizer_example.py
@@ -20,8 +20,6 @@
 Run with:
   bin/spark-submit examples/src/main/python/ml/summarizer_example.py
 """
-from __future__ import print_function
-
 from pyspark.sql import SparkSession
 # $example on$
 from pyspark.ml.stat import Summarizer
diff --git a/examples/src/main/python/ml/tf_idf_example.py b/examples/src/main/python/ml/tf_idf_example.py
index d43244fa68e97..b4bb0dfa3183c 100644
--- a/examples/src/main/python/ml/tf_idf_example.py
+++ b/examples/src/main/python/ml/tf_idf_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.feature import HashingTF, IDF, Tokenizer
 # $example off$
diff --git a/examples/src/main/python/ml/tokenizer_example.py b/examples/src/main/python/ml/tokenizer_example.py
index 5c65c5c9f8260..c6b5fac227315 100644
--- a/examples/src/main/python/ml/tokenizer_example.py
+++ b/examples/src/main/python/ml/tokenizer_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.feature import Tokenizer, RegexTokenizer
 from pyspark.sql.functions import col, udf
diff --git a/examples/src/main/python/ml/train_validation_split.py b/examples/src/main/python/ml/train_validation_split.py
index d4f9184bf576e..5e3dc7b3ec2fa 100644
--- a/examples/src/main/python/ml/train_validation_split.py
+++ b/examples/src/main/python/ml/train_validation_split.py
@@ -17,7 +17,7 @@
 
 """
 This example demonstrates applying TrainValidationSplit to split data
-and preform model selection.
+and perform model selection.
 Run with:
 
   bin/spark-submit examples/src/main/python/ml/train_validation_split.py
diff --git a/examples/src/main/python/ml/univariate_feature_selector_example.py b/examples/src/main/python/ml/univariate_feature_selector_example.py
new file mode 100644
index 0000000000000..6dc293e49643b
--- /dev/null
+++ b/examples/src/main/python/ml/univariate_feature_selector_example.py
@@ -0,0 +1,55 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+An example for UnivariateFeatureSelector.
+Run with:
+  bin/spark-submit examples/src/main/python/ml/univariate_feature_selector_example.py
+"""
+from pyspark.sql import SparkSession
+# $example on$
+from pyspark.ml.feature import UnivariateFeatureSelector
+from pyspark.ml.linalg import Vectors
+# $example off$
+
+if __name__ == "__main__":
+    spark = SparkSession\
+        .builder\
+        .appName("UnivariateFeatureSelectorExample")\
+        .getOrCreate()
+
+    # $example on$
+    df = spark.createDataFrame([
+        (1, Vectors.dense([1.7, 4.4, 7.6, 5.8, 9.6, 2.3]), 3.0,),
+        (2, Vectors.dense([8.8, 7.3, 5.7, 7.3, 2.2, 4.1]), 2.0,),
+        (3, Vectors.dense([1.2, 9.5, 2.5, 3.1, 8.7, 2.5]), 3.0,),
+        (4, Vectors.dense([3.7, 9.2, 6.1, 4.1, 7.5, 3.8]), 2.0,),
+        (5, Vectors.dense([8.9, 5.2, 7.8, 8.3, 5.2, 3.0]), 4.0,),
+        (6, Vectors.dense([7.9, 8.5, 9.2, 4.0, 9.4, 2.1]), 4.0,)], ["id", "features", "label"])
+
+    selector = UnivariateFeatureSelector(featuresCol="features", outputCol="selectedFeatures",
+                                         labelCol="label", selectionMode="numTopFeatures")
+    selector.setFeatureType("continuous").setLabelType("categorical").setSelectionThreshold(1)
+
+    result = selector.fit(df).transform(df)
+
+    print("UnivariateFeatureSelector output with top %d features selected using f_classif"
+          % selector.getSelectionThreshold())
+    result.show()
+    # $example off$
+
+    spark.stop()
diff --git a/examples/src/main/python/ml/variance_threshold_selector_example.py b/examples/src/main/python/ml/variance_threshold_selector_example.py
index b7edb86653530..0a996e0e28264 100644
--- a/examples/src/main/python/ml/variance_threshold_selector_example.py
+++ b/examples/src/main/python/ml/variance_threshold_selector_example.py
@@ -20,8 +20,6 @@
 Run with:
   bin/spark-submit examples/src/main/python/ml/variance_threshold_selector_example.py
 """
-from __future__ import print_function
-
 from pyspark.sql import SparkSession
 # $example on$
 from pyspark.ml.feature import VarianceThresholdSelector
diff --git a/examples/src/main/python/ml/vector_assembler_example.py b/examples/src/main/python/ml/vector_assembler_example.py
index 98de1d5ea7dac..0ce31cf0eabc9 100644
--- a/examples/src/main/python/ml/vector_assembler_example.py
+++ b/examples/src/main/python/ml/vector_assembler_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.linalg import Vectors
 from pyspark.ml.feature import VectorAssembler
diff --git a/examples/src/main/python/ml/vector_indexer_example.py b/examples/src/main/python/ml/vector_indexer_example.py
index 5c2956077d6ce..51a4191606fb8 100644
--- a/examples/src/main/python/ml/vector_indexer_example.py
+++ b/examples/src/main/python/ml/vector_indexer_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.feature import VectorIndexer
 # $example off$
diff --git a/examples/src/main/python/ml/vector_size_hint_example.py b/examples/src/main/python/ml/vector_size_hint_example.py
index fb77dacec629d..355d85aee8729 100644
--- a/examples/src/main/python/ml/vector_size_hint_example.py
+++ b/examples/src/main/python/ml/vector_size_hint_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.linalg import Vectors
 from pyspark.ml.feature import (VectorSizeHint, VectorAssembler)
diff --git a/examples/src/main/python/ml/vector_slicer_example.py b/examples/src/main/python/ml/vector_slicer_example.py
index 68c8cfe27e375..86e089d152c5a 100644
--- a/examples/src/main/python/ml/vector_slicer_example.py
+++ b/examples/src/main/python/ml/vector_slicer_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.feature import VectorSlicer
 from pyspark.ml.linalg import Vectors
diff --git a/examples/src/main/python/ml/word2vec_example.py b/examples/src/main/python/ml/word2vec_example.py
index 77f8951df0883..0eabeda3dce4b 100644
--- a/examples/src/main/python/ml/word2vec_example.py
+++ b/examples/src/main/python/ml/word2vec_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from pyspark.ml.feature import Word2Vec
 # $example off$
diff --git a/examples/src/main/python/mllib/binary_classification_metrics_example.py b/examples/src/main/python/mllib/binary_classification_metrics_example.py
index d14ce7982e24f..741746e6e35ae 100644
--- a/examples/src/main/python/mllib/binary_classification_metrics_example.py
+++ b/examples/src/main/python/mllib/binary_classification_metrics_example.py
@@ -17,7 +17,6 @@
 """
 Binary Classification Metrics Example.
 """
-from __future__ import print_function
 from pyspark import SparkContext
 # $example on$
 from pyspark.mllib.classification import LogisticRegressionWithLBFGS
diff --git a/examples/src/main/python/mllib/bisecting_k_means_example.py b/examples/src/main/python/mllib/bisecting_k_means_example.py
index 36e36fc6897f3..d7b6ad9d424a6 100644
--- a/examples/src/main/python/mllib/bisecting_k_means_example.py
+++ b/examples/src/main/python/mllib/bisecting_k_means_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from numpy import array
 # $example off$
diff --git a/examples/src/main/python/mllib/correlations.py b/examples/src/main/python/mllib/correlations.py
index 089504fa7064b..27d07b22a5645 100755
--- a/examples/src/main/python/mllib/correlations.py
+++ b/examples/src/main/python/mllib/correlations.py
@@ -18,8 +18,6 @@
 """
 Correlations using MLlib.
 """
-from __future__ import print_function
-
 import sys
 
 from pyspark import SparkContext
diff --git a/examples/src/main/python/mllib/correlations_example.py b/examples/src/main/python/mllib/correlations_example.py
index 66d18f6e5df17..bb71b968687cb 100644
--- a/examples/src/main/python/mllib/correlations_example.py
+++ b/examples/src/main/python/mllib/correlations_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 import numpy as np
 
 from pyspark import SparkContext
diff --git a/examples/src/main/python/mllib/decision_tree_classification_example.py b/examples/src/main/python/mllib/decision_tree_classification_example.py
index 7eecf500584ad..009e393226c01 100644
--- a/examples/src/main/python/mllib/decision_tree_classification_example.py
+++ b/examples/src/main/python/mllib/decision_tree_classification_example.py
@@ -18,8 +18,6 @@
 """
 Decision Tree Classification Example.
 """
-from __future__ import print_function
-
 from pyspark import SparkContext
 # $example on$
 from pyspark.mllib.tree import DecisionTree, DecisionTreeModel
diff --git a/examples/src/main/python/mllib/decision_tree_regression_example.py b/examples/src/main/python/mllib/decision_tree_regression_example.py
index acf9e25fdf31c..71dfbf0790175 100644
--- a/examples/src/main/python/mllib/decision_tree_regression_example.py
+++ b/examples/src/main/python/mllib/decision_tree_regression_example.py
@@ -18,8 +18,6 @@
 """
 Decision Tree Regression Example.
 """
-from __future__ import print_function
-
 from pyspark import SparkContext
 # $example on$
 from pyspark.mllib.tree import DecisionTree, DecisionTreeModel
diff --git a/examples/src/main/python/mllib/elementwise_product_example.py b/examples/src/main/python/mllib/elementwise_product_example.py
index 8ae9afb1dc477..15e6a43f736cf 100644
--- a/examples/src/main/python/mllib/elementwise_product_example.py
+++ b/examples/src/main/python/mllib/elementwise_product_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 from pyspark import SparkContext
 # $example on$
 from pyspark.mllib.feature import ElementwiseProduct
diff --git a/examples/src/main/python/mllib/gaussian_mixture_example.py b/examples/src/main/python/mllib/gaussian_mixture_example.py
index a60e799d62eb1..3b19478f457ec 100644
--- a/examples/src/main/python/mllib/gaussian_mixture_example.py
+++ b/examples/src/main/python/mllib/gaussian_mixture_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from numpy import array
 # $example off$
diff --git a/examples/src/main/python/mllib/gaussian_mixture_model.py b/examples/src/main/python/mllib/gaussian_mixture_model.py
index 6b46e27ddaaa8..96ce6b6f6ab25 100644
--- a/examples/src/main/python/mllib/gaussian_mixture_model.py
+++ b/examples/src/main/python/mllib/gaussian_mixture_model.py
@@ -18,11 +18,6 @@
 """
 A Gaussian Mixture Model clustering program using MLlib.
 """
-from __future__ import print_function
-
-import sys
-if sys.version >= '3':
-    long = int
 
 import random
 import argparse
@@ -53,7 +48,7 @@ def parseVector(line):
     parser.add_argument('--convergenceTol', default=1e-3, type=float, help='convergence threshold')
     parser.add_argument('--maxIterations', default=100, type=int, help='Number of iterations')
     parser.add_argument('--seed', default=random.getrandbits(19),
-                        type=long, help='Random seed')
+                        type=int, help='Random seed')
     args = parser.parse_args()
 
     conf = SparkConf().setAppName("GMM")
diff --git a/examples/src/main/python/mllib/gradient_boosting_classification_example.py b/examples/src/main/python/mllib/gradient_boosting_classification_example.py
index 65a03572be9b5..eb12f206196fe 100644
--- a/examples/src/main/python/mllib/gradient_boosting_classification_example.py
+++ b/examples/src/main/python/mllib/gradient_boosting_classification_example.py
@@ -18,8 +18,6 @@
 """
 Gradient Boosted Trees Classification Example.
 """
-from __future__ import print_function
-
 from pyspark import SparkContext
 # $example on$
 from pyspark.mllib.tree import GradientBoostedTrees, GradientBoostedTreesModel
diff --git a/examples/src/main/python/mllib/gradient_boosting_regression_example.py b/examples/src/main/python/mllib/gradient_boosting_regression_example.py
index 877f8ab461ccd..eb59a992df539 100644
--- a/examples/src/main/python/mllib/gradient_boosting_regression_example.py
+++ b/examples/src/main/python/mllib/gradient_boosting_regression_example.py
@@ -18,8 +18,6 @@
 """
 Gradient Boosted Trees Regression Example.
 """
-from __future__ import print_function
-
 from pyspark import SparkContext
 # $example on$
 from pyspark.mllib.tree import GradientBoostedTrees, GradientBoostedTreesModel
diff --git a/examples/src/main/python/mllib/hypothesis_testing_example.py b/examples/src/main/python/mllib/hypothesis_testing_example.py
index 21a5584fd6e06..321be8b76f1b9 100644
--- a/examples/src/main/python/mllib/hypothesis_testing_example.py
+++ b/examples/src/main/python/mllib/hypothesis_testing_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 from pyspark import SparkContext
 # $example on$
 from pyspark.mllib.linalg import Matrices, Vectors
diff --git a/examples/src/main/python/mllib/hypothesis_testing_kolmogorov_smirnov_test_example.py b/examples/src/main/python/mllib/hypothesis_testing_kolmogorov_smirnov_test_example.py
index ef380dee79d3d..12a186900e358 100644
--- a/examples/src/main/python/mllib/hypothesis_testing_kolmogorov_smirnov_test_example.py
+++ b/examples/src/main/python/mllib/hypothesis_testing_kolmogorov_smirnov_test_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 from pyspark import SparkContext
 # $example on$
 from pyspark.mllib.stat import Statistics
diff --git a/examples/src/main/python/mllib/isotonic_regression_example.py b/examples/src/main/python/mllib/isotonic_regression_example.py
index f5322d79c45ba..a5a0cfeae9d75 100644
--- a/examples/src/main/python/mllib/isotonic_regression_example.py
+++ b/examples/src/main/python/mllib/isotonic_regression_example.py
@@ -18,8 +18,6 @@
 """
 Isotonic Regression Example.
 """
-from __future__ import print_function
-
 from pyspark import SparkContext
 # $example on$
 import math
diff --git a/examples/src/main/python/mllib/k_means_example.py b/examples/src/main/python/mllib/k_means_example.py
index d6058f45020c4..ead1e56de55c6 100644
--- a/examples/src/main/python/mllib/k_means_example.py
+++ b/examples/src/main/python/mllib/k_means_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 # $example on$
 from numpy import array
 from math import sqrt
diff --git a/examples/src/main/python/mllib/kernel_density_estimation_example.py b/examples/src/main/python/mllib/kernel_density_estimation_example.py
index 3e8f7241a4a1e..22d191716057c 100644
--- a/examples/src/main/python/mllib/kernel_density_estimation_example.py
+++ b/examples/src/main/python/mllib/kernel_density_estimation_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 from pyspark import SparkContext
 # $example on$
 from pyspark.mllib.stat import KernelDensity
diff --git a/examples/src/main/python/mllib/kmeans.py b/examples/src/main/python/mllib/kmeans.py
index 1bdb3e9b4a2af..2560384b6a0e2 100755
--- a/examples/src/main/python/mllib/kmeans.py
+++ b/examples/src/main/python/mllib/kmeans.py
@@ -20,8 +20,6 @@
 
 This example requires NumPy (http://www.numpy.org/).
 """
-from __future__ import print_function
-
 import sys
 
 import numpy as np
diff --git a/examples/src/main/python/mllib/latent_dirichlet_allocation_example.py b/examples/src/main/python/mllib/latent_dirichlet_allocation_example.py
index 2a1bef5f207b7..f82a28aadc5a3 100644
--- a/examples/src/main/python/mllib/latent_dirichlet_allocation_example.py
+++ b/examples/src/main/python/mllib/latent_dirichlet_allocation_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 from pyspark import SparkContext
 # $example on$
 from pyspark.mllib.clustering import LDA, LDAModel
diff --git a/examples/src/main/python/mllib/linear_regression_with_sgd_example.py b/examples/src/main/python/mllib/linear_regression_with_sgd_example.py
index 6744463d40ef1..cb67396332312 100644
--- a/examples/src/main/python/mllib/linear_regression_with_sgd_example.py
+++ b/examples/src/main/python/mllib/linear_regression_with_sgd_example.py
@@ -18,8 +18,6 @@
 """
 Linear Regression With SGD Example.
 """
-from __future__ import print_function
-
 from pyspark import SparkContext
 # $example on$
 from pyspark.mllib.regression import LabeledPoint, LinearRegressionWithSGD, LinearRegressionModel
diff --git a/examples/src/main/python/mllib/logistic_regression.py b/examples/src/main/python/mllib/logistic_regression.py
index 87efe17375226..7b90615a53424 100755
--- a/examples/src/main/python/mllib/logistic_regression.py
+++ b/examples/src/main/python/mllib/logistic_regression.py
@@ -20,8 +20,6 @@
 
 This example requires NumPy (http://www.numpy.org/).
 """
-from __future__ import print_function
-
 import sys
 
 from pyspark import SparkContext
diff --git a/examples/src/main/python/mllib/logistic_regression_with_lbfgs_example.py b/examples/src/main/python/mllib/logistic_regression_with_lbfgs_example.py
index c9b768b3147d2..ac5ab1d1b5d91 100644
--- a/examples/src/main/python/mllib/logistic_regression_with_lbfgs_example.py
+++ b/examples/src/main/python/mllib/logistic_regression_with_lbfgs_example.py
@@ -18,8 +18,6 @@
 """
 Logistic Regression With LBFGS Example.
 """
-from __future__ import print_function
-
 from pyspark import SparkContext
 # $example on$
 from pyspark.mllib.classification import LogisticRegressionWithLBFGS, LogisticRegressionModel
diff --git a/examples/src/main/python/mllib/naive_bayes_example.py b/examples/src/main/python/mllib/naive_bayes_example.py
index a29fcccac5bfc..74d18233d533a 100644
--- a/examples/src/main/python/mllib/naive_bayes_example.py
+++ b/examples/src/main/python/mllib/naive_bayes_example.py
@@ -22,8 +22,6 @@
   `spark-submit --master local[4] examples/src/main/python/mllib/naive_bayes_example.py`
 """
 
-from __future__ import print_function
-
 import shutil
 
 from pyspark import SparkContext
diff --git a/examples/src/main/python/mllib/normalizer_example.py b/examples/src/main/python/mllib/normalizer_example.py
index a4e028ca9af8b..d46110d9a0300 100644
--- a/examples/src/main/python/mllib/normalizer_example.py
+++ b/examples/src/main/python/mllib/normalizer_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 from pyspark import SparkContext
 # $example on$
 from pyspark.mllib.feature import Normalizer
diff --git a/examples/src/main/python/mllib/power_iteration_clustering_example.py b/examples/src/main/python/mllib/power_iteration_clustering_example.py
index ca19c0ccb60c8..60eedef5fab30 100644
--- a/examples/src/main/python/mllib/power_iteration_clustering_example.py
+++ b/examples/src/main/python/mllib/power_iteration_clustering_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 from pyspark import SparkContext
 # $example on$
 from pyspark.mllib.clustering import PowerIterationClustering, PowerIterationClusteringModel
diff --git a/examples/src/main/python/mllib/random_forest_classification_example.py b/examples/src/main/python/mllib/random_forest_classification_example.py
index 5ac67520daee0..a929c10d5a573 100644
--- a/examples/src/main/python/mllib/random_forest_classification_example.py
+++ b/examples/src/main/python/mllib/random_forest_classification_example.py
@@ -18,8 +18,6 @@
 """
 Random Forest Classification Example.
 """
-from __future__ import print_function
-
 from pyspark import SparkContext
 # $example on$
 from pyspark.mllib.tree import RandomForest, RandomForestModel
diff --git a/examples/src/main/python/mllib/random_forest_regression_example.py b/examples/src/main/python/mllib/random_forest_regression_example.py
index 7e986a0d307f0..4e05937768211 100644
--- a/examples/src/main/python/mllib/random_forest_regression_example.py
+++ b/examples/src/main/python/mllib/random_forest_regression_example.py
@@ -18,8 +18,6 @@
 """
 Random Forest Regression Example.
 """
-from __future__ import print_function
-
 from pyspark import SparkContext
 # $example on$
 from pyspark.mllib.tree import RandomForest, RandomForestModel
diff --git a/examples/src/main/python/mllib/random_rdd_generation.py b/examples/src/main/python/mllib/random_rdd_generation.py
index 9a429b5f8abdf..49afcfe9391ab 100755
--- a/examples/src/main/python/mllib/random_rdd_generation.py
+++ b/examples/src/main/python/mllib/random_rdd_generation.py
@@ -18,8 +18,6 @@
 """
 Randomly generated RDDs.
 """
-from __future__ import print_function
-
 import sys
 
 from pyspark import SparkContext
diff --git a/examples/src/main/python/mllib/recommendation_example.py b/examples/src/main/python/mllib/recommendation_example.py
index 00e683c3ae938..719f3f904b246 100644
--- a/examples/src/main/python/mllib/recommendation_example.py
+++ b/examples/src/main/python/mllib/recommendation_example.py
@@ -18,8 +18,6 @@
 """
 Collaborative Filtering Classification Example.
 """
-from __future__ import print_function
-
 from pyspark import SparkContext
 
 # $example on$
diff --git a/examples/src/main/python/mllib/sampled_rdds.py b/examples/src/main/python/mllib/sampled_rdds.py
index 00e7cf4bbcdbf..9095c2b2d70d6 100755
--- a/examples/src/main/python/mllib/sampled_rdds.py
+++ b/examples/src/main/python/mllib/sampled_rdds.py
@@ -18,8 +18,6 @@
 """
 Randomly sampled RDDs.
 """
-from __future__ import print_function
-
 import sys
 
 from pyspark import SparkContext
diff --git a/examples/src/main/python/mllib/standard_scaler_example.py b/examples/src/main/python/mllib/standard_scaler_example.py
index 11ed34427dfe2..c8fd64dfbbf4a 100644
--- a/examples/src/main/python/mllib/standard_scaler_example.py
+++ b/examples/src/main/python/mllib/standard_scaler_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 from pyspark import SparkContext
 # $example on$
 from pyspark.mllib.feature import StandardScaler
diff --git a/examples/src/main/python/mllib/stratified_sampling_example.py b/examples/src/main/python/mllib/stratified_sampling_example.py
index a13f8f08dd68b..2d29f74a19c1a 100644
--- a/examples/src/main/python/mllib/stratified_sampling_example.py
+++ b/examples/src/main/python/mllib/stratified_sampling_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 from pyspark import SparkContext
 
 if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/streaming_k_means_example.py b/examples/src/main/python/mllib/streaming_k_means_example.py
index e82509ad3ffb6..4904a9ebcf544 100644
--- a/examples/src/main/python/mllib/streaming_k_means_example.py
+++ b/examples/src/main/python/mllib/streaming_k_means_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 from pyspark import SparkContext
 from pyspark.streaming import StreamingContext
 # $example on$
diff --git a/examples/src/main/python/mllib/streaming_linear_regression_example.py b/examples/src/main/python/mllib/streaming_linear_regression_example.py
index 714c9a0de7217..1d52e00fbfb5e 100644
--- a/examples/src/main/python/mllib/streaming_linear_regression_example.py
+++ b/examples/src/main/python/mllib/streaming_linear_regression_example.py
@@ -18,8 +18,6 @@
 """
 Streaming Linear Regression Example.
 """
-from __future__ import print_function
-
 # $example on$
 import sys
 # $example off$
diff --git a/examples/src/main/python/mllib/summary_statistics_example.py b/examples/src/main/python/mllib/summary_statistics_example.py
index d55d1a2c2d0e1..d86e841145501 100644
--- a/examples/src/main/python/mllib/summary_statistics_example.py
+++ b/examples/src/main/python/mllib/summary_statistics_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 from pyspark import SparkContext
 # $example on$
 import numpy as np
diff --git a/examples/src/main/python/mllib/tf_idf_example.py b/examples/src/main/python/mllib/tf_idf_example.py
index b66412b2334e7..4449066f5b0a6 100644
--- a/examples/src/main/python/mllib/tf_idf_example.py
+++ b/examples/src/main/python/mllib/tf_idf_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 from pyspark import SparkContext
 # $example on$
 from pyspark.mllib.feature import HashingTF, IDF
diff --git a/examples/src/main/python/mllib/word2vec.py b/examples/src/main/python/mllib/word2vec.py
index 4e7d4f7610c24..3e5720b4df4d6 100644
--- a/examples/src/main/python/mllib/word2vec.py
+++ b/examples/src/main/python/mllib/word2vec.py
@@ -23,8 +23,6 @@
 # grep -o -E '\w+(\W+\w+){0,15}' text8 > text8_lines
 # This was done so that the example can be run in local mode
 
-from __future__ import print_function
-
 import sys
 
 from pyspark import SparkContext
diff --git a/examples/src/main/python/mllib/word2vec_example.py b/examples/src/main/python/mllib/word2vec_example.py
index ad1090c77ee11..d37a6e7137b8f 100644
--- a/examples/src/main/python/mllib/word2vec_example.py
+++ b/examples/src/main/python/mllib/word2vec_example.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 from pyspark import SparkContext
 # $example on$
 from pyspark.mllib.feature import Word2Vec
diff --git a/examples/src/main/python/pagerank.py b/examples/src/main/python/pagerank.py
index 2c19e8700ab16..0ab7249a82185 100755
--- a/examples/src/main/python/pagerank.py
+++ b/examples/src/main/python/pagerank.py
@@ -22,8 +22,6 @@
 Example Usage:
 bin/spark-submit examples/src/main/python/pagerank.py data/mllib/pagerank_data.txt 10
 """
-from __future__ import print_function
-
 import re
 import sys
 from operator import add
diff --git a/examples/src/main/python/parquet_inputformat.py b/examples/src/main/python/parquet_inputformat.py
index 83041f0040a0c..ca8dd25e6dabf 100644
--- a/examples/src/main/python/parquet_inputformat.py
+++ b/examples/src/main/python/parquet_inputformat.py
@@ -29,8 +29,6 @@
 {u'favorite_color': u'red', u'name': u'Ben', u'favorite_numbers': []}
 <...more log output...>
 """
-from __future__ import print_function
-
 import sys
 
 from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/pi.py b/examples/src/main/python/pi.py
index 5839cc2874956..e646722533f68 100755
--- a/examples/src/main/python/pi.py
+++ b/examples/src/main/python/pi.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 import sys
 from random import random
 from operator import add
diff --git a/examples/src/main/python/sort.py b/examples/src/main/python/sort.py
index d3cd985d197e3..9efb00a6f1532 100755
--- a/examples/src/main/python/sort.py
+++ b/examples/src/main/python/sort.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 import sys
 
 from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/sql/arrow.py b/examples/src/main/python/sql/arrow.py
index b7d8467172fab..a0eba0fbede73 100644
--- a/examples/src/main/python/sql/arrow.py
+++ b/examples/src/main/python/sql/arrow.py
@@ -21,9 +21,8 @@
   ./bin/spark-submit examples/src/main/python/sql/arrow.py
 """
 
-from __future__ import print_function
-
-import sys
+# NOTE that this file is imported in user guide in PySpark documentation.
+# The codes are referred via line numbers. See also `literalinclude` directive in Sphinx.
 
 from pyspark.sql import SparkSession
 from pyspark.sql.pandas.utils import require_minimum_pandas_version, require_minimum_pyarrow_version
@@ -31,16 +30,10 @@
 require_minimum_pandas_version()
 require_minimum_pyarrow_version()
 
-if sys.version_info < (3, 6):
-    raise Exception(
-        "Running this example file requires Python 3.6+; however, "
-        "your Python version was:\n %s" % sys.version)
-
 
 def dataframe_with_arrow_example(spark):
-    # $example on:dataframe_with_arrow$
-    import numpy as np
-    import pandas as pd
+    import numpy as np  # type: ignore[import]
+    import pandas as pd  # type: ignore[import]
 
     # Enable Arrow-based columnar data transfers
     spark.conf.set("spark.sql.execution.arrow.pyspark.enabled", "true")
@@ -53,12 +46,11 @@ def dataframe_with_arrow_example(spark):
 
     # Convert the Spark DataFrame back to a Pandas DataFrame using Arrow
     result_pdf = df.select("*").toPandas()
-    # $example off:dataframe_with_arrow$
+
     print("Pandas DataFrame result statistics:\n%s\n" % str(result_pdf.describe()))
 
 
 def ser_to_frame_pandas_udf_example(spark):
-    # $example on:ser_to_frame_pandas_udf$
     import pandas as pd
 
     from pyspark.sql.functions import pandas_udf
@@ -68,7 +60,7 @@ def func(s1: pd.Series, s2: pd.Series, s3: pd.DataFrame) -> pd.DataFrame:
         s3['col2'] = s1 + s2.str.len()
         return s3
 
-    # Create a Spark DataFrame that has three columns including a sturct column.
+    # Create a Spark DataFrame that has three columns including a struct column.
     df = spark.createDataFrame(
         [[1, "a string", ("a nested string",)]],
         "long_col long, string_col string, struct_col struct<col1:string>")
@@ -84,11 +76,9 @@ def func(s1: pd.Series, s2: pd.Series, s3: pd.DataFrame) -> pd.DataFrame:
     # |-- func(long_col, string_col, struct_col): struct (nullable = true)
     # |    |-- col1: string (nullable = true)
     # |    |-- col2: long (nullable = true)
-    # $example off:ser_to_frame_pandas_udf$$
 
 
 def ser_to_ser_pandas_udf_example(spark):
-    # $example on:ser_to_ser_pandas_udf$
     import pandas as pd
 
     from pyspark.sql.functions import col, pandas_udf
@@ -120,11 +110,9 @@ def multiply_func(a: pd.Series, b: pd.Series) -> pd.Series:
     # |                  4|
     # |                  9|
     # +-------------------+
-    # $example off:ser_to_ser_pandas_udf$
 
 
 def iter_ser_to_iter_ser_pandas_udf_example(spark):
-    # $example on:iter_ser_to_iter_ser_pandas_udf$
     from typing import Iterator
 
     import pandas as pd
@@ -148,11 +136,9 @@ def plus_one(iterator: Iterator[pd.Series]) -> Iterator[pd.Series]:
     # |          3|
     # |          4|
     # +-----------+
-    # $example off:iter_ser_to_iter_ser_pandas_udf$
 
 
 def iter_sers_to_iter_ser_pandas_udf_example(spark):
-    # $example on:iter_sers_to_iter_ser_pandas_udf$
     from typing import Iterator, Tuple
 
     import pandas as pd
@@ -177,11 +163,9 @@ def multiply_two_cols(
     # |                      4|
     # |                      9|
     # +-----------------------+
-    # $example off:iter_sers_to_iter_ser_pandas_udf$
 
 
 def ser_to_scalar_pandas_udf_example(spark):
-    # $example on:ser_to_scalar_pandas_udf$
     import pandas as pd
 
     from pyspark.sql.functions import pandas_udf
@@ -224,11 +208,9 @@ def mean_udf(v: pd.Series) -> float:
     # |  2| 5.0|   6.0|
     # |  2|10.0|   6.0|
     # +---+----+------+
-    # $example off:ser_to_scalar_pandas_udf$
 
 
 def grouped_apply_in_pandas_example(spark):
-    # $example on:grouped_apply_in_pandas$
     df = spark.createDataFrame(
         [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)],
         ("id", "v"))
@@ -248,11 +230,9 @@ def subtract_mean(pdf):
     # |  2|-1.0|
     # |  2| 4.0|
     # +---+----+
-    # $example off:grouped_apply_in_pandas$
 
 
 def map_in_pandas_example(spark):
-    # $example on:map_in_pandas$
     df = spark.createDataFrame([(1, 21), (2, 30)], ("id", "age"))
 
     def filter_func(iterator):
@@ -265,11 +245,9 @@ def filter_func(iterator):
     # +---+---+
     # |  1| 21|
     # +---+---+
-    # $example off:map_in_pandas$
 
 
 def cogrouped_apply_in_pandas_example(spark):
-    # $example on:cogrouped_apply_in_pandas$
     import pandas as pd
 
     df1 = spark.createDataFrame(
@@ -293,7 +271,6 @@ def asof_join(l, r):
     # |20000101|  2|2.0|  y|
     # |20000102|  2|4.0|  y|
     # +--------+---+---+---+
-    # $example off:cogrouped_apply_in_pandas$
 
 
 if __name__ == "__main__":
@@ -308,7 +285,7 @@ def asof_join(l, r):
     ser_to_frame_pandas_udf_example(spark)
     print("Running pandas_udf example: Series to Series")
     ser_to_ser_pandas_udf_example(spark)
-    print("Running pandas_udf example: Iterator of Series to Iterator of Seires")
+    print("Running pandas_udf example: Iterator of Series to Iterator of Series")
     iter_ser_to_iter_ser_pandas_udf_example(spark)
     print("Running pandas_udf example: Iterator of Multiple Series to Iterator of Series")
     iter_sers_to_iter_ser_pandas_udf_example(spark)
diff --git a/examples/src/main/python/sql/basic.py b/examples/src/main/python/sql/basic.py
index c8fb25d0533b5..2667f9acab521 100644
--- a/examples/src/main/python/sql/basic.py
+++ b/examples/src/main/python/sql/basic.py
@@ -20,8 +20,6 @@
 Run with:
   ./bin/spark-submit examples/src/main/python/sql/basic.py
 """
-from __future__ import print_function
-
 # $example on:init_session$
 from pyspark.sql import SparkSession
 # $example off:init_session$
@@ -32,7 +30,7 @@
 
 # $example on:programmatic_schema$
 # Import data types
-from pyspark.sql.types import *
+from pyspark.sql.types import StringType, StructType, StructField
 # $example off:programmatic_schema$
 
 
diff --git a/examples/src/main/python/sql/datasource.py b/examples/src/main/python/sql/datasource.py
index 265f135e1e5f2..4d7aa045b4b87 100644
--- a/examples/src/main/python/sql/datasource.py
+++ b/examples/src/main/python/sql/datasource.py
@@ -20,8 +20,6 @@
 Run with:
   ./bin/spark-submit examples/src/main/python/sql/datasource.py
 """
-from __future__ import print_function
-
 from pyspark.sql import SparkSession
 # $example on:schema_merging$
 from pyspark.sql import Row
@@ -69,6 +67,26 @@ def generic_file_source_options_example(spark):
     # +-------------+
     # $example off:load_with_path_glob_filter$
 
+    # $example on:load_with_modified_time_filter$
+    # Only load files modified before 07/1/2050 @ 08:30:00
+    df = spark.read.load("examples/src/main/resources/dir1",
+                         format="parquet", modifiedBefore="2050-07-01T08:30:00")
+    df.show()
+    # +-------------+
+    # |         file|
+    # +-------------+
+    # |file1.parquet|
+    # +-------------+
+    # Only load files modified after 06/01/2050 @ 08:30:00
+    df = spark.read.load("examples/src/main/resources/dir1",
+                         format="parquet", modifiedAfter="2050-06-01T08:30:00")
+    df.show()
+    # +-------------+
+    # |         file|
+    # +-------------+
+    # +-------------+
+    # $example off:load_with_modified_time_filter$
+
 
 def basic_datasource_example(spark):
     # $example on:generic_load_save_functions$
@@ -86,7 +104,7 @@ def basic_datasource_example(spark):
         .write
         .partitionBy("favorite_color")
         .bucketBy(42, "name")
-        .saveAsTable("people_partitioned_bucketed"))
+        .saveAsTable("users_partitioned_bucketed"))
     # $example off:write_partition_and_bucket$
 
     # $example on:manual_load_options$
@@ -96,7 +114,7 @@ def basic_datasource_example(spark):
 
     # $example on:manual_load_options_csv$
     df = spark.read.load("examples/src/main/resources/people.csv",
-                         format="csv", sep=":", inferSchema="true", header="true")
+                         format="csv", sep=";", inferSchema="true", header="true")
     # $example off:manual_load_options_csv$
 
     # $example on:manual_save_options_orc$
@@ -108,6 +126,16 @@ def basic_datasource_example(spark):
         .save("users_with_options.orc"))
     # $example off:manual_save_options_orc$
 
+    # $example on:manual_save_options_parquet$
+    df = spark.read.parquet("examples/src/main/resources/users.parquet")
+    (df.write.format("parquet")
+        .option("parquet.bloom.filter.enabled#favorite_color", "true")
+        .option("parquet.bloom.filter.expected.ndv#favorite_color", "1000000")
+        .option("parquet.enable.dictionary", "true")
+        .option("parquet.page.write-checksum.enabled", "false")
+        .save("users_with_options.parquet"))
+    # $example off:manual_save_options_parquet$
+
     # $example on:write_sorting_and_bucketing$
     df.write.bucketBy(42, "name").sortBy("age").saveAsTable("people_bucketed")
     # $example off:write_sorting_and_bucketing$
@@ -117,7 +145,7 @@ def basic_datasource_example(spark):
     # $example off:direct_sql$
 
     spark.sql("DROP TABLE IF EXISTS people_bucketed")
-    spark.sql("DROP TABLE IF EXISTS people_partitioned_bucketed")
+    spark.sql("DROP TABLE IF EXISTS users_partitioned_bucketed")
 
 
 def parquet_example(spark):
@@ -216,6 +244,120 @@ def json_dataset_example(spark):
     # $example off:json_dataset$
 
 
+def csv_dataset_example(spark):
+    # $example on:csv_dataset$
+    # spark is from the previous example
+    sc = spark.sparkContext
+
+    # A CSV dataset is pointed to by path.
+    # The path can be either a single CSV file or a directory of CSV files
+    path = "examples/src/main/resources/people.csv"
+
+    df = spark.read.csv(path)
+    df.show()
+    # +------------------+
+    # |               _c0|
+    # +------------------+
+    # |      name;age;job|
+    # |Jorge;30;Developer|
+    # |  Bob;32;Developer|
+    # +------------------+
+
+    # Read a csv with delimiter, the default delimiter is ","
+    df2 = spark.read.option(delimiter=';').csv(path)
+    df2.show()
+    # +-----+---+---------+
+    # |  _c0|_c1|      _c2|
+    # +-----+---+---------+
+    # | name|age|      job|
+    # |Jorge| 30|Developer|
+    # |  Bob| 32|Developer|
+    # +-----+---+---------+
+
+    # Read a csv with delimiter and a header
+    df3 = spark.read.option("delimiter", ";").option("header", True).csv(path)
+    df3.show()
+    # +-----+---+---------+
+    # | name|age|      job|
+    # +-----+---+---------+
+    # |Jorge| 30|Developer|
+    # |  Bob| 32|Developer|
+    # +-----+---+---------+
+
+    # You can also use options() to use multiple options
+    df4 = spark.read.options(delimiter=";", header=True).csv(path)
+
+    # "output" is a folder which contains multiple csv files and a _SUCCESS file.
+    df3.write.csv("output")
+
+    # Read all files in a folder, please make sure only CSV files should present in the folder.
+    folderPath = "examples/src/main/resources"
+    df5 = spark.read.csv(folderPath)
+    df5.show()
+    # Wrong schema because non-CSV files are read
+    # +-----------+
+    # |        _c0|
+    # +-----------+
+    # |238val_238|
+    # |  86val_86|
+    # |311val_311|
+    # |  27val_27|
+    # |165val_165|
+    # +-----------+
+
+    # $example off:csv_dataset$
+
+
+def text_dataset_example(spark):
+    # $example on:text_dataset$
+    # spark is from the previous example
+    sc = spark.sparkContext
+
+    # A text dataset is pointed to by path.
+    # The path can be either a single text file or a directory of text files
+    path = "examples/src/main/resources/people.txt"
+
+    df1 = spark.read.text(path)
+    df1.show()
+    # +-----------+
+    # |      value|
+    # +-----------+
+    # |Michael, 29|
+    # |   Andy, 30|
+    # | Justin, 19|
+    # +-----------+
+
+    # You can use 'lineSep' option to define the line separator.
+    # The line separator handles all `\r`, `\r\n` and `\n` by default.
+    df2 = spark.read.text(path, lineSep=",")
+    df2.show()
+    # +-----------+
+    # |      value|
+    # +-----------+
+    # |    Michael|
+    # |   29\nAndy|
+    # | 30\nJustin|
+    # |       19\n|
+    # +-----------+
+
+    # You can also use 'wholetext' option to read each input file as a single row.
+    df3 = spark.read.text(path, wholetext=True)
+    df3.show()
+    # +--------------------+
+    # |               value|
+    # +--------------------+
+    # |Michael, 29\nAndy...|
+    # +--------------------+
+
+    # "output" is a folder which contains multiple text files and a _SUCCESS file.
+    df1.write.csv("output")
+
+    # You can specify the compression format using the 'compression' option.
+    df1.write.text("output_compressed", compression="gzip")
+
+    # $example off:text_dataset$
+
+
 def jdbc_dataset_example(spark):
     # $example on:jdbc_dataset$
     # Note: JDBC loading and saving can be achieved via either the load/save or jdbc methods
@@ -274,6 +416,8 @@ def jdbc_dataset_example(spark):
     parquet_example(spark)
     parquet_schema_merging_example(spark)
     json_dataset_example(spark)
+    csv_dataset_example(spark)
+    text_dataset_example(spark)
     jdbc_dataset_example(spark)
 
     spark.stop()
diff --git a/examples/src/main/python/sql/hive.py b/examples/src/main/python/sql/hive.py
index e96a8af71adc3..fa1b975e2bfdc 100644
--- a/examples/src/main/python/sql/hive.py
+++ b/examples/src/main/python/sql/hive.py
@@ -20,10 +20,8 @@
 Run with:
   ./bin/spark-submit examples/src/main/python/sql/hive.py
 """
-from __future__ import print_function
-
 # $example on:spark_hive$
-from os.path import join, abspath
+from os.path import abspath
 
 from pyspark.sql import SparkSession
 from pyspark.sql import Row
diff --git a/examples/src/main/python/sql/streaming/structured_kafka_wordcount.py b/examples/src/main/python/sql/streaming/structured_kafka_wordcount.py
index 921067891352a..40a955a46c9b9 100644
--- a/examples/src/main/python/sql/streaming/structured_kafka_wordcount.py
+++ b/examples/src/main/python/sql/streaming/structured_kafka_wordcount.py
@@ -36,8 +36,6 @@
     `$ bin/spark-submit examples/src/main/python/sql/streaming/structured_kafka_wordcount.py \
     host1:port1,host2:port2 subscribe topic1,topic2`
 """
-from __future__ import print_function
-
 import sys
 
 from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/sql/streaming/structured_network_wordcount.py b/examples/src/main/python/sql/streaming/structured_network_wordcount.py
index 9ac392164735b..c8f43c9dcf2eb 100644
--- a/examples/src/main/python/sql/streaming/structured_network_wordcount.py
+++ b/examples/src/main/python/sql/streaming/structured_network_wordcount.py
@@ -27,8 +27,6 @@
     `$ bin/spark-submit examples/src/main/python/sql/streaming/structured_network_wordcount.py
     localhost 9999`
 """
-from __future__ import print_function
-
 import sys
 
 from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/sql/streaming/structured_network_wordcount_windowed.py b/examples/src/main/python/sql/streaming/structured_network_wordcount_windowed.py
index c4e3bbf44cd5a..cc39d8afa6be9 100644
--- a/examples/src/main/python/sql/streaming/structured_network_wordcount_windowed.py
+++ b/examples/src/main/python/sql/streaming/structured_network_wordcount_windowed.py
@@ -39,8 +39,6 @@
 
  One recommended <window duration>, <slide duration> pair is 10, 5
 """
-from __future__ import print_function
-
 import sys
 
 from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/status_api_demo.py b/examples/src/main/python/status_api_demo.py
index 8cc8cc820cfce..ae39cef6ebbad 100644
--- a/examples/src/main/python/status_api_demo.py
+++ b/examples/src/main/python/status_api_demo.py
@@ -15,15 +15,9 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 import time
 import threading
-import sys
-if sys.version >= '3':
-    import queue as Queue
-else:
-    import Queue
+import queue as Queue
 
 from pyspark import SparkConf, SparkContext
 
diff --git a/examples/src/main/python/streaming/hdfs_wordcount.py b/examples/src/main/python/streaming/hdfs_wordcount.py
index f9a5c43a8eaa9..fac07727b7b12 100644
--- a/examples/src/main/python/streaming/hdfs_wordcount.py
+++ b/examples/src/main/python/streaming/hdfs_wordcount.py
@@ -25,8 +25,6 @@
 
  Then create a text file in `localdir` and the words in the file will get counted.
 """
-from __future__ import print_function
-
 import sys
 
 from pyspark import SparkContext
diff --git a/examples/src/main/python/streaming/network_wordcount.py b/examples/src/main/python/streaming/network_wordcount.py
index f3099d2517cd5..b57f4e9e38b82 100644
--- a/examples/src/main/python/streaming/network_wordcount.py
+++ b/examples/src/main/python/streaming/network_wordcount.py
@@ -25,8 +25,6 @@
  and then run the example
     `$ bin/spark-submit examples/src/main/python/streaming/network_wordcount.py localhost 9999`
 """
-from __future__ import print_function
-
 import sys
 
 from pyspark import SparkContext
diff --git a/examples/src/main/python/streaming/network_wordjoinsentiments.py b/examples/src/main/python/streaming/network_wordjoinsentiments.py
index 2b5434c0c845a..5b03546fb4d83 100644
--- a/examples/src/main/python/streaming/network_wordjoinsentiments.py
+++ b/examples/src/main/python/streaming/network_wordjoinsentiments.py
@@ -30,8 +30,6 @@
     localhost 9999`
 """
 
-from __future__ import print_function
-
 import sys
 
 from pyspark import SparkContext
diff --git a/examples/src/main/python/streaming/recoverable_network_wordcount.py b/examples/src/main/python/streaming/recoverable_network_wordcount.py
index a39c4d0b5b8cd..567f9c819e3ad 100644
--- a/examples/src/main/python/streaming/recoverable_network_wordcount.py
+++ b/examples/src/main/python/streaming/recoverable_network_wordcount.py
@@ -35,8 +35,6 @@
  checkpoint data exists in ~/checkpoint/, then it will create StreamingContext from
  the checkpoint data.
 """
-from __future__ import print_function
-
 import os
 import sys
 
@@ -45,10 +43,10 @@
 
 
 # Get or register a Broadcast variable
-def getWordBlacklist(sparkContext):
-    if ('wordBlacklist' not in globals()):
-        globals()['wordBlacklist'] = sparkContext.broadcast(["a", "b", "c"])
-    return globals()['wordBlacklist']
+def getWordExcludeList(sparkContext):
+    if ('wordExcludeList' not in globals()):
+        globals()['wordExcludeList'] = sparkContext.broadcast(["a", "b", "c"])
+    return globals()['wordExcludeList']
 
 
 # Get or register an Accumulator
@@ -68,20 +66,20 @@ def createContext(host, port, outputPath):
     ssc = StreamingContext(sc, 1)
 
     # Create a socket stream on target ip:port and count the
-    # words in input stream of \n delimited text (eg. generated by 'nc')
+    # words in input stream of \n delimited text (e.g. generated by 'nc')
     lines = ssc.socketTextStream(host, port)
     words = lines.flatMap(lambda line: line.split(" "))
     wordCounts = words.map(lambda x: (x, 1)).reduceByKey(lambda x, y: x + y)
 
     def echo(time, rdd):
-        # Get or register the blacklist Broadcast
-        blacklist = getWordBlacklist(rdd.context)
+        # Get or register the excludeList Broadcast
+        excludeList = getWordExcludeList(rdd.context)
         # Get or register the droppedWordsCounter Accumulator
         droppedWordsCounter = getDroppedWordsCounter(rdd.context)
 
-        # Use blacklist to drop words and use droppedWordsCounter to count them
+        # Use excludeList to drop words and use droppedWordsCounter to count them
         def filterFunc(wordCount):
-            if wordCount[0] in blacklist.value:
+            if wordCount[0] in excludeList.value:
                 droppedWordsCounter.add(wordCount[1])
                 return False
             else:
diff --git a/examples/src/main/python/streaming/sql_network_wordcount.py b/examples/src/main/python/streaming/sql_network_wordcount.py
index ab3cfc067994d..2965ea8fb1872 100644
--- a/examples/src/main/python/streaming/sql_network_wordcount.py
+++ b/examples/src/main/python/streaming/sql_network_wordcount.py
@@ -27,8 +27,6 @@
  and then run the example
     `$ bin/spark-submit examples/src/main/python/streaming/sql_network_wordcount.py localhost 9999`
 """
-from __future__ import print_function
-
 import sys
 
 from pyspark import SparkContext
@@ -54,7 +52,7 @@ def getSparkSessionInstance(sparkConf):
     ssc = StreamingContext(sc, 1)
 
     # Create a socket stream on target ip:port and count the
-    # words in input stream of \n delimited text (eg. generated by 'nc')
+    # words in input stream of \n delimited text (e.g. generated by 'nc')
     lines = ssc.socketTextStream(host, int(port))
     words = lines.flatMap(lambda line: line.split(" "))
 
diff --git a/examples/src/main/python/streaming/stateful_network_wordcount.py b/examples/src/main/python/streaming/stateful_network_wordcount.py
index d5d1eba6c5969..7a45be663a765 100644
--- a/examples/src/main/python/streaming/stateful_network_wordcount.py
+++ b/examples/src/main/python/streaming/stateful_network_wordcount.py
@@ -29,8 +29,6 @@
     `$ bin/spark-submit examples/src/main/python/streaming/stateful_network_wordcount.py \
         localhost 9999`
 """
-from __future__ import print_function
-
 import sys
 
 from pyspark import SparkContext
diff --git a/examples/src/main/python/transitive_closure.py b/examples/src/main/python/transitive_closure.py
index 49551d40851cc..9f543daecd3dd 100755
--- a/examples/src/main/python/transitive_closure.py
+++ b/examples/src/main/python/transitive_closure.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 import sys
 from random import Random
 
diff --git a/examples/src/main/python/wordcount.py b/examples/src/main/python/wordcount.py
index a05e24ff3ff95..037c1e8aa379d 100755
--- a/examples/src/main/python/wordcount.py
+++ b/examples/src/main/python/wordcount.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 import sys
 from operator import add
 
diff --git a/examples/src/main/r/RSparkSQLExample.R b/examples/src/main/r/RSparkSQLExample.R
index 8685cfb5c05f2..15118e118ab3a 100644
--- a/examples/src/main/r/RSparkSQLExample.R
+++ b/examples/src/main/r/RSparkSQLExample.R
@@ -144,11 +144,24 @@ df <- read.df("examples/src/main/resources/dir1", "parquet", pathGlobFilter = "*
 # 1 file1.parquet
 # $example off:load_with_path_glob_filter$
 
+# $example on:load_with_modified_time_filter$
+beforeDF <- read.df("examples/src/main/resources/dir1", "parquet", modifiedBefore= "2020-07-01T05:30:00")
+#            file
+# 1 file1.parquet
+afterDF <- read.df("examples/src/main/resources/dir1", "parquet", modifiedAfter = "2020-06-01T05:30:00")
+#            file
+# $example off:load_with_modified_time_filter$
+
 # $example on:manual_save_options_orc$
 df <- read.df("examples/src/main/resources/users.orc", "orc")
 write.orc(df, "users_with_options.orc", orc.bloom.filter.columns = "favorite_color", orc.dictionary.key.threshold = 1.0, orc.column.encoding.direct = "name")
 # $example off:manual_save_options_orc$
 
+# $example on:manual_save_options_parquet$
+df <- read.df("examples/src/main/resources/users.parquet", "parquet")
+write.parquet(df, "users_with_options.parquet", parquet.bloom.filter.enabled#favorite_color = true, parquet.bloom.filter.expected.ndv#favorite_color = 1000000, parquet.enable.dictionary = true, parquet.page.write-checksum.enabled = false)
+# $example off:manual_save_options_parquet$
+
 # $example on:direct_sql$
 df <- sql("SELECT * FROM parquet.`examples/src/main/resources/users.parquet`")
 # $example off:direct_sql$
diff --git a/examples/src/main/resources/META-INF/services/org.apache.spark.sql.SparkSessionExtensionsProvider b/examples/src/main/resources/META-INF/services/org.apache.spark.sql.SparkSessionExtensionsProvider
new file mode 100644
index 0000000000000..c239843a3b502
--- /dev/null
+++ b/examples/src/main/resources/META-INF/services/org.apache.spark.sql.SparkSessionExtensionsProvider
@@ -0,0 +1 @@
+org.apache.spark.examples.extensions.SessionExtensionsWithLoader
diff --git a/examples/src/main/resources/META-INF/services/org.apache.spark.sql.jdbc.JdbcConnectionProvider b/examples/src/main/resources/META-INF/services/org.apache.spark.sql.jdbc.JdbcConnectionProvider
new file mode 100644
index 0000000000000..776948cc04de7
--- /dev/null
+++ b/examples/src/main/resources/META-INF/services/org.apache.spark.sql.jdbc.JdbcConnectionProvider
@@ -0,0 +1 @@
+org.apache.spark.examples.sql.jdbc.ExampleJdbcConnectionProvider
\ No newline at end of file
diff --git a/examples/src/main/scala/org/apache/spark/examples/DFSReadWriteTest.scala b/examples/src/main/scala/org/apache/spark/examples/DFSReadWriteTest.scala
index a73859891a860..323bab427e55e 100644
--- a/examples/src/main/scala/org/apache/spark/examples/DFSReadWriteTest.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/DFSReadWriteTest.scala
@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.FileSystem
 import org.apache.hadoop.fs.Path
 
 import org.apache.spark.sql.SparkSession
+import org.apache.spark.util.Utils
 
 /**
  * Simple test for reading and writing to a distributed
@@ -46,9 +47,7 @@ object DFSReadWriteTest {
   private val NPARAMS = 2
 
   private def readFile(filename: String): List[String] = {
-    val lineIter: Iterator[String] = fromFile(filename).getLines()
-    val lineList: List[String] = lineIter.toList
-    lineList
+    Utils.tryWithResource(fromFile(filename))(_.getLines().toList)
   }
 
   private def printUsage(): Unit = {
diff --git a/examples/src/main/scala/org/apache/spark/examples/LocalKMeans.scala b/examples/src/main/scala/org/apache/spark/examples/LocalKMeans.scala
index 4a73466841f69..f649c310b34ac 100644
--- a/examples/src/main/scala/org/apache/spark/examples/LocalKMeans.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/LocalKMeans.scala
@@ -101,7 +101,7 @@ object LocalKMeans {
         }
       }
 
-      var newPoints = pointStats.map {mapping =>
+      val newPoints = pointStats.map { mapping =>
         (mapping._1, mapping._2._1 * (1.0 / mapping._2._2))}
 
       tempDist = 0.0
diff --git a/examples/src/main/scala/org/apache/spark/examples/SparkKMeans.scala b/examples/src/main/scala/org/apache/spark/examples/SparkKMeans.scala
index ec9b44ce6e3b7..cf03e0203f771 100644
--- a/examples/src/main/scala/org/apache/spark/examples/SparkKMeans.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/SparkKMeans.scala
@@ -82,7 +82,7 @@ object SparkKMeans {
     while(tempDist > convergeDist) {
       val closest = data.map (p => (closestPoint(p, kPoints), (p, 1)))
 
-      val pointStats = closest.reduceByKey{case ((p1, c1), (p2, c2)) => (p1 + p2, c1 + c2)}
+      val pointStats = closest.reduceByKey(mergeResults)
 
       val newPoints = pointStats.map {pair =>
         (pair._1, pair._2._1 * (1.0 / pair._2._2))}.collectAsMap()
@@ -102,5 +102,11 @@ object SparkKMeans {
     kPoints.foreach(println)
     spark.stop()
   }
+
+  private def mergeResults(
+      a: (Vector[Double], Int),
+      b: (Vector[Double], Int)): (Vector[Double], Int) = {
+    (a._1 + b._1, a._2 + b._2)
+  }
 }
 // scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/extensions/AgeExample.scala b/examples/src/main/scala/org/apache/spark/examples/extensions/AgeExample.scala
new file mode 100644
index 0000000000000..d25f2204994c7
--- /dev/null
+++ b/examples/src/main/scala/org/apache/spark/examples/extensions/AgeExample.scala
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.examples.extensions
+
+import org.apache.spark.sql.catalyst.expressions.{CurrentDate, Expression, RuntimeReplaceable, SubtractDates}
+
+/**
+ * How old are you in days?
+ */
+case class AgeExample(birthday: Expression, child: Expression) extends RuntimeReplaceable {
+
+  def this(birthday: Expression) = this(birthday, SubtractDates(CurrentDate(), birthday))
+  override def exprsReplaced: Seq[Expression] = Seq(birthday)
+
+  override protected def withNewChildInternal(newChild: Expression): Expression = copy(newChild)
+}
diff --git a/examples/src/main/scala/org/apache/spark/examples/extensions/SessionExtensionsWithLoader.scala b/examples/src/main/scala/org/apache/spark/examples/extensions/SessionExtensionsWithLoader.scala
new file mode 100644
index 0000000000000..0daf7346bc1d1
--- /dev/null
+++ b/examples/src/main/scala/org/apache/spark/examples/extensions/SessionExtensionsWithLoader.scala
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.examples.extensions
+
+import org.apache.spark.sql.{SparkSessionExtensions, SparkSessionExtensionsProvider}
+import org.apache.spark.sql.catalyst.FunctionIdentifier
+import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionInfo}
+
+class SessionExtensionsWithLoader extends SparkSessionExtensionsProvider {
+  override def apply(v1: SparkSessionExtensions): Unit = {
+    v1.injectFunction(
+      (new FunctionIdentifier("age_two"),
+        new ExpressionInfo(classOf[AgeExample].getName,
+        "age_two"), (children: Seq[Expression]) => new AgeExample(children.head)))
+  }
+}
diff --git a/examples/src/main/scala/org/apache/spark/examples/extensions/SessionExtensionsWithoutLoader.scala b/examples/src/main/scala/org/apache/spark/examples/extensions/SessionExtensionsWithoutLoader.scala
new file mode 100644
index 0000000000000..5194c43297473
--- /dev/null
+++ b/examples/src/main/scala/org/apache/spark/examples/extensions/SessionExtensionsWithoutLoader.scala
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.examples.extensions
+
+import org.apache.spark.sql.{SparkSessionExtensions, SparkSessionExtensionsProvider}
+import org.apache.spark.sql.catalyst.FunctionIdentifier
+import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionInfo}
+
+class SessionExtensionsWithoutLoader extends SparkSessionExtensionsProvider {
+  override def apply(v1: SparkSessionExtensions): Unit = {
+    v1.injectFunction(
+      (new FunctionIdentifier("age_one"),
+        new ExpressionInfo(classOf[AgeExample].getName,
+        "age_one"), (children: Seq[Expression]) => new AgeExample(children.head)))
+  }
+}
diff --git a/examples/src/main/scala/org/apache/spark/examples/extensions/SparkSessionExtensionsTest.scala b/examples/src/main/scala/org/apache/spark/examples/extensions/SparkSessionExtensionsTest.scala
new file mode 100644
index 0000000000000..8a906964a4018
--- /dev/null
+++ b/examples/src/main/scala/org/apache/spark/examples/extensions/SparkSessionExtensionsTest.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.examples.extensions
+
+import org.apache.spark.sql.SparkSession
+
+/**
+ * [[SessionExtensionsWithLoader]] is registered in
+ * src/main/resources/META-INF/services/org.apache.spark.sql.SparkSessionExtensionsProvider
+ *
+ * [[SessionExtensionsWithoutLoader]] is registered via spark.sql.extensions
+ */
+object SparkSessionExtensionsTest {
+
+  def main(args: Array[String]): Unit = {
+    val spark = SparkSession
+      .builder()
+      .appName("SparkSessionExtensionsTest")
+      .config("spark.sql.extensions", classOf[SessionExtensionsWithoutLoader].getName)
+      .getOrCreate()
+    spark.sql("SELECT age_one('2018-11-17'), age_two('2018-11-17')").show()
+  }
+}
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/ANOVASelectorExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/ANOVASelectorExample.scala
deleted file mode 100644
index 46803cc78e767..0000000000000
--- a/examples/src/main/scala/org/apache/spark/examples/ml/ANOVASelectorExample.scala
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package org.apache.spark.examples.ml
-
-// $example on$
-import org.apache.spark.ml.feature.ANOVASelector
-import org.apache.spark.ml.linalg.Vectors
-// $example off$
-import org.apache.spark.sql.SparkSession
-
-/**
- * An example for ANOVASelector.
- * Run with
- * {{{
- * bin/run-example ml.ANOVASelectorExample
- * }}}
- */
-object ANOVASelectorExample {
-  def main(args: Array[String]): Unit = {
-    val spark = SparkSession
-      .builder
-      .appName("ANOVASelectorExample")
-      .getOrCreate()
-    import spark.implicits._
-
-    // $example on$
-    val data = Seq(
-      (1, Vectors.dense(1.7, 4.4, 7.6, 5.8, 9.6, 2.3), 3.0),
-      (2, Vectors.dense(8.8, 7.3, 5.7, 7.3, 2.2, 4.1), 2.0),
-      (3, Vectors.dense(1.2, 9.5, 2.5, 3.1, 8.7, 2.5), 3.0),
-      (4, Vectors.dense(3.7, 9.2, 6.1, 4.1, 7.5, 3.8), 2.0),
-      (5, Vectors.dense(8.9, 5.2, 7.8, 8.3, 5.2, 3.0), 4.0),
-      (6, Vectors.dense(7.9, 8.5, 9.2, 4.0, 9.4, 2.1), 4.0)
-    )
-
-    val df = spark.createDataset(data).toDF("id", "features", "label")
-
-    val selector = new ANOVASelector()
-      .setNumTopFeatures(1)
-      .setFeaturesCol("features")
-      .setLabelCol("label")
-      .setOutputCol("selectedFeatures")
-
-    val result = selector.fit(df).transform(df)
-
-    println(s"ANOVASelector output with top ${selector.getNumTopFeatures} features selected")
-    result.show()
-    // $example off$
-
-    spark.stop()
-  }
-}
-// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/ANOVATestExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/ANOVATestExample.scala
deleted file mode 100644
index f0b9f23514d93..0000000000000
--- a/examples/src/main/scala/org/apache/spark/examples/ml/ANOVATestExample.scala
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package org.apache.spark.examples.ml
-
-// $example on$
-import org.apache.spark.ml.linalg.{Vector, Vectors}
-import org.apache.spark.ml.stat.ANOVATest
-// $example off$
-import org.apache.spark.sql.SparkSession
-
-/**
- * An example for ANOVA testing.
- * Run with
- * {{{
- * bin/run-example ml.ANOVATestExample
- * }}}
- */
-object ANOVATestExample {
-
-  def main(args: Array[String]): Unit = {
-    val spark = SparkSession
-      .builder
-      .appName("ANOVATestExample")
-      .getOrCreate()
-    import spark.implicits._
-
-    // $example on$
-    val data = Seq(
-      (3.0, Vectors.dense(1.7, 4.4, 7.6, 5.8, 9.6, 2.3)),
-      (2.0, Vectors.dense(8.8, 7.3, 5.7, 7.3, 2.2, 4.1)),
-      (3.0, Vectors.dense(1.2, 9.5, 2.5, 3.1, 8.7, 2.5)),
-      (2.0, Vectors.dense(3.7, 9.2, 6.1, 4.1, 7.5, 3.8)),
-      (4.0, Vectors.dense(8.9, 5.2, 7.8, 8.3, 5.2, 3.0)),
-      (4.0, Vectors.dense(7.9, 8.5, 9.2, 4.0, 9.4, 2.1))
-    )
-
-    val df = data.toDF("label", "features")
-    val anova = ANOVATest.test(df, "features", "label").head
-    println(s"pValues = ${anova.getAs[Vector](0)}")
-    println(s"degreesOfFreedom ${anova.getSeq[Int](1).mkString("[", ",", "]")}")
-    println(s"fValues ${anova.getAs[Vector](2)}")
-    // $example off$
-
-    spark.stop()
-  }
-}
-// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/DeveloperApiExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/DeveloperApiExample.scala
index 86d00cac9485f..487cb27b93fe8 100644
--- a/examples/src/main/scala/org/apache/spark/examples/ml/DeveloperApiExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/ml/DeveloperApiExample.scala
@@ -41,7 +41,6 @@ object DeveloperApiExample {
       .builder
       .appName("DeveloperApiExample")
       .getOrCreate()
-    import spark.implicits._
 
     // Prepare training data.
     val training = spark.createDataFrame(Seq(
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/FValueSelectorExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/FValueSelectorExample.scala
deleted file mode 100644
index 914d81b79c997..0000000000000
--- a/examples/src/main/scala/org/apache/spark/examples/ml/FValueSelectorExample.scala
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package org.apache.spark.examples.ml
-
-// $example on$
-import org.apache.spark.ml.feature.FValueSelector
-import org.apache.spark.ml.linalg.Vectors
-// $example off$
-import org.apache.spark.sql.SparkSession
-
-/**
- * An example for FValueSelector.
- * Run with
- * {{{
- * bin/run-example ml.FValueSelectorExample
- * }}}
- */
-object FValueSelectorExample {
-  def main(args: Array[String]): Unit = {
-    val spark = SparkSession
-      .builder
-      .appName("FValueSelectorExample")
-      .getOrCreate()
-    import spark.implicits._
-
-    // $example on$
-    val data = Seq(
-      (1, Vectors.dense(6.0, 7.0, 0.0, 7.0, 6.0, 0.0), 4.6),
-      (2, Vectors.dense(0.0, 9.0, 6.0, 0.0, 5.0, 9.0), 6.6),
-      (3, Vectors.dense(0.0, 9.0, 3.0, 0.0, 5.0, 5.0), 5.1),
-      (4, Vectors.dense(0.0, 9.0, 8.0, 5.0, 6.0, 4.0), 7.6),
-      (5, Vectors.dense(8.0, 9.0, 6.0, 5.0, 4.0, 4.0), 9.0),
-      (6, Vectors.dense(8.0, 9.0, 6.0, 4.0, 0.0, 0.0), 9.0)
-    )
-
-    val df = spark.createDataset(data).toDF("id", "features", "label")
-
-    val selector = new FValueSelector()
-      .setNumTopFeatures(1)
-      .setFeaturesCol("features")
-      .setLabelCol("label")
-      .setOutputCol("selectedFeatures")
-
-    val result = selector.fit(df).transform(df)
-
-    println(s"FValueSelector output with top ${selector.getNumTopFeatures} features selected")
-    result.show()
-    // $example off$
-
-    spark.stop()
-  }
-}
-// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/FValueTestExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/FValueTestExample.scala
deleted file mode 100644
index 08ec22c90512a..0000000000000
--- a/examples/src/main/scala/org/apache/spark/examples/ml/FValueTestExample.scala
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package org.apache.spark.examples.ml
-
-// $example on$
-import org.apache.spark.ml.linalg.{Vector, Vectors}
-import org.apache.spark.ml.stat.FValueTest
-// $example off$
-import org.apache.spark.sql.SparkSession
-
-/**
- * An example for FValue testing.
- * Run with
- * {{{
- * bin/run-example ml.FValueTestExample
- * }}}
- */
-object FValueTestExample {
-
-  def main(args: Array[String]): Unit = {
-    val spark = SparkSession
-      .builder
-      .appName("FValueTestExample")
-      .getOrCreate()
-    import spark.implicits._
-
-    // $example on$
-    val data = Seq(
-      (4.6, Vectors.dense(6.0, 7.0, 0.0, 7.0, 6.0, 0.0)),
-      (6.6, Vectors.dense(0.0, 9.0, 6.0, 0.0, 5.0, 9.0)),
-      (5.1, Vectors.dense(0.0, 9.0, 3.0, 0.0, 5.0, 5.0)),
-      (7.6, Vectors.dense(0.0, 9.0, 8.0, 5.0, 6.0, 4.0)),
-      (9.0, Vectors.dense(8.0, 9.0, 6.0, 5.0, 4.0, 4.0)),
-      (9.0, Vectors.dense(8.0, 9.0, 6.0, 4.0, 0.0, 0.0))
-    )
-
-    val df = data.toDF("label", "features")
-    val fValue = FValueTest.test(df, "features", "label").head
-    println(s"pValues ${fValue.getAs[Vector](0)}")
-    println(s"degreesOfFreedom ${fValue.getSeq[Int](1).mkString("[", ",", "]")}")
-    println(s"fValues ${fValue.getAs[Vector](2)}")
-    // $example off$
-
-    spark.stop()
-  }
-}
-// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/ModelSelectionViaRandomHyperparametersExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/ModelSelectionViaRandomHyperparametersExample.scala
new file mode 100644
index 0000000000000..9d2c58bbf9c7f
--- /dev/null
+++ b/examples/src/main/scala/org/apache/spark/examples/ml/ModelSelectionViaRandomHyperparametersExample.scala
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.examples.ml
+
+// $example on$
+import org.apache.spark.ml.evaluation.RegressionEvaluator
+import org.apache.spark.ml.regression.LinearRegression
+import org.apache.spark.ml.tuning.{CrossValidator, CrossValidatorModel, Limits, ParamRandomBuilder}
+import org.apache.spark.ml.tuning.RandomRanges._
+// $example off$
+import org.apache.spark.sql.SparkSession
+
+/**
+ * A simple example demonstrating model selection using ParamRandomBuilder.
+ *
+ * Run with
+ * {{{
+ * bin/run-example ml.ModelSelectionViaRandomHyperparametersExample
+ * }}}
+ */
+object ModelSelectionViaRandomHyperparametersExample {
+  def main(args: Array[String]): Unit = {
+    val spark = SparkSession
+      .builder
+      .appName("ModelSelectionViaTrainValidationSplitExample")
+      .getOrCreate()
+    // scalastyle:off println
+    // $example on$
+    // Prepare training and test data.
+    val data = spark.read.format("libsvm").load("data/mllib/sample_linear_regression_data.txt")
+
+    val lr = new LinearRegression().setMaxIter(10)
+
+    // We sample the regularization parameter logarithmically over the range [0.01, 1.0].
+    // This means that values around 0.01, 0.1 and 1.0 are roughly equally likely.
+    // Note that both parameters must be greater than zero as otherwise we'll get an infinity.
+    // We sample the the ElasticNet mixing parameter uniformly over the range [0, 1]
+    // Note that in real life, you'd choose more than the 5 samples we see below.
+    val hyperparameters = new ParamRandomBuilder()
+      .addLog10Random(lr.regParam, Limits(0.01, 1.0), 5)
+      .addGrid(lr.fitIntercept)
+      .addRandom(lr.elasticNetParam, Limits(0.0, 1.0), 5)
+      .build()
+
+    println(s"hyperparameters:\n${hyperparameters.mkString("\n")}")
+
+    val cv: CrossValidator = new CrossValidator()
+      .setEstimator(lr)
+      .setEstimatorParamMaps(hyperparameters)
+      .setEvaluator(new RegressionEvaluator)
+      .setNumFolds(3)
+    val cvModel: CrossValidatorModel = cv.fit(data)
+    val parent: LinearRegression = cvModel.bestModel.parent.asInstanceOf[LinearRegression]
+
+    println(s"""Optimal model has:
+         |${lr.regParam}        = ${parent.getRegParam}
+         |${lr.elasticNetParam} = ${parent.getElasticNetParam}
+         |${lr.fitIntercept}    = ${parent.getFitIntercept}""".stripMargin)
+    // $example off$
+
+    spark.stop()
+  }
+  // scalastyle:on println
+}
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/UnivariateFeatureSelectorExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/UnivariateFeatureSelectorExample.scala
new file mode 100644
index 0000000000000..e4932db5e3765
--- /dev/null
+++ b/examples/src/main/scala/org/apache/spark/examples/ml/UnivariateFeatureSelectorExample.scala
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// scalastyle:off println
+package org.apache.spark.examples.ml
+
+// $example on$
+import org.apache.spark.ml.feature.UnivariateFeatureSelector
+import org.apache.spark.ml.linalg.Vectors
+// $example off$
+import org.apache.spark.sql.SparkSession
+
+/**
+ * An example for UnivariateFeatureSelector.
+ * Run with
+ * {{{
+ * bin/run-example ml.UnivariateFeatureSelectorExample
+ * }}}
+ */
+object UnivariateFeatureSelectorExample {
+  def main(args: Array[String]): Unit = {
+    val spark = SparkSession
+      .builder
+      .appName("UnivariateFeatureSelectorExample")
+      .getOrCreate()
+    import spark.implicits._
+
+    // $example on$
+    val data = Seq(
+      (1, Vectors.dense(1.7, 4.4, 7.6, 5.8, 9.6, 2.3), 3.0),
+      (2, Vectors.dense(8.8, 7.3, 5.7, 7.3, 2.2, 4.1), 2.0),
+      (3, Vectors.dense(1.2, 9.5, 2.5, 3.1, 8.7, 2.5), 3.0),
+      (4, Vectors.dense(3.7, 9.2, 6.1, 4.1, 7.5, 3.8), 2.0),
+      (5, Vectors.dense(8.9, 5.2, 7.8, 8.3, 5.2, 3.0), 4.0),
+      (6, Vectors.dense(7.9, 8.5, 9.2, 4.0, 9.4, 2.1), 4.0)
+    )
+
+    val df = spark.createDataset(data).toDF("id", "features", "label")
+
+    val selector = new UnivariateFeatureSelector()
+      .setFeatureType("continuous")
+      .setLabelType("categorical")
+      .setSelectionMode("numTopFeatures")
+      .setSelectionThreshold(1)
+      .setFeaturesCol("features")
+      .setLabelCol("label")
+      .setOutputCol("selectedFeatures")
+
+    val result = selector.fit(df).transform(df)
+
+    println(s"UnivariateFeatureSelector output with top ${selector.getSelectionThreshold}" +
+      s" features selected using f_classif")
+    result.show()
+    // $example off$
+
+    spark.stop()
+  }
+}
+// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassificationMetricsExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassificationMetricsExample.scala
index c6312d71cc912..a606cc4218778 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassificationMetricsExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassificationMetricsExample.scala
@@ -59,13 +59,13 @@ object BinaryClassificationMetricsExample {
 
     // Precision by threshold
     val precision = metrics.precisionByThreshold
-    precision.foreach { case (t, p) =>
+    precision.collect.foreach { case (t, p) =>
       println(s"Threshold: $t, Precision: $p")
     }
 
     // Recall by threshold
     val recall = metrics.recallByThreshold
-    recall.foreach { case (t, r) =>
+    recall.collect.foreach { case (t, r) =>
       println(s"Threshold: $t, Recall: $r")
     }
 
@@ -74,13 +74,13 @@ object BinaryClassificationMetricsExample {
 
     // F-measure
     val f1Score = metrics.fMeasureByThreshold
-    f1Score.foreach { case (t, f) =>
+    f1Score.collect.foreach { case (t, f) =>
       println(s"Threshold: $t, F-score: $f, Beta = 1")
     }
 
     val beta = 0.5
     val fScore = metrics.fMeasureByThreshold(beta)
-    f1Score.foreach { case (t, f) =>
+    fScore.collect.foreach { case (t, f) =>
       println(s"Threshold: $t, F-score: $f, Beta = 0.5")
     }
 
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/ChiSqSelectorExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/ChiSqSelectorExample.scala
index 5e400b7d715b4..6ed59a3366d93 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/ChiSqSelectorExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/ChiSqSelectorExample.scala
@@ -53,7 +53,7 @@ object ChiSqSelectorExample {
     // $example off$
 
     println("filtered data: ")
-    filteredData.foreach(x => println(x))
+    filteredData.collect.foreach(x => println(x))
 
     sc.stop()
   }
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/ElementwiseProductExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/ElementwiseProductExample.scala
index 1855058bb4f3c..d6ec678b422f9 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/ElementwiseProductExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/ElementwiseProductExample.scala
@@ -45,10 +45,10 @@ object ElementwiseProductExample {
     // $example off$
 
     println("transformedData: ")
-    transformedData.foreach(x => println(x))
+    transformedData.collect.foreach(x => println(x))
 
     println("transformedData2: ")
-    transformedData2.foreach(x => println(x))
+    transformedData2.collect.foreach(x => println(x))
 
     sc.stop()
   }
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/NormalizerExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/NormalizerExample.scala
index b3a9604c2be3e..b1cad7bf8125f 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/NormalizerExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/NormalizerExample.scala
@@ -46,10 +46,10 @@ object NormalizerExample {
     // $example off$
 
     println("data1: ")
-    data1.foreach(x => println(x))
+    data1.collect.foreach(x => println(x))
 
     println("data2: ")
-    data2.foreach(x => println(x))
+    data2.collect.foreach(x => println(x))
 
     sc.stop()
   }
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/RankingMetricsExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/RankingMetricsExample.scala
index 2845028dd0814..7a7501ee84526 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/RankingMetricsExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/RankingMetricsExample.scala
@@ -30,7 +30,6 @@ object RankingMetricsExample {
       .builder
       .appName("RankingMetricsExample")
       .getOrCreate()
-    import spark.implicits._
     // $example on$
     // Read in the ratings data
     val ratings = spark.read.textFile("data/mllib/sample_movielens_data.txt").rdd.map { line =>
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/StandardScalerExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/StandardScalerExample.scala
index 769fc17b3dc65..66a608ced0f83 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/StandardScalerExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/StandardScalerExample.scala
@@ -49,10 +49,10 @@ object StandardScalerExample {
     // $example off$
 
     println("data1: ")
-    data1.foreach(x => println(x))
+    data1.collect.foreach(x => println(x))
 
     println("data2: ")
-    data2.foreach(x => println(x))
+    data2.collect.foreach(x => println(x))
 
     sc.stop()
   }
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/TFIDFExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/TFIDFExample.scala
index a5bdcd8f2ed32..14b2a2015c241 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/TFIDFExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/TFIDFExample.scala
@@ -55,10 +55,10 @@ object TFIDFExample {
     // $example off$
 
     println("tfidf: ")
-    tfidf.foreach(x => println(x))
+    tfidf.collect.foreach(x => println(x))
 
     println("tfidfIgnore: ")
-    tfidfIgnore.foreach(x => println(x))
+    tfidfIgnore.collect.foreach(x => println(x))
 
     sc.stop()
   }
diff --git a/examples/src/main/scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala b/examples/src/main/scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala
index 2c7abfcd335d1..6bd2bd6d3bf5e 100644
--- a/examples/src/main/scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala
@@ -36,6 +36,8 @@ object SQLDataSourceExample {
     runBasicParquetExample(spark)
     runParquetSchemaMergingExample(spark)
     runJsonDatasetExample(spark)
+    runCsvDatasetExample(spark)
+    runTextDatasetExample(spark)
     runJdbcDatasetExample(spark)
 
     spark.stop()
@@ -81,6 +83,27 @@ object SQLDataSourceExample {
     // |file1.parquet|
     // +-------------+
     // $example off:load_with_path_glob_filter$
+    // $example on:load_with_modified_time_filter$
+    val beforeFilterDF = spark.read.format("parquet")
+      // Files modified before 07/01/2020 at 05:30 are allowed
+      .option("modifiedBefore", "2020-07-01T05:30:00")
+      .load("examples/src/main/resources/dir1");
+    beforeFilterDF.show();
+    // +-------------+
+    // |         file|
+    // +-------------+
+    // |file1.parquet|
+    // +-------------+
+    val afterFilterDF = spark.read.format("parquet")
+       // Files modified after 06/01/2020 at 05:30 are allowed
+      .option("modifiedAfter", "2020-06-01T05:30:00")
+      .load("examples/src/main/resources/dir1");
+    afterFilterDF.show();
+    // +-------------+
+    // |         file|
+    // +-------------+
+    // +-------------+
+    // $example off:load_with_modified_time_filter$
   }
 
   private def runBasicDataSourceExample(spark: SparkSession): Unit = {
@@ -106,6 +129,14 @@ object SQLDataSourceExample {
       .option("orc.column.encoding.direct", "name")
       .save("users_with_options.orc")
     // $example off:manual_save_options_orc$
+    // $example on:manual_save_options_parquet$
+    usersDF.write.format("parquet")
+      .option("parquet.bloom.filter.enabled#favorite_color", "true")
+      .option("parquet.bloom.filter.expected.ndv#favorite_color", "1000000")
+      .option("parquet.enable.dictionary", "true")
+      .option("parquet.page.write-checksum.enabled", "false")
+      .save("users_with_options.parquet")
+    // $example off:manual_save_options_parquet$
 
     // $example on:direct_sql$
     val sqlDF = spark.sql("SELECT * FROM parquet.`examples/src/main/resources/users.parquet`")
@@ -226,6 +257,114 @@ object SQLDataSourceExample {
     // $example off:json_dataset$
   }
 
+  private def runCsvDatasetExample(spark: SparkSession): Unit = {
+    // $example on:csv_dataset$
+    // A CSV dataset is pointed to by path.
+    // The path can be either a single CSV file or a directory of CSV files
+    val path = "examples/src/main/resources/people.csv"
+
+    val df = spark.read.csv(path)
+    df.show()
+    // +------------------+
+    // |               _c0|
+    // +------------------+
+    // |      name;age;job|
+    // |Jorge;30;Developer|
+    // |  Bob;32;Developer|
+    // +------------------+
+
+    // Read a csv with delimiter, the default delimiter is ","
+    val df2 = spark.read.option("delimiter", ";").csv(path)
+    df2.show()
+    // +-----+---+---------+
+    // |  _c0|_c1|      _c2|
+    // +-----+---+---------+
+    // | name|age|      job|
+    // |Jorge| 30|Developer|
+    // |  Bob| 32|Developer|
+    // +-----+---+---------+
+
+    // Read a csv with delimiter and a header
+    val df3 = spark.read.option("delimiter", ";").option("header", "true").csv(path)
+    df3.show()
+    // +-----+---+---------+
+    // | name|age|      job|
+    // +-----+---+---------+
+    // |Jorge| 30|Developer|
+    // |  Bob| 32|Developer|
+    // +-----+---+---------+
+
+    // You can also use options() to use multiple options
+    val df4 = spark.read.options(Map("delimiter"->";", "header"->"true")).csv(path)
+
+    // "output" is a folder which contains multiple csv files and a _SUCCESS file.
+    df3.write.csv("output")
+
+    // Read all files in a folder, please make sure only CSV files should present in the folder.
+    val folderPath = "examples/src/main/resources";
+    val df5 = spark.read.csv(folderPath);
+    df5.show();
+    // Wrong schema because non-CSV files are read
+    // +-----------+
+    // |        _c0|
+    // +-----------+
+    // |238val_238|
+    // |  86val_86|
+    // |311val_311|
+    // |  27val_27|
+    // |165val_165|
+    // +-----------+
+
+    // $example off:csv_dataset$
+  }
+
+  private def runTextDatasetExample(spark: SparkSession): Unit = {
+    // $example on:text_dataset$
+    // A text dataset is pointed to by path.
+    // The path can be either a single text file or a directory of text files
+    val path = "examples/src/main/resources/people.txt"
+
+    val df1 = spark.read.text(path)
+    df1.show()
+    // +-----------+
+    // |      value|
+    // +-----------+
+    // |Michael, 29|
+    // |   Andy, 30|
+    // | Justin, 19|
+    // +-----------+
+
+    // You can use 'lineSep' option to define the line separator.
+    // The line separator handles all `\r`, `\r\n` and `\n` by default.
+    val df2 = spark.read.option("lineSep", ",").text(path)
+    df2.show()
+    // +-----------+
+    // |      value|
+    // +-----------+
+    // |    Michael|
+    // |   29\nAndy|
+    // | 30\nJustin|
+    // |       19\n|
+    // +-----------+
+
+    // You can also use 'wholetext' option to read each input file as a single row.
+    val df3 = spark.read.option("wholetext", true).text(path)
+    df3.show()
+    //  +--------------------+
+    //  |               value|
+    //  +--------------------+
+    //  |Michael, 29\nAndy...|
+    //  +--------------------+
+
+    // "output" is a folder which contains multiple text files and a _SUCCESS file.
+    df1.write.text("output")
+
+    // You can specify the compression format using the 'compression' option.
+    df1.write.option("compression", "gzip").text("output_compressed")
+
+    // $example off:text_dataset$
+  }
+
   private def runJdbcDatasetExample(spark: SparkSession): Unit = {
     // $example on:jdbc_dataset$
     // Note: JDBC loading and saving can be achieved via either the load/save or jdbc methods
diff --git a/examples/src/main/scala/org/apache/spark/examples/sql/SimpleTypedAggregator.scala b/examples/src/main/scala/org/apache/spark/examples/sql/SimpleTypedAggregator.scala
index 5510f0019353b..5d11fb2fc96e5 100644
--- a/examples/src/main/scala/org/apache/spark/examples/sql/SimpleTypedAggregator.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/sql/SimpleTypedAggregator.scala
@@ -31,7 +31,7 @@ object SimpleTypedAggregator {
       .getOrCreate()
 
     import spark.implicits._
-    val ds = spark.range(20).select(('id % 3).as("key"), 'id).as[(Long, Long)]
+    val ds = spark.range(20).select(($"id" % 3).as("key"), $"id").as[(Long, Long)]
     println("input data:")
     ds.show()
 
diff --git a/examples/src/main/scala/org/apache/spark/examples/sql/SparkSQLExample.scala b/examples/src/main/scala/org/apache/spark/examples/sql/SparkSQLExample.scala
index fde281087c267..b17b86c08314b 100644
--- a/examples/src/main/scala/org/apache/spark/examples/sql/SparkSQLExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/sql/SparkSQLExample.scala
@@ -42,8 +42,6 @@ object SparkSQLExample {
       .config("spark.some.config.option", "some-value")
       .getOrCreate()
 
-    // For implicit conversions like converting RDDs to DataFrames
-    import spark.implicits._
     // $example off:init_session$
 
     runBasicDataFrameExample(spark)
diff --git a/examples/src/main/scala/org/apache/spark/examples/sql/jdbc/ExampleJdbcConnectionProvider.scala b/examples/src/main/scala/org/apache/spark/examples/sql/jdbc/ExampleJdbcConnectionProvider.scala
new file mode 100644
index 0000000000000..6d275d4912bc9
--- /dev/null
+++ b/examples/src/main/scala/org/apache/spark/examples/sql/jdbc/ExampleJdbcConnectionProvider.scala
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.examples.sql.jdbc
+
+import java.sql.{Connection, Driver}
+
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.jdbc.JdbcConnectionProvider
+
+class ExampleJdbcConnectionProvider extends JdbcConnectionProvider with Logging {
+  logInfo("ExampleJdbcConnectionProvider instantiated")
+
+  override val name: String = "ExampleJdbcConnectionProvider"
+
+  override def canHandle(driver: Driver, options: Map[String, String]): Boolean = false
+
+  override def getConnection(driver: Driver, options: Map[String, String]): Connection = null
+}
diff --git a/examples/src/main/scala/org/apache/spark/examples/streaming/CustomReceiver.scala b/examples/src/main/scala/org/apache/spark/examples/streaming/CustomReceiver.scala
index 0f47deaf1021b..626f4b4d3ccdf 100644
--- a/examples/src/main/scala/org/apache/spark/examples/streaming/CustomReceiver.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/streaming/CustomReceiver.scala
@@ -50,7 +50,7 @@ object CustomReceiver {
     val ssc = new StreamingContext(sparkConf, Seconds(1))
 
     // Create an input stream with the custom receiver on target ip:port and count the
-    // words in input stream of \n delimited text (eg. generated by 'nc')
+    // words in input stream of \n delimited text (e.g. generated by 'nc')
     val lines = ssc.receiverStream(new CustomReceiver(args(0), args(1).toInt))
     val words = lines.flatMap(_.split(" "))
     val wordCounts = words.map(x => (x, 1)).reduceByKey(_ + _)
diff --git a/examples/src/main/scala/org/apache/spark/examples/streaming/NetworkWordCount.scala b/examples/src/main/scala/org/apache/spark/examples/streaming/NetworkWordCount.scala
index 26bb51dde3a1d..7d981dfb949ea 100644
--- a/examples/src/main/scala/org/apache/spark/examples/streaming/NetworkWordCount.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/streaming/NetworkWordCount.scala
@@ -47,7 +47,7 @@ object NetworkWordCount {
     val ssc = new StreamingContext(sparkConf, Seconds(1))
 
     // Create a socket stream on target ip:port and count the
-    // words in input stream of \n delimited text (eg. generated by 'nc')
+    // words in input stream of \n delimited text (e.g. generated by 'nc')
     // Note that no duplication in storage level only for running locally.
     // Replication necessary in distributed scenario for fault tolerance.
     val lines = ssc.socketTextStream(args(0), args(1).toInt, StorageLevel.MEMORY_AND_DISK_SER)
diff --git a/examples/src/main/scala/org/apache/spark/examples/streaming/RecoverableNetworkWordCount.scala b/examples/src/main/scala/org/apache/spark/examples/streaming/RecoverableNetworkWordCount.scala
index 243c22e71275c..98539d6494231 100644
--- a/examples/src/main/scala/org/apache/spark/examples/streaming/RecoverableNetworkWordCount.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/streaming/RecoverableNetworkWordCount.scala
@@ -32,7 +32,7 @@ import org.apache.spark.util.{IntParam, LongAccumulator}
 /**
  * Use this singleton to get or register a Broadcast variable.
  */
-object WordBlacklist {
+object WordExcludeList {
 
   @volatile private var instance: Broadcast[Seq[String]] = null
 
@@ -40,8 +40,8 @@ object WordBlacklist {
     if (instance == null) {
       synchronized {
         if (instance == null) {
-          val wordBlacklist = Seq("a", "b", "c")
-          instance = sc.broadcast(wordBlacklist)
+          val wordExcludeList = Seq("a", "b", "c")
+          instance = sc.broadcast(wordExcludeList)
         }
       }
     }
@@ -60,7 +60,7 @@ object DroppedWordsCounter {
     if (instance == null) {
       synchronized {
         if (instance == null) {
-          instance = sc.longAccumulator("WordsInBlacklistCounter")
+          instance = sc.longAccumulator("DroppedWordsCounter")
         }
       }
     }
@@ -112,18 +112,18 @@ object RecoverableNetworkWordCount {
     ssc.checkpoint(checkpointDirectory)
 
     // Create a socket stream on target ip:port and count the
-    // words in input stream of \n delimited text (eg. generated by 'nc')
+    // words in input stream of \n delimited text (e.g. generated by 'nc')
     val lines = ssc.socketTextStream(ip, port)
     val words = lines.flatMap(_.split(" "))
     val wordCounts = words.map((_, 1)).reduceByKey(_ + _)
     wordCounts.foreachRDD { (rdd: RDD[(String, Int)], time: Time) =>
-      // Get or register the blacklist Broadcast
-      val blacklist = WordBlacklist.getInstance(rdd.sparkContext)
+      // Get or register the excludeList Broadcast
+      val excludeList = WordExcludeList.getInstance(rdd.sparkContext)
       // Get or register the droppedWordsCounter Accumulator
       val droppedWordsCounter = DroppedWordsCounter.getInstance(rdd.sparkContext)
-      // Use blacklist to drop words and use droppedWordsCounter to count them
+      // Use excludeList to drop words and use droppedWordsCounter to count them
       val counts = rdd.filter { case (word, count) =>
-        if (blacklist.value.contains(word)) {
+        if (excludeList.value.contains(word)) {
           droppedWordsCounter.add(count)
           false
         } else {
diff --git a/examples/src/main/scala/org/apache/spark/examples/streaming/SqlNetworkWordCount.scala b/examples/src/main/scala/org/apache/spark/examples/streaming/SqlNetworkWordCount.scala
index 778be7baaeeac..7daa0014e0f1c 100644
--- a/examples/src/main/scala/org/apache/spark/examples/streaming/SqlNetworkWordCount.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/streaming/SqlNetworkWordCount.scala
@@ -51,7 +51,7 @@ object SqlNetworkWordCount {
     val ssc = new StreamingContext(sparkConf, Seconds(2))
 
     // Create a socket stream on target ip:port and count the
-    // words in input stream of \n delimited text (eg. generated by 'nc')
+    // words in input stream of \n delimited text (e.g. generated by 'nc')
     // Note that no duplication in storage level only for running locally.
     // Replication necessary in distributed scenario for fault tolerance.
     val lines = ssc.socketTextStream(args(0), args(1).toInt, StorageLevel.MEMORY_AND_DISK_SER)
diff --git a/examples/src/main/scala/org/apache/spark/examples/streaming/StatefulNetworkWordCount.scala b/examples/src/main/scala/org/apache/spark/examples/streaming/StatefulNetworkWordCount.scala
index 46f01edf7deec..8a5fcda9cd990 100644
--- a/examples/src/main/scala/org/apache/spark/examples/streaming/StatefulNetworkWordCount.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/streaming/StatefulNetworkWordCount.scala
@@ -52,7 +52,7 @@ object StatefulNetworkWordCount {
     val initialRDD = ssc.sparkContext.parallelize(List(("hello", 1), ("world", 1)))
 
     // Create a ReceiverInputDStream on target ip:port and count the
-    // words in input stream of \n delimited test (eg. generated by 'nc')
+    // words in input stream of \n delimited test (e.g. generated by 'nc')
     val lines = ssc.socketTextStream(args(0), args(1).toInt)
     val words = lines.flatMap(_.split(" "))
     val wordDstream = words.map(x => (x, 1))
diff --git a/external/avro/benchmarks/AvroReadBenchmark-jdk11-results.txt b/external/avro/benchmarks/AvroReadBenchmark-jdk11-results.txt
index 3c1b5af0d5986..0a7f6edaba266 100644
--- a/external/avro/benchmarks/AvroReadBenchmark-jdk11-results.txt
+++ b/external/avro/benchmarks/AvroReadBenchmark-jdk11-results.txt
@@ -2,121 +2,129 @@
 SQL Single Numeric Column Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 SQL Single TINYINT Column Scan:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum                                                2689           2694           7          5.8         170.9       1.0X
+Sum                                                3468           3495          39          4.5         220.5       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 SQL Single SMALLINT Column Scan:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum                                                2741           2759          26          5.7         174.2       1.0X
+Sum                                                3378           3388          13          4.7         214.8       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 SQL Single INT Column Scan:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum                                                2736           2748          17          5.7         173.9       1.0X
+Sum                                                3353           3371          26          4.7         213.2       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 SQL Single BIGINT Column Scan:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum                                                3305           3317          17          4.8         210.2       1.0X
+Sum                                                3712           3739          38          4.2         236.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 SQL Single FLOAT Column Scan:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum                                                2904           2952          68          5.4         184.6       1.0X
+Sum                                                3290           3322          45          4.8         209.2       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 SQL Single DOUBLE Column Scan:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum                                                3090           3093           4          5.1         196.5       1.0X
+Sum                                                3475           3493          25          4.5         220.9       1.0X
 
 
 ================================================================================================
 Int and String Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Int and String Scan:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum of columns                                     5351           5365          20          2.0         510.3       1.0X
+Sum of columns                                     5032           5053          29          2.1         479.9       1.0X
 
 
 ================================================================================================
 Partitioned Table Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Partitioned Table:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Data column                                        3278           3288          14          4.8         208.4       1.0X
-Partition column                                   3149           3193          62          5.0         200.2       1.0X
-Both columns                                       3198           3204           7          4.9         203.4       1.0X
+Data column                                        3819           3820           1          4.1         242.8       1.0X
+Partition column                                   3285           3340          77          4.8         208.8       1.2X
+Both columns                                       4051           4066          21          3.9         257.6       0.9X
 
 
 ================================================================================================
 Repeated String Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Repeated String:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum of string length                               3435           3438           5          3.1         327.6       1.0X
+Sum of string length                               3631           3653          32          2.9         346.2       1.0X
 
 
 ================================================================================================
 String with Nulls Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 String with Nulls Scan (0.0%):            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum of string length                               5634           5650          23          1.9         537.3       1.0X
+Sum of string length                               5786           5801          21          1.8         551.8       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 String with Nulls Scan (50.0%):           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum of string length                               4725           4752          39          2.2         450.6       1.0X
+Sum of string length                               3767           3791          34          2.8         359.2       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 String with Nulls Scan (95.0%):           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum of string length                               3550           3566          23          3.0         338.6       1.0X
+Sum of string length                               2472           2498          36          4.2         235.8       1.0X
 
 
 ================================================================================================
 Single Column Scan From Wide Columns
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Single Column Scan from 100 columns:      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum of single column                               5271           5279          11          0.2        5027.0       1.0X
+Sum of single column                               5366           5366           0          0.2        5117.5       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Single Column Scan from 200 columns:      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum of single column                              10393          10516         174          0.1        9911.3       1.0X
+Sum of single column                              10223          10234          15          0.1        9749.7       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Single Column Scan from 300 columns:      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum of single column                              15330          15343          19          0.1       14619.6       1.0X
+Sum of single column                              15283          15328          64          0.1       14575.0       1.0X
 
 
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+Filters pushdown:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+w/o filters                                        9724           9765          46          0.1        9724.4       1.0X
+pushdown disabled                                  9628           9726          99          0.1        9627.6       1.0X
+w/ filters                                         4833           4879          40          0.2        4833.5       2.0X
+
diff --git a/external/avro/benchmarks/AvroReadBenchmark-results.txt b/external/avro/benchmarks/AvroReadBenchmark-results.txt
index 0ab611a0f9a29..f77db2d5e4e67 100644
--- a/external/avro/benchmarks/AvroReadBenchmark-results.txt
+++ b/external/avro/benchmarks/AvroReadBenchmark-results.txt
@@ -2,121 +2,129 @@
 SQL Single Numeric Column Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 SQL Single TINYINT Column Scan:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum                                                3049           3071          32          5.2         193.8       1.0X
+Sum                                                2802           2826          34          5.6         178.1       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 SQL Single SMALLINT Column Scan:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum                                                2982           2992          13          5.3         189.6       1.0X
+Sum                                                2786           2810          35          5.6         177.1       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 SQL Single INT Column Scan:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum                                                2984           2989           7          5.3         189.7       1.0X
+Sum                                                2808           2817          13          5.6         178.5       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 SQL Single BIGINT Column Scan:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum                                                3262           3353         128          4.8         207.4       1.0X
+Sum                                                3222           3224           3          4.9         204.9       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 SQL Single FLOAT Column Scan:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum                                                2716           2723          10          5.8         172.7       1.0X
+Sum                                                2827           2844          24          5.6         179.7       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 SQL Single DOUBLE Column Scan:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum                                                2868           2870           3          5.5         182.4       1.0X
+Sum                                                2910           2924          20          5.4         185.0       1.0X
 
 
 ================================================================================================
 Int and String Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Int and String Scan:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum of columns                                     4714           4739          35          2.2         449.6       1.0X
+Sum of columns                                     4575           4580           7          2.3         436.3       1.0X
 
 
 ================================================================================================
 Partitioned Table Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Partitioned Table:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Data column                                        3257           3286          41          4.8         207.1       1.0X
-Partition column                                   3258           3277          27          4.8         207.2       1.0X
-Both columns                                       3399           3405           9          4.6         216.1       1.0X
+Data column                                        3252           3271          27          4.8         206.8       1.0X
+Partition column                                   2905           2907           3          5.4         184.7       1.1X
+Both columns                                       3385           3398          18          4.6         215.2       1.0X
 
 
 ================================================================================================
 Repeated String Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Repeated String:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum of string length                               3292           3316          33          3.2         314.0       1.0X
+Sum of string length                               3275           3278           3          3.2         312.4       1.0X
 
 
 ================================================================================================
 String with Nulls Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 String with Nulls Scan (0.0%):            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum of string length                               5450           5456           9          1.9         519.7       1.0X
+Sum of string length                               5202           5219          24          2.0         496.1       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 String with Nulls Scan (50.0%):           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum of string length                               4410           4435          35          2.4         420.6       1.0X
+Sum of string length                               3360           3381          29          3.1         320.5       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 String with Nulls Scan (95.0%):           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum of string length                               3074           3122          68          3.4         293.2       1.0X
+Sum of string length                               1917           1936          28          5.5         182.8       1.0X
 
 
 ================================================================================================
 Single Column Scan From Wide Columns
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Single Column Scan from 100 columns:      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum of single column                               5120           5136          23          0.2        4882.7       1.0X
+Sum of single column                               4348           4424         107          0.2        4146.5       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Single Column Scan from 200 columns:      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum of single column                               9952          10002          71          0.1        9490.7       1.0X
+Sum of single column                               8799           8806          10          0.1        8391.2       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Single Column Scan from 300 columns:      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Sum of single column                              14973          14978           7          0.1       14279.8       1.0X
+Sum of single column                              12956          12990          49          0.1       12355.5       1.0X
 
 
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+Filters pushdown:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+w/o filters                                        9208           9269          63          0.1        9207.5       1.0X
+pushdown disabled                                  9073           9111          59          0.1        9072.7       1.0X
+w/ filters                                         3929           3947          18          0.3        3928.8       2.3X
+
diff --git a/external/avro/benchmarks/AvroWriteBenchmark-jdk11-results.txt b/external/avro/benchmarks/AvroWriteBenchmark-jdk11-results.txt
index 2cf1835013821..4cdb05dffcaaa 100644
--- a/external/avro/benchmarks/AvroWriteBenchmark-jdk11-results.txt
+++ b/external/avro/benchmarks/AvroWriteBenchmark-jdk11-results.txt
@@ -1,10 +1,10 @@
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Avro writer benchmark:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Output Single Int Column                           3026           3142         164          5.2         192.4       1.0X
-Output Single Double Column                        3157           3260         145          5.0         200.7       1.0X
-Output Int and String Column                       6123           6190          94          2.6         389.3       0.5X
-Output Partitions                                  5197           5733         758          3.0         330.4       0.6X
-Output Buckets                                     7074           7285         298          2.2         449.7       0.4X
+Output Single Int Column                           3307           3313           8          4.8         210.3       1.0X
+Output Single Double Column                        3566           3584          26          4.4         226.7       0.9X
+Output Int and String Column                       6913           6988         106          2.3         439.5       0.5X
+Output Partitions                                  5984           6112         181          2.6         380.5       0.6X
+Output Buckets                                     8403           8505         144          1.9         534.3       0.4X
 
diff --git a/external/avro/benchmarks/AvroWriteBenchmark-results.txt b/external/avro/benchmarks/AvroWriteBenchmark-results.txt
index 20f6ae9099a4d..26bb126cf182d 100644
--- a/external/avro/benchmarks/AvroWriteBenchmark-results.txt
+++ b/external/avro/benchmarks/AvroWriteBenchmark-results.txt
@@ -1,10 +1,10 @@
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 Avro writer benchmark:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Output Single Int Column                           3080           3137          82          5.1         195.8       1.0X
-Output Single Double Column                        3595           3595           0          4.4         228.6       0.9X
-Output Int and String Column                       7491           7504          18          2.1         476.3       0.4X
-Output Partitions                                  5518           5663         205          2.9         350.8       0.6X
-Output Buckets                                     7467           7581         161          2.1         474.7       0.4X
+Output Single Int Column                           2478           2537          83          6.3         157.6       1.0X
+Output Single Double Column                        2636           2652          21          6.0         167.6       0.9X
+Output Int and String Column                       5922           6039         166          2.7         376.5       0.4X
+Output Partitions                                  4158           4305         207          3.8         264.3       0.6X
+Output Buckets                                     5486           5534          68          2.9         348.8       0.5X
 
diff --git a/external/avro/pom.xml b/external/avro/pom.xml
index 98036846eb2a8..b4ec472639c0a 100644
--- a/external/avro/pom.xml
+++ b/external/avro/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
@@ -70,6 +70,10 @@
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-tags_${scala.binary.version}</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.tukaani</groupId>
+      <artifactId>xz</artifactId>
+    </dependency>
   </dependencies>
   <build>
     <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
diff --git a/external/avro/src/main/java/org/apache/spark/sql/avro/SparkAvroKeyOutputFormat.java b/external/avro/src/main/java/org/apache/spark/sql/avro/SparkAvroKeyOutputFormat.java
index 55696a6ac2e0f..a4555844b5117 100644
--- a/external/avro/src/main/java/org/apache/spark/sql/avro/SparkAvroKeyOutputFormat.java
+++ b/external/avro/src/main/java/org/apache/spark/sql/avro/SparkAvroKeyOutputFormat.java
@@ -35,8 +35,8 @@
 
 // A variant of `AvroKeyOutputFormat`, which is used to inject the custom `RecordWriterFactory` so
 // that we can set avro file metadata.
-public class SparkAvroKeyOutputFormat extends AvroKeyOutputFormat<GenericRecord> {
-  public SparkAvroKeyOutputFormat(Map<String, String> metadata) {
+class SparkAvroKeyOutputFormat extends AvroKeyOutputFormat<GenericRecord> {
+  SparkAvroKeyOutputFormat(Map<String, String> metadata) {
     super(new SparkRecordWriterFactory(metadata));
   }
 
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala
index 79c72057c5823..b4965003ba33d 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala
@@ -30,7 +30,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodeGe
 import org.apache.spark.sql.catalyst.util.{FailFastMode, ParseMode, PermissiveMode}
 import org.apache.spark.sql.types._
 
-case class AvroDataToCatalyst(
+private[avro] case class AvroDataToCatalyst(
     child: Expression,
     jsonFormatSchema: String,
     options: Map[String, String])
@@ -55,13 +55,12 @@ case class AvroDataToCatalyst(
 
   @transient private lazy val actualSchema = new Schema.Parser().parse(jsonFormatSchema)
 
-  @transient private lazy val expectedSchema = avroOptions.schema
-    .map(expectedSchema => new Schema.Parser().parse(expectedSchema))
-    .getOrElse(actualSchema)
+  @transient private lazy val expectedSchema = avroOptions.schema.getOrElse(actualSchema)
 
   @transient private lazy val reader = new GenericDatumReader[Any](actualSchema, expectedSchema)
 
-  @transient private lazy val deserializer = new AvroDeserializer(expectedSchema, dataType)
+  @transient private lazy val deserializer =
+    new AvroDeserializer(expectedSchema, dataType, avroOptions.datetimeRebaseModeInRead)
 
   @transient private var decoder: BinaryDecoder = _
 
@@ -98,7 +97,10 @@ case class AvroDataToCatalyst(
     try {
       decoder = DecoderFactory.get().binaryDecoder(binary, 0, binary.length, decoder)
       result = reader.read(result, decoder)
-      deserializer.deserialize(result)
+      val deserialized = deserializer.deserialize(result)
+      assert(deserialized.isDefined,
+        "Avro deserializer cannot return an empty result because filters are not pushed down")
+      deserialized.get
     } catch {
       // There could be multiple possible exceptions here, e.g. java.io.IOException,
       // AvroRuntimeException, ArrayIndexOutOfBoundsException, etc.
@@ -132,4 +134,7 @@ case class AvroDataToCatalyst(
       """
     })
   }
+
+  override protected def withNewChildInternal(newChild: Expression): AvroDataToCatalyst =
+    copy(child = newChild)
 }
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
index 1d18594fd349c..a19a7b0d0edd1 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
@@ -30,27 +30,33 @@ import org.apache.avro.Schema.Type._
 import org.apache.avro.generic._
 import org.apache.avro.util.Utf8
 
-import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.avro.AvroUtils.toFieldStr
+import org.apache.spark.sql.catalyst.{InternalRow, NoopFilters, StructFilters}
 import org.apache.spark.sql.catalyst.expressions.{SpecificInternalRow, UnsafeArrayData}
 import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, DateTimeUtils, GenericArrayData}
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.MILLIS_PER_DAY
 import org.apache.spark.sql.execution.datasources.DataSourceUtils
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 /**
  * A deserializer to deserialize data in avro format to data in catalyst format.
  */
-class AvroDeserializer(
+private[sql] class AvroDeserializer(
     rootAvroType: Schema,
     rootCatalystType: DataType,
-    datetimeRebaseMode: LegacyBehaviorPolicy.Value) {
-
-  def this(rootAvroType: Schema, rootCatalystType: DataType) {
-    this(rootAvroType, rootCatalystType,
-      LegacyBehaviorPolicy.withName(
-        SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ)))
+    datetimeRebaseMode: LegacyBehaviorPolicy.Value,
+    filters: StructFilters) {
+
+  def this(
+      rootAvroType: Schema,
+      rootCatalystType: DataType,
+      datetimeRebaseMode: String) = {
+    this(
+      rootAvroType,
+      rootCatalystType,
+      LegacyBehaviorPolicy.withName(datetimeRebaseMode),
+      new NoopFilters)
   }
 
   private lazy val decimalConversions = new DecimalConversion()
@@ -61,32 +67,38 @@ class AvroDeserializer(
   private val timestampRebaseFunc = DataSourceUtils.creteTimestampRebaseFuncInRead(
     datetimeRebaseMode, "Avro")
 
-  private val converter: Any => Any = rootCatalystType match {
-    // A shortcut for empty schema.
-    case st: StructType if st.isEmpty =>
-      (data: Any) => InternalRow.empty
-
-    case st: StructType =>
-      val resultRow = new SpecificInternalRow(st.map(_.dataType))
-      val fieldUpdater = new RowUpdater(resultRow)
-      val writer = getRecordWriter(rootAvroType, st, Nil)
-      (data: Any) => {
-        val record = data.asInstanceOf[GenericRecord]
-        writer(fieldUpdater, record)
-        resultRow
-      }
+  private val converter: Any => Option[Any] = try {
+    rootCatalystType match {
+      // A shortcut for empty schema.
+      case st: StructType if st.isEmpty =>
+        (_: Any) => Some(InternalRow.empty)
+
+      case st: StructType =>
+        val resultRow = new SpecificInternalRow(st.map(_.dataType))
+        val fieldUpdater = new RowUpdater(resultRow)
+        val applyFilters = filters.skipRow(resultRow, _)
+        val writer = getRecordWriter(rootAvroType, st, Nil, Nil, applyFilters)
+        (data: Any) => {
+          val record = data.asInstanceOf[GenericRecord]
+          val skipRow = writer(fieldUpdater, record)
+          if (skipRow) None else Some(resultRow)
+        }
 
-    case _ =>
-      val tmpRow = new SpecificInternalRow(Seq(rootCatalystType))
-      val fieldUpdater = new RowUpdater(tmpRow)
-      val writer = newWriter(rootAvroType, rootCatalystType, Nil)
-      (data: Any) => {
-        writer(fieldUpdater, 0, data)
-        tmpRow.get(0, rootCatalystType)
-      }
+      case _ =>
+        val tmpRow = new SpecificInternalRow(Seq(rootCatalystType))
+        val fieldUpdater = new RowUpdater(tmpRow)
+        val writer = newWriter(rootAvroType, rootCatalystType, Nil, Nil)
+        (data: Any) => {
+          writer(fieldUpdater, 0, data)
+          Some(tmpRow.get(0, rootCatalystType))
+        }
+    }
+  } catch {
+    case ise: IncompatibleSchemaException => throw new IncompatibleSchemaException(
+      s"Cannot convert Avro type $rootAvroType to SQL type ${rootCatalystType.sql}.", ise)
   }
 
-  def deserialize(data: Any): Any = converter(data)
+  def deserialize(data: Any): Option[Any] = converter(data)
 
   /**
    * Creates a writer to write avro values to Catalyst values at the given ordinal with the given
@@ -95,7 +107,13 @@ class AvroDeserializer(
   private def newWriter(
       avroType: Schema,
       catalystType: DataType,
-      path: List[String]): (CatalystDataUpdater, Int, Any) => Unit =
+      avroPath: Seq[String],
+      catalystPath: Seq[String]): (CatalystDataUpdater, Int, Any) => Unit = {
+    val errorPrefix = s"Cannot convert Avro ${toFieldStr(avroPath)} to " +
+        s"SQL ${toFieldStr(catalystPath)} because "
+    val incompatibleMsg = errorPrefix +
+        s"schema is incompatible (avroType = $avroType, sqlType = ${catalystType.sql})"
+
     (avroType.getType, catalystType) match {
       case (NULL, NullType) => (updater, ordinal, _) =>
         updater.setNullAt(ordinal)
@@ -123,8 +141,8 @@ class AvroDeserializer(
         case _: TimestampMicros => (updater, ordinal, value) =>
           val micros = value.asInstanceOf[Long]
           updater.setLong(ordinal, timestampRebaseFunc(micros))
-        case other => throw new IncompatibleSchemaException(
-          s"Cannot convert Avro logical type ${other} to Catalyst Timestamp type.")
+        case other => throw new IncompatibleSchemaException(errorPrefix +
+          s"Avro logical type $other cannot be converted to SQL type ${TimestampType.sql}.")
       }
 
       // Before we upgrade Avro to 1.8 for logical type support, spark-avro converts Long to Date.
@@ -161,31 +179,37 @@ class AvroDeserializer(
             b.get(bytes)
             bytes
           case b: Array[Byte] => b
-          case other => throw new RuntimeException(s"$other is not a valid avro binary.")
+          case other =>
+            throw new RuntimeException(errorPrefix + s"$other is not a valid avro binary.")
         }
         updater.set(ordinal, bytes)
 
-      case (FIXED, d: DecimalType) => (updater, ordinal, value) =>
-        val bigDecimal = decimalConversions.fromFixed(value.asInstanceOf[GenericFixed], avroType,
-          LogicalTypes.decimal(d.precision, d.scale))
-        val decimal = createDecimal(bigDecimal, d.precision, d.scale)
+      case (FIXED, _: DecimalType) => (updater, ordinal, value) =>
+        val d = avroType.getLogicalType.asInstanceOf[LogicalTypes.Decimal]
+        val bigDecimal = decimalConversions.fromFixed(value.asInstanceOf[GenericFixed], avroType, d)
+        val decimal = createDecimal(bigDecimal, d.getPrecision, d.getScale)
         updater.setDecimal(ordinal, decimal)
 
-      case (BYTES, d: DecimalType) => (updater, ordinal, value) =>
-        val bigDecimal = decimalConversions.fromBytes(value.asInstanceOf[ByteBuffer], avroType,
-          LogicalTypes.decimal(d.precision, d.scale))
-        val decimal = createDecimal(bigDecimal, d.precision, d.scale)
+      case (BYTES, _: DecimalType) => (updater, ordinal, value) =>
+        val d = avroType.getLogicalType.asInstanceOf[LogicalTypes.Decimal]
+        val bigDecimal = decimalConversions.fromBytes(value.asInstanceOf[ByteBuffer], avroType, d)
+        val decimal = createDecimal(bigDecimal, d.getPrecision, d.getScale)
         updater.setDecimal(ordinal, decimal)
 
       case (RECORD, st: StructType) =>
-        val writeRecord = getRecordWriter(avroType, st, path)
+        // Avro datasource doesn't accept filters with nested attributes. See SPARK-32328.
+        // We can always return `false` from `applyFilters` for nested records.
+        val writeRecord =
+          getRecordWriter(avroType, st, avroPath, catalystPath, applyFilters = _ => false)
         (updater, ordinal, value) =>
           val row = new SpecificInternalRow(st)
           writeRecord(new RowUpdater(row), value.asInstanceOf[GenericRecord])
           updater.set(ordinal, row)
 
       case (ARRAY, ArrayType(elementType, containsNull)) =>
-        val elementWriter = newWriter(avroType.getElementType, elementType, path)
+        val avroElementPath = avroPath :+ "element"
+        val elementWriter = newWriter(avroType.getElementType, elementType,
+          avroElementPath, catalystPath :+ "element")
         (updater, ordinal, value) =>
           val collection = value.asInstanceOf[java.util.Collection[Any]]
           val result = createArrayData(elementType, collection.size())
@@ -197,8 +221,8 @@ class AvroDeserializer(
             val element = iter.next()
             if (element == null) {
               if (!containsNull) {
-                throw new RuntimeException(s"Array value at path ${path.mkString(".")} is not " +
-                  "allowed to be null")
+                throw new RuntimeException(
+                  s"Array value at path ${toFieldStr(avroElementPath)} is not allowed to be null")
               } else {
                 elementUpdater.setNullAt(i)
               }
@@ -211,8 +235,10 @@ class AvroDeserializer(
           updater.set(ordinal, result)
 
       case (MAP, MapType(keyType, valueType, valueContainsNull)) if keyType == StringType =>
-        val keyWriter = newWriter(SchemaBuilder.builder().stringType(), StringType, path)
-        val valueWriter = newWriter(avroType.getValueType, valueType, path)
+        val keyWriter = newWriter(SchemaBuilder.builder().stringType(), StringType,
+          avroPath :+ "key", catalystPath :+ "key")
+        val valueWriter = newWriter(avroType.getValueType, valueType,
+          avroPath :+ "value", catalystPath :+ "value")
         (updater, ordinal, value) =>
           val map = value.asInstanceOf[java.util.Map[AnyRef, AnyRef]]
           val keyArray = createArrayData(keyType, map.size())
@@ -227,8 +253,8 @@ class AvroDeserializer(
             keyWriter(keyUpdater, i, entry.getKey)
             if (entry.getValue == null) {
               if (!valueContainsNull) {
-                throw new RuntimeException(s"Map value at path ${path.mkString(".")} is not " +
-                  "allowed to be null")
+                throw new RuntimeException(
+                  s"Map value at path ${toFieldStr(avroPath :+ "value")} is not allowed to be null")
               } else {
                 valueUpdater.setNullAt(i)
               }
@@ -248,9 +274,9 @@ class AvroDeserializer(
         val nonNullAvroType = Schema.createUnion(nonNullTypes.asJava)
         if (nonNullTypes.nonEmpty) {
           if (nonNullTypes.length == 1) {
-            newWriter(nonNullTypes.head, catalystType, path)
+            newWriter(nonNullTypes.head, catalystType, avroPath, catalystPath)
           } else {
-            nonNullTypes.map(_.getType) match {
+            nonNullTypes.map(_.getType).toSeq match {
               case Seq(a, b) if Set(a, b) == Set(INT, LONG) && catalystType == LongType =>
                 (updater, ordinal, value) => value match {
                   case null => updater.setNullAt(ordinal)
@@ -269,7 +295,8 @@ class AvroDeserializer(
                 catalystType match {
                   case st: StructType if st.length == nonNullTypes.size =>
                     val fieldWriters = nonNullTypes.zip(st.fields).map {
-                      case (schema, field) => newWriter(schema, field.dataType, path :+ field.name)
+                      case (schema, field) =>
+                        newWriter(schema, field.dataType, avroPath, catalystPath :+ field.name)
                     }.toArray
                     (updater, ordinal, value) => {
                       val row = new SpecificInternalRow(st)
@@ -279,27 +306,17 @@ class AvroDeserializer(
                       updater.set(ordinal, row)
                     }
 
-                  case _ =>
-                    throw new IncompatibleSchemaException(
-                      s"Cannot convert Avro to catalyst because schema at path " +
-                        s"${path.mkString(".")} is not compatible " +
-                        s"(avroType = $avroType, sqlType = $catalystType).\n" +
-                        s"Source Avro schema: $rootAvroType.\n" +
-                        s"Target Catalyst type: $rootCatalystType")
+                  case _ => throw new IncompatibleSchemaException(incompatibleMsg)
                 }
             }
           }
         } else {
-          (updater, ordinal, value) => updater.setNullAt(ordinal)
+          (updater, ordinal, _) => updater.setNullAt(ordinal)
         }
 
-      case _ =>
-        throw new IncompatibleSchemaException(
-          s"Cannot convert Avro to catalyst because schema at path ${path.mkString(".")} " +
-            s"is not compatible (avroType = $avroType, sqlType = $catalystType).\n" +
-            s"Source Avro schema: $rootAvroType.\n" +
-            s"Target Catalyst type: $rootCatalystType")
+      case _ => throw new IncompatibleSchemaException(incompatibleMsg)
     }
+  }
 
   // TODO: move the following method in Decimal object on creating Decimal from BigDecimal?
   private def createDecimal(decimal: BigDecimal, precision: Int, scale: Int): Decimal = {
@@ -314,46 +331,49 @@ class AvroDeserializer(
 
   private def getRecordWriter(
       avroType: Schema,
-      sqlType: StructType,
-      path: List[String]): (CatalystDataUpdater, GenericRecord) => Unit = {
+      catalystType: StructType,
+      avroPath: Seq[String],
+      catalystPath: Seq[String],
+      applyFilters: Int => Boolean): (CatalystDataUpdater, GenericRecord) => Boolean = {
     val validFieldIndexes = ArrayBuffer.empty[Int]
     val fieldWriters = ArrayBuffer.empty[(CatalystDataUpdater, Any) => Unit]
 
-    val length = sqlType.length
+    val length = catalystType.length
     var i = 0
     while (i < length) {
-      val sqlField = sqlType.fields(i)
-      val avroField = avroType.getField(sqlField.name)
-      if (avroField != null) {
-        validFieldIndexes += avroField.pos()
-
-        val baseWriter = newWriter(avroField.schema(), sqlField.dataType, path :+ sqlField.name)
-        val ordinal = i
-        val fieldWriter = (fieldUpdater: CatalystDataUpdater, value: Any) => {
-          if (value == null) {
-            fieldUpdater.setNullAt(ordinal)
-          } else {
-            baseWriter(fieldUpdater, ordinal, value)
+      val catalystField = catalystType.fields(i)
+      AvroUtils.getAvroFieldByName(avroType, catalystField.name, avroPath) match {
+        case Some(avroField) =>
+          validFieldIndexes += avroField.pos()
+
+          val baseWriter = newWriter(avroField.schema(), catalystField.dataType,
+            avroPath :+ avroField.name, catalystPath :+ catalystField.name)
+          val ordinal = i
+          val fieldWriter = (fieldUpdater: CatalystDataUpdater, value: Any) => {
+            if (value == null) {
+              fieldUpdater.setNullAt(ordinal)
+            } else {
+              baseWriter(fieldUpdater, ordinal, value)
+            }
           }
-        }
-        fieldWriters += fieldWriter
-      } else if (!sqlField.nullable) {
-        throw new IncompatibleSchemaException(
-          s"""
-             |Cannot find non-nullable field ${path.mkString(".")}.${sqlField.name} in Avro schema.
-             |Source Avro schema: $rootAvroType.
-             |Target Catalyst type: $rootCatalystType.
-           """.stripMargin)
+          fieldWriters += fieldWriter
+        case None if !catalystField.nullable =>
+          throw new IncompatibleSchemaException(s"Cannot find non-nullable " +
+              s"${toFieldStr(catalystPath :+ catalystField.name)} in Avro schema.")
+        case _ => // nothing to do
       }
       i += 1
     }
 
     (fieldUpdater, record) => {
       var i = 0
-      while (i < validFieldIndexes.length) {
+      var skipRow = false
+      while (i < validFieldIndexes.length && !skipRow) {
         fieldWriters(i)(fieldUpdater, record.get(validFieldIndexes(i)))
+        skipRow = applyFilters(i)
         i += 1
       }
+      skipRow
     }
   }
 
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala
index 59d54bc433f8b..11f14c2a8013c 100755
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala
@@ -22,7 +22,6 @@ import java.net.URI
 
 import scala.util.control.NonFatal
 
-import org.apache.avro.Schema
 import org.apache.avro.file.DataFileReader
 import org.apache.avro.generic.{GenericDatumReader, GenericRecord}
 import org.apache.avro.mapred.FsInput
@@ -33,7 +32,7 @@ import org.apache.hadoop.mapreduce.Job
 import org.apache.spark.TaskContext
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.{InternalRow, NoopFilters, OrderedFilters}
 import org.apache.spark.sql.execution.datasources.{DataSourceUtils, FileFormat, OutputWriterFactory, PartitionedFile}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.sources.{DataSourceRegister, Filter}
@@ -87,10 +86,11 @@ private[sql] class AvroFileFormat extends FileFormat
     val broadcastedConf =
       spark.sparkContext.broadcast(new SerializableConfiguration(hadoopConf))
     val parsedOptions = new AvroOptions(options, hadoopConf)
+    val datetimeRebaseModeInRead = parsedOptions.datetimeRebaseModeInRead
 
     (file: PartitionedFile) => {
       val conf = broadcastedConf.value.value
-      val userProvidedSchema = parsedOptions.schema.map(new Schema.Parser().parse)
+      val userProvidedSchema = parsedOptions.schema
 
       // TODO Removes this check once `FileFormat` gets a general file filtering interface method.
       // Doing input file filtering is improper because we may generate empty tasks that process no
@@ -122,38 +122,28 @@ private[sql] class AvroFileFormat extends FileFormat
         }
 
         reader.sync(file.start)
-        val stop = file.start + file.length
 
         val datetimeRebaseMode = DataSourceUtils.datetimeRebaseMode(
           reader.asInstanceOf[DataFileReader[_]].getMetaString,
-          SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ))
-
-        val deserializer = new AvroDeserializer(
-          userProvidedSchema.getOrElse(reader.getSchema), requiredSchema, datetimeRebaseMode)
-
-        new Iterator[InternalRow] {
-          private[this] var completed = false
-
-          override def hasNext: Boolean = {
-            if (completed) {
-              false
-            } else {
-              val r = reader.hasNext && !reader.pastSync(stop)
-              if (!r) {
-                reader.close()
-                completed = true
-              }
-              r
-            }
-          }
+          datetimeRebaseModeInRead)
 
-          override def next(): InternalRow = {
-            if (!hasNext) {
-              throw new NoSuchElementException("next on empty iterator")
-            }
-            val record = reader.next()
-            deserializer.deserialize(record).asInstanceOf[InternalRow]
-          }
+        val avroFilters = if (SQLConf.get.avroFilterPushDown) {
+          new OrderedFilters(filters, requiredSchema)
+        } else {
+          new NoopFilters
+        }
+
+        new Iterator[InternalRow] with AvroUtils.RowReader {
+          override val fileReader = reader
+          override val deserializer = new AvroDeserializer(
+            userProvidedSchema.getOrElse(reader.getSchema),
+            requiredSchema,
+            datetimeRebaseMode,
+            avroFilters)
+          override val stopPosition = file.start + file.length
+
+          override def hasNext: Boolean = hasNextRow
+          override def next(): InternalRow = nextRow
         }
       } else {
         Iterator.empty
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala
index f3ea78583fbc0..7f8d7f492abce 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala
@@ -17,7 +17,11 @@
 
 package org.apache.spark.sql.avro
 
+import java.net.URI
+
+import org.apache.avro.Schema
 import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.{FileSystem, Path}
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.SparkSession
@@ -27,7 +31,7 @@ import org.apache.spark.sql.internal.SQLConf
 /**
  * Options for Avro Reader and Writer stored in case insensitive manner.
  */
-class AvroOptions(
+private[sql] class AvroOptions(
     @transient val parameters: CaseInsensitiveMap[String],
     @transient val conf: Configuration) extends Logging with Serializable {
 
@@ -36,7 +40,7 @@ class AvroOptions(
   }
 
   /**
-   * Optional schema provided by a user in JSON format.
+   * Optional schema provided by a user in schema file or in JSON format.
    *
    * When reading Avro, this option can be set to an evolved schema, which is compatible but
    * different with the actual Avro schema. The deserialization schema will be consistent with
@@ -47,18 +51,32 @@ class AvroOptions(
    * schema converted by Spark. For example, the expected schema of one column is of "enum" type,
    * instead of "string" type in the default converted schema.
    */
-  val schema: Option[String] = parameters.get("avroSchema")
+  val schema: Option[Schema] = {
+    parameters.get("avroSchema").map(new Schema.Parser().parse).orElse({
+      val avroUrlSchema = parameters.get("avroSchemaUrl").map(url => {
+        log.debug("loading avro schema from url: " + url)
+        val fs = FileSystem.get(new URI(url), conf)
+        val in = fs.open(new Path(url))
+        try {
+          new Schema.Parser().parse(in)
+        } finally {
+          in.close()
+        }
+      })
+      avroUrlSchema
+    })
+  }
 
   /**
    * Top level record name in write result, which is required in Avro spec.
-   * See https://avro.apache.org/docs/1.8.2/spec.html#schema_record .
+   * See https://avro.apache.org/docs/1.10.2/spec.html#schema_record .
    * Default value is "topLevelRecord"
    */
   val recordName: String = parameters.getOrElse("recordName", "topLevelRecord")
 
   /**
    * Record namespace in write result. Default value is "".
-   * See Avro spec for details: https://avro.apache.org/docs/1.8.2/spec.html#schema_record .
+   * See Avro spec for details: https://avro.apache.org/docs/1.10.2/spec.html#schema_record .
    */
   val recordNamespace: String = parameters.getOrElse("recordNamespace", "")
 
@@ -83,9 +101,9 @@ class AvroOptions(
 
   /**
    * The `compression` option allows to specify a compression codec used in write.
-   * Currently supported codecs are `uncompressed`, `snappy`, `deflate`, `bzip2` and `xz`.
-   * If the option is not set, the `spark.sql.avro.compression.codec` config is taken into
-   * account. If the former one is not set too, the `snappy` codec is used by default.
+   * Currently supported codecs are `uncompressed`, `snappy`, `deflate`, `bzip2`, `xz` and
+   * `zstandard`. If the option is not set, the `spark.sql.avro.compression.codec` config is
+   * taken into account. If the former one is not set too, the `snappy` codec is used by default.
    */
   val compression: String = {
     parameters.get("compression").getOrElse(SQLConf.get.avroCompressionCodec)
@@ -93,9 +111,16 @@ class AvroOptions(
 
   val parseMode: ParseMode =
     parameters.get("mode").map(ParseMode.fromString).getOrElse(FailFastMode)
+
+  /**
+   * The rebasing mode for the DATE and TIMESTAMP_MICROS, TIMESTAMP_MILLIS values in reads.
+   */
+  val datetimeRebaseModeInRead: String = parameters
+    .get(AvroOptions.DATETIME_REBASE_MODE)
+    .getOrElse(SQLConf.get.getConf(SQLConf.AVRO_REBASE_MODE_IN_READ))
 }
 
-object AvroOptions {
+private[sql] object AvroOptions {
   def apply(parameters: Map[String, String]): AvroOptions = {
     val hadoopConf = SparkSession
       .getActiveSession
@@ -105,4 +130,10 @@ object AvroOptions {
   }
 
   val ignoreExtensionKey = "ignoreExtension"
+
+  // The option controls rebasing of the DATE and TIMESTAMP values between
+  // Julian and Proleptic Gregorian calendars. It impacts on the behaviour of the Avro
+  // datasource similarly to the SQL config `spark.sql.avro.datetimeRebaseModeInRead`,
+  // and can be set to the same values: `EXCEPTION`, `LEGACY` or `CORRECTED`.
+  val DATETIME_REBASE_MODE = "datetimeRebaseMode"
 }
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOutputWriter.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOutputWriter.scala
index ac9608c867937..424526eafdfaa 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOutputWriter.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOutputWriter.scala
@@ -38,14 +38,14 @@ import org.apache.spark.sql.types._
 
 // NOTE: This class is instantiated and used on executor side only, no need to be serializable.
 private[avro] class AvroOutputWriter(
-    path: String,
+    val path: String,
     context: TaskAttemptContext,
     schema: StructType,
     avroSchema: Schema) extends OutputWriter {
 
   // Whether to rebase datetimes from Gregorian to Julian calendar in write
   private val datetimeRebaseMode = LegacyBehaviorPolicy.withName(
-    SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE))
+    SQLConf.get.getConf(SQLConf.AVRO_REBASE_MODE_IN_WRITE))
 
   // The input rows will never be null.
   private lazy val serializer =
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
index 21c5dec6239bd..866575726b7a8 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
@@ -32,6 +32,7 @@ import org.apache.avro.generic.GenericData.Record
 import org.apache.avro.util.Utf8
 
 import org.apache.spark.internal.Logging
+import org.apache.spark.sql.avro.AvroUtils.toFieldStr
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{SpecializedGetters, SpecificInternalRow}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
@@ -43,16 +44,16 @@ import org.apache.spark.sql.types._
 /**
  * A serializer to serialize data in catalyst format to data in avro format.
  */
-class AvroSerializer(
+private[sql] class AvroSerializer(
     rootCatalystType: DataType,
     rootAvroType: Schema,
     nullable: Boolean,
     datetimeRebaseMode: LegacyBehaviorPolicy.Value) extends Logging {
 
-  def this(rootCatalystType: DataType, rootAvroType: Schema, nullable: Boolean) {
+  def this(rootCatalystType: DataType, rootAvroType: Schema, nullable: Boolean) = {
     this(rootCatalystType, rootAvroType, nullable,
       LegacyBehaviorPolicy.withName(SQLConf.get.getConf(
-        SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE)))
+        SQLConf.AVRO_REBASE_MODE_IN_WRITE)))
   }
 
   def serialize(catalystData: Any): Any = {
@@ -67,15 +68,20 @@ class AvroSerializer(
 
   private val converter: Any => Any = {
     val actualAvroType = resolveNullableType(rootAvroType, nullable)
-    val baseConverter = rootCatalystType match {
-      case st: StructType =>
-        newStructConverter(st, actualAvroType).asInstanceOf[Any => Any]
-      case _ =>
-        val tmpRow = new SpecificInternalRow(Seq(rootCatalystType))
-        val converter = newConverter(rootCatalystType, actualAvroType)
-        (data: Any) =>
-          tmpRow.update(0, data)
-          converter.apply(tmpRow, 0)
+    val baseConverter = try {
+      rootCatalystType match {
+        case st: StructType =>
+          newStructConverter(st, actualAvroType, Nil, Nil).asInstanceOf[Any => Any]
+        case _ =>
+          val tmpRow = new SpecificInternalRow(Seq(rootCatalystType))
+          val converter = newConverter(rootCatalystType, actualAvroType, Nil, Nil)
+          (data: Any) =>
+            tmpRow.update(0, data)
+            converter.apply(tmpRow, 0)
+      }
+    } catch {
+      case ise: IncompatibleSchemaException => throw new IncompatibleSchemaException(
+        s"Cannot convert SQL type ${rootCatalystType.sql} to Avro type $rootAvroType.", ise)
     }
     if (nullable) {
       (data: Any) =>
@@ -93,7 +99,13 @@ class AvroSerializer(
 
   private lazy val decimalConversions = new DecimalConversion()
 
-  private def newConverter(catalystType: DataType, avroType: Schema): Converter = {
+  private def newConverter(
+      catalystType: DataType,
+      avroType: Schema,
+      catalystPath: Seq[String],
+      avroPath: Seq[String]): Converter = {
+    val errorPrefix = s"Cannot convert SQL ${toFieldStr(catalystPath)} " +
+      s"to Avro ${toFieldStr(avroPath)} because "
     (catalystType, avroType.getType) match {
       case (NullType, NULL) =>
         (getter, ordinal) => null
@@ -130,9 +142,9 @@ class AvroSerializer(
         (getter, ordinal) =>
           val data = getter.getUTF8String(ordinal).toString
           if (!enumSymbols.contains(data)) {
-            throw new IncompatibleSchemaException(
-              "Cannot write \"" + data + "\" since it's not defined in enum \"" +
-                enumSymbols.mkString("\", \"") + "\"")
+            throw new IncompatibleSchemaException(errorPrefix +
+              s""""$data" cannot be written since it's not defined in enum """ +
+                enumSymbols.mkString("\"", "\", \"", "\""))
           }
           new EnumSymbol(avroType, data)
 
@@ -140,14 +152,13 @@ class AvroSerializer(
         (getter, ordinal) => new Utf8(getter.getUTF8String(ordinal).getBytes)
 
       case (BinaryType, FIXED) =>
-        val size = avroType.getFixedSize()
+        val size = avroType.getFixedSize
         (getter, ordinal) =>
           val data: Array[Byte] = getter.getBinary(ordinal)
           if (data.length != size) {
-            throw new IncompatibleSchemaException(
-              s"Cannot write ${data.length} ${if (data.length > 1) "bytes" else "byte"} of " +
-                "binary data into FIXED Type with size of " +
-                s"$size ${if (size > 1) "bytes" else "byte"}")
+            def len2str(len: Int): String = s"$len ${if (len > 1) "bytes" else "byte"}"
+            throw new IncompatibleSchemaException(errorPrefix + len2str(data.length) +
+                " of binary data cannot be written into FIXED type with size of " + len2str(size))
           }
           new Fixed(avroType, data)
 
@@ -164,13 +175,14 @@ class AvroSerializer(
             DateTimeUtils.microsToMillis(timestampRebaseFunc(getter.getLong(ordinal)))
           case _: TimestampMicros => (getter, ordinal) =>
             timestampRebaseFunc(getter.getLong(ordinal))
-          case other => throw new IncompatibleSchemaException(
-            s"Cannot convert Catalyst Timestamp type to Avro logical type ${other}")
+          case other => throw new IncompatibleSchemaException(errorPrefix +
+            s"SQL type ${TimestampType.sql} cannot be converted to Avro logical type $other")
         }
 
       case (ArrayType(et, containsNull), ARRAY) =>
         val elementConverter = newConverter(
-          et, resolveNullableType(avroType.getElementType, containsNull))
+          et, resolveNullableType(avroType.getElementType, containsNull),
+          catalystPath :+ "element", avroPath :+ "element")
         (getter, ordinal) => {
           val arrayData = getter.getArray(ordinal)
           val len = arrayData.numElements()
@@ -190,13 +202,14 @@ class AvroSerializer(
         }
 
       case (st: StructType, RECORD) =>
-        val structConverter = newStructConverter(st, avroType)
+        val structConverter = newStructConverter(st, avroType, catalystPath, avroPath)
         val numFields = st.length
         (getter, ordinal) => structConverter(getter.getStruct(ordinal, numFields))
 
       case (MapType(kt, vt, valueContainsNull), MAP) if kt == StringType =>
         val valueConverter = newConverter(
-          vt, resolveNullableType(avroType.getValueType, valueContainsNull))
+          vt, resolveNullableType(avroType.getValueType, valueContainsNull),
+          catalystPath :+ "value", avroPath :+ "value")
         (getter, ordinal) =>
           val mapData = getter.getMap(ordinal)
           val len = mapData.numElements()
@@ -215,28 +228,40 @@ class AvroSerializer(
           }
           result
 
-      case other =>
-        throw new IncompatibleSchemaException(s"Cannot convert Catalyst type $catalystType to " +
-          s"Avro type $avroType.")
+      case _ =>
+        throw new IncompatibleSchemaException(errorPrefix +
+          s"schema is incompatible (sqlType = ${catalystType.sql}, avroType = $avroType)")
     }
   }
 
   private def newStructConverter(
-      catalystStruct: StructType, avroStruct: Schema): InternalRow => Record = {
-    if (avroStruct.getType != RECORD || avroStruct.getFields.size() != catalystStruct.length) {
-      throw new IncompatibleSchemaException(s"Cannot convert Catalyst type $catalystStruct to " +
-        s"Avro type $avroStruct.")
+      catalystStruct: StructType,
+      avroStruct: Schema,
+      catalystPath: Seq[String],
+      avroPath: Seq[String]): InternalRow => Record = {
+
+    val avroPathStr = toFieldStr(avroPath)
+    if (avroStruct.getType != RECORD) {
+      throw new IncompatibleSchemaException(s"$avroPathStr was not a RECORD")
+    }
+    val avroFields = avroStruct.getFields.asScala
+    if (avroFields.size != catalystStruct.length) {
+      throw new IncompatibleSchemaException(
+        s"Avro $avroPathStr schema length (${avroFields.size}) doesn't match " +
+        s"SQL ${toFieldStr(catalystPath)} schema length (${catalystStruct.length})")
     }
 
     val (avroIndices: Array[Int], fieldConverters: Array[Converter]) =
       catalystStruct.map { catalystField =>
-        val avroField = avroStruct.getField(catalystField.name)
-        if (avroField == null) {
-          throw new IncompatibleSchemaException(
-            s"Cannot convert Catalyst type $catalystStruct to Avro type $avroStruct.")
+        val avroField = AvroUtils
+            .getAvroFieldByName(avroStruct, catalystField.name, avroPath) match {
+          case Some(f) => f
+          case None => throw new IncompatibleSchemaException(s"Cannot find " +
+              s"${toFieldStr(catalystPath :+ catalystField.name)} in Avro schema at $avroPathStr")
         }
-        val converter = newConverter(catalystField.dataType, resolveNullableType(
-          avroField.schema(), catalystField.nullable))
+        val converter = newConverter(catalystField.dataType,
+          resolveNullableType(avroField.schema(), catalystField.nullable),
+          catalystPath :+ catalystField.name, avroPath :+ avroField.name)
         (avroField.pos(), converter)
       }.toArray.unzip
 
@@ -255,20 +280,54 @@ class AvroSerializer(
       result
   }
 
+  /**
+   * Resolve a possibly nullable Avro Type.
+   *
+   * An Avro type is nullable when it is a [[UNION]] of two types: one null type and another
+   * non-null type. This method will check the nullability of the input Avro type and return the
+   * non-null type within when it is nullable. Otherwise it will return the input Avro type
+   * unchanged. It will throw an [[UnsupportedAvroTypeException]] when the input Avro type is an
+   * unsupported nullable type.
+   *
+   * It will also log a warning message if the nullability for Avro and catalyst types are
+   * different.
+   */
   private def resolveNullableType(avroType: Schema, nullable: Boolean): Schema = {
-    if (avroType.getType == Type.UNION && nullable) {
-      // avro uses union to represent nullable type.
+    val (avroNullable, resolvedAvroType) = resolveAvroType(avroType)
+    warnNullabilityDifference(avroNullable, nullable)
+    resolvedAvroType
+  }
+
+  /**
+   * Check the nullability of the input Avro type and resolve it when it is nullable. The first
+   * return value is a [[Boolean]] indicating if the input Avro type is nullable. The second
+   * return value is the possibly resolved type.
+   */
+  private def resolveAvroType(avroType: Schema): (Boolean, Schema) = {
+    if (avroType.getType == Type.UNION) {
       val fields = avroType.getTypes.asScala
-      assert(fields.length == 2)
       val actualType = fields.filter(_.getType != Type.NULL)
-      assert(actualType.length == 1)
-      actualType.head
-    } else {
-      if (nullable) {
-        logWarning("Writing avro files with non-nullable avro schema with nullable catalyst " +
-          "schema will throw runtime exception if there is a record with null value.")
+      if (fields.length != 2 || actualType.length != 1) {
+        throw new UnsupportedAvroTypeException(
+          s"Unsupported Avro UNION type $avroType: Only UNION of a null type and a non-null " +
+            "type is supported")
       }
-      avroType
+      (true, actualType.head)
+    } else {
+      (false, avroType)
+    }
+  }
+
+  /**
+   * log a warning message if the nullability for Avro and catalyst types are different.
+   */
+  private def warnNullabilityDifference(avroNullable: Boolean, catalystNullable: Boolean): Unit = {
+    if (avroNullable && !catalystNullable) {
+      logWarning("Writing Avro files with nullable Avro schema and non-nullable catalyst schema.")
+    }
+    if (!avroNullable && catalystNullable) {
+      logWarning("Writing Avro files with non-nullable Avro schema and nullable catalyst " +
+        "schema will throw runtime exception if there is a record with null value.")
     }
   }
 }
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala
index 70dcd58a600fc..74f4d0e649587 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala
@@ -18,9 +18,11 @@ package org.apache.spark.sql.avro
 
 import java.io.{FileNotFoundException, IOException}
 
+import scala.collection.JavaConverters._
+
 import org.apache.avro.Schema
-import org.apache.avro.file.DataFileConstants.{BZIP2_CODEC, DEFLATE_CODEC, SNAPPY_CODEC, XZ_CODEC}
-import org.apache.avro.file.DataFileReader
+import org.apache.avro.file.{DataFileReader, FileReader}
+import org.apache.avro.file.DataFileConstants.{BZIP2_CODEC, DEFLATE_CODEC, SNAPPY_CODEC, XZ_CODEC, ZSTANDARD_CODEC}
 import org.apache.avro.generic.{GenericDatumReader, GenericRecord}
 import org.apache.avro.mapred.{AvroOutputFormat, FsInput}
 import org.apache.avro.mapreduce.AvroJob
@@ -32,17 +34,18 @@ import org.apache.spark.SparkException
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.avro.AvroOptions.ignoreExtensionKey
+import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.execution.datasources.OutputWriterFactory
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.util.Utils
 
-object AvroUtils extends Logging {
+private[sql] object AvroUtils extends Logging {
   def inferSchema(
       spark: SparkSession,
       options: Map[String, String],
       files: Seq[FileStatus]): Option[StructType] = {
-    val conf = spark.sessionState.newHadoopConf()
+    val conf = spark.sessionState.newHadoopConfWithOptions(options)
     val parsedOptions = new AvroOptions(options, conf)
 
     if (parsedOptions.parameters.contains(ignoreExtensionKey)) {
@@ -51,7 +54,6 @@ object AvroUtils extends Logging {
     }
     // User can specify an optional avro json schema.
     val avroSchema = parsedOptions.schema
-      .map(new Schema.Parser().parse)
       .getOrElse {
         inferAvroSchemaFromFiles(files, conf, parsedOptions.ignoreExtension,
           spark.sessionState.conf.ignoreCorruptFiles)
@@ -91,7 +93,6 @@ object AvroUtils extends Logging {
       dataSchema: StructType): OutputWriterFactory = {
     val parsedOptions = new AvroOptions(options, job.getConfiguration)
     val outputAvroSchema: Schema = parsedOptions.schema
-      .map(new Schema.Parser().parse)
       .getOrElse(SchemaConverters.toAvroType(dataSchema, nullable = false,
         parsedOptions.recordName, parsedOptions.recordNamespace))
 
@@ -108,7 +109,7 @@ object AvroUtils extends Logging {
           logInfo(s"Avro compression level $deflateLevel will be used for $DEFLATE_CODEC codec.")
           job.getConfiguration.setInt(AvroOutputFormat.DEFLATE_LEVEL_KEY, deflateLevel)
           DEFLATE_CODEC
-        case codec @ (SNAPPY_CODEC | BZIP2_CODEC | XZ_CODEC) => codec
+        case codec @ (SNAPPY_CODEC | BZIP2_CODEC | XZ_CODEC | ZSTANDARD_CODEC) => codec
         case unknown => throw new IllegalArgumentException(s"Invalid compression codec: $unknown")
       }
       job.getConfiguration.set(AvroJob.CONF_OUTPUT_CODEC, codec)
@@ -161,4 +162,84 @@ object AvroUtils extends Logging {
           "No Avro files found. If files don't have .avro extension, set ignoreExtension to true")
     }
   }
+
+  // The trait provides iterator-like interface for reading records from an Avro file,
+  // deserializing and returning them as internal rows.
+  trait RowReader {
+    protected val fileReader: FileReader[GenericRecord]
+    protected val deserializer: AvroDeserializer
+    protected val stopPosition: Long
+
+    private[this] var completed = false
+    private[this] var currentRow: Option[InternalRow] = None
+
+    def hasNextRow: Boolean = {
+      while (!completed && currentRow.isEmpty) {
+        val r = fileReader.hasNext && !fileReader.pastSync(stopPosition)
+        if (!r) {
+          fileReader.close()
+          completed = true
+          currentRow = None
+        } else {
+          val record = fileReader.next()
+          // the row must be deserialized in hasNextRow, because AvroDeserializer#deserialize
+          // potentially filters rows
+          currentRow = deserializer.deserialize(record).asInstanceOf[Option[InternalRow]]
+        }
+      }
+      currentRow.isDefined
+    }
+
+    def nextRow: InternalRow = {
+      if (currentRow.isEmpty) {
+        hasNextRow
+      }
+      val returnRow = currentRow
+      currentRow = None // free up hasNextRow to consume more Avro records, if not exhausted
+      returnRow.getOrElse {
+        throw new NoSuchElementException("next on empty iterator")
+      }
+    }
+  }
+
+  /**
+   * Extract a single field from `avroSchema` which has the desired field name,
+   * performing the matching with proper case sensitivity according to [[SQLConf.resolver]].
+   *
+   * @param avroSchema The schema in which to search for the field. Must be of type RECORD.
+   * @param name The name of the field to search for.
+   * @param avroPath The seq of parent field names leading to `avroSchema`.
+   * @return `Some(match)` if a matching Avro field is found, otherwise `None`.
+   * @throws IncompatibleSchemaException if `avroSchema` is not a RECORD or contains multiple
+   *                                     fields matching `name` (i.e., case-insensitive matching
+   *                                     is used and `avroSchema` has two or more fields that have
+   *                                     the same name with difference case).
+   */
+  private[avro] def getAvroFieldByName(
+      avroSchema: Schema,
+      name: String,
+      avroPath: Seq[String]): Option[Schema.Field] = {
+    if (avroSchema.getType != Schema.Type.RECORD) {
+      throw new IncompatibleSchemaException(
+        s"Attempting to treat ${avroSchema.getName} as a RECORD, but it was: ${avroSchema.getType}")
+    }
+    avroSchema.getFields.asScala.filter(f => SQLConf.get.resolver(f.name(), name)).toSeq match {
+      case Seq(avroField) => Some(avroField)
+      case Seq() => None
+      case matches => throw new IncompatibleSchemaException(s"Searching for '$name' in Avro " +
+          s"schema at ${toFieldStr(avroPath)} gave ${matches.size} matches. Candidates: " +
+          matches.map(_.name()).mkString("[", ", ", "]")
+      )
+    }
+  }
+
+  /**
+   * Convert a sequence of hierarchical field names (like `Seq(foo, bar)`) into a human-readable
+   * string representing the field, like "field 'foo.bar'". If `names` is empty, the string
+   * "top-level record" is returned.
+   */
+  private[avro] def toFieldStr(names: Seq[String]): String = names match {
+    case Seq() => "top-level record"
+    case n => s"field '${n.mkString(".")}'"
+  }
 }
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala
index 7732c83aaf7c4..5d79c44ad422e 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala
@@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, UnaryExpression}
 import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
 import org.apache.spark.sql.types.{BinaryType, DataType}
 
-case class CatalystDataToAvro(
+private[avro] case class CatalystDataToAvro(
     child: Expression,
     jsonFormatSchema: Option[String]) extends UnaryExpression {
 
@@ -64,4 +64,7 @@ case class CatalystDataToAvro(
     defineCodeGen(ctx, ev, input =>
       s"(byte[]) $expr.nullSafeEval($input)")
   }
+
+  override protected def withNewChildInternal(newChild: Expression): CatalystDataToAvro =
+    copy(child = newChild)
 }
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/SchemaConverters.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/SchemaConverters.scala
index 3947d327dfac6..09c849960c1b5 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/SchemaConverters.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/SchemaConverters.scala
@@ -24,29 +24,38 @@ import org.apache.avro.{LogicalTypes, Schema, SchemaBuilder}
 import org.apache.avro.LogicalTypes.{Date, Decimal, TimestampMicros, TimestampMillis}
 import org.apache.avro.Schema.Type._
 
+import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.sql.catalyst.util.RandomUUIDGenerator
 import org.apache.spark.sql.types._
-import org.apache.spark.sql.types.Decimal.{maxPrecisionForBytes, minBytesForPrecision}
+import org.apache.spark.sql.types.Decimal.minBytesForPrecision
 
 /**
  * This object contains method that are used to convert sparkSQL schemas to avro schemas and vice
  * versa.
  */
+@DeveloperApi
 object SchemaConverters {
   private lazy val uuidGenerator = RandomUUIDGenerator(new Random().nextLong())
 
   private lazy val nullSchema = Schema.create(Schema.Type.NULL)
 
+  /**
+   * Internal wrapper for SQL data type and nullability.
+   *
+   * @since 2.4.0
+   */
   case class SchemaType(dataType: DataType, nullable: Boolean)
 
   /**
-   * This function takes an avro schema and returns a sql schema.
+   * Converts an Avro schema to a corresponding Spark SQL schema.
+   *
+   * @since 2.4.0
    */
   def toSqlType(avroSchema: Schema): SchemaType = {
     toSqlTypeHelper(avroSchema, Set.empty)
   }
 
-  def toSqlTypeHelper(avroSchema: Schema, existingRecordNames: Set[String]): SchemaType = {
+  private def toSqlTypeHelper(avroSchema: Schema, existingRecordNames: Set[String]): SchemaType = {
     avroSchema.getType match {
       case INT => avroSchema.getLogicalType match {
         case _: Date => SchemaType(DateType, nullable = false)
@@ -85,7 +94,7 @@ object SchemaConverters {
           StructField(f.name, schemaType.dataType, schemaType.nullable)
         }
 
-        SchemaType(StructType(fields), nullable = false)
+        SchemaType(StructType(fields.toSeq), nullable = false)
 
       case ARRAY =>
         val schemaType = toSqlTypeHelper(avroSchema.getElementType, existingRecordNames)
@@ -109,7 +118,7 @@ object SchemaConverters {
             toSqlTypeHelper(Schema.createUnion(remainingUnionTypes.asJava), existingRecordNames)
               .copy(nullable = true)
           }
-        } else avroSchema.getTypes.asScala.map(_.getType) match {
+        } else avroSchema.getTypes.asScala.map(_.getType).toSeq match {
           case Seq(t1) =>
             toSqlTypeHelper(avroSchema.getTypes.get(0), existingRecordNames)
           case Seq(t1, t2) if Set(t1, t2) == Set(INT, LONG) =>
@@ -126,13 +135,18 @@ object SchemaConverters {
                 StructField(s"member$i", schemaType.dataType, nullable = true)
             }
 
-            SchemaType(StructType(fields), nullable = false)
+            SchemaType(StructType(fields.toSeq), nullable = false)
         }
 
       case other => throw new IncompatibleSchemaException(s"Unsupported type $other")
     }
   }
 
+  /**
+   * Converts a Spark SQL schema to a corresponding Avro schema.
+   *
+   * @since 2.4.0
+   */
   def toAvroType(
       catalystType: DataType,
       nullable: Boolean = false,
@@ -192,4 +206,7 @@ object SchemaConverters {
   }
 }
 
-class IncompatibleSchemaException(msg: String, ex: Throwable = null) extends Exception(msg, ex)
+private[avro] class IncompatibleSchemaException(
+  msg: String, ex: Throwable = null) extends Exception(msg, ex)
+
+private[avro] class UnsupportedAvroTypeException(msg: String) extends Exception(msg)
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroDataSourceV2.scala b/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroDataSourceV2.scala
index c6f52d676422c..2d21c493d5711 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroDataSourceV2.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroDataSourceV2.scala
@@ -31,13 +31,15 @@ class AvroDataSourceV2 extends FileDataSourceV2 {
 
   override def getTable(options: CaseInsensitiveStringMap): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
-    AvroTable(tableName, sparkSession, options, paths, None, fallbackFileFormat)
+    val tableName = getTableName(options, paths)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    AvroTable(tableName, sparkSession, optionsWithoutPaths, paths, None, fallbackFileFormat)
   }
 
   override def getTable(options: CaseInsensitiveStringMap, schema: StructType): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
-    AvroTable(tableName, sparkSession, options, paths, Some(schema), fallbackFileFormat)
+    val tableName = getTableName(options, paths)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    AvroTable(tableName, sparkSession, optionsWithoutPaths, paths, Some(schema), fallbackFileFormat)
   }
 }
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroPartitionReaderFactory.scala b/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroPartitionReaderFactory.scala
index 15918f46a83bb..00c3f038c8118 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroPartitionReaderFactory.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroPartitionReaderFactory.scala
@@ -20,7 +20,6 @@ import java.net.URI
 
 import scala.util.control.NonFatal
 
-import org.apache.avro.Schema
 import org.apache.avro.file.DataFileReader
 import org.apache.avro.generic.{GenericDatumReader, GenericRecord}
 import org.apache.avro.mapred.FsInput
@@ -29,12 +28,13 @@ import org.apache.hadoop.fs.Path
 import org.apache.spark.TaskContext
 import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.avro.{AvroDeserializer, AvroOptions}
-import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.avro.{AvroDeserializer, AvroOptions, AvroUtils}
+import org.apache.spark.sql.catalyst.{InternalRow, NoopFilters, OrderedFilters}
 import org.apache.spark.sql.connector.read.PartitionReader
 import org.apache.spark.sql.execution.datasources.{DataSourceUtils, PartitionedFile}
 import org.apache.spark.sql.execution.datasources.v2.{EmptyPartitionReader, FilePartitionReaderFactory, PartitionReaderWithPartitionValues}
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.util.SerializableConfiguration
 
@@ -54,11 +54,13 @@ case class AvroPartitionReaderFactory(
     dataSchema: StructType,
     readDataSchema: StructType,
     partitionSchema: StructType,
-    parsedOptions: AvroOptions) extends FilePartitionReaderFactory with Logging {
+    parsedOptions: AvroOptions,
+    filters: Seq[Filter]) extends FilePartitionReaderFactory with Logging {
+  private val datetimeRebaseModeInRead = parsedOptions.datetimeRebaseModeInRead
 
   override def buildReader(partitionedFile: PartitionedFile): PartitionReader[InternalRow] = {
     val conf = broadcastedConf.value.value
-    val userProvidedSchema = parsedOptions.schema.map(new Schema.Parser().parse)
+    val userProvidedSchema = parsedOptions.schema
 
     if (parsedOptions.ignoreExtension || partitionedFile.filePath.endsWith(".avro")) {
       val reader = {
@@ -86,38 +88,28 @@ case class AvroPartitionReaderFactory(
       }
 
       reader.sync(partitionedFile.start)
-      val stop = partitionedFile.start + partitionedFile.length
 
       val datetimeRebaseMode = DataSourceUtils.datetimeRebaseMode(
         reader.asInstanceOf[DataFileReader[_]].getMetaString,
-        SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ))
-      val deserializer = new AvroDeserializer(
-        userProvidedSchema.getOrElse(reader.getSchema), readDataSchema, datetimeRebaseMode)
+        datetimeRebaseModeInRead)
 
-      val fileReader = new PartitionReader[InternalRow] {
-        private[this] var completed = false
-
-        override def next(): Boolean = {
-          if (completed) {
-            false
-          } else {
-            val r = reader.hasNext && !reader.pastSync(stop)
-            if (!r) {
-              reader.close()
-              completed = true
-            }
-            r
-          }
-        }
+      val avroFilters = if (SQLConf.get.avroFilterPushDown) {
+        new OrderedFilters(filters, readDataSchema)
+      } else {
+        new NoopFilters
+      }
 
-        override def get(): InternalRow = {
-          if (!next) {
-            throw new NoSuchElementException("next on empty iterator")
-          }
-          val record = reader.next()
-          deserializer.deserialize(record).asInstanceOf[InternalRow]
-        }
+      val fileReader = new PartitionReader[InternalRow] with AvroUtils.RowReader {
+        override val fileReader = reader
+        override val deserializer = new AvroDeserializer(
+          userProvidedSchema.getOrElse(reader.getSchema),
+          readDataSchema,
+          datetimeRebaseMode,
+          avroFilters)
+        override val stopPosition = partitionedFile.start + partitionedFile.length
 
+        override def next(): Boolean = hasNextRow
+        override def get(): InternalRow = nextRow
         override def close(): Unit = reader.close()
       }
       new PartitionReaderWithPartitionValues(fileReader, readDataSchema,
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroScan.scala b/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroScan.scala
index fe7315c739296..144e9ad129feb 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroScan.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroScan.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql.catalyst.expressions.Expression
 import org.apache.spark.sql.connector.read.PartitionReaderFactory
 import org.apache.spark.sql.execution.datasources.PartitioningAwareFileIndex
 import org.apache.spark.sql.execution.datasources.v2.FileScan
+import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 import org.apache.spark.util.SerializableConfiguration
@@ -37,6 +38,7 @@ case class AvroScan(
     readDataSchema: StructType,
     readPartitionSchema: StructType,
     options: CaseInsensitiveStringMap,
+    pushedFilters: Array[Filter],
     partitionFilters: Seq[Expression] = Seq.empty,
     dataFilters: Seq[Expression] = Seq.empty) extends FileScan {
   override def isSplitable(path: Path): Boolean = true
@@ -50,8 +52,14 @@ case class AvroScan(
     val parsedOptions = new AvroOptions(caseSensitiveMap, hadoopConf)
     // The partition values are already truncated in `FileScan.partitions`.
     // We should use `readPartitionSchema` as the partition schema here.
-    AvroPartitionReaderFactory(sparkSession.sessionState.conf, broadcastedConf,
-      dataSchema, readDataSchema, readPartitionSchema, parsedOptions)
+    AvroPartitionReaderFactory(
+      sparkSession.sessionState.conf,
+      broadcastedConf,
+      dataSchema,
+      readDataSchema,
+      readPartitionSchema,
+      parsedOptions,
+      pushedFilters)
   }
 
   override def withFilters(
@@ -59,10 +67,18 @@ case class AvroScan(
     this.copy(partitionFilters = partitionFilters, dataFilters = dataFilters)
 
   override def equals(obj: Any): Boolean = obj match {
-    case a: AvroScan => super.equals(a) && dataSchema == a.dataSchema && options == a.options
-
+    case a: AvroScan => super.equals(a) && dataSchema == a.dataSchema && options == a.options &&
+      equivalentFilters(pushedFilters, a.pushedFilters)
     case _ => false
   }
 
   override def hashCode(): Int = super.hashCode()
+
+  override def description(): String = {
+    super.description() + ", PushedFilters: " + pushedFilters.mkString("[", ", ", "]")
+  }
+
+  override def getMetaData(): Map[String, String] = {
+    super.getMetaData() ++ Map("PushedFilters" -> seqToString(pushedFilters))
+  }
 }
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroScanBuilder.scala b/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroScanBuilder.scala
index e36c71ef4b1f7..9420608bb22ce 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroScanBuilder.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroScanBuilder.scala
@@ -17,9 +17,11 @@
 package org.apache.spark.sql.v2.avro
 
 import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.connector.read.Scan
+import org.apache.spark.sql.catalyst.StructFilters
+import org.apache.spark.sql.connector.read.{Scan, SupportsPushDownFilters}
 import org.apache.spark.sql.execution.datasources.PartitioningAwareFileIndex
 import org.apache.spark.sql.execution.datasources.v2.FileScanBuilder
+import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
@@ -29,8 +31,27 @@ class AvroScanBuilder (
     schema: StructType,
     dataSchema: StructType,
     options: CaseInsensitiveStringMap)
-  extends FileScanBuilder(sparkSession, fileIndex, dataSchema) {
+  extends FileScanBuilder(sparkSession, fileIndex, dataSchema) with SupportsPushDownFilters {
+
   override def build(): Scan = {
-    AvroScan(sparkSession, fileIndex, dataSchema, readDataSchema(), readPartitionSchema(), options)
+    AvroScan(
+      sparkSession,
+      fileIndex,
+      dataSchema,
+      readDataSchema(),
+      readPartitionSchema(),
+      options,
+      pushedFilters())
   }
+
+  private var _pushedFilters: Array[Filter] = Array.empty
+
+  override def pushFilters(filters: Array[Filter]): Array[Filter] = {
+    if (sparkSession.sessionState.conf.avroFilterPushDown) {
+      _pushedFilters = StructFilters.pushedFilters(filters, dataSchema)
+    }
+    filters
+  }
+
+  override def pushedFilters(): Array[Filter] = _pushedFilters
 }
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroTable.scala b/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroTable.scala
index 2096f1a08a0d1..f19d856252b50 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroTable.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroTable.scala
@@ -22,7 +22,7 @@ import org.apache.hadoop.fs.FileStatus
 
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.avro.AvroUtils
-import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder}
+import org.apache.spark.sql.connector.write.{LogicalWriteInfo, Write, WriteBuilder}
 import org.apache.spark.sql.execution.datasources.FileFormat
 import org.apache.spark.sql.execution.datasources.v2.FileTable
 import org.apache.spark.sql.types.{DataType, StructType}
@@ -43,7 +43,9 @@ case class AvroTable(
     AvroUtils.inferSchema(sparkSession, options.asScala.toMap, files)
 
   override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder =
-    new AvroWriteBuilder(paths, formatName, supportsDataType, info)
+    new WriteBuilder {
+      override def build(): Write = AvroWrite(paths, formatName, supportsDataType, info)
+    }
 
   override def supportsDataType(dataType: DataType): Boolean = AvroUtils.supportsDataType(dataType)
 
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroWrite.scala b/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroWrite.scala
new file mode 100644
index 0000000000000..3a91fd0c73d1a
--- /dev/null
+++ b/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroWrite.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.v2.avro
+
+import org.apache.hadoop.mapreduce.Job
+
+import org.apache.spark.sql.avro.AvroUtils
+import org.apache.spark.sql.connector.write.LogicalWriteInfo
+import org.apache.spark.sql.execution.datasources.OutputWriterFactory
+import org.apache.spark.sql.execution.datasources.v2.FileWrite
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types._
+
+case class AvroWrite(
+    paths: Seq[String],
+    formatName: String,
+    supportsDataType: DataType => Boolean,
+    info: LogicalWriteInfo) extends FileWrite {
+  override def prepareWrite(
+      sqlConf: SQLConf,
+      job: Job,
+      options: Map[String, String],
+      dataSchema: StructType): OutputWriterFactory = {
+    AvroUtils.prepareWrite(sqlConf, job, options, dataSchema)
+  }
+}
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroWriteBuilder.scala b/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroWriteBuilder.scala
deleted file mode 100644
index c4defb9f065e3..0000000000000
--- a/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroWriteBuilder.scala
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.v2.avro
-
-import org.apache.hadoop.mapreduce.Job
-
-import org.apache.spark.sql.avro.AvroUtils
-import org.apache.spark.sql.connector.write.LogicalWriteInfo
-import org.apache.spark.sql.execution.datasources.OutputWriterFactory
-import org.apache.spark.sql.execution.datasources.v2.FileWriteBuilder
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types._
-
-class AvroWriteBuilder(
-    paths: Seq[String],
-    formatName: String,
-    supportsDataType: DataType => Boolean,
-    info: LogicalWriteInfo)
-  extends FileWriteBuilder(paths, formatName, supportsDataType, info) {
-  override def prepareWrite(
-      sqlConf: SQLConf,
-      job: Job,
-      options: Map[String, String],
-      dataSchema: StructType): OutputWriterFactory = {
-    AvroUtils.prepareWrite(sqlConf, job, options, dataSchema)
-  }
-}
diff --git a/external/avro/src/test/resources/before_1582_date_v2_4.avro b/external/avro/src/test/resources/before_1582_date_v2_4.avro
deleted file mode 100644
index 96aa7cbf176a5..0000000000000
Binary files a/external/avro/src/test/resources/before_1582_date_v2_4.avro and /dev/null differ
diff --git a/external/avro/src/test/resources/before_1582_date_v2_4_5.avro b/external/avro/src/test/resources/before_1582_date_v2_4_5.avro
new file mode 100644
index 0000000000000..5c15601f7ee4b
Binary files /dev/null and b/external/avro/src/test/resources/before_1582_date_v2_4_5.avro differ
diff --git a/external/avro/src/test/resources/before_1582_date_v2_4_6.avro b/external/avro/src/test/resources/before_1582_date_v2_4_6.avro
new file mode 100644
index 0000000000000..212ea1d5efa5c
Binary files /dev/null and b/external/avro/src/test/resources/before_1582_date_v2_4_6.avro differ
diff --git a/external/avro/src/test/resources/before_1582_timestamp_micros_v2_4_5.avro b/external/avro/src/test/resources/before_1582_timestamp_micros_v2_4_5.avro
new file mode 100644
index 0000000000000..c3445e3999bc1
Binary files /dev/null and b/external/avro/src/test/resources/before_1582_timestamp_micros_v2_4_5.avro differ
diff --git a/external/avro/src/test/resources/before_1582_timestamp_micros_v2_4_6.avro b/external/avro/src/test/resources/before_1582_timestamp_micros_v2_4_6.avro
new file mode 100644
index 0000000000000..96008d2378b1f
Binary files /dev/null and b/external/avro/src/test/resources/before_1582_timestamp_micros_v2_4_6.avro differ
diff --git a/external/avro/src/test/resources/before_1582_timestamp_millis_v2_4_5.avro b/external/avro/src/test/resources/before_1582_timestamp_millis_v2_4_5.avro
new file mode 100644
index 0000000000000..be12a0782073c
Binary files /dev/null and b/external/avro/src/test/resources/before_1582_timestamp_millis_v2_4_5.avro differ
diff --git a/external/avro/src/test/resources/before_1582_timestamp_millis_v2_4_6.avro b/external/avro/src/test/resources/before_1582_timestamp_millis_v2_4_6.avro
new file mode 100644
index 0000000000000..262f5dd6e77a4
Binary files /dev/null and b/external/avro/src/test/resources/before_1582_timestamp_millis_v2_4_6.avro differ
diff --git a/external/avro/src/test/resources/before_1582_ts_micros_v2_4.avro b/external/avro/src/test/resources/before_1582_ts_micros_v2_4.avro
deleted file mode 100644
index efe5e71a58813..0000000000000
Binary files a/external/avro/src/test/resources/before_1582_ts_micros_v2_4.avro and /dev/null differ
diff --git a/external/avro/src/test/resources/before_1582_ts_millis_v2_4.avro b/external/avro/src/test/resources/before_1582_ts_millis_v2_4.avro
deleted file mode 100644
index dbaec814eb954..0000000000000
Binary files a/external/avro/src/test/resources/before_1582_ts_millis_v2_4.avro and /dev/null differ
diff --git a/external/avro/src/test/resources/test_sub.avsc b/external/avro/src/test/resources/test_sub.avsc
new file mode 100644
index 0000000000000..311b9933c7ceb
--- /dev/null
+++ b/external/avro/src/test/resources/test_sub.avsc
@@ -0,0 +1,9 @@
+{
+  "type" : "record",
+  "name" : "test_schema",
+  "fields" : [{
+    "name" : "string",
+    "type" : "string",
+    "doc"  : "Meaningless string of characters"
+  }]
+}
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroCatalystDataConversionSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroCatalystDataConversionSuite.scala
index c8a1f670bda9e..ebf0a074381df 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroCatalystDataConversionSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroCatalystDataConversionSuite.scala
@@ -26,11 +26,14 @@ import org.apache.avro.message.{BinaryMessageDecoder, BinaryMessageEncoder}
 
 import org.apache.spark.{SparkException, SparkFunSuite}
 import org.apache.spark.sql.{RandomDataGenerator, Row}
-import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
+import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow, NoopFilters, OrderedFilters, StructFilters}
 import org.apache.spark.sql.catalyst.expressions.{ExpressionEvalHelper, GenericInternalRow, Literal}
 import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, GenericArrayData, MapData}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.sources.{EqualTo, Not}
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.UTF8String
 
 class AvroCatalystDataConversionSuite extends SparkFunSuite
   with SharedSparkSession
@@ -269,7 +272,26 @@ class AvroCatalystDataConversionSuite extends SparkFunSuite
     val message = intercept[IncompatibleSchemaException] {
       CatalystDataToAvro(Literal("SPADES"), Some("\"long\"")).eval()
     }.getMessage
-    assert(message ==  "Cannot convert Catalyst type StringType to Avro type \"long\".")
+    assert(message === "Cannot convert SQL type STRING to Avro type \"long\".")
+  }
+
+  private def checkDeserialization(
+      schema: Schema,
+      data: GenericData.Record,
+      expected: Option[Any],
+      filters: StructFilters = new NoopFilters): Unit = {
+    val dataType = SchemaConverters.toSqlType(schema).dataType
+    val deserializer = new AvroDeserializer(
+      schema,
+      dataType,
+      SQLConf.LegacyBehaviorPolicy.CORRECTED,
+      filters)
+    val deserialized = deserializer.deserialize(data)
+    expected match {
+      case None => assert(deserialized == None)
+      case Some(d) =>
+        assert(checkResult(d, deserialized.get, dataType, exprNullable = false))
+    }
   }
 
   test("avro array can be generic java collection") {
@@ -287,30 +309,53 @@ class AvroCatalystDataConversionSuite extends SparkFunSuite
         |}
       """.stripMargin
     val avroSchema = new Schema.Parser().parse(jsonFormatSchema)
-    val dataType = SchemaConverters.toSqlType(avroSchema).dataType
-    val deserializer = new AvroDeserializer(avroSchema, dataType)
-
-    def checkDeserialization(data: GenericData.Record, expected: Any): Unit = {
-      assert(checkResult(
-        expected,
-        deserializer.deserialize(data),
-        dataType, exprNullable = false
-      ))
-    }
 
     def validateDeserialization(array: java.util.Collection[Integer]): Unit = {
       val data = new GenericRecordBuilder(avroSchema)
         .set("array", array)
         .build()
       val expected = InternalRow(new GenericArrayData(new util.ArrayList[Any](array)))
-      checkDeserialization(data, expected)
+      checkDeserialization(avroSchema, data, Some(expected))
 
       val reEncoded = new BinaryMessageDecoder[GenericData.Record](new GenericData(), avroSchema)
         .decode(new BinaryMessageEncoder(new GenericData(), avroSchema).encode(data))
-      checkDeserialization(reEncoded, expected)
+      checkDeserialization(avroSchema, reEncoded, Some(expected))
     }
 
     validateDeserialization(Collections.emptySet())
     validateDeserialization(util.Arrays.asList(1, null, 3))
   }
+
+  test("SPARK-32346: filter pushdown to Avro deserializer") {
+    val schema =
+      """
+        |{
+        |  "type" : "record",
+        |  "name" : "test_schema",
+        |  "fields" : [
+        |    {"name": "Age", "type": "int"},
+        |    {"name": "Name", "type": "string"}
+        |  ]
+        |}
+        """.stripMargin
+    val avroSchema = new Schema.Parser().parse(schema)
+    val sqlSchema = new StructType().add("Age", "int").add("Name", "string")
+    val data = new GenericRecordBuilder(avroSchema)
+      .set("Age", 39)
+      .set("Name", "Maxim")
+      .build()
+    val expectedRow = Some(InternalRow(39, UTF8String.fromString("Maxim")))
+
+    checkDeserialization(avroSchema, data, expectedRow)
+    checkDeserialization(
+      avroSchema,
+      data,
+      expectedRow,
+      new OrderedFilters(Seq(EqualTo("Age", 39)), sqlSchema))
+    checkDeserialization(
+      avroSchema,
+      data,
+      None,
+      new OrderedFilters(Seq(Not(EqualTo("Age", 39))), sqlSchema))
+  }
 }
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroFunctionsSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroFunctionsSuite.scala
index 7f14efe15ad55..c9e0d4344691a 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroFunctionsSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroFunctionsSuite.scala
@@ -201,4 +201,41 @@ class AvroFunctionsSuite extends QueryTest with SharedSparkSession {
           Map("avroSchema" -> evolvedAvroSchema).asJava)),
       expected)
   }
+
+  test("roundtrip in to_avro and from_avro - struct with nullable Avro schema") {
+    val df = spark.range(10).select(struct('id, 'id.cast("string").as("str")).as("struct"))
+    val avroTypeStruct = s"""
+      |{
+      |  "type": "record",
+      |  "name": "struct",
+      |  "fields": [
+      |    {"name": "id", "type": "long"},
+      |    {"name": "str", "type": ["null", "string"]}
+      |  ]
+      |}
+    """.stripMargin
+    val avroStructDF = df.select(functions.to_avro('struct, avroTypeStruct).as("avro"))
+    checkAnswer(avroStructDF.select(
+      functions.from_avro('avro, avroTypeStruct)), df)
+  }
+
+  test("to_avro with unsupported nullable Avro schema") {
+    val df = spark.range(10).select(struct('id, 'id.cast("string").as("str")).as("struct"))
+    for (unsupportedAvroType <- Seq("""["null", "int", "long"]""", """["int", "long"]""")) {
+      val avroTypeStruct = s"""
+        |{
+        |  "type": "record",
+        |  "name": "struct",
+        |  "fields": [
+        |    {"name": "id", "type": $unsupportedAvroType},
+        |    {"name": "str", "type": ["null", "string"]}
+        |  ]
+        |}
+      """.stripMargin
+      val message = intercept[SparkException] {
+        df.select(functions.to_avro('struct, avroTypeStruct).as("avro")).show()
+      }.getCause.getMessage
+      assert(message.contains("Only UNION of a null type and a non-null type is supported"))
+    }
+  }
 }
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala
index 82569653c1f23..12ebddf72b03d 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala
@@ -246,7 +246,7 @@ abstract class AvroLogicalTypeSuite extends QueryTest with SharedSparkSession {
     dataFileWriter.create(schema, new File(avroFile))
     val logicalType = LogicalTypes.decimal(precision, scale)
 
-    decimalInputData.map { x =>
+    decimalInputData.foreach { x =>
       val avroRec = new GenericData.Record(schema)
       val decimal = new java.math.BigDecimal(x).setScale(scale)
       val bytes =
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroScanSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroScanSuite.scala
new file mode 100644
index 0000000000000..98a7190ba984e
--- /dev/null
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroScanSuite.scala
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.avro
+
+import org.apache.spark.sql.FileScanSuiteBase
+import org.apache.spark.sql.v2.avro.AvroScan
+
+class AvroScanSuite extends FileScanSuiteBase {
+  val scanBuilders = Seq[(String, ScanBuilder, Seq[String])](
+    ("AvroScan",
+      (s, fi, ds, rds, rps, f, o, pf, df) => AvroScan(s, fi, ds, rds, rps, o, f, pf, df),
+      Seq.empty))
+
+  run(scanBuilders)
+}
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSerdeSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSerdeSuite.scala
new file mode 100644
index 0000000000000..5be760f0056ba
--- /dev/null
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSerdeSuite.scala
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.avro
+
+import org.apache.avro.{Schema, SchemaBuilder}
+import org.apache.avro.generic.GenericRecordBuilder
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
+import org.apache.spark.sql.types.{IntegerType, StructType}
+
+/**
+ * Tests for [[AvroSerializer]] and [[AvroDeserializer]], complementing those in [[AvroSuite]]
+ * with a more specific focus on those classes.
+ */
+class AvroSerdeSuite extends SparkFunSuite {
+  import AvroSerdeSuite._
+
+  private val defaultRebaseMode = LegacyBehaviorPolicy.CORRECTED.toString
+
+  test("Test basic conversion") {
+    val avro = createNestedAvroSchemaWithFields("foo", _.optionalInt("bar"))
+    val record = new GenericRecordBuilder(avro)
+        .set("foo", new GenericRecordBuilder(avro.getField("foo").schema()).set("bar", 42).build())
+        .build()
+    val serializer = new AvroSerializer(CATALYST_STRUCT, avro, false)
+    val deserializer = new AvroDeserializer(avro, CATALYST_STRUCT, defaultRebaseMode)
+    assert(serializer.serialize(deserializer.deserialize(record).get) === record)
+  }
+
+  test("Fail to convert with field type mismatch") {
+    val avro = createAvroSchemaWithTopLevelFields(_.requiredInt("foo"))
+
+    assertFailedConversionMessage(avro, deserialize = true,
+      "Cannot convert Avro field 'foo' to SQL field 'foo' because schema is incompatible " +
+          s"""(avroType = "int", sqlType = ${CATALYST_STRUCT.head.dataType.sql})""")
+
+    assertFailedConversionMessage(avro, deserialize = false,
+      s"Cannot convert SQL field 'foo' to Avro field 'foo' because schema is incompatible " +
+          s"""(sqlType = ${CATALYST_STRUCT.head.dataType.sql}, avroType = "int")""")
+  }
+
+  test("Fail to convert with nested field type mismatch") {
+    val avro = createNestedAvroSchemaWithFields("foo", _.optionalFloat("bar"))
+
+    assertFailedConversionMessage(avro, deserialize = true,
+      "Cannot convert Avro field 'foo.bar' to SQL field 'foo.bar' because schema is " +
+          """incompatible (avroType = "float", sqlType = INT)""")
+
+    assertFailedConversionMessage(avro, deserialize = false,
+      "Cannot convert SQL field 'foo.bar' to Avro field 'foo.bar' because " +
+        """schema is incompatible (sqlType = INT, avroType = "float")""")
+  }
+
+  test("Fail to convert with nested field name mismatch") {
+    val avro = createNestedAvroSchemaWithFields("foo", _.optionalInt("NOTbar"))
+    val nonnullCatalyst = new StructType()
+        .add("foo", new StructType().add("bar", IntegerType, nullable = false))
+
+    // deserialize should have no issues when 'bar' is nullable but fail when it is nonnull
+    new AvroDeserializer(avro, CATALYST_STRUCT, defaultRebaseMode)
+    assertFailedConversionMessage(avro, deserialize = true,
+      "Cannot find non-nullable field 'foo.bar' in Avro schema.",
+      nonnullCatalyst)
+
+    // serialize fails whether or not 'bar' is nullable
+    val expectMsg = "Cannot find field 'foo.bar' in Avro schema at field 'foo'"
+    assertFailedConversionMessage(avro, deserialize = false, expectMsg)
+    assertFailedConversionMessage(avro, deserialize = false, expectMsg, nonnullCatalyst)
+  }
+
+  test("Fail to convert with deeply nested field type mismatch") {
+    val avro = SchemaBuilder.builder().record("toptest").fields()
+        .name("top").`type`(createNestedAvroSchemaWithFields("foo", _.optionalFloat("bar")))
+        .noDefault().endRecord()
+    val catalyst = new StructType().add("top", CATALYST_STRUCT)
+
+    assertFailedConversionMessage(avro, deserialize = true,
+      "Cannot convert Avro field 'top.foo.bar' to SQL field 'top.foo.bar' because schema " +
+        """is incompatible (avroType = "float", sqlType = INT)""",
+      catalyst)
+
+    assertFailedConversionMessage(avro, deserialize = false,
+      "Cannot convert SQL field 'top.foo.bar' to Avro field 'top.foo.bar' because schema is " +
+          """incompatible (sqlType = INT, avroType = "float")""",
+      catalyst)
+  }
+
+  test("Fail to convert for serialization with field count mismatch") {
+    val tooManyFields = createAvroSchemaWithTopLevelFields(_.optionalInt("foo").optionalLong("bar"))
+    assertFailedConversionMessage(tooManyFields, deserialize = false,
+      "Avro top-level record schema length (2) " +
+        "doesn't match SQL top-level record schema length (1)")
+
+    val tooFewFields = createAvroSchemaWithTopLevelFields(f => f)
+    assertFailedConversionMessage(tooFewFields, deserialize = false,
+      "Avro top-level record schema length (0) " +
+        "doesn't match SQL top-level record schema length (1)")
+  }
+
+  /**
+   * Attempt to convert `catalystSchema` to `avroSchema` (or vice-versa if `deserialize` is true),
+   * assert that it fails, and assert that the _cause_ of the thrown exception has a message
+   * matching `expectedCauseMessage`.
+   */
+  private def assertFailedConversionMessage(avroSchema: Schema,
+      deserialize: Boolean,
+      expectedCauseMessage: String,
+      catalystSchema: StructType = CATALYST_STRUCT): Unit = {
+    val e = intercept[IncompatibleSchemaException] {
+      if (deserialize) {
+        new AvroDeserializer(avroSchema, catalystSchema, defaultRebaseMode)
+      } else {
+        new AvroSerializer(catalystSchema, avroSchema, false)
+      }
+    }
+    val expectMsg = if (deserialize) {
+      s"Cannot convert Avro type $avroSchema to SQL type ${catalystSchema.sql}."
+    } else {
+      s"Cannot convert SQL type ${catalystSchema.sql} to Avro type $avroSchema."
+    }
+    assert(e.getMessage === expectMsg)
+    assert(e.getCause.getMessage === expectedCauseMessage)
+  }
+}
+
+
+object AvroSerdeSuite {
+
+  private val CATALYST_STRUCT =
+    new StructType().add("foo", new StructType().add("bar", IntegerType))
+
+  /**
+   * Convenience method to create a top-level Avro schema with a single nested record
+   * (at field name `nestedRecordFieldName`) which has fields as defined by those set
+   * on the field assembler using `f`.
+   */
+  private def createNestedAvroSchemaWithFields(
+      nestedRecordFieldName: String,
+      f: SchemaBuilder.FieldAssembler[Schema] => SchemaBuilder.FieldAssembler[Schema]): Schema = {
+    createAvroSchemaWithTopLevelFields(_.name(nestedRecordFieldName)
+        .`type`(f(SchemaBuilder.builder().record("test").fields()).endRecord())
+        .noDefault())
+  }
+
+  /**
+   * Convenience method to create a top-level Avro schema with fields as defined by those set
+   * on the field assembler using `f`.
+   */
+  private def createAvroSchemaWithTopLevelFields(
+      f: SchemaBuilder.FieldAssembler[Schema] => SchemaBuilder.FieldAssembler[Schema]): Schema = {
+    f(SchemaBuilder.builder().record("top").fields()).endRecord()
+  }
+}
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
index a5c1fb15add5c..6c04417289292 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
@@ -18,40 +18,54 @@
 package org.apache.spark.sql.avro
 
 import java.io._
-import java.net.URL
-import java.nio.file.{Files, Paths}
+import java.net.{URI, URL}
+import java.nio.file.{Files, Paths, StandardCopyOption}
 import java.sql.{Date, Timestamp}
 import java.util.{Locale, UUID}
 
 import scala.collection.JavaConverters._
 
-import org.apache.avro.Schema
+import org.apache.avro.{AvroTypeException, Schema}
 import org.apache.avro.Schema.{Field, Type}
 import org.apache.avro.Schema.Type._
 import org.apache.avro.file.{DataFileReader, DataFileWriter}
 import org.apache.avro.generic.{GenericData, GenericDatumReader, GenericDatumWriter, GenericRecord}
 import org.apache.avro.generic.GenericData.{EnumSymbol, Fixed}
+import org.apache.avro.mapred.FsInput
 import org.apache.commons.io.FileUtils
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.Path
 
 import org.apache.spark.{SPARK_VERSION_SHORT, SparkConf, SparkException, SparkUpgradeException}
+import org.apache.spark.TestUtils.assertExceptionMsg
 import org.apache.spark.sql._
 import org.apache.spark.sql.TestingUDT.IntervalData
+import org.apache.spark.sql.catalyst.{InternalRow, NoopFilters}
 import org.apache.spark.sql.catalyst.expressions.AttributeReference
 import org.apache.spark.sql.catalyst.plans.logical.Filter
-import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, UTC}
-import org.apache.spark.sql.execution.SparkPlan
-import org.apache.spark.sql.execution.datasources.{DataSource, FilePartition}
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, LA, UTC}
+import org.apache.spark.sql.execution.{FormattedMode, SparkPlan}
+import org.apache.spark.sql.execution.datasources.{CommonFileDataSourceSuite, DataSource, FilePartition}
 import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
+import org.apache.spark.sql.functions.col
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy._
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.v2.avro.AvroScan
 import org.apache.spark.util.Utils
 
-abstract class AvroSuite extends QueryTest with SharedSparkSession {
+abstract class AvroSuite
+  extends QueryTest
+  with SharedSparkSession
+  with CommonFileDataSourceSuite
+  with NestedDataSourceSuiteBase {
+
   import testImplicits._
 
+  override protected def dataSourceFormat = "avro"
+  override val nestedDataSources = Seq("avro")
   val episodesAvro = testFile("episodes.avro")
   val testAvro = testFile("test.avro")
 
@@ -379,11 +393,11 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
         }.getMessage
         assert(message.contains("No Avro files found."))
 
-        val srcFile = new File("src/test/resources/episodes.avro")
-        val destFile = new File(dir, "episodes.avro")
-        FileUtils.copyFile(srcFile, destFile)
+        Files.copy(
+          Paths.get(new URL(episodesAvro).toURI),
+          Paths.get(dir.getCanonicalPath, "episodes.avro"))
 
-        val result = spark.read.format("avro").load(srcFile.getAbsolutePath).collect()
+        val result = spark.read.format("avro").load(episodesAvro).collect()
         checkAnswer(spark.read.format("avro").load(dir.getAbsolutePath), result)
       }
     }
@@ -465,6 +479,7 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
       val xzDir = s"$dir/xz"
       val deflateDir = s"$dir/deflate"
       val snappyDir = s"$dir/snappy"
+      val zstandardDir = s"$dir/zstandard"
 
       val df = spark.read.format("avro").load(testAvro)
       spark.conf.set(SQLConf.AVRO_COMPRESSION_CODEC.key, "uncompressed")
@@ -478,17 +493,21 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
       df.write.format("avro").save(deflateDir)
       spark.conf.set(SQLConf.AVRO_COMPRESSION_CODEC.key, "snappy")
       df.write.format("avro").save(snappyDir)
+      spark.conf.set(SQLConf.AVRO_COMPRESSION_CODEC.key, "zstandard")
+      df.write.format("avro").save(zstandardDir)
 
       val uncompressSize = FileUtils.sizeOfDirectory(new File(uncompressDir))
       val bzip2Size = FileUtils.sizeOfDirectory(new File(bzip2Dir))
       val xzSize = FileUtils.sizeOfDirectory(new File(xzDir))
       val deflateSize = FileUtils.sizeOfDirectory(new File(deflateDir))
       val snappySize = FileUtils.sizeOfDirectory(new File(snappyDir))
+      val zstandardSize = FileUtils.sizeOfDirectory(new File(zstandardDir))
 
       assert(uncompressSize > deflateSize)
       assert(snappySize > deflateSize)
       assert(snappySize > bzip2Size)
       assert(bzip2Size > xzSize)
+      assert(uncompressSize > zstandardSize)
     }
   }
 
@@ -541,7 +560,8 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
 
     val array_of_boolean =
       spark.read.format("avro").load(testAvro).select("array_of_boolean").collect()
-    assert(array_of_boolean.map(_(0).asInstanceOf[Seq[Boolean]].size).toSet == Set(3, 1, 0))
+    assert(array_of_boolean.map(_(0).asInstanceOf[scala.collection.Seq[Boolean]].size).toSet ==
+      Set(3, 1, 0))
 
     val bytes = spark.read.format("avro").load(testAvro).select("bytes").collect()
     assert(bytes.map(_(0).asInstanceOf[Array[Byte]].length).toSet == Set(3, 1, 0))
@@ -589,6 +609,14 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
     }
   }
 
+  test("SPARK-34229: Avro should read decimal values with the file schema") {
+    withTempPath { path =>
+      sql("SELECT 3.14 a").write.format("avro").save(path.toString)
+      val data = spark.read.schema("a DECIMAL(4, 3)").format("avro").load(path.toString).collect()
+      assert(data.map(_ (0)).contains(new java.math.BigDecimal("3.140")))
+    }
+  }
+
   test("converting some specific sparkSQL types to avro") {
     withTempPath { tempDir =>
       val testSchema = StructType(Seq(
@@ -702,6 +730,52 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
     assert(result.sameElements(expected))
   }
 
+  test("SPARK-34416: support user provided avro schema url") {
+    val avroSchemaUrl = testFile("test_sub.avsc")
+    val result = spark.read.option("avroSchemaUrl", avroSchemaUrl)
+      .format("avro")
+      .load(testAvro)
+      .collect()
+    val expected = spark.read.format("avro").load(testAvro).select("string").collect()
+    assert(result.sameElements(expected))
+  }
+
+  test("SPARK-34416: support user provided both avro schema and avro schema url") {
+    val avroSchemaUrl = testFile("test_sub.avsc")
+    val avroSchema =
+      """
+        |{
+        |  "type" : "record",
+        |  "name" : "test_schema",
+        |  "fields" : [{
+        |    "name" : "union_int_long_null",
+        |    "type" : ["int", "long", "null"]
+        |  }]
+        |}
+      """.stripMargin
+
+    val result = spark.read
+      .option("avroSchema", avroSchema)
+      .option("avroSchemaUrl", avroSchemaUrl)
+      .format("avro")
+      .load(testAvro)
+      .collect()
+    val expected = spark.read.format("avro").load(testAvro).select("union_int_long_null").collect()
+    assert(result.sameElements(expected))
+  }
+
+  test("SPARK-34416: support user provided wrong avro schema url") {
+    val e = intercept[FileNotFoundException] {
+      spark.read
+        .option("avroSchemaUrl", "not_exists.avsc")
+        .format("avro")
+        .load(testAvro)
+        .collect()
+    }
+
+    assertExceptionMsg[FileNotFoundException](e, "File not_exists.avsc does not exist")
+  }
+
   test("support user provided avro schema with defaults for missing fields") {
     val avroSchema =
       """
@@ -752,15 +826,15 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
       checkAvroSchemaEquals(avroSchema, getAvroSchemaStringFromFiles(tempSaveDir))
 
       // Writing df containing data not in the enum will throw an exception
-      val message = intercept[SparkException] {
+      val e = intercept[SparkException] {
         spark.createDataFrame(spark.sparkContext.parallelize(Seq(
           Row("SPADES"), Row("NOT-IN-ENUM"), Row("HEARTS"), Row("DIAMONDS"))),
           StructType(Seq(StructField("Suit", StringType, true))))
           .write.format("avro").option("avroSchema", avroSchema)
           .save(s"$tempDir/${UUID.randomUUID()}")
-      }.getCause.getMessage
-      assert(message.contains("org.apache.spark.sql.avro.IncompatibleSchemaException: " +
-        "Cannot write \"NOT-IN-ENUM\" since it's not defined in enum"))
+      }
+      assertExceptionMsg[IncompatibleSchemaException](e,
+        """"NOT-IN-ENUM" cannot be written since it's not defined in enum""")
     }
   }
 
@@ -798,22 +872,22 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
 
       // Writing df containing nulls without using avro union type will
       // throw an exception as avro uses union type to handle null.
-      val message1 = intercept[SparkException] {
+      val e1 = intercept[SparkException] {
         dfWithNull.write.format("avro")
           .option("avroSchema", avroSchema).save(s"$tempDir/${UUID.randomUUID()}")
-      }.getCause.getMessage
-      assert(message1.contains("org.apache.avro.AvroTypeException: Not an enum: null"))
+      }
+      assertExceptionMsg[AvroTypeException](e1, "Not an enum: null")
 
       // Writing df containing data not in the enum will throw an exception
-      val message2 = intercept[SparkException] {
+      val e2 = intercept[SparkException] {
         spark.createDataFrame(spark.sparkContext.parallelize(Seq(
           Row("SPADES"), Row("NOT-IN-ENUM"), Row("HEARTS"), Row("DIAMONDS"))),
           StructType(Seq(StructField("Suit", StringType, false))))
           .write.format("avro").option("avroSchema", avroSchema)
           .save(s"$tempDir/${UUID.randomUUID()}")
-      }.getCause.getMessage
-      assert(message2.contains("org.apache.spark.sql.avro.IncompatibleSchemaException: " +
-        "Cannot write \"NOT-IN-ENUM\" since it's not defined in enum"))
+      }
+      assertExceptionMsg[IncompatibleSchemaException](e2,
+        """"NOT-IN-ENUM" cannot be written since it's not defined in enum""")
     }
   }
 
@@ -846,26 +920,26 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
       checkAvroSchemaEquals(avroSchema, getAvroSchemaStringFromFiles(tempSaveDir))
 
       // Writing df containing binary data that doesn't fit FIXED size will throw an exception
-      val message1 = intercept[SparkException] {
+      val e1 = intercept[SparkException] {
         spark.createDataFrame(spark.sparkContext.parallelize(Seq(
           Row(Array(192, 168, 1).map(_.toByte)))),
           StructType(Seq(StructField("fixed2", BinaryType, true))))
           .write.format("avro").option("avroSchema", avroSchema)
           .save(s"$tempDir/${UUID.randomUUID()}")
-      }.getCause.getMessage
-      assert(message1.contains("org.apache.spark.sql.avro.IncompatibleSchemaException: " +
-        "Cannot write 3 bytes of binary data into FIXED Type with size of 2 bytes"))
+      }
+      assertExceptionMsg[IncompatibleSchemaException](e1,
+        "3 bytes of binary data cannot be written into FIXED type with size of 2 bytes")
 
       // Writing df containing binary data that doesn't fit FIXED size will throw an exception
-      val message2 = intercept[SparkException] {
+      val e2 = intercept[SparkException] {
         spark.createDataFrame(spark.sparkContext.parallelize(Seq(
           Row(Array(192).map(_.toByte)))),
           StructType(Seq(StructField("fixed2", BinaryType, true))))
           .write.format("avro").option("avroSchema", avroSchema)
           .save(s"$tempDir/${UUID.randomUUID()}")
-      }.getCause.getMessage
-      assert(message2.contains("org.apache.spark.sql.avro.IncompatibleSchemaException: " +
-        "Cannot write 1 byte of binary data into FIXED Type with size of 2 bytes"))
+      }
+      assertExceptionMsg[IncompatibleSchemaException](e2,
+        "1 byte of binary data cannot be written into FIXED type with size of 2 bytes")
     }
   }
 
@@ -898,26 +972,26 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
       checkAvroSchemaEquals(avroSchema, getAvroSchemaStringFromFiles(tempSaveDir))
 
       // Writing df containing binary data that doesn't fit FIXED size will throw an exception
-      val message1 = intercept[SparkException] {
+      val e1 = intercept[SparkException] {
         spark.createDataFrame(spark.sparkContext.parallelize(Seq(
           Row(Array(192, 168, 1).map(_.toByte)))),
           StructType(Seq(StructField("fixed2", BinaryType, false))))
           .write.format("avro").option("avroSchema", avroSchema)
           .save(s"$tempDir/${UUID.randomUUID()}")
-      }.getCause.getMessage
-      assert(message1.contains("org.apache.spark.sql.avro.IncompatibleSchemaException: " +
-        "Cannot write 3 bytes of binary data into FIXED Type with size of 2 bytes"))
+      }
+      assertExceptionMsg[IncompatibleSchemaException](e1,
+        "3 bytes of binary data cannot be written into FIXED type with size of 2 bytes")
 
       // Writing df containing binary data that doesn't fit FIXED size will throw an exception
-      val message2 = intercept[SparkException] {
+      val e2 = intercept[SparkException] {
         spark.createDataFrame(spark.sparkContext.parallelize(Seq(
           Row(Array(192).map(_.toByte)))),
           StructType(Seq(StructField("fixed2", BinaryType, false))))
           .write.format("avro").option("avroSchema", avroSchema)
           .save(s"$tempDir/${UUID.randomUUID()}")
-      }.getCause.getMessage
-      assert(message2.contains("org.apache.spark.sql.avro.IncompatibleSchemaException: " +
-        "Cannot write 1 byte of binary data into FIXED Type with size of 2 bytes"))
+      }
+      assertExceptionMsg[IncompatibleSchemaException](e2,
+        "1 byte of binary data cannot be written into FIXED type with size of 2 bytes")
     }
   }
 
@@ -1002,7 +1076,64 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
           .save(s"$tempDir/${UUID.randomUUID()}")
       }.getCause.getMessage
       assert(message.contains("Caused by: java.lang.NullPointerException: " +
-        "in test_schema in string null of string in field Name"))
+        "null of string in string in field Name of test_schema in test_schema"))
+    }
+  }
+
+  test("support user provided nullable avro schema " +
+    "for non-nullable catalyst schema without any null record") {
+    val catalystSchema =
+      StructType(Seq(
+        StructField("Age", IntegerType, nullable = false),
+        StructField("Name", StringType, nullable = false)))
+
+    val avroSchema = """
+      |{
+      |  "type" : "record",
+      |  "name" : "test_schema",
+      |  "fields" : [
+      |    {"name": "Age", "type": ["null", "int"]},
+      |    {"name": "Name", "type": ["null", "string"]}
+      |  ]
+      |}
+    """.stripMargin
+
+    val df = spark.createDataFrame(
+      spark.sparkContext.parallelize(Seq(Row(2, "Aurora"))), catalystSchema)
+
+    withTempPath { tempDir =>
+      df.write.format("avro").option("avroSchema", avroSchema).save(tempDir.getPath)
+      checkAvroSchemaEquals(avroSchema, getAvroSchemaStringFromFiles(tempDir.getPath))
+    }
+  }
+
+  test("unsupported nullable avro type") {
+    val catalystSchema =
+      StructType(Seq(
+        StructField("Age", IntegerType, nullable = false),
+        StructField("Name", StringType, nullable = false)))
+
+    for (unsupportedAvroType <- Seq("""["null", "int", "long"]""", """["int", "long"]""")) {
+      val avroSchema = s"""
+        |{
+        |  "type" : "record",
+        |  "name" : "test_schema",
+        |  "fields" : [
+        |    {"name": "Age", "type": $unsupportedAvroType},
+        |    {"name": "Name", "type": ["null", "string"]}
+        |  ]
+        |}
+      """.stripMargin
+
+      val df = spark.createDataFrame(
+        spark.sparkContext.parallelize(Seq(Row(2, "Aurora"))), catalystSchema)
+
+      withTempPath { tempDir =>
+        val message = intercept[SparkException] {
+          df.write.format("avro").option("avroSchema", avroSchema).save(tempDir.getPath)
+        }.getCause.getMessage
+        assert(message.contains("Only UNION of a null type and a non-null type is supported"))
+      }
     }
   }
 
@@ -1052,7 +1183,7 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
       val message = intercept[org.apache.spark.sql.avro.IncompatibleSchemaException] {
         f()
       }.getMessage
-      assert(message.contains("Cannot convert Catalyst type"))
+      assert(message.contains("Cannot convert SQL type"))
     }
 
     def resolveNullable(schema: Schema, nullable: Boolean): Schema = {
@@ -1191,6 +1322,96 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
     }
   }
 
+  test("SPARK-34133: Reading user provided schema respects case sensitivity for field matching") {
+    val wrongCaseSchema = new StructType()
+        .add("STRING", StringType, nullable = false)
+        .add("UNION_STRING_NULL", StringType, nullable = true)
+    val withSchema = spark.read
+        .schema(wrongCaseSchema)
+        .format("avro").load(testAvro).collect()
+
+    val withOutSchema = spark.read.format("avro").load(testAvro)
+        .select("STRING", "UNION_STRING_NULL")
+        .collect()
+    assert(withSchema.sameElements(withOutSchema))
+
+    withSQLConf((SQLConf.CASE_SENSITIVE.key, "true")) {
+      val  out = spark.read.format("avro").schema(wrongCaseSchema).load(testAvro).collect()
+      assert(out.forall(_.isNullAt(0)))
+      assert(out.forall(_.isNullAt(1)))
+    }
+  }
+
+  test("SPARK-34133: Writing user provided schema respects case sensitivity for field matching") {
+    withTempDir { tempDir =>
+      val avroSchema =
+        """
+          |{
+          |  "type" : "record",
+          |  "name" : "test_schema",
+          |  "fields" : [
+          |    {"name": "foo", "type": "int"},
+          |    {"name": "BAR", "type": "int"}
+          |  ]
+          |}
+      """.stripMargin
+      val df = Seq((1, 3), (2, 4)).toDF("FOO", "bar")
+
+      val savePath = s"$tempDir/save"
+      df.write.option("avroSchema", avroSchema).format("avro").save(savePath)
+
+      val loaded = spark.read.format("avro").load(savePath)
+      assert(loaded.schema === new StructType().add("foo", IntegerType).add("BAR", IntegerType))
+      assert(loaded.collect().map(_.getInt(0)).toSet === Set(1, 2))
+      assert(loaded.collect().map(_.getInt(1)).toSet === Set(3, 4))
+
+      withSQLConf((SQLConf.CASE_SENSITIVE.key, "true")) {
+        val e = intercept[SparkException] {
+          df.write.option("avroSchema", avroSchema).format("avro").save(s"$tempDir/save2")
+        }
+        assertExceptionMsg[IncompatibleSchemaException](e,
+          "Cannot find field 'FOO' in Avro schema at top-level record")
+      }
+    }
+  }
+
+  test("SPARK-34133: Writing user provided schema with multiple matching Avro fields fails") {
+    withTempDir { tempDir =>
+      val avroSchema =
+        """
+          |{
+          |  "type" : "record",
+          |  "name" : "test_schema",
+          |  "fields" : [
+          |    {"name": "foo", "type": "int"},
+          |    {"name": "FOO", "type": "string"}
+          |  ]
+          |}
+      """.stripMargin
+
+      val errorMsg = "Searching for 'foo' in Avro schema at top-level record gave 2 matches. " +
+          "Candidates: [foo, FOO]"
+      assertExceptionMsg(intercept[SparkException] {
+        val fooBarDf = Seq((1, "3"), (2, "4")).toDF("foo", "bar")
+        fooBarDf.write.option("avroSchema", avroSchema).format("avro").save(s"$tempDir/save-fail")
+      }, errorMsg)
+
+      val savePath = s"$tempDir/save"
+      withSQLConf((SQLConf.CASE_SENSITIVE.key, "true")) {
+        val fooFooDf = Seq((1, "3"), (2, "4")).toDF("foo", "FOO")
+        fooFooDf.write.option("avroSchema", avroSchema).format("avro").save(savePath)
+
+        val loadedDf = spark.read.format("avro").schema(fooFooDf.schema).load(savePath)
+        assert(loadedDf.collect().toSet === fooFooDf.collect().toSet)
+      }
+
+      assertExceptionMsg(intercept[SparkException] {
+        val fooSchema = new StructType().add("foo", IntegerType)
+        spark.read.format("avro").schema(fooSchema).load(savePath).collect()
+      }, errorMsg)
+    }
+  }
+
   test("read avro with user defined schema: read partial columns") {
     val partialColumns = StructType(Seq(
       StructField("string", StringType, false),
@@ -1529,82 +1750,175 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
     }
   }
 
+  // It generates input files for the test below:
+  // "SPARK-31183: compatibility with Spark 2.4 in reading dates/timestamps"
+  ignore("SPARK-31855: generate test files for checking compatibility with Spark 2.4") {
+    val resourceDir = "external/avro/src/test/resources"
+    val version = "2_4_6"
+    def save(
+      in: Seq[String],
+      t: String,
+      dstFile: String,
+      options: Map[String, String] = Map.empty): Unit = {
+      withTempDir { dir =>
+        in.toDF("dt")
+          .select($"dt".cast(t))
+          .repartition(1)
+          .write
+          .mode("overwrite")
+          .options(options)
+          .format("avro")
+          .save(dir.getCanonicalPath)
+        Files.copy(
+          dir.listFiles().filter(_.getName.endsWith(".avro")).head.toPath,
+          Paths.get(resourceDir, dstFile),
+          StandardCopyOption.REPLACE_EXISTING)
+      }
+    }
+    withDefaultTimeZone(LA) {
+      withSQLConf(
+        SQLConf.SESSION_LOCAL_TIMEZONE.key -> LA.getId) {
+        save(
+          Seq("1001-01-01"),
+          "date",
+          s"before_1582_date_v$version.avro")
+        save(
+          Seq("1001-01-01 01:02:03.123"),
+          "timestamp",
+          s"before_1582_timestamp_millis_v$version.avro",
+          // scalastyle:off line.size.limit
+          Map("avroSchema" ->
+            s"""
+               |  {
+               |    "namespace": "logical",
+               |    "type": "record",
+               |    "name": "test",
+               |    "fields": [
+               |      {"name": "dt", "type": ["null", {"type": "long","logicalType": "timestamp-millis"}], "default": null}
+               |    ]
+               |  }
+               |""".stripMargin))
+        // scalastyle:on line.size.limit
+        save(
+          Seq("1001-01-01 01:02:03.123456"),
+          "timestamp",
+          s"before_1582_timestamp_micros_v$version.avro")
+      }
+    }
+  }
+
+  private def runInMode(
+      modes: Seq[LegacyBehaviorPolicy.Value])(f: Map[String, String] => Unit): Unit = {
+    modes.foreach { mode =>
+      withSQLConf(SQLConf.AVRO_REBASE_MODE_IN_READ.key -> mode.toString) {
+        f(Map.empty)
+      }
+    }
+    withSQLConf(SQLConf.AVRO_REBASE_MODE_IN_READ.key -> EXCEPTION.toString) {
+      modes.foreach { mode =>
+        f(Map(AvroOptions.DATETIME_REBASE_MODE -> mode.toString))
+      }
+    }
+  }
+
   test("SPARK-31183: compatibility with Spark 2.4 in reading dates/timestamps") {
     // test reading the existing 2.4 files and new 3.0 files (with rebase on/off) together.
-    def checkReadMixedFiles(fileName: String, dt: String, dataStr: String): Unit = {
+    def checkReadMixedFiles(
+        fileName: String,
+        dt: String,
+        dataStr: String,
+        checkDefaultLegacyRead: String => Unit): Unit = {
       withTempPaths(2) { paths =>
         paths.foreach(_.delete())
         val path2_4 = getResourceAvroFilePath(fileName)
         val path3_0 = paths(0).getCanonicalPath
         val path3_0_rebase = paths(1).getCanonicalPath
         if (dt == "date") {
-          val df = Seq(dataStr).toDF("str").select($"str".cast("date").as("date"))
+          val df = Seq(dataStr).toDF("str").select($"str".cast("date").as("dt"))
 
           // By default we should fail to write ancient datetime values.
-          var e = intercept[SparkException](df.write.format("avro").save(path3_0))
+          val e = intercept[SparkException](df.write.format("avro").save(path3_0))
           assert(e.getCause.getCause.getCause.isInstanceOf[SparkUpgradeException])
-          // By default we should fail to read ancient datetime values.
-          e = intercept[SparkException](spark.read.format("avro").load(path2_4).collect())
-          assert(e.getCause.isInstanceOf[SparkUpgradeException])
+          checkDefaultLegacyRead(path2_4)
 
-          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString) {
+          withSQLConf(SQLConf.AVRO_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString) {
             df.write.format("avro").mode("overwrite").save(path3_0)
           }
-          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
+          withSQLConf(SQLConf.AVRO_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
             df.write.format("avro").save(path3_0_rebase)
           }
 
           // For Avro files written by Spark 3.0, we know the writer info and don't need the config
           // to guide the rebase behavior.
-          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ.key -> LEGACY.toString) {
+          runInMode(Seq(LEGACY)) { options =>
             checkAnswer(
-              spark.read.format("avro").load(path2_4, path3_0, path3_0_rebase),
+              spark.read.options(options).format("avro").load(path2_4, path3_0, path3_0_rebase),
               1.to(3).map(_ => Row(java.sql.Date.valueOf(dataStr))))
           }
         } else {
-          val df = Seq(dataStr).toDF("str").select($"str".cast("timestamp").as("ts"))
+          val df = Seq(dataStr).toDF("str").select($"str".cast("timestamp").as("dt"))
           val avroSchema =
             s"""
               |{
               |  "type" : "record",
               |  "name" : "test_schema",
               |  "fields" : [
-              |    {"name": "ts", "type": {"type": "long", "logicalType": "$dt"}}
+              |    {"name": "dt", "type": {"type": "long", "logicalType": "$dt"}}
               |  ]
               |}""".stripMargin
 
           // By default we should fail to write ancient datetime values.
-          var e = intercept[SparkException] {
+          val e = intercept[SparkException] {
             df.write.format("avro").option("avroSchema", avroSchema).save(path3_0)
           }
           assert(e.getCause.getCause.getCause.isInstanceOf[SparkUpgradeException])
-          // By default we should fail to read ancient datetime values.
-          e = intercept[SparkException](spark.read.format("avro").load(path2_4).collect())
-          assert(e.getCause.isInstanceOf[SparkUpgradeException])
+          checkDefaultLegacyRead(path2_4)
 
-          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString) {
+          withSQLConf(SQLConf.AVRO_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString) {
             df.write.format("avro").option("avroSchema", avroSchema).mode("overwrite").save(path3_0)
           }
-          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
+          withSQLConf(SQLConf.AVRO_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
             df.write.format("avro").option("avroSchema", avroSchema).save(path3_0_rebase)
           }
 
           // For Avro files written by Spark 3.0, we know the writer info and don't need the config
           // to guide the rebase behavior.
-          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ.key -> LEGACY.toString) {
+          runInMode(Seq(LEGACY)) { options =>
             checkAnswer(
-              spark.read.format("avro").load(path2_4, path3_0, path3_0_rebase),
+              spark.read.options(options).format("avro").load(path2_4, path3_0, path3_0_rebase),
               1.to(3).map(_ => Row(java.sql.Timestamp.valueOf(dataStr))))
           }
         }
       }
     }
 
-    checkReadMixedFiles("before_1582_date_v2_4.avro", "date", "1001-01-01")
-    checkReadMixedFiles(
-      "before_1582_ts_micros_v2_4.avro", "timestamp-micros", "1001-01-01 01:02:03.123456")
-    checkReadMixedFiles(
-      "before_1582_ts_millis_v2_4.avro", "timestamp-millis", "1001-01-01 01:02:03.124")
+    def failInRead(path: String): Unit = {
+      val e = intercept[SparkException](spark.read.format("avro").load(path).collect())
+      assert(e.getCause.isInstanceOf[SparkUpgradeException])
+    }
+    def successInRead(path: String): Unit = spark.read.format("avro").load(path).collect()
+    Seq(
+      // By default we should fail to read ancient datetime values when parquet files don't
+      // contain Spark version.
+      "2_4_5" -> failInRead _,
+      "2_4_6" -> successInRead _
+    ).foreach { case (version, checkDefaultRead) =>
+      checkReadMixedFiles(
+        s"before_1582_date_v$version.avro",
+        "date",
+        "1001-01-01",
+        checkDefaultRead)
+      checkReadMixedFiles(
+        s"before_1582_timestamp_micros_v$version.avro",
+        "timestamp-micros",
+        "1001-01-01 01:02:03.123456",
+        checkDefaultRead)
+      checkReadMixedFiles(
+        s"before_1582_timestamp_millis_v$version.avro",
+        "timestamp-millis",
+        "1001-01-01 01:02:03.123",
+        checkDefaultRead)
+    }
   }
 
   test("SPARK-31183: rebasing microseconds timestamps in write") {
@@ -1612,7 +1926,7 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
     val nonRebased = "1001-01-07 01:09:05.123456"
     withTempPath { dir =>
       val path = dir.getAbsolutePath
-      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
+      withSQLConf(SQLConf.AVRO_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
         Seq(tsStr).toDF("tsS")
           .select($"tsS".cast("timestamp").as("ts"))
           .write.format("avro")
@@ -1621,10 +1935,10 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
 
       // The file metadata indicates if it needs rebase or not, so we can always get the correct
       // result regardless of the "rebase mode" config.
-      Seq(LEGACY, CORRECTED, EXCEPTION).foreach { mode =>
-        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ.key -> mode.toString) {
-          checkAnswer(spark.read.format("avro").load(path), Row(Timestamp.valueOf(tsStr)))
-        }
+      runInMode(Seq(LEGACY, CORRECTED, EXCEPTION)) { options =>
+        checkAnswer(
+          spark.read.options(options).format("avro").load(path),
+          Row(Timestamp.valueOf(tsStr)))
       }
 
       // Force to not rebase to prove the written datetime values are rebased and we will get
@@ -1653,7 +1967,7 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
         |}""".stripMargin
       withTempPath { dir =>
         val path = dir.getAbsolutePath
-        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
+        withSQLConf(SQLConf.AVRO_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
           Seq(tsStr).toDF("tsS")
             .select($"tsS".cast("timestamp").as("ts"))
             .write
@@ -1664,12 +1978,10 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
 
         // The file metadata indicates if it needs rebase or not, so we can always get the correct
         // result regardless of the "rebase mode" config.
-        Seq(LEGACY, CORRECTED, EXCEPTION).foreach { mode =>
-          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ.key -> mode.toString) {
-            checkAnswer(
-              spark.read.schema("ts timestamp").format("avro").load(path),
-              Row(Timestamp.valueOf(rebased)))
-          }
+        runInMode(Seq(LEGACY, CORRECTED, EXCEPTION)) { options =>
+          checkAnswer(
+            spark.read.options(options).schema("ts timestamp").format("avro").load(path),
+            Row(Timestamp.valueOf(rebased)))
         }
 
         // Force to not rebase to prove the written datetime values are rebased and we will get
@@ -1686,7 +1998,7 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
   test("SPARK-31183: rebasing dates in write") {
     withTempPath { dir =>
       val path = dir.getAbsolutePath
-      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
+      withSQLConf(SQLConf.AVRO_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
         Seq("1001-01-01").toDF("dateS")
           .select($"dateS".cast("date").as("date"))
           .write.format("avro")
@@ -1695,10 +2007,10 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
 
       // The file metadata indicates if it needs rebase or not, so we can always get the correct
       // result regardless of the "rebase mode" config.
-      Seq(LEGACY, CORRECTED, EXCEPTION).foreach { mode =>
-        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ.key -> mode.toString) {
-          checkAnswer(spark.read.format("avro").load(path), Row(Date.valueOf("1001-01-01")))
-        }
+      runInMode(Seq(LEGACY, CORRECTED, EXCEPTION)) { options =>
+        checkAnswer(
+          spark.read.options(options).format("avro").load(path),
+          Row(Date.valueOf("1001-01-01")))
       }
 
       // Force to not rebase to prove the written datetime values are rebased and we will get
@@ -1709,15 +2021,61 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
     }
   }
 
+  private def checkMetaData(path: java.io.File, key: String, expectedValue: String): Unit = {
+    val avroFiles = path.listFiles()
+      .filter(f => f.isFile && !f.getName.startsWith(".") && !f.getName.startsWith("_"))
+    assert(avroFiles.length === 1)
+    val reader = DataFileReader.openReader(avroFiles(0), new GenericDatumReader[GenericRecord]())
+    val value = reader.asInstanceOf[DataFileReader[_]].getMetaString(key)
+    assert(value === expectedValue)
+  }
+
   test("SPARK-31327: Write Spark version into Avro file metadata") {
     withTempPath { path =>
       spark.range(1).repartition(1).write.format("avro").save(path.getCanonicalPath)
-      val avroFiles = path.listFiles()
-        .filter(f => f.isFile && !f.getName.startsWith(".") && !f.getName.startsWith("_"))
-      assert(avroFiles.length === 1)
-      val reader = DataFileReader.openReader(avroFiles(0), new GenericDatumReader[GenericRecord]())
-      val version = reader.asInstanceOf[DataFileReader[_]].getMetaString(SPARK_VERSION_METADATA_KEY)
-      assert(version === SPARK_VERSION_SHORT)
+      checkMetaData(path, SPARK_VERSION_METADATA_KEY, SPARK_VERSION_SHORT)
+    }
+  }
+
+  test("SPARK-33163: write the metadata key 'org.apache.spark.legacyDateTime'") {
+    def saveTs(dir: java.io.File): Unit = {
+      Seq(Timestamp.valueOf("2020-10-15 01:02:03")).toDF()
+        .repartition(1)
+        .write
+        .format("avro")
+        .save(dir.getAbsolutePath)
+    }
+    withSQLConf(SQLConf.AVRO_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
+      withTempPath { dir =>
+        saveTs(dir)
+        checkMetaData(dir, SPARK_LEGACY_DATETIME, "")
+      }
+    }
+    Seq(CORRECTED, EXCEPTION).foreach { mode =>
+      withSQLConf(SQLConf.AVRO_REBASE_MODE_IN_WRITE.key -> mode.toString) {
+        withTempPath { dir =>
+          saveTs(dir)
+          checkMetaData(dir, SPARK_LEGACY_DATETIME, null)
+        }
+      }
+    }
+  }
+
+  test("SPARK-33314: RowReader doesn't over-consume when hasNextRow called twice") {
+    withTempPath { dir =>
+      Seq((1), (2), (3))
+        .toDF("index")
+        .write
+        .format("avro")
+        .save(dir.getCanonicalPath)
+      val df = spark
+        .read
+        .format("avro")
+        .load(dir.getCanonicalPath)
+        .orderBy("index")
+
+      checkAnswer(df,
+        Seq(Row(1), Row(2), Row(3)))
     }
   }
 }
@@ -1729,7 +2087,7 @@ class AvroV1Suite extends AvroSuite {
       .set(SQLConf.USE_V1_SOURCE_LIST, "avro")
 }
 
-class AvroV2Suite extends AvroSuite {
+class AvroV2Suite extends AvroSuite with ExplainSuiteHelper {
   import testImplicits._
 
   override protected def sparkConf: SparkConf =
@@ -1828,4 +2186,122 @@ class AvroV2Suite extends AvroSuite {
       assert(scan1.sameResult(scan2))
     }
   }
+
+  test("explain formatted on an avro data source v2") {
+    withTempDir { dir =>
+      val basePath = dir.getCanonicalPath + "/avro"
+      val expected_plan_fragment =
+        s"""
+           |\\(1\\) BatchScan
+           |Output \\[2\\]: \\[value#xL, id#x\\]
+           |DataFilters: \\[isnotnull\\(value#xL\\), \\(value#xL > 2\\)\\]
+           |Format: avro
+           |Location: InMemoryFileIndex\\([0-9]+ paths\\)\\[.*\\]
+           |PartitionFilters: \\[isnotnull\\(id#x\\), \\(id#x > 1\\)\\]
+           |PushedFilters: \\[IsNotNull\\(value\\), GreaterThan\\(value,2\\)\\]
+           |ReadSchema: struct\\<value:bigint\\>
+           |""".stripMargin.trim
+      spark.range(10)
+        .select(col("id"), col("id").as("value"))
+        .write.option("header", true)
+        .partitionBy("id")
+        .format("avro")
+        .save(basePath)
+      val df = spark
+        .read
+        .format("avro")
+        .load(basePath).where($"id" > 1 && $"value" > 2)
+      val normalizedOutput = getNormalizedExplain(df, FormattedMode)
+      assert(expected_plan_fragment.r.findAllMatchIn(normalizedOutput).length == 1,
+        normalizedOutput)
+    }
+  }
+
+  test("SPARK-32346: filters pushdown to Avro datasource v2") {
+    Seq(true, false).foreach { filtersPushdown =>
+      withSQLConf(SQLConf.AVRO_FILTER_PUSHDOWN_ENABLED.key -> filtersPushdown.toString) {
+        withTempPath { dir =>
+          Seq(("a", 1, 2), ("b", 1, 2), ("c", 2, 1))
+            .toDF("value", "p1", "p2")
+            .write
+            .format("avro")
+            .save(dir.getCanonicalPath)
+          val df = spark
+            .read
+            .format("avro")
+            .load(dir.getCanonicalPath)
+            .where("value = 'a'")
+
+          val fileScan = df.queryExecution.executedPlan collectFirst {
+            case BatchScanExec(_, f: AvroScan) => f
+          }
+          assert(fileScan.nonEmpty)
+          if (filtersPushdown) {
+            assert(fileScan.get.pushedFilters.nonEmpty)
+          } else {
+            assert(fileScan.get.pushedFilters.isEmpty)
+          }
+          checkAnswer(df, Row("a", 1, 2))
+        }
+      }
+    }
+  }
+}
+
+class AvroRowReaderSuite
+  extends QueryTest
+  with SharedSparkSession {
+
+  import testImplicits._
+
+  override protected def sparkConf: SparkConf =
+    super
+      .sparkConf
+      .set(SQLConf.USE_V1_SOURCE_LIST, "") // need this for BatchScanExec
+
+  test("SPARK-33314: hasNextRow and nextRow properly handle consecutive calls") {
+    withTempPath { dir =>
+      Seq((1), (2), (3))
+        .toDF("value")
+        .coalesce(1)
+        .write
+        .format("avro")
+        .save(dir.getCanonicalPath)
+
+      val df = spark.read.format("avro").load(dir.getCanonicalPath)
+      val fileScan = df.queryExecution.executedPlan collectFirst {
+        case BatchScanExec(_, f: AvroScan) => f
+      }
+      val filePath = fileScan.get.fileIndex.inputFiles(0)
+      val fileSize = new File(new URI(filePath)).length
+      val in = new FsInput(new Path(new URI(filePath)), new Configuration())
+      val reader = DataFileReader.openReader(in, new GenericDatumReader[GenericRecord]())
+
+      val it = new Iterator[InternalRow] with AvroUtils.RowReader {
+        override val fileReader = reader
+        override val deserializer = new AvroDeserializer(
+          reader.getSchema,
+          StructType(new StructField("value", IntegerType, true) :: Nil),
+          CORRECTED,
+          new NoopFilters)
+        override val stopPosition = fileSize
+
+        override def hasNext: Boolean = hasNextRow
+
+        override def next: InternalRow = nextRow
+      }
+      assert(it.hasNext == true)
+      assert(it.next.getInt(0) == 1)
+      // test no intervening next
+      assert(it.hasNext == true)
+      assert(it.hasNext == true)
+      // test no intervening hasNext
+      assert(it.next.getInt(0) == 2)
+      assert(it.next.getInt(0) == 3)
+      assert(it.hasNext == false)
+      assertThrows[NoSuchElementException] {
+        it.next
+      }
+    }
+  }
 }
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/execution/benchmark/AvroReadBenchmark.scala b/external/avro/src/test/scala/org/apache/spark/sql/execution/benchmark/AvroReadBenchmark.scala
index dc9606f405191..01a78dc7e58a7 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/execution/benchmark/AvroReadBenchmark.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/execution/benchmark/AvroReadBenchmark.scala
@@ -17,11 +17,14 @@
 package org.apache.spark.sql.execution.benchmark
 
 import java.io.File
+import java.time.Instant
 
 import scala.util.Random
 
 import org.apache.spark.benchmark.Benchmark
-import org.apache.spark.sql.DataFrame
+import org.apache.spark.sql.{Column, DataFrame}
+import org.apache.spark.sql.functions.lit
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 /**
@@ -29,13 +32,15 @@ import org.apache.spark.sql.types._
  * {{{
  *   To run this benchmark:
  *   1. without sbt: bin/spark-submit --class <this class>
- *        --jars <catalyst test jar>,<core test jar>,<spark-avro jar> <avro test jar>
+ *        --jars <catalyst test jar>,<core test jar>,<sql test jar>,<spark-avro jar> <avro test jar>
  *   2. build/sbt "avro/test:runMain <this class>"
  *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "avro/test:runMain <this class>"
  *      Results will be written to "benchmarks/AvroReadBenchmark-results.txt".
  * }}}
  */
 object AvroReadBenchmark extends SqlBasedBenchmark {
+  import spark.implicits._
+
   def withTempTable(tableNames: String*)(f: => Unit): Unit = {
     try f finally tableNames.foreach(spark.catalog.dropTempView)
   }
@@ -186,6 +191,60 @@ object AvroReadBenchmark extends SqlBasedBenchmark {
     }
   }
 
+  private def filtersPushdownBenchmark(rowsNum: Int, numIters: Int): Unit = {
+    val benchmark = new Benchmark("Filters pushdown", rowsNum, output = output)
+    val colsNum = 100
+    val fields = Seq.tabulate(colsNum)(i => StructField(s"col$i", TimestampType))
+    val schema = StructType(StructField("key", LongType) +: fields)
+    def columns(): Seq[Column] = {
+      val ts = Seq.tabulate(colsNum) { i =>
+        lit(Instant.ofEpochSecond(-30610224000L + i * 123456)).as(s"col$i")
+      }
+      ($"id" % 1000).as("key") +: ts
+    }
+    withTempPath { path =>
+      // Write and read timestamp in the LEGACY mode to make timestamp conversions more expensive
+      withSQLConf(SQLConf.AVRO_REBASE_MODE_IN_WRITE.key -> "LEGACY") {
+        spark.range(rowsNum).select(columns(): _*)
+          .write
+          .format("avro")
+          .save(path.getAbsolutePath)
+      }
+      def readback = {
+        spark.read
+          .schema(schema)
+          .format("avro")
+          .load(path.getAbsolutePath)
+      }
+
+      benchmark.addCase("w/o filters", numIters) { _ =>
+        withSQLConf(SQLConf.AVRO_REBASE_MODE_IN_READ.key -> "LEGACY") {
+          readback.noop()
+        }
+      }
+
+      def withFilter(configEnabled: Boolean): Unit = {
+        withSQLConf(
+          SQLConf.AVRO_REBASE_MODE_IN_READ.key -> "LEGACY",
+          SQLConf.AVRO_FILTER_PUSHDOWN_ENABLED.key -> configEnabled.toString()) {
+          readback.filter($"key" === 0).noop()
+        }
+      }
+
+      benchmark.addCase("pushdown disabled", numIters) { _ =>
+        withSQLConf(SQLConf.AVRO_REBASE_MODE_IN_READ.key -> "LEGACY") {
+          withFilter(configEnabled = false)
+        }
+      }
+
+      benchmark.addCase("w/ filters", numIters) { _ =>
+        withFilter(configEnabled = true)
+      }
+
+      benchmark.run()
+    }
+  }
+
   override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
     runBenchmark("SQL Single Numeric Column Scan") {
       Seq(ByteType, ShortType, IntegerType, LongType, FloatType, DoubleType).foreach { dataType =>
@@ -211,5 +270,8 @@ object AvroReadBenchmark extends SqlBasedBenchmark {
       columnsBenchmark(1024 * 1024 * 1, 200)
       columnsBenchmark(1024 * 1024 * 1, 300)
     }
+    // Benchmark pushdown filters that refer to top-level columns.
+    // TODO (SPARK-32328): Add benchmarks for filters with nested column attributes.
+    filtersPushdownBenchmark(rowsNum = 1000 * 1000, numIters = 3)
   }
 }
diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml
index 7f9e92f585168..808f48f18e1ff 100644
--- a/external/docker-integration-tests/pom.xml
+++ b/external/docker-integration-tests/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
@@ -130,15 +130,9 @@
       <artifactId>postgresql</artifactId>
       <scope>test</scope>
     </dependency>
-    <!-- Oracle ojdbc jar, used for oracle  integration suite for docker testing.
-     See https://github.com/apache/spark/pull/11306 for background on why we need
-     to use a an ojdbc jar for the testcase. The maven dependency here is commented
-     because currently the maven repository does not contain the ojdbc jar mentioned.
-     Once the jar is available in maven, this could be uncommented. -->
     <dependency>
-      <groupId>com.oracle</groupId>
-      <artifactId>ojdbc6</artifactId>
-      <version>11.2.0.1.0</version>
+      <groupId>com.oracle.database.jdbc</groupId>
+      <artifactId>ojdbc8</artifactId>
       <scope>test</scope>
     </dependency>
 
@@ -166,7 +160,6 @@
     <dependency>
       <groupId>com.microsoft.sqlserver</groupId>
       <artifactId>mssql-jdbc</artifactId>
-      <version>7.2.1.jre8</version>
       <scope>test</scope>
     </dependency>
   </dependencies>
diff --git a/external/docker-integration-tests/src/test/resources/mariadb_docker_entrypoint.sh b/external/docker-integration-tests/src/test/resources/mariadb_docker_entrypoint.sh
index 00885a3b62327..ab7d967a927d0 100755
--- a/external/docker-integration-tests/src/test/resources/mariadb_docker_entrypoint.sh
+++ b/external/docker-integration-tests/src/test/resources/mariadb_docker_entrypoint.sh
@@ -18,7 +18,9 @@
 
 dpkg-divert --add /bin/systemctl && ln -sT /bin/true /bin/systemctl
 apt update
-apt install -y mariadb-plugin-gssapi-server
+GSSAPI_PLUGIN=mariadb-plugin-gssapi-server=$(dpkg -s mariadb-server | sed -n "s/^Version: \(.*\)/\1/p")
+echo "Installing $GSSAPI_PLUGIN"
+apt install -y "$GSSAPI_PLUGIN"
 echo "gssapi_keytab_path=/docker-entrypoint-initdb.d/mariadb.keytab" >> /etc/mysql/mariadb.conf.d/auth_gssapi.cnf
 echo "gssapi_principal_name=mariadb/__IP_ADDRESS_REPLACE_ME__@EXAMPLE.COM" >> /etc/mysql/mariadb.conf.d/auth_gssapi.cnf
 docker-entrypoint.sh mysqld
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala
index 02a7ff8f16073..62bba797413a1 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala
@@ -21,14 +21,25 @@ import java.math.BigDecimal
 import java.sql.{Connection, Date, Timestamp}
 import java.util.Properties
 
+import org.scalatest.time.SpanSugar._
+
 import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
 import org.apache.spark.sql.types.{BooleanType, ByteType, ShortType, StructType}
 import org.apache.spark.tags.DockerTest
 
+/**
+ * To run this test suite for a specific version (e.g., ibmcom/db2:11.5.4.0):
+ * {{{
+ *   DB2_DOCKER_IMAGE_NAME=ibmcom/db2:11.5.4.0
+ *     ./build/sbt -Pdocker-integration-tests
+ *     "testOnly org.apache.spark.sql.jdbc.DB2IntegrationSuite"
+ * }}}
+ */
 @DockerTest
 class DB2IntegrationSuite extends DockerJDBCIntegrationSuite {
   override val db = new DatabaseOnDocker {
-    override val imageName = "ibmcom/db2:11.5.0.0a"
+    override val imageName = sys.env.getOrElse("DB2_DOCKER_IMAGE_NAME", "ibmcom/db2:11.5.4.0")
     override val env = Map(
       "DB2INST1_PASSWORD" -> "rootpass",
       "LICENSE" -> "accept",
@@ -43,6 +54,8 @@ class DB2IntegrationSuite extends DockerJDBCIntegrationSuite {
       s"jdbc:db2://$ip:$port/foo:user=db2inst1;password=rootpass;retrieveMessagesFromServerOnGetMessage=true;" //scalastyle:ignore
   }
 
+  override val connectionTimeout = timeout(3.minutes)
+
   override def dataPreparation(conn: Connection): Unit = {
     conn.prepareStatement("CREATE TABLE tbl (x INTEGER, y VARCHAR(8))").executeUpdate()
     conn.prepareStatement("INSERT INTO tbl VALUES (42,'fred')").executeUpdate()
@@ -107,17 +120,19 @@ class DB2IntegrationSuite extends DockerJDBCIntegrationSuite {
   }
 
   test("Date types") {
-    val df = sqlContext.read.jdbc(jdbcUrl, "dates", new Properties)
-    val rows = df.collect()
-    assert(rows.length == 1)
-    val types = rows(0).toSeq.map(x => x.getClass.toString)
-    assert(types.length == 3)
-    assert(types(0).equals("class java.sql.Date"))
-    assert(types(1).equals("class java.sql.Timestamp"))
-    assert(types(2).equals("class java.sql.Timestamp"))
-    assert(rows(0).getAs[Date](0).equals(Date.valueOf("1991-11-09")))
-    assert(rows(0).getAs[Timestamp](1).equals(Timestamp.valueOf("1970-01-01 13:31:24")))
-    assert(rows(0).getAs[Timestamp](2).equals(Timestamp.valueOf("2009-02-13 23:31:30")))
+    withDefaultTimeZone(UTC) {
+      val df = sqlContext.read.jdbc(jdbcUrl, "dates", new Properties)
+      val rows = df.collect()
+      assert(rows.length == 1)
+      val types = rows(0).toSeq.map(x => x.getClass.toString)
+      assert(types.length == 3)
+      assert(types(0).equals("class java.sql.Date"))
+      assert(types(1).equals("class java.sql.Timestamp"))
+      assert(types(2).equals("class java.sql.Timestamp"))
+      assert(rows(0).getAs[Date](0).equals(Date.valueOf("1991-11-09")))
+      assert(rows(0).getAs[Timestamp](1).equals(Timestamp.valueOf("1970-01-01 13:31:24")))
+      assert(rows(0).getAs[Timestamp](2).equals(Timestamp.valueOf("2009-02-13 23:31:30")))
+    }
   }
 
   test("String types") {
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2KrbIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2KrbIntegrationSuite.scala
index fc88985cf2ec7..f79809f355c0f 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2KrbIntegrationSuite.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2KrbIntegrationSuite.scala
@@ -24,18 +24,26 @@ import javax.security.auth.login.Configuration
 import com.spotify.docker.client.messages.{ContainerConfig, HostConfig}
 import org.apache.hadoop.security.{SecurityUtil, UserGroupInformation}
 import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS
+import org.scalatest.time.SpanSugar._
 
 import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions
 import org.apache.spark.sql.execution.datasources.jdbc.connection.{DB2ConnectionProvider, SecureConnectionProvider}
 import org.apache.spark.tags.DockerTest
 
+/**
+ * To run this test suite for a specific version (e.g., ibmcom/db2:11.5.4.0):
+ * {{{
+ *   DB2_DOCKER_IMAGE_NAME=ibmcom/db2:11.5.4.0
+ *     ./build/sbt -Pdocker-integration-tests "testOnly *DB2KrbIntegrationSuite"
+ * }}}
+ */
 @DockerTest
 class DB2KrbIntegrationSuite extends DockerKrbJDBCIntegrationSuite {
   override protected val userName = s"db2/$dockerIp"
   override protected val keytabFileName = "db2.keytab"
 
   override val db = new DatabaseOnDocker {
-    override val imageName = "ibmcom/db2:11.5.0.0a"
+    override val imageName = sys.env.getOrElse("DB2_DOCKER_IMAGE_NAME", "ibmcom/db2:11.5.4.0")
     override val env = Map(
       "DB2INST1_PASSWORD" -> "rootpass",
       "LICENSE" -> "accept",
@@ -54,7 +62,7 @@ class DB2KrbIntegrationSuite extends DockerKrbJDBCIntegrationSuite {
         JDBCOptions.JDBC_KEYTAB -> keytabFileName,
         JDBCOptions.JDBC_PRINCIPAL -> principal
       ))
-      new DB2ConnectionProvider(null, options).getAdditionalProperties()
+      new DB2ConnectionProvider().getAdditionalProperties(options)
     }
 
     override def beforeContainerStart(
@@ -69,9 +77,11 @@ class DB2KrbIntegrationSuite extends DockerKrbJDBCIntegrationSuite {
     }
   }
 
+  override val connectionTimeout = timeout(3.minutes)
+
   override protected def setAuthentication(keytabFile: String, principal: String): Unit = {
     val config = new SecureConnectionProvider.JDBCConfiguration(
-      Configuration.getConfiguration, "JaasClient", keytabFile, principal)
+      Configuration.getConfiguration, "JaasClient", keytabFile, principal, true)
     Configuration.setConfiguration(config)
   }
 
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
index d15b366bfc9b0..d6270313cabea 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
@@ -25,6 +25,7 @@ import scala.collection.JavaConverters._
 import scala.util.control.NonFatal
 
 import com.spotify.docker.client._
+import com.spotify.docker.client.DockerClient.ListContainersParam
 import com.spotify.docker.client.exceptions.ImageNotFoundException
 import com.spotify.docker.client.messages.{ContainerConfig, HostConfig, PortBinding}
 import org.scalatest.concurrent.Eventually
@@ -45,7 +46,7 @@ abstract class DatabaseOnDocker {
   val env: Map[String, String]
 
   /**
-   * Wheather or not to use ipc mode for shared memory when starting docker image
+   * Whether or not to use ipc mode for shared memory when starting docker image
    */
   val usesIpc: Boolean
 
@@ -95,13 +96,22 @@ abstract class DockerJDBCIntegrationSuite extends SharedSparkSession with Eventu
 
   protected val dockerIp = DockerUtils.getDockerIp()
   val db: DatabaseOnDocker
+  val connectionTimeout = timeout(5.minutes)
+  val keepContainer =
+    sys.props.getOrElse("spark.test.docker.keepContainer", "false").toBoolean
 
   private var docker: DockerClient = _
-  protected var externalPort: Int = _
+  // Configure networking (necessary for boot2docker / Docker Machine)
+  protected lazy val externalPort: Int = {
+    val sock = new ServerSocket(0)
+    val port = sock.getLocalPort
+    sock.close()
+    port
+  }
   private var containerId: String = _
   protected var jdbcUrl: String = _
 
-  override def beforeAll() {
+  override def beforeAll(): Unit = {
     super.beforeAll()
     try {
       docker = DefaultDockerClient.fromEnv.build()
@@ -121,13 +131,6 @@ abstract class DockerJDBCIntegrationSuite extends SharedSparkSession with Eventu
           log.warn(s"Docker image ${db.imageName} not found; pulling image from registry")
           docker.pull(db.imageName)
       }
-      // Configure networking (necessary for boot2docker / Docker Machine)
-      externalPort = {
-        val sock = new ServerSocket(0)
-        val port = sock.getLocalPort
-        sock.close()
-        port
-      }
       val hostConfigBuilder = HostConfig.builder()
         .privileged(db.privileged)
         .networkMode("bridge")
@@ -155,7 +158,7 @@ abstract class DockerJDBCIntegrationSuite extends SharedSparkSession with Eventu
       docker.startContainer(containerId)
       jdbcUrl = db.getJdbcUrl(dockerIp, externalPort)
       var conn: Connection = null
-      eventually(timeout(2.minutes), interval(1.second)) {
+      eventually(connectionTimeout, interval(1.second)) {
         conn = getConnection()
       }
       // Run any setup queries:
@@ -174,22 +177,13 @@ abstract class DockerJDBCIntegrationSuite extends SharedSparkSession with Eventu
     }
   }
 
-  override def afterAll() {
+  override def afterAll(): Unit = {
     try {
+      cleanupContainer()
+    } finally {
       if (docker != null) {
-        try {
-          if (containerId != null) {
-            docker.killContainer(containerId)
-            docker.removeContainer(containerId)
-          }
-        } catch {
-          case NonFatal(e) =>
-            logWarning(s"Could not stop container $containerId", e)
-        } finally {
-          docker.close()
-        }
+        docker.close()
       }
-    } finally {
       super.afterAll()
     }
   }
@@ -205,4 +199,23 @@ abstract class DockerJDBCIntegrationSuite extends SharedSparkSession with Eventu
    * Prepare databases and tables for testing.
    */
   def dataPreparation(connection: Connection): Unit
+
+  private def cleanupContainer(): Unit = {
+    if (docker != null && containerId != null && !keepContainer) {
+      try {
+        docker.killContainer(containerId)
+      } catch {
+        case NonFatal(e) =>
+          val exitContainerIds =
+            docker.listContainers(ListContainersParam.withStatusExited()).asScala.map(_.id())
+          if (exitContainerIds.contains(containerId)) {
+            logWarning(s"Container $containerId already stopped")
+          } else {
+            logWarning(s"Could not stop container $containerId", e)
+          }
+      } finally {
+        docker.removeContainer(containerId)
+      }
+    }
+  }
 }
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerKrbJDBCIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerKrbJDBCIntegrationSuite.scala
index c20c006f3b8b6..4a828ae7ee3f9 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerKrbJDBCIntegrationSuite.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerKrbJDBCIntegrationSuite.scala
@@ -32,6 +32,7 @@ import org.apache.spark.util.{SecurityUtils, Utils}
 
 abstract class DockerKrbJDBCIntegrationSuite extends DockerJDBCIntegrationSuite {
   private var kdc: MiniKdc = _
+  private val KRB5_CONF_PROP = "java.security.krb5.conf"
   protected var entryPointDir: File = _
   protected var initDbDir: File = _
   protected val userName: String
@@ -160,4 +161,53 @@ abstract class DockerKrbJDBCIntegrationSuite extends DockerJDBCIntegrationSuite
     assert(rows(0).getString(0) === "foo")
     assert(rows(0).getString(1) === "bar")
   }
+
+  test("SPARK-35226: JDBCOption should accept refreshKrb5Config parameter") {
+    // This makes sure Spark must do authentication
+    Configuration.setConfiguration(null)
+    withTempDir { dir =>
+      val dummyKrb5Conf = File.createTempFile("dummy", "krb5.conf", dir)
+      val origKrb5Conf = sys.props(KRB5_CONF_PROP)
+      try {
+        // Set dummy krb5.conf and refresh config so this assertion is expected to fail.
+        // The thrown exception is dependent on the actual JDBC driver class.
+        intercept[Exception] {
+          sys.props(KRB5_CONF_PROP) = dummyKrb5Conf.getAbsolutePath
+          spark.read.format("jdbc")
+            .option("url", jdbcUrl)
+            .option("keytab", keytabFullPath)
+            .option("principal", principal)
+            .option("refreshKrb5Config", "true")
+            .option("query", "SELECT 1")
+            .load()
+        }
+
+        // Set the authentic krb5.conf but doesn't refresh config
+        // so this assertion is expected to fail.
+        intercept[Exception] {
+          sys.props(KRB5_CONF_PROP) = origKrb5Conf
+          spark.read.format("jdbc")
+            .option("url", jdbcUrl)
+            .option("keytab", keytabFullPath)
+            .option("principal", principal)
+            .option("query", "SELECT 1")
+            .load()
+        }
+
+        sys.props(KRB5_CONF_PROP) = origKrb5Conf
+        val df = spark.read.format("jdbc")
+          .option("url", jdbcUrl)
+          .option("keytab", keytabFullPath)
+          .option("principal", principal)
+          .option("refreshKrb5Config", "true")
+          .option("query", "SELECT 1")
+          .load()
+        val result = df.collect().map(_.getInt(0))
+        assert(result.length === 1)
+        assert(result(0) === 1)
+      } finally {
+        sys.props(KRB5_CONF_PROP) = origKrb5Conf
+      }
+    }
+  }
 }
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala
index 9b9d15517d572..9b653f81afe9f 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala
@@ -24,13 +24,21 @@ import com.spotify.docker.client.messages.{ContainerConfig, HostConfig}
 import org.apache.spark.sql.execution.datasources.jdbc.connection.SecureConnectionProvider
 import org.apache.spark.tags.DockerTest
 
+/**
+ * To run this test suite for a specific version (e.g., mariadb:10.5.8):
+ * {{{
+ *   MARIADB_DOCKER_IMAGE_NAME=mariadb:10.5.8
+ *     ./build/sbt -Pdocker-integration-tests
+ *     "testOnly org.apache.spark.sql.jdbc.MariaDBKrbIntegrationSuite"
+ * }}}
+ */
 @DockerTest
 class MariaDBKrbIntegrationSuite extends DockerKrbJDBCIntegrationSuite {
   override protected val userName = s"mariadb/$dockerIp"
   override protected val keytabFileName = "mariadb.keytab"
 
   override val db = new DatabaseOnDocker {
-    override val imageName = "mariadb:10.4"
+    override val imageName = sys.env.getOrElse("MARIADB_DOCKER_IMAGE_NAME", "mariadb:10.5")
     override val env = Map(
       "MYSQL_ROOT_PASSWORD" -> "rootpass"
     )
@@ -60,7 +68,7 @@ class MariaDBKrbIntegrationSuite extends DockerKrbJDBCIntegrationSuite {
 
   override protected def setAuthentication(keytabFile: String, principal: String): Unit = {
     val config = new SecureConnectionProvider.JDBCConfiguration(
-      Configuration.getConfiguration, "Krb5ConnectorContext", keytabFile, principal)
+      Configuration.getConfiguration, "Krb5ConnectorContext", keytabFile, principal, true)
     Configuration.setConfiguration(config)
   }
 }
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala
index 42d64873c44d9..c6e5cd26958ed 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala
@@ -21,13 +21,23 @@ import java.math.BigDecimal
 import java.sql.{Connection, Date, Timestamp}
 import java.util.Properties
 
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.tags.DockerTest
 
+/**
+ * To run this test suite for a specific version (e.g., 2019-GA-ubuntu-16.04):
+ * {{{
+ *   MSSQLSERVER_DOCKER_IMAGE_NAME=2019-GA-ubuntu-16.04
+ *     ./build/sbt -Pdocker-integration-tests
+ *     "testOnly org.apache.spark.sql.jdbc.MsSqlServerIntegrationSuite"
+ * }}}
+ */
 @DockerTest
 class MsSqlServerIntegrationSuite extends DockerJDBCIntegrationSuite {
   override val db = new DatabaseOnDocker {
-    override val imageName = "mcr.microsoft.com/mssql/server:2017-GA-ubuntu"
+    override val imageName = sys.env.getOrElse("MSSQLSERVER_DOCKER_IMAGE_NAME",
+      "mcr.microsoft.com/mssql/server:2019-GA-ubuntu-16.04")
     override val env = Map(
       "SA_PASSWORD" -> "Sapass123",
       "ACCEPT_EULA" -> "Y"
@@ -98,6 +108,37 @@ class MsSqlServerIntegrationSuite extends DockerJDBCIntegrationSuite {
         |'the', 'lazy',
         |'dog')
       """.stripMargin).executeUpdate()
+    conn.prepareStatement(
+      """
+        |CREATE TABLE spatials (
+        |point geometry,
+        |line geometry,
+        |circle geometry,
+        |curve geography,
+        |polygon geometry,
+        |curve_polygon geography,
+        |multi_point geometry,
+        |multi_line geometry,
+        |multi_polygon geometry,
+        |geometry_collection geometry)
+      """.stripMargin).executeUpdate()
+    conn.prepareStatement(
+      """
+        |INSERT INTO spatials VALUES (
+        |'POINT(3 4 7 2.5)',
+        |'LINESTRING(1 0, 0 1, -1 0)',
+        |'CIRCULARSTRING(
+        |  -122.358 47.653, -122.348 47.649, -122.348 47.658, -122.358 47.658, -122.358 47.653)',
+        |'COMPOUNDCURVE(
+        |  CIRCULARSTRING(-122.358 47.653, -122.348 47.649,
+        |    -122.348 47.658, -122.358 47.658, -122.358 47.653))',
+        |'POLYGON((-20 -20, -20 20, 20 20, 20 -20, -20 -20), (10 0, 0 10, 0 -10, 10 0))',
+        |'CURVEPOLYGON((-122.3 47, 122.3 47, 125.7 49, 121 38, -122.3 47))',
+        |'MULTIPOINT((2 3), (7 8 9.5))',
+        |'MULTILINESTRING((0 2, 1 1), (1 0, 1 1))',
+        |'MULTIPOLYGON(((2 2, 2 -2, -2 -2, -2 2, 2 2)),((1 1, 3 1, 3 3, 1 3, 1 1)))',
+        |'GEOMETRYCOLLECTION(LINESTRING(1 1, 3 5),POLYGON((-1 -1, -1 -5, -5 -5, -5 -1, -1 -1)))')
+      """.stripMargin).executeUpdate()
   }
 
   test("Basic test") {
@@ -166,24 +207,26 @@ class MsSqlServerIntegrationSuite extends DockerJDBCIntegrationSuite {
   }
 
   test("Date types") {
-    val df = spark.read.jdbc(jdbcUrl, "dates", new Properties)
-    val rows = df.collect()
-    assert(rows.length == 1)
-    val row = rows(0)
-    val types = row.toSeq.map(x => x.getClass.toString)
-    assert(types.length == 6)
-    assert(types(0).equals("class java.sql.Date"))
-    assert(types(1).equals("class java.sql.Timestamp"))
-    assert(types(2).equals("class java.sql.Timestamp"))
-    assert(types(3).equals("class java.lang.String"))
-    assert(types(4).equals("class java.sql.Timestamp"))
-    assert(types(5).equals("class java.sql.Timestamp"))
-    assert(row.getAs[Date](0).equals(Date.valueOf("1991-11-09")))
-    assert(row.getAs[Timestamp](1).equals(Timestamp.valueOf("1999-01-01 13:23:35.0")))
-    assert(row.getAs[Timestamp](2).equals(Timestamp.valueOf("9999-12-31 23:59:59.0")))
-    assert(row.getString(3).equals("1901-05-09 23:59:59.0000000 +14:00"))
-    assert(row.getAs[Timestamp](4).equals(Timestamp.valueOf("1996-01-01 23:24:00.0")))
-    assert(row.getAs[Timestamp](5).equals(Timestamp.valueOf("1900-01-01 13:31:24.0")))
+    withDefaultTimeZone(UTC) {
+      val df = spark.read.jdbc(jdbcUrl, "dates", new Properties)
+      val rows = df.collect()
+      assert(rows.length == 1)
+      val row = rows(0)
+      val types = row.toSeq.map(x => x.getClass.toString)
+      assert(types.length == 6)
+      assert(types(0).equals("class java.sql.Date"))
+      assert(types(1).equals("class java.sql.Timestamp"))
+      assert(types(2).equals("class java.sql.Timestamp"))
+      assert(types(3).equals("class java.lang.String"))
+      assert(types(4).equals("class java.sql.Timestamp"))
+      assert(types(5).equals("class java.sql.Timestamp"))
+      assert(row.getAs[Date](0).equals(Date.valueOf("1991-11-09")))
+      assert(row.getAs[Timestamp](1).equals(Timestamp.valueOf("1999-01-01 13:23:35.0")))
+      assert(row.getAs[Timestamp](2).equals(Timestamp.valueOf("9999-12-31 23:59:59.0")))
+      assert(row.getString(3).equals("1901-05-09 23:59:59.0000000 +14:00"))
+      assert(row.getAs[Timestamp](4).equals(Timestamp.valueOf("1996-01-01 23:24:00.0")))
+      assert(row.getAs[Timestamp](5).equals(Timestamp.valueOf("1970-01-01 13:31:24.0")))
+    }
   }
 
   test("String types") {
@@ -223,4 +266,94 @@ class MsSqlServerIntegrationSuite extends DockerJDBCIntegrationSuite {
     df2.write.jdbc(jdbcUrl, "datescopy", new Properties)
     df3.write.jdbc(jdbcUrl, "stringscopy", new Properties)
   }
+
+  test("SPARK-33813: MsSqlServerDialect should support spatial types") {
+    val df = spark.read.jdbc(jdbcUrl, "spatials", new Properties)
+    val rows = df.collect()
+    assert(rows.length == 1)
+    val row = rows(0)
+    val types = row.toSeq.map(x => x.getClass.toString)
+    assert(types.length == 10)
+    assert(types(0) == "class [B")
+    assert(row.getAs[Array[Byte]](0) ===
+      Array(0, 0, 0, 0, 1, 15, 0, 0, 0, 0, 0, 0, 8, 64, 0, 0, 0, 0, 0, 0,
+        16, 64, 0, 0, 0, 0, 0, 0, 28, 64, 0, 0, 0, 0, 0, 0, 4, 64))
+    assert(types(1) == "class [B")
+    assert(row.getAs[Array[Byte]](1) ===
+      Array[Byte](0, 0, 0, 0, 1, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, -16, 63, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        -16, 63, 0, 0, 0, 0, 0, 0, -16, -65, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
+        0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, -1, -1, -1, -1, 0, 0, 0, 0, 2))
+    assert(types(2) == "class [B")
+    assert(row.getAs[Array[Byte]](2) ===
+      Array[Byte](0, 0, 0, 0, 2, 4, 5, 0, 0, 0, -12, -3, -44, 120, -23, -106,
+        94, -64, -35, 36, 6, -127, -107, -45, 71, 64, -125, -64, -54, -95, 69,
+        -106, 94, -64, 80, -115, -105, 110, 18, -45, 71, 64, -125, -64, -54,
+        -95, 69, -106, 94, -64, 78, 98, 16, 88, 57, -44, 71, 64, -12, -3, -44,
+        120, -23, -106, 94, -64, 78, 98, 16, 88, 57, -44, 71, 64, -12, -3, -44,
+        120, -23, -106, 94, -64, -35, 36, 6, -127, -107, -45, 71, 64, 1, 0, 0,
+        0, 2, 0, 0, 0, 0, 1, 0, 0, 0, -1, -1, -1, -1, 0, 0, 0, 0, 8))
+    assert(types(3) == "class [B")
+    assert(row.getAs[Array[Byte]](3) ===
+      Array[Byte](-26, 16, 0, 0, 2, 4, 5, 0, 0, 0, -35, 36, 6, -127, -107, -45,
+        71, 64, -12, -3, -44, 120, -23, -106, 94, -64, 80, -115, -105, 110, 18,
+        -45, 71, 64, -125, -64, -54, -95, 69, -106, 94, -64, 78, 98, 16, 88, 57,
+        -44, 71, 64, -125, -64, -54, -95, 69, -106, 94, -64, 78, 98, 16, 88, 57,
+        -44, 71, 64, -12, -3, -44, 120, -23, -106, 94, -64, -35, 36, 6, -127, -107,
+        -45, 71, 64, -12, -3, -44, 120, -23, -106, 94, -64, 1, 0, 0, 0, 3, 0, 0,
+        0, 0, 1, 0, 0, 0, -1, -1, -1, -1, 0, 0, 0, 0, 9, 2, 0, 0, 0, 3, 1))
+    assert(types(5) == "class [B")
+    assert(row.getAs[Array[Byte]](4) ===
+      Array[Byte](0, 0, 0, 0, 1, 4, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, -64, 0, 0,
+        0, 0, 0, 0, 52, -64, 0, 0, 0, 0, 0, 0, 52, -64, 0, 0, 0, 0, 0, 0, 52, 64,
+        0, 0, 0, 0, 0, 0, 52, 64, 0, 0, 0, 0, 0, 0, 52, 64, 0, 0, 0, 0, 0, 0, 52,
+        64, 0, 0, 0, 0, 0, 0, 52, -64, 0, 0, 0, 0, 0, 0, 52, -64, 0, 0, 0, 0, 0,
+        0, 52, -64, 0, 0, 0, 0, 0, 0, 36, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 36, -64, 0, 0, 0, 0, 0, 0, 36, 64, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0,
+        0, 2, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, -1, -1, -1, -1, 0, 0, 0, 0, 3))
+    assert(types(6) === "class [B")
+    assert(row.getAs[Array[Byte]](5) ===
+      Array[Byte](-26, 16, 0, 0, 2, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, -128, 71, 64, 51,
+        51, 51, 51, 51, -109, 94, -64, 0, 0, 0, 0, 0, -128, 71, 64, 51, 51, 51, 51,
+        51, -109, 94, 64, 0, 0, 0, 0, 0, -128, 72, 64, -51, -52, -52, -52, -52, 108,
+        95, 64, 0, 0, 0, 0, 0, 0, 67, 64, 0, 0, 0, 0, 0, 64, 94, 64, 0, 0, 0, 0, 0,
+        -128, 71, 64, 51, 51, 51, 51, 51, -109, 94, -64, 1, 0, 0, 0, 1, 0, 0, 0, 0,
+        1, 0, 0, 0, -1, -1, -1, -1, 0, 0, 0, 0, 10))
+    assert(types(6) === "class [B")
+    assert(row.getAs[Array[Byte]](6) ===
+      Array[Byte](0, 0, 0, 0, 1, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0,
+        0, 0, 8, 64, 0, 0, 0, 0, 0, 0, 28, 64, 0, 0, 0, 0, 0, 0, 32, 64, 0, 0, 0, 0,
+        0, 0, -8, -1, 0, 0, 0, 0, 0, 0, 35, 64, 2, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0,
+        0, 0, 3, 0, 0, 0, -1, -1, -1, -1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1,
+        0, 0, 0, 0, 1, 0, 0, 0, 1))
+    assert(types(6) === "class [B")
+    assert(row.getAs[Array[Byte]](7) ===
+      Array[Byte](0, 0, 0, 0, 1, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 64, 0, 0, 0, 0, 0, 0, -16, 63, 0, 0, 0, 0, 0, 0, -16, 63, 0, 0, 0,
+        0, 0, 0, -16, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -16, 63, 0, 0,
+        0, 0, 0, 0, -16, 63, 2, 0, 0, 0, 1, 0, 0, 0, 0, 1, 2, 0, 0, 0, 3, 0, 0, 0,
+        -1, -1, -1, -1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 2))
+    assert(types(6) === "class [B")
+    assert(row.getAs[Array[Byte]](8) ===
+      Array[Byte](0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0,
+        0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, -64, 0, 0, 0,
+        0, 0, 0, 0, -64, 0, 0, 0, 0, 0, 0, 0, -64, 0, 0, 0, 0, 0, 0, 0, -64, 0, 0,
+        0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0,
+        0, 0, 0, 0, -16, 63, 0, 0, 0, 0, 0, 0, -16, 63, 0, 0, 0, 0, 0, 0, 8, 64, 0,
+        0, 0, 0, 0, 0, -16, 63, 0, 0, 0, 0, 0, 0, 8, 64, 0, 0, 0, 0, 0, 0, 8, 64, 0,
+        0, 0, 0, 0, 0, -16, 63, 0, 0, 0, 0, 0, 0, 8, 64, 0, 0, 0, 0, 0, 0, -16, 63,
+        0, 0, 0, 0, 0, 0, -16, 63, 2, 0, 0, 0, 2, 0, 0, 0, 0, 2, 5, 0, 0, 0, 3, 0,
+        0, 0, -1, -1, -1, -1, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 3))
+    assert(types(6) === "class [B")
+    assert(row.getAs[Array[Byte]](9) ===
+      Array[Byte](0, 0, 0, 0, 1, 4, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, -16, 63, 0, 0, 0,
+        0, 0, 0, -16, 63, 0, 0, 0, 0, 0, 0, 8, 64, 0, 0, 0, 0, 0, 0, 20, 64, 0, 0,
+        0, 0, 0, 0, -16, -65, 0, 0, 0, 0, 0, 0, -16, -65, 0, 0, 0, 0, 0, 0, -16, -65,
+        0, 0, 0, 0, 0, 0, 20, -64, 0, 0, 0, 0, 0, 0, 20, -64, 0, 0, 0, 0, 0, 0, 20,
+        -64, 0, 0, 0, 0, 0, 0, 20, -64, 0, 0, 0, 0, 0, 0, -16, -65, 0, 0, 0, 0, 0, 0,
+        -16, -65, 0, 0, 0, 0, 0, 0, -16, -65, 2, 0, 0, 0, 1, 0, 0, 0, 0, 2, 2, 0, 0,
+        0, 3, 0, 0, 0, -1, -1, -1, -1, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0,
+        0, 0, 0, 1, 0, 0, 0, 3))
+  }
 }
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
index 4cbcb59e02de1..5d4da7d7ed794 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
@@ -21,13 +21,22 @@ import java.math.BigDecimal
 import java.sql.{Connection, Date, Timestamp}
 import java.util.Properties
 
-import org.apache.spark.sql.{Row, SaveMode}
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
 import org.apache.spark.tags.DockerTest
 
+/**
+ * To run this test suite for a specific version (e.g., mysql:5.7.31):
+ * {{{
+ *   MYSQL_DOCKER_IMAGE_NAME=mysql:5.7.31
+ *     ./build/sbt -Pdocker-integration-tests
+ *     "testOnly org.apache.spark.sql.jdbc.MySQLIntegrationSuite"
+ * }}}
+ */
 @DockerTest
 class MySQLIntegrationSuite extends DockerJDBCIntegrationSuite {
   override val db = new DatabaseOnDocker {
-    override val imageName = "mysql:5.7.28"
+    override val imageName = sys.env.getOrElse("MYSQL_DOCKER_IMAGE_NAME", "mysql:5.7.31")
     override val env = Map(
       "MYSQL_ROOT_PASSWORD" -> "rootpass"
     )
@@ -103,21 +112,24 @@ class MySQLIntegrationSuite extends DockerJDBCIntegrationSuite {
   }
 
   test("Date types") {
-    val df = sqlContext.read.jdbc(jdbcUrl, "dates", new Properties)
-    val rows = df.collect()
-    assert(rows.length == 1)
-    val types = rows(0).toSeq.map(x => x.getClass.toString)
-    assert(types.length == 5)
-    assert(types(0).equals("class java.sql.Date"))
-    assert(types(1).equals("class java.sql.Timestamp"))
-    assert(types(2).equals("class java.sql.Timestamp"))
-    assert(types(3).equals("class java.sql.Timestamp"))
-    assert(types(4).equals("class java.sql.Date"))
-    assert(rows(0).getAs[Date](0).equals(Date.valueOf("1991-11-09")))
-    assert(rows(0).getAs[Timestamp](1).equals(Timestamp.valueOf("1970-01-01 13:31:24")))
-    assert(rows(0).getAs[Timestamp](2).equals(Timestamp.valueOf("1996-01-01 01:23:45")))
-    assert(rows(0).getAs[Timestamp](3).equals(Timestamp.valueOf("2009-02-13 23:31:30")))
-    assert(rows(0).getAs[Date](4).equals(Date.valueOf("2001-01-01")))
+    withDefaultTimeZone(UTC) {
+      val df = sqlContext.read.jdbc(jdbcUrl, "dates", new Properties)
+      val rows = df.collect()
+      assert(rows.length == 1)
+      val types = rows(0).toSeq.map(x => x.getClass.toString)
+      assert(types.length == 5)
+      assert(types(0).equals("class java.sql.Date"))
+      assert(types(1).equals("class java.sql.Timestamp"))
+      assert(types(2).equals("class java.sql.Timestamp"))
+      assert(types(3).equals("class java.sql.Timestamp"))
+      assert(types(4).equals("class java.sql.Date"))
+      assert(rows(0).getAs[Date](0).equals(Date.valueOf("1991-11-09")))
+      assert(
+        rows(0).getAs[Timestamp](1) === Timestamp.valueOf("1970-01-01 13:31:24"))
+      assert(rows(0).getAs[Timestamp](2).equals(Timestamp.valueOf("1996-01-01 01:23:45")))
+      assert(rows(0).getAs[Timestamp](3).equals(Timestamp.valueOf("2009-02-13 23:31:30")))
+      assert(rows(0).getAs[Date](4).equals(Date.valueOf("2001-01-01")))
+    }
   }
 
   test("String types") {
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
index 24c3adb9c0153..7f8e7c48462f5 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
@@ -21,7 +21,10 @@ import java.math.BigDecimal
 import java.sql.{Connection, Date, Timestamp}
 import java.util.{Properties, TimeZone}
 
+import org.scalatest.time.SpanSugar._
+
 import org.apache.spark.sql.{Row, SaveMode}
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
 import org.apache.spark.sql.execution.{RowDataSourceScanExec, WholeStageCodegenExec}
 import org.apache.spark.sql.execution.datasources.LogicalRelation
 import org.apache.spark.sql.execution.datasources.jdbc.{JDBCPartition, JDBCRelation}
@@ -31,27 +34,27 @@ import org.apache.spark.sql.types._
 import org.apache.spark.tags.DockerTest
 
 /**
- * This patch was tested using the Oracle docker. Created this integration suite for the same.
- * The ojdbc6-11.2.0.2.0.jar was to be downloaded from the maven repository. Since there was
- * no jdbc jar available in the maven repository, the jar was downloaded from oracle site
- * manually and installed in the local; thus tested. So, for SparkQA test case run, the
- * ojdbc jar might be manually placed in the local maven repository(com/oracle/ojdbc6/11.2.0.2.0)
- * while Spark QA test run.
- *
  * The following would be the steps to test this
  * 1. Build Oracle database in Docker, please refer below link about how to.
  *    https://github.com/oracle/docker-images/blob/master/OracleDatabase/SingleInstance/README.md
  * 2. export ORACLE_DOCKER_IMAGE_NAME=$ORACLE_DOCKER_IMAGE_NAME
  *    Pull oracle $ORACLE_DOCKER_IMAGE_NAME image - docker pull $ORACLE_DOCKER_IMAGE_NAME
  * 3. Start docker - sudo service docker start
- * 4. Download oracle 11g driver jar and put it in maven local repo:
- *    (com/oracle/ojdbc6/11.2.0.2.0/ojdbc6-11.2.0.2.0.jar)
- * 5. The timeout and interval parameter to be increased from 60,1 to a high value for oracle test
- *    in DockerJDBCIntegrationSuite.scala (Locally tested with 200,200 and executed successfully).
- * 6. Run spark test - ./build/sbt "test-only org.apache.spark.sql.jdbc.OracleIntegrationSuite"
+ * 4. Run spark test - ./build/sbt -Pdocker-integration-tests
+ *    "testOnly org.apache.spark.sql.jdbc.OracleIntegrationSuite"
+ *
+ * An actual sequence of commands to run the test is as follows
+ *
+ *  $ git clone https://github.com/oracle/docker-images.git
+ *  // Head SHA: 3e352a22618070595f823977a0fd1a3a8071a83c
+ *  $ cd docker-images/OracleDatabase/SingleInstance/dockerfiles
+ *  $ ./buildDockerImage.sh -v 18.4.0 -x
+ *  $ export ORACLE_DOCKER_IMAGE_NAME=oracle/database:18.4.0-xe
+ *  $ cd $SPARK_HOME
+ *  $ ./build/sbt -Pdocker-integration-tests
+ *    "testOnly org.apache.spark.sql.jdbc.OracleIntegrationSuite"
  *
- * All tests in this suite are ignored because of the dependency with the oracle jar from maven
- * repository.
+ * It has been validated with 18.4.0 Express Edition.
  */
 @DockerTest
 class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSparkSession {
@@ -60,7 +63,7 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSpark
   override val db = new DatabaseOnDocker {
     override val imageName = sys.env("ORACLE_DOCKER_IMAGE_NAME")
     override val env = Map(
-      "ORACLE_ROOT_PASSWORD" -> "oracle"
+      "ORACLE_PWD" -> "oracle"
     )
     override val usesIpc = false
     override val jdbcPort: Int = 1521
@@ -68,7 +71,11 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSpark
       s"jdbc:oracle:thin:system/oracle@//$ip:$port/xe"
   }
 
+  override val connectionTimeout = timeout(7.minutes)
+
   override def dataPreparation(conn: Connection): Unit = {
+    // In 18.4.0 Express Edition auto commit is enabled by default.
+    conn.setAutoCommit(false)
     conn.prepareStatement("CREATE TABLE datetime (id NUMBER(10), d DATE, t TIMESTAMP)")
       .executeUpdate()
     conn.prepareStatement(
@@ -282,23 +289,6 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSpark
     }
   }
 
-  /**
-   * Change the Time Zone `timeZoneId` of JVM before executing `f`, then switches back to the
-   * original after `f` returns.
-   * @param timeZoneId the ID for a TimeZone, either an abbreviation such as "PST", a full name such
-   *                   as "America/Los_Angeles", or a custom ID such as "GMT-8:00".
-   */
-  private def withTimeZone(timeZoneId: String)(f: => Unit): Unit = {
-    val originalLocale = TimeZone.getDefault
-    try {
-      // Add Locale setting
-      TimeZone.setDefault(TimeZone.getTimeZone(timeZoneId))
-      f
-    } finally {
-      TimeZone.setDefault(originalLocale)
-    }
-  }
-
   test("Column TIMESTAMP with TIME ZONE(JVM timezone)") {
     def checkRow(row: Row, ts: String): Unit = {
       assert(row.getTimestamp(1).equals(Timestamp.valueOf(ts)))
@@ -306,14 +296,14 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSpark
 
     withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> TimeZone.getDefault.getID) {
       val dfRead = sqlContext.read.jdbc(jdbcUrl, "ts_with_timezone", new Properties)
-      withTimeZone("PST") {
+      withDefaultTimeZone(PST) {
         assert(dfRead.collect().toSet ===
           Set(
             Row(BigDecimal.valueOf(1), java.sql.Timestamp.valueOf("1999-12-01 03:00:00")),
             Row(BigDecimal.valueOf(2), java.sql.Timestamp.valueOf("1999-12-01 12:00:00"))))
       }
 
-      withTimeZone("UTC") {
+      withDefaultTimeZone(UTC) {
         assert(dfRead.collect().toSet ===
           Set(
             Row(BigDecimal.valueOf(1), java.sql.Timestamp.valueOf("1999-12-01 11:00:00")),
@@ -395,7 +385,7 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSpark
     val values = rows(0)
     assert(values.getDecimal(0).equals(new java.math.BigDecimal("12312321321321312312312312123")))
     assert(values.getInt(1).equals(1))
-    assert(values.getBoolean(2).equals(false))
+    assert(values.getBoolean(2) == false)
   }
 
   test("SPARK-22303: handle BINARY_DOUBLE and BINARY_FLOAT as DoubleType and FloatType") {
@@ -456,9 +446,9 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSpark
       case LogicalRelation(JDBCRelation(_, parts, _), _, _, _) =>
         val whereClauses = parts.map(_.asInstanceOf[JDBCPartition].whereClause).toSet
         assert(whereClauses === Set(
-          """"D" < '2018-07-10' or "D" is null""",
-          """"D" >= '2018-07-10' AND "D" < '2018-07-14'""",
-          """"D" >= '2018-07-14'"""))
+          """"D" < '2018-07-11' or "D" is null""",
+          """"D" >= '2018-07-11' AND "D" < '2018-07-15'""",
+          """"D" >= '2018-07-15'"""))
     }
     assert(df1.collect.toSet === expectedResult)
 
@@ -512,4 +502,15 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSpark
        """.stripMargin.replaceAll("\n", " "))
     assert(sql("select id, d, t from queryOption").collect.toSet == expectedResult)
   }
+
+  test("SPARK-32992: map Oracle's ROWID type to StringType") {
+    val rows = spark.read.format("jdbc")
+      .option("url", jdbcUrl)
+      .option("query", "SELECT ROWID from datetime")
+      .load()
+      .collect()
+    val types = rows(0).toSeq.map(x => x.getClass.toString)
+    assert(types(0).equals("class java.lang.String"))
+    assert(!rows(0).getString(0).isEmpty)
+  }
 }
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
index 6611bc2d19ed8..39450d1f56799 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
@@ -17,7 +17,9 @@
 
 package org.apache.spark.sql.jdbc
 
-import java.sql.Connection
+import java.math.{BigDecimal => JBigDecimal}
+import java.sql.{Connection, Date, Timestamp}
+import java.text.SimpleDateFormat
 import java.util.Properties
 
 import org.apache.spark.sql.Column
@@ -26,10 +28,18 @@ import org.apache.spark.sql.catalyst.expressions.Literal
 import org.apache.spark.sql.types.{ArrayType, DecimalType, FloatType, ShortType}
 import org.apache.spark.tags.DockerTest
 
+/**
+ * To run this test suite for a specific version (e.g., postgres:13.0):
+ * {{{
+ *   POSTGRES_DOCKER_IMAGE_NAME=postgres:13.0
+ *     ./build/sbt -Pdocker-integration-tests
+ *     "testOnly org.apache.spark.sql.jdbc.PostgresIntegrationSuite"
+ * }}}
+ */
 @DockerTest
 class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite {
   override val db = new DatabaseOnDocker {
-    override val imageName = "postgres:12.0-alpine"
+    override val imageName = sys.env.getOrElse("POSTGRES_DOCKER_IMAGE_NAME", "postgres:13.0-alpine")
     override val env = Map(
       "POSTGRES_PASSWORD" -> "rootpass"
     )
@@ -46,15 +56,28 @@ class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite {
     conn.prepareStatement("CREATE TABLE bar (c0 text, c1 integer, c2 double precision, c3 bigint, "
       + "c4 bit(1), c5 bit(10), c6 bytea, c7 boolean, c8 inet, c9 cidr, "
       + "c10 integer[], c11 text[], c12 real[], c13 numeric(2,2)[], c14 enum_type, "
-      + "c15 float4, c16 smallint, c17 numeric[])").executeUpdate()
+      + "c15 float4, c16 smallint, c17 numeric[], c18 bit varying(6), c19 point, c20 line, "
+      + "c21 lseg, c22 box, c23 path, c24 polygon, c25 circle, c26 pg_lsn, "
+      + "c27 character(2), c28 character varying(3), c29 date, c30 interval, "
+      + "c31 macaddr, c32 macaddr8, c33 numeric(6,4), c34 pg_snapshot, "
+      + "c35 real, c36 time, c37 timestamp, c38 tsquery, c39 tsvector, c40 txid_snapshot, "
+      + "c41 xml)").executeUpdate()
     conn.prepareStatement("INSERT INTO bar VALUES ('hello', 42, 1.25, 123456789012345, B'0', "
       + "B'1000100101', E'\\\\xDEADBEEF', true, '172.16.0.42', '192.168.0.0/16', "
       + """'{1, 2}', '{"a", null, "b"}', '{0.11, 0.22}', '{0.11, 0.22}', 'd1', 1.01, 1, """
-      + "'{111.2222, 333.4444}')"
+      + "'{111.2222, 333.4444}', B'101010', '(800, 600)', '(23.8, 56.2), (16.23, 89.2)', "
+      + "'[(80.12, 131.24), (201.5, 503.33)]', '(19.84, 11.23), (20.21, 2.1)', "
+      + "'(10.2, 30.4), (50.6, 70.8), (90.1, 11.3)', "
+      + "'((100.3, 40.2), (20.198, 83.1), (500.821, 311.38))', '<500, 200, 100>', '16/B374D848', "
+      + "'ab', 'efg', '2021-02-02', '1 minute', '00:11:22:33:44:55', "
+      + "'00:11:22:33:44:55:66:77', 12.3456, '10:20:10,14,15', 1E+37, "
+      + "'17:22:31', '2016-08-12 10:22:31.949271', 'cat:AB & dog:CD', "
+      + "'dog and cat and fox', '10:20:10,14,15', '<key>id</key><value>10</value>')"
     ).executeUpdate()
     conn.prepareStatement("INSERT INTO bar VALUES (null, null, null, null, null, "
-      + "null, null, null, null, null, "
-      + "null, null, null, null, null, null, null, null)"
+      + "null, null, null, null, null, null, null, null, null, null, null, null, "
+      + "null, null, null, null, null, null, null, null, null, null, null, null, "
+      + "null, null, null, null, null, null, null, null, null, null, null, null, null)"
     ).executeUpdate()
 
     conn.prepareStatement("CREATE TABLE ts_with_timezone " +
@@ -66,7 +89,10 @@ class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite {
       .executeUpdate()
 
     conn.prepareStatement("CREATE TABLE st_with_array (c0 uuid, c1 inet, c2 cidr," +
-      "c3 json, c4 jsonb, c5 uuid[], c6 inet[], c7 cidr[], c8 json[], c9 jsonb[])")
+      "c3 json, c4 jsonb, c5 uuid[], c6 inet[], c7 cidr[], c8 json[], c9 jsonb[], c10 xml[], " +
+      "c11 tsvector[], c12 tsquery[], c13 macaddr[], c14 txid_snapshot[], c15 point[], " +
+      "c16 line[], c17 lseg[], c18 box[], c19 path[], c20 polygon[], c21 circle[], c22 pg_lsn[], " +
+      "c23 bit varying(6)[], c24 interval[], c25 macaddr8[], c26 pg_snapshot[])")
       .executeUpdate()
     conn.prepareStatement("INSERT INTO st_with_array VALUES ( " +
       "'0a532531-cdf1-45e3-963d-5de90b6a30f1', '172.168.22.1', '192.168.100.128/25', " +
@@ -75,9 +101,43 @@ class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite {
       "'205f9bfc-018c-4452-a605-609c0cfad228']::uuid[], ARRAY['172.16.0.41', " +
       "'172.16.0.42']::inet[], ARRAY['192.168.0.0/24', '10.1.0.0/16']::cidr[], " +
       """ARRAY['{"a": "foo", "b": "bar"}', '{"a": 1, "b": 2}']::json[], """ +
-      """ARRAY['{"a": 1, "b": 2, "c": 3}']::jsonb[])"""
-    )
-      .executeUpdate()
+      """ARRAY['{"a": 1, "b": 2, "c": 3}']::jsonb[], """ +
+      """ARRAY['<key>id</key><value>10</value>']::xml[], ARRAY['The dog laying on the grass', """ +
+      """'the:1 cat:2 is:3 on:4 the:5 table:6']::tsvector[], """ +
+      """ARRAY['programming & language & ! interpreter', 'cat:AB & dog:CD']::tsquery[], """ +
+      """ARRAY['12:34:56:78:90:ab', 'cd-ef-12-34-56-78']::macaddr[], """ +
+      """ARRAY['10:20:10,14,15']::txid_snapshot[], """ +
+      """ARRAY['(800, 600)', '83.24, 5.10']::point[], """ +
+      """ARRAY['(23.8, 56.2), (16.23, 89.2)', '{23.85, 10.87, 5.92}']::line[], """ +
+      """ARRAY['[(80.12, 131.24), (201.5, 503.33)]']::lseg[], """ +
+      """ARRAY['(19.84, 11.23), (20.21, 2.1)']::box[], """ +
+      """ARRAY['(10.2, 30.4), (50.6, 70.8), (90.1, 11.3)']::path[], """ +
+      """ARRAY['((100.3, 40.2), (20.198, 83.1), (500.821, 311.38))']::polygon[], """ +
+      """ARRAY['<500, 200, 100>']::circle[], """ +
+      """ARRAY['16/B374D848']::pg_lsn[], """ +
+      """ARRAY[B'101010']::bit varying(6)[], """ +
+      """ARRAY['1 day', '2 minutes']::interval[], """ +
+      """ARRAY['08:00:2b:01:02:03:04:05']::macaddr8[], """ +
+      """ARRAY['10:20:10,14,15']::pg_snapshot[])"""
+    ).executeUpdate()
+
+    conn.prepareStatement("CREATE TABLE char_types (" +
+      "c0 char(4), c1 character(4), c2 character varying(4), c3 varchar(4), c4 bpchar)"
+    ).executeUpdate()
+    conn.prepareStatement("INSERT INTO char_types VALUES " +
+      "('abcd', 'efgh', 'ijkl', 'mnop', 'q')").executeUpdate()
+
+    conn.prepareStatement("CREATE TABLE char_array_types (" +
+      "c0 char(4)[], c1 character(4)[], c2 character varying(4)[], c3 varchar(4)[], c4 bpchar[])"
+    ).executeUpdate()
+    conn.prepareStatement("INSERT INTO char_array_types VALUES " +
+      """('{"a", "bcd"}', '{"ef", "gh"}', '{"i", "j", "kl"}', '{"mnop"}', '{"q", "r"}')"""
+    ).executeUpdate()
+
+    conn.prepareStatement("CREATE TABLE money_types (" +
+      "c0 money)").executeUpdate()
+    conn.prepareStatement("INSERT INTO money_types VALUES " +
+      "('$1,000.00')").executeUpdate()
   }
 
   test("Type mapping for various types") {
@@ -86,7 +146,7 @@ class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite {
     assert(rows.length == 2)
     // Test the types, and values using the first row.
     val types = rows(0).toSeq.map(x => x.getClass)
-    assert(types.length == 18)
+    assert(types.length == 42)
     assert(classOf[String].isAssignableFrom(types(0)))
     assert(classOf[java.lang.Integer].isAssignableFrom(types(1)))
     assert(classOf[java.lang.Double].isAssignableFrom(types(2)))
@@ -97,14 +157,38 @@ class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite {
     assert(classOf[java.lang.Boolean].isAssignableFrom(types(7)))
     assert(classOf[String].isAssignableFrom(types(8)))
     assert(classOf[String].isAssignableFrom(types(9)))
-    assert(classOf[Seq[Int]].isAssignableFrom(types(10)))
-    assert(classOf[Seq[String]].isAssignableFrom(types(11)))
-    assert(classOf[Seq[Double]].isAssignableFrom(types(12)))
-    assert(classOf[Seq[BigDecimal]].isAssignableFrom(types(13)))
+    assert(classOf[scala.collection.Seq[Int]].isAssignableFrom(types(10)))
+    assert(classOf[scala.collection.Seq[String]].isAssignableFrom(types(11)))
+    assert(classOf[scala.collection.Seq[Double]].isAssignableFrom(types(12)))
+    assert(classOf[scala.collection.Seq[BigDecimal]].isAssignableFrom(types(13)))
     assert(classOf[String].isAssignableFrom(types(14)))
     assert(classOf[java.lang.Float].isAssignableFrom(types(15)))
     assert(classOf[java.lang.Short].isAssignableFrom(types(16)))
-    assert(classOf[Seq[BigDecimal]].isAssignableFrom(types(17)))
+    assert(classOf[scala.collection.Seq[BigDecimal]].isAssignableFrom(types(17)))
+    assert(classOf[String].isAssignableFrom(types(18)))
+    assert(classOf[String].isAssignableFrom(types(19)))
+    assert(classOf[String].isAssignableFrom(types(20)))
+    assert(classOf[String].isAssignableFrom(types(21)))
+    assert(classOf[String].isAssignableFrom(types(22)))
+    assert(classOf[String].isAssignableFrom(types(23)))
+    assert(classOf[String].isAssignableFrom(types(24)))
+    assert(classOf[String].isAssignableFrom(types(25)))
+    assert(classOf[String].isAssignableFrom(types(26)))
+    assert(classOf[String].isAssignableFrom(types(27)))
+    assert(classOf[String].isAssignableFrom(types(28)))
+    assert(classOf[Date].isAssignableFrom(types(29)))
+    assert(classOf[String].isAssignableFrom(types(30)))
+    assert(classOf[String].isAssignableFrom(types(31)))
+    assert(classOf[String].isAssignableFrom(types(32)))
+    assert(classOf[JBigDecimal].isAssignableFrom(types(33)))
+    assert(classOf[String].isAssignableFrom(types(34)))
+    assert(classOf[java.lang.Float].isAssignableFrom(types(35)))
+    assert(classOf[java.sql.Timestamp].isAssignableFrom(types(36)))
+    assert(classOf[java.sql.Timestamp].isAssignableFrom(types(37)))
+    assert(classOf[String].isAssignableFrom(types(38)))
+    assert(classOf[String].isAssignableFrom(types(39)))
+    assert(classOf[String].isAssignableFrom(types(40)))
+    assert(classOf[String].isAssignableFrom(types(41)))
     assert(rows(0).getString(0).equals("hello"))
     assert(rows(0).getInt(1) == 42)
     assert(rows(0).getDouble(2) == 1.25)
@@ -127,6 +211,30 @@ class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite {
     assert(rows(0).getShort(16) == 1)
     assert(rows(0).getSeq(17) ==
       Seq("111.222200000000000000", "333.444400000000000000").map(BigDecimal(_).bigDecimal))
+    assert(rows(0).getString(18) == "101010")
+    assert(rows(0).getString(19) == "(800,600)")
+    assert(rows(0).getString(20) == "{-4.359313077939234,-1,159.9516512549538}")
+    assert(rows(0).getString(21) == "[(80.12,131.24),(201.5,503.33)]")
+    assert(rows(0).getString(22) == "(20.21,11.23),(19.84,2.1)")
+    assert(rows(0).getString(23) == "((10.2,30.4),(50.6,70.8),(90.1,11.3))")
+    assert(rows(0).getString(24) == "((100.3,40.2),(20.198,83.1),(500.821,311.38))")
+    assert(rows(0).getString(25) == "<(500,200),100>")
+    assert(rows(0).getString(26) == "16/B374D848")
+    assert(rows(0).getString(27) == "ab")
+    assert(rows(0).getString(28) == "efg")
+    assert(rows(0).getDate(29) == new SimpleDateFormat("yyyy-MM-dd").parse("2021-02-02"))
+    assert(rows(0).getString(30) == "00:01:00")
+    assert(rows(0).getString(31) == "00:11:22:33:44:55")
+    assert(rows(0).getString(32) == "00:11:22:33:44:55:66:77")
+    assert(rows(0).getDecimal(33) == new JBigDecimal("12.3456"))
+    assert(rows(0).getString(34) == "10:20:10,14,15")
+    assert(rows(0).getFloat(35) == 1E+37F)
+    assert(rows(0).getTimestamp(36) == Timestamp.valueOf("1970-01-01 17:22:31.0"))
+    assert(rows(0).getTimestamp(37) == Timestamp.valueOf("2016-08-12 10:22:31.949271"))
+    assert(rows(0).getString(38) == "'cat':AB & 'dog':CD")
+    assert(rows(0).getString(39) == "'and' 'cat' 'dog' 'fox'")
+    assert(rows(0).getString(40) == "10:20:10,14,15")
+    assert(rows(0).getString(41) == "<key>id</key><value>10</value>")
 
     // Test reading null values using the second row.
     assert(0.until(16).forall(rows(1).isNullAt(_)))
@@ -179,6 +287,27 @@ class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite {
     assert(rows(0).getSeq(7) == Seq("192.168.0.0/24", "10.1.0.0/16"))
     assert(rows(0).getSeq(8) == Seq("""{"a": "foo", "b": "bar"}""", """{"a": 1, "b": 2}"""))
     assert(rows(0).getSeq(9) == Seq("""{"a": 1, "b": 2, "c": 3}"""))
+    assert(rows(0).getSeq(10) == Seq("""<key>id</key><value>10</value>"""))
+    assert(rows(0).getSeq(11) == Seq("'The' 'dog' 'grass' 'laying' 'on' 'the'",
+      "'cat':2 'is':3 'on':4 'table':6 'the':1,5"))
+    assert(rows(0).getSeq(12) == Seq("'programming' & 'language' & !'interpreter'",
+      "'cat':AB & 'dog':CD"))
+    assert(rows(0).getSeq(13) == Seq("12:34:56:78:90:ab", "cd:ef:12:34:56:78"))
+    assert(rows(0).getSeq(14) == Seq("10:20:10,14,15"))
+    assert(rows(0).getSeq(15) == Seq("(800.0,600.0)", "(83.24,5.1)"))
+    assert(rows(0).getSeq(16) == Seq("{-4.359313077939234,-1.0,159.9516512549538}",
+      "{23.85,10.87,5.92}"))
+    assert(rows(0).getSeq(17) == Seq("[(80.12,131.24),(201.5,503.33)]"))
+    assert(rows(0).getSeq(18) == Seq("(20.21,11.23),(19.84,2.1)"))
+    assert(rows(0).getSeq(19) == Seq("((10.2,30.4),(50.6,70.8),(90.1,11.3))"))
+    assert(rows(0).getSeq(20) == Seq("((100.3,40.2),(20.198,83.1),(500.821,311.38))"))
+    assert(rows(0).getSeq(21) == Seq("<(500.0,200.0),100.0>"))
+    assert(rows(0).getSeq(22) == Seq("16/B374D848"))
+    assert(rows(0).getSeq(23) == Seq("101010"))
+    assert(rows(0).getSeq(24) == Seq("0 years 0 mons 1 days 0 hours 0 mins 0.0 secs",
+      "0 years 0 mons 0 days 0 hours 2 mins 0.0 secs"))
+    assert(rows(0).getSeq(25) == Seq("08:00:2b:01:02:03:04:05"))
+    assert(rows(0).getSeq(26) == Seq("10:20:10,14,15"))
   }
 
   test("query JDBC option") {
@@ -218,4 +347,36 @@ class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite {
     assert(rows(0).getShort(0) === 1)
     assert(rows(0).getShort(1) === 2)
   }
+
+  test("character type tests") {
+    val df = sqlContext.read.jdbc(jdbcUrl, "char_types", new Properties)
+    val row = df.collect()
+    assert(row.length == 1)
+    assert(row(0).length === 5)
+    assert(row(0).getString(0) === "abcd")
+    assert(row(0).getString(1) === "efgh")
+    assert(row(0).getString(2) === "ijkl")
+    assert(row(0).getString(3) === "mnop")
+    assert(row(0).getString(4) === "q")
+  }
+
+  test("SPARK-32576: character array type tests") {
+    val df = sqlContext.read.jdbc(jdbcUrl, "char_array_types", new Properties)
+    val row = df.collect()
+    assert(row.length == 1)
+    assert(row(0).length === 5)
+    assert(row(0).getSeq[String](0) === Seq("a   ", "bcd "))
+    assert(row(0).getSeq[String](1) === Seq("ef  ", "gh  "))
+    assert(row(0).getSeq[String](2) === Seq("i", "j", "kl"))
+    assert(row(0).getSeq[String](3) === Seq("mnop"))
+    assert(row(0).getSeq[String](4) === Seq("q", "r"))
+  }
+
+  test("SPARK-34333: money type tests") {
+    val df = sqlContext.read.jdbc(jdbcUrl, "money_types", new Properties)
+    val row = df.collect()
+    assert(row.length === 1)
+    assert(row(0).length === 1)
+    assert(row(0).getString(0) === "$1,000.00")
+  }
 }
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresKrbIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresKrbIntegrationSuite.scala
index e94bf3dd588aa..1198ba8a3e465 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresKrbIntegrationSuite.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresKrbIntegrationSuite.scala
@@ -24,13 +24,20 @@ import com.spotify.docker.client.messages.{ContainerConfig, HostConfig}
 import org.apache.spark.sql.execution.datasources.jdbc.connection.SecureConnectionProvider
 import org.apache.spark.tags.DockerTest
 
+/**
+ * To run this test suite for a specific version (e.g., postgres:13.0):
+ * {{{
+ *   POSTGRES_DOCKER_IMAGE_NAME=postgres:13.0
+ *     ./build/sbt -Pdocker-integration-tests "testOnly *PostgresKrbIntegrationSuite"
+ * }}}
+ */
 @DockerTest
 class PostgresKrbIntegrationSuite extends DockerKrbJDBCIntegrationSuite {
   override protected val userName = s"postgres/$dockerIp"
   override protected val keytabFileName = "postgres.keytab"
 
   override val db = new DatabaseOnDocker {
-    override val imageName = "postgres:12.0"
+    override val imageName = sys.env.getOrElse("POSTGRES_DOCKER_IMAGE_NAME", "postgres:13.0")
     override val env = Map(
       "POSTGRES_PASSWORD" -> "rootpass"
     )
@@ -54,7 +61,7 @@ class PostgresKrbIntegrationSuite extends DockerKrbJDBCIntegrationSuite {
 
   override protected def setAuthentication(keytabFile: String, principal: String): Unit = {
     val config = new SecureConnectionProvider.JDBCConfiguration(
-      Configuration.getConfiguration, "pgjdbc", keytabFile, principal)
+      Configuration.getConfiguration, "pgjdbc", keytabFile, principal, true)
     Configuration.setConfiguration(config)
   }
 }
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DB2IntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DB2IntegrationSuite.scala
new file mode 100644
index 0000000000000..3b8008aca1c93
--- /dev/null
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DB2IntegrationSuite.scala
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc.v2
+
+import java.sql.Connection
+
+import org.scalatest.time.SpanSugar._
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTableCatalog
+import org.apache.spark.sql.jdbc.{DatabaseOnDocker, DockerJDBCIntegrationSuite}
+import org.apache.spark.sql.types._
+import org.apache.spark.tags.DockerTest
+
+/**
+ * To run this test suite for a specific version (e.g., ibmcom/db2:11.5.4.0):
+ * {{{
+ *   DB2_DOCKER_IMAGE_NAME=ibmcom/db2:11.5.4.0
+ *     ./build/sbt -Pdocker-integration-tests "testOnly *v2.DB2IntegrationSuite"
+ * }}}
+ */
+@DockerTest
+class DB2IntegrationSuite extends DockerJDBCIntegrationSuite with V2JDBCTest {
+  override val catalogName: String = "db2"
+  override val db = new DatabaseOnDocker {
+    override val imageName = sys.env.getOrElse("DB2_DOCKER_IMAGE_NAME", "ibmcom/db2:11.5.4.0")
+    override val env = Map(
+      "DB2INST1_PASSWORD" -> "rootpass",
+      "LICENSE" -> "accept",
+      "DBNAME" -> "foo",
+      "ARCHIVE_LOGS" -> "false",
+      "AUTOCONFIG" -> "false"
+    )
+    override val usesIpc = false
+    override val jdbcPort: Int = 50000
+    override val privileged = true
+    override def getJdbcUrl(ip: String, port: Int): String =
+      s"jdbc:db2://$ip:$port/foo:user=db2inst1;password=rootpass;retrieveMessagesFromServerOnGetMessage=true;" //scalastyle:ignore
+  }
+
+  override val connectionTimeout = timeout(3.minutes)
+
+  override def sparkConf: SparkConf = super.sparkConf
+    .set("spark.sql.catalog.db2", classOf[JDBCTableCatalog].getName)
+    .set("spark.sql.catalog.db2.url", db.getJdbcUrl(dockerIp, externalPort))
+
+  override def dataPreparation(conn: Connection): Unit = {}
+
+  override def testUpdateColumnType(tbl: String): Unit = {
+    sql(s"CREATE TABLE $tbl (ID INTEGER)")
+    var t = spark.table(tbl)
+    var expectedSchema = new StructType().add("ID", IntegerType, true, defaultMetadata)
+    assert(t.schema === expectedSchema)
+    sql(s"ALTER TABLE $tbl ALTER COLUMN id TYPE DOUBLE")
+    t = spark.table(tbl)
+    expectedSchema = new StructType().add("ID", DoubleType, true, defaultMetadata)
+    assert(t.schema === expectedSchema)
+    // Update column type from DOUBLE to STRING
+    val msg1 = intercept[AnalysisException] {
+      sql(s"ALTER TABLE $tbl ALTER COLUMN id TYPE VARCHAR(10)")
+    }.getMessage
+    assert(msg1.contains("Cannot update alt_table field ID: double cannot be cast to varchar"))
+  }
+
+  override def testCreateTableWithProperty(tbl: String): Unit = {
+    sql(s"CREATE TABLE $tbl (ID INT)" +
+      s" TBLPROPERTIES('CCSID'='UNICODE')")
+    val t = spark.table(tbl)
+    val expectedSchema = new StructType().add("ID", IntegerType, true, defaultMetadata)
+    assert(t.schema === expectedSchema)
+  }
+}
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MsSqlServerIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MsSqlServerIntegrationSuite.scala
new file mode 100644
index 0000000000000..a756516457d16
--- /dev/null
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MsSqlServerIntegrationSuite.scala
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc.v2
+
+import java.sql.{Connection, SQLFeatureNotSupportedException}
+
+import org.scalatest.time.SpanSugar._
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTableCatalog
+import org.apache.spark.sql.jdbc.{DatabaseOnDocker, DockerJDBCIntegrationSuite}
+import org.apache.spark.sql.types._
+import org.apache.spark.tags.DockerTest
+
+/**
+ * To run this test suite for a specific version (e.g., 2019-GA-ubuntu-16.04):
+ * {{{
+ *   MSSQLSERVER_DOCKER_IMAGE_NAME=2019-GA-ubuntu-16.04
+ *     ./build/sbt -Pdocker-integration-tests "testOnly *v2*MsSqlServerIntegrationSuite"
+ * }}}
+ */
+@DockerTest
+class MsSqlServerIntegrationSuite extends DockerJDBCIntegrationSuite with V2JDBCTest {
+
+  override val catalogName: String = "mssql"
+
+  override val db = new DatabaseOnDocker {
+    override val imageName = sys.env.getOrElse("MSSQLSERVER_DOCKER_IMAGE_NAME",
+      "mcr.microsoft.com/mssql/server:2019-GA-ubuntu-16.04")
+    override val env = Map(
+      "SA_PASSWORD" -> "Sapass123",
+      "ACCEPT_EULA" -> "Y"
+    )
+    override val usesIpc = false
+    override val jdbcPort: Int = 1433
+
+    override def getJdbcUrl(ip: String, port: Int): String =
+      s"jdbc:sqlserver://$ip:$port;user=sa;password=Sapass123;"
+  }
+
+  override def sparkConf: SparkConf = super.sparkConf
+    .set("spark.sql.catalog.mssql", classOf[JDBCTableCatalog].getName)
+    .set("spark.sql.catalog.mssql.url", db.getJdbcUrl(dockerIp, externalPort))
+
+  override val connectionTimeout = timeout(7.minutes)
+
+  override def dataPreparation(conn: Connection): Unit = {}
+
+  override def notSupportsTableComment: Boolean = true
+
+  override def testUpdateColumnType(tbl: String): Unit = {
+    sql(s"CREATE TABLE $tbl (ID INTEGER)")
+    var t = spark.table(tbl)
+    var expectedSchema = new StructType().add("ID", IntegerType, true, defaultMetadata)
+    assert(t.schema === expectedSchema)
+    sql(s"ALTER TABLE $tbl ALTER COLUMN id TYPE STRING")
+    t = spark.table(tbl)
+    expectedSchema = new StructType().add("ID", StringType, true, defaultMetadata)
+    assert(t.schema === expectedSchema)
+    // Update column type from STRING to INTEGER
+    val msg1 = intercept[AnalysisException] {
+      sql(s"ALTER TABLE $tbl ALTER COLUMN id TYPE INTEGER")
+    }.getMessage
+    assert(msg1.contains("Cannot update alt_table field ID: string cannot be cast to int"))
+  }
+
+  override def testUpdateColumnNullability(tbl: String): Unit = {
+    sql(s"CREATE TABLE $tbl (ID STRING NOT NULL)")
+    // Update nullability is unsupported for mssql db.
+    val msg = intercept[AnalysisException] {
+      sql(s"ALTER TABLE $tbl ALTER COLUMN ID DROP NOT NULL")
+    }.getCause.asInstanceOf[SQLFeatureNotSupportedException].getMessage
+
+    assert(msg.contains("UpdateColumnNullability is not supported"))
+  }
+}
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala
new file mode 100644
index 0000000000000..a567ab3b82362
--- /dev/null
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc.v2
+
+import java.sql.{Connection, SQLFeatureNotSupportedException}
+
+import org.scalatest.time.SpanSugar._
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTableCatalog
+import org.apache.spark.sql.jdbc.{DatabaseOnDocker, DockerJDBCIntegrationSuite}
+import org.apache.spark.sql.types._
+import org.apache.spark.tags.DockerTest
+
+/**
+ *
+ * To run this test suite for a specific version (e.g., mysql:5.7.31):
+ * {{{
+ * MYSQL_DOCKER_IMAGE_NAME=mysql:5.7.31
+ *         ./build/sbt -Pdocker-integration-tests "testOnly *v2*MySQLIntegrationSuite"
+ *
+ * }}}
+ *
+ */
+@DockerTest
+class MySQLIntegrationSuite extends DockerJDBCIntegrationSuite with V2JDBCTest {
+  override val catalogName: String = "mysql"
+  override val db = new DatabaseOnDocker {
+    override val imageName = sys.env.getOrElse("MYSQL_DOCKER_IMAGE_NAME", "mysql:5.7.31")
+    override val env = Map(
+      "MYSQL_ROOT_PASSWORD" -> "rootpass"
+    )
+    override val usesIpc = false
+    override val jdbcPort: Int = 3306
+
+    override def getJdbcUrl(ip: String, port: Int): String =
+      s"jdbc:mysql://$ip:$port/" +
+        s"mysql?user=root&password=rootpass&allowPublicKeyRetrieval=true&useSSL=false"
+  }
+
+  override def sparkConf: SparkConf = super.sparkConf
+    .set("spark.sql.catalog.mysql", classOf[JDBCTableCatalog].getName)
+    .set("spark.sql.catalog.mysql.url", db.getJdbcUrl(dockerIp, externalPort))
+
+  override val connectionTimeout = timeout(7.minutes)
+
+  private var mySQLVersion = -1
+
+  override def dataPreparation(conn: Connection): Unit = {
+    mySQLVersion = conn.getMetaData.getDatabaseMajorVersion
+  }
+
+  override def testUpdateColumnType(tbl: String): Unit = {
+    sql(s"CREATE TABLE $tbl (ID INTEGER)")
+    var t = spark.table(tbl)
+    var expectedSchema = new StructType().add("ID", IntegerType, true, defaultMetadata)
+    assert(t.schema === expectedSchema)
+    sql(s"ALTER TABLE $tbl ALTER COLUMN id TYPE STRING")
+    t = spark.table(tbl)
+    expectedSchema = new StructType().add("ID", StringType, true, defaultMetadata)
+    assert(t.schema === expectedSchema)
+    // Update column type from STRING to INTEGER
+    val msg1 = intercept[AnalysisException] {
+      sql(s"ALTER TABLE $tbl ALTER COLUMN id TYPE INTEGER")
+    }.getMessage
+    assert(msg1.contains("Cannot update alt_table field ID: string cannot be cast to int"))
+  }
+
+  override def testRenameColumn(tbl: String): Unit = {
+    assert(mySQLVersion > 0)
+    if (mySQLVersion < 8) {
+      // Rename is unsupported for mysql versions < 8.0.
+      val exception = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $tbl RENAME COLUMN ID TO RENAMED")
+      }
+      assert(exception.getCause != null, s"Wrong exception thrown: $exception")
+      val msg = exception.getCause.asInstanceOf[SQLFeatureNotSupportedException].getMessage
+      assert(msg.contains("Rename column is only supported for MySQL version 8.0 and above."))
+    } else {
+      super.testRenameColumn(tbl)
+    }
+  }
+
+  override def testUpdateColumnNullability(tbl: String): Unit = {
+    sql(s"CREATE TABLE $tbl (ID STRING NOT NULL)")
+    // Update nullability is unsupported for mysql db.
+    val msg = intercept[AnalysisException] {
+      sql(s"ALTER TABLE $tbl ALTER COLUMN ID DROP NOT NULL")
+    }.getCause.asInstanceOf[SQLFeatureNotSupportedException].getMessage
+
+    assert(msg.contains("UpdateColumnNullability is not supported"))
+  }
+
+  override def testCreateTableWithProperty(tbl: String): Unit = {
+    sql(s"CREATE TABLE $tbl (ID INT)" +
+      s" TBLPROPERTIES('ENGINE'='InnoDB', 'DEFAULT CHARACTER SET'='utf8')")
+    val t = spark.table(tbl)
+    val expectedSchema = new StructType().add("ID", IntegerType, true, defaultMetadata)
+    assert(t.schema === expectedSchema)
+  }
+}
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleIntegrationSuite.scala
new file mode 100644
index 0000000000000..84b952937d0c1
--- /dev/null
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleIntegrationSuite.scala
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc.v2
+
+import java.sql.Connection
+
+import org.scalatest.time.SpanSugar._
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTableCatalog
+import org.apache.spark.sql.jdbc.{DatabaseOnDocker, DockerJDBCIntegrationSuite}
+import org.apache.spark.sql.types._
+import org.apache.spark.tags.DockerTest
+
+/**
+ * The following would be the steps to test this
+ * 1. Build Oracle database in Docker, please refer below link about how to.
+ *    https://github.com/oracle/docker-images/blob/master/OracleDatabase/SingleInstance/README.md
+ * 2. export ORACLE_DOCKER_IMAGE_NAME=$ORACLE_DOCKER_IMAGE_NAME
+ *    Pull oracle $ORACLE_DOCKER_IMAGE_NAME image - docker pull $ORACLE_DOCKER_IMAGE_NAME
+ * 3. Start docker - sudo service docker start
+ * 4. Run spark test - ./build/sbt -Pdocker-integration-tests
+ *    "testOnly org.apache.spark.sql.jdbc.v2.OracleIntegrationSuite"
+ *
+ * An actual sequence of commands to run the test is as follows
+ *
+ *  $ git clone https://github.com/oracle/docker-images.git
+ *  // Head SHA: 3e352a22618070595f823977a0fd1a3a8071a83c
+ *  $ cd docker-images/OracleDatabase/SingleInstance/dockerfiles
+ *  $ ./buildDockerImage.sh -v 18.4.0 -x
+ *  $ export ORACLE_DOCKER_IMAGE_NAME=oracle/database:18.4.0-xe
+ *  $ cd $SPARK_HOME
+ *  $ ./build/sbt -Pdocker-integration-tests
+ *    "testOnly org.apache.spark.sql.jdbc.v2.OracleIntegrationSuite"
+ *
+ * It has been validated with 18.4.0 Express Edition.
+ */
+@DockerTest
+class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with V2JDBCTest {
+  override val catalogName: String = "oracle"
+  override val db = new DatabaseOnDocker {
+    override val imageName = sys.env("ORACLE_DOCKER_IMAGE_NAME")
+    override val env = Map(
+      "ORACLE_PWD" -> "oracle"
+    )
+    override val usesIpc = false
+    override val jdbcPort: Int = 1521
+    override def getJdbcUrl(ip: String, port: Int): String =
+      s"jdbc:oracle:thin:system/oracle@//$ip:$port/xe"
+  }
+
+  override def sparkConf: SparkConf = super.sparkConf
+    .set("spark.sql.catalog.oracle", classOf[JDBCTableCatalog].getName)
+    .set("spark.sql.catalog.oracle.url", db.getJdbcUrl(dockerIp, externalPort))
+
+  override val connectionTimeout = timeout(7.minutes)
+  override def dataPreparation(conn: Connection): Unit = {}
+
+  override def testUpdateColumnType(tbl: String): Unit = {
+    sql(s"CREATE TABLE $tbl (ID INTEGER)")
+    var t = spark.table(tbl)
+    var expectedSchema = new StructType().add("ID", DecimalType(10, 0), true, defaultMetadata)
+    assert(t.schema === expectedSchema)
+    sql(s"ALTER TABLE $tbl ALTER COLUMN id TYPE STRING")
+    t = spark.table(tbl)
+    expectedSchema = new StructType().add("ID", StringType, true, defaultMetadata)
+    assert(t.schema === expectedSchema)
+    // Update column type from STRING to INTEGER
+    val msg1 = intercept[AnalysisException] {
+      sql(s"ALTER TABLE $tbl ALTER COLUMN id TYPE INTEGER")
+    }.getMessage
+    assert(msg1.contains("Cannot update alt_table field ID: string cannot be cast to int"))
+  }
+}
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/PostgresIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/PostgresIntegrationSuite.scala
new file mode 100644
index 0000000000000..eded03afdaa32
--- /dev/null
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/PostgresIntegrationSuite.scala
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc.v2
+
+import java.sql.Connection
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTableCatalog
+import org.apache.spark.sql.jdbc.{DatabaseOnDocker, DockerJDBCIntegrationSuite}
+import org.apache.spark.sql.types._
+import org.apache.spark.tags.DockerTest
+
+/**
+ * To run this test suite for a specific version (e.g., postgres:13.0):
+ * {{{
+ *   POSTGRES_DOCKER_IMAGE_NAME=postgres:13.0
+ *     ./build/sbt -Pdocker-integration-tests "testOnly *v2.PostgresIntegrationSuite"
+ * }}}
+ */
+@DockerTest
+class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite with V2JDBCTest {
+  override val catalogName: String = "postgresql"
+  override val db = new DatabaseOnDocker {
+    override val imageName = sys.env.getOrElse("POSTGRES_DOCKER_IMAGE_NAME", "postgres:13.0-alpine")
+    override val env = Map(
+      "POSTGRES_PASSWORD" -> "rootpass"
+    )
+    override val usesIpc = false
+    override val jdbcPort = 5432
+    override def getJdbcUrl(ip: String, port: Int): String =
+      s"jdbc:postgresql://$ip:$port/postgres?user=postgres&password=rootpass"
+  }
+  override def sparkConf: SparkConf = super.sparkConf
+    .set("spark.sql.catalog.postgresql", classOf[JDBCTableCatalog].getName)
+    .set("spark.sql.catalog.postgresql.url", db.getJdbcUrl(dockerIp, externalPort))
+  override def dataPreparation(conn: Connection): Unit = {}
+
+  override def testUpdateColumnType(tbl: String): Unit = {
+    sql(s"CREATE TABLE $tbl (ID INTEGER)")
+    var t = spark.table(tbl)
+    var expectedSchema = new StructType().add("ID", IntegerType, true, defaultMetadata)
+    assert(t.schema === expectedSchema)
+    sql(s"ALTER TABLE $tbl ALTER COLUMN id TYPE STRING")
+    t = spark.table(tbl)
+    expectedSchema = new StructType().add("ID", StringType, true, defaultMetadata)
+    assert(t.schema === expectedSchema)
+    // Update column type from STRING to INTEGER
+    val msg = intercept[AnalysisException] {
+      sql(s"ALTER TABLE $tbl ALTER COLUMN id TYPE INTEGER")
+    }.getMessage
+    assert(msg.contains("Cannot update alt_table field ID: string cannot be cast to int"))
+  }
+
+  override def testCreateTableWithProperty(tbl: String): Unit = {
+    sql(s"CREATE TABLE $tbl (ID INT)" +
+      s" TBLPROPERTIES('TABLESPACE'='pg_default')")
+    val t = spark.table(tbl)
+    val expectedSchema = new StructType().add("ID", IntegerType, true, defaultMetadata)
+    assert(t.schema === expectedSchema)
+  }
+}
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/PostgresNamespaceSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/PostgresNamespaceSuite.scala
new file mode 100644
index 0000000000000..e534df84ce6fa
--- /dev/null
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/PostgresNamespaceSuite.scala
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc.v2
+
+import java.sql.Connection
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.sql.jdbc.{DatabaseOnDocker, DockerJDBCIntegrationSuite}
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
+import org.apache.spark.tags.DockerTest
+
+/**
+ * To run this test suite for a specific version (e.g., postgres:13.0):
+ * {{{
+ *   POSTGRES_DOCKER_IMAGE_NAME=postgres:13.0
+ *     ./build/sbt -Pdocker-integration-tests "testOnly *v2.PostgresNamespaceSuite"
+ * }}}
+ */
+@DockerTest
+class PostgresNamespaceSuite extends DockerJDBCIntegrationSuite with V2JDBCNamespaceTest {
+  override val db = new DatabaseOnDocker {
+    override val imageName = sys.env.getOrElse("POSTGRES_DOCKER_IMAGE_NAME", "postgres:13.0-alpine")
+    override val env = Map(
+      "POSTGRES_PASSWORD" -> "rootpass"
+    )
+    override val usesIpc = false
+    override val jdbcPort = 5432
+    override def getJdbcUrl(ip: String, port: Int): String =
+      s"jdbc:postgresql://$ip:$port/postgres?user=postgres&password=rootpass"
+  }
+
+  val map = new CaseInsensitiveStringMap(
+    Map("url" -> db.getJdbcUrl(dockerIp, externalPort),
+      "driver" -> "org.postgresql.Driver").asJava)
+
+  catalog.initialize("postgresql", map)
+
+  override def dataPreparation(conn: Connection): Unit = {}
+
+  override def builtinNamespaces: Array[Array[String]] = {
+    Array(Array("information_schema"), Array("pg_catalog"), Array("public"))
+  }
+}
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCNamespaceTest.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCNamespaceTest.scala
new file mode 100644
index 0000000000000..979b0784f0448
--- /dev/null
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCNamespaceTest.scala
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc.v2
+
+import scala.collection.JavaConverters._
+
+import org.apache.log4j.Level
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.connector.catalog.NamespaceChange
+import org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTableCatalog
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.tags.DockerTest
+
+@DockerTest
+private[v2] trait V2JDBCNamespaceTest extends SharedSparkSession {
+  val catalog = new JDBCTableCatalog()
+
+  def builtinNamespaces: Array[Array[String]]
+
+  test("listNamespaces: basic behavior") {
+    catalog.createNamespace(Array("foo"), Map("comment" -> "test comment").asJava)
+    assert(catalog.listNamespaces() === Array(Array("foo")) ++ builtinNamespaces)
+    assert(catalog.listNamespaces(Array("foo")) === Array())
+    assert(catalog.namespaceExists(Array("foo")) === true)
+
+    val logAppender = new LogAppender("catalog comment")
+    withLogAppender(logAppender) {
+      catalog.alterNamespace(Array("foo"), NamespaceChange
+        .setProperty("comment", "comment for foo"))
+      catalog.alterNamespace(Array("foo"), NamespaceChange.removeProperty("comment"))
+    }
+    val createCommentWarning = logAppender.loggingEvents
+      .filter(_.getLevel == Level.WARN)
+      .map(_.getRenderedMessage)
+      .exists(_.contains("catalog comment"))
+    assert(createCommentWarning === false)
+
+    catalog.dropNamespace(Array("foo"))
+    assert(catalog.namespaceExists(Array("foo")) === false)
+    assert(catalog.listNamespaces() === builtinNamespaces)
+    val msg = intercept[AnalysisException] {
+      catalog.listNamespaces(Array("foo"))
+    }.getMessage
+    assert(msg.contains("Namespace 'foo' not found"))
+  }
+}
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala
new file mode 100644
index 0000000000000..3807eb732a1a5
--- /dev/null
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc.v2
+
+import org.apache.log4j.Level
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types._
+import org.apache.spark.tags.DockerTest
+
+@DockerTest
+private[v2] trait V2JDBCTest extends SharedSparkSession {
+  val catalogName: String
+  // dialect specific update column type test
+  def testUpdateColumnType(tbl: String): Unit
+
+  def notSupportsTableComment: Boolean = false
+
+  val defaultMetadata = new MetadataBuilder().putLong("scale", 0).build()
+
+  def testUpdateColumnNullability(tbl: String): Unit = {
+    sql(s"CREATE TABLE $catalogName.alt_table (ID STRING NOT NULL)")
+    var t = spark.table(s"$catalogName.alt_table")
+    // nullable is true in the expectedSchema because Spark always sets nullable to true
+    // regardless of the JDBC metadata https://github.com/apache/spark/pull/18445
+    var expectedSchema = new StructType().add("ID", StringType, true, defaultMetadata)
+    assert(t.schema === expectedSchema)
+    sql(s"ALTER TABLE $catalogName.alt_table ALTER COLUMN ID DROP NOT NULL")
+    t = spark.table(s"$catalogName.alt_table")
+    expectedSchema = new StructType().add("ID", StringType, true, defaultMetadata)
+    assert(t.schema === expectedSchema)
+    // Update nullability of not existing column
+    val msg = intercept[AnalysisException] {
+      sql(s"ALTER TABLE $catalogName.alt_table ALTER COLUMN bad_column DROP NOT NULL")
+    }.getMessage
+    assert(msg.contains("Cannot update missing field bad_column"))
+  }
+
+  def testRenameColumn(tbl: String): Unit = {
+    sql(s"ALTER TABLE $tbl RENAME COLUMN ID TO RENAMED")
+    val t = spark.table(s"$tbl")
+    val expectedSchema = new StructType().add("RENAMED", StringType, true, defaultMetadata)
+      .add("ID1", StringType, true, defaultMetadata).add("ID2", StringType, true, defaultMetadata)
+    assert(t.schema === expectedSchema)
+  }
+
+  def testCreateTableWithProperty(tbl: String): Unit = {}
+
+  test("SPARK-33034: ALTER TABLE ... add new columns") {
+    withTable(s"$catalogName.alt_table") {
+      sql(s"CREATE TABLE $catalogName.alt_table (ID STRING)")
+      var t = spark.table(s"$catalogName.alt_table")
+      var expectedSchema = new StructType().add("ID", StringType, true, defaultMetadata)
+      assert(t.schema === expectedSchema)
+      sql(s"ALTER TABLE $catalogName.alt_table ADD COLUMNS (C1 STRING, C2 STRING)")
+      t = spark.table(s"$catalogName.alt_table")
+      expectedSchema = expectedSchema.add("C1", StringType, true, defaultMetadata)
+        .add("C2", StringType, true, defaultMetadata)
+      assert(t.schema === expectedSchema)
+      sql(s"ALTER TABLE $catalogName.alt_table ADD COLUMNS (C3 STRING)")
+      t = spark.table(s"$catalogName.alt_table")
+      expectedSchema = expectedSchema.add("C3", StringType, true, defaultMetadata)
+      assert(t.schema === expectedSchema)
+      // Add already existing column
+      val msg = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $catalogName.alt_table ADD COLUMNS (C3 DOUBLE)")
+      }.getMessage
+      assert(msg.contains("Cannot add column, because C3 already exists"))
+    }
+    // Add a column to not existing table
+    val msg = intercept[AnalysisException] {
+      sql(s"ALTER TABLE $catalogName.not_existing_table ADD COLUMNS (C4 STRING)")
+    }.getMessage
+    assert(msg.contains("Table not found"))
+  }
+
+  test("SPARK-33034: ALTER TABLE ... drop column") {
+    withTable(s"$catalogName.alt_table") {
+      sql(s"CREATE TABLE $catalogName.alt_table (C1 INTEGER, C2 STRING, c3 INTEGER)")
+      sql(s"ALTER TABLE $catalogName.alt_table DROP COLUMN C1")
+      sql(s"ALTER TABLE $catalogName.alt_table DROP COLUMN c3")
+      val t = spark.table(s"$catalogName.alt_table")
+      val expectedSchema = new StructType().add("C2", StringType, true, defaultMetadata)
+      assert(t.schema === expectedSchema)
+      // Drop not existing column
+      val msg = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $catalogName.alt_table DROP COLUMN bad_column")
+      }.getMessage
+      assert(msg.contains("Cannot delete missing field bad_column in alt_table schema"))
+    }
+    // Drop a column from a not existing table
+    val msg = intercept[AnalysisException] {
+      sql(s"ALTER TABLE $catalogName.not_existing_table DROP COLUMN C1")
+    }.getMessage
+    assert(msg.contains("Table not found"))
+  }
+
+  test("SPARK-33034: ALTER TABLE ... update column type") {
+    withTable(s"$catalogName.alt_table") {
+      testUpdateColumnType(s"$catalogName.alt_table")
+      // Update not existing column
+      val msg2 = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $catalogName.alt_table ALTER COLUMN bad_column TYPE DOUBLE")
+      }.getMessage
+      assert(msg2.contains("Cannot update missing field bad_column"))
+    }
+    // Update column type in not existing table
+    val msg = intercept[AnalysisException] {
+      sql(s"ALTER TABLE $catalogName.not_existing_table ALTER COLUMN id TYPE DOUBLE")
+    }.getMessage
+    assert(msg.contains("Table not found"))
+  }
+
+  test("SPARK-33034: ALTER TABLE ... rename column") {
+    withTable(s"$catalogName.alt_table") {
+      sql(s"CREATE TABLE $catalogName.alt_table (ID STRING NOT NULL," +
+        s" ID1 STRING NOT NULL, ID2 STRING NOT NULL)")
+      testRenameColumn(s"$catalogName.alt_table")
+      // Rename to already existing column
+      val msg = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $catalogName.alt_table RENAME COLUMN ID1 TO ID2")
+      }.getMessage
+      assert(msg.contains("Cannot rename column, because ID2 already exists"))
+    }
+    // Rename a column in a not existing table
+    val msg = intercept[AnalysisException] {
+      sql(s"ALTER TABLE $catalogName.not_existing_table RENAME COLUMN ID TO C")
+    }.getMessage
+    assert(msg.contains("Table not found"))
+  }
+
+  test("SPARK-33034: ALTER TABLE ... update column nullability") {
+    withTable(s"$catalogName.alt_table") {
+      testUpdateColumnNullability(s"$catalogName.alt_table")
+    }
+    // Update column nullability in not existing table
+    val msg = intercept[AnalysisException] {
+      sql(s"ALTER TABLE $catalogName.not_existing_table ALTER COLUMN ID DROP NOT NULL")
+    }.getMessage
+    assert(msg.contains("Table not found"))
+  }
+
+  test("CREATE TABLE with table comment") {
+    withTable(s"$catalogName.new_table") {
+      val logAppender = new LogAppender("table comment")
+      withLogAppender(logAppender) {
+        sql(s"CREATE TABLE $catalogName.new_table(i INT) COMMENT 'this is a comment'")
+      }
+      val createCommentWarning = logAppender.loggingEvents
+        .filter(_.getLevel == Level.WARN)
+        .map(_.getRenderedMessage)
+        .exists(_.contains("Cannot create JDBC table comment"))
+      assert(createCommentWarning === notSupportsTableComment)
+    }
+  }
+
+  test("CREATE TABLE with table property") {
+    withTable(s"$catalogName.new_table") {
+      val m = intercept[AnalysisException] {
+        sql(s"CREATE TABLE $catalogName.new_table (i INT) TBLPROPERTIES('a'='1')")
+      }.message
+      assert(m.contains("Failed table creation"))
+      testCreateTableWithProperty(s"$catalogName.new_table")
+    }
+  }
+}
+
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/util/DockerUtils.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/util/DockerUtils.scala
index fda377e032350..5abca8df77dcd 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/util/DockerUtils.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/util/DockerUtils.scala
@@ -36,7 +36,7 @@ private[spark] object DockerUtils {
       .orElse(findFromDockerMachine())
       .orElse(Try(Seq("/bin/bash", "-c", "boot2docker ip 2>/dev/null").!!.trim).toOption)
       .getOrElse {
-        // This block of code is based on Utils.findLocalInetAddress(), but is modified to blacklist
+        // This block of code is based on Utils.findLocalInetAddress(), but is modified to exclude
         // certain interfaces.
         val address = InetAddress.getLocalHost
         // Address resolves to something like 127.0.1.1, which happens on Debian; try to find
@@ -44,12 +44,12 @@ private[spark] object DockerUtils {
         // getNetworkInterfaces returns ifs in reverse order compared to ifconfig output order
         // on unix-like system. On windows, it returns in index order.
         // It's more proper to pick ip address following system output order.
-        val blackListedIFs = Seq(
+        val excludedIFs = Seq(
           "vboxnet0",  // Mac
           "docker0"    // Linux
         )
         val activeNetworkIFs = NetworkInterface.getNetworkInterfaces.asScala.toSeq.filter { i =>
-          !blackListedIFs.contains(i.getName)
+          !excludedIFs.contains(i.getName)
         }
         val reOrderedNetworkIFs = activeNetworkIFs.reverse
         for (ni <- reOrderedNetworkIFs) {
diff --git a/external/docker/spark-test/base/Dockerfile b/external/docker/spark-test/base/Dockerfile
index 5bec5d3f16548..d4a30c4681cba 100644
--- a/external/docker/spark-test/base/Dockerfile
+++ b/external/docker/spark-test/base/Dockerfile
@@ -15,23 +15,14 @@
 # limitations under the License.
 #
 
-FROM ubuntu:xenial
+FROM ubuntu:20.04
 
 # Upgrade package index
-# install a few other useful packages plus Open Jdk 8
+# install a few other useful packages plus Open Java 11
 # Remove unneeded /var/lib/apt/lists/* after install to reduce the
 # docker image size (by ~30MB)
 RUN apt-get update && \
-    apt-get install -y less openjdk-8-jre-headless iproute2 vim-tiny sudo openssh-server && \
+    apt-get install -y less openjdk-11-jre-headless iproute2 vim-tiny sudo openssh-server && \
     rm -rf /var/lib/apt/lists/*
 
-ENV SCALA_VERSION 2.12.10
-ENV CDH_VERSION cdh4
-ENV SCALA_HOME /opt/scala-$SCALA_VERSION
 ENV SPARK_HOME /opt/spark
-ENV PATH $SPARK_HOME:$SCALA_HOME/bin:$PATH
-
-# Install Scala
-ADD https://www.scala-lang.org/files/archive/scala-$SCALA_VERSION.tgz /
-RUN (cd / && gunzip < scala-$SCALA_VERSION.tgz)|(cd /opt && tar -xvf -)
-RUN rm /scala-$SCALA_VERSION.tgz
diff --git a/external/docker/spark-test/master/default_cmd b/external/docker/spark-test/master/default_cmd
index 5a7da3446f6d2..96a36cd0bb682 100755
--- a/external/docker/spark-test/master/default_cmd
+++ b/external/docker/spark-test/master/default_cmd
@@ -22,7 +22,4 @@ echo "CONTAINER_IP=$IP"
 export SPARK_LOCAL_IP=$IP
 export SPARK_PUBLIC_DNS=$IP
 
-# Avoid the default Docker behavior of mapping our IP address to an unreachable host name
-umount /etc/hosts
-
 /opt/spark/bin/spark-class org.apache.spark.deploy.master.Master -i $IP
diff --git a/external/docker/spark-test/worker/default_cmd b/external/docker/spark-test/worker/default_cmd
index 31b06cb0eb047..2401f5565aa0b 100755
--- a/external/docker/spark-test/worker/default_cmd
+++ b/external/docker/spark-test/worker/default_cmd
@@ -22,7 +22,4 @@ echo "CONTAINER_IP=$IP"
 export SPARK_LOCAL_IP=$IP
 export SPARK_PUBLIC_DNS=$IP
 
-# Avoid the default Docker behavior of mapping our IP address to an unreachable host name
-umount /etc/hosts
-
 /opt/spark/bin/spark-class org.apache.spark.deploy.worker.Worker $1
diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml
index d9d9fb7f55c77..bc5c77f3c18c7 100644
--- a/external/kafka-0-10-assembly/pom.xml
+++ b/external/kafka-0-10-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
@@ -71,13 +71,18 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-client</artifactId>
+      <artifactId>${hadoop-client-api.artifact}</artifactId>
+      <version>${hadoop.version}</version>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>${hadoop-client-runtime.artifact}</artifactId>
+      <version>${hadoop.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro-mapred</artifactId>
-      <classifier>${avro.mapred.classifier}</classifier>
       <scope>provided</scope>
     </dependency>
     <dependency>
diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml
index 95a99ac88412e..3ca0c964de937 100644
--- a/external/kafka-0-10-sql/pom.xml
+++ b/external/kafka-0-10-sql/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
@@ -78,6 +78,16 @@
       <groupId>org.apache.kafka</groupId>
       <artifactId>kafka-clients</artifactId>
       <version>${kafka.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>com.github.luben</groupId>
+          <artifactId>zstd-jni</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>com.google.code.findbugs</groupId>
+      <artifactId>jsr305</artifactId>
     </dependency>
     <dependency>
       <groupId>org.apache.commons</groupId>
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/ConsumerStrategy.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/ConsumerStrategy.scala
index 7bb829c282eba..a0331d7889e04 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/ConsumerStrategy.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/ConsumerStrategy.scala
@@ -20,12 +20,15 @@ package org.apache.spark.sql.kafka010
 import java.{util => ju}
 
 import scala.collection.JavaConverters._
+import scala.collection.mutable
 
+import org.apache.kafka.clients.admin.Admin
 import org.apache.kafka.clients.consumer.{Consumer, KafkaConsumer}
 import org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener
 import org.apache.kafka.common.TopicPartition
 
-import org.apache.spark.kafka010.KafkaConfigUpdater
+import org.apache.spark.internal.Logging
+import org.apache.spark.kafka010.{KafkaConfigUpdater, KafkaRedactionUtil}
 
 /**
  * Subscribe allows you to subscribe to a fixed collection of topics.
@@ -36,10 +39,20 @@ import org.apache.spark.kafka010.KafkaConfigUpdater
  * All three strategies have overloaded constructors that allow you to specify
  * the starting offset for a particular partition.
  */
-private[kafka010] sealed trait ConsumerStrategy {
+private[kafka010] sealed trait ConsumerStrategy extends Logging {
   /** Create a [[KafkaConsumer]] and subscribe to topics according to a desired strategy */
   def createConsumer(kafkaParams: ju.Map[String, Object]): Consumer[Array[Byte], Array[Byte]]
 
+  /** Creates an [[org.apache.kafka.clients.admin.AdminClient]] */
+  def createAdmin(kafkaParams: ju.Map[String, Object]): Admin = {
+    val updatedKafkaParams = setAuthenticationConfigIfNeeded(kafkaParams)
+    logDebug(s"Admin params: ${KafkaRedactionUtil.redactParams(updatedKafkaParams.asScala.toSeq)}")
+    Admin.create(updatedKafkaParams)
+  }
+
+  /** Returns the assigned or subscribed [[TopicPartition]] */
+  def assignedTopicPartitions(admin: Admin): Set[TopicPartition]
+
   /**
    * Updates the parameters with security if needed.
    * Added a function to hide internals and reduce code duplications because all strategy uses it.
@@ -48,13 +61,24 @@ private[kafka010] sealed trait ConsumerStrategy {
     KafkaConfigUpdater("source", kafkaParams.asScala.toMap)
       .setAuthenticationConfigIfNeeded()
       .build()
+
+  protected def retrieveAllPartitions(admin: Admin, topics: Set[String]): Set[TopicPartition] = {
+    admin.describeTopics(topics.asJava).all().get().asScala.filterNot(_._2.isInternal).flatMap {
+      case (topic, topicDescription) =>
+        topicDescription.partitions().asScala.map { topicPartitionInfo =>
+          val partition = topicPartitionInfo.partition()
+          logDebug(s"Partition found: $topic:$partition")
+          new TopicPartition(topic, partition)
+        }
+    }.toSet
+  }
 }
 
 /**
  * Specify a fixed collection of partitions.
  */
 private[kafka010] case class AssignStrategy(partitions: Array[TopicPartition])
-    extends ConsumerStrategy {
+    extends ConsumerStrategy with Logging {
   override def createConsumer(
       kafkaParams: ju.Map[String, Object]): Consumer[Array[Byte], Array[Byte]] = {
     val updatedKafkaParams = setAuthenticationConfigIfNeeded(kafkaParams)
@@ -63,13 +87,20 @@ private[kafka010] case class AssignStrategy(partitions: Array[TopicPartition])
     consumer
   }
 
+  override def assignedTopicPartitions(admin: Admin): Set[TopicPartition] = {
+    val topics = partitions.map(_.topic()).toSet
+    logDebug(s"Topics for assignment: $topics")
+    retrieveAllPartitions(admin, topics).filter(partitions.contains(_))
+  }
+
   override def toString: String = s"Assign[${partitions.mkString(", ")}]"
 }
 
 /**
  * Subscribe to a fixed collection of topics.
  */
-private[kafka010] case class SubscribeStrategy(topics: Seq[String]) extends ConsumerStrategy {
+private[kafka010] case class SubscribeStrategy(topics: Seq[String])
+    extends ConsumerStrategy with Logging {
   override def createConsumer(
       kafkaParams: ju.Map[String, Object]): Consumer[Array[Byte], Array[Byte]] = {
     val updatedKafkaParams = setAuthenticationConfigIfNeeded(kafkaParams)
@@ -78,6 +109,10 @@ private[kafka010] case class SubscribeStrategy(topics: Seq[String]) extends Cons
     consumer
   }
 
+  override def assignedTopicPartitions(admin: Admin): Set[TopicPartition] = {
+    retrieveAllPartitions(admin, topics.toSet)
+  }
+
   override def toString: String = s"Subscribe[${topics.mkString(", ")}]"
 }
 
@@ -85,16 +120,30 @@ private[kafka010] case class SubscribeStrategy(topics: Seq[String]) extends Cons
  * Use a regex to specify topics of interest.
  */
 private[kafka010] case class SubscribePatternStrategy(topicPattern: String)
-    extends ConsumerStrategy {
+    extends ConsumerStrategy with Logging {
+  private val topicRegex = topicPattern.r
+
   override def createConsumer(
       kafkaParams: ju.Map[String, Object]): Consumer[Array[Byte], Array[Byte]] = {
     val updatedKafkaParams = setAuthenticationConfigIfNeeded(kafkaParams)
     val consumer = new KafkaConsumer[Array[Byte], Array[Byte]](updatedKafkaParams)
-    consumer.subscribe(
-      ju.regex.Pattern.compile(topicPattern),
-      new NoOpConsumerRebalanceListener())
+    consumer.subscribe(ju.regex.Pattern.compile(topicPattern), new NoOpConsumerRebalanceListener())
     consumer
   }
 
+  override def assignedTopicPartitions(admin: Admin): Set[TopicPartition] = {
+    logDebug(s"Topic pattern: $topicPattern")
+    var topics = mutable.Seq.empty[String]
+    // listTopics is not listing internal topics by default so no filter needed
+    admin.listTopics().listings().get().asScala.foreach { topicListing =>
+      val name = topicListing.name()
+      if (topicRegex.findFirstIn(name).isDefined) {
+        logDebug(s"Topic matches pattern: $name")
+        topics :+= name
+      }
+    }
+    retrieveAllPartitions(admin, topics.toSet)
+  }
+
   override def toString: String = s"SubscribePattern[$topicPattern]"
 }
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaBatch.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaBatch.scala
index 9ad083f1cfde5..268719d6aed2c 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaBatch.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaBatch.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.sql.kafka010
 
-import org.apache.kafka.common.TopicPartition
-
 import org.apache.spark.SparkEnv
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.Network.NETWORK_TIMEOUT
@@ -50,7 +48,7 @@ private[kafka010] class KafkaBatch(
     // id. Hence, we should generate a unique id for each query.
     val uniqueGroupId = KafkaSourceProvider.batchUniqueGroupId(sourceOptions)
 
-    val kafkaOffsetReader = new KafkaOffsetReader(
+    val kafkaOffsetReader = KafkaOffsetReader.build(
       strategy,
       KafkaSourceProvider.kafkaParamsForDriver(specifiedKafkaParams),
       sourceOptions,
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaBatchPartitionReader.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaBatchPartitionReader.scala
index 8b37fd6e7e2b3..b6d64c79b1df4 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaBatchPartitionReader.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaBatchPartitionReader.scala
@@ -22,6 +22,7 @@ import java.{util => ju}
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.UnsafeRow
+import org.apache.spark.sql.connector.metric.CustomTaskMetric
 import org.apache.spark.sql.connector.read.{InputPartition, PartitionReader, PartitionReaderFactory}
 import org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer
 
@@ -105,4 +106,16 @@ private case class KafkaBatchPartitionReader(
       range
     }
   }
+
+  override def currentMetricsValues(): Array[CustomTaskMetric] = {
+    val offsetOutOfRange = new CustomTaskMetric {
+      override def name(): String = "offsetOutOfRange"
+      override def value(): Long = consumer.getNumOffsetOutOfRange()
+    }
+    val dataLoss = new CustomTaskMetric {
+      override def name(): String = "dataLoss"
+      override def value(): Long = consumer.getNumDataLoss()
+    }
+    Array(offsetOutOfRange, dataLoss)
+  }
 }
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchStream.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchStream.scala
index 6599e7e0fe707..822184842deb2 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchStream.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchStream.scala
@@ -18,16 +18,16 @@
 package org.apache.spark.sql.kafka010
 
 import java.{util => ju}
+import java.util.Optional
 
-import org.apache.kafka.clients.consumer.ConsumerConfig
+import scala.collection.JavaConverters._
 
 import org.apache.spark.SparkEnv
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.Network.NETWORK_TIMEOUT
-import org.apache.spark.scheduler.ExecutorCacheTaskLocation
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.connector.read.{InputPartition, PartitionReaderFactory}
-import org.apache.spark.sql.connector.read.streaming.{MicroBatchStream, Offset, ReadAllAvailable, ReadLimit, ReadMaxRows, SupportsAdmissionControl}
+import org.apache.spark.sql.connector.read.streaming.{MicroBatchStream, Offset, ReadAllAvailable, ReadLimit, ReadMaxRows, ReportsSourceMetrics, SupportsAdmissionControl}
 import org.apache.spark.sql.kafka010.KafkaSourceProvider._
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 import org.apache.spark.util.UninterruptibleThread
@@ -54,7 +54,8 @@ private[kafka010] class KafkaMicroBatchStream(
     options: CaseInsensitiveStringMap,
     metadataPath: String,
     startingOffsets: KafkaOffsetRangeLimit,
-    failOnDataLoss: Boolean) extends SupportsAdmissionControl with MicroBatchStream with Logging {
+    failOnDataLoss: Boolean)
+  extends SupportsAdmissionControl with ReportsSourceMetrics with MicroBatchStream with Logging {
 
   private[kafka010] val pollTimeoutMs = options.getLong(
     KafkaSourceProvider.CONSUMER_POLL_TIMEOUT,
@@ -67,6 +68,8 @@ private[kafka010] class KafkaMicroBatchStream(
 
   private var endPartitionOffsets: KafkaSourceOffset = _
 
+  private var latestPartitionOffsets: PartitionOffsetMap = _
+
   /**
    * Lazily initialize `initialPartitionOffsets` to make sure that `KafkaConsumer.poll` is only
    * called in StreamExecutionThread. Otherwise, interrupting a thread while running
@@ -80,6 +83,10 @@ private[kafka010] class KafkaMicroBatchStream(
     maxOffsetsPerTrigger.map(ReadLimit.maxRows).getOrElse(super.getDefaultReadLimit)
   }
 
+  override def reportLatestOffset(): Offset = {
+    KafkaSourceOffset(latestPartitionOffsets)
+  }
+
   override def latestOffset(): Offset = {
     throw new UnsupportedOperationException(
       "latestOffset(Offset, ReadLimit) should be called instead of this method")
@@ -87,7 +94,7 @@ private[kafka010] class KafkaMicroBatchStream(
 
   override def latestOffset(start: Offset, readLimit: ReadLimit): Offset = {
     val startPartitionOffsets = start.asInstanceOf[KafkaSourceOffset].partitionToOffsets
-    val latestPartitionOffsets = kafkaOffsetReader.fetchLatestOffsets(Some(startPartitionOffsets))
+    latestPartitionOffsets = kafkaOffsetReader.fetchLatestOffsets(Some(startPartitionOffsets))
     endPartitionOffsets = KafkaSourceOffset(readLimit match {
       case rows: ReadMaxRows =>
         rateLimit(rows.maxRows(), startPartitionOffsets, latestPartitionOffsets)
@@ -130,6 +137,10 @@ private[kafka010] class KafkaMicroBatchStream(
 
   override def toString(): String = s"KafkaV2[$kafkaOffsetReader]"
 
+  override def metrics(latestConsumedOffset: Optional[Offset]): ju.Map[String, String] = {
+    KafkaMicroBatchStream.metrics(latestConsumedOffset, latestPartitionOffsets)
+  }
+
   /**
    * Read initial partition offsets from the checkpoint, or decide the offsets and write them to
    * the checkpoint.
@@ -167,7 +178,7 @@ private[kafka010] class KafkaMicroBatchStream(
       limit: Long,
       from: PartitionOffsetMap,
       until: PartitionOffsetMap): PartitionOffsetMap = {
-    val fromNew = kafkaOffsetReader.fetchEarliestOffsets(until.keySet.diff(from.keySet).toSeq)
+    lazy val fromNew = kafkaOffsetReader.fetchEarliestOffsets(until.keySet.diff(from.keySet).toSeq)
     val sizes = until.flatMap {
       case (tp, end) =>
         // If begin isn't defined, something's wrong, but let alert logic in getBatch handle it
@@ -215,3 +226,37 @@ private[kafka010] class KafkaMicroBatchStream(
     }
   }
 }
+
+object KafkaMicroBatchStream extends Logging {
+
+  /**
+   * Compute the difference of offset per partition between latestAvailablePartitionOffsets
+   * and partition offsets in the latestConsumedOffset.
+   * Report min/max/avg offsets behind the latest for all the partitions in the Kafka stream.
+   *
+   * Because of rate limit, latest consumed offset per partition can be smaller than
+   * the latest available offset per partition.
+   * @param latestConsumedOffset latest consumed offset
+   * @param latestAvailablePartitionOffsets latest available offset per partition
+   * @return the generated metrics map
+   */
+  def metrics(
+      latestConsumedOffset: Optional[Offset],
+      latestAvailablePartitionOffsets: PartitionOffsetMap): ju.Map[String, String] = {
+    val offset = Option(latestConsumedOffset.orElse(null))
+
+    if (offset.nonEmpty && latestAvailablePartitionOffsets != null) {
+      val consumedPartitionOffsets = offset.map(KafkaSourceOffset(_)).get.partitionToOffsets
+      val offsetsBehindLatest = latestAvailablePartitionOffsets
+        .map(partitionOffset => partitionOffset._2 - consumedPartitionOffsets(partitionOffset._1))
+      if (offsetsBehindLatest.nonEmpty) {
+        val avgOffsetBehindLatest = offsetsBehindLatest.sum.toDouble / offsetsBehindLatest.size
+        return Map[String, String](
+          "minOffsetsBehindLatest" -> offsetsBehindLatest.min.toString,
+          "maxOffsetsBehindLatest" -> offsetsBehindLatest.max.toString,
+          "avgOffsetsBehindLatest" -> avgOffsetBehindLatest.toString).asJava
+      }
+    }
+    ju.Collections.emptyMap()
+  }
+}
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetRangeCalculator.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetRangeCalculator.scala
index f7183f7add14b..1e9a62ecce025 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetRangeCalculator.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetRangeCalculator.scala
@@ -46,7 +46,7 @@ private[kafka010] class KafkaOffsetRangeCalculator(val minPartitions: Option[Int
     val offsetRanges = ranges.filter(_.size > 0)
 
     // If minPartitions not set or there are enough partitions to satisfy minPartitions
-    if (minPartitions.isEmpty || offsetRanges.size > minPartitions.get) {
+    if (minPartitions.isEmpty || offsetRanges.size >= minPartitions.get) {
       // Assign preferred executor locations to each range such that the same topic-partition is
       // preferentially read from the same executor and the KafkaConsumer can be reused.
       offsetRanges.map { range =>
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReader.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReader.scala
index 216e74a85c2ae..546970507a2ed 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReader.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReader.scala
@@ -18,129 +18,30 @@
 package org.apache.spark.sql.kafka010
 
 import java.{util => ju}
-import java.util.concurrent.Executors
 
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
-import scala.concurrent.{ExecutionContext, Future}
-import scala.concurrent.duration.Duration
-import scala.util.control.NonFatal
-
-import org.apache.kafka.clients.consumer.{Consumer, ConsumerConfig, KafkaConsumer, OffsetAndTimestamp}
 import org.apache.kafka.common.TopicPartition
 
-import org.apache.spark.SparkEnv
 import org.apache.spark.internal.Logging
-import org.apache.spark.scheduler.ExecutorCacheTaskLocation
 import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
-import org.apache.spark.util.{ThreadUtils, UninterruptibleThread}
+import org.apache.spark.sql.internal.SQLConf
 
 /**
- * This class uses Kafka's own [[KafkaConsumer]] API to read data offsets from Kafka.
- * The [[ConsumerStrategy]] class defines which Kafka topics and partitions should be read
- * by this source. These strategies directly correspond to the different consumption options
- * in. This class is designed to return a configured [[KafkaConsumer]] that is used by the
- * [[KafkaSource]] to query for the offsets. See the docs on
- * [[org.apache.spark.sql.kafka010.ConsumerStrategy]]
- * for more details.
- *
- * Note: This class is not ThreadSafe
+ * Base trait to fetch offsets from Kafka. The implementations are
+ * [[KafkaOffsetReaderConsumer]] and [[KafkaOffsetReaderAdmin]].
  */
-private[kafka010] class KafkaOffsetReader(
-    consumerStrategy: ConsumerStrategy,
-    val driverKafkaParams: ju.Map[String, Object],
-    readerOptions: CaseInsensitiveMap[String],
-    driverGroupIdPrefix: String) extends Logging {
-  /**
-   * Used to ensure execute fetch operations execute in an UninterruptibleThread
-   */
-  val kafkaReaderThread = Executors.newSingleThreadExecutor((r: Runnable) => {
-    val t = new UninterruptibleThread("Kafka Offset Reader") {
-      override def run(): Unit = {
-        r.run()
-      }
-    }
-    t.setDaemon(true)
-    t
-  })
-  val execContext = ExecutionContext.fromExecutorService(kafkaReaderThread)
-
-  /**
-   * Place [[groupId]] and [[nextId]] here so that they are initialized before any consumer is
-   * created -- see SPARK-19564.
-   */
-  private var groupId: String = null
-  private var nextId = 0
-
-  /**
-   * A KafkaConsumer used in the driver to query the latest Kafka offsets. This only queries the
-   * offsets and never commits them.
-   */
-  @volatile protected var _consumer: Consumer[Array[Byte], Array[Byte]] = null
-
-  protected def consumer: Consumer[Array[Byte], Array[Byte]] = synchronized {
-    assert(Thread.currentThread().isInstanceOf[UninterruptibleThread])
-    if (_consumer == null) {
-      val newKafkaParams = new ju.HashMap[String, Object](driverKafkaParams)
-      if (driverKafkaParams.get(ConsumerConfig.GROUP_ID_CONFIG) == null) {
-        newKafkaParams.put(ConsumerConfig.GROUP_ID_CONFIG, nextGroupId())
-      }
-      _consumer = consumerStrategy.createConsumer(newKafkaParams)
-    }
-    _consumer
-  }
+private[kafka010] trait KafkaOffsetReader {
 
-  private[kafka010] val maxOffsetFetchAttempts =
-    readerOptions.getOrElse(KafkaSourceProvider.FETCH_OFFSET_NUM_RETRY, "3").toInt
+  // These are needed here because of KafkaSourceProviderSuite
+  private[kafka010] val maxOffsetFetchAttempts: Int
+  private[kafka010] val offsetFetchAttemptIntervalMs: Long
 
-  /**
-   * Number of partitions to read from Kafka. If this value is greater than the number of Kafka
-   * topicPartitions, we will split up  the read tasks of the skewed partitions to multiple Spark
-   * tasks. The number of Spark tasks will be *approximately* `numPartitions`. It can be less or
-   * more depending on rounding errors or Kafka partitions that didn't receive any new data.
-   */
-  private val minPartitions =
-    readerOptions.get(KafkaSourceProvider.MIN_PARTITIONS_OPTION_KEY).map(_.toInt)
-
-  private val rangeCalculator = new KafkaOffsetRangeCalculator(minPartitions)
-
-  private[kafka010] val offsetFetchAttemptIntervalMs =
-    readerOptions.getOrElse(KafkaSourceProvider.FETCH_OFFSET_RETRY_INTERVAL_MS, "1000").toLong
-
-  /**
-   * Whether we should divide Kafka TopicPartitions with a lot of data into smaller Spark tasks.
-   */
-  private def shouldDivvyUpLargePartitions(numTopicPartitions: Int): Boolean = {
-    minPartitions.map(_ > numTopicPartitions).getOrElse(false)
-  }
-
-  private def nextGroupId(): String = {
-    groupId = driverGroupIdPrefix + "-" + nextId
-    nextId += 1
-    groupId
-  }
-
-  override def toString(): String = consumerStrategy.toString
+  // This is needed here because of KafkaContinuousStream
+  val driverKafkaParams: ju.Map[String, Object]
 
   /**
    * Closes the connection to Kafka, and cleans up state.
    */
-  def close(): Unit = {
-    if (_consumer != null) runUninterruptibly { stopConsumer() }
-    kafkaReaderThread.shutdown()
-  }
-
-  /**
-   * @return The Set of TopicPartitions for a given topic
-   */
-  def fetchTopicPartitions(): Set[TopicPartition] = runUninterruptibly {
-    assert(Thread.currentThread().isInstanceOf[UninterruptibleThread])
-    // Poll to get the latest assigned partitions
-    consumer.poll(0)
-    val partitions = consumer.assignment()
-    consumer.pause(partitions)
-    partitions.asScala.toSet
-  }
+  def close(): Unit
 
   /**
    * Fetch the partition offsets for the topic partitions that are indicated
@@ -148,32 +49,7 @@ private[kafka010] class KafkaOffsetReader(
    */
   def fetchPartitionOffsets(
       offsetRangeLimit: KafkaOffsetRangeLimit,
-      isStartingOffsets: Boolean): Map[TopicPartition, Long] = {
-    def validateTopicPartitions(partitions: Set[TopicPartition],
-      partitionOffsets: Map[TopicPartition, Long]): Map[TopicPartition, Long] = {
-      assert(partitions == partitionOffsets.keySet,
-        "If startingOffsets contains specific offsets, you must specify all TopicPartitions.\n" +
-          "Use -1 for latest, -2 for earliest.\n" +
-          s"Specified: ${partitionOffsets.keySet} Assigned: ${partitions}")
-      logDebug(s"Partitions assigned to consumer: $partitions. Seeking to $partitionOffsets")
-      partitionOffsets
-    }
-    val partitions = fetchTopicPartitions()
-    // Obtain TopicPartition offsets with late binding support
-    offsetRangeLimit match {
-      case EarliestOffsetRangeLimit => partitions.map {
-        case tp => tp -> KafkaOffsetRangeLimit.EARLIEST
-      }.toMap
-      case LatestOffsetRangeLimit => partitions.map {
-        case tp => tp -> KafkaOffsetRangeLimit.LATEST
-      }.toMap
-      case SpecificOffsetRangeLimit(partitionOffsets) =>
-        validateTopicPartitions(partitions, partitionOffsets)
-      case SpecificTimestampRangeLimit(partitionTimestamps) =>
-        fetchSpecificTimestampBasedOffsets(partitionTimestamps,
-          failsOnNoMatchingOffset = isStartingOffsets).partitionToOffsets
-    }
-  }
+      isStartingOffsets: Boolean): Map[TopicPartition, Long]
 
   /**
    * Resolves the specific offsets based on Kafka seek positions.
@@ -185,212 +61,46 @@ private[kafka010] class KafkaOffsetReader(
    */
   def fetchSpecificOffsets(
       partitionOffsets: Map[TopicPartition, Long],
-      reportDataLoss: String => Unit): KafkaSourceOffset = {
-    val fnAssertParametersWithPartitions: ju.Set[TopicPartition] => Unit = { partitions =>
-      assert(partitions.asScala == partitionOffsets.keySet,
-        "If startingOffsets contains specific offsets, you must specify all TopicPartitions.\n" +
-          "Use -1 for latest, -2 for earliest, if you don't care.\n" +
-          s"Specified: ${partitionOffsets.keySet} Assigned: ${partitions.asScala}")
-      logDebug(s"Partitions assigned to consumer: $partitions. Seeking to $partitionOffsets")
-    }
-
-    val fnRetrievePartitionOffsets: ju.Set[TopicPartition] => Map[TopicPartition, Long] = { _ =>
-      partitionOffsets
-    }
-
-    val fnAssertFetchedOffsets: Map[TopicPartition, Long] => Unit = { fetched =>
-      partitionOffsets.foreach {
-        case (tp, off) if off != KafkaOffsetRangeLimit.LATEST &&
-          off != KafkaOffsetRangeLimit.EARLIEST =>
-          if (fetched(tp) != off) {
-            reportDataLoss(
-              s"startingOffsets for $tp was $off but consumer reset to ${fetched(tp)}")
-          }
-        case _ =>
-        // no real way to check that beginning or end is reasonable
-      }
-    }
-
-    fetchSpecificOffsets0(fnAssertParametersWithPartitions, fnRetrievePartitionOffsets,
-      fnAssertFetchedOffsets)
-  }
+      reportDataLoss: String => Unit): KafkaSourceOffset
 
+  /**
+   * Resolves the specific offsets based on timestamp per topic-partition.
+   * The returned offset for each partition is the earliest offset whose timestamp is greater
+   * than or equal to the given timestamp in the corresponding partition. If the matched offset
+   * doesn't exist, depending on `failsOnNoMatchingOffset` parameter, the offset will be set to
+   * latest or this method throws an error.
+   *
+   * @param partitionTimestamps the timestamp per topic-partition.
+   * @param failsOnNoMatchingOffset whether to fail the query when no matched offset can be found.
+   */
   def fetchSpecificTimestampBasedOffsets(
       partitionTimestamps: Map[TopicPartition, Long],
-      failsOnNoMatchingOffset: Boolean): KafkaSourceOffset = {
-    val fnAssertParametersWithPartitions: ju.Set[TopicPartition] => Unit = { partitions =>
-      assert(partitions.asScala == partitionTimestamps.keySet,
-        "If starting/endingOffsetsByTimestamp contains specific offsets, you must specify all " +
-          s"topics. Specified: ${partitionTimestamps.keySet} Assigned: ${partitions.asScala}")
-      logDebug(s"Partitions assigned to consumer: $partitions. Seeking to $partitionTimestamps")
-    }
-
-    val fnRetrievePartitionOffsets: ju.Set[TopicPartition] => Map[TopicPartition, Long] = { _ => {
-        val converted = partitionTimestamps.map { case (tp, timestamp) =>
-          tp -> java.lang.Long.valueOf(timestamp)
-        }.asJava
-
-        val offsetForTime: ju.Map[TopicPartition, OffsetAndTimestamp] =
-          consumer.offsetsForTimes(converted)
-
-        offsetForTime.asScala.map { case (tp, offsetAndTimestamp) =>
-          if (failsOnNoMatchingOffset) {
-            assert(offsetAndTimestamp != null, "No offset matched from request of " +
-              s"topic-partition $tp and timestamp ${partitionTimestamps(tp)}.")
-          }
-
-          if (offsetAndTimestamp == null) {
-            tp -> KafkaOffsetRangeLimit.LATEST
-          } else {
-            tp -> offsetAndTimestamp.offset()
-          }
-        }.toMap
-      }
-    }
-
-    val fnAssertFetchedOffsets: Map[TopicPartition, Long] => Unit = { _ => }
-
-    fetchSpecificOffsets0(fnAssertParametersWithPartitions, fnRetrievePartitionOffsets,
-      fnAssertFetchedOffsets)
-  }
-
-  private def fetchSpecificOffsets0(
-      fnAssertParametersWithPartitions: ju.Set[TopicPartition] => Unit,
-      fnRetrievePartitionOffsets: ju.Set[TopicPartition] => Map[TopicPartition, Long],
-      fnAssertFetchedOffsets: Map[TopicPartition, Long] => Unit): KafkaSourceOffset = {
-    val fetched = partitionsAssignedToConsumer {
-      partitions => {
-        fnAssertParametersWithPartitions(partitions)
-
-        val partitionOffsets = fnRetrievePartitionOffsets(partitions)
-
-        partitionOffsets.foreach {
-          case (tp, KafkaOffsetRangeLimit.LATEST) =>
-            consumer.seekToEnd(ju.Arrays.asList(tp))
-          case (tp, KafkaOffsetRangeLimit.EARLIEST) =>
-            consumer.seekToBeginning(ju.Arrays.asList(tp))
-          case (tp, off) => consumer.seek(tp, off)
-        }
-
-        partitionOffsets.map {
-          case (tp, _) => tp -> consumer.position(tp)
-        }
-      }
-    }
-
-    fnAssertFetchedOffsets(fetched)
-
-    KafkaSourceOffset(fetched)
-  }
+      failsOnNoMatchingOffset: Boolean): KafkaSourceOffset
 
   /**
    * Fetch the earliest offsets for the topic partitions that are indicated
    * in the [[ConsumerStrategy]].
    */
-  def fetchEarliestOffsets(): Map[TopicPartition, Long] = partitionsAssignedToConsumer(
-    partitions => {
-      logDebug("Seeking to the beginning")
-
-      consumer.seekToBeginning(partitions)
-      val partitionOffsets = partitions.asScala.map(p => p -> consumer.position(p)).toMap
-      logDebug(s"Got earliest offsets for partition : $partitionOffsets")
-      partitionOffsets
-    }, fetchingEarliestOffset = true)
+  def fetchEarliestOffsets(): Map[TopicPartition, Long]
 
   /**
    * Fetch the latest offsets for the topic partitions that are indicated
    * in the [[ConsumerStrategy]].
    *
-   * Kafka may return earliest offsets when we are requesting latest offsets if `poll` is called
-   * right before `seekToEnd` (KAFKA-7703). As a workaround, we will call `position` right after
-   * `poll` to wait until the potential offset request triggered by `poll(0)` is done.
-   *
-   * In addition, to avoid other unknown issues, we also use the given `knownOffsets` to audit the
+   * In order to avoid unknown issues, we use the given `knownOffsets` to audit the
    * latest offsets returned by Kafka. If we find some incorrect offsets (a latest offset is less
    * than an offset in `knownOffsets`), we will retry at most `maxOffsetFetchAttempts` times. When
    * a topic is recreated, the latest offsets may be less than offsets in `knownOffsets`. We cannot
-   * distinguish this with KAFKA-7703, so we just return whatever we get from Kafka after retrying.
+   * distinguish this with issues like KAFKA-7703, so we just return whatever we get from Kafka
+   * after retrying.
    */
-  def fetchLatestOffsets(
-      knownOffsets: Option[PartitionOffsetMap]): PartitionOffsetMap =
-    partitionsAssignedToConsumer { partitions => {
-      logDebug("Seeking to the end.")
-
-      if (knownOffsets.isEmpty) {
-        consumer.seekToEnd(partitions)
-        partitions.asScala.map(p => p -> consumer.position(p)).toMap
-      } else {
-        var partitionOffsets: PartitionOffsetMap = Map.empty
-
-        /**
-         * Compare `knownOffsets` and `partitionOffsets`. Returns all partitions that have incorrect
-         * latest offset (offset in `knownOffsets` is great than the one in `partitionOffsets`).
-         */
-        def findIncorrectOffsets(): Seq[(TopicPartition, Long, Long)] = {
-          var incorrectOffsets = ArrayBuffer[(TopicPartition, Long, Long)]()
-          partitionOffsets.foreach { case (tp, offset) =>
-            knownOffsets.foreach(_.get(tp).foreach { knownOffset =>
-              if (knownOffset > offset) {
-                val incorrectOffset = (tp, knownOffset, offset)
-                incorrectOffsets += incorrectOffset
-              }
-            })
-          }
-          incorrectOffsets
-        }
-
-        // Retry to fetch latest offsets when detecting incorrect offsets. We don't use
-        // `withRetriesWithoutInterrupt` to retry because:
-        //
-        // - `withRetriesWithoutInterrupt` will reset the consumer for each attempt but a fresh
-        //    consumer has a much bigger chance to hit KAFKA-7703.
-        // - Avoid calling `consumer.poll(0)` which may cause KAFKA-7703.
-        var incorrectOffsets: Seq[(TopicPartition, Long, Long)] = Nil
-        var attempt = 0
-        do {
-          consumer.seekToEnd(partitions)
-          partitionOffsets = partitions.asScala.map(p => p -> consumer.position(p)).toMap
-          attempt += 1
-
-          incorrectOffsets = findIncorrectOffsets()
-          if (incorrectOffsets.nonEmpty) {
-            logWarning("Found incorrect offsets in some partitions " +
-              s"(partition, previous offset, fetched offset): $incorrectOffsets")
-            if (attempt < maxOffsetFetchAttempts) {
-              logWarning("Retrying to fetch latest offsets because of incorrect offsets")
-              Thread.sleep(offsetFetchAttemptIntervalMs)
-            }
-          }
-        } while (incorrectOffsets.nonEmpty && attempt < maxOffsetFetchAttempts)
-
-        logDebug(s"Got latest offsets for partition : $partitionOffsets")
-        partitionOffsets
-      }
-    }
-  }
+  def fetchLatestOffsets(knownOffsets: Option[PartitionOffsetMap]): PartitionOffsetMap
 
   /**
    * Fetch the earliest offsets for specific topic partitions.
    * The return result may not contain some partitions if they are deleted.
    */
-  def fetchEarliestOffsets(
-      newPartitions: Seq[TopicPartition]): Map[TopicPartition, Long] = {
-    if (newPartitions.isEmpty) {
-      Map.empty[TopicPartition, Long]
-    } else {
-      partitionsAssignedToConsumer(partitions => {
-        // Get the earliest offset of each partition
-        consumer.seekToBeginning(partitions)
-        val partitionOffsets = newPartitions.filter { p =>
-          // When deleting topics happen at the same time, some partitions may not be in
-          // `partitions`. So we need to ignore them
-          partitions.contains(p)
-        }.map(p => p -> consumer.position(p)).toMap
-        logDebug(s"Got earliest offsets for new partitions: $partitionOffsets")
-        partitionOffsets
-      }, fetchingEarliestOffset = true)
-    }
-  }
+  def fetchEarliestOffsets(newPartitions: Seq[TopicPartition]): Map[TopicPartition, Long]
 
   /**
    * Return the offset ranges for a Kafka batch query. If `minPartitions` is set, this method may
@@ -402,75 +112,7 @@ private[kafka010] class KafkaOffsetReader(
    */
   def getOffsetRangesFromUnresolvedOffsets(
       startingOffsets: KafkaOffsetRangeLimit,
-      endingOffsets: KafkaOffsetRangeLimit): Seq[KafkaOffsetRange] = {
-    val fromPartitionOffsets = fetchPartitionOffsets(startingOffsets, isStartingOffsets = true)
-    val untilPartitionOffsets = fetchPartitionOffsets(endingOffsets, isStartingOffsets = false)
-
-    // Obtain topicPartitions in both from and until partition offset, ignoring
-    // topic partitions that were added and/or deleted between the two above calls.
-    if (fromPartitionOffsets.keySet != untilPartitionOffsets.keySet) {
-      implicit val topicOrdering: Ordering[TopicPartition] = Ordering.by(t => t.topic())
-      val fromTopics = fromPartitionOffsets.keySet.toList.sorted.mkString(",")
-      val untilTopics = untilPartitionOffsets.keySet.toList.sorted.mkString(",")
-      throw new IllegalStateException("different topic partitions " +
-        s"for starting offsets topics[${fromTopics}] and " +
-        s"ending offsets topics[${untilTopics}]")
-    }
-
-    // Calculate offset ranges
-    val offsetRangesBase = untilPartitionOffsets.keySet.map { tp =>
-      val fromOffset = fromPartitionOffsets.get(tp).getOrElse {
-        // This should not happen since topicPartitions contains all partitions not in
-        // fromPartitionOffsets
-        throw new IllegalStateException(s"$tp doesn't have a from offset")
-      }
-      val untilOffset = untilPartitionOffsets(tp)
-      KafkaOffsetRange(tp, fromOffset, untilOffset, None)
-    }.toSeq
-
-    if (shouldDivvyUpLargePartitions(offsetRangesBase.size)) {
-      val fromOffsetsMap =
-        offsetRangesBase.map(range => (range.topicPartition, range.fromOffset)).toMap
-      val untilOffsetsMap =
-        offsetRangesBase.map(range => (range.topicPartition, range.untilOffset)).toMap
-
-      // No need to report data loss here
-      val resolvedFromOffsets = fetchSpecificOffsets(fromOffsetsMap, _ => ()).partitionToOffsets
-      val resolvedUntilOffsets = fetchSpecificOffsets(untilOffsetsMap, _ => ()).partitionToOffsets
-      val ranges = offsetRangesBase.map(_.topicPartition).map { tp =>
-        KafkaOffsetRange(tp, resolvedFromOffsets(tp), resolvedUntilOffsets(tp), preferredLoc = None)
-      }
-      val divvied = rangeCalculator.getRanges(ranges).groupBy(_.topicPartition)
-      divvied.flatMap { case (tp, splitOffsetRanges) =>
-        if (splitOffsetRanges.length == 1) {
-          Seq(KafkaOffsetRange(tp, fromOffsetsMap(tp), untilOffsetsMap(tp), None))
-        } else {
-          // the list can't be empty
-          val first = splitOffsetRanges.head.copy(fromOffset = fromOffsetsMap(tp))
-          val end = splitOffsetRanges.last.copy(untilOffset = untilOffsetsMap(tp))
-          Seq(first) ++ splitOffsetRanges.drop(1).dropRight(1) :+ end
-        }
-      }.toArray.toSeq
-    } else {
-      offsetRangesBase
-    }
-  }
-
-  private def getSortedExecutorList(): Array[String] = {
-    def compare(a: ExecutorCacheTaskLocation, b: ExecutorCacheTaskLocation): Boolean = {
-      if (a.host == b.host) {
-        a.executorId > b.executorId
-      } else {
-        a.host > b.host
-      }
-    }
-
-    val bm = SparkEnv.get.blockManager
-    bm.master.getPeers(bm.blockManagerId).toArray
-      .map(x => ExecutorCacheTaskLocation(x.host, x.executorId))
-      .sortWith(compare)
-      .map(_.toString)
-  }
+      endingOffsets: KafkaOffsetRangeLimit): Seq[KafkaOffsetRange]
 
   /**
    * Return the offset ranges for a Kafka streaming batch. If `minPartitions` is set, this method
@@ -480,157 +122,23 @@ private[kafka010] class KafkaOffsetReader(
   def getOffsetRangesFromResolvedOffsets(
       fromPartitionOffsets: PartitionOffsetMap,
       untilPartitionOffsets: PartitionOffsetMap,
-      reportDataLoss: String => Unit): Seq[KafkaOffsetRange] = {
-    // Find the new partitions, and get their earliest offsets
-    val newPartitions = untilPartitionOffsets.keySet.diff(fromPartitionOffsets.keySet)
-    val newPartitionInitialOffsets = fetchEarliestOffsets(newPartitions.toSeq)
-    if (newPartitionInitialOffsets.keySet != newPartitions) {
-      // We cannot get from offsets for some partitions. It means they got deleted.
-      val deletedPartitions = newPartitions.diff(newPartitionInitialOffsets.keySet)
-      reportDataLoss(
-        s"Cannot find earliest offsets of ${deletedPartitions}. Some data may have been missed")
-    }
-    logInfo(s"Partitions added: $newPartitionInitialOffsets")
-    newPartitionInitialOffsets.filter(_._2 != 0).foreach { case (p, o) =>
-      reportDataLoss(
-        s"Added partition $p starts from $o instead of 0. Some data may have been missed")
-    }
-
-    val deletedPartitions = fromPartitionOffsets.keySet.diff(untilPartitionOffsets.keySet)
-    if (deletedPartitions.nonEmpty) {
-      val message = if (driverKafkaParams.containsKey(ConsumerConfig.GROUP_ID_CONFIG)) {
-        s"$deletedPartitions are gone. ${KafkaSourceProvider.CUSTOM_GROUP_ID_ERROR_MESSAGE}"
-      } else {
-        s"$deletedPartitions are gone. Some data may have been missed."
-      }
-      reportDataLoss(message)
-    }
-
-    // Use the until partitions to calculate offset ranges to ignore partitions that have
-    // been deleted
-    val topicPartitions = untilPartitionOffsets.keySet.filter { tp =>
-      // Ignore partitions that we don't know the from offsets.
-      newPartitionInitialOffsets.contains(tp) || fromPartitionOffsets.contains(tp)
-    }.toSeq
-    logDebug("TopicPartitions: " + topicPartitions.mkString(", "))
-
-    val fromOffsets = fromPartitionOffsets ++ newPartitionInitialOffsets
-    val untilOffsets = untilPartitionOffsets
-    val ranges = topicPartitions.map { tp =>
-      val fromOffset = fromOffsets(tp)
-      val untilOffset = untilOffsets(tp)
-      if (untilOffset < fromOffset) {
-        reportDataLoss(s"Partition $tp's offset was changed from " +
-          s"$fromOffset to $untilOffset, some data may have been missed")
-      }
-      KafkaOffsetRange(tp, fromOffset, untilOffset, preferredLoc = None)
-    }
-    rangeCalculator.getRanges(ranges, getSortedExecutorList)
-  }
-
-  private def partitionsAssignedToConsumer(
-      body: ju.Set[TopicPartition] => Map[TopicPartition, Long],
-      fetchingEarliestOffset: Boolean = false)
-    : Map[TopicPartition, Long] = runUninterruptibly {
-
-    withRetriesWithoutInterrupt {
-      // Poll to get the latest assigned partitions
-      consumer.poll(0)
-      val partitions = consumer.assignment()
-
-      if (!fetchingEarliestOffset) {
-        // Call `position` to wait until the potential offset request triggered by `poll(0)` is
-        // done. This is a workaround for KAFKA-7703, which an async `seekToBeginning` triggered by
-        // `poll(0)` may reset offsets that should have been set by another request.
-        partitions.asScala.map(p => p -> consumer.position(p)).foreach(_ => {})
-      }
-
-      consumer.pause(partitions)
-      logDebug(s"Partitions assigned to consumer: $partitions.")
-      body(partitions)
-    }
-  }
+      reportDataLoss: String => Unit): Seq[KafkaOffsetRange]
+}
 
-  /**
-   * This method ensures that the closure is called in an [[UninterruptibleThread]].
-   * This is required when communicating with the [[KafkaConsumer]]. In the case
-   * of streaming queries, we are already running in an [[UninterruptibleThread]],
-   * however for batch mode this is not the case.
-   */
-  private def runUninterruptibly[T](body: => T): T = {
-    if (!Thread.currentThread.isInstanceOf[UninterruptibleThread]) {
-      val future = Future {
-        body
-      }(execContext)
-      ThreadUtils.awaitResult(future, Duration.Inf)
+private[kafka010] object KafkaOffsetReader extends Logging {
+  def build(
+      consumerStrategy: ConsumerStrategy,
+      driverKafkaParams: ju.Map[String, Object],
+      readerOptions: CaseInsensitiveMap[String],
+      driverGroupIdPrefix: String): KafkaOffsetReader = {
+    if (SQLConf.get.useDeprecatedKafkaOffsetFetching) {
+      logDebug("Creating old and deprecated Consumer based offset reader")
+      new KafkaOffsetReaderConsumer(consumerStrategy, driverKafkaParams, readerOptions,
+        driverGroupIdPrefix)
     } else {
-      body
-    }
-  }
-
-  /**
-   * Helper function that does multiple retries on a body of code that returns offsets.
-   * Retries are needed to handle transient failures. For e.g. race conditions between getting
-   * assignment and getting position while topics/partitions are deleted can cause NPEs.
-   *
-   * This method also makes sure `body` won't be interrupted to workaround a potential issue in
-   * `KafkaConsumer.poll`. (KAFKA-1894)
-   */
-  private def withRetriesWithoutInterrupt(
-      body: => Map[TopicPartition, Long]): Map[TopicPartition, Long] = {
-    // Make sure `KafkaConsumer.poll` won't be interrupted (KAFKA-1894)
-    assert(Thread.currentThread().isInstanceOf[UninterruptibleThread])
-
-    synchronized {
-      var result: Option[Map[TopicPartition, Long]] = None
-      var attempt = 1
-      var lastException: Throwable = null
-      while (result.isEmpty && attempt <= maxOffsetFetchAttempts
-        && !Thread.currentThread().isInterrupted) {
-        Thread.currentThread match {
-          case ut: UninterruptibleThread =>
-            // "KafkaConsumer.poll" may hang forever if the thread is interrupted (E.g., the query
-            // is stopped)(KAFKA-1894). Hence, we just make sure we don't interrupt it.
-            //
-            // If the broker addresses are wrong, or Kafka cluster is down, "KafkaConsumer.poll" may
-            // hang forever as well. This cannot be resolved in KafkaSource until Kafka fixes the
-            // issue.
-            ut.runUninterruptibly {
-              try {
-                result = Some(body)
-              } catch {
-                case NonFatal(e) =>
-                  lastException = e
-                  logWarning(s"Error in attempt $attempt getting Kafka offsets: ", e)
-                  attempt += 1
-                  Thread.sleep(offsetFetchAttemptIntervalMs)
-                  resetConsumer()
-              }
-            }
-          case _ =>
-            throw new IllegalStateException(
-              "Kafka APIs must be executed on a o.a.spark.util.UninterruptibleThread")
-        }
-      }
-      if (Thread.interrupted()) {
-        throw new InterruptedException()
-      }
-      if (result.isEmpty) {
-        assert(attempt > maxOffsetFetchAttempts)
-        assert(lastException != null)
-        throw lastException
-      }
-      result.get
+      logDebug("Creating new Admin based offset reader")
+      new KafkaOffsetReaderAdmin(consumerStrategy, driverKafkaParams, readerOptions,
+        driverGroupIdPrefix)
     }
   }
-
-  private def stopConsumer(): Unit = synchronized {
-    assert(Thread.currentThread().isInstanceOf[UninterruptibleThread])
-    if (_consumer != null) _consumer.close()
-  }
-
-  private def resetConsumer(): Unit = synchronized {
-    stopConsumer()
-    _consumer = null  // will automatically get reinitialized again
-  }
 }
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderAdmin.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderAdmin.scala
new file mode 100644
index 0000000000000..6f4cb895f363d
--- /dev/null
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderAdmin.scala
@@ -0,0 +1,495 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.kafka010
+
+import java.{util => ju}
+import java.util.Locale
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+import scala.util.control.NonFatal
+
+import org.apache.kafka.clients.admin.{Admin, ListOffsetsOptions, OffsetSpec}
+import org.apache.kafka.clients.consumer.ConsumerConfig
+import org.apache.kafka.common.{IsolationLevel, TopicPartition}
+import org.apache.kafka.common.requests.OffsetFetchResponse
+
+import org.apache.spark.SparkEnv
+import org.apache.spark.internal.Logging
+import org.apache.spark.scheduler.ExecutorCacheTaskLocation
+import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
+
+/**
+ * This class uses Kafka's own [[Admin]] API to read data offsets from Kafka.
+ * The [[ConsumerStrategy]] class defines which Kafka topics and partitions should be read
+ * by this source. These strategies directly correspond to the different consumption options
+ * in. This class is designed to return a configured [[Admin]] that is used by the
+ * [[KafkaSource]] to query for the offsets. See the docs on
+ * [[org.apache.spark.sql.kafka010.ConsumerStrategy]]
+ * for more details.
+ *
+ * Note: This class is not ThreadSafe
+ */
+private[kafka010] class KafkaOffsetReaderAdmin(
+    consumerStrategy: ConsumerStrategy,
+    override val driverKafkaParams: ju.Map[String, Object],
+    readerOptions: CaseInsensitiveMap[String],
+    driverGroupIdPrefix: String) extends KafkaOffsetReader with Logging {
+
+  private[kafka010] val maxOffsetFetchAttempts =
+    readerOptions.getOrElse(KafkaSourceProvider.FETCH_OFFSET_NUM_RETRY, "3").toInt
+
+  private[kafka010] val offsetFetchAttemptIntervalMs =
+    readerOptions.getOrElse(KafkaSourceProvider.FETCH_OFFSET_RETRY_INTERVAL_MS, "1000").toLong
+
+  /**
+   * An AdminClient used in the driver to query the latest Kafka offsets.
+   * This only queries the offsets because AdminClient has no functionality to commit offsets like
+   * KafkaConsumer.
+   */
+  @volatile protected var _admin: Admin = null
+
+  protected def admin: Admin = synchronized {
+    if (_admin == null) {
+      _admin = consumerStrategy.createAdmin(driverKafkaParams)
+    }
+    _admin
+  }
+
+  lazy val isolationLevel: IsolationLevel = {
+    Option(driverKafkaParams.get(ConsumerConfig.ISOLATION_LEVEL_CONFIG)) match {
+      case Some(s: String) => IsolationLevel.valueOf(s.toUpperCase(Locale.ROOT))
+      case None => IsolationLevel.valueOf(
+        ConsumerConfig.DEFAULT_ISOLATION_LEVEL.toUpperCase(Locale.ROOT))
+      case _ => throw new IllegalArgumentException(s"${ConsumerConfig.ISOLATION_LEVEL_CONFIG} " +
+        "must be either not defined or with type String")
+    }
+  }
+
+  private lazy val listOffsetsOptions = new ListOffsetsOptions(isolationLevel)
+
+  private def listOffsets(admin: Admin, listOffsetsParams: ju.Map[TopicPartition, OffsetSpec]) = {
+    admin.listOffsets(listOffsetsParams, listOffsetsOptions).all().get().asScala
+      .map(result => result._1 -> result._2.offset()).toMap
+  }
+
+  /**
+   * Number of partitions to read from Kafka. If this value is greater than the number of Kafka
+   * topicPartitions, we will split up  the read tasks of the skewed partitions to multiple Spark
+   * tasks. The number of Spark tasks will be *approximately* `numPartitions`. It can be less or
+   * more depending on rounding errors or Kafka partitions that didn't receive any new data.
+   */
+  private val minPartitions =
+    readerOptions.get(KafkaSourceProvider.MIN_PARTITIONS_OPTION_KEY).map(_.toInt)
+
+  private val rangeCalculator = new KafkaOffsetRangeCalculator(minPartitions)
+
+  /**
+   * Whether we should divide Kafka TopicPartitions with a lot of data into smaller Spark tasks.
+   */
+  private def shouldDivvyUpLargePartitions(numTopicPartitions: Int): Boolean = {
+    minPartitions.map(_ > numTopicPartitions).getOrElse(false)
+  }
+
+  override def toString(): String = consumerStrategy.toString
+
+  override def close(): Unit = {
+    stopAdmin()
+  }
+
+  override def fetchPartitionOffsets(
+      offsetRangeLimit: KafkaOffsetRangeLimit,
+      isStartingOffsets: Boolean): Map[TopicPartition, Long] = {
+    def validateTopicPartitions(partitions: Set[TopicPartition],
+      partitionOffsets: Map[TopicPartition, Long]): Map[TopicPartition, Long] = {
+      assert(partitions == partitionOffsets.keySet,
+        "If startingOffsets contains specific offsets, you must specify all TopicPartitions.\n" +
+          "Use -1 for latest, -2 for earliest.\n" +
+          s"Specified: ${partitionOffsets.keySet} Assigned: ${partitions}")
+      logDebug(s"Assigned partitions: $partitions. Seeking to $partitionOffsets")
+      partitionOffsets
+    }
+    val partitions = consumerStrategy.assignedTopicPartitions(admin)
+    // Obtain TopicPartition offsets with late binding support
+    offsetRangeLimit match {
+      case EarliestOffsetRangeLimit => partitions.map {
+        case tp => tp -> KafkaOffsetRangeLimit.EARLIEST
+      }.toMap
+      case LatestOffsetRangeLimit => partitions.map {
+        case tp => tp -> KafkaOffsetRangeLimit.LATEST
+      }.toMap
+      case SpecificOffsetRangeLimit(partitionOffsets) =>
+        validateTopicPartitions(partitions, partitionOffsets)
+      case SpecificTimestampRangeLimit(partitionTimestamps) =>
+        fetchSpecificTimestampBasedOffsets(partitionTimestamps,
+          failsOnNoMatchingOffset = isStartingOffsets).partitionToOffsets
+    }
+  }
+
+  override def fetchSpecificOffsets(
+      partitionOffsets: Map[TopicPartition, Long],
+      reportDataLoss: String => Unit): KafkaSourceOffset = {
+    val fnAssertParametersWithPartitions: ju.Set[TopicPartition] => Unit = { partitions =>
+      assert(partitions.asScala == partitionOffsets.keySet,
+        "If startingOffsets contains specific offsets, you must specify all TopicPartitions.\n" +
+          "Use -1 for latest, -2 for earliest, if you don't care.\n" +
+          s"Specified: ${partitionOffsets.keySet} Assigned: ${partitions.asScala}")
+      logDebug(s"Assigned partitions: $partitions. Seeking to $partitionOffsets")
+    }
+
+    val fnRetrievePartitionOffsets: ju.Set[TopicPartition] => Map[TopicPartition, Long] = { _ =>
+      partitionOffsets
+    }
+
+    fetchSpecificOffsets0(fnAssertParametersWithPartitions, fnRetrievePartitionOffsets)
+  }
+
+  override def fetchSpecificTimestampBasedOffsets(
+      partitionTimestamps: Map[TopicPartition, Long],
+      failsOnNoMatchingOffset: Boolean): KafkaSourceOffset = {
+    val fnAssertParametersWithPartitions: ju.Set[TopicPartition] => Unit = { partitions =>
+      assert(partitions.asScala == partitionTimestamps.keySet,
+        "If starting/endingOffsetsByTimestamp contains specific offsets, you must specify all " +
+          s"topics. Specified: ${partitionTimestamps.keySet} Assigned: ${partitions.asScala}")
+      logDebug(s"Assigned partitions: $partitions. Seeking to $partitionTimestamps")
+    }
+
+    val fnRetrievePartitionOffsets: ju.Set[TopicPartition] => Map[TopicPartition, Long] = { _ => {
+        val listOffsetsParams = partitionTimestamps.map { case (tp, timestamp) =>
+          tp -> OffsetSpec.forTimestamp(timestamp)
+        }.asJava
+        admin.listOffsets(listOffsetsParams, listOffsetsOptions).all().get().asScala.map {
+          case (tp, offsetSpec) =>
+            if (failsOnNoMatchingOffset) {
+              assert(offsetSpec.offset() != OffsetFetchResponse.INVALID_OFFSET, "No offset " +
+                s"matched from request of topic-partition $tp and timestamp " +
+                s"${partitionTimestamps(tp)}.")
+            }
+
+            if (offsetSpec.offset() == OffsetFetchResponse.INVALID_OFFSET) {
+              tp -> KafkaOffsetRangeLimit.LATEST
+            } else {
+              tp -> offsetSpec.offset()
+            }
+        }.toMap
+      }
+    }
+
+    fetchSpecificOffsets0(fnAssertParametersWithPartitions, fnRetrievePartitionOffsets)
+  }
+
+  private def fetchSpecificOffsets0(
+      fnAssertParametersWithPartitions: ju.Set[TopicPartition] => Unit,
+      fnRetrievePartitionOffsets: ju.Set[TopicPartition] => Map[TopicPartition, Long]
+    ): KafkaSourceOffset = {
+    val fetched = partitionsAssignedToAdmin {
+      partitions => {
+        fnAssertParametersWithPartitions(partitions)
+
+        val partitionOffsets = fnRetrievePartitionOffsets(partitions)
+
+        val listOffsetsParams = partitionOffsets.filter { case (_, off) =>
+          off == KafkaOffsetRangeLimit.LATEST || off == KafkaOffsetRangeLimit.EARLIEST
+        }.map { case (tp, off) =>
+          off match {
+            case KafkaOffsetRangeLimit.LATEST =>
+              tp -> OffsetSpec.latest()
+            case KafkaOffsetRangeLimit.EARLIEST =>
+              tp -> OffsetSpec.earliest()
+          }
+        }
+        val resolvedPartitionOffsets = listOffsets(admin, listOffsetsParams.asJava)
+
+        partitionOffsets.map { case (tp, off) =>
+          off match {
+            case KafkaOffsetRangeLimit.LATEST =>
+              tp -> resolvedPartitionOffsets(tp)
+            case KafkaOffsetRangeLimit.EARLIEST =>
+              tp -> resolvedPartitionOffsets(tp)
+            case _ =>
+              tp -> off
+          }
+        }
+      }
+    }
+
+    KafkaSourceOffset(fetched)
+  }
+
+  override def fetchEarliestOffsets(): Map[TopicPartition, Long] = partitionsAssignedToAdmin(
+    partitions => {
+      val listOffsetsParams = partitions.asScala.map(p => p -> OffsetSpec.earliest()).toMap.asJava
+      val partitionOffsets = listOffsets(admin, listOffsetsParams)
+      logDebug(s"Got earliest offsets for partitions: $partitionOffsets")
+      partitionOffsets
+    })
+
+  override def fetchLatestOffsets(
+      knownOffsets: Option[PartitionOffsetMap]): PartitionOffsetMap =
+    partitionsAssignedToAdmin { partitions => {
+      val listOffsetsParams = partitions.asScala.map(_ -> OffsetSpec.latest()).toMap.asJava
+      if (knownOffsets.isEmpty) {
+        val partitionOffsets = listOffsets(admin, listOffsetsParams)
+        logDebug(s"Got latest offsets for partitions: $partitionOffsets")
+        partitionOffsets
+      } else {
+        var partitionOffsets: PartitionOffsetMap = Map.empty
+
+        /**
+         * Compare `knownOffsets` and `partitionOffsets`. Returns all partitions that have incorrect
+         * latest offset (offset in `knownOffsets` is great than the one in `partitionOffsets`).
+         */
+        def findIncorrectOffsets(): Seq[(TopicPartition, Long, Long)] = {
+          var incorrectOffsets = ArrayBuffer[(TopicPartition, Long, Long)]()
+          partitionOffsets.foreach { case (tp, offset) =>
+            knownOffsets.foreach(_.get(tp).foreach { knownOffset =>
+              if (knownOffset > offset) {
+                val incorrectOffset = (tp, knownOffset, offset)
+                incorrectOffsets += incorrectOffset
+              }
+            })
+          }
+          // toSeq seems redundant but it's needed for Scala 2.13
+          incorrectOffsets.toSeq
+        }
+
+        // Retry to fetch latest offsets when detecting incorrect offsets. We don't use
+        // `withRetries` to retry because:
+        //
+        // - `withRetries` will reset the admin for each attempt but a fresh
+        //    admin has a much bigger chance to hit KAFKA-7703 like issues.
+        var incorrectOffsets: Seq[(TopicPartition, Long, Long)] = Nil
+        var attempt = 0
+        do {
+          partitionOffsets = listOffsets(admin, listOffsetsParams)
+          attempt += 1
+
+          incorrectOffsets = findIncorrectOffsets()
+          if (incorrectOffsets.nonEmpty) {
+            logWarning("Found incorrect offsets in some partitions " +
+              s"(partition, previous offset, fetched offset): $incorrectOffsets")
+            if (attempt < maxOffsetFetchAttempts) {
+              logWarning("Retrying to fetch latest offsets because of incorrect offsets")
+              Thread.sleep(offsetFetchAttemptIntervalMs)
+            }
+          }
+        } while (incorrectOffsets.nonEmpty && attempt < maxOffsetFetchAttempts)
+
+        logDebug(s"Got latest offsets for partitions: $partitionOffsets")
+        partitionOffsets
+      }
+    }
+  }
+
+  override def fetchEarliestOffsets(
+      newPartitions: Seq[TopicPartition]): Map[TopicPartition, Long] = {
+    if (newPartitions.isEmpty) {
+      Map.empty[TopicPartition, Long]
+    } else {
+      partitionsAssignedToAdmin(partitions => {
+        // Get the earliest offset of each partition
+        val listOffsetsParams = newPartitions.filter { newPartition =>
+          // When deleting topics happen at the same time, some partitions may not be in
+          // `partitions`. So we need to ignore them
+          partitions.contains(newPartition)
+        }.map(partition => partition -> OffsetSpec.earliest()).toMap.asJava
+        val partitionOffsets = listOffsets(admin, listOffsetsParams)
+        logDebug(s"Got earliest offsets for new partitions: $partitionOffsets")
+        partitionOffsets
+      })
+    }
+  }
+
+  override def getOffsetRangesFromUnresolvedOffsets(
+      startingOffsets: KafkaOffsetRangeLimit,
+      endingOffsets: KafkaOffsetRangeLimit): Seq[KafkaOffsetRange] = {
+    val fromPartitionOffsets = fetchPartitionOffsets(startingOffsets, isStartingOffsets = true)
+    val untilPartitionOffsets = fetchPartitionOffsets(endingOffsets, isStartingOffsets = false)
+
+    // Obtain topicPartitions in both from and until partition offset, ignoring
+    // topic partitions that were added and/or deleted between the two above calls.
+    if (fromPartitionOffsets.keySet != untilPartitionOffsets.keySet) {
+      implicit val topicOrdering: Ordering[TopicPartition] = Ordering.by(t => t.topic())
+      val fromTopics = fromPartitionOffsets.keySet.toList.sorted.mkString(",")
+      val untilTopics = untilPartitionOffsets.keySet.toList.sorted.mkString(",")
+      throw new IllegalStateException("different topic partitions " +
+        s"for starting offsets topics[${fromTopics}] and " +
+        s"ending offsets topics[${untilTopics}]")
+    }
+
+    // Calculate offset ranges
+    val offsetRangesBase = untilPartitionOffsets.keySet.map { tp =>
+      val fromOffset = fromPartitionOffsets.get(tp).getOrElse {
+        // This should not happen since topicPartitions contains all partitions not in
+        // fromPartitionOffsets
+        throw new IllegalStateException(s"$tp doesn't have a from offset")
+      }
+      val untilOffset = untilPartitionOffsets(tp)
+      KafkaOffsetRange(tp, fromOffset, untilOffset, None)
+    }.toSeq
+
+    if (shouldDivvyUpLargePartitions(offsetRangesBase.size)) {
+      val fromOffsetsMap =
+        offsetRangesBase.map(range => (range.topicPartition, range.fromOffset)).toMap
+      val untilOffsetsMap =
+        offsetRangesBase.map(range => (range.topicPartition, range.untilOffset)).toMap
+
+      // No need to report data loss here
+      val resolvedFromOffsets = fetchSpecificOffsets(fromOffsetsMap, _ => ()).partitionToOffsets
+      val resolvedUntilOffsets = fetchSpecificOffsets(untilOffsetsMap, _ => ()).partitionToOffsets
+      val ranges = offsetRangesBase.map(_.topicPartition).map { tp =>
+        KafkaOffsetRange(tp, resolvedFromOffsets(tp), resolvedUntilOffsets(tp), preferredLoc = None)
+      }
+      val divvied = rangeCalculator.getRanges(ranges).groupBy(_.topicPartition)
+      divvied.flatMap { case (tp, splitOffsetRanges) =>
+        if (splitOffsetRanges.length == 1) {
+          Seq(KafkaOffsetRange(tp, fromOffsetsMap(tp), untilOffsetsMap(tp), None))
+        } else {
+          // the list can't be empty
+          val first = splitOffsetRanges.head.copy(fromOffset = fromOffsetsMap(tp))
+          val end = splitOffsetRanges.last.copy(untilOffset = untilOffsetsMap(tp))
+          Seq(first) ++ splitOffsetRanges.drop(1).dropRight(1) :+ end
+        }
+      }.toArray.toSeq
+    } else {
+      offsetRangesBase
+    }
+  }
+
+  private def getSortedExecutorList: Array[String] = {
+    def compare(a: ExecutorCacheTaskLocation, b: ExecutorCacheTaskLocation): Boolean = {
+      if (a.host == b.host) {
+        a.executorId > b.executorId
+      } else {
+        a.host > b.host
+      }
+    }
+
+    val bm = SparkEnv.get.blockManager
+    bm.master.getPeers(bm.blockManagerId).toArray
+      .map(x => ExecutorCacheTaskLocation(x.host, x.executorId))
+      .sortWith(compare)
+      .map(_.toString)
+  }
+
+  override def getOffsetRangesFromResolvedOffsets(
+      fromPartitionOffsets: PartitionOffsetMap,
+      untilPartitionOffsets: PartitionOffsetMap,
+      reportDataLoss: String => Unit): Seq[KafkaOffsetRange] = {
+    // Find the new partitions, and get their earliest offsets
+    val newPartitions = untilPartitionOffsets.keySet.diff(fromPartitionOffsets.keySet)
+    val newPartitionInitialOffsets = fetchEarliestOffsets(newPartitions.toSeq)
+    if (newPartitionInitialOffsets.keySet != newPartitions) {
+      // We cannot get from offsets for some partitions. It means they got deleted.
+      val deletedPartitions = newPartitions.diff(newPartitionInitialOffsets.keySet)
+      reportDataLoss(
+        s"Cannot find earliest offsets of ${deletedPartitions}. Some data may have been missed")
+    }
+    logInfo(s"Partitions added: $newPartitionInitialOffsets")
+    newPartitionInitialOffsets.filter(_._2 != 0).foreach { case (p, o) =>
+      reportDataLoss(
+        s"Added partition $p starts from $o instead of 0. Some data may have been missed")
+    }
+
+    val deletedPartitions = fromPartitionOffsets.keySet.diff(untilPartitionOffsets.keySet)
+    if (deletedPartitions.nonEmpty) {
+      val message = if (driverKafkaParams.containsKey(ConsumerConfig.GROUP_ID_CONFIG)) {
+        s"$deletedPartitions are gone. ${KafkaSourceProvider.CUSTOM_GROUP_ID_ERROR_MESSAGE}"
+      } else {
+        s"$deletedPartitions are gone. Some data may have been missed."
+      }
+      reportDataLoss(message)
+    }
+
+    // Use the until partitions to calculate offset ranges to ignore partitions that have
+    // been deleted
+    val topicPartitions = untilPartitionOffsets.keySet.filter { tp =>
+      // Ignore partitions that we don't know the from offsets.
+      newPartitionInitialOffsets.contains(tp) || fromPartitionOffsets.contains(tp)
+    }.toSeq
+    logDebug("TopicPartitions: " + topicPartitions.mkString(", "))
+
+    val fromOffsets = fromPartitionOffsets ++ newPartitionInitialOffsets
+    val untilOffsets = untilPartitionOffsets
+    val ranges = topicPartitions.map { tp =>
+      val fromOffset = fromOffsets(tp)
+      val untilOffset = untilOffsets(tp)
+      if (untilOffset < fromOffset) {
+        reportDataLoss(s"Partition $tp's offset was changed from " +
+          s"$fromOffset to $untilOffset, some data may have been missed")
+      }
+      KafkaOffsetRange(tp, fromOffset, untilOffset, preferredLoc = None)
+    }
+    rangeCalculator.getRanges(ranges, getSortedExecutorList)
+  }
+
+  private def partitionsAssignedToAdmin(
+      body: ju.Set[TopicPartition] => Map[TopicPartition, Long])
+    : Map[TopicPartition, Long] = {
+
+    withRetries {
+      val partitions = consumerStrategy.assignedTopicPartitions(admin).asJava
+      logDebug(s"Partitions assigned: $partitions.")
+      body(partitions)
+    }
+  }
+
+  /**
+   * Helper function that does multiple retries on a body of code that returns offsets.
+   * Retries are needed to handle transient failures. For e.g. race conditions between getting
+   * assignment and getting position while topics/partitions are deleted can cause NPEs.
+   */
+  private def withRetries(body: => Map[TopicPartition, Long]): Map[TopicPartition, Long] = {
+    synchronized {
+      var result: Option[Map[TopicPartition, Long]] = None
+      var attempt = 1
+      var lastException: Throwable = null
+      while (result.isEmpty && attempt <= maxOffsetFetchAttempts
+        && !Thread.currentThread().isInterrupted) {
+        try {
+          result = Some(body)
+        } catch {
+          case NonFatal(e) =>
+            lastException = e
+            logWarning(s"Error in attempt $attempt getting Kafka offsets: ", e)
+            attempt += 1
+            Thread.sleep(offsetFetchAttemptIntervalMs)
+            resetAdmin()
+        }
+      }
+      if (Thread.interrupted()) {
+        throw new InterruptedException()
+      }
+      if (result.isEmpty) {
+        assert(attempt > maxOffsetFetchAttempts)
+        assert(lastException != null)
+        throw lastException
+      }
+      result.get
+    }
+  }
+
+  private def stopAdmin(): Unit = synchronized {
+    if (_admin != null) _admin.close()
+  }
+
+  private def resetAdmin(): Unit = synchronized {
+    stopAdmin()
+    _admin = null  // will automatically get reinitialized again
+  }
+}
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderConsumer.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderConsumer.scala
new file mode 100644
index 0000000000000..ead819e4c27aa
--- /dev/null
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderConsumer.scala
@@ -0,0 +1,570 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.kafka010
+
+import java.{util => ju}
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+import scala.util.control.NonFatal
+
+import org.apache.kafka.clients.consumer.{Consumer, ConsumerConfig, OffsetAndTimestamp}
+import org.apache.kafka.common.TopicPartition
+
+import org.apache.spark.SparkEnv
+import org.apache.spark.internal.Logging
+import org.apache.spark.scheduler.ExecutorCacheTaskLocation
+import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
+import org.apache.spark.util.{UninterruptibleThread, UninterruptibleThreadRunner}
+
+/**
+ * This class uses Kafka's own [[org.apache.kafka.clients.consumer.KafkaConsumer]] API to
+ * read data offsets from Kafka.
+ * The [[ConsumerStrategy]] class defines which Kafka topics and partitions should be read
+ * by this source. These strategies directly correspond to the different consumption options
+ * in. This class is designed to return a configured
+ * [[org.apache.kafka.clients.consumer.KafkaConsumer]] that is used by the
+ * [[KafkaSource]] to query for the offsets. See the docs on
+ * [[org.apache.spark.sql.kafka010.ConsumerStrategy]]
+ * for more details.
+ *
+ * Note: This class is not ThreadSafe
+ */
+private[kafka010] class KafkaOffsetReaderConsumer(
+    consumerStrategy: ConsumerStrategy,
+    override val driverKafkaParams: ju.Map[String, Object],
+    readerOptions: CaseInsensitiveMap[String],
+    driverGroupIdPrefix: String) extends KafkaOffsetReader with Logging {
+
+  /**
+   * [[UninterruptibleThreadRunner]] ensures that all
+   * [[org.apache.kafka.clients.consumer.KafkaConsumer]] communication called in an
+   * [[UninterruptibleThread]]. In the case of streaming queries, we are already running in an
+   * [[UninterruptibleThread]], however for batch mode this is not the case.
+   */
+  val uninterruptibleThreadRunner = new UninterruptibleThreadRunner("Kafka Offset Reader")
+
+  /**
+   * Place [[groupId]] and [[nextId]] here so that they are initialized before any consumer is
+   * created -- see SPARK-19564.
+   */
+  private var groupId: String = null
+  private var nextId = 0
+
+  /**
+   * A KafkaConsumer used in the driver to query the latest Kafka offsets. This only queries the
+   * offsets and never commits them.
+   */
+  @volatile protected var _consumer: Consumer[Array[Byte], Array[Byte]] = null
+
+  protected def consumer: Consumer[Array[Byte], Array[Byte]] = synchronized {
+    assert(Thread.currentThread().isInstanceOf[UninterruptibleThread])
+    if (_consumer == null) {
+      val newKafkaParams = new ju.HashMap[String, Object](driverKafkaParams)
+      if (driverKafkaParams.get(ConsumerConfig.GROUP_ID_CONFIG) == null) {
+        newKafkaParams.put(ConsumerConfig.GROUP_ID_CONFIG, nextGroupId())
+      }
+      _consumer = consumerStrategy.createConsumer(newKafkaParams)
+    }
+    _consumer
+  }
+
+  private[kafka010] val maxOffsetFetchAttempts =
+    readerOptions.getOrElse(KafkaSourceProvider.FETCH_OFFSET_NUM_RETRY, "3").toInt
+
+  /**
+   * Number of partitions to read from Kafka. If this value is greater than the number of Kafka
+   * topicPartitions, we will split up  the read tasks of the skewed partitions to multiple Spark
+   * tasks. The number of Spark tasks will be *approximately* `numPartitions`. It can be less or
+   * more depending on rounding errors or Kafka partitions that didn't receive any new data.
+   */
+  private val minPartitions =
+    readerOptions.get(KafkaSourceProvider.MIN_PARTITIONS_OPTION_KEY).map(_.toInt)
+
+  private val rangeCalculator = new KafkaOffsetRangeCalculator(minPartitions)
+
+  private[kafka010] val offsetFetchAttemptIntervalMs =
+    readerOptions.getOrElse(KafkaSourceProvider.FETCH_OFFSET_RETRY_INTERVAL_MS, "1000").toLong
+
+  /**
+   * Whether we should divide Kafka TopicPartitions with a lot of data into smaller Spark tasks.
+   */
+  private def shouldDivvyUpLargePartitions(numTopicPartitions: Int): Boolean = {
+    minPartitions.map(_ > numTopicPartitions).getOrElse(false)
+  }
+
+  private def nextGroupId(): String = {
+    groupId = driverGroupIdPrefix + "-" + nextId
+    nextId += 1
+    groupId
+  }
+
+  override def toString(): String = consumerStrategy.toString
+
+  override def close(): Unit = {
+    if (_consumer != null) uninterruptibleThreadRunner.runUninterruptibly { stopConsumer() }
+    uninterruptibleThreadRunner.shutdown()
+  }
+
+  /**
+   * @return The Set of TopicPartitions for a given topic
+   */
+  private def fetchTopicPartitions(): Set[TopicPartition] =
+    uninterruptibleThreadRunner.runUninterruptibly {
+      assert(Thread.currentThread().isInstanceOf[UninterruptibleThread])
+      // Poll to get the latest assigned partitions
+      consumer.poll(0)
+      val partitions = consumer.assignment()
+      consumer.pause(partitions)
+      partitions.asScala.toSet
+  }
+
+  override def fetchPartitionOffsets(
+      offsetRangeLimit: KafkaOffsetRangeLimit,
+      isStartingOffsets: Boolean): Map[TopicPartition, Long] = {
+    def validateTopicPartitions(partitions: Set[TopicPartition],
+      partitionOffsets: Map[TopicPartition, Long]): Map[TopicPartition, Long] = {
+      assert(partitions == partitionOffsets.keySet,
+        "If startingOffsets contains specific offsets, you must specify all TopicPartitions.\n" +
+          "Use -1 for latest, -2 for earliest.\n" +
+          s"Specified: ${partitionOffsets.keySet} Assigned: ${partitions}")
+      logDebug(s"Partitions assigned to consumer: $partitions. Seeking to $partitionOffsets")
+      partitionOffsets
+    }
+    val partitions = fetchTopicPartitions()
+    // Obtain TopicPartition offsets with late binding support
+    offsetRangeLimit match {
+      case EarliestOffsetRangeLimit => partitions.map {
+        case tp => tp -> KafkaOffsetRangeLimit.EARLIEST
+      }.toMap
+      case LatestOffsetRangeLimit => partitions.map {
+        case tp => tp -> KafkaOffsetRangeLimit.LATEST
+      }.toMap
+      case SpecificOffsetRangeLimit(partitionOffsets) =>
+        validateTopicPartitions(partitions, partitionOffsets)
+      case SpecificTimestampRangeLimit(partitionTimestamps) =>
+        fetchSpecificTimestampBasedOffsets(partitionTimestamps,
+          failsOnNoMatchingOffset = isStartingOffsets).partitionToOffsets
+    }
+  }
+
+  override def fetchSpecificOffsets(
+      partitionOffsets: Map[TopicPartition, Long],
+      reportDataLoss: String => Unit): KafkaSourceOffset = {
+    val fnAssertParametersWithPartitions: ju.Set[TopicPartition] => Unit = { partitions =>
+      assert(partitions.asScala == partitionOffsets.keySet,
+        "If startingOffsets contains specific offsets, you must specify all TopicPartitions.\n" +
+          "Use -1 for latest, -2 for earliest, if you don't care.\n" +
+          s"Specified: ${partitionOffsets.keySet} Assigned: ${partitions.asScala}")
+      logDebug(s"Partitions assigned to consumer: $partitions. Seeking to $partitionOffsets")
+    }
+
+    val fnRetrievePartitionOffsets: ju.Set[TopicPartition] => Map[TopicPartition, Long] = { _ =>
+      partitionOffsets
+    }
+
+    val fnAssertFetchedOffsets: Map[TopicPartition, Long] => Unit = { fetched =>
+      partitionOffsets.foreach {
+        case (tp, off) if off != KafkaOffsetRangeLimit.LATEST &&
+          off != KafkaOffsetRangeLimit.EARLIEST =>
+          if (fetched(tp) != off) {
+            reportDataLoss(
+              s"startingOffsets for $tp was $off but consumer reset to ${fetched(tp)}")
+          }
+        case _ =>
+        // no real way to check that beginning or end is reasonable
+      }
+    }
+
+    fetchSpecificOffsets0(fnAssertParametersWithPartitions, fnRetrievePartitionOffsets,
+      fnAssertFetchedOffsets)
+  }
+
+  override def fetchSpecificTimestampBasedOffsets(
+      partitionTimestamps: Map[TopicPartition, Long],
+      failsOnNoMatchingOffset: Boolean): KafkaSourceOffset = {
+    val fnAssertParametersWithPartitions: ju.Set[TopicPartition] => Unit = { partitions =>
+      assert(partitions.asScala == partitionTimestamps.keySet,
+        "If starting/endingOffsetsByTimestamp contains specific offsets, you must specify all " +
+          s"topics. Specified: ${partitionTimestamps.keySet} Assigned: ${partitions.asScala}")
+      logDebug(s"Partitions assigned to consumer: $partitions. Seeking to $partitionTimestamps")
+    }
+
+    val fnRetrievePartitionOffsets: ju.Set[TopicPartition] => Map[TopicPartition, Long] = { _ => {
+        val converted = partitionTimestamps.map { case (tp, timestamp) =>
+          tp -> java.lang.Long.valueOf(timestamp)
+        }.asJava
+
+        val offsetForTime: ju.Map[TopicPartition, OffsetAndTimestamp] =
+          consumer.offsetsForTimes(converted)
+
+        offsetForTime.asScala.map { case (tp, offsetAndTimestamp) =>
+          if (failsOnNoMatchingOffset) {
+            assert(offsetAndTimestamp != null, "No offset matched from request of " +
+              s"topic-partition $tp and timestamp ${partitionTimestamps(tp)}.")
+          }
+
+          if (offsetAndTimestamp == null) {
+            tp -> KafkaOffsetRangeLimit.LATEST
+          } else {
+            tp -> offsetAndTimestamp.offset()
+          }
+        }.toMap
+      }
+    }
+
+    val fnAssertFetchedOffsets: Map[TopicPartition, Long] => Unit = { _ => }
+
+    fetchSpecificOffsets0(fnAssertParametersWithPartitions, fnRetrievePartitionOffsets,
+      fnAssertFetchedOffsets)
+  }
+
+  private def fetchSpecificOffsets0(
+      fnAssertParametersWithPartitions: ju.Set[TopicPartition] => Unit,
+      fnRetrievePartitionOffsets: ju.Set[TopicPartition] => Map[TopicPartition, Long],
+      fnAssertFetchedOffsets: Map[TopicPartition, Long] => Unit): KafkaSourceOffset = {
+    val fetched = partitionsAssignedToConsumer {
+      partitions => {
+        fnAssertParametersWithPartitions(partitions)
+
+        val partitionOffsets = fnRetrievePartitionOffsets(partitions)
+
+        partitionOffsets.foreach {
+          case (tp, KafkaOffsetRangeLimit.LATEST) =>
+            consumer.seekToEnd(ju.Arrays.asList(tp))
+          case (tp, KafkaOffsetRangeLimit.EARLIEST) =>
+            consumer.seekToBeginning(ju.Arrays.asList(tp))
+          case (tp, off) => consumer.seek(tp, off)
+        }
+
+        partitionOffsets.map {
+          case (tp, _) => tp -> consumer.position(tp)
+        }
+      }
+    }
+
+    fnAssertFetchedOffsets(fetched)
+
+    KafkaSourceOffset(fetched)
+  }
+
+  override def fetchEarliestOffsets(): Map[TopicPartition, Long] = partitionsAssignedToConsumer(
+    partitions => {
+      logDebug("Seeking to the beginning")
+
+      consumer.seekToBeginning(partitions)
+      val partitionOffsets = partitions.asScala.map(p => p -> consumer.position(p)).toMap
+      logDebug(s"Got earliest offsets for partition : $partitionOffsets")
+      partitionOffsets
+    }, fetchingEarliestOffset = true)
+
+  /**
+   * Specific to `KafkaOffsetReaderConsumer`:
+   * Kafka may return earliest offsets when we are requesting latest offsets if `poll` is called
+   * right before `seekToEnd` (KAFKA-7703). As a workaround, we will call `position` right after
+   * `poll` to wait until the potential offset request triggered by `poll(0)` is done.
+   */
+  override def fetchLatestOffsets(
+      knownOffsets: Option[PartitionOffsetMap]): PartitionOffsetMap =
+    partitionsAssignedToConsumer { partitions => {
+      logDebug("Seeking to the end.")
+
+      if (knownOffsets.isEmpty) {
+        consumer.seekToEnd(partitions)
+        partitions.asScala.map(p => p -> consumer.position(p)).toMap
+      } else {
+        var partitionOffsets: PartitionOffsetMap = Map.empty
+
+        /**
+         * Compare `knownOffsets` and `partitionOffsets`. Returns all partitions that have incorrect
+         * latest offset (offset in `knownOffsets` is great than the one in `partitionOffsets`).
+         */
+        def findIncorrectOffsets(): Seq[(TopicPartition, Long, Long)] = {
+          var incorrectOffsets = ArrayBuffer[(TopicPartition, Long, Long)]()
+          partitionOffsets.foreach { case (tp, offset) =>
+            knownOffsets.foreach(_.get(tp).foreach { knownOffset =>
+              if (knownOffset > offset) {
+                val incorrectOffset = (tp, knownOffset, offset)
+                incorrectOffsets += incorrectOffset
+              }
+            })
+          }
+          incorrectOffsets.toSeq
+        }
+
+        // Retry to fetch latest offsets when detecting incorrect offsets. We don't use
+        // `withRetriesWithoutInterrupt` to retry because:
+        //
+        // - `withRetriesWithoutInterrupt` will reset the consumer for each attempt but a fresh
+        //    consumer has a much bigger chance to hit KAFKA-7703.
+        // - Avoid calling `consumer.poll(0)` which may cause KAFKA-7703.
+        var incorrectOffsets: Seq[(TopicPartition, Long, Long)] = Nil
+        var attempt = 0
+        do {
+          consumer.seekToEnd(partitions)
+          partitionOffsets = partitions.asScala.map(p => p -> consumer.position(p)).toMap
+          attempt += 1
+
+          incorrectOffsets = findIncorrectOffsets()
+          if (incorrectOffsets.nonEmpty) {
+            logWarning("Found incorrect offsets in some partitions " +
+              s"(partition, previous offset, fetched offset): $incorrectOffsets")
+            if (attempt < maxOffsetFetchAttempts) {
+              logWarning("Retrying to fetch latest offsets because of incorrect offsets")
+              Thread.sleep(offsetFetchAttemptIntervalMs)
+            }
+          }
+        } while (incorrectOffsets.nonEmpty && attempt < maxOffsetFetchAttempts)
+
+        logDebug(s"Got latest offsets for partition : $partitionOffsets")
+        partitionOffsets
+      }
+    }
+  }
+
+  override def fetchEarliestOffsets(
+      newPartitions: Seq[TopicPartition]): Map[TopicPartition, Long] = {
+    if (newPartitions.isEmpty) {
+      Map.empty[TopicPartition, Long]
+    } else {
+      partitionsAssignedToConsumer(partitions => {
+        // Get the earliest offset of each partition
+        consumer.seekToBeginning(partitions)
+        val partitionOffsets = newPartitions.filter { p =>
+          // When deleting topics happen at the same time, some partitions may not be in
+          // `partitions`. So we need to ignore them
+          partitions.contains(p)
+        }.map(p => p -> consumer.position(p)).toMap
+        logDebug(s"Got earliest offsets for new partitions: $partitionOffsets")
+        partitionOffsets
+      }, fetchingEarliestOffset = true)
+    }
+  }
+
+  override def getOffsetRangesFromUnresolvedOffsets(
+      startingOffsets: KafkaOffsetRangeLimit,
+      endingOffsets: KafkaOffsetRangeLimit): Seq[KafkaOffsetRange] = {
+    val fromPartitionOffsets = fetchPartitionOffsets(startingOffsets, isStartingOffsets = true)
+    val untilPartitionOffsets = fetchPartitionOffsets(endingOffsets, isStartingOffsets = false)
+
+    // Obtain topicPartitions in both from and until partition offset, ignoring
+    // topic partitions that were added and/or deleted between the two above calls.
+    if (fromPartitionOffsets.keySet != untilPartitionOffsets.keySet) {
+      implicit val topicOrdering: Ordering[TopicPartition] = Ordering.by(t => t.topic())
+      val fromTopics = fromPartitionOffsets.keySet.toList.sorted.mkString(",")
+      val untilTopics = untilPartitionOffsets.keySet.toList.sorted.mkString(",")
+      throw new IllegalStateException("different topic partitions " +
+        s"for starting offsets topics[${fromTopics}] and " +
+        s"ending offsets topics[${untilTopics}]")
+    }
+
+    // Calculate offset ranges
+    val offsetRangesBase = untilPartitionOffsets.keySet.map { tp =>
+      val fromOffset = fromPartitionOffsets.get(tp).getOrElse {
+        // This should not happen since topicPartitions contains all partitions not in
+        // fromPartitionOffsets
+        throw new IllegalStateException(s"$tp doesn't have a from offset")
+      }
+      val untilOffset = untilPartitionOffsets(tp)
+      KafkaOffsetRange(tp, fromOffset, untilOffset, None)
+    }.toSeq
+
+    if (shouldDivvyUpLargePartitions(offsetRangesBase.size)) {
+      val fromOffsetsMap =
+        offsetRangesBase.map(range => (range.topicPartition, range.fromOffset)).toMap
+      val untilOffsetsMap =
+        offsetRangesBase.map(range => (range.topicPartition, range.untilOffset)).toMap
+
+      // No need to report data loss here
+      val resolvedFromOffsets = fetchSpecificOffsets(fromOffsetsMap, _ => ()).partitionToOffsets
+      val resolvedUntilOffsets = fetchSpecificOffsets(untilOffsetsMap, _ => ()).partitionToOffsets
+      val ranges = offsetRangesBase.map(_.topicPartition).map { tp =>
+        KafkaOffsetRange(tp, resolvedFromOffsets(tp), resolvedUntilOffsets(tp), preferredLoc = None)
+      }
+      val divvied = rangeCalculator.getRanges(ranges).groupBy(_.topicPartition)
+      divvied.flatMap { case (tp, splitOffsetRanges) =>
+        if (splitOffsetRanges.length == 1) {
+          Seq(KafkaOffsetRange(tp, fromOffsetsMap(tp), untilOffsetsMap(tp), None))
+        } else {
+          // the list can't be empty
+          val first = splitOffsetRanges.head.copy(fromOffset = fromOffsetsMap(tp))
+          val end = splitOffsetRanges.last.copy(untilOffset = untilOffsetsMap(tp))
+          Seq(first) ++ splitOffsetRanges.drop(1).dropRight(1) :+ end
+        }
+      }.toArray.toSeq
+    } else {
+      offsetRangesBase
+    }
+  }
+
+  private def getSortedExecutorList(): Array[String] = {
+    def compare(a: ExecutorCacheTaskLocation, b: ExecutorCacheTaskLocation): Boolean = {
+      if (a.host == b.host) {
+        a.executorId > b.executorId
+      } else {
+        a.host > b.host
+      }
+    }
+
+    val bm = SparkEnv.get.blockManager
+    bm.master.getPeers(bm.blockManagerId).toArray
+      .map(x => ExecutorCacheTaskLocation(x.host, x.executorId))
+      .sortWith(compare)
+      .map(_.toString)
+  }
+
+  override def getOffsetRangesFromResolvedOffsets(
+      fromPartitionOffsets: PartitionOffsetMap,
+      untilPartitionOffsets: PartitionOffsetMap,
+      reportDataLoss: String => Unit): Seq[KafkaOffsetRange] = {
+    // Find the new partitions, and get their earliest offsets
+    val newPartitions = untilPartitionOffsets.keySet.diff(fromPartitionOffsets.keySet)
+    val newPartitionInitialOffsets = fetchEarliestOffsets(newPartitions.toSeq)
+    if (newPartitionInitialOffsets.keySet != newPartitions) {
+      // We cannot get from offsets for some partitions. It means they got deleted.
+      val deletedPartitions = newPartitions.diff(newPartitionInitialOffsets.keySet)
+      reportDataLoss(
+        s"Cannot find earliest offsets of ${deletedPartitions}. Some data may have been missed")
+    }
+    logInfo(s"Partitions added: $newPartitionInitialOffsets")
+    newPartitionInitialOffsets.filter(_._2 != 0).foreach { case (p, o) =>
+      reportDataLoss(
+        s"Added partition $p starts from $o instead of 0. Some data may have been missed")
+    }
+
+    val deletedPartitions = fromPartitionOffsets.keySet.diff(untilPartitionOffsets.keySet)
+    if (deletedPartitions.nonEmpty) {
+      val message = if (driverKafkaParams.containsKey(ConsumerConfig.GROUP_ID_CONFIG)) {
+        s"$deletedPartitions are gone. ${KafkaSourceProvider.CUSTOM_GROUP_ID_ERROR_MESSAGE}"
+      } else {
+        s"$deletedPartitions are gone. Some data may have been missed."
+      }
+      reportDataLoss(message)
+    }
+
+    // Use the until partitions to calculate offset ranges to ignore partitions that have
+    // been deleted
+    val topicPartitions = untilPartitionOffsets.keySet.filter { tp =>
+      // Ignore partitions that we don't know the from offsets.
+      newPartitionInitialOffsets.contains(tp) || fromPartitionOffsets.contains(tp)
+    }.toSeq
+    logDebug("TopicPartitions: " + topicPartitions.mkString(", "))
+
+    val fromOffsets = fromPartitionOffsets ++ newPartitionInitialOffsets
+    val untilOffsets = untilPartitionOffsets
+    val ranges = topicPartitions.map { tp =>
+      val fromOffset = fromOffsets(tp)
+      val untilOffset = untilOffsets(tp)
+      if (untilOffset < fromOffset) {
+        reportDataLoss(s"Partition $tp's offset was changed from " +
+          s"$fromOffset to $untilOffset, some data may have been missed")
+      }
+      KafkaOffsetRange(tp, fromOffset, untilOffset, preferredLoc = None)
+    }
+    rangeCalculator.getRanges(ranges, getSortedExecutorList)
+  }
+
+  private def partitionsAssignedToConsumer(
+      body: ju.Set[TopicPartition] => Map[TopicPartition, Long],
+      fetchingEarliestOffset: Boolean = false)
+    : Map[TopicPartition, Long] = uninterruptibleThreadRunner.runUninterruptibly {
+
+    withRetriesWithoutInterrupt {
+      // Poll to get the latest assigned partitions
+      consumer.poll(0)
+      val partitions = consumer.assignment()
+
+      if (!fetchingEarliestOffset) {
+        // Call `position` to wait until the potential offset request triggered by `poll(0)` is
+        // done. This is a workaround for KAFKA-7703, which an async `seekToBeginning` triggered by
+        // `poll(0)` may reset offsets that should have been set by another request.
+        partitions.asScala.map(p => p -> consumer.position(p)).foreach(_ => {})
+      }
+
+      consumer.pause(partitions)
+      logDebug(s"Partitions assigned to consumer: $partitions.")
+      body(partitions)
+    }
+  }
+
+  /**
+   * Helper function that does multiple retries on a body of code that returns offsets.
+   * Retries are needed to handle transient failures. For e.g. race conditions between getting
+   * assignment and getting position while topics/partitions are deleted can cause NPEs.
+   *
+   * This method also makes sure `body` won't be interrupted to workaround a potential issue in
+   * `KafkaConsumer.poll`. (KAFKA-1894)
+   */
+  private def withRetriesWithoutInterrupt(
+      body: => Map[TopicPartition, Long]): Map[TopicPartition, Long] = {
+    // Make sure `KafkaConsumer.poll` won't be interrupted (KAFKA-1894)
+    assert(Thread.currentThread().isInstanceOf[UninterruptibleThread])
+
+    synchronized {
+      var result: Option[Map[TopicPartition, Long]] = None
+      var attempt = 1
+      var lastException: Throwable = null
+      while (result.isEmpty && attempt <= maxOffsetFetchAttempts
+        && !Thread.currentThread().isInterrupted) {
+        Thread.currentThread match {
+          case ut: UninterruptibleThread =>
+            // "KafkaConsumer.poll" may hang forever if the thread is interrupted (E.g., the query
+            // is stopped)(KAFKA-1894). Hence, we just make sure we don't interrupt it.
+            //
+            // If the broker addresses are wrong, or Kafka cluster is down, "KafkaConsumer.poll" may
+            // hang forever as well. This cannot be resolved in KafkaSource until Kafka fixes the
+            // issue.
+            ut.runUninterruptibly {
+              try {
+                result = Some(body)
+              } catch {
+                case NonFatal(e) =>
+                  lastException = e
+                  logWarning(s"Error in attempt $attempt getting Kafka offsets: ", e)
+                  attempt += 1
+                  Thread.sleep(offsetFetchAttemptIntervalMs)
+                  resetConsumer()
+              }
+            }
+          case _ =>
+            throw new IllegalStateException(
+              "Kafka APIs must be executed on a o.a.spark.util.UninterruptibleThread")
+        }
+      }
+      if (Thread.interrupted()) {
+        throw new InterruptedException()
+      }
+      if (result.isEmpty) {
+        assert(attempt > maxOffsetFetchAttempts)
+        assert(lastException != null)
+        throw lastException
+      }
+      result.get
+    }
+  }
+
+  private def stopConsumer(): Unit = synchronized {
+    assert(Thread.currentThread().isInstanceOf[UninterruptibleThread])
+    if (_consumer != null) _consumer.close()
+  }
+
+  private def resetConsumer(): Unit = synchronized {
+    stopConsumer()
+    _consumer = null  // will automatically get reinitialized again
+  }
+}
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaRelation.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaRelation.scala
index 413a0c4de8bea..ed3407c822b96 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaRelation.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaRelation.scala
@@ -17,13 +17,10 @@
 
 package org.apache.spark.sql.kafka010
 
-import org.apache.kafka.common.TopicPartition
-
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.Network.NETWORK_TIMEOUT
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{Row, SQLContext}
-import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
 import org.apache.spark.sql.sources.{BaseRelation, TableScan}
 import org.apache.spark.sql.types.StructType
@@ -59,7 +56,7 @@ private[kafka010] class KafkaRelation(
     // id. Hence, we should generate a unique id for each query.
     val uniqueGroupId = KafkaSourceProvider.batchUniqueGroupId(sourceOptions)
 
-    val kafkaOffsetReader = new KafkaOffsetReader(
+    val kafkaOffsetReader = KafkaOffsetReader.build(
       strategy,
       KafkaSourceProvider.kafkaParamsForDriver(specifiedKafkaParams),
       sourceOptions,
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSource.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSource.scala
index 57879c7ca31cf..1e17f9a7407a9 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSource.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSource.scala
@@ -18,11 +18,7 @@
 package org.apache.spark.sql.kafka010
 
 import java.{util => ju}
-import java.io._
-import java.nio.charset.StandardCharsets
 
-import org.apache.commons.io.IOUtils
-import org.apache.kafka.clients.consumer.ConsumerConfig
 import org.apache.kafka.common.TopicPartition
 
 import org.apache.spark.SparkContext
@@ -35,7 +31,6 @@ import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
 import org.apache.spark.sql.connector.read.streaming
 import org.apache.spark.sql.connector.read.streaming.{ReadAllAvailable, ReadLimit, ReadMaxRows, SupportsAdmissionControl}
 import org.apache.spark.sql.execution.streaming._
-import org.apache.spark.sql.kafka010.KafkaSource._
 import org.apache.spark.sql.kafka010.KafkaSourceProvider._
 import org.apache.spark.sql.types._
 
@@ -120,8 +115,13 @@ private[kafka010] class KafkaSource(
     maxOffsetsPerTrigger.map(ReadLimit.maxRows).getOrElse(super.getDefaultReadLimit)
   }
 
+  // The offsets for each topic-partition currently read to process. Note this maybe not necessarily
+  // to be latest offsets because we possibly apply a read limit.
   private var currentPartitionOffsets: Option[Map[TopicPartition, Long]] = None
 
+  // The latest offsets for each topic-partition.
+  private var latestPartitionOffsets: Option[Map[TopicPartition, Long]] = None
+
   private val converter = new KafkaRecordToRowConverter()
 
   override def schema: StructType = KafkaRecordToRowConverter.kafkaSchema(includeHeaders)
@@ -132,6 +132,10 @@ private[kafka010] class KafkaSource(
       "latestOffset(Offset, ReadLimit) should be called instead of this method")
   }
 
+  override def reportLatestOffset(): streaming.Offset = {
+    latestPartitionOffsets.map(KafkaSourceOffset(_)).getOrElse(null)
+  }
+
   override def latestOffset(startOffset: streaming.Offset, limit: ReadLimit): streaming.Offset = {
     // Make sure initialPartitionOffsets is initialized
     initialPartitionOffsets
@@ -150,6 +154,7 @@ private[kafka010] class KafkaSource(
     }
 
     currentPartitionOffsets = Some(offsets)
+    latestPartitionOffsets = Some(latest)
     logDebug(s"GetOffset: ${offsets.toSeq.map(_.toString).sorted}")
     KafkaSourceOffset(offsets)
   }
@@ -159,7 +164,7 @@ private[kafka010] class KafkaSource(
       limit: Long,
       from: Map[TopicPartition, Long],
       until: Map[TopicPartition, Long]): Map[TopicPartition, Long] = {
-    val fromNew = kafkaReader.fetchEarliestOffsets(until.keySet.diff(from.keySet).toSeq)
+    lazy val fromNew = kafkaReader.fetchEarliestOffsets(until.keySet.diff(from.keySet).toSeq)
     val sizes = until.flatMap {
       case (tp, end) =>
         // If begin isn't defined, something's wrong, but let alert logic in getBatch handle it
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceOffset.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceOffset.scala
index b9674a30aee39..8790a77e5d895 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceOffset.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceOffset.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.kafka010
 
 import org.apache.kafka.common.TopicPartition
 
+import org.apache.spark.sql.connector.read.streaming
 import org.apache.spark.sql.connector.read.streaming.PartitionOffset
 import org.apache.spark.sql.execution.streaming.{Offset, SerializedOffset}
 
@@ -62,4 +63,17 @@ private[kafka010] object KafkaSourceOffset {
    */
   def apply(offset: SerializedOffset): KafkaSourceOffset =
     KafkaSourceOffset(JsonUtils.partitionOffsets(offset.json))
+
+  /**
+   * Returns [[KafkaSourceOffset]] from a streaming.Offset
+   */
+  def apply(offset: streaming.Offset): KafkaSourceOffset = {
+    offset match {
+      case k: KafkaSourceOffset => k
+      case so: SerializedOffset => apply(so)
+      case _ =>
+        throw new IllegalArgumentException(
+          s"Invalid conversion from offset of ${offset.getClass} to KafkaSourceOffset")
+    }
+  }
 }
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala
index a5e5d01152db8..0c891cd725ae3 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala
@@ -30,13 +30,13 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.kafka010.KafkaConfigUpdater
 import org.apache.spark.sql.{AnalysisException, DataFrame, SaveMode, SQLContext}
 import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
-import org.apache.spark.sql.connector.catalog.{SupportsRead, SupportsWrite, Table, TableCapability, TableProvider}
+import org.apache.spark.sql.connector.catalog.{SupportsRead, SupportsWrite, Table, TableCapability}
+import org.apache.spark.sql.connector.metric.{CustomMetric, CustomSumMetric}
 import org.apache.spark.sql.connector.read.{Batch, Scan, ScanBuilder}
 import org.apache.spark.sql.connector.read.streaming.{ContinuousStream, MicroBatchStream}
-import org.apache.spark.sql.connector.write.{BatchWrite, LogicalWriteInfo, WriteBuilder}
-import org.apache.spark.sql.connector.write.streaming.StreamingWrite
+import org.apache.spark.sql.connector.write.{LogicalWriteInfo, SupportsTruncate, Write, WriteBuilder}
 import org.apache.spark.sql.execution.streaming.{Sink, Source}
-import org.apache.spark.sql.internal.connector.SimpleTableProvider
+import org.apache.spark.sql.internal.connector.{SimpleTableProvider, SupportsStreamingUpdateAsAppend}
 import org.apache.spark.sql.sources._
 import org.apache.spark.sql.streaming.OutputMode
 import org.apache.spark.sql.types.StructType
@@ -93,7 +93,7 @@ private[kafka010] class KafkaSourceProvider extends DataSourceRegister
       caseInsensitiveParameters, STARTING_OFFSETS_BY_TIMESTAMP_OPTION_KEY,
       STARTING_OFFSETS_OPTION_KEY, LatestOffsetRangeLimit)
 
-    val kafkaOffsetReader = new KafkaOffsetReader(
+    val kafkaOffsetReader = KafkaOffsetReader.build(
       strategy(caseInsensitiveParameters),
       kafkaParamsForDriver(specifiedKafkaParams),
       caseInsensitiveParameters,
@@ -394,22 +394,16 @@ private[kafka010] class KafkaSourceProvider extends DataSourceRegister
       () => new KafkaScan(options)
 
     override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
-      new WriteBuilder {
+      new WriteBuilder with SupportsTruncate with SupportsStreamingUpdateAsAppend {
         private val options = info.options
         private val inputSchema: StructType = info.schema()
         private val topic = Option(options.get(TOPIC_OPTION_KEY)).map(_.trim)
         private val producerParams =
           kafkaParamsForProducer(CaseInsensitiveMap(options.asScala.toMap))
 
-        override def buildForBatch(): BatchWrite = {
-          assert(inputSchema != null)
-          new KafkaBatchWrite(topic, producerParams, inputSchema)
-        }
+        override def build(): Write = KafkaWrite(topic, producerParams, inputSchema)
 
-        override def buildForStreaming(): StreamingWrite = {
-          assert(inputSchema != null)
-          new KafkaStreamingWrite(topic, producerParams, inputSchema)
-        }
+        override def truncate(): WriteBuilder = this
       }
     }
   }
@@ -458,7 +452,7 @@ private[kafka010] class KafkaSourceProvider extends DataSourceRegister
         caseInsensitiveOptions, STARTING_OFFSETS_BY_TIMESTAMP_OPTION_KEY,
         STARTING_OFFSETS_OPTION_KEY, LatestOffsetRangeLimit)
 
-      val kafkaOffsetReader = new KafkaOffsetReader(
+      val kafkaOffsetReader = KafkaOffsetReader.build(
         strategy(caseInsensitiveOptions),
         kafkaParamsForDriver(specifiedKafkaParams),
         caseInsensitiveOptions,
@@ -487,7 +481,7 @@ private[kafka010] class KafkaSourceProvider extends DataSourceRegister
         caseInsensitiveOptions, STARTING_OFFSETS_BY_TIMESTAMP_OPTION_KEY,
         STARTING_OFFSETS_OPTION_KEY, LatestOffsetRangeLimit)
 
-      val kafkaOffsetReader = new KafkaOffsetReader(
+      val kafkaOffsetReader = KafkaOffsetReader.build(
         strategy(caseInsensitiveOptions),
         kafkaParamsForDriver(specifiedKafkaParams),
         caseInsensitiveOptions,
@@ -501,9 +495,23 @@ private[kafka010] class KafkaSourceProvider extends DataSourceRegister
         startingStreamOffsets,
         failOnDataLoss(caseInsensitiveOptions))
     }
+
+    override def supportedCustomMetrics(): Array[CustomMetric] = {
+      Array(new OffsetOutOfRangeMetric, new DataLossMetric)
+    }
   }
 }
 
+private[spark] class OffsetOutOfRangeMetric extends CustomSumMetric {
+  override def name(): String = "offsetOutOfRange"
+  override def description(): String = "estimated number of fetched offsets out of range"
+}
+
+private[spark] class DataLossMetric extends CustomSumMetric {
+  override def name(): String = "dataLoss"
+  override def description(): String = "number of data loss error"
+}
+
 private[kafka010] object KafkaSourceProvider extends Logging {
   private val ASSIGN = "assign"
   private val SUBSCRIBE_PATTERN = "subscribepattern"
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaWrite.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaWrite.scala
new file mode 100644
index 0000000000000..8e0e0516d871a
--- /dev/null
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaWrite.scala
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.kafka010
+
+import java.{util => ju}
+
+import org.apache.spark.sql.connector.write.{BatchWrite, Write}
+import org.apache.spark.sql.connector.write.streaming.StreamingWrite
+import org.apache.spark.sql.types.StructType
+
+case class KafkaWrite(
+    topic: Option[String],
+    producerParams: ju.Map[String, Object],
+    schema: StructType) extends Write {
+
+  override def description(): String = "Kafka"
+
+  override def toBatch: BatchWrite = {
+    assert(schema != null)
+    new KafkaBatchWrite(topic, producerParams, schema)
+  }
+
+  override def toStreaming: StreamingWrite = {
+    assert(schema != null)
+    new KafkaStreamingWrite(topic, producerParams, schema)
+  }
+}
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/FetchedDataPool.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/FetchedDataPool.scala
index 6174bfb203429..3e6831770a674 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/FetchedDataPool.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/FetchedDataPool.scala
@@ -28,7 +28,7 @@ import org.apache.kafka.clients.consumer.ConsumerRecord
 import org.apache.spark.SparkConf
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.kafka010.{FETCHED_DATA_CACHE_EVICTOR_THREAD_RUN_INTERVAL, FETCHED_DATA_CACHE_TIMEOUT}
-import org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.{CacheKey, UNKNOWN_OFFSET}
+import org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.{AvailableOffsetRange, CacheKey, UNKNOWN_OFFSET}
 import org.apache.spark.util.{Clock, SystemClock, ThreadUtils, Utils}
 
 /**
@@ -174,7 +174,8 @@ private[consumer] object FetchedDataPool {
       val emptyData = FetchedData(
         ju.Collections.emptyListIterator[ConsumerRecord[Array[Byte], Array[Byte]]],
         UNKNOWN_OFFSET,
-        UNKNOWN_OFFSET)
+        UNKNOWN_OFFSET,
+        AvailableOffsetRange(UNKNOWN_OFFSET, UNKNOWN_OFFSET))
 
       CachedFetchedData(emptyData)
     }
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/KafkaDataConsumer.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/KafkaDataConsumer.scala
index 5f23029d9fed3..37fe38ea94ece 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/KafkaDataConsumer.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/KafkaDataConsumer.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.kafka010.consumer
 
 import java.{util => ju}
 import java.io.Closeable
+import java.time.Duration
 import java.util.concurrent.TimeoutException
 
 import scala.collection.JavaConverters._
@@ -28,6 +29,7 @@ import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord, KafkaC
 import org.apache.kafka.common.TopicPartition
 
 import org.apache.spark.{SparkEnv, TaskContext}
+import org.apache.spark.deploy.security.HadoopDelegationTokenManager
 import org.apache.spark.internal.Logging
 import org.apache.spark.kafka010.{KafkaConfigUpdater, KafkaTokenUtil}
 import org.apache.spark.sql.kafka010.KafkaSourceProvider._
@@ -69,16 +71,17 @@ private[kafka010] class InternalKafkaConsumer(
    *                          consumer polls nothing before timeout.
    */
   def fetch(offset: Long, pollTimeoutMs: Long):
-      (ju.List[ConsumerRecord[Array[Byte], Array[Byte]]], Long) = {
+      (ju.List[ConsumerRecord[Array[Byte], Array[Byte]]], Long, AvailableOffsetRange) = {
 
     // Seek to the offset because we may call seekToBeginning or seekToEnd before this.
     seek(offset)
-    val p = consumer.poll(pollTimeoutMs)
+    val p = consumer.poll(Duration.ofMillis(pollTimeoutMs))
     val r = p.records(topicPartition)
     logDebug(s"Polled $groupId ${p.partitions()}  ${r.size}")
     val offsetAfterPoll = consumer.position(topicPartition)
     logDebug(s"Offset changed from $offset to $offsetAfterPoll after polling")
-    val fetchedData = (r, offsetAfterPoll)
+    val range = getAvailableOffsetRange()
+    val fetchedData = (r, offsetAfterPoll, range)
     if (r.isEmpty) {
       // We cannot fetch anything after `poll`. Two possible cases:
       // - `offset` is out of range so that Kafka returns nothing. `OffsetOutOfRangeException` will
@@ -86,7 +89,6 @@ private[kafka010] class InternalKafkaConsumer(
       // - Cannot fetch any data before timeout. `TimeoutException` will be thrown.
       // - Fetched something but all of them are not invisible. This is a valid case and let the
       //   caller handles this.
-      val range = getAvailableOffsetRange()
       if (offset < range.earliest || offset >= range.latest) {
         throw new OffsetOutOfRangeException(
           Map(topicPartition -> java.lang.Long.valueOf(offset)).asJava)
@@ -140,18 +142,22 @@ private[kafka010] class InternalKafkaConsumer(
  *                                 should check if the pre-fetched data is still valid.
  * @param _offsetAfterPoll the Kafka offset after calling `poll`. We will use this offset to
  *                           poll when `records` is drained.
+ * @param _availableOffsetRange the available offset range in Kafka when polling the records.
  */
 private[consumer] case class FetchedData(
     private var _records: ju.ListIterator[ConsumerRecord[Array[Byte], Array[Byte]]],
     private var _nextOffsetInFetchedData: Long,
-    private var _offsetAfterPoll: Long) {
+    private var _offsetAfterPoll: Long,
+    private var _availableOffsetRange: AvailableOffsetRange) {
 
   def withNewPoll(
       records: ju.ListIterator[ConsumerRecord[Array[Byte], Array[Byte]]],
-      offsetAfterPoll: Long): FetchedData = {
+      offsetAfterPoll: Long,
+      availableOffsetRange: AvailableOffsetRange): FetchedData = {
     this._records = records
     this._nextOffsetInFetchedData = UNKNOWN_OFFSET
     this._offsetAfterPoll = offsetAfterPoll
+    this._availableOffsetRange = availableOffsetRange
     this
   }
 
@@ -178,6 +184,7 @@ private[consumer] case class FetchedData(
     _records = ju.Collections.emptyListIterator()
     _nextOffsetInFetchedData = UNKNOWN_OFFSET
     _offsetAfterPoll = UNKNOWN_OFFSET
+    _availableOffsetRange = AvailableOffsetRange(UNKNOWN_OFFSET, UNKNOWN_OFFSET)
   }
 
   /**
@@ -190,6 +197,13 @@ private[consumer] case class FetchedData(
    * Returns the next offset to poll after draining the pre-fetched records.
    */
   def offsetAfterPoll: Long = _offsetAfterPoll
+
+  /**
+   * Returns the tuple of earliest and latest offsets that is the available offset range when
+   * polling the records.
+   */
+  def availableOffsetRange: (Long, Long) =
+    (_availableOffsetRange.earliest, _availableOffsetRange.latest)
 }
 
 /**
@@ -225,6 +239,12 @@ private[kafka010] class KafkaDataConsumer(
     fetchedDataPool: FetchedDataPool) extends Logging {
   import KafkaDataConsumer._
 
+  private var offsetOutOfRange = 0L
+  private var dataLoss = 0L
+
+  private val isTokenProviderEnabled =
+    HadoopDelegationTokenManager.isServiceEnabled(SparkEnv.get.conf, "kafka")
+
   // Exposed for testing
   @volatile private[consumer] var _consumer: Option[InternalKafkaConsumer] = None
   @volatile private var _fetchedData: Option[FetchedData] = None
@@ -275,7 +295,7 @@ private[kafka010] class KafkaDataConsumer(
     val fetchedData = getOrRetrieveFetchedData(offset)
 
     logDebug(s"Get $groupId $topicPartition nextOffset ${fetchedData.nextOffsetInFetchedData} " +
-      "requested $offset")
+      s"requested $offset")
 
     // The following loop is basically for `failOnDataLoss = false`. When `failOnDataLoss` is
     // `false`, first, we will try to fetch the record at `offset`. If no such record exists, then
@@ -312,7 +332,14 @@ private[kafka010] class KafkaDataConsumer(
 
           reportDataLoss(topicPartition, groupId, failOnDataLoss,
             s"Cannot fetch offset $toFetchOffset", e)
+
+          val oldToFetchOffsetd = toFetchOffset
           toFetchOffset = getEarliestAvailableOffsetBetween(consumer, toFetchOffset, untilOffset)
+          if (toFetchOffset == UNKNOWN_OFFSET) {
+            offsetOutOfRange += (untilOffset - oldToFetchOffsetd)
+          } else {
+            offsetOutOfRange += (toFetchOffset - oldToFetchOffsetd)
+          }
       }
     }
 
@@ -333,6 +360,9 @@ private[kafka010] class KafkaDataConsumer(
     consumer.getAvailableOffsetRange()
   }
 
+  def getNumOffsetOutOfRange(): Long = offsetOutOfRange
+  def getNumDataLoss(): Long = dataLoss
+
   /**
    * Release borrowed objects in data reader to the pool. Once the instance is created, caller
    * must call method after using the instance to make sure resources are not leaked.
@@ -469,8 +499,8 @@ private[kafka010] class KafkaDataConsumer(
       // In general, Kafka uses the specified offset as the start point, and tries to fetch the next
       // available offset. Hence we need to handle offset mismatch.
       if (record.offset > offset) {
-        val range = consumer.getAvailableOffsetRange()
-        if (range.earliest <= offset) {
+        val (earliestOffset, _) = fetchedData.availableOffsetRange
+        if (earliestOffset <= offset) {
           // `offset` is still valid but the corresponding message is invisible. We should skip it
           // and jump to `record.offset`. Here we move `fetchedData` back so that the next call of
           // `fetchRecord` can just return `record` directly.
@@ -519,8 +549,8 @@ private[kafka010] class KafkaDataConsumer(
       fetchedData: FetchedData,
       offset: Long,
       pollTimeoutMs: Long): Unit = {
-    val (records, offsetAfterPoll) = consumer.fetch(offset, pollTimeoutMs)
-    fetchedData.withNewPoll(records.listIterator, offsetAfterPoll)
+    val (records, offsetAfterPoll, range) = consumer.fetch(offset, pollTimeoutMs)
+    fetchedData.withNewPoll(records.listIterator, offsetAfterPoll, range)
   }
 
   private[kafka010] def getOrRetrieveConsumer(): InternalKafkaConsumer = {
@@ -528,8 +558,8 @@ private[kafka010] class KafkaDataConsumer(
       retrieveConsumer()
     }
     require(_consumer.isDefined, "Consumer must be defined")
-    if (KafkaTokenUtil.needTokenUpdate(SparkEnv.get.conf, _consumer.get.kafkaParamsWithSecurity,
-        _consumer.get.clusterConfig)) {
+    if (isTokenProviderEnabled && KafkaTokenUtil.needTokenUpdate(
+        _consumer.get.kafkaParamsWithSecurity, _consumer.get.clusterConfig)) {
       logDebug("Cached consumer uses an old delegation token, invalidating.")
       releaseConsumer()
       consumerPool.invalidateKey(cacheKey)
@@ -579,6 +609,7 @@ private[kafka010] class KafkaDataConsumer(
       message: String,
       cause: Throwable = null): Unit = {
     val finalMessage = s"$message ${additionalMessage(topicPartition, groupId, failOnDataLoss)}"
+    dataLoss += 1
     reportDataLoss0(failOnDataLoss, finalMessage, cause)
   }
 
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/ConsumerStrategySuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/ConsumerStrategySuite.scala
new file mode 100644
index 0000000000000..939cf0bb36a8c
--- /dev/null
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/ConsumerStrategySuite.scala
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.kafka010
+
+import java.util.UUID
+
+import scala.collection.JavaConverters._
+
+import org.apache.kafka.clients.CommonClientConfigs
+import org.apache.kafka.clients.admin.Admin
+import org.apache.kafka.common.TopicPartition
+import org.mockito.Mockito.mock
+
+import org.apache.spark.{SparkConf, SparkEnv, SparkFunSuite}
+
+class ConsumerStrategySuite extends SparkFunSuite {
+  private var testUtils: KafkaTestUtils = _
+
+  private def doReturn(value: Any) = org.mockito.Mockito.doReturn(value, Seq.empty: _*)
+
+  protected def newTopic(prefix: String = "topic") = s"$prefix-${UUID.randomUUID().toString}"
+
+  private def setSparkEnv(settings: Iterable[(String, String)]): Unit = {
+    val conf = new SparkConf().setAll(settings)
+    val env = mock(classOf[SparkEnv])
+    doReturn(conf).when(env).conf
+    SparkEnv.set(env)
+  }
+
+  private def adminProps = {
+    Map[String, Object](
+      CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG -> testUtils.brokerAddress
+    ).asJava
+  }
+
+  private def admin(strategy: ConsumerStrategy): Admin = {
+    strategy.createAdmin(adminProps)
+  }
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    testUtils = new KafkaTestUtils(Map.empty)
+    testUtils.setup()
+    setSparkEnv(Map.empty)
+  }
+
+  override def afterAll(): Unit = {
+    if (testUtils != null) {
+      testUtils.teardown()
+      testUtils = null
+    }
+    super.afterAll()
+  }
+
+  test("createAdmin must create admin properly") {
+    val strategy = AssignStrategy(Array.empty)
+    assert(strategy.createAdmin(adminProps) != null)
+  }
+
+  test("AssignStrategy.assignedTopicPartitions must give back all assigned") {
+    val assignedTopic = newTopic()
+    testUtils.createTopic(assignedTopic, partitions = 3)
+    val otherExistingTopic = newTopic()
+    testUtils.createTopic(otherExistingTopic, partitions = 2)
+
+    val partitions = Array(
+      new TopicPartition(assignedTopic, 0),
+      new TopicPartition(assignedTopic, 2)
+    )
+    val strategy = AssignStrategy(partitions)
+    assert(strategy.assignedTopicPartitions(admin(strategy)) === partitions.toSet)
+
+    testUtils.deleteTopic(assignedTopic)
+    testUtils.deleteTopic(otherExistingTopic)
+  }
+
+  test("AssignStrategy.assignedTopicPartitions must skip invalid partitions") {
+    val assignedTopic = newTopic()
+    testUtils.createTopic(assignedTopic, partitions = 1)
+
+    val partitions = Array(new TopicPartition(assignedTopic, 1))
+    val strategy = AssignStrategy(partitions)
+    assert(strategy.assignedTopicPartitions(admin(strategy)) === Set.empty)
+
+    testUtils.deleteTopic(assignedTopic)
+  }
+
+  test("SubscribeStrategy.assignedTopicPartitions must give back all assigned") {
+    val subscribedTopic1 = newTopic()
+    testUtils.createTopic(subscribedTopic1, partitions = 2)
+    val subscribedTopic2 = newTopic()
+    testUtils.createTopic(subscribedTopic2, partitions = 2)
+    val otherExistingTopic = newTopic()
+    testUtils.createTopic(otherExistingTopic, partitions = 2)
+
+    val partitions = Set(
+      new TopicPartition(subscribedTopic1, 0),
+      new TopicPartition(subscribedTopic1, 1),
+      new TopicPartition(subscribedTopic2, 0),
+      new TopicPartition(subscribedTopic2, 1)
+    )
+    val strategy = SubscribeStrategy(Seq(subscribedTopic1, subscribedTopic2))
+    assert(strategy.assignedTopicPartitions(admin(strategy)) === partitions)
+
+    testUtils.deleteTopic(subscribedTopic1)
+    testUtils.deleteTopic(subscribedTopic2)
+    testUtils.deleteTopic(otherExistingTopic)
+  }
+
+  test("SubscribePatternStrategy.assignedTopicPartitions must give back all assigned") {
+    val subscribePattern = "subscribePattern"
+    val subscribedTopic1 = newTopic(subscribePattern)
+    testUtils.createTopic(subscribedTopic1, partitions = 2)
+    val subscribedTopic2 = newTopic(subscribePattern)
+    testUtils.createTopic(subscribedTopic2, partitions = 2)
+    val otherExistingTopic = newTopic("other")
+    testUtils.createTopic(otherExistingTopic, partitions = 2)
+
+    val partitions = Set(
+      new TopicPartition(subscribedTopic1, 0),
+      new TopicPartition(subscribedTopic1, 1),
+      new TopicPartition(subscribedTopic2, 0),
+      new TopicPartition(subscribedTopic2, 1)
+    )
+    val strategy = SubscribePatternStrategy(s"$subscribePattern.*")
+    assert(strategy.assignedTopicPartitions(admin(strategy)) === partitions)
+
+    testUtils.deleteTopic(subscribedTopic1)
+    testUtils.deleteTopic(subscribedTopic2)
+    testUtils.deleteTopic(otherExistingTopic)
+  }
+}
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousSourceSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousSourceSuite.scala
index 76c25980fc33f..6801d14d036dd 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousSourceSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousSourceSuite.scala
@@ -33,7 +33,7 @@ class KafkaContinuousSourceSuite extends KafkaSourceSuiteBase with KafkaContinuo
     withTable(table) {
       val topic = newTopic()
       testUtils.createTopic(topic)
-      testUtils.withTranscationalProducer { producer =>
+      testUtils.withTransactionalProducer { producer =>
         val df = spark
           .readStream
           .format("kafka")
@@ -99,7 +99,7 @@ class KafkaContinuousSourceSuite extends KafkaSourceSuiteBase with KafkaContinuo
     withTable(table) {
       val topic = newTopic()
       testUtils.createTopic(topic)
-      testUtils.withTranscationalProducer { producer =>
+      testUtils.withTransactionalProducer { producer =>
         val df = spark
           .readStream
           .format("kafka")
@@ -193,6 +193,7 @@ class KafkaContinuousSourceTopicDeletionSuite extends KafkaContinuousTest {
       .format("kafka")
       .option("kafka.bootstrap.servers", testUtils.brokerAddress)
       .option("kafka.metadata.max.age.ms", "1")
+      .option("kafka.request.timeout.ms", "3000")
       .option("kafka.default.api.timeout.ms", "3000")
       .option("subscribePattern", s"$topicPrefix-.*")
       .option("failOnDataLoss", "false")
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaDontFailOnDataLossSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaDontFailOnDataLossSuite.scala
index a1c7b5ae13474..54ce1717acc71 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaDontFailOnDataLossSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaDontFailOnDataLossSuite.scala
@@ -223,6 +223,7 @@ class KafkaSourceStressForDontFailOnDataLossSuite extends StreamTest with KafkaM
       .format("kafka")
       .option("kafka.bootstrap.servers", testUtils.brokerAddress)
       .option("kafka.metadata.max.age.ms", "1")
+      .option("kafka.request.timeout.ms", "3000")
       .option("kafka.default.api.timeout.ms", "3000")
       .option("subscribePattern", "failOnDataLoss.*")
       .option("startingOffsets", "earliest")
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala
index bdad214a91343..4826e7a251447 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.kafka010
 import java.io._
 import java.nio.charset.StandardCharsets.UTF_8
 import java.nio.file.{Files, Paths}
-import java.util.Locale
+import java.util.{Locale, Optional}
 import java.util.concurrent.ConcurrentLinkedQueue
 import java.util.concurrent.atomic.AtomicInteger
 
@@ -349,8 +349,7 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
     )
   }
 
-  // TODO (SPARK-31731): re-enable it
-  ignore("subscribing topic by pattern with topic deletions") {
+  test("subscribing topic by pattern with topic deletions") {
     val topicPrefix = newTopic()
     val topic = topicPrefix + "-seems"
     val topic2 = topicPrefix + "-bad"
@@ -363,6 +362,7 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
       .format("kafka")
       .option("kafka.bootstrap.servers", testUtils.brokerAddress)
       .option("kafka.metadata.max.age.ms", "1")
+      .option("kafka.request.timeout.ms", "3000")
       .option("kafka.default.api.timeout.ms", "3000")
       .option("subscribePattern", s"$topicPrefix-.*")
       .option("failOnDataLoss", "false")
@@ -400,6 +400,7 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
       .format("kafka")
       .option("kafka.bootstrap.servers", testUtils.brokerAddress)
       .option("kafka.metadata.max.age.ms", "1")
+      .option("kafka.request.timeout.ms", "3000")
       .option("kafka.default.api.timeout.ms", "3000")
       .option("startingOffsets", "earliest")
       .option("subscribePattern", s"$topicPrefix-.*")
@@ -567,7 +568,7 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
     val rows = spark.table("kafkaWatermark").collect()
     assert(rows.length === 1, s"Unexpected results: ${rows.toList}")
     val row = rows(0)
-    // We cannot check the exact window start time as it depands on the time that messages were
+    // We cannot check the exact window start time as it depends on the time that messages were
     // inserted by the producer. So here we just use a low bound to make sure the internal
     // conversion works.
     assert(
@@ -589,6 +590,7 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
       .format("kafka")
       .option("kafka.bootstrap.servers", testUtils.brokerAddress)
       .option("kafka.metadata.max.age.ms", "1")
+      .option("kafka.request.timeout.ms", "3000")
       .option("kafka.default.api.timeout.ms", "3000")
       .option("subscribe", topic)
       // If a topic is deleted and we try to poll data starting from offset 0,
@@ -605,7 +607,9 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
     // in executors.
     val query = kafka.map(kv => kv._2.toInt).writeStream.foreach(new ForeachWriter[Int] {
       override def open(partitionId: Long, version: Long): Boolean = {
+        // Re-create topic since Kafka auto topic creation is not supported by Spark
         KafkaSourceSuite.globalTestUtils.deleteTopic(topic)
+        KafkaSourceSuite.globalTestUtils.createTopic(topic)
         true
       }
 
@@ -687,19 +691,25 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
   }
 
   test("allow group.id prefix") {
-    testGroupId("groupIdPrefix", (expected, actual) => {
-      assert(actual.exists(_.startsWith(expected)) && !actual.exists(_ === expected),
-        "Valid consumer groups don't contain the expected group id - " +
-        s"Valid consumer groups: $actual / expected group id: $expected")
-    })
+    // Group ID prefix is only supported by consumer based offset reader
+    if (spark.conf.get(SQLConf.USE_DEPRECATED_KAFKA_OFFSET_FETCHING)) {
+      testGroupId("groupIdPrefix", (expected, actual) => {
+        assert(actual.exists(_.startsWith(expected)) && !actual.exists(_ === expected),
+          "Valid consumer groups don't contain the expected group id - " +
+            s"Valid consumer groups: $actual / expected group id: $expected")
+      })
+    }
   }
 
   test("allow group.id override") {
-    testGroupId("kafka.group.id", (expected, actual) => {
-      assert(actual.exists(_ === expected), "Valid consumer groups don't " +
-        s"contain the expected group id - Valid consumer groups: $actual / " +
-        s"expected group id: $expected")
-    })
+    // Group ID override is only supported by consumer based offset reader
+    if (spark.conf.get(SQLConf.USE_DEPRECATED_KAFKA_OFFSET_FETCHING)) {
+      testGroupId("kafka.group.id", (expected, actual) => {
+        assert(actual.exists(_ === expected), "Valid consumer groups don't " +
+          s"contain the expected group id - Valid consumer groups: $actual / " +
+          s"expected group id: $expected")
+      })
+    }
   }
 
   private def testGroupId(groupIdKey: String,
@@ -833,7 +843,7 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
 
     val topicPartition = new TopicPartition(topic, 0)
     // The message values are the same as their offsets to make the test easy to follow
-    testUtils.withTranscationalProducer { producer =>
+    testUtils.withTransactionalProducer { producer =>
       testStream(mapped)(
         StartStream(Trigger.ProcessingTime(100), clock),
         waitUntilBatchProcessed,
@@ -956,7 +966,7 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
 
     val topicPartition = new TopicPartition(topic, 0)
     // The message values are the same as their offsets to make the test easy to follow
-    testUtils.withTranscationalProducer { producer =>
+    testUtils.withTransactionalProducer { producer =>
       testStream(mapped)(
         StartStream(Trigger.ProcessingTime(100), clock),
         waitUntilBatchProcessed,
@@ -1047,7 +1057,7 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
       .load()
       .select($"value".as[String])
 
-    testUtils.withTranscationalProducer { producer =>
+    testUtils.withTransactionalProducer { producer =>
       producer.beginTransaction()
       (0 to 3).foreach { i =>
         producer.send(new ProducerRecord[String, String](topic, i.toString)).get()
@@ -1063,7 +1073,7 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
         // this case, if we forget to reset `FetchedData._nextOffsetInFetchedData` or
         // `FetchedData._offsetAfterPoll` (See SPARK-25495), the next batch will see incorrect
         // values and return wrong results hence fail the test.
-        testUtils.withTranscationalProducer { producer =>
+        testUtils.withTransactionalProducer { producer =>
           producer.beginTransaction()
           (4 to 7).foreach { i =>
             producer.send(new ProducerRecord[String, String](topic, i.toString)).get()
@@ -1118,6 +1128,20 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
 }
 
 
+class KafkaMicroBatchV1SourceWithAdminSuite extends KafkaMicroBatchV1SourceSuite {
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    spark.conf.set(SQLConf.USE_DEPRECATED_KAFKA_OFFSET_FETCHING.key, "false")
+  }
+}
+
+class KafkaMicroBatchV2SourceWithAdminSuite extends KafkaMicroBatchV2SourceSuite {
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    spark.conf.set(SQLConf.USE_DEPRECATED_KAFKA_OFFSET_FETCHING.key, "false")
+  }
+}
+
 class KafkaMicroBatchV1SourceSuite extends KafkaMicroBatchSourceSuiteBase {
   override def beforeAll(): Unit = {
     super.beforeAll()
@@ -1175,7 +1199,6 @@ class KafkaMicroBatchV2SourceSuite extends KafkaMicroBatchSourceSuiteBase {
   }
 
   testWithUninterruptibleThread("minPartitions is supported") {
-    import testImplicits._
 
     val topic = newTopic()
     val tp = new TopicPartition(topic, 0)
@@ -1274,6 +1297,112 @@ class KafkaMicroBatchV2SourceSuite extends KafkaMicroBatchSourceSuiteBase {
       CheckNewAnswer(32, 33, 34, 35, 36)
     )
   }
+
+  test("test custom metrics - with rate limit") {
+    import testImplicits._
+
+    val topic = newTopic()
+    val data = 1 to 10
+    testUtils.createTopic(topic, partitions = 2)
+    testUtils.sendMessages(topic, (1 to 5).map(_.toString).toArray, Some(0))
+    testUtils.sendMessages(topic, (6 to 10).map(_.toString).toArray, Some(1))
+
+    val kafka = spark
+      .readStream
+      .format("kafka")
+      .option("kafka.bootstrap.servers", testUtils.brokerAddress)
+      .option("subscribe", topic)
+      .option("maxOffsetsPerTrigger", 1)
+      .option(STARTING_OFFSETS_OPTION_KEY, "earliest")
+      .load()
+      .selectExpr("CAST(value AS STRING)")
+      .as[String]
+      .map(_.toInt)
+
+    testStream(kafka)(
+      StartStream(),
+      makeSureGetOffsetCalled,
+      CheckAnswer(data: _*),
+      Execute { query =>
+        // The rate limit is 1, so there must be some delay in offsets per partition.
+        val progressWithDelay = query.recentProgress.map(_.sources.head).reverse.find { progress =>
+          // find the metrics that has non-zero average offsetsBehindLatest greater than 0.
+          !progress.metrics.isEmpty && progress.metrics.get("avgOffsetsBehindLatest").toDouble > 0
+        }
+        assert(progressWithDelay.nonEmpty)
+        val metrics = progressWithDelay.get.metrics
+        assert(metrics.keySet() ===
+          Set("minOffsetsBehindLatest",
+            "maxOffsetsBehindLatest",
+            "avgOffsetsBehindLatest").asJava)
+        assert(metrics.get("minOffsetsBehindLatest").toLong > 0)
+        assert(metrics.get("maxOffsetsBehindLatest").toLong > 0)
+        assert(metrics.get("avgOffsetsBehindLatest").toDouble > 0)
+      }
+    )
+  }
+
+  test("test custom metrics - no rate limit") {
+    import testImplicits._
+
+    val topic = newTopic()
+    val data = 1 to 10
+    testUtils.createTopic(topic, partitions = 2)
+    testUtils.sendMessages(topic, (1 to 5).map(_.toString).toArray, Some(0))
+    testUtils.sendMessages(topic, (6 to 10).map(_.toString).toArray, Some(1))
+
+    val kafka = spark
+      .readStream
+      .format("kafka")
+      .option("kafka.bootstrap.servers", testUtils.brokerAddress)
+      .option("subscribe", topic)
+      .option(STARTING_OFFSETS_OPTION_KEY, "earliest")
+      .load()
+      .selectExpr("CAST(value AS STRING)")
+      .as[String]
+      .map(_.toInt)
+
+    testStream(kafka)(
+      StartStream(),
+      makeSureGetOffsetCalled,
+      CheckAnswer(data: _*),
+      Execute { query =>
+        val progress = query.recentProgress.map(_.sources.head).lastOption
+        assert(progress.nonEmpty)
+        val metrics = progress.get.metrics
+        // When there is no rate limit, there shouldn't be any delay in the current stream.
+        assert(metrics.keySet() ===
+          Set("minOffsetsBehindLatest",
+            "maxOffsetsBehindLatest",
+            "avgOffsetsBehindLatest").asJava)
+        assert(metrics.get("minOffsetsBehindLatest").toLong === 0)
+        assert(metrics.get("maxOffsetsBehindLatest").toLong === 0)
+        assert(metrics.get("avgOffsetsBehindLatest").toDouble === 0)
+      }
+    )
+  }
+
+  test("test custom metrics - corner cases") {
+    val topicPartition1 = new TopicPartition(newTopic(), 0)
+    val topicPartition2 = new TopicPartition(newTopic(), 0)
+    val latestOffset = Map[TopicPartition, Long]((topicPartition1, 3L), (topicPartition2, 6L))
+
+    // test empty offset.
+    assert(KafkaMicroBatchStream.metrics(Optional.ofNullable(null), latestOffset).isEmpty)
+
+    // test valid offsetsBehindLatest
+    val offset = KafkaSourceOffset(
+      Map[TopicPartition, Long]((topicPartition1, 1L), (topicPartition2, 2L)))
+    assert(
+      KafkaMicroBatchStream.metrics(Optional.ofNullable(offset), latestOffset) ===
+        Map[String, String](
+          "minOffsetsBehindLatest" -> "2",
+          "maxOffsetsBehindLatest" -> "4",
+          "avgOffsetsBehindLatest" -> "3.0").asJava)
+
+    // test null latestAvailablePartitionOffsets
+    assert(KafkaMicroBatchStream.metrics(Optional.ofNullable(offset), null).isEmpty)
+  }
 }
 
 abstract class KafkaSourceSuiteBase extends KafkaSourceTest {
@@ -1540,8 +1669,8 @@ abstract class KafkaSourceSuiteBase extends KafkaSourceTest {
       makeSureGetOffsetCalled,
       Execute { q =>
         // wait to reach the last offset in every partition
-        q.awaitOffset(
-          0, KafkaSourceOffset(partitionOffsets.mapValues(_ => 3L)), streamingTimeout.toMillis)
+        q.awaitOffset(0,
+          KafkaSourceOffset(partitionOffsets.mapValues(_ => 3L).toMap), streamingTimeout.toMillis)
       },
       CheckAnswer(-20, -21, -22, 0, 1, 2, 11, 12, 22),
       StopStream,
@@ -1776,7 +1905,7 @@ abstract class KafkaSourceSuiteBase extends KafkaSourceTest {
     withTable(table) {
       val topic = newTopic()
       testUtils.createTopic(topic)
-      testUtils.withTranscationalProducer { producer =>
+      testUtils.withTransactionalProducer { producer =>
         val df = spark
           .readStream
           .format("kafka")
@@ -1860,6 +1989,7 @@ class KafkaSourceStressSuite extends KafkaSourceTest {
         .option("kafka.metadata.max.age.ms", "1")
         .option("subscribePattern", "stress.*")
         .option("failOnDataLoss", "false")
+        .option("kafka.request.timeout.ms", "3000")
         .option("kafka.default.api.timeout.ms", "3000")
         .load()
         .selectExpr("CAST(key AS STRING)", "CAST(value AS STRING)")
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaOffsetRangeCalculatorSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaOffsetRangeCalculatorSuite.scala
index 5d010cd553521..751b877df9c78 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaOffsetRangeCalculatorSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaOffsetRangeCalculatorSuite.scala
@@ -71,6 +71,20 @@ class KafkaOffsetRangeCalculatorSuite extends SparkFunSuite {
         KafkaOffsetRange(tp3, 1, 2, None)))
   }
 
+  testWithMinPartitions("N TopicPartitions to N offset ranges with executors", 3) { calc =>
+    assert(
+      calc.getRanges(
+        Seq(
+          KafkaOffsetRange(tp1, 1, 2),
+          KafkaOffsetRange(tp2, 1, 2),
+          KafkaOffsetRange(tp3, 1, 2)),
+        Seq("exec1", "exec2", "exec3")) ===
+        Seq(
+          KafkaOffsetRange(tp1, 1, 2, Some("exec3")),
+          KafkaOffsetRange(tp2, 1, 2, Some("exec1")),
+          KafkaOffsetRange(tp3, 1, 2, Some("exec2"))))
+  }
+
   testWithMinPartitions("1 TopicPartition to N offset ranges", 4) { calc =>
     assert(
       calc.getRanges(
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderSuite.scala
index ad22a56d9157f..d1e49b0e14314 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderSuite.scala
@@ -17,13 +17,17 @@
 
 package org.apache.spark.sql.kafka010
 
+import java.util.Locale
 import java.util.UUID
 import java.util.concurrent.atomic.AtomicInteger
 
-import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.clients.CommonClientConfigs
+import org.apache.kafka.clients.consumer.ConsumerConfig
+import org.apache.kafka.common.{IsolationLevel, TopicPartition}
 
 import org.apache.spark.sql.QueryTest
 import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.kafka010.KafkaOffsetRangeLimit.{EARLIEST, LATEST}
 import org.apache.spark.sql.test.SharedSparkSession
 
@@ -53,9 +57,9 @@ class KafkaOffsetReaderSuite extends QueryTest with SharedSparkSession with Kafk
   }
 
   private def createKafkaReader(topic: String, minPartitions: Option[Int]): KafkaOffsetReader = {
-    new KafkaOffsetReader(
+    KafkaOffsetReader.build(
       SubscribeStrategy(Seq(topic)),
-      org.apache.spark.sql.kafka010.KafkaSourceProvider.kafkaParamsForDriver(
+      KafkaSourceProvider.kafkaParamsForDriver(
         Map(
         "bootstrap.servers" ->
          testUtils.brokerAddress
@@ -66,7 +70,39 @@ class KafkaOffsetReaderSuite extends QueryTest with SharedSparkSession with Kafk
     )
   }
 
-  test("SPARK-30656: getOffsetRangesFromUnresolvedOffsets - using specific offsets") {
+  test("isolationLevel must give back default isolation level when not set") {
+    testIsolationLevel(None,
+      IsolationLevel.valueOf(ConsumerConfig.DEFAULT_ISOLATION_LEVEL.toUpperCase(Locale.ROOT)))
+  }
+
+  test("isolationLevel must give back READ_UNCOMMITTED when set") {
+    testIsolationLevel(Some("read_uncommitted"), IsolationLevel.READ_UNCOMMITTED)
+  }
+
+  test("isolationLevel must give back READ_COMMITTED when set") {
+    testIsolationLevel(Some("read_committed"), IsolationLevel.READ_COMMITTED)
+  }
+
+  test("isolationLevel must throw exception when invalid isolation level set") {
+    intercept[IllegalArgumentException] {
+      testIsolationLevel(Some("intentionally_invalid"), IsolationLevel.READ_COMMITTED)
+    }
+  }
+
+  private def testIsolationLevel(kafkaParam: Option[String], isolationLevel: IsolationLevel) = {
+    var kafkaParams = Map(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG -> testUtils.brokerAddress)
+    kafkaParam.foreach(p => kafkaParams ++= Map(ConsumerConfig.ISOLATION_LEVEL_CONFIG -> p))
+    val reader = new KafkaOffsetReaderAdmin(
+      SubscribeStrategy(Seq()),
+      KafkaSourceProvider.kafkaParamsForDriver(kafkaParams),
+      CaseInsensitiveMap(Map.empty),
+      ""
+    )
+    assert(reader.isolationLevel === isolationLevel)
+  }
+
+  testWithAllOffsetFetchingSQLConf("SPARK-30656: getOffsetRangesFromUnresolvedOffsets - " +
+    "using specific offsets") {
     val topic = newTopic()
     testUtils.createTopic(topic, partitions = 1)
     testUtils.sendMessages(topic, (0 until 10).map(_.toString).toArray, Some(0))
@@ -74,14 +110,16 @@ class KafkaOffsetReaderSuite extends QueryTest with SharedSparkSession with Kafk
     val reader = createKafkaReader(topic, minPartitions = Some(3))
     val startingOffsets = SpecificOffsetRangeLimit(Map(tp -> 1))
     val endingOffsets = SpecificOffsetRangeLimit(Map(tp -> 4))
-    val offsetRanges = reader.getOffsetRangesFromUnresolvedOffsets(startingOffsets, endingOffsets)
-    assert(offsetRanges === Seq(
+    val offsetRanges = reader.getOffsetRangesFromUnresolvedOffsets(startingOffsets,
+      endingOffsets)
+    assert(offsetRanges.sortBy(_.topicPartition.toString) === Seq(
       KafkaOffsetRange(tp, 1, 2, None),
       KafkaOffsetRange(tp, 2, 3, None),
-      KafkaOffsetRange(tp, 3, 4, None)))
+      KafkaOffsetRange(tp, 3, 4, None)).sortBy(_.topicPartition.toString))
   }
 
-  test("SPARK-30656: getOffsetRangesFromUnresolvedOffsets - using special offsets") {
+  testWithAllOffsetFetchingSQLConf("SPARK-30656: getOffsetRangesFromUnresolvedOffsets - " +
+    "using special offsets") {
     val topic = newTopic()
     testUtils.createTopic(topic, partitions = 1)
     testUtils.sendMessages(topic, (0 until 4).map(_.toString).toArray, Some(0))
@@ -89,14 +127,16 @@ class KafkaOffsetReaderSuite extends QueryTest with SharedSparkSession with Kafk
     val reader = createKafkaReader(topic, minPartitions = Some(3))
     val startingOffsets = EarliestOffsetRangeLimit
     val endingOffsets = LatestOffsetRangeLimit
-    val offsetRanges = reader.getOffsetRangesFromUnresolvedOffsets(startingOffsets, endingOffsets)
-    assert(offsetRanges === Seq(
+    val offsetRanges = reader.getOffsetRangesFromUnresolvedOffsets(startingOffsets,
+      endingOffsets)
+    assert(offsetRanges.sortBy(_.topicPartition.toString) === Seq(
       KafkaOffsetRange(tp, EARLIEST, 1, None),
       KafkaOffsetRange(tp, 1, 2, None),
-      KafkaOffsetRange(tp, 2, LATEST, None)))
+      KafkaOffsetRange(tp, 2, LATEST, None)).sortBy(_.topicPartition.toString))
   }
 
-  test("SPARK-30656: getOffsetRangesFromUnresolvedOffsets - multiple topic partitions") {
+  testWithAllOffsetFetchingSQLConf("SPARK-30656: getOffsetRangesFromUnresolvedOffsets - " +
+    "multiple topic partitions") {
     val topic = newTopic()
     testUtils.createTopic(topic, partitions = 2)
     testUtils.sendMessages(topic, (0 until 100).map(_.toString).toArray, Some(0))
@@ -107,15 +147,16 @@ class KafkaOffsetReaderSuite extends QueryTest with SharedSparkSession with Kafk
 
     val startingOffsets = SpecificOffsetRangeLimit(Map(tp1 -> EARLIEST, tp2 -> EARLIEST))
     val endingOffsets = SpecificOffsetRangeLimit(Map(tp1 -> LATEST, tp2 -> 3))
-    val offsetRanges = reader.getOffsetRangesFromUnresolvedOffsets(startingOffsets, endingOffsets)
-    assert(offsetRanges === Seq(
+    val offsetRanges = reader.getOffsetRangesFromUnresolvedOffsets(startingOffsets,
+      endingOffsets)
+    assert(offsetRanges.sortBy(_.topicPartition.toString) === Seq(
       KafkaOffsetRange(tp2, EARLIEST, 3, None),
       KafkaOffsetRange(tp1, EARLIEST, 33, None),
       KafkaOffsetRange(tp1, 33, 66, None),
-      KafkaOffsetRange(tp1, 66, LATEST, None)))
+      KafkaOffsetRange(tp1, 66, LATEST, None)).sortBy(_.topicPartition.toString))
   }
 
-  test("SPARK-30656: getOffsetRangesFromResolvedOffsets") {
+  testWithAllOffsetFetchingSQLConf("SPARK-30656: getOffsetRangesFromResolvedOffsets") {
     val topic = newTopic()
     testUtils.createTopic(topic, partitions = 2)
     testUtils.sendMessages(topic, (0 until 100).map(_.toString).toArray, Some(0))
@@ -130,10 +171,28 @@ class KafkaOffsetReaderSuite extends QueryTest with SharedSparkSession with Kafk
       fromPartitionOffsets,
       untilPartitionOffsets,
       _ => {})
-    assert(offsetRanges === Seq(
+    assert(offsetRanges.sortBy(_.topicPartition.toString) === Seq(
       KafkaOffsetRange(tp1, 0, 33, None),
       KafkaOffsetRange(tp1, 33, 66, None),
       KafkaOffsetRange(tp1, 66, 100, None),
-      KafkaOffsetRange(tp2, 0, 3, None)))
+      KafkaOffsetRange(tp2, 0, 3, None)).sortBy(_.topicPartition.toString))
+  }
+
+  private def testWithAllOffsetFetchingSQLConf(name: String)(func: => Any): Unit = {
+    Seq("true", "false").foreach { useDeprecatedOffsetFetching =>
+      val testName = s"$name with useDeprecatedOffsetFetching $useDeprecatedOffsetFetching"
+      executeFuncWithSQLConf(testName, useDeprecatedOffsetFetching, func)
+    }
+  }
+
+  private def executeFuncWithSQLConf(
+      name: String,
+      useDeprecatedOffsetFetching: String,
+      func: => Any): Unit = {
+    test(name) {
+      withSQLConf(SQLConf.USE_DEPRECATED_KAFKA_OFFSET_FETCHING.key -> useDeprecatedOffsetFetching) {
+        func
+      }
+    }
   }
 }
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRelationSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRelationSuite.scala
index e5f3a229622e1..9cec37e708dbb 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRelationSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRelationSuite.scala
@@ -21,15 +21,10 @@ import java.nio.charset.StandardCharsets.UTF_8
 import java.util.Locale
 import java.util.concurrent.atomic.AtomicInteger
 
-import scala.annotation.tailrec
-import scala.collection.JavaConverters._
-import scala.util.Random
-
 import org.apache.kafka.clients.producer.ProducerRecord
 import org.apache.kafka.common.TopicPartition
 
-import org.apache.spark.SparkConf
-import org.apache.spark.SparkException
+import org.apache.spark.{SparkConf, TestUtils}
 import org.apache.spark.sql.{DataFrameReader, QueryTest}
 import org.apache.spark.sql.execution.datasources.LogicalRelation
 import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
@@ -179,8 +174,7 @@ abstract class KafkaRelationSuiteBase extends QueryTest with SharedSparkSession
       ("3", Seq(("e", "f".getBytes(UTF_8)), ("e", "g".getBytes(UTF_8))))).toDF)
   }
 
-  // TODO (SPARK-31729): re-enable it
-  ignore("timestamp provided for starting and ending") {
+  test("timestamp provided for starting and ending") {
     val (topic, timestamps) = prepareTimestampRelatedUnitTest
 
     // timestamp both presented: starting "first" ending "finalized"
@@ -272,7 +266,9 @@ abstract class KafkaRelationSuiteBase extends QueryTest with SharedSparkSession
   test("no matched offset for timestamp - startingOffsets") {
     val (topic, timestamps) = prepareTimestampRelatedUnitTest
 
-    val e = intercept[SparkException] {
+    // KafkaOffsetReaderConsumer and KafkaOffsetReaderAdmin both throws AssertionError
+    // but the UninterruptibleThread used by KafkaOffsetReaderConsumer wraps it with SparkException
+    val e = intercept[Throwable] {
       verifyTimestampRelatedQueryResult({ df =>
         // partition 2 will make query fail
         val startTopicTimestamps = Map(
@@ -285,19 +281,7 @@ abstract class KafkaRelationSuiteBase extends QueryTest with SharedSparkSession
       }, topic, Seq.empty)
     }
 
-    @tailrec
-    def assertionErrorInExceptionChain(e: Throwable): Boolean = {
-      if (e.isInstanceOf[AssertionError]) {
-        true
-      } else if (e.getCause == null) {
-        false
-      } else {
-        assertionErrorInExceptionChain(e.getCause)
-      }
-    }
-
-    assert(assertionErrorInExceptionChain(e),
-      "Cannot find expected AssertionError in chained exceptions")
+    TestUtils.assertExceptionMsg(e, "No offset matched from request")
   }
 
   test("no matched offset for timestamp - endingOffsets") {
@@ -465,45 +449,10 @@ abstract class KafkaRelationSuiteBase extends QueryTest with SharedSparkSession
     testBadOptions("subscribePattern" -> "")("pattern to subscribe is empty")
   }
 
-  test("allow group.id prefix") {
-    testGroupId("groupIdPrefix", (expected, actual) => {
-      assert(actual.exists(_.startsWith(expected)) && !actual.exists(_ === expected),
-        "Valid consumer groups don't contain the expected group id - " +
-        s"Valid consumer groups: $actual / expected group id: $expected")
-    })
-  }
-
-  test("allow group.id override") {
-    testGroupId("kafka.group.id", (expected, actual) => {
-      assert(actual.exists(_ === expected), "Valid consumer groups don't " +
-        s"contain the expected group id - Valid consumer groups: $actual / " +
-        s"expected group id: $expected")
-    })
-  }
-
-  private def testGroupId(groupIdKey: String,
-      validateGroupId: (String, Iterable[String]) => Unit): Unit = {
-    // Tests code path KafkaSourceProvider.createRelation(.)
-    val topic = newTopic()
-    testUtils.createTopic(topic, partitions = 3)
-    testUtils.sendMessages(topic, (1 to 10).map(_.toString).toArray, Some(0))
-    testUtils.sendMessages(topic, (11 to 20).map(_.toString).toArray, Some(1))
-    testUtils.sendMessages(topic, (21 to 30).map(_.toString).toArray, Some(2))
-
-    val customGroupId = "id-" + Random.nextInt()
-    val df = createDF(topic, withOptions = Map(groupIdKey -> customGroupId))
-    checkAnswer(df, (1 to 30).map(_.toString).toDF())
-
-    val consumerGroups = testUtils.listConsumerGroups()
-    val validGroups = consumerGroups.valid().get()
-    val validGroupsId = validGroups.asScala.map(_.groupId())
-    validateGroupId(customGroupId, validGroupsId)
-  }
-
   test("read Kafka transactional messages: read_committed") {
     val topic = newTopic()
     testUtils.createTopic(topic)
-    testUtils.withTranscationalProducer { producer =>
+    testUtils.withTransactionalProducer { producer =>
       val df = spark
         .read
         .format("kafka")
@@ -552,7 +501,7 @@ abstract class KafkaRelationSuiteBase extends QueryTest with SharedSparkSession
   test("read Kafka transactional messages: read_uncommitted") {
     val topic = newTopic()
     testUtils.createTopic(topic)
-    testUtils.withTranscationalProducer { producer =>
+    testUtils.withTransactionalProducer { producer =>
       val df = spark
         .read
         .format("kafka")
@@ -622,6 +571,16 @@ abstract class KafkaRelationSuiteBase extends QueryTest with SharedSparkSession
   }
 }
 
+class KafkaRelationSuiteWithAdminV1 extends KafkaRelationSuiteV1 {
+  override protected def sparkConf: SparkConf =
+    super.sparkConf.set(SQLConf.USE_DEPRECATED_KAFKA_OFFSET_FETCHING.key, "false")
+}
+
+class KafkaRelationSuiteWithAdminV2 extends KafkaRelationSuiteV2 {
+  override protected def sparkConf: SparkConf =
+    super.sparkConf.set(SQLConf.USE_DEPRECATED_KAFKA_OFFSET_FETCHING.key, "false")
+}
+
 class KafkaRelationSuiteV1 extends KafkaRelationSuiteBase {
   override protected def sparkConf: SparkConf =
     super
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSourceOffsetSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSourceOffsetSuite.scala
index ef902fcab3b50..553ab42b9c8b1 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSourceOffsetSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSourceOffsetSuite.scala
@@ -22,6 +22,7 @@ import java.io.File
 import org.apache.spark.sql.execution.streaming._
 import org.apache.spark.sql.streaming.OffsetSuite
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.util.Utils
 
 class KafkaSourceOffsetSuite extends OffsetSuite with SharedSparkSession {
 
@@ -99,7 +100,6 @@ class KafkaSourceOffsetSuite extends OffsetSuite with SharedSparkSession {
   private def readFromResource(file: String): SerializedOffset = {
     import scala.io.Source
     val input = getClass.getResource(s"/$file").toURI
-    val str = Source.fromFile(input).mkString
-    SerializedOffset(str)
+    SerializedOffset(Utils.tryWithResource(Source.fromFile(input))(_.mkString))
   }
 }
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
index 275a8170182fe..058563dfa167d 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
@@ -26,7 +26,6 @@ import javax.security.auth.login.Configuration
 
 import scala.collection.JavaConverters._
 import scala.io.Source
-import scala.util.Random
 import scala.util.control.NonFatal
 
 import com.google.common.io.Files
@@ -38,13 +37,12 @@ import org.apache.hadoop.minikdc.MiniKdc
 import org.apache.hadoop.security.UserGroupInformation
 import org.apache.kafka.clients.CommonClientConfigs
 import org.apache.kafka.clients.admin._
-import org.apache.kafka.clients.consumer.KafkaConsumer
 import org.apache.kafka.clients.producer._
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.config.SaslConfigs
 import org.apache.kafka.common.network.ListenerName
 import org.apache.kafka.common.security.auth.SecurityProtocol.{PLAINTEXT, SASL_PLAINTEXT}
-import org.apache.kafka.common.serialization.{StringDeserializer, StringSerializer}
+import org.apache.kafka.common.serialization.StringSerializer
 import org.apache.kafka.common.utils.SystemTime
 import org.apache.zookeeper.server.{NIOServerCnxnFactory, ZooKeeperServer}
 import org.apache.zookeeper.server.auth.SASLAuthenticationProvider
@@ -169,7 +167,8 @@ class KafkaTestUtils(
    * In this method we rewrite krb5.conf to make kdc and client use the same enctypes
    */
   private def rewriteKrb5Conf(): Unit = {
-    val krb5Conf = Source.fromFile(kdc.getKrb5conf, "UTF-8").getLines()
+    val krb5Conf = Utils
+      .tryWithResource(Source.fromFile(kdc.getKrb5conf, "UTF-8"))(_.getLines().toList)
     var rewritten = false
     val addedConfig =
       addedKrb5Config("default_tkt_enctypes", "aes128-cts-hmac-sha1-96") +
@@ -395,7 +394,7 @@ class KafkaTestUtils(
   }
 
   def getAllTopicsAndPartitionSize(): Seq[(String, Int)] = {
-    zkClient.getPartitionsForTopics(zkClient.getAllTopicsInCluster).mapValues(_.size).toSeq
+    zkClient.getPartitionsForTopics(zkClient.getAllTopicsInCluster()).mapValues(_.size).toSeq
   }
 
   /** Create a Kafka topic and wait until it is propagated to the whole cluster */
@@ -462,32 +461,24 @@ class KafkaTestUtils(
     server.logManager.cleanupLogs()
   }
 
+  private def getOffsets(topics: Set[String], offsetSpec: OffsetSpec): Map[TopicPartition, Long] = {
+    val listOffsetsParams = adminClient.describeTopics(topics.asJava).all().get().asScala
+      .flatMap { topicDescription =>
+        topicDescription._2.partitions().asScala.map { topicPartitionInfo =>
+          new TopicPartition(topicDescription._1, topicPartitionInfo.partition())
+        }
+      }.map(_ -> offsetSpec).toMap.asJava
+    val partitionOffsets = adminClient.listOffsets(listOffsetsParams).all().get().asScala
+      .map(result => result._1 -> result._2.offset()).toMap
+    partitionOffsets
+  }
+
   def getEarliestOffsets(topics: Set[String]): Map[TopicPartition, Long] = {
-    val kc = new KafkaConsumer[String, String](consumerConfiguration)
-    logInfo("Created consumer to get earliest offsets")
-    kc.subscribe(topics.asJavaCollection)
-    kc.poll(0)
-    val partitions = kc.assignment()
-    kc.pause(partitions)
-    kc.seekToBeginning(partitions)
-    val offsets = partitions.asScala.map(p => p -> kc.position(p)).toMap
-    kc.close()
-    logInfo("Closed consumer to get earliest offsets")
-    offsets
+    getOffsets(topics, OffsetSpec.earliest())
   }
 
   def getLatestOffsets(topics: Set[String]): Map[TopicPartition, Long] = {
-    val kc = new KafkaConsumer[String, String](consumerConfiguration)
-    logInfo("Created consumer to get latest offsets")
-    kc.subscribe(topics.asJavaCollection)
-    kc.poll(0)
-    val partitions = kc.assignment()
-    kc.pause(partitions)
-    kc.seekToEnd(partitions)
-    val offsets = partitions.asScala.map(p => p -> kc.position(p)).toMap
-    kc.close()
-    logInfo("Closed consumer to get latest offsets")
-    offsets
+    getOffsets(topics, OffsetSpec.latest())
   }
 
   def listConsumerGroups(): ListConsumerGroupsResult = {
@@ -547,7 +538,7 @@ class KafkaTestUtils(
   }
 
   /** Call `f` with a `KafkaProducer` that has initialized transactions. */
-  def withTranscationalProducer(f: KafkaProducer[String, String] => Unit): Unit = {
+  def withTransactionalProducer(f: KafkaProducer[String, String] => Unit): Unit = {
     val props = producerConfiguration
     props.put("transactional.id", UUID.randomUUID().toString)
     val producer = new KafkaProducer[String, String](props)
@@ -559,17 +550,6 @@ class KafkaTestUtils(
     }
   }
 
-  private def consumerConfiguration: Properties = {
-    val props = new Properties()
-    props.put("bootstrap.servers", brokerAddress)
-    props.put("group.id", "group-KafkaTestUtils-" + Random.nextInt)
-    props.put("value.deserializer", classOf[StringDeserializer].getName)
-    props.put("key.deserializer", classOf[StringDeserializer].getName)
-    props.put("enable.auto.commit", "false")
-    setAuthenticationConfigIfNeeded(props)
-    props
-  }
-
   private def setAuthenticationConfigIfNeeded(props: Properties): Unit = {
     if (secure) {
       val jaasParams = KafkaTokenUtil.getKeytabJaasParams(
@@ -595,18 +575,18 @@ class KafkaTestUtils(
       s"topic $topic still exists in the replica manager")
     // ensure that logs from all replicas are deleted if delete topic is marked successful
     assert(servers.forall(server => topicAndPartitions.forall(tp =>
-      server.getLogManager().getLog(tp).isEmpty)),
-      s"topic $topic still exists in log mananger")
+      server.getLogManager.getLog(tp).isEmpty)),
+      s"topic $topic still exists in log manager")
     // ensure that topic is removed from all cleaner offsets
     assert(servers.forall(server => topicAndPartitions.forall { tp =>
-      val checkpoints = server.getLogManager().liveLogDirs.map { logDir =>
+      val checkpoints = server.getLogManager.liveLogDirs.map { logDir =>
         new OffsetCheckpointFile(new File(logDir, "cleaner-offset-checkpoint")).read()
       }
       checkpoints.forall(checkpointsPerLogDir => !checkpointsPerLogDir.contains(tp))
     }), s"checkpoint for topic $topic still exists")
     // ensure the topic is gone
     assert(
-      !zkClient.getAllTopicsInCluster.contains(topic),
+      !zkClient.getAllTopicsInCluster().contains(topic),
       s"topic $topic still exists on zookeeper")
   }
 
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/consumer/FetchedDataPoolSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/consumer/FetchedDataPoolSuite.scala
index 23bab5cd48083..09d50ef0660cf 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/consumer/FetchedDataPoolSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/consumer/FetchedDataPoolSuite.scala
@@ -30,7 +30,7 @@ import org.scalatest.PrivateMethodTester
 
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.kafka010.{FETCHED_DATA_CACHE_EVICTOR_THREAD_RUN_INTERVAL, FETCHED_DATA_CACHE_TIMEOUT}
-import org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.CacheKey
+import org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.{AvailableOffsetRange, CacheKey}
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.util.ManualClock
 
@@ -69,7 +69,7 @@ class FetchedDataPoolSuite extends SharedSparkSession with PrivateMethodTester {
     assertFetchedDataPoolStatistic(dataPool, expectedNumCreated = 10, expectedNumTotal = 10)
 
     dataList.map { case (_, data) =>
-      data.withNewPoll(testRecords(0, 5).listIterator, 5)
+      data.withNewPoll(testRecords(0, 5).listIterator, 5, AvailableOffsetRange(0, 4))
     }
 
     dataList.foreach { case (key, data) =>
@@ -91,7 +91,7 @@ class FetchedDataPoolSuite extends SharedSparkSession with PrivateMethodTester {
     val cacheKey = CacheKey("testgroup", new TopicPartition("topic", 0))
 
     val data = dataPool.acquire(cacheKey, 0)
-    data.withNewPoll(testRecords(0, 5).listIterator, 5)
+    data.withNewPoll(testRecords(0, 5).listIterator, 5, AvailableOffsetRange(0, 4))
 
     (0 to 3).foreach { _ => data.next() }
 
@@ -130,14 +130,14 @@ class FetchedDataPoolSuite extends SharedSparkSession with PrivateMethodTester {
     assert(getCache(dataPool)(cacheKey)(1).inUse)
 
     // reading from task 1
-    dataFromTask1.withNewPoll(testRecords(0, 5).listIterator, 5)
+    dataFromTask1.withNewPoll(testRecords(0, 5).listIterator, 5, AvailableOffsetRange(0, 4))
 
     (0 to 3).foreach { _ => dataFromTask1.next() }
 
     dataPool.release(cacheKey, dataFromTask1)
 
     // reading from task 2
-    dataFromTask2.withNewPoll(testRecords(0, 30).listIterator, 30)
+    dataFromTask2.withNewPoll(testRecords(0, 30).listIterator, 30, AvailableOffsetRange(0, 29))
 
     (0 to 5).foreach { _ => dataFromTask2.next() }
 
@@ -189,7 +189,7 @@ class FetchedDataPoolSuite extends SharedSparkSession with PrivateMethodTester {
     assertFetchedDataPoolStatistic(dataPool, expectedNumCreated = 10, expectedNumTotal = 10)
 
     dataList.map { case (_, data) =>
-      data.withNewPoll(testRecords(0, 5).listIterator, 5)
+      data.withNewPoll(testRecords(0, 5).listIterator, 5, AvailableOffsetRange(0, 4))
     }
 
     val dataToEvict = dataList.take(3)
diff --git a/external/kafka-0-10-token-provider/pom.xml b/external/kafka-0-10-token-provider/pom.xml
index 941946f30e96f..688d8cd0e8c3e 100644
--- a/external/kafka-0-10-token-provider/pom.xml
+++ b/external/kafka-0-10-token-provider/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
@@ -52,12 +52,23 @@
       <groupId>org.apache.kafka</groupId>
       <artifactId>kafka-clients</artifactId>
       <version>${kafka.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>com.github.luben</groupId>
+          <artifactId>zstd-jni</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>${hadoop-client-runtime.artifact}</artifactId>
+      <scope>${hadoop.deps.scope}</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-tags_${scala.binary.version}</artifactId>
diff --git a/external/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaTokenSparkConf.scala b/external/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaTokenSparkConf.scala
index ed4a6f1e34c55..21ba7b21ed9d6 100644
--- a/external/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaTokenSparkConf.scala
+++ b/external/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaTokenSparkConf.scala
@@ -31,8 +31,10 @@ private[spark] case class KafkaTokenClusterConf(
     targetServersRegex: String,
     securityProtocol: String,
     kerberosServiceName: String,
+    trustStoreType: Option[String],
     trustStoreLocation: Option[String],
     trustStorePassword: Option[String],
+    keyStoreType: Option[String],
     keyStoreLocation: Option[String],
     keyStorePassword: Option[String],
     keyPassword: Option[String],
@@ -44,8 +46,10 @@ private[spark] case class KafkaTokenClusterConf(
     s"targetServersRegex=$targetServersRegex, " +
     s"securityProtocol=$securityProtocol, " +
     s"kerberosServiceName=$kerberosServiceName, " +
+    s"trustStoreType=$trustStoreType, " +
     s"trustStoreLocation=$trustStoreLocation, " +
     s"trustStorePassword=${trustStorePassword.map(_ => REDACTION_REPLACEMENT_TEXT)}, " +
+    s"keyStoreType=$keyStoreType, " +
     s"keyStoreLocation=$keyStoreLocation, " +
     s"keyStorePassword=${keyStorePassword.map(_ => REDACTION_REPLACEMENT_TEXT)}, " +
     s"keyPassword=${keyPassword.map(_ => REDACTION_REPLACEMENT_TEXT)}, " +
@@ -77,8 +81,10 @@ private [kafka010] object KafkaTokenSparkConf extends Logging {
         DEFAULT_SECURITY_PROTOCOL_CONFIG),
       sparkClusterConf.getOrElse(SaslConfigs.SASL_KERBEROS_SERVICE_NAME,
         KafkaTokenSparkConf.DEFAULT_SASL_KERBEROS_SERVICE_NAME),
+      sparkClusterConf.get(SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG),
       sparkClusterConf.get(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG),
       sparkClusterConf.get(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG),
+      sparkClusterConf.get(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG),
       sparkClusterConf.get(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG),
       sparkClusterConf.get(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG),
       sparkClusterConf.get(SslConfigs.SSL_KEY_PASSWORD_CONFIG),
diff --git a/external/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaTokenUtil.scala b/external/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaTokenUtil.scala
index 307a69f9b84c5..2b28ccaddc636 100644
--- a/external/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaTokenUtil.scala
+++ b/external/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaTokenUtil.scala
@@ -36,9 +36,8 @@ import org.apache.kafka.common.security.auth.SecurityProtocol.{SASL_PLAINTEXT, S
 import org.apache.kafka.common.security.scram.ScramLoginModule
 import org.apache.kafka.common.security.token.delegation.DelegationToken
 
-import org.apache.spark.{SparkConf, SparkEnv}
+import org.apache.spark.SparkConf
 import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.deploy.security.HadoopDelegationTokenManager
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config._
 import org.apache.spark.util.{SecurityUtils, Utils}
@@ -162,6 +161,9 @@ private[spark] object KafkaTokenUtil extends Logging {
   private def setTrustStoreProperties(
       clusterConf: KafkaTokenClusterConf,
       properties: ju.Properties): Unit = {
+    clusterConf.trustStoreType.foreach { truststoreType =>
+      properties.put(SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG, truststoreType)
+    }
     clusterConf.trustStoreLocation.foreach { truststoreLocation =>
       properties.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, truststoreLocation)
     }
@@ -173,6 +175,9 @@ private[spark] object KafkaTokenUtil extends Logging {
   private def setKeyStoreProperties(
       clusterConf: KafkaTokenClusterConf,
       properties: ju.Properties): Unit = {
+    clusterConf.keyStoreType.foreach { keystoreType =>
+      properties.put(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG, keystoreType)
+    }
     clusterConf.keyStoreLocation.foreach { keystoreLocation =>
       properties.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, keystoreLocation)
     }
@@ -270,11 +275,9 @@ private[spark] object KafkaTokenUtil extends Logging {
   }
 
   def needTokenUpdate(
-      sparkConf: SparkConf,
       params: ju.Map[String, Object],
       clusterConfig: Option[KafkaTokenClusterConf]): Boolean = {
-    if (HadoopDelegationTokenManager.isServiceEnabled(sparkConf, "kafka") &&
-        clusterConfig.isDefined && params.containsKey(SaslConfigs.SASL_JAAS_CONFIG)) {
+    if (clusterConfig.isDefined && params.containsKey(SaslConfigs.SASL_JAAS_CONFIG)) {
       logDebug("Delegation token used by connector, checking if uses the latest token.")
       val connectorJaasParams = params.get(SaslConfigs.SASL_JAAS_CONFIG).asInstanceOf[String]
       getTokenJaasParams(clusterConfig.get) != connectorJaasParams
diff --git a/external/kafka-0-10-token-provider/src/test/scala/org/apache/spark/kafka010/KafkaDelegationTokenTest.scala b/external/kafka-0-10-token-provider/src/test/scala/org/apache/spark/kafka010/KafkaDelegationTokenTest.scala
index 19335f4221e40..8271acdc7dfb6 100644
--- a/external/kafka-0-10-token-provider/src/test/scala/org/apache/spark/kafka010/KafkaDelegationTokenTest.scala
+++ b/external/kafka-0-10-token-provider/src/test/scala/org/apache/spark/kafka010/KafkaDelegationTokenTest.scala
@@ -51,8 +51,10 @@ trait KafkaDelegationTokenTest extends BeforeAndAfterEach {
   protected val bootStrapServers = "127.0.0.1:0"
   protected val matchingTargetServersRegex = "127.0.0.*:0"
   protected val nonMatchingTargetServersRegex = "127.0.intentionally_non_matching.*:0"
+  protected val trustStoreType = "customTrustStoreType"
   protected val trustStoreLocation = "/path/to/trustStore"
   protected val trustStorePassword = "trustStoreSecret"
+  protected val keyStoreType = "customKeyStoreType"
   protected val keyStoreLocation = "/path/to/keyStore"
   protected val keyStorePassword = "keyStoreSecret"
   protected val keyPassword = "keySecret"
@@ -124,8 +126,10 @@ trait KafkaDelegationTokenTest extends BeforeAndAfterEach {
       KafkaTokenSparkConf.DEFAULT_TARGET_SERVERS_REGEX,
       securityProtocol,
       KafkaTokenSparkConf.DEFAULT_SASL_KERBEROS_SERVICE_NAME,
+      Some(trustStoreType),
       Some(trustStoreLocation),
       Some(trustStorePassword),
+      Some(keyStoreType),
       Some(keyStoreLocation),
       Some(keyStorePassword),
       Some(keyPassword),
diff --git a/external/kafka-0-10-token-provider/src/test/scala/org/apache/spark/kafka010/KafkaTokenSparkConfSuite.scala b/external/kafka-0-10-token-provider/src/test/scala/org/apache/spark/kafka010/KafkaTokenSparkConfSuite.scala
index 61184a6fac33d..17caf96818e47 100644
--- a/external/kafka-0-10-token-provider/src/test/scala/org/apache/spark/kafka010/KafkaTokenSparkConfSuite.scala
+++ b/external/kafka-0-10-token-provider/src/test/scala/org/apache/spark/kafka010/KafkaTokenSparkConfSuite.scala
@@ -29,8 +29,10 @@ class KafkaTokenSparkConfSuite extends SparkFunSuite with BeforeAndAfterEach {
   private val targetServersRegex = "127.0.0.*:0"
   private val securityProtocol = SSL.name
   private val kerberosServiceName = "kafka1"
+  private val trustStoreType = "customTrustStoreType"
   private val trustStoreLocation = "/path/to/trustStore"
   private val trustStorePassword = "trustStoreSecret"
+  private val keyStoreType = "customKeyStoreType"
   private val keyStoreLocation = "/path/to/keyStore"
   private val keyStorePassword = "keyStoreSecret"
   private val keyPassword = "keySecret"
@@ -60,8 +62,10 @@ class KafkaTokenSparkConfSuite extends SparkFunSuite with BeforeAndAfterEach {
     assert(clusterConfig.securityProtocol === SASL_SSL.name)
     assert(clusterConfig.kerberosServiceName ===
       KafkaTokenSparkConf.DEFAULT_SASL_KERBEROS_SERVICE_NAME)
+    assert(clusterConfig.trustStoreType === None)
     assert(clusterConfig.trustStoreLocation === None)
     assert(clusterConfig.trustStorePassword === None)
+    assert(clusterConfig.keyStoreType === None)
     assert(clusterConfig.keyStoreLocation === None)
     assert(clusterConfig.keyStorePassword === None)
     assert(clusterConfig.keyPassword === None)
@@ -75,8 +79,10 @@ class KafkaTokenSparkConfSuite extends SparkFunSuite with BeforeAndAfterEach {
     sparkConf.set(s"spark.kafka.clusters.$identifier1.security.protocol", securityProtocol)
     sparkConf.set(s"spark.kafka.clusters.$identifier1.sasl.kerberos.service.name",
       kerberosServiceName)
+    sparkConf.set(s"spark.kafka.clusters.$identifier1.ssl.truststore.type", trustStoreType)
     sparkConf.set(s"spark.kafka.clusters.$identifier1.ssl.truststore.location", trustStoreLocation)
     sparkConf.set(s"spark.kafka.clusters.$identifier1.ssl.truststore.password", trustStorePassword)
+    sparkConf.set(s"spark.kafka.clusters.$identifier1.ssl.keystore.type", keyStoreType)
     sparkConf.set(s"spark.kafka.clusters.$identifier1.ssl.keystore.location", keyStoreLocation)
     sparkConf.set(s"spark.kafka.clusters.$identifier1.ssl.keystore.password", keyStorePassword)
     sparkConf.set(s"spark.kafka.clusters.$identifier1.ssl.key.password", keyPassword)
@@ -88,8 +94,10 @@ class KafkaTokenSparkConfSuite extends SparkFunSuite with BeforeAndAfterEach {
     assert(clusterConfig.targetServersRegex === targetServersRegex)
     assert(clusterConfig.securityProtocol === securityProtocol)
     assert(clusterConfig.kerberosServiceName === kerberosServiceName)
+    assert(clusterConfig.trustStoreType === Some(trustStoreType))
     assert(clusterConfig.trustStoreLocation === Some(trustStoreLocation))
     assert(clusterConfig.trustStorePassword === Some(trustStorePassword))
+    assert(clusterConfig.keyStoreType === Some(keyStoreType))
     assert(clusterConfig.keyStoreLocation === Some(keyStoreLocation))
     assert(clusterConfig.keyStorePassword === Some(keyStorePassword))
     assert(clusterConfig.keyPassword === Some(keyPassword))
@@ -127,8 +135,10 @@ class KafkaTokenSparkConfSuite extends SparkFunSuite with BeforeAndAfterEach {
       assert(clusterConfig.securityProtocol === SASL_SSL.name)
       assert(clusterConfig.kerberosServiceName ===
         KafkaTokenSparkConf.DEFAULT_SASL_KERBEROS_SERVICE_NAME)
+      assert(clusterConfig.trustStoreType === None)
       assert(clusterConfig.trustStoreLocation === None)
       assert(clusterConfig.trustStorePassword === None)
+      assert(clusterConfig.keyStoreType === None)
       assert(clusterConfig.keyStoreLocation === None)
       assert(clusterConfig.keyStorePassword === None)
       assert(clusterConfig.keyPassword === None)
diff --git a/external/kafka-0-10-token-provider/src/test/scala/org/apache/spark/kafka010/KafkaTokenUtilSuite.scala b/external/kafka-0-10-token-provider/src/test/scala/org/apache/spark/kafka010/KafkaTokenUtilSuite.scala
index 94f7853003bd9..7e4efb948b77f 100644
--- a/external/kafka-0-10-token-provider/src/test/scala/org/apache/spark/kafka010/KafkaTokenUtilSuite.scala
+++ b/external/kafka-0-10-token-provider/src/test/scala/org/apache/spark/kafka010/KafkaTokenUtilSuite.scala
@@ -28,7 +28,7 @@ import org.apache.kafka.clients.CommonClientConfigs
 import org.apache.kafka.common.config.{SaslConfigs, SslConfigs}
 import org.apache.kafka.common.security.auth.SecurityProtocol.{SASL_PLAINTEXT, SASL_SSL, SSL}
 
-import org.apache.spark.{SparkConf, SparkEnv, SparkFunSuite}
+import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.internal.config._
 
 class KafkaTokenUtilSuite extends SparkFunSuite with KafkaDelegationTokenTest {
@@ -64,8 +64,10 @@ class KafkaTokenUtilSuite extends SparkFunSuite with KafkaDelegationTokenTest {
       === bootStrapServers)
     assert(adminClientProperties.get(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG)
       === SASL_PLAINTEXT.name)
+    assert(!adminClientProperties.containsKey(SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG))
     assert(!adminClientProperties.containsKey(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG))
     assert(!adminClientProperties.containsKey(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG))
+    assert(!adminClientProperties.containsKey(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG))
     assert(!adminClientProperties.containsKey(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG))
     assert(!adminClientProperties.containsKey(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG))
     assert(!adminClientProperties.containsKey(SslConfigs.SSL_KEY_PASSWORD_CONFIG))
@@ -80,10 +82,12 @@ class KafkaTokenUtilSuite extends SparkFunSuite with KafkaDelegationTokenTest {
       === bootStrapServers)
     assert(adminClientProperties.get(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG)
       === SASL_SSL.name)
+    assert(adminClientProperties.get(SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG) === trustStoreType)
     assert(adminClientProperties.get(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG)
       === trustStoreLocation)
     assert(adminClientProperties.get(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG)
       === trustStorePassword)
+    assert(!adminClientProperties.containsKey(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG))
     assert(!adminClientProperties.containsKey(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG))
     assert(!adminClientProperties.containsKey(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG))
     assert(!adminClientProperties.containsKey(SslConfigs.SSL_KEY_PASSWORD_CONFIG))
@@ -99,10 +103,12 @@ class KafkaTokenUtilSuite extends SparkFunSuite with KafkaDelegationTokenTest {
       === bootStrapServers)
     assert(adminClientProperties.get(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG)
       === SSL.name)
+    assert(adminClientProperties.get(SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG) === trustStoreType)
     assert(adminClientProperties.get(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG)
       === trustStoreLocation)
     assert(adminClientProperties.get(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG)
       === trustStorePassword)
+    assert(adminClientProperties.get(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG) === keyStoreType)
     assert(adminClientProperties.get(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG) === keyStoreLocation)
     assert(adminClientProperties.get(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG) === keyStorePassword)
     assert(adminClientProperties.get(SslConfigs.SSL_KEY_PASSWORD_CONFIG) === keyPassword)
@@ -233,24 +239,17 @@ class KafkaTokenUtilSuite extends SparkFunSuite with KafkaDelegationTokenTest {
     assert(jaasParams.contains(tokenPassword1))
   }
 
-  test("needTokenUpdate without security credentials enabled should return false") {
-    sparkConf.set(s"spark.security.credentials.kafka.enabled", "false")
-    val kafkaParams = getKafkaParams(addJaasConfig = true, Some("custom_jaas_config"))
-
-    assert(!KafkaTokenUtil.needTokenUpdate(sparkConf, kafkaParams, None))
-  }
-
   test("needTokenUpdate without cluster config should return false") {
     val kafkaParams = getKafkaParams(addJaasConfig = true, Some("custom_jaas_config"))
 
-    assert(!KafkaTokenUtil.needTokenUpdate(sparkConf, kafkaParams, None))
+    assert(!KafkaTokenUtil.needTokenUpdate(kafkaParams, None))
   }
 
   test("needTokenUpdate without jaas config should return false") {
     setSparkEnv(Map.empty)
     val kafkaParams = getKafkaParams(addJaasConfig = false)
 
-    assert(!KafkaTokenUtil.needTokenUpdate(SparkEnv.get.conf, kafkaParams, None))
+    assert(!KafkaTokenUtil.needTokenUpdate(kafkaParams, None))
   }
 
   test("needTokenUpdate with same token should return false") {
@@ -260,7 +259,7 @@ class KafkaTokenUtilSuite extends SparkFunSuite with KafkaDelegationTokenTest {
     val clusterConfig = KafkaTokenUtil.findMatchingTokenClusterConfig(sparkConf,
       kafkaParams.get(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG).asInstanceOf[String])
 
-    assert(!KafkaTokenUtil.needTokenUpdate(sparkConf, kafkaParams, clusterConfig))
+    assert(!KafkaTokenUtil.needTokenUpdate(kafkaParams, clusterConfig))
   }
 
   test("needTokenUpdate with different token should return true") {
@@ -271,7 +270,7 @@ class KafkaTokenUtilSuite extends SparkFunSuite with KafkaDelegationTokenTest {
     val clusterConfig = KafkaTokenUtil.findMatchingTokenClusterConfig(sparkConf,
       kafkaParams.get(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG).asInstanceOf[String])
 
-    assert(KafkaTokenUtil.needTokenUpdate(sparkConf, kafkaParams, clusterConfig))
+    assert(KafkaTokenUtil.needTokenUpdate(kafkaParams, clusterConfig))
   }
 
   private def getKafkaParams(
diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml
index 024fdb26d5bf4..65ff607dafa93 100644
--- a/external/kafka-0-10/pom.xml
+++ b/external/kafka-0-10/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
@@ -63,6 +63,12 @@
       <groupId>org.apache.kafka</groupId>
       <artifactId>kafka-clients</artifactId>
       <version>${kafka.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>com.github.luben</groupId>
+          <artifactId>zstd-jni</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.kafka</groupId>
diff --git a/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/ConsumerStrategy.scala b/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/ConsumerStrategy.scala
index 3e32b592b3a3a..ab6550ddf2fb3 100644
--- a/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/ConsumerStrategy.scala
+++ b/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/ConsumerStrategy.scala
@@ -241,7 +241,7 @@ object ConsumerStrategies {
     new Subscribe[K, V](
       new ju.ArrayList(topics.asJavaCollection),
       new ju.HashMap[String, Object](kafkaParams.asJava),
-      new ju.HashMap[TopicPartition, jl.Long](offsets.mapValues(jl.Long.valueOf).asJava))
+      new ju.HashMap[TopicPartition, jl.Long](offsets.mapValues(jl.Long.valueOf).toMap.asJava))
   }
 
   /**
@@ -320,7 +320,7 @@ object ConsumerStrategies {
     new SubscribePattern[K, V](
       pattern,
       new ju.HashMap[String, Object](kafkaParams.asJava),
-      new ju.HashMap[TopicPartition, jl.Long](offsets.mapValues(jl.Long.valueOf).asJava))
+      new ju.HashMap[TopicPartition, jl.Long](offsets.mapValues(jl.Long.valueOf).toMap.asJava))
   }
 
   /**
@@ -404,7 +404,7 @@ object ConsumerStrategies {
     new Assign[K, V](
       new ju.ArrayList(topicPartitions.asJavaCollection),
       new ju.HashMap[String, Object](kafkaParams.asJava),
-      new ju.HashMap[TopicPartition, jl.Long](offsets.mapValues(jl.Long.valueOf).asJava))
+      new ju.HashMap[TopicPartition, jl.Long](offsets.mapValues(jl.Long.valueOf).toMap.asJava))
   }
 
   /**
diff --git a/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/DirectKafkaInputDStream.scala b/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/DirectKafkaInputDStream.scala
index a449a8bb7213e..fb86111cb1876 100644
--- a/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/DirectKafkaInputDStream.scala
+++ b/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/DirectKafkaInputDStream.scala
@@ -70,7 +70,8 @@ private[spark] class DirectKafkaInputDStream[K, V](
   @transient private var kc: Consumer[K, V] = null
   def consumer(): Consumer[K, V] = this.synchronized {
     if (null == kc) {
-      kc = consumerStrategy.onStart(currentOffsets.mapValues(l => java.lang.Long.valueOf(l)).asJava)
+      kc = consumerStrategy.onStart(
+        currentOffsets.mapValues(l => java.lang.Long.valueOf(l)).toMap.asJava)
     }
     kc
   }
@@ -315,7 +316,7 @@ private[spark] class DirectKafkaInputDStream[K, V](
     override def update(time: Time): Unit = {
       batchForTime.clear()
       generatedRDDs.foreach { kv =>
-        val a = kv._2.asInstanceOf[KafkaRDD[K, V]].offsetRanges.map(_.toTuple).toArray
+        val a = kv._2.asInstanceOf[KafkaRDD[K, V]].offsetRanges.map(_.toTuple)
         batchForTime += kv._1 -> a
       }
     }
diff --git a/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaDataConsumer.scala b/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaDataConsumer.scala
index 142e946188ace..09af5a0815147 100644
--- a/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaDataConsumer.scala
+++ b/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaDataConsumer.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.streaming.kafka010
 
 import java.{util => ju}
+import java.time.Duration
 
 import scala.collection.JavaConverters._
 
@@ -203,7 +204,7 @@ private[kafka010] class InternalKafkaConsumer[K, V](
   }
 
   private def poll(timeout: Long): Unit = {
-    val p = consumer.poll(timeout)
+    val p = consumer.poll(Duration.ofMillis(timeout))
     val r = p.records(topicPartition)
     logDebug(s"Polled ${p.partitions()}  ${r.size}")
     buffer = r.listIterator
diff --git a/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaRDD.scala b/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaRDD.scala
index bd2e7e11b7383..46164e9b63365 100644
--- a/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaRDD.scala
+++ b/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaRDD.scala
@@ -248,7 +248,7 @@ private class KafkaRDDIterator[K, V](
     }
   }
 
-  override def hasNext(): Boolean = requestOffset < part.untilOffset
+  override def hasNext: Boolean = requestOffset < part.untilOffset
 
   override def next(): ConsumerRecord[K, V] = {
     if (!hasNext) {
@@ -291,7 +291,7 @@ private class CompactedKafkaRDDIterator[K, V](
 
   private var okNext: Boolean = true
 
-  override def hasNext(): Boolean = okNext
+  override def hasNext: Boolean = okNext
 
   override def next(): ConsumerRecord[K, V] = {
     if (!hasNext) {
diff --git a/external/kafka-0-10/src/test/java/org/apache/spark/streaming/kafka010/JavaConsumerStrategySuite.java b/external/kafka-0-10/src/test/java/org/apache/spark/streaming/kafka010/JavaConsumerStrategySuite.java
index dc364aca9bd3b..3d6e5ebe978e8 100644
--- a/external/kafka-0-10/src/test/java/org/apache/spark/streaming/kafka010/JavaConsumerStrategySuite.java
+++ b/external/kafka-0-10/src/test/java/org/apache/spark/streaming/kafka010/JavaConsumerStrategySuite.java
@@ -48,15 +48,12 @@ public void testConsumerStrategyConstructors() {
       JavaConverters.mapAsScalaMapConverter(kafkaParams).asScala();
     final Map<TopicPartition, Long> offsets = new HashMap<>();
     offsets.put(tp1, 23L);
+    final Map<TopicPartition, Object> dummyOffsets = new HashMap<>();
+    for (Map.Entry<TopicPartition, Long> kv : offsets.entrySet()) {
+      dummyOffsets.put(kv.getKey(), kv.getValue());
+    }
     final scala.collection.Map<TopicPartition, Object> sOffsets =
-      JavaConverters.mapAsScalaMapConverter(offsets).asScala().mapValues(
-        new scala.runtime.AbstractFunction1<Long, Object>() {
-          @Override
-          public Object apply(Long x) {
-            return (Object) x;
-          }
-        }
-      );
+      JavaConverters.mapAsScalaMap(dummyOffsets);
 
     final ConsumerStrategy<String, String> sub1 =
       ConsumerStrategies.Subscribe(sTopics, sKafkaParams, sOffsets);
diff --git a/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala b/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala
index 72cf3e8118228..2b7fef1e0fde3 100644
--- a/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala
+++ b/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala
@@ -332,8 +332,7 @@ class DirectKafkaStreamSuite
   }
 
   // Test to verify the offset ranges can be recovered from the checkpoints
-  // TODO (SPARK-31722): re-enable it
-  ignore("offset recovery") {
+  test("offset recovery") {
     val topic = "recovery"
     kafkaTestUtils.createTopic(topic)
     testDir = Utils.createTempDir()
@@ -420,8 +419,7 @@ class DirectKafkaStreamSuite
   }
 
   // Test to verify the offsets can be recovered from Kafka
-  // TODO (SPARK-31722): re-enable it
-  ignore("offset recovery from kafka") {
+  test("offset recovery from kafka") {
     val topic = "recoveryfromkafka"
     kafkaTestUtils.createTopic(topic)
 
diff --git a/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala b/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala
index d6123e16dd238..b9ef16fb58cb9 100644
--- a/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala
+++ b/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala
@@ -42,7 +42,7 @@ class KafkaRDDSuite extends SparkFunSuite with BeforeAndAfterAll {
   private val sparkConf = new SparkConf().setMaster("local[4]")
     .setAppName(this.getClass.getSimpleName)
     // Set a timeout of 10 seconds that's going to be used to fetch topics/partitions from kafka.
-    // Othewise the poll timeout defaults to 2 minutes and causes test cases to run longer.
+    // Otherwise the poll timeout defaults to 2 minutes and causes test cases to run longer.
     .set("spark.streaming.kafka.consumer.poll.ms", "10000")
 
   private var sc: SparkContext = _
@@ -236,7 +236,7 @@ class KafkaRDDSuite extends SparkFunSuite with BeforeAndAfterAll {
 
     // this is the "lots of messages" case
     kafkaTestUtils.sendMessages(topic, sent)
-    var sentCount = sent.values.sum
+    val sentCount = sent.values.sum
 
     val rdd = KafkaUtils.createRDD[String, String](sc, kafkaParams,
       Array(OffsetRange(topic, 0, 0, sentCount)), preferredHosts)
diff --git a/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/mocks/MockScheduler.scala b/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/mocks/MockScheduler.scala
index ac81f92f86109..c0724909bc350 100644
--- a/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/mocks/MockScheduler.scala
+++ b/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/mocks/MockScheduler.scala
@@ -19,8 +19,6 @@ package org.apache.spark.streaming.kafka010.mocks
 
 import java.util.concurrent.{ScheduledFuture, TimeUnit}
 
-import scala.collection.mutable.PriorityQueue
-
 import kafka.utils.Scheduler
 import org.apache.kafka.common.utils.Time
 import org.jmock.lib.concurrent.DeterministicScheduler
diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml
index 76ee5bb7b2f85..ae41cf69fc704 100644
--- a/external/kinesis-asl-assembly/pom.xml
+++ b/external/kinesis-asl-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
@@ -91,18 +91,18 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-client</artifactId>
+      <artifactId>${hadoop-client-api.artifact}</artifactId>
+      <version>${hadoop.version}</version>
       <scope>provided</scope>
     </dependency>
     <dependency>
-      <groupId>org.apache.avro</groupId>
-      <artifactId>avro-ipc</artifactId>
-      <scope>provided</scope>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>${hadoop-client-runtime.artifact}</artifactId>
+      <version>${hadoop.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro-mapred</artifactId>
-      <classifier>${avro.mapred.classifier}</classifier>
       <scope>provided</scope>
     </dependency>
     <dependency>
diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml
index 7e80bd28c19e8..b54ad71eba305 100644
--- a/external/kinesis-asl/pom.xml
+++ b/external/kinesis-asl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java b/external/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java
index 31ca2fe5c95ff..244873af70de9 100644
--- a/external/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java
+++ b/external/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java
@@ -49,7 +49,7 @@
  *
  * Usage: JavaKinesisWordCountASL [app-name] [stream-name] [endpoint-url] [region-name]
  *   [app-name] is the name of the consumer app, used to track the read data in DynamoDB
- *   [stream-name] name of the Kinesis stream (ie. mySparkStream)
+ *   [stream-name] name of the Kinesis stream (i.e. mySparkStream)
  *   [endpoint-url] endpoint of the Kinesis service
  *     (e.g. https://kinesis.us-east-1.amazonaws.com)
  *
@@ -57,7 +57,7 @@
  * Example:
  *      # export AWS keys if necessary
  *      $ export AWS_ACCESS_KEY_ID=[your-access-key]
- *      $ export AWS_SECRET_KEY=<your-secret-key>
+ *      $ export AWS_SECRET_ACCESS_KEY=<your-secret-key>
  *
  *      # run the example
  *      $ SPARK_HOME/bin/run-example   streaming.JavaKinesisWordCountASL myAppName  mySparkStream \
@@ -68,7 +68,7 @@
  *
  * This code uses the DefaultAWSCredentialsProviderChain to find credentials
  * in the following order:
- *    Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_KEY
+ *    Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
  *    Java System Properties - aws.accessKeyId and aws.secretKey
  *    Credential profiles file - default location (~/.aws/credentials) shared by all AWS SDKs
  *    Instance profile credentials - delivered through the Amazon EC2 metadata service
diff --git a/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py b/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
index 777a33270c415..06ada13b52399 100644
--- a/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
+++ b/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
@@ -23,7 +23,7 @@
 
   Usage: kinesis_wordcount_asl.py <app-name> <stream-name> <endpoint-url> <region-name>
     <app-name> is the name of the consumer app, used to track the read data in DynamoDB
-    <stream-name> name of the Kinesis stream (ie. mySparkStream)
+    <stream-name> name of the Kinesis stream (i.e. mySparkStream)
     <endpoint-url> endpoint of the Kinesis service
       (e.g. https://kinesis.us-east-1.amazonaws.com)
     <region-name> region name of the Kinesis endpoint (e.g. us-east-1)
@@ -32,7 +32,7 @@
   Example:
       # export AWS keys if necessary
       $ export AWS_ACCESS_KEY_ID=<your-access-key>
-      $ export AWS_SECRET_KEY=<your-secret-key>
+      $ export AWS_SECRET_ACCESS_KEY=<your-secret-key>
 
       # run the example
       $ bin/spark-submit --jars \
@@ -45,7 +45,7 @@
 
   This code uses the DefaultAWSCredentialsProviderChain to find credentials
   in the following order:
-      Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_KEY
+      Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
       Java System Properties - aws.accessKeyId and aws.secretKey
       Credential profiles file - default location (~/.aws/credentials) shared by all AWS SDKs
       Instance profile credentials - delivered through the Amazon EC2 metadata service
@@ -55,8 +55,6 @@
   See http://spark.apache.org/docs/latest/streaming-kinesis-integration.html for more details on
   the Kinesis Spark Streaming integration.
 """
-from __future__ import print_function
-
 import sys
 
 from pyspark import SparkContext
diff --git a/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
index 32f4a6759474f..d6a9160eed98e 100644
--- a/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
+++ b/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
@@ -43,7 +43,7 @@ import org.apache.spark.streaming.kinesis.KinesisInputDStream
  *
  * Usage: KinesisWordCountASL <app-name> <stream-name> <endpoint-url> <region-name>
  *   <app-name> is the name of the consumer app, used to track the read data in DynamoDB
- *   <stream-name> name of the Kinesis stream (ie. mySparkStream)
+ *   <stream-name> name of the Kinesis stream (i.e. mySparkStream)
  *   <endpoint-url> endpoint of the Kinesis service
  *     (e.g. https://kinesis.us-east-1.amazonaws.com)
  *
@@ -51,7 +51,7 @@ import org.apache.spark.streaming.kinesis.KinesisInputDStream
  * Example:
  *      # export AWS keys if necessary
  *      $ export AWS_ACCESS_KEY_ID=<your-access-key>
- *      $ export AWS_SECRET_KEY=<your-secret-key>
+ *      $ export AWS_SECRET_ACCESS_KEY=<your-secret-key>
  *
  *      # run the example
  *      $ SPARK_HOME/bin/run-example  streaming.KinesisWordCountASL myAppName  mySparkStream \
@@ -62,7 +62,7 @@ import org.apache.spark.streaming.kinesis.KinesisInputDStream
  *
  * This code uses the DefaultAWSCredentialsProviderChain to find credentials
  * in the following order:
- *    Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_KEY
+ *    Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
  *    Java System Properties - aws.accessKeyId and aws.secretKey
  *    Credential profiles file - default location (~/.aws/credentials) shared by all AWS SDKs
  *    Instance profile credentials - delivered through the Amazon EC2 metadata service
@@ -167,9 +167,9 @@ object KinesisWordCountASL extends Logging {
  * Usage: KinesisWordProducerASL <stream-name> <endpoint-url> \
  *   <records-per-sec> <words-per-record>
  *
- *   <stream-name> is the name of the Kinesis stream (ie. mySparkStream)
+ *   <stream-name> is the name of the Kinesis stream (i.e. mySparkStream)
  *   <endpoint-url> is the endpoint of the Kinesis service
- *     (ie. https://kinesis.us-east-1.amazonaws.com)
+ *     (i.e. https://kinesis.us-east-1.amazonaws.com)
  *   <records-per-sec> is the rate of records per second to put onto the stream
  *   <words-per-record> is the number of words per record
  *
diff --git a/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala
index 5072b3ae21d87..ab55d545770e9 100644
--- a/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala
+++ b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala
@@ -91,7 +91,7 @@ class KinesisBackedBlockRDD[T: ClassTag](
   require(_blockIds.length == arrayOfseqNumberRanges.length,
     "Number of blockIds is not equal to the number of sequence number ranges")
 
-  override def isValid(): Boolean = true
+  override def isValid: Boolean = true
 
   override def getPartitions: Array[Partition] = {
     Array.tabulate(_blockIds.length) { i =>
diff --git a/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala
index 11e949536f2b6..770eb2d89d522 100644
--- a/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala
+++ b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala
@@ -21,7 +21,6 @@ import java.util.concurrent._
 import scala.util.control.NonFatal
 
 import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer
-import com.amazonaws.services.kinesis.clientlibrary.lib.worker.ShutdownReason
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.streaming.Duration
diff --git a/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisInputDStream.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisInputDStream.scala
index 8c3931a1c87fd..e778d083b3f70 100644
--- a/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisInputDStream.scala
+++ b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisInputDStream.scala
@@ -50,8 +50,6 @@ private[kinesis] class KinesisInputDStream[T: ClassTag](
     val metricsEnabledDimensions: Set[String]
   ) extends ReceiverInputDStream[T](_ssc) {
 
-  import KinesisReadConfigurations._
-
   private[streaming]
   override def createBlockRDD(time: Time, blockInfos: Seq[ReceivedBlockInfo]): RDD[T] = {
 
diff --git a/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala
index 2ac83c8ee6b38..4dacc03c72719 100644
--- a/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala
+++ b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala
@@ -97,7 +97,7 @@ private[kinesis] class KinesisTestUtils(streamShardCount: Int = 2) extends Loggi
   }
 
   def getShards(): Seq[Shard] = {
-    kinesisClient.describeStream(_streamName).getStreamDescription.getShards.asScala
+    kinesisClient.describeStream(_streamName).getStreamDescription.getShards.asScala.toSeq
   }
 
   def splitShard(shardId: String): Unit = {
@@ -137,7 +137,7 @@ private[kinesis] class KinesisTestUtils(streamShardCount: Int = 2) extends Loggi
    * Expose a Python friendly API.
    */
   def pushData(testData: java.util.List[Int]): Unit = {
-    pushData(testData.asScala, aggregate = false)
+    pushData(testData.asScala.toSeq, aggregate = false)
   }
 
   def deleteStream(): Unit = {
@@ -289,6 +289,6 @@ private[kinesis] class SimpleDataGenerator(
       sentSeqNumbers += ((num, seqNumber))
     }
 
-    shardIdToSeqNumbers.toMap
+    shardIdToSeqNumbers.mapValues(_.toSeq).toMap
   }
 }
diff --git a/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisUtilsPythonHelper.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisUtilsPythonHelper.scala
index c89dedd3366d1..0056438c4eefb 100644
--- a/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisUtilsPythonHelper.scala
+++ b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisUtilsPythonHelper.scala
@@ -46,7 +46,7 @@ private class KinesisUtilsPythonHelper {
     // scalastyle:on
     if (!(stsAssumeRoleArn != null && stsSessionName != null && stsExternalId != null)
         && !(stsAssumeRoleArn == null && stsSessionName == null && stsExternalId == null)) {
-      throw new IllegalArgumentException("stsAssumeRoleArn, stsSessionName, and stsExtenalId " +
+      throw new IllegalArgumentException("stsAssumeRoleArn, stsSessionName, and stsExternalId " +
         "must all be defined or all be null")
     }
     if (awsAccessKeyId == null && awsSecretKey != null) {
diff --git a/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KPLBasedKinesisTestUtils.scala b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KPLBasedKinesisTestUtils.scala
index af84498d5e47e..c76eb7c29dd94 100644
--- a/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KPLBasedKinesisTestUtils.scala
+++ b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KPLBasedKinesisTestUtils.scala
@@ -72,6 +72,6 @@ private[kinesis] class KPLDataGenerator(regionName: String) extends KinesisDataG
       Futures.addCallback(future, kinesisCallBack, ThreadUtils.sameThreadExecutorService)
     }
     producer.flushSync()
-    shardIdToSeqNumbers.toMap
+    shardIdToSeqNumbers.mapValues(_.toSeq).toMap
   }
 }
diff --git a/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDDSuite.scala b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDDSuite.scala
index 2c7b9c58e6fa6..12d950096b4c2 100644
--- a/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDDSuite.scala
+++ b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDDSuite.scala
@@ -47,8 +47,8 @@ abstract class KinesisBackedBlockRDDTests(aggregateTestData: Boolean)
       require(shardIdToDataAndSeqNumbers.size > 1, "Need data to be sent to multiple shards")
 
       shardIds = shardIdToDataAndSeqNumbers.keySet.toSeq
-      shardIdToData = shardIdToDataAndSeqNumbers.mapValues { _.map { _._1 }}
-      shardIdToSeqNumbers = shardIdToDataAndSeqNumbers.mapValues { _.map { _._2 }}
+      shardIdToData = shardIdToDataAndSeqNumbers.mapValues(_.map(_._1)).toMap
+      shardIdToSeqNumbers = shardIdToDataAndSeqNumbers.mapValues(_.map(_._2)).toMap
       shardIdToRange = shardIdToSeqNumbers.map { case (shardId, seqNumbers) =>
         val seqNumRange = SequenceNumberRange(
           testUtils.streamName, shardId, seqNumbers.head, seqNumbers.last, seqNumbers.size)
diff --git a/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala
index 470a8cecc8fd9..d008de3b3f1c4 100644
--- a/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala
+++ b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala
@@ -26,7 +26,9 @@ import com.amazonaws.services.kinesis.clientlibrary.lib.worker.ShutdownReason
 import com.amazonaws.services.kinesis.model.Record
 import org.mockito.ArgumentMatchers.{anyList, anyString, eq => meq}
 import org.mockito.Mockito.{never, times, verify, when}
-import org.scalatest.{BeforeAndAfter, Matchers}
+import org.scalatest.BeforeAndAfter
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 import org.scalatestplus.mockito.MockitoSugar
 
 import org.apache.spark.streaming.{Duration, TestSuiteBase}
diff --git a/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisStreamSuite.scala b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisStreamSuite.scala
index eee62d25e62bb..647f17859d041 100644
--- a/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisStreamSuite.scala
+++ b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisStreamSuite.scala
@@ -23,8 +23,8 @@ import scala.util.Random
 
 import com.amazonaws.services.kinesis.model.Record
 import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll}
-import org.scalatest.Matchers._
 import org.scalatest.concurrent.Eventually
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.{SparkConf, SparkContext}
 import org.apache.spark.network.util.JavaUtils
diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml
index 728b489da6785..bbb71035c3e19 100644
--- a/external/spark-ganglia-lgpl/pom.xml
+++ b/external/spark-ganglia-lgpl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/spark-ganglia-lgpl/src/main/scala/org/apache/spark/metrics/sink/GangliaSink.scala b/external/spark-ganglia-lgpl/src/main/scala/org/apache/spark/metrics/sink/GangliaSink.scala
index 4fb9f2f849085..2b48a34abb8fe 100644
--- a/external/spark-ganglia-lgpl/src/main/scala/org/apache/spark/metrics/sink/GangliaSink.scala
+++ b/external/spark-ganglia-lgpl/src/main/scala/org/apache/spark/metrics/sink/GangliaSink.scala
@@ -25,11 +25,10 @@ import com.codahale.metrics.ganglia.GangliaReporter
 import info.ganglia.gmetric4j.gmetric.GMetric
 import info.ganglia.gmetric4j.gmetric.GMetric.UDPAddressingMode
 
-import org.apache.spark.SecurityManager
 import org.apache.spark.metrics.MetricsSystem
 
-class GangliaSink(val property: Properties, val registry: MetricRegistry,
-    securityMgr: SecurityManager) extends Sink {
+class GangliaSink(
+    val property: Properties, val registry: MetricRegistry) extends Sink {
   val GANGLIA_KEY_PERIOD = "period"
   val GANGLIA_DEFAULT_PERIOD = 10
 
@@ -81,15 +80,15 @@ class GangliaSink(val property: Properties, val registry: MetricRegistry,
       .withDMax(dmax)
       .build(ganglia)
 
-  override def start() {
+  override def start(): Unit = {
     reporter.start(pollPeriod, pollUnit)
   }
 
-  override def stop() {
+  override def stop(): Unit = {
     reporter.stop()
   }
 
-  override def report() {
+  override def report(): Unit = {
     reporter.report()
   }
 }
diff --git a/graphx/pom.xml b/graphx/pom.xml
index 38836db01553a..c4fa38a1dc9e5 100644
--- a/graphx/pom.xml
+++ b/graphx/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
@@ -60,9 +60,8 @@
       <artifactId>guava</artifactId>
     </dependency>
     <dependency>
-      <groupId>com.github.fommil.netlib</groupId>
-      <artifactId>core</artifactId>
-      <version>${netlib.java.version}</version>
+      <groupId>dev.ludovic.netlib</groupId>
+      <artifactId>blas</artifactId>
     </dependency>
     <dependency>
       <groupId>net.sourceforge.f2j</groupId>
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/lib/PageRank.scala b/graphx/src/main/scala/org/apache/spark/graphx/lib/PageRank.scala
index 102dc2d2dd4b0..5dd6f13235ccb 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/lib/PageRank.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/lib/PageRank.scala
@@ -140,8 +140,32 @@ object PageRank extends Logging {
    */
   def runWithOptions[VD: ClassTag, ED: ClassTag](
       graph: Graph[VD, ED], numIter: Int, resetProb: Double = 0.15,
-      srcId: Option[VertexId] = None): Graph[Double, Double] =
-  {
+      srcId: Option[VertexId] = None): Graph[Double, Double] = {
+    runWithOptions(graph, numIter, resetProb, srcId, normalized = true)
+  }
+
+  /**
+   * Run PageRank for a fixed number of iterations returning a graph
+   * with vertex attributes containing the PageRank and edge
+   * attributes the normalized edge weight.
+   *
+   * @tparam VD the original vertex attribute (not used)
+   * @tparam ED the original edge attribute (not used)
+   *
+   * @param graph the graph on which to compute PageRank
+   * @param numIter the number of iterations of PageRank to run
+   * @param resetProb the random reset probability (alpha)
+   * @param srcId the source vertex for a Personalized Page Rank (optional)
+   * @param normalized whether or not to normalize rank sum
+   *
+   * @return the graph containing with each vertex containing the PageRank and each edge
+   *         containing the normalized weight.
+   *
+   * @since 3.2.0
+   */
+  def runWithOptions[VD: ClassTag, ED: ClassTag](
+      graph: Graph[VD, ED], numIter: Int, resetProb: Double,
+      srcId: Option[VertexId], normalized: Boolean): Graph[Double, Double] = {
     require(numIter > 0, s"Number of iterations must be greater than 0," +
       s" but got ${numIter}")
     require(resetProb >= 0 && resetProb <= 1, s"Random reset probability must belong" +
@@ -179,8 +203,13 @@ object PageRank extends Logging {
       iteration += 1
     }
 
-    // SPARK-18847 If the graph has sinks (vertices with no outgoing edges) correct the sum of ranks
-    normalizeRankSum(rankGraph, personalized)
+    if (normalized) {
+      // SPARK-18847 If the graph has sinks (vertices with no outgoing edges),
+      // correct the sum of ranks
+      normalizeRankSum(rankGraph, personalized)
+    } else {
+      rankGraph
+    }
   }
 
   /**
@@ -204,6 +233,34 @@ object PageRank extends Logging {
   def runWithOptionsWithPreviousPageRank[VD: ClassTag, ED: ClassTag](
       graph: Graph[VD, ED], numIter: Int, resetProb: Double, srcId: Option[VertexId],
       preRankGraph: Graph[Double, Double]): Graph[Double, Double] = {
+    runWithOptionsWithPreviousPageRank(
+      graph, numIter, resetProb, srcId, normalized = true, preRankGraph
+    )
+  }
+
+  /**
+   * Run PageRank for a fixed number of iterations returning a graph
+   * with vertex attributes containing the PageRank and edge
+   * attributes the normalized edge weight.
+   *
+   * @tparam VD the original vertex attribute (not used)
+   * @tparam ED the original edge attribute (not used)
+   *
+   * @param graph the graph on which to compute PageRank
+   * @param numIter the number of iterations of PageRank to run
+   * @param resetProb the random reset probability (alpha)
+   * @param srcId the source vertex for a Personalized Page Rank (optional)
+   * @param normalized whether or not to normalize rank sum
+   * @param preRankGraph PageRank graph from which to keep iterating
+   *
+   * @return the graph containing with each vertex containing the PageRank and each edge
+   *         containing the normalized weight.
+   *
+   * @since 3.2.0
+   */
+  def runWithOptionsWithPreviousPageRank[VD: ClassTag, ED: ClassTag](
+      graph: Graph[VD, ED], numIter: Int, resetProb: Double, srcId: Option[VertexId],
+      normalized: Boolean, preRankGraph: Graph[Double, Double]): Graph[Double, Double] = {
     require(numIter > 0, s"Number of iterations must be greater than 0," +
       s" but got ${numIter}")
     require(resetProb >= 0 && resetProb <= 1, s"Random reset probability must belong" +
@@ -238,8 +295,13 @@ object PageRank extends Logging {
       iteration += 1
     }
 
-    // SPARK-18847 If the graph has sinks (vertices with no outgoing edges) correct the sum of ranks
-    normalizeRankSum(rankGraph, personalized)
+    if (normalized) {
+      // SPARK-18847 If the graph has sinks (vertices with no outgoing edges),
+      // correct the sum of ranks
+      normalizeRankSum(rankGraph, personalized)
+    } else {
+      rankGraph
+    }
   }
 
   /**
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/lib/SVDPlusPlus.scala b/graphx/src/main/scala/org/apache/spark/graphx/lib/SVDPlusPlus.scala
index c508056fe3ae3..d7099c5c953c1 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/lib/SVDPlusPlus.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/lib/SVDPlusPlus.scala
@@ -19,9 +19,8 @@ package org.apache.spark.graphx.lib
 
 import scala.util.Random
 
-import com.github.fommil.netlib.BLAS.{getInstance => blas}
-
 import org.apache.spark.graphx._
+import org.apache.spark.ml.linalg.BLAS
 import org.apache.spark.rdd._
 
 /** Implementation of SVD++ algorithm. */
@@ -102,22 +101,22 @@ object SVDPlusPlus {
       val (usr, itm) = (ctx.srcAttr, ctx.dstAttr)
       val (p, q) = (usr._1, itm._1)
       val rank = p.length
-      var pred = u + usr._3 + itm._3 + blas.ddot(rank, q, 1, usr._2, 1)
+      var pred = u + usr._3 + itm._3 + BLAS.nativeBLAS.ddot(rank, q, 1, usr._2, 1)
       pred = math.max(pred, conf.minVal)
       pred = math.min(pred, conf.maxVal)
       val err = ctx.attr - pred
       // updateP = (err * q - conf.gamma7 * p) * conf.gamma2
       val updateP = q.clone()
-      blas.dscal(rank, err * conf.gamma2, updateP, 1)
-      blas.daxpy(rank, -conf.gamma7 * conf.gamma2, p, 1, updateP, 1)
+      BLAS.nativeBLAS.dscal(rank, err * conf.gamma2, updateP, 1)
+      BLAS.nativeBLAS.daxpy(rank, -conf.gamma7 * conf.gamma2, p, 1, updateP, 1)
       // updateQ = (err * usr._2 - conf.gamma7 * q) * conf.gamma2
       val updateQ = usr._2.clone()
-      blas.dscal(rank, err * conf.gamma2, updateQ, 1)
-      blas.daxpy(rank, -conf.gamma7 * conf.gamma2, q, 1, updateQ, 1)
+      BLAS.nativeBLAS.dscal(rank, err * conf.gamma2, updateQ, 1)
+      BLAS.nativeBLAS.daxpy(rank, -conf.gamma7 * conf.gamma2, q, 1, updateQ, 1)
       // updateY = (err * usr._4 * q - conf.gamma7 * itm._2) * conf.gamma2
       val updateY = q.clone()
-      blas.dscal(rank, err * usr._4 * conf.gamma2, updateY, 1)
-      blas.daxpy(rank, -conf.gamma7 * conf.gamma2, itm._2, 1, updateY, 1)
+      BLAS.nativeBLAS.dscal(rank, err * usr._4 * conf.gamma2, updateY, 1)
+      BLAS.nativeBLAS.daxpy(rank, -conf.gamma7 * conf.gamma2, itm._2, 1, updateY, 1)
       ctx.sendToSrc((updateP, updateY, (err - conf.gamma6 * usr._3) * conf.gamma1))
       ctx.sendToDst((updateQ, updateY, (err - conf.gamma6 * itm._3) * conf.gamma1))
     }
@@ -129,7 +128,7 @@ object SVDPlusPlus {
         ctx => ctx.sendToSrc(ctx.dstAttr._2),
         (g1, g2) => {
           val out = g1.clone()
-          blas.daxpy(out.length, 1.0, g2, 1, out, 1)
+          BLAS.nativeBLAS.daxpy(out.length, 1.0, g2, 1, out, 1)
           out
         })
       val gJoinT1 = g.outerJoinVertices(t1) {
@@ -137,7 +136,7 @@ object SVDPlusPlus {
          msg: Option[Array[Double]]) =>
           if (msg.isDefined) {
             val out = vd._1.clone()
-            blas.daxpy(out.length, vd._4, msg.get, 1, out, 1)
+            BLAS.nativeBLAS.daxpy(out.length, vd._4, msg.get, 1, out, 1)
             (vd._1, out, vd._3, vd._4)
           } else {
             vd
@@ -154,9 +153,9 @@ object SVDPlusPlus {
         (g1: (Array[Double], Array[Double], Double), g2: (Array[Double], Array[Double], Double)) =>
         {
           val out1 = g1._1.clone()
-          blas.daxpy(out1.length, 1.0, g2._1, 1, out1, 1)
+          BLAS.nativeBLAS.daxpy(out1.length, 1.0, g2._1, 1, out1, 1)
           val out2 = g2._2.clone()
-          blas.daxpy(out2.length, 1.0, g2._2, 1, out2, 1)
+          BLAS.nativeBLAS.daxpy(out2.length, 1.0, g2._2, 1, out2, 1)
           (out1, out2, g1._3 + g2._3)
         })
       val gJoinT2 = g.outerJoinVertices(t2) {
@@ -164,9 +163,9 @@ object SVDPlusPlus {
          vd: (Array[Double], Array[Double], Double, Double),
          msg: Option[(Array[Double], Array[Double], Double)]) => {
           val out1 = vd._1.clone()
-          blas.daxpy(out1.length, 1.0, msg.get._1, 1, out1, 1)
+          BLAS.nativeBLAS.daxpy(out1.length, 1.0, msg.get._1, 1, out1, 1)
           val out2 = vd._2.clone()
-          blas.daxpy(out2.length, 1.0, msg.get._2, 1, out2, 1)
+          BLAS.nativeBLAS.daxpy(out2.length, 1.0, msg.get._2, 1, out2, 1)
           (out1, out2, vd._3 + msg.get._3, vd._4)
         }
       }.cache()
@@ -180,7 +179,7 @@ object SVDPlusPlus {
         (ctx: EdgeContext[(Array[Double], Array[Double], Double, Double), Double, Double]): Unit = {
       val (usr, itm) = (ctx.srcAttr, ctx.dstAttr)
       val (p, q) = (usr._1, itm._1)
-      var pred = u + usr._3 + itm._3 + blas.ddot(q.length, q, 1, usr._2, 1)
+      var pred = u + usr._3 + itm._3 + BLAS.nativeBLAS.ddot(q.length, q, 1, usr._2, 1)
       pred = math.max(pred, conf.minVal)
       pred = math.min(pred, conf.maxVal)
       val err = (ctx.attr - pred) * (ctx.attr - pred)
@@ -198,7 +197,7 @@ object SVDPlusPlus {
     g = gJoinT3
 
     // Convert DoubleMatrix to Array[Double]:
-    val newVertices = g.vertices.mapValues(v => (v._1.toArray, v._2.toArray, v._3, v._4))
+    val newVertices = g.vertices.mapValues(v => (v._1, v._2, v._3, v._4))
     (Graph(newVertices, g.edges), u)
   }
 
diff --git a/graphx/src/test/scala/org/apache/spark/graphx/lib/PageRankSuite.scala b/graphx/src/test/scala/org/apache/spark/graphx/lib/PageRankSuite.scala
index a5e2fc5c9a74f..caa2fdcdf5d2b 100644
--- a/graphx/src/test/scala/org/apache/spark/graphx/lib/PageRankSuite.scala
+++ b/graphx/src/test/scala/org/apache/spark/graphx/lib/PageRankSuite.scala
@@ -233,7 +233,37 @@ class PageRankSuite extends SparkFunSuite with LocalSparkContext {
       assert(totalIters == 19)
       assert(iterAfterHalfCheckPoint == 18)
     }
-  } // end of Grid PageRank
+  } // end of Grid PageRank with checkpoint
+
+  test("Grid PageRank with checkpoint without intermediate normalization") {
+    withSpark { sc =>
+      // Check that 6 iterations in a row are equivalent
+      // to 3 times 2 iterations without intermediate normalization
+      val rows = 10
+      val cols = 10
+      val resetProb = 0.15
+      val gridGraph = GraphGenerators.gridGraph(sc, rows, cols).cache()
+
+      val ranksA: Array[(VertexId, Double)] = PageRank.runWithOptions(
+        gridGraph, numIter = 6, resetProb, srcId = None, normalized = true
+      ).vertices.collect()
+
+      val preRankGraph1 = PageRank.runWithOptions(
+        gridGraph, numIter = 2, resetProb, srcId = None, normalized = false
+      )
+
+      val preRankGraph2 = PageRank.runWithOptionsWithPreviousPageRank(
+        gridGraph, numIter = 2, resetProb, srcId = None, normalized = false, preRankGraph1
+      )
+
+      val ranksB: Array[(VertexId, Double)] = PageRank.runWithOptionsWithPreviousPageRank(
+        gridGraph, numIter = 2, resetProb, srcId = None, normalized = true, preRankGraph2
+      ).vertices.collect()
+
+      // assert that all scores are equal
+      assert(ranksA.zip(ranksB).forall { case(rankA, rankB) => rankA == rankB })
+    }
+  } // end of Grid PageRank with checkpoint without intermediate normalization
 
   test("Chain PageRank") {
     withSpark { sc =>
@@ -274,8 +304,8 @@ class PageRankSuite extends SparkFunSuite with LocalSparkContext {
     withSpark { sc =>
       // Check that implementation can handle large vertexIds, SPARK-25149
       val vertexIdOffset = Int.MaxValue.toLong + 1
-      val sourceOffest = 4
-      val source = vertexIdOffset + sourceOffest
+      val sourceOffset = 4
+      val source = vertexIdOffset + sourceOffset
       val numIter = 10
       val vertices = vertexIdOffset until vertexIdOffset + numIter
       val chain1 = vertices.zip(vertices.tail)
@@ -285,7 +315,7 @@ class PageRankSuite extends SparkFunSuite with LocalSparkContext {
       val tol = 0.0001
       val errorTol = 1.0e-1
 
-      val a = resetProb / (1 - Math.pow(1 - resetProb, numIter - sourceOffest))
+      val a = resetProb / (1 - Math.pow(1 - resetProb, numIter - sourceOffset))
       // We expect the rank to decay as (1 - resetProb) ^ distance
       val expectedRanks = sc.parallelize(vertices).map { vid =>
         val rank = if (vid < source) {
diff --git a/graphx/src/test/scala/org/apache/spark/graphx/lib/ShortestPathsSuite.scala b/graphx/src/test/scala/org/apache/spark/graphx/lib/ShortestPathsSuite.scala
index 994395bbffa56..f909b703ac6e0 100644
--- a/graphx/src/test/scala/org/apache/spark/graphx/lib/ShortestPathsSuite.scala
+++ b/graphx/src/test/scala/org/apache/spark/graphx/lib/ShortestPathsSuite.scala
@@ -34,7 +34,7 @@ class ShortestPathsSuite extends SparkFunSuite with LocalSparkContext {
       val graph = Graph.fromEdgeTuples(edges, 1)
       val landmarks = Seq(1, 4).map(_.toLong)
       val results = ShortestPaths.run(graph, landmarks).vertices.collect.map {
-        case (v, spMap) => (v, spMap.mapValues(i => i))
+        case (v, spMap) => (v, spMap.mapValues(i => i).toMap)
       }
       assert(results.toSet === shortestPaths)
     }
diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml
index 8689e0b8a9ea8..c0997e5e0a1cc 100644
--- a/hadoop-cloud/pom.xml
+++ b/hadoop-cloud/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
@@ -58,10 +58,15 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-client</artifactId>
+      <artifactId>${hadoop-client-api.artifact}</artifactId>
       <version>${hadoop.version}</version>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>${hadoop-client-runtime.artifact}</artifactId>
+      <version>${hadoop.version}</version>
+    </dependency>
     <!--
       the AWS module pulls in jackson; its transitive dependencies can create
       intra-jackson-module version problems.
diff --git a/launcher/pom.xml b/launcher/pom.xml
index a2550ac939e83..dd872f4648132 100644
--- a/launcher/pom.xml
+++ b/launcher/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
@@ -81,7 +81,14 @@
     <!-- Not needed by the test code, but referenced by SparkSubmit which is used by the tests. -->
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-client</artifactId>
+      <artifactId>${hadoop-client-api.artifact}</artifactId>
+      <version>${hadoop.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>${hadoop-client-runtime.artifact}</artifactId>
+      <version>${hadoop.version}</version>
       <scope>test</scope>
     </dependency>
   </dependencies>
diff --git a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
index 3ae4633c79b04..24ad9cbdba087 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
@@ -92,17 +92,13 @@ abstract List<String> buildCommand(Map<String, String> env)
   List<String> buildJavaCommand(String extraClassPath) throws IOException {
     List<String> cmd = new ArrayList<>();
 
-    String[] candidateJavaHomes = new String[] {
-      javaHome,
+    String firstJavaHome = firstNonEmpty(javaHome,
       childEnv.get("JAVA_HOME"),
       System.getenv("JAVA_HOME"),
-      System.getProperty("java.home")
-    };
-    for (String javaHome : candidateJavaHomes) {
-      if (javaHome != null) {
-        cmd.add(join(File.separator, javaHome, "bin", "java"));
-        break;
-      }
+      System.getProperty("java.home"));
+
+    if (firstJavaHome != null) {
+      cmd.add(join(File.separator, firstJavaHome, "bin", "java"));
     }
 
     // Load extra JAVA_OPTS from conf/java-opts, if it exists.
@@ -235,20 +231,17 @@ String getScalaVersion() {
       return scala;
     }
     String sparkHome = getSparkHome();
-    // TODO: revisit for Scala 2.13 support
     File scala212 = new File(sparkHome, "launcher/target/scala-2.12");
-    // File scala211 = new File(sparkHome, "launcher/target/scala-2.11");
-    // checkState(!scala212.isDirectory() || !scala211.isDirectory(),
-    //   "Presence of build for multiple Scala versions detected.\n" +
-    //   "Either clean one of them or set SPARK_SCALA_VERSION in your environment.");
-    // if (scala212.isDirectory()) {
-    //   return "2.12";
-    // } else {
-    //   checkState(scala211.isDirectory(), "Cannot find any build directories.");
-    //   return "2.11";
-    // }
-    checkState(scala212.isDirectory(), "Cannot find any build directories.");
-    return "2.12";
+    File scala213 = new File(sparkHome, "launcher/target/scala-2.13");
+    checkState(!scala212.isDirectory() || !scala213.isDirectory(),
+      "Presence of build for multiple Scala versions detected.\n" +
+      "Either clean one of them or set SPARK_SCALA_VERSION in your environment.");
+    if (scala213.isDirectory()) {
+      return "2.13";
+    } else {
+      checkState(scala212.isDirectory(), "Cannot find any build directories.");
+      return "2.12";
+    }
   }
 
   String getSparkHome() {
diff --git a/launcher/src/main/java/org/apache/spark/launcher/LauncherServer.java b/launcher/src/main/java/org/apache/spark/launcher/LauncherServer.java
index 3ff77878f68a8..125205f416d35 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/LauncherServer.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/LauncherServer.java
@@ -263,7 +263,19 @@ public void run() {
 
   private long getConnectionTimeout() {
     String value = SparkLauncher.launcherConfig.get(SparkLauncher.CHILD_CONNECTION_TIMEOUT);
-    return (value != null) ? Long.parseLong(value) : DEFAULT_CONNECT_TIMEOUT;
+    if (value != null) {
+        return Long.parseLong(value);
+    }
+
+    value = SparkLauncher.launcherConfig.get(SparkLauncher.DEPRECATED_CHILD_CONNECTION_TIMEOUT);
+    if (value != null) {
+        LOG.log(Level.WARNING,
+                "Property '" + SparkLauncher.DEPRECATED_CHILD_CONNECTION_TIMEOUT +
+                "' is deprecated, please switch to '" + SparkLauncher.CHILD_CONNECTION_TIMEOUT +
+                "'.");
+        return Long.parseLong(value);
+    }
+    return DEFAULT_CONNECT_TIMEOUT;
   }
 
   private String createSecret() {
@@ -364,7 +376,7 @@ public void close() throws IOException {
      *
      * This method allows a short period for the above to happen (same amount of time as the
      * connection timeout, which is configurable). This should be fine for well-behaved
-     * applications, where they close the connection arond the same time the app handle detects the
+     * applications, where they close the connection around the same time the app handle detects the
      * app has finished.
      *
      * In case the connection is not closed within the grace period, this method forcefully closes
diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java b/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java
index 32844104c1deb..12febc5441bd6 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java
@@ -89,11 +89,21 @@ public class SparkLauncher extends AbstractLauncher<SparkLauncher> {
    */
   public static final String NO_RESOURCE = "spark-internal";
 
+  /**
+   * Maximum time (in ms) to wait for a child process to connect back to the launcher server
+   * when using @link{#start()}.
+   *
+   * @deprecated use `CHILD_CONNECTION_TIMEOUT`
+   * @since 1.6.0
+   */
+  public static final String DEPRECATED_CHILD_CONNECTION_TIMEOUT =
+    "spark.launcher.childConectionTimeout";
+
   /**
    * Maximum time (in ms) to wait for a child process to connect back to the launcher server
    * when using @link{#start()}.
    */
-  public static final String CHILD_CONNECTION_TIMEOUT = "spark.launcher.childConectionTimeout";
+  public static final String CHILD_CONNECTION_TIMEOUT = "spark.launcher.childConnectionTimeout";
 
   /** Used internally to create unique logger names. */
   private static final AtomicInteger COUNTER = new AtomicInteger();
diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
index 383c3f60a595b..b2c12973bcabd 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
@@ -139,7 +139,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder {
 
         case RUN_EXAMPLE:
           isExample = true;
-          appResource = SparkLauncher.NO_RESOURCE;
+          appResource = findExamplesAppJar();
           submitArgs = args.subList(1, args.size());
       }
 
@@ -241,9 +241,11 @@ List<String> buildSparkSubmitArgs() {
     }
 
     args.addAll(parsedArgs);
+
     if (appResource != null) {
       args.add(appResource);
     }
+
     args.addAll(appArgs);
 
     return args;
@@ -334,7 +336,7 @@ private List<String> buildPySparkShellCommand(Map<String, String> env) throws IO
       conf.get(SparkLauncher.PYSPARK_PYTHON),
       System.getenv("PYSPARK_DRIVER_PYTHON"),
       System.getenv("PYSPARK_PYTHON"),
-      "python"));
+      "python3"));
     String pyOpts = System.getenv("PYSPARK_DRIVER_PYTHON_OPTS");
     if (conf.containsKey(SparkLauncher.PYSPARK_PYTHON)) {
       // pass conf spark.pyspark.python to python by environment variable.
@@ -401,6 +403,20 @@ private boolean isThriftServer(String mainClass) {
       mainClass.equals("org.apache.spark.sql.hive.thriftserver.HiveThriftServer2"));
   }
 
+  private String findExamplesAppJar() {
+    boolean isTesting = "1".equals(getenv("SPARK_TESTING"));
+    if (isTesting) {
+      return SparkLauncher.NO_RESOURCE;
+    } else {
+      for (String exampleJar : findExamplesJars()) {
+        if (new File(exampleJar).getName().startsWith("spark-examples")) {
+          return exampleJar;
+        }
+      }
+      throw new IllegalStateException("Failed to find examples' main app jar.");
+    }
+  }
+
   private List<String> findExamplesJars() {
     boolean isTesting = "1".equals(getenv("SPARK_TESTING"));
     List<String> examplesJars = new ArrayList<>();
@@ -513,7 +529,7 @@ protected boolean handleUnknown(String opt) {
           className = EXAMPLE_CLASS_PREFIX + className;
         }
         mainClass = className;
-        appResource = SparkLauncher.NO_RESOURCE;
+        appResource = findExamplesAppJar();
         return false;
       } else if (errorOnUnknownArgs) {
         checkArgument(!opt.startsWith("-"), "Unrecognized option: %s", opt);
diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
index 752e8d4c23f8b..d892ebcf3b250 100644
--- a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
+++ b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
@@ -26,11 +26,7 @@
 import java.util.Map;
 import java.util.regex.Pattern;
 
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
+import org.junit.*;
 
 import static org.junit.Assert.*;
 
@@ -39,9 +35,6 @@ public class SparkSubmitCommandBuilderSuite extends BaseSuite {
   private static File dummyPropsFile;
   private static SparkSubmitOptionParser parser;
 
-  @Rule
-  public ExpectedException expectedException = ExpectedException.none();
-
   @BeforeClass
   public static void setUp() throws Exception {
     dummyPropsFile = File.createTempFile("spark", "properties");
@@ -216,15 +209,13 @@ public void testExamplesRunnerNoMainClass() throws Exception {
 
   @Test
   public void testExamplesRunnerWithMasterNoMainClass() throws Exception {
-    expectedException.expect(IllegalArgumentException.class);
-    expectedException.expectMessage("Missing example class name.");
-
     List<String> sparkSubmitArgs = Arrays.asList(
       SparkSubmitCommandBuilder.RUN_EXAMPLE,
       parser.MASTER + "=foo"
     );
     Map<String, String> env = new HashMap<>();
-    buildCommand(sparkSubmitArgs, env);
+    Assert.assertThrows("Missing example class name.", IllegalArgumentException.class,
+      () -> buildCommand(sparkSubmitArgs, env));
   }
 
   @Test
@@ -245,6 +236,24 @@ public void testExamplesRunner() throws Exception {
     assertEquals("42", cmd.get(cmd.size() - 1));
   }
 
+  @Test
+  public void testExamplesRunnerPrimaryResource() throws Exception {
+    List<String> sparkSubmitArgs = Arrays.asList(
+            SparkSubmitCommandBuilder.RUN_EXAMPLE,
+            parser.MASTER + "=foo",
+            parser.DEPLOY_MODE + "=cluster",
+            "SparkPi",
+            "100");
+
+    List<String> cmd = newCommandBuilder(sparkSubmitArgs).buildSparkSubmitArgs();
+    assertEquals(SparkSubmitCommandBuilder.EXAMPLE_CLASS_PREFIX + "SparkPi",
+            findArgValue(cmd, parser.CLASS));
+    assertEquals("cluster", findArgValue(cmd, parser.DEPLOY_MODE));
+    String primaryResource = cmd.get(cmd.size() - 2);
+    assertTrue(primaryResource.equals(SparkLauncher.NO_RESOURCE)
+            || new File(primaryResource).getName().startsWith("spark-examples"));
+  }
+
   @Test(expected = IllegalArgumentException.class)
   public void testMissingAppResource() {
     new SparkSubmitCommandBuilder().buildSparkSubmitArgs();
diff --git a/licenses-binary/LICENSE-blas.txt b/licenses-binary/LICENSE-blas.txt
new file mode 100644
index 0000000000000..2b8bec28b0d3b
--- /dev/null
+++ b/licenses-binary/LICENSE-blas.txt
@@ -0,0 +1,25 @@
+MIT License
+-----------
+
+Copyright 2020, 2021, Ludovic Henry
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+Please contact git@ludovic.dev or visit ludovic.dev if you need additional
+information or have any questions.
diff --git a/licenses-binary/LICENSE-heapq.txt b/licenses-binary/LICENSE-heapq.txt
deleted file mode 100644
index 0c4c4b954bea4..0000000000000
--- a/licenses-binary/LICENSE-heapq.txt
+++ /dev/null
@@ -1,280 +0,0 @@
-
-# A. HISTORY OF THE SOFTWARE
-# ==========================
-#
-# Python was created in the early 1990s by Guido van Rossum at Stichting
-# Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands
-# as a successor of a language called ABC.  Guido remains Python's
-# principal author, although it includes many contributions from others.
-#
-# In 1995, Guido continued his work on Python at the Corporation for
-#     National Research Initiatives (CNRI, see http://www.cnri.reston.va.us)
-# in Reston, Virginia where he released several versions of the
-# software.
-#
-# In May 2000, Guido and the Python core development team moved to
-# BeOpen.com to form the BeOpen PythonLabs team.  In October of the same
-# year, the PythonLabs team moved to Digital Creations (now Zope
-# Corporation, see http://www.zope.com).  In 2001, the Python Software
-# Foundation (PSF, see http://www.python.org/psf/) was formed, a
-# non-profit organization created specifically to own Python-related
-# Intellectual Property.  Zope Corporation is a sponsoring member of
-# the PSF.
-#
-# All Python releases are Open Source (see http://www.opensource.org for
-# the Open Source Definition).  Historically, most, but not all, Python
-# releases have also been GPL-compatible; the table below summarizes
-# the various releases.
-#
-# Release         Derived     Year        Owner       GPL-
-# from                                compatible? (1)
-#
-# 0.9.0 thru 1.2              1991-1995   CWI         yes
-# 1.3 thru 1.5.2  1.2         1995-1999   CNRI        yes
-# 1.6             1.5.2       2000        CNRI        no
-# 2.0             1.6         2000        BeOpen.com  no
-# 1.6.1           1.6         2001        CNRI        yes (2)
-# 2.1             2.0+1.6.1   2001        PSF         no
-# 2.0.1           2.0+1.6.1   2001        PSF         yes
-# 2.1.1           2.1+2.0.1   2001        PSF         yes
-# 2.2             2.1.1       2001        PSF         yes
-# 2.1.2           2.1.1       2002        PSF         yes
-# 2.1.3           2.1.2       2002        PSF         yes
-# 2.2.1           2.2         2002        PSF         yes
-# 2.2.2           2.2.1       2002        PSF         yes
-# 2.2.3           2.2.2       2003        PSF         yes
-# 2.3             2.2.2       2002-2003   PSF         yes
-# 2.3.1           2.3         2002-2003   PSF         yes
-# 2.3.2           2.3.1       2002-2003   PSF         yes
-# 2.3.3           2.3.2       2002-2003   PSF         yes
-# 2.3.4           2.3.3       2004        PSF         yes
-# 2.3.5           2.3.4       2005        PSF         yes
-# 2.4             2.3         2004        PSF         yes
-# 2.4.1           2.4         2005        PSF         yes
-# 2.4.2           2.4.1       2005        PSF         yes
-# 2.4.3           2.4.2       2006        PSF         yes
-# 2.4.4           2.4.3       2006        PSF         yes
-# 2.5             2.4         2006        PSF         yes
-# 2.5.1           2.5         2007        PSF         yes
-# 2.5.2           2.5.1       2008        PSF         yes
-# 2.5.3           2.5.2       2008        PSF         yes
-# 2.6             2.5         2008        PSF         yes
-# 2.6.1           2.6         2008        PSF         yes
-# 2.6.2           2.6.1       2009        PSF         yes
-# 2.6.3           2.6.2       2009        PSF         yes
-# 2.6.4           2.6.3       2009        PSF         yes
-# 2.6.5           2.6.4       2010        PSF         yes
-# 2.7             2.6         2010        PSF         yes
-#
-# Footnotes:
-#
-# (1) GPL-compatible doesn't mean that we're distributing Python under
-# the GPL.  All Python licenses, unlike the GPL, let you distribute
-# a modified version without making your changes open source.  The
-# GPL-compatible licenses make it possible to combine Python with
-#     other software that is released under the GPL; the others don't.
-#
-# (2) According to Richard Stallman, 1.6.1 is not GPL-compatible,
-# because its license has a choice of law clause.  According to
-# CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1
-# is "not incompatible" with the GPL.
-#
-# Thanks to the many outside volunteers who have worked under Guido's
-# direction to make these releases possible.
-#
-#
-# B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON
-# ===============================================================
-#
-# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
-# --------------------------------------------
-#
-# 1. This LICENSE AGREEMENT is between the Python Software Foundation
-# ("PSF"), and the Individual or Organization ("Licensee") accessing and
-# otherwise using this software ("Python") in source or binary form and
-# its associated documentation.
-#
-# 2. Subject to the terms and conditions of this License Agreement, PSF hereby
-# grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
-# analyze, test, perform and/or display publicly, prepare derivative works,
-# distribute, and otherwise use Python alone or in any derivative version,
-# provided, however, that PSF's License Agreement and PSF's notice of copyright,
-# i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
-# 2011, 2012, 2013 Python Software Foundation; All Rights Reserved" are retained
-# in Python alone or in any derivative version prepared by Licensee.
-#
-# 3. In the event Licensee prepares a derivative work that is based on
-# or incorporates Python or any part thereof, and wants to make
-# the derivative work available to others as provided herein, then
-# Licensee hereby agrees to include in any such work a brief summary of
-# the changes made to Python.
-#
-# 4. PSF is making Python available to Licensee on an "AS IS"
-# basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
-# IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
-# DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
-# FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
-# INFRINGE ANY THIRD PARTY RIGHTS.
-#
-# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
-# FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
-# A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
-# OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
-#
-# 6. This License Agreement will automatically terminate upon a material
-# breach of its terms and conditions.
-#
-# 7. Nothing in this License Agreement shall be deemed to create any
-# relationship of agency, partnership, or joint venture between PSF and
-# Licensee.  This License Agreement does not grant permission to use PSF
-# trademarks or trade name in a trademark sense to endorse or promote
-# products or services of Licensee, or any third party.
-#
-# 8. By copying, installing or otherwise using Python, Licensee
-# agrees to be bound by the terms and conditions of this License
-# Agreement.
-#
-#
-# BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0
-# -------------------------------------------
-#
-# BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1
-#
-# 1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an
-# office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the
-# Individual or Organization ("Licensee") accessing and otherwise using
-# this software in source or binary form and its associated
-# documentation ("the Software").
-#
-# 2. Subject to the terms and conditions of this BeOpen Python License
-# Agreement, BeOpen hereby grants Licensee a non-exclusive,
-# royalty-free, world-wide license to reproduce, analyze, test, perform
-# and/or display publicly, prepare derivative works, distribute, and
-# otherwise use the Software alone or in any derivative version,
-# provided, however, that the BeOpen Python License is retained in the
-# Software, alone or in any derivative version prepared by Licensee.
-#
-# 3. BeOpen is making the Software available to Licensee on an "AS IS"
-# basis.  BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
-# IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND
-# DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
-# FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT
-# INFRINGE ANY THIRD PARTY RIGHTS.
-#
-# 4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE
-# SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS
-# AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY
-# DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
-#
-# 5. This License Agreement will automatically terminate upon a material
-# breach of its terms and conditions.
-#
-# 6. This License Agreement shall be governed by and interpreted in all
-# respects by the law of the State of California, excluding conflict of
-# law provisions.  Nothing in this License Agreement shall be deemed to
-# create any relationship of agency, partnership, or joint venture
-# between BeOpen and Licensee.  This License Agreement does not grant
-# permission to use BeOpen trademarks or trade names in a trademark
-# sense to endorse or promote products or services of Licensee, or any
-# third party.  As an exception, the "BeOpen Python" logos available at
-# http://www.pythonlabs.com/logos.html may be used according to the
-# permissions granted on that web page.
-#
-# 7. By copying, installing or otherwise using the software, Licensee
-# agrees to be bound by the terms and conditions of this License
-# Agreement.
-#
-#
-# CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1
-# ---------------------------------------
-#
-# 1. This LICENSE AGREEMENT is between the Corporation for National
-#     Research Initiatives, having an office at 1895 Preston White Drive,
-# Reston, VA 20191 ("CNRI"), and the Individual or Organization
-# ("Licensee") accessing and otherwise using Python 1.6.1 software in
-# source or binary form and its associated documentation.
-#
-# 2. Subject to the terms and conditions of this License Agreement, CNRI
-# hereby grants Licensee a nonexclusive, royalty-free, world-wide
-# license to reproduce, analyze, test, perform and/or display publicly,
-# prepare derivative works, distribute, and otherwise use Python 1.6.1
-# alone or in any derivative version, provided, however, that CNRI's
-# License Agreement and CNRI's notice of copyright, i.e., "Copyright (c)
-# 1995-2001 Corporation for National Research Initiatives; All Rights
-# Reserved" are retained in Python 1.6.1 alone or in any derivative
-# version prepared by Licensee.  Alternately, in lieu of CNRI's License
-# Agreement, Licensee may substitute the following text (omitting the
-# quotes): "Python 1.6.1 is made available subject to the terms and
-# conditions in CNRI's License Agreement.  This Agreement together with
-# Python 1.6.1 may be located on the Internet using the following
-# unique, persistent identifier (known as a handle): 1895.22/1013.  This
-# Agreement may also be obtained from a proxy server on the Internet
-# using the following URL: http://hdl.handle.net/1895.22/1013".
-#
-# 3. In the event Licensee prepares a derivative work that is based on
-# or incorporates Python 1.6.1 or any part thereof, and wants to make
-# the derivative work available to others as provided herein, then
-# Licensee hereby agrees to include in any such work a brief summary of
-# the changes made to Python 1.6.1.
-#
-# 4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS"
-# basis.  CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
-# IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND
-# DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
-# FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT
-# INFRINGE ANY THIRD PARTY RIGHTS.
-#
-# 5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
-# 1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
-# A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1,
-# OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
-#
-# 6. This License Agreement will automatically terminate upon a material
-# breach of its terms and conditions.
-#
-# 7. This License Agreement shall be governed by the federal
-# intellectual property law of the United States, including without
-# limitation the federal copyright law, and, to the extent such
-# U.S. federal law does not apply, by the law of the Commonwealth of
-# Virginia, excluding Virginia's conflict of law provisions.
-# Notwithstanding the foregoing, with regard to derivative works based
-# on Python 1.6.1 that incorporate non-separable material that was
-# previously distributed under the GNU General Public License (GPL), the
-# law of the Commonwealth of Virginia shall govern this License
-# Agreement only as to issues arising under or with respect to
-# Paragraphs 4, 5, and 7 of this License Agreement.  Nothing in this
-# License Agreement shall be deemed to create any relationship of
-# agency, partnership, or joint venture between CNRI and Licensee.  This
-# License Agreement does not grant permission to use CNRI trademarks or
-# trade name in a trademark sense to endorse or promote products or
-# services of Licensee, or any third party.
-#
-# 8. By clicking on the "ACCEPT" button where indicated, or by copying,
-# installing or otherwise using Python 1.6.1, Licensee agrees to be
-# bound by the terms and conditions of this License Agreement.
-#
-# ACCEPT
-#
-#
-# CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2
-# --------------------------------------------------
-#
-# Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam,
-# The Netherlands.  All rights reserved.
-#
-# Permission to use, copy, modify, and distribute this software and its
-# documentation for any purpose and without fee is hereby granted,
-# provided that the above copyright notice appear in all copies and that
-# both that copyright notice and this permission notice appear in
-# supporting documentation, and that the name of Stichting Mathematisch
-# Centrum or CWI not be used in advertising or publicity pertaining to
-# distribution of the software without specific, written prior
-# permission.
-#
-# STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO
-# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
-# FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE
-# FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
-# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
-# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
-# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
\ No newline at end of file
diff --git a/licenses-binary/LICENSE-vis-timeline.txt b/licenses-binary/LICENSE-vis-timeline.txt
index d343783c16791..324402de4dc09 100644
--- a/licenses-binary/LICENSE-vis-timeline.txt
+++ b/licenses-binary/LICENSE-vis-timeline.txt
@@ -1,23 +1,22 @@
-vis-timeline and vis-graph2d
-https://visjs.github.io/vis-timeline/
+vis.js
+https://github.com/almende/vis
 
-Create a fully customizable, interactive timeline with items and ranges.
+A dynamic, browser-based visualization library.
 
-@version 7.3.4
-@date    2020-03-18T17:03:58.105Z
-
-@copyright (c) 2011-2017 Almende B.V, http://almende.com
-@copyright (c) 2017-2019 visjs contributors, https://github.com/visjs
+@version 4.20.1-SNAPSHOT
+@date    2017-10-12
 
 @license
-vis.js is dual licensed under both
+Copyright (C) 2011-2017 Almende B.V, http://almende.com
+
+Vis.js is dual licensed under both
 
-  1. The Apache 2.0 License
-     http://www.apache.org/licenses/LICENSE-2.0
+* The Apache 2.0 License
+  http://www.apache.org/licenses/LICENSE-2.0
 
-  and
+and
 
-  2. The MIT License
-     http://opensource.org/licenses/MIT
+* The MIT License
+  http://opensource.org/licenses/MIT
 
-vis.js may be distributed under either license.
+Vis.js may be distributed under either license.
diff --git a/licenses/LICENSE-heapq.txt b/licenses/LICENSE-heapq.txt
deleted file mode 100644
index 45be6b83a53be..0000000000000
--- a/licenses/LICENSE-heapq.txt
+++ /dev/null
@@ -1,49 +0,0 @@
-PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
---------------------------------------------
-
-1. This LICENSE AGREEMENT is between the Python Software Foundation
-("PSF"), and the Individual or Organization ("Licensee") accessing and
-otherwise using this software ("Python") in source or binary form and
-its associated documentation.
-
-2. Subject to the terms and conditions of this License Agreement, PSF hereby
-grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
-analyze, test, perform and/or display publicly, prepare derivative works,
-distribute, and otherwise use Python alone or in any derivative version,
-provided, however, that PSF's License Agreement and PSF's notice of copyright,
-i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
-2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019 Python Software Foundation;
-All Rights Reserved" are retained in Python alone or in any derivative version
-prepared by Licensee.
-
-3. In the event Licensee prepares a derivative work that is based on
-or incorporates Python or any part thereof, and wants to make
-the derivative work available to others as provided herein, then
-Licensee hereby agrees to include in any such work a brief summary of
-the changes made to Python.
-
-4. PSF is making Python available to Licensee on an "AS IS"
-basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
-IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
-DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
-FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
-INFRINGE ANY THIRD PARTY RIGHTS.
-
-5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
-FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
-A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
-OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
-
-6. This License Agreement will automatically terminate upon a material
-breach of its terms and conditions.
-
-7. Nothing in this License Agreement shall be deemed to create any
-relationship of agency, partnership, or joint venture between PSF and
-Licensee.  This License Agreement does not grant permission to use PSF
-trademarks or trade name in a trademark sense to endorse or promote
-products or services of Licensee, or any third party.
-
-8. By copying, installing or otherwise using Python, Licensee
-agrees to be bound by the terms and conditions of this License
-Agreement.
-
diff --git a/licenses/LICENSE-vis-timeline.txt b/licenses/LICENSE-vis-timeline.txt
index d343783c16791..324402de4dc09 100644
--- a/licenses/LICENSE-vis-timeline.txt
+++ b/licenses/LICENSE-vis-timeline.txt
@@ -1,23 +1,22 @@
-vis-timeline and vis-graph2d
-https://visjs.github.io/vis-timeline/
+vis.js
+https://github.com/almende/vis
 
-Create a fully customizable, interactive timeline with items and ranges.
+A dynamic, browser-based visualization library.
 
-@version 7.3.4
-@date    2020-03-18T17:03:58.105Z
-
-@copyright (c) 2011-2017 Almende B.V, http://almende.com
-@copyright (c) 2017-2019 visjs contributors, https://github.com/visjs
+@version 4.20.1-SNAPSHOT
+@date    2017-10-12
 
 @license
-vis.js is dual licensed under both
+Copyright (C) 2011-2017 Almende B.V, http://almende.com
+
+Vis.js is dual licensed under both
 
-  1. The Apache 2.0 License
-     http://www.apache.org/licenses/LICENSE-2.0
+* The Apache 2.0 License
+  http://www.apache.org/licenses/LICENSE-2.0
 
-  and
+and
 
-  2. The MIT License
-     http://opensource.org/licenses/MIT
+* The MIT License
+  http://opensource.org/licenses/MIT
 
-vis.js may be distributed under either license.
+Vis.js may be distributed under either license.
diff --git a/mllib-local/benchmarks/BLASBenchmark-jdk11-results.txt b/mllib-local/benchmarks/BLASBenchmark-jdk11-results.txt
new file mode 100644
index 0000000000000..e68e80a6ed5b9
--- /dev/null
+++ b/mllib-local/benchmarks/BLASBenchmark-jdk11-results.txt
@@ -0,0 +1,252 @@
+================================================================================================
+daxpy
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+daxpy:                                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                 530            554          11        188.8           5.3       1.0X
+java                                                533            563          11        187.5           5.3       1.0X
+
+
+================================================================================================
+saxpy
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+saxpy:                                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                 295            309           5        338.8           3.0       1.0X
+java                                                290            300           6        345.2           2.9       1.0X
+
+
+================================================================================================
+ddot
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+ddot:                                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                 155            165           6        643.5           1.6       1.0X
+java                                                131            142           4        764.1           1.3       1.2X
+
+
+================================================================================================
+sdot
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+sdot:                                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                 121            131           4        827.9           1.2       1.0X
+java                                                 91             99           3       1101.7           0.9       1.3X
+
+
+================================================================================================
+dscal
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+dscal:                                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                 486            502           9        205.7           4.9       1.0X
+java                                                486            501           8        205.7           4.9       1.0X
+
+
+================================================================================================
+sscal
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+sscal:                                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                 257            265           5        389.8           2.6       1.0X
+java                                                253            267           4        395.1           2.5       1.0X
+
+
+================================================================================================
+dspmv[U]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+dspmv[U]:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                   1              1           0        884.8           1.1       1.0X
+java                                                  0              1           0       1064.7           0.9       1.2X
+
+
+================================================================================================
+dspr[U]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+dspr[U]:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                   1              1           0        533.3           1.9       1.0X
+java                                                  1              1           0        523.9           1.9       1.0X
+
+
+================================================================================================
+dsyr[U]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+dsyr[U]:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                   1              2           1        340.5           2.9       1.0X
+java                                                  1              2           1        344.2           2.9       1.0X
+
+
+================================================================================================
+dgemv[N]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+dgemv[N]:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                   1              1           0       1491.4           0.7       1.0X
+java                                                  0              1           0       2554.3           0.4       1.7X
+
+
+================================================================================================
+dgemv[T]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+dgemv[T]:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                   1              1           0       1189.3           0.8       1.0X
+java                                                  0              0           0       2770.9           0.4       2.3X
+
+
+================================================================================================
+sgemv[N]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+sgemv[N]:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                   1              1           0       1580.8           0.6       1.0X
+java                                                  0              1           0       2091.6           0.5       1.3X
+
+
+================================================================================================
+sgemv[T]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+sgemv[T]:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                   1              1           0       1190.6           0.8       1.0X
+java                                                  0              1           0       2551.7           0.4       2.1X
+
+
+================================================================================================
+dgemm[N,N]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+dgemm[N,N]:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                 914            956          25       1094.1           0.9       1.0X
+java                                                347            370          11       2881.8           0.3       2.6X
+
+
+================================================================================================
+dgemm[N,T]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+dgemm[N,T]:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                 927            968          22       1078.4           0.9       1.0X
+java                                                348            370          10       2872.6           0.3       2.7X
+
+
+================================================================================================
+dgemm[T,N]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+dgemm[T,N]:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                1085           1126          23        921.7           1.1       1.0X
+java                                                341            366          10       2930.8           0.3       3.2X
+
+
+================================================================================================
+dgemm[T,T]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+dgemm[T,T]:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                1535           1596          35        651.3           1.5       1.0X
+java                                                354            375          12       2821.1           0.4       4.3X
+
+
+================================================================================================
+sgemm[N,N]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+sgemm[N,N]:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                 835            875          21       1197.9           0.8       1.0X
+java                                                339            365          10       2948.7           0.3       2.5X
+
+
+================================================================================================
+sgemm[N,T]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+sgemm[N,T]:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                 840            874          17       1190.7           0.8       1.0X
+java                                                351            366          10       2852.8           0.4       2.4X
+
+
+================================================================================================
+sgemm[T,N]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+sgemm[T,N]:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                1032           1114          39        969.1           1.0       1.0X
+java                                                311            344          16       3210.9           0.3       3.3X
+
+
+================================================================================================
+sgemm[T,T]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+sgemm[T,T]:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                1316           1425          45        759.8           1.3       1.0X
+java                                                313            346          17       3194.0           0.3       4.2X
+
+
diff --git a/mllib-local/benchmarks/BLASBenchmark-results.txt b/mllib-local/benchmarks/BLASBenchmark-results.txt
new file mode 100644
index 0000000000000..6484c73b97243
--- /dev/null
+++ b/mllib-local/benchmarks/BLASBenchmark-results.txt
@@ -0,0 +1,252 @@
+================================================================================================
+daxpy
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+daxpy:                                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                 398            422          13        251.1           4.0       1.0X
+java                                                397            413          11        252.0           4.0       1.0X
+
+
+================================================================================================
+saxpy
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+saxpy:                                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                 217            230           7        460.7           2.2       1.0X
+java                                                214            226           8        466.7           2.1       1.0X
+
+
+================================================================================================
+ddot
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+ddot:                                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                 147            156           6        679.4           1.5       1.0X
+java                                                121            128           4        824.8           1.2       1.2X
+
+
+================================================================================================
+sdot
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+sdot:                                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                 118            126           5        849.2           1.2       1.0X
+java                                                 79             85           3       1263.7           0.8       1.5X
+
+
+================================================================================================
+dscal
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+dscal:                                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                 357            371           9        279.7           3.6       1.0X
+java                                                355            370           9        282.0           3.5       1.0X
+
+
+================================================================================================
+sscal
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+sscal:                                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                 184            195           6        543.4           1.8       1.0X
+java                                                183            196           6        547.4           1.8       1.0X
+
+
+================================================================================================
+dspmv[U]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+dspmv[U]:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                   1              1           0        604.2           1.7       1.0X
+java                                                  1              1           0        975.4           1.0       1.6X
+
+
+================================================================================================
+dspr[U]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+dspr[U]:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                   1              1           0        553.8           1.8       1.0X
+java                                                  1              1           0        556.5           1.8       1.0X
+
+
+================================================================================================
+dsyr[U]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+dsyr[U]:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                   1              2           0        370.5           2.7       1.0X
+java                                                  1              2           0        371.3           2.7       1.0X
+
+
+================================================================================================
+dgemv[N]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+dgemv[N]:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                   1              1           0       1194.5           0.8       1.0X
+java                                                  1              1           0       1729.5           0.6       1.4X
+
+
+================================================================================================
+dgemv[T]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+dgemv[T]:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                   1              1           0       1024.0           1.0       1.0X
+java                                                  1              1           0       1740.9           0.6       1.7X
+
+
+================================================================================================
+sgemv[N]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+sgemv[N]:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                   1              1           0       1164.4           0.9       1.0X
+java                                                  1              1           0       1863.9           0.5       1.6X
+
+
+================================================================================================
+sgemv[T]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+sgemv[T]:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                   1              1           0       1025.7           1.0       1.0X
+java                                                  1              1           0       1876.5           0.5       1.8X
+
+
+================================================================================================
+dgemm[N,N]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+dgemm[N,N]:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                 872            916          24       1146.7           0.9       1.0X
+java                                                415            435          14       2409.7           0.4       2.1X
+
+
+================================================================================================
+dgemm[N,T]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+dgemm[N,T]:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                 897            940          22       1115.0           0.9       1.0X
+java                                                424            455          15       2357.7           0.4       2.1X
+
+
+================================================================================================
+dgemm[T,N]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+dgemm[T,N]:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                1026           1086          29        974.7           1.0       1.0X
+java                                                415            437          14       2406.9           0.4       2.5X
+
+
+================================================================================================
+dgemm[T,T]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+dgemm[T,T]:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                1355           1408          33        737.9           1.4       1.0X
+java                                                415            436          13       2412.4           0.4       3.3X
+
+
+================================================================================================
+sgemm[N,N]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+sgemm[N,N]:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                 870            919          24       1149.8           0.9       1.0X
+java                                                379            403          15       2635.2           0.4       2.3X
+
+
+================================================================================================
+sgemm[N,T]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+sgemm[N,T]:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                 887            933          28       1127.3           0.9       1.0X
+java                                                393            440          18       2543.4           0.4       2.3X
+
+
+================================================================================================
+sgemm[T,N]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+sgemm[T,N]:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                1024           1088          33        976.7           1.0       1.0X
+java                                                402            421          12       2487.7           0.4       2.5X
+
+
+================================================================================================
+sgemm[T,T]
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+sgemm[T,T]:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+f2j                                                1304           1363          33        767.0           1.3       1.0X
+java                                                402            426          13       2486.8           0.4       3.2X
+
+
diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml
index 851af8d52a3ee..bc3b702f296c6 100644
--- a/mllib-local/pom.xml
+++ b/mllib-local/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
@@ -68,20 +68,19 @@
       <scope>test</scope>
     </dependency>
 
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-core_${scala.binary.version}</artifactId>
+      <version>${project.version}</version>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>dev.ludovic.netlib</groupId>
+      <artifactId>blas</artifactId>
+    </dependency>
   </dependencies>
-  <profiles>
-    <profile>
-      <id>netlib-lgpl</id>
-      <dependencies>
-        <dependency>
-          <groupId>com.github.fommil.netlib</groupId>
-          <artifactId>all</artifactId>
-          <version>${netlib.java.version}</version>
-          <type>pom</type>
-        </dependency>
-      </dependencies>
-    </profile>
-  </profiles>
   <build>
     <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
     <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/BLAS.scala b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/BLAS.scala
index 368f177cda828..5a6bee3e74ead 100644
--- a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/BLAS.scala
+++ b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/BLAS.scala
@@ -17,29 +17,39 @@
 
 package org.apache.spark.ml.linalg
 
-import com.github.fommil.netlib.{BLAS => NetlibBLAS, F2jBLAS}
-import com.github.fommil.netlib.BLAS.{getInstance => NativeBLAS}
+import dev.ludovic.netlib.{BLAS => NetlibBLAS,
+                           JavaBLAS => NetlibJavaBLAS,
+                           NativeBLAS => NetlibNativeBLAS}
 
 /**
  * BLAS routines for MLlib's vectors and matrices.
  */
 private[spark] object BLAS extends Serializable {
 
-  @transient private var _f2jBLAS: NetlibBLAS = _
+  @transient private var _javaBLAS: NetlibBLAS = _
   @transient private var _nativeBLAS: NetlibBLAS = _
   private val nativeL1Threshold: Int = 256
 
-  // For level-1 function dspmv, use f2jBLAS for better performance.
-  private[ml] def f2jBLAS: NetlibBLAS = {
-    if (_f2jBLAS == null) {
-      _f2jBLAS = new F2jBLAS
+  // For level-1 function dspmv, use javaBLAS for better performance.
+  private[spark] def javaBLAS: NetlibBLAS = {
+    if (_javaBLAS == null) {
+      _javaBLAS = NetlibJavaBLAS.getInstance
     }
-    _f2jBLAS
+    _javaBLAS
   }
 
-  private[ml] def getBLAS(vectorSize: Int): NetlibBLAS = {
+  // For level-3 routines, we use the native BLAS.
+  private[spark] def nativeBLAS: NetlibBLAS = {
+    if (_nativeBLAS == null) {
+      _nativeBLAS =
+        try { NetlibNativeBLAS.getInstance } catch { case _: Throwable => javaBLAS }
+    }
+    _nativeBLAS
+  }
+
+  private[spark] def getBLAS(vectorSize: Int): NetlibBLAS = {
     if (vectorSize < nativeL1Threshold) {
-      f2jBLAS
+      javaBLAS
     } else {
       nativeBLAS
     }
@@ -235,14 +245,6 @@ private[spark] object BLAS extends Serializable {
     }
   }
 
-  // For level-3 routines, we use the native BLAS.
-  private[ml] def nativeBLAS: NetlibBLAS = {
-    if (_nativeBLAS == null) {
-      _nativeBLAS = NativeBLAS
-    }
-    _nativeBLAS
-  }
-
   /**
    * Adds alpha * x * x.t to a matrix in-place. This is the same as BLAS's ?SPR.
    *
@@ -267,7 +269,7 @@ private[spark] object BLAS extends Serializable {
       x: DenseVector,
       beta: Double,
       y: DenseVector): Unit = {
-    f2jBLAS.dspmv("U", n, alpha, A.values, x.values, 1, beta, y.values, 1)
+    javaBLAS.dspmv("U", n, alpha, A.values, x.values, 1, beta, y.values, 1)
   }
 
   /**
@@ -279,7 +281,7 @@ private[spark] object BLAS extends Serializable {
     val n = v.size
     v match {
       case DenseVector(values) =>
-        NativeBLAS.dspr("U", n, alpha, values, 1, U)
+        nativeBLAS.dspr("U", n, alpha, values, 1, U)
       case SparseVector(size, indices, values) =>
         val nnz = indices.length
         var colStartIdx = 0
@@ -302,6 +304,8 @@ private[spark] object BLAS extends Serializable {
           j += 1
           prevCol = col
         }
+      case _ =>
+        throw new IllegalArgumentException(s"spr doesn't support vector type ${v.getClass}.")
     }
   }
 
diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Matrices.scala b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Matrices.scala
index 1254ed747aeb4..3272a50508a3d 100644
--- a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Matrices.scala
+++ b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Matrices.scala
@@ -22,7 +22,6 @@ import java.util.{Arrays, Random}
 import scala.collection.mutable.{ArrayBuffer, ArrayBuilder => MArrayBuilder, HashSet => MHashSet}
 
 import breeze.linalg.{CSCMatrix => BSM, DenseMatrix => BDM, Matrix => BM}
-import com.github.fommil.netlib.BLAS.{getInstance => blas}
 
 import org.apache.spark.annotation.Since
 
@@ -457,7 +456,7 @@ class DenseMatrix @Since("2.0.0") (
     if (isTransposed) {
       Iterator.tabulate(numCols) { j =>
         val col = new Array[Double](numRows)
-        blas.dcopy(numRows, values, j, numCols, col, 0, 1)
+        BLAS.nativeBLAS.dcopy(numRows, values, j, numCols, col, 0, 1)
         new DenseVector(col)
       }
     } else {
diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala
index 83973bcffef05..cf5c4ff514d32 100644
--- a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala
+++ b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala
@@ -22,6 +22,7 @@ import java.util
 
 import scala.annotation.varargs
 import scala.collection.JavaConverters._
+import scala.collection.mutable
 
 import breeze.linalg.{DenseVector => BDV, SparseVector => BSV, Vector => BV}
 
@@ -753,24 +754,51 @@ class SparseVector @Since("2.0.0") (
    * Create a slice of this vector based on the given indices.
    * @param selectedIndices Unsorted list of indices into the vector.
    *                        This does NOT do bound checking.
+   * @param sorted Whether the input indices are already sorted.
+   *               This does NOT do ordering checking.
    * @return  New SparseVector with values in the order specified by the given indices.
    *
    * NOTE: The API needs to be discussed before making this public.
-   *       Also, if we have a version assuming indices are sorted, we should optimize it.
-   */
-  private[spark] def slice(selectedIndices: Array[Int]): SparseVector = {
-    var currentIdx = 0
-    val (sliceInds, sliceVals) = selectedIndices.flatMap { origIdx =>
-      val iIdx = java.util.Arrays.binarySearch(this.indices, origIdx)
-      val i_v = if (iIdx >= 0) {
-        Iterator((currentIdx, this.values(iIdx)))
-      } else {
-        Iterator()
+   */
+  private[spark] def slice(selectedIndices: Array[Int], sorted: Boolean = false): SparseVector = {
+    val localIndices = indices
+    val localValues = values
+    val ns = selectedIndices.length
+    val indexBuff = mutable.ArrayBuilder.make[Int]
+    val valueBuff = mutable.ArrayBuilder.make[Double]
+
+    if (sorted) {
+      val nk = localIndices.length
+      var k = 0
+      var s = 0
+      while (k < nk && s < ns) {
+        val i = localIndices(k)
+        val v = localValues(k)
+        if (v != 0) {
+          while (s < ns && selectedIndices(s) < i) { s += 1 }
+          if (s < ns && selectedIndices(s) == i) {
+            indexBuff += s
+            valueBuff += v
+            s += 1
+          }
+        }
+        k += 1
       }
-      currentIdx += 1
-      i_v
-    }.unzip
-    new SparseVector(selectedIndices.length, sliceInds.toArray, sliceVals.toArray)
+    } else {
+      var s = 0
+      while (s < ns) {
+        val j = java.util.Arrays.binarySearch(localIndices, selectedIndices(s))
+        if (j >= 0) {
+          val v = localValues(j)
+          if (v != 0) {
+            indexBuff += s
+            valueBuff += v
+          }
+        }
+        s += 1
+      }
+    }
+    new SparseVector(ns, indexBuff.result, valueBuff.result)
   }
 
   private[spark] override def iterator: Iterator[(Int, Double)] = {
@@ -786,7 +814,7 @@ class SparseVector @Since("2.0.0") (
 
       override def hasNext: Boolean = i < localSize
 
-      override def next: (Int, Double) = {
+      override def next(): (Int, Double) = {
         val v = if (i == k) {
           j += 1
           k = if (j < localNumActives) localIndices(j) else -1
diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/stat/distribution/MultivariateGaussian.scala b/mllib-local/src/main/scala/org/apache/spark/ml/stat/distribution/MultivariateGaussian.scala
index a08b8af0fcbfd..42746b5727029 100644
--- a/mllib-local/src/main/scala/org/apache/spark/ml/stat/distribution/MultivariateGaussian.scala
+++ b/mllib-local/src/main/scala/org/apache/spark/ml/stat/distribution/MultivariateGaussian.scala
@@ -55,7 +55,7 @@ class MultivariateGaussian @Since("2.0.0") (
    */
   @transient private lazy val tuple = {
     val (rootSigmaInv, u) = calculateCovarianceConstants
-    val rootSigmaInvMat = Matrices.fromBreeze(rootSigmaInv).toDense
+    val rootSigmaInvMat = Matrices.fromBreeze(rootSigmaInv)
     val rootSigmaInvMulMu = rootSigmaInvMat.multiply(mean)
     (rootSigmaInvMat, u, rootSigmaInvMulMu)
   }
@@ -81,36 +81,6 @@ class MultivariateGaussian @Since("2.0.0") (
     u - 0.5 * BLAS.dot(v, v)
   }
 
-  private[ml] def pdf(X: Matrix): DenseVector = {
-    val mat = DenseMatrix.zeros(X.numRows, X.numCols)
-    pdf(X, mat)
-  }
-
-  private[ml] def pdf(X: Matrix, mat: DenseMatrix): DenseVector = {
-    require(!mat.isTransposed)
-
-    BLAS.gemm(1.0, X, rootSigmaInvMat.transpose, 0.0, mat)
-    val m = mat.numRows
-    val n = mat.numCols
-
-    val pdfVec = mat.multiply(rootSigmaInvMulMu)
-
-    val blas = BLAS.getBLAS(n)
-    val squared1 = blas.ddot(n, rootSigmaInvMulMu.values, 1, rootSigmaInvMulMu.values, 1)
-
-    val localU = u
-    var i = 0
-    while (i < m) {
-      val squared2 = blas.ddot(n, mat.values, i, m, mat.values, i, m)
-      val dot = pdfVec(i)
-      val squaredSum = squared1 + squared2 - dot - dot
-      pdfVec.values(i) = math.exp(localU - 0.5 * squaredSum)
-      i += 1
-    }
-
-    pdfVec
-  }
-
   /**
    * Calculate distribution dependent components used for the density function:
    *    pdf(x) = (2*pi)^(-k/2)^ * det(sigma)^(-1/2)^ * exp((-1/2) * (x-mu).t * inv(sigma) * (x-mu))
diff --git a/mllib-local/src/test/scala/org/apache/spark/ml/SparkMLFunSuite.scala b/mllib-local/src/test/scala/org/apache/spark/ml/SparkMLFunSuite.scala
index cb3b56bba87b0..d5ba1f906d389 100644
--- a/mllib-local/src/test/scala/org/apache/spark/ml/SparkMLFunSuite.scala
+++ b/mllib-local/src/test/scala/org/apache/spark/ml/SparkMLFunSuite.scala
@@ -18,13 +18,14 @@
 package org.apache.spark.ml
 
 // scalastyle:off
-import org.scalatest.{BeforeAndAfterAll, FunSuite}
+import org.scalatest.BeforeAndAfterAll
+import org.scalatest.funsuite.AnyFunSuite
 
 /**
  * Base abstract class for all unit tests in Spark for handling common functionality.
  */
 private[spark] abstract class SparkMLFunSuite
-  extends FunSuite
+  extends AnyFunSuite
   with BeforeAndAfterAll {
   // scalastyle:on
 }
diff --git a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/BLASBenchmark.scala b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/BLASBenchmark.scala
new file mode 100644
index 0000000000000..86a576826b715
--- /dev/null
+++ b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/BLASBenchmark.scala
@@ -0,0 +1,430 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.linalg
+
+import dev.ludovic.netlib.{BLAS => NetlibBLAS}
+import dev.ludovic.netlib.blas.F2jBLAS
+import scala.concurrent.duration._
+
+import org.apache.spark.benchmark.{Benchmark, BenchmarkBase}
+
+/**
+ * Serialization benchmark for BLAS.
+ * To run this benchmark:
+ * {{{
+ * 1. without sbt: bin/spark-submit --class <this class> <spark mllib test jar>
+ * 2. build/sbt "mllib-local/test:runMain <this class>"
+ * 3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "mllib/test:runMain <this class>"
+ *    Results will be written to "benchmarks/BLASBenchmark-results.txt".
+ * }}}
+ */
+object BLASBenchmark extends BenchmarkBase {
+
+  override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
+
+    val iters = 1e2.toInt
+    val rnd = new scala.util.Random(0)
+
+    val f2jBLAS = F2jBLAS.getInstance
+    val javaBLAS = BLAS.javaBLAS
+    val nativeBLAS = BLAS.nativeBLAS
+
+    // scalastyle:off println
+    println("f2jBLAS    = " + f2jBLAS.getClass.getName)
+    println("javaBLAS   = " + javaBLAS.getClass.getName)
+    println("nativeBLAS = " + nativeBLAS.getClass.getName)
+    // scalastyle:on println
+
+    def runBLASBenchmark(name: String, n: Int)(bench: NetlibBLAS => Unit) {
+      val benchmark = new Benchmark(name, n, iters,
+                                    warmupTime = 30.seconds,
+                                    minTime = 30.seconds,
+                                    output = output)
+
+      benchmark.addCase("f2j") { _ =>
+        bench(f2jBLAS)
+      }
+
+      benchmark.addCase("java") { _ =>
+        bench(javaBLAS)
+      }
+
+      if (nativeBLAS != javaBLAS) {
+        benchmark.addCase("native") { _ =>
+          bench(nativeBLAS)
+        }
+      }
+
+      benchmark.run()
+    }
+
+    runBenchmark("daxpy") {
+      val n = 1e8.toInt
+      val alpha = rnd.nextDouble
+      val x = Array.fill(n) { rnd.nextDouble }
+      val y = Array.fill(n) { rnd.nextDouble }
+
+      runBLASBenchmark("daxpy", n) { impl =>
+        impl.daxpy(n, alpha, x, 1, y.clone, 1)
+      }
+    }
+
+    runBenchmark("saxpy") {
+      val n = 1e8.toInt
+      val alpha = rnd.nextFloat
+      val x = Array.fill(n) { rnd.nextFloat }
+      val y = Array.fill(n) { rnd.nextFloat }
+
+      runBLASBenchmark("saxpy", n) { impl =>
+        impl.saxpy(n, alpha, x, 1, y.clone, 1)
+      }
+    }
+
+    runBenchmark("dcopy") {
+      val n = 1e8.toInt
+      val x = Array.fill(n) { rnd.nextDouble }
+      val y = Array.fill(n) { 0.0 }
+
+      runBLASBenchmark("dcopy", n) { impl =>
+        impl.dcopy(n, x, 1, y.clone, 1)
+      }
+    }
+
+    runBenchmark("scopy") {
+      val n = 1e8.toInt
+      val x = Array.fill(n) { rnd.nextFloat }
+      val y = Array.fill(n) { 0.0f }
+
+      runBLASBenchmark("scopy", n) { impl =>
+        impl.scopy(n, x, 1, y.clone, 1)
+      }
+    }
+
+    runBenchmark("ddot") {
+      val n = 1e8.toInt
+      val x = Array.fill(n) { rnd.nextDouble }
+      val y = Array.fill(n) { rnd.nextDouble }
+
+      runBLASBenchmark("ddot", n) { impl =>
+        impl.ddot(n, x, 1, y, 1)
+      }
+    }
+
+    runBenchmark("sdot") {
+      val n = 1e8.toInt
+      val x = Array.fill(n) { rnd.nextFloat }
+      val y = Array.fill(n) { rnd.nextFloat }
+
+      runBLASBenchmark("sdot", n) { impl =>
+        impl.sdot(n, x, 1, y, 1)
+      }
+    }
+
+    runBenchmark("dnrm2") {
+      val n = 1e8.toInt
+      val x = Array.fill(n) { rnd.nextDouble }
+
+      runBLASBenchmark("dnrm2", n) { impl =>
+        impl.dnrm2(n, x, 1)
+      }
+    }
+
+    runBenchmark("snrm2") {
+      val n = 1e8.toInt
+      val x = Array.fill(n) { rnd.nextFloat }
+
+      runBLASBenchmark("snrm2", n) { impl =>
+        impl.snrm2(n, x, 1)
+      }
+    }
+
+    runBenchmark("dscal") {
+      val n = 1e8.toInt
+      val alpha = rnd.nextDouble
+      val x = Array.fill(n) { rnd.nextDouble }
+
+      runBLASBenchmark("dscal", n) { impl =>
+        impl.dscal(n, alpha, x.clone, 1)
+      }
+    }
+
+    runBenchmark("sscal") {
+      val n = 1e8.toInt
+      val alpha = rnd.nextFloat
+      val x = Array.fill(n) { rnd.nextFloat }
+
+      runBLASBenchmark("sscal", n) { impl =>
+        impl.sscal(n, alpha, x.clone, 1)
+      }
+    }
+
+    runBenchmark("dgemv[N]") {
+      val m = 1e4.toInt
+      val n = 1e4.toInt
+      val alpha = rnd.nextDouble
+      val a = Array.fill(m * n) { rnd.nextDouble }
+      val lda = m
+      val x = Array.fill(n) { rnd.nextDouble }
+      val beta = rnd.nextDouble
+      val y = Array.fill(m) { rnd.nextDouble }
+
+      runBLASBenchmark("dgemv[N]", m * n) { impl =>
+        impl.dgemv("N", m, n, alpha, a, lda, x, 1, beta, y.clone, 1)
+      }
+    }
+
+    runBenchmark("dgemv[T]") {
+      val m = 1e4.toInt
+      val n = 1e4.toInt
+      val alpha = rnd.nextDouble
+      val a = Array.fill(m * n) { rnd.nextDouble }
+      val lda = m
+      val x = Array.fill(m) { rnd.nextDouble }
+      val beta = rnd.nextDouble
+      val y = Array.fill(n) { rnd.nextDouble }
+
+      runBLASBenchmark("dgemv[T]", m * n) { impl =>
+        impl.dgemv("T", m, n, alpha, a, lda, x, 1, beta, y.clone, 1)
+      }
+    }
+
+    runBenchmark("sgemv[N]") {
+      val m = 1e4.toInt
+      val n = 1e4.toInt
+      val alpha = rnd.nextFloat
+      val a = Array.fill(m * n) { rnd.nextFloat }
+      val lda = m
+      val x = Array.fill(n) { rnd.nextFloat }
+      val beta = rnd.nextFloat
+      val y = Array.fill(m) { rnd.nextFloat }
+
+      runBLASBenchmark("sgemv[N]", m * n) { impl =>
+        impl.sgemv("N", m, n, alpha, a, lda, x, 1, beta, y.clone, 1)
+      }
+    }
+
+    runBenchmark("sgemv[T]") {
+      val m = 1e4.toInt
+      val n = 1e4.toInt
+      val alpha = rnd.nextFloat
+      val a = Array.fill(m * n) { rnd.nextFloat }
+      val lda = m
+      val x = Array.fill(m) { rnd.nextFloat }
+      val beta = rnd.nextFloat
+      val y = Array.fill(n) { rnd.nextFloat }
+
+      runBLASBenchmark("sgemv[T]", m * n) { impl =>
+        impl.sgemv("T", m, n, alpha, a, lda, x, 1, beta, y.clone, 1)
+      }
+    }
+
+    runBenchmark("dger") {
+      val m = 1e4.toInt
+      val n = 1e4.toInt
+      val alpha = rnd.nextDouble
+      val a = Array.fill(m * n) { rnd.nextDouble }
+      val lda = m
+      val x = Array.fill(n) { rnd.nextDouble }
+      val beta = rnd.nextDouble
+      val y = Array.fill(m) { rnd.nextDouble }
+
+      runBLASBenchmark("dger", m * n) { impl =>
+        impl.dger(m, n, alpha, x, 1, y, 1, a.clone(), m)
+      }
+    }
+
+    runBenchmark("dspmv[U]") {
+      val n = 1e4.toInt
+      val alpha = rnd.nextDouble
+      val a = Array.fill(n * (n + 1) / 2) { rnd.nextDouble }
+      val x = Array.fill(n) { rnd.nextDouble }
+      val beta = rnd.nextDouble
+      val y = Array.fill(n) { rnd.nextDouble }
+
+      runBLASBenchmark("dspmv[U]", n * (n + 1) / 2) { impl =>
+        impl.dspmv("U", n, alpha, a, x, 1, beta, y.clone, 1)
+      }
+    }
+
+    runBenchmark("dspr[U]") {
+      val n = 1e4.toInt
+      val alpha = rnd.nextDouble
+      val x = Array.fill(n) { rnd.nextDouble }
+      val a = Array.fill(n * (n + 1) / 2) { rnd.nextDouble }
+
+      runBLASBenchmark("dspr[U]", n * (n + 1) / 2) { impl =>
+        impl.dspr("U", n, alpha, x, 1, a.clone)
+      }
+    }
+
+    runBenchmark("dsyr[U]") {
+      val n = 1e4.toInt
+      val alpha = rnd.nextDouble
+      val x = Array.fill(n) { rnd.nextDouble }
+      val a = Array.fill(n * n) { rnd.nextDouble }
+
+      runBLASBenchmark("dsyr[U]", n * (n + 1) / 2) { impl =>
+        impl.dsyr("U", n, alpha, x, 1, a.clone, n)
+      }
+    }
+
+    runBenchmark("dgemm[N,N]") {
+      val m = 1e3.toInt
+      val n = 1e3.toInt
+      val k = 1e3.toInt
+      val alpha = rnd.nextDouble
+      val a = Array.fill(m * k) { rnd.nextDouble }
+      val lda = m
+      val b = Array.fill(k * n) { rnd.nextDouble }
+      val ldb = k
+      val beta = rnd.nextDouble
+      val c = Array.fill(m * n) { rnd.nextDouble }
+      var ldc = m
+
+      runBLASBenchmark("dgemm[N,N]", m * n * k) { impl =>
+        impl.dgemm("N", "N", m, n, k, alpha, a, lda, b, ldb, beta, c.clone, ldc)
+      }
+    }
+
+    runBenchmark("dgemm[N,T]") {
+      val m = 1e3.toInt
+      val n = 1e3.toInt
+      val k = 1e3.toInt
+      val alpha = rnd.nextDouble
+      val a = Array.fill(m * k) { rnd.nextDouble }
+      val lda = m
+      val b = Array.fill(k * n) { rnd.nextDouble }
+      val ldb = n
+      val beta = rnd.nextDouble
+      val c = Array.fill(m * n) { rnd.nextDouble }
+      var ldc = m
+
+      runBLASBenchmark("dgemm[N,T]", m * n * k) { impl =>
+        impl.dgemm("N", "T", m, n, k, alpha, a, lda, b, ldb, beta, c.clone, ldc)
+      }
+    }
+
+    runBenchmark("dgemm[T,N]") {
+      val m = 1e3.toInt
+      val n = 1e3.toInt
+      val k = 1e3.toInt
+      val alpha = rnd.nextDouble
+      val a = Array.fill(m * k) { rnd.nextDouble }
+      val lda = k
+      val b = Array.fill(k * n) { rnd.nextDouble }
+      val ldb = k
+      val beta = rnd.nextDouble
+      val c = Array.fill(m * n) { rnd.nextDouble }
+      var ldc = m
+
+      runBLASBenchmark("dgemm[T,N]", m * n * k) { impl =>
+        impl.dgemm("T", "N", m, n, k, alpha, a, lda, b, ldb, beta, c.clone, ldc)
+      }
+    }
+
+    runBenchmark("dgemm[T,T]") {
+      val m = 1e3.toInt
+      val n = 1e3.toInt
+      val k = 1e3.toInt
+      val alpha = rnd.nextDouble
+      val a = Array.fill(m * k) { rnd.nextDouble }
+      val lda = k
+      val b = Array.fill(k * n) { rnd.nextDouble }
+      val ldb = n
+      val beta = rnd.nextDouble
+      val c = Array.fill(m * n) { rnd.nextDouble }
+      var ldc = m
+
+      runBLASBenchmark("dgemm[T,T]", m * n * k) { impl =>
+        impl.dgemm("T", "T", m, n, k, alpha, a, lda, b, ldb, beta, c.clone, ldc)
+      }
+    }
+
+    runBenchmark("sgemm[N,N]") {
+      val m = 1e3.toInt
+      val n = 1e3.toInt
+      val k = 1e3.toInt
+      val alpha = rnd.nextFloat
+      val a = Array.fill(m * k) { rnd.nextFloat }
+      val lda = m
+      val b = Array.fill(k * n) { rnd.nextFloat }
+      val ldb = k
+      val beta = rnd.nextFloat
+      val c = Array.fill(m * n) { rnd.nextFloat }
+      var ldc = m
+
+      runBLASBenchmark("sgemm[N,N]", m * n * k) { impl =>
+        impl.sgemm("N", "N", m, n, k, alpha, a, lda, b, ldb, beta, c.clone, ldc)
+      }
+    }
+
+    runBenchmark("sgemm[N,T]") {
+      val m = 1e3.toInt
+      val n = 1e3.toInt
+      val k = 1e3.toInt
+      val alpha = rnd.nextFloat
+      val a = Array.fill(m * k) { rnd.nextFloat }
+      val lda = m
+      val b = Array.fill(k * n) { rnd.nextFloat }
+      val ldb = n
+      val beta = rnd.nextFloat
+      val c = Array.fill(m * n) { rnd.nextFloat }
+      var ldc = m
+
+      runBLASBenchmark("sgemm[N,T]", m * n * k) { impl =>
+        impl.sgemm("N", "T", m, n, k, alpha, a, lda, b, ldb, beta, c.clone, ldc)
+      }
+    }
+
+    runBenchmark("sgemm[T,N]") {
+      val m = 1e3.toInt
+      val n = 1e3.toInt
+      val k = 1e3.toInt
+      val alpha = rnd.nextFloat
+      val a = Array.fill(m * k) { rnd.nextFloat }
+      val lda = k
+      val b = Array.fill(k * n) { rnd.nextFloat }
+      val ldb = k
+      val beta = rnd.nextFloat
+      val c = Array.fill(m * n) { rnd.nextFloat }
+      var ldc = m
+
+      runBLASBenchmark("sgemm[T,N]", m * n * k) { impl =>
+        impl.sgemm("T", "N", m, n, k, alpha, a, lda, b, ldb, beta, c.clone, ldc)
+      }
+    }
+
+    runBenchmark("sgemm[T,T]") {
+      val m = 1e3.toInt
+      val n = 1e3.toInt
+      val k = 1e3.toInt
+      val alpha = rnd.nextFloat
+      val a = Array.fill(m * k) { rnd.nextFloat }
+      val lda = k
+      val b = Array.fill(k * n) { rnd.nextFloat }
+      val ldb = n
+      val beta = rnd.nextFloat
+      val c = Array.fill(m * n) { rnd.nextFloat }
+      var ldc = m
+
+      runBLASBenchmark("sgemm[T,T]", m * n * k) { impl =>
+        impl.sgemm("T", "T", m, n, k, alpha, a, lda, b, ldb, beta, c.clone, ldc)
+      }
+    }
+  }
+}
diff --git a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/BLASSuite.scala b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/BLASSuite.scala
index 781f3da313d82..ce177666f34e6 100644
--- a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/BLASSuite.scala
+++ b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/BLASSuite.scala
@@ -24,7 +24,7 @@ import org.apache.spark.ml.util.TestingUtils._
 class BLASSuite extends SparkMLFunSuite {
 
   test("nativeL1Threshold") {
-    assert(getBLAS(128) == BLAS.f2jBLAS)
+    assert(getBLAS(128) == BLAS.javaBLAS)
     assert(getBLAS(256) == BLAS.nativeBLAS)
     assert(getBLAS(512) == BLAS.nativeBLAS)
   }
diff --git a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala
index b9471f51038ac..f08669781e917 100644
--- a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala
+++ b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala
@@ -383,6 +383,13 @@ class VectorsSuite extends SparkMLFunSuite {
     assert(v.slice(Array(2, 0, 3, 4)) === new SparseVector(4, Array(0, 3), Array(2.2, 4.4)))
   }
 
+  test("SparseVector.slice with sorted indices") {
+    val v = new SparseVector(5, Array(1, 2, 4), Array(1.1, 2.2, 4.4))
+    assert(v.slice(Array(0, 2), true) === v.slice(Array(0, 2), false))
+    assert(v.slice(Array(0, 2, 4), true) === v.slice(Array(0, 2, 4), false))
+    assert(v.slice(Array(1, 3), true) === v.slice(Array(1, 3), false))
+  }
+
   test("sparse vector only support non-negative length") {
     val v1 = Vectors.sparse(0, Array.emptyIntArray, Array.emptyDoubleArray)
     val v2 = Vectors.sparse(0, Array.empty[(Int, Double)])
diff --git a/mllib-local/src/test/scala/org/apache/spark/ml/stat/distribution/MultivariateGaussianSuite.scala b/mllib-local/src/test/scala/org/apache/spark/ml/stat/distribution/MultivariateGaussianSuite.scala
index 8652d317a85c4..f2ecff1cc58bd 100644
--- a/mllib-local/src/test/scala/org/apache/spark/ml/stat/distribution/MultivariateGaussianSuite.scala
+++ b/mllib-local/src/test/scala/org/apache/spark/ml/stat/distribution/MultivariateGaussianSuite.scala
@@ -27,7 +27,6 @@ class MultivariateGaussianSuite extends SparkMLFunSuite {
   test("univariate") {
     val x1 = Vectors.dense(0.0)
     val x2 = Vectors.dense(1.5)
-    val mat = Matrices.fromVectors(Seq(x1, x2))
 
     val mu = Vectors.dense(0.0)
     val sigma1 = Matrices.dense(1, 1, Array(1.0))
@@ -36,7 +35,6 @@ class MultivariateGaussianSuite extends SparkMLFunSuite {
     assert(dist1.logpdf(x2) ~== -2.0439385332046727 absTol 1E-5)
     assert(dist1.pdf(x1) ~== 0.39894 absTol 1E-5)
     assert(dist1.pdf(x2) ~== 0.12952 absTol 1E-5)
-    assert(dist1.pdf(mat) ~== Vectors.dense(0.39894, 0.12952) absTol 1E-5)
 
     val sigma2 = Matrices.dense(1, 1, Array(4.0))
     val dist2 = new MultivariateGaussian(mu, sigma2)
@@ -44,13 +42,11 @@ class MultivariateGaussianSuite extends SparkMLFunSuite {
     assert(dist2.logpdf(x2) ~== -1.893335713764618 absTol 1E-5)
     assert(dist2.pdf(x1) ~== 0.19947 absTol 1E-5)
     assert(dist2.pdf(x2) ~== 0.15057 absTol 1E-5)
-    assert(dist2.pdf(mat) ~== Vectors.dense(0.19947, 0.15057) absTol 1E-5)
   }
 
   test("multivariate") {
     val x1 = Vectors.dense(0.0, 0.0)
     val x2 = Vectors.dense(1.0, 1.0)
-    val mat = Matrices.fromVectors(Seq(x1, x2))
 
     val mu = Vectors.dense(0.0, 0.0)
     val sigma1 = Matrices.dense(2, 2, Array(1.0, 0.0, 0.0, 1.0))
@@ -59,7 +55,6 @@ class MultivariateGaussianSuite extends SparkMLFunSuite {
     assert(dist1.logpdf(x2) ~== -2.8378770664093453 absTol 1E-5)
     assert(dist1.pdf(x1) ~== 0.15915 absTol 1E-5)
     assert(dist1.pdf(x2) ~== 0.05855 absTol 1E-5)
-    assert(dist1.pdf(mat) ~== Vectors.dense(0.15915, 0.05855) absTol 1E-5)
 
     val sigma2 = Matrices.dense(2, 2, Array(4.0, -1.0, -1.0, 2.0))
     val dist2 = new MultivariateGaussian(mu, sigma2)
@@ -67,25 +62,21 @@ class MultivariateGaussianSuite extends SparkMLFunSuite {
     assert(dist2.logpdf(x2) ~== -3.3822607123655732 absTol 1E-5)
     assert(dist2.pdf(x1) ~== 0.060155 absTol 1E-5)
     assert(dist2.pdf(x2) ~== 0.033971 absTol 1E-5)
-    assert(dist2.pdf(mat) ~== Vectors.dense(0.060155, 0.033971) absTol 1E-5)
   }
 
   test("multivariate degenerate") {
     val x1 = Vectors.dense(0.0, 0.0)
     val x2 = Vectors.dense(1.0, 1.0)
-    val mat = Matrices.fromVectors(Seq(x1, x2))
 
     val mu = Vectors.dense(0.0, 0.0)
     val sigma = Matrices.dense(2, 2, Array(1.0, 1.0, 1.0, 1.0))
     val dist = new MultivariateGaussian(mu, sigma)
     assert(dist.pdf(x1) ~== 0.11254 absTol 1E-5)
     assert(dist.pdf(x2) ~== 0.068259 absTol 1E-5)
-    assert(dist.pdf(mat) ~== Vectors.dense(0.11254, 0.068259) absTol 1E-5)
   }
 
   test("SPARK-11302") {
     val x = Vectors.dense(629, 640, 1.7188, 618.19)
-    val mat = Matrices.fromVectors(Seq(x))
     val mu = Vectors.dense(
       1055.3910505836575, 1070.489299610895, 1.39020554474708, 1040.5907503867697)
     val sigma = Matrices.dense(4, 4, Array(
@@ -96,6 +87,5 @@ class MultivariateGaussianSuite extends SparkMLFunSuite {
     val dist = new MultivariateGaussian(mu, sigma)
     // Agrees with R's dmvnorm: 7.154782e-05
     assert(dist.pdf(x) ~== 7.154782224045512E-5 absTol 1E-9)
-    assert(dist.pdf(mat) ~== Vectors.dense(7.154782224045512E-5) absTol 1E-5)
   }
 }
diff --git a/mllib/benchmarks/UDTSerializationBenchmark-jdk11-results.txt b/mllib/benchmarks/UDTSerializationBenchmark-jdk11-results.txt
index 6f671405b8343..8354132d7e5c7 100644
--- a/mllib/benchmarks/UDTSerializationBenchmark-jdk11-results.txt
+++ b/mllib/benchmarks/UDTSerializationBenchmark-jdk11-results.txt
@@ -2,11 +2,11 @@
 VectorUDT de/serialization
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 VectorUDT de/serialization:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-serialize                                           269            292          13          0.0      269441.1       1.0X
-deserialize                                         164            191           9          0.0      164314.6       1.6X
+serialize                                           482            507          11          0.0      481956.1       1.0X
+deserialize                                         276            291           9          0.0      276403.3       1.7X
 
 
diff --git a/mllib/benchmarks/UDTSerializationBenchmark-results.txt b/mllib/benchmarks/UDTSerializationBenchmark-results.txt
index a0c853e99014b..8a8e904430172 100644
--- a/mllib/benchmarks/UDTSerializationBenchmark-results.txt
+++ b/mllib/benchmarks/UDTSerializationBenchmark-results.txt
@@ -2,11 +2,11 @@
 VectorUDT de/serialization
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 VectorUDT de/serialization:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-serialize                                           271            294          12          0.0      271054.3       1.0X
-deserialize                                         190            192           2          0.0      189706.1       1.4X
+serialize                                           248            271          16          0.0      248244.3       1.0X
+deserialize                                         135            165          19          0.0      135445.9       1.8X
 
 
diff --git a/mllib/pom.xml b/mllib/pom.xml
index 9eacf380e17f2..c0e0574b00056 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
@@ -142,20 +142,20 @@
       <scope>test</scope>
     </dependency>
 
+    <dependency>
+      <groupId>dev.ludovic.netlib</groupId>
+      <artifactId>blas</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>dev.ludovic.netlib</groupId>
+      <artifactId>lapack</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>dev.ludovic.netlib</groupId>
+      <artifactId>arpack</artifactId>
+    </dependency>
+
   </dependencies>
-  <profiles>
-    <profile>
-      <id>netlib-lgpl</id>
-      <dependencies>
-        <dependency>
-          <groupId>com.github.fommil.netlib</groupId>
-          <artifactId>all</artifactId>
-          <version>${netlib.java.version}</version>
-          <type>pom</type>
-        </dependency>
-      </dependencies>
-    </profile>
-  </profiles>
 
   <build>
     <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
diff --git a/mllib/src/main/scala/org/apache/spark/ml/Estimator.scala b/mllib/src/main/scala/org/apache/spark/ml/Estimator.scala
index 8815eb29bc860..3a02e2be6fe04 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/Estimator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/Estimator.scala
@@ -76,7 +76,7 @@ abstract class Estimator[M <: Model[M]] extends PipelineStage {
    * @return fitted models, matching the input parameter maps
    */
   @Since("2.0.0")
-  def fit(dataset: Dataset[_], paramMaps: Array[ParamMap]): Seq[M] = {
+  def fit(dataset: Dataset[_], paramMaps: Seq[ParamMap]): Seq[M] = {
     paramMaps.map(fit(dataset, _))
   }
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/Predictor.scala b/mllib/src/main/scala/org/apache/spark/ml/Predictor.scala
index febeba7e13fcb..e0b128e369816 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/Predictor.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/Predictor.scala
@@ -19,6 +19,7 @@ package org.apache.spark.ml
 
 import org.apache.spark.annotation.Since
 import org.apache.spark.ml.feature.{Instance, LabeledPoint}
+import org.apache.spark.ml.functions.checkNonNegativeWeight
 import org.apache.spark.ml.linalg.{Vector, VectorUDT}
 import org.apache.spark.ml.param._
 import org.apache.spark.ml.param.shared._
@@ -71,7 +72,7 @@ private[ml] trait PredictorParams extends Params
     val w = this match {
       case p: HasWeightCol =>
         if (isDefined(p.weightCol) && $(p.weightCol).nonEmpty) {
-          col($(p.weightCol)).cast(DoubleType)
+          checkNonNegativeWeight((col($(p.weightCol)).cast(DoubleType)))
         } else {
           lit(1.0)
         }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/ann/BreezeUtil.scala b/mllib/src/main/scala/org/apache/spark/ml/ann/BreezeUtil.scala
index 6bbe7e1cb2134..ce9093a6238ee 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/ann/BreezeUtil.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/ann/BreezeUtil.scala
@@ -18,7 +18,8 @@
 package org.apache.spark.ml.ann
 
 import breeze.linalg.{DenseMatrix => BDM, DenseVector => BDV}
-import com.github.fommil.netlib.BLAS.{getInstance => NativeBLAS}
+
+import org.apache.spark.ml.linalg.BLAS
 
 /**
  * In-place DGEMM and DGEMV for Breeze
@@ -41,7 +42,7 @@ private[ann] object BreezeUtil {
     require(A.cols == B.rows, "A & B Dimension mismatch!")
     require(A.rows == C.rows, "A & C Dimension mismatch!")
     require(B.cols == C.cols, "A & C Dimension mismatch!")
-    NativeBLAS.dgemm(transposeString(A), transposeString(B), C.rows, C.cols, A.cols,
+    BLAS.nativeBLAS.dgemm(transposeString(A), transposeString(B), C.rows, C.cols, A.cols,
       alpha, A.data, A.offset, A.majorStride, B.data, B.offset, B.majorStride,
       beta, C.data, C.offset, C.rows)
   }
@@ -57,7 +58,7 @@ private[ann] object BreezeUtil {
   def dgemv(alpha: Double, A: BDM[Double], x: BDV[Double], beta: Double, y: BDV[Double]): Unit = {
     require(A.cols == x.length, "A & x Dimension mismatch!")
     require(A.rows == y.length, "A & y Dimension mismatch!")
-    NativeBLAS.dgemv(transposeString(A), A.rows, A.cols,
+    BLAS.nativeBLAS.dgemv(transposeString(A), A.rows, A.cols,
       alpha, A.data, A.offset, A.majorStride, x.data, x.offset, x.stride,
       beta, y.data, y.offset, y.stride)
   }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/ann/Layer.scala b/mllib/src/main/scala/org/apache/spark/ml/ann/Layer.scala
index 2b4b0fc55b955..253d4083de7d4 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/ann/Layer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/ann/Layer.scala
@@ -838,7 +838,7 @@ private[ml] class FeedForwardTrainer(
    * @param data RDD of input and output vector pairs
    * @return model
    */
-  def train(data: RDD[(Vector, Vector)]): TopologyModel = {
+  def train(data: RDD[(Vector, Vector)]): (TopologyModel, Array[Double]) = {
     val w = if (getWeights == null) {
       // TODO: will make a copy if vector is a subvector of BDV (see Vectors code)
       topology.model(_seed).weights
@@ -851,9 +851,14 @@ private[ml] class FeedForwardTrainer(
     }
     val handlePersistence = trainData.getStorageLevel == StorageLevel.NONE
     if (handlePersistence) trainData.persist(StorageLevel.MEMORY_AND_DISK)
-    val newWeights = optimizer.optimize(trainData, w)
+    val (newWeights, lossHistory) = optimizer match {
+      case lbfgs: LBFGS => lbfgs.optimizeWithLossReturned(trainData, w)
+      case sgd: GradientDescent => sgd.optimizeWithLossReturned(trainData, w)
+      case other => throw new UnsupportedOperationException(
+        s"Only LBFGS and GradientDescent are supported but got ${other.getClass}.")
+    }
     if (handlePersistence) trainData.unpersist()
-    topology.model(newWeights)
+    (topology.model(newWeights), lossHistory)
   }
 
 }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/attribute/package.scala b/mllib/src/main/scala/org/apache/spark/ml/attribute/package.scala
index d26acf924c0a3..7bc86c4871cfb 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/attribute/package.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/attribute/package.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.ml
 
-import org.apache.spark.ml.attribute.{Attribute, AttributeGroup}
-
 /**
  * ==ML attributes==
  *
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/ClassificationSummary.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/ClassificationSummary.scala
new file mode 100644
index 0000000000000..9f3428db484c2
--- /dev/null
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/ClassificationSummary.scala
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.classification
+
+import org.apache.spark.annotation.Since
+import org.apache.spark.ml.linalg.Vector
+import org.apache.spark.mllib.evaluation.{BinaryClassificationMetrics, MulticlassMetrics}
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions.{col, lit}
+import org.apache.spark.sql.types.DoubleType
+
+
+/**
+ * Abstraction for multiclass classification results for a given model.
+ */
+private[classification] trait ClassificationSummary extends Serializable {
+
+  /**
+   * Dataframe output by the model's `transform` method.
+   */
+  @Since("3.1.0")
+  def predictions: DataFrame
+
+  /** Field in "predictions" which gives the prediction of each class. */
+  @Since("3.1.0")
+  def predictionCol: String
+
+  /** Field in "predictions" which gives the true label of each instance (if available). */
+  @Since("3.1.0")
+  def labelCol: String
+
+  /** Field in "predictions" which gives the weight of each instance. */
+  @Since("3.1.0")
+  def weightCol: String
+
+  @transient private val multiclassMetrics = {
+    val weightColumn = if (predictions.schema.fieldNames.contains(weightCol)) {
+      col(weightCol).cast(DoubleType)
+    } else {
+      lit(1.0)
+    }
+    new MulticlassMetrics(
+      predictions.select(col(predictionCol), col(labelCol).cast(DoubleType), weightColumn)
+        .rdd.map {
+          case Row(prediction: Double, label: Double, weight: Double) => (prediction, label, weight)
+      })
+  }
+
+  /**
+   * Returns the sequence of labels in ascending order. This order matches the order used
+   * in metrics which are specified as arrays over labels, e.g., truePositiveRateByLabel.
+   *
+   * Note: In most cases, it will be values {0.0, 1.0, ..., numClasses-1}, However, if the
+   * training set is missing a label, then all of the arrays over labels
+   * (e.g., from truePositiveRateByLabel) will be of length numClasses-1 instead of the
+   * expected numClasses.
+   */
+  @Since("3.1.0")
+  def labels: Array[Double] = multiclassMetrics.labels
+
+  /** Returns true positive rate for each label (category). */
+  @Since("3.1.0")
+  def truePositiveRateByLabel: Array[Double] = recallByLabel
+
+  /** Returns false positive rate for each label (category). */
+  @Since("3.1.0")
+  def falsePositiveRateByLabel: Array[Double] = {
+    multiclassMetrics.labels.map(label => multiclassMetrics.falsePositiveRate(label))
+  }
+
+  /** Returns precision for each label (category). */
+  @Since("3.1.0")
+  def precisionByLabel: Array[Double] = {
+    multiclassMetrics.labels.map(label => multiclassMetrics.precision(label))
+  }
+
+  /** Returns recall for each label (category). */
+  @Since("3.1.0")
+  def recallByLabel: Array[Double] = {
+    multiclassMetrics.labels.map(label => multiclassMetrics.recall(label))
+  }
+
+  /** Returns f-measure for each label (category). */
+  @Since("3.1.0")
+  def fMeasureByLabel(beta: Double): Array[Double] = {
+    multiclassMetrics.labels.map(label => multiclassMetrics.fMeasure(label, beta))
+  }
+
+  /** Returns f1-measure for each label (category). */
+  @Since("3.1.0")
+  def fMeasureByLabel: Array[Double] = fMeasureByLabel(1.0)
+
+  /**
+   * Returns accuracy.
+   * (equals to the total number of correctly classified instances
+   * out of the total number of instances.)
+   */
+  @Since("3.1.0")
+  def accuracy: Double = multiclassMetrics.accuracy
+
+  /**
+   * Returns weighted true positive rate.
+   * (equals to precision, recall and f-measure)
+   */
+  @Since("3.1.0")
+  def weightedTruePositiveRate: Double = weightedRecall
+
+  /** Returns weighted false positive rate. */
+  @Since("3.1.0")
+  def weightedFalsePositiveRate: Double = multiclassMetrics.weightedFalsePositiveRate
+
+  /**
+   * Returns weighted averaged recall.
+   * (equals to precision, recall and f-measure)
+   */
+  @Since("3.1.0")
+  def weightedRecall: Double = multiclassMetrics.weightedRecall
+
+  /** Returns weighted averaged precision. */
+  @Since("3.1.0")
+  def weightedPrecision: Double = multiclassMetrics.weightedPrecision
+
+  /** Returns weighted averaged f-measure. */
+  @Since("3.1.0")
+  def weightedFMeasure(beta: Double): Double = multiclassMetrics.weightedFMeasure(beta)
+
+  /** Returns weighted averaged f1-measure. */
+  @Since("3.1.0")
+  def weightedFMeasure: Double = multiclassMetrics.weightedFMeasure(1.0)
+}
+
+/**
+ * Abstraction for training results.
+ */
+private[classification] trait TrainingSummary {
+
+  /**
+   *  objective function (scaled loss + regularization) at each iteration.
+   *  It contains one more element, the initial state, than number of iterations.
+   */
+  @Since("3.1.0")
+  def objectiveHistory: Array[Double]
+
+  /** Number of training iterations. */
+  @Since("3.1.0")
+  def totalIterations: Int = {
+    assert(objectiveHistory.length > 0, "objectiveHistory length should be greater than 0.")
+    objectiveHistory.length - 1
+  }
+}
+
+/**
+ * Abstraction for binary classification results for a given model.
+ */
+private[classification] trait BinaryClassificationSummary extends ClassificationSummary {
+
+  private val sparkSession = predictions.sparkSession
+  import sparkSession.implicits._
+
+  /**
+   *  Field in "predictions" which gives the probability or rawPrediction of each class as a
+   *  vector.
+   */
+  def scoreCol: String = null
+
+  @transient private val binaryMetrics = {
+    val weightColumn = if (predictions.schema.fieldNames.contains(weightCol)) {
+      col(weightCol).cast(DoubleType)
+    } else {
+      lit(1.0)
+    }
+
+    // TODO: Allow the user to vary the number of bins using a setBins method in
+    // BinaryClassificationMetrics. For now the default is set to 1000.
+    new BinaryClassificationMetrics(
+      predictions.select(col(scoreCol), col(labelCol).cast(DoubleType), weightColumn).rdd.map {
+        case Row(score: Vector, label: Double, weight: Double) => (score(1), label, weight)
+      }, 1000
+    )
+  }
+
+  /**
+   * Returns the receiver operating characteristic (ROC) curve,
+   * which is a Dataframe having two fields (FPR, TPR)
+   * with (0.0, 0.0) prepended and (1.0, 1.0) appended to it.
+   * See http://en.wikipedia.org/wiki/Receiver_operating_characteristic
+   */
+  @Since("3.1.0")
+  @transient lazy val roc: DataFrame = binaryMetrics.roc().toDF("FPR", "TPR")
+
+  /**
+   * Computes the area under the receiver operating characteristic (ROC) curve.
+   */
+  @Since("3.1.0")
+  lazy val areaUnderROC: Double = binaryMetrics.areaUnderROC()
+
+  /**
+   * Returns the precision-recall curve, which is a Dataframe containing
+   * two fields recall, precision with (0.0, 1.0) prepended to it.
+   */
+  @Since("3.1.0")
+  @transient lazy val pr: DataFrame = binaryMetrics.pr().toDF("recall", "precision")
+
+  /**
+   * Returns a dataframe with two fields (threshold, F-Measure) curve with beta = 1.0.
+   */
+  @Since("3.1.0")
+  @transient lazy val fMeasureByThreshold: DataFrame = {
+    binaryMetrics.fMeasureByThreshold().toDF("threshold", "F-Measure")
+  }
+
+  /**
+   * Returns a dataframe with two fields (threshold, precision) curve.
+   * Every possible probability obtained in transforming the dataset are used
+   * as thresholds used in calculating the precision.
+   */
+  @Since("3.1.0")
+  @transient lazy val precisionByThreshold: DataFrame = {
+    binaryMetrics.precisionByThreshold().toDF("threshold", "precision")
+  }
+
+  /**
+   * Returns a dataframe with two fields (threshold, recall) curve.
+   * Every possible probability obtained in transforming the dataset are used
+   * as thresholds used in calculating the recall.
+   */
+  @Since("3.1.0")
+  @transient lazy val recallByThreshold: DataFrame = {
+    binaryMetrics.recallByThreshold().toDF("threshold", "recall")
+  }
+}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala
index 1f3f291644f93..233e8e5bcdc88 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala
@@ -22,6 +22,7 @@ import org.apache.spark.annotation.Since
 import org.apache.spark.ml.{PredictionModel, Predictor, PredictorParams}
 import org.apache.spark.ml.feature.{Instance, LabeledPoint}
 import org.apache.spark.ml.linalg.{Vector, VectorUDT}
+import org.apache.spark.ml.param.ParamMap
 import org.apache.spark.ml.param.shared.HasRawPredictionCol
 import org.apache.spark.ml.util.{MetadataUtils, SchemaUtils}
 import org.apache.spark.rdd.RDD
@@ -269,4 +270,26 @@ abstract class ClassificationModel[FeaturesType, M <: ClassificationModel[Featur
    * @return  predicted label
    */
   protected def raw2prediction(rawPrediction: Vector): Double = rawPrediction.argmax
+
+  /**
+   * If the rawPrediction and prediction columns are set, this method returns the current model,
+   * otherwise it generates new columns for them and sets them as columns on a new copy of
+   * the current model
+   */
+  private[classification] def findSummaryModel():
+  (ClassificationModel[FeaturesType, M], String, String) = {
+    val model = if ($(rawPredictionCol).isEmpty && $(predictionCol).isEmpty) {
+      copy(ParamMap.empty)
+        .setRawPredictionCol("rawPrediction_" + java.util.UUID.randomUUID.toString)
+        .setPredictionCol("prediction_" + java.util.UUID.randomUUID.toString)
+    } else if ($(rawPredictionCol).isEmpty) {
+      copy(ParamMap.empty).setRawPredictionCol("rawPrediction_" +
+        java.util.UUID.randomUUID.toString)
+    } else if ($(predictionCol).isEmpty) {
+      copy(ParamMap.empty).setPredictionCol("prediction_" + java.util.UUID.randomUUID.toString)
+    } else {
+      this
+    }
+    (model, model.getRawPredictionCol, model.getPredictionCol)
+  }
 }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/DecisionTreeClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/DecisionTreeClassifier.scala
index 530943c910d7d..4a2bc19426ef3 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/DecisionTreeClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/DecisionTreeClassifier.scala
@@ -129,7 +129,7 @@ class DecisionTreeClassifier @Since("1.4.0") (
     instr.logNumClasses(numClasses)
     instr.logParams(this, labelCol, featuresCol, predictionCol, rawPredictionCol,
       probabilityCol, leafCol, maxDepth, maxBins, minInstancesPerNode, minInfoGain,
-      maxMemoryInMB, cacheNodeIds, checkpointInterval, impurity, seed)
+      maxMemoryInMB, cacheNodeIds, checkpointInterval, impurity, seed, thresholds)
 
     val trees = RandomForest.run(instances, strategy, numTrees = 1, featureSubsetStrategy = "all",
       seed = $(seed), instr = Some(instr), parentUID = Some(uid))
@@ -288,7 +288,8 @@ object DecisionTreeClassificationModel extends MLReadable[DecisionTreeClassifica
       DefaultParamsWriter.saveMetadata(instance, path, sc, Some(extraMetadata))
       val (nodeData, _) = NodeData.build(instance.rootNode, 0)
       val dataPath = new Path(path, "data").toString
-      sparkSession.createDataFrame(nodeData).write.parquet(dataPath)
+      val numDataParts = NodeData.inferNumPartitions(instance.numNodes)
+      sparkSession.createDataFrame(nodeData).repartition(numDataParts).write.parquet(dataPath)
     }
   }
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala
index 6ef42500f86f7..4188f6893ea37 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala
@@ -30,7 +30,7 @@ import org.apache.spark.ml.util.Instrumentation.instrumented
 import org.apache.spark.mllib.linalg.{Vector => OldVector}
 import org.apache.spark.mllib.linalg.VectorImplicits._
 import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.{Dataset, Row}
+import org.apache.spark.sql._
 import org.apache.spark.storage.StorageLevel
 
 /**
@@ -85,7 +85,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setFactorSize(value: Int): this.type = set(factorSize, value)
-  setDefault(factorSize -> 8)
 
   /**
    * Set whether to fit intercept term.
@@ -95,7 +94,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
-  setDefault(fitIntercept -> true)
 
   /**
    * Set whether to fit linear term.
@@ -105,7 +103,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setFitLinear(value: Boolean): this.type = set(fitLinear, value)
-  setDefault(fitLinear -> true)
 
   /**
    * Set the L2 regularization parameter.
@@ -115,7 +112,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setRegParam(value: Double): this.type = set(regParam, value)
-  setDefault(regParam -> 0.0)
 
   /**
    * Set the mini-batch fraction parameter.
@@ -125,7 +121,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setMiniBatchFraction(value: Double): this.type = set(miniBatchFraction, value)
-  setDefault(miniBatchFraction -> 1.0)
 
   /**
    * Set the standard deviation of initial coefficients.
@@ -135,7 +130,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setInitStd(value: Double): this.type = set(initStd, value)
-  setDefault(initStd -> 0.01)
 
   /**
    * Set the maximum number of iterations.
@@ -145,7 +139,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setMaxIter(value: Int): this.type = set(maxIter, value)
-  setDefault(maxIter -> 100)
 
   /**
    * Set the initial step size for the first step (like learning rate).
@@ -155,7 +148,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setStepSize(value: Double): this.type = set(stepSize, value)
-  setDefault(stepSize -> 1.0)
 
   /**
    * Set the convergence tolerance of iterations.
@@ -165,7 +157,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setTol(value: Double): this.type = set(tol, value)
-  setDefault(tol -> 1E-6)
 
   /**
    * Set the solver algorithm used for optimization.
@@ -176,7 +167,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setSolver(value: String): this.type = set(solver, value)
-  setDefault(solver -> AdamW)
 
   /**
    * Set the random seed for weight initialization.
@@ -187,9 +177,7 @@ class FMClassifier @Since("3.0.0") (
   def setSeed(value: Long): this.type = set(seed, value)
 
   override protected def train(
-      dataset: Dataset[_]
-    ): FMClassificationModel = instrumented { instr =>
-
+      dataset: Dataset[_]): FMClassificationModel = instrumented { instr =>
     val numClasses = 2
     if (isDefined(thresholds)) {
       require($(thresholds).length == numClasses, this.getClass.getSimpleName +
@@ -200,7 +188,7 @@ class FMClassifier @Since("3.0.0") (
     instr.logPipelineStage(this)
     instr.logDataset(dataset)
     instr.logParams(this, factorSize, fitIntercept, fitLinear, regParam,
-      miniBatchFraction, initStd, maxIter, stepSize, tol, solver)
+      miniBatchFraction, initStd, maxIter, stepSize, tol, solver, thresholds)
     instr.logNumClasses(numClasses)
 
     val numFeatures = MetadataUtils.getNumFeatures(dataset, $(featuresCol))
@@ -212,14 +200,34 @@ class FMClassifier @Since("3.0.0") (
 
     if (handlePersistence) data.persist(StorageLevel.MEMORY_AND_DISK)
 
-    val coefficients = trainImpl(data, numFeatures, LogisticLoss)
+    val (coefficients, objectiveHistory) = trainImpl(data, numFeatures, LogisticLoss)
 
     val (intercept, linear, factors) = splitCoefficients(
       coefficients, numFeatures, $(factorSize), $(fitIntercept), $(fitLinear))
 
     if (handlePersistence) data.unpersist()
 
-    copyValues(new FMClassificationModel(uid, intercept, linear, factors))
+    createModel(dataset, intercept, linear, factors, objectiveHistory)
+  }
+
+  private def createModel(
+    dataset: Dataset[_],
+    intercept: Double,
+    linear: Vector,
+    factors: Matrix,
+    objectiveHistory: Array[Double]): FMClassificationModel = {
+    val model = copyValues(new FMClassificationModel(uid, intercept, linear, factors))
+    val weightColName = if (!isDefined(weightCol)) "weightCol" else $(weightCol)
+
+    val (summaryModel, probabilityColName, predictionColName) = model.findSummaryModel()
+    val summary = new FMClassificationTrainingSummaryImpl(
+      summaryModel.transform(dataset),
+      probabilityColName,
+      predictionColName,
+      $(labelCol),
+      weightColName,
+      objectiveHistory)
+    model.setSummary(Some(summary))
   }
 
   @Since("3.0.0")
@@ -243,7 +251,8 @@ class FMClassificationModel private[classification] (
   @Since("3.0.0") val linear: Vector,
   @Since("3.0.0") val factors: Matrix)
   extends ProbabilisticClassificationModel[Vector, FMClassificationModel]
-    with FMClassifierParams with MLWritable {
+    with FMClassifierParams with MLWritable
+    with HasTrainingSummary[FMClassificationTrainingSummary]{
 
   @Since("3.0.0")
   override val numClasses: Int = 2
@@ -251,6 +260,27 @@ class FMClassificationModel private[classification] (
   @Since("3.0.0")
   override val numFeatures: Int = linear.size
 
+  /**
+   * Gets summary of model on training set. An exception is thrown
+   * if `hasSummary` is false.
+   */
+  @Since("3.1.0")
+  override def summary: FMClassificationTrainingSummary = super.summary
+
+  /**
+   * Evaluates the model on a test dataset.
+   *
+   * @param dataset Test dataset to evaluate model on.
+   */
+  @Since("3.1.0")
+  def evaluate(dataset: Dataset[_]): FMClassificationSummary = {
+    val weightColName = if (!isDefined(weightCol)) "weightCol" else $(weightCol)
+    // Handle possible missing or invalid probability or prediction columns
+    val (summaryModel, probability, predictionColName) = findSummaryModel()
+    new FMClassificationSummaryImpl(summaryModel.transform(dataset),
+      probability, predictionColName, $(labelCol), weightColName)
+  }
+
   @Since("3.0.0")
   override def predictRaw(features: Vector): Vector = {
     val rawPrediction = getRawPrediction(features, intercept, linear, factors)
@@ -328,3 +358,53 @@ object FMClassificationModel extends MLReadable[FMClassificationModel] {
     }
   }
 }
+
+/**
+ * Abstraction for FMClassifier results for a given model.
+ */
+sealed trait FMClassificationSummary extends BinaryClassificationSummary
+
+/**
+ * Abstraction for FMClassifier training results.
+ */
+sealed trait FMClassificationTrainingSummary extends FMClassificationSummary with TrainingSummary
+
+/**
+ * FMClassifier results for a given model.
+ *
+ * @param predictions dataframe output by the model's `transform` method.
+ * @param scoreCol field in "predictions" which gives the probability of each instance.
+ * @param predictionCol field in "predictions" which gives the prediction for a data instance as a
+ *                      double.
+ * @param labelCol field in "predictions" which gives the true label of each instance.
+ * @param weightCol field in "predictions" which gives the weight of each instance.
+ */
+private class FMClassificationSummaryImpl(
+    @transient override val predictions: DataFrame,
+    override val scoreCol: String,
+    override val predictionCol: String,
+    override val labelCol: String,
+    override val weightCol: String)
+  extends FMClassificationSummary
+
+/**
+ * FMClassifier training results.
+ *
+ * @param predictions dataframe output by the model's `transform` method.
+ * @param scoreCol field in "predictions" which gives the probability of each instance.
+ * @param predictionCol field in "predictions" which gives the prediction for a data instance as a
+ *                      double.
+ * @param labelCol field in "predictions" which gives the true label of each instance.
+ * @param weightCol field in "predictions" which gives the weight of each instance.
+ * @param objectiveHistory objective function (scaled loss + regularization) at each iteration.
+ */
+private class FMClassificationTrainingSummaryImpl(
+    predictions: DataFrame,
+    scoreCol: String,
+    predictionCol: String,
+    labelCol: String,
+    weightCol: String,
+    override val objectiveHistory: Array[Double])
+  extends FMClassificationSummaryImpl(
+    predictions, scoreCol, predictionCol, labelCol, weightCol)
+    with FMClassificationTrainingSummary
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/GBTClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/GBTClassifier.scala
index 46810bccc8e69..453e28609a0c2 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/GBTClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/GBTClassifier.scala
@@ -17,14 +17,13 @@
 
 package org.apache.spark.ml.classification
 
-import com.github.fommil.netlib.BLAS.{getInstance => blas}
 import org.json4s.{DefaultFormats, JObject}
 import org.json4s.JsonDSL._
 
 import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.feature.Instance
-import org.apache.spark.ml.linalg.{DenseVector, SparseVector, Vector, Vectors}
+import org.apache.spark.ml.linalg.{BLAS, DenseVector, SparseVector, Vector, Vectors}
 import org.apache.spark.ml.param.ParamMap
 import org.apache.spark.ml.regression.DecisionTreeRegressionModel
 import org.apache.spark.ml.tree._
@@ -197,7 +196,7 @@ class GBTClassifier @Since("1.4.0") (
     instr.logParams(this, labelCol, weightCol, featuresCol, predictionCol, leafCol,
       impurity, lossType, maxDepth, maxBins, maxIter, maxMemoryInMB, minInfoGain,
       minInstancesPerNode, minWeightFractionPerNode, seed, stepSize, subsamplingRate, cacheNodeIds,
-      checkpointInterval, featureSubsetStrategy, validationIndicatorCol, validationTol)
+      checkpointInterval, featureSubsetStrategy, validationIndicatorCol, validationTol, thresholds)
     instr.logNumClasses(numClasses)
 
     val categoricalFeatures = MetadataUtils.getCategoricalFeatures(dataset.schema($(featuresCol)))
@@ -371,7 +370,7 @@ class GBTClassificationModel private[ml](
   /** Raw prediction for the positive class. */
   private def margin(features: Vector): Double = {
     val treePredictions = _trees.map(_.rootNode.predictImpl(features).prediction)
-    blas.ddot(getNumTrees, treePredictions, 1, _treeWeights, 1)
+    BLAS.nativeBLAS.ddot(getNumTrees, treePredictions, 1, _treeWeights, 1)
   }
 
   /** (private[ml]) Convert to a model in the old API */
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
index 217398c51b393..9214f55130856 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
@@ -36,13 +36,13 @@ import org.apache.spark.ml.stat._
 import org.apache.spark.ml.util._
 import org.apache.spark.ml.util.Instrumentation.instrumented
 import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.{Dataset, Row}
+import org.apache.spark.sql._
 import org.apache.spark.storage.StorageLevel
 
 /** Params for linear SVM Classifier. */
 private[classification] trait LinearSVCParams extends ClassifierParams with HasRegParam
   with HasMaxIter with HasFitIntercept with HasTol with HasStandardization with HasWeightCol
-  with HasAggregationDepth with HasThreshold with HasBlockSize {
+  with HasAggregationDepth with HasThreshold with HasMaxBlockSizeInMB {
 
   /**
    * Param for threshold in binary classification prediction.
@@ -55,6 +55,9 @@ private[classification] trait LinearSVCParams extends ClassifierParams with HasR
    */
   final override val threshold: DoubleParam = new DoubleParam(this, "threshold",
     "threshold in binary classification prediction applied to rawPrediction")
+
+  setDefault(regParam -> 0.0, maxIter -> 100, fitIntercept -> true, tol -> 1E-6,
+    standardization -> true, threshold -> 0.0, aggregationDepth -> 2, maxBlockSizeInMB -> 0.0)
 }
 
 /**
@@ -64,6 +67,10 @@ private[classification] trait LinearSVCParams extends ClassifierParams with HasR
  * This binary classifier optimizes the Hinge Loss using the OWLQN optimizer.
  * Only supports L2 regularization currently.
  *
+ * Since 3.1.0, it supports stacking instances into blocks and using GEMV for
+ * better performance.
+ * The block size will be 1.0 MB, if param maxBlockSizeInMB is set 0.0 by default.
+ *
  */
 @Since("2.2.0")
 class LinearSVC @Since("2.2.0") (
@@ -82,7 +89,6 @@ class LinearSVC @Since("2.2.0") (
    */
   @Since("2.2.0")
   def setRegParam(value: Double): this.type = set(regParam, value)
-  setDefault(regParam -> 0.0)
 
   /**
    * Set the maximum number of iterations.
@@ -92,7 +98,6 @@ class LinearSVC @Since("2.2.0") (
    */
   @Since("2.2.0")
   def setMaxIter(value: Int): this.type = set(maxIter, value)
-  setDefault(maxIter -> 100)
 
   /**
    * Whether to fit an intercept term.
@@ -102,7 +107,6 @@ class LinearSVC @Since("2.2.0") (
    */
   @Since("2.2.0")
   def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
-  setDefault(fitIntercept -> true)
 
   /**
    * Set the convergence tolerance of iterations.
@@ -113,7 +117,6 @@ class LinearSVC @Since("2.2.0") (
    */
   @Since("2.2.0")
   def setTol(value: Double): this.type = set(tol, value)
-  setDefault(tol -> 1E-6)
 
   /**
    * Whether to standardize the training features before fitting the model.
@@ -123,7 +126,6 @@ class LinearSVC @Since("2.2.0") (
    */
   @Since("2.2.0")
   def setStandardization(value: Boolean): this.type = set(standardization, value)
-  setDefault(standardization -> true)
 
   /**
    * Set the value of param [[weightCol]].
@@ -142,7 +144,6 @@ class LinearSVC @Since("2.2.0") (
    */
   @Since("2.2.0")
   def setThreshold(value: Double): this.type = set(threshold, value)
-  setDefault(threshold -> 0.0)
 
   /**
    * Suggested depth for treeAggregate (greater than or equal to 2).
@@ -154,26 +155,15 @@ class LinearSVC @Since("2.2.0") (
    */
   @Since("2.2.0")
   def setAggregationDepth(value: Int): this.type = set(aggregationDepth, value)
-  setDefault(aggregationDepth -> 2)
 
   /**
-   * Set block size for stacking input data in matrices.
-   * If blockSize == 1, then stacking will be skipped, and each vector is treated individually;
-   * If blockSize &gt; 1, then vectors will be stacked to blocks, and high-level BLAS routines
-   * will be used if possible (for example, GEMV instead of DOT, GEMM instead of GEMV).
-   * Recommended size is between 10 and 1000. An appropriate choice of the block size depends
-   * on the sparsity and dim of input datasets, the underlying BLAS implementation (for example,
-   * f2jBLAS, OpenBLAS, intel MKL) and its configuration (for example, number of threads).
-   * Note that existing BLAS implementations are mainly optimized for dense matrices, if the
-   * input dataset is sparse, stacking may bring no performance gain, the worse is possible
-   * performance regression.
-   * Default is 1.
+   * Sets the value of param [[maxBlockSizeInMB]].
+   * Default is 0.0, then 1.0 MB will be chosen.
    *
    * @group expertSetParam
    */
   @Since("3.1.0")
-  def setBlockSize(value: Int): this.type = set(blockSize, value)
-  setDefault(blockSize -> 1)
+  def setMaxBlockSizeInMB(value: Double): this.type = set(maxBlockSizeInMB, value)
 
   @Since("2.2.0")
   override def copy(extra: ParamMap): LinearSVC = defaultCopy(extra)
@@ -182,19 +172,19 @@ class LinearSVC @Since("2.2.0") (
     instr.logPipelineStage(this)
     instr.logDataset(dataset)
     instr.logParams(this, labelCol, weightCol, featuresCol, predictionCol, rawPredictionCol,
-      regParam, maxIter, fitIntercept, tol, standardization, threshold, aggregationDepth, blockSize)
+      regParam, maxIter, fitIntercept, tol, standardization, threshold, aggregationDepth,
+      maxBlockSizeInMB)
+
+    if (dataset.storageLevel != StorageLevel.NONE) {
+      instr.logWarning(s"Input instances will be standardized, blockified to blocks, and " +
+        s"then cached during training. Be careful of double caching!")
+    }
 
     val instances = extractInstances(dataset)
       .setName("training instances")
 
-    if (dataset.storageLevel == StorageLevel.NONE && $(blockSize) == 1) {
-      instances.persist(StorageLevel.MEMORY_AND_DISK)
-    }
-
-    var requestedMetrics = Seq("mean", "std", "count")
-    if ($(blockSize) != 1) requestedMetrics +:= "numNonZeros"
     val (summarizer, labelSummarizer) = Summarizer
-      .getClassificationSummarizers(instances, $(aggregationDepth), requestedMetrics)
+      .getClassificationSummarizers(instances, $(aggregationDepth), Seq("mean", "std", "count"))
 
     val histogram = labelSummarizer.histogram
     val numInvalid = labelSummarizer.countInvalid
@@ -204,14 +194,12 @@ class LinearSVC @Since("2.2.0") (
     instr.logNamedValue("lowestLabelWeight", labelSummarizer.histogram.min.toString)
     instr.logNamedValue("highestLabelWeight", labelSummarizer.histogram.max.toString)
     instr.logSumOfWeights(summarizer.weightSum)
-    if ($(blockSize) > 1) {
-      val scale = 1.0 / summarizer.count / numFeatures
-      val sparsity = 1 - summarizer.numNonzeros.toArray.map(_ * scale).sum
-      instr.logNamedValue("sparsity", sparsity.toString)
-      if (sparsity > 0.5) {
-        instr.logWarning(s"sparsity of input dataset is $sparsity, " +
-          s"which may hurt performance in high-level BLAS.")
-      }
+
+    var actualBlockSizeInMB = $(maxBlockSizeInMB)
+    if (actualBlockSizeInMB == 0) {
+      actualBlockSizeInMB = InstanceBlock.DefaultBlockSizeInMB
+      require(actualBlockSizeInMB > 0, "inferred actual BlockSizeInMB must > 0")
+      instr.logNamedValue("actualBlockSizeInMB", actualBlockSizeInMB.toString)
     }
 
     val numClasses = MetadataUtils.getNumClasses(dataset.schema($(labelCol))) match {
@@ -234,6 +222,7 @@ class LinearSVC @Since("2.2.0") (
     }
 
     val featuresStd = summarizer.std.toArray
+    val featuresMean = summarizer.mean.toArray
     val getFeaturesStd = (j: Int) => featuresStd(j)
     val regularization = if ($(regParam) != 0.0) {
       val shouldApply = (idx: Int) => idx >= 0 && idx < numFeatures
@@ -250,12 +239,9 @@ class LinearSVC @Since("2.2.0") (
        Note that the intercept in scaled space and original space is the same;
        as a result, no scaling is needed.
      */
-    val (rawCoefficients, objectiveHistory) = if ($(blockSize) == 1) {
-      trainOnRows(instances, featuresStd, regularization, optimizer)
-    } else {
-      trainOnBlocks(instances, featuresStd, regularization, optimizer)
-    }
-    if (instances.getStorageLevel != StorageLevel.NONE) instances.unpersist()
+    val (rawCoefficients, objectiveHistory) =
+      trainImpl(instances, actualBlockSizeInMB, featuresStd, featuresMean,
+        regularization, optimizer)
 
     if (rawCoefficients == null) {
       val msg = s"${optimizer.getClass.getName} failed."
@@ -267,62 +253,71 @@ class LinearSVC @Since("2.2.0") (
       if (featuresStd(i) != 0.0) rawCoefficients(i) / featuresStd(i) else 0.0
     }
     val intercept = if ($(fitIntercept)) rawCoefficients.last else 0.0
-    copyValues(new LinearSVCModel(uid, Vectors.dense(coefficientArray), intercept))
+    createModel(dataset, Vectors.dense(coefficientArray), intercept, objectiveHistory)
   }
 
-  private def trainOnRows(
-      instances: RDD[Instance],
-      featuresStd: Array[Double],
-      regularization: Option[L2Regularization],
-      optimizer: BreezeOWLQN[Int, BDV[Double]]): (Array[Double], Array[Double]) = {
-    val numFeatures = featuresStd.length
-    val numFeaturesPlusIntercept = if ($(fitIntercept)) numFeatures + 1 else numFeatures
-
-    val bcFeaturesStd = instances.context.broadcast(featuresStd)
-    val getAggregatorFunc = new HingeAggregator(bcFeaturesStd, $(fitIntercept))(_)
-    val costFun = new RDDLossFunction(instances, getAggregatorFunc,
-      regularization, $(aggregationDepth))
-
-    val states = optimizer.iterations(new CachedDiffFunction(costFun),
-      Vectors.zeros(numFeaturesPlusIntercept).asBreeze.toDenseVector)
-
-    val arrayBuilder = mutable.ArrayBuilder.make[Double]
-    var state: optimizer.State = null
-    while (states.hasNext) {
-      state = states.next()
-      arrayBuilder += state.adjustedValue
-    }
-    bcFeaturesStd.destroy()
-
-    (if (state != null) state.x.toArray else null, arrayBuilder.result)
+  private def createModel(
+      dataset: Dataset[_],
+      coefficients: Vector,
+      intercept: Double,
+      objectiveHistory: Array[Double]): LinearSVCModel = {
+    val model = copyValues(new LinearSVCModel(uid, coefficients, intercept))
+    val weightColName = if (!isDefined(weightCol)) "weightCol" else $(weightCol)
+
+    val (summaryModel, rawPredictionColName, predictionColName) = model.findSummaryModel()
+    val summary = new LinearSVCTrainingSummaryImpl(
+      summaryModel.transform(dataset),
+      rawPredictionColName,
+      predictionColName,
+      $(labelCol),
+      weightColName,
+      objectiveHistory)
+    model.setSummary(Some(summary))
   }
 
-  private def trainOnBlocks(
+  private def trainImpl(
       instances: RDD[Instance],
+      actualBlockSizeInMB: Double,
       featuresStd: Array[Double],
+      featuresMean: Array[Double],
       regularization: Option[L2Regularization],
       optimizer: BreezeOWLQN[Int, BDV[Double]]): (Array[Double], Array[Double]) = {
     val numFeatures = featuresStd.length
     val numFeaturesPlusIntercept = if ($(fitIntercept)) numFeatures + 1 else numFeatures
 
-    val bcFeaturesStd = instances.context.broadcast(featuresStd)
+    val inverseStd = featuresStd.map(std => if (std != 0) 1.0 / std else 0.0)
+    val scaledMean = Array.tabulate(numFeatures)(i => inverseStd(i) * featuresMean(i))
+    val bcInverseStd = instances.context.broadcast(inverseStd)
+    val bcScaledMean = instances.context.broadcast(scaledMean)
 
     val standardized = instances.mapPartitions { iter =>
-      val inverseStd = bcFeaturesStd.value.map { std => if (std != 0) 1.0 / std else 0.0 }
-      val func = StandardScalerModel.getTransformFunc(Array.empty, inverseStd, false, true)
+      val func = StandardScalerModel.getTransformFunc(Array.empty, bcInverseStd.value, false, true)
       iter.map { case Instance(label, weight, vec) => Instance(label, weight, func(vec)) }
     }
-    val blocks = InstanceBlock.blokify(standardized, $(blockSize))
+
+    val maxMemUsage = (actualBlockSizeInMB * 1024L * 1024L).ceil.toLong
+    val blocks = InstanceBlock.blokifyWithMaxMemUsage(standardized, maxMemUsage)
       .persist(StorageLevel.MEMORY_AND_DISK)
-      .setName(s"training blocks (blockSize=${$(blockSize)})")
+      .setName(s"training blocks (blockSizeInMB=$actualBlockSizeInMB)")
 
-    val getAggregatorFunc = new BlockHingeAggregator($(fitIntercept))(_)
+    val getAggregatorFunc = new HingeBlockAggregator(bcInverseStd, bcScaledMean,
+      $(fitIntercept))(_)
     val costFun = new RDDLossFunction(blocks, getAggregatorFunc,
       regularization, $(aggregationDepth))
 
-    val states = optimizer.iterations(new CachedDiffFunction(costFun),
-      Vectors.zeros(numFeaturesPlusIntercept).asBreeze.toDenseVector)
+    val initialSolution = Array.ofDim[Double](numFeaturesPlusIntercept)
+    if ($(fitIntercept)) {
+      // orginal `initialSolution` is for problem:
+      // y = f(w1 * x1 / std_x1, w2 * x2 / std_x2, ..., intercept)
+      // we should adjust it to the initial solution for problem:
+      // y = f(w1 * (x1 - avg_x1) / std_x1, w2 * (x2 - avg_x2) / std_x2, ..., intercept)
+      // NOTE: this is NOOP before we finally support model initialization
+      val adapt = BLAS.javaBLAS.ddot(numFeatures, initialSolution, 1, scaledMean, 1)
+      initialSolution(numFeatures) += adapt
+    }
 
+    val states = optimizer.iterations(new CachedDiffFunction(costFun),
+      new BDV[Double](initialSolution))
     val arrayBuilder = mutable.ArrayBuilder.make[Double]
     var state: optimizer.State = null
     while (states.hasNext) {
@@ -330,9 +325,19 @@ class LinearSVC @Since("2.2.0") (
       arrayBuilder += state.adjustedValue
     }
     blocks.unpersist()
-    bcFeaturesStd.destroy()
-
-    (if (state != null) state.x.toArray else null, arrayBuilder.result)
+    bcInverseStd.destroy()
+    bcScaledMean.destroy()
+
+    val solution = if (state == null) null else state.x.toArray
+    if ($(fitIntercept) && solution != null) {
+      // the final solution is for problem:
+      // y = f(w1 * (x1 - avg_x1) / std_x1, w2 * (x2 - avg_x2) / std_x2, ..., intercept)
+      // we should adjust it back for original problem:
+      // y = f(w1 * x1 / std_x1, w2 * x2 / std_x2, ..., intercept)
+      val adapt = BLAS.javaBLAS.ddot(numFeatures, solution, 1, scaledMean, 1)
+      solution(numFeatures) -= adapt
+    }
+    (solution, arrayBuilder.result)
   }
 }
 
@@ -352,7 +357,7 @@ class LinearSVCModel private[classification] (
     @Since("2.2.0") val coefficients: Vector,
     @Since("2.2.0") val intercept: Double)
   extends ClassificationModel[Vector, LinearSVCModel]
-  with LinearSVCParams with MLWritable {
+  with LinearSVCParams with MLWritable with HasTrainingSummary[LinearSVCTrainingSummary] {
 
   @Since("2.2.0")
   override val numClasses: Int = 2
@@ -362,12 +367,32 @@ class LinearSVCModel private[classification] (
 
   @Since("2.2.0")
   def setThreshold(value: Double): this.type = set(threshold, value)
-  setDefault(threshold, 0.0)
 
   private val margin: Vector => Double = (features) => {
     BLAS.dot(features, coefficients) + intercept
   }
 
+  /**
+   * Gets summary of model on training set. An exception is thrown
+   * if `hasSummary` is false.
+   */
+  @Since("3.1.0")
+  override def summary: LinearSVCTrainingSummary = super.summary
+
+  /**
+   * Evaluates the model on a test dataset.
+   *
+   * @param dataset Test dataset to evaluate model on.
+   */
+  @Since("3.1.0")
+  def evaluate(dataset: Dataset[_]): LinearSVCSummary = {
+    val weightColName = if (!isDefined(weightCol)) "weightCol" else $(weightCol)
+    // Handle possible missing or invalid rawPrediction or prediction columns
+    val (summaryModel, rawPrediction, predictionColName) = findSummaryModel()
+    new LinearSVCSummaryImpl(summaryModel.transform(dataset),
+      rawPrediction, predictionColName, $(labelCol), weightColName)
+  }
+
   override def predict(features: Vector): Double = {
     if (margin(features) > $(threshold)) 1.0 else 0.0
   }
@@ -439,3 +464,53 @@ object LinearSVCModel extends MLReadable[LinearSVCModel] {
     }
   }
 }
+
+/**
+ * Abstraction for LinearSVC results for a given model.
+ */
+sealed trait LinearSVCSummary extends BinaryClassificationSummary
+
+/**
+ * Abstraction for LinearSVC training results.
+ */
+sealed trait LinearSVCTrainingSummary extends LinearSVCSummary with TrainingSummary
+
+/**
+ * LinearSVC results for a given model.
+ *
+ * @param predictions dataframe output by the model's `transform` method.
+ * @param scoreCol field in "predictions" which gives the rawPrediction of each instance.
+ * @param predictionCol field in "predictions" which gives the prediction for a data instance as a
+ *                      double.
+ * @param labelCol field in "predictions" which gives the true label of each instance.
+ * @param weightCol field in "predictions" which gives the weight of each instance.
+ */
+private class LinearSVCSummaryImpl(
+    @transient override val predictions: DataFrame,
+    override val scoreCol: String,
+    override val predictionCol: String,
+    override val labelCol: String,
+    override val weightCol: String)
+  extends LinearSVCSummary
+
+/**
+ * LinearSVC training results.
+ *
+ * @param predictions dataframe output by the model's `transform` method.
+ * @param scoreCol field in "predictions" which gives the rawPrediction of each instance.
+ * @param predictionCol field in "predictions" which gives the prediction for a data instance as a
+ *                      double.
+ * @param labelCol field in "predictions" which gives the true label of each instance.
+ * @param weightCol field in "predictions" which gives the weight of each instance.
+ * @param objectiveHistory objective function (scaled loss + regularization) at each iteration.
+ */
+private class LinearSVCTrainingSummaryImpl(
+    predictions: DataFrame,
+    scoreCol: String,
+    predictionCol: String,
+    labelCol: String,
+    weightCol: String,
+    override val objectiveHistory: Array[Double])
+  extends LinearSVCSummaryImpl(
+    predictions, scoreCol, predictionCol, labelCol, weightCol)
+    with LinearSVCTrainingSummary
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
index c1dd677f08888..c3c54651bad7f 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
@@ -37,12 +37,10 @@ import org.apache.spark.ml.param.shared._
 import org.apache.spark.ml.stat._
 import org.apache.spark.ml.util._
 import org.apache.spark.ml.util.Instrumentation.instrumented
-import org.apache.spark.mllib.evaluation.{BinaryClassificationMetrics, MulticlassMetrics}
 import org.apache.spark.mllib.util.MLUtils
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{DataFrame, Dataset, Row}
-import org.apache.spark.sql.functions.col
-import org.apache.spark.sql.types.{DataType, DoubleType, StructType}
+import org.apache.spark.sql.types.{DataType, StructType}
 import org.apache.spark.storage.StorageLevel
 import org.apache.spark.util.VersionUtils
 
@@ -52,7 +50,7 @@ import org.apache.spark.util.VersionUtils
 private[classification] trait LogisticRegressionParams extends ProbabilisticClassifierParams
   with HasRegParam with HasElasticNetParam with HasMaxIter with HasFitIntercept with HasTol
   with HasStandardization with HasWeightCol with HasThreshold with HasAggregationDepth
-  with HasBlockSize {
+  with HasMaxBlockSizeInMB {
 
   import org.apache.spark.ml.classification.LogisticRegression.supportedFamilyNames
 
@@ -245,6 +243,10 @@ private[classification] trait LogisticRegressionParams extends ProbabilisticClas
   @Since("2.2.0")
   def getUpperBoundsOnIntercepts: Vector = $(upperBoundsOnIntercepts)
 
+  setDefault(regParam -> 0.0, elasticNetParam -> 0.0, maxIter -> 100, tol -> 1E-6,
+    fitIntercept -> true, family -> "auto", standardization -> true, threshold -> 0.5,
+    aggregationDepth -> 2, maxBlockSizeInMB -> 0.0)
+
   protected def usingBoundConstrainedOptimization: Boolean = {
     isSet(lowerBoundsOnCoefficients) || isSet(upperBoundsOnCoefficients) ||
       isSet(lowerBoundsOnIntercepts) || isSet(upperBoundsOnIntercepts)
@@ -274,6 +276,10 @@ private[classification] trait LogisticRegressionParams extends ProbabilisticClas
  *
  * This class supports fitting traditional logistic regression model by LBFGS/OWLQN and
  * bound (box) constrained logistic regression model by LBFGSB.
+ *
+ * Since 3.1.0, it supports stacking instances into blocks and using GEMV/GEMM for
+ * better performance.
+ * The block size will be 1.0 MB, if param maxBlockSizeInMB is set 0.0 by default.
  */
 @Since("1.2.0")
 class LogisticRegression @Since("1.2.0") (
@@ -292,7 +298,6 @@ class LogisticRegression @Since("1.2.0") (
    */
   @Since("1.2.0")
   def setRegParam(value: Double): this.type = set(regParam, value)
-  setDefault(regParam -> 0.0)
 
   /**
    * Set the ElasticNet mixing parameter.
@@ -308,7 +313,6 @@ class LogisticRegression @Since("1.2.0") (
    */
   @Since("1.4.0")
   def setElasticNetParam(value: Double): this.type = set(elasticNetParam, value)
-  setDefault(elasticNetParam -> 0.0)
 
   /**
    * Set the maximum number of iterations.
@@ -318,7 +322,6 @@ class LogisticRegression @Since("1.2.0") (
    */
   @Since("1.2.0")
   def setMaxIter(value: Int): this.type = set(maxIter, value)
-  setDefault(maxIter -> 100)
 
   /**
    * Set the convergence tolerance of iterations.
@@ -329,7 +332,6 @@ class LogisticRegression @Since("1.2.0") (
    */
   @Since("1.4.0")
   def setTol(value: Double): this.type = set(tol, value)
-  setDefault(tol -> 1E-6)
 
   /**
    * Whether to fit an intercept term.
@@ -339,7 +341,6 @@ class LogisticRegression @Since("1.2.0") (
    */
   @Since("1.4.0")
   def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
-  setDefault(fitIntercept -> true)
 
   /**
    * Sets the value of param [[family]].
@@ -349,7 +350,6 @@ class LogisticRegression @Since("1.2.0") (
    */
   @Since("2.1.0")
   def setFamily(value: String): this.type = set(family, value)
-  setDefault(family -> "auto")
 
   /**
    * Whether to standardize the training features before fitting the model.
@@ -363,11 +363,9 @@ class LogisticRegression @Since("1.2.0") (
    */
   @Since("1.5.0")
   def setStandardization(value: Boolean): this.type = set(standardization, value)
-  setDefault(standardization -> true)
 
   @Since("1.5.0")
   override def setThreshold(value: Double): this.type = super.setThreshold(value)
-  setDefault(threshold -> 0.5)
 
   @Since("1.5.0")
   override def getThreshold: Double = super.getThreshold
@@ -398,7 +396,6 @@ class LogisticRegression @Since("1.2.0") (
    */
   @Since("2.1.0")
   def setAggregationDepth(value: Int): this.type = set(aggregationDepth, value)
-  setDefault(aggregationDepth -> 2)
 
   /**
    * Set the lower bounds on coefficients if fitting under bound constrained optimization.
@@ -433,23 +430,13 @@ class LogisticRegression @Since("1.2.0") (
   def setUpperBoundsOnIntercepts(value: Vector): this.type = set(upperBoundsOnIntercepts, value)
 
   /**
-   * Set block size for stacking input data in matrices.
-   * If blockSize == 1, then stacking will be skipped, and each vector is treated individually;
-   * If blockSize &gt; 1, then vectors will be stacked to blocks, and high-level BLAS routines
-   * will be used if possible (for example, GEMV instead of DOT, GEMM instead of GEMV).
-   * Recommended size is between 10 and 1000. An appropriate choice of the block size depends
-   * on the sparsity and dim of input datasets, the underlying BLAS implementation (for example,
-   * f2jBLAS, OpenBLAS, intel MKL) and its configuration (for example, number of threads).
-   * Note that existing BLAS implementations are mainly optimized for dense matrices, if the
-   * input dataset is sparse, stacking may bring no performance gain, the worse is possible
-   * performance regression.
-   * Default is 1.
+   * Sets the value of param [[maxBlockSizeInMB]].
+   * Default is 0.0, then 1.0 MB will be chosen.
    *
    * @group expertSetParam
    */
   @Since("3.1.0")
-  def setBlockSize(value: Int): this.type = set(blockSize, value)
-  setDefault(blockSize -> 1)
+  def setMaxBlockSizeInMB(value: Double): this.type = set(maxBlockSizeInMB, value)
 
   private def assertBoundConstrainedOptimizationParamsValid(
       numCoefficientSets: Int,
@@ -503,31 +490,24 @@ class LogisticRegression @Since("1.2.0") (
     this
   }
 
-  override protected[spark] def train(dataset: Dataset[_]): LogisticRegressionModel = {
-    val handlePersistence = dataset.storageLevel == StorageLevel.NONE
-    train(dataset, handlePersistence)
-  }
-
   protected[spark] def train(
-      dataset: Dataset[_],
-      handlePersistence: Boolean): LogisticRegressionModel = instrumented { instr =>
+      dataset: Dataset[_]): LogisticRegressionModel = instrumented { instr =>
     instr.logPipelineStage(this)
     instr.logDataset(dataset)
     instr.logParams(this, labelCol, weightCol, featuresCol, predictionCol, rawPredictionCol,
-      probabilityCol, regParam, elasticNetParam, standardization, threshold, maxIter, tol,
-      fitIntercept, blockSize)
+      probabilityCol, regParam, elasticNetParam, standardization, threshold, thresholds, maxIter,
+      tol, fitIntercept, maxBlockSizeInMB)
+
+    if (dataset.storageLevel != StorageLevel.NONE) {
+      instr.logWarning(s"Input instances will be standardized, blockified to blocks, and " +
+        s"then cached during training. Be careful of double caching!")
+    }
 
     val instances = extractInstances(dataset)
       .setName("training instances")
 
-    if (handlePersistence && $(blockSize) == 1) {
-      instances.persist(StorageLevel.MEMORY_AND_DISK)
-    }
-
-    var requestedMetrics = Seq("mean", "std", "count")
-    if ($(blockSize) != 1) requestedMetrics +:= "numNonZeros"
     val (summarizer, labelSummarizer) = Summarizer
-      .getClassificationSummarizers(instances, $(aggregationDepth), requestedMetrics)
+      .getClassificationSummarizers(instances, $(aggregationDepth), Seq("mean", "std", "count"))
 
     val numFeatures = summarizer.mean.size
     val histogram = labelSummarizer.histogram
@@ -555,14 +535,13 @@ class LogisticRegression @Since("1.2.0") (
     instr.logNamedValue("lowestLabelWeight", labelSummarizer.histogram.min.toString)
     instr.logNamedValue("highestLabelWeight", labelSummarizer.histogram.max.toString)
     instr.logSumOfWeights(summarizer.weightSum)
-    if ($(blockSize) > 1) {
-      val scale = 1.0 / summarizer.count / numFeatures
-      val sparsity = 1 - summarizer.numNonzeros.toArray.map(_ * scale).sum
-      instr.logNamedValue("sparsity", sparsity.toString)
-      if (sparsity > 0.5) {
-        instr.logWarning(s"sparsity of input dataset is $sparsity, " +
-          s"which may hurt performance in high-level BLAS.")
-      }
+
+    var actualBlockSizeInMB = $(maxBlockSizeInMB)
+    if (actualBlockSizeInMB == 0) {
+      // TODO: for Multinomial logistic regression, take numClasses into account
+      actualBlockSizeInMB = InstanceBlock.DefaultBlockSizeInMB
+      require(actualBlockSizeInMB > 0, "inferred actual BlockSizeInMB must > 0")
+      instr.logNamedValue("actualBlockSizeInMB", actualBlockSizeInMB.toString)
     }
 
     val isMultinomial = checkMultinomial(numClasses)
@@ -592,8 +571,7 @@ class LogisticRegression @Since("1.2.0") (
       } else {
         Vectors.dense(if (numClasses == 2) Double.PositiveInfinity else Double.NegativeInfinity)
       }
-      if (instances.getStorageLevel != StorageLevel.NONE) instances.unpersist()
-      return createModel(dataset, numClasses, coefMatrix, interceptVec, Array.empty)
+      return createModel(dataset, numClasses, coefMatrix, interceptVec, Array(0.0))
     }
 
     if (!$(fitIntercept) && isConstantLabel) {
@@ -630,7 +608,7 @@ class LogisticRegression @Since("1.2.0") (
       The coefficients are laid out in column major order during training. Here we initialize
       a column major matrix of initial coefficients.
      */
-    val initialCoefWithInterceptMatrix = createInitCoefWithInterceptMatrix(
+    val initialSolution = createInitialSolution(
       numClasses, numFeatures, histogram, featuresStd, lowerBounds, upperBounds, instr)
 
     /*
@@ -644,14 +622,9 @@ class LogisticRegression @Since("1.2.0") (
        Note that the intercept in scaled space and original space is the same;
        as a result, no scaling is needed.
      */
-    val (allCoefficients, objectiveHistory) = if ($(blockSize) == 1) {
-      trainOnRows(instances, featuresStd, numClasses, initialCoefWithInterceptMatrix,
-        regularization, optimizer)
-    } else {
-      trainOnBlocks(instances, featuresStd, numClasses, initialCoefWithInterceptMatrix,
-        regularization, optimizer)
-    }
-    if (instances.getStorageLevel != StorageLevel.NONE) instances.unpersist()
+    val (allCoefficients, objectiveHistory) =
+      trainImpl(instances, actualBlockSizeInMB, featuresStd, featuresMean, numClasses,
+        initialSolution.toArray, regularization, optimizer)
 
     if (allCoefficients == null) {
       val msg = s"${optimizer.getClass.getName} failed."
@@ -729,6 +702,7 @@ class LogisticRegression @Since("1.2.0") (
       objectiveHistory: Array[Double]): LogisticRegressionModel = {
     val model = copyValues(new LogisticRegressionModel(uid, coefficientMatrix, interceptVector,
       numClasses, checkMultinomial(numClasses)))
+    val weightColName = if (!isDefined(weightCol)) "weightCol" else $(weightCol)
 
     val (summaryModel, probabilityColName, predictionColName) = model.findSummaryModel()
     val logRegSummary = if (numClasses <= 2) {
@@ -738,6 +712,7 @@ class LogisticRegression @Since("1.2.0") (
         predictionColName,
         $(labelCol),
         $(featuresCol),
+        weightColName,
         objectiveHistory)
     } else {
       new LogisticRegressionTrainingSummaryImpl(
@@ -746,6 +721,7 @@ class LogisticRegression @Since("1.2.0") (
         predictionColName,
         $(labelCol),
         $(featuresCol),
+        weightColName,
         objectiveHistory)
     }
     model.setSummary(Some(logRegSummary))
@@ -841,7 +817,7 @@ class LogisticRegression @Since("1.2.0") (
    * The coefficients are laid out in column major order during training. Here we initialize
    * a column major matrix of initial coefficients.
    */
-  private def createInitCoefWithInterceptMatrix(
+  private def createInitialSolution(
       numClasses: Int,
       numFeatures: Int,
       histogram: Array[Double],
@@ -954,64 +930,72 @@ class LogisticRegression @Since("1.2.0") (
     initialCoefWithInterceptMatrix
   }
 
-  private def trainOnRows(
+  private def trainImpl(
       instances: RDD[Instance],
+      actualBlockSizeInMB: Double,
       featuresStd: Array[Double],
+      featuresMean: Array[Double],
       numClasses: Int,
-      initialCoefWithInterceptMatrix: Matrix,
+      initialSolution: Array[Double],
       regularization: Option[L2Regularization],
       optimizer: FirstOrderMinimizer[BDV[Double], DiffFunction[BDV[Double]]]) = {
-    val bcFeaturesStd = instances.context.broadcast(featuresStd)
-    val getAggregatorFunc = new LogisticAggregator(bcFeaturesStd, numClasses, $(fitIntercept),
-      checkMultinomial(numClasses))(_)
+    val multinomial = checkMultinomial(numClasses)
+
+    // for LR, we can center the input vector, if and only if:
+    // 1, fitIntercept is true;
+    // 2, no penalty on the intercept, which is always true in existing impl;
+    // 3, no bounds on the intercept.
+    val fitWithMean = $(fitIntercept) &&
+      (!isSet(lowerBoundsOnIntercepts) ||
+        $(lowerBoundsOnIntercepts).toArray.forall(_.isNegInfinity)) &&
+      (!isSet(upperBoundsOnIntercepts) ||
+        $(upperBoundsOnIntercepts).toArray.forall(_.isPosInfinity))
 
-    val costFun = new RDDLossFunction(instances, getAggregatorFunc,
-      regularization, $(aggregationDepth))
-    val states = optimizer.iterations(new CachedDiffFunction(costFun),
-      new BDV[Double](initialCoefWithInterceptMatrix.toArray))
-
-    /*
-       Note that in Logistic Regression, the objective history (loss + regularization)
-       is log-likelihood which is invariant under feature standardization. As a result,
-       the objective history from optimizer is the same as the one in the original space.
-     */
-    val arrayBuilder = mutable.ArrayBuilder.make[Double]
-    var state: optimizer.State = null
-    while (states.hasNext) {
-      state = states.next()
-      arrayBuilder += state.adjustedValue
-    }
-    bcFeaturesStd.destroy()
-
-    (if (state == null) null else state.x.toArray, arrayBuilder.result)
-  }
-
-  private def trainOnBlocks(
-      instances: RDD[Instance],
-      featuresStd: Array[Double],
-      numClasses: Int,
-      initialCoefWithInterceptMatrix: Matrix,
-      regularization: Option[L2Regularization],
-      optimizer: FirstOrderMinimizer[BDV[Double], DiffFunction[BDV[Double]]]) = {
     val numFeatures = featuresStd.length
-    val bcFeaturesStd = instances.context.broadcast(featuresStd)
+    val inverseStd = featuresStd.map(std => if (std != 0) 1.0 / std else 0.0)
+    val scaledMean = Array.tabulate(numFeatures)(i => inverseStd(i) * featuresMean(i))
+    val bcInverseStd = instances.context.broadcast(inverseStd)
+    val bcScaledMean = instances.context.broadcast(scaledMean)
 
-    val standardized = instances.mapPartitions { iter =>
-      val inverseStd = bcFeaturesStd.value.map { std => if (std != 0) 1.0 / std else 0.0 }
-      val func = StandardScalerModel.getTransformFunc(Array.empty, inverseStd, false, true)
+    val scaled = instances.mapPartitions { iter =>
+      val func = StandardScalerModel.getTransformFunc(Array.empty, bcInverseStd.value, false, true)
       iter.map { case Instance(label, weight, vec) => Instance(label, weight, func(vec)) }
     }
-    val blocks = InstanceBlock.blokify(standardized, $(blockSize))
+
+    val maxMemUsage = (actualBlockSizeInMB * 1024L * 1024L).ceil.toLong
+    val blocks = InstanceBlock.blokifyWithMaxMemUsage(scaled, maxMemUsage)
       .persist(StorageLevel.MEMORY_AND_DISK)
-      .setName(s"training blocks (blockSize=${$(blockSize)})")
+      .setName(s"$uid: training blocks (blockSizeInMB=$actualBlockSizeInMB)")
+
+    val costFun = if (multinomial) {
+      val getAggregatorFunc = new MultinomialLogisticBlockAggregator(bcInverseStd, bcScaledMean,
+         $(fitIntercept), fitWithMean)(_)
+      new RDDLossFunction(blocks, getAggregatorFunc, regularization, $(aggregationDepth))
+    } else {
+      val getAggregatorFunc = new BinaryLogisticBlockAggregator(bcInverseStd, bcScaledMean,
+         $(fitIntercept), fitWithMean)(_)
+      new RDDLossFunction(blocks, getAggregatorFunc, regularization, $(aggregationDepth))
+    }
 
-    val getAggregatorFunc = new BlockLogisticAggregator(numFeatures, numClasses, $(fitIntercept),
-      checkMultinomial(numClasses))(_)
+    if (fitWithMean) {
+      if (multinomial) {
+        val adapt = Array.ofDim[Double](numClasses)
+        BLAS.javaBLAS.dgemv("N", numClasses, numFeatures, 1.0,
+          initialSolution, numClasses, scaledMean, 1, 0.0, adapt, 1)
+        BLAS.javaBLAS.daxpy(numClasses, 1.0, adapt, 0, 1,
+          initialSolution, numClasses * numFeatures, 1)
+      } else {
+        // original `initialSolution` is for problem:
+        // y = f(w1 * x1 / std_x1, w2 * x2 / std_x2, ..., intercept)
+        // we should adjust it to the initial solution for problem:
+        // y = f(w1 * (x1 - avg_x1) / std_x1, w2 * (x2 - avg_x2) / std_x2, ..., intercept)
+        val adapt = BLAS.javaBLAS.ddot(numFeatures, initialSolution, 1, scaledMean, 1)
+        initialSolution(numFeatures) += adapt
+      }
+    }
 
-    val costFun = new RDDLossFunction(blocks, getAggregatorFunc,
-      regularization, $(aggregationDepth))
     val states = optimizer.iterations(new CachedDiffFunction(costFun),
-      new BDV[Double](initialCoefWithInterceptMatrix.toArray))
+      new BDV[Double](initialSolution))
 
     /*
        Note that in Logistic Regression, the objective history (loss + regularization)
@@ -1025,9 +1009,27 @@ class LogisticRegression @Since("1.2.0") (
       arrayBuilder += state.adjustedValue
     }
     blocks.unpersist()
-    bcFeaturesStd.destroy()
-
-    (if (state == null) null else state.x.toArray, arrayBuilder.result)
+    bcInverseStd.destroy()
+    bcScaledMean.destroy()
+
+    val solution = if (state == null) null else state.x.toArray
+    if (fitWithMean && solution != null) {
+      if (multinomial) {
+        val adapt = Array.ofDim[Double](numClasses)
+        BLAS.javaBLAS.dgemv("N", numClasses, numFeatures, 1.0,
+          solution, numClasses, scaledMean, 1, 0.0, adapt, 1)
+        BLAS.javaBLAS.daxpy(numClasses, -1.0, adapt, 0, 1,
+          solution, numClasses * numFeatures, 1)
+      } else {
+        // the final solution is for problem:
+        // y = f(w1 * (x1 - avg_x1) / std_x1, w2 * (x2 - avg_x2) / std_x2, ..., intercept)
+        // we should adjust it back for original problem:
+        // y = f(w1 * x1 / std_x1, w2 * x2 / std_x2, ..., intercept)
+        val adapt = BLAS.javaBLAS.ddot(numFeatures, solution, 1, scaledMean, 1)
+        solution(numFeatures) -= adapt
+      }
+    }
+    (solution, arrayBuilder.result)
   }
 
   @Since("1.4.0")
@@ -1103,16 +1105,46 @@ class LogisticRegressionModel private[spark] (
     _intercept
   }
 
-  private lazy val _intercept = interceptVector.toArray.head
+  private lazy val _intercept = interceptVector(0)
+  private lazy val _interceptVector = interceptVector.toDense
+  private lazy val _binaryThresholdArray = {
+    val array = Array(Double.NaN, Double.NaN)
+    updateBinaryThresholds(array)
+    array
+  }
+  private def _threshold: Double = _binaryThresholdArray(0)
+  private def _rawThreshold: Double = _binaryThresholdArray(1)
+
+  private def updateBinaryThresholds(array: Array[Double]): Unit = {
+    if (!isMultinomial) {
+      val _threshold = getThreshold
+      array(0) = _threshold
+      if (_threshold == 0.0) {
+        array(1) = Double.NegativeInfinity
+      } else if (_threshold == 1.0) {
+        array(1) = Double.PositiveInfinity
+      } else {
+        array(1) = math.log(_threshold / (1.0 - _threshold))
+      }
+    }
+  }
 
   @Since("1.5.0")
-  override def setThreshold(value: Double): this.type = super.setThreshold(value)
+  override def setThreshold(value: Double): this.type = {
+    super.setThreshold(value)
+    updateBinaryThresholds(_binaryThresholdArray)
+    this
+  }
 
   @Since("1.5.0")
   override def getThreshold: Double = super.getThreshold
 
   @Since("1.5.0")
-  override def setThresholds(value: Array[Double]): this.type = super.setThresholds(value)
+  override def setThresholds(value: Array[Double]): this.type = {
+    super.setThresholds(value)
+    updateBinaryThresholds(_binaryThresholdArray)
+    this
+  }
 
   @Since("1.5.0")
   override def getThresholds: Array[Double] = super.getThresholds
@@ -1124,7 +1156,7 @@ class LogisticRegressionModel private[spark] (
 
   /** Margin (rawPrediction) for each class label. */
   private val margins: Vector => Vector = (features) => {
-    val m = interceptVector.toDense.copy
+    val m = _interceptVector.copy
     BLAS.gemv(1.0, coefficientMatrix, features, 1.0, m)
     m
   }
@@ -1157,27 +1189,6 @@ class LogisticRegressionModel private[spark] (
         s"(numClasses=${numClasses}), use summary instead.")
   }
 
-  /**
-   * If the probability and prediction columns are set, this method returns the current model,
-   * otherwise it generates new columns for them and sets them as columns on a new copy of
-   * the current model
-   */
-  private[classification] def findSummaryModel():
-      (LogisticRegressionModel, String, String) = {
-    val model = if ($(probabilityCol).isEmpty && $(predictionCol).isEmpty) {
-      copy(ParamMap.empty)
-        .setProbabilityCol("probability_" + java.util.UUID.randomUUID.toString)
-        .setPredictionCol("prediction_" + java.util.UUID.randomUUID.toString)
-    } else if ($(probabilityCol).isEmpty) {
-      copy(ParamMap.empty).setProbabilityCol("probability_" + java.util.UUID.randomUUID.toString)
-    } else if ($(predictionCol).isEmpty) {
-      copy(ParamMap.empty).setPredictionCol("prediction_" + java.util.UUID.randomUUID.toString)
-    } else {
-      this
-    }
-    (model, model.getProbabilityCol, model.getPredictionCol)
-  }
-
   /**
    * Evaluates the model on a test dataset.
    *
@@ -1185,14 +1196,15 @@ class LogisticRegressionModel private[spark] (
    */
   @Since("2.0.0")
   def evaluate(dataset: Dataset[_]): LogisticRegressionSummary = {
+    val weightColName = if (!isDefined(weightCol)) "weightCol" else $(weightCol)
     // Handle possible missing or invalid prediction columns
     val (summaryModel, probabilityColName, predictionColName) = findSummaryModel()
     if (numClasses > 2) {
       new LogisticRegressionSummaryImpl(summaryModel.transform(dataset),
-        probabilityColName, predictionColName, $(labelCol), $(featuresCol))
+        probabilityColName, predictionColName, $(labelCol), $(featuresCol), weightColName)
     } else {
       new BinaryLogisticRegressionSummaryImpl(summaryModel.transform(dataset),
-        probabilityColName, predictionColName, $(labelCol), $(featuresCol))
+        probabilityColName, predictionColName, $(labelCol), $(featuresCol), weightColName)
     }
   }
 
@@ -1203,52 +1215,43 @@ class LogisticRegressionModel private[spark] (
   override def predict(features: Vector): Double = if (isMultinomial) {
     super.predict(features)
   } else {
-    // Note: We should use getThreshold instead of $(threshold) since getThreshold is overridden.
-    if (score(features) > getThreshold) 1 else 0
+    // Note: We should use _threshold instead of $(threshold) since getThreshold is overridden.
+    if (score(features) > _threshold) 1 else 0
   }
 
   override protected def raw2probabilityInPlace(rawPrediction: Vector): Vector = {
     rawPrediction match {
       case dv: DenseVector =>
+        val values = dv.values
         if (isMultinomial) {
-          val size = dv.size
-          val values = dv.values
-
           // get the maximum margin
           val maxMarginIndex = rawPrediction.argmax
           val maxMargin = rawPrediction(maxMarginIndex)
 
           if (maxMargin == Double.PositiveInfinity) {
             var k = 0
-            while (k < size) {
+            while (k < numClasses) {
               values(k) = if (k == maxMarginIndex) 1.0 else 0.0
               k += 1
             }
           } else {
-            val sum = {
-              var temp = 0.0
-              var k = 0
-              while (k < numClasses) {
-                values(k) = if (maxMargin > 0) {
-                  math.exp(values(k) - maxMargin)
-                } else {
-                  math.exp(values(k))
-                }
-                temp += values(k)
-                k += 1
+            var sum = 0.0
+            var k = 0
+            while (k < numClasses) {
+              values(k) = if (maxMargin > 0) {
+                math.exp(values(k) - maxMargin)
+              } else {
+                math.exp(values(k))
               }
-              temp
+              sum += values(k)
+              k += 1
             }
             BLAS.scal(1 / sum, dv)
           }
           dv
         } else {
-          var i = 0
-          val size = dv.size
-          while (i < size) {
-            dv.values(i) = 1.0 / (1.0 + math.exp(-dv.values(i)))
-            i += 1
-          }
+          values(0) = 1.0 / (1.0 + math.exp(-values(0)))
+          values(1) = 1.0 - values(0)
           dv
         }
       case sv: SparseVector =>
@@ -1278,16 +1281,8 @@ class LogisticRegressionModel private[spark] (
     if (isMultinomial) {
       super.raw2prediction(rawPrediction)
     } else {
-      // Note: We should use getThreshold instead of $(threshold) since getThreshold is overridden.
-      val t = getThreshold
-      val rawThreshold = if (t == 0.0) {
-        Double.NegativeInfinity
-      } else if (t == 1.0) {
-        Double.PositiveInfinity
-      } else {
-        math.log(t / (1.0 - t))
-      }
-      if (rawPrediction(1) > rawThreshold) 1 else 0
+      // Note: We should use _threshold instead of $(threshold) since getThreshold is overridden.
+      if (rawPrediction(1) > _rawThreshold) 1.0 else 0.0
     }
   }
 
@@ -1295,8 +1290,8 @@ class LogisticRegressionModel private[spark] (
     if (isMultinomial) {
       super.probability2prediction(probability)
     } else {
-      // Note: We should use getThreshold instead of $(threshold) since getThreshold is overridden.
-      if (probability(1) > getThreshold) 1 else 0
+      // Note: We should use _threshold instead of $(threshold) since getThreshold is overridden.
+      if (probability(1) > _threshold) 1.0 else 0.0
     }
   }
 
@@ -1390,123 +1385,17 @@ object LogisticRegressionModel extends MLReadable[LogisticRegressionModel] {
 
 /**
  * Abstraction for logistic regression results for a given model.
- *
- * Currently, the summary ignores the instance weights.
  */
-sealed trait LogisticRegressionSummary extends Serializable {
-
-  /**
-   * Dataframe output by the model's `transform` method.
-   */
-  @Since("1.5.0")
-  def predictions: DataFrame
+sealed trait LogisticRegressionSummary extends ClassificationSummary {
 
   /** Field in "predictions" which gives the probability of each class as a vector. */
   @Since("1.5.0")
   def probabilityCol: String
 
-  /** Field in "predictions" which gives the prediction of each class. */
-  @Since("2.3.0")
-  def predictionCol: String
-
-  /** Field in "predictions" which gives the true label of each instance (if available). */
-  @Since("1.5.0")
-  def labelCol: String
-
   /** Field in "predictions" which gives the features of each instance as a vector. */
   @Since("1.6.0")
   def featuresCol: String
 
-  @transient private val multiclassMetrics = {
-    new MulticlassMetrics(
-      predictions.select(
-        col(predictionCol),
-        col(labelCol).cast(DoubleType))
-        .rdd.map { case Row(prediction: Double, label: Double) => (prediction, label) })
-  }
-
-  /**
-   * Returns the sequence of labels in ascending order. This order matches the order used
-   * in metrics which are specified as arrays over labels, e.g., truePositiveRateByLabel.
-   *
-   * Note: In most cases, it will be values {0.0, 1.0, ..., numClasses-1}, However, if the
-   * training set is missing a label, then all of the arrays over labels
-   * (e.g., from truePositiveRateByLabel) will be of length numClasses-1 instead of the
-   * expected numClasses.
-   */
-  @Since("2.3.0")
-  def labels: Array[Double] = multiclassMetrics.labels
-
-  /** Returns true positive rate for each label (category). */
-  @Since("2.3.0")
-  def truePositiveRateByLabel: Array[Double] = recallByLabel
-
-  /** Returns false positive rate for each label (category). */
-  @Since("2.3.0")
-  def falsePositiveRateByLabel: Array[Double] = {
-    multiclassMetrics.labels.map(label => multiclassMetrics.falsePositiveRate(label))
-  }
-
-  /** Returns precision for each label (category). */
-  @Since("2.3.0")
-  def precisionByLabel: Array[Double] = {
-    multiclassMetrics.labels.map(label => multiclassMetrics.precision(label))
-  }
-
-  /** Returns recall for each label (category). */
-  @Since("2.3.0")
-  def recallByLabel: Array[Double] = {
-    multiclassMetrics.labels.map(label => multiclassMetrics.recall(label))
-  }
-
-  /** Returns f-measure for each label (category). */
-  @Since("2.3.0")
-  def fMeasureByLabel(beta: Double): Array[Double] = {
-    multiclassMetrics.labels.map(label => multiclassMetrics.fMeasure(label, beta))
-  }
-
-  /** Returns f1-measure for each label (category). */
-  @Since("2.3.0")
-  def fMeasureByLabel: Array[Double] = fMeasureByLabel(1.0)
-
-  /**
-   * Returns accuracy.
-   * (equals to the total number of correctly classified instances
-   * out of the total number of instances.)
-   */
-  @Since("2.3.0")
-  def accuracy: Double = multiclassMetrics.accuracy
-
-  /**
-   * Returns weighted true positive rate.
-   * (equals to precision, recall and f-measure)
-   */
-  @Since("2.3.0")
-  def weightedTruePositiveRate: Double = weightedRecall
-
-  /** Returns weighted false positive rate. */
-  @Since("2.3.0")
-  def weightedFalsePositiveRate: Double = multiclassMetrics.weightedFalsePositiveRate
-
-  /**
-   * Returns weighted averaged recall.
-   * (equals to precision, recall and f-measure)
-   */
-  @Since("2.3.0")
-  def weightedRecall: Double = multiclassMetrics.weightedRecall
-
-  /** Returns weighted averaged precision. */
-  @Since("2.3.0")
-  def weightedPrecision: Double = multiclassMetrics.weightedPrecision
-
-  /** Returns weighted averaged f-measure. */
-  @Since("2.3.0")
-  def weightedFMeasure(beta: Double): Double = multiclassMetrics.weightedFMeasure(beta)
-
-  /** Returns weighted averaged f1-measure. */
-  @Since("2.3.0")
-  def weightedFMeasure: Double = multiclassMetrics.weightedFMeasure(1.0)
-
   /**
    * Convenient method for casting to binary logistic regression summary.
    * This method will throw an Exception if the summary is not a binary summary.
@@ -1521,112 +1410,26 @@ sealed trait LogisticRegressionSummary extends Serializable {
 
 /**
  * Abstraction for multiclass logistic regression training results.
- * Currently, the training summary ignores the training weights except
- * for the objective trace.
  */
-sealed trait LogisticRegressionTrainingSummary extends LogisticRegressionSummary {
-
-  /** objective function (scaled loss + regularization) at each iteration. */
-  @Since("1.5.0")
-  def objectiveHistory: Array[Double]
-
-  /** Number of training iterations. */
-  @Since("1.5.0")
-  def totalIterations: Int = objectiveHistory.length
-
+sealed trait LogisticRegressionTrainingSummary extends LogisticRegressionSummary
+  with TrainingSummary {
 }
 
 /**
  * Abstraction for binary logistic regression results for a given model.
- *
- * Currently, the summary ignores the instance weights.
  */
-sealed trait BinaryLogisticRegressionSummary extends LogisticRegressionSummary {
-
-  private val sparkSession = predictions.sparkSession
-  import sparkSession.implicits._
-
-  // TODO: Allow the user to vary the number of bins using a setBins method in
-  // BinaryClassificationMetrics. For now the default is set to 100.
-  @transient private val binaryMetrics = new BinaryClassificationMetrics(
-    predictions.select(col(probabilityCol), col(labelCol).cast(DoubleType)).rdd.map {
-      case Row(score: Vector, label: Double) => (score(1), label)
-    }, 100
-  )
-
-  /**
-   * Returns the receiver operating characteristic (ROC) curve,
-   * which is a Dataframe having two fields (FPR, TPR)
-   * with (0.0, 0.0) prepended and (1.0, 1.0) appended to it.
-   * See http://en.wikipedia.org/wiki/Receiver_operating_characteristic
-   *
-   * @note This ignores instance weights (setting all to 1.0) from `LogisticRegression.weightCol`.
-   * This will change in later Spark versions.
-   */
-  @Since("1.5.0")
-  @transient lazy val roc: DataFrame = binaryMetrics.roc().toDF("FPR", "TPR")
-
-  /**
-   * Computes the area under the receiver operating characteristic (ROC) curve.
-   *
-   * @note This ignores instance weights (setting all to 1.0) from `LogisticRegression.weightCol`.
-   * This will change in later Spark versions.
-   */
-  @Since("1.5.0")
-  lazy val areaUnderROC: Double = binaryMetrics.areaUnderROC()
-
-  /**
-   * Returns the precision-recall curve, which is a Dataframe containing
-   * two fields recall, precision with (0.0, 1.0) prepended to it.
-   *
-   * @note This ignores instance weights (setting all to 1.0) from `LogisticRegression.weightCol`.
-   * This will change in later Spark versions.
-   */
-  @Since("1.5.0")
-  @transient lazy val pr: DataFrame = binaryMetrics.pr().toDF("recall", "precision")
-
-  /**
-   * Returns a dataframe with two fields (threshold, F-Measure) curve with beta = 1.0.
-   *
-   * @note This ignores instance weights (setting all to 1.0) from `LogisticRegression.weightCol`.
-   * This will change in later Spark versions.
-   */
-  @Since("1.5.0")
-  @transient lazy val fMeasureByThreshold: DataFrame = {
-    binaryMetrics.fMeasureByThreshold().toDF("threshold", "F-Measure")
-  }
-
-  /**
-   * Returns a dataframe with two fields (threshold, precision) curve.
-   * Every possible probability obtained in transforming the dataset are used
-   * as thresholds used in calculating the precision.
-   *
-   * @note This ignores instance weights (setting all to 1.0) from `LogisticRegression.weightCol`.
-   * This will change in later Spark versions.
-   */
-  @Since("1.5.0")
-  @transient lazy val precisionByThreshold: DataFrame = {
-    binaryMetrics.precisionByThreshold().toDF("threshold", "precision")
-  }
+sealed trait BinaryLogisticRegressionSummary extends LogisticRegressionSummary
+  with BinaryClassificationSummary {
 
-  /**
-   * Returns a dataframe with two fields (threshold, recall) curve.
-   * Every possible probability obtained in transforming the dataset are used
-   * as thresholds used in calculating the recall.
-   *
-   * @note This ignores instance weights (setting all to 1.0) from `LogisticRegression.weightCol`.
-   * This will change in later Spark versions.
-   */
-  @Since("1.5.0")
-  @transient lazy val recallByThreshold: DataFrame = {
-    binaryMetrics.recallByThreshold().toDF("threshold", "recall")
+  override def scoreCol: String = if (probabilityCol.nonEmpty) {
+    probabilityCol
+  } else {
+    throw new SparkException("probabilityCol is required for BinaryLogisticRegressionSummary.")
   }
 }
 
 /**
  * Abstraction for binary logistic regression training results.
- * Currently, the training summary ignores the training weights except
- * for the objective trace.
  */
 sealed trait BinaryLogisticRegressionTrainingSummary extends BinaryLogisticRegressionSummary
   with LogisticRegressionTrainingSummary
@@ -1641,6 +1444,7 @@ sealed trait BinaryLogisticRegressionTrainingSummary extends BinaryLogisticRegre
  *                      double.
  * @param labelCol field in "predictions" which gives the true label of each instance.
  * @param featuresCol field in "predictions" which gives the features of each instance as a vector.
+ * @param weightCol field in "predictions" which gives the weight of each instance.
  * @param objectiveHistory objective function (scaled loss + regularization) at each iteration.
  */
 private class LogisticRegressionTrainingSummaryImpl(
@@ -1649,9 +1453,10 @@ private class LogisticRegressionTrainingSummaryImpl(
     predictionCol: String,
     labelCol: String,
     featuresCol: String,
+    weightCol: String,
     override val objectiveHistory: Array[Double])
   extends LogisticRegressionSummaryImpl(
-    predictions, probabilityCol, predictionCol, labelCol, featuresCol)
+    predictions, probabilityCol, predictionCol, labelCol, featuresCol, weightCol)
   with LogisticRegressionTrainingSummary
 
 /**
@@ -1659,18 +1464,20 @@ private class LogisticRegressionTrainingSummaryImpl(
  *
  * @param predictions dataframe output by the model's `transform` method.
  * @param probabilityCol field in "predictions" which gives the probability of
- *                       each class as a vector.
+ *                 each class as a vector.
  * @param predictionCol field in "predictions" which gives the prediction for a data instance as a
  *                      double.
  * @param labelCol field in "predictions" which gives the true label of each instance.
  * @param featuresCol field in "predictions" which gives the features of each instance as a vector.
+ * @param weightCol field in "predictions" which gives the weight of each instance.
  */
 private class LogisticRegressionSummaryImpl(
     @transient override val predictions: DataFrame,
     override val probabilityCol: String,
     override val predictionCol: String,
     override val labelCol: String,
-    override val featuresCol: String)
+    override val featuresCol: String,
+    override val weightCol: String)
   extends LogisticRegressionSummary
 
 /**
@@ -1683,6 +1490,7 @@ private class LogisticRegressionSummaryImpl(
  *                      double.
  * @param labelCol field in "predictions" which gives the true label of each instance.
  * @param featuresCol field in "predictions" which gives the features of each instance as a vector.
+ * @param weightCol field in "predictions" which gives the weight of each instance.
  * @param objectiveHistory objective function (scaled loss + regularization) at each iteration.
  */
 private class BinaryLogisticRegressionTrainingSummaryImpl(
@@ -1691,9 +1499,10 @@ private class BinaryLogisticRegressionTrainingSummaryImpl(
     predictionCol: String,
     labelCol: String,
     featuresCol: String,
+    weightCol: String,
     override val objectiveHistory: Array[Double])
   extends BinaryLogisticRegressionSummaryImpl(
-    predictions, probabilityCol, predictionCol, labelCol, featuresCol)
+    predictions, probabilityCol, predictionCol, labelCol, featuresCol, weightCol)
   with BinaryLogisticRegressionTrainingSummary
 
 /**
@@ -1706,13 +1515,15 @@ private class BinaryLogisticRegressionTrainingSummaryImpl(
  *                      each class as a double.
  * @param labelCol field in "predictions" which gives the true label of each instance.
  * @param featuresCol field in "predictions" which gives the features of each instance as a vector.
+ * @param weightCol field in "predictions" which gives the weight of each instance.
  */
 private class BinaryLogisticRegressionSummaryImpl(
     predictions: DataFrame,
     probabilityCol: String,
     predictionCol: String,
     labelCol: String,
-    featuresCol: String)
+    featuresCol: String,
+    weightCol: String)
   extends LogisticRegressionSummaryImpl(
-    predictions, probabilityCol, predictionCol, labelCol, featuresCol)
+    predictions, probabilityCol, predictionCol, labelCol, featuresCol, weightCol)
   with BinaryLogisticRegressionSummary
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala
index 6bffc372b68fe..58fc53517c9b0 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala
@@ -27,7 +27,7 @@ import org.apache.spark.ml.param._
 import org.apache.spark.ml.param.shared._
 import org.apache.spark.ml.util._
 import org.apache.spark.ml.util.Instrumentation.instrumented
-import org.apache.spark.sql.{Dataset, Row}
+import org.apache.spark.sql._
 import org.apache.spark.util.VersionUtils.majorMinorVersion
 
 /** Params for Multilayer Perceptron. */
@@ -185,7 +185,7 @@ class MultilayerPerceptronClassifier @Since("1.5.0") (
     instr.logPipelineStage(this)
     instr.logDataset(dataset)
     instr.logParams(this, labelCol, featuresCol, predictionCol, rawPredictionCol, layers, maxIter,
-      tol, blockSize, solver, stepSize, seed)
+      tol, blockSize, solver, stepSize, seed, thresholds)
 
     val myLayers = $(layers)
     val labels = myLayers.last
@@ -225,8 +225,24 @@ class MultilayerPerceptronClassifier @Since("1.5.0") (
         s"The solver $solver is not supported by MultilayerPerceptronClassifier.")
     }
     trainer.setStackSize($(blockSize))
-    val mlpModel = trainer.train(data)
-    new MultilayerPerceptronClassificationModel(uid, mlpModel.weights)
+    val (mlpModel, objectiveHistory) = trainer.train(data)
+    createModel(dataset, mlpModel.weights, objectiveHistory)
+  }
+
+  private def createModel(
+      dataset: Dataset[_],
+      weights: Vector,
+      objectiveHistory: Array[Double]): MultilayerPerceptronClassificationModel = {
+    val model = copyValues(new MultilayerPerceptronClassificationModel(uid, weights))
+
+    val (summaryModel, _, predictionColName) = model.findSummaryModel()
+    val summary = new MultilayerPerceptronClassificationTrainingSummaryImpl(
+      summaryModel.transform(dataset),
+      predictionColName,
+      $(labelCol),
+      "",
+      objectiveHistory)
+    model.setSummary(Some(summary))
   }
 }
 
@@ -259,7 +275,8 @@ class MultilayerPerceptronClassificationModel private[ml] (
     @Since("1.5.0") override val uid: String,
     @Since("2.0.0") val weights: Vector)
   extends ProbabilisticClassificationModel[Vector, MultilayerPerceptronClassificationModel]
-  with MultilayerPerceptronParams with Serializable with MLWritable {
+  with MultilayerPerceptronParams with Serializable with MLWritable
+  with HasTrainingSummary[MultilayerPerceptronClassificationTrainingSummary]{
 
   @Since("1.6.0")
   override lazy val numFeatures: Int = $(layers).head
@@ -268,6 +285,26 @@ class MultilayerPerceptronClassificationModel private[ml] (
     .multiLayerPerceptron($(layers), softmaxOnTop = true)
     .model(weights)
 
+  /**
+   * Gets summary of model on training set. An exception is thrown
+   * if `hasSummary` is false.
+   */
+  @Since("3.1.0")
+  override def summary: MultilayerPerceptronClassificationTrainingSummary = super.summary
+
+  /**
+   * Evaluates the model on a test dataset.
+   *
+   * @param dataset Test dataset to evaluate model on.
+   */
+  @Since("3.1.0")
+  def evaluate(dataset: Dataset[_]): MultilayerPerceptronClassificationSummary = {
+    // Handle possible missing or invalid probability or prediction columns
+    val (summaryModel, _, predictionColName) = findSummaryModel()
+    new MultilayerPerceptronClassificationSummaryImpl(summaryModel.transform(dataset),
+      predictionColName, $(labelCol), "")
+  }
+
   /**
    * Predict label for the given features.
    * This internal method is used to implement `transform()` and output [[predictionCol]].
@@ -359,3 +396,51 @@ object MultilayerPerceptronClassificationModel
     }
   }
 }
+
+
+/**
+ * Abstraction for MultilayerPerceptronClassification results for a given model.
+ */
+sealed trait MultilayerPerceptronClassificationSummary extends ClassificationSummary
+
+/**
+ * Abstraction for MultilayerPerceptronClassification training results.
+ */
+sealed trait MultilayerPerceptronClassificationTrainingSummary
+  extends MultilayerPerceptronClassificationSummary with TrainingSummary
+
+/**
+ * MultilayerPerceptronClassification training results.
+ *
+ * @param predictions dataframe output by the model's `transform` method.
+ * @param predictionCol field in "predictions" which gives the prediction for a data instance as a
+ *                      double.
+ * @param labelCol field in "predictions" which gives the true label of each instance.
+ * @param weightCol field in "predictions" which gives the weight of each instance.
+ * @param objectiveHistory objective function (scaled loss + regularization) at each iteration.
+ */
+private class MultilayerPerceptronClassificationTrainingSummaryImpl(
+    predictions: DataFrame,
+    predictionCol: String,
+    labelCol: String,
+    weightCol: String,
+    override val objectiveHistory: Array[Double])
+  extends MultilayerPerceptronClassificationSummaryImpl(
+    predictions, predictionCol, labelCol, weightCol)
+    with MultilayerPerceptronClassificationTrainingSummary
+
+/**
+ * MultilayerPerceptronClassification results for a given model.
+ *
+ * @param predictions dataframe output by the model's `transform` method.
+ * @param predictionCol field in "predictions" which gives the prediction for a data instance as a
+ *                      double.
+ * @param labelCol field in "predictions" which gives the true label of each instance.
+ * @param weightCol field in "predictions" which gives the weight of each instance.
+ */
+private class MultilayerPerceptronClassificationSummaryImpl(
+    @transient override val predictions: DataFrame,
+    override val predictionCol: String,
+    override val labelCol: String,
+    override val weightCol: String)
+  extends MultilayerPerceptronClassificationSummary
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala
index 5459a0fab9135..6b1537bcc5069 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala
@@ -22,6 +22,7 @@ import org.json4s.DefaultFormats
 
 import org.apache.spark.annotation.Since
 import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.functions.checkNonNegativeWeight
 import org.apache.spark.ml.linalg._
 import org.apache.spark.ml.param.{DoubleParam, Param, ParamMap, ParamValidators}
 import org.apache.spark.ml.param.shared.HasWeightCol
@@ -63,6 +64,8 @@ private[classification] trait NaiveBayesParams extends PredictorParams with HasW
 
   /** @group getParam */
   final def getModelType: String = $(modelType)
+
+  setDefault(smoothing -> 1.0, modelType -> NaiveBayes.Multinomial)
 }
 
 // scalastyle:off line.size.limit
@@ -106,7 +109,6 @@ class NaiveBayes @Since("1.5.0") (
    */
   @Since("1.5.0")
   def setSmoothing(value: Double): this.type = set(smoothing, value)
-  setDefault(smoothing -> 1.0)
 
   /**
    * Set the model type using a string (case-sensitive).
@@ -116,7 +118,6 @@ class NaiveBayes @Since("1.5.0") (
    */
   @Since("1.5.0")
   def setModelType(value: String): this.type = set(modelType, value)
-  setDefault(modelType -> Multinomial)
 
   /**
    * Sets the value of param [[weightCol]].
@@ -179,7 +180,7 @@ class NaiveBayes @Since("1.5.0") (
     }
 
     val w = if (isDefined(weightCol) && $(weightCol).nonEmpty) {
-      col($(weightCol)).cast(DoubleType)
+      checkNonNegativeWeight(col($(weightCol)).cast(DoubleType))
     } else {
       lit(1.0)
     }
@@ -259,7 +260,7 @@ class NaiveBayes @Since("1.5.0") (
     import spark.implicits._
 
     val w = if (isDefined(weightCol) && $(weightCol).nonEmpty) {
-      col($(weightCol)).cast(DoubleType)
+      checkNonNegativeWeight(col($(weightCol)).cast(DoubleType))
     } else {
       lit(1.0)
     }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/OneVsRest.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/OneVsRest.scala
index 94bf9a7fa9395..cf94c9fd36a40 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/OneVsRest.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/OneVsRest.scala
@@ -185,13 +185,16 @@ final class OneVsRestModel private[ml] (
       return dataset.toDF
     }
 
-    // determine the input columns: these need to be passed through
-    val origCols = dataset.schema.map(f => col(f.name))
+    val isProbModel = models.head.isInstanceOf[ProbabilisticClassificationModel[_, _]]
+
+    // use a temporary raw prediction column to avoid column conflict
+    val tmpRawPredName = "mbc$raw" + UUID.randomUUID().toString
 
     // add an accumulator column to store predictions of all the models
     val accColName = "mbc$acc" + UUID.randomUUID().toString
-    val initUDF = udf { () => Map[Int, Double]() }
-    val newDataset = dataset.withColumn(accColName, initUDF())
+    val newDataset = dataset.withColumn(accColName, lit(Array.emptyDoubleArray))
+    val columns = newDataset.schema.fieldNames.map(col)
+    val updateUDF = udf { (preds: Array[Double], pred: Vector) => preds :+ pred(1) }
 
     // persist if underlying dataset is not persistent.
     val handlePersistence = !dataset.isStreaming && dataset.storageLevel == StorageLevel.NONE
@@ -200,25 +203,19 @@ final class OneVsRestModel private[ml] (
     }
 
     // update the accumulator column with the result of prediction of models
-    val aggregatedDataset = models.zipWithIndex.foldLeft[DataFrame](newDataset) {
-      case (df, (model, index)) =>
-        val rawPredictionCol = model.getRawPredictionCol
-        val columns = origCols ++ List(col(rawPredictionCol), col(accColName))
-
-        // add temporary column to store intermediate scores and update
-        val tmpColName = "mbc$tmp" + UUID.randomUUID().toString
-        val updateUDF = udf { (predictions: Map[Int, Double], prediction: Vector) =>
-          predictions + ((index, prediction(1)))
-        }
-
-        model.setFeaturesCol($(featuresCol))
-        val transformedDataset = model.transform(df).select(columns: _*)
-        val updatedDataset = transformedDataset
-          .withColumn(tmpColName, updateUDF(col(accColName), col(rawPredictionCol)))
-        val newColumns = origCols ++ List(col(tmpColName))
+    val aggregatedDataset = models.foldLeft[DataFrame](newDataset) { case (df, model) =>
+      // avoid calling directly setter of model
+      val tmpModel = model.copy(ParamMap.empty).asInstanceOf[ClassificationModel[_, _]]
+      tmpModel.setFeaturesCol($(featuresCol))
+      tmpModel.setRawPredictionCol(tmpRawPredName)
+      tmpModel.setPredictionCol("")
+      if (isProbModel) {
+        tmpModel.asInstanceOf[ProbabilisticClassificationModel[_, _]].setProbabilityCol("")
+      }
 
-        // switch out the intermediate column with the accumulator column
-        updatedDataset.select(newColumns: _*).withColumnRenamed(tmpColName, accColName)
+      tmpModel.transform(df)
+        .withColumn(accColName, updateUDF(col(accColName), col(tmpRawPredName)))
+        .select(columns: _*)
     }
 
     if (handlePersistence) {
@@ -229,15 +226,8 @@ final class OneVsRestModel private[ml] (
     var predictionColumns = Seq.empty[Column]
 
     if (getRawPredictionCol.nonEmpty) {
-      val numClass = models.length
-
       // output the RawPrediction as vector
-      val rawPredictionUDF = udf { predictions: Map[Int, Double] =>
-        val predArray = Array.ofDim[Double](numClass)
-        predictions.foreach { case (idx, value) => predArray(idx) = value }
-        Vectors.dense(predArray)
-      }
-
+      val rawPredictionUDF = udf { preds: Array[Double] => Vectors.dense(preds) }
       predictionColNames :+= getRawPredictionCol
       predictionColumns :+= rawPredictionUDF(col(accColName))
         .as($(rawPredictionCol), outputSchema($(rawPredictionCol)).metadata)
@@ -245,10 +235,7 @@ final class OneVsRestModel private[ml] (
 
     if (getPredictionCol.nonEmpty) {
       // output the index of the classifier with highest confidence as prediction
-      val labelUDF = udf { (predictions: Map[Int, Double]) =>
-        predictions.maxBy(_._2)._1.toDouble
-      }
-
+      val labelUDF = udf { (preds: Array[Double]) => preds.indices.maxBy(preds.apply).toDouble }
       predictionColNames :+= getPredictionCol
       predictionColumns :+= labelUDF(col(accColName))
         .as(getPredictionCol, labelMetadata)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/ProbabilisticClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/ProbabilisticClassifier.scala
index 9758e3ca72c38..1caaeccd7b0d8 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/ProbabilisticClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/ProbabilisticClassifier.scala
@@ -19,6 +19,7 @@ package org.apache.spark.ml.classification
 
 import org.apache.spark.annotation.Since
 import org.apache.spark.ml.linalg.{DenseVector, Vector, VectorUDT}
+import org.apache.spark.ml.param.ParamMap
 import org.apache.spark.ml.param.shared._
 import org.apache.spark.ml.util.SchemaUtils
 import org.apache.spark.sql.{DataFrame, Dataset}
@@ -229,6 +230,27 @@ abstract class ProbabilisticClassificationModel[
       argMax
     }
   }
+
+  /**
+   *If the probability and prediction columns are set, this method returns the current model,
+   * otherwise it generates new columns for them and sets them as columns on a new copy of
+   * the current model
+   */
+  override private[classification] def findSummaryModel():
+  (ProbabilisticClassificationModel[FeaturesType, M], String, String) = {
+    val model = if ($(probabilityCol).isEmpty && $(predictionCol).isEmpty) {
+      copy(ParamMap.empty)
+        .setProbabilityCol("probability_" + java.util.UUID.randomUUID.toString)
+        .setPredictionCol("prediction_" + java.util.UUID.randomUUID.toString)
+    } else if ($(probabilityCol).isEmpty) {
+      copy(ParamMap.empty).setProbabilityCol("probability_" + java.util.UUID.randomUUID.toString)
+    } else if ($(predictionCol).isEmpty) {
+      copy(ParamMap.empty).setPredictionCol("prediction_" + java.util.UUID.randomUUID.toString)
+    } else {
+      this
+    }
+    (model, model.getProbabilityCol, model.getPredictionCol)
+  }
 }
 
 private[ml] object ProbabilisticClassificationModel {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/RandomForestClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/RandomForestClassifier.scala
index a316e472d9674..f9ce62b91924b 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/RandomForestClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/RandomForestClassifier.scala
@@ -166,7 +166,35 @@ class RandomForestClassifier @Since("1.4.0") (
     val numFeatures = trees.head.numFeatures
     instr.logNumClasses(numClasses)
     instr.logNumFeatures(numFeatures)
-    new RandomForestClassificationModel(uid, trees, numFeatures, numClasses)
+    createModel(dataset, trees, numFeatures, numClasses)
+  }
+
+  private def createModel(
+      dataset: Dataset[_],
+      trees: Array[DecisionTreeClassificationModel],
+      numFeatures: Int,
+      numClasses: Int): RandomForestClassificationModel = {
+    val model = copyValues(new RandomForestClassificationModel(uid, trees, numFeatures, numClasses))
+    val weightColName = if (!isDefined(weightCol)) "weightCol" else $(weightCol)
+
+    val (summaryModel, probabilityColName, predictionColName) = model.findSummaryModel()
+    val rfSummary = if (numClasses <= 2) {
+      new BinaryRandomForestClassificationTrainingSummaryImpl(
+        summaryModel.transform(dataset),
+        probabilityColName,
+        predictionColName,
+        $(labelCol),
+        weightColName,
+        Array(0.0))
+    } else {
+      new RandomForestClassificationTrainingSummaryImpl(
+        summaryModel.transform(dataset),
+        predictionColName,
+        $(labelCol),
+        weightColName,
+        Array(0.0))
+    }
+    model.setSummary(Some(rfSummary))
   }
 
   @Since("1.4.1")
@@ -204,7 +232,8 @@ class RandomForestClassificationModel private[ml] (
     @Since("1.5.0") override val numClasses: Int)
   extends ProbabilisticClassificationModel[Vector, RandomForestClassificationModel]
   with RandomForestClassifierParams with TreeEnsembleModel[DecisionTreeClassificationModel]
-  with MLWritable with Serializable {
+  with MLWritable with Serializable
+  with HasTrainingSummary[RandomForestClassificationTrainingSummary] {
 
   require(_trees.nonEmpty, "RandomForestClassificationModel requires at least 1 tree.")
 
@@ -228,6 +257,44 @@ class RandomForestClassificationModel private[ml] (
   @Since("1.4.0")
   override def treeWeights: Array[Double] = _treeWeights
 
+  /**
+   * Gets summary of model on training set. An exception is thrown
+   * if `hasSummary` is false.
+   */
+  @Since("3.1.0")
+  override def summary: RandomForestClassificationTrainingSummary = super.summary
+
+  /**
+   * Gets summary of model on training set. An exception is thrown
+   * if `hasSummary` is false or it is a multiclass model.
+   */
+  @Since("3.1.0")
+  def binarySummary: BinaryRandomForestClassificationTrainingSummary = summary match {
+    case b: BinaryRandomForestClassificationTrainingSummary => b
+    case _ =>
+      throw new RuntimeException("Cannot create a binary summary for a non-binary model" +
+        s"(numClasses=${numClasses}), use summary instead.")
+  }
+
+  /**
+   * Evaluates the model on a test dataset.
+   *
+   * @param dataset Test dataset to evaluate model on.
+   */
+  @Since("3.1.0")
+  def evaluate(dataset: Dataset[_]): RandomForestClassificationSummary = {
+    val weightColName = if (!isDefined(weightCol)) "weightCol" else $(weightCol)
+    // Handle possible missing or invalid prediction columns
+    val (summaryModel, probabilityColName, predictionColName) = findSummaryModel()
+    if (numClasses > 2) {
+      new RandomForestClassificationSummaryImpl(summaryModel.transform(dataset),
+        predictionColName, $(labelCol), weightColName)
+    } else {
+      new BinaryRandomForestClassificationSummaryImpl(summaryModel.transform(dataset),
+        probabilityColName, predictionColName, $(labelCol), weightColName)
+    }
+  }
+
   @Since("1.4.0")
   override def transformSchema(schema: StructType): StructType = {
     var outputSchema = super.transformSchema(schema)
@@ -388,3 +455,113 @@ object RandomForestClassificationModel extends MLReadable[RandomForestClassifica
     new RandomForestClassificationModel(uid, newTrees, numFeatures, numClasses)
   }
 }
+
+/**
+ * Abstraction for multiclass RandomForestClassification results for a given model.
+ */
+sealed trait RandomForestClassificationSummary extends ClassificationSummary {
+  /**
+   * Convenient method for casting to BinaryRandomForestClassificationSummary.
+   * This method will throw an Exception if the summary is not a binary summary.
+   */
+  @Since("3.1.0")
+  def asBinary: BinaryRandomForestClassificationSummary = this match {
+    case b: BinaryRandomForestClassificationSummary => b
+    case _ =>
+      throw new RuntimeException("Cannot cast to a binary summary.")
+  }
+}
+
+/**
+ * Abstraction for multiclass RandomForestClassification training results.
+ */
+sealed trait RandomForestClassificationTrainingSummary extends RandomForestClassificationSummary
+  with TrainingSummary
+
+/**
+ * Abstraction for BinaryRandomForestClassification results for a given model.
+ */
+sealed trait BinaryRandomForestClassificationSummary extends BinaryClassificationSummary
+
+/**
+ * Abstraction for BinaryRandomForestClassification training results.
+ */
+sealed trait BinaryRandomForestClassificationTrainingSummary extends
+  BinaryRandomForestClassificationSummary with RandomForestClassificationTrainingSummary
+
+/**
+ * Multiclass RandomForestClassification training results.
+ *
+ * @param predictions dataframe output by the model's `transform` method.
+ * @param predictionCol field in "predictions" which gives the prediction for a data instance as a
+ *                      double.
+ * @param labelCol field in "predictions" which gives the true label of each instance.
+ * @param weightCol field in "predictions" which gives the weight of each instance.
+ * @param objectiveHistory objective function (scaled loss + regularization) at each iteration.
+ */
+private class RandomForestClassificationTrainingSummaryImpl(
+    predictions: DataFrame,
+    predictionCol: String,
+    labelCol: String,
+    weightCol: String,
+    override val objectiveHistory: Array[Double])
+  extends RandomForestClassificationSummaryImpl(
+    predictions, predictionCol, labelCol, weightCol)
+    with RandomForestClassificationTrainingSummary
+
+/**
+ * Multiclass RandomForestClassification results for a given model.
+ *
+ * @param predictions dataframe output by the model's `transform` method.
+ * @param predictionCol field in "predictions" which gives the prediction for a data instance as a
+ *                      double.
+ * @param labelCol field in "predictions" which gives the true label of each instance.
+ * @param weightCol field in "predictions" which gives the weight of each instance.
+ */
+private class RandomForestClassificationSummaryImpl(
+    @transient override val predictions: DataFrame,
+    override val predictionCol: String,
+    override val labelCol: String,
+    override val weightCol: String)
+  extends RandomForestClassificationSummary
+
+/**
+ * Binary RandomForestClassification training results.
+ *
+ * @param predictions dataframe output by the model's `transform` method.
+ * @param scoreCol field in "predictions" which gives the probability of each class as a vector.
+ * @param predictionCol field in "predictions" which gives the prediction for a data instance as a
+ *                      double.
+ * @param labelCol field in "predictions" which gives the true label of each instance.
+ * @param weightCol field in "predictions" which gives the weight of each instance.
+ * @param objectiveHistory objective function (scaled loss + regularization) at each iteration.
+ */
+private class BinaryRandomForestClassificationTrainingSummaryImpl(
+    predictions: DataFrame,
+    scoreCol: String,
+    predictionCol: String,
+    labelCol: String,
+    weightCol: String,
+    override val objectiveHistory: Array[Double])
+  extends BinaryRandomForestClassificationSummaryImpl(
+    predictions, scoreCol, predictionCol, labelCol, weightCol)
+    with BinaryRandomForestClassificationTrainingSummary
+
+/**
+ * Binary RandomForestClassification for a given model.
+ *
+ * @param predictions dataframe output by the model's `transform` method.
+ * @param scoreCol field in "predictions" which gives the prediction of
+ *                 each class as a vector.
+ * @param labelCol field in "predictions" which gives the true label of each instance.
+ * @param weightCol field in "predictions" which gives the weight of each instance.
+ */
+private class BinaryRandomForestClassificationSummaryImpl(
+    predictions: DataFrame,
+    override val scoreCol: String,
+    predictionCol: String,
+    labelCol: String,
+    weightCol: String)
+  extends RandomForestClassificationSummaryImpl(
+    predictions, predictionCol, labelCol, weightCol)
+    with BinaryRandomForestClassificationSummary
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
index 6c7112b80569f..061091cfbe49c 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
@@ -21,6 +21,7 @@ import org.apache.hadoop.fs.Path
 
 import org.apache.spark.annotation.Since
 import org.apache.spark.ml.{Estimator, Model}
+import org.apache.spark.ml.functions.checkNonNegativeWeight
 import org.apache.spark.ml.linalg.Vector
 import org.apache.spark.ml.param._
 import org.apache.spark.ml.param.shared._
@@ -28,9 +29,8 @@ import org.apache.spark.ml.util._
 import org.apache.spark.ml.util.Instrumentation.instrumented
 import org.apache.spark.mllib.clustering.{BisectingKMeans => MLlibBisectingKMeans,
   BisectingKMeansModel => MLlibBisectingKMeansModel}
-import org.apache.spark.mllib.linalg.{Vector => OldVector, Vectors => OldVectors}
+import org.apache.spark.mllib.linalg.{Vectors => OldVectors}
 import org.apache.spark.mllib.linalg.VectorImplicits._
-import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{DataFrame, Dataset, Row}
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types.{DoubleType, IntegerType, StructType}
@@ -71,6 +71,8 @@ private[clustering] trait BisectingKMeansParams extends Params with HasMaxIter
   @Since("2.0.0")
   def getMinDivisibleClusterSize: Double = $(minDivisibleClusterSize)
 
+  setDefault(k -> 4, maxIter -> 20, minDivisibleClusterSize -> 1.0)
+
   /**
    * Validates and transforms the input schema.
    * @param schema input schema
@@ -225,11 +227,6 @@ class BisectingKMeans @Since("2.0.0") (
     @Since("2.0.0") override val uid: String)
   extends Estimator[BisectingKMeansModel] with BisectingKMeansParams with DefaultParamsWritable {
 
-  setDefault(
-    k -> 4,
-    maxIter -> 20,
-    minDivisibleClusterSize -> 1.0)
-
   @Since("2.0.0")
   override def copy(extra: ParamMap): BisectingKMeans = defaultCopy(extra)
 
@@ -278,21 +275,6 @@ class BisectingKMeans @Since("2.0.0") (
   override def fit(dataset: Dataset[_]): BisectingKMeansModel = instrumented { instr =>
     transformSchema(dataset.schema, logging = true)
 
-    val handlePersistence = dataset.storageLevel == StorageLevel.NONE
-    val w = if (isDefined(weightCol) && $(weightCol).nonEmpty) {
-      col($(weightCol)).cast(DoubleType)
-    } else {
-      lit(1.0)
-    }
-
-    val instances: RDD[(OldVector, Double)] = dataset
-      .select(DatasetUtils.columnToVector(dataset, getFeaturesCol), w).rdd.map {
-      case Row(point: Vector, weight: Double) => (OldVectors.fromML(point), weight)
-    }
-    if (handlePersistence) {
-      instances.persist(StorageLevel.MEMORY_AND_DISK)
-    }
-
     instr.logPipelineStage(this)
     instr.logDataset(dataset)
     instr.logParams(this, featuresCol, predictionCol, k, maxIter, seed,
@@ -304,11 +286,18 @@ class BisectingKMeans @Since("2.0.0") (
       .setMinDivisibleClusterSize($(minDivisibleClusterSize))
       .setSeed($(seed))
       .setDistanceMeasure($(distanceMeasure))
-    val parentModel = bkm.runWithWeight(instances, Some(instr))
-    val model = copyValues(new BisectingKMeansModel(uid, parentModel).setParent(this))
-    if (handlePersistence) {
-      instances.unpersist()
+
+    val w = if (isDefined(weightCol) && $(weightCol).nonEmpty) {
+      checkNonNegativeWeight(col($(weightCol)).cast(DoubleType))
+    } else {
+      lit(1.0)
     }
+    val instances = dataset.select(DatasetUtils.columnToVector(dataset, getFeaturesCol), w)
+      .rdd.map { case Row(point: Vector, weight: Double) => (OldVectors.fromML(point), weight) }
+
+    val handlePersistence = dataset.storageLevel == StorageLevel.NONE
+    val parentModel = bkm.runWithWeight(instances, handlePersistence, Some(instr))
+    val model = copyValues(new BisectingKMeansModel(uid, parentModel).setParent(this))
 
     val summary = new BisectingKMeansSummary(
       model.transform(dataset),
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
index 6d4137b638dcc..6568b36fb0e7c 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
@@ -22,6 +22,7 @@ import org.apache.hadoop.fs.Path
 import org.apache.spark.annotation.Since
 import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.ml.{Estimator, Model}
+import org.apache.spark.ml.functions.checkNonNegativeWeight
 import org.apache.spark.ml.impl.Utils.{unpackUpperTriangular, EPSILON}
 import org.apache.spark.ml.linalg._
 import org.apache.spark.ml.param._
@@ -43,7 +44,7 @@ import org.apache.spark.storage.StorageLevel
  */
 private[clustering] trait GaussianMixtureParams extends Params with HasMaxIter with HasFeaturesCol
   with HasSeed with HasPredictionCol with HasWeightCol with HasProbabilityCol with HasTol
-  with HasAggregationDepth with HasBlockSize {
+  with HasAggregationDepth {
 
   /**
    * Number of independent Gaussians in the mixture model. Must be greater than 1. Default: 2.
@@ -58,6 +59,8 @@ private[clustering] trait GaussianMixtureParams extends Params with HasMaxIter w
   @Since("2.0.0")
   def getK: Int = $(k)
 
+  setDefault(k -> 2, maxIter -> 100, tol -> 0.01)
+
   /**
    * Validates and transforms the input schema.
    *
@@ -167,8 +170,7 @@ class GaussianMixtureModel private[ml] (
 
   @Since("3.0.0")
   def predictProbability(features: Vector): Vector = {
-    val probs = GaussianMixtureModel
-      .computeProbabilities(features, gaussians, weights)
+    val probs = GaussianMixtureModel.computeProbabilities(features, gaussians, weights)
     Vectors.dense(probs)
   }
 
@@ -260,10 +262,8 @@ object GaussianMixtureModel extends MLReadable[GaussianMixtureModel] {
       require(mus.length == sigmas.length, "Length of Mu and Sigma array must match")
       require(mus.length == weights.length, "Length of weight and Gaussian array must match")
 
-      val gaussians = mus.zip(sigmas).map {
-        case (mu, sigma) =>
-          new MultivariateGaussian(mu.asML, sigma.asML)
-      }
+      val gaussians = mus.zip(sigmas)
+        .map { case (mu, sigma) => new MultivariateGaussian(mu.asML, sigma.asML) }
       val model = new GaussianMixtureModel(metadata.uid, weights, gaussians)
 
       metadata.getAndSetParams(model)
@@ -327,11 +327,6 @@ class GaussianMixture @Since("2.0.0") (
     @Since("2.0.0") override val uid: String)
   extends Estimator[GaussianMixtureModel] with GaussianMixtureParams with DefaultParamsWritable {
 
-  setDefault(
-    k -> 2,
-    maxIter -> 100,
-    tol -> 0.01)
-
   @Since("2.0.0")
   override def copy(extra: ParamMap): GaussianMixture = defaultCopy(extra)
 
@@ -374,25 +369,6 @@ class GaussianMixture @Since("2.0.0") (
   @Since("3.0.0")
   def setAggregationDepth(value: Int): this.type = set(aggregationDepth, value)
 
-  /**
-   * Set block size for stacking input data in matrices.
-   * If blockSize == 1, then stacking will be skipped, and each vector is treated individually;
-   * If blockSize &gt; 1, then vectors will be stacked to blocks, and high-level BLAS routines
-   * will be used if possible (for example, GEMV instead of DOT, GEMM instead of GEMV).
-   * Recommended size is between 10 and 1000. An appropriate choice of the block size depends
-   * on the sparsity and dim of input datasets, the underlying BLAS implementation (for example,
-   * f2jBLAS, OpenBLAS, intel MKL) and its configuration (for example, number of threads).
-   * Note that existing BLAS implementations are mainly optimized for dense matrices, if the
-   * input dataset is sparse, stacking may bring no performance gain, the worse is possible
-   * performance regression.
-   * Default is 1.
-   *
-   * @group expertSetParam
-   */
-  @Since("3.1.0")
-  def setBlockSize(value: Int): this.type = set(blockSize, value)
-  setDefault(blockSize -> 1)
-
   /**
    * Number of samples per cluster to use when initializing Gaussians.
    */
@@ -413,11 +389,11 @@ class GaussianMixture @Since("2.0.0") (
     instr.logPipelineStage(this)
     instr.logDataset(dataset)
     instr.logParams(this, featuresCol, predictionCol, probabilityCol, weightCol, k, maxIter,
-      seed, tol, aggregationDepth, blockSize)
+      seed, tol, aggregationDepth)
     instr.logNumFeatures(numFeatures)
 
     val w = if (isDefined(weightCol) && $(weightCol).nonEmpty) {
-      col($(weightCol)).cast(DoubleType)
+      checkNonNegativeWeight(col($(weightCol)).cast(DoubleType))
     } else {
       lit(1.0)
     }
@@ -426,25 +402,12 @@ class GaussianMixture @Since("2.0.0") (
       .as[(Vector, Double)].rdd
       .setName("training instances")
 
-    if ($(blockSize) == 1 && dataset.storageLevel == StorageLevel.NONE) {
-      instances.persist(StorageLevel.MEMORY_AND_DISK)
-    }
-
+    val handlePersistence = dataset.storageLevel == StorageLevel.NONE
+    if (handlePersistence) { instances.persist(StorageLevel.MEMORY_AND_DISK) }
     // TODO: SPARK-15785 Support users supplied initial GMM.
     val (weights, gaussians) = initRandom(instances, $(k), numFeatures)
-
-    val (logLikelihood, iteration) = if ($(blockSize) == 1) {
-      trainOnRows(instances, weights, gaussians, numFeatures, instr)
-    } else {
-      val sparsity = 1 - instances.map { case (v, _) => v.numNonzeros.toDouble / v.size }.mean()
-      instr.logNamedValue("sparsity", sparsity.toString)
-      if (sparsity > 0.5) {
-        logWarning(s"sparsity of input dataset is $sparsity, " +
-          s"which may hurt performance in high-level BLAS.")
-      }
-      trainOnBlocks(instances, weights, gaussians, numFeatures, instr)
-    }
-    if (instances.getStorageLevel != StorageLevel.NONE) instances.unpersist()
+    val (logLikelihood, iteration) = trainImpl(instances, weights, gaussians, numFeatures, instr)
+    if (handlePersistence) { instances.unpersist() }
 
     val gaussianDists = gaussians.map { case (mean, covVec) =>
       val cov = GaussianMixture.unpackUpperTriangularMatrix(numFeatures, covVec.values)
@@ -460,7 +423,7 @@ class GaussianMixture @Since("2.0.0") (
     model.setSummary(Some(summary))
   }
 
-  private def trainOnRows(
+  private def trainImpl(
       instances: RDD[(Vector, Double)],
       weights: Array[Double],
       gaussians: Array[(DenseVector, DenseVector)],
@@ -491,80 +454,7 @@ class GaussianMixture @Since("2.0.0") (
             (i, (agg.means(i), agg.covs(i), agg.weights(i), ws))
           }
         } else Iterator.empty
-      }.reduceByKey { case ((mean1, cov1, w1, ws1), (mean2, cov2, w2, ws2)) =>
-        // update the weights, means and covariances for i-th distributions
-        BLAS.axpy(1.0, mean2, mean1)
-        BLAS.axpy(1.0, cov2, cov1)
-        (mean1, cov1, w1 + w2, ws1 + ws2)
-      }.mapValues { case (mean, cov, w, ws) =>
-        // Create new distributions based on the partial assignments
-        // (often referred to as the "M" step in literature)
-        GaussianMixture.updateWeightsAndGaussians(mean, cov, w, ws)
-      }.collect().foreach { case (i, (weight, gaussian)) =>
-        weights(i) = weight
-        gaussians(i) = gaussian
-      }
-
-      bcWeights.destroy()
-      bcGaussians.destroy()
-
-      if (iteration == 0) {
-        instr.logNumExamples(weightSumAccum.count)
-        instr.logSumOfWeights(weightSumAccum.value)
-      }
-
-      logLikelihoodPrev = logLikelihood         // current becomes previous
-      logLikelihood = logLikelihoodAccum.value  // this is the freshly computed log-likelihood
-      instr.logNamedValue(s"logLikelihood@iter$iteration", logLikelihood)
-      iteration += 1
-    }
-
-    (logLikelihood, iteration)
-  }
-
-  private def trainOnBlocks(
-      instances: RDD[(Vector, Double)],
-      weights: Array[Double],
-      gaussians: Array[(DenseVector, DenseVector)],
-      numFeatures: Int,
-      instr: Instrumentation): (Double, Int) = {
-    val blocks = instances.mapPartitions { iter =>
-      iter.grouped($(blockSize))
-        .map { seq => (Matrices.fromVectors(seq.map(_._1)), seq.map(_._2).toArray) }
-    }.persist(StorageLevel.MEMORY_AND_DISK)
-     .setName(s"training dataset (blockSize=${$(blockSize)})")
-
-    val sc = instances.sparkContext
-    var logLikelihood = Double.MinValue
-    var logLikelihoodPrev = 0.0
-
-    var iteration = 0
-    while (iteration < $(maxIter) && math.abs(logLikelihood - logLikelihoodPrev) > $(tol)) {
-      val weightSumAccum = if (iteration == 0) sc.doubleAccumulator else null
-      val logLikelihoodAccum = sc.doubleAccumulator
-      val bcWeights = sc.broadcast(weights)
-      val bcGaussians = sc.broadcast(gaussians)
-
-      // aggregate the cluster contribution for all sample points,
-      // and then compute the new distributions
-      blocks.mapPartitions { iter =>
-        if (iter.nonEmpty) {
-          val agg = new BlockExpectationAggregator(numFeatures,
-            $(blockSize), bcWeights, bcGaussians)
-          while (iter.hasNext) { agg.add(iter.next) }
-          // sum of weights in this partition
-          val ws = agg.weights.sum
-          if (iteration == 0) weightSumAccum.add(ws)
-          logLikelihoodAccum.add(agg.logLikelihood)
-          agg.meanIter.zip(agg.covIter).zipWithIndex
-            .map { case ((mean, cov), i) => (i, (mean, cov, agg.weights(i), ws)) }
-        } else Iterator.empty
-      }.reduceByKey { case ((mean1, cov1, w1, ws1), (mean2, cov2, w2, ws2)) =>
-        // update the weights, means and covariances for i-th distributions
-        BLAS.axpy(1.0, mean2, mean1)
-        BLAS.axpy(1.0, cov2, cov1)
-        (mean1, cov1, w1 + w2, ws1 + ws2)
-      }.mapValues { case (mean, cov, w, ws) =>
+      }.reduceByKey(GaussianMixture.mergeWeightsMeans).mapValues { case (mean, cov, w, ws) =>
         // Create new distributions based on the partial assignments
         // (often referred to as the "M" step in literature)
         GaussianMixture.updateWeightsAndGaussians(mean, cov, w, ws)
@@ -586,7 +476,6 @@ class GaussianMixture @Since("2.0.0") (
       instr.logNamedValue(s"logLikelihood@iter$iteration", logLikelihood)
       iteration += 1
     }
-    blocks.unpersist()
 
     (logLikelihood, iteration)
   }
@@ -623,8 +512,8 @@ class GaussianMixture @Since("2.0.0") (
     val gaussians = Array.tabulate(numClusters) { i =>
       val start = i * numSamples
       val end = start + numSamples
-      val sampleSlice = samples.view(start, end)
-      val weightSlice = sampleWeights.view(start, end)
+      val sampleSlice = samples.view.slice(start, end)
+      val weightSlice = sampleWeights.view.slice(start, end)
       val localWeightSum = weightSlice.sum
       weights(i) = localWeightSum / weightSum
 
@@ -690,6 +579,16 @@ object GaussianMixture extends DefaultParamsReadable[GaussianMixture] {
     new DenseMatrix(n, n, symmetricValues)
   }
 
+  private def mergeWeightsMeans(
+      a: (DenseVector, DenseVector, Double, Double),
+      b: (DenseVector, DenseVector, Double, Double)): (DenseVector, DenseVector, Double, Double) =
+  {
+    // update the weights, means and covariances for i-th distributions
+    BLAS.axpy(1.0, b._1, a._1)
+    BLAS.axpy(1.0, b._2, a._2)
+    (a._1, a._2, a._3 + b._3, a._4 + b._4)
+  }
+
   /**
    * Update the weight, mean and covariance of gaussian distribution.
    *
@@ -795,121 +694,6 @@ private class ExpectationAggregator(
 }
 
 
-/**
- * BlockExpectationAggregator computes the partial expectation results.
- *
- * @param numFeatures The number of features.
- * @param bcWeights The broadcast weights for each Gaussian distribution in the mixture.
- * @param bcGaussians The broadcast array of Multivariate Gaussian (Normal) Distribution
- *                    in the mixture. Note only upper triangular part of the covariance
- *                    matrix of each distribution is stored as dense vector (column major)
- *                    in order to reduce shuffled data size.
- */
-private class BlockExpectationAggregator(
-    numFeatures: Int,
-    blockSize: Int,
-    bcWeights: Broadcast[Array[Double]],
-    bcGaussians: Broadcast[Array[(DenseVector, DenseVector)]]) extends Serializable {
-
-  private val k = bcWeights.value.length
-  private var totalCnt = 0L
-  private var newLogLikelihood = 0.0
-  private val covSize = numFeatures * (numFeatures + 1) / 2
-  private lazy val newWeights = Array.ofDim[Double](k)
-  @transient private lazy val newMeansMat = DenseMatrix.zeros(numFeatures, k)
-  @transient private lazy val newCovsMat = DenseMatrix.zeros(covSize, k)
-  @transient private lazy val auxiliaryProbMat = DenseMatrix.zeros(blockSize, k)
-  @transient private lazy val auxiliaryPDFMat = DenseMatrix.zeros(blockSize, numFeatures)
-  @transient private lazy val auxiliaryCovVec = Vectors.zeros(covSize).toDense
-
-  @transient private lazy val gaussians = {
-    bcGaussians.value.map { case (mean, covVec) =>
-      val cov = GaussianMixture.unpackUpperTriangularMatrix(numFeatures, covVec.values)
-      new MultivariateGaussian(mean, cov)
-    }
-  }
-
-  def count: Long = totalCnt
-
-  def logLikelihood: Double = newLogLikelihood
-
-  def weights: Array[Double] = newWeights
-
-  def meanIter: Iterator[DenseVector] = newMeansMat.colIter.map(_.toDense)
-
-  def covIter: Iterator[DenseVector] = newCovsMat.colIter.map(_.toDense)
-
-  /**
-   * Add a new training instance block to this BlockExpectationAggregator, update the weights,
-   * means and covariances for each distributions, and update the log likelihood.
-   *
-   * @param block The instance block of data point to be added.
-   * @return This BlockExpectationAggregator object.
-   */
-  def add(block: (Matrix, Array[Double])): this.type = {
-    val (matrix: Matrix, weights: Array[Double]) = block
-    require(matrix.isTransposed)
-    val size = matrix.numRows
-    require(weights.length == size)
-
-    val blas1 = BLAS.getBLAS(size)
-    val blas2 = BLAS.getBLAS(k)
-
-    val probMat = if (blockSize == size) auxiliaryProbMat else DenseMatrix.zeros(size, k)
-    require(!probMat.isTransposed)
-    java.util.Arrays.fill(probMat.values, EPSILON)
-
-    val pdfMat = if (blockSize == size) auxiliaryPDFMat else DenseMatrix.zeros(size, numFeatures)
-    var j = 0
-    while (j < k) {
-      val pdfVec = gaussians(j).pdf(matrix, pdfMat)
-      blas1.daxpy(size, bcWeights.value(j), pdfVec.values, 0, 1, probMat.values, j * size, 1)
-      j += 1
-    }
-
-    var i = 0
-    while (i < size) {
-      val weight = weights(i)
-      val probSum = blas2.dasum(k, probMat.values, i, size)
-      blas2.dscal(k, weight / probSum, probMat.values, i, size)
-      blas2.daxpy(k, 1.0, probMat.values, i, size, newWeights, 0, 1)
-      newLogLikelihood += math.log(probSum) * weight
-      i += 1
-    }
-
-    BLAS.gemm(1.0, matrix.transpose, probMat, 1.0, newMeansMat)
-
-    // compute the cov vector for each row vector
-    val covVec = auxiliaryCovVec
-    val covVecIter = matrix match {
-      case dm: DenseMatrix =>
-        Iterator.tabulate(size) { i =>
-          java.util.Arrays.fill(covVec.values, 0.0)
-          // when input block is dense, directly use nativeBLAS to avoid array copy
-          BLAS.nativeBLAS.dspr("U", numFeatures, 1.0, dm.values, i * numFeatures, 1,
-            covVec.values, 0)
-          covVec
-        }
-
-      case sm: SparseMatrix =>
-        sm.rowIter.map { vec =>
-          java.util.Arrays.fill(covVec.values, 0.0)
-          BLAS.spr(1.0, vec, covVec)
-          covVec
-        }
-    }
-
-    covVecIter.zipWithIndex.foreach { case (covVec, i) =>
-      BLAS.nativeBLAS.dger(covSize, k, 1.0, covVec.values, 0, 1,
-        probMat.values, i, size, newCovsMat.values, 0, covSize)
-    }
-
-    totalCnt += size
-
-    this
-  }
-}
-
 /**
  * Summary of GaussianMixture.
  *
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
index a42c920e24987..f6f6eb7fa6d31 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
@@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path
 
 import org.apache.spark.annotation.Since
 import org.apache.spark.ml.{Estimator, Model, PipelineStage}
+import org.apache.spark.ml.functions.checkNonNegativeWeight
 import org.apache.spark.ml.linalg.Vector
 import org.apache.spark.ml.param._
 import org.apache.spark.ml.param.shared._
@@ -31,7 +32,6 @@ import org.apache.spark.ml.util.Instrumentation.instrumented
 import org.apache.spark.mllib.clustering.{DistanceMeasure, KMeans => MLlibKMeans, KMeansModel => MLlibKMeansModel}
 import org.apache.spark.mllib.linalg.{Vector => OldVector, Vectors => OldVectors}
 import org.apache.spark.mllib.linalg.VectorImplicits._
-import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types.{DoubleType, IntegerType, StructType}
@@ -86,6 +86,9 @@ private[clustering] trait KMeansParams extends Params with HasMaxIter with HasFe
   @Since("1.5.0")
   def getInitSteps: Int = $(initSteps)
 
+  setDefault(k -> 2, maxIter -> 20, initMode -> MLlibKMeans.K_MEANS_PARALLEL, initSteps -> 2,
+    tol -> 1e-4, distanceMeasure -> DistanceMeasure.EUCLIDEAN)
+
   /**
    * Validates and transforms the input schema.
    * @param schema input schema
@@ -270,14 +273,6 @@ class KMeans @Since("1.5.0") (
     @Since("1.5.0") override val uid: String)
   extends Estimator[KMeansModel] with KMeansParams with DefaultParamsWritable {
 
-  setDefault(
-    k -> 2,
-    maxIter -> 20,
-    initMode -> MLlibKMeans.K_MEANS_PARALLEL,
-    initSteps -> 2,
-    tol -> 1e-4,
-    distanceMeasure -> DistanceMeasure.EUCLIDEAN)
-
   @Since("1.5.0")
   override def copy(extra: ParamMap): KMeans = defaultCopy(extra)
 
@@ -334,22 +329,6 @@ class KMeans @Since("1.5.0") (
   override def fit(dataset: Dataset[_]): KMeansModel = instrumented { instr =>
     transformSchema(dataset.schema, logging = true)
 
-    val handlePersistence = dataset.storageLevel == StorageLevel.NONE
-    val w = if (isDefined(weightCol) && $(weightCol).nonEmpty) {
-      col($(weightCol)).cast(DoubleType)
-    } else {
-      lit(1.0)
-    }
-
-    val instances: RDD[(OldVector, Double)] = dataset
-      .select(DatasetUtils.columnToVector(dataset, getFeaturesCol), w).rdd.map {
-      case Row(point: Vector, weight: Double) => (OldVectors.fromML(point), weight)
-    }
-
-    if (handlePersistence) {
-      instances.persist(StorageLevel.MEMORY_AND_DISK)
-    }
-
     instr.logPipelineStage(this)
     instr.logDataset(dataset)
     instr.logParams(this, featuresCol, predictionCol, k, initMode, initSteps, distanceMeasure,
@@ -362,8 +341,19 @@ class KMeans @Since("1.5.0") (
       .setSeed($(seed))
       .setEpsilon($(tol))
       .setDistanceMeasure($(distanceMeasure))
-    val parentModel = algo.runWithWeight(instances, Option(instr))
+
+    val w = if (isDefined(weightCol) && $(weightCol).nonEmpty) {
+      checkNonNegativeWeight(col($(weightCol)).cast(DoubleType))
+    } else {
+      lit(1.0)
+    }
+    val instances = dataset.select(DatasetUtils.columnToVector(dataset, getFeaturesCol), w)
+      .rdd.map { case Row(point: Vector, weight: Double) => (OldVectors.fromML(point), weight) }
+
+    val handlePersistence = dataset.storageLevel == StorageLevel.NONE
+    val parentModel = algo.runWithWeight(instances, handlePersistence, Some(instr))
     val model = copyValues(new KMeansModel(uid, parentModel).setParent(this))
+
     val summary = new KMeansSummary(
       model.transform(dataset),
       $(predictionCol),
@@ -374,9 +364,6 @@ class KMeans @Since("1.5.0") (
 
     model.setSummary(Some(summary))
     instr.logNamedValue("clusterSizes", summary.clusterSizes)
-    if (handlePersistence) {
-      instances.unpersist()
-    }
     model
   }
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
index 4e86b3b247ace..c1b76fb40b2f6 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
@@ -199,8 +199,6 @@ private[clustering] trait LDAParams extends Params with HasFeaturesCol with HasM
     " with estimates of the topic mixture distribution for each document (often called \"theta\"" +
     " in the literature).  Returns a vector of zeros for an empty document.")
 
-  setDefault(topicDistributionCol -> "topicDistribution")
-
   /** @group getParam */
   @Since("1.6.0")
   def getTopicDistributionCol: String = $(topicDistributionCol)
@@ -315,6 +313,11 @@ private[clustering] trait LDAParams extends Params with HasFeaturesCol with HasM
   @Since("2.0.0")
   def getKeepLastCheckpoint: Boolean = $(keepLastCheckpoint)
 
+  setDefault(maxIter -> 20, k -> 10, optimizer -> "online", checkpointInterval -> 10,
+    learningOffset -> 1024, learningDecay -> 0.51, subsamplingRate -> 0.05,
+    optimizeDocConcentration -> true, keepLastCheckpoint -> true,
+    topicDistributionCol -> "topicDistribution")
+
   /**
    * Validates and transforms the input schema.
    *
@@ -863,10 +866,6 @@ class LDA @Since("1.6.0") (
   @Since("1.6.0")
   def this() = this(Identifiable.randomUID("lda"))
 
-  setDefault(maxIter -> 20, k -> 10, optimizer -> "online", checkpointInterval -> 10,
-    learningOffset -> 1024, learningDecay -> 0.51, subsamplingRate -> 0.05,
-    optimizeDocConcentration -> true, keepLastCheckpoint -> true)
-
   /**
    * The features for LDA should be a `Vector` representing the word counts in a document.
    * The vector should be of length vocabSize, with counts for each term (word).
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/PowerIterationClustering.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/PowerIterationClustering.scala
index 812a426a062c1..1466b32bef530 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/PowerIterationClustering.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/PowerIterationClustering.scala
@@ -91,7 +91,7 @@ private[clustering] trait PowerIterationClusteringParams extends Params with Has
   @Since("2.4.0")
   def getDstCol: String = $(dstCol)
 
-  setDefault(srcCol -> "src", dstCol -> "dst")
+  setDefault(srcCol -> "src", dstCol -> "dst", k -> 2, maxIter -> 20, initMode -> "random")
 }
 
 /**
@@ -111,11 +111,6 @@ class PowerIterationClustering private[clustering] (
     @Since("2.4.0") override val uid: String)
   extends PowerIterationClusteringParams with DefaultParamsWritable {
 
-  setDefault(
-    k -> 2,
-    maxIter -> 20,
-    initMode -> "random")
-
   @Since("2.4.0")
   def this() = this(Identifiable.randomUID("PowerIterationClustering"))
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluator.scala
index 82b8e14f010af..93b66f3ab7007 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluator.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.ml.evaluation
 
 import org.apache.spark.annotation.Since
+import org.apache.spark.ml.functions.checkNonNegativeWeight
 import org.apache.spark.ml.linalg.{Vector, VectorUDT}
 import org.apache.spark.ml.param._
 import org.apache.spark.ml.param.shared._
@@ -80,8 +81,6 @@ class BinaryClassificationEvaluator @Since("1.4.0") (@Since("1.4.0") override va
   @Since("3.0.0")
   def setNumBins(value: Int): this.type = set(numBins, value)
 
-  setDefault(numBins -> 1000)
-
   /** @group setParam */
   @Since("1.5.0")
   def setRawPredictionCol(value: String): this.type = set(rawPredictionCol, value)
@@ -94,10 +93,28 @@ class BinaryClassificationEvaluator @Since("1.4.0") (@Since("1.4.0") override va
   @Since("3.0.0")
   def setWeightCol(value: String): this.type = set(weightCol, value)
 
-  setDefault(metricName -> "areaUnderROC")
+  setDefault(metricName -> "areaUnderROC", numBins -> 1000)
 
   @Since("2.0.0")
   override def evaluate(dataset: Dataset[_]): Double = {
+    val metrics = getMetrics(dataset)
+    val metric = $(metricName) match {
+      case "areaUnderROC" => metrics.areaUnderROC()
+      case "areaUnderPR" => metrics.areaUnderPR()
+    }
+    metrics.unpersist()
+    metric
+  }
+
+  /**
+   * Get a BinaryClassificationMetrics, which can be used to get binary classification
+   * metrics such as areaUnderROC and areaUnderPR.
+   *
+   * @param dataset a dataset that contains labels/observations and predictions.
+   * @return BinaryClassificationMetrics
+   */
+  @Since("3.1.0")
+  def getMetrics(dataset: Dataset[_]): BinaryClassificationMetrics = {
     val schema = dataset.schema
     SchemaUtils.checkColumnTypes(schema, $(rawPredictionCol), Seq(DoubleType, new VectorUDT))
     SchemaUtils.checkNumericType(schema, $(labelCol))
@@ -113,19 +130,13 @@ class BinaryClassificationEvaluator @Since("1.4.0") (@Since("1.4.0") override va
         col($(rawPredictionCol)),
         col($(labelCol)).cast(DoubleType),
         if (!isDefined(weightCol) || $(weightCol).isEmpty) lit(1.0)
-        else col($(weightCol)).cast(DoubleType)).rdd.map {
+        else checkNonNegativeWeight(col($(weightCol)).cast(DoubleType))).rdd.map {
         case Row(rawPrediction: Vector, label: Double, weight: Double) =>
           (rawPrediction(1), label, weight)
         case Row(rawPrediction: Double, label: Double, weight: Double) =>
           (rawPrediction, label, weight)
       }
-    val metrics = new BinaryClassificationMetrics(scoreAndLabelsWithWeights, $(numBins))
-    val metric = $(metricName) match {
-      case "areaUnderROC" => metrics.areaUnderROC()
-      case "areaUnderPR" => metrics.areaUnderPR()
-    }
-    metrics.unpersist()
-    metric
+    new BinaryClassificationMetrics(scoreAndLabelsWithWeights, $(numBins))
   }
 
   @Since("1.5.0")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/ClusteringEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/ClusteringEvaluator.scala
index 641a1eb5f61db..fa2c25a5912a7 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/ClusteringEvaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/ClusteringEvaluator.scala
@@ -17,15 +17,13 @@
 
 package org.apache.spark.ml.evaluation
 
-import org.apache.spark.SparkContext
 import org.apache.spark.annotation.Since
-import org.apache.spark.broadcast.Broadcast
-import org.apache.spark.ml.linalg.{BLAS, DenseVector, Vector, Vectors}
+import org.apache.spark.ml.functions.checkNonNegativeWeight
 import org.apache.spark.ml.param.{Param, ParamMap, ParamValidators}
-import org.apache.spark.ml.param.shared.{HasFeaturesCol, HasPredictionCol}
+import org.apache.spark.ml.param.shared.{HasFeaturesCol, HasPredictionCol, HasWeightCol}
 import org.apache.spark.ml.util._
-import org.apache.spark.sql.{Column, DataFrame, Dataset}
-import org.apache.spark.sql.functions.{avg, col, udf}
+import org.apache.spark.sql.Dataset
+import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types.DoubleType
 
 /**
@@ -38,7 +36,8 @@ import org.apache.spark.sql.types.DoubleType
  */
 @Since("2.3.0")
 class ClusteringEvaluator @Since("2.3.0") (@Since("2.3.0") override val uid: String)
-  extends Evaluator with HasPredictionCol with HasFeaturesCol with DefaultParamsWritable {
+  extends Evaluator with HasPredictionCol with HasFeaturesCol with HasWeightCol
+    with DefaultParamsWritable {
 
   @Since("2.3.0")
   def this() = this(Identifiable.randomUID("cluEval"))
@@ -57,6 +56,10 @@ class ClusteringEvaluator @Since("2.3.0") (@Since("2.3.0") override val uid: Str
   @Since("2.3.0")
   def setFeaturesCol(value: String): this.type = set(featuresCol, value)
 
+  /** @group setParam */
+  @Since("3.1.0")
+  def setWeightCol(value: String): this.type = set(weightCol, value)
+
   /**
    * param for metric name in evaluation
    * (supports `"silhouette"` (default))
@@ -102,557 +105,62 @@ class ClusteringEvaluator @Since("2.3.0") (@Since("2.3.0") override val uid: Str
 
   @Since("2.3.0")
   override def evaluate(dataset: Dataset[_]): Double = {
-    SchemaUtils.validateVectorCompatibleColumn(dataset.schema, $(featuresCol))
-    SchemaUtils.checkNumericType(dataset.schema, $(predictionCol))
+    val metrics = getMetrics(dataset)
 
-    val vectorCol = DatasetUtils.columnToVector(dataset, $(featuresCol))
-    val df = dataset.select(col($(predictionCol)),
-      vectorCol.as($(featuresCol), dataset.schema($(featuresCol)).metadata))
-
-    ($(metricName), $(distanceMeasure)) match {
-      case ("silhouette", "squaredEuclidean") =>
-        SquaredEuclideanSilhouette.computeSilhouetteScore(
-          df, $(predictionCol), $(featuresCol))
-      case ("silhouette", "cosine") =>
-        CosineSilhouette.computeSilhouetteScore(df, $(predictionCol), $(featuresCol))
-      case (mn, dm) =>
-        throw new IllegalArgumentException(s"No support for metric $mn, distance $dm")
+    $(metricName) match {
+      case ("silhouette") => metrics.silhouette
+      case (other) =>
+        throw new IllegalArgumentException(s"No support for metric $other")
     }
   }
 
-  @Since("3.0.0")
-  override def toString: String = {
-    s"ClusteringEvaluator: uid=$uid, metricName=${$(metricName)}, " +
-      s"distanceMeasure=${$(distanceMeasure)}"
-  }
-}
-
-
-@Since("2.3.0")
-object ClusteringEvaluator
-  extends DefaultParamsReadable[ClusteringEvaluator] {
-
-  @Since("2.3.0")
-  override def load(path: String): ClusteringEvaluator = super.load(path)
-
-}
-
-
-private[evaluation] abstract class Silhouette {
-
-  /**
-   * It computes the Silhouette coefficient for a point.
-   */
-  def pointSilhouetteCoefficient(
-      clusterIds: Set[Double],
-      pointClusterId: Double,
-      pointClusterNumOfPoints: Long,
-      averageDistanceToCluster: (Double) => Double): Double = {
-    if (pointClusterNumOfPoints == 1) {
-      // Single-element clusters have silhouette 0
-      0.0
-    } else {
-      // Here we compute the average dissimilarity of the current point to any cluster of which the
-      // point is not a member.
-      // The cluster with the lowest average dissimilarity - i.e. the nearest cluster to the current
-      // point - is said to be the "neighboring cluster".
-      val otherClusterIds = clusterIds.filter(_ != pointClusterId)
-      val neighboringClusterDissimilarity = otherClusterIds.map(averageDistanceToCluster).min
-      // adjustment for excluding the node itself from the computation of the average dissimilarity
-      val currentClusterDissimilarity =
-        averageDistanceToCluster(pointClusterId) * pointClusterNumOfPoints /
-          (pointClusterNumOfPoints - 1)
-      if (currentClusterDissimilarity < neighboringClusterDissimilarity) {
-        1 - (currentClusterDissimilarity / neighboringClusterDissimilarity)
-      } else if (currentClusterDissimilarity > neighboringClusterDissimilarity) {
-        (neighboringClusterDissimilarity / currentClusterDissimilarity) - 1
-      } else {
-        0.0
-      }
-    }
-  }
-
-  /**
-   * Compute the mean Silhouette values of all samples.
-   */
-  def overallScore(df: DataFrame, scoreColumn: Column): Double = {
-    df.select(avg(scoreColumn)).collect()(0).getDouble(0)
-  }
-}
-
-/**
- * SquaredEuclideanSilhouette computes the average of the
- * Silhouette over all the data of the dataset, which is
- * a measure of how appropriately the data have been clustered.
- *
- * The Silhouette for each point `i` is defined as:
- *
- * <blockquote>
- *   $$
- *   s_{i} = \frac{b_{i}-a_{i}}{max\{a_{i},b_{i}\}}
- *   $$
- * </blockquote>
- *
- * which can be rewritten as
- *
- * <blockquote>
- *   $$
- *   s_{i}= \begin{cases}
- *   1-\frac{a_{i}}{b_{i}} & \text{if } a_{i} \leq b_{i} \\
- *   \frac{b_{i}}{a_{i}}-1 & \text{if } a_{i} \gt b_{i} \end{cases}
- *   $$
- * </blockquote>
- *
- * where `$a_{i}$` is the average dissimilarity of `i` with all other data
- * within the same cluster, `$b_{i}$` is the lowest average dissimilarity
- * of `i` to any other cluster, of which `i` is not a member.
- * `$a_{i}$` can be interpreted as how well `i` is assigned to its cluster
- * (the smaller the value, the better the assignment), while `$b_{i}$` is
- * a measure of how well `i` has not been assigned to its "neighboring cluster",
- * ie. the nearest cluster to `i`.
- *
- * Unfortunately, the naive implementation of the algorithm requires to compute
- * the distance of each couple of points in the dataset. Since the computation of
- * the distance measure takes `D` operations - if `D` is the number of dimensions
- * of each point, the computational complexity of the algorithm is `O(N^2^*D)`, where
- * `N` is the cardinality of the dataset. Of course this is not scalable in `N`,
- * which is the critical number in a Big Data context.
- *
- * The algorithm which is implemented in this object, instead, is an efficient
- * and parallel implementation of the Silhouette using the squared Euclidean
- * distance measure.
- *
- * With this assumption, the total distance of the point `X`
- * to the points `$C_{i}$` belonging to the cluster `$\Gamma$` is:
- *
- * <blockquote>
- *   $$
- *   \sum\limits_{i=1}^N d(X, C_{i} ) =
- *   \sum\limits_{i=1}^N \Big( \sum\limits_{j=1}^D (x_{j}-c_{ij})^2 \Big)
- *   = \sum\limits_{i=1}^N \Big( \sum\limits_{j=1}^D x_{j}^2 +
- *   \sum\limits_{j=1}^D c_{ij}^2 -2\sum\limits_{j=1}^D x_{j}c_{ij} \Big)
- *   = \sum\limits_{i=1}^N \sum\limits_{j=1}^D x_{j}^2 +
- *   \sum\limits_{i=1}^N \sum\limits_{j=1}^D c_{ij}^2
- *   -2 \sum\limits_{i=1}^N \sum\limits_{j=1}^D x_{j}c_{ij}
- *   $$
- * </blockquote>
- *
- * where `$x_{j}$` is the `j`-th dimension of the point `X` and
- * `$c_{ij}$` is the `j`-th dimension of the `i`-th point in cluster `$\Gamma$`.
- *
- * Then, the first term of the equation can be rewritten as:
- *
- * <blockquote>
- *   $$
- *   \sum\limits_{i=1}^N \sum\limits_{j=1}^D x_{j}^2 = N \xi_{X} \text{ ,
- *   with } \xi_{X} = \sum\limits_{j=1}^D x_{j}^2
- *   $$
- * </blockquote>
- *
- * where `$\xi_{X}$` is fixed for each point and it can be precomputed.
- *
- * Moreover, the second term is fixed for each cluster too,
- * thus we can name it `$\Psi_{\Gamma}$`
- *
- * <blockquote>
- *   $$
- *   \sum\limits_{i=1}^N \sum\limits_{j=1}^D c_{ij}^2 =
- *   \sum\limits_{i=1}^N \xi_{C_{i}} = \Psi_{\Gamma}
- *   $$
- * </blockquote>
- *
- * Last, the third element becomes
- *
- * <blockquote>
- *   $$
- *   \sum\limits_{i=1}^N \sum\limits_{j=1}^D x_{j}c_{ij} =
- *   \sum\limits_{j=1}^D \Big(\sum\limits_{i=1}^N c_{ij} \Big) x_{j}
- *   $$
- * </blockquote>
- *
- * thus defining the vector
- *
- * <blockquote>
- *   $$
- *   Y_{\Gamma}:Y_{\Gamma j} = \sum\limits_{i=1}^N c_{ij} , j=0, ..., D
- *   $$
- * </blockquote>
- *
- * which is fixed for each cluster `$\Gamma$`, we have
- *
- * <blockquote>
- *   $$
- *   \sum\limits_{j=1}^D \Big(\sum\limits_{i=1}^N c_{ij} \Big) x_{j} =
- *   \sum\limits_{j=1}^D Y_{\Gamma j} x_{j}
- *   $$
- * </blockquote>
- *
- * In this way, the previous equation becomes
- *
- * <blockquote>
- *   $$
- *   N\xi_{X} + \Psi_{\Gamma} - 2 \sum\limits_{j=1}^D Y_{\Gamma j} x_{j}
- *   $$
- * </blockquote>
- *
- * and the average distance of a point to a cluster can be computed as
- *
- * <blockquote>
- *   $$
- *   \frac{\sum\limits_{i=1}^N d(X, C_{i} )}{N} =
- *   \frac{N\xi_{X} + \Psi_{\Gamma} - 2 \sum\limits_{j=1}^D Y_{\Gamma j} x_{j}}{N} =
- *   \xi_{X} + \frac{\Psi_{\Gamma} }{N} - 2 \frac{\sum\limits_{j=1}^D Y_{\Gamma j} x_{j}}{N}
- *   $$
- * </blockquote>
- *
- * Thus, it is enough to precompute: the constant `$\xi_{X}$` for each point `X`; the
- * constants `$\Psi_{\Gamma}$`, `N` and the vector `$Y_{\Gamma}$` for
- * each cluster `$\Gamma$`.
- *
- * In the implementation, the precomputed values for the clusters
- * are distributed among the worker nodes via broadcasted variables,
- * because we can assume that the clusters are limited in number and
- * anyway they are much fewer than the points.
- *
- * The main strengths of this algorithm are the low computational complexity
- * and the intrinsic parallelism. The precomputed information for each point
- * and for each cluster can be computed with a computational complexity
- * which is `O(N/W)`, where `N` is the number of points in the dataset and
- * `W` is the number of worker nodes. After that, every point can be
- * analyzed independently of the others.
- *
- * For every point we need to compute the average distance to all the clusters.
- * Since the formula above requires `O(D)` operations, this phase has a
- * computational complexity which is `O(C*D*N/W)` where `C` is the number of
- * clusters (which we assume quite low), `D` is the number of dimensions,
- * `N` is the number of points in the dataset and `W` is the number
- * of worker nodes.
- */
-private[evaluation] object SquaredEuclideanSilhouette extends Silhouette {
-
-  private[this] var kryoRegistrationPerformed: Boolean = false
-
   /**
-   * This method registers the class
-   * [[org.apache.spark.ml.evaluation.SquaredEuclideanSilhouette.ClusterStats]]
-   * for kryo serialization.
+   * Get a ClusteringMetrics, which can be used to get clustering metrics such as
+   * silhouette score.
    *
-   * @param sc `SparkContext` to be used
+   * @param dataset a dataset that contains labels/observations and predictions.
+   * @return ClusteringMetrics
    */
-  def registerKryoClasses(sc: SparkContext): Unit = {
-    if (!kryoRegistrationPerformed) {
-      sc.getConf.registerKryoClasses(
-        Array(
-          classOf[SquaredEuclideanSilhouette.ClusterStats]
-        )
-      )
-      kryoRegistrationPerformed = true
+  @Since("3.1.0")
+  def getMetrics(dataset: Dataset[_]): ClusteringMetrics = {
+    val schema = dataset.schema
+    SchemaUtils.validateVectorCompatibleColumn(schema, $(featuresCol))
+    SchemaUtils.checkNumericType(schema, $(predictionCol))
+    if (isDefined(weightCol)) {
+      SchemaUtils.checkNumericType(schema, $(weightCol))
     }
-  }
 
-  case class ClusterStats(featureSum: Vector, squaredNormSum: Double, numOfPoints: Long)
-
-  /**
-   * The method takes the input dataset and computes the aggregated values
-   * about a cluster which are needed by the algorithm.
-   *
-   * @param df The DataFrame which contains the input data
-   * @param predictionCol The name of the column which contains the predicted cluster id
-   *                      for the point.
-   * @param featuresCol The name of the column which contains the feature vector of the point.
-   * @return A [[scala.collection.immutable.Map]] which associates each cluster id
-   *         to a [[ClusterStats]] object (which contains the precomputed values `N`,
-   *         `$\Psi_{\Gamma}$` and `$Y_{\Gamma}$` for a cluster).
-   */
-  def computeClusterStats(
-    df: DataFrame,
-    predictionCol: String,
-    featuresCol: String): Map[Double, ClusterStats] = {
-    val numFeatures = MetadataUtils.getNumFeatures(df, featuresCol)
-    val clustersStatsRDD = df.select(
-        col(predictionCol).cast(DoubleType), col(featuresCol), col("squaredNorm"))
-      .rdd
-      .map { row => (row.getDouble(0), (row.getAs[Vector](1), row.getDouble(2))) }
-      .aggregateByKey[(DenseVector, Double, Long)]((Vectors.zeros(numFeatures).toDense, 0.0, 0L))(
-        seqOp = {
-          case (
-              (featureSum: DenseVector, squaredNormSum: Double, numOfPoints: Long),
-              (features, squaredNorm)
-            ) =>
-            BLAS.axpy(1.0, features, featureSum)
-            (featureSum, squaredNormSum + squaredNorm, numOfPoints + 1)
-        },
-        combOp = {
-          case (
-              (featureSum1, squaredNormSum1, numOfPoints1),
-              (featureSum2, squaredNormSum2, numOfPoints2)
-            ) =>
-            BLAS.axpy(1.0, featureSum2, featureSum1)
-            (featureSum1, squaredNormSum1 + squaredNormSum2, numOfPoints1 + numOfPoints2)
-        }
-      )
-
-    clustersStatsRDD
-      .collectAsMap()
-      .mapValues {
-        case (featureSum: DenseVector, squaredNormSum: Double, numOfPoints: Long) =>
-          SquaredEuclideanSilhouette.ClusterStats(featureSum, squaredNormSum, numOfPoints)
-      }
-      .toMap
-  }
+    val weightColName = if (!isDefined(weightCol)) "weightCol" else $(weightCol)
 
-  /**
-   * It computes the Silhouette coefficient for a point.
-   *
-   * @param broadcastedClustersMap A map of the precomputed values for each cluster.
-   * @param point The [[org.apache.spark.ml.linalg.Vector]] representing the current point.
-   * @param clusterId The id of the cluster the current point belongs to.
-   * @param squaredNorm The `$\Xi_{X}$` (which is the squared norm) precomputed for the point.
-   * @return The Silhouette for the point.
-   */
-  def computeSilhouetteCoefficient(
-     broadcastedClustersMap: Broadcast[Map[Double, ClusterStats]],
-     point: Vector,
-     clusterId: Double,
-     squaredNorm: Double): Double = {
-
-    def compute(targetClusterId: Double): Double = {
-      val clusterStats = broadcastedClustersMap.value(targetClusterId)
-      val pointDotClusterFeaturesSum = BLAS.dot(point, clusterStats.featureSum)
-
-      squaredNorm +
-        clusterStats.squaredNormSum / clusterStats.numOfPoints -
-        2 * pointDotClusterFeaturesSum / clusterStats.numOfPoints
+    val vectorCol = DatasetUtils.columnToVector(dataset, $(featuresCol))
+    val df = if (!isDefined(weightCol) || $(weightCol).isEmpty) {
+      dataset.select(col($(predictionCol)),
+        vectorCol.as($(featuresCol), dataset.schema($(featuresCol)).metadata),
+        lit(1.0).as(weightColName))
+    } else {
+      dataset.select(col($(predictionCol)),
+        vectorCol.as($(featuresCol), dataset.schema($(featuresCol)).metadata),
+        checkNonNegativeWeight(col(weightColName).cast(DoubleType)))
     }
 
-    pointSilhouetteCoefficient(broadcastedClustersMap.value.keySet,
-      clusterId,
-      broadcastedClustersMap.value(clusterId).numOfPoints,
-      compute)
+    val metrics = new ClusteringMetrics(df)
+    metrics.setDistanceMeasure($(distanceMeasure))
+    metrics
   }
 
-  /**
-   * Compute the Silhouette score of the dataset using squared Euclidean distance measure.
-   *
-   * @param dataset The input dataset (previously clustered) on which compute the Silhouette.
-   * @param predictionCol The name of the column which contains the predicted cluster id
-   *                      for the point.
-   * @param featuresCol The name of the column which contains the feature vector of the point.
-   * @return The average of the Silhouette values of the clustered data.
-   */
-  def computeSilhouetteScore(
-      dataset: Dataset[_],
-      predictionCol: String,
-      featuresCol: String): Double = {
-    SquaredEuclideanSilhouette.registerKryoClasses(dataset.sparkSession.sparkContext)
-
-    val squaredNormUDF = udf {
-      features: Vector => math.pow(Vectors.norm(features, 2.0), 2.0)
-    }
-    val dfWithSquaredNorm = dataset.withColumn("squaredNorm", squaredNormUDF(col(featuresCol)))
-
-    // compute aggregate values for clusters needed by the algorithm
-    val clustersStatsMap = SquaredEuclideanSilhouette
-      .computeClusterStats(dfWithSquaredNorm, predictionCol, featuresCol)
-
-    // Silhouette is reasonable only when the number of clusters is greater then 1
-    assert(clustersStatsMap.size > 1, "Number of clusters must be greater than one.")
-
-    val bClustersStatsMap = dataset.sparkSession.sparkContext.broadcast(clustersStatsMap)
-
-    val computeSilhouetteCoefficientUDF = udf {
-      computeSilhouetteCoefficient(bClustersStatsMap, _: Vector, _: Double, _: Double)
-    }
-
-    val silhouetteScore = overallScore(dfWithSquaredNorm,
-      computeSilhouetteCoefficientUDF(col(featuresCol), col(predictionCol).cast(DoubleType),
-        col("squaredNorm")))
-
-    bClustersStatsMap.destroy()
-
-    silhouetteScore
+  @Since("3.0.0")
+  override def toString: String = {
+    s"ClusteringEvaluator: uid=$uid, metricName=${$(metricName)}, " +
+      s"distanceMeasure=${$(distanceMeasure)}"
   }
 }
 
 
-/**
- * The algorithm which is implemented in this object, instead, is an efficient and parallel
- * implementation of the Silhouette using the cosine distance measure. The cosine distance
- * measure is defined as `1 - s` where `s` is the cosine similarity between two points.
- *
- * The total distance of the point `X` to the points `$C_{i}$` belonging to the cluster `$\Gamma$`
- * is:
- *
- * <blockquote>
- *   $$
- *   \sum\limits_{i=1}^N d(X, C_{i} ) =
- *   \sum\limits_{i=1}^N \Big( 1 - \frac{\sum\limits_{j=1}^D x_{j}c_{ij} }{ \|X\|\|C_{i}\|} \Big)
- *   = \sum\limits_{i=1}^N 1 - \sum\limits_{i=1}^N \sum\limits_{j=1}^D \frac{x_{j}}{\|X\|}
- *   \frac{c_{ij}}{\|C_{i}\|}
- *   = N - \sum\limits_{j=1}^D \frac{x_{j}}{\|X\|} \Big( \sum\limits_{i=1}^N
- *   \frac{c_{ij}}{\|C_{i}\|} \Big)
- *   $$
- * </blockquote>
- *
- * where `$x_{j}$` is the `j`-th dimension of the point `X` and `$c_{ij}$` is the `j`-th dimension
- * of the `i`-th point in cluster `$\Gamma$`.
- *
- * Then, we can define the vector:
- *
- * <blockquote>
- *   $$
- *   \xi_{X} : \xi_{X i} = \frac{x_{i}}{\|X\|}, i = 1, ..., D
- *   $$
- * </blockquote>
- *
- * which can be precomputed for each point and the vector
- *
- * <blockquote>
- *   $$
- *   \Omega_{\Gamma} : \Omega_{\Gamma i} = \sum\limits_{j=1}^N \xi_{C_{j}i}, i = 1, ..., D
- *   $$
- * </blockquote>
- *
- * which can be precomputed too for each cluster `$\Gamma$` by its points `$C_{i}$`.
- *
- * With these definitions, the numerator becomes:
- *
- * <blockquote>
- *   $$
- *   N - \sum\limits_{j=1}^D \xi_{X j} \Omega_{\Gamma j}
- *   $$
- * </blockquote>
- *
- * Thus the average distance of a point `X` to the points of the cluster `$\Gamma$` is:
- *
- * <blockquote>
- *   $$
- *   1 - \frac{\sum\limits_{j=1}^D \xi_{X j} \Omega_{\Gamma j}}{N}
- *   $$
- * </blockquote>
- *
- * In the implementation, the precomputed values for the clusters are distributed among the worker
- * nodes via broadcasted variables, because we can assume that the clusters are limited in number.
- *
- * The main strengths of this algorithm are the low computational complexity and the intrinsic
- * parallelism. The precomputed information for each point and for each cluster can be computed
- * with a computational complexity which is `O(N/W)`, where `N` is the number of points in the
- * dataset and `W` is the number of worker nodes. After that, every point can be analyzed
- * independently from the others.
- *
- * For every point we need to compute the average distance to all the clusters. Since the formula
- * above requires `O(D)` operations, this phase has a computational complexity which is
- * `O(C*D*N/W)` where `C` is the number of clusters (which we assume quite low), `D` is the number
- * of dimensions, `N` is the number of points in the dataset and `W` is the number of worker
- * nodes.
- */
-private[evaluation] object CosineSilhouette extends Silhouette {
-
-  private[this] val normalizedFeaturesColName = "normalizedFeatures"
-
-  /**
-   * The method takes the input dataset and computes the aggregated values
-   * about a cluster which are needed by the algorithm.
-   *
-   * @param df The DataFrame which contains the input data
-   * @param predictionCol The name of the column which contains the predicted cluster id
-   *                      for the point.
-   * @return A [[scala.collection.immutable.Map]] which associates each cluster id to a
-   *         its statistics (ie. the precomputed values `N` and `$\Omega_{\Gamma}$`).
-   */
-  def computeClusterStats(
-      df: DataFrame,
-      featuresCol: String,
-      predictionCol: String): Map[Double, (Vector, Long)] = {
-    val numFeatures = MetadataUtils.getNumFeatures(df, featuresCol)
-    val clustersStatsRDD = df.select(
-      col(predictionCol).cast(DoubleType), col(normalizedFeaturesColName))
-      .rdd
-      .map { row => (row.getDouble(0), row.getAs[Vector](1)) }
-      .aggregateByKey[(DenseVector, Long)]((Vectors.zeros(numFeatures).toDense, 0L))(
-      seqOp = {
-        case ((normalizedFeaturesSum: DenseVector, numOfPoints: Long), (normalizedFeatures)) =>
-          BLAS.axpy(1.0, normalizedFeatures, normalizedFeaturesSum)
-          (normalizedFeaturesSum, numOfPoints + 1)
-      },
-      combOp = {
-        case ((normalizedFeaturesSum1, numOfPoints1), (normalizedFeaturesSum2, numOfPoints2)) =>
-          BLAS.axpy(1.0, normalizedFeaturesSum2, normalizedFeaturesSum1)
-          (normalizedFeaturesSum1, numOfPoints1 + numOfPoints2)
-      }
-    )
-
-    clustersStatsRDD
-      .collectAsMap()
-      .toMap
-  }
-
-  /**
-   * It computes the Silhouette coefficient for a point.
-   *
-   * @param broadcastedClustersMap A map of the precomputed values for each cluster.
-   * @param normalizedFeatures The [[org.apache.spark.ml.linalg.Vector]] representing the
-   *                           normalized features of the current point.
-   * @param clusterId The id of the cluster the current point belongs to.
-   */
-  def computeSilhouetteCoefficient(
-      broadcastedClustersMap: Broadcast[Map[Double, (Vector, Long)]],
-      normalizedFeatures: Vector,
-      clusterId: Double): Double = {
-
-    def compute(targetClusterId: Double): Double = {
-      val (normalizedFeatureSum, numOfPoints) = broadcastedClustersMap.value(targetClusterId)
-      1 - BLAS.dot(normalizedFeatures, normalizedFeatureSum) / numOfPoints
-    }
-
-    pointSilhouetteCoefficient(broadcastedClustersMap.value.keySet,
-      clusterId,
-      broadcastedClustersMap.value(clusterId)._2,
-      compute)
-  }
-
-  /**
-   * Compute the Silhouette score of the dataset using the cosine distance measure.
-   *
-   * @param dataset The input dataset (previously clustered) on which compute the Silhouette.
-   * @param predictionCol The name of the column which contains the predicted cluster id
-   *                      for the point.
-   * @param featuresCol The name of the column which contains the feature vector of the point.
-   * @return The average of the Silhouette values of the clustered data.
-   */
-  def computeSilhouetteScore(
-      dataset: Dataset[_],
-      predictionCol: String,
-      featuresCol: String): Double = {
-    val normalizeFeatureUDF = udf {
-      features: Vector => {
-        val norm = Vectors.norm(features, 2.0)
-        BLAS.scal(1.0 / norm, features)
-        features
-      }
-    }
-    val dfWithNormalizedFeatures = dataset.withColumn(normalizedFeaturesColName,
-      normalizeFeatureUDF(col(featuresCol)))
-
-    // compute aggregate values for clusters needed by the algorithm
-    val clustersStatsMap = computeClusterStats(dfWithNormalizedFeatures, featuresCol,
-      predictionCol)
-
-    // Silhouette is reasonable only when the number of clusters is greater then 1
-    assert(clustersStatsMap.size > 1, "Number of clusters must be greater than one.")
-
-    val bClustersStatsMap = dataset.sparkSession.sparkContext.broadcast(clustersStatsMap)
-
-    val computeSilhouetteCoefficientUDF = udf {
-      computeSilhouetteCoefficient(bClustersStatsMap, _: Vector, _: Double)
-    }
-
-    val silhouetteScore = overallScore(dfWithNormalizedFeatures,
-      computeSilhouetteCoefficientUDF(col(normalizedFeaturesColName),
-        col(predictionCol).cast(DoubleType)))
+@Since("2.3.0")
+object ClusteringEvaluator
+  extends DefaultParamsReadable[ClusteringEvaluator] {
 
-    bClustersStatsMap.destroy()
+  @Since("2.3.0")
+  override def load(path: String): ClusteringEvaluator = super.load(path)
 
-    silhouetteScore
-  }
 }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/ClusteringMetrics.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/ClusteringMetrics.scala
new file mode 100644
index 0000000000000..3035688709301
--- /dev/null
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/ClusteringMetrics.scala
@@ -0,0 +1,598 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.evaluation
+
+import org.apache.spark.SparkContext
+import org.apache.spark.annotation.Since
+import org.apache.spark.broadcast.Broadcast
+import org.apache.spark.ml.linalg.{BLAS, DenseVector, Vector, Vectors}
+import org.apache.spark.ml.util.MetadataUtils
+import org.apache.spark.sql.{Column, DataFrame, Dataset}
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql.types.DoubleType
+
+
+/**
+ * Metrics for clustering, which expects two input columns: prediction and label.
+ */
+@Since("3.1.0")
+class ClusteringMetrics private[spark](dataset: Dataset[_]) {
+
+  private var distanceMeasure: String = "squaredEuclidean"
+
+  def getDistanceMeasure: String = distanceMeasure
+
+  def setDistanceMeasure(value: String) : this.type = {
+    require(value.equalsIgnoreCase("squaredEuclidean") ||
+      value.equalsIgnoreCase("cosine"))
+    distanceMeasure = value
+    this
+  }
+
+  /**
+   * Returns the silhouette score
+   */
+  @Since("3.1.0")
+  def silhouette(): Double = {
+    val columns = dataset.columns.toSeq
+    if (distanceMeasure.equalsIgnoreCase("squaredEuclidean")) {
+      SquaredEuclideanSilhouette.computeSilhouetteScore(
+        dataset, columns(0), columns(1), columns(2))
+    } else {
+      CosineSilhouette.computeSilhouetteScore(dataset, columns(0), columns(1), columns(2))
+    }
+  }
+}
+
+
+private[evaluation] abstract class Silhouette {
+
+  /**
+   * It computes the Silhouette coefficient for a point.
+   */
+  def pointSilhouetteCoefficient(
+      clusterIds: Set[Double],
+      pointClusterId: Double,
+      weightSum: Double,
+      weight: Double,
+      averageDistanceToCluster: (Double) => Double): Double = {
+    if (weightSum == weight) {
+      // Single-element clusters have silhouette 0
+      0.0
+    } else {
+      // Here we compute the average dissimilarity of the current point to any cluster of which the
+      // point is not a member.
+      // The cluster with the lowest average dissimilarity - i.e. the nearest cluster to the current
+      // point - is said to be the "neighboring cluster".
+      val otherClusterIds = clusterIds.filter(_ != pointClusterId)
+      val neighboringClusterDissimilarity = otherClusterIds.map(averageDistanceToCluster).min
+      // adjustment for excluding the node itself from the computation of the average dissimilarity
+      val currentClusterDissimilarity =
+      averageDistanceToCluster(pointClusterId) * weightSum /
+        (weightSum - weight)
+      if (currentClusterDissimilarity < neighboringClusterDissimilarity) {
+        1 - (currentClusterDissimilarity / neighboringClusterDissimilarity)
+      } else if (currentClusterDissimilarity > neighboringClusterDissimilarity) {
+        (neighboringClusterDissimilarity / currentClusterDissimilarity) - 1
+      } else {
+        0.0
+      }
+    }
+  }
+
+  /**
+   * Compute the mean Silhouette values of all samples.
+   */
+  def overallScore(df: DataFrame, scoreColumn: Column, weightColumn: Column): Double = {
+    df.select(sum(scoreColumn * weightColumn) / sum(weightColumn)).collect()(0).getDouble(0)
+  }
+}
+
+/**
+ * SquaredEuclideanSilhouette computes the average of the
+ * Silhouette over all the data of the dataset, which is
+ * a measure of how appropriately the data have been clustered.
+ *
+ * The Silhouette for each point `i` is defined as:
+ *
+ * <blockquote>
+ *   $$
+ *   s_{i} = \frac{b_{i}-a_{i}}{max\{a_{i},b_{i}\}}
+ *   $$
+ * </blockquote>
+ *
+ * which can be rewritten as
+ *
+ * <blockquote>
+ *   $$
+ *   s_{i}= \begin{cases}
+ *   1-\frac{a_{i}}{b_{i}} & \text{if } a_{i} \leq b_{i} \\
+ *   \frac{b_{i}}{a_{i}}-1 & \text{if } a_{i} \gt b_{i} \end{cases}
+ *   $$
+ * </blockquote>
+ *
+ * where `$a_{i}$` is the average dissimilarity of `i` with all other data
+ * within the same cluster, `$b_{i}$` is the lowest average dissimilarity
+ * of `i` to any other cluster, of which `i` is not a member.
+ * `$a_{i}$` can be interpreted as how well `i` is assigned to its cluster
+ * (the smaller the value, the better the assignment), while `$b_{i}$` is
+ * a measure of how well `i` has not been assigned to its "neighboring cluster",
+ * i.e. the nearest cluster to `i`.
+ *
+ * Unfortunately, the naive implementation of the algorithm requires to compute
+ * the distance of each couple of points in the dataset. Since the computation of
+ * the distance measure takes `D` operations - if `D` is the number of dimensions
+ * of each point, the computational complexity of the algorithm is `O(N^2^*D)`, where
+ * `N` is the cardinality of the dataset. Of course this is not scalable in `N`,
+ * which is the critical number in a Big Data context.
+ *
+ * The algorithm which is implemented in this object, instead, is an efficient
+ * and parallel implementation of the Silhouette using the squared Euclidean
+ * distance measure.
+ *
+ * With this assumption, the total distance of the point `X`
+ * to the points `$C_{i}$` belonging to the cluster `$\Gamma$` is:
+ *
+ * <blockquote>
+ *   $$
+ *   \sum\limits_{i=1}^N d(X, C_{i} ) =
+ *   \sum\limits_{i=1}^N \Big( \sum\limits_{j=1}^D (x_{j}-c_{ij})^2 \Big)
+ *   = \sum\limits_{i=1}^N \Big( \sum\limits_{j=1}^D x_{j}^2 +
+ *   \sum\limits_{j=1}^D c_{ij}^2 -2\sum\limits_{j=1}^D x_{j}c_{ij} \Big)
+ *   = \sum\limits_{i=1}^N \sum\limits_{j=1}^D x_{j}^2 +
+ *   \sum\limits_{i=1}^N \sum\limits_{j=1}^D c_{ij}^2
+ *   -2 \sum\limits_{i=1}^N \sum\limits_{j=1}^D x_{j}c_{ij}
+ *   $$
+ * </blockquote>
+ *
+ * where `$x_{j}$` is the `j`-th dimension of the point `X` and
+ * `$c_{ij}$` is the `j`-th dimension of the `i`-th point in cluster `$\Gamma$`.
+ *
+ * Then, the first term of the equation can be rewritten as:
+ *
+ * <blockquote>
+ *   $$
+ *   \sum\limits_{i=1}^N \sum\limits_{j=1}^D x_{j}^2 = N \xi_{X} \text{ ,
+ *   with } \xi_{X} = \sum\limits_{j=1}^D x_{j}^2
+ *   $$
+ * </blockquote>
+ *
+ * where `$\xi_{X}$` is fixed for each point and it can be precomputed.
+ *
+ * Moreover, the second term is fixed for each cluster too,
+ * thus we can name it `$\Psi_{\Gamma}$`
+ *
+ * <blockquote>
+ *   $$
+ *   \sum\limits_{i=1}^N \sum\limits_{j=1}^D c_{ij}^2 =
+ *   \sum\limits_{i=1}^N \xi_{C_{i}} = \Psi_{\Gamma}
+ *   $$
+ * </blockquote>
+ *
+ * Last, the third element becomes
+ *
+ * <blockquote>
+ *   $$
+ *   \sum\limits_{i=1}^N \sum\limits_{j=1}^D x_{j}c_{ij} =
+ *   \sum\limits_{j=1}^D \Big(\sum\limits_{i=1}^N c_{ij} \Big) x_{j}
+ *   $$
+ * </blockquote>
+ *
+ * thus defining the vector
+ *
+ * <blockquote>
+ *   $$
+ *   Y_{\Gamma}:Y_{\Gamma j} = \sum\limits_{i=1}^N c_{ij} , j=0, ..., D
+ *   $$
+ * </blockquote>
+ *
+ * which is fixed for each cluster `$\Gamma$`, we have
+ *
+ * <blockquote>
+ *   $$
+ *   \sum\limits_{j=1}^D \Big(\sum\limits_{i=1}^N c_{ij} \Big) x_{j} =
+ *   \sum\limits_{j=1}^D Y_{\Gamma j} x_{j}
+ *   $$
+ * </blockquote>
+ *
+ * In this way, the previous equation becomes
+ *
+ * <blockquote>
+ *   $$
+ *   N\xi_{X} + \Psi_{\Gamma} - 2 \sum\limits_{j=1}^D Y_{\Gamma j} x_{j}
+ *   $$
+ * </blockquote>
+ *
+ * and the average distance of a point to a cluster can be computed as
+ *
+ * <blockquote>
+ *   $$
+ *   \frac{\sum\limits_{i=1}^N d(X, C_{i} )}{N} =
+ *   \frac{N\xi_{X} + \Psi_{\Gamma} - 2 \sum\limits_{j=1}^D Y_{\Gamma j} x_{j}}{N} =
+ *   \xi_{X} + \frac{\Psi_{\Gamma} }{N} - 2 \frac{\sum\limits_{j=1}^D Y_{\Gamma j} x_{j}}{N}
+ *   $$
+ * </blockquote>
+ *
+ * Thus, it is enough to precompute: the constant `$\xi_{X}$` for each point `X`; the
+ * constants `$\Psi_{\Gamma}$`, `N` and the vector `$Y_{\Gamma}$` for
+ * each cluster `$\Gamma$`.
+ *
+ * In the implementation, the precomputed values for the clusters
+ * are distributed among the worker nodes via broadcasted variables,
+ * because we can assume that the clusters are limited in number and
+ * anyway they are much fewer than the points.
+ *
+ * The main strengths of this algorithm are the low computational complexity
+ * and the intrinsic parallelism. The precomputed information for each point
+ * and for each cluster can be computed with a computational complexity
+ * which is `O(N/W)`, where `N` is the number of points in the dataset and
+ * `W` is the number of worker nodes. After that, every point can be
+ * analyzed independently of the others.
+ *
+ * For every point we need to compute the average distance to all the clusters.
+ * Since the formula above requires `O(D)` operations, this phase has a
+ * computational complexity which is `O(C*D*N/W)` where `C` is the number of
+ * clusters (which we assume quite low), `D` is the number of dimensions,
+ * `N` is the number of points in the dataset and `W` is the number
+ * of worker nodes.
+ */
+private[evaluation] object SquaredEuclideanSilhouette extends Silhouette {
+
+  private[this] var kryoRegistrationPerformed: Boolean = false
+
+  /**
+   * This method registers the class
+   * [[org.apache.spark.ml.evaluation.SquaredEuclideanSilhouette.ClusterStats]]
+   * for kryo serialization.
+   *
+   * @param sc `SparkContext` to be used
+   */
+  def registerKryoClasses(sc: SparkContext): Unit = {
+    if (!kryoRegistrationPerformed) {
+      sc.getConf.registerKryoClasses(
+        Array(
+          classOf[SquaredEuclideanSilhouette.ClusterStats]
+        )
+      )
+      kryoRegistrationPerformed = true
+    }
+  }
+
+  case class ClusterStats(featureSum: Vector, squaredNormSum: Double, weightSum: Double)
+
+  /**
+   * The method takes the input dataset and computes the aggregated values
+   * about a cluster which are needed by the algorithm.
+   *
+   * @param df The DataFrame which contains the input data
+   * @param predictionCol The name of the column which contains the predicted cluster id
+   *                      for the point.
+   * @param featuresCol The name of the column which contains the feature vector of the point.
+   * @param weightCol The name of the column which contains the instance weight.
+   * @return A [[scala.collection.immutable.Map]] which associates each cluster id
+   *         to a [[ClusterStats]] object (which contains the precomputed values `N`,
+   *         `$\Psi_{\Gamma}$` and `$Y_{\Gamma}$` for a cluster).
+   */
+  def computeClusterStats(
+      df: DataFrame,
+      predictionCol: String,
+      featuresCol: String,
+      weightCol: String): Map[Double, ClusterStats] = {
+    val numFeatures = MetadataUtils.getNumFeatures(df, featuresCol)
+    val clustersStatsRDD = df.select(
+      col(predictionCol).cast(DoubleType), col(featuresCol), col("squaredNorm"), col(weightCol))
+      .rdd
+      .map { row => (row.getDouble(0), (row.getAs[Vector](1), row.getDouble(2), row.getDouble(3))) }
+      .aggregateByKey
+      [(DenseVector, Double, Double)]((Vectors.zeros(numFeatures).toDense, 0.0, 0.0))(
+        seqOp = {
+          case (
+            (featureSum: DenseVector, squaredNormSum: Double, weightSum: Double),
+            (features, squaredNorm, weight)
+            ) =>
+            BLAS.axpy(weight, features, featureSum)
+            (featureSum, squaredNormSum + squaredNorm * weight, weightSum + weight)
+        },
+        combOp = {
+          case (
+            (featureSum1, squaredNormSum1, weightSum1),
+            (featureSum2, squaredNormSum2, weightSum2)
+            ) =>
+            BLAS.axpy(1.0, featureSum2, featureSum1)
+            (featureSum1, squaredNormSum1 + squaredNormSum2, weightSum1 + weightSum2)
+        }
+      )
+
+    clustersStatsRDD
+      .collectAsMap()
+      .mapValues {
+        case (featureSum: DenseVector, squaredNormSum: Double, weightSum: Double) =>
+          SquaredEuclideanSilhouette.ClusterStats(featureSum, squaredNormSum, weightSum)
+      }
+      .toMap
+  }
+
+  /**
+   * It computes the Silhouette coefficient for a point.
+   *
+   * @param broadcastedClustersMap A map of the precomputed values for each cluster.
+   * @param point The [[org.apache.spark.ml.linalg.Vector]] representing the current point.
+   * @param clusterId The id of the cluster the current point belongs to.
+   * @param weight The instance weight of the current point.
+   * @param squaredNorm The `$\Xi_{X}$` (which is the squared norm) precomputed for the point.
+   * @return The Silhouette for the point.
+   */
+  def computeSilhouetteCoefficient(
+      broadcastedClustersMap: Broadcast[Map[Double, ClusterStats]],
+      point: Vector,
+      clusterId: Double,
+      weight: Double,
+      squaredNorm: Double): Double = {
+
+    def compute(targetClusterId: Double): Double = {
+      val clusterStats = broadcastedClustersMap.value(targetClusterId)
+      val pointDotClusterFeaturesSum = BLAS.dot(point, clusterStats.featureSum)
+
+      squaredNorm +
+        clusterStats.squaredNormSum / clusterStats.weightSum -
+        2 * pointDotClusterFeaturesSum / clusterStats.weightSum
+    }
+
+    pointSilhouetteCoefficient(broadcastedClustersMap.value.keySet,
+      clusterId,
+      broadcastedClustersMap.value(clusterId).weightSum,
+      weight,
+      compute)
+  }
+
+  /**
+   * Compute the Silhouette score of the dataset using squared Euclidean distance measure.
+   *
+   * @param dataset The input dataset (previously clustered) on which compute the Silhouette.
+   * @param predictionCol The name of the column which contains the predicted cluster id
+   *                      for the point.
+   * @param featuresCol The name of the column which contains the feature vector of the point.
+   * @param weightCol The name of the column which contains instance weight.
+   * @return The average of the Silhouette values of the clustered data.
+   */
+  def computeSilhouetteScore(
+      dataset: Dataset[_],
+      predictionCol: String,
+      featuresCol: String,
+      weightCol: String): Double = {
+    SquaredEuclideanSilhouette.registerKryoClasses(dataset.sparkSession.sparkContext)
+
+    val squaredNormUDF = udf {
+      features: Vector => math.pow(Vectors.norm(features, 2.0), 2.0)
+    }
+    val dfWithSquaredNorm = dataset.withColumn("squaredNorm", squaredNormUDF(col(featuresCol)))
+
+    // compute aggregate values for clusters needed by the algorithm
+    val clustersStatsMap = SquaredEuclideanSilhouette
+      .computeClusterStats(dfWithSquaredNorm, predictionCol, featuresCol, weightCol)
+
+    // Silhouette is reasonable only when the number of clusters is greater then 1
+    assert(clustersStatsMap.size > 1, "Number of clusters must be greater than one.")
+
+    val bClustersStatsMap = dataset.sparkSession.sparkContext.broadcast(clustersStatsMap)
+
+    val computeSilhouetteCoefficientUDF = udf {
+      computeSilhouetteCoefficient(bClustersStatsMap, _: Vector, _: Double, _: Double, _: Double)
+    }
+
+    val silhouetteScore = overallScore(dfWithSquaredNorm,
+      computeSilhouetteCoefficientUDF(col(featuresCol), col(predictionCol).cast(DoubleType),
+        col(weightCol), col("squaredNorm")), col(weightCol))
+
+    bClustersStatsMap.destroy()
+
+    silhouetteScore
+  }
+}
+
+
+/**
+ * The algorithm which is implemented in this object, instead, is an efficient and parallel
+ * implementation of the Silhouette using the cosine distance measure. The cosine distance
+ * measure is defined as `1 - s` where `s` is the cosine similarity between two points.
+ *
+ * The total distance of the point `X` to the points `$C_{i}$` belonging to the cluster `$\Gamma$`
+ * is:
+ *
+ * <blockquote>
+ *   $$
+ *   \sum\limits_{i=1}^N d(X, C_{i} ) =
+ *   \sum\limits_{i=1}^N \Big( 1 - \frac{\sum\limits_{j=1}^D x_{j}c_{ij} }{ \|X\|\|C_{i}\|} \Big)
+ *   = \sum\limits_{i=1}^N 1 - \sum\limits_{i=1}^N \sum\limits_{j=1}^D \frac{x_{j}}{\|X\|}
+ *   \frac{c_{ij}}{\|C_{i}\|}
+ *   = N - \sum\limits_{j=1}^D \frac{x_{j}}{\|X\|} \Big( \sum\limits_{i=1}^N
+ *   \frac{c_{ij}}{\|C_{i}\|} \Big)
+ *   $$
+ * </blockquote>
+ *
+ * where `$x_{j}$` is the `j`-th dimension of the point `X` and `$c_{ij}$` is the `j`-th dimension
+ * of the `i`-th point in cluster `$\Gamma$`.
+ *
+ * Then, we can define the vector:
+ *
+ * <blockquote>
+ *   $$
+ *   \xi_{X} : \xi_{X i} = \frac{x_{i}}{\|X\|}, i = 1, ..., D
+ *   $$
+ * </blockquote>
+ *
+ * which can be precomputed for each point and the vector
+ *
+ * <blockquote>
+ *   $$
+ *   \Omega_{\Gamma} : \Omega_{\Gamma i} = \sum\limits_{j=1}^N \xi_{C_{j}i}, i = 1, ..., D
+ *   $$
+ * </blockquote>
+ *
+ * which can be precomputed too for each cluster `$\Gamma$` by its points `$C_{i}$`.
+ *
+ * With these definitions, the numerator becomes:
+ *
+ * <blockquote>
+ *   $$
+ *   N - \sum\limits_{j=1}^D \xi_{X j} \Omega_{\Gamma j}
+ *   $$
+ * </blockquote>
+ *
+ * Thus the average distance of a point `X` to the points of the cluster `$\Gamma$` is:
+ *
+ * <blockquote>
+ *   $$
+ *   1 - \frac{\sum\limits_{j=1}^D \xi_{X j} \Omega_{\Gamma j}}{N}
+ *   $$
+ * </blockquote>
+ *
+ * In the implementation, the precomputed values for the clusters are distributed among the worker
+ * nodes via broadcasted variables, because we can assume that the clusters are limited in number.
+ *
+ * The main strengths of this algorithm are the low computational complexity and the intrinsic
+ * parallelism. The precomputed information for each point and for each cluster can be computed
+ * with a computational complexity which is `O(N/W)`, where `N` is the number of points in the
+ * dataset and `W` is the number of worker nodes. After that, every point can be analyzed
+ * independently from the others.
+ *
+ * For every point we need to compute the average distance to all the clusters. Since the formula
+ * above requires `O(D)` operations, this phase has a computational complexity which is
+ * `O(C*D*N/W)` where `C` is the number of clusters (which we assume quite low), `D` is the number
+ * of dimensions, `N` is the number of points in the dataset and `W` is the number of worker
+ * nodes.
+ */
+private[evaluation] object CosineSilhouette extends Silhouette {
+
+  private[this] val normalizedFeaturesColName = "normalizedFeatures"
+
+  /**
+   * The method takes the input dataset and computes the aggregated values
+   * about a cluster which are needed by the algorithm.
+   *
+   * @param df The DataFrame which contains the input data
+   * @param featuresCol The name of the column which contains the feature vector of the point.
+   * @param predictionCol The name of the column which contains the predicted cluster id
+   *                      for the point.
+   * @param weightCol The name of the column which contains the instance weight.
+   * @return A [[scala.collection.immutable.Map]] which associates each cluster id to a
+   *         its statistics (i.e. the precomputed values `N` and `$\Omega_{\Gamma}$`).
+   */
+  def computeClusterStats(
+      df: DataFrame,
+      featuresCol: String,
+      predictionCol: String,
+      weightCol: String): Map[Double, (Vector, Double)] = {
+    val numFeatures = MetadataUtils.getNumFeatures(df, featuresCol)
+    val clustersStatsRDD = df.select(
+      col(predictionCol).cast(DoubleType), col(normalizedFeaturesColName), col(weightCol))
+      .rdd
+      .map { row => (row.getDouble(0), (row.getAs[Vector](1), row.getDouble(2))) }
+      .aggregateByKey[(DenseVector, Double)]((Vectors.zeros(numFeatures).toDense, 0.0))(
+      seqOp = {
+        case ((normalizedFeaturesSum: DenseVector, weightSum: Double),
+        (normalizedFeatures, weight)) =>
+          BLAS.axpy(weight, normalizedFeatures, normalizedFeaturesSum)
+          (normalizedFeaturesSum, weightSum + weight)
+      },
+      combOp = {
+        case ((normalizedFeaturesSum1, weightSum1), (normalizedFeaturesSum2, weightSum2)) =>
+          BLAS.axpy(1.0, normalizedFeaturesSum2, normalizedFeaturesSum1)
+          (normalizedFeaturesSum1, weightSum1 + weightSum2)
+      }
+    )
+
+    clustersStatsRDD
+      .collectAsMap()
+      .toMap
+  }
+
+  /**
+   * It computes the Silhouette coefficient for a point.
+   *
+   * @param broadcastedClustersMap A map of the precomputed values for each cluster.
+   * @param normalizedFeatures The [[org.apache.spark.ml.linalg.Vector]] representing the
+   *                           normalized features of the current point.
+   * @param clusterId The id of the cluster the current point belongs to.
+   * @param weight The instance weight of the current point.
+   */
+  def computeSilhouetteCoefficient(
+      broadcastedClustersMap: Broadcast[Map[Double, (Vector, Double)]],
+      normalizedFeatures: Vector,
+      clusterId: Double,
+      weight: Double): Double = {
+
+    def compute(targetClusterId: Double): Double = {
+      val (normalizedFeatureSum, numOfPoints) = broadcastedClustersMap.value(targetClusterId)
+      1 - BLAS.dot(normalizedFeatures, normalizedFeatureSum) / numOfPoints
+    }
+
+    pointSilhouetteCoefficient(broadcastedClustersMap.value.keySet,
+      clusterId,
+      broadcastedClustersMap.value(clusterId)._2,
+      weight,
+      compute)
+  }
+
+  /**
+   * Compute the Silhouette score of the dataset using the cosine distance measure.
+   *
+   * @param dataset The input dataset (previously clustered) on which compute the Silhouette.
+   * @param predictionCol The name of the column which contains the predicted cluster id
+   *                      for the point.
+   * @param featuresCol The name of the column which contains the feature vector of the point.
+   * @param weightCol The name of the column which contains the instance weight.
+   * @return The average of the Silhouette values of the clustered data.
+   */
+  def computeSilhouetteScore(
+      dataset: Dataset[_],
+      predictionCol: String,
+      featuresCol: String,
+      weightCol: String): Double = {
+    val normalizeFeatureUDF = udf {
+      features: Vector => {
+        val norm = Vectors.norm(features, 2.0)
+        BLAS.scal(1.0 / norm, features)
+        features
+      }
+    }
+    val dfWithNormalizedFeatures = dataset.withColumn(normalizedFeaturesColName,
+      normalizeFeatureUDF(col(featuresCol)))
+
+    // compute aggregate values for clusters needed by the algorithm
+    val clustersStatsMap = computeClusterStats(dfWithNormalizedFeatures, featuresCol,
+      predictionCol, weightCol)
+
+    // Silhouette is reasonable only when the number of clusters is greater then 1
+    assert(clustersStatsMap.size > 1, "Number of clusters must be greater than one.")
+
+    val bClustersStatsMap = dataset.sparkSession.sparkContext.broadcast(clustersStatsMap)
+
+    val computeSilhouetteCoefficientUDF = udf {
+      computeSilhouetteCoefficient(bClustersStatsMap, _: Vector, _: Double, _: Double)
+    }
+
+    val silhouetteScore = overallScore(dfWithNormalizedFeatures,
+      computeSilhouetteCoefficientUDF(col(normalizedFeaturesColName),
+        col(predictionCol).cast(DoubleType), col(weightCol)), col(weightCol))
+
+    bClustersStatsMap.destroy()
+
+    silhouetteScore
+  }
+}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/MulticlassClassificationEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/MulticlassClassificationEvaluator.scala
index 1d6540e970383..beeefde8c5fac 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/MulticlassClassificationEvaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/MulticlassClassificationEvaluator.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.ml.evaluation
 
 import org.apache.spark.annotation.Since
+import org.apache.spark.ml.functions.checkNonNegativeWeight
 import org.apache.spark.ml.linalg.Vector
 import org.apache.spark.ml.param._
 import org.apache.spark.ml.param.shared._
@@ -63,8 +64,6 @@ class MulticlassClassificationEvaluator @Since("1.5.0") (@Since("1.5.0") overrid
   @Since("1.5.0")
   def setMetricName(value: String): this.type = set(metricName, value)
 
-  setDefault(metricName -> "f1")
-
   /** @group setParam */
   @Since("1.5.0")
   def setPredictionCol(value: String): this.type = set(predictionCol, value)
@@ -104,8 +103,6 @@ class MulticlassClassificationEvaluator @Since("1.5.0") (@Since("1.5.0") overrid
   @Since("3.0.0")
   def setMetricLabel(value: Double): this.type = set(metricLabel, value)
 
-  setDefault(metricLabel -> 0.0)
-
   /**
    * The beta value, which controls precision vs recall weighting,
    * used in `"weightedFMeasure"`, `"fMeasureByLabel"`.
@@ -127,8 +124,6 @@ class MulticlassClassificationEvaluator @Since("1.5.0") (@Since("1.5.0") overrid
   @Since("3.0.0")
   def setBeta(value: Double): this.type = set(beta, value)
 
-  setDefault(beta -> 1.0)
-
   /**
    * param for eps. log-loss is undefined for p=0 or p=1, so probabilities are clipped to
    * max(eps, min(1 - eps, p)). Must be in range (0, 0.5). The default value is 1e-15.
@@ -149,23 +144,55 @@ class MulticlassClassificationEvaluator @Since("1.5.0") (@Since("1.5.0") overrid
   @Since("3.0.0")
   def setEps(value: Double): this.type = set(eps, value)
 
-  setDefault(eps -> 1e-15)
+  setDefault(metricName -> "f1", eps -> 1e-15, metricLabel -> 0.0, beta -> 1.0)
 
   @Since("2.0.0")
   override def evaluate(dataset: Dataset[_]): Double = {
+    val metrics = getMetrics(dataset)
+    $(metricName) match {
+      case "f1" => metrics.weightedFMeasure
+      case "accuracy" => metrics.accuracy
+      case "weightedPrecision" => metrics.weightedPrecision
+      case "weightedRecall" => metrics.weightedRecall
+      case "weightedTruePositiveRate" => metrics.weightedTruePositiveRate
+      case "weightedFalsePositiveRate" => metrics.weightedFalsePositiveRate
+      case "weightedFMeasure" => metrics.weightedFMeasure($(beta))
+      case "truePositiveRateByLabel" => metrics.truePositiveRate($(metricLabel))
+      case "falsePositiveRateByLabel" => metrics.falsePositiveRate($(metricLabel))
+      case "precisionByLabel" => metrics.precision($(metricLabel))
+      case "recallByLabel" => metrics.recall($(metricLabel))
+      case "fMeasureByLabel" => metrics.fMeasure($(metricLabel), $(beta))
+      case "hammingLoss" => metrics.hammingLoss
+      case "logLoss" => metrics.logLoss($(eps))
+    }
+  }
+
+  /**
+   * Get a MulticlassMetrics, which can be used to get multiclass classification
+   * metrics such as accuracy, weightedPrecision, etc.
+   *
+   * @param dataset a dataset that contains labels/observations and predictions.
+   * @return MulticlassMetrics
+   */
+  @Since("3.1.0")
+  def getMetrics(dataset: Dataset[_]): MulticlassMetrics = {
     val schema = dataset.schema
     SchemaUtils.checkColumnType(schema, $(predictionCol), DoubleType)
     SchemaUtils.checkNumericType(schema, $(labelCol))
 
     val w = if (isDefined(weightCol) && $(weightCol).nonEmpty) {
-      col($(weightCol)).cast(DoubleType)
+      checkNonNegativeWeight(col($(weightCol)).cast(DoubleType))
     } else {
       lit(1.0)
     }
 
-    val rdd = if ($(metricName) == "logLoss") {
+    if ($(metricName) == "logLoss") {
       // probabilityCol is only needed to compute logloss
-      require(isDefined(probabilityCol) && $(probabilityCol).nonEmpty)
+      require(schema.fieldNames.contains($(probabilityCol)),
+        "probabilityCol is needed to compute logloss")
+    }
+
+    val rdd = if (schema.fieldNames.contains($(probabilityCol))) {
       val p = DatasetUtils.columnToVector(dataset, $(probabilityCol))
       dataset.select(col($(predictionCol)), col($(labelCol)).cast(DoubleType), w, p)
         .rdd.map {
@@ -179,23 +206,7 @@ class MulticlassClassificationEvaluator @Since("1.5.0") (@Since("1.5.0") overrid
       }
     }
 
-    val metrics = new MulticlassMetrics(rdd)
-    $(metricName) match {
-      case "f1" => metrics.weightedFMeasure
-      case "accuracy" => metrics.accuracy
-      case "weightedPrecision" => metrics.weightedPrecision
-      case "weightedRecall" => metrics.weightedRecall
-      case "weightedTruePositiveRate" => metrics.weightedTruePositiveRate
-      case "weightedFalsePositiveRate" => metrics.weightedFalsePositiveRate
-      case "weightedFMeasure" => metrics.weightedFMeasure($(beta))
-      case "truePositiveRateByLabel" => metrics.truePositiveRate($(metricLabel))
-      case "falsePositiveRateByLabel" => metrics.falsePositiveRate($(metricLabel))
-      case "precisionByLabel" => metrics.precision($(metricLabel))
-      case "recallByLabel" => metrics.recall($(metricLabel))
-      case "fMeasureByLabel" => metrics.fMeasure($(metricLabel), $(beta))
-      case "hammingLoss" => metrics.hammingLoss
-      case "logLoss" => metrics.logLoss($(eps))
-    }
+    new MulticlassMetrics(rdd)
   }
 
   @Since("1.5.0")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.scala
index a8db5452bd56c..8ed26502407a8 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.scala
@@ -65,8 +65,6 @@ class MultilabelClassificationEvaluator @Since("3.0.0") (@Since("3.0.0") overrid
   @Since("3.0.0")
   def setMetricName(value: String): this.type = set(metricName, value)
 
-  setDefault(metricName -> "f1Measure")
-
   /**
    * param for the class whose metric will be computed in `"precisionByLabel"`, `"recallByLabel"`,
    * `"f1MeasureByLabel"`.
@@ -86,8 +84,6 @@ class MultilabelClassificationEvaluator @Since("3.0.0") (@Since("3.0.0") overrid
   /** @group setParam */
   def setMetricLabel(value: Double): this.type = set(metricLabel, value)
 
-  setDefault(metricLabel -> 0.0)
-
   /** @group setParam */
   @Since("3.0.0")
   def setPredictionCol(value: String): this.type = set(predictionCol, value)
@@ -96,20 +92,11 @@ class MultilabelClassificationEvaluator @Since("3.0.0") (@Since("3.0.0") overrid
   @Since("3.0.0")
   def setLabelCol(value: String): this.type = set(labelCol, value)
 
+  setDefault(metricLabel -> 0.0, metricName -> "f1Measure")
+
   @Since("3.0.0")
   override def evaluate(dataset: Dataset[_]): Double = {
-    val schema = dataset.schema
-    SchemaUtils.checkColumnTypes(schema, $(predictionCol),
-      Seq(ArrayType(DoubleType, false), ArrayType(DoubleType, true)))
-    SchemaUtils.checkColumnTypes(schema, $(labelCol),
-      Seq(ArrayType(DoubleType, false), ArrayType(DoubleType, true)))
-
-    val predictionAndLabels =
-      dataset.select(col($(predictionCol)), col($(labelCol)))
-        .rdd.map { row =>
-        (row.getSeq[Double](0).toArray, row.getSeq[Double](1).toArray)
-      }
-    val metrics = new MultilabelMetrics(predictionAndLabels)
+    val metrics = getMetrics(dataset)
     $(metricName) match {
       case "subsetAccuracy" => metrics.subsetAccuracy
       case "accuracy" => metrics.accuracy
@@ -126,6 +113,29 @@ class MultilabelClassificationEvaluator @Since("3.0.0") (@Since("3.0.0") overrid
     }
   }
 
+  /**
+   * Get a MultilabelMetrics, which can be used to get multilabel classification
+   * metrics such as accuracy, precision, precisionByLabel, etc.
+   *
+   * @param dataset a dataset that contains labels/observations and predictions.
+   * @return MultilabelMetrics
+   */
+  @Since("3.1.0")
+  def getMetrics(dataset: Dataset[_]): MultilabelMetrics = {
+    val schema = dataset.schema
+    SchemaUtils.checkColumnTypes(schema, $(predictionCol),
+      Seq(ArrayType(DoubleType, false), ArrayType(DoubleType, true)))
+    SchemaUtils.checkColumnTypes(schema, $(labelCol),
+      Seq(ArrayType(DoubleType, false), ArrayType(DoubleType, true)))
+
+    val predictionAndLabels =
+      dataset.select(col($(predictionCol)), col($(labelCol)))
+        .rdd.map { row =>
+        (row.getSeq[Double](0).toArray, row.getSeq[Double](1).toArray)
+      }
+    new MultilabelMetrics(predictionAndLabels)
+  }
+
   @Since("3.0.0")
   override def isLargerBetter: Boolean = {
     $(metricName) match {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/RankingEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/RankingEvaluator.scala
index c5dea6c177e21..01fb0599160b0 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/RankingEvaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/RankingEvaluator.scala
@@ -61,8 +61,6 @@ class RankingEvaluator @Since("3.0.0") (@Since("3.0.0") override val uid: String
   @Since("3.0.0")
   def setMetricName(value: String): this.type = set(metricName, value)
 
-  setDefault(metricName -> "meanAveragePrecision")
-
   /**
    * param for ranking position value used in `"meanAveragePrecisionAtK"`, `"precisionAtK"`,
    * `"ndcgAtK"`, `"recallAtK"`. Must be &gt; 0. The default value is 10.
@@ -83,8 +81,6 @@ class RankingEvaluator @Since("3.0.0") (@Since("3.0.0") override val uid: String
   @Since("3.0.0")
   def setK(value: Int): this.type = set(k, value)
 
-  setDefault(k -> 10)
-
   /** @group setParam */
   @Since("3.0.0")
   def setPredictionCol(value: String): this.type = set(predictionCol, value)
@@ -93,8 +89,29 @@ class RankingEvaluator @Since("3.0.0") (@Since("3.0.0") override val uid: String
   @Since("3.0.0")
   def setLabelCol(value: String): this.type = set(labelCol, value)
 
+  setDefault(k -> 10, metricName -> "meanAveragePrecision")
+
   @Since("3.0.0")
   override def evaluate(dataset: Dataset[_]): Double = {
+    val metrics = getMetrics(dataset)
+    $(metricName) match {
+      case "meanAveragePrecision" => metrics.meanAveragePrecision
+      case "meanAveragePrecisionAtK" => metrics.meanAveragePrecisionAt($(k))
+      case "precisionAtK" => metrics.precisionAt($(k))
+      case "ndcgAtK" => metrics.ndcgAt($(k))
+      case "recallAtK" => metrics.recallAt($(k))
+    }
+  }
+
+  /**
+   * Get a RankingMetrics, which can be used to get ranking metrics
+   * such as meanAveragePrecision, meanAveragePrecisionAtK, etc.
+   *
+   * @param dataset a dataset that contains labels/observations and predictions.
+   * @return RankingMetrics
+   */
+  @Since("3.1.0")
+  def getMetrics(dataset: Dataset[_]): RankingMetrics[Double] = {
     val schema = dataset.schema
     SchemaUtils.checkColumnTypes(schema, $(predictionCol),
       Seq(ArrayType(DoubleType, false), ArrayType(DoubleType, true)))
@@ -106,14 +123,7 @@ class RankingEvaluator @Since("3.0.0") (@Since("3.0.0") override val uid: String
         .rdd.map { row =>
         (row.getSeq[Double](0).toArray, row.getSeq[Double](1).toArray)
       }
-    val metrics = new RankingMetrics[Double](predictionAndLabels)
-    $(metricName) match {
-      case "meanAveragePrecision" => metrics.meanAveragePrecision
-      case "meanAveragePrecisionAtK" => metrics.meanAveragePrecisionAt($(k))
-      case "precisionAtK" => metrics.precisionAt($(k))
-      case "ndcgAtK" => metrics.ndcgAt($(k))
-      case "recallAtK" => metrics.recallAt($(k))
-    }
+    new RankingMetrics[Double](predictionAndLabels)
   }
 
   @Since("3.0.0")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/RegressionEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/RegressionEvaluator.scala
index 18a8dda0c76ef..902869cc681b8 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/RegressionEvaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/RegressionEvaluator.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.ml.evaluation
 
 import org.apache.spark.annotation.Since
+import org.apache.spark.ml.functions.checkNonNegativeWeight
 import org.apache.spark.ml.param.{BooleanParam, Param, ParamMap, ParamValidators}
 import org.apache.spark.ml.param.shared.{HasLabelCol, HasPredictionCol, HasWeightCol}
 import org.apache.spark.ml.util.{DefaultParamsReadable, DefaultParamsWritable, Identifiable, SchemaUtils}
@@ -79,8 +80,6 @@ final class RegressionEvaluator @Since("1.4.0") (@Since("1.4.0") override val ui
   @Since("3.0.0")
   def setThroughOrigin(value: Boolean): this.type = set(throughOrigin, value)
 
-  setDefault(throughOrigin -> false)
-
   /** @group setParam */
   @Since("1.4.0")
   def setPredictionCol(value: String): this.type = set(predictionCol, value)
@@ -93,28 +92,41 @@ final class RegressionEvaluator @Since("1.4.0") (@Since("1.4.0") override val ui
   @Since("3.0.0")
   def setWeightCol(value: String): this.type = set(weightCol, value)
 
-  setDefault(metricName -> "rmse")
+  setDefault(metricName -> "rmse", throughOrigin -> false)
 
   @Since("2.0.0")
   override def evaluate(dataset: Dataset[_]): Double = {
+    val metrics = getMetrics(dataset)
+    $(metricName) match {
+      case "rmse" => metrics.rootMeanSquaredError
+      case "mse" => metrics.meanSquaredError
+      case "r2" => metrics.r2
+      case "mae" => metrics.meanAbsoluteError
+      case "var" => metrics.explainedVariance
+    }
+  }
+
+  /**
+   * Get a RegressionMetrics, which can be used to get regression
+   * metrics such as rootMeanSquaredError, meanSquaredError, etc.
+   *
+   * @param dataset a dataset that contains labels/observations and predictions.
+   * @return RegressionMetrics
+   */
+  @Since("3.1.0")
+  def getMetrics(dataset: Dataset[_]): RegressionMetrics = {
     val schema = dataset.schema
     SchemaUtils.checkColumnTypes(schema, $(predictionCol), Seq(DoubleType, FloatType))
     SchemaUtils.checkNumericType(schema, $(labelCol))
 
     val predictionAndLabelsWithWeights = dataset
       .select(col($(predictionCol)).cast(DoubleType), col($(labelCol)).cast(DoubleType),
-        if (!isDefined(weightCol) || $(weightCol).isEmpty) lit(1.0) else col($(weightCol)))
+        if (!isDefined(weightCol) || $(weightCol).isEmpty) lit(1.0)
+        else checkNonNegativeWeight(col($(weightCol)).cast(DoubleType)))
       .rdd
       .map { case Row(prediction: Double, label: Double, weight: Double) =>
         (prediction, label, weight) }
-    val metrics = new RegressionMetrics(predictionAndLabelsWithWeights, $(throughOrigin))
-    $(metricName) match {
-      case "rmse" => metrics.rootMeanSquaredError
-      case "mse" => metrics.meanSquaredError
-      case "r2" => metrics.r2
-      case "mae" => metrics.meanAbsoluteError
-      case "var" => metrics.explainedVariance
-    }
+    new RegressionMetrics(predictionAndLabelsWithWeights, $(throughOrigin))
   }
 
   @Since("1.4.0")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/events.scala b/mllib/src/main/scala/org/apache/spark/ml/events.scala
index dc4be4dd9efda..f221183369dfd 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/events.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/events.scala
@@ -20,7 +20,7 @@ package org.apache.spark.ml
 import com.fasterxml.jackson.annotation.JsonIgnore
 
 import org.apache.spark.SparkContext
-import org.apache.spark.annotation.Unstable
+import org.apache.spark.annotation.Evolving
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.util.{MLReader, MLWriter}
 import org.apache.spark.scheduler.SparkListenerEvent
@@ -31,10 +31,8 @@ import org.apache.spark.sql.{DataFrame, Dataset}
  * after each operation (the event should document this).
  *
  * @note This is supported via [[Pipeline]] and [[PipelineModel]].
- * @note This is experimental and unstable. Do not use this unless you fully
- *   understand what `Unstable` means.
  */
-@Unstable
+@Evolving
 sealed trait MLEvent extends SparkListenerEvent {
   // Do not log ML events in event log. It should be revisited to see
   // how it works with history server.
@@ -44,7 +42,7 @@ sealed trait MLEvent extends SparkListenerEvent {
 /**
  * Event fired before `Transformer.transform`.
  */
-@Unstable
+@Evolving
 case class TransformStart() extends MLEvent {
   @JsonIgnore var transformer: Transformer = _
   @JsonIgnore var input: Dataset[_] = _
@@ -53,7 +51,7 @@ case class TransformStart() extends MLEvent {
 /**
  * Event fired after `Transformer.transform`.
  */
-@Unstable
+@Evolving
 case class TransformEnd() extends MLEvent {
   @JsonIgnore var transformer: Transformer = _
   @JsonIgnore var output: Dataset[_] = _
@@ -62,7 +60,7 @@ case class TransformEnd() extends MLEvent {
 /**
  * Event fired before `Estimator.fit`.
  */
-@Unstable
+@Evolving
 case class FitStart[M <: Model[M]]() extends MLEvent {
   @JsonIgnore var estimator: Estimator[M] = _
   @JsonIgnore var dataset: Dataset[_] = _
@@ -71,7 +69,7 @@ case class FitStart[M <: Model[M]]() extends MLEvent {
 /**
  * Event fired after `Estimator.fit`.
  */
-@Unstable
+@Evolving
 case class FitEnd[M <: Model[M]]() extends MLEvent {
   @JsonIgnore var estimator: Estimator[M] = _
   @JsonIgnore var model: M = _
@@ -80,7 +78,7 @@ case class FitEnd[M <: Model[M]]() extends MLEvent {
 /**
  * Event fired before `MLReader.load`.
  */
-@Unstable
+@Evolving
 case class LoadInstanceStart[T](path: String) extends MLEvent {
   @JsonIgnore var reader: MLReader[T] = _
 }
@@ -88,7 +86,7 @@ case class LoadInstanceStart[T](path: String) extends MLEvent {
 /**
  * Event fired after `MLReader.load`.
  */
-@Unstable
+@Evolving
 case class LoadInstanceEnd[T]() extends MLEvent {
   @JsonIgnore var reader: MLReader[T] = _
   @JsonIgnore var instance: T = _
@@ -97,7 +95,7 @@ case class LoadInstanceEnd[T]() extends MLEvent {
 /**
  * Event fired before `MLWriter.save`.
  */
-@Unstable
+@Evolving
 case class SaveInstanceStart(path: String) extends MLEvent {
   @JsonIgnore var writer: MLWriter = _
 }
@@ -105,7 +103,7 @@ case class SaveInstanceStart(path: String) extends MLEvent {
 /**
  * Event fired after `MLWriter.save`.
  */
-@Unstable
+@Evolving
 case class SaveInstanceEnd(path: String) extends MLEvent {
   @JsonIgnore var writer: MLWriter = _
 }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/ANOVASelector.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/ANOVASelector.scala
deleted file mode 100644
index 81ffd01caea9d..0000000000000
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/ANOVASelector.scala
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.ml.feature
-
-import org.apache.hadoop.fs.Path
-
-import org.apache.spark.annotation.Since
-import org.apache.spark.ml.param._
-import org.apache.spark.ml.stat.ANOVATest
-import org.apache.spark.ml.util._
-import org.apache.spark.sql.{DataFrame, Dataset}
-
-
-/**
- * ANOVA F-value Classification selector, which selects continuous features to use for predicting a
- * categorical label.
- * The selector supports different selection methods: `numTopFeatures`, `percentile`, `fpr`,
- * `fdr`, `fwe`.
- *  - `numTopFeatures` chooses a fixed number of top features according to a F value classification
- *     test.
- *  - `percentile` is similar but chooses a fraction of all features instead of a fixed number.
- *  - `fpr` chooses all features whose p-value are below a threshold, thus controlling the false
- *    positive rate of selection.
- *  - `fdr` uses the [Benjamini-Hochberg procedure]
- *    (https://en.wikipedia.org/wiki/False_discovery_rate#Benjamini.E2.80.93Hochberg_procedure)
- *    to choose all features whose false discovery rate is below a threshold.
- *  - `fwe` chooses all features whose p-values are below a threshold. The threshold is scaled by
- *    1/numFeatures, thus controlling the family-wise error rate of selection.
- * By default, the selection method is `numTopFeatures`, with the default number of top features
- * set to 50.
- */
-@Since("3.1.0")
-final class ANOVASelector @Since("3.1.0")(@Since("3.1.0") override val uid: String)
-  extends Selector[ANOVASelectorModel] {
-
-  @Since("3.1.0")
-  def this() = this(Identifiable.randomUID("ANOVASelector"))
-
-  /** @group setParam */
-  @Since("3.1.0")
-  override def setNumTopFeatures(value: Int): this.type = super.setNumTopFeatures(value)
-
-  /** @group setParam */
-  @Since("3.1.0")
-  override def setPercentile(value: Double): this.type = super.setPercentile(value)
-
-  /** @group setParam */
-  @Since("3.1.0")
-  override def setFpr(value: Double): this.type = super.setFpr(value)
-
-  /** @group setParam */
-  @Since("3.1.0")
-  override def setFdr(value: Double): this.type = super.setFdr(value)
-
-  /** @group setParam */
-  @Since("3.1.0")
-  override def setFwe(value: Double): this.type = super.setFwe(value)
-
-  /** @group setParam */
-  @Since("3.1.0")
-  override def setSelectorType(value: String): this.type = super.setSelectorType(value)
-
-  /** @group setParam */
-  @Since("3.1.0")
-  override def setFeaturesCol(value: String): this.type = super.setFeaturesCol(value)
-
-  /** @group setParam */
-  @Since("3.1.0")
-  override def setOutputCol(value: String): this.type = super.setOutputCol(value)
-
-  /** @group setParam */
-  @Since("3.1.0")
-  override def setLabelCol(value: String): this.type = super.setLabelCol(value)
-
-  /**
-   * get the SelectionTestResult for every feature against the label
-   */
-  protected[this] override def getSelectionTestResult(df: DataFrame): DataFrame = {
-    ANOVATest.test(df, getFeaturesCol, getLabelCol, true)
-  }
-
-  /**
-   * Create a new instance of concrete SelectorModel.
-   * @param indices The indices of the selected features
-   * @return A new SelectorModel instance
-   */
-  protected[this] def createSelectorModel(
-      uid: String,
-      indices: Array[Int]): ANOVASelectorModel = {
-    new ANOVASelectorModel(uid, indices)
-  }
-
-  @Since("3.1.0")
-  override def fit(dataset: Dataset[_]): ANOVASelectorModel = {
-    super.fit(dataset)
-  }
-
-  @Since("3.1.0")
-  override def copy(extra: ParamMap): this.type = defaultCopy(extra)
-}
-
-@Since("3.1.0")
-object ANOVASelector extends DefaultParamsReadable[ANOVASelector] {
-
-  @Since("3.1.0")
-  override def load(path: String): ANOVASelector = super.load(path)
-}
-
-/**
- * Model fitted by [[ANOVASelector]].
- */
-@Since("3.1.0")
-class ANOVASelectorModel private[ml](
-    @Since("3.1.0") override val uid: String,
-    @Since("3.1.0") override val selectedFeatures: Array[Int])
-  extends SelectorModel[ANOVASelectorModel] (uid, selectedFeatures) {
-
-  /** @group setParam */
-  @Since("3.1.0")
-  override def setFeaturesCol(value: String): this.type = super.setFeaturesCol(value)
-
-  /** @group setParam */
-  @Since("3.1.0")
-  override def setOutputCol(value: String): this.type = super.setOutputCol(value)
-
-  @Since("3.1.0")
-  override def copy(extra: ParamMap): ANOVASelectorModel = {
-    val copied = new ANOVASelectorModel(uid, selectedFeatures)
-      .setParent(parent)
-    copyValues(copied, extra)
-  }
-
-  @Since("3.1.0")
-  override def write: MLWriter = new ANOVASelectorModel.ANOVASelectorModelWriter(this)
-
-  @Since("3.1.0")
-  override def toString: String = {
-    s"ANOVASelectorModel: uid=$uid, numSelectedFeatures=${selectedFeatures.length}"
-  }
-}
-
-@Since("3.1.0")
-object ANOVASelectorModel extends MLReadable[ANOVASelectorModel] {
-
-  @Since("3.1.0")
-  override def read: MLReader[ANOVASelectorModel] = new ANOVASelectorModelReader
-
-  @Since("3.1.0")
-  override def load(path: String): ANOVASelectorModel = super.load(path)
-
-  private[ANOVASelectorModel] class ANOVASelectorModelWriter(
-      instance: ANOVASelectorModel) extends MLWriter {
-
-    private case class Data(selectedFeatures: Seq[Int])
-
-    override protected def saveImpl(path: String): Unit = {
-      DefaultParamsWriter.saveMetadata(instance, path, sc)
-      val data = Data(instance.selectedFeatures.toSeq)
-      val dataPath = new Path(path, "data").toString
-      sparkSession.createDataFrame(Seq(data)).repartition(1).write.parquet(dataPath)
-    }
-  }
-
-  private class ANOVASelectorModelReader extends MLReader[ANOVASelectorModel] {
-
-    /** Checked against metadata when loading model */
-    private val className = classOf[ANOVASelectorModel].getName
-
-    override def load(path: String): ANOVASelectorModel = {
-      val metadata = DefaultParamsReader.loadMetadata(path, sc, className)
-      val dataPath = new Path(path, "data").toString
-      val data = sparkSession.read.parquet(dataPath)
-        .select("selectedFeatures").head()
-      val selectedFeatures = data.getAs[Seq[Int]](0).toArray
-      val model = new ANOVASelectorModel(metadata.uid, selectedFeatures)
-      metadata.getAndSetParams(model)
-      model
-    }
-  }
-}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/Binarizer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/Binarizer.scala
index 5ed7619fce5dc..2ec7a8632e39d 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/Binarizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/Binarizer.scala
@@ -112,7 +112,7 @@ final class Binarizer @Since("1.4.0") (@Since("1.4.0") override val uid: String)
         (Seq($(inputCol)), Seq($(outputCol)), Seq($(threshold)))
       }
 
-    val ouputCols = inputColNames.zip(tds).map { case (inputColName, td) =>
+    val mappedOutputCols = inputColNames.zip(tds).map { case (inputColName, td) =>
       val binarizerUDF = dataset.schema(inputColName).dataType match {
         case DoubleType =>
           udf { in: Double => if (in > td) 1.0 else 0.0 }
@@ -147,8 +147,8 @@ final class Binarizer @Since("1.4.0") (@Since("1.4.0") override val uid: String)
       binarizerUDF(col(inputColName))
     }
 
-    val ouputMetadata = outputColNames.map(outputSchema(_).metadata)
-    dataset.withColumns(outputColNames, ouputCols, ouputMetadata)
+    val outputMetadata = outputColNames.map(outputSchema(_).metadata)
+    dataset.withColumns(outputColNames, mappedOutputCols, outputMetadata)
   }
 
   @Since("1.4.0")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/BucketedRandomProjectionLSH.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/BucketedRandomProjectionLSH.scala
index 4e266fbc1ec13..16d711197fdde 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/BucketedRandomProjectionLSH.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/BucketedRandomProjectionLSH.scala
@@ -19,7 +19,6 @@ package org.apache.spark.ml.feature
 
 import scala.util.Random
 
-import breeze.linalg.normalize
 import org.apache.hadoop.fs.Path
 
 import org.apache.spark.annotation.Since
@@ -60,14 +59,20 @@ private[ml] trait BucketedRandomProjectionLSHParams extends Params {
  * where `r_i` is the i-th random unit vector. The number of buckets will be `(max L2 norm of input
  * vectors) / bucketLength`.
  *
- * @param randUnitVectors An array of random unit vectors. Each vector represents a hash function.
+ * @param randMatrix A matrix with each row representing a hash function.
  */
 @Since("2.1.0")
 class BucketedRandomProjectionLSHModel private[ml](
     override val uid: String,
-    private[ml] val randUnitVectors: Array[Vector])
+    private[ml] val randMatrix: Matrix)
   extends LSHModel[BucketedRandomProjectionLSHModel] with BucketedRandomProjectionLSHParams {
 
+  private[ml] def this(uid: String, randUnitVectors: Array[Vector]) = {
+    this(uid, Matrices.fromVectors(randUnitVectors))
+  }
+
+  private[ml] def randUnitVectors: Array[Vector] = randMatrix.rowIter.toArray
+
   /** @group setParam */
   @Since("2.4.0")
   override def setInputCol(value: String): this.type = super.set(inputCol, value)
@@ -78,11 +83,10 @@ class BucketedRandomProjectionLSHModel private[ml](
 
   @Since("2.1.0")
   override protected[ml] def hashFunction(elems: Vector): Array[Vector] = {
-    val hashValues = randUnitVectors.map(
-      randUnitVector => Math.floor(BLAS.dot(elems, randUnitVector) / $(bucketLength))
-    )
+    val hashVec = new DenseVector(Array.ofDim[Double](randMatrix.numRows))
+    BLAS.gemv(1.0 / $(bucketLength), randMatrix, elems, 0.0, hashVec)
     // TODO: Output vectors of dimension numHashFunctions in SPARK-18450
-    hashValues.map(Vectors.dense(_))
+    hashVec.values.map(h => Vectors.dense(h.floor))
   }
 
   @Since("2.1.0")
@@ -91,14 +95,30 @@ class BucketedRandomProjectionLSHModel private[ml](
   }
 
   @Since("2.1.0")
-  override protected[ml] def hashDistance(x: Seq[Vector], y: Seq[Vector]): Double = {
+  override protected[ml] def hashDistance(x: Array[Vector], y: Array[Vector]): Double = {
     // Since it's generated by hashing, it will be a pair of dense vectors.
-    x.zip(y).map(vectorPair => Vectors.sqdist(vectorPair._1, vectorPair._2)).min
+    var distance = Double.MaxValue
+    var i = 0
+    while (i < x.length) {
+      val vx = x(i).toArray
+      val vy = y(i).toArray
+      var j = 0
+      var d = 0.0
+      while (j < vx.length && d < distance) {
+        val diff = vx(j) - vy(j)
+        d += diff * diff
+        j += 1
+      }
+      if (d == 0) return 0.0
+      if (d < distance) distance = d
+      i += 1
+    }
+    distance
   }
 
   @Since("2.1.0")
   override def copy(extra: ParamMap): BucketedRandomProjectionLSHModel = {
-    val copied = new BucketedRandomProjectionLSHModel(uid, randUnitVectors).setParent(parent)
+    val copied = new BucketedRandomProjectionLSHModel(uid, randMatrix).setParent(parent)
     copyValues(copied, extra)
   }
 
@@ -144,9 +164,7 @@ class BucketedRandomProjectionLSH(override val uid: String)
   override def setNumHashTables(value: Int): this.type = super.setNumHashTables(value)
 
   @Since("2.1.0")
-  def this() = {
-    this(Identifiable.randomUID("brp-lsh"))
-  }
+  def this() = this(Identifiable.randomUID("brp-lsh"))
 
   /** @group setParam */
   @Since("2.1.0")
@@ -159,14 +177,18 @@ class BucketedRandomProjectionLSH(override val uid: String)
   @Since("2.1.0")
   override protected[this] def createRawLSHModel(
     inputDim: Int): BucketedRandomProjectionLSHModel = {
-    val rand = new Random($(seed))
-    val randUnitVectors: Array[Vector] = {
-      Array.fill($(numHashTables)) {
-        val randArray = Array.fill(inputDim)(rand.nextGaussian())
-        Vectors.fromBreeze(normalize(breeze.linalg.Vector(randArray)))
-      }
+    val rng = new Random($(seed))
+    val localNumHashTables = $(numHashTables)
+    val values = Array.fill(localNumHashTables * inputDim)(rng.nextGaussian)
+    var i = 0
+    while (i < localNumHashTables) {
+      val offset = i * inputDim
+      val norm = BLAS.javaBLAS.dnrm2(inputDim, values, offset, 1)
+      if (norm != 0) BLAS.javaBLAS.dscal(inputDim, 1.0 / norm, values, offset, 1)
+      i += 1
     }
-    new BucketedRandomProjectionLSHModel(uid, randUnitVectors)
+    val randMatrix = new DenseMatrix(localNumHashTables, inputDim, values, true)
+    new BucketedRandomProjectionLSHModel(uid, randMatrix)
   }
 
   @Since("2.1.0")
@@ -205,12 +227,7 @@ object BucketedRandomProjectionLSHModel extends MLReadable[BucketedRandomProject
 
     override protected def saveImpl(path: String): Unit = {
       DefaultParamsWriter.saveMetadata(instance, path, sc)
-      val numRows = instance.randUnitVectors.length
-      require(numRows > 0)
-      val numCols = instance.randUnitVectors.head.size
-      val values = instance.randUnitVectors.map(_.toArray).reduce(Array.concat(_, _))
-      val randMatrix = Matrices.dense(numRows, numCols, values)
-      val data = Data(randMatrix)
+      val data = Data(instance.randMatrix)
       val dataPath = new Path(path, "data").toString
       sparkSession.createDataFrame(Seq(data)).repartition(1).write.parquet(dataPath)
     }
@@ -227,11 +244,10 @@ object BucketedRandomProjectionLSHModel extends MLReadable[BucketedRandomProject
 
       val dataPath = new Path(path, "data").toString
       val data = sparkSession.read.parquet(dataPath)
-      val Row(randUnitVectors: Matrix) = MLUtils.convertMatrixColumnsToML(data, "randUnitVectors")
+      val Row(randMatrix: Matrix) = MLUtils.convertMatrixColumnsToML(data, "randUnitVectors")
         .select("randUnitVectors")
         .head()
-      val model = new BucketedRandomProjectionLSHModel(metadata.uid,
-        randUnitVectors.rowIter.toArray)
+      val model = new BucketedRandomProjectionLSHModel(metadata.uid, randMatrix)
 
       metadata.getAndSetParams(model)
       model
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/ChiSqSelector.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/ChiSqSelector.scala
index 7f83b69ae73ac..fc6c6157f850b 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/ChiSqSelector.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/ChiSqSelector.scala
@@ -44,6 +44,7 @@ import org.apache.spark.sql.types.StructType
  * By default, the selection method is `numTopFeatures`, with the default number of top features
  * set to 50.
  */
+@deprecated("use UnivariateFeatureSelector instead", "3.1.1")
 @Since("1.6.0")
 final class ChiSqSelector @Since("1.6.0") (@Since("1.6.0") override val uid: String)
   extends Selector[ChiSqSelectorModel] {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/CountVectorizer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/CountVectorizer.scala
index 6d39f18df60ea..fd07073c306e3 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/CountVectorizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/CountVectorizer.scala
@@ -241,7 +241,10 @@ class CountVectorizer @Since("1.5.0") (@Since("1.5.0") override val uid: String)
     }
     wordCounts.unpersist()
 
-    require(vocab.length > 0, "The vocabulary size should be > 0. Lower minDF as necessary.")
+    if (vocab.isEmpty) {
+      this.logWarning("The vocabulary size is empty. " +
+        "If this was unexpected, you may wish to lower minDF (or) increase maxDF.")
+    }
     copyValues(new CountVectorizerModel(uid, vocab).setParent(this))
   }
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/FValueSelector.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/FValueSelector.scala
deleted file mode 100644
index d177555c5ddee..0000000000000
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/FValueSelector.scala
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.ml.feature
-
-import org.apache.hadoop.fs.Path
-
-import org.apache.spark.annotation.Since
-import org.apache.spark.ml.param.ParamMap
-import org.apache.spark.ml.stat.FValueTest
-import org.apache.spark.ml.util._
-import org.apache.spark.sql.{DataFrame, Dataset}
-
-
-/**
- * F Value Regression feature selector, which selects continuous features to use for predicting a
- * continuous label.
- * The selector supports different selection methods: `numTopFeatures`, `percentile`, `fpr`,
- * `fdr`, `fwe`.
- *  - `numTopFeatures` chooses a fixed number of top features according to a F value regression
- *  test.
- *  - `percentile` is similar but chooses a fraction of all features instead of a fixed number.
- *  - `fpr` chooses all features whose p-value are below a threshold, thus controlling the false
- *    positive rate of selection.
- *  - `fdr` uses the [Benjamini-Hochberg procedure]
- *    (https://en.wikipedia.org/wiki/False_discovery_rate#Benjamini.E2.80.93Hochberg_procedure)
- *    to choose all features whose false discovery rate is below a threshold.
- *  - `fwe` chooses all features whose p-values are below a threshold. The threshold is scaled by
- *    1/numFeatures, thus controlling the family-wise error rate of selection.
- * By default, the selection method is `numTopFeatures`, with the default number of top features
- * set to 50.
- */
-@Since("3.1.0")
-final class FValueSelector @Since("3.1.0") (@Since("3.1.0") override val uid: String) extends
-  Selector[FValueSelectorModel] {
-
-  @Since("3.1.0")
-  def this() = this(Identifiable.randomUID("FValueSelector"))
-
-  /** @group setParam */
-  @Since("3.1.0")
-  override def setNumTopFeatures(value: Int): this.type = super.setNumTopFeatures(value)
-
-  /** @group setParam */
-  @Since("3.1.0")
-  override def setPercentile(value: Double): this.type = super.setPercentile(value)
-
-  /** @group setParam */
-  @Since("3.1.0")
-  override def setFpr(value: Double): this.type = super.setFpr(value)
-
-  /** @group setParam */
-  @Since("3.1.0")
-  override def setFdr(value: Double): this.type = super.setFdr(value)
-
-  /** @group setParam */
-  @Since("3.1.0")
-  override def setFwe(value: Double): this.type = super.setFwe(value)
-
-  /** @group setParam */
-  @Since("3.1.0")
-  override def setSelectorType(value: String): this.type = super.setSelectorType(value)
-
-  /** @group setParam */
-  @Since("3.1.0")
-  override def setFeaturesCol(value: String): this.type = super.setFeaturesCol(value)
-
-  /** @group setParam */
-  @Since("3.1.0")
-  override def setOutputCol(value: String): this.type = super.setOutputCol(value)
-
-  /** @group setParam */
-  @Since("3.1.0")
-  override def setLabelCol(value: String): this.type = super.setLabelCol(value)
-
-  /**
-   * get the SelectionTestResult for every feature against the label
-   */
-  protected[this] override def getSelectionTestResult(df: DataFrame): DataFrame = {
-    FValueTest.test(df, getFeaturesCol, getLabelCol, true)
-  }
-
-  /**
-   * Create a new instance of concrete SelectorModel.
-   * @param indices The indices of the selected features
-   * @return A new SelectorModel instance
-   */
-  protected[this] def createSelectorModel(
-      uid: String,
-      indices: Array[Int]): FValueSelectorModel = {
-    new FValueSelectorModel(uid, indices)
-  }
-
-  @Since("3.1.0")
-  override def fit(dataset: Dataset[_]): FValueSelectorModel = {
-    super.fit(dataset)
-  }
-
-  @Since("3.1.0")
-  override def copy(extra: ParamMap): this.type = defaultCopy(extra)
-}
-
-@Since("3.1.0")
-object FValueSelector extends DefaultParamsReadable[FValueSelector] {
-
-  @Since("3.1.0")
-  override def load(path: String): FValueSelector = super.load(path)
-}
-
-/**
- * Model fitted by [[FValueSelector]]
- */
-@Since("3.1.0")
-class FValueSelectorModel private[ml](
-    @Since("3.1.0") override val uid: String,
-    @Since("3.1.0") override val selectedFeatures: Array[Int])
-  extends SelectorModel[FValueSelectorModel] (uid, selectedFeatures) {
-
-  /** @group setParam */
-  @Since("3.1.0")
-  override def setFeaturesCol(value: String): this.type = super.setFeaturesCol(value)
-
-  /** @group setParam */
-  @Since("3.1.0")
-  override def setOutputCol(value: String): this.type = super.setOutputCol(value)
-
-  @Since("3.1.0")
-  override def copy(extra: ParamMap): FValueSelectorModel = {
-    val copied = new FValueSelectorModel(uid, selectedFeatures)
-      .setParent(parent)
-    copyValues(copied, extra)
-  }
-
-  @Since("3.1.0")
-  override def write: MLWriter = new FValueSelectorModel.FValueSelectorModelWriter(this)
-
-  @Since("3.1.0")
-  override def toString: String = {
-    s"FValueSelectorModel: uid=$uid, numSelectedFeatures=${selectedFeatures.length}"
-  }
-}
-
-@Since("3.1.0")
-object FValueSelectorModel extends MLReadable[FValueSelectorModel] {
-
-  @Since("3.1.0")
-  override def read: MLReader[FValueSelectorModel] = new FValueSelectorModelReader
-
-  @Since("3.1.0")
-  override def load(path: String): FValueSelectorModel = super.load(path)
-
-  private[FValueSelectorModel] class FValueSelectorModelWriter(
-      instance: FValueSelectorModel) extends MLWriter {
-
-    private case class Data(selectedFeatures: Seq[Int])
-
-    override protected def saveImpl(path: String): Unit = {
-      DefaultParamsWriter.saveMetadata(instance, path, sc)
-      val data = Data(instance.selectedFeatures.toSeq)
-      val dataPath = new Path(path, "data").toString
-      sparkSession.createDataFrame(Seq(data)).repartition(1).write.parquet(dataPath)
-    }
-  }
-
-  private class FValueSelectorModelReader extends MLReader[FValueSelectorModel] {
-
-    /** Checked against metadata when loading model */
-    private val className = classOf[FValueSelectorModel].getName
-
-    override def load(path: String): FValueSelectorModel = {
-      val metadata = DefaultParamsReader.loadMetadata(path, sc, className)
-      val dataPath = new Path(path, "data").toString
-      val data = sparkSession.read.parquet(dataPath)
-        .select("selectedFeatures").head()
-      val selectedFeatures = data.getAs[Seq[Int]](0).toArray
-      val model = new FValueSelectorModel(metadata.uid, selectedFeatures)
-      metadata.getAndSetParams(model)
-      model
-    }
-  }
-}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/FeatureHasher.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/FeatureHasher.scala
index 39862554c5d8d..f1268bdf6bd89 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/FeatureHasher.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/FeatureHasher.scala
@@ -91,8 +91,8 @@ class FeatureHasher(@Since("2.3.0") override val uid: String) extends Transforme
   /**
    * Numeric columns to treat as categorical features. By default only string and boolean
    * columns are treated as categorical, so this param can be used to explicitly specify the
-   * numerical columns to treat as categorical. Note, the relevant columns must also be set in
-   * `inputCols`.
+   * numerical columns to treat as categorical. Note, the relevant columns should also be set in
+   * `inputCols`, categorical columns not set in `inputCols` will be listed in a warning.
    * @group param
    */
   @Since("2.3.0")
@@ -125,19 +125,24 @@ class FeatureHasher(@Since("2.3.0") override val uid: String) extends Transforme
 
   @Since("2.3.0")
   override def transform(dataset: Dataset[_]): DataFrame = {
+    val outputSchema = transformSchema(dataset.schema)
     val hashFunc: Any => Int = FeatureHasher.murmur3Hash
+
     val n = $(numFeatures)
     val localInputCols = $(inputCols)
-    val catCols = if (isSet(categoricalCols)) {
-      $(categoricalCols).toSet
-    } else {
-      Set[String]()
+
+    var catCols = dataset.schema(localInputCols.toSet)
+      .filterNot(_.dataType.isInstanceOf[NumericType]).map(_.name).toArray
+    if (isSet(categoricalCols)) {
+      // categoricalCols may contain columns not set in inputCols
+      catCols = (catCols ++ $(categoricalCols).intersect(localInputCols)).distinct
     }
+    val catIndices = catCols.map(c => localInputCols.indexOf(c))
 
-    val outputSchema = transformSchema(dataset.schema)
-    val realFields = outputSchema.fields.filter { f =>
-      f.dataType.isInstanceOf[NumericType] && !catCols.contains(f.name)
-    }.map(_.name).toSet
+    val realCols = (localInputCols.toSet -- catCols).toArray
+    val realIndices = realCols.map(c => localInputCols.indexOf(c))
+    // pre-compute output indices of real columns
+    val realOutputIndices = realCols.map(c => Utils.nonNegativeMod(hashFunc(c), n))
 
     def getDouble(x: Any): Double = {
       x match {
@@ -151,33 +156,38 @@ class FeatureHasher(@Since("2.3.0") override val uid: String) extends Transforme
 
     val hashFeatures = udf { row: Row =>
       val map = new OpenHashMap[Int, Double]()
-      localInputCols.foreach { colName =>
-        val fieldIndex = row.fieldIndex(colName)
-        if (!row.isNullAt(fieldIndex)) {
-          val (rawIdx, value) = if (realFields(colName)) {
-            // numeric values are kept as is, with vector index based on hash of "column_name"
-            val value = getDouble(row.get(fieldIndex))
-            val hash = hashFunc(colName)
-            (hash, value)
-          } else {
-            // string, boolean and numeric values that are in catCols are treated as categorical,
-            // with an indicator value of 1.0 and vector index based on hash of "column_name=value"
-            val value = row.get(fieldIndex).toString
-            val fieldName = s"$colName=$value"
-            val hash = hashFunc(fieldName)
-            (hash, 1.0)
-          }
-          val idx = Utils.nonNegativeMod(rawIdx, n)
+
+      var i = 0
+      while (i < realIndices.length) {
+        val realIdx = realIndices(i)
+        if (!row.isNullAt(realIdx)) {
+          // numeric values are kept as is, with vector index based on hash of "column_name"
+          val value = getDouble(row.get(realIdx))
+          val idx = realOutputIndices(i)
           map.changeValue(idx, value, v => v + value)
         }
+        i += 1
       }
+
+      i = 0
+      while (i < catIndices.length) {
+        val catIdx = catIndices(i)
+        if (!row.isNullAt(catIdx)) {
+          // string, boolean and numeric values that are in catCols are treated as categorical,
+          // with an indicator value of 1.0 and vector index based on hash of "column_name=value"
+          val string = row.get(catIdx).toString
+          val rawIdx = hashFunc(s"${catCols(i)}=$string")
+          val idx = Utils.nonNegativeMod(rawIdx, n)
+          map.changeValue(idx, 1.0, v => v + 1.0)
+        }
+        i += 1
+      }
+
       Vectors.sparse(n, map.toSeq)
     }
 
     val metadata = outputSchema($(outputCol)).metadata
-    dataset.select(
-      col("*"),
-      hashFeatures(struct($(inputCols).map(col): _*)).as($(outputCol), metadata))
+    dataset.withColumn($(outputCol), hashFeatures(struct($(inputCols).map(col): _*)), metadata)
   }
 
   @Since("2.3.0")
@@ -185,7 +195,14 @@ class FeatureHasher(@Since("2.3.0") override val uid: String) extends Transforme
 
   @Since("2.3.0")
   override def transformSchema(schema: StructType): StructType = {
-    val fields = schema($(inputCols).toSet)
+    val localInputCols = $(inputCols).toSet
+    if (isSet(categoricalCols)) {
+      val set = $(categoricalCols).filterNot(c => localInputCols.contains(c))
+      if (set.nonEmpty) {
+        log.warn(s"categoricalCols ${set.mkString("[", ",", "]")} do not exist in inputCols")
+      }
+    }
+    val fields = schema(localInputCols)
     fields.foreach { fieldSchema =>
       val dataType = fieldSchema.dataType
       val fieldName = fieldSchema.name
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/HashingTF.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/HashingTF.scala
index d2bb013448aae..f4223bc85943d 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/HashingTF.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/HashingTF.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.ml.feature
 
-import scala.collection.mutable
-
 import org.apache.spark.annotation.Since
 import org.apache.spark.ml.Transformer
 import org.apache.spark.ml.attribute.AttributeGroup
@@ -32,6 +30,7 @@ import org.apache.spark.sql.functions.{col, udf}
 import org.apache.spark.sql.types.{ArrayType, StructType}
 import org.apache.spark.util.Utils
 import org.apache.spark.util.VersionUtils.majorMinorVersion
+import org.apache.spark.util.collection.OpenHashMap
 
 /**
  * Maps a sequence of terms to their term frequencies using the hashing trick.
@@ -91,20 +90,13 @@ class HashingTF @Since("3.0.0") private[ml] (
   @Since("2.0.0")
   override def transform(dataset: Dataset[_]): DataFrame = {
     val outputSchema = transformSchema(dataset.schema)
-    val localNumFeatures = $(numFeatures)
-    val localBinary = $(binary)
+    val n = $(numFeatures)
+    val updateFunc = if ($(binary)) (v: Double) => 1.0 else (v: Double) => v + 1.0
 
     val hashUDF = udf { terms: Seq[_] =>
-      val termFrequencies = mutable.HashMap.empty[Int, Double].withDefaultValue(0.0)
-      terms.foreach { term =>
-        val i = indexOf(term)
-        if (localBinary) {
-          termFrequencies(i) = 1.0
-        } else {
-          termFrequencies(i) += 1.0
-        }
-      }
-      Vectors.sparse(localNumFeatures, termFrequencies.toSeq)
+      val map = new OpenHashMap[Int, Double]()
+      terms.foreach { term => map.changeValue(indexOf(term), 1.0, updateFunc) }
+      Vectors.sparse(n, map.toSeq)
     }
 
     dataset.withColumn($(outputCol), hashUDF(col($(inputCol))),
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/IDF.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/IDF.scala
index e6f124ef7d666..e451d4daffbc7 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/IDF.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/IDF.scala
@@ -215,10 +215,10 @@ object IDFModel extends MLReadable[IDFModel] {
       val data = sparkSession.read.parquet(dataPath)
 
       val model = if (majorVersion(metadata.sparkVersion) >= 3) {
-        val Row(idf: Vector, df: Seq[_], numDocs: Long) = data.select("idf", "docFreq", "numDocs")
-          .head()
+        val Row(idf: Vector, df: scala.collection.Seq[_], numDocs: Long) =
+          data.select("idf", "docFreq", "numDocs").head()
         new IDFModel(metadata.uid, new feature.IDFModel(OldVectors.fromML(idf),
-          df.asInstanceOf[Seq[Long]].toArray, numDocs))
+          df.asInstanceOf[scala.collection.Seq[Long]].toArray, numDocs))
       } else {
         val Row(idf: Vector) = MLUtils.convertVectorColumnsToML(data, "idf")
           .select("idf")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/Imputer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/Imputer.scala
index 64f1722f5fcb8..71403acc91b55 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/Imputer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/Imputer.scala
@@ -39,14 +39,16 @@ private[feature] trait ImputerParams extends Params with HasInputCol with HasInp
    * The imputation strategy. Currently only "mean" and "median" are supported.
    * If "mean", then replace missing values using the mean value of the feature.
    * If "median", then replace missing values using the approximate median value of the feature.
+   * If "mode", then replace missing using the most frequent value of the feature.
    * Default: mean
    *
    * @group param
    */
   final val strategy: Param[String] = new Param(this, "strategy", s"strategy for imputation. " +
     s"If ${Imputer.mean}, then replace missing values using the mean value of the feature. " +
-    s"If ${Imputer.median}, then replace missing values using the median value of the feature.",
-    ParamValidators.inArray[String](Array(Imputer.mean, Imputer.median)))
+    s"If ${Imputer.median}, then replace missing values using the median value of the feature. " +
+    s"If ${Imputer.mode}, then replace missing values using the most frequent value of " +
+    s"the feature.", ParamValidators.inArray[String](Imputer.supportedStrategies))
 
   /** @group getParam */
   def getStrategy: String = $(strategy)
@@ -64,6 +66,8 @@ private[feature] trait ImputerParams extends Params with HasInputCol with HasInp
   /** @group getParam */
   def getMissingValue: Double = $(missingValue)
 
+  setDefault(strategy -> Imputer.mean, missingValue -> Double.NaN)
+
   /** Returns the input and output column names corresponding in pair. */
   private[feature] def getInOutCols(): (Array[String], Array[String]) = {
     if (isSet(inputCol)) {
@@ -93,7 +97,7 @@ private[feature] trait ImputerParams extends Params with HasInputCol with HasInp
 }
 
 /**
- * Imputation estimator for completing missing values, either using the mean or the median
+ * Imputation estimator for completing missing values, using the mean, median or mode
  * of the columns in which the missing values are located. The input columns should be of
  * numeric type. Currently Imputer does not support categorical features
  * (SPARK-15041) and possibly creates incorrect values for a categorical feature.
@@ -102,7 +106,7 @@ private[feature] trait ImputerParams extends Params with HasInputCol with HasInp
  * For example, if the input column is IntegerType (1, 2, 4, null),
  * the output will be IntegerType (1, 2, 4, 2) after mean imputation.
  *
- * Note that the mean/median value is computed after filtering out missing values.
+ * Note that the mean/median/mode value is computed after filtering out missing values.
  * All Null values in the input columns are treated as missing, and so are also imputed. For
  * computing median, DataFrameStatFunctions.approxQuantile is used with a relative error of 0.001.
  */
@@ -130,7 +134,7 @@ class Imputer @Since("2.2.0") (@Since("2.2.0") override val uid: String)
   def setOutputCols(value: Array[String]): this.type = set(outputCols, value)
 
   /**
-   * Imputation strategy. Available options are ["mean", "median"].
+   * Imputation strategy. Available options are ["mean", "median", "mode"].
    * @group setParam
    */
   @Since("2.2.0")
@@ -144,46 +148,47 @@ class Imputer @Since("2.2.0") (@Since("2.2.0") override val uid: String)
   @Since("3.0.0")
   def setRelativeError(value: Double): this.type = set(relativeError, value)
 
-  setDefault(strategy -> Imputer.mean, missingValue -> Double.NaN)
-
   override def fit(dataset: Dataset[_]): ImputerModel = {
     transformSchema(dataset.schema, logging = true)
     val spark = dataset.sparkSession
 
     val (inputColumns, _) = getInOutCols()
-
     val cols = inputColumns.map { inputCol =>
       when(col(inputCol).equalTo($(missingValue)), null)
         .when(col(inputCol).isNaN, null)
         .otherwise(col(inputCol))
-        .cast("double")
+        .cast(DoubleType)
         .as(inputCol)
     }
+    val numCols = cols.length
 
     val results = $(strategy) match {
       case Imputer.mean =>
         // Function avg will ignore null automatically.
         // For a column only containing null, avg will return null.
         val row = dataset.select(cols.map(avg): _*).head()
-        Array.range(0, inputColumns.length).map { i =>
-          if (row.isNullAt(i)) {
-            Double.NaN
-          } else {
-            row.getDouble(i)
-          }
-        }
+        Array.tabulate(numCols)(i => if (row.isNullAt(i)) Double.NaN else row.getDouble(i))
 
       case Imputer.median =>
         // Function approxQuantile will ignore null automatically.
         // For a column only containing null, approxQuantile will return an empty array.
         dataset.select(cols: _*).stat.approxQuantile(inputColumns, Array(0.5), $(relativeError))
-          .map { array =>
-            if (array.isEmpty) {
-              Double.NaN
-            } else {
-              array.head
-            }
-          }
+          .map(_.headOption.getOrElse(Double.NaN))
+
+      case Imputer.mode =>
+        import spark.implicits._
+        // If there is more than one mode, choose the smallest one to keep in line
+        // with sklearn.impute.SimpleImputer (using scipy.stats.mode).
+        val modes = dataset.select(cols: _*).flatMap { row =>
+          // Ignore null.
+          Iterator.range(0, numCols)
+            .flatMap(i => if (row.isNullAt(i)) None else Some((i, row.getDouble(i))))
+        }.toDF("index", "value")
+         .groupBy("index", "value").agg(negate(count(lit(0))).as("negative_count"))
+         .groupBy("index").agg(min(struct("negative_count", "value")).as("mode"))
+         .select("index", "mode.value")
+         .as[(Int, Double)].collect().toMap
+        Array.tabulate(numCols)(i => modes.getOrElse(i, Double.NaN))
     }
 
     val emptyCols = inputColumns.zip(results).filter(_._2.isNaN).map(_._1)
@@ -212,6 +217,10 @@ object Imputer extends DefaultParamsReadable[Imputer] {
   /** strategy names that Imputer currently supports. */
   private[feature] val mean = "mean"
   private[feature] val median = "median"
+  private[feature] val mode = "mode"
+
+  /* Set of strategies that Imputer supports */
+  private[feature] val supportedStrategies = Array(mean, median, mode)
 
   @Since("2.2.0")
   override def load(path: String): Imputer = super.load(path)
@@ -245,20 +254,25 @@ class ImputerModel private[ml] (
   /** @group setParam */
   def setOutputCols(value: Array[String]): this.type = set(outputCols, value)
 
+  @transient private lazy val surrogates = {
+    val row = surrogateDF.head()
+    row.schema.fieldNames.zipWithIndex
+      .map { case (name, index) => (name, row.getDouble(index)) }
+      .toMap
+  }
+
   override def transform(dataset: Dataset[_]): DataFrame = {
     transformSchema(dataset.schema, logging = true)
-    val (inputColumns, outputColumns) = getInOutCols
-    val surrogates = surrogateDF.select(inputColumns.map(col): _*).head().toSeq
-
-
-    val newCols = inputColumns.zip(outputColumns).zip(surrogates).map {
-      case ((inputCol, outputCol), surrogate) =>
-        val inputType = dataset.schema(inputCol).dataType
-        val ic = col(inputCol).cast(DoubleType)
-        when(ic.isNull, surrogate)
-          .when(ic === $(missingValue), surrogate)
-          .otherwise(ic)
-          .cast(inputType)
+    val (inputColumns, outputColumns) = getInOutCols()
+
+    val newCols = inputColumns.map { inputCol =>
+      val surrogate = surrogates(inputCol)
+      val inputType = dataset.schema(inputCol).dataType
+      val ic = col(inputCol).cast(DoubleType)
+      when(ic.isNull, surrogate)
+        .when(ic === $(missingValue), surrogate)
+        .otherwise(ic)
+        .cast(inputType)
     }
     dataset.withColumns(outputColumns, newCols).toDF()
   }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/Instance.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/Instance.scala
index db5f88d5dddc8..c237366ec5c3d 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/Instance.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/Instance.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.ml.feature
 
+import scala.collection.mutable
+
 import org.apache.spark.ml.linalg._
 import org.apache.spark.rdd.RDD
 
@@ -100,6 +102,32 @@ private[spark] case class InstanceBlock(
 
 private[spark] object InstanceBlock {
 
+  /**
+   * Suggested value for BlockSizeInMB in Level-2 routine cases.
+   * According to performance tests of BLAS routine (see SPARK-31714) and
+   * LinearSVC (see SPARK-32907), 1.0 MB should be an acceptable value for
+   * linear models using Level-2 routine (GEMV) to perform prediction and
+   * gradient computation.
+   */
+  val DefaultBlockSizeInMB = 1.0
+
+  private def getBlockMemUsage(
+      numCols: Long,
+      numRows: Long,
+      nnz: Long,
+      allUnitWeight: Boolean): Long = {
+    val doubleBytes = java.lang.Double.BYTES
+    val arrayHeader = 12L
+    val denseSize = Matrices.getDenseSize(numCols, numRows)
+    val sparseSize = Matrices.getSparseSize(nnz, numRows + 1)
+    val matrixSize = math.min(denseSize, sparseSize)
+    if (allUnitWeight) {
+      matrixSize + doubleBytes * numRows + arrayHeader * 2
+    } else {
+      matrixSize + doubleBytes * numRows * 2 + arrayHeader * 2
+    }
+  }
+
   def fromInstances(instances: Seq[Instance]): InstanceBlock = {
     val labels = instances.map(_.label).toArray
     val weights = if (instances.exists(_.weight != 1)) {
@@ -114,6 +142,49 @@ private[spark] object InstanceBlock {
   def blokify(instances: RDD[Instance], blockSize: Int): RDD[InstanceBlock] = {
     instances.mapPartitions(_.grouped(blockSize).map(InstanceBlock.fromInstances))
   }
+
+  def blokifyWithMaxMemUsage(
+      instanceIterator: Iterator[Instance],
+      maxMemUsage: Long): Iterator[InstanceBlock] = {
+    require(maxMemUsage > 0)
+
+    new Iterator[InstanceBlock]() {
+      private var numCols = -1L
+
+      override def hasNext: Boolean = instanceIterator.hasNext
+
+      override def next(): InstanceBlock = {
+        val buff = mutable.ArrayBuilder.make[Instance]
+        var buffCnt = 0L
+        var buffNnz = 0L
+        var buffUnitWeight = true
+        var blockMemUsage = 0L
+
+        while (instanceIterator.hasNext && blockMemUsage < maxMemUsage) {
+          val instance = instanceIterator.next()
+          if (numCols < 0L) numCols = instance.features.size
+          require(numCols == instance.features.size)
+
+          buff += instance
+          buffCnt += 1L
+          buffNnz += instance.features.numNonzeros
+          buffUnitWeight &&= (instance.weight == 1)
+          blockMemUsage = getBlockMemUsage(numCols, buffCnt, buffNnz, buffUnitWeight)
+        }
+
+        // the block memory usage may slightly exceed threshold, not a big issue.
+        // and this ensure even if one row exceed block limit, each block has one row.
+        InstanceBlock.fromInstances(buff.result())
+      }
+    }
+  }
+
+  def blokifyWithMaxMemUsage(
+      instances: RDD[Instance],
+      maxMemUsage: Long): RDD[InstanceBlock] = {
+    require(maxMemUsage > 0)
+    instances.mapPartitions(iter => blokifyWithMaxMemUsage(iter, maxMemUsage))
+  }
 }
 
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/LSH.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/LSH.scala
index 6d5c7c50dbacc..c3304047fce90 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/LSH.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/LSH.scala
@@ -94,7 +94,7 @@ private[ml] abstract class LSHModel[T <: LSHModel[T]]
    * @param y Another hash vector.
    * @return The distance between hash vectors x and y.
    */
-  protected[ml] def hashDistance(x: Seq[Vector], y: Seq[Vector]): Double
+  protected[ml] def hashDistance(x: Array[Vector], y: Array[Vector]): Double
 
   override def transform(dataset: Dataset[_]): DataFrame = {
     transformSchema(dataset.schema, logging = true)
@@ -116,44 +116,44 @@ private[ml] abstract class LSHModel[T <: LSHModel[T]]
     require(numNearestNeighbors > 0, "The number of nearest neighbors cannot be less than 1")
     // Get Hash Value of the key
     val keyHash = hashFunction(key)
-    val modelDataset: DataFrame = if (!dataset.columns.contains($(outputCol))) {
+    val modelDataset = if (!dataset.columns.contains($(outputCol))) {
         transform(dataset)
       } else {
         dataset.toDF()
       }
 
     val modelSubset = if (singleProbe) {
-      def sameBucket(x: Seq[Vector], y: Seq[Vector]): Boolean = {
-        x.zip(y).exists(tuple => tuple._1 == tuple._2)
+      def sameBucket(x: Array[Vector], y: Array[Vector]): Boolean = {
+        x.iterator.zip(y.iterator).exists(tuple => tuple._1 == tuple._2)
       }
 
       // In the origin dataset, find the hash value that hash the same bucket with the key
-      val sameBucketWithKeyUDF = udf((x: Seq[Vector]) => sameBucket(x, keyHash))
+      val sameBucketWithKeyUDF = udf((x: Array[Vector]) => sameBucket(x, keyHash))
 
       modelDataset.filter(sameBucketWithKeyUDF(col($(outputCol))))
     } else {
       // In the origin dataset, find the hash value that is closest to the key
       // Limit the use of hashDist since it's controversial
-      val hashDistUDF = udf((x: Seq[Vector]) => hashDistance(x, keyHash))
+      val hashDistUDF = udf((x: Array[Vector]) => hashDistance(x, keyHash))
       val hashDistCol = hashDistUDF(col($(outputCol)))
       val modelDatasetWithDist = modelDataset.withColumn(distCol, hashDistCol)
 
       val relativeError = 0.05
-      val (summary, count) = modelDatasetWithDist.select(distCol)
-        .rdd
-        .mapPartitions { iter =>
-          if (iter.hasNext) {
-            var s = new QuantileSummaries(
-              QuantileSummaries.defaultCompressThreshold, relativeError)
-            var c = 0L
-            while (iter.hasNext) {
-              val Row(dist: Double) = iter.next
-              s = s.insert(dist)
-              c += 1
+      val summary = modelDatasetWithDist.select(distCol).rdd.mapPartitions { iter =>
+        if (iter.hasNext) {
+          var s = new QuantileSummaries(
+            QuantileSummaries.defaultCompressThreshold, relativeError)
+          while (iter.hasNext) {
+            val row = iter.next
+            if (!row.isNullAt(0)) {
+              val v = row.getDouble(0)
+              if (!v.isNaN) s = s.insert(v)
             }
-            Iterator.single((s.compress, c))
-          } else Iterator.empty
-        }.treeReduce { case ((s1, c1), (s2, c2)) => (s1.merge(s2), c1 + c2) }
+          }
+          Iterator.single(s.compress)
+        } else Iterator.empty
+      }.treeReduce((s1, s2) => s1.merge(s2))
+      val count = summary.count
 
       // Compute threshold to get around k elements.
       // To guarantee to have enough neighbors in one pass, we need (p - err) * N >= M
@@ -223,7 +223,7 @@ private[ml] abstract class LSHModel[T <: LSHModel[T]]
       inputName: String,
       explodeCols: Seq[String]): Dataset[_] = {
     require(explodeCols.size == 2, "explodeCols must be two strings.")
-    val modelDataset: DataFrame = if (!dataset.columns.contains($(outputCol))) {
+    val modelDataset = if (!dataset.columns.contains($(outputCol))) {
       transform(dataset)
     } else {
       dataset.toDF()
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/MinHashLSH.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/MinHashLSH.scala
index be467c654aaa1..d189edcb4e558 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/MinHashLSH.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/MinHashLSH.scala
@@ -106,12 +106,25 @@ class MinHashLSHModel private[ml](
   }
 
   @Since("2.1.0")
-  override protected[ml] def hashDistance(x: Seq[Vector], y: Seq[Vector]): Double = {
+  override protected[ml] def hashDistance(x: Array[Vector], y: Array[Vector]): Double = {
     // Since it's generated by hashing, it will be a pair of dense vectors.
     // TODO: This hashDistance function requires more discussion in SPARK-18454
-    x.iterator.zip(y.iterator).map(vectorPair =>
-      vectorPair._1.toArray.zip(vectorPair._2.toArray).count(pair => pair._1 != pair._2)
-    ).min
+    var distance = Int.MaxValue
+    var i = 0
+    while (i < x.length) {
+      val vx = x(i).toArray
+      val vy = y(i).toArray
+      var j = 0
+      var d = 0
+      while (j < vx.length && d < distance) {
+        if (vx(j) != vy(j)) d += 1
+        j += 1
+      }
+      if (d == 0) return 0.0
+      if (d < distance) distance = d
+      i += 1
+    }
+    distance
   }
 
   @Since("2.1.0")
@@ -224,7 +237,7 @@ object MinHashLSHModel extends MLReadable[MinHashLSHModel] {
 
       val dataPath = new Path(path, "data").toString
       val data = sparkSession.read.parquet(dataPath).select("randCoefficients").head()
-      val randCoefficients = data.getAs[Seq[Int]](0).grouped(2)
+      val randCoefficients = data.getSeq[Int](0).grouped(2)
         .map(tuple => (tuple(0), tuple(1))).toArray
       val model = new MinHashLSHModel(metadata.uid, randCoefficients)
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/MinMaxScaler.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/MinMaxScaler.scala
index 90187c331e835..22c4ca9cddf4b 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/MinMaxScaler.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/MinMaxScaler.scala
@@ -58,6 +58,8 @@ private[feature] trait MinMaxScalerParams extends Params with HasInputCol with H
   /** @group getParam */
   def getMax: Double = $(max)
 
+  setDefault(min -> 0.0, max -> 1.0)
+
   /** Validates and transforms the input schema. */
   protected def validateAndTransformSchema(schema: StructType): StructType = {
     require($(min) < $(max), s"The specified min(${$(min)}) is larger or equal to max(${$(max)})")
@@ -93,8 +95,6 @@ class MinMaxScaler @Since("1.5.0") (@Since("1.5.0") override val uid: String)
   @Since("1.5.0")
   def this() = this(Identifiable.randomUID("minMaxScal"))
 
-  setDefault(min -> 0.0, max -> 1.0)
-
   /** @group setParam */
   @Since("1.5.0")
   def setInputCol(value: String): this.type = set(inputCol, value)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/OneHotEncoder.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/OneHotEncoder.scala
index 0ef092f6be463..5a500fefb57ec 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/OneHotEncoder.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/OneHotEncoder.scala
@@ -53,8 +53,6 @@ private[ml] trait OneHotEncoderBase extends Params with HasHandleInvalid
     "during fitting, invalid data will result in an error.",
     ParamValidators.inArray(OneHotEncoder.supportedHandleInvalids))
 
-  setDefault(handleInvalid, OneHotEncoder.ERROR_INVALID)
-
   /**
    * Whether to drop the last category in the encoded vector (default: true)
    * @group param
@@ -62,12 +60,13 @@ private[ml] trait OneHotEncoderBase extends Params with HasHandleInvalid
   @Since("2.3.0")
   final val dropLast: BooleanParam =
     new BooleanParam(this, "dropLast", "whether to drop the last category")
-  setDefault(dropLast -> true)
 
   /** @group getParam */
   @Since("2.3.0")
   def getDropLast: Boolean = $(dropLast)
 
+  setDefault(handleInvalid -> OneHotEncoder.ERROR_INVALID, dropLast -> true)
+
   /** Returns the input and output column names corresponding in pair. */
   private[feature] def getInOutCols(): (Array[String], Array[String]) = {
     if (isSet(inputCol)) {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/QuantileDiscretizer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/QuantileDiscretizer.scala
index 4eedfc4dc0efa..b93c9b1fcd204 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/QuantileDiscretizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/QuantileDiscretizer.scala
@@ -46,7 +46,6 @@ private[feature] trait QuantileDiscretizerBase extends Params
   val numBuckets = new IntParam(this, "numBuckets", "Number of buckets (quantiles, or " +
     "categories) into which data points are grouped. Must be >= 2.",
     ParamValidators.gtEq(2))
-  setDefault(numBuckets -> 2)
 
   /** @group getParam */
   def getNumBuckets: Int = getOrDefault(numBuckets)
@@ -82,7 +81,8 @@ private[feature] trait QuantileDiscretizerBase extends Params
     "how to handle invalid entries. Options are skip (filter out rows with invalid values), " +
     "error (throw an error), or keep (keep invalid values in a special additional bucket).",
     ParamValidators.inArray(Bucketizer.supportedHandleInvalids))
-  setDefault(handleInvalid, Bucketizer.ERROR_INVALID)
+
+  setDefault(handleInvalid -> Bucketizer.ERROR_INVALID, numBuckets -> 2)
 }
 
 /**
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala
index 7ccfafa4ac813..563e1708acdf1 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala
@@ -60,7 +60,6 @@ private[feature] trait RFormulaBase extends HasFeaturesCol with HasLabelCol with
   @Since("2.1.0")
   val forceIndexLabel: BooleanParam = new BooleanParam(this, "forceIndexLabel",
     "Force to index label whether it is numeric or string")
-  setDefault(forceIndexLabel -> false)
 
   /** @group getParam */
   @Since("2.1.0")
@@ -80,7 +79,6 @@ private[feature] trait RFormulaBase extends HasFeaturesCol with HasLabelCol with
     "type. Options are 'skip' (filter out rows with invalid data), error (throw an error), " +
     "or 'keep' (put invalid data in a special additional bucket, at index numLabels).",
     ParamValidators.inArray(StringIndexer.supportedHandleInvalids))
-  setDefault(handleInvalid, StringIndexer.ERROR_INVALID)
 
   /**
    * Param for how to order categories of a string FEATURE column used by `StringIndexer`.
@@ -113,12 +111,14 @@ private[feature] trait RFormulaBase extends HasFeaturesCol with HasLabelCol with
     "The default value is 'frequencyDesc'. When the ordering is set to 'alphabetDesc', " +
     "RFormula drops the same category as R when encoding strings.",
     ParamValidators.inArray(StringIndexer.supportedStringOrderType))
-  setDefault(stringIndexerOrderType, StringIndexer.frequencyDesc)
 
   /** @group getParam */
   @Since("2.3.0")
   def getStringIndexerOrderType: String = $(stringIndexerOrderType)
 
+  setDefault(forceIndexLabel -> false, handleInvalid -> StringIndexer.ERROR_INVALID,
+    stringIndexerOrderType -> StringIndexer.frequencyDesc)
+
   protected def hasLabelCol(schema: StructType): Boolean = {
     schema.map(_.name).contains($(labelCol))
   }
@@ -449,7 +449,7 @@ object RFormulaModel extends MLReadable[RFormulaModel] {
       val dataPath = new Path(path, "data").toString
       val data = sparkSession.read.parquet(dataPath).select("label", "terms", "hasIntercept").head()
       val label = data.getString(0)
-      val terms = data.getAs[Seq[Seq[String]]](1)
+      val terms = data.getSeq[Seq[String]](1)
       val hasIntercept = data.getBoolean(2)
       val resolvedRFormula = ResolvedRFormula(label, terms, hasIntercept)
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/RFormulaParser.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/RFormulaParser.scala
index dbbfd8f329431..c5b28c95eb7c9 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/RFormulaParser.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/RFormulaParser.scala
@@ -286,6 +286,7 @@ private[ml] object RFormulaParser extends RegexParsers {
 
   private val pow: Parser[Term] = term ~ "^" ~ "^[1-9]\\d*".r ^^ {
     case base ~ "^" ~ degree => power(base, degree.toInt)
+    case t => throw new IllegalArgumentException(s"Invalid term: $t")
   } | term
 
   private val interaction: Parser[Term] = pow * (":" ^^^ { interact _ })
@@ -298,7 +299,10 @@ private[ml] object RFormulaParser extends RegexParsers {
   private val expr = (sum | term)
 
   private val formula: Parser[ParsedRFormula] =
-    (label ~ "~" ~ expr) ^^ { case r ~ "~" ~ t => ParsedRFormula(r, t.asTerms.terms) }
+    (label ~ "~" ~ expr) ^^ {
+      case r ~ "~" ~ t => ParsedRFormula(r, t.asTerms.terms)
+      case t => throw new IllegalArgumentException(s"Invalid term: $t")
+    }
 
   def parse(value: String): ParsedRFormula = parseAll(formula, value) match {
     case Success(result, _) => result
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/RobustScaler.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/RobustScaler.scala
index bd9be779fedbd..e8f325ec58432 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/RobustScaler.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/RobustScaler.scala
@@ -50,8 +50,6 @@ private[feature] trait RobustScalerParams extends Params with HasInputCol with H
   /** @group getParam */
   def getLower: Double = $(lower)
 
-  setDefault(lower -> 0.25)
-
   /**
    * Upper quantile to calculate quantile range, shared by all features
    * Default: 0.75
@@ -64,8 +62,6 @@ private[feature] trait RobustScalerParams extends Params with HasInputCol with H
   /** @group getParam */
   def getUpper: Double = $(upper)
 
-  setDefault(upper -> 0.75)
-
   /**
    * Whether to center the data with median before scaling.
    * It will build a dense output, so take care when applying to sparse input.
@@ -78,8 +74,6 @@ private[feature] trait RobustScalerParams extends Params with HasInputCol with H
   /** @group getParam */
   def getWithCentering: Boolean = $(withCentering)
 
-  setDefault(withCentering -> false)
-
   /**
    * Whether to scale the data to quantile range.
    * Default: true
@@ -91,7 +85,7 @@ private[feature] trait RobustScalerParams extends Params with HasInputCol with H
   /** @group getParam */
   def getWithScaling: Boolean = $(withScaling)
 
-  setDefault(withScaling -> true)
+  setDefault(withScaling -> true, lower -> 0.25, upper -> 0.75, withCentering -> false)
 
   /** Validates and transforms the input schema. */
   protected def validateAndTransformSchema(schema: StructType): StructType = {
@@ -201,7 +195,7 @@ object RobustScaler extends DefaultParamsReadable[RobustScaler] {
           }
           Iterator.tabulate(numFeatures)(i => (i, summaries(i).compress))
         } else Iterator.empty
-      }.reduceByKey { case (s1, s2) => s1.merge(s2) }
+      }.reduceByKey { (s1, s2) => s1.merge(s2) }
     } else {
       val scale = math.max(math.ceil(math.sqrt(vectors.getNumPartitions)).toInt, 2)
       vectors.mapPartitionsWithIndex { case (pid, iter) =>
@@ -214,7 +208,7 @@ object RobustScaler extends DefaultParamsReadable[RobustScaler] {
         seqOp = (s, v) => s.insert(v),
         combOp = (s1, s2) => s1.compress.merge(s2.compress)
       ).map { case ((_, i), s) => (i, s)
-      }.reduceByKey { case (s1, s2) => s1.compress.merge(s2.compress) }
+      }.reduceByKey { (s1, s2) => s1.compress.merge(s2.compress) }
     }
   }
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/Selector.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/Selector.scala
index 627133968d149..e24593a01b629 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/Selector.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/Selector.scala
@@ -50,7 +50,6 @@ private[feature] trait SelectorParams extends Params
     "Number of features that selector will select, ordered by ascending p-value. If the" +
       " number of features is < numTopFeatures, then this will select all features.",
     ParamValidators.gtEq(1))
-  setDefault(numTopFeatures -> 50)
 
   /** @group getParam */
   @Since("3.1.0")
@@ -66,7 +65,6 @@ private[feature] trait SelectorParams extends Params
   final val percentile = new DoubleParam(this, "percentile",
     "Percentile of features that selector will select, ordered by ascending p-value.",
     ParamValidators.inRange(0, 1))
-  setDefault(percentile -> 0.1)
 
   /** @group getParam */
   @Since("3.1.0")
@@ -79,9 +77,8 @@ private[feature] trait SelectorParams extends Params
    * @group param
    */
   @Since("3.1.0")
-  final val fpr = new DoubleParam(this, "fpr", "The higest p-value for features to be kept.",
+  final val fpr = new DoubleParam(this, "fpr", "The highest p-value for features to be kept.",
     ParamValidators.inRange(0, 1))
-  setDefault(fpr -> 0.05)
 
   /** @group getParam */
   @Since("3.1.0")
@@ -96,7 +93,6 @@ private[feature] trait SelectorParams extends Params
   @Since("3.1.0")
   final val fdr = new DoubleParam(this, "fdr",
     "The upper bound of the expected false discovery rate.", ParamValidators.inRange(0, 1))
-  setDefault(fdr -> 0.05)
 
   /** @group getParam */
   def getFdr: Double = $(fdr)
@@ -110,7 +106,6 @@ private[feature] trait SelectorParams extends Params
   @Since("3.1.0")
   final val fwe = new DoubleParam(this, "fwe",
     "The upper bound of the expected family-wise error rate.", ParamValidators.inRange(0, 1))
-  setDefault(fwe -> 0.05)
 
   /** @group getParam */
   def getFwe: Double = $(fwe)
@@ -125,22 +120,19 @@ private[feature] trait SelectorParams extends Params
     "The selector type. Supported options: numTopFeatures, percentile, fpr, fdr, fwe",
     ParamValidators.inArray(Array("numTopFeatures", "percentile", "fpr", "fdr",
       "fwe")))
-  setDefault(selectorType -> "numTopFeatures")
 
   /** @group getParam */
   @Since("3.1.0")
   def getSelectorType: String = $(selectorType)
 
+  setDefault(numTopFeatures -> 50, percentile -> 0.1, fpr -> 0.05, fdr -> 0.05, fwe -> 0.05,
+    selectorType -> "numTopFeatures")
 }
 
 /**
  * Super class for feature selectors.
  * 1. Chi-Square Selector
  * This feature selector is for categorical features and categorical labels.
- * 2. ANOVA F-value Classification Selector
- * This feature selector is for continuous features and categorical labels.
- * 3. Regression F-value Selector
- * This feature selector is for continuous features and continuous labels.
  * The selector supports different selection methods: `numTopFeatures`, `percentile`, `fpr`,
  * `fdr`, `fwe`.
  *  - `numTopFeatures` chooses a fixed number of top features according to a hypothesis.
@@ -255,7 +247,7 @@ private[ml] abstract class Selector[T <: SelectorModel[T]]
           .where(col("pValue") < $(fwe) / numFeatures)
           .as[Int].collect()
       case errorType =>
-        throw new IllegalStateException(s"Unknown Selector Type: $errorType")
+        throw new IllegalArgumentException(s"Unknown Selector Type: $errorType")
     }
 
     copyValues(createSelectorModel(uid, indices.sorted)
@@ -283,11 +275,6 @@ private[ml] abstract class SelectorModel[T <: SelectorModel[T]] (
   extends Model[T] with SelectorParams with MLWritable {
   self: T =>
 
-  if (selectedFeatures.length >= 2) {
-    require(selectedFeatures.sliding(2).forall(l => l(0) < l(1)),
-      "Index should be strictly increasing.")
-  }
-
   /** @group setParam */
   @Since("3.1.0")
   def setFeaturesCol(value: String): this.type = set(featuresCol, value)
@@ -302,7 +289,8 @@ private[ml] abstract class SelectorModel[T <: SelectorModel[T]] (
   override def transform(dataset: Dataset[_]): DataFrame = {
     val outputSchema = transformSchema(dataset.schema, logging = true)
 
-    SelectorModel.transform(dataset, selectedFeatures, outputSchema, $(outputCol), $(featuresCol))
+    SelectorModel.transform(dataset, selectedFeatures.sorted, outputSchema,
+      $(outputCol), $(featuresCol))
   }
 
   @Since("3.1.0")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/StandardScaler.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/StandardScaler.scala
index 7434b1adb2ff2..92dee46ad0055 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/StandardScaler.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/StandardScaler.scala
@@ -314,6 +314,8 @@ object StandardScalerModel extends MLReadable[StandardScalerModel] {
             case SparseVector(size, indices, values) =>
               val newValues = transformSparseWithScale(scale, indices, values.clone())
               Vectors.sparse(size, indices, newValues)
+            case v =>
+              throw new IllegalArgumentException(s"Unknown vector type ${v.getClass}.")
           }
 
       case (false, false) =>
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/StopWordsRemover.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/StopWordsRemover.scala
index b6ed4f2b000cc..8bcd7909b6078 100755
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/StopWordsRemover.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/StopWordsRemover.scala
@@ -166,11 +166,11 @@ class StopWordsRemover @Since("1.5.0") (@Since("1.5.0") override val uid: String
     }
 
     val (inputColNames, outputColNames) = getInOutCols()
-    val ouputCols = inputColNames.map { inputColName =>
+    val outputCols = inputColNames.map { inputColName =>
       t(col(inputColName))
     }
-    val ouputMetadata = outputColNames.map(outputSchema(_).metadata)
-    dataset.withColumns(outputColNames, ouputCols, ouputMetadata)
+    val outputMetadata = outputColNames.map(outputSchema(_).metadata)
+    dataset.withColumns(outputColNames, outputCols, outputMetadata)
   }
 
   @Since("1.5.0")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala
index be32f44287b6a..0ca88b8e61e29 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala
@@ -56,8 +56,6 @@ private[feature] trait StringIndexerBase extends Params with HasHandleInvalid wi
     "or 'keep' (put invalid data in a special additional bucket, at index numLabels).",
     ParamValidators.inArray(StringIndexer.supportedHandleInvalids))
 
-  setDefault(handleInvalid, StringIndexer.ERROR_INVALID)
-
   /**
    * Param for how to order labels of string column. The first label after ordering is assigned
    * an index of 0.
@@ -80,6 +78,9 @@ private[feature] trait StringIndexerBase extends Params with HasHandleInvalid wi
     s"Supported options: ${StringIndexer.supportedStringOrderType.mkString(", ")}.",
     ParamValidators.inArray(StringIndexer.supportedStringOrderType))
 
+  setDefault(handleInvalid -> StringIndexer.ERROR_INVALID,
+    stringOrderType -> StringIndexer.frequencyDesc)
+
   /** @group getParam */
   @Since("2.3.0")
   def getStringOrderType: String = $(stringOrderType)
@@ -155,7 +156,6 @@ class StringIndexer @Since("1.4.0") (
   /** @group setParam */
   @Since("2.3.0")
   def setStringOrderType(value: String): this.type = set(stringOrderType, value)
-  setDefault(stringOrderType, StringIndexer.frequencyDesc)
 
   /** @group setParam */
   @Since("1.4.0")
@@ -220,7 +220,8 @@ class StringIndexer @Since("1.4.0") (
 
     val selectedCols = getSelectedCols(dataset, inputCols).map(collect_set(_))
     val allLabels = dataset.select(selectedCols: _*)
-      .collect().toSeq.flatMap(_.toSeq).asInstanceOf[Seq[Seq[String]]]
+      .collect().toSeq.flatMap(_.toSeq)
+      .asInstanceOf[scala.collection.Seq[scala.collection.Seq[String]]].toSeq
     ThreadUtils.parmap(allLabels, "sortingStringLabels", 8) { labels =>
       val sorted = labels.filter(_ != null).sorted
       if (ascending) {
@@ -522,7 +523,7 @@ object StringIndexerModel extends MLReadable[StringIndexerModel] {
         val data = sparkSession.read.parquet(dataPath)
           .select("labelsArray")
           .head()
-        data.getAs[Seq[Seq[String]]](0).map(_.toArray).toArray
+        data.getSeq[scala.collection.Seq[String]](0).map(_.toArray).toArray
       }
       val model = new StringIndexerModel(metadata.uid, labelsArray)
       metadata.getAndSetParams(model)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/UnivariateFeatureSelector.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/UnivariateFeatureSelector.scala
new file mode 100644
index 0000000000000..7fff1599594f7
--- /dev/null
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/UnivariateFeatureSelector.scala
@@ -0,0 +1,450 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.feature
+
+import scala.collection.mutable.ArrayBuilder
+
+import org.apache.hadoop.fs.Path
+
+import org.apache.spark.annotation.Since
+import org.apache.spark.ml.{Estimator, Model}
+import org.apache.spark.ml.attribute.{Attribute, AttributeGroup, NominalAttribute, NumericAttribute}
+import org.apache.spark.ml.linalg.{DenseVector, SparseVector, Vector, Vectors, VectorUDT}
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared.{HasFeaturesCol, HasLabelCol, HasOutputCol}
+import org.apache.spark.ml.stat.{ANOVATest, ChiSquareTest, FValueTest}
+import org.apache.spark.ml.util._
+import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}
+import org.apache.spark.sql.functions.{col, udf}
+import org.apache.spark.sql.types.{StructField, StructType}
+
+
+/**
+ * Params for [[UnivariateFeatureSelector]] and [[UnivariateFeatureSelectorModel]].
+ */
+private[feature] trait UnivariateFeatureSelectorParams extends Params
+  with HasFeaturesCol with HasLabelCol with HasOutputCol {
+
+  /**
+   * The feature type.
+   * Supported options: "categorical", "continuous"
+   * @group param
+   */
+  @Since("3.1.1")
+  final val featureType = new Param[String](this, "featureType",
+    "Feature type. Supported options: categorical, continuous.",
+    ParamValidators.inArray(Array("categorical", "continuous")))
+
+  /** @group getParam */
+  @Since("3.1.1")
+  def getFeatureType: String = $(featureType)
+
+  /**
+   * The label type.
+   * Supported options: "categorical", "continuous"
+   * @group param
+   */
+  @Since("3.1.1")
+  final val labelType = new Param[String](this, "labelType",
+    "Label type. Supported options: categorical, continuous.",
+    ParamValidators.inArray(Array("categorical", "continuous")))
+
+  /** @group getParam */
+  @Since("3.1.1")
+  def getLabelType: String = $(labelType)
+
+  /**
+   * The selection mode.
+   * Supported options: "numTopFeatures" (default), "percentile", "fpr", "fdr", "fwe"
+   * @group param
+   */
+  @Since("3.1.1")
+  final val selectionMode = new Param[String](this, "selectionMode",
+    "The selection mode. Supported options: numTopFeatures, percentile, fpr, fdr, fwe",
+    ParamValidators.inArray(Array("numTopFeatures", "percentile", "fpr", "fdr", "fwe")))
+
+  /** @group getParam */
+  @Since("3.1.1")
+  def getSelectionMode: String = $(selectionMode)
+
+  /**
+   * The upper bound of the features that selector will select.
+   * @group param
+   */
+  @Since("3.1.1")
+  final val selectionThreshold = new DoubleParam(this, "selectionThreshold",
+    "The upper bound of the features that selector will select.")
+
+  /** @group getParam */
+  def getSelectionThreshold: Double = $(selectionThreshold)
+
+  setDefault(selectionMode -> "numTopFeatures")
+}
+
+/**
+ * The user can set `featureType` and labelType`, and Spark will pick the score function based on
+ * the specified `featureType` and labelType`.
+ * The following combination of `featureType` and `labelType` are supported:
+ *  - `featureType` `categorical` and `labelType` `categorical`: Spark uses chi-squared,
+ *    i.e. chi2 in sklearn.
+ *  - `featureType` `continuous` and `labelType` `categorical`: Spark uses ANOVATest,
+ *    i.e. f_classif in sklearn.
+ *  - `featureType` `continuous` and `labelType` `continuous`: Spark uses F-value,
+ *    i.e. f_regression in sklearn.
+ *
+ * The `UnivariateFeatureSelector` supports different selection modes: `numTopFeatures`,
+ * `percentile`, `fpr`, `fdr`, `fwe`.
+ *  - `numTopFeatures` chooses a fixed number of top features according to a hypothesis.
+ *  - `percentile` is similar but chooses a fraction of all features instead of a fixed number.
+ *  - `fpr` chooses all features whose p-value are below a threshold, thus controlling the false
+ *    positive rate of selection.
+ *  - `fdr` uses the <a href=
+ *  "https://en.wikipedia.org/wiki/False_discovery_rate#Benjamini.E2.80.93Hochberg_procedure">
+ *  Benjamini-Hochberg procedure</a>
+ *    to choose all features whose false discovery rate is below a threshold.
+ *  - `fwe` chooses all features whose p-values are below a threshold. The threshold is scaled by
+ *    1/numFeatures, thus controlling the family-wise error rate of selection.
+ *
+ * By default, the selection mode is `numTopFeatures`.
+ */
+@Since("3.1.1")
+final class UnivariateFeatureSelector @Since("3.1.1")(@Since("3.1.1") override val uid: String)
+  extends Estimator[UnivariateFeatureSelectorModel] with UnivariateFeatureSelectorParams
+    with DefaultParamsWritable {
+
+  @Since("3.1.1")
+  def this() = this(Identifiable.randomUID("UnivariateFeatureSelector"))
+
+  /** @group setParam */
+  @Since("3.1.1")
+  def setSelectionMode(value: String): this.type = set(selectionMode, value)
+
+  /** @group setParam */
+  @Since("3.1.1")
+  def setSelectionThreshold(value: Double): this.type = set(selectionThreshold, value)
+
+  /** @group setParam */
+  @Since("3.1.1")
+  def setFeatureType(value: String): this.type = set(featureType, value)
+
+  /** @group setParam */
+  @Since("3.1.1")
+  def setLabelType(value: String): this.type = set(labelType, value)
+
+  /** @group setParam */
+  @Since("3.1.1")
+  def setFeaturesCol(value: String): this.type = set(featuresCol, value)
+
+  /** @group setParam */
+  @Since("3.1.1")
+  def setOutputCol(value: String): this.type = set(outputCol, value)
+
+  /** @group setParam */
+  @Since("3.1.1")
+  def setLabelCol(value: String): this.type = set(labelCol, value)
+
+  @Since("3.1.1")
+  override def fit(dataset: Dataset[_]): UnivariateFeatureSelectorModel = {
+    transformSchema(dataset.schema, logging = true)
+    val numFeatures = MetadataUtils.getNumFeatures(dataset, $(featuresCol))
+
+    var threshold = Double.NaN
+    if (isSet(selectionThreshold)) {
+      threshold = $(selectionThreshold)
+    } else {
+      $(selectionMode) match {
+        case "numTopFeatures" => threshold = 50
+        case "percentile" => threshold = 0.1
+        case "fpr" | "fdr" | "fwe" => threshold = 0.05
+      }
+    }
+
+    val resultDF = ($(featureType), $(labelType)) match {
+      case ("categorical", "categorical") =>
+        ChiSquareTest.test(dataset.toDF, getFeaturesCol, getLabelCol, true)
+      case ("continuous", "categorical") =>
+        ANOVATest.test(dataset.toDF, getFeaturesCol, getLabelCol, true)
+      case ("continuous", "continuous") =>
+        FValueTest.test(dataset.toDF, getFeaturesCol, getLabelCol, true)
+      case _ =>
+        throw new IllegalArgumentException(s"Unsupported combination:" +
+          s" featureType=${$(featureType)}, labelType=${$(labelType)}")
+    }
+
+    val indices = selectIndicesFromPValues(numFeatures, resultDF, $(selectionMode), threshold)
+    copyValues(new UnivariateFeatureSelectorModel(uid, indices)
+      .setParent(this))
+  }
+
+  private def getTopIndices(df: DataFrame, k: Int): Array[Int] = {
+    val spark = SparkSession.builder().getOrCreate()
+    import spark.implicits._
+    df.sort("pValue", "featureIndex")
+      .select("featureIndex")
+      .limit(k)
+      .as[Int]
+      .collect()
+  }
+
+  private[feature] def selectIndicesFromPValues(
+      numFeatures: Int,
+      resultDF: DataFrame,
+      selectionMode: String,
+      selectionThreshold: Double): Array[Int] = {
+    val spark = SparkSession.builder().getOrCreate()
+    import spark.implicits._
+    val indices = selectionMode match {
+      case "numTopFeatures" =>
+        getTopIndices(resultDF, selectionThreshold.toInt)
+      case "percentile" =>
+        getTopIndices(resultDF, (numFeatures * selectionThreshold).toInt)
+      case "fpr" =>
+        resultDF.select("featureIndex")
+          .where(col("pValue") < selectionThreshold)
+          .as[Int].collect()
+      case "fdr" =>
+        // This uses the Benjamini-Hochberg procedure.
+        // https://en.wikipedia.org/wiki/False_discovery_rate#Benjamini.E2.80.93Hochberg_procedure
+        val f = selectionThreshold / numFeatures
+        val maxIndex = resultDF.sort("pValue", "featureIndex")
+          .select("pValue")
+          .as[Double].rdd
+          .zipWithIndex
+          .flatMap { case (pValue, index) =>
+            if (pValue <= f * (index + 1)) {
+              Iterator.single(index.toInt)
+            } else Iterator.empty
+          }.fold(-1)(math.max)
+        if (maxIndex >= 0) {
+          getTopIndices(resultDF, maxIndex + 1)
+        } else Array.emptyIntArray
+      case "fwe" =>
+        resultDF.select("featureIndex")
+          .where(col("pValue") < selectionThreshold / numFeatures)
+          .as[Int].collect()
+      case errorType =>
+        throw new IllegalArgumentException(s"Unknown Selector Type: $errorType")
+    }
+    indices
+  }
+
+  @Since("3.1.1")
+  override def transformSchema(schema: StructType): StructType = {
+    if (isSet(selectionThreshold)) {
+      val threshold = $(selectionThreshold)
+      $(selectionMode) match {
+        case "numTopFeatures" =>
+          require(threshold >= 1 && threshold.toInt == threshold,
+            s"selectionThreshold needs to be a positive Integer for selection mode " +
+              s"numTopFeatures, but got $threshold")
+        case "percentile" | "fpr" | "fdr" | "fwe" =>
+          require(0 <= threshold && threshold <= 1,
+            s"selectionThreshold needs to be in the range [0, 1] for selection mode " +
+              s"${$(selectionMode)}, but got $threshold")
+      }
+    }
+    require(isSet(featureType) && isSet(labelType), "featureType and labelType need to be set")
+    SchemaUtils.checkColumnType(schema, $(featuresCol), new VectorUDT)
+    SchemaUtils.checkNumericType(schema, $(labelCol))
+    SchemaUtils.appendColumn(schema, $(outputCol), new VectorUDT)
+  }
+
+  @Since("3.1.1")
+  override def copy(extra: ParamMap): UnivariateFeatureSelector = defaultCopy(extra)
+}
+
+@Since("3.1.1")
+object UnivariateFeatureSelector extends DefaultParamsReadable[UnivariateFeatureSelector] {
+
+  @Since("3.1.1")
+  override def load(path: String): UnivariateFeatureSelector = super.load(path)
+}
+
+/**
+ * Model fitted by [[UnivariateFeatureSelectorModel]].
+ */
+@Since("3.1.1")
+class UnivariateFeatureSelectorModel private[ml](
+    @Since("3.1.1") override val uid: String,
+    @Since("3.1.1") val selectedFeatures: Array[Int])
+  extends Model[UnivariateFeatureSelectorModel] with UnivariateFeatureSelectorParams
+    with MLWritable {
+
+  /** @group setParam */
+  @Since("3.1.1")
+  def setFeaturesCol(value: String): this.type = set(featuresCol, value)
+
+  /** @group setParam */
+  @Since("3.1.1")
+  def setOutputCol(value: String): this.type = set(outputCol, value)
+
+  protected def isNumericAttribute = true
+
+  @Since("3.1.1")
+  override def transform(dataset: Dataset[_]): DataFrame = {
+    val outputSchema = transformSchema(dataset.schema, logging = true)
+
+    UnivariateFeatureSelectorModel
+      .transform(dataset, selectedFeatures.sorted, outputSchema, $(outputCol), $(featuresCol))
+  }
+
+  @Since("3.1.1")
+  override def transformSchema(schema: StructType): StructType = {
+    SchemaUtils.checkColumnType(schema, $(featuresCol), new VectorUDT)
+    val newField =
+      UnivariateFeatureSelectorModel
+        .prepOutputField(schema, selectedFeatures, $(outputCol), $(featuresCol), isNumericAttribute)
+    SchemaUtils.appendColumn(schema, newField)
+  }
+
+  @Since("3.1.1")
+  override def copy(extra: ParamMap): UnivariateFeatureSelectorModel = {
+    val copied = new UnivariateFeatureSelectorModel(uid, selectedFeatures)
+      .setParent(parent)
+    copyValues(copied, extra)
+  }
+
+  @Since("3.1.1")
+  override def write: MLWriter =
+    new UnivariateFeatureSelectorModel.UnivariateFeatureSelectorModelWriter(this)
+
+  @Since("3.1.1")
+  override def toString: String = {
+    s"UnivariateFeatureSelectorModel: uid=$uid, numSelectedFeatures=${selectedFeatures.length}"
+  }
+}
+
+@Since("3.1.1")
+object UnivariateFeatureSelectorModel extends MLReadable[UnivariateFeatureSelectorModel] {
+
+  @Since("3.1.1")
+  override def read: MLReader[UnivariateFeatureSelectorModel] =
+    new UnivariateFeatureSelectorModelReader
+
+  @Since("3.1.1")
+  override def load(path: String): UnivariateFeatureSelectorModel = super.load(path)
+
+  private[UnivariateFeatureSelectorModel] class UnivariateFeatureSelectorModelWriter(
+      instance: UnivariateFeatureSelectorModel) extends MLWriter {
+
+    private case class Data(selectedFeatures: Seq[Int])
+
+    override protected def saveImpl(path: String): Unit = {
+      DefaultParamsWriter.saveMetadata(instance, path, sc)
+      val data = Data(instance.selectedFeatures.toSeq)
+      val dataPath = new Path(path, "data").toString
+      sparkSession.createDataFrame(Seq(data)).repartition(1).write.parquet(dataPath)
+    }
+  }
+
+  private class UnivariateFeatureSelectorModelReader
+    extends MLReader[UnivariateFeatureSelectorModel] {
+
+    /** Checked against metadata when loading model */
+    private val className = classOf[UnivariateFeatureSelectorModel].getName
+
+    override def load(path: String): UnivariateFeatureSelectorModel = {
+      val metadata = DefaultParamsReader.loadMetadata(path, sc, className)
+      val dataPath = new Path(path, "data").toString
+      val data = sparkSession.read.parquet(dataPath)
+        .select("selectedFeatures").head()
+      val selectedFeatures = data.getAs[Seq[Int]](0).toArray
+      val model = new UnivariateFeatureSelectorModel(metadata.uid, selectedFeatures)
+      metadata.getAndSetParams(model)
+      model
+    }
+  }
+
+  private def transform(
+      dataset: Dataset[_],
+      selectedFeatures: Array[Int],
+      outputSchema: StructType,
+      outputCol: String,
+      featuresCol: String): DataFrame = {
+    val newSize = selectedFeatures.length
+    val func = { vector: Vector =>
+      vector match {
+        case SparseVector(_, indices, values) =>
+          val (newIndices, newValues) =
+            compressSparse(indices, values, selectedFeatures)
+          Vectors.sparse(newSize, newIndices, newValues)
+        case DenseVector(values) =>
+          Vectors.dense(selectedFeatures.map(values))
+        case other =>
+          throw new UnsupportedOperationException(
+            s"Only sparse and dense vectors are supported but got ${other.getClass}.")
+      }
+    }
+
+    val transformer = udf(func)
+    dataset.withColumn(outputCol, transformer(col(featuresCol)),
+      outputSchema(outputCol).metadata)
+  }
+
+  /**
+   * Prepare the output column field, including per-feature metadata.
+   */
+  private def prepOutputField(
+      schema: StructType,
+      selectedFeatures: Array[Int],
+      outputCol: String,
+      featuresCol: String,
+      isNumericAttribute: Boolean): StructField = {
+    val selector = selectedFeatures.toSet
+    val origAttrGroup = AttributeGroup.fromStructField(schema(featuresCol))
+    val featureAttributes: Array[Attribute] = if (origAttrGroup.attributes.nonEmpty) {
+      origAttrGroup.attributes.get.zipWithIndex.filter(x => selector.contains(x._2)).map(_._1)
+    } else {
+      if (isNumericAttribute) {
+        Array.fill[Attribute](selector.size)(NumericAttribute.defaultAttr)
+      } else {
+        Array.fill[Attribute](selector.size)(NominalAttribute.defaultAttr)
+      }
+    }
+    val newAttributeGroup = new AttributeGroup(outputCol, featureAttributes)
+    newAttributeGroup.toStructField()
+  }
+
+  private def compressSparse(
+      indices: Array[Int],
+      values: Array[Double],
+      selectedFeatures: Array[Int]): (Array[Int], Array[Double]) = {
+    val newValues = new ArrayBuilder.ofDouble
+    val newIndices = new ArrayBuilder.ofInt
+    var i = 0
+    var j = 0
+    while (i < indices.length && j < selectedFeatures.length) {
+      val indicesIdx = indices(i)
+      val filterIndicesIdx = selectedFeatures(j)
+      if (indicesIdx == filterIndicesIdx) {
+        newIndices += j
+        newValues += values(i)
+        j += 1
+        i += 1
+      } else {
+        if (indicesIdx > filterIndicesIdx) {
+          j += 1
+        } else {
+          i += 1
+        }
+      }
+    }
+    // TODO: Sparse representation might be ineffective if (newSize ~= newValues.size)
+    (newIndices.result(), newValues.result())
+  }
+}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/VarianceThresholdSelector.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/VarianceThresholdSelector.scala
index cd245dd723348..2c7186015d400 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/VarianceThresholdSelector.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/VarianceThresholdSelector.scala
@@ -17,13 +17,10 @@
 
 package org.apache.spark.ml.feature
 
-import scala.collection.mutable.ArrayBuilder
-
 import org.apache.hadoop.fs.Path
 
 import org.apache.spark.annotation.Since
 import org.apache.spark.ml._
-import org.apache.spark.ml.attribute.{Attribute, AttributeGroup, NominalAttribute}
 import org.apache.spark.ml.linalg._
 import org.apache.spark.ml.param._
 import org.apache.spark.ml.param.shared._
@@ -31,7 +28,7 @@ import org.apache.spark.ml.stat.Summarizer
 import org.apache.spark.ml.util._
 import org.apache.spark.sql._
 import org.apache.spark.sql.functions._
-import org.apache.spark.sql.types.{StructField, StructType}
+import org.apache.spark.sql.types.StructType
 
 
 /**
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorIndexer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorIndexer.scala
index 866074fb1453e..874b421387279 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorIndexer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorIndexer.scala
@@ -60,8 +60,6 @@ private[ml] trait VectorIndexerParams extends Params with HasInputCol with HasOu
     "number of categories of the feature).",
     ParamValidators.inArray(VectorIndexer.supportedHandleInvalids))
 
-  setDefault(handleInvalid, VectorIndexer.ERROR_INVALID)
-
   /**
    * Threshold for the number of values a categorical feature can take.
    * If a feature is found to have {@literal >} maxCategories values, then it is declared
@@ -75,10 +73,10 @@ private[ml] trait VectorIndexerParams extends Params with HasInputCol with HasOu
       " If a feature is found to have > maxCategories values, then it is declared continuous.",
     ParamValidators.gtEq(2))
 
-  setDefault(maxCategories -> 20)
-
   /** @group getParam */
   def getMaxCategories: Int = $(maxCategories)
+
+  setDefault(maxCategories -> 20, handleInvalid -> VectorIndexer.ERROR_INVALID)
 }
 
 /**
@@ -302,7 +300,7 @@ class VectorIndexerModel private[ml] (
   /** Java-friendly version of [[categoryMaps]] */
   @Since("1.4.0")
   def javaCategoryMaps: JMap[JInt, JMap[JDouble, JInt]] = {
-    categoryMaps.mapValues(_.asJava).asJava.asInstanceOf[JMap[JInt, JMap[JDouble, JInt]]]
+    categoryMaps.mapValues(_.asJava).toMap.asJava.asInstanceOf[JMap[JInt, JMap[JDouble, JInt]]]
   }
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorSlicer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorSlicer.scala
index be91844ba39e6..fd44b1c622ddd 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorSlicer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorSlicer.scala
@@ -19,7 +19,7 @@ package org.apache.spark.ml.feature
 
 import org.apache.spark.annotation.Since
 import org.apache.spark.ml.Transformer
-import org.apache.spark.ml.attribute.{Attribute, AttributeGroup}
+import org.apache.spark.ml.attribute.AttributeGroup
 import org.apache.spark.ml.linalg._
 import org.apache.spark.ml.param.{IntArrayParam, ParamMap, StringArrayParam}
 import org.apache.spark.ml.param.shared.{HasInputCol, HasOutputCol}
@@ -57,8 +57,6 @@ final class VectorSlicer @Since("1.5.0") (@Since("1.5.0") override val uid: Stri
     "An array of indices to select features from a vector column." +
       " There can be no overlap with names.", VectorSlicer.validIndices)
 
-  setDefault(indices -> Array.emptyIntArray)
-
   /** @group getParam */
   @Since("1.5.0")
   def getIndices: Array[Int] = $(indices)
@@ -79,8 +77,6 @@ final class VectorSlicer @Since("1.5.0") (@Since("1.5.0") override val uid: Stri
     "An array of feature names to select features from a vector column." +
       " There can be no overlap with indices.", VectorSlicer.validNames)
 
-  setDefault(names -> Array.empty[String])
-
   /** @group getParam */
   @Since("1.5.0")
   def getNames: Array[String] = $(names)
@@ -97,34 +93,38 @@ final class VectorSlicer @Since("1.5.0") (@Since("1.5.0") override val uid: Stri
   @Since("1.5.0")
   def setOutputCol(value: String): this.type = set(outputCol, value)
 
+  setDefault(indices -> Array.emptyIntArray, names -> Array.empty[String])
+
   @Since("2.0.0")
   override def transform(dataset: Dataset[_]): DataFrame = {
     // Validity checks
     transformSchema(dataset.schema)
     val inputAttr = AttributeGroup.fromStructField(dataset.schema($(inputCol)))
-    inputAttr.numAttributes.foreach { numFeatures =>
-      val maxIndex = $(indices).max
-      require(maxIndex < numFeatures,
-        s"Selected feature index $maxIndex invalid for only $numFeatures input features.")
+    if ($(indices).nonEmpty) {
+      val size = inputAttr.size
+      if (size >= 0) {
+        val maxIndex = $(indices).max
+        require(maxIndex < size,
+          s"Selected feature index $maxIndex invalid for only $size input features.")
+      }
     }
 
     // Prepare output attributes
-    val inds = getSelectedFeatureIndices(dataset.schema)
-    val selectedAttrs: Option[Array[Attribute]] = inputAttr.attributes.map { attrs =>
-      inds.map(index => attrs(index))
-    }
+    val selectedIndices = getSelectedFeatureIndices(dataset.schema)
+    val selectedAttrs = inputAttr.attributes.map { attrs => selectedIndices.map(attrs.apply) }
     val outputAttr = selectedAttrs match {
       case Some(attrs) => new AttributeGroup($(outputCol), attrs)
-      case None => new AttributeGroup($(outputCol), inds.length)
+      case None => new AttributeGroup($(outputCol), selectedIndices.length)
     }
 
-    // Select features
+    val sorted = selectedIndices.length > 1 && selectedIndices.sliding(2).forall(t => t(1) > t(0))
     val slicer = udf { vec: Vector =>
       vec match {
-        case features: DenseVector => Vectors.dense(inds.map(features.apply))
-        case features: SparseVector => features.slice(inds)
+        case dv: DenseVector => Vectors.dense(selectedIndices.map(dv.apply))
+        case sv: SparseVector => sv.slice(selectedIndices, sorted)
       }
     }
+
     dataset.withColumn($(outputCol), slicer(dataset($(inputCol))), outputAttr.toMetadata())
   }
 
@@ -161,8 +161,13 @@ final class VectorSlicer @Since("1.5.0") (@Since("1.5.0") override val uid: Stri
 
   @Since("3.0.0")
   override def toString: String = {
-    s"VectorSlicer: uid=$uid" +
-      get(indices).map(i => s", numSelectedFeatures=${i.length}").getOrElse("")
+    val numSelectedFeatures =
+      get(indices).map(_.length).getOrElse(0) + get(names).map(_.length).getOrElse(0)
+    if (numSelectedFeatures > 0) {
+      s"VectorSlicer: uid=$uid, numSelectedFeatures=$numSelectedFeatures"
+    } else {
+      s"VectorSlicer: uid=$uid"
+    }
   }
 }
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/Word2Vec.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/Word2Vec.scala
index bbfcbfbe038ef..6a22c3580e377 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/Word2Vec.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/Word2Vec.scala
@@ -27,7 +27,6 @@ import org.apache.spark.ml.param._
 import org.apache.spark.ml.param.shared._
 import org.apache.spark.ml.util._
 import org.apache.spark.mllib.feature
-import org.apache.spark.mllib.linalg.VectorImplicits._
 import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types._
@@ -47,7 +46,6 @@ private[feature] trait Word2VecBase extends Params
   final val vectorSize = new IntParam(
     this, "vectorSize", "the dimension of codes after transforming from words (> 0)",
     ParamValidators.gt(0))
-  setDefault(vectorSize -> 100)
 
   /** @group getParam */
   def getVectorSize: Int = $(vectorSize)
@@ -60,7 +58,6 @@ private[feature] trait Word2VecBase extends Params
   final val windowSize = new IntParam(
     this, "windowSize", "the window size (context words from [-window, window]) (> 0)",
     ParamValidators.gt(0))
-  setDefault(windowSize -> 5)
 
   /** @group expertGetParam */
   def getWindowSize: Int = $(windowSize)
@@ -73,7 +70,6 @@ private[feature] trait Word2VecBase extends Params
   final val numPartitions = new IntParam(
     this, "numPartitions", "number of partitions for sentences of words (> 0)",
     ParamValidators.gt(0))
-  setDefault(numPartitions -> 1)
 
   /** @group getParam */
   def getNumPartitions: Int = $(numPartitions)
@@ -86,7 +82,6 @@ private[feature] trait Word2VecBase extends Params
    */
   final val minCount = new IntParam(this, "minCount", "the minimum number of times a token must " +
     "appear to be included in the word2vec model's vocabulary (>= 0)", ParamValidators.gtEq(0))
-  setDefault(minCount -> 5)
 
   /** @group getParam */
   def getMinCount: Int = $(minCount)
@@ -101,13 +96,12 @@ private[feature] trait Word2VecBase extends Params
   final val maxSentenceLength = new IntParam(this, "maxSentenceLength", "Maximum length " +
     "(in words) of each sentence in the input data. Any sentence longer than this threshold will " +
     "be divided into chunks up to the size (> 0)", ParamValidators.gt(0))
-  setDefault(maxSentenceLength -> 1000)
 
   /** @group getParam */
   def getMaxSentenceLength: Int = $(maxSentenceLength)
 
-  setDefault(stepSize -> 0.025)
-  setDefault(maxIter -> 1)
+  setDefault(vectorSize -> 100, windowSize -> 5, numPartitions -> 1, minCount -> 5,
+    maxSentenceLength -> 1000, stepSize -> 0.025, maxIter -> 1)
 
   /**
    * Validate and transform the input schema.
@@ -174,7 +168,8 @@ final class Word2Vec @Since("1.4.0") (
   @Since("2.0.0")
   override def fit(dataset: Dataset[_]): Word2VecModel = {
     transformSchema(dataset.schema, logging = true)
-    val input = dataset.select($(inputCol)).rdd.map(_.getAs[Seq[String]](0))
+    val input =
+      dataset.select($(inputCol)).rdd.map(_.getSeq[String](0))
     val wordVectors = new feature.Word2Vec()
       .setLearningRate($(stepSize))
       .setMinCount($(minCount))
@@ -260,7 +255,7 @@ class Word2VecModel private[ml] (
    */
   @Since("2.2.0")
   def findSynonymsArray(vec: Vector, num: Int): Array[(String, Double)] = {
-    wordVectors.findSynonyms(vec, num)
+    wordVectors.findSynonyms(vec.toArray, num, None)
   }
 
   /**
@@ -289,27 +284,33 @@ class Word2VecModel private[ml] (
   @Since("2.0.0")
   override def transform(dataset: Dataset[_]): DataFrame = {
     val outputSchema = transformSchema(dataset.schema, logging = true)
-    val vectors = wordVectors.getVectors
-      .mapValues(vv => Vectors.dense(vv.map(_.toDouble)))
-      .map(identity) // mapValues doesn't return a serializable map (SI-7005)
-    val bVectors = dataset.sparkSession.sparkContext.broadcast(vectors)
-    val d = $(vectorSize)
-    val emptyVec = Vectors.sparse(d, Array.emptyIntArray, Array.emptyDoubleArray)
-    val word2Vec = udf { sentence: Seq[String] =>
+
+    val bcModel = dataset.sparkSession.sparkContext.broadcast(this.wordVectors)
+    val size = $(vectorSize)
+    val emptyVec = Vectors.sparse(size, Array.emptyIntArray, Array.emptyDoubleArray)
+    val transformer = udf { sentence: Seq[String] =>
       if (sentence.isEmpty) {
         emptyVec
       } else {
-        val sum = Vectors.zeros(d)
+        val wordIndices = bcModel.value.wordIndex
+        val wordVectors = bcModel.value.wordVectors
+        val array = Array.ofDim[Double](size)
+        var count = 0
         sentence.foreach { word =>
-          bVectors.value.get(word).foreach { v =>
-            BLAS.axpy(1.0, v, sum)
+          wordIndices.get(word).foreach { index =>
+            val offset = index * size
+            var i = 0
+            while (i < size) { array(i) += wordVectors(offset + i); i += 1 }
           }
+          count += 1
         }
-        BLAS.scal(1.0 / sentence.size, sum)
-        sum
+        val vec = Vectors.dense(array)
+        BLAS.scal(1.0 / count, vec)
+        vec
       }
     }
-    dataset.withColumn($(outputCol), word2Vec(col($(inputCol))),
+
+    dataset.withColumn($(outputCol), transformer(col($(inputCol))),
       outputSchema($(outputCol)).metadata)
   }
 
@@ -342,7 +343,7 @@ class Word2VecModel private[ml] (
 @Since("1.6.0")
 object Word2VecModel extends MLReadable[Word2VecModel] {
 
-  private case class Data(word: String, vector: Array[Float])
+  private[Word2VecModel] case class Data(word: String, vector: Array[Float])
 
   private[Word2VecModel]
   class Word2VecModelWriter(instance: Word2VecModel) extends MLWriter {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/package.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/package.scala
index 6ff970cc72dfd..ac63024768d77 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/package.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/package.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.ml
 
-import org.apache.spark.ml.feature.{HashingTF, IDF, IDFModel, VectorAssembler}
-
 /**
  * == Feature transformers ==
  *
diff --git a/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala b/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala
index e50d4255b1f37..8aaa5efdf06c5 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala
@@ -50,7 +50,6 @@ private[fpm] trait FPGrowthParams extends Params with HasPredictionCol {
    */
   @Since("2.2.0")
   val itemsCol: Param[String] = new Param[String](this, "itemsCol", "items column name")
-  setDefault(itemsCol -> "items")
 
   /** @group getParam */
   @Since("2.2.0")
@@ -66,7 +65,6 @@ private[fpm] trait FPGrowthParams extends Params with HasPredictionCol {
   val minSupport: DoubleParam = new DoubleParam(this, "minSupport",
     "the minimal support level of a frequent pattern",
     ParamValidators.inRange(0.0, 1.0))
-  setDefault(minSupport -> 0.3)
 
   /** @group getParam */
   @Since("2.2.0")
@@ -95,12 +93,13 @@ private[fpm] trait FPGrowthParams extends Params with HasPredictionCol {
   val minConfidence: DoubleParam = new DoubleParam(this, "minConfidence",
     "minimal confidence for generating Association Rule",
     ParamValidators.inRange(0.0, 1.0))
-  setDefault(minConfidence -> 0.8)
 
   /** @group getParam */
   @Since("2.2.0")
   def getMinConfidence: Double = $(minConfidence)
 
+  setDefault(minSupport -> 0.3, itemsCol -> "items", minConfidence -> 0.8)
+
   /**
    * Validates and transforms the input schema.
    * @param schema input schema
@@ -244,9 +243,9 @@ class FPGrowthModel private[ml] (
   @transient private var _cachedRules: DataFrame = _
 
   /**
-   * Get association rules fitted using the minConfidence. Returns a dataframe with four fields,
-   * "antecedent", "consequent", "confidence" and "lift", where "antecedent" and "consequent" are
-   * Array[T], whereas "confidence" and "lift" are Double.
+   * Get association rules fitted using the minConfidence. Returns a dataframe with five fields,
+   * "antecedent", "consequent", "confidence", "lift" and "support", where "antecedent" and
+   *  "consequent" are Array[T], whereas "confidence", "lift" and "support" are Double.
    */
   @Since("2.2.0")
   @transient def associationRules: DataFrame = {
@@ -254,7 +253,8 @@ class FPGrowthModel private[ml] (
       _cachedRules
     } else {
       _cachedRules = AssociationRules
-        .getAssociationRulesFromFP(freqItemsets, "items", "freq", $(minConfidence), itemSupport)
+        .getAssociationRulesFromFP(freqItemsets, "items", "freq", $(minConfidence), itemSupport,
+          numTrainingRecords)
       _cachedMinConf = $(minConfidence)
       _cachedRules
     }
@@ -363,7 +363,7 @@ object FPGrowthModel extends MLReadable[FPGrowthModel] {
         Map.empty[Any, Double]
       } else {
         frequentItems.rdd.flatMap {
-            case Row(items: Seq[_], count: Long) if items.length == 1 =>
+            case Row(items: scala.collection.Seq[_], count: Long) if items.length == 1 =>
               Some(items.head -> count.toDouble / numTrainingRecords)
             case _ => None
           }.collectAsMap()
@@ -385,6 +385,7 @@ private[fpm] object AssociationRules {
    * @param freqCol column name for appearance count of the frequent itemsets
    * @param minConfidence minimum confidence for generating the association rules
    * @param itemSupport map containing an item and its support
+   * @param numTrainingRecords count of training Dataset
    * @return a DataFrame("antecedent"[Array], "consequent"[Array], "confidence"[Double],
    *         "lift" [Double]) containing the association rules.
    */
@@ -393,21 +394,23 @@ private[fpm] object AssociationRules {
         itemsCol: String,
         freqCol: String,
         minConfidence: Double,
-        itemSupport: scala.collection.Map[T, Double]): DataFrame = {
-
+        itemSupport: scala.collection.Map[T, Double],
+        numTrainingRecords: Long): DataFrame = {
     val freqItemSetRdd = dataset.select(itemsCol, freqCol).rdd
       .map(row => new FreqItemset(row.getSeq[T](0).toArray, row.getLong(1)))
     val rows = new MLlibAssociationRules()
       .setMinConfidence(minConfidence)
       .run(freqItemSetRdd, itemSupport)
-      .map(r => Row(r.antecedent, r.consequent, r.confidence, r.lift.orNull))
+      .map(r => Row(r.antecedent, r.consequent, r.confidence, r.lift.orNull,
+        r.freqUnion / numTrainingRecords))
 
     val dt = dataset.schema(itemsCol).dataType
     val schema = StructType(Seq(
       StructField("antecedent", dt, nullable = false),
       StructField("consequent", dt, nullable = false),
       StructField("confidence", DoubleType, nullable = false),
-      StructField("lift", DoubleType)))
+      StructField("lift", DoubleType),
+      StructField("support", DoubleType, nullable = false)))
     val rules = dataset.sparkSession.createDataFrame(rows, schema)
     rules
   }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/fpm/PrefixSpan.scala b/mllib/src/main/scala/org/apache/spark/ml/fpm/PrefixSpan.scala
index c9c049248f70c..10a569a8ff88b 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/fpm/PrefixSpan.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/fpm/PrefixSpan.scala
@@ -146,7 +146,7 @@ final class PrefixSpan(@Since("2.4.0") override val uid: String) extends Params
 
     val data = dataset.select(sequenceColParam)
     val sequences = data.where(col(sequenceColParam).isNotNull).rdd
-      .map(r => r.getAs[Seq[Seq[Any]]](0).map(_.toArray).toArray)
+      .map(r => r.getSeq[scala.collection.Seq[Any]](0).map(_.toArray).toArray)
 
     val mllibPrefixSpan = new mllibPrefixSpan()
       .setMinSupport($(minSupport))
diff --git a/mllib/src/main/scala/org/apache/spark/ml/functions.scala b/mllib/src/main/scala/org/apache/spark/ml/functions.scala
index 0f03231079866..43622a4f3edfb 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/functions.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/functions.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.ml
 
 import org.apache.spark.annotation.Since
-import org.apache.spark.ml.linalg.{SparseVector, Vector}
+import org.apache.spark.ml.linalg.{SparseVector, Vector, Vectors}
 import org.apache.spark.mllib.linalg.{Vector => OldVector}
 import org.apache.spark.sql.Column
 import org.apache.spark.sql.functions.udf
@@ -71,4 +71,24 @@ object functions {
       )
     }
   }
+
+  private val arrayToVectorUdf = udf { array: Seq[Double] =>
+    Vectors.dense(array.toArray)
+  }
+
+  /**
+   * Converts a column of array of numeric type into a column of dense vectors in MLlib.
+   * @param v: the column of array&lt;NumericType&gt type
+   * @return a column of type `org.apache.spark.ml.linalg.Vector`
+   * @since 3.1.0
+   */
+  def array_to_vector(v: Column): Column = {
+    arrayToVectorUdf(v)
+  }
+
+  private[ml] def checkNonNegativeWeight = udf {
+    value: Double =>
+      require(value >= 0, s"illegal weight value: $value. weight must be >= 0.0.")
+      value
+  }
 }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/image/ImageSchema.scala b/mllib/src/main/scala/org/apache/spark/ml/image/ImageSchema.scala
index 5efcf0dce68a2..37b715930a501 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/image/ImageSchema.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/image/ImageSchema.scala
@@ -133,7 +133,7 @@ object ImageSchema {
     val img = try {
       ImageIO.read(new ByteArrayInputStream(bytes))
     } catch {
-      // Catch runtime exception because `ImageIO` may throw unexcepted `RuntimeException`.
+      // Catch runtime exception because `ImageIO` may throw unexpected `RuntimeException`.
       // But do not catch the declared `IOException` (regarded as FileSystem failure)
       case _: RuntimeException => null
     }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/linalg/JsonMatrixConverter.scala b/mllib/src/main/scala/org/apache/spark/ml/linalg/JsonMatrixConverter.scala
index 0bee643412b3f..8f03a29eb991a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/linalg/JsonMatrixConverter.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/linalg/JsonMatrixConverter.scala
@@ -74,6 +74,8 @@ private[ml] object JsonMatrixConverter {
           ("values" -> values.toSeq) ~
           ("isTransposed" -> isTransposed)
         compact(render(jValue))
+      case _ =>
+        throw new IllegalArgumentException(s"Unknown matrix type ${m.getClass}.")
     }
   }
 }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/linalg/JsonVectorConverter.scala b/mllib/src/main/scala/org/apache/spark/ml/linalg/JsonVectorConverter.scala
index 781e69f8d63db..1b949d75eeaa0 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/linalg/JsonVectorConverter.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/linalg/JsonVectorConverter.scala
@@ -57,6 +57,8 @@ private[ml] object JsonVectorConverter {
       case DenseVector(values) =>
         val jValue = ("type" -> 1) ~ ("values" -> values.toSeq)
         compact(render(jValue))
+      case _ =>
+        throw new IllegalArgumentException(s"Unknown vector type ${v.getClass}.")
     }
   }
 }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/linalg/MatrixUDT.scala b/mllib/src/main/scala/org/apache/spark/ml/linalg/MatrixUDT.scala
index f4a8556c71f6e..838b51a07eb0a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/linalg/MatrixUDT.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/linalg/MatrixUDT.scala
@@ -22,7 +22,7 @@ import org.apache.spark.sql.catalyst.expressions.{GenericInternalRow, UnsafeArra
 import org.apache.spark.sql.types._
 
 /**
- * User-defined type for [[Matrix]] in [[mllib-local]] which allows easy interaction with SQL
+ * User-defined type for [[Matrix]] in mllib-local which allows easy interaction with SQL
  * via [[org.apache.spark.sql.Dataset]].
  */
 private[spark] class MatrixUDT extends UserDefinedType[Matrix] {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/linalg/VectorUDT.scala b/mllib/src/main/scala/org/apache/spark/ml/linalg/VectorUDT.scala
index 37f173bc20469..8c81acae34164 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/linalg/VectorUDT.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/linalg/VectorUDT.scala
@@ -22,7 +22,7 @@ import org.apache.spark.sql.catalyst.expressions.{GenericInternalRow, UnsafeArra
 import org.apache.spark.sql.types._
 
 /**
- * User-defined type for [[Vector]] in [[mllib-local]] which allows easy interaction with SQL
+ * User-defined type for [[Vector]] in mllib-local which allows easy interaction with SQL
  * via [[org.apache.spark.sql.Dataset]].
  */
 private[spark] class VectorUDT extends UserDefinedType[Vector] {
@@ -45,6 +45,8 @@ private[spark] class VectorUDT extends UserDefinedType[Vector] {
         row.setNullAt(2)
         row.update(3, UnsafeArrayData.fromPrimitiveArray(values))
         row
+      case v =>
+        throw new IllegalArgumentException(s"Unknown vector type ${v.getClass}.")
     }
   }
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/AFTAggregator.scala b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/AFTAggregator.scala
index 8a5d7fe34e7a0..fd59b4b71c41b 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/AFTAggregator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/AFTAggregator.scala
@@ -18,8 +18,8 @@
 package org.apache.spark.ml.optim.aggregator
 
 import org.apache.spark.broadcast.Broadcast
+import org.apache.spark.ml.feature._
 import org.apache.spark.ml.linalg._
-import org.apache.spark.ml.regression.AFTPoint
 
 /**
  * AFTAggregator computes the gradient and loss for a AFT loss function,
@@ -108,7 +108,7 @@ import org.apache.spark.ml.regression.AFTPoint
 private[ml] class AFTAggregator(
     bcFeaturesStd: Broadcast[Array[Double]],
     fitIntercept: Boolean)(bcCoefficients: Broadcast[Vector])
-  extends DifferentiableLossAggregator[AFTPoint, AFTAggregator] {
+  extends DifferentiableLossAggregator[Instance, AFTAggregator] {
 
   protected override val dim: Int = bcCoefficients.value.size
 
@@ -116,10 +116,10 @@ private[ml] class AFTAggregator(
    * Add a new training data to this AFTAggregator, and update the loss and gradient
    * of the objective function.
    *
-   * @param data The AFTPoint representation for one data point to be added into this aggregator.
+   * @param data The Instance representation for one data point to be added into this aggregator.
    * @return This AFTAggregator object.
    */
-  def add(data: AFTPoint): this.type = {
+  def add(data: Instance): this.type = {
     val coefficients = bcCoefficients.value.toArray
     val intercept = coefficients(dim - 2)
     // sigma is the scale parameter of the AFT model
@@ -127,7 +127,7 @@ private[ml] class AFTAggregator(
 
     val xi = data.features
     val ti = data.label
-    val delta = data.censor
+    val delta = data.weight
 
     require(ti > 0.0, "The lifetime or label should be  greater than 0.")
 
@@ -176,7 +176,7 @@ private[ml] class AFTAggregator(
  */
 private[ml] class BlockAFTAggregator(
     fitIntercept: Boolean)(bcCoefficients: Broadcast[Vector])
-  extends DifferentiableLossAggregator[(Matrix, Array[Double], Array[Double]),
+  extends DifferentiableLossAggregator[InstanceBlock,
     BlockAFTAggregator] {
 
   protected override val dim: Int = bcCoefficients.value.size
@@ -196,16 +196,13 @@ private[ml] class BlockAFTAggregator(
    *
    * @return This BlockAFTAggregator object.
    */
-  def add(block: (Matrix, Array[Double], Array[Double])): this.type = {
-    val (matrix, labels, censors) = block
-    require(matrix.isTransposed)
-    require(numFeatures == matrix.numCols, s"Dimensions mismatch when adding new " +
-      s"instance. Expecting $numFeatures but got ${matrix.numCols}.")
-    require(labels.forall(_ > 0.0), "The lifetime or label should be  greater than 0.")
-
-    val size = matrix.numRows
-    require(labels.length == size && censors.length == size)
+  def add(block: InstanceBlock): this.type = {
+    require(block.matrix.isTransposed)
+    require(numFeatures == block.numFeatures, s"Dimensions mismatch when adding new " +
+      s"instance. Expecting $numFeatures but got ${block.numFeatures}.")
+    require(block.labels.forall(_ > 0.0), "The lifetime or label should be  greater than 0.")
 
+    val size = block.size
     val intercept = coefficientsArray(dim - 2)
     // sigma is the scale parameter of the AFT model
     val sigma = math.exp(coefficientsArray(dim - 1))
@@ -216,26 +213,30 @@ private[ml] class BlockAFTAggregator(
     } else {
       Vectors.zeros(size).toDense
     }
-    BLAS.gemv(1.0, matrix, linear, 1.0, vec)
+    BLAS.gemv(1.0, block.matrix, linear, 1.0, vec)
 
     // in-place convert margins to gradient scales
     // then, vec represents gradient scales
+    var localLossSum = 0.0
     var i = 0
     var sigmaGradSum = 0.0
     while (i < size) {
-      val ti = labels(i)
-      val delta = censors(i)
+      val ti = block.getLabel(i)
+      // here use Instance.weight to store censor for convenience
+      val delta = block.getWeight(i)
       val margin = vec(i)
       val epsilon = (math.log(ti) - margin) / sigma
       val expEpsilon = math.exp(epsilon)
-      lossSum += delta * math.log(sigma) - delta * epsilon + expEpsilon
+      localLossSum += delta * math.log(sigma) - delta * epsilon + expEpsilon
       val multiplier = (delta - expEpsilon) / sigma
       vec.values(i) = multiplier
       sigmaGradSum += delta + multiplier * sigma * epsilon
       i += 1
     }
+    lossSum += localLossSum
+    weightSum += size
 
-    matrix match {
+    block.matrix match {
       case dm: DenseMatrix =>
         BLAS.nativeBLAS.dgemv("N", dm.numCols, dm.numRows, 1.0, dm.values, dm.numCols,
           vec.values, 1, 1.0, gradientSumArray, 1)
@@ -249,7 +250,6 @@ private[ml] class BlockAFTAggregator(
 
     if (fitIntercept) gradientSumArray(dim - 2) += vec.values.sum
     gradientSumArray(dim - 1) += sigmaGradSum
-    weightSum += size
 
     this
   }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/BinaryLogisticBlockAggregator.scala b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/BinaryLogisticBlockAggregator.scala
new file mode 100644
index 0000000000000..09a4335dad669
--- /dev/null
+++ b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/BinaryLogisticBlockAggregator.scala
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.ml.optim.aggregator
+
+import org.apache.spark.broadcast.Broadcast
+import org.apache.spark.internal.Logging
+import org.apache.spark.ml.feature.InstanceBlock
+import org.apache.spark.ml.impl.Utils
+import org.apache.spark.ml.linalg._
+
+/**
+ * BinaryLogisticBlockAggregator computes the gradient and loss used in binary logistic
+ * classification for blocks in sparse or dense matrix in an online fashion.
+ *
+ * Two BinaryLogisticBlockAggregator can be merged together to have a summary of loss and
+ * gradient of the corresponding joint dataset.
+ *
+ * NOTE: The feature values are expected to already have be scaled (multiplied by bcInverseStd,
+ * but NOT centered) before computation.
+ *
+ * @param bcCoefficients The coefficients corresponding to the features.
+ * @param fitIntercept Whether to fit an intercept term.
+ * @param fitWithMean Whether to center the data with mean before training, in a virtual way.
+ *                    If true, we MUST adjust the intercept of both initial coefficients and
+ *                    final solution in the caller.
+ */
+private[ml] class BinaryLogisticBlockAggregator(
+    bcInverseStd: Broadcast[Array[Double]],
+    bcScaledMean: Broadcast[Array[Double]],
+    fitIntercept: Boolean,
+    fitWithMean: Boolean)(bcCoefficients: Broadcast[Vector])
+  extends DifferentiableLossAggregator[InstanceBlock, BinaryLogisticBlockAggregator]
+  with Logging {
+
+  if (fitWithMean) {
+    require(fitIntercept, s"for training without intercept, should not center the vectors")
+    require(bcScaledMean != null && bcScaledMean.value.length == bcInverseStd.value.length,
+      "scaled means is required when center the vectors")
+  }
+
+  private val numFeatures = bcInverseStd.value.length
+  protected override val dim: Int = bcCoefficients.value.size
+
+  @transient private lazy val coefficientsArray = bcCoefficients.value match {
+    case DenseVector(values) => values
+    case _ => throw new IllegalArgumentException(s"coefficients only supports dense vector but " +
+      s"got type ${bcCoefficients.value.getClass}.)")
+  }
+
+  @transient private lazy val linear = if (fitIntercept) {
+    new DenseVector(coefficientsArray.take(numFeatures))
+  } else {
+    new DenseVector(coefficientsArray)
+  }
+
+  // pre-computed margin of an empty vector.
+  // with this variable as an offset, for a sparse vector, we only need to
+  // deal with non-zero values in prediction.
+  private val marginOffset = if (fitWithMean) {
+    coefficientsArray.last -
+      BLAS.javaBLAS.ddot(numFeatures, coefficientsArray, 1, bcScaledMean.value, 1)
+  } else {
+    Double.NaN
+  }
+
+  /**
+   * Add a new training instance block to this BinaryLogisticBlockAggregator, and update the loss
+   * and gradient of the objective function.
+   *
+   * @param block The instance block of data point to be added.
+   * @return This BinaryLogisticBlockAggregator object.
+   */
+  def add(block: InstanceBlock): this.type = {
+    require(block.matrix.isTransposed)
+    require(numFeatures == block.numFeatures, s"Dimensions mismatch when adding new " +
+      s"instance. Expecting $numFeatures but got ${block.numFeatures}.")
+    require(block.weightIter.forall(_ >= 0),
+      s"instance weights ${block.weightIter.mkString("[", ",", "]")} has to be >= 0.0")
+
+    if (block.weightIter.forall(_ == 0)) return this
+    val size = block.size
+
+    // vec/arr here represents margins
+    val vec = new DenseVector(Array.ofDim[Double](size))
+    val arr = vec.values
+    if (fitIntercept) {
+      val offset = if (fitWithMean) marginOffset else coefficientsArray.last
+      java.util.Arrays.fill(arr, offset)
+    }
+    BLAS.gemv(1.0, block.matrix, linear, 1.0, vec)
+
+    // in-place convert margins to multiplier
+    // then, vec/arr represents multiplier
+    var localLossSum = 0.0
+    var localWeightSum = 0.0
+    var multiplierSum = 0.0
+    var i = 0
+    while (i < size) {
+      val weight = block.getWeight(i)
+      localWeightSum += weight
+      if (weight > 0) {
+        val label = block.getLabel(i)
+        val margin = arr(i)
+        if (label > 0) {
+          // The following is equivalent to log(1 + exp(-margin)) but more numerically stable.
+          localLossSum += weight * Utils.log1pExp(-margin)
+        } else {
+          localLossSum += weight * (Utils.log1pExp(-margin) + margin)
+        }
+        val multiplier = weight * (1.0 / (1.0 + math.exp(-margin)) - label)
+        arr(i) = multiplier
+        multiplierSum += multiplier
+      } else { arr(i) = 0.0 }
+      i += 1
+    }
+    lossSum += localLossSum
+    weightSum += localWeightSum
+
+    // predictions are all correct, no gradient signal
+    if (arr.forall(_ == 0)) return this
+
+    // update the linear part of gradientSumArray
+    block.matrix match {
+      case dm: DenseMatrix =>
+        BLAS.nativeBLAS.dgemv("N", dm.numCols, dm.numRows, 1.0, dm.values, dm.numCols,
+          vec.values, 1, 1.0, gradientSumArray, 1)
+
+      case sm: SparseMatrix if fitIntercept =>
+        val linearGradSumVec = new DenseVector(Array.ofDim[Double](numFeatures))
+        BLAS.gemv(1.0, sm.transpose, vec, 0.0, linearGradSumVec)
+        BLAS.javaBLAS.daxpy(numFeatures, 1.0, linearGradSumVec.values, 1,
+          gradientSumArray, 1)
+
+      case sm: SparseMatrix if !fitIntercept =>
+        val gradSumVec = new DenseVector(gradientSumArray)
+        BLAS.gemv(1.0, sm.transpose, vec, 1.0, gradSumVec)
+
+      case m =>
+        throw new IllegalArgumentException(s"Unknown matrix type ${m.getClass}.")
+    }
+
+    if (fitWithMean) {
+      // above update of the linear part of gradientSumArray does NOT take the centering
+      // into account, here we need to adjust this part.
+      BLAS.javaBLAS.daxpy(numFeatures, -multiplierSum, bcScaledMean.value, 1,
+        gradientSumArray, 1)
+    }
+
+    if (fitIntercept) {
+      // update the intercept part of gradientSumArray
+      gradientSumArray(numFeatures) += multiplierSum
+    }
+
+    this
+  }
+}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HingeAggregator.scala b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HingeAggregator.scala
deleted file mode 100644
index b1990f7c60f64..0000000000000
--- a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HingeAggregator.scala
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.ml.optim.aggregator
-
-import org.apache.spark.broadcast.Broadcast
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
-import org.apache.spark.ml.linalg._
-
-/**
- * HingeAggregator computes the gradient and loss for Hinge loss function as used in
- * binary classification for instances in sparse or dense vector in an online fashion.
- *
- * Two HingeAggregators can be merged together to have a summary of loss and gradient of
- * the corresponding joint dataset.
- *
- * This class standardizes feature values during computation using bcFeaturesStd.
- *
- * @param bcCoefficients The coefficients corresponding to the features.
- * @param fitIntercept Whether to fit an intercept term.
- * @param bcFeaturesStd The standard deviation values of the features.
- */
-private[ml] class HingeAggregator(
-    bcFeaturesStd: Broadcast[Array[Double]],
-    fitIntercept: Boolean)(bcCoefficients: Broadcast[Vector])
-  extends DifferentiableLossAggregator[Instance, HingeAggregator] {
-
-  private val numFeatures = bcFeaturesStd.value.length
-  private val numFeaturesPlusIntercept = if (fitIntercept) numFeatures + 1 else numFeatures
-  @transient private lazy val coefficientsArray = bcCoefficients.value match {
-    case DenseVector(values) => values
-    case _ => throw new IllegalArgumentException(s"coefficients only supports dense vector" +
-      s" but got type ${bcCoefficients.value.getClass}.")
-  }
-  protected override val dim: Int = numFeaturesPlusIntercept
-
-  /**
-   * Add a new training instance to this HingeAggregator, and update the loss and gradient
-   * of the objective function.
-   *
-   * @param instance The instance of data point to be added.
-   * @return This HingeAggregator object.
-   */
-  def add(instance: Instance): this.type = {
-    instance match { case Instance(label, weight, features) =>
-      require(numFeatures == features.size, s"Dimensions mismatch when adding new instance." +
-        s" Expecting $numFeatures but got ${features.size}.")
-      require(weight >= 0.0, s"instance weight, $weight has to be >= 0.0")
-
-      if (weight == 0.0) return this
-      val localFeaturesStd = bcFeaturesStd.value
-      val localCoefficients = coefficientsArray
-      val localGradientSumArray = gradientSumArray
-
-      val dotProduct = {
-        var sum = 0.0
-        features.foreachNonZero { (index, value) =>
-          if (localFeaturesStd(index) != 0.0) {
-            sum += localCoefficients(index) * value / localFeaturesStd(index)
-          }
-        }
-        if (fitIntercept) sum += localCoefficients(numFeaturesPlusIntercept - 1)
-        sum
-      }
-      // Our loss function with {0, 1} labels is max(0, 1 - (2y - 1) (f_w(x)))
-      // Therefore the gradient is -(2y - 1)*x
-      val labelScaled = 2 * label - 1.0
-      val loss = if (1.0 > labelScaled * dotProduct) {
-        (1.0 - labelScaled * dotProduct) * weight
-      } else {
-        0.0
-      }
-
-      if (1.0 > labelScaled * dotProduct) {
-        val gradientScale = -labelScaled * weight
-        features.foreachNonZero { (index, value) =>
-          if (localFeaturesStd(index) != 0.0) {
-            localGradientSumArray(index) += value * gradientScale / localFeaturesStd(index)
-          }
-        }
-        if (fitIntercept) {
-          localGradientSumArray(localGradientSumArray.length - 1) += gradientScale
-        }
-      }
-
-      lossSum += loss
-      weightSum += weight
-      this
-    }
-  }
-}
-
-
-/**
- * BlockHingeAggregator computes the gradient and loss for Hinge loss function as used in
- * binary classification for blocks in sparse or dense matrix in an online fashion.
- *
- * Two BlockHingeAggregators can be merged together to have a summary of loss and gradient of
- * the corresponding joint dataset.
- *
- * NOTE: The feature values are expected to be standardized before computation.
- *
- * @param bcCoefficients The coefficients corresponding to the features.
- * @param fitIntercept Whether to fit an intercept term.
- */
-private[ml] class BlockHingeAggregator(
-    fitIntercept: Boolean)(bcCoefficients: Broadcast[Vector])
-  extends DifferentiableLossAggregator[InstanceBlock, BlockHingeAggregator] {
-
-  protected override val dim: Int = bcCoefficients.value.size
-  private val numFeatures = if (fitIntercept) dim - 1 else dim
-
-  @transient private lazy val coefficientsArray = bcCoefficients.value match {
-    case DenseVector(values) => values
-    case _ => throw new IllegalArgumentException(s"coefficients only supports dense vector" +
-      s" but got type ${bcCoefficients.value.getClass}.")
-  }
-
-  @transient private lazy val linear = {
-    val linear = if (fitIntercept) coefficientsArray.take(numFeatures) else coefficientsArray
-    Vectors.dense(linear)
-  }
-
-  /**
-   * Add a new training instance block to this BlockHingeAggregator, and update the loss and
-   * gradient of the objective function.
-   *
-   * @param block The InstanceBlock to be added.
-   * @return This BlockHingeAggregator object.
-   */
-  def add(block: InstanceBlock): this.type = {
-    require(block.matrix.isTransposed)
-    require(numFeatures == block.numFeatures, s"Dimensions mismatch when adding new " +
-      s"instance. Expecting $numFeatures but got ${block.numFeatures}.")
-    require(block.weightIter.forall(_ >= 0),
-      s"instance weights ${block.weightIter.mkString("[", ",", "]")} has to be >= 0.0")
-
-    if (block.weightIter.forall(_ == 0)) return this
-    val size = block.size
-
-    // vec here represents dotProducts
-    val vec = if (fitIntercept) {
-      Vectors.dense(Array.fill(size)(coefficientsArray.last)).toDense
-    } else {
-      Vectors.zeros(size).toDense
-    }
-    BLAS.gemv(1.0, block.matrix, linear, 1.0, vec)
-
-    // in-place convert dotProducts to gradient scales
-    // then, vec represents gradient scales
-    var i = 0
-    while (i < size) {
-      val weight = block.getWeight(i)
-      if (weight > 0) {
-        weightSum += weight
-        // Our loss function with {0, 1} labels is max(0, 1 - (2y - 1) (f_w(x)))
-        // Therefore the gradient is -(2y - 1)*x
-        val label = block.getLabel(i)
-        val labelScaled = label + label - 1.0
-        val loss = (1.0 - labelScaled * vec(i)) * weight
-        if (loss > 0) {
-          lossSum += loss
-          val gradScale = -labelScaled * weight
-          vec.values(i) = gradScale
-        } else { vec.values(i) = 0.0 }
-      } else { vec.values(i) = 0.0 }
-      i += 1
-    }
-
-    // predictions are all correct, no gradient signal
-    if (vec.values.forall(_ == 0)) return this
-
-    block.matrix match {
-      case dm: DenseMatrix =>
-        BLAS.nativeBLAS.dgemv("N", dm.numCols, dm.numRows, 1.0, dm.values, dm.numCols,
-          vec.values, 1, 1.0, gradientSumArray, 1)
-
-      case sm: SparseMatrix if fitIntercept =>
-        val linearGradSumVec = Vectors.zeros(numFeatures).toDense
-        BLAS.gemv(1.0, sm.transpose, vec, 0.0, linearGradSumVec)
-        BLAS.getBLAS(numFeatures).daxpy(numFeatures, 1.0, linearGradSumVec.values, 1,
-          gradientSumArray, 1)
-
-      case sm: SparseMatrix if !fitIntercept =>
-        val gradSumVec = new DenseVector(gradientSumArray)
-        BLAS.gemv(1.0, sm.transpose, vec, 1.0, gradSumVec)
-    }
-
-    if (fitIntercept) gradientSumArray(numFeatures) += vec.values.sum
-
-    this
-  }
-}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HingeBlockAggregator.scala b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HingeBlockAggregator.scala
new file mode 100644
index 0000000000000..f99c531c96b9d
--- /dev/null
+++ b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HingeBlockAggregator.scala
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.ml.optim.aggregator
+
+import org.apache.spark.broadcast.Broadcast
+import org.apache.spark.internal.Logging
+import org.apache.spark.ml.feature.InstanceBlock
+import org.apache.spark.ml.linalg._
+
+
+/**
+ * HingeBlockAggregator computes the gradient and loss for Huber loss function
+ * as used in linear regression for blocks in sparse or dense matrix in an online fashion.
+ *
+ * Two BlockHuberAggregators can be merged together to have a summary of loss and gradient
+ * of the corresponding joint dataset.
+ *
+ * NOTE: The feature values are expected to already have be scaled (multiplied by bcInverseStd,
+ * but NOT centered) before computation.
+ *
+ * @param bcCoefficients The coefficients corresponding to the features.
+ * @param fitIntercept Whether to fit an intercept term. When true, will perform data centering
+ *                     in a virtual way. Then we MUST adjust the intercept of both initial
+ *                     coefficients and final solution in the caller.
+ */
+private[ml] class HingeBlockAggregator(
+    bcInverseStd: Broadcast[Array[Double]],
+    bcScaledMean: Broadcast[Array[Double]],
+    fitIntercept: Boolean)(bcCoefficients: Broadcast[Vector])
+  extends DifferentiableLossAggregator[InstanceBlock, HingeBlockAggregator]
+    with Logging {
+
+  if (fitIntercept) {
+    require(bcScaledMean != null && bcScaledMean.value.length == bcInverseStd.value.length,
+      "scaled means is required when center the vectors")
+  }
+
+  private val numFeatures = bcInverseStd.value.length
+  protected override val dim: Int = bcCoefficients.value.size
+
+  @transient private lazy val coefficientsArray = bcCoefficients.value match {
+    case DenseVector(values) => values
+    case _ => throw new IllegalArgumentException(s"coefficients only supports dense vector but " +
+      s"got type ${bcCoefficients.value.getClass}.)")
+  }
+
+  @transient private lazy val linear = if (fitIntercept) {
+    new DenseVector(coefficientsArray.take(numFeatures))
+  } else {
+    new DenseVector(coefficientsArray)
+  }
+
+  // pre-computed margin of an empty vector.
+  // with this variable as an offset, for a sparse vector, we only need to
+  // deal with non-zero values in prediction.
+  private val marginOffset = if (fitIntercept) {
+    coefficientsArray.last -
+      BLAS.javaBLAS.ddot(numFeatures, coefficientsArray, 1, bcScaledMean.value, 1)
+  } else {
+    Double.NaN
+  }
+
+  /**
+   * Add a new training instance block to this HingeBlockAggregator, and update the loss
+   * and gradient of the objective function.
+   *
+   * @param block The instance block of data point to be added.
+   * @return This HingeBlockAggregator object.
+   */
+  def add(block: InstanceBlock): this.type = {
+    require(block.matrix.isTransposed)
+    require(numFeatures == block.numFeatures, s"Dimensions mismatch when adding new " +
+      s"instance. Expecting $numFeatures but got ${block.numFeatures}.")
+    require(block.weightIter.forall(_ >= 0),
+      s"instance weights ${block.weightIter.mkString("[", ",", "]")} has to be >= 0.0")
+
+    if (block.weightIter.forall(_ == 0)) return this
+    val size = block.size
+
+    // vec/arr here represents margins
+    val vec = new DenseVector(Array.ofDim[Double](size))
+    val arr = vec.values
+    if (fitIntercept) java.util.Arrays.fill(arr, marginOffset)
+    BLAS.gemv(1.0, block.matrix, linear, 1.0, vec)
+
+    // in-place convert margins to multiplier
+    // then, vec/arr represents multiplier
+    var localLossSum = 0.0
+    var localWeightSum = 0.0
+    var multiplierSum = 0.0
+    var i = 0
+    while (i < size) {
+      val weight = block.getWeight(i)
+      localWeightSum += weight
+      if (weight > 0) {
+        // Our loss function with {0, 1} labels is max(0, 1 - (2y - 1) (f_w(x)))
+        // Therefore the gradient is -(2y - 1)*x
+        val label = block.getLabel(i)
+        val labelScaled = label + label - 1.0
+        val loss = (1.0 - labelScaled * arr(i)) * weight
+        if (loss > 0) {
+          localLossSum += loss
+          val multiplier = -labelScaled * weight
+          arr(i) = multiplier
+          multiplierSum += multiplier
+        } else { arr(i) = 0.0 }
+      } else { arr(i) = 0.0 }
+      i += 1
+    }
+    lossSum += localLossSum
+    weightSum += localWeightSum
+
+    // predictions are all correct, no gradient signal
+    if (arr.forall(_ == 0)) return this
+
+    // update the linear part of gradientSumArray
+    block.matrix match {
+      case dm: DenseMatrix =>
+        BLAS.nativeBLAS.dgemv("N", dm.numCols, dm.numRows, 1.0, dm.values, dm.numCols,
+          vec.values, 1, 1.0, gradientSumArray, 1)
+
+      case sm: SparseMatrix if fitIntercept =>
+        val linearGradSumVec = new DenseVector(Array.ofDim[Double](numFeatures))
+        BLAS.gemv(1.0, sm.transpose, vec, 0.0, linearGradSumVec)
+        BLAS.javaBLAS.daxpy(numFeatures, 1.0, linearGradSumVec.values, 1,
+          gradientSumArray, 1)
+
+      case sm: SparseMatrix if !fitIntercept =>
+        val gradSumVec = new DenseVector(gradientSumArray)
+        BLAS.gemv(1.0, sm.transpose, vec, 1.0, gradSumVec)
+
+      case m =>
+        throw new IllegalArgumentException(s"Unknown matrix type ${m.getClass}.")
+    }
+
+    if (fitIntercept) {
+      // above update of the linear part of gradientSumArray does NOT take the centering
+      // into account, here we need to adjust this part.
+      BLAS.javaBLAS.daxpy(numFeatures, -multiplierSum, bcScaledMean.value, 1,
+        gradientSumArray, 1)
+
+      // update the intercept part of gradientSumArray
+      gradientSumArray(numFeatures) += multiplierSum
+    }
+
+    this
+  }
+}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HuberAggregator.scala b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HuberAggregator.scala
index 59ecc038e5569..35582dbc990e6 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HuberAggregator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HuberAggregator.scala
@@ -167,7 +167,6 @@ private[ml] class BlockHuberAggregator(
 
   protected override val dim: Int = bcParameters.value.size
   private val numFeatures = if (fitIntercept) dim - 2 else dim - 1
-  private val sigma = bcParameters.value(dim - 1)
   private val intercept = if (fitIntercept) bcParameters.value(dim - 2) else 0.0
   // make transient so we do not serialize between aggregation stages
   @transient private lazy val linear = Vectors.dense(bcParameters.value.toArray.take(numFeatures))
@@ -187,7 +186,9 @@ private[ml] class BlockHuberAggregator(
       s"instance weights ${block.weightIter.mkString("[", ",", "]")} has to be >= 0.0")
 
     if (block.weightIter.forall(_ == 0)) return this
+
     val size = block.size
+    val sigma = bcParameters.value(dim - 1)
 
     // vec here represents margins or dotProducts
     val vec = if (fitIntercept) {
@@ -200,23 +201,23 @@ private[ml] class BlockHuberAggregator(
     // in-place convert margins to multipliers
     // then, vec represents multipliers
     var sigmaGradSum = 0.0
+    var localLossSum = 0.0
     var i = 0
     while (i < size) {
       val weight = block.getWeight(i)
       if (weight > 0) {
-        weightSum += weight
         val label = block.getLabel(i)
         val margin = vec(i)
         val linearLoss = label - margin
 
         if (math.abs(linearLoss) <= sigma * epsilon) {
-          lossSum += 0.5 * weight * (sigma + math.pow(linearLoss, 2.0) / sigma)
+          localLossSum += 0.5 * weight * (sigma + math.pow(linearLoss, 2.0) / sigma)
           val linearLossDivSigma = linearLoss / sigma
           val multiplier = -1.0 * weight * linearLossDivSigma
           vec.values(i) = multiplier
           sigmaGradSum += 0.5 * weight * (1.0 - math.pow(linearLossDivSigma, 2.0))
         } else {
-          lossSum += 0.5 * weight *
+          localLossSum += 0.5 * weight *
             (sigma + 2.0 * epsilon * math.abs(linearLoss) - sigma * epsilon * epsilon)
           val sign = if (linearLoss >= 0) -1.0 else 1.0
           val multiplier = weight * sign * epsilon
@@ -226,6 +227,8 @@ private[ml] class BlockHuberAggregator(
       } else { vec.values(i) = 0.0 }
       i += 1
     }
+    lossSum += localLossSum
+    weightSum += block.weightIter.sum
 
     block.matrix match {
       case dm: DenseMatrix =>
diff --git a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/LeastSquaresAggregator.scala b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/LeastSquaresAggregator.scala
index fa3bda00d802d..d5e1ea980840b 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/LeastSquaresAggregator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/LeastSquaresAggregator.scala
@@ -267,9 +267,6 @@ private[ml] class BlockLeastSquaresAggregator(
     val offset = if (fitIntercept) labelMean / labelStd - sum else 0.0
     (Vectors.dense(coefficientsArray), offset)
   }
-  // do not use tuple assignment above because it will circumvent the @transient tag
-  @transient private lazy val effectiveCoefficientsVec = effectiveCoefAndOffset._1
-  @transient private lazy val offset = effectiveCoefAndOffset._2
 
   /**
    * Add a new training instance block to this BlockLeastSquaresAggregator, and update the loss
@@ -286,7 +283,9 @@ private[ml] class BlockLeastSquaresAggregator(
       s"instance weights ${block.weightIter.mkString("[", ",", "]")} has to be >= 0.0")
 
     if (block.weightIter.forall(_ == 0)) return this
+
     val size = block.size
+    val (effectiveCoefficientsVec, offset) = effectiveCoefAndOffset
 
     // vec here represents diffs
     val vec = new DenseVector(Array.tabulate(size)(i => offset - block.getLabel(i) / labelStd))
@@ -294,16 +293,18 @@ private[ml] class BlockLeastSquaresAggregator(
 
     // in-place convert diffs to multipliers
     // then, vec represents multipliers
+    var localLossSum = 0.0
     var i = 0
     while (i < size) {
       val weight = block.getWeight(i)
       val diff = vec(i)
-      lossSum += weight * diff * diff / 2
-      weightSum += weight
+      localLossSum += weight * diff * diff / 2
       val multiplier = weight * diff
       vec.values(i) = multiplier
       i += 1
     }
+    lossSum += localLossSum
+    weightSum += block.weightIter.sum
 
     val gradSumVec = new DenseVector(gradientSumArray)
     BLAS.gemv(1.0, block.matrix.transpose, vec, 1.0, gradSumVec)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/LogisticAggregator.scala b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/LogisticAggregator.scala
deleted file mode 100644
index a331122776b5c..0000000000000
--- a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/LogisticAggregator.scala
+++ /dev/null
@@ -1,600 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.ml.optim.aggregator
-
-import org.apache.spark.broadcast.Broadcast
-import org.apache.spark.internal.Logging
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
-import org.apache.spark.ml.impl.Utils
-import org.apache.spark.ml.linalg._
-
-/**
- * LogisticAggregator computes the gradient and loss for binary or multinomial logistic (softmax)
- * loss function, as used in classification for instances in sparse or dense vector in an online
- * fashion.
- *
- * Two LogisticAggregators can be merged together to have a summary of loss and gradient of
- * the corresponding joint dataset.
- *
- * For improving the convergence rate during the optimization process and also to prevent against
- * features with very large variances exerting an overly large influence during model training,
- * packages like R's GLMNET perform the scaling to unit variance and remove the mean in order to
- * reduce the condition number. The model is then trained in this scaled space, but returns the
- * coefficients in the original scale. See page 9 in
- * http://cran.r-project.org/web/packages/glmnet/glmnet.pdf
- *
- * However, we don't want to apply the [[org.apache.spark.ml.feature.StandardScaler]] on the
- * training dataset, and then cache the standardized dataset since it will create a lot of overhead.
- * As a result, we perform the scaling implicitly when we compute the objective function (though
- * we do not subtract the mean).
- *
- * Note that there is a difference between multinomial (softmax) and binary loss. The binary case
- * uses one outcome class as a "pivot" and regresses the other class against the pivot. In the
- * multinomial case, the softmax loss function is used to model each class probability
- * independently. Using softmax loss produces `K` sets of coefficients, while using a pivot class
- * produces `K - 1` sets of coefficients (a single coefficient vector in the binary case). In the
- * binary case, we can say that the coefficients are shared between the positive and negative
- * classes. When regularization is applied, multinomial (softmax) loss will produce a result
- * different from binary loss since the positive and negative don't share the coefficients while the
- * binary regression shares the coefficients between positive and negative.
- *
- * The following is a mathematical derivation for the multinomial (softmax) loss.
- *
- * The probability of the multinomial outcome $y$ taking on any of the K possible outcomes is:
- *
- * <blockquote>
- *    $$
- *    P(y_i=0|\vec{x}_i, \beta) = \frac{e^{\vec{x}_i^T \vec{\beta}_0}}{\sum_{k=0}^{K-1}
- *       e^{\vec{x}_i^T \vec{\beta}_k}} \\
- *    P(y_i=1|\vec{x}_i, \beta) = \frac{e^{\vec{x}_i^T \vec{\beta}_1}}{\sum_{k=0}^{K-1}
- *       e^{\vec{x}_i^T \vec{\beta}_k}}\\
- *    P(y_i=K-1|\vec{x}_i, \beta) = \frac{e^{\vec{x}_i^T \vec{\beta}_{K-1}}\,}{\sum_{k=0}^{K-1}
- *       e^{\vec{x}_i^T \vec{\beta}_k}}
- *    $$
- * </blockquote>
- *
- * The model coefficients $\beta = (\beta_0, \beta_1, \beta_2, ..., \beta_{K-1})$ become a matrix
- * which has dimension of $K \times (N+1)$ if the intercepts are added. If the intercepts are not
- * added, the dimension will be $K \times N$.
- *
- * Note that the coefficients in the model above lack identifiability. That is, any constant scalar
- * can be added to all of the coefficients and the probabilities remain the same.
- *
- * <blockquote>
- *    $$
- *    \begin{align}
- *    \frac{e^{\vec{x}_i^T \left(\vec{\beta}_0 + \vec{c}\right)}}{\sum_{k=0}^{K-1}
- *       e^{\vec{x}_i^T \left(\vec{\beta}_k + \vec{c}\right)}}
- *    = \frac{e^{\vec{x}_i^T \vec{\beta}_0}e^{\vec{x}_i^T \vec{c}}\,}{e^{\vec{x}_i^T \vec{c}}
- *       \sum_{k=0}^{K-1} e^{\vec{x}_i^T \vec{\beta}_k}}
- *    = \frac{e^{\vec{x}_i^T \vec{\beta}_0}}{\sum_{k=0}^{K-1} e^{\vec{x}_i^T \vec{\beta}_k}}
- *    \end{align}
- *    $$
- * </blockquote>
- *
- * However, when regularization is added to the loss function, the coefficients are indeed
- * identifiable because there is only one set of coefficients which minimizes the regularization
- * term. When no regularization is applied, we choose the coefficients with the minimum L2
- * penalty for consistency and reproducibility. For further discussion see:
- *
- * Friedman, et al. "Regularization Paths for Generalized Linear Models via Coordinate Descent"
- *
- * The loss of objective function for a single instance of data (we do not include the
- * regularization term here for simplicity) can be written as
- *
- * <blockquote>
- *    $$
- *    \begin{align}
- *    \ell\left(\beta, x_i\right) &= -log{P\left(y_i \middle| \vec{x}_i, \beta\right)} \\
- *    &= log\left(\sum_{k=0}^{K-1}e^{\vec{x}_i^T \vec{\beta}_k}\right) - \vec{x}_i^T \vec{\beta}_y\\
- *    &= log\left(\sum_{k=0}^{K-1} e^{margins_k}\right) - margins_y
- *    \end{align}
- *    $$
- * </blockquote>
- *
- * where ${margins}_k = \vec{x}_i^T \vec{\beta}_k$.
- *
- * For optimization, we have to calculate the first derivative of the loss function, and a simple
- * calculation shows that
- *
- * <blockquote>
- *    $$
- *    \begin{align}
- *    \frac{\partial \ell(\beta, \vec{x}_i, w_i)}{\partial \beta_{j, k}}
- *    &= x_{i,j} \cdot w_i \cdot \left(\frac{e^{\vec{x}_i \cdot \vec{\beta}_k}}{\sum_{k'=0}^{K-1}
- *      e^{\vec{x}_i \cdot \vec{\beta}_{k'}}\,} - I_{y=k}\right) \\
- *    &= x_{i, j} \cdot w_i \cdot multiplier_k
- *    \end{align}
- *    $$
- * </blockquote>
- *
- * where $w_i$ is the sample weight, $I_{y=k}$ is an indicator function
- *
- *  <blockquote>
- *    $$
- *    I_{y=k} = \begin{cases}
- *          1 & y = k \\
- *          0 & else
- *       \end{cases}
- *    $$
- * </blockquote>
- *
- * and
- *
- * <blockquote>
- *    $$
- *    multiplier_k = \left(\frac{e^{\vec{x}_i \cdot \vec{\beta}_k}}{\sum_{k=0}^{K-1}
- *       e^{\vec{x}_i \cdot \vec{\beta}_k}} - I_{y=k}\right)
- *    $$
- * </blockquote>
- *
- * If any of margins is larger than 709.78, the numerical computation of multiplier and loss
- * function will suffer from arithmetic overflow. This issue occurs when there are outliers in
- * data which are far away from the hyperplane, and this will cause the failing of training once
- * infinity is introduced. Note that this is only a concern when max(margins) &gt; 0.
- *
- * Fortunately, when max(margins) = maxMargin &gt; 0, the loss function and the multiplier can
- * easily be rewritten into the following equivalent numerically stable formula.
- *
- * <blockquote>
- *    $$
- *    \ell\left(\beta, x\right) = log\left(\sum_{k=0}^{K-1} e^{margins_k - maxMargin}\right) -
- *       margins_{y} + maxMargin
- *    $$
- * </blockquote>
- *
- * Note that each term, $(margins_k - maxMargin)$ in the exponential is no greater than zero; as a
- * result, overflow will not happen with this formula.
- *
- * For $multiplier$, a similar trick can be applied as the following,
- *
- * <blockquote>
- *    $$
- *    multiplier_k = \left(\frac{e^{\vec{x}_i \cdot \vec{\beta}_k - maxMargin}}{\sum_{k'=0}^{K-1}
- *       e^{\vec{x}_i \cdot \vec{\beta}_{k'} - maxMargin}} - I_{y=k}\right)
- *    $$
- * </blockquote>
- *
- *
- * @param bcCoefficients The broadcast coefficients corresponding to the features.
- * @param bcFeaturesStd The broadcast standard deviation values of the features.
- * @param numClasses the number of possible outcomes for k classes classification problem in
- *                   Multinomial Logistic Regression.
- * @param fitIntercept Whether to fit an intercept term.
- * @param multinomial Whether to use multinomial (softmax) or binary loss
- * @note In order to avoid unnecessary computation during calculation of the gradient updates
- * we lay out the coefficients in column major order during training. This allows us to
- * perform feature standardization once, while still retaining sequential memory access
- * for speed. We convert back to row major order when we create the model,
- * since this form is optimal for the matrix operations used for prediction.
- */
-private[ml] class LogisticAggregator(
-    bcFeaturesStd: Broadcast[Array[Double]],
-    numClasses: Int,
-    fitIntercept: Boolean,
-    multinomial: Boolean)(bcCoefficients: Broadcast[Vector])
-  extends DifferentiableLossAggregator[Instance, LogisticAggregator] with Logging {
-
-  private val numFeatures = bcFeaturesStd.value.length
-  private val numFeaturesPlusIntercept = if (fitIntercept) numFeatures + 1 else numFeatures
-  private val coefficientSize = bcCoefficients.value.size
-  protected override val dim: Int = coefficientSize
-  if (multinomial) {
-    require(numClasses ==  coefficientSize / numFeaturesPlusIntercept, s"The number of " +
-      s"coefficients should be ${numClasses * numFeaturesPlusIntercept} but was $coefficientSize")
-  } else {
-    require(coefficientSize == numFeaturesPlusIntercept, s"Expected $numFeaturesPlusIntercept " +
-      s"coefficients but got $coefficientSize")
-    require(numClasses == 1 || numClasses == 2, s"Binary logistic aggregator requires numClasses " +
-      s"in {1, 2} but found $numClasses.")
-  }
-
-  @transient private lazy val coefficientsArray = bcCoefficients.value match {
-    case DenseVector(values) => values
-    case _ => throw new IllegalArgumentException(s"coefficients only supports dense vector but " +
-      s"got type ${bcCoefficients.value.getClass}.)")
-  }
-
-  if (multinomial && numClasses <= 2) {
-    logInfo(s"Multinomial logistic regression for binary classification yields separate " +
-      s"coefficients for positive and negative classes. When no regularization is applied, the" +
-      s"result will be effectively the same as binary logistic regression. When regularization" +
-      s"is applied, multinomial loss will produce a result different from binary loss.")
-  }
-
-  /** Update gradient and loss using binary loss function. */
-  private def binaryUpdateInPlace(features: Vector, weight: Double, label: Double): Unit = {
-
-    val localFeaturesStd = bcFeaturesStd.value
-    val localCoefficients = coefficientsArray
-    val localGradientArray = gradientSumArray
-    val margin = - {
-      var sum = 0.0
-      features.foreachNonZero { (index, value) =>
-        if (localFeaturesStd(index) != 0.0) {
-          sum += localCoefficients(index) * value / localFeaturesStd(index)
-        }
-      }
-      if (fitIntercept) sum += localCoefficients(numFeaturesPlusIntercept - 1)
-      sum
-    }
-
-    val multiplier = weight * (1.0 / (1.0 + math.exp(margin)) - label)
-
-    features.foreachNonZero { (index, value) =>
-      if (localFeaturesStd(index) != 0.0) {
-        localGradientArray(index) += multiplier * value / localFeaturesStd(index)
-      }
-    }
-
-    if (fitIntercept) {
-      localGradientArray(numFeaturesPlusIntercept - 1) += multiplier
-    }
-
-    if (label > 0) {
-      // The following is equivalent to log(1 + exp(margin)) but more numerically stable.
-      lossSum += weight * Utils.log1pExp(margin)
-    } else {
-      lossSum += weight * (Utils.log1pExp(margin) - margin)
-    }
-  }
-
-  /** Update gradient and loss using multinomial (softmax) loss function. */
-  private def multinomialUpdateInPlace(features: Vector, weight: Double, label: Double): Unit = {
-    // TODO: use level 2 BLAS operations
-    /*
-      Note: this can still be used when numClasses = 2 for binary
-      logistic regression without pivoting.
-     */
-    val localFeaturesStd = bcFeaturesStd.value
-    val localCoefficients = coefficientsArray
-    val localGradientArray = gradientSumArray
-
-    // marginOfLabel is margins(label) in the formula
-    var marginOfLabel = 0.0
-    var maxMargin = Double.NegativeInfinity
-
-    val margins = new Array[Double](numClasses)
-    features.foreachNonZero { (index, value) =>
-      if (localFeaturesStd(index) != 0.0) {
-        val stdValue = value / localFeaturesStd(index)
-        var j = 0
-        while (j < numClasses) {
-          margins(j) += localCoefficients(index * numClasses + j) * stdValue
-          j += 1
-        }
-      }
-    }
-    var i = 0
-    while (i < numClasses) {
-      if (fitIntercept) {
-        margins(i) += localCoefficients(numClasses * numFeatures + i)
-      }
-      if (i == label.toInt) marginOfLabel = margins(i)
-      if (margins(i) > maxMargin) {
-        maxMargin = margins(i)
-      }
-      i += 1
-    }
-
-    /**
-     * When maxMargin is greater than 0, the original formula could cause overflow.
-     * We address this by subtracting maxMargin from all the margins, so it's guaranteed
-     * that all of the new margins will be smaller than zero to prevent arithmetic overflow.
-     */
-    val multipliers = new Array[Double](numClasses)
-    val sum = {
-      var temp = 0.0
-      var i = 0
-      while (i < numClasses) {
-        if (maxMargin > 0) margins(i) -= maxMargin
-        val exp = math.exp(margins(i))
-        temp += exp
-        multipliers(i) = exp
-        i += 1
-      }
-      temp
-    }
-
-    margins.indices.foreach { i =>
-      multipliers(i) = multipliers(i) / sum - (if (label == i) 1.0 else 0.0)
-    }
-    features.foreachNonZero { (index, value) =>
-      if (localFeaturesStd(index) != 0.0) {
-        val stdValue = value / localFeaturesStd(index)
-        var j = 0
-        while (j < numClasses) {
-          localGradientArray(index * numClasses + j) += weight * multipliers(j) * stdValue
-          j += 1
-        }
-      }
-    }
-    if (fitIntercept) {
-      var i = 0
-      while (i < numClasses) {
-        localGradientArray(numFeatures * numClasses + i) += weight * multipliers(i)
-        i += 1
-      }
-    }
-
-    val loss = if (maxMargin > 0) {
-      math.log(sum) - marginOfLabel + maxMargin
-    } else {
-      math.log(sum) - marginOfLabel
-    }
-    lossSum += weight * loss
-  }
-
-  /**
-   * Add a new training instance to this LogisticAggregator, and update the loss and gradient
-   * of the objective function.
-   *
-   * @param instance The instance of data point to be added.
-   * @return This LogisticAggregator object.
-   */
-  def add(instance: Instance): this.type = {
-    instance match { case Instance(label, weight, features) =>
-      require(numFeatures == features.size, s"Dimensions mismatch when adding new instance." +
-        s" Expecting $numFeatures but got ${features.size}.")
-      require(weight >= 0.0, s"instance weight, $weight has to be >= 0.0")
-
-      if (weight == 0.0) return this
-
-      if (multinomial) {
-        multinomialUpdateInPlace(features, weight, label)
-      } else {
-        binaryUpdateInPlace(features, weight, label)
-      }
-      weightSum += weight
-      this
-    }
-  }
-}
-
-
-/**
- * BlockLogisticAggregator computes the gradient and loss used in Logistic classification
- * for blocks in sparse or dense matrix in an online fashion.
- *
- * Two BlockLogisticAggregators can be merged together to have a summary of loss and gradient of
- * the corresponding joint dataset.
- *
- * NOTE: The feature values are expected to be standardized before computation.
- *
- * @param bcCoefficients The coefficients corresponding to the features.
- * @param fitIntercept Whether to fit an intercept term.
- */
-private[ml] class BlockLogisticAggregator(
-    numFeatures: Int,
-    numClasses: Int,
-    fitIntercept: Boolean,
-    multinomial: Boolean)(bcCoefficients: Broadcast[Vector])
-  extends DifferentiableLossAggregator[InstanceBlock, BlockLogisticAggregator] with Logging {
-
-  if (multinomial && numClasses <= 2) {
-    logInfo(s"Multinomial logistic regression for binary classification yields separate " +
-      s"coefficients for positive and negative classes. When no regularization is applied, the" +
-      s"result will be effectively the same as binary logistic regression. When regularization" +
-      s"is applied, multinomial loss will produce a result different from binary loss.")
-  }
-
-  private val numFeaturesPlusIntercept = if (fitIntercept) numFeatures + 1 else numFeatures
-  private val coefficientSize = bcCoefficients.value.size
-  protected override val dim: Int = coefficientSize
-
-  if (multinomial) {
-    require(numClasses ==  coefficientSize / numFeaturesPlusIntercept, s"The number of " +
-      s"coefficients should be ${numClasses * numFeaturesPlusIntercept} but was $coefficientSize")
-  } else {
-    require(coefficientSize == numFeaturesPlusIntercept, s"Expected $numFeaturesPlusIntercept " +
-      s"coefficients but got $coefficientSize")
-    require(numClasses == 1 || numClasses == 2, s"Binary logistic aggregator requires numClasses " +
-      s"in {1, 2} but found $numClasses.")
-  }
-
-  @transient private lazy val coefficientsArray = bcCoefficients.value match {
-    case DenseVector(values) => values
-    case _ => throw new IllegalArgumentException(s"coefficients only supports dense vector but " +
-      s"got type ${bcCoefficients.value.getClass}.)")
-  }
-
-  @transient private lazy val binaryLinear = (multinomial, fitIntercept) match {
-    case (false, true) => Vectors.dense(coefficientsArray.take(numFeatures))
-    case (false, false) => Vectors.dense(coefficientsArray)
-    case _ => null
-  }
-
-  @transient private lazy val multinomialLinear = (multinomial, fitIntercept) match {
-    case (true, true) =>
-      Matrices.dense(numClasses, numFeatures,
-        coefficientsArray.take(numClasses * numFeatures)).toDense
-    case (true, false) =>
-      Matrices.dense(numClasses, numFeatures, coefficientsArray).toDense
-    case _ => null
-  }
-
-  /**
-   * Add a new training instance block to this BlockLogisticAggregator, and update the loss and
-   * gradient of the objective function.
-   *
-   * @param block The instance block of data point to be added.
-   * @return This BlockLogisticAggregator object.
-   */
-  def add(block: InstanceBlock): this.type = {
-    require(block.matrix.isTransposed)
-    require(numFeatures == block.numFeatures, s"Dimensions mismatch when adding new " +
-      s"instance. Expecting $numFeatures but got ${block.numFeatures}.")
-    require(block.weightIter.forall(_ >= 0),
-      s"instance weights ${block.weightIter.mkString("[", ",", "]")} has to be >= 0.0")
-
-    if (block.weightIter.forall(_ == 0)) return this
-
-    if (multinomial) {
-      multinomialUpdateInPlace(block)
-    } else {
-      binaryUpdateInPlace(block)
-    }
-
-    this
-  }
-
-  /** Update gradient and loss using binary loss function. */
-  private def binaryUpdateInPlace(block: InstanceBlock): Unit = {
-    val size = block.size
-
-    // vec here represents margins or negative dotProducts
-    val vec = if (fitIntercept) {
-      Vectors.dense(Array.fill(size)(coefficientsArray.last)).toDense
-    } else {
-      Vectors.zeros(size).toDense
-    }
-    BLAS.gemv(-1.0, block.matrix, binaryLinear, -1.0, vec)
-
-    // in-place convert margins to multiplier
-    // then, vec represents multiplier
-    var i = 0
-    while (i < size) {
-      val weight = block.getWeight(i)
-      if (weight > 0) {
-        weightSum += weight
-        val label = block.getLabel(i)
-        val margin = vec(i)
-        if (label > 0) {
-          // The following is equivalent to log(1 + exp(margin)) but more numerically stable.
-          lossSum += weight * Utils.log1pExp(margin)
-        } else {
-          lossSum += weight * (Utils.log1pExp(margin) - margin)
-        }
-        val multiplier = weight * (1.0 / (1.0 + math.exp(margin)) - label)
-        vec.values(i) = multiplier
-      } else { vec.values(i) = 0.0 }
-      i += 1
-    }
-
-    // predictions are all correct, no gradient signal
-    if (vec.values.forall(_ == 0)) return
-
-    block.matrix match {
-      case dm: DenseMatrix =>
-        BLAS.nativeBLAS.dgemv("N", dm.numCols, dm.numRows, 1.0, dm.values, dm.numCols,
-          vec.values, 1, 1.0, gradientSumArray, 1)
-
-      case sm: SparseMatrix if fitIntercept =>
-        val linearGradSumVec = Vectors.zeros(numFeatures).toDense
-        BLAS.gemv(1.0, sm.transpose, vec, 0.0, linearGradSumVec)
-        BLAS.getBLAS(numFeatures).daxpy(numFeatures, 1.0, linearGradSumVec.values, 1,
-          gradientSumArray, 1)
-
-      case sm: SparseMatrix if !fitIntercept =>
-        val gradSumVec = new DenseVector(gradientSumArray)
-        BLAS.gemv(1.0, sm.transpose, vec, 1.0, gradSumVec)
-    }
-
-    if (fitIntercept) gradientSumArray(numFeatures) += vec.values.sum
-  }
-
-  /** Update gradient and loss using multinomial (softmax) loss function. */
-  private def multinomialUpdateInPlace(block: InstanceBlock): Unit = {
-    val size = block.size
-
-    // mat here represents margins, shape: S X C
-    val mat = DenseMatrix.zeros(size, numClasses)
-    if (fitIntercept) {
-      val offset = numClasses * numFeatures
-      var j = 0
-      while (j < numClasses) {
-        val intercept = coefficientsArray(offset + j)
-        var i = 0
-        while (i < size) { mat.update(i, j, intercept); i += 1 }
-        j += 1
-      }
-    }
-    BLAS.gemm(1.0, block.matrix, multinomialLinear.transpose, 1.0, mat)
-
-    // in-place convert margins to multipliers
-    // then, mat represents multipliers
-    var i = 0
-    val tmp = Array.ofDim[Double](numClasses)
-    val interceptGradSumArr = if (fitIntercept) Array.ofDim[Double](numClasses) else null
-    while (i < size) {
-      val weight = block.getWeight(i)
-      if (weight > 0) {
-        weightSum += weight
-        val label = block.getLabel(i)
-
-        var maxMargin = Double.NegativeInfinity
-        var j = 0
-        while (j < numClasses) {
-          tmp(j) = mat(i, j)
-          maxMargin = math.max(maxMargin, tmp(j))
-          j += 1
-        }
-
-        // marginOfLabel is margins(label) in the formula
-        val marginOfLabel = tmp(label.toInt)
-
-        var sum = 0.0
-        j = 0
-        while (j < numClasses) {
-          if (maxMargin > 0) tmp(j) -= maxMargin
-          val exp = math.exp(tmp(j))
-          sum += exp
-          tmp(j) = exp
-          j += 1
-        }
-
-        j = 0
-        while (j < numClasses) {
-          val multiplier = weight * (tmp(j) / sum - (if (label == j) 1.0 else 0.0))
-          mat.update(i, j, multiplier)
-          if (fitIntercept) interceptGradSumArr(j) += multiplier
-          j += 1
-        }
-
-        if (maxMargin > 0) {
-          lossSum += weight * (math.log(sum) - marginOfLabel + maxMargin)
-        } else {
-          lossSum += weight * (math.log(sum) - marginOfLabel)
-        }
-      } else {
-        var j = 0; while (j < numClasses) { mat.update(i, j, 0.0); j += 1 }
-      }
-      i += 1
-    }
-
-    // mat (multipliers):             S X C, dense                                N
-    // mat.transpose (multipliers):   C X S, dense                                T
-    // block.matrix:                  S X F, unknown type                         T
-    // gradSumMat(gradientSumArray):  C X FPI (numFeaturesPlusIntercept), dense   N
-    block.matrix match {
-      case dm: DenseMatrix =>
-        BLAS.nativeBLAS.dgemm("T", "T", numClasses, numFeatures, size, 1.0,
-          mat.values, size, dm.values, numFeatures, 1.0, gradientSumArray, numClasses)
-
-      case sm: SparseMatrix =>
-        // linearGradSumMat = matrix.T X mat
-        val linearGradSumMat = DenseMatrix.zeros(numFeatures, numClasses)
-        BLAS.gemm(1.0, sm.transpose, mat, 0.0, linearGradSumMat)
-        linearGradSumMat.foreachActive { (i, j, v) => gradientSumArray(i * numClasses + j) += v }
-    }
-
-    if (fitIntercept) {
-      BLAS.getBLAS(numClasses).daxpy(numClasses, 1.0, interceptGradSumArr, 0, 1,
-        gradientSumArray, numClasses * numFeatures, 1)
-    }
-  }
-}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/MultinomialLogisticBlockAggregator.scala b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/MultinomialLogisticBlockAggregator.scala
new file mode 100644
index 0000000000000..0683cec628849
--- /dev/null
+++ b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/MultinomialLogisticBlockAggregator.scala
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.ml.optim.aggregator
+
+import org.apache.spark.broadcast.Broadcast
+import org.apache.spark.internal.Logging
+import org.apache.spark.ml.feature.InstanceBlock
+import org.apache.spark.ml.linalg._
+
+/**
+ * MultinomialLogisticBlockAggregator computes the gradient and loss used in multinomial logistic
+ * classification for blocks in sparse or dense matrix in an online fashion.
+ *
+ * Two MultinomialLogisticBlockAggregator can be merged together to have a summary of loss and
+ * gradient of the corresponding joint dataset.
+ *
+ * NOTE: The feature values are expected to already have be scaled (multiplied by bcInverseStd,
+ * but NOT centered) before computation.
+ *
+ * @param bcCoefficients The coefficients corresponding to the features.
+ * @param fitIntercept   Whether to fit an intercept term.
+ * @param fitWithMean    Whether to center the data with mean before training, in a virtual way.
+ *                       If true, we MUST adjust the intercept of both initial coefficients and
+ *                       final solution in the caller.
+ * @note In order to avoid unnecessary computation during calculation of the gradient updates
+ * we lay out the coefficients in column major order during training. We convert back to row
+ * major order when we create the model, since this form is optimal for the matrix operations
+ * used for prediction.
+ */
+private[ml] class MultinomialLogisticBlockAggregator(
+    bcInverseStd: Broadcast[Array[Double]],
+    bcScaledMean: Broadcast[Array[Double]],
+    fitIntercept: Boolean,
+    fitWithMean: Boolean)(bcCoefficients: Broadcast[Vector])
+  extends DifferentiableLossAggregator[InstanceBlock, MultinomialLogisticBlockAggregator]
+    with Logging {
+
+  if (fitWithMean) {
+    require(fitIntercept, s"for training without intercept, should not center the vectors")
+    require(bcScaledMean != null && bcScaledMean.value.length == bcInverseStd.value.length,
+      "scaled means is required when center the vectors")
+  }
+
+  private val numFeatures = bcInverseStd.value.length
+  protected override val dim: Int = bcCoefficients.value.size
+  private val numFeaturesPlusIntercept = if (fitIntercept) numFeatures + 1 else numFeatures
+  private val numClasses = dim / numFeaturesPlusIntercept
+  require(dim == numClasses * numFeaturesPlusIntercept)
+
+  @transient private lazy val coefficientsArray = bcCoefficients.value match {
+    case DenseVector(values) => values
+    case _ => throw new IllegalArgumentException(s"coefficients only supports dense vector but " +
+      s"got type ${bcCoefficients.value.getClass}.)")
+  }
+
+  @transient private lazy val linear = if (fitIntercept) {
+    new DenseMatrix(numClasses, numFeatures, coefficientsArray.take(numClasses * numFeatures))
+  } else {
+    new DenseMatrix(numClasses, numFeatures, coefficientsArray)
+  }
+
+  private lazy val intercept = if (fitIntercept) {
+    new DenseVector(coefficientsArray.takeRight(numClasses))
+  } else {
+    null
+  }
+
+  // pre-computed margin of an empty vector.
+  // with this variable as an offset, for a sparse vector, we only need to
+  // deal with non-zero values in prediction.
+  private val marginOffset = if (fitWithMean) {
+    val offset = intercept.copy
+    BLAS.gemv(-1.0, linear, Vectors.dense(bcScaledMean.value), 1.0, offset)
+    offset
+  } else {
+    null
+  }
+
+  /**
+   * Add a new training instance block to this BinaryLogisticBlockAggregator, and update the loss
+   * and gradient of the objective function.
+   *
+   * @param block The instance block of data point to be added.
+   * @return This BinaryLogisticBlockAggregator object.
+   */
+  def add(block: InstanceBlock): this.type = {
+    require(block.matrix.isTransposed)
+    require(numFeatures == block.numFeatures, s"Dimensions mismatch when adding new " +
+      s"instance. Expecting $numFeatures but got ${block.numFeatures}.")
+    require(block.weightIter.forall(_ >= 0),
+      s"instance weights ${block.weightIter.mkString("[", ",", "]")} has to be >= 0.0")
+
+    if (block.weightIter.forall(_ == 0)) return this
+    val size = block.size
+
+    // mat/arr here represents margins, shape: S X C
+    val mat = DenseMatrix.zeros(size, numClasses)
+    val arr = mat.values
+    if (fitIntercept) {
+      val offset = if (fitWithMean) marginOffset else intercept
+      var j = 0
+      while (j < numClasses) {
+        java.util.Arrays.fill(arr, j * size, (j + 1) * size, offset(j))
+        j += 1
+      }
+    }
+    BLAS.gemm(1.0, block.matrix, linear.transpose, 1.0, mat)
+
+    // in-place convert margins to multipliers
+    // then, mat/arr represents multipliers
+    var localLossSum = 0.0
+    var localWeightSum = 0.0
+    var i = 0
+    val tmp = Array.ofDim[Double](numClasses)
+    val multiplierSum = Array.ofDim[Double](numClasses)
+    while (i < size) {
+      val weight = block.getWeight(i)
+      localWeightSum += weight
+      if (weight > 0) {
+        val label = block.getLabel(i)
+        var maxMargin = Double.NegativeInfinity
+        var j = 0
+        while (j < numClasses) {
+          tmp(j) = mat(i, j)
+          maxMargin = math.max(maxMargin, tmp(j))
+          j += 1
+        }
+
+        // marginOfLabel is margins(label) in the formula
+        val marginOfLabel = tmp(label.toInt)
+
+        var sum = 0.0
+        j = 0
+        while (j < numClasses) {
+          if (maxMargin > 0) tmp(j) -= maxMargin
+          val exp = math.exp(tmp(j))
+          sum += exp
+          tmp(j) = exp
+          j += 1
+        }
+
+        j = 0
+        while (j < numClasses) {
+          val multiplier = weight * (tmp(j) / sum - (if (label == j) 1.0 else 0.0))
+          mat.update(i, j, multiplier)
+          multiplierSum(j) += multiplier
+          j += 1
+        }
+
+        if (maxMargin > 0) {
+          localLossSum += weight * (math.log(sum) - marginOfLabel + maxMargin)
+        } else {
+          localLossSum += weight * (math.log(sum) - marginOfLabel)
+        }
+      } else {
+        var j = 0; while (j < numClasses) { mat.update(i, j, 0.0); j += 1 }
+      }
+      i += 1
+    }
+    lossSum += localLossSum
+    weightSum += localWeightSum
+
+    // mat (multipliers):             S X C, dense                                N
+    // mat.transpose (multipliers):   C X S, dense                                T
+    // block.matrix (data):           S X F, unknown type                         T
+    // gradSumMat (gradientSumArray): C X FPI (numFeaturesPlusIntercept), dense   N
+    block.matrix match {
+      case dm: DenseMatrix =>
+        // gradientSumArray[0 : F X C] += mat.T X dm
+        BLAS.nativeBLAS.dgemm("T", "T", numClasses, numFeatures, size, 1.0,
+          mat.values, size, dm.values, numFeatures, 1.0, gradientSumArray, numClasses)
+
+      case sm: SparseMatrix =>
+        // TODO: convert Coefficients to row major order to simplify BLAS operations?
+        // linearGradSumMat = sm.T X mat
+        // existing BLAS.gemm requires linearGradSumMat is NOT Transposed.
+        val linearGradSumMat = DenseMatrix.zeros(numFeatures, numClasses)
+        BLAS.gemm(1.0, sm.transpose, mat, 0.0, linearGradSumMat)
+        linearGradSumMat.foreachActive { (i, j, v) => gradientSumArray(i * numClasses + j) += v }
+    }
+
+    if (fitWithMean) {
+      // above update of the linear part of gradientSumArray does NOT take the centering
+      // into account, here we need to adjust this part.
+      // following BLAS.dger operation equals to: gradientSumArray[0 : F X C] -= mat.T X _mm_,
+      // where _mm_ is a matrix of size S X F with each row equals to array ScaledMean.
+      BLAS.nativeBLAS.dger(numClasses, numFeatures, -1.0, multiplierSum, 1,
+        bcScaledMean.value, 1, gradientSumArray, numClasses)
+    }
+
+    if (fitIntercept) {
+      BLAS.javaBLAS.daxpy(numClasses, 1.0, multiplierSum, 0, 1,
+        gradientSumArray, numClasses * numFeatures, 1)
+    }
+
+    this
+  }
+}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/param/params.scala b/mllib/src/main/scala/org/apache/spark/ml/param/params.scala
index 53ca35ccd0073..f12c1f995b7d7 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/param/params.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/param/params.scala
@@ -937,7 +937,7 @@ final class ParamMap private[ml] (private val map: mutable.Map[Param[Any], Any])
 
   /** Put param pairs with a `java.util.List` of values for Python. */
   private[ml] def put(paramPairs: JList[ParamPair[_]]): this.type = {
-    put(paramPairs.asScala: _*)
+    put(paramPairs.asScala.toSeq: _*)
   }
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala b/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala
index 7fd5f5938b565..2f6b9c1e11aac 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala
@@ -108,7 +108,12 @@ private[shared] object SharedParamsCodeGen {
       ParamDesc[Int]("blockSize", "block size for stacking input data in matrices. Data is " +
         "stacked within partitions. If block size is more than remaining data in a partition " +
         "then it is adjusted to the size of this data.",
-        isValid = "ParamValidators.gt(0)", isExpertParam = true)
+        isValid = "ParamValidators.gt(0)", isExpertParam = true),
+      ParamDesc[Double]("maxBlockSizeInMB", "Maximum memory in MB for stacking input data " +
+        "into blocks. Data is stacked within partitions. If more than remaining data size in a " +
+        "partition then it is adjusted to the data size. Default 0.0 represents choosing " +
+        "optimal value, depends on specific algorithm. Must be >= 0.",
+        Some("0.0"), isValid = "ParamValidators.gtEq(0.0)", isExpertParam = true)
     )
 
     val code = genSharedParams(params)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala b/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala
index 60203eba61ea5..425bf91fd00ba 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala
@@ -562,4 +562,22 @@ trait HasBlockSize extends Params {
   /** @group expertGetParam */
   final def getBlockSize: Int = $(blockSize)
 }
+
+/**
+ * Trait for shared param maxBlockSizeInMB (default: 0.0). This trait may be changed or
+ * removed between minor versions.
+ */
+trait HasMaxBlockSizeInMB extends Params {
+
+  /**
+   * Param for Maximum memory in MB for stacking input data into blocks. Data is stacked within partitions. If more than remaining data size in a partition then it is adjusted to the data size. Default 0.0 represents choosing optimal value, depends on specific algorithm. Must be &gt;= 0..
+   * @group expertParam
+   */
+  final val maxBlockSizeInMB: DoubleParam = new DoubleParam(this, "maxBlockSizeInMB", "Maximum memory in MB for stacking input data into blocks. Data is stacked within partitions. If more than remaining data size in a partition then it is adjusted to the data size. Default 0.0 represents choosing optimal value, depends on specific algorithm. Must be >= 0.", ParamValidators.gtEq(0.0))
+
+  setDefault(maxBlockSizeInMB, 0.0)
+
+  /** @group expertGetParam */
+  final def getMaxBlockSizeInMB: Double = $(maxBlockSizeInMB)
+}
 // scalastyle:on
diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/AFTSurvivalRegressionWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/AFTSurvivalRegressionWrapper.scala
index 1b5f77a9ae897..594d9f315f508 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/r/AFTSurvivalRegressionWrapper.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/r/AFTSurvivalRegressionWrapper.scala
@@ -88,9 +88,9 @@ private[r] object AFTSurvivalRegressionWrapper extends MLReadable[AFTSurvivalReg
       aggregationDepth: Int,
       stringIndexerOrderType: String): AFTSurvivalRegressionWrapper = {
 
-    val (rewritedFormula, censorCol) = formulaRewrite(formula)
+    val (rewrittenFormula, censorCol) = formulaRewrite(formula)
 
-    val rFormula = new RFormula().setFormula(rewritedFormula)
+    val rFormula = new RFormula().setFormula(rewrittenFormula)
       .setStringIndexerOrderType(stringIndexerOrderType)
     RWrapperUtils.checkDataColumns(rFormula, data)
     val rFormulaModel = rFormula.fit(data)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
index a0e5924a7ee3a..a0ddf7129c9b5 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
@@ -26,12 +26,12 @@ import scala.reflect.ClassTag
 import scala.util.{Sorting, Try}
 import scala.util.hashing.byteswap64
 
-import com.github.fommil.netlib.BLAS.{getInstance => blas}
+import com.google.common.collect.{Ordering => GuavaOrdering}
 import org.apache.hadoop.fs.Path
 import org.json4s.DefaultFormats
 import org.json4s.JsonDSL._
 
-import org.apache.spark.{Dependency, Partitioner, ShuffleDependency, SparkContext, SparkException}
+import org.apache.spark.{Partitioner, SparkException}
 import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.{Estimator, Model}
@@ -47,7 +47,7 @@ import org.apache.spark.sql.{DataFrame, Dataset}
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types._
 import org.apache.spark.storage.StorageLevel
-import org.apache.spark.util.{BoundedPriorityQueue, Utils}
+import org.apache.spark.util.Utils
 import org.apache.spark.util.collection.{OpenHashMap, OpenHashSet, SortDataFormat, Sorter}
 import org.apache.spark.util.random.XORShiftRandom
 
@@ -434,7 +434,8 @@ class ALSModel private[ml] (
    * relatively efficient, the approach implemented here is significantly more efficient.
    *
    * This approach groups factors into blocks and computes the top-k elements per block,
-   * using dot product and an efficient [[BoundedPriorityQueue]] (instead of gemm).
+   * using GEMV (it use less memory compared with GEMM, and is much faster than DOT) and
+   * an efficient selection based on [[GuavaOrdering]] (instead of [[BoundedPriorityQueue]]).
    * It then computes the global top-k by aggregating the per block top-k elements with
    * a [[TopByKeyAggregator]]. This significantly reduces the size of intermediate and shuffle data.
    * This is the DataFrame equivalent to the approach used in
@@ -456,30 +457,39 @@ class ALSModel private[ml] (
       num: Int,
       blockSize: Int): DataFrame = {
     import srcFactors.sparkSession.implicits._
+    import scala.collection.JavaConverters._
 
     val srcFactorsBlocked = blockify(srcFactors.as[(Int, Array[Float])], blockSize)
     val dstFactorsBlocked = blockify(dstFactors.as[(Int, Array[Float])], blockSize)
     val ratings = srcFactorsBlocked.crossJoin(dstFactorsBlocked)
-      .as[(Seq[(Int, Array[Float])], Seq[(Int, Array[Float])])]
-      .flatMap { case (srcIter, dstIter) =>
-        val m = srcIter.size
-        val n = math.min(dstIter.size, num)
-        val output = new Array[(Int, Int, Float)](m * n)
-        var i = 0
-        val pq = new BoundedPriorityQueue[(Int, Float)](num)(Ordering.by(_._2))
-        srcIter.foreach { case (srcId, srcFactor) =>
-          dstIter.foreach { case (dstId, dstFactor) =>
-            // We use F2jBLAS which is faster than a call to native BLAS for vector dot product
-            val score = BLAS.f2jBLAS.sdot(rank, srcFactor, 1, dstFactor, 1)
-            pq += dstId -> score
+      .as[(Array[Int], Array[Float], Array[Int], Array[Float])]
+      .mapPartitions { iter =>
+        var scores: Array[Float] = null
+        var idxOrd: GuavaOrdering[Int] = null
+        iter.flatMap { case (srcIds, srcMat, dstIds, dstMat) =>
+          require(srcMat.length == srcIds.length * rank)
+          require(dstMat.length == dstIds.length * rank)
+          val m = srcIds.length
+          val n = dstIds.length
+          if (scores == null || scores.length < n) {
+            scores = Array.ofDim[Float](n)
+            idxOrd = new GuavaOrdering[Int] {
+              override def compare(left: Int, right: Int): Int = {
+                Ordering[Float].compare(scores(left), scores(right))
+              }
+            }
           }
-          pq.foreach { case (dstId, score) =>
-            output(i) = (srcId, dstId, score)
-            i += 1
+
+          Iterator.range(0, m).flatMap { i =>
+            // scores = i-th vec in srcMat * dstMat
+            BLAS.javaBLAS.sgemv("T", rank, n, 1.0F, dstMat, 0, rank,
+              srcMat, i * rank, 1, 0.0F, scores, 0, 1)
+
+            val srcId = srcIds(i)
+            idxOrd.greatestOf(Iterator.range(0, n).asJava, num).asScala
+              .iterator.map { j => (srcId, dstIds(j), scores(j)) }
           }
-          pq.clear()
         }
-        output.toSeq
       }
     // We'll force the IDs to be Int. Unfortunately this converts IDs to Int in the output.
     val topKAggregator = new TopByKeyAggregator[Int, Int, Float](num, Ordering.by(_._2))
@@ -499,9 +509,12 @@ class ALSModel private[ml] (
    */
   private def blockify(
       factors: Dataset[(Int, Array[Float])],
-      blockSize: Int): Dataset[Seq[(Int, Array[Float])]] = {
+      blockSize: Int): Dataset[(Array[Int], Array[Float])] = {
     import factors.sparkSession.implicits._
-    factors.mapPartitions(_.grouped(blockSize))
+    factors.mapPartitions { iter =>
+      iter.grouped(blockSize)
+        .map(block => (block.map(_._1).toArray, block.flatMap(_._2).toArray))
+    }
   }
 
 }
@@ -881,9 +894,9 @@ object ALS extends DefaultParamsReadable[ALS] with Logging {
       require(c >= 0.0)
       require(a.length == k)
       copyToDouble(a)
-      blas.dspr(upper, k, c, da, 1, ata)
+      BLAS.nativeBLAS.dspr(upper, k, c, da, 1, ata)
       if (b != 0.0) {
-        blas.daxpy(k, b, da, 1, atb, 1)
+        BLAS.nativeBLAS.daxpy(k, b, da, 1, atb, 1)
       }
       this
     }
@@ -891,8 +904,8 @@ object ALS extends DefaultParamsReadable[ALS] with Logging {
     /** Merges another normal equation object. */
     def merge(other: NormalEquation): NormalEquation = {
       require(other.k == k)
-      blas.daxpy(ata.length, 1.0, other.ata, 1, ata, 1)
-      blas.daxpy(atb.length, 1.0, other.atb, 1, atb, 1)
+      BLAS.nativeBLAS.daxpy(ata.length, 1.0, other.ata, 1, ata, 1)
+      BLAS.nativeBLAS.daxpy(atb.length, 1.0, other.atb, 1, atb, 1)
       this
     }
 
@@ -1266,8 +1279,8 @@ object ALS extends DefaultParamsReadable[ALS] with Logging {
           val random = new XORShiftRandom(byteswap64(seed ^ srcBlockId))
           val factors = Array.fill(inBlock.srcIds.length) {
             val factor = Array.fill(rank)(random.nextGaussian().toFloat)
-            val nrm = blas.snrm2(rank, factor, 1)
-            blas.sscal(rank, 1.0f / nrm, factor, 1)
+            val nrm = BLAS.nativeBLAS.snrm2(rank, factor, 1)
+            BLAS.nativeBLAS.sscal(rank, 1.0f / nrm, factor, 1)
             factor
           }
           (srcBlockId, factors)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/TopByKeyAggregator.scala b/mllib/src/main/scala/org/apache/spark/ml/recommendation/TopByKeyAggregator.scala
index 517179c0eb9ae..ed41169070c59 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/TopByKeyAggregator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/TopByKeyAggregator.scala
@@ -17,7 +17,6 @@
 
 package org.apache.spark.ml.recommendation
 
-import scala.language.implicitConversions
 import scala.reflect.runtime.universe.TypeTag
 
 import org.apache.spark.sql.{Encoder, Encoders}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
index 2c30e44b93467..4d214dc74ed8b 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
@@ -27,7 +27,7 @@ import org.apache.spark.SparkException
 import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.PredictorParams
-import org.apache.spark.ml.feature.StandardScalerModel
+import org.apache.spark.ml.feature._
 import org.apache.spark.ml.linalg._
 import org.apache.spark.ml.optim.aggregator._
 import org.apache.spark.ml.optim.loss.RDDLossFunction
@@ -47,8 +47,8 @@ import org.apache.spark.storage.StorageLevel
  * Params for accelerated failure time (AFT) regression.
  */
 private[regression] trait AFTSurvivalRegressionParams extends PredictorParams
-  with HasMaxIter with HasTol with HasFitIntercept with HasAggregationDepth with HasBlockSize
-  with Logging {
+  with HasMaxIter with HasTol with HasFitIntercept with HasAggregationDepth
+  with HasMaxBlockSizeInMB with Logging {
 
   /**
    * Param for censor column name.
@@ -62,7 +62,6 @@ private[regression] trait AFTSurvivalRegressionParams extends PredictorParams
   /** @group getParam */
   @Since("1.6.0")
   def getCensorCol: String = $(censorCol)
-  setDefault(censorCol -> "censor")
 
   /**
    * Param for quantile probabilities array.
@@ -78,7 +77,6 @@ private[regression] trait AFTSurvivalRegressionParams extends PredictorParams
   /** @group getParam */
   @Since("1.6.0")
   def getQuantileProbabilities: Array[Double] = $(quantileProbabilities)
-  setDefault(quantileProbabilities -> Array(0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99))
 
   /**
    * Param for quantiles column name.
@@ -92,6 +90,11 @@ private[regression] trait AFTSurvivalRegressionParams extends PredictorParams
   @Since("1.6.0")
   def getQuantilesCol: String = $(quantilesCol)
 
+  setDefault(censorCol -> "censor",
+    quantileProbabilities -> Array(0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99),
+    fitIntercept -> true, maxIter -> 100, tol -> 1E-6, aggregationDepth -> 2,
+    maxBlockSizeInMB -> 0.0)
+
   /** Checks whether the input has quantiles column name. */
   private[regression] def hasQuantilesCol: Boolean = {
     isDefined(quantilesCol) && $(quantilesCol).nonEmpty
@@ -125,6 +128,10 @@ private[regression] trait AFTSurvivalRegressionParams extends PredictorParams
  * (see <a href="https://en.wikipedia.org/wiki/Accelerated_failure_time_model">
  * Accelerated failure time model (Wikipedia)</a>)
  * based on the Weibull distribution of the survival time.
+ *
+ * Since 3.1.0, it supports stacking instances into blocks and using GEMV for
+ * better performance.
+ * The block size will be 1.0 MB, if param maxBlockSizeInMB is set 0.0 by default.
  */
 @Since("1.6.0")
 class AFTSurvivalRegression @Since("1.6.0") (@Since("1.6.0") override val uid: String)
@@ -153,7 +160,6 @@ class AFTSurvivalRegression @Since("1.6.0") (@Since("1.6.0") override val uid: S
    */
   @Since("1.6.0")
   def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
-  setDefault(fitIntercept -> true)
 
   /**
    * Set the maximum number of iterations.
@@ -162,7 +168,6 @@ class AFTSurvivalRegression @Since("1.6.0") (@Since("1.6.0") override val uid: S
    */
   @Since("1.6.0")
   def setMaxIter(value: Int): this.type = set(maxIter, value)
-  setDefault(maxIter -> 100)
 
   /**
    * Set the convergence tolerance of iterations.
@@ -172,7 +177,6 @@ class AFTSurvivalRegression @Since("1.6.0") (@Since("1.6.0") override val uid: S
    */
   @Since("1.6.0")
   def setTol(value: Double): this.type = set(tol, value)
-  setDefault(tol -> 1E-6)
 
   /**
    * Suggested depth for treeAggregate (greater than or equal to 2).
@@ -183,59 +187,41 @@ class AFTSurvivalRegression @Since("1.6.0") (@Since("1.6.0") override val uid: S
    */
   @Since("2.1.0")
   def setAggregationDepth(value: Int): this.type = set(aggregationDepth, value)
-  setDefault(aggregationDepth -> 2)
 
   /**
-   * Set block size for stacking input data in matrices.
-   * If blockSize == 1, then stacking will be skipped, and each vector is treated individually;
-   * If blockSize &gt; 1, then vectors will be stacked to blocks, and high-level BLAS routines
-   * will be used if possible (for example, GEMV instead of DOT, GEMM instead of GEMV).
-   * Recommended size is between 10 and 1000. An appropriate choice of the block size depends
-   * on the sparsity and dim of input datasets, the underlying BLAS implementation (for example,
-   * f2jBLAS, OpenBLAS, intel MKL) and its configuration (for example, number of threads).
-   * Note that existing BLAS implementations are mainly optimized for dense matrices, if the
-   * input dataset is sparse, stacking may bring no performance gain, the worse is possible
-   * performance regression.
-   * Default is 1.
+   * Sets the value of param [[maxBlockSizeInMB]].
+   * Default is 0.0, then 1.0 MB will be chosen.
    *
    * @group expertSetParam
    */
   @Since("3.1.0")
-  def setBlockSize(value: Int): this.type = set(blockSize, value)
-  setDefault(blockSize -> 1)
-
-  /**
-   * Extract [[featuresCol]], [[labelCol]] and [[censorCol]] from input dataset,
-   * and put it in an RDD with strong types.
-   */
-  protected[ml] def extractAFTPoints(dataset: Dataset[_]): RDD[AFTPoint] = {
-    dataset.select(col($(featuresCol)), col($(labelCol)).cast(DoubleType),
-      col($(censorCol)).cast(DoubleType)).rdd.map {
-        case Row(features: Vector, label: Double, censor: Double) =>
-          AFTPoint(features, label, censor)
-      }
-  }
+  def setMaxBlockSizeInMB(value: Double): this.type = set(maxBlockSizeInMB, value)
 
   override protected def train(
       dataset: Dataset[_]): AFTSurvivalRegressionModel = instrumented { instr =>
     instr.logPipelineStage(this)
     instr.logDataset(dataset)
     instr.logParams(this, labelCol, featuresCol, censorCol, predictionCol, quantilesCol,
-      fitIntercept, maxIter, tol, aggregationDepth, blockSize)
+      fitIntercept, maxIter, tol, aggregationDepth, maxBlockSizeInMB)
     instr.logNamedValue("quantileProbabilities.size", $(quantileProbabilities).length)
 
-    val instances = extractAFTPoints(dataset)
-      .setName("training instances")
-
-    if ($(blockSize) == 1 && dataset.storageLevel == StorageLevel.NONE) {
-      instances.persist(StorageLevel.MEMORY_AND_DISK)
+    if (dataset.storageLevel != StorageLevel.NONE) {
+      instr.logWarning(s"Input instances will be standardized, blockified to blocks, and " +
+        s"then cached during training. Be careful of double caching!")
     }
 
-    var requestedMetrics = Seq("mean", "std", "count")
-    if ($(blockSize) != 1) requestedMetrics +:= "numNonZeros"
+    val instances = dataset.select(col($(featuresCol)), col($(labelCol)).cast(DoubleType),
+      col($(censorCol)).cast(DoubleType))
+      .rdd.map { case Row(features: Vector, label: Double, censor: Double) =>
+        require(censor == 1.0 || censor == 0.0, "censor must be 1.0 or 0.0")
+        // AFT does not support instance weighting,
+        // here use Instance.weight to store censor for convenience
+        Instance(label, censor, features)
+      }.setName("training instances")
+
     val summarizer = instances.treeAggregate(
-      Summarizer.createSummarizerBuffer(requestedMetrics: _*))(
-      seqOp = (c: SummarizerBuffer, v: AFTPoint) => c.add(v.features),
+      Summarizer.createSummarizerBuffer("mean", "std", "count"))(
+      seqOp = (c: SummarizerBuffer, i: Instance) => c.add(i.features),
       combOp = (c1: SummarizerBuffer, c2: SummarizerBuffer) => c1.merge(c2),
       depth = $(aggregationDepth)
     )
@@ -244,14 +230,12 @@ class AFTSurvivalRegression @Since("1.6.0") (@Since("1.6.0") override val uid: S
     val numFeatures = featuresStd.length
     instr.logNumFeatures(numFeatures)
     instr.logNumExamples(summarizer.count)
-    if ($(blockSize) > 1) {
-      val scale = 1.0 / summarizer.count / numFeatures
-      val sparsity = 1 - summarizer.numNonzeros.toArray.map(_ * scale).sum
-      instr.logNamedValue("sparsity", sparsity.toString)
-      if (sparsity > 0.5) {
-        instr.logWarning(s"sparsity of input dataset is $sparsity, " +
-          s"which may hurt performance in high-level BLAS.")
-      }
+
+    var actualBlockSizeInMB = $(maxBlockSizeInMB)
+    if (actualBlockSizeInMB == 0) {
+      actualBlockSizeInMB = InstanceBlock.DefaultBlockSizeInMB
+      require(actualBlockSizeInMB > 0, "inferred actual BlockSizeInMB must > 0")
+      instr.logNamedValue("actualBlockSizeInMB", actualBlockSizeInMB.toString)
     }
 
     if (!$(fitIntercept) && (0 until numFeatures).exists { i =>
@@ -271,12 +255,8 @@ class AFTSurvivalRegression @Since("1.6.0") (@Since("1.6.0") override val uid: S
      */
     val initialParameters = Vectors.zeros(numFeatures + 2)
 
-    val (rawCoefficients, objectiveHistory) = if ($(blockSize) == 1) {
-      trainOnRows(instances, featuresStd, optimizer, initialParameters)
-    } else {
-      trainOnBlocks(instances, featuresStd, optimizer, initialParameters)
-    }
-    if (instances.getStorageLevel != StorageLevel.NONE) instances.unpersist()
+    val (rawCoefficients, objectiveHistory) =
+      trainImpl(instances, actualBlockSizeInMB, featuresStd, optimizer, initialParameters)
 
     if (rawCoefficients == null) {
       val msg = s"${optimizer.getClass.getName} failed."
@@ -293,47 +273,24 @@ class AFTSurvivalRegression @Since("1.6.0") (@Since("1.6.0") override val uid: S
     new AFTSurvivalRegressionModel(uid, coefficients, intercept, scale)
   }
 
-  private def trainOnRows(
-      instances: RDD[AFTPoint],
+  private def trainImpl(
+      instances: RDD[Instance],
+      actualBlockSizeInMB: Double,
       featuresStd: Array[Double],
       optimizer: BreezeLBFGS[BDV[Double]],
       initialParameters: Vector): (Array[Double], Array[Double]) = {
     val bcFeaturesStd = instances.context.broadcast(featuresStd)
-    val getAggregatorFunc = new AFTAggregator(bcFeaturesStd, $(fitIntercept))(_)
-    val costFun = new RDDLossFunction(instances, getAggregatorFunc, None, $(aggregationDepth))
-
-    val states = optimizer.iterations(new CachedDiffFunction(costFun),
-      initialParameters.asBreeze.toDenseVector)
 
-    val arrayBuilder = mutable.ArrayBuilder.make[Double]
-    var state: optimizer.State = null
-    while (states.hasNext) {
-      state = states.next()
-      arrayBuilder += state.adjustedValue
-    }
-    bcFeaturesStd.destroy()
-
-    (if (state != null) state.x.toArray else null, arrayBuilder.result)
-  }
-
-  private def trainOnBlocks(
-      instances: RDD[AFTPoint],
-      featuresStd: Array[Double],
-      optimizer: BreezeLBFGS[BDV[Double]],
-      initialParameters: Vector): (Array[Double], Array[Double]) = {
-    val bcFeaturesStd = instances.context.broadcast(featuresStd)
-    val blocks = instances.mapPartitions { iter =>
+    val standardized = instances.mapPartitions { iter =>
       val inverseStd = bcFeaturesStd.value.map { std => if (std != 0) 1.0 / std else 0.0 }
       val func = StandardScalerModel.getTransformFunc(Array.empty, inverseStd, false, true)
-      iter.grouped($(blockSize)).map { seq =>
-        val matrix = Matrices.fromVectors(seq.map(point => func(point.features)))
-        val labels = seq.map(_.label).toArray
-        val censors = seq.map(_.censor).toArray
-        (matrix, labels, censors)
-      }
+      iter.map { case Instance(label, weight, vec) => Instance(label, weight, func(vec)) }
     }
-    blocks.persist(StorageLevel.MEMORY_AND_DISK)
-      .setName(s"training blocks (blockSize=${$(blockSize)})")
+
+    val maxMemUsage = (actualBlockSizeInMB * 1024L * 1024L).ceil.toLong
+    val blocks = InstanceBlock.blokifyWithMaxMemUsage(standardized, maxMemUsage)
+      .persist(StorageLevel.MEMORY_AND_DISK)
+      .setName(s"training blocks (blockSizeInMB=$actualBlockSizeInMB)")
 
     val getAggregatorFunc = new BlockAFTAggregator($(fitIntercept))(_)
     val costFun = new RDDLossFunction(blocks, getAggregatorFunc, None, $(aggregationDepth))
@@ -386,22 +343,32 @@ class AFTSurvivalRegressionModel private[ml] (
 
   /** @group setParam */
   @Since("1.6.0")
-  def setQuantileProbabilities(value: Array[Double]): this.type = set(quantileProbabilities, value)
+  def setQuantileProbabilities(value: Array[Double]): this.type = {
+    set(quantileProbabilities, value)
+    _quantiles(0) = $(quantileProbabilities).map(q => math.exp(math.log(-math.log1p(-q)) * scale))
+    this
+  }
 
   /** @group setParam */
   @Since("1.6.0")
   def setQuantilesCol(value: String): this.type = set(quantilesCol, value)
 
+  private lazy val _quantiles = {
+    Array($(quantileProbabilities).map(q => math.exp(math.log(-math.log1p(-q)) * scale)))
+  }
+
+  private def lambda2Quantiles(lambda: Double): Vector = {
+    val quantiles = _quantiles(0).clone()
+    var i = 0
+    while (i < quantiles.length) { quantiles(i) *= lambda; i += 1 }
+    Vectors.dense(quantiles)
+  }
+
   @Since("2.0.0")
   def predictQuantiles(features: Vector): Vector = {
     // scale parameter for the Weibull distribution of lifetime
-    val lambda = math.exp(BLAS.dot(coefficients, features) + intercept)
-    // shape parameter for the Weibull distribution of lifetime
-    val k = 1 / scale
-    val quantiles = $(quantileProbabilities).map {
-      q => lambda * math.exp(math.log(-math.log1p(-q)) / k)
-    }
-    Vectors.dense(quantiles)
+    val lambda = predict(features)
+    lambda2Quantiles(lambda)
   }
 
   @Since("2.0.0")
@@ -417,16 +384,20 @@ class AFTSurvivalRegressionModel private[ml] (
     var predictionColumns = Seq.empty[Column]
 
     if ($(predictionCol).nonEmpty) {
-      val predictUDF = udf { features: Vector => predict(features) }
+      val predCol = udf(predict _).apply(col($(featuresCol)))
       predictionColNames :+= $(predictionCol)
-      predictionColumns :+= predictUDF(col($(featuresCol)))
+      predictionColumns :+= predCol
         .as($(predictionCol), outputSchema($(predictionCol)).metadata)
     }
 
     if (hasQuantilesCol) {
-      val predictQuantilesUDF = udf { features: Vector => predictQuantiles(features)}
+      val quanCol = if ($(predictionCol).nonEmpty) {
+        udf(lambda2Quantiles _).apply(predictionColumns.head)
+      } else {
+        udf(predictQuantiles _).apply(col($(featuresCol)))
+      }
       predictionColNames :+= $(quantilesCol)
-      predictionColumns :+= predictQuantilesUDF(col($(featuresCol)))
+      predictionColumns :+= quanCol
         .as($(quantilesCol), outputSchema($(quantilesCol)).metadata)
     }
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/DecisionTreeRegressor.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/DecisionTreeRegressor.scala
index 3cb69e7fe8212..6913718bba996 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/DecisionTreeRegressor.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/DecisionTreeRegressor.scala
@@ -294,7 +294,8 @@ object DecisionTreeRegressionModel extends MLReadable[DecisionTreeRegressionMode
       DefaultParamsWriter.saveMetadata(instance, path, sc, Some(extraMetadata))
       val (nodeData, _) = NodeData.build(instance.rootNode, 0)
       val dataPath = new Path(path, "data").toString
-      sparkSession.createDataFrame(nodeData).write.parquet(dataPath)
+      val numDataParts = NodeData.inferNumPartitions(instance.numNodes)
+      sparkSession.createDataFrame(nodeData).repartition(numDataParts).write.parquet(dataPath)
     }
   }
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/FMRegressor.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/FMRegressor.scala
index df4dac1e240e2..f70baa4ddd393 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/FMRegressor.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/FMRegressor.scala
@@ -47,7 +47,7 @@ import org.apache.spark.storage.StorageLevel
  */
 private[ml] trait FactorizationMachinesParams extends PredictorParams
   with HasMaxIter with HasStepSize with HasTol with HasSolver with HasSeed
-  with HasFitIntercept with HasRegParam {
+  with HasFitIntercept with HasRegParam with HasWeightCol {
 
   /**
    * Param for dimensionality of the factors (&gt;= 0)
@@ -112,6 +112,10 @@ private[ml] trait FactorizationMachinesParams extends PredictorParams
     "The solver algorithm for optimization. Supported options: " +
       s"${supportedSolvers.mkString(", ")}. (Default adamW)",
     ParamValidators.inArray[String](supportedSolvers))
+
+  setDefault(factorSize -> 8, fitIntercept -> true, fitLinear -> true, regParam -> 0.0,
+    miniBatchFraction -> 1.0, initStd -> 0.01, maxIter -> 100, stepSize -> 1.0, tol -> 1E-6,
+    solver -> AdamW)
 }
 
 private[ml] trait FactorizationMachines extends FactorizationMachinesParams {
@@ -130,7 +134,7 @@ private[ml] trait FactorizationMachines extends FactorizationMachinesParams {
       data: RDD[(Double, OldVector)],
       numFeatures: Int,
       loss: String
-    ): Vector = {
+    ): (Vector, Array[Double]) = {
 
     // initialize coefficients
     val initialCoefficients = initCoefficients(numFeatures)
@@ -147,8 +151,8 @@ private[ml] trait FactorizationMachines extends FactorizationMachinesParams {
       .setRegParam($(regParam))
       .setMiniBatchFraction($(miniBatchFraction))
       .setConvergenceTol($(tol))
-    val coefficients = optimizer.optimize(data, initialCoefficients)
-    coefficients.asML
+    val (coefficients, lossHistory) = optimizer.optimizeWithLossReturned(data, initialCoefficients)
+    (coefficients.asML, lossHistory)
   }
 }
 
@@ -308,7 +312,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setFactorSize(value: Int): this.type = set(factorSize, value)
-  setDefault(factorSize -> 8)
 
   /**
    * Set whether to fit intercept term.
@@ -318,7 +321,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
-  setDefault(fitIntercept -> true)
 
   /**
    * Set whether to fit linear term.
@@ -328,7 +330,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setFitLinear(value: Boolean): this.type = set(fitLinear, value)
-  setDefault(fitLinear -> true)
 
   /**
    * Set the L2 regularization parameter.
@@ -338,7 +339,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setRegParam(value: Double): this.type = set(regParam, value)
-  setDefault(regParam -> 0.0)
 
   /**
    * Set the mini-batch fraction parameter.
@@ -348,7 +348,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setMiniBatchFraction(value: Double): this.type = set(miniBatchFraction, value)
-  setDefault(miniBatchFraction -> 1.0)
 
   /**
    * Set the standard deviation of initial coefficients.
@@ -358,7 +357,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setInitStd(value: Double): this.type = set(initStd, value)
-  setDefault(initStd -> 0.01)
 
   /**
    * Set the maximum number of iterations.
@@ -368,7 +366,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setMaxIter(value: Int): this.type = set(maxIter, value)
-  setDefault(maxIter -> 100)
 
   /**
    * Set the initial step size for the first step (like learning rate).
@@ -378,7 +375,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setStepSize(value: Double): this.type = set(stepSize, value)
-  setDefault(stepSize -> 1.0)
 
   /**
    * Set the convergence tolerance of iterations.
@@ -388,7 +384,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setTol(value: Double): this.type = set(tol, value)
-  setDefault(tol -> 1E-6)
 
   /**
    * Set the solver algorithm used for optimization.
@@ -399,7 +394,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setSolver(value: String): this.type = set(solver, value)
-  setDefault(solver -> AdamW)
 
   /**
    * Set the random seed for weight initialization.
@@ -427,7 +421,7 @@ class FMRegressor @Since("3.0.0") (
 
     if (handlePersistence) data.persist(StorageLevel.MEMORY_AND_DISK)
 
-    val coefficients = trainImpl(data, numFeatures, SquaredError)
+    val (coefficients, _) = trainImpl(data, numFeatures, SquaredError)
 
     val (intercept, linear, factors) = splitCoefficients(
       coefficients, numFeatures, $(factorSize), $(fitIntercept), $(fitLinear))
@@ -561,7 +555,7 @@ object FMRegressionModel extends MLReadable[FMRegressionModel] {
  *   \hat{y} = p\left( y_{fm} \right)
  * }}}
  * p is the prediction function, for binary classification task is sigmoid.
- * The loss funcation gradient formula:
+ * The loss function gradient formula:
  * {{{
  *   \frac{\partial}{\partial\theta} l\left( \hat{y},y \right) =
  *   \frac{\partial}{\partial\theta} l\left( p\left( y_{fm} \right),y \right) =
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/GBTRegressor.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/GBTRegressor.scala
index 78d5ddaa2758b..fd8af71d43568 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/GBTRegressor.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/GBTRegressor.scala
@@ -17,13 +17,12 @@
 
 package org.apache.spark.ml.regression
 
-import com.github.fommil.netlib.BLAS.{getInstance => blas}
 import org.json4s.{DefaultFormats, JObject}
 import org.json4s.JsonDSL._
 
 import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
-import org.apache.spark.ml.linalg.Vector
+import org.apache.spark.ml.linalg.{BLAS, Vector}
 import org.apache.spark.ml.param.ParamMap
 import org.apache.spark.ml.tree._
 import org.apache.spark.ml.tree.impl.GradientBoostedTrees
@@ -299,7 +298,7 @@ class GBTRegressionModel private[ml](
     // TODO: When we add a generic Boosting class, handle transform there?  SPARK-7129
     // Classifies by thresholding sum of weighted tree predictions
     val treePredictions = _trees.map(_.rootNode.predictImpl(features).prediction)
-    blas.ddot(getNumTrees, treePredictions, 1, _treeWeights, 1)
+    BLAS.nativeBLAS.ddot(getNumTrees, treePredictions, 1, _treeWeights, 1)
   }
 
   @Since("1.4.0")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
index fa41a98749f32..f7dfda81d4e6f 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
@@ -29,6 +29,7 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.ml.PredictorParams
 import org.apache.spark.ml.attribute._
 import org.apache.spark.ml.feature.{Instance, OffsetInstance}
+import org.apache.spark.ml.functions.checkNonNegativeWeight
 import org.apache.spark.ml.linalg.{BLAS, Vector, Vectors}
 import org.apache.spark.ml.optim._
 import org.apache.spark.ml.param._
@@ -180,6 +181,9 @@ private[regression] trait GeneralizedLinearRegressionBase extends PredictorParam
       s"${supportedSolvers.mkString(", ")}. (Default irls)",
     ParamValidators.inArray[String](supportedSolvers))
 
+  setDefault(family -> Gaussian.name, variancePower -> 0.0, maxIter -> 25, tol -> 1E-6,
+    regParam -> 0.0, solver -> IRLS)
+
   @Since("2.0.0")
   override def validateAndTransformSchema(
       schema: StructType,
@@ -256,7 +260,6 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val
    */
   @Since("2.0.0")
   def setFamily(value: String): this.type = set(family, value)
-  setDefault(family -> Gaussian.name)
 
   /**
    * Sets the value of param [[variancePower]].
@@ -267,7 +270,6 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val
    */
   @Since("2.2.0")
   def setVariancePower(value: Double): this.type = set(variancePower, value)
-  setDefault(variancePower -> 0.0)
 
   /**
    * Sets the value of param [[linkPower]].
@@ -304,7 +306,6 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val
    */
   @Since("2.0.0")
   def setMaxIter(value: Int): this.type = set(maxIter, value)
-  setDefault(maxIter -> 25)
 
   /**
    * Sets the convergence tolerance of iterations.
@@ -315,7 +316,6 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val
    */
   @Since("2.0.0")
   def setTol(value: Double): this.type = set(tol, value)
-  setDefault(tol -> 1E-6)
 
   /**
    * Sets the regularization parameter for L2 regularization.
@@ -331,7 +331,6 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val
    */
   @Since("2.0.0")
   def setRegParam(value: Double): this.type = set(regParam, value)
-  setDefault(regParam -> 0.0)
 
   /**
    * Sets the value of param [[weightCol]].
@@ -363,7 +362,6 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val
    */
   @Since("2.0.0")
   def setSolver(value: String): this.type = set(solver, value)
-  setDefault(solver -> IRLS)
 
   /**
    * Sets the link prediction (linear predictor) column name.
@@ -399,7 +397,7 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val
       "GeneralizedLinearRegression was given data with 0 features, and with Param fitIntercept " +
         "set to false. To fit a model with 0 features, fitIntercept must be set to true." )
 
-    val w = if (!hasWeightCol) lit(1.0) else col($(weightCol))
+    val w = if (!hasWeightCol) lit(1.0) else checkNonNegativeWeight(col($(weightCol)))
     val offset = if (!hasOffsetCol) lit(0.0) else col($(offsetCol)).cast(DoubleType)
 
     val model = if (familyAndLink.family == Gaussian && familyAndLink.link == Identity) {
@@ -1219,10 +1217,41 @@ class GeneralizedLinearRegressionSummary private[regression] (
 
   private[regression] lazy val link: Link = familyLink.link
 
+  /**
+   * summary row containing:
+   *  numInstances, weightSum, deviance, rss, weighted average of label - offset.
+   */
+  private lazy val glrSummary = {
+    val devUDF = udf { (label: Double, pred: Double, weight: Double) =>
+      family.deviance(label, pred, weight)
+    }
+    val devCol = sum(devUDF(label, prediction, weight))
+
+    val rssCol = if (model.getFamily.toLowerCase(Locale.ROOT) != Binomial.name &&
+      model.getFamily.toLowerCase(Locale.ROOT) != Poisson.name) {
+      val rssUDF = udf { (label: Double, pred: Double, weight: Double) =>
+        (label - pred) * (label - pred) * weight / family.variance(pred)
+      }
+      sum(rssUDF(label, prediction, weight))
+    } else {
+      lit(Double.NaN)
+    }
+
+    val avgCol = if (model.getFitIntercept &&
+      (!model.hasOffsetCol || (model.hasOffsetCol && family == Gaussian && link == Identity))) {
+      sum((label - offset) * weight) / sum(weight)
+    } else {
+      lit(Double.NaN)
+    }
+
+    predictions
+      .select(count(label), sum(weight), devCol, rssCol, avgCol)
+      .head()
+  }
+
   /** Number of instances in DataFrame predictions. */
   @Since("2.2.0")
-  lazy val numInstances: Long = predictions.count()
-
+  lazy val numInstances: Long = glrSummary.getLong(0)
 
   /**
    * Name of features. If the name cannot be retrieved from attributes,
@@ -1334,9 +1363,7 @@ class GeneralizedLinearRegressionSummary private[regression] (
        */
       if (!model.hasOffsetCol ||
         (model.hasOffsetCol && family == Gaussian && link == Identity)) {
-        val agg = predictions.agg(sum(weight.multiply(
-          label.minus(offset))), sum(weight)).first()
-        link.link(agg.getDouble(0) / agg.getDouble(1))
+        link.link(glrSummary.getDouble(4))
       } else {
         // Create empty feature column and fit intercept only model using param setting from model
         val featureNull = "feature_" + java.util.UUID.randomUUID.toString
@@ -1361,12 +1388,7 @@ class GeneralizedLinearRegressionSummary private[regression] (
    * The deviance for the fitted model.
    */
   @Since("2.0.0")
-  lazy val deviance: Double = {
-    predictions.select(label, prediction, weight).rdd.map {
-      case Row(label: Double, pred: Double, weight: Double) =>
-        family.deviance(label, pred, weight)
-    }.sum()
-  }
+  lazy val deviance: Double = glrSummary.getDouble(2)
 
   /**
    * The dispersion of the fitted model.
@@ -1380,14 +1402,14 @@ class GeneralizedLinearRegressionSummary private[regression] (
       model.getFamily.toLowerCase(Locale.ROOT) == Poisson.name) {
     1.0
   } else {
-    val rss = pearsonResiduals.agg(sum(pow(col("pearsonResiduals"), 2.0))).first().getDouble(0)
+    val rss = glrSummary.getDouble(3)
     rss / degreesOfFreedom
   }
 
   /** Akaike Information Criterion (AIC) for the fitted model. */
   @Since("2.0.0")
   lazy val aic: Double = {
-    val weightSum = predictions.select(weight).agg(sum(weight)).first().getDouble(0)
+    val weightSum = glrSummary.getDouble(1)
     val t = predictions.select(
       label, prediction, weight).rdd.map {
         case Row(label: Double, pred: Double, weight: Double) =>
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/IsotonicRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/IsotonicRegression.scala
index fe4de57de60f2..ec2640e9ef225 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/IsotonicRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/IsotonicRegression.scala
@@ -22,6 +22,7 @@ import org.apache.hadoop.fs.Path
 import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.{Estimator, Model}
+import org.apache.spark.ml.functions.checkNonNegativeWeight
 import org.apache.spark.ml.linalg.{Vector, Vectors, VectorUDT}
 import org.apache.spark.ml.param._
 import org.apache.spark.ml.param.shared._
@@ -87,11 +88,11 @@ private[regression] trait IsotonicRegressionBase extends Params with HasFeatures
     } else {
       col($(featuresCol))
     }
-    val w = if (hasWeightCol) col($(weightCol)).cast(DoubleType) else lit(1.0)
+    val w =
+      if (hasWeightCol) checkNonNegativeWeight(col($(weightCol)).cast(DoubleType)) else lit(1.0)
 
     dataset.select(col($(labelCol)).cast(DoubleType), f, w).rdd.map {
-      case Row(label: Double, feature: Double, weight: Double) =>
-        (label, feature, weight)
+      case Row(label: Double, feature: Double, weight: Double) => (label, feature, weight)
     }
   }
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala
index bcf9b7c0426cd..11a1984b0ab4c 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala
@@ -56,7 +56,7 @@ import org.apache.spark.util.VersionUtils.majorMinorVersion
 private[regression] trait LinearRegressionParams extends PredictorParams
     with HasRegParam with HasElasticNetParam with HasMaxIter with HasTol
     with HasFitIntercept with HasStandardization with HasWeightCol with HasSolver
-    with HasAggregationDepth with HasLoss with HasBlockSize {
+    with HasAggregationDepth with HasLoss with HasMaxBlockSizeInMB {
 
   import LinearRegression._
 
@@ -105,6 +105,10 @@ private[regression] trait LinearRegressionParams extends PredictorParams
   @Since("2.3.0")
   def getEpsilon: Double = $(epsilon)
 
+  setDefault(regParam -> 0.0, fitIntercept -> true, standardization -> true,
+    elasticNetParam -> 0.0, maxIter -> 100, tol -> 1E-6, solver -> Auto,
+    aggregationDepth -> 2, loss -> SquaredError, epsilon -> 1.35, maxBlockSizeInMB -> 0.0)
+
   override protected def validateAndTransformSchema(
       schema: StructType,
       fitting: Boolean,
@@ -171,6 +175,10 @@ private[regression] trait LinearRegressionParams extends PredictorParams
  *   $$
  * </blockquote>
  *
+ * Since 3.1.0, it supports stacking instances into blocks and using GEMV for
+ * better performance.
+ * The block size will be 1.0 MB, if param maxBlockSizeInMB is set 0.0 by default.
+ *
  * Note: Fitting with huber loss only supports none and L2 regularization.
  */
 @Since("1.3.0")
@@ -191,7 +199,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("1.3.0")
   def setRegParam(value: Double): this.type = set(regParam, value)
-  setDefault(regParam -> 0.0)
 
   /**
    * Set if we should fit the intercept.
@@ -201,7 +208,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("1.5.0")
   def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
-  setDefault(fitIntercept -> true)
 
   /**
    * Whether to standardize the training features before fitting the model.
@@ -217,7 +223,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("1.5.0")
   def setStandardization(value: Boolean): this.type = set(standardization, value)
-  setDefault(standardization -> true)
 
   /**
    * Set the ElasticNet mixing parameter.
@@ -233,7 +238,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("1.4.0")
   def setElasticNetParam(value: Double): this.type = set(elasticNetParam, value)
-  setDefault(elasticNetParam -> 0.0)
 
   /**
    * Set the maximum number of iterations.
@@ -243,7 +247,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("1.3.0")
   def setMaxIter(value: Int): this.type = set(maxIter, value)
-  setDefault(maxIter -> 100)
 
   /**
    * Set the convergence tolerance of iterations.
@@ -254,7 +257,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("1.4.0")
   def setTol(value: Double): this.type = set(tol, value)
-  setDefault(tol -> 1E-6)
 
   /**
    * Whether to over-/under-sample training instances according to the given weights in weightCol.
@@ -283,7 +285,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("1.6.0")
   def setSolver(value: String): this.type = set(solver, value)
-  setDefault(solver -> Auto)
 
   /**
    * Suggested depth for treeAggregate (greater than or equal to 2).
@@ -295,7 +296,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("2.1.0")
   def setAggregationDepth(value: Int): this.type = set(aggregationDepth, value)
-  setDefault(aggregationDepth -> 2)
 
   /**
    * Sets the value of param [[loss]].
@@ -305,7 +305,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("2.3.0")
   def setLoss(value: String): this.type = set(loss, value)
-  setDefault(loss -> SquaredError)
 
   /**
    * Sets the value of param [[epsilon]].
@@ -315,33 +314,28 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("2.3.0")
   def setEpsilon(value: Double): this.type = set(epsilon, value)
-  setDefault(epsilon -> 1.35)
 
   /**
-   * Set block size for stacking input data in matrices.
-   * If blockSize == 1, then stacking will be skipped, and each vector is treated individually;
-   * If blockSize &gt; 1, then vectors will be stacked to blocks, and high-level BLAS routines
-   * will be used if possible (for example, GEMV instead of DOT, GEMM instead of GEMV).
-   * Recommended size is between 10 and 1000. An appropriate choice of the block size depends
-   * on the sparsity and dim of input datasets, the underlying BLAS implementation (for example,
-   * f2jBLAS, OpenBLAS, intel MKL) and its configuration (for example, number of threads).
-   * Note that existing BLAS implementations are mainly optimized for dense matrices, if the
-   * input dataset is sparse, stacking may bring no performance gain, the worse is possible
-   * performance regression.
-   * Default is 1.
+   * Sets the value of param [[maxBlockSizeInMB]].
+   * Default is 0.0, then 1.0 MB will be chosen.
    *
    * @group expertSetParam
    */
   @Since("3.1.0")
-  def setBlockSize(value: Int): this.type = set(blockSize, value)
-  setDefault(blockSize -> 1)
+  def setMaxBlockSizeInMB(value: Double): this.type = set(maxBlockSizeInMB, value)
 
-  override protected def train(dataset: Dataset[_]): LinearRegressionModel = instrumented { instr =>
+  override protected def train(
+      dataset: Dataset[_]): LinearRegressionModel = instrumented { instr =>
     instr.logPipelineStage(this)
     instr.logDataset(dataset)
     instr.logParams(this, labelCol, featuresCol, weightCol, predictionCol, solver, tol,
       elasticNetParam, fitIntercept, maxIter, regParam, standardization, aggregationDepth, loss,
-      epsilon, blockSize)
+      epsilon, maxBlockSizeInMB)
+
+    if (dataset.storageLevel != StorageLevel.NONE) {
+      instr.logWarning(s"Input instances will be standardized, blockified to blocks, and " +
+        s"then cached during training. Be careful of double caching!")
+    }
 
     // Extract the number of features before deciding optimization solver.
     val numFeatures = MetadataUtils.getNumFeatures(dataset, $(featuresCol))
@@ -355,35 +349,26 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
     val instances = extractInstances(dataset)
       .setName("training instances")
 
-    if (dataset.storageLevel == StorageLevel.NONE && $(blockSize) == 1) {
-      instances.persist(StorageLevel.MEMORY_AND_DISK)
-    }
+    val (summarizer, labelSummarizer) = Summarizer
+      .getRegressionSummarizers(instances, $(aggregationDepth), Seq("mean", "std", "count"))
 
-    var requestedMetrics = Seq("mean", "std", "count")
-    if ($(blockSize) != 1) requestedMetrics +:= "numNonZeros"
-    val (featuresSummarizer, ySummarizer) = Summarizer
-      .getRegressionSummarizers(instances, $(aggregationDepth), requestedMetrics)
+    val yMean = labelSummarizer.mean(0)
+    val rawYStd = labelSummarizer.std(0)
 
-    val yMean = ySummarizer.mean(0)
-    val rawYStd = ySummarizer.std(0)
-
-    instr.logNumExamples(ySummarizer.count)
+    instr.logNumExamples(labelSummarizer.count)
     instr.logNamedValue(Instrumentation.loggerTags.meanOfLabels, yMean)
     instr.logNamedValue(Instrumentation.loggerTags.varianceOfLabels, rawYStd)
-    instr.logSumOfWeights(featuresSummarizer.weightSum)
-    if ($(blockSize) > 1) {
-      val scale = 1.0 / featuresSummarizer.count / numFeatures
-      val sparsity = 1 - featuresSummarizer.numNonzeros.toArray.map(_ * scale).sum
-      instr.logNamedValue("sparsity", sparsity.toString)
-      if (sparsity > 0.5) {
-        instr.logWarning(s"sparsity of input dataset is $sparsity, " +
-          s"which may hurt performance in high-level BLAS.")
-      }
+    instr.logSumOfWeights(summarizer.weightSum)
+
+    var actualBlockSizeInMB = $(maxBlockSizeInMB)
+    if (actualBlockSizeInMB == 0) {
+      actualBlockSizeInMB = InstanceBlock.DefaultBlockSizeInMB
+      require(actualBlockSizeInMB > 0, "inferred actual BlockSizeInMB must > 0")
+      instr.logNamedValue("actualBlockSizeInMB", actualBlockSizeInMB.toString)
     }
 
     if (rawYStd == 0.0) {
       if ($(fitIntercept) || yMean == 0.0) {
-        if (instances.getStorageLevel != StorageLevel.NONE) instances.unpersist()
         return trainWithConstantLabel(dataset, instr, numFeatures, yMean)
       } else {
         require($(regParam) == 0.0, "The standard deviation of the label is zero. " +
@@ -396,8 +381,8 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
     // if y is constant (rawYStd is zero), then y cannot be scaled. In this case
     // setting yStd=abs(yMean) ensures that y is not scaled anymore in l-bfgs algorithm.
     val yStd = if (rawYStd > 0) rawYStd else math.abs(yMean)
-    val featuresMean = featuresSummarizer.mean.toArray
-    val featuresStd = featuresSummarizer.std.toArray
+    val featuresMean = summarizer.mean.toArray
+    val featuresStd = summarizer.std.toArray
 
     if (!$(fitIntercept) && (0 until numFeatures).exists { i =>
       featuresStd(i) == 0.0 && featuresMean(i) != 0.0 }) {
@@ -433,14 +418,9 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
         Vectors.dense(Array.fill(dim)(1.0))
     }
 
-    val  (parameters, objectiveHistory) = if ($(blockSize) == 1) {
-      trainOnRows(instances, yMean, yStd, featuresMean, featuresStd,
-        initialValues, regularization, optimizer)
-    } else {
-      trainOnBlocks(instances, yMean, yStd, featuresMean, featuresStd,
-        initialValues, regularization, optimizer)
-    }
-    if (instances.getStorageLevel != StorageLevel.NONE) instances.unpersist()
+    val (parameters, objectiveHistory) =
+      trainImpl(instances, actualBlockSizeInMB, yMean, yStd,
+        featuresMean, featuresStd, initialValues, regularization, optimizer)
 
     if (parameters == null) {
       val msg = s"${optimizer.getClass.getName} failed."
@@ -548,56 +528,9 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
     }
   }
 
-  private def trainOnRows(
-      instances: RDD[Instance],
-      yMean: Double,
-      yStd: Double,
-      featuresMean: Array[Double],
-      featuresStd: Array[Double],
-      initialValues: Vector,
-      regularization: Option[L2Regularization],
-      optimizer: FirstOrderMinimizer[BDV[Double], DiffFunction[BDV[Double]]]) = {
-    val bcFeaturesMean = instances.context.broadcast(featuresMean)
-    val bcFeaturesStd = instances.context.broadcast(featuresStd)
-
-    val costFun = $(loss) match {
-      case SquaredError =>
-        val getAggregatorFunc = new LeastSquaresAggregator(yStd, yMean, $(fitIntercept),
-          bcFeaturesStd, bcFeaturesMean)(_)
-        new RDDLossFunction(instances, getAggregatorFunc, regularization, $(aggregationDepth))
-      case Huber =>
-        val getAggregatorFunc = new HuberAggregator($(fitIntercept), $(epsilon), bcFeaturesStd)(_)
-        new RDDLossFunction(instances, getAggregatorFunc, regularization, $(aggregationDepth))
-    }
-
-    val states = optimizer.iterations(new CachedDiffFunction(costFun),
-      initialValues.asBreeze.toDenseVector)
-
-    /*
-       Note that in Linear Regression, the objective history (loss + regularization) returned
-       from optimizer is computed in the scaled space given by the following formula.
-       <blockquote>
-          $$
-          L &= 1/2n||\sum_i w_i(x_i - \bar{x_i}) / \hat{x_i} - (y - \bar{y}) / \hat{y}||^2
-               + regTerms \\
-          $$
-       </blockquote>
-     */
-    val arrayBuilder = mutable.ArrayBuilder.make[Double]
-    var state: optimizer.State = null
-    while (states.hasNext) {
-      state = states.next()
-      arrayBuilder += state.adjustedValue
-    }
-
-    bcFeaturesMean.destroy()
-    bcFeaturesStd.destroy()
-
-    (if (state == null) null else state.x.toArray, arrayBuilder.result)
-  }
-
-  private def trainOnBlocks(
+  private def trainImpl(
       instances: RDD[Instance],
+      actualBlockSizeInMB: Double,
       yMean: Double,
       yStd: Double,
       featuresMean: Array[Double],
@@ -613,9 +546,11 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
       val func = StandardScalerModel.getTransformFunc(Array.empty, inverseStd, false, true)
       iter.map { case Instance(label, weight, vec) => Instance(label, weight, func(vec)) }
     }
-    val blocks = InstanceBlock.blokify(standardized, $(blockSize))
+
+    val maxMemUsage = (actualBlockSizeInMB * 1024L * 1024L).ceil.toLong
+    val blocks = InstanceBlock.blokifyWithMaxMemUsage(standardized, maxMemUsage)
       .persist(StorageLevel.MEMORY_AND_DISK)
-      .setName(s"training blocks (blockSize=${$(blockSize)})")
+      .setName(s"training blocks (blockSizeInMB=$actualBlockSizeInMB)")
 
     val costFun = $(loss) match {
       case SquaredError =>
@@ -939,8 +874,10 @@ class LinearRegressionTrainingSummary private[regression] (
    * @see `LinearRegression.solver`
    */
   @Since("1.5.0")
-  val totalIterations = objectiveHistory.length
-
+  val totalIterations = {
+    assert(objectiveHistory.length > 0, s"objectiveHistory length should be greater than 1.")
+    objectiveHistory.length - 1
+  }
 }
 
 /**
@@ -961,21 +898,27 @@ class LinearRegressionSummary private[regression] (
     private val privateModel: LinearRegressionModel,
     private val diagInvAtWA: Array[Double]) extends Serializable {
 
-  @transient private val metrics = new RegressionMetrics(
-    predictions
-      .select(col(predictionCol), col(labelCol).cast(DoubleType))
-      .rdd
-      .map { case Row(pred: Double, label: Double) => (pred, label) },
-    !privateModel.getFitIntercept)
+  @transient private val metrics = {
+    val weightCol =
+      if (!privateModel.isDefined(privateModel.weightCol) || privateModel.getWeightCol.isEmpty) {
+        lit(1.0)
+      } else {
+        col(privateModel.getWeightCol).cast(DoubleType)
+      }
+
+    new RegressionMetrics(
+      predictions
+        .select(col(predictionCol), col(labelCol).cast(DoubleType), weightCol)
+        .rdd
+        .map { case Row(pred: Double, label: Double, weight: Double) => (pred, label, weight) },
+      !privateModel.getFitIntercept)
+  }
 
   /**
    * Returns the explained variance regression score.
    * explainedVariance = 1 - variance(y - \hat{y}) / variance(y)
    * Reference: <a href="http://en.wikipedia.org/wiki/Explained_variation">
    * Wikipedia explain variation</a>
-   *
-   * @note This ignores instance weights (setting all to 1.0) from `LinearRegression.weightCol`.
-   * This will change in later Spark versions.
    */
   @Since("1.5.0")
   val explainedVariance: Double = metrics.explainedVariance
@@ -983,9 +926,6 @@ class LinearRegressionSummary private[regression] (
   /**
    * Returns the mean absolute error, which is a risk function corresponding to the
    * expected value of the absolute error loss or l1-norm loss.
-   *
-   * @note This ignores instance weights (setting all to 1.0) from `LinearRegression.weightCol`.
-   * This will change in later Spark versions.
    */
   @Since("1.5.0")
   val meanAbsoluteError: Double = metrics.meanAbsoluteError
@@ -993,9 +933,6 @@ class LinearRegressionSummary private[regression] (
   /**
    * Returns the mean squared error, which is a risk function corresponding to the
    * expected value of the squared error loss or quadratic loss.
-   *
-   * @note This ignores instance weights (setting all to 1.0) from `LinearRegression.weightCol`.
-   * This will change in later Spark versions.
    */
   @Since("1.5.0")
   val meanSquaredError: Double = metrics.meanSquaredError
@@ -1003,9 +940,6 @@ class LinearRegressionSummary private[regression] (
   /**
    * Returns the root mean squared error, which is defined as the square root of
    * the mean squared error.
-   *
-   * @note This ignores instance weights (setting all to 1.0) from `LinearRegression.weightCol`.
-   * This will change in later Spark versions.
    */
   @Since("1.5.0")
   val rootMeanSquaredError: Double = metrics.rootMeanSquaredError
@@ -1014,9 +948,6 @@ class LinearRegressionSummary private[regression] (
    * Returns R^2^, the coefficient of determination.
    * Reference: <a href="http://en.wikipedia.org/wiki/Coefficient_of_determination">
    * Wikipedia coefficient of determination</a>
-   *
-   * @note This ignores instance weights (setting all to 1.0) from `LinearRegression.weightCol`.
-   * This will change in later Spark versions.
    */
   @Since("1.5.0")
   val r2: Double = metrics.r2
@@ -1025,9 +956,6 @@ class LinearRegressionSummary private[regression] (
    * Returns Adjusted R^2^, the adjusted coefficient of determination.
    * Reference: <a href="https://en.wikipedia.org/wiki/Coefficient_of_determination#Adjusted_R2">
    * Wikipedia coefficient of determination</a>
-   *
-   * @note This ignores instance weights (setting all to 1.0) from `LinearRegression.weightCol`.
-   * This will change in later Spark versions.
    */
   @Since("2.3.0")
   val r2adj: Double = {
@@ -1044,7 +972,7 @@ class LinearRegressionSummary private[regression] (
   }
 
   /** Number of instances in DataFrame predictions */
-  lazy val numInstances: Long = predictions.count()
+  lazy val numInstances: Long = metrics.count
 
   /** Degrees of freedom */
   @Since("2.2.0")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala b/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala
index da8f3a24ff27e..837883e53d306 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala
@@ -40,7 +40,7 @@ import org.apache.spark.sql.types._
 import org.apache.spark.util.SerializableConfiguration
 
 private[libsvm] class LibSVMOutputWriter(
-    path: String,
+    val path: String,
     dataSchema: StructType,
     context: TaskAttemptContext)
   extends OutputWriter {
@@ -100,8 +100,8 @@ private[libsvm] class LibSVMFileFormat
         "though the input. If you know the number in advance, please specify it via " +
         "'numFeatures' option to avoid the extra scan.")
 
-      val paths = files.map(_.getPath.toUri.toString)
-      val parsed = MLUtils.parseLibSVMFile(sparkSession, paths)
+      val paths = files.map(_.getPath.toString)
+      val parsed = MLUtils.parseLibSVMFile(sparkSession, paths, options)
       MLUtils.computeNumFeatures(parsed)
     }
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/stat/ANOVATest.scala b/mllib/src/main/scala/org/apache/spark/ml/stat/ANOVATest.scala
index f14f63b674f8e..7a7e76c457dc6 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/stat/ANOVATest.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/stat/ANOVATest.scala
@@ -35,7 +35,7 @@ import org.apache.spark.util.collection.OpenHashMap
  * information on ANOVA test.
  */
 @Since("3.1.0")
-object ANOVATest {
+private[ml] object ANOVATest {
 
   /**
    * @param dataset  DataFrame of categorical labels and continuous features.
diff --git a/mllib/src/main/scala/org/apache/spark/ml/stat/FValueTest.scala b/mllib/src/main/scala/org/apache/spark/ml/stat/FValueTest.scala
index ad506ab337720..f315e92e86d3d 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/stat/FValueTest.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/stat/FValueTest.scala
@@ -30,7 +30,7 @@ import org.apache.spark.sql.functions._
  * FValue test for continuous data.
  */
 @Since("3.1.0")
-object FValueTest {
+private[ml] object FValueTest {
 
   /** Used to construct output schema of tests */
   private  case class FValueResult(
diff --git a/mllib/src/main/scala/org/apache/spark/ml/stat/Summarizer.scala b/mllib/src/main/scala/org/apache/spark/ml/stat/Summarizer.scala
index 4db518bd4f9ba..a3dd133a4ce8d 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/stat/Summarizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/stat/Summarizer.scala
@@ -28,6 +28,7 @@ import org.apache.spark.sql.Column
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{Expression, ImplicitCastInputTypes}
 import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Complete, TypedImperativeAggregate}
+import org.apache.spark.sql.catalyst.trees.BinaryLike
 import org.apache.spark.sql.functions.lit
 import org.apache.spark.sql.types._
 
@@ -348,7 +349,9 @@ private[spark] object SummaryBuilderImpl extends Logging {
       weightExpr: Expression,
       mutableAggBufferOffset: Int,
       inputAggBufferOffset: Int)
-    extends TypedImperativeAggregate[SummarizerBuffer] with ImplicitCastInputTypes {
+    extends TypedImperativeAggregate[SummarizerBuffer]
+    with ImplicitCastInputTypes
+    with BinaryLike[Expression] {
 
     override def eval(state: SummarizerBuffer): Any = {
       val metrics = requestedMetrics.map {
@@ -368,7 +371,12 @@ private[spark] object SummaryBuilderImpl extends Logging {
 
     override def inputTypes: Seq[DataType] = vectorUDT :: DoubleType :: Nil
 
-    override def children: Seq[Expression] = featuresExpr :: weightExpr :: Nil
+    override def left: Expression = featuresExpr
+    override def right: Expression = weightExpr
+
+    override protected def withNewChildrenInternal(
+        newLeft: Expression, newRight: Expression): MetricsAggregate =
+      copy(featuresExpr = newLeft, weightExpr = newRight)
 
     override def update(state: SummarizerBuffer, row: InternalRow): SummarizerBuffer = {
       val features = vectorUDT.deserialize(featuresExpr.eval(row))
@@ -435,7 +443,7 @@ private[spark] class SummarizerBuffer(
   private var currMax: Array[Double] = null
   private var currMin: Array[Double] = null
 
-  def this() {
+  def this() = {
     this(
       Seq(
         SummaryBuilderImpl.Mean,
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/Node.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/Node.scala
index cc21e9cbbe8de..dfa4961d9ffb8 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/Node.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/Node.scala
@@ -174,22 +174,32 @@ class InternalNode private[ml] (
   }
 
   override private[ml] def predictImpl(features: Vector): LeafNode = {
-    if (split.shouldGoLeft(features)) {
-      leftChild.predictImpl(features)
-    } else {
-      rightChild.predictImpl(features)
+    var node: Node = this
+    while (node.isInstanceOf[InternalNode]) {
+      val n = node.asInstanceOf[InternalNode]
+      if (n.split.shouldGoLeft(features)) {
+        node = n.leftChild
+      } else {
+        node = n.rightChild
+      }
     }
+    node.asInstanceOf[LeafNode]
   }
 
   override private[ml] def predictBinned(
       binned: Array[Int],
       splits: Array[Array[Split]]): LeafNode = {
-    val i = split.featureIndex
-    if (split.shouldGoLeft(binned(i), splits(i))) {
-      leftChild.predictBinned(binned, splits)
-    } else {
-      rightChild.predictBinned(binned, splits)
+    var node: Node = this
+    while (node.isInstanceOf[InternalNode]) {
+      val n = node.asInstanceOf[InternalNode]
+      val i = n.split.featureIndex
+      if (n.split.shouldGoLeft(binned(i), splits(i))) {
+        node = n.leftChild
+      } else {
+        node = n.rightChild
+      }
     }
+    node.asInstanceOf[LeafNode]
   }
 
   override private[tree] def numDescendants: Int = {
@@ -326,27 +336,27 @@ private[tree] class LearningNode(
    *         [[org.apache.spark.ml.tree.impl.RandomForest.findBestSplits()]].
    */
   def predictImpl(binnedFeatures: Array[Int], splits: Array[Array[Split]]): Int = {
-    if (this.isLeaf || this.split.isEmpty) {
-      this.id
-    } else {
-      val split = this.split.get
+    var node = this
+    while (!node.isLeaf && node.split.nonEmpty) {
+      val split = node.split.get
       val featureIndex = split.featureIndex
       val splitLeft = split.shouldGoLeft(binnedFeatures(featureIndex), splits(featureIndex))
-      if (this.leftChild.isEmpty) {
+      if (node.leftChild.isEmpty) {
         // Not yet split. Return next layer of nodes to train
         if (splitLeft) {
-          LearningNode.leftChildIndex(this.id)
+          return LearningNode.leftChildIndex(node.id)
         } else {
-          LearningNode.rightChildIndex(this.id)
+          return LearningNode.rightChildIndex(node.id)
         }
       } else {
         if (splitLeft) {
-          this.leftChild.get.predictImpl(binnedFeatures, splits)
+          node = node.leftChild.get
         } else {
-          this.rightChild.get.predictImpl(binnedFeatures, splits)
+          node = node.rightChild.get
         }
       }
     }
+    node.id
   }
 
 }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
index 162641f605264..4858189dea825 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
@@ -31,8 +31,10 @@ import org.apache.spark.ml.util.{DefaultParamsReader, DefaultParamsWriter}
 import org.apache.spark.ml.util.DefaultParamsReader.Metadata
 import org.apache.spark.mllib.tree.impurity.ImpurityCalculator
 import org.apache.spark.mllib.tree.model.{DecisionTreeModel => OldDecisionTreeModel}
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.{Dataset, SparkSession}
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.functions.{col, lit, struct}
+import org.apache.spark.sql.types.StructType
+import org.apache.spark.util.VersionUtils
 import org.apache.spark.util.collection.OpenHashMap
 
 /**
@@ -381,6 +383,15 @@ private[ml] object DecisionTreeModelReadWrite {
           node.impurityStats.rawCount, -1.0, -1, -1, SplitData(-1, Array.emptyDoubleArray, -1))),
           id)
     }
+
+    /**
+     * When save a tree model, infer the number of partitions based on number of nodes.
+     */
+    def inferNumPartitions(numNodes: Long): Int = {
+      require(numNodes > 0)
+      // 7,280,000 nodes is about 128MB
+      (numNodes / 7280000.0).ceil.toInt
+    }
   }
 
   /**
@@ -401,8 +412,13 @@ private[ml] object DecisionTreeModelReadWrite {
     }
 
     val dataPath = new Path(path, "data").toString
-    val data = sparkSession.read.parquet(dataPath).as[NodeData]
-    buildTreeFromNodes(data.collect(), impurityType)
+    var df = sparkSession.read.parquet(dataPath)
+    val (major, minor) = VersionUtils.majorMinorVersion(metadata.sparkVersion)
+    if (major.toInt < 3) {
+      df = df.withColumn("rawCount", lit(-1L))
+    }
+
+    buildTreeFromNodes(df.as[NodeData].collect(), impurityType)
   }
 
   /**
@@ -452,23 +468,27 @@ private[ml] object EnsembleModelReadWrite {
   def saveImpl[M <: Params with TreeEnsembleModel[_ <: DecisionTreeModel]](
       instance: M,
       path: String,
-      sql: SparkSession,
+      sparkSession: SparkSession,
       extraMetadata: JObject): Unit = {
-    DefaultParamsWriter.saveMetadata(instance, path, sql.sparkContext, Some(extraMetadata))
-    val treesMetadataWeights: Array[(Int, String, Double)] = instance.trees.zipWithIndex.map {
-      case (tree, treeID) =>
-        (treeID,
-          DefaultParamsWriter.getMetadataToSave(tree.asInstanceOf[Params], sql.sparkContext),
-          instance.treeWeights(treeID))
+    DefaultParamsWriter.saveMetadata(instance, path, sparkSession.sparkContext, Some(extraMetadata))
+    val treesMetadataWeights = instance.trees.zipWithIndex.map { case (tree, treeID) =>
+      (treeID,
+        DefaultParamsWriter.getMetadataToSave(tree.asInstanceOf[Params], sparkSession.sparkContext),
+        instance.treeWeights(treeID))
     }
     val treesMetadataPath = new Path(path, "treesMetadata").toString
-    sql.createDataFrame(treesMetadataWeights).toDF("treeID", "metadata", "weights")
+    sparkSession.createDataFrame(treesMetadataWeights)
+      .toDF("treeID", "metadata", "weights")
+      .repartition(1)
       .write.parquet(treesMetadataPath)
+
     val dataPath = new Path(path, "data").toString
-    val nodeDataRDD = sql.sparkContext.parallelize(instance.trees.zipWithIndex).flatMap {
-      case (tree, treeID) => EnsembleNodeData.build(tree, treeID)
-    }
-    sql.createDataFrame(nodeDataRDD).write.parquet(dataPath)
+    val numDataParts = NodeData.inferNumPartitions(instance.trees.map(_.numNodes.toLong).sum)
+    val nodeDataRDD = sparkSession.sparkContext.parallelize(instance.trees.zipWithIndex)
+      .flatMap { case (tree, treeID) => EnsembleNodeData.build(tree, treeID) }
+    sparkSession.createDataFrame(nodeDataRDD)
+      .repartition(numDataParts)
+      .write.parquet(dataPath)
   }
 
   /**
@@ -483,12 +503,12 @@ private[ml] object EnsembleModelReadWrite {
    */
   def loadImpl(
       path: String,
-      sql: SparkSession,
+      sparkSession: SparkSession,
       className: String,
       treeClassName: String): (Metadata, Array[(Metadata, Node)], Array[Double]) = {
-    import sql.implicits._
+    import sparkSession.implicits._
     implicit val format = DefaultFormats
-    val metadata = DefaultParamsReader.loadMetadata(path, sql.sparkContext, className)
+    val metadata = DefaultParamsReader.loadMetadata(path, sparkSession.sparkContext, className)
 
     // Get impurity to construct ImpurityCalculator for each node
     val impurityType: String = {
@@ -497,25 +517,36 @@ private[ml] object EnsembleModelReadWrite {
     }
 
     val treesMetadataPath = new Path(path, "treesMetadata").toString
-    val treesMetadataRDD: RDD[(Int, (Metadata, Double))] = sql.read.parquet(treesMetadataPath)
-      .select("treeID", "metadata", "weights").as[(Int, String, Double)].rdd.map {
-      case (treeID: Int, json: String, weights: Double) =>
+    val treesMetadataRDD = sparkSession.read.parquet(treesMetadataPath)
+      .select("treeID", "metadata", "weights")
+      .as[(Int, String, Double)].rdd
+      .map { case (treeID: Int, json: String, weights: Double) =>
         treeID -> ((DefaultParamsReader.parseMetadata(json, treeClassName), weights))
-    }
+      }
 
     val treesMetadataWeights = treesMetadataRDD.sortByKey().values.collect()
     val treesMetadata = treesMetadataWeights.map(_._1)
     val treesWeights = treesMetadataWeights.map(_._2)
 
     val dataPath = new Path(path, "data").toString
-    val nodeData: Dataset[EnsembleNodeData] =
-      sql.read.parquet(dataPath).as[EnsembleNodeData]
-    val rootNodesRDD: RDD[(Int, Node)] =
-      nodeData.rdd.map(d => (d.treeID, d.nodeData)).groupByKey().map {
-        case (treeID: Int, nodeData: Iterable[NodeData]) =>
-          treeID -> DecisionTreeModelReadWrite.buildTreeFromNodes(nodeData.toArray, impurityType)
+    var df = sparkSession.read.parquet(dataPath)
+    val (major, minor) = VersionUtils.majorMinorVersion(metadata.sparkVersion)
+    if (major.toInt < 3) {
+      val newNodeDataCol = df.schema("nodeData").dataType match {
+        case StructType(fields) =>
+          val cols = fields.map(f => col(s"nodeData.${f.name}")) :+ lit(-1L).as("rawCount")
+          struct(cols: _*)
+      }
+      df = df.withColumn("nodeData", newNodeDataCol)
+    }
+
+    val rootNodesRDD = df.as[EnsembleNodeData].rdd
+      .map(d => (d.treeID, d.nodeData))
+      .groupByKey()
+      .map { case (treeID: Int, nodeData: Iterable[NodeData]) =>
+        treeID -> DecisionTreeModelReadWrite.buildTreeFromNodes(nodeData.toArray, impurityType)
       }
-    val rootNodes: Array[Node] = rootNodesRDD.sortByKey().values.collect()
+    val rootNodes = rootNodesRDD.sortByKey().values.collect()
     (metadata, treesMetadata.zip(rootNodes), treesWeights)
   }
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala
index 7e2c287f146fb..768e14f4b74e4 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala
@@ -60,8 +60,9 @@ private[ml] trait DecisionTreeParams extends PredictorParams
    */
   final val maxDepth: IntParam =
     new IntParam(this, "maxDepth", "Maximum depth of the tree. (Nonnegative)" +
-      " E.g., depth 0 means 1 leaf node; depth 1 means 1 internal node + 2 leaf nodes.",
-      ParamValidators.gtEq(0))
+      " E.g., depth 0 means 1 leaf node; depth 1 means 1 internal node + 2 leaf nodes." +
+      " Must be in range [0, 30].",
+      ParamValidators.inRange(0, 30))
 
   /**
    * Maximum number of bins used for discretizing continuous features and for choosing how to split
@@ -333,8 +334,6 @@ private[ml] trait TreeEnsembleParams extends DecisionTreeParams {
     "Fraction of the training data used for learning each decision tree, in range (0, 1].",
     ParamValidators.inRange(0, 1, lowerInclusive = false, upperInclusive = true))
 
-  setDefault(subsamplingRate -> 1.0)
-
   /** @group getParam */
   final def getSubsamplingRate: Double = $(subsamplingRate)
 
@@ -386,10 +385,10 @@ private[ml] trait TreeEnsembleParams extends DecisionTreeParams {
       || Try(value.toInt).filter(_ > 0).isSuccess
       || Try(value.toDouble).filter(_ > 0).filter(_ <= 1.0).isSuccess)
 
-  setDefault(featureSubsetStrategy -> "auto")
-
   /** @group getParam */
   final def getFeatureSubsetStrategy: String = $(featureSubsetStrategy).toLowerCase(Locale.ROOT)
+
+  setDefault(subsamplingRate -> 1.0, featureSubsetStrategy -> "auto")
 }
 
 /**
@@ -448,8 +447,6 @@ private[ml] trait RandomForestParams extends TreeEnsembleParams {
     new IntParam(this, "numTrees", "Number of trees to train (at least 1)",
     ParamValidators.gtEq(1))
 
-  setDefault(numTrees -> 20)
-
   /** @group getParam */
   final def getNumTrees: Int = $(numTrees)
 
@@ -461,11 +458,11 @@ private[ml] trait RandomForestParams extends TreeEnsembleParams {
   final val bootstrap: BooleanParam = new BooleanParam(this, "bootstrap",
     "Whether bootstrap samples are used when building trees.")
 
-  setDefault(bootstrap -> true)
-
   /** @group getParam */
   @Since("3.0.0")
   final def getBootstrap: Boolean = $(bootstrap)
+
+  setDefault(numTrees -> 20, bootstrap -> true)
 }
 
 private[ml] trait RandomForestClassifierParams
@@ -518,9 +515,7 @@ private[ml] trait GBTParams extends TreeEnsembleParams with HasMaxIter with HasS
     "(a.k.a. learning rate) in interval (0, 1] for shrinking the contribution of each estimator.",
     ParamValidators.inRange(0, 1, lowerInclusive = false, upperInclusive = true))
 
-  setDefault(maxIter -> 20, stepSize -> 0.1, validationTol -> 0.01)
-
-  setDefault(featureSubsetStrategy -> "all")
+  setDefault(maxIter -> 20, stepSize -> 0.1, validationTol -> 0.01, featureSubsetStrategy -> "all")
 
   /** (private[ml]) Create a BoostingStrategy instance to use with the old API. */
   private[ml] def getOldBoostingStrategy(
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala b/mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala
index 858cc57982a3f..275d3c5510f7d 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala
@@ -30,13 +30,13 @@ import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.{Estimator, Model}
 import org.apache.spark.ml.evaluation.Evaluator
-import org.apache.spark.ml.param.{IntParam, ParamMap, ParamValidators}
+import org.apache.spark.ml.param.{IntParam, Param, ParamMap, ParamValidators}
 import org.apache.spark.ml.param.shared.{HasCollectSubModels, HasParallelism}
 import org.apache.spark.ml.util._
 import org.apache.spark.ml.util.Instrumentation.instrumented
 import org.apache.spark.mllib.util.MLUtils
 import org.apache.spark.sql.{DataFrame, Dataset}
-import org.apache.spark.sql.types.StructType
+import org.apache.spark.sql.types.{IntegerType, StructType}
 import org.apache.spark.util.ThreadUtils
 
 /**
@@ -55,7 +55,18 @@ private[ml] trait CrossValidatorParams extends ValidatorParams {
   /** @group getParam */
   def getNumFolds: Int = $(numFolds)
 
-  setDefault(numFolds -> 3)
+  /**
+   * Param for the column name of user specified fold number. Once this is specified,
+   * `CrossValidator` won't do random k-fold split. Note that this column should be
+   * integer type with range [0, numFolds) and Spark will throw exception on out-of-range
+   * fold numbers.
+   */
+  val foldCol: Param[String] = new Param[String](this, "foldCol",
+    "the column name of user specified fold number")
+
+  def getFoldCol: String = $(foldCol)
+
+  setDefault(foldCol -> "", numFolds -> 3)
 }
 
 /**
@@ -94,6 +105,10 @@ class CrossValidator @Since("1.2.0") (@Since("1.4.0") override val uid: String)
   @Since("2.0.0")
   def setSeed(value: Long): this.type = set(seed, value)
 
+  /** @group setParam */
+  @Since("3.1.0")
+  def setFoldCol(value: String): this.type = set(foldCol, value)
+
   /**
    * Set the maximum level of parallelism to evaluate models in parallel.
    * Default is 1 for serial evaluation
@@ -132,7 +147,7 @@ class CrossValidator @Since("1.2.0") (@Since("1.4.0") override val uid: String)
 
     instr.logPipelineStage(this)
     instr.logDataset(dataset)
-    instr.logParams(this, numFolds, seed, parallelism)
+    instr.logParams(this, numFolds, seed, parallelism, foldCol)
     logTuningParams(instr)
 
     val collectSubModelsParam = $(collectSubModels)
@@ -142,10 +157,15 @@ class CrossValidator @Since("1.2.0") (@Since("1.4.0") override val uid: String)
     } else None
 
     // Compute metrics for each model over each split
-    val splits = MLUtils.kFold(dataset.toDF.rdd, $(numFolds), $(seed))
+    val (splits, schemaWithoutFold) = if ($(foldCol) == "") {
+      (MLUtils.kFold(dataset.toDF.rdd, $(numFolds), $(seed)), schema)
+    } else {
+      val filteredSchema = StructType(schema.filter(_.name != $(foldCol)).toArray)
+      (MLUtils.kFold(dataset.toDF, $(numFolds), $(foldCol)), filteredSchema)
+    }
     val metrics = splits.zipWithIndex.map { case ((training, validation), splitIndex) =>
-      val trainingDataset = sparkSession.createDataFrame(training, schema).cache()
-      val validationDataset = sparkSession.createDataFrame(validation, schema).cache()
+      val trainingDataset = sparkSession.createDataFrame(training, schemaWithoutFold).cache()
+      val validationDataset = sparkSession.createDataFrame(validation, schemaWithoutFold).cache()
       instr.logDebug(s"Train split $splitIndex with multiple sets of parameters.")
 
       // Fit models in a Future for training in parallel
@@ -183,7 +203,14 @@ class CrossValidator @Since("1.2.0") (@Since("1.4.0") override val uid: String)
   }
 
   @Since("1.4.0")
-  override def transformSchema(schema: StructType): StructType = transformSchemaImpl(schema)
+  override def transformSchema(schema: StructType): StructType = {
+    if ($(foldCol) != "") {
+      val foldColDt = schema.apply($(foldCol)).dataType
+      require(foldColDt.isInstanceOf[IntegerType],
+        s"The specified `foldCol` column ${$(foldCol)} must be integer type, but got $foldColDt.")
+    }
+    transformSchemaImpl(schema)
+  }
 
   @Since("1.4.0")
   override def copy(extra: ParamMap): CrossValidator = {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tuning/ParamRandomBuilder.scala b/mllib/src/main/scala/org/apache/spark/ml/tuning/ParamRandomBuilder.scala
new file mode 100644
index 0000000000000..9c296bbc95224
--- /dev/null
+++ b/mllib/src/main/scala/org/apache/spark/ml/tuning/ParamRandomBuilder.scala
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.tuning
+
+import org.apache.spark.annotation.Since
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.tuning.RandomRanges._
+
+case class Limits[T: Numeric](x: T, y: T)
+
+private[ml] abstract class RandomT[T: Numeric] {
+  def randomT(): T
+  def randomTLog(n: Int): T
+}
+
+abstract class Generator[T: Numeric] {
+  def apply(lim: Limits[T]): RandomT[T]
+}
+
+object RandomRanges {
+
+  private val rnd = new scala.util.Random
+
+  private[tuning] def randomBigInt0To(x: BigInt): BigInt = {
+    var randVal = BigInt(x.bitLength, rnd)
+    while (randVal > x) {
+      randVal = BigInt(x.bitLength, rnd)
+    }
+    randVal
+  }
+
+  private[ml] def bigIntBetween(lower: BigInt, upper: BigInt): BigInt = {
+    val diff: BigInt = upper - lower
+    randomBigInt0To(diff) + lower
+  }
+
+  private def randomBigDecimalBetween(lower: BigDecimal, upper: BigDecimal): BigDecimal = {
+    val zeroCenteredRnd: BigDecimal = BigDecimal(rnd.nextDouble() - 0.5)
+    val range: BigDecimal = upper - lower
+    val halfWay: BigDecimal = lower + range / 2
+    (zeroCenteredRnd * range) + halfWay
+  }
+
+  implicit object DoubleGenerator extends Generator[Double] {
+    def apply(limits: Limits[Double]): RandomT[Double] = new RandomT[Double] {
+      import limits._
+      val lower: Double = math.min(x, y)
+      val upper: Double = math.max(x, y)
+
+      override def randomTLog(n: Int): Double =
+        RandomRanges.randomLog(lower, upper, n)
+
+      override def randomT(): Double =
+        randomBigDecimalBetween(BigDecimal(lower), BigDecimal(upper)).doubleValue
+    }
+  }
+
+  implicit object FloatGenerator extends Generator[Float] {
+    def apply(limits: Limits[Float]): RandomT[Float] = new RandomT[Float] {
+      import limits._
+      val lower: Float = math.min(x, y)
+      val upper: Float = math.max(x, y)
+
+      override def randomTLog(n: Int): Float =
+        RandomRanges.randomLog(lower, upper, n).toFloat
+
+      override def randomT(): Float =
+        randomBigDecimalBetween(BigDecimal(lower), BigDecimal(upper)).floatValue
+    }
+  }
+
+  implicit object IntGenerator extends Generator[Int] {
+    def apply(limits: Limits[Int]): RandomT[Int] = new RandomT[Int] {
+      import limits._
+      val lower: Int = math.min(x, y)
+      val upper: Int = math.max(x, y)
+
+      override def randomTLog(n: Int): Int =
+        RandomRanges.randomLog(lower, upper, n).toInt
+
+      override def randomT(): Int =
+        bigIntBetween(BigInt(lower), BigInt(upper)).intValue
+    }
+  }
+
+  private[ml] def logN(x: Double, base: Int): Double = math.log(x) / math.log(base)
+
+  private[ml] def randomLog(lower: Double, upper: Double, n: Int): Double = {
+    val logLower: Double = logN(lower, n)
+    val logUpper: Double = logN(upper, n)
+    val logLimits: Limits[Double] = Limits(logLower, logUpper)
+    val rndLogged: RandomT[Double] = RandomRanges(logLimits)
+    math.pow(n, rndLogged.randomT())
+  }
+
+  private[ml] def apply[T: Generator](lim: Limits[T])(implicit t: Generator[T]): RandomT[T] = t(lim)
+
+}
+
+/**
+ * "For any distribution over a sample space with a finite maximum, the maximum of 60 random
+ * observations lies within the top 5% of the true maximum, with 95% probability"
+ * - Evaluating Machine Learning Models by Alice Zheng
+ * https://www.oreilly.com/library/view/evaluating-machine-learning/9781492048756/ch04.html
+ *
+ * Note: if you want more sophisticated hyperparameter tuning, consider Python libraries
+ * such as Hyperopt.
+ */
+@Since("3.2.0")
+class ParamRandomBuilder extends ParamGridBuilder {
+  def addRandom[T: Generator](param: Param[T], lim: Limits[T], n: Int): this.type = {
+    val gen: RandomT[T] = RandomRanges(lim)
+    addGrid(param, (1 to n).map { _: Int => gen.randomT() })
+  }
+
+  def addLog10Random[T: Generator](param: Param[T], lim: Limits[T], n: Int): this.type =
+    addLogRandom(param, lim, n, 10)
+
+  private def addLogRandom[T: Generator](param: Param[T], lim: Limits[T],
+                                         n: Int, base: Int): this.type = {
+    val gen: RandomT[T] = RandomRanges(lim)
+    addGrid(param, (1 to n).map { _: Int => gen.randomTLog(base) })
+  }
+
+  // specialized versions for Java.
+
+  def addRandom(param: DoubleParam, x: Double, y: Double, n: Int): this.type =
+    addRandom(param, Limits(x, y), n)(DoubleGenerator)
+
+  def addLog10Random(param: DoubleParam, x: Double, y: Double, n: Int): this.type =
+    addLogRandom(param, Limits(x, y), n, 10)(DoubleGenerator)
+
+  def addRandom(param: FloatParam, x: Float, y: Float, n: Int): this.type =
+    addRandom(param, Limits(x, y), n)(FloatGenerator)
+
+  def addLog10Random(param: FloatParam, x: Float, y: Float, n: Int): this.type =
+    addLogRandom(param, Limits(x, y), n, 10)(FloatGenerator)
+
+  def addRandom(param: IntParam, x: Int, y: Int, n: Int): this.type =
+    addRandom(param, Limits(x, y), n)(IntGenerator)
+
+  def addLog10Random(param: IntParam, x: Int, y: Int, n: Int): this.type =
+    addLogRandom(param, Limits(x, y), n, 10)(IntGenerator)
+
+}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/util/Instrumentation.scala b/mllib/src/main/scala/org/apache/spark/ml/util/Instrumentation.scala
index d4b39e11fd1d7..2215c2b071584 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/util/Instrumentation.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/util/Instrumentation.scala
@@ -192,6 +192,8 @@ private[spark] object Instrumentation {
       case Failure(NonFatal(e)) =>
         instr.logFailure(e)
         throw e
+      case Failure(e) =>
+        throw e
       case Success(result) =>
         instr.logSuccess()
         result
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/api/python/PythonMLLibAPI.scala b/mllib/src/main/scala/org/apache/spark/mllib/api/python/PythonMLLibAPI.scala
index 259ecb3a1762f..92ae6b5bf8a03 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/api/python/PythonMLLibAPI.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/api/python/PythonMLLibAPI.scala
@@ -1223,28 +1223,28 @@ private[python] class PythonMLLibAPI extends Serializable {
    * Python-friendly version of [[MLUtils.convertVectorColumnsToML()]].
    */
   def convertVectorColumnsToML(dataset: DataFrame, cols: JArrayList[String]): DataFrame = {
-    MLUtils.convertVectorColumnsToML(dataset, cols.asScala: _*)
+    MLUtils.convertVectorColumnsToML(dataset, cols.asScala.toSeq: _*)
   }
 
   /**
    * Python-friendly version of [[MLUtils.convertVectorColumnsFromML()]]
    */
   def convertVectorColumnsFromML(dataset: DataFrame, cols: JArrayList[String]): DataFrame = {
-    MLUtils.convertVectorColumnsFromML(dataset, cols.asScala: _*)
+    MLUtils.convertVectorColumnsFromML(dataset, cols.asScala.toSeq: _*)
   }
 
   /**
    * Python-friendly version of [[MLUtils.convertMatrixColumnsToML()]].
    */
   def convertMatrixColumnsToML(dataset: DataFrame, cols: JArrayList[String]): DataFrame = {
-    MLUtils.convertMatrixColumnsToML(dataset, cols.asScala: _*)
+    MLUtils.convertMatrixColumnsToML(dataset, cols.asScala.toSeq: _*)
   }
 
   /**
    * Python-friendly version of [[MLUtils.convertMatrixColumnsFromML()]]
    */
   def convertMatrixColumnsFromML(dataset: DataFrame, cols: JArrayList[String]): DataFrame = {
-    MLUtils.convertMatrixColumnsFromML(dataset, cols.asScala: _*)
+    MLUtils.convertMatrixColumnsFromML(dataset, cols.asScala.toSeq: _*)
   }
 }
 
@@ -1313,8 +1313,10 @@ private[spark] abstract class SerDeBase {
   def dumps(obj: AnyRef): Array[Byte] = {
     obj match {
       // Pickler in Python side cannot deserialize Scala Array normally. See SPARK-12834.
-      case array: Array[_] => new Pickler().dumps(array.toSeq.asJava)
-      case _ => new Pickler().dumps(obj)
+      case array: Array[_] => new Pickler(/* useMemo = */ true,
+        /* valueCompare = */ false).dumps(array.toSeq.asJava)
+      case _ => new Pickler(/* useMemo = */ true,
+        /* valueCompare = */ false).dumps(obj)
     }
   }
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/classification/LogisticRegression.scala b/mllib/src/main/scala/org/apache/spark/mllib/classification/LogisticRegression.scala
index 21eb17dfaacb3..75262ac4fe06b 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/classification/LogisticRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/classification/LogisticRegression.scala
@@ -30,7 +30,6 @@ import org.apache.spark.mllib.regression._
 import org.apache.spark.mllib.util.{DataValidators, Loader, Saveable}
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.SparkSession
-import org.apache.spark.storage.StorageLevel
 
 /**
  * Classification model trained using Multinomial/Binary Logistic Regression.
@@ -339,10 +338,8 @@ class LogisticRegressionWithLBFGS
         // Convert our input into a DataFrame
         val spark = SparkSession.builder().sparkContext(input.context).getOrCreate()
         val df = spark.createDataFrame(input.map(_.asML))
-        // Determine if we should cache the DF
-        val handlePersistence = input.getStorageLevel == StorageLevel.NONE
         // Train our model
-        val mlLogisticRegressionModel = lr.train(df, handlePersistence)
+        val mlLogisticRegressionModel = lr.train(df)
         // convert the model
         val weights = Vectors.dense(mlLogisticRegressionModel.coefficients.toArray)
         createModel(weights, mlLogisticRegressionModel.intercept)
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/classification/NaiveBayes.scala b/mllib/src/main/scala/org/apache/spark/mllib/classification/NaiveBayes.scala
index 586f622fc47c5..5b13deffcf056 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/classification/NaiveBayes.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/classification/NaiveBayes.scala
@@ -216,7 +216,7 @@ object NaiveBayesModel extends Loader[NaiveBayesModel] {
       val data = dataArray(0)
       val labels = data.getAs[Seq[Double]](0).toArray
       val pi = data.getAs[Seq[Double]](1).toArray
-      val theta = data.getAs[Seq[Seq[Double]]](2).map(_.toArray).toArray
+      val theta = data.getSeq[scala.collection.Seq[Double]](2).map(_.toArray).toArray
       val modelType = data.getString(3)
       new NaiveBayesModel(labels, pi, theta, modelType)
     }
@@ -260,7 +260,7 @@ object NaiveBayesModel extends Loader[NaiveBayesModel] {
       val data = dataArray(0)
       val labels = data.getAs[Seq[Double]](0).toArray
       val pi = data.getAs[Seq[Double]](1).toArray
-      val theta = data.getAs[Seq[Seq[Double]]](2).map(_.toArray).toArray
+      val theta = data.getSeq[scala.collection.Seq[Double]](2).map(_.toArray).toArray
       new NaiveBayesModel(labels, pi, theta)
     }
   }
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/classification/SVM.scala b/mllib/src/main/scala/org/apache/spark/mllib/classification/SVM.scala
index 087c2c2639831..90cc4fb13b995 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/classification/SVM.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/classification/SVM.scala
@@ -146,7 +146,7 @@ class SVMWithSGD private (
 
   /**
    * Construct a SVM object with default parameters: {stepSize: 1.0, numIterations: 100,
-   * regParm: 0.01, miniBatchFraction: 1.0}.
+   * regParam: 0.01, miniBatchFraction: 1.0}.
    */
   @Since("0.8.0")
   def this() = this(1.0, 100, 0.01, 1.0)
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeans.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeans.scala
index 7c12697be95c8..6be32ab104389 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeans.scala
@@ -153,30 +153,25 @@ class BisectingKMeans private (
     this
   }
 
-  private[spark] def run(
-      input: RDD[Vector],
-      instr: Option[Instrumentation]): BisectingKMeansModel = {
-    val instances: RDD[(Vector, Double)] = input.map {
-      case (point) => (point, 1.0)
-    }
-    runWithWeight(instances, None)
-  }
-
   private[spark] def runWithWeight(
-      input: RDD[(Vector, Double)],
+      instances: RDD[(Vector, Double)],
+      handlePersistence: Boolean,
       instr: Option[Instrumentation]): BisectingKMeansModel = {
-    val d = input.map(_._1.size).first
+    val d = instances.map(_._1.size).first
     logInfo(s"Feature dimension: $d.")
 
-    val dMeasure: DistanceMeasure = DistanceMeasure.decodeFromString(this.distanceMeasure)
-    // Compute and cache vector norms for fast distance computation.
-    val norms = input.map(d => Vectors.norm(d._1, 2.0))
-    val vectors = input.zip(norms).map {
-      case ((x, weight), norm) => new VectorWithNorm(x, norm, weight)
-    }
-    if (input.getStorageLevel == StorageLevel.NONE) {
+    val dMeasure = DistanceMeasure.decodeFromString(this.distanceMeasure)
+    val norms = instances.map(d => Vectors.norm(d._1, 2.0))
+    val vectors = instances.zip(norms)
+      .map { case ((x, weight), norm) => new VectorWithNorm(x, norm, weight) }
+
+    if (handlePersistence) {
       vectors.persist(StorageLevel.MEMORY_AND_DISK)
+    } else {
+      // Compute and cache vector norms for fast distance computation.
+      norms.persist(StorageLevel.MEMORY_AND_DISK)
     }
+
     var assignments = vectors.map(v => (ROOT_INDEX, v))
     var activeClusters = summarize(d, assignments, dMeasure)
     instr.foreach(_.logNumExamples(activeClusters.values.map(_.size).sum))
@@ -225,7 +220,7 @@ class BisectingKMeans private (
             divisibleIndices.contains(parentIndex(index))
           }
           newClusters = summarize(d, newAssignments, dMeasure)
-          newClusterCenters = newClusters.mapValues(_.center).map(identity)
+          newClusterCenters = newClusters.mapValues(_.center).map(identity).toMap
         }
         if (preIndices != null) {
           preIndices.unpersist()
@@ -244,13 +239,11 @@ class BisectingKMeans private (
       }
       level += 1
     }
-    if (preIndices != null) {
-      preIndices.unpersist()
-    }
-    if (indices != null) {
-      indices.unpersist()
-    }
-    vectors.unpersist()
+
+    if (preIndices != null) { preIndices.unpersist() }
+    if (indices != null) { indices.unpersist() }
+    if (handlePersistence) { vectors.unpersist() } else { norms.unpersist() }
+
     val clusters = activeClusters ++ inactiveClusters
     val root = buildTree(clusters, dMeasure)
     val totalCost = root.leafNodes.map(_.cost).sum
@@ -264,7 +257,9 @@ class BisectingKMeans private (
    */
   @Since("1.6.0")
   def run(input: RDD[Vector]): BisectingKMeansModel = {
-    run(input, None)
+    val instances = input.map(point => (point, 1.0))
+    val handlePersistence = input.getStorageLevel == StorageLevel.NONE
+    runWithWeight(instances, handlePersistence, None)
   }
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/DistanceMeasure.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/DistanceMeasure.scala
index bffed61c291ea..9ac473aabecea 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/DistanceMeasure.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/DistanceMeasure.scala
@@ -41,7 +41,7 @@ private[spark] abstract class DistanceMeasure extends Serializable {
    *         1, if i != j: a bound r = matrix(i,j) to help avoiding unnecessary distance
    *         computation. Given point x, let i be current closest center, and d be current best
    *         distance, if d < f(r), then we no longer need to compute the distance to center j;
-   *         2, if i == j: a bound r = matrix(i,i) = min_k{maxtrix(i,k)|k!=i}. If distance
+   *         2, if i == j: a bound r = matrix(i,i) = min_k{matrix(i,k)|k!=i}. If distance
    *         between point x and center i is less than f(r), then center i is the closest center
    *         to point x.
    */
@@ -268,7 +268,7 @@ private[spark] class EuclideanDistanceMeasure extends DistanceMeasure {
    *         squared distance, if d < r, then we no longer need to compute the distance to center
    *         j. matrix(i,j) equals to squared of half of Euclidean distance between centers i
    *         and j;
-   *         2, if i == j: a bound r = matrix(i,i) = min_k{maxtrix(i,k)|k!=i}. If squared
+   *         2, if i == j: a bound r = matrix(i,i) = min_k{matrix(i,k)|k!=i}. If squared
    *         distance between point x and center i is less than r, then center i is the closest
    *         center to point x.
    */
@@ -405,7 +405,7 @@ private[spark] class CosineDistanceMeasure extends DistanceMeasure {
    *         is used instead of Cosine distance to compute matrix(i,j): for centers i and j,
    *         compute the radian/angle between them, halving it, and converting it back to Cosine
    *         distance at the end;
-   *         2, if i == j: a bound r = matrix(i,i) = min_k{maxtrix(i,k)|k!=i}. If Cosine
+   *         2, if i == j: a bound r = matrix(i,i) = min_k{matrix(i,k)|k!=i}. If Cosine
    *         distance between point x and center i is less than r, then center i is the closest
    *         center to point x.
    */
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/GaussianMixture.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/GaussianMixture.scala
index 4d98ba41bbb7b..d5a7882614546 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/GaussianMixture.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/GaussianMixture.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.mllib.clustering
 
-import scala.collection.mutable.IndexedSeq
-
 import breeze.linalg.{diag, DenseMatrix => BreezeMatrix, DenseVector => BDV, Vector => BV}
 
 import org.apache.spark.annotation.Since
@@ -189,8 +187,8 @@ class GaussianMixture private (
       case None =>
         val samples = breezeData.takeSample(withReplacement = true, k * nSamples, seed)
         (Array.fill(k)(1.0 / k), Array.tabulate(k) { i =>
-          val slice = samples.view(i * nSamples, (i + 1) * nSamples)
-          new MultivariateGaussian(vectorMean(slice), initCovariance(slice))
+          val slice = samples.view.slice(i * nSamples, (i + 1) * nSamples)
+          new MultivariateGaussian(vectorMean(slice.toSeq), initCovariance(slice.toSeq))
         })
     }
 
@@ -259,7 +257,7 @@ class GaussianMixture private (
   }
 
   /** Average of dense breeze vectors */
-  private def vectorMean(x: IndexedSeq[BV[Double]]): BDV[Double] = {
+  private def vectorMean(x: Seq[BV[Double]]): BDV[Double] = {
     val v = BDV.zeros[Double](x(0).length)
     x.foreach(xi => v += xi)
     v / x.length.toDouble
@@ -269,7 +267,7 @@ class GaussianMixture private (
    * Construct matrix where diagonal entries are element-wise
    * variance of input vectors (computes biased variance)
    */
-  private def initCovariance(x: IndexedSeq[BV[Double]]): BreezeMatrix[Double] = {
+  private def initCovariance(x: Seq[BV[Double]]): BreezeMatrix[Double] = {
     val mu = vectorMean(x)
     val ss = BDV.zeros[Double](x(0).length)
     x.foreach { xi =>
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/GaussianMixtureModel.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/GaussianMixtureModel.scala
index 5d2ecf33129c6..0c9c6ab826e62 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/GaussianMixtureModel.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/GaussianMixtureModel.scala
@@ -168,7 +168,7 @@ object GaussianMixtureModel extends Loader[GaussianMixtureModel] {
           (weight, new MultivariateGaussian(mu, sigma))
       }.unzip
 
-      new GaussianMixtureModel(weights.toArray, gaussians.toArray)
+      new GaussianMixtureModel(weights, gaussians)
     }
   }
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
index 1c5de5a092d6e..76e2928f12236 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
@@ -210,27 +210,26 @@ class KMeans private (
   @Since("0.8.0")
   def run(data: RDD[Vector]): KMeansModel = {
     val instances = data.map(point => (point, 1.0))
-    runWithWeight(instances, None)
+    val handlePersistence = data.getStorageLevel == StorageLevel.NONE
+    runWithWeight(instances, handlePersistence, None)
   }
 
   private[spark] def runWithWeight(
-      data: RDD[(Vector, Double)],
+      instances: RDD[(Vector, Double)],
+      handlePersistence: Boolean,
       instr: Option[Instrumentation]): KMeansModel = {
+    val norms = instances.map { case (v, _) => Vectors.norm(v, 2.0) }
+    val vectors = instances.zip(norms)
+      .map { case ((v, w), norm) => new VectorWithNorm(v, norm, w) }
 
-    // Compute squared norms and cache them.
-    val norms = data.map { case (v, _) =>
-      Vectors.norm(v, 2.0)
-    }
-
-    val zippedData = data.zip(norms).map { case ((v, w), norm) =>
-      new VectorWithNorm(v, norm, w)
-    }
-
-    if (data.getStorageLevel == StorageLevel.NONE) {
-      zippedData.persist(StorageLevel.MEMORY_AND_DISK)
+    if (handlePersistence) {
+      vectors.persist(StorageLevel.MEMORY_AND_DISK)
+    } else {
+      // Compute squared norms and cache them.
+      norms.persist(StorageLevel.MEMORY_AND_DISK)
     }
-    val model = runAlgorithmWithWeight(zippedData, instr)
-    zippedData.unpersist()
+    val model = runAlgorithmWithWeight(vectors, instr)
+    if (handlePersistence) { vectors.unpersist() } else { norms.unpersist() }
 
     model
   }
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeansModel.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeansModel.scala
index 04a3b6dd413b4..a24493bb7a8f9 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeansModel.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeansModel.scala
@@ -145,9 +145,9 @@ object KMeansModel extends Loader[KMeansModel] {
     }
   }
 
-  private case class Cluster(id: Int, point: Vector)
+  private[KMeansModel] case class Cluster(id: Int, point: Vector)
 
-  private object Cluster {
+  private[KMeansModel] object Cluster {
     def apply(r: Row): Cluster = {
       Cluster(r.getInt(0), r.getAs[Vector](1))
     }
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAModel.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAModel.scala
index 2c2eacbeb9407..ec952520eb9c0 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAModel.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAModel.scala
@@ -212,7 +212,7 @@ class LocalLDAModel private[spark] (
       val topic = normalize(brzTopics(::, topicIndex), 1.0)
       val (termWeights, terms) =
         topic.toArray.zipWithIndex.sortBy(-_._1).take(maxTermsPerTopic).unzip
-      (terms.toArray, termWeights.toArray)
+      (terms, termWeights)
     }.toArray
   }
 
@@ -606,7 +606,7 @@ class DistributedLDAModel private[clustering] (
       }
     topicsInQueues.map { q =>
       val (termWeights, terms) = q.toArray.sortBy(-_._1).unzip
-      (terms.toArray, termWeights.toArray)
+      (terms, termWeights)
     }
   }
 
@@ -641,7 +641,7 @@ class DistributedLDAModel private[clustering] (
       }
     topicsInQueues.map { q =>
       val (docTopics, docs) = q.toArray.sortBy(-_._1).unzip
-      (docs.toArray, docTopics.toArray)
+      (docs, docTopics)
     }
   }
 
@@ -681,7 +681,7 @@ class DistributedLDAModel private[clustering] (
     perDocAssignments.map { case (docID: Long, (terms: Array[Int], topics: Array[Int])) =>
       // TODO: Avoid zip, which is inefficient.
       val (sortedTerms, sortedTopics) = terms.zip(topics).sortBy(_._1).unzip
-      (docID, sortedTerms.toArray, sortedTopics.toArray)
+      (docID, sortedTerms, sortedTopics)
     }
   }
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala
index b2742ee6ecb5b..c9f6d789d6740 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala
@@ -466,7 +466,7 @@ final class OnlineLDAOptimizer extends LDAOptimizer with Logging {
     val seed = randomGenerator.nextLong()
     // If and only if optimizeDocConcentration is set true,
     // we calculate logphat in the same pass as other statistics.
-    // No calculation of loghat happens otherwise.
+    // No calculation of logphat happens otherwise.
     val logphatPartOptionBase = () => if (optimizeDocConcentration) {
                                         Some(BDV.zeros[Double](k))
                                       } else {
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/StreamingKMeans.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/StreamingKMeans.scala
index 3c9b806d616fc..111030dada491 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/StreamingKMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/StreamingKMeans.scala
@@ -36,7 +36,7 @@ import org.apache.spark.util.random.XORShiftRandom
  * doing a single iteration of the standard k-means algorithm.
  *
  * The update algorithm uses the "mini-batch" KMeans rule,
- * generalized to incorporate forgetfullness (i.e. decay).
+ * generalized to incorporate forgetfulness (i.e. decay).
  * The update rule (for each cluster) is:
  *
  * <blockquote>
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MulticlassMetrics.scala b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MulticlassMetrics.scala
index 050ebb0fa4fbd..1a91801a9da28 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MulticlassMetrics.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MulticlassMetrics.scala
@@ -283,7 +283,8 @@ class MulticlassMetrics @Since("1.1.0") (predictionAndLabels: RDD[_ <: Product])
         (loss * weight, weight)
 
       case other =>
-        throw new IllegalArgumentException(s"Expected quadruples, got $other")
+        throw new IllegalArgumentException(s"Invalid RDD value for MulticlassMetrics.logLoss. " +
+          s"Expected quadruples, got $other")
     }.treeReduce { case ((l1, w1), (l2, w2)) =>
       (l1 + l2, w1 + w2)
     }
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/RegressionMetrics.scala b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/RegressionMetrics.scala
index b697d2746ce7b..7938427544bd9 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/RegressionMetrics.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/RegressionMetrics.scala
@@ -131,4 +131,6 @@ class RegressionMetrics @Since("2.0.0") (
       1 - SSerr / SStot
     }
   }
+
+  private[spark] def count: Long = summary.count
 }
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/feature/ChiSqSelector.scala b/mllib/src/main/scala/org/apache/spark/mllib/feature/ChiSqSelector.scala
index d970c3c3d6131..3202f08e220b0 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/feature/ChiSqSelector.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/feature/ChiSqSelector.scala
@@ -195,7 +195,7 @@ class ChiSqSelector @Since("2.1.0") () extends Serializable {
    * The is the same to call this() and setNumTopFeatures(numTopFeatures)
    */
   @Since("1.3.0")
-  def this(numTopFeatures: Int) {
+  def this(numTopFeatures: Int) = {
     this()
     this.numTopFeatures = numTopFeatures
   }
@@ -283,7 +283,7 @@ class ChiSqSelector @Since("2.1.0") () extends Serializable {
         chiSqTestResult
           .filter { case (res, _) => res.pValue < fwe / chiSqTestResult.length }
       case errorType =>
-        throw new IllegalStateException(s"Unknown ChiSqSelector Type: $errorType")
+        throw new IllegalArgumentException(s"Unknown ChiSqSelector Type: $errorType")
     }
     val indices = features.map { case (_, index) => index }
     new ChiSqSelectorModel(indices)
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/feature/PCA.scala b/mllib/src/main/scala/org/apache/spark/mllib/feature/PCA.scala
index 356ed48e99387..f7c6d09f5e437 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/feature/PCA.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/feature/PCA.scala
@@ -48,11 +48,11 @@ class PCA @Since("1.4.0") (@Since("1.4.0") val k: Int) {
     val mat = if (numFeatures > 65535) {
       val summary = Statistics.colStats(sources.map((_, 1.0)), Seq("mean"))
       val mean = Vectors.fromML(summary.mean)
-      val meanCentredRdd = sources.map { row =>
+      val meanCenteredRdd = sources.map { row =>
         BLAS.axpy(-1, mean, row)
         row
       }
-      new RowMatrix(meanCentredRdd)
+      new RowMatrix(meanCenteredRdd)
     } else {
       require(PCAUtil.memoryCost(k, numFeatures) < Int.MaxValue,
         "The param k and numFeatures is too large for SVD computation. " +
@@ -124,9 +124,9 @@ private[feature] object PCAUtil {
   // 6e541be066d547a097f5089165cd7c38c3ca276d/math/src/main/scala/breeze/linalg/
   // functions/svd.scala#L87
   def memoryCost(k: Int, numFeatures: Int): Long = {
-    3L * math.min(k, numFeatures) * math.min(k, numFeatures)
-    + math.max(math.max(k, numFeatures), 4L * math.min(k, numFeatures)
-    * math.min(k, numFeatures) + 4L * math.min(k, numFeatures))
+    3L * math.min(k, numFeatures) * math.min(k, numFeatures) +
+      math.max(math.max(k, numFeatures), 4L * math.min(k, numFeatures) *
+      math.min(k, numFeatures) + 4L * math.min(k, numFeatures))
   }
 
 }
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/feature/StandardScaler.scala b/mllib/src/main/scala/org/apache/spark/mllib/feature/StandardScaler.scala
index 78c974e22f2cf..12a5a0f2b2189 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/feature/StandardScaler.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/feature/StandardScaler.scala
@@ -83,7 +83,7 @@ class StandardScalerModel @Since("1.3.0") (
   /**
    */
   @Since("1.3.0")
-  def this(std: Vector, mean: Vector) {
+  def this(std: Vector, mean: Vector) = {
     this(std, mean, withStd = std != null, withMean = mean != null)
     require(this.withStd || this.withMean,
       "at least one of std or mean vectors must be provided")
@@ -167,6 +167,8 @@ class StandardScalerModel @Since("1.3.0") (
             val newValues = NewStandardScalerModel
               .transformSparseWithScale(localScale, indices, values.clone())
             Vectors.sparse(size, indices, newValues)
+          case v =>
+            throw new IllegalArgumentException(s"Unknown vector type ${v.getClass}.")
         }
 
       case _ => vector
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/feature/Word2Vec.scala b/mllib/src/main/scala/org/apache/spark/mllib/feature/Word2Vec.scala
index 13899fa8296f6..d2cfedcc33e88 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/feature/Word2Vec.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/feature/Word2Vec.scala
@@ -22,7 +22,7 @@ import java.lang.{Iterable => JavaIterable}
 import scala.collection.JavaConverters._
 import scala.collection.mutable
 
-import com.github.fommil.netlib.BLAS.{getInstance => blas}
+import com.google.common.collect.{Ordering => GuavaOrdering}
 import org.json4s.DefaultFormats
 import org.json4s.JsonDSL._
 import org.json4s.jackson.JsonMethods._
@@ -33,11 +33,11 @@ import org.apache.spark.api.java.JavaRDD
 import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.Kryo.KRYO_SERIALIZER_MAX_BUFFER_SIZE
+import org.apache.spark.ml.linalg.BLAS
 import org.apache.spark.mllib.linalg.{Vector, Vectors}
 import org.apache.spark.mllib.util.{Loader, Saveable}
 import org.apache.spark.rdd._
 import org.apache.spark.sql.SparkSession
-import org.apache.spark.util.BoundedPriorityQueue
 import org.apache.spark.util.Utils
 import org.apache.spark.util.random.XORShiftRandom
 
@@ -401,18 +401,18 @@ class Word2Vec extends Serializable with Logging {
                       val inner = bcVocab.value(word).point(d)
                       val l2 = inner * vectorSize
                       // Propagate hidden -> output
-                      var f = blas.sdot(vectorSize, syn0, l1, 1, syn1, l2, 1)
+                      var f = BLAS.nativeBLAS.sdot(vectorSize, syn0, l1, 1, syn1, l2, 1)
                       if (f > -MAX_EXP && f < MAX_EXP) {
                         val ind = ((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2.0)).toInt
                         f = expTable.value(ind)
                         val g = ((1 - bcVocab.value(word).code(d) - f) * alpha).toFloat
-                        blas.saxpy(vectorSize, g, syn1, l2, 1, neu1e, 0, 1)
-                        blas.saxpy(vectorSize, g, syn0, l1, 1, syn1, l2, 1)
+                        BLAS.nativeBLAS.saxpy(vectorSize, g, syn1, l2, 1, neu1e, 0, 1)
+                        BLAS.nativeBLAS.saxpy(vectorSize, g, syn0, l1, 1, syn1, l2, 1)
                         syn1Modify(inner) += 1
                       }
                       d += 1
                     }
-                    blas.saxpy(vectorSize, 1.0f, neu1e, 0, 1, syn0, l1, 1)
+                    BLAS.nativeBLAS.saxpy(vectorSize, 1.0f, neu1e, 0, 1, syn0, l1, 1)
                     syn0Modify(lastWord) += 1
                   }
                 }
@@ -448,10 +448,10 @@ class Word2Vec extends Serializable with Logging {
           (id, (vec, 1))
         }
       }.reduceByKey { (vc1, vc2) =>
-        blas.saxpy(vectorSize, 1.0f, vc2._1, 1, vc1._1, 1)
+        BLAS.nativeBLAS.saxpy(vectorSize, 1.0f, vc2._1, 1, vc1._1, 1)
         (vc1._1, vc1._2 + vc2._2)
       }.map { case (id, (vec, count)) =>
-        blas.sscal(vectorSize, 1.0f / count, vec, 1)
+        BLAS.nativeBLAS.sscal(vectorSize, 1.0f / count, vec, 1)
         (id, vec)
       }.collect()
       var i = 0
@@ -502,22 +502,18 @@ class Word2VecModel private[spark] (
   private val vectorSize = wordVectors.length / numWords
 
   // wordList: Ordered list of words obtained from wordIndex.
-  private val wordList: Array[String] = {
-    val (wl, _) = wordIndex.toSeq.sortBy(_._2).unzip
-    wl.toArray
+  private lazy val wordList: Array[String] = {
+    wordIndex.toSeq.sortBy(_._2).iterator.map(_._1).toArray
   }
 
-  // wordVecNorms: Array of length numWords, each value being the Euclidean norm
-  //               of the wordVector.
-  private val wordVecNorms: Array[Float] = {
-    val wordVecNorms = new Array[Float](numWords)
-    var i = 0
-    while (i < numWords) {
-      val vec = wordVectors.slice(i * vectorSize, i * vectorSize + vectorSize)
-      wordVecNorms(i) = blas.snrm2(vectorSize, vec, 1)
-      i += 1
+  // wordVecInvNorms: Array of length numWords, each value being the inverse of
+  //                  Euclidean norm of the wordVector.
+  private lazy val wordVecInvNorms: Array[Float] = {
+    val size = vectorSize
+    Array.tabulate(numWords) { i =>
+      val norm = BLAS.nativeBLAS.snrm2(size, wordVectors, i * size, 1)
+      if (norm != 0) 1 / norm else 0.0F
     }
-    wordVecNorms
   }
 
   @Since("1.5.0")
@@ -538,9 +534,13 @@ class Word2VecModel private[spark] (
   @Since("1.1.0")
   def transform(word: String): Vector = {
     wordIndex.get(word) match {
-      case Some(ind) =>
-        val vec = wordVectors.slice(ind * vectorSize, ind * vectorSize + vectorSize)
-        Vectors.dense(vec.map(_.toDouble))
+      case Some(index) =>
+        val size = vectorSize
+        val offset = index * size
+        val array = Array.ofDim[Double](size)
+        var i = 0
+        while (i < size) { array(i) = wordVectors(offset + i); i += 1 }
+        Vectors.dense(array)
       case None =>
         throw new IllegalStateException(s"$word not in vocabulary")
     }
@@ -555,12 +555,12 @@ class Word2VecModel private[spark] (
   @Since("1.1.0")
   def findSynonyms(word: String, num: Int): Array[(String, Double)] = {
     val vector = transform(word)
-    findSynonyms(vector, num, Some(word))
+    findSynonyms(vector.toArray, num, Some(word))
   }
 
   /**
    * Find synonyms of the vector representation of a word, possibly
-   * including any words in the model vocabulary whose vector respresentation
+   * including any words in the model vocabulary whose vector representation
    * is the supplied vector.
    * @param vector vector representation of a word
    * @param num number of synonyms to find
@@ -568,7 +568,7 @@ class Word2VecModel private[spark] (
    */
   @Since("1.1.0")
   def findSynonyms(vector: Vector, num: Int): Array[(String, Double)] = {
-    findSynonyms(vector, num, None)
+    findSynonyms(vector.toArray, num, None)
   }
 
   /**
@@ -579,54 +579,48 @@ class Word2VecModel private[spark] (
    * @param wordOpt optionally, a word to reject from the results list
    * @return array of (word, cosineSimilarity)
    */
-  private def findSynonyms(
-      vector: Vector,
+  private[spark] def findSynonyms(
+      vector: Array[Double],
       num: Int,
       wordOpt: Option[String]): Array[(String, Double)] = {
     require(num > 0, "Number of similar words should > 0")
+    val localVectorSize = vectorSize
+
+    val floatVec = vector.map(_.toFloat)
+    val vecNorm = BLAS.nativeBLAS.snrm2(localVectorSize, floatVec, 1)
+
+    val localWordList = wordList
+    val localNumWords = numWords
+    if (vecNorm == 0) {
+      Iterator.tabulate(num + 1)(i => (localWordList(i), 0.0))
+        .filterNot(t => wordOpt.contains(t._1))
+        .take(num)
+        .toArray
+    } else {
+      // Normalize input vector before BLAS.nativeBLAS.sgemv to avoid Inf value
+      BLAS.nativeBLAS.sscal(localVectorSize, 1 / vecNorm, floatVec, 0, 1)
+
+      val cosineVec = Array.ofDim[Float](localNumWords)
+      BLAS.nativeBLAS.sgemv("T", localVectorSize, localNumWords, 1.0F, wordVectors, localVectorSize,
+        floatVec, 1, 0.0F, cosineVec, 1)
+
+      val localWordVecInvNorms = wordVecInvNorms
+      var i = 0
+      while (i < cosineVec.length) { cosineVec(i) *= localWordVecInvNorms(i); i += 1 }
 
-    val fVector = vector.toArray.map(_.toFloat)
-    val cosineVec = new Array[Float](numWords)
-    val alpha: Float = 1
-    val beta: Float = 0
-    // Normalize input vector before blas.sgemv to avoid Inf value
-    val vecNorm = blas.snrm2(vectorSize, fVector, 1)
-    if (vecNorm != 0.0f) {
-      blas.sscal(vectorSize, 1 / vecNorm, fVector, 0, 1)
-    }
-    blas.sgemv(
-      "T", vectorSize, numWords, alpha, wordVectors, vectorSize, fVector, 1, beta, cosineVec, 1)
-
-    var i = 0
-    while (i < numWords) {
-      val norm = wordVecNorms(i)
-      if (norm == 0.0f) {
-        cosineVec(i) = 0.0f
-      } else {
-        cosineVec(i) /= norm
+      val idxOrd = new GuavaOrdering[Int] {
+        override def compare(left: Int, right: Int): Int = {
+          Ordering[Float].compare(cosineVec(left), cosineVec(right))
+        }
       }
-      i += 1
-    }
-
-    val pq = new BoundedPriorityQueue[(String, Float)](num + 1)(Ordering.by(_._2))
 
-    var j = 0
-    while (j < numWords) {
-      pq += Tuple2(wordList(j), cosineVec(j))
-      j += 1
+      idxOrd.greatestOf(Iterator.range(0, localNumWords).asJava, num + 1)
+        .iterator.asScala
+        .map(i => (localWordList(i), cosineVec(i).toDouble))
+        .filterNot(t => wordOpt.contains(t._1))
+        .take(num)
+        .toArray
     }
-
-    val scored = pq.toSeq.sortBy(-_._2)
-
-    val filtered = wordOpt match {
-      case Some(w) => scored.filter(tup => w != tup._1)
-      case None => scored
-    }
-
-    filtered
-      .take(num)
-      .map { case (word, score) => (word, score.toDouble) }
-      .toArray
   }
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/fpm/AssociationRules.scala b/mllib/src/main/scala/org/apache/spark/mllib/fpm/AssociationRules.scala
index 43d256bbc46c3..606e2f2f212ca 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/fpm/AssociationRules.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/fpm/AssociationRules.scala
@@ -88,8 +88,8 @@ class AssociationRules private[fpm] (
 
     // Join to get (X, ((Y, freq(X union Y)), freq(X))), generate rules, and filter by confidence
     candidates.join(freqItemsets.map(x => (x.items.toSeq, x.freq)))
-      .map { case (antecendent, ((consequent, freqUnion), freqAntecedent)) =>
-        new Rule(antecendent.toArray,
+      .map { case (antecedent, ((consequent, freqUnion), freqAntecedent)) =>
+        new Rule(antecedent.toArray,
           consequent.toArray,
           freqUnion,
           freqAntecedent,
@@ -124,7 +124,7 @@ object AssociationRules {
   class Rule[Item] private[fpm] (
       @Since("1.5.0") val antecedent: Array[Item],
       @Since("1.5.0") val consequent: Array[Item],
-      freqUnion: Double,
+      private[spark] val freqUnion: Double,
       freqAntecedent: Double,
       freqConsequent: Option[Double]) extends Serializable {
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/fpm/PrefixSpan.scala b/mllib/src/main/scala/org/apache/spark/mllib/fpm/PrefixSpan.scala
index ac2b576f4ac4e..cd71aac34c268 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/fpm/PrefixSpan.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/fpm/PrefixSpan.scala
@@ -335,7 +335,7 @@ object PrefixSpan extends Logging {
       largePrefixes = newLargePrefixes
     }
 
-    var freqPatterns = sc.parallelize(localFreqPatterns, 1)
+    var freqPatterns = sc.parallelize(localFreqPatterns.toSeq, 1)
 
     val numSmallPrefixes = smallPrefixes.size
     logInfo(s"number of small prefixes for local processing: $numSmallPrefixes")
@@ -683,7 +683,7 @@ object PrefixSpanModel extends Loader[PrefixSpanModel[_]] {
 
     def loadImpl[Item: ClassTag](freqSequences: DataFrame, sample: Item): PrefixSpanModel[Item] = {
       val freqSequencesRDD = freqSequences.select("sequence", "freq").rdd.map { x =>
-        val sequence = x.getAs[Seq[Seq[Item]]](0).map(_.toArray).toArray
+        val sequence = x.getSeq[scala.collection.Seq[Item]](0).map(_.toArray).toArray
         val freq = x.getLong(1)
         new PrefixSpan.FreqSequence(sequence, freq)
       }
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/ARPACK.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/ARPACK.scala
new file mode 100644
index 0000000000000..fb0f6ddd470b4
--- /dev/null
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/ARPACK.scala
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.mllib.linalg
+
+import dev.ludovic.netlib.{ARPACK => NetlibARPACK,
+                           JavaARPACK => NetlibJavaARPACK,
+                           NativeARPACK => NetlibNativeARPACK}
+
+/**
+ * ARPACK routines for MLlib's vectors and matrices.
+ */
+private[spark] object ARPACK extends Serializable {
+
+  @transient private var _javaARPACK: NetlibARPACK = _
+  @transient private var _nativeARPACK: NetlibARPACK = _
+
+  private[spark] def javaARPACK: NetlibARPACK = {
+    if (_javaARPACK == null) {
+      _javaARPACK = NetlibJavaARPACK.getInstance
+    }
+    _javaARPACK
+  }
+
+  private[spark] def nativeARPACK: NetlibARPACK = {
+    if (_nativeARPACK == null) {
+      _nativeARPACK =
+        try { NetlibNativeARPACK.getInstance } catch { case _: Throwable => javaARPACK }
+    }
+    _nativeARPACK
+  }
+}
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/BLAS.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/BLAS.scala
index da486010cfa9e..e38cfe4e18d40 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/BLAS.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/BLAS.scala
@@ -17,8 +17,9 @@
 
 package org.apache.spark.mllib.linalg
 
-import com.github.fommil.netlib.{BLAS => NetlibBLAS, F2jBLAS}
-import com.github.fommil.netlib.BLAS.{getInstance => NativeBLAS}
+import dev.ludovic.netlib.{BLAS => NetlibBLAS,
+                           JavaBLAS => NetlibJavaBLAS,
+                           NativeBLAS => NetlibNativeBLAS}
 
 import org.apache.spark.internal.Logging
 
@@ -27,21 +28,30 @@ import org.apache.spark.internal.Logging
  */
 private[spark] object BLAS extends Serializable with Logging {
 
-  @transient private var _f2jBLAS: NetlibBLAS = _
+  @transient private var _javaBLAS: NetlibBLAS = _
   @transient private var _nativeBLAS: NetlibBLAS = _
   private val nativeL1Threshold: Int = 256
 
-  // For level-1 function dspmv, use f2jBLAS for better performance.
-  private[mllib] def f2jBLAS: NetlibBLAS = {
-    if (_f2jBLAS == null) {
-      _f2jBLAS = new F2jBLAS
+  // For level-1 function dspmv, use javaBLAS for better performance.
+  private[spark] def javaBLAS: NetlibBLAS = {
+    if (_javaBLAS == null) {
+      _javaBLAS = NetlibJavaBLAS.getInstance
     }
-    _f2jBLAS
+    _javaBLAS
   }
 
-  private[mllib] def getBLAS(vectorSize: Int): NetlibBLAS = {
+  // For level-3 routines, we use the native BLAS.
+  private[spark] def nativeBLAS: NetlibBLAS = {
+    if (_nativeBLAS == null) {
+      _nativeBLAS =
+        try { NetlibNativeBLAS.getInstance } catch { case _: Throwable => javaBLAS }
+    }
+    _nativeBLAS
+  }
+
+  private[spark] def getBLAS(vectorSize: Int): NetlibBLAS = {
     if (vectorSize < nativeL1Threshold) {
-      f2jBLAS
+      javaBLAS
     } else {
       nativeBLAS
     }
@@ -237,14 +247,6 @@ private[spark] object BLAS extends Serializable with Logging {
     }
   }
 
-  // For level-3 routines, we use the native BLAS.
-  private[mllib] def nativeBLAS: NetlibBLAS = {
-    if (_nativeBLAS == null) {
-      _nativeBLAS = NativeBLAS
-    }
-    _nativeBLAS
-  }
-
   /**
    * Adds alpha * v * v.t to a matrix in-place. This is the same as BLAS's ?SPR.
    *
@@ -263,7 +265,7 @@ private[spark] object BLAS extends Serializable with Logging {
     val n = v.size
     v match {
       case DenseVector(values) =>
-        NativeBLAS.dspr("U", n, alpha, values, 1, U)
+        nativeBLAS.dspr("U", n, alpha, values, 1, U)
       case SparseVector(size, indices, values) =>
         val nnz = indices.length
         var colStartIdx = 0
@@ -285,6 +287,8 @@ private[spark] object BLAS extends Serializable with Logging {
           j += 1
           prevCol = col
         }
+      case _ =>
+        throw new IllegalArgumentException(s"Unknown vector type ${v.getClass}.")
     }
   }
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/CholeskyDecomposition.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/CholeskyDecomposition.scala
index 68771f1afbe8c..f06ea9418f252 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/CholeskyDecomposition.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/CholeskyDecomposition.scala
@@ -17,7 +17,6 @@
 
 package org.apache.spark.mllib.linalg
 
-import com.github.fommil.netlib.LAPACK.{getInstance => lapack}
 import org.netlib.util.intW
 
 import org.apache.spark.ml.optim.SingularMatrixException
@@ -37,7 +36,7 @@ private[spark] object CholeskyDecomposition {
   def solve(A: Array[Double], bx: Array[Double]): Array[Double] = {
     val k = bx.length
     val info = new intW(0)
-    lapack.dppsv("U", k, 1, A, bx, k, info)
+    LAPACK.nativeLAPACK.dppsv("U", k, 1, A, bx, k, info)
     checkReturnValue(info, "dppsv")
     bx
   }
@@ -52,7 +51,7 @@ private[spark] object CholeskyDecomposition {
    */
   def inverse(UAi: Array[Double], k: Int): Array[Double] = {
     val info = new intW(0)
-    lapack.dpptri("U", k, UAi, info)
+    LAPACK.nativeLAPACK.dpptri("U", k, UAi, info)
     checkReturnValue(info, "dpptri")
     UAi
   }
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/EigenValueDecomposition.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/EigenValueDecomposition.scala
index c7c1a5404e5e8..2cbf5d09dc56f 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/EigenValueDecomposition.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/EigenValueDecomposition.scala
@@ -18,7 +18,6 @@
 package org.apache.spark.mllib.linalg
 
 import breeze.linalg.{DenseMatrix => BDM, DenseVector => BDV}
-import com.github.fommil.netlib.ARPACK
 import org.netlib.util.{doubleW, intW}
 
 /**
@@ -51,8 +50,6 @@ private[mllib] object EigenValueDecomposition {
     // TODO: remove this function and use eigs in breeze when switching breeze version
     require(n > k, s"Number of required eigenvalues $k must be smaller than matrix dimension $n")
 
-    val arpack = ARPACK.getInstance()
-
     // tolerance used in stopping criterion
     val tolW = new doubleW(tol)
     // number of desired eigenvalues, 0 < nev < n
@@ -67,7 +64,7 @@ private[mllib] object EigenValueDecomposition {
     // "LM" : compute the NEV largest (in magnitude) eigenvalues
     val which = "LM"
 
-    var iparam = new Array[Int](11)
+    val iparam = new Array[Int](11)
     // use exact shift in each iteration
     iparam(0) = 1
     // maximum number of Arnoldi update iterations, or the actual number of iterations on output
@@ -87,8 +84,8 @@ private[mllib] object EigenValueDecomposition {
     val ipntr = new Array[Int](11)
 
     // call ARPACK's reverse communication, first iteration with ido = 0
-    arpack.dsaupd(ido, bmat, n, which, nev.`val`, tolW, resid, ncv, v, n, iparam, ipntr, workd,
-      workl, workl.length, info)
+    ARPACK.nativeARPACK.dsaupd(ido, bmat, n, which, nev.`val`, tolW, resid, ncv,
+      v, n, iparam, ipntr, workd, workl, workl.length, info)
 
     val w = BDV(workd)
 
@@ -105,8 +102,8 @@ private[mllib] object EigenValueDecomposition {
       val y = w.slice(outputOffset, outputOffset + n)
       y := mul(x)
       // call ARPACK's reverse communication
-      arpack.dsaupd(ido, bmat, n, which, nev.`val`, tolW, resid, ncv, v, n, iparam, ipntr,
-        workd, workl, workl.length, info)
+      ARPACK.nativeARPACK.dsaupd(ido, bmat, n, which, nev.`val`, tolW, resid, ncv,
+        v, n, iparam, ipntr, workd, workl, workl.length, info)
     }
 
     if (info.`val` != 0) {
@@ -127,8 +124,8 @@ private[mllib] object EigenValueDecomposition {
     val z = java.util.Arrays.copyOfRange(v, 0, nev.`val` * n)
 
     // call ARPACK's post-processing for eigenvectors
-    arpack.dseupd(true, "A", select, d, z, n, 0.0, bmat, n, which, nev, tol, resid, ncv, v, n,
-      iparam, ipntr, workd, workl, workl.length, info)
+    ARPACK.nativeARPACK.dseupd(true, "A", select, d, z, n, 0.0, bmat, n, which, nev, tol, resid,
+      ncv, v, n, iparam, ipntr, workd, workl, workl.length, info)
 
     // number of computed eigenvalues, might be smaller than k
     val computed = iparam(4)
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/LAPACK.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/LAPACK.scala
new file mode 100644
index 0000000000000..4d25aed2835cb
--- /dev/null
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/LAPACK.scala
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.mllib.linalg
+
+import dev.ludovic.netlib.{JavaLAPACK => NetlibJavaLAPACK,
+                           LAPACK => NetlibLAPACK,
+                           NativeLAPACK => NetlibNativeLAPACK}
+
+/**
+ * LAPACK routines for MLlib's vectors and matrices.
+ */
+private[spark] object LAPACK extends Serializable {
+
+  @transient private var _javaLAPACK: NetlibLAPACK = _
+  @transient private var _nativeLAPACK: NetlibLAPACK = _
+
+  private[spark] def javaLAPACK: NetlibLAPACK = {
+    if (_javaLAPACK == null) {
+      _javaLAPACK = NetlibJavaLAPACK.getInstance
+    }
+    _javaLAPACK
+  }
+
+  private[spark] def nativeLAPACK: NetlibLAPACK = {
+    if (_nativeLAPACK == null) {
+      _nativeLAPACK =
+        try { NetlibNativeLAPACK.getInstance } catch { case _: Throwable => javaLAPACK }
+    }
+    _nativeLAPACK
+  }
+}
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
index 57edc965112ef..e4f64b4e34864 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
@@ -23,7 +23,6 @@ import scala.collection.mutable.{ArrayBuffer, ArrayBuilder => MArrayBuilder, Has
 import scala.language.implicitConversions
 
 import breeze.linalg.{CSCMatrix => BSM, DenseMatrix => BDM, Matrix => BM}
-import com.github.fommil.netlib.BLAS.{getInstance => blas}
 
 import org.apache.spark.annotation.Since
 import org.apache.spark.ml.{linalg => newlinalg}
@@ -427,7 +426,7 @@ class DenseMatrix @Since("1.3.0") (
     if (isTransposed) {
       Iterator.tabulate(numCols) { j =>
         val col = new Array[Double](numRows)
-        blas.dcopy(numRows, values, j, numCols, col, 0, 1)
+        BLAS.nativeBLAS.dcopy(numRows, values, j, numCols, col, 0, 1)
         new DenseVector(col)
       }
     } else {
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala
index c23088de85b8a..806cbab607af0 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala
@@ -289,6 +289,8 @@ class VectorUDT extends UserDefinedType[Vector] {
         row.setNullAt(2)
         row.update(3, UnsafeArrayData.fromPrimitiveArray(values))
         row
+      case v =>
+        throw new IllegalArgumentException(s"Unknown vector type ${v.getClass}.")
     }
   }
 
@@ -954,7 +956,7 @@ class SparseVector @Since("1.0.0") (
       currentIdx += 1
       i_v
     }.unzip
-    new SparseVector(selectedIndices.length, sliceInds.toArray, sliceVals.toArray)
+    new SparseVector(selectedIndices.length, sliceInds, sliceVals)
   }
 
   @Since("1.6.0")
@@ -984,7 +986,7 @@ class SparseVector @Since("1.0.0") (
 
       override def hasNext: Boolean = i < localSize
 
-      override def next: (Int, Double) = {
+      override def next(): (Int, Double) = {
         val v = if (i == k) {
           j += 1
           k = if (j < localNumActives) localIndices(j) else -1
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/IndexedRowMatrix.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/IndexedRowMatrix.scala
index ad79230c7513c..da5d1650694d6 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/IndexedRowMatrix.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/IndexedRowMatrix.scala
@@ -145,6 +145,8 @@ class IndexedRowMatrix @Since("1.0.0") (
             .map { case (values, blockColumn) =>
               ((blockRow.toInt, blockColumn), (rowInBlock.toInt, values.zipWithIndex))
             }
+        case v =>
+          throw new IllegalArgumentException(s"Unknown vector type ${v.getClass}.")
       }
     }.groupByKey(GridPartitioner(numRowBlocks, numColBlocks, rows.getNumPartitions)).map {
       case ((blockRow, blockColumn), itr) =>
@@ -187,6 +189,8 @@ class IndexedRowMatrix @Since("1.0.0") (
           Iterator.tabulate(indices.length)(i => MatrixEntry(rowIndex, indices(i), values(i)))
         case DenseVector(values) =>
           Iterator.tabulate(values.length)(i => MatrixEntry(rowIndex, i, values(i)))
+        case v =>
+          throw new IllegalArgumentException(s"Unknown vector type ${v.getClass}.")
       }
     }
     new CoordinateMatrix(entries, numRows(), numCols())
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
index 20e26cee9e0d6..d546f0c1a8e19 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
@@ -693,11 +693,11 @@ class RowMatrix @Since("1.0.0") (
     val pBV = sc.broadcast(colMagsCorrected.map(c => sg / c))
     val qBV = sc.broadcast(colMagsCorrected.map(c => math.min(sg, c)))
 
-    val sims = rows.mapPartitionsWithIndex { (indx, iter) =>
+    val sims = rows.mapPartitionsWithIndex { (index, iter) =>
       val p = pBV.value
       val q = qBV.value
 
-      val rand = new XORShiftRandom(indx)
+      val rand = new XORShiftRandom(index)
       val scaled = new Array[Double](p.size)
       iter.flatMap { row =>
         row match {
@@ -748,6 +748,8 @@ class RowMatrix @Since("1.0.0") (
               }
               buf
             }.flatten
+          case v =>
+            throw new IllegalArgumentException(s"Unknown vector type ${v.getClass}.")
         }
       }
     }.reduceByKey(_ + _).map { case ((i, j), sim) =>
@@ -786,11 +788,15 @@ class RowMatrix @Since("1.0.0") (
    * Based on the formulae: (numPartitions)^(1/depth) * objectSize <= DriverMaxResultSize
    * @param aggregatedObjectSizeInBytes the size, in megabytes, of the object being tree aggregated
    */
-  private[spark] def getTreeAggregateIdealDepth(aggregatedObjectSizeInBytes: Long) = {
+  private[spark] def getTreeAggregateIdealDepth(aggregatedObjectSizeInBytes: Long): Int = {
     require(aggregatedObjectSizeInBytes > 0,
       "Cannot compute aggregate depth heuristic based on a zero-size object to aggregate")
 
     val maxDriverResultSizeInBytes = rows.conf.get[Long](MAX_RESULT_SIZE)
+    if (maxDriverResultSizeInBytes <= 0) {
+      // Unlimited result size, so 1 is OK
+      return 1
+    }
 
     require(maxDriverResultSizeInBytes > aggregatedObjectSizeInBytes,
       s"Cannot aggregate object of size $aggregatedObjectSizeInBytes Bytes, "
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala
index 1336ffd2f7d5e..796a787e77db4 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala
@@ -129,7 +129,20 @@ class GradientDescent private[spark] (private var gradient: Gradient, private va
    * @return solution vector
    */
   def optimize(data: RDD[(Double, Vector)], initialWeights: Vector): Vector = {
-    val (weights, _) = GradientDescent.runMiniBatchSGD(
+    val (weights, _) = optimizeWithLossReturned(data, initialWeights)
+    weights
+  }
+
+  /**
+   * Runs gradient descent on the given training data.
+   * @param data training data
+   * @param initialWeights initial weights
+   * @return solution vector and loss value in an array
+   */
+  def optimizeWithLossReturned(
+      data: RDD[(Double, Vector)],
+      initialWeights: Vector): (Vector, Array[Double]) = {
+    GradientDescent.runMiniBatchSGD(
       data,
       gradient,
       updater,
@@ -139,7 +152,6 @@ class GradientDescent private[spark] (private var gradient: Gradient, private va
       miniBatchFraction,
       initialWeights,
       convergenceTol)
-    weights
   }
 
 }
@@ -195,7 +207,7 @@ object GradientDescent extends Logging {
         s"numIterations=$numIterations and miniBatchFraction=$miniBatchFraction")
     }
 
-    val stochasticLossHistory = new ArrayBuffer[Double](numIterations)
+    val stochasticLossHistory = new ArrayBuffer[Double](numIterations + 1)
     // Record previous weight and current one to calculate solution vector difference
 
     var previousWeights: Option[Vector] = None
@@ -226,7 +238,7 @@ object GradientDescent extends Logging {
 
     var converged = false // indicates whether converged based on convergenceTol
     var i = 1
-    while (!converged && i <= numIterations) {
+    while (!converged && (i <= numIterations + 1)) {
       val bcWeights = data.context.broadcast(weights)
       // Sample a subset (fraction miniBatchFraction) of the total data
       // compute and sum up the subgradients on this subset (this is one map-reduce)
@@ -249,17 +261,19 @@ object GradientDescent extends Logging {
          * and regVal is the regularization value computed in the previous iteration as well.
          */
         stochasticLossHistory += lossSum / miniBatchSize + regVal
-        val update = updater.compute(
-          weights, Vectors.fromBreeze(gradientSum / miniBatchSize.toDouble),
-          stepSize, i, regParam)
-        weights = update._1
-        regVal = update._2
-
-        previousWeights = currentWeights
-        currentWeights = Some(weights)
-        if (previousWeights != None && currentWeights != None) {
-          converged = isConverged(previousWeights.get,
-            currentWeights.get, convergenceTol)
+        if (i != (numIterations + 1)) {
+          val update = updater.compute(
+            weights, Vectors.fromBreeze(gradientSum / miniBatchSize.toDouble),
+            stepSize, i, regParam)
+          weights = update._1
+          regVal = update._2
+
+          previousWeights = currentWeights
+          currentWeights = Some(weights)
+          if (previousWeights != None && currentWeights != None) {
+            converged = isConverged(previousWeights.get,
+              currentWeights.get, convergenceTol)
+          }
         }
       } else {
         logWarning(s"Iteration ($i/$numIterations). The size of sampled batch is zero")
@@ -271,7 +285,6 @@ object GradientDescent extends Logging {
       stochasticLossHistory.takeRight(10).mkString(", ")))
 
     (weights, stochasticLossHistory.toArray)
-
   }
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala
index 1ee9241104f87..4fc297560c088 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala
@@ -136,7 +136,14 @@ class LBFGS(private var gradient: Gradient, private var updater: Updater)
   }
 
   override def optimize(data: RDD[(Double, Vector)], initialWeights: Vector): Vector = {
-    val (weights, _) = LBFGS.runLBFGS(
+    val (weights, _) = optimizeWithLossReturned(data, initialWeights)
+    weights
+  }
+
+  def optimizeWithLossReturned(
+      data: RDD[(Double, Vector)],
+      initialWeights: Vector): (Vector, Array[Double]) = {
+    LBFGS.runLBFGS(
       data,
       gradient,
       updater,
@@ -145,9 +152,7 @@ class LBFGS(private var gradient: Gradient, private var updater: Updater)
       maxNumIterations,
       regParam,
       initialWeights)
-    weights
   }
-
 }
 
 /**
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/NNLS.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/NNLS.scala
index 86632ae335957..e070d605b1647 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/NNLS.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/NNLS.scala
@@ -19,7 +19,7 @@ package org.apache.spark.mllib.optimization
 
 import java.{util => ju}
 
-import com.github.fommil.netlib.BLAS.{getInstance => blas}
+import org.apache.spark.ml.linalg.BLAS
 
 /**
  * Object used to solve nonnegative least squares problems using a modified
@@ -75,10 +75,10 @@ private[spark] object NNLS {
 
     // find the optimal unconstrained step
     def steplen(dir: Array[Double], res: Array[Double]): Double = {
-      val top = blas.ddot(n, dir, 1, res, 1)
-      blas.dgemv("N", n, n, 1.0, ata, n, dir, 1, 0.0, scratch, 1)
+      val top = BLAS.nativeBLAS.ddot(n, dir, 1, res, 1)
+      BLAS.nativeBLAS.dgemv("N", n, n, 1.0, ata, n, dir, 1, 0.0, scratch, 1)
       // Push the denominator upward very slightly to avoid infinities and silliness
-      top / (blas.ddot(n, scratch, 1, dir, 1) + 1e-20)
+      top / (BLAS.nativeBLAS.ddot(n, scratch, 1, dir, 1) + 1e-20)
     }
 
     // stopping condition
@@ -103,9 +103,9 @@ private[spark] object NNLS {
     var i = 0
     while (iterno < iterMax) {
       // find the residual
-      blas.dgemv("N", n, n, 1.0, ata, n, x, 1, 0.0, res, 1)
-      blas.daxpy(n, -1.0, atb, 1, res, 1)
-      blas.dcopy(n, res, 1, grad, 1)
+      BLAS.nativeBLAS.dgemv("N", n, n, 1.0, ata, n, x, 1, 0.0, res, 1)
+      BLAS.nativeBLAS.daxpy(n, -1.0, atb, 1, res, 1)
+      BLAS.nativeBLAS.dcopy(n, res, 1, grad, 1)
 
       // project the gradient
       i = 0
@@ -115,28 +115,28 @@ private[spark] object NNLS {
         }
         i = i + 1
       }
-      val ngrad = blas.ddot(n, grad, 1, grad, 1)
+      val ngrad = BLAS.nativeBLAS.ddot(n, grad, 1, grad, 1)
 
-      blas.dcopy(n, grad, 1, dir, 1)
+      BLAS.nativeBLAS.dcopy(n, grad, 1, dir, 1)
 
       // use a CG direction under certain conditions
       var step = steplen(grad, res)
       var ndir = 0.0
-      val nx = blas.ddot(n, x, 1, x, 1)
+      val nx = BLAS.nativeBLAS.ddot(n, x, 1, x, 1)
       if (iterno > lastWall + 1) {
         val alpha = ngrad / lastNorm
-        blas.daxpy(n, alpha, lastDir, 1, dir, 1)
+        BLAS.nativeBLAS.daxpy(n, alpha, lastDir, 1, dir, 1)
         val dstep = steplen(dir, res)
-        ndir = blas.ddot(n, dir, 1, dir, 1)
+        ndir = BLAS.nativeBLAS.ddot(n, dir, 1, dir, 1)
         if (stop(dstep, ndir, nx)) {
           // reject the CG step if it could lead to premature termination
-          blas.dcopy(n, grad, 1, dir, 1)
-          ndir = blas.ddot(n, dir, 1, dir, 1)
+          BLAS.nativeBLAS.dcopy(n, grad, 1, dir, 1)
+          ndir = BLAS.nativeBLAS.ddot(n, dir, 1, dir, 1)
         } else {
           step = dstep
         }
       } else {
-        ndir = blas.ddot(n, dir, 1, dir, 1)
+        ndir = BLAS.nativeBLAS.ddot(n, dir, 1, dir, 1)
       }
 
       // terminate?
@@ -166,7 +166,7 @@ private[spark] object NNLS {
       }
 
       iterno = iterno + 1
-      blas.dcopy(n, dir, 1, lastDir, 1)
+      BLAS.nativeBLAS.dcopy(n, dir, 1, lastDir, 1)
       lastNorm = ngrad
     }
     x.clone
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/rdd/SlidingRDD.scala b/mllib/src/main/scala/org/apache/spark/mllib/rdd/SlidingRDD.scala
index 365b2a06110f6..c669ced61d2f7 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/rdd/SlidingRDD.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/rdd/SlidingRDD.scala
@@ -97,7 +97,7 @@ class SlidingRDD[T: ClassTag](@transient val parent: RDD[T], val windowSize: Int
           }
           if (sizes(i) + tail.length >= offset + windowSize) {
             partitions +=
-              new SlidingRDDPartition[T](partitionIndex, parentPartitions(i), tail, offset)
+              new SlidingRDDPartition[T](partitionIndex, parentPartitions(i), tail.toSeq, offset)
             partitionIndex += 1
           }
         }
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModel.scala b/mllib/src/main/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModel.scala
index e5e82d19f1cbd..3276513213f5d 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModel.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModel.scala
@@ -21,7 +21,7 @@ import java.io.IOException
 import java.lang.{Integer => JavaInteger}
 
 import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus
-import com.github.fommil.netlib.BLAS.{getInstance => blas}
+import com.google.common.collect.{Ordering => GuavaOrdering}
 import org.apache.hadoop.fs.Path
 import org.json4s._
 import org.json4s.JsonDSL._
@@ -37,7 +37,6 @@ import org.apache.spark.mllib.util.{Loader, Saveable}
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{Row, SparkSession}
 import org.apache.spark.storage.StorageLevel
-import org.apache.spark.util.BoundedPriorityQueue
 
 /**
  * Model representing the result of matrix factorization.
@@ -85,7 +84,7 @@ class MatrixFactorizationModel @Since("0.8.0") (
 
     val userVector = userFeatureSeq.head
     val productVector = productFeatureSeq.head
-    blas.ddot(rank, userVector, 1, productVector, 1)
+    BLAS.nativeBLAS.ddot(rank, userVector, 1, productVector, 1)
   }
 
   /**
@@ -136,7 +135,7 @@ class MatrixFactorizationModel @Since("0.8.0") (
       }
       users.join(productFeatures).map {
         case (product, ((user, uFeatures), pFeatures)) =>
-          Rating(user, product, blas.ddot(uFeatures.length, uFeatures, 1, pFeatures, 1))
+          Rating(user, product, BLAS.nativeBLAS.ddot(uFeatures.length, uFeatures, 1, pFeatures, 1))
       }
     } else {
       val products = productFeatures.join(usersProducts.map(_.swap)).map {
@@ -144,7 +143,7 @@ class MatrixFactorizationModel @Since("0.8.0") (
       }
       products.join(userFeatures).map {
         case (user, ((product, pFeatures), uFeatures)) =>
-          Rating(user, product, blas.ddot(uFeatures.length, uFeatures, 1, pFeatures, 1))
+          Rating(user, product, BLAS.nativeBLAS.ddot(uFeatures.length, uFeatures, 1, pFeatures, 1))
       }
     }
   }
@@ -263,7 +262,7 @@ object MatrixFactorizationModel extends Loader[MatrixFactorizationModel] {
       recommendableFeatures: RDD[(Int, Array[Double])],
       num: Int): Array[(Int, Double)] = {
     val scored = recommendableFeatures.map { case (id, features) =>
-      (id, blas.ddot(features.length, recommendToFeatures, 1, features, 1))
+      (id, BLAS.nativeBLAS.ddot(features.length, recommendToFeatures, 1, features, 1))
     }
     scored.top(num)(Ordering.by(_._2))
   }
@@ -277,11 +276,12 @@ object MatrixFactorizationModel extends Loader[MatrixFactorizationModel] {
    * arrays required for intermediate result storage, as well as a high sensitivity to the
    * block size used.
    *
-   * The following approach still groups factors into blocks, but instead computes the
-   * top-k elements per block, using dot product and an efficient [[BoundedPriorityQueue]]
-   * (instead of gemm). This avoids any large intermediate data structures and results
-   * in significantly reduced GC pressure as well as shuffle data, which far outweighs
-   * any cost incurred from not using Level 3 BLAS operations.
+   * This approach groups factors into blocks and computes the top-k elements per block,
+   * using GEMV (it use less memory compared with GEMM, and is much faster than DOT) and
+   * an efficient selection based on [[GuavaOrdering]] (instead of [[BoundedPriorityQueue]]).
+   * It then computes the global top-k by aggregating the per block top-k elements with
+   * a [[BoundedPriorityQueue]]. This significantly reduces the size of intermediate and
+   * shuffle data.
    *
    * @param rank rank
    * @param srcFeatures src features to receive recommendations
@@ -295,28 +295,40 @@ object MatrixFactorizationModel extends Loader[MatrixFactorizationModel] {
       srcFeatures: RDD[(Int, Array[Double])],
       dstFeatures: RDD[(Int, Array[Double])],
       num: Int): RDD[(Int, Array[(Int, Double)])] = {
+    import scala.collection.JavaConverters._
     val srcBlocks = blockify(srcFeatures)
     val dstBlocks = blockify(dstFeatures)
-    val ratings = srcBlocks.cartesian(dstBlocks).flatMap { case (srcIter, dstIter) =>
-      val m = srcIter.size
-      val n = math.min(dstIter.size, num)
-      val output = new Array[(Int, (Int, Double))](m * n)
-      var i = 0
-      val pq = new BoundedPriorityQueue[(Int, Double)](n)(Ordering.by(_._2))
-      srcIter.foreach { case (srcId, srcFactor) =>
-        dstIter.foreach { case (dstId, dstFactor) =>
-          // We use F2jBLAS which is faster than a call to native BLAS for vector dot product
-          val score = BLAS.f2jBLAS.ddot(rank, srcFactor, 1, dstFactor, 1)
-          pq += dstId -> score
-        }
-        pq.foreach { case (dstId, score) =>
-          output(i) = (srcId, (dstId, score))
-          i += 1
+
+    val ratings = srcBlocks.cartesian(dstBlocks)
+      .mapPartitions { iter =>
+        var scores: Array[Double] = null
+        var idxOrd: GuavaOrdering[Int] = null
+        iter.flatMap { case ((srcIds, srcMat), (dstIds, dstMat)) =>
+          require(srcMat.length == srcIds.length * rank)
+          require(dstMat.length == dstIds.length * rank)
+          val m = srcIds.length
+          val n = dstIds.length
+          if (scores == null || scores.length < n) {
+            scores = Array.ofDim[Double](n)
+            idxOrd = new GuavaOrdering[Int] {
+              override def compare(left: Int, right: Int): Int = {
+                Ordering[Double].compare(scores(left), scores(right))
+              }
+            }
+          }
+
+          Iterator.range(0, m).flatMap { i =>
+            // scores = i-th vec in srcMat * dstMat
+            BLAS.javaBLAS.dgemv("T", rank, n, 1.0F, dstMat, 0, rank,
+              srcMat, i * rank, 1, 0.0F, scores, 0, 1)
+
+            val srcId = srcIds(i)
+            idxOrd.greatestOf(Iterator.range(0, n).asJava, num).asScala
+              .iterator.map { j => (srcId, (dstIds(j), scores(j))) }
+          }
         }
-        pq.clear()
       }
-      output.toSeq
-    }
+
     ratings.topByKey(num)(Ordering.by(_._2))
   }
 
@@ -326,9 +338,10 @@ object MatrixFactorizationModel extends Loader[MatrixFactorizationModel] {
    */
   private def blockify(
       features: RDD[(Int, Array[Double])],
-      blockSize: Int = 4096): RDD[Seq[(Int, Array[Double])]] = {
+      blockSize: Int = 4096): RDD[(Array[Int], Array[Double])] = {
     features.mapPartitions { iter =>
       iter.grouped(blockSize)
+        .map(block => (block.map(_._1).toArray, block.flatMap(_._2).toArray))
     }
   }
 
@@ -386,12 +399,12 @@ object MatrixFactorizationModel extends Loader[MatrixFactorizationModel] {
       assert(formatVersion == thisFormatVersion)
       val rank = (metadata \ "rank").extract[Int]
       val userFeatures = spark.read.parquet(userPath(path)).rdd.map {
-        case Row(id: Int, features: Seq[_]) =>
-          (id, features.asInstanceOf[Seq[Double]].toArray)
+        case Row(id: Int, features: scala.collection.Seq[_]) =>
+          (id, features.asInstanceOf[scala.collection.Seq[Double]].toArray)
       }
       val productFeatures = spark.read.parquet(productPath(path)).rdd.map {
-        case Row(id: Int, features: Seq[_]) =>
-          (id, features.asInstanceOf[Seq[Double]].toArray)
+        case Row(id: Int, features: scala.collection.Seq[_]) =>
+          (id, features.asInstanceOf[scala.collection.Seq[Double]].toArray)
       }
       new MatrixFactorizationModel(rank, userFeatures, productFeatures)
     }
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/stat/KernelDensity.scala b/mllib/src/main/scala/org/apache/spark/mllib/stat/KernelDensity.scala
index f253963270bc4..f0236f0528a21 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/stat/KernelDensity.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/stat/KernelDensity.scala
@@ -17,10 +17,9 @@
 
 package org.apache.spark.mllib.stat
 
-import com.github.fommil.netlib.BLAS.{getInstance => blas}
-
 import org.apache.spark.annotation.Since
 import org.apache.spark.api.java.JavaRDD
+import org.apache.spark.ml.linalg.BLAS
 import org.apache.spark.rdd.RDD
 
 /**
@@ -99,10 +98,10 @@ class KernelDensity extends Serializable {
         (x._1, x._2 + 1)
       },
       (x, y) => {
-        blas.daxpy(n, 1.0, y._1, 1, x._1, 1)
+        BLAS.nativeBLAS.daxpy(n, 1.0, y._1, 1, x._1, 1)
         (x._1, x._2 + y._2)
       })
-    blas.dscal(n, 1.0 / count, densities, 1)
+    BLAS.nativeBLAS.dscal(n, 1.0 / count, densities, 1)
     densities
   }
 }
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/stat/test/KolmogorovSmirnovTest.scala b/mllib/src/main/scala/org/apache/spark/mllib/stat/test/KolmogorovSmirnovTest.scala
index d17f7047c5b2b..778de30e756c0 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/stat/test/KolmogorovSmirnovTest.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/stat/test/KolmogorovSmirnovTest.scala
@@ -91,7 +91,7 @@ private[stat] object KolmogorovSmirnovTest extends Logging {
    * @param partData `Iterator[Double]` 1 partition of a sorted RDD
    * @param n `Double` the total size of the RDD
    * @param cdf `Double => Double` a function the calculates the theoretical CDF of a value
-   * @return `Iterator[(Double, Double)] `Unadjusted (ie. off by a constant) potential extrema
+   * @return `Iterator[(Double, Double)] `Unadjusted (i.e. off by a constant) potential extrema
    *        in a partition. The first element corresponds to the (empirical CDF - 1/N) - CDF,
    *        the second element corresponds to empirical CDF - CDF.  We can then search the resulting
    *        iterator for the minimum of the first and the maximum of the second element, and provide
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/configuration/Strategy.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/configuration/Strategy.scala
index 09e3e22030546..0f6c7033687fa 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/configuration/Strategy.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/configuration/Strategy.scala
@@ -116,7 +116,7 @@ class Strategy @Since("1.3.0") (
       maxMemoryInMB: Int,
       subsamplingRate: Double,
       useNodeIdCache: Boolean,
-      checkpointInterval: Int) {
+      checkpointInterval: Int) = {
     this(algo, impurity, maxDepth, numClasses, maxBins, quantileCalculationStrategy,
       categoricalFeaturesInfo, minInstancesPerNode, minInfoGain, maxMemoryInMB,
       subsamplingRate, useNodeIdCache, checkpointInterval, 0.0)
@@ -133,7 +133,7 @@ class Strategy @Since("1.3.0") (
       maxDepth: Int,
       numClasses: Int,
       maxBins: Int,
-      categoricalFeaturesInfo: java.util.Map[java.lang.Integer, java.lang.Integer]) {
+      categoricalFeaturesInfo: java.util.Map[java.lang.Integer, java.lang.Integer]) = {
     this(algo, impurity, maxDepth, numClasses, maxBins, Sort,
       categoricalFeaturesInfo.asInstanceOf[java.util.Map[Int, Int]].asScala.toMap,
       minWeightFractionPerNode = 0.0)
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Entropy.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Entropy.scala
index 6e2732f7ae7aa..c3bda99786310 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Entropy.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Entropy.scala
@@ -112,7 +112,7 @@ private[spark] class EntropyAggregator(numClasses: Int)
    * @param offset    Start index of stats for this (node, feature, bin).
    */
   def getCalculator(allStats: Array[Double], offset: Int): EntropyCalculator = {
-    new EntropyCalculator(allStats.view(offset, offset + statsSize - 1).toArray,
+    new EntropyCalculator(allStats.view.slice(offset, offset + statsSize - 1).toArray,
       allStats(offset + statsSize - 1).toLong)
   }
 }
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Gini.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Gini.scala
index 5983118c05754..70163b56408a4 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Gini.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Gini.scala
@@ -107,7 +107,7 @@ private[spark] class GiniAggregator(numClasses: Int)
    * @param offset    Start index of stats for this (node, feature, bin).
    */
   def getCalculator(allStats: Array[Double], offset: Int): GiniCalculator = {
-    new GiniCalculator(allStats.view(offset, offset + statsSize - 1).toArray,
+    new GiniCalculator(allStats.view.slice(offset, offset + statsSize - 1).toArray,
       allStats(offset + statsSize - 1).toLong)
   }
 }
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Variance.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Variance.scala
index f5b2f8d514c7e..7143fd07d7333 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Variance.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Variance.scala
@@ -95,7 +95,7 @@ private[spark] class VarianceAggregator()
    * @param offset    Start index of stats for this (node, feature, bin).
    */
   def getCalculator(allStats: Array[Double], offset: Int): VarianceCalculator = {
-    new VarianceCalculator(allStats.view(offset, offset + statsSize - 1).toArray,
+    new VarianceCalculator(allStats.view.slice(offset, offset + statsSize - 1).toArray,
       allStats(offset + statsSize - 1).toLong)
   }
 }
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/DecisionTreeModel.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/DecisionTreeModel.scala
index 9983ca7dc5e87..cdc998000c2fc 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/DecisionTreeModel.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/DecisionTreeModel.scala
@@ -164,7 +164,7 @@ object DecisionTreeModel extends Loader[DecisionTreeModel] with Logging {
       }
 
       def apply(r: Row): SplitData = {
-        SplitData(r.getInt(0), r.getDouble(1), r.getInt(2), r.getAs[Seq[Double]](3))
+        SplitData(r.getInt(0), r.getDouble(1), r.getInt(2), r.getSeq[Double](3))
       }
     }
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala
index c5069277fad68..1f879a4d9dfbb 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala
@@ -19,7 +19,6 @@ package org.apache.spark.mllib.tree.model
 
 import scala.collection.mutable
 
-import com.github.fommil.netlib.BLAS.{getInstance => blas}
 import org.json4s._
 import org.json4s.JsonDSL._
 import org.json4s.jackson.JsonMethods._
@@ -28,6 +27,7 @@ import org.apache.spark.SparkContext
 import org.apache.spark.annotation.Since
 import org.apache.spark.api.java.JavaRDD
 import org.apache.spark.internal.Logging
+import org.apache.spark.ml.linalg.BLAS
 import org.apache.spark.mllib.linalg.Vector
 import org.apache.spark.mllib.regression.LabeledPoint
 import org.apache.spark.mllib.tree.configuration.Algo
@@ -280,7 +280,7 @@ private[tree] sealed class TreeEnsembleModel(
    */
   private def predictBySumming(features: Vector): Double = {
     val treePredictions = trees.map(_.predict(features))
-    blas.ddot(numTrees, treePredictions, 1, treeWeights, 1)
+    BLAS.nativeBLAS.ddot(numTrees, treePredictions, 1, treeWeights, 1)
   }
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala
index 689ed0f842189..1ad8d93a445a2 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala
@@ -179,7 +179,7 @@ object LinearDataGenerator {
     val data: RDD[LabeledPoint] = sc.parallelize(0 until nparts, nparts).flatMap { p =>
       val seed = 42 + p
       val examplesInPartition = nexamples / nparts
-      generateLinearInput(intercept, w.toArray, examplesInPartition, seed, eps)
+      generateLinearInput(intercept, w, examplesInPartition, seed, eps)
     }
     data
   }
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala
index 9198334ba02a1..7bce38cc38a08 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala
@@ -20,7 +20,7 @@ package org.apache.spark.mllib.util
 import scala.annotation.varargs
 import scala.reflect.ClassTag
 
-import org.apache.spark.SparkContext
+import org.apache.spark.{SparkContext, SparkException}
 import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.linalg.{MatrixUDT => MLMatrixUDT, VectorUDT => MLVectorUDT}
@@ -28,7 +28,7 @@ import org.apache.spark.mllib.linalg._
 import org.apache.spark.mllib.linalg.BLAS.dot
 import org.apache.spark.mllib.regression.LabeledPoint
 import org.apache.spark.rdd.{PartitionwiseSampledRDD, RDD}
-import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}
+import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
 import org.apache.spark.sql.execution.datasources.DataSource
 import org.apache.spark.sql.execution.datasources.text.TextFileFormat
 import org.apache.spark.sql.functions._
@@ -105,12 +105,15 @@ object MLUtils extends Logging {
   }
 
   private[spark] def parseLibSVMFile(
-      sparkSession: SparkSession, paths: Seq[String]): RDD[(Double, Array[Int], Array[Double])] = {
+      sparkSession: SparkSession,
+      paths: Seq[String],
+      options: Map[String, String]): RDD[(Double, Array[Int], Array[Double])] = {
     val lines = sparkSession.baseRelationToDataFrame(
       DataSource.apply(
         sparkSession,
         paths = paths,
-        className = classOf[TextFileFormat].getName
+        className = classOf[TextFileFormat].getName,
+        options = options ++ Map(DataSource.GLOB_PATHS_KEY -> "false")
       ).resolveRelation(checkFilesExist = false))
       .select("value")
 
@@ -144,7 +147,7 @@ object MLUtils extends Logging {
       previous = current
       i += 1
     }
-    (label, indices.toArray, values.toArray)
+    (label, indices, values)
   }
 
   /**
@@ -248,6 +251,36 @@ object MLUtils extends Logging {
     }.toArray
   }
 
+  /**
+   * Version of `kFold()` taking a fold column name.
+   */
+  @Since("3.1.0")
+  def kFold(df: DataFrame, numFolds: Int, foldColName: String): Array[(RDD[Row], RDD[Row])] = {
+    val foldCol = df.col(foldColName)
+    val checker = udf { foldNum: Int =>
+      // Valid fold number is in range [0, numFolds).
+      if (foldNum < 0 || foldNum >= numFolds) {
+        throw new SparkException(s"Fold number must be in range [0, $numFolds), but got $foldNum.")
+      }
+      true
+    }
+    (0 until numFolds).map { fold =>
+      val training = df
+        .filter(checker(foldCol) && foldCol =!= fold)
+        .drop(foldColName).rdd
+      val validation = df
+        .filter(checker(foldCol) && foldCol === fold)
+        .drop(foldColName).rdd
+      if (training.isEmpty()) {
+        throw new SparkException(s"The training data at fold $fold is empty.")
+      }
+      if (validation.isEmpty()) {
+        throw new SparkException(s"The validation data at fold $fold is empty.")
+      }
+      (training, validation)
+    }.toArray
+  }
+
   /**
    * Returns a new vector with `1.0` (bias) appended to the input vector.
    */
@@ -504,8 +537,10 @@ object MLUtils extends Logging {
       norm2: Double,
       precision: Double = 1e-6): Double = {
     val n = v1.size
-    require(v2.size == n)
-    require(norm1 >= 0.0 && norm2 >= 0.0)
+    require(v2.size == n,
+      s"Both vectors should have same length, found v1 is $n while v2 is ${v2.size}")
+    require(norm1 >= 0.0 && norm2 >= 0.0,
+      s"Both norms should be greater or equal to 0.0, found norm1=$norm1, norm2=$norm2")
     var sqDist = 0.0
     /*
      * The relative error is
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/NumericParser.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/NumericParser.scala
index 2c613348c2d92..959e54e4c7169 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/NumericParser.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/NumericParser.scala
@@ -85,10 +85,10 @@ private[mllib] object NumericParser {
     while (parsing && tokenizer.hasMoreTokens()) {
       token = tokenizer.nextToken()
       if (token == "(") {
-        items.append(parseTuple(tokenizer))
+        items += parseTuple(tokenizer)
         allowComma = true
       } else if (token == "[") {
-        items.append(parseArray(tokenizer))
+        items += parseArray(tokenizer)
         allowComma = true
       } else if (token == ",") {
         if (allowComma) {
@@ -102,14 +102,14 @@ private[mllib] object NumericParser {
           // ignore whitespaces between delim chars, e.g. ", ["
       } else {
         // expecting a number
-        items.append(parseDouble(token))
+        items += parseDouble(token)
         allowComma = true
       }
     }
     if (parsing) {
       throw new SparkException(s"A tuple must end with ')'.")
     }
-    items
+    items.toSeq
   }
 
   private def parseDouble(s: String): Double = {
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/SVMDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/SVMDataGenerator.scala
index 9fffa508afbfb..0f99cef665eaf 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/SVMDataGenerator.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/SVMDataGenerator.scala
@@ -19,10 +19,9 @@ package org.apache.spark.mllib.util
 
 import scala.util.Random
 
-import com.github.fommil.netlib.BLAS.{getInstance => blas}
-
 import org.apache.spark.SparkContext
 import org.apache.spark.annotation.Since
+import org.apache.spark.ml.linalg.BLAS
 import org.apache.spark.mllib.linalg.Vectors
 import org.apache.spark.mllib.regression.LabeledPoint
 import org.apache.spark.rdd.RDD
@@ -61,7 +60,8 @@ object SVMDataGenerator {
       val x = Array.fill[Double](nfeatures) {
         rnd.nextDouble() * 2.0 - 1.0
       }
-      val yD = blas.ddot(trueWeights.length, x, 1, trueWeights, 1) + rnd.nextGaussian() * 0.1
+      val yD = BLAS.nativeBLAS.ddot(trueWeights.length, x, 1, trueWeights, 1)
+                + rnd.nextGaussian() * 0.1
       val y = if (yD < 0) 0.0 else 1.0
       LabeledPoint(y, Vectors.dense(x))
     }
diff --git a/mllib/src/test/java/org/apache/spark/ml/classification/JavaLogisticRegressionSuite.java b/mllib/src/test/java/org/apache/spark/ml/classification/JavaLogisticRegressionSuite.java
index 49ac49339415a..7c63a8755b4f3 100644
--- a/mllib/src/test/java/org/apache/spark/ml/classification/JavaLogisticRegressionSuite.java
+++ b/mllib/src/test/java/org/apache/spark/ml/classification/JavaLogisticRegressionSuite.java
@@ -143,6 +143,6 @@ public void logisticRegressionTrainingSummary() {
     LogisticRegressionModel model = lr.fit(dataset);
 
     LogisticRegressionTrainingSummary summary = model.summary();
-    Assert.assertEquals(summary.totalIterations(), summary.objectiveHistory().length);
+    Assert.assertEquals(summary.totalIterations(), summary.objectiveHistory().length - 1);
   }
 }
diff --git a/mllib/src/test/java/org/apache/spark/ml/feature/JavaStopWordsRemoverSuite.java b/mllib/src/test/java/org/apache/spark/ml/feature/JavaStopWordsRemoverSuite.java
index 6480b57e1f796..af32e03854b53 100644
--- a/mllib/src/test/java/org/apache/spark/ml/feature/JavaStopWordsRemoverSuite.java
+++ b/mllib/src/test/java/org/apache/spark/ml/feature/JavaStopWordsRemoverSuite.java
@@ -41,7 +41,7 @@ public void javaCompatibilityTest() {
       .setOutputCol("filtered");
 
     List<Row> data = Arrays.asList(
-      RowFactory.create(Arrays.asList("I", "saw", "the", "red", "baloon")),
+      RowFactory.create(Arrays.asList("I", "saw", "the", "red", "balloon")),
       RowFactory.create(Arrays.asList("Mary", "had", "a", "little", "lamb"))
     );
     StructType schema = new StructType(new StructField[]{
diff --git a/mllib/src/test/resources/ml-models/dtc-2.4.7/data/._SUCCESS.crc b/mllib/src/test/resources/ml-models/dtc-2.4.7/data/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/dtc-2.4.7/data/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/dtc-2.4.7/data/.part-00000-bd7ae42f-c890-406c-894c-ca4eac67c690-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/dtc-2.4.7/data/.part-00000-bd7ae42f-c890-406c-894c-ca4eac67c690-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000..3ac562a8b2b84
Binary files /dev/null and b/mllib/src/test/resources/ml-models/dtc-2.4.7/data/.part-00000-bd7ae42f-c890-406c-894c-ca4eac67c690-c000.snappy.parquet.crc differ
diff --git a/mllib/src/test/resources/ml-models/dtc-2.4.7/data/_SUCCESS b/mllib/src/test/resources/ml-models/dtc-2.4.7/data/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/dtc-2.4.7/data/part-00000-bd7ae42f-c890-406c-894c-ca4eac67c690-c000.snappy.parquet b/mllib/src/test/resources/ml-models/dtc-2.4.7/data/part-00000-bd7ae42f-c890-406c-894c-ca4eac67c690-c000.snappy.parquet
new file mode 100644
index 0000000000000..09c38d602c105
Binary files /dev/null and b/mllib/src/test/resources/ml-models/dtc-2.4.7/data/part-00000-bd7ae42f-c890-406c-894c-ca4eac67c690-c000.snappy.parquet differ
diff --git a/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/.part-00000.crc b/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/.part-00000.crc
new file mode 100644
index 0000000000000..22b1eb8089ce3
Binary files /dev/null and b/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/.part-00000.crc differ
diff --git a/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/_SUCCESS b/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/part-00000 b/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/part-00000
new file mode 100644
index 0000000000000..ef92265f89fff
--- /dev/null
+++ b/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/part-00000
@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.classification.DecisionTreeClassificationModel","timestamp":1608687929358,"sparkVersion":"2.4.7","uid":"dtc_bc7ad285bb73","paramMap":{},"defaultParamMap":{"impurity":"gini","maxDepth":5,"labelCol":"label","maxMemoryInMB":256,"featuresCol":"features","predictionCol":"prediction","minInfoGain":0.0,"seed":159147643,"rawPredictionCol":"rawPrediction","minInstancesPerNode":1,"cacheNodeIds":false,"probabilityCol":"probability","maxBins":32,"checkpointInterval":10},"numFeatures":692,"numClasses":2}
diff --git a/mllib/src/test/resources/ml-models/dtr-2.4.7/data/._SUCCESS.crc b/mllib/src/test/resources/ml-models/dtr-2.4.7/data/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/dtr-2.4.7/data/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/dtr-2.4.7/data/.part-00000-39b027f0-a437-4b3d-84af-d861adcb9ca8-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/dtr-2.4.7/data/.part-00000-39b027f0-a437-4b3d-84af-d861adcb9ca8-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000..f6465e27d902b
Binary files /dev/null and b/mllib/src/test/resources/ml-models/dtr-2.4.7/data/.part-00000-39b027f0-a437-4b3d-84af-d861adcb9ca8-c000.snappy.parquet.crc differ
diff --git a/mllib/src/test/resources/ml-models/dtr-2.4.7/data/_SUCCESS b/mllib/src/test/resources/ml-models/dtr-2.4.7/data/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/dtr-2.4.7/data/part-00000-39b027f0-a437-4b3d-84af-d861adcb9ca8-c000.snappy.parquet b/mllib/src/test/resources/ml-models/dtr-2.4.7/data/part-00000-39b027f0-a437-4b3d-84af-d861adcb9ca8-c000.snappy.parquet
new file mode 100644
index 0000000000000..2904f8433b78f
Binary files /dev/null and b/mllib/src/test/resources/ml-models/dtr-2.4.7/data/part-00000-39b027f0-a437-4b3d-84af-d861adcb9ca8-c000.snappy.parquet differ
diff --git a/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/.part-00000.crc b/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/.part-00000.crc
new file mode 100644
index 0000000000000..bbad108766e14
Binary files /dev/null and b/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/.part-00000.crc differ
diff --git a/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/_SUCCESS b/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/part-00000 b/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/part-00000
new file mode 100644
index 0000000000000..2895223cffde6
--- /dev/null
+++ b/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/part-00000
@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.regression.DecisionTreeRegressionModel","timestamp":1608687932847,"sparkVersion":"2.4.7","uid":"dtr_c16a90fcdaf8","paramMap":{},"defaultParamMap":{"labelCol":"label","checkpointInterval":10,"minInfoGain":0.0,"maxMemoryInMB":256,"minInstancesPerNode":1,"maxBins":32,"seed":926680331,"cacheNodeIds":false,"maxDepth":5,"predictionCol":"prediction","featuresCol":"features","impurity":"variance"},"numFeatures":692}
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/._SUCCESS.crc b/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/.part-00000-dacbde64-c861-41c7-91c0-6da8cc01fb43-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/.part-00000-dacbde64-c861-41c7-91c0-6da8cc01fb43-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000..13fc4ed696eff
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/.part-00000-dacbde64-c861-41c7-91c0-6da8cc01fb43-c000.snappy.parquet.crc differ
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/_SUCCESS b/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/part-00000-dacbde64-c861-41c7-91c0-6da8cc01fb43-c000.snappy.parquet b/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/part-00000-dacbde64-c861-41c7-91c0-6da8cc01fb43-c000.snappy.parquet
new file mode 100644
index 0000000000000..5682038fcc764
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/part-00000-dacbde64-c861-41c7-91c0-6da8cc01fb43-c000.snappy.parquet differ
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/.part-00000.crc b/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/.part-00000.crc
new file mode 100644
index 0000000000000..a810dd9107fe7
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/.part-00000.crc differ
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/_SUCCESS b/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/part-00000 b/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/part-00000
new file mode 100644
index 0000000000000..675fea29ba9e2
--- /dev/null
+++ b/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/part-00000
@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.classification.GBTClassificationModel","timestamp":1608687932103,"sparkVersion":"2.4.7","uid":"gbtc_81db008b4f25","paramMap":{"maxIter":2},"defaultParamMap":{"seed":-1287390502,"maxMemoryInMB":256,"stepSize":0.1,"validationTol":0.01,"maxBins":32,"checkpointInterval":10,"predictionCol":"prediction","lossType":"logistic","rawPredictionCol":"rawPrediction","featuresCol":"features","cacheNodeIds":false,"maxIter":20,"featureSubsetStrategy":"all","impurity":"gini","minInstancesPerNode":1,"minInfoGain":0.0,"maxDepth":5,"subsamplingRate":1.0,"labelCol":"label","probabilityCol":"probability"},"numFeatures":692,"numTrees":2}
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/.part-00000-81137d9f-31e3-4a90-813c-ddc394101e21-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/.part-00000-81137d9f-31e3-4a90-813c-ddc394101e21-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000..101c207119388
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/.part-00000-81137d9f-31e3-4a90-813c-ddc394101e21-c000.snappy.parquet.crc differ
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/_SUCCESS b/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/part-00000-81137d9f-31e3-4a90-813c-ddc394101e21-c000.snappy.parquet b/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/part-00000-81137d9f-31e3-4a90-813c-ddc394101e21-c000.snappy.parquet
new file mode 100644
index 0000000000000..e232de3400423
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/part-00000-81137d9f-31e3-4a90-813c-ddc394101e21-c000.snappy.parquet differ
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/._SUCCESS.crc b/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/.part-00000-3b5433ff-d346-4511-9aab-639288bfae6d-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/.part-00000-3b5433ff-d346-4511-9aab-639288bfae6d-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000..c35b81f9cb25f
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/.part-00000-3b5433ff-d346-4511-9aab-639288bfae6d-c000.snappy.parquet.crc differ
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/_SUCCESS b/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/part-00000-3b5433ff-d346-4511-9aab-639288bfae6d-c000.snappy.parquet b/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/part-00000-3b5433ff-d346-4511-9aab-639288bfae6d-c000.snappy.parquet
new file mode 100644
index 0000000000000..ba26a44f46f11
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/part-00000-3b5433ff-d346-4511-9aab-639288bfae6d-c000.snappy.parquet differ
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/.part-00000.crc b/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/.part-00000.crc
new file mode 100644
index 0000000000000..7dc6e149db71e
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/.part-00000.crc differ
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/_SUCCESS b/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/part-00000 b/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/part-00000
new file mode 100644
index 0000000000000..a9a712e626510
--- /dev/null
+++ b/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/part-00000
@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.regression.GBTRegressionModel","timestamp":1608687942434,"sparkVersion":"2.4.7","uid":"gbtr_0a74cb2536ff","paramMap":{"maxIter":2},"defaultParamMap":{"impurity":"variance","maxMemoryInMB":256,"maxDepth":5,"subsamplingRate":1.0,"validationTol":0.01,"labelCol":"label","maxIter":20,"checkpointInterval":10,"minInfoGain":0.0,"predictionCol":"prediction","stepSize":0.1,"cacheNodeIds":false,"lossType":"squared","seed":-131597770,"featureSubsetStrategy":"all","featuresCol":"features","minInstancesPerNode":1,"maxBins":32},"numFeatures":692,"numTrees":2}
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/.part-00000-6b9124f5-87fe-4fd8-ad9c-4be239c2215a-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/.part-00000-6b9124f5-87fe-4fd8-ad9c-4be239c2215a-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000..b681b9f615bf5
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/.part-00000-6b9124f5-87fe-4fd8-ad9c-4be239c2215a-c000.snappy.parquet.crc differ
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/_SUCCESS b/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/part-00000-6b9124f5-87fe-4fd8-ad9c-4be239c2215a-c000.snappy.parquet b/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/part-00000-6b9124f5-87fe-4fd8-ad9c-4be239c2215a-c000.snappy.parquet
new file mode 100644
index 0000000000000..9a7e77acf5464
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/part-00000-6b9124f5-87fe-4fd8-ad9c-4be239c2215a-c000.snappy.parquet differ
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/data/._SUCCESS.crc b/mllib/src/test/resources/ml-models/rfc-2.4.7/data/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfc-2.4.7/data/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/data/.part-00000-e41a7b98-91f8-4485-b112-25b4b11c9009-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/rfc-2.4.7/data/.part-00000-e41a7b98-91f8-4485-b112-25b4b11c9009-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000..5bb3a22617543
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfc-2.4.7/data/.part-00000-e41a7b98-91f8-4485-b112-25b4b11c9009-c000.snappy.parquet.crc differ
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/data/_SUCCESS b/mllib/src/test/resources/ml-models/rfc-2.4.7/data/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/data/part-00000-e41a7b98-91f8-4485-b112-25b4b11c9009-c000.snappy.parquet b/mllib/src/test/resources/ml-models/rfc-2.4.7/data/part-00000-e41a7b98-91f8-4485-b112-25b4b11c9009-c000.snappy.parquet
new file mode 100644
index 0000000000000..d9ec35a4de580
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfc-2.4.7/data/part-00000-e41a7b98-91f8-4485-b112-25b4b11c9009-c000.snappy.parquet differ
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/.part-00000.crc b/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/.part-00000.crc
new file mode 100644
index 0000000000000..58bda6df9cca7
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/.part-00000.crc differ
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/_SUCCESS b/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/part-00000 b/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/part-00000
new file mode 100644
index 0000000000000..07748b070ee84
--- /dev/null
+++ b/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/part-00000
@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.classification.RandomForestClassificationModel","timestamp":1608687930713,"sparkVersion":"2.4.7","uid":"rfc_db1adb353f1e","paramMap":{"numTrees":2},"defaultParamMap":{"impurity":"gini","predictionCol":"prediction","numTrees":20,"maxDepth":5,"featureSubsetStrategy":"auto","subsamplingRate":1.0,"featuresCol":"features","checkpointInterval":10,"rawPredictionCol":"rawPrediction","cacheNodeIds":false,"labelCol":"label","seed":207336481,"probabilityCol":"probability","maxBins":32,"minInstancesPerNode":1,"minInfoGain":0.0,"maxMemoryInMB":256},"numFeatures":692,"numClasses":2,"numTrees":2}
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/.part-00000-21082d24-b666-4c4e-a823-70c7afdcbdc5-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/.part-00000-21082d24-b666-4c4e-a823-70c7afdcbdc5-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000..729c5bb30d7fe
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/.part-00000-21082d24-b666-4c4e-a823-70c7afdcbdc5-c000.snappy.parquet.crc differ
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/_SUCCESS b/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/part-00000-21082d24-b666-4c4e-a823-70c7afdcbdc5-c000.snappy.parquet b/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/part-00000-21082d24-b666-4c4e-a823-70c7afdcbdc5-c000.snappy.parquet
new file mode 100644
index 0000000000000..610882198c790
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/part-00000-21082d24-b666-4c4e-a823-70c7afdcbdc5-c000.snappy.parquet differ
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/data/._SUCCESS.crc b/mllib/src/test/resources/ml-models/rfr-2.4.7/data/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfr-2.4.7/data/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/data/.part-00000-4a69607d-6edb-40fc-b681-981caaeca996-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/rfr-2.4.7/data/.part-00000-4a69607d-6edb-40fc-b681-981caaeca996-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000..52cf21f6701ca
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfr-2.4.7/data/.part-00000-4a69607d-6edb-40fc-b681-981caaeca996-c000.snappy.parquet.crc differ
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/data/_SUCCESS b/mllib/src/test/resources/ml-models/rfr-2.4.7/data/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/data/part-00000-4a69607d-6edb-40fc-b681-981caaeca996-c000.snappy.parquet b/mllib/src/test/resources/ml-models/rfr-2.4.7/data/part-00000-4a69607d-6edb-40fc-b681-981caaeca996-c000.snappy.parquet
new file mode 100644
index 0000000000000..75a3f0459864c
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfr-2.4.7/data/part-00000-4a69607d-6edb-40fc-b681-981caaeca996-c000.snappy.parquet differ
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/.part-00000.crc b/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/.part-00000.crc
new file mode 100644
index 0000000000000..1a72b8e029844
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/.part-00000.crc differ
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/_SUCCESS b/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/part-00000 b/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/part-00000
new file mode 100644
index 0000000000000..cccbb8f9f3b05
--- /dev/null
+++ b/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/part-00000
@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.regression.RandomForestRegressionModel","timestamp":1608687933536,"sparkVersion":"2.4.7","uid":"rfr_d946d96b7ff0","paramMap":{"numTrees":2},"defaultParamMap":{"numTrees":20,"featureSubsetStrategy":"auto","maxDepth":5,"minInstancesPerNode":1,"labelCol":"label","cacheNodeIds":false,"checkpointInterval":10,"featuresCol":"features","maxMemoryInMB":256,"predictionCol":"prediction","minInfoGain":0.0,"subsamplingRate":1.0,"impurity":"variance","seed":235498149,"maxBins":32},"numFeatures":692,"numTrees":2}
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/.part-00000-dfe4db51-d349-447a-9b86-d95edaabcde8-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/.part-00000-dfe4db51-d349-447a-9b86-d95edaabcde8-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000..8081f8842670e
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/.part-00000-dfe4db51-d349-447a-9b86-d95edaabcde8-c000.snappy.parquet.crc differ
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/_SUCCESS b/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/part-00000-dfe4db51-d349-447a-9b86-d95edaabcde8-c000.snappy.parquet b/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/part-00000-dfe4db51-d349-447a-9b86-d95edaabcde8-c000.snappy.parquet
new file mode 100644
index 0000000000000..093c3468db301
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/part-00000-dfe4db51-d349-447a-9b86-d95edaabcde8-c000.snappy.parquet differ
diff --git a/mllib/src/test/scala/org/apache/spark/ml/FunctionsSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/FunctionsSuite.scala
index 3dd9a7d8ec85d..21b823383d233 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/FunctionsSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/FunctionsSuite.scala
@@ -18,8 +18,8 @@
 package org.apache.spark.ml
 
 import org.apache.spark.SparkException
-import org.apache.spark.ml.functions.vector_to_array
-import org.apache.spark.ml.linalg.Vectors
+import org.apache.spark.ml.functions.{array_to_vector, vector_to_array}
+import org.apache.spark.ml.linalg.{Vector, Vectors}
 import org.apache.spark.ml.util.MLTest
 import org.apache.spark.mllib.linalg.{Vectors => OldVectors}
 import org.apache.spark.sql.functions.col
@@ -87,4 +87,18 @@ class FunctionsSuite extends MLTest {
     assert(thrown2.getMessage.contains(
       s"Unsupported dtype: float16. Valid values: float64, float32."))
   }
+
+  test("test array_to_vector") {
+    val df1 = Seq(Tuple1(Array(0.5, 1.5))).toDF("c1")
+    val resultVec = df1.select(array_to_vector(col("c1"))).collect()(0)(0).asInstanceOf[Vector]
+    assert(resultVec === Vectors.dense(Array(0.5, 1.5)))
+
+    val df2 = Seq(Tuple1(Array(1.5f, 2.5f))).toDF("c1")
+    val resultVec2 = df2.select(array_to_vector(col("c1"))).collect()(0)(0).asInstanceOf[Vector]
+    assert(resultVec2 === Vectors.dense(Array(1.5, 2.5)))
+
+    val df3 = Seq(Tuple1(Array(1, 2))).toDF("c1")
+    val resultVec3 = df3.select(array_to_vector(col("c1"))).collect()(0)(0).asInstanceOf[Vector]
+    assert(resultVec3 === Vectors.dense(Array(1.0, 2.0)))
+  }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/ann/ANNSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/ann/ANNSuite.scala
index 35586320cb82b..fdd6e352fa639 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/ann/ANNSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/ann/ANNSuite.scala
@@ -45,7 +45,7 @@ class ANNSuite extends SparkFunSuite with MLlibTestSparkContext {
     val trainer = new FeedForwardTrainer(topology, 2, 1)
     trainer.setWeights(initialWeights)
     trainer.LBFGSOptimizer.setNumIterations(20)
-    val model = trainer.train(rddData)
+    val (model, _) = trainer.train(rddData)
     val predictionAndLabels = rddData.map { case (input, label) =>
       (model.predict(input)(0), label(0))
     }.collect()
@@ -80,7 +80,7 @@ class ANNSuite extends SparkFunSuite with MLlibTestSparkContext {
     // TODO: add a test for SGD
     trainer.LBFGSOptimizer.setConvergenceTol(1e-4).setNumIterations(20)
     trainer.setWeights(initialWeights).setStackSize(1)
-    val model = trainer.train(rddData)
+    val (model, _) = trainer.train(rddData)
     val predictionAndLabels = rddData.map { case (input, label) =>
       (model.predict(input), label)
     }.collect()
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/DecisionTreeClassifierSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/DecisionTreeClassifierSuite.scala
index d1ade85cea049..13efdf13d9e54 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/DecisionTreeClassifierSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/DecisionTreeClassifierSuite.scala
@@ -446,6 +446,18 @@ class DecisionTreeClassifierSuite extends MLTest with DefaultReadWriteTest {
 
     testDefaultReadWrite(model)
   }
+
+  test("SPARK-33398: Load DecisionTreeClassificationModel prior to Spark 3.0") {
+    val path = testFile("ml-models/dtc-2.4.7")
+    val model = DecisionTreeClassificationModel.load(path)
+    assert(model.numClasses === 2)
+    assert(model.numFeatures === 692)
+    assert(model.numNodes === 5)
+
+    val metadata = spark.read.json(s"$path/metadata")
+    val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
+    assert(sparkVersionStr === "2.4.7")
+  }
 }
 
 private[ml] object DecisionTreeClassifierSuite extends SparkFunSuite {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/FMClassifierSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/FMClassifierSuite.scala
index d477049824b19..9a04bdc39718c 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/FMClassifierSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/FMClassifierSuite.scala
@@ -194,6 +194,32 @@ class FMClassifierSuite extends MLTest with DefaultReadWriteTest {
     testPredictionModelSinglePrediction(fmModel, smallBinaryDataset)
   }
 
+  test("summary and training summary") {
+    val fm = new FMClassifier()
+    val model = fm.setMaxIter(5).fit(smallBinaryDataset)
+
+    val summary = model.evaluate(smallBinaryDataset)
+
+    assert(model.summary.accuracy === summary.accuracy)
+    assert(model.summary.weightedPrecision === summary.weightedPrecision)
+    assert(model.summary.weightedRecall === summary.weightedRecall)
+    assert(model.summary.pr.collect() === summary.pr.collect())
+    assert(model.summary.roc.collect() === summary.roc.collect())
+    assert(model.summary.areaUnderROC === summary.areaUnderROC)
+  }
+
+  test("FMClassifier training summary totalIterations") {
+    Seq(1, 5, 10, 20, 100).foreach { maxIter =>
+      val trainer = new FMClassifier().setMaxIter(maxIter)
+      val model = trainer.fit(smallBinaryDataset)
+      if (maxIter == 1) {
+        assert(model.summary.totalIterations === maxIter)
+      } else {
+        assert(model.summary.totalIterations <= maxIter)
+      }
+    }
+  }
+
   test("read/write") {
     def checkModelData(
       model: FMClassificationModel,
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/GBTClassifierSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/GBTClassifierSuite.scala
index a2208edcb8394..c86bc0d9a36f3 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/GBTClassifierSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/GBTClassifierSuite.scala
@@ -17,12 +17,10 @@
 
 package org.apache.spark.ml.classification
 
-import com.github.fommil.netlib.BLAS
-
 import org.apache.spark.{SparkException, SparkFunSuite}
 import org.apache.spark.ml.classification.LinearSVCSuite.generateSVMInput
 import org.apache.spark.ml.feature.{Instance, LabeledPoint}
-import org.apache.spark.ml.linalg.{Vector, Vectors}
+import org.apache.spark.ml.linalg.{BLAS, Vector, Vectors}
 import org.apache.spark.ml.param.ParamsSuite
 import org.apache.spark.ml.regression.DecisionTreeRegressionModel
 import org.apache.spark.ml.tree._
@@ -170,8 +168,6 @@ class GBTClassifierSuite extends MLTest with DefaultReadWriteTest {
     val numFeatures = trainingDataset.select(featuresCol).first().getAs[Vector](0).size
     assert(gbtModel.numFeatures === numFeatures)
 
-    val blas = BLAS.getInstance()
-
     val validationDataset = validationData.toDF(labelCol, featuresCol)
     testTransformer[(Double, Vector)](validationDataset, gbtModel,
       "rawPrediction", "features", "probability", "prediction") {
@@ -179,7 +175,7 @@ class GBTClassifierSuite extends MLTest with DefaultReadWriteTest {
         assert(raw.size === 2)
         // check that raw prediction is tree predictions dot tree weights
         val treePredictions = gbtModel.trees.map(_.rootNode.predictImpl(features).prediction)
-        val prediction = blas.ddot(gbtModel.getNumTrees, treePredictions, 1,
+        val prediction = BLAS.nativeBLAS.ddot(gbtModel.getNumTrees, treePredictions, 1,
           gbtModel.treeWeights, 1)
         assert(raw ~== Vectors.dense(-prediction, prediction) relTol eps)
 
@@ -545,6 +541,20 @@ class GBTClassifierSuite extends MLTest with DefaultReadWriteTest {
     testEstimatorAndModelReadWrite(gbt, continuousData, allParamSettings,
       allParamSettings, checkModelData)
   }
+
+  test("SPARK-33398: Load GBTClassificationModel prior to Spark 3.0") {
+    val path = testFile("ml-models/gbtc-2.4.7")
+    val model = GBTClassificationModel.load(path)
+    assert(model.numClasses === 2)
+    assert(model.numFeatures === 692)
+    assert(model.getNumTrees === 2)
+    assert(model.totalNumNodes === 22)
+    assert(model.trees.map(_.numNodes) === Array(5, 17))
+
+    val metadata = spark.read.json(s"$path/metadata")
+    val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
+    assert(sparkVersionStr === "2.4.7")
+  }
 }
 
 private object GBTClassifierSuite extends SparkFunSuite {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/LinearSVCSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/LinearSVCSuite.scala
index 579d6b12ab99f..d18a950a01ab4 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/LinearSVCSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/LinearSVCSuite.scala
@@ -23,14 +23,13 @@ import breeze.linalg.{DenseVector => BDV}
 import org.scalatest.Assertions._
 
 import org.apache.spark.ml.classification.LinearSVCSuite._
-import org.apache.spark.ml.feature.{Instance, LabeledPoint}
+import org.apache.spark.ml.feature.LabeledPoint
 import org.apache.spark.ml.linalg.{DenseVector, SparseVector, Vector, Vectors}
-import org.apache.spark.ml.optim.aggregator.HingeAggregator
 import org.apache.spark.ml.param.ParamsSuite
 import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTest, MLTestingUtils}
 import org.apache.spark.ml.util.TestingUtils._
 import org.apache.spark.sql.{Dataset, Row}
-import org.apache.spark.sql.functions.udf
+import org.apache.spark.sql.functions._
 
 
 class LinearSVCSuite extends MLTest with DefaultReadWriteTest {
@@ -176,28 +175,13 @@ class LinearSVCSuite extends MLTest with DefaultReadWriteTest {
     assert(model2.intercept !== 0.0)
   }
 
-  test("sparse coefficients in HingeAggregator") {
-    val bcCoefficients = spark.sparkContext.broadcast(Vectors.sparse(2, Array(0), Array(1.0)))
-    val bcFeaturesStd = spark.sparkContext.broadcast(Array(1.0))
-    val agg = new HingeAggregator(bcFeaturesStd, true)(bcCoefficients)
-    val thrown = withClue("LinearSVCAggregator cannot handle sparse coefficients") {
-      intercept[IllegalArgumentException] {
-        agg.add(Instance(1.0, 1.0, Vectors.dense(1.0)))
-      }
-    }
-    assert(thrown.getMessage.contains("coefficients only supports dense"))
-
-    bcCoefficients.destroy()
-    bcFeaturesStd.destroy()
-  }
-
   test("linearSVC with sample weights") {
     def modelEquals(m1: LinearSVCModel, m2: LinearSVCModel): Unit = {
-      assert(m1.coefficients ~== m2.coefficients absTol 0.05)
+      assert(m1.coefficients ~== m2.coefficients relTol 0.05)
       assert(m1.intercept ~== m2.intercept absTol 0.05)
     }
 
-    val estimator = new LinearSVC().setRegParam(0.01).setTol(0.01)
+    val estimator = new LinearSVC().setRegParam(0.01).setTol(0.001)
     val dataset = smallBinaryDataset
     MLTestingUtils.testArbitrarilyScaledWeights[LinearSVCModel, LinearSVC](
       dataset.as[LabeledPoint], estimator, modelEquals)
@@ -214,8 +198,8 @@ class LinearSVCSuite extends MLTest with DefaultReadWriteTest {
         .setFitIntercept(fitIntercept)
         .setMaxIter(5)
       val model = lsvc.fit(dataset)
-      Seq(4, 16, 64).foreach { blockSize =>
-        val model2 = lsvc.setBlockSize(blockSize).fit(dataset)
+      Seq(0, 0.01, 0.1, 1, 2, 4).foreach { s =>
+        val model2 = lsvc.setMaxBlockSizeInMB(s).fit(dataset)
         assert(model.intercept ~== model2.intercept relTol 1e-9)
         assert(model.coefficients ~== model2.coefficients relTol 1e-9)
       }
@@ -237,7 +221,7 @@ class LinearSVCSuite extends MLTest with DefaultReadWriteTest {
     val model1 = trainer1.fit(binaryDataset)
 
     /*
-      Use the following R code to load the data and train the model using glmnet package.
+      Use the following R code to load the data and train the model using e1071 package.
 
       library(e1071)
       data <- read.csv("path/target/tmp/LinearSVC/binaryDataset/part-00000", header=FALSE)
@@ -257,8 +241,8 @@ class LinearSVCSuite extends MLTest with DefaultReadWriteTest {
      */
     val coefficientsR = Vectors.dense(7.310338, 14.89741, 22.21005, 29.83508)
     val interceptR = 7.440177
-    assert(model1.intercept ~== interceptR relTol 1E-2)
-    assert(model1.coefficients ~== coefficientsR relTol 1E-2)
+    assert(model1.intercept ~== interceptR relTol 1E-3)
+    assert(model1.coefficients ~== coefficientsR relTol 5E-3)
 
     /*
       Use the following python code to load the data and train the model using scikit-learn package.
@@ -280,8 +264,59 @@ class LinearSVCSuite extends MLTest with DefaultReadWriteTest {
 
     val coefficientsSK = Vectors.dense(7.24690165, 14.77029087, 21.99924004, 29.5575729)
     val interceptSK = 7.36947518
-    assert(model1.intercept ~== interceptSK relTol 1E-3)
-    assert(model1.coefficients ~== coefficientsSK relTol 4E-3)
+    assert(model1.intercept ~== interceptSK relTol 1E-2)
+    assert(model1.coefficients ~== coefficientsSK relTol 1E-2)
+  }
+
+  test("summary and training summary") {
+    val lsvc = new LinearSVC()
+    val model = lsvc.setMaxIter(5).fit(smallBinaryDataset)
+
+    val summary = model.evaluate(smallBinaryDataset)
+
+    assert(model.summary.accuracy === summary.accuracy)
+    assert(model.summary.weightedPrecision === summary.weightedPrecision)
+    assert(model.summary.weightedRecall === summary.weightedRecall)
+    assert(model.summary.pr.collect() === summary.pr.collect())
+    assert(model.summary.roc.collect() === summary.roc.collect())
+    assert(model.summary.areaUnderROC === summary.areaUnderROC)
+
+    // verify instance weight works
+    val lsvc2 = new LinearSVC()
+      .setMaxIter(5)
+      .setWeightCol("weight")
+
+    val smallBinaryDatasetWithWeight =
+      smallBinaryDataset.select(col("label"), col("features"), lit(2.5).as("weight"))
+
+    val summary2 = model.evaluate(smallBinaryDatasetWithWeight)
+
+    val model2 = lsvc2.fit(smallBinaryDatasetWithWeight)
+    assert(model2.summary.accuracy === summary2.accuracy)
+    assert(model2.summary.weightedPrecision ~== summary2.weightedPrecision relTol 1e-6)
+    assert(model2.summary.weightedRecall === summary2.weightedRecall)
+    assert(model2.summary.pr.collect() === summary2.pr.collect())
+    assert(model2.summary.roc.collect() === summary2.roc.collect())
+    assert(model2.summary.areaUnderROC === summary2.areaUnderROC)
+
+    assert(model2.summary.accuracy === model.summary.accuracy)
+    assert(model2.summary.weightedPrecision ~== model.summary.weightedPrecision relTol 1e-6)
+    assert(model2.summary.weightedRecall === model.summary.weightedRecall)
+    assert(model2.summary.pr.collect() === model.summary.pr.collect())
+    assert(model2.summary.roc.collect() === model.summary.roc.collect())
+    assert(model2.summary.areaUnderROC === model.summary.areaUnderROC)
+  }
+
+  test("linearSVC training summary totalIterations") {
+    Seq(1, 5, 10, 20, 100).foreach { maxIter =>
+      val trainer = new LinearSVC().setMaxIter(maxIter)
+      val model = trainer.fit(smallBinaryDataset)
+      if (maxIter == 1) {
+        assert(model.summary.totalIterations === maxIter)
+      } else {
+        assert(model.summary.totalIterations <= maxIter)
+      }
+    }
   }
 
   test("read/write: SVM") {
@@ -328,8 +363,8 @@ object LinearSVCSuite {
   }
 
   def checkModels(model1: LinearSVCModel, model2: LinearSVCModel): Unit = {
-    assert(model1.intercept == model2.intercept)
-    assert(model1.coefficients.equals(model2.coefficients))
+    assert(model1.intercept ~== model2.intercept relTol 1e-9)
+    assert(model1.coefficients  ~==  model2.coefficients relTol 1e-9)
   }
 
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
index 933a63b40fcf8..60b2bd8e53090 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
@@ -28,7 +28,6 @@ import org.apache.spark.ml.attribute.NominalAttribute
 import org.apache.spark.ml.classification.LogisticRegressionSuite._
 import org.apache.spark.ml.feature.{Instance, LabeledPoint}
 import org.apache.spark.ml.linalg.{DenseMatrix, Matrices, Matrix, SparseMatrix, Vector, Vectors}
-import org.apache.spark.ml.optim.aggregator.LogisticAggregator
 import org.apache.spark.ml.param.{ParamMap, ParamsSuite}
 import org.apache.spark.ml.stat.MultiClassSummarizer
 import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTest, MLTestingUtils}
@@ -45,7 +44,9 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
   @transient var smallBinaryDataset: DataFrame = _
   @transient var smallMultinomialDataset: DataFrame = _
   @transient var binaryDataset: DataFrame = _
+  @transient var binaryDatasetWithSmallVar: DataFrame = _
   @transient var multinomialDataset: DataFrame = _
+  @transient var multinomialDatasetWithSmallVar: DataFrame = _
   @transient var multinomialDatasetWithZeroVar: DataFrame = _
   private val eps: Double = 1e-5
 
@@ -86,6 +87,21 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
       df
     }
 
+    binaryDatasetWithSmallVar = {
+      val nPoints = 10000
+      val coefficients = Array(-0.57997, 0.912083, -0.371077, -0.819866, 2.688191)
+      val xMean = Array(5.843, 3.057, 3.758, 10.199)
+      val xVariance = Array(0.6856, 0.1899, 3.116, 0.0001)
+
+      val testData =
+        generateMultinomialLogisticInput(coefficients, xMean, xVariance,
+          addIntercept = true, nPoints, seed)
+
+      val df = sc.parallelize(testData, 4).toDF().withColumn("weight", rand(seed))
+      df.cache()
+      df
+    }
+
     multinomialDataset = {
       val nPoints = 10000
       val coefficients = Array(
@@ -103,6 +119,23 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
       df
     }
 
+    multinomialDatasetWithSmallVar = {
+      val nPoints = 50000
+      val coefficients = Array(
+        -0.57997, 0.912083, -0.371077, -0.819866, 2.688191,
+        -0.16624, -0.84355, -0.048509, -0.301789, 4.170682)
+
+      val xMean = Array(5.843, 3.057, 3.758, 10.199)
+      val xVariance = Array(0.6856, 0.1899, 3.116, 0.001)
+
+      val testData = generateMultinomialLogisticInput(
+        coefficients, xMean, xVariance, addIntercept = true, nPoints, seed)
+
+      val df = sc.parallelize(testData, 4).toDF().withColumn("weight", rand(seed))
+      df.cache()
+      df
+    }
+
     multinomialDatasetWithZeroVar = {
       val nPoints = 100
       val coefficients = Array(
@@ -126,15 +159,21 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
    * so we can validate the training accuracy compared with R's glmnet package.
    */
   ignore("export test data into CSV format") {
-    binaryDataset.rdd.map { case Row(label: Double, features: Vector, weight: Double) =>
-      label + "," + weight + "," + features.toArray.mkString(",")
+    binaryDataset.rdd.map { case Row(l: Double, f: Vector, w: Double) =>
+      l + "," + w + "," + f.toArray.mkString(",")
     }.repartition(1).saveAsTextFile("target/tmp/LogisticRegressionSuite/binaryDataset")
-    multinomialDataset.rdd.map { case Row(label: Double, features: Vector, weight: Double) =>
-      label + "," + weight + "," + features.toArray.mkString(",")
+    binaryDatasetWithSmallVar.rdd.map { case Row(l: Double, f: Vector, w: Double) =>
+      l + "," + w + "," + f.toArray.mkString(",")
+    }.repartition(1).saveAsTextFile("target/tmp/LogisticRegressionSuite/binaryDatasetWithSmallVar")
+    multinomialDataset.rdd.map { case Row(l: Double, f: Vector, w: Double) =>
+      l + "," + w + "," + f.toArray.mkString(",")
     }.repartition(1).saveAsTextFile("target/tmp/LogisticRegressionSuite/multinomialDataset")
-    multinomialDatasetWithZeroVar.rdd.map {
-      case Row(label: Double, features: Vector, weight: Double) =>
-        label + "," + weight + "," + features.toArray.mkString(",")
+    multinomialDatasetWithSmallVar.rdd.map { case Row(l: Double, f: Vector, w: Double) =>
+      l + "," + w + "," + f.toArray.mkString(",")
+    }.repartition(1)
+     .saveAsTextFile("target/tmp/LogisticRegressionSuite/multinomialDatasetWithSmallVar")
+    multinomialDatasetWithZeroVar.rdd.map { case Row(l: Double, f: Vector, w: Double) =>
+        l + "," + w + "," + f.toArray.mkString(",")
     }.repartition(1)
      .saveAsTextFile("target/tmp/LogisticRegressionSuite/multinomialDatasetWithZeroVar")
   }
@@ -266,6 +305,8 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
     assert(blorModel.summary.isInstanceOf[BinaryLogisticRegressionTrainingSummary])
     assert(blorModel.summary.asBinary.isInstanceOf[BinaryLogisticRegressionSummary])
     assert(blorModel.binarySummary.isInstanceOf[BinaryLogisticRegressionTrainingSummary])
+    assert(blorModel.summary.totalIterations == 1)
+    assert(blorModel.binarySummary.totalIterations == 1)
 
     val mlorModel = lr.setFamily("multinomial").fit(smallMultinomialDataset)
     assert(mlorModel.summary.isInstanceOf[LogisticRegressionTrainingSummary])
@@ -279,6 +320,7 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
         mlorModel.summary.asBinary
       }
     }
+    assert(mlorModel.summary.totalIterations == 1)
 
     val mlorBinaryModel = lr.setFamily("multinomial").fit(smallBinaryDataset)
     assert(mlorBinaryModel.summary.isInstanceOf[BinaryLogisticRegressionTrainingSummary])
@@ -288,6 +330,67 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
     val mlorSummary = mlorModel.evaluate(smallMultinomialDataset)
     assert(blorSummary.isInstanceOf[BinaryLogisticRegressionSummary])
     assert(mlorSummary.isInstanceOf[LogisticRegressionSummary])
+
+    // verify instance weight works
+    val lr2 = new LogisticRegression()
+      .setFamily("binomial")
+      .setMaxIter(1)
+      .setWeightCol("weight")
+
+    val smallBinaryDatasetWithWeight =
+      smallBinaryDataset.select(col("label"), col("features"), lit(2.5).as("weight"))
+
+    val smallMultinomialDatasetWithWeight =
+      smallMultinomialDataset.select(col("label"), col("features"), lit(10.0).as("weight"))
+
+    val blorModel2 = lr2.fit(smallBinaryDatasetWithWeight)
+    assert(blorModel2.summary.isInstanceOf[BinaryLogisticRegressionTrainingSummary])
+    assert(blorModel2.summary.asBinary.isInstanceOf[BinaryLogisticRegressionSummary])
+    assert(blorModel2.binarySummary.isInstanceOf[BinaryLogisticRegressionTrainingSummary])
+
+    val mlorModel2 = lr2.setFamily("multinomial").fit(smallMultinomialDatasetWithWeight)
+    assert(mlorModel2.summary.isInstanceOf[LogisticRegressionTrainingSummary])
+    withClue("cannot get binary summary for multiclass model") {
+      intercept[RuntimeException] {
+        mlorModel2.binarySummary
+      }
+    }
+    withClue("cannot cast summary to binary summary multiclass model") {
+      intercept[RuntimeException] {
+        mlorModel2.summary.asBinary
+      }
+    }
+
+    val mlorBinaryModel2 = lr2.setFamily("multinomial").fit(smallBinaryDatasetWithWeight)
+    assert(mlorBinaryModel2.summary.isInstanceOf[BinaryLogisticRegressionTrainingSummary])
+    assert(mlorBinaryModel2.binarySummary.isInstanceOf[BinaryLogisticRegressionTrainingSummary])
+
+    val blorSummary2 = blorModel2.evaluate(smallBinaryDatasetWithWeight)
+    val mlorSummary2 = mlorModel2.evaluate(smallMultinomialDatasetWithWeight)
+    assert(blorSummary2.isInstanceOf[BinaryLogisticRegressionSummary])
+    assert(mlorSummary2.isInstanceOf[LogisticRegressionSummary])
+
+    assert(blorSummary.accuracy ~== blorSummary2.accuracy relTol 1e-6)
+    assert(blorSummary.weightedPrecision ~== blorSummary2.weightedPrecision relTol 1e-6)
+    assert(blorSummary.weightedRecall ~== blorSummary2.weightedRecall relTol 1e-6)
+    assert(blorSummary.asBinary.areaUnderROC ~== blorSummary2.asBinary.areaUnderROC relTol 1e-6)
+
+    assert(blorModel.summary.asBinary.accuracy ~==
+      blorModel2.summary.asBinary.accuracy relTol 1e-6)
+    assert(blorModel.summary.asBinary.weightedPrecision ~==
+      blorModel2.summary.asBinary.weightedPrecision relTol 1e-6)
+    assert(blorModel.summary.asBinary.weightedRecall ~==
+      blorModel2.summary.asBinary.weightedRecall relTol 1e-6)
+    assert(blorModel.summary.asBinary.areaUnderROC ~==
+      blorModel2.summary.asBinary.areaUnderROC relTol 1e-6)
+
+    assert(mlorSummary.accuracy ~== mlorSummary2.accuracy relTol 1e-6)
+    assert(mlorSummary.weightedPrecision ~== mlorSummary2.weightedPrecision relTol 1e-6)
+    assert(mlorSummary.weightedRecall ~== mlorSummary2.weightedRecall relTol 1e-6)
+
+    assert(mlorModel.summary.accuracy ~== mlorModel2.summary.accuracy relTol 1e-6)
+    assert(mlorModel.summary.weightedPrecision ~== mlorModel2.summary.weightedPrecision relTol 1e-6)
+    assert(mlorModel.summary.weightedRecall ~==mlorModel2.summary.weightedRecall relTol 1e-6)
   }
 
   test("setThreshold, getThreshold") {
@@ -336,10 +439,9 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
   }
 
   test("thresholds prediction") {
-    val blr = new LogisticRegression().setFamily("binomial")
+    val blr = new LogisticRegression().setFamily("binomial").setThreshold(1.0)
     val binaryModel = blr.fit(smallBinaryDataset)
 
-    binaryModel.setThreshold(1.0)
     testTransformer[(Double, Vector)](smallBinaryDataset.toDF(), binaryModel, "prediction") {
       row => assert(row.getDouble(0) === 0.0)
     }
@@ -530,8 +632,8 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
         .setMaxIter(5)
         .setFamily("multinomial")
       val model = mlor.fit(dataset)
-      Seq(4, 16, 64).foreach { blockSize =>
-        val model2 = mlor.setBlockSize(blockSize).fit(dataset)
+      Seq(0, 0.01, 0.1, 1, 2, 4).foreach { s =>
+        val model2 = mlor.setMaxBlockSizeInMB(s).fit(dataset)
         assert(model.interceptVector ~== model2.interceptVector relTol 1e-6)
         assert(model.coefficientMatrix ~== model2.coefficientMatrix relTol 1e-6)
       }
@@ -543,8 +645,8 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
         .setMaxIter(5)
         .setFamily("binomial")
       val model = blor.fit(dataset)
-      Seq(4, 16, 64).foreach { blockSize =>
-        val model2 = blor.setBlockSize(blockSize).fit(dataset)
+      Seq(0, 0.01, 0.1, 1, 2, 4).foreach { s =>
+        val model2 = blor.setMaxBlockSizeInMB(s).fit(dataset)
         assert(model.intercept ~== model2.intercept relTol 1e-6)
         assert(model.coefficients ~== model2.coefficients relTol 1e-6)
       }
@@ -569,32 +671,6 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
     assert(blrModel.intercept !== 0.0)
   }
 
-  test("sparse coefficients in LogisticAggregator") {
-    val bcCoefficientsBinary = spark.sparkContext.broadcast(Vectors.sparse(2, Array(0), Array(1.0)))
-    val bcFeaturesStd = spark.sparkContext.broadcast(Array(1.0))
-    val binaryAgg = new LogisticAggregator(bcFeaturesStd, 2,
-      fitIntercept = true, multinomial = false)(bcCoefficientsBinary)
-    val thrownBinary = withClue("binary logistic aggregator cannot handle sparse coefficients") {
-      intercept[IllegalArgumentException] {
-        binaryAgg.add(Instance(1.0, 1.0, Vectors.dense(1.0)))
-      }
-    }
-    assert(thrownBinary.getMessage.contains("coefficients only supports dense"))
-
-    val bcCoefficientsMulti = spark.sparkContext.broadcast(Vectors.sparse(6, Array(0), Array(1.0)))
-    val multinomialAgg = new LogisticAggregator(bcFeaturesStd, 3,
-      fitIntercept = true, multinomial = true)(bcCoefficientsMulti)
-    val thrown = withClue("multinomial logistic aggregator cannot handle sparse coefficients") {
-      intercept[IllegalArgumentException] {
-        multinomialAgg.add(Instance(1.0, 1.0, Vectors.dense(1.0)))
-      }
-    }
-    assert(thrown.getMessage.contains("coefficients only supports dense"))
-    bcCoefficientsBinary.destroy()
-    bcFeaturesStd.destroy()
-    bcCoefficientsMulti.destroy()
-  }
-
   test("overflow prediction for multiclass") {
     val model = new LogisticRegressionModel("mLogReg",
       Matrices.dense(3, 2, Array(0.0, 0.0, 0.0, 1.0, 2.0, 3.0)),
@@ -761,6 +837,64 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
     assert(model6.coefficients ~= coefficientsExpected5 relTol 1E-3)
   }
 
+  test("SPARK-34448: binary logistic regression with intercept, with features with small var") {
+    val trainer1 = new LogisticRegression().setFitIntercept(true).setStandardization(true)
+      .setWeightCol("weight")
+    val trainer2 = new LogisticRegression().setFitIntercept(true).setStandardization(false)
+      .setWeightCol("weight")
+    val trainer3 = new LogisticRegression().setFitIntercept(true).setStandardization(true)
+      .setElasticNetParam(0.0001).setRegParam(0.5).setWeightCol("weight")
+
+    val model1 = trainer1.fit(binaryDatasetWithSmallVar)
+    val model2 = trainer2.fit(binaryDatasetWithSmallVar)
+    val model3 = trainer3.fit(binaryDatasetWithSmallVar)
+
+    /*
+      Use the following R code to load the data and train the model using glmnet package.
+      library("glmnet")
+      data <- read.csv("path", header=FALSE)
+      label = factor(data$V1)
+      w = data$V2
+      features = as.matrix(data.frame(data$V3, data$V4, data$V5, data$V6))
+      coefficients = coef(glmnet(features, label, weights=w, family="binomial", alpha = 0,
+      lambda = 0))
+      coefficients
+      5 x 1 sparse Matrix of class "dgCMatrix"
+                          s0
+      (Intercept) -348.2955812
+      data.V3       -0.8145023
+      data.V4        0.8979252
+      data.V5       -0.6082397
+      data.V6       33.8070109
+
+      coefficientsStd = coef(glmnet(features, label, weights=w, family="binomial", alpha = 0.0001,
+      lambda = 0.5, standardize=T))
+      coefficientsStd
+      5 x 1 sparse Matrix of class "dgCMatrix"
+                            s0
+      (Intercept) -7.403746510
+      data.V3     -0.001443382
+      data.V4      0.001454470
+      data.V5     -0.001097110
+      data.V6      0.048747722
+     */
+    val coefficientsR = Vectors.dense(-0.8145023, 0.8979252, -0.6082397, 33.8070109)
+    val interceptR = -348.2955812
+
+    assert(model1.intercept ~== interceptR relTol 1E-3)
+    assert(model1.coefficients ~= coefficientsR relTol 1E-3)
+
+    // Without regularization, with or without standardization will converge to the same solution.
+    assert(model2.intercept ~== interceptR relTol 1E-3)
+    assert(model2.coefficients ~= coefficientsR relTol 1E-3)
+
+    val coefficientsR2 = Vectors.dense(-0.001443382, 0.001454470, -0.001097110, 0.048747722)
+    val interceptR2 = -7.403746510
+
+    assert(model3.intercept ~== interceptR2 relTol 1E-3)
+    assert(model3.coefficients ~= coefficientsR2 relTol 1E-3)
+  }
+
   test("binary logistic regression without intercept without regularization") {
     val trainer1 = (new LogisticRegression).setFitIntercept(false).setStandardization(true)
       .setWeightCol("weight")
@@ -1485,9 +1619,9 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
     val interceptsExpected1 = Vectors.dense(
       1.0000152482448372, 3.591773288423673, 5.079685953744937)
 
-    checkCoefficientsEquivalent(model1.coefficientMatrix, coefficientsExpected1)
+    checkBoundedMLORCoefficientsEquivalent(model1.coefficientMatrix, coefficientsExpected1)
     assert(model1.interceptVector ~== interceptsExpected1 relTol 0.01)
-    checkCoefficientsEquivalent(model2.coefficientMatrix, coefficientsExpected1)
+    checkBoundedMLORCoefficientsEquivalent(model2.coefficientMatrix, coefficientsExpected1)
     assert(model2.interceptVector ~== interceptsExpected1 relTol 0.01)
 
     // Bound constrained optimization with bound on both side.
@@ -1522,9 +1656,9 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
       isTransposed = true)
     val interceptsExpected3 = Vectors.dense(1.0, 2.0, 2.0)
 
-    checkCoefficientsEquivalent(model3.coefficientMatrix, coefficientsExpected3)
+    checkBoundedMLORCoefficientsEquivalent(model3.coefficientMatrix, coefficientsExpected3)
     assert(model3.interceptVector ~== interceptsExpected3 relTol 0.01)
-    checkCoefficientsEquivalent(model4.coefficientMatrix, coefficientsExpected3)
+    checkBoundedMLORCoefficientsEquivalent(model4.coefficientMatrix, coefficientsExpected3)
     assert(model4.interceptVector ~== interceptsExpected3 relTol 0.01)
 
     // Bound constrained optimization with infinite bound on both side.
@@ -1558,9 +1692,9 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
     val interceptsExpected5 = Vectors.dense(
       -2.2231282183460723, 0.3669496747012527, 1.856178543644802)
 
-    checkCoefficientsEquivalent(model5.coefficientMatrix, coefficientsExpected5)
+    checkBoundedMLORCoefficientsEquivalent(model5.coefficientMatrix, coefficientsExpected5)
     assert(model5.interceptVector ~== interceptsExpected5 relTol 0.01)
-    checkCoefficientsEquivalent(model6.coefficientMatrix, coefficientsExpected5)
+    checkBoundedMLORCoefficientsEquivalent(model6.coefficientMatrix, coefficientsExpected5)
     assert(model6.interceptVector ~== interceptsExpected5 relTol 0.01)
   }
 
@@ -1656,9 +1790,9 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
       1.7524631428961193, 1.2292565990448736, 1.3433784431904323, 1.5846063017678864),
       isTransposed = true)
 
-    checkCoefficientsEquivalent(model1.coefficientMatrix, coefficientsExpected)
+    checkBoundedMLORCoefficientsEquivalent(model1.coefficientMatrix, coefficientsExpected)
     assert(model1.interceptVector.toArray === Array.fill(3)(0.0))
-    checkCoefficientsEquivalent(model2.coefficientMatrix, coefficientsExpected)
+    checkBoundedMLORCoefficientsEquivalent(model2.coefficientMatrix, coefficientsExpected)
     assert(model2.interceptVector.toArray === Array.fill(3)(0.0))
   }
 
@@ -1750,21 +1884,125 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
       0.0, 0.0, 0.0, 0.09064661,
       -0.1144333, 0.3204703, -0.1621061, -0.2308192,
       0.0, -0.4832131, 0.0, 0.0), isTransposed = true)
-    val interceptsRStd = Vectors.dense(-0.72638218, -0.01737265, 0.74375484)
+    val interceptsRStd = Vectors.dense(-0.69265374, -0.2260274, 0.9186811)
     val coefficientsR = new DenseMatrix(3, 4, Array(
       0.0, 0.0, 0.01641412, 0.03570376,
       -0.05110822, 0.0, -0.21595670, -0.16162836,
       0.0, 0.0, 0.0, 0.0), isTransposed = true)
     val interceptsR = Vectors.dense(-0.44707756, 0.75180900, -0.3047314)
 
-    assert(model1.coefficientMatrix ~== coefficientsRStd absTol 0.05)
-    assert(model1.interceptVector ~== interceptsRStd relTol 0.1)
+    assert(model1.coefficientMatrix ~== coefficientsRStd absTol 1e-3)
+    assert(model1.interceptVector ~== interceptsRStd relTol 1e-3)
     assert(model1.interceptVector.toArray.sum ~== 0.0 absTol eps)
-    assert(model2.coefficientMatrix ~== coefficientsR absTol 0.02)
-    assert(model2.interceptVector ~== interceptsR relTol 0.1)
+    assert(model2.coefficientMatrix ~== coefficientsR absTol 1e-3)
+    assert(model2.interceptVector ~== interceptsR relTol 1e-3)
     assert(model2.interceptVector.toArray.sum ~== 0.0 absTol eps)
   }
 
+  test("SPARK-34860: multinomial logistic regression with intercept, with small var") {
+    val trainer1 = new LogisticRegression().setFitIntercept(true).setStandardization(true)
+      .setWeightCol("weight")
+    val trainer2 = new LogisticRegression().setFitIntercept(true).setStandardization(false)
+      .setWeightCol("weight")
+    val trainer3 = new LogisticRegression().setFitIntercept(true).setStandardization(true)
+      .setElasticNetParam(0.0001).setRegParam(0.5).setWeightCol("weight")
+
+    val model1 = trainer1.fit(multinomialDatasetWithSmallVar)
+    val model2 = trainer2.fit(multinomialDatasetWithSmallVar)
+    val model3 = trainer3.fit(multinomialDatasetWithSmallVar)
+
+    /*
+      Use the following R code to load the data and train the model using glmnet package.
+      library("glmnet")
+      data <- read.csv("path", header=FALSE)
+      label = factor(data$V1)
+      w = data$V2
+      features = as.matrix(data.frame(data$V3, data$V4, data$V5, data$V6))
+      coefficients = coef(glmnet(features, label, weights=w, family="multinomial", alpha = 0,
+      lambda = 0))
+      coefficients
+      $`0`
+      5 x 1 sparse Matrix of class "dgCMatrix"
+                       s0
+               2.91748298
+      data.V3  0.21755977
+      data.V4  0.01647541
+      data.V5  0.16507778
+      data.V6 -0.14016680
+
+      $`1`
+      5 x 1 sparse Matrix of class "dgCMatrix"
+                       s0
+              -17.5107460
+      data.V3  -0.2443600
+      data.V4   0.7564655
+      data.V5  -0.2955698
+      data.V6   1.3262009
+
+      $`2`
+      5 x 1 sparse Matrix of class "dgCMatrix"
+                       s0
+              14.59326301
+      data.V3  0.02680026
+      data.V4 -0.77294095
+      data.V5  0.13049206
+      data.V6 -1.18603411
+
+
+
+      coefficientsStd = coef(glmnet(features, label, weights=w, family="multinomial",
+      alpha = 0.0001, lambda = 0.5, standardize=T))
+      coefficientsStd
+      $`0`
+      5 x 1 sparse Matrix of class "dgCMatrix"
+                       s0
+              1.751626027
+      data.V3 0.019970169
+      data.V4 0.079611293
+      data.V5 0.003959452
+      data.V6 0.110024399
+
+      $`1`
+      5 x 1 sparse Matrix of class "dgCMatrix"
+                         s0
+              -3.9297124987
+      data.V3 -0.0004788494
+      data.V4  0.0010097453
+      data.V5 -0.0005832701
+      data.V6  .
+
+      $`2`
+      5 x 1 sparse Matrix of class "dgCMatrix"
+                        s0
+               2.178086472
+      data.V3 -0.019369990
+      data.V4 -0.080851149
+      data.V5 -0.003319687
+      data.V6 -0.112435972
+     */
+    val interceptsR = Vectors.dense(2.91748298, -17.5107460, 14.59326301)
+    val coefficientsR = new DenseMatrix(3, 4, Array(
+      0.21755977, 0.01647541, 0.16507778, -0.14016680,
+      -0.2443600, 0.7564655, -0.2955698, 1.3262009,
+      0.02680026, -0.77294095, 0.13049206, -1.18603411), isTransposed = true)
+
+    assert(model1.interceptVector ~== interceptsR relTol 1e-2)
+    assert(model1.coefficientMatrix ~= coefficientsR relTol 1e-1)
+
+    // Without regularization, with or without standardization will converge to the same solution.
+    assert(model2.interceptVector ~== interceptsR relTol 1e-2)
+    assert(model2.coefficientMatrix ~= coefficientsR relTol 1e-1)
+
+    val interceptsR2 = Vectors.dense(1.751626027, -3.9297124987, 2.178086472)
+    val coefficientsR2 = new DenseMatrix(3, 4, Array(
+      0.019970169, 0.079611293, 0.003959452, 0.110024399,
+      -0.0004788494, 0.0010097453, -0.0005832701, 0.0,
+      -0.019369990, -0.080851149, -0.003319687, -0.112435972), isTransposed = true)
+
+    assert(model3.interceptVector ~== interceptsR2 relTol 1e-3)
+    assert(model3.coefficientMatrix ~= coefficientsR2 relTol 1e-2)
+  }
+
   test("multinomial logistic regression without intercept with L1 regularization") {
     val trainer1 = (new LogisticRegression).setFitIntercept(false)
       .setElasticNetParam(1.0).setRegParam(0.05).setStandardization(true).setWeightCol("weight")
@@ -2509,7 +2747,7 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
         rows.map(_.getDouble(0)).toArray === binaryExpected
       }
     }
-    assert(model2.summary.totalIterations === 1)
+    assert(model2.summary.totalIterations === 0)
 
     val lr3 = new LogisticRegression().setFamily("multinomial")
     val model3 = lr3.fit(smallMultinomialDataset)
@@ -2524,7 +2762,7 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
         rows.map(_.getDouble(0)).toArray === multinomialExpected
       }
     }
-    assert(model4.summary.totalIterations === 1)
+    assert(model4.summary.totalIterations === 0)
   }
 
   test("binary logistic regression with all labels the same") {
@@ -2544,6 +2782,7 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
     assert(allZeroInterceptModel.coefficients ~== Vectors.dense(0.0) absTol 1E-3)
     assert(allZeroInterceptModel.intercept === Double.NegativeInfinity)
     assert(allZeroInterceptModel.summary.totalIterations === 0)
+    assert(allZeroInterceptModel.summary.objectiveHistory(0) ~== 0.0 absTol 1e-4)
 
     val allOneInterceptModel = lrIntercept
       .setLabelCol("oneLabel")
@@ -2551,6 +2790,7 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
     assert(allOneInterceptModel.coefficients ~== Vectors.dense(0.0) absTol 1E-3)
     assert(allOneInterceptModel.intercept === Double.PositiveInfinity)
     assert(allOneInterceptModel.summary.totalIterations === 0)
+    assert(allOneInterceptModel.summary.objectiveHistory(0) ~== 0.0 absTol 1e-4)
 
     // fitIntercept=false
     val lrNoIntercept = new LogisticRegression()
@@ -2586,6 +2826,7 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
         assert(pred === 4.0)
     }
     assert(model.summary.totalIterations === 0)
+    assert(model.summary.objectiveHistory(0) ~== 0.0 absTol 1e-4)
 
     // force the model to be trained with only one class
     val constantZeroData = Seq(
@@ -2599,7 +2840,7 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
         assert(prob === Vectors.dense(Array(1.0)))
         assert(pred === 0.0)
     }
-    assert(modelZeroLabel.summary.totalIterations > 0)
+    assert(modelZeroLabel.summary.totalIterations === 0)
 
     // ensure that the correct value is predicted when numClasses passed through metadata
     val labelMeta = NominalAttribute.defaultAttr.withName("label").withNumValues(6).toMetadata()
@@ -2614,6 +2855,7 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
         assert(pred === 4.0)
     }
     require(modelWithMetadata.summary.totalIterations === 0)
+    assert(model.summary.objectiveHistory(0) ~== 0.0 absTol 1e-4)
   }
 
   test("compressed storage for constant label") {
@@ -2886,16 +3128,17 @@ object LogisticRegressionSuite {
   }
 
   /**
+   * Note: This method is only used in Bounded MLOR (without regularization) test
    * When no regularization is applied, the multinomial coefficients lack identifiability
    * because we do not use a pivot class. We can add any constant value to the coefficients
    * and get the same likelihood. If fitting under bound constrained optimization, we don't
    * choose the mean centered coefficients like what we do for unbound problems, since they
    * may out of the bounds. We use this function to check whether two coefficients are equivalent.
    */
-  def checkCoefficientsEquivalent(coefficients1: Matrix, coefficients2: Matrix): Unit = {
+  def checkBoundedMLORCoefficientsEquivalent(coefficients1: Matrix, coefficients2: Matrix): Unit = {
     coefficients1.colIter.zip(coefficients2.colIter).foreach { case (col1: Vector, col2: Vector) =>
       (col1.asBreeze - col2.asBreeze).toArray.toSeq.sliding(2).foreach {
-        case Seq(v1, v2) => assert(v1 ~= v2 absTol 1E-3)
+        case Seq(v1, v2) => assert(v1 ~= v2 absTol 1E-2)
       }
     }
   }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifierSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifierSuite.scala
index 902af71e42f86..cc52bd8cf2c7e 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifierSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifierSuite.scala
@@ -240,6 +240,38 @@ class MultilayerPerceptronClassifierSuite extends MLTest with DefaultReadWriteTe
 
     val metadata = spark.read.json(s"$mlpPath/metadata")
     val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
-    assert(sparkVersionStr == "2.4.4")
+    assert(sparkVersionStr === "2.4.4")
+  }
+
+  test("summary and training summary") {
+    val mlp = new MultilayerPerceptronClassifier()
+    val model = mlp.setMaxIter(5).setLayers(Array(2, 3, 2)).fit(dataset)
+    val summary = model.evaluate(dataset)
+
+    assert(model.summary.truePositiveRateByLabel === summary.truePositiveRateByLabel)
+    assert(model.summary.falsePositiveRateByLabel === summary.falsePositiveRateByLabel)
+    assert(model.summary.precisionByLabel === summary.precisionByLabel)
+    assert(model.summary.recallByLabel === summary.recallByLabel)
+    assert(model.summary.fMeasureByLabel === summary.fMeasureByLabel)
+    assert(model.summary.accuracy === summary.accuracy)
+    assert(model.summary.weightedFalsePositiveRate === summary.weightedFalsePositiveRate)
+    assert(model.summary.weightedTruePositiveRate === summary.weightedTruePositiveRate)
+    assert(model.summary.weightedPrecision === summary.weightedPrecision)
+    assert(model.summary.weightedRecall === summary.weightedRecall)
+    assert(model.summary.weightedFMeasure === summary.weightedFMeasure)
+  }
+
+  test("MultilayerPerceptron training summary totalIterations") {
+    Seq(1, 5, 10, 20, 100).foreach { maxIter =>
+      val trainer = new MultilayerPerceptronClassifier()
+        .setMaxIter(maxIter)
+        .setLayers(Array(2, 3, 2))
+      val model = trainer.fit(dataset)
+      if (maxIter == 1) {
+        assert(model.summary.totalIterations === maxIter)
+      } else {
+        assert(model.summary.totalIterations <= maxIter)
+      }
+    }
   }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/NaiveBayesSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/NaiveBayesSuite.scala
index af76f045fb84b..6742d612a492d 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/NaiveBayesSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/NaiveBayesSuite.scala
@@ -80,10 +80,10 @@ class NaiveBayesSuite extends MLTest with DefaultReadWriteTest {
   }
 
   def validatePrediction(predictionAndLabels: Seq[Row]): Unit = {
-    val numOfErrorPredictions = predictionAndLabels.filter {
+    val numOfErrorPredictions = predictionAndLabels.count {
       case Row(prediction: Double, label: Double) =>
         prediction != label
-    }.length
+    }
     // At least 80% of the predictions should be on.
     assert(numOfErrorPredictions < predictionAndLabels.length / 5)
   }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/OneVsRestSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/OneVsRestSuite.scala
index a4b1a5f18141a..61b5a50fea69a 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/OneVsRestSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/OneVsRestSuite.scala
@@ -210,6 +210,26 @@ class OneVsRestSuite extends MLTest with DefaultReadWriteTest {
     assert(ova2.fit(dataset2) !== null)
   }
 
+  test("SPARK-34045: OneVsRestModel.transform should not call setter of submodels") {
+    val logReg = new LogisticRegression().setMaxIter(1)
+    val ovr = new OneVsRest().setClassifier(logReg)
+    val ovrm = ovr.fit(dataset)
+    val dataset2 = dataset.withColumnRenamed("features", "features2")
+    ovrm.setFeaturesCol("features2")
+
+    val oldCols = ovrm.models.map(_.getFeaturesCol)
+    ovrm.transform(dataset2)
+    val newCols = ovrm.models.map(_.getFeaturesCol)
+    assert(oldCols === newCols)
+  }
+
+  test("SPARK-34356: OneVsRestModel.transform should avoid potential column conflict") {
+    val logReg = new LogisticRegression().setMaxIter(1)
+    val ovr = new OneVsRest().setClassifier(logReg)
+    val ovrm = ovr.fit(dataset)
+    assert(ovrm.transform(dataset.withColumn("probability", lit(0.0))).count() === dataset.count())
+  }
+
   test("OneVsRest.copy and OneVsRestModel.copy") {
     val lr = new LogisticRegression()
       .setMaxIter(1)
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/RandomForestClassifierSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/RandomForestClassifierSuite.scala
index e30e93ad4628c..7be007a89f6fc 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/RandomForestClassifierSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/RandomForestClassifierSuite.scala
@@ -25,12 +25,13 @@ import org.apache.spark.ml.param.ParamsSuite
 import org.apache.spark.ml.tree._
 import org.apache.spark.ml.tree.impl.TreeTests
 import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTest, MLTestingUtils}
+import org.apache.spark.ml.util.TestingUtils._
 import org.apache.spark.mllib.regression.{LabeledPoint => OldLabeledPoint}
 import org.apache.spark.mllib.tree.{EnsembleTestHelper, RandomForest => OldRandomForest}
 import org.apache.spark.mllib.tree.configuration.{Algo => OldAlgo}
-import org.apache.spark.mllib.util.TestingUtils._
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
 
 /**
  * Test suite for [[RandomForestClassifier]].
@@ -296,6 +297,115 @@ class RandomForestClassifierSuite extends MLTest with DefaultReadWriteTest {
     }
   }
 
+  test("summary for binary and multiclass") {
+    val arr = new Array[LabeledPoint](300)
+      for (i <- 0 until 300) {
+        if (i < 100) {
+          arr(i) = new LabeledPoint(0.0, Vectors.dense(2.0, 2.0))
+        } else if (i < 200) {
+          arr(i) = new LabeledPoint(1.0, Vectors.dense(1.0, 2.0))
+        } else {
+          arr(i) = new LabeledPoint(2.0, Vectors.dense(0.0, 2.0))
+        }
+      }
+    val rdd = sc.parallelize(arr)
+    val multinomialDataset = spark.createDataFrame(rdd)
+
+    val rf = new RandomForestClassifier()
+
+    val brfModel = rf.fit(binaryDataset)
+    assert(brfModel.summary.isInstanceOf[BinaryRandomForestClassificationTrainingSummary])
+    assert(brfModel.summary.asBinary.isInstanceOf[BinaryRandomForestClassificationTrainingSummary])
+    assert(brfModel.binarySummary.isInstanceOf[RandomForestClassificationTrainingSummary])
+    assert(brfModel.summary.totalIterations === 0)
+    assert(brfModel.binarySummary.totalIterations === 0)
+
+    val mrfModel = rf.fit(multinomialDataset)
+    assert(mrfModel.summary.isInstanceOf[RandomForestClassificationTrainingSummary])
+    withClue("cannot get binary summary for multiclass model") {
+      intercept[RuntimeException] {
+        mrfModel.binarySummary
+      }
+    }
+    withClue("cannot cast summary to binary summary multiclass model") {
+      intercept[RuntimeException] {
+        mrfModel.summary.asBinary
+      }
+    }
+    assert(mrfModel.summary.totalIterations === 0)
+
+    val brfSummary = brfModel.evaluate(binaryDataset)
+    val mrfSummary = mrfModel.evaluate(multinomialDataset)
+    assert(brfSummary.isInstanceOf[BinaryRandomForestClassificationSummary])
+    assert(mrfSummary.isInstanceOf[RandomForestClassificationSummary])
+
+    assert(brfSummary.accuracy === brfModel.summary.accuracy)
+    assert(brfSummary.weightedPrecision === brfModel.summary.weightedPrecision)
+    assert(brfSummary.weightedRecall === brfModel.summary.weightedRecall)
+    assert(brfSummary.asBinary.areaUnderROC ~== brfModel.summary.asBinary.areaUnderROC relTol 1e-6)
+
+    // verify instance weight works
+    val rf2 = new RandomForestClassifier()
+      .setWeightCol("weight")
+
+    val binaryDatasetWithWeight =
+      binaryDataset.select(col("label"), col("features"), lit(2.5).as("weight"))
+
+    val multinomialDatasetWithWeight =
+      multinomialDataset.select(col("label"), col("features"), lit(10.0).as("weight"))
+
+    val brfModel2 = rf2.fit(binaryDatasetWithWeight)
+    assert(brfModel2.summary.isInstanceOf[BinaryRandomForestClassificationTrainingSummary])
+    assert(brfModel2.summary.asBinary.isInstanceOf[BinaryRandomForestClassificationTrainingSummary])
+    assert(brfModel2.binarySummary.isInstanceOf[BinaryRandomForestClassificationTrainingSummary])
+
+    val mrfModel2 = rf2.fit(multinomialDatasetWithWeight)
+    assert(mrfModel2.summary.isInstanceOf[RandomForestClassificationTrainingSummary])
+    withClue("cannot get binary summary for multiclass model") {
+      intercept[RuntimeException] {
+        mrfModel2.binarySummary
+      }
+    }
+    withClue("cannot cast summary to binary summary multiclass model") {
+      intercept[RuntimeException] {
+        mrfModel2.summary.asBinary
+      }
+    }
+
+    val brfSummary2 = brfModel2.evaluate(binaryDatasetWithWeight)
+    val mrfSummary2 = mrfModel2.evaluate(multinomialDatasetWithWeight)
+    assert(brfSummary2.isInstanceOf[BinaryRandomForestClassificationSummary])
+    assert(mrfSummary2.isInstanceOf[RandomForestClassificationSummary])
+
+    assert(brfSummary2.accuracy === brfModel2.summary.accuracy)
+    assert(brfSummary2.weightedPrecision === brfModel2.summary.weightedPrecision)
+    assert(brfSummary2.weightedRecall === brfModel2.summary.weightedRecall)
+    assert(brfSummary2.asBinary.areaUnderROC ~==
+      brfModel2.summary.asBinary.areaUnderROC relTol 1e-6)
+
+    assert(brfSummary.accuracy ~== brfSummary2.accuracy relTol 1e-6)
+    assert(brfSummary.weightedPrecision ~== brfSummary2.weightedPrecision relTol 1e-6)
+    assert(brfSummary.weightedRecall ~== brfSummary2.weightedRecall relTol 1e-6)
+    assert(brfSummary.asBinary.areaUnderROC ~== brfSummary2.asBinary.areaUnderROC relTol 1e-6)
+
+    assert(brfModel.summary.asBinary.accuracy ~==
+      brfModel2.summary.asBinary.accuracy relTol 1e-6)
+    assert(brfModel.summary.asBinary.weightedPrecision ~==
+      brfModel2.summary.asBinary.weightedPrecision relTol 1e-6)
+    assert(brfModel.summary.asBinary.weightedRecall ~==
+      brfModel2.summary.asBinary.weightedRecall relTol 1e-6)
+    assert(brfModel.summary.asBinary.areaUnderROC ~==
+      brfModel2.summary.asBinary.areaUnderROC relTol 1e-6)
+
+    assert(mrfSummary.accuracy ~== mrfSummary2.accuracy relTol 1e-6)
+    assert(mrfSummary.weightedPrecision ~== mrfSummary2.weightedPrecision relTol 1e-6)
+    assert(mrfSummary.weightedRecall ~== mrfSummary2.weightedRecall relTol 1e-6)
+
+    assert(mrfModel.summary.accuracy ~== mrfModel2.summary.accuracy relTol 1e-6)
+    assert(mrfModel.summary.weightedPrecision ~== mrfModel2.summary.weightedPrecision relTol 1e-6)
+    assert(mrfModel.summary.weightedRecall ~==mrfModel2.summary.weightedRecall relTol 1e-6)
+  }
+
   /////////////////////////////////////////////////////////////////////////////
   // Tests of model save/load
   /////////////////////////////////////////////////////////////////////////////
@@ -319,6 +429,20 @@ class RandomForestClassifierSuite extends MLTest with DefaultReadWriteTest {
     testEstimatorAndModelReadWrite(rf, continuousData, allParamSettings,
       allParamSettings, checkModelData)
   }
+
+  test("SPARK-33398: Load RandomForestClassificationModel prior to Spark 3.0") {
+    val path = testFile("ml-models/rfc-2.4.7")
+    val model = RandomForestClassificationModel.load(path)
+    assert(model.numClasses === 2)
+    assert(model.numFeatures === 692)
+    assert(model.getNumTrees === 2)
+    assert(model.totalNumNodes === 10)
+    assert(model.trees.map(_.numNodes) === Array(3, 7))
+
+    val metadata = spark.read.json(s"$path/metadata")
+    val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
+    assert(sparkVersionStr === "2.4.7")
+  }
 }
 
 private object RandomForestClassifierSuite extends SparkFunSuite {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/clustering/BisectingKMeansSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/clustering/BisectingKMeansSuite.scala
index debd0dd65d0c8..04b20d1e58dd3 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/clustering/BisectingKMeansSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/clustering/BisectingKMeansSuite.scala
@@ -219,7 +219,7 @@ class BisectingKMeansSuite extends MLTest with DefaultReadWriteTest {
 
     model1.clusterCenters.forall(Vectors.norm(_, 2) == 1.0)
 
-    val df2 = spark.createDataFrame(spark.sparkContext.parallelize(Array(
+    val df2 = spark.createDataFrame(spark.sparkContext.parallelize(Seq(
       (Vectors.dense(1.0, 1.0), 2.0), (Vectors.dense(10.0, 10.0), 2.0),
       (Vectors.dense(1.0, 0.5), 2.0), (Vectors.dense(10.0, 4.4), 2.0),
       (Vectors.dense(-1.0, 1.0), 2.0), (Vectors.dense(-100.0, 90.0), 2.0))))
@@ -286,7 +286,7 @@ class BisectingKMeansSuite extends MLTest with DefaultReadWriteTest {
 
     model1.clusterCenters.forall(Vectors.norm(_, 2) == 1.0)
 
-    val df2 = spark.createDataFrame(spark.sparkContext.parallelize(Array(
+    val df2 = spark.createDataFrame(spark.sparkContext.parallelize(Seq(
       (Vectors.dense(1.0, 1.0), 1.0), (Vectors.dense(10.0, 10.0), 2.0),
       (Vectors.dense(1.0, 0.5), 2.0), (Vectors.dense(10.0, 4.4), 3.0),
       (Vectors.dense(-1.0, 1.0), 3.0), (Vectors.dense(-100.0, 90.0), 4.0))))
diff --git a/mllib/src/test/scala/org/apache/spark/ml/clustering/GaussianMixtureSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/clustering/GaussianMixtureSuite.scala
index d848d5a5ee452..0eae23df8358d 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/clustering/GaussianMixtureSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/clustering/GaussianMixtureSuite.scala
@@ -181,7 +181,7 @@ class GaussianMixtureSuite extends MLTest with DefaultReadWriteTest {
     }
   }
 
-  test("multivariate data and check againt R mvnormalmixEM") {
+  test("multivariate data and check against R mvnormalmixEM") {
     /*
       Using the following R code to generate data and train the model using mixtools package.
       library(mvtnorm)
@@ -285,17 +285,6 @@ class GaussianMixtureSuite extends MLTest with DefaultReadWriteTest {
     testClusteringModelSingleProbabilisticPrediction(model, model.predictProbability, dataset,
       model.getFeaturesCol, model.getProbabilityCol)
   }
-
-  test("GMM on blocks") {
-    Seq(dataset, sparseDataset, denseDataset, rDataset).foreach { dataset =>
-      val gmm = new GaussianMixture().setK(k).setMaxIter(20).setBlockSize(1).setSeed(seed)
-      val model = gmm.fit(dataset)
-      Seq(2, 4, 8, 16, 32).foreach { blockSize =>
-        val model2 = gmm.setBlockSize(blockSize).fit(dataset)
-        modelEquals(model, model2)
-      }
-    }
-  }
 }
 
 object GaussianMixtureSuite extends SparkFunSuite {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/clustering/KMeansSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/clustering/KMeansSuite.scala
index 584594436267f..61f4359d99ea9 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/clustering/KMeansSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/clustering/KMeansSuite.scala
@@ -255,7 +255,7 @@ class KMeansSuite extends MLTest with DefaultReadWriteTest with PMMLReadWriteTes
   }
 
   test("compare with weightCol and without weightCol") {
-    val df1 = spark.createDataFrame(spark.sparkContext.parallelize(Array(
+    val df1 = spark.createDataFrame(spark.sparkContext.parallelize(Seq(
       Vectors.dense(1.0, 1.0),
       Vectors.dense(10.0, 10.0), Vectors.dense(10.0, 10.0),
       Vectors.dense(1.0, 0.5),
@@ -285,7 +285,7 @@ class KMeansSuite extends MLTest with DefaultReadWriteTest with PMMLReadWriteTes
 
     model1.clusterCenters.forall(Vectors.norm(_, 2) == 1.0)
 
-    val df2 = spark.createDataFrame(spark.sparkContext.parallelize(Array(
+    val df2 = spark.createDataFrame(spark.sparkContext.parallelize(Seq(
       (Vectors.dense(1.0, 1.0), 1.0),
       (Vectors.dense(10.0, 10.0), 2.0),
       (Vectors.dense(1.0, 0.5), 1.0),
@@ -322,7 +322,7 @@ class KMeansSuite extends MLTest with DefaultReadWriteTest with PMMLReadWriteTes
 
   test("Two centers with weightCol") {
     // use the same weight for all samples.
-    val df1 = spark.createDataFrame(spark.sparkContext.parallelize(Array(
+    val df1 = spark.createDataFrame(spark.sparkContext.parallelize(Seq(
       (Vectors.dense(0.0, 0.0), 2.0),
       (Vectors.dense(0.0, 0.1), 2.0),
       (Vectors.dense(0.1, 0.0), 2.0),
@@ -366,7 +366,7 @@ class KMeansSuite extends MLTest with DefaultReadWriteTest with PMMLReadWriteTes
     assert(model1.clusterCenters(1) === model1_center2)
 
     // use different weight
-    val df2 = spark.createDataFrame(spark.sparkContext.parallelize(Array(
+    val df2 = spark.createDataFrame(spark.sparkContext.parallelize(Seq(
       (Vectors.dense(0.0, 0.0), 1.0),
       (Vectors.dense(0.0, 0.1), 2.0),
       (Vectors.dense(0.1, 0.0), 3.0),
@@ -412,7 +412,7 @@ class KMeansSuite extends MLTest with DefaultReadWriteTest with PMMLReadWriteTes
 
   test("Four centers with weightCol") {
     // no weight
-    val df1 = spark.createDataFrame(spark.sparkContext.parallelize(Array(
+    val df1 = spark.createDataFrame(spark.sparkContext.parallelize(Seq(
       Vectors.dense(0.1, 0.1),
       Vectors.dense(5.0, 0.2),
       Vectors.dense(10.0, 0.0),
@@ -444,7 +444,7 @@ class KMeansSuite extends MLTest with DefaultReadWriteTest with PMMLReadWriteTes
     model1.clusterCenters.forall(Vectors.norm(_, 2) == 1.0)
 
     // use same weight, should have the same result as no weight
-    val df2 = spark.createDataFrame(spark.sparkContext.parallelize(Array(
+    val df2 = spark.createDataFrame(spark.sparkContext.parallelize(Seq(
       (Vectors.dense(0.1, 0.1), 2.0),
       (Vectors.dense(5.0, 0.2), 2.0),
       (Vectors.dense(10.0, 0.0), 2.0),
diff --git a/mllib/src/test/scala/org/apache/spark/ml/clustering/LDASuite.scala b/mllib/src/test/scala/org/apache/spark/ml/clustering/LDASuite.scala
index d0898220b80de..e05d76cf70ed3 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/clustering/LDASuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/clustering/LDASuite.scala
@@ -199,11 +199,11 @@ class LDASuite extends MLTest with DefaultReadWriteTest {
     assert(topics.count() === k)
     assert(topics.select("topic").rdd.map(_.getInt(0)).collect().toSet === Range(0, k).toSet)
     topics.select("termIndices").collect().foreach { case r: Row =>
-      val termIndices = r.getAs[Seq[Int]](0)
+      val termIndices = r.getSeq[Int](0)
       assert(termIndices.length === 3 && termIndices.toSet.size === 3)
     }
     topics.select("termWeights").collect().foreach { case r: Row =>
-      val termWeights = r.getAs[Seq[Double]](0)
+      val termWeights = r.getSeq[Double](0)
       assert(termWeights.length === 3 && termWeights.forall(w => w >= 0.0 && w <= 1.0))
     }
   }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala
index 6b3970def12ec..0ffef986fbdbc 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala
@@ -224,7 +224,7 @@ class PowerIterationClusteringSuite extends SparkFunSuite
       (3, 4)
     )).toDF("src", "dst").repartition(1)
 
-    var assignments2 = new PowerIterationClustering()
+    val assignments2 = new PowerIterationClustering()
       .setInitMode("random")
       .setK(2)
       .assignClusters(data2)
@@ -238,7 +238,7 @@ class PowerIterationClusteringSuite extends SparkFunSuite
     assert(Set(predictions2(0).size, predictions2(1).size) !== Set(2, 3))
 
 
-    var assignments3 = new PowerIterationClustering()
+    val assignments3 = new PowerIterationClustering()
       .setInitMode("degree")
       .setK(2)
       .assignClusters(data2)
diff --git a/mllib/src/test/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluatorSuite.scala
index 83b213ab51d43..008bf0e108e13 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluatorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluatorSuite.scala
@@ -102,4 +102,27 @@ class BinaryClassificationEvaluatorSuite
     val evaluator = new BinaryClassificationEvaluator().setRawPredictionCol("prediction")
     MLTestingUtils.checkNumericTypes(evaluator, spark)
   }
+
+  test("getMetrics") {
+    val weightCol = "weight"
+    // get metric with weight column
+    val evaluator = new BinaryClassificationEvaluator()
+      .setWeightCol(weightCol)
+    val vectorDF = Seq(
+      (0.0, Vectors.dense(2.5, 12), 1.0),
+      (1.0, Vectors.dense(1, 3), 1.0),
+      (0.0, Vectors.dense(10, 2), 1.0)
+    ).toDF("label", "rawPrediction", weightCol)
+
+    val metrics = evaluator.getMetrics(vectorDF)
+    val roc = metrics.areaUnderROC()
+    val pr = metrics.areaUnderPR()
+
+    // default = areaUnderROC
+    assert(evaluator.evaluate(vectorDF) == roc)
+
+    // areaUnderPR
+    evaluator.setMetricName("areaUnderPR")
+    assert(evaluator.evaluate(vectorDF) == pr)
+  }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/evaluation/ClusteringEvaluatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/evaluation/ClusteringEvaluatorSuite.scala
index 6cf3b1deeac93..06f2cb2b9788b 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/evaluation/ClusteringEvaluatorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/evaluation/ClusteringEvaluatorSuite.scala
@@ -19,12 +19,13 @@ package org.apache.spark.ml.evaluation
 
 import org.apache.spark.{SparkException, SparkFunSuite}
 import org.apache.spark.ml.attribute.AttributeGroup
-import org.apache.spark.ml.linalg.Vector
+import org.apache.spark.ml.linalg.{Vector, Vectors}
 import org.apache.spark.ml.param.ParamsSuite
 import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTestingUtils}
 import org.apache.spark.ml.util.TestingUtils._
 import org.apache.spark.mllib.util.MLlibTestSparkContext
 import org.apache.spark.sql.DataFrame
+import org.apache.spark.sql.functions.lit
 
 
 class ClusteringEvaluatorSuite
@@ -145,4 +146,60 @@ class ClusteringEvaluatorSuite
     assert(evaluator.evaluate(twoSingleItemClusters) === 0.0)
   }
 
+  test("getMetrics") {
+    val evaluator = new ClusteringEvaluator()
+      .setFeaturesCol("features")
+      .setPredictionCol("label")
+
+    val metrics1 = evaluator.getMetrics(irisDataset)
+    val silhouetteScoreEuclidean = metrics1.silhouette
+
+    assert(evaluator.evaluate(irisDataset) == silhouetteScoreEuclidean)
+
+    evaluator.setDistanceMeasure("cosine")
+    val metrics2 = evaluator.getMetrics(irisDataset)
+    val silhouetteScoreCosin = metrics2.silhouette
+
+    assert(evaluator.evaluate(irisDataset) == silhouetteScoreCosin)
+  }
+
+  test("test weight support") {
+    Seq("squaredEuclidean", "cosine").foreach { distanceMeasure =>
+      val evaluator1 = new ClusteringEvaluator()
+        .setFeaturesCol("features")
+        .setPredictionCol("label")
+        .setDistanceMeasure(distanceMeasure)
+
+      val evaluator2 = new ClusteringEvaluator()
+        .setFeaturesCol("features")
+        .setPredictionCol("label")
+        .setDistanceMeasure(distanceMeasure)
+        .setWeightCol("weight")
+
+      Seq(0.25, 1.0, 10.0, 99.99).foreach { w =>
+        var score1 = evaluator1.evaluate(irisDataset)
+        var score2 = evaluator2.evaluate(irisDataset.withColumn("weight", lit(w)))
+        assert(score1 ~== score2 relTol 1e-6)
+
+        score1 = evaluator1.evaluate(newIrisDataset)
+        score2 = evaluator2.evaluate(newIrisDataset.withColumn("weight", lit(w)))
+        assert(score1 ~== score2 relTol 1e-6)
+      }
+    }
+  }
+
+  test("single-element clusters with weight") {
+    val singleItemClusters = spark.createDataFrame(spark.sparkContext.parallelize(Seq(
+      (0.0, Vectors.dense(5.1, 3.5, 1.4, 0.2), 6.0),
+      (1.0, Vectors.dense(7.0, 3.2, 4.7, 1.4), 0.25),
+      (2.0, Vectors.dense(6.3, 3.3, 6.0, 2.5), 9.99)))).toDF("label", "features", "weight")
+    Seq("squaredEuclidean", "cosine").foreach { distanceMeasure =>
+      val evaluator = new ClusteringEvaluator()
+        .setFeaturesCol("features")
+        .setPredictionCol("label")
+        .setDistanceMeasure(distanceMeasure)
+        .setWeightCol("weight")
+      assert(evaluator.evaluate(singleItemClusters) === 0.0)
+    }
+  }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/evaluation/MulticlassClassificationEvaluatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/evaluation/MulticlassClassificationEvaluatorSuite.scala
index 5b5212abdf7cc..3dfd860a5b9d8 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/evaluation/MulticlassClassificationEvaluatorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/evaluation/MulticlassClassificationEvaluatorSuite.scala
@@ -80,4 +80,33 @@ class MulticlassClassificationEvaluatorSuite
       .setMetricName("logLoss")
     assert(evaluator.evaluate(df) ~== 0.9682005730687164 absTol 1e-5)
   }
+
+  test("getMetrics") {
+    val predictionAndLabels = Seq((0.0, 0.0), (0.0, 1.0),
+      (0.0, 0.0), (1.0, 0.0), (1.0, 1.0),
+      (1.0, 1.0), (1.0, 1.0), (2.0, 2.0), (2.0, 0.0)).toDF("prediction", "label")
+
+    val evaluator = new MulticlassClassificationEvaluator()
+
+    val metrics = evaluator.getMetrics(predictionAndLabels)
+    val f1 = metrics.weightedFMeasure
+    val accuracy = metrics.accuracy
+    val precisionByLabel = metrics.precision(evaluator.getMetricLabel)
+
+    // default = f1
+    assert(evaluator.evaluate(predictionAndLabels) == f1)
+
+    // accuracy
+    evaluator.setMetricName("accuracy")
+    assert(evaluator.evaluate(predictionAndLabels) == accuracy)
+
+    // precisionByLabel
+    evaluator.setMetricName("precisionByLabel")
+    assert(evaluator.evaluate(predictionAndLabels) == precisionByLabel)
+
+    // truePositiveRateByLabel
+    evaluator.setMetricName("truePositiveRateByLabel").setMetricLabel(1.0)
+    assert(evaluator.evaluate(predictionAndLabels) ==
+      metrics.truePositiveRate(evaluator.getMetricLabel))
+  }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluatorSuite.scala
index f41fc04a5faed..520103d6aed92 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluatorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluatorSuite.scala
@@ -59,4 +59,52 @@ class MultilabelClassificationEvaluatorSuite
       .setMetricName("precisionByLabel")
     testDefaultReadWrite(evaluator)
   }
+
+  test("getMetrics") {
+    val scoreAndLabels = Seq((Array(0.0, 1.0), Array(0.0, 2.0)),
+      (Array(0.0, 2.0), Array(0.0, 1.0)),
+      (Array.empty[Double], Array(0.0)),
+      (Array(2.0), Array(2.0)),
+      (Array(2.0, 0.0), Array(2.0, 0.0)),
+      (Array(0.0, 1.0, 2.0), Array(0.0, 1.0)),
+      (Array(1.0), Array(1.0, 2.0))).toDF("prediction", "label")
+
+    val evaluator = new MultilabelClassificationEvaluator()
+
+    val metrics = evaluator.getMetrics(scoreAndLabels)
+    val f1 = metrics.f1Measure
+    val accuracy = metrics.accuracy
+    val precision = metrics.precision
+    val recall = metrics.recall
+    val hammingLoss = metrics.hammingLoss
+    val precisionByLabel = metrics.precision(evaluator.getMetricLabel)
+
+    // default = f1
+    assert(evaluator.evaluate(scoreAndLabels) == f1)
+
+    // accuracy
+    evaluator.setMetricName("accuracy")
+    assert(evaluator.evaluate(scoreAndLabels) == accuracy)
+
+    // precision
+    evaluator.setMetricName("precision")
+    assert(evaluator.evaluate(scoreAndLabels) == precision)
+
+    // recall
+    evaluator.setMetricName("recall")
+    assert(evaluator.evaluate(scoreAndLabels) == recall)
+
+    // hammingLoss
+    evaluator.setMetricName("hammingLoss")
+    assert(evaluator.evaluate(scoreAndLabels) == hammingLoss)
+
+    // precisionByLabel
+    evaluator.setMetricName("precisionByLabel")
+    assert(evaluator.evaluate(scoreAndLabels) == precisionByLabel)
+
+    // truePositiveRateByLabel
+    evaluator.setMetricName("recallByLabel").setMetricLabel(1.0)
+    assert(evaluator.evaluate(scoreAndLabels) ==
+      metrics.recall(evaluator.getMetricLabel))
+  }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/evaluation/RankingEvaluatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/evaluation/RankingEvaluatorSuite.scala
index 02d26d7eb351f..b3457981a08e9 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/evaluation/RankingEvaluatorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/evaluation/RankingEvaluatorSuite.scala
@@ -59,4 +59,42 @@ class RankingEvaluatorSuite
       .setK(2)
     assert(evaluator.evaluate(scoreAndLabels) ~== 1.0 / 3 absTol 1e-5)
   }
+
+  test("getMetrics") {
+    val scoreAndLabels = Seq(
+      (Array(1.0, 6.0, 2.0, 7.0, 8.0, 3.0, 9.0, 10.0, 4.0, 5.0),
+        Array(1.0, 2.0, 3.0, 4.0, 5.0)),
+      (Array(4.0, 1.0, 5.0, 6.0, 2.0, 7.0, 3.0, 8.0, 9.0, 10.0),
+        Array(1.0, 2.0, 3.0)),
+      (Array(1.0, 2.0, 3.0, 4.0, 5.0), Array.empty[Double])
+    ).toDF("prediction", "label")
+
+    val evaluator = new RankingEvaluator().setK(5)
+
+    val metrics = evaluator.getMetrics(scoreAndLabels)
+    val meanAveragePrecision = metrics.meanAveragePrecision
+    val meanAveragePrecisionAtK = metrics.meanAveragePrecisionAt(evaluator.getK)
+    val precisionAtK = metrics.precisionAt(evaluator.getK)
+    val ndcgAtK = metrics.ndcgAt(evaluator.getK)
+    val recallAtK = metrics.recallAt(evaluator.getK)
+
+    // default = meanAveragePrecision
+    assert(evaluator.evaluate(scoreAndLabels) == meanAveragePrecision)
+
+    // meanAveragePrecisionAtK
+    evaluator.setMetricName("meanAveragePrecisionAtK")
+    assert(evaluator.evaluate(scoreAndLabels) == meanAveragePrecisionAtK)
+
+    // precisionAtK
+    evaluator.setMetricName("precisionAtK")
+    assert(evaluator.evaluate(scoreAndLabels) == precisionAtK)
+
+    // ndcgAtK
+    evaluator.setMetricName("ndcgAtK")
+    assert(evaluator.evaluate(scoreAndLabels) == ndcgAtK)
+
+    // recallAtK
+    evaluator.setMetricName("recallAtK")
+    assert(evaluator.evaluate(scoreAndLabels) == recallAtK)
+  }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/evaluation/RegressionEvaluatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/evaluation/RegressionEvaluatorSuite.scala
index f4f858c3e92dc..deaad2bd54d0e 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/evaluation/RegressionEvaluatorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/evaluation/RegressionEvaluatorSuite.scala
@@ -36,7 +36,7 @@ class RegressionEvaluatorSuite
   test("Regression Evaluator: default params") {
     /**
      * Here is the instruction describing how to export the test data into CSV format
-     * so we can validate the metrics compared with R's mmetric package.
+     * so we can validate the metrics compared with R's mmetric function.
      *
      * import org.apache.spark.mllib.util.LinearDataGenerator
      * val data = sc.parallelize(LinearDataGenerator.generateLinearInput(6.3,
@@ -93,4 +93,37 @@ class RegressionEvaluatorSuite
   test("should support all NumericType labels and not support other types") {
     MLTestingUtils.checkNumericTypes(new RegressionEvaluator, spark)
   }
+
+  test("getMetrics") {
+    val dataset = LinearDataGenerator.generateLinearInput(
+      6.3, Array(4.7, 7.2), Array(0.9, -1.3), Array(0.7, 1.2), 100, 42, 0.1)
+      .map(_.asML).toDF()
+
+    val trainer = new LinearRegression
+    val model = trainer.fit(dataset)
+    val predictions = model.transform(dataset)
+
+    val evaluator = new RegressionEvaluator()
+
+    val metrics = evaluator.getMetrics(predictions)
+    val rmse = metrics.rootMeanSquaredError
+    val r2 = metrics.r2
+    val mae = metrics.meanAbsoluteError
+    val variance = metrics.explainedVariance
+
+    // default = rmse
+    assert(evaluator.evaluate(predictions) == rmse)
+
+    // r2 score
+    evaluator.setMetricName("r2")
+    assert(evaluator.evaluate(predictions) == r2)
+
+    // mae
+    evaluator.setMetricName("mae")
+    assert(evaluator.evaluate(predictions) == mae)
+
+    // var
+    evaluator.setMetricName("var")
+    assert(evaluator.evaluate(predictions) == variance)
+  }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/ANOVASelectorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/ANOVASelectorSuite.scala
deleted file mode 100644
index 1e1ab206cc1c2..0000000000000
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/ANOVASelectorSuite.scala
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.ml.feature
-
-import org.apache.spark.ml.linalg.{Vector, Vectors}
-import org.apache.spark.ml.param.ParamsSuite
-import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTest, MLTestingUtils}
-import org.apache.spark.ml.util.TestingUtils._
-import org.apache.spark.sql.{Dataset, Row}
-
-class ANOVASelectorSuite extends MLTest with DefaultReadWriteTest {
-
-  import testImplicits._
-
-  @transient var dataset: Dataset[_] = _
-
-  override def beforeAll(): Unit = {
-    super.beforeAll()
-
-    // scalastyle:off
-    /*
-      X:
-      array([[4.65415496e-03, 1.03550567e-01, -1.17358140e+00,
-      1.61408773e-01,  3.92492111e-01,  7.31240882e-01],
-      [-9.01651741e-01, -5.28905302e-01,  1.27636785e+00,
-      7.02154563e-01,  6.21348351e-01,  1.88397353e-01],
-      [ 3.85692159e-01, -9.04639637e-01,  5.09782604e-02,
-      8.40043971e-01,  7.45977857e-01,  8.78402288e-01],
-      [ 1.36264353e+00,  2.62454094e-01,  7.96306202e-01,
-      6.14948000e-01,  7.44948187e-01,  9.74034830e-01],
-      [ 9.65874070e-01,  2.52773665e+00, -2.19380094e+00,
-      2.33408080e-01,  1.86340919e-01,  8.23390433e-01],
-      [ 1.12324305e+01, -2.77121515e-01,  1.12740513e-01,
-      2.35184013e-01,  3.46668895e-01,  9.38500782e-02],
-      [ 1.06195839e+01, -1.82891238e+00,  2.25085601e-01,
-      9.09979851e-01,  6.80257535e-02,  8.24017480e-01],
-      [ 1.12806837e+01,  1.30686889e+00,  9.32839108e-02,
-      3.49784755e-01,  1.71322408e-02,  7.48465194e-02],
-      [ 9.98689462e+00,  9.50808938e-01, -2.90786359e-01,
-      2.31253009e-01,  7.46270968e-01,  1.60308169e-01],
-      [ 1.08428551e+01, -1.02749936e+00,  1.73951508e-01,
-      8.92482744e-02,  1.42651730e-01,  7.66751625e-01],
-      [-1.98641448e+00,  1.12811990e+01, -2.35246756e-01,
-      8.22809049e-01,  3.26739456e-01,  7.88268404e-01],
-      [-6.09864090e-01,  1.07346276e+01, -2.18805509e-01,
-      7.33931213e-01,  1.42554396e-01,  7.11225605e-01],
-      [-1.58481268e+00,  9.19364039e+00, -5.87490459e-02,
-      2.51532056e-01,  2.82729807e-01,  7.16245686e-01],
-      [-2.50949277e-01,  1.12815254e+01, -6.94806734e-01,
-      5.93898886e-01,  5.68425656e-01,  8.49762330e-01],
-      [ 7.63485129e-01,  1.02605138e+01,  1.32617719e+00,
-      5.49682879e-01,  8.59931442e-01,  4.88677978e-02],
-      [ 9.34900015e-01,  4.11379043e-01,  8.65010205e+00,
-      9.23509168e-01,  1.16995043e-01,  5.91894106e-03],
-      [ 4.73734933e-01, -1.48321181e+00,  9.73349621e+00,
-      4.09421563e-01,  5.09375719e-01,  5.93157850e-01],
-      [ 3.41470679e-01, -6.88972582e-01,  9.60347938e+00,
-      3.62654055e-01,  2.43437468e-01,  7.13052838e-01],
-      [-5.29614251e-01, -1.39262856e+00,  1.01354144e+01,
-      8.24123861e-01,  5.84074506e-01,  6.54461558e-01],
-      [-2.99454508e-01,  2.20457263e+00,  1.14586015e+01,
-      5.16336729e-01,  9.99776159e-01,  3.15769738e-01]])
-      y:
-      array([1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4])
-      scikit-learn result:
-      >>> f_classif(X, y)
-      (array([228.27701422,  84.33070501, 134.25330675,   0.82211775, 0.82991363,   1.08478943]),
-       array([2.43864448e-13, 5.09088367e-10, 1.49033067e-11, 5.00596446e-01, 4.96684374e-01, 3.83798191e-01]))
-    */
-    // scalastyle:on
-
-    val data = Seq(
-      (1, Vectors.dense(4.65415496e-03, 1.03550567e-01, -1.17358140e+00,
-      1.61408773e-01, 3.92492111e-01, 7.31240882e-01), Vectors.dense(4.65415496e-03)),
-      (1, Vectors.dense(-9.01651741e-01, -5.28905302e-01, 1.27636785e+00,
-      7.02154563e-01, 6.21348351e-01, 1.88397353e-01), Vectors.dense(-9.01651741e-01)),
-      (1, Vectors.dense(3.85692159e-01, -9.04639637e-01, 5.09782604e-02,
-      8.40043971e-01, 7.45977857e-01, 8.78402288e-01), Vectors.dense(3.85692159e-01)),
-      (1, Vectors.dense(1.36264353e+00, 2.62454094e-01, 7.96306202e-01,
-      6.14948000e-01, 7.44948187e-01, 9.74034830e-01), Vectors.dense(1.36264353e+00)),
-      (1, Vectors.dense(9.65874070e-01, 2.52773665e+00, -2.19380094e+00,
-        2.33408080e-01, 1.86340919e-01, 8.23390433e-01), Vectors.dense(9.65874070e-01)),
-      (2, Vectors.dense(1.12324305e+01, -2.77121515e-01, 1.12740513e-01,
-        2.35184013e-01, 3.46668895e-01, 9.38500782e-02), Vectors.dense(1.12324305e+01)),
-      (2, Vectors.dense(1.06195839e+01, -1.82891238e+00, 2.25085601e-01,
-        9.09979851e-01, 6.80257535e-02, 8.24017480e-01), Vectors.dense(1.06195839e+01)),
-      (2, Vectors.dense(1.12806837e+01, 1.30686889e+00, 9.32839108e-02,
-        3.49784755e-01, 1.71322408e-02, 7.48465194e-02), Vectors.dense(1.12806837e+01)),
-      (2, Vectors.dense(9.98689462e+00, 9.50808938e-01, -2.90786359e-01,
-        2.31253009e-01, 7.46270968e-01, 1.60308169e-01), Vectors.dense(9.98689462e+00)),
-      (2, Vectors.dense(1.08428551e+01, -1.02749936e+00, 1.73951508e-01,
-        8.92482744e-02, 1.42651730e-01, 7.66751625e-01), Vectors.dense(1.08428551e+01)),
-      (3, Vectors.dense(-1.98641448e+00, 1.12811990e+01, -2.35246756e-01,
-        8.22809049e-01, 3.26739456e-01, 7.88268404e-01), Vectors.dense(-1.98641448e+00)),
-      (3, Vectors.dense(-6.09864090e-01, 1.07346276e+01, -2.18805509e-01,
-        7.33931213e-01, 1.42554396e-01, 7.11225605e-01), Vectors.dense(-6.09864090e-01)),
-      (3, Vectors.dense(-1.58481268e+00, 9.19364039e+00, -5.87490459e-02,
-        2.51532056e-01, 2.82729807e-01, 7.16245686e-01), Vectors.dense(-1.58481268e+00)),
-      (3, Vectors.dense(-2.50949277e-01, 1.12815254e+01, -6.94806734e-01,
-        5.93898886e-01, 5.68425656e-01, 8.49762330e-01), Vectors.dense(-2.50949277e-01)),
-      (3, Vectors.dense(7.63485129e-01, 1.02605138e+01, 1.32617719e+00,
-        5.49682879e-01, 8.59931442e-01, 4.88677978e-02), Vectors.dense(7.63485129e-01)),
-      (4, Vectors.dense(9.34900015e-01, 4.11379043e-01, 8.65010205e+00,
-        9.23509168e-01, 1.16995043e-01, 5.91894106e-03), Vectors.dense(9.34900015e-01)),
-      (4, Vectors.dense(4.73734933e-01, -1.48321181e+00, 9.73349621e+00,
-        4.09421563e-01, 5.09375719e-01, 5.93157850e-01), Vectors.dense(4.73734933e-01)),
-      (4, Vectors.dense(3.41470679e-01, -6.88972582e-01, 9.60347938e+00,
-        3.62654055e-01, 2.43437468e-01, 7.13052838e-01), Vectors.dense(3.41470679e-01)),
-      (4, Vectors.dense(-5.29614251e-01, -1.39262856e+00, 1.01354144e+01,
-        8.24123861e-01, 5.84074506e-01, 6.54461558e-01), Vectors.dense(-5.29614251e-01)),
-      (4, Vectors.dense(-2.99454508e-01, 2.20457263e+00, 1.14586015e+01,
-        5.16336729e-01, 9.99776159e-01, 3.15769738e-01), Vectors.dense(-2.99454508e-01)))
-
-    dataset = spark.createDataFrame(data).toDF("label", "features", "topFeature")
-  }
-
-  test("params") {
-    ParamsSuite.checkParams(new ANOVASelector())
-  }
-
-  test("Test ANOVAFValue calssification selector: numTopFeatures") {
-    val selector = new ANOVASelector()
-      .setOutputCol("filtered").setSelectorType("numTopFeatures").setNumTopFeatures(1)
-    val model = testSelector(selector, dataset)
-    MLTestingUtils.checkCopyAndUids(selector, model)
-  }
-
-  test("Test ANOVAFValue calssification selector: percentile") {
-    val selector = new ANOVASelector()
-      .setOutputCol("filtered").setSelectorType("percentile").setPercentile(0.17)
-    val model = testSelector(selector, dataset)
-    MLTestingUtils.checkCopyAndUids(selector, model)
-  }
-
-  test("Test ANOVAFValue calssification selector: fpr") {
-    val selector = new ANOVASelector()
-      .setOutputCol("filtered").setSelectorType("fpr").setFpr(1.0E-12)
-    val model = testSelector(selector, dataset)
-    MLTestingUtils.checkCopyAndUids(selector, model)
-  }
-
-  test("Test ANOVAFValue calssification selector: fdr") {
-    val selector = new ANOVASelector()
-      .setOutputCol("filtered").setSelectorType("fdr").setFdr(6.0E-12)
-    val model = testSelector(selector, dataset)
-    MLTestingUtils.checkCopyAndUids(selector, model)
-  }
-
-  test("Test ANOVAFValue calssification selector: fwe") {
-    val selector = new ANOVASelector()
-      .setOutputCol("filtered").setSelectorType("fwe").setFwe(6.0E-12)
-    val model = testSelector(selector, dataset)
-    MLTestingUtils.checkCopyAndUids(selector, model)
-  }
-
-  test("read/write") {
-    def checkModelData(model: ANOVASelectorModel, model2: ANOVASelectorModel): Unit = {
-      assert(model.selectedFeatures === model2.selectedFeatures)
-    }
-    val anovaSelector = new ANOVASelector()
-    testEstimatorAndModelReadWrite(anovaSelector, dataset,
-      ANOVASelectorSuite.allParamSettings,
-      ANOVASelectorSuite.allParamSettings, checkModelData)
-  }
-
-  private def testSelector(selector: ANOVASelector, data: Dataset[_]):
-  ANOVASelectorModel = {
-    val selectorModel = selector.fit(data)
-    testTransformer[(Double, Vector, Vector)](data.toDF(), selectorModel,
-      "filtered", "topFeature") {
-      case Row(vec1: Vector, vec2: Vector) =>
-        assert(vec1 ~== vec2 absTol 1e-1)
-    }
-    selectorModel
-  }
-}
-
-object ANOVASelectorSuite {
-
-  /**
-   * Mapping from all Params to valid settings which differ from the defaults.
-   * This is useful for tests which need to exercise all Params, such as save/load.
-   * This excludes input columns to simplify some tests.
-   */
-  val allParamSettings: Map[String, Any] = Map(
-    "selectorType" -> "percentile",
-    "numTopFeatures" -> 1,
-    "percentile" -> 0.12,
-    "outputCol" -> "myOutput"
-  )
-}
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/BucketedRandomProjectionLSHSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/BucketedRandomProjectionLSHSuite.scala
index 9b823259b1deb..a7d320e8164b6 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/BucketedRandomProjectionLSHSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/BucketedRandomProjectionLSHSuite.scala
@@ -115,7 +115,7 @@ class BucketedRandomProjectionLSHSuite extends MLTest with DefaultReadWriteTest
     val brpModel = brp.fit(dataset)
 
     testTransformer[Tuple1[Vector]](dataset.toDF(), brpModel, "values") {
-      case Row(values: Seq[_]) =>
+      case Row(values: scala.collection.Seq[_]) =>
         assert(values.length === brp.getNumHashTables)
     }
   }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/CountVectorizerSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/CountVectorizerSuite.scala
index bca580d411373..5e32a654c130b 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/CountVectorizerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/CountVectorizerSuite.scala
@@ -188,21 +188,6 @@ class CountVectorizerSuite extends MLTest with DefaultReadWriteTest {
     }
   }
 
-  test("CountVectorizer throws exception when vocab is empty") {
-    intercept[IllegalArgumentException] {
-      val df = Seq(
-        (0, split("a a b b c c")),
-        (1, split("aa bb cc"))
-      ).toDF("id", "words")
-      val cvModel = new CountVectorizer()
-        .setInputCol("words")
-        .setOutputCol("features")
-        .setVocabSize(3) // limit vocab size to 3
-        .setMinDF(3)
-        .fit(df)
-    }
-  }
-
   test("CountVectorizerModel with minTF count") {
     val df = Seq(
       (0, split("a a a b b c c c d "), Vectors.sparse(4, Seq((0, 3.0), (2, 3.0)))),
@@ -305,4 +290,63 @@ class CountVectorizerSuite extends MLTest with DefaultReadWriteTest {
       .setOutputCol("features")
     interaction.transform(df1)
   }
+
+  test("SPARK-32662: Test on empty dataset") {
+    val df = Seq[(Int, Array[String])]().toDF("id", "words")
+    val cvModel = new CountVectorizer()
+      .setInputCol("words")
+      .setOutputCol("features")
+      .fit(df)
+    assert(cvModel.vocabulary.isEmpty === true)
+    val ans = cvModel.transform(df).select("features").collect()
+    assert(ans.length === 0)
+  }
+
+  test("SPARK-32662: Remove requirement for minimum vocabulary size") {
+    val df = Seq(
+      (0, Array[String]()),
+      (1, Array[String]())
+    ).toDF("id", "words")
+    val cvModel = new CountVectorizer()
+      .setInputCol("words")
+      .setOutputCol("features")
+      .fit(df)
+    assert(cvModel.vocabulary.isEmpty === true)
+    testTransformer[(Int, Seq[String])](df, cvModel, "features") {
+      case Row(features: Vector) =>
+        assert(features === Vectors.sparse(0, Seq()))
+    }
+
+    val df2 = Seq(
+      (0, Array("a", "b", "c")),
+      (1, Array("d", "e")),
+      (2, Array[String]())
+    ).toDF("id", "words")
+    val cvModel2 = new CountVectorizer()
+      .setInputCol("words")
+      .setOutputCol("features")
+      .setMinDF(2)
+      .fit(df2)
+    assert(cvModel2.vocabulary.isEmpty === true)
+    testTransformer[(Int, Seq[String])](df2, cvModel2, "features") {
+      case Row(features: Vector) =>
+        assert(features === Vectors.sparse(0, Seq()))
+    }
+
+    val df3 = Seq(
+      (0, Array("a")),
+      (1, Array("a")),
+      (2, Array("a"))
+    ).toDF("id", "words")
+    val cvModel3 = new CountVectorizer()
+      .setInputCol("words")
+      .setOutputCol("features")
+      .setMaxDF(2)
+      .fit(df3)
+    assert(cvModel3.vocabulary.isEmpty === true)
+    testTransformer[(Int, Seq[String])](df3, cvModel3, "features") {
+      case Row(features: Vector) =>
+        assert(features === Vectors.sparse(0, Seq()))
+    }
+  }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/DCTSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/DCTSuite.scala
index 19645b517d79c..8f8365a59082b 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/DCTSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/DCTSuite.scala
@@ -81,7 +81,7 @@ class DCTSuite extends MLTest with DefaultReadWriteTest {
       .map { case Row(vec: Vector) => vec.size }
       .head()
 
-    // Can not infer size of ouput vector, since no metadata is provided
+    // Can not infer size of output vector, since no metadata is provided
     intercept[TestFailedException] {
       val transformed = transformer.transform(dataset)
       checkVectorSizeOnDF(transformed, "resultVec", vectorSize)
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/FValueSelectorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/FValueSelectorSuite.scala
deleted file mode 100644
index 5c1200188bb34..0000000000000
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/FValueSelectorSuite.scala
+++ /dev/null
@@ -1,238 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.ml.feature
-
-import org.apache.spark.ml.linalg.{Vector, Vectors}
-import org.apache.spark.ml.param.ParamsSuite
-import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTest, MLTestingUtils}
-import org.apache.spark.ml.util.TestingUtils._
-import org.apache.spark.sql.{Dataset, Row}
-
-class FValueSelectorSuite extends MLTest with DefaultReadWriteTest {
-
-  import testImplicits._
-
-  @transient var dataset: Dataset[_] = _
-
-  override def beforeAll(): Unit = {
-    super.beforeAll()
-
-    // scalastyle:off
-    /*
-    Use the following sklearn data in this test
-
-    >>> from sklearn.feature_selection import f_regression
-    >>> import numpy as np
-    >>> np.random.seed(777)
-    >>> X = np.random.rand(20, 6)
-    >>> w = np.array([0.3, 0.4, 0.5, 0, 0, 0])
-    >>> y = X @ w
-    >>> X
-    array([[0.19151945, 0.62210877, 0.43772774, 0.78535858, 0.77997581,
-            0.27259261],
-           [0.27646426, 0.80187218, 0.95813935, 0.87593263, 0.35781727,
-            0.50099513],
-           [0.68346294, 0.71270203, 0.37025075, 0.56119619, 0.50308317,
-            0.01376845],
-           [0.77282662, 0.88264119, 0.36488598, 0.61539618, 0.07538124,
-            0.36882401],
-           [0.9331401 , 0.65137814, 0.39720258, 0.78873014, 0.31683612,
-            0.56809865],
-           [0.86912739, 0.43617342, 0.80214764, 0.14376682, 0.70426097,
-            0.70458131],
-           [0.21879211, 0.92486763, 0.44214076, 0.90931596, 0.05980922,
-            0.18428708],
-           [0.04735528, 0.67488094, 0.59462478, 0.53331016, 0.04332406,
-            0.56143308],
-           [0.32966845, 0.50296683, 0.11189432, 0.60719371, 0.56594464,
-            0.00676406],
-           [0.61744171, 0.91212289, 0.79052413, 0.99208147, 0.95880176,
-            0.79196414],
-           [0.28525096, 0.62491671, 0.4780938 , 0.19567518, 0.38231745,
-            0.05387369],
-           [0.45164841, 0.98200474, 0.1239427 , 0.1193809 , 0.73852306,
-            0.58730363],
-           [0.47163253, 0.10712682, 0.22921857, 0.89996519, 0.41675354,
-            0.53585166],
-           [0.00620852, 0.30064171, 0.43689317, 0.612149  , 0.91819808,
-            0.62573667],
-           [0.70599757, 0.14983372, 0.74606341, 0.83100699, 0.63372577,
-            0.43830988],
-           [0.15257277, 0.56840962, 0.52822428, 0.95142876, 0.48035918,
-            0.50255956],
-           [0.53687819, 0.81920207, 0.05711564, 0.66942174, 0.76711663,
-             0.70811536],
-           [0.79686718, 0.55776083, 0.96583653, 0.1471569 , 0.029647  ,
-            0.59389349],
-           [0.1140657 , 0.95080985, 0.32570741, 0.19361869, 0.45781165,
-            0.92040257],
-           [0.87906916, 0.25261576, 0.34800879, 0.18258873, 0.90179605,
-            0.70652816]])
-    >>> y
-    array([0.52516321, 0.88275782, 0.67524507, 0.76734745, 0.73909458,
-           0.83628141, 0.65665506, 0.58147135, 0.35603443, 0.94534373,
-           0.57458887, 0.59026777, 0.29894977, 0.34056582, 0.64476446,
-           0.53724782, 0.5173021 , 0.94508275, 0.57739736, 0.53877145])
-    >>> f_regression(X, y)
-    (array([5.58025504,  3.98311705, 20.59605518,  0.07993376,  1.25127646,
-            0.7676937 ]),
-    array([2.96302196e-02, 6.13173918e-02, 2.54580618e-04, 7.80612726e-01,
-    2.78015517e-01, 3.92474567e-01]))
-    */
-    // scalastyle:on
-
-    val data = Seq(
-      (0.52516321, Vectors.dense(0.19151945, 0.62210877, 0.43772774, 0.78535858, 0.77997581,
-        0.27259261), Vectors.dense(0.43772774)),
-      (0.88275782, Vectors.dense(0.27646426, 0.80187218, 0.95813935, 0.87593263, 0.35781727,
-        0.50099513), Vectors.dense(0.95813935)),
-      (0.67524507, Vectors.dense(0.68346294, 0.71270203, 0.37025075, 0.56119619, 0.50308317,
-        0.01376845), Vectors.dense(0.37025075)),
-      (0.76734745, Vectors.dense(0.77282662, 0.88264119, 0.36488598, 0.61539618, 0.07538124,
-        0.36882401), Vectors.dense(0.36488598)),
-      (0.73909458, Vectors.dense(0.9331401, 0.65137814, 0.39720258, 0.78873014, 0.31683612,
-        0.56809865), Vectors.dense(0.39720258)),
-
-      (0.83628141, Vectors.dense(0.86912739, 0.43617342, 0.80214764, 0.14376682, 0.70426097,
-        0.70458131), Vectors.dense(0.80214764)),
-      (0.65665506, Vectors.dense(0.21879211, 0.92486763, 0.44214076, 0.90931596, 0.05980922,
-        0.18428708), Vectors.dense(0.44214076)),
-      (0.58147135, Vectors.dense(0.04735528, 0.67488094, 0.59462478, 0.53331016, 0.04332406,
-        0.56143308), Vectors.dense(0.59462478)),
-      (0.35603443, Vectors.dense(0.32966845, 0.50296683, 0.11189432, 0.60719371, 0.56594464,
-        0.00676406), Vectors.dense(0.11189432)),
-      (0.94534373, Vectors.dense(0.61744171, 0.91212289, 0.79052413, 0.99208147, 0.95880176,
-        0.79196414), Vectors.dense(0.79052413)),
-
-      (0.57458887, Vectors.dense(0.28525096, 0.62491671, 0.4780938, 0.19567518, 0.38231745,
-        0.05387369), Vectors.dense(0.4780938)),
-      (0.59026777, Vectors.dense(0.45164841, 0.98200474, 0.1239427, 0.1193809, 0.73852306,
-        0.58730363), Vectors.dense(0.1239427)),
-      (0.29894977, Vectors.dense(0.47163253, 0.10712682, 0.22921857, 0.89996519, 0.41675354,
-        0.53585166), Vectors.dense(0.22921857)),
-      (0.34056582, Vectors.dense(0.00620852, 0.30064171, 0.43689317, 0.612149, 0.91819808,
-        0.62573667), Vectors.dense(0.43689317)),
-      (0.64476446, Vectors.dense(0.70599757, 0.14983372, 0.74606341, 0.83100699, 0.63372577,
-        0.43830988), Vectors.dense(0.74606341)),
-
-      (0.53724782, Vectors.dense(0.15257277, 0.56840962, 0.52822428, 0.95142876, 0.48035918,
-        0.50255956), Vectors.dense(0.52822428)),
-      (0.5173021, Vectors.dense(0.53687819, 0.81920207, 0.05711564, 0.66942174, 0.76711663,
-        0.70811536), Vectors.dense(0.05711564)),
-      (0.94508275, Vectors.dense(0.79686718, 0.55776083, 0.96583653, 0.1471569, 0.029647,
-        0.59389349), Vectors.dense(0.96583653)),
-      (0.57739736, Vectors.dense(0.1140657, 0.95080985, 0.96583653, 0.19361869, 0.45781165,
-        0.92040257), Vectors.dense(0.96583653)),
-      (0.53877145, Vectors.dense(0.87906916, 0.25261576, 0.34800879, 0.18258873, 0.90179605,
-        0.70652816), Vectors.dense(0.34800879)))
-
-    dataset = spark.createDataFrame(data).toDF("label", "features", "topFeature")
-  }
-
-  test("params") {
-    ParamsSuite.checkParams(new FValueSelector)
-  }
-
-  test("Test FValue selector: numTopFeatures") {
-    val selector = new FValueSelector()
-      .setOutputCol("filtered").setSelectorType("numTopFeatures").setNumTopFeatures(1)
-    val model = testSelector(selector, dataset)
-    MLTestingUtils.checkCopyAndUids(selector, model)
-  }
-
-  test("Test F Value selector: percentile") {
-    val selector = new FValueSelector()
-      .setOutputCol("filtered").setSelectorType("percentile").setPercentile(0.17)
-    val model = testSelector(selector, dataset)
-    MLTestingUtils.checkCopyAndUids(selector, model)
-  }
-
-  test("Test F Value selector: fpr") {
-    val selector = new FValueSelector()
-      .setOutputCol("filtered").setSelectorType("fpr").setFpr(0.01)
-    val model = testSelector(selector, dataset)
-    MLTestingUtils.checkCopyAndUids(selector, model)
-  }
-
-  test("Test F Value selector: fdr") {
-    val selector = new FValueSelector()
-      .setOutputCol("filtered").setSelectorType("fdr").setFdr(0.03)
-    val model = testSelector(selector, dataset)
-    MLTestingUtils.checkCopyAndUids(selector, model)
-  }
-
-  test("Test F Value selector: fwe") {
-    val selector = new FValueSelector()
-      .setOutputCol("filtered").setSelectorType("fwe").setFwe(0.03)
-    val model = testSelector(selector, dataset)
-    MLTestingUtils.checkCopyAndUids(selector, model)
-  }
-
-  test("Test FValue selector with sparse vector") {
-    val df = spark.createDataFrame(Seq(
-      (4.6, Vectors.sparse(6, Array((0, 6.0), (1, 7.0), (3, 7.0), (4, 6.0))), Vectors.dense(0.0)),
-      (6.6, Vectors.sparse(6, Array((1, 9.0), (2, 6.0), (4, 5.0), (5, 9.0))), Vectors.dense(6.0)),
-      (5.1, Vectors.sparse(6, Array((1, 9.0), (2, 3.0), (4, 5.0), (5, 5.0))), Vectors.dense(3.0)),
-      (7.6, Vectors.dense(Array(0.0, 9.0, 8.0, 5.0, 6.0, 4.0)), Vectors.dense(8.0)),
-      (9.0, Vectors.dense(Array(8.0, 9.0, 6.0, 5.0, 4.0, 4.0)), Vectors.dense(6.0)),
-      (9.0, Vectors.dense(Array(8.0, 9.0, 6.0, 4.0, 0.0, 0.0)), Vectors.dense(6.0))
-    )).toDF("label", "features", "topFeature")
-
-    val selector = new FValueSelector()
-      .setOutputCol("filtered").setSelectorType("numTopFeatures").setNumTopFeatures(1)
-    val model = testSelector(selector, df)
-    MLTestingUtils.checkCopyAndUids(selector, model)
-  }
-
-  test("read/write") {
-    def checkModelData(model: FValueSelectorModel, model2:
-      FValueSelectorModel): Unit = {
-      assert(model.selectedFeatures === model2.selectedFeatures)
-    }
-    val fSelector = new FValueSelector
-    testEstimatorAndModelReadWrite(fSelector, dataset,
-      FValueSelectorSuite.allParamSettings,
-      FValueSelectorSuite.allParamSettings, checkModelData)
-  }
-
-  private def testSelector(selector: FValueSelector, data: Dataset[_]):
-      FValueSelectorModel = {
-    val selectorModel = selector.fit(data)
-    testTransformer[(Double, Vector, Vector)](data.toDF(), selectorModel,
-      "filtered", "topFeature") {
-      case Row(vec1: Vector, vec2: Vector) =>
-        assert(vec1 ~== vec2 absTol 1e-6)
-    }
-    selectorModel
-  }
-}
-
-object FValueSelectorSuite {
-
-  /**
-   * Mapping from all Params to valid settings which differ from the defaults.
-   * This is useful for tests which need to exercise all Params, such as save/load.
-   * This excludes input columns to simplify some tests.
-   */
-  val allParamSettings: Map[String, Any] = Map(
-    "selectorType" -> "percentile",
-    "numTopFeatures" -> 1,
-    "percentile" -> 0.12,
-    "outputCol" -> "myOutput"
-  )
-}
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/HashingTFSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/HashingTFSuite.scala
index 8fd192fa56500..861bf1e0b1292 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/HashingTFSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/HashingTFSuite.scala
@@ -99,7 +99,7 @@ class HashingTFSuite extends MLTest with DefaultReadWriteTest {
 
     val metadata = spark.read.json(s"$hashingTFPath/metadata")
     val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
-    assert(sparkVersionStr == "2.4.4")
+    assert(sparkVersionStr === "2.4.4")
 
     intercept[IllegalArgumentException] {
       loadedHashingTF.save(hashingTFPath)
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/ImputerSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/ImputerSuite.scala
index dfee2b4029c8b..30887f55638f9 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/ImputerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/ImputerSuite.scala
@@ -28,13 +28,14 @@ import org.apache.spark.sql.types._
 class ImputerSuite extends MLTest with DefaultReadWriteTest {
 
   test("Imputer for Double with default missing Value NaN") {
-    val df = spark.createDataFrame( Seq(
-      (0, 1.0, 4.0, 1.0, 1.0, 4.0, 4.0),
-      (1, 11.0, 12.0, 11.0, 11.0, 12.0, 12.0),
-      (2, 3.0, Double.NaN, 3.0, 3.0, 10.0, 12.0),
-      (3, Double.NaN, 14.0, 5.0, 3.0, 14.0, 14.0)
-    )).toDF("id", "value1", "value2", "expected_mean_value1", "expected_median_value1",
-      "expected_mean_value2", "expected_median_value2")
+    val df = spark.createDataFrame(Seq(
+      (0, 1.0, 4.0, 1.0, 1.0, 1.0, 4.0, 4.0, 4.0),
+      (1, 11.0, 12.0, 11.0, 11.0, 11.0, 12.0, 12.0, 12.0),
+      (2, 3.0, Double.NaN, 3.0, 3.0, 3.0, 10.0, 12.0, 4.0),
+      (3, Double.NaN, 14.0, 5.0, 3.0, 1.0, 14.0, 14.0, 14.0)
+    )).toDF("id", "value1", "value2",
+      "expected_mean_value1", "expected_median_value1", "expected_mode_value1",
+      "expected_mean_value2", "expected_median_value2", "expected_mode_value2")
     val imputer = new Imputer()
       .setInputCols(Array("value1", "value2"))
       .setOutputCols(Array("out1", "out2"))
@@ -42,23 +43,25 @@ class ImputerSuite extends MLTest with DefaultReadWriteTest {
   }
 
   test("Single Column: Imputer for Double with default missing Value NaN") {
-    val df1 = spark.createDataFrame( Seq(
-      (0, 1.0, 1.0, 1.0),
-      (1, 11.0, 11.0, 11.0),
-      (2, 3.0, 3.0, 3.0),
-      (3, Double.NaN, 5.0, 3.0)
-    )).toDF("id", "value", "expected_mean_value", "expected_median_value")
+    val df1 = spark.createDataFrame(Seq(
+      (0, 1.0, 1.0, 1.0, 1.0),
+      (1, 11.0, 11.0, 11.0, 11.0),
+      (2, 3.0, 3.0, 3.0, 3.0),
+      (3, Double.NaN, 5.0, 3.0, 1.0)
+    )).toDF("id", "value",
+      "expected_mean_value", "expected_median_value", "expected_mode_value")
     val imputer1 = new Imputer()
       .setInputCol("value")
       .setOutputCol("out")
     ImputerSuite.iterateStrategyTest(false, imputer1, df1)
 
-    val df2 = spark.createDataFrame( Seq(
-      (0, 4.0, 4.0, 4.0),
-      (1, 12.0, 12.0, 12.0),
-      (2, Double.NaN, 10.0, 12.0),
-      (3, 14.0, 14.0, 14.0)
-    )).toDF("id", "value", "expected_mean_value", "expected_median_value")
+    val df2 = spark.createDataFrame(Seq(
+      (0, 4.0, 4.0, 4.0, 4.0),
+      (1, 12.0, 12.0, 12.0, 12.0),
+      (2, Double.NaN, 10.0, 12.0, 4.0),
+      (3, 14.0, 14.0, 14.0, 14.0)
+    )).toDF("id", "value",
+      "expected_mean_value", "expected_median_value", "expected_mode_value")
     val imputer2 = new Imputer()
       .setInputCol("value")
       .setOutputCol("out")
@@ -66,12 +69,13 @@ class ImputerSuite extends MLTest with DefaultReadWriteTest {
   }
 
   test("Imputer should handle NaNs when computing surrogate value, if missingValue is not NaN") {
-    val df = spark.createDataFrame( Seq(
-      (0, 1.0, 1.0, 1.0),
-      (1, 3.0, 3.0, 3.0),
-      (2, Double.NaN, Double.NaN, Double.NaN),
-      (3, -1.0, 2.0, 1.0)
-    )).toDF("id", "value", "expected_mean_value", "expected_median_value")
+    val df = spark.createDataFrame(Seq(
+      (0, 1.0, 1.0, 1.0, 1.0),
+      (1, 3.0, 3.0, 3.0, 3.0),
+      (2, Double.NaN, Double.NaN, Double.NaN, Double.NaN),
+      (3, -1.0, 2.0, 1.0, 1.0)
+    )).toDF("id", "value",
+      "expected_mean_value", "expected_median_value", "expected_mode_value")
     val imputer = new Imputer().setInputCols(Array("value")).setOutputCols(Array("out"))
       .setMissingValue(-1.0)
     ImputerSuite.iterateStrategyTest(true, imputer, df)
@@ -79,64 +83,69 @@ class ImputerSuite extends MLTest with DefaultReadWriteTest {
 
   test("Single Column: Imputer should handle NaNs when computing surrogate value," +
     " if missingValue is not NaN") {
-    val df = spark.createDataFrame( Seq(
-      (0, 1.0, 1.0, 1.0),
-      (1, 3.0, 3.0, 3.0),
-      (2, Double.NaN, Double.NaN, Double.NaN),
-      (3, -1.0, 2.0, 1.0)
-    )).toDF("id", "value", "expected_mean_value", "expected_median_value")
+    val df = spark.createDataFrame(Seq(
+      (0, 1.0, 1.0, 1.0, 1.0),
+      (1, 3.0, 3.0, 3.0, 3.0),
+      (2, Double.NaN, Double.NaN, Double.NaN, Double.NaN),
+      (3, -1.0, 2.0, 1.0, 1.0)
+    )).toDF("id", "value",
+      "expected_mean_value", "expected_median_value", "expected_mode_value")
     val imputer = new Imputer().setInputCol("value").setOutputCol("out")
       .setMissingValue(-1.0)
     ImputerSuite.iterateStrategyTest(false, imputer, df)
   }
 
   test("Imputer for Float with missing Value -1.0") {
-    val df = spark.createDataFrame( Seq(
-      (0, 1.0F, 1.0F, 1.0F),
-      (1, 3.0F, 3.0F, 3.0F),
-      (2, 10.0F, 10.0F, 10.0F),
-      (3, 10.0F, 10.0F, 10.0F),
-      (4, -1.0F, 6.0F, 3.0F)
-    )).toDF("id", "value", "expected_mean_value", "expected_median_value")
+    val df = spark.createDataFrame(Seq(
+      (0, 1.0F, 1.0F, 1.0F, 1.0F),
+      (1, 3.0F, 3.0F, 3.0F, 3.0F),
+      (2, 10.0F, 10.0F, 10.0F, 10.0F),
+      (3, 10.0F, 10.0F, 10.0F, 10.0F),
+      (4, -1.0F, 6.0F, 3.0F, 10.0F)
+    )).toDF("id", "value",
+      "expected_mean_value", "expected_median_value", "expected_mode_value")
     val imputer = new Imputer().setInputCols(Array("value")).setOutputCols(Array("out"))
       .setMissingValue(-1)
     ImputerSuite.iterateStrategyTest(true, imputer, df)
   }
 
   test("Single Column: Imputer for Float with missing Value -1.0") {
-    val df = spark.createDataFrame( Seq(
-      (0, 1.0F, 1.0F, 1.0F),
-      (1, 3.0F, 3.0F, 3.0F),
-      (2, 10.0F, 10.0F, 10.0F),
-      (3, 10.0F, 10.0F, 10.0F),
-      (4, -1.0F, 6.0F, 3.0F)
-    )).toDF("id", "value", "expected_mean_value", "expected_median_value")
+    val df = spark.createDataFrame(Seq(
+      (0, 1.0F, 1.0F, 1.0F, 1.0F),
+      (1, 3.0F, 3.0F, 3.0F, 3.0F),
+      (2, 10.0F, 10.0F, 10.0F, 10.0F),
+      (3, 10.0F, 10.0F, 10.0F, 10.0F),
+      (4, -1.0F, 6.0F, 3.0F, 10.0F)
+    )).toDF("id", "value",
+      "expected_mean_value", "expected_median_value", "expected_mode_value")
     val imputer = new Imputer().setInputCol("value").setOutputCol("out")
       .setMissingValue(-1)
     ImputerSuite.iterateStrategyTest(false, imputer, df)
   }
 
   test("Imputer should impute null as well as 'missingValue'") {
-    val rawDf = spark.createDataFrame( Seq(
-      (0, 4.0, 4.0, 4.0),
-      (1, 10.0, 10.0, 10.0),
-      (2, 10.0, 10.0, 10.0),
-      (3, Double.NaN, 8.0, 10.0),
-      (4, -1.0, 8.0, 10.0)
-    )).toDF("id", "rawValue", "expected_mean_value", "expected_median_value")
+    val rawDf = spark.createDataFrame(Seq(
+      (0, 4.0, 4.0, 4.0, 4.0),
+      (1, 10.0, 10.0, 10.0, 10.0),
+      (2, 10.0, 10.0, 10.0, 10.0),
+      (3, Double.NaN, 8.0, 10.0, 10.0),
+      (4, -1.0, 8.0, 10.0, 10.0)
+    )).toDF("id", "rawValue",
+      "expected_mean_value", "expected_median_value", "expected_mode_value")
     val df = rawDf.selectExpr("*", "IF(rawValue=-1.0, null, rawValue) as value")
     val imputer = new Imputer().setInputCols(Array("value")).setOutputCols(Array("out"))
     ImputerSuite.iterateStrategyTest(true, imputer, df)
   }
 
   test("Single Column: Imputer should impute null as well as 'missingValue'") {
-    val rawDf = spark.createDataFrame( Seq(
-      (0, 4.0, 4.0, 4.0),
-      (1, 10.0, 10.0, 10.0),
-      (2, 10.0, 10.0, 10.0),
-      (3, Double.NaN, 8.0, 10.0),
-      (4, -1.0, 8.0, 10.0)
-    )).toDF("id", "rawValue", "expected_mean_value", "expected_median_value")
+    val rawDf = spark.createDataFrame(Seq(
+      (0, 4.0, 4.0, 4.0, 4.0),
+      (1, 10.0, 10.0, 10.0, 10.0),
+      (2, 10.0, 10.0, 10.0, 10.0),
+      (3, Double.NaN, 8.0, 10.0, 10.0),
+      (4, -1.0, 8.0, 10.0, 10.0)
+    )).toDF("id", "rawValue",
+      "expected_mean_value", "expected_median_value", "expected_mode_value")
     val df = rawDf.selectExpr("*", "IF(rawValue=-1.0, null, rawValue) as value")
     val imputer = new Imputer().setInputCol("value").setOutputCol("out")
     ImputerSuite.iterateStrategyTest(false, imputer, df)
@@ -187,7 +196,7 @@ class ImputerSuite extends MLTest with DefaultReadWriteTest {
   }
 
   test("Imputer throws exception when surrogate cannot be computed") {
-    val df = spark.createDataFrame( Seq(
+    val df = spark.createDataFrame(Seq(
       (0, Double.NaN, 1.0, 1.0),
       (1, Double.NaN, 3.0, 3.0),
       (2, Double.NaN, Double.NaN, Double.NaN)
@@ -205,12 +214,13 @@ class ImputerSuite extends MLTest with DefaultReadWriteTest {
   }
 
   test("Single Column: Imputer throws exception when surrogate cannot be computed") {
-    val df = spark.createDataFrame( Seq(
-      (0, Double.NaN, 1.0, 1.0),
-      (1, Double.NaN, 3.0, 3.0),
-      (2, Double.NaN, Double.NaN, Double.NaN)
-    )).toDF("id", "value", "expected_mean_value", "expected_median_value")
-    Seq("mean", "median").foreach { strategy =>
+    val df = spark.createDataFrame(Seq(
+      (0, Double.NaN, 1.0, 1.0, 1.0),
+      (1, Double.NaN, 3.0, 3.0, 3.0),
+      (2, Double.NaN, Double.NaN, Double.NaN, Double.NaN)
+    )).toDF("id", "value",
+      "expected_mean_value", "expected_median_value", "expected_mode_value")
+    Seq("mean", "median", "mode").foreach { strategy =>
       val imputer = new Imputer().setInputCol("value").setOutputCol("out")
         .setStrategy(strategy)
       withClue("Imputer should fail all the values are invalid") {
@@ -223,12 +233,12 @@ class ImputerSuite extends MLTest with DefaultReadWriteTest {
   }
 
   test("Imputer input & output column validation") {
-    val df = spark.createDataFrame( Seq(
+    val df = spark.createDataFrame(Seq(
       (0, 1.0, 1.0, 1.0),
       (1, Double.NaN, 3.0, 3.0),
       (2, Double.NaN, Double.NaN, Double.NaN)
     )).toDF("id", "value1", "value2", "value3")
-    Seq("mean", "median").foreach { strategy =>
+    Seq("mean", "median", "mode").foreach { strategy =>
       withClue("Imputer should fail if inputCols and outputCols are different length") {
         val e: IllegalArgumentException = intercept[IllegalArgumentException] {
           val imputer = new Imputer().setStrategy(strategy)
@@ -306,13 +316,13 @@ class ImputerSuite extends MLTest with DefaultReadWriteTest {
   }
 
   test("Imputer for IntegerType with default missing value null") {
-
-    val df = spark.createDataFrame(Seq[(Integer, Integer, Integer)](
-      (1, 1, 1),
-      (11, 11, 11),
-      (3, 3, 3),
-      (null, 5, 3)
-    )).toDF("value1", "expected_mean_value1", "expected_median_value1")
+    val df = spark.createDataFrame(Seq[(Integer, Integer, Integer, Integer)](
+      (1, 1, 1, 1),
+      (11, 11, 11, 11),
+      (3, 3, 3, 3),
+      (null, 5, 3, 1)
+    )).toDF("value1",
+      "expected_mean_value1", "expected_median_value1", "expected_mode_value1")
 
     val imputer = new Imputer()
       .setInputCols(Array("value1"))
@@ -327,12 +337,13 @@ class ImputerSuite extends MLTest with DefaultReadWriteTest {
   }
 
   test("Single Column Imputer for IntegerType with default missing value null") {
-    val df = spark.createDataFrame(Seq[(Integer, Integer, Integer)](
-      (1, 1, 1),
-      (11, 11, 11),
-      (3, 3, 3),
-      (null, 5, 3)
-    )).toDF("value", "expected_mean_value", "expected_median_value")
+    val df = spark.createDataFrame(Seq[(Integer, Integer, Integer, Integer)](
+      (1, 1, 1, 1),
+      (11, 11, 11, 11),
+      (3, 3, 3, 3),
+      (null, 5, 3, 1)
+    )).toDF("value",
+      "expected_mean_value", "expected_median_value", "expected_mode_value")
 
     val imputer = new Imputer()
       .setInputCol("value")
@@ -347,13 +358,13 @@ class ImputerSuite extends MLTest with DefaultReadWriteTest {
   }
 
   test("Imputer for IntegerType with missing value -1") {
-
-    val df = spark.createDataFrame(Seq[(Integer, Integer, Integer)](
-      (1, 1, 1),
-      (11, 11, 11),
-      (3, 3, 3),
-      (-1, 5, 3)
-    )).toDF("value1", "expected_mean_value1", "expected_median_value1")
+    val df = spark.createDataFrame(Seq[(Integer, Integer, Integer, Integer)](
+      (1, 1, 1, 1),
+      (11, 11, 11, 11),
+      (3, 3, 3, 3),
+      (-1, 5, 3, 1)
+    )).toDF("value1",
+      "expected_mean_value1", "expected_median_value1", "expected_mode_value1")
 
     val imputer = new Imputer()
       .setInputCols(Array("value1"))
@@ -369,12 +380,13 @@ class ImputerSuite extends MLTest with DefaultReadWriteTest {
   }
 
   test("Single Column: Imputer for IntegerType with missing value -1") {
-    val df = spark.createDataFrame(Seq[(Integer, Integer, Integer)](
-      (1, 1, 1),
-      (11, 11, 11),
-      (3, 3, 3),
-      (-1, 5, 3)
-    )).toDF("value", "expected_mean_value", "expected_median_value")
+    val df = spark.createDataFrame(Seq[(Integer, Integer, Integer, Integer)](
+      (1, 1, 1, 1),
+      (11, 11, 11, 11),
+      (3, 3, 3, 3),
+      (-1, 5, 3, 1)
+    )).toDF("value",
+      "expected_mean_value", "expected_median_value", "expected_mode_value")
 
     val imputer = new Imputer()
       .setInputCol("value")
@@ -402,13 +414,13 @@ class ImputerSuite extends MLTest with DefaultReadWriteTest {
   }
 
   test("Compare single/multiple column(s) Imputer in pipeline") {
-    val df = spark.createDataFrame( Seq(
+    val df = spark.createDataFrame(Seq(
       (0, 1.0, 4.0),
       (1, 11.0, 12.0),
       (2, 3.0, Double.NaN),
       (3, Double.NaN, 14.0)
     )).toDF("id", "value1", "value2")
-    Seq("mean", "median").foreach { strategy =>
+    Seq("mean", "median", "mode").foreach { strategy =>
       val multiColsImputer = new Imputer()
         .setInputCols(Array("value1", "value2"))
         .setOutputCols(Array("result1", "result2"))
@@ -450,11 +462,12 @@ class ImputerSuite extends MLTest with DefaultReadWriteTest {
 object ImputerSuite {
 
   /**
-   * Imputation strategy. Available options are ["mean", "median"].
-   * @param df DataFrame with columns "id", "value", "expected_mean", "expected_median"
+   * Imputation strategy. Available options are ["mean", "median", "mode"].
+   * @param df DataFrame with columns "id", "value", "expected_mean", "expected_median",
+   *           "expected_mode".
    */
   def iterateStrategyTest(isMultiCol: Boolean, imputer: Imputer, df: DataFrame): Unit = {
-    Seq("mean", "median").foreach { strategy =>
+    Seq("mean", "median", "mode").foreach { strategy =>
       imputer.setStrategy(strategy)
       val model = imputer.fit(df)
       val resultDF = model.transform(df)
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/InstanceSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/InstanceSuite.scala
index d780bdf5f5dc8..f1e071357bab7 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/InstanceSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/InstanceSuite.scala
@@ -74,4 +74,58 @@ class InstanceSuite extends SparkFunSuite{
     }
   }
 
+  test("InstanceBlock: blokify with max memory usage") {
+    val instance1 = Instance(19.0, 2.0, Vectors.dense(1.0, 7.0))
+    val instance2 = Instance(17.0, 1.0, Vectors.dense(0.0, 5.0).toSparse)
+    val instances = Seq(instance1, instance2)
+
+    val blocks = InstanceBlock
+      .blokifyWithMaxMemUsage(Iterator.apply(instance1, instance2), 128).toArray
+    require(blocks.length == 1)
+    val block = blocks.head
+    assert(block.size === 2)
+    assert(block.numFeatures === 2)
+    block.instanceIterator.zipWithIndex.foreach {
+      case (instance, i) =>
+        assert(instance.label === instances(i).label)
+        assert(instance.weight === instances(i).weight)
+        assert(instance.features.toArray === instances(i).features.toArray)
+    }
+    Seq(0, 1).foreach { i =>
+      val nzIter = block.getNonZeroIter(i)
+      val vec = Vectors.sparse(2, nzIter.toSeq)
+      assert(vec.toArray === instances(i).features.toArray)
+    }
+
+    // instances larger than maxMemUsage
+    val denseInstance = Instance(-1.0, 2.0, Vectors.dense(Array.fill(1000)(1.0)))
+    InstanceBlock.blokifyWithMaxMemUsage(Iterator.single(denseInstance), 64).size
+    InstanceBlock.blokifyWithMaxMemUsage(Iterator.fill(10)(denseInstance), 64).size
+
+    // different numFeatures
+    intercept[IllegalArgumentException] {
+      InstanceBlock.blokifyWithMaxMemUsage(Iterator.apply(instance1, denseInstance), 64).size
+    }
+
+    // nnz = 10
+    val sparseInstance = Instance(-2.0, 3.0,
+      Vectors.sparse(1000, Array.range(0, 1000, 100), Array.fill(10)(0.1)))
+
+    // normally, memory usage of a block does not exceed maxMemUsage too much
+    val maxMemUsage = 1 << 18
+    val mixedIter = Iterator.fill(100)(denseInstance) ++
+      Iterator.fill(1000)(sparseInstance) ++
+      Iterator.fill(10)(denseInstance) ++
+      Iterator.fill(10)(sparseInstance) ++
+      Iterator.fill(100)(denseInstance) ++
+      Iterator.fill(100)(sparseInstance)
+    InstanceBlock.blokifyWithMaxMemUsage(mixedIter, maxMemUsage)
+      .foreach { block =>
+        val doubleBytes = java.lang.Double.BYTES
+        val arrayHeader = 12L
+        val blockMemUsage = block.matrix.getSizeInBytes +
+          (block.labels.length + block.weights.length) * doubleBytes + arrayHeader * 2
+        require(blockMemUsage < maxMemUsage * 1.05)
+      }
+  }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/LSHTest.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/LSHTest.scala
index 1d052fbebd92d..2815adb75adf3 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/LSHTest.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/LSHTest.scala
@@ -27,7 +27,7 @@ import org.apache.spark.sql.types.DataTypes
 
 private[ml] object LSHTest {
   /**
-   * For any locality sensitive function h in a metric space, we meed to verify whether
+   * For any locality sensitive function h in a metric space, we need to verify whether
    * the following property is satisfied.
    *
    * There exist dist1, dist2, p1, p2, so that for any two elements e1 and e2,
@@ -71,13 +71,14 @@ private[ml] object LSHTest {
       transformedData.schema, model.getOutputCol, DataTypes.createArrayType(new VectorUDT))
 
     // Check output column dimensions
-    val headHashValue = transformedData.select(outputCol).head().get(0).asInstanceOf[Seq[Vector]]
+    val headHashValue =
+      transformedData.select(outputCol).head().get(0).asInstanceOf[scala.collection.Seq[Vector]]
     assert(headHashValue.length == model.getNumHashTables)
 
     // Perform a cross join and label each pair of same_bucket and distance
     val pairs = transformedData.as("a").crossJoin(transformedData.as("b"))
     val distUDF = udf((x: Vector, y: Vector) => model.keyDistance(x, y))
-    val sameBucket = udf((x: Seq[Vector], y: Seq[Vector]) => model.hashDistance(x, y) == 0.0)
+    val sameBucket = udf((x: Array[Vector], y: Array[Vector]) => model.hashDistance(x, y) == 0.0)
     val result = pairs
       .withColumn("same_bucket", sameBucket(col(s"a.$outputCol"), col(s"b.$outputCol")))
       .withColumn("distance", distUDF(col(s"a.$inputCol"), col(s"b.$inputCol")))
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/MinHashLSHSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/MinHashLSHSuite.scala
index 1c2956cb82908..c99e0fa3f8623 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/MinHashLSHSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/MinHashLSHSuite.scala
@@ -182,7 +182,7 @@ class MinHashLSHSuite extends MLTest with DefaultReadWriteTest {
     val model = new MinHashLSHModel("mh", randCoefficients = Array((1, 0)))
     model.set(model.inputCol, "keys")
     testTransformer[Tuple1[Vector]](dataset.toDF(), model, "keys", model.getOutputCol) {
-      case Row(_: Vector, output: Seq[_]) =>
+      case Row(_: Vector, output: scala.collection.Seq[_]) =>
         assert(output.length === model.randCoefficients.length)
         // no AND-amplification yet: SPARK-18450, so each hash output is of length 1
         output.foreach {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/NGramSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/NGramSuite.scala
index 1483d5df4d224..bf276ceed2097 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/NGramSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/NGramSuite.scala
@@ -83,7 +83,7 @@ class NGramSuite extends MLTest with DefaultReadWriteTest {
 
   def testNGram(t: NGram, dataFrame: DataFrame): Unit = {
     testTransformer[(Seq[String], Seq[String])](dataFrame, t, "nGrams", "wantedNGrams") {
-      case Row(actualNGrams : Seq[_], wantedNGrams: Seq[_]) =>
+      case Row(actualNGrams : scala.collection.Seq[_], wantedNGrams: scala.collection.Seq[_]) =>
         assert(actualNGrams === wantedNGrams)
     }
   }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/NormalizerSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/NormalizerSuite.scala
index d97df0050d74e..1c602cd7d9a4f 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/NormalizerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/NormalizerSuite.scala
@@ -29,14 +29,14 @@ class NormalizerSuite extends MLTest with DefaultReadWriteTest {
 
   import testImplicits._
 
-  @transient var data: Array[Vector] = _
-  @transient var l1Normalized: Array[Vector] = _
-  @transient var l2Normalized: Array[Vector] = _
+  @transient var data: Seq[Vector] = _
+  @transient var l1Normalized: Seq[Vector] = _
+  @transient var l2Normalized: Seq[Vector] = _
 
   override def beforeAll(): Unit = {
     super.beforeAll()
 
-    data = Array(
+    data = Seq(
       Vectors.sparse(3, Seq((0, -2.0), (1, 2.3))),
       Vectors.dense(0.0, 0.0, 0.0),
       Vectors.dense(0.6, -1.1, -3.0),
@@ -44,7 +44,7 @@ class NormalizerSuite extends MLTest with DefaultReadWriteTest {
       Vectors.sparse(3, Seq((0, 5.7), (1, 0.72), (2, 2.7))),
       Vectors.sparse(3, Seq())
     )
-    l1Normalized = Array(
+    l1Normalized = Seq(
       Vectors.sparse(3, Seq((0, -0.465116279), (1, 0.53488372))),
       Vectors.dense(0.0, 0.0, 0.0),
       Vectors.dense(0.12765957, -0.23404255, -0.63829787),
@@ -52,7 +52,7 @@ class NormalizerSuite extends MLTest with DefaultReadWriteTest {
       Vectors.dense(0.625, 0.07894737, 0.29605263),
       Vectors.sparse(3, Seq())
     )
-    l2Normalized = Array(
+    l2Normalized = Seq(
       Vectors.sparse(3, Seq((0, -0.65617871), (1, 0.75460552))),
       Vectors.dense(0.0, 0.0, 0.0),
       Vectors.dense(0.184549876, -0.3383414, -0.922749378),
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/StopWordsRemoverSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/StopWordsRemoverSuite.scala
index dc6fb31a1f8e4..eaf91769a08dd 100755
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/StopWordsRemoverSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/StopWordsRemoverSuite.scala
@@ -29,7 +29,7 @@ class StopWordsRemoverSuite extends MLTest with DefaultReadWriteTest {
 
   def testStopWordsRemover(t: StopWordsRemover, dataFrame: DataFrame): Unit = {
     testTransformer[(Array[String], Array[String])](dataFrame, t, "filtered", "expected") {
-       case Row(tokens: Seq[_], wantedTokens: Seq[_]) =>
+       case Row(tokens: scala.collection.Seq[_], wantedTokens: scala.collection.Seq[_]) =>
          assert(tokens === wantedTokens)
     }
   }
@@ -242,7 +242,8 @@ class StopWordsRemoverSuite extends MLTest with DefaultReadWriteTest {
     remover.transform(df)
       .select("filtered1", "expected1", "filtered2", "expected2")
       .collect().foreach {
-        case Row(r1: Seq[_], e1: Seq[_], r2: Seq[_], e2: Seq[_]) =>
+        case Row(r1: scala.collection.Seq[_], e1: scala.collection.Seq[_],
+          r2: scala.collection.Seq[_], e2: scala.collection.Seq[_]) =>
           assert(r1 === e1,
             s"The result value is not correct after bucketing. Expected $e1 but found $r1")
           assert(r2 === e2,
@@ -268,7 +269,8 @@ class StopWordsRemoverSuite extends MLTest with DefaultReadWriteTest {
     remover.transform(df)
       .select("filtered1", "expected1", "filtered2", "expected2")
       .collect().foreach {
-        case Row(r1: Seq[_], e1: Seq[_], r2: Seq[_], e2: Seq[_]) =>
+        case Row(r1: scala.collection.Seq[_], e1: scala.collection.Seq[_],
+          r2: scala.collection.Seq[_], e2: scala.collection.Seq[_]) =>
           assert(r1 === e1,
             s"The result value is not correct after bucketing. Expected $e1 but found $r1")
           assert(r2 === e2,
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala
index 948140897d8cc..c8247b9c8f3bf 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala
@@ -483,6 +483,6 @@ class StringIndexerSuite extends MLTest with DefaultReadWriteTest {
 
     val metadata = spark.read.json(s"$modelPath/metadata")
     val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
-    assert(sparkVersionStr == "2.4.4")
+    assert(sparkVersionStr === "2.4.4")
   }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/UnivariateFeatureSelectorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/UnivariateFeatureSelectorSuite.scala
new file mode 100644
index 0000000000000..84868dcfb795b
--- /dev/null
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/UnivariateFeatureSelectorSuite.scala
@@ -0,0 +1,685 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.feature
+
+import org.apache.spark.ml.linalg.{Vector, Vectors}
+import org.apache.spark.ml.param.ParamsSuite
+import org.apache.spark.ml.stat.{ANOVATest, FValueTest}
+import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTest, MLTestingUtils}
+import org.apache.spark.ml.util.TestingUtils._
+import org.apache.spark.sql.{Dataset, Row}
+
+class UnivariateFeatureSelectorSuite extends MLTest with DefaultReadWriteTest {
+
+  import testImplicits._
+
+  @transient var datasetChi2: Dataset[_] = _
+  @transient var datasetAnova: Dataset[_] = _
+  @transient var datasetFRegression: Dataset[_] = _
+
+  private var selector1: UnivariateFeatureSelector = _
+  private var selector2: UnivariateFeatureSelector = _
+  private var selector3: UnivariateFeatureSelector = _
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    // Toy dataset, including the top feature for a chi-squared test.
+    // These data are chosen such that each feature's test has a distinct p-value.
+    /*
+     *  Contingency tables
+     *  feature1 = {6.0, 0.0, 8.0}
+     *  class  0 1 2
+     *    6.0||1|0|0|
+     *    0.0||0|3|0|
+     *    8.0||0|0|2|
+     *  degree of freedom = 4, statistic = 12, pValue = 0.017
+     *
+     *  feature2 = {7.0, 9.0}
+     *  class  0 1 2
+     *    7.0||1|0|0|
+     *    9.0||0|3|2|
+     *  degree of freedom = 2, statistic = 6, pValue = 0.049
+     *
+     *  feature3 = {0.0, 6.0, 3.0, 8.0}
+     *  class  0 1 2
+     *    0.0||1|0|0|
+     *    6.0||0|1|2|
+     *    3.0||0|1|0|
+     *    8.0||0|1|0|
+     *  degree of freedom = 6, statistic = 8.66, pValue = 0.193
+     *
+     *  feature4 = {7.0, 0.0, 5.0, 4.0}
+     *  class  0 1 2
+     *    7.0||1|0|0|
+     *    0.0||0|2|0|
+     *    5.0||0|1|1|
+     *    4.0||0|0|1|
+     *  degree of freedom = 6, statistic = 9.5, pValue = 0.147
+     *
+     *  feature5 = {6.0, 5.0, 4.0, 0.0}
+     *  class  0 1 2
+     *    6.0||1|1|0|
+     *    5.0||0|2|0|
+     *    4.0||0|0|1|
+     *    0.0||0|0|1|
+     *  degree of freedom = 6, statistic = 8.0, pValue = 0.238
+     *
+     *  feature6 = {0.0, 9.0, 5.0, 4.0}
+     *  class  0 1 2
+     *    0.0||1|0|1|
+     *    9.0||0|1|0|
+     *    5.0||0|1|0|
+     *    4.0||0|1|1|
+     *  degree of freedom = 6, statistic = 5, pValue = 0.54
+     *
+     *  To verify the results with R, run:
+     *  library(stats)
+     *  x1 <- c(6.0, 0.0, 0.0, 0.0, 8.0, 8.0)
+     *  x2 <- c(7.0, 9.0, 9.0, 9.0, 9.0, 9.0)
+     *  x3 <- c(0.0, 6.0, 3.0, 8.0, 6.0, 6.0)
+     *  x4 <- c(7.0, 0.0, 0.0, 5.0, 5.0, 4.0)
+     *  x5 <- c(6.0, 5.0, 5.0, 6.0, 4.0, 0.0)
+     *  x6 <- c(0.0, 9.0, 5.0, 4.0, 4.0, 0.0)
+     *  y <- c(0.0, 1.0, 1.0, 1.0, 2.0, 2.0)
+     *  chisq.test(x1,y)
+     *  chisq.test(x2,y)
+     *  chisq.test(x3,y)
+     *  chisq.test(x4,y)
+     *  chisq.test(x5,y)
+     *  chisq.test(x6,y)
+     */
+
+    datasetChi2 = spark.createDataFrame(Seq(
+      (0.0, Vectors.sparse(6, Array((0, 6.0), (1, 7.0), (3, 7.0), (4, 6.0))), Vectors.dense(6.0)),
+      (1.0, Vectors.sparse(6, Array((1, 9.0), (2, 6.0), (4, 5.0), (5, 9.0))), Vectors.dense(0.0)),
+      (1.0, Vectors.sparse(6, Array((1, 9.0), (2, 3.0), (4, 5.0), (5, 5.0))), Vectors.dense(0.0)),
+      (1.0, Vectors.dense(Array(0.0, 9.0, 8.0, 5.0, 6.0, 4.0)), Vectors.dense(0.0)),
+      (2.0, Vectors.dense(Array(8.0, 9.0, 6.0, 5.0, 4.0, 4.0)), Vectors.dense(8.0)),
+      (2.0, Vectors.dense(Array(8.0, 9.0, 6.0, 4.0, 0.0, 0.0)), Vectors.dense(8.0))
+    )).toDF("label", "features", "topFeature")
+
+    // scalastyle:off
+    /*
+      X:
+      array([[4.65415496e-03, 1.03550567e-01, -1.17358140e+00,
+      1.61408773e-01,  3.92492111e-01,  7.31240882e-01],
+      [-9.01651741e-01, -5.28905302e-01,  1.27636785e+00,
+      7.02154563e-01,  6.21348351e-01,  1.88397353e-01],
+      [ 3.85692159e-01, -9.04639637e-01,  5.09782604e-02,
+      8.40043971e-01,  7.45977857e-01,  8.78402288e-01],
+      [ 1.36264353e+00,  2.62454094e-01,  7.96306202e-01,
+      6.14948000e-01,  7.44948187e-01,  9.74034830e-01],
+      [ 9.65874070e-01,  2.52773665e+00, -2.19380094e+00,
+      2.33408080e-01,  1.86340919e-01,  8.23390433e-01],
+      [ 1.12324305e+01, -2.77121515e-01,  1.12740513e-01,
+      2.35184013e-01,  3.46668895e-01,  9.38500782e-02],
+      [ 1.06195839e+01, -1.82891238e+00,  2.25085601e-01,
+      9.09979851e-01,  6.80257535e-02,  8.24017480e-01],
+      [ 1.12806837e+01,  1.30686889e+00,  9.32839108e-02,
+      3.49784755e-01,  1.71322408e-02,  7.48465194e-02],
+      [ 9.98689462e+00,  9.50808938e-01, -2.90786359e-01,
+      2.31253009e-01,  7.46270968e-01,  1.60308169e-01],
+      [ 1.08428551e+01, -1.02749936e+00,  1.73951508e-01,
+      8.92482744e-02,  1.42651730e-01,  7.66751625e-01],
+      [-1.98641448e+00,  1.12811990e+01, -2.35246756e-01,
+      8.22809049e-01,  3.26739456e-01,  7.88268404e-01],
+      [-6.09864090e-01,  1.07346276e+01, -2.18805509e-01,
+      7.33931213e-01,  1.42554396e-01,  7.11225605e-01],
+      [-1.58481268e+00,  9.19364039e+00, -5.87490459e-02,
+      2.51532056e-01,  2.82729807e-01,  7.16245686e-01],
+      [-2.50949277e-01,  1.12815254e+01, -6.94806734e-01,
+      5.93898886e-01,  5.68425656e-01,  8.49762330e-01],
+      [ 7.63485129e-01,  1.02605138e+01,  1.32617719e+00,
+      5.49682879e-01,  8.59931442e-01,  4.88677978e-02],
+      [ 9.34900015e-01,  4.11379043e-01,  8.65010205e+00,
+      9.23509168e-01,  1.16995043e-01,  5.91894106e-03],
+      [ 4.73734933e-01, -1.48321181e+00,  9.73349621e+00,
+      4.09421563e-01,  5.09375719e-01,  5.93157850e-01],
+      [ 3.41470679e-01, -6.88972582e-01,  9.60347938e+00,
+      3.62654055e-01,  2.43437468e-01,  7.13052838e-01],
+      [-5.29614251e-01, -1.39262856e+00,  1.01354144e+01,
+      8.24123861e-01,  5.84074506e-01,  6.54461558e-01],
+      [-2.99454508e-01,  2.20457263e+00,  1.14586015e+01,
+      5.16336729e-01,  9.99776159e-01,  3.15769738e-01]])
+      y:
+      array([1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4])
+      scikit-learn result:
+      >>> f_classif(X, y)
+      (array([228.27701422,  84.33070501, 134.25330675,   0.82211775, 0.82991363,   1.08478943]),
+       array([2.43864448e-13, 5.09088367e-10, 1.49033067e-11, 5.00596446e-01, 4.96684374e-01, 3.83798191e-01]))
+    */
+    // scalastyle:on
+
+    val dataAnova = Seq(
+      (1, Vectors.dense(4.65415496e-03, 1.03550567e-01, -1.17358140e+00,
+      1.61408773e-01, 3.92492111e-01, 7.31240882e-01), Vectors.dense(4.65415496e-03)),
+      (1, Vectors.dense(-9.01651741e-01, -5.28905302e-01, 1.27636785e+00,
+      7.02154563e-01, 6.21348351e-01, 1.88397353e-01), Vectors.dense(-9.01651741e-01)),
+      (1, Vectors.dense(3.85692159e-01, -9.04639637e-01, 5.09782604e-02,
+      8.40043971e-01, 7.45977857e-01, 8.78402288e-01), Vectors.dense(3.85692159e-01)),
+      (1, Vectors.dense(1.36264353e+00, 2.62454094e-01, 7.96306202e-01,
+      6.14948000e-01, 7.44948187e-01, 9.74034830e-01), Vectors.dense(1.36264353e+00)),
+      (1, Vectors.dense(9.65874070e-01, 2.52773665e+00, -2.19380094e+00,
+        2.33408080e-01, 1.86340919e-01, 8.23390433e-01), Vectors.dense(9.65874070e-01)),
+      (2, Vectors.dense(1.12324305e+01, -2.77121515e-01, 1.12740513e-01,
+        2.35184013e-01, 3.46668895e-01, 9.38500782e-02), Vectors.dense(1.12324305e+01)),
+      (2, Vectors.dense(1.06195839e+01, -1.82891238e+00, 2.25085601e-01,
+        9.09979851e-01, 6.80257535e-02, 8.24017480e-01), Vectors.dense(1.06195839e+01)),
+      (2, Vectors.dense(1.12806837e+01, 1.30686889e+00, 9.32839108e-02,
+        3.49784755e-01, 1.71322408e-02, 7.48465194e-02), Vectors.dense(1.12806837e+01)),
+      (2, Vectors.dense(9.98689462e+00, 9.50808938e-01, -2.90786359e-01,
+        2.31253009e-01, 7.46270968e-01, 1.60308169e-01), Vectors.dense(9.98689462e+00)),
+      (2, Vectors.dense(1.08428551e+01, -1.02749936e+00, 1.73951508e-01,
+        8.92482744e-02, 1.42651730e-01, 7.66751625e-01), Vectors.dense(1.08428551e+01)),
+      (3, Vectors.dense(-1.98641448e+00, 1.12811990e+01, -2.35246756e-01,
+        8.22809049e-01, 3.26739456e-01, 7.88268404e-01), Vectors.dense(-1.98641448e+00)),
+      (3, Vectors.dense(-6.09864090e-01, 1.07346276e+01, -2.18805509e-01,
+        7.33931213e-01, 1.42554396e-01, 7.11225605e-01), Vectors.dense(-6.09864090e-01)),
+      (3, Vectors.dense(-1.58481268e+00, 9.19364039e+00, -5.87490459e-02,
+        2.51532056e-01, 2.82729807e-01, 7.16245686e-01), Vectors.dense(-1.58481268e+00)),
+      (3, Vectors.dense(-2.50949277e-01, 1.12815254e+01, -6.94806734e-01,
+        5.93898886e-01, 5.68425656e-01, 8.49762330e-01), Vectors.dense(-2.50949277e-01)),
+      (3, Vectors.dense(7.63485129e-01, 1.02605138e+01, 1.32617719e+00,
+        5.49682879e-01, 8.59931442e-01, 4.88677978e-02), Vectors.dense(7.63485129e-01)),
+      (4, Vectors.dense(9.34900015e-01, 4.11379043e-01, 8.65010205e+00,
+        9.23509168e-01, 1.16995043e-01, 5.91894106e-03), Vectors.dense(9.34900015e-01)),
+      (4, Vectors.dense(4.73734933e-01, -1.48321181e+00, 9.73349621e+00,
+        4.09421563e-01, 5.09375719e-01, 5.93157850e-01), Vectors.dense(4.73734933e-01)),
+      (4, Vectors.dense(3.41470679e-01, -6.88972582e-01, 9.60347938e+00,
+        3.62654055e-01, 2.43437468e-01, 7.13052838e-01), Vectors.dense(3.41470679e-01)),
+      (4, Vectors.dense(-5.29614251e-01, -1.39262856e+00, 1.01354144e+01,
+        8.24123861e-01, 5.84074506e-01, 6.54461558e-01), Vectors.dense(-5.29614251e-01)),
+      (4, Vectors.dense(-2.99454508e-01, 2.20457263e+00, 1.14586015e+01,
+        5.16336729e-01, 9.99776159e-01, 3.15769738e-01), Vectors.dense(-2.99454508e-01)))
+
+    datasetAnova = spark.createDataFrame(dataAnova).toDF("label", "features", "topFeature")
+
+    // scalastyle:off
+    /*
+    Use the following sklearn data in this test
+
+    >>> from sklearn.feature_selection import f_regression
+    >>> import numpy as np
+    >>> np.random.seed(777)
+    >>> X = np.random.rand(20, 6)
+    >>> w = np.array([0.3, 0.4, 0.5, 0, 0, 0])
+    >>> y = X @ w
+    >>> X
+    array([[0.19151945, 0.62210877, 0.43772774, 0.78535858, 0.77997581,
+            0.27259261],
+           [0.27646426, 0.80187218, 0.95813935, 0.87593263, 0.35781727,
+            0.50099513],
+           [0.68346294, 0.71270203, 0.37025075, 0.56119619, 0.50308317,
+            0.01376845],
+           [0.77282662, 0.88264119, 0.36488598, 0.61539618, 0.07538124,
+            0.36882401],
+           [0.9331401 , 0.65137814, 0.39720258, 0.78873014, 0.31683612,
+            0.56809865],
+           [0.86912739, 0.43617342, 0.80214764, 0.14376682, 0.70426097,
+            0.70458131],
+           [0.21879211, 0.92486763, 0.44214076, 0.90931596, 0.05980922,
+            0.18428708],
+           [0.04735528, 0.67488094, 0.59462478, 0.53331016, 0.04332406,
+            0.56143308],
+           [0.32966845, 0.50296683, 0.11189432, 0.60719371, 0.56594464,
+            0.00676406],
+           [0.61744171, 0.91212289, 0.79052413, 0.99208147, 0.95880176,
+            0.79196414],
+           [0.28525096, 0.62491671, 0.4780938 , 0.19567518, 0.38231745,
+            0.05387369],
+           [0.45164841, 0.98200474, 0.1239427 , 0.1193809 , 0.73852306,
+            0.58730363],
+           [0.47163253, 0.10712682, 0.22921857, 0.89996519, 0.41675354,
+            0.53585166],
+           [0.00620852, 0.30064171, 0.43689317, 0.612149  , 0.91819808,
+            0.62573667],
+           [0.70599757, 0.14983372, 0.74606341, 0.83100699, 0.63372577,
+            0.43830988],
+           [0.15257277, 0.56840962, 0.52822428, 0.95142876, 0.48035918,
+            0.50255956],
+           [0.53687819, 0.81920207, 0.05711564, 0.66942174, 0.76711663,
+             0.70811536],
+           [0.79686718, 0.55776083, 0.96583653, 0.1471569 , 0.029647  ,
+            0.59389349],
+           [0.1140657 , 0.95080985, 0.32570741, 0.19361869, 0.45781165,
+            0.92040257],
+           [0.87906916, 0.25261576, 0.34800879, 0.18258873, 0.90179605,
+            0.70652816]])
+    >>> y
+    array([0.52516321, 0.88275782, 0.67524507, 0.76734745, 0.73909458,
+           0.83628141, 0.65665506, 0.58147135, 0.35603443, 0.94534373,
+           0.57458887, 0.59026777, 0.29894977, 0.34056582, 0.64476446,
+           0.53724782, 0.5173021 , 0.94508275, 0.57739736, 0.53877145])
+    >>> f_regression(X, y)
+    (array([5.58025504,  3.98311705, 20.59605518,  0.07993376,  1.25127646,
+            0.7676937 ]),
+    array([2.96302196e-02, 6.13173918e-02, 2.54580618e-04, 7.80612726e-01,
+    2.78015517e-01, 3.92474567e-01]))
+    */
+    // scalastyle:on
+
+    val dataFRegression = Seq(
+      (0.52516321, Vectors.dense(0.19151945, 0.62210877, 0.43772774, 0.78535858, 0.77997581,
+        0.27259261), Vectors.dense(0.43772774)),
+      (0.88275782, Vectors.dense(0.27646426, 0.80187218, 0.95813935, 0.87593263, 0.35781727,
+        0.50099513), Vectors.dense(0.95813935)),
+      (0.67524507, Vectors.dense(0.68346294, 0.71270203, 0.37025075, 0.56119619, 0.50308317,
+        0.01376845), Vectors.dense(0.37025075)),
+      (0.76734745, Vectors.dense(0.77282662, 0.88264119, 0.36488598, 0.61539618, 0.07538124,
+        0.36882401), Vectors.dense(0.36488598)),
+      (0.73909458, Vectors.dense(0.9331401, 0.65137814, 0.39720258, 0.78873014, 0.31683612,
+        0.56809865), Vectors.dense(0.39720258)),
+
+      (0.83628141, Vectors.dense(0.86912739, 0.43617342, 0.80214764, 0.14376682, 0.70426097,
+        0.70458131), Vectors.dense(0.80214764)),
+      (0.65665506, Vectors.dense(0.21879211, 0.92486763, 0.44214076, 0.90931596, 0.05980922,
+        0.18428708), Vectors.dense(0.44214076)),
+      (0.58147135, Vectors.dense(0.04735528, 0.67488094, 0.59462478, 0.53331016, 0.04332406,
+        0.56143308), Vectors.dense(0.59462478)),
+      (0.35603443, Vectors.dense(0.32966845, 0.50296683, 0.11189432, 0.60719371, 0.56594464,
+        0.00676406), Vectors.dense(0.11189432)),
+      (0.94534373, Vectors.dense(0.61744171, 0.91212289, 0.79052413, 0.99208147, 0.95880176,
+        0.79196414), Vectors.dense(0.79052413)),
+
+      (0.57458887, Vectors.dense(0.28525096, 0.62491671, 0.4780938, 0.19567518, 0.38231745,
+        0.05387369), Vectors.dense(0.4780938)),
+      (0.59026777, Vectors.dense(0.45164841, 0.98200474, 0.1239427, 0.1193809, 0.73852306,
+        0.58730363), Vectors.dense(0.1239427)),
+      (0.29894977, Vectors.dense(0.47163253, 0.10712682, 0.22921857, 0.89996519, 0.41675354,
+        0.53585166), Vectors.dense(0.22921857)),
+      (0.34056582, Vectors.dense(0.00620852, 0.30064171, 0.43689317, 0.612149, 0.91819808,
+        0.62573667), Vectors.dense(0.43689317)),
+      (0.64476446, Vectors.dense(0.70599757, 0.14983372, 0.74606341, 0.83100699, 0.63372577,
+        0.43830988), Vectors.dense(0.74606341)),
+
+      (0.53724782, Vectors.dense(0.15257277, 0.56840962, 0.52822428, 0.95142876, 0.48035918,
+        0.50255956), Vectors.dense(0.52822428)),
+      (0.5173021, Vectors.dense(0.53687819, 0.81920207, 0.05711564, 0.66942174, 0.76711663,
+        0.70811536), Vectors.dense(0.05711564)),
+      (0.94508275, Vectors.dense(0.79686718, 0.55776083, 0.96583653, 0.1471569, 0.029647,
+        0.59389349), Vectors.dense(0.96583653)),
+      (0.57739736, Vectors.dense(0.1140657, 0.95080985, 0.96583653, 0.19361869, 0.45781165,
+        0.92040257), Vectors.dense(0.96583653)),
+      (0.53877145, Vectors.dense(0.87906916, 0.25261576, 0.34800879, 0.18258873, 0.90179605,
+        0.70652816), Vectors.dense(0.34800879)))
+
+    datasetFRegression = spark.createDataFrame(dataFRegression)
+      .toDF("label", "features", "topFeature")
+
+    selector1 = new UnivariateFeatureSelector()
+      .setOutputCol("filtered")
+      .setFeatureType("continuous")
+      .setLabelType("categorical")
+    selector2 = new UnivariateFeatureSelector()
+      .setOutputCol("filtered")
+      .setFeatureType("continuous")
+      .setLabelType("continuous")
+    selector3 = new UnivariateFeatureSelector()
+      .setOutputCol("filtered")
+      .setFeatureType("categorical")
+      .setLabelType("categorical")
+  }
+
+  test("params") {
+    ParamsSuite.checkParams(new UnivariateFeatureSelector())
+  }
+
+  test("Test numTopFeatures") {
+    val testParams: Seq[(UnivariateFeatureSelector, Dataset[_])] = Seq(
+      (selector1.setSelectionMode("numTopFeatures").setSelectionThreshold(1), datasetAnova),
+      (selector2.setSelectionMode("numTopFeatures").setSelectionThreshold(1), datasetFRegression),
+      (selector3.setSelectionMode("numTopFeatures").setSelectionThreshold(1), datasetChi2)
+    )
+    for ((sel, dataset) <- testParams) {
+      val model = testSelector(sel, dataset)
+      MLTestingUtils.checkCopyAndUids(sel, model)
+    }
+  }
+
+  test("Test percentile") {
+    val testParams: Seq[(UnivariateFeatureSelector, Dataset[_])] = Seq(
+      (selector1.setSelectionMode("percentile").setSelectionThreshold(0.17), datasetAnova),
+      (selector2.setSelectionMode("percentile").setSelectionThreshold(0.17), datasetFRegression),
+      (selector3.setSelectionMode("percentile").setSelectionThreshold(0.17), datasetChi2)
+    )
+    for ((sel, dataset) <- testParams) {
+      val model = testSelector(sel, dataset)
+      MLTestingUtils.checkCopyAndUids(sel, model)
+    }
+  }
+
+  test("Test fpr") {
+    val testParams: Seq[(UnivariateFeatureSelector, Dataset[_])] = Seq(
+      (selector1.setSelectionMode("fpr").setSelectionThreshold(1.0E-12), datasetAnova),
+      (selector2.setSelectionMode("fpr").setSelectionThreshold(0.01), datasetFRegression),
+      (selector3.setSelectionMode("fpr").setSelectionThreshold(0.02), datasetChi2)
+    )
+    for ((sel, dataset) <- testParams) {
+      val model = testSelector(sel, dataset)
+      MLTestingUtils.checkCopyAndUids(sel, model)
+    }
+  }
+
+  test("Test fdr") {
+    val testParams: Seq[(UnivariateFeatureSelector, Dataset[_])] = Seq(
+      (selector1.setSelectionMode("fdr").setSelectionThreshold(6.0E-12), datasetAnova),
+      (selector2.setSelectionMode("fdr").setSelectionThreshold(0.03), datasetFRegression),
+      (selector3.setSelectionMode("fdr").setSelectionThreshold(0.12), datasetChi2)
+    )
+    for ((sel, dataset) <- testParams) {
+      val model = testSelector(sel, dataset)
+      MLTestingUtils.checkCopyAndUids(sel, model)
+    }
+  }
+
+  test("Test fwe") {
+    val testParams: Seq[(UnivariateFeatureSelector, Dataset[_])] = Seq(
+      (selector1.setSelectionMode("fwe").setSelectionThreshold(6.0E-12), datasetAnova),
+      (selector2.setSelectionMode("fwe").setSelectionThreshold(0.03), datasetFRegression),
+      (selector3.setSelectionMode("fwe").setSelectionThreshold(0.12), datasetChi2)
+    )
+    for ((sel, dataset) <- testParams) {
+      val model = testSelector(sel, dataset)
+      MLTestingUtils.checkCopyAndUids(sel, model)
+    }
+  }
+
+  // use the following sklean program to verify the test
+  // scalastyle:off
+  /*
+  import numpy as np
+    from sklearn.feature_selection import SelectFdr, f_classif
+
+  X = np.random.rand(10, 6)
+  w = np.array([5, 5, 0.0, 0, 0, 0]).reshape((-1, 1))
+  y = np.rint(0.1 * (X @ w)).flatten()
+  print(X)
+  print(y)
+
+  F, p = f_classif(X, y)
+  print('F', F)
+  print('p', p)
+  selected = SelectFdr(f_classif, alpha=0.25).fit(X, y).get_support(True)
+
+  print(selected)
+  */
+
+  /*
+  sklearn result
+  [[0.92166066 0.82295823 0.31276624 0.63069973 0.64679537 0.94138368]
+  [0.47027783 0.74907889 0.43660557 0.93212582 0.5654378  0.531748  ]
+  [0.67771108 0.23926502 0.66906295 0.73117095 0.67340005 0.52864934]
+  [0.84565144 0.28050298 0.94137135 0.42479664 0.21600724 0.98956871]
+  [0.58818255 0.32223507 0.13727654 0.80948059 0.94617741 0.48460179]
+  [0.59528639 0.75838511 0.98648654 0.65561948 0.83818237 0.30178127]
+  [0.00264811 0.46492597 0.71428557 0.94708987 0.54587827 0.9484639 ]
+  [0.94604186 0.43187098 0.42135172 0.77256283 0.44334613 0.1514674 ]
+  [0.45694004 0.00273459 0.14580367 0.74278963 0.57819284 0.99413419]
+  [0.02256925 0.56136702 0.0629738  0.64130602 0.01536191 0.56638321]]
+  [1. 1. 0. 1. 0. 1. 0. 1. 0. 0.]
+  F [5.66456136e+00 4.08120006e+00 1.85418412e+00 8.67095392e-01
+  2.87769237e-03 3.66010633e-01]
+  p [0.04454332 0.07803464 0.21040406 0.37900428 0.95853411 0.56195058]
+  [0 1]
+
+  [[0.27976711 0.48397753 0.18451698 0.59844137 0.01459805 0.98895542]
+  [0.97192726 0.46737333 0.08048093 0.38253056 0.04776121 0.55949538]
+  [0.62559834 0.44102192 0.19199043 0.959706   0.5332824  0.78621594]
+  [0.91649448 0.76501992 0.58678528 0.75239909 0.33179368 0.00893317]
+  [0.14086806 0.21876364 0.31767297 0.53061653 0.02786653 0.20021944]
+  [0.15214833 0.03028593 0.12326784 0.55663152 0.8333684  0.76923807]
+  [0.88178287 0.8492688  0.29417221 0.98122401 0.44103191 0.32709781]
+  [0.06686689 0.05834763 0.41316273 0.92850555 0.77308549 0.2931857 ]
+  [0.94747449 0.78336777 0.76096282 0.52368192 0.64814324 0.60455684]
+  [0.83382261 0.31412713 0.62490246 0.43896432 0.35390503 0.02316754]]
+  [0. 1. 1. 1. 0. 0. 1. 0. 1. 1.]
+  F [9.22227201e+01 8.36710241e+00 1.22217112e+00 1.63526175e-02
+  8.91954821e-03 6.44534477e-01]
+  p [1.14739663e-05 2.01189199e-02 3.01070031e-01 9.01402125e-01
+  9.27079623e-01 4.45267639e-01]
+  [0 1]
+  */
+  // scalastyle:on
+  test("Test selectIndicesFromPValues f_classif") {
+    val data_f_classif1 = Seq(
+      (1, Vectors.dense(0.92166066, 0.82295823, 0.31276624, 0.63069973, 0.64679537, 0.94138368),
+        Vectors.dense(0.92166066, 0.82295823)),
+      (1, Vectors.dense(0.47027783, 0.74907889, 0.43660557, 0.93212582, 0.5654378, 0.531748),
+        Vectors.dense(0.47027783, 0.74907889)),
+      (0, Vectors.dense(0.67771108, 0.23926502, 0.66906295, 0.73117095, 0.67340005, 0.52864934),
+        Vectors.dense(0.67771108, 0.23926502)),
+      (1, Vectors.dense(0.84565144, 0.28050298, 0.94137135, 0.42479664, 0.21600724, 0.98956871),
+        Vectors.dense(0.84565144, 0.28050298)),
+      (0, Vectors.dense(0.58818255, 0.32223507, 0.13727654, 0.80948059, 0.94617741, 0.48460179),
+        Vectors.dense(0.58818255, 0.32223507)),
+      (1, Vectors.dense(0.59528639, 0.75838511, 0.98648654, 0.65561948, 0.83818237, 0.30178127),
+        Vectors.dense(0.59528639, 0.75838511)),
+      (0, Vectors.dense(0.00264811, 0.46492597, 0.71428557, 0.94708987, 0.54587827, 0.9484639),
+        Vectors.dense(0.00264811, 0.46492597)),
+      (1, Vectors.dense(0.94604186, 0.43187098, 0.42135172, 0.77256283, 0.44334613, 0.1514674),
+        Vectors.dense(0.94604186, 0.43187098)),
+      (0, Vectors.dense(0.45694004, 0.00273459, 0.14580367, 0.74278963, 0.57819284, 0.99413419),
+        Vectors.dense(0.45694004, 0.00273459)),
+      (0, Vectors.dense(0.02256925, 0.56136702, 0.0629738, 0.64130602, 0.01536191, 0.56638321),
+        Vectors.dense(0.02256925, 0.56136702)))
+
+    val data_f_classif2 = Seq(
+      (0, Vectors.dense(0.27976711, 0.48397753, 0.18451698, 0.59844137, 0.01459805, 0.98895542),
+        Vectors.dense(0.27976711, 0.48397753)),
+      (1, Vectors.dense(0.97192726, 0.46737333, 0.08048093, 0.38253056, 0.04776121, 0.55949538),
+        Vectors.dense(0.97192726, 0.46737333)),
+      (1, Vectors.dense(0.62559834, 0.44102192, 0.19199043, 0.959706, 0.5332824, 0.78621594),
+        Vectors.dense(0.62559834, 0.44102192)),
+      (1, Vectors.dense(0.91649448, 0.76501992, 0.58678528, 0.75239909, 0.33179368, 0.00893317),
+        Vectors.dense(0.91649448, 0.76501992)),
+      (0, Vectors.dense(0.14086806, 0.21876364, 0.31767297, 0.53061653, 0.02786653, 0.20021944),
+        Vectors.dense(0.14086806, 0.21876364)),
+      (0, Vectors.dense(0.15214833, 0.03028593, 0.12326784, 0.55663152, 0.8333684, 0.76923807),
+        Vectors.dense(0.15214833, 0.03028593)),
+      (1, Vectors.dense(0.88178287, 0.8492688, 0.29417221, 0.98122401, 0.44103191, 0.32709781),
+        Vectors.dense(0.88178287, 0.8492688)),
+      (0, Vectors.dense(0.06686689, 0.05834763, 0.41316273, 0.92850555, 0.77308549, 0.2931857),
+        Vectors.dense(0.06686689, 0.05834763)),
+      (1, Vectors.dense(0.94747449, 0.78336777, 0.76096282, 0.52368192, 0.64814324, 0.60455684),
+        Vectors.dense(0.94747449, 0.78336777)),
+      (1, Vectors.dense(0.83382261, 0.31412713, 0.62490246, 0.43896432, 0.35390503, 0.02316754),
+        Vectors.dense(0.83382261, 0.31412713)))
+
+    val dataset_f_classification1 =
+      spark.createDataFrame(data_f_classif1).toDF("label", "features", "topFeature")
+
+    val dataset_f_classification2 =
+      spark.createDataFrame(data_f_classif2).toDF("label", "features", "topFeature")
+
+    val resultDF1 = ANOVATest.test(dataset_f_classification1.toDF, "features", "label", true)
+    val resultDF2 = ANOVATest.test(dataset_f_classification2.toDF, "features", "label", true)
+    val selector = new UnivariateFeatureSelector()
+      .setOutputCol("filtered")
+      .setFeatureType("continuous")
+      .setLabelType("categorical")
+    val indices1 = selector.selectIndicesFromPValues(6, resultDF1, "fdr", 0.25)
+    val indices2 = selector.selectIndicesFromPValues(6, resultDF2, "fdr", 0.25)
+    assert(indices1(0) === 0 && indices1(1) === 1)
+    assert(indices2(0) === 0 && indices1(1) === 1)
+  }
+
+  // use the following sklean program to verify the test
+  // scalastyle:off
+  /* import numpy as np
+    from sklearn.feature_selection import SelectFdr, f_regression
+
+  X = np.random.rand(10, 6)
+  w = np.array([5, 5, 0.0, 0, 0, 0]).reshape((-1, 1))
+  y = (X @ w).flatten()
+  print(X)
+  print(y)
+
+  F, p = f_regression(X, y)
+  print('F', F)
+  print('p', p)
+  selected = SelectFdr(f_regression, alpha=0.1).fit(X, y).get_support(True)
+
+  print(selected) */
+
+  /* sklean result
+  [[5.19537247e-01 4.53144603e-01 2.10190418e-01 9.76237361e-01
+  9.05792824e-01 9.34081024e-01]
+  [8.68906163e-01 5.49099467e-01 6.73567960e-01 3.94736897e-01
+  9.98764158e-01 1.14285918e-01]
+  [2.56211244e-01 5.21857152e-01 6.55000402e-01 4.81092256e-01
+  4.05802734e-02 1.59811005e-01]
+  [9.03076723e-01 1.80316576e-01 8.13131160e-01 6.92327901e-01
+  4.77693321e-01 2.17284784e-01]
+  [4.75926597e-01 6.80511651e-01 9.55843875e-01 1.52627108e-01
+  1.72766587e-01 6.45234673e-01]
+  [6.05829005e-01 8.43879811e-01 4.48596383e-01 7.25003439e-01
+  2.83962640e-02 5.14414827e-01]
+  [8.57631869e-01 1.18279868e-01 2.84428492e-01 8.51544596e-01
+  1.33220409e-02 1.87044251e-01]
+  [2.43360773e-01 4.83288948e-02 1.10430569e-01 4.33097852e-01
+  5.63452248e-02 8.24333214e-01]
+  [2.18226531e-01 5.28477779e-01 3.01852956e-01 6.31664822e-04
+  8.97463990e-01 8.25297034e-01]
+  [6.95170305e-01 7.35775299e-01 4.32188618e-01 2.26744166e-01
+  5.13186095e-01 2.91635657e-01]]
+  [4.86340925 7.09002815 3.89034198 5.4169665  5.78219124 7.24854408
+  4.87955868 1.45844834 3.73352155 7.15472802]
+  F [6.79932587 7.09311449 2.25262252 0.02652918 0.40812054 2.14464201]
+  p [0.03124895 0.02865887 0.17178184 0.87465381 0.54077957 0.18122753]
+  [0 1]
+  */
+
+  /* SKLearn result
+  [[0.21557113 0.66070242 0.89964323 0.1569332  0.84097522 0.61614986]
+  [0.14790391 0.40356507 0.2973803  0.53051143 0.35408457 0.88180598]
+  [0.39333276 0.42790148 0.41415147 0.82478069 0.57201431 0.49972278]
+  [0.46189165 0.460305   0.21054573 0.16588781 0.72898672 0.41290627]
+  [0.42527082 0.83902909 0.97275171 0.76947383 0.24470714 0.57847281]
+  [0.56185556 0.94463811 0.97741409 0.27233834 0.76460529 0.53085766]
+  [0.5828694  0.45827703 0.49305311 0.13803643 0.18242319 0.14182515]
+  [0.98848811 0.43453809 0.11712213 0.4849829  0.06431555 0.76125387]
+  [0.1181108  0.43820753 0.49576967 0.75729578 0.35355208 0.48165022]
+  [0.44250624 0.24310088 0.03976366 0.24023351 0.91659502 0.75260252]]
+  [4.38136774 2.7573449  4.10617119 4.61098326 6.32149954 7.53246836
+  5.20573215 7.11513098 2.78159163 3.42803558]
+  F [11.90962327  6.49595546  1.51054886  0.17751367  0.40829523  0.1797005 ]
+  p [0.0086816  0.03424301 0.25397764 0.68461076 0.54069506 0.68279904]
+  [0] */
+  // scalastyle:on
+  test("Test selectIndicesFromPValues f_regression") {
+    val data_f_regression1 = Seq(
+      (4.86340925, Vectors.dense(5.19537247e-01, 4.53144603e-01, 2.10190418e-01, 9.76237361e-01,
+        9.05792824e-01, 9.34081024e-01), Vectors.dense(5.19537247e-01, 4.53144603e-01)),
+      (7.09002815, Vectors.dense(8.68906163e-01, 5.49099467e-01, 6.73567960e-01, 3.94736897e-01,
+        9.98764158e-01, 1.14285918e-01), Vectors.dense(8.68906163e-01, 5.49099467e-01)),
+      (3.89034198, Vectors.dense(2.56211244e-01, 5.21857152e-01, 6.55000402e-01, 4.81092256e-01,
+        4.05802734e-02, 1.59811005e-01), Vectors.dense(2.56211244e-01, 5.21857152e-01)),
+      (5.4169665, Vectors.dense(9.03076723e-01, 1.80316576e-01, 8.13131160e-01, 6.92327901e-01,
+        4.77693321e-01, 2.17284784e-01), Vectors.dense(9.03076723e-01, 1.80316576e-01)),
+      (5.78219124, Vectors.dense(4.75926597e-01, 6.80511651e-01, 9.55843875e-01, 1.52627108e-01,
+        1.72766587e-01, 6.45234673e-01), Vectors.dense(4.75926597e-01, 6.80511651e-01)),
+      (7.24854408, Vectors.dense(6.05829005e-01, 8.43879811e-01, 4.48596383e-01, 7.25003439e-01,
+        2.83962640e-02, 5.14414827e-01), Vectors.dense(6.05829005e-01, 8.43879811e-01)),
+      (4.87955868, Vectors.dense(8.57631869e-01, 1.18279868e-01, 2.84428492e-01, 8.51544596e-01,
+        1.33220409e-02, 1.87044251e-01), Vectors.dense(8.57631869e-01, 1.18279868e-01)),
+      (1.45844834, Vectors.dense(2.43360773e-01, 4.83288948e-02, 1.10430569e-01, 4.33097852e-01,
+        5.63452248e-02, 8.24333214e-01), Vectors.dense(2.43360773e-01, 4.83288948e-02)),
+      (3.73352155, Vectors.dense(2.18226531e-01, 5.28477779e-01, 3.01852956e-01, 6.31664822e-04,
+        8.97463990e-01, 8.25297034e-01), Vectors.dense(2.18226531e-01, 5.28477779e-01)),
+      (7.15472802, Vectors.dense(6.95170305e-01, 7.35775299e-01, 4.32188618e-01, 2.26744166e-01,
+        5.13186095e-01, 2.91635657e-01), Vectors.dense(6.95170305e-01, 7.35775299e-01)))
+
+    val data_f_regression2 = Seq(
+      (4.38136774, Vectors.dense(0.21557113, 0.66070242, 0.89964323, 0.1569332, 0.84097522,
+        0.61614986), Vectors.dense(0.21557113)),
+      (2.7573449, Vectors.dense(0.14790391, 0.40356507, 0.2973803, 0.53051143, 0.35408457,
+        0.88180598), Vectors.dense(0.14790391)),
+      (4.10617119, Vectors.dense(0.39333276, 0.42790148, 0.41415147, 0.82478069, 0.57201431,
+        0.49972278), Vectors.dense(0.39333276)),
+      (4.61098326, Vectors.dense(0.46189165, 0.460305, 0.21054573, 0.16588781, 0.72898672,
+        0.41290627), Vectors.dense(0.46189165)),
+      (6.32149954, Vectors.dense(0.42527082, 0.83902909, 0.97275171, 0.76947383, 0.24470714,
+        0.57847281), Vectors.dense(0.42527082)),
+      (7.53246836, Vectors.dense(0.56185556, 0.94463811, 0.97741409, 0.27233834, 0.76460529,
+        0.53085766), Vectors.dense(0.56185556)),
+      (5.20573215, Vectors.dense(0.5828694, 0.45827703, 0.49305311, 0.13803643, 0.18242319,
+        0.14182515), Vectors.dense(0.5828694)),
+      (7.11513098, Vectors.dense(0.98848811, 0.43453809, 0.11712213, 0.4849829, 0.06431555,
+        0.76125387), Vectors.dense(0.98848811)),
+      (2.78159163, Vectors.dense(0.1181108, 0.43820753, 0.49576967, 0.75729578, 0.35355208,
+        0.48165022), Vectors.dense(0.1181108)),
+      (3.42803558, Vectors.dense(0.44250624, 0.24310088, 0.03976366, 0.24023351, 0.91659502,
+        0.75260252), Vectors.dense(0.44250624)))
+
+    val dataset_f_regression1 =
+      spark.createDataFrame(data_f_regression1).toDF("label", "features", "topFeature")
+
+    val dataset_f_regression2 =
+      spark.createDataFrame(data_f_regression2).toDF("label", "features", "topFeature")
+
+    val resultDF1 = FValueTest.test(dataset_f_regression1.toDF, "features", "label", true)
+    val resultDF2 = FValueTest.test(dataset_f_regression2.toDF, "features", "label", true)
+    val selector = new UnivariateFeatureSelector()
+      .setOutputCol("filtered")
+      .setFeatureType("continuous")
+      .setLabelType("continuous")
+    val indices1 = selector.selectIndicesFromPValues(6, resultDF1, "fdr", 0.1)
+    val indices2 = selector.selectIndicesFromPValues(6, resultDF2, "fdr", 0.1)
+    assert(indices1(0) === 1 && indices1(1) === 0)
+    assert(indices2(0) === 0)
+  }
+
+  test("read/write") {
+    def checkModelData(
+        model: UnivariateFeatureSelectorModel,
+        model2: UnivariateFeatureSelectorModel): Unit = {
+      assert(model.selectedFeatures === model2.selectedFeatures)
+    }
+    val selector = new UnivariateFeatureSelector()
+      .setFeatureType("continuous")
+      .setLabelType("categorical")
+    testEstimatorAndModelReadWrite(selector, datasetAnova,
+      UnivariateFeatureSelectorSuite.allParamSettings,
+      UnivariateFeatureSelectorSuite.allParamSettings, checkModelData)
+  }
+
+  private def testSelector(selector: UnivariateFeatureSelector, data: Dataset[_]):
+  UnivariateFeatureSelectorModel = {
+    val selectorModel = selector.fit(data)
+    testTransformer[(Double, Vector, Vector)](data.toDF(), selectorModel,
+      "filtered", "topFeature") {
+      case Row(vec1: Vector, vec2: Vector) =>
+        assert(vec1 ~== vec2 absTol 1e-1)
+    }
+    selectorModel
+  }
+}
+
+object UnivariateFeatureSelectorSuite {
+
+  /**
+   * Mapping from all Params to valid settings which differ from the defaults.
+   * This is useful for tests which need to exercise all Params, such as save/load.
+   * This excludes input columns to simplify some tests.
+   */
+  val allParamSettings: Map[String, Any] = Map(
+    "selectionMode" -> "percentile",
+    "selectionThreshold" -> 0.12,
+    "outputCol" -> "myOutput"
+  )
+}
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/VarianceThresholdSelectorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/VarianceThresholdSelectorSuite.scala
index cc451c0b60379..142abf2ccdfb9 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/VarianceThresholdSelectorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/VarianceThresholdSelectorSuite.scala
@@ -53,7 +53,7 @@ class VarianceThresholdSelectorSuite extends MLTest with DefaultReadWriteTest {
     ParamsSuite.checkParams(new VarianceThresholdSelector)
   }
 
-  test("Test VarianceThresholdSelector: varainceThreshold not set") {
+  test("Test VarianceThresholdSelector: varianceThreshold not set") {
     val selector = new VarianceThresholdSelector().setOutputCol("filtered")
     val model = testSelector(selector, dataset)
     MLTestingUtils.checkCopyAndUids(selector, model)
diff --git a/mllib/src/test/scala/org/apache/spark/ml/fpm/FPGrowthSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/fpm/FPGrowthSuite.scala
index b75526a48371a..3d994366b8918 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/fpm/FPGrowthSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/fpm/FPGrowthSuite.scala
@@ -39,9 +39,9 @@ class FPGrowthSuite extends SparkFunSuite with MLlibTestSparkContext with Defaul
       val model = new FPGrowth().setMinSupport(0.5).fit(data)
       val generatedRules = model.setMinConfidence(0.5).associationRules
       val expectedRules = spark.createDataFrame(Seq(
-        (Array("2"), Array("1"), 1.0, 1.0),
-        (Array("1"), Array("2"), 0.75, 1.0)
-      )).toDF("antecedent", "consequent", "confidence", "lift")
+        (Array("2"), Array("1"), 1.0, 1.0, 0.75),
+        (Array("1"), Array("2"), 0.75, 1.0, 0.75)
+      )).toDF("antecedent", "consequent", "confidence", "lift", "support")
         .withColumn("antecedent", col("antecedent").cast(ArrayType(dt)))
         .withColumn("consequent", col("consequent").cast(ArrayType(dt)))
       assert(expectedRules.sort("antecedent").rdd.collect().sameElements(
@@ -61,6 +61,31 @@ class FPGrowthSuite extends SparkFunSuite with MLlibTestSparkContext with Defaul
     }
   }
 
+  test("FPGrowth associationRules") {
+    val dataset = spark.createDataFrame(Seq(
+        (1, Array("1", "2")),
+        (2, Array("3")),
+        (3, Array("4", "5")),
+        (4, Array("1", "2", "3")),
+        (5, Array("2"))
+      )).toDF("id", "items")
+    val model = new FPGrowth().setMinSupport(0.1).setMinConfidence(0.1).fit(dataset)
+    val expectedRules = spark.createDataFrame(Seq(
+      (Array("2"), Array("1"), 0.6666666666666666, 1.6666666666666665, 0.4),
+      (Array("2"), Array("3"), 0.3333333333333333, 0.8333333333333333, 0.2),
+      (Array("3"), Array("1"), 0.5, 1.25, 0.2),
+      (Array("3"), Array("2"), 0.5, 0.8333333333333334, 0.2),
+      (Array("1", "3"), Array("2"), 1.0, 1.6666666666666667, 0.2),
+      (Array("1", "2"), Array("3"), 0.5, 1.25, 0.2),
+      (Array("4"), Array("5"), 1.0, 5.0, 0.2),
+      (Array("5"), Array("4"), 1.0, 5.0, 0.2),
+      (Array("1"), Array("3"), 0.5, 1.25, 0.2),
+      (Array("1"), Array("2"), 1.0, 1.6666666666666667, 0.4),
+      (Array("3", "2"), Array("1"), 1.0, 2.5, 0.2)
+    )).toDF("antecedent", "consequent", "confidence", "lift", "support")
+    assert(expectedRules.collect().toSet.equals(model.associationRules.collect().toSet))
+  }
+
   test("FPGrowth getFreqItems") {
     val model = new FPGrowth().setMinSupport(0.7).fit(dataset)
     val expectedFreq = spark.createDataFrame(Seq(
@@ -96,7 +121,7 @@ class FPGrowthSuite extends SparkFunSuite with MLlibTestSparkContext with Defaul
 
     val prediction = model.transform(
       spark.createDataFrame(Seq(Tuple1(Array("1", "2")))).toDF("items")
-    ).first().getAs[Seq[String]]("prediction")
+    ).first().getAs[scala.collection.Seq[String]]("prediction")
 
     assert(prediction === Seq("3"))
   }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/fpm/PrefixSpanSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/fpm/PrefixSpanSuite.scala
index 2252151af306b..cc8982f338702 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/fpm/PrefixSpanSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/fpm/PrefixSpanSuite.scala
@@ -17,7 +17,6 @@
 package org.apache.spark.ml.fpm
 
 import org.apache.spark.ml.util.MLTest
-import org.apache.spark.sql.DataFrame
 
 class PrefixSpanSuite extends MLTest {
 
diff --git a/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/BinaryLogisticBlockAggregatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/BinaryLogisticBlockAggregatorSuite.scala
new file mode 100644
index 0000000000000..f5ae22d228cdd
--- /dev/null
+++ b/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/BinaryLogisticBlockAggregatorSuite.scala
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.ml.optim.aggregator
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.ml.feature.{Instance, InstanceBlock}
+import org.apache.spark.ml.linalg._
+import org.apache.spark.ml.stat.Summarizer
+import org.apache.spark.ml.util.TestingUtils._
+import org.apache.spark.mllib.util.MLlibTestSparkContext
+
+class BinaryLogisticBlockAggregatorSuite extends SparkFunSuite with MLlibTestSparkContext {
+
+  @transient var instances: Array[Instance] = _
+  @transient var instancesConstantFeature: Array[Instance] = _
+  @transient var instancesConstantFeatureFiltered: Array[Instance] = _
+  @transient var scaledInstances: Array[Instance] = _
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    instances = Array(
+      Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
+      Instance(1.0, 0.5, Vectors.dense(1.5, 1.0)),
+      Instance(0.0, 0.3, Vectors.dense(4.0, 0.5))
+    )
+    instancesConstantFeature = Array(
+      Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
+      Instance(1.0, 0.5, Vectors.dense(1.0, 1.0)),
+      Instance(0.0, 0.3, Vectors.dense(1.0, 0.5))
+    )
+    instancesConstantFeatureFiltered = Array(
+      Instance(0.0, 0.1, Vectors.dense(2.0)),
+      Instance(1.0, 0.5, Vectors.dense(1.0)),
+      Instance(0.0, 0.3, Vectors.dense(0.5))
+    )
+    scaledInstances = standardize(instances)
+  }
+
+  /** Get summary statistics for some data and create a new BinaryLogisticBlockAggregator. */
+  private def getNewAggregator(
+      instances: Array[Instance],
+      coefficients: Vector,
+      fitIntercept: Boolean,
+      fitWithMean: Boolean): BinaryLogisticBlockAggregator = {
+    val (featuresSummarizer, _) =
+      Summarizer.getClassificationSummarizers(sc.parallelize(instances))
+    val featuresStd = featuresSummarizer.std.toArray
+    val featuresMean = featuresSummarizer.mean.toArray
+    val inverseStd = featuresStd.map(std => if (std != 0) 1.0 / std else 0.0)
+    val scaledMean = inverseStd.zip(featuresMean).map(t => t._1 * t._2)
+    val bcInverseStd = sc.broadcast(inverseStd)
+    val bcScaledMean = sc.broadcast(scaledMean)
+    val bcCoefficients = sc.broadcast(coefficients)
+    new BinaryLogisticBlockAggregator(bcInverseStd,
+      bcScaledMean, fitIntercept, fitWithMean)(bcCoefficients)
+  }
+
+  test("sparse coefficients") {
+    val bcInverseStd = sc.broadcast(Array(1.0))
+    val bcScaledMean = sc.broadcast(Array(2.0))
+    val bcCoefficients = sc.broadcast(Vectors.sparse(2, Array(0), Array(1.0)))
+    val binaryAgg = new BinaryLogisticBlockAggregator(bcInverseStd, bcScaledMean,
+      fitIntercept = true, fitWithMean = false)(bcCoefficients)
+    val block = InstanceBlock.fromInstances(Seq(Instance(1.0, 1.0, Vectors.dense(1.0))))
+    val thrownBinary = withClue("aggregator cannot handle sparse coefficients") {
+      intercept[IllegalArgumentException] {
+        binaryAgg.add(block)
+      }
+    }
+    assert(thrownBinary.getMessage.contains("coefficients only supports dense"))
+  }
+
+  test("aggregator add method input size") {
+    val coefArray = Array(1.0, 2.0)
+    val interceptValue = 4.0
+    val agg = getNewAggregator(instances, Vectors.dense(coefArray :+ interceptValue),
+      fitIntercept = true, fitWithMean = true)
+    val block = InstanceBlock.fromInstances(Seq(Instance(1.0, 1.0, Vectors.dense(2.0))))
+    withClue("BinaryLogisticBlockAggregator features dimension must match coefficients dimension") {
+      intercept[IllegalArgumentException] {
+        agg.add(block)
+      }
+    }
+  }
+
+  test("negative weight") {
+    val coefArray = Array(1.0, 2.0)
+    val interceptValue = 4.0
+    val agg = getNewAggregator(instances, Vectors.dense(coefArray :+ interceptValue),
+      fitIntercept = true, fitWithMean = true)
+    val block = InstanceBlock.fromInstances(Seq(Instance(1.0, -1.0, Vectors.dense(2.0, 1.0))))
+    withClue("BinaryLogisticBlockAggregator does not support negative instance weights") {
+      intercept[IllegalArgumentException] {
+        agg.add(block)
+      }
+    }
+  }
+
+  test("check sizes") {
+    val rng = new scala.util.Random
+    val numFeatures = instances.head.features.size
+    val coefWithIntercept = Vectors.dense(Array.fill(numFeatures + 1)(rng.nextDouble))
+    val coefWithoutIntercept = Vectors.dense(Array.fill(numFeatures)(rng.nextDouble))
+    val block = InstanceBlock.fromInstances(instances)
+
+    val aggIntercept = getNewAggregator(instances, coefWithIntercept,
+      fitIntercept = true, fitWithMean = false)
+    aggIntercept.add(block)
+    assert(aggIntercept.gradient.size === numFeatures + 1)
+
+    val aggNoIntercept = getNewAggregator(instances, coefWithoutIntercept,
+      fitIntercept = false, fitWithMean = false)
+    aggNoIntercept.add(block)
+    assert(aggNoIntercept.gradient.size === numFeatures)
+  }
+
+  test("check correctness: fitIntercept = false") {
+    val coefVec = Vectors.dense(1.0, 2.0)
+    val numFeatures = instances.head.features.size
+    val (featuresSummarizer, _) =
+      Summarizer.getClassificationSummarizers(sc.parallelize(instances))
+    val featuresStd = featuresSummarizer.std
+    val stdCoefVec = Vectors.dense(Array.tabulate(coefVec.size)(i => coefVec(i) / featuresStd(i)))
+    val weightSum = instances.map(_.weight).sum
+
+    // compute the loss and the gradients
+    var lossSum = 0.0
+    val gradientCoef = Array.ofDim[Double](numFeatures)
+    instances.foreach { case Instance(l, w, f) =>
+      val margin = BLAS.dot(stdCoefVec, f)
+      val prob = 1.0 / (1.0 + math.exp(-margin))
+      lossSum += -w * l * math.log(prob) - w * (1.0 - l) * math.log1p(-prob)
+      gradientCoef.indices.foreach { i =>
+        gradientCoef(i) += w * (prob - l) * f(i) / featuresStd(i)
+      }
+    }
+    val loss = lossSum / weightSum
+    val gradient = Vectors.dense(gradientCoef.map(_ / weightSum))
+
+    Seq(1, 2, 4).foreach { blockSize =>
+      val blocks1 = scaledInstances
+        .grouped(blockSize)
+        .map(seq => InstanceBlock.fromInstances(seq))
+        .toArray
+      val blocks2 = blocks1.map { block =>
+        new InstanceBlock(block.labels, block.weights, block.matrix.toSparseRowMajor)
+      }
+
+      Seq(blocks1, blocks2).foreach { blocks =>
+        val agg = getNewAggregator(instances, coefVec,
+          fitIntercept = false, fitWithMean = false)
+        blocks.foreach(agg.add)
+        assert(agg.loss ~== loss relTol 1e-9)
+        assert(agg.gradient ~== gradient relTol 1e-9)
+      }
+    }
+  }
+
+  test("check correctness: fitIntercept = true, fitWithMean = false") {
+    val coefVec = Vectors.dense(1.0, 2.0)
+    val interceptValue = 1.0
+    val numFeatures = instances.head.features.size
+    val (featuresSummarizer, _) =
+      Summarizer.getClassificationSummarizers(sc.parallelize(instances))
+    val featuresStd = featuresSummarizer.std
+    val stdCoefVec = Vectors.dense(Array.tabulate(coefVec.size)(i => coefVec(i) / featuresStd(i)))
+    val weightSum = instances.map(_.weight).sum
+
+    // compute the loss and the gradients
+    var lossSum = 0.0
+    val gradientCoef = Array.ofDim[Double](numFeatures)
+    var gradientIntercept = 0.0
+    instances.foreach { case Instance(l, w, f) =>
+      val margin = BLAS.dot(stdCoefVec, f) + interceptValue
+      val prob = 1.0 / (1.0 + math.exp(-margin))
+      lossSum += -w * l * math.log(prob) - w * (1.0 - l) * math.log1p(-prob)
+      gradientCoef.indices.foreach { i =>
+        gradientCoef(i) += w * (prob - l) * f(i) / featuresStd(i)
+      }
+      gradientIntercept += w * (prob - l)
+    }
+    val loss = lossSum / weightSum
+    val gradient = Vectors.dense((gradientCoef :+ gradientIntercept).map(_ / weightSum))
+
+    Seq(1, 2, 4).foreach { blockSize =>
+      val blocks1 = scaledInstances
+        .grouped(blockSize)
+        .map(seq => InstanceBlock.fromInstances(seq))
+        .toArray
+      val blocks2 = blocks1.map { block =>
+        new InstanceBlock(block.labels, block.weights, block.matrix.toSparseRowMajor)
+      }
+
+      Seq(blocks1, blocks2).foreach { blocks =>
+        val agg = getNewAggregator(instances, Vectors.dense(coefVec.toArray :+ interceptValue),
+          fitIntercept = true, fitWithMean = false)
+        blocks.foreach(agg.add)
+        assert(agg.loss ~== loss relTol 1e-9)
+        assert(agg.gradient ~== gradient relTol 1e-9)
+      }
+    }
+  }
+
+  test("check correctness: fitIntercept = true, fitWithMean = true") {
+    val coefVec = Vectors.dense(1.0, 2.0)
+    val interceptValue = 1.0
+    val numFeatures = instances.head.features.size
+    val (featuresSummarizer, _) =
+      Summarizer.getClassificationSummarizers(sc.parallelize(instances))
+    val featuresStd = featuresSummarizer.std
+    val featuresMean = featuresSummarizer.mean
+    val stdCoefVec = Vectors.dense(Array.tabulate(coefVec.size)(i => coefVec(i) / featuresStd(i)))
+    val weightSum = instances.map(_.weight).sum
+
+    // compute the loss and the gradients
+    var lossSum = 0.0
+    val gradientCoef = Array.ofDim[Double](numFeatures)
+    var gradientIntercept = 0.0
+    instances.foreach { case Instance(l, w, f) =>
+      val centered = f.toDense.copy
+      BLAS.axpy(-1.0, featuresMean, centered)
+      val margin = BLAS.dot(stdCoefVec, centered) + interceptValue
+      val prob = 1.0 / (1.0 + math.exp(-margin))
+      lossSum += -w * l * math.log(prob) - w * (1.0 - l) * math.log1p(-prob)
+      gradientCoef.indices.foreach { i =>
+        gradientCoef(i) += w * (prob - l) * (f(i) - featuresMean(i)) / featuresStd(i)
+      }
+      gradientIntercept += w * (prob - l)
+    }
+    val loss = lossSum / weightSum
+    val gradient = Vectors.dense((gradientCoef :+ gradientIntercept).map(_ / weightSum))
+
+    Seq(1, 2, 4).foreach { blockSize =>
+      val blocks1 = scaledInstances
+        .grouped(blockSize)
+        .map(seq => InstanceBlock.fromInstances(seq))
+        .toArray
+      val blocks2 = blocks1.map { block =>
+        new InstanceBlock(block.labels, block.weights, block.matrix.toSparseRowMajor)
+      }
+
+      Seq(blocks1, blocks2).foreach { blocks =>
+        val agg = getNewAggregator(instances, Vectors.dense(coefVec.toArray :+ interceptValue),
+          fitIntercept = true, fitWithMean = true)
+        blocks.foreach(agg.add)
+        assert(agg.loss ~== loss relTol 1e-9)
+        assert(agg.gradient ~== gradient relTol 1e-9)
+      }
+    }
+  }
+
+  test("check with zero standard deviation") {
+    val coefArray = Array(1.0, 2.0)
+    val coefArrayFiltered = Array(2.0)
+    val interceptValue = 1.0
+
+    Seq((false, false), (true, false), (true, true)).foreach { case (fitIntercept, fitWithMean) =>
+      val coefVec = if (fitIntercept) {
+        Vectors.dense(coefArray :+ interceptValue)
+      } else {
+        Vectors.dense(coefArray)
+      }
+      val aggConstantFeature = getNewAggregator(instancesConstantFeature,
+        coefVec, fitIntercept = fitIntercept, fitWithMean = fitWithMean)
+      aggConstantFeature
+        .add(InstanceBlock.fromInstances(standardize(instancesConstantFeature)))
+      val grad = aggConstantFeature.gradient
+
+      val coefVecFiltered = if (fitIntercept) {
+        Vectors.dense(coefArrayFiltered :+ interceptValue)
+      } else {
+        Vectors.dense(coefArrayFiltered)
+      }
+      val aggConstantFeatureFiltered = getNewAggregator(instancesConstantFeatureFiltered,
+        coefVecFiltered, fitIntercept = fitIntercept, fitWithMean = fitWithMean)
+      aggConstantFeatureFiltered
+        .add(InstanceBlock.fromInstances(standardize(instancesConstantFeatureFiltered)))
+      val gradFiltered = aggConstantFeatureFiltered.gradient
+
+      // constant features should not affect gradient
+      assert(aggConstantFeature.loss ~== aggConstantFeatureFiltered.loss relTol 1e-9)
+      assert(grad(0) === 0)
+      assert(grad(1) ~== gradFiltered(0) relTol 1e-9)
+      if (fitIntercept) {
+        assert(grad.toArray.last ~== gradFiltered.toArray.last relTol 1e-9)
+      }
+    }
+  }
+}
diff --git a/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/HingeAggregatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/HingeAggregatorSuite.scala
deleted file mode 100644
index 425a5eb26ab67..0000000000000
--- a/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/HingeAggregatorSuite.scala
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.ml.optim.aggregator
-
-import org.apache.spark.SparkFunSuite
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
-import org.apache.spark.ml.linalg.{BLAS, Vector, Vectors}
-import org.apache.spark.ml.stat.Summarizer
-import org.apache.spark.ml.util.TestingUtils._
-import org.apache.spark.mllib.util.MLlibTestSparkContext
-
-class HingeAggregatorSuite extends SparkFunSuite with MLlibTestSparkContext {
-
-  @transient var instances: Array[Instance] = _
-  @transient var instancesConstantFeature: Array[Instance] = _
-  @transient var instancesConstantFeatureFiltered: Array[Instance] = _
-  @transient var standardizedInstances: Array[Instance] = _
-
-  override def beforeAll(): Unit = {
-    super.beforeAll()
-    instances = Array(
-      Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
-      Instance(1.0, 0.5, Vectors.dense(1.5, 1.0)),
-      Instance(0.0, 0.3, Vectors.dense(4.0, 0.5))
-    )
-    instancesConstantFeature = Array(
-      Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
-      Instance(1.0, 0.5, Vectors.dense(1.0, 1.0)),
-      Instance(1.0, 0.3, Vectors.dense(1.0, 0.5))
-    )
-    instancesConstantFeatureFiltered = Array(
-      Instance(0.0, 0.1, Vectors.dense(2.0)),
-      Instance(1.0, 0.5, Vectors.dense(1.0)),
-      Instance(2.0, 0.3, Vectors.dense(0.5))
-    )
-    standardizedInstances = standardize(instances)
-  }
-
-   /** Get summary statistics for some data and create a new HingeAggregator. */
-  private def getNewAggregator(
-      instances: Array[Instance],
-      coefficients: Vector,
-      fitIntercept: Boolean): HingeAggregator = {
-    val (featuresSummarizer, ySummarizer) =
-      Summarizer.getClassificationSummarizers(sc.parallelize(instances))
-    val featuresStd = featuresSummarizer.std.toArray
-    val bcFeaturesStd = spark.sparkContext.broadcast(featuresStd)
-    val bcCoefficients = spark.sparkContext.broadcast(coefficients)
-    new HingeAggregator(bcFeaturesStd, fitIntercept)(bcCoefficients)
-  }
-
-  /** Get summary statistics for some data and create a new BlockHingeAggregator. */
-  private def getNewBlockAggregator(
-      coefficients: Vector,
-      fitIntercept: Boolean): BlockHingeAggregator = {
-    val bcCoefficients = spark.sparkContext.broadcast(coefficients)
-    new BlockHingeAggregator(fitIntercept)(bcCoefficients)
-  }
-
-  test("aggregator add method input size") {
-    val coefArray = Array(1.0, 2.0)
-    val interceptArray = Array(2.0)
-    val agg = getNewAggregator(instances, Vectors.dense(coefArray ++ interceptArray),
-      fitIntercept = true)
-    withClue("HingeAggregator features dimension must match coefficients dimension") {
-      intercept[IllegalArgumentException] {
-        agg.add(Instance(1.0, 1.0, Vectors.dense(2.0)))
-      }
-    }
-  }
-
-  test("negative weight") {
-    val coefArray = Array(1.0, 2.0)
-    val interceptArray = Array(2.0)
-    val agg = getNewAggregator(instances, Vectors.dense(coefArray ++ interceptArray),
-      fitIntercept = true)
-    withClue("HingeAggregator does not support negative instance weights") {
-      intercept[IllegalArgumentException] {
-        agg.add(Instance(1.0, -1.0, Vectors.dense(2.0, 1.0)))
-      }
-    }
-  }
-
-  test("check sizes") {
-    val rng = new scala.util.Random
-    val numFeatures = instances.head.features.size
-    val coefWithIntercept = Vectors.dense(Array.fill(numFeatures + 1)(rng.nextDouble))
-    val coefWithoutIntercept = Vectors.dense(Array.fill(numFeatures)(rng.nextDouble))
-    val aggIntercept = getNewAggregator(instances, coefWithIntercept, fitIntercept = true)
-    val aggNoIntercept = getNewAggregator(instances, coefWithoutIntercept,
-      fitIntercept = false)
-    instances.foreach(aggIntercept.add)
-    instances.foreach(aggNoIntercept.add)
-
-    assert(aggIntercept.gradient.size === numFeatures + 1)
-    assert(aggNoIntercept.gradient.size === numFeatures)
-  }
-
-  test("check correctness") {
-    val coefArray = Array(1.0, 2.0)
-    val intercept = 1.0
-    val numFeatures = instances.head.features.size
-    val (featuresSummarizer, _) = Summarizer.getClassificationSummarizers(sc.parallelize(instances))
-    val featuresStd = featuresSummarizer.std.toArray
-    val weightSum = instances.map(_.weight).sum
-
-    val agg = getNewAggregator(instances, Vectors.dense(coefArray ++ Array(intercept)),
-      fitIntercept = true)
-    instances.foreach(agg.add)
-
-    // compute the loss
-    val stdCoef = coefArray.indices.map(i => coefArray(i) / featuresStd(i)).toArray
-    val lossSum = instances.map { case Instance(l, w, f) =>
-      val margin = BLAS.dot(Vectors.dense(stdCoef), f) + intercept
-      val labelScaled = 2 * l - 1.0
-      if (1.0 > labelScaled * margin) {
-        (1.0 - labelScaled * margin) * w
-      } else {
-        0.0
-      }
-    }.sum
-    val loss = lossSum / weightSum
-
-    // compute the gradients
-    val gradientCoef = new Array[Double](numFeatures)
-    var gradientIntercept = 0.0
-    instances.foreach { case Instance(l, w, f) =>
-      val margin = BLAS.dot(f, Vectors.dense(coefArray)) + intercept
-      if (1.0 > (2 * l - 1.0) * margin) {
-        gradientCoef.indices.foreach { i =>
-          gradientCoef(i) += f(i) * -(2 * l - 1.0) * w / featuresStd(i)
-        }
-        gradientIntercept += -(2 * l - 1.0) * w
-      }
-    }
-    val gradient = Vectors.dense((gradientCoef ++ Array(gradientIntercept)).map(_ / weightSum))
-
-    assert(loss ~== agg.loss relTol 1e-9)
-    assert(gradient ~== agg.gradient relTol 1e-9)
-
-    Seq(1, 2, 4).foreach { blockSize =>
-      val blocks1 = standardizedInstances
-        .grouped(blockSize)
-        .map(seq => InstanceBlock.fromInstances(seq))
-        .toArray
-      val blocks2 = blocks1.map { block =>
-        new InstanceBlock(block.labels, block.weights, block.matrix.toSparseRowMajor)
-      }
-
-      Seq(blocks1, blocks2).foreach { blocks =>
-        val blockAgg = getNewBlockAggregator(Vectors.dense(coefArray ++ Array(intercept)),
-          fitIntercept = true)
-        blocks.foreach(blockAgg.add)
-        assert(agg.loss ~== blockAgg.loss relTol 1e-9)
-        assert(agg.gradient ~== blockAgg.gradient relTol 1e-9)
-      }
-    }
-  }
-
-  test("check with zero standard deviation") {
-    val binaryCoefArray = Array(1.0, 2.0)
-    val intercept = 1.0
-    val aggConstantFeatureBinary = getNewAggregator(instancesConstantFeature,
-      Vectors.dense(binaryCoefArray ++ Array(intercept)), fitIntercept = true)
-    instancesConstantFeature.foreach(aggConstantFeatureBinary.add)
-
-    val aggConstantFeatureBinaryFiltered = getNewAggregator(instancesConstantFeatureFiltered,
-      Vectors.dense(binaryCoefArray ++ Array(intercept)), fitIntercept = true)
-    instancesConstantFeatureFiltered.foreach(aggConstantFeatureBinaryFiltered.add)
-
-    // constant features should not affect gradient
-    assert(aggConstantFeatureBinary.gradient(0) === 0.0)
-    assert(aggConstantFeatureBinary.gradient(1) == aggConstantFeatureBinaryFiltered.gradient(0))
-  }
-}
diff --git a/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/HingeBlockAggregatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/HingeBlockAggregatorSuite.scala
new file mode 100644
index 0000000000000..029911adb46aa
--- /dev/null
+++ b/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/HingeBlockAggregatorSuite.scala
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.ml.optim.aggregator
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.ml.feature.{Instance, InstanceBlock}
+import org.apache.spark.ml.linalg._
+import org.apache.spark.ml.stat.Summarizer
+import org.apache.spark.ml.util.TestingUtils._
+import org.apache.spark.mllib.util.MLlibTestSparkContext
+
+class HingeBlockAggregatorSuite extends SparkFunSuite with MLlibTestSparkContext {
+
+  @transient var instances: Array[Instance] = _
+  @transient var instancesConstantFeature: Array[Instance] = _
+  @transient var instancesConstantFeatureFiltered: Array[Instance] = _
+  @transient var scaledInstances: Array[Instance] = _
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    instances = Array(
+      Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
+      Instance(1.0, 0.5, Vectors.dense(1.5, 1.0)),
+      Instance(0.0, 0.3, Vectors.dense(4.0, 0.5))
+    )
+    instancesConstantFeature = Array(
+      Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
+      Instance(1.0, 0.5, Vectors.dense(1.0, 1.0)),
+      Instance(1.0, 0.3, Vectors.dense(1.0, 0.5))
+    )
+    instancesConstantFeatureFiltered = Array(
+      Instance(0.0, 0.1, Vectors.dense(2.0)),
+      Instance(1.0, 0.5, Vectors.dense(1.0)),
+      Instance(1.0, 0.3, Vectors.dense(0.5))
+    )
+    scaledInstances = standardize(instances)
+  }
+
+
+  /** Get summary statistics for some data and create a new HingeBlockAggregator. */
+  private def getNewAggregator(
+      instances: Array[Instance],
+      coefficients: Vector,
+      fitIntercept: Boolean): HingeBlockAggregator = {
+    val (featuresSummarizer, _) =
+      Summarizer.getClassificationSummarizers(sc.parallelize(instances))
+    val featuresStd = featuresSummarizer.std.toArray
+    val featuresMean = featuresSummarizer.mean.toArray
+    val inverseStd = featuresStd.map(std => if (std != 0) 1.0 / std else 0.0)
+    val scaledMean = inverseStd.zip(featuresMean).map(t => t._1 * t._2)
+    val bcInverseStd = sc.broadcast(inverseStd)
+    val bcScaledMean = sc.broadcast(scaledMean)
+    val bcCoefficients = sc.broadcast(coefficients)
+    new HingeBlockAggregator(bcInverseStd, bcScaledMean, fitIntercept)(bcCoefficients)
+  }
+
+  test("sparse coefficients") {
+    val bcInverseStd = sc.broadcast(Array(1.0))
+    val bcScaledMean = sc.broadcast(Array(2.0))
+    val bcCoefficients = sc.broadcast(Vectors.sparse(2, Array(0), Array(1.0)))
+    val binaryAgg = new HingeBlockAggregator(bcInverseStd, bcScaledMean,
+      fitIntercept = false)(bcCoefficients)
+    val block = InstanceBlock.fromInstances(Seq(Instance(1.0, 1.0, Vectors.dense(1.0))))
+    val thrownBinary = withClue("aggregator cannot handle sparse coefficients") {
+      intercept[IllegalArgumentException] {
+        binaryAgg.add(block)
+      }
+    }
+    assert(thrownBinary.getMessage.contains("coefficients only supports dense"))
+  }
+
+  test("aggregator add method input size") {
+    val coefArray = Array(1.0, 2.0)
+    val interceptValue = 4.0
+    val agg = getNewAggregator(instances, Vectors.dense(coefArray :+ interceptValue),
+      fitIntercept = true)
+    val block = InstanceBlock.fromInstances(Seq(Instance(1.0, 1.0, Vectors.dense(2.0))))
+    withClue("BinaryLogisticBlockAggregator features dimension must match coefficients dimension") {
+      intercept[IllegalArgumentException] {
+        agg.add(block)
+      }
+    }
+  }
+
+  test("negative weight") {
+    val coefArray = Array(1.0, 2.0)
+    val interceptValue = 4.0
+    val agg = getNewAggregator(instances, Vectors.dense(coefArray :+ interceptValue),
+      fitIntercept = true)
+    val block = InstanceBlock.fromInstances(Seq(Instance(1.0, -1.0, Vectors.dense(2.0, 1.0))))
+    withClue("BinaryLogisticBlockAggregator does not support negative instance weights") {
+      intercept[IllegalArgumentException] {
+        agg.add(block)
+      }
+    }
+  }
+
+  test("check sizes") {
+    val rng = new scala.util.Random
+    val numFeatures = instances.head.features.size
+    val coefWithIntercept = Vectors.dense(Array.fill(numFeatures + 1)(rng.nextDouble))
+    val coefWithoutIntercept = Vectors.dense(Array.fill(numFeatures)(rng.nextDouble))
+    val block = InstanceBlock.fromInstances(instances)
+
+    val aggIntercept = getNewAggregator(instances, coefWithIntercept, fitIntercept = true)
+    aggIntercept.add(block)
+    assert(aggIntercept.gradient.size === numFeatures + 1)
+
+    val aggNoIntercept = getNewAggregator(instances, coefWithoutIntercept, fitIntercept = false)
+    aggNoIntercept.add(block)
+    assert(aggNoIntercept.gradient.size === numFeatures)
+  }
+
+  test("check correctness: fitIntercept = false") {
+    val coefVec = Vectors.dense(1.0, 2.0)
+    val numFeatures = instances.head.features.size
+    val (featuresSummarizer, _) =
+      Summarizer.getClassificationSummarizers(sc.parallelize(instances))
+    val featuresStd = featuresSummarizer.std
+    val stdCoefVec = Vectors.dense(Array.tabulate(coefVec.size)(i => coefVec(i) / featuresStd(i)))
+    val weightSum = instances.map(_.weight).sum
+
+    // compute the loss and the gradients
+    var lossSum = 0.0
+    val gradientCoef = Array.ofDim[Double](numFeatures)
+    instances.foreach { case Instance(l, w, f) =>
+      val margin = BLAS.dot(stdCoefVec, f)
+      val labelScaled = 2 * l - 1.0
+      if (1.0 > labelScaled * margin) {
+        lossSum += (1.0 - labelScaled * margin) * w
+        gradientCoef.indices.foreach { i =>
+          gradientCoef(i) += f(i) * -(2 * l - 1.0) * w / featuresStd(i)
+        }
+      }
+    }
+    val loss = lossSum / weightSum
+    val gradient = Vectors.dense(gradientCoef.map(_ / weightSum))
+
+    Seq(1, 2, 4).foreach { blockSize =>
+      val blocks1 = scaledInstances
+        .grouped(blockSize)
+        .map(seq => InstanceBlock.fromInstances(seq))
+        .toArray
+      val blocks2 = blocks1.map { block =>
+        new InstanceBlock(block.labels, block.weights, block.matrix.toSparseRowMajor)
+      }
+
+      Seq(blocks1, blocks2).foreach { blocks =>
+        val agg = getNewAggregator(instances, coefVec, fitIntercept = false)
+        blocks.foreach(agg.add)
+        assert(agg.loss ~== loss relTol 1e-9)
+        assert(agg.gradient ~== gradient relTol 1e-9)
+      }
+    }
+  }
+
+  test("check correctness: fitIntercept = true") {
+    val coefVec = Vectors.dense(1.0, 2.0)
+    val interceptValue = 1.0
+    val numFeatures = instances.head.features.size
+    val (featuresSummarizer, _) =
+      Summarizer.getClassificationSummarizers(sc.parallelize(instances))
+    val featuresStd = featuresSummarizer.std
+    val featuresMean = featuresSummarizer.mean
+    val stdCoefVec = Vectors.dense(Array.tabulate(coefVec.size)(i => coefVec(i) / featuresStd(i)))
+    val weightSum = instances.map(_.weight).sum
+
+    // compute the loss and the gradients
+    var lossSum = 0.0
+    val gradientCoef = Array.ofDim[Double](numFeatures)
+    var gradientIntercept = 0.0
+    instances.foreach { case Instance(l, w, f) =>
+      val centered = f.toDense.copy
+      BLAS.axpy(-1.0, featuresMean, centered)
+      val margin = BLAS.dot(stdCoefVec, centered) + interceptValue
+      val labelScaled = 2 * l - 1.0
+      if (1.0 > labelScaled * margin) {
+        lossSum += (1.0 - labelScaled * margin) * w
+        gradientCoef.indices.foreach { i =>
+          gradientCoef(i) += (f(i) - featuresMean(i)) * -(2 * l - 1.0) * w / featuresStd(i)
+        }
+        gradientIntercept += -(2 * l - 1.0) * w
+      }
+    }
+    val loss = lossSum / weightSum
+    val gradient = Vectors.dense((gradientCoef :+ gradientIntercept).map(_ / weightSum))
+
+    Seq(1, 2, 4).foreach { blockSize =>
+      val blocks1 = scaledInstances
+        .grouped(blockSize)
+        .map(seq => InstanceBlock.fromInstances(seq))
+        .toArray
+      val blocks2 = blocks1.map { block =>
+        new InstanceBlock(block.labels, block.weights, block.matrix.toSparseRowMajor)
+      }
+
+      Seq(blocks1, blocks2).foreach { blocks =>
+        val agg = getNewAggregator(instances, Vectors.dense(coefVec.toArray :+ interceptValue),
+          fitIntercept = true)
+        blocks.foreach(agg.add)
+        assert(agg.loss ~== loss relTol 1e-9)
+        assert(agg.gradient ~== gradient relTol 1e-9)
+      }
+    }
+  }
+
+  test("check with zero standard deviation") {
+    val coefArray = Array(1.0, 2.0)
+    val coefArrayFiltered = Array(2.0)
+    val interceptValue = 1.0
+
+    Seq(false, true).foreach { fitIntercept =>
+      val coefVec = if (fitIntercept) {
+        Vectors.dense(coefArray :+ interceptValue)
+      } else {
+        Vectors.dense(coefArray)
+      }
+      val aggConstantFeature = getNewAggregator(instancesConstantFeature,
+        coefVec, fitIntercept = fitIntercept)
+      aggConstantFeature
+        .add(InstanceBlock.fromInstances(standardize(instancesConstantFeature)))
+      val grad = aggConstantFeature.gradient
+
+      val coefVecFiltered = if (fitIntercept) {
+        Vectors.dense(coefArrayFiltered :+ interceptValue)
+      } else {
+        Vectors.dense(coefArrayFiltered)
+      }
+      val aggConstantFeatureFiltered = getNewAggregator(instancesConstantFeatureFiltered,
+        coefVecFiltered, fitIntercept = fitIntercept)
+      aggConstantFeatureFiltered
+        .add(InstanceBlock.fromInstances(standardize(instancesConstantFeatureFiltered)))
+      val gradFiltered = aggConstantFeatureFiltered.gradient
+
+      // constant features should not affect gradient
+      assert(aggConstantFeature.loss ~== aggConstantFeatureFiltered.loss relTol 1e-9)
+      assert(grad(0) === 0)
+      assert(grad(1) ~== gradFiltered(0) relTol 1e-9)
+      if (fitIntercept) {
+        assert(grad.toArray.last ~== gradFiltered.toArray.last relTol 1e-9)
+      }
+    }
+  }
+}
diff --git a/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/LogisticAggregatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/LogisticAggregatorSuite.scala
deleted file mode 100644
index e3e39c691b8a3..0000000000000
--- a/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/LogisticAggregatorSuite.scala
+++ /dev/null
@@ -1,333 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.ml.optim.aggregator
-
-import org.apache.spark.SparkFunSuite
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
-import org.apache.spark.ml.linalg.{BLAS, Matrices, Vector, Vectors}
-import org.apache.spark.ml.stat.Summarizer
-import org.apache.spark.ml.util.TestingUtils._
-import org.apache.spark.mllib.util.MLlibTestSparkContext
-
-class LogisticAggregatorSuite extends SparkFunSuite with MLlibTestSparkContext {
-
-  @transient var instances: Array[Instance] = _
-  @transient var instancesConstantFeature: Array[Instance] = _
-  @transient var instancesConstantFeatureFiltered: Array[Instance] = _
-  @transient var standardizedInstances: Array[Instance] = _
-
-  override def beforeAll(): Unit = {
-    super.beforeAll()
-    instances = Array(
-      Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
-      Instance(1.0, 0.5, Vectors.dense(1.5, 1.0)),
-      Instance(2.0, 0.3, Vectors.dense(4.0, 0.5))
-    )
-    instancesConstantFeature = Array(
-      Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
-      Instance(1.0, 0.5, Vectors.dense(1.0, 1.0)),
-      Instance(2.0, 0.3, Vectors.dense(1.0, 0.5))
-    )
-    instancesConstantFeatureFiltered = Array(
-      Instance(0.0, 0.1, Vectors.dense(2.0)),
-      Instance(1.0, 0.5, Vectors.dense(1.0)),
-      Instance(2.0, 0.3, Vectors.dense(0.5))
-    )
-    standardizedInstances = standardize(instances)
-  }
-
-  /** Get summary statistics for some data and create a new LogisticAggregator. */
-  private def getNewAggregator(
-      instances: Array[Instance],
-      coefficients: Vector,
-      fitIntercept: Boolean,
-      isMultinomial: Boolean): LogisticAggregator = {
-    val (featuresSummarizer, ySummarizer) =
-      Summarizer.getClassificationSummarizers(sc.parallelize(instances))
-    val numClasses = ySummarizer.histogram.length
-    val featuresStd = featuresSummarizer.std.toArray
-    val bcFeaturesStd = spark.sparkContext.broadcast(featuresStd)
-    val bcCoefficients = spark.sparkContext.broadcast(coefficients)
-    new LogisticAggregator(bcFeaturesStd, numClasses, fitIntercept, isMultinomial)(bcCoefficients)
-  }
-
-  /** Get summary statistics for some data and create a new BlockHingeAggregator. */
-  private def getNewBlockAggregator(
-      instances: Array[Instance],
-      coefficients: Vector,
-      fitIntercept: Boolean,
-      multinomial: Boolean): BlockLogisticAggregator = {
-    val (_, ySummarizer) =
-      Summarizer.getClassificationSummarizers(sc.parallelize(instances))
-    val numFeatures = instances.head.features.size
-    val numClasses = ySummarizer.histogram.length
-    val bcCoefficients = spark.sparkContext.broadcast(coefficients)
-    new BlockLogisticAggregator(numFeatures, numClasses, fitIntercept, multinomial)(bcCoefficients)
-  }
-
-  test("aggregator add method input size") {
-    val coefArray = Array(1.0, 2.0, -2.0, 3.0, 0.0, -1.0)
-    val interceptArray = Array(4.0, 2.0, -3.0)
-    val agg = getNewAggregator(instances, Vectors.dense(coefArray ++ interceptArray),
-      fitIntercept = true, isMultinomial = true)
-    withClue("LogisticAggregator features dimension must match coefficients dimension") {
-      intercept[IllegalArgumentException] {
-        agg.add(Instance(1.0, 1.0, Vectors.dense(2.0)))
-      }
-    }
-  }
-
-  test("negative weight") {
-    val coefArray = Array(1.0, 2.0, -2.0, 3.0, 0.0, -1.0)
-    val interceptArray = Array(4.0, 2.0, -3.0)
-    val agg = getNewAggregator(instances, Vectors.dense(coefArray ++ interceptArray),
-      fitIntercept = true, isMultinomial = true)
-    withClue("LogisticAggregator does not support negative instance weights") {
-      intercept[IllegalArgumentException] {
-        agg.add(Instance(1.0, -1.0, Vectors.dense(2.0, 1.0)))
-      }
-    }
-  }
-
-  test("check sizes multinomial") {
-    val rng = new scala.util.Random
-    val numFeatures = instances.head.features.size
-    val numClasses = instances.map(_.label).toSet.size
-    val coefWithIntercept = Vectors.dense(
-      Array.fill(numClasses * (numFeatures + 1))(rng.nextDouble))
-    val coefWithoutIntercept = Vectors.dense(
-      Array.fill(numClasses * numFeatures)(rng.nextDouble))
-    val aggIntercept = getNewAggregator(instances, coefWithIntercept, fitIntercept = true,
-      isMultinomial = true)
-    val aggNoIntercept = getNewAggregator(instances, coefWithoutIntercept, fitIntercept = false,
-      isMultinomial = true)
-    instances.foreach(aggIntercept.add)
-    instances.foreach(aggNoIntercept.add)
-
-    assert(aggIntercept.gradient.size === (numFeatures + 1) * numClasses)
-    assert(aggNoIntercept.gradient.size === numFeatures * numClasses)
-  }
-
-  test("check sizes binomial") {
-    val rng = new scala.util.Random
-    val binaryInstances = instances.filter(_.label < 2.0)
-    val numFeatures = binaryInstances.head.features.size
-    val coefWithIntercept = Vectors.dense(Array.fill(numFeatures + 1)(rng.nextDouble))
-    val coefWithoutIntercept = Vectors.dense(Array.fill(numFeatures)(rng.nextDouble))
-    val aggIntercept = getNewAggregator(binaryInstances, coefWithIntercept, fitIntercept = true,
-      isMultinomial = false)
-    val aggNoIntercept = getNewAggregator(binaryInstances, coefWithoutIntercept,
-      fitIntercept = false, isMultinomial = false)
-    binaryInstances.foreach(aggIntercept.add)
-    binaryInstances.foreach(aggNoIntercept.add)
-
-    assert(aggIntercept.gradient.size === numFeatures + 1)
-    assert(aggNoIntercept.gradient.size === numFeatures)
-  }
-
-  test("check correctness multinomial") {
-    /*
-    Check that the aggregator computes loss/gradient for:
-      -sum_i w_i * (beta_y dot x_i - log(sum_k e^(beta_k dot x_i)))
-     */
-    val coefArray = Array(1.0, 2.0, -2.0, 3.0, 0.0, -1.0)
-    val interceptArray = Array(4.0, 2.0, -3.0)
-    val numFeatures = instances.head.features.size
-    val numClasses = instances.map(_.label).toSet.size
-    val intercepts = Vectors.dense(interceptArray)
-    val (featuresSummarizer, ySummarizer) =
-      Summarizer.getClassificationSummarizers(sc.parallelize(instances))
-    val featuresStd = featuresSummarizer.std.toArray
-    val weightSum = instances.map(_.weight).sum
-
-    val agg = getNewAggregator(instances, Vectors.dense(coefArray ++ interceptArray),
-      fitIntercept = true, isMultinomial = true)
-    instances.foreach(agg.add)
-
-    // compute the loss
-    val stdCoef = coefArray.indices.map(i => coefArray(i) / featuresStd(i / numClasses)).toArray
-    val linearPredictors = instances.map { case Instance(l, w, f) =>
-      val result = intercepts.copy.toDense
-      BLAS.gemv(1.0, Matrices.dense(numClasses, numFeatures, stdCoef), f, 1.0, result)
-      (l, w, result)
-    }
-
-    // sum_i w * beta_k dot x_i
-    val sumLinear = linearPredictors.map { case (l, w, p) =>
-      w * p(l.toInt)
-    }.sum
-
-    // sum_i w * log(sum_k e^(beta_K dot x_i))
-    val sumLogs = linearPredictors.map { case (l, w, p) =>
-      w * math.log(p.values.map(math.exp).sum)
-    }.sum
-    val loss = (sumLogs - sumLinear) / weightSum
-
-
-    // compute the gradients
-    val gradientCoef = new Array[Double](numFeatures * numClasses)
-    val gradientIntercept = new Array[Double](numClasses)
-    instances.foreach { case Instance(l, w, f) =>
-      val margin = intercepts.copy.toDense
-      BLAS.gemv(1.0, Matrices.dense(numClasses, numFeatures, stdCoef), f, 1.0, margin)
-      val sum = margin.values.map(math.exp).sum
-
-      gradientCoef.indices.foreach { i =>
-        val fStd = f(i / numClasses) / featuresStd(i / numClasses)
-        val cidx = i % numClasses
-        if (cidx == l.toInt) gradientCoef(i) -= w * fStd
-        gradientCoef(i) += w * math.exp(margin(cidx)) / sum * fStd
-      }
-
-      gradientIntercept.indices.foreach { i =>
-        val cidx = i % numClasses
-        if (cidx == l.toInt) gradientIntercept(i) -= w
-        gradientIntercept(i) += w * math.exp(margin(cidx)) / sum
-      }
-    }
-    val gradient = Vectors.dense((gradientCoef ++ gradientIntercept).map(_ / weightSum))
-
-    assert(loss ~== agg.loss relTol 0.01)
-    assert(gradient ~== agg.gradient relTol 0.01)
-
-    Seq(1, 2, 4).foreach { blockSize =>
-      val blocks1 = standardizedInstances
-        .grouped(blockSize)
-        .map(seq => InstanceBlock.fromInstances(seq))
-        .toArray
-      val blocks2 = blocks1.map { block =>
-        new InstanceBlock(block.labels, block.weights, block.matrix.toSparseRowMajor)
-      }
-
-      Seq(blocks1, blocks2).foreach { blocks =>
-        val blockAgg = getNewBlockAggregator(standardizedInstances,
-          Vectors.dense(coefArray ++ interceptArray), true, true)
-        blocks.foreach(blockAgg.add)
-        assert(agg.loss ~== blockAgg.loss relTol 1e-9)
-        assert(agg.gradient ~== blockAgg.gradient relTol 1e-9)
-      }
-    }
-  }
-
-  test("check correctness binomial") {
-    /*
-    Check that the aggregator computes loss/gradient for:
-      -sum_i y_i * log(1 / (1 + e^(-beta dot x_i)) + (1 - y_i) * log(1 - 1 / (1 + e^(-beta dot x_i))
-     */
-    val binaryInstances = instances.map { instance =>
-      if (instance.label <= 1.0) instance else Instance(0.0, instance.weight, instance.features)
-    }
-    val coefArray = Array(1.0, 2.0)
-    val intercept = 1.0
-    val numFeatures = binaryInstances.head.features.size
-    val (featuresSummarizer, _) =
-      Summarizer.getClassificationSummarizers(sc.parallelize(binaryInstances))
-    val featuresStd = featuresSummarizer.std.toArray
-    val weightSum = binaryInstances.map(_.weight).sum
-
-    val agg = getNewAggregator(binaryInstances, Vectors.dense(coefArray ++ Array(intercept)),
-      fitIntercept = true, isMultinomial = false)
-    binaryInstances.foreach(agg.add)
-
-    // compute the loss
-    val stdCoef = coefArray.indices.map(i => coefArray(i) / featuresStd(i)).toArray
-    val lossSum = binaryInstances.map { case Instance(l, w, f) =>
-      val margin = BLAS.dot(Vectors.dense(stdCoef), f) + intercept
-      val prob = 1.0 / (1.0 + math.exp(-margin))
-      -w * l * math.log(prob) - w * (1.0 - l) * math.log1p(-prob)
-    }.sum
-    val loss = lossSum / weightSum
-
-    // compute the gradients
-    val gradientCoef = new Array[Double](numFeatures)
-    var gradientIntercept = 0.0
-    binaryInstances.foreach { case Instance(l, w, f) =>
-      val margin = BLAS.dot(f, Vectors.dense(coefArray)) + intercept
-      gradientCoef.indices.foreach { i =>
-        gradientCoef(i) += w * (1.0 / (1.0 + math.exp(-margin)) - l) * f(i) / featuresStd(i)
-      }
-      gradientIntercept += w * (1.0 / (1.0 + math.exp(-margin)) - l)
-    }
-    val gradient = Vectors.dense((gradientCoef ++ Array(gradientIntercept)).map(_ / weightSum))
-
-    assert(loss ~== agg.loss relTol 0.01)
-    assert(gradient ~== agg.gradient relTol 0.01)
-
-    Seq(1, 2, 4).foreach { blockSize =>
-      val blocks1 = standardize(binaryInstances)
-        .grouped(blockSize)
-        .map(seq => InstanceBlock.fromInstances(seq))
-        .toArray
-      val blocks2 = blocks1.map { block =>
-        new InstanceBlock(block.labels, block.weights, block.matrix.toSparseRowMajor)
-      }
-
-      Seq(blocks1, blocks2).foreach { blocks =>
-        val blockAgg = getNewBlockAggregator(binaryInstances,
-          Vectors.dense(coefArray ++ Array(intercept)), true, false)
-        blocks.foreach(blockAgg.add)
-        assert(agg.loss ~== blockAgg.loss relTol 1e-9)
-        assert(agg.gradient ~== blockAgg.gradient relTol 1e-9)
-      }
-    }
-  }
-
-  test("check with zero standard deviation") {
-    val binaryInstances = instancesConstantFeature.map { instance =>
-      if (instance.label <= 1.0) instance else Instance(0.0, instance.weight, instance.features)
-    }
-    val binaryInstancesFiltered = instancesConstantFeatureFiltered.map { instance =>
-      if (instance.label <= 1.0) instance else Instance(0.0, instance.weight, instance.features)
-    }
-    val coefArray = Array(1.0, 2.0, -2.0, 3.0, 0.0, -1.0)
-    val coefArrayFiltered = Array(3.0, 0.0, -1.0)
-    val interceptArray = Array(4.0, 2.0, -3.0)
-    val aggConstantFeature = getNewAggregator(instancesConstantFeature,
-      Vectors.dense(coefArray ++ interceptArray), fitIntercept = true, isMultinomial = true)
-    val aggConstantFeatureFiltered = getNewAggregator(instancesConstantFeatureFiltered,
-      Vectors.dense(coefArrayFiltered ++ interceptArray), fitIntercept = true, isMultinomial = true)
-
-    instancesConstantFeature.foreach(aggConstantFeature.add)
-    instancesConstantFeatureFiltered.foreach(aggConstantFeatureFiltered.add)
-
-    // constant features should not affect gradient
-    def validateGradient(grad: Vector, gradFiltered: Vector, numCoefficientSets: Int): Unit = {
-      for (i <- 0 until numCoefficientSets) {
-        assert(grad(i) === 0.0)
-        assert(grad(numCoefficientSets + i) == gradFiltered(i))
-      }
-    }
-
-    validateGradient(aggConstantFeature.gradient, aggConstantFeatureFiltered.gradient, 3)
-
-    val binaryCoefArray = Array(1.0, 2.0)
-    val binaryCoefArrayFiltered = Array(2.0)
-    val intercept = 1.0
-    val aggConstantFeatureBinary = getNewAggregator(binaryInstances,
-      Vectors.dense(binaryCoefArray ++ Array(intercept)), fitIntercept = true,
-      isMultinomial = false)
-    val aggConstantFeatureBinaryFiltered = getNewAggregator(binaryInstancesFiltered,
-      Vectors.dense(binaryCoefArrayFiltered ++ Array(intercept)), fitIntercept = true,
-      isMultinomial = false)
-    binaryInstances.foreach(aggConstantFeatureBinary.add)
-    binaryInstancesFiltered.foreach(aggConstantFeatureBinaryFiltered.add)
-
-    // constant features should not affect gradient
-    validateGradient(aggConstantFeatureBinary.gradient,
-      aggConstantFeatureBinaryFiltered.gradient, 1)
-  }
-}
diff --git a/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/MultinomialLogisticBlockAggregatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/MultinomialLogisticBlockAggregatorSuite.scala
new file mode 100644
index 0000000000000..d00fdaca15828
--- /dev/null
+++ b/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/MultinomialLogisticBlockAggregatorSuite.scala
@@ -0,0 +1,387 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.ml.optim.aggregator
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.ml.feature.{Instance, InstanceBlock}
+import org.apache.spark.ml.linalg._
+import org.apache.spark.ml.stat.Summarizer
+import org.apache.spark.ml.util.TestingUtils._
+import org.apache.spark.mllib.util.MLlibTestSparkContext
+
+class MultinomialLogisticBlockAggregatorSuite extends SparkFunSuite with MLlibTestSparkContext {
+
+  @transient var instances: Array[Instance] = _
+  @transient var instancesConstantFeature: Array[Instance] = _
+  @transient var instancesConstantFeatureFiltered: Array[Instance] = _
+  @transient var scaledInstances: Array[Instance] = _
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    instances = Array(
+      Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
+      Instance(1.0, 0.5, Vectors.dense(1.5, 1.0)),
+      Instance(2.0, 0.3, Vectors.dense(4.0, 0.5))
+    )
+    instancesConstantFeature = Array(
+      Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
+      Instance(1.0, 0.5, Vectors.dense(1.0, 1.0)),
+      Instance(2.0, 0.3, Vectors.dense(1.0, 0.5))
+    )
+    instancesConstantFeatureFiltered = Array(
+      Instance(0.0, 0.1, Vectors.dense(2.0)),
+      Instance(1.0, 0.5, Vectors.dense(1.0)),
+      Instance(2.0, 0.3, Vectors.dense(0.5))
+    )
+    scaledInstances = standardize(instances)
+  }
+
+  /** Get summary statistics for some data and create a new MultinomialLogisticBlockAggregator. */
+  private def getNewAggregator(
+      instances: Array[Instance],
+      coefficients: Vector,
+      fitIntercept: Boolean,
+      fitWithMean: Boolean): MultinomialLogisticBlockAggregator = {
+    val (featuresSummarizer, _) =
+      Summarizer.getClassificationSummarizers(sc.parallelize(instances))
+    val featuresStd = featuresSummarizer.std.toArray
+    val featuresMean = featuresSummarizer.mean.toArray
+    val inverseStd = featuresStd.map(std => if (std != 0) 1.0 / std else 0.0)
+    val scaledMean = inverseStd.zip(featuresMean).map(t => t._1 * t._2)
+    val bcInverseStd = sc.broadcast(inverseStd)
+    val bcScaledMean = sc.broadcast(scaledMean)
+    val bcCoefficients = sc.broadcast(coefficients)
+    new MultinomialLogisticBlockAggregator(bcInverseStd,
+      bcScaledMean, fitIntercept, fitWithMean)(bcCoefficients)
+  }
+
+  test("sparse coefficients") {
+    val bcInverseStd = sc.broadcast(Array(1.0))
+    val bcScaledMean = sc.broadcast(Array(2.0))
+    val bcCoefficients = sc.broadcast(Vectors.sparse(4, Array(0), Array(1.0)))
+    val binaryAgg = new MultinomialLogisticBlockAggregator(bcInverseStd, bcScaledMean,
+      fitIntercept = true, fitWithMean = false)(bcCoefficients)
+    val block = InstanceBlock.fromInstances(Seq(Instance(1.0, 1.0, Vectors.dense(1.0))))
+    val thrownBinary = withClue("aggregator cannot handle sparse coefficients") {
+      intercept[IllegalArgumentException] {
+        binaryAgg.add(block)
+      }
+    }
+    assert(thrownBinary.getMessage.contains("coefficients only supports dense"))
+  }
+
+  test("aggregator add method input size") {
+    val coefArray = Array(1.0, 2.0, 3.0, 4.0, 5.0, 6.0)
+    val interceptArray = Array(7.0, 8.0, 9.0)
+    val agg = getNewAggregator(instances, Vectors.dense(coefArray ++ interceptArray),
+      fitIntercept = true, fitWithMean = true)
+    val block = InstanceBlock.fromInstances(Seq(Instance(1.0, 1.0, Vectors.dense(2.0))))
+    withClue("BinaryLogisticBlockAggregator features dimension must match coefficients dimension") {
+      intercept[IllegalArgumentException] {
+        agg.add(block)
+      }
+    }
+  }
+
+  test("negative weight") {
+    val coefArray = Array(1.0, 2.0, 3.0, 4.0, 5.0, 6.0)
+    val interceptArray = Array(7.0, 8.0, 9.0)
+    val agg = getNewAggregator(instances, Vectors.dense(coefArray ++ interceptArray),
+      fitIntercept = true, fitWithMean = true)
+    val block = InstanceBlock.fromInstances(Seq(Instance(1.0, -1.0, Vectors.dense(2.0, 1.0))))
+    withClue("BinaryLogisticBlockAggregator does not support negative instance weights") {
+      intercept[IllegalArgumentException] {
+        agg.add(block)
+      }
+    }
+  }
+
+  test("check sizes") {
+    val rng = new scala.util.Random
+    val numFeatures = instances.head.features.size
+    val numClasses = instances.map(_.label).distinct.size
+    val coefWithIntercept = Vectors.dense(
+      Array.fill(numClasses * (numFeatures + 1))(rng.nextDouble))
+    val coefWithoutIntercept = Vectors.dense(
+      Array.fill(numClasses * numFeatures)(rng.nextDouble))
+    val block = InstanceBlock.fromInstances(instances)
+
+    val aggIntercept = getNewAggregator(instances, coefWithIntercept,
+      fitIntercept = true, fitWithMean = false)
+    aggIntercept.add(block)
+    assert(aggIntercept.gradient.size === (numFeatures + 1) * numClasses)
+
+    val aggNoIntercept = getNewAggregator(instances, coefWithoutIntercept,
+      fitIntercept = false, fitWithMean = false)
+    aggNoIntercept.add(block)
+    assert(aggNoIntercept.gradient.size === numFeatures * numClasses)
+  }
+
+  test("check correctness: fitIntercept = false") {
+    val coefArray = Array(1.0, 2.0, -2.0, 3.0, 0.0, -1.0)
+    val numFeatures = instances.head.features.size
+    val numClasses = instances.map(_.label).toSet.size
+    val (featuresSummarizer, _) =
+      Summarizer.getClassificationSummarizers(sc.parallelize(instances))
+    val featuresStd = featuresSummarizer.std
+    val stdCoefMat = Matrices.dense(numClasses, numFeatures,
+      Array.tabulate(coefArray.size)(i => coefArray(i) / featuresStd(i / numClasses)))
+    val weightSum = instances.map(_.weight).sum
+
+    // compute the loss
+    val linearPredictors = instances.map { case Instance(l, w, f) =>
+      val result = new DenseVector(Array.ofDim[Double](numClasses))
+      BLAS.gemv(1.0, stdCoefMat, f, 1.0, result)
+      (l, w, result)
+    }
+
+    // sum_i w * beta_k dot x_i
+    val sumLinear = linearPredictors.map { case (l, w, p) =>
+      w * p(l.toInt)
+    }.sum
+
+    // sum_i w * log(sum_k e^(beta_K dot x_i))
+    val sumLogs = linearPredictors.map { case (l, w, p) =>
+      w * math.log(p.values.map(math.exp).sum)
+    }.sum
+    val loss = (sumLogs - sumLinear) / weightSum
+
+    // compute the gradients
+    val gradientCoef = new Array[Double](numFeatures * numClasses)
+    instances.foreach { case Instance(l, w, f) =>
+      val margin = new DenseVector(Array.ofDim[Double](numClasses))
+      BLAS.gemv(1.0, stdCoefMat, f, 1.0, margin)
+      val sum = margin.values.map(math.exp).sum
+
+      gradientCoef.indices.foreach { i =>
+        val fStd = f(i / numClasses) / featuresStd(i / numClasses)
+        val cidx = i % numClasses
+        if (cidx == l.toInt) gradientCoef(i) -= w * fStd
+        gradientCoef(i) += w * math.exp(margin(cidx)) / sum * fStd
+      }
+    }
+    val gradient = Vectors.dense((gradientCoef).map(_ / weightSum))
+
+    Seq(1, 2, 4).foreach { blockSize =>
+      val blocks1 = scaledInstances
+        .grouped(blockSize)
+        .map(seq => InstanceBlock.fromInstances(seq))
+        .toArray
+      val blocks2 = blocks1.map { block =>
+        new InstanceBlock(block.labels, block.weights, block.matrix.toSparseRowMajor)
+      }
+
+      Seq(blocks1, blocks2).foreach { blocks =>
+        val agg = getNewAggregator(instances, Vectors.dense(coefArray),
+          fitIntercept = false, fitWithMean = false)
+        blocks.foreach(agg.add)
+        assert(agg.loss ~== loss relTol 1e-9)
+        assert(agg.gradient ~== gradient relTol 1e-9)
+      }
+    }
+  }
+
+  test("check correctness: fitIntercept = true, fitWithMean = false") {
+    val coefArray = Array(1.0, 2.0, -2.0, 3.0, 0.0, -1.0)
+    val interceptArray = Array(4.0, 2.0, -3.0)
+    val numFeatures = instances.head.features.size
+    val numClasses = instances.map(_.label).toSet.size
+    val intercepts = Vectors.dense(interceptArray)
+    val (featuresSummarizer, _) =
+      Summarizer.getClassificationSummarizers(sc.parallelize(instances))
+    val featuresStd = featuresSummarizer.std
+    val stdCoefMat = Matrices.dense(numClasses, numFeatures,
+      Array.tabulate(coefArray.size)(i => coefArray(i) / featuresStd(i / numClasses)))
+    val weightSum = instances.map(_.weight).sum
+
+    // compute the loss
+    val linearPredictors = instances.map { case Instance(l, w, f) =>
+      val result = intercepts.copy.toDense
+      BLAS.gemv(1.0, stdCoefMat, f, 1.0, result)
+      (l, w, result)
+    }
+
+    // sum_i w * beta_k dot x_i
+    val sumLinear = linearPredictors.map { case (l, w, p) =>
+      w * p(l.toInt)
+    }.sum
+
+    // sum_i w * log(sum_k e^(beta_K dot x_i))
+    val sumLogs = linearPredictors.map { case (l, w, p) =>
+      w * math.log(p.values.map(math.exp).sum)
+    }.sum
+    val loss = (sumLogs - sumLinear) / weightSum
+
+    // compute the gradients
+    val gradientCoef = new Array[Double](numFeatures * numClasses)
+    val gradientIntercept = new Array[Double](numClasses)
+    instances.foreach { case Instance(l, w, f) =>
+      val margin = intercepts.copy.toDense
+      BLAS.gemv(1.0, stdCoefMat, f, 1.0, margin)
+      val sum = margin.values.map(math.exp).sum
+
+      gradientCoef.indices.foreach { i =>
+        val fStd = f(i / numClasses) / featuresStd(i / numClasses)
+        val cidx = i % numClasses
+        if (cidx == l.toInt) gradientCoef(i) -= w * fStd
+        gradientCoef(i) += w * math.exp(margin(cidx)) / sum * fStd
+      }
+
+      gradientIntercept.indices.foreach { i =>
+        val cidx = i % numClasses
+        if (cidx == l.toInt) gradientIntercept(i) -= w
+        gradientIntercept(i) += w * math.exp(margin(cidx)) / sum
+      }
+    }
+    val gradient = Vectors.dense((gradientCoef ++ gradientIntercept).map(_ / weightSum))
+
+    Seq(1, 2, 4).foreach { blockSize =>
+      val blocks1 = scaledInstances
+        .grouped(blockSize)
+        .map(seq => InstanceBlock.fromInstances(seq))
+        .toArray
+      val blocks2 = blocks1.map { block =>
+        new InstanceBlock(block.labels, block.weights, block.matrix.toSparseRowMajor)
+      }
+
+      Seq(blocks1, blocks2).foreach { blocks =>
+        val agg = getNewAggregator(instances, Vectors.dense(coefArray ++ interceptArray),
+          fitIntercept = true, fitWithMean = false)
+        blocks.foreach(agg.add)
+        assert(agg.loss ~== loss relTol 1e-9)
+        assert(agg.gradient ~== gradient relTol 1e-9)
+      }
+    }
+  }
+
+  test("check correctness: fitIntercept = true, fitWithMean = true") {
+    val coefArray = Array(1.0, 2.0, -2.0, 3.0, 0.0, -1.0)
+    val interceptArray = Array(4.0, 2.0, -3.0)
+    val numFeatures = instances.head.features.size
+    val numClasses = instances.map(_.label).toSet.size
+    val intercepts = Vectors.dense(interceptArray)
+    val (featuresSummarizer, _) =
+      Summarizer.getClassificationSummarizers(sc.parallelize(instances))
+    val featuresStd = featuresSummarizer.std
+    val featuresMean = featuresSummarizer.mean
+    val stdCoefMat = Matrices.dense(numClasses, numFeatures,
+      Array.tabulate(coefArray.size)(i => coefArray(i) / featuresStd(i / numClasses)))
+    val weightSum = instances.map(_.weight).sum
+
+    // compute the loss
+    val linearPredictors = instances.map { case Instance(l, w, f) =>
+      val centered = f.toDense.copy
+      BLAS.axpy(-1.0, featuresMean, centered)
+      val result = intercepts.copy.toDense
+      BLAS.gemv(1.0, stdCoefMat, centered, 1.0, result)
+      (l, w, result)
+    }
+
+    // sum_i w * beta_k dot x_i
+    val sumLinear = linearPredictors.map { case (l, w, p) =>
+      w * p(l.toInt)
+    }.sum
+
+    // sum_i w * log(sum_k e^(beta_K dot x_i))
+    val sumLogs = linearPredictors.map { case (l, w, p) =>
+      w * math.log(p.values.map(math.exp).sum)
+    }.sum
+    val loss = (sumLogs - sumLinear) / weightSum
+
+    // compute the gradients
+    val gradientCoef = new Array[Double](numFeatures * numClasses)
+    val gradientIntercept = new Array[Double](numClasses)
+    instances.foreach { case Instance(l, w, f) =>
+      val centered = f.toDense.copy
+      BLAS.axpy(-1.0, featuresMean, centered)
+      val margin = intercepts.copy.toDense
+      BLAS.gemv(1.0, stdCoefMat, centered, 1.0, margin)
+      val sum = margin.values.map(math.exp).sum
+
+      gradientCoef.indices.foreach { i =>
+        val fStd = centered(i / numClasses) / featuresStd(i / numClasses)
+        val cidx = i % numClasses
+        if (cidx == l.toInt) gradientCoef(i) -= w * fStd
+        gradientCoef(i) += w * math.exp(margin(cidx)) / sum * fStd
+      }
+
+      gradientIntercept.indices.foreach { i =>
+        val cidx = i % numClasses
+        if (cidx == l.toInt) gradientIntercept(i) -= w
+        gradientIntercept(i) += w * math.exp(margin(cidx)) / sum
+      }
+    }
+    val gradient = Vectors.dense((gradientCoef ++ gradientIntercept).map(_ / weightSum))
+
+    Seq(1, 2, 4).foreach { blockSize =>
+      val blocks1 = scaledInstances
+        .grouped(blockSize)
+        .map(seq => InstanceBlock.fromInstances(seq))
+        .toArray
+      val blocks2 = blocks1.map { block =>
+        new InstanceBlock(block.labels, block.weights, block.matrix.toSparseRowMajor)
+      }
+
+      Seq(blocks1, blocks2).foreach { blocks =>
+        val agg = getNewAggregator(instances, Vectors.dense(coefArray ++ interceptArray),
+          fitIntercept = true, fitWithMean = true)
+        blocks.foreach(agg.add)
+        assert(agg.loss ~== loss relTol 1e-9)
+        assert(agg.gradient ~== gradient relTol 1e-9)
+      }
+    }
+  }
+
+  test("check with zero standard deviation") {
+    val coefArray = Array(1.0, 2.0, -2.0, 3.0, 0.0, -1.0)
+    val coefArrayFiltered = Array(3.0, 0.0, -1.0)
+    val interceptArray = Array(4.0, 2.0, -3.0)
+
+    Seq((false, false), (true, false), (true, true)).foreach { case (fitIntercept, fitWithMean) =>
+      val coefVec = if (fitIntercept) {
+        Vectors.dense(coefArray ++ interceptArray)
+      } else {
+        Vectors.dense(coefArray)
+      }
+      val aggConstantFeature = getNewAggregator(instancesConstantFeature,
+        coefVec, fitIntercept = fitIntercept, fitWithMean = fitWithMean)
+      aggConstantFeature
+        .add(InstanceBlock.fromInstances(standardize(instancesConstantFeature)))
+      val grad = aggConstantFeature.gradient
+
+      val coefVecFiltered = if (fitIntercept) {
+        Vectors.dense(coefArrayFiltered ++ interceptArray)
+      } else {
+        Vectors.dense(coefArrayFiltered)
+      }
+      val aggConstantFeatureFiltered = getNewAggregator(instancesConstantFeatureFiltered,
+        coefVecFiltered, fitIntercept = fitIntercept, fitWithMean = fitWithMean)
+      aggConstantFeatureFiltered
+        .add(InstanceBlock.fromInstances(standardize(instancesConstantFeatureFiltered)))
+      val gradFiltered = aggConstantFeatureFiltered.gradient
+
+      // constant features should not affect gradient
+      assert(Vectors.dense(grad.toArray.take(3)) === Vectors.zeros(3))
+      assert(Vectors.dense(grad.toArray.slice(3, 6)) ~==
+        Vectors.dense(gradFiltered.toArray.take(3)) relTol 1e-9)
+      if (fitIntercept) {
+        assert(Vectors.dense(grad.toArray.takeRight(3)) ~==
+          Vectors.dense(gradFiltered.toArray.takeRight(3)) relTol 1e-9)
+      }
+    }
+  }
+}
diff --git a/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
index 9029fc96b36a8..cebd8cac057f1 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
@@ -24,14 +24,13 @@ import scala.collection.JavaConverters._
 import scala.collection.mutable
 import scala.collection.mutable.{ArrayBuffer, WrappedArray}
 
-import com.github.fommil.netlib.BLAS.{getInstance => blas}
 import org.apache.commons.io.FileUtils
 import org.apache.commons.io.filefilter.TrueFileFilter
 import org.scalatest.BeforeAndAfterEach
 
 import org.apache.spark._
 import org.apache.spark.internal.Logging
-import org.apache.spark.ml.linalg.Vectors
+import org.apache.spark.ml.linalg.{BLAS, Vectors}
 import org.apache.spark.ml.recommendation.ALS._
 import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTest, MLTestingUtils}
 import org.apache.spark.ml.util.TestingUtils._
@@ -296,7 +295,7 @@ class ALSSuite extends MLTest with DefaultReadWriteTest with Logging {
     for ((userId, userFactor) <- userFactors; (itemId, itemFactor) <- itemFactors) {
       val x = random.nextDouble()
       if (x < totalFraction) {
-        val rating = blas.sdot(rank, userFactor, 1, itemFactor, 1)
+        val rating = BLAS.nativeBLAS.sdot(rank, userFactor, 1, itemFactor, 1)
         if (x < trainingFraction) {
           val noise = noiseStd * random.nextGaussian()
           training += Rating(userId, itemId, rating + noise.toFloat)
@@ -307,7 +306,7 @@ class ALSSuite extends MLTest with DefaultReadWriteTest with Logging {
     }
     logInfo(s"Generated an explicit feedback dataset with ${training.size} ratings for training " +
       s"and ${test.size} for test.")
-    (sc.parallelize(training, 2), sc.parallelize(test, 2))
+    (sc.parallelize(training.toSeq, 2), sc.parallelize(test.toSeq, 2))
   }
 
   /**
@@ -810,7 +809,7 @@ class ALSSuite extends MLTest with DefaultReadWriteTest with Logging {
       val topItems = model.recommendForAllUsers(k)
       assert(topItems.count() == numUsers)
       assert(topItems.columns.contains("user"))
-      checkRecommendations(topItems, expectedUpToN, "item")
+      checkRecommendations(topItems, expectedUpToN.toMap, "item")
     }
   }
 
@@ -831,7 +830,7 @@ class ALSSuite extends MLTest with DefaultReadWriteTest with Logging {
       val topUsers = getALSModel.recommendForAllItems(k)
       assert(topUsers.count() == numItems)
       assert(topUsers.columns.contains("item"))
-      checkRecommendations(topUsers, expectedUpToN, "user")
+      checkRecommendations(topUsers, expectedUpToN.toMap, "user")
     }
   }
 
@@ -853,7 +852,7 @@ class ALSSuite extends MLTest with DefaultReadWriteTest with Logging {
       val topItems = model.recommendForUserSubset(userSubset, k)
       assert(topItems.count() == numUsersSubset)
       assert(topItems.columns.contains("user"))
-      checkRecommendations(topItems, expectedUpToN, "item")
+      checkRecommendations(topItems, expectedUpToN.toMap, "item")
     }
   }
 
@@ -875,7 +874,7 @@ class ALSSuite extends MLTest with DefaultReadWriteTest with Logging {
       val topUsers = model.recommendForItemSubset(itemSubset, k)
       assert(topUsers.count() == numItemsSubset)
       assert(topUsers.columns.contains("item"))
-      checkRecommendations(topUsers, expectedUpToN, "user")
+      checkRecommendations(topUsers, expectedUpToN.toMap, "user")
     }
   }
 
@@ -1194,7 +1193,7 @@ object ALSSuite extends Logging {
     val training = ArrayBuffer.empty[Rating[Int]]
     val test = ArrayBuffer.empty[Rating[Int]]
     for ((userId, userFactor) <- userFactors; (itemId, itemFactor) <- itemFactors) {
-      val rating = blas.sdot(rank, userFactor, 1, itemFactor, 1)
+      val rating = BLAS.nativeBLAS.sdot(rank, userFactor, 1, itemFactor, 1)
       val threshold = if (rating > 0) positiveFraction else negativeFraction
       val observed = random.nextDouble() < threshold
       if (observed) {
@@ -1211,6 +1210,6 @@ object ALSSuite extends Logging {
     }
     logInfo(s"Generated an implicit feedback dataset with ${training.size} ratings for training " +
       s"and ${test.size} for test.")
-    (sc.parallelize(training, 2), sc.parallelize(test, 2))
+    (sc.parallelize(training.toSeq, 2), sc.parallelize(test.toSeq, 2))
   }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/regression/AFTSurvivalRegressionSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/regression/AFTSurvivalRegressionSuite.scala
index a66143ab12e49..e745e7f67df98 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/regression/AFTSurvivalRegressionSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/regression/AFTSurvivalRegressionSuite.scala
@@ -130,9 +130,9 @@ class AFTSurvivalRegressionSuite extends MLTest with DefaultReadWriteTest {
   test("aft survival regression with univariate") {
     val quantileProbabilities = Array(0.1, 0.5, 0.9)
     val trainer = new AFTSurvivalRegression()
-      .setQuantileProbabilities(quantileProbabilities)
       .setQuantilesCol("quantiles")
     val model = trainer.fit(datasetUnivariate)
+    model.setQuantileProbabilities(quantileProbabilities)
 
     /*
        Using the following R code to load the data and train the model using survival package.
@@ -436,8 +436,8 @@ class AFTSurvivalRegressionSuite extends MLTest with DefaultReadWriteTest {
         .setQuantileProbabilities(quantileProbabilities)
         .setQuantilesCol("quantiles")
       val model = aft.fit(dataset)
-      Seq(4, 16, 64).foreach { blockSize =>
-        val model2 = aft.setBlockSize(blockSize).fit(dataset)
+      Seq(0, 0.01, 0.1, 1, 2, 4).foreach { s =>
+        val model2 = aft.setMaxBlockSizeInMB(s).fit(dataset)
         assert(model.coefficients ~== model2.coefficients relTol 1e-9)
         assert(model.intercept ~== model2.intercept relTol 1e-9)
         assert(model.scale ~== model2.scale relTol 1e-9)
diff --git a/mllib/src/test/scala/org/apache/spark/ml/regression/DecisionTreeRegressorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/regression/DecisionTreeRegressorSuite.scala
index 49ebcb385640e..9cb0345400bc4 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/regression/DecisionTreeRegressorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/regression/DecisionTreeRegressorSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.ml.regression
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.ml.feature.LabeledPoint
-import org.apache.spark.ml.linalg.Vector
+import org.apache.spark.ml.linalg.{Vector, Vectors}
 import org.apache.spark.ml.tree.impl.TreeTests
 import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTest, MLTestingUtils}
 import org.apache.spark.ml.util.TestingUtils._
@@ -236,6 +236,20 @@ class DecisionTreeRegressorSuite extends MLTest with DefaultReadWriteTest {
       TreeTests.allParamSettings ++ Map("maxDepth" -> 0),
       TreeTests.allParamSettings ++ Map("maxDepth" -> 0), checkModelData)
   }
+
+  test("SPARK-33398: Load DecisionTreeRegressionModel prior to Spark 3.0") {
+    val path = testFile("ml-models/dtr-2.4.7")
+    val model = DecisionTreeRegressionModel.load(path)
+    assert(model.numFeatures === 692)
+    assert(model.numNodes === 5)
+    assert(model.featureImportances ~==
+      Vectors.sparse(692, Array(100, 434),
+        Array(0.03987240829346093, 0.960127591706539)) absTol 1e-4)
+
+    val metadata = spark.read.json(s"$path/metadata")
+    val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
+    assert(sparkVersionStr === "2.4.7")
+  }
 }
 
 private[ml] object DecisionTreeRegressorSuite extends SparkFunSuite {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/regression/GBTRegressorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/regression/GBTRegressorSuite.scala
index 04b0d4b8470f3..7d84df6326397 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/regression/GBTRegressorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/regression/GBTRegressorSuite.scala
@@ -370,6 +370,18 @@ class GBTRegressorSuite extends MLTest with DefaultReadWriteTest {
     testEstimatorAndModelReadWrite(gbt, continuousData, allParamSettings,
       allParamSettings, checkModelData)
   }
+
+  test("SPARK-33398: Load GBTRegressionModel prior to Spark 3.0") {
+    val path = testFile("ml-models/gbtr-2.4.7")
+    val model = GBTRegressionModel.load(path)
+    assert(model.numFeatures === 692)
+    assert(model.totalNumNodes === 6)
+    assert(model.trees.map(_.numNodes) === Array(5, 1))
+
+    val metadata = spark.read.json(s"$path/metadata")
+    val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
+    assert(sparkVersionStr === "2.4.7")
+  }
 }
 
 private object GBTRegressorSuite extends SparkFunSuite {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala
index a30c47293c543..bfa9f4b59511c 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala
@@ -494,7 +494,7 @@ class GeneralizedLinearRegressionSuite extends MLTest with DefaultReadWriteTest
        [1] -0.0457441 -0.6833928
        [1] 1.8121235  -0.1747493  -0.5815417
 
-       R code for deivance calculation:
+       R code for deviance calculation:
        data = cbind(y=c(0,1,0,0,0,1), x1=c(18, 12, 15, 13, 15, 16), x2=c(1,0,0,2,1,1))
        summary(glm(y~x1+x2, family=poisson, data=data.frame(data)))$deviance
        [1] 3.70055
@@ -507,8 +507,6 @@ class GeneralizedLinearRegressionSuite extends MLTest with DefaultReadWriteTest
 
     val residualDeviancesR = Array(3.809296, 3.70055)
 
-    import GeneralizedLinearRegression._
-
     var idx = 0
     val link = "log"
     val dataset = datasetPoissonLogWithZero
@@ -790,8 +788,6 @@ class GeneralizedLinearRegressionSuite extends MLTest with DefaultReadWriteTest
     val expected = Seq(0.5108256, 0.1201443, 1.600000, 1.886792, 0.625, 0.530,
       -0.4700036, -0.6348783, 1.325782, 1.463641)
 
-    import GeneralizedLinearRegression._
-
     var idx = 0
     for (family <- GeneralizedLinearRegression.supportedFamilyNames.sortWith(_ < _)) {
       for (useWeight <- Seq(false, true)) {
@@ -1665,7 +1661,7 @@ class GeneralizedLinearRegressionSuite extends MLTest with DefaultReadWriteTest
   }
 
   test("evaluate with labels that are not doubles") {
-    // Evaulate with a dataset that contains Labels not as doubles to verify correct casting
+    // Evaluate with a dataset that contains Labels not as doubles to verify correct casting
     val dataset = Seq(
       Instance(17.0, 1.0, Vectors.dense(0.0, 5.0).toSparse),
       Instance(19.0, 1.0, Vectors.dense(1.0, 7.0)),
diff --git a/mllib/src/test/scala/org/apache/spark/ml/regression/LinearRegressionSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/regression/LinearRegressionSuite.scala
index df9a66b49fe48..b3098be0a36fb 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/regression/LinearRegressionSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/regression/LinearRegressionSuite.scala
@@ -31,6 +31,7 @@ import org.apache.spark.ml.util._
 import org.apache.spark.ml.util.TestingUtils._
 import org.apache.spark.mllib.util.LinearDataGenerator
 import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions.lit
 
 
 class LinearRegressionSuite extends MLTest with DefaultReadWriteTest with PMMLReadWriteTest {
@@ -671,8 +672,8 @@ class LinearRegressionSuite extends MLTest with DefaultReadWriteTest with PMMLRe
         .setLoss(loss)
         .setMaxIter(3)
       val model = lir.fit(dataset)
-      Seq(4, 16, 64).foreach { blockSize =>
-        val model2 = lir.setBlockSize(blockSize).fit(dataset)
+      Seq(0, 0.01, 0.1, 1, 2, 4).foreach { s =>
+        val model2 = lir.setMaxBlockSizeInMB(s).fit(dataset)
         assert(model.intercept ~== model2.intercept relTol 1e-9)
         assert(model.coefficients ~== model2.coefficients relTol 1e-9)
         assert(model.scale ~== model2.scale relTol 1e-9)
@@ -760,6 +761,7 @@ class LinearRegressionSuite extends MLTest with DefaultReadWriteTest with PMMLRe
           .fit(datasetWithWeightConstantLabel)
         if (fitIntercept) {
           assert(model1.summary.objectiveHistory(0) ~== 0.0 absTol 1e-4)
+          assert(model1.summary.totalIterations === 0)
         }
         val model2 = new LinearRegression()
           .setFitIntercept(fitIntercept)
@@ -767,6 +769,7 @@ class LinearRegressionSuite extends MLTest with DefaultReadWriteTest with PMMLRe
           .setSolver("l-bfgs")
           .fit(datasetWithWeightZeroLabel)
         assert(model2.summary.objectiveHistory(0) ~== 0.0 absTol 1e-4)
+        assert(model2.summary.totalIterations === 0)
       }
     }
   }
@@ -899,6 +902,59 @@ class LinearRegressionSuite extends MLTest with DefaultReadWriteTest with PMMLRe
     }
   }
 
+  test("linear regression model training summary with weighted samples") {
+    Seq("auto", "l-bfgs", "normal").foreach { solver =>
+      val trainer1 = new LinearRegression().setSolver(solver)
+      val trainer2 = new LinearRegression().setSolver(solver).setWeightCol("weight")
+
+      Seq(0.25, 1.0, 10.0, 50.00).foreach { w =>
+        val model1 = trainer1.fit(datasetWithDenseFeature)
+        val model2 = trainer2.fit(datasetWithDenseFeature.withColumn("weight", lit(w)))
+        assert(model1.summary.explainedVariance ~== model2.summary.explainedVariance relTol 1e-6)
+        assert(model1.summary.meanAbsoluteError ~== model2.summary.meanAbsoluteError relTol 1e-6)
+        assert(model1.summary.meanSquaredError ~== model2.summary.meanSquaredError relTol 1e-6)
+        assert(model1.summary.rootMeanSquaredError ~==
+          model2.summary.rootMeanSquaredError relTol 1e-6)
+        assert(model1.summary.r2 ~== model2.summary.r2 relTol 1e-6)
+        assert(model1.summary.r2adj ~== model2.summary.r2adj relTol 1e-6)
+      }
+    }
+  }
+
+  test("linear regression model testset evaluation summary with weighted samples") {
+    Seq("auto", "l-bfgs", "normal").foreach { solver =>
+      val trainer1 = new LinearRegression().setSolver(solver)
+      val trainer2 = new LinearRegression().setSolver(solver).setWeightCol("weight")
+
+      Seq(0.25, 1.0, 10.0, 50.00).foreach { w =>
+        val model1 = trainer1.fit(datasetWithDenseFeature)
+        val model2 = trainer2.fit(datasetWithDenseFeature.withColumn("weight", lit(w)))
+        val testSummary1 = model1.evaluate(datasetWithDenseFeature)
+        val testSummary2 = model2.evaluate(datasetWithDenseFeature.withColumn("weight", lit(w)))
+        assert(testSummary1.explainedVariance ~== testSummary2.explainedVariance relTol 1e-6)
+        assert(testSummary1.meanAbsoluteError ~== testSummary2.meanAbsoluteError relTol 1e-6)
+        assert(testSummary1.meanSquaredError ~== testSummary2.meanSquaredError relTol 1e-6)
+        assert(testSummary1.rootMeanSquaredError ~==
+          testSummary2.rootMeanSquaredError relTol 1e-6)
+        assert(testSummary1.r2 ~== testSummary2.r2 relTol 1e-6)
+        assert(testSummary1.r2adj ~== testSummary2.r2adj relTol 1e-6)
+      }
+    }
+  }
+
+  test("linear regression training summary totalIterations") {
+    Seq(1, 5, 10, 20).foreach { maxIter =>
+      val trainer = new LinearRegression().setSolver("l-bfgs").setMaxIter(maxIter)
+      val model = trainer.fit(datasetWithDenseFeature)
+      assert(model.summary.totalIterations <= maxIter)
+    }
+    Seq("auto", "normal").foreach { solver =>
+      val trainer = new LinearRegression().setSolver(solver)
+      val model = trainer.fit(datasetWithDenseFeature)
+      assert(model.summary.totalIterations === 0)
+    }
+  }
+
   test("linear regression with weighted samples") {
     val sqlContext = spark.sqlContext
     import sqlContext.implicits._
diff --git a/mllib/src/test/scala/org/apache/spark/ml/regression/RandomForestRegressorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/regression/RandomForestRegressorSuite.scala
index 31dc6d379e76c..7ec30de301779 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/regression/RandomForestRegressorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/regression/RandomForestRegressorSuite.scala
@@ -175,7 +175,7 @@ class RandomForestRegressorSuite extends MLTest with DefaultReadWriteTest{
     val testParams = Seq(
       (50, 5, 1.0, 0.75),
       (50, 10, 1.0, 0.75),
-      (50, 10, 0.95, 0.78)
+      (50, 10, 0.95, 0.75)
     )
 
     for ((numTrees, maxDepth, subsamplingRate, tol) <- testParams) {
@@ -221,6 +221,18 @@ class RandomForestRegressorSuite extends MLTest with DefaultReadWriteTest{
     testEstimatorAndModelReadWrite(rf, continuousData, allParamSettings,
       allParamSettings, checkModelData)
   }
+
+  test("SPARK-33398: Load RandomForestRegressionModel prior to Spark 3.0") {
+    val path = testFile("ml-models/rfr-2.4.7")
+    val model = RandomForestRegressionModel.load(path)
+    assert(model.numFeatures === 692)
+    assert(model.totalNumNodes === 8)
+    assert(model.trees.map(_.numNodes) === Array(5, 3))
+
+    val metadata = spark.read.json(s"$path/metadata")
+    val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
+    assert(sparkVersionStr === "2.4.7")
+  }
 }
 
 private object RandomForestRegressorSuite extends SparkFunSuite {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala
index 263ad26657545..a456409cfe3bc 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala
@@ -28,11 +28,25 @@ import org.apache.spark.ml.linalg.{DenseVector, SparseVector, Vector, Vectors}
 import org.apache.spark.ml.linalg.SQLDataTypes.VectorType
 import org.apache.spark.mllib.util.MLlibTestSparkContext
 import org.apache.spark.sql.{Row, SaveMode}
+import org.apache.spark.sql.execution.datasources.CommonFileDataSourceSuite
 import org.apache.spark.sql.types.{DoubleType, StructField, StructType}
 import org.apache.spark.util.Utils
 
+class LibSVMRelationSuite
+  extends SparkFunSuite
+  with MLlibTestSparkContext
+  with CommonFileDataSourceSuite {
+
+  override protected def dataSourceFormat = "libsvm"
+  override protected def inputDataset = {
+    val rawData = new java.util.ArrayList[Row]()
+    rawData.add(Row(1.0, Vectors.sparse(1, Seq((0, 1.0)))))
+    val struct = new StructType()
+      .add("labelFoo", DoubleType, false)
+      .add("featuresBar", VectorType, false)
+    spark.createDataFrame(rawData, struct)
+  }
 
-class LibSVMRelationSuite extends SparkFunSuite with MLlibTestSparkContext {
   // Path for dataset
   var path: String = _
 
@@ -191,4 +205,24 @@ class LibSVMRelationSuite extends SparkFunSuite with MLlibTestSparkContext {
       spark.sql("DROP TABLE IF EXISTS libsvmTable")
     }
   }
+
+  test("SPARK-32815: Test LibSVM data source on file paths with glob metacharacters") {
+    withTempDir { dir =>
+      val basePath = dir.getCanonicalPath
+      // test libsvm writer / reader without specifying schema
+      val svmFileName = "[abc]"
+      val escapedSvmFileName = "\\[abc\\]"
+      val rawData = new java.util.ArrayList[Row]()
+      rawData.add(Row(1.0, Vectors.sparse(2, Seq((0, 2.0), (1, 3.0)))))
+      val struct = new StructType()
+        .add("labelFoo", DoubleType, false)
+        .add("featuresBar", VectorType, false)
+      val df = spark.createDataFrame(rawData, struct)
+      df.write.format("libsvm").save(s"$basePath/$svmFileName")
+      val df2 = spark.read.format("libsvm").load(s"$basePath/$escapedSvmFileName")
+      val row1 = df2.first()
+      val v = row1.getAs[SparseVector](1)
+      assert(v == Vectors.sparse(2, Seq((0, 2.0), (1, 3.0))))
+    }
+  }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/stat/SummarizerSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/stat/SummarizerSuite.scala
index 68ba57c0d5fc8..e438a4135908e 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/stat/SummarizerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/stat/SummarizerSuite.scala
@@ -29,7 +29,6 @@ class SummarizerSuite extends SparkFunSuite with MLlibTestSparkContext {
 
   import testImplicits._
   import Summarizer._
-  import SummaryBuilderImpl._
 
   private case class ExpectedMetrics(
       mean: Vector,
diff --git a/mllib/src/test/scala/org/apache/spark/ml/tree/impl/RandomForestSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/tree/impl/RandomForestSuite.scala
index 2a83d0aaf9699..3ca6816ce7c0d 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/tree/impl/RandomForestSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/tree/impl/RandomForestSuite.scala
@@ -19,7 +19,6 @@ package org.apache.spark.ml.tree.impl
 
 import scala.annotation.tailrec
 import scala.collection.mutable
-import scala.language.implicitConversions
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.ml.classification.DecisionTreeClassificationModel
diff --git a/mllib/src/test/scala/org/apache/spark/ml/tuning/CrossValidatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/tuning/CrossValidatorSuite.scala
index a30428ec2d283..d7cbfa8e2e2c5 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/tuning/CrossValidatorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/tuning/CrossValidatorSuite.scala
@@ -32,6 +32,7 @@ import org.apache.spark.ml.regression.LinearRegression
 import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTest, MLTestingUtils}
 import org.apache.spark.mllib.util.LinearDataGenerator
 import org.apache.spark.sql.Dataset
+import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types.StructType
 
 class CrossValidatorSuite
@@ -40,10 +41,14 @@ class CrossValidatorSuite
   import testImplicits._
 
   @transient var dataset: Dataset[_] = _
+  @transient var datasetWithFold: Dataset[_] = _
 
   override def beforeAll(): Unit = {
     super.beforeAll()
     dataset = sc.parallelize(generateLogisticInput(1.0, 1.0, 100, 42), 2).toDF()
+    val dfWithRandom = dataset.repartition(1).withColumn("random", rand(100L))
+    val foldCol = when(col("random") < 0.33, 0).when(col("random") < 0.66, 1).otherwise(2)
+    datasetWithFold = dfWithRandom.withColumn("fold", foldCol).drop("random").repartition(2)
   }
 
   test("cross validation with logistic regression") {
@@ -75,6 +80,65 @@ class CrossValidatorSuite
     }
   }
 
+  test("cross validation with logistic regression with fold col") {
+    val lr = new LogisticRegression
+    val lrParamMaps = new ParamGridBuilder()
+      .addGrid(lr.regParam, Array(0.001, 1000.0))
+      .addGrid(lr.maxIter, Array(0, 10))
+      .build()
+    val eval = new BinaryClassificationEvaluator
+    val cv = new CrossValidator()
+      .setEstimator(lr)
+      .setEstimatorParamMaps(lrParamMaps)
+      .setEvaluator(eval)
+      .setNumFolds(3)
+      .setFoldCol("fold")
+    val cvModel = cv.fit(datasetWithFold)
+
+    MLTestingUtils.checkCopyAndUids(cv, cvModel)
+
+    val parent = cvModel.bestModel.parent.asInstanceOf[LogisticRegression]
+    assert(parent.getRegParam === 0.001)
+    assert(parent.getMaxIter === 10)
+    assert(cvModel.avgMetrics.length === lrParamMaps.length)
+
+    val result = cvModel.transform(dataset).select("prediction").as[Double].collect()
+    testTransformerByGlobalCheckFunc[(Double, Vector)](dataset.toDF(), cvModel, "prediction") {
+      rows =>
+        val result2 = rows.map(_.getDouble(0))
+        assert(result === result2)
+    }
+  }
+
+  test("cross validation with logistic regression with wrong fold col") {
+    val lr = new LogisticRegression
+    val lrParamMaps = new ParamGridBuilder()
+      .addGrid(lr.regParam, Array(0.001, 1000.0))
+      .addGrid(lr.maxIter, Array(0, 10))
+      .build()
+    val eval = new BinaryClassificationEvaluator
+    val cv = new CrossValidator()
+      .setEstimator(lr)
+      .setEstimatorParamMaps(lrParamMaps)
+      .setEvaluator(eval)
+      .setNumFolds(3)
+      .setFoldCol("fold1")
+    val err1 = intercept[IllegalArgumentException] {
+      cv.fit(datasetWithFold)
+    }
+    assert(err1.getMessage.contains("fold1 does not exist. Available: label, features, fold"))
+
+    // Fold column must be integer type.
+    val foldCol = udf(() => 1L)
+    val datasetWithWrongFoldType = dataset.withColumn("fold1", foldCol())
+    val err2 = intercept[IllegalArgumentException] {
+      cv.fit(datasetWithWrongFoldType)
+    }
+    assert(err2
+      .getMessage
+      .contains("The specified `foldCol` column fold1 must be integer type, but got LongType."))
+  }
+
   test("cross validation with linear regression") {
     val dataset = sc.parallelize(
       LinearDataGenerator.generateLinearInput(
diff --git a/mllib/src/test/scala/org/apache/spark/ml/tuning/ParamRandomBuilderSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/tuning/ParamRandomBuilderSuite.scala
new file mode 100644
index 0000000000000..e17c48e4d991d
--- /dev/null
+++ b/mllib/src/test/scala/org/apache/spark/ml/tuning/ParamRandomBuilderSuite.scala
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.tuning
+
+import org.scalatest.matchers.must.Matchers
+import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.ml.param._
+
+class ParamRandomBuilderSuite extends SparkFunSuite with ScalaCheckDrivenPropertyChecks
+  with Matchers {
+
+  val solver = new TestParams() {
+    private val randomColName = "randomVal"
+    val DummyDoubleParam = new DoubleParam(this, randomColName, "doc")
+    val DummyFloatParam = new FloatParam(this, randomColName, "doc")
+    val DummyIntParam = new IntParam(this, randomColName, "doc")
+  }
+  import solver._
+
+  val DoubleLimits: Limits[Double] = Limits(1d, 100d)
+  val FloatLimits: Limits[Float] = Limits(1f, 100f)
+  val IntLimits: Limits[Int] = Limits(1, 100)
+  val nRandoms: Int = 5
+
+  // Java API
+
+  test("Java API random Double linear params mixed with fixed values") {
+    checkRangeAndCardinality(
+      _.addRandom(DummyDoubleParam, DoubleLimits.x, DoubleLimits.y, nRandoms),
+      DoubleLimits,
+      DummyDoubleParam)
+  }
+
+  test("Java API random Double log10 params mixed with fixed values") {
+    checkRangeAndCardinality(
+      _.addLog10Random(DummyDoubleParam, DoubleLimits.x, DoubleLimits.y, nRandoms),
+      DoubleLimits,
+      DummyDoubleParam)
+  }
+
+  test("Java API random Float linear params mixed with fixed values") {
+    checkRangeAndCardinality(
+      _.addRandom(DummyFloatParam, FloatLimits.x, FloatLimits.y, nRandoms),
+      FloatLimits,
+      DummyFloatParam)
+  }
+
+  test("Java API random Float log10 params mixed with fixed values") {
+    checkRangeAndCardinality(
+      _.addLog10Random(DummyFloatParam, FloatLimits.x, FloatLimits.y, nRandoms),
+      FloatLimits,
+      DummyFloatParam)
+  }
+
+  test("Java API random Int linear params mixed with fixed values") {
+    checkRangeAndCardinality(
+      _.addRandom(DummyIntParam, IntLimits.x, IntLimits.y, nRandoms),
+      IntLimits,
+      DummyIntParam)
+  }
+
+  test("Java API random Int log10 params mixed with fixed values") {
+    checkRangeAndCardinality(
+      _.addLog10Random(DummyIntParam, IntLimits.x, IntLimits.y, nRandoms),
+      IntLimits,
+      DummyIntParam)
+  }
+
+  // Scala API
+
+  test("random linear params mixed with fixed values") {
+    import RandomRanges._
+    checkRangeAndCardinality(_.addRandom(DummyDoubleParam, DoubleLimits, nRandoms),
+      DoubleLimits,
+      DummyDoubleParam)
+  }
+
+  test("random log10 params mixed with fixed values") {
+    import RandomRanges._
+    checkRangeAndCardinality(_.addLog10Random(DummyDoubleParam, DoubleLimits, nRandoms),
+      DoubleLimits,
+      DummyDoubleParam)
+  }
+
+  def checkRangeAndCardinality[T: Numeric](addFn: ParamRandomBuilder => ParamRandomBuilder,
+                               lim: Limits[T],
+                               randomCol: Param[T]): Unit = {
+    val maxIterations: Int = 10
+    val basedOn: Array[ParamPair[_]] = Array(maxIter -> maxIterations)
+    val inputCols: Array[String] = Array("input0", "input1")
+    val ops: Numeric[T] = implicitly[Numeric[T]]
+
+    val builder: ParamRandomBuilder = new ParamRandomBuilder()
+      .baseOn(basedOn: _*)
+      .addGrid(inputCol, inputCols)
+    val paramMap: Array[ParamMap] = addFn(builder).build()
+    assert(paramMap.length == inputCols.length * nRandoms * basedOn.length)
+    paramMap.foreach { m: ParamMap =>
+      assert(m(maxIter) == maxIterations)
+      assert(inputCols contains  m(inputCol))
+      assert(ops.gteq(m(randomCol), lim.x))
+      assert(ops.lteq(m(randomCol), lim.y))
+    }
+  }
+
+}
diff --git a/mllib/src/test/scala/org/apache/spark/ml/tuning/RandomRangesSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/tuning/RandomRangesSuite.scala
new file mode 100644
index 0000000000000..afcbc033956b5
--- /dev/null
+++ b/mllib/src/test/scala/org/apache/spark/ml/tuning/RandomRangesSuite.scala
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.tuning
+
+import scala.reflect.runtime.universe.TypeTag
+
+import org.scalacheck.{Arbitrary, Gen}
+import org.scalacheck.Arbitrary._
+import org.scalacheck.Gen.Choose
+import org.scalatest.{Assertion, Succeeded}
+import org.scalatest.matchers.must.Matchers
+import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks
+
+import org.apache.spark.SparkFunSuite
+
+class RandomRangesSuite extends SparkFunSuite with ScalaCheckDrivenPropertyChecks with Matchers {
+
+  import RandomRanges._
+
+  test("log of any base") {
+    assert(logN(16, 4) == 2d)
+    assert(logN(1000, 10) === (3d +- 0.000001))
+    assert(logN(256, 2) == 8d)
+  }
+
+  test("random doubles in log space") {
+    val gen: Gen[(Double, Double, Int)] = for {
+      x <- Gen.choose(0d, Double.MaxValue)
+      y <- Gen.choose(0d, Double.MaxValue)
+      n <- Gen.choose(0, Int.MaxValue)
+    } yield (x, y, n)
+    forAll(gen) { case (x, y, n) =>
+      val lower = math.min(x, y)
+      val upper = math.max(x, y)
+      val result = randomLog(x, y, n)
+      assert(result >= lower && result <= upper)
+    }
+  }
+
+  test("random BigInt generation does not go into infinite loop") {
+    assert(randomBigInt0To(0) == BigInt(0))
+  }
+
+  test("random ints") {
+    checkRange(Linear[Int])
+  }
+
+  test("random log ints") {
+    checkRange(Log10[Int])
+  }
+
+  test("random int distribution") {
+    checkDistributionOf(1000)
+  }
+
+  test("random doubles") {
+    checkRange(Linear[Double])
+  }
+
+  test("random log doubles") {
+    checkRange(Log10[Double])
+  }
+
+  test("random double distribution") {
+    checkDistributionOf(1000d)
+  }
+
+  test("random floats") {
+    checkRange(Linear[Float])
+  }
+
+  test("random log floats") {
+    checkRange(Log10[Float])
+  }
+
+  test("random float distribution") {
+    checkDistributionOf(1000f)
+  }
+
+  private abstract class RandomFn[T: Numeric: Generator] {
+    def apply(genRandom: RandomT[T]): T = genRandom.randomT()
+    def appropriate(x: T, y: T): Boolean
+  }
+
+  private def Linear[T: Numeric: Generator]: RandomFn[T] = new RandomFn {
+    override def apply(genRandom: RandomT[T]): T = genRandom.randomT()
+    override def appropriate(x: T, y: T): Boolean = true
+  }
+
+  private def Log10[T: Numeric: Generator]: RandomFn[T] = new RandomFn {
+    override def apply(genRandom: RandomT[T]): T = genRandom.randomTLog(10)
+    val ops: Numeric[T] = implicitly[Numeric[T]]
+    override def appropriate(x: T, y: T): Boolean = {
+      ops.gt(x, ops.zero) && ops.gt(y, ops.zero) && x != y
+    }
+  }
+
+  private def checkRange[T: Numeric: Generator: Choose: TypeTag: Arbitrary]
+  (rand: RandomFn[T]): Assertion =
+    forAll { (x: T, y: T) =>
+      if (rand.appropriate(x, y)) {
+        val ops: Numeric[T] = implicitly[Numeric[T]]
+        val limit: Limits[T] = Limits(x, y)
+        val gen: RandomT[T] = RandomRanges(limit)
+        val result: T = rand(gen)
+        val ordered: (T, T) = lowerUpper(x, y)
+        assert(ops.gteq(result, ordered._1) && ops.lteq(result, ordered._2))
+      } else Succeeded
+    }
+
+  private def checkDistributionOf[T: Numeric: Generator: Choose](range: T): Unit = {
+    val ops: Numeric[T] = implicitly[Numeric[T]]
+    import ops._
+    val gen: Gen[(T, T)] = for {
+      x <- Gen.choose(negate(range), range)
+      y <- Gen.choose(range, times(range, plus(one, one)))
+    } yield (x, y)
+    forAll(gen) { case (x, y) =>
+      assertEvenDistribution(10000, Limits(x, y))
+    }
+  }
+
+  private def meanAndStandardDeviation[T: Numeric](xs: Seq[T]): (Double, Double) = {
+    val ops: Numeric[T] = implicitly[Numeric[T]]
+    val n: Int = xs.length
+    val mean: Double = ops.toDouble(xs.sum) / n
+    val squaredDiff: Seq[Double] = xs.map { x: T => math.pow(ops.toDouble(x) - mean, 2) }
+    val stdDev: Double = math.pow(squaredDiff.sum / n - 1, 0.5)
+    (mean, stdDev)
+  }
+
+  private def lowerUpper[T: Numeric](x: T, y: T): (T, T) = {
+    val ops: Numeric[T] = implicitly[Numeric[T]]
+    (ops.min(x, y), ops.max(x, y))
+  }
+
+  private def midPointOf[T: Numeric : Generator](lim: Limits[T]): Double = {
+    val ordered: (T, T) = lowerUpper(lim.x, lim.y)
+    val ops: Numeric[T] = implicitly[Numeric[T]]
+    val range: T = ops.minus(ordered._2, ordered._1)
+    (ops.toDouble(range) / 2) + ops.toDouble(ordered._1)
+  }
+
+  private def assertEvenDistribution[T: Numeric: Generator](n: Int, lim: Limits[T]): Assertion = {
+    val gen: RandomT[T] = RandomRanges(lim)
+    val xs: Seq[T] = (0 to n).map { _: Int => gen.randomT() }
+    val (mean, stdDev) = meanAndStandardDeviation(xs)
+    val tolerance: Double = 4 * stdDev
+    val halfWay: Double = midPointOf(lim)
+    assert(mean > halfWay - tolerance && mean < halfWay + tolerance)
+  }
+
+}
diff --git a/mllib/src/test/scala/org/apache/spark/ml/util/DefaultReadWriteTest.scala b/mllib/src/test/scala/org/apache/spark/ml/util/DefaultReadWriteTest.scala
index 4d9e664850c12..c5bf202a2d337 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/util/DefaultReadWriteTest.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/util/DefaultReadWriteTest.scala
@@ -19,7 +19,6 @@ package org.apache.spark.ml.util
 
 import java.io.{File, IOException}
 
-import org.json4s.JNothing
 import org.scalatest.Suite
 
 import org.apache.spark.{SparkException, SparkFunSuite}
@@ -63,6 +62,9 @@ trait DefaultReadWriteTest extends TempDirectory { self: Suite =>
           (instance.getOrDefault(p), newInstance.getOrDefault(p)) match {
             case (Array(values), Array(newValues)) =>
               assert(values === newValues, s"Values do not match on param ${p.name}.")
+            case (value: Double, newValue: Double) =>
+              assert(value.isNaN && newValue.isNaN || value == newValue,
+                s"Values do not match on param ${p.name}.")
             case (value, newValue) =>
               assert(value === newValue, s"Values do not match on param ${p.name}.")
           }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/util/MLTestSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/util/MLTestSuite.scala
index 20c5b5395f6a4..1732469ccf590 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/util/MLTestSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/util/MLTestSuite.scala
@@ -47,7 +47,7 @@ class MLTestSuite extends MLTest {
     }
     intercept[Exception] {
       testTransformerOnStreamData[(Int, String)](data, indexerModel, "id", "indexed") {
-        rows: Seq[Row] =>
+        rows: scala.collection.Seq[Row] =>
           assert(rows.map(_.getDouble(1)).max === 1.0)
       }
     }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/util/PMMLReadWriteTest.scala b/mllib/src/test/scala/org/apache/spark/ml/util/PMMLReadWriteTest.scala
index d2c4832b12bac..19e9fe4bdb30e 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/util/PMMLReadWriteTest.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/util/PMMLReadWriteTest.scala
@@ -23,10 +23,7 @@ import org.dmg.pmml.PMML
 import org.scalatest.Suite
 
 import org.apache.spark.SparkContext
-import org.apache.spark.ml.{Estimator, Model}
 import org.apache.spark.ml.param._
-import org.apache.spark.mllib.util.MLlibTestSparkContext
-import org.apache.spark.sql.Dataset
 
 trait PMMLReadWriteTest extends TempDirectory { self: Suite =>
   /**
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/classification/LogisticRegressionSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/classification/LogisticRegressionSuite.scala
index d96a4da46a630..584e7555eb0d9 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/classification/LogisticRegressionSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/classification/LogisticRegressionSuite.scala
@@ -21,8 +21,8 @@ import scala.collection.JavaConverters._
 import scala.util.Random
 import scala.util.control.Breaks._
 
-import org.scalatest.Assertions._
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.mllib.linalg.{Vector, Vectors}
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/classification/NaiveBayesSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/classification/NaiveBayesSuite.scala
index 47dac3ec29a5c..b9d83dd2b81f8 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/classification/NaiveBayesSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/classification/NaiveBayesSuite.scala
@@ -107,9 +107,9 @@ class NaiveBayesSuite extends SparkFunSuite with MLlibTestSparkContext {
     val modelIndex = piData.indices.zip(model.labels.map(_.toInt))
     try {
       for (i <- modelIndex) {
-        assert(math.exp(piData(i._2)) ~== math.exp(model.pi(i._1)) absTol 0.05)
+        assert(piData(i._2) ~== model.pi(i._1) relTol 0.35)
         for (j <- thetaData(i._2).indices) {
-          assert(math.exp(thetaData(i._2)(j)) ~== math.exp(model.theta(i._1)(j)) absTol 0.05)
+          assert(thetaData(i._2)(j) ~== model.theta(i._1)(j) relTol 0.35)
         }
       }
     } catch {
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/clustering/LDASuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/clustering/LDASuite.scala
index 56d41403f74cc..8f311bbf9f840 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/clustering/LDASuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/clustering/LDASuite.scala
@@ -20,7 +20,6 @@ package org.apache.spark.mllib.clustering
 import java.util.{ArrayList => JArrayList}
 
 import breeze.linalg.{argmax, argtopk, max, DenseMatrix => BDM}
-import org.scalatest.Assertions
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.graphx.Edge
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/feature/Word2VecSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/feature/Word2VecSuite.scala
index a679fe43414f2..e4cd492be3d2e 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/feature/Word2VecSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/feature/Word2VecSuite.scala
@@ -43,7 +43,8 @@ class Word2VecSuite extends SparkFunSuite with MLlibTestSparkContext {
     // and a Word2VecMap give the same values.
     val word2VecMap = model.getVectors
     val newModel = new Word2VecModel(word2VecMap)
-    assert(newModel.getVectors.mapValues(_.toSeq) === word2VecMap.mapValues(_.toSeq))
+    assert(newModel.getVectors.mapValues(_.toSeq).toMap ===
+      word2VecMap.mapValues(_.toSeq).toMap)
   }
 
   test("Word2Vec throws exception when vocabulary is empty") {
@@ -102,7 +103,8 @@ class Word2VecSuite extends SparkFunSuite with MLlibTestSparkContext {
     try {
       model.save(sc, path)
       val sameModel = Word2VecModel.load(sc, path)
-      assert(sameModel.getVectors.mapValues(_.toSeq) === model.getVectors.mapValues(_.toSeq))
+      assert(sameModel.getVectors.mapValues(_.toSeq).toMap ===
+        model.getVectors.mapValues(_.toSeq).toMap)
     } finally {
       Utils.deleteRecursively(tempDir)
     }
@@ -136,7 +138,8 @@ class Word2VecSuite extends SparkFunSuite with MLlibTestSparkContext {
     try {
       model.save(sc, path)
       val sameModel = Word2VecModel.load(sc, path)
-      assert(sameModel.getVectors.mapValues(_.toSeq) === model.getVectors.mapValues(_.toSeq))
+      assert(sameModel.getVectors.mapValues(_.toSeq).toMap ===
+        model.getVectors.mapValues(_.toSeq).toMap)
     }
     catch {
       case t: Throwable => fail("exception thrown persisting a model " +
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/BLASSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/BLASSuite.scala
index 12ab2ac3cc698..91d1e9a44791e 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/BLASSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/BLASSuite.scala
@@ -24,7 +24,7 @@ import org.apache.spark.mllib.util.TestingUtils._
 class BLASSuite extends SparkFunSuite {
 
   test("nativeL1Threshold") {
-    assert(getBLAS(128) == BLAS.f2jBLAS)
+    assert(getBLAS(128) == BLAS.javaBLAS)
     assert(getBLAS(256) == BLAS.nativeBLAS)
     assert(getBLAS(512) == BLAS.nativeBLAS)
   }
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/UDTSerializationBenchmark.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/UDTSerializationBenchmark.scala
index 3caa8f6d5b1e5..e748e3288b643 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/UDTSerializationBenchmark.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/UDTSerializationBenchmark.scala
@@ -24,7 +24,9 @@ import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
  * Serialization benchmark for VectorUDT.
  * To run this benchmark:
  * {{{
- * 1. without sbt: bin/spark-submit --class <this class> <spark mllib test jar>
+ * 1. without sbt:
+ *    bin/spark-submit --class <this class>
+ *      --jars <spark core test jar> <spark mllib test jar>
  * 2. build/sbt "mllib/test:runMain <this class>"
  * 3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "mllib/test:runMain <this class>"
  *    Results will be written to "benchmarks/UDTSerializationBenchmark-results.txt".
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/distributed/BlockMatrixSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/distributed/BlockMatrixSuite.scala
index 9d7177e0a149e..0e789821aa5f3 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/distributed/BlockMatrixSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/distributed/BlockMatrixSuite.scala
@@ -22,7 +22,7 @@ import java.{util => ju}
 import breeze.linalg.{DenseMatrix => BDM, DenseVector => BDV, SparseVector => BSV}
 
 import org.apache.spark.{SparkException, SparkFunSuite}
-import org.apache.spark.mllib.linalg.{DenseMatrix, DenseVector, Matrices, Matrix, SparseMatrix, SparseVector, Vectors}
+import org.apache.spark.mllib.linalg.{DenseMatrix, Matrices, Matrix, SparseMatrix}
 import org.apache.spark.mllib.util.MLlibTestSparkContext
 import org.apache.spark.mllib.util.TestingUtils._
 
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/distributed/RowMatrixSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/distributed/RowMatrixSuite.scala
index 0a4b11935580a..adc4eeef91bb1 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/distributed/RowMatrixSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/distributed/RowMatrixSuite.scala
@@ -25,6 +25,7 @@ import breeze.linalg.{norm => brzNorm, svd => brzSvd, DenseMatrix => BDM, DenseV
 import breeze.numerics.abs
 
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.internal.config.MAX_RESULT_SIZE
 import org.apache.spark.mllib.linalg.{Matrices, Vector, Vectors}
 import org.apache.spark.mllib.random.RandomRDDs
 import org.apache.spark.mllib.util.{LocalClusterSparkContext, MLlibTestSparkContext}
@@ -121,6 +122,20 @@ class RowMatrixSuite extends SparkFunSuite with MLlibTestSparkContext {
     assert(objectBiggerThanResultSize.getMessage.contains("it's bigger than maxResultSize"))
   }
 
+  test("SPARK-33043: getTreeAggregateIdealDepth with unlimited driver size") {
+    val originalMaxResultSize = sc.conf.get[Long](MAX_RESULT_SIZE)
+    sc.conf.set(MAX_RESULT_SIZE, 0L)
+    try {
+      val nbPartitions = 100
+      val vectors = sc.emptyRDD[Vector]
+        .repartition(nbPartitions)
+      val rowMat = new RowMatrix(vectors)
+      assert(rowMat.getTreeAggregateIdealDepth(700 * 1024 * 1024) === 1)
+    } finally {
+      sc.conf.set(MAX_RESULT_SIZE, originalMaxResultSize)
+    }
+  }
+
   test("similar columns") {
     val colMags = Vectors.dense(math.sqrt(126), math.sqrt(66), math.sqrt(94))
     val expected = BDM(
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/optimization/GradientDescentSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/optimization/GradientDescentSuite.scala
index a5542565c7401..edea67e524aad 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/optimization/GradientDescentSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/optimization/GradientDescentSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.mllib.optimization
 import scala.collection.JavaConverters._
 import scala.util.Random
 
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.mllib.linalg.Vectors
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/optimization/LBFGSSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/optimization/LBFGSSuite.scala
index 69c303ee932e0..27e21acc275c3 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/optimization/LBFGSSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/optimization/LBFGSSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.mllib.optimization
 
 import scala.util.Random
 
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.mllib.linalg.Vectors
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/regression/IsotonicRegressionSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/regression/IsotonicRegressionSuite.scala
index 02ea74b87f684..8066900dfa011 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/regression/IsotonicRegressionSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/regression/IsotonicRegressionSuite.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.mllib.regression
 
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
 
 import org.apache.spark.{SparkException, SparkFunSuite}
 import org.apache.spark.mllib.util.MLlibTestSparkContext
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/regression/StreamingLinearRegressionSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/regression/StreamingLinearRegressionSuite.scala
index 8e2d7d10f2ce2..b8342f84be44b 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/regression/StreamingLinearRegressionSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/regression/StreamingLinearRegressionSuite.scala
@@ -31,7 +31,7 @@ class StreamingLinearRegressionSuite
   with TestSuiteBase {
 
   // use longer wait time to ensure job completion
-  override def maxWaitTimeMillis: Int = 20000
+  override def maxWaitTimeMillis: Int = 60000
 
   // Assert that two values are equal within tolerance epsilon
   def assertEqual(v1: Double, v2: Double, epsilon: Double): Unit = {
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/util/MLUtilsSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/util/MLUtilsSuite.scala
index 665708a780c48..fb3bc9f798490 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/util/MLUtilsSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/util/MLUtilsSuite.scala
@@ -353,4 +353,34 @@ class MLUtilsSuite extends SparkFunSuite with MLlibTestSparkContext {
       convertMatrixColumnsFromML(df, "p._2")
     }
   }
+
+  test("kFold with fold column") {
+    val data = sc.parallelize(1 to 100, 2).map(x => (x, if (x <= 50) 0 else 1)).toDF("i", "fold")
+    val collectedData = data.collect().map(_.getInt(0)).sorted
+    val twoFoldedRdd = kFold(data, 2, "fold")
+    assert(twoFoldedRdd(0)._1.collect().map(_.getInt(0)).sorted ===
+      twoFoldedRdd(1)._2.collect().map(_.getInt(0)).sorted)
+    assert(twoFoldedRdd(0)._2.collect().map(_.getInt(0)).sorted ===
+      twoFoldedRdd(1)._1.collect().map(_.getInt(0)).sorted)
+
+    val result1 = twoFoldedRdd(0)._1.union(twoFoldedRdd(0)._2).collect().map(_.getInt(0)).sorted
+    assert(result1 ===  collectedData,
+      "Each training+validation set combined should contain all of the data.")
+    val result2 = twoFoldedRdd(1)._1.union(twoFoldedRdd(1)._2).collect().map(_.getInt(0)).sorted
+    assert(result2 ===  collectedData,
+      "Each training+validation set combined should contain all of the data.")
+  }
+
+  test("kFold with fold column: invalid fold numbers") {
+    val data = sc.parallelize(Seq(0, 1, 2), 2).toDF( "fold")
+    val err1 = intercept[SparkException] {
+      kFold(data, 2, "fold")(0)._1.collect()
+    }
+    assert(err1.getMessage.contains("Fold number must be in range [0, 2), but got 2."))
+
+    val err2 = intercept[SparkException] {
+      kFold(data, 4, "fold")(0)._1.collect()
+    }
+    assert(err2.getMessage.contains("The validation data at fold 3 is empty."))
+  }
 }
diff --git a/pom.xml b/pom.xml
index fd4cebcd37319..d9d21cd3d9b4e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -26,7 +26,7 @@
   </parent>
   <groupId>org.apache.spark</groupId>
   <artifactId>spark-parent_2.12</artifactId>
-  <version>3.1.0-SNAPSHOT</version>
+  <version>3.2.0-SNAPSHOT</version>
   <packaging>pom</packaging>
   <name>Spark Project Parent POM</name>
   <url>http://spark.apache.org/</url>
@@ -48,7 +48,7 @@
       <id>matei</id>
       <name>Matei Zaharia</name>
       <email>matei.zaharia@gmail.com</email>
-      <url>http://www.cs.berkeley.edu/~matei</url>
+      <url>https://cs.stanford.edu/people/matei</url>
       <organization>Apache Software Foundation</organization>
       <organizationUrl>http://spark.apache.org</organizationUrl>
     </developer>
@@ -116,44 +116,46 @@
     <maven.compiler.source>${java.version}</maven.compiler.source>
     <maven.compiler.target>${java.version}</maven.compiler.target>
     <maven.version>3.6.3</maven.version>
+    <exec-maven-plugin.version>1.6.0</exec-maven-plugin.version>
     <sbt.project.name>spark</sbt.project.name>
     <slf4j.version>1.7.30</slf4j.version>
     <log4j.version>1.2.17</log4j.version>
-    <hadoop.version>2.7.4</hadoop.version>
+    <hadoop.version>3.2.2</hadoop.version>
     <protobuf.version>2.5.0</protobuf.version>
     <yarn.version>${hadoop.version}</yarn.version>
-    <zookeeper.version>3.4.14</zookeeper.version>
-    <curator.version>2.7.1</curator.version>
+    <zookeeper.version>3.6.2</zookeeper.version>
+    <curator.version>2.13.0</curator.version>
     <hive.group>org.apache.hive</hive.group>
     <hive.classifier>core</hive.classifier>
     <!-- Version used in Maven Hive dependency -->
-    <hive.version>2.3.7</hive.version>
-    <hive23.version>2.3.7</hive23.version>
+    <hive.version>2.3.8</hive.version>
+    <hive23.version>2.3.8</hive23.version>
     <!-- Version used for internal directory structure -->
     <hive.version.short>2.3</hive.version.short>
     <!-- note that this should be compatible with Kafka brokers version 0.10 and up -->
-    <kafka.version>2.5.0</kafka.version>
-    <derby.version>10.12.1.1</derby.version>
-    <parquet.version>1.10.1</parquet.version>
-    <orc.version>1.5.10</orc.version>
-    <orc.classifier></orc.classifier>
-    <hive.parquet.group>com.twitter</hive.parquet.group>
-    <hive.parquet.version>1.6.0</hive.parquet.version>
-    <jetty.version>9.4.18.v20190429</jetty.version>
-    <javaxservlet.version>3.1.0</javaxservlet.version>
+    <kafka.version>2.8.0</kafka.version>
+    <!-- After 10.15.1.3, the minimum required version is JDK9 -->
+    <derby.version>10.14.2.0</derby.version>
+    <parquet.version>1.12.0</parquet.version>
+    <orc.version>1.6.7</orc.version>
+    <jetty.version>9.4.40.v20210413</jetty.version>
+    <jakartaservlet.version>4.0.3</jakartaservlet.version>
     <chill.version>0.9.5</chill.version>
     <ivy.version>2.4.0</ivy.version>
     <oro.version>2.0.8</oro.version>
+    <!--
+    If you changes codahale.metrics.version, you also need to change
+    the link to metrics.dropwizard.io in docs/monitoring.md.
+    -->
     <codahale.metrics.version>4.1.1</codahale.metrics.version>
-    <avro.version>1.8.2</avro.version>
-    <avro.mapred.classifier>hadoop2</avro.mapred.classifier>
-    <aws.kinesis.client.version>1.12.0</aws.kinesis.client.version>
+    <avro.version>1.10.2</avro.version>
+    <aws.kinesis.client.version>1.14.0</aws.kinesis.client.version>
     <!-- Should be consistent with Kinesis client dependency -->
-    <aws.java.sdk.version>1.11.655</aws.java.sdk.version>
+    <aws.java.sdk.version>1.11.844</aws.java.sdk.version>
     <!-- the producer is used in tests -->
     <aws.kinesis.producer.version>0.12.8</aws.kinesis.producer.version>
     <!--  org.apache.httpcomponents/httpclient-->
-    <commons.httpclient.version>4.5.6</commons.httpclient.version>
+    <commons.httpclient.version>4.5.13</commons.httpclient.version>
     <commons.httpcore.version>4.4.12</commons.httpcore.version>
     <!--  commons-httpclient/commons-httpclient-->
     <httpclient.classic.version>3.1</httpclient.classic.version>
@@ -162,51 +164,60 @@
     <commons.collections.version>3.2.2</commons.collections.version>
     <scala.version>2.12.10</scala.version>
     <scala.binary.version>2.12</scala.binary.version>
+    <scalatest-maven-plugin.version>2.0.0</scalatest-maven-plugin.version>
     <scalafmt.parameters>--test</scalafmt.parameters>
     <!-- for now, not running scalafmt as part of default verify pipeline -->
     <scalafmt.skip>true</scalafmt.skip>
     <codehaus.jackson.version>1.9.13</codehaus.jackson.version>
-    <fasterxml.jackson.version>2.10.0</fasterxml.jackson.version>
-    <snappy.version>1.1.7.5</snappy.version>
+    <fasterxml.jackson.version>2.12.2</fasterxml.jackson.version>
+    <snappy.version>1.1.8.4</snappy.version>
     <netlib.java.version>1.1.2</netlib.java.version>
-    <commons-codec.version>1.10</commons-codec.version>
-    <commons-io.version>2.4</commons-io.version>
+    <netlib.ludovic.dev.version>2.2.0</netlib.ludovic.dev.version>
+    <commons-codec.version>1.15</commons-codec.version>
+    <commons-compress.version>1.20</commons-compress.version>
+    <commons-io.version>2.8.0</commons-io.version>
     <!-- org.apache.commons/commons-lang/-->
     <commons-lang2.version>2.6</commons-lang2.version>
     <!-- org.apache.commons/commons-lang3/-->
-    <commons-lang3.version>3.9</commons-lang3.version>
+    <commons-lang3.version>3.12.0</commons-lang3.version>
     <!-- org.apache.commons/commons-pool2/-->
     <commons-pool2.version>2.6.2</commons-pool2.version>
     <datanucleus-core.version>4.1.17</datanucleus-core.version>
     <guava.version>14.0.1</guava.version>
-    <janino.version>3.0.16</janino.version>
-    <jersey.version>2.30</jersey.version>
+    <janino.version>3.1.4</janino.version>
+    <jersey.version>2.34</jersey.version>
     <joda.version>2.10.5</joda.version>
     <jodd.version>3.5.2</jodd.version>
     <jsr305.version>3.0.0</jsr305.version>
     <libthrift.version>0.12.0</libthrift.version>
-    <antlr4.version>4.7.1</antlr4.version>
+    <antlr4.version>4.8-1</antlr4.version>
     <jpam.version>1.1</jpam.version>
-    <selenium.version>2.52.0</selenium.version>
-    <htmlunit.version>2.22</htmlunit.version>
-    <!--
-    Managed up from older version from Avro; sync with jackson-module-paranamer dependency version
-    -->
-    <paranamer.version>2.8</paranamer.version>
+    <selenium.version>3.141.59</selenium.version>
+    <htmlunit.version>2.40.0</htmlunit.version>
     <maven-antrun.version>1.8</maven-antrun.version>
-    <commons-crypto.version>1.0.0</commons-crypto.version>
+    <commons-crypto.version>1.1.0</commons-crypto.version>
+    <commons-cli.version>1.2</commons-cli.version>
     <!--
-    If you are changing Arrow version specification, please check ./python/pyspark/sql/utils.py,
-    and ./python/setup.py too.
+    If you are changing Arrow version specification, please check
+    ./python/pyspark/sql/pandas/utils.py, and ./python/setup.py too.
     -->
-    <arrow.version>0.15.1</arrow.version>
+    <arrow.version>2.0.0</arrow.version>
     <!-- org.fusesource.leveldbjni will be used except on arm64 platform. -->
     <leveldbjni.group>org.fusesource.leveldbjni</leveldbjni.group>
+    <kubernetes-client.version>5.3.1</kubernetes-client.version>
 
     <test.java.home>${java.home}</test.java.home>
+
+    <!-- Some UI tests require Chrome and Chrome driver installed so those tests are disabled by default. -->
+    <test.default.exclude.tags>org.apache.spark.tags.ChromeUITest</test.default.exclude.tags>
     <test.exclude.tags></test.exclude.tags>
     <test.include.tags></test.include.tags>
 
+    <test.jdwp.address>localhost:0</test.jdwp.address>
+    <test.jdwp.suspend>y</test.jdwp.suspend>
+    <test.jdwp.server>y</test.jdwp.server>
+    <test.debug.suite>false</test.debug.suite>
+
     <!-- Package to use when relocating shaded classes. -->
     <spark.shade.packageName>org.sparkproject</spark.shade.packageName>
 
@@ -222,13 +233,12 @@
       declared in the projects that build assemblies.
 
       For other projects the scope should remain as "compile", otherwise they are not available
-      during compilation if the dependency is transivite (e.g. "graphx/" depending on "core/" and
+      during compilation if the dependency is transitive (e.g. "graphx/" depending on "core/" and
       needing Hadoop classes in the classpath to compile).
     -->
     <hadoop.deps.scope>compile</hadoop.deps.scope>
     <hive.deps.scope>compile</hive.deps.scope>
-    <hive.parquet.scope>provided</hive.parquet.scope>
-    <hive.storage.version>2.7.1</hive.storage.version>
+    <hive.storage.version>2.7.2</hive.storage.version>
     <hive.storage.scope>compile</hive.storage.scope>
     <hive.common.scope>compile</hive.common.scope>
     <hive.llap.scope>compile</hive.llap.scope>
@@ -238,13 +248,26 @@
     <parquet.deps.scope>compile</parquet.deps.scope>
     <parquet.test.deps.scope>test</parquet.test.deps.scope>
 
+    <!--
+      These default to Hadoop 3.x shaded client/minicluster jars, but are switched to hadoop-client
+      when the Hadoop profile is hadoop-2.7, because these are only available in 3.x. Note that,
+      as result we have to include the same hadoop-client dependency multiple times in hadoop-2.7.
+    -->
+    <hadoop-client-api.artifact>hadoop-client-api</hadoop-client-api.artifact>
+    <hadoop-client-runtime.artifact>hadoop-client-runtime</hadoop-client-runtime.artifact>
+    <hadoop-client-minicluster.artifact>hadoop-client-minicluster</hadoop-client-minicluster.artifact>
+
     <!--
       Overridable test home. So that you can call individual pom files directly without
       things breaking.
     -->
     <spark.test.home>${session.executionRootDirectory}</spark.test.home>
+    <spark.test.webdriver.chrome.driver></spark.test.webdriver.chrome.driver>
+    <spark.test.docker.keepContainer>false</spark.test.docker.keepContainer>
 
     <CodeCacheSize>1g</CodeCacheSize>
+    <!-- Needed for consistent times -->
+    <maven.build.timestamp.format>yyyy-MM-dd HH:mm:ss z</maven.build.timestamp.format>
   </properties>
   <repositories>
     <repository>
@@ -330,6 +353,21 @@
       <artifactId>scalatest_${scala.binary.version}</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.scalatestplus</groupId>
+      <artifactId>scalatestplus-scalacheck_${scala.binary.version}</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.scalatestplus</groupId>
+      <artifactId>scalatestplus-mockito_${scala.binary.version}</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.scalatestplus</groupId>
+      <artifactId>scalatestplus-selenium_${scala.binary.version}</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
@@ -517,6 +555,11 @@
         <artifactId>commons-codec</artifactId>
         <version>${commons-codec.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.commons</groupId>
+        <artifactId>commons-compress</artifactId>
+        <version>${commons-compress.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.apache.commons</groupId>
         <artifactId>commons-math3</artifactId>
@@ -587,12 +630,16 @@
             <groupId>io.netty</groupId>
             <artifactId>netty</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>net.bytebuddy</groupId>
+            <artifactId>byte-buddy</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
       <dependency>
         <groupId>org.seleniumhq.selenium</groupId>
-        <artifactId>selenium-htmlunit-driver</artifactId>
-        <version>${selenium.version}</version>
+        <artifactId>htmlunit-driver</artifactId>
+        <version>${htmlunit.version}</version>
         <scope>test</scope>
       </dependency>
       <!-- Update htmlunit dependency that selenium uses for better JS support -->
@@ -661,7 +708,7 @@
       <dependency>
         <groupId>com.github.luben</groupId>
         <artifactId>zstd-jni</artifactId>
-        <version>1.4.4-3</version>
+        <version>1.4.9-1</version>
       </dependency>
       <dependency>
         <groupId>com.clearspring.analytics</groupId>
@@ -689,7 +736,7 @@
       <dependency>
         <groupId>org.roaringbitmap</groupId>
         <artifactId>RoaringBitmap</artifactId>
-        <version>0.7.45</version>
+        <version>0.9.0</version>
       </dependency>
       <dependency>
         <groupId>commons-net</groupId>
@@ -699,7 +746,7 @@
       <dependency>
         <groupId>io.netty</groupId>
         <artifactId>netty-all</artifactId>
-        <version>4.1.47.Final</version>
+        <version>4.1.63.Final</version>
       </dependency>
       <dependency>
         <groupId>org.apache.derby</groupId>
@@ -824,6 +871,11 @@
         <artifactId>javax.ws.rs-api</artifactId>
         <version>2.0.1</version>
       </dependency>
+      <dependency>
+        <groupId>javax.xml.bind</groupId>
+        <artifactId>jaxb-api</artifactId>
+        <version>2.2.11</version>
+      </dependency>
       <dependency>
         <groupId>org.scalanlp</groupId>
         <artifactId>breeze_${scala.binary.version}</artifactId>
@@ -838,7 +890,7 @@
       <dependency>
         <groupId>org.json4s</groupId>
         <artifactId>json4s-jackson_${scala.binary.version}</artifactId>
-        <version>3.6.6</version>
+        <version>3.7.0-M5</version>
         <exclusions>
           <exclusion>
             <groupId>com.fasterxml.jackson.core</groupId>
@@ -884,20 +936,38 @@
       <dependency>
         <groupId>org.scalatest</groupId>
         <artifactId>scalatest_${scala.binary.version}</artifactId>
-        <version>3.0.8</version>
+        <version>3.2.3</version>
+        <scope>test</scope>
+      </dependency>
+      <dependency>
+        <groupId>org.scalatestplus</groupId>
+        <artifactId>scalatestplus-scalacheck_${scala.binary.version}</artifactId>
+        <version>3.1.0.0-RC2</version>
+        <scope>test</scope>
+      </dependency>
+      <dependency>
+        <groupId>org.scalatestplus</groupId>
+        <artifactId>scalatestplus-mockito_${scala.binary.version}</artifactId>
+        <version>1.0.0-SNAP5</version>
+        <scope>test</scope>
+      </dependency>
+      <dependency>
+        <groupId>org.scalatestplus</groupId>
+        <artifactId>scalatestplus-selenium_${scala.binary.version}</artifactId>
+        <version>1.0.0-SNAP5</version>
         <scope>test</scope>
       </dependency>
       <dependency>
         <groupId>org.mockito</groupId>
         <artifactId>mockito-core</artifactId>
-        <version>3.1.0</version>
+        <version>3.4.6</version>
         <scope>test</scope>
       </dependency>
       <dependency>
         <groupId>org.jmock</groupId>
         <artifactId>jmock-junit4</artifactId>
         <scope>test</scope>
-        <version>2.8.4</version>
+        <version>2.12.0</version>
       </dependency>
       <dependency>
         <groupId>org.scalacheck</groupId>
@@ -908,7 +978,7 @@
       <dependency>
         <groupId>junit</groupId>
         <artifactId>junit</artifactId>
-        <version>4.12</version>
+        <version>4.13.1</version>
         <scope>test</scope>
       </dependency>
       <dependency>
@@ -961,7 +1031,7 @@
       <dependency>
         <groupId>org.postgresql</groupId>
         <artifactId>postgresql</artifactId>
-        <version>42.2.6</version>
+        <version>42.2.19</version>
         <scope>test</scope>
       </dependency>
       <dependency>
@@ -970,6 +1040,18 @@
         <version>11.5.0.0</version>
         <scope>test</scope>
       </dependency>
+      <dependency>
+        <groupId>com.microsoft.sqlserver</groupId>
+        <artifactId>mssql-jdbc</artifactId>
+        <version>8.2.2.jre8</version>
+        <scope>test</scope>
+      </dependency>
+      <dependency>
+        <groupId>com.oracle.database.jdbc</groupId>
+        <artifactId>ojdbc8</artifactId>
+        <version>19.6.0.0</version>
+        <scope>test</scope>
+      </dependency>
       <dependency>
         <groupId>org.apache.curator</groupId>
         <artifactId>curator-recipes</artifactId>
@@ -1002,6 +1084,26 @@
         <version>${curator.version}</version>
         <scope>test</scope>
       </dependency>
+      <!-- Hadoop 3.x dependencies -->
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>${hadoop-client-api.artifact}</artifactId>
+        <version>${hadoop.version}</version>
+        <scope>${hadoop.deps.scope}</scope>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>${hadoop-client-runtime.artifact}</artifactId>
+        <version>${hadoop.version}</version>
+        <scope>${hadoop.deps.scope}</scope>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>${hadoop-client-minicluster.artifact}</artifactId>
+        <version>${yarn.version}</version>
+        <scope>test</scope>
+      </dependency>
+      <!-- End of Hadoop 3.x dependencies -->
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-client</artifactId>
@@ -1109,9 +1211,14 @@
       </dependency>
       <dependency>
         <groupId>org.apache.avro</groupId>
-        <artifactId>avro-ipc</artifactId>
+        <artifactId>avro-mapred</artifactId>
         <version>${avro.version}</version>
+        <scope>${hive.deps.scope}</scope>
         <exclusions>
+          <exclusion>
+            <groupId>org.apache.avro</groupId>
+            <artifactId>avro-ipc-jetty</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>io.netty</groupId>
             <artifactId>netty</artifactId>
@@ -1130,47 +1237,23 @@
           </exclusion>
           <exclusion>
             <groupId>org.apache.velocity</groupId>
-            <artifactId>velocity</artifactId>
-          </exclusion>
-        </exclusions>
-      </dependency>
-      <!-- avro-mapred for some reason depends on avro-ipc's test jar, so undo that. -->
-      <dependency>
-        <groupId>org.apache.avro</groupId>
-        <artifactId>avro-ipc</artifactId>
-        <classifier>tests</classifier>
-        <version>${avro.version}</version>
-        <scope>test</scope>
-      </dependency>
-      <dependency>
-        <groupId>org.apache.avro</groupId>
-        <artifactId>avro-mapred</artifactId>
-        <version>${avro.version}</version>
-        <classifier>${avro.mapred.classifier}</classifier>
-        <scope>${hive.deps.scope}</scope>
-        <exclusions>
-          <exclusion>
-            <groupId>io.netty</groupId>
-            <artifactId>netty</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>org.mortbay.jetty</groupId>
-            <artifactId>jetty</artifactId>
+            <artifactId>velocity-engine-core</artifactId>
           </exclusion>
           <exclusion>
-            <groupId>org.mortbay.jetty</groupId>
-            <artifactId>jetty-util</artifactId>
+            <groupId>javax.annotation</groupId>
+            <artifactId>javax.annotation-api</artifactId>
           </exclusion>
           <exclusion>
-            <groupId>org.mortbay.jetty</groupId>
-            <artifactId>servlet-api</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>org.apache.velocity</groupId>
-            <artifactId>velocity</artifactId>
+            <groupId>com.github.luben</groupId>
+            <artifactId>zstd-jni</artifactId>
           </exclusion>
         </exclusions>
       </dependency>
+      <dependency>
+        <groupId>org.tukaani</groupId>
+        <artifactId>xz</artifactId>
+        <version>1.8</version>
+      </dependency>
       <!-- See SPARK-23654 for info on this dependency;
       It is used to keep javax.activation at v1.1.1 after dropping
       jets3t as a dependency.
@@ -1308,14 +1391,50 @@
             <groupId>com.sun.jersey.contribs</groupId>
             <artifactId>*</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
+      <!--
+        Hack to exclude org.apache.hadoop:hadoop-yarn-server-resourcemanager:jar:tests.
+        For some reasons, SBT starts to pull the dependencies of 'hadoop-yarn-server-tests' above
+        with 'tests' classifier after upgrading SBT 1.3 (SPARK-21708). Otherwise, some tests might
+        fail, see also SPARK-33104.
+      -->
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+        <version>${yarn.version}</version>
+        <scope>test</scope>
+      </dependency>
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-yarn-server-web-proxy</artifactId>
         <version>${yarn.version}</version>
         <scope>${hadoop.deps.scope}</scope>
         <exclusions>
+          <exclusion>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-yarn-server-common</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-yarn-common</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-yarn-api</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.bouncycastle</groupId>
+            <artifactId>bcprov-jdk15on</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.bouncycastle</groupId>
+            <artifactId>bcpkix-jdk15on</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>org.fusesource.leveldbjni</groupId>
             <artifactId>leveldbjni-all</artifactId>
@@ -1336,6 +1455,10 @@
             <groupId>javax.servlet</groupId>
             <artifactId>servlet-api</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>javax.servlet</groupId>
+            <artifactId>javax.servlet-api</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>commons-logging</groupId>
             <artifactId>commons-logging</artifactId>
@@ -1357,6 +1480,10 @@
             <groupId>com.zaxxer</groupId>
             <artifactId>HikariCP-java7</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>com.microsoft.sqlserver</groupId>
+            <artifactId>mssql-jdbc</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
       <dependency>
@@ -1415,7 +1542,11 @@
           </exclusion>
           <exclusion>
             <groupId>io.netty</groupId>
-            <artifactId>netty</artifactId>
+            <artifactId>netty-handler</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>io.netty</groupId>
+            <artifactId>netty-transport-native-epoll</artifactId>
           </exclusion>
           <exclusion>
             <groupId>com.github.spotbugs</groupId>
@@ -1472,7 +1603,6 @@
             <artifactId>hive-service</artifactId>
           </exclusion>
           <exclusion>
-            <!-- All classes are covered by spark's hive-thriftserver module -->
             <groupId>${hive.group}</groupId>
             <artifactId>hive-service-rpc</artifactId>
           </exclusion>
@@ -1532,11 +1662,6 @@
             <groupId>${hive.group}</groupId>
             <artifactId>hive-service</artifactId>
           </exclusion>
-          <exclusion>
-            <!-- All classes are covered by spark's hive-thriftserver module -->
-            <groupId>${hive.group}</groupId>
-            <artifactId>hive-service-rpc</artifactId>
-          </exclusion>
           <exclusion>
             <groupId>${hive.group}</groupId>
             <artifactId>hive-shims</artifactId>
@@ -1577,6 +1702,14 @@
             <groupId>org.apache.ant</groupId>
             <artifactId>ant</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-auth</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>org.apache.zookeeper</groupId>
             <artifactId>zookeeper</artifactId>
@@ -1639,7 +1772,7 @@
         <scope>${hive.deps.scope}</scope>
         <exclusions>
 
-          <!-- pull this in when needed; the explicit definition culls the surplis-->
+          <!-- pull this in when needed; the explicit definition culls the surplus-->
           <exclusion>
             <groupId>${hive.group}</groupId>
             <artifactId>hive-metastore</artifactId>
@@ -1766,6 +1899,22 @@
             <groupId>org.apache.logging.log4j</groupId>
             <artifactId>*</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>net.hydromatic</groupId>
+            <artifactId>eigenbase-properties</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.codehaus.janino</groupId>
+            <artifactId>commons-compiler</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.codehaus.janino</groupId>
+            <artifactId>janino</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.pentaho</groupId>
+            <artifactId>pentaho-aggdesigner-algorithm</artifactId>
+          </exclusion>
           <!-- End of Hive 2.3 exclusion -->
         </exclusions>
       </dependency>
@@ -1791,7 +1940,6 @@
             <artifactId>hive-service</artifactId>
           </exclusion>
           <exclusion>
-            <!-- All classes are covered by spark's hive-thriftserver module -->
             <groupId>${hive.group}</groupId>
             <artifactId>hive-service-rpc</artifactId>
           </exclusion>
@@ -1946,11 +2094,10 @@
           </exclusion>
           <!-- Begin of Hive 2.3 exclusion -->
           <exclusion>
-            <!-- All classes are covered by spark's hive-thriftserver module -->
             <groupId>${hive.group}</groupId>
             <artifactId>hive-service-rpc</artifactId>
           </exclusion>
-          <!-- parquet-hadoop-bundle:1.8.1 conflict with 1.10.1 -->
+          <!-- parquet-hadoop-bundle:1.8.1 conflict with 1.12.0 -->
           <exclusion>
             <groupId>org.apache.parquet</groupId>
             <artifactId>parquet-hadoop-bundle</artifactId>
@@ -1968,6 +2115,17 @@
         </exclusions>
       </dependency>
 
+      <dependency>
+        <groupId>${hive.group}</groupId>
+        <artifactId>hive-service-rpc</artifactId>
+        <version>3.1.2</version>
+        <exclusions>
+          <exclusion>
+            <groupId>*</groupId>
+            <artifactId>*</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
       <dependency>
         <groupId>net.sf.jpam</groupId>
         <artifactId>jpam</artifactId>
@@ -2104,7 +2262,6 @@
         <groupId>org.apache.orc</groupId>
         <artifactId>orc-core</artifactId>
         <version>${orc.version}</version>
-        <classifier>${orc.classifier}</classifier>
         <scope>${orc.deps.scope}</scope>
         <exclusions>
           <exclusion>
@@ -2129,7 +2286,6 @@
         <groupId>org.apache.orc</groupId>
         <artifactId>orc-mapreduce</artifactId>
         <version>${orc.version}</version>
-        <classifier>${orc.classifier}</classifier>
         <scope>${orc.deps.scope}</scope>
         <exclusions>
           <exclusion>
@@ -2170,6 +2326,10 @@
             <groupId>commons-pool</groupId>
             <artifactId>commons-pool</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>javax.annotation</groupId>
+            <artifactId>javax.annotation-api</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
       <dependency>
@@ -2178,12 +2338,6 @@
         <version>${parquet.version}</version>
         <scope>${parquet.test.deps.scope}</scope>
       </dependency>
-      <dependency>
-        <groupId>${hive.parquet.group}</groupId>
-        <artifactId>parquet-hadoop-bundle</artifactId>
-        <version>${hive.parquet.version}</version>
-        <scope>${hive.parquet.scope}</scope>
-      </dependency>
       <dependency>
         <groupId>org.codehaus.janino</groupId>
         <artifactId>janino</artifactId>
@@ -2247,12 +2401,6 @@
           </exclusion>
         </exclusions>
       </dependency>
-      <dependency>
-        <groupId>com.thoughtworks.paranamer</groupId>
-        <artifactId>paranamer</artifactId>
-        <version>${paranamer.version}</version>
-        <scope>runtime</scope>
-      </dependency>
       <dependency>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-vector</artifactId>
@@ -2264,26 +2412,33 @@
           </exclusion>
           <exclusion>
             <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
+            <artifactId>jackson-core</artifactId>
           </exclusion>
           <exclusion>
             <groupId>io.netty</groupId>
-            <artifactId>netty-buffer</artifactId>
+            <artifactId>netty-common</artifactId>
           </exclusion>
+        </exclusions>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.arrow</groupId>
+        <artifactId>arrow-memory-netty</artifactId>
+        <version>${arrow.version}</version>
+        <exclusions>
           <exclusion>
             <groupId>io.netty</groupId>
-            <artifactId>netty-common</artifactId>
+            <artifactId>netty-buffer</artifactId>
           </exclusion>
           <exclusion>
             <groupId>io.netty</groupId>
-            <artifactId>netty-handler</artifactId>
+            <artifactId>netty-common</artifactId>
           </exclusion>
         </exclusions>
       </dependency>
       <dependency>
         <groupId>com.univocity</groupId>
         <artifactId>univocity-parsers</artifactId>
-        <version>2.8.3</version>
+        <version>2.9.1</version>
       </dependency>
       <dependency>
         <groupId>org.apache.hive</groupId>
@@ -2297,6 +2452,26 @@
           </exclusion>
         </exclusions>
       </dependency>
+      <dependency>
+        <groupId>commons-cli</groupId>
+        <artifactId>commons-cli</artifactId>
+        <version>${commons-cli.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>dev.ludovic.netlib</groupId>
+        <artifactId>blas</artifactId>
+        <version>${netlib.ludovic.dev.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>dev.ludovic.netlib</groupId>
+        <artifactId>lapack</artifactId>
+        <version>${netlib.ludovic.dev.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>dev.ludovic.netlib</groupId>
+        <artifactId>arpack</artifactId>
+        <version>${netlib.ludovic.dev.version}</version>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 
@@ -2346,11 +2521,39 @@
             </execution>
           </executions>
         </plugin>
-        <plugin>
-          <groupId>org.codehaus.mojo</groupId>
-          <artifactId>build-helper-maven-plugin</artifactId>
-          <version>3.0.0</version>
-        </plugin>
+	<plugin>
+	  <groupId>org.codehaus.mojo</groupId>
+	  <artifactId>build-helper-maven-plugin</artifactId>
+	  <version>3.2.0</version>
+	  <executions>
+	    <execution>
+	      <id>module-timestamp-property</id>
+	      <phase>validate</phase>
+	      <goals>
+		<goal>timestamp-property</goal>
+	      </goals>
+	      <configuration>
+		<name>module.build.timestamp</name>
+		<pattern>${maven.build.timestamp.format}</pattern>
+		<timeSource>current</timeSource>
+		<timeZone>America/Los_Angeles</timeZone>
+	      </configuration>
+	    </execution>
+	    <execution>
+	      <id>local-timestamp-property</id>
+	      <phase>validate</phase>
+	      <goals>
+		<goal>timestamp-property</goal>
+	      </goals>
+	      <configuration>
+		<name>local.build.timestamp</name>
+		<pattern>${maven.build.timestamp.format}</pattern>
+		<timeSource>build</timeSource>
+		<timeZone>America/Los_Angeles</timeZone>
+	      </configuration>
+	    </execution>
+	  </executions>
+	</plugin>
         <plugin>
           <groupId>net.alchim31.maven</groupId>
           <artifactId>scala-maven-plugin</artifactId>
@@ -2387,15 +2590,18 @@
             <checkMultipleScalaVersions>true</checkMultipleScalaVersions>
             <failOnMultipleScalaVersions>true</failOnMultipleScalaVersions>
             <recompileMode>incremental</recompileMode>
-            <useZincServer>true</useZincServer>
             <args>
               <arg>-unchecked</arg>
               <arg>-deprecation</arg>
               <arg>-feature</arg>
               <arg>-explaintypes</arg>
               <arg>-target:jvm-1.8</arg>
+              <arg>-Xfatal-warnings</arg>
+              <arg>-Ywarn-unused:imports</arg>
+              <arg>-P:silencer:globalFilters=.*deprecated.*</arg>
             </args>
             <jvmArgs>
+              <jvmArg>-Xss4m</jvmArg>
               <jvmArg>-Xms1024m</jvmArg>
               <jvmArg>-Xmx1024m</jvmArg>
               <jvmArg>-XX:ReservedCodeCacheSize=${CodeCacheSize}</jvmArg>
@@ -2407,6 +2613,13 @@
               <javacArg>${java.version}</javacArg>
               <javacArg>-Xlint:all,-serial,-path,-try</javacArg>
             </javacArgs>
+            <compilerPlugins>
+              <compilerPlugin>
+                <groupId>com.github.ghik</groupId>
+                <artifactId>silencer-plugin_${scala.version}</artifactId>
+                <version>1.6.0</version>
+              </compilerPlugin>
+            </compilerPlugins>
           </configuration>
         </plugin>
         <plugin>
@@ -2429,7 +2642,7 @@
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-surefire-plugin</artifactId>
-          <version>3.0.0-M3</version>
+          <version>3.0.0-M5</version>
           <!-- Note config is repeated in scalatest config -->
           <configuration>
             <includes>
@@ -2463,6 +2676,7 @@
               <spark.ui.showConsoleProgress>false</spark.ui.showConsoleProgress>
               <spark.unsafe.exceptionOnMemoryLeak>true</spark.unsafe.exceptionOnMemoryLeak>
               <spark.memory.debugFill>true</spark.memory.debugFill>
+              <spark.hadoop.hadoop.security.key.provider.path>test:///</spark.hadoop.hadoop.security.key.provider.path>
               <!-- Needed by sql/hive tests. -->
               <test.src.tables>src</test.src.tables>
             </systemProperties>
@@ -2483,7 +2697,7 @@
         <plugin>
           <groupId>org.scalatest</groupId>
           <artifactId>scalatest-maven-plugin</artifactId>
-          <version>2.0.0</version>
+          <version>${scalatest-maven-plugin.version}</version>
           <!-- Note config is repeated in surefire config -->
           <configuration>
             <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
@@ -2512,10 +2726,12 @@
               <spark.ui.enabled>false</spark.ui.enabled>
               <spark.ui.showConsoleProgress>false</spark.ui.showConsoleProgress>
               <spark.unsafe.exceptionOnMemoryLeak>true</spark.unsafe.exceptionOnMemoryLeak>
+              <spark.test.webdriver.chrome.driver>${spark.test.webdriver.chrome.driver}</spark.test.webdriver.chrome.driver>
+              <spark.test.docker.keepContainer>${spark.test.docker.keepContainer}</spark.test.docker.keepContainer>
               <!-- Needed by sql/hive tests. -->
               <test.src.tables>__not_used__</test.src.tables>
             </systemProperties>
-            <tagsToExclude>${test.exclude.tags}</tagsToExclude>
+            <tagsToExclude>${test.exclude.tags},${test.default.exclude.tags}</tagsToExclude>
             <tagsToInclude>${test.include.tags}</tagsToInclude>
           </configuration>
           <executions>
@@ -2569,6 +2785,12 @@
               <fileset>
                 <directory>lib_managed</directory>
               </fileset>
+              <fileset>
+                <directory>metastore_db</directory>
+              </fileset>
+              <fileset>
+                <directory>spark-warehouse</directory>
+              </fileset>
             </filesets>
           </configuration>
         </plugin>
@@ -2618,7 +2840,7 @@
         <plugin>
           <groupId>org.codehaus.mojo</groupId>
           <artifactId>exec-maven-plugin</artifactId>
-          <version>1.6.0</version>
+          <version>${exec-maven-plugin.version}</version>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
@@ -2768,6 +2990,7 @@
         <artifactId>maven-shade-plugin</artifactId>
         <configuration>
           <shadedArtifactAttached>false</shadedArtifactAttached>
+          <createDependencyReducedPom>false</createDependencyReducedPom>
           <artifactSet>
             <includes>
               <include>org.spark-project.spark:unused</include>
@@ -2856,20 +3079,16 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-checkstyle-plugin</artifactId>
-        <version>3.1.0</version>
+        <version>3.1.1</version>
         <configuration>
           <failOnViolation>false</failOnViolation>
           <includeTestSourceDirectory>true</includeTestSourceDirectory>
           <sourceDirectories>
             <directory>${basedir}/src/main/java</directory>
             <directory>${basedir}/src/main/scala</directory>
-            <directory>${basedir}/v${hive.version.short}/src/main/java</directory>
-            <directory>${basedir}/v${hive.version.short}/src/main/scala</directory>
           </sourceDirectories>
           <testSourceDirectories>
             <directory>${basedir}/src/test/java</directory>
-            <directory>${basedir}/v${hive.version.short}/src/test/java</directory>
-            <directory>${basedir}/v${hive.version.short}/src/test/scala</directory>
           </testSourceDirectories>
           <configLocation>dev/checkstyle.xml</configLocation>
           <outputFile>${basedir}/target/checkstyle-output.xml</outputFile>
@@ -2880,7 +3099,7 @@
           <dependency>
             <groupId>com.puppycrawl.tools</groupId>
             <artifactId>checkstyle</artifactId>
-            <version>8.29</version>
+            <version>8.39</version>
           </dependency>
         </dependencies>
         <executions>
@@ -2942,7 +3161,7 @@
       <plugin>
         <groupId>org.antipathy</groupId>
         <artifactId>mvn-scalafmt_${scala.binary.version}</artifactId>
-        <version>1.0.3</version>
+        <version>1.0.4</version>
         <configuration>
           <parameters>${scalafmt.parameters}</parameters> <!-- (Optional) Additional command line arguments -->
           <skip>${scalafmt.skip}</skip> <!-- (Optional) skip formatting -->
@@ -3029,36 +3248,19 @@
 
     <profile>
       <id>hadoop-2.7</id>
-      <!-- Default hadoop profile. Uses global properties. -->
-    </profile>
-
-    <profile>
-      <id>hadoop-3.2</id>
       <properties>
-        <hadoop.version>3.2.0</hadoop.version>
-        <curator.version>2.13.0</curator.version>
+        <hadoop.version>2.7.4</hadoop.version>
+        <curator.version>2.7.1</curator.version>
+        <commons-io.version>2.4</commons-io.version>
+        <hadoop-client-api.artifact>hadoop-client</hadoop-client-api.artifact>
+        <hadoop-client-runtime.artifact>hadoop-client</hadoop-client-runtime.artifact>
+        <hadoop-client-minicluster.artifact>hadoop-client</hadoop-client-minicluster.artifact>
       </properties>
     </profile>
 
     <profile>
-      <id>hive-1.2</id>
-      <properties>
-        <hive.group>org.spark-project.hive</hive.group>
-        <hive.classifier></hive.classifier>
-        <!-- Version used in Maven Hive dependency -->
-        <hive.version>1.2.1.spark2</hive.version>
-        <!-- Version used for internal directory structure -->
-        <hive.version.short>1.2</hive.version.short>
-        <hive.parquet.scope>${hive.deps.scope}</hive.parquet.scope>
-        <hive.storage.version>2.6.0</hive.storage.version>
-        <hive.storage.scope>provided</hive.storage.scope>
-        <hive.common.scope>provided</hive.common.scope>
-        <hive.llap.scope>provided</hive.llap.scope>
-        <hive.serde.scope>provided</hive.serde.scope>
-        <hive.shims.scope>provided</hive.shims.scope>
-        <orc.classifier>nohive</orc.classifier>
-        <datanucleus-core.version>3.2.10</datanucleus-core.version>
-      </properties>
+      <id>hadoop-3.2</id>
+      <!-- Default hadoop profile. Uses global properties. -->
     </profile>
 
     <profile>
@@ -3122,6 +3324,13 @@
 
     <profile>
       <id>scala-2.12</id>
+      <properties>
+        <!-- 
+         SPARK-34774 Add this property to ensure change-scala-version.sh can replace the public `scala.version`
+         property correctly. 
+        -->
+        <scala.version>2.12.10</scala.version>
+      </properties>
       <build>
         <pluginManagement>
           <plugins>
@@ -3129,11 +3338,11 @@
         </pluginManagement>
       </build>
     </profile>
-    
+
     <profile>
       <id>scala-2.13</id>
       <properties>
-        <scala.version>2.13.1</scala.version>
+        <scala.version>2.13.5</scala.version>
         <scala.binary.version>2.13</scala.binary.version>
       </properties>
       <dependencyManagement>
@@ -3145,6 +3354,48 @@
           </dependency>
         </dependencies>
       </dependencyManagement>
+      <build>
+        <pluginManagement>
+          <plugins>
+            <plugin>
+              <groupId>net.alchim31.maven</groupId>
+              <artifactId>scala-maven-plugin</artifactId>
+              <configuration>
+                <args>
+                  <arg>-unchecked</arg>
+                  <arg>-deprecation</arg>
+                  <arg>-feature</arg>
+                  <arg>-explaintypes</arg>
+                  <arg>-target:jvm-1.8</arg>
+                  <arg>-Wconf:cat=deprecation:wv,any:e</arg>
+                  <!--
+                    TODO(SPARK-33805): Undo the corresponding deprecated usage suppression rule after fixed
+                    <arg>-Wunused:imports</arg>
+                  -->
+                  <arg>-Wconf:cat=scaladoc:wv</arg>
+                  <arg>-Wconf:cat=lint-multiarg-infix:wv</arg>
+                  <arg>-Wconf:cat=other-nullary-override:wv</arg>
+                  <arg>-Wconf:cat=other-match-analysis&amp;site=org.apache.spark.sql.catalyst.catalog.SessionCatalog.lookupFunction.catalogFunction:wv</arg>
+                  <arg>-Wconf:cat=other-pure-statement&amp;site=org.apache.spark.streaming.util.FileBasedWriteAheadLog.readAll.readFile:wv</arg>
+                  <arg>-Wconf:cat=other-pure-statement&amp;site=org.apache.spark.scheduler.OutputCommitCoordinatorSuite.&lt;local OutputCommitCoordinatorSuite&gt;.futureAction:wv</arg>
+                  <!--
+                    SPARK-33775 Suppress compilation warnings that contain the following contents.
+                    TODO(SPARK-33805): Undo the corresponding deprecated usage suppression rule after fixed.
+                  -->
+                  <arg>-Wconf:msg=^(?=.*?method|value|type|object|trait|inheritance)(?=.*?deprecated)(?=.*?since 2.13).+$:s</arg>
+                  <arg>-Wconf:msg=^(?=.*?Widening conversion from)(?=.*?is deprecated because it loses precision).+$:s</arg>
+                  <arg>-Wconf:msg=Auto-application to \`\(\)\` is deprecated:s</arg>
+                  <arg>-Wconf:msg=method with a single empty parameter list overrides method without any parameter list:s</arg>
+                  <arg>-Wconf:msg=method without a parameter list overrides a method with a single empty one:s</arg>
+                </args>
+                <compilerPlugins combine.self="override">
+                </compilerPlugins>
+              </configuration>
+            </plugin>
+          </plugins>
+        </pluginManagement>
+      </build>
+
     </profile>
 
     <!--
@@ -3221,6 +3472,41 @@
       <properties>
         <leveldbjni.group>org.openlabtesting.leveldbjni</leveldbjni.group>
       </properties>
+      <activation>
+        <os>
+          <family>linux</family>
+          <arch>aarch64</arch>
+        </os>
+      </activation>
+    </profile>
+    <profile>
+      <id>jdwp-test-debug</id>
+      <properties>
+        <jdwp.arg.line>-agentlib:jdwp=transport=dt_socket,suspend=${test.jdwp.suspend},server=${test.jdwp.server},address=${test.jdwp.address}</jdwp.arg.line>
+        <debugArgLine>${jdwp.arg.line}</debugArgLine>
+        <maven.surefire.debug>${jdwp.arg.line}</maven.surefire.debug>
+        <debugForkedProcess>${test.debug.suite}</debugForkedProcess>
+      </properties>
+    </profile>
+
+    <!--
+      Deprecated: com.github.fommil.netlib has been replaced by dev.ludovic.netlib which
+      doesn't package or distribute any GPL/LGPL dependencies. There should now be hardware
+      acceleration out-of-the-box without enabling any additional profile.
+      However, we need to keep the dependency for now because the last release of
+      org.scalanlp:breeze still depends on `com.github.fommil.netlib`. It's been updated with
+      https://github.com/scalanlp/breeze/pull/811 but it hasn't been released yet.
+    -->
+    <profile>
+      <id>netlib-lgpl</id>
+      <dependencies>
+        <dependency>
+          <groupId>com.github.fommil.netlib</groupId>
+          <artifactId>all</artifactId>
+          <version>${netlib.java.version}</version>
+          <type>pom</type>
+        </dependency>
+      </dependencies>
     </profile>
   </profiles>
 </project>
diff --git a/project/MimaBuild.scala b/project/MimaBuild.scala
index 10c02103aeddb..badcdf34a2ad0 100644
--- a/project/MimaBuild.scala
+++ b/project/MimaBuild.scala
@@ -22,9 +22,7 @@ import com.typesafe.tools.mima.core._
 import com.typesafe.tools.mima.core.MissingClassProblem
 import com.typesafe.tools.mima.core.MissingTypesProblem
 import com.typesafe.tools.mima.core.ProblemFilters._
-import com.typesafe.tools.mima.plugin.MimaKeys.{mimaBinaryIssueFilters, mimaPreviousArtifacts}
-import com.typesafe.tools.mima.plugin.MimaPlugin.mimaDefaultSettings
-
+import com.typesafe.tools.mima.plugin.MimaKeys.{mimaBinaryIssueFilters, mimaPreviousArtifacts, mimaFailOnNoPrevious}
 
 object MimaBuild {
 
@@ -86,14 +84,17 @@ object MimaBuild {
     ignoredMembers.flatMap(excludeMember) ++ MimaExcludes.excludes(currentSparkVersion)
   }
 
-  def mimaSettings(sparkHome: File, projectRef: ProjectRef) = {
+  def mimaSettings(sparkHome: File, projectRef: ProjectRef): Seq[Setting[_]] = {
     val organization = "org.apache.spark"
-    val previousSparkVersion = "2.4.0"
+    val previousSparkVersion = "3.0.0"
     val project = projectRef.project
     val fullId = "spark-" + project + "_2.12"
-    mimaDefaultSettings ++
-    Seq(mimaPreviousArtifacts := Set(organization % fullId % previousSparkVersion),
-      mimaBinaryIssueFilters ++= ignoredABIProblems(sparkHome, version.value))
+
+    Seq(
+      mimaFailOnNoPrevious := true,
+      mimaPreviousArtifacts := Set(organization % fullId % previousSparkVersion),
+      mimaBinaryIssueFilters ++= ignoredABIProblems(sparkHome, version.value)
+    )
   }
 
 }
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index e2228b0b45075..7c168c14b0279 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -34,22 +34,115 @@ import com.typesafe.tools.mima.core.ProblemFilters._
  */
 object MimaExcludes {
 
+  // Exclude rules for 3.2.x
+  lazy val v32excludes = v31excludes ++ Seq(
+    // [SPARK-33808][SQL] DataSource V2: Build logical writes in the optimizer
+    ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.connector.write.V1WriteBuilder"),
+
+    // [SPARK-33955] Add latest offsets to source progress
+    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.streaming.SourceProgress.this"),
+
+    // [SPARK-34862][SQL] Support nested column in ORC vectorized reader
+    ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getBoolean"),
+    ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getByte"),
+    ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getShort"),
+    ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getInt"),
+    ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getLong"),
+    ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getFloat"),
+    ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getDouble"),
+    ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getDecimal"),
+    ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getUTF8String"),
+    ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getBinary"),
+    ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getArray"),
+    ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getMap"),
+    ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getChild"),
+
+    // [SPARK-35135][CORE] Turn WritablePartitionedIterator from trait into a default implementation class
+    ProblemFilters.exclude[IncompatibleTemplateDefProblem]("org.apache.spark.util.collection.WritablePartitionedIterator")
+  )
+
   // Exclude rules for 3.1.x
   lazy val v31excludes = v30excludes ++ Seq(
+    // mima plugin update caused new incompatibilities to be detected
+    // core module
+    ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.shuffle.sort.io.LocalDiskShuffleMapOutputWriter.commitAllPartitions"),
+    ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.shuffle.api.ShuffleMapOutputWriter.commitAllPartitions"),
+    ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.shuffle.api.ShuffleMapOutputWriter.commitAllPartitions"),
+    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.SparkEnv.environmentDetails"),
+    // mllib module
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionTrainingSummary.totalIterations"),
+    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.classification.LogisticRegressionTrainingSummary.$init$"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.labels"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.truePositiveRateByLabel"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.falsePositiveRateByLabel"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.precisionByLabel"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.recallByLabel"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.fMeasureByLabel"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.fMeasureByLabel"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.accuracy"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.weightedTruePositiveRate"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.weightedFalsePositiveRate"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.weightedRecall"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.weightedPrecision"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.weightedFMeasure"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.weightedFMeasure"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionSummary.roc"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionSummary.areaUnderROC"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionSummary.pr"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionSummary.fMeasureByThreshold"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionSummary.precisionByThreshold"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionSummary.recallByThreshold"),
+    ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.ml.classification.FMClassifier.trainImpl"),
+    ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.ml.regression.FMRegressor.trainImpl"),
     // [SPARK-31077] Remove ChiSqSelector dependency on mllib.ChiSqSelectorModel
     // private constructor
     ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.ml.feature.ChiSqSelectorModel.this"),
+
     // [SPARK-31127] Implement abstract Selector
     // org.apache.spark.ml.feature.ChiSqSelectorModel type hierarchy change
     // before: class ChiSqSelector extends Estimator with ChiSqSelectorParams
     // after: class ChiSqSelector extends PSelector
     // false positive, no binary incompatibility
     ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.ml.feature.ChiSqSelectorModel"),
-    ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.ml.feature.ChiSqSelector")
+    ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.ml.feature.ChiSqSelector"),
+
+    // [SPARK-24634] Add a new metric regarding number of inputs later than watermark plus allowed delay
+    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.streaming.StateOperatorProgress.<init>$default$4"),
+
+    //[SPARK-31893] Add a generic ClassificationSummary trait
+    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.classification.LogisticRegressionTrainingSummary.org$apache$spark$ml$classification$ClassificationSummary$_setter_$org$apache$spark$ml$classification$ClassificationSummary$$multiclassMetrics_="),
+    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.classification.LogisticRegressionTrainingSummary.org$apache$spark$ml$classification$ClassificationSummary$$multiclassMetrics"),
+    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.classification.LogisticRegressionTrainingSummary.weightCol"),
+    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionTrainingSummary.org$apache$spark$ml$classification$BinaryClassificationSummary$_setter_$org$apache$spark$ml$classification$BinaryClassificationSummary$$sparkSession_="),
+    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionTrainingSummary.org$apache$spark$ml$classification$BinaryClassificationSummary$_setter_$org$apache$spark$ml$classification$BinaryClassificationSummary$$binaryMetrics_="),
+    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionTrainingSummary.org$apache$spark$ml$classification$BinaryClassificationSummary$$binaryMetrics"),
+    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionTrainingSummary.org$apache$spark$ml$classification$BinaryClassificationSummary$$sparkSession"),
+    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionTrainingSummary.org$apache$spark$ml$classification$ClassificationSummary$_setter_$org$apache$spark$ml$classification$ClassificationSummary$$multiclassMetrics_="),
+    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionTrainingSummary.org$apache$spark$ml$classification$ClassificationSummary$$multiclassMetrics"),
+    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionTrainingSummary.weightCol"),
+    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.org$apache$spark$ml$classification$ClassificationSummary$_setter_$org$apache$spark$ml$classification$ClassificationSummary$$multiclassMetrics_="),
+    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.org$apache$spark$ml$classification$ClassificationSummary$$multiclassMetrics"),
+    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.weightCol"),
+    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionSummary.org$apache$spark$ml$classification$BinaryClassificationSummary$_setter_$org$apache$spark$ml$classification$BinaryClassificationSummary$$sparkSession_="),
+    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionSummary.org$apache$spark$ml$classification$BinaryClassificationSummary$_setter_$org$apache$spark$ml$classification$BinaryClassificationSummary$$binaryMetrics_="),
+    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionSummary.org$apache$spark$ml$classification$BinaryClassificationSummary$$binaryMetrics"),
+    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionSummary.org$apache$spark$ml$classification$BinaryClassificationSummary$$sparkSession"),
+    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionSummary.org$apache$spark$ml$classification$ClassificationSummary$_setter_$org$apache$spark$ml$classification$ClassificationSummary$$multiclassMetrics_="),
+    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionSummary.org$apache$spark$ml$classification$ClassificationSummary$$multiclassMetrics"),
+    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionSummary.weightCol"),
+
+    // [SPARK-32879] Pass SparkSession.Builder options explicitly to SparkSession
+    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.SparkSession.this")
   )
 
   // Exclude rules for 3.0.x
   lazy val v30excludes = v24excludes ++ Seq(
+    // [SPARK-23429][CORE] Add executor memory metrics to heartbeat and expose in executors REST API
+    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate.apply"),
+    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate.copy"),
+    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate.this"),
+    ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate$"),
+
     // [SPARK-29306] Add support for Stage level scheduling for executors
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages#RetrieveSparkAppConfig.productElement"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages#RetrieveSparkAppConfig.productArity"),
@@ -474,12 +567,6 @@ object MimaExcludes {
 
   // Exclude rules for 2.4.x
   lazy val v24excludes = v23excludes ++ Seq(
-    // [SPARK-23429][CORE] Add executor memory metrics to heartbeat and expose in executors REST API
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate.apply"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate.copy"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate.this"),
-    ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate$"),
-
     // [SPARK-25248] add package private methods to TaskContext
     ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.TaskContext.markTaskFailed"),
     ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.TaskContext.markInterrupted"),
@@ -1670,7 +1757,7 @@ object MimaExcludes {
       ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.ml.regression.RandomForestRegressionModel.numTrees"),
       ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.ml.regression.RandomForestRegressionModel.setFeatureSubsetStrategy")
     ) ++ Seq(
-      // [SPARK-21680][ML][MLLIB]optimzie Vector coompress
+      // [SPARK-21680][ML][MLLIB]optimize Vector compress
       ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.mllib.linalg.Vector.toSparseWithSize"),
       ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.ml.linalg.Vector.toSparseWithSize")
     ) ++ Seq(
@@ -1682,6 +1769,7 @@ object MimaExcludes {
   }
 
   def excludes(version: String) = version match {
+    case v if v.startsWith("3.2") => v32excludes
     case v if v.startsWith("3.1") => v31excludes
     case v if v.startsWith("3.0") => v30excludes
     case v if v.startsWith("2.4") => v24excludes
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 65937ad3cefe3..e2e18d557273e 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -23,18 +23,19 @@ import java.util.Locale
 import scala.io.Source
 import scala.util.Properties
 import scala.collection.JavaConverters._
-import scala.collection.mutable.Stack
+import scala.collection.mutable.ListBuffer
 
 import sbt._
 import sbt.Classpaths.publishTask
 import sbt.Keys._
-import sbtunidoc.Plugin.UnidocKeys.unidocGenjavadocVersion
+import sbt.librarymanagement.{ VersionNumber, SemanticSelector }
 import com.etsy.sbt.checkstyle.CheckstylePlugin.autoImport._
 import com.simplytyped.Antlr4Plugin._
 import com.typesafe.sbt.pom.{PomBuild, SbtPomKeys}
 import com.typesafe.tools.mima.plugin.MimaKeys
 import org.scalastyle.sbt.ScalastylePlugin.autoImport._
 import org.scalastyle.sbt.Tasks
+import sbtassembly.AssemblyPlugin.autoImport._
 
 import spray.revolver.RevolverPlugin._
 
@@ -83,6 +84,8 @@ object BuildCommons {
 object SparkBuild extends PomBuild {
 
   import BuildCommons._
+  import sbtunidoc.GenJavadocPlugin
+  import sbtunidoc.GenJavadocPlugin.autoImport._
   import scala.collection.mutable.Map
 
   val projectsMap: Map[String, Seq[Setting[_]]] = Map.empty
@@ -94,21 +97,9 @@ object SparkBuild extends PomBuild {
       case Some(v) =>
         v.split("(\\s+|,)").filterNot(_.isEmpty).map(_.trim.replaceAll("-P", "")).toSeq
     }
-
-    // TODO: revisit for Scala 2.13 support
-    /*
-    Option(System.getProperty("scala.version"))
-      .filter(_.startsWith("2.11"))
-      .foreach { versionString =>
-        System.setProperty("scala-2.11", "true")
-      }
-    if (System.getProperty("scala-2.11") == "") {
-      // To activate scala-2.10 profile, replace empty property value to non-empty value
-      // in the same way as Maven which handles -Dname as -Dname=true before executes build process.
-      // see: https://github.com/apache/maven/blob/maven-3.0.4/maven-embedder/src/main/java/org/apache/maven/cli/MavenCli.java#L1082
-      System.setProperty("scala-2.11", "true")
+    if (profiles.contains("jdwp-test-debug")) {
+      sys.props.put("test.jdwp.enabled", "true")
     }
-     */
     profiles
   }
 
@@ -121,13 +112,10 @@ object SparkBuild extends PomBuild {
   override val userPropertiesMap = System.getProperties.asScala.toMap
 
   lazy val MavenCompile = config("m2r") extend(Compile)
-  lazy val publishLocalBoth = TaskKey[Unit]("publish-local", "publish local for m2 and ivy")
+  lazy val SbtCompile = config("sbt") extend(Compile)
 
-  lazy val sparkGenjavadocSettings: Seq[sbt.Def.Setting[_]] = Seq(
-    libraryDependencies += compilerPlugin(
-      "com.typesafe.genjavadoc" %% "genjavadoc-plugin" % unidocGenjavadocVersion.value cross CrossVersion.full),
+  lazy val sparkGenjavadocSettings: Seq[sbt.Def.Setting[_]] = GenJavadocPlugin.projectSettings ++ Seq(
     scalacOptions ++= Seq(
-      "-P:genjavadoc:out=" + (target.value / "java"),
       "-P:genjavadoc:strictVisibility=true" // hide package private types
     )
   )
@@ -169,16 +157,16 @@ object SparkBuild extends PomBuild {
     val cachedFun = FileFunction.cached(cacheDir, FilesInfo.lastModified, FilesInfo.exists) {
       (inFiles: Set[File]) => {
         val args: Seq[String] = Seq.empty
-        val scalaSourceV = Seq(file(scalaSource.in(config).value.getAbsolutePath))
-        val configV = (baseDirectory in ThisBuild).value / scalaStyleOnCompileConfig
-        val configUrlV = scalastyleConfigUrl.in(config).value
-        val streamsV = streams.in(config).value
+        val scalaSourceV = Seq(file((config / scalaSource).value.getAbsolutePath))
+        val configV = (ThisBuild / baseDirectory).value / scalaStyleOnCompileConfig
+        val configUrlV = (config / scalastyleConfigUrl).value
+        val streamsV = ((config / streams).value: @sbtUnchecked)
         val failOnErrorV = true
         val failOnWarningV = false
-        val scalastyleTargetV = scalastyleTarget.in(config).value
-        val configRefreshHoursV = scalastyleConfigRefreshHours.in(config).value
-        val targetV = target.in(config).value
-        val configCacheFileV = scalastyleConfigUrlCacheFile.in(config).value
+        val scalastyleTargetV = (config / scalastyleTarget).value
+        val configRefreshHoursV = (config / scalastyleConfigRefreshHours).value
+        val targetV = (config / target).value
+        val configCacheFileV = (config / scalastyleConfigUrlCacheFile).value
 
         logger.info(s"Running scalastyle on ${name.value} in ${config.name}")
         Tasks.doScalastyle(args, configV, configUrlV, failOnErrorV, failOnWarningV, scalaSourceV,
@@ -188,7 +176,7 @@ object SparkBuild extends PomBuild {
       }
     }
 
-    cachedFun(findFiles(scalaSource.in(config).value))
+    cachedFun(findFiles((config / scalaSource).value))
   }
 
   private def findFiles(file: File): Set[File] = if (file.isDirectory) {
@@ -200,28 +188,83 @@ object SparkBuild extends PomBuild {
   def enableScalaStyle: Seq[sbt.Def.Setting[_]] = Seq(
     scalaStyleOnCompile := cachedScalaStyle(Compile).value,
     scalaStyleOnTest := cachedScalaStyle(Test).value,
-    logLevel in scalaStyleOnCompile := Level.Warn,
-    logLevel in scalaStyleOnTest := Level.Warn,
-    (compile in Compile) := {
+    (scalaStyleOnCompile / logLevel) := Level.Warn,
+    (scalaStyleOnTest / logLevel) := Level.Warn,
+    (Compile / compile) := {
       scalaStyleOnCompile.value
-      (compile in Compile).value
+      (Compile / compile).value
     },
-    (compile in Test) := {
+    (Test / compile) := {
       scalaStyleOnTest.value
-      (compile in Test).value
+      (Test / compile).value
+    }
+  )
+
+  // Silencer: Scala compiler plugin for warning suppression
+  // Aim: enable fatal warnings, but suppress ones related to using of deprecated APIs
+  // depends on scala version:
+  // <2.13 - silencer 1.6.0 and compiler settings to enable fatal warnings
+  // 2.13.0,2.13.1 - silencer 1.7.1 and compiler settings to enable fatal warnings
+  // 2.13.2+ - no silencer and configured warnings to achieve the same
+  lazy val compilerWarningSettings: Seq[sbt.Def.Setting[_]] = Seq(
+    libraryDependencies ++= {
+      if (VersionNumber(scalaVersion.value).matchesSemVer(SemanticSelector("<2.13.2"))) {
+        val silencerVersion = if (scalaBinaryVersion.value == "2.13") "1.7.1" else "1.6.0"
+        Seq(
+          "org.scala-lang.modules" %% "scala-collection-compat" % "2.2.0",
+          compilerPlugin("com.github.ghik" % "silencer-plugin" % silencerVersion cross CrossVersion.full),
+          "com.github.ghik" % "silencer-lib" % silencerVersion % Provided cross CrossVersion.full
+        )
+      } else {
+        Seq.empty
+      }
+    },
+    (Compile / scalacOptions) ++= {
+      if (VersionNumber(scalaVersion.value).matchesSemVer(SemanticSelector("<2.13.2"))) {
+        Seq(
+          "-Xfatal-warnings",
+          "-deprecation",
+          "-Ywarn-unused-import",
+          "-P:silencer:globalFilters=.*deprecated.*" //regex to catch deprecation warnings and suppress them
+        )
+      } else {
+        Seq(
+          // replace -Xfatal-warnings with fine-grained configuration, since 2.13.2
+          // verbose warning on deprecation, error on all others
+          // see `scalac -Wconf:help` for details
+          "-Wconf:cat=deprecation:wv,any:e",
+          // 2.13-specific warning hits to be muted (as narrowly as possible) and addressed separately
+          // TODO(SPARK-33499): Enable this option when Scala 2.12 is no longer supported.
+          // "-Wunused:imports",
+          "-Wconf:cat=lint-multiarg-infix:wv",
+          "-Wconf:cat=other-nullary-override:wv",
+          "-Wconf:cat=other-match-analysis&site=org.apache.spark.sql.catalyst.catalog.SessionCatalog.lookupFunction.catalogFunction:wv",
+          "-Wconf:cat=other-pure-statement&site=org.apache.spark.streaming.util.FileBasedWriteAheadLog.readAll.readFile:wv",
+          "-Wconf:cat=other-pure-statement&site=org.apache.spark.scheduler.OutputCommitCoordinatorSuite.<local OutputCommitCoordinatorSuite>.futureAction:wv",
+          "-Wconf:cat=other-pure-statement&site=org.apache.spark.sql.streaming.sources.StreamingDataSourceV2Suite.testPositiveCase.\\$anonfun:wv",
+          // SPARK-33775 Suppress compilation warnings that contain the following contents.
+          // TODO(SPARK-33805): Undo the corresponding deprecated usage suppression rule after
+          //  fixed.
+          "-Wconf:msg=^(?=.*?method|value|type|object|trait|inheritance)(?=.*?deprecated)(?=.*?since 2.13).+$:s",
+          "-Wconf:msg=^(?=.*?Widening conversion from)(?=.*?is deprecated because it loses precision).+$:s",
+          "-Wconf:msg=Auto-application to \\`\\(\\)\\` is deprecated:s",
+          "-Wconf:msg=method with a single empty parameter list overrides method without any parameter list:s",
+          "-Wconf:msg=method without a parameter list overrides a method with a single empty one:s"
+        )
+      }
     }
   )
 
   lazy val sharedSettings = sparkGenjavadocSettings ++
+                            compilerWarningSettings ++
       (if (sys.env.contains("NOLINT_ON_COMPILE")) Nil else enableScalaStyle) ++ Seq(
-    exportJars in Compile := true,
-    exportJars in Test := false,
+    (Compile / exportJars) := true,
+    (Test / exportJars) := false,
     javaHome := sys.env.get("JAVA_HOME")
       .orElse(sys.props.get("java.home").map { p => new File(p).getParentFile().getAbsolutePath() })
       .map(file),
-    incOptions := incOptions.value.withNameHashing(true),
     publishMavenStyle := true,
-    unidocGenjavadocVersion := "0.16",
+    unidocGenjavadocVersion := "0.17",
 
     // Override SBT's default resolvers:
     resolvers := Seq(
@@ -230,17 +273,31 @@ object SparkBuild extends PomBuild {
       "gcs-maven-central-mirror" at "https://maven-central.storage-download.googleapis.com/maven2/",
       DefaultMavenRepository,
       Resolver.mavenLocal,
-      Resolver.file("local", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns)
+      Resolver.file("ivyLocal", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns)
     ),
     externalResolvers := resolvers.value,
     otherResolvers := SbtPomKeys.mvnLocalRepository(dotM2 => Seq(Resolver.file("dotM2", dotM2))).value,
-    publishLocalConfiguration in MavenCompile :=
-      new PublishConfiguration(None, "dotM2", packagedArtifacts.value, Seq(), ivyLoggingLevel.value),
-    publishMavenStyle in MavenCompile := true,
-    publishLocal in MavenCompile := publishTask(publishLocalConfiguration in MavenCompile, deliverLocal).value,
-    publishLocalBoth := Seq(publishLocal in MavenCompile, publishLocal).dependOn.value,
+    (MavenCompile / publishLocalConfiguration) := PublishConfiguration()
+        .withResolverName("dotM2")
+        .withArtifacts(packagedArtifacts.value.toVector)
+        .withLogging(ivyLoggingLevel.value),
+    (SbtCompile / publishLocalConfiguration) := PublishConfiguration()
+        .withResolverName("ivyLocal")
+        .withArtifacts(packagedArtifacts.value.toVector)
+        .withLogging(ivyLoggingLevel.value),
+    (MavenCompile / publishMavenStyle) := true,
+    (SbtCompile / publishMavenStyle) := false,
+    (MavenCompile / publishLocal) := publishTask((MavenCompile / publishLocalConfiguration)).value,
+    (SbtCompile / publishLocal) := publishTask((SbtCompile / publishLocalConfiguration)).value,
+    publishLocal := Seq((MavenCompile / publishLocal), (SbtCompile / publishLocal)).dependOn.value,
+
+    javaOptions ++= {
+      val versionParts = System.getProperty("java.version").split("[+.\\-]+", 3)
+      var major = versionParts(0).toInt
+      if (major >= 16) Seq("--add-modules=jdk.incubator.vector,jdk.incubator.foreign", "-Dforeign.restricted=warn") else Seq.empty
+    },
 
-    javacOptions in (Compile, doc) ++= {
+    (Compile / doc / javacOptions) ++= {
       val versionParts = System.getProperty("java.version").split("[+.\\-]+", 3)
       var major = versionParts(0).toInt
       if (major == 1) major = versionParts(1).toInt
@@ -249,25 +306,27 @@ object SparkBuild extends PomBuild {
 
     javaVersion := SbtPomKeys.effectivePom.value.getProperties.get("java.version").asInstanceOf[String],
 
-    javacOptions in Compile ++= Seq(
+    (Compile / javacOptions) ++= Seq(
       "-encoding", UTF_8.name(),
       "-source", javaVersion.value
     ),
     // This -target and Xlint:unchecked options cannot be set in the Compile configuration scope since
     // `javadoc` doesn't play nicely with them; see https://github.com/sbt/sbt/issues/355#issuecomment-3817629
     // for additional discussion and explanation.
-    javacOptions in (Compile, compile) ++= Seq(
+    (Compile / compile / javacOptions) ++= Seq(
       "-target", javaVersion.value,
       "-Xlint:unchecked"
     ),
 
-    scalacOptions in Compile ++= Seq(
+    (Compile / scalacOptions) ++= Seq(
       s"-target:jvm-${javaVersion.value}",
-      "-sourcepath", (baseDirectory in ThisBuild).value.getAbsolutePath  // Required for relative source links in scaladoc
+      "-sourcepath", (ThisBuild / baseDirectory).value.getAbsolutePath  // Required for relative source links in scaladoc
     ),
 
+    SbtPomKeys.profiles := profiles,
+
     // Remove certain packages from Scaladoc
-    scalacOptions in (Compile, doc) := Seq(
+    (Compile / doc / scalacOptions) := Seq(
       "-groups",
       "-skip-packages", Seq(
         "org.apache.spark.api.python",
@@ -281,43 +340,13 @@ object SparkBuild extends PomBuild {
       if (scalaBinaryVersion.value == "2.12") Seq("-no-java-comments") else Seq.empty
     },
 
-    // Implements -Xfatal-warnings, ignoring deprecation warnings.
-    // Code snippet taken from https://issues.scala-lang.org/browse/SI-8410.
-    compile in Compile := {
-      val analysis = (compile in Compile).value
-      val out = streams.value
+    // disable Mima check for all modules,
+    // to be enabled in specific ones that have previous artifacts
+    MimaKeys.mimaFailOnNoPrevious := false,
 
-      def logProblem(l: (=> String) => Unit, f: File, p: xsbti.Problem) = {
-        l(f.toString + ":" + p.position.line.fold("")(_ + ":") + " " + p.message)
-        l(p.position.lineContent)
-        l("")
-      }
-
-      var failed = 0
-      analysis.infos.allInfos.foreach { case (k, i) =>
-        i.reportedProblems foreach { p =>
-          val deprecation = p.message.contains("is deprecated")
-
-          if (!deprecation) {
-            failed = failed + 1
-          }
-
-          val printer: (=> String) => Unit = s => if (deprecation) {
-            out.log.warn(s)
-          } else {
-            out.log.error("[warn] " + s)
-          }
-
-          logProblem(printer, k, p)
-
-        }
-      }
-
-      if (failed > 0) {
-        sys.error(s"$failed fatal warnings")
-      }
-      analysis
-    }
+    // To prevent intermittent compilation failures, see also SPARK-33297
+    // Apparently we can remove this when we use JDK 11.
+    Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat
   )
 
   def enable(settings: Seq[Setting[_]])(projectRef: ProjectRef) = {
@@ -383,6 +412,12 @@ object SparkBuild extends PomBuild {
 
   enable(KubernetesIntegrationTests.settings)(kubernetesIntegrationTests)
 
+  enable(YARN.settings)(yarn)
+
+  if (profiles.contains("sparkr")) {
+    enable(SparkR.settings)(core)
+  }
+
   /**
    * Adds the ability to run the spark shell directly from SBT without building an assembly
    * jar.
@@ -398,35 +433,35 @@ object SparkBuild extends PomBuild {
   val sparkSql = taskKey[Unit]("starts the spark sql CLI.")
 
   enable(Seq(
-    connectInput in run := true,
+    (run / connectInput) := true,
     fork := true,
-    outputStrategy in run := Some (StdoutOutput),
+    (run / outputStrategy) := Some (StdoutOutput),
 
     javaOptions += "-Xmx2g",
 
     sparkShell := {
-      (runMain in Compile).toTask(" org.apache.spark.repl.Main -usejavacp").value
+      (Compile / runMain).toTask(" org.apache.spark.repl.Main -usejavacp").value
     },
 
     sparkPackage := {
       import complete.DefaultParsers._
       val packages :: className :: otherArgs = spaceDelimited("<group:artifact:version> <MainClass> [args]").parsed.toList
-      val scalaRun = (runner in run).value
-      val classpath = (fullClasspath in Runtime).value
-      val args = Seq("--packages", packages, "--class", className, (Keys.`package` in Compile in LocalProject("core"))
+      val scalaRun = (run / runner).value
+      val classpath = (Runtime / fullClasspath).value
+      val args = Seq("--packages", packages, "--class", className, (LocalProject("core") / Compile / Keys.`package`)
         .value.getCanonicalPath) ++ otherArgs
       println(args)
       scalaRun.run("org.apache.spark.deploy.SparkSubmit", classpath.map(_.data), args, streams.value.log)
     },
 
-    javaOptions in Compile += "-Dspark.master=local",
+    (Compile / javaOptions) += "-Dspark.master=local",
 
     sparkSql := {
-      (runMain in Compile).toTask(" org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver").value
+      (Compile / runMain).toTask(" org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver").value
     }
   ))(assembly)
 
-  enable(Seq(sparkShell := sparkShell in LocalProject("assembly")))(spark)
+  enable(Seq(sparkShell := (LocalProject("assembly") / sparkShell).value))(spark)
 
   // TODO: move this to its upstream project.
   override def projectDefinitions(baseDirectory: File): Seq[Project] = {
@@ -444,10 +479,10 @@ object SparkBuild extends PomBuild {
 object SparkParallelTestGrouping {
   // Settings for parallelizing tests. The basic strategy here is to run the slowest suites (or
   // collections of suites) in their own forked JVMs, allowing us to gain parallelism within a
-  // SBT project. Here, we take a whitelisting approach where the default behavior is to run all
+  // SBT project. Here, we take an opt-in approach where the default behavior is to run all
   // tests sequentially in a single JVM, requiring us to manually opt-in to the extra parallelism.
   //
-  // There are a reasons why such a whitelist approach is good:
+  // There are a reasons why such an opt-in approach is good:
   //
   //    1. Launching one JVM per suite adds significant overhead for short-running suites. In
   //       addition to JVM startup time and JIT warmup, it appears that initialization of Derby
@@ -469,46 +504,51 @@ object SparkParallelTestGrouping {
     "org.apache.spark.sql.catalyst.expressions.MathExpressionsSuite",
     "org.apache.spark.sql.hive.HiveExternalCatalogSuite",
     "org.apache.spark.sql.hive.StatisticsSuite",
-    "org.apache.spark.sql.hive.execution.HiveCompatibilitySuite",
     "org.apache.spark.sql.hive.client.VersionsSuite",
     "org.apache.spark.sql.hive.client.HiveClientVersions",
     "org.apache.spark.sql.hive.HiveExternalCatalogVersionsSuite",
     "org.apache.spark.ml.classification.LogisticRegressionSuite",
     "org.apache.spark.ml.classification.LinearSVCSuite",
     "org.apache.spark.sql.SQLQueryTestSuite",
+    "org.apache.spark.sql.hive.client.HadoopVersionInfoSuite",
+    "org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperationSuite",
     "org.apache.spark.sql.hive.thriftserver.ThriftServerQueryTestSuite",
     "org.apache.spark.sql.hive.thriftserver.SparkSQLEnvSuite",
     "org.apache.spark.sql.hive.thriftserver.ui.ThriftServerPageSuite",
     "org.apache.spark.sql.hive.thriftserver.ui.HiveThriftServer2ListenerSuite",
-    "org.apache.spark.sql.hive.thriftserver.ThriftServerWithSparkContextSuite",
     "org.apache.spark.sql.kafka010.KafkaDelegationTokenSuite"
   )
 
   private val DEFAULT_TEST_GROUP = "default_test_group"
+  private val HIVE_EXECUTION_TEST_GROUP = "hive_execution_test_group"
 
   private def testNameToTestGroup(name: String): String = name match {
     case _ if testsWhichShouldRunInTheirOwnDedicatedJvm.contains(name) => name
+    // Different with the cases in testsWhichShouldRunInTheirOwnDedicatedJvm, here we are grouping
+    // all suites of `org.apache.spark.sql.hive.execution.*` into a single group, instead of
+    // launching one JVM per suite.
+    case _ if name.contains("org.apache.spark.sql.hive.execution") => HIVE_EXECUTION_TEST_GROUP
     case _ => DEFAULT_TEST_GROUP
   }
 
   lazy val settings = Seq(
-    testGrouping in Test := {
-      val tests: Seq[TestDefinition] = (definedTests in Test).value
+    (Test / testGrouping) := {
+      val tests: Seq[TestDefinition] = (Test / definedTests).value
       val defaultForkOptions = ForkOptions(
-        bootJars = Nil,
         javaHome = javaHome.value,
-        connectInput = connectInput.value,
         outputStrategy = outputStrategy.value,
-        runJVMOptions = (javaOptions in Test).value,
+        bootJars = Vector.empty[java.io.File],
         workingDirectory = Some(baseDirectory.value),
-        envVars = (envVars in Test).value
+        runJVMOptions = (Test / javaOptions).value.toVector,
+        connectInput = connectInput.value,
+        envVars = (Test / envVars).value
       )
       tests.groupBy(test => testNameToTestGroup(test.name)).map { case (groupName, groupTests) =>
         val forkOptions = {
           if (groupName == DEFAULT_TEST_GROUP) {
             defaultForkOptions
           } else {
-            defaultForkOptions.copy(runJVMOptions = defaultForkOptions.runJVMOptions ++
+            defaultForkOptions.withRunJVMOptions(defaultForkOptions.runJVMOptions ++
               Seq(s"-Djava.io.tmpdir=${baseDirectory.value}/target/tmp/$groupName"))
           }
         }
@@ -522,8 +562,9 @@ object SparkParallelTestGrouping {
 }
 
 object Core {
+  import scala.sys.process.Process
   lazy val settings = Seq(
-    resourceGenerators in Compile += Def.task {
+    (Compile / resourceGenerators) += Def.task {
       val buildScript = baseDirectory.value + "/../build/spark-build-info"
       val targetDir = baseDirectory.value + "/target/extra-resources/"
       val command = Seq("bash", buildScript, targetDir, version.value)
@@ -537,7 +578,7 @@ object Core {
 object Unsafe {
   lazy val settings = Seq(
     // This option is needed to suppress warnings from sun.misc.Unsafe usage
-    javacOptions in Compile += "-XDignore.symbol.file"
+    (Compile / javacOptions) += "-XDignore.symbol.file"
   )
 }
 
@@ -567,6 +608,7 @@ object DockerIntegrationTests {
  */
 object KubernetesIntegrationTests {
   import BuildCommons._
+  import scala.sys.process.Process
 
   val dockerBuild = TaskKey[Unit]("docker-imgs", "Build the docker images for ITs.")
   val runITs = TaskKey[Unit]("run-its", "Only run ITs, skip image build.")
@@ -601,19 +643,21 @@ object KubernetesIntegrationTests {
     runITs := Def.taskDyn {
       shouldBuildImage = false
       Def.task {
-        (test in Test).value
+        (Test / test).value
       }
     }.value,
-    test in Test := (test in Test).dependsOn(dockerBuild).value,
-    javaOptions in Test ++= Seq(
+    (Test / test) := (Test / test).dependsOn(dockerBuild).value,
+    (Test / javaOptions) ++= Seq(
       "-Dspark.kubernetes.test.deployMode=minikube",
       s"-Dspark.kubernetes.test.imageTag=${imageTag.value}",
       s"-Dspark.kubernetes.test.namespace=${namespace.value}",
       s"-Dspark.kubernetes.test.unpackSparkDir=$sparkHome"
     ),
     // Force packaging before building images, so that the latest code is tested.
-    dockerBuild := dockerBuild.dependsOn(packageBin in Compile in assembly)
-      .dependsOn(packageBin in Compile in examples).value
+    dockerBuild := dockerBuild
+      .dependsOn(assembly / Compile / packageBin)
+      .dependsOn(examples / Compile / packageBin)
+      .value
   )
 }
 
@@ -621,12 +665,12 @@ object KubernetesIntegrationTests {
  * Overrides to work around sbt's dependency resolution being different from Maven's.
  */
 object DependencyOverrides {
+  lazy val guavaVersion = sys.props.get("guava.version").getOrElse("14.0.1")
   lazy val settings = Seq(
-    dependencyOverrides += "com.google.guava" % "guava" % "14.0.1",
-    dependencyOverrides += "commons-io" % "commons-io" % "2.4",
+    dependencyOverrides += "com.google.guava" % "guava" % guavaVersion,
     dependencyOverrides += "xerces" % "xercesImpl" % "2.12.0",
     dependencyOverrides += "jline" % "jline" % "2.14.6",
-    dependencyOverrides += "org.apache.avro" % "avro" % "1.8.2")
+    dependencyOverrides += "org.apache.avro" % "avro" % "1.10.2")
 }
 
 /**
@@ -635,7 +679,23 @@ object DependencyOverrides {
  */
 object ExcludedDependencies {
   lazy val settings = Seq(
-    libraryDependencies ~= { libs => libs.filterNot(_.name == "groovy-all") }
+    libraryDependencies ~= { libs => libs.filterNot(_.name == "groovy-all") },
+    // SPARK-33705: Due to sbt compiler issues, it brings exclusions defined in maven pom back to
+    // the classpath directly and assemble test scope artifacts to assembly/target/scala-xx/jars,
+    // which is also will be added to the classpath of some unit tests that will build a subprocess
+    // to run `spark-submit`, e.g. HiveThriftServer2Test.
+    //
+    // These artifacts are for the jersey-1 API but Spark use jersey-2 ones, so it cause test
+    // flakiness w/ jar conflicts issues.
+    //
+    // Also jersey-1 is only used by yarn module(see resource-managers/yarn/pom.xml) for testing
+    // purpose only. Here we exclude them from the whole project scope and add them w/ yarn only.
+    excludeDependencies ++= Seq(
+      ExclusionRule(organization = "com.sun.jersey"),
+      ExclusionRule("javax.servlet", "javax.servlet-api"),
+      ExclusionRule("javax.ws.rs", "jsr311-api"),
+      ExclusionRule("io.netty", "netty-handler"),
+      ExclusionRule("io.netty", "netty-transport-native-epoll"))
   )
 }
 
@@ -644,11 +704,13 @@ object ExcludedDependencies {
  */
 object OldDeps {
 
-  lazy val project = Project("oldDeps", file("dev"), settings = oldDepsSettings)
+  lazy val project = Project("oldDeps", file("dev"))
+    .settings(oldDepsSettings)
+    .disablePlugins(com.typesafe.sbt.pom.PomReaderPlugin)
 
   lazy val allPreviousArtifactKeys = Def.settingDyn[Seq[Set[ModuleID]]] {
     SparkBuild.mimaProjects
-      .map { project => MimaKeys.mimaPreviousArtifacts in project }
+      .map { project => (project / MimaKeys.mimaPreviousArtifacts) }
       .map(k => Def.setting(k.value))
       .join
   }
@@ -660,18 +722,24 @@ object OldDeps {
 }
 
 object Catalyst {
-  lazy val settings = antlr4Settings ++ Seq(
-    antlr4Version in Antlr4 := SbtPomKeys.effectivePom.value.getProperties.get("antlr4.version").asInstanceOf[String],
-    antlr4PackageName in Antlr4 := Some("org.apache.spark.sql.catalyst.parser"),
-    antlr4GenListener in Antlr4 := true,
-    antlr4GenVisitor in Antlr4 := true,
-    antlr4TreatWarningsAsErrors in Antlr4 := true
+  import com.simplytyped.Antlr4Plugin
+  import com.simplytyped.Antlr4Plugin.autoImport._
+
+  lazy val settings = Antlr4Plugin.projectSettings ++ Seq(
+    (Antlr4 / antlr4Version) := SbtPomKeys.effectivePom.value.getProperties.get("antlr4.version").asInstanceOf[String],
+    (Antlr4 / antlr4PackageName) := Some("org.apache.spark.sql.catalyst.parser"),
+    (Antlr4 / antlr4GenListener) := true,
+    (Antlr4 / antlr4GenVisitor) := true,
+    (Antlr4 / antlr4TreatWarningsAsErrors) := true
   )
 }
 
 object SQL {
+
+  import sbtavro.SbtAvro.autoImport._
+
   lazy val settings = Seq(
-    initialCommands in console :=
+    (console / initialCommands) :=
       """
         |import org.apache.spark.SparkContext
         |import org.apache.spark.sql.SQLContext
@@ -691,21 +759,23 @@ object SQL {
         |import sqlContext.implicits._
         |import sqlContext._
       """.stripMargin,
-    cleanupCommands in console := "sc.stop()"
+    (console / cleanupCommands) := "sc.stop()",
+    Test / avroGenerate := (Compile / avroGenerate).value
   )
+
 }
 
 object Hive {
 
   lazy val settings = Seq(
     // Specially disable assertions since some Hive tests fail them
-    javaOptions in Test := (javaOptions in Test).value.filterNot(_ == "-ea"),
+    (Test / javaOptions) := (Test / javaOptions).value.filterNot(_ == "-ea"),
     // Supporting all SerDes requires us to depend on deprecated APIs, so we turn off the warnings
     // only for this subproject.
     scalacOptions := (scalacOptions map { currentOpts: Seq[String] =>
       currentOpts.filterNot(_ == "-deprecation")
     }).value,
-    initialCommands in console :=
+    (console / initialCommands) :=
       """
         |import org.apache.spark.SparkContext
         |import org.apache.spark.sql.catalyst.analysis._
@@ -721,37 +791,46 @@ object Hive {
         |import org.apache.spark.sql.hive.test.TestHive._
         |import org.apache.spark.sql.hive.test.TestHive.implicits._
         |import org.apache.spark.sql.types._""".stripMargin,
-    cleanupCommands in console := "sparkContext.stop()",
+    (console / cleanupCommands) := "sparkContext.stop()",
     // Some of our log4j jars make it impossible to submit jobs from this JVM to Hive Map/Reduce
     // in order to generate golden files.  This is only required for developers who are adding new
     // new query tests.
-    fullClasspath in Test := (fullClasspath in Test).value.filterNot { f => f.toString.contains("jcl-over") }
+    (Test / fullClasspath) := (Test / fullClasspath).value.filterNot { f => f.toString.contains("jcl-over") }
+  )
+}
+
+object YARN {
+  lazy val settings = Seq(
+    excludeDependencies --= Seq(
+      ExclusionRule(organization = "com.sun.jersey"),
+      ExclusionRule("javax.servlet", "javax.servlet-api"),
+      ExclusionRule("javax.ws.rs", "jsr311-api"))
   )
 }
 
 object Assembly {
   import sbtassembly.AssemblyUtils._
-  import sbtassembly.Plugin._
-  import AssemblyKeys._
+  import sbtassembly.AssemblyPlugin.autoImport._
 
   val hadoopVersion = taskKey[String]("The version of hadoop that spark is compiled against.")
 
-  lazy val settings = assemblySettings ++ Seq(
-    test in assembly := {},
+  lazy val settings = baseAssemblySettings ++ Seq(
+    (assembly / test) := {},
     hadoopVersion := {
       sys.props.get("hadoop.version")
         .getOrElse(SbtPomKeys.effectivePom.value.getProperties.get("hadoop.version").asInstanceOf[String])
     },
-    jarName in assembly := {
+    (assembly / assemblyJarName) := {
+      lazy val hadoopVersionValue = hadoopVersion.value
       if (moduleName.value.contains("streaming-kafka-0-10-assembly")
         || moduleName.value.contains("streaming-kinesis-asl-assembly")) {
         s"${moduleName.value}-${version.value}.jar"
       } else {
-        s"${moduleName.value}-${version.value}-hadoop${hadoopVersion.value}.jar"
+        s"${moduleName.value}-${version.value}-hadoop${hadoopVersionValue}.jar"
       }
     },
-    jarName in (Test, assembly) := s"${moduleName.value}-test-${version.value}.jar",
-    mergeStrategy in assembly := {
+    (Test / assembly / assemblyJarName) := s"${moduleName.value}-test-${version.value}.jar",
+    (assembly / assemblyMergeStrategy) := {
       case m if m.toLowerCase(Locale.ROOT).endsWith("manifest.mf")
                                                                => MergeStrategy.discard
       case m if m.toLowerCase(Locale.ROOT).matches("meta-inf.*\\.sf$")
@@ -766,15 +845,14 @@ object Assembly {
 }
 
 object PySparkAssembly {
-  import sbtassembly.Plugin._
-  import AssemblyKeys._
+  import sbtassembly.AssemblyPlugin.autoImport._
   import java.util.zip.{ZipOutputStream, ZipEntry}
 
   lazy val settings = Seq(
     // Use a resource generator to copy all .py files from python/pyspark into a managed directory
     // to be included in the assembly. We can't just add "python/" to the assembly's resource dir
     // list since that will copy unneeded / unwanted files.
-    resourceGenerators in Compile += Def.macroValueI(resourceManaged in Compile map { outDir: File =>
+    (Compile / resourceGenerators) += Def.macroValueI((Compile / resourceManaged) map { outDir: File =>
       val src = new File(BuildCommons.sparkHome, "python/pyspark")
       val zipFile = new File(BuildCommons.sparkHome , "python/lib/pyspark.zip")
       zipFile.delete()
@@ -814,11 +892,35 @@ object PySparkAssembly {
 
 }
 
+object SparkR {
+  import scala.sys.process.Process
+
+  val buildRPackage = taskKey[Unit]("Build the R package")
+  lazy val settings = Seq(
+    buildRPackage := {
+      val command = baseDirectory.value / ".." / "R" / "install-dev.sh"
+      Process(command.toString).!!
+    },
+    (Compile / compile) := (Def.taskDyn {
+      val c = (Compile / compile).value
+      Def.task {
+        (Compile / buildRPackage).value
+        c
+      }
+    }).value
+  )
+}
+
 object Unidoc {
 
   import BuildCommons._
-  import sbtunidoc.Plugin._
-  import UnidocKeys._
+  import sbtunidoc.BaseUnidocPlugin
+  import sbtunidoc.JavaUnidocPlugin
+  import sbtunidoc.ScalaUnidocPlugin
+  import sbtunidoc.BaseUnidocPlugin.autoImport._
+  import sbtunidoc.GenJavadocPlugin.autoImport._
+  import sbtunidoc.JavaUnidocPlugin.autoImport._
+  import sbtunidoc.ScalaUnidocPlugin.autoImport._
 
   private def ignoreUndocumentedPackages(packages: Seq[Seq[File]]): Seq[Seq[File]] = {
     packages
@@ -833,6 +935,8 @@ object Unidoc {
         f.getCanonicalPath.contains("org/apache/spark/shuffle") &&
         !f.getCanonicalPath.contains("org/apache/spark/shuffle/api")))
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/executor")))
+      .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/ExecutorAllocationClient")))
+      .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend")))
       .map(_.filterNot(f =>
         f.getCanonicalPath.contains("org/apache/spark/unsafe") &&
         !f.getCanonicalPath.contains("org/apache/spark/unsafe/types/CalendarInterval")))
@@ -842,9 +946,11 @@ object Unidoc {
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/catalyst")))
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/execution")))
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/internal")))
-      .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/hive/test")))
+      .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/hive")))
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/catalog/v2/utils")))
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/hive")))
+      .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/v2/avro")))
+      .map(_.filterNot(_.getCanonicalPath.contains("SSLOptions")))
   }
 
   private def ignoreClasspaths(classpaths: Seq[Classpath]): Seq[Classpath] = {
@@ -855,56 +961,65 @@ object Unidoc {
 
   val unidocSourceBase = settingKey[String]("Base URL of source links in Scaladoc.")
 
-  lazy val settings = scalaJavaUnidocSettings ++ Seq (
+  lazy val settings = BaseUnidocPlugin.projectSettings ++
+                      ScalaUnidocPlugin.projectSettings ++
+                      JavaUnidocPlugin.projectSettings ++
+                      Seq (
     publish := {},
 
-    unidocProjectFilter in(ScalaUnidoc, unidoc) :=
+    (ScalaUnidoc / unidoc / unidocProjectFilter) :=
       inAnyProject -- inProjects(OldDeps.project, repl, examples, tools, kubernetes,
-        yarn, tags, streamingKafka010, sqlKafka010, avro),
-    unidocProjectFilter in(JavaUnidoc, unidoc) :=
+        yarn, tags, streamingKafka010, sqlKafka010),
+    (JavaUnidoc / unidoc / unidocProjectFilter) :=
       inAnyProject -- inProjects(OldDeps.project, repl, examples, tools, kubernetes,
-        yarn, tags, streamingKafka010, sqlKafka010, avro),
+        yarn, tags, streamingKafka010, sqlKafka010),
 
-    unidocAllClasspaths in (ScalaUnidoc, unidoc) := {
-      ignoreClasspaths((unidocAllClasspaths in (ScalaUnidoc, unidoc)).value)
+    (ScalaUnidoc / unidoc / unidocAllClasspaths) := {
+      ignoreClasspaths((ScalaUnidoc / unidoc / unidocAllClasspaths).value)
     },
 
-    unidocAllClasspaths in (JavaUnidoc, unidoc) := {
-      ignoreClasspaths((unidocAllClasspaths in (JavaUnidoc, unidoc)).value)
+    (JavaUnidoc / unidoc / unidocAllClasspaths) := {
+      ignoreClasspaths((JavaUnidoc / unidoc / unidocAllClasspaths).value)
     },
 
     // Skip actual catalyst, but include the subproject.
     // Catalyst is not public API and contains quasiquotes which break scaladoc.
-    unidocAllSources in (ScalaUnidoc, unidoc) := {
-      ignoreUndocumentedPackages((unidocAllSources in (ScalaUnidoc, unidoc)).value)
+    (ScalaUnidoc / unidoc / unidocAllSources) := {
+      ignoreUndocumentedPackages((ScalaUnidoc / unidoc / unidocAllSources).value)
     },
 
     // Skip class names containing $ and some internal packages in Javadocs
-    unidocAllSources in (JavaUnidoc, unidoc) := {
-      ignoreUndocumentedPackages((unidocAllSources in (JavaUnidoc, unidoc)).value)
+    (JavaUnidoc / unidoc / unidocAllSources) := {
+      ignoreUndocumentedPackages((JavaUnidoc / unidoc / unidocAllSources).value)
         .map(_.filterNot(_.getCanonicalPath.contains("org/apache/hadoop")))
     },
 
-    javacOptions in (JavaUnidoc, unidoc) := Seq(
-      "-windowtitle", "Spark " + version.value.replaceAll("-SNAPSHOT", "") + " JavaDoc",
-      "-public",
-      "-noqualifier", "java.lang",
-      "-tag", """example:a:Example\:""",
-      "-tag", """note:a:Note\:""",
-      "-tag", "group:X",
-      "-tag", "tparam:X",
-      "-tag", "constructor:X",
-      "-tag", "todo:X",
-      "-tag", "groupname:X"
-    ),
+    (JavaUnidoc / unidoc / javacOptions) := {
+      val versionParts = System.getProperty("java.version").split("[+.\\-]+", 3)
+      var major = versionParts(0).toInt
+      if (major == 1) major = versionParts(1).toInt
+
+      Seq(
+        "-windowtitle", "Spark " + version.value.replaceAll("-SNAPSHOT", "") + " JavaDoc",
+        "-public",
+        "-noqualifier", "java.lang",
+        "-tag", """example:a:Example\:""",
+        "-tag", """note:a:Note\:""",
+        "-tag", "group:X",
+        "-tag", "tparam:X",
+        "-tag", "constructor:X",
+        "-tag", "todo:X",
+        "-tag", "groupname:X",
+      ) ++ { if (major >= 9) Seq("--ignore-source-errors", "-notree") else Seq.empty }
+    },
 
     // Use GitHub repository for Scaladoc source links
     unidocSourceBase := s"https://github.com/apache/spark/tree/v${version.value}",
 
-    scalacOptions in (ScalaUnidoc, unidoc) ++= Seq(
+    (ScalaUnidoc / unidoc / scalacOptions) ++= Seq(
       "-groups", // Group similar methods together based on the @group annotation.
       "-skip-packages", "org.apache.hadoop",
-      "-sourcepath", (baseDirectory in ThisBuild).value.getAbsolutePath
+      "-sourcepath", (ThisBuild / baseDirectory).value.getAbsolutePath
     ) ++ (
       // Add links to sources when generating Scaladoc for a non-snapshot release
       if (!isSnapshot.value) {
@@ -919,18 +1034,18 @@ object Unidoc {
 object Checkstyle {
   lazy val settings = Seq(
     checkstyleSeverityLevel := Some(CheckstyleSeverityLevel.Error),
-    javaSource in (Compile, checkstyle) := baseDirectory.value / "src/main/java",
-    javaSource in (Test, checkstyle) := baseDirectory.value / "src/test/java",
+    (Compile / checkstyle / javaSource) := baseDirectory.value / "src/main/java",
+    (Test / checkstyle / javaSource) := baseDirectory.value / "src/test/java",
     checkstyleConfigLocation := CheckstyleConfigLocation.File("dev/checkstyle.xml"),
     checkstyleOutputFile := baseDirectory.value / "target/checkstyle-output.xml",
-    checkstyleOutputFile in Test := baseDirectory.value / "target/checkstyle-output.xml"
+    (Test / checkstyleOutputFile) := baseDirectory.value / "target/checkstyle-output.xml"
   )
 }
 
 object CopyDependencies {
 
   val copyDeps = TaskKey[Unit]("copyDeps", "Copies needed dependencies to the build directory.")
-  val destPath = (crossTarget in Compile) { _ / "jars"}
+  val destPath = (Compile / crossTarget) { _ / "jars"}
 
   lazy val settings = Seq(
     copyDeps := {
@@ -939,7 +1054,7 @@ object CopyDependencies {
         throw new IOException("Failed to create jars directory.")
       }
 
-      (dependencyClasspath in Compile).value.map(_.data)
+      (Compile / dependencyClasspath).value.map(_.data)
         .filter { jar => jar.isFile() }
         .foreach { jar =>
           val destJar = new File(dest, jar.getName())
@@ -949,68 +1064,90 @@ object CopyDependencies {
           Files.copy(jar.toPath(), destJar.toPath())
         }
     },
-    crossTarget in (Compile, packageBin) := destPath.value,
-    packageBin in Compile := (packageBin in Compile).dependsOn(copyDeps).value
+    (Compile / packageBin / crossTarget) := destPath.value,
+    (Compile / packageBin) := (Compile / packageBin).dependsOn(copyDeps).value
   )
 
 }
 
 object TestSettings {
   import BuildCommons._
+  private val defaultExcludedTags = Seq("org.apache.spark.tags.ChromeUITest")
 
-  // TODO revisit for Scala 2.13 support
-  private val scalaBinaryVersion = "2.12"
-    /*
-    if (System.getProperty("scala-2.11") == "true") {
-      "2.11"
-    } else {
-      "2.12"
-    }
-     */
   lazy val settings = Seq (
     // Fork new JVMs for tests and set Java options for those
     fork := true,
     // Setting SPARK_DIST_CLASSPATH is a simple way to make sure any child processes
     // launched by the tests have access to the correct test-time classpath.
-    envVars in Test ++= Map(
+    (Test / envVars) ++= Map(
       "SPARK_DIST_CLASSPATH" ->
-        (fullClasspath in Test).value.files.map(_.getAbsolutePath)
+        (Test / fullClasspath).value.files.map(_.getAbsolutePath)
           .mkString(File.pathSeparator).stripSuffix(File.pathSeparator),
       "SPARK_PREPEND_CLASSES" -> "1",
-      "SPARK_SCALA_VERSION" -> scalaBinaryVersion,
+      "SPARK_SCALA_VERSION" -> scalaBinaryVersion.value,
       "SPARK_TESTING" -> "1",
       "JAVA_HOME" -> sys.env.get("JAVA_HOME").getOrElse(sys.props("java.home"))),
-    javaOptions in Test += s"-Djava.io.tmpdir=$testTempDir",
-    javaOptions in Test += "-Dspark.test.home=" + sparkHome,
-    javaOptions in Test += "-Dspark.testing=1",
-    javaOptions in Test += "-Dspark.port.maxRetries=100",
-    javaOptions in Test += "-Dspark.master.rest.enabled=false",
-    javaOptions in Test += "-Dspark.memory.debugFill=true",
-    javaOptions in Test += "-Dspark.ui.enabled=false",
-    javaOptions in Test += "-Dspark.ui.showConsoleProgress=false",
-    javaOptions in Test += "-Dspark.unsafe.exceptionOnMemoryLeak=true",
-    javaOptions in Test += "-Dsun.io.serialization.extendedDebugInfo=false",
-    javaOptions in Test += "-Dderby.system.durability=test",
-    javaOptions in Test += "-Dio.netty.tryReflectionSetAccessible=true",
-    javaOptions in Test ++= System.getProperties.asScala.filter(_._1.startsWith("spark"))
+    (Test / javaOptions) += s"-Djava.io.tmpdir=$testTempDir",
+    (Test / javaOptions) += "-Dspark.test.home=" + sparkHome,
+    (Test / javaOptions) += "-Dspark.testing=1",
+    (Test / javaOptions) += "-Dspark.port.maxRetries=100",
+    (Test / javaOptions) += "-Dspark.master.rest.enabled=false",
+    (Test / javaOptions) += "-Dspark.memory.debugFill=true",
+    (Test / javaOptions) += "-Dspark.ui.enabled=false",
+    (Test / javaOptions) += "-Dspark.ui.showConsoleProgress=false",
+    (Test / javaOptions) += "-Dspark.unsafe.exceptionOnMemoryLeak=true",
+    (Test / javaOptions) += "-Dspark.hadoop.hadoop.security.key.provider.path=test:///",
+    (Test / javaOptions) += "-Dsun.io.serialization.extendedDebugInfo=false",
+    (Test / javaOptions) += "-Dderby.system.durability=test",
+    (Test / javaOptions) += "-Dio.netty.tryReflectionSetAccessible=true",
+    (Test / javaOptions) ++= System.getProperties.asScala.filter(_._1.startsWith("spark"))
       .map { case (k,v) => s"-D$k=$v" }.toSeq,
-    javaOptions in Test += "-ea",
+    (Test / javaOptions) += "-ea",
     // SPARK-29282 This is for consistency between JDK8 and JDK11.
-    javaOptions in Test ++= "-Xmx4g -Xss4m -XX:+UseParallelGC -XX:-UseDynamicNumberOfGCThreads"
+    (Test / javaOptions) ++= "-Xmx4g -Xss4m -XX:+UseParallelGC -XX:-UseDynamicNumberOfGCThreads"
       .split(" ").toSeq,
     javaOptions += "-Xmx3g",
+    (Test / javaOptions) ++= {
+      val jdwpEnabled = sys.props.getOrElse("test.jdwp.enabled", "false").toBoolean
+
+      if (jdwpEnabled) {
+        val jdwpAddr = sys.props.getOrElse("test.jdwp.address", "localhost:0")
+        val jdwpServer = sys.props.getOrElse("test.jdwp.server", "y")
+        val jdwpSuspend = sys.props.getOrElse("test.jdwp.suspend", "y")
+        ("-agentlib:jdwp=transport=dt_socket," +
+          s"suspend=$jdwpSuspend,server=$jdwpServer,address=$jdwpAddr").split(" ").toSeq
+      } else {
+        Seq.empty
+      }
+    },
     // Exclude tags defined in a system property
-    testOptions in Test += Tests.Argument(TestFrameworks.ScalaTest,
+    (Test / testOptions) += Tests.Argument(TestFrameworks.ScalaTest,
       sys.props.get("test.exclude.tags").map { tags =>
         tags.split(",").flatMap { tag => Seq("-l", tag) }.toSeq
       }.getOrElse(Nil): _*),
-    testOptions in Test += Tests.Argument(TestFrameworks.JUnit,
+    (Test / testOptions) += Tests.Argument(TestFrameworks.ScalaTest,
+      sys.props.get("test.default.exclude.tags").map(tags => tags.split(",").toSeq)
+        .map(tags => tags.filter(!_.trim.isEmpty)).getOrElse(defaultExcludedTags)
+        .flatMap(tag => Seq("-l", tag)): _*),
+    (Test / testOptions) += Tests.Argument(TestFrameworks.JUnit,
       sys.props.get("test.exclude.tags").map { tags =>
         Seq("--exclude-categories=" + tags)
       }.getOrElse(Nil): _*),
+    // Include tags defined in a system property
+    (Test / testOptions) += Tests.Argument(TestFrameworks.ScalaTest,
+      sys.props.get("test.include.tags").map { tags =>
+        tags.split(",").flatMap { tag => Seq("-n", tag) }.toSeq
+      }.getOrElse(Nil): _*),
+    (Test / testOptions) += Tests.Argument(TestFrameworks.JUnit,
+      sys.props.get("test.include.tags").map { tags =>
+        Seq("--include-categories=" + tags)
+      }.getOrElse(Nil): _*),
     // Show full stack trace and duration in test cases.
-    testOptions in Test += Tests.Argument("-oDF"),
-    testOptions in Test += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"),
+    (Test / testOptions) += Tests.Argument("-oDF"),
+    // Slowpoke notifications: receive notifications every 5 minute of tests that have been running
+    // longer than two minutes.
+    (Test / testOptions) += Tests.Argument(TestFrameworks.ScalaTest, "-W", "120", "300"),
+    (Test / testOptions) += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"),
     // Enable Junit testing.
     libraryDependencies += "com.novocode" % "junit-interface" % "0.11" % "test",
     // `parallelExecutionInTest` controls whether test suites belonging to the same SBT project
@@ -1020,31 +1157,31 @@ object TestSettings {
     // to the underlying runner but is not a SBT-level configuration). This needs to be `true` in
     // order for the extra parallelism enabled by `SparkParallelTestGrouping` to take effect.
     // The `SERIAL_SBT_TESTS` check is here so the extra parallelism can be feature-flagged.
-    parallelExecution in Test := { if (sys.env.contains("SERIAL_SBT_TESTS")) false else true },
+    (Test / parallelExecution) := { if (sys.env.contains("SERIAL_SBT_TESTS")) false else true },
     // Make sure the test temp directory exists.
-    resourceGenerators in Test += Def.macroValueI(resourceManaged in Test map { outDir: File =>
+    (Test / resourceGenerators) += Def.macroValueI((Test / resourceManaged) map { outDir: File =>
       var dir = new File(testTempDir)
       if (!dir.isDirectory()) {
         // Because File.mkdirs() can fail if multiple callers are trying to create the same
         // parent directory, this code tries to create parents one at a time, and avoids
         // failures when the directories have been created by somebody else.
-        val stack = new Stack[File]()
+        val stack = new ListBuffer[File]()
         while (!dir.isDirectory()) {
-          stack.push(dir)
+          stack.prepend(dir)
           dir = dir.getParentFile()
         }
 
         while (stack.nonEmpty) {
-          val d = stack.pop()
+          val d = stack.remove(0)
           require(d.mkdir() || d.isDirectory(), s"Failed to create directory $d")
         }
       }
       Seq.empty[File]
     }).value,
-    concurrentRestrictions in Global := {
+    (Global / concurrentRestrictions) := {
       // The number of concurrent test groups is empirically chosen based on experience
       // with Jenkins flakiness.
-      if (sys.env.contains("SERIAL_SBT_TESTS")) (concurrentRestrictions in Global).value
+      if (sys.env.contains("SERIAL_SBT_TESTS")) (Global / concurrentRestrictions).value
       else Seq(Tags.limit(Tags.ForkedTestGroup, 4))
     }
   )
diff --git a/project/build.properties b/project/build.properties
index 23aa187fb35a7..a35acae0f31a7 100644
--- a/project/build.properties
+++ b/project/build.properties
@@ -14,4 +14,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-sbt.version=0.13.18
+sbt.version=1.5.1
diff --git a/project/plugins.sbt b/project/plugins.sbt
index 5f21d8126e48a..a6e7e94e67621 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -23,28 +23,18 @@ libraryDependencies += "com.puppycrawl.tools" % "checkstyle" % "8.25"
 // checkstyle uses guava 23.0.
 libraryDependencies += "com.google.guava" % "guava" % "23.0"
 
-// need to make changes to uptake sbt 1.0 support in "com.eed3si9n" % "sbt-assembly" % "1.14.5"
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.15.0")
 
 addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "5.2.4")
 
-addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.9.2")
-
 addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0")
 
-// SPARK-29560 Only sbt-mima-plugin needs this repo
-resolvers += Resolver.url("bintray",
-  new java.net.URL("https://dl.bintray.com/typesafe/sbt-plugins"))(Resolver.defaultIvyPatterns)
-addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.3.0")
-
-// sbt 1.0.0 support: https://github.com/AlpineNow/junit_xml_listener/issues/6
-addSbtPlugin("com.alpinenow" % "junit_xml_listener" % "0.5.1")
+addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.8.0")
 
-// need to make changes to uptake sbt 1.0 support in "com.eed3si9n" % "sbt-unidoc" % "0.4.1"
-addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.3.3")
+addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.3")
 
-// need to make changes to uptake sbt 1.0 support in "com.cavorite" % "sbt-avro-1-7" % "1.1.2"
-addSbtPlugin("com.cavorite" % "sbt-avro" % "0.3.2")
+addSbtPlugin("com.cavorite" % "sbt-avro" % "2.1.1")
+libraryDependencies += "org.apache.avro" % "avro-compiler" % "1.10.2"
 
 addSbtPlugin("io.spray" % "sbt-revolver" % "0.9.1")
 
@@ -52,14 +42,6 @@ libraryDependencies += "org.ow2.asm"  % "asm" % "7.2"
 
 libraryDependencies += "org.ow2.asm"  % "asm-commons" % "7.2"
 
-// sbt 1.0.0 support: https://github.com/ihji/sbt-antlr4/issues/14
-addSbtPlugin("com.simplytyped" % "sbt-antlr4" % "0.7.13")
-
-// Spark uses a custom fork of the sbt-pom-reader plugin which contains a patch to fix issues
-// related to test-jar dependencies (https://github.com/sbt/sbt-pom-reader/pull/14). The source for
-// this fork is published at https://github.com/JoshRosen/sbt-pom-reader/tree/v1.0.0-spark
-// and corresponds to commit b160317fcb0b9d1009635a7c5aa05d0f3be61936 in that repository.
-// In the long run, we should try to merge our patch upstream and switch to an upstream version of
-// the plugin; this is tracked at SPARK-14401.
+addSbtPlugin("com.simplytyped" % "sbt-antlr4" % "0.8.2")
 
-addSbtPlugin("org.spark-project" % "sbt-pom-reader" % "1.0.0-spark")
+addSbtPlugin("com.typesafe.sbt" % "sbt-pom-reader" % "2.2.0")
diff --git a/python/MANIFEST.in b/python/MANIFEST.in
index 2d78a001a4d98..862d62b1d3b29 100644
--- a/python/MANIFEST.in
+++ b/python/MANIFEST.in
@@ -22,4 +22,5 @@ recursive-include deps/data *.data *.txt
 recursive-include deps/licenses *.txt
 recursive-include deps/examples *.py
 recursive-include lib *.zip
+recursive-include pyspark *.pyi py.typed
 include README.md
diff --git a/python/docs/Makefile b/python/docs/Makefile
index 4272b7488d9a0..86bce0f88f959 100644
--- a/python/docs/Makefile
+++ b/python/docs/Makefile
@@ -3,10 +3,10 @@
 # You can set these variables from the command line.
 SPHINXOPTS    ?=
 SPHINXBUILD   ?= sphinx-build
-SOURCEDIR     ?= .
-BUILDDIR      ?= _build
+SOURCEDIR     ?= source
+BUILDDIR      ?= build
 
-export PYTHONPATH=$(realpath ..):$(realpath ../lib/py4j-0.10.9-src.zip)
+export PYTHONPATH=$(realpath ..):$(realpath ../lib/py4j-0.10.9.2-src.zip)
 
 # Put it first so that "make" without argument is like "make help".
 help:
diff --git a/python/docs/_static/pyspark.css b/python/docs/_static/pyspark.css
deleted file mode 100644
index 41106f2f6e26d..0000000000000
--- a/python/docs/_static/pyspark.css
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements.  See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-*/
-
-body {
-    background-color: #ffffff;
-}
-
-div.sphinxsidebar {
-    width: 274px;
-}
-
-div.bodywrapper {
-    margin: 0 0 0 274px;
-}
-
-div.sphinxsidebar ul {
-    margin-right: 10px;
-}
-
-div.sphinxsidebar li a {
-    word-break: break-all;
-}
-
-span.pys-tag {
-    font-size: 11px;
-    font-weight: bold;
-    margin: 0 0 0 2px;
-    padding: 1px 3px 1px 3px;
-    -moz-border-radius: 3px;
-    -webkit-border-radius: 3px;
-    border-radius: 3px;
-    text-align: center;
-    text-decoration: none;
-}
-
-span.pys-tag-experimental {
-    background-color: rgb(37, 112, 128);
-    color: rgb(255, 255, 255);
-}
-
-span.pys-tag-deprecated {
-    background-color: rgb(238, 238, 238);
-    color: rgb(62, 67, 73);
-}
-
-div.pys-note-experimental {
-    background-color: rgb(88, 151, 165);
-    border-color: rgb(59, 115, 127);
-    color: rgb(255, 255, 255);
-}
-
-div.pys-note-deprecated {
-}
-
-.hasTooltip {
-    position:relative;
-}
-.hasTooltip span {
-    display:none;
-}
-
-.hasTooltip:hover span.tooltip {
-    display: inline-block;
-    -moz-border-radius: 2px;
-    -webkit-border-radius: 2px;
-    border-radius: 2px;
-    background-color: rgb(250, 250, 250);
-    color: rgb(68, 68, 68);
-    font-weight: normal;
-    box-shadow: 1px 1px 3px rgb(127, 127, 127);
-    position: absolute;
-    padding: 0 3px 0 3px;
-    top: 1.3em;
-    left: 14px;
-    z-index: 9999
-}
diff --git a/python/docs/_static/pyspark.js b/python/docs/_static/pyspark.js
deleted file mode 100644
index 75e4c42492a48..0000000000000
--- a/python/docs/_static/pyspark.js
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements.  See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-*/
-
-$(function (){
-
-    function startsWith(s, prefix) {
-        return s && s.indexOf(prefix) === 0;
-    }
-
-    function buildSidebarLinkMap() {
-        var linkMap = {};
-        $('div.sphinxsidebar a.reference.internal').each(function (i,a)  {
-            var href = $(a).attr('href');
-            if (startsWith(href, '#module-')) {
-                var id = href.substr(8);
-                linkMap[id] = [$(a), null];
-            }
-        })
-        return linkMap;
-    };
-
-    function getAdNoteDivs(dd) {
-        var noteDivs = {};
-        dd.find('> div.admonition.note > p.last').each(function (i, p) {
-            var text = $(p).text();
-            if (!noteDivs.experimental && startsWith(text, 'Experimental')) {
-                noteDivs.experimental = $(p).parent();
-            }
-            if (!noteDivs.deprecated && startsWith(text, 'Deprecated')) {
-                noteDivs.deprecated = $(p).parent();
-            }
-        });
-        return noteDivs;
-    }
-
-    function getParentId(name) {
-        var last_idx = name.lastIndexOf('.');
-        return last_idx == -1? '': name.substr(0, last_idx);
-    }
-
-    function buildTag(text, cls, tooltip) {
-        return '<span class="pys-tag ' + cls + ' hasTooltip">' + text + '<span class="tooltip">'
-            + tooltip + '</span></span>'
-    }
-
-
-    var sidebarLinkMap = buildSidebarLinkMap();
-
-    $('dl.class, dl.function').each(function (i,dl)  {
-
-        dl = $(dl);
-        dt = dl.children('dt').eq(0);
-        dd = dl.children('dd').eq(0);
-        var id = dt.attr('id');
-        var desc = dt.find('> .descname').text();
-        var adNoteDivs = getAdNoteDivs(dd);
-
-        if (id) {
-            var parent_id = getParentId(id);
-
-            var r = sidebarLinkMap[parent_id];
-            if (r) {
-                if (r[1] === null) {
-                    r[1] = $('<ul/>');
-                    r[0].parent().append(r[1]);
-                }
-                var tags = '';
-                if (adNoteDivs.experimental) {
-                    tags += buildTag('E', 'pys-tag-experimental', 'Experimental');
-                    adNoteDivs.experimental.addClass('pys-note pys-note-experimental');
-                }
-                if (adNoteDivs.deprecated) {
-                    tags += buildTag('D', 'pys-tag-deprecated', 'Deprecated');
-                    adNoteDivs.deprecated.addClass('pys-note pys-note-deprecated');
-                }
-                var li = $('<li/>');
-                var a = $('<a href="#' + id + '">' + desc + '</a>');
-                li.append(a);
-                li.append(tags);
-                r[1].append(li);
-                sidebarLinkMap[id] = [a, null];
-            }
-        }
-    });
-});
diff --git a/python/docs/_templates/layout.html b/python/docs/_templates/layout.html
deleted file mode 100644
index ab36ebababf88..0000000000000
--- a/python/docs/_templates/layout.html
+++ /dev/null
@@ -1,6 +0,0 @@
-{% extends "!layout.html" %}
-{% set script_files = script_files + ["_static/pyspark.js"] %}
-{% set css_files = css_files + ['_static/pyspark.css'] %}
-{% block rootrellink %}
-    {{ super() }}
-{% endblock %}
diff --git a/python/docs/index.rst b/python/docs/index.rst
deleted file mode 100644
index 0e7b62361802a..0000000000000
--- a/python/docs/index.rst
+++ /dev/null
@@ -1,52 +0,0 @@
-.. pyspark documentation master file, created by
-   sphinx-quickstart on Thu Aug 28 15:17:47 2014.
-   You can adapt this file completely to your liking, but it should at least
-   contain the root `toctree` directive.
-
-Welcome to Spark Python API Docs!
-===================================
-
-Contents:
-
-.. toctree::
-   :maxdepth: 2
-
-   pyspark
-   pyspark.sql
-   pyspark.streaming
-   pyspark.ml
-   pyspark.mllib
-
-
-Core classes:
----------------
-
-    :class:`pyspark.SparkContext`
-
-    Main entry point for Spark functionality.
-
-    :class:`pyspark.RDD`
-
-    A Resilient Distributed Dataset (RDD), the basic abstraction in Spark.
-
-    :class:`pyspark.streaming.StreamingContext`
-
-    Main entry point for Spark Streaming functionality.
-
-    :class:`pyspark.streaming.DStream`
-
-    A Discretized Stream (DStream), the basic abstraction in Spark Streaming.
-
-    :class:`pyspark.sql.SparkSession`
-
-    Main entry point for DataFrame and SQL functionality.
-
-    :class:`pyspark.sql.DataFrame`
-
-    A distributed collection of data grouped into named columns.
-
-
-Indices and tables
-==================
-
-* :ref:`search`
diff --git a/python/docs/make2.bat b/python/docs/make2.bat
index 7955a83051b8e..26364c1313c33 100644
--- a/python/docs/make2.bat
+++ b/python/docs/make2.bat
@@ -5,10 +5,10 @@ REM Command file for Sphinx documentation
 if "%SPHINXBUILD%" == "" (
 	set SPHINXBUILD=sphinx-build
 )
-set SOURCEDIR=.
-set BUILDDIR=_build
+set SOURCEDIR=source
+set BUILDDIR=build
 
-set PYTHONPATH=..;..\lib\py4j-0.10.9-src.zip
+set PYTHONPATH=..;..\lib\py4j-0.10.9.2-src.zip
 
 if "%1" == "" goto help
 
diff --git a/python/docs/pyspark.ml.rst b/python/docs/pyspark.ml.rst
deleted file mode 100644
index e31dfddd5988e..0000000000000
--- a/python/docs/pyspark.ml.rst
+++ /dev/null
@@ -1,122 +0,0 @@
-pyspark.ml package
-==================
-
-ML Pipeline APIs
-----------------
-
-.. automodule:: pyspark.ml
-    :members:
-    :undoc-members:
-    :inherited-members:
-
-pyspark.ml.param module
------------------------
-
-.. automodule:: pyspark.ml.param
-    :members:
-    :undoc-members:
-    :inherited-members:
-
-pyspark.ml.feature module
--------------------------
-
-.. automodule:: pyspark.ml.feature
-    :members:
-    :undoc-members:
-    :inherited-members:
-
-pyspark.ml.classification module
---------------------------------
-
-.. automodule:: pyspark.ml.classification
-    :members:
-    :undoc-members:
-    :inherited-members:
-
-pyspark.ml.clustering module
-----------------------------
-
-.. automodule:: pyspark.ml.clustering
-    :members:
-    :undoc-members:
-    :inherited-members:
-
-pyspark.ml.functions module
-----------------------------
-
-.. automodule:: pyspark.ml.functions
-    :members:
-    :undoc-members:
-    :inherited-members:
-
-pyspark.ml.linalg module
-----------------------------
-
-.. automodule:: pyspark.ml.linalg
-    :members:
-    :undoc-members:
-    :inherited-members:
-
-pyspark.ml.recommendation module
---------------------------------
-
-.. automodule:: pyspark.ml.recommendation
-    :members:
-    :undoc-members:
-    :inherited-members:
-
-pyspark.ml.regression module
-----------------------------
-
-.. automodule:: pyspark.ml.regression
-    :members:
-    :undoc-members:
-    :inherited-members:
-
-pyspark.ml.stat module
-----------------------
-
-.. automodule:: pyspark.ml.stat
-    :members:
-    :undoc-members:
-    :inherited-members:
-
-pyspark.ml.tuning module
-------------------------
-
-.. automodule:: pyspark.ml.tuning
-    :members:
-    :undoc-members:
-    :inherited-members:
-
-pyspark.ml.evaluation module
-----------------------------
-
-.. automodule:: pyspark.ml.evaluation
-    :members:
-    :undoc-members:
-    :inherited-members:
-
-pyspark.ml.fpm module
-----------------------------
-
-.. automodule:: pyspark.ml.fpm
-    :members:
-    :undoc-members:
-    :inherited-members:
-
-pyspark.ml.image module
-----------------------------
-
-.. automodule:: pyspark.ml.image
-    :members:
-    :undoc-members:
-    :inherited-members:
-
-pyspark.ml.util module
-----------------------------
-
-.. automodule:: pyspark.ml.util
-    :members:
-    :undoc-members:
-    :inherited-members:
diff --git a/python/docs/pyspark.mllib.rst b/python/docs/pyspark.mllib.rst
deleted file mode 100644
index 2d54ab118b94b..0000000000000
--- a/python/docs/pyspark.mllib.rst
+++ /dev/null
@@ -1,99 +0,0 @@
-pyspark.mllib package
-=====================
-
-pyspark.mllib.classification module
------------------------------------
-
-.. automodule:: pyspark.mllib.classification
-    :members:
-    :undoc-members:
-    :inherited-members:
-
-pyspark.mllib.clustering module
--------------------------------
-
-.. automodule:: pyspark.mllib.clustering
-    :members:
-    :undoc-members:
-
-pyspark.mllib.evaluation module
--------------------------------
-
-.. automodule:: pyspark.mllib.evaluation
-      :members:
-      :undoc-members:
-
-pyspark.mllib.feature module
--------------------------------
-
-.. automodule:: pyspark.mllib.feature
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pyspark.mllib.fpm module
-------------------------
-
-.. automodule:: pyspark.mllib.fpm
-    :members:
-    :undoc-members:
-
-pyspark.mllib.linalg module
----------------------------
-
-.. automodule:: pyspark.mllib.linalg
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pyspark.mllib.linalg.distributed module
----------------------------------------
-
-.. automodule:: pyspark.mllib.linalg.distributed
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pyspark.mllib.random module
----------------------------
-
-.. automodule:: pyspark.mllib.random
-    :members:
-    :undoc-members:
-
-pyspark.mllib.recommendation module
------------------------------------
-
-.. automodule:: pyspark.mllib.recommendation
-    :members:
-    :undoc-members:
-
-pyspark.mllib.regression module
--------------------------------
-
-.. automodule:: pyspark.mllib.regression
-    :members:
-    :undoc-members:
-    :inherited-members:
-
-pyspark.mllib.stat module
--------------------------
-
-.. automodule:: pyspark.mllib.stat
-    :members:
-    :undoc-members:
-
-pyspark.mllib.tree module
--------------------------
-
-.. automodule:: pyspark.mllib.tree
-    :members:
-    :undoc-members:
-    :inherited-members:
-
-pyspark.mllib.util module
--------------------------
-
-.. automodule:: pyspark.mllib.util
-    :members:
-    :undoc-members:
diff --git a/python/docs/pyspark.rst b/python/docs/pyspark.rst
deleted file mode 100644
index 0df12c49ad033..0000000000000
--- a/python/docs/pyspark.rst
+++ /dev/null
@@ -1,20 +0,0 @@
-pyspark package
-===============
-
-Subpackages
------------
-
-.. toctree::
-    :maxdepth: 1
-    
-    pyspark.sql
-    pyspark.streaming
-    pyspark.ml
-    pyspark.mllib
-
-Contents
---------
-
-.. automodule:: pyspark
-    :members:
-    :undoc-members:
diff --git a/python/docs/pyspark.sql.rst b/python/docs/pyspark.sql.rst
deleted file mode 100644
index b69562e845920..0000000000000
--- a/python/docs/pyspark.sql.rst
+++ /dev/null
@@ -1,37 +0,0 @@
-pyspark.sql module
-==================
-
-Module Context
---------------
-
-.. automodule:: pyspark.sql
-    :members:
-    :undoc-members:
-    :inherited-members:
-    :exclude-members: builder
-.. We need `exclude-members` to prevent default description generations
-   as a workaround for old Sphinx (< 1.6.6).
-
-pyspark.sql.types module
-------------------------
-.. automodule:: pyspark.sql.types
-    :members:
-    :undoc-members:
-
-pyspark.sql.functions module
-----------------------------
-.. automodule:: pyspark.sql.functions
-    :members:
-    :undoc-members:
-
-pyspark.sql.avro.functions module
----------------------------------
-.. automodule:: pyspark.sql.avro.functions
-    :members:
-    :undoc-members:
-
-pyspark.sql.streaming module
-----------------------------
-.. automodule:: pyspark.sql.streaming
-    :members:
-    :undoc-members:
diff --git a/python/docs/pyspark.streaming.rst b/python/docs/pyspark.streaming.rst
deleted file mode 100644
index f7df6438b9169..0000000000000
--- a/python/docs/pyspark.streaming.rst
+++ /dev/null
@@ -1,17 +0,0 @@
-pyspark.streaming module
-========================
-
-Module contents
----------------
-
-.. automodule:: pyspark.streaming
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pyspark.streaming.kinesis module
---------------------------------
-.. automodule:: pyspark.streaming.kinesis
-    :members:
-    :undoc-members:
-    :show-inheritance:
diff --git a/python/docs/_static/copybutton.js b/python/docs/source/_static/copybutton.js
similarity index 100%
rename from python/docs/_static/copybutton.js
rename to python/docs/source/_static/copybutton.js
diff --git a/python/docs/source/_static/css/pyspark.css b/python/docs/source/_static/css/pyspark.css
new file mode 100644
index 0000000000000..1e493c4c868e6
--- /dev/null
+++ b/python/docs/source/_static/css/pyspark.css
@@ -0,0 +1,94 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+/* PySpark style CSS overwrite */
+
+/* Lato font (from its parent) does not make any difference when it's bold. Defaults to 'Source Sans Pro' */
+body {
+    font-family:"Source Sans Pro",sans-serif!important;
+}
+
+h1,h2 {
+    color:#1B5162!important;
+}
+
+h3 {
+    color: #555555
+}
+
+/* Top menu */
+#navbar-main {
+    background: #1B5162!important;
+    box-shadow: 0px 2px 4px rgba(0, 0, 0, 0.11);
+}
+
+#navbar-main-elements li.nav-item a {
+    color: rgba(255, 255, 255, 0.8);
+}
+
+#navbar-main-elements li.active a {
+    font-weight: 600;
+    color: #FFFFFF!important;
+}
+
+.col-9 {
+    flex: 0 0 80%;
+    max-width: 80%;
+}
+
+/* Left panel size */
+@media (min-width: 768px) {
+    .col-md-3 {
+        flex: 0 0 20%;
+        max-width: 20%;
+    }
+}
+
+/* Top menu right button */
+.navbar-toggler {
+    color:rgba(255,255,255,.5)!important;
+    border-color:rgba(255,255,255,.5)!important;
+}
+
+.navbar-toggler-icon {
+    background-image:url("data:image/svg+xml;charset=utf-8,%3Csvg xmlns='http://www.w3.org/2000/svg' width='30' height='30'%3E%3Cpath stroke='rgba(255,255,255,.5)' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3E%3C/svg%3E")!important;
+}
+
+/* Left bar list colors */
+.bd-sidebar .nav>.active>a {
+    color: #1B5162!important;
+}
+
+.bd-sidebar .nav>li>a:hover {
+    color: #1B5162!important;
+}
+
+.bd-sidebar .nav>.active:hover>a,.bd-sidebar .nav>.active>a {
+    color: #1B5162!important;
+}
+
+u.bd-sidebar .nav>li>ul>.active:hover>a,.bd-sidebar .nav>li>ul>.active>a {
+    color: #1B5162!important;
+}
+
+/* Right bar list colors */
+.toc-entry>.nav-link.active {
+    color: #1B5162!important;
+    border-left: 2px solid #1B5162!important;
+}
+
diff --git a/python/docs/source/_templates/autosummary/class.rst b/python/docs/source/_templates/autosummary/class.rst
new file mode 100644
index 0000000000000..b5f62677ee0ed
--- /dev/null
+++ b/python/docs/source/_templates/autosummary/class.rst
@@ -0,0 +1,38 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+.. Workaround to avoid documenting __init__.
+
+{% extends "!autosummary/class.rst" %}
+
+{% if '__init__' in methods %}
+{% set caught_result = methods.remove('__init__') %}
+{% endif %}
+    
+{% block methods %}
+{% if methods %}
+
+   .. rubric:: Methods
+
+   .. autosummary::
+      {% for item in methods %}
+      ~{{ name }}.{{ item }}
+      {%- endfor %}
+
+{% endif %}
+{% endblock %}
+
diff --git a/python/docs/source/_templates/autosummary/class_with_docs.rst b/python/docs/source/_templates/autosummary/class_with_docs.rst
new file mode 100644
index 0000000000000..7c37b83c0e90e
--- /dev/null
+++ b/python/docs/source/_templates/autosummary/class_with_docs.rst
@@ -0,0 +1,79 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+{{ objname }}
+{{ underline }}
+
+.. currentmodule:: {{ module }}
+
+.. autoclass:: {{ objname }}
+
+    {% if '__init__' in methods %}
+      {% set caught_result = methods.remove('__init__') %}
+    {% endif %}
+
+    {% block methods_summary %}
+    {% if methods %}
+
+    .. rubric:: Methods
+
+    .. autosummary::
+    {% for item in methods %}
+       ~{{ name }}.{{ item }}
+    {%- endfor %}
+
+    {% endif %}
+    {% endblock %}
+
+    {% block attributes_summary %}
+    {% if attributes %}
+
+    .. rubric:: Attributes
+
+    .. autosummary::
+    {% for item in attributes %}
+       ~{{ name }}.{{ item }}
+    {%- endfor %}
+
+    {% endif %}
+    {% endblock %}
+
+    {% block methods_documentation %}
+    {% if methods %}
+
+    .. rubric:: Methods Documentation
+
+    {% for item in methods %}
+    .. automethod:: {{ item }}
+    {%- endfor %}
+
+    {% endif %}
+    {% endblock %}
+
+    {% block attributes_documentation %}
+    {% if attributes %}
+
+    .. rubric:: Attributes Documentation
+
+    {% for item in attributes %}
+    .. autoattribute:: {{ item }}
+    {%- endfor %}
+
+    {% endif %}
+    {% endblock %}
+
diff --git a/python/docs/conf.py b/python/docs/source/conf.py
similarity index 81%
rename from python/docs/conf.py
rename to python/docs/source/conf.py
index 9e7afb7c07298..58ff5d968bc66 100644
--- a/python/docs/conf.py
+++ b/python/docs/source/conf.py
@@ -14,12 +14,24 @@
 
 import sys
 import os
+import shutil
+import errno
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 sys.path.insert(0, os.path.abspath('.'))
 
+# Remove previously generated rst files. Ignore errors just in case it stops
+# generating whole docs.
+shutil.rmtree(
+    "%s/reference/api" % os.path.dirname(os.path.abspath(__file__)), ignore_errors=True)
+try:
+    os.mkdir("%s/reference/api" % os.path.dirname(os.path.abspath(__file__)))
+except OSError as e:
+    if e.errno != errno.EEXIST:
+        raise
+
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -32,8 +44,31 @@
     'sphinx.ext.autodoc',
     'sphinx.ext.viewcode',
     'sphinx.ext.mathjax',
+    'sphinx.ext.autosummary',
+    'nbsphinx',  # Converts Jupyter Notebook to reStructuredText files for Sphinx.
+    # For ipython directive in reStructuredText files. It is generated by the notebook.
+    'IPython.sphinxext.ipython_console_highlighting',
+    'numpydoc',  # handle NumPy documentation formatted docstrings.
 ]
 
+numpydoc_show_class_members = False
+
+# Links used globally in the RST files.
+# These are defined here to allow link substitutions dynamically.
+rst_epilog = """
+.. |binder| replace:: Live Notebook
+.. _binder: https://mybinder.org/v2/gh/apache/spark/{0}?filepath=python%2Fdocs%2Fsource%2Fgetting_started%2Fquickstart.ipynb
+.. |examples| replace:: Examples
+.. _examples: https://github.com/apache/spark/tree/{0}/examples/src/main/python
+.. |downloading| replace:: Downloading
+.. _downloading: https://spark.apache.org/docs/{1}/building-spark.html
+.. |building_spark| replace:: Building Spark
+.. _building_spark: https://spark.apache.org/docs/{1}/#downloading
+""".format(
+    os.environ.get("GIT_HASH", "master"),
+    os.environ.get("RELEASE_VERSION", "latest"),
+)
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
@@ -47,8 +82,8 @@
 master_doc = 'index'
 
 # General information about the project.
-project = u'PySpark'
-copyright = u''
+project = 'PySpark'
+copyright = ''
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -71,7 +106,7 @@
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ['_build']
+exclude_patterns = ['_build', '.DS_Store', '**.ipynb_checkpoints']
 
 # The reST default role (used for this markup: `text`) to use for all
 # documents.
@@ -101,12 +136,13 @@
 
 # Look at the first line of the docstring for function and method signatures.
 autodoc_docstring_signature = True
+autosummary_generate = True
 
 # -- Options for HTML output ----------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = 'nature'
+html_theme = 'pydata_sphinx_theme'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -125,7 +161,7 @@
 
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
-html_logo = "../../docs/img/spark-logo-hd.png"
+html_logo = "../../../docs/img/spark-logo-reverse.png"
 
 # The name of an image file (within the static path) to use as favicon of the
 # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
@@ -137,6 +173,10 @@
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
 
+html_css_files = [
+    'css/pyspark.css',
+]
+
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
 # directly to the root of the documentation.
@@ -204,8 +244,8 @@
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-  ('index', 'pyspark.tex', u'pyspark Documentation',
-   u'Author', 'manual'),
+  ('index', 'pyspark.tex', 'pyspark Documentation',
+   'Author', 'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -234,8 +274,8 @@
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    ('index', 'pyspark', u'pyspark Documentation',
-     [u'Author'], 1)
+    ('index', 'pyspark', 'pyspark Documentation',
+     ['Author'], 1)
 ]
 
 # If true, show URL addresses after external links.
@@ -248,8 +288,8 @@
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  ('index', 'pyspark', u'pyspark Documentation',
-   u'Author', 'pyspark', 'One line description of project.',
+  ('index', 'pyspark', 'pyspark Documentation',
+   'Author', 'pyspark', 'One line description of project.',
    'Miscellaneous'),
 ]
 
@@ -269,13 +309,13 @@
 # -- Options for Epub output ----------------------------------------------
 
 # Bibliographic Dublin Core info.
-epub_title = u'pyspark'
-epub_author = u'Author'
-epub_publisher = u'Author'
-epub_copyright = u'2014, Author'
+epub_title = 'pyspark'
+epub_author = 'Author'
+epub_publisher = 'Author'
+epub_copyright = '2014, Author'
 
 # The basename for the epub file. It defaults to the project name.
-#epub_basename = u'pyspark'
+#epub_basename = 'pyspark'
 
 # The HTML theme for the epub output. Since the default themes are not optimized
 # for small screen space, using the same theme for HTML and epub output is
@@ -335,7 +375,8 @@
 # If false, no index is generated.
 #epub_use_index = True
 def setup(app):
-    app.add_javascript('copybutton.js')
+    # The app.add_javascript() is deprecated.
+    getattr(app, "add_js_file", getattr(app, "add_javascript"))('copybutton.js')
 
 # Skip sample endpoint link (not expected to resolve)
 linkcheck_ignore = [r'https://kinesis.us-east-1.amazonaws.com']
diff --git a/python/docs/source/development/contributing.rst b/python/docs/source/development/contributing.rst
new file mode 100644
index 0000000000000..4f0f9ae998f34
--- /dev/null
+++ b/python/docs/source/development/contributing.rst
@@ -0,0 +1,137 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+=======================
+Contributing to PySpark
+=======================
+
+There are many types of contribution, for example, helping other users, testing releases, reviewing changes,
+documentation contribution, bug reporting, JIRA maintenance, code changes, etc.
+These are documented at `the general guidelines <https://spark.apache.org/contributing.html>`_.
+This page focuses on PySpark and includes additional details specifically for PySpark.
+
+
+Contributing by Testing Releases
+--------------------------------
+
+Before the official release, PySpark release candidates are shared in the `dev@spark.apache.org <https://mail-archives.apache.org/mod_mbox/spark-dev/>`_ mailing list to vote on.
+This release candidates can be easily installed via pip. For example, in case of Spark 3.0.0 RC1, you can install as below:
+
+.. code-block:: bash
+
+    pip install https://dist.apache.org/repos/dist/dev/spark/v3.0.0-rc1-bin/pyspark-3.0.0.tar.gz
+
+The link for release files such as ``https://dist.apache.org/repos/dist/dev/spark/v3.0.0-rc1-bin`` can be found in the vote thread.
+
+Testing and verifying users' existing workloads against release candidates is one of the vital contributions to PySpark.
+It prevents breaking users' existing workloads before the official release.
+When there is an issue such as a regression, correctness problem or performance degradation worth enough to drop the release candidate,
+usually the release candidate is dropped and the community focuses on fixing it to include in the next release candidate.
+
+
+Contributing Documentation Changes
+----------------------------------
+
+The release documentation is located under Spark's `docs <https://github.com/apache/spark/tree/master/docs>`_ directory.
+`README.md <https://github.com/apache/spark/blob/master/docs/README.md>`_ describes the required dependencies and steps
+to generate the documentations. Usually, PySpark documentation is tested with the command below
+under the `docs <https://github.com/apache/spark/tree/master/docs>`_ directory:
+
+.. code-block:: bash
+
+    SKIP_SCALADOC=1 SKIP_RDOC=1 SKIP_SQLDOC=1 bundle exec jekyll serve --watch
+
+PySpark uses Sphinx to generate its release PySpark documentation. Therefore, if you want to build only PySpark documentation alone,
+you can build under `python/docs <https://github.com/apache/spark/tree/master/python>`_ directory by:
+
+.. code-block:: bash
+
+    make html
+
+It generates the corresponding HTMLs under ``python/docs/build/html``.
+
+Lastly, please make sure that the new APIs are documented by manually adding methods and/or classes at the corresponding RST files
+under ``python/docs/source/reference``. Otherwise, they would not be documented in PySpark documentation.
+
+
+Preparing to Contribute Code Changes
+------------------------------------
+
+Before starting to work on codes in PySpark, it is recommended to read `the general guidelines <https://spark.apache.org/contributing.html>`_.
+There are a couple of additional notes to keep in mind when contributing to codes in PySpark:
+
+* Be Pythonic.
+* APIs are matched with Scala and Java sides in general.
+* PySpark specific APIs can still be considered as long as they are Pythonic and do not conflict with other existent APIs, for example, decorator usage of UDFs.
+* If you extend or modify public API, please adjust corresponding type hints. See `Contributing and Maintaining Type Hints`_ for details.
+
+Contributing and Maintaining Type Hints
+----------------------------------------
+
+PySpark type hints are provided using stub files, placed in the same directory as the annotated module, with exception to ``# type: ignore`` in modules which don't have their own stubs (tests, examples and non-public API).
+As a rule of thumb, only public API is annotated.
+
+Annotations should, when possible:
+
+* Reflect expectations of the underlying JVM API, to help avoid type related failures outside Python interpreter.
+* In case of conflict between too broad (``Any``) and too narrow argument annotations, prefer the latter as one, as long as it is covering most of the typical use cases.
+* Indicate nonsensical combinations of arguments using ``@overload``  annotations. For example, to indicate that ``*Col`` and ``*Cols`` arguments are mutually exclusive:
+
+  .. code-block:: python
+
+    @overload
+    def __init__(
+        self,
+        *,
+        threshold: float = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> None: ...
+    @overload
+    def __init__(
+        self,
+        *,
+        thresholds: Optional[List[float]] = ...,
+        inputCols: Optional[List[str]] = ...,
+        outputCols: Optional[List[str]] = ...
+    ) -> None: ...
+
+* Be compatible with the current stable MyPy release.
+
+
+Complex supporting type definitions, should be placed in dedicated ``_typing.pyi`` stubs. See for example `pyspark.sql._typing.pyi <https://github.com/apache/spark/blob/master/python/pyspark/sql/_typing.pyi>`_.
+
+Annotations can be validated using ``dev/lint-python`` script or by invoking mypy directly:
+
+.. code-block:: bash
+
+    mypy --config python/mypy.ini python/pyspark
+
+
+
+Code and Docstring Guide
+----------------------------------
+
+Please follow the style of the existing codebase as is, which is virtually PEP 8 with one exception: lines can be up
+to 100 characters in length, not 79.
+For the docstring style, PySpark follows `NumPy documentation style <https://numpydoc.readthedocs.io/en/latest/format.html>`_.
+
+Note that the method and variable names in PySpark are the similar case is ``threading`` library in Python itself where
+the APIs were inspired by Java. PySpark also follows `camelCase` for exposed APIs that match with Scala and Java.
+There is an exception ``functions.py`` that uses `snake_case`. It was in order to make APIs SQL (and Python) friendly.
+
+PySpark leverages linters such as `pycodestyle <https://pycodestyle.pycqa.org/en/latest/>`_ and `flake8 <https://flake8.pycqa.org/en/latest/>`_, which ``dev/lint-python`` runs. Therefore, make sure to run that script to double check.
diff --git a/python/docs/source/development/debugging.rst b/python/docs/source/development/debugging.rst
new file mode 100644
index 0000000000000..829919858f67a
--- /dev/null
+++ b/python/docs/source/development/debugging.rst
@@ -0,0 +1,280 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+=================
+Debugging PySpark
+=================
+
+PySpark uses Spark as an engine. PySpark uses `Py4J <https://www.py4j.org/>`_ to leverage Spark to submit and computes the jobs.
+
+On the driver side, PySpark communicates with the driver on JVM by using `Py4J <https://www.py4j.org/>`_.
+When :class:`pyspark.sql.SparkSession` or :class:`pyspark.SparkContext` is created and initialized, PySpark launches a JVM
+to communicate.
+
+On the executor side, Python workers execute and handle Python native functions or data. They are not launched if
+a PySpark application does not require interaction between Python workers and JVMs. They are lazily launched only when
+Python native functions or data have to be handled, for example, when you execute pandas UDFs or
+PySpark RDD APIs.
+
+This page focuses on debugging Python side of PySpark on both driver and executor sides instead of focusing on debugging
+with JVM. Profiling and debugging JVM is described at `Useful Developer Tools <https://spark.apache.org/developer-tools.html>`_.
+
+Note that,
+
+- If you are running locally, you can directly debug the driver side via using your IDE without the remote debug feature. Setting PySpark with IDEs is documented `here <setting_ide.rst#pycharm>`__.
+- *There are many other ways of debugging PySpark applications*. For example, you can remotely debug by using the open source `Remote Debugger <https://www.pydev.org/manual_adv_remote_debugger.html>`_ instead of using PyCharm Professional documented here.
+
+
+Remote Debugging (PyCharm Professional)
+---------------------------------------
+
+This section describes remote debugging on both driver and executor sides within a single machine to demonstrate easily.
+The ways of debugging PySpark on the executor side is different from doing in the driver. Therefore, they will be demonstrated respectively.
+In order to debug PySpark applications on other machines, please refer to the full instructions that are specific
+to PyCharm, documented `here <https://www.jetbrains.com/help/pycharm/remote-debugging-with-product.html>`_.
+
+Firstly, choose **Edit Configuration...** from the *Run* menu. It opens the **Run/Debug Configurations dialog**.
+You have to click ``+`` configuration on the toolbar, and from the list of available configurations, select **Python Debug Server**.
+Enter the name of this new configuration, for example, ``MyRemoteDebugger`` and also specify the port number, for example ``12345``.
+
+.. image:: ../../../../docs/img/pyspark-remote-debug1.png
+    :alt: PyCharm remote debugger setting
+
+| After that, you should install the corresponding version of the ``pydevd-pycharm`` package in all the machines which will connect to your PyCharm debugger. In the previous dialog, it shows the command to install.
+
+.. code-block:: text
+
+    pip install pydevd-pycharm~=<version of PyCharm on the local machine>
+
+Driver Side
+~~~~~~~~~~~
+
+To debug on the driver side, your application should be able to connect to the debugging server. Copy and paste the codes
+with ``pydevd_pycharm.settrace`` to the top of your PySpark script. Suppose the script name is ``app.py``:
+
+.. code-block:: bash
+
+    echo "#======================Copy and paste from the previous dialog===========================
+    import pydevd_pycharm
+    pydevd_pycharm.settrace('localhost', port=12345, stdoutToServer=True, stderrToServer=True)
+    #========================================================================================
+    # Your PySpark application codes:
+    from pyspark.sql import SparkSession
+    spark = SparkSession.builder.getOrCreate()
+    spark.range(10).show()" > app.py
+
+Start to debug with your ``MyRemoteDebugger``.
+
+.. image:: ../../../../docs/img/pyspark-remote-debug2.png
+    :alt: PyCharm run remote debugger
+
+| After that, submit your application. This will connect to your PyCharm debugging server and enable you to debug on the driver side remotely.
+
+.. code-block:: bash
+
+    spark-submit app.py
+
+Executor Side
+~~~~~~~~~~~~~
+
+To debug on the executor side, prepare a Python file as below in your current working directory.
+
+.. code-block:: bash
+
+    echo "from pyspark import daemon, worker
+    def remote_debug_wrapped(*args, **kwargs):
+        #======================Copy and paste from the previous dialog===========================
+        import pydevd_pycharm
+        pydevd_pycharm.settrace('localhost', port=12345, stdoutToServer=True, stderrToServer=True)
+        #========================================================================================
+        worker.main(*args, **kwargs)
+    daemon.worker_main = remote_debug_wrapped
+    if __name__ == '__main__':
+        daemon.manager()" > remote_debug.py
+
+You will use this file as the Python worker in your PySpark applications by using the ``spark.python.daemon.module`` configuration.
+Run the ``pyspark`` shell with the configuration below:
+
+.. code-block:: bash
+
+    pyspark --conf spark.python.daemon.module=remote_debug
+
+Now you're ready to remotely debug. Start to debug with your ``MyRemoteDebugger``.
+
+.. image:: ../../../../docs/img/pyspark-remote-debug2.png
+    :alt: PyCharm run remote debugger
+
+| After that, run a job that creates Python workers, for example, as below:
+
+.. code-block:: python
+
+    spark.range(10).repartition(1).rdd.map(lambda x: x).collect()
+
+
+Checking Resource Usage (``top`` and ``ps``)
+--------------------------------------------
+
+The Python processes on the driver and executor can be checked via typical ways such as ``top`` and ``ps`` commands.
+
+Driver Side
+~~~~~~~~~~~
+
+On the driver side, you can get the process id from your PySpark shell easily as below to know the process id and resources.
+
+.. code-block:: python
+
+    >>> import os; os.getpid()
+    18482
+
+.. code-block:: bash
+
+    ps -fe 18482
+
+.. code-block:: text
+
+    UID   PID  PPID   C STIME  TTY           TIME CMD
+    000 18482 12345   0 0:00PM ttys001    0:00.00 /.../python
+
+Executor Side
+~~~~~~~~~~~~~
+
+To check on the executor side, you can simply ``grep`` them to figure out the process
+ids and relevant resources because Python workers are forked from ``pyspark.daemon``.
+
+.. code-block:: bash
+
+    ps -fe | grep pyspark.daemon
+
+.. code-block:: text
+
+    000 12345     1   0  0:00PM ttys000    0:00.00 /.../python -m pyspark.daemon
+    000 12345     1   0  0:00PM ttys000    0:00.00 /.../python -m pyspark.daemon
+    000 12345     1   0  0:00PM ttys000    0:00.00 /.../python -m pyspark.daemon
+    000 12345     1   0  0:00PM ttys000    0:00.00 /.../python -m pyspark.daemon
+    ...
+
+
+Profiling Memory Usage (Memory Profiler)
+----------------------------------------
+
+`memory_profiler <https://github.com/pythonprofilers/memory_profiler>`_ is one of the profilers that allow you to
+check the memory usage line by line. This method documented here *only works for the driver side*.
+
+Unless you are running your driver program in another machine (e.g., YARN cluster mode), this useful tool can be used
+to debug the memory usage on driver side easily. Suppose your PySpark script name is ``profile_memory.py``.
+You can profile it as below.
+
+.. code-block:: bash
+
+    echo "from pyspark.sql import SparkSession
+    #===Your function should be decorated with @profile===
+    from memory_profiler import profile
+    @profile
+    #=====================================================
+    def my_func():
+        session = SparkSession.builder.getOrCreate()
+        df = session.range(10000)
+        return df.collect()
+    if __name__ == '__main__':
+        my_func()" > profile_memory.py
+
+.. code-block:: bash
+
+    python -m memory_profiler profile_memory.py
+
+.. code-block:: text
+
+    Filename: profile_memory.py
+
+    Line #    Mem usage    Increment   Line Contents
+    ================================================
+    ...
+         6                             def my_func():
+         7     51.5 MiB      0.6 MiB       session = SparkSession.builder.getOrCreate()
+         8     51.5 MiB      0.0 MiB       df = session.range(10000)
+         9     54.4 MiB      2.8 MiB       return df.collect()
+
+
+Identifying Hot Loops (Python Profilers)
+----------------------------------------
+
+`Python Profilers <https://docs.python.org/3/library/profile.html>`_ are useful built-in features in Python itself. These
+provide deterministic profiling of Python programs with a lot of useful statistics. This section describes how to use it on
+both driver and executor sides in order to identify expensive or hot code paths.
+
+Driver Side
+~~~~~~~~~~~
+
+To use this on driver side, you can use it as you would do for regular Python programs because PySpark on driver side is a
+regular Python process unless you are running your driver program in another machine (e.g., YARN cluster mode).
+
+.. code-block:: bash
+
+    echo "from pyspark.sql import SparkSession
+    spark = SparkSession.builder.getOrCreate()
+    spark.range(10).show()" > app.py
+
+.. code-block:: bash
+
+    python -m cProfile app.py
+
+.. code-block:: text
+
+    ...
+         129215 function calls (125446 primitive calls) in 5.926 seconds
+
+       Ordered by: standard name
+
+       ncalls  tottime  percall  cumtime  percall filename:lineno(function)
+     1198/405    0.001    0.000    0.083    0.000 <frozen importlib._bootstrap>:1009(_handle_fromlist)
+          561    0.001    0.000    0.001    0.000 <frozen importlib._bootstrap>:103(release)
+          276    0.000    0.000    0.000    0.000 <frozen importlib._bootstrap>:143(__init__)
+          276    0.000    0.000    0.002    0.000 <frozen importlib._bootstrap>:147(__enter__)
+    ...
+
+Executor Side
+~~~~~~~~~~~~~
+
+To use this on executor side, PySpark provides remote `Python Profilers <https://docs.python.org/3/library/profile.html>`_ for
+executor side, which can be enabled by setting ``spark.python.profile`` configuration to ``true``.
+
+.. code-block:: bash
+
+    pyspark --conf spark.python.profile=true
+
+
+.. code-block:: python
+
+    >>> rdd = sc.parallelize(range(100)).map(str)
+    >>> rdd.count()
+    100
+    >>> sc.show_profiles()
+    ============================================================
+    Profile of RDD<id=1>
+    ============================================================
+             728 function calls (692 primitive calls) in 0.004 seconds
+
+       Ordered by: internal time, cumulative time
+
+       ncalls  tottime  percall  cumtime  percall filename:lineno(function)
+           12    0.001    0.000    0.001    0.000 serializers.py:210(load_stream)
+           12    0.000    0.000    0.000    0.000 {built-in method _pickle.dumps}
+           12    0.000    0.000    0.001    0.000 serializers.py:252(dump_stream)
+           12    0.000    0.000    0.001    0.000 context.py:506(f)
+    ...
+
+This feature is supported only with RDD APIs.
diff --git a/python/docs/source/development/index.rst b/python/docs/source/development/index.rst
new file mode 100644
index 0000000000000..bf8b2ccafaa9e
--- /dev/null
+++ b/python/docs/source/development/index.rst
@@ -0,0 +1,28 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+===========
+Development
+===========
+
+.. toctree::
+    :maxdepth: 2
+
+    contributing
+    testing
+    debugging
+    setting_ide
diff --git a/python/docs/source/development/setting_ide.rst b/python/docs/source/development/setting_ide.rst
new file mode 100644
index 0000000000000..6e8f0148c6eb3
--- /dev/null
+++ b/python/docs/source/development/setting_ide.rst
@@ -0,0 +1,62 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+===============
+Setting up IDEs
+===============
+
+
+PyCharm
+-------
+
+This section describes how to setup PySpark on PyCharm.
+It guides step by step to the process of downloading the source code from GitHub and running the test code successfully.
+
+Firstly, download the Spark source code from GitHub using git url. You can download the source code by simply using ``git clone`` command as shown below.
+If you want to download the code from any forked repository rather than Spark original repository, please change the url properly.
+
+.. code-block:: bash
+
+    git clone https://github.com/apache/spark.git
+
+When the download is completed, go to the ``spark`` directory and build the package.
+SBT build is generally much faster than Maven. More details about the build are documented `here <https://spark.apache.org/docs/latest/building-spark.html>`_.
+
+.. code-block:: bash
+
+    build/sbt package
+
+After building is finished, run PyCharm and select the path ``spark/python``.
+
+.. image:: ../../../../docs/img/pycharm-with-pyspark1.png
+    :alt: Select the Spark path
+
+
+Let's go to the path ``python/pyspark/tests`` in PyCharm and try to run the any test like ``test_join.py``.
+You might can see the ``KeyError: 'SPARK_HOME'`` because the environment variable has not been set yet.
+
+Go **Run -> Edit Configurations**, and set the environment variables as below.
+Please make sure to specify your own path for ``SPARK_HOME`` rather than ``/.../spark``. After completing the variable, click **OK** to apply the changes.
+
+.. image:: ../../../../docs/img/pycharm-with-pyspark2.png
+    :alt: Setting up SPARK_HOME
+
+
+Once ``SPARK_HOME`` is set properly, you'll be able to run the tests properly as below:
+
+.. image:: ../../../../docs/img/pycharm-with-pyspark3.png
+    :alt: Running tests properly
diff --git a/python/docs/source/development/testing.rst b/python/docs/source/development/testing.rst
new file mode 100644
index 0000000000000..3eab8d04511d6
--- /dev/null
+++ b/python/docs/source/development/testing.rst
@@ -0,0 +1,57 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+===============
+Testing PySpark
+===============
+
+In order to run PySpark tests, you should build Spark itself first via Maven or SBT. For example,
+
+.. code-block:: bash
+
+    build/mvn -DskipTests clean package
+
+After that, the PySpark test cases can be run via using ``python/run-tests``. For example,
+
+.. code-block:: bash
+
+    python/run-tests --python-executable=python3
+
+Note that you may set ``OBJC_DISABLE_INITIALIZE_FORK_SAFETY`` environment variable to ``YES`` if you are running tests on Mac OS.
+
+Please see the guidance on how to `build Spark <https://github.com/apache/spark#building-spark>`_,
+`run tests for a module, or individual tests <https://spark.apache.org/developer-tools.html>`_.
+
+
+Running Individual PySpark Tests
+--------------------------------
+
+You can run a specific test via using ``python/run-tests``, for example, as below:
+
+.. code-block:: bash
+
+    python/run-tests --testnames pyspark.sql.tests.test_arrow
+
+Please refer to `Testing PySpark <https://spark.apache.org/developer-tools.html>`_ for more details.
+
+
+Running tests using GitHub Actions
+----------------------------------
+
+You can run the full PySpark tests by using GitHub Actions in your own forked GitHub
+repository with a few clicks. Please refer to
+`Running tests in your forked repository using GitHub Actions <https://spark.apache.org/developer-tools.html>`_ for more details.
diff --git a/python/docs/source/getting_started/index.rst b/python/docs/source/getting_started/index.rst
new file mode 100644
index 0000000000000..f6d7a92ced03f
--- /dev/null
+++ b/python/docs/source/getting_started/index.rst
@@ -0,0 +1,32 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+===============
+Getting Started
+===============
+
+This page summarizes the basic steps required to setup and get started with PySpark.
+There are more guides shared with other languages such as
+`Quick Start <https://spark.apache.org/docs/latest/quick-start.html>`_ in Programming Guides
+at `the Spark documentation <https://spark.apache.org/docs/latest/index.html#where-to-go-from-here>`_.
+
+.. toctree::
+    :maxdepth: 2
+
+    install
+    quickstart
diff --git a/python/docs/source/getting_started/install.rst b/python/docs/source/getting_started/install.rst
new file mode 100644
index 0000000000000..1d008c101e408
--- /dev/null
+++ b/python/docs/source/getting_started/install.rst
@@ -0,0 +1,177 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+============
+Installation
+============
+
+PySpark is included in the official releases of Spark available in the `Apache Spark website <https://spark.apache.org/downloads.html>`_.
+For Python users, PySpark also provides ``pip`` installation from PyPI. This is usually for local usage or as
+a client to connect to a cluster instead of setting up a cluster itself.
+ 
+This page includes instructions for installing PySpark by using pip, Conda, downloading manually,
+and building from the source.
+
+
+Python Version Supported
+------------------------
+
+Python 3.6 and above.
+
+
+Using PyPI
+----------
+
+PySpark installation using `PyPI <https://pypi.org/project/pyspark/>`_ is as follows:
+
+.. code-block:: bash
+
+    pip install pyspark
+
+If you want to install extra dependencies for a specific component, you can install it as below:
+
+.. code-block:: bash
+
+    pip install pyspark[sql]
+
+For PySpark with/without a specific Hadoop version, you can install it by using ``PYSPARK_HADOOP_VERSION`` environment variables as below:
+
+.. code-block:: bash
+
+    PYSPARK_HADOOP_VERSION=2.7 pip install pyspark
+
+The default distribution uses Hadoop 3.2 and Hive 2.3. If users specify different versions of Hadoop, the pip installation automatically
+downloads a different version and use it in PySpark. Downloading it can take a while depending on
+the network and the mirror chosen. ``PYSPARK_RELEASE_MIRROR`` can be set to manually choose the mirror for faster downloading.
+
+.. code-block:: bash
+
+    PYSPARK_RELEASE_MIRROR=http://mirror.apache-kr.org PYSPARK_HADOOP_VERSION=2.7 pip install
+
+It is recommended to use ``-v`` option in ``pip`` to track the installation and download status.
+
+.. code-block:: bash
+
+    PYSPARK_HADOOP_VERSION=2.7 pip install pyspark -v
+
+Supported values in ``PYSPARK_HADOOP_VERSION`` are:
+
+- ``without``: Spark pre-built with user-provided Apache Hadoop
+- ``2.7``: Spark pre-built for Apache Hadoop 2.7
+- ``3.2``: Spark pre-built for Apache Hadoop 3.2 and later (default)
+
+Note that this installation way of PySpark with/without a specific Hadoop version is experimental. It can change or be removed between minor releases.
+
+
+Using Conda
+-----------
+
+Conda is an open-source package management and environment management system which is a part of
+the `Anaconda <https://docs.continuum.io/anaconda/>`_ distribution. It is both cross-platform and
+language agnostic. In practice, Conda can replace both `pip <https://pip.pypa.io/en/latest/>`_ and
+`virtualenv <https://virtualenv.pypa.io/en/latest/>`_.
+
+Create new virtual environment from your terminal as shown below:
+
+.. code-block:: bash
+
+    conda create -n pyspark_env
+
+After the virtual environment is created, it should be visible under the list of Conda environments
+which can be seen using the following command:
+
+.. code-block:: bash
+
+    conda env list
+
+Now activate the newly created environment with the following command:
+
+.. code-block:: bash
+
+    conda activate pyspark_env
+
+You can install pyspark by `Using PyPI <#using-pypi>`_ to install PySpark in the newly created
+environment, for example as below. It will install PySpark under the new virtual environment
+``pyspark_env`` created above.
+
+.. code-block:: bash
+
+    pip install pyspark
+
+Alternatively, you can install PySpark from Conda itself as below:
+
+.. code-block:: bash
+
+    conda install pyspark
+
+However, note that `PySpark at Conda <https://anaconda.org/conda-forge/pyspark>`_ is not necessarily
+synced with PySpark release cycle because it is maintained by the community separately.
+
+
+Manually Downloading
+--------------------
+
+PySpark is included in the distributions available at the `Apache Spark website <https://spark.apache.org/downloads.html>`_.
+You can download a distribution you want from the site. After that, uncompress the tar file into the directory where you want
+to install Spark, for example, as below:
+
+.. code-block:: bash
+
+    tar xzvf spark-3.0.0-bin-hadoop2.7.tgz
+
+Ensure the ``SPARK_HOME`` environment variable points to the directory where the tar file has been extracted.
+Update ``PYTHONPATH`` environment variable such that it can find the PySpark and Py4J under ``SPARK_HOME/python/lib``.
+One example of doing this is shown below:
+
+.. code-block:: bash
+
+    cd spark-3.0.0-bin-hadoop2.7
+    export SPARK_HOME=`pwd`
+    export PYTHONPATH=$(ZIPS=("$SPARK_HOME"/python/lib/*.zip); IFS=:; echo "${ZIPS[*]}"):$PYTHONPATH
+
+
+Installing from Source
+----------------------
+
+To install PySpark from source, refer to |building_spark|_.
+
+
+Dependencies
+------------
+============= ========================= ======================================
+Package       Minimum supported version Note
+============= ========================= ======================================
+`pandas`      0.23.2                    Optional for Spark SQL
+`NumPy`       1.7                       Required for MLlib DataFrame-based API
+`pyarrow`     1.0.0                     Optional for Spark SQL
+`Py4J`        0.10.9.2                  Required
+`pandas`      0.23.2                    Required for pandas APIs on Spark
+`pyarrow`     1.0.0                     Required for pandas APIs on Spark
+`Numpy`       1.14(<1.20.0)             Required for pandas APIs on Spark
+============= ========================= ======================================
+
+Note that PySpark requires Java 8 or later with ``JAVA_HOME`` properly set.  
+If using JDK 11, set ``-Dio.netty.tryReflectionSetAccessible=true`` for Arrow related features and refer
+to |downloading|_.
+
+Note for AArch64 (ARM64) users: PyArrow is required by PySpark SQL, but PyArrow support for AArch64
+is introduced in PyArrow 4.0.0. If PySpark installation fails on AArch64 due to PyArrow
+installation errors, you can install PyArrow >= 4.0.0 as below:
+
+.. code-block:: bash
+
+    pip install "pyarrow>=4.0.0" --prefer-binary
diff --git a/python/docs/source/getting_started/quickstart.ipynb b/python/docs/source/getting_started/quickstart.ipynb
new file mode 100644
index 0000000000000..550b532fefc14
--- /dev/null
+++ b/python/docs/source/getting_started/quickstart.ipynb
@@ -0,0 +1,1177 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Quickstart\n",
+    "\n",
+    "This is a short introduction and quickstart for the PySpark DataFrame API. PySpark DataFrames are lazily evaluated. They are implemented on top of [RDD](https://spark.apache.org/docs/latest/rdd-programming-guide.html#overview)s. When Spark [transforms](https://spark.apache.org/docs/latest/rdd-programming-guide.html#transformations) data, it does not immediately compute the transformation but plans how to compute later. When [actions](https://spark.apache.org/docs/latest/rdd-programming-guide.html#actions) such as `collect()` are explicitly called, the computation starts.\n",
+    "This notebook shows the basic usages of the DataFrame, geared mainly for new users. You can run the latest version of these examples by yourself on a live notebook [here](https://mybinder.org/v2/gh/apache/spark/master?filepath=python%2Fdocs%2Fsource%2Fgetting_started%2Fquickstart.ipynb).\n",
+    "\n",
+    "There is also other useful information in Apache Spark documentation site, see the latest version of [Spark SQL and DataFrames](https://spark.apache.org/docs/latest/sql-programming-guide.html), [RDD Programming Guide](https://spark.apache.org/docs/latest/rdd-programming-guide.html), [Structured Streaming Programming Guide](https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html), [Spark Streaming Programming Guide](https://spark.apache.org/docs/latest/streaming-programming-guide.html) and [Machine Learning Library (MLlib) Guide](https://spark.apache.org/docs/latest/ml-guide.html).\n",
+    "\n",
+    "PySpark applications start with initializing `SparkSession` which is the entry point of PySpark as below. In case of running it in PySpark shell via <code>pyspark</code> executable, the shell automatically creates the session in the variable <code>spark</code> for users."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pyspark.sql import SparkSession\n",
+    "\n",
+    "spark = SparkSession.builder.getOrCreate()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## DataFrame Creation\n",
+    "\n",
+    "A PySpark DataFrame can be created via `pyspark.sql.SparkSession.createDataFrame` typically by passing a list of lists, tuples, dictionaries and `pyspark.sql.Row`s, a [pandas DataFrame](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html) and an RDD consisting of such a list.\n",
+    "`pyspark.sql.SparkSession.createDataFrame` takes the `schema` argument to specify the schema of the DataFrame. When it is omitted, PySpark infers the corresponding schema by taking a sample from the data.\n",
+    "\n",
+    "Firstly, you can create a PySpark DataFrame from a list of rows"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DataFrame[a: bigint, b: double, c: string, d: date, e: timestamp]"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from datetime import datetime, date\n",
+    "import pandas as pd\n",
+    "from pyspark.sql import Row\n",
+    "\n",
+    "df = spark.createDataFrame([\n",
+    "    Row(a=1, b=2., c='string1', d=date(2000, 1, 1), e=datetime(2000, 1, 1, 12, 0)),\n",
+    "    Row(a=2, b=3., c='string2', d=date(2000, 2, 1), e=datetime(2000, 1, 2, 12, 0)),\n",
+    "    Row(a=4, b=5., c='string3', d=date(2000, 3, 1), e=datetime(2000, 1, 3, 12, 0))\n",
+    "])\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Create a PySpark DataFrame with an explicit schema."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DataFrame[a: bigint, b: double, c: string, d: date, e: timestamp]"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df = spark.createDataFrame([\n",
+    "    (1, 2., 'string1', date(2000, 1, 1), datetime(2000, 1, 1, 12, 0)),\n",
+    "    (2, 3., 'string2', date(2000, 2, 1), datetime(2000, 1, 2, 12, 0)),\n",
+    "    (3, 4., 'string3', date(2000, 3, 1), datetime(2000, 1, 3, 12, 0))\n",
+    "], schema='a long, b double, c string, d date, e timestamp')\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Create a PySpark DataFrame from a pandas DataFrame"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DataFrame[a: bigint, b: double, c: string, d: date, e: timestamp]"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "pandas_df = pd.DataFrame({\n",
+    "    'a': [1, 2, 3],\n",
+    "    'b': [2., 3., 4.],\n",
+    "    'c': ['string1', 'string2', 'string3'],\n",
+    "    'd': [date(2000, 1, 1), date(2000, 2, 1), date(2000, 3, 1)],\n",
+    "    'e': [datetime(2000, 1, 1, 12, 0), datetime(2000, 1, 2, 12, 0), datetime(2000, 1, 3, 12, 0)]\n",
+    "})\n",
+    "df = spark.createDataFrame(pandas_df)\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Create a PySpark DataFrame from an RDD consisting of a list of tuples."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DataFrame[a: bigint, b: double, c: string, d: date, e: timestamp]"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "rdd = spark.sparkContext.parallelize([\n",
+    "    (1, 2., 'string1', date(2000, 1, 1), datetime(2000, 1, 1, 12, 0)),\n",
+    "    (2, 3., 'string2', date(2000, 2, 1), datetime(2000, 1, 2, 12, 0)),\n",
+    "    (3, 4., 'string3', date(2000, 3, 1), datetime(2000, 1, 3, 12, 0))\n",
+    "])\n",
+    "df = spark.createDataFrame(rdd, schema=['a', 'b', 'c', 'd', 'e'])\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The DataFrames created above all have the same results and schema."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "+---+---+-------+----------+-------------------+\n",
+      "|  a|  b|      c|         d|                  e|\n",
+      "+---+---+-------+----------+-------------------+\n",
+      "|  1|2.0|string1|2000-01-01|2000-01-01 12:00:00|\n",
+      "|  2|3.0|string2|2000-02-01|2000-01-02 12:00:00|\n",
+      "|  3|4.0|string3|2000-03-01|2000-01-03 12:00:00|\n",
+      "+---+---+-------+----------+-------------------+\n",
+      "\n",
+      "root\n",
+      " |-- a: long (nullable = true)\n",
+      " |-- b: double (nullable = true)\n",
+      " |-- c: string (nullable = true)\n",
+      " |-- d: date (nullable = true)\n",
+      " |-- e: timestamp (nullable = true)\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "# All DataFrames above result same.\n",
+    "df.show()\n",
+    "df.printSchema()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Viewing Data\n",
+    "\n",
+    "The top rows of a DataFrame can be displayed using `DataFrame.show()`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "+---+---+-------+----------+-------------------+\n",
+      "|  a|  b|      c|         d|                  e|\n",
+      "+---+---+-------+----------+-------------------+\n",
+      "|  1|2.0|string1|2000-01-01|2000-01-01 12:00:00|\n",
+      "+---+---+-------+----------+-------------------+\n",
+      "only showing top 1 row\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "df.show(1)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Alternatively, you can enable `spark.sql.repl.eagerEval.enabled` configuration for the eager evaluation of PySpark DataFrame in notebooks such as Jupyter. The number of rows to show can be controlled via `spark.sql.repl.eagerEval.maxNumRows` configuration."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table border='1'>\n",
+       "<tr><th>a</th><th>b</th><th>c</th><th>d</th><th>e</th></tr>\n",
+       "<tr><td>1</td><td>2.0</td><td>string1</td><td>2000-01-01</td><td>2000-01-01 12:00:00</td></tr>\n",
+       "<tr><td>2</td><td>3.0</td><td>string2</td><td>2000-02-01</td><td>2000-01-02 12:00:00</td></tr>\n",
+       "<tr><td>3</td><td>4.0</td><td>string3</td><td>2000-03-01</td><td>2000-01-03 12:00:00</td></tr>\n",
+       "</table>\n"
+      ],
+      "text/plain": [
+       "DataFrame[a: bigint, b: double, c: string, d: date, e: timestamp]"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "spark.conf.set('spark.sql.repl.eagerEval.enabled', True)\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The rows can also be shown vertically. This is useful when rows are too long to show horizontally."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "-RECORD 0------------------\n",
+      " a   | 1                   \n",
+      " b   | 2.0                 \n",
+      " c   | string1             \n",
+      " d   | 2000-01-01          \n",
+      " e   | 2000-01-01 12:00:00 \n",
+      "only showing top 1 row\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "df.show(1, vertical=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You can see the DataFrame's schema and column names as follows:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['a', 'b', 'c', 'd', 'e']"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.columns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "root\n",
+      " |-- a: long (nullable = true)\n",
+      " |-- b: double (nullable = true)\n",
+      " |-- c: string (nullable = true)\n",
+      " |-- d: date (nullable = true)\n",
+      " |-- e: timestamp (nullable = true)\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "df.printSchema()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Show the summary of the DataFrame"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "+-------+---+---+-------+\n",
+      "|summary|  a|  b|      c|\n",
+      "+-------+---+---+-------+\n",
+      "|  count|  3|  3|      3|\n",
+      "|   mean|2.0|3.0|   null|\n",
+      "| stddev|1.0|1.0|   null|\n",
+      "|    min|  1|2.0|string1|\n",
+      "|    max|  3|4.0|string3|\n",
+      "+-------+---+---+-------+\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "df.select(\"a\", \"b\", \"c\").describe().show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "`DataFrame.collect()` collects the distributed data to the driver side as the local data in Python. Note that this can throw an out-of-memory error when the dataset is too large to fit in the driver side because it collects all the data from executors to the driver side."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Row(a=1, b=2.0, c='string1', d=datetime.date(2000, 1, 1), e=datetime.datetime(2000, 1, 1, 12, 0)),\n",
+       " Row(a=2, b=3.0, c='string2', d=datetime.date(2000, 2, 1), e=datetime.datetime(2000, 1, 2, 12, 0)),\n",
+       " Row(a=3, b=4.0, c='string3', d=datetime.date(2000, 3, 1), e=datetime.datetime(2000, 1, 3, 12, 0))]"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.collect()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "In order to avoid throwing an out-of-memory exception, use `DataFrame.take()` or `DataFrame.tail()`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Row(a=1, b=2.0, c='string1', d=datetime.date(2000, 1, 1), e=datetime.datetime(2000, 1, 1, 12, 0))]"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.take(1)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "PySpark DataFrame also provides the conversion back to a [pandas DataFrame](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html) to leverage pandas APIs. Note that `toPandas` also collects all data into the driver side that can easily cause an out-of-memory-error when the data is too large to fit into the driver side."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>a</th>\n",
+       "      <th>b</th>\n",
+       "      <th>c</th>\n",
+       "      <th>d</th>\n",
+       "      <th>e</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>1</td>\n",
+       "      <td>2.0</td>\n",
+       "      <td>string1</td>\n",
+       "      <td>2000-01-01</td>\n",
+       "      <td>2000-01-01 12:00:00</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2</td>\n",
+       "      <td>3.0</td>\n",
+       "      <td>string2</td>\n",
+       "      <td>2000-02-01</td>\n",
+       "      <td>2000-01-02 12:00:00</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>3</td>\n",
+       "      <td>4.0</td>\n",
+       "      <td>string3</td>\n",
+       "      <td>2000-03-01</td>\n",
+       "      <td>2000-01-03 12:00:00</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   a    b        c           d                   e\n",
+       "0  1  2.0  string1  2000-01-01 2000-01-01 12:00:00\n",
+       "1  2  3.0  string2  2000-02-01 2000-01-02 12:00:00\n",
+       "2  3  4.0  string3  2000-03-01 2000-01-03 12:00:00"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.toPandas()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Selecting and Accessing Data\n",
+    "\n",
+    "PySpark DataFrame is lazily evaluated and simply selecting a column does not trigger the computation but it returns a `Column` instance."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Column<b'a'>"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.a"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "In fact, most of column-wise operations return `Column`s."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from pyspark.sql import Column\n",
+    "from pyspark.sql.functions import upper\n",
+    "\n",
+    "type(df.c) == type(upper(df.c)) == type(df.c.isNull())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "These `Column`s can be used to select the columns from a DataFrame. For example, `DataFrame.select()` takes the `Column` instances that returns another DataFrame."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "+-------+\n",
+      "|      c|\n",
+      "+-------+\n",
+      "|string1|\n",
+      "|string2|\n",
+      "|string3|\n",
+      "+-------+\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "df.select(df.c).show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Assign new `Column` instance."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "+---+---+-------+----------+-------------------+-------+\n",
+      "|  a|  b|      c|         d|                  e|upper_c|\n",
+      "+---+---+-------+----------+-------------------+-------+\n",
+      "|  1|2.0|string1|2000-01-01|2000-01-01 12:00:00|STRING1|\n",
+      "|  2|3.0|string2|2000-02-01|2000-01-02 12:00:00|STRING2|\n",
+      "|  3|4.0|string3|2000-03-01|2000-01-03 12:00:00|STRING3|\n",
+      "+---+---+-------+----------+-------------------+-------+\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "df.withColumn('upper_c', upper(df.c)).show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "To select a subset of rows, use `DataFrame.filter()`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "+---+---+-------+----------+-------------------+\n",
+      "|  a|  b|      c|         d|                  e|\n",
+      "+---+---+-------+----------+-------------------+\n",
+      "|  1|2.0|string1|2000-01-01|2000-01-01 12:00:00|\n",
+      "+---+---+-------+----------+-------------------+\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "df.filter(df.a == 1).show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Applying a Function\n",
+    "\n",
+    "PySpark supports various UDFs and APIs to allow users to execute Python native functions. See also the latest [Pandas UDFs](https://spark.apache.org/docs/latest/sql-pyspark-pandas-with-arrow.html#pandas-udfs-aka-vectorized-udfs) and [Pandas Function APIs](https://spark.apache.org/docs/latest/sql-pyspark-pandas-with-arrow.html#pandas-function-apis). For instance, the example below allows users to directly use the APIs in [a pandas Series](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.html) within Python native function."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "+------------------+\n",
+      "|pandas_plus_one(a)|\n",
+      "+------------------+\n",
+      "|                 2|\n",
+      "|                 3|\n",
+      "|                 4|\n",
+      "+------------------+\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas\n",
+    "from pyspark.sql.functions import pandas_udf\n",
+    "\n",
+    "@pandas_udf('long')\n",
+    "def pandas_plus_one(series: pd.Series) -> pd.Series:\n",
+    "    # Simply plus one by using pandas Series.\n",
+    "    return series + 1\n",
+    "\n",
+    "df.select(pandas_plus_one(df.a)).show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Another example is `DataFrame.mapInPandas` which allows users directly use the APIs in a [pandas DataFrame](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html) without any restrictions such as the result length."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "+---+---+-------+----------+-------------------+\n",
+      "|  a|  b|      c|         d|                  e|\n",
+      "+---+---+-------+----------+-------------------+\n",
+      "|  1|2.0|string1|2000-01-01|2000-01-01 12:00:00|\n",
+      "+---+---+-------+----------+-------------------+\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "def pandas_filter_func(iterator):\n",
+    "    for pandas_df in iterator:\n",
+    "        yield pandas_df[pandas_df.a == 1]\n",
+    "\n",
+    "df.mapInPandas(pandas_filter_func, schema=df.schema).show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Grouping Data\n",
+    "\n",
+    "PySpark DataFrame also provides a way of handling grouped data by using the common approach, split-apply-combine strategy.\n",
+    "It groups the data by a certain condition applies a function to each group and then combines them back to the DataFrame."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "+-----+------+---+---+\n",
+      "|color| fruit| v1| v2|\n",
+      "+-----+------+---+---+\n",
+      "|  red|banana|  1| 10|\n",
+      "| blue|banana|  2| 20|\n",
+      "|  red|carrot|  3| 30|\n",
+      "| blue| grape|  4| 40|\n",
+      "|  red|carrot|  5| 50|\n",
+      "|black|carrot|  6| 60|\n",
+      "|  red|banana|  7| 70|\n",
+      "|  red| grape|  8| 80|\n",
+      "+-----+------+---+---+\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "df = spark.createDataFrame([\n",
+    "    ['red', 'banana', 1, 10], ['blue', 'banana', 2, 20], ['red', 'carrot', 3, 30],\n",
+    "    ['blue', 'grape', 4, 40], ['red', 'carrot', 5, 50], ['black', 'carrot', 6, 60],\n",
+    "    ['red', 'banana', 7, 70], ['red', 'grape', 8, 80]], schema=['color', 'fruit', 'v1', 'v2'])\n",
+    "df.show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Grouping and then applying the `avg()` function to the resulting groups."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "+-----+-------+-------+\n",
+      "|color|avg(v1)|avg(v2)|\n",
+      "+-----+-------+-------+\n",
+      "|  red|    4.8|   48.0|\n",
+      "|black|    6.0|   60.0|\n",
+      "| blue|    3.0|   30.0|\n",
+      "+-----+-------+-------+\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "df.groupby('color').avg().show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You can also apply a Python native function against each group by using pandas APIs."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "+-----+------+---+---+\n",
+      "|color| fruit| v1| v2|\n",
+      "+-----+------+---+---+\n",
+      "|  red|banana| -3| 10|\n",
+      "|  red|carrot| -1| 30|\n",
+      "|  red|carrot|  0| 50|\n",
+      "|  red|banana|  2| 70|\n",
+      "|  red| grape|  3| 80|\n",
+      "|black|carrot|  0| 60|\n",
+      "| blue|banana| -1| 20|\n",
+      "| blue| grape|  1| 40|\n",
+      "+-----+------+---+---+\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "def plus_mean(pandas_df):\n",
+    "    return pandas_df.assign(v1=pandas_df.v1 - pandas_df.v1.mean())\n",
+    "\n",
+    "df.groupby('color').applyInPandas(plus_mean, schema=df.schema).show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Co-grouping and applying a function."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "+--------+---+---+---+\n",
+      "|    time| id| v1| v2|\n",
+      "+--------+---+---+---+\n",
+      "|20000101|  1|1.0|  x|\n",
+      "|20000102|  1|3.0|  x|\n",
+      "|20000101|  2|2.0|  y|\n",
+      "|20000102|  2|4.0|  y|\n",
+      "+--------+---+---+---+\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "df1 = spark.createDataFrame(\n",
+    "    [(20000101, 1, 1.0), (20000101, 2, 2.0), (20000102, 1, 3.0), (20000102, 2, 4.0)],\n",
+    "    ('time', 'id', 'v1'))\n",
+    "\n",
+    "df2 = spark.createDataFrame(\n",
+    "    [(20000101, 1, 'x'), (20000101, 2, 'y')],\n",
+    "    ('time', 'id', 'v2'))\n",
+    "\n",
+    "def asof_join(l, r):\n",
+    "    return pd.merge_asof(l, r, on='time', by='id')\n",
+    "\n",
+    "df1.groupby('id').cogroup(df2.groupby('id')).applyInPandas(\n",
+    "    asof_join, schema='time int, id int, v1 double, v2 string').show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Getting Data in/out\n",
+    "\n",
+    "CSV is straightforward and easy to use. Parquet and ORC are efficient and compact file formats to read and write faster.\n",
+    "\n",
+    "There are many other data sources available in PySpark such as JDBC, text, binaryFile, Avro, etc. See also the latest [Spark SQL, DataFrames and Datasets Guide](https://spark.apache.org/docs/latest/sql-programming-guide.html) in Apache Spark documentation."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### CSV"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "+-----+------+---+---+\n",
+      "|color| fruit| v1| v2|\n",
+      "+-----+------+---+---+\n",
+      "|  red|banana|  1| 10|\n",
+      "| blue|banana|  2| 20|\n",
+      "|  red|carrot|  3| 30|\n",
+      "| blue| grape|  4| 40|\n",
+      "|  red|carrot|  5| 50|\n",
+      "|black|carrot|  6| 60|\n",
+      "|  red|banana|  7| 70|\n",
+      "|  red| grape|  8| 80|\n",
+      "+-----+------+---+---+\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "df.write.csv('foo.csv', header=True)\n",
+    "spark.read.csv('foo.csv', header=True).show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Parquet"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "+-----+------+---+---+\n",
+      "|color| fruit| v1| v2|\n",
+      "+-----+------+---+---+\n",
+      "|  red|banana|  1| 10|\n",
+      "| blue|banana|  2| 20|\n",
+      "|  red|carrot|  3| 30|\n",
+      "| blue| grape|  4| 40|\n",
+      "|  red|carrot|  5| 50|\n",
+      "|black|carrot|  6| 60|\n",
+      "|  red|banana|  7| 70|\n",
+      "|  red| grape|  8| 80|\n",
+      "+-----+------+---+---+\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "df.write.parquet('bar.parquet')\n",
+    "spark.read.parquet('bar.parquet').show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### ORC"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "+-----+------+---+---+\n",
+      "|color| fruit| v1| v2|\n",
+      "+-----+------+---+---+\n",
+      "|  red|banana|  1| 10|\n",
+      "| blue|banana|  2| 20|\n",
+      "|  red|carrot|  3| 30|\n",
+      "| blue| grape|  4| 40|\n",
+      "|  red|carrot|  5| 50|\n",
+      "|black|carrot|  6| 60|\n",
+      "|  red|banana|  7| 70|\n",
+      "|  red| grape|  8| 80|\n",
+      "+-----+------+---+---+\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "df.write.orc('zoo.orc')\n",
+    "spark.read.orc('zoo.orc').show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Working with SQL\n",
+    "\n",
+    "DataFrame and Spark SQL share the same execution engine so they can be interchangeably used seamlessly. For example, you can register the DataFrame as a table and run a SQL easily as below:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "+--------+\n",
+      "|count(1)|\n",
+      "+--------+\n",
+      "|       8|\n",
+      "+--------+\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "df.createOrReplaceTempView(\"tableA\")\n",
+    "spark.sql(\"SELECT count(*) from tableA\").show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "In addition, UDFs can be registered and invoked in SQL out of the box:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "+-----------+\n",
+      "|add_one(v1)|\n",
+      "+-----------+\n",
+      "|          2|\n",
+      "|          3|\n",
+      "|          4|\n",
+      "|          5|\n",
+      "|          6|\n",
+      "|          7|\n",
+      "|          8|\n",
+      "|          9|\n",
+      "+-----------+\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "@pandas_udf(\"integer\")\n",
+    "def add_one(s: pd.Series) -> pd.Series:\n",
+    "    return s + 1\n",
+    "\n",
+    "spark.udf.register(\"add_one\", add_one)\n",
+    "spark.sql(\"SELECT add_one(v1) FROM tableA\").show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "These SQL expressions can directly be mixed and used as PySpark columns."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "+-----------+\n",
+      "|add_one(v1)|\n",
+      "+-----------+\n",
+      "|          2|\n",
+      "|          3|\n",
+      "|          4|\n",
+      "|          5|\n",
+      "|          6|\n",
+      "|          7|\n",
+      "|          8|\n",
+      "|          9|\n",
+      "+-----------+\n",
+      "\n",
+      "+--------------+\n",
+      "|(count(1) > 0)|\n",
+      "+--------------+\n",
+      "|          true|\n",
+      "+--------------+\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pyspark.sql.functions import expr\n",
+    "\n",
+    "df.selectExpr('add_one(v1)').show()\n",
+    "df.select(expr('count(*)') > 0).show()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.8"
+  },
+  "name": "quickstart",
+  "notebookId": 1927513300154480
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
diff --git a/python/docs/source/index.rst b/python/docs/source/index.rst
new file mode 100644
index 0000000000000..1c10099143c5e
--- /dev/null
+++ b/python/docs/source/index.rst
@@ -0,0 +1,68 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+.. PySpark documentation master file
+
+=====================
+PySpark Documentation
+=====================
+
+|binder|_ | `GitHub <https://github.com/apache/spark>`_ | `Issues <https://issues.apache.org/jira/projects/SPARK/issues>`_ | |examples|_ | `Community <https://spark.apache.org/community.html>`_
+
+PySpark is an interface for Apache Spark in Python. It not only allows you to write
+Spark applications using Python APIs, but also provides the PySpark shell for
+interactively analyzing your data in a distributed environment. PySpark supports most
+of Spark's features such as Spark SQL, DataFrame, Streaming, MLlib
+(Machine Learning) and Spark Core.
+
+.. image:: ../../../docs/img/pyspark-components.png
+  :alt: PySpark Components
+
+**Spark SQL and DataFrame**
+
+Spark SQL is a Spark module for structured data processing. It provides
+a programming abstraction called DataFrame and can also act as distributed
+SQL query engine.
+
+**Streaming**
+
+Running on top of Spark, the streaming feature in Apache Spark enables powerful
+interactive and analytical applications across both streaming and historical data,
+while inheriting Spark's ease of use and fault tolerance characteristics.
+
+**MLlib**
+
+Built on top of Spark, MLlib is a scalable machine learning library that provides
+a uniform set of high-level APIs that help users create and tune practical machine
+learning pipelines.
+
+**Spark Core**
+
+Spark Core is the underlying general execution engine for the Spark platform that all
+other functionality is built on top of. It provides an RDD (Resilient Distributed Dataset)
+and in-memory computing capabilities.
+
+.. toctree::
+    :maxdepth: 2
+    :hidden:
+
+    getting_started/index
+    user_guide/index
+    reference/index
+    development/index
+    migration_guide/index
+
diff --git a/python/docs/source/migration_guide/index.rst b/python/docs/source/migration_guide/index.rst
new file mode 100644
index 0000000000000..5d30b585dcd1d
--- /dev/null
+++ b/python/docs/source/migration_guide/index.rst
@@ -0,0 +1,44 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+===============
+Migration Guide
+===============
+
+This page describes the migration guide specific to PySpark.
+
+.. toctree::
+   :maxdepth: 2
+
+   pyspark_3.1_to_3.2
+   pyspark_2.4_to_3.0
+   pyspark_2.3_to_2.4
+   pyspark_2.3.0_to_2.3.1_above
+   pyspark_2.2_to_2.3
+   pyspark_1.4_to_1.5
+   pyspark_1.0_1.2_to_1.3
+
+
+Many items of other migration guides can also be applied when migrating PySpark to higher versions because PySpark internally shares other components.
+Please also refer other migration guides:
+
+- `Migration Guide: Spark Core <https://spark.apache.org/docs/latest/core-migration-guide.html>`_
+- `Migration Guide: SQL, Datasets and DataFrame <https://spark.apache.org/docs/latest/sql-migration-guide.html>`_
+- `Migration Guide: Structured Streaming <https://spark.apache.org/docs/latest/ss-migration-guide.html>`_
+- `Migration Guide: MLlib (Machine Learning) <https://spark.apache.org/docs/latest/ml-migration-guide.html>`_
+
diff --git a/python/docs/source/migration_guide/pyspark_1.0_1.2_to_1.3.rst b/python/docs/source/migration_guide/pyspark_1.0_1.2_to_1.3.rst
new file mode 100644
index 0000000000000..ef2b74b566aee
--- /dev/null
+++ b/python/docs/source/migration_guide/pyspark_1.0_1.2_to_1.3.rst
@@ -0,0 +1,23 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+=====================================
+Upgrading from PySpark 1.0-1.2 to 1.3
+=====================================
+
+* When using DataTypes in Python you will need to construct them (i.e. ``StringType()``) instead of referencing a singleton.
\ No newline at end of file
diff --git a/python/docs/source/migration_guide/pyspark_1.4_to_1.5.rst b/python/docs/source/migration_guide/pyspark_1.4_to_1.5.rst
new file mode 100644
index 0000000000000..2b42b17defc9c
--- /dev/null
+++ b/python/docs/source/migration_guide/pyspark_1.4_to_1.5.rst
@@ -0,0 +1,25 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+=================================
+Upgrading from PySpark 1.4 to 1.5
+=================================
+
+* Resolution of strings to columns in Python now supports using dots (.) to qualify the column or access nested values. For example ``df['table.column.nestedField']``. However, this means that if your column name contains any dots you must now escape them using backticks (e.g., ``table.`column.with.dots`.nested``).
+
+* DataFrame.withColumn method in PySpark supports adding a new column or replacing existing columns of the same name.
diff --git a/python/docs/source/migration_guide/pyspark_2.2_to_2.3.rst b/python/docs/source/migration_guide/pyspark_2.2_to_2.3.rst
new file mode 100644
index 0000000000000..e1b9bffe5a659
--- /dev/null
+++ b/python/docs/source/migration_guide/pyspark_2.2_to_2.3.rst
@@ -0,0 +1,30 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+=================================
+Upgrading from PySpark 2.2 to 2.3
+=================================
+
+* In PySpark, now we need Pandas 0.19.2 or upper if you want to use Pandas related functionalities, such as ``toPandas``, ``createDataFrame`` from Pandas DataFrame, etc.
+
+* In PySpark, the behavior of timestamp values for Pandas related functionalities was changed to respect session timezone. If you want to use the old behavior, you need to set a configuration ``spark.sql.execution.pandas.respectSessionTimeZone`` to False. See `SPARK-22395 <https://issues.apache.org/jira/browse/SPARK-22395>`_ for details.
+
+* In PySpark, ``na.fill()`` or ``fillna`` also accepts boolean and replaces nulls with booleans. In prior Spark versions, PySpark just ignores it and returns the original Dataset/DataFrame.
+
+* In PySpark, ``df.replace`` does not allow to omit value when ``to_replace`` is not a dictionary. Previously, value could be omitted in the other cases and had None by default, which is counterintuitive and error-prone.
+
diff --git a/python/docs/source/migration_guide/pyspark_2.3.0_to_2.3.1_above.rst b/python/docs/source/migration_guide/pyspark_2.3.0_to_2.3.1_above.rst
new file mode 100644
index 0000000000000..4de43b340eb40
--- /dev/null
+++ b/python/docs/source/migration_guide/pyspark_2.3.0_to_2.3.1_above.rst
@@ -0,0 +1,23 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+===============================================
+Upgrading from PySpark 2.3.0 to 2.3.1 and above
+===============================================
+
+* As of version 2.3.1 Arrow functionality, including ``pandas_udf`` and ``toPandas()``/``createDataFrame()`` with ``spark.sql.execution.arrow.enabled`` set to ``True``, has been marked as experimental. These are still evolving and not currently recommended for use in production.
\ No newline at end of file
diff --git a/python/docs/source/migration_guide/pyspark_2.3_to_2.4.rst b/python/docs/source/migration_guide/pyspark_2.3_to_2.4.rst
new file mode 100644
index 0000000000000..394d2bd3bbe3f
--- /dev/null
+++ b/python/docs/source/migration_guide/pyspark_2.3_to_2.4.rst
@@ -0,0 +1,23 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+=================================
+Upgrading from PySpark 2.3 to 2.4
+=================================
+
+* In PySpark, when Arrow optimization is enabled, previously ``toPandas`` just failed when Arrow optimization is unable to be used whereas ``createDataFrame`` from Pandas DataFrame allowed the fallback to non-optimization. Now, both ``toPandas`` and ``createDataFrame`` from Pandas DataFrame allow the fallback by default, which can be switched off by ``spark.sql.execution.arrow.fallback.enabled``.
diff --git a/python/docs/source/migration_guide/pyspark_2.4_to_3.0.rst b/python/docs/source/migration_guide/pyspark_2.4_to_3.0.rst
new file mode 100644
index 0000000000000..ad800ddfc342a
--- /dev/null
+++ b/python/docs/source/migration_guide/pyspark_2.4_to_3.0.rst
@@ -0,0 +1,44 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+=================================
+Upgrading from PySpark 2.4 to 3.0
+=================================
+
+* In Spark 3.0, PySpark requires a pandas version of 0.23.2 or higher to use pandas related functionality, such as ``toPandas``, ``createDataFrame`` from pandas DataFrame, and so on.
+
+* In Spark 3.0, PySpark requires a PyArrow version of 0.12.1 or higher to use PyArrow related functionality, such as ``pandas_udf``, ``toPandas`` and ``createDataFrame`` with "spark.sql.execution.arrow.enabled=true", etc.
+
+* In PySpark, when creating a ``SparkSession`` with ``SparkSession.builder.getOrCreate()``, if there is an existing ``SparkContext``, the builder was trying to update the ``SparkConf`` of the existing ``SparkContext`` with configurations specified to the builder, but the ``SparkContext`` is shared by all ``SparkSession`` s, so we should not update them. In 3.0, the builder comes to not update the configurations. This is the same behavior as Java/Scala API in 2.3 and above. If you want to update them, you need to update them prior to creating a ``SparkSession``.
+
+* In PySpark, when Arrow optimization is enabled, if Arrow version is higher than 0.11.0, Arrow can perform safe type conversion when converting pandas.Series to an Arrow array during serialization. Arrow raises errors when detecting unsafe type conversions like overflow. You enable it by setting ``spark.sql.execution.pandas.convertToArrowArraySafely`` to true. The default setting is false. PySpark behavior for Arrow versions is illustrated in the following table:
+
+    =======================================  ================  =========================
+    PyArrow version                          Integer overflow  Floating point truncation
+    =======================================  ================  =========================
+    0.11.0 and below                         Raise error       Silently allows
+    > 0.11.0, arrowSafeTypeConversion=false  Silent overflow   Silently allows
+    > 0.11.0, arrowSafeTypeConversion=true   Raise error       Raise error
+    =======================================  ================  =========================
+
+* In Spark 3.0, ``createDataFrame(..., verifySchema=True)`` validates LongType as well in PySpark. Previously, LongType was not verified and resulted in None in case the value overflows. To restore this behavior, verifySchema can be set to False to disable the validation.
+
+* As of Spark 3.0, ``Row`` field names are no longer sorted alphabetically when constructing with named arguments for Python versions 3.6 and above, and the order of fields will match that as entered. To enable sorted fields by default, as in Spark 2.4, set the environment variable ``PYSPARK_ROW_FIELD_SORTING_ENABLED`` to true for both executors and driver - this environment variable must be consistent on all executors and driver; otherwise, it may cause failures or incorrect answers. For Python versions less than 3.6, the field names will be sorted alphabetically as the only option.
+
+* In Spark 3.0, ``pyspark.ml.param.shared.Has*`` mixins do not provide any ``set*(self, value)`` setter methods anymore, use the respective ``self.set(self.*, value)`` instead. See `SPARK-29093 <https://issues.apache.org/jira/browse/SPARK-29093>`_ for details.
+
diff --git a/python/docs/source/migration_guide/pyspark_3.1_to_3.2.rst b/python/docs/source/migration_guide/pyspark_3.1_to_3.2.rst
new file mode 100644
index 0000000000000..1537ef8e51572
--- /dev/null
+++ b/python/docs/source/migration_guide/pyspark_3.1_to_3.2.rst
@@ -0,0 +1,23 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+=================================
+Upgrading from PySpark 3.1 to 3.2
+=================================
+
+* In Spark 3.2, the PySpark methods from sql, ml, spark_on_pandas modules raise the ``TypeError`` instead of ``ValueError`` when are applied to an param of inappropriate type.
diff --git a/python/docs/source/reference/index.rst b/python/docs/source/reference/index.rst
new file mode 100644
index 0000000000000..77f17da720dd5
--- /dev/null
+++ b/python/docs/source/reference/index.rst
@@ -0,0 +1,34 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+=============
+API Reference
+=============
+
+This page lists an overview of all public PySpark modules, classes, functions and methods.
+
+.. toctree::
+   :maxdepth: 2
+
+   pyspark.sql
+   pyspark.ss
+   pyspark.ml
+   pyspark.streaming
+   pyspark.mllib
+   pyspark
+   pyspark.resource
diff --git a/python/docs/source/reference/pyspark.ml.rst b/python/docs/source/reference/pyspark.ml.rst
new file mode 100644
index 0000000000000..fc6060c979d1e
--- /dev/null
+++ b/python/docs/source/reference/pyspark.ml.rst
@@ -0,0 +1,364 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+MLlib (DataFrame-based)
+=======================
+
+Pipeline APIs
+-------------
+
+.. currentmodule:: pyspark.ml
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    Transformer
+    UnaryTransformer
+    Estimator
+    Model
+    Predictor
+    PredictionModel
+    Pipeline
+    PipelineModel
+
+
+Parameters
+----------
+
+.. currentmodule:: pyspark.ml.param
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    Param
+    Params
+    TypeConverters
+
+
+Feature
+-------
+
+.. currentmodule:: pyspark.ml.feature
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    Binarizer
+    BucketedRandomProjectionLSH
+    BucketedRandomProjectionLSHModel
+    Bucketizer
+    ChiSqSelector
+    ChiSqSelectorModel
+    CountVectorizer
+    CountVectorizerModel
+    DCT
+    ElementwiseProduct
+    FeatureHasher
+    HashingTF
+    IDF
+    IDFModel
+    Imputer
+    ImputerModel
+    IndexToString
+    Interaction
+    MaxAbsScaler
+    MaxAbsScalerModel
+    MinHashLSH
+    MinHashLSHModel
+    MinMaxScaler
+    MinMaxScalerModel
+    NGram
+    Normalizer
+    OneHotEncoder
+    OneHotEncoderModel
+    PCA
+    PCAModel
+    PolynomialExpansion
+    QuantileDiscretizer
+    RobustScaler
+    RobustScalerModel
+    RegexTokenizer
+    RFormula
+    RFormulaModel
+    SQLTransformer
+    StandardScaler
+    StandardScalerModel
+    StopWordsRemover
+    StringIndexer
+    StringIndexerModel
+    Tokenizer
+    UnivariateFeatureSelector
+    UnivariateFeatureSelectorModel
+    VarianceThresholdSelector
+    VarianceThresholdSelectorModel
+    VectorAssembler
+    VectorIndexer
+    VectorIndexerModel
+    VectorSizeHint
+    VectorSlicer
+    Word2Vec
+    Word2VecModel
+
+
+Classification
+--------------
+
+.. currentmodule:: pyspark.ml.classification
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    LinearSVC
+    LinearSVCModel
+    LinearSVCSummary
+    LinearSVCTrainingSummary
+    LogisticRegression
+    LogisticRegressionModel
+    LogisticRegressionSummary
+    LogisticRegressionTrainingSummary
+    BinaryLogisticRegressionSummary
+    BinaryLogisticRegressionTrainingSummary
+    DecisionTreeClassifier
+    DecisionTreeClassificationModel
+    GBTClassifier
+    GBTClassificationModel
+    RandomForestClassifier
+    RandomForestClassificationModel
+    RandomForestClassificationSummary
+    RandomForestClassificationTrainingSummary
+    BinaryRandomForestClassificationSummary
+    BinaryRandomForestClassificationTrainingSummary
+    NaiveBayes
+    NaiveBayesModel
+    MultilayerPerceptronClassifier
+    MultilayerPerceptronClassificationModel
+    MultilayerPerceptronClassificationSummary
+    MultilayerPerceptronClassificationTrainingSummary
+    OneVsRest
+    OneVsRestModel
+    FMClassifier
+    FMClassificationModel
+    FMClassificationSummary
+    FMClassificationTrainingSummary
+
+
+Clustering
+----------
+
+.. currentmodule:: pyspark.ml.clustering
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    BisectingKMeans
+    BisectingKMeansModel
+    BisectingKMeansSummary
+    KMeans
+    KMeansModel
+    KMeansSummary
+    GaussianMixture
+    GaussianMixtureModel
+    GaussianMixtureSummary
+    LDA
+    LDAModel
+    LocalLDAModel
+    DistributedLDAModel
+    PowerIterationClustering
+
+
+Functions
+---------
+
+.. currentmodule:: pyspark.ml.functions
+
+.. autosummary::
+    :toctree: api/
+
+    array_to_vector
+    vector_to_array
+
+
+Vector and Matrix
+-----------------
+
+.. currentmodule:: pyspark.ml.linalg
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    Vector
+    DenseVector
+    SparseVector
+    Vectors
+    Matrix
+    DenseMatrix
+    SparseMatrix
+    Matrices
+
+
+Recommendation
+--------------
+
+.. currentmodule:: pyspark.ml.recommendation
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    ALS
+    ALSModel
+
+
+Regression
+----------
+
+.. currentmodule:: pyspark.ml.regression
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    AFTSurvivalRegression
+    AFTSurvivalRegressionModel
+    DecisionTreeRegressor
+    DecisionTreeRegressionModel
+    GBTRegressor
+    GBTRegressionModel
+    GeneralizedLinearRegression
+    GeneralizedLinearRegressionModel
+    GeneralizedLinearRegressionSummary
+    GeneralizedLinearRegressionTrainingSummary
+    IsotonicRegression
+    IsotonicRegressionModel
+    LinearRegression
+    LinearRegressionModel
+    LinearRegressionSummary
+    LinearRegressionTrainingSummary
+    RandomForestRegressor
+    RandomForestRegressionModel
+    FMRegressor
+    FMRegressionModel
+
+
+Statistics
+----------
+
+.. currentmodule:: pyspark.ml.stat
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    ChiSquareTest
+    Correlation
+    KolmogorovSmirnovTest
+    MultivariateGaussian
+    Summarizer
+    SummaryBuilder
+
+
+Tuning
+------
+
+.. currentmodule:: pyspark.ml.tuning
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    ParamGridBuilder
+    ParamRandomBuilder
+    CrossValidator
+    CrossValidatorModel
+    TrainValidationSplit
+    TrainValidationSplitModel
+
+
+Evaluation
+----------
+
+.. currentmodule:: pyspark.ml.evaluation
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    Evaluator
+    BinaryClassificationEvaluator
+    RegressionEvaluator
+    MulticlassClassificationEvaluator
+    MultilabelClassificationEvaluator
+    ClusteringEvaluator
+    RankingEvaluator
+
+
+Frequency Pattern Mining
+----------------------------
+
+.. currentmodule:: pyspark.ml.fpm
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    FPGrowth
+    FPGrowthModel
+    PrefixSpan
+
+
+Image
+-----
+
+.. currentmodule:: pyspark.ml.image
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    ImageSchema
+    _ImageSchema
+
+
+Utilities
+---------
+
+.. currentmodule:: pyspark.ml.util
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    BaseReadWrite
+    DefaultParamsReadable
+    DefaultParamsReader
+    DefaultParamsWritable
+    DefaultParamsWriter
+    GeneralMLWriter
+    HasTrainingSummary
+    Identifiable
+    MLReadable
+    MLReader
+    MLWritable
+    MLWriter
+
diff --git a/python/docs/source/reference/pyspark.mllib.rst b/python/docs/source/reference/pyspark.mllib.rst
new file mode 100644
index 0000000000000..12fc4798dd8de
--- /dev/null
+++ b/python/docs/source/reference/pyspark.mllib.rst
@@ -0,0 +1,254 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+MLlib (RDD-based)
+=================
+
+Classification
+--------------
+
+.. currentmodule:: pyspark.mllib.classification
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    LogisticRegressionModel
+    LogisticRegressionWithSGD
+    LogisticRegressionWithLBFGS
+    SVMModel
+    SVMWithSGD
+    NaiveBayesModel
+    NaiveBayes
+    StreamingLogisticRegressionWithSGD
+
+
+Clustering
+----------
+
+.. currentmodule:: pyspark.mllib.clustering
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+
+    BisectingKMeansModel
+    BisectingKMeans
+    KMeansModel
+    KMeans
+    GaussianMixtureModel
+    GaussianMixture
+    PowerIterationClusteringModel
+    PowerIterationClustering
+    StreamingKMeans
+    StreamingKMeansModel
+    LDA
+    LDAModel
+
+
+Evaluation
+----------
+
+.. currentmodule:: pyspark.mllib.evaluation
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    BinaryClassificationMetrics
+    RegressionMetrics
+    MulticlassMetrics
+    RankingMetrics
+
+
+Feature
+-------
+
+.. currentmodule:: pyspark.mllib.feature
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    Normalizer
+    StandardScalerModel
+    StandardScaler
+    HashingTF
+    IDFModel
+    IDF
+    Word2Vec
+    Word2VecModel
+    ChiSqSelector
+    ChiSqSelectorModel
+    ElementwiseProduct
+
+
+Frequency Pattern Mining
+------------------------
+
+.. currentmodule:: pyspark.mllib.fpm
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    FPGrowth
+    FPGrowthModel
+    PrefixSpan
+    PrefixSpanModel
+
+
+Vector and Matrix
+-----------------
+
+.. currentmodule:: pyspark.mllib.linalg
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    Vector
+    DenseVector
+    SparseVector
+    Vectors
+    Matrix
+    DenseMatrix
+    SparseMatrix
+    Matrices
+    QRDecomposition
+
+
+Distributed Representation
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. currentmodule:: pyspark.mllib.linalg.distributed
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    BlockMatrix
+    CoordinateMatrix
+    DistributedMatrix
+    IndexedRow
+    IndexedRowMatrix
+    MatrixEntry
+    RowMatrix
+    SingularValueDecomposition
+
+
+Random
+------
+
+.. currentmodule:: pyspark.mllib.random
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    RandomRDDs
+
+
+Recommendation
+--------------
+
+.. currentmodule:: pyspark.mllib.recommendation
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    MatrixFactorizationModel
+    ALS
+    Rating
+
+
+Regression
+----------
+
+.. currentmodule:: pyspark.mllib.regression
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    LabeledPoint
+    LinearModel
+    LinearRegressionModel
+    LinearRegressionWithSGD
+    RidgeRegressionModel
+    RidgeRegressionWithSGD
+    LassoModel
+    LassoWithSGD
+    IsotonicRegressionModel
+    IsotonicRegression
+    StreamingLinearAlgorithm
+    StreamingLinearRegressionWithSGD
+
+
+Statistics
+----------
+
+.. currentmodule:: pyspark.mllib.stat
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    Statistics
+    MultivariateStatisticalSummary
+    ChiSqTestResult
+    MultivariateGaussian
+    KernelDensity
+    ChiSqTestResult
+    KolmogorovSmirnovTestResult
+
+
+Tree
+----
+
+.. currentmodule:: pyspark.mllib.tree
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    DecisionTreeModel
+    DecisionTree
+    RandomForestModel
+    RandomForest
+    GradientBoostedTreesModel
+    GradientBoostedTrees
+
+
+Utilities
+---------
+
+.. currentmodule:: pyspark.mllib.util
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    JavaLoader
+    JavaSaveable
+    LinearDataGenerator
+    Loader
+    MLUtils
+    Saveable
diff --git a/python/docs/source/reference/pyspark.resource.rst b/python/docs/source/reference/pyspark.resource.rst
new file mode 100644
index 0000000000000..a1d885c44c480
--- /dev/null
+++ b/python/docs/source/reference/pyspark.resource.rst
@@ -0,0 +1,38 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+===================
+Resource Management
+===================
+
+Core Classes
+------------
+
+.. currentmodule:: pyspark.resource
+
+.. autosummary::
+    :toctree: api/
+
+    ResourceInformation
+    ResourceProfile
+    ResourceProfileBuilder
+    ExecutorResourceRequest
+    ExecutorResourceRequests
+    TaskResourceRequest
+    TaskResourceRequests
+
diff --git a/python/docs/source/reference/pyspark.rst b/python/docs/source/reference/pyspark.rst
new file mode 100644
index 0000000000000..49a8ff59aca0d
--- /dev/null
+++ b/python/docs/source/reference/pyspark.rst
@@ -0,0 +1,281 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+==========
+Spark Core
+==========
+
+Public Classes
+--------------
+
+.. currentmodule:: pyspark
+
+.. autosummary::
+    :toctree: api/
+
+    SparkContext
+    RDD
+    Broadcast
+    Accumulator
+    AccumulatorParam
+    SparkConf
+    SparkFiles
+    StorageLevel
+    TaskContext
+    RDDBarrier
+    BarrierTaskContext
+    BarrierTaskInfo
+    InheritableThread
+
+
+Spark Context APIs
+------------------
+
+.. currentmodule:: pyspark
+
+.. autosummary::
+    :toctree: api/
+
+    SparkContext.PACKAGE_EXTENSIONS
+    SparkContext.accumulator
+    SparkContext.addFile
+    SparkContext.addPyFile
+    SparkContext.applicationId
+    SparkContext.binaryFiles
+    SparkContext.binaryRecords
+    SparkContext.broadcast
+    SparkContext.cancelAllJobs
+    SparkContext.cancelJobGroup
+    SparkContext.defaultMinPartitions
+    SparkContext.defaultParallelism
+    SparkContext.dump_profiles
+    SparkContext.emptyRDD
+    SparkContext.getCheckpointDir
+    SparkContext.getConf
+    SparkContext.getLocalProperty
+    SparkContext.getOrCreate
+    SparkContext.hadoopFile
+    SparkContext.hadoopRDD
+    SparkContext.newAPIHadoopFile
+    SparkContext.newAPIHadoopRDD
+    SparkContext.parallelize
+    SparkContext.pickleFile
+    SparkContext.range
+    SparkContext.resources
+    SparkContext.runJob
+    SparkContext.sequenceFile
+    SparkContext.setCheckpointDir
+    SparkContext.setJobDescription
+    SparkContext.setJobGroup
+    SparkContext.setLocalProperty
+    SparkContext.setLogLevel
+    SparkContext.setSystemProperty
+    SparkContext.show_profiles
+    SparkContext.sparkUser
+    SparkContext.startTime
+    SparkContext.statusTracker
+    SparkContext.stop
+    SparkContext.textFile
+    SparkContext.uiWebUrl
+    SparkContext.union
+    SparkContext.version
+    SparkContext.wholeTextFiles
+
+
+RDD APIs
+--------
+
+.. currentmodule:: pyspark
+
+.. autosummary::
+    :toctree: api/
+
+    RDD.aggregate
+    RDD.aggregateByKey
+    RDD.barrier
+    RDD.cache
+    RDD.cartesian
+    RDD.checkpoint
+    RDD.coalesce
+    RDD.cogroup
+    RDD.collect
+    RDD.collectAsMap
+    RDD.collectWithJobGroup
+    RDD.combineByKey
+    RDD.context
+    RDD.count
+    RDD.countApprox
+    RDD.countApproxDistinct
+    RDD.countByKey
+    RDD.countByValue
+    RDD.distinct
+    RDD.filter
+    RDD.first
+    RDD.flatMap
+    RDD.flatMapValues
+    RDD.fold
+    RDD.foldByKey
+    RDD.foreach
+    RDD.foreachPartition
+    RDD.fullOuterJoin
+    RDD.getCheckpointFile
+    RDD.getNumPartitions
+    RDD.getResourceProfile
+    RDD.getStorageLevel
+    RDD.glom
+    RDD.groupBy
+    RDD.groupByKey
+    RDD.groupWith
+    RDD.histogram
+    RDD.id
+    RDD.intersection
+    RDD.isCheckpointed
+    RDD.isEmpty
+    RDD.isLocallyCheckpointed
+    RDD.join
+    RDD.keyBy
+    RDD.keys
+    RDD.leftOuterJoin
+    RDD.localCheckpoint
+    RDD.lookup
+    RDD.map
+    RDD.mapPartitions
+    RDD.mapPartitionsWithIndex
+    RDD.mapPartitionsWithSplit
+    RDD.mapValues
+    RDD.max
+    RDD.mean
+    RDD.meanApprox
+    RDD.min
+    RDD.name
+    RDD.partitionBy
+    RDD.persist
+    RDD.pipe
+    RDD.randomSplit
+    RDD.reduce
+    RDD.reduceByKey
+    RDD.reduceByKeyLocally
+    RDD.repartition
+    RDD.repartitionAndSortWithinPartitions
+    RDD.rightOuterJoin
+    RDD.sample
+    RDD.sampleByKey
+    RDD.sampleStdev
+    RDD.sampleVariance
+    RDD.saveAsHadoopDataset
+    RDD.saveAsHadoopFile
+    RDD.saveAsNewAPIHadoopDataset
+    RDD.saveAsNewAPIHadoopFile
+    RDD.saveAsPickleFile
+    RDD.saveAsSequenceFile
+    RDD.saveAsTextFile
+    RDD.setName
+    RDD.sortBy
+    RDD.sortByKey
+    RDD.stats
+    RDD.stdev
+    RDD.subtract
+    RDD.subtractByKey
+    RDD.sum
+    RDD.sumApprox
+    RDD.take
+    RDD.takeOrdered
+    RDD.takeSample
+    RDD.toDebugString
+    RDD.toLocalIterator
+    RDD.top
+    RDD.treeAggregate
+    RDD.treeReduce
+    RDD.union
+    RDD.unpersist
+    RDD.values
+    RDD.variance
+    RDD.withResources
+    RDD.zip
+    RDD.zipWithIndex
+    RDD.zipWithUniqueId
+
+
+Broadcast and Accumulator
+-------------------------
+
+.. currentmodule:: pyspark
+
+.. autosummary::
+    :toctree: api/
+
+    Broadcast.destroy
+    Broadcast.dump
+    Broadcast.load
+    Broadcast.load_from_path
+    Broadcast.unpersist
+    Broadcast.value
+    Accumulator.add
+    Accumulator.value
+    AccumulatorParam.addInPlace
+    AccumulatorParam.zero
+
+
+Management
+----------
+
+.. currentmodule:: pyspark
+
+.. autosummary::
+    :toctree: api/
+
+    SparkConf.contains
+    SparkConf.get
+    SparkConf.getAll
+    SparkConf.set
+    SparkConf.setAll
+    SparkConf.setAppName
+    SparkConf.setExecutorEnv
+    SparkConf.setIfMissing
+    SparkConf.setMaster
+    SparkConf.setSparkHome
+    SparkConf.toDebugString
+    SparkFiles.get
+    SparkFiles.getRootDirectory
+    StorageLevel.DISK_ONLY
+    StorageLevel.DISK_ONLY_2
+    StorageLevel.DISK_ONLY_3
+    StorageLevel.MEMORY_AND_DISK
+    StorageLevel.MEMORY_AND_DISK_2
+    StorageLevel.MEMORY_ONLY
+    StorageLevel.MEMORY_ONLY_2
+    StorageLevel.OFF_HEAP
+    TaskContext.attemptNumber
+    TaskContext.get
+    TaskContext.getLocalProperty
+    TaskContext.partitionId
+    TaskContext.resources
+    TaskContext.stageId
+    TaskContext.taskAttemptId
+    RDDBarrier.mapPartitions
+    RDDBarrier.mapPartitionsWithIndex
+    BarrierTaskContext.allGather
+    BarrierTaskContext.attemptNumber
+    BarrierTaskContext.barrier
+    BarrierTaskContext.get
+    BarrierTaskContext.getLocalProperty
+    BarrierTaskContext.getTaskInfos
+    BarrierTaskContext.partitionId
+    BarrierTaskContext.resources
+    BarrierTaskContext.stageId
+    BarrierTaskContext.taskAttemptId
diff --git a/python/docs/source/reference/pyspark.sql.rst b/python/docs/source/reference/pyspark.sql.rst
new file mode 100644
index 0000000000000..19e4e00960734
--- /dev/null
+++ b/python/docs/source/reference/pyspark.sql.rst
@@ -0,0 +1,603 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+=========
+Spark SQL
+=========
+
+Core Classes
+------------
+
+.. currentmodule:: pyspark.sql
+
+.. autosummary::
+    :toctree: api/
+
+    SparkSession
+    DataFrame
+    Column
+    Row
+    GroupedData
+    PandasCogroupedOps
+    DataFrameNaFunctions
+    DataFrameStatFunctions
+    Window
+
+
+Spark Session APIs
+------------------
+
+.. currentmodule:: pyspark.sql
+
+The entry point to programming Spark with the Dataset and DataFrame API.
+To create a Spark session, you should use ``SparkSession.builder`` attribute.
+See also :class:`SparkSession`.
+
+.. autosummary::
+    :toctree: api/
+
+    SparkSession.builder.appName
+    SparkSession.builder.config
+    SparkSession.builder.enableHiveSupport
+    SparkSession.builder.getOrCreate
+    SparkSession.builder.master
+    SparkSession.catalog
+    SparkSession.conf
+    SparkSession.createDataFrame
+    SparkSession.getActiveSession
+    SparkSession.newSession
+    SparkSession.range
+    SparkSession.read
+    SparkSession.readStream
+    SparkSession.sparkContext
+    SparkSession.sql
+    SparkSession.stop
+    SparkSession.streams
+    SparkSession.table
+    SparkSession.udf
+    SparkSession.version
+
+
+Configuration
+-------------
+
+.. currentmodule:: pyspark.sql.conf
+
+.. autosummary::
+    :toctree: api/
+
+    RuntimeConfig
+
+
+Input and Output
+----------------
+
+.. currentmodule:: pyspark.sql
+
+.. autosummary::
+    :toctree: api/
+
+    DataFrameReader.csv
+    DataFrameReader.format
+    DataFrameReader.jdbc
+    DataFrameReader.json
+    DataFrameReader.load
+    DataFrameReader.option
+    DataFrameReader.options
+    DataFrameReader.orc
+    DataFrameReader.parquet
+    DataFrameReader.schema
+    DataFrameReader.table
+    DataFrameWriter.bucketBy
+    DataFrameWriter.csv
+    DataFrameWriter.format
+    DataFrameWriter.insertInto
+    DataFrameWriter.jdbc
+    DataFrameWriter.json
+    DataFrameWriter.mode
+    DataFrameWriter.option
+    DataFrameWriter.options
+    DataFrameWriter.orc
+    DataFrameWriter.parquet
+    DataFrameWriter.partitionBy
+    DataFrameWriter.save
+    DataFrameWriter.saveAsTable
+    DataFrameWriter.sortBy
+    DataFrameWriter.text
+
+
+DataFrame APIs
+--------------
+
+.. currentmodule:: pyspark.sql
+
+.. autosummary::
+    :toctree: api/
+
+    DataFrame.agg
+    DataFrame.alias
+    DataFrame.approxQuantile
+    DataFrame.cache
+    DataFrame.checkpoint
+    DataFrame.coalesce
+    DataFrame.colRegex
+    DataFrame.collect
+    DataFrame.columns
+    DataFrame.corr
+    DataFrame.count
+    DataFrame.cov
+    DataFrame.createGlobalTempView
+    DataFrame.createOrReplaceGlobalTempView
+    DataFrame.createOrReplaceTempView
+    DataFrame.createTempView
+    DataFrame.crossJoin
+    DataFrame.crosstab
+    DataFrame.cube
+    DataFrame.describe
+    DataFrame.distinct
+    DataFrame.drop
+    DataFrame.dropDuplicates
+    DataFrame.drop_duplicates
+    DataFrame.dropna
+    DataFrame.dtypes
+    DataFrame.exceptAll
+    DataFrame.explain
+    DataFrame.fillna
+    DataFrame.filter
+    DataFrame.first
+    DataFrame.foreach
+    DataFrame.foreachPartition
+    DataFrame.freqItems
+    DataFrame.groupBy
+    DataFrame.head
+    DataFrame.hint
+    DataFrame.inputFiles
+    DataFrame.intersect
+    DataFrame.intersectAll
+    DataFrame.isLocal
+    DataFrame.isStreaming
+    DataFrame.join
+    DataFrame.limit
+    DataFrame.localCheckpoint
+    DataFrame.mapInPandas
+    DataFrame.na
+    DataFrame.orderBy
+    DataFrame.persist
+    DataFrame.printSchema
+    DataFrame.randomSplit
+    DataFrame.rdd
+    DataFrame.registerTempTable
+    DataFrame.repartition
+    DataFrame.repartitionByRange
+    DataFrame.replace
+    DataFrame.rollup
+    DataFrame.sameSemantics
+    DataFrame.sample
+    DataFrame.sampleBy
+    DataFrame.schema
+    DataFrame.select
+    DataFrame.selectExpr
+    DataFrame.semanticHash
+    DataFrame.show
+    DataFrame.sort
+    DataFrame.sortWithinPartitions
+    DataFrame.stat
+    DataFrame.storageLevel
+    DataFrame.subtract
+    DataFrame.summary
+    DataFrame.tail
+    DataFrame.take
+    DataFrame.toDF
+    DataFrame.toJSON
+    DataFrame.toLocalIterator
+    DataFrame.toPandas
+    DataFrame.transform
+    DataFrame.union
+    DataFrame.unionAll
+    DataFrame.unionByName
+    DataFrame.unpersist
+    DataFrame.where
+    DataFrame.withColumn
+    DataFrame.withColumnRenamed
+    DataFrame.withWatermark
+    DataFrame.write
+    DataFrame.writeStream
+    DataFrame.writeTo
+    DataFrameNaFunctions.drop
+    DataFrameNaFunctions.fill
+    DataFrameNaFunctions.replace
+    DataFrameStatFunctions.approxQuantile
+    DataFrameStatFunctions.corr
+    DataFrameStatFunctions.cov
+    DataFrameStatFunctions.crosstab
+    DataFrameStatFunctions.freqItems
+    DataFrameStatFunctions.sampleBy
+
+Column APIs
+-----------
+
+.. currentmodule:: pyspark.sql
+
+.. autosummary::
+    :toctree: api/
+
+    Column.alias
+    Column.asc
+    Column.asc_nulls_first
+    Column.asc_nulls_last
+    Column.astype
+    Column.between
+    Column.bitwiseAND
+    Column.bitwiseOR
+    Column.bitwiseXOR
+    Column.cast
+    Column.contains
+    Column.desc
+    Column.desc_nulls_first
+    Column.desc_nulls_last
+    Column.dropFields
+    Column.endswith
+    Column.eqNullSafe
+    Column.getField
+    Column.getItem
+    Column.isNotNull
+    Column.isNull
+    Column.isin
+    Column.like
+    Column.name
+    Column.otherwise
+    Column.over
+    Column.rlike
+    Column.startswith
+    Column.substr
+    Column.when
+    Column.withField
+
+Data Types
+----------
+
+.. currentmodule:: pyspark.sql.types
+
+.. autosummary::
+    :template: autosummary/class_with_docs.rst
+    :toctree: api/
+
+    ArrayType
+    BinaryType
+    BooleanType
+    ByteType
+    DataType
+    DateType
+    DecimalType
+    DoubleType
+    FloatType
+    IntegerType
+    LongType
+    MapType
+    NullType
+    ShortType
+    StringType
+    StructField
+    StructType
+    TimestampType
+
+
+Row
+---
+
+.. currentmodule:: pyspark.sql
+
+.. autosummary::
+    :toctree: api/
+
+    Row.asDict
+
+
+Functions
+---------
+
+.. currentmodule:: pyspark.sql.functions
+
+.. autosummary::
+    :toctree: api/
+
+    abs
+    acos
+    acosh
+    add_months
+    aggregate
+    approxCountDistinct
+    approx_count_distinct
+    array
+    array_contains
+    array_distinct
+    array_except
+    array_intersect
+    array_join
+    array_max
+    array_min
+    array_position
+    array_remove
+    array_repeat
+    array_sort
+    array_union
+    arrays_overlap
+    arrays_zip
+    asc
+    asc_nulls_first
+    asc_nulls_last
+    ascii
+    asin
+    asinh
+    assert_true
+    atan
+    atanh
+    atan2
+    avg
+    base64
+    bin
+    bitwise_not
+    bitwiseNOT
+    broadcast
+    bround
+    bucket
+    cbrt
+    ceil
+    coalesce
+    col
+    collect_list
+    collect_set
+    column
+    concat
+    concat_ws
+    conv
+    corr
+    cos
+    cosh
+    count
+    count_distinct
+    countDistinct
+    covar_pop
+    covar_samp
+    crc32
+    create_map
+    cume_dist
+    current_date
+    current_timestamp
+    date_add
+    date_format
+    date_sub
+    date_trunc
+    datediff
+    dayofmonth
+    dayofweek
+    dayofyear
+    days
+    decode
+    degrees
+    dense_rank
+    desc
+    desc_nulls_first
+    desc_nulls_last
+    element_at
+    encode
+    exists
+    exp
+    explode
+    explode_outer
+    expm1
+    expr
+    factorial
+    filter
+    first
+    flatten
+    floor
+    forall
+    format_number
+    format_string
+    from_csv
+    from_json
+    from_unixtime
+    from_utc_timestamp
+    get_json_object
+    greatest
+    grouping
+    grouping_id
+    hash
+    hex
+    hour
+    hours
+    hypot
+    initcap
+    input_file_name
+    instr
+    isnan
+    isnull
+    json_tuple
+    kurtosis
+    lag
+    last
+    last_day
+    lead
+    least
+    length
+    levenshtein
+    lit
+    locate
+    log
+    log10
+    log1p
+    log2
+    lower
+    lpad
+    ltrim
+    map_concat
+    map_entries
+    map_filter
+    map_from_arrays
+    map_from_entries
+    map_keys
+    map_values
+    map_zip_with
+    max
+    md5
+    mean
+    min
+    minute
+    monotonically_increasing_id
+    month
+    months
+    months_between
+    nanvl
+    next_day
+    nth_value
+    ntile
+    overlay
+    pandas_udf
+    percent_rank
+    percentile_approx
+    posexplode
+    posexplode_outer
+    pow
+    product
+    quarter
+    radians
+    raise_error
+    rand
+    randn
+    rank
+    regexp_extract
+    regexp_replace
+    repeat
+    reverse
+    rint
+    round
+    row_number
+    rpad
+    rtrim
+    schema_of_csv
+    schema_of_json
+    second
+    sequence
+    sha1
+    sha2
+    shiftleft
+    shiftright
+    shiftrightunsigned
+    shuffle
+    signum
+    sin
+    sinh
+    size
+    skewness
+    slice
+    sort_array
+    soundex
+    spark_partition_id
+    split
+    sqrt
+    stddev
+    stddev_pop
+    stddev_samp
+    struct
+    substring
+    substring_index
+    sum
+    sum_distinct
+    sumDistinct
+    tan
+    tanh
+    timestamp_seconds
+    toDegrees
+    toRadians
+    to_csv
+    to_date
+    to_json
+    to_timestamp
+    to_utc_timestamp
+    transform
+    transform_keys
+    transform_values
+    translate
+    trim
+    trunc
+    udf
+    unbase64
+    unhex
+    unix_timestamp
+    upper
+    var_pop
+    var_samp
+    variance
+    weekofyear
+    when
+    window
+    xxhash64
+    year
+    years
+    zip_with
+
+
+.. currentmodule:: pyspark.sql.avro.functions
+
+.. autosummary::
+    :toctree: api/
+
+    from_avro
+    to_avro
+
+Window
+------
+
+.. currentmodule:: pyspark.sql
+
+.. autosummary::
+    :toctree: api/
+
+    Window.currentRow
+    Window.orderBy
+    Window.partitionBy
+    Window.rangeBetween
+    Window.rowsBetween
+    Window.unboundedFollowing
+    Window.unboundedPreceding
+    WindowSpec.orderBy
+    WindowSpec.partitionBy
+    WindowSpec.rangeBetween
+    WindowSpec.rowsBetween
+
+Grouping
+--------
+
+.. currentmodule:: pyspark.sql
+
+.. autosummary::
+    :toctree: api/
+
+    GroupedData.agg
+    GroupedData.apply
+    GroupedData.applyInPandas
+    GroupedData.avg
+    GroupedData.cogroup
+    GroupedData.count
+    GroupedData.max
+    GroupedData.mean
+    GroupedData.min
+    GroupedData.pivot
+    GroupedData.sum
+    PandasCogroupedOps.applyInPandas
diff --git a/python/docs/source/reference/pyspark.ss.rst b/python/docs/source/reference/pyspark.ss.rst
new file mode 100644
index 0000000000000..a7936a4f2a59c
--- /dev/null
+++ b/python/docs/source/reference/pyspark.ss.rst
@@ -0,0 +1,90 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+====================
+Structured Streaming
+====================
+
+Core Classes
+------------
+
+.. currentmodule:: pyspark.sql.streaming
+
+.. autosummary::
+    :toctree: api/
+
+    DataStreamReader
+    DataStreamWriter
+    ForeachBatchFunction
+    StreamingQuery
+    StreamingQueryException
+    StreamingQueryManager
+
+Input and Output
+----------------
+
+.. currentmodule:: pyspark.sql.streaming
+
+.. autosummary::
+    :toctree: api/
+
+    DataStreamReader.csv
+    DataStreamReader.format
+    DataStreamReader.json
+    DataStreamReader.load
+    DataStreamReader.option
+    DataStreamReader.options
+    DataStreamReader.orc
+    DataStreamReader.parquet
+    DataStreamReader.schema
+    DataStreamReader.text
+    DataStreamWriter.foreach
+    DataStreamWriter.foreachBatch
+    DataStreamWriter.format
+    DataStreamWriter.option
+    DataStreamWriter.options
+    DataStreamWriter.outputMode
+    DataStreamWriter.partitionBy
+    DataStreamWriter.queryName
+    DataStreamWriter.start
+    DataStreamWriter.trigger
+
+Query Management
+----------------
+
+.. currentmodule:: pyspark.sql.streaming
+
+.. autosummary::
+    :toctree: api/
+
+    StreamingQuery.awaitTermination
+    StreamingQuery.exception
+    StreamingQuery.explain
+    StreamingQuery.id
+    StreamingQuery.isActive
+    StreamingQuery.lastProgress
+    StreamingQuery.name
+    StreamingQuery.processAllAvailable
+    StreamingQuery.recentProgress
+    StreamingQuery.runId
+    StreamingQuery.status
+    StreamingQuery.stop
+    StreamingQueryManager.active
+    StreamingQueryManager.awaitAnyTermination
+    StreamingQueryManager.get
+    StreamingQueryManager.resetTerminated
diff --git a/python/docs/source/reference/pyspark.streaming.rst b/python/docs/source/reference/pyspark.streaming.rst
new file mode 100644
index 0000000000000..57cbd00b67e4c
--- /dev/null
+++ b/python/docs/source/reference/pyspark.streaming.rst
@@ -0,0 +1,130 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+===============
+Spark Streaming
+===============
+
+Core Classes
+------------
+
+.. currentmodule:: pyspark.streaming
+
+.. autosummary::
+    :toctree: api/
+
+    StreamingContext
+    DStream
+
+
+Streaming Management
+--------------------
+
+.. currentmodule:: pyspark.streaming
+
+.. autosummary::
+    :toctree: api/
+
+    StreamingContext.addStreamingListener
+    StreamingContext.awaitTermination
+    StreamingContext.awaitTerminationOrTimeout
+    StreamingContext.checkpoint
+    StreamingContext.getActive
+    StreamingContext.getActiveOrCreate
+    StreamingContext.getOrCreate
+    StreamingContext.remember
+    StreamingContext.sparkContext
+    StreamingContext.start
+    StreamingContext.stop
+    StreamingContext.transform
+    StreamingContext.union
+
+
+Input and Output
+----------------
+
+.. autosummary::
+    :toctree: api/
+
+    StreamingContext.binaryRecordsStream
+    StreamingContext.queueStream
+    StreamingContext.socketTextStream
+    StreamingContext.textFileStream
+    DStream.pprint
+    DStream.saveAsTextFiles
+
+
+Transformations and Actions
+---------------------------
+
+.. currentmodule:: pyspark.streaming
+
+.. autosummary::
+    :toctree: api/
+
+    DStream.cache
+    DStream.checkpoint
+    DStream.cogroup
+    DStream.combineByKey
+    DStream.context
+    DStream.count
+    DStream.countByValue
+    DStream.countByValueAndWindow
+    DStream.countByWindow
+    DStream.filter
+    DStream.flatMap
+    DStream.flatMapValues
+    DStream.foreachRDD
+    DStream.fullOuterJoin
+    DStream.glom
+    DStream.groupByKey
+    DStream.groupByKeyAndWindow
+    DStream.join
+    DStream.leftOuterJoin
+    DStream.map
+    DStream.mapPartitions
+    DStream.mapPartitionsWithIndex
+    DStream.mapValues
+    DStream.partitionBy
+    DStream.persist
+    DStream.reduce
+    DStream.reduceByKey
+    DStream.reduceByKeyAndWindow
+    DStream.reduceByWindow
+    DStream.repartition
+    DStream.rightOuterJoin
+    DStream.slice
+    DStream.transform
+    DStream.transformWith
+    DStream.union
+    DStream.updateStateByKey
+    DStream.window
+
+
+Kinesis
+-------
+
+.. currentmodule:: pyspark.streaming.kinesis
+
+.. autosummary::
+    :toctree: api/
+
+    KinesisUtils.createStream
+    InitialPositionInStream.LATEST
+    InitialPositionInStream.TRIM_HORIZON
+
diff --git a/python/docs/source/user_guide/arrow_pandas.rst b/python/docs/source/user_guide/arrow_pandas.rst
new file mode 100644
index 0000000000000..9907db0d3ff76
--- /dev/null
+++ b/python/docs/source/user_guide/arrow_pandas.rst
@@ -0,0 +1,421 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+=======================
+Apache Arrow in PySpark
+=======================
+
+.. currentmodule:: pyspark.sql
+
+Apache Arrow is an in-memory columnar data format that is used in Spark to efficiently transfer
+data between JVM and Python processes. This currently is most beneficial to Python users that
+work with Pandas/NumPy data. Its usage is not automatic and might require some minor
+changes to configuration or code to take full advantage and ensure compatibility. This guide will
+give a high-level description of how to use Arrow in Spark and highlight any differences when
+working with Arrow-enabled data.
+
+Ensure PyArrow Installed
+------------------------
+
+To use Apache Arrow in PySpark, `the recommended version of PyArrow <arrow_pandas.rst#recommended-pandas-and-pyarrow-versions>`_
+should be installed.
+If you install PySpark using pip, then PyArrow can be brought in as an extra dependency of the
+SQL module with the command ``pip install pyspark[sql]``. Otherwise, you must ensure that PyArrow
+is installed and available on all cluster nodes.
+You can install using pip or conda from the conda-forge channel. See PyArrow
+`installation <https://arrow.apache.org/docs/python/install.html>`_ for details.
+
+Enabling for Conversion to/from Pandas
+--------------------------------------
+
+Arrow is available as an optimization when converting a Spark DataFrame to a Pandas DataFrame
+using the call :meth:`DataFrame.toPandas` and when creating a Spark DataFrame from a Pandas DataFrame with
+:meth:`SparkSession.createDataFrame`. To use Arrow when executing these calls, users need to first set
+the Spark configuration ``spark.sql.execution.arrow.pyspark.enabled`` to ``true``. This is disabled by default.
+
+In addition, optimizations enabled by ``spark.sql.execution.arrow.pyspark.enabled`` could fallback automatically
+to non-Arrow optimization implementation if an error occurs before the actual computation within Spark.
+This can be controlled by ``spark.sql.execution.arrow.pyspark.fallback.enabled``.
+
+.. literalinclude:: ../../../../examples/src/main/python/sql/arrow.py
+    :language: python
+    :lines: 35-48
+    :dedent: 4
+
+Using the above optimizations with Arrow will produce the same results as when Arrow is not
+enabled.
+
+Note that even with Arrow, :meth:`DataFrame.toPandas` results in the collection of all records in the
+DataFrame to the driver program and should be done on a small subset of the data. Not all Spark
+data types are currently supported and an error can be raised if a column has an unsupported type.
+If an error occurs during :meth:`SparkSession.createDataFrame`, Spark will fall back to create the
+DataFrame without Arrow.
+
+Pandas UDFs (a.k.a. Vectorized UDFs)
+------------------------------------
+
+.. currentmodule:: pyspark.sql.functions
+
+Pandas UDFs are user defined functions that are executed by Spark using
+Arrow to transfer data and Pandas to work with the data, which allows vectorized operations. A Pandas
+UDF is defined using the :meth:`pandas_udf` as a decorator or to wrap the function, and no additional
+configuration is required. A Pandas UDF behaves as a regular PySpark function API in general.
+
+Before Spark 3.0, Pandas UDFs used to be defined with ``pyspark.sql.functions.PandasUDFType``. From Spark 3.0
+with Python 3.6+, you can also use `Python type hints <https://www.python.org/dev/peps/pep-0484>`_.
+Using Python type hints is preferred and using ``pyspark.sql.functions.PandasUDFType`` will be deprecated in
+the future release.
+
+.. currentmodule:: pyspark.sql.types
+
+Note that the type hint should use ``pandas.Series`` in all cases but there is one variant
+that ``pandas.DataFrame`` should be used for its input or output type hint instead when the input
+or output column is of :class:`StructType`. The following example shows a Pandas UDF which takes long
+column, string column and struct column, and outputs a struct column. It requires the function to
+specify the type hints of ``pandas.Series`` and ``pandas.DataFrame`` as below:
+
+.. literalinclude:: ../../../../examples/src/main/python/sql/arrow.py
+    :language: python
+    :lines: 54-78
+    :dedent: 4
+
+In the following sections, it describes the combinations of the supported type hints. For simplicity,
+``pandas.DataFrame`` variant is omitted.
+
+Series to Series
+~~~~~~~~~~~~~~~~
+
+.. currentmodule:: pyspark.sql.functions
+
+The type hint can be expressed as ``pandas.Series``, ... -> ``pandas.Series``.
+
+By using :func:`pandas_udf` with the function having such type hints above, it creates a Pandas UDF where the given
+function takes one or more ``pandas.Series`` and outputs one ``pandas.Series``. The output of the function should
+always be of the same length as the input. Internally, PySpark will execute a Pandas UDF by splitting
+columns into batches and calling the function for each batch as a subset of the data, then concatenating
+the results together.
+
+The following example shows how to create this Pandas UDF that computes the product of 2 columns.
+
+.. literalinclude:: ../../../../examples/src/main/python/sql/arrow.py
+    :language: python
+    :lines: 82-112
+    :dedent: 4
+
+For detailed usage, please see :func:`pandas_udf`.
+
+Iterator of Series to Iterator of Series
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. currentmodule:: pyspark.sql.functions
+
+The type hint can be expressed as ``Iterator[pandas.Series]`` -> ``Iterator[pandas.Series]``.
+
+By using :func:`pandas_udf` with the function having such type hints above, it creates a Pandas UDF where the given
+function takes an iterator of ``pandas.Series`` and outputs an iterator of ``pandas.Series``. The
+length of the entire output from the function should be the same length of the entire input; therefore, it can
+prefetch the data from the input iterator as long as the lengths are the same.
+In this case, the created Pandas UDF requires one input column when the Pandas UDF is called. To use
+multiple input columns, a different type hint is required. See Iterator of Multiple Series to Iterator
+of Series.
+
+It is also useful when the UDF execution requires initializing some states although internally it works
+identically as Series to Series case. The pseudocode below illustrates the example.
+
+.. code-block:: python
+
+    @pandas_udf("long")
+    def calculate(iterator: Iterator[pd.Series]) -> Iterator[pd.Series]:
+        # Do some expensive initialization with a state
+        state = very_expensive_initialization()
+        for x in iterator:
+            # Use that state for whole iterator.
+            yield calculate_with_state(x, state)
+
+    df.select(calculate("value")).show()
+
+The following example shows how to create this Pandas UDF:
+
+.. literalinclude:: ../../../../examples/src/main/python/sql/arrow.py
+    :language: python
+    :lines: 116-138
+    :dedent: 4
+
+For detailed usage, please see :func:`pandas_udf`.
+
+Iterator of Multiple Series to Iterator of Series
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. currentmodule:: pyspark.sql.functions
+
+The type hint can be expressed as ``Iterator[Tuple[pandas.Series, ...]]`` -> ``Iterator[pandas.Series]``.
+
+By using :func:`pandas_udf` with the function having such type hints above, it creates a Pandas UDF where the
+given function takes an iterator of a tuple of multiple ``pandas.Series`` and outputs an iterator of ``pandas.Series``.
+In this case, the created pandas UDF requires multiple input columns as many as the series in the tuple
+when the Pandas UDF is called. Otherwise, it has the same characteristics and restrictions as Iterator of Series
+to Iterator of Series case.
+
+The following example shows how to create this Pandas UDF:
+
+.. literalinclude:: ../../../../examples/src/main/python/sql/arrow.py
+    :language: python
+    :lines: 142-165
+    :dedent: 4
+
+For detailed usage, please see :func:`pandas_udf`.
+
+Series to Scalar
+~~~~~~~~~~~~~~~~
+
+.. currentmodule:: pyspark.sql.functions
+
+The type hint can be expressed as ``pandas.Series``, ... -> ``Any``.
+
+By using :func:`pandas_udf` with the function having such type hints above, it creates a Pandas UDF similar
+to PySpark's aggregate functions. The given function takes `pandas.Series` and returns a scalar value.
+The return type should be a primitive data type, and the returned scalar can be either a python
+primitive type, e.g., ``int`` or ``float`` or a numpy data type, e.g., ``numpy.int64`` or ``numpy.float64``.
+``Any`` should ideally be a specific scalar type accordingly.
+
+.. currentmodule:: pyspark.sql
+
+This UDF can be also used with :meth:`GroupedData.agg` and `Window`.
+It defines an aggregation from one or more ``pandas.Series`` to a scalar value, where each ``pandas.Series``
+represents a column within the group or window.
+
+Note that this type of UDF does not support partial aggregation and all data for a group or window
+will be loaded into memory. Also, only unbounded window is supported with Grouped aggregate Pandas
+UDFs currently. The following example shows how to use this type of UDF to compute mean with a group-by
+and window operations:
+
+.. literalinclude:: ../../../../examples/src/main/python/sql/arrow.py
+    :language: python
+    :lines: 169-210
+    :dedent: 4
+
+.. currentmodule:: pyspark.sql.functions
+
+For detailed usage, please see :func:`pandas_udf`.
+
+Pandas Function APIs
+--------------------
+
+.. currentmodule:: pyspark.sql
+
+Pandas Function APIs can directly apply a Python native function against the whole :class:`DataFrame` by
+using Pandas instances. Internally it works similarly with Pandas UDFs by using Arrow to transfer
+data and Pandas to work with the data, which allows vectorized operations. However, a Pandas Function
+API behaves as a regular API under PySpark :class:`DataFrame` instead of :class:`Column`, and Python type hints in Pandas
+Functions APIs are optional and do not affect how it works internally at this moment although they
+might be required in the future.
+
+.. currentmodule:: pyspark.sql.functions
+
+From Spark 3.0, grouped map pandas UDF is now categorized as a separate Pandas Function API,
+``DataFrame.groupby().applyInPandas()``. It is still possible to use it with ``pyspark.sql.functions.PandasUDFType``
+and ``DataFrame.groupby().apply()`` as it was; however, it is preferred to use
+``DataFrame.groupby().applyInPandas()`` directly. Using ``pyspark.sql.functions.PandasUDFType`` will be deprecated
+in the future.
+
+Grouped Map
+~~~~~~~~~~~
+
+.. currentmodule:: pyspark.sql
+
+Grouped map operations with Pandas instances are supported by ``DataFrame.groupby().applyInPandas()``
+which requires a Python function that takes a ``pandas.DataFrame`` and return another ``pandas.DataFrame``.
+It maps each group to each ``pandas.DataFrame`` in the Python function.
+
+This API implements the "split-apply-combine" pattern which consists of three steps:
+
+* Split the data into groups by using :meth:`DataFrame.groupBy`.
+
+* Apply a function on each group. The input and output of the function are both ``pandas.DataFrame``. The input data contains all the rows and columns for each group.
+
+* Combine the results into a new PySpark :class:`DataFrame`.
+
+To use ``DataFrame.groupBy().applyInPandas()``, the user needs to define the following:
+
+* A Python function that defines the computation for each group.
+
+* A ``StructType`` object or a string that defines the schema of the output PySpark :class:`DataFrame`.
+
+The column labels of the returned ``pandas.DataFrame`` must either match the field names in the
+defined output schema if specified as strings, or match the field data types by position if not
+strings, e.g. integer indices. See `pandas.DataFrame <https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html#pandas.DataFrame>`_
+on how to label columns when constructing a ``pandas.DataFrame``.
+
+Note that all data for a group will be loaded into memory before the function is applied. This can
+lead to out of memory exceptions, especially if the group sizes are skewed. The configuration for
+`maxRecordsPerBatch <arrow_pandas.rst#setting-arrow-batch-size>`_ is not applied on groups and it is up to the user
+to ensure that the grouped data will fit into the available memory.
+
+The following example shows how to use ``DataFrame.groupby().applyInPandas()`` to subtract the mean from each value
+in the group.
+
+.. literalinclude:: ../../../../examples/src/main/python/sql/arrow.py
+    :language: python
+    :lines: 214-232
+    :dedent: 4
+
+For detailed usage, please see  please see :meth:`GroupedData.applyInPandas`
+
+Map
+~~~
+
+Map operations with Pandas instances are supported by :meth:`DataFrame.mapInPandas` which maps an iterator
+of ``pandas.DataFrame``\s to another iterator of ``pandas.DataFrame``\s that represents the current
+PySpark :class:`DataFrame` and returns the result as a PySpark :class:`DataFrame`. The function takes and outputs
+an iterator of ``pandas.DataFrame``. It can return the output of arbitrary length in contrast to some
+Pandas UDFs although internally it works similarly with Series to Series Pandas UDF.
+
+The following example shows how to use :meth:`DataFrame.mapInPandas`:
+
+.. literalinclude:: ../../../../examples/src/main/python/sql/arrow.py
+    :language: python
+    :lines: 236-247
+    :dedent: 4
+
+For detailed usage, please see :meth:`DataFrame.mapInPandas`.
+
+Co-grouped Map
+~~~~~~~~~~~~~~
+
+.. currentmodule:: pyspark.sql
+
+Co-grouped map operations with Pandas instances are supported by ``DataFrame.groupby().cogroup().applyInPandas()`` which
+allows two PySpark :class:`DataFrame`\s to be cogrouped by a common key and then a Python function applied to each
+cogroup. It consists of the following steps:
+
+* Shuffle the data such that the groups of each dataframe which share a key are cogrouped together.
+
+* Apply a function to each cogroup. The input of the function is two ``pandas.DataFrame`` (with an optional tuple representing the key). The output of the function is a ``pandas.DataFrame``.
+
+* Combine the ``pandas.DataFrame``\s from all groups into a new PySpark :class:`DataFrame`. 
+
+To use ``groupBy().cogroup().applyInPandas()``, the user needs to define the following:
+
+* A Python function that defines the computation for each cogroup.
+
+* A ``StructType`` object or a string that defines the schema of the output PySpark :class:`DataFrame`.
+
+The column labels of the returned ``pandas.DataFrame`` must either match the field names in the
+defined output schema if specified as strings, or match the field data types by position if not
+strings, e.g. integer indices. See `pandas.DataFrame <https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html#pandas.DataFrame>`_.
+on how to label columns when constructing a ``pandas.DataFrame``.
+
+Note that all data for a cogroup will be loaded into memory before the function is applied. This can lead to out of
+memory exceptions, especially if the group sizes are skewed. The configuration for `maxRecordsPerBatch <arrow_pandas.rst#setting-arrow-batch-size>`_
+is not applied and it is up to the user to ensure that the cogrouped data will fit into the available memory.
+
+The following example shows how to use ``DataFrame.groupby().cogroup().applyInPandas()`` to perform an asof join between two datasets.
+
+.. literalinclude:: ../../../../examples/src/main/python/sql/arrow.py
+    :language: python
+    :lines: 251-273
+    :dedent: 4
+
+
+For detailed usage, please see :meth:`PandasCogroupedOps.applyInPandas`
+
+Usage Notes
+-----------
+
+Supported SQL Types
+~~~~~~~~~~~~~~~~~~~
+
+.. currentmodule:: pyspark.sql.types
+
+Currently, all Spark SQL data types are supported by Arrow-based conversion except
+:class:`ArrayType` of :class:`TimestampType`, and nested :class:`StructType`.
+:class: `MapType` is only supported when using PyArrow 2.0.0 and above.
+
+Setting Arrow Batch Size
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Data partitions in Spark are converted into Arrow record batches, which can temporarily lead to
+high memory usage in the JVM. To avoid possible out of memory exceptions, the size of the Arrow
+record batches can be adjusted by setting the conf ``spark.sql.execution.arrow.maxRecordsPerBatch``
+to an integer that will determine the maximum number of rows for each batch. The default value is
+10,000 records per batch. If the number of columns is large, the value should be adjusted
+accordingly. Using this limit, each data partition will be made into 1 or more record batches for
+processing.
+
+Timestamp with Time Zone Semantics
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. currentmodule:: pyspark.sql
+
+Spark internally stores timestamps as UTC values, and timestamp data that is brought in without
+a specified time zone is converted as local time to UTC with microsecond resolution. When timestamp
+data is exported or displayed in Spark, the session time zone is used to localize the timestamp
+values. The session time zone is set with the configuration ``spark.sql.session.timeZone`` and will
+default to the JVM system local time zone if not set. Pandas uses a ``datetime64`` type with nanosecond
+resolution, ``datetime64[ns]``, with optional time zone on a per-column basis.
+
+When timestamp data is transferred from Spark to Pandas it will be converted to nanoseconds
+and each column will be converted to the Spark session time zone then localized to that time
+zone, which removes the time zone and displays values as local time. This will occur
+when calling :meth:`DataFrame.toPandas()` or ``pandas_udf`` with timestamp columns.
+
+When timestamp data is transferred from Pandas to Spark, it will be converted to UTC microseconds. This
+occurs when calling :meth:`SparkSession.createDataFrame` with a Pandas DataFrame or when returning a timestamp from a
+``pandas_udf``. These conversions are done automatically to ensure Spark will have data in the
+expected format, so it is not necessary to do any of these conversions yourself. Any nanosecond
+values will be truncated.
+
+Note that a standard UDF (non-Pandas) will load timestamp data as Python datetime objects, which is
+different than a Pandas timestamp. It is recommended to use Pandas time series functionality when
+working with timestamps in ``pandas_udf``\s to get the best performance, see
+`here <https://pandas.pydata.org/pandas-docs/stable/timeseries.html>`_ for details.
+
+Recommended Pandas and PyArrow Versions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+For usage with pyspark.sql, the minimum supported versions of Pandas is 0.23.2 and PyArrow is 1.0.0.
+Higher versions may be used, however, compatibility and data correctness can not be guaranteed and should
+be verified by the user.
+
+Compatibility Setting for PyArrow >= 0.15.0 and Spark 2.3.x, 2.4.x
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Since Arrow 0.15.0, a change in the binary IPC format requires an environment variable to be
+compatible with previous versions of Arrow <= 0.14.1. This is only necessary to do for PySpark
+users with versions 2.3.x and 2.4.x that have manually upgraded PyArrow to 0.15.0. The following
+can be added to ``conf/spark-env.sh`` to use the legacy Arrow IPC format:
+
+.. code-block:: bash
+
+    ARROW_PRE_0_15_IPC_FORMAT=1
+
+
+This will instruct PyArrow >= 0.15.0 to use the legacy IPC format with the older Arrow Java that
+is in Spark 2.3.x and 2.4.x. Not setting this environment variable will lead to a similar error as
+described in `SPARK-29367 <https://issues.apache.org/jira/browse/SPARK-29367>`_ when running
+``pandas_udf``\s or :meth:`DataFrame.toPandas` with Arrow enabled. More information about the Arrow IPC change can
+be read on the Arrow 0.15.0 release `blog <https://arrow.apache.org/blog/2019/10/06/0.15.0-release/#columnar-streaming-protocol-change-since-0140>`_.
+
+Setting Arrow ``self_destruct`` for memory savings
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Since Spark 3.2, the Spark configuration ``spark.sql.execution.arrow.pyspark.selfDestruct.enabled`` can be used to enable PyArrow's ``self_destruct`` feature, which can save memory when creating a Pandas DataFrame via ``toPandas`` by freeing Arrow-allocated memory while building the Pandas DataFrame.
+This option is experimental, and some operations may fail on the resulting Pandas DataFrame due to immutable backing arrays.
+Typically, you would see the error ``ValueError: buffer source array is read-only``.
+Newer versions of Pandas may fix these errors by improving support for such cases.
+You can work around this error by copying the column(s) beforehand.
+Additionally, this conversion may be slower because it is single-threaded.
diff --git a/python/docs/source/user_guide/index.rst b/python/docs/source/user_guide/index.rst
new file mode 100644
index 0000000000000..383effba81ee3
--- /dev/null
+++ b/python/docs/source/user_guide/index.rst
@@ -0,0 +1,38 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+==========
+User Guide
+==========
+
+There are basic guides shared with other languages in Programming Guides
+at `the Spark documentation <https://spark.apache.org/docs/latest/index.html#where-to-go-from-here>`_ as below:
+
+- `RDD Programming Guide <https://spark.apache.org/docs/latest/rdd-programming-guide.html>`_
+- `Spark SQL, DataFrames and Datasets Guide <https://spark.apache.org/docs/latest/sql-programming-guide.html>`_
+- `Structured Streaming Programming Guide <https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html>`_
+- `Spark Streaming Programming Guide <https://spark.apache.org/docs/latest/streaming-programming-guide.html>`_
+- `Machine Learning Library (MLlib) Guide <https://spark.apache.org/docs/latest/ml-guide.html>`_
+
+PySpark specific user guide is as follows:
+
+.. toctree::
+    :maxdepth: 2
+
+    python_packaging
+    arrow_pandas
diff --git a/python/docs/source/user_guide/python_packaging.rst b/python/docs/source/user_guide/python_packaging.rst
new file mode 100644
index 0000000000000..e61cd338f8c36
--- /dev/null
+++ b/python/docs/source/user_guide/python_packaging.rst
@@ -0,0 +1,253 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+=========================
+Python Package Management
+=========================
+
+When you want to run your PySpark application on a cluster such as YARN, Kubernetes, Mesos, etc., you need to make
+sure that your code and all used libraries are available on the executors.
+
+As an example let's say you may want to run the `Pandas UDF's examples <arrow_pandas.rst#series-to-scalar>`_.
+As it uses pyarrow as an underlying implementation we need to make sure to have pyarrow installed on each executor
+on the cluster. Otherwise you may get errors such as ``ModuleNotFoundError: No module named 'pyarrow'``.
+
+Here is the script ``app.py`` from the previous example that will be executed on the cluster:
+
+.. code-block:: python
+
+    import pandas as pd
+    from pyspark.sql.functions import pandas_udf
+    from pyspark.sql import SparkSession
+
+    def main(spark):
+        df = spark.createDataFrame(
+            [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)],
+            ("id", "v"))
+
+        @pandas_udf("double")
+        def mean_udf(v: pd.Series) -> float:
+            return v.mean()
+
+        print(df.groupby("id").agg(mean_udf(df['v'])).collect())
+
+
+    if __name__ == "__main__":
+        main(SparkSession.builder.getOrCreate())
+
+
+There are multiple ways to manage Python dependencies in the cluster:
+
+- Using PySpark Native Features
+- Using Conda
+- Using Virtualenv
+- Using PEX
+
+
+Using PySpark Native Features
+-----------------------------
+
+PySpark allows to upload Python files (``.py``), zipped Python packages (``.zip``), and Egg files (``.egg``)
+to the executors by:
+
+- Setting the configuration setting ``spark.submit.pyFiles``
+- Setting ``--py-files`` option in Spark scripts
+- Directly calling :meth:`pyspark.SparkContext.addPyFile` in applications
+
+This is a straightforward method to ship additional custom Python code to the cluster. You can just add individual files or zip whole
+packages and upload them. Using :meth:`pyspark.SparkContext.addPyFile` allows to upload code even after having started your job.
+
+However, it does not allow to add packages built as `Wheels <https://www.python.org/dev/peps/pep-0427/>`_ and therefore
+does not allow to include dependencies with native code.
+
+
+Using Conda
+-----------
+
+`Conda <https://docs.conda.io/en/latest/>`_ is one of the most widely-used Python package management systems. PySpark users can directly
+use a Conda environment to ship their third-party Python packages by leveraging
+`conda-pack <https://conda.github.io/conda-pack/spark.html>`_ which is a command line tool creating
+relocatable Conda environments.
+
+The example below creates a Conda environment to use on both the driver and executor and packs
+it into an archive file. This archive file captures the Conda environment for Python and stores
+both Python interpreter and all its relevant dependencies.
+
+.. code-block:: bash
+
+    conda create -y -n pyspark_conda_env -c conda-forge pyarrow pandas conda-pack
+    conda activate pyspark_conda_env
+    conda pack -f -o pyspark_conda_env.tar.gz
+
+After that, you can ship it together with scripts or in the code by using the ``--archives`` option
+or ``spark.archives`` configuration (``spark.yarn.dist.archives`` in YARN). It automatically unpacks the archive on executors.
+
+In the case of a ``spark-submit`` script, you can use it as follows:
+
+.. code-block:: bash
+
+    export PYSPARK_DRIVER_PYTHON=python # Do not set in cluster modes.
+    export PYSPARK_PYTHON=./environment/bin/python
+    spark-submit --archives pyspark_conda_env.tar.gz#environment app.py
+
+Note that ``PYSPARK_DRIVER_PYTHON`` above should not be set for cluster modes in YARN or Kubernetes.
+
+If you're on a regular Python shell or notebook, you can try it as shown below:
+
+.. code-block:: python
+
+    import os
+    from pyspark.sql import SparkSession
+    from app import main
+
+    os.environ['PYSPARK_PYTHON'] = "./environment/bin/python"
+    spark = SparkSession.builder.config(
+        "spark.archives",  # 'spark.yarn.dist.archives' in YARN.
+        "pyspark_conda_env.tar.gz#environment").getOrCreate()
+    main(spark)
+
+For a pyspark shell:
+
+.. code-block:: bash
+
+    export PYSPARK_DRIVER_PYTHON=python
+    export PYSPARK_PYTHON=./environment/bin/python
+    pyspark --archives pyspark_conda_env.tar.gz#environment
+
+
+Using Virtualenv
+----------------
+
+`Virtualenv <https://virtualenv.pypa.io/en/latest/>`_  is a Python tool to create isolated Python environments.
+Since Python 3.3, a subset of its features has been integrated into Python as a standard library under
+the `venv <https://docs.python.org/3/library/venv.html>`_ module. PySpark users can use virtualenv to manage
+Python dependencies in their clusters by using `venv-pack <https://jcristharif.com/venv-pack/index.html>`_
+in a similar way as conda-pack.
+
+A virtual environment to use on both driver and executor can be created as demonstrated below.
+It packs the current virtual environment to an archive file, and it contains both Python interpreter and the dependencies.
+However, it requires all nodes in a cluster to have the same Python interpreter installed because
+`venv-pack packs Python interpreter as a symbolic link <https://github.com/jcrist/venv-pack/issues/5>`_.
+
+
+.. code-block:: bash
+
+    python -m venv pyspark_venv
+    source pyspark_venv/bin/activate
+    pip install pyarrow pandas venv-pack
+    venv-pack -o pyspark_venv.tar.gz
+
+You can directly pass/unpack the archive file and enable the environment on executors by leveraging
+the ``--archives`` option or ``spark.archives`` configuration (``spark.yarn.dist.archives`` in YARN).
+
+For ``spark-submit``, you can use it by running the command as follows. Also, notice that
+``PYSPARK_DRIVER_PYTHON`` has to be unset in Kubernetes or YARN cluster modes.
+
+.. code-block:: bash
+
+    export PYSPARK_DRIVER_PYTHON=python # Do not set in cluster modes.
+    export PYSPARK_PYTHON=./environment/bin/python
+    spark-submit --archives pyspark_venv.tar.gz#environment app.py
+
+For regular Python shells or notebooks:
+
+.. code-block:: bash
+
+    import os
+    from pyspark.sql import SparkSession
+    from app import main
+
+    os.environ['PYSPARK_PYTHON'] = "./environment/bin/python"
+    spark = SparkSession.builder.config(
+        "spark.archives",  # 'spark.yarn.dist.archives' in YARN.
+        "pyspark_venv.tar.gz#environment").getOrCreate()
+    main(spark)
+
+In the case of a pyspark shell:
+
+.. code-block:: bash
+
+    export PYSPARK_DRIVER_PYTHON=python
+    export PYSPARK_PYTHON=./environment/bin/python
+    pyspark --archives pyspark_venv.tar.gz#environment
+
+
+Using PEX
+---------
+
+PySpark can also use `PEX <https://github.com/pantsbuild/pex>`_ to ship the Python packages
+together. PEX is a tool that creates a self-contained Python environment. This is similar
+to Conda or virtualenv, but a ``.pex`` file is executable by itself.
+
+The following example creates a ``.pex`` file for the driver and executor to use.
+The file contains the Python dependencies specified with the ``pex`` command.
+
+.. code-block:: bash
+
+    pip install pyarrow pandas pex
+    pex pyspark pyarrow pandas -o pyspark_pex_env.pex
+
+This file behaves similarly with a regular Python interpreter.
+
+.. code-block:: bash
+
+    ./pyspark_pex_env.pex -c "import pandas; print(pandas.__version__)"
+    1.1.5
+
+However, ``.pex`` file does not include a Python interpreter itself under the hood so all
+nodes in a cluster should have the same Python interpreter installed.
+
+In order to transfer and use the ``.pex`` file in a cluster, you should ship it via the
+``spark.files`` configuration (``spark.yarn.dist.files`` in YARN) or ``--files`` option because they are regular files instead
+of directories or archive files.
+
+For application submission, you run the commands as shown below.
+Note that ``PYSPARK_DRIVER_PYTHON`` should not be set for cluster modes in YARN or Kubernetes.
+
+.. code-block:: bash
+
+    export PYSPARK_DRIVER_PYTHON=python  # Do not set in cluster modes.
+    export PYSPARK_PYTHON=./pyspark_pex_env.pex
+    spark-submit --files pyspark_pex_env.pex app.py
+
+For regular Python shells or notebooks:
+
+.. code-block:: python
+
+    import os
+    from pyspark.sql import SparkSession
+    from app import main
+
+    os.environ['PYSPARK_PYTHON'] = "./pyspark_pex_env.pex"
+    spark = SparkSession.builder.config(
+        "spark.files",  # 'spark.yarn.dist.files' in YARN.
+        "pyspark_pex_env.pex").getOrCreate()
+    main(spark)
+
+For the interactive pyspark shell, the commands are almost the same:
+
+.. code-block:: bash
+
+    export PYSPARK_DRIVER_PYTHON=python
+    export PYSPARK_PYTHON=./pyspark_pex_env.pex
+    pyspark --files pyspark_pex_env.pex
+
+An end-to-end Docker example for deploying a standalone PySpark with ``SparkSession.builder`` and PEX
+can be found `here <https://github.com/criteo/cluster-pack/blob/master/examples/spark-with-S3/README.md>`_
+- it uses cluster-pack, a library on top of PEX that automatizes the the intermediate step of having
+to create & upload the PEX manually.
diff --git a/python/lib/py4j-0.10.9-src.zip b/python/lib/py4j-0.10.9-src.zip
deleted file mode 100644
index 2c49836147030..0000000000000
Binary files a/python/lib/py4j-0.10.9-src.zip and /dev/null differ
diff --git a/python/lib/py4j-0.10.9.2-src.zip b/python/lib/py4j-0.10.9.2-src.zip
new file mode 100644
index 0000000000000..f82d64efb24b5
Binary files /dev/null and b/python/lib/py4j-0.10.9.2-src.zip differ
diff --git a/python/mypy.ini b/python/mypy.ini
new file mode 100644
index 0000000000000..bbc4faba5f61c
--- /dev/null
+++ b/python/mypy.ini
@@ -0,0 +1,130 @@
+;
+; Licensed to the Apache Software Foundation (ASF) under one or more
+; contributor license agreements.  See the NOTICE file distributed with
+; this work for additional information regarding copyright ownership.
+; The ASF licenses this file to You under the Apache License, Version 2.0
+; (the "License"); you may not use this file except in compliance with
+; the License.  You may obtain a copy of the License at
+;
+;    http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing, software
+; distributed under the License is distributed on an "AS IS" BASIS,
+; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+; See the License for the specific language governing permissions and
+; limitations under the License.
+;
+
+[mypy]
+strict_optional = True
+no_implicit_optional = True
+disallow_untyped_defs = True
+
+; Allow untyped def in internal modules and tests
+
+[mypy-pyspark.daemon]
+disallow_untyped_defs = False
+
+[mypy-pyspark.find_spark_home]
+disallow_untyped_defs = False
+
+[mypy-pyspark._globals]
+disallow_untyped_defs = False
+
+[mypy-pyspark.install]
+disallow_untyped_defs = False
+
+[mypy-pyspark.java_gateway]
+disallow_untyped_defs = False
+
+[mypy-pyspark.join]
+disallow_untyped_defs = False
+
+[mypy-pyspark.ml.tests.*]
+disallow_untyped_defs = False
+
+[mypy-pyspark.mllib.tests.*]
+disallow_untyped_defs = False
+
+[mypy-pyspark.rddsampler]
+disallow_untyped_defs = False
+
+[mypy-pyspark.resource.tests.*]
+disallow_untyped_defs = False
+
+[mypy-pyspark.serializers]
+disallow_untyped_defs = False
+
+[mypy-pyspark.shuffle]
+disallow_untyped_defs = False
+
+[mypy-pyspark.streaming.tests.*]
+disallow_untyped_defs = False
+
+[mypy-pyspark.streaming.util]
+disallow_untyped_defs = False
+
+[mypy-pyspark.sql.tests.*]
+disallow_untyped_defs = False
+
+[mypy-pyspark.sql.pandas.serializers]
+disallow_untyped_defs = False
+
+[mypy-pyspark.sql.pandas.types]
+disallow_untyped_defs = False
+
+[mypy-pyspark.sql.pandas.typehints]
+disallow_untyped_defs = False
+
+[mypy-pyspark.sql.pandas.utils]
+disallow_untyped_defs = False
+
+[mypy-pyspark.sql.pandas._typing.protocols.*]
+disallow_untyped_defs = False
+
+[mypy-pyspark.sql.utils]
+disallow_untyped_defs = False
+
+[mypy-pyspark.tests.*]
+disallow_untyped_defs = False
+
+[mypy-pyspark.testing.*]
+disallow_untyped_defs = False
+
+[mypy-pyspark.traceback_utils]
+disallow_untyped_defs = False
+
+[mypy-pyspark.util]
+disallow_untyped_defs = False
+
+[mypy-pyspark.worker]
+disallow_untyped_defs = False
+
+; Ignore errors in embedded third party code
+
+[mypy-pyspark.cloudpickle.*]
+ignore_errors = True
+
+; Ignore missing imports for external untyped packages
+
+[mypy-py4j.*]
+ignore_missing_imports = True
+
+[mypy-numpy]
+ignore_missing_imports = True
+
+[mypy-scipy.*]
+ignore_missing_imports = True
+
+[mypy-pandas.*]
+ignore_missing_imports = True
+
+[mypy-pyarrow.*]
+ignore_missing_imports = True
+
+[mypy-psutil.*]
+ignore_missing_imports = True
+
+# TODO(SPARK-34941): Enable mypy for pandas-on-Spark
+[mypy-pyspark.pandas.*]
+ignore_errors = True
diff --git a/python/pylintrc b/python/pylintrc
index 6a675770da69a..548377416ca3d 100644
--- a/python/pylintrc
+++ b/python/pylintrc
@@ -27,9 +27,9 @@
 # Profiled execution.
 profile=no
 
-# Add files or directories to the blacklist. They should be base names, not
+# Add files or directories to the ignoreList. They should be base names, not
 # paths.
-ignore=pyspark.heapq3
+#ignore=
 
 # Pickle collected data for later comparisons.
 persistent=yes
diff --git a/python/pyspark/__init__.py b/python/pyspark/__init__.py
index 70c0b27a6aa33..8e36d9fc941af 100644
--- a/python/pyspark/__init__.py
+++ b/python/pyspark/__init__.py
@@ -42,25 +42,26 @@
       A :class:`TaskContext` that provides extra info and tooling for barrier execution.
   - :class:`BarrierTaskInfo`:
       Information about a barrier task.
+  - :class:`InheritableThread`:
+      A inheritable thread to use in Spark when the pinned thread mode is on.
 """
 
 from functools import wraps
 import types
 
 from pyspark.conf import SparkConf
-from pyspark.context import SparkContext
 from pyspark.rdd import RDD, RDDBarrier
 from pyspark.files import SparkFiles
+from pyspark.status import StatusTracker, SparkJobInfo, SparkStageInfo
+from pyspark.util import InheritableThread
 from pyspark.storagelevel import StorageLevel
 from pyspark.accumulators import Accumulator, AccumulatorParam
 from pyspark.broadcast import Broadcast
-from pyspark.resourceinformation import ResourceInformation
 from pyspark.serializers import MarshalSerializer, PickleSerializer
-from pyspark.status import *
 from pyspark.taskcontext import TaskContext, BarrierTaskContext, BarrierTaskInfo
 from pyspark.profiler import Profiler, BasicProfiler
 from pyspark.version import __version__
-from pyspark._globals import _NoValue
+from pyspark._globals import _NoValue  # noqa: F401
 
 
 def since(version):
@@ -101,7 +102,9 @@ def keyword_only(func):
     A decorator that forces keyword arguments in the wrapped method
     and saves actual input keyword arguments in `_input_kwargs`.
 
-    .. note:: Should only be used to wrap a method where first arg is `self`
+    Notes
+    -----
+    Should only be used to wrap a method where first arg is `self`
     """
     @wraps(func)
     def wrapper(self, *args, **kwargs):
@@ -111,13 +114,16 @@ def wrapper(self, *args, **kwargs):
         return func(self, **kwargs)
     return wrapper
 
+# To avoid circular dependencies
+from pyspark.context import SparkContext
 
 # for back compatibility
-from pyspark.sql import SQLContext, HiveContext, Row
+from pyspark.sql import SQLContext, HiveContext, Row  # noqa: F401
 
 __all__ = [
     "SparkConf", "SparkContext", "SparkFiles", "RDD", "StorageLevel", "Broadcast",
     "Accumulator", "AccumulatorParam", "MarshalSerializer", "PickleSerializer",
     "StatusTracker", "SparkJobInfo", "SparkStageInfo", "Profiler", "BasicProfiler", "TaskContext",
-    "RDDBarrier", "BarrierTaskContext", "BarrierTaskInfo", "ResourceInformation",
+    "RDDBarrier", "BarrierTaskContext", "BarrierTaskInfo", "InheritableThread",
+    "__version__",
 ]
diff --git a/python/pyspark/__init__.pyi b/python/pyspark/__init__.pyi
new file mode 100644
index 0000000000000..852de35496046
--- /dev/null
+++ b/python/pyspark/__init__.pyi
@@ -0,0 +1,74 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Callable, Optional, TypeVar
+
+from pyspark.accumulators import (  # noqa: F401
+    Accumulator as Accumulator,
+    AccumulatorParam as AccumulatorParam,
+)
+from pyspark.broadcast import Broadcast as Broadcast  # noqa: F401
+from pyspark.conf import SparkConf as SparkConf  # noqa: F401
+from pyspark.context import SparkContext as SparkContext  # noqa: F401
+from pyspark.files import SparkFiles as SparkFiles  # noqa: F401
+from pyspark.status import (
+    StatusTracker as StatusTracker,
+    SparkJobInfo as SparkJobInfo,
+    SparkStageInfo as SparkStageInfo,
+)  # noqa: F401
+from pyspark.profiler import (  # noqa: F401
+    BasicProfiler as BasicProfiler,
+    Profiler as Profiler,
+)
+from pyspark.rdd import RDD as RDD, RDDBarrier as RDDBarrier  # noqa: F401
+from pyspark.serializers import (  # noqa: F401
+    MarshalSerializer as MarshalSerializer,
+    PickleSerializer as PickleSerializer,
+)
+from pyspark.status import (  # noqa: F401
+    SparkJobInfo as SparkJobInfo,
+    SparkStageInfo as SparkStageInfo,
+    StatusTracker as StatusTracker,
+)
+from pyspark.storagelevel import StorageLevel as StorageLevel  # noqa: F401
+from pyspark.taskcontext import (  # noqa: F401
+    BarrierTaskContext as BarrierTaskContext,
+    BarrierTaskInfo as BarrierTaskInfo,
+    TaskContext as TaskContext,
+)
+from pyspark.util import InheritableThread as InheritableThread  # noqa: F401
+from pyspark.version import __version__ as __version__
+
+# Compatibility imports
+from pyspark.sql import (  # noqa: F401
+    SQLContext as SQLContext,
+    HiveContext as HiveContext,
+    Row as Row,
+)
+
+T = TypeVar("T")
+F = TypeVar("F", bound=Callable)
+
+def since(version: str) -> Callable[[T], T]: ...
+def copy_func(
+    f: F,
+    name: Optional[str] = ...,
+    sinceversion: Optional[str] = ...,
+    doc: Optional[str] = ...,
+) -> F: ...
+def keyword_only(func: F) -> F: ...
diff --git a/python/pyspark/_typing.pyi b/python/pyspark/_typing.pyi
new file mode 100644
index 0000000000000..637e4cb4fbccc
--- /dev/null
+++ b/python/pyspark/_typing.pyi
@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Callable, Iterable, Sized, TypeVar, Union
+from typing_extensions import Protocol
+
+F = TypeVar("F", bound=Callable)
+T = TypeVar("T", covariant=True)
+
+PrimitiveType = Union[bool, float, int, str]
+
+class SupportsIAdd(Protocol):
+    def __iadd__(self, other: SupportsIAdd) -> SupportsIAdd: ...
+
+class SupportsOrdering(Protocol):
+    def __le__(self, other: SupportsOrdering) -> bool: ...
+
+class SizedIterable(Protocol, Sized, Iterable[T]): ...
diff --git a/python/pyspark/accumulators.py b/python/pyspark/accumulators.py
index a5d513262b266..91e2b6abdd771 100644
--- a/python/pyspark/accumulators.py
+++ b/python/pyspark/accumulators.py
@@ -15,84 +15,10 @@
 # limitations under the License.
 #
 
-"""
->>> from pyspark.context import SparkContext
->>> sc = SparkContext('local', 'test')
->>> a = sc.accumulator(1)
->>> a.value
-1
->>> a.value = 2
->>> a.value
-2
->>> a += 5
->>> a.value
-7
-
->>> sc.accumulator(1.0).value
-1.0
-
->>> sc.accumulator(1j).value
-1j
-
->>> rdd = sc.parallelize([1,2,3])
->>> def f(x):
-...     global a
-...     a += x
->>> rdd.foreach(f)
->>> a.value
-13
-
->>> b = sc.accumulator(0)
->>> def g(x):
-...     b.add(x)
->>> rdd.foreach(g)
->>> b.value
-6
-
->>> from pyspark.accumulators import AccumulatorParam
->>> class VectorAccumulatorParam(AccumulatorParam):
-...     def zero(self, value):
-...         return [0.0] * len(value)
-...     def addInPlace(self, val1, val2):
-...         for i in range(len(val1)):
-...              val1[i] += val2[i]
-...         return val1
->>> va = sc.accumulator([1.0, 2.0, 3.0], VectorAccumulatorParam())
->>> va.value
-[1.0, 2.0, 3.0]
->>> def g(x):
-...     global va
-...     va += [x] * 3
->>> rdd.foreach(g)
->>> va.value
-[7.0, 8.0, 9.0]
-
->>> rdd.map(lambda x: a.value).collect() # doctest: +IGNORE_EXCEPTION_DETAIL
-Traceback (most recent call last):
-    ...
-Py4JJavaError:...
-
->>> def h(x):
-...     global a
-...     a.value = 7
->>> rdd.foreach(h) # doctest: +IGNORE_EXCEPTION_DETAIL
-Traceback (most recent call last):
-    ...
-Py4JJavaError:...
-
->>> sc.accumulator([1.0, 2.0, 3.0]) # doctest: +IGNORE_EXCEPTION_DETAIL
-Traceback (most recent call last):
-    ...
-TypeError:...
-"""
-
 import sys
 import select
 import struct
-if sys.version < '3':
-    import SocketServer
-else:
-    import socketserver as SocketServer
+import socketserver as SocketServer
 import threading
 from pyspark.serializers import read_int, PickleSerializer
 
@@ -129,7 +55,54 @@ class Accumulator(object):
 
     While :class:`SparkContext` supports accumulators for primitive data types like :class:`int` and
     :class:`float`, users can also define accumulators for custom types by providing a custom
-    :class:`AccumulatorParam` object. Refer to the doctest of this module for an example.
+    :py:class:`AccumulatorParam` object. Refer to its doctest for an example.
+
+    Examples
+    --------
+    >>> a = sc.accumulator(1)
+    >>> a.value
+    1
+    >>> a.value = 2
+    >>> a.value
+    2
+    >>> a += 5
+    >>> a.value
+    7
+    >>> sc.accumulator(1.0).value
+    1.0
+    >>> sc.accumulator(1j).value
+    1j
+    >>> rdd = sc.parallelize([1,2,3])
+    >>> def f(x):
+    ...     global a
+    ...     a += x
+    >>> rdd.foreach(f)
+    >>> a.value
+    13
+    >>> b = sc.accumulator(0)
+    >>> def g(x):
+    ...     b.add(x)
+    >>> rdd.foreach(g)
+    >>> b.value
+    6
+
+    >>> rdd.map(lambda x: a.value).collect() # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    Py4JJavaError: ...
+
+    >>> def h(x):
+    ...     global a
+    ...     a.value = 7
+    >>> rdd.foreach(h) # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    Py4JJavaError: ...
+
+    >>> sc.accumulator([1.0, 2.0, 3.0]) # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    TypeError: ...
     """
 
     def __init__(self, aid, value, accum_param):
@@ -180,6 +153,27 @@ class AccumulatorParam(object):
 
     """
     Helper object that defines how to accumulate values of a given type.
+
+    Examples
+    --------
+    >>> from pyspark.accumulators import AccumulatorParam
+    >>> class VectorAccumulatorParam(AccumulatorParam):
+    ...     def zero(self, value):
+    ...         return [0.0] * len(value)
+    ...     def addInPlace(self, val1, val2):
+    ...         for i in range(len(val1)):
+    ...              val1[i] += val2[i]
+    ...         return val1
+    >>> va = sc.accumulator([1.0, 2.0, 3.0], VectorAccumulatorParam())
+    >>> va.value
+    [1.0, 2.0, 3.0]
+    >>> def g(x):
+    ...     global va
+    ...     va += [x] * 3
+    >>> rdd = sc.parallelize([1,2,3])
+    >>> rdd.foreach(g)
+    >>> va.value
+    [7.0, 8.0, 9.0]
     """
 
     def zero(self, value):
@@ -296,6 +290,14 @@ def _start_update_server(auth_token):
 
 if __name__ == "__main__":
     import doctest
-    (failure_count, test_count) = doctest.testmod()
+
+    from pyspark.context import SparkContext
+    globs = globals().copy()
+    # The small batch size here ensures that we see multiple batches,
+    # even in these small test examples:
+    globs['sc'] = SparkContext('local', 'test')
+    (failure_count, test_count) = doctest.testmod(
+        globs=globs, optionflags=doctest.ELLIPSIS)
+    globs['sc'].stop()
     if failure_count:
         sys.exit(-1)
diff --git a/python/pyspark/accumulators.pyi b/python/pyspark/accumulators.pyi
new file mode 100644
index 0000000000000..13a1792cd247d
--- /dev/null
+++ b/python/pyspark/accumulators.pyi
@@ -0,0 +1,73 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Callable, Dict, Generic, Tuple, Type, TypeVar
+
+import socketserver.BaseRequestHandler  # type: ignore
+
+from pyspark._typing import SupportsIAdd
+
+T = TypeVar("T")
+U = TypeVar("U", bound=SupportsIAdd)
+
+import socketserver as SocketServer
+
+_accumulatorRegistry: Dict[int, Accumulator]
+
+class Accumulator(Generic[T]):
+    aid: int
+    accum_param: AccumulatorParam[T]
+    def __init__(
+        self, aid: int, value: T, accum_param: AccumulatorParam[T]
+    ) -> None: ...
+    def __reduce__(
+        self,
+    ) -> Tuple[
+        Callable[[int, int, AccumulatorParam[T]], Accumulator[T]],
+        Tuple[int, int, AccumulatorParam[T]],
+    ]: ...
+    @property
+    def value(self) -> T: ...
+    @value.setter
+    def value(self, value: T) -> None: ...
+    def add(self, term: T) -> None: ...
+    def __iadd__(self, term: T) -> Accumulator[T]: ...
+
+class AccumulatorParam(Generic[T]):
+    def zero(self, value: T) -> T: ...
+    def addInPlace(self, value1: T, value2: T) -> T: ...
+
+class AddingAccumulatorParam(AccumulatorParam[U]):
+    zero_value: U
+    def __init__(self, zero_value: U) -> None: ...
+    def zero(self, value: U) -> U: ...
+    def addInPlace(self, value1: U, value2: U) -> U: ...
+
+class _UpdateRequestHandler(SocketServer.StreamRequestHandler):
+    def handle(self) -> None: ...
+
+class AccumulatorServer(SocketServer.TCPServer):
+    auth_token: str
+    def __init__(
+        self,
+        server_address: Tuple[str, int],
+        RequestHandlerClass: Type[socketserver.BaseRequestHandler],
+        auth_token: str,
+    ) -> None: ...
+    server_shutdown: bool
+    def shutdown(self) -> None: ...
diff --git a/python/pyspark/broadcast.py b/python/pyspark/broadcast.py
index 803d857055dc0..1f3b053c05d19 100644
--- a/python/pyspark/broadcast.py
+++ b/python/pyspark/broadcast.py
@@ -20,16 +20,12 @@
 import sys
 from tempfile import NamedTemporaryFile
 import threading
+import pickle
 
 from pyspark.java_gateway import local_connect_and_auth
 from pyspark.serializers import ChunkedStream, pickle_protocol
-from pyspark.util import _exception_message, print_exec
+from pyspark.util import print_exec
 
-if sys.version < '3':
-    import cPickle as pickle
-else:
-    import pickle
-    unicode = str
 
 __all__ = ['Broadcast']
 
@@ -51,8 +47,8 @@ class Broadcast(object):
     A broadcast variable created with :meth:`SparkContext.broadcast`.
     Access its value through :attr:`value`.
 
-    Examples:
-
+    Examples
+    --------
     >>> from pyspark.context import SparkContext
     >>> sc = SparkContext('local', 'test')
     >>> b = sc.broadcast([1, 2, 3, 4, 5])
@@ -113,7 +109,7 @@ def dump(self, value, f):
             raise
         except Exception as e:
             msg = "Could not serialize broadcast: %s: %s" \
-                  % (e.__class__.__name__, _exception_message(e))
+                  % (e.__class__.__name__, str(e))
             print_exec(sys.stderr)
             raise pickle.PicklingError(msg)
         f.close()
@@ -152,7 +148,10 @@ def unpersist(self, blocking=False):
         broadcast is used after this is called, it will need to be
         re-sent to each executor.
 
-        :param blocking: Whether to block until unpersisting has completed
+        Parameters
+        ----------
+        blocking : bool, optional
+            Whether to block until unpersisting has completed
         """
         if self._jbroadcast is None:
             raise Exception("Broadcast can only be unpersisted in driver")
@@ -167,6 +166,11 @@ def destroy(self, blocking=False):
         .. versionchanged:: 3.0.0
            Added optional argument `blocking` to specify whether to block until all
            blocks are deleted.
+
+        Parameters
+        ----------
+        blocking : bool, optional
+            Whether to block until unpersisting has completed
         """
         if self._jbroadcast is None:
             raise Exception("Broadcast can only be destroyed in driver")
diff --git a/python/pyspark/broadcast.pyi b/python/pyspark/broadcast.pyi
new file mode 100644
index 0000000000000..944cb06d4178c
--- /dev/null
+++ b/python/pyspark/broadcast.pyi
@@ -0,0 +1,48 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import threading
+from typing import Any, Callable, Dict, Generic, Optional, Tuple, TypeVar
+
+T = TypeVar("T")
+
+_broadcastRegistry: Dict[int, Broadcast]
+
+class Broadcast(Generic[T]):
+    def __init__(
+        self,
+        sc: Optional[Any] = ...,
+        value: Optional[T] = ...,
+        pickle_registry: Optional[Any] = ...,
+        path: Optional[Any] = ...,
+        sock_file: Optional[Any] = ...,
+    ) -> None: ...
+    def dump(self, value: T, f: Any) -> None: ...
+    def load_from_path(self, path: Any) -> T: ...
+    def load(self, file: Any) -> T: ...
+    @property
+    def value(self) -> T: ...
+    def unpersist(self, blocking: bool = ...) -> None: ...
+    def destroy(self, blocking: bool = ...) -> None: ...
+    def __reduce__(self) -> Tuple[Callable[[int], T], Tuple[int]]: ...
+
+class BroadcastPickleRegistry(threading.local):
+    def __init__(self) -> None: ...
+    def __iter__(self) -> None: ...
+    def add(self, bcast: Any) -> None: ...
+    def clear(self) -> None: ...
diff --git a/python/pyspark/cloudpickle.py b/python/pyspark/cloudpickle.py
deleted file mode 100644
index 09d3a5e7cfb6f..0000000000000
--- a/python/pyspark/cloudpickle.py
+++ /dev/null
@@ -1,1362 +0,0 @@
-"""
-This class is defined to override standard pickle functionality
-
-The goals of it follow:
--Serialize lambdas and nested functions to compiled byte code
--Deal with main module correctly
--Deal with other non-serializable objects
-
-It does not include an unpickler, as standard python unpickling suffices.
-
-This module was extracted from the `cloud` package, developed by `PiCloud, Inc.
-<https://web.archive.org/web/20140626004012/http://www.picloud.com/>`_.
-
-Copyright (c) 2012, Regents of the University of California.
-Copyright (c) 2009 `PiCloud, Inc. <https://web.archive.org/web/20140626004012/http://www.picloud.com/>`_.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-    * Redistributions of source code must retain the above copyright
-      notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above copyright
-      notice, this list of conditions and the following disclaimer in the
-      documentation and/or other materials provided with the distribution.
-    * Neither the name of the University of California, Berkeley nor the
-      names of its contributors may be used to endorse or promote
-      products derived from this software without specific prior written
-      permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
-TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-"""
-from __future__ import print_function
-
-import dis
-from functools import partial
-import io
-import itertools
-import logging
-import opcode
-import operator
-import pickle
-import struct
-import sys
-import traceback
-import types
-import weakref
-import uuid
-import threading
-
-
-try:
-    from enum import Enum
-except ImportError:
-    Enum = None
-
-# cloudpickle is meant for inter process communication: we expect all
-# communicating processes to run the same Python version hence we favor
-# communication speed over compatibility:
-DEFAULT_PROTOCOL = pickle.HIGHEST_PROTOCOL
-
-# Track the provenance of reconstructed dynamic classes to make it possible to
-# recontruct instances from the matching singleton class definition when
-# appropriate and preserve the usual "isinstance" semantics of Python objects.
-_DYNAMIC_CLASS_TRACKER_BY_CLASS = weakref.WeakKeyDictionary()
-_DYNAMIC_CLASS_TRACKER_BY_ID = weakref.WeakValueDictionary()
-_DYNAMIC_CLASS_TRACKER_LOCK = threading.Lock()
-
-if sys.version_info[0] < 3:  # pragma: no branch
-    from pickle import Pickler
-    try:
-        from cStringIO import StringIO
-    except ImportError:
-        from StringIO import StringIO
-    string_types = (basestring,)  # noqa
-    PY3 = False
-    PY2 = True
-    PY2_WRAPPER_DESCRIPTOR_TYPE = type(object.__init__)
-    PY2_METHOD_WRAPPER_TYPE = type(object.__eq__)
-    PY2_CLASS_DICT_BLACKLIST = (PY2_METHOD_WRAPPER_TYPE,
-                                PY2_WRAPPER_DESCRIPTOR_TYPE)
-else:
-    types.ClassType = type
-    from pickle import _Pickler as Pickler
-    from io import BytesIO as StringIO
-    string_types = (str,)
-    PY3 = True
-    PY2 = False
-
-
-def _ensure_tracking(class_def):
-    with _DYNAMIC_CLASS_TRACKER_LOCK:
-        class_tracker_id = _DYNAMIC_CLASS_TRACKER_BY_CLASS.get(class_def)
-        if class_tracker_id is None:
-            class_tracker_id = uuid.uuid4().hex
-            _DYNAMIC_CLASS_TRACKER_BY_CLASS[class_def] = class_tracker_id
-            _DYNAMIC_CLASS_TRACKER_BY_ID[class_tracker_id] = class_def
-    return class_tracker_id
-
-
-def _lookup_class_or_track(class_tracker_id, class_def):
-    if class_tracker_id is not None:
-        with _DYNAMIC_CLASS_TRACKER_LOCK:
-            class_def = _DYNAMIC_CLASS_TRACKER_BY_ID.setdefault(
-                class_tracker_id, class_def)
-            _DYNAMIC_CLASS_TRACKER_BY_CLASS[class_def] = class_tracker_id
-    return class_def
-
-
-def _make_cell_set_template_code():
-    """Get the Python compiler to emit LOAD_FAST(arg); STORE_DEREF
-
-    Notes
-    -----
-    In Python 3, we could use an easier function:
-
-    .. code-block:: python
-
-       def f():
-           cell = None
-
-           def _stub(value):
-               nonlocal cell
-               cell = value
-
-           return _stub
-
-        _cell_set_template_code = f().__code__
-
-    This function is _only_ a LOAD_FAST(arg); STORE_DEREF, but that is
-    invalid syntax on Python 2. If we use this function we also don't need
-    to do the weird freevars/cellvars swap below
-    """
-    def inner(value):
-        lambda: cell  # make ``cell`` a closure so that we get a STORE_DEREF
-        cell = value
-
-    co = inner.__code__
-
-    # NOTE: we are marking the cell variable as a free variable intentionally
-    # so that we simulate an inner function instead of the outer function. This
-    # is what gives us the ``nonlocal`` behavior in a Python 2 compatible way.
-    if PY2:  # pragma: no branch
-        return types.CodeType(
-            co.co_argcount,
-            co.co_nlocals,
-            co.co_stacksize,
-            co.co_flags,
-            co.co_code,
-            co.co_consts,
-            co.co_names,
-            co.co_varnames,
-            co.co_filename,
-            co.co_name,
-            co.co_firstlineno,
-            co.co_lnotab,
-            co.co_cellvars,  # this is the trickery
-            (),
-        )
-    else:
-        if hasattr(types.CodeType, "co_posonlyargcount"):  # pragma: no branch
-            return types.CodeType(
-                co.co_argcount,
-                co.co_posonlyargcount,  # Python3.8 with PEP570
-                co.co_kwonlyargcount,
-                co.co_nlocals,
-                co.co_stacksize,
-                co.co_flags,
-                co.co_code,
-                co.co_consts,
-                co.co_names,
-                co.co_varnames,
-                co.co_filename,
-                co.co_name,
-                co.co_firstlineno,
-                co.co_lnotab,
-                co.co_cellvars,  # this is the trickery
-                (),
-            )
-        else:
-            return types.CodeType(
-                co.co_argcount,
-                co.co_kwonlyargcount,
-                co.co_nlocals,
-                co.co_stacksize,
-                co.co_flags,
-                co.co_code,
-                co.co_consts,
-                co.co_names,
-                co.co_varnames,
-                co.co_filename,
-                co.co_name,
-                co.co_firstlineno,
-                co.co_lnotab,
-                co.co_cellvars,  # this is the trickery
-                (),
-            )
-
-_cell_set_template_code = _make_cell_set_template_code()
-
-
-def cell_set(cell, value):
-    """Set the value of a closure cell.
-    """
-    return types.FunctionType(
-        _cell_set_template_code,
-        {},
-        '_cell_set_inner',
-        (),
-        (cell,),
-    )(value)
-
-
-# relevant opcodes
-STORE_GLOBAL = opcode.opmap['STORE_GLOBAL']
-DELETE_GLOBAL = opcode.opmap['DELETE_GLOBAL']
-LOAD_GLOBAL = opcode.opmap['LOAD_GLOBAL']
-GLOBAL_OPS = (STORE_GLOBAL, DELETE_GLOBAL, LOAD_GLOBAL)
-HAVE_ARGUMENT = dis.HAVE_ARGUMENT
-EXTENDED_ARG = dis.EXTENDED_ARG
-
-
-def islambda(func):
-    return getattr(func, '__name__') == '<lambda>'
-
-
-_BUILTIN_TYPE_NAMES = {}
-for k, v in types.__dict__.items():
-    if type(v) is type:
-        _BUILTIN_TYPE_NAMES[v] = k
-
-
-def _builtin_type(name):
-    return getattr(types, name)
-
-
-def _make__new__factory(type_):
-    def _factory():
-        return type_.__new__
-    return _factory
-
-
-# NOTE: These need to be module globals so that they're pickleable as globals.
-_get_dict_new = _make__new__factory(dict)
-_get_frozenset_new = _make__new__factory(frozenset)
-_get_list_new = _make__new__factory(list)
-_get_set_new = _make__new__factory(set)
-_get_tuple_new = _make__new__factory(tuple)
-_get_object_new = _make__new__factory(object)
-
-# Pre-defined set of builtin_function_or_method instances that can be
-# serialized.
-_BUILTIN_TYPE_CONSTRUCTORS = {
-    dict.__new__: _get_dict_new,
-    frozenset.__new__: _get_frozenset_new,
-    set.__new__: _get_set_new,
-    list.__new__: _get_list_new,
-    tuple.__new__: _get_tuple_new,
-    object.__new__: _get_object_new,
-}
-
-
-if sys.version_info < (3, 4):  # pragma: no branch
-    def _walk_global_ops(code):
-        """
-        Yield (opcode, argument number) tuples for all
-        global-referencing instructions in *code*.
-        """
-        code = getattr(code, 'co_code', b'')
-        if PY2:  # pragma: no branch
-            code = map(ord, code)
-
-        n = len(code)
-        i = 0
-        extended_arg = 0
-        while i < n:
-            op = code[i]
-            i += 1
-            if op >= HAVE_ARGUMENT:
-                oparg = code[i] + code[i + 1] * 256 + extended_arg
-                extended_arg = 0
-                i += 2
-                if op == EXTENDED_ARG:
-                    extended_arg = oparg * 65536
-                if op in GLOBAL_OPS:
-                    yield op, oparg
-
-else:
-    def _walk_global_ops(code):
-        """
-        Yield (opcode, argument number) tuples for all
-        global-referencing instructions in *code*.
-        """
-        for instr in dis.get_instructions(code):
-            op = instr.opcode
-            if op in GLOBAL_OPS:
-                yield op, instr.arg
-
-
-def _extract_class_dict(cls):
-    """Retrieve a copy of the dict of a class without the inherited methods"""
-    clsdict = dict(cls.__dict__)  # copy dict proxy to a dict
-    if len(cls.__bases__) == 1:
-        inherited_dict = cls.__bases__[0].__dict__
-    else:
-        inherited_dict = {}
-        for base in reversed(cls.__bases__):
-            inherited_dict.update(base.__dict__)
-    to_remove = []
-    for name, value in clsdict.items():
-        try:
-            base_value = inherited_dict[name]
-            if value is base_value:
-                to_remove.append(name)
-            elif PY2:
-                # backward compat for Python 2
-                if hasattr(value, "im_func"):
-                    if value.im_func is getattr(base_value, "im_func", None):
-                        to_remove.append(name)
-                elif isinstance(value, PY2_CLASS_DICT_BLACKLIST):
-                    # On Python 2 we have no way to pickle those specific
-                    # methods types nor to check that they are actually
-                    # inherited. So we assume that they are always inherited
-                    # from builtin types.
-                    to_remove.append(name)
-        except KeyError:
-            pass
-    for name in to_remove:
-        clsdict.pop(name)
-    return clsdict
-
-
-class CloudPickler(Pickler):
-
-    dispatch = Pickler.dispatch.copy()
-
-    def __init__(self, file, protocol=None):
-        if protocol is None:
-            protocol = DEFAULT_PROTOCOL
-        Pickler.__init__(self, file, protocol=protocol)
-        # map ids to dictionary. used to ensure that functions can share global env
-        self.globals_ref = {}
-
-    def dump(self, obj):
-        self.inject_addons()
-        try:
-            return Pickler.dump(self, obj)
-        except RuntimeError as e:
-            if 'recursion' in e.args[0]:
-                msg = """Could not pickle object as excessively deep recursion required."""
-                raise pickle.PicklingError(msg)
-            else:
-                raise
-
-    def save_memoryview(self, obj):
-        self.save(obj.tobytes())
-
-    dispatch[memoryview] = save_memoryview
-
-    if PY2:  # pragma: no branch
-        def save_buffer(self, obj):
-            self.save(str(obj))
-
-        dispatch[buffer] = save_buffer  # noqa: F821 'buffer' was removed in Python 3
-
-    def save_module(self, obj):
-        """
-        Save a module as an import
-        """
-        if _is_dynamic(obj):
-            self.save_reduce(dynamic_subimport, (obj.__name__, vars(obj)),
-                             obj=obj)
-        else:
-            self.save_reduce(subimport, (obj.__name__,), obj=obj)
-
-    dispatch[types.ModuleType] = save_module
-
-    def save_codeobject(self, obj):
-        """
-        Save a code object
-        """
-        if PY3:  # pragma: no branch
-            if hasattr(obj, "co_posonlyargcount"):  # pragma: no branch
-                args = (
-                    obj.co_argcount, obj.co_posonlyargcount,
-                    obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
-                    obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
-                    obj.co_varnames, obj.co_filename, obj.co_name,
-                    obj.co_firstlineno, obj.co_lnotab, obj.co_freevars,
-                    obj.co_cellvars
-                )
-            else:
-                args = (
-                    obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals,
-                    obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts,
-                    obj.co_names, obj.co_varnames, obj.co_filename,
-                    obj.co_name, obj.co_firstlineno, obj.co_lnotab,
-                    obj.co_freevars, obj.co_cellvars
-                )
-        else:
-            args = (
-                obj.co_argcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, obj.co_code,
-                obj.co_consts, obj.co_names, obj.co_varnames, obj.co_filename, obj.co_name,
-                obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, obj.co_cellvars
-            )
-        self.save_reduce(types.CodeType, args, obj=obj)
-
-    dispatch[types.CodeType] = save_codeobject
-
-    def save_function(self, obj, name=None):
-        """ Registered with the dispatch to handle all function types.
-
-        Determines what kind of function obj is (e.g. lambda, defined at
-        interactive prompt, etc) and handles the pickling appropriately.
-        """
-        try:
-            should_special_case = obj in _BUILTIN_TYPE_CONSTRUCTORS
-        except TypeError:
-            # Methods of builtin types aren't hashable in python 2.
-            should_special_case = False
-
-        if should_special_case:
-            # We keep a special-cased cache of built-in type constructors at
-            # global scope, because these functions are structured very
-            # differently in different python versions and implementations (for
-            # example, they're instances of types.BuiltinFunctionType in
-            # CPython, but they're ordinary types.FunctionType instances in
-            # PyPy).
-            #
-            # If the function we've received is in that cache, we just
-            # serialize it as a lookup into the cache.
-            return self.save_reduce(_BUILTIN_TYPE_CONSTRUCTORS[obj], (), obj=obj)
-
-        write = self.write
-
-        if name is None:
-            name = obj.__name__
-        try:
-            # whichmodule() could fail, see
-            # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling
-            modname = pickle.whichmodule(obj, name)
-        except Exception:
-            modname = None
-        # print('which gives %s %s %s' % (modname, obj, name))
-        try:
-            themodule = sys.modules[modname]
-        except KeyError:
-            # eval'd items such as namedtuple give invalid items for their function __module__
-            modname = '__main__'
-
-        if modname == '__main__':
-            themodule = None
-
-        try:
-            lookedup_by_name = getattr(themodule, name, None)
-        except Exception:
-            lookedup_by_name = None
-
-        if themodule:
-            if lookedup_by_name is obj:
-                return self.save_global(obj, name)
-
-        # a builtin_function_or_method which comes in as an attribute of some
-        # object (e.g., itertools.chain.from_iterable) will end
-        # up with modname "__main__" and so end up here. But these functions
-        # have no __code__ attribute in CPython, so the handling for
-        # user-defined functions below will fail.
-        # So we pickle them here using save_reduce; have to do it differently
-        # for different python versions.
-        if not hasattr(obj, '__code__'):
-            if PY3:  # pragma: no branch
-                rv = obj.__reduce_ex__(self.proto)
-            else:
-                if hasattr(obj, '__self__'):
-                    rv = (getattr, (obj.__self__, name))
-                else:
-                    raise pickle.PicklingError("Can't pickle %r" % obj)
-            return self.save_reduce(obj=obj, *rv)
-
-        # if func is lambda, def'ed at prompt, is in main, or is nested, then
-        # we'll pickle the actual function object rather than simply saving a
-        # reference (as is done in default pickler), via save_function_tuple.
-        if (islambda(obj)
-                or getattr(obj.__code__, 'co_filename', None) == '<stdin>'
-                or themodule is None):
-            self.save_function_tuple(obj)
-            return
-        else:
-            # func is nested
-            if lookedup_by_name is None or lookedup_by_name is not obj:
-                self.save_function_tuple(obj)
-                return
-
-        if obj.__dict__:
-            # essentially save_reduce, but workaround needed to avoid recursion
-            self.save(_restore_attr)
-            write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n')
-            self.memoize(obj)
-            self.save(obj.__dict__)
-            write(pickle.TUPLE + pickle.REDUCE)
-        else:
-            write(pickle.GLOBAL + modname + '\n' + name + '\n')
-            self.memoize(obj)
-
-    dispatch[types.FunctionType] = save_function
-
-    def _save_subimports(self, code, top_level_dependencies):
-        """
-        Save submodules used by a function but not listed in its globals.
-
-        In the example below:
-
-        ```
-        import concurrent.futures
-        import cloudpickle
-
-
-        def func():
-            x = concurrent.futures.ThreadPoolExecutor
-
-
-        if __name__ == '__main__':
-            cloudpickle.dumps(func)
-        ```
-
-        the globals extracted by cloudpickle in the function's state include
-        the concurrent module, but not its submodule (here,
-        concurrent.futures), which is the module used by func.
-
-        To ensure that calling the depickled function does not raise an
-        AttributeError, this function looks for any currently loaded submodule
-        that the function uses and whose parent is present in the function
-        globals, and saves it before saving the function.
-        """
-
-        # check if any known dependency is an imported package
-        for x in top_level_dependencies:
-            if isinstance(x, types.ModuleType) and hasattr(x, '__package__') and x.__package__:
-                # check if the package has any currently loaded sub-imports
-                prefix = x.__name__ + '.'
-                # A concurrent thread could mutate sys.modules,
-                # make sure we iterate over a copy to avoid exceptions
-                for name in list(sys.modules):
-                    # Older versions of pytest will add a "None" module to sys.modules.
-                    if name is not None and name.startswith(prefix):
-                        # check whether the function can address the sub-module
-                        tokens = set(name[len(prefix):].split('.'))
-                        if not tokens - set(code.co_names):
-                            # ensure unpickler executes this import
-                            self.save(sys.modules[name])
-                            # then discards the reference to it
-                            self.write(pickle.POP)
-
-    def _save_dynamic_enum(self, obj, clsdict):
-        """Special handling for dynamic Enum subclasses
-
-        Use a dedicated Enum constructor (inspired by EnumMeta.__call__) as the
-        EnumMeta metaclass has complex initialization that makes the Enum
-        subclasses hold references to their own instances.
-        """
-        members = dict((e.name, e.value) for e in obj)
-
-        # Python 2.7 with enum34 can have no qualname:
-        qualname = getattr(obj, "__qualname__", None)
-
-        self.save_reduce(_make_skeleton_enum,
-                         (obj.__bases__, obj.__name__, qualname, members,
-                          obj.__module__, _ensure_tracking(obj), None),
-                         obj=obj)
-
-        # Cleanup the clsdict that will be passed to _rehydrate_skeleton_class:
-        # Those attributes are already handled by the metaclass.
-        for attrname in ["_generate_next_value_", "_member_names_",
-                         "_member_map_", "_member_type_",
-                         "_value2member_map_"]:
-            clsdict.pop(attrname, None)
-        for member in members:
-            clsdict.pop(member)
-
-    def save_dynamic_class(self, obj):
-        """Save a class that can't be stored as module global.
-
-        This method is used to serialize classes that are defined inside
-        functions, or that otherwise can't be serialized as attribute lookups
-        from global modules.
-        """
-        clsdict = _extract_class_dict(obj)
-        clsdict.pop('__weakref__', None)
-
-        # For ABCMeta in python3.7+, remove _abc_impl as it is not picklable.
-        # This is a fix which breaks the cache but this only makes the first
-        # calls to issubclass slower.
-        if "_abc_impl" in clsdict:
-            import abc
-            (registry, _, _, _) = abc._get_dump(obj)
-            clsdict["_abc_impl"] = [subclass_weakref()
-                                    for subclass_weakref in registry]
-
-        # On PyPy, __doc__ is a readonly attribute, so we need to include it in
-        # the initial skeleton class.  This is safe because we know that the
-        # doc can't participate in a cycle with the original class.
-        type_kwargs = {'__doc__': clsdict.pop('__doc__', None)}
-
-        if hasattr(obj, "__slots__"):
-            type_kwargs['__slots__'] = obj.__slots__
-            # pickle string length optimization: member descriptors of obj are
-            # created automatically from obj's __slots__ attribute, no need to
-            # save them in obj's state
-            if isinstance(obj.__slots__, string_types):
-                clsdict.pop(obj.__slots__)
-            else:
-                for k in obj.__slots__:
-                    clsdict.pop(k, None)
-
-        # If type overrides __dict__ as a property, include it in the type
-        # kwargs. In Python 2, we can't set this attribute after construction.
-        __dict__ = clsdict.pop('__dict__', None)
-        if isinstance(__dict__, property):
-            type_kwargs['__dict__'] = __dict__
-
-        save = self.save
-        write = self.write
-
-        # We write pickle instructions explicitly here to handle the
-        # possibility that the type object participates in a cycle with its own
-        # __dict__. We first write an empty "skeleton" version of the class and
-        # memoize it before writing the class' __dict__ itself. We then write
-        # instructions to "rehydrate" the skeleton class by restoring the
-        # attributes from the __dict__.
-        #
-        # A type can appear in a cycle with its __dict__ if an instance of the
-        # type appears in the type's __dict__ (which happens for the stdlib
-        # Enum class), or if the type defines methods that close over the name
-        # of the type, (which is common for Python 2-style super() calls).
-
-        # Push the rehydration function.
-        save(_rehydrate_skeleton_class)
-
-        # Mark the start of the args tuple for the rehydration function.
-        write(pickle.MARK)
-
-        # Create and memoize an skeleton class with obj's name and bases.
-        if Enum is not None and issubclass(obj, Enum):
-            # Special handling of Enum subclasses
-            self._save_dynamic_enum(obj, clsdict)
-        else:
-            # "Regular" class definition:
-            tp = type(obj)
-            self.save_reduce(_make_skeleton_class,
-                             (tp, obj.__name__, obj.__bases__, type_kwargs,
-                              _ensure_tracking(obj), None),
-                             obj=obj)
-
-        # Now save the rest of obj's __dict__. Any references to obj
-        # encountered while saving will point to the skeleton class.
-        save(clsdict)
-
-        # Write a tuple of (skeleton_class, clsdict).
-        write(pickle.TUPLE)
-
-        # Call _rehydrate_skeleton_class(skeleton_class, clsdict)
-        write(pickle.REDUCE)
-
-    def save_function_tuple(self, func):
-        """  Pickles an actual func object.
-
-        A func comprises: code, globals, defaults, closure, and dict.  We
-        extract and save these, injecting reducing functions at certain points
-        to recreate the func object.  Keep in mind that some of these pieces
-        can contain a ref to the func itself.  Thus, a naive save on these
-        pieces could trigger an infinite loop of save's.  To get around that,
-        we first create a skeleton func object using just the code (this is
-        safe, since this won't contain a ref to the func), and memoize it as
-        soon as it's created.  The other stuff can then be filled in later.
-        """
-        if is_tornado_coroutine(func):
-            self.save_reduce(_rebuild_tornado_coroutine, (func.__wrapped__,),
-                             obj=func)
-            return
-
-        save = self.save
-        write = self.write
-
-        code, f_globals, defaults, closure_values, dct, base_globals = self.extract_func_data(func)
-
-        save(_fill_function)  # skeleton function updater
-        write(pickle.MARK)    # beginning of tuple that _fill_function expects
-
-        self._save_subimports(
-            code,
-            itertools.chain(f_globals.values(), closure_values or ()),
-        )
-
-        # create a skeleton function object and memoize it
-        save(_make_skel_func)
-        save((
-            code,
-            len(closure_values) if closure_values is not None else -1,
-            base_globals,
-        ))
-        write(pickle.REDUCE)
-        self.memoize(func)
-
-        # save the rest of the func data needed by _fill_function
-        state = {
-            'globals': f_globals,
-            'defaults': defaults,
-            'dict': dct,
-            'closure_values': closure_values,
-            'module': func.__module__,
-            'name': func.__name__,
-            'doc': func.__doc__,
-        }
-        if hasattr(func, '__annotations__') and sys.version_info >= (3, 7):
-            state['annotations'] = func.__annotations__
-        if hasattr(func, '__qualname__'):
-            state['qualname'] = func.__qualname__
-        if hasattr(func, '__kwdefaults__'):
-            state['kwdefaults'] = func.__kwdefaults__
-        save(state)
-        write(pickle.TUPLE)
-        write(pickle.REDUCE)  # applies _fill_function on the tuple
-
-    _extract_code_globals_cache = (
-        weakref.WeakKeyDictionary()
-        if not hasattr(sys, "pypy_version_info")
-        else {})
-
-    @classmethod
-    def extract_code_globals(cls, co):
-        """
-        Find all globals names read or written to by codeblock co
-        """
-        out_names = cls._extract_code_globals_cache.get(co)
-        if out_names is None:
-            try:
-                names = co.co_names
-            except AttributeError:
-                # PyPy "builtin-code" object
-                out_names = set()
-            else:
-                out_names = {names[oparg] for _, oparg in _walk_global_ops(co)}
-
-                # see if nested function have any global refs
-                if co.co_consts:
-                    for const in co.co_consts:
-                        if type(const) is types.CodeType:
-                            out_names |= cls.extract_code_globals(const)
-
-            cls._extract_code_globals_cache[co] = out_names
-
-        return out_names
-
-    def extract_func_data(self, func):
-        """
-        Turn the function into a tuple of data necessary to recreate it:
-            code, globals, defaults, closure_values, dict
-        """
-        code = func.__code__
-
-        # extract all global ref's
-        func_global_refs = self.extract_code_globals(code)
-
-        # process all variables referenced by global environment
-        f_globals = {}
-        for var in func_global_refs:
-            if var in func.__globals__:
-                f_globals[var] = func.__globals__[var]
-
-        # defaults requires no processing
-        defaults = func.__defaults__
-
-        # process closure
-        closure = (
-            list(map(_get_cell_contents, func.__closure__))
-            if func.__closure__ is not None
-            else None
-        )
-
-        # save the dict
-        dct = func.__dict__
-
-        # base_globals represents the future global namespace of func at
-        # unpickling time. Looking it up and storing it in globals_ref allow
-        # functions sharing the same globals at pickling time to also
-        # share them once unpickled, at one condition: since globals_ref is
-        # an attribute of a Cloudpickler instance, and that a new CloudPickler is
-        # created each time pickle.dump or pickle.dumps is called, functions
-        # also need to be saved within the same invokation of
-        # cloudpickle.dump/cloudpickle.dumps (for example: cloudpickle.dumps([f1, f2])). There
-        # is no such limitation when using Cloudpickler.dump, as long as the
-        # multiple invokations are bound to the same Cloudpickler.
-        base_globals = self.globals_ref.setdefault(id(func.__globals__), {})
-
-        if base_globals == {}:
-            # Add module attributes used to resolve relative imports
-            # instructions inside func.
-            for k in ["__package__", "__name__", "__path__", "__file__"]:
-                # Some built-in functions/methods such as object.__new__  have
-                # their __globals__ set to None in PyPy
-                if func.__globals__ is not None and k in func.__globals__:
-                    base_globals[k] = func.__globals__[k]
-
-        return (code, f_globals, defaults, closure, dct, base_globals)
-
-    def save_builtin_function(self, obj):
-        if obj.__module__ == "__builtin__":
-            return self.save_global(obj)
-        return self.save_function(obj)
-
-    dispatch[types.BuiltinFunctionType] = save_builtin_function
-
-    def save_global(self, obj, name=None, pack=struct.pack):
-        """
-        Save a "global".
-
-        The name of this method is somewhat misleading: all types get
-        dispatched here.
-        """
-        if obj is type(None):
-            return self.save_reduce(type, (None,), obj=obj)
-        elif obj is type(Ellipsis):
-            return self.save_reduce(type, (Ellipsis,), obj=obj)
-        elif obj is type(NotImplemented):
-            return self.save_reduce(type, (NotImplemented,), obj=obj)
-
-        if obj.__module__ == "__main__":
-            return self.save_dynamic_class(obj)
-
-        try:
-            return Pickler.save_global(self, obj, name=name)
-        except Exception:
-            if obj.__module__ == "__builtin__" or obj.__module__ == "builtins":
-                if obj in _BUILTIN_TYPE_NAMES:
-                    return self.save_reduce(
-                        _builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj)
-
-            typ = type(obj)
-            if typ is not obj and isinstance(obj, (type, types.ClassType)):
-                return self.save_dynamic_class(obj)
-
-            raise
-
-    dispatch[type] = save_global
-    dispatch[types.ClassType] = save_global
-
-    def save_instancemethod(self, obj):
-        # Memoization rarely is ever useful due to python bounding
-        if obj.__self__ is None:
-            self.save_reduce(getattr, (obj.im_class, obj.__name__))
-        else:
-            if PY3:  # pragma: no branch
-                self.save_reduce(types.MethodType, (obj.__func__, obj.__self__), obj=obj)
-            else:
-                self.save_reduce(types.MethodType, (obj.__func__, obj.__self__, obj.__self__.__class__),
-                                 obj=obj)
-
-    dispatch[types.MethodType] = save_instancemethod
-
-    def save_inst(self, obj):
-        """Inner logic to save instance. Based off pickle.save_inst"""
-        cls = obj.__class__
-
-        # Try the dispatch table (pickle module doesn't do it)
-        f = self.dispatch.get(cls)
-        if f:
-            f(self, obj)  # Call unbound method with explicit self
-            return
-
-        memo = self.memo
-        write = self.write
-        save = self.save
-
-        if hasattr(obj, '__getinitargs__'):
-            args = obj.__getinitargs__()
-            len(args)  # XXX Assert it's a sequence
-            pickle._keep_alive(args, memo)
-        else:
-            args = ()
-
-        write(pickle.MARK)
-
-        if self.bin:
-            save(cls)
-            for arg in args:
-                save(arg)
-            write(pickle.OBJ)
-        else:
-            for arg in args:
-                save(arg)
-            write(pickle.INST + cls.__module__ + '\n' + cls.__name__ + '\n')
-
-        self.memoize(obj)
-
-        try:
-            getstate = obj.__getstate__
-        except AttributeError:
-            stuff = obj.__dict__
-        else:
-            stuff = getstate()
-            pickle._keep_alive(stuff, memo)
-        save(stuff)
-        write(pickle.BUILD)
-
-    if PY2:  # pragma: no branch
-        dispatch[types.InstanceType] = save_inst
-
-    def save_property(self, obj):
-        # properties not correctly saved in python
-        self.save_reduce(property, (obj.fget, obj.fset, obj.fdel, obj.__doc__), obj=obj)
-
-    dispatch[property] = save_property
-
-    def save_classmethod(self, obj):
-        orig_func = obj.__func__
-        self.save_reduce(type(obj), (orig_func,), obj=obj)
-
-    dispatch[classmethod] = save_classmethod
-    dispatch[staticmethod] = save_classmethod
-
-    def save_itemgetter(self, obj):
-        """itemgetter serializer (needed for namedtuple support)"""
-        class Dummy:
-            def __getitem__(self, item):
-                return item
-        items = obj(Dummy())
-        if not isinstance(items, tuple):
-            items = (items,)
-        return self.save_reduce(operator.itemgetter, items)
-
-    if type(operator.itemgetter) is type:
-        dispatch[operator.itemgetter] = save_itemgetter
-
-    def save_attrgetter(self, obj):
-        """attrgetter serializer"""
-        class Dummy(object):
-            def __init__(self, attrs, index=None):
-                self.attrs = attrs
-                self.index = index
-            def __getattribute__(self, item):
-                attrs = object.__getattribute__(self, "attrs")
-                index = object.__getattribute__(self, "index")
-                if index is None:
-                    index = len(attrs)
-                    attrs.append(item)
-                else:
-                    attrs[index] = ".".join([attrs[index], item])
-                return type(self)(attrs, index)
-        attrs = []
-        obj(Dummy(attrs))
-        return self.save_reduce(operator.attrgetter, tuple(attrs))
-
-    if type(operator.attrgetter) is type:
-        dispatch[operator.attrgetter] = save_attrgetter
-
-    def save_file(self, obj):
-        """Save a file"""
-        try:
-            import StringIO as pystringIO  # we can't use cStringIO as it lacks the name attribute
-        except ImportError:
-            import io as pystringIO
-
-        if not hasattr(obj, 'name') or not hasattr(obj, 'mode'):
-            raise pickle.PicklingError("Cannot pickle files that do not map to an actual file")
-        if obj is sys.stdout:
-            return self.save_reduce(getattr, (sys, 'stdout'), obj=obj)
-        if obj is sys.stderr:
-            return self.save_reduce(getattr, (sys, 'stderr'), obj=obj)
-        if obj is sys.stdin:
-            raise pickle.PicklingError("Cannot pickle standard input")
-        if obj.closed:
-            raise pickle.PicklingError("Cannot pickle closed files")
-        if hasattr(obj, 'isatty') and obj.isatty():
-            raise pickle.PicklingError("Cannot pickle files that map to tty objects")
-        if 'r' not in obj.mode and '+' not in obj.mode:
-            raise pickle.PicklingError("Cannot pickle files that are not opened for reading: %s" % obj.mode)
-
-        name = obj.name
-
-        retval = pystringIO.StringIO()
-
-        try:
-            # Read the whole file
-            curloc = obj.tell()
-            obj.seek(0)
-            contents = obj.read()
-            obj.seek(curloc)
-        except IOError:
-            raise pickle.PicklingError("Cannot pickle file %s as it cannot be read" % name)
-        retval.write(contents)
-        retval.seek(curloc)
-
-        retval.name = name
-        self.save(retval)
-        self.memoize(obj)
-
-    def save_ellipsis(self, obj):
-        self.save_reduce(_gen_ellipsis, ())
-
-    def save_not_implemented(self, obj):
-        self.save_reduce(_gen_not_implemented, ())
-
-    try:               # Python 2
-        dispatch[file] = save_file
-    except NameError:  # Python 3  # pragma: no branch
-        dispatch[io.TextIOWrapper] = save_file
-
-    dispatch[type(Ellipsis)] = save_ellipsis
-    dispatch[type(NotImplemented)] = save_not_implemented
-
-    def save_weakset(self, obj):
-        self.save_reduce(weakref.WeakSet, (list(obj),))
-
-    dispatch[weakref.WeakSet] = save_weakset
-
-    def save_logger(self, obj):
-        self.save_reduce(logging.getLogger, (obj.name,), obj=obj)
-
-    dispatch[logging.Logger] = save_logger
-
-    def save_root_logger(self, obj):
-        self.save_reduce(logging.getLogger, (), obj=obj)
-
-    dispatch[logging.RootLogger] = save_root_logger
-
-    if hasattr(types, "MappingProxyType"):  # pragma: no branch
-        def save_mappingproxy(self, obj):
-            self.save_reduce(types.MappingProxyType, (dict(obj),), obj=obj)
-
-        dispatch[types.MappingProxyType] = save_mappingproxy
-
-    """Special functions for Add-on libraries"""
-    def inject_addons(self):
-        """Plug in system. Register additional pickling functions if modules already loaded"""
-        pass
-
-
-# Tornado support
-
-def is_tornado_coroutine(func):
-    """
-    Return whether *func* is a Tornado coroutine function.
-    Running coroutines are not supported.
-    """
-    if 'tornado.gen' not in sys.modules:
-        return False
-    gen = sys.modules['tornado.gen']
-    if not hasattr(gen, "is_coroutine_function"):
-        # Tornado version is too old
-        return False
-    return gen.is_coroutine_function(func)
-
-
-def _rebuild_tornado_coroutine(func):
-    from tornado import gen
-    return gen.coroutine(func)
-
-
-# Shorthands for legacy support
-
-def dump(obj, file, protocol=None):
-    """Serialize obj as bytes streamed into file
-
-    protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
-    pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed
-    between processes running the same Python version.
-
-    Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
-    compatibility with older versions of Python.
-    """
-    CloudPickler(file, protocol=protocol).dump(obj)
-
-
-def dumps(obj, protocol=None):
-    """Serialize obj as a string of bytes allocated in memory
-
-    protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
-    pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed
-    between processes running the same Python version.
-
-    Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
-    compatibility with older versions of Python.
-    """
-    file = StringIO()
-    try:
-        cp = CloudPickler(file, protocol=protocol)
-        cp.dump(obj)
-        return file.getvalue()
-    finally:
-        file.close()
-
-
-# including pickles unloading functions in this namespace
-load = pickle.load
-loads = pickle.loads
-
-
-# hack for __import__ not working as desired
-def subimport(name):
-    __import__(name)
-    return sys.modules[name]
-
-
-def dynamic_subimport(name, vars):
-    mod = types.ModuleType(name)
-    mod.__dict__.update(vars)
-    return mod
-
-
-# restores function attributes
-def _restore_attr(obj, attr):
-    for key, val in attr.items():
-        setattr(obj, key, val)
-    return obj
-
-
-def _gen_ellipsis():
-    return Ellipsis
-
-
-def _gen_not_implemented():
-    return NotImplemented
-
-
-def _get_cell_contents(cell):
-    try:
-        return cell.cell_contents
-    except ValueError:
-        # sentinel used by ``_fill_function`` which will leave the cell empty
-        return _empty_cell_value
-
-
-def instance(cls):
-    """Create a new instance of a class.
-
-    Parameters
-    ----------
-    cls : type
-        The class to create an instance of.
-
-    Returns
-    -------
-    instance : cls
-        A new instance of ``cls``.
-    """
-    return cls()
-
-
-@instance
-class _empty_cell_value(object):
-    """sentinel for empty closures
-    """
-    @classmethod
-    def __reduce__(cls):
-        return cls.__name__
-
-
-def _fill_function(*args):
-    """Fills in the rest of function data into the skeleton function object
-
-    The skeleton itself is create by _make_skel_func().
-    """
-    if len(args) == 2:
-        func = args[0]
-        state = args[1]
-    elif len(args) == 5:
-        # Backwards compat for cloudpickle v0.4.0, after which the `module`
-        # argument was introduced
-        func = args[0]
-        keys = ['globals', 'defaults', 'dict', 'closure_values']
-        state = dict(zip(keys, args[1:]))
-    elif len(args) == 6:
-        # Backwards compat for cloudpickle v0.4.1, after which the function
-        # state was passed as a dict to the _fill_function it-self.
-        func = args[0]
-        keys = ['globals', 'defaults', 'dict', 'module', 'closure_values']
-        state = dict(zip(keys, args[1:]))
-    else:
-        raise ValueError('Unexpected _fill_value arguments: %r' % (args,))
-
-    # - At pickling time, any dynamic global variable used by func is
-    #   serialized by value (in state['globals']).
-    # - At unpickling time, func's __globals__ attribute is initialized by
-    #   first retrieving an empty isolated namespace that will be shared
-    #   with other functions pickled from the same original module
-    #   by the same CloudPickler instance and then updated with the
-    #   content of state['globals'] to populate the shared isolated
-    #   namespace with all the global variables that are specifically
-    #   referenced for this function.
-    func.__globals__.update(state['globals'])
-
-    func.__defaults__ = state['defaults']
-    func.__dict__ = state['dict']
-    if 'annotations' in state:
-        func.__annotations__ = state['annotations']
-    if 'doc' in state:
-        func.__doc__  = state['doc']
-    if 'name' in state:
-        func.__name__ = state['name']
-    if 'module' in state:
-        func.__module__ = state['module']
-    if 'qualname' in state:
-        func.__qualname__ = state['qualname']
-    if 'kwdefaults' in state:
-        func.__kwdefaults__ = state['kwdefaults']
-
-    cells = func.__closure__
-    if cells is not None:
-        for cell, value in zip(cells, state['closure_values']):
-            if value is not _empty_cell_value:
-                cell_set(cell, value)
-
-    return func
-
-
-def _make_empty_cell():
-    if False:
-        # trick the compiler into creating an empty cell in our lambda
-        cell = None
-        raise AssertionError('this route should not be executed')
-
-    return (lambda: cell).__closure__[0]
-
-
-def _make_skel_func(code, cell_count, base_globals=None):
-    """ Creates a skeleton function object that contains just the provided
-        code and the correct number of cells in func_closure.  All other
-        func attributes (e.g. func_globals) are empty.
-    """
-    # This is backward-compatibility code: for cloudpickle versions between
-    # 0.5.4 and 0.7, base_globals could be a string or None. base_globals
-    # should now always be a dictionary.
-    if base_globals is None or isinstance(base_globals, str):
-        base_globals = {}
-
-    base_globals['__builtins__'] = __builtins__
-
-    closure = (
-        tuple(_make_empty_cell() for _ in range(cell_count))
-        if cell_count >= 0 else
-        None
-    )
-    return types.FunctionType(code, base_globals, None, None, closure)
-
-
-def _make_skeleton_class(type_constructor, name, bases, type_kwargs,
-                         class_tracker_id, extra):
-    """Build dynamic class with an empty __dict__ to be filled once memoized
-
-    If class_tracker_id is not None, try to lookup an existing class definition
-    matching that id. If none is found, track a newly reconstructed class
-    definition under that id so that other instances stemming from the same
-    class id will also reuse this class definition.
-
-    The "extra" variable is meant to be a dict (or None) that can be used for
-    forward compatibility shall the need arise.
-    """
-    skeleton_class = type_constructor(name, bases, type_kwargs)
-    return _lookup_class_or_track(class_tracker_id, skeleton_class)
-
-
-def _rehydrate_skeleton_class(skeleton_class, class_dict):
-    """Put attributes from `class_dict` back on `skeleton_class`.
-
-    See CloudPickler.save_dynamic_class for more info.
-    """
-    registry = None
-    for attrname, attr in class_dict.items():
-        if attrname == "_abc_impl":
-            registry = attr
-        else:
-            setattr(skeleton_class, attrname, attr)
-    if registry is not None:
-        for subclass in registry:
-            skeleton_class.register(subclass)
-
-    return skeleton_class
-
-
-def _make_skeleton_enum(bases, name, qualname, members, module,
-                        class_tracker_id, extra):
-    """Build dynamic enum with an empty __dict__ to be filled once memoized
-
-    The creation of the enum class is inspired by the code of
-    EnumMeta._create_.
-
-    If class_tracker_id is not None, try to lookup an existing enum definition
-    matching that id. If none is found, track a newly reconstructed enum
-    definition under that id so that other instances stemming from the same
-    class id will also reuse this enum definition.
-
-    The "extra" variable is meant to be a dict (or None) that can be used for
-    forward compatibility shall the need arise.
-    """
-    # enums always inherit from their base Enum class at the last position in
-    # the list of base classes:
-    enum_base = bases[-1]
-    metacls = enum_base.__class__
-    classdict = metacls.__prepare__(name, bases)
-
-    for member_name, member_value in members.items():
-        classdict[member_name] = member_value
-    enum_class = metacls.__new__(metacls, name, bases, classdict)
-    enum_class.__module__ = module
-
-    # Python 2.7 compat
-    if qualname is not None:
-        enum_class.__qualname__ = qualname
-
-    return _lookup_class_or_track(class_tracker_id, enum_class)
-
-
-def _is_dynamic(module):
-    """
-    Return True if the module is special module that cannot be imported by its
-    name.
-    """
-    # Quick check: module that have __file__ attribute are not dynamic modules.
-    if hasattr(module, '__file__'):
-        return False
-
-    if hasattr(module, '__spec__'):
-        return module.__spec__ is None
-    else:
-        # Backward compat for Python 2
-        import imp
-        try:
-            path = None
-            for part in module.__name__.split('.'):
-                if path is not None:
-                    path = [path]
-                f, path, description = imp.find_module(part, path)
-                if f is not None:
-                    f.close()
-        except ImportError:
-            return True
-        return False
-
-
-""" Use copy_reg to extend global pickle definitions """
-
-if sys.version_info < (3, 4):  # pragma: no branch
-    method_descriptor = type(str.upper)
-
-    def _reduce_method_descriptor(obj):
-        return (getattr, (obj.__objclass__, obj.__name__))
-
-    try:
-        import copy_reg as copyreg
-    except ImportError:
-        import copyreg
-    copyreg.pickle(method_descriptor, _reduce_method_descriptor)
\ No newline at end of file
diff --git a/python/pyspark/cloudpickle/__init__.py b/python/pyspark/cloudpickle/__init__.py
new file mode 100644
index 0000000000000..56506d95fa1be
--- /dev/null
+++ b/python/pyspark/cloudpickle/__init__.py
@@ -0,0 +1,11 @@
+from __future__ import absolute_import
+
+
+from pyspark.cloudpickle.cloudpickle import *  # noqa
+from pyspark.cloudpickle.cloudpickle_fast import CloudPickler, dumps, dump  # noqa
+
+# Conform to the convention used by python serialization libraries, which
+# expose their Pickler subclass at top-level under the  "Pickler" name.
+Pickler = CloudPickler
+
+__version__ = '1.6.0'
diff --git a/python/pyspark/cloudpickle/cloudpickle.py b/python/pyspark/cloudpickle/cloudpickle.py
new file mode 100644
index 0000000000000..05d52afa0da96
--- /dev/null
+++ b/python/pyspark/cloudpickle/cloudpickle.py
@@ -0,0 +1,842 @@
+"""
+This class is defined to override standard pickle functionality
+
+The goals of it follow:
+-Serialize lambdas and nested functions to compiled byte code
+-Deal with main module correctly
+-Deal with other non-serializable objects
+
+It does not include an unpickler, as standard python unpickling suffices.
+
+This module was extracted from the `cloud` package, developed by `PiCloud, Inc.
+<https://web.archive.org/web/20140626004012/http://www.picloud.com/>`_.
+
+Copyright (c) 2012, Regents of the University of California.
+Copyright (c) 2009 `PiCloud, Inc. <https://web.archive.org/web/20140626004012/http://www.picloud.com/>`_.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * Neither the name of the University of California, Berkeley nor the
+      names of its contributors may be used to endorse or promote
+      products derived from this software without specific prior written
+      permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""
+from __future__ import print_function
+
+import builtins
+import dis
+import opcode
+import platform
+import sys
+import types
+import weakref
+import uuid
+import threading
+import typing
+import warnings
+
+from .compat import pickle
+from typing import Generic, Union, Tuple, Callable
+from pickle import _getattribute
+from importlib._bootstrap import _find_spec
+
+try:  # pragma: no branch
+    import typing_extensions as _typing_extensions
+    from typing_extensions import Literal, Final
+except ImportError:
+    _typing_extensions = Literal = Final = None
+
+if sys.version_info >= (3, 5, 3):
+    from typing import ClassVar
+else:  # pragma: no cover
+    ClassVar = None
+
+if sys.version_info >= (3, 8):
+    from types import CellType
+else:
+    def f():
+        a = 1
+
+        def g():
+            return a
+        return g
+    CellType = type(f().__closure__[0])
+
+
+# cloudpickle is meant for inter process communication: we expect all
+# communicating processes to run the same Python version hence we favor
+# communication speed over compatibility:
+DEFAULT_PROTOCOL = pickle.HIGHEST_PROTOCOL
+
+# Track the provenance of reconstructed dynamic classes to make it possible to
+# recontruct instances from the matching singleton class definition when
+# appropriate and preserve the usual "isinstance" semantics of Python objects.
+_DYNAMIC_CLASS_TRACKER_BY_CLASS = weakref.WeakKeyDictionary()
+_DYNAMIC_CLASS_TRACKER_BY_ID = weakref.WeakValueDictionary()
+_DYNAMIC_CLASS_TRACKER_LOCK = threading.Lock()
+
+PYPY = platform.python_implementation() == "PyPy"
+
+builtin_code_type = None
+if PYPY:
+    # builtin-code objects only exist in pypy
+    builtin_code_type = type(float.__new__.__code__)
+
+_extract_code_globals_cache = weakref.WeakKeyDictionary()
+
+
+def _get_or_create_tracker_id(class_def):
+    with _DYNAMIC_CLASS_TRACKER_LOCK:
+        class_tracker_id = _DYNAMIC_CLASS_TRACKER_BY_CLASS.get(class_def)
+        if class_tracker_id is None:
+            class_tracker_id = uuid.uuid4().hex
+            _DYNAMIC_CLASS_TRACKER_BY_CLASS[class_def] = class_tracker_id
+            _DYNAMIC_CLASS_TRACKER_BY_ID[class_tracker_id] = class_def
+    return class_tracker_id
+
+
+def _lookup_class_or_track(class_tracker_id, class_def):
+    if class_tracker_id is not None:
+        with _DYNAMIC_CLASS_TRACKER_LOCK:
+            class_def = _DYNAMIC_CLASS_TRACKER_BY_ID.setdefault(
+                class_tracker_id, class_def)
+            _DYNAMIC_CLASS_TRACKER_BY_CLASS[class_def] = class_tracker_id
+    return class_def
+
+
+def _whichmodule(obj, name):
+    """Find the module an object belongs to.
+
+    This function differs from ``pickle.whichmodule`` in two ways:
+    - it does not mangle the cases where obj's module is __main__ and obj was
+      not found in any module.
+    - Errors arising during module introspection are ignored, as those errors
+      are considered unwanted side effects.
+    """
+    if sys.version_info[:2] < (3, 7) and isinstance(obj, typing.TypeVar):  # pragma: no branch  # noqa
+        # Workaround bug in old Python versions: prior to Python 3.7,
+        # T.__module__ would always be set to "typing" even when the TypeVar T
+        # would be defined in a different module.
+        #
+        # For such older Python versions, we ignore the __module__ attribute of
+        # TypeVar instances and instead exhaustively lookup those instances in
+        # all currently imported modules.
+        module_name = None
+    else:
+        module_name = getattr(obj, '__module__', None)
+
+    if module_name is not None:
+        return module_name
+    # Protect the iteration by using a copy of sys.modules against dynamic
+    # modules that trigger imports of other modules upon calls to getattr or
+    # other threads importing at the same time.
+    for module_name, module in sys.modules.copy().items():
+        # Some modules such as coverage can inject non-module objects inside
+        # sys.modules
+        if (
+                module_name == '__main__' or
+                module is None or
+                not isinstance(module, types.ModuleType)
+        ):
+            continue
+        try:
+            if _getattribute(module, name)[0] is obj:
+                return module_name
+        except Exception:
+            pass
+    return None
+
+
+def _is_importable(obj, name=None):
+    """Dispatcher utility to test the importability of various constructs."""
+    if isinstance(obj, types.FunctionType):
+        return _lookup_module_and_qualname(obj, name=name) is not None
+    elif issubclass(type(obj), type):
+        return _lookup_module_and_qualname(obj, name=name) is not None
+    elif isinstance(obj, types.ModuleType):
+        # We assume that sys.modules is primarily used as a cache mechanism for
+        # the Python import machinery. Checking if a module has been added in
+        # is sys.modules therefore a cheap and simple heuristic to tell us whether
+        # we can assume  that a given module could be imported by name in
+        # another Python process.
+        return obj.__name__ in sys.modules
+    else:
+        raise TypeError(
+            "cannot check importability of {} instances".format(
+                type(obj).__name__)
+        )
+
+
+def _lookup_module_and_qualname(obj, name=None):
+    if name is None:
+        name = getattr(obj, '__qualname__', None)
+    if name is None:  # pragma: no cover
+        # This used to be needed for Python 2.7 support but is probably not
+        # needed anymore. However we keep the __name__ introspection in case
+        # users of cloudpickle rely on this old behavior for unknown reasons.
+        name = getattr(obj, '__name__', None)
+
+    module_name = _whichmodule(obj, name)
+
+    if module_name is None:
+        # In this case, obj.__module__ is None AND obj was not found in any
+        # imported module. obj is thus treated as dynamic.
+        return None
+
+    if module_name == "__main__":
+        return None
+
+    # Note: if module_name is in sys.modules, the corresponding module is
+    # assumed importable at unpickling time. See #357
+    module = sys.modules.get(module_name, None)
+    if module is None:
+        # The main reason why obj's module would not be imported is that this
+        # module has been dynamically created, using for example
+        # types.ModuleType. The other possibility is that module was removed
+        # from sys.modules after obj was created/imported. But this case is not
+        # supported, as the standard pickle does not support it either.
+        return None
+
+    try:
+        obj2, parent = _getattribute(module, name)
+    except AttributeError:
+        # obj was not found inside the module it points to
+        return None
+    if obj2 is not obj:
+        return None
+    return module, name
+
+
+def _extract_code_globals(co):
+    """
+    Find all globals names read or written to by codeblock co
+    """
+    out_names = _extract_code_globals_cache.get(co)
+    if out_names is None:
+        names = co.co_names
+        out_names = {names[oparg] for _, oparg in _walk_global_ops(co)}
+
+        # Declaring a function inside another one using the "def ..."
+        # syntax generates a constant code object corresonding to the one
+        # of the nested function's As the nested function may itself need
+        # global variables, we need to introspect its code, extract its
+        # globals, (look for code object in it's co_consts attribute..) and
+        # add the result to code_globals
+        if co.co_consts:
+            for const in co.co_consts:
+                if isinstance(const, types.CodeType):
+                    out_names |= _extract_code_globals(const)
+
+        _extract_code_globals_cache[co] = out_names
+
+    return out_names
+
+
+def _find_imported_submodules(code, top_level_dependencies):
+    """
+    Find currently imported submodules used by a function.
+
+    Submodules used by a function need to be detected and referenced for the
+    function to work correctly at depickling time. Because submodules can be
+    referenced as attribute of their parent package (``package.submodule``), we
+    need a special introspection technique that does not rely on GLOBAL-related
+    opcodes to find references of them in a code object.
+
+    Example:
+    ```
+    import concurrent.futures
+    import cloudpickle
+    def func():
+        x = concurrent.futures.ThreadPoolExecutor
+    if __name__ == '__main__':
+        cloudpickle.dumps(func)
+    ```
+    The globals extracted by cloudpickle in the function's state include the
+    concurrent package, but not its submodule (here, concurrent.futures), which
+    is the module used by func. Find_imported_submodules will detect the usage
+    of concurrent.futures. Saving this module alongside with func will ensure
+    that calling func once depickled does not fail due to concurrent.futures
+    not being imported
+    """
+
+    subimports = []
+    # check if any known dependency is an imported package
+    for x in top_level_dependencies:
+        if (isinstance(x, types.ModuleType) and
+                hasattr(x, '__package__') and x.__package__):
+            # check if the package has any currently loaded sub-imports
+            prefix = x.__name__ + '.'
+            # A concurrent thread could mutate sys.modules,
+            # make sure we iterate over a copy to avoid exceptions
+            for name in list(sys.modules):
+                # Older versions of pytest will add a "None" module to
+                # sys.modules.
+                if name is not None and name.startswith(prefix):
+                    # check whether the function can address the sub-module
+                    tokens = set(name[len(prefix):].split('.'))
+                    if not tokens - set(code.co_names):
+                        subimports.append(sys.modules[name])
+    return subimports
+
+
+def cell_set(cell, value):
+    """Set the value of a closure cell.
+
+    The point of this function is to set the cell_contents attribute of a cell
+    after its creation. This operation is necessary in case the cell contains a
+    reference to the function the cell belongs to, as when calling the
+    function's constructor
+    ``f = types.FunctionType(code, globals, name, argdefs, closure)``,
+    closure will not be able to contain the yet-to-be-created f.
+
+    In Python3.7, cell_contents is writeable, so setting the contents of a cell
+    can be done simply using
+    >>> cell.cell_contents = value
+
+    In earlier Python3 versions, the cell_contents attribute of a cell is read
+    only, but this limitation can be worked around by leveraging the Python 3
+    ``nonlocal`` keyword.
+
+    In Python2 however, this attribute is read only, and there is no
+    ``nonlocal`` keyword. For this reason, we need to come up with more
+    complicated hacks to set this attribute.
+
+    The chosen approach is to create a function with a STORE_DEREF opcode,
+    which sets the content of a closure variable. Typically:
+
+    >>> def inner(value):
+    ...     lambda: cell  # the lambda makes cell a closure
+    ...     cell = value  # cell is a closure, so this triggers a STORE_DEREF
+
+    (Note that in Python2, A STORE_DEREF can never be triggered from an inner
+    function. The function g for example here
+    >>> def f(var):
+    ...     def g():
+    ...         var += 1
+    ...     return g
+
+    will not modify the closure variable ``var```inplace, but instead try to
+    load a local variable var and increment it. As g does not assign the local
+    variable ``var`` any initial value, calling f(1)() will fail at runtime.)
+
+    Our objective is to set the value of a given cell ``cell``. So we need to
+    somewhat reference our ``cell`` object into the ``inner`` function so that
+    this object (and not the smoke cell of the lambda function) gets affected
+    by the STORE_DEREF operation.
+
+    In inner, ``cell`` is referenced as a cell variable (an enclosing variable
+    that is referenced by the inner function). If we create a new function
+    cell_set with the exact same code as ``inner``, but with ``cell`` marked as
+    a free variable instead, the STORE_DEREF will be applied on its closure -
+    ``cell``, which we can specify explicitly during construction! The new
+    cell_set variable thus actually sets the contents of a specified cell!
+
+    Note: we do not make use of the ``nonlocal`` keyword to set the contents of
+    a cell in early python3 versions to limit possible syntax errors in case
+    test and checker libraries decide to parse the whole file.
+    """
+
+    if sys.version_info[:2] >= (3, 7):  # pragma: no branch
+        cell.cell_contents = value
+    else:
+        _cell_set = types.FunctionType(
+            _cell_set_template_code, {}, '_cell_set', (), (cell,),)
+        _cell_set(value)
+
+
+def _make_cell_set_template_code():
+    def _cell_set_factory(value):
+        lambda: cell
+        cell = value
+
+    co = _cell_set_factory.__code__
+
+    _cell_set_template_code = types.CodeType(
+        co.co_argcount,
+        co.co_kwonlyargcount,   # Python 3 only argument
+        co.co_nlocals,
+        co.co_stacksize,
+        co.co_flags,
+        co.co_code,
+        co.co_consts,
+        co.co_names,
+        co.co_varnames,
+        co.co_filename,
+        co.co_name,
+        co.co_firstlineno,
+        co.co_lnotab,
+        co.co_cellvars,  # co_freevars is initialized with co_cellvars
+        (),  # co_cellvars is made empty
+    )
+    return _cell_set_template_code
+
+
+if sys.version_info[:2] < (3, 7):
+    _cell_set_template_code = _make_cell_set_template_code()
+
+# relevant opcodes
+STORE_GLOBAL = opcode.opmap['STORE_GLOBAL']
+DELETE_GLOBAL = opcode.opmap['DELETE_GLOBAL']
+LOAD_GLOBAL = opcode.opmap['LOAD_GLOBAL']
+GLOBAL_OPS = (STORE_GLOBAL, DELETE_GLOBAL, LOAD_GLOBAL)
+HAVE_ARGUMENT = dis.HAVE_ARGUMENT
+EXTENDED_ARG = dis.EXTENDED_ARG
+
+
+_BUILTIN_TYPE_NAMES = {}
+for k, v in types.__dict__.items():
+    if type(v) is type:
+        _BUILTIN_TYPE_NAMES[v] = k
+
+
+def _builtin_type(name):
+    if name == "ClassType":  # pragma: no cover
+        # Backward compat to load pickle files generated with cloudpickle
+        # < 1.3 even if loading pickle files from older versions is not
+        # officially supported.
+        return type
+    return getattr(types, name)
+
+
+def _walk_global_ops(code):
+    """
+    Yield (opcode, argument number) tuples for all
+    global-referencing instructions in *code*.
+    """
+    for instr in dis.get_instructions(code):
+        op = instr.opcode
+        if op in GLOBAL_OPS:
+            yield op, instr.arg
+
+
+def _extract_class_dict(cls):
+    """Retrieve a copy of the dict of a class without the inherited methods"""
+    clsdict = dict(cls.__dict__)  # copy dict proxy to a dict
+    if len(cls.__bases__) == 1:
+        inherited_dict = cls.__bases__[0].__dict__
+    else:
+        inherited_dict = {}
+        for base in reversed(cls.__bases__):
+            inherited_dict.update(base.__dict__)
+    to_remove = []
+    for name, value in clsdict.items():
+        try:
+            base_value = inherited_dict[name]
+            if value is base_value:
+                to_remove.append(name)
+        except KeyError:
+            pass
+    for name in to_remove:
+        clsdict.pop(name)
+    return clsdict
+
+
+if sys.version_info[:2] < (3, 7):  # pragma: no branch
+    def _is_parametrized_type_hint(obj):
+        # This is very cheap but might generate false positives.
+        # general typing Constructs
+        is_typing = getattr(obj, '__origin__', None) is not None
+
+        # typing_extensions.Literal
+        is_litteral = getattr(obj, '__values__', None) is not None
+
+        # typing_extensions.Final
+        is_final = getattr(obj, '__type__', None) is not None
+
+        # typing.Union/Tuple for old Python 3.5
+        is_union = getattr(obj, '__union_params__', None) is not None
+        is_tuple = getattr(obj, '__tuple_params__', None) is not None
+        is_callable = (
+            getattr(obj, '__result__', None) is not None and
+            getattr(obj, '__args__', None) is not None
+        )
+        return any((is_typing, is_litteral, is_final, is_union, is_tuple,
+                    is_callable))
+
+    def _create_parametrized_type_hint(origin, args):
+        return origin[args]
+else:
+    _is_parametrized_type_hint = None
+    _create_parametrized_type_hint = None
+
+
+def parametrized_type_hint_getinitargs(obj):
+    # The distorted type check sematic for typing construct becomes:
+    # ``type(obj) is type(TypeHint)``, which means "obj is a
+    # parametrized TypeHint"
+    if type(obj) is type(Literal):  # pragma: no branch
+        initargs = (Literal, obj.__values__)
+    elif type(obj) is type(Final):  # pragma: no branch
+        initargs = (Final, obj.__type__)
+    elif type(obj) is type(ClassVar):
+        initargs = (ClassVar, obj.__type__)
+    elif type(obj) is type(Generic):
+        parameters = obj.__parameters__
+        if len(obj.__parameters__) > 0:
+            # in early Python 3.5, __parameters__ was sometimes
+            # preferred to __args__
+            initargs = (obj.__origin__, parameters)
+
+        else:
+            initargs = (obj.__origin__, obj.__args__)
+    elif type(obj) is type(Union):
+        if sys.version_info < (3, 5, 3):  # pragma: no cover
+            initargs = (Union, obj.__union_params__)
+        else:
+            initargs = (Union, obj.__args__)
+    elif type(obj) is type(Tuple):
+        if sys.version_info < (3, 5, 3):  # pragma: no cover
+            initargs = (Tuple, obj.__tuple_params__)
+        else:
+            initargs = (Tuple, obj.__args__)
+    elif type(obj) is type(Callable):
+        if sys.version_info < (3, 5, 3):  # pragma: no cover
+            args = obj.__args__
+            result = obj.__result__
+            if args != Ellipsis:
+                if isinstance(args, tuple):
+                    args = list(args)
+                else:
+                    args = [args]
+        else:
+            (*args, result) = obj.__args__
+            if len(args) == 1 and args[0] is Ellipsis:
+                args = Ellipsis
+            else:
+                args = list(args)
+        initargs = (Callable, (args, result))
+    else:  # pragma: no cover
+        raise pickle.PicklingError(
+            "Cloudpickle Error: Unknown type {}".format(type(obj))
+        )
+    return initargs
+
+
+# Tornado support
+
+def is_tornado_coroutine(func):
+    """
+    Return whether *func* is a Tornado coroutine function.
+    Running coroutines are not supported.
+    """
+    if 'tornado.gen' not in sys.modules:
+        return False
+    gen = sys.modules['tornado.gen']
+    if not hasattr(gen, "is_coroutine_function"):
+        # Tornado version is too old
+        return False
+    return gen.is_coroutine_function(func)
+
+
+def _rebuild_tornado_coroutine(func):
+    from tornado import gen
+    return gen.coroutine(func)
+
+
+# including pickles unloading functions in this namespace
+load = pickle.load
+loads = pickle.loads
+
+
+# hack for __import__ not working as desired
+def subimport(name):
+    __import__(name)
+    return sys.modules[name]
+
+
+def dynamic_subimport(name, vars):
+    mod = types.ModuleType(name)
+    mod.__dict__.update(vars)
+    mod.__dict__['__builtins__'] = builtins.__dict__
+    return mod
+
+
+def _gen_ellipsis():
+    return Ellipsis
+
+
+def _gen_not_implemented():
+    return NotImplemented
+
+
+def _get_cell_contents(cell):
+    try:
+        return cell.cell_contents
+    except ValueError:
+        # sentinel used by ``_fill_function`` which will leave the cell empty
+        return _empty_cell_value
+
+
+def instance(cls):
+    """Create a new instance of a class.
+
+    Parameters
+    ----------
+    cls : type
+        The class to create an instance of.
+
+    Returns
+    -------
+    instance : cls
+        A new instance of ``cls``.
+    """
+    return cls()
+
+
+@instance
+class _empty_cell_value(object):
+    """sentinel for empty closures
+    """
+    @classmethod
+    def __reduce__(cls):
+        return cls.__name__
+
+
+def _fill_function(*args):
+    """Fills in the rest of function data into the skeleton function object
+
+    The skeleton itself is create by _make_skel_func().
+    """
+    if len(args) == 2:
+        func = args[0]
+        state = args[1]
+    elif len(args) == 5:
+        # Backwards compat for cloudpickle v0.4.0, after which the `module`
+        # argument was introduced
+        func = args[0]
+        keys = ['globals', 'defaults', 'dict', 'closure_values']
+        state = dict(zip(keys, args[1:]))
+    elif len(args) == 6:
+        # Backwards compat for cloudpickle v0.4.1, after which the function
+        # state was passed as a dict to the _fill_function it-self.
+        func = args[0]
+        keys = ['globals', 'defaults', 'dict', 'module', 'closure_values']
+        state = dict(zip(keys, args[1:]))
+    else:
+        raise ValueError('Unexpected _fill_value arguments: %r' % (args,))
+
+    # - At pickling time, any dynamic global variable used by func is
+    #   serialized by value (in state['globals']).
+    # - At unpickling time, func's __globals__ attribute is initialized by
+    #   first retrieving an empty isolated namespace that will be shared
+    #   with other functions pickled from the same original module
+    #   by the same CloudPickler instance and then updated with the
+    #   content of state['globals'] to populate the shared isolated
+    #   namespace with all the global variables that are specifically
+    #   referenced for this function.
+    func.__globals__.update(state['globals'])
+
+    func.__defaults__ = state['defaults']
+    func.__dict__ = state['dict']
+    if 'annotations' in state:
+        func.__annotations__ = state['annotations']
+    if 'doc' in state:
+        func.__doc__ = state['doc']
+    if 'name' in state:
+        func.__name__ = state['name']
+    if 'module' in state:
+        func.__module__ = state['module']
+    if 'qualname' in state:
+        func.__qualname__ = state['qualname']
+    if 'kwdefaults' in state:
+        func.__kwdefaults__ = state['kwdefaults']
+    # _cloudpickle_subimports is a set of submodules that must be loaded for
+    # the pickled function to work correctly at unpickling time. Now that these
+    # submodules are depickled (hence imported), they can be removed from the
+    # object's state (the object state only served as a reference holder to
+    # these submodules)
+    if '_cloudpickle_submodules' in state:
+        state.pop('_cloudpickle_submodules')
+
+    cells = func.__closure__
+    if cells is not None:
+        for cell, value in zip(cells, state['closure_values']):
+            if value is not _empty_cell_value:
+                cell_set(cell, value)
+
+    return func
+
+
+def _make_empty_cell():
+    if False:
+        # trick the compiler into creating an empty cell in our lambda
+        cell = None
+        raise AssertionError('this route should not be executed')
+
+    return (lambda: cell).__closure__[0]
+
+
+def _make_cell(value=_empty_cell_value):
+    cell = _make_empty_cell()
+    if value is not _empty_cell_value:
+        cell_set(cell, value)
+    return cell
+
+
+def _make_skel_func(code, cell_count, base_globals=None):
+    """ Creates a skeleton function object that contains just the provided
+        code and the correct number of cells in func_closure.  All other
+        func attributes (e.g. func_globals) are empty.
+    """
+    # This function is deprecated and should be removed in cloudpickle 1.7
+    warnings.warn(
+        "A pickle file created using an old (<=1.4.1) version of cloudpicke "
+        "is currently being loaded. This is not supported by cloudpickle and "
+        "will break in cloudpickle 1.7", category=UserWarning
+    )
+    # This is backward-compatibility code: for cloudpickle versions between
+    # 0.5.4 and 0.7, base_globals could be a string or None. base_globals
+    # should now always be a dictionary.
+    if base_globals is None or isinstance(base_globals, str):
+        base_globals = {}
+
+    base_globals['__builtins__'] = __builtins__
+
+    closure = (
+        tuple(_make_empty_cell() for _ in range(cell_count))
+        if cell_count >= 0 else
+        None
+    )
+    return types.FunctionType(code, base_globals, None, None, closure)
+
+
+def _make_skeleton_class(type_constructor, name, bases, type_kwargs,
+                         class_tracker_id, extra):
+    """Build dynamic class with an empty __dict__ to be filled once memoized
+
+    If class_tracker_id is not None, try to lookup an existing class definition
+    matching that id. If none is found, track a newly reconstructed class
+    definition under that id so that other instances stemming from the same
+    class id will also reuse this class definition.
+
+    The "extra" variable is meant to be a dict (or None) that can be used for
+    forward compatibility shall the need arise.
+    """
+    skeleton_class = types.new_class(
+        name, bases, {'metaclass': type_constructor},
+        lambda ns: ns.update(type_kwargs)
+    )
+    return _lookup_class_or_track(class_tracker_id, skeleton_class)
+
+
+def _rehydrate_skeleton_class(skeleton_class, class_dict):
+    """Put attributes from `class_dict` back on `skeleton_class`.
+
+    See CloudPickler.save_dynamic_class for more info.
+    """
+    registry = None
+    for attrname, attr in class_dict.items():
+        if attrname == "_abc_impl":
+            registry = attr
+        else:
+            setattr(skeleton_class, attrname, attr)
+    if registry is not None:
+        for subclass in registry:
+            skeleton_class.register(subclass)
+
+    return skeleton_class
+
+
+def _make_skeleton_enum(bases, name, qualname, members, module,
+                        class_tracker_id, extra):
+    """Build dynamic enum with an empty __dict__ to be filled once memoized
+
+    The creation of the enum class is inspired by the code of
+    EnumMeta._create_.
+
+    If class_tracker_id is not None, try to lookup an existing enum definition
+    matching that id. If none is found, track a newly reconstructed enum
+    definition under that id so that other instances stemming from the same
+    class id will also reuse this enum definition.
+
+    The "extra" variable is meant to be a dict (or None) that can be used for
+    forward compatibility shall the need arise.
+    """
+    # enums always inherit from their base Enum class at the last position in
+    # the list of base classes:
+    enum_base = bases[-1]
+    metacls = enum_base.__class__
+    classdict = metacls.__prepare__(name, bases)
+
+    for member_name, member_value in members.items():
+        classdict[member_name] = member_value
+    enum_class = metacls.__new__(metacls, name, bases, classdict)
+    enum_class.__module__ = module
+    enum_class.__qualname__ = qualname
+
+    return _lookup_class_or_track(class_tracker_id, enum_class)
+
+
+def _make_typevar(name, bound, constraints, covariant, contravariant,
+                  class_tracker_id):
+    tv = typing.TypeVar(
+        name, *constraints, bound=bound,
+        covariant=covariant, contravariant=contravariant
+    )
+    if class_tracker_id is not None:
+        return _lookup_class_or_track(class_tracker_id, tv)
+    else:  # pragma: nocover
+        # Only for Python 3.5.3 compat.
+        return tv
+
+
+def _decompose_typevar(obj):
+    try:
+        class_tracker_id = _get_or_create_tracker_id(obj)
+    except TypeError:  # pragma: nocover
+        # TypeVar instances are not weakref-able in Python 3.5.3
+        class_tracker_id = None
+    return (
+        obj.__name__, obj.__bound__, obj.__constraints__,
+        obj.__covariant__, obj.__contravariant__,
+        class_tracker_id,
+    )
+
+
+def _typevar_reduce(obj):
+    # TypeVar instances have no __qualname__ hence we pass the name explicitly.
+    module_and_name = _lookup_module_and_qualname(obj, name=obj.__name__)
+    if module_and_name is None:
+        return (_make_typevar, _decompose_typevar(obj))
+    return (getattr, module_and_name)
+
+
+def _get_bases(typ):
+    if hasattr(typ, '__orig_bases__'):
+        # For generic types (see PEP 560)
+        bases_attr = '__orig_bases__'
+    else:
+        # For regular class objects
+        bases_attr = '__bases__'
+    return getattr(typ, bases_attr)
+
+
+def _make_dict_keys(obj):
+    return dict.fromkeys(obj).keys()
+
+
+def _make_dict_values(obj):
+    return {i: _ for i, _ in enumerate(obj)}.values()
+
+
+def _make_dict_items(obj):
+    return obj.items()
diff --git a/python/pyspark/cloudpickle/cloudpickle_fast.py b/python/pyspark/cloudpickle/cloudpickle_fast.py
new file mode 100644
index 0000000000000..fa8da0f635c49
--- /dev/null
+++ b/python/pyspark/cloudpickle/cloudpickle_fast.py
@@ -0,0 +1,770 @@
+"""
+New, fast version of the CloudPickler.
+
+This new CloudPickler class can now extend the fast C Pickler instead of the
+previous Python implementation of the Pickler class. Because this functionality
+is only available for Python versions 3.8+, a lot of backward-compatibility
+code is also removed.
+
+Note that the C Pickler sublassing API is CPython-specific. Therefore, some
+guards present in cloudpickle.py that were written to handle PyPy specificities
+are not present in cloudpickle_fast.py
+"""
+import _collections_abc
+import abc
+import copyreg
+import io
+import itertools
+import logging
+import sys
+import struct
+import types
+import weakref
+import typing
+
+from enum import Enum
+from collections import ChainMap
+
+from .compat import pickle, Pickler
+from .cloudpickle import (
+    _extract_code_globals, _BUILTIN_TYPE_NAMES, DEFAULT_PROTOCOL,
+    _find_imported_submodules, _get_cell_contents, _is_importable,
+    _builtin_type, _get_or_create_tracker_id,  _make_skeleton_class,
+    _make_skeleton_enum, _extract_class_dict, dynamic_subimport, subimport,
+    _typevar_reduce, _get_bases, _make_cell, _make_empty_cell, CellType,
+    _is_parametrized_type_hint, PYPY, cell_set,
+    parametrized_type_hint_getinitargs, _create_parametrized_type_hint,
+    builtin_code_type,
+    _make_dict_keys, _make_dict_values, _make_dict_items,
+)
+
+
+if pickle.HIGHEST_PROTOCOL >= 5 and not PYPY:
+    # Shorthands similar to pickle.dump/pickle.dumps
+
+    def dump(obj, file, protocol=None, buffer_callback=None):
+        """Serialize obj as bytes streamed into file
+
+        protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
+        pickle.HIGHEST_PROTOCOL. This setting favors maximum communication
+        speed between processes running the same Python version.
+
+        Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
+        compatibility with older versions of Python.
+        """
+        CloudPickler(
+            file, protocol=protocol, buffer_callback=buffer_callback
+        ).dump(obj)
+
+    def dumps(obj, protocol=None, buffer_callback=None):
+        """Serialize obj as a string of bytes allocated in memory
+
+        protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
+        pickle.HIGHEST_PROTOCOL. This setting favors maximum communication
+        speed between processes running the same Python version.
+
+        Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
+        compatibility with older versions of Python.
+        """
+        with io.BytesIO() as file:
+            cp = CloudPickler(
+                file, protocol=protocol, buffer_callback=buffer_callback
+            )
+            cp.dump(obj)
+            return file.getvalue()
+
+else:
+    # Shorthands similar to pickle.dump/pickle.dumps
+    def dump(obj, file, protocol=None):
+        """Serialize obj as bytes streamed into file
+
+        protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
+        pickle.HIGHEST_PROTOCOL. This setting favors maximum communication
+        speed between processes running the same Python version.
+
+        Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
+        compatibility with older versions of Python.
+        """
+        CloudPickler(file, protocol=protocol).dump(obj)
+
+    def dumps(obj, protocol=None):
+        """Serialize obj as a string of bytes allocated in memory
+
+        protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
+        pickle.HIGHEST_PROTOCOL. This setting favors maximum communication
+        speed between processes running the same Python version.
+
+        Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
+        compatibility with older versions of Python.
+        """
+        with io.BytesIO() as file:
+            cp = CloudPickler(file, protocol=protocol)
+            cp.dump(obj)
+            return file.getvalue()
+
+
+load, loads = pickle.load, pickle.loads
+
+
+# COLLECTION OF OBJECTS __getnewargs__-LIKE METHODS
+# -------------------------------------------------
+
+def _class_getnewargs(obj):
+    type_kwargs = {}
+    if "__slots__" in obj.__dict__:
+        type_kwargs["__slots__"] = obj.__slots__
+
+    __dict__ = obj.__dict__.get('__dict__', None)
+    if isinstance(__dict__, property):
+        type_kwargs['__dict__'] = __dict__
+
+    return (type(obj), obj.__name__, _get_bases(obj), type_kwargs,
+            _get_or_create_tracker_id(obj), None)
+
+
+def _enum_getnewargs(obj):
+    members = dict((e.name, e.value) for e in obj)
+    return (obj.__bases__, obj.__name__, obj.__qualname__, members,
+            obj.__module__, _get_or_create_tracker_id(obj), None)
+
+
+# COLLECTION OF OBJECTS RECONSTRUCTORS
+# ------------------------------------
+def _file_reconstructor(retval):
+    return retval
+
+
+# COLLECTION OF OBJECTS STATE GETTERS
+# -----------------------------------
+def _function_getstate(func):
+    # - Put func's dynamic attributes (stored in func.__dict__) in state. These
+    #   attributes will be restored at unpickling time using
+    #   f.__dict__.update(state)
+    # - Put func's members into slotstate. Such attributes will be restored at
+    #   unpickling time by iterating over slotstate and calling setattr(func,
+    #   slotname, slotvalue)
+    slotstate = {
+        "__name__": func.__name__,
+        "__qualname__": func.__qualname__,
+        "__annotations__": func.__annotations__,
+        "__kwdefaults__": func.__kwdefaults__,
+        "__defaults__": func.__defaults__,
+        "__module__": func.__module__,
+        "__doc__": func.__doc__,
+        "__closure__": func.__closure__,
+    }
+
+    f_globals_ref = _extract_code_globals(func.__code__)
+    f_globals = {k: func.__globals__[k] for k in f_globals_ref if k in
+                 func.__globals__}
+
+    closure_values = (
+        list(map(_get_cell_contents, func.__closure__))
+        if func.__closure__ is not None else ()
+    )
+
+    # Extract currently-imported submodules used by func. Storing these modules
+    # in a smoke _cloudpickle_subimports attribute of the object's state will
+    # trigger the side effect of importing these modules at unpickling time
+    # (which is necessary for func to work correctly once depickled)
+    slotstate["_cloudpickle_submodules"] = _find_imported_submodules(
+        func.__code__, itertools.chain(f_globals.values(), closure_values))
+    slotstate["__globals__"] = f_globals
+
+    state = func.__dict__
+    return state, slotstate
+
+
+def _class_getstate(obj):
+    clsdict = _extract_class_dict(obj)
+    clsdict.pop('__weakref__', None)
+
+    if issubclass(type(obj), abc.ABCMeta):
+        # If obj is an instance of an ABCMeta subclass, dont pickle the
+        # cache/negative caches populated during isinstance/issubclass
+        # checks, but pickle the list of registered subclasses of obj.
+        clsdict.pop('_abc_cache', None)
+        clsdict.pop('_abc_negative_cache', None)
+        clsdict.pop('_abc_negative_cache_version', None)
+        registry = clsdict.pop('_abc_registry', None)
+        if registry is None:
+            # in Python3.7+, the abc caches and registered subclasses of a
+            # class are bundled into the single _abc_impl attribute
+            clsdict.pop('_abc_impl', None)
+            (registry, _, _, _) = abc._get_dump(obj)
+
+            clsdict["_abc_impl"] = [subclass_weakref()
+                                    for subclass_weakref in registry]
+        else:
+            # In the above if clause, registry is a set of weakrefs -- in
+            # this case, registry is a WeakSet
+            clsdict["_abc_impl"] = [type_ for type_ in registry]
+
+    if "__slots__" in clsdict:
+        # pickle string length optimization: member descriptors of obj are
+        # created automatically from obj's __slots__ attribute, no need to
+        # save them in obj's state
+        if isinstance(obj.__slots__, str):
+            clsdict.pop(obj.__slots__)
+        else:
+            for k in obj.__slots__:
+                clsdict.pop(k, None)
+
+    clsdict.pop('__dict__', None)  # unpicklable property object
+
+    return (clsdict, {})
+
+
+def _enum_getstate(obj):
+    clsdict, slotstate = _class_getstate(obj)
+
+    members = dict((e.name, e.value) for e in obj)
+    # Cleanup the clsdict that will be passed to _rehydrate_skeleton_class:
+    # Those attributes are already handled by the metaclass.
+    for attrname in ["_generate_next_value_", "_member_names_",
+                     "_member_map_", "_member_type_",
+                     "_value2member_map_"]:
+        clsdict.pop(attrname, None)
+    for member in members:
+        clsdict.pop(member)
+        # Special handling of Enum subclasses
+    return clsdict, slotstate
+
+
+# COLLECTIONS OF OBJECTS REDUCERS
+# -------------------------------
+# A reducer is a function taking a single argument (obj), and that returns a
+# tuple with all the necessary data to re-construct obj. Apart from a few
+# exceptions (list, dict, bytes, int, etc.), a reducer is necessary to
+# correctly pickle an object.
+# While many built-in objects (Exceptions objects, instances of the "object"
+# class, etc), are shipped with their own built-in reducer (invoked using
+# obj.__reduce__), some do not. The following methods were created to "fill
+# these holes".
+
+def _code_reduce(obj):
+    """codeobject reducer"""
+    if hasattr(obj, "co_posonlyargcount"):  # pragma: no branch
+        args = (
+            obj.co_argcount, obj.co_posonlyargcount,
+            obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
+            obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
+            obj.co_varnames, obj.co_filename, obj.co_name,
+            obj.co_firstlineno, obj.co_lnotab, obj.co_freevars,
+            obj.co_cellvars
+        )
+    else:
+        args = (
+            obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals,
+            obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts,
+            obj.co_names, obj.co_varnames, obj.co_filename,
+            obj.co_name, obj.co_firstlineno, obj.co_lnotab,
+            obj.co_freevars, obj.co_cellvars
+        )
+    return types.CodeType, args
+
+
+def _cell_reduce(obj):
+    """Cell (containing values of a function's free variables) reducer"""
+    try:
+        obj.cell_contents
+    except ValueError:  # cell is empty
+        return _make_empty_cell, ()
+    else:
+        return _make_cell, (obj.cell_contents, )
+
+
+def _classmethod_reduce(obj):
+    orig_func = obj.__func__
+    return type(obj), (orig_func,)
+
+
+def _file_reduce(obj):
+    """Save a file"""
+    import io
+
+    if not hasattr(obj, "name") or not hasattr(obj, "mode"):
+        raise pickle.PicklingError(
+            "Cannot pickle files that do not map to an actual file"
+        )
+    if obj is sys.stdout:
+        return getattr, (sys, "stdout")
+    if obj is sys.stderr:
+        return getattr, (sys, "stderr")
+    if obj is sys.stdin:
+        raise pickle.PicklingError("Cannot pickle standard input")
+    if obj.closed:
+        raise pickle.PicklingError("Cannot pickle closed files")
+    if hasattr(obj, "isatty") and obj.isatty():
+        raise pickle.PicklingError(
+            "Cannot pickle files that map to tty objects"
+        )
+    if "r" not in obj.mode and "+" not in obj.mode:
+        raise pickle.PicklingError(
+            "Cannot pickle files that are not opened for reading: %s"
+            % obj.mode
+        )
+
+    name = obj.name
+
+    retval = io.StringIO()
+
+    try:
+        # Read the whole file
+        curloc = obj.tell()
+        obj.seek(0)
+        contents = obj.read()
+        obj.seek(curloc)
+    except IOError as e:
+        raise pickle.PicklingError(
+            "Cannot pickle file %s as it cannot be read" % name
+        ) from e
+    retval.write(contents)
+    retval.seek(curloc)
+
+    retval.name = name
+    return _file_reconstructor, (retval,)
+
+
+def _getset_descriptor_reduce(obj):
+    return getattr, (obj.__objclass__, obj.__name__)
+
+
+def _mappingproxy_reduce(obj):
+    return types.MappingProxyType, (dict(obj),)
+
+
+def _memoryview_reduce(obj):
+    return bytes, (obj.tobytes(),)
+
+
+def _module_reduce(obj):
+    if _is_importable(obj):
+        return subimport, (obj.__name__,)
+    else:
+        obj.__dict__.pop('__builtins__', None)
+        return dynamic_subimport, (obj.__name__, vars(obj))
+
+
+def _method_reduce(obj):
+    return (types.MethodType, (obj.__func__, obj.__self__))
+
+
+def _logger_reduce(obj):
+    return logging.getLogger, (obj.name,)
+
+
+def _root_logger_reduce(obj):
+    return logging.getLogger, ()
+
+
+def _property_reduce(obj):
+    return property, (obj.fget, obj.fset, obj.fdel, obj.__doc__)
+
+
+def _weakset_reduce(obj):
+    return weakref.WeakSet, (list(obj),)
+
+
+def _dynamic_class_reduce(obj):
+    """
+    Save a class that can't be stored as module global.
+
+    This method is used to serialize classes that are defined inside
+    functions, or that otherwise can't be serialized as attribute lookups
+    from global modules.
+    """
+    if Enum is not None and issubclass(obj, Enum):
+        return (
+            _make_skeleton_enum, _enum_getnewargs(obj), _enum_getstate(obj),
+            None, None, _class_setstate
+        )
+    else:
+        return (
+            _make_skeleton_class, _class_getnewargs(obj), _class_getstate(obj),
+            None, None, _class_setstate
+        )
+
+
+def _class_reduce(obj):
+    """Select the reducer depending on the dynamic nature of the class obj"""
+    if obj is type(None):  # noqa
+        return type, (None,)
+    elif obj is type(Ellipsis):
+        return type, (Ellipsis,)
+    elif obj is type(NotImplemented):
+        return type, (NotImplemented,)
+    elif obj in _BUILTIN_TYPE_NAMES:
+        return _builtin_type, (_BUILTIN_TYPE_NAMES[obj],)
+    elif not _is_importable(obj):
+        return _dynamic_class_reduce(obj)
+    return NotImplemented
+
+
+def _dict_keys_reduce(obj):
+    # Safer not to ship the full dict as sending the rest might
+    # be unintended and could potentially cause leaking of
+    # sensitive information
+    return _make_dict_keys, (list(obj), )
+
+
+def _dict_values_reduce(obj):
+    # Safer not to ship the full dict as sending the rest might
+    # be unintended and could potentially cause leaking of
+    # sensitive information
+    return _make_dict_values, (list(obj), )
+
+
+def _dict_items_reduce(obj):
+    return _make_dict_items, (dict(obj), )
+
+
+# COLLECTIONS OF OBJECTS STATE SETTERS
+# ------------------------------------
+# state setters are called at unpickling time, once the object is created and
+# it has to be updated to how it was at unpickling time.
+
+
+def _function_setstate(obj, state):
+    """Update the state of a dynaamic function.
+
+    As __closure__ and __globals__ are readonly attributes of a function, we
+    cannot rely on the native setstate routine of pickle.load_build, that calls
+    setattr on items of the slotstate. Instead, we have to modify them inplace.
+    """
+    state, slotstate = state
+    obj.__dict__.update(state)
+
+    obj_globals = slotstate.pop("__globals__")
+    obj_closure = slotstate.pop("__closure__")
+    # _cloudpickle_subimports is a set of submodules that must be loaded for
+    # the pickled function to work correctly at unpickling time. Now that these
+    # submodules are depickled (hence imported), they can be removed from the
+    # object's state (the object state only served as a reference holder to
+    # these submodules)
+    slotstate.pop("_cloudpickle_submodules")
+
+    obj.__globals__.update(obj_globals)
+    obj.__globals__["__builtins__"] = __builtins__
+
+    if obj_closure is not None:
+        for i, cell in enumerate(obj_closure):
+            try:
+                value = cell.cell_contents
+            except ValueError:  # cell is empty
+                continue
+            cell_set(obj.__closure__[i], value)
+
+    for k, v in slotstate.items():
+        setattr(obj, k, v)
+
+
+def _class_setstate(obj, state):
+    state, slotstate = state
+    registry = None
+    for attrname, attr in state.items():
+        if attrname == "_abc_impl":
+            registry = attr
+        else:
+            setattr(obj, attrname, attr)
+    if registry is not None:
+        for subclass in registry:
+            obj.register(subclass)
+
+    return obj
+
+
+class CloudPickler(Pickler):
+    # set of reducers defined and used by cloudpickle (private)
+    _dispatch_table = {}
+    _dispatch_table[classmethod] = _classmethod_reduce
+    _dispatch_table[io.TextIOWrapper] = _file_reduce
+    _dispatch_table[logging.Logger] = _logger_reduce
+    _dispatch_table[logging.RootLogger] = _root_logger_reduce
+    _dispatch_table[memoryview] = _memoryview_reduce
+    _dispatch_table[property] = _property_reduce
+    _dispatch_table[staticmethod] = _classmethod_reduce
+    _dispatch_table[CellType] = _cell_reduce
+    _dispatch_table[types.CodeType] = _code_reduce
+    _dispatch_table[types.GetSetDescriptorType] = _getset_descriptor_reduce
+    _dispatch_table[types.ModuleType] = _module_reduce
+    _dispatch_table[types.MethodType] = _method_reduce
+    _dispatch_table[types.MappingProxyType] = _mappingproxy_reduce
+    _dispatch_table[weakref.WeakSet] = _weakset_reduce
+    _dispatch_table[typing.TypeVar] = _typevar_reduce
+    _dispatch_table[_collections_abc.dict_keys] = _dict_keys_reduce
+    _dispatch_table[_collections_abc.dict_values] = _dict_values_reduce
+    _dispatch_table[_collections_abc.dict_items] = _dict_items_reduce
+
+
+    dispatch_table = ChainMap(_dispatch_table, copyreg.dispatch_table)
+
+    # function reducers are defined as instance methods of CloudPickler
+    # objects, as they rely on a CloudPickler attribute (globals_ref)
+    def _dynamic_function_reduce(self, func):
+        """Reduce a function that is not pickleable via attribute lookup."""
+        newargs = self._function_getnewargs(func)
+        state = _function_getstate(func)
+        return (types.FunctionType, newargs, state, None, None,
+                _function_setstate)
+
+    def _function_reduce(self, obj):
+        """Reducer for function objects.
+
+        If obj is a top-level attribute of a file-backed module, this
+        reducer returns NotImplemented, making the CloudPickler fallback to
+        traditional _pickle.Pickler routines to save obj. Otherwise, it reduces
+        obj using a custom cloudpickle reducer designed specifically to handle
+        dynamic functions.
+
+        As opposed to cloudpickle.py, There no special handling for builtin
+        pypy functions because cloudpickle_fast is CPython-specific.
+        """
+        if _is_importable(obj):
+            return NotImplemented
+        else:
+            return self._dynamic_function_reduce(obj)
+
+    def _function_getnewargs(self, func):
+        code = func.__code__
+
+        # base_globals represents the future global namespace of func at
+        # unpickling time. Looking it up and storing it in
+        # CloudpiPickler.globals_ref allow functions sharing the same globals
+        # at pickling time to also share them once unpickled, at one condition:
+        # since globals_ref is an attribute of a CloudPickler instance, and
+        # that a new CloudPickler is created each time pickle.dump or
+        # pickle.dumps is called, functions also need to be saved within the
+        # same invocation of cloudpickle.dump/cloudpickle.dumps (for example:
+        # cloudpickle.dumps([f1, f2])). There is no such limitation when using
+        # CloudPickler.dump, as long as the multiple invocations are bound to
+        # the same CloudPickler.
+        base_globals = self.globals_ref.setdefault(id(func.__globals__), {})
+
+        if base_globals == {}:
+            # Add module attributes used to resolve relative imports
+            # instructions inside func.
+            for k in ["__package__", "__name__", "__path__", "__file__"]:
+                if k in func.__globals__:
+                    base_globals[k] = func.__globals__[k]
+
+        # Do not bind the free variables before the function is created to
+        # avoid infinite recursion.
+        if func.__closure__ is None:
+            closure = None
+        else:
+            closure = tuple(
+                _make_empty_cell() for _ in range(len(code.co_freevars)))
+
+        return code, base_globals, None, None, closure
+
+    def dump(self, obj):
+        try:
+            return Pickler.dump(self, obj)
+        except RuntimeError as e:
+            if "recursion" in e.args[0]:
+                msg = (
+                    "Could not pickle object as excessively deep recursion "
+                    "required."
+                )
+                raise pickle.PicklingError(msg) from e
+            else:
+                raise
+
+    if pickle.HIGHEST_PROTOCOL >= 5:
+        # `CloudPickler.dispatch` is only left for backward compatibility - note
+        # that when using protocol 5, `CloudPickler.dispatch` is not an
+        # extension of `Pickler.dispatch` dictionary, because CloudPickler
+        # subclasses the C-implemented Pickler, which does not expose a
+        # `dispatch` attribute.  Earlier versions of the protocol 5 CloudPickler
+        # used `CloudPickler.dispatch` as a class-level attribute storing all
+        # reducers implemented by cloudpickle, but the attribute name was not a
+        # great choice given the meaning of `Cloudpickler.dispatch` when
+        # `CloudPickler` extends the pure-python pickler.
+        dispatch = dispatch_table
+
+        # Implementation of the reducer_override callback, in order to
+        # efficiently serialize dynamic functions and classes by subclassing
+        # the C-implemented Pickler.
+        # TODO: decorrelate reducer_override (which is tied to CPython's
+        # implementation - would it make sense to backport it to pypy? - and
+        # pickle's protocol 5 which is implementation agnostic. Currently, the
+        # availability of both notions coincide on CPython's pickle and the
+        # pickle5 backport, but it may not be the case anymore when pypy
+        # implements protocol 5
+        def __init__(self, file, protocol=None, buffer_callback=None):
+            if protocol is None:
+                protocol = DEFAULT_PROTOCOL
+            Pickler.__init__(
+                self, file, protocol=protocol, buffer_callback=buffer_callback
+            )
+            # map functions __globals__ attribute ids, to ensure that functions
+            # sharing the same global namespace at pickling time also share
+            # their global namespace at unpickling time.
+            self.globals_ref = {}
+            self.proto = int(protocol)
+
+        def reducer_override(self, obj):
+            """Type-agnostic reducing callback for function and classes.
+
+            For performance reasons, subclasses of the C _pickle.Pickler class
+            cannot register custom reducers for functions and classes in the
+            dispatch_table. Reducer for such types must instead implemented in
+            the special reducer_override method.
+
+            Note that method will be called for any object except a few
+            builtin-types (int, lists, dicts etc.), which differs from reducers
+            in the Pickler's dispatch_table, each of them being invoked for
+            objects of a specific type only.
+
+            This property comes in handy for classes: although most classes are
+            instances of the ``type`` metaclass, some of them can be instances
+            of other custom metaclasses (such as enum.EnumMeta for example). In
+            particular, the metaclass will likely not be known in advance, and
+            thus cannot be special-cased using an entry in the dispatch_table.
+            reducer_override, among other things, allows us to register a
+            reducer that will be called for any class, independently of its
+            type.
+
+
+            Notes:
+
+            * reducer_override has the priority over dispatch_table-registered
+            reducers.
+            * reducer_override can be used to fix other limitations of
+              cloudpickle for other types that suffered from type-specific
+              reducers, such as Exceptions. See
+              https://github.com/cloudpipe/cloudpickle/issues/248
+            """
+            if sys.version_info[:2] < (3, 7) and _is_parametrized_type_hint(obj):  # noqa  # pragma: no branch
+                return (
+                    _create_parametrized_type_hint,
+                    parametrized_type_hint_getinitargs(obj)
+                )
+            t = type(obj)
+            try:
+                is_anyclass = issubclass(t, type)
+            except TypeError:  # t is not a class (old Boost; see SF #502085)
+                is_anyclass = False
+
+            if is_anyclass:
+                return _class_reduce(obj)
+            elif isinstance(obj, types.FunctionType):
+                return self._function_reduce(obj)
+            else:
+                # fallback to save_global, including the Pickler's
+                # distpatch_table
+                return NotImplemented
+
+    else:
+        # When reducer_override is not available, hack the pure-Python
+        # Pickler's types.FunctionType and type savers. Note: the type saver
+        # must override Pickler.save_global, because pickle.py contains a
+        # hard-coded call to save_global when pickling meta-classes.
+        dispatch = Pickler.dispatch.copy()
+
+        def __init__(self, file, protocol=None):
+            if protocol is None:
+                protocol = DEFAULT_PROTOCOL
+            Pickler.__init__(self, file, protocol=protocol)
+            # map functions __globals__ attribute ids, to ensure that functions
+            # sharing the same global namespace at pickling time also share
+            # their global namespace at unpickling time.
+            self.globals_ref = {}
+            assert hasattr(self, 'proto')
+
+        def _save_reduce_pickle5(self, func, args, state=None, listitems=None,
+                                 dictitems=None, state_setter=None, obj=None):
+            save = self.save
+            write = self.write
+            self.save_reduce(
+                func, args, state=None, listitems=listitems,
+                dictitems=dictitems, obj=obj
+            )
+            # backport of the Python 3.8 state_setter pickle operations
+            save(state_setter)
+            save(obj)  # simple BINGET opcode as obj is already memoized.
+            save(state)
+            write(pickle.TUPLE2)
+            # Trigger a state_setter(obj, state) function call.
+            write(pickle.REDUCE)
+            # The purpose of state_setter is to carry-out an
+            # inplace modification of obj. We do not care about what the
+            # method might return, so its output is eventually removed from
+            # the stack.
+            write(pickle.POP)
+
+        def save_global(self, obj, name=None, pack=struct.pack):
+            """
+            Save a "global".
+
+            The name of this method is somewhat misleading: all types get
+            dispatched here.
+            """
+            if obj is type(None):  # noqa
+                return self.save_reduce(type, (None,), obj=obj)
+            elif obj is type(Ellipsis):
+                return self.save_reduce(type, (Ellipsis,), obj=obj)
+            elif obj is type(NotImplemented):
+                return self.save_reduce(type, (NotImplemented,), obj=obj)
+            elif obj in _BUILTIN_TYPE_NAMES:
+                return self.save_reduce(
+                    _builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj)
+
+            if sys.version_info[:2] < (3, 7) and _is_parametrized_type_hint(obj):  # noqa  # pragma: no branch
+                # Parametrized typing constructs in Python < 3.7 are not
+                # compatible with type checks and ``isinstance`` semantics. For
+                # this reason, it is easier to detect them using a
+                # duck-typing-based check (``_is_parametrized_type_hint``) than
+                # to populate the Pickler's dispatch with type-specific savers.
+                self.save_reduce(
+                    _create_parametrized_type_hint,
+                    parametrized_type_hint_getinitargs(obj),
+                    obj=obj
+                )
+            elif name is not None:
+                Pickler.save_global(self, obj, name=name)
+            elif not _is_importable(obj, name=name):
+                self._save_reduce_pickle5(*_dynamic_class_reduce(obj), obj=obj)
+            else:
+                Pickler.save_global(self, obj, name=name)
+        dispatch[type] = save_global
+
+        def save_function(self, obj, name=None):
+            """ Registered with the dispatch to handle all function types.
+
+            Determines what kind of function obj is (e.g. lambda, defined at
+            interactive prompt, etc) and handles the pickling appropriately.
+            """
+            if _is_importable(obj, name=name):
+                return Pickler.save_global(self, obj, name=name)
+            elif PYPY and isinstance(obj.__code__, builtin_code_type):
+                return self.save_pypy_builtin_func(obj)
+            else:
+                return self._save_reduce_pickle5(
+                    *self._dynamic_function_reduce(obj), obj=obj
+                )
+
+        def save_pypy_builtin_func(self, obj):
+            """Save pypy equivalent of builtin functions.
+            PyPy does not have the concept of builtin-functions. Instead,
+            builtin-functions are simple function instances, but with a
+            builtin-code attribute.
+            Most of the time, builtin functions should be pickled by attribute.
+            But PyPy has flaky support for __qualname__, so some builtin
+            functions such as float.__new__ will be classified as dynamic. For
+            this reason only, we created this special routine. Because
+            builtin-functions are not expected to have closure or globals,
+            there is no additional hack (compared the one already implemented
+            in pickle) to protect ourselves from reference cycles. A simple
+            (reconstructor, newargs, obj.__dict__) tuple is save_reduced.  Note
+            also that PyPy improved their support for __qualname__ in v3.6, so
+            this routing should be removed when cloudpickle supports only PyPy
+            3.6 and later.
+            """
+            rv = (types.FunctionType, (obj.__code__, {}, obj.__name__,
+                                       obj.__defaults__, obj.__closure__),
+                  obj.__dict__)
+            self.save_reduce(*rv, obj=obj)
+
+        dispatch[types.FunctionType] = save_function
diff --git a/python/pyspark/cloudpickle/compat.py b/python/pyspark/cloudpickle/compat.py
new file mode 100644
index 0000000000000..afa285f62903d
--- /dev/null
+++ b/python/pyspark/cloudpickle/compat.py
@@ -0,0 +1,13 @@
+import sys
+
+
+if sys.version_info < (3, 8):
+    try:
+        import pickle5 as pickle  # noqa: F401
+        from pickle5 import Pickler  # noqa: F401
+    except ImportError:
+        import pickle  # noqa: F401
+        from pickle import _Pickler as Pickler  # noqa: F401
+else:
+    import pickle  # noqa: F401
+    from _pickle import Pickler  # noqa: F401
diff --git a/python/pyspark/conf.py b/python/pyspark/conf.py
index 2024260868197..c4c3e43de60bf 100644
--- a/python/pyspark/conf.py
+++ b/python/pyspark/conf.py
@@ -15,61 +15,9 @@
 # limitations under the License.
 #
 
-"""
->>> from pyspark.conf import SparkConf
->>> from pyspark.context import SparkContext
->>> conf = SparkConf()
->>> conf.setMaster("local").setAppName("My app")
-<pyspark.conf.SparkConf object at ...>
->>> conf.get("spark.master")
-u'local'
->>> conf.get("spark.app.name")
-u'My app'
->>> sc = SparkContext(conf=conf)
->>> sc.master
-u'local'
->>> sc.appName
-u'My app'
->>> sc.sparkHome is None
-True
-
->>> conf = SparkConf(loadDefaults=False)
->>> conf.setSparkHome("/path")
-<pyspark.conf.SparkConf object at ...>
->>> conf.get("spark.home")
-u'/path'
->>> conf.setExecutorEnv("VAR1", "value1")
-<pyspark.conf.SparkConf object at ...>
->>> conf.setExecutorEnv(pairs = [("VAR3", "value3"), ("VAR4", "value4")])
-<pyspark.conf.SparkConf object at ...>
->>> conf.get("spark.executorEnv.VAR1")
-u'value1'
->>> print(conf.toDebugString())
-spark.executorEnv.VAR1=value1
-spark.executorEnv.VAR3=value3
-spark.executorEnv.VAR4=value4
-spark.home=/path
->>> sorted(conf.getAll(), key=lambda p: p[0])
-[(u'spark.executorEnv.VAR1', u'value1'), (u'spark.executorEnv.VAR3', u'value3'), \
-(u'spark.executorEnv.VAR4', u'value4'), (u'spark.home', u'/path')]
->>> conf._jconf.setExecutorEnv("VAR5", "value5")
-JavaObject id...
->>> print(conf.toDebugString())
-spark.executorEnv.VAR1=value1
-spark.executorEnv.VAR3=value3
-spark.executorEnv.VAR4=value4
-spark.executorEnv.VAR5=value5
-spark.home=/path
-"""
-
 __all__ = ['SparkConf']
 
 import sys
-import re
-
-if sys.version > '3':
-    unicode = str
-    __doc__ = re.sub(r"(\W|^)[uU](['])", r'\1\2', __doc__)
 
 
 class SparkConf(object):
@@ -90,20 +38,76 @@ class SparkConf(object):
     All setter methods in this class support chaining. For example,
     you can write ``conf.setMaster("local").setAppName("My app")``.
 
-    .. note:: Once a SparkConf object is passed to Spark, it is cloned
-        and can no longer be modified by the user.
+    Parameters
+    ----------
+    loadDefaults : bool
+        whether to load values from Java system properties (True by default)
+    _jvm : class:`py4j.java_gateway.JVMView`
+        internal parameter used to pass a handle to the
+        Java VM; does not need to be set by users
+    _jconf : class:`py4j.java_gateway.JavaObject`
+        Optionally pass in an existing SparkConf handle
+        to use its parameters
+
+    Notes
+    -----
+    Once a SparkConf object is passed to Spark, it is cloned
+    and can no longer be modified by the user.
+
+    Examples
+    --------
+    >>> from pyspark.conf import SparkConf
+    >>> from pyspark.context import SparkContext
+    >>> conf = SparkConf()
+    >>> conf.setMaster("local").setAppName("My app")
+    <pyspark.conf.SparkConf object at ...>
+    >>> conf.get("spark.master")
+    'local'
+    >>> conf.get("spark.app.name")
+    'My app'
+    >>> sc = SparkContext(conf=conf)
+    >>> sc.master
+    'local'
+    >>> sc.appName
+    'My app'
+    >>> sc.sparkHome is None
+    True
+
+    >>> conf = SparkConf(loadDefaults=False)
+    >>> conf.setSparkHome("/path")
+    <pyspark.conf.SparkConf object at ...>
+    >>> conf.get("spark.home")
+    '/path'
+    >>> conf.setExecutorEnv("VAR1", "value1")
+    <pyspark.conf.SparkConf object at ...>
+    >>> conf.setExecutorEnv(pairs = [("VAR3", "value3"), ("VAR4", "value4")])
+    <pyspark.conf.SparkConf object at ...>
+    >>> conf.get("spark.executorEnv.VAR1")
+    'value1'
+    >>> print(conf.toDebugString())
+    spark.executorEnv.VAR1=value1
+    spark.executorEnv.VAR3=value3
+    spark.executorEnv.VAR4=value4
+    spark.home=/path
+    >>> for p in sorted(conf.getAll(), key=lambda p: p[0]):
+    ...     print(p)
+    ('spark.executorEnv.VAR1', 'value1')
+    ('spark.executorEnv.VAR3', 'value3')
+    ('spark.executorEnv.VAR4', 'value4')
+    ('spark.home', '/path')
+    >>> conf._jconf.setExecutorEnv("VAR5", "value5")
+    JavaObject id...
+    >>> print(conf.toDebugString())
+    spark.executorEnv.VAR1=value1
+    spark.executorEnv.VAR3=value3
+    spark.executorEnv.VAR4=value4
+    spark.executorEnv.VAR5=value5
+    spark.home=/path
     """
 
     def __init__(self, loadDefaults=True, _jvm=None, _jconf=None):
         """
         Create a new Spark configuration.
-
-        :param loadDefaults: whether to load values from Java system
-               properties (True by default)
-        :param _jvm: internal parameter used to pass a handle to the
-               Java VM; does not need to be set by users
-        :param _jconf: Optionally pass in an existing SparkConf handle
-               to use its parameters
         """
         if _jconf:
             self._jconf = _jconf
@@ -124,9 +128,9 @@ def set(self, key, value):
         """Set a configuration property."""
         # Try to set self._jconf first if JVM is created, set self._conf if JVM is not created yet.
         if self._jconf is not None:
-            self._jconf.set(key, unicode(value))
+            self._jconf.set(key, str(value))
         else:
-            self._conf[key] = unicode(value)
+            self._conf[key] = str(value)
         return self
 
     def setIfMissing(self, key, value):
@@ -165,7 +169,10 @@ def setAll(self, pairs):
         """
         Set multiple parameters, passed as a list of key-value pairs.
 
-        :param pairs: list of key-value pairs to set
+        Parameters
+        ----------
+        pairs : iterable of tuples
+            list of key-value pairs to set
         """
         for (k, v) in pairs:
             self.set(k, v)
diff --git a/python/pyspark/conf.pyi b/python/pyspark/conf.pyi
new file mode 100644
index 0000000000000..f7ca61dea9cd2
--- /dev/null
+++ b/python/pyspark/conf.pyi
@@ -0,0 +1,44 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import overload
+from typing import List, Optional, Tuple
+
+from py4j.java_gateway import JVMView, JavaObject  # type: ignore[import]
+
+class SparkConf:
+    def __init__(
+        self,
+        loadDefaults: bool = ...,
+        _jvm: Optional[JVMView] = ...,
+        _jconf: Optional[JavaObject] = ...,
+    ) -> None: ...
+    def set(self, key: str, value: str) -> SparkConf: ...
+    def setIfMissing(self, key: str, value: str) -> SparkConf: ...
+    def setMaster(self, value: str) -> SparkConf: ...
+    def setAppName(self, value: str) -> SparkConf: ...
+    def setSparkHome(self, value: str) -> SparkConf: ...
+    @overload
+    def setExecutorEnv(self, key: str, value: str) -> SparkConf: ...
+    @overload
+    def setExecutorEnv(self, *, pairs: List[Tuple[str, str]]) -> SparkConf: ...
+    def setAll(self, pairs: List[Tuple[str, str]]) -> SparkConf: ...
+    def get(self, key: str, defaultValue: Optional[str] = ...) -> str: ...
+    def getAll(self) -> List[Tuple[str, str]]: ...
+    def contains(self, key: str) -> bool: ...
+    def toDebugString(self) -> str: ...
diff --git a/python/pyspark/context.py b/python/pyspark/context.py
index 6cc343e3e495c..79fdd22ab13fd 100644
--- a/python/pyspark/context.py
+++ b/python/pyspark/context.py
@@ -21,12 +21,14 @@
 import sys
 import threading
 import warnings
+import importlib
 from threading import RLock
 from tempfile import NamedTemporaryFile
 
 from py4j.protocol import Py4JError
+from py4j.java_gateway import is_instance_of
 
-from pyspark import accumulators
+from pyspark import accumulators, since
 from pyspark.accumulators import Accumulator
 from pyspark.broadcast import Broadcast, BroadcastPickleRegistry
 from pyspark.conf import SparkConf
@@ -35,15 +37,12 @@
 from pyspark.serializers import PickleSerializer, BatchedSerializer, UTF8Deserializer, \
     PairDeserializer, AutoBatchedSerializer, NoOpSerializer, ChunkedStream
 from pyspark.storagelevel import StorageLevel
-from pyspark.resourceinformation import ResourceInformation
-from pyspark.rdd import RDD, _load_from_socket, ignore_unicode_prefix
+from pyspark.resource.information import ResourceInformation
+from pyspark.rdd import RDD, _load_from_socket
+from pyspark.taskcontext import TaskContext
 from pyspark.traceback_utils import CallSite, first_spark_call
 from pyspark.status import StatusTracker
 from pyspark.profiler import ProfilerCollector, BasicProfiler
-from pyspark.util import _warn_pin_thread
-
-if sys.version > '3':
-    xrange = range
 
 
 __all__ = ['SparkContext']
@@ -64,12 +63,59 @@ class SparkContext(object):
     connection to a Spark cluster, and can be used to create :class:`RDD` and
     broadcast variables on that cluster.
 
-    .. note:: Only one :class:`SparkContext` should be active per JVM. You must `stop()`
-        the active :class:`SparkContext` before creating a new one.
-
-    .. note:: :class:`SparkContext` instance is not supported to share across multiple
-        processes out of the box, and PySpark does not guarantee multi-processing execution.
-        Use threads instead for concurrent processing purpose.
+    When you create a new SparkContext, at least the master and app name should
+    be set, either through the named parameters here or through `conf`.
+
+    Parameters
+    ----------
+    master : str, optional
+        Cluster URL to connect to (e.g. mesos://host:port, spark://host:port, local[4]).
+    appName : str, optional
+        A name for your job, to display on the cluster web UI.
+    sparkHome : str, optional
+        Location where Spark is installed on cluster nodes.
+    pyFiles : list, optional
+        Collection of .zip or .py files to send to the cluster
+        and add to PYTHONPATH.  These can be paths on the local file
+        system or HDFS, HTTP, HTTPS, or FTP URLs.
+    environment : dict, optional
+        A dictionary of environment variables to set on
+        worker nodes.
+    batchSize : int, optional
+        The number of Python objects represented as a single
+        Java object. Set 1 to disable batching, 0 to automatically choose
+        the batch size based on object sizes, or -1 to use an unlimited
+        batch size
+    serializer : :class:`pyspark.serializers.Serializer`, optional
+        The serializer for RDDs.
+    conf : :py:class:`pyspark.SparkConf`, optional
+        An object setting Spark properties.
+    gateway : :py:class:`py4j.java_gateway.JavaGateway`,  optional
+        Use an existing gateway and JVM, otherwise a new JVM
+        will be instantiated. This is only used internally.
+    jsc : :py:class:`py4j.java_gateway.JavaObject`, optional
+        The JavaSparkContext instance. This is only used internally.
+    profiler_cls : type, optional
+        A class of custom Profiler used to do profiling
+        (default is :class:`pyspark.profiler.BasicProfiler`).
+
+    Notes
+    -----
+    Only one :class:`SparkContext` should be active per JVM. You must `stop()`
+    the active :class:`SparkContext` before creating a new one.
+
+    :class:`SparkContext` instance is not supported to share across multiple
+    processes out of the box, and PySpark does not guarantee multi-processing execution.
+    Use threads instead for concurrent processing purpose.
+
+    Examples
+    --------
+    >>> from pyspark.context import SparkContext
+    >>> sc = SparkContext('local', 'test')
+    >>> sc2 = SparkContext('local', 'test2') # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    ValueError: ...
     """
 
     _gateway = None
@@ -84,40 +130,11 @@ class SparkContext(object):
     def __init__(self, master=None, appName=None, sparkHome=None, pyFiles=None,
                  environment=None, batchSize=0, serializer=PickleSerializer(), conf=None,
                  gateway=None, jsc=None, profiler_cls=BasicProfiler):
-        """
-        Create a new SparkContext. At least the master and app name should be set,
-        either through the named parameters here or through `conf`.
-
-        :param master: Cluster URL to connect to
-               (e.g. mesos://host:port, spark://host:port, local[4]).
-        :param appName: A name for your job, to display on the cluster web UI.
-        :param sparkHome: Location where Spark is installed on cluster nodes.
-        :param pyFiles: Collection of .zip or .py files to send to the cluster
-               and add to PYTHONPATH.  These can be paths on the local file
-               system or HDFS, HTTP, HTTPS, or FTP URLs.
-        :param environment: A dictionary of environment variables to set on
-               worker nodes.
-        :param batchSize: The number of Python objects represented as a single
-               Java object. Set 1 to disable batching, 0 to automatically choose
-               the batch size based on object sizes, or -1 to use an unlimited
-               batch size
-        :param serializer: The serializer for RDDs.
-        :param conf: A :class:`SparkConf` object setting Spark properties.
-        :param gateway: Use an existing gateway and JVM, otherwise a new JVM
-               will be instantiated.
-        :param jsc: The JavaSparkContext instance (optional).
-        :param profiler_cls: A class of custom Profiler used to do profiling
-               (default is pyspark.profiler.BasicProfiler).
-
-
-        >>> from pyspark.context import SparkContext
-        >>> sc = SparkContext('local', 'test')
-
-        >>> sc2 = SparkContext('local', 'test2') # doctest: +IGNORE_EXCEPTION_DETAIL
-        Traceback (most recent call last):
-            ...
-        ValueError:...
-        """
+        if (conf is None or
+                conf.get("spark.executor.allowSparkContext", "false").lower() != "true"):
+            # In order to prevent SparkContext from being created in executors.
+            SparkContext._assert_on_driver()
+
         self._callsite = first_spark_call() or CallSite(None, None, None)
         if gateway is not None and gateway.gateway_parameters.auth_token is None:
             raise ValueError(
@@ -205,19 +222,14 @@ def _do_init(self, master, appName, sparkHome, pyFiles, environment, batchSize,
         # data via a socket.
         # scala's mangled names w/ $ in them require special treatment.
         self._encryption_enabled = self._jvm.PythonUtils.isEncryptionEnabled(self._jsc)
+        os.environ["SPARK_AUTH_SOCKET_TIMEOUT"] = \
+            str(self._jvm.PythonUtils.getPythonAuthSocketTimeout(self._jsc))
+        os.environ["SPARK_BUFFER_SIZE"] = \
+            str(self._jvm.PythonUtils.getSparkBufferSize(self._jsc))
 
-        self.pythonExec = os.environ.get("PYSPARK_PYTHON", 'python')
+        self.pythonExec = os.environ.get("PYSPARK_PYTHON", 'python3')
         self.pythonVer = "%d.%d" % sys.version_info[:2]
 
-        if sys.version_info < (3, 6):
-            with warnings.catch_warnings():
-                warnings.simplefilter("once")
-                warnings.warn(
-                    "Support for Python 2 and Python 3 prior to version 3.6 is deprecated as "
-                    "of Spark 3.0. See also the plan for dropping Python 2 support at "
-                    "https://spark.apache.org/news/plan-for-dropping-python-2-support.html.",
-                    DeprecationWarning)
-
         # Broadcast's __reduce__ method stores Broadcast instances here.
         # This allows other code to determine which Broadcast instances have
         # been pickled, so it can determine which Java broadcast objects to
@@ -250,7 +262,7 @@ def _do_init(self, master, appName, sparkHome, pyFiles, environment, batchSize,
                         sys.path.insert(1, filepath)
                 except Exception:
                     warnings.warn(
-                        "Failed to add file [%s] speficied in 'spark.submit.pyFiles' to "
+                        "Failed to add file [%s] specified in 'spark.submit.pyFiles' to "
                         "Python path:\n  %s" % (path, "\n  ".join(sys.path)),
                         RuntimeWarning)
 
@@ -363,7 +375,9 @@ def getOrCreate(cls, conf=None):
         """
         Get or instantiate a SparkContext and register it as a singleton object.
 
-        :param conf: SparkConf (optional)
+        Parameters
+        ----------
+        conf : :py:class:`pyspark.SparkConf`, optional
         """
         with SparkContext._lock:
             if SparkContext._active_spark_context is None:
@@ -394,7 +408,6 @@ def version(self):
         return self._jsc.version()
 
     @property
-    @ignore_unicode_prefix
     def applicationId(self):
         """
         A unique identifier for the Spark application.
@@ -403,8 +416,10 @@ def applicationId(self):
         * in case of local spark app something like 'local-1433865536131'
         * in case of YARN something like 'application_1433865536131_34483'
 
+        Examples
+        --------
         >>> sc.applicationId  # doctest: +ELLIPSIS
-        u'local-...'
+        'local-...'
         """
         return self._jsc.sc().applicationId()
 
@@ -469,12 +484,24 @@ def range(self, start, end=None, step=1, numSlices=None):
         way as python's built-in range() function. If called with a single argument,
         the argument is interpreted as `end`, and `start` is set to 0.
 
-        :param start: the start value
-        :param end: the end value (exclusive)
-        :param step: the incremental step (default: 1)
-        :param numSlices: the number of partitions of the new RDD
-        :return: An RDD of int
-
+        Parameters
+        ----------
+        start : int
+            the start value
+        end : int, optional
+            the end value (exclusive)
+        step : int, optional
+            the incremental step (default: 1)
+        numSlices : int, optional
+            the number of partitions of the new RDD
+
+        Returns
+        -------
+        :py:class:`pyspark.RDD`
+            An RDD of int
+
+        Examples
+        --------
         >>> sc.range(5).collect()
         [0, 1, 2, 3, 4]
         >>> sc.range(2, 4).collect()
@@ -486,20 +513,22 @@ def range(self, start, end=None, step=1, numSlices=None):
             end = start
             start = 0
 
-        return self.parallelize(xrange(start, end, step), numSlices)
+        return self.parallelize(range(start, end, step), numSlices)
 
     def parallelize(self, c, numSlices=None):
         """
-        Distribute a local Python collection to form an RDD. Using xrange
+        Distribute a local Python collection to form an RDD. Using range
         is recommended if the input represents a range for performance.
 
+        Examples
+        --------
         >>> sc.parallelize([0, 2, 3, 4, 6], 5).glom().collect()
         [[0], [2], [3], [4], [6]]
-        >>> sc.parallelize(xrange(0, 6, 2), 5).glom().collect()
+        >>> sc.parallelize(range(0, 6, 2), 5).glom().collect()
         [[], [0], [], [2], [4]]
         """
         numSlices = int(numSlices) if numSlices is not None else self.defaultParallelism
-        if isinstance(c, xrange):
+        if isinstance(c, range):
             size = len(c)
             if size == 0:
                 return self.parallelize([], numSlices)
@@ -518,7 +547,7 @@ def f(split, iterator):
                 # the empty iterator to a list, thus make sure worker reuse takes effect.
                 # See more details in SPARK-26549.
                 assert len(list(iterator)) == 0
-                return xrange(getStart(split), getStart(split + 1), step)
+                return range(getStart(split), getStart(split + 1), step)
 
             return self.parallelize([], numSlices).mapPartitionsWithIndex(f)
 
@@ -541,13 +570,18 @@ def _serialize_to_jvm(self, data, serializer, reader_func, createRDDServer):
         """
         Using py4j to send a large dataset to the jvm is really slow, so we use either a file
         or a socket if we have encryption enabled.
-        :param data:
-        :param serializer:
-        :param reader_func:  A function which takes a filename and reads in the data in the jvm and
-                returns a JavaRDD. Only used when encryption is disabled.
-        :param createRDDServer:  A function which creates a PythonRDDServer in the jvm to
-               accept the serialized data, for use when encryption is enabled.
-        :return:
+
+        Examples
+        --------
+        data
+            object to be serialized
+        serializer : :py:class:`pyspark.serializers.Serializer`
+        reader_func : function
+            A function which takes a filename and reads in the data in the jvm and
+            returns a JavaRDD. Only used when encryption is disabled.
+        createRDDServer : function
+            A function which creates a PythonRDDServer in the jvm to
+            accept the serialized data, for use when encryption is enabled.
         """
         if self._encryption_enabled:
             # with encryption, we open a server in java and send the data directly
@@ -571,13 +605,15 @@ def _serialize_to_jvm(self, data, serializer, reader_func, createRDDServer):
                     tempFile.close()
                 return reader_func(tempFile.name)
             finally:
-                # we eagerily reads the file so we can delete right after.
+                # we eagerly reads the file so we can delete right after.
                 os.unlink(tempFile.name)
 
     def pickleFile(self, name, minPartitions=None):
         """
         Load an RDD previously saved using :meth:`RDD.saveAsPickleFile` method.
 
+        Examples
+        --------
         >>> tmpFile = NamedTemporaryFile(delete=True)
         >>> tmpFile.close()
         >>> sc.parallelize(range(10)).saveAsPickleFile(tmpFile.name, 5)
@@ -587,7 +623,6 @@ def pickleFile(self, name, minPartitions=None):
         minPartitions = minPartitions or self.defaultMinPartitions
         return RDD(self._jsc.objectFile(name, minPartitions), self)
 
-    @ignore_unicode_prefix
     def textFile(self, name, minPartitions=None, use_unicode=True):
         """
         Read a text file from HDFS, a local file system (available on all
@@ -599,18 +634,19 @@ def textFile(self, name, minPartitions=None, use_unicode=True):
         as `utf-8`), which is faster and smaller than unicode. (Added in
         Spark 1.2)
 
+        Examples
+        --------
         >>> path = os.path.join(tempdir, "sample-text.txt")
         >>> with open(path, "w") as testFile:
         ...    _ = testFile.write("Hello world!")
         >>> textFile = sc.textFile(path)
         >>> textFile.collect()
-        [u'Hello world!']
+        ['Hello world!']
         """
         minPartitions = minPartitions or min(self.defaultParallelism, 2)
         return RDD(self._jsc.textFile(name, minPartitions), self,
                    UTF8Deserializer(use_unicode))
 
-    @ignore_unicode_prefix
     def wholeTextFiles(self, path, minPartitions=None, use_unicode=True):
         """
         Read a directory of text files from HDFS, a local file system
@@ -620,7 +656,7 @@ def wholeTextFiles(self, path, minPartitions=None, use_unicode=True):
         value is the content of each file.
         The text files must be encoded as UTF-8.
 
-        If use_unicode is False, the strings will be kept as `str` (encoding
+        If `use_unicode` is False, the strings will be kept as `str` (encoding
         as `utf-8`), which is faster and smaller than unicode. (Added in
         Spark 1.2)
 
@@ -643,9 +679,12 @@ def wholeTextFiles(self, path, minPartitions=None, use_unicode=True):
             ...
             (a-hdfs-path/part-nnnnn, its content)
 
-        .. note:: Small files are preferred, as each file will be loaded
-            fully in memory.
+        Notes
+        -----
+        Small files are preferred, as each file will be loaded fully in memory.
 
+        Examples
+        --------
         >>> dirPath = os.path.join(tempdir, "files")
         >>> os.mkdir(dirPath)
         >>> with open(os.path.join(dirPath, "1.txt"), "w") as file1:
@@ -654,7 +693,7 @@ def wholeTextFiles(self, path, minPartitions=None, use_unicode=True):
         ...    _ = file2.write("2")
         >>> textFiles = sc.wholeTextFiles(dirPath)
         >>> sorted(textFiles.collect())
-        [(u'.../1.txt', u'1'), (u'.../2.txt', u'2')]
+        [('.../1.txt', '1'), ('.../2.txt', '2')]
         """
         minPartitions = minPartitions or self.defaultMinPartitions
         return RDD(self._jsc.wholeTextFiles(path, minPartitions), self,
@@ -668,8 +707,9 @@ def binaryFiles(self, path, minPartitions=None):
         in a key-value pair, where the key is the path of each file, the
         value is the content of each file.
 
-        .. note:: Small files are preferred, large file is also allowable, but
-            may cause bad performance.
+        Notes
+        -----
+        Small files are preferred, large file is also allowable, but may cause bad performance.
         """
         minPartitions = minPartitions or self.defaultMinPartitions
         return RDD(self._jsc.binaryFiles(path, minPartitions), self,
@@ -681,8 +721,12 @@ def binaryRecords(self, path, recordLength):
         with the specified numerical format (see ByteBuffer), and the number of
         bytes per record is constant.
 
-        :param path: Directory to the input data files
-        :param recordLength: The length at which to split the records
+        Parameters
+        ----------
+        path : str
+            Directory to the input data files
+        recordLength : int
+            The length at which to split the records
         """
         return RDD(self._jsc.binaryRecords(path, recordLength), self, NoOpSerializer())
 
@@ -707,17 +751,24 @@ def sequenceFile(self, path, keyClass=None, valueClass=None, keyConverter=None,
             3. If this fails, the fallback is to call 'toString' on each key and value
             4. :class:`PickleSerializer` is used to deserialize pickled objects on the Python side
 
-        :param path: path to sequncefile
-        :param keyClass: fully qualified classname of key Writable class
-               (e.g. "org.apache.hadoop.io.Text")
-        :param valueClass: fully qualified classname of value Writable class
-               (e.g. "org.apache.hadoop.io.LongWritable")
-        :param keyConverter:
-        :param valueConverter:
-        :param minSplits: minimum splits in dataset
-               (default min(2, sc.defaultParallelism))
-        :param batchSize: The number of Python objects represented as a single
-               Java object. (default 0, choose batchSize automatically)
+        Parameters
+        ----------
+        path : str
+            path to sequencefile
+        keyClass: str, optional
+            fully qualified classname of key Writable class (e.g. "org.apache.hadoop.io.Text")
+        valueClass : str, optional
+            fully qualified classname of value Writable class
+            (e.g. "org.apache.hadoop.io.LongWritable")
+        keyConverter : str, optional
+            fully qualified name of a function returning key WritableConverter
+        valueConverter : str, optional
+            fully qualifiedname of a function returning value WritableConverter
+        minSplits : int, optional
+            minimum splits in dataset (default min(2, sc.defaultParallelism))
+        batchSize : int, optional
+            The number of Python objects represented as a single
+            Java object. (default 0, choose batchSize automatically)
         """
         minSplits = minSplits or min(self.defaultParallelism, 2)
         jrdd = self._jvm.PythonRDD.sequenceFile(self._jsc, path, keyClass, valueClass,
@@ -729,24 +780,36 @@ def newAPIHadoopFile(self, path, inputFormatClass, keyClass, valueClass, keyConv
         """
         Read a 'new API' Hadoop InputFormat with arbitrary key and value class from HDFS,
         a local file system (available on all nodes), or any Hadoop-supported file system URI.
-        The mechanism is the same as for sc.sequenceFile.
+        The mechanism is the same as for :py:meth:`SparkContext.sequenceFile`.
 
         A Hadoop configuration can be passed in as a Python dict. This will be converted into a
         Configuration in Java
 
-        :param path: path to Hadoop file
-        :param inputFormatClass: fully qualified classname of Hadoop InputFormat
-               (e.g. "org.apache.hadoop.mapreduce.lib.input.TextInputFormat")
-        :param keyClass: fully qualified classname of key Writable class
-               (e.g. "org.apache.hadoop.io.Text")
-        :param valueClass: fully qualified classname of value Writable class
-               (e.g. "org.apache.hadoop.io.LongWritable")
-        :param keyConverter: (None by default)
-        :param valueConverter: (None by default)
-        :param conf: Hadoop configuration, passed in as a dict
-               (None by default)
-        :param batchSize: The number of Python objects represented as a single
-               Java object. (default 0, choose batchSize automatically)
+        Parameters
+        ----------
+        path : str
+            path to Hadoop file
+        inputFormatClass : str
+            fully qualified classname of Hadoop InputFormat
+            (e.g. "org.apache.hadoop.mapreduce.lib.input.TextInputFormat")
+        keyClass : str
+            fully qualified classname of key Writable class
+            (e.g. "org.apache.hadoop.io.Text")
+        valueClass : str
+            fully qualified classname of value Writable class
+            (e.g. "org.apache.hadoop.io.LongWritable")
+        keyConverter : str, optional
+            fully qualified name of a function returning key WritableConverter
+            None by default
+        valueConverter : str, optional
+            fully qualified name of a function returning value WritableConverter
+            None by default
+        conf : dict, optional
+            Hadoop configuration, passed in as a dict
+            None by default
+        batchSize : int, optional
+            The number of Python objects represented as a single
+            Java object. (default 0, choose batchSize automatically)
         """
         jconf = self._dictToJavaMap(conf)
         jrdd = self._jvm.PythonRDD.newAPIHadoopFile(self._jsc, path, inputFormatClass, keyClass,
@@ -760,20 +823,29 @@ def newAPIHadoopRDD(self, inputFormatClass, keyClass, valueClass, keyConverter=N
         Read a 'new API' Hadoop InputFormat with arbitrary key and value class, from an arbitrary
         Hadoop configuration, which is passed in as a Python dict.
         This will be converted into a Configuration in Java.
-        The mechanism is the same as for sc.sequenceFile.
-
-        :param inputFormatClass: fully qualified classname of Hadoop InputFormat
-               (e.g. "org.apache.hadoop.mapreduce.lib.input.TextInputFormat")
-        :param keyClass: fully qualified classname of key Writable class
-               (e.g. "org.apache.hadoop.io.Text")
-        :param valueClass: fully qualified classname of value Writable class
-               (e.g. "org.apache.hadoop.io.LongWritable")
-        :param keyConverter: (None by default)
-        :param valueConverter: (None by default)
-        :param conf: Hadoop configuration, passed in as a dict
-               (None by default)
-        :param batchSize: The number of Python objects represented as a single
-               Java object. (default 0, choose batchSize automatically)
+        The mechanism is the same as for :py:meth:`SparkContext.sequenceFile`.
+
+        Parameters
+        ----------
+        inputFormatClass : str
+            fully qualified classname of Hadoop InputFormat
+            (e.g. "org.apache.hadoop.mapreduce.lib.input.TextInputFormat")
+        keyClass : str
+            fully qualified classname of key Writable class (e.g. "org.apache.hadoop.io.Text")
+        valueClass : str
+            fully qualified classname of value Writable class
+            (e.g. "org.apache.hadoop.io.LongWritable")
+        keyConverter : str, optional
+            fully qualified name of a function returning key WritableConverter
+            (None by default)
+        valueConverter : str, optional
+            fully qualified name of a function returning value WritableConverter
+            (None by default)
+        conf : dict, optional
+            Hadoop configuration, passed in as a dict (None by default)
+        batchSize : int, optional
+            The number of Python objects represented as a single
+            Java object. (default 0, choose batchSize automatically)
         """
         jconf = self._dictToJavaMap(conf)
         jrdd = self._jvm.PythonRDD.newAPIHadoopRDD(self._jsc, inputFormatClass, keyClass,
@@ -786,24 +858,32 @@ def hadoopFile(self, path, inputFormatClass, keyClass, valueClass, keyConverter=
         """
         Read an 'old' Hadoop InputFormat with arbitrary key and value class from HDFS,
         a local file system (available on all nodes), or any Hadoop-supported file system URI.
-        The mechanism is the same as for sc.sequenceFile.
+        The mechanism is the same as for :py:meth:`SparkContext.sequenceFile`.
 
         A Hadoop configuration can be passed in as a Python dict. This will be converted into a
         Configuration in Java.
 
-        :param path: path to Hadoop file
-        :param inputFormatClass: fully qualified classname of Hadoop InputFormat
-               (e.g. "org.apache.hadoop.mapred.TextInputFormat")
-        :param keyClass: fully qualified classname of key Writable class
-               (e.g. "org.apache.hadoop.io.Text")
-        :param valueClass: fully qualified classname of value Writable class
-               (e.g. "org.apache.hadoop.io.LongWritable")
-        :param keyConverter: (None by default)
-        :param valueConverter: (None by default)
-        :param conf: Hadoop configuration, passed in as a dict
-               (None by default)
-        :param batchSize: The number of Python objects represented as a single
-               Java object. (default 0, choose batchSize automatically)
+        path : str
+            path to Hadoop file
+        inputFormatClass : str
+            fully qualified classname of Hadoop InputFormat
+            (e.g. "org.apache.hadoop.mapreduce.lib.input.TextInputFormat")
+        keyClass : str
+            fully qualified classname of key Writable class (e.g. "org.apache.hadoop.io.Text")
+        valueClass : str
+            fully qualified classname of value Writable class
+            (e.g. "org.apache.hadoop.io.LongWritable")
+        keyConverter : str, optional
+            fully qualified name of a function returning key WritableConverter
+            (None by default)
+        valueConverter : str, optional
+            fully qualified name of a function returning value WritableConverter
+            (None by default)
+        conf : dict, optional
+            Hadoop configuration, passed in as a dict (None by default)
+        batchSize : int, optional
+            The number of Python objects represented as a single
+            Java object. (default 0, choose batchSize automatically)
         """
         jconf = self._dictToJavaMap(conf)
         jrdd = self._jvm.PythonRDD.hadoopFile(self._jsc, path, inputFormatClass, keyClass,
@@ -817,20 +897,29 @@ def hadoopRDD(self, inputFormatClass, keyClass, valueClass, keyConverter=None,
         Read an 'old' Hadoop InputFormat with arbitrary key and value class, from an arbitrary
         Hadoop configuration, which is passed in as a Python dict.
         This will be converted into a Configuration in Java.
-        The mechanism is the same as for sc.sequenceFile.
-
-        :param inputFormatClass: fully qualified classname of Hadoop InputFormat
-               (e.g. "org.apache.hadoop.mapred.TextInputFormat")
-        :param keyClass: fully qualified classname of key Writable class
-               (e.g. "org.apache.hadoop.io.Text")
-        :param valueClass: fully qualified classname of value Writable class
-               (e.g. "org.apache.hadoop.io.LongWritable")
-        :param keyConverter: (None by default)
-        :param valueConverter: (None by default)
-        :param conf: Hadoop configuration, passed in as a dict
-               (None by default)
-        :param batchSize: The number of Python objects represented as a single
-               Java object. (default 0, choose batchSize automatically)
+        The mechanism is the same as for :py:meth:`SparkContext.sequenceFile`.
+
+        Parameters
+        ----------
+        inputFormatClass : str
+            fully qualified classname of Hadoop InputFormat
+            (e.g. "org.apache.hadoop.mapreduce.lib.input.TextInputFormat")
+        keyClass : str
+            fully qualified classname of key Writable class (e.g. "org.apache.hadoop.io.Text")
+        valueClass : str
+            fully qualified classname of value Writable class
+            (e.g. "org.apache.hadoop.io.LongWritable")
+        keyConverter : str, optional
+            fully qualified name of a function returning key WritableConverter
+            (None by default)
+        valueConverter : str, optional
+            fully qualified name of a function returning value WritableConverter
+            (None by default)
+        conf : dict, optional
+            Hadoop configuration, passed in as a dict (None by default)
+        batchSize : int, optional
+            The number of Python objects represented as a single
+            Java object. (default 0, choose batchSize automatically)
         """
         jconf = self._dictToJavaMap(conf)
         jrdd = self._jvm.PythonRDD.hadoopRDD(self._jsc, inputFormatClass, keyClass,
@@ -842,7 +931,6 @@ def _checkpointFile(self, name, input_deserializer):
         jrdd = self._jsc.checkpointFile(name)
         return RDD(jrdd, self, input_deserializer)
 
-    @ignore_unicode_prefix
     def union(self, rdds):
         """
         Build the union of a list of RDDs.
@@ -851,21 +939,36 @@ def union(self, rdds):
         although this forces them to be reserialized using the default
         serializer:
 
+        Examples
+        --------
         >>> path = os.path.join(tempdir, "union-text.txt")
         >>> with open(path, "w") as testFile:
         ...    _ = testFile.write("Hello")
         >>> textFile = sc.textFile(path)
         >>> textFile.collect()
-        [u'Hello']
+        ['Hello']
         >>> parallelized = sc.parallelize(["World!"])
         >>> sorted(sc.union([textFile, parallelized]).collect())
-        [u'Hello', 'World!']
+        ['Hello', 'World!']
         """
         first_jrdd_deserializer = rdds[0]._jrdd_deserializer
         if any(x._jrdd_deserializer != first_jrdd_deserializer for x in rdds):
             rdds = [x._reserialize() for x in rdds]
-        cls = SparkContext._jvm.org.apache.spark.api.java.JavaRDD
-        jrdds = SparkContext._gateway.new_array(cls, len(rdds))
+        gw = SparkContext._gateway
+        jvm = SparkContext._jvm
+        jrdd_cls = jvm.org.apache.spark.api.java.JavaRDD
+        jpair_rdd_cls = jvm.org.apache.spark.api.java.JavaPairRDD
+        jdouble_rdd_cls = jvm.org.apache.spark.api.java.JavaDoubleRDD
+        if is_instance_of(gw, rdds[0]._jrdd, jrdd_cls):
+            cls = jrdd_cls
+        elif is_instance_of(gw, rdds[0]._jrdd, jpair_rdd_cls):
+            cls = jpair_rdd_cls
+        elif is_instance_of(gw, rdds[0]._jrdd, jdouble_rdd_cls):
+            cls = jdouble_rdd_cls
+        else:
+            cls_name = rdds[0]._jrdd.getClass().getCanonicalName()
+            raise TypeError("Unsupported Java RDD class %s" % cls_name)
+        jrdds = gw.new_array(cls, len(rdds))
         for i in range(0, len(rdds)):
             jrdds[i] = rdds[i]._jrdd
         return RDD(self._jsc.union(jrdds), self, rdds[0]._jrdd_deserializer)
@@ -911,8 +1014,12 @@ def addFile(self, path, recursive=False):
         A directory can be given if the recursive option is set to True.
         Currently directories are only supported for Hadoop-supported filesystems.
 
-        .. note:: A path can be added only once. Subsequent additions of the same path are ignored.
+        Notes
+        -----
+        A path can be added only once. Subsequent additions of the same path are ignored.
 
+        Examples
+        --------
         >>> from pyspark import SparkFiles
         >>> path = os.path.join(tempdir, "test.txt")
         >>> with open(path, "w") as testFile:
@@ -934,7 +1041,9 @@ def addPyFile(self, path):
         file, a file in HDFS (or other Hadoop-supported filesystems), or an
         HTTP, HTTPS or FTP URI.
 
-        .. note:: A path can be added only once. Subsequent additions of the same path are ignored.
+        Notes
+        -----
+        A path can be added only once. Subsequent additions of the same path are ignored.
         """
         self.addFile(path)
         (dirname, filename) = os.path.split(path)  # dirname may be directory or HDFS/S3 prefix
@@ -942,9 +1051,8 @@ def addPyFile(self, path):
             self._python_includes.append(filename)
             # for tests in local mode
             sys.path.insert(1, os.path.join(SparkFiles.getRootDirectory(), filename))
-        if sys.version > '3':
-            import importlib
-            importlib.invalidate_caches()
+
+        importlib.invalidate_caches()
 
     def setCheckpointDir(self, dirName):
         """
@@ -953,6 +1061,16 @@ def setCheckpointDir(self, dirName):
         """
         self._jsc.sc().setCheckpointDir(dirName)
 
+    @since(3.1)
+    def getCheckpointDir(self):
+        """
+        Return the directory where RDDs are checkpointed. Returns None if no
+        checkpoint directory has been set.
+        """
+        if not self._jsc.sc().getCheckpointDir().isEmpty():
+            return self._jsc.sc().getCheckpointDir().get()
+        return None
+
     def _getJavaStorageLevel(self, storageLevel):
         """
         Returns a Java StorageLevel based on a pyspark.StorageLevel.
@@ -979,6 +1097,23 @@ def setJobGroup(self, groupId, description, interruptOnCancel=False):
         The application can use :meth:`SparkContext.cancelJobGroup` to cancel all
         running jobs in this group.
 
+        Notes
+        -----
+        If interruptOnCancel is set to true for the job group, then job cancellation will result
+        in Thread.interrupt() being called on the job's executor threads. This is useful to help
+        ensure that the tasks are actually stopped in a timely manner, but is off by default due
+        to HDFS-1208, where HDFS may respond to Thread.interrupt() by marking nodes as dead.
+
+        Currently, setting a group ID (set to local properties) with multiple threads
+        does not properly work. Internally threads on PVM and JVM are not synced, and JVM
+        thread can be reused for multiple threads on PVM, which fails to isolate local
+        properties for each thread on PVM.
+
+        To avoid this, enable the pinned thread mode by setting ``PYSPARK_PIN_THREAD``
+        environment variable to ``true`` and uses :class:`pyspark.InheritableThread`.
+
+        Examples
+        --------
         >>> import threading
         >>> from time import sleep
         >>> result = "Not Set"
@@ -1003,26 +1138,7 @@ def setJobGroup(self, groupId, description, interruptOnCancel=False):
         >>> suppress = lock.acquire()
         >>> print(result)
         Cancelled
-
-        If interruptOnCancel is set to true for the job group, then job cancellation will result
-        in Thread.interrupt() being called on the job's executor threads. This is useful to help
-        ensure that the tasks are actually stopped in a timely manner, but is off by default due
-        to HDFS-1208, where HDFS may respond to Thread.interrupt() by marking nodes as dead.
-
-        .. note:: Currently, setting a group ID (set to local properties) with multiple threads
-            does not properly work. Internally threads on PVM and JVM are not synced, and JVM
-            thread can be reused for multiple threads on PVM, which fails to isolate local
-            properties for each thread on PVM.
-
-            To work around this, you can set `PYSPARK_PIN_THREAD` to
-            `'true'` (see SPARK-22340). However, note that it cannot inherit the local properties
-            from the parent thread although it isolates each thread on PVM and JVM with its own
-            local properties.
-
-            To work around this, you should manually copy and set the local
-            properties from the parent thread to the child thread when you create another thread.
         """
-        _warn_pin_thread("setJobGroup")
         self._jsc.setJobGroup(groupId, description, interruptOnCancel)
 
     def setLocalProperty(self, key, value):
@@ -1030,20 +1146,16 @@ def setLocalProperty(self, key, value):
         Set a local property that affects jobs submitted from this thread, such as the
         Spark fair scheduler pool.
 
-        .. note:: Currently, setting a local property with multiple threads does not properly work.
-            Internally threads on PVM and JVM are not synced, and JVM thread
-            can be reused for multiple threads on PVM, which fails to isolate local properties
-            for each thread on PVM.
+        Notes
+        -----
+        Currently, setting a local property with multiple threads does not properly work.
+        Internally threads on PVM and JVM are not synced, and JVM thread
+        can be reused for multiple threads on PVM, which fails to isolate local properties
+        for each thread on PVM.
 
-            To work around this, you can set `PYSPARK_PIN_THREAD` to
-            `'true'` (see SPARK-22340). However, note that it cannot inherit the local properties
-            from the parent thread although it isolates each thread on PVM and JVM with its own
-            local properties.
-
-            To work around this, you should manually copy and set the local
-            properties from the parent thread to the child thread when you create another thread.
+        To avoid this, enable the pinned thread mode by setting ``PYSPARK_PIN_THREAD``
+        environment variable to ``true`` and uses :class:`pyspark.InheritableThread`.
         """
-        _warn_pin_thread("setLocalProperty")
         self._jsc.setLocalProperty(key, value)
 
     def getLocalProperty(self, key):
@@ -1057,20 +1169,16 @@ def setJobDescription(self, value):
         """
         Set a human readable description of the current job.
 
-        .. note:: Currently, setting a job description (set to local properties) with multiple
-            threads does not properly work. Internally threads on PVM and JVM are not synced,
-            and JVM thread can be reused for multiple threads on PVM, which fails to isolate
-            local properties for each thread on PVM.
-
-            To work around this, you can set `PYSPARK_PIN_THREAD` to
-            `'true'` (see SPARK-22340). However, note that it cannot inherit the local properties
-            from the parent thread although it isolates each thread on PVM and JVM with its own
-            local properties.
+        Notes
+        -----
+        Currently, setting a job description (set to local properties) with multiple
+        threads does not properly work. Internally threads on PVM and JVM are not synced,
+        and JVM thread can be reused for multiple threads on PVM, which fails to isolate
+        local properties for each thread on PVM.
 
-            To work around this, you should manually copy and set the local
-            properties from the parent thread to the child thread when you create another thread.
+        To avoid this, enable the pinned thread mode by setting ``PYSPARK_PIN_THREAD``
+        environment variable to ``true`` and uses :class:`pyspark.InheritableThread`.
         """
-        _warn_pin_thread("setJobDescription")
         self._jsc.setJobDescription(value)
 
     def sparkUser(self):
@@ -1105,6 +1213,8 @@ def runJob(self, rdd, partitionFunc, partitions=None, allowLocal=False):
 
         If 'partitions' is not specified, this will run over all partitions.
 
+        Examples
+        --------
         >>> myRDD = sc.parallelize(range(6), 3)
         >>> sc.runJob(myRDD, lambda part: [x * x for x in part])
         [0, 1, 4, 9, 16, 25]
@@ -1156,6 +1266,16 @@ def resources(self):
             resources[name] = ResourceInformation(name, addrs)
         return resources
 
+    @staticmethod
+    def _assert_on_driver():
+        """
+        Called to ensure that SparkContext is created only on the Driver.
+
+        Throws an exception if a SparkContext is about to be created in executors.
+        """
+        if TaskContext.get() is not None:
+            raise Exception("SparkContext should only be created and accessed on the driver.")
+
 
 def _test():
     import atexit
diff --git a/python/pyspark/context.pyi b/python/pyspark/context.pyi
new file mode 100644
index 0000000000000..640a69cad08ab
--- /dev/null
+++ b/python/pyspark/context.pyi
@@ -0,0 +1,194 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Iterable,
+    List,
+    NoReturn,
+    Optional,
+    Tuple,
+    Type,
+    TypeVar,
+)
+from types import TracebackType
+
+from py4j.java_gateway import JavaGateway, JavaObject  # type: ignore[import]
+
+from pyspark.accumulators import Accumulator, AccumulatorParam
+from pyspark.broadcast import Broadcast
+from pyspark.conf import SparkConf
+from pyspark.profiler import Profiler  # noqa: F401
+from pyspark.resource.information import ResourceInformation
+from pyspark.rdd import RDD
+from pyspark.serializers import Serializer
+from pyspark.status import StatusTracker
+
+T = TypeVar("T")
+U = TypeVar("U")
+
+class SparkContext:
+    master: str
+    appName: str
+    sparkHome: str
+    PACKAGE_EXTENSIONS: Iterable[str]
+    def __init__(
+        self,
+        master: Optional[str] = ...,
+        appName: Optional[str] = ...,
+        sparkHome: Optional[str] = ...,
+        pyFiles: Optional[List[str]] = ...,
+        environment: Optional[Dict[str, str]] = ...,
+        batchSize: int = ...,
+        serializer: Serializer = ...,
+        conf: Optional[SparkConf] = ...,
+        gateway: Optional[JavaGateway] = ...,
+        jsc: Optional[JavaObject] = ...,
+        profiler_cls: type = ...,
+    ) -> None: ...
+    def __getnewargs__(self) -> NoReturn: ...
+    def __enter__(self) -> SparkContext: ...
+    def __exit__(
+        self,
+        type: Optional[Type[BaseException]],
+        value: Optional[BaseException],
+        trace: Optional[TracebackType],
+    ) -> None: ...
+    @classmethod
+    def getOrCreate(cls, conf: Optional[SparkConf] = ...) -> SparkContext: ...
+    def setLogLevel(self, logLevel: str) -> None: ...
+    @classmethod
+    def setSystemProperty(cls, key: str, value: str) -> None: ...
+    @property
+    def version(self) -> str: ...
+    @property
+    def applicationId(self) -> str: ...
+    @property
+    def uiWebUrl(self) -> str: ...
+    @property
+    def startTime(self) -> int: ...
+    @property
+    def defaultParallelism(self) -> int: ...
+    @property
+    def defaultMinPartitions(self) -> int: ...
+    def stop(self) -> None: ...
+    def emptyRDD(self) -> RDD[Any]: ...
+    def range(
+        self,
+        start: int,
+        end: Optional[int] = ...,
+        step: int = ...,
+        numSlices: Optional[int] = ...,
+    ) -> RDD[int]: ...
+    def parallelize(self, c: Iterable[T], numSlices: Optional[int] = ...) -> RDD[T]: ...
+    def pickleFile(self, name: str, minPartitions: Optional[int] = ...) -> RDD[Any]: ...
+    def textFile(
+        self, name: str, minPartitions: Optional[int] = ..., use_unicode: bool = ...
+    ) -> RDD[str]: ...
+    def wholeTextFiles(
+        self, path: str, minPartitions: Optional[int] = ..., use_unicode: bool = ...
+    ) -> RDD[Tuple[str, str]]: ...
+    def binaryFiles(
+        self, path: str, minPartitions: Optional[int] = ...
+    ) -> RDD[Tuple[str, bytes]]: ...
+    def binaryRecords(self, path: str, recordLength: int) -> RDD[bytes]: ...
+    def sequenceFile(
+        self,
+        path: str,
+        keyClass: Optional[str] = ...,
+        valueClass: Optional[str] = ...,
+        keyConverter: Optional[str] = ...,
+        valueConverter: Optional[str] = ...,
+        minSplits: Optional[int] = ...,
+        batchSize: int = ...,
+    ) -> RDD[Tuple[T, U]]: ...
+    def newAPIHadoopFile(
+        self,
+        path: str,
+        inputFormatClass: str,
+        keyClass: str,
+        valueClass: str,
+        keyConverter: Optional[str] = ...,
+        valueConverter: Optional[str] = ...,
+        conf: Optional[Dict[str, str]] = ...,
+        batchSize: int = ...,
+    ) -> RDD[Tuple[T, U]]: ...
+    def newAPIHadoopRDD(
+        self,
+        inputFormatClass: str,
+        keyClass: str,
+        valueClass: str,
+        keyConverter: Optional[str] = ...,
+        valueConverter: Optional[str] = ...,
+        conf: Optional[Dict[str, str]] = ...,
+        batchSize: int = ...,
+    ) -> RDD[Tuple[T, U]]: ...
+    def hadoopFile(
+        self,
+        path: str,
+        inputFormatClass: str,
+        keyClass: str,
+        valueClass: str,
+        keyConverter: Optional[str] = ...,
+        valueConverter: Optional[str] = ...,
+        conf: Optional[Dict[str, str]] = ...,
+        batchSize: int = ...,
+    ) -> RDD[Tuple[T, U]]: ...
+    def hadoopRDD(
+        self,
+        inputFormatClass: str,
+        keyClass: str,
+        valueClass: str,
+        keyConverter: Optional[str] = ...,
+        valueConverter: Optional[str] = ...,
+        conf: Optional[Dict[str, str]] = ...,
+        batchSize: int = ...,
+    ) -> RDD[Tuple[T, U]]: ...
+    def union(self, rdds: Iterable[RDD[T]]) -> RDD[T]: ...
+    def broadcast(self, value: T) -> Broadcast[T]: ...
+    def accumulator(
+        self, value: T, accum_param: Optional[AccumulatorParam[T]] = ...
+    ) -> Accumulator[T]: ...
+    def addFile(self, path: str, recursive: bool = ...) -> None: ...
+    def addPyFile(self, path: str) -> None: ...
+    def setCheckpointDir(self, dirName: str) -> None: ...
+    def getCheckpointDir(self) -> Optional[str]: ...
+    def setJobGroup(
+        self, groupId: str, description: str, interruptOnCancel: bool = ...
+    ) -> None: ...
+    def setLocalProperty(self, key: str, value: str) -> None: ...
+    def getLocalProperty(self, key: str) -> Optional[str]: ...
+    def sparkUser(self) -> str: ...
+    def setJobDescription(self, value: str) -> None: ...
+    def cancelJobGroup(self, groupId: str) -> None: ...
+    def cancelAllJobs(self) -> None: ...
+    def statusTracker(self) -> StatusTracker: ...
+    def runJob(
+        self,
+        rdd: RDD[T],
+        partitionFunc: Callable[[Iterable[T]], Iterable[U]],
+        partitions: Optional[List[int]] = ...,
+        allowLocal: bool = ...,
+    ) -> List[U]: ...
+    def show_profiles(self) -> None: ...
+    def dump_profiles(self, path: str) -> None: ...
+    def getConf(self) -> SparkConf: ...
+    @property
+    def resources(self) -> Dict[str, ResourceInformation]: ...
diff --git a/python/pyspark/files.pyi b/python/pyspark/files.pyi
new file mode 100644
index 0000000000000..9e7cad17ebbdb
--- /dev/null
+++ b/python/pyspark/files.pyi
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class SparkFiles:
+    def __init__(self) -> None: ...
+    @classmethod
+    def get(cls, filename: str) -> str: ...
+    @classmethod
+    def getRootDirectory(cls) -> str: ...
diff --git a/python/pyspark/find_spark_home.py b/python/pyspark/find_spark_home.py
index 52f6ea9a37100..62a36d42ebc72 100755
--- a/python/pyspark/find_spark_home.py
+++ b/python/pyspark/find_spark_home.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
@@ -20,7 +20,6 @@
 # This script attempt to determine the correct setting for SPARK_HOME given
 # that Spark may have been installed on the system with pip.
 
-from __future__ import print_function
 import os
 import sys
 
@@ -37,30 +36,28 @@ def is_spark_home(path):
                 (os.path.isdir(os.path.join(path, "jars")) or
                  os.path.isdir(os.path.join(path, "assembly"))))
 
-    paths = ["../", os.path.dirname(os.path.realpath(__file__))]
+    # Spark distribution can be downloaded when PYSPARK_HADOOP_VERSION environment variable is set.
+    # We should look up this directory first, see also SPARK-32017.
+    spark_dist_dir = "spark-distribution"
+    paths = [
+        "../",  # When we're in spark/python.
+        # Two case belows are valid when the current script is called as a library.
+        os.path.join(os.path.dirname(os.path.realpath(__file__)), spark_dist_dir),
+        os.path.dirname(os.path.realpath(__file__))]
 
     # Add the path of the PySpark module if it exists
     import_error_raised = False
-    if sys.version < "3":
-        import imp
-        try:
-            module_home = imp.find_module("pyspark")[1]
-            paths.append(module_home)
-            # If we are installed in edit mode also look two dirs up
-            paths.append(os.path.join(module_home, "../../"))
-        except ImportError:
-            # Not pip installed no worries
-            import_error_raised = True
-    else:
-        from importlib.util import find_spec
-        try:
-            module_home = os.path.dirname(find_spec("pyspark").origin)
-            paths.append(module_home)
-            # If we are installed in edit mode also look two dirs up
-            paths.append(os.path.join(module_home, "../../"))
-        except ImportError:
-            # Not pip installed no worries
-            import_error_raised = True
+    from importlib.util import find_spec
+    try:
+        module_home = os.path.dirname(find_spec("pyspark").origin)
+        paths.append(os.path.join(module_home, spark_dist_dir))
+        paths.append(module_home)
+        # If we are installed in edit mode also look two dirs up
+        # Downloading different versions are not supported in edit mode.
+        paths.append(os.path.join(module_home, "../../"))
+    except ImportError:
+        # Not pip installed no worries
+        import_error_raised = True
 
     # Normalize the paths
     paths = [os.path.abspath(p) for p in paths]
@@ -84,5 +81,6 @@ def is_spark_home(path):
                 "'PYSPARK_PYTHON=python3 pyspark'.\n", file=sys.stderr)
         sys.exit(-1)
 
+
 if __name__ == "__main__":
     print(_find_spark_home())
diff --git a/python/pyspark/heapq3.py b/python/pyspark/heapq3.py
deleted file mode 100644
index 37a2914ebac05..0000000000000
--- a/python/pyspark/heapq3.py
+++ /dev/null
@@ -1,890 +0,0 @@
-# -*- encoding: utf-8 -*-
-#  back ported from CPython 3
-# A. HISTORY OF THE SOFTWARE
-# ==========================
-#
-# Python was created in the early 1990s by Guido van Rossum at Stichting
-# Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands
-# as a successor of a language called ABC.  Guido remains Python's
-# principal author, although it includes many contributions from others.
-#
-# In 1995, Guido continued his work on Python at the Corporation for
-#     National Research Initiatives (CNRI, see http://www.cnri.reston.va.us)
-# in Reston, Virginia where he released several versions of the
-# software.
-#
-# In May 2000, Guido and the Python core development team moved to
-# BeOpen.com to form the BeOpen PythonLabs team.  In October of the same
-# year, the PythonLabs team moved to Digital Creations (now Zope
-# Corporation, see http://www.zope.com).  In 2001, the Python Software
-# Foundation (PSF, see http://www.python.org/psf/) was formed, a
-# non-profit organization created specifically to own Python-related
-# Intellectual Property.  Zope Corporation is a sponsoring member of
-# the PSF.
-#
-# All Python releases are Open Source (see http://www.opensource.org for
-# the Open Source Definition).  Historically, most, but not all, Python
-# releases have also been GPL-compatible; the table below summarizes
-# the various releases.
-#
-# Release         Derived     Year        Owner       GPL-
-# from                                compatible? (1)
-#
-# 0.9.0 thru 1.2              1991-1995   CWI         yes
-# 1.3 thru 1.5.2  1.2         1995-1999   CNRI        yes
-# 1.6             1.5.2       2000        CNRI        no
-# 2.0             1.6         2000        BeOpen.com  no
-# 1.6.1           1.6         2001        CNRI        yes (2)
-# 2.1             2.0+1.6.1   2001        PSF         no
-# 2.0.1           2.0+1.6.1   2001        PSF         yes
-# 2.1.1           2.1+2.0.1   2001        PSF         yes
-# 2.2             2.1.1       2001        PSF         yes
-# 2.1.2           2.1.1       2002        PSF         yes
-# 2.1.3           2.1.2       2002        PSF         yes
-# 2.2.1           2.2         2002        PSF         yes
-# 2.2.2           2.2.1       2002        PSF         yes
-# 2.2.3           2.2.2       2003        PSF         yes
-# 2.3             2.2.2       2002-2003   PSF         yes
-# 2.3.1           2.3         2002-2003   PSF         yes
-# 2.3.2           2.3.1       2002-2003   PSF         yes
-# 2.3.3           2.3.2       2002-2003   PSF         yes
-# 2.3.4           2.3.3       2004        PSF         yes
-# 2.3.5           2.3.4       2005        PSF         yes
-# 2.4             2.3         2004        PSF         yes
-# 2.4.1           2.4         2005        PSF         yes
-# 2.4.2           2.4.1       2005        PSF         yes
-# 2.4.3           2.4.2       2006        PSF         yes
-# 2.4.4           2.4.3       2006        PSF         yes
-# 2.5             2.4         2006        PSF         yes
-# 2.5.1           2.5         2007        PSF         yes
-# 2.5.2           2.5.1       2008        PSF         yes
-# 2.5.3           2.5.2       2008        PSF         yes
-# 2.6             2.5         2008        PSF         yes
-# 2.6.1           2.6         2008        PSF         yes
-# 2.6.2           2.6.1       2009        PSF         yes
-# 2.6.3           2.6.2       2009        PSF         yes
-# 2.6.4           2.6.3       2009        PSF         yes
-# 2.6.5           2.6.4       2010        PSF         yes
-# 2.7             2.6         2010        PSF         yes
-#
-# Footnotes:
-#
-# (1) GPL-compatible doesn't mean that we're distributing Python under
-# the GPL.  All Python licenses, unlike the GPL, let you distribute
-# a modified version without making your changes open source.  The
-# GPL-compatible licenses make it possible to combine Python with
-#     other software that is released under the GPL; the others don't.
-#
-# (2) According to Richard Stallman, 1.6.1 is not GPL-compatible,
-# because its license has a choice of law clause.  According to
-# CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1
-# is "not incompatible" with the GPL.
-#
-# Thanks to the many outside volunteers who have worked under Guido's
-# direction to make these releases possible.
-#
-#
-# B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON
-# ===============================================================
-#
-# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
-# --------------------------------------------
-#
-# 1. This LICENSE AGREEMENT is between the Python Software Foundation
-# ("PSF"), and the Individual or Organization ("Licensee") accessing and
-# otherwise using this software ("Python") in source or binary form and
-# its associated documentation.
-#
-# 2. Subject to the terms and conditions of this License Agreement, PSF hereby
-# grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
-# analyze, test, perform and/or display publicly, prepare derivative works,
-# distribute, and otherwise use Python alone or in any derivative version,
-# provided, however, that PSF's License Agreement and PSF's notice of copyright,
-# i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
-# 2011, 2012, 2013 Python Software Foundation; All Rights Reserved" are retained
-# in Python alone or in any derivative version prepared by Licensee.
-#
-# 3. In the event Licensee prepares a derivative work that is based on
-# or incorporates Python or any part thereof, and wants to make
-# the derivative work available to others as provided herein, then
-# Licensee hereby agrees to include in any such work a brief summary of
-# the changes made to Python.
-#
-# 4. PSF is making Python available to Licensee on an "AS IS"
-# basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
-# IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
-# DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
-# FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
-# INFRINGE ANY THIRD PARTY RIGHTS.
-#
-# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
-# FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
-# A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
-# OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
-#
-# 6. This License Agreement will automatically terminate upon a material
-# breach of its terms and conditions.
-#
-# 7. Nothing in this License Agreement shall be deemed to create any
-# relationship of agency, partnership, or joint venture between PSF and
-# Licensee.  This License Agreement does not grant permission to use PSF
-# trademarks or trade name in a trademark sense to endorse or promote
-# products or services of Licensee, or any third party.
-#
-# 8. By copying, installing or otherwise using Python, Licensee
-# agrees to be bound by the terms and conditions of this License
-# Agreement.
-#
-#
-# BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0
-# -------------------------------------------
-#
-# BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1
-#
-# 1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an
-# office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the
-# Individual or Organization ("Licensee") accessing and otherwise using
-# this software in source or binary form and its associated
-# documentation ("the Software").
-#
-# 2. Subject to the terms and conditions of this BeOpen Python License
-# Agreement, BeOpen hereby grants Licensee a non-exclusive,
-# royalty-free, world-wide license to reproduce, analyze, test, perform
-# and/or display publicly, prepare derivative works, distribute, and
-# otherwise use the Software alone or in any derivative version,
-# provided, however, that the BeOpen Python License is retained in the
-# Software, alone or in any derivative version prepared by Licensee.
-#
-# 3. BeOpen is making the Software available to Licensee on an "AS IS"
-# basis.  BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
-# IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND
-# DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
-# FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT
-# INFRINGE ANY THIRD PARTY RIGHTS.
-#
-# 4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE
-# SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS
-# AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY
-# DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
-#
-# 5. This License Agreement will automatically terminate upon a material
-# breach of its terms and conditions.
-#
-# 6. This License Agreement shall be governed by and interpreted in all
-# respects by the law of the State of California, excluding conflict of
-# law provisions.  Nothing in this License Agreement shall be deemed to
-# create any relationship of agency, partnership, or joint venture
-# between BeOpen and Licensee.  This License Agreement does not grant
-# permission to use BeOpen trademarks or trade names in a trademark
-# sense to endorse or promote products or services of Licensee, or any
-# third party.  As an exception, the "BeOpen Python" logos available at
-# http://www.pythonlabs.com/logos.html may be used according to the
-# permissions granted on that web page.
-#
-# 7. By copying, installing or otherwise using the software, Licensee
-# agrees to be bound by the terms and conditions of this License
-# Agreement.
-#
-#
-# CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1
-# ---------------------------------------
-#
-# 1. This LICENSE AGREEMENT is between the Corporation for National
-#     Research Initiatives, having an office at 1895 Preston White Drive,
-# Reston, VA 20191 ("CNRI"), and the Individual or Organization
-# ("Licensee") accessing and otherwise using Python 1.6.1 software in
-# source or binary form and its associated documentation.
-#
-# 2. Subject to the terms and conditions of this License Agreement, CNRI
-# hereby grants Licensee a nonexclusive, royalty-free, world-wide
-# license to reproduce, analyze, test, perform and/or display publicly,
-# prepare derivative works, distribute, and otherwise use Python 1.6.1
-# alone or in any derivative version, provided, however, that CNRI's
-# License Agreement and CNRI's notice of copyright, i.e., "Copyright (c)
-# 1995-2001 Corporation for National Research Initiatives; All Rights
-# Reserved" are retained in Python 1.6.1 alone or in any derivative
-# version prepared by Licensee.  Alternately, in lieu of CNRI's License
-# Agreement, Licensee may substitute the following text (omitting the
-# quotes): "Python 1.6.1 is made available subject to the terms and
-# conditions in CNRI's License Agreement.  This Agreement together with
-# Python 1.6.1 may be located on the Internet using the following
-# unique, persistent identifier (known as a handle): 1895.22/1013.  This
-# Agreement may also be obtained from a proxy server on the Internet
-# using the following URL: http://hdl.handle.net/1895.22/1013".
-#
-# 3. In the event Licensee prepares a derivative work that is based on
-# or incorporates Python 1.6.1 or any part thereof, and wants to make
-# the derivative work available to others as provided herein, then
-# Licensee hereby agrees to include in any such work a brief summary of
-# the changes made to Python 1.6.1.
-#
-# 4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS"
-# basis.  CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
-# IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND
-# DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
-# FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT
-# INFRINGE ANY THIRD PARTY RIGHTS.
-#
-# 5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
-# 1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
-# A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1,
-# OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
-#
-# 6. This License Agreement will automatically terminate upon a material
-# breach of its terms and conditions.
-#
-# 7. This License Agreement shall be governed by the federal
-# intellectual property law of the United States, including without
-# limitation the federal copyright law, and, to the extent such
-# U.S. federal law does not apply, by the law of the Commonwealth of
-# Virginia, excluding Virginia's conflict of law provisions.
-# Notwithstanding the foregoing, with regard to derivative works based
-# on Python 1.6.1 that incorporate non-separable material that was
-# previously distributed under the GNU General Public License (GPL), the
-# law of the Commonwealth of Virginia shall govern this License
-# Agreement only as to issues arising under or with respect to
-# Paragraphs 4, 5, and 7 of this License Agreement.  Nothing in this
-# License Agreement shall be deemed to create any relationship of
-# agency, partnership, or joint venture between CNRI and Licensee.  This
-# License Agreement does not grant permission to use CNRI trademarks or
-# trade name in a trademark sense to endorse or promote products or
-# services of Licensee, or any third party.
-#
-# 8. By clicking on the "ACCEPT" button where indicated, or by copying,
-# installing or otherwise using Python 1.6.1, Licensee agrees to be
-# bound by the terms and conditions of this License Agreement.
-#
-# ACCEPT
-#
-#
-# CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2
-# --------------------------------------------------
-#
-# Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam,
-# The Netherlands.  All rights reserved.
-#
-# Permission to use, copy, modify, and distribute this software and its
-# documentation for any purpose and without fee is hereby granted,
-# provided that the above copyright notice appear in all copies and that
-# both that copyright notice and this permission notice appear in
-# supporting documentation, and that the name of Stichting Mathematisch
-# Centrum or CWI not be used in advertising or publicity pertaining to
-# distribution of the software without specific, written prior
-# permission.
-#
-# STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO
-# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
-# FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE
-# FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
-# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
-# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
-# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-"""Heap queue algorithm (a.k.a. priority queue).
-
-Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for
-all k, counting elements from 0.  For the sake of comparison,
-non-existing elements are considered to be infinite.  The interesting
-property of a heap is that a[0] is always its smallest element.
-
-Usage:
-
-heap = []            # creates an empty heap
-heappush(heap, item) # pushes a new item on the heap
-item = heappop(heap) # pops the smallest item from the heap
-item = heap[0]       # smallest item on the heap without popping it
-heapify(x)           # transforms list into a heap, in-place, in linear time
-item = heapreplace(heap, item) # pops and returns smallest item, and adds
-                               # new item; the heap size is unchanged
-
-Our API differs from textbook heap algorithms as follows:
-
-- We use 0-based indexing.  This makes the relationship between the
-  index for a node and the indexes for its children slightly less
-  obvious, but is more suitable since Python uses 0-based indexing.
-
-- Our heappop() method returns the smallest item, not the largest.
-
-These two make it possible to view the heap as a regular Python list
-without surprises: heap[0] is the smallest item, and heap.sort()
-maintains the heap invariant!
-"""
-
-# Original code by Kevin O'Connor, augmented by Tim Peters and Raymond Hettinger
-
-__about__ = """Heap queues
-
-[explanation by François Pinard]
-
-Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for
-all k, counting elements from 0.  For the sake of comparison,
-non-existing elements are considered to be infinite.  The interesting
-property of a heap is that a[0] is always its smallest element.
-
-The strange invariant above is meant to be an efficient memory
-representation for a tournament.  The numbers below are `k', not a[k]:
-
-                                   0
-
-                  1                                 2
-
-          3               4                5               6
-
-      7       8       9       10      11      12      13      14
-
-    15 16   17 18   19 20   21 22   23 24   25 26   27 28   29 30
-
-
-In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'.  In
-an usual binary tournament we see in sports, each cell is the winner
-over the two cells it tops, and we can trace the winner down the tree
-to see all opponents s/he had.  However, in many computer applications
-of such tournaments, we do not need to trace the history of a winner.
-To be more memory efficient, when a winner is promoted, we try to
-replace it by something else at a lower level, and the rule becomes
-that a cell and the two cells it tops contain three different items,
-but the top cell "wins" over the two topped cells.
-
-If this heap invariant is protected at all time, index 0 is clearly
-the overall winner.  The simplest algorithmic way to remove it and
-find the "next" winner is to move some loser (let's say cell 30 in the
-diagram above) into the 0 position, and then percolate this new 0 down
-the tree, exchanging values, until the invariant is re-established.
-This is clearly logarithmic on the total number of items in the tree.
-By iterating over all items, you get an O(n ln n) sort.
-
-A nice feature of this sort is that you can efficiently insert new
-items while the sort is going on, provided that the inserted items are
-not "better" than the last 0'th element you extracted.  This is
-especially useful in simulation contexts, where the tree holds all
-incoming events, and the "win" condition means the smallest scheduled
-time.  When an event schedule other events for execution, they are
-scheduled into the future, so they can easily go into the heap.  So, a
-heap is a good structure for implementing schedulers (this is what I
-used for my MIDI sequencer :-).
-
-Various structures for implementing schedulers have been extensively
-studied, and heaps are good for this, as they are reasonably speedy,
-the speed is almost constant, and the worst case is not much different
-than the average case.  However, there are other representations which
-are more efficient overall, yet the worst cases might be terrible.
-
-Heaps are also very useful in big disk sorts.  You most probably all
-know that a big sort implies producing "runs" (which are pre-sorted
-sequences, which size is usually related to the amount of CPU memory),
-followed by a merging passes for these runs, which merging is often
-very cleverly organised[1].  It is very important that the initial
-sort produces the longest runs possible.  Tournaments are a good way
-to that.  If, using all the memory available to hold a tournament, you
-replace and percolate items that happen to fit the current run, you'll
-produce runs which are twice the size of the memory for random input,
-and much better for input fuzzily ordered.
-
-Moreover, if you output the 0'th item on disk and get an input which
-may not fit in the current tournament (because the value "wins" over
-the last output value), it cannot fit in the heap, so the size of the
-heap decreases.  The freed memory could be cleverly reused immediately
-for progressively building a second heap, which grows at exactly the
-same rate the first heap is melting.  When the first heap completely
-vanishes, you switch heaps and start a new run.  Clever and quite
-effective!
-
-In a word, heaps are useful memory structures to know.  I use them in
-a few applications, and I think it is good to keep a `heap' module
-around. :-)
-
---------------------
-[1] The disk balancing algorithms which are current, nowadays, are
-more annoying than clever, and this is a consequence of the seeking
-capabilities of the disks.  On devices which cannot seek, like big
-tape drives, the story was quite different, and one had to be very
-clever to ensure (far in advance) that each tape movement will be the
-most effective possible (that is, will best participate at
-"progressing" the merge).  Some tapes were even able to read
-backwards, and this was also used to avoid the rewinding time.
-Believe me, real good tape sorts were quite spectacular to watch!
-From all times, sorting has always been a Great Art! :-)
-"""
-
-__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge',
-           'nlargest', 'nsmallest', 'heappushpop']
-
-def heappush(heap, item):
-    """Push item onto heap, maintaining the heap invariant."""
-    heap.append(item)
-    _siftdown(heap, 0, len(heap)-1)
-
-def heappop(heap):
-    """Pop the smallest item off the heap, maintaining the heap invariant."""
-    lastelt = heap.pop()    # raises appropriate IndexError if heap is empty
-    if heap:
-        returnitem = heap[0]
-        heap[0] = lastelt
-        _siftup(heap, 0)
-        return returnitem
-    return lastelt
-
-def heapreplace(heap, item):
-    """Pop and return the current smallest value, and add the new item.
-
-    This is more efficient than heappop() followed by heappush(), and can be
-    more appropriate when using a fixed-size heap.  Note that the value
-    returned may be larger than item!  That constrains reasonable uses of
-    this routine unless written as part of a conditional replacement:
-
-        if item > heap[0]:
-            item = heapreplace(heap, item)
-    """
-    returnitem = heap[0]    # raises appropriate IndexError if heap is empty
-    heap[0] = item
-    _siftup(heap, 0)
-    return returnitem
-
-def heappushpop(heap, item):
-    """Fast version of a heappush followed by a heappop."""
-    if heap and heap[0] < item:
-        item, heap[0] = heap[0], item
-        _siftup(heap, 0)
-    return item
-
-def heapify(x):
-    """Transform list into a heap, in-place, in O(len(x)) time."""
-    n = len(x)
-    # Transform bottom-up.  The largest index there's any point to looking at
-    # is the largest with a child index in-range, so must have 2*i + 1 < n,
-    # or i < (n-1)/2.  If n is even = 2*j, this is (2*j-1)/2 = j-1/2 so
-    # j-1 is the largest, which is n//2 - 1.  If n is odd = 2*j+1, this is
-    # (2*j+1-1)/2 = j so j-1 is the largest, and that's again n//2-1.
-    for i in reversed(range(n//2)):
-        _siftup(x, i)
-
-def _heappop_max(heap):
-    """Maxheap version of a heappop."""
-    lastelt = heap.pop()    # raises appropriate IndexError if heap is empty
-    if heap:
-        returnitem = heap[0]
-        heap[0] = lastelt
-        _siftup_max(heap, 0)
-        return returnitem
-    return lastelt
-
-def _heapreplace_max(heap, item):
-    """Maxheap version of a heappop followed by a heappush."""
-    returnitem = heap[0]    # raises appropriate IndexError if heap is empty
-    heap[0] = item
-    _siftup_max(heap, 0)
-    return returnitem
-
-def _heapify_max(x):
-    """Transform list into a maxheap, in-place, in O(len(x)) time."""
-    n = len(x)
-    for i in reversed(range(n//2)):
-        _siftup_max(x, i)
-
-# 'heap' is a heap at all indices >= startpos, except possibly for pos.  pos
-# is the index of a leaf with a possibly out-of-order value.  Restore the
-# heap invariant.
-def _siftdown(heap, startpos, pos):
-    newitem = heap[pos]
-    # Follow the path to the root, moving parents down until finding a place
-    # newitem fits.
-    while pos > startpos:
-        parentpos = (pos - 1) >> 1
-        parent = heap[parentpos]
-        if newitem < parent:
-            heap[pos] = parent
-            pos = parentpos
-            continue
-        break
-    heap[pos] = newitem
-
-# The child indices of heap index pos are already heaps, and we want to make
-# a heap at index pos too.  We do this by bubbling the smaller child of
-# pos up (and so on with that child's children, etc) until hitting a leaf,
-# then using _siftdown to move the oddball originally at index pos into place.
-#
-# We *could* break out of the loop as soon as we find a pos where newitem <=
-# both its children, but turns out that's not a good idea, and despite that
-# many books write the algorithm that way.  During a heap pop, the last array
-# element is sifted in, and that tends to be large, so that comparing it
-# against values starting from the root usually doesn't pay (= usually doesn't
-# get us out of the loop early).  See Knuth, Volume 3, where this is
-# explained and quantified in an exercise.
-#
-# Cutting the # of comparisons is important, since these routines have no
-# way to extract "the priority" from an array element, so that intelligence
-# is likely to be hiding in custom comparison methods, or in array elements
-# storing (priority, record) tuples.  Comparisons are thus potentially
-# expensive.
-#
-# On random arrays of length 1000, making this change cut the number of
-# comparisons made by heapify() a little, and those made by exhaustive
-# heappop() a lot, in accord with theory.  Here are typical results from 3
-# runs (3 just to demonstrate how small the variance is):
-#
-# Compares needed by heapify     Compares needed by 1000 heappops
-# --------------------------     --------------------------------
-# 1837 cut to 1663               14996 cut to 8680
-# 1855 cut to 1659               14966 cut to 8678
-# 1847 cut to 1660               15024 cut to 8703
-#
-# Building the heap by using heappush() 1000 times instead required
-# 2198, 2148, and 2219 compares:  heapify() is more efficient, when
-# you can use it.
-#
-# The total compares needed by list.sort() on the same lists were 8627,
-# 8627, and 8632 (this should be compared to the sum of heapify() and
-# heappop() compares):  list.sort() is (unsurprisingly!) more efficient
-# for sorting.
-
-def _siftup(heap, pos):
-    endpos = len(heap)
-    startpos = pos
-    newitem = heap[pos]
-    # Bubble up the smaller child until hitting a leaf.
-    childpos = 2*pos + 1    # leftmost child position
-    while childpos < endpos:
-        # Set childpos to index of smaller child.
-        rightpos = childpos + 1
-        if rightpos < endpos and not heap[childpos] < heap[rightpos]:
-            childpos = rightpos
-        # Move the smaller child up.
-        heap[pos] = heap[childpos]
-        pos = childpos
-        childpos = 2*pos + 1
-    # The leaf at pos is empty now.  Put newitem there, and bubble it up
-    # to its final resting place (by sifting its parents down).
-    heap[pos] = newitem
-    _siftdown(heap, startpos, pos)
-
-def _siftdown_max(heap, startpos, pos):
-    'Maxheap variant of _siftdown'
-    newitem = heap[pos]
-    # Follow the path to the root, moving parents down until finding a place
-    # newitem fits.
-    while pos > startpos:
-        parentpos = (pos - 1) >> 1
-        parent = heap[parentpos]
-        if parent < newitem:
-            heap[pos] = parent
-            pos = parentpos
-            continue
-        break
-    heap[pos] = newitem
-
-def _siftup_max(heap, pos):
-    'Maxheap variant of _siftup'
-    endpos = len(heap)
-    startpos = pos
-    newitem = heap[pos]
-    # Bubble up the larger child until hitting a leaf.
-    childpos = 2*pos + 1    # leftmost child position
-    while childpos < endpos:
-        # Set childpos to index of larger child.
-        rightpos = childpos + 1
-        if rightpos < endpos and not heap[rightpos] < heap[childpos]:
-            childpos = rightpos
-        # Move the larger child up.
-        heap[pos] = heap[childpos]
-        pos = childpos
-        childpos = 2*pos + 1
-    # The leaf at pos is empty now.  Put newitem there, and bubble it up
-    # to its final resting place (by sifting its parents down).
-    heap[pos] = newitem
-    _siftdown_max(heap, startpos, pos)
-
-def merge(iterables, key=None, reverse=False):
-    '''Merge multiple sorted inputs into a single sorted output.
-
-    Similar to sorted(itertools.chain(*iterables)) but returns a generator,
-    does not pull the data into memory all at once, and assumes that each of
-    the input streams is already sorted (smallest to largest).
-
-    >>> list(merge([1,3,5,7], [0,2,4,8], [5,10,15,20], [], [25]))
-    [0, 1, 2, 3, 4, 5, 5, 7, 8, 10, 15, 20, 25]
-
-    If *key* is not None, applies a key function to each element to determine
-    its sort order.
-
-    >>> list(merge(['dog', 'horse'], ['cat', 'fish', 'kangaroo'], key=len))
-    ['dog', 'cat', 'fish', 'horse', 'kangaroo']
-
-    '''
-
-    h = []
-    h_append = h.append
-
-    if reverse:
-        _heapify = _heapify_max
-        _heappop = _heappop_max
-        _heapreplace = _heapreplace_max
-        direction = -1
-    else:
-        _heapify = heapify
-        _heappop = heappop
-        _heapreplace = heapreplace
-        direction = 1
-
-    if key is None:
-        for order, it in enumerate(map(iter, iterables)):
-            try:
-                h_append([next(it), order * direction, it])
-            except StopIteration:
-                pass
-        _heapify(h)
-        while len(h) > 1:
-            try:
-                while True:
-                    value, order, it = s = h[0]
-                    yield value
-                    s[0] = next(it)           # raises StopIteration when exhausted
-                    _heapreplace(h, s)      # restore heap condition
-            except StopIteration:
-                _heappop(h)                 # remove empty iterator
-        if h:
-            # fast case when only a single iterator remains
-            value, order, it = h[0]
-            yield value
-            for value in it:
-                yield value
-        return
-
-    for order, it in enumerate(map(iter, iterables)):
-        try:
-            value = next(it)
-            h_append([key(value), order * direction, value, it])
-        except StopIteration:
-            pass
-    _heapify(h)
-    while len(h) > 1:
-        try:
-            while True:
-                key_value, order, value, it = s = h[0]
-                yield value
-                value = next(it)
-                s[0] = key(value)
-                s[2] = value
-                _heapreplace(h, s)
-        except StopIteration:
-            _heappop(h)
-    if h:
-        key_value, order, value, it = h[0]
-        yield value
-        for value in it:
-            yield value
-
-
-# Algorithm notes for nlargest() and nsmallest()
-# ==============================================
-#
-# Make a single pass over the data while keeping the k most extreme values
-# in a heap.  Memory consumption is limited to keeping k values in a list.
-#
-# Measured performance for random inputs:
-#
-#                                   number of comparisons
-#    n inputs     k-extreme values  (average of 5 trials)   % more than min()
-# -------------   ----------------  ---------------------   -----------------
-#      1,000           100                  3,317               231.7%
-#     10,000           100                 14,046                40.5%
-#    100,000           100                105,749                 5.7%
-#  1,000,000           100              1,007,751                 0.8%
-# 10,000,000           100             10,009,401                 0.1%
-#
-# Theoretical number of comparisons for k smallest of n random inputs:
-#
-# Step   Comparisons                  Action
-# ----   --------------------------   ---------------------------
-#  1     1.66 * k                     heapify the first k-inputs
-#  2     n - k                        compare remaining elements to top of heap
-#  3     k * (1 + lg2(k)) * ln(n/k)   replace the topmost value on the heap
-#  4     k * lg2(k) - (k/2)           final sort of the k most extreme values
-#
-# Combining and simplifying for a rough estimate gives:
-#
-#        comparisons = n + k * (log(k, 2) * log(n/k) + log(k, 2) + log(n/k))
-#
-# Computing the number of comparisons for step 3:
-# -----------------------------------------------
-# * For the i-th new value from the iterable, the probability of being in the
-#   k most extreme values is k/i.  For example, the probability of the 101st
-#   value seen being in the 100 most extreme values is 100/101.
-# * If the value is a new extreme value, the cost of inserting it into the
-#   heap is 1 + log(k, 2).
-# * The probability times the cost gives:
-#            (k/i) * (1 + log(k, 2))
-# * Summing across the remaining n-k elements gives:
-#            sum((k/i) * (1 + log(k, 2)) for i in range(k+1, n+1))
-# * This reduces to:
-#            (H(n) - H(k)) * k * (1 + log(k, 2))
-# * Where H(n) is the n-th harmonic number estimated by:
-#            gamma = 0.5772156649
-#            H(n) = log(n, e) + gamma + 1 / (2 * n)
-#   http://en.wikipedia.org/wiki/Harmonic_series_(mathematics)#Rate_of_divergence
-# * Substituting the H(n) formula:
-#            comparisons = k * (1 + log(k, 2)) * (log(n/k, e) + (1/n - 1/k) / 2)
-#
-# Worst-case for step 3:
-# ----------------------
-# In the worst case, the input data is reversed sorted so that every new element
-# must be inserted in the heap:
-#
-#             comparisons = 1.66 * k + log(k, 2) * (n - k)
-#
-# Alternative Algorithms
-# ----------------------
-# Other algorithms were not used because they:
-# 1) Took much more auxiliary memory,
-# 2) Made multiple passes over the data.
-# 3) Made more comparisons in common cases (small k, large n, semi-random input).
-# See the more detailed comparison of approach at:
-# http://code.activestate.com/recipes/577573-compare-algorithms-for-heapqsmallest
-
-def nsmallest(n, iterable, key=None):
-    """Find the n smallest elements in a dataset.
-
-    Equivalent to:  sorted(iterable, key=key)[:n]
-    """
-
-    # Short-cut for n==1 is to use min()
-    if n == 1:
-        it = iter(iterable)
-        sentinel = object()
-        if key is None:
-            result = min(it, default=sentinel)
-        else:
-            result = min(it, default=sentinel, key=key)
-        return [] if result is sentinel else [result]
-
-    # When n>=size, it's faster to use sorted()
-    try:
-        size = len(iterable)
-    except (TypeError, AttributeError):
-        pass
-    else:
-        if n >= size:
-            return sorted(iterable, key=key)[:n]
-
-    # When key is none, use simpler decoration
-    if key is None:
-        it = iter(iterable)
-        # put the range(n) first so that zip() doesn't
-        # consume one too many elements from the iterator
-        result = [(elem, i) for i, elem in zip(range(n), it)]
-        if not result:
-            return result
-        _heapify_max(result)
-        top = result[0][0]
-        order = n
-        _heapreplace = _heapreplace_max
-        for elem in it:
-            if elem < top:
-                _heapreplace(result, (elem, order))
-                top = result[0][0]
-                order += 1
-        result.sort()
-        return [r[0] for r in result]
-
-    # General case, slowest method
-    it = iter(iterable)
-    result = [(key(elem), i, elem) for i, elem in zip(range(n), it)]
-    if not result:
-        return result
-    _heapify_max(result)
-    top = result[0][0]
-    order = n
-    _heapreplace = _heapreplace_max
-    for elem in it:
-        k = key(elem)
-        if k < top:
-            _heapreplace(result, (k, order, elem))
-            top = result[0][0]
-            order += 1
-    result.sort()
-    return [r[2] for r in result]
-
-def nlargest(n, iterable, key=None):
-    """Find the n largest elements in a dataset.
-
-    Equivalent to:  sorted(iterable, key=key, reverse=True)[:n]
-    """
-
-    # Short-cut for n==1 is to use max()
-    if n == 1:
-        it = iter(iterable)
-        sentinel = object()
-        if key is None:
-            result = max(it, default=sentinel)
-        else:
-            result = max(it, default=sentinel, key=key)
-        return [] if result is sentinel else [result]
-
-    # When n>=size, it's faster to use sorted()
-    try:
-        size = len(iterable)
-    except (TypeError, AttributeError):
-        pass
-    else:
-        if n >= size:
-            return sorted(iterable, key=key, reverse=True)[:n]
-
-    # When key is none, use simpler decoration
-    if key is None:
-        it = iter(iterable)
-        result = [(elem, i) for i, elem in zip(range(0, -n, -1), it)]
-        if not result:
-            return result
-        heapify(result)
-        top = result[0][0]
-        order = -n
-        _heapreplace = heapreplace
-        for elem in it:
-            if top < elem:
-                _heapreplace(result, (elem, order))
-                top = result[0][0]
-                order -= 1
-        result.sort(reverse=True)
-        return [r[0] for r in result]
-
-    # General case, slowest method
-    it = iter(iterable)
-    result = [(key(elem), i, elem) for i, elem in zip(range(0, -n, -1), it)]
-    if not result:
-        return result
-    heapify(result)
-    top = result[0][0]
-    order = -n
-    _heapreplace = heapreplace
-    for elem in it:
-        k = key(elem)
-        if top < k:
-            _heapreplace(result, (k, order, elem))
-            top = result[0][0]
-            order -= 1
-    result.sort(reverse=True)
-    return [r[2] for r in result]
-
-# If available, use C implementation
-try:
-    from _heapq import *
-except ImportError:
-    pass
-try:
-    from _heapq import _heapreplace_max
-except ImportError:
-    pass
-try:
-    from _heapq import _heapify_max
-except ImportError:
-    pass
-try:
-    from _heapq import _heappop_max
-except ImportError:
-    pass
-
-
-if __name__ == "__main__":
-    import doctest
-    import sys
-    (failure_count, test_count) = doctest.testmod()
-    if failure_count:
-        sys.exit(-1)
diff --git a/python/pyspark/install.py b/python/pyspark/install.py
new file mode 100644
index 0000000000000..7efee42bac609
--- /dev/null
+++ b/python/pyspark/install.py
@@ -0,0 +1,179 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import os
+import re
+import tarfile
+import traceback
+import urllib.request
+from shutil import rmtree
+# NOTE that we shouldn't import pyspark here because this is used in
+# setup.py, and assume there's no PySpark imported.
+
+DEFAULT_HADOOP = "hadoop3.2"
+DEFAULT_HIVE = "hive2.3"
+SUPPORTED_HADOOP_VERSIONS = ["hadoop2.7", "hadoop3.2", "without-hadoop"]
+SUPPORTED_HIVE_VERSIONS = ["hive2.3"]
+UNSUPPORTED_COMBINATIONS = [  # type: ignore
+]
+
+
+def checked_package_name(spark_version, hadoop_version, hive_version):
+    return "%s-bin-%s" % (spark_version, hadoop_version)
+
+
+def checked_versions(spark_version, hadoop_version, hive_version):
+    """
+    Check the valid combinations of supported versions in Spark distributions.
+
+    Parameters
+    ----------
+    spark_version : str
+        Spark version. It should be X.X.X such as '3.0.0' or spark-3.0.0.
+    hadoop_version : str
+        Hadoop version. It should be X.X such as '2.7' or 'hadoop2.7'.
+        'without' and 'without-hadoop' are supported as special keywords for Hadoop free
+        distribution.
+    hive_version : str
+        Hive version. It should be X.X such as '2.3' or 'hive2.3'.
+
+    Parameters
+    ----------
+    tuple
+        fully-qualified versions of Spark, Hadoop and Hive in a tuple.
+        For example, spark-3.0.0, hadoop3.2 and hive2.3.
+    """
+    if re.match("^[0-9]+\\.[0-9]+\\.[0-9]+$", spark_version):
+        spark_version = "spark-%s" % spark_version
+    if not spark_version.startswith("spark-"):
+        raise RuntimeError(
+            "Spark version should start with 'spark-' prefix; however, "
+            "got %s" % spark_version)
+
+    if hadoop_version == "without":
+        hadoop_version = "without-hadoop"
+    elif re.match("^[0-9]+\\.[0-9]+$", hadoop_version):
+        hadoop_version = "hadoop%s" % hadoop_version
+
+    if hadoop_version not in SUPPORTED_HADOOP_VERSIONS:
+        raise RuntimeError(
+            "Spark distribution of %s is not supported. Hadoop version should be "
+            "one of [%s]" % (hadoop_version, ", ".join(
+                SUPPORTED_HADOOP_VERSIONS)))
+
+    if re.match("^[0-9]+\\.[0-9]+$", hive_version):
+        hive_version = "hive%s" % hive_version
+
+    if hive_version not in SUPPORTED_HIVE_VERSIONS:
+        raise RuntimeError(
+            "Spark distribution of %s is not supported. Hive version should be "
+            "one of [%s]" % (hive_version, ", ".join(
+                SUPPORTED_HADOOP_VERSIONS)))
+
+    return spark_version, hadoop_version, hive_version
+
+
+def install_spark(dest, spark_version, hadoop_version, hive_version):
+    """
+    Installs Spark that corresponds to the given Hadoop version in the current
+    library directory.
+
+    Parameters
+    ----------
+    dest : str
+        The location to download and install the Spark.
+    spark_version : str
+        Spark version. It should be spark-X.X.X form.
+    hadoop_version : str
+        Hadoop version. It should be hadoopX.X
+        such as 'hadoop2.7' or 'without-hadoop'.
+    hive_version : str
+        Hive version. It should be hiveX.X such as 'hive2.3'.
+    """
+
+    package_name = checked_package_name(spark_version, hadoop_version, hive_version)
+    package_local_path = os.path.join(dest, "%s.tgz" % package_name)
+    if "PYSPARK_RELEASE_MIRROR" in os.environ:
+        sites = [os.environ["PYSPARK_RELEASE_MIRROR"]]
+    else:
+        sites = get_preferred_mirrors()
+    print("Trying to download Spark %s from [%s]" % (spark_version, ", ".join(sites)))
+
+    pretty_pkg_name = "%s for Hadoop %s" % (
+        spark_version,
+        "Free build" if hadoop_version == "without" else hadoop_version)
+
+    for site in sites:
+        os.makedirs(dest, exist_ok=True)
+        url = "%s/spark/%s/%s.tgz" % (site, spark_version, package_name)
+
+        tar = None
+        try:
+            print("Downloading %s from:\n- %s" % (pretty_pkg_name, url))
+            download_to_file(urllib.request.urlopen(url), package_local_path)
+
+            print("Installing to %s" % dest)
+            tar = tarfile.open(package_local_path, "r:gz")
+            for member in tar.getmembers():
+                if member.name == package_name:
+                    # Skip the root directory.
+                    continue
+                member.name = os.path.relpath(member.name, package_name + os.path.sep)
+                tar.extract(member, dest)
+            return
+        except Exception:
+            print("Failed to download %s from %s:" % (pretty_pkg_name, url))
+            traceback.print_exc()
+            rmtree(dest, ignore_errors=True)
+        finally:
+            if tar is not None:
+                tar.close()
+            if os.path.exists(package_local_path):
+                os.remove(package_local_path)
+    raise IOError("Unable to download %s." % pretty_pkg_name)
+
+
+def get_preferred_mirrors():
+    mirror_urls = []
+    for _ in range(3):
+        try:
+            response = urllib.request.urlopen(
+                "https://www.apache.org/dyn/closer.lua?preferred=true")
+            mirror_urls.append(response.read().decode('utf-8'))
+        except Exception:
+            # If we can't get a mirror URL, skip it. No retry.
+            pass
+
+    default_sites = [
+        "https://archive.apache.org/dist", "https://dist.apache.org/repos/dist/release"]
+    return list(set(mirror_urls)) + default_sites
+
+
+def download_to_file(response, path, chunk_size=1024 * 1024):
+    total_size = int(response.info().get('Content-Length').strip())
+    bytes_so_far = 0
+
+    with open(path, mode="wb") as dest:
+        while True:
+            chunk = response.read(chunk_size)
+            bytes_so_far += len(chunk)
+            if not chunk:
+                break
+            dest.write(chunk)
+            print("Downloaded %d of %d bytes (%0.2f%%)" % (
+                bytes_so_far,
+                total_size,
+                round(float(bytes_so_far) / total_size * 100, 2)))
diff --git a/python/pyspark/java_gateway.py b/python/pyspark/java_gateway.py
index 0daf09b17a82a..cc0c3a8888a66 100644
--- a/python/pyspark/java_gateway.py
+++ b/python/pyspark/java_gateway.py
@@ -17,7 +17,6 @@
 
 import atexit
 import os
-import sys
 import signal
 import shlex
 import shutil
@@ -27,25 +26,29 @@
 import time
 from subprocess import Popen, PIPE
 
-if sys.version >= '3':
-    xrange = range
-
 from py4j.java_gateway import java_import, JavaGateway, JavaObject, GatewayParameters
 from py4j.clientserver import ClientServer, JavaParameters, PythonParameters
 from pyspark.find_spark_home import _find_spark_home
 from pyspark.serializers import read_int, write_with_length, UTF8Deserializer
-from pyspark.util import _exception_message
 
 
 def launch_gateway(conf=None, popen_kwargs=None):
     """
     launch jvm gateway
-    :param conf: spark configuration passed to spark-submit
-    :param popen_kwargs: Dictionary of kwargs to pass to Popen when spawning
+
+    Parameters
+    ----------
+    conf : :py:class:`pyspark.SparkConf`
+        spark configuration passed to spark-submit
+    popen_kwargs : dict
+        Dictionary of kwargs to pass to Popen when spawning
         the py4j JVM. This is a developer feature intended for use in
         customizing how pyspark interacts with the py4j JVM (e.g., capturing
         stdout/stderr).
-    :return:
+
+    Returns
+    -------
+    ClientServer or JavaGateway
     """
     if "PYSPARK_GATEWAY_PORT" in os.environ:
         gateway_port = int(os.environ["PYSPARK_GATEWAY_PORT"])
@@ -179,9 +182,16 @@ def local_connect_and_auth(port, auth_secret):
     """
     Connect to local host, authenticate with it, and return a (sockfile,sock) for that connection.
     Handles IPV4 & IPV6, does some error handling.
-    :param port
-    :param auth_secret
-    :return: a tuple with (sockfile, sock)
+
+    Parameters
+    ----------
+    port : str or int or None
+    auth_secret : str
+
+    Returns
+    -------
+    tuple
+        with (sockfile, sock)
     """
     sock = None
     errors = []
@@ -191,14 +201,14 @@ def local_connect_and_auth(port, auth_secret):
         af, socktype, proto, _, sa = res
         try:
             sock = socket.socket(af, socktype, proto)
-            sock.settimeout(15)
+            sock.settimeout(int(os.environ.get("SPARK_AUTH_SOCKET_TIMEOUT", 15)))
             sock.connect(sa)
             sockfile = sock.makefile("rwb", int(os.environ.get("SPARK_BUFFER_SIZE", 65536)))
             _do_server_auth(sockfile, auth_secret)
             return (sockfile, sock)
         except socket.error as e:
-            emsg = _exception_message(e)
-            errors.append("tried to connect to %s, but an error occured: %s" % (sa, emsg))
+            emsg = str(e)
+            errors.append("tried to connect to %s, but an error occurred: %s" % (sa, emsg))
             sock.close()
             sock = None
     raise Exception("could not open socket: %s" % errors)
diff --git a/python/pyspark/ml/__init__.py b/python/pyspark/ml/__init__.py
index 47fc78e83f217..7d0e55a922629 100644
--- a/python/pyspark/ml/__init__.py
+++ b/python/pyspark/ml/__init__.py
@@ -23,7 +23,7 @@
     Transformer, UnaryTransformer
 from pyspark.ml.pipeline import Pipeline, PipelineModel
 from pyspark.ml import classification, clustering, evaluation, feature, fpm, \
-    image, pipeline, recommendation, regression, stat, tuning, util, linalg, param
+    image, recommendation, regression, stat, tuning, util, linalg, param
 
 __all__ = [
     "Transformer", "UnaryTransformer", "Estimator", "Model",
diff --git a/python/pyspark/ml/_typing.pyi b/python/pyspark/ml/_typing.pyi
new file mode 100644
index 0000000000000..d966a787c0fca
--- /dev/null
+++ b/python/pyspark/ml/_typing.pyi
@@ -0,0 +1,76 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Any, Dict, TypeVar, Union
+from typing_extensions import Literal
+
+import pyspark.ml.base
+import pyspark.ml.param
+import pyspark.ml.util
+import pyspark.ml.wrapper
+
+ParamMap = Dict[pyspark.ml.param.Param, Any]
+PipelineStage = Union[pyspark.ml.base.Estimator, pyspark.ml.base.Transformer]
+
+T = TypeVar("T")
+P = TypeVar("P", bound=pyspark.ml.param.Params)
+M = TypeVar("M", bound=pyspark.ml.base.Transformer)
+JM = TypeVar("JM", bound=pyspark.ml.wrapper.JavaTransformer)
+
+BinaryClassificationEvaluatorMetricType = Union[
+    Literal["areaUnderROC"], Literal["areaUnderPR"]
+]
+RegressionEvaluatorMetricType = Union[
+    Literal["rmse"], Literal["mse"], Literal["r2"], Literal["mae"], Literal["var"]
+]
+MulticlassClassificationEvaluatorMetricType = Union[
+    Literal["f1"],
+    Literal["accuracy"],
+    Literal["weightedPrecision"],
+    Literal["weightedRecall"],
+    Literal["weightedTruePositiveRate"],
+    Literal["weightedFalsePositiveRate"],
+    Literal["weightedFMeasure"],
+    Literal["truePositiveRateByLabel"],
+    Literal["falsePositiveRateByLabel"],
+    Literal["precisionByLabel"],
+    Literal["recallByLabel"],
+    Literal["fMeasureByLabel"],
+]
+MultilabelClassificationEvaluatorMetricType = Union[
+    Literal["subsetAccuracy"],
+    Literal["accuracy"],
+    Literal["hammingLoss"],
+    Literal["precision"],
+    Literal["recall"],
+    Literal["f1Measure"],
+    Literal["precisionByLabel"],
+    Literal["recallByLabel"],
+    Literal["f1MeasureByLabel"],
+    Literal["microPrecision"],
+    Literal["microRecall"],
+    Literal["microF1Measure"],
+]
+ClusteringEvaluatorMetricType = Union[Literal["silhouette"]]
+RankingEvaluatorMetricType = Union[
+    Literal["meanAveragePrecision"],
+    Literal["meanAveragePrecisionAtK"],
+    Literal["precisionAtK"],
+    Literal["ndcgAtK"],
+    Literal["recallAtK"],
+]
diff --git a/python/pyspark/ml/base.py b/python/pyspark/ml/base.py
index a24b1ca5122db..31ce93d2e6485 100644
--- a/python/pyspark/ml/base.py
+++ b/python/pyspark/ml/base.py
@@ -21,8 +21,9 @@
 import threading
 
 from pyspark import since
-from pyspark.ml.param.shared import *
 from pyspark.ml.common import inherit_doc
+from pyspark.ml.param.shared import HasInputCol, HasOutputCol, HasLabelCol, HasFeaturesCol, \
+    HasPredictionCol, Params
 from pyspark.sql.functions import udf
 from pyspark.sql.types import StructField, StructType
 
@@ -33,13 +34,19 @@ class _FitMultipleIterator(object):
     iterator. This class handles the simple case of fitMultiple where each param map should be
     fit independently.
 
-    :param fitSingleModel: Function: (int => Model) which fits an estimator to a dataset.
+    Parameters
+    ----------
+    fitSingleModel : function
+        Callable[[int], Transformer] which fits an estimator to a dataset.
         `fitSingleModel` may be called up to `numModels` times, with a unique index each time.
         Each call to `fitSingleModel` with an index should return the Model associated with
         that index.
-    :param numModel: Number of models this iterator should produce.
+    numModel : int
+        Number of models this iterator should produce.
 
-    See Estimator.fitMultiple for more info.
+    Notes
+    -----
+    See :py:meth:`Estimator.fitMultiple` for more info.
     """
     def __init__(self, fitSingleModel, numModels):
         """
@@ -67,35 +74,51 @@ def next(self):
 
 
 @inherit_doc
-class Estimator(Params):
+class Estimator(Params, metaclass=ABCMeta):
     """
     Abstract class for estimators that fit models to data.
 
     .. versionadded:: 1.3.0
     """
-
-    __metaclass__ = ABCMeta
+    pass
 
     @abstractmethod
     def _fit(self, dataset):
         """
         Fits a model to the input dataset. This is called by the default implementation of fit.
 
-        :param dataset: input dataset, which is an instance of :py:class:`pyspark.sql.DataFrame`
-        :returns: fitted model
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            input dataset
+
+        Returns
+        -------
+        :class:`Transformer`
+            fitted model
         """
         raise NotImplementedError()
 
-    @since("2.3.0")
     def fitMultiple(self, dataset, paramMaps):
         """
         Fits a model to the input dataset for each param map in `paramMaps`.
 
-        :param dataset: input dataset, which is an instance of :py:class:`pyspark.sql.DataFrame`.
-        :param paramMaps: A Sequence of param maps.
-        :return: A thread safe iterable which contains one model for each param map. Each
-                 call to `next(modelIterator)` will return `(index, model)` where model was fit
-                 using `paramMaps[index]`. `index` values may not be sequential.
+        .. versionadded:: 2.3.0
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            input dataset.
+        paramMaps : :py:class:`collections.abc.Sequence`
+            A Sequence of param maps.
+
+        Returns
+        -------
+        :py:class:`_FitMultipleIterator`
+            A thread safe iterable which contains one model for each param map. Each
+            call to `next(modelIterator)` will return `(index, model)` where model was fit
+            using `paramMaps[index]`. `index` values may not be sequential.
         """
         estimator = self.copy()
 
@@ -104,16 +127,25 @@ def fitSingleModel(index):
 
         return _FitMultipleIterator(fitSingleModel, len(paramMaps))
 
-    @since("1.3.0")
     def fit(self, dataset, params=None):
         """
         Fits a model to the input dataset with optional parameters.
 
-        :param dataset: input dataset, which is an instance of :py:class:`pyspark.sql.DataFrame`
-        :param params: an optional param map that overrides embedded params. If a list/tuple of
-                       param maps is given, this calls fit on each param map and returns a list of
-                       models.
-        :returns: fitted model(s)
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            input dataset.
+        params : dict or list or tuple, optional
+            an optional param map that overrides embedded params. If a list/tuple of
+            param maps is given, this calls fit on each param map and returns a list of
+            models.
+
+        Returns
+        -------
+        :py:class:`Transformer` or a list of :py:class:`Transformer`
+            fitted model(s)
         """
         if params is None:
             params = dict()
@@ -128,38 +160,53 @@ def fit(self, dataset, params=None):
             else:
                 return self._fit(dataset)
         else:
-            raise ValueError("Params must be either a param map or a list/tuple of param maps, "
-                             "but got %s." % type(params))
+            raise TypeError("Params must be either a param map or a list/tuple of param maps, "
+                            "but got %s." % type(params))
 
 
 @inherit_doc
-class Transformer(Params):
+class Transformer(Params, metaclass=ABCMeta):
     """
     Abstract class for transformers that transform one dataset into another.
 
     .. versionadded:: 1.3.0
     """
-
-    __metaclass__ = ABCMeta
+    pass
 
     @abstractmethod
     def _transform(self, dataset):
         """
         Transforms the input dataset.
 
-        :param dataset: input dataset, which is an instance of :py:class:`pyspark.sql.DataFrame`
-        :returns: transformed dataset
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            input dataset.
+
+        Returns
+        -------
+        :py:class:`pyspark.sql.DataFrame`
+            transformed dataset
         """
         raise NotImplementedError()
 
-    @since("1.3.0")
     def transform(self, dataset, params=None):
         """
         Transforms the input dataset with optional parameters.
 
-        :param dataset: input dataset, which is an instance of :py:class:`pyspark.sql.DataFrame`
-        :param params: an optional param map that overrides embedded params.
-        :returns: transformed dataset
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            input dataset
+        params : dict, optional
+            an optional param map that overrides embedded params.
+
+        Returns
+        -------
+        :py:class:`pyspark.sql.DataFrame`
+            transformed dataset
         """
         if params is None:
             params = dict()
@@ -169,18 +216,17 @@ def transform(self, dataset, params=None):
             else:
                 return self._transform(dataset)
         else:
-            raise ValueError("Params must be a param map but got %s." % type(params))
+            raise TypeError("Params must be a param map but got %s." % type(params))
 
 
 @inherit_doc
-class Model(Transformer):
+class Model(Transformer, metaclass=ABCMeta):
     """
     Abstract class for models that are fitted by estimators.
 
     .. versionadded:: 1.4.0
     """
-
-    __metaclass__ = ABCMeta
+    pass
 
 
 @inherit_doc
@@ -257,13 +303,11 @@ class _PredictorParams(HasLabelCol, HasFeaturesCol, HasPredictionCol):
 
 
 @inherit_doc
-class Predictor(Estimator, _PredictorParams):
+class Predictor(Estimator, _PredictorParams, metaclass=ABCMeta):
     """
     Estimator for prediction tasks (regression and classification).
     """
 
-    __metaclass__ = ABCMeta
-
     @since("3.0.0")
     def setLabelCol(self, value):
         """
@@ -287,13 +331,11 @@ def setPredictionCol(self, value):
 
 
 @inherit_doc
-class PredictionModel(Model, _PredictorParams):
+class PredictionModel(Model, _PredictorParams, metaclass=ABCMeta):
     """
     Model for prediction tasks (regression and classification).
     """
 
-    __metaclass__ = ABCMeta
-
     @since("3.0.0")
     def setFeaturesCol(self, value):
         """
diff --git a/python/pyspark/ml/base.pyi b/python/pyspark/ml/base.pyi
new file mode 100644
index 0000000000000..4f1c6f98ebc49
--- /dev/null
+++ b/python/pyspark/ml/base.pyi
@@ -0,0 +1,107 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import overload
+from typing import (
+    Callable,
+    Generic,
+    Iterable,
+    List,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+)
+from pyspark.ml._typing import M, P, T, ParamMap
+
+import _thread
+
+import abc
+from abc import abstractmethod
+from pyspark import since as since  # noqa: F401
+from pyspark.ml.common import inherit_doc as inherit_doc  # noqa: F401
+from pyspark.ml.param.shared import (
+    HasFeaturesCol as HasFeaturesCol,
+    HasInputCol as HasInputCol,
+    HasLabelCol as HasLabelCol,
+    HasOutputCol as HasOutputCol,
+    HasPredictionCol as HasPredictionCol,
+    Params as Params,
+)
+from pyspark.sql.functions import udf as udf  # noqa: F401
+from pyspark.sql.types import (  # noqa: F401
+    DataType,
+    StructField as StructField,
+    StructType as StructType,
+)
+
+from pyspark.sql.dataframe import DataFrame
+
+class _FitMultipleIterator:
+    fitSingleModel: Callable[[int], Transformer]
+    numModel: int
+    counter: int = ...
+    lock: _thread.LockType
+    def __init__(
+        self, fitSingleModel: Callable[[int], Transformer], numModels: int
+    ) -> None: ...
+    def __iter__(self) -> _FitMultipleIterator: ...
+    def __next__(self) -> Tuple[int, Transformer]: ...
+    def next(self) -> Tuple[int, Transformer]: ...
+
+class Estimator(Generic[M], Params, metaclass=abc.ABCMeta):
+    @overload
+    def fit(self, dataset: DataFrame, params: Optional[ParamMap] = ...) -> M: ...
+    @overload
+    def fit(
+        self, dataset: DataFrame, params: Union[List[ParamMap], Tuple[ParamMap]]
+    ) -> List[M]: ...
+    def fitMultiple(
+        self, dataset: DataFrame, params: Sequence[ParamMap]
+    ) -> Iterable[Tuple[int, M]]: ...
+
+class Transformer(Params, metaclass=abc.ABCMeta):
+    def transform(
+        self, dataset: DataFrame, params: Optional[ParamMap] = ...
+    ) -> DataFrame: ...
+
+class Model(Transformer, metaclass=abc.ABCMeta): ...
+
+class UnaryTransformer(HasInputCol, HasOutputCol, Transformer, metaclass=abc.ABCMeta):
+    def createTransformFunc(self) -> Callable: ...
+    def outputDataType(self) -> DataType: ...
+    def validateInputType(self, inputType: DataType) -> None: ...
+    def transformSchema(self, schema: StructType) -> StructType: ...
+    def setInputCol(self: M, value: str) -> M: ...
+    def setOutputCol(self: M, value: str) -> M: ...
+
+class _PredictorParams(HasLabelCol, HasFeaturesCol, HasPredictionCol): ...
+
+class Predictor(Estimator[M], _PredictorParams, metaclass=abc.ABCMeta):
+    def setLabelCol(self: P, value: str) -> P: ...
+    def setFeaturesCol(self: P, value: str) -> P: ...
+    def setPredictionCol(self: P, value: str) -> P: ...
+
+class PredictionModel(Generic[T], Model, _PredictorParams, metaclass=abc.ABCMeta):
+    def setFeaturesCol(self: M, value: str) -> M: ...
+    def setPredictionCol(self: M, value: str) -> M: ...
+    @property
+    @abc.abstractmethod
+    def numFeatures(self) -> int: ...
+    @abstractmethod
+    def predict(self, value: T) -> float: ...
diff --git a/python/pyspark/ml/classification.py b/python/pyspark/ml/classification.py
index c43a79fac3312..92f4070b822e9 100644
--- a/python/pyspark/ml/classification.py
+++ b/python/pyspark/ml/classification.py
@@ -15,40 +15,55 @@
 # limitations under the License.
 #
 
+import os
 import operator
 import sys
+import uuid
+import warnings
 from abc import ABCMeta, abstractmethod, abstractproperty
 from multiprocessing.pool import ThreadPool
 
-from pyspark import since, keyword_only
+from pyspark import keyword_only, since, SparkContext
 from pyspark.ml import Estimator, Predictor, PredictionModel, Model
-from pyspark.ml.param.shared import *
+from pyspark.ml.param.shared import HasRawPredictionCol, HasProbabilityCol, HasThresholds, \
+    HasRegParam, HasMaxIter, HasFitIntercept, HasTol, HasStandardization, HasWeightCol, \
+    HasAggregationDepth, HasThreshold, HasBlockSize, HasMaxBlockSizeInMB, Param, Params, \
+    TypeConverters, HasElasticNetParam, HasSeed, HasStepSize, HasSolver, HasParallelism
 from pyspark.ml.tree import _DecisionTreeModel, _DecisionTreeParams, \
     _TreeEnsembleModel, _RandomForestParams, _GBTParams, \
-    _HasVarianceImpurity, _TreeClassifierParams, _TreeEnsembleParams
+    _HasVarianceImpurity, _TreeClassifierParams
 from pyspark.ml.regression import _FactorizationMachinesParams, DecisionTreeRegressionModel
-from pyspark.ml.util import *
 from pyspark.ml.base import _PredictorParams
-from pyspark.ml.wrapper import JavaEstimator, JavaModel, JavaParams, \
+from pyspark.ml.util import DefaultParamsReader, DefaultParamsWriter, \
+    JavaMLReadable, JavaMLReader, JavaMLWritable, JavaMLWriter, \
+    MLReader, MLReadable, MLWriter, MLWritable, HasTrainingSummary
+from pyspark.ml.wrapper import JavaParams, \
     JavaPredictor, JavaPredictionModel, JavaWrapper
-from pyspark.ml.common import inherit_doc, _java2py, _py2java
-from pyspark.ml.linalg import Vectors
+from pyspark.ml.common import inherit_doc
+from pyspark.ml.linalg import Vectors, VectorUDT
 from pyspark.sql import DataFrame
 from pyspark.sql.functions import udf, when
 from pyspark.sql.types import ArrayType, DoubleType
 from pyspark.storagelevel import StorageLevel
 
 __all__ = ['LinearSVC', 'LinearSVCModel',
+           'LinearSVCSummary', 'LinearSVCTrainingSummary',
            'LogisticRegression', 'LogisticRegressionModel',
            'LogisticRegressionSummary', 'LogisticRegressionTrainingSummary',
            'BinaryLogisticRegressionSummary', 'BinaryLogisticRegressionTrainingSummary',
            'DecisionTreeClassifier', 'DecisionTreeClassificationModel',
            'GBTClassifier', 'GBTClassificationModel',
            'RandomForestClassifier', 'RandomForestClassificationModel',
+           'RandomForestClassificationSummary', 'RandomForestClassificationTrainingSummary',
+           'BinaryRandomForestClassificationSummary',
+           'BinaryRandomForestClassificationTrainingSummary',
            'NaiveBayes', 'NaiveBayesModel',
            'MultilayerPerceptronClassifier', 'MultilayerPerceptronClassificationModel',
+           'MultilayerPerceptronClassificationSummary',
+           'MultilayerPerceptronClassificationTrainingSummary',
            'OneVsRest', 'OneVsRestModel',
-           'FMClassifier', 'FMClassificationModel']
+           'FMClassifier', 'FMClassificationModel', 'FMClassificationSummary',
+           'FMClassificationTrainingSummary']
 
 
 class _ClassifierParams(HasRawPredictionCol, _PredictorParams):
@@ -61,14 +76,12 @@ class _ClassifierParams(HasRawPredictionCol, _PredictorParams):
 
 
 @inherit_doc
-class Classifier(Predictor, _ClassifierParams):
+class Classifier(Predictor, _ClassifierParams, metaclass=ABCMeta):
     """
     Classifier for classification tasks.
     Classes are indexed {0, 1, ..., numClasses - 1}.
     """
 
-    __metaclass__ = ABCMeta
-
     @since("3.0.0")
     def setRawPredictionCol(self, value):
         """
@@ -78,14 +91,12 @@ def setRawPredictionCol(self, value):
 
 
 @inherit_doc
-class ClassificationModel(PredictionModel, _ClassifierParams):
+class ClassificationModel(PredictionModel, _ClassifierParams, metaclass=ABCMeta):
     """
     Model produced by a ``Classifier``.
     Classes are indexed {0, 1, ..., numClasses - 1}.
     """
 
-    __metaclass__ = ABCMeta
-
     @since("3.0.0")
     def setRawPredictionCol(self, value):
         """
@@ -121,13 +132,12 @@ class _ProbabilisticClassifierParams(HasProbabilityCol, HasThresholds, _Classifi
 
 
 @inherit_doc
-class ProbabilisticClassifier(Classifier, _ProbabilisticClassifierParams):
+class ProbabilisticClassifier(Classifier, _ProbabilisticClassifierParams,
+                              metaclass=ABCMeta):
     """
     Probabilistic Classifier for classification tasks.
     """
 
-    __metaclass__ = ABCMeta
-
     @since("3.0.0")
     def setProbabilityCol(self, value):
         """
@@ -145,13 +155,12 @@ def setThresholds(self, value):
 
 @inherit_doc
 class ProbabilisticClassificationModel(ClassificationModel,
-                                       _ProbabilisticClassifierParams):
+                                       _ProbabilisticClassifierParams,
+                                       metaclass=ABCMeta):
     """
     Model produced by a ``ProbabilisticClassifier``.
     """
 
-    __metaclass__ = ABCMeta
-
     @since("3.0.0")
     def setProbabilityCol(self, value):
         """
@@ -176,14 +185,12 @@ def predictProbability(self, value):
 
 
 @inherit_doc
-class _JavaClassifier(Classifier, JavaPredictor):
+class _JavaClassifier(Classifier, JavaPredictor, metaclass=ABCMeta):
     """
     Java Classifier for classification tasks.
     Classes are indexed {0, 1, ..., numClasses - 1}.
     """
 
-    __metaclass__ = ABCMeta
-
     @since("3.0.0")
     def setRawPredictionCol(self, value):
         """
@@ -217,12 +224,12 @@ def predictRaw(self, value):
 
 
 @inherit_doc
-class _JavaProbabilisticClassifier(ProbabilisticClassifier, _JavaClassifier):
+class _JavaProbabilisticClassifier(ProbabilisticClassifier, _JavaClassifier,
+                                   metaclass=ABCMeta):
     """
     Java Probabilistic Classifier for classification tasks.
     """
-
-    __metaclass__ = ABCMeta
+    pass
 
 
 @inherit_doc
@@ -240,9 +247,267 @@ def predictProbability(self, value):
         return self._call_java("predictProbability", value)
 
 
+@inherit_doc
+class _ClassificationSummary(JavaWrapper):
+    """
+    Abstraction for multiclass classification results for a given model.
+
+    .. versionadded:: 3.1.0
+    """
+
+    @property
+    @since("3.1.0")
+    def predictions(self):
+        """
+        Dataframe outputted by the model's `transform` method.
+        """
+        return self._call_java("predictions")
+
+    @property
+    @since("3.1.0")
+    def predictionCol(self):
+        """
+        Field in "predictions" which gives the prediction of each class.
+        """
+        return self._call_java("predictionCol")
+
+    @property
+    @since("3.1.0")
+    def labelCol(self):
+        """
+        Field in "predictions" which gives the true label of each
+        instance.
+        """
+        return self._call_java("labelCol")
+
+    @property
+    @since("3.1.0")
+    def weightCol(self):
+        """
+        Field in "predictions" which gives the weight of each instance
+        as a vector.
+        """
+        return self._call_java("weightCol")
+
+    @property
+    def labels(self):
+        """
+        Returns the sequence of labels in ascending order. This order matches the order used
+        in metrics which are specified as arrays over labels, e.g., truePositiveRateByLabel.
+
+        .. versionadded:: 3.1.0
+
+        Notes
+        -----
+        In most cases, it will be values {0.0, 1.0, ..., numClasses-1}, However, if the
+        training set is missing a label, then all of the arrays over labels
+        (e.g., from truePositiveRateByLabel) will be of length numClasses-1 instead of the
+        expected numClasses.
+        """
+        return self._call_java("labels")
+
+    @property
+    @since("3.1.0")
+    def truePositiveRateByLabel(self):
+        """
+        Returns true positive rate for each label (category).
+        """
+        return self._call_java("truePositiveRateByLabel")
+
+    @property
+    @since("3.1.0")
+    def falsePositiveRateByLabel(self):
+        """
+        Returns false positive rate for each label (category).
+        """
+        return self._call_java("falsePositiveRateByLabel")
+
+    @property
+    @since("3.1.0")
+    def precisionByLabel(self):
+        """
+        Returns precision for each label (category).
+        """
+        return self._call_java("precisionByLabel")
+
+    @property
+    @since("3.1.0")
+    def recallByLabel(self):
+        """
+        Returns recall for each label (category).
+        """
+        return self._call_java("recallByLabel")
+
+    @since("3.1.0")
+    def fMeasureByLabel(self, beta=1.0):
+        """
+        Returns f-measure for each label (category).
+        """
+        return self._call_java("fMeasureByLabel", beta)
+
+    @property
+    @since("3.1.0")
+    def accuracy(self):
+        """
+        Returns accuracy.
+        (equals to the total number of correctly classified instances
+        out of the total number of instances.)
+        """
+        return self._call_java("accuracy")
+
+    @property
+    @since("3.1.0")
+    def weightedTruePositiveRate(self):
+        """
+        Returns weighted true positive rate.
+        (equals to precision, recall and f-measure)
+        """
+        return self._call_java("weightedTruePositiveRate")
+
+    @property
+    @since("3.1.0")
+    def weightedFalsePositiveRate(self):
+        """
+        Returns weighted false positive rate.
+        """
+        return self._call_java("weightedFalsePositiveRate")
+
+    @property
+    @since("3.1.0")
+    def weightedRecall(self):
+        """
+        Returns weighted averaged recall.
+        (equals to precision, recall and f-measure)
+        """
+        return self._call_java("weightedRecall")
+
+    @property
+    @since("3.1.0")
+    def weightedPrecision(self):
+        """
+        Returns weighted averaged precision.
+        """
+        return self._call_java("weightedPrecision")
+
+    @since("3.1.0")
+    def weightedFMeasure(self, beta=1.0):
+        """
+        Returns weighted averaged f-measure.
+        """
+        return self._call_java("weightedFMeasure", beta)
+
+
+@inherit_doc
+class _TrainingSummary(JavaWrapper):
+    """
+    Abstraction for Training results.
+
+    .. versionadded:: 3.1.0
+    """
+
+    @property
+    @since("3.1.0")
+    def objectiveHistory(self):
+        """
+        Objective function (scaled loss + regularization) at each
+        iteration. It contains one more element, the initial state,
+        than number of iterations.
+        """
+        return self._call_java("objectiveHistory")
+
+    @property
+    @since("3.1.0")
+    def totalIterations(self):
+        """
+        Number of training iterations until termination.
+        """
+        return self._call_java("totalIterations")
+
+
+@inherit_doc
+class _BinaryClassificationSummary(_ClassificationSummary):
+    """
+    Binary classification results for a given model.
+
+    .. versionadded:: 3.1.0
+    """
+
+    @property
+    @since("3.1.0")
+    def scoreCol(self):
+        """
+        Field in "predictions" which gives the probability or raw prediction
+        of each class as a vector.
+        """
+        return self._call_java("scoreCol")
+
+    @property
+    def roc(self):
+        """
+        Returns the receiver operating characteristic (ROC) curve,
+        which is a Dataframe having two fields (FPR, TPR) with
+        (0.0, 0.0) prepended and (1.0, 1.0) appended to it.
+
+        .. versionadded:: 3.1.0
+
+        Notes
+        -----
+        `Wikipedia reference <http://en.wikipedia.org/wiki/Receiver_operating_characteristic>`_
+        """
+        return self._call_java("roc")
+
+    @property
+    @since("3.1.0")
+    def areaUnderROC(self):
+        """
+        Computes the area under the receiver operating characteristic
+        (ROC) curve.
+        """
+        return self._call_java("areaUnderROC")
+
+    @property
+    @since("3.1.0")
+    def pr(self):
+        """
+        Returns the precision-recall curve, which is a Dataframe
+        containing two fields recall, precision with (0.0, 1.0) prepended
+        to it.
+        """
+        return self._call_java("pr")
+
+    @property
+    @since("3.1.0")
+    def fMeasureByThreshold(self):
+        """
+        Returns a dataframe with two fields (threshold, F-Measure) curve
+        with beta = 1.0.
+        """
+        return self._call_java("fMeasureByThreshold")
+
+    @property
+    @since("3.1.0")
+    def precisionByThreshold(self):
+        """
+        Returns a dataframe with two fields (threshold, precision) curve.
+        Every possible probability obtained in transforming the dataset
+        are used as thresholds used in calculating the precision.
+        """
+        return self._call_java("precisionByThreshold")
+
+    @property
+    @since("3.1.0")
+    def recallByThreshold(self):
+        """
+        Returns a dataframe with two fields (threshold, recall) curve.
+        Every possible probability obtained in transforming the dataset
+        are used as thresholds used in calculating the recall.
+        """
+        return self._call_java("recallByThreshold")
+
+
 class _LinearSVCParams(_ClassifierParams, HasRegParam, HasMaxIter, HasFitIntercept, HasTol,
                        HasStandardization, HasWeightCol, HasAggregationDepth, HasThreshold,
-                       HasBlockSize):
+                       HasMaxBlockSizeInMB):
     """
     Params for :py:class:`LinearSVC` and :py:class:`LinearSVCModel`.
 
@@ -255,15 +520,27 @@ class _LinearSVCParams(_ClassifierParams, HasRegParam, HasMaxIter, HasFitInterce
                       " all predictions 0.0 and -Inf will make all predictions 1.0.",
                       typeConverter=TypeConverters.toFloat)
 
+    def __init__(self, *args):
+        super(_LinearSVCParams, self).__init__(*args)
+        self._setDefault(maxIter=100, regParam=0.0, tol=1e-6, fitIntercept=True,
+                         standardization=True, threshold=0.0, aggregationDepth=2,
+                         maxBlockSizeInMB=0.0)
+
 
 @inherit_doc
 class LinearSVC(_JavaClassifier, _LinearSVCParams, JavaMLWritable, JavaMLReadable):
     """
-    `Linear SVM Classifier <https://en.wikipedia.org/wiki/Support_vector_machine#Linear_SVM>`_
-
     This binary classifier optimizes the Hinge Loss using the OWLQN optimizer.
     Only supports L2 regularization currently.
 
+    .. versionadded:: 2.2.0
+
+    Notes
+    -----
+    `Linear SVM Classifier <https://en.wikipedia.org/wiki/Support_vector_machine#Linear_SVM>`_
+
+    Examples
+    --------
     >>> from pyspark.sql import Row
     >>> from pyspark.ml.linalg import Vectors
     >>> df = sc.parallelize([
@@ -291,12 +568,12 @@ class LinearSVC(_JavaClassifier, _LinearSVCParams, JavaMLWritable, JavaMLReadabl
     LinearSVCModel...
     >>> model.getThreshold()
     0.5
-    >>> model.getBlockSize()
-    1
+    >>> model.getMaxBlockSizeInMB()
+    0.0
     >>> model.coefficients
-    DenseVector([0.0, -0.2792, -0.1833])
+    DenseVector([0.0, -1.0319, -0.5159])
     >>> model.intercept
-    1.0206118982229047
+    2.579645978780695
     >>> model.numClasses
     2
     >>> model.numFeatures
@@ -305,12 +582,12 @@ class LinearSVC(_JavaClassifier, _LinearSVCParams, JavaMLWritable, JavaMLReadabl
     >>> model.predict(test0.head().features)
     1.0
     >>> model.predictRaw(test0.head().features)
-    DenseVector([-1.4831, 1.4831])
+    DenseVector([-4.1274, 4.1274])
     >>> result = model.transform(test0).head()
     >>> result.newPrediction
     1.0
     >>> result.rawPrediction
-    DenseVector([-1.4831, 1.4831])
+    DenseVector([-4.1274, 4.1274])
     >>> svm_path = temp_path + "/svm"
     >>> svm.save(svm_path)
     >>> svm2 = LinearSVC.load(svm_path)
@@ -323,41 +600,38 @@ class LinearSVC(_JavaClassifier, _LinearSVCParams, JavaMLWritable, JavaMLReadabl
     True
     >>> model.intercept == model2.intercept
     True
-
-    .. versionadded:: 2.2.0
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     """
 
     @keyword_only
-    def __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def __init__(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                  maxIter=100, regParam=0.0, tol=1e-6, rawPredictionCol="rawPrediction",
                  fitIntercept=True, standardization=True, threshold=0.0, weightCol=None,
-                 aggregationDepth=2, blockSize=1):
+                 aggregationDepth=2, maxBlockSizeInMB=0.0):
         """
-        __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                  maxIter=100, regParam=0.0, tol=1e-6, rawPredictionCol="rawPrediction", \
                  fitIntercept=True, standardization=True, threshold=0.0, weightCol=None, \
-                 aggregationDepth=2, blockSize=1):
+                 aggregationDepth=2, maxBlockSizeInMB=0.0):
         """
         super(LinearSVC, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.LinearSVC", self.uid)
-        self._setDefault(maxIter=100, regParam=0.0, tol=1e-6, fitIntercept=True,
-                         standardization=True, threshold=0.0, aggregationDepth=2,
-                         blockSize=1)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("2.2.0")
-    def setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def setParams(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                   maxIter=100, regParam=0.0, tol=1e-6, rawPredictionCol="rawPrediction",
                   fitIntercept=True, standardization=True, threshold=0.0, weightCol=None,
-                  aggregationDepth=2, blockSize=1):
+                  aggregationDepth=2, maxBlockSizeInMB=0.0):
         """
-        setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                   maxIter=100, regParam=0.0, tol=1e-6, rawPredictionCol="rawPrediction", \
                   fitIntercept=True, standardization=True, threshold=0.0, weightCol=None, \
-                  aggregationDepth=2, blockSize=1):
+                  aggregationDepth=2, maxBlockSizeInMB=0.0):
         Sets params for Linear SVM Classifier.
         """
         kwargs = self._input_kwargs
@@ -423,48 +697,96 @@ def setAggregationDepth(self, value):
         return self._set(aggregationDepth=value)
 
     @since("3.1.0")
-    def setBlockSize(self, value):
+    def setMaxBlockSizeInMB(self, value):
         """
-        Sets the value of :py:attr:`blockSize`.
+        Sets the value of :py:attr:`maxBlockSizeInMB`.
         """
-        return self._set(blockSize=value)
+        return self._set(maxBlockSizeInMB=value)
 
 
-class LinearSVCModel(_JavaClassificationModel, _LinearSVCParams, JavaMLWritable, JavaMLReadable):
+class LinearSVCModel(_JavaClassificationModel, _LinearSVCParams, JavaMLWritable, JavaMLReadable,
+                     HasTrainingSummary):
     """
     Model fitted by LinearSVC.
 
-    .. versionadded:: 2.2.0
+    .. versionadded:: 2.2.0
+    """
+
+    @since("3.0.0")
+    def setThreshold(self, value):
+        """
+        Sets the value of :py:attr:`threshold`.
+        """
+        return self._set(threshold=value)
+
+    @property
+    @since("2.2.0")
+    def coefficients(self):
+        """
+        Model coefficients of Linear SVM Classifier.
+        """
+        return self._call_java("coefficients")
+
+    @property
+    @since("2.2.0")
+    def intercept(self):
+        """
+        Model intercept of Linear SVM Classifier.
+        """
+        return self._call_java("intercept")
+
+    @since("3.1.0")
+    def summary(self):
+        """
+        Gets summary (accuracy/precision/recall, objective history, total iterations) of model
+        trained on the training set. An exception is thrown if `trainingSummary is None`.
+        """
+        if self.hasSummary:
+            return LinearSVCTrainingSummary(super(LinearSVCModel, self).summary)
+        else:
+            raise RuntimeError("No training summary available for this %s" %
+                               self.__class__.__name__)
+
+    def evaluate(self, dataset):
+        """
+        Evaluates the model on a test dataset.
+
+        .. versionadded:: 3.1.0
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            Test dataset to evaluate model on.
+        """
+        if not isinstance(dataset, DataFrame):
+            raise TypeError("dataset must be a DataFrame but got %s." % type(dataset))
+        java_lsvc_summary = self._call_java("evaluate", dataset)
+        return LinearSVCSummary(java_lsvc_summary)
+
+
+class LinearSVCSummary(_BinaryClassificationSummary):
+    """
+    Abstraction for LinearSVC Results for a given model.
+
+    .. versionadded:: 3.1.0
     """
+    pass
 
-    @since("3.0.0")
-    def setThreshold(self, value):
-        """
-        Sets the value of :py:attr:`threshold`.
-        """
-        return self._set(threshold=value)
 
-    @property
-    @since("2.2.0")
-    def coefficients(self):
-        """
-        Model coefficients of Linear SVM Classifier.
-        """
-        return self._call_java("coefficients")
+@inherit_doc
+class LinearSVCTrainingSummary(LinearSVCSummary, _TrainingSummary):
+    """
+    Abstraction for LinearSVC Training results.
 
-    @property
-    @since("2.2.0")
-    def intercept(self):
-        """
-        Model intercept of Linear SVM Classifier.
-        """
-        return self._call_java("intercept")
+    .. versionadded:: 3.1.0
+    """
+    pass
 
 
 class _LogisticRegressionParams(_ProbabilisticClassifierParams, HasRegParam,
                                 HasElasticNetParam, HasMaxIter, HasFitIntercept, HasTol,
                                 HasStandardization, HasWeightCol, HasAggregationDepth,
-                                HasThreshold, HasBlockSize):
+                                HasThreshold, HasMaxBlockSizeInMB):
     """
     Params for :py:class:`LogisticRegression` and :py:class:`LogisticRegressionModel`.
 
@@ -514,6 +836,11 @@ class _LogisticRegressionParams(_ProbabilisticClassifierParams, HasRegParam,
                                     "classes for multinomial regression.",
                                     typeConverter=TypeConverters.toVector)
 
+    def __init__(self, *args):
+        super(_LogisticRegressionParams, self).__init__(*args)
+        self._setDefault(maxIter=100, regParam=0.0, tol=1E-6, threshold=0.5, family="auto",
+                         maxBlockSizeInMB=0.0)
+
     @since("1.4.0")
     def setThreshold(self, value):
         """
@@ -626,6 +953,10 @@ class LogisticRegression(_JavaProbabilisticClassifier, _LogisticRegressionParams
     Logistic regression.
     This class supports multinomial logistic (softmax) and binomial logistic regression.
 
+    .. versionadded:: 1.3.0
+
+    Examples
+    --------
     >>> from pyspark.sql import Row
     >>> from pyspark.ml.linalg import Vectors
     >>> bdf = sc.parallelize([
@@ -652,8 +983,8 @@ class LogisticRegression(_JavaProbabilisticClassifier, _LogisticRegressionParams
     LogisticRegressionModel...
     >>> blorModel.getProbabilityCol()
     'newProbability'
-    >>> blorModel.getBlockSize()
-    1
+    >>> blorModel.getMaxBlockSizeInMB()
+    0.0
     >>> blorModel.setThreshold(0.1)
     LogisticRegressionModel...
     >>> blorModel.getThreshold()
@@ -662,6 +993,8 @@ class LogisticRegression(_JavaProbabilisticClassifier, _LogisticRegressionParams
     DenseVector([-1.080..., -0.646...])
     >>> blorModel.intercept
     3.112...
+    >>> blorModel.evaluate(bdf).accuracy == blorModel.summary.accuracy
+    True
     >>> data_path = "data/mllib/sample_multiclass_classification_data.txt"
     >>> mdf = spark.read.format("libsvm").load(data_path)
     >>> mlor = LogisticRegression(regParam=0.1, elasticNetParam=1.0, family="multinomial")
@@ -705,59 +1038,57 @@ class LogisticRegression(_JavaProbabilisticClassifier, _LogisticRegressionParams
     True
     >>> model2
     LogisticRegressionModel: uid=..., numClasses=2, numFeatures=2
-
-    .. versionadded:: 1.3.0
+    >>> blorModel.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     """
 
     @keyword_only
-    def __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def __init__(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                  maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True,
                  threshold=0.5, thresholds=None, probabilityCol="probability",
                  rawPredictionCol="rawPrediction", standardization=True, weightCol=None,
                  aggregationDepth=2, family="auto",
                  lowerBoundsOnCoefficients=None, upperBoundsOnCoefficients=None,
                  lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None,
-                 blockSize=1):
+                 maxBlockSizeInMB=0.0):
 
         """
-        __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                  maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True, \
                  threshold=0.5, thresholds=None, probabilityCol="probability", \
                  rawPredictionCol="rawPrediction", standardization=True, weightCol=None, \
                  aggregationDepth=2, family="auto", \
                  lowerBoundsOnCoefficients=None, upperBoundsOnCoefficients=None, \
                  lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None, \
-                 blockSize=1):
+                 maxBlockSizeInMB=0.0):
         If the threshold and thresholds Params are both set, they must be equivalent.
         """
         super(LogisticRegression, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.LogisticRegression", self.uid)
-        self._setDefault(maxIter=100, regParam=0.0, tol=1E-6, threshold=0.5, family="auto",
-                         blockSize=1)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
         self._checkThresholdConsistency()
 
     @keyword_only
     @since("1.3.0")
-    def setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def setParams(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                   maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True,
                   threshold=0.5, thresholds=None, probabilityCol="probability",
                   rawPredictionCol="rawPrediction", standardization=True, weightCol=None,
                   aggregationDepth=2, family="auto",
                   lowerBoundsOnCoefficients=None, upperBoundsOnCoefficients=None,
                   lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None,
-                  blockSize=1):
+                  maxBlockSizeInMB=0.0):
         """
-        setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                   maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True, \
                   threshold=0.5, thresholds=None, probabilityCol="probability", \
                   rawPredictionCol="rawPrediction", standardization=True, weightCol=None, \
                   aggregationDepth=2, family="auto", \
                   lowerBoundsOnCoefficients=None, upperBoundsOnCoefficients=None, \
                   lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None, \
-                  blockSize=1):
+                  maxBlockSizeInMB=0.0):
         Sets params for logistic regression.
         If the threshold and thresholds Params are both set, they must be equivalent.
         """
@@ -853,11 +1184,11 @@ def setAggregationDepth(self, value):
         return self._set(aggregationDepth=value)
 
     @since("3.1.0")
-    def setBlockSize(self, value):
+    def setMaxBlockSizeInMB(self, value):
         """
-        Sets the value of :py:attr:`blockSize`.
+        Sets the value of :py:attr:`maxBlockSizeInMB`.
         """
-        return self._set(blockSize=value)
+        return self._set(maxBlockSizeInMB=value)
 
 
 class LogisticRegressionModel(_JavaProbabilisticClassificationModel, _LogisticRegressionParams,
@@ -906,7 +1237,7 @@ def interceptVector(self):
     @since("2.0.0")
     def summary(self):
         """
-        Gets summary (e.g. accuracy/precision/recall, objective history, total iterations) of model
+        Gets summary (accuracy/precision/recall, objective history, total iterations) of model
         trained on the training set. An exception is thrown if `trainingSummary is None`.
         """
         if self.hasSummary:
@@ -920,17 +1251,19 @@ def summary(self):
             raise RuntimeError("No training summary available for this %s" %
                                self.__class__.__name__)
 
-    @since("2.0.0")
     def evaluate(self, dataset):
         """
         Evaluates the model on a test dataset.
 
-        :param dataset:
-          Test dataset to evaluate model on, where dataset is an
-          instance of :py:class:`pyspark.sql.DataFrame`
+        .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            Test dataset to evaluate model on.
         """
         if not isinstance(dataset, DataFrame):
-            raise ValueError("dataset must be a DataFrame but got %s." % type(dataset))
+            raise TypeError("dataset must be a DataFrame but got %s." % type(dataset))
         java_blr_summary = self._call_java("evaluate", dataset)
         if self.numClasses <= 2:
             return BinaryLogisticRegressionSummary(java_blr_summary)
@@ -938,21 +1271,13 @@ def evaluate(self, dataset):
             return LogisticRegressionSummary(java_blr_summary)
 
 
-class LogisticRegressionSummary(JavaWrapper):
+class LogisticRegressionSummary(_ClassificationSummary):
     """
     Abstraction for Logistic Regression Results for a given model.
 
     .. versionadded:: 2.0.0
     """
 
-    @property
-    @since("2.0.0")
-    def predictions(self):
-        """
-        Dataframe outputted by the model's `transform` method.
-        """
-        return self._call_java("predictions")
-
     @property
     @since("2.0.0")
     def probabilityCol(self):
@@ -962,23 +1287,6 @@ def probabilityCol(self):
         """
         return self._call_java("probabilityCol")
 
-    @property
-    @since("2.3.0")
-    def predictionCol(self):
-        """
-        Field in "predictions" which gives the prediction of each class.
-        """
-        return self._call_java("predictionCol")
-
-    @property
-    @since("2.0.0")
-    def labelCol(self):
-        """
-        Field in "predictions" which gives the true label of each
-        instance.
-        """
-        return self._call_java("labelCol")
-
     @property
     @since("2.0.0")
     def featuresCol(self):
@@ -988,231 +1296,26 @@ def featuresCol(self):
         """
         return self._call_java("featuresCol")
 
-    @property
-    @since("2.3.0")
-    def labels(self):
-        """
-        Returns the sequence of labels in ascending order. This order matches the order used
-        in metrics which are specified as arrays over labels, e.g., truePositiveRateByLabel.
-
-        Note: In most cases, it will be values {0.0, 1.0, ..., numClasses-1}, However, if the
-        training set is missing a label, then all of the arrays over labels
-        (e.g., from truePositiveRateByLabel) will be of length numClasses-1 instead of the
-        expected numClasses.
-        """
-        return self._call_java("labels")
-
-    @property
-    @since("2.3.0")
-    def truePositiveRateByLabel(self):
-        """
-        Returns true positive rate for each label (category).
-        """
-        return self._call_java("truePositiveRateByLabel")
-
-    @property
-    @since("2.3.0")
-    def falsePositiveRateByLabel(self):
-        """
-        Returns false positive rate for each label (category).
-        """
-        return self._call_java("falsePositiveRateByLabel")
-
-    @property
-    @since("2.3.0")
-    def precisionByLabel(self):
-        """
-        Returns precision for each label (category).
-        """
-        return self._call_java("precisionByLabel")
-
-    @property
-    @since("2.3.0")
-    def recallByLabel(self):
-        """
-        Returns recall for each label (category).
-        """
-        return self._call_java("recallByLabel")
-
-    @since("2.3.0")
-    def fMeasureByLabel(self, beta=1.0):
-        """
-        Returns f-measure for each label (category).
-        """
-        return self._call_java("fMeasureByLabel", beta)
-
-    @property
-    @since("2.3.0")
-    def accuracy(self):
-        """
-        Returns accuracy.
-        (equals to the total number of correctly classified instances
-        out of the total number of instances.)
-        """
-        return self._call_java("accuracy")
-
-    @property
-    @since("2.3.0")
-    def weightedTruePositiveRate(self):
-        """
-        Returns weighted true positive rate.
-        (equals to precision, recall and f-measure)
-        """
-        return self._call_java("weightedTruePositiveRate")
-
-    @property
-    @since("2.3.0")
-    def weightedFalsePositiveRate(self):
-        """
-        Returns weighted false positive rate.
-        """
-        return self._call_java("weightedFalsePositiveRate")
-
-    @property
-    @since("2.3.0")
-    def weightedRecall(self):
-        """
-        Returns weighted averaged recall.
-        (equals to precision, recall and f-measure)
-        """
-        return self._call_java("weightedRecall")
-
-    @property
-    @since("2.3.0")
-    def weightedPrecision(self):
-        """
-        Returns weighted averaged precision.
-        """
-        return self._call_java("weightedPrecision")
-
-    @since("2.3.0")
-    def weightedFMeasure(self, beta=1.0):
-        """
-        Returns weighted averaged f-measure.
-        """
-        return self._call_java("weightedFMeasure", beta)
-
 
 @inherit_doc
-class LogisticRegressionTrainingSummary(LogisticRegressionSummary):
+class LogisticRegressionTrainingSummary(LogisticRegressionSummary, _TrainingSummary):
     """
     Abstraction for multinomial Logistic Regression Training results.
-    Currently, the training summary ignores the training weights except
-    for the objective trace.
-
-    .. versionadded:: 2.0.0
-    """
-
-    @property
-    @since("2.0.0")
-    def objectiveHistory(self):
-        """
-        Objective function (scaled loss + regularization) at each
-        iteration.
-        """
-        return self._call_java("objectiveHistory")
-
-    @property
-    @since("2.0.0")
-    def totalIterations(self):
-        """
-        Number of training iterations until termination.
-        """
-        return self._call_java("totalIterations")
-
-
-@inherit_doc
-class BinaryLogisticRegressionSummary(LogisticRegressionSummary):
-    """
-    Binary Logistic regression results for a given model.
 
     .. versionadded:: 2.0.0
     """
+    pass
 
-    @property
-    @since("2.0.0")
-    def roc(self):
-        """
-        Returns the receiver operating characteristic (ROC) curve,
-        which is a Dataframe having two fields (FPR, TPR) with
-        (0.0, 0.0) prepended and (1.0, 1.0) appended to it.
-
-        .. seealso:: `Wikipedia reference
-            <http://en.wikipedia.org/wiki/Receiver_operating_characteristic>`_
-
-        .. note:: This ignores instance weights (setting all to 1.0) from
-            `LogisticRegression.weightCol`. This will change in later Spark
-            versions.
-        """
-        return self._call_java("roc")
-
-    @property
-    @since("2.0.0")
-    def areaUnderROC(self):
-        """
-        Computes the area under the receiver operating characteristic
-        (ROC) curve.
-
-        .. note:: This ignores instance weights (setting all to 1.0) from
-            `LogisticRegression.weightCol`. This will change in later Spark
-            versions.
-        """
-        return self._call_java("areaUnderROC")
-
-    @property
-    @since("2.0.0")
-    def pr(self):
-        """
-        Returns the precision-recall curve, which is a Dataframe
-        containing two fields recall, precision with (0.0, 1.0) prepended
-        to it.
-
-        .. note:: This ignores instance weights (setting all to 1.0) from
-            `LogisticRegression.weightCol`. This will change in later Spark
-            versions.
-        """
-        return self._call_java("pr")
-
-    @property
-    @since("2.0.0")
-    def fMeasureByThreshold(self):
-        """
-        Returns a dataframe with two fields (threshold, F-Measure) curve
-        with beta = 1.0.
-
-        .. note:: This ignores instance weights (setting all to 1.0) from
-            `LogisticRegression.weightCol`. This will change in later Spark
-            versions.
-        """
-        return self._call_java("fMeasureByThreshold")
-
-    @property
-    @since("2.0.0")
-    def precisionByThreshold(self):
-        """
-        Returns a dataframe with two fields (threshold, precision) curve.
-        Every possible probability obtained in transforming the dataset
-        are used as thresholds used in calculating the precision.
-
-        .. note:: This ignores instance weights (setting all to 1.0) from
-            `LogisticRegression.weightCol`. This will change in later Spark
-            versions.
-        """
-        return self._call_java("precisionByThreshold")
 
-    @property
-    @since("2.0.0")
-    def recallByThreshold(self):
-        """
-        Returns a dataframe with two fields (threshold, recall) curve.
-        Every possible probability obtained in transforming the dataset
-        are used as thresholds used in calculating the recall.
+@inherit_doc
+class BinaryLogisticRegressionSummary(_BinaryClassificationSummary,
+                                      LogisticRegressionSummary):
+    """
+    Binary Logistic regression results for a given model.
 
-        .. note:: This ignores instance weights (setting all to 1.0) from
-            `LogisticRegression.weightCol`. This will change in later Spark
-            versions.
-        """
-        return self._call_java("recallByThreshold")
+    .. versionadded:: 2.0.0
+    """
+    pass
 
 
 @inherit_doc
@@ -1231,7 +1334,12 @@ class _DecisionTreeClassifierParams(_DecisionTreeParams, _TreeClassifierParams):
     """
     Params for :py:class:`DecisionTreeClassifier` and :py:class:`DecisionTreeClassificationModel`.
     """
-    pass
+
+    def __init__(self, *args):
+        super(_DecisionTreeClassifierParams, self).__init__(*args)
+        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
+                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
+                         impurity="gini", leafCol="", minWeightFractionPerNode=0.0)
 
 
 @inherit_doc
@@ -1243,6 +1351,10 @@ class DecisionTreeClassifier(_JavaProbabilisticClassifier, _DecisionTreeClassifi
     It supports both binary and multiclass labels, as well as both continuous and categorical
     features.
 
+    .. versionadded:: 1.4.0
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> from pyspark.ml.feature import StringIndexer
     >>> df = spark.createDataFrame([
@@ -1298,7 +1410,8 @@ class DecisionTreeClassifier(_JavaProbabilisticClassifier, _DecisionTreeClassifi
     >>> model2 = DecisionTreeClassificationModel.load(model_path)
     >>> model.featureImportances == model2.featureImportances
     True
-
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     >>> df3 = spark.createDataFrame([
     ...     (1.0, 0.2, Vectors.dense(1.0)),
     ...     (1.0, 0.8, Vectors.dense(1.0)),
@@ -1310,18 +1423,16 @@ class DecisionTreeClassifier(_JavaProbabilisticClassifier, _DecisionTreeClassifi
     >>> model3 = dt3.fit(td3)
     >>> print(model3.toDebugString)
     DecisionTreeClassificationModel...depth=1, numNodes=3...
-
-    .. versionadded:: 1.4.0
     """
 
     @keyword_only
-    def __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def __init__(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                  probabilityCol="probability", rawPredictionCol="rawPrediction",
                  maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
                  maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, impurity="gini",
                  seed=None, weightCol=None, leafCol="", minWeightFractionPerNode=0.0):
         """
-        __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                  probabilityCol="probability", rawPredictionCol="rawPrediction", \
                  maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \
                  maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, impurity="gini", \
@@ -1330,22 +1441,19 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(DecisionTreeClassifier, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.DecisionTreeClassifier", self.uid)
-        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
-                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
-                         impurity="gini", leafCol="", minWeightFractionPerNode=0.0)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("1.4.0")
-    def setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def setParams(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                   probabilityCol="probability", rawPredictionCol="rawPrediction",
                   maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
                   maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
                   impurity="gini", seed=None, weightCol=None, leafCol="",
                   minWeightFractionPerNode=0.0):
         """
-        setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                   probabilityCol="probability", rawPredictionCol="rawPrediction", \
                   maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \
                   maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, impurity="gini", \
@@ -1440,7 +1548,6 @@ class DecisionTreeClassificationModel(_DecisionTreeModel, _JavaProbabilisticClas
     """
 
     @property
-    @since("2.0.0")
     def featureImportances(self):
         """
         Estimate of the importance of each feature.
@@ -1454,9 +1561,13 @@ def featureImportances(self):
             where gain is scaled by the number of instances passing through node
           - Normalize importances for tree to sum to 1.
 
-        .. note:: Feature importance for single decision trees can have high variance due to
-            correlated predictor variables. Consider using a :py:class:`RandomForestClassifier`
-            to determine feature importance instead.
+        .. versionadded:: 2.0.0
+
+        Notes
+        -----
+        Feature importance for single decision trees can have high variance due to
+        correlated predictor variables. Consider using a :py:class:`RandomForestClassifier`
+        to determine feature importance instead.
         """
         return self._call_java("featureImportances")
 
@@ -1466,7 +1577,14 @@ class _RandomForestClassifierParams(_RandomForestParams, _TreeClassifierParams):
     """
     Params for :py:class:`RandomForestClassifier` and :py:class:`RandomForestClassificationModel`.
     """
-    pass
+
+    def __init__(self, *args):
+        super(_RandomForestClassifierParams, self).__init__(*args)
+        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
+                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
+                         impurity="gini", numTrees=20, featureSubsetStrategy="auto",
+                         subsamplingRate=1.0, leafCol="", minWeightFractionPerNode=0.0,
+                         bootstrap=True)
 
 
 @inherit_doc
@@ -1478,6 +1596,10 @@ class RandomForestClassifier(_JavaProbabilisticClassifier, _RandomForestClassifi
     It supports both binary and multiclass labels, as well as both continuous and categorical
     features.
 
+    .. versionadded:: 1.4.0
+
+    Examples
+    --------
     >>> import numpy
     >>> from numpy import allclose
     >>> from pyspark.ml.linalg import Vectors
@@ -1538,19 +1660,19 @@ class RandomForestClassifier(_JavaProbabilisticClassifier, _RandomForestClassifi
     >>> model2 = RandomForestClassificationModel.load(model_path)
     >>> model.featureImportances == model2.featureImportances
     True
-
-    .. versionadded:: 1.4.0
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     """
 
     @keyword_only
-    def __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def __init__(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                  probabilityCol="probability", rawPredictionCol="rawPrediction",
                  maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
                  maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, impurity="gini",
                  numTrees=20, featureSubsetStrategy="auto", seed=None, subsamplingRate=1.0,
                  leafCol="", minWeightFractionPerNode=0.0, weightCol=None, bootstrap=True):
         """
-        __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                  probabilityCol="probability", rawPredictionCol="rawPrediction", \
                  maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \
                  maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, impurity="gini", \
@@ -1560,17 +1682,12 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(RandomForestClassifier, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.RandomForestClassifier", self.uid)
-        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
-                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
-                         impurity="gini", numTrees=20, featureSubsetStrategy="auto",
-                         subsamplingRate=1.0, leafCol="", minWeightFractionPerNode=0.0,
-                         bootstrap=True)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("1.4.0")
-    def setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def setParams(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                   probabilityCol="probability", rawPredictionCol="rawPrediction",
                   maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
                   maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, seed=None,
@@ -1691,7 +1808,7 @@ def setMinWeightFractionPerNode(self, value):
 
 class RandomForestClassificationModel(_TreeEnsembleModel, _JavaProbabilisticClassificationModel,
                                       _RandomForestClassifierParams, JavaMLWritable,
-                                      JavaMLReadable):
+                                      JavaMLReadable, HasTrainingSummary):
     """
     Model fitted by RandomForestClassifier.
 
@@ -1699,7 +1816,6 @@ class RandomForestClassificationModel(_TreeEnsembleModel, _JavaProbabilisticClas
     """
 
     @property
-    @since("2.0.0")
     def featureImportances(self):
         """
         Estimate of the importance of each feature.
@@ -1709,7 +1825,11 @@ def featureImportances(self):
         (Hastie, Tibshirani, Friedman. "The Elements of Statistical Learning, 2nd Edition." 2001.)
         and follows the implementation from scikit-learn.
 
-        .. seealso:: :py:attr:`DecisionTreeClassificationModel.featureImportances`
+        .. versionadded:: 2.0.0
+
+        See Also
+        --------
+        DecisionTreeClassificationModel.featureImportances
         """
         return self._call_java("featureImportances")
 
@@ -1719,6 +1839,84 @@ def trees(self):
         """Trees in this ensemble. Warning: These have null parent Estimators."""
         return [DecisionTreeClassificationModel(m) for m in list(self._call_java("trees"))]
 
+    @property
+    @since("3.1.0")
+    def summary(self):
+        """
+        Gets summary (accuracy/precision/recall, objective history, total iterations) of model
+        trained on the training set. An exception is thrown if `trainingSummary is None`.
+        """
+        if self.hasSummary:
+            if self.numClasses <= 2:
+                return BinaryRandomForestClassificationTrainingSummary(
+                    super(RandomForestClassificationModel, self).summary)
+            else:
+                return RandomForestClassificationTrainingSummary(
+                    super(RandomForestClassificationModel, self).summary)
+        else:
+            raise RuntimeError("No training summary available for this %s" %
+                               self.__class__.__name__)
+
+    def evaluate(self, dataset):
+        """
+        Evaluates the model on a test dataset.
+
+        .. versionadded:: 3.1.0
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            Test dataset to evaluate model on.
+        """
+        if not isinstance(dataset, DataFrame):
+            raise TypeError("dataset must be a DataFrame but got %s." % type(dataset))
+        java_rf_summary = self._call_java("evaluate", dataset)
+        if self.numClasses <= 2:
+            return BinaryRandomForestClassificationSummary(java_rf_summary)
+        else:
+            return RandomForestClassificationSummary(java_rf_summary)
+
+
+class RandomForestClassificationSummary(_ClassificationSummary):
+    """
+    Abstraction for RandomForestClassification Results for a given model.
+
+    .. versionadded:: 3.1.0
+    """
+    pass
+
+
+@inherit_doc
+class RandomForestClassificationTrainingSummary(RandomForestClassificationSummary,
+                                                _TrainingSummary):
+    """
+    Abstraction for RandomForestClassificationTraining Training results.
+
+    .. versionadded:: 3.1.0
+    """
+    pass
+
+
+@inherit_doc
+class BinaryRandomForestClassificationSummary(_BinaryClassificationSummary):
+    """
+    BinaryRandomForestClassification results for a given model.
+
+    .. versionadded:: 3.1.0
+    """
+    pass
+
+
+@inherit_doc
+class BinaryRandomForestClassificationTrainingSummary(BinaryRandomForestClassificationSummary,
+                                                      RandomForestClassificationTrainingSummary):
+    """
+    BinaryRandomForestClassification training results for a given model.
+
+    .. versionadded:: 3.1.0
+    """
+    pass
+
 
 class _GBTClassifierParams(_GBTParams, _HasVarianceImpurity):
     """
@@ -1734,6 +1932,14 @@ class _GBTClassifierParams(_GBTParams, _HasVarianceImpurity):
                      "Supported options: " + ", ".join(supportedLossTypes),
                      typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_GBTClassifierParams, self).__init__(*args)
+        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
+                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
+                         lossType="logistic", maxIter=20, stepSize=0.1, subsamplingRate=1.0,
+                         impurity="variance", featureSubsetStrategy="all", validationTol=0.01,
+                         leafCol="", minWeightFractionPerNode=0.0)
+
     @since("1.4.0")
     def getLossType(self):
         """
@@ -1750,18 +1956,25 @@ class GBTClassifier(_JavaProbabilisticClassifier, _GBTClassifierParams,
     learning algorithm for classification.
     It supports binary labels, as well as both continuous and categorical features.
 
+    .. versionadded:: 1.4.0
+
+    Notes
+    -----
+    Multiclass labels are not currently supported.
+
     The implementation is based upon: J.H. Friedman. "Stochastic Gradient Boosting." 1999.
 
-    Notes on Gradient Boosting vs. TreeBoost:
+    Gradient Boosting vs. TreeBoost:
+
     - This implementation is for Stochastic Gradient Boosting, not for TreeBoost.
     - Both algorithms learn tree ensembles by minimizing loss functions.
     - TreeBoost (Friedman, 1999) additionally modifies the outputs at tree leaf nodes
-    based on the loss function, whereas the original gradient boosting method does not.
+      based on the loss function, whereas the original gradient boosting method does not.
     - We expect to implement TreeBoost in the future:
-    `SPARK-4240 <https://issues.apache.org/jira/browse/SPARK-4240>`_
-
-    .. note:: Multiclass labels are not currently supported.
+      `SPARK-4240 <https://issues.apache.org/jira/browse/SPARK-4240>`_
 
+    Examples
+    --------
     >>> from numpy import allclose
     >>> from pyspark.ml.linalg import Vectors
     >>> from pyspark.ml.feature import StringIndexer
@@ -1825,6 +2038,8 @@ class GBTClassifier(_JavaProbabilisticClassifier, _GBTClassifierParams,
     True
     >>> model.treeWeights == model2.treeWeights
     True
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     >>> model.trees
     [DecisionTreeRegressionModel...depth=..., DecisionTreeRegressionModel...]
     >>> validation = spark.createDataFrame([(0.0, Vectors.dense(-1.0),)],
@@ -1838,19 +2053,17 @@ class GBTClassifier(_JavaProbabilisticClassifier, _GBTClassifierParams,
     'validationIndicator'
     >>> gbt.getValidationTol()
     0.01
-
-    .. versionadded:: 1.4.0
     """
 
     @keyword_only
-    def __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def __init__(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                  maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
                  maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, lossType="logistic",
                  maxIter=20, stepSize=0.1, seed=None, subsamplingRate=1.0, impurity="variance",
                  featureSubsetStrategy="all", validationTol=0.01, validationIndicatorCol=None,
                  leafCol="", minWeightFractionPerNode=0.0, weightCol=None):
         """
-        __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                  maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \
                  maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, \
                  lossType="logistic", maxIter=20, stepSize=0.1, seed=None, subsamplingRate=1.0, \
@@ -1861,17 +2074,12 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(GBTClassifier, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.GBTClassifier", self.uid)
-        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
-                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
-                         lossType="logistic", maxIter=20, stepSize=0.1, subsamplingRate=1.0,
-                         impurity="variance", featureSubsetStrategy="all", validationTol=0.01,
-                         leafCol="", minWeightFractionPerNode=0.0)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("1.4.0")
-    def setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def setParams(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                   maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
                   maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
                   lossType="logistic", maxIter=20, stepSize=0.1, seed=None, subsamplingRate=1.0,
@@ -1879,7 +2087,7 @@ def setParams(self, featuresCol="features", labelCol="label", predictionCol="pre
                   validationIndicatorCol=None, leafCol="", minWeightFractionPerNode=0.0,
                   weightCol=None):
         """
-        setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                   maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \
                   maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, \
                   lossType="logistic", maxIter=20, stepSize=0.1, seed=None, subsamplingRate=1.0, \
@@ -2017,7 +2225,6 @@ class GBTClassificationModel(_TreeEnsembleModel, _JavaProbabilisticClassificatio
     """
 
     @property
-    @since("2.0.0")
     def featureImportances(self):
         """
         Estimate of the importance of each feature.
@@ -2027,7 +2234,11 @@ def featureImportances(self):
         (Hastie, Tibshirani, Friedman. "The Elements of Statistical Learning, 2nd Edition." 2001.)
         and follows the implementation from scikit-learn.
 
-        .. seealso:: :py:attr:`DecisionTreeClassificationModel.featureImportances`
+        .. versionadded:: 2.0.0
+
+        See Also
+        --------
+        DecisionTreeClassificationModel.featureImportances
         """
         return self._call_java("featureImportances")
 
@@ -2037,14 +2248,16 @@ def trees(self):
         """Trees in this ensemble. Warning: These have null parent Estimators."""
         return [DecisionTreeRegressionModel(m) for m in list(self._call_java("trees"))]
 
-    @since("2.4.0")
     def evaluateEachIteration(self, dataset):
         """
         Method to compute error or loss for every iteration of gradient boosting.
 
-        :param dataset:
-            Test dataset to evaluate model on, where dataset is an
-            instance of :py:class:`pyspark.sql.DataFrame`
+        .. versionadded:: 2.4.0
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            Test dataset to evaluate model on.
         """
         return self._call_java("evaluateEachIteration", dataset)
 
@@ -2063,6 +2276,10 @@ class _NaiveBayesParams(_PredictorParams, HasWeightCol):
                       "and gaussian.",
                       typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_NaiveBayesParams, self).__init__(*args)
+        self._setDefault(smoothing=1.0, modelType="multinomial")
+
     @since("1.5.0")
     def getSmoothing(self):
         """
@@ -2083,22 +2300,27 @@ class NaiveBayes(_JavaProbabilisticClassifier, _NaiveBayesParams, HasThresholds,
                  JavaMLWritable, JavaMLReadable):
     """
     Naive Bayes Classifiers.
-    It supports both Multinomial and Bernoulli NB. `Multinomial NB
+    It supports both Multinomial and Bernoulli NB. `Multinomial NB \
     <http://nlp.stanford.edu/IR-book/html/htmledition/naive-bayes-text-classification-1.html>`_
     can handle finitely supported discrete data. For example, by converting documents into
     TF-IDF vectors, it can be used for document classification. By making every vector a
-    binary (0/1) data, it can also be used as `Bernoulli NB
+    binary (0/1) data, it can also be used as `Bernoulli NB \
     <http://nlp.stanford.edu/IR-book/html/htmledition/the-bernoulli-model-1.html>`_.
+
     The input feature values for Multinomial NB and Bernoulli NB must be nonnegative.
     Since 3.0.0, it supports Complement NB which is an adaptation of the Multinomial NB.
     Specifically, Complement NB uses statistics from the complement of each class to compute
     the model's coefficients. The inventors of Complement NB show empirically that the parameter
     estimates for CNB are more stable than those for Multinomial NB. Like Multinomial NB, the
     input feature values for Complement NB must be nonnegative.
-    Since 3.0.0, it also supports Gaussian NB
+    Since 3.0.0, it also supports `Gaussian NB \
     <https://en.wikipedia.org/wiki/Naive_Bayes_classifier#Gaussian_naive_Bayes>`_.
     which can handle continuous data.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> from pyspark.sql import Row
     >>> from pyspark.ml.linalg import Vectors
     >>> df = spark.createDataFrame([
@@ -2146,6 +2368,8 @@ class NaiveBayes(_JavaProbabilisticClassifier, _NaiveBayesParams, HasThresholds,
     True
     >>> model.theta == model2.theta
     True
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     >>> nb = nb.setThresholds([0.01, 10.00])
     >>> model3 = nb.fit(df)
     >>> result = model3.transform(test0).head()
@@ -2165,33 +2389,30 @@ class NaiveBayes(_JavaProbabilisticClassifier, _NaiveBayesParams, HasThresholds,
     DenseMatrix(2, 2, [...], 1)
     >>> model5.sigma
     DenseMatrix(0, 0, [...], ...)
-
-    .. versionadded:: 1.5.0
     """
 
     @keyword_only
-    def __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def __init__(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                  probabilityCol="probability", rawPredictionCol="rawPrediction", smoothing=1.0,
                  modelType="multinomial", thresholds=None, weightCol=None):
         """
-        __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                  probabilityCol="probability", rawPredictionCol="rawPrediction", smoothing=1.0, \
                  modelType="multinomial", thresholds=None, weightCol=None)
         """
         super(NaiveBayes, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.NaiveBayes", self.uid)
-        self._setDefault(smoothing=1.0, modelType="multinomial")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("1.5.0")
-    def setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def setParams(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                   probabilityCol="probability", rawPredictionCol="rawPrediction", smoothing=1.0,
                   modelType="multinomial", thresholds=None, weightCol=None):
         """
-        setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                   probabilityCol="probability", rawPredictionCol="rawPrediction", smoothing=1.0, \
                   modelType="multinomial", thresholds=None, weightCol=None)
         Sets params for Naive Bayes.
@@ -2273,6 +2494,10 @@ class _MultilayerPerceptronParams(_ProbabilisticClassifierParams, HasSeed, HasMa
     initialWeights = Param(Params._dummy(), "initialWeights", "The initial weights of the model.",
                            typeConverter=TypeConverters.toVector)
 
+    def __init__(self, *args):
+        super(_MultilayerPerceptronParams, self).__init__(*args)
+        self._setDefault(maxIter=100, tol=1E-6, blockSize=128, stepSize=0.03, solver="l-bfgs")
+
     @since("1.6.0")
     def getLayers(self):
         """
@@ -2297,6 +2522,10 @@ class MultilayerPerceptronClassifier(_JavaProbabilisticClassifier, _MultilayerPe
     Number of inputs has to be equal to the size of feature vectors.
     Number of outputs has to be equal to the total number of labels.
 
+    .. versionadded:: 1.6.0
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> df = spark.createDataFrame([
     ...     (0.0, Vectors.dense([0.0, 0.0])),
@@ -2352,23 +2581,23 @@ class MultilayerPerceptronClassifier(_JavaProbabilisticClassifier, _MultilayerPe
     True
     >>> model.weights == model2.weights
     True
+    >>> model.transform(testDF).take(1) == model2.transform(testDF).take(1)
+    True
     >>> mlp2 = mlp2.setInitialWeights(list(range(0, 12)))
     >>> model3 = mlp2.fit(df)
     >>> model3.weights != model2.weights
     True
     >>> model3.getLayers() == model.getLayers()
     True
-
-    .. versionadded:: 1.6.0
     """
 
     @keyword_only
-    def __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def __init__(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                  maxIter=100, tol=1e-6, seed=None, layers=None, blockSize=128, stepSize=0.03,
                  solver="l-bfgs", initialWeights=None, probabilityCol="probability",
                  rawPredictionCol="rawPrediction"):
         """
-        __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                  maxIter=100, tol=1e-6, seed=None, layers=None, blockSize=128, stepSize=0.03, \
                  solver="l-bfgs", initialWeights=None, probabilityCol="probability", \
                  rawPredictionCol="rawPrediction")
@@ -2376,18 +2605,17 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(MultilayerPerceptronClassifier, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.MultilayerPerceptronClassifier", self.uid)
-        self._setDefault(maxIter=100, tol=1E-6, blockSize=128, stepSize=0.03, solver="l-bfgs")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("1.6.0")
-    def setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def setParams(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                   maxIter=100, tol=1e-6, seed=None, layers=None, blockSize=128, stepSize=0.03,
                   solver="l-bfgs", initialWeights=None, probabilityCol="probability",
                   rawPredictionCol="rawPrediction"):
         """
-        setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                   maxIter=100, tol=1e-6, seed=None, layers=None, blockSize=128, stepSize=0.03, \
                   solver="l-bfgs", initialWeights=None, probabilityCol="probability", \
                   rawPredictionCol="rawPrediction"):
@@ -2454,7 +2682,7 @@ def setSolver(self, value):
 
 class MultilayerPerceptronClassificationModel(_JavaProbabilisticClassificationModel,
                                               _MultilayerPerceptronParams, JavaMLWritable,
-                                              JavaMLReadable):
+                                              JavaMLReadable, HasTrainingSummary):
     """
     Model fitted by MultilayerPerceptronClassifier.
 
@@ -2469,6 +2697,55 @@ def weights(self):
         """
         return self._call_java("weights")
 
+    @since("3.1.0")
+    def summary(self):
+        """
+        Gets summary (accuracy/precision/recall, objective history, total iterations) of model
+        trained on the training set. An exception is thrown if `trainingSummary is None`.
+        """
+        if self.hasSummary:
+            return MultilayerPerceptronClassificationTrainingSummary(
+                super(MultilayerPerceptronClassificationModel, self).summary)
+        else:
+            raise RuntimeError("No training summary available for this %s" %
+                               self.__class__.__name__)
+
+    def evaluate(self, dataset):
+        """
+        Evaluates the model on a test dataset.
+
+        .. versionadded:: 3.1.0
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            Test dataset to evaluate model on.
+        """
+        if not isinstance(dataset, DataFrame):
+            raise TypeError("dataset must be a DataFrame but got %s." % type(dataset))
+        java_mlp_summary = self._call_java("evaluate", dataset)
+        return MultilayerPerceptronClassificationSummary(java_mlp_summary)
+
+
+class MultilayerPerceptronClassificationSummary(_ClassificationSummary):
+    """
+    Abstraction for MultilayerPerceptronClassifier Results for a given model.
+
+    .. versionadded:: 3.1.0
+    """
+    pass
+
+
+@inherit_doc
+class MultilayerPerceptronClassificationTrainingSummary(MultilayerPerceptronClassificationSummary,
+                                                        _TrainingSummary):
+    """
+    Abstraction for MultilayerPerceptronClassifier Training results.
+
+    .. versionadded:: 3.1.0
+    """
+    pass
+
 
 class _OneVsRestParams(_ClassifierParams, HasWeightCol):
     """
@@ -2486,7 +2763,7 @@ def getClassifier(self):
 
 
 @inherit_doc
-class OneVsRest(Estimator, _OneVsRestParams, HasParallelism, JavaMLReadable, JavaMLWritable):
+class OneVsRest(Estimator, _OneVsRestParams, HasParallelism, MLReadable, MLWritable):
     """
     Reduction of Multiclass Classification to Binary Classification.
     Performs reduction using one against all strategy.
@@ -2494,6 +2771,10 @@ class OneVsRest(Estimator, _OneVsRestParams, HasParallelism, JavaMLReadable, Jav
     Each example is scored against all k models and the model with highest score
     is picked to label the example.
 
+    .. versionadded:: 2.0.0
+
+    Examples
+    --------
     >>> from pyspark.sql import Row
     >>> from pyspark.ml.linalg import Vectors
     >>> data_path = "data/mllib/sample_multiclass_classification_data.txt"
@@ -2527,17 +2808,17 @@ class OneVsRest(Estimator, _OneVsRestParams, HasParallelism, JavaMLReadable, Jav
     >>> model2 = OneVsRestModel.load(model_path)
     >>> model2.transform(test0).head().newPrediction
     0.0
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     >>> model.transform(test2).columns
     ['features', 'rawPrediction', 'newPrediction']
-
-    .. versionadded:: 2.0.0
     """
 
     @keyword_only
-    def __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def __init__(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                  rawPredictionCol="rawPrediction", classifier=None, weightCol=None, parallelism=1):
         """
-        __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                  rawPredictionCol="rawPrediction", classifier=None, weightCol=None, parallelism=1):
         """
         super(OneVsRest, self).__init__()
@@ -2547,10 +2828,10 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
 
     @keyword_only
     @since("2.0.0")
-    def setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def setParams(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                   rawPredictionCol="rawPrediction", classifier=None, weightCol=None, parallelism=1):
         """
-        setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                   rawPredictionCol="rawPrediction", classifier=None, weightCol=None, parallelism=1):
         Sets params for OneVsRest.
         """
@@ -2647,15 +2928,23 @@ def trainSingleClass(index):
 
         return self._copyValues(OneVsRestModel(models=models))
 
-    @since("2.0.0")
     def copy(self, extra=None):
         """
         Creates a copy of this instance with a randomly generated uid
         and some extra params. This creates a deep copy of the embedded paramMap,
         and copies the embedded and extra parameters over.
 
-        :param extra: Extra parameters to copy to the new instance
-        :return: Copy of this instance
+        .. versionadded:: 2.0.0
+
+        Examples
+        --------
+        extra : dict, optional
+            Extra parameters to copy to the new instance
+
+        Returns
+        -------
+        :py:class:`OneVsRest`
+            Copy of this instance
         """
         if extra is None:
             extra = dict()
@@ -2688,7 +2977,10 @@ def _to_java(self):
         """
         Transfer this instance to a Java OneVsRest. Used for ML persistence.
 
-        :return: Java object equivalent to this instance.
+        Returns
+        -------
+        py4j.java_gateway.JavaObject
+            Java object equivalent to this instance.
         """
         _java_obj = JavaParams._new_java_obj("org.apache.spark.ml.classification.OneVsRest",
                                              self.uid)
@@ -2702,52 +2994,73 @@ def _to_java(self):
         _java_obj.setRawPredictionCol(self.getRawPredictionCol())
         return _java_obj
 
-    def _make_java_param_pair(self, param, value):
-        """
-        Makes a Java param pair.
-        """
-        sc = SparkContext._active_spark_context
-        param = self._resolveParam(param)
-        _java_obj = JavaParams._new_java_obj("org.apache.spark.ml.classification.OneVsRest",
-                                             self.uid)
-        java_param = _java_obj.getParam(param.name)
-        if isinstance(value, JavaParams):
-            # used in the case of an estimator having another estimator as a parameter
-            # the reason why this is not in _py2java in common.py is that importing
-            # Estimator and Model in common.py results in a circular import with inherit_doc
-            java_value = value._to_java()
+    @classmethod
+    def read(cls):
+        return OneVsRestReader(cls)
+
+    def write(self):
+        if isinstance(self.getClassifier(), JavaMLWritable):
+            return JavaMLWriter(self)
         else:
-            java_value = _py2java(sc, value)
-        return java_param.w(java_value)
+            return OneVsRestWriter(self)
 
-    def _transfer_param_map_to_java(self, pyParamMap):
-        """
-        Transforms a Python ParamMap into a Java ParamMap.
-        """
-        paramMap = JavaWrapper._new_java_obj("org.apache.spark.ml.param.ParamMap")
-        for param in self.params:
-            if param in pyParamMap:
-                pair = self._make_java_param_pair(param, pyParamMap[param])
-                paramMap.put([pair])
-        return paramMap
 
-    def _transfer_param_map_from_java(self, javaParamMap):
-        """
-        Transforms a Java ParamMap into a Python ParamMap.
-        """
-        sc = SparkContext._active_spark_context
-        paramMap = dict()
-        for pair in javaParamMap.toList():
-            param = pair.param()
-            if self.hasParam(str(param.name())):
-                if param.name() == "classifier":
-                    paramMap[self.getParam(param.name())] = JavaParams._from_java(pair.value())
-                else:
-                    paramMap[self.getParam(param.name())] = _java2py(sc, pair.value())
-        return paramMap
+class _OneVsRestSharedReadWrite:
+    @staticmethod
+    def saveImpl(instance, sc, path, extraMetadata=None):
+        skipParams = ['classifier']
+        jsonParams = DefaultParamsWriter.extractJsonParams(instance, skipParams)
+        DefaultParamsWriter.saveMetadata(instance, path, sc, paramMap=jsonParams,
+                                         extraMetadata=extraMetadata)
+        classifierPath = os.path.join(path, 'classifier')
+        instance.getClassifier().save(classifierPath)
+
+    @staticmethod
+    def loadClassifier(path, sc):
+        classifierPath = os.path.join(path, 'classifier')
+        return DefaultParamsReader.loadParamsInstance(classifierPath, sc)
+
+    @staticmethod
+    def validateParams(instance):
+        elems_to_check = [instance.getClassifier()]
+        if isinstance(instance, OneVsRestModel):
+            elems_to_check.extend(instance.models)
 
+        for elem in elems_to_check:
+            if not isinstance(elem, MLWritable):
+                raise ValueError(f'OneVsRest write will fail because it contains {elem.uid} '
+                                 f'which is not writable.')
 
-class OneVsRestModel(Model, _OneVsRestParams, JavaMLReadable, JavaMLWritable):
+
+@inherit_doc
+class OneVsRestReader(MLReader):
+    def __init__(self, cls):
+        super(OneVsRestReader, self).__init__()
+        self.cls = cls
+
+    def load(self, path):
+        metadata = DefaultParamsReader.loadMetadata(path, self.sc)
+        if not DefaultParamsReader.isPythonParamsInstance(metadata):
+            return JavaMLReader(self.cls).load(path)
+        else:
+            classifier = _OneVsRestSharedReadWrite.loadClassifier(path, self.sc)
+            ova = OneVsRest(classifier=classifier)._resetUid(metadata['uid'])
+            DefaultParamsReader.getAndSetParams(ova, metadata, skipParams=['classifier'])
+            return ova
+
+
+@inherit_doc
+class OneVsRestWriter(MLWriter):
+    def __init__(self, instance):
+        super(OneVsRestWriter, self).__init__()
+        self.instance = instance
+
+    def saveImpl(self, path):
+        _OneVsRestSharedReadWrite.validateParams(self.instance)
+        _OneVsRestSharedReadWrite.saveImpl(self.instance, self.sc, path)
+
+
+class OneVsRestModel(Model, _OneVsRestParams, MLReadable, MLWritable):
     """
     Model fitted by OneVsRest.
     This stores the models resulting from training k binary classifiers: one for each class.
@@ -2778,6 +3091,9 @@ def setRawPredictionCol(self, value):
     def __init__(self, models):
         super(OneVsRestModel, self).__init__()
         self.models = models
+        if not isinstance(models[0], JavaMLWritable):
+            return
+        # set java instance
         java_models = [model._to_java() for model in self.models]
         sc = SparkContext._active_spark_context
         java_models_array = JavaWrapper._new_java_array(java_models,
@@ -2835,7 +3151,7 @@ def func(predictions):
                     predArray.append(x)
                 return Vectors.dense(predArray)
 
-            rawPredictionUDF = udf(func)
+            rawPredictionUDF = udf(func, VectorUDT())
             aggregatedDataset = aggregatedDataset.withColumn(
                 self.getRawPredictionCol(), rawPredictionUDF(aggregatedDataset[accColName]))
 
@@ -2847,15 +3163,23 @@ def func(predictions):
                 self.getPredictionCol(), labelUDF(aggregatedDataset[accColName]))
         return aggregatedDataset.drop(accColName)
 
-    @since("2.0.0")
     def copy(self, extra=None):
         """
         Creates a copy of this instance with a randomly generated uid
         and some extra params. This creates a deep copy of the embedded paramMap,
         and copies the embedded and extra parameters over.
 
-        :param extra: Extra parameters to copy to the new instance
-        :return: Copy of this instance
+        .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        extra : dict, optional
+            Extra parameters to copy to the new instance
+
+        Returns
+        -------
+        :py:class:`OneVsRestModel`
+            Copy of this instance
         """
         if extra is None:
             extra = dict()
@@ -2887,7 +3211,10 @@ def _to_java(self):
         """
         Transfer this instance to a Java OneVsRestModel. Used for ML persistence.
 
-        :return: Java object equivalent to this instance.
+        Returns
+        -------
+        py4j.java_gateway.JavaObject
+            Java object equivalent to this instance.
         """
         sc = SparkContext._active_spark_context
         java_models = [model._to_java() for model in self.models]
@@ -2904,6 +3231,57 @@ def _to_java(self):
             _java_obj.set("weightCol", self.getWeightCol())
         return _java_obj
 
+    @classmethod
+    def read(cls):
+        return OneVsRestModelReader(cls)
+
+    def write(self):
+        if all(map(lambda elem: isinstance(elem, JavaMLWritable),
+                   [self.getClassifier()] + self.models)):
+            return JavaMLWriter(self)
+        else:
+            return OneVsRestModelWriter(self)
+
+
+@inherit_doc
+class OneVsRestModelReader(MLReader):
+    def __init__(self, cls):
+        super(OneVsRestModelReader, self).__init__()
+        self.cls = cls
+
+    def load(self, path):
+        metadata = DefaultParamsReader.loadMetadata(path, self.sc)
+        if not DefaultParamsReader.isPythonParamsInstance(metadata):
+            return JavaMLReader(self.cls).load(path)
+        else:
+            classifier = _OneVsRestSharedReadWrite.loadClassifier(path, self.sc)
+            numClasses = metadata['numClasses']
+            subModels = [None] * numClasses
+            for idx in range(numClasses):
+                subModelPath = os.path.join(path, f'model_{idx}')
+                subModels[idx] = DefaultParamsReader.loadParamsInstance(subModelPath, self.sc)
+            ovaModel = OneVsRestModel(subModels)._resetUid(metadata['uid'])
+            ovaModel.set(ovaModel.classifier, classifier)
+            DefaultParamsReader.getAndSetParams(ovaModel, metadata, skipParams=['classifier'])
+            return ovaModel
+
+
+@inherit_doc
+class OneVsRestModelWriter(MLWriter):
+    def __init__(self, instance):
+        super(OneVsRestModelWriter, self).__init__()
+        self.instance = instance
+
+    def saveImpl(self, path):
+        _OneVsRestSharedReadWrite.validateParams(self.instance)
+        instance = self.instance
+        numClasses = len(instance.models)
+        extraMetadata = {'numClasses': numClasses}
+        _OneVsRestSharedReadWrite.saveImpl(instance, self.sc, path, extraMetadata=extraMetadata)
+        for idx in range(numClasses):
+            subModelPath = os.path.join(path, f'model_{idx}')
+            instance.models[idx].save(subModelPath)
+
 
 @inherit_doc
 class FMClassifier(_JavaProbabilisticClassifier, _FactorizationMachinesParams, JavaMLWritable,
@@ -2911,11 +3289,15 @@ class FMClassifier(_JavaProbabilisticClassifier, _FactorizationMachinesParams, J
     """
     Factorization Machines learning algorithm for classification.
 
-    solver Supports:
+    Solver supports:
 
     * gd (normal mini-batch gradient descent)
     * adamW (default)
 
+    .. versionadded:: 3.0.0
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> from pyspark.ml.classification import FMClassifier
     >>> df = spark.createDataFrame([
@@ -2952,18 +3334,27 @@ class FMClassifier(_JavaProbabilisticClassifier, _FactorizationMachinesParams, J
     DenseVector([14.8232])
     >>> model.factors
     DenseMatrix(1, 2, [0.0163, -0.0051], 1)
-
-    .. versionadded:: 3.0.0
+    >>> model_path = temp_path + "/fm_model"
+    >>> model.save(model_path)
+    >>> model2 = FMClassificationModel.load(model_path)
+    >>> model2.intercept
+    -7.316665276826291
+    >>> model2.linear
+    DenseVector([14.8232])
+    >>> model2.factors
+    DenseMatrix(1, 2, [0.0163, -0.0051], 1)
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     """
 
     @keyword_only
-    def __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def __init__(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                  probabilityCol="probability", rawPredictionCol="rawPrediction",
                  factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0,
                  miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0,
                  tol=1e-6, solver="adamW", thresholds=None, seed=None):
         """
-        __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                  probabilityCol="probability", rawPredictionCol="rawPrediction", \
                  factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0, \
                  miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0, \
@@ -2972,21 +3363,18 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(FMClassifier, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.FMClassifier", self.uid)
-        self._setDefault(factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0,
-                         miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0,
-                         tol=1e-6, solver="adamW")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("3.0.0")
-    def setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def setParams(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                   probabilityCol="probability", rawPredictionCol="rawPrediction",
                   factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0,
                   miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0,
                   tol=1e-6, solver="adamW", thresholds=None, seed=None):
         """
-        setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                   probabilityCol="probability", rawPredictionCol="rawPrediction", \
                   factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0, \
                   miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0, \
@@ -3078,7 +3466,7 @@ def setRegParam(self, value):
 
 
 class FMClassificationModel(_JavaProbabilisticClassificationModel, _FactorizationMachinesParams,
-                            JavaMLWritable, JavaMLReadable):
+                            JavaMLWritable, JavaMLReadable, HasTrainingSummary):
     """
     Model fitted by :class:`FMClassifier`.
 
@@ -3109,6 +3497,53 @@ def factors(self):
         """
         return self._call_java("factors")
 
+    @since("3.1.0")
+    def summary(self):
+        """
+        Gets summary (accuracy/precision/recall, objective history, total iterations) of model
+        trained on the training set. An exception is thrown if `trainingSummary is None`.
+        """
+        if self.hasSummary:
+            return FMClassificationTrainingSummary(super(FMClassificationModel, self).summary)
+        else:
+            raise RuntimeError("No training summary available for this %s" %
+                               self.__class__.__name__)
+
+    def evaluate(self, dataset):
+        """
+        Evaluates the model on a test dataset.
+
+        .. versionadded:: 3.1.0
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            Test dataset to evaluate model on.
+        """
+        if not isinstance(dataset, DataFrame):
+            raise TypeError("dataset must be a DataFrame but got %s." % type(dataset))
+        java_fm_summary = self._call_java("evaluate", dataset)
+        return FMClassificationSummary(java_fm_summary)
+
+
+class FMClassificationSummary(_BinaryClassificationSummary):
+    """
+    Abstraction for FMClassifier Results for a given model.
+
+    .. versionadded:: 3.1.0
+    """
+    pass
+
+
+@inherit_doc
+class FMClassificationTrainingSummary(FMClassificationSummary, _TrainingSummary):
+    """
+    Abstraction for FMClassifier Training results.
+
+    .. versionadded:: 3.1.0
+    """
+    pass
+
 
 if __name__ == "__main__":
     import doctest
diff --git a/python/pyspark/ml/classification.pyi b/python/pyspark/ml/classification.pyi
new file mode 100644
index 0000000000000..a4a3d21018ad9
--- /dev/null
+++ b/python/pyspark/ml/classification.pyi
@@ -0,0 +1,944 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Any, List, Optional, Type
+from pyspark.ml._typing import JM, M, P, T, ParamMap
+
+import abc
+from abc import abstractmethod
+from pyspark.ml import Estimator, Model, PredictionModel, Predictor, Transformer
+from pyspark.ml.base import _PredictorParams
+from pyspark.ml.param.shared import (
+    HasAggregationDepth,
+    HasBlockSize,
+    HasMaxBlockSizeInMB,
+    HasElasticNetParam,
+    HasFitIntercept,
+    HasMaxIter,
+    HasParallelism,
+    HasProbabilityCol,
+    HasRawPredictionCol,
+    HasRegParam,
+    HasSeed,
+    HasSolver,
+    HasStandardization,
+    HasStepSize,
+    HasThreshold,
+    HasThresholds,
+    HasTol,
+    HasWeightCol,
+)
+from pyspark.ml.regression import _FactorizationMachinesParams
+from pyspark.ml.tree import (
+    _DecisionTreeModel,
+    _DecisionTreeParams,
+    _GBTParams,
+    _HasVarianceImpurity,
+    _RandomForestParams,
+    _TreeClassifierParams,
+    _TreeEnsembleModel,
+)
+from pyspark.ml.util import HasTrainingSummary, JavaMLReadable, JavaMLWritable, \
+    MLReader, MLReadable, MLWriter, MLWritable
+from pyspark.ml.wrapper import JavaPredictionModel, JavaPredictor, JavaWrapper
+
+from pyspark.ml.linalg import Matrix, Vector
+from pyspark.ml.param import Param
+from pyspark.ml.regression import DecisionTreeRegressionModel
+from pyspark.sql.dataframe import DataFrame
+
+class _ClassifierParams(HasRawPredictionCol, _PredictorParams): ...
+
+class Classifier(Predictor, _ClassifierParams, metaclass=abc.ABCMeta):
+    def setRawPredictionCol(self: P, value: str) -> P: ...
+
+class ClassificationModel(PredictionModel, _ClassifierParams, metaclass=abc.ABCMeta):
+    def setRawPredictionCol(self: P, value: str) -> P: ...
+    @property
+    @abc.abstractmethod
+    def numClasses(self) -> int: ...
+    @abstractmethod
+    def predictRaw(self, value: Vector) -> Vector: ...
+
+class _ProbabilisticClassifierParams(
+    HasProbabilityCol, HasThresholds, _ClassifierParams
+): ...
+
+class ProbabilisticClassifier(
+    Classifier, _ProbabilisticClassifierParams, metaclass=abc.ABCMeta
+):
+    def setProbabilityCol(self: P, value: str) -> P: ...
+    def setThresholds(self: P, value: List[float]) -> P: ...
+
+class ProbabilisticClassificationModel(
+    ClassificationModel, _ProbabilisticClassifierParams, metaclass=abc.ABCMeta
+):
+    def setProbabilityCol(self: M, value: str) -> M: ...
+    def setThresholds(self: M, value: List[float]) -> M: ...
+    @abstractmethod
+    def predictProbability(self, value: Vector) -> Vector: ...
+
+class _JavaClassifier(Classifier, JavaPredictor[JM], metaclass=abc.ABCMeta):
+    def setRawPredictionCol(self: P, value: str) -> P: ...
+
+class _JavaClassificationModel(ClassificationModel, JavaPredictionModel[T]):
+    @property
+    def numClasses(self) -> int: ...
+    def predictRaw(self, value: Vector) -> Vector: ...
+
+class _JavaProbabilisticClassifier(
+    ProbabilisticClassifier, _JavaClassifier[JM], metaclass=abc.ABCMeta
+): ...
+
+class _JavaProbabilisticClassificationModel(
+    ProbabilisticClassificationModel, _JavaClassificationModel[T]
+):
+    def predictProbability(self, value: Vector) -> Vector: ...
+
+class _ClassificationSummary(JavaWrapper):
+    @property
+    def predictions(self) -> DataFrame: ...
+    @property
+    def predictionCol(self) -> str: ...
+    @property
+    def labelCol(self) -> str: ...
+    @property
+    def weightCol(self) -> str: ...
+    @property
+    def labels(self) -> List[str]: ...
+    @property
+    def truePositiveRateByLabel(self) -> List[float]: ...
+    @property
+    def falsePositiveRateByLabel(self) -> List[float]: ...
+    @property
+    def precisionByLabel(self) -> List[float]: ...
+    @property
+    def recallByLabel(self) -> List[float]: ...
+    def fMeasureByLabel(self, beta: float = ...) -> List[float]: ...
+    @property
+    def accuracy(self) -> float: ...
+    @property
+    def weightedTruePositiveRate(self) -> float: ...
+    @property
+    def weightedFalsePositiveRate(self) -> float: ...
+    @property
+    def weightedRecall(self) -> float: ...
+    @property
+    def weightedPrecision(self) -> float: ...
+    def weightedFMeasure(self, beta: float = ...) -> float: ...
+
+class _TrainingSummary(JavaWrapper):
+    @property
+    def objectiveHistory(self) -> List[float]: ...
+    @property
+    def totalIterations(self) -> int: ...
+
+class _BinaryClassificationSummary(_ClassificationSummary):
+    @property
+    def scoreCol(self) -> str: ...
+    @property
+    def roc(self) -> DataFrame: ...
+    @property
+    def areaUnderROC(self) -> float: ...
+    @property
+    def pr(self) -> DataFrame: ...
+    @property
+    def fMeasureByThreshold(self) -> DataFrame: ...
+    @property
+    def precisionByThreshold(self) -> DataFrame: ...
+    @property
+    def recallByThreshold(self) -> DataFrame: ...
+
+class _LinearSVCParams(
+    _ClassifierParams,
+    HasRegParam,
+    HasMaxIter,
+    HasFitIntercept,
+    HasTol,
+    HasStandardization,
+    HasWeightCol,
+    HasAggregationDepth,
+    HasThreshold,
+    HasMaxBlockSizeInMB,
+):
+    threshold: Param[float]
+    def __init__(self, *args: Any) -> None: ...
+
+class LinearSVC(
+    _JavaClassifier[LinearSVCModel],
+    _LinearSVCParams,
+    JavaMLWritable,
+    JavaMLReadable[LinearSVC],
+):
+    def __init__(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        maxIter: int = ...,
+        regParam: float = ...,
+        tol: float = ...,
+        rawPredictionCol: str = ...,
+        fitIntercept: bool = ...,
+        standardization: bool = ...,
+        threshold: float = ...,
+        weightCol: Optional[str] = ...,
+        aggregationDepth: int = ...,
+        maxBlockSizeInMB: float = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        maxIter: int = ...,
+        regParam: float = ...,
+        tol: float = ...,
+        rawPredictionCol: str = ...,
+        fitIntercept: bool = ...,
+        standardization: bool = ...,
+        threshold: float = ...,
+        weightCol: Optional[str] = ...,
+        aggregationDepth: int = ...,
+        maxBlockSizeInMB: float = ...
+    ) -> LinearSVC: ...
+    def setMaxIter(self, value: int) -> LinearSVC: ...
+    def setRegParam(self, value: float) -> LinearSVC: ...
+    def setTol(self, value: float) -> LinearSVC: ...
+    def setFitIntercept(self, value: bool) -> LinearSVC: ...
+    def setStandardization(self, value: bool) -> LinearSVC: ...
+    def setThreshold(self, value: float) -> LinearSVC: ...
+    def setWeightCol(self, value: str) -> LinearSVC: ...
+    def setAggregationDepth(self, value: int) -> LinearSVC: ...
+    def setMaxBlockSizeInMB(self, value: float) -> LinearSVC: ...
+
+class LinearSVCModel(
+    _JavaClassificationModel[Vector],
+    _LinearSVCParams,
+    JavaMLWritable,
+    JavaMLReadable[LinearSVCModel],
+    HasTrainingSummary[LinearSVCTrainingSummary],
+):
+    def setThreshold(self, value: float) -> LinearSVCModel: ...
+    @property
+    def coefficients(self) -> Vector: ...
+    @property
+    def intercept(self) -> float: ...
+    def summary(self) -> LinearSVCTrainingSummary: ...
+    def evaluate(self, dataset: DataFrame) -> LinearSVCSummary: ...
+
+class LinearSVCSummary(_BinaryClassificationSummary): ...
+class LinearSVCTrainingSummary(LinearSVCSummary, _TrainingSummary): ...
+
+class _LogisticRegressionParams(
+    _ProbabilisticClassifierParams,
+    HasRegParam,
+    HasElasticNetParam,
+    HasMaxIter,
+    HasFitIntercept,
+    HasTol,
+    HasStandardization,
+    HasWeightCol,
+    HasAggregationDepth,
+    HasThreshold,
+    HasMaxBlockSizeInMB,
+):
+    threshold: Param[float]
+    family: Param[str]
+    lowerBoundsOnCoefficients: Param[Matrix]
+    upperBoundsOnCoefficients: Param[Matrix]
+    lowerBoundsOnIntercepts: Param[Vector]
+    upperBoundsOnIntercepts: Param[Vector]
+    def __init__(self, *args: Any): ...
+    def setThreshold(self: P, value: float) -> P: ...
+    def getThreshold(self) -> float: ...
+    def setThresholds(self: P, value: List[float]) -> P: ...
+    def getThresholds(self) -> List[float]: ...
+    def getFamily(self) -> str: ...
+    def getLowerBoundsOnCoefficients(self) -> Matrix: ...
+    def getUpperBoundsOnCoefficients(self) -> Matrix: ...
+    def getLowerBoundsOnIntercepts(self) -> Vector: ...
+    def getUpperBoundsOnIntercepts(self) -> Vector: ...
+
+class LogisticRegression(
+    _JavaProbabilisticClassifier[LogisticRegressionModel],
+    _LogisticRegressionParams,
+    JavaMLWritable,
+    JavaMLReadable[LogisticRegression],
+):
+    def __init__(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        maxIter: int = ...,
+        regParam: float = ...,
+        elasticNetParam: float = ...,
+        tol: float = ...,
+        fitIntercept: bool = ...,
+        threshold: float = ...,
+        thresholds: Optional[List[float]] = ...,
+        probabilityCol: str = ...,
+        rawPredictionCol: str = ...,
+        standardization: bool = ...,
+        weightCol: Optional[str] = ...,
+        aggregationDepth: int = ...,
+        family: str = ...,
+        lowerBoundsOnCoefficients: Optional[Matrix] = ...,
+        upperBoundsOnCoefficients: Optional[Matrix] = ...,
+        lowerBoundsOnIntercepts: Optional[Vector] = ...,
+        upperBoundsOnIntercepts: Optional[Vector] = ...,
+        maxBlockSizeInMB: float = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        maxIter: int = ...,
+        regParam: float = ...,
+        elasticNetParam: float = ...,
+        tol: float = ...,
+        fitIntercept: bool = ...,
+        threshold: float = ...,
+        thresholds: Optional[List[float]] = ...,
+        probabilityCol: str = ...,
+        rawPredictionCol: str = ...,
+        standardization: bool = ...,
+        weightCol: Optional[str] = ...,
+        aggregationDepth: int = ...,
+        family: str = ...,
+        lowerBoundsOnCoefficients: Optional[Matrix] = ...,
+        upperBoundsOnCoefficients: Optional[Matrix] = ...,
+        lowerBoundsOnIntercepts: Optional[Vector] = ...,
+        upperBoundsOnIntercepts: Optional[Vector] = ...,
+        maxBlockSizeInMB: float = ...
+    ) -> LogisticRegression: ...
+    def setFamily(self, value: str) -> LogisticRegression: ...
+    def setLowerBoundsOnCoefficients(self, value: Matrix) -> LogisticRegression: ...
+    def setUpperBoundsOnCoefficients(self, value: Matrix) -> LogisticRegression: ...
+    def setLowerBoundsOnIntercepts(self, value: Vector) -> LogisticRegression: ...
+    def setUpperBoundsOnIntercepts(self, value: Vector) -> LogisticRegression: ...
+    def setMaxIter(self, value: int) -> LogisticRegression: ...
+    def setRegParam(self, value: float) -> LogisticRegression: ...
+    def setTol(self, value: float) -> LogisticRegression: ...
+    def setElasticNetParam(self, value: float) -> LogisticRegression: ...
+    def setFitIntercept(self, value: bool) -> LogisticRegression: ...
+    def setStandardization(self, value: bool) -> LogisticRegression: ...
+    def setWeightCol(self, value: str) -> LogisticRegression: ...
+    def setAggregationDepth(self, value: int) -> LogisticRegression: ...
+    def setMaxBlockSizeInMB(self, value: float) -> LogisticRegression: ...
+
+class LogisticRegressionModel(
+    _JavaProbabilisticClassificationModel[Vector],
+    _LogisticRegressionParams,
+    JavaMLWritable,
+    JavaMLReadable[LogisticRegressionModel],
+    HasTrainingSummary[LogisticRegressionTrainingSummary],
+):
+    @property
+    def coefficients(self) -> Vector: ...
+    @property
+    def intercept(self) -> float: ...
+    @property
+    def coefficientMatrix(self) -> Matrix: ...
+    @property
+    def interceptVector(self) -> Vector: ...
+    @property
+    def summary(self) -> LogisticRegressionTrainingSummary: ...
+    def evaluate(self, dataset: DataFrame) -> LogisticRegressionSummary: ...
+
+class LogisticRegressionSummary(_ClassificationSummary):
+    @property
+    def probabilityCol(self) -> str: ...
+    @property
+    def featuresCol(self) -> str: ...
+
+class LogisticRegressionTrainingSummary(
+    LogisticRegressionSummary, _TrainingSummary
+): ...
+class BinaryLogisticRegressionSummary(
+    _BinaryClassificationSummary, LogisticRegressionSummary
+): ...
+class BinaryLogisticRegressionTrainingSummary(
+    BinaryLogisticRegressionSummary, LogisticRegressionTrainingSummary
+): ...
+
+class _DecisionTreeClassifierParams(_DecisionTreeParams, _TreeClassifierParams):
+    def __init__(self, *args: Any): ...
+
+class DecisionTreeClassifier(
+    _JavaProbabilisticClassifier[DecisionTreeClassificationModel],
+    _DecisionTreeClassifierParams,
+    JavaMLWritable,
+    JavaMLReadable[DecisionTreeClassifier],
+):
+    def __init__(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        probabilityCol: str = ...,
+        rawPredictionCol: str = ...,
+        maxDepth: int = ...,
+        maxBins: int = ...,
+        minInstancesPerNode: int = ...,
+        minInfoGain: float = ...,
+        maxMemoryInMB: int = ...,
+        cacheNodeIds: bool = ...,
+        checkpointInterval: int = ...,
+        impurity: str = ...,
+        seed: Optional[int] = ...,
+        weightCol: Optional[str] = ...,
+        leafCol: str = ...,
+        minWeightFractionPerNode: float = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        probabilityCol: str = ...,
+        rawPredictionCol: str = ...,
+        maxDepth: int = ...,
+        maxBins: int = ...,
+        minInstancesPerNode: int = ...,
+        minInfoGain: float = ...,
+        maxMemoryInMB: int = ...,
+        cacheNodeIds: bool = ...,
+        checkpointInterval: int = ...,
+        impurity: str = ...,
+        seed: Optional[int] = ...,
+        weightCol: Optional[str] = ...,
+        leafCol: str = ...,
+        minWeightFractionPerNode: float = ...
+    ) -> DecisionTreeClassifier: ...
+    def setMaxDepth(self, value: int) -> DecisionTreeClassifier: ...
+    def setMaxBins(self, value: int) -> DecisionTreeClassifier: ...
+    def setMinInstancesPerNode(self, value: int) -> DecisionTreeClassifier: ...
+    def setMinWeightFractionPerNode(self, value: float) -> DecisionTreeClassifier: ...
+    def setMinInfoGain(self, value: float) -> DecisionTreeClassifier: ...
+    def setMaxMemoryInMB(self, value: int) -> DecisionTreeClassifier: ...
+    def setCacheNodeIds(self, value: bool) -> DecisionTreeClassifier: ...
+    def setImpurity(self, value: str) -> DecisionTreeClassifier: ...
+    def setCheckpointInterval(self, value: int) -> DecisionTreeClassifier: ...
+    def setSeed(self, value: int) -> DecisionTreeClassifier: ...
+    def setWeightCol(self, value: str) -> DecisionTreeClassifier: ...
+
+class DecisionTreeClassificationModel(
+    _DecisionTreeModel,
+    _JavaProbabilisticClassificationModel[Vector],
+    _DecisionTreeClassifierParams,
+    JavaMLWritable,
+    JavaMLReadable[DecisionTreeClassificationModel],
+):
+    @property
+    def featureImportances(self) -> Vector: ...
+
+class _RandomForestClassifierParams(_RandomForestParams, _TreeClassifierParams):
+    def __init__(self, *args: Any): ...
+
+class RandomForestClassifier(
+    _JavaProbabilisticClassifier[RandomForestClassificationModel],
+    _RandomForestClassifierParams,
+    JavaMLWritable,
+    JavaMLReadable[RandomForestClassifier],
+):
+    def __init__(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        probabilityCol: str = ...,
+        rawPredictionCol: str = ...,
+        maxDepth: int = ...,
+        maxBins: int = ...,
+        minInstancesPerNode: int = ...,
+        minInfoGain: float = ...,
+        maxMemoryInMB: int = ...,
+        cacheNodeIds: bool = ...,
+        checkpointInterval: int = ...,
+        impurity: str = ...,
+        numTrees: int = ...,
+        featureSubsetStrategy: str = ...,
+        seed: Optional[int] = ...,
+        subsamplingRate: float = ...,
+        leafCol: str = ...,
+        minWeightFractionPerNode: float = ...,
+        weightCol: Optional[str] = ...,
+        bootstrap: Optional[bool] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        probabilityCol: str = ...,
+        rawPredictionCol: str = ...,
+        maxDepth: int = ...,
+        maxBins: int = ...,
+        minInstancesPerNode: int = ...,
+        minInfoGain: float = ...,
+        maxMemoryInMB: int = ...,
+        cacheNodeIds: bool = ...,
+        checkpointInterval: int = ...,
+        seed: Optional[int] = ...,
+        impurity: str = ...,
+        numTrees: int = ...,
+        featureSubsetStrategy: str = ...,
+        subsamplingRate: float = ...,
+        leafCol: str = ...,
+        minWeightFractionPerNode: float = ...,
+        weightCol: Optional[str] = ...,
+        bootstrap: Optional[bool] = ...
+    ) -> RandomForestClassifier: ...
+    def setMaxDepth(self, value: int) -> RandomForestClassifier: ...
+    def setMaxBins(self, value: int) -> RandomForestClassifier: ...
+    def setMinInstancesPerNode(self, value: int) -> RandomForestClassifier: ...
+    def setMinInfoGain(self, value: float) -> RandomForestClassifier: ...
+    def setMaxMemoryInMB(self, value: int) -> RandomForestClassifier: ...
+    def setCacheNodeIds(self, value: bool) -> RandomForestClassifier: ...
+    def setImpurity(self, value: str) -> RandomForestClassifier: ...
+    def setNumTrees(self, value: int) -> RandomForestClassifier: ...
+    def setBootstrap(self, value: bool) -> RandomForestClassifier: ...
+    def setSubsamplingRate(self, value: float) -> RandomForestClassifier: ...
+    def setFeatureSubsetStrategy(self, value: str) -> RandomForestClassifier: ...
+    def setSeed(self, value: int) -> RandomForestClassifier: ...
+    def setCheckpointInterval(self, value: int) -> RandomForestClassifier: ...
+    def setWeightCol(self, value: str) -> RandomForestClassifier: ...
+    def setMinWeightFractionPerNode(self, value: float) -> RandomForestClassifier: ...
+
+class RandomForestClassificationModel(
+    _TreeEnsembleModel,
+    _JavaProbabilisticClassificationModel[Vector],
+    _RandomForestClassifierParams,
+    JavaMLWritable,
+    JavaMLReadable[RandomForestClassificationModel],
+    HasTrainingSummary[RandomForestClassificationTrainingSummary],
+):
+    @property
+    def featureImportances(self) -> Vector: ...
+    @property
+    def trees(self) -> List[DecisionTreeClassificationModel]: ...
+    def summary(self) -> RandomForestClassificationTrainingSummary: ...
+    def evaluate(self, dataset: DataFrame) -> RandomForestClassificationSummary: ...
+
+class RandomForestClassificationSummary(_ClassificationSummary): ...
+class RandomForestClassificationTrainingSummary(
+    RandomForestClassificationSummary, _TrainingSummary
+): ...
+class BinaryRandomForestClassificationSummary(_BinaryClassificationSummary): ...
+class BinaryRandomForestClassificationTrainingSummary(
+    BinaryRandomForestClassificationSummary, RandomForestClassificationTrainingSummary
+): ...
+
+class _GBTClassifierParams(_GBTParams, _HasVarianceImpurity):
+    supportedLossTypes: List[str]
+    lossType: Param[str]
+    def __init__(self, *args: Any): ...
+    def getLossType(self) -> str: ...
+
+class GBTClassifier(
+    _JavaProbabilisticClassifier[GBTClassificationModel],
+    _GBTClassifierParams,
+    JavaMLWritable,
+    JavaMLReadable[GBTClassifier],
+):
+    def __init__(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        maxDepth: int = ...,
+        maxBins: int = ...,
+        minInstancesPerNode: int = ...,
+        minInfoGain: float = ...,
+        maxMemoryInMB: int = ...,
+        cacheNodeIds: bool = ...,
+        checkpointInterval: int = ...,
+        lossType: str = ...,
+        maxIter: int = ...,
+        stepSize: float = ...,
+        seed: Optional[int] = ...,
+        subsamplingRate: float = ...,
+        featureSubsetStrategy: str = ...,
+        validationTol: float = ...,
+        validationIndicatorCol: Optional[str] = ...,
+        leafCol: str = ...,
+        minWeightFractionPerNode: float = ...,
+        weightCol: Optional[str] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        maxDepth: int = ...,
+        maxBins: int = ...,
+        minInstancesPerNode: int = ...,
+        minInfoGain: float = ...,
+        maxMemoryInMB: int = ...,
+        cacheNodeIds: bool = ...,
+        checkpointInterval: int = ...,
+        lossType: str = ...,
+        maxIter: int = ...,
+        stepSize: float = ...,
+        seed: Optional[int] = ...,
+        subsamplingRate: float = ...,
+        featureSubsetStrategy: str = ...,
+        validationTol: float = ...,
+        validationIndicatorCol: Optional[str] = ...,
+        leafCol: str = ...,
+        minWeightFractionPerNode: float = ...,
+        weightCol: Optional[str] = ...
+    ) -> GBTClassifier: ...
+    def setMaxDepth(self, value: int) -> GBTClassifier: ...
+    def setMaxBins(self, value: int) -> GBTClassifier: ...
+    def setMinInstancesPerNode(self, value: int) -> GBTClassifier: ...
+    def setMinInfoGain(self, value: float) -> GBTClassifier: ...
+    def setMaxMemoryInMB(self, value: int) -> GBTClassifier: ...
+    def setCacheNodeIds(self, value: bool) -> GBTClassifier: ...
+    def setImpurity(self, value: str) -> GBTClassifier: ...
+    def setLossType(self, value: str) -> GBTClassifier: ...
+    def setSubsamplingRate(self, value: float) -> GBTClassifier: ...
+    def setFeatureSubsetStrategy(self, value: str) -> GBTClassifier: ...
+    def setValidationIndicatorCol(self, value: str) -> GBTClassifier: ...
+    def setMaxIter(self, value: int) -> GBTClassifier: ...
+    def setCheckpointInterval(self, value: int) -> GBTClassifier: ...
+    def setSeed(self, value: int) -> GBTClassifier: ...
+    def setStepSize(self, value: float) -> GBTClassifier: ...
+    def setWeightCol(self, value: str) -> GBTClassifier: ...
+    def setMinWeightFractionPerNode(self, value: float) -> GBTClassifier: ...
+
+class GBTClassificationModel(
+    _TreeEnsembleModel,
+    _JavaProbabilisticClassificationModel[Vector],
+    _GBTClassifierParams,
+    JavaMLWritable,
+    JavaMLReadable[GBTClassificationModel],
+):
+    @property
+    def featureImportances(self) -> Vector: ...
+    @property
+    def trees(self) -> List[DecisionTreeRegressionModel]: ...
+    def evaluateEachIteration(self, dataset: DataFrame) -> List[float]: ...
+
+class _NaiveBayesParams(_PredictorParams, HasWeightCol):
+    smoothing: Param[float]
+    modelType: Param[str]
+    def __init__(self, *args: Any): ...
+    def getSmoothing(self) -> float: ...
+    def getModelType(self) -> str: ...
+
+class NaiveBayes(
+    _JavaProbabilisticClassifier[NaiveBayesModel],
+    _NaiveBayesParams,
+    HasThresholds,
+    HasWeightCol,
+    JavaMLWritable,
+    JavaMLReadable[NaiveBayes],
+):
+    def __init__(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        probabilityCol: str = ...,
+        rawPredictionCol: str = ...,
+        smoothing: float = ...,
+        modelType: str = ...,
+        thresholds: Optional[List[float]] = ...,
+        weightCol: Optional[str] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        probabilityCol: str = ...,
+        rawPredictionCol: str = ...,
+        smoothing: float = ...,
+        modelType: str = ...,
+        thresholds: Optional[List[float]] = ...,
+        weightCol: Optional[str] = ...
+    ) -> NaiveBayes: ...
+    def setSmoothing(self, value: float) -> NaiveBayes: ...
+    def setModelType(self, value: str) -> NaiveBayes: ...
+    def setWeightCol(self, value: str) -> NaiveBayes: ...
+
+class NaiveBayesModel(
+    _JavaProbabilisticClassificationModel[Vector],
+    _NaiveBayesParams,
+    JavaMLWritable,
+    JavaMLReadable[NaiveBayesModel],
+):
+    @property
+    def pi(self) -> Vector: ...
+    @property
+    def theta(self) -> Matrix: ...
+    @property
+    def sigma(self) -> Matrix: ...
+
+class _MultilayerPerceptronParams(
+    _ProbabilisticClassifierParams,
+    HasSeed,
+    HasMaxIter,
+    HasTol,
+    HasStepSize,
+    HasSolver,
+    HasBlockSize,
+):
+    layers: Param[List[int]]
+    solver: Param[str]
+    initialWeights: Param[Vector]
+    def __init__(self, *args: Any): ...
+    def getLayers(self) -> List[int]: ...
+    def getInitialWeights(self) -> Vector: ...
+
+class MultilayerPerceptronClassifier(
+    _JavaProbabilisticClassifier[MultilayerPerceptronClassificationModel],
+    _MultilayerPerceptronParams,
+    JavaMLWritable,
+    JavaMLReadable[MultilayerPerceptronClassifier],
+):
+    def __init__(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        maxIter: int = ...,
+        tol: float = ...,
+        seed: Optional[int] = ...,
+        layers: Optional[List[int]] = ...,
+        blockSize: int = ...,
+        stepSize: float = ...,
+        solver: str = ...,
+        initialWeights: Optional[Vector] = ...,
+        probabilityCol: str = ...,
+        rawPredictionCol: str = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        maxIter: int = ...,
+        tol: float = ...,
+        seed: Optional[int] = ...,
+        layers: Optional[List[int]] = ...,
+        blockSize: int = ...,
+        stepSize: float = ...,
+        solver: str = ...,
+        initialWeights: Optional[Vector] = ...,
+        probabilityCol: str = ...,
+        rawPredictionCol: str = ...
+    ) -> MultilayerPerceptronClassifier: ...
+    def setLayers(self, value: List[int]) -> MultilayerPerceptronClassifier: ...
+    def setBlockSize(self, value: int) -> MultilayerPerceptronClassifier: ...
+    def setInitialWeights(self, value: Vector) -> MultilayerPerceptronClassifier: ...
+    def setMaxIter(self, value: int) -> MultilayerPerceptronClassifier: ...
+    def setSeed(self, value: int) -> MultilayerPerceptronClassifier: ...
+    def setTol(self, value: float) -> MultilayerPerceptronClassifier: ...
+    def setStepSize(self, value: float) -> MultilayerPerceptronClassifier: ...
+    def setSolver(self, value: str) -> MultilayerPerceptronClassifier: ...
+
+class MultilayerPerceptronClassificationModel(
+    _JavaProbabilisticClassificationModel[Vector],
+    _MultilayerPerceptronParams,
+    JavaMLWritable,
+    JavaMLReadable[MultilayerPerceptronClassificationModel],
+    HasTrainingSummary[MultilayerPerceptronClassificationTrainingSummary],
+):
+    @property
+    def weights(self) -> Vector: ...
+    def summary(self) -> MultilayerPerceptronClassificationTrainingSummary: ...
+    def evaluate(
+        self, dataset: DataFrame
+    ) -> MultilayerPerceptronClassificationSummary: ...
+
+class MultilayerPerceptronClassificationSummary(_ClassificationSummary): ...
+class MultilayerPerceptronClassificationTrainingSummary(
+    MultilayerPerceptronClassificationSummary, _TrainingSummary
+): ...
+
+class _OneVsRestParams(_ClassifierParams, HasWeightCol):
+    classifier: Param[Estimator]
+    def getClassifier(self) -> Estimator[M]: ...
+
+class OneVsRest(
+    Estimator[OneVsRestModel],
+    _OneVsRestParams,
+    HasParallelism,
+    MLReadable[OneVsRest],
+    MLWritable,
+):
+    def __init__(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        rawPredictionCol: str = ...,
+        classifier: Optional[Estimator[M]] = ...,
+        weightCol: Optional[str] = ...,
+        parallelism: int = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        featuresCol: Optional[str] = ...,
+        labelCol: Optional[str] = ...,
+        predictionCol: Optional[str] = ...,
+        rawPredictionCol: str = ...,
+        classifier: Optional[Estimator[M]] = ...,
+        weightCol: Optional[str] = ...,
+        parallelism: int = ...
+    ) -> OneVsRest: ...
+    def setClassifier(self, value: Estimator[M]) -> OneVsRest: ...
+    def setLabelCol(self, value: str) -> OneVsRest: ...
+    def setFeaturesCol(self, value: str) -> OneVsRest: ...
+    def setPredictionCol(self, value: str) -> OneVsRest: ...
+    def setRawPredictionCol(self, value: str) -> OneVsRest: ...
+    def setWeightCol(self, value: str) -> OneVsRest: ...
+    def setParallelism(self, value: int) -> OneVsRest: ...
+    def copy(self, extra: Optional[ParamMap] = ...) -> OneVsRest: ...
+
+class OneVsRestModel(
+    Model, _OneVsRestParams, MLReadable[OneVsRestModel], MLWritable
+):
+    models: List[Transformer]
+    def __init__(self, models: List[Transformer]) -> None: ...
+    def setFeaturesCol(self, value: str) -> OneVsRestModel: ...
+    def setPredictionCol(self, value: str) -> OneVsRestModel: ...
+    def setRawPredictionCol(self, value: str) -> OneVsRestModel: ...
+    def copy(self, extra: Optional[ParamMap] = ...) -> OneVsRestModel: ...
+
+class OneVsRestWriter(MLWriter):
+    instance: OneVsRest
+    def __init__(self, instance: OneVsRest) -> None: ...
+    def saveImpl(self, path: str) -> None: ...
+
+class OneVsRestReader(MLReader[OneVsRest]):
+    cls: Type[OneVsRest]
+    def __init__(self, cls: Type[OneVsRest]) -> None: ...
+    def load(self, path: str) -> OneVsRest: ...
+
+class OneVsRestModelWriter(MLWriter):
+    instance: OneVsRestModel
+    def __init__(self, instance: OneVsRestModel) -> None: ...
+    def saveImpl(self, path: str) -> None: ...
+
+class OneVsRestModelReader(MLReader[OneVsRestModel]):
+    cls: Type[OneVsRestModel]
+    def __init__(self, cls: Type[OneVsRestModel]) -> None: ...
+    def load(self, path: str) -> OneVsRestModel: ...
+
+class FMClassifier(
+    _JavaProbabilisticClassifier[FMClassificationModel],
+    _FactorizationMachinesParams,
+    JavaMLWritable,
+    JavaMLReadable[FMClassifier],
+):
+    factorSize: Param[int]
+    fitLinear: Param[bool]
+    miniBatchFraction: Param[float]
+    initStd: Param[float]
+    solver: Param[str]
+    def __init__(
+        self,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        probabilityCol: str = ...,
+        rawPredictionCol: str = ...,
+        factorSize: int = ...,
+        fitIntercept: bool = ...,
+        fitLinear: bool = ...,
+        regParam: float = ...,
+        miniBatchFraction: float = ...,
+        initStd: float = ...,
+        maxIter: int = ...,
+        stepSize: float = ...,
+        tol: float = ...,
+        solver: str = ...,
+        thresholds: Optional[Any] = ...,
+        seed: Optional[Any] = ...,
+    ) -> None: ...
+    def setParams(
+        self,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        probabilityCol: str = ...,
+        rawPredictionCol: str = ...,
+        factorSize: int = ...,
+        fitIntercept: bool = ...,
+        fitLinear: bool = ...,
+        regParam: float = ...,
+        miniBatchFraction: float = ...,
+        initStd: float = ...,
+        maxIter: int = ...,
+        stepSize: float = ...,
+        tol: float = ...,
+        solver: str = ...,
+        thresholds: Optional[Any] = ...,
+        seed: Optional[Any] = ...,
+    ) -> FMClassifier: ...
+    def setFactorSize(self, value: int) -> FMClassifier: ...
+    def setFitLinear(self, value: bool) -> FMClassifier: ...
+    def setMiniBatchFraction(self, value: float) -> FMClassifier: ...
+    def setInitStd(self, value: float) -> FMClassifier: ...
+    def setMaxIter(self, value: int) -> FMClassifier: ...
+    def setStepSize(self, value: float) -> FMClassifier: ...
+    def setTol(self, value: float) -> FMClassifier: ...
+    def setSolver(self, value: str) -> FMClassifier: ...
+    def setSeed(self, value: int) -> FMClassifier: ...
+    def setFitIntercept(self, value: bool) -> FMClassifier: ...
+    def setRegParam(self, value: float) -> FMClassifier: ...
+
+class FMClassificationModel(
+    _JavaProbabilisticClassificationModel[Vector],
+    _FactorizationMachinesParams,
+    JavaMLWritable,
+    JavaMLReadable[FMClassificationModel],
+):
+    @property
+    def intercept(self) -> float: ...
+    @property
+    def linear(self) -> Vector: ...
+    @property
+    def factors(self) -> Matrix: ...
+    def summary(self) -> FMClassificationTrainingSummary: ...
+    def evaluate(self, dataset: DataFrame) -> FMClassificationSummary: ...
+
+class FMClassificationSummary(_BinaryClassificationSummary): ...
+class FMClassificationTrainingSummary(FMClassificationSummary, _TrainingSummary): ...
diff --git a/python/pyspark/ml/clustering.py b/python/pyspark/ml/clustering.py
index 54a184bc081ee..f2a248c33883c 100644
--- a/python/pyspark/ml/clustering.py
+++ b/python/pyspark/ml/clustering.py
@@ -19,15 +19,18 @@
 import warnings
 
 from pyspark import since, keyword_only
-from pyspark.ml.util import *
+from pyspark.ml.param.shared import HasMaxIter, HasFeaturesCol, HasSeed, HasPredictionCol, \
+    HasAggregationDepth, HasWeightCol, HasTol, HasProbabilityCol, HasDistanceMeasure, \
+    HasCheckpointInterval, Param, Params, TypeConverters
+from pyspark.ml.util import JavaMLWritable, JavaMLReadable, GeneralJavaMLWritable, \
+    HasTrainingSummary, SparkContext
 from pyspark.ml.wrapper import JavaEstimator, JavaModel, JavaParams, JavaWrapper
-from pyspark.ml.param.shared import *
 from pyspark.ml.common import inherit_doc, _java2py
 from pyspark.ml.stat import MultivariateGaussian
 from pyspark.sql import DataFrame
 
 __all__ = ['BisectingKMeans', 'BisectingKMeansModel', 'BisectingKMeansSummary',
-           'KMeans', 'KMeansModel',
+           'KMeans', 'KMeansModel', 'KMeansSummary',
            'GaussianMixture', 'GaussianMixtureModel', 'GaussianMixtureSummary',
            'LDA', 'LDAModel', 'LocalLDAModel', 'DistributedLDAModel', 'PowerIterationClustering']
 
@@ -98,8 +101,7 @@ def numIter(self):
 
 @inherit_doc
 class _GaussianMixtureParams(HasMaxIter, HasFeaturesCol, HasSeed, HasPredictionCol,
-                             HasProbabilityCol, HasTol, HasAggregationDepth, HasWeightCol,
-                             HasBlockSize):
+                             HasProbabilityCol, HasTol, HasAggregationDepth, HasWeightCol):
     """
     Params for :py:class:`GaussianMixture` and :py:class:`GaussianMixtureModel`.
 
@@ -109,6 +111,10 @@ class _GaussianMixtureParams(HasMaxIter, HasFeaturesCol, HasSeed, HasPredictionC
     k = Param(Params._dummy(), "k", "Number of independent Gaussians in the mixture model. " +
               "Must be > 1.", typeConverter=TypeConverters.toInt)
 
+    def __init__(self, *args):
+        super(_GaussianMixtureParams, self).__init__(*args)
+        self._setDefault(k=2, tol=0.01, maxIter=100, aggregationDepth=2)
+
     @since("2.0.0")
     def getK(self):
         """
@@ -183,7 +189,7 @@ def gaussiansDF(self):
     @since("2.1.0")
     def summary(self):
         """
-        Gets summary (e.g. cluster assignments, cluster sizes) of the model trained on the
+        Gets summary (cluster assignments, cluster sizes) of the model trained on the
         training set. An exception is thrown if no summary exists.
         """
         if self.hasSummary:
@@ -222,11 +228,17 @@ class GaussianMixture(JavaEstimator, _GaussianMixtureParams, JavaMLWritable, Jav
     While this process is generally guaranteed to converge, it is not guaranteed
     to find a global optimum.
 
-    .. note:: For high-dimensional data (with many features), this algorithm may perform poorly.
-          This is due to high-dimensional data (a) making it difficult to cluster at all
-          (based on statistical/theoretical arguments) and (b) numerical issues with
-          Gaussian distributions.
+    .. versionadded:: 2.0.0
+
+    Notes
+    -----
+    For high-dimensional data (with many features), this algorithm may perform poorly.
+    This is due to high-dimensional data (a) making it difficult to cluster at all
+    (based on statistical/theoretical arguments) and (b) numerical issues with
+    Gaussian distributions.
 
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
 
     >>> data = [(Vectors.dense([-0.1, -0.05 ]),),
@@ -244,8 +256,6 @@ class GaussianMixture(JavaEstimator, _GaussianMixtureParams, JavaMLWritable, Jav
     >>> gm.getMaxIter()
     30
     >>> model = gm.fit(df)
-    >>> model.getBlockSize()
-    1
     >>> model.getAggregationDepth()
     2
     >>> model.getFeaturesCol()
@@ -263,8 +273,6 @@ class GaussianMixture(JavaEstimator, _GaussianMixtureParams, JavaMLWritable, Jav
     3
     >>> summary.clusterSizes
     [2, 2, 2]
-    >>> summary.logLikelihood
-    65.02945...
     >>> weights = model.weights
     >>> len(weights)
     3
@@ -321,25 +329,24 @@ class GaussianMixture(JavaEstimator, _GaussianMixtureParams, JavaMLWritable, Jav
     Row(mean=DenseVector([0.825, 0.8675]))
     >>> model2.gaussiansDF.select("cov").head()
     Row(cov=DenseMatrix(2, 2, [0.0056, -0.0051, -0.0051, 0.0046], False))
+    >>> model.transform(df).take(1) == model2.transform(df).take(1)
+    True
     >>> gm2.setWeightCol("weight")
     GaussianMixture...
-
-    .. versionadded:: 2.0.0
     """
 
     @keyword_only
-    def __init__(self, featuresCol="features", predictionCol="prediction", k=2,
+    def __init__(self, *, featuresCol="features", predictionCol="prediction", k=2,
                  probabilityCol="probability", tol=0.01, maxIter=100, seed=None,
-                 aggregationDepth=2, weightCol=None, blockSize=1):
+                 aggregationDepth=2, weightCol=None):
         """
-        __init__(self, featuresCol="features", predictionCol="prediction", k=2, \
+        __init__(self, \\*, featuresCol="features", predictionCol="prediction", k=2, \
                  probabilityCol="probability", tol=0.01, maxIter=100, seed=None, \
-                 aggregationDepth=2, weightCol=None, blockSize=1)
+                 aggregationDepth=2, weightCol=None)
         """
         super(GaussianMixture, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.clustering.GaussianMixture",
                                             self.uid)
-        self._setDefault(k=2, tol=0.01, maxIter=100, aggregationDepth=2, blockSize=1)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -348,13 +355,13 @@ def _create_model(self, java_model):
 
     @keyword_only
     @since("2.0.0")
-    def setParams(self, featuresCol="features", predictionCol="prediction", k=2,
+    def setParams(self, *, featuresCol="features", predictionCol="prediction", k=2,
                   probabilityCol="probability", tol=0.01, maxIter=100, seed=None,
-                  aggregationDepth=2, weightCol=None, blockSize=1):
+                  aggregationDepth=2, weightCol=None):
         """
-        setParams(self, featuresCol="features", predictionCol="prediction", k=2, \
+        setParams(self, \\*, featuresCol="features", predictionCol="prediction", k=2, \
                   probabilityCol="probability", tol=0.01, maxIter=100, seed=None, \
-                  aggregationDepth=2, weightCol=None, blockSize=1)
+                  aggregationDepth=2, weightCol=None)
 
         Sets params for GaussianMixture.
         """
@@ -424,13 +431,6 @@ def setAggregationDepth(self, value):
         """
         return self._set(aggregationDepth=value)
 
-    @since("3.1.0")
-    def setBlockSize(self, value):
-        """
-        Sets the value of :py:attr:`blockSize`.
-        """
-        return self._set(blockSize=value)
-
 
 class GaussianMixtureSummary(ClusteringSummary):
     """
@@ -500,6 +500,11 @@ class _KMeansParams(HasMaxIter, HasFeaturesCol, HasSeed, HasPredictionCol, HasTo
     initSteps = Param(Params._dummy(), "initSteps", "The number of steps for k-means|| " +
                       "initialization mode. Must be > 0.", typeConverter=TypeConverters.toInt)
 
+    def __init__(self, *args):
+        super(_KMeansParams, self).__init__(*args)
+        self._setDefault(k=2, initMode="k-means||", initSteps=2, tol=1e-4, maxIter=20,
+                         distanceMeasure="euclidean")
+
     @since("1.5.0")
     def getK(self):
         """
@@ -553,7 +558,7 @@ def clusterCenters(self):
     @since("2.1.0")
     def summary(self):
         """
-        Gets summary (e.g. cluster assignments, cluster sizes) of the model trained on the
+        Gets summary (cluster assignments, cluster sizes) of the model trained on the
         training set. An exception is thrown if no summary exists.
         """
         if self.hasSummary:
@@ -576,6 +581,10 @@ class KMeans(JavaEstimator, _KMeansParams, JavaMLWritable, JavaMLReadable):
     K-means clustering with a k-means++ like initialization mode
     (the k-means|| algorithm by Bahmani et al).
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> data = [(Vectors.dense([0.0, 0.0]), 2.0), (Vectors.dense([1.0, 1.0]), 2.0),
     ...         (Vectors.dense([9.0, 8.0]), 2.0), (Vectors.dense([8.0, 9.0]), 2.0)]
@@ -629,23 +638,21 @@ class KMeans(JavaEstimator, _KMeansParams, JavaMLWritable, JavaMLReadable):
     array([ True,  True], dtype=bool)
     >>> model.clusterCenters()[1] == model2.clusterCenters()[1]
     array([ True,  True], dtype=bool)
-
-    .. versionadded:: 1.5.0
+    >>> model.transform(df).take(1) == model2.transform(df).take(1)
+    True
     """
 
     @keyword_only
-    def __init__(self, featuresCol="features", predictionCol="prediction", k=2,
+    def __init__(self, *, featuresCol="features", predictionCol="prediction", k=2,
                  initMode="k-means||", initSteps=2, tol=1e-4, maxIter=20, seed=None,
                  distanceMeasure="euclidean", weightCol=None):
         """
-        __init__(self, featuresCol="features", predictionCol="prediction", k=2, \
+        __init__(self, \\*, featuresCol="features", predictionCol="prediction", k=2, \
                  initMode="k-means||", initSteps=2, tol=1e-4, maxIter=20, seed=None, \
                  distanceMeasure="euclidean", weightCol=None)
         """
         super(KMeans, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.clustering.KMeans", self.uid)
-        self._setDefault(k=2, initMode="k-means||", initSteps=2, tol=1e-4, maxIter=20,
-                         distanceMeasure="euclidean")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -654,11 +661,11 @@ def _create_model(self, java_model):
 
     @keyword_only
     @since("1.5.0")
-    def setParams(self, featuresCol="features", predictionCol="prediction", k=2,
+    def setParams(self, *, featuresCol="features", predictionCol="prediction", k=2,
                   initMode="k-means||", initSteps=2, tol=1e-4, maxIter=20, seed=None,
                   distanceMeasure="euclidean", weightCol=None):
         """
-        setParams(self, featuresCol="features", predictionCol="prediction", k=2, \
+        setParams(self, \\*, featuresCol="features", predictionCol="prediction", k=2, \
                   initMode="k-means||", initSteps=2, tol=1e-4, maxIter=20, seed=None, \
                   distanceMeasure="euclidean", weightCol=None)
 
@@ -754,6 +761,10 @@ class _BisectingKMeansParams(HasMaxIter, HasFeaturesCol, HasSeed, HasPredictionC
                                     "proportion of points (if < 1.0) of a divisible cluster.",
                                     typeConverter=TypeConverters.toFloat)
 
+    def __init__(self, *args):
+        super(_BisectingKMeansParams, self).__init__(*args)
+        self._setDefault(maxIter=20, k=4, minDivisibleClusterSize=1.0)
+
     @since("2.0.0")
     def getK(self):
         """
@@ -802,20 +813,20 @@ def computeCost(self, dataset):
         Computes the sum of squared distances between the input points
         and their corresponding cluster centers.
 
-        .. note:: Deprecated in 3.0.0. It will be removed in future versions. Use
-           ClusteringEvaluator instead. You can also get the cost on the training dataset in the
-           summary.
+        .. deprecated:: 3.0.0
+            It will be removed in future versions. Use :py:class:`ClusteringEvaluator` instead.
+            You can also get the cost on the training dataset in the summary.
         """
         warnings.warn("Deprecated in 3.0.0. It will be removed in future versions. Use "
                       "ClusteringEvaluator instead. You can also get the cost on the training "
-                      "dataset in the summary.", DeprecationWarning)
+                      "dataset in the summary.", FutureWarning)
         return self._call_java("computeCost", dataset)
 
     @property
     @since("2.1.0")
     def summary(self):
         """
-        Gets summary (e.g. cluster assignments, cluster sizes) of the model trained on the
+        Gets summary (cluster assignments, cluster sizes) of the model trained on the
         training set. An exception is thrown if no summary exists.
         """
         if self.hasSummary:
@@ -844,6 +855,10 @@ class BisectingKMeans(JavaEstimator, _BisectingKMeansParams, JavaMLWritable, Jav
     If bisecting all divisible clusters on the bottom level would result more than `k` leaf
     clusters, larger clusters get higher priority.
 
+    .. versionadded:: 2.0.0
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> data = [(Vectors.dense([0.0, 0.0]), 2.0), (Vectors.dense([1.0, 1.0]), 2.0),
     ...         (Vectors.dense([9.0, 8.0]), 2.0), (Vectors.dense([8.0, 9.0]), 2.0)]
@@ -904,33 +919,32 @@ class BisectingKMeans(JavaEstimator, _BisectingKMeansParams, JavaMLWritable, Jav
     array([ True,  True], dtype=bool)
     >>> model.clusterCenters()[1] == model2.clusterCenters()[1]
     array([ True,  True], dtype=bool)
-
-    .. versionadded:: 2.0.0
+    >>> model.transform(df).take(1) == model2.transform(df).take(1)
+    True
     """
 
     @keyword_only
-    def __init__(self, featuresCol="features", predictionCol="prediction", maxIter=20,
+    def __init__(self, *, featuresCol="features", predictionCol="prediction", maxIter=20,
                  seed=None, k=4, minDivisibleClusterSize=1.0, distanceMeasure="euclidean",
                  weightCol=None):
         """
-        __init__(self, featuresCol="features", predictionCol="prediction", maxIter=20, \
+        __init__(self, \\*, featuresCol="features", predictionCol="prediction", maxIter=20, \
                  seed=None, k=4, minDivisibleClusterSize=1.0, distanceMeasure="euclidean", \
                  weightCol=None)
         """
         super(BisectingKMeans, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.clustering.BisectingKMeans",
                                             self.uid)
-        self._setDefault(maxIter=20, k=4, minDivisibleClusterSize=1.0)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("2.0.0")
-    def setParams(self, featuresCol="features", predictionCol="prediction", maxIter=20,
+    def setParams(self, *, featuresCol="features", predictionCol="prediction", maxIter=20,
                   seed=None, k=4, minDivisibleClusterSize=1.0, distanceMeasure="euclidean",
                   weightCol=None):
         """
-        setParams(self, featuresCol="features", predictionCol="prediction", maxIter=20, \
+        setParams(self, \\*, featuresCol="features", predictionCol="prediction", maxIter=20, \
                   seed=None, k=4, minDivisibleClusterSize=1.0, distanceMeasure="euclidean", \
                   weightCol=None)
         Sets params for BisectingKMeans.
@@ -1063,6 +1077,13 @@ class _LDAParams(HasMaxIter, HasFeaturesCol, HasSeed, HasCheckpointInterval):
                                " partition is lost, so set this bit with care.",
                                TypeConverters.toBoolean)
 
+    def __init__(self, *args):
+        super(_LDAParams, self).__init__(*args)
+        self._setDefault(maxIter=20, checkpointInterval=10,
+                         k=10, optimizer="online", learningOffset=1024.0, learningDecay=0.51,
+                         subsamplingRate=0.05, optimizeDocConcentration=True,
+                         topicDistributionCol="topicDistribution", keepLastCheckpoint=True)
+
     @since("2.0.0")
     def getK(self):
         """
@@ -1184,9 +1205,10 @@ def topicsMatrix(self):
         This is a matrix of size vocabSize x k, where each column is a topic.
         No guarantees are given about the ordering of the topics.
 
-        WARNING: If this model is actually a :py:class:`DistributedLDAModel` instance produced by
-        the Expectation-Maximization ("em") `optimizer`, then this method could involve
-        collecting a large amount of data to the driver (on the order of vocabSize x k).
+        .. warning:: If this model is actually a :py:class:`DistributedLDAModel`
+            instance produced by the Expectation-Maximization ("em") `optimizer`,
+            then this method could involve collecting a large amount of data
+            to the driver (on the order of vocabSize x k).
         """
         return self._call_java("topicsMatrix")
 
@@ -1196,9 +1218,9 @@ def logLikelihood(self, dataset):
         Calculates a lower bound on the log likelihood of the entire corpus.
         See Equation (16) in the Online LDA paper (Hoffman et al., 2010).
 
-        WARNING: If this model is an instance of :py:class:`DistributedLDAModel` (produced when
-        :py:attr:`optimizer` is set to "em"), this involves collecting a large
-        :py:func:`topicsMatrix` to the driver. This implementation may be changed in the future.
+        .. warning:: If this model is an instance of :py:class:`DistributedLDAModel` (produced when
+            :py:attr:`optimizer` is set to "em"), this involves collecting a large
+            :py:func:`topicsMatrix` to the driver. This implementation may be changed in the future.
         """
         return self._call_java("logLikelihood", dataset)
 
@@ -1208,9 +1230,9 @@ def logPerplexity(self, dataset):
         Calculate an upper bound on perplexity.  (Lower is better.)
         See Equation (16) in the Online LDA paper (Hoffman et al., 2010).
 
-        WARNING: If this model is an instance of :py:class:`DistributedLDAModel` (produced when
-        :py:attr:`optimizer` is set to "em"), this involves collecting a large
-        :py:func:`topicsMatrix` to the driver. This implementation may be changed in the future.
+        .. warning:: If this model is an instance of :py:class:`DistributedLDAModel` (produced when
+            :py:attr:`optimizer` is set to "em"), this involves collecting a large
+            :py:func:`topicsMatrix` to the driver. This implementation may be changed in the future.
         """
         return self._call_java("logPerplexity", dataset)
 
@@ -1249,7 +1271,7 @@ def toLocal(self):
         Convert this distributed model to a local representation.  This discards info about the
         training dataset.
 
-        WARNING: This involves collecting a large :py:func:`topicsMatrix` to the driver.
+        .. warning:: This involves collecting a large :py:func:`topicsMatrix` to the driver.
         """
         model = LocalLDAModel(self._call_java("toLocal"))
 
@@ -1266,11 +1288,12 @@ def trainingLogLikelihood(self):
         given the current parameter estimates:
         log P(docs | topics, topic distributions for docs, Dirichlet hyperparameters)
 
-        Notes:
-          - This excludes the prior; for that, use :py:func:`logPrior`.
-          - Even with :py:func:`logPrior`, this is NOT the same as the data log likelihood given
+        Notes
+        -----
+        - This excludes the prior; for that, use :py:func:`logPrior`.
+        - Even with :py:func:`logPrior`, this is NOT the same as the data log likelihood given
             the hyperparameters.
-          - This is computed from the topic distributions computed during training. If you call
+        - This is computed from the topic distributions computed during training. If you call
             :py:func:`logLikelihood` on the same training dataset, the topic distributions
             will be computed again, possibly giving different results.
         """
@@ -1284,17 +1307,23 @@ def logPrior(self):
         """
         return self._call_java("logPrior")
 
-    @since("2.0.0")
     def getCheckpointFiles(self):
         """
         If using checkpointing and :py:attr:`LDA.keepLastCheckpoint` is set to true, then there may
         be saved checkpoint files.  This method is provided so that users can manage those files.
 
-        .. note:: Removing the checkpoints can cause failures if a partition is lost and is needed
-            by certain :py:class:`DistributedLDAModel` methods.  Reference counting will clean up
-            the checkpoints when this model and derivative data go out of scope.
+        .. versionadded:: 2.0.0
 
-        :return  List of checkpoint files from training
+        Returns
+        -------
+        list
+            List of checkpoint files from training
+
+        Notes
+        -----
+        Removing the checkpoints can cause failures if a partition is lost and is needed
+        by certain :py:class:`DistributedLDAModel` methods.  Reference counting will clean up
+        the checkpoints when this model and derivative data go out of scope.
         """
         return self._call_java("getCheckpointFiles")
 
@@ -1332,6 +1361,10 @@ class LDA(JavaEstimator, _LDAParams, JavaMLReadable, JavaMLWritable):
     :py:class:`pyspark.ml.feature.Tokenizer` and :py:class:`pyspark.ml.feature.CountVectorizer`
     can be useful for converting text to word count vectors.
 
+    .. versionadded:: 2.0.0
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors, SparseVector
     >>> from pyspark.ml.clustering import LDA
     >>> df = spark.createDataFrame([[1, Vectors.dense([0.0, 1.0])],
@@ -1373,18 +1406,18 @@ class LDA(JavaEstimator, _LDAParams, JavaMLReadable, JavaMLWritable):
     >>> local_model_path = temp_path + "/lda_local_model"
     >>> localModel.save(local_model_path)
     >>> sameLocalModel = LocalLDAModel.load(local_model_path)
-
-    .. versionadded:: 2.0.0
+    >>> model.transform(df).take(1) == sameLocalModel.transform(df).take(1)
+    True
     """
 
     @keyword_only
-    def __init__(self, featuresCol="features", maxIter=20, seed=None, checkpointInterval=10,
+    def __init__(self, *, featuresCol="features", maxIter=20, seed=None, checkpointInterval=10,
                  k=10, optimizer="online", learningOffset=1024.0, learningDecay=0.51,
                  subsamplingRate=0.05, optimizeDocConcentration=True,
                  docConcentration=None, topicConcentration=None,
                  topicDistributionCol="topicDistribution", keepLastCheckpoint=True):
         """
-        __init__(self, featuresCol="features", maxIter=20, seed=None, checkpointInterval=10,\
+        __init__(self, \\*, featuresCol="features", maxIter=20, seed=None, checkpointInterval=10,\
                   k=10, optimizer="online", learningOffset=1024.0, learningDecay=0.51,\
                   subsamplingRate=0.05, optimizeDocConcentration=True,\
                   docConcentration=None, topicConcentration=None,\
@@ -1392,10 +1425,6 @@ def __init__(self, featuresCol="features", maxIter=20, seed=None, checkpointInte
         """
         super(LDA, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.clustering.LDA", self.uid)
-        self._setDefault(maxIter=20, checkpointInterval=10,
-                         k=10, optimizer="online", learningOffset=1024.0, learningDecay=0.51,
-                         subsamplingRate=0.05, optimizeDocConcentration=True,
-                         topicDistributionCol="topicDistribution", keepLastCheckpoint=True)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -1407,13 +1436,13 @@ def _create_model(self, java_model):
 
     @keyword_only
     @since("2.0.0")
-    def setParams(self, featuresCol="features", maxIter=20, seed=None, checkpointInterval=10,
+    def setParams(self, *, featuresCol="features", maxIter=20, seed=None, checkpointInterval=10,
                   k=10, optimizer="online", learningOffset=1024.0, learningDecay=0.51,
                   subsamplingRate=0.05, optimizeDocConcentration=True,
                   docConcentration=None, topicConcentration=None,
                   topicDistributionCol="topicDistribution", keepLastCheckpoint=True):
         """
-        setParams(self, featuresCol="features", maxIter=20, seed=None, checkpointInterval=10,\
+        setParams(self, \\*, featuresCol="features", maxIter=20, seed=None, checkpointInterval=10,\
                   k=10, optimizer="online", learningOffset=1024.0, learningDecay=0.51,\
                   subsamplingRate=0.05, optimizeDocConcentration=True,\
                   docConcentration=None, topicConcentration=None,\
@@ -1455,6 +1484,8 @@ def setOptimizer(self, value):
         Sets the value of :py:attr:`optimizer`.
         Currently only support 'em' and 'online'.
 
+        Examples
+        --------
         >>> algo = LDA().setOptimizer("em")
         >>> algo.getOptimizer()
         'em'
@@ -1466,6 +1497,8 @@ def setLearningOffset(self, value):
         """
         Sets the value of :py:attr:`learningOffset`.
 
+        Examples
+        --------
         >>> algo = LDA().setLearningOffset(100)
         >>> algo.getLearningOffset()
         100.0
@@ -1477,6 +1510,8 @@ def setLearningDecay(self, value):
         """
         Sets the value of :py:attr:`learningDecay`.
 
+        Examples
+        --------
         >>> algo = LDA().setLearningDecay(0.1)
         >>> algo.getLearningDecay()
         0.1...
@@ -1488,6 +1523,8 @@ def setSubsamplingRate(self, value):
         """
         Sets the value of :py:attr:`subsamplingRate`.
 
+        Examples
+        --------
         >>> algo = LDA().setSubsamplingRate(0.1)
         >>> algo.getSubsamplingRate()
         0.1...
@@ -1499,6 +1536,8 @@ def setOptimizeDocConcentration(self, value):
         """
         Sets the value of :py:attr:`optimizeDocConcentration`.
 
+        Examples
+        --------
         >>> algo = LDA().setOptimizeDocConcentration(True)
         >>> algo.getOptimizeDocConcentration()
         True
@@ -1510,6 +1549,8 @@ def setDocConcentration(self, value):
         """
         Sets the value of :py:attr:`docConcentration`.
 
+        Examples
+        --------
         >>> algo = LDA().setDocConcentration([0.1, 0.2])
         >>> algo.getDocConcentration()
         [0.1..., 0.2...]
@@ -1521,6 +1562,8 @@ def setTopicConcentration(self, value):
         """
         Sets the value of :py:attr:`topicConcentration`.
 
+        Examples
+        --------
         >>> algo = LDA().setTopicConcentration(0.5)
         >>> algo.getTopicConcentration()
         0.5...
@@ -1532,6 +1575,8 @@ def setTopicDistributionCol(self, value):
         """
         Sets the value of :py:attr:`topicDistributionCol`.
 
+        Examples
+        --------
         >>> algo = LDA().setTopicDistributionCol("topicDistributionCol")
         >>> algo.getTopicDistributionCol()
         'topicDistributionCol'
@@ -1543,6 +1588,8 @@ def setKeepLastCheckpoint(self, value):
         """
         Sets the value of :py:attr:`keepLastCheckpoint`.
 
+        Examples
+        --------
         >>> algo = LDA().setKeepLastCheckpoint(False)
         >>> algo.getKeepLastCheckpoint()
         False
@@ -1588,6 +1635,10 @@ class _PowerIterationClusteringParams(HasMaxIter, HasWeightCol):
                    "Name of the input column for destination vertex IDs.",
                    typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_PowerIterationClusteringParams, self).__init__(*args)
+        self._setDefault(k=2, maxIter=20, initMode="random", srcCol="src", dstCol="dst")
+
     @since("2.4.0")
     def getK(self):
         """
@@ -1629,9 +1680,14 @@ class PowerIterationClustering(_PowerIterationClusteringParams, JavaParams, Java
     This class is not yet an Estimator/Transformer, use :py:func:`assignClusters` method
     to run the PowerIterationClustering algorithm.
 
-    .. seealso:: `Wikipedia on Spectral clustering
-        <http://en.wikipedia.org/wiki/Spectral_clustering>`_
+    .. versionadded:: 2.4.0
 
+    Notes
+    -----
+    See `Wikipedia on Spectral clustering <http://en.wikipedia.org/wiki/Spectral_clustering>`_
+
+    Examples
+    --------
     >>> data = [(1, 0, 0.5),
     ...         (2, 0, 0.5), (2, 1, 0.7),
     ...         (3, 0, 0.5), (3, 1, 0.7), (3, 2, 0.9),
@@ -1661,30 +1717,29 @@ class PowerIterationClustering(_PowerIterationClusteringParams, JavaParams, Java
     2
     >>> pic2.getMaxIter()
     40
-
-    .. versionadded:: 2.4.0
+    >>> pic2.assignClusters(df).take(6) == assignments.take(6)
+    True
     """
 
     @keyword_only
-    def __init__(self, k=2, maxIter=20, initMode="random", srcCol="src", dstCol="dst",
+    def __init__(self, *, k=2, maxIter=20, initMode="random", srcCol="src", dstCol="dst",
                  weightCol=None):
         """
-        __init__(self, k=2, maxIter=20, initMode="random", srcCol="src", dstCol="dst",\
+        __init__(self, \\*, k=2, maxIter=20, initMode="random", srcCol="src", dstCol="dst",\
                  weightCol=None)
         """
         super(PowerIterationClustering, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.clustering.PowerIterationClustering", self.uid)
-        self._setDefault(k=2, maxIter=20, initMode="random", srcCol="src", dstCol="dst")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("2.4.0")
-    def setParams(self, k=2, maxIter=20, initMode="random", srcCol="src", dstCol="dst",
+    def setParams(self, *, k=2, maxIter=20, initMode="random", srcCol="src", dstCol="dst",
                   weightCol=None):
         """
-        setParams(self, k=2, maxIter=20, initMode="random", srcCol="src", dstCol="dst",\
+        setParams(self, \\*, k=2, maxIter=20, initMode="random", srcCol="src", dstCol="dst",\
                   weightCol=None)
         Sets params for PowerIterationClustering.
         """
@@ -1738,7 +1793,9 @@ def assignClusters(self, dataset):
         """
         Run the PIC algorithm and returns a cluster assignment for each input vertex.
 
-        :param dataset:
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
           A dataset with columns src, dst, weight representing the affinity matrix,
           which is the matrix A in the PIC paper. Suppose the src column value is i,
           the dst column value is j, the weight column value is similarity s,,ij,,
@@ -1747,13 +1804,13 @@ def assignClusters(self, dataset):
           either (i, j, s,,ij,,) or (j, i, s,,ji,,) in the input. Rows with i = j are
           ignored, because we assume s,,ij,, = 0.0.
 
-        :return:
-          A dataset that contains columns of vertex id and the corresponding cluster for
-          the id. The schema of it will be:
-          - id: Long
-          - cluster: Int
-
-        .. versionadded:: 2.4.0
+        Returns
+        -------
+        :py:class:`pyspark.sql.DataFrame`
+            A dataset that contains columns of vertex id and the corresponding cluster for
+            the id. The schema of it will be:
+            - id: Long
+            - cluster: Int
         """
         self._transfer_params_to_java()
         jdf = self._java_obj.assignClusters(dataset._jdf)
diff --git a/python/pyspark/ml/clustering.pyi b/python/pyspark/ml/clustering.pyi
new file mode 100644
index 0000000000000..e2a2d7e888367
--- /dev/null
+++ b/python/pyspark/ml/clustering.pyi
@@ -0,0 +1,437 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Any, List, Optional
+
+from pyspark.ml.linalg import Matrix, Vector
+from pyspark.ml.util import (
+    GeneralJavaMLWritable,
+    HasTrainingSummary,
+    JavaMLReadable,
+    JavaMLWritable,
+)
+from pyspark.ml.wrapper import JavaEstimator, JavaModel, JavaParams, JavaWrapper
+from pyspark.ml.param.shared import (
+    HasAggregationDepth,
+    HasCheckpointInterval,
+    HasDistanceMeasure,
+    HasFeaturesCol,
+    HasMaxIter,
+    HasPredictionCol,
+    HasProbabilityCol,
+    HasSeed,
+    HasTol,
+    HasWeightCol,
+)
+
+from pyspark.ml.param import Param
+from pyspark.ml.stat import MultivariateGaussian
+from pyspark.sql.dataframe import DataFrame
+
+from numpy import ndarray  # type: ignore[import]
+
+class ClusteringSummary(JavaWrapper):
+    @property
+    def predictionCol(self) -> str: ...
+    @property
+    def predictions(self) -> DataFrame: ...
+    @property
+    def featuresCol(self) -> str: ...
+    @property
+    def k(self) -> int: ...
+    @property
+    def cluster(self) -> DataFrame: ...
+    @property
+    def clusterSizes(self) -> List[int]: ...
+    @property
+    def numIter(self) -> int: ...
+
+class _GaussianMixtureParams(
+    HasMaxIter,
+    HasFeaturesCol,
+    HasSeed,
+    HasPredictionCol,
+    HasProbabilityCol,
+    HasTol,
+    HasAggregationDepth,
+    HasWeightCol,
+):
+    k: Param[int]
+    def __init__(self, *args: Any): ...
+    def getK(self) -> int: ...
+
+class GaussianMixtureModel(
+    JavaModel,
+    _GaussianMixtureParams,
+    JavaMLWritable,
+    JavaMLReadable[GaussianMixtureModel],
+    HasTrainingSummary[GaussianMixtureSummary],
+):
+    def setFeaturesCol(self, value: str) -> GaussianMixtureModel: ...
+    def setPredictionCol(self, value: str) -> GaussianMixtureModel: ...
+    def setProbabilityCol(self, value: str) -> GaussianMixtureModel: ...
+    @property
+    def weights(self) -> List[float]: ...
+    @property
+    def gaussians(self) -> List[MultivariateGaussian]: ...
+    @property
+    def gaussiansDF(self) -> DataFrame: ...
+    @property
+    def summary(self) -> GaussianMixtureSummary: ...
+    def predict(self, value: Vector) -> int: ...
+    def predictProbability(self, value: Vector) -> Vector: ...
+
+class GaussianMixture(
+    JavaEstimator[GaussianMixtureModel],
+    _GaussianMixtureParams,
+    JavaMLWritable,
+    JavaMLReadable[GaussianMixture],
+):
+    def __init__(
+        self,
+        *,
+        featuresCol: str = ...,
+        predictionCol: str = ...,
+        k: int = ...,
+        probabilityCol: str = ...,
+        tol: float = ...,
+        maxIter: int = ...,
+        seed: Optional[int] = ...,
+        aggregationDepth: int = ...,
+        weightCol: Optional[str] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        featuresCol: str = ...,
+        predictionCol: str = ...,
+        k: int = ...,
+        probabilityCol: str = ...,
+        tol: float = ...,
+        maxIter: int = ...,
+        seed: Optional[int] = ...,
+        aggregationDepth: int = ...,
+        weightCol: Optional[str] = ...
+    ) -> GaussianMixture: ...
+    def setK(self, value: int) -> GaussianMixture: ...
+    def setMaxIter(self, value: int) -> GaussianMixture: ...
+    def setFeaturesCol(self, value: str) -> GaussianMixture: ...
+    def setPredictionCol(self, value: str) -> GaussianMixture: ...
+    def setProbabilityCol(self, value: str) -> GaussianMixture: ...
+    def setWeightCol(self, value: str) -> GaussianMixture: ...
+    def setSeed(self, value: int) -> GaussianMixture: ...
+    def setTol(self, value: float) -> GaussianMixture: ...
+    def setAggregationDepth(self, value: int) -> GaussianMixture: ...
+
+class GaussianMixtureSummary(ClusteringSummary):
+    @property
+    def probabilityCol(self) -> str: ...
+    @property
+    def probability(self) -> DataFrame: ...
+    @property
+    def logLikelihood(self) -> float: ...
+
+class KMeansSummary(ClusteringSummary):
+    def trainingCost(self) -> float: ...
+
+class _KMeansParams(
+    HasMaxIter,
+    HasFeaturesCol,
+    HasSeed,
+    HasPredictionCol,
+    HasTol,
+    HasDistanceMeasure,
+    HasWeightCol,
+):
+    k: Param[int]
+    initMode: Param[str]
+    initSteps: Param[int]
+    def __init__(self, *args: Any): ...
+    def getK(self) -> int: ...
+    def getInitMode(self) -> str: ...
+    def getInitSteps(self) -> int: ...
+
+class KMeansModel(
+    JavaModel,
+    _KMeansParams,
+    GeneralJavaMLWritable,
+    JavaMLReadable[KMeansModel],
+    HasTrainingSummary[KMeansSummary],
+):
+    def setFeaturesCol(self, value: str) -> KMeansModel: ...
+    def setPredictionCol(self, value: str) -> KMeansModel: ...
+    def clusterCenters(self) -> List[ndarray]: ...
+    @property
+    def summary(self) -> KMeansSummary: ...
+    def predict(self, value: Vector) -> int: ...
+
+class KMeans(
+    JavaEstimator[KMeansModel], _KMeansParams, JavaMLWritable, JavaMLReadable[KMeans]
+):
+    def __init__(
+        self,
+        *,
+        featuresCol: str = ...,
+        predictionCol: str = ...,
+        k: int = ...,
+        initMode: str = ...,
+        initSteps: int = ...,
+        tol: float = ...,
+        maxIter: int = ...,
+        seed: Optional[int] = ...,
+        distanceMeasure: str = ...,
+        weightCol: Optional[str] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        featuresCol: str = ...,
+        predictionCol: str = ...,
+        k: int = ...,
+        initMode: str = ...,
+        initSteps: int = ...,
+        tol: float = ...,
+        maxIter: int = ...,
+        seed: Optional[int] = ...,
+        distanceMeasure: str = ...,
+        weightCol: Optional[str] = ...
+    ) -> KMeans: ...
+    def setK(self, value: int) -> KMeans: ...
+    def setInitMode(self, value: str) -> KMeans: ...
+    def setInitSteps(self, value: int) -> KMeans: ...
+    def setDistanceMeasure(self, value: str) -> KMeans: ...
+    def setMaxIter(self, value: int) -> KMeans: ...
+    def setFeaturesCol(self, value: str) -> KMeans: ...
+    def setPredictionCol(self, value: str) -> KMeans: ...
+    def setSeed(self, value: int) -> KMeans: ...
+    def setTol(self, value: float) -> KMeans: ...
+    def setWeightCol(self, value: str) -> KMeans: ...
+
+class _BisectingKMeansParams(
+    HasMaxIter,
+    HasFeaturesCol,
+    HasSeed,
+    HasPredictionCol,
+    HasDistanceMeasure,
+    HasWeightCol,
+):
+    k: Param[int]
+    minDivisibleClusterSize: Param[float]
+    def __init__(self, *args: Any): ...
+    def getK(self) -> int: ...
+    def getMinDivisibleClusterSize(self) -> float: ...
+
+class BisectingKMeansModel(
+    JavaModel,
+    _BisectingKMeansParams,
+    JavaMLWritable,
+    JavaMLReadable[BisectingKMeansModel],
+    HasTrainingSummary[BisectingKMeansSummary],
+):
+    def setFeaturesCol(self, value: str) -> BisectingKMeansModel: ...
+    def setPredictionCol(self, value: str) -> BisectingKMeansModel: ...
+    def clusterCenters(self) -> List[ndarray]: ...
+    def computeCost(self, dataset: DataFrame) -> float: ...
+    @property
+    def summary(self) -> BisectingKMeansSummary: ...
+    def predict(self, value: Vector) -> int: ...
+
+class BisectingKMeans(
+    JavaEstimator[BisectingKMeansModel],
+    _BisectingKMeansParams,
+    JavaMLWritable,
+    JavaMLReadable[BisectingKMeans],
+):
+    def __init__(
+        self,
+        *,
+        featuresCol: str = ...,
+        predictionCol: str = ...,
+        maxIter: int = ...,
+        seed: Optional[int] = ...,
+        k: int = ...,
+        minDivisibleClusterSize: float = ...,
+        distanceMeasure: str = ...,
+        weightCol: Optional[str] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        featuresCol: str = ...,
+        predictionCol: str = ...,
+        maxIter: int = ...,
+        seed: Optional[int] = ...,
+        k: int = ...,
+        minDivisibleClusterSize: float = ...,
+        distanceMeasure: str = ...,
+        weightCol: Optional[str] = ...
+    ) -> BisectingKMeans: ...
+    def setK(self, value: int) -> BisectingKMeans: ...
+    def setMinDivisibleClusterSize(self, value: float) -> BisectingKMeans: ...
+    def setDistanceMeasure(self, value: str) -> BisectingKMeans: ...
+    def setMaxIter(self, value: int) -> BisectingKMeans: ...
+    def setFeaturesCol(self, value: str) -> BisectingKMeans: ...
+    def setPredictionCol(self, value: str) -> BisectingKMeans: ...
+    def setSeed(self, value: int) -> BisectingKMeans: ...
+    def setWeightCol(self, value: str) -> BisectingKMeans: ...
+
+class BisectingKMeansSummary(ClusteringSummary):
+    @property
+    def trainingCost(self) -> float: ...
+
+class _LDAParams(HasMaxIter, HasFeaturesCol, HasSeed, HasCheckpointInterval):
+    k: Param[int]
+    optimizer: Param[str]
+    learningOffset: Param[float]
+    learningDecay: Param[float]
+    subsamplingRate: Param[float]
+    optimizeDocConcentration: Param[bool]
+    docConcentration: Param[List[float]]
+    topicConcentration: Param[float]
+    topicDistributionCol: Param[str]
+    keepLastCheckpoint: Param[bool]
+    def __init__(self, *args: Any): ...
+    def setK(self, value: int) -> LDA: ...
+    def getOptimizer(self) -> str: ...
+    def getLearningOffset(self) -> float: ...
+    def getLearningDecay(self) -> float: ...
+    def getSubsamplingRate(self) -> float: ...
+    def getOptimizeDocConcentration(self) -> bool: ...
+    def getDocConcentration(self) -> List[float]: ...
+    def getTopicConcentration(self) -> float: ...
+    def getTopicDistributionCol(self) -> str: ...
+    def getKeepLastCheckpoint(self) -> bool: ...
+
+class LDAModel(JavaModel, _LDAParams):
+    def setFeaturesCol(self, value: str) -> LDAModel: ...
+    def setSeed(self, value: int) -> LDAModel: ...
+    def setTopicDistributionCol(self, value: str) -> LDAModel: ...
+    def isDistributed(self) -> bool: ...
+    def vocabSize(self) -> int: ...
+    def topicsMatrix(self) -> Matrix: ...
+    def logLikelihood(self, dataset: DataFrame) -> float: ...
+    def logPerplexity(self, dataset: DataFrame) -> float: ...
+    def describeTopics(self, maxTermsPerTopic: int = ...) -> DataFrame: ...
+    def estimatedDocConcentration(self) -> Vector: ...
+
+class DistributedLDAModel(
+    LDAModel, JavaMLReadable[DistributedLDAModel], JavaMLWritable
+):
+    def toLocal(self) -> LDAModel: ...
+    def trainingLogLikelihood(self) -> float: ...
+    def logPrior(self) -> float: ...
+    def getCheckpointFiles(self) -> List[str]: ...
+
+class LocalLDAModel(LDAModel, JavaMLReadable[LocalLDAModel], JavaMLWritable): ...
+
+class LDA(JavaEstimator[LDAModel], _LDAParams, JavaMLReadable[LDA], JavaMLWritable):
+    def __init__(
+        self,
+        *,
+        featuresCol: str = ...,
+        maxIter: int = ...,
+        seed: Optional[int] = ...,
+        checkpointInterval: int = ...,
+        k: int = ...,
+        optimizer: str = ...,
+        learningOffset: float = ...,
+        learningDecay: float = ...,
+        subsamplingRate: float = ...,
+        optimizeDocConcentration: bool = ...,
+        docConcentration: Optional[List[float]] = ...,
+        topicConcentration: Optional[float] = ...,
+        topicDistributionCol: str = ...,
+        keepLastCheckpoint: bool = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        featuresCol: str = ...,
+        maxIter: int = ...,
+        seed: Optional[int] = ...,
+        checkpointInterval: int = ...,
+        k: int = ...,
+        optimizer: str = ...,
+        learningOffset: float = ...,
+        learningDecay: float = ...,
+        subsamplingRate: float = ...,
+        optimizeDocConcentration: bool = ...,
+        docConcentration: Optional[List[float]] = ...,
+        topicConcentration: Optional[float] = ...,
+        topicDistributionCol: str = ...,
+        keepLastCheckpoint: bool = ...
+    ) -> LDA: ...
+    def setCheckpointInterval(self, value: int) -> LDA: ...
+    def setSeed(self, value: int) -> LDA: ...
+    def setK(self, value: int) -> LDA: ...
+    def setOptimizer(self, value: str) -> LDA: ...
+    def setLearningOffset(self, value: float) -> LDA: ...
+    def setLearningDecay(self, value: float) -> LDA: ...
+    def setSubsamplingRate(self, value: float) -> LDA: ...
+    def setOptimizeDocConcentration(self, value: bool) -> LDA: ...
+    def setDocConcentration(self, value: List[float]) -> LDA: ...
+    def setTopicConcentration(self, value: float) -> LDA: ...
+    def setTopicDistributionCol(self, value: str) -> LDA: ...
+    def setKeepLastCheckpoint(self, value: bool) -> LDA: ...
+    def setMaxIter(self, value: int) -> LDA: ...
+    def setFeaturesCol(self, value: str) -> LDA: ...
+
+class _PowerIterationClusteringParams(HasMaxIter, HasWeightCol):
+    k: Param[int]
+    initMode: Param[str]
+    srcCol: Param[str]
+    dstCol: Param[str]
+    def __init__(self, *args: Any): ...
+    def getK(self) -> int: ...
+    def getInitMode(self) -> str: ...
+    def getSrcCol(self) -> str: ...
+    def getDstCol(self) -> str: ...
+
+class PowerIterationClustering(
+    _PowerIterationClusteringParams,
+    JavaParams,
+    JavaMLReadable[PowerIterationClustering],
+    JavaMLWritable,
+):
+    def __init__(
+        self,
+        *,
+        k: int = ...,
+        maxIter: int = ...,
+        initMode: str = ...,
+        srcCol: str = ...,
+        dstCol: str = ...,
+        weightCol: Optional[str] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        k: int = ...,
+        maxIter: int = ...,
+        initMode: str = ...,
+        srcCol: str = ...,
+        dstCol: str = ...,
+        weightCol: Optional[str] = ...
+    ) -> PowerIterationClustering: ...
+    def setK(self, value: int) -> PowerIterationClustering: ...
+    def setInitMode(self, value: str) -> PowerIterationClustering: ...
+    def setSrcCol(self, value: str) -> str: ...
+    def setDstCol(self, value: str) -> PowerIterationClustering: ...
+    def setMaxIter(self, value: int) -> PowerIterationClustering: ...
+    def setWeightCol(self, value: str) -> PowerIterationClustering: ...
+    def assignClusters(self, dataset: DataFrame) -> DataFrame: ...
diff --git a/python/pyspark/ml/common.py b/python/pyspark/ml/common.py
index 387c5d7309dea..4e1d7f93aef9b 100644
--- a/python/pyspark/ml/common.py
+++ b/python/pyspark/ml/common.py
@@ -15,11 +15,6 @@
 # limitations under the License.
 #
 
-import sys
-if sys.version >= '3':
-    long = int
-    unicode = str
-
 import py4j.protocol
 from py4j.protocol import Py4JJavaError
 from py4j.java_gateway import JavaObject
@@ -79,7 +74,7 @@ def _py2java(sc, obj):
         obj = [_py2java(sc, x) for x in obj]
     elif isinstance(obj, JavaObject):
         pass
-    elif isinstance(obj, (int, long, float, bool, bytes, unicode)):
+    elif isinstance(obj, (int, float, bool, bytes, str)):
         pass
     else:
         data = bytearray(PickleSerializer().dumps(obj))
diff --git a/python/pyspark/ml/common.pyi b/python/pyspark/ml/common.pyi
new file mode 100644
index 0000000000000..a38fc5734f466
--- /dev/null
+++ b/python/pyspark/ml/common.pyi
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Any, TypeVar
+
+import pyspark.context
+
+C = TypeVar("C", bound=type)
+
+def callJavaFunc(sc: pyspark.context.SparkContext, func: Any, *args: Any) -> Any: ...
+def inherit_doc(cls: C) -> C: ...
diff --git a/python/pyspark/ml/evaluation.py b/python/pyspark/ml/evaluation.py
index 265f02c1a03ac..e8cada907577d 100644
--- a/python/pyspark/ml/evaluation.py
+++ b/python/pyspark/ml/evaluation.py
@@ -32,36 +32,48 @@
 
 
 @inherit_doc
-class Evaluator(Params):
+class Evaluator(Params, metaclass=ABCMeta):
     """
     Base class for evaluators that compute metrics from predictions.
 
     .. versionadded:: 1.4.0
     """
-
-    __metaclass__ = ABCMeta
+    pass
 
     @abstractmethod
     def _evaluate(self, dataset):
         """
         Evaluates the output.
 
-        :param dataset: a dataset that contains labels/observations and
-               predictions
-        :return: metric
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            a dataset that contains labels/observations and predictions
+
+        Returns
+        -------
+        float
+            metric
         """
         raise NotImplementedError()
 
-    @since("1.4.0")
     def evaluate(self, dataset, params=None):
         """
         Evaluates the output with optional parameters.
 
-        :param dataset: a dataset that contains labels/observations and
-                        predictions
-        :param params: an optional param map that overrides embedded
-                       params
-        :return: metric
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            a dataset that contains labels/observations and predictions
+        params : dict, optional
+            an optional param map that overrides embedded params
+
+        Returns
+        -------
+        float
+            metric
         """
         if params is None:
             params = dict()
@@ -71,7 +83,7 @@ def evaluate(self, dataset, params=None):
             else:
                 return self._evaluate(dataset)
         else:
-            raise ValueError("Params must be a param map but got %s." % type(params))
+            raise TypeError("Params must be a param map but got %s." % type(params))
 
     @since("1.5.0")
     def isLargerBetter(self):
@@ -84,19 +96,25 @@ def isLargerBetter(self):
 
 
 @inherit_doc
-class JavaEvaluator(JavaParams, Evaluator):
+class JavaEvaluator(JavaParams, Evaluator, metaclass=ABCMeta):
     """
     Base class for :py:class:`Evaluator`s that wrap Java/Scala
     implementations.
     """
 
-    __metaclass__ = ABCMeta
-
     def _evaluate(self, dataset):
         """
         Evaluates the output.
-        :param dataset: a dataset that contains labels/observations and predictions.
-        :return: evaluation metric
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            a dataset that contains labels/observations and predictions
+
+        Returns
+        -------
+        float
+            evaluation metric
         """
         self._transfer_params_to_java()
         return self._java_obj.evaluate(dataset._jdf)
@@ -115,6 +133,10 @@ class BinaryClassificationEvaluator(JavaEvaluator, HasLabelCol, HasRawPrediction
     The rawPrediction column can be of type double (binary 0/1 prediction, or probability of label
     1) or of type vector (length-2 vector of raw predictions, scores, or label probabilities).
 
+    .. versionadded:: 1.4.0
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> scoreAndLabels = map(lambda x: (Vectors.dense([1.0 - x[0], x[0]]), x[1]),
     ...    [(0.1, 0.0), (0.1, 1.0), (0.4, 0.0), (0.6, 0.0), (0.6, 1.0), (0.6, 1.0), (0.8, 1.0)])
@@ -144,8 +166,6 @@ class BinaryClassificationEvaluator(JavaEvaluator, HasLabelCol, HasRawPrediction
     0.82...
     >>> evaluator.getNumBins()
     1000
-
-    .. versionadded:: 1.4.0
     """
 
     metricName = Param(Params._dummy(), "metricName",
@@ -158,10 +178,10 @@ class BinaryClassificationEvaluator(JavaEvaluator, HasLabelCol, HasRawPrediction
                     typeConverter=TypeConverters.toInt)
 
     @keyword_only
-    def __init__(self, rawPredictionCol="rawPrediction", labelCol="label",
+    def __init__(self, *, rawPredictionCol="rawPrediction", labelCol="label",
                  metricName="areaUnderROC", weightCol=None, numBins=1000):
         """
-        __init__(self, rawPredictionCol="rawPrediction", labelCol="label", \
+        __init__(self, \\*, rawPredictionCol="rawPrediction", labelCol="label", \
                  metricName="areaUnderROC", weightCol=None, numBins=1000)
         """
         super(BinaryClassificationEvaluator, self).__init__()
@@ -220,10 +240,10 @@ def setWeightCol(self, value):
 
     @keyword_only
     @since("1.4.0")
-    def setParams(self, rawPredictionCol="rawPrediction", labelCol="label",
+    def setParams(self, *, rawPredictionCol="rawPrediction", labelCol="label",
                   metricName="areaUnderROC", weightCol=None, numBins=1000):
         """
-        setParams(self, rawPredictionCol="rawPrediction", labelCol="label", \
+        setParams(self, \\*, rawPredictionCol="rawPrediction", labelCol="label", \
                   metricName="areaUnderROC", weightCol=None, numBins=1000)
         Sets params for binary classification evaluator.
         """
@@ -238,6 +258,10 @@ class RegressionEvaluator(JavaEvaluator, HasLabelCol, HasPredictionCol, HasWeigh
     Evaluator for Regression, which expects input columns prediction, label
     and an optional weight column.
 
+    .. versionadded:: 1.4.0
+
+    Examples
+    --------
     >>> scoreAndLabels = [(-28.98343821, -27.0), (20.21491975, 21.5),
     ...   (-25.98418959, -22.0), (30.69731842, 33.0), (74.69283752, 71.0)]
     >>> dataset = spark.createDataFrame(scoreAndLabels, ["raw", "label"])
@@ -265,8 +289,6 @@ class RegressionEvaluator(JavaEvaluator, HasLabelCol, HasPredictionCol, HasWeigh
     2.740...
     >>> evaluator.getThroughOrigin()
     False
-
-    .. versionadded:: 1.4.0
     """
     metricName = Param(Params._dummy(), "metricName",
                        """metric name in evaluation - one of:
@@ -282,10 +304,10 @@ class RegressionEvaluator(JavaEvaluator, HasLabelCol, HasPredictionCol, HasWeigh
                           typeConverter=TypeConverters.toBoolean)
 
     @keyword_only
-    def __init__(self, predictionCol="prediction", labelCol="label",
+    def __init__(self, *, predictionCol="prediction", labelCol="label",
                  metricName="rmse", weightCol=None, throughOrigin=False):
         """
-        __init__(self, predictionCol="prediction", labelCol="label", \
+        __init__(self, \\*, predictionCol="prediction", labelCol="label", \
                  metricName="rmse", weightCol=None, throughOrigin=False)
         """
         super(RegressionEvaluator, self).__init__()
@@ -344,10 +366,10 @@ def setWeightCol(self, value):
 
     @keyword_only
     @since("1.4.0")
-    def setParams(self, predictionCol="prediction", labelCol="label",
+    def setParams(self, *, predictionCol="prediction", labelCol="label",
                   metricName="rmse", weightCol=None, throughOrigin=False):
         """
-        setParams(self, predictionCol="prediction", labelCol="label", \
+        setParams(self, \\*, predictionCol="prediction", labelCol="label", \
                   metricName="rmse", weightCol=None, throughOrigin=False)
         Sets params for regression evaluator.
         """
@@ -362,6 +384,10 @@ class MulticlassClassificationEvaluator(JavaEvaluator, HasLabelCol, HasPredictio
     Evaluator for Multiclass Classification, which expects input
     columns: prediction, label, weight (optional) and probabilityCol (only for logLoss).
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> scoreAndLabels = [(0.0, 0.0), (0.0, 1.0), (0.0, 0.0),
     ...     (1.0, 0.0), (1.0, 1.0), (1.0, 1.0), (1.0, 1.0), (2.0, 2.0), (2.0, 0.0)]
     >>> dataset = spark.createDataFrame(scoreAndLabels, ["prediction", "label"])
@@ -405,8 +431,6 @@ class MulticlassClassificationEvaluator(JavaEvaluator, HasLabelCol, HasPredictio
     MulticlassClassificationEvaluator...
     >>> evaluator.evaluate(dataset)
     0.9682...
-
-    .. versionadded:: 1.5.0
     """
     metricName = Param(Params._dummy(), "metricName",
                        "metric name in evaluation "
@@ -431,11 +455,11 @@ class MulticlassClassificationEvaluator(JavaEvaluator, HasLabelCol, HasPredictio
                 typeConverter=TypeConverters.toFloat)
 
     @keyword_only
-    def __init__(self, predictionCol="prediction", labelCol="label",
+    def __init__(self, *, predictionCol="prediction", labelCol="label",
                  metricName="f1", weightCol=None, metricLabel=0.0, beta=1.0,
                  probabilityCol="probability", eps=1e-15):
         """
-        __init__(self, predictionCol="prediction", labelCol="label", \
+        __init__(self, \\*, predictionCol="prediction", labelCol="label", \
                  metricName="f1", weightCol=None, metricLabel=0.0, beta=1.0, \
                  probabilityCol="probability", eps=1e-15)
         """
@@ -530,11 +554,11 @@ def setWeightCol(self, value):
 
     @keyword_only
     @since("1.5.0")
-    def setParams(self, predictionCol="prediction", labelCol="label",
+    def setParams(self, *, predictionCol="prediction", labelCol="label",
                   metricName="f1", weightCol=None, metricLabel=0.0, beta=1.0,
                   probabilityCol="probability", eps=1e-15):
         """
-        setParams(self, predictionCol="prediction", labelCol="label", \
+        setParams(self, \\*, predictionCol="prediction", labelCol="label", \
                   metricName="f1", weightCol=None, metricLabel=0.0, beta=1.0, \
                   probabilityCol="probability", eps=1e-15)
         Sets params for multiclass classification evaluator.
@@ -547,11 +571,17 @@ def setParams(self, predictionCol="prediction", labelCol="label",
 class MultilabelClassificationEvaluator(JavaEvaluator, HasLabelCol, HasPredictionCol,
                                         JavaMLReadable, JavaMLWritable):
     """
-    .. note:: Experimental
-
     Evaluator for Multilabel Classification, which expects two input
     columns: prediction and label.
 
+    .. versionadded:: 3.0.0
+
+    Notes
+    -----
+    Experimental
+
+    Examples
+    --------
     >>> scoreAndLabels = [([0.0, 1.0], [0.0, 2.0]), ([0.0, 2.0], [0.0, 1.0]),
     ...     ([], [0.0]), ([2.0], [2.0]), ([2.0, 0.0], [2.0, 0.0]),
     ...     ([0.0, 1.0, 2.0], [0.0, 1.0]), ([1.0], [1.0, 2.0])]
@@ -569,8 +599,6 @@ class MultilabelClassificationEvaluator(JavaEvaluator, HasLabelCol, HasPredictio
     >>> evaluator2 = MultilabelClassificationEvaluator.load(mlce_path)
     >>> str(evaluator2.getPredictionCol())
     'prediction'
-
-    .. versionadded:: 3.0.0
     """
     metricName = Param(Params._dummy(), "metricName",
                        "metric name in evaluation "
@@ -585,10 +613,10 @@ class MultilabelClassificationEvaluator(JavaEvaluator, HasLabelCol, HasPredictio
                         typeConverter=TypeConverters.toFloat)
 
     @keyword_only
-    def __init__(self, predictionCol="prediction", labelCol="label",
+    def __init__(self, *, predictionCol="prediction", labelCol="label",
                  metricName="f1Measure", metricLabel=0.0):
         """
-        __init__(self, predictionCol="prediction", labelCol="label", \
+        __init__(self, \\*, predictionCol="prediction", labelCol="label", \
                  metricName="f1Measure", metricLabel=0.0)
         """
         super(MultilabelClassificationEvaluator, self).__init__()
@@ -642,10 +670,10 @@ def setPredictionCol(self, value):
 
     @keyword_only
     @since("3.0.0")
-    def setParams(self, predictionCol="prediction", labelCol="label",
+    def setParams(self, *, predictionCol="prediction", labelCol="label",
                   metricName="f1Measure", metricLabel=0.0):
         """
-        setParams(self, predictionCol="prediction", labelCol="label", \
+        setParams(self, \\*, predictionCol="prediction", labelCol="label", \
                   metricName="f1Measure", metricLabel=0.0)
         Sets params for multilabel classification evaluator.
         """
@@ -654,7 +682,7 @@ def setParams(self, predictionCol="prediction", labelCol="label",
 
 
 @inherit_doc
-class ClusteringEvaluator(JavaEvaluator, HasPredictionCol, HasFeaturesCol,
+class ClusteringEvaluator(JavaEvaluator, HasPredictionCol, HasFeaturesCol, HasWeightCol,
                           JavaMLReadable, JavaMLWritable):
     """
     Evaluator for Clustering results, which expects two input
@@ -666,6 +694,10 @@ class ClusteringEvaluator(JavaEvaluator, HasPredictionCol, HasFeaturesCol,
     1 means that the points in a cluster are close to the other points
     in the same cluster and far from the points of the other clusters.
 
+    .. versionadded:: 2.3.0
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> featureAndPredictions = map(lambda x: (Vectors.dense(x[0]), x[1]),
     ...     [([0.0, 0.5], 0.0), ([0.5, 0.0], 0.0), ([10.0, 11.0], 1.0),
@@ -677,13 +709,23 @@ class ClusteringEvaluator(JavaEvaluator, HasPredictionCol, HasFeaturesCol,
     ClusteringEvaluator...
     >>> evaluator.evaluate(dataset)
     0.9079...
+    >>> featureAndPredictionsWithWeight = map(lambda x: (Vectors.dense(x[0]), x[1], x[2]),
+    ...     [([0.0, 0.5], 0.0, 2.5), ([0.5, 0.0], 0.0, 2.5), ([10.0, 11.0], 1.0, 2.5),
+    ...     ([10.5, 11.5], 1.0, 2.5), ([1.0, 1.0], 0.0, 2.5), ([8.0, 6.0], 1.0, 2.5)])
+    >>> dataset = spark.createDataFrame(
+    ...     featureAndPredictionsWithWeight, ["features", "prediction", "weight"])
+    >>> evaluator = ClusteringEvaluator()
+    >>> evaluator.setPredictionCol("prediction")
+    ClusteringEvaluator...
+    >>> evaluator.setWeightCol("weight")
+    ClusteringEvaluator...
+    >>> evaluator.evaluate(dataset)
+    0.9079...
     >>> ce_path = temp_path + "/ce"
     >>> evaluator.save(ce_path)
     >>> evaluator2 = ClusteringEvaluator.load(ce_path)
     >>> str(evaluator2.getPredictionCol())
     'prediction'
-
-    .. versionadded:: 2.3.0
     """
     metricName = Param(Params._dummy(), "metricName",
                        "metric name in evaluation (silhouette)",
@@ -693,11 +735,11 @@ class ClusteringEvaluator(JavaEvaluator, HasPredictionCol, HasFeaturesCol,
                             typeConverter=TypeConverters.toString)
 
     @keyword_only
-    def __init__(self, predictionCol="prediction", featuresCol="features",
-                 metricName="silhouette", distanceMeasure="squaredEuclidean"):
+    def __init__(self, *, predictionCol="prediction", featuresCol="features",
+                 metricName="silhouette", distanceMeasure="squaredEuclidean", weightCol=None):
         """
-        __init__(self, predictionCol="prediction", featuresCol="features", \
-                 metricName="silhouette", distanceMeasure="squaredEuclidean")
+        __init__(self, \\*, predictionCol="prediction", featuresCol="features", \
+                 metricName="silhouette", distanceMeasure="squaredEuclidean", weightCol=None)
         """
         super(ClusteringEvaluator, self).__init__()
         self._java_obj = self._new_java_obj(
@@ -708,11 +750,11 @@ def __init__(self, predictionCol="prediction", featuresCol="features",
 
     @keyword_only
     @since("2.3.0")
-    def setParams(self, predictionCol="prediction", featuresCol="features",
-                  metricName="silhouette", distanceMeasure="squaredEuclidean"):
+    def setParams(self, *, predictionCol="prediction", featuresCol="features",
+                  metricName="silhouette", distanceMeasure="squaredEuclidean", weightCol=None):
         """
-        setParams(self, predictionCol="prediction", featuresCol="features", \
-                  metricName="silhouette", distanceMeasure="squaredEuclidean")
+        setParams(self, \\*, predictionCol="prediction", featuresCol="features", \
+                  metricName="silhouette", distanceMeasure="squaredEuclidean", weightCol=None)
         Sets params for clustering evaluator.
         """
         kwargs = self._input_kwargs
@@ -758,16 +800,29 @@ def setPredictionCol(self, value):
         """
         return self._set(predictionCol=value)
 
+    @since("3.1.0")
+    def setWeightCol(self, value):
+        """
+        Sets the value of :py:attr:`weightCol`.
+        """
+        return self._set(weightCol=value)
+
 
 @inherit_doc
 class RankingEvaluator(JavaEvaluator, HasLabelCol, HasPredictionCol,
                        JavaMLReadable, JavaMLWritable):
     """
-    .. note:: Experimental
-
     Evaluator for Ranking, which expects two input
     columns: prediction and label.
 
+    .. versionadded:: 3.0.0
+
+    Notes
+    -----
+    Experimental
+
+    Examples
+    --------
     >>> scoreAndLabels = [([1.0, 6.0, 2.0, 7.0, 8.0, 3.0, 9.0, 10.0, 4.0, 5.0],
     ...     [1.0, 2.0, 3.0, 4.0, 5.0]),
     ...     ([4.0, 1.0, 5.0, 6.0, 2.0, 7.0, 3.0, 8.0, 9.0, 10.0], [1.0, 2.0, 3.0]),
@@ -786,8 +841,6 @@ class RankingEvaluator(JavaEvaluator, HasLabelCol, HasPredictionCol,
     >>> evaluator2 = RankingEvaluator.load(ranke_path)
     >>> str(evaluator2.getPredictionCol())
     'prediction'
-
-    .. versionadded:: 3.0.0
     """
     metricName = Param(Params._dummy(), "metricName",
                        "metric name in evaluation "
@@ -800,10 +853,10 @@ class RankingEvaluator(JavaEvaluator, HasLabelCol, HasPredictionCol,
               typeConverter=TypeConverters.toInt)
 
     @keyword_only
-    def __init__(self, predictionCol="prediction", labelCol="label",
+    def __init__(self, *, predictionCol="prediction", labelCol="label",
                  metricName="meanAveragePrecision", k=10):
         """
-        __init__(self, predictionCol="prediction", labelCol="label", \
+        __init__(self, \\*, predictionCol="prediction", labelCol="label", \
                  metricName="meanAveragePrecision", k=10)
         """
         super(RankingEvaluator, self).__init__()
@@ -857,10 +910,10 @@ def setPredictionCol(self, value):
 
     @keyword_only
     @since("3.0.0")
-    def setParams(self, predictionCol="prediction", labelCol="label",
+    def setParams(self, *, predictionCol="prediction", labelCol="label",
                   metricName="meanAveragePrecision", k=10):
         """
-        setParams(self, predictionCol="prediction", labelCol="label", \
+        setParams(self, \\*, predictionCol="prediction", labelCol="label", \
                   metricName="meanAveragePrecision", k=10)
         Sets params for ranking evaluator.
         """
diff --git a/python/pyspark/ml/evaluation.pyi b/python/pyspark/ml/evaluation.pyi
new file mode 100644
index 0000000000000..55a3ae2774115
--- /dev/null
+++ b/python/pyspark/ml/evaluation.pyi
@@ -0,0 +1,283 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import abc
+from typing import Optional
+from pyspark.ml._typing import (
+    ParamMap,
+    BinaryClassificationEvaluatorMetricType,
+    ClusteringEvaluatorMetricType,
+    MulticlassClassificationEvaluatorMetricType,
+    MultilabelClassificationEvaluatorMetricType,
+    RankingEvaluatorMetricType,
+    RegressionEvaluatorMetricType,
+)
+
+from pyspark.ml.wrapper import JavaParams
+from pyspark.ml.param import Param, Params
+from pyspark.ml.param.shared import (
+    HasFeaturesCol,
+    HasLabelCol,
+    HasPredictionCol,
+    HasProbabilityCol,
+    HasRawPredictionCol,
+    HasWeightCol,
+)
+from pyspark.ml.util import JavaMLReadable, JavaMLWritable
+from pyspark.sql.dataframe import DataFrame
+
+class Evaluator(Params, metaclass=abc.ABCMeta):
+    def evaluate(
+        self, dataset: DataFrame, params: Optional[ParamMap] = ...
+    ) -> float: ...
+    def isLargerBetter(self) -> bool: ...
+
+class JavaEvaluator(JavaParams, Evaluator, metaclass=abc.ABCMeta):
+    def isLargerBetter(self) -> bool: ...
+
+class BinaryClassificationEvaluator(
+    JavaEvaluator,
+    HasLabelCol,
+    HasRawPredictionCol,
+    HasWeightCol,
+    JavaMLReadable[BinaryClassificationEvaluator],
+    JavaMLWritable,
+):
+    metricName: Param[BinaryClassificationEvaluatorMetricType]
+    numBins: Param[int]
+    def __init__(
+        self,
+        *,
+        rawPredictionCol: str = ...,
+        labelCol: str = ...,
+        metricName: BinaryClassificationEvaluatorMetricType = ...,
+        weightCol: Optional[str] = ...,
+        numBins: int = ...
+    ) -> None: ...
+    def setMetricName(
+        self, value: BinaryClassificationEvaluatorMetricType
+    ) -> BinaryClassificationEvaluator: ...
+    def getMetricName(self) -> BinaryClassificationEvaluatorMetricType: ...
+    def setNumBins(self, value: int) -> BinaryClassificationEvaluator: ...
+    def getNumBins(self) -> int: ...
+    def setLabelCol(self, value: str) -> BinaryClassificationEvaluator: ...
+    def setRawPredictionCol(self, value: str) -> BinaryClassificationEvaluator: ...
+    def setWeightCol(self, value: str) -> BinaryClassificationEvaluator: ...
+    def setParams(
+        self,
+        *,
+        rawPredictionCol: str = ...,
+        labelCol: str = ...,
+        metricName: BinaryClassificationEvaluatorMetricType = ...,
+        weightCol: Optional[str] = ...,
+        numBins: int = ...
+    ) -> BinaryClassificationEvaluator: ...
+
+class RegressionEvaluator(
+    JavaEvaluator,
+    HasLabelCol,
+    HasPredictionCol,
+    HasWeightCol,
+    JavaMLReadable[RegressionEvaluator],
+    JavaMLWritable,
+):
+    metricName: Param[RegressionEvaluatorMetricType]
+    throughOrigin: Param[bool]
+    def __init__(
+        self,
+        *,
+        predictionCol: str = ...,
+        labelCol: str = ...,
+        metricName: RegressionEvaluatorMetricType = ...,
+        weightCol: Optional[str] = ...,
+        throughOrigin: bool = ...
+    ) -> None: ...
+    def setMetricName(
+        self, value: RegressionEvaluatorMetricType
+    ) -> RegressionEvaluator: ...
+    def getMetricName(self) -> RegressionEvaluatorMetricType: ...
+    def setThroughOrigin(self, value: bool) -> RegressionEvaluator: ...
+    def getThroughOrigin(self) -> bool: ...
+    def setLabelCol(self, value: str) -> RegressionEvaluator: ...
+    def setPredictionCol(self, value: str) -> RegressionEvaluator: ...
+    def setWeightCol(self, value: str) -> RegressionEvaluator: ...
+    def setParams(
+        self,
+        *,
+        predictionCol: str = ...,
+        labelCol: str = ...,
+        metricName: RegressionEvaluatorMetricType = ...,
+        weightCol: Optional[str] = ...,
+        throughOrigin: bool = ...
+    ) -> RegressionEvaluator: ...
+
+class MulticlassClassificationEvaluator(
+    JavaEvaluator,
+    HasLabelCol,
+    HasPredictionCol,
+    HasWeightCol,
+    HasProbabilityCol,
+    JavaMLReadable[MulticlassClassificationEvaluator],
+    JavaMLWritable,
+):
+    metricName: Param[MulticlassClassificationEvaluatorMetricType]
+    metricLabel: Param[float]
+    beta: Param[float]
+    eps: Param[float]
+    def __init__(
+        self,
+        *,
+        predictionCol: str = ...,
+        labelCol: str = ...,
+        metricName: MulticlassClassificationEvaluatorMetricType = ...,
+        weightCol: Optional[str] = ...,
+        metricLabel: float = ...,
+        beta: float = ...,
+        probabilityCol: str = ...,
+        eps: float = ...
+    ) -> None: ...
+    def setMetricName(
+        self, value: MulticlassClassificationEvaluatorMetricType
+    ) -> MulticlassClassificationEvaluator: ...
+    def getMetricName(self) -> MulticlassClassificationEvaluatorMetricType: ...
+    def setMetricLabel(self, value: float) -> MulticlassClassificationEvaluator: ...
+    def getMetricLabel(self) -> float: ...
+    def setBeta(self, value: float) -> MulticlassClassificationEvaluator: ...
+    def getBeta(self) -> float: ...
+    def setEps(self, value: float) -> MulticlassClassificationEvaluator: ...
+    def getEps(self) -> float: ...
+    def setLabelCol(self, value: str) -> MulticlassClassificationEvaluator: ...
+    def setPredictionCol(self, value: str) -> MulticlassClassificationEvaluator: ...
+    def setProbabilityCol(self, value: str) -> MulticlassClassificationEvaluator: ...
+    def setWeightCol(self, value: str) -> MulticlassClassificationEvaluator: ...
+    def setParams(
+        self,
+        *,
+        predictionCol: str = ...,
+        labelCol: str = ...,
+        metricName: MulticlassClassificationEvaluatorMetricType = ...,
+        weightCol: Optional[str] = ...,
+        metricLabel: float = ...,
+        beta: float = ...,
+        probabilityCol: str = ...,
+        eps: float = ...
+    ) -> MulticlassClassificationEvaluator: ...
+
+class MultilabelClassificationEvaluator(
+    JavaEvaluator,
+    HasLabelCol,
+    HasPredictionCol,
+    JavaMLReadable[MultilabelClassificationEvaluator],
+    JavaMLWritable,
+):
+    metricName: Param[MultilabelClassificationEvaluatorMetricType]
+    metricLabel: Param[float]
+    def __init__(
+        self,
+        *,
+        predictionCol: str = ...,
+        labelCol: str = ...,
+        metricName: MultilabelClassificationEvaluatorMetricType = ...,
+        metricLabel: float = ...
+    ) -> None: ...
+    def setMetricName(
+        self, value: MultilabelClassificationEvaluatorMetricType
+    ) -> MultilabelClassificationEvaluator: ...
+    def getMetricName(self) -> MultilabelClassificationEvaluatorMetricType: ...
+    def setMetricLabel(self, value: float) -> MultilabelClassificationEvaluator: ...
+    def getMetricLabel(self) -> float: ...
+    def setLabelCol(self, value: str) -> MultilabelClassificationEvaluator: ...
+    def setPredictionCol(self, value: str) -> MultilabelClassificationEvaluator: ...
+    def setParams(
+        self,
+        *,
+        predictionCol: str = ...,
+        labelCol: str = ...,
+        metricName: MultilabelClassificationEvaluatorMetricType = ...,
+        metricLabel: float = ...
+    ) -> MultilabelClassificationEvaluator: ...
+
+class ClusteringEvaluator(
+    JavaEvaluator,
+    HasPredictionCol,
+    HasFeaturesCol,
+    HasWeightCol,
+    JavaMLReadable[ClusteringEvaluator],
+    JavaMLWritable,
+):
+    metricName: Param[ClusteringEvaluatorMetricType]
+    distanceMeasure: Param[str]
+    def __init__(
+        self,
+        *,
+        predictionCol: str = ...,
+        featuresCol: str = ...,
+        metricName: ClusteringEvaluatorMetricType = ...,
+        distanceMeasure: str = ...,
+        weightCol: Optional[str] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        predictionCol: str = ...,
+        featuresCol: str = ...,
+        metricName: ClusteringEvaluatorMetricType = ...,
+        distanceMeasure: str = ...,
+        weightCol: Optional[str] = ...
+    ) -> ClusteringEvaluator: ...
+    def setMetricName(
+        self, value: ClusteringEvaluatorMetricType
+    ) -> ClusteringEvaluator: ...
+    def getMetricName(self) -> ClusteringEvaluatorMetricType: ...
+    def setDistanceMeasure(self, value: str) -> ClusteringEvaluator: ...
+    def getDistanceMeasure(self) -> str: ...
+    def setFeaturesCol(self, value: str) -> ClusteringEvaluator: ...
+    def setPredictionCol(self, value: str) -> ClusteringEvaluator: ...
+    def setWeightCol(self, value: str) -> ClusteringEvaluator: ...
+
+class RankingEvaluator(
+    JavaEvaluator,
+    HasLabelCol,
+    HasPredictionCol,
+    JavaMLReadable[RankingEvaluator],
+    JavaMLWritable,
+):
+    metricName: Param[RankingEvaluatorMetricType]
+    k: Param[int]
+    def __init__(
+        self,
+        *,
+        predictionCol: str = ...,
+        labelCol: str = ...,
+        metricName: RankingEvaluatorMetricType = ...,
+        k: int = ...
+    ) -> None: ...
+    def setMetricName(self, value: RankingEvaluatorMetricType) -> RankingEvaluator: ...
+    def getMetricName(self) -> RankingEvaluatorMetricType: ...
+    def setK(self, value: int) -> RankingEvaluator: ...
+    def getK(self) -> int: ...
+    def setLabelCol(self, value: str) -> RankingEvaluator: ...
+    def setPredictionCol(self, value: str) -> RankingEvaluator: ...
+    def setParams(
+        self,
+        *,
+        predictionCol: str = ...,
+        labelCol: str = ...,
+        metricName: RankingEvaluatorMetricType = ...,
+        k: int = ...
+    ) -> RankingEvaluator: ...
diff --git a/python/pyspark/ml/feature.py b/python/pyspark/ml/feature.py
index 498629cea846c..e066788152970 100755
--- a/python/pyspark/ml/feature.py
+++ b/python/pyspark/ml/feature.py
@@ -15,20 +15,16 @@
 # limitations under the License.
 #
 
-import sys
-if sys.version > '3':
-    basestring = str
-
 from pyspark import since, keyword_only, SparkContext
-from pyspark.rdd import ignore_unicode_prefix
 from pyspark.ml.linalg import _convert_to_vector
-from pyspark.ml.param.shared import *
+from pyspark.ml.param.shared import HasThreshold, HasThresholds, HasInputCol, HasOutputCol, \
+    HasInputCols, HasOutputCols, HasHandleInvalid, HasRelativeError, HasFeaturesCol, HasLabelCol, \
+    HasSeed, HasNumFeatures, HasStepSize, HasMaxIter, TypeConverters, Param, Params
 from pyspark.ml.util import JavaMLReadable, JavaMLWritable
 from pyspark.ml.wrapper import JavaEstimator, JavaModel, JavaParams, JavaTransformer, _jvm
 from pyspark.ml.common import inherit_doc
 
-__all__ = ['ANOVASelector', 'ANOVASelectorModel',
-           'Binarizer',
+__all__ = ['Binarizer',
            'BucketedRandomProjectionLSH', 'BucketedRandomProjectionLSHModel',
            'Bucketizer',
            'ChiSqSelector', 'ChiSqSelectorModel',
@@ -36,7 +32,6 @@
            'DCT',
            'ElementwiseProduct',
            'FeatureHasher',
-           'FValueSelector', 'FValueSelectorModel',
            'HashingTF',
            'IDF', 'IDFModel',
            'Imputer', 'ImputerModel',
@@ -59,6 +54,7 @@
            'StopWordsRemover',
            'StringIndexer', 'StringIndexerModel',
            'Tokenizer',
+           'UnivariateFeatureSelector', 'UnivariateFeatureSelectorModel',
            'VarianceThresholdSelector', 'VarianceThresholdSelectorModel',
            'VectorAssembler',
            'VectorIndexer', 'VectorIndexerModel',
@@ -77,6 +73,10 @@ class Binarizer(JavaTransformer, HasThreshold, HasThresholds, HasInputCol, HasOu
     are set, an Exception will be thrown. The :py:attr:`threshold` parameter is used for
     single column usage, and :py:attr:`thresholds` is for multiple columns.
 
+    .. versionadded:: 1.4.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([(0.5,)], ["values"])
     >>> binarizer = Binarizer(threshold=1.0, inputCol="values", outputCol="features")
     >>> binarizer.setThreshold(1.0)
@@ -97,6 +97,8 @@ class Binarizer(JavaTransformer, HasThreshold, HasThresholds, HasInputCol, HasOu
     >>> loadedBinarizer = Binarizer.load(binarizerPath)
     >>> loadedBinarizer.getThreshold() == binarizer.getThreshold()
     True
+    >>> loadedBinarizer.transform(df).take(1) == binarizer.transform(df).take(1)
+    True
     >>> df2 = spark.createDataFrame([(0.5, 0.3)], ["values1", "values2"])
     >>> binarizer2 = Binarizer(thresholds=[0.0, 1.0])
     >>> binarizer2.setInputCols(["values1", "values2"]).setOutputCols(["output1", "output2"])
@@ -108,8 +110,6 @@ class Binarizer(JavaTransformer, HasThreshold, HasThresholds, HasInputCol, HasOu
     |    0.5|    0.3|    1.0|    0.0|
     +-------+-------+-------+-------+
     ...
-
-    .. versionadded:: 1.4.0
     """
 
     threshold = Param(Params._dummy(), "threshold",
@@ -125,10 +125,10 @@ class Binarizer(JavaTransformer, HasThreshold, HasThresholds, HasInputCol, HasOu
                        typeConverter=TypeConverters.toListFloat)
 
     @keyword_only
-    def __init__(self, threshold=0.0, inputCol=None, outputCol=None, thresholds=None,
+    def __init__(self, *, threshold=0.0, inputCol=None, outputCol=None, thresholds=None,
                  inputCols=None, outputCols=None):
         """
-        __init__(self, threshold=0.0, inputCol=None, outputCol=None, thresholds=None, \
+        __init__(self, \\*, threshold=0.0, inputCol=None, outputCol=None, thresholds=None, \
                  inputCols=None, outputCols=None)
         """
         super(Binarizer, self).__init__()
@@ -139,10 +139,10 @@ def __init__(self, threshold=0.0, inputCol=None, outputCol=None, thresholds=None
 
     @keyword_only
     @since("1.4.0")
-    def setParams(self, threshold=0.0, inputCol=None, outputCol=None, thresholds=None,
+    def setParams(self, *, threshold=0.0, inputCol=None, outputCol=None, thresholds=None,
                   inputCols=None, outputCols=None):
         """
-        setParams(self, threshold=0.0, inputCol=None, outputCol=None, thresholds=None, \
+        setParams(self, \\*, threshold=0.0, inputCol=None, outputCol=None, thresholds=None, \
                   inputCols=None, outputCols=None)
         Sets params for this Binarizer.
         """
@@ -200,6 +200,10 @@ class _LSHParams(HasInputCol, HasOutputCol):
                           "and decreasing it improves the running performance.",
                           typeConverter=TypeConverters.toInt)
 
+    def __init__(self, *args):
+        super(_LSHParams, self).__init__(*args)
+        self._setDefault(numHashTables=1)
+
     def getNumHashTables(self):
         """
         Gets the value of numHashTables or its default value.
@@ -255,15 +259,27 @@ def approxNearestNeighbors(self, dataset, key, numNearestNeighbors, distCol="dis
         transform the data; if the :py:attr:`outputCol` exists, it will use that. This allows
         caching of the transformed data when necessary.
 
-        .. note:: This method is experimental and will likely change behavior in the next release.
-
-        :param dataset: The dataset to search for nearest neighbors of the key.
-        :param key: Feature vector representing the item to search for.
-        :param numNearestNeighbors: The maximum number of nearest neighbors.
-        :param distCol: Output column for storing the distance between each result row and the key.
-                        Use "distCol" as default value if it's not specified.
-        :return: A dataset containing at most k items closest to the key. A column "distCol" is
-                 added to show the distance between each row and the key.
+        Notes
+        -----
+        This method is experimental and will likely change behavior in the next release.
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            The dataset to search for nearest neighbors of the key.
+        key :  :py:class:`pyspark.ml.linalg.Vector`
+            Feature vector representing the item to search for.
+        numNearestNeighbors : int
+            The maximum number of nearest neighbors.
+        distCol : str
+            Output column for storing the distance between each result row and the key.
+            Use "distCol" as default value if it's not specified.
+
+        Returns
+        -------
+        :py:class:`pyspark.sql.DataFrame`
+            A dataset containing at most k items closest to the key. A column "distCol" is
+            added to show the distance between each row and the key.
         """
         return self._call_java("approxNearestNeighbors", dataset, key, numNearestNeighbors,
                                distCol)
@@ -275,14 +291,24 @@ def approxSimilarityJoin(self, datasetA, datasetB, threshold, distCol="distCol")
         if the :py:attr:`outputCol` exists, it will use that. This allows caching of the
         transformed data when necessary.
 
-        :param datasetA: One of the datasets to join.
-        :param datasetB: Another dataset to join.
-        :param threshold: The threshold for the distance of row pairs.
-        :param distCol: Output column for storing the distance between each pair of rows. Use
-                        "distCol" as default value if it's not specified.
-        :return: A joined dataset containing pairs of rows. The original rows are in columns
-                 "datasetA" and "datasetB", and a column "distCol" is added to show the distance
-                 between each pair.
+        Parameters
+        ----------
+        datasetA : :py:class:`pyspark.sql.DataFrame`
+            One of the datasets to join.
+        datasetB : :py:class:`pyspark.sql.DataFrame`
+            Another dataset to join.
+        threshold : float
+            The threshold for the distance of row pairs.
+        distCol : str, optional
+            Output column for storing the distance between each pair of rows. Use
+            "distCol" as default value if it's not specified.
+
+        Returns
+        -------
+        :py:class:`pyspark.sql.DataFrame`
+            A joined dataset containing pairs of rows. The original rows are in columns
+            "datasetA" and "datasetB", and a column "distCol" is added to show the distance
+            between each pair.
         """
         threshold = TypeConverters.toFloat(threshold)
         return self._call_java("approxSimilarityJoin", datasetA, datasetB, threshold, distCol)
@@ -317,10 +343,17 @@ class BucketedRandomProjectionLSH(_LSH, _BucketedRandomProjectionLSHParams,
     distance space. The output will be vectors of configurable dimension. Hash values in the same
     dimension are calculated by the same hash function.
 
-    .. seealso:: `Stable Distributions
-        <https://en.wikipedia.org/wiki/Locality-sensitive_hashing#Stable_distributions>`_
-    .. seealso:: `Hashing for Similarity Search: A Survey <https://arxiv.org/abs/1408.2927>`_
+    .. versionadded:: 2.2.0
+
+    Notes
+    -----
 
+    - `Stable Distributions in Wikipedia article on Locality-sensitive hashing \
+      <https://en.wikipedia.org/wiki/Locality-sensitive_hashing#Stable_distributions>`_
+    - `Hashing for Similarity Search: A Survey <https://arxiv.org/abs/1408.2927>`_
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> from pyspark.sql.functions import col
     >>> data = [(0, Vectors.dense([-1.0, -1.0 ]),),
@@ -381,30 +414,27 @@ class BucketedRandomProjectionLSH(_LSH, _BucketedRandomProjectionLSHParams,
     >>> model2 = BucketedRandomProjectionLSHModel.load(modelPath)
     >>> model.transform(df).head().hashes == model2.transform(df).head().hashes
     True
-
-    .. versionadded:: 2.2.0
     """
 
     @keyword_only
-    def __init__(self, inputCol=None, outputCol=None, seed=None, numHashTables=1,
+    def __init__(self, *, inputCol=None, outputCol=None, seed=None, numHashTables=1,
                  bucketLength=None):
         """
-        __init__(self, inputCol=None, outputCol=None, seed=None, numHashTables=1, \
+        __init__(self, \\*, inputCol=None, outputCol=None, seed=None, numHashTables=1, \
                  bucketLength=None)
         """
         super(BucketedRandomProjectionLSH, self).__init__()
         self._java_obj = \
             self._new_java_obj("org.apache.spark.ml.feature.BucketedRandomProjectionLSH", self.uid)
-        self._setDefault(numHashTables=1)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("2.2.0")
-    def setParams(self, inputCol=None, outputCol=None, seed=None, numHashTables=1,
+    def setParams(self, *, inputCol=None, outputCol=None, seed=None, numHashTables=1,
                   bucketLength=None):
         """
-        setParams(self, inputCol=None, outputCol=None, seed=None, numHashTables=1, \
+        setParams(self, \\*, inputCol=None, outputCol=None, seed=None, numHashTables=1, \
                   bucketLength=None)
         Sets params for this BucketedRandomProjectionLSH.
         """
@@ -451,6 +481,10 @@ class Bucketizer(JavaTransformer, HasInputCol, HasOutputCol, HasInputCols, HasOu
     are set, an Exception will be thrown. The :py:attr:`splits` parameter is only used for single
     column usage, and :py:attr:`splitsArray` is for multiple columns.
 
+    .. versionadded:: 1.4.0
+
+    Examples
+    --------
     >>> values = [(0.1, 0.0), (0.4, 1.0), (1.2, 1.3), (1.5, float("nan")),
     ...     (float("nan"), 1.0), (float("nan"), 0.0)]
     >>> df = spark.createDataFrame(values, ["values1", "values2"])
@@ -482,6 +516,8 @@ class Bucketizer(JavaTransformer, HasInputCol, HasOutputCol, HasInputCols, HasOu
     >>> loadedBucketizer = Bucketizer.load(bucketizerPath)
     >>> loadedBucketizer.getSplits() == bucketizer.getSplits()
     True
+    >>> loadedBucketizer.transform(df).take(1) == bucketizer.transform(df).take(1)
+    True
     >>> bucketed = bucketizer.setHandleInvalid("skip").transform(df).collect()
     >>> len(bucketed)
     4
@@ -501,8 +537,6 @@ class Bucketizer(JavaTransformer, HasInputCol, HasOutputCol, HasInputCols, HasOu
     |NaN    |0.0    |3.0     |0.0     |
     +-------+-------+--------+--------+
     ...
-
-    .. versionadded:: 1.4.0
     """
 
     splits = \
@@ -537,10 +571,10 @@ class Bucketizer(JavaTransformer, HasInputCol, HasOutputCol, HasInputCols, HasOu
                         typeConverter=TypeConverters.toListListFloat)
 
     @keyword_only
-    def __init__(self, splits=None, inputCol=None, outputCol=None, handleInvalid="error",
+    def __init__(self, *, splits=None, inputCol=None, outputCol=None, handleInvalid="error",
                  splitsArray=None, inputCols=None, outputCols=None):
         """
-        __init__(self, splits=None, inputCol=None, outputCol=None, handleInvalid="error", \
+        __init__(self, \\*, splits=None, inputCol=None, outputCol=None, handleInvalid="error", \
                  splitsArray=None, inputCols=None, outputCols=None)
         """
         super(Bucketizer, self).__init__()
@@ -551,10 +585,10 @@ def __init__(self, splits=None, inputCol=None, outputCol=None, handleInvalid="er
 
     @keyword_only
     @since("1.4.0")
-    def setParams(self, splits=None, inputCol=None, outputCol=None, handleInvalid="error",
+    def setParams(self, *, splits=None, inputCol=None, outputCol=None, handleInvalid="error",
                   splitsArray=None, inputCols=None, outputCols=None):
         """
-        setParams(self, splits=None, inputCol=None, outputCol=None, handleInvalid="error", \
+        setParams(self, \\*, splits=None, inputCol=None, outputCol=None, handleInvalid="error", \
                   splitsArray=None, inputCols=None, outputCols=None)
         Sets params for this Bucketizer.
         """
@@ -703,6 +737,10 @@ class CountVectorizer(JavaEstimator, _CountVectorizerParams, JavaMLReadable, Jav
     """
     Extracts a vocabulary from document collections and generates a :py:attr:`CountVectorizerModel`.
 
+    .. versionadded:: 1.6.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame(
     ...    [(0, ["a", "b", "c"]), (1, ["a", "b", "b", "c", "a"])],
     ...    ["label", "raw"])
@@ -738,6 +776,8 @@ class CountVectorizer(JavaEstimator, _CountVectorizerParams, JavaMLReadable, Jav
     >>> loadedModel = CountVectorizerModel.load(modelPath)
     >>> loadedModel.vocabulary == model.vocabulary
     True
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
     >>> fromVocabModel = CountVectorizerModel.from_vocabulary(["a", "b", "c"],
     ...     inputCol="raw", outputCol="vectors")
     >>> fromVocabModel.transform(df).show(truncate=False)
@@ -748,16 +788,14 @@ class CountVectorizer(JavaEstimator, _CountVectorizerParams, JavaMLReadable, Jav
     |1    |[a, b, b, c, a]|(3,[0,1,2],[2.0,2.0,1.0])|
     +-----+---------------+-------------------------+
     ...
-
-    .. versionadded:: 1.6.0
     """
 
     @keyword_only
-    def __init__(self, minTF=1.0, minDF=1.0, maxDF=2 ** 63 - 1, vocabSize=1 << 18, binary=False,
-                 inputCol=None, outputCol=None):
+    def __init__(self, *, minTF=1.0, minDF=1.0, maxDF=2 ** 63 - 1, vocabSize=1 << 18,
+                 binary=False, inputCol=None, outputCol=None):
         """
-        __init__(self, minTF=1.0, minDF=1.0, maxDF=2 ** 63 - 1, vocabSize=1 << 18, binary=False,\
-                 inputCol=None,outputCol=None)
+        __init__(self, \\*, minTF=1.0, minDF=1.0, maxDF=2 ** 63 - 1, vocabSize=1 << 18,\
+                 binary=False, inputCol=None,outputCol=None)
         """
         super(CountVectorizer, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.CountVectorizer",
@@ -767,11 +805,11 @@ def __init__(self, minTF=1.0, minDF=1.0, maxDF=2 ** 63 - 1, vocabSize=1 << 18, b
 
     @keyword_only
     @since("1.6.0")
-    def setParams(self, minTF=1.0, minDF=1.0, maxDF=2 ** 63 - 1, vocabSize=1 << 18, binary=False,
-                  inputCol=None, outputCol=None):
+    def setParams(self, *, minTF=1.0, minDF=1.0, maxDF=2 ** 63 - 1, vocabSize=1 << 18,
+                  binary=False, inputCol=None, outputCol=None):
         """
-        setParams(self, minTF=1.0, minDF=1.0, maxDF=2 ** 63 - 1, vocabSize=1 << 18, binary=False,\
-                  inputCol=None, outputCol=None)
+        setParams(self, \\*, minTF=1.0, minDF=1.0, maxDF=2 ** 63 - 1, vocabSize=1 << 18,\
+                  binary=False, inputCol=None, outputCol=None)
         Set the params for the CountVectorizer
         """
         kwargs = self._input_kwargs
@@ -904,9 +942,15 @@ class DCT(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWrit
     The return vector is scaled such that the transform matrix is
     unitary (aka scaled DCT-II).
 
-    .. seealso:: `More information on Wikipedia
-        <https://en.wikipedia.org/wiki/Discrete_cosine_transform#DCT-II Wikipedia>`_.
+    .. versionadded:: 1.6.0
+
+    Notes
+    -----
+    `More information on Wikipedia \
+      <https://en.wikipedia.org/wiki/Discrete_cosine_transform#DCT-II Wikipedia>`_.
 
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> df1 = spark.createDataFrame([(Vectors.dense([5.0, 8.0, 6.0]),)], ["vec"])
     >>> dct = DCT( )
@@ -925,19 +969,19 @@ class DCT(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWrit
     >>> dctPath = temp_path + "/dct"
     >>> dct.save(dctPath)
     >>> loadedDtc = DCT.load(dctPath)
+    >>> loadedDtc.transform(df1).take(1) == dct.transform(df1).take(1)
+    True
     >>> loadedDtc.getInverse()
     False
-
-    .. versionadded:: 1.6.0
     """
 
     inverse = Param(Params._dummy(), "inverse", "Set transformer to perform inverse DCT, " +
                     "default False.", typeConverter=TypeConverters.toBoolean)
 
     @keyword_only
-    def __init__(self, inverse=False, inputCol=None, outputCol=None):
+    def __init__(self, *, inverse=False, inputCol=None, outputCol=None):
         """
-        __init__(self, inverse=False, inputCol=None, outputCol=None)
+        __init__(self, \\*, inverse=False, inputCol=None, outputCol=None)
         """
         super(DCT, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.DCT", self.uid)
@@ -947,9 +991,9 @@ def __init__(self, inverse=False, inputCol=None, outputCol=None):
 
     @keyword_only
     @since("1.6.0")
-    def setParams(self, inverse=False, inputCol=None, outputCol=None):
+    def setParams(self, *, inverse=False, inputCol=None, outputCol=None):
         """
-        setParams(self, inverse=False, inputCol=None, outputCol=None)
+        setParams(self, \\*, inverse=False, inputCol=None, outputCol=None)
         Sets params for this DCT.
         """
         kwargs = self._input_kwargs
@@ -990,6 +1034,10 @@ class ElementwiseProduct(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReada
     with a provided "weight" vector. In other words, it scales each column of the dataset
     by a scalar multiplier.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> df = spark.createDataFrame([(Vectors.dense([2.0, 1.0, 3.0]),)], ["values"])
     >>> ep = ElementwiseProduct()
@@ -1008,17 +1056,17 @@ class ElementwiseProduct(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReada
     >>> loadedEp = ElementwiseProduct.load(elementwiseProductPath)
     >>> loadedEp.getScalingVec() == ep.getScalingVec()
     True
-
-    .. versionadded:: 1.5.0
+    >>> loadedEp.transform(df).take(1) == ep.transform(df).take(1)
+    True
     """
 
     scalingVec = Param(Params._dummy(), "scalingVec", "Vector for hadamard product.",
                        typeConverter=TypeConverters.toVector)
 
     @keyword_only
-    def __init__(self, scalingVec=None, inputCol=None, outputCol=None):
+    def __init__(self, *, scalingVec=None, inputCol=None, outputCol=None):
         """
-        __init__(self, scalingVec=None, inputCol=None, outputCol=None)
+        __init__(self, \\*, scalingVec=None, inputCol=None, outputCol=None)
         """
         super(ElementwiseProduct, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.ElementwiseProduct",
@@ -1028,9 +1076,9 @@ def __init__(self, scalingVec=None, inputCol=None, outputCol=None):
 
     @keyword_only
     @since("1.5.0")
-    def setParams(self, scalingVec=None, inputCol=None, outputCol=None):
+    def setParams(self, *, scalingVec=None, inputCol=None, outputCol=None):
         """
-        setParams(self, scalingVec=None, inputCol=None, outputCol=None)
+        setParams(self, \\*, scalingVec=None, inputCol=None, outputCol=None)
         Sets params for this ElementwiseProduct.
         """
         kwargs = self._input_kwargs
@@ -1098,6 +1146,10 @@ class FeatureHasher(JavaTransformer, HasInputCols, HasOutputCol, HasNumFeatures,
     it is advisable to use a power of two as the `numFeatures` parameter;
     otherwise the features will not be mapped evenly to the vector indices.
 
+    .. versionadded:: 2.3.0
+
+    Examples
+    --------
     >>> data = [(2.0, True, "1", "foo"), (3.0, False, "2", "bar")]
     >>> cols = ["real", "bool", "stringNum", "string"]
     >>> df = spark.createDataFrame(data, cols)
@@ -1117,8 +1169,6 @@ class FeatureHasher(JavaTransformer, HasInputCols, HasOutputCol, HasNumFeatures,
     True
     >>> loadedHasher.transform(df).head().features == hasher.transform(df).head().features
     True
-
-    .. versionadded:: 2.3.0
     """
 
     categoricalCols = Param(Params._dummy(), "categoricalCols",
@@ -1126,9 +1176,11 @@ class FeatureHasher(JavaTransformer, HasInputCols, HasOutputCol, HasNumFeatures,
                             typeConverter=TypeConverters.toListString)
 
     @keyword_only
-    def __init__(self, numFeatures=1 << 18, inputCols=None, outputCol=None, categoricalCols=None):
+    def __init__(self, *, numFeatures=1 << 18, inputCols=None, outputCol=None,
+                 categoricalCols=None):
         """
-        __init__(self, numFeatures=1 << 18, inputCols=None, outputCol=None, categoricalCols=None)
+        __init__(self, \\*, numFeatures=1 << 18, inputCols=None, outputCol=None, \
+                 categoricalCols=None)
         """
         super(FeatureHasher, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.FeatureHasher", self.uid)
@@ -1138,9 +1190,11 @@ def __init__(self, numFeatures=1 << 18, inputCols=None, outputCol=None, categori
 
     @keyword_only
     @since("2.3.0")
-    def setParams(self, numFeatures=1 << 18, inputCols=None, outputCol=None, categoricalCols=None):
+    def setParams(self, *, numFeatures=1 << 18, inputCols=None, outputCol=None,
+                  categoricalCols=None):
         """
-        setParams(self, numFeatures=1 << 18, inputCols=None, outputCol=None, categoricalCols=None)
+        setParams(self, \\*, numFeatures=1 << 18, inputCols=None, outputCol=None, \
+                  categoricalCols=None)
         Sets params for this FeatureHasher.
         """
         kwargs = self._input_kwargs
@@ -1190,6 +1244,10 @@ class HashingTF(JavaTransformer, HasInputCol, HasOutputCol, HasNumFeatures, Java
     it is advisable to use a power of two as the numFeatures parameter;
     otherwise the features will not be mapped evenly to the columns.
 
+    .. versionadded:: 1.3.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([(["a", "b", "c"],)], ["words"])
     >>> hashingTF = HashingTF(inputCol="words", outputCol="features")
     >>> hashingTF.setNumFeatures(10)
@@ -1206,10 +1264,10 @@ class HashingTF(JavaTransformer, HasInputCol, HasOutputCol, HasNumFeatures, Java
     >>> loadedHashingTF = HashingTF.load(hashingTFPath)
     >>> loadedHashingTF.getNumFeatures() == hashingTF.getNumFeatures()
     True
+    >>> loadedHashingTF.transform(df).take(1) == hashingTF.transform(df).take(1)
+    True
     >>> hashingTF.indexOf("b")
     5
-
-    .. versionadded:: 1.3.0
     """
 
     binary = Param(Params._dummy(), "binary", "If True, all non zero counts are set to 1. " +
@@ -1218,9 +1276,9 @@ class HashingTF(JavaTransformer, HasInputCol, HasOutputCol, HasNumFeatures, Java
                    typeConverter=TypeConverters.toBoolean)
 
     @keyword_only
-    def __init__(self, numFeatures=1 << 18, binary=False, inputCol=None, outputCol=None):
+    def __init__(self, *, numFeatures=1 << 18, binary=False, inputCol=None, outputCol=None):
         """
-        __init__(self, numFeatures=1 << 18, binary=False, inputCol=None, outputCol=None)
+        __init__(self, \\*, numFeatures=1 << 18, binary=False, inputCol=None, outputCol=None)
         """
         super(HashingTF, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.HashingTF", self.uid)
@@ -1230,9 +1288,9 @@ def __init__(self, numFeatures=1 << 18, binary=False, inputCol=None, outputCol=N
 
     @keyword_only
     @since("1.3.0")
-    def setParams(self, numFeatures=1 << 18, binary=False, inputCol=None, outputCol=None):
+    def setParams(self, *, numFeatures=1 << 18, binary=False, inputCol=None, outputCol=None):
         """
-        setParams(self, numFeatures=1 << 18, binary=False, inputCol=None, outputCol=None)
+        setParams(self, \\*, numFeatures=1 << 18, binary=False, inputCol=None, outputCol=None)
         Sets params for this HashingTF.
         """
         kwargs = self._input_kwargs
@@ -1297,12 +1355,20 @@ def getMinDocFreq(self):
         """
         return self.getOrDefault(self.minDocFreq)
 
+    def __init__(self, *args):
+        super(_IDFParams, self).__init__(*args)
+        self._setDefault(minDocFreq=0)
+
 
 @inherit_doc
 class IDF(JavaEstimator, _IDFParams, JavaMLReadable, JavaMLWritable):
     """
     Compute the Inverse Document Frequency (IDF) given a collection of documents.
 
+    .. versionadded:: 1.4.0
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import DenseVector
     >>> df = spark.createDataFrame([(DenseVector([1.0, 2.0]),),
     ...     (DenseVector([0.0, 1.0]),), (DenseVector([3.0, 0.2]),)], ["tf"])
@@ -1339,26 +1405,23 @@ class IDF(JavaEstimator, _IDFParams, JavaMLReadable, JavaMLWritable):
     >>> loadedModel = IDFModel.load(modelPath)
     >>> loadedModel.transform(df).head().idf == model.transform(df).head().idf
     True
-
-    .. versionadded:: 1.4.0
     """
 
     @keyword_only
-    def __init__(self, minDocFreq=0, inputCol=None, outputCol=None):
+    def __init__(self, *, minDocFreq=0, inputCol=None, outputCol=None):
         """
-        __init__(self, minDocFreq=0, inputCol=None, outputCol=None)
+        __init__(self, \\*, minDocFreq=0, inputCol=None, outputCol=None)
         """
         super(IDF, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.IDF", self.uid)
-        self._setDefault(minDocFreq=0)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("1.4.0")
-    def setParams(self, minDocFreq=0, inputCol=None, outputCol=None):
+    def setParams(self, *, minDocFreq=0, inputCol=None, outputCol=None):
         """
-        setParams(self, minDocFreq=0, inputCol=None, outputCol=None)
+        setParams(self, \\*, minDocFreq=0, inputCol=None, outputCol=None)
         Sets params for this IDF.
         """
         kwargs = self._input_kwargs
@@ -1443,13 +1506,18 @@ class _ImputerParams(HasInputCol, HasInputCols, HasOutputCol, HasOutputCols, Has
     strategy = Param(Params._dummy(), "strategy",
                      "strategy for imputation. If mean, then replace missing values using the mean "
                      "value of the feature. If median, then replace missing values using the "
-                     "median value of the feature.",
+                     "median value of the feature. If mode, then replace missing using the most "
+                     "frequent value of the feature.",
                      typeConverter=TypeConverters.toString)
 
     missingValue = Param(Params._dummy(), "missingValue",
                          "The placeholder for the missing values. All occurrences of missingValue "
                          "will be imputed.", typeConverter=TypeConverters.toFloat)
 
+    def __init__(self, *args):
+        super(_ImputerParams, self).__init__(*args)
+        self._setDefault(strategy="mean", missingValue=float("nan"), relativeError=0.001)
+
     @since("2.2.0")
     def getStrategy(self):
         """
@@ -1468,16 +1536,20 @@ def getMissingValue(self):
 @inherit_doc
 class Imputer(JavaEstimator, _ImputerParams, JavaMLReadable, JavaMLWritable):
     """
-    Imputation estimator for completing missing values, either using the mean or the median
+    Imputation estimator for completing missing values, using the mean, median or mode
     of the columns in which the missing values are located. The input columns should be of
     numeric type. Currently Imputer does not support categorical features and
     possibly creates incorrect values for a categorical feature.
 
-    Note that the mean/median value is computed after filtering out missing values.
+    Note that the mean/median/mode value is computed after filtering out missing values.
     All Null values in the input columns are treated as missing, and so are also imputed. For
     computing median, :py:meth:`pyspark.sql.DataFrame.approxQuantile` is used with a
     relative error of `0.001`.
 
+    .. versionadded:: 2.2.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([(1.0, float("nan")), (2.0, float("nan")), (float("nan"), 3.0),
     ...                             (4.0, 4.0), (5.0, 5.0)], ["a", "b"])
     >>> imputer = Imputer()
@@ -1574,29 +1646,26 @@ class Imputer(JavaEstimator, _ImputerParams, JavaMLReadable, JavaMLWritable):
     >>> loadedModel = ImputerModel.load(modelPath)
     >>> loadedModel.transform(df).head().out_a == model.transform(df).head().out_a
     True
-
-    .. versionadded:: 2.2.0
     """
 
     @keyword_only
-    def __init__(self, strategy="mean", missingValue=float("nan"), inputCols=None,
+    def __init__(self, *, strategy="mean", missingValue=float("nan"), inputCols=None,
                  outputCols=None, inputCol=None, outputCol=None, relativeError=0.001):
         """
-        __init__(self, strategy="mean", missingValue=float("nan"), inputCols=None, \
+        __init__(self, \\*, strategy="mean", missingValue=float("nan"), inputCols=None, \
                  outputCols=None, inputCol=None, outputCol=None, relativeError=0.001):
         """
         super(Imputer, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Imputer", self.uid)
-        self._setDefault(strategy="mean", missingValue=float("nan"), relativeError=0.001)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("2.2.0")
-    def setParams(self, strategy="mean", missingValue=float("nan"), inputCols=None,
+    def setParams(self, *, strategy="mean", missingValue=float("nan"), inputCols=None,
                   outputCols=None, inputCol=None, outputCol=None, relativeError=0.001):
         """
-        setParams(self, strategy="mean", missingValue=float("nan"), inputCols=None, \
+        setParams(self, \\*, strategy="mean", missingValue=float("nan"), inputCols=None, \
                   outputCols=None, inputCol=None, outputCol=None, relativeError=0.001)
         Sets params for this Imputer.
         """
@@ -1713,6 +1782,10 @@ class Interaction(JavaTransformer, HasInputCols, HasOutputCol, JavaMLReadable, J
     `Vector(6, 8)` if all input features were numeric. If the first feature was instead nominal
     with four categories, the output would then be `Vector(0, 0, 0, 0, 3, 4, 0, 0)`.
 
+    .. versionadded:: 3.0.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([(0.0, 1.0), (2.0, 3.0)], ["a", "b"])
     >>> interaction = Interaction()
     >>> interaction.setInputCols(["a", "b"])
@@ -1732,14 +1805,12 @@ class Interaction(JavaTransformer, HasInputCols, HasOutputCol, JavaMLReadable, J
     >>> loadedInteraction = Interaction.load(interactionPath)
     >>> loadedInteraction.transform(df).head().ab == interaction.transform(df).head().ab
     True
-
-    .. versionadded:: 3.0.0
     """
 
     @keyword_only
-    def __init__(self, inputCols=None, outputCol=None):
+    def __init__(self, *, inputCols=None, outputCol=None):
         """
-        __init__(self, inputCols=None, outputCol=None):
+        __init__(self, \\*, inputCols=None, outputCol=None):
         """
         super(Interaction, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Interaction", self.uid)
@@ -1749,9 +1820,9 @@ def __init__(self, inputCols=None, outputCol=None):
 
     @keyword_only
     @since("3.0.0")
-    def setParams(self, inputCols=None, outputCol=None):
+    def setParams(self, *, inputCols=None, outputCol=None):
         """
-        setParams(self, inputCols=None, outputCol=None)
+        setParams(self, \\*, inputCols=None, outputCol=None)
         Sets params for this Interaction.
         """
         kwargs = self._input_kwargs
@@ -1788,6 +1859,10 @@ class MaxAbsScaler(JavaEstimator, _MaxAbsScalerParams, JavaMLReadable, JavaMLWri
     absolute value in each feature. It does not shift/center the data, and thus does not destroy
     any sparsity.
 
+    .. versionadded:: 2.0.0
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> df = spark.createDataFrame([(Vectors.dense([1.0]),), (Vectors.dense([2.0]),)], ["a"])
     >>> maScaler = MaxAbsScaler(outputCol="scaled")
@@ -1816,14 +1891,14 @@ class MaxAbsScaler(JavaEstimator, _MaxAbsScalerParams, JavaMLReadable, JavaMLWri
     >>> loadedModel = MaxAbsScalerModel.load(modelPath)
     >>> loadedModel.maxAbs == model.maxAbs
     True
-
-    .. versionadded:: 2.0.0
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
     """
 
     @keyword_only
-    def __init__(self, inputCol=None, outputCol=None):
+    def __init__(self, *, inputCol=None, outputCol=None):
         """
-        __init__(self, inputCol=None, outputCol=None)
+        __init__(self, \\*, inputCol=None, outputCol=None)
         """
         super(MaxAbsScaler, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.MaxAbsScaler", self.uid)
@@ -1833,9 +1908,9 @@ def __init__(self, inputCol=None, outputCol=None):
 
     @keyword_only
     @since("2.0.0")
-    def setParams(self, inputCol=None, outputCol=None):
+    def setParams(self, *, inputCol=None, outputCol=None):
         """
-        setParams(self, inputCol=None, outputCol=None)
+        setParams(self, \\*, inputCol=None, outputCol=None)
         Sets params for this MaxAbsScaler.
         """
         kwargs = self._input_kwargs
@@ -1897,8 +1972,14 @@ class MinHashLSH(_LSH, HasInputCol, HasOutputCol, HasSeed, JavaMLReadable, JavaM
     in the space. This set contains elements 2, 3, and 5. Also, any input vector must have at
     least 1 non-zero index, and all non-zero values are treated as binary "1" values.
 
-    .. seealso:: `Wikipedia on MinHash <https://en.wikipedia.org/wiki/MinHash>`_
+    .. versionadded:: 2.2.0
+
+    Notes
+    -----
+    See `Wikipedia on MinHash <https://en.wikipedia.org/wiki/MinHash>`_
 
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> from pyspark.sql.functions import col
     >>> data = [(0, Vectors.sparse(6, [0, 1, 2], [1.0, 1.0, 1.0]),),
@@ -1945,26 +2026,23 @@ class MinHashLSH(_LSH, HasInputCol, HasOutputCol, HasSeed, JavaMLReadable, JavaM
     >>> model2 = MinHashLSHModel.load(modelPath)
     >>> model.transform(df).head().hashes == model2.transform(df).head().hashes
     True
-
-    .. versionadded:: 2.2.0
     """
 
     @keyword_only
-    def __init__(self, inputCol=None, outputCol=None, seed=None, numHashTables=1):
+    def __init__(self, *, inputCol=None, outputCol=None, seed=None, numHashTables=1):
         """
-        __init__(self, inputCol=None, outputCol=None, seed=None, numHashTables=1)
+        __init__(self, \\*, inputCol=None, outputCol=None, seed=None, numHashTables=1)
         """
         super(MinHashLSH, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.MinHashLSH", self.uid)
-        self._setDefault(numHashTables=1)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("2.2.0")
-    def setParams(self, inputCol=None, outputCol=None, seed=None, numHashTables=1):
+    def setParams(self, *, inputCol=None, outputCol=None, seed=None, numHashTables=1):
         """
-        setParams(self, inputCol=None, outputCol=None, seed=None, numHashTables=1)
+        setParams(self, \\*, inputCol=None, outputCol=None, seed=None, numHashTables=1)
         Sets params for this MinHashLSH.
         """
         kwargs = self._input_kwargs
@@ -1988,10 +2066,12 @@ class MinHashLSHModel(_LSHModel, JavaMLReadable, JavaMLWritable):
     :math:`h_i(x) = ((x \cdot a_i + b_i) \mod prime)` This hash family is approximately min-wise
     independent according to the reference.
 
-    .. seealso:: Tom Bohman, Colin Cooper, and Alan Frieze. "Min-wise independent linear
-        permutations." Electronic Journal of Combinatorics 7 (2000): R26.
-
     .. versionadded:: 2.2.0
+
+    Notes
+    -----
+    See Tom Bohman, Colin Cooper, and Alan Frieze. "Min-wise independent linear permutations."
+    Electronic Journal of Combinatorics 7 (2000): R26.
     """
 
 
@@ -2007,6 +2087,10 @@ class _MinMaxScalerParams(HasInputCol, HasOutputCol):
     max = Param(Params._dummy(), "max", "Upper bound of the output feature range",
                 typeConverter=TypeConverters.toFloat)
 
+    def __init__(self, *args):
+        super(_MinMaxScalerParams, self).__init__(*args)
+        self._setDefault(min=0.0, max=1.0)
+
     @since("1.6.0")
     def getMin(self):
         """
@@ -2033,9 +2117,15 @@ class MinMaxScaler(JavaEstimator, _MinMaxScalerParams, JavaMLReadable, JavaMLWri
 
     For the case E_max == E_min, Rescaled(e_i) = 0.5 * (max + min)
 
-    .. note:: Since zero values will probably be transformed to non-zero values, output of the
-        transformer will be DenseVector even for sparse input.
+    .. versionadded:: 1.6.0
+
+    Notes
+    -----
+    Since zero values will probably be transformed to non-zero values, output of the
+    transformer will be DenseVector even for sparse input.
 
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> df = spark.createDataFrame([(Vectors.dense([0.0]),), (Vectors.dense([2.0]),)], ["a"])
     >>> mmScaler = MinMaxScaler(outputCol="scaled")
@@ -2070,26 +2160,25 @@ class MinMaxScaler(JavaEstimator, _MinMaxScalerParams, JavaMLReadable, JavaMLWri
     True
     >>> loadedModel.originalMax == model.originalMax
     True
-
-    .. versionadded:: 1.6.0
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
     """
 
     @keyword_only
-    def __init__(self, min=0.0, max=1.0, inputCol=None, outputCol=None):
+    def __init__(self, *, min=0.0, max=1.0, inputCol=None, outputCol=None):
         """
-        __init__(self, min=0.0, max=1.0, inputCol=None, outputCol=None)
+        __init__(self, \\*, min=0.0, max=1.0, inputCol=None, outputCol=None)
         """
         super(MinMaxScaler, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.MinMaxScaler", self.uid)
-        self._setDefault(min=0.0, max=1.0)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("1.6.0")
-    def setParams(self, min=0.0, max=1.0, inputCol=None, outputCol=None):
+    def setParams(self, *, min=0.0, max=1.0, inputCol=None, outputCol=None):
         """
-        setParams(self, min=0.0, max=1.0, inputCol=None, outputCol=None)
+        setParams(self, \\*, min=0.0, max=1.0, inputCol=None, outputCol=None)
         Sets params for this MinMaxScaler.
         """
         kwargs = self._input_kwargs
@@ -2178,7 +2267,6 @@ def originalMax(self):
 
 
 @inherit_doc
-@ignore_unicode_prefix
 class NGram(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWritable):
     """
     A feature transformer that converts the input array of strings into an array of n-grams. Null
@@ -2189,6 +2277,10 @@ class NGram(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWr
     When the input array length is less than n (number of elements per n-gram), no n-grams are
     returned.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([Row(inputTokens=["a", "b", "c", "d", "e"])])
     >>> ngram = NGram(n=2)
     >>> ngram.setInputCol("inputTokens")
@@ -2196,15 +2288,15 @@ class NGram(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWr
     >>> ngram.setOutputCol("nGrams")
     NGram...
     >>> ngram.transform(df).head()
-    Row(inputTokens=[u'a', u'b', u'c', u'd', u'e'], nGrams=[u'a b', u'b c', u'c d', u'd e'])
+    Row(inputTokens=['a', 'b', 'c', 'd', 'e'], nGrams=['a b', 'b c', 'c d', 'd e'])
     >>> # Change n-gram length
     >>> ngram.setParams(n=4).transform(df).head()
-    Row(inputTokens=[u'a', u'b', u'c', u'd', u'e'], nGrams=[u'a b c d', u'b c d e'])
+    Row(inputTokens=['a', 'b', 'c', 'd', 'e'], nGrams=['a b c d', 'b c d e'])
     >>> # Temporarily modify output column.
     >>> ngram.transform(df, {ngram.outputCol: "output"}).head()
-    Row(inputTokens=[u'a', u'b', u'c', u'd', u'e'], output=[u'a b c d', u'b c d e'])
+    Row(inputTokens=['a', 'b', 'c', 'd', 'e'], output=['a b c d', 'b c d e'])
     >>> ngram.transform(df).head()
-    Row(inputTokens=[u'a', u'b', u'c', u'd', u'e'], nGrams=[u'a b c d', u'b c d e'])
+    Row(inputTokens=['a', 'b', 'c', 'd', 'e'], nGrams=['a b c d', 'b c d e'])
     >>> # Must use keyword arguments to specify params.
     >>> ngram.setParams("text")
     Traceback (most recent call last):
@@ -2215,17 +2307,17 @@ class NGram(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWr
     >>> loadedNGram = NGram.load(ngramPath)
     >>> loadedNGram.getN() == ngram.getN()
     True
-
-    .. versionadded:: 1.5.0
+    >>> loadedNGram.transform(df).take(1) == ngram.transform(df).take(1)
+    True
     """
 
     n = Param(Params._dummy(), "n", "number of elements per n-gram (>=1)",
               typeConverter=TypeConverters.toInt)
 
     @keyword_only
-    def __init__(self, n=2, inputCol=None, outputCol=None):
+    def __init__(self, *, n=2, inputCol=None, outputCol=None):
         """
-        __init__(self, n=2, inputCol=None, outputCol=None)
+        __init__(self, \\*, n=2, inputCol=None, outputCol=None)
         """
         super(NGram, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.NGram", self.uid)
@@ -2235,9 +2327,9 @@ def __init__(self, n=2, inputCol=None, outputCol=None):
 
     @keyword_only
     @since("1.5.0")
-    def setParams(self, n=2, inputCol=None, outputCol=None):
+    def setParams(self, *, n=2, inputCol=None, outputCol=None):
         """
-        setParams(self, n=2, inputCol=None, outputCol=None)
+        setParams(self, \\*, n=2, inputCol=None, outputCol=None)
         Sets params for this NGram.
         """
         kwargs = self._input_kwargs
@@ -2275,6 +2367,10 @@ class Normalizer(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, Jav
     """
      Normalize a vector to have unit norm using the given p-norm.
 
+    .. versionadded:: 1.4.0
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> svec = Vectors.sparse(4, {1: 4.0, 3: 3.0})
     >>> df = spark.createDataFrame([(Vectors.dense([3.0, -4.0]), svec)], ["dense", "sparse"])
@@ -2295,17 +2391,17 @@ class Normalizer(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, Jav
     >>> loadedNormalizer = Normalizer.load(normalizerPath)
     >>> loadedNormalizer.getP() == normalizer.getP()
     True
-
-    .. versionadded:: 1.4.0
+    >>> loadedNormalizer.transform(df).take(1) == normalizer.transform(df).take(1)
+    True
     """
 
     p = Param(Params._dummy(), "p", "the p norm value.",
               typeConverter=TypeConverters.toFloat)
 
     @keyword_only
-    def __init__(self, p=2.0, inputCol=None, outputCol=None):
+    def __init__(self, *, p=2.0, inputCol=None, outputCol=None):
         """
-        __init__(self, p=2.0, inputCol=None, outputCol=None)
+        __init__(self, \\*, p=2.0, inputCol=None, outputCol=None)
         """
         super(Normalizer, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Normalizer", self.uid)
@@ -2315,9 +2411,9 @@ def __init__(self, p=2.0, inputCol=None, outputCol=None):
 
     @keyword_only
     @since("1.4.0")
-    def setParams(self, p=2.0, inputCol=None, outputCol=None):
+    def setParams(self, *, p=2.0, inputCol=None, outputCol=None):
         """
-        setParams(self, p=2.0, inputCol=None, outputCol=None)
+        setParams(self, \\*, p=2.0, inputCol=None, outputCol=None)
         Sets params for this Normalizer.
         """
         kwargs = self._input_kwargs
@@ -2368,6 +2464,10 @@ class _OneHotEncoderParams(HasInputCol, HasInputCols, HasOutputCol, HasOutputCol
     dropLast = Param(Params._dummy(), "dropLast", "whether to drop the last category",
                      typeConverter=TypeConverters.toBoolean)
 
+    def __init__(self, *args):
+        super(_OneHotEncoderParams, self).__init__(*args)
+        self._setDefault(handleInvalid="error", dropLast=True)
+
     @since("2.3.0")
     def getDropLast(self):
         """
@@ -2387,19 +2487,27 @@ class OneHotEncoder(JavaEstimator, _OneHotEncoderParams, JavaMLReadable, JavaMLW
     because it makes the vector entries sum up to one, and hence linearly dependent.
     So an input value of 4.0 maps to `[0.0, 0.0, 0.0, 0.0]`.
 
-    .. note:: This is different from scikit-learn's OneHotEncoder, which keeps all categories.
-        The output vectors are sparse.
-
     When :py:attr:`handleInvalid` is configured to 'keep', an extra "category" indicating invalid
     values is added as last category. So when :py:attr:`dropLast` is true, invalid values are
     encoded as all-zeros vector.
 
-    .. note:: When encoding multi-column by using :py:attr:`inputCols` and
-        :py:attr:`outputCols` params, input/output cols come in pairs, specified by the order in
-        the arrays, and each pair is treated independently.
+    .. versionadded:: 2.3.0
+
+    Notes
+    -----
+    This is different from scikit-learn's OneHotEncoder, which keeps all categories.
+    The output vectors are sparse.
+
+    When encoding multi-column by using :py:attr:`inputCols` and
+    :py:attr:`outputCols` params, input/output cols come in pairs, specified by the order in
+    the arrays, and each pair is treated independently.
 
-    .. seealso:: :py:class:`StringIndexer` for converting categorical values into category indices
+    See Also
+    --------
+    StringIndexer : for converting categorical values into category indices
 
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> df = spark.createDataFrame([(0.0,), (1.0,), (2.0,)], ["input"])
     >>> ohe = OneHotEncoder()
@@ -2428,31 +2536,30 @@ class OneHotEncoder(JavaEstimator, _OneHotEncoderParams, JavaMLReadable, JavaMLW
     >>> loadedModel = OneHotEncoderModel.load(modelPath)
     >>> loadedModel.categorySizes == model.categorySizes
     True
-
-    .. versionadded:: 2.3.0
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
     """
 
     @keyword_only
-    def __init__(self, inputCols=None, outputCols=None, handleInvalid="error", dropLast=True,
+    def __init__(self, *, inputCols=None, outputCols=None, handleInvalid="error", dropLast=True,
                  inputCol=None, outputCol=None):
         """
-        __init__(self, inputCols=None, outputCols=None, handleInvalid="error", dropLast=True, \
+        __init__(self, \\*, inputCols=None, outputCols=None, handleInvalid="error", dropLast=True, \
                  inputCol=None, outputCol=None)
         """
         super(OneHotEncoder, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.feature.OneHotEncoder", self.uid)
-        self._setDefault(handleInvalid="error", dropLast=True)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("2.3.0")
-    def setParams(self, inputCols=None, outputCols=None, handleInvalid="error", dropLast=True,
-                  inputCol=None, outputCol=None):
+    def setParams(self, *, inputCols=None, outputCols=None, handleInvalid="error",
+                  dropLast=True, inputCol=None, outputCol=None):
         """
-        setParams(self, inputCols=None, outputCols=None, handleInvalid="error", dropLast=True, \
-                  inputCol=None, outputCol=None)
+        setParams(self, \\*, inputCols=None, outputCols=None, handleInvalid="error", \
+                  dropLast=True, inputCol=None, outputCol=None)
         Sets params for this OneHotEncoder.
         """
         kwargs = self._input_kwargs
@@ -2573,6 +2680,10 @@ class PolynomialExpansion(JavaTransformer, HasInputCol, HasOutputCol, JavaMLRead
     multiplication distributes over addition". Take a 2-variable feature vector as an example:
     `(x, y)`, if we want to expand it with degree 2, then we get `(x, x * x, y, x * y, y * y)`.
 
+    .. versionadded:: 1.4.0
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> df = spark.createDataFrame([(Vectors.dense([0.5, 2.0]),)], ["dense"])
     >>> px = PolynomialExpansion(degree=2)
@@ -2589,17 +2700,17 @@ class PolynomialExpansion(JavaTransformer, HasInputCol, HasOutputCol, JavaMLRead
     >>> loadedPx = PolynomialExpansion.load(polyExpansionPath)
     >>> loadedPx.getDegree() == px.getDegree()
     True
-
-    .. versionadded:: 1.4.0
+    >>> loadedPx.transform(df).take(1) == px.transform(df).take(1)
+    True
     """
 
     degree = Param(Params._dummy(), "degree", "the polynomial degree to expand (>= 1)",
                    typeConverter=TypeConverters.toInt)
 
     @keyword_only
-    def __init__(self, degree=2, inputCol=None, outputCol=None):
+    def __init__(self, *, degree=2, inputCol=None, outputCol=None):
         """
-        __init__(self, degree=2, inputCol=None, outputCol=None)
+        __init__(self, \\*, degree=2, inputCol=None, outputCol=None)
         """
         super(PolynomialExpansion, self).__init__()
         self._java_obj = self._new_java_obj(
@@ -2610,9 +2721,9 @@ def __init__(self, degree=2, inputCol=None, outputCol=None):
 
     @keyword_only
     @since("1.4.0")
-    def setParams(self, degree=2, inputCol=None, outputCol=None):
+    def setParams(self, *, degree=2, inputCol=None, outputCol=None):
         """
-        setParams(self, degree=2, inputCol=None, outputCol=None)
+        setParams(self, \\*, degree=2, inputCol=None, outputCol=None)
         Sets params for this PolynomialExpansion.
         """
         kwargs = self._input_kwargs
@@ -2659,6 +2770,10 @@ class QuantileDiscretizer(JavaEstimator, HasInputCol, HasOutputCol, HasInputCols
     column, the :py:attr:`numBucketsArray` parameter can be set, or if the number of buckets
     should be the same across columns, :py:attr:`numBuckets` can be set as a convenience.
 
+    .. versionadded:: 2.0.0
+
+    Notes
+    -----
     NaN handling: Note also that
     :py:class:`QuantileDiscretizer` will raise an error when it finds NaN values in the dataset,
     but the user can also choose to either keep or remove NaN values within the dataset by setting
@@ -2672,6 +2787,8 @@ class QuantileDiscretizer(JavaEstimator, HasInputCol, HasOutputCol, HasInputCols
     :py:attr:`relativeError` parameter.
     The lower and upper bin bounds will be `-Infinity` and `+Infinity`, covering all real values.
 
+    Examples
+    --------
     >>> values = [(0.1,), (0.4,), (1.2,), (1.5,), (float("nan"),), (float("nan"),)]
     >>> df1 = spark.createDataFrame(values, ["values"])
     >>> qds1 = QuantileDiscretizer(inputCol="values", outputCol="buckets")
@@ -2733,8 +2850,6 @@ class QuantileDiscretizer(JavaEstimator, HasInputCol, HasOutputCol, HasInputCols
     |   1.5|   1.5|    4.0|    4.0|
     +------+------+-------+-------+
     ...
-
-    .. versionadded:: 2.0.0
     """
 
     numBuckets = Param(Params._dummy(), "numBuckets",
@@ -2760,10 +2875,10 @@ class QuantileDiscretizer(JavaEstimator, HasInputCol, HasOutputCol, HasInputCols
                             typeConverter=TypeConverters.toListInt)
 
     @keyword_only
-    def __init__(self, numBuckets=2, inputCol=None, outputCol=None, relativeError=0.001,
+    def __init__(self, *, numBuckets=2, inputCol=None, outputCol=None, relativeError=0.001,
                  handleInvalid="error", numBucketsArray=None, inputCols=None, outputCols=None):
         """
-        __init__(self, numBuckets=2, inputCol=None, outputCol=None, relativeError=0.001, \
+        __init__(self, \\*, numBuckets=2, inputCol=None, outputCol=None, relativeError=0.001, \
                  handleInvalid="error", numBucketsArray=None, inputCols=None, outputCols=None)
         """
         super(QuantileDiscretizer, self).__init__()
@@ -2775,10 +2890,10 @@ def __init__(self, numBuckets=2, inputCol=None, outputCol=None, relativeError=0.
 
     @keyword_only
     @since("2.0.0")
-    def setParams(self, numBuckets=2, inputCol=None, outputCol=None, relativeError=0.001,
+    def setParams(self, *, numBuckets=2, inputCol=None, outputCol=None, relativeError=0.001,
                   handleInvalid="error", numBucketsArray=None, inputCols=None, outputCols=None):
         """
-        setParams(self, numBuckets=2, inputCol=None, outputCol=None, relativeError=0.001, \
+        setParams(self, \\*, numBuckets=2, inputCol=None, outputCol=None, relativeError=0.001, \
                   handleInvalid="error", numBucketsArray=None, inputCols=None, outputCols=None)
         Set the params for the QuantileDiscretizer
         """
@@ -2885,6 +3000,11 @@ class _RobustScalerParams(HasInputCol, HasOutputCol, HasRelativeError):
     withScaling = Param(Params._dummy(), "withScaling", "Whether to scale the data to "
                         "quantile range", typeConverter=TypeConverters.toBoolean)
 
+    def __init__(self, *args):
+        super(_RobustScalerParams, self).__init__(*args)
+        self._setDefault(lower=0.25, upper=0.75, withCentering=False, withScaling=True,
+                         relativeError=0.001)
+
     @since("3.0.0")
     def getLower(self):
         """
@@ -2925,6 +3045,10 @@ class RobustScaler(JavaEstimator, _RobustScalerParams, JavaMLReadable, JavaMLWri
     stored to be used on later data using the transform method.
     Note that NaN values are ignored in the computation of medians and ranges.
 
+    .. versionadded:: 3.0.0
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> data = [(0, Vectors.dense([0.0, 0.0]),),
     ...         (1, Vectors.dense([1.0, -1.0]),),
@@ -2960,30 +3084,28 @@ class RobustScaler(JavaEstimator, _RobustScalerParams, JavaMLReadable, JavaMLWri
     True
     >>> loadedModel.range == model.range
     True
-
-    .. versionadded:: 3.0.0
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
     """
 
     @keyword_only
-    def __init__(self, lower=0.25, upper=0.75, withCentering=False, withScaling=True,
+    def __init__(self, *, lower=0.25, upper=0.75, withCentering=False, withScaling=True,
                  inputCol=None, outputCol=None, relativeError=0.001):
         """
-        __init__(self, lower=0.25, upper=0.75, withCentering=False, withScaling=True, \
+        __init__(self, \\*, lower=0.25, upper=0.75, withCentering=False, withScaling=True, \
                  inputCol=None, outputCol=None, relativeError=0.001)
         """
         super(RobustScaler, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.RobustScaler", self.uid)
-        self._setDefault(lower=0.25, upper=0.75, withCentering=False, withScaling=True,
-                         relativeError=0.001)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("3.0.0")
-    def setParams(self, lower=0.25, upper=0.75, withCentering=False, withScaling=True,
+    def setParams(self, *, lower=0.25, upper=0.75, withCentering=False, withScaling=True,
                   inputCol=None, outputCol=None, relativeError=0.001):
         """
-        setParams(self, lower=0.25, upper=0.75, withCentering=False, withScaling=True, \
+        setParams(self, \\*, lower=0.25, upper=0.75, withCentering=False, withScaling=True, \
                   inputCol=None, outputCol=None, relativeError=0.001)
         Sets params for this RobustScaler.
         """
@@ -3082,7 +3204,6 @@ def range(self):
 
 
 @inherit_doc
-@ignore_unicode_prefix
 class RegexTokenizer(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWritable):
     """
     A regex based tokenizer that extracts tokens either by using the
@@ -3092,6 +3213,10 @@ class RegexTokenizer(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable,
     length.
     It returns an array of strings that can be empty.
 
+    .. versionadded:: 1.4.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([("A B  c",)], ["text"])
     >>> reTokenizer = RegexTokenizer()
     >>> reTokenizer.setInputCol("text")
@@ -3099,15 +3224,15 @@ class RegexTokenizer(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable,
     >>> reTokenizer.setOutputCol("words")
     RegexTokenizer...
     >>> reTokenizer.transform(df).head()
-    Row(text=u'A B  c', words=[u'a', u'b', u'c'])
+    Row(text='A B  c', words=['a', 'b', 'c'])
     >>> # Change a parameter.
     >>> reTokenizer.setParams(outputCol="tokens").transform(df).head()
-    Row(text=u'A B  c', tokens=[u'a', u'b', u'c'])
+    Row(text='A B  c', tokens=['a', 'b', 'c'])
     >>> # Temporarily modify a parameter.
     >>> reTokenizer.transform(df, {reTokenizer.outputCol: "words"}).head()
-    Row(text=u'A B  c', words=[u'a', u'b', u'c'])
+    Row(text='A B  c', words=['a', 'b', 'c'])
     >>> reTokenizer.transform(df).head()
-    Row(text=u'A B  c', tokens=[u'a', u'b', u'c'])
+    Row(text='A B  c', tokens=['a', 'b', 'c'])
     >>> # Must use keyword arguments to specify params.
     >>> reTokenizer.setParams("text")
     Traceback (most recent call last):
@@ -3120,8 +3245,8 @@ class RegexTokenizer(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable,
     True
     >>> loadedReTokenizer.getGaps() == reTokenizer.getGaps()
     True
-
-    .. versionadded:: 1.4.0
+    >>> loadedReTokenizer.transform(df).take(1) == reTokenizer.transform(df).take(1)
+    True
     """
 
     minTokenLength = Param(Params._dummy(), "minTokenLength", "minimum token length (>= 0)",
@@ -3134,10 +3259,10 @@ class RegexTokenizer(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable,
                         "lowercase before tokenizing", typeConverter=TypeConverters.toBoolean)
 
     @keyword_only
-    def __init__(self, minTokenLength=1, gaps=True, pattern="\\s+", inputCol=None,
+    def __init__(self, *, minTokenLength=1, gaps=True, pattern="\\s+", inputCol=None,
                  outputCol=None, toLowercase=True):
         """
-        __init__(self, minTokenLength=1, gaps=True, pattern="\\s+", inputCol=None, \
+        __init__(self, \\*, minTokenLength=1, gaps=True, pattern="\\s+", inputCol=None, \
                  outputCol=None, toLowercase=True)
         """
         super(RegexTokenizer, self).__init__()
@@ -3148,10 +3273,10 @@ def __init__(self, minTokenLength=1, gaps=True, pattern="\\s+", inputCol=None,
 
     @keyword_only
     @since("1.4.0")
-    def setParams(self, minTokenLength=1, gaps=True, pattern="\\s+", inputCol=None,
+    def setParams(self, *, minTokenLength=1, gaps=True, pattern="\\s+", inputCol=None,
                   outputCol=None, toLowercase=True):
         """
-        setParams(self, minTokenLength=1, gaps=True, pattern="\\s+", inputCol=None, \
+        setParams(self, \\*, minTokenLength=1, gaps=True, pattern="\\s+", inputCol=None, \
                   outputCol=None, toLowercase=True)
         Sets params for this RegexTokenizer.
         """
@@ -3231,9 +3356,13 @@ def setOutputCol(self, value):
 class SQLTransformer(JavaTransformer, JavaMLReadable, JavaMLWritable):
     """
     Implements the transforms which are defined by SQL statement.
-    Currently we only support SQL syntax like 'SELECT ... FROM __THIS__'
-    where '__THIS__' represents the underlying table of the input dataset.
+    Currently we only support SQL syntax like `SELECT ... FROM __THIS__`
+    where `__THIS__` represents the underlying table of the input dataset.
+
+    .. versionadded:: 1.6.0
 
+    Examples
+    --------
     >>> df = spark.createDataFrame([(0, 1.0, 3.0), (2, 2.0, 5.0)], ["id", "v1", "v2"])
     >>> sqlTrans = SQLTransformer(
     ...     statement="SELECT *, (v1 + v2) AS v3, (v1 * v2) AS v4 FROM __THIS__")
@@ -3244,17 +3373,17 @@ class SQLTransformer(JavaTransformer, JavaMLReadable, JavaMLWritable):
     >>> loadedSqlTrans = SQLTransformer.load(sqlTransformerPath)
     >>> loadedSqlTrans.getStatement() == sqlTrans.getStatement()
     True
-
-    .. versionadded:: 1.6.0
+    >>> loadedSqlTrans.transform(df).take(1) == sqlTrans.transform(df).take(1)
+    True
     """
 
     statement = Param(Params._dummy(), "statement", "SQL statement",
                       typeConverter=TypeConverters.toString)
 
     @keyword_only
-    def __init__(self, statement=None):
+    def __init__(self, *, statement=None):
         """
-        __init__(self, statement=None)
+        __init__(self, \\*, statement=None)
         """
         super(SQLTransformer, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.SQLTransformer", self.uid)
@@ -3263,9 +3392,9 @@ def __init__(self, statement=None):
 
     @keyword_only
     @since("1.6.0")
-    def setParams(self, statement=None):
+    def setParams(self, *, statement=None):
         """
-        setParams(self, statement=None)
+        setParams(self, \\*, statement=None)
         Sets params for this SQLTransformer.
         """
         kwargs = self._input_kwargs
@@ -3298,6 +3427,10 @@ class _StandardScalerParams(HasInputCol, HasOutputCol):
     withStd = Param(Params._dummy(), "withStd", "Scale to unit standard deviation",
                     typeConverter=TypeConverters.toBoolean)
 
+    def __init__(self, *args):
+        super(_StandardScalerParams, self).__init__(*args)
+        self._setDefault(withMean=False, withStd=True)
+
     @since("1.4.0")
     def getWithMean(self):
         """
@@ -3323,6 +3456,10 @@ class StandardScaler(JavaEstimator, _StandardScalerParams, JavaMLReadable, JavaM
     <https://en.wikipedia.org/wiki/Standard_deviation#Corrected_sample_standard_deviation>`_,
     which is computed as the square root of the unbiased sample variance.
 
+    .. versionadded:: 1.4.0
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> df = spark.createDataFrame([(Vectors.dense([0.0]),), (Vectors.dense([2.0]),)], ["a"])
     >>> standardScaler = StandardScaler()
@@ -3355,26 +3492,25 @@ class StandardScaler(JavaEstimator, _StandardScalerParams, JavaMLReadable, JavaM
     True
     >>> loadedModel.mean == model.mean
     True
-
-    .. versionadded:: 1.4.0
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
     """
 
     @keyword_only
-    def __init__(self, withMean=False, withStd=True, inputCol=None, outputCol=None):
+    def __init__(self, *, withMean=False, withStd=True, inputCol=None, outputCol=None):
         """
-        __init__(self, withMean=False, withStd=True, inputCol=None, outputCol=None)
+        __init__(self, \\*, withMean=False, withStd=True, inputCol=None, outputCol=None)
         """
         super(StandardScaler, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.StandardScaler", self.uid)
-        self._setDefault(withMean=False, withStd=True)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("1.4.0")
-    def setParams(self, withMean=False, withStd=True, inputCol=None, outputCol=None):
+    def setParams(self, *, withMean=False, withStd=True, inputCol=None, outputCol=None):
         """
-        setParams(self, withMean=False, withStd=True, inputCol=None, outputCol=None)
+        setParams(self, \\*, withMean=False, withStd=True, inputCol=None, outputCol=None)
         Sets params for this StandardScaler.
         """
         kwargs = self._input_kwargs
@@ -3489,6 +3625,10 @@ class StringIndexer(JavaEstimator, _StringIndexerParams, JavaMLReadable, JavaMLW
     so the most frequent label gets index 0. The ordering behavior is controlled by
     setting :py:attr:`stringOrderType`. Its default value is 'frequencyDesc'.
 
+    .. versionadded:: 1.4.0
+
+    Examples
+    --------
     >>> stringIndexer = StringIndexer(inputCol="label", outputCol="indexed",
     ...     stringOrderType="frequencyDesc")
     >>> stringIndexer.setHandleInvalid("error")
@@ -3520,6 +3660,8 @@ class StringIndexer(JavaEstimator, _StringIndexerParams, JavaMLReadable, JavaMLW
     >>> loadedInverter = IndexToString.load(indexToStringPath)
     >>> loadedInverter.getLabels() == inverter.getLabels()
     True
+    >>> loadedModel.transform(stringIndDf).take(1) == model.transform(stringIndDf).take(1)
+    True
     >>> stringIndexer.getStringOrderType()
     'frequencyDesc'
     >>> stringIndexer = StringIndexer(inputCol="label", outputCol="indexed", handleInvalid="error",
@@ -3556,15 +3698,13 @@ class StringIndexer(JavaEstimator, _StringIndexerParams, JavaMLReadable, JavaMLW
     >>> sorted(set([(i[0], i[1], i[2]) for i in result.select(result.id, result.index1,
     ...     result.index2).collect()]), key=lambda x: x[0])
     [(0, 0.0, 0.0), (1, 1.0, 1.0), (2, 2.0, 0.0), (3, 0.0, 1.0), (4, 0.0, 1.0), (5, 2.0, 1.0)]
-
-    .. versionadded:: 1.4.0
     """
 
     @keyword_only
-    def __init__(self, inputCol=None, outputCol=None, inputCols=None, outputCols=None,
+    def __init__(self, *, inputCol=None, outputCol=None, inputCols=None, outputCols=None,
                  handleInvalid="error", stringOrderType="frequencyDesc"):
         """
-        __init__(self, inputCol=None, outputCol=None, inputCols=None, outputCols=None, \
+        __init__(self, \\*, inputCol=None, outputCol=None, inputCols=None, outputCols=None, \
                  handleInvalid="error", stringOrderType="frequencyDesc")
         """
         super(StringIndexer, self).__init__()
@@ -3574,10 +3714,10 @@ def __init__(self, inputCol=None, outputCol=None, inputCols=None, outputCols=Non
 
     @keyword_only
     @since("1.4.0")
-    def setParams(self, inputCol=None, outputCol=None, inputCols=None, outputCols=None,
+    def setParams(self, *, inputCol=None, outputCol=None, inputCols=None, outputCols=None,
                   handleInvalid="error", stringOrderType="frequencyDesc"):
         """
-        setParams(self, inputCol=None, outputCol=None, inputCols=None, outputCols=None, \
+        setParams(self, \\*, inputCol=None, outputCol=None, inputCols=None, outputCols=None, \
                   handleInvalid="error", stringOrderType="frequencyDesc")
         Sets params for this StringIndexer.
         """
@@ -3711,20 +3851,35 @@ def from_arrays_of_labels(cls, arrayOfLabels, inputCols, outputCols=None,
     def labels(self):
         """
         Ordered list of labels, corresponding to indices to be assigned.
+
+        .. deprecated:: 3.1.0
+            It will be removed in future versions. Use `labelsArray` method instead.
         """
         return self._call_java("labels")
 
+    @property
+    @since("3.0.2")
+    def labelsArray(self):
+        """
+        Array of ordered list of labels, corresponding to indices to be assigned
+        for each input column.
+        """
+        return self._call_java("labelsArray")
+
 
 @inherit_doc
 class IndexToString(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWritable):
     """
-    A :py:class:`Transformer` that maps a column of indices back to a new column of
+    A :py:class:`pyspark.ml.base.Transformer` that maps a column of indices back to a new column of
     corresponding string values.
     The index-string mapping is either from the ML attributes of the input column,
     or from user-supplied labels (which take precedence over ML attributes).
-    See :class:`StringIndexer` for converting strings into indices.
 
     .. versionadded:: 1.6.0
+
+    See Also
+    --------
+    StringIndexer : for converting categorical values into category indices
     """
 
     labels = Param(Params._dummy(), "labels",
@@ -3733,9 +3888,9 @@ class IndexToString(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable,
                    typeConverter=TypeConverters.toListString)
 
     @keyword_only
-    def __init__(self, inputCol=None, outputCol=None, labels=None):
+    def __init__(self, *, inputCol=None, outputCol=None, labels=None):
         """
-        __init__(self, inputCol=None, outputCol=None, labels=None)
+        __init__(self, \\*, inputCol=None, outputCol=None, labels=None)
         """
         super(IndexToString, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.IndexToString",
@@ -3745,9 +3900,9 @@ def __init__(self, inputCol=None, outputCol=None, labels=None):
 
     @keyword_only
     @since("1.6.0")
-    def setParams(self, inputCol=None, outputCol=None, labels=None):
+    def setParams(self, *, inputCol=None, outputCol=None, labels=None):
         """
-        setParams(self, inputCol=None, outputCol=None, labels=None)
+        setParams(self, \\*, inputCol=None, outputCol=None, labels=None)
         Sets params for this IndexToString.
         """
         kwargs = self._input_kwargs
@@ -3788,8 +3943,14 @@ class StopWordsRemover(JavaTransformer, HasInputCol, HasOutputCol, HasInputCols,
     the :py:attr:`inputCols` parameter. Note that when both the :py:attr:`inputCol` and
     :py:attr:`inputCols` parameters are set, an Exception will be thrown.
 
-    .. note:: null values from input array are preserved unless adding null to stopWords explicitly.
+    .. versionadded:: 1.6.0
+
+    Notes
+    -----
+    null values from input array are preserved unless adding null to stopWords explicitly.
 
+    Examples
+    --------
     >>> df = spark.createDataFrame([(["a", "b", "c"],)], ["text"])
     >>> remover = StopWordsRemover(stopWords=["b"])
     >>> remover.setInputCol("text")
@@ -3805,6 +3966,8 @@ class StopWordsRemover(JavaTransformer, HasInputCol, HasOutputCol, HasInputCols,
     True
     >>> loadedRemover.getCaseSensitive() == remover.getCaseSensitive()
     True
+    >>> loadedRemover.transform(df).take(1) == remover.transform(df).take(1)
+    True
     >>> df2 = spark.createDataFrame([(["a", "b", "c"], ["a", "b"])], ["text1", "text2"])
     >>> remover2 = StopWordsRemover(stopWords=["b"])
     >>> remover2.setInputCols(["text1", "text2"]).setOutputCols(["words1", "words2"])
@@ -3816,8 +3979,6 @@ class StopWordsRemover(JavaTransformer, HasInputCol, HasOutputCol, HasInputCols,
     |[a, b, c]|[a, b]|[a, c]|   [a]|
     +---------+------+------+------+
     ...
-
-    .. versionadded:: 1.6.0
     """
 
     stopWords = Param(Params._dummy(), "stopWords", "The words to be filtered out",
@@ -3828,10 +3989,10 @@ class StopWordsRemover(JavaTransformer, HasInputCol, HasOutputCol, HasInputCols,
                    "is true", typeConverter=TypeConverters.toString)
 
     @keyword_only
-    def __init__(self, inputCol=None, outputCol=None, stopWords=None, caseSensitive=False,
+    def __init__(self, *, inputCol=None, outputCol=None, stopWords=None, caseSensitive=False,
                  locale=None, inputCols=None, outputCols=None):
         """
-        __init__(self, inputCol=None, outputCol=None, stopWords=None, caseSensitive=false, \
+        __init__(self, \\*, inputCol=None, outputCol=None, stopWords=None, caseSensitive=false, \
                  locale=None, inputCols=None, outputCols=None)
         """
         super(StopWordsRemover, self).__init__()
@@ -3844,10 +4005,10 @@ def __init__(self, inputCol=None, outputCol=None, stopWords=None, caseSensitive=
 
     @keyword_only
     @since("1.6.0")
-    def setParams(self, inputCol=None, outputCol=None, stopWords=None, caseSensitive=False,
+    def setParams(self, *, inputCol=None, outputCol=None, stopWords=None, caseSensitive=False,
                   locale=None, inputCols=None, outputCols=None):
         """
-        setParams(self, inputCol=None, outputCol=None, stopWords=None, caseSensitive=false, \
+        setParams(self, \\*, inputCol=None, outputCol=None, stopWords=None, caseSensitive=false, \
                   locale=None, inputCols=None, outputCols=None)
         Sets params for this StopWordRemover.
         """
@@ -3935,26 +4096,29 @@ def loadDefaultStopWords(language):
 
 
 @inherit_doc
-@ignore_unicode_prefix
 class Tokenizer(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWritable):
     """
     A tokenizer that converts the input string to lowercase and then
     splits it by white spaces.
 
+    .. versionadded:: 1.3.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([("a b c",)], ["text"])
     >>> tokenizer = Tokenizer(outputCol="words")
     >>> tokenizer.setInputCol("text")
     Tokenizer...
     >>> tokenizer.transform(df).head()
-    Row(text=u'a b c', words=[u'a', u'b', u'c'])
+    Row(text='a b c', words=['a', 'b', 'c'])
     >>> # Change a parameter.
     >>> tokenizer.setParams(outputCol="tokens").transform(df).head()
-    Row(text=u'a b c', tokens=[u'a', u'b', u'c'])
+    Row(text='a b c', tokens=['a', 'b', 'c'])
     >>> # Temporarily modify a parameter.
     >>> tokenizer.transform(df, {tokenizer.outputCol: "words"}).head()
-    Row(text=u'a b c', words=[u'a', u'b', u'c'])
+    Row(text='a b c', words=['a', 'b', 'c'])
     >>> tokenizer.transform(df).head()
-    Row(text=u'a b c', tokens=[u'a', u'b', u'c'])
+    Row(text='a b c', tokens=['a', 'b', 'c'])
     >>> # Must use keyword arguments to specify params.
     >>> tokenizer.setParams("text")
     Traceback (most recent call last):
@@ -3965,14 +4129,12 @@ class Tokenizer(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, Java
     >>> loadedTokenizer = Tokenizer.load(tokenizerPath)
     >>> loadedTokenizer.transform(df).head().tokens == tokenizer.transform(df).head().tokens
     True
-
-    .. versionadded:: 1.3.0
     """
 
     @keyword_only
-    def __init__(self, inputCol=None, outputCol=None):
+    def __init__(self, *, inputCol=None, outputCol=None):
         """
-        __init__(self, inputCol=None, outputCol=None)
+        __init__(self, \\*, inputCol=None, outputCol=None)
         """
         super(Tokenizer, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Tokenizer", self.uid)
@@ -3981,9 +4143,9 @@ def __init__(self, inputCol=None, outputCol=None):
 
     @keyword_only
     @since("1.3.0")
-    def setParams(self, inputCol=None, outputCol=None):
+    def setParams(self, *, inputCol=None, outputCol=None):
         """
-        setParams(self, inputCol=None, outputCol=None)
+        setParams(self, \\*, inputCol=None, outputCol=None)
         Sets params for this Tokenizer.
         """
         kwargs = self._input_kwargs
@@ -4008,6 +4170,10 @@ class VectorAssembler(JavaTransformer, HasInputCols, HasOutputCol, HasHandleInva
     """
     A feature transformer that merges multiple columns into a vector column.
 
+    .. versionadded:: 1.4.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([(1, 0, 3)], ["a", "b", "c"])
     >>> vecAssembler = VectorAssembler(outputCol="features")
     >>> vecAssembler.setInputCols(["a", "b", "c"])
@@ -4044,8 +4210,6 @@ class VectorAssembler(JavaTransformer, HasInputCols, HasOutputCol, HasHandleInva
     |5.0|6.0|7.0|[5.0,6.0,7.0]|
     +---+---+---+-------------+
     ...
-
-    .. versionadded:: 1.4.0
     """
 
     handleInvalid = Param(Params._dummy(), "handleInvalid", "How to handle invalid data (NULL " +
@@ -4059,9 +4223,9 @@ class VectorAssembler(JavaTransformer, HasInputCols, HasOutputCol, HasHandleInva
                           typeConverter=TypeConverters.toString)
 
     @keyword_only
-    def __init__(self, inputCols=None, outputCol=None, handleInvalid="error"):
+    def __init__(self, *, inputCols=None, outputCol=None, handleInvalid="error"):
         """
-        __init__(self, inputCols=None, outputCol=None, handleInvalid="error")
+        __init__(self, \\*, inputCols=None, outputCol=None, handleInvalid="error")
         """
         super(VectorAssembler, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.VectorAssembler", self.uid)
@@ -4071,9 +4235,9 @@ def __init__(self, inputCols=None, outputCol=None, handleInvalid="error"):
 
     @keyword_only
     @since("1.4.0")
-    def setParams(self, inputCols=None, outputCol=None, handleInvalid="error"):
+    def setParams(self, *, inputCols=None, outputCol=None, handleInvalid="error"):
         """
-        setParams(self, inputCols=None, outputCol=None, handleInvalid="error")
+        setParams(self, \\*, inputCols=None, outputCol=None, handleInvalid="error")
         Sets params for this VectorAssembler.
         """
         kwargs = self._input_kwargs
@@ -4117,6 +4281,10 @@ class _VectorIndexerParams(HasInputCol, HasOutputCol, HasHandleInvalid):
                           "of categories of the feature).",
                           typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_VectorIndexerParams, self).__init__(*args)
+        self._setDefault(maxCategories=20, handleInvalid="error")
+
     @since("1.4.0")
     def getMaxCategories(self):
         """
@@ -4156,12 +4324,16 @@ class VectorIndexer(JavaEstimator, _VectorIndexerParams, JavaMLReadable, JavaMLW
         index 0. This maintains vector sparsity.
       - More stability may be added in the future.
 
-     TODO: Future extensions: The following functionality is planned for the future:
+    TODO: Future extensions: The following functionality is planned for the future:
       - Preserve metadata in transform; if a feature's metadata is already present,
         do not recompute.
       - Specify certain features to not index, either via a parameter or via existing metadata.
       - Add warning if a categorical feature has only 1 category.
 
+    .. versionadded:: 1.4.0
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> df = spark.createDataFrame([(Vectors.dense([-1.0, 0.0]),),
     ...     (Vectors.dense([0.0, 1.0]),), (Vectors.dense([0.0, 2.0]),)], ["a"])
@@ -4197,6 +4369,8 @@ class VectorIndexer(JavaEstimator, _VectorIndexerParams, JavaMLReadable, JavaMLW
     True
     >>> loadedModel.categoryMaps == model.categoryMaps
     True
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
     >>> dfWithInvalid = spark.createDataFrame([(Vectors.dense([3.0, 1.0]),)], ["a"])
     >>> indexer.getHandleInvalid()
     'error'
@@ -4206,26 +4380,23 @@ class VectorIndexer(JavaEstimator, _VectorIndexerParams, JavaMLReadable, JavaMLW
     >>> model4 = indexer.setParams(handleInvalid="keep", outputCol="indexed").fit(df)
     >>> model4.transform(dfWithInvalid).head().indexed
     DenseVector([2.0, 1.0])
-
-    .. versionadded:: 1.4.0
     """
 
     @keyword_only
-    def __init__(self, maxCategories=20, inputCol=None, outputCol=None, handleInvalid="error"):
+    def __init__(self, *, maxCategories=20, inputCol=None, outputCol=None, handleInvalid="error"):
         """
-        __init__(self, maxCategories=20, inputCol=None, outputCol=None, handleInvalid="error")
+        __init__(self, \\*, maxCategories=20, inputCol=None, outputCol=None, handleInvalid="error")
         """
         super(VectorIndexer, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.VectorIndexer", self.uid)
-        self._setDefault(maxCategories=20, handleInvalid="error")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("1.4.0")
-    def setParams(self, maxCategories=20, inputCol=None, outputCol=None, handleInvalid="error"):
+    def setParams(self, *, maxCategories=20, inputCol=None, outputCol=None, handleInvalid="error"):
         """
-        setParams(self, maxCategories=20, inputCol=None, outputCol=None, handleInvalid="error")
+        setParams(self, \\*, maxCategories=20, inputCol=None, outputCol=None, handleInvalid="error")
         Sets params for this VectorIndexer.
         """
         kwargs = self._input_kwargs
@@ -4323,6 +4494,10 @@ class VectorSlicer(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, J
     The output vector will order features with the selected indices first (in the order given),
     followed by the selected names (in the order given).
 
+    .. versionadded:: 1.6.0
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> df = spark.createDataFrame([
     ...     (Vectors.dense([-2.0, 2.3, 0.0, 0.0, 1.0]),),
@@ -4340,8 +4515,8 @@ class VectorSlicer(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, J
     True
     >>> loadedVs.getNames() == vs.getNames()
     True
-
-    .. versionadded:: 1.6.0
+    >>> loadedVs.transform(df).take(1) == vs.transform(df).take(1)
+    True
     """
 
     indices = Param(Params._dummy(), "indices", "An array of indices to select features from " +
@@ -4353,9 +4528,9 @@ class VectorSlicer(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, J
                   "indices.", typeConverter=TypeConverters.toListString)
 
     @keyword_only
-    def __init__(self, inputCol=None, outputCol=None, indices=None, names=None):
+    def __init__(self, *, inputCol=None, outputCol=None, indices=None, names=None):
         """
-        __init__(self, inputCol=None, outputCol=None, indices=None, names=None)
+        __init__(self, \\*, inputCol=None, outputCol=None, indices=None, names=None)
         """
         super(VectorSlicer, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.VectorSlicer", self.uid)
@@ -4365,9 +4540,9 @@ def __init__(self, inputCol=None, outputCol=None, indices=None, names=None):
 
     @keyword_only
     @since("1.6.0")
-    def setParams(self, inputCol=None, outputCol=None, indices=None, names=None):
+    def setParams(self, *, inputCol=None, outputCol=None, indices=None, names=None):
         """
-        setParams(self, inputCol=None, outputCol=None, indices=None, names=None):
+        setParams(self, \\*, inputCol=None, outputCol=None, indices=None, names=None):
         Sets params for this VectorSlicer.
         """
         kwargs = self._input_kwargs
@@ -4439,6 +4614,11 @@ class _Word2VecParams(HasStepSize, HasMaxIter, HasSeed, HasInputCol, HasOutputCo
                               "be divided into chunks up to the size.",
                               typeConverter=TypeConverters.toInt)
 
+    def __init__(self, *args):
+        super(_Word2VecParams, self).__init__(*args)
+        self._setDefault(vectorSize=100, minCount=5, numPartitions=1, stepSize=0.025, maxIter=1,
+                         windowSize=5, maxSentenceLength=1000)
+
     @since("1.4.0")
     def getVectorSize(self):
         """
@@ -4476,12 +4656,15 @@ def getMaxSentenceLength(self):
 
 
 @inherit_doc
-@ignore_unicode_prefix
 class Word2Vec(JavaEstimator, _Word2VecParams, JavaMLReadable, JavaMLWritable):
     """
     Word2Vec trains a model of `Map(String, Vector)`, i.e. transforms a word into a code for further
     natural language processing or machine learning process.
 
+    .. versionadded:: 1.4.0
+
+    Examples
+    --------
     >>> sent = ("a b " * 100 + "a c " * 10).split(" ")
     >>> doc = spark.createDataFrame([(sent,), (sent,)], ["sentence"])
     >>> word2Vec = Word2Vec(vectorSize=5, seed=42, inputCol="sentence", outputCol="model")
@@ -4499,13 +4682,13 @@ class Word2Vec(JavaEstimator, _Word2VecParams, JavaMLReadable, JavaMLWritable):
     +----+--------------------+
     |word|              vector|
     +----+--------------------+
-    |   a|[0.09511678665876...|
-    |   b|[-1.2028766870498...|
-    |   c|[0.30153277516365...|
+    |   a|[0.0951...
+    |   b|[-1.202...
+    |   c|[0.3015...
     +----+--------------------+
     ...
     >>> model.findSynonymsArray("a", 2)
-    [(u'b', 0.015859870240092278), (u'c', -0.5680795907974243)]
+    [('b', 0.015859...), ('c', -0.568079...)]
     >>> from pyspark.sql.functions import format_number as fmt
     >>> model.findSynonyms("a", 2).select("word", fmt("similarity", 5).alias("similarity")).show()
     +----+----------+
@@ -4533,31 +4716,33 @@ class Word2Vec(JavaEstimator, _Word2VecParams, JavaMLReadable, JavaMLWritable):
     True
     >>> loadedModel.getVectors().first().vector == model.getVectors().first().vector
     True
-
-    .. versionadded:: 1.4.0
+    >>> loadedModel.transform(doc).take(1) == model.transform(doc).take(1)
+    True
     """
 
     @keyword_only
-    def __init__(self, vectorSize=100, minCount=5, numPartitions=1, stepSize=0.025, maxIter=1,
-                 seed=None, inputCol=None, outputCol=None, windowSize=5, maxSentenceLength=1000):
+    def __init__(self, *, vectorSize=100, minCount=5, numPartitions=1, stepSize=0.025,
+                 maxIter=1, seed=None, inputCol=None, outputCol=None, windowSize=5,
+                 maxSentenceLength=1000):
         """
-        __init__(self, vectorSize=100, minCount=5, numPartitions=1, stepSize=0.025, maxIter=1, \
-                 seed=None, inputCol=None, outputCol=None, windowSize=5, maxSentenceLength=1000)
+        __init__(self, \\*, vectorSize=100, minCount=5, numPartitions=1, stepSize=0.025, \
+                 maxIter=1, seed=None, inputCol=None, outputCol=None, windowSize=5, \
+                 maxSentenceLength=1000)
         """
         super(Word2Vec, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Word2Vec", self.uid)
-        self._setDefault(vectorSize=100, minCount=5, numPartitions=1, stepSize=0.025, maxIter=1,
-                         windowSize=5, maxSentenceLength=1000)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("1.4.0")
-    def setParams(self, vectorSize=100, minCount=5, numPartitions=1, stepSize=0.025, maxIter=1,
-                  seed=None, inputCol=None, outputCol=None, windowSize=5, maxSentenceLength=1000):
+    def setParams(self, *, vectorSize=100, minCount=5, numPartitions=1, stepSize=0.025,
+                  maxIter=1, seed=None, inputCol=None, outputCol=None, windowSize=5,
+                  maxSentenceLength=1000):
         """
-        setParams(self, minCount=5, numPartitions=1, stepSize=0.025, maxIter=1, seed=None, \
-                 inputCol=None, outputCol=None, windowSize=5, maxSentenceLength=1000)
+        setParams(self, \\*, minCount=5, numPartitions=1, stepSize=0.025, maxIter=1, \
+                  seed=None, inputCol=None, outputCol=None, windowSize=5, \
+                  maxSentenceLength=1000)
         Sets params for this Word2Vec.
         """
         kwargs = self._input_kwargs
@@ -4668,7 +4853,7 @@ def findSynonyms(self, word, num):
         Returns a dataframe with two fields word and similarity (which
         gives the cosine similarity).
         """
-        if not isinstance(word, basestring):
+        if not isinstance(word, str):
             word = _convert_to_vector(word)
         return self._call_java("findSynonyms", word, num)
 
@@ -4680,7 +4865,7 @@ def findSynonymsArray(self, word, num):
         Returns an array with two fields word and similarity (which
         gives the cosine similarity).
         """
-        if not isinstance(word, basestring):
+        if not isinstance(word, str):
             word = _convert_to_vector(word)
         tuples = self._java_obj.findSynonymsArray(word, num)
         return list(map(lambda st: (st._1(), st._2()), list(tuples)))
@@ -4710,6 +4895,10 @@ class PCA(JavaEstimator, _PCAParams, JavaMLReadable, JavaMLWritable):
     PCA trains a model to project vectors to a lower dimensional space of the
     top :py:attr:`k` principal components.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> data = [(Vectors.sparse(5, [(1, 1.0), (3, 7.0)]),),
     ...     (Vectors.dense([2.0, 0.0, 3.0, 4.0, 5.0]),),
@@ -4739,14 +4928,14 @@ class PCA(JavaEstimator, _PCAParams, JavaMLReadable, JavaMLWritable):
     True
     >>> loadedModel.explainedVariance == model.explainedVariance
     True
-
-    .. versionadded:: 1.5.0
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
     """
 
     @keyword_only
-    def __init__(self, k=None, inputCol=None, outputCol=None):
+    def __init__(self, *, k=None, inputCol=None, outputCol=None):
         """
-        __init__(self, k=None, inputCol=None, outputCol=None)
+        __init__(self, \\*, k=None, inputCol=None, outputCol=None)
         """
         super(PCA, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.PCA", self.uid)
@@ -4755,9 +4944,9 @@ def __init__(self, k=None, inputCol=None, outputCol=None):
 
     @keyword_only
     @since("1.5.0")
-    def setParams(self, k=None, inputCol=None, outputCol=None):
+    def setParams(self, *, k=None, inputCol=None, outputCol=None):
         """
-        setParams(self, k=None, inputCol=None, outputCol=None)
+        setParams(self, \\*, k=None, inputCol=None, outputCol=None)
         Set params for this PCA.
         """
         kwargs = self._input_kwargs
@@ -4855,6 +5044,11 @@ class _RFormulaParams(HasFeaturesCol, HasLabelCol, HasHandleInvalid):
                           "additional bucket, at index numLabels).",
                           typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_RFormulaParams, self).__init__(*args)
+        self._setDefault(forceIndexLabel=False, stringIndexerOrderType="frequencyDesc",
+                         handleInvalid="error")
+
     @since("1.5.0")
     def getFormula(self):
         """
@@ -4883,9 +5077,16 @@ class RFormula(JavaEstimator, _RFormulaParams, JavaMLReadable, JavaMLWritable):
     Implements the transforms required for fitting a dataset against an
     R model formula. Currently we support a limited subset of the R
     operators, including '~', '.', ':', '+', '-', '*', and '^'.
+
+    .. versionadded:: 1.5.0
+
+    Notes
+    -----
     Also see the `R formula docs
     <http://stat.ethz.ch/R-manual/R-patched/library/stats/html/formula.html>`_.
 
+    Examples
+    --------
     >>> df = spark.createDataFrame([
     ...     (1.0, 1.0, "a"),
     ...     (0.0, 2.0, "b"),
@@ -4942,33 +5143,29 @@ class RFormula(JavaEstimator, _RFormulaParams, JavaMLReadable, JavaMLWritable):
     ...
     >>> str(loadedModel)
     'RFormulaModel(ResolvedRFormula(label=y, terms=[x,s], hasIntercept=true)) (uid=...)'
-
-    .. versionadded:: 1.5.0
     """
 
     @keyword_only
-    def __init__(self, formula=None, featuresCol="features", labelCol="label",
+    def __init__(self, *, formula=None, featuresCol="features", labelCol="label",
                  forceIndexLabel=False, stringIndexerOrderType="frequencyDesc",
                  handleInvalid="error"):
         """
-        __init__(self, formula=None, featuresCol="features", labelCol="label", \
+        __init__(self, \\*, formula=None, featuresCol="features", labelCol="label", \
                  forceIndexLabel=False, stringIndexerOrderType="frequencyDesc", \
                  handleInvalid="error")
         """
         super(RFormula, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.RFormula", self.uid)
-        self._setDefault(forceIndexLabel=False, stringIndexerOrderType="frequencyDesc",
-                         handleInvalid="error")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("1.5.0")
-    def setParams(self, formula=None, featuresCol="features", labelCol="label",
+    def setParams(self, *, formula=None, featuresCol="features", labelCol="label",
                   forceIndexLabel=False, stringIndexerOrderType="frequencyDesc",
                   handleInvalid="error"):
         """
-        setParams(self, formula=None, featuresCol="features", labelCol="label", \
+        setParams(self, \\*, formula=None, featuresCol="features", labelCol="label", \
                   forceIndexLabel=False, stringIndexerOrderType="frequencyDesc", \
                   handleInvalid="error")
         Sets params for RFormula.
@@ -5067,6 +5264,11 @@ class _SelectorParams(HasFeaturesCol, HasOutputCol, HasLabelCol):
     fwe = Param(Params._dummy(), "fwe", "The upper bound of the expected family-wise error rate.",
                 typeConverter=TypeConverters.toFloat)
 
+    def __init__(self, *args):
+        super(_SelectorParams, self).__init__(*args)
+        self._setDefault(numTopFeatures=50, selectorType="numTopFeatures", percentile=0.1,
+                         fpr=0.05, fdr=0.05, fwe=0.05)
+
     @since("2.1.0")
     def getSelectorType(self):
         """
@@ -5210,15 +5412,14 @@ def selectedFeatures(self):
 
 
 @inherit_doc
-class ANOVASelector(_Selector, JavaMLReadable, JavaMLWritable):
+class ChiSqSelector(_Selector, JavaMLReadable, JavaMLWritable):
     """
-    ANOVA F-value Classification selector, which selects continuous features to use for predicting
-    a categorical label.
+    Chi-Squared feature selection, which selects categorical features to use for predicting a
+    categorical label.
     The selector supports different selection methods: `numTopFeatures`, `percentile`, `fpr`,
     `fdr`, `fwe`.
 
-     * `numTopFeatures` chooses a fixed number of top features according to a F value
-        classification test.
+     * `numTopFeatures` chooses a fixed number of top features according to a chi-squared test.
 
      * `percentile` is similar but chooses a fraction of all features
        instead of a fixed number.
@@ -5236,281 +5437,80 @@ class ANOVASelector(_Selector, JavaMLReadable, JavaMLWritable):
     By default, the selection method is `numTopFeatures`, with the default number of top features
     set to 50.
 
+    .. deprecated:: 3.1.0
+        Use UnivariateFeatureSelector
 
+    .. versionadded:: 2.0.0
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> df = spark.createDataFrame(
-    ...    [(Vectors.dense([1.7, 4.4, 7.6, 5.8, 9.6, 2.3]), 3.0),
-    ...     (Vectors.dense([8.8, 7.3, 5.7, 7.3, 2.2, 4.1]), 2.0),
-    ...     (Vectors.dense([1.2, 9.5, 2.5, 3.1, 8.7, 2.5]), 1.0),
-    ...     (Vectors.dense([3.7, 9.2, 6.1, 4.1, 7.5, 3.8]), 2.0),
-    ...     (Vectors.dense([8.9, 5.2, 7.8, 8.3, 5.2, 3.0]), 4.0),
-    ...     (Vectors.dense([7.9, 8.5, 9.2, 4.0, 9.4, 2.1]), 4.0)],
+    ...    [(Vectors.dense([0.0, 0.0, 18.0, 1.0]), 1.0),
+    ...     (Vectors.dense([0.0, 1.0, 12.0, 0.0]), 0.0),
+    ...     (Vectors.dense([1.0, 0.0, 15.0, 0.1]), 0.0)],
     ...    ["features", "label"])
-    >>> selector = ANOVASelector(numTopFeatures=1, outputCol="selectedFeatures")
+    >>> selector = ChiSqSelector(numTopFeatures=1, outputCol="selectedFeatures")
     >>> model = selector.fit(df)
     >>> model.getFeaturesCol()
     'features'
     >>> model.setFeaturesCol("features")
-    ANOVASelectorModel...
+    ChiSqSelectorModel...
     >>> model.transform(df).head().selectedFeatures
-    DenseVector([7.6])
+    DenseVector([18.0])
     >>> model.selectedFeatures
     [2]
-    >>> anovaSelectorPath = temp_path + "/anova-selector"
-    >>> selector.save(anovaSelectorPath)
-    >>> loadedSelector = ANOVASelector.load(anovaSelectorPath)
+    >>> chiSqSelectorPath = temp_path + "/chi-sq-selector"
+    >>> selector.save(chiSqSelectorPath)
+    >>> loadedSelector = ChiSqSelector.load(chiSqSelectorPath)
     >>> loadedSelector.getNumTopFeatures() == selector.getNumTopFeatures()
     True
-    >>> modelPath = temp_path + "/anova-selector-model"
+    >>> modelPath = temp_path + "/chi-sq-selector-model"
     >>> model.save(modelPath)
-    >>> loadedModel = ANOVASelectorModel.load(modelPath)
+    >>> loadedModel = ChiSqSelectorModel.load(modelPath)
     >>> loadedModel.selectedFeatures == model.selectedFeatures
     True
-
-    .. versionadded:: 3.1.0
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
     """
 
     @keyword_only
-    def __init__(self, numTopFeatures=50, featuresCol="features", outputCol=None,
+    def __init__(self, *, numTopFeatures=50, featuresCol="features", outputCol=None,
                  labelCol="label", selectorType="numTopFeatures", percentile=0.1, fpr=0.05,
                  fdr=0.05, fwe=0.05):
         """
-        __init__(self, numTopFeatures=50, featuresCol="features", outputCol=None, \
+        __init__(self, \\*, numTopFeatures=50, featuresCol="features", outputCol=None, \
                  labelCol="label", selectorType="numTopFeatures", percentile=0.1, fpr=0.05, \
                  fdr=0.05, fwe=0.05)
         """
-        super(ANOVASelector, self).__init__()
-        self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.ANOVASelector", self.uid)
-        self._setDefault(numTopFeatures=50, selectorType="numTopFeatures", percentile=0.1,
-                         fpr=0.05, fdr=0.05, fwe=0.05)
+        super(ChiSqSelector, self).__init__()
+        self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.ChiSqSelector", self.uid)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
-    @since("3.1.0")
-    def setParams(self, numTopFeatures=50, featuresCol="features", outputCol=None,
+    @since("2.0.0")
+    def setParams(self, *, numTopFeatures=50, featuresCol="features", outputCol=None,
                   labelCol="labels", selectorType="numTopFeatures", percentile=0.1, fpr=0.05,
                   fdr=0.05, fwe=0.05):
         """
-        setParams(self, numTopFeatures=50, featuresCol="features", outputCol=None, \
+        setParams(self, \\*, numTopFeatures=50, featuresCol="features", outputCol=None, \
                   labelCol="labels", selectorType="numTopFeatures", percentile=0.1, fpr=0.05, \
                   fdr=0.05, fwe=0.05)
-        Sets params for this ANOVASelector.
+        Sets params for this ChiSqSelector.
         """
         kwargs = self._input_kwargs
         return self._set(**kwargs)
 
     def _create_model(self, java_model):
-        return ANOVASelectorModel(java_model)
+        return ChiSqSelectorModel(java_model)
 
 
-class ANOVASelectorModel(_SelectorModel, JavaMLReadable, JavaMLWritable):
+class ChiSqSelectorModel(_SelectorModel, JavaMLReadable, JavaMLWritable):
     """
-    Model fitted by :py:class:`ANOVASelector`.
+    Model fitted by :py:class:`ChiSqSelector`.
 
-    .. versionadded:: 3.1.0
-    """
-
-
-@inherit_doc
-class ChiSqSelector(_Selector, JavaMLReadable, JavaMLWritable):
-    """
-    Chi-Squared feature selection, which selects categorical features to use for predicting a
-    categorical label.
-    The selector supports different selection methods: `numTopFeatures`, `percentile`, `fpr`,
-    `fdr`, `fwe`.
-
-     * `numTopFeatures` chooses a fixed number of top features according to a chi-squared test.
-
-     * `percentile` is similar but chooses a fraction of all features
-       instead of a fixed number.
-
-     * `fpr` chooses all features whose p-values are below a threshold,
-       thus controlling the false positive rate of selection.
-
-     * `fdr` uses the `Benjamini-Hochberg procedure <https://en.wikipedia.org/wiki/
-       False_discovery_rate#Benjamini.E2.80.93Hochberg_procedure>`_
-       to choose all features whose false discovery rate is below a threshold.
-
-     * `fwe` chooses all features whose p-values are below a threshold. The threshold is scaled by
-       1/numFeatures, thus controlling the family-wise error rate of selection.
-
-    By default, the selection method is `numTopFeatures`, with the default number of top features
-    set to 50.
-
-
-    >>> from pyspark.ml.linalg import Vectors
-    >>> df = spark.createDataFrame(
-    ...    [(Vectors.dense([0.0, 0.0, 18.0, 1.0]), 1.0),
-    ...     (Vectors.dense([0.0, 1.0, 12.0, 0.0]), 0.0),
-    ...     (Vectors.dense([1.0, 0.0, 15.0, 0.1]), 0.0)],
-    ...    ["features", "label"])
-    >>> selector = ChiSqSelector(numTopFeatures=1, outputCol="selectedFeatures")
-    >>> model = selector.fit(df)
-    >>> model.getFeaturesCol()
-    'features'
-    >>> model.setFeaturesCol("features")
-    ChiSqSelectorModel...
-    >>> model.transform(df).head().selectedFeatures
-    DenseVector([18.0])
-    >>> model.selectedFeatures
-    [2]
-    >>> chiSqSelectorPath = temp_path + "/chi-sq-selector"
-    >>> selector.save(chiSqSelectorPath)
-    >>> loadedSelector = ChiSqSelector.load(chiSqSelectorPath)
-    >>> loadedSelector.getNumTopFeatures() == selector.getNumTopFeatures()
-    True
-    >>> modelPath = temp_path + "/chi-sq-selector-model"
-    >>> model.save(modelPath)
-    >>> loadedModel = ChiSqSelectorModel.load(modelPath)
-    >>> loadedModel.selectedFeatures == model.selectedFeatures
-    True
-
-    .. versionadded:: 2.0.0
-    """
-
-    @keyword_only
-    def __init__(self, numTopFeatures=50, featuresCol="features", outputCol=None,
-                 labelCol="label", selectorType="numTopFeatures", percentile=0.1, fpr=0.05,
-                 fdr=0.05, fwe=0.05):
-        """
-        __init__(self, numTopFeatures=50, featuresCol="features", outputCol=None, \
-                 labelCol="label", selectorType="numTopFeatures", percentile=0.1, fpr=0.05, \
-                 fdr=0.05, fwe=0.05)
-        """
-        super(ChiSqSelector, self).__init__()
-        self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.ChiSqSelector", self.uid)
-        self._setDefault(numTopFeatures=50, selectorType="numTopFeatures", percentile=0.1,
-                         fpr=0.05, fdr=0.05, fwe=0.05)
-        kwargs = self._input_kwargs
-        self.setParams(**kwargs)
-
-    @keyword_only
-    @since("2.0.0")
-    def setParams(self, numTopFeatures=50, featuresCol="features", outputCol=None,
-                  labelCol="labels", selectorType="numTopFeatures", percentile=0.1, fpr=0.05,
-                  fdr=0.05, fwe=0.05):
-        """
-        setParams(self, numTopFeatures=50, featuresCol="features", outputCol=None, \
-                  labelCol="labels", selectorType="numTopFeatures", percentile=0.1, fpr=0.05, \
-                  fdr=0.05, fwe=0.05)
-        Sets params for this ChiSqSelector.
-        """
-        kwargs = self._input_kwargs
-        return self._set(**kwargs)
-
-    def _create_model(self, java_model):
-        return ChiSqSelectorModel(java_model)
-
-
-class ChiSqSelectorModel(_SelectorModel, JavaMLReadable, JavaMLWritable):
-    """
-    Model fitted by :py:class:`ChiSqSelector`.
-
-    .. versionadded:: 2.0.0
-    """
-
-
-@inherit_doc
-class FValueSelector(_Selector, JavaMLReadable, JavaMLWritable):
-    """
-    F Value Regression feature selector, which selects continuous features to use for predicting a
-    continuous label.
-    The selector supports different selection methods: `numTopFeatures`, `percentile`, `fpr`,
-    `fdr`, `fwe`.
-
-     * `numTopFeatures` chooses a fixed number of top features according to a F value
-        regression test.
-
-     * `percentile` is similar but chooses a fraction of all features
-       instead of a fixed number.
-
-     * `fpr` chooses all features whose p-values are below a threshold,
-       thus controlling the false positive rate of selection.
-
-     * `fdr` uses the `Benjamini-Hochberg procedure <https://en.wikipedia.org/wiki/
-       False_discovery_rate#Benjamini.E2.80.93Hochberg_procedure>`_
-       to choose all features whose false discovery rate is below a threshold.
-
-     * `fwe` chooses all features whose p-values are below a threshold. The threshold is scaled by
-       1/numFeatures, thus controlling the family-wise error rate of selection.
-
-    By default, the selection method is `numTopFeatures`, with the default number of top features
-    set to 50.
-
-
-    >>> from pyspark.ml.linalg import Vectors
-    >>> df = spark.createDataFrame(
-    ...    [(Vectors.dense([6.0, 7.0, 0.0, 7.0, 6.0, 0.0]), 4.6),
-    ...     (Vectors.dense([0.0, 9.0, 6.0, 0.0, 5.0, 9.0]), 6.6),
-    ...     (Vectors.dense([0.0, 9.0, 3.0, 0.0, 5.0, 5.0]), 5.1),
-    ...     (Vectors.dense([0.0, 9.0, 8.0, 5.0, 6.0, 4.0]), 7.6),
-    ...     (Vectors.dense([8.0, 9.0, 6.0, 5.0, 4.0, 4.0]), 9.0),
-    ...     (Vectors.dense([8.0, 9.0, 6.0, 4.0, 0.0, 0.0]), 9.0)],
-    ...    ["features", "label"])
-    >>> selector = FValueSelector(numTopFeatures=1, outputCol="selectedFeatures")
-    >>> model = selector.fit(df)
-    >>> model.getFeaturesCol()
-    'features'
-    >>> model.setFeaturesCol("features")
-    FValueSelectorModel...
-    >>> model.transform(df).head().selectedFeatures
-    DenseVector([0.0])
-    >>> model.selectedFeatures
-    [2]
-    >>> fvalueSelectorPath = temp_path + "/fvalue-selector"
-    >>> selector.save(fvalueSelectorPath)
-    >>> loadedSelector = FValueSelector.load(fvalueSelectorPath)
-    >>> loadedSelector.getNumTopFeatures() == selector.getNumTopFeatures()
-    True
-    >>> modelPath = temp_path + "/fvalue-selector-model"
-    >>> model.save(modelPath)
-    >>> loadedModel = FValueSelectorModel.load(modelPath)
-    >>> loadedModel.selectedFeatures == model.selectedFeatures
-    True
-
-    .. versionadded:: 3.1.0
-    """
-
-    @keyword_only
-    def __init__(self, numTopFeatures=50, featuresCol="features", outputCol=None,
-                 labelCol="label", selectorType="numTopFeatures", percentile=0.1, fpr=0.05,
-                 fdr=0.05, fwe=0.05):
-        """
-        __init__(self, numTopFeatures=50, featuresCol="features", outputCol=None, \
-                 labelCol="label", selectorType="numTopFeatures", percentile=0.1, fpr=0.05, \
-                 fdr=0.05, fwe=0.05)
-        """
-        super(FValueSelector, self).__init__()
-        self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.FValueSelector", self.uid)
-        self._setDefault(numTopFeatures=50, selectorType="numTopFeatures", percentile=0.1,
-                         fpr=0.05, fdr=0.05, fwe=0.05)
-        kwargs = self._input_kwargs
-        self.setParams(**kwargs)
-
-    @keyword_only
-    @since("3.1.0")
-    def setParams(self, numTopFeatures=50, featuresCol="features", outputCol=None,
-                  labelCol="labels", selectorType="numTopFeatures", percentile=0.1, fpr=0.05,
-                  fdr=0.05, fwe=0.05):
-        """
-        setParams(self, numTopFeatures=50, featuresCol="features", outputCol=None, \
-                  labelCol="labels", selectorType="numTopFeatures", percentile=0.1, fpr=0.05, \
-                  fdr=0.05, fwe=0.05)
-        Sets params for this FValueSelector.
-        """
-        kwargs = self._input_kwargs
-        return self._set(**kwargs)
-
-    def _create_model(self, java_model):
-        return FValueSelectorModel(java_model)
-
-
-class FValueSelectorModel(_SelectorModel, JavaMLReadable, JavaMLWritable):
-    """
-    Model fitted by :py:class:`FValueSelector`.
-
-    .. versionadded:: 3.1.0
+    .. versionadded:: 2.0.0
     """
 
 
@@ -5522,9 +5522,14 @@ class VectorSizeHint(JavaTransformer, HasInputCol, HasHandleInvalid, JavaMLReada
     VectorAssembler needs size information for its input columns and cannot be used on streaming
     dataframes without this metadata.
 
-    .. note:: VectorSizeHint modifies `inputCol` to include size metadata and does not have an
-        outputCol.
+    .. versionadded:: 2.3.0
+
+    Notes
+    -----
+    VectorSizeHint modifies `inputCol` to include size metadata and does not have an outputCol.
 
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> from pyspark.ml import Pipeline, PipelineModel
     >>> data = [(Vectors.dense([1., 2., 3.]), 4.)]
@@ -5544,8 +5549,6 @@ class VectorSizeHint(JavaTransformer, HasInputCol, HasHandleInvalid, JavaMLReada
     >>> expected = pipelineModel.transform(df).head().assembled
     >>> loaded == expected
     True
-
-    .. versionadded:: 2.3.0
     """
 
     size = Param(Params._dummy(), "size", "Size of vectors in column.",
@@ -5560,9 +5563,9 @@ class VectorSizeHint(JavaTransformer, HasInputCol, HasHandleInvalid, JavaMLReada
                           TypeConverters.toString)
 
     @keyword_only
-    def __init__(self, inputCol=None, size=None, handleInvalid="error"):
+    def __init__(self, *, inputCol=None, size=None, handleInvalid="error"):
         """
-        __init__(self, inputCol=None, size=None, handleInvalid="error")
+        __init__(self, \\*, inputCol=None, size=None, handleInvalid="error")
         """
         super(VectorSizeHint, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.VectorSizeHint", self.uid)
@@ -5571,9 +5574,9 @@ def __init__(self, inputCol=None, size=None, handleInvalid="error"):
 
     @keyword_only
     @since("2.3.0")
-    def setParams(self, inputCol=None, size=None, handleInvalid="error"):
+    def setParams(self, *, inputCol=None, size=None, handleInvalid="error"):
         """
-        setParams(self, inputCol=None, size=None, handleInvalid="error")
+        setParams(self, \\*, inputCol=None, size=None, handleInvalid="error")
         Sets params for this VectorSizeHint.
         """
         kwargs = self._input_kwargs
@@ -5605,7 +5608,7 @@ def setHandleInvalid(self, value):
 class _VarianceThresholdSelectorParams(HasFeaturesCol, HasOutputCol):
     """
     Params for :py:class:`VarianceThresholdSelector` and
-    :py:class:`VarianceThresholdSelectorrModel`.
+    :py:class:`VarianceThresholdSelectorModel`.
 
     .. versionadded:: 3.1.0
     """
@@ -5632,6 +5635,10 @@ class VarianceThresholdSelector(JavaEstimator, _VarianceThresholdSelectorParams,
     all features with non-zero variance, i.e. remove the features that have the
     same value in all samples.
 
+    .. versionadded:: 3.1.0
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> df = spark.createDataFrame(
     ...    [(Vectors.dense([6.0, 7.0, 0.0, 7.0, 6.0, 0.0]),),
@@ -5661,14 +5668,14 @@ class VarianceThresholdSelector(JavaEstimator, _VarianceThresholdSelectorParams,
     >>> loadedModel = VarianceThresholdSelectorModel.load(modelPath)
     >>> loadedModel.selectedFeatures == model.selectedFeatures
     True
-
-    .. versionadded:: 3.1.0
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
     """
 
     @keyword_only
-    def __init__(self, featuresCol="features", outputCol=None, varianceThreshold=0.0):
+    def __init__(self, *, featuresCol="features", outputCol=None, varianceThreshold=0.0):
         """
-        __init__(self, featuresCol="features", outputCol=None, varianceThreshold=0.0)
+        __init__(self, \\*, featuresCol="features", outputCol=None, varianceThreshold=0.0)
         """
         super(VarianceThresholdSelector, self).__init__()
         self._java_obj = self._new_java_obj(
@@ -5679,9 +5686,9 @@ def __init__(self, featuresCol="features", outputCol=None, varianceThreshold=0.0
 
     @keyword_only
     @since("3.1.0")
-    def setParams(self, featuresCol="features", outputCol=None, varianceThreshold=0.0):
+    def setParams(self, *, featuresCol="features", outputCol=None, varianceThreshold=0.0):
         """
-        setParams(self, featuresCol="features", outputCol=None, varianceThreshold=0.0)
+        setParams(self, \\*, featuresCol="features", outputCol=None, varianceThreshold=0.0)
         Sets params for this VarianceThresholdSelector.
         """
         kwargs = self._input_kwargs
@@ -5743,8 +5750,249 @@ def selectedFeatures(self):
         return self._call_java("selectedFeatures")
 
 
+class _UnivariateFeatureSelectorParams(HasFeaturesCol, HasOutputCol, HasLabelCol):
+    """
+    Params for :py:class:`UnivariateFeatureSelector` and
+    :py:class:`UnivariateFeatureSelectorModel`.
+
+    .. versionadded:: 3.1.0
+    """
+
+    featureType = Param(Params._dummy(), "featureType",
+                        "The feature type. " +
+                        "Supported options: categorical, continuous.",
+                        typeConverter=TypeConverters.toString)
+
+    labelType = Param(Params._dummy(), "labelType",
+                      "The label type. " +
+                      "Supported options: categorical, continuous.",
+                      typeConverter=TypeConverters.toString)
+
+    selectionMode = Param(Params._dummy(), "selectionMode",
+                          "The selection mode. " +
+                          "Supported options: numTopFeatures (default), percentile, fpr, " +
+                          "fdr, fwe.",
+                          typeConverter=TypeConverters.toString)
+
+    selectionThreshold = Param(Params._dummy(), "selectionThreshold", "The upper bound of the " +
+                               "features that selector will select.",
+                               typeConverter=TypeConverters.toFloat)
+
+    def __init__(self, *args):
+        super(_UnivariateFeatureSelectorParams, self).__init__(*args)
+        self._setDefault(selectionMode="numTopFeatures")
+
+    @since("3.1.1")
+    def getFeatureType(self):
+        """
+        Gets the value of featureType or its default value.
+        """
+        return self.getOrDefault(self.featureType)
+
+    @since("3.1.1")
+    def getLabelType(self):
+        """
+        Gets the value of labelType or its default value.
+        """
+        return self.getOrDefault(self.labelType)
+
+    @since("3.1.1")
+    def getSelectionMode(self):
+        """
+        Gets the value of selectionMode or its default value.
+        """
+        return self.getOrDefault(self.selectionMode)
+
+    @since("3.1.1")
+    def getSelectionThreshold(self):
+        """
+        Gets the value of selectionThreshold or its default value.
+        """
+        return self.getOrDefault(self.selectionThreshold)
+
+
+@inherit_doc
+class UnivariateFeatureSelector(JavaEstimator, _UnivariateFeatureSelectorParams, JavaMLReadable,
+                                JavaMLWritable):
+    """
+    UnivariateFeatureSelector
+    The user can set `featureType` and `labelType`, and Spark will pick the score function based on
+    the specified `featureType` and `labelType`.
+
+    The following combination of `featureType` and `labelType` are supported:
+
+    - `featureType` `categorical` and `labelType` `categorical`, Spark uses chi-squared,
+      i.e. chi2 in sklearn.
+    - `featureType` `continuous` and `labelType` `categorical`, Spark uses ANOVATest,
+      i.e. f_classif in sklearn.
+    - `featureType` `continuous` and `labelType` `continuous`, Spark uses F-value,
+      i.e. f_regression in sklearn.
+
+    The `UnivariateFeatureSelector` supports different selection modes: `numTopFeatures`,
+    `percentile`, `fpr`, `fdr`, `fwe`.
+
+    - `numTopFeatures` chooses a fixed number of top features according to a according to a
+      hypothesis.
+    - `percentile` is similar but chooses a fraction of all features
+      instead of a fixed number.
+    - `fpr` chooses all features whose p-values are below a threshold,
+      thus controlling the false positive rate of selection.
+    - `fdr` uses the `Benjamini-Hochberg procedure \
+      <https://en.wikipedia.org/wiki/False_discovery_rate#Benjamini.E2.80.93Hochberg_procedure>`_
+      to choose all features whose false discovery rate is below a threshold.
+    - `fwe` chooses all features whose p-values are below a threshold. The threshold is scaled by
+      1 / `numFeatures`, thus controlling the family-wise error rate of selection.
+
+    By default, the selection mode is `numTopFeatures`.
+
+    .. versionadded:: 3.1.1
+
+    Examples
+    --------
+    >>> from pyspark.ml.linalg import Vectors
+    >>> df = spark.createDataFrame(
+    ...    [(Vectors.dense([1.7, 4.4, 7.6, 5.8, 9.6, 2.3]), 3.0),
+    ...     (Vectors.dense([8.8, 7.3, 5.7, 7.3, 2.2, 4.1]), 2.0),
+    ...     (Vectors.dense([1.2, 9.5, 2.5, 3.1, 8.7, 2.5]), 1.0),
+    ...     (Vectors.dense([3.7, 9.2, 6.1, 4.1, 7.5, 3.8]), 2.0),
+    ...     (Vectors.dense([8.9, 5.2, 7.8, 8.3, 5.2, 3.0]), 4.0),
+    ...     (Vectors.dense([7.9, 8.5, 9.2, 4.0, 9.4, 2.1]), 4.0)],
+    ...    ["features", "label"])
+    >>> selector = UnivariateFeatureSelector(outputCol="selectedFeatures")
+    >>> selector.setFeatureType("continuous").setLabelType("categorical").setSelectionThreshold(1)
+    UnivariateFeatureSelector...
+    >>> model = selector.fit(df)
+    >>> model.getFeaturesCol()
+    'features'
+    >>> model.setFeaturesCol("features")
+    UnivariateFeatureSelectorModel...
+    >>> model.transform(df).head().selectedFeatures
+    DenseVector([7.6])
+    >>> model.selectedFeatures
+    [2]
+    >>> selectorPath = temp_path + "/selector"
+    >>> selector.save(selectorPath)
+    >>> loadedSelector = UnivariateFeatureSelector.load(selectorPath)
+    >>> loadedSelector.getSelectionThreshold() == selector.getSelectionThreshold()
+    True
+    >>> modelPath = temp_path + "/selector-model"
+    >>> model.save(modelPath)
+    >>> loadedModel = UnivariateFeatureSelectorModel.load(modelPath)
+    >>> loadedModel.selectedFeatures == model.selectedFeatures
+    True
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
+    """
+
+    @keyword_only
+    def __init__(self, *, featuresCol="features", outputCol=None,
+                 labelCol="label", selectionMode="numTopFeatures"):
+        """
+        __init__(self, \\*, featuresCol="features", outputCol=None, \
+                 labelCol="label", selectionMode="numTopFeatures")
+        """
+        super(UnivariateFeatureSelector, self).__init__()
+        self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.UnivariateFeatureSelector",
+                                            self.uid)
+        kwargs = self._input_kwargs
+        self.setParams(**kwargs)
+
+    @keyword_only
+    @since("3.1.1")
+    def setParams(self, *, featuresCol="features", outputCol=None,
+                  labelCol="labels", selectionMode="numTopFeatures"):
+        """
+        setParams(self, \\*, featuresCol="features", outputCol=None, \
+                  labelCol="labels", selectionMode="numTopFeatures")
+        Sets params for this UnivariateFeatureSelector.
+        """
+        kwargs = self._input_kwargs
+        return self._set(**kwargs)
+
+    @since("3.1.1")
+    def setFeatureType(self, value):
+        """
+        Sets the value of :py:attr:`featureType`.
+        """
+        return self._set(featureType=value)
+
+    @since("3.1.1")
+    def setLabelType(self, value):
+        """
+        Sets the value of :py:attr:`labelType`.
+        """
+        return self._set(labelType=value)
+
+    @since("3.1.1")
+    def setSelectionMode(self, value):
+        """
+        Sets the value of :py:attr:`selectionMode`.
+        """
+        return self._set(selectionMode=value)
+
+    @since("3.1.1")
+    def setSelectionThreshold(self, value):
+        """
+        Sets the value of :py:attr:`selectionThreshold`.
+        """
+        return self._set(selectionThreshold=value)
+
+    def setFeaturesCol(self, value):
+        """
+        Sets the value of :py:attr:`featuresCol`.
+        """
+        return self._set(featuresCol=value)
+
+    def setOutputCol(self, value):
+        """
+        Sets the value of :py:attr:`outputCol`.
+        """
+        return self._set(outputCol=value)
+
+    def setLabelCol(self, value):
+        """
+        Sets the value of :py:attr:`labelCol`.
+        """
+        return self._set(labelCol=value)
+
+    def _create_model(self, java_model):
+        return UnivariateFeatureSelectorModel(java_model)
+
+
+class UnivariateFeatureSelectorModel(JavaModel, _UnivariateFeatureSelectorParams, JavaMLReadable,
+                                     JavaMLWritable):
+    """
+    Model fitted by :py:class:`UnivariateFeatureSelector`.
+
+    .. versionadded:: 3.1.1
+    """
+
+    @since("3.1.1")
+    def setFeaturesCol(self, value):
+        """
+        Sets the value of :py:attr:`featuresCol`.
+        """
+        return self._set(featuresCol=value)
+
+    @since("3.1.1")
+    def setOutputCol(self, value):
+        """
+        Sets the value of :py:attr:`outputCol`.
+        """
+        return self._set(outputCol=value)
+
+    @property
+    @since("3.1.1")
+    def selectedFeatures(self):
+        """
+        List of indices to select (filter).
+        """
+        return self._call_java("selectedFeatures")
+
+
 if __name__ == "__main__":
     import doctest
+    import sys
     import tempfile
 
     import pyspark.ml.feature
diff --git a/python/pyspark/ml/feature.pyi b/python/pyspark/ml/feature.pyi
new file mode 100644
index 0000000000000..33e4691d97352
--- /dev/null
+++ b/python/pyspark/ml/feature.pyi
@@ -0,0 +1,1623 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import overload
+from typing import Any, Dict, Generic, List, Optional, Tuple
+from pyspark.ml._typing import JM, P
+
+from pyspark.ml.param.shared import (
+    HasFeaturesCol,
+    HasHandleInvalid,
+    HasInputCol,
+    HasInputCols,
+    HasLabelCol,
+    HasMaxIter,
+    HasNumFeatures,
+    HasOutputCol,
+    HasOutputCols,
+    HasRelativeError,
+    HasSeed,
+    HasStepSize,
+    HasThreshold,
+    HasThresholds,
+)
+from pyspark.ml.util import JavaMLReadable, JavaMLWritable
+from pyspark.ml.wrapper import JavaEstimator, JavaModel, JavaParams, JavaTransformer
+from pyspark.ml.linalg import Vector, DenseVector, DenseMatrix
+from pyspark.sql.dataframe import DataFrame
+from pyspark.ml.param import Param
+
+class Binarizer(
+    JavaTransformer,
+    HasThreshold,
+    HasThresholds,
+    HasInputCol,
+    HasOutputCol,
+    HasInputCols,
+    HasOutputCols,
+    JavaMLReadable[Binarizer],
+    JavaMLWritable,
+):
+    threshold: Param[float]
+    thresholds: Param[List[float]]
+    @overload
+    def __init__(
+        self,
+        *,
+        threshold: float = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> None: ...
+    @overload
+    def __init__(
+        self,
+        *,
+        thresholds: Optional[List[float]] = ...,
+        inputCols: Optional[List[str]] = ...,
+        outputCols: Optional[List[str]] = ...
+    ) -> None: ...
+    @overload
+    def setParams(
+        self,
+        *,
+        threshold: float = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> Binarizer: ...
+    @overload
+    def setParams(
+        self,
+        *,
+        thresholds: Optional[List[float]] = ...,
+        inputCols: Optional[List[str]] = ...,
+        outputCols: Optional[List[str]] = ...
+    ) -> Binarizer: ...
+    def setThreshold(self, value: float) -> Binarizer: ...
+    def setThresholds(self, value: List[float]) -> Binarizer: ...
+    def setInputCol(self, value: str) -> Binarizer: ...
+    def setInputCols(self, value: List[str]) -> Binarizer: ...
+    def setOutputCol(self, value: str) -> Binarizer: ...
+    def setOutputCols(self, value: List[str]) -> Binarizer: ...
+
+class _LSHParams(HasInputCol, HasOutputCol):
+    numHashTables: Param[int]
+    def __init__(self, *args: Any): ...
+    def getNumHashTables(self) -> int: ...
+
+class _LSH(Generic[JM], JavaEstimator[JM], _LSHParams, JavaMLReadable, JavaMLWritable):
+    def setNumHashTables(self: P, value: int) -> P: ...
+    def setInputCol(self: P, value: str) -> P: ...
+    def setOutputCol(self: P, value: str) -> P: ...
+
+class _LSHModel(JavaModel, _LSHParams):
+    def setInputCol(self: P, value: str) -> P: ...
+    def setOutputCol(self: P, value: str) -> P: ...
+    def approxNearestNeighbors(
+        self,
+        dataset: DataFrame,
+        key: Vector,
+        numNearestNeighbors: int,
+        distCol: str = ...,
+    ) -> DataFrame: ...
+    def approxSimilarityJoin(
+        self,
+        datasetA: DataFrame,
+        datasetB: DataFrame,
+        threshold: float,
+        distCol: str = ...,
+    ) -> DataFrame: ...
+
+class _BucketedRandomProjectionLSHParams:
+    bucketLength: Param[float]
+    def getBucketLength(self) -> float: ...
+
+class BucketedRandomProjectionLSH(
+    _LSH[BucketedRandomProjectionLSHModel],
+    _LSHParams,
+    HasSeed,
+    JavaMLReadable[BucketedRandomProjectionLSH],
+    JavaMLWritable,
+):
+    def __init__(
+        self,
+        *,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...,
+        seed: Optional[int] = ...,
+        numHashTables: int = ...,
+        bucketLength: Optional[float] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...,
+        seed: Optional[int] = ...,
+        numHashTables: int = ...,
+        bucketLength: Optional[float] = ...
+    ) -> BucketedRandomProjectionLSH: ...
+    def setBucketLength(self, value: float) -> BucketedRandomProjectionLSH: ...
+    def setSeed(self, value: int) -> BucketedRandomProjectionLSH: ...
+
+class BucketedRandomProjectionLSHModel(
+    _LSHModel,
+    _BucketedRandomProjectionLSHParams,
+    JavaMLReadable[BucketedRandomProjectionLSHModel],
+    JavaMLWritable,
+): ...
+
+class Bucketizer(
+    JavaTransformer,
+    HasInputCol,
+    HasOutputCol,
+    HasInputCols,
+    HasOutputCols,
+    HasHandleInvalid,
+    JavaMLReadable[Bucketizer],
+    JavaMLWritable,
+):
+    splits: Param[List[float]]
+    handleInvalid: Param[str]
+    splitsArray: Param[List[List[float]]]
+    @overload
+    def __init__(
+        self,
+        *,
+        splits: Optional[List[float]] = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...,
+        handleInvalid: str = ...
+    ) -> None: ...
+    @overload
+    def __init__(
+        self,
+        *,
+        handleInvalid: str = ...,
+        splitsArray: Optional[List[List[float]]] = ...,
+        inputCols: Optional[List[str]] = ...,
+        outputCols: Optional[List[str]] = ...
+    ) -> None: ...
+    @overload
+    def setParams(
+        self,
+        *,
+        splits: Optional[List[float]] = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...,
+        handleInvalid: str = ...
+    ) -> Bucketizer: ...
+    @overload
+    def setParams(
+        self,
+        *,
+        handleInvalid: str = ...,
+        splitsArray: Optional[List[List[float]]] = ...,
+        inputCols: Optional[List[str]] = ...,
+        outputCols: Optional[List[str]] = ...
+    ) -> Bucketizer: ...
+    def setSplits(self, value: List[float]) -> Bucketizer: ...
+    def getSplits(self) -> List[float]: ...
+    def setSplitsArray(self, value: List[List[float]]) -> Bucketizer: ...
+    def getSplitsArray(self) -> List[List[float]]: ...
+    def setInputCol(self, value: str) -> Bucketizer: ...
+    def setInputCols(self, value: List[str]) -> Bucketizer: ...
+    def setOutputCol(self, value: str) -> Bucketizer: ...
+    def setOutputCols(self, value: List[str]) -> Bucketizer: ...
+    def setHandleInvalid(self, value: str) -> Bucketizer: ...
+
+class _CountVectorizerParams(JavaParams, HasInputCol, HasOutputCol):
+    minTF: Param[float]
+    minDF: Param[float]
+    maxDF: Param[float]
+    vocabSize: Param[int]
+    binary: Param[bool]
+    def __init__(self, *args: Any) -> None: ...
+    def getMinTF(self) -> float: ...
+    def getMinDF(self) -> float: ...
+    def getMaxDF(self) -> float: ...
+    def getVocabSize(self) -> int: ...
+    def getBinary(self) -> bool: ...
+
+class CountVectorizer(
+    JavaEstimator[CountVectorizerModel],
+    _CountVectorizerParams,
+    JavaMLReadable[CountVectorizer],
+    JavaMLWritable,
+):
+    def __init__(
+        self,
+        *,
+        minTF: float = ...,
+        minDF: float = ...,
+        maxDF: float = ...,
+        vocabSize: int = ...,
+        binary: bool = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        minTF: float = ...,
+        minDF: float = ...,
+        maxDF: float = ...,
+        vocabSize: int = ...,
+        binary: bool = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> CountVectorizer: ...
+    def setMinTF(self, value: float) -> CountVectorizer: ...
+    def setMinDF(self, value: float) -> CountVectorizer: ...
+    def setMaxDF(self, value: float) -> CountVectorizer: ...
+    def setVocabSize(self, value: int) -> CountVectorizer: ...
+    def setBinary(self, value: bool) -> CountVectorizer: ...
+    def setInputCol(self, value: str) -> CountVectorizer: ...
+    def setOutputCol(self, value: str) -> CountVectorizer: ...
+
+class CountVectorizerModel(
+    JavaModel, JavaMLReadable[CountVectorizerModel], JavaMLWritable
+):
+    def setInputCol(self, value: str) -> CountVectorizerModel: ...
+    def setOutputCol(self, value: str) -> CountVectorizerModel: ...
+    def setMinTF(self, value: float) -> CountVectorizerModel: ...
+    def setBinary(self, value: bool) -> CountVectorizerModel: ...
+    @classmethod
+    def from_vocabulary(
+        cls,
+        vocabulary: List[str],
+        inputCol: str,
+        outputCol: Optional[str] = ...,
+        minTF: Optional[float] = ...,
+        binary: Optional[bool] = ...,
+    ) -> CountVectorizerModel: ...
+    @property
+    def vocabulary(self) -> List[str]: ...
+
+class DCT(
+    JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable[DCT], JavaMLWritable
+):
+    inverse: Param[bool]
+    def __init__(
+        self,
+        *,
+        inverse: bool = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        inverse: bool = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> DCT: ...
+    def setInverse(self, value: bool) -> DCT: ...
+    def getInverse(self) -> bool: ...
+    def setInputCol(self, value: str) -> DCT: ...
+    def setOutputCol(self, value: str) -> DCT: ...
+
+class ElementwiseProduct(
+    JavaTransformer,
+    HasInputCol,
+    HasOutputCol,
+    JavaMLReadable[ElementwiseProduct],
+    JavaMLWritable,
+):
+    scalingVec: Param[Vector]
+    def __init__(
+        self,
+        *,
+        scalingVec: Optional[Vector] = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        scalingVec: Optional[Vector] = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> ElementwiseProduct: ...
+    def setScalingVec(self, value: Vector) -> ElementwiseProduct: ...
+    def getScalingVec(self) -> Vector: ...
+    def setInputCol(self, value: str) -> ElementwiseProduct: ...
+    def setOutputCol(self, value: str) -> ElementwiseProduct: ...
+
+class FeatureHasher(
+    JavaTransformer,
+    HasInputCols,
+    HasOutputCol,
+    HasNumFeatures,
+    JavaMLReadable[FeatureHasher],
+    JavaMLWritable,
+):
+    categoricalCols: Param[List[str]]
+    def __init__(
+        self,
+        *,
+        numFeatures: int = ...,
+        inputCols: Optional[List[str]] = ...,
+        outputCol: Optional[str] = ...,
+        categoricalCols: Optional[List[str]] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        numFeatures: int = ...,
+        inputCols: Optional[List[str]] = ...,
+        outputCol: Optional[str] = ...,
+        categoricalCols: Optional[List[str]] = ...
+    ) -> FeatureHasher: ...
+    def setCategoricalCols(self, value: List[str]) -> FeatureHasher: ...
+    def getCategoricalCols(self) -> List[str]: ...
+    def setInputCols(self, value: List[str]) -> FeatureHasher: ...
+    def setOutputCol(self, value: str) -> FeatureHasher: ...
+    def setNumFeatures(self, value: int) -> FeatureHasher: ...
+
+class HashingTF(
+    JavaTransformer,
+    HasInputCol,
+    HasOutputCol,
+    HasNumFeatures,
+    JavaMLReadable[HashingTF],
+    JavaMLWritable,
+):
+    binary: Param[bool]
+    def __init__(
+        self,
+        *,
+        numFeatures: int = ...,
+        binary: bool = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        numFeatures: int = ...,
+        binary: bool = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> HashingTF: ...
+    def setBinary(self, value: bool) -> HashingTF: ...
+    def getBinary(self) -> bool: ...
+    def setInputCol(self, value: str) -> HashingTF: ...
+    def setOutputCol(self, value: str) -> HashingTF: ...
+    def setNumFeatures(self, value: int) -> HashingTF: ...
+    def indexOf(self, term: Any) -> int: ...
+
+class _IDFParams(HasInputCol, HasOutputCol):
+    minDocFreq: Param[int]
+    def __init__(self, *args: Any): ...
+    def getMinDocFreq(self) -> int: ...
+
+class IDF(JavaEstimator[IDFModel], _IDFParams, JavaMLReadable[IDF], JavaMLWritable):
+    def __init__(
+        self,
+        *,
+        minDocFreq: int = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        minDocFreq: int = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> IDF: ...
+    def setMinDocFreq(self, value: int) -> IDF: ...
+    def setInputCol(self, value: str) -> IDF: ...
+    def setOutputCol(self, value: str) -> IDF: ...
+
+class IDFModel(JavaModel, _IDFParams, JavaMLReadable[IDFModel], JavaMLWritable):
+    def setInputCol(self, value: str) -> IDFModel: ...
+    def setOutputCol(self, value: str) -> IDFModel: ...
+    @property
+    def idf(self) -> Vector: ...
+    @property
+    def docFreq(self) -> List[int]: ...
+    @property
+    def numDocs(self) -> int: ...
+
+class _ImputerParams(
+    HasInputCol, HasInputCols, HasOutputCol, HasOutputCols, HasRelativeError
+):
+    strategy: Param[str]
+    missingValue: Param[float]
+    def getStrategy(self) -> str: ...
+    def getMissingValue(self) -> float: ...
+
+class Imputer(
+    JavaEstimator[ImputerModel], _ImputerParams, JavaMLReadable[Imputer], JavaMLWritable
+):
+    @overload
+    def __init__(
+        self,
+        *,
+        strategy: str = ...,
+        missingValue: float = ...,
+        inputCols: Optional[List[str]] = ...,
+        outputCols: Optional[List[str]] = ...,
+        relativeError: float = ...
+    ) -> None: ...
+    @overload
+    def __init__(
+        self,
+        *,
+        strategy: str = ...,
+        missingValue: float = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...,
+        relativeError: float = ...
+    ) -> None: ...
+    @overload
+    def setParams(
+        self,
+        *,
+        strategy: str = ...,
+        missingValue: float = ...,
+        inputCols: Optional[List[str]] = ...,
+        outputCols: Optional[List[str]] = ...,
+        relativeError: float = ...
+    ) -> Imputer: ...
+    @overload
+    def setParams(
+        self,
+        *,
+        strategy: str = ...,
+        missingValue: float = ...,
+        inputCol: Optional[str] = ...,
+        outputCols: Optional[str] = ...,
+        relativeError: float = ...
+    ) -> Imputer: ...
+    def setStrategy(self, value: str) -> Imputer: ...
+    def setMissingValue(self, value: float) -> Imputer: ...
+    def setInputCols(self, value: List[str]) -> Imputer: ...
+    def setOutputCols(self, value: List[str]) -> Imputer: ...
+    def setInputCol(self, value: str) -> Imputer: ...
+    def setOutputCol(self, value: str) -> Imputer: ...
+    def setRelativeError(self, value: float) -> Imputer: ...
+
+class ImputerModel(
+    JavaModel, _ImputerParams, JavaMLReadable[ImputerModel], JavaMLWritable
+):
+    def setInputCols(self, value: List[str]) -> ImputerModel: ...
+    def setOutputCols(self, value: List[str]) -> ImputerModel: ...
+    def setInputCol(self, value: str) -> ImputerModel: ...
+    def setOutputCol(self, value: str) -> ImputerModel: ...
+    @property
+    def surrogateDF(self) -> DataFrame: ...
+
+class Interaction(
+    JavaTransformer,
+    HasInputCols,
+    HasOutputCol,
+    JavaMLReadable[Interaction],
+    JavaMLWritable,
+):
+    def __init__(
+        self, *, inputCols: Optional[List[str]] = ..., outputCol: Optional[str] = ...
+    ) -> None: ...
+    def setParams(
+        self, *, inputCols: Optional[List[str]] = ..., outputCol: Optional[str] = ...
+    ) -> Interaction: ...
+    def setInputCols(self, value: List[str]) -> Interaction: ...
+    def setOutputCol(self, value: str) -> Interaction: ...
+
+class _MaxAbsScalerParams(HasInputCol, HasOutputCol): ...
+
+class MaxAbsScaler(
+    JavaEstimator[MaxAbsScalerModel],
+    _MaxAbsScalerParams,
+    JavaMLReadable[MaxAbsScaler],
+    JavaMLWritable,
+):
+    def __init__(
+        self, *, inputCol: Optional[str] = ..., outputCol: Optional[str] = ...
+    ) -> None: ...
+    def setParams(
+        self, *, inputCol: Optional[str] = ..., outputCol: Optional[str] = ...
+    ) -> MaxAbsScaler: ...
+    def setInputCol(self, value: str) -> MaxAbsScaler: ...
+    def setOutputCol(self, value: str) -> MaxAbsScaler: ...
+
+class MaxAbsScalerModel(
+    JavaModel, _MaxAbsScalerParams, JavaMLReadable[MaxAbsScalerModel], JavaMLWritable
+):
+    def setInputCol(self, value: str) -> MaxAbsScalerModel: ...
+    def setOutputCol(self, value: str) -> MaxAbsScalerModel: ...
+    @property
+    def maxAbs(self) -> Vector: ...
+
+class MinHashLSH(
+    _LSH[MinHashLSHModel],
+    HasInputCol,
+    HasOutputCol,
+    HasSeed,
+    JavaMLReadable[MinHashLSH],
+    JavaMLWritable,
+):
+    def __init__(
+        self,
+        *,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...,
+        seed: Optional[int] = ...,
+        numHashTables: int = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...,
+        seed: Optional[int] = ...,
+        numHashTables: int = ...
+    ) -> MinHashLSH: ...
+    def setSeed(self, value: int) -> MinHashLSH: ...
+
+class MinHashLSHModel(_LSHModel, JavaMLReadable[MinHashLSHModel], JavaMLWritable): ...
+
+class _MinMaxScalerParams(HasInputCol, HasOutputCol):
+    min: Param[float]
+    max: Param[float]
+    def __init__(self, *args: Any): ...
+    def getMin(self) -> float: ...
+    def getMax(self) -> float: ...
+
+class MinMaxScaler(
+    JavaEstimator[MinMaxScalerModel],
+    _MinMaxScalerParams,
+    JavaMLReadable[MinMaxScaler],
+    JavaMLWritable,
+):
+    def __init__(
+        self,
+        *,
+        min: float = ...,
+        max: float = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        min: float = ...,
+        max: float = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> MinMaxScaler: ...
+    def setMin(self, value: float) -> MinMaxScaler: ...
+    def setMax(self, value: float) -> MinMaxScaler: ...
+    def setInputCol(self, value: str) -> MinMaxScaler: ...
+    def setOutputCol(self, value: str) -> MinMaxScaler: ...
+
+class MinMaxScalerModel(
+    JavaModel, _MinMaxScalerParams, JavaMLReadable[MinMaxScalerModel], JavaMLWritable
+):
+    def setInputCol(self, value: str) -> MinMaxScalerModel: ...
+    def setOutputCol(self, value: str) -> MinMaxScalerModel: ...
+    def setMin(self, value: float) -> MinMaxScalerModel: ...
+    def setMax(self, value: float) -> MinMaxScalerModel: ...
+    @property
+    def originalMin(self) -> Vector: ...
+    @property
+    def originalMax(self) -> Vector: ...
+
+class NGram(
+    JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable[NGram], JavaMLWritable
+):
+    n: Param[int]
+    def __init__(
+        self,
+        *,
+        n: int = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        n: int = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> NGram: ...
+    def setN(self, value: int) -> NGram: ...
+    def getN(self) -> int: ...
+    def setInputCol(self, value: str) -> NGram: ...
+    def setOutputCol(self, value: str) -> NGram: ...
+
+class Normalizer(
+    JavaTransformer,
+    HasInputCol,
+    HasOutputCol,
+    JavaMLReadable[Normalizer],
+    JavaMLWritable,
+):
+    p: Param[float]
+    def __init__(
+        self,
+        *,
+        p: float = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        p: float = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> Normalizer: ...
+    def setP(self, value: float) -> Normalizer: ...
+    def getP(self) -> float: ...
+    def setInputCol(self, value: str) -> Normalizer: ...
+    def setOutputCol(self, value: str) -> Normalizer: ...
+
+class _OneHotEncoderParams(HasInputCols, HasOutputCols, HasHandleInvalid):
+    handleInvalid: Param[str]
+    dropLast: Param[bool]
+    def __init__(self, *args: Any): ...
+    def getDropLast(self) -> bool: ...
+
+class OneHotEncoder(
+    JavaEstimator[OneHotEncoderModel],
+    _OneHotEncoderParams,
+    JavaMLReadable[OneHotEncoder],
+    JavaMLWritable,
+):
+    @overload
+    def __init__(
+        self,
+        *,
+        inputCols: Optional[List[str]] = ...,
+        outputCols: Optional[List[str]] = ...,
+        handleInvalid: str = ...,
+        dropLast: bool = ...
+    ) -> None: ...
+    @overload
+    def __init__(
+        self,
+        *,
+        handleInvalid: str = ...,
+        dropLast: bool = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> None: ...
+    @overload
+    def setParams(
+        self,
+        *,
+        inputCols: Optional[List[str]] = ...,
+        outputCols: Optional[List[str]] = ...,
+        handleInvalid: str = ...,
+        dropLast: bool = ...
+    ) -> OneHotEncoder: ...
+    @overload
+    def setParams(
+        self,
+        *,
+        handleInvalid: str = ...,
+        dropLast: bool = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> OneHotEncoder: ...
+    def setDropLast(self, value: bool) -> OneHotEncoder: ...
+    def setInputCols(self, value: List[str]) -> OneHotEncoder: ...
+    def setOutputCols(self, value: List[str]) -> OneHotEncoder: ...
+    def setHandleInvalid(self, value: str) -> OneHotEncoder: ...
+    def setInputCol(self, value: str) -> OneHotEncoder: ...
+    def setOutputCol(self, value: str) -> OneHotEncoder: ...
+
+class OneHotEncoderModel(
+    JavaModel, _OneHotEncoderParams, JavaMLReadable[OneHotEncoderModel], JavaMLWritable
+):
+    def setDropLast(self, value: bool) -> OneHotEncoderModel: ...
+    def setInputCols(self, value: List[str]) -> OneHotEncoderModel: ...
+    def setOutputCols(self, value: List[str]) -> OneHotEncoderModel: ...
+    def setInputCol(self, value: str) -> OneHotEncoderModel: ...
+    def setOutputCol(self, value: str) -> OneHotEncoderModel: ...
+    def setHandleInvalid(self, value: str) -> OneHotEncoderModel: ...
+    @property
+    def categorySizes(self) -> List[int]: ...
+
+class PolynomialExpansion(
+    JavaTransformer,
+    HasInputCol,
+    HasOutputCol,
+    JavaMLReadable[PolynomialExpansion],
+    JavaMLWritable,
+):
+    degree: Param[int]
+    def __init__(
+        self,
+        *,
+        degree: int = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        degree: int = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> PolynomialExpansion: ...
+    def setDegree(self, value: int) -> PolynomialExpansion: ...
+    def getDegree(self) -> int: ...
+    def setInputCol(self, value: str) -> PolynomialExpansion: ...
+    def setOutputCol(self, value: str) -> PolynomialExpansion: ...
+
+class QuantileDiscretizer(
+    JavaEstimator[Bucketizer],
+    HasInputCol,
+    HasOutputCol,
+    HasInputCols,
+    HasOutputCols,
+    HasHandleInvalid,
+    HasRelativeError,
+    JavaMLReadable[QuantileDiscretizer],
+    JavaMLWritable,
+):
+    numBuckets: Param[int]
+    handleInvalid: Param[str]
+    numBucketsArray: Param[List[int]]
+    @overload
+    def __init__(
+        self,
+        *,
+        numBuckets: int = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...,
+        relativeError: float = ...,
+        handleInvalid: str = ...
+    ) -> None: ...
+    @overload
+    def __init__(
+        self,
+        *,
+        relativeError: float = ...,
+        handleInvalid: str = ...,
+        numBucketsArray: Optional[List[int]] = ...,
+        inputCols: Optional[List[str]] = ...,
+        outputCols: Optional[List[str]] = ...
+    ) -> None: ...
+    @overload
+    def setParams(
+        self,
+        *,
+        numBuckets: int = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...,
+        relativeError: float = ...,
+        handleInvalid: str = ...
+    ) -> QuantileDiscretizer: ...
+    @overload
+    def setParams(
+        self,
+        *,
+        relativeError: float = ...,
+        handleInvalid: str = ...,
+        numBucketsArray: Optional[List[int]] = ...,
+        inputCols: Optional[List[str]] = ...,
+        outputCols: Optional[List[str]] = ...
+    ) -> QuantileDiscretizer: ...
+    def setNumBuckets(self, value: int) -> QuantileDiscretizer: ...
+    def getNumBuckets(self) -> int: ...
+    def setNumBucketsArray(self, value: List[int]) -> QuantileDiscretizer: ...
+    def getNumBucketsArray(self) -> List[int]: ...
+    def setRelativeError(self, value: float) -> QuantileDiscretizer: ...
+    def setInputCol(self, value: str) -> QuantileDiscretizer: ...
+    def setInputCols(self, value: List[str]) -> QuantileDiscretizer: ...
+    def setOutputCol(self, value: str) -> QuantileDiscretizer: ...
+    def setOutputCols(self, value: List[str]) -> QuantileDiscretizer: ...
+    def setHandleInvalid(self, value: str) -> QuantileDiscretizer: ...
+
+class _RobustScalerParams(HasInputCol, HasOutputCol, HasRelativeError):
+    lower: Param[float]
+    upper: Param[float]
+    withCentering: Param[bool]
+    withScaling: Param[bool]
+    def __init__(self, *args: Any): ...
+    def getLower(self) -> float: ...
+    def getUpper(self) -> float: ...
+    def getWithCentering(self) -> bool: ...
+    def getWithScaling(self) -> bool: ...
+
+class RobustScaler(
+    JavaEstimator, _RobustScalerParams, JavaMLReadable[RobustScaler], JavaMLWritable
+):
+    def __init__(
+        self,
+        *,
+        lower: float = ...,
+        upper: float = ...,
+        withCentering: bool = ...,
+        withScaling: bool = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...,
+        relativeError: float = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        lower: float = ...,
+        upper: float = ...,
+        withCentering: bool = ...,
+        withScaling: bool = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...,
+        relativeError: float = ...
+    ) -> RobustScaler: ...
+    def setLower(self, value: float) -> RobustScaler: ...
+    def setUpper(self, value: float) -> RobustScaler: ...
+    def setWithCentering(self, value: bool) -> RobustScaler: ...
+    def setWithScaling(self, value: bool) -> RobustScaler: ...
+    def setInputCol(self, value: str) -> RobustScaler: ...
+    def setOutputCol(self, value: str) -> RobustScaler: ...
+    def setRelativeError(self, value: float) -> RobustScaler: ...
+
+class RobustScalerModel(
+    JavaModel, _RobustScalerParams, JavaMLReadable[RobustScalerModel], JavaMLWritable
+):
+    def setInputCol(self, value: str) -> RobustScalerModel: ...
+    def setOutputCol(self, value: str) -> RobustScalerModel: ...
+    @property
+    def median(self) -> Vector: ...
+    @property
+    def range(self) -> Vector: ...
+
+class RegexTokenizer(
+    JavaTransformer,
+    HasInputCol,
+    HasOutputCol,
+    JavaMLReadable[RegexTokenizer],
+    JavaMLWritable,
+):
+    minTokenLength: Param[int]
+    gaps: Param[bool]
+    pattern: Param[str]
+    toLowercase: Param[bool]
+    def __init__(
+        self,
+        *,
+        minTokenLength: int = ...,
+        gaps: bool = ...,
+        pattern: str = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...,
+        toLowercase: bool = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        minTokenLength: int = ...,
+        gaps: bool = ...,
+        pattern: str = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...,
+        toLowercase: bool = ...
+    ) -> RegexTokenizer: ...
+    def setMinTokenLength(self, value: int) -> RegexTokenizer: ...
+    def getMinTokenLength(self) -> int: ...
+    def setGaps(self, value: bool) -> RegexTokenizer: ...
+    def getGaps(self) -> bool: ...
+    def setPattern(self, value: str) -> RegexTokenizer: ...
+    def getPattern(self) -> str: ...
+    def setToLowercase(self, value: bool) -> RegexTokenizer: ...
+    def getToLowercase(self) -> bool: ...
+    def setInputCol(self, value: str) -> RegexTokenizer: ...
+    def setOutputCol(self, value: str) -> RegexTokenizer: ...
+
+class SQLTransformer(JavaTransformer, JavaMLReadable[SQLTransformer], JavaMLWritable):
+    statement: Param[str]
+    def __init__(self, *, statement: Optional[str] = ...) -> None: ...
+    def setParams(self, *, statement: Optional[str] = ...) -> SQLTransformer: ...
+    def setStatement(self, value: str) -> SQLTransformer: ...
+    def getStatement(self) -> str: ...
+
+class _StandardScalerParams(HasInputCol, HasOutputCol):
+    withMean: Param[bool]
+    withStd: Param[bool]
+    def __init__(self, *args: Any): ...
+    def getWithMean(self) -> bool: ...
+    def getWithStd(self) -> bool: ...
+
+class StandardScaler(
+    JavaEstimator[StandardScalerModel],
+    _StandardScalerParams,
+    JavaMLReadable[StandardScaler],
+    JavaMLWritable,
+):
+    def __init__(
+        self,
+        *,
+        withMean: bool = ...,
+        withStd: bool = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        withMean: bool = ...,
+        withStd: bool = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> StandardScaler: ...
+    def setWithMean(self, value: bool) -> StandardScaler: ...
+    def setWithStd(self, value: bool) -> StandardScaler: ...
+    def setInputCol(self, value: str) -> StandardScaler: ...
+    def setOutputCol(self, value: str) -> StandardScaler: ...
+
+class StandardScalerModel(
+    JavaModel,
+    _StandardScalerParams,
+    JavaMLReadable[StandardScalerModel],
+    JavaMLWritable,
+):
+    def setInputCol(self, value: str) -> StandardScalerModel: ...
+    def setOutputCol(self, value: str) -> StandardScalerModel: ...
+    @property
+    def std(self) -> Vector: ...
+    @property
+    def mean(self) -> Vector: ...
+
+class _StringIndexerParams(
+    JavaParams, HasHandleInvalid, HasInputCol, HasOutputCol, HasInputCols, HasOutputCols
+):
+    stringOrderType: Param[str]
+    handleInvalid: Param[str]
+    def __init__(self, *args: Any) -> None: ...
+    def getStringOrderType(self) -> str: ...
+
+class StringIndexer(
+    JavaEstimator[StringIndexerModel],
+    _StringIndexerParams,
+    JavaMLReadable[StringIndexer],
+    JavaMLWritable,
+):
+    @overload
+    def __init__(
+        self,
+        *,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...,
+        handleInvalid: str = ...,
+        stringOrderType: str = ...
+    ) -> None: ...
+    @overload
+    def __init__(
+        self,
+        *,
+        inputCols: Optional[List[str]] = ...,
+        outputCols: Optional[List[str]] = ...,
+        handleInvalid: str = ...,
+        stringOrderType: str = ...
+    ) -> None: ...
+    @overload
+    def setParams(
+        self,
+        *,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...,
+        handleInvalid: str = ...,
+        stringOrderType: str = ...
+    ) -> StringIndexer: ...
+    @overload
+    def setParams(
+        self,
+        *,
+        inputCols: Optional[List[str]] = ...,
+        outputCols: Optional[List[str]] = ...,
+        handleInvalid: str = ...,
+        stringOrderType: str = ...
+    ) -> StringIndexer: ...
+    def setStringOrderType(self, value: str) -> StringIndexer: ...
+    def setInputCol(self, value: str) -> StringIndexer: ...
+    def setInputCols(self, value: List[str]) -> StringIndexer: ...
+    def setOutputCol(self, value: str) -> StringIndexer: ...
+    def setOutputCols(self, value: List[str]) -> StringIndexer: ...
+    def setHandleInvalid(self, value: str) -> StringIndexer: ...
+
+class StringIndexerModel(
+    JavaModel, _StringIndexerParams, JavaMLReadable[StringIndexerModel], JavaMLWritable
+):
+    def setInputCol(self, value: str) -> StringIndexerModel: ...
+    def setInputCols(self, value: List[str]) -> StringIndexerModel: ...
+    def setOutputCol(self, value: str) -> StringIndexerModel: ...
+    def setOutputCols(self, value: List[str]) -> StringIndexerModel: ...
+    def setHandleInvalid(self, value: str) -> StringIndexerModel: ...
+    @classmethod
+    def from_labels(
+        cls,
+        labels: List[str],
+        inputCol: str,
+        outputCol: Optional[str] = ...,
+        handleInvalid: Optional[str] = ...,
+    ) -> StringIndexerModel: ...
+    @classmethod
+    def from_arrays_of_labels(
+        cls,
+        arrayOfLabels: List[List[str]],
+        inputCols: List[str],
+        outputCols: Optional[List[str]] = ...,
+        handleInvalid: Optional[str] = ...,
+    ) -> StringIndexerModel: ...
+    @property
+    def labels(self) -> List[str]: ...
+
+class IndexToString(
+    JavaTransformer,
+    HasInputCol,
+    HasOutputCol,
+    JavaMLReadable[IndexToString],
+    JavaMLWritable,
+):
+    labels: Param[List[str]]
+    def __init__(
+        self,
+        *,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...,
+        labels: Optional[List[str]] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...,
+        labels: Optional[List[str]] = ...
+    ) -> IndexToString: ...
+    def setLabels(self, value: List[str]) -> IndexToString: ...
+    def getLabels(self) -> List[str]: ...
+    def setInputCol(self, value: str) -> IndexToString: ...
+    def setOutputCol(self, value: str) -> IndexToString: ...
+
+class StopWordsRemover(
+    JavaTransformer,
+    HasInputCol,
+    HasOutputCol,
+    HasInputCols,
+    HasOutputCols,
+    JavaMLReadable[StopWordsRemover],
+    JavaMLWritable,
+):
+    stopWords: Param[List[str]]
+    caseSensitive: Param[bool]
+    locale: Param[str]
+    @overload
+    def __init__(
+        self,
+        *,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...,
+        stopWords: Optional[List[str]] = ...,
+        caseSensitive: bool = ...,
+        locale: Optional[str] = ...
+    ) -> None: ...
+    @overload
+    def __init__(
+        self,
+        *,
+        stopWords: Optional[List[str]] = ...,
+        caseSensitive: bool = ...,
+        locale: Optional[str] = ...,
+        inputCols: Optional[List[str]] = ...,
+        outputCols: Optional[List[str]] = ...
+    ) -> None: ...
+    @overload
+    def setParams(
+        self,
+        *,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...,
+        stopWords: Optional[List[str]] = ...,
+        caseSensitive: bool = ...,
+        locale: Optional[str] = ...
+    ) -> StopWordsRemover: ...
+    @overload
+    def setParams(
+        self,
+        *,
+        stopWords: Optional[List[str]] = ...,
+        caseSensitive: bool = ...,
+        locale: Optional[str] = ...,
+        inputCols: Optional[List[str]] = ...,
+        outputCols: Optional[List[str]] = ...
+    ) -> StopWordsRemover: ...
+    def setStopWords(self, value: List[str]) -> StopWordsRemover: ...
+    def getStopWords(self) -> List[str]: ...
+    def setCaseSensitive(self, value: bool) -> StopWordsRemover: ...
+    def getCaseSensitive(self) -> bool: ...
+    def setLocale(self, value: str) -> StopWordsRemover: ...
+    def getLocale(self) -> str: ...
+    def setInputCol(self, value: str) -> StopWordsRemover: ...
+    def setOutputCol(self, value: str) -> StopWordsRemover: ...
+    def setInputCols(self, value: List[str]) -> StopWordsRemover: ...
+    def setOutputCols(self, value: List[str]) -> StopWordsRemover: ...
+    @staticmethod
+    def loadDefaultStopWords(language: str) -> List[str]: ...
+
+class Tokenizer(
+    JavaTransformer,
+    HasInputCol,
+    HasOutputCol,
+    JavaMLReadable[Tokenizer],
+    JavaMLWritable,
+):
+    def __init__(
+        self, *, inputCol: Optional[str] = ..., outputCol: Optional[str] = ...
+    ) -> None: ...
+    def setParams(
+        self, *, inputCol: Optional[str] = ..., outputCol: Optional[str] = ...
+    ) -> Tokenizer: ...
+    def setInputCol(self, value: str) -> Tokenizer: ...
+    def setOutputCol(self, value: str) -> Tokenizer: ...
+
+class VectorAssembler(
+    JavaTransformer,
+    HasInputCols,
+    HasOutputCol,
+    HasHandleInvalid,
+    JavaMLReadable[VectorAssembler],
+    JavaMLWritable,
+):
+    handleInvalid: Param[str]
+    def __init__(
+        self,
+        *,
+        inputCols: Optional[List[str]] = ...,
+        outputCol: Optional[str] = ...,
+        handleInvalid: str = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        inputCols: Optional[List[str]] = ...,
+        outputCol: Optional[str] = ...,
+        handleInvalid: str = ...
+    ) -> VectorAssembler: ...
+    def setInputCols(self, value: List[str]) -> VectorAssembler: ...
+    def setOutputCol(self, value: str) -> VectorAssembler: ...
+    def setHandleInvalid(self, value: str) -> VectorAssembler: ...
+
+class _VectorIndexerParams(HasInputCol, HasOutputCol, HasHandleInvalid):
+    maxCategories: Param[int]
+    handleInvalid: Param[str]
+    def __init__(self, *args: Any): ...
+    def getMaxCategories(self) -> int: ...
+
+class VectorIndexer(
+    JavaEstimator[VectorIndexerModel],
+    _VectorIndexerParams,
+    HasHandleInvalid,
+    JavaMLReadable[VectorIndexer],
+    JavaMLWritable,
+):
+    def __init__(
+        self,
+        *,
+        maxCategories: int = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...,
+        handleInvalid: str = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        maxCategories: int = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...,
+        handleInvalid: str = ...
+    ) -> VectorIndexer: ...
+    def setMaxCategories(self, value: int) -> VectorIndexer: ...
+    def setInputCol(self, value: str) -> VectorIndexer: ...
+    def setOutputCol(self, value: str) -> VectorIndexer: ...
+    def setHandleInvalid(self, value: str) -> VectorIndexer: ...
+
+class VectorIndexerModel(
+    JavaModel, _VectorIndexerParams, JavaMLReadable[VectorIndexerModel], JavaMLWritable
+):
+    def setInputCol(self, value: str) -> VectorIndexerModel: ...
+    def setOutputCol(self, value: str) -> VectorIndexerModel: ...
+    @property
+    def numFeatures(self) -> int: ...
+    @property
+    def categoryMaps(self) -> Dict[int, Tuple[float, int]]: ...
+
+class VectorSlicer(
+    JavaTransformer,
+    HasInputCol,
+    HasOutputCol,
+    JavaMLReadable[VectorSlicer],
+    JavaMLWritable,
+):
+    indices: Param[List[int]]
+    names: Param[List[str]]
+    def __init__(
+        self,
+        *,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...,
+        indices: Optional[List[int]] = ...,
+        names: Optional[List[str]] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...,
+        indices: Optional[List[int]] = ...,
+        names: Optional[List[str]] = ...
+    ) -> VectorSlicer: ...
+    def setIndices(self, value: List[int]) -> VectorSlicer: ...
+    def getIndices(self) -> List[int]: ...
+    def setNames(self, value: List[str]) -> VectorSlicer: ...
+    def getNames(self) -> List[str]: ...
+    def setInputCol(self, value: str) -> VectorSlicer: ...
+    def setOutputCol(self, value: str) -> VectorSlicer: ...
+
+class _Word2VecParams(HasStepSize, HasMaxIter, HasSeed, HasInputCol, HasOutputCol):
+    vectorSize: Param[int]
+    numPartitions: Param[int]
+    minCount: Param[int]
+    windowSize: Param[int]
+    maxSentenceLength: Param[int]
+    def __init__(self, *args: Any): ...
+    def getVectorSize(self) -> int: ...
+    def getNumPartitions(self) -> int: ...
+    def getMinCount(self) -> int: ...
+    def getWindowSize(self) -> int: ...
+    def getMaxSentenceLength(self) -> int: ...
+
+class Word2Vec(
+    JavaEstimator[Word2VecModel],
+    _Word2VecParams,
+    JavaMLReadable[Word2Vec],
+    JavaMLWritable,
+):
+    def __init__(
+        self,
+        *,
+        vectorSize: int = ...,
+        minCount: int = ...,
+        numPartitions: int = ...,
+        stepSize: float = ...,
+        maxIter: int = ...,
+        seed: Optional[int] = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...,
+        windowSize: int = ...,
+        maxSentenceLength: int = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        vectorSize: int = ...,
+        minCount: int = ...,
+        numPartitions: int = ...,
+        stepSize: float = ...,
+        maxIter: int = ...,
+        seed: Optional[int] = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...,
+        windowSize: int = ...,
+        maxSentenceLength: int = ...
+    ) -> Word2Vec: ...
+    def setVectorSize(self, value: int) -> Word2Vec: ...
+    def setNumPartitions(self, value: int) -> Word2Vec: ...
+    def setMinCount(self, value: int) -> Word2Vec: ...
+    def setWindowSize(self, value: int) -> Word2Vec: ...
+    def setMaxSentenceLength(self, value: int) -> Word2Vec: ...
+    def setMaxIter(self, value: int) -> Word2Vec: ...
+    def setInputCol(self, value: str) -> Word2Vec: ...
+    def setOutputCol(self, value: str) -> Word2Vec: ...
+    def setSeed(self, value: int) -> Word2Vec: ...
+    def setStepSize(self, value: float) -> Word2Vec: ...
+
+class Word2VecModel(
+    JavaModel, _Word2VecParams, JavaMLReadable[Word2VecModel], JavaMLWritable
+):
+    def getVectors(self) -> DataFrame: ...
+    def setInputCol(self, value: str) -> Word2VecModel: ...
+    def setOutputCol(self, value: str) -> Word2VecModel: ...
+    @overload
+    def findSynonyms(self, word: str, num: int) -> DataFrame: ...
+    @overload
+    def findSynonyms(self, word: Vector, num: int) -> DataFrame: ...
+    @overload
+    def findSynonymsArray(self, word: str, num: int) -> List[Tuple[str, float]]: ...
+    @overload
+    def findSynonymsArray(self, word: Vector, num: int) -> List[Tuple[str, float]]: ...
+
+class _PCAParams(HasInputCol, HasOutputCol):
+    k: Param[int]
+    def getK(self) -> int: ...
+
+class PCA(JavaEstimator[PCAModel], _PCAParams, JavaMLReadable[PCA], JavaMLWritable):
+    def __init__(
+        self,
+        *,
+        k: Optional[int] = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        k: Optional[int] = ...,
+        inputCol: Optional[str] = ...,
+        outputCol: Optional[str] = ...
+    ) -> PCA: ...
+    def setK(self, value: int) -> PCA: ...
+    def setInputCol(self, value: str) -> PCA: ...
+    def setOutputCol(self, value: str) -> PCA: ...
+
+class PCAModel(JavaModel, _PCAParams, JavaMLReadable[PCAModel], JavaMLWritable):
+    def setInputCol(self, value: str) -> PCAModel: ...
+    def setOutputCol(self, value: str) -> PCAModel: ...
+    @property
+    def pc(self) -> DenseMatrix: ...
+    @property
+    def explainedVariance(self) -> DenseVector: ...
+
+class _RFormulaParams(HasFeaturesCol, HasLabelCol, HasHandleInvalid):
+    formula: Param[str]
+    forceIndexLabel: Param[bool]
+    stringIndexerOrderType: Param[str]
+    handleInvalid: Param[str]
+    def __init__(self, *args: Any): ...
+    def getFormula(self) -> str: ...
+    def getForceIndexLabel(self) -> bool: ...
+    def getStringIndexerOrderType(self) -> str: ...
+
+class RFormula(
+    JavaEstimator[RFormulaModel],
+    _RFormulaParams,
+    JavaMLReadable[RFormula],
+    JavaMLWritable,
+):
+    def __init__(
+        self,
+        *,
+        formula: Optional[str] = ...,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        forceIndexLabel: bool = ...,
+        stringIndexerOrderType: str = ...,
+        handleInvalid: str = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        formula: Optional[str] = ...,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        forceIndexLabel: bool = ...,
+        stringIndexerOrderType: str = ...,
+        handleInvalid: str = ...
+    ) -> RFormula: ...
+    def setFormula(self, value: str) -> RFormula: ...
+    def setForceIndexLabel(self, value: bool) -> RFormula: ...
+    def setStringIndexerOrderType(self, value: str) -> RFormula: ...
+    def setFeaturesCol(self, value: str) -> RFormula: ...
+    def setLabelCol(self, value: str) -> RFormula: ...
+    def setHandleInvalid(self, value: str) -> RFormula: ...
+
+class RFormulaModel(
+    JavaModel, _RFormulaParams, JavaMLReadable[RFormulaModel], JavaMLWritable
+): ...
+
+class _SelectorParams(HasFeaturesCol, HasOutputCol, HasLabelCol):
+    selectorType: Param[str]
+    numTopFeatures: Param[int]
+    percentile: Param[float]
+    fpr: Param[float]
+    fdr: Param[float]
+    fwe: Param[float]
+    def __init__(self, *args: Any): ...
+    def getSelectorType(self) -> str: ...
+    def getNumTopFeatures(self) -> int: ...
+    def getPercentile(self) -> float: ...
+    def getFpr(self) -> float: ...
+    def getFdr(self) -> float: ...
+    def getFwe(self) -> float: ...
+
+class _Selector(JavaEstimator[JM], _SelectorParams, JavaMLReadable, JavaMLWritable):
+    def setSelectorType(self: P, value: str) -> P: ...
+    def setNumTopFeatures(self: P, value: int) -> P: ...
+    def setPercentile(self: P, value: float) -> P: ...
+    def setFpr(self: P, value: float) -> P: ...
+    def setFdr(self: P, value: float) -> P: ...
+    def setFwe(self: P, value: float) -> P: ...
+    def setFeaturesCol(self: P, value: str) -> P: ...
+    def setOutputCol(self: P, value: str) -> P: ...
+    def setLabelCol(self: P, value: str) -> P: ...
+
+class _SelectorModel(JavaModel, _SelectorParams):
+    def setFeaturesCol(self: P, value: str) -> P: ...
+    def setOutputCol(self: P, value: str) -> P: ...
+    @property
+    def selectedFeatures(self) -> List[int]: ...
+
+class ChiSqSelector(
+    _Selector[ChiSqSelectorModel],
+    JavaMLReadable[ChiSqSelector],
+    JavaMLWritable,
+):
+    def __init__(
+        self,
+        *,
+        numTopFeatures: int = ...,
+        featuresCol: str = ...,
+        outputCol: Optional[str] = ...,
+        labelCol: str = ...,
+        selectorType: str = ...,
+        percentile: float = ...,
+        fpr: float = ...,
+        fdr: float = ...,
+        fwe: float = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        numTopFeatures: int = ...,
+        featuresCol: str = ...,
+        outputCol: Optional[str] = ...,
+        labelCol: str = ...,
+        selectorType: str = ...,
+        percentile: float = ...,
+        fpr: float = ...,
+        fdr: float = ...,
+        fwe: float = ...
+    ) -> ChiSqSelector: ...
+    def setSelectorType(self, value: str) -> ChiSqSelector: ...
+    def setNumTopFeatures(self, value: int) -> ChiSqSelector: ...
+    def setPercentile(self, value: float) -> ChiSqSelector: ...
+    def setFpr(self, value: float) -> ChiSqSelector: ...
+    def setFdr(self, value: float) -> ChiSqSelector: ...
+    def setFwe(self, value: float) -> ChiSqSelector: ...
+    def setFeaturesCol(self, value: str) -> ChiSqSelector: ...
+    def setOutputCol(self, value: str) -> ChiSqSelector: ...
+    def setLabelCol(self, value: str) -> ChiSqSelector: ...
+
+class ChiSqSelectorModel(
+    _SelectorModel, JavaMLReadable[ChiSqSelectorModel], JavaMLWritable
+):
+    def setFeaturesCol(self, value: str) -> ChiSqSelectorModel: ...
+    def setOutputCol(self, value: str) -> ChiSqSelectorModel: ...
+    @property
+    def selectedFeatures(self) -> List[int]: ...
+
+class VectorSizeHint(
+    JavaTransformer,
+    HasInputCol,
+    HasHandleInvalid,
+    JavaMLReadable[VectorSizeHint],
+    JavaMLWritable,
+):
+    size: Param[int]
+    handleInvalid: Param[str]
+    def __init__(
+        self,
+        *,
+        inputCol: Optional[str] = ...,
+        size: Optional[int] = ...,
+        handleInvalid: str = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        inputCol: Optional[str] = ...,
+        size: Optional[int] = ...,
+        handleInvalid: str = ...
+    ) -> VectorSizeHint: ...
+    def setSize(self, value: int) -> VectorSizeHint: ...
+    def getSize(self) -> int: ...
+    def setInputCol(self, value: str) -> VectorSizeHint: ...
+    def setHandleInvalid(self, value: str) -> VectorSizeHint: ...
+
+class _VarianceThresholdSelectorParams(HasFeaturesCol, HasOutputCol):
+    varianceThreshold: Param[float] = ...
+    def getVarianceThreshold(self) -> float: ...
+
+class VarianceThresholdSelector(
+    JavaEstimator[VarianceThresholdSelectorModel],
+    _VarianceThresholdSelectorParams,
+    JavaMLReadable[VarianceThresholdSelector],
+    JavaMLWritable,
+):
+    def __init__(
+        self,
+        featuresCol: str = ...,
+        outputCol: Optional[str] = ...,
+        varianceThreshold: float = ...,
+    ) -> None: ...
+    def setParams(
+        self,
+        featuresCol: str = ...,
+        outputCol: Optional[str] = ...,
+        varianceThreshold: float = ...,
+    ) -> VarianceThresholdSelector: ...
+    def setVarianceThreshold(self, value: float) -> VarianceThresholdSelector: ...
+    def setFeaturesCol(self, value: str) -> VarianceThresholdSelector: ...
+    def setOutputCol(self, value: str) -> VarianceThresholdSelector: ...
+
+class VarianceThresholdSelectorModel(
+    JavaModel,
+    _VarianceThresholdSelectorParams,
+    JavaMLReadable[VarianceThresholdSelectorModel],
+    JavaMLWritable,
+):
+    def setFeaturesCol(self, value: str) -> VarianceThresholdSelectorModel: ...
+    def setOutputCol(self, value: str) -> VarianceThresholdSelectorModel: ...
+    @property
+    def selectedFeatures(self) -> List[int]: ...
+
+class _UnivariateFeatureSelectorParams(HasFeaturesCol, HasOutputCol, HasLabelCol):
+    featureType: Param[str] = ...
+    labelType: Param[str] = ...
+    selectionMode: Param[str] = ...
+    selectionThreshold: Param[float] = ...
+    def __init__(self, *args: Any): ...
+    def getFeatureType(self) -> str: ...
+    def getLabelType(self) -> str: ...
+    def getSelectionMode(self) -> str: ...
+    def getSelectionThreshold(self) -> float: ...
+
+class UnivariateFeatureSelector(
+    JavaEstimator[UnivariateFeatureSelectorModel],
+    _UnivariateFeatureSelectorParams,
+    JavaMLReadable[UnivariateFeatureSelector],
+    JavaMLWritable,
+):
+    def __init__(
+        self,
+        *,
+        featuresCol: str = ...,
+        outputCol: Optional[str] = ...,
+        labelCol: str = ...,
+        selectionMode: str = ...,
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        featuresCol: str = ...,
+        outputCol: Optional[str] = ...,
+        labelCol: str = ...,
+        selectionMode: str = ...,
+    ) -> UnivariateFeatureSelector: ...
+    def setFeatureType(self, value: str) -> UnivariateFeatureSelector: ...
+    def setLabelType(self, value: str) -> UnivariateFeatureSelector: ...
+    def setSelectionMode(self, value: str) -> UnivariateFeatureSelector: ...
+    def setSelectionThreshold(self, value: float) -> UnivariateFeatureSelector: ...
+    def setFeaturesCol(self, value: str) -> UnivariateFeatureSelector: ...
+    def setOutputCol(self, value: str) -> UnivariateFeatureSelector: ...
+    def setLabelCol(self, value: str) -> UnivariateFeatureSelector: ...
+
+class UnivariateFeatureSelectorModel(
+    JavaModel,
+    _UnivariateFeatureSelectorParams,
+    JavaMLReadable[UnivariateFeatureSelectorModel],
+    JavaMLWritable,
+):
+    def setFeaturesCol(self, value: str) -> UnivariateFeatureSelectorModel: ...
+    def setOutputCol(self, value: str) -> UnivariateFeatureSelectorModel: ...
+    @property
+    def selectedFeatures(self) -> List[int]: ...
diff --git a/python/pyspark/ml/fpm.py b/python/pyspark/ml/fpm.py
index 7d933daf9e032..7c02199370eec 100644
--- a/python/pyspark/ml/fpm.py
+++ b/python/pyspark/ml/fpm.py
@@ -15,12 +15,13 @@
 # limitations under the License.
 #
 
+import sys
+
 from pyspark import keyword_only, since
-from pyspark.rdd import ignore_unicode_prefix
 from pyspark.sql import DataFrame
-from pyspark.ml.util import *
+from pyspark.ml.util import JavaMLWritable, JavaMLReadable
 from pyspark.ml.wrapper import JavaEstimator, JavaModel, JavaParams
-from pyspark.ml.param.shared import *
+from pyspark.ml.param.shared import HasPredictionCol, Param, TypeConverters, Params
 
 __all__ = ["FPGrowth", "FPGrowthModel", "PrefixSpan"]
 
@@ -56,6 +57,11 @@ class _FPGrowthParams(HasPredictionCol):
         "but will affect the association rules generation.",
         typeConverter=TypeConverters.toFloat)
 
+    def __init__(self, *args):
+        super(_FPGrowthParams, self).__init__(*args)
+        self._setDefault(minSupport=0.3, minConfidence=0.8,
+                         itemsCol="items", predictionCol="prediction")
+
     def getItemsCol(self):
         """
         Gets the value of itemsCol or its default value.
@@ -132,21 +138,39 @@ def associationRules(self):
         return self._call_java("associationRules")
 
 
-@ignore_unicode_prefix
 class FPGrowth(JavaEstimator, _FPGrowthParams, JavaMLWritable, JavaMLReadable):
     r"""
-    A parallel FP-growth algorithm to mine frequent itemsets. The algorithm is described in
-    Li et al., PFP: Parallel FP-Growth for Query Recommendation [LI2008]_.
+    A parallel FP-growth algorithm to mine frequent itemsets.
+
+    .. versionadded:: 2.2.0
+
+    Notes
+    -----
+
+    The algorithm is described in
+    Li et al., PFP: Parallel FP-Growth for Query Recommendation [1]_.
     PFP distributes computation in such a way that each worker executes an
     independent group of mining tasks. The FP-Growth algorithm is described in
-    Han et al., Mining frequent patterns without candidate generation [HAN2000]_
+    Han et al., Mining frequent patterns without candidate generation [2]_
+
+    NULL values in the feature column are ignored during `fit()`.
 
-    .. [LI2008] https://doi.org/10.1145/1454008.1454027
-    .. [HAN2000] https://doi.org/10.1145/335191.335372
+    Internally `transform` `collects` and `broadcasts` association rules.
 
-    .. note:: null values in the feature column are ignored during fit().
-    .. note:: Internally `transform` `collects` and `broadcasts` association rules.
 
+    .. [1] Haoyuan Li, Yi Wang, Dong Zhang, Ming Zhang, and Edward Y. Chang. 2008.
+        Pfp: parallel fp-growth for query recommendation.
+        In Proceedings of the 2008 ACM conference on Recommender systems (RecSys '08).
+        Association for Computing Machinery, New York, NY, USA, 107-114.
+        DOI: https://doi.org/10.1145/1454008.1454027
+    .. [2] Jiawei Han, Jian Pei, and Yiwen Yin. 2000.
+        Mining frequent patterns without candidate generation.
+        SIGMOD Rec. 29, 2 (June 2000), 1-12.
+        DOI: https://doi.org/10.1145/335191.335372
+
+
+    Examples
+    --------
     >>> from pyspark.sql.functions import split
     >>> data = (spark.read
     ...     .text("data/mllib/sample_fpgrowth.txt")
@@ -180,43 +204,44 @@ class FPGrowth(JavaEstimator, _FPGrowthParams, JavaMLWritable, JavaMLReadable):
     only showing top 5 rows
     ...
     >>> fpm.associationRules.show(5)
-    +----------+----------+----------+----+
-    |antecedent|consequent|confidence|lift|
-    +----------+----------+----------+----+
-    |    [t, s]|       [y]|       1.0| 2.0|
-    |    [t, s]|       [x]|       1.0| 1.5|
-    |    [t, s]|       [z]|       1.0| 1.2|
-    |       [p]|       [r]|       1.0| 2.0|
-    |       [p]|       [z]|       1.0| 1.2|
-    +----------+----------+----------+----+
+    +----------+----------+----------+----+------------------+
+    |antecedent|consequent|confidence|lift|           support|
+    +----------+----------+----------+----+------------------+
+    |    [t, s]|       [y]|       1.0| 2.0|0.3333333333333333|
+    |    [t, s]|       [x]|       1.0| 1.5|0.3333333333333333|
+    |    [t, s]|       [z]|       1.0| 1.2|0.3333333333333333|
+    |       [p]|       [r]|       1.0| 2.0|0.3333333333333333|
+    |       [p]|       [z]|       1.0| 1.2|0.3333333333333333|
+    +----------+----------+----------+----+------------------+
     only showing top 5 rows
     ...
     >>> new_data = spark.createDataFrame([(["t", "s"], )], ["items"])
     >>> sorted(fpm.transform(new_data).first().newPrediction)
-    [u'x', u'y', u'z']
-
-    .. versionadded:: 2.2.0
+    ['x', 'y', 'z']
+    >>> model_path = temp_path + "/fpm_model"
+    >>> fpm.save(model_path)
+    >>> model2 = FPGrowthModel.load(model_path)
+    >>> fpm.transform(data).take(1) == model2.transform(data).take(1)
+    True
     """
     @keyword_only
-    def __init__(self, minSupport=0.3, minConfidence=0.8, itemsCol="items",
+    def __init__(self, *, minSupport=0.3, minConfidence=0.8, itemsCol="items",
                  predictionCol="prediction", numPartitions=None):
         """
-        __init__(self, minSupport=0.3, minConfidence=0.8, itemsCol="items", \
+        __init__(self, \\*, minSupport=0.3, minConfidence=0.8, itemsCol="items", \
                  predictionCol="prediction", numPartitions=None)
         """
         super(FPGrowth, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.fpm.FPGrowth", self.uid)
-        self._setDefault(minSupport=0.3, minConfidence=0.8,
-                         itemsCol="items", predictionCol="prediction")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("2.2.0")
-    def setParams(self, minSupport=0.3, minConfidence=0.8, itemsCol="items",
+    def setParams(self, *, minSupport=0.3, minConfidence=0.8, itemsCol="items",
                   predictionCol="prediction", numPartitions=None):
         """
-        setParams(self, minSupport=0.3, minConfidence=0.8, itemsCol="items", \
+        setParams(self, \\*, minSupport=0.3, minConfidence=0.8, itemsCol="items", \
                   predictionCol="prediction", numPartitions=None)
         """
         kwargs = self._input_kwargs
@@ -261,13 +286,19 @@ class PrefixSpan(JavaParams):
     A parallel PrefixSpan algorithm to mine frequent sequential patterns.
     The PrefixSpan algorithm is described in J. Pei, et al., PrefixSpan: Mining Sequential Patterns
     Efficiently by Prefix-Projected Pattern Growth
-    (see <a href="https://doi.org/10.1109/ICDE.2001.914830">here</a>).
+    (see `here <https://doi.org/10.1109/ICDE.2001.914830">`_).
     This class is not yet an Estimator/Transformer, use :py:func:`findFrequentSequentialPatterns`
     method to run the PrefixSpan algorithm.
 
-    @see <a href="https://en.wikipedia.org/wiki/Sequential_Pattern_Mining">Sequential Pattern Mining
-    (Wikipedia)</a>
+    .. versionadded:: 2.4.0
+
+    Notes
+    -----
+    See `Sequential Pattern Mining (Wikipedia) \
+      <https://en.wikipedia.org/wiki/Sequential_Pattern_Mining>`_
 
+    Examples
+    --------
     >>> from pyspark.ml.fpm import PrefixSpan
     >>> from pyspark.sql import Row
     >>> df = sc.parallelize([Row(sequence=[[1, 2], [3]]),
@@ -294,8 +325,6 @@ class PrefixSpan(JavaParams):
     |[[3]]     |2   |
     +----------+----+
     ...
-
-    .. versionadded:: 2.4.0
     """
 
     minSupport = Param(Params._dummy(), "minSupport", "The minimal support level of the " +
@@ -320,10 +349,10 @@ class PrefixSpan(JavaParams):
                         typeConverter=TypeConverters.toString)
 
     @keyword_only
-    def __init__(self, minSupport=0.1, maxPatternLength=10, maxLocalProjDBSize=32000000,
+    def __init__(self, *, minSupport=0.1, maxPatternLength=10, maxLocalProjDBSize=32000000,
                  sequenceCol="sequence"):
         """
-        __init__(self, minSupport=0.1, maxPatternLength=10, maxLocalProjDBSize=32000000, \
+        __init__(self, \\*, minSupport=0.1, maxPatternLength=10, maxLocalProjDBSize=32000000, \
                  sequenceCol="sequence")
         """
         super(PrefixSpan, self).__init__()
@@ -335,10 +364,10 @@ def __init__(self, minSupport=0.1, maxPatternLength=10, maxLocalProjDBSize=32000
 
     @keyword_only
     @since("2.4.0")
-    def setParams(self, minSupport=0.1, maxPatternLength=10, maxLocalProjDBSize=32000000,
+    def setParams(self, *, minSupport=0.1, maxPatternLength=10, maxLocalProjDBSize=32000000,
                   sequenceCol="sequence"):
         """
-        setParams(self, minSupport=0.1, maxPatternLength=10, maxLocalProjDBSize=32000000, \
+        setParams(self, \\*, minSupport=0.1, maxPatternLength=10, maxLocalProjDBSize=32000000, \
                   sequenceCol="sequence")
         """
         kwargs = self._input_kwargs
@@ -400,19 +429,26 @@ def getSequenceCol(self):
         """
         return self.getOrDefault(self.sequenceCol)
 
-    @since("2.4.0")
     def findFrequentSequentialPatterns(self, dataset):
         """
         Finds the complete set of frequent sequential patterns in the input sequences of itemsets.
 
-        :param dataset: A dataframe containing a sequence column which is
-                        `ArrayType(ArrayType(T))` type, T is the item type for the input dataset.
-        :return: A `DataFrame` that contains columns of sequence and corresponding frequency.
-                 The schema of it will be:
-                 - `sequence: ArrayType(ArrayType(T))` (T is the item type)
-                 - `freq: Long`
-
         .. versionadded:: 2.4.0
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            A dataframe containing a sequence column which is
+            `ArrayType(ArrayType(T))` type, T is the item type for the input dataset.
+
+        Returns
+        -------
+        :py:class:`pyspark.sql.DataFrame`
+            A `DataFrame` that contains columns of sequence and corresponding frequency.
+            The schema of it will be:
+
+            - `sequence: ArrayType(ArrayType(T))` (T is the item type)
+            - `freq: Long`
         """
 
         self._transfer_params_to_java()
diff --git a/python/pyspark/ml/fpm.pyi b/python/pyspark/ml/fpm.pyi
new file mode 100644
index 0000000000000..7cc304a2ffa39
--- /dev/null
+++ b/python/pyspark/ml/fpm.pyi
@@ -0,0 +1,109 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Any, Optional
+
+from pyspark.ml.util import JavaMLReadable, JavaMLWritable
+from pyspark.ml.wrapper import JavaEstimator, JavaParams, JavaModel
+from pyspark.ml.param.shared import HasPredictionCol
+from pyspark.sql.dataframe import DataFrame
+
+from pyspark.ml.param import Param
+
+class _FPGrowthParams(HasPredictionCol):
+    itemsCol: Param[str]
+    minSupport: Param[float]
+    numPartitions: Param[int]
+    minConfidence: Param[float]
+    def __init__(self, *args: Any): ...
+    def getItemsCol(self) -> str: ...
+    def getMinSupport(self) -> float: ...
+    def getNumPartitions(self) -> int: ...
+    def getMinConfidence(self) -> float: ...
+
+class FPGrowthModel(
+    JavaModel, _FPGrowthParams, JavaMLWritable, JavaMLReadable[FPGrowthModel]
+):
+    def setItemsCol(self, value: str) -> FPGrowthModel: ...
+    def setMinConfidence(self, value: float) -> FPGrowthModel: ...
+    def setPredictionCol(self, value: str) -> FPGrowthModel: ...
+    @property
+    def freqItemsets(self) -> DataFrame: ...
+    @property
+    def associationRules(self) -> DataFrame: ...
+
+class FPGrowth(
+    JavaEstimator[FPGrowthModel],
+    _FPGrowthParams,
+    JavaMLWritable,
+    JavaMLReadable[FPGrowth],
+):
+    def __init__(
+        self,
+        *,
+        minSupport: float = ...,
+        minConfidence: float = ...,
+        itemsCol: str = ...,
+        predictionCol: str = ...,
+        numPartitions: Optional[int] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        minSupport: float = ...,
+        minConfidence: float = ...,
+        itemsCol: str = ...,
+        predictionCol: str = ...,
+        numPartitions: Optional[int] = ...
+    ) -> FPGrowth: ...
+    def setItemsCol(self, value: str) -> FPGrowth: ...
+    def setMinSupport(self, value: float) -> FPGrowth: ...
+    def setNumPartitions(self, value: int) -> FPGrowth: ...
+    def setMinConfidence(self, value: float) -> FPGrowth: ...
+    def setPredictionCol(self, value: str) -> FPGrowth: ...
+
+class PrefixSpan(JavaParams):
+    minSupport: Param[float]
+    maxPatternLength: Param[int]
+    maxLocalProjDBSize: Param[int]
+    sequenceCol: Param[str]
+    def __init__(
+        self,
+        *,
+        minSupport: float = ...,
+        maxPatternLength: int = ...,
+        maxLocalProjDBSize: int = ...,
+        sequenceCol: str = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        minSupport: float = ...,
+        maxPatternLength: int = ...,
+        maxLocalProjDBSize: int = ...,
+        sequenceCol: str = ...
+    ) -> PrefixSpan: ...
+    def setMinSupport(self, value: float) -> PrefixSpan: ...
+    def getMinSupport(self) -> float: ...
+    def setMaxPatternLength(self, value: int) -> PrefixSpan: ...
+    def getMaxPatternLength(self) -> int: ...
+    def setMaxLocalProjDBSize(self, value: int) -> PrefixSpan: ...
+    def getMaxLocalProjDBSize(self) -> int: ...
+    def setSequenceCol(self, value: str) -> PrefixSpan: ...
+    def getSequenceCol(self) -> str: ...
+    def findFrequentSequentialPatterns(self, dataset: DataFrame) -> DataFrame: ...
diff --git a/python/pyspark/ml/functions.py b/python/pyspark/ml/functions.py
index 65b0558b282a4..1eadbd694210e 100644
--- a/python/pyspark/ml/functions.py
+++ b/python/pyspark/ml/functions.py
@@ -15,21 +15,30 @@
 # limitations under the License.
 #
 
-from pyspark import since, SparkContext
+from pyspark import SparkContext
 from pyspark.sql.column import Column, _to_java_column
 
 
-@since("3.0.0")
 def vector_to_array(col, dtype="float64"):
     """
     Converts a column of MLlib sparse/dense vectors into a column of dense arrays.
 
-    :param col: A string of the column name or a Column
-    :param dtype: The data type of the output array. Valid values: "float64" or "float32".
-    :return: The converted column of dense arrays.
-
     .. versionadded:: 3.0.0
 
+    Parameters
+    ----------
+    col : :py:class:`pyspark.sql.Column` or str
+        Input column
+    dtype : str, optional
+        The data type of the output array. Valid values: "float64" or "float32".
+
+    Returns
+    -------
+    :py:class:`pyspark.sql.Column`
+        The converted column of dense arrays.
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> from pyspark.ml.functions import vector_to_array
     >>> from pyspark.mllib.linalg import Vectors as OldVectors
@@ -60,6 +69,41 @@ def vector_to_array(col, dtype="float64"):
         sc._jvm.org.apache.spark.ml.functions.vector_to_array(_to_java_column(col), dtype))
 
 
+def array_to_vector(col):
+    """
+    Converts a column of array of numeric type into a column of pyspark.ml.linalg.DenseVector
+    instances
+
+    .. versionadded:: 3.1.0
+
+    Parameters
+    ----------
+    col : :py:class:`pyspark.sql.Column` or str
+        Input column
+
+    Returns
+    -------
+    :py:class:`pyspark.sql.Column`
+        The converted column of dense vectors.
+
+    Examples
+    --------
+    >>> from pyspark.ml.functions import array_to_vector
+    >>> df1 = spark.createDataFrame([([1.5, 2.5],),], schema='v1 array<double>')
+    >>> df1.select(array_to_vector('v1').alias('vec1')).collect()
+    [Row(vec1=DenseVector([1.5, 2.5]))]
+    >>> df2 = spark.createDataFrame([([1.5, 3.5],),], schema='v1 array<float>')
+    >>> df2.select(array_to_vector('v1').alias('vec1')).collect()
+    [Row(vec1=DenseVector([1.5, 3.5]))]
+    >>> df3 = spark.createDataFrame([([1, 3],),], schema='v1 array<int>')
+    >>> df3.select(array_to_vector('v1').alias('vec1')).collect()
+    [Row(vec1=DenseVector([1.0, 3.0]))]
+    """
+    sc = SparkContext._active_spark_context
+    return Column(
+        sc._jvm.org.apache.spark.ml.functions.array_to_vector(_to_java_column(col)))
+
+
 def _test():
     import doctest
     from pyspark.sql import SparkSession
diff --git a/python/pyspark/ml/functions.pyi b/python/pyspark/ml/functions.pyi
new file mode 100644
index 0000000000000..12b44fc63b5b7
--- /dev/null
+++ b/python/pyspark/ml/functions.pyi
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from pyspark import SparkContext as SparkContext, since as since  # noqa: F401
+from pyspark.sql.column import Column as Column
+
+def vector_to_array(col: Column) -> Column: ...
+
+def array_to_vector(col: Column) -> Column: ...
diff --git a/python/pyspark/ml/image.py b/python/pyspark/ml/image.py
index 4fb1036fbab89..728e9a356ed36 100644
--- a/python/pyspark/ml/image.py
+++ b/python/pyspark/ml/image.py
@@ -25,14 +25,13 @@
 """
 
 import sys
-import warnings
 
 import numpy as np
 from distutils.version import LooseVersion
 
 from pyspark import SparkContext
 from pyspark.sql.types import Row, _create_row, _parse_datatype_json_string
-from pyspark.sql import DataFrame, SparkSession
+from pyspark.sql import SparkSession
 
 __all__ = ["ImageSchema"]
 
@@ -56,8 +55,11 @@ def imageSchema(self):
         """
         Returns the image schema.
 
-        :return: a :class:`StructType` with a single column of images
-               named "image" (nullable) and having the same type returned by :meth:`columnSchema`.
+        Returns
+        -------
+        :class:`StructType`
+            with a single column of images named "image" (nullable)
+            and having the same type returned by :meth:`columnSchema`.
 
         .. versionadded:: 2.3.0
         """
@@ -73,7 +75,10 @@ def ocvTypes(self):
         """
         Returns the OpenCV type mapping supported.
 
-        :return: a dictionary containing the OpenCV type mapping supported.
+        Returns
+        -------
+        dict
+            a dictionary containing the OpenCV type mapping supported.
 
         .. versionadded:: 2.3.0
         """
@@ -88,7 +93,10 @@ def columnSchema(self):
         """
         Returns the schema for the image column.
 
-        :return: a :class:`StructType` for image column,
+        Returns
+        -------
+        :class:`StructType`
+            a schema for image column,
             ``struct<origin:string, height:int, width:int, nChannels:int, mode:int, data:binary>``.
 
         .. versionadded:: 2.4.0
@@ -105,7 +113,10 @@ def imageFields(self):
         """
         Returns field names of image columns.
 
-        :return: a list of field names.
+        Returns
+        -------
+        list
+            a list of field names.
 
         .. versionadded:: 2.3.0
         """
@@ -133,9 +144,16 @@ def toNDArray(self, image):
         """
         Converts an image to an array with metadata.
 
-        :param `Row` image: A row that contains the image to be converted. It should
+        Parameters
+        ----------
+        image : :class:`Row`
+            image: A row that contains the image to be converted. It should
             have the attributes specified in `ImageSchema.imageSchema`.
-        :return: a `numpy.ndarray` that is an image.
+
+        Returns
+        -------
+        :class:`numpy.ndarray`
+            that is an image.
 
         .. versionadded:: 2.3.0
         """
@@ -163,9 +181,17 @@ def toImage(self, array, origin=""):
         """
         Converts an array with metadata to a two-dimensional image.
 
-        :param `numpy.ndarray` array: The array to convert to image.
-        :param str origin: Path to the image, optional.
-        :return: a :class:`Row` that is a two dimensional image.
+        Parameters
+        ----------
+        array : :class:`numpy.ndarray`
+            The array to convert to image.
+        origin : str
+            Path to the image, optional.
+
+        Returns
+        -------
+        :class:`Row`
+            that is a two dimensional image.
 
         .. versionadded:: 2.3.0
         """
diff --git a/python/pyspark/ml/image.pyi b/python/pyspark/ml/image.pyi
new file mode 100644
index 0000000000000..9ff3a8817aadd
--- /dev/null
+++ b/python/pyspark/ml/image.pyi
@@ -0,0 +1,40 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Dict, List
+
+from pyspark.sql.types import Row, StructType
+
+from numpy import ndarray  # type: ignore[import]
+
+class _ImageSchema:
+    def __init__(self) -> None: ...
+    @property
+    def imageSchema(self) -> StructType: ...
+    @property
+    def ocvTypes(self) -> Dict[str, int]: ...
+    @property
+    def columnSchema(self) -> StructType: ...
+    @property
+    def imageFields(self) -> List[str]: ...
+    @property
+    def undefinedImageType(self) -> str: ...
+    def toNDArray(self, image: Row) -> ndarray: ...
+    def toImage(self, array: ndarray, origin: str = ...) -> Row: ...
+
+ImageSchema: _ImageSchema
diff --git a/python/pyspark/ml/linalg/__init__.py b/python/pyspark/ml/linalg/__init__.py
index a79d5e5dcbb16..221b8f4204683 100644
--- a/python/pyspark/ml/linalg/__init__.py
+++ b/python/pyspark/ml/linalg/__init__.py
@@ -27,18 +27,8 @@
 import array
 import struct
 
-if sys.version >= '3':
-    basestring = str
-    xrange = range
-    import copyreg as copy_reg
-    long = int
-else:
-    from itertools import izip as zip
-    import copy_reg
-
 import numpy as np
 
-from pyspark import since
 from pyspark.sql.types import UserDefinedType, StructField, StructType, ArrayType, DoubleType, \
     IntegerType, ByteType, BooleanType
 
@@ -47,13 +37,6 @@
            'Matrix', 'DenseMatrix', 'SparseMatrix', 'Matrices']
 
 
-if sys.version_info[:2] == (2, 7):
-    # speed up pickling array in Python 2.7
-    def fast_pickle_array(ar):
-        return array.array, (ar.typecode, ar.tostring())
-    copy_reg.pickle(array.array, fast_pickle_array)
-
-
 # Check whether we have SciPy. MLlib works without it too, but if we have it, some methods,
 # such as _dot and _serialize_double_vector, start to support scipy.sparse matrices.
 
@@ -68,7 +51,7 @@ def fast_pickle_array(ar):
 def _convert_to_vector(l):
     if isinstance(l, Vector):
         return l
-    elif type(l) in (array.array, np.array, np.ndarray, list, tuple, xrange):
+    elif type(l) in (array.array, np.array, np.ndarray, list, tuple, range):
         return DenseVector(l)
     elif _have_scipy and scipy.sparse.issparse(l):
         assert l.shape[1] == 1, "Expected column vector"
@@ -85,6 +68,8 @@ def _vector_size(v):
     """
     Returns the size of the vector.
 
+    Examples
+    --------
     >>> _vector_size([1., 2., 3.])
     3
     >>> _vector_size((1., 2., 3.))
@@ -102,7 +87,7 @@ def _vector_size(v):
     """
     if isinstance(v, Vector):
         return len(v)
-    elif type(v) in (array.array, list, tuple, xrange):
+    elif type(v) in (array.array, list, tuple, range):
         return len(v)
     elif type(v) == np.ndarray:
         if v.ndim == 1 or (v.ndim == 2 and v.shape[1] == 1):
@@ -256,6 +241,8 @@ class DenseVector(Vector):
     storage and arithmetics will be delegated to the underlying numpy
     array.
 
+    Examples
+    --------
     >>> v = Vectors.dense([1.0, 2.0])
     >>> u = Vectors.dense([3.0, 4.0])
     >>> v + u
@@ -295,6 +282,8 @@ def norm(self, p):
         """
         Calculates the norm of a DenseVector.
 
+        Examples
+        --------
         >>> a = DenseVector([0, -1, 2, -3])
         >>> a.norm(2)
         3.7...
@@ -310,6 +299,8 @@ def dot(self, other):
         and a target NumPy array that is either 1- or 2-dimensional.
         Equivalent to calling numpy.dot of the two vectors.
 
+        Examples
+        --------
         >>> dense = DenseVector(array.array('d', [1., 2.]))
         >>> dense.dot(dense)
         5.0
@@ -350,6 +341,8 @@ def squared_distance(self, other):
         """
         Squared distance of two Vectors.
 
+        Examples
+        --------
         >>> dense1 = DenseVector(array.array('d', [1., 2.]))
         >>> dense1.squared_distance(dense1)
         0.0
@@ -415,7 +408,7 @@ def __eq__(self, other):
         elif isinstance(other, SparseVector):
             if len(self) != other.size:
                 return False
-            return Vectors._equals(list(xrange(len(self))), self.array, other.indices, other.values)
+            return Vectors._equals(list(range(len(self))), self.array, other.indices, other.values)
         return False
 
     def __ne__(self, other):
@@ -473,12 +466,18 @@ def __init__(self, size, *args):
         (index, value) pairs, or two separate arrays of indices and
         values (sorted by index).
 
-        :param size: Size of the vector.
-        :param args: Active entries, as a dictionary {index: value, ...},
-          a list of tuples [(index, value), ...], or a list of strictly
-          increasing indices and a list of corresponding values [index, ...],
-          [value, ...]. Inactive entries are treated as zeros.
-
+        Examples
+        --------
+        size : int
+            Size of the vector.
+        args
+            Active entries, as a dictionary {index: value, ...},
+            a list of tuples [(index, value), ...], or a list of strictly
+            increasing indices and a list of corresponding values [index, ...],
+            [value, ...]. Inactive entries are treated as zeros.
+
+        Examples
+        --------
         >>> SparseVector(4, {1: 1.0, 3: 5.5})
         SparseVector(4, {1: 1.0, 3: 5.5})
         >>> SparseVector(4, [(1, 1.0), (3, 5.5)])
@@ -520,7 +519,7 @@ def __init__(self, size, *args):
                 self.indices = np.array(args[0], dtype=np.int32)
                 self.values = np.array(args[1], dtype=np.float64)
             assert len(self.indices) == len(self.values), "index and value arrays not same length"
-            for i in xrange(len(self.indices) - 1):
+            for i in range(len(self.indices) - 1):
                 if self.indices[i] >= self.indices[i + 1]:
                     raise TypeError(
                         "Indices %s and %s are not strictly increasing"
@@ -543,6 +542,8 @@ def norm(self, p):
         """
         Calculates the norm of a SparseVector.
 
+        Examples
+        --------
         >>> a = SparseVector(4, [0, 1], [3., -4.])
         >>> a.norm(1)
         7.0
@@ -560,6 +561,8 @@ def dot(self, other):
         """
         Dot product with a SparseVector or 1- or 2-dimensional Numpy array.
 
+        Examples
+        --------
         >>> a = SparseVector(4, [1, 3], [3.0, 4.0])
         >>> a.dot(a)
         25.0
@@ -616,6 +619,8 @@ def squared_distance(self, other):
         """
         Squared distance from a SparseVector or 1-dimensional NumPy array.
 
+        Examples
+        --------
         >>> a = SparseVector(4, [1, 3], [3.0, 4.0])
         >>> a.squared_distance(a)
         0.0
@@ -699,7 +704,7 @@ def __repr__(self):
         inds = self.indices
         vals = self.values
         entries = ", ".join(["{0}: {1}".format(inds[i], _format_float(vals[i]))
-                             for i in xrange(len(inds))])
+                             for i in range(len(inds))])
         return "SparseVector({0}, {{{1}}})".format(self.size, entries)
 
     def __eq__(self, other):
@@ -709,7 +714,7 @@ def __eq__(self, other):
         elif isinstance(other, DenseVector):
             if self.size != len(other):
                 return False
-            return Vectors._equals(self.indices, self.values, list(xrange(len(other))), other.array)
+            return Vectors._equals(self.indices, self.values, list(range(len(other))), other.array)
         return False
 
     def __getitem__(self, index):
@@ -755,10 +760,12 @@ class Vectors(object):
     """
     Factory methods for working with vectors.
 
-    .. note:: Dense vectors are simply represented as NumPy array objects,
-        so there is no need to covert them for use in MLlib. For sparse vectors,
-        the factory methods in this class create an MLlib-compatible type, or users
-        can pass in SciPy's `scipy.sparse` column vectors.
+    Notes
+    -----
+    Dense vectors are simply represented as NumPy array objects,
+    so there is no need to covert them for use in MLlib. For sparse vectors,
+    the factory methods in this class create an MLlib-compatible type, or users
+    can pass in SciPy's `scipy.sparse` column vectors.
     """
 
     @staticmethod
@@ -768,10 +775,16 @@ def sparse(size, *args):
         (index, value) pairs, or two separate arrays of indices and
         values (sorted by index).
 
-        :param size: Size of the vector.
-        :param args: Non-zero entries, as a dictionary, list of tuples,
-                     or two sorted lists containing indices and values.
+        Parameters
+        ----------
+        size : int
+            Size of the vector.
+        args
+            Non-zero entries, as a dictionary, list of tuples,
+            or two sorted lists containing indices and values.
 
+        Examples
+        --------
         >>> Vectors.sparse(4, {1: 1.0, 3: 5.5})
         SparseVector(4, {1: 1.0, 3: 5.5})
         >>> Vectors.sparse(4, [(1, 1.0), (3, 5.5)])
@@ -786,12 +799,14 @@ def dense(*elements):
         """
         Create a dense vector of 64-bit floats from a Python list or numbers.
 
+        Examples
+        --------
         >>> Vectors.dense([1, 2, 3])
         DenseVector([1.0, 2.0, 3.0])
         >>> Vectors.dense(1.0, 2.0)
         DenseVector([1.0, 2.0])
         """
-        if len(elements) == 1 and not isinstance(elements[0], (float, int, long)):
+        if len(elements) == 1 and not isinstance(elements[0], (float, int)):
             # it's list, numpy.array or other iterable object.
             elements = elements[0]
         return DenseVector(elements)
@@ -803,6 +818,8 @@ def squared_distance(v1, v2):
         a and b can be of type SparseVector, DenseVector, np.ndarray
         or array.array.
 
+        Examples
+        --------
         >>> a = Vectors.sparse(4, [(0, 1), (3, 4)])
         >>> b = Vectors.dense([2, 5, 4, 1])
         >>> a.squared_distance(b)
@@ -895,6 +912,8 @@ def __str__(self):
         """
         Pretty printing of a DenseMatrix
 
+        Examples
+        --------
         >>> dm = DenseMatrix(2, 2, range(4))
         >>> print(dm)
         DenseMatrix([[ 0.,  2.],
@@ -916,6 +935,8 @@ def __repr__(self):
         """
         Representation of a DenseMatrix
 
+        Examples
+        --------
         >>> dm = DenseMatrix(2, 2, range(4))
         >>> dm
         DenseMatrix(2, 2, [0.0, 1.0, 2.0, 3.0], False)
@@ -937,8 +958,10 @@ def __repr__(self):
 
     def toArray(self):
         """
-        Return a numpy.ndarray
+        Return a :py:class:`numpy.ndarray`
 
+        Examples
+        --------
         >>> m = DenseMatrix(2, 2, range(4))
         >>> m.toArray()
         array([[ 0.,  2.],
@@ -1015,6 +1038,8 @@ def __str__(self):
         """
         Pretty printing of a SparseMatrix
 
+        Examples
+        --------
         >>> sm1 = SparseMatrix(2, 2, [0, 2, 3], [0, 1, 1], [2, 3, 4])
         >>> print(sm1)
         2 X 2 CSCMatrix
@@ -1061,6 +1086,8 @@ def __repr__(self):
         """
         Representation of a SparseMatrix
 
+        Examples
+        --------
         >>> sm1 = SparseMatrix(2, 2, [0, 2, 3], [0, 1, 1], [2, 3, 4])
         >>> sm1
         SparseMatrix(2, 2, [0, 2, 3], [0, 1, 1], [2.0, 3.0, 4.0], False)
@@ -1124,7 +1151,7 @@ def toArray(self):
         Return a numpy.ndarray
         """
         A = np.zeros((self.numRows, self.numCols), dtype=np.float64, order='F')
-        for k in xrange(self.colPtrs.size - 1):
+        for k in range(self.colPtrs.size - 1):
             startptr = self.colPtrs[k]
             endptr = self.colPtrs[k + 1]
             if self.isTransposed:
diff --git a/python/pyspark/ml/linalg/__init__.pyi b/python/pyspark/ml/linalg/__init__.pyi
new file mode 100644
index 0000000000000..b4fba8823b678
--- /dev/null
+++ b/python/pyspark/ml/linalg/__init__.pyi
@@ -0,0 +1,251 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import overload
+from typing import Any, Dict, Iterable, List, NoReturn, Optional, Tuple, Type, Union
+
+from pyspark.ml import linalg as newlinalg  # noqa: F401
+from pyspark.sql.types import StructType, UserDefinedType
+
+from numpy import float64, ndarray  # type: ignore[import]
+
+class VectorUDT(UserDefinedType):
+    @classmethod
+    def sqlType(cls) -> StructType: ...
+    @classmethod
+    def module(cls) -> str: ...
+    @classmethod
+    def scalaUDT(cls) -> str: ...
+    def serialize(
+        self, obj: Vector
+    ) -> Tuple[int, Optional[int], Optional[List[int]], List[float]]: ...
+    def deserialize(self, datum: Any) -> Vector: ...
+    def simpleString(self) -> str: ...
+
+class MatrixUDT(UserDefinedType):
+    @classmethod
+    def sqlType(cls) -> StructType: ...
+    @classmethod
+    def module(cls) -> str: ...
+    @classmethod
+    def scalaUDT(cls) -> str: ...
+    def serialize(
+        self, obj: Matrix
+    ) -> Tuple[
+        int, int, int, Optional[List[int]], Optional[List[int]], List[float], bool
+    ]: ...
+    def deserialize(self, datum: Any) -> Matrix: ...
+    def simpleString(self) -> str: ...
+
+class Vector:
+    __UDT__: VectorUDT
+    def toArray(self) -> ndarray: ...
+
+class DenseVector(Vector):
+    array: ndarray
+    @overload
+    def __init__(self, *elements: float) -> None: ...
+    @overload
+    def __init__(self, __arr: bytes) -> None: ...
+    @overload
+    def __init__(self, __arr: Iterable[float]) -> None: ...
+    def __reduce__(self) -> Tuple[Type[DenseVector], bytes]: ...
+    def numNonzeros(self) -> int: ...
+    def norm(self, p: Union[float, str]) -> float64: ...
+    def dot(self, other: Iterable[float]) -> float64: ...
+    def squared_distance(self, other: Iterable[float]) -> float64: ...
+    def toArray(self) -> ndarray: ...
+    @property
+    def values(self) -> ndarray: ...
+    def __getitem__(self, item: int) -> float64: ...
+    def __len__(self) -> int: ...
+    def __eq__(self, other: Any) -> bool: ...
+    def __ne__(self, other: Any) -> bool: ...
+    def __hash__(self) -> int: ...
+    def __getattr__(self, item: str) -> Any: ...
+    def __neg__(self) -> DenseVector: ...
+    def __add__(self, other: Union[float, Iterable[float]]) -> DenseVector: ...
+    def __sub__(self, other: Union[float, Iterable[float]]) -> DenseVector: ...
+    def __mul__(self, other: Union[float, Iterable[float]]) -> DenseVector: ...
+    def __div__(self, other: Union[float, Iterable[float]]) -> DenseVector: ...
+    def __truediv__(self, other: Union[float, Iterable[float]]) -> DenseVector: ...
+    def __mod__(self, other: Union[float, Iterable[float]]) -> DenseVector: ...
+    def __radd__(self, other: Union[float, Iterable[float]]) -> DenseVector: ...
+    def __rsub__(self, other: Union[float, Iterable[float]]) -> DenseVector: ...
+    def __rmul__(self, other: Union[float, Iterable[float]]) -> DenseVector: ...
+    def __rdiv__(self, other: Union[float, Iterable[float]]) -> DenseVector: ...
+    def __rtruediv__(self, other: Union[float, Iterable[float]]) -> DenseVector: ...
+    def __rmod__(self, other: Union[float, Iterable[float]]) -> DenseVector: ...
+
+class SparseVector(Vector):
+    size: int
+    indices: ndarray
+    values: ndarray
+    @overload
+    def __init__(self, size: int, *args: Tuple[int, float]) -> None: ...
+    @overload
+    def __init__(self, size: int, __indices: bytes, __values: bytes) -> None: ...
+    @overload
+    def __init__(
+        self, size: int, __indices: Iterable[int], __values: Iterable[float]
+    ) -> None: ...
+    @overload
+    def __init__(self, size: int, __pairs: Iterable[Tuple[int, float]]) -> None: ...
+    @overload
+    def __init__(self, size: int, __map: Dict[int, float]) -> None: ...
+    def numNonzeros(self) -> int: ...
+    def norm(self, p: Union[float, str]) -> float64: ...
+    def __reduce__(self) -> Tuple[Type[SparseVector], Tuple[int, bytes, bytes]]: ...
+    def dot(self, other: Iterable[float]) -> float64: ...
+    def squared_distance(self, other: Iterable[float]) -> float64: ...
+    def toArray(self) -> ndarray: ...
+    def __len__(self) -> int: ...
+    def __eq__(self, other: Any) -> bool: ...
+    def __getitem__(self, index: int) -> float64: ...
+    def __ne__(self, other: Any) -> bool: ...
+    def __hash__(self) -> int: ...
+
+class Vectors:
+    @overload
+    @staticmethod
+    def sparse(size: int, *args: Tuple[int, float]) -> SparseVector: ...
+    @overload
+    @staticmethod
+    def sparse(size: int, __indices: bytes, __values: bytes) -> SparseVector: ...
+    @overload
+    @staticmethod
+    def sparse(
+        size: int, __indices: Iterable[int], __values: Iterable[float]
+    ) -> SparseVector: ...
+    @overload
+    @staticmethod
+    def sparse(size: int, __pairs: Iterable[Tuple[int, float]]) -> SparseVector: ...
+    @overload
+    @staticmethod
+    def sparse(size: int, __map: Dict[int, float]) -> SparseVector: ...
+    @overload
+    @staticmethod
+    def dense(*elements: float) -> DenseVector: ...
+    @overload
+    @staticmethod
+    def dense(__arr: bytes) -> DenseVector: ...
+    @overload
+    @staticmethod
+    def dense(__arr: Iterable[float]) -> DenseVector: ...
+    @staticmethod
+    def stringify(vector: Vector) -> str: ...
+    @staticmethod
+    def squared_distance(v1: Vector, v2: Vector) -> float64: ...
+    @staticmethod
+    def norm(vector: Vector, p: Union[float, str]) -> float64: ...
+    @staticmethod
+    def zeros(size: int) -> DenseVector: ...
+
+class Matrix:
+    __UDT__: MatrixUDT
+    numRows: int
+    numCols: int
+    isTransposed: bool
+    def __init__(
+        self, numRows: int, numCols: int, isTransposed: bool = ...
+    ) -> None: ...
+    def toArray(self) -> NoReturn: ...
+
+class DenseMatrix(Matrix):
+    values: Any
+    @overload
+    def __init__(
+        self, numRows: int, numCols: int, values: bytes, isTransposed: bool = ...
+    ) -> None: ...
+    @overload
+    def __init__(
+        self,
+        numRows: int,
+        numCols: int,
+        values: Iterable[float],
+        isTransposed: bool = ...,
+    ) -> None: ...
+    def __reduce__(self) -> Tuple[Type[DenseMatrix], Tuple[int, int, bytes, int]]: ...
+    def toArray(self) -> ndarray: ...
+    def toSparse(self) -> SparseMatrix: ...
+    def __getitem__(self, indices: Tuple[int, int]) -> float64: ...
+    def __eq__(self, other: Any) -> bool: ...
+
+class SparseMatrix(Matrix):
+    colPtrs: ndarray
+    rowIndices: ndarray
+    values: ndarray
+    @overload
+    def __init__(
+        self,
+        numRows: int,
+        numCols: int,
+        colPtrs: bytes,
+        rowIndices: bytes,
+        values: bytes,
+        isTransposed: bool = ...,
+    ) -> None: ...
+    @overload
+    def __init__(
+        self,
+        numRows: int,
+        numCols: int,
+        colPtrs: Iterable[int],
+        rowIndices: Iterable[int],
+        values: Iterable[float],
+        isTransposed: bool = ...,
+    ) -> None: ...
+    def __reduce__(
+        self,
+    ) -> Tuple[Type[SparseMatrix], Tuple[int, int, bytes, bytes, bytes, int]]: ...
+    def __getitem__(self, indices: Tuple[int, int]) -> float64: ...
+    def toArray(self) -> ndarray: ...
+    def toDense(self) -> DenseMatrix: ...
+    def __eq__(self, other: Any) -> bool: ...
+
+class Matrices:
+    @overload
+    @staticmethod
+    def dense(
+        numRows: int, numCols: int, values: bytes, isTransposed: bool = ...
+    ) -> DenseMatrix: ...
+    @overload
+    @staticmethod
+    def dense(
+        numRows: int, numCols: int, values: Iterable[float], isTransposed: bool = ...
+    ) -> DenseMatrix: ...
+    @overload
+    @staticmethod
+    def sparse(
+        numRows: int,
+        numCols: int,
+        colPtrs: bytes,
+        rowIndices: bytes,
+        values: bytes,
+        isTransposed: bool = ...,
+    ) -> SparseMatrix: ...
+    @overload
+    @staticmethod
+    def sparse(
+        numRows: int,
+        numCols: int,
+        colPtrs: Iterable[int],
+        rowIndices: Iterable[int],
+        values: Iterable[float],
+        isTransposed: bool = ...,
+    ) -> SparseMatrix: ...
diff --git a/python/pyspark/ml/param/__init__.py b/python/pyspark/ml/param/__init__.py
index 1be8755c7b982..ab3491c059568 100644
--- a/python/pyspark/ml/param/__init__.py
+++ b/python/pyspark/ml/param/__init__.py
@@ -15,16 +15,10 @@
 # limitations under the License.
 #
 import array
-import sys
-if sys.version > '3':
-    basestring = str
-    xrange = range
-    unicode = str
-
 from abc import ABCMeta
 import copy
-import numpy as np
 
+import numpy as np
 from py4j.java_gateway import JavaObject
 
 from pyspark.ml.linalg import DenseVector, Vector, Matrix
@@ -93,12 +87,12 @@ def _is_integer(value):
     @staticmethod
     def _can_convert_to_list(value):
         vtype = type(value)
-        return vtype in [list, np.ndarray, tuple, xrange, array.array] or isinstance(value, Vector)
+        return vtype in [list, np.ndarray, tuple, range, array.array] or isinstance(value, Vector)
 
     @staticmethod
     def _can_convert_to_string(value):
         vtype = type(value)
-        return isinstance(value, basestring) or vtype in [np.unicode_, np.string_, np.str_]
+        return isinstance(value, str) or vtype in [np.unicode_, np.string_, np.str_]
 
     @staticmethod
     def identity(value):
@@ -114,7 +108,7 @@ def toList(value):
         """
         if type(value) == list:
             return value
-        elif type(value) in [np.ndarray, tuple, xrange, array.array]:
+        elif type(value) in [np.ndarray, tuple, range, array.array]:
             return list(value)
         elif isinstance(value, Vector):
             return list(value.toArray())
@@ -211,12 +205,10 @@ def toString(value):
         """
         Convert a value to a string, if possible.
         """
-        if isinstance(value, basestring):
+        if isinstance(value, str):
             return value
-        elif type(value) in [np.string_, np.str_]:
+        elif type(value) in [np.string_, np.str_, np.unicode_]:
             return str(value)
-        elif type(value) == np.unicode_:
-            return unicode(value)
         else:
             raise TypeError("Could not convert %s to string type" % type(value))
 
@@ -231,7 +223,7 @@ def toBoolean(value):
             raise TypeError("Boolean Param requires value of type bool. Found %s." % type(value))
 
 
-class Params(Identifiable):
+class Params(Identifiable, metaclass=ABCMeta):
     """
     Components that take parameters. This also provides an internal
     param map to store parameter values attached to the instance.
@@ -239,8 +231,6 @@ class Params(Identifiable):
     .. versionadded:: 1.3.0
     """
 
-    __metaclass__ = ABCMeta
-
     def __init__(self):
         super(Params, self).__init__()
         #: internal param map for user-supplied values param map
@@ -338,7 +328,7 @@ def hasParam(self, paramName):
         Tests whether this instance contains a param with a given
         (string) name.
         """
-        if isinstance(paramName, basestring):
+        if isinstance(paramName, str):
             p = getattr(self, paramName, None)
             return isinstance(p, Param)
         else:
@@ -363,8 +353,15 @@ def extractParamMap(self, extra=None):
         conflicts, i.e., with ordering: default param values <
         user-supplied values < extra.
 
-        :param extra: extra param values
-        :return: merged param map
+        Parameters
+        ----------
+        extra : dict, optional
+            extra param values
+
+        Returns
+        -------
+        dict
+            merged param map
         """
         if extra is None:
             extra = dict()
@@ -382,8 +379,15 @@ def copy(self, extra=None):
         Subclasses should override this method if the default approach
         is not sufficient.
 
-        :param extra: Extra parameters to copy to the new instance
-        :return: Copy of this instance
+        Parameters
+        ----------
+        extra : dict, optional
+            Extra parameters to copy to the new instance
+
+        Returns
+        -------
+        :py:class:`Params`
+            Copy of this instance
         """
         if extra is None:
             extra = dict()
@@ -414,17 +418,30 @@ def _resolveParam(self, param):
         """
         Resolves a param and validates the ownership.
 
-        :param param: param name or the param instance, which must
-                      belong to this Params instance
-        :return: resolved param instance
+        Parameters
+        ----------
+        param : str or :py:class:`Param`
+            param name or the param instance, which must
+            belong to this Params instance
+
+        Returns
+        -------
+        :py:class:`Param`
+            resolved param instance
         """
         if isinstance(param, Param):
             self._shouldOwn(param)
             return param
-        elif isinstance(param, basestring):
+        elif isinstance(param, str):
             return self.getParam(param)
         else:
-            raise ValueError("Cannot resolve %r as a param." % param)
+            raise TypeError("Cannot resolve %r as a param." % param)
+
+    def _testOwnParam(self, param_parent, param_name):
+        """
+        Test the ownership. Return True or False
+        """
+        return self.uid == param_parent and self.hasParam(param_name)
 
     @staticmethod
     def _dummy():
@@ -477,9 +494,17 @@ def _copyValues(self, to, extra=None):
         Copies param values from this instance to another instance for
         params shared by them.
 
-        :param to: the target instance
-        :param extra: extra params to be copied
-        :return: the target instance with param values copied
+        Parameters
+        ----------
+        to : :py:class:`Params`
+            the target instance
+        extra : dict, optional
+            extra params to be copied
+
+        Returns
+        -------
+        :py:class:`Params`
+            the target instance with param values copied
         """
         paramMap = self._paramMap.copy()
         if isinstance(extra, dict):
@@ -506,11 +531,19 @@ def _resetUid(self, newUid):
         Changes the uid of this instance. This updates both
         the stored uid and the parent uid of params and param maps.
         This is used by persistence (loading).
-        :param newUid: new uid to use, which is converted to unicode
-        :return: same instance, but with the uid and Param.parent values
-                 updated, including within param maps
+
+        Parameters
+        ----------
+        newUid
+            new uid to use, which is converted to unicode
+
+        Returns
+        -------
+        :py:class:`Params`
+            same instance, but with the uid and Param.parent values
+            updated, including within param maps
         """
-        newUid = unicode(newUid)
+        newUid = str(newUid)
         self.uid = newUid
         newDefaultParamMap = dict()
         newParamMap = dict()
diff --git a/python/pyspark/ml/param/__init__.pyi b/python/pyspark/ml/param/__init__.pyi
new file mode 100644
index 0000000000000..23a63c573e452
--- /dev/null
+++ b/python/pyspark/ml/param/__init__.pyi
@@ -0,0 +1,96 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import abc
+from typing import overload
+from typing import Any, Callable, Generic, List, Optional
+from pyspark.ml._typing import T
+import pyspark.ml._typing
+
+import pyspark.ml.util
+from pyspark.ml.linalg import DenseVector, Matrix
+
+class Param(Generic[T]):
+    parent: str
+    name: str
+    doc: str
+    typeConverter: Callable[[Any], T]
+    def __init__(
+        self,
+        parent: pyspark.ml.util.Identifiable,
+        name: str,
+        doc: str,
+        typeConverter: Optional[Callable[[Any], T]] = ...,
+    ) -> None: ...
+    def __hash__(self) -> int: ...
+    def __eq__(self, other: Any) -> bool: ...
+
+class TypeConverters:
+    @staticmethod
+    def identity(value: T) -> T: ...
+    @staticmethod
+    def toList(value: Any) -> List: ...
+    @staticmethod
+    def toListFloat(value: Any) -> List[float]: ...
+    @staticmethod
+    def toListInt(value: Any) -> List[int]: ...
+    @staticmethod
+    def toListString(value: Any) -> List[str]: ...
+    @staticmethod
+    def toVector(value: Any) -> DenseVector: ...
+    @staticmethod
+    def toMatrix(value: Any) -> Matrix: ...
+    @staticmethod
+    def toFloat(value: Any) -> float: ...
+    @staticmethod
+    def toInt(value: Any) -> int: ...
+    @staticmethod
+    def toString(value: Any) -> str: ...
+    @staticmethod
+    def toBoolean(value: Any) -> bool: ...
+
+class Params(pyspark.ml.util.Identifiable, metaclass=abc.ABCMeta):
+    def __init__(self) -> None: ...
+    @property
+    def params(self) -> List[Param]: ...
+    def explainParam(self, param: str) -> str: ...
+    def explainParams(self) -> str: ...
+    def getParam(self, paramName: str) -> Param: ...
+    @overload
+    def isSet(self, param: str) -> bool: ...
+    @overload
+    def isSet(self, param: Param[Any]) -> bool: ...
+    @overload
+    def hasDefault(self, param: str) -> bool: ...
+    @overload
+    def hasDefault(self, param: Param[Any]) -> bool: ...
+    @overload
+    def isDefined(self, param: str) -> bool: ...
+    @overload
+    def isDefined(self, param: Param[Any]) -> bool: ...
+    def hasParam(self, paramName: str) -> bool: ...
+    @overload
+    def getOrDefault(self, param: str) -> Any: ...
+    @overload
+    def getOrDefault(self, param: Param[T]) -> T: ...
+    def extractParamMap(
+        self, extra: Optional[pyspark.ml._typing.ParamMap] = ...
+    ) -> pyspark.ml._typing.ParamMap: ...
+    def copy(self, extra: Optional[pyspark.ml._typing.ParamMap] = ...) -> Params: ...
+    def set(self, param: Param, value: Any) -> None: ...
+    def clear(self, param: Param) -> None: ...
diff --git a/python/pyspark/ml/param/_shared_params_code_gen.py b/python/pyspark/ml/param/_shared_params_code_gen.py
index 2086e831f4282..bcab51f76bd49 100644
--- a/python/pyspark/ml/param/_shared_params_code_gen.py
+++ b/python/pyspark/ml/param/_shared_params_code_gen.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 header = """#
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
@@ -167,7 +165,11 @@ def get$Name(self):
          None, "TypeConverters.toString"),
         ("blockSize", "block size for stacking input data in matrices. Data is stacked within "
          "partitions. If block size is more than remaining data in a partition then it is "
-         "adjusted to the size of this data.", None, "TypeConverters.toInt")]
+         "adjusted to the size of this data.", None, "TypeConverters.toInt"),
+        ("maxBlockSizeInMB", "maximum memory in MB for stacking input data into blocks. Data is " +
+         "stacked within partitions. If more than remaining data size in a partition then it " +
+         "is adjusted to the data size. Default 0.0 represents choosing optimal value, depends " +
+         "on specific algorithm. Must be >= 0.", "0.0", "TypeConverters.toFloat")]
 
     code = []
     for name, doc, defaultValueStr, typeConverter in shared:
diff --git a/python/pyspark/ml/param/_shared_params_code_gen.pyi b/python/pyspark/ml/param/_shared_params_code_gen.pyi
new file mode 100644
index 0000000000000..e436a54c0eaa4
--- /dev/null
+++ b/python/pyspark/ml/param/_shared_params_code_gen.pyi
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+header: str
diff --git a/python/pyspark/ml/param/shared.py b/python/pyspark/ml/param/shared.py
index 24fb0d3e2554d..9311e4481e2b4 100644
--- a/python/pyspark/ml/param/shared.py
+++ b/python/pyspark/ml/param/shared.py
@@ -597,3 +597,21 @@ def getBlockSize(self):
         Gets the value of blockSize or its default value.
         """
         return self.getOrDefault(self.blockSize)
+
+
+class HasMaxBlockSizeInMB(Params):
+    """
+    Mixin for param maxBlockSizeInMB: maximum memory in MB for stacking input data into blocks. Data is stacked within partitions. If more than remaining data size in a partition then it is adjusted to the data size. Default 0.0 represents choosing optimal value, depends on specific algorithm. Must be >= 0.
+    """
+
+    maxBlockSizeInMB = Param(Params._dummy(), "maxBlockSizeInMB", "maximum memory in MB for stacking input data into blocks. Data is stacked within partitions. If more than remaining data size in a partition then it is adjusted to the data size. Default 0.0 represents choosing optimal value, depends on specific algorithm. Must be >= 0.", typeConverter=TypeConverters.toFloat)
+
+    def __init__(self):
+        super(HasMaxBlockSizeInMB, self).__init__()
+        self._setDefault(maxBlockSizeInMB=0.0)
+
+    def getMaxBlockSizeInMB(self):
+        """
+        Gets the value of maxBlockSizeInMB or its default value.
+        """
+        return self.getOrDefault(self.maxBlockSizeInMB)
diff --git a/python/pyspark/ml/param/shared.pyi b/python/pyspark/ml/param/shared.pyi
new file mode 100644
index 0000000000000..0ff4d544205bc
--- /dev/null
+++ b/python/pyspark/ml/param/shared.pyi
@@ -0,0 +1,192 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Any, Generic, List
+from pyspark.ml._typing import T
+
+from pyspark.ml.param import *
+
+class HasMaxIter(Params):
+    maxIter: Param[int]
+    def __init__(self) -> None: ...
+    def getMaxIter(self) -> int: ...
+
+class HasRegParam(Params):
+    regParam: Param[float]
+    def __init__(self) -> None: ...
+    def getRegParam(self) -> float: ...
+
+class HasFeaturesCol(Params):
+    featuresCol: Param[str]
+    def __init__(self) -> None: ...
+    def getFeaturesCol(self) -> str: ...
+
+class HasLabelCol(Params):
+    labelCol: Param[str]
+    def __init__(self) -> None: ...
+    def getLabelCol(self) -> str: ...
+
+class HasPredictionCol(Params):
+    predictionCol: Param[str]
+    def __init__(self) -> None: ...
+    def getPredictionCol(self) -> str: ...
+
+class HasProbabilityCol(Params):
+    probabilityCol: Param[str]
+    def __init__(self) -> None: ...
+    def getProbabilityCol(self) -> str: ...
+
+class HasRawPredictionCol(Params):
+    rawPredictionCol: Param[str]
+    def __init__(self) -> None: ...
+    def getRawPredictionCol(self) -> str: ...
+
+class HasInputCol(Params):
+    inputCol: Param[str]
+    def __init__(self) -> None: ...
+    def getInputCol(self) -> str: ...
+
+class HasInputCols(Params):
+    inputCols: Param[List[str]]
+    def __init__(self) -> None: ...
+    def getInputCols(self) -> List[str]: ...
+
+class HasOutputCol(Params):
+    outputCol: Param[str]
+    def __init__(self) -> None: ...
+    def getOutputCol(self) -> str: ...
+
+class HasOutputCols(Params):
+    outputCols: Param[List[str]]
+    def __init__(self) -> None: ...
+    def getOutputCols(self) -> List[str]: ...
+
+class HasNumFeatures(Params):
+    numFeatures: Param[int]
+    def __init__(self) -> None: ...
+    def getNumFeatures(self) -> int: ...
+
+class HasCheckpointInterval(Params):
+    checkpointInterval: Param[int]
+    def __init__(self) -> None: ...
+    def getCheckpointInterval(self) -> int: ...
+
+class HasSeed(Params):
+    seed: Param[int]
+    def __init__(self) -> None: ...
+    def getSeed(self) -> int: ...
+
+class HasTol(Params):
+    tol: Param[float]
+    def __init__(self) -> None: ...
+    def getTol(self) -> float: ...
+
+class HasRelativeError(Params):
+    relativeError: Param[float]
+    def __init__(self) -> None: ...
+    def getRelativeError(self) -> float: ...
+
+class HasStepSize(Params):
+    stepSize: Param[float]
+    def __init__(self) -> None: ...
+    def getStepSize(self) -> float: ...
+
+class HasHandleInvalid(Params):
+    handleInvalid: Param[str]
+    def __init__(self) -> None: ...
+    def getHandleInvalid(self) -> str: ...
+
+class HasElasticNetParam(Params):
+    elasticNetParam: Param[float]
+    def __init__(self) -> None: ...
+    def getElasticNetParam(self) -> float: ...
+
+class HasFitIntercept(Params):
+    fitIntercept: Param[bool]
+    def __init__(self) -> None: ...
+    def getFitIntercept(self) -> bool: ...
+
+class HasStandardization(Params):
+    standardization: Param[bool]
+    def __init__(self) -> None: ...
+    def getStandardization(self) -> bool: ...
+
+class HasThresholds(Params):
+    thresholds: Param[List[float]]
+    def __init__(self) -> None: ...
+    def getThresholds(self) -> List[float]: ...
+
+class HasThreshold(Params):
+    threshold: Param[float]
+    def __init__(self) -> None: ...
+    def getThreshold(self) -> float: ...
+
+class HasWeightCol(Params):
+    weightCol: Param[str]
+    def __init__(self) -> None: ...
+    def getWeightCol(self) -> str: ...
+
+class HasSolver(Params):
+    solver: Param[str]
+    def __init__(self) -> None: ...
+    def getSolver(self) -> str: ...
+
+class HasVarianceCol(Params):
+    varianceCol: Param[str]
+    def __init__(self) -> None: ...
+    def getVarianceCol(self) -> str: ...
+
+class HasAggregationDepth(Params):
+    aggregationDepth: Param[int]
+    def __init__(self) -> None: ...
+    def getAggregationDepth(self) -> int: ...
+
+class HasParallelism(Params):
+    parallelism: Param[int]
+    def __init__(self) -> None: ...
+    def getParallelism(self) -> int: ...
+
+class HasCollectSubModels(Params):
+    collectSubModels: Param[bool]
+    def __init__(self) -> None: ...
+    def getCollectSubModels(self) -> bool: ...
+
+class HasLoss(Params):
+    loss: Param[str]
+    def __init__(self) -> None: ...
+    def getLoss(self) -> str: ...
+
+class HasValidationIndicatorCol(Params):
+    validationIndicatorCol: Param[str]
+    def __init__(self) -> None: ...
+    def getValidationIndicatorCol(self) -> str: ...
+
+class HasDistanceMeasure(Params):
+    distanceMeasure: Param[str]
+    def __init__(self) -> None: ...
+    def getDistanceMeasure(self) -> str: ...
+
+class HasBlockSize(Params):
+    blockSize: Param[int]
+    def __init__(self) -> None: ...
+    def getBlockSize(self) -> int: ...
+
+class HasMaxBlockSizeInMB(Params):
+    maxBlockSizeInMB: Param[float]
+    def __init__(self) -> None: ...
+    def getMaxBlockSizeInMB(self) -> float: ...
diff --git a/python/pyspark/ml/pipeline.py b/python/pyspark/ml/pipeline.py
index 53d07ec9660d9..b0aa735709e8d 100644
--- a/python/pyspark/ml/pipeline.py
+++ b/python/pyspark/ml/pipeline.py
@@ -14,19 +14,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
-import sys
 import os
 
-if sys.version > '3':
-    basestring = str
-
-from pyspark import since, keyword_only, SparkContext
+from pyspark import keyword_only, since, SparkContext
 from pyspark.ml.base import Estimator, Model, Transformer
 from pyspark.ml.param import Param, Params
-from pyspark.ml.util import *
-from pyspark.ml.wrapper import JavaParams, JavaWrapper
-from pyspark.ml.common import inherit_doc, _java2py, _py2java
+from pyspark.ml.util import MLReadable, MLWritable, JavaMLWriter, JavaMLReader, \
+    DefaultParamsReader, DefaultParamsWriter, MLWriter, MLReader, JavaMLWritable
+from pyspark.ml.wrapper import JavaParams
+from pyspark.ml.common import inherit_doc
 
 
 @inherit_doc
@@ -54,21 +50,30 @@ class Pipeline(Estimator, MLReadable, MLWritable):
     stages = Param(Params._dummy(), "stages", "a list of pipeline stages")
 
     @keyword_only
-    def __init__(self, stages=None):
+    def __init__(self, *, stages=None):
         """
-        __init__(self, stages=None)
+        __init__(self, \\*, stages=None)
         """
         super(Pipeline, self).__init__()
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
-    @since("1.3.0")
     def setStages(self, value):
         """
         Set pipeline stages.
 
-        :param value: a list of transformers or estimators
-        :return: the pipeline instance
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        value : list
+            of :py:class:`pyspark.ml.Transformer`
+            or :py:class:`pyspark.ml.Estimator`
+
+        Returns
+        -------
+        :py:class:`Pipeline`
+            the pipeline instance
         """
         return self._set(stages=value)
 
@@ -81,9 +86,9 @@ def getStages(self):
 
     @keyword_only
     @since("1.3.0")
-    def setParams(self, stages=None):
+    def setParams(self, *, stages=None):
         """
-        setParams(self, stages=None)
+        setParams(self, \\*, stages=None)
         Sets params for Pipeline.
         """
         kwargs = self._input_kwargs
@@ -114,13 +119,21 @@ def _fit(self, dataset):
                 transformers.append(stage)
         return PipelineModel(transformers)
 
-    @since("1.4.0")
     def copy(self, extra=None):
         """
         Creates a copy of this instance.
 
-        :param extra: extra parameters
-        :returns: new instance
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        extra : dict, optional
+            extra parameters
+
+        Returns
+        -------
+        :py:class:`Pipeline`
+            new instance
         """
         if extra is None:
             extra = dict()
@@ -160,7 +173,10 @@ def _to_java(self):
         """
         Transfer this instance to a Java Pipeline.  Used for ML persistence.
 
-        :return: Java object equivalent to this instance.
+        Returns
+        -------
+        py4j.java_gateway.JavaObject
+            Java object equivalent to this instance.
         """
 
         gateway = SparkContext._gateway
@@ -174,55 +190,6 @@ def _to_java(self):
 
         return _java_obj
 
-    def _make_java_param_pair(self, param, value):
-        """
-        Makes a Java param pair.
-        """
-        sc = SparkContext._active_spark_context
-        param = self._resolveParam(param)
-        java_param = sc._jvm.org.apache.spark.ml.param.Param(param.parent, param.name, param.doc)
-        if isinstance(value, Params) and hasattr(value, "_to_java"):
-            # Convert JavaEstimator/JavaTransformer object or Estimator/Transformer object which
-            # implements `_to_java` method (such as OneVsRest, Pipeline object) to java object.
-            # used in the case of an estimator having another estimator as a parameter
-            # the reason why this is not in _py2java in common.py is that importing
-            # Estimator and Model in common.py results in a circular import with inherit_doc
-            java_value = value._to_java()
-        else:
-            java_value = _py2java(sc, value)
-        return java_param.w(java_value)
-
-    def _transfer_param_map_to_java(self, pyParamMap):
-        """
-        Transforms a Python ParamMap into a Java ParamMap.
-        """
-        paramMap = JavaWrapper._new_java_obj("org.apache.spark.ml.param.ParamMap")
-        for param in self.params:
-            if param in pyParamMap:
-                pair = self._make_java_param_pair(param, pyParamMap[param])
-                paramMap.put([pair])
-        return paramMap
-
-    def _transfer_param_map_from_java(self, javaParamMap):
-        """
-        Transforms a Java ParamMap into a Python ParamMap.
-        """
-        sc = SparkContext._active_spark_context
-        paramMap = dict()
-        for pair in javaParamMap.toList():
-            param = pair.param()
-            if self.hasParam(str(param.name())):
-                java_obj = pair.value()
-                if sc._jvm.Class.forName("org.apache.spark.ml.PipelineStage").isInstance(java_obj):
-                    # Note: JavaParams._from_java support both JavaEstimator/JavaTransformer class
-                    # and Estimator/Transformer class which implements `_from_java` static method
-                    # (such as OneVsRest, Pipeline class).
-                    py_obj = JavaParams._from_java(java_obj)
-                else:
-                    py_obj = _java2py(sc, java_obj)
-                paramMap[self.getParam(param.name())] = py_obj
-        return paramMap
-
 
 @inherit_doc
 class PipelineWriter(MLWriter):
@@ -311,11 +278,12 @@ def _transform(self, dataset):
             dataset = t.transform(dataset)
         return dataset
 
-    @since("1.4.0")
     def copy(self, extra=None):
         """
         Creates a copy of this instance.
 
+        .. versionadded:: 1.4.0
+
         :param extra: extra parameters
         :returns: new instance
         """
@@ -414,7 +382,10 @@ def load(metadata, sc, path):
         """
         Load metadata and stages for a :py:class:`Pipeline` or :py:class:`PipelineModel`
 
-        :return: (UID, list of stages)
+        Returns
+        -------
+        tuple
+            (UID, list of stages)
         """
         stagesDir = os.path.join(path, "stages")
         stageUids = metadata['paramMap']['stageUids']
diff --git a/python/pyspark/ml/pipeline.pyi b/python/pyspark/ml/pipeline.pyi
new file mode 100644
index 0000000000000..f47e9e012ae14
--- /dev/null
+++ b/python/pyspark/ml/pipeline.pyi
@@ -0,0 +1,97 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Any, Dict, List, Optional, Tuple, Type, Union
+
+from pyspark.ml._typing import PipelineStage
+from pyspark.context import SparkContext
+from pyspark.ml.base import Estimator, Model, Transformer
+from pyspark.ml.param import Param
+from pyspark.ml.util import (  # noqa: F401
+    DefaultParamsReader as DefaultParamsReader,
+    DefaultParamsWriter as DefaultParamsWriter,
+    JavaMLReader as JavaMLReader,
+    JavaMLWritable as JavaMLWritable,
+    JavaMLWriter as JavaMLWriter,
+    MLReadable as MLReadable,
+    MLReader as MLReader,
+    MLWritable as MLWritable,
+    MLWriter as MLWriter,
+)
+
+class Pipeline(Estimator[PipelineModel], MLReadable[Pipeline], MLWritable):
+    stages: List[PipelineStage]
+    def __init__(self, *, stages: Optional[List[PipelineStage]] = ...) -> None: ...
+    def setStages(self, stages: List[PipelineStage]) -> Pipeline: ...
+    def getStages(self) -> List[PipelineStage]: ...
+    def setParams(self, *, stages: Optional[List[PipelineStage]] = ...) -> Pipeline: ...
+    def copy(self, extra: Optional[Dict[Param, str]] = ...) -> Pipeline: ...
+    def write(self) -> JavaMLWriter: ...
+    def save(self, path: str) -> None: ...
+    @classmethod
+    def read(cls) -> PipelineReader: ...
+
+class PipelineWriter(MLWriter):
+    instance: Pipeline
+    def __init__(self, instance: Pipeline) -> None: ...
+    def saveImpl(self, path: str) -> None: ...
+
+class PipelineReader(MLReader[Pipeline]):
+    cls: Type[Pipeline]
+    def __init__(self, cls: Type[Pipeline]) -> None: ...
+    def load(self, path: str) -> Pipeline: ...
+
+class PipelineModelWriter(MLWriter):
+    instance: PipelineModel
+    def __init__(self, instance: PipelineModel) -> None: ...
+    def saveImpl(self, path: str) -> None: ...
+
+class PipelineModelReader(MLReader[PipelineModel]):
+    cls: Type[PipelineModel]
+    def __init__(self, cls: Type[PipelineModel]) -> None: ...
+    def load(self, path: str) -> PipelineModel: ...
+
+class PipelineModel(Model, MLReadable[PipelineModel], MLWritable):
+    stages: List[PipelineStage]
+    def __init__(self, stages: List[Transformer]) -> None: ...
+    def copy(self, extra: Optional[Dict[Param, Any]] = ...) -> PipelineModel: ...
+    def write(self) -> JavaMLWriter: ...
+    def save(self, path: str) -> None: ...
+    @classmethod
+    def read(cls) -> PipelineModelReader: ...
+
+class PipelineSharedReadWrite:
+    @staticmethod
+    def checkStagesForJava(stages: List[PipelineStage]) -> bool: ...
+    @staticmethod
+    def validateStages(stages: List[PipelineStage]) -> None: ...
+    @staticmethod
+    def saveImpl(
+        instance: Union[Pipeline, PipelineModel],
+        stages: List[PipelineStage],
+        sc: SparkContext,
+        path: str,
+    ) -> None: ...
+    @staticmethod
+    def load(
+        metadata: Dict[str, Any], sc: SparkContext, path: str
+    ) -> Tuple[str, List[PipelineStage]]: ...
+    @staticmethod
+    def getStagePath(
+        stageUid: str, stageIdx: int, numStages: int, stagesDir: str
+    ) -> str: ...
diff --git a/python/pyspark/ml/recommendation.py b/python/pyspark/ml/recommendation.py
index 99d80aa867bda..5bc1801a0c957 100644
--- a/python/pyspark/ml/recommendation.py
+++ b/python/pyspark/ml/recommendation.py
@@ -18,10 +18,12 @@
 import sys
 
 from pyspark import since, keyword_only
-from pyspark.ml.util import *
+from pyspark.ml.param.shared import HasPredictionCol, HasBlockSize, HasMaxIter, HasRegParam, \
+    HasCheckpointInterval, HasSeed
 from pyspark.ml.wrapper import JavaEstimator, JavaModel
-from pyspark.ml.param.shared import *
 from pyspark.ml.common import inherit_doc
+from pyspark.ml.param import Params, TypeConverters, Param
+from pyspark.ml.util import JavaMLWritable, JavaMLReadable
 
 
 __all__ = ['ALS', 'ALSModel']
@@ -46,6 +48,10 @@ class _ALSModelParams(HasPredictionCol, HasBlockSize):
                               "Supported values: 'nan', 'drop'.",
                               typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_ALSModelParams, self).__init__(*args)
+        self._setDefault(blockSize=4096)
+
     @since("1.4.0")
     def getUserCol(self):
         """
@@ -99,6 +105,14 @@ class _ALSParams(_ALSModelParams, HasMaxIter, HasRegParam, HasCheckpointInterval
                               "StorageLevel for ALS model factors.",
                               typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_ALSParams, self).__init__(*args)
+        self._setDefault(rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, numItemBlocks=10,
+                         implicitPrefs=False, alpha=1.0, userCol="user", itemCol="item",
+                         ratingCol="rating", nonnegative=False, checkpointInterval=10,
+                         intermediateStorageLevel="MEMORY_AND_DISK",
+                         finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan")
+
     @since("1.4.0")
     def getRank(self):
         """
@@ -202,13 +216,19 @@ class ALS(JavaEstimator, _ALSParams, JavaMLWritable, JavaMLReadable):
     indicated user preferences rather than explicit ratings given to
     items.
 
-    .. note:: the input rating dataframe to the ALS implementation should be deterministic.
-              Nondeterministic data can cause failure during fitting ALS model.
-              For example, an order-sensitive operation like sampling after a repartition makes
-              dataframe output nondeterministic, like `df.repartition(2).sample(False, 0.5, 1618)`.
-              Checkpointing sampled dataframe or adding a sort before sampling can help make the
-              dataframe deterministic.
+    .. versionadded:: 1.4.0
 
+    Notes
+    -----
+    The input rating dataframe to the ALS implementation should be deterministic.
+    Nondeterministic data can cause failure during fitting ALS model.
+    For example, an order-sensitive operation like sampling after a repartition makes
+    dataframe output nondeterministic, like `df.repartition(2).sample(False, 0.5, 1618)`.
+    Checkpointing sampled dataframe or adding a sort before sampling can help make the
+    dataframe deterministic.
+
+    Examples
+    --------
     >>> df = spark.createDataFrame(
     ...     [(0, 0, 4.0), (0, 1, 2.0), (1, 1, 3.0), (1, 2, 4.0), (2, 1, 1.0), (2, 2, 5.0)],
     ...     ["user", "item", "rating"])
@@ -240,11 +260,11 @@ class ALS(JavaEstimator, _ALSParams, JavaMLWritable, JavaMLReadable):
     >>> test = spark.createDataFrame([(0, 2), (1, 0), (2, 0)], ["user", "item"])
     >>> predictions = sorted(model.transform(test).collect(), key=lambda r: r[0])
     >>> predictions[0]
-    Row(user=0, item=2, newPrediction=0.6929101347923279)
+    Row(user=0, item=2, newPrediction=0.69291...)
     >>> predictions[1]
-    Row(user=1, item=0, newPrediction=3.47356915473938)
+    Row(user=1, item=0, newPrediction=3.47356...)
     >>> predictions[2]
-    Row(user=2, item=0, newPrediction=-0.8991986513137817)
+    Row(user=2, item=0, newPrediction=-0.899198...)
     >>> user_recs = model.recommendForAllUsers(3)
     >>> user_recs.where(user_recs.user == 0)\
         .select("recommendations.item", "recommendations.rating").collect()
@@ -275,45 +295,39 @@ class ALS(JavaEstimator, _ALSParams, JavaMLWritable, JavaMLReadable):
     True
     >>> sorted(model.itemFactors.collect()) == sorted(model2.itemFactors.collect())
     True
-
-    .. versionadded:: 1.4.0
+    >>> model.transform(test).take(1) == model2.transform(test).take(1)
+    True
     """
 
     @keyword_only
-    def __init__(self, rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, numItemBlocks=10,
-                 implicitPrefs=False, alpha=1.0, userCol="user", itemCol="item", seed=None,
-                 ratingCol="rating", nonnegative=False, checkpointInterval=10,
+    def __init__(self, *, rank=10, maxIter=10, regParam=0.1, numUserBlocks=10,
+                 numItemBlocks=10, implicitPrefs=False, alpha=1.0, userCol="user", itemCol="item",
+                 seed=None, ratingCol="rating", nonnegative=False, checkpointInterval=10,
                  intermediateStorageLevel="MEMORY_AND_DISK",
                  finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan", blockSize=4096):
         """
-        __init__(self, rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, numItemBlocks=10, \
-                 implicitPrefs=false, alpha=1.0, userCol="user", itemCol="item", seed=None, \
-                 ratingCol="rating", nonnegative=false, checkpointInterval=10, \
+        __init__(self, \\*, rank=10, maxIter=10, regParam=0.1, numUserBlocks=10,
+                 numItemBlocks=10, implicitPrefs=False, alpha=1.0, userCol="user", itemCol="item", \
+                 seed=None, ratingCol="rating", nonnegative=False, checkpointInterval=10, \
                  intermediateStorageLevel="MEMORY_AND_DISK", \
                  finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan", blockSize=4096)
         """
         super(ALS, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.recommendation.ALS", self.uid)
-        self._setDefault(rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, numItemBlocks=10,
-                         implicitPrefs=False, alpha=1.0, userCol="user", itemCol="item",
-                         ratingCol="rating", nonnegative=False, checkpointInterval=10,
-                         intermediateStorageLevel="MEMORY_AND_DISK",
-                         finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan",
-                         blockSize=4096)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("1.4.0")
-    def setParams(self, rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, numItemBlocks=10,
-                  implicitPrefs=False, alpha=1.0, userCol="user", itemCol="item", seed=None,
-                  ratingCol="rating", nonnegative=False, checkpointInterval=10,
+    def setParams(self, *, rank=10, maxIter=10, regParam=0.1, numUserBlocks=10,
+                  numItemBlocks=10, implicitPrefs=False, alpha=1.0, userCol="user", itemCol="item",
+                  seed=None, ratingCol="rating", nonnegative=False, checkpointInterval=10,
                   intermediateStorageLevel="MEMORY_AND_DISK",
                   finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan", blockSize=4096):
         """
-        setParams(self, rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, numItemBlocks=10, \
-                 implicitPrefs=False, alpha=1.0, userCol="user", itemCol="item", seed=None, \
-                 ratingCol="rating", nonnegative=False, checkpointInterval=10, \
+        setParams(self, \\*, rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, \
+                 numItemBlocks=10, implicitPrefs=False, alpha=1.0, userCol="user", itemCol="item", \
+                 seed=None, ratingCol="rating", nonnegative=False, checkpointInterval=10, \
                  intermediateStorageLevel="MEMORY_AND_DISK", \
                  finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan", blockSize=4096)
         Sets params for ALS.
@@ -520,55 +534,87 @@ def itemFactors(self):
         """
         return self._call_java("itemFactors")
 
-    @since("2.2.0")
     def recommendForAllUsers(self, numItems):
         """
         Returns top `numItems` items recommended for each user, for all users.
 
-        :param numItems: max number of recommendations for each user
-        :return: a DataFrame of (userCol, recommendations), where recommendations are
-                 stored as an array of (itemCol, rating) Rows.
+        .. versionadded:: 2.2.0
+
+        Parameters
+        ----------
+        numItems : int
+            max number of recommendations for each user
+
+        Returns
+        -------
+        :py:class:`pyspark.sql.DataFrame`
+            a DataFrame of (userCol, recommendations), where recommendations are
+            stored as an array of (itemCol, rating) Rows.
         """
         return self._call_java("recommendForAllUsers", numItems)
 
-    @since("2.2.0")
     def recommendForAllItems(self, numUsers):
         """
         Returns top `numUsers` users recommended for each item, for all items.
 
-        :param numUsers: max number of recommendations for each item
-        :return: a DataFrame of (itemCol, recommendations), where recommendations are
-                 stored as an array of (userCol, rating) Rows.
+        .. versionadded:: 2.2.0
+
+        Parameters
+        ----------
+        numUsers : int
+            max number of recommendations for each item
+
+        Returns
+        -------
+        :py:class:`pyspark.sql.DataFrame`
+            a DataFrame of (itemCol, recommendations), where recommendations are
+            stored as an array of (userCol, rating) Rows.
         """
         return self._call_java("recommendForAllItems", numUsers)
 
-    @since("2.3.0")
     def recommendForUserSubset(self, dataset, numItems):
         """
         Returns top `numItems` items recommended for each user id in the input data set. Note that
         if there are duplicate ids in the input dataset, only one set of recommendations per unique
         id will be returned.
 
-        :param dataset: a Dataset containing a column of user ids. The column name must match
-                        `userCol`.
-        :param numItems: max number of recommendations for each user
-        :return: a DataFrame of (userCol, recommendations), where recommendations are
-                 stored as an array of (itemCol, rating) Rows.
+        .. versionadded:: 2.3.0
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            a DataFrame containing a column of user ids. The column name must match `userCol`.
+        numItems : int
+            max number of recommendations for each user
+
+        Returns
+        -------
+        :py:class:`pyspark.sql.DataFrame`
+            a DataFrame of (userCol, recommendations), where recommendations are
+            stored as an array of (itemCol, rating) Rows.
         """
         return self._call_java("recommendForUserSubset", dataset, numItems)
 
-    @since("2.3.0")
     def recommendForItemSubset(self, dataset, numUsers):
         """
         Returns top `numUsers` users recommended for each item id in the input data set. Note that
         if there are duplicate ids in the input dataset, only one set of recommendations per unique
         id will be returned.
 
-        :param dataset: a Dataset containing a column of item ids. The column name must match
-                        `itemCol`.
-        :param numUsers: max number of recommendations for each item
-        :return: a DataFrame of (itemCol, recommendations), where recommendations are
-                 stored as an array of (userCol, rating) Rows.
+        .. versionadded:: 2.3.0
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            a DataFrame containing a column of item ids. The column name must match `itemCol`.
+        numUsers : int
+            max number of recommendations for each item
+
+        Returns
+        -------
+        :py:class:`pyspark.sql.DataFrame`
+            a DataFrame of (itemCol, recommendations), where recommendations are
+            stored as an array of (userCol, rating) Rows.
         """
         return self._call_java("recommendForItemSubset", dataset, numUsers)
 
diff --git a/python/pyspark/ml/recommendation.pyi b/python/pyspark/ml/recommendation.pyi
new file mode 100644
index 0000000000000..390486b45c5e6
--- /dev/null
+++ b/python/pyspark/ml/recommendation.pyi
@@ -0,0 +1,152 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Any, Optional
+
+import sys  # noqa: F401
+
+from pyspark import since, keyword_only  # noqa: F401
+from pyspark.ml.param.shared import (
+    HasBlockSize,
+    HasCheckpointInterval,
+    HasMaxIter,
+    HasPredictionCol,
+    HasRegParam,
+    HasSeed,
+)
+from pyspark.ml.wrapper import JavaEstimator, JavaModel
+from pyspark.ml.common import inherit_doc  # noqa: F401
+from pyspark.ml.param import Param
+from pyspark.ml.util import JavaMLWritable, JavaMLReadable
+
+from pyspark.sql.dataframe import DataFrame
+
+class _ALSModelParams(HasPredictionCol, HasBlockSize):
+    userCol: Param[str]
+    itemCol: Param[str]
+    coldStartStrategy: Param[str]
+    def getUserCol(self) -> str: ...
+    def getItemCol(self) -> str: ...
+    def getColdStartStrategy(self) -> str: ...
+
+class _ALSParams(
+    _ALSModelParams, HasMaxIter, HasRegParam, HasCheckpointInterval, HasSeed
+):
+    rank: Param[int]
+    numUserBlocks: Param[int]
+    numItemBlocks: Param[int]
+    implicitPrefs: Param[bool]
+    alpha: Param[float]
+    ratingCol: Param[str]
+    nonnegative: Param[bool]
+    intermediateStorageLevel: Param[str]
+    finalStorageLevel: Param[str]
+    def __init__(self, *args: Any): ...
+    def getRank(self) -> int: ...
+    def getNumUserBlocks(self) -> int: ...
+    def getNumItemBlocks(self) -> int: ...
+    def getImplicitPrefs(self) -> bool: ...
+    def getAlpha(self) -> float: ...
+    def getRatingCol(self) -> str: ...
+    def getNonnegative(self) -> bool: ...
+    def getIntermediateStorageLevel(self) -> str: ...
+    def getFinalStorageLevel(self) -> str: ...
+
+class ALS(JavaEstimator[ALSModel], _ALSParams, JavaMLWritable, JavaMLReadable[ALS]):
+    def __init__(
+        self,
+        *,
+        rank: int = ...,
+        maxIter: int = ...,
+        regParam: float = ...,
+        numUserBlocks: int = ...,
+        numItemBlocks: int = ...,
+        implicitPrefs: bool = ...,
+        alpha: float = ...,
+        userCol: str = ...,
+        itemCol: str = ...,
+        seed: Optional[int] = ...,
+        ratingCol: str = ...,
+        nonnegative: bool = ...,
+        checkpointInterval: int = ...,
+        intermediateStorageLevel: str = ...,
+        finalStorageLevel: str = ...,
+        coldStartStrategy: str = ...,
+        blockSize: int = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        rank: int = ...,
+        maxIter: int = ...,
+        regParam: float = ...,
+        numUserBlocks: int = ...,
+        numItemBlocks: int = ...,
+        implicitPrefs: bool = ...,
+        alpha: float = ...,
+        userCol: str = ...,
+        itemCol: str = ...,
+        seed: Optional[int] = ...,
+        ratingCol: str = ...,
+        nonnegative: bool = ...,
+        checkpointInterval: int = ...,
+        intermediateStorageLevel: str = ...,
+        finalStorageLevel: str = ...,
+        coldStartStrategy: str = ...,
+        blockSize: int = ...
+    ) -> ALS: ...
+    def setRank(self, value: int) -> ALS: ...
+    def setNumUserBlocks(self, value: int) -> ALS: ...
+    def setNumItemBlocks(self, value: int) -> ALS: ...
+    def setNumBlocks(self, value: int) -> ALS: ...
+    def setImplicitPrefs(self, value: bool) -> ALS: ...
+    def setAlpha(self, value: float) -> ALS: ...
+    def setUserCol(self, value: str) -> ALS: ...
+    def setItemCol(self, value: str) -> ALS: ...
+    def setRatingCol(self, value: str) -> ALS: ...
+    def setNonnegative(self, value: bool) -> ALS: ...
+    def setIntermediateStorageLevel(self, value: str) -> ALS: ...
+    def setFinalStorageLevel(self, value: str) -> ALS: ...
+    def setColdStartStrategy(self, value: str) -> ALS: ...
+    def setMaxIter(self, value: int) -> ALS: ...
+    def setRegParam(self, value: float) -> ALS: ...
+    def setPredictionCol(self, value: str) -> ALS: ...
+    def setCheckpointInterval(self, value: int) -> ALS: ...
+    def setSeed(self, value: int) -> ALS: ...
+    def setBlockSize(self, value: int) -> ALS: ...
+
+class ALSModel(JavaModel, _ALSModelParams, JavaMLWritable, JavaMLReadable[ALSModel]):
+    def setUserCol(self, value: str) -> ALSModel: ...
+    def setItemCol(self, value: str) -> ALSModel: ...
+    def setColdStartStrategy(self, value: str) -> ALSModel: ...
+    def setPredictionCol(self, value: str) -> ALSModel: ...
+    def setBlockSize(self, value: int) -> ALSModel: ...
+    @property
+    def rank(self) -> int: ...
+    @property
+    def userFactors(self) -> DataFrame: ...
+    @property
+    def itemFactors(self) -> DataFrame: ...
+    def recommendForAllUsers(self, numItems: int) -> DataFrame: ...
+    def recommendForAllItems(self, numUsers: int) -> DataFrame: ...
+    def recommendForUserSubset(
+        self, dataset: DataFrame, numItems: int
+    ) -> DataFrame: ...
+    def recommendForItemSubset(
+        self, dataset: DataFrame, numUsers: int
+    ) -> DataFrame: ...
diff --git a/python/pyspark/ml/regression.py b/python/pyspark/ml/regression.py
index b58255ea12afc..90bc06a2da73a 100644
--- a/python/pyspark/ml/regression.py
+++ b/python/pyspark/ml/regression.py
@@ -16,17 +16,21 @@
 #
 
 import sys
+
 from abc import ABCMeta
 
-from pyspark import since, keyword_only
+from pyspark import keyword_only, since
 from pyspark.ml import Predictor, PredictionModel
 from pyspark.ml.base import _PredictorParams
-from pyspark.ml.param.shared import *
+from pyspark.ml.param.shared import HasFeaturesCol, HasLabelCol, HasPredictionCol, HasWeightCol, \
+    Param, Params, TypeConverters, HasMaxIter, HasTol, HasFitIntercept, HasAggregationDepth, \
+    HasMaxBlockSizeInMB, HasRegParam, HasSolver, HasStepSize, HasSeed, HasElasticNetParam, \
+    HasStandardization, HasLoss, HasVarianceCol
 from pyspark.ml.tree import _DecisionTreeModel, _DecisionTreeParams, \
-    _TreeEnsembleModel, _TreeEnsembleParams, _RandomForestParams, _GBTParams, \
-    _HasVarianceImpurity, _TreeRegressorParams
-from pyspark.ml.util import *
-from pyspark.ml.wrapper import JavaEstimator, JavaModel, JavaParams, \
+    _TreeEnsembleModel, _RandomForestParams, _GBTParams, _TreeRegressorParams
+from pyspark.ml.util import JavaMLWritable, JavaMLReadable, HasTrainingSummary, \
+    GeneralJavaMLWritable
+from pyspark.ml.wrapper import JavaEstimator, JavaModel, \
     JavaPredictor, JavaPredictionModel, JavaWrapper
 from pyspark.ml.common import inherit_doc
 from pyspark.sql import DataFrame
@@ -44,50 +48,46 @@
            'FMRegressor', 'FMRegressionModel']
 
 
-class Regressor(Predictor, _PredictorParams):
+class Regressor(Predictor, _PredictorParams, metaclass=ABCMeta):
     """
     Regressor for regression tasks.
 
     .. versionadded:: 3.0.0
     """
-
-    __metaclass__ = ABCMeta
+    pass
 
 
-class RegressionModel(PredictionModel, _PredictorParams):
+class RegressionModel(PredictionModel, _PredictorParams, metaclass=ABCMeta):
     """
     Model produced by a ``Regressor``.
 
     .. versionadded:: 3.0.0
     """
-
-    __metaclass__ = ABCMeta
+    pass
 
 
-class _JavaRegressor(Regressor, JavaPredictor):
+class _JavaRegressor(Regressor, JavaPredictor, metaclass=ABCMeta):
     """
     Java Regressor for regression tasks.
 
     .. versionadded:: 3.0.0
     """
-
-    __metaclass__ = ABCMeta
+    pass
 
 
-class _JavaRegressionModel(RegressionModel, JavaPredictionModel):
+class _JavaRegressionModel(RegressionModel, JavaPredictionModel, metaclass=ABCMeta):
     """
     Java Model produced by a ``_JavaRegressor``.
     To be mixed in with :class:`pyspark.ml.JavaModel`
 
     .. versionadded:: 3.0.0
     """
-
-    __metaclass__ = ABCMeta
+    pass
 
 
 class _LinearRegressionParams(_PredictorParams, HasRegParam, HasElasticNetParam, HasMaxIter,
                               HasTol, HasFitIntercept, HasStandardization, HasWeightCol, HasSolver,
-                              HasAggregationDepth, HasLoss, HasBlockSize):
+                              HasAggregationDepth, HasLoss, HasMaxBlockSizeInMB):
     """
     Params for :py:class:`LinearRegression` and :py:class:`LinearRegressionModel`.
 
@@ -104,6 +104,11 @@ class _LinearRegressionParams(_PredictorParams, HasRegParam, HasElasticNetParam,
                     "robustness. Must be > 1.0. Only valid when loss is huber",
                     typeConverter=TypeConverters.toFloat)
 
+    def __init__(self, *args):
+        super(_LinearRegressionParams, self).__init__(*args)
+        self._setDefault(maxIter=100, regParam=0.0, tol=1e-6, loss="squaredError", epsilon=1.35,
+                         maxBlockSizeInMB=0.0)
+
     @since("2.3.0")
     def getEpsilon(self):
         """
@@ -131,8 +136,14 @@ class LinearRegression(_JavaRegressor, _LinearRegressionParams, JavaMLWritable,
     * L1 (Lasso)
     * L2 + L1 (elastic net)
 
-    Note: Fitting with huber loss only supports none and L2 regularization.
+    .. versionadded:: 1.4.0
 
+    Notes
+    -----
+    Fitting with huber loss only supports none and L2 regularization.
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> df = spark.createDataFrame([
     ...     (1.0, 2.0, Vectors.dense(1.0)),
@@ -155,8 +166,8 @@ class LinearRegression(_JavaRegressor, _LinearRegressionParams, JavaMLWritable,
     LinearRegressionModel...
     >>> model.getMaxIter()
     5
-    >>> model.getBlockSize()
-    1
+    >>> model.getMaxBlockSizeInMB()
+    0.0
     >>> test0 = spark.createDataFrame([(Vectors.dense(-1.0),)], ["features"])
     >>> abs(model.predict(test0.head().features) - (-1.0)) < 0.001
     True
@@ -169,10 +180,8 @@ class LinearRegression(_JavaRegressor, _LinearRegressionParams, JavaMLWritable,
     >>> test1 = spark.createDataFrame([(Vectors.sparse(1, [0], [1.0]),)], ["features"])
     >>> abs(model.transform(test1).head().newPrediction - 1.0) < 0.001
     True
-    >>> lr.setParams("vector")
-    Traceback (most recent call last):
-        ...
-    TypeError: Method setParams forces keyword arguments.
+    >>> lr.setParams(featuresCol="vector")
+    LinearRegression...
     >>> lr_path = temp_path + "/lr"
     >>> lr.save(lr_path)
     >>> lr2 = LinearRegression.load(lr_path)
@@ -185,43 +194,41 @@ class LinearRegression(_JavaRegressor, _LinearRegressionParams, JavaMLWritable,
     True
     >>> model.intercept == model2.intercept
     True
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     >>> model.numFeatures
     1
     >>> model.write().format("pmml").save(model_path + "_2")
-
-    .. versionadded:: 1.4.0
     """
 
     @keyword_only
-    def __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def __init__(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                  maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True,
                  standardization=True, solver="auto", weightCol=None, aggregationDepth=2,
-                 loss="squaredError", epsilon=1.35, blockSize=1):
+                 loss="squaredError", epsilon=1.35, maxBlockSizeInMB=0.0):
         """
-        __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                  maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True, \
                  standardization=True, solver="auto", weightCol=None, aggregationDepth=2, \
-                 loss="squaredError", epsilon=1.35, blockSize=1)
+                 loss="squaredError", epsilon=1.35, maxBlockSizeInMB=0.0)
         """
         super(LinearRegression, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.regression.LinearRegression", self.uid)
-        self._setDefault(maxIter=100, regParam=0.0, tol=1e-6, loss="squaredError", epsilon=1.35,
-                         blockSize=1)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("1.4.0")
-    def setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def setParams(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                   maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True,
                   standardization=True, solver="auto", weightCol=None, aggregationDepth=2,
-                  loss="squaredError", epsilon=1.35, blockSize=1):
+                  loss="squaredError", epsilon=1.35, maxBlockSizeInMB=0.0):
         """
-        setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                   maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True, \
                   standardization=True, solver="auto", weightCol=None, aggregationDepth=2, \
-                  loss="squaredError", epsilon=1.35, blockSize=1)
+                  loss="squaredError", epsilon=1.35, maxBlockSizeInMB=0.0)
         Sets params for linear regression.
         """
         kwargs = self._input_kwargs
@@ -298,11 +305,11 @@ def setLoss(self, value):
         return self._set(lossType=value)
 
     @since("3.1.0")
-    def setBlockSize(self, value):
+    def setMaxBlockSizeInMB(self, value):
         """
-        Sets the value of :py:attr:`blockSize`.
+        Sets the value of :py:attr:`maxBlockSizeInMB`.
         """
-        return self._set(blockSize=value)
+        return self._set(maxBlockSizeInMB=value)
 
 
 class LinearRegressionModel(_JavaRegressionModel, _LinearRegressionParams, GeneralJavaMLWritable,
@@ -341,7 +348,7 @@ def scale(self):
     @since("2.0.0")
     def summary(self):
         """
-        Gets summary (e.g. residuals, mse, r-squared ) of model on
+        Gets summary (residuals, MSE, r-squared ) of model on
         training set. An exception is thrown if
         `trainingSummary is None`.
         """
@@ -351,17 +358,20 @@ def summary(self):
             raise RuntimeError("No training summary available for this %s" %
                                self.__class__.__name__)
 
-    @since("2.0.0")
     def evaluate(self, dataset):
         """
         Evaluates the model on a test dataset.
 
-        :param dataset:
-          Test dataset to evaluate model on, where dataset is an
-          instance of :py:class:`pyspark.sql.DataFrame`
+        .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            Test dataset to evaluate model on, where dataset is an
+            instance of :py:class:`pyspark.sql.DataFrame`
         """
         if not isinstance(dataset, DataFrame):
-            raise ValueError("dataset must be a DataFrame but got %s." % type(dataset))
+            raise TypeError("dataset must be a DataFrame but got %s." % type(dataset))
         java_lr_summary = self._call_java("evaluate", dataset)
         return LinearRegressionSummary(java_lr_summary)
 
@@ -415,12 +425,15 @@ def explainedVariance(self):
         Returns the explained variance regression score.
         explainedVariance = :math:`1 - \frac{variance(y - \hat{y})}{variance(y)}`
 
-        .. seealso:: `Wikipedia explain variation
-            <http://en.wikipedia.org/wiki/Explained_variation>`_
+        Notes
+        -----
+        This ignores instance weights (setting all to 1.0) from
+        `LinearRegression.weightCol`. This will change in later Spark
+        versions.
 
-        .. note:: This ignores instance weights (setting all to 1.0) from
-            `LinearRegression.weightCol`. This will change in later Spark
-            versions.
+        For additional information see
+        `Explained variation on Wikipedia \
+        <http://en.wikipedia.org/wiki/Explained_variation>`_
         """
         return self._call_java("explainedVariance")
 
@@ -432,9 +445,11 @@ def meanAbsoluteError(self):
         corresponding to the expected value of the absolute error
         loss or l1-norm loss.
 
-        .. note:: This ignores instance weights (setting all to 1.0) from
-            `LinearRegression.weightCol`. This will change in later Spark
-            versions.
+        Notes
+        -----
+        This ignores instance weights (setting all to 1.0) from
+        `LinearRegression.weightCol`. This will change in later Spark
+        versions.
         """
         return self._call_java("meanAbsoluteError")
 
@@ -446,9 +461,11 @@ def meanSquaredError(self):
         corresponding to the expected value of the squared error
         loss or quadratic loss.
 
-        .. note:: This ignores instance weights (setting all to 1.0) from
-            `LinearRegression.weightCol`. This will change in later Spark
-            versions.
+        Notes
+        -----
+        This ignores instance weights (setting all to 1.0) from
+        `LinearRegression.weightCol`. This will change in later Spark
+        versions.
         """
         return self._call_java("meanSquaredError")
 
@@ -459,9 +476,11 @@ def rootMeanSquaredError(self):
         Returns the root mean squared error, which is defined as the
         square root of the mean squared error.
 
-        .. note:: This ignores instance weights (setting all to 1.0) from
-            `LinearRegression.weightCol`. This will change in later Spark
-            versions.
+        Notes
+        -----
+        This ignores instance weights (setting all to 1.0) from
+        `LinearRegression.weightCol`. This will change in later Spark
+        versions.
         """
         return self._call_java("rootMeanSquaredError")
 
@@ -471,12 +490,14 @@ def r2(self):
         """
         Returns R^2, the coefficient of determination.
 
-        .. seealso:: `Wikipedia coefficient of determination
-            <http://en.wikipedia.org/wiki/Coefficient_of_determination>`_
+        Notes
+        -----
+        This ignores instance weights (setting all to 1.0) from
+        `LinearRegression.weightCol`. This will change in later Spark
+        versions.
 
-        .. note:: This ignores instance weights (setting all to 1.0) from
-            `LinearRegression.weightCol`. This will change in later Spark
-            versions.
+        See also `Wikipedia coefficient of determination \
+        <http://en.wikipedia.org/wiki/Coefficient_of_determination>`_
         """
         return self._call_java("r2")
 
@@ -486,11 +507,13 @@ def r2adj(self):
         """
         Returns Adjusted R^2, the adjusted coefficient of determination.
 
-        .. seealso:: `Wikipedia coefficient of determination, Adjusted R^2
-            <https://en.wikipedia.org/wiki/Coefficient_of_determination#Adjusted_R2>`_
+        Notes
+        -----
+        This ignores instance weights (setting all to 1.0) from
+        `LinearRegression.weightCol`. This will change in later Spark versions.
 
-        .. note:: This ignores instance weights (setting all to 1.0) from
-            `LinearRegression.weightCol`. This will change in later Spark versions.
+        `Wikipedia coefficient of determination, Adjusted R^2 \
+        <https://en.wikipedia.org/wiki/Coefficient_of_determination#Adjusted_R2>`_
         """
         return self._call_java("r2adj")
 
@@ -528,7 +551,6 @@ def devianceResiduals(self):
         return self._call_java("devianceResiduals")
 
     @property
-    @since("2.0.0")
     def coefficientStandardErrors(self):
         """
         Standard error of estimated coefficients and intercept.
@@ -537,12 +559,15 @@ def coefficientStandardErrors(self):
         If :py:attr:`LinearRegression.fitIntercept` is set to True,
         then the last element returned corresponds to the intercept.
 
-        .. seealso:: :py:attr:`LinearRegression.solver`
+        .. versionadded:: 2.0.0
+
+        See Also
+        --------
+        LinearRegression.solver
         """
         return self._call_java("coefficientStandardErrors")
 
     @property
-    @since("2.0.0")
     def tValues(self):
         """
         T-statistic of estimated coefficients and intercept.
@@ -551,12 +576,15 @@ def tValues(self):
         If :py:attr:`LinearRegression.fitIntercept` is set to True,
         then the last element returned corresponds to the intercept.
 
-        .. seealso:: :py:attr:`LinearRegression.solver`
+        .. versionadded:: 2.0.0
+
+        See Also
+        --------
+        LinearRegression.solver
         """
         return self._call_java("tValues")
 
     @property
-    @since("2.0.0")
     def pValues(self):
         """
         Two-sided p-value of estimated coefficients and intercept.
@@ -565,7 +593,11 @@ def pValues(self):
         If :py:attr:`LinearRegression.fitIntercept` is set to True,
         then the last element returned corresponds to the intercept.
 
-        .. seealso:: :py:attr:`LinearRegression.solver`
+        .. versionadded:: 2.0.0
+
+        See Also
+        --------
+        LinearRegression.solver
         """
         return self._call_java("pValues")
 
@@ -580,25 +612,31 @@ class LinearRegressionTrainingSummary(LinearRegressionSummary):
     """
 
     @property
-    @since("2.0.0")
     def objectiveHistory(self):
         """
         Objective function (scaled loss + regularization) at each
         iteration.
         This value is only available when using the "l-bfgs" solver.
 
-        .. seealso:: :py:attr:`LinearRegression.solver`
+        .. versionadded:: 2.0.0
+
+        See Also
+        --------
+        LinearRegression.solver
         """
         return self._call_java("objectiveHistory")
 
     @property
-    @since("2.0.0")
     def totalIterations(self):
         """
         Number of training iterations until termination.
         This value is only available when using the "l-bfgs" solver.
 
-        .. seealso:: :py:attr:`LinearRegression.solver`
+        .. versionadded:: 2.0.0
+
+        See Also
+        --------
+        LinearRegression.solver
         """
         return self._call_java("totalIterations")
 
@@ -619,6 +657,10 @@ class _IsotonicRegressionParams(HasFeaturesCol, HasLabelCol, HasPredictionCol, H
         "The index of the feature if featuresCol is a vector column, no effect otherwise.",
         typeConverter=TypeConverters.toInt)
 
+    def __init__(self, *args):
+        super(_IsotonicRegressionParams, self).__init__(*args)
+        self._setDefault(isotonic=True, featureIndex=0)
+
     def getIsotonic(self):
         """
         Gets the value of isotonic or its default value.
@@ -639,6 +681,10 @@ class IsotonicRegression(JavaEstimator, _IsotonicRegressionParams, HasWeightCol,
     Currently implemented using parallelized pool adjacent violators algorithm.
     Only univariate (single feature) algorithm supported.
 
+    .. versionadded:: 1.6.0
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> df = spark.createDataFrame([
     ...     (1.0, Vectors.dense(1.0)),
@@ -668,28 +714,27 @@ class IsotonicRegression(JavaEstimator, _IsotonicRegressionParams, HasWeightCol,
     True
     >>> model.predictions == model2.predictions
     True
-
-    .. versionadded:: 1.6.0
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     """
     @keyword_only
-    def __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def __init__(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                  weightCol=None, isotonic=True, featureIndex=0):
         """
-        __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                  weightCol=None, isotonic=True, featureIndex=0):
         """
         super(IsotonicRegression, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.regression.IsotonicRegression", self.uid)
-        self._setDefault(isotonic=True, featureIndex=0)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
-    def setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def setParams(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                   weightCol=None, isotonic=True, featureIndex=0):
         """
-        setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                  weightCol=None, isotonic=True, featureIndex=0):
         Set the params for IsotonicRegression.
         """
@@ -808,7 +853,11 @@ class _DecisionTreeRegressorParams(_DecisionTreeParams, _TreeRegressorParams, Ha
     .. versionadded:: 3.0.0
     """
 
-    pass
+    def __init__(self, *args):
+        super(_DecisionTreeRegressorParams, self).__init__(*args)
+        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
+                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
+                         impurity="variance", leafCol="", minWeightFractionPerNode=0.0)
 
 
 @inherit_doc
@@ -819,6 +868,10 @@ class DecisionTreeRegressor(_JavaRegressor, _DecisionTreeRegressorParams, JavaML
     learning algorithm for regression.
     It supports both continuous and categorical features.
 
+    .. versionadded:: 1.4.0
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> df = spark.createDataFrame([
     ...     (1.0, Vectors.dense(1.0)),
@@ -866,7 +919,8 @@ class DecisionTreeRegressor(_JavaRegressor, _DecisionTreeRegressorParams, JavaML
     True
     >>> model.transform(test1).head().variance
     0.0
-
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     >>> df3 = spark.createDataFrame([
     ...     (1.0, 0.2, Vectors.dense(1.0)),
     ...     (1.0, 0.8, Vectors.dense(1.0)),
@@ -875,18 +929,16 @@ class DecisionTreeRegressor(_JavaRegressor, _DecisionTreeRegressorParams, JavaML
     >>> model3 = dt3.fit(df3)
     >>> print(model3.toDebugString)
     DecisionTreeRegressionModel...depth=1, numNodes=3...
-
-    .. versionadded:: 1.4.0
     """
 
     @keyword_only
-    def __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def __init__(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                  maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
                  maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, impurity="variance",
                  seed=None, varianceCol=None, weightCol=None, leafCol="",
                  minWeightFractionPerNode=0.0):
         """
-        __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                  maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \
                  maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, \
                  impurity="variance", seed=None, varianceCol=None, weightCol=None, \
@@ -895,21 +947,18 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(DecisionTreeRegressor, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.regression.DecisionTreeRegressor", self.uid)
-        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
-                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
-                         impurity="variance", leafCol="", minWeightFractionPerNode=0.0)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("1.4.0")
-    def setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def setParams(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                   maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
                   maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
                   impurity="variance", seed=None, varianceCol=None, weightCol=None,
                   leafCol="", minWeightFractionPerNode=0.0):
         """
-        setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                   maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \
                   maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, \
                   impurity="variance", seed=None, varianceCol=None, weightCol=None, \
@@ -1025,7 +1074,6 @@ def setVarianceCol(self, value):
         return self._set(varianceCol=value)
 
     @property
-    @since("2.0.0")
     def featureImportances(self):
         """
         Estimate of the importance of each feature.
@@ -1039,9 +1087,13 @@ def featureImportances(self):
             where gain is scaled by the number of instances passing through node
           - Normalize importances for tree to sum to 1.
 
-        .. note:: Feature importance for single decision trees can have high variance due to
-              correlated predictor variables. Consider using a :py:class:`RandomForestRegressor`
-              to determine feature importance instead.
+        .. versionadded:: 2.0.0
+
+        Notes
+        -----
+        Feature importance for single decision trees can have high variance due to
+        correlated predictor variables. Consider using a :py:class:`RandomForestRegressor`
+        to determine feature importance instead.
         """
         return self._call_java("featureImportances")
 
@@ -1052,7 +1104,14 @@ class _RandomForestRegressorParams(_RandomForestParams, _TreeRegressorParams):
 
     .. versionadded:: 3.0.0
     """
-    pass
+
+    def __init__(self, *args):
+        super(_RandomForestRegressorParams, self).__init__(*args)
+        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
+                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
+                         impurity="variance", subsamplingRate=1.0, numTrees=20,
+                         featureSubsetStrategy="auto", leafCol="", minWeightFractionPerNode=0.0,
+                         bootstrap=True)
 
 
 @inherit_doc
@@ -1063,6 +1122,10 @@ class RandomForestRegressor(_JavaRegressor, _RandomForestRegressorParams, JavaML
     learning algorithm for regression.
     It supports both continuous and categorical features.
 
+    .. versionadded:: 1.4.0
+
+    Examples
+    --------
     >>> from numpy import allclose
     >>> from pyspark.ml.linalg import Vectors
     >>> df = spark.createDataFrame([
@@ -1113,19 +1176,19 @@ class RandomForestRegressor(_JavaRegressor, _RandomForestRegressorParams, JavaML
     >>> model2 = RandomForestRegressionModel.load(model_path)
     >>> model.featureImportances == model2.featureImportances
     True
-
-    .. versionadded:: 1.4.0
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     """
 
     @keyword_only
-    def __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def __init__(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                  maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
                  maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
                  impurity="variance", subsamplingRate=1.0, seed=None, numTrees=20,
                  featureSubsetStrategy="auto", leafCol="", minWeightFractionPerNode=0.0,
                  weightCol=None, bootstrap=True):
         """
-        __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                  maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \
                  maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, \
                  impurity="variance", subsamplingRate=1.0, seed=None, numTrees=20, \
@@ -1135,24 +1198,19 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(RandomForestRegressor, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.regression.RandomForestRegressor", self.uid)
-        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
-                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
-                         impurity="variance", subsamplingRate=1.0, numTrees=20,
-                         featureSubsetStrategy="auto", leafCol="", minWeightFractionPerNode=0.0,
-                         bootstrap=True)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("1.4.0")
-    def setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def setParams(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                   maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
                   maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
                   impurity="variance", subsamplingRate=1.0, seed=None, numTrees=20,
                   featureSubsetStrategy="auto", leafCol="", minWeightFractionPerNode=0.0,
                   weightCol=None, bootstrap=True):
         """
-        setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                   maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \
                   maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, \
                   impurity="variance", subsamplingRate=1.0, seed=None, numTrees=20, \
@@ -1281,7 +1339,6 @@ def trees(self):
         return [DecisionTreeRegressionModel(m) for m in list(self._call_java("trees"))]
 
     @property
-    @since("2.0.0")
     def featureImportances(self):
         """
         Estimate of the importance of each feature.
@@ -1291,7 +1348,11 @@ def featureImportances(self):
         (Hastie, Tibshirani, Friedman. "The Elements of Statistical Learning, 2nd Edition." 2001.)
         and follows the implementation from scikit-learn.
 
-        .. seealso:: :py:attr:`DecisionTreeRegressionModel.featureImportances`
+        .. versionadded:: 2.0.0
+
+        Examples
+        --------
+        DecisionTreeRegressionModel.featureImportances
         """
         return self._call_java("featureImportances")
 
@@ -1310,6 +1371,14 @@ class _GBTRegressorParams(_GBTParams, _TreeRegressorParams):
                      "Supported options: " + ", ".join(supportedLossTypes),
                      typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_GBTRegressorParams, self).__init__(*args)
+        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
+                         maxMemoryInMB=256, cacheNodeIds=False, subsamplingRate=1.0,
+                         checkpointInterval=10, lossType="squared", maxIter=20, stepSize=0.1,
+                         impurity="variance", featureSubsetStrategy="all", validationTol=0.01,
+                         leafCol="", minWeightFractionPerNode=0.0)
+
     @since("1.4.0")
     def getLossType(self):
         """
@@ -1325,6 +1394,10 @@ class GBTRegressor(_JavaRegressor, _GBTRegressorParams, JavaMLWritable, JavaMLRe
     learning algorithm for regression.
     It supports both continuous and categorical features.
 
+    .. versionadded:: 1.4.0
+
+    Examples
+    --------
     >>> from numpy import allclose
     >>> from pyspark.ml.linalg import Vectors
     >>> df = spark.createDataFrame([
@@ -1373,6 +1446,8 @@ class GBTRegressor(_JavaRegressor, _GBTRegressorParams, JavaMLWritable, JavaMLRe
     True
     >>> model.treeWeights == model2.treeWeights
     True
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     >>> model.trees
     [DecisionTreeRegressionModel...depth=..., DecisionTreeRegressionModel...]
     >>> validation = spark.createDataFrame([(0.0, Vectors.dense(-1.0))],
@@ -1384,12 +1459,10 @@ class GBTRegressor(_JavaRegressor, _GBTRegressorParams, JavaMLWritable, JavaMLRe
     'validationIndicator'
     >>> gbt.getValidationTol()
     0.01
-
-    .. versionadded:: 1.4.0
     """
 
     @keyword_only
-    def __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def __init__(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                  maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
                  maxMemoryInMB=256, cacheNodeIds=False, subsamplingRate=1.0,
                  checkpointInterval=10, lossType="squared", maxIter=20, stepSize=0.1, seed=None,
@@ -1397,7 +1470,7 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
                  validationIndicatorCol=None, leafCol="", minWeightFractionPerNode=0.0,
                  weightCol=None):
         """
-        __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                  maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \
                  maxMemoryInMB=256, cacheNodeIds=False, subsamplingRate=1.0, \
                  checkpointInterval=10, lossType="squared", maxIter=20, stepSize=0.1, seed=None, \
@@ -1407,25 +1480,20 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         """
         super(GBTRegressor, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.regression.GBTRegressor", self.uid)
-        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
-                         maxMemoryInMB=256, cacheNodeIds=False, subsamplingRate=1.0,
-                         checkpointInterval=10, lossType="squared", maxIter=20, stepSize=0.1,
-                         impurity="variance", featureSubsetStrategy="all", validationTol=0.01,
-                         leafCol="", minWeightFractionPerNode=0.0)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("1.4.0")
-    def setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def setParams(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                   maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
                   maxMemoryInMB=256, cacheNodeIds=False, subsamplingRate=1.0,
                   checkpointInterval=10, lossType="squared", maxIter=20, stepSize=0.1, seed=None,
-                  impuriy="variance", featureSubsetStrategy="all", validationTol=0.01,
+                  impurity="variance", featureSubsetStrategy="all", validationTol=0.01,
                   validationIndicatorCol=None, leafCol="", minWeightFractionPerNode=0.0,
                   weightCol=None):
         """
-        setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                   maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, \
                   maxMemoryInMB=256, cacheNodeIds=False, subsamplingRate=1.0, \
                   checkpointInterval=10, lossType="squared", maxIter=20, stepSize=0.1, seed=None, \
@@ -1571,7 +1639,6 @@ class GBTRegressionModel(
     """
 
     @property
-    @since("2.0.0")
     def featureImportances(self):
         """
         Estimate of the importance of each feature.
@@ -1581,7 +1648,11 @@ def featureImportances(self):
         (Hastie, Tibshirani, Friedman. "The Elements of Statistical Learning, 2nd Edition." 2001.)
         and follows the implementation from scikit-learn.
 
-        .. seealso:: :py:attr:`DecisionTreeRegressionModel.featureImportances`
+        .. versionadded:: 2.0.0
+
+        Examples
+        --------
+        DecisionTreeRegressionModel.featureImportances
         """
         return self._call_java("featureImportances")
 
@@ -1591,15 +1662,18 @@ def trees(self):
         """Trees in this ensemble. Warning: These have null parent Estimators."""
         return [DecisionTreeRegressionModel(m) for m in list(self._call_java("trees"))]
 
-    @since("2.4.0")
     def evaluateEachIteration(self, dataset, loss):
         """
         Method to compute error or loss for every iteration of gradient boosting.
 
-        :param dataset:
+        .. versionadded:: 2.4.0
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
             Test dataset to evaluate model on, where dataset is an
             instance of :py:class:`pyspark.sql.DataFrame`
-        :param loss:
+        loss : str
             The loss function used to compute error.
             Supported options: squared, absolute
         """
@@ -1607,7 +1681,7 @@ def evaluateEachIteration(self, dataset, loss):
 
 
 class _AFTSurvivalRegressionParams(_PredictorParams, HasMaxIter, HasTol, HasFitIntercept,
-                                   HasAggregationDepth, HasBlockSize):
+                                   HasAggregationDepth, HasMaxBlockSizeInMB):
     """
     Params for :py:class:`AFTSurvivalRegression` and :py:class:`AFTSurvivalRegressionModel`.
 
@@ -1630,6 +1704,12 @@ class _AFTSurvivalRegressionParams(_PredictorParams, HasMaxIter, HasTol, HasFitI
         "corresponding quantileProbabilities if it is set.",
         typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_AFTSurvivalRegressionParams, self).__init__(*args)
+        self._setDefault(censorCol="censor",
+                         quantileProbabilities=[0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99],
+                         maxIter=100, tol=1E-6, maxBlockSizeInMB=0.0)
+
     @since("1.6.0")
     def getCensorCol(self):
         """
@@ -1661,8 +1741,14 @@ class AFTSurvivalRegression(_JavaRegressor, _AFTSurvivalRegressionParams,
     Fit a parametric AFT survival regression model based on the Weibull distribution
     of the survival time.
 
-    .. seealso:: `AFT Model <https://en.wikipedia.org/wiki/Accelerated_failure_time_model>`_
+    Notes
+    -----
+    For more information see Wikipedia page on
+    `AFT Model <https://en.wikipedia.org/wiki/Accelerated_failure_time_model>`_
 
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> df = spark.createDataFrame([
     ...     (1.0, Vectors.dense(1.0), 1.0),
@@ -1674,8 +1760,8 @@ class AFTSurvivalRegression(_JavaRegressor, _AFTSurvivalRegressionParams,
     10
     >>> aftsr.clear(aftsr.maxIter)
     >>> model = aftsr.fit(df)
-    >>> model.getBlockSize()
-    1
+    >>> model.getMaxBlockSizeInMB()
+    0.0
     >>> model.setFeaturesCol("features")
     AFTSurvivalRegressionModel...
     >>> model.predict(Vectors.dense(6.3))
@@ -1704,41 +1790,40 @@ class AFTSurvivalRegression(_JavaRegressor, _AFTSurvivalRegressionParams,
     True
     >>> model.scale == model2.scale
     True
+    >>> model.transform(df).take(1) == model2.transform(df).take(1)
+    True
 
     .. versionadded:: 1.6.0
     """
 
     @keyword_only
-    def __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def __init__(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                  fitIntercept=True, maxIter=100, tol=1E-6, censorCol="censor",
-                 quantileProbabilities=list([0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99]),
-                 quantilesCol=None, aggregationDepth=2, blockSize=1):
+                 quantileProbabilities=list([0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99]),  # noqa: B005
+                 quantilesCol=None, aggregationDepth=2, maxBlockSizeInMB=0.0):
         """
-        __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                  fitIntercept=True, maxIter=100, tol=1E-6, censorCol="censor", \
                  quantileProbabilities=[0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99], \
-                 quantilesCol=None, aggregationDepth=2, blockSize=1)
+                 quantilesCol=None, aggregationDepth=2, maxBlockSizeInMB=0.0)
         """
         super(AFTSurvivalRegression, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.regression.AFTSurvivalRegression", self.uid)
-        self._setDefault(censorCol="censor",
-                         quantileProbabilities=[0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99],
-                         maxIter=100, tol=1E-6, blockSize=1)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("1.6.0")
-    def setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def setParams(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                   fitIntercept=True, maxIter=100, tol=1E-6, censorCol="censor",
-                  quantileProbabilities=list([0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99]),
-                  quantilesCol=None, aggregationDepth=2, blockSize=1):
+                  quantileProbabilities=list([0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99]),  # noqa: B005
+                  quantilesCol=None, aggregationDepth=2, maxBlockSizeInMB=0.0):
         """
-        setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                   fitIntercept=True, maxIter=100, tol=1E-6, censorCol="censor", \
                   quantileProbabilities=[0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99], \
-                  quantilesCol=None, aggregationDepth=2, blockSize=1):
+                  quantilesCol=None, aggregationDepth=2, maxBlockSizeInMB=0.0):
         """
         kwargs = self._input_kwargs
         return self._set(**kwargs)
@@ -1796,11 +1881,11 @@ def setAggregationDepth(self, value):
         return self._set(aggregationDepth=value)
 
     @since("3.1.0")
-    def setBlockSize(self, value):
+    def setMaxBlockSizeInMB(self, value):
         """
-        Sets the value of :py:attr:`blockSize`.
+        Sets the value of :py:attr:`maxBlockSizeInMB`.
         """
-        return self._set(blockSize=value)
+        return self._set(maxBlockSizeInMB=value)
 
 
 class AFTSurvivalRegressionModel(_JavaRegressionModel, _AFTSurvivalRegressionParams,
@@ -1891,6 +1976,11 @@ class _GeneralizedLinearRegressionParams(_PredictorParams, HasFitIntercept, HasM
                       "or empty, we treat all instance offsets as 0.0",
                       typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_GeneralizedLinearRegressionParams, self).__init__(*args)
+        self._setDefault(family="gaussian", maxIter=25, tol=1e-6, regParam=0.0, solver="irls",
+                         variancePower=0.0, aggregationDepth=2)
+
     @since("2.0.0")
     def getFamily(self):
         """
@@ -1956,8 +2046,15 @@ class GeneralizedLinearRegression(_JavaRegressor, _GeneralizedLinearRegressionPa
     * "tweedie"  -> power link function specified through "linkPower". \
                     The default link power in the tweedie family is 1 - variancePower.
 
-    .. seealso:: `GLM <https://en.wikipedia.org/wiki/Generalized_linear_model>`_
+    .. versionadded:: 2.0.0
+
+    Notes
+    -----
+    For more information see Wikipedia page on
+    `GLM <https://en.wikipedia.org/wiki/Generalized_linear_model>`_
 
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> df = spark.createDataFrame([
     ...     (1.0, Vectors.dense(0.0, 0.0)),
@@ -2005,17 +2102,17 @@ class GeneralizedLinearRegression(_JavaRegressor, _GeneralizedLinearRegressionPa
     True
     >>> model.coefficients[0] == model2.coefficients[0]
     True
-
-    .. versionadded:: 2.0.0
+    >>> model.transform(df).take(1) == model2.transform(df).take(1)
+    True
     """
 
     @keyword_only
-    def __init__(self, labelCol="label", featuresCol="features", predictionCol="prediction",
+    def __init__(self, *, labelCol="label", featuresCol="features", predictionCol="prediction",
                  family="gaussian", link=None, fitIntercept=True, maxIter=25, tol=1e-6,
                  regParam=0.0, weightCol=None, solver="irls", linkPredictionCol=None,
                  variancePower=0.0, linkPower=None, offsetCol=None, aggregationDepth=2):
         """
-        __init__(self, labelCol="label", featuresCol="features", predictionCol="prediction", \
+        __init__(self, \\*, labelCol="label", featuresCol="features", predictionCol="prediction", \
                  family="gaussian", link=None, fitIntercept=True, maxIter=25, tol=1e-6, \
                  regParam=0.0, weightCol=None, solver="irls", linkPredictionCol=None, \
                  variancePower=0.0, linkPower=None, offsetCol=None, aggregationDepth=2)
@@ -2023,20 +2120,18 @@ def __init__(self, labelCol="label", featuresCol="features", predictionCol="pred
         super(GeneralizedLinearRegression, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.regression.GeneralizedLinearRegression", self.uid)
-        self._setDefault(family="gaussian", maxIter=25, tol=1e-6, regParam=0.0, solver="irls",
-                         variancePower=0.0, aggregationDepth=2)
         kwargs = self._input_kwargs
 
         self.setParams(**kwargs)
 
     @keyword_only
     @since("2.0.0")
-    def setParams(self, labelCol="label", featuresCol="features", predictionCol="prediction",
+    def setParams(self, *, labelCol="label", featuresCol="features", predictionCol="prediction",
                   family="gaussian", link=None, fitIntercept=True, maxIter=25, tol=1e-6,
                   regParam=0.0, weightCol=None, solver="irls", linkPredictionCol=None,
                   variancePower=0.0, linkPower=None, offsetCol=None, aggregationDepth=2):
         """
-        setParams(self, labelCol="label", featuresCol="features", predictionCol="prediction", \
+        setParams(self, \\*, labelCol="label", featuresCol="features", predictionCol="prediction", \
                   family="gaussian", link=None, fitIntercept=True, maxIter=25, tol=1e-6, \
                   regParam=0.0, weightCol=None, solver="irls", linkPredictionCol=None, \
                   variancePower=0.0, linkPower=None, offsetCol=None, aggregationDepth=2)
@@ -2175,7 +2270,7 @@ def intercept(self):
     @since("2.0.0")
     def summary(self):
         """
-        Gets summary (e.g. residuals, deviance, pValues) of model on
+        Gets summary (residuals, deviance, p-values) of model on
         training set. An exception is thrown if
         `trainingSummary is None`.
         """
@@ -2186,17 +2281,20 @@ def summary(self):
             raise RuntimeError("No training summary available for this %s" %
                                self.__class__.__name__)
 
-    @since("2.0.0")
     def evaluate(self, dataset):
         """
         Evaluates the model on a test dataset.
 
-        :param dataset:
-          Test dataset to evaluate model on, where dataset is an
-          instance of :py:class:`pyspark.sql.DataFrame`
+        .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            Test dataset to evaluate model on, where dataset is an
+            instance of :py:class:`pyspark.sql.DataFrame`
         """
         if not isinstance(dataset, DataFrame):
-            raise ValueError("dataset must be a DataFrame but got %s." % type(dataset))
+            raise TypeError("dataset must be a DataFrame but got %s." % type(dataset))
         java_glr_summary = self._call_java("evaluate", dataset)
         return GeneralizedLinearRegressionSummary(java_glr_summary)
 
@@ -2265,13 +2363,17 @@ def residualDegreeOfFreedomNull(self):
         """
         return self._call_java("residualDegreeOfFreedomNull")
 
-    @since("2.0.0")
     def residuals(self, residualsType="deviance"):
         """
         Get the residuals of the fitted model by type.
 
-        :param residualsType: The type of residuals which should be returned.
-                              Supported options: deviance (default), pearson, working, and response.
+        .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        residualsType : str, optional
+            The type of residuals which should be returned.
+            Supported options: deviance (default), pearson, working, and response.
         """
         return self._call_java("residuals", residualsType)
 
@@ -2373,7 +2475,7 @@ def __repr__(self):
 
 
 class _FactorizationMachinesParams(_PredictorParams, HasMaxIter, HasStepSize, HasTol,
-                                   HasSolver, HasSeed, HasFitIntercept, HasRegParam):
+                                   HasSolver, HasSeed, HasFitIntercept, HasRegParam, HasWeightCol):
     """
     Params for :py:class:`FMRegressor`, :py:class:`FMRegressionModel`, :py:class:`FMClassifier`
     and :py:class:`FMClassifierModel`.
@@ -2398,6 +2500,12 @@ class _FactorizationMachinesParams(_PredictorParams, HasMaxIter, HasStepSize, Ha
     solver = Param(Params._dummy(), "solver", "The solver algorithm for optimization. Supported " +
                    "options: gd, adamW. (Default adamW)", typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_FactorizationMachinesParams, self).__init__(*args)
+        self._setDefault(factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0,
+                         miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0,
+                         tol=1e-6, solver="adamW")
+
     @since("3.0.0")
     def getFactorSize(self):
         """
@@ -2437,6 +2545,10 @@ class FMRegressor(_JavaRegressor, _FactorizationMachinesParams, JavaMLWritable,
     * gd (normal mini-batch gradient descent)
     * adamW (default)
 
+    .. versionadded:: 3.0.0
+
+    Examples
+    --------
     >>> from pyspark.ml.linalg import Vectors
     >>> from pyspark.ml.regression import FMRegressor
     >>> df = spark.createDataFrame([
@@ -2471,17 +2583,26 @@ class FMRegressor(_JavaRegressor, _FactorizationMachinesParams, JavaMLWritable,
     DenseVector([0.9978])
     >>> model.factors
     DenseMatrix(1, 2, [0.0173, 0.0021], 1)
-
-    .. versionadded:: 3.0.0
+    >>> model_path = temp_path + "/fm_model"
+    >>> model.save(model_path)
+    >>> model2 = FMRegressionModel.load(model_path)
+    >>> model2.intercept
+    -0.0032501766849261557
+    >>> model2.linear
+    DenseVector([0.9978])
+    >>> model2.factors
+    DenseMatrix(1, 2, [0.0173, 0.0021], 1)
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     """
 
     @keyword_only
-    def __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def __init__(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                  factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0,
                  miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0,
                  tol=1e-6, solver="adamW", seed=None):
         """
-        __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                  factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0, \
                  miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0, \
                  tol=1e-6, solver="adamW", seed=None)
@@ -2489,20 +2610,17 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(FMRegressor, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.regression.FMRegressor", self.uid)
-        self._setDefault(factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0,
-                         miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0,
-                         tol=1e-6, solver="adamW")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
     @keyword_only
     @since("3.0.0")
-    def setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+    def setParams(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
                   factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0,
                   miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0,
                   tol=1e-6, solver="adamW", seed=None):
         """
-        setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+        setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                   factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0, \
                   miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0, \
                   tol=1e-6, solver="adamW", seed=None)
diff --git a/python/pyspark/ml/regression.pyi b/python/pyspark/ml/regression.pyi
new file mode 100644
index 0000000000000..61172305a3726
--- /dev/null
+++ b/python/pyspark/ml/regression.pyi
@@ -0,0 +1,825 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Any, List, Optional
+from pyspark.ml._typing import JM, M, T
+
+import abc
+from pyspark.ml import PredictionModel, Predictor
+from pyspark.ml.base import _PredictorParams
+from pyspark.ml.param.shared import (
+    HasAggregationDepth,
+    HasMaxBlockSizeInMB,
+    HasElasticNetParam,
+    HasFeaturesCol,
+    HasFitIntercept,
+    HasLabelCol,
+    HasLoss,
+    HasMaxIter,
+    HasPredictionCol,
+    HasRegParam,
+    HasSeed,
+    HasSolver,
+    HasStandardization,
+    HasStepSize,
+    HasTol,
+    HasVarianceCol,
+    HasWeightCol,
+)
+from pyspark.ml.tree import (
+    _DecisionTreeModel,
+    _DecisionTreeParams,
+    _GBTParams,
+    _RandomForestParams,
+    _TreeEnsembleModel,
+    _TreeRegressorParams,
+)
+from pyspark.ml.util import (
+    GeneralJavaMLWritable,
+    HasTrainingSummary,
+    JavaMLReadable,
+    JavaMLWritable,
+)
+from pyspark.ml.wrapper import (
+    JavaEstimator,
+    JavaModel,
+    JavaPredictionModel,
+    JavaPredictor,
+    JavaWrapper,
+)
+
+from pyspark.ml.linalg import Matrix, Vector
+from pyspark.ml.param import Param
+from pyspark.sql.dataframe import DataFrame
+
+class Regressor(Predictor[M], _PredictorParams, metaclass=abc.ABCMeta): ...
+class RegressionModel(PredictionModel[T], _PredictorParams, metaclass=abc.ABCMeta): ...
+class _JavaRegressor(Regressor, JavaPredictor[JM], metaclass=abc.ABCMeta): ...
+class _JavaRegressionModel(
+    RegressionModel, JavaPredictionModel[T], metaclass=abc.ABCMeta
+): ...
+
+class _LinearRegressionParams(
+    _PredictorParams,
+    HasRegParam,
+    HasElasticNetParam,
+    HasMaxIter,
+    HasTol,
+    HasFitIntercept,
+    HasStandardization,
+    HasWeightCol,
+    HasSolver,
+    HasAggregationDepth,
+    HasLoss,
+    HasMaxBlockSizeInMB,
+):
+    solver: Param[str]
+    loss: Param[str]
+    epsilon: Param[float]
+    def __init__(self, *args: Any): ...
+    def getEpsilon(self) -> float: ...
+
+class LinearRegression(
+    _JavaRegressor[LinearRegressionModel],
+    _LinearRegressionParams,
+    JavaMLWritable,
+    JavaMLReadable[LinearRegression],
+):
+    def __init__(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        maxIter: int = ...,
+        regParam: float = ...,
+        elasticNetParam: float = ...,
+        tol: float = ...,
+        fitIntercept: bool = ...,
+        standardization: bool = ...,
+        solver: str = ...,
+        weightCol: Optional[str] = ...,
+        aggregationDepth: int = ...,
+        epsilon: float = ...,
+        maxBlockSizeInMB: float = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        maxIter: int = ...,
+        regParam: float = ...,
+        elasticNetParam: float = ...,
+        tol: float = ...,
+        fitIntercept: bool = ...,
+        standardization: bool = ...,
+        solver: str = ...,
+        weightCol: Optional[str] = ...,
+        aggregationDepth: int = ...,
+        epsilon: float = ...,
+        maxBlockSizeInMB: float = ...
+    ) -> LinearRegression: ...
+    def setEpsilon(self, value: float) -> LinearRegression: ...
+    def setMaxIter(self, value: int) -> LinearRegression: ...
+    def setRegParam(self, value: float) -> LinearRegression: ...
+    def setTol(self, value: float) -> LinearRegression: ...
+    def setElasticNetParam(self, value: float) -> LinearRegression: ...
+    def setFitIntercept(self, value: bool) -> LinearRegression: ...
+    def setStandardization(self, value: bool) -> LinearRegression: ...
+    def setWeightCol(self, value: str) -> LinearRegression: ...
+    def setSolver(self, value: str) -> LinearRegression: ...
+    def setAggregationDepth(self, value: int) -> LinearRegression: ...
+    def setLoss(self, value: str) -> LinearRegression: ...
+    def setMaxBlockSizeInMB(self, value: float) -> LinearRegression: ...
+
+class LinearRegressionModel(
+    _JavaRegressionModel[Vector],
+    _LinearRegressionParams,
+    GeneralJavaMLWritable,
+    JavaMLReadable[LinearRegressionModel],
+    HasTrainingSummary[LinearRegressionSummary],
+):
+    @property
+    def coefficients(self) -> Vector: ...
+    @property
+    def intercept(self) -> float: ...
+    @property
+    def summary(self) -> LinearRegressionTrainingSummary: ...
+    def evaluate(self, dataset: DataFrame) -> LinearRegressionSummary: ...
+
+class LinearRegressionSummary(JavaWrapper):
+    @property
+    def predictions(self) -> DataFrame: ...
+    @property
+    def predictionCol(self) -> str: ...
+    @property
+    def labelCol(self) -> str: ...
+    @property
+    def featuresCol(self) -> str: ...
+    @property
+    def explainedVariance(self) -> float: ...
+    @property
+    def meanAbsoluteError(self) -> float: ...
+    @property
+    def meanSquaredError(self) -> float: ...
+    @property
+    def rootMeanSquaredError(self) -> float: ...
+    @property
+    def r2(self) -> float: ...
+    @property
+    def r2adj(self) -> float: ...
+    @property
+    def residuals(self) -> DataFrame: ...
+    @property
+    def numInstances(self) -> int: ...
+    @property
+    def devianceResiduals(self) -> List[float]: ...
+    @property
+    def coefficientStandardErrors(self) -> List[float]: ...
+    @property
+    def tValues(self) -> List[float]: ...
+    @property
+    def pValues(self) -> List[float]: ...
+
+class LinearRegressionTrainingSummary(LinearRegressionSummary):
+    @property
+    def objectiveHistory(self) -> List[float]: ...
+    @property
+    def totalIterations(self) -> int: ...
+
+class _IsotonicRegressionParams(
+    HasFeaturesCol, HasLabelCol, HasPredictionCol, HasWeightCol
+):
+    isotonic: Param[bool]
+    featureIndex: Param[int]
+    def getIsotonic(self) -> bool: ...
+    def getFeatureIndex(self) -> int: ...
+
+class IsotonicRegression(
+    JavaEstimator[IsotonicRegressionModel],
+    _IsotonicRegressionParams,
+    HasWeightCol,
+    JavaMLWritable,
+    JavaMLReadable[IsotonicRegression],
+):
+    def __init__(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        weightCol: Optional[str] = ...,
+        isotonic: bool = ...,
+        featureIndex: int = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        weightCol: Optional[str] = ...,
+        isotonic: bool = ...,
+        featureIndex: int = ...
+    ) -> IsotonicRegression: ...
+    def setIsotonic(self, value: bool) -> IsotonicRegression: ...
+    def setFeatureIndex(self, value: int) -> IsotonicRegression: ...
+    def setFeaturesCol(self, value: str) -> IsotonicRegression: ...
+    def setPredictionCol(self, value: str) -> IsotonicRegression: ...
+    def setLabelCol(self, value: str) -> IsotonicRegression: ...
+    def setWeightCol(self, value: str) -> IsotonicRegression: ...
+
+class IsotonicRegressionModel(
+    JavaModel,
+    _IsotonicRegressionParams,
+    JavaMLWritable,
+    JavaMLReadable[IsotonicRegressionModel],
+):
+    def setFeaturesCol(self, value: str) -> IsotonicRegressionModel: ...
+    def setPredictionCol(self, value: str) -> IsotonicRegressionModel: ...
+    def setFeatureIndex(self, value: int) -> IsotonicRegressionModel: ...
+    @property
+    def boundaries(self) -> Vector: ...
+    @property
+    def predictions(self) -> Vector: ...
+    @property
+    def numFeatures(self) -> int: ...
+    def predict(self, value: float) -> float: ...
+
+class _DecisionTreeRegressorParams(
+    _DecisionTreeParams, _TreeRegressorParams, HasVarianceCol
+):
+    def __init__(self, *args: Any): ...
+
+class DecisionTreeRegressor(
+    _JavaRegressor[DecisionTreeRegressionModel],
+    _DecisionTreeRegressorParams,
+    JavaMLWritable,
+    JavaMLReadable[DecisionTreeRegressor],
+):
+    def __init__(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        maxDepth: int = ...,
+        maxBins: int = ...,
+        minInstancesPerNode: int = ...,
+        minInfoGain: float = ...,
+        maxMemoryInMB: int = ...,
+        cacheNodeIds: bool = ...,
+        checkpointInterval: int = ...,
+        impurity: str = ...,
+        seed: Optional[int] = ...,
+        varianceCol: Optional[str] = ...,
+        weightCol: Optional[str] = ...,
+        leafCol: str = ...,
+        minWeightFractionPerNode: float = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        maxDepth: int = ...,
+        maxBins: int = ...,
+        minInstancesPerNode: int = ...,
+        minInfoGain: float = ...,
+        maxMemoryInMB: int = ...,
+        cacheNodeIds: bool = ...,
+        checkpointInterval: int = ...,
+        impurity: str = ...,
+        seed: Optional[int] = ...,
+        varianceCol: Optional[str] = ...,
+        weightCol: Optional[str] = ...,
+        leafCol: str = ...,
+        minWeightFractionPerNode: float = ...
+    ) -> DecisionTreeRegressor: ...
+    def setMaxDepth(self, value: int) -> DecisionTreeRegressor: ...
+    def setMaxBins(self, value: int) -> DecisionTreeRegressor: ...
+    def setMinInstancesPerNode(self, value: int) -> DecisionTreeRegressor: ...
+    def setMinWeightFractionPerNode(self, value: float) -> DecisionTreeRegressor: ...
+    def setMinInfoGain(self, value: float) -> DecisionTreeRegressor: ...
+    def setMaxMemoryInMB(self, value: int) -> DecisionTreeRegressor: ...
+    def setCacheNodeIds(self, value: bool) -> DecisionTreeRegressor: ...
+    def setImpurity(self, value: str) -> DecisionTreeRegressor: ...
+    def setCheckpointInterval(self, value: int) -> DecisionTreeRegressor: ...
+    def setSeed(self, value: int) -> DecisionTreeRegressor: ...
+    def setWeightCol(self, value: str) -> DecisionTreeRegressor: ...
+    def setVarianceCol(self, value: str) -> DecisionTreeRegressor: ...
+
+class DecisionTreeRegressionModel(
+    _JavaRegressionModel[Vector],
+    _DecisionTreeModel,
+    _DecisionTreeRegressorParams,
+    JavaMLWritable,
+    JavaMLReadable[DecisionTreeRegressionModel],
+):
+    def setVarianceCol(self, value: str) -> DecisionTreeRegressionModel: ...
+    @property
+    def featureImportances(self) -> Vector: ...
+
+class _RandomForestRegressorParams(_RandomForestParams, _TreeRegressorParams):
+    def __init__(self, *args: Any): ...
+
+class RandomForestRegressor(
+    _JavaRegressor[RandomForestRegressionModel],
+    _RandomForestRegressorParams,
+    JavaMLWritable,
+    JavaMLReadable[RandomForestRegressor],
+):
+    def __init__(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        maxDepth: int = ...,
+        maxBins: int = ...,
+        minInstancesPerNode: int = ...,
+        minInfoGain: float = ...,
+        maxMemoryInMB: int = ...,
+        cacheNodeIds: bool = ...,
+        checkpointInterval: int = ...,
+        impurity: str = ...,
+        subsamplingRate: float = ...,
+        seed: Optional[int] = ...,
+        numTrees: int = ...,
+        featureSubsetStrategy: str = ...,
+        leafCol: str = ...,
+        minWeightFractionPerNode: float = ...,
+        weightCol: Optional[str] = ...,
+        bootstrap: Optional[bool] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        maxDepth: int = ...,
+        maxBins: int = ...,
+        minInstancesPerNode: int = ...,
+        minInfoGain: float = ...,
+        maxMemoryInMB: int = ...,
+        cacheNodeIds: bool = ...,
+        checkpointInterval: int = ...,
+        impurity: str = ...,
+        subsamplingRate: float = ...,
+        seed: Optional[int] = ...,
+        numTrees: int = ...,
+        featureSubsetStrategy: str = ...,
+        leafCol: str = ...,
+        minWeightFractionPerNode: float = ...,
+        weightCol: Optional[str] = ...,
+        bootstrap: Optional[bool] = ...
+    ) -> RandomForestRegressor: ...
+    def setMaxDepth(self, value: int) -> RandomForestRegressor: ...
+    def setMaxBins(self, value: int) -> RandomForestRegressor: ...
+    def setMinInstancesPerNode(self, value: int) -> RandomForestRegressor: ...
+    def setMinInfoGain(self, value: float) -> RandomForestRegressor: ...
+    def setMaxMemoryInMB(self, value: int) -> RandomForestRegressor: ...
+    def setCacheNodeIds(self, value: bool) -> RandomForestRegressor: ...
+    def setImpurity(self, value: str) -> RandomForestRegressor: ...
+    def setNumTrees(self, value: int) -> RandomForestRegressor: ...
+    def setBootstrap(self, value: bool) -> RandomForestRegressor: ...
+    def setSubsamplingRate(self, value: float) -> RandomForestRegressor: ...
+    def setFeatureSubsetStrategy(self, value: str) -> RandomForestRegressor: ...
+    def setCheckpointInterval(self, value: int) -> RandomForestRegressor: ...
+    def setSeed(self, value: int) -> RandomForestRegressor: ...
+    def setWeightCol(self, value: str) -> RandomForestRegressor: ...
+    def setMinWeightFractionPerNode(self, value: float) -> RandomForestRegressor: ...
+
+class RandomForestRegressionModel(
+    _JavaRegressionModel[Vector],
+    _TreeEnsembleModel,
+    _RandomForestRegressorParams,
+    JavaMLWritable,
+    JavaMLReadable[RandomForestRegressionModel],
+):
+    @property
+    def trees(self) -> List[DecisionTreeRegressionModel]: ...
+    @property
+    def featureImportances(self) -> Vector: ...
+
+class _GBTRegressorParams(_GBTParams, _TreeRegressorParams):
+    supportedLossTypes: List[str]
+    lossType: Param[str]
+    def __init__(self, *args: Any): ...
+    def getLossType(self) -> str: ...
+
+class GBTRegressor(
+    _JavaRegressor[GBTRegressionModel],
+    _GBTRegressorParams,
+    JavaMLWritable,
+    JavaMLReadable[GBTRegressor],
+):
+    def __init__(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        maxDepth: int = ...,
+        maxBins: int = ...,
+        minInstancesPerNode: int = ...,
+        minInfoGain: float = ...,
+        maxMemoryInMB: int = ...,
+        cacheNodeIds: bool = ...,
+        subsamplingRate: float = ...,
+        checkpointInterval: int = ...,
+        lossType: str = ...,
+        maxIter: int = ...,
+        stepSize: float = ...,
+        seed: Optional[int] = ...,
+        impurity: str = ...,
+        featureSubsetStrategy: str = ...,
+        validationTol: float = ...,
+        validationIndicatorCol: Optional[str] = ...,
+        leafCol: str = ...,
+        minWeightFractionPerNode: float = ...,
+        weightCol: Optional[str] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        maxDepth: int = ...,
+        maxBins: int = ...,
+        minInstancesPerNode: int = ...,
+        minInfoGain: float = ...,
+        maxMemoryInMB: int = ...,
+        cacheNodeIds: bool = ...,
+        subsamplingRate: float = ...,
+        checkpointInterval: int = ...,
+        lossType: str = ...,
+        maxIter: int = ...,
+        stepSize: float = ...,
+        seed: Optional[int] = ...,
+        impurity: str = ...,
+        featureSubsetStrategy: str = ...,
+        validationTol: float = ...,
+        validationIndicatorCol: Optional[str] = ...,
+        leafCol: str = ...,
+        minWeightFractionPerNode: float = ...,
+        weightCol: Optional[str] = ...
+    ) -> GBTRegressor: ...
+    def setMaxDepth(self, value: int) -> GBTRegressor: ...
+    def setMaxBins(self, value: int) -> GBTRegressor: ...
+    def setMinInstancesPerNode(self, value: int) -> GBTRegressor: ...
+    def setMinInfoGain(self, value: float) -> GBTRegressor: ...
+    def setMaxMemoryInMB(self, value: int) -> GBTRegressor: ...
+    def setCacheNodeIds(self, value: bool) -> GBTRegressor: ...
+    def setImpurity(self, value: str) -> GBTRegressor: ...
+    def setLossType(self, value: str) -> GBTRegressor: ...
+    def setSubsamplingRate(self, value: float) -> GBTRegressor: ...
+    def setFeatureSubsetStrategy(self, value: str) -> GBTRegressor: ...
+    def setValidationIndicatorCol(self, value: str) -> GBTRegressor: ...
+    def setMaxIter(self, value: int) -> GBTRegressor: ...
+    def setCheckpointInterval(self, value: int) -> GBTRegressor: ...
+    def setSeed(self, value: int) -> GBTRegressor: ...
+    def setStepSize(self, value: float) -> GBTRegressor: ...
+    def setWeightCol(self, value: str) -> GBTRegressor: ...
+    def setMinWeightFractionPerNode(self, value: float) -> GBTRegressor: ...
+
+class GBTRegressionModel(
+    _JavaRegressionModel[Vector],
+    _TreeEnsembleModel,
+    _GBTRegressorParams,
+    JavaMLWritable,
+    JavaMLReadable[GBTRegressionModel],
+):
+    @property
+    def featureImportances(self) -> Vector: ...
+    @property
+    def trees(self) -> List[DecisionTreeRegressionModel]: ...
+    def evaluateEachIteration(self, dataset: DataFrame, loss: str) -> List[float]: ...
+
+class _AFTSurvivalRegressionParams(
+    _PredictorParams,
+    HasMaxIter,
+    HasTol,
+    HasFitIntercept,
+    HasAggregationDepth,
+    HasMaxBlockSizeInMB,
+):
+    censorCol: Param[str]
+    quantileProbabilities: Param[List[float]]
+    quantilesCol: Param[str]
+    def __init__(self, *args: Any): ...
+    def getCensorCol(self) -> str: ...
+    def getQuantileProbabilities(self) -> List[float]: ...
+    def getQuantilesCol(self) -> str: ...
+
+class AFTSurvivalRegression(
+    _JavaRegressor[AFTSurvivalRegressionModel],
+    _AFTSurvivalRegressionParams,
+    JavaMLWritable,
+    JavaMLReadable[AFTSurvivalRegression],
+):
+    def __init__(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        fitIntercept: bool = ...,
+        maxIter: int = ...,
+        tol: float = ...,
+        censorCol: str = ...,
+        quantileProbabilities: List[float] = ...,
+        quantilesCol: Optional[str] = ...,
+        aggregationDepth: int = ...,
+        maxBlockSizeInMB: float = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        fitIntercept: bool = ...,
+        maxIter: int = ...,
+        tol: float = ...,
+        censorCol: str = ...,
+        quantileProbabilities: List[float] = ...,
+        quantilesCol: Optional[str] = ...,
+        aggregationDepth: int = ...,
+        maxBlockSizeInMB: float = ...
+    ) -> AFTSurvivalRegression: ...
+    def setCensorCol(self, value: str) -> AFTSurvivalRegression: ...
+    def setQuantileProbabilities(self, value: List[float]) -> AFTSurvivalRegression: ...
+    def setQuantilesCol(self, value: str) -> AFTSurvivalRegression: ...
+    def setMaxIter(self, value: int) -> AFTSurvivalRegression: ...
+    def setTol(self, value: float) -> AFTSurvivalRegression: ...
+    def setFitIntercept(self, value: bool) -> AFTSurvivalRegression: ...
+    def setAggregationDepth(self, value: int) -> AFTSurvivalRegression: ...
+    def setMaxBlockSizeInMB(self, value: float) -> AFTSurvivalRegression: ...
+
+class AFTSurvivalRegressionModel(
+    _JavaRegressionModel[Vector],
+    _AFTSurvivalRegressionParams,
+    JavaMLWritable,
+    JavaMLReadable[AFTSurvivalRegressionModel],
+):
+    def setQuantileProbabilities(
+        self, value: List[float]
+    ) -> AFTSurvivalRegressionModel: ...
+    def setQuantilesCol(self, value: str) -> AFTSurvivalRegressionModel: ...
+    @property
+    def coefficients(self) -> Vector: ...
+    @property
+    def intercept(self) -> float: ...
+    @property
+    def scale(self) -> float: ...
+    def predictQuantiles(self, features: Vector) -> Vector: ...
+    def predict(self, features: Vector) -> float: ...
+
+class _GeneralizedLinearRegressionParams(
+    _PredictorParams,
+    HasFitIntercept,
+    HasMaxIter,
+    HasTol,
+    HasRegParam,
+    HasWeightCol,
+    HasSolver,
+    HasAggregationDepth,
+):
+    family: Param[str]
+    link: Param[str]
+    linkPredictionCol: Param[str]
+    variancePower: Param[float]
+    linkPower: Param[float]
+    solver: Param[str]
+    offsetCol: Param[str]
+    def __init__(self, *args: Any): ...
+    def getFamily(self) -> str: ...
+    def getLinkPredictionCol(self) -> str: ...
+    def getLink(self) -> str: ...
+    def getVariancePower(self) -> float: ...
+    def getLinkPower(self) -> float: ...
+    def getOffsetCol(self) -> str: ...
+
+class GeneralizedLinearRegression(
+    _JavaRegressor[GeneralizedLinearRegressionModel],
+    _GeneralizedLinearRegressionParams,
+    JavaMLWritable,
+    JavaMLReadable[GeneralizedLinearRegression],
+):
+    def __init__(
+        self,
+        *,
+        labelCol: str = ...,
+        featuresCol: str = ...,
+        predictionCol: str = ...,
+        family: str = ...,
+        link: Optional[str] = ...,
+        fitIntercept: bool = ...,
+        maxIter: int = ...,
+        tol: float = ...,
+        regParam: float = ...,
+        weightCol: Optional[str] = ...,
+        solver: str = ...,
+        linkPredictionCol: Optional[str] = ...,
+        variancePower: float = ...,
+        linkPower: Optional[float] = ...,
+        offsetCol: Optional[str] = ...,
+        aggregationDepth: int = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        labelCol: str = ...,
+        featuresCol: str = ...,
+        predictionCol: str = ...,
+        family: str = ...,
+        link: Optional[str] = ...,
+        fitIntercept: bool = ...,
+        maxIter: int = ...,
+        tol: float = ...,
+        regParam: float = ...,
+        weightCol: Optional[str] = ...,
+        solver: str = ...,
+        linkPredictionCol: Optional[str] = ...,
+        variancePower: float = ...,
+        linkPower: Optional[float] = ...,
+        offsetCol: Optional[str] = ...,
+        aggregationDepth: int = ...
+    ) -> GeneralizedLinearRegression: ...
+    def setFamily(self, value: str) -> GeneralizedLinearRegression: ...
+    def setLinkPredictionCol(self, value: str) -> GeneralizedLinearRegression: ...
+    def setLink(self, value: str) -> GeneralizedLinearRegression: ...
+    def setVariancePower(self, value: float) -> GeneralizedLinearRegression: ...
+    def setLinkPower(self, value: float) -> GeneralizedLinearRegression: ...
+    def setOffsetCol(self, value: str) -> GeneralizedLinearRegression: ...
+    def setMaxIter(self, value: int) -> GeneralizedLinearRegression: ...
+    def setRegParam(self, value: float) -> GeneralizedLinearRegression: ...
+    def setTol(self, value: float) -> GeneralizedLinearRegression: ...
+    def setFitIntercept(self, value: bool) -> GeneralizedLinearRegression: ...
+    def setWeightCol(self, value: str) -> GeneralizedLinearRegression: ...
+    def setSolver(self, value: str) -> GeneralizedLinearRegression: ...
+    def setAggregationDepth(self, value: int) -> GeneralizedLinearRegression: ...
+
+class GeneralizedLinearRegressionModel(
+    _JavaRegressionModel[Vector],
+    _GeneralizedLinearRegressionParams,
+    JavaMLWritable,
+    JavaMLReadable[GeneralizedLinearRegressionModel],
+    HasTrainingSummary[GeneralizedLinearRegressionTrainingSummary],
+):
+    def setLinkPredictionCol(self, value: str) -> GeneralizedLinearRegressionModel: ...
+    @property
+    def coefficients(self) -> Vector: ...
+    @property
+    def intercept(self) -> float: ...
+    @property
+    def summary(self) -> GeneralizedLinearRegressionTrainingSummary: ...
+    def evaluate(self, dataset: DataFrame) -> GeneralizedLinearRegressionSummary: ...
+
+class GeneralizedLinearRegressionSummary(JavaWrapper):
+    @property
+    def predictions(self) -> DataFrame: ...
+    @property
+    def predictionCol(self) -> str: ...
+    @property
+    def rank(self) -> int: ...
+    @property
+    def degreesOfFreedom(self) -> int: ...
+    @property
+    def residualDegreeOfFreedom(self) -> int: ...
+    @property
+    def residualDegreeOfFreedomNull(self) -> int: ...
+    def residuals(self, residualsType: str = ...) -> DataFrame: ...
+    @property
+    def nullDeviance(self) -> float: ...
+    @property
+    def deviance(self) -> float: ...
+    @property
+    def dispersion(self) -> float: ...
+    @property
+    def aic(self) -> float: ...
+
+class GeneralizedLinearRegressionTrainingSummary(GeneralizedLinearRegressionSummary):
+    @property
+    def numIterations(self) -> int: ...
+    @property
+    def solver(self) -> str: ...
+    @property
+    def coefficientStandardErrors(self) -> List[float]: ...
+    @property
+    def tValues(self) -> List[float]: ...
+    @property
+    def pValues(self) -> List[float]: ...
+
+class _FactorizationMachinesParams(
+    _PredictorParams,
+    HasMaxIter,
+    HasStepSize,
+    HasTol,
+    HasSolver,
+    HasSeed,
+    HasFitIntercept,
+    HasRegParam,
+    HasWeightCol,
+):
+    factorSize: Param[int]
+    fitLinear: Param[bool]
+    miniBatchFraction: Param[float]
+    initStd: Param[float]
+    solver: Param[str]
+    def __init__(self, *args: Any): ...
+    def getFactorSize(self) -> int: ...
+    def getFitLinear(self) -> bool: ...
+    def getMiniBatchFraction(self) -> float: ...
+    def getInitStd(self) -> float: ...
+
+class FMRegressor(
+    _JavaRegressor[FMRegressionModel],
+    _FactorizationMachinesParams,
+    JavaMLWritable,
+    JavaMLReadable[FMRegressor],
+):
+    factorSize: Param[int]
+    fitLinear: Param[bool]
+    miniBatchFraction: Param[float]
+    initStd: Param[float]
+    solver: Param[str]
+    def __init__(
+        self,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        factorSize: int = ...,
+        fitIntercept: bool = ...,
+        fitLinear: bool = ...,
+        regParam: float = ...,
+        miniBatchFraction: float = ...,
+        initStd: float = ...,
+        maxIter: int = ...,
+        stepSize: float = ...,
+        tol: float = ...,
+        solver: str = ...,
+        seed: Optional[int] = ...,
+    ) -> None: ...
+    def setParams(
+        self,
+        featuresCol: str = ...,
+        labelCol: str = ...,
+        predictionCol: str = ...,
+        factorSize: int = ...,
+        fitIntercept: bool = ...,
+        fitLinear: bool = ...,
+        regParam: float = ...,
+        miniBatchFraction: float = ...,
+        initStd: float = ...,
+        maxIter: int = ...,
+        stepSize: float = ...,
+        tol: float = ...,
+        solver: str = ...,
+        seed: Optional[int] = ...,
+    ) -> FMRegressor: ...
+    def setFactorSize(self, value: int) -> FMRegressor: ...
+    def setFitLinear(self, value: bool) -> FMRegressor: ...
+    def setMiniBatchFraction(self, value: float) -> FMRegressor: ...
+    def setInitStd(self, value: float) -> FMRegressor: ...
+    def setMaxIter(self, value: int) -> FMRegressor: ...
+    def setStepSize(self, value: float) -> FMRegressor: ...
+    def setTol(self, value: float) -> FMRegressor: ...
+    def setSolver(self, value: str) -> FMRegressor: ...
+    def setSeed(self, value: int) -> FMRegressor: ...
+    def setFitIntercept(self, value: bool) -> FMRegressor: ...
+    def setRegParam(self, value: float) -> FMRegressor: ...
+
+class FMRegressionModel(
+    _JavaRegressionModel,
+    _FactorizationMachinesParams,
+    JavaMLWritable,
+    JavaMLReadable[FMRegressionModel],
+):
+    @property
+    def intercept(self) -> float: ...
+    @property
+    def linear(self) -> Vector: ...
+    @property
+    def factors(self) -> Matrix: ...
diff --git a/python/pyspark/ml/stat.py b/python/pyspark/ml/stat.py
index 70de8425613ec..60eeb68462f63 100644
--- a/python/pyspark/ml/stat.py
+++ b/python/pyspark/ml/stat.py
@@ -19,7 +19,6 @@
 
 from pyspark import since, SparkContext
 from pyspark.ml.common import _java2py, _py2java
-from pyspark.ml.linalg import DenseMatrix, Vectors
 from pyspark.ml.wrapper import JavaWrapper, _jvm
 from pyspark.sql.column import Column, _to_seq
 from pyspark.sql.functions import lit
@@ -37,36 +36,48 @@ class ChiSquareTest(object):
 
     """
     @staticmethod
-    @since("2.2.0")
     def test(dataset, featuresCol, labelCol, flatten=False):
         """
         Perform a Pearson's independence test using dataset.
 
-        :param dataset:
-          DataFrame of categorical labels and categorical features.
-          Real-valued features will be treated as categorical for each distinct value.
-        :param featuresCol:
-          Name of features column in dataset, of type `Vector` (`VectorUDT`).
-        :param labelCol:
-          Name of label column in dataset, of any numerical type.
-        :param flatten: if True, flattens the returned dataframe.
-        :return:
-          DataFrame containing the test result for every feature against the label.
-          If flatten is True, this DataFrame will contain one row per feature with the following
-          fields:
-          - `featureIndex: int`
-          - `pValue: float`
-          - `degreesOfFreedom: int`
-          - `statistic: float`
-          If flatten is False, this DataFrame will contain a single Row with the following fields:
-          - `pValues: Vector`
-          - `degreesOfFreedom: Array[int]`
-          - `statistics: Vector`
-          Each of these fields has one value per feature.
-
+        .. versionadded:: 2.2.0
         .. versionchanged:: 3.1.0
            Added optional ``flatten`` argument.
 
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            DataFrame of categorical labels and categorical features.
+            Real-valued features will be treated as categorical for each distinct value.
+        featuresCol : str
+            Name of features column in dataset, of type `Vector` (`VectorUDT`).
+        labelCol : str
+            Name of label column in dataset, of any numerical type.
+        flatten : bool, optional
+            if True, flattens the returned dataframe.
+
+        Returns
+        -------
+        :py:class:`pyspark.sql.DataFrame`
+            DataFrame containing the test result for every feature against the label.
+            If flatten is True, this DataFrame will contain one row per feature with the following
+            fields:
+
+            - `featureIndex: int`
+            - `pValue: float`
+            - `degreesOfFreedom: int`
+            - `statistic: float`
+
+            If flatten is False, this DataFrame will contain a single Row with the following fields:
+
+            - `pValues: Vector`
+            - `degreesOfFreedom: Array[int]`
+            - `statistics: Vector`
+
+            Each of these fields has one value per feature.
+
+        Examples
+        --------
         >>> from pyspark.ml.linalg import Vectors
         >>> from pyspark.ml.stat import ChiSquareTest
         >>> dataset = [[0, Vectors.dense([0, 0, 1])],
@@ -93,35 +104,42 @@ class Correlation(object):
     Compute the correlation matrix for the input dataset of Vectors using the specified method.
     Methods currently supported: `pearson` (default), `spearman`.
 
-    .. note:: For Spearman, a rank correlation, we need to create an RDD[Double] for each column
-      and sort it in order to retrieve the ranks and then join the columns back into an RDD[Vector],
-      which is fairly costly. Cache the input Dataset before calling corr with `method = 'spearman'`
-      to avoid recomputing the common lineage.
-
     .. versionadded:: 2.2.0
 
+    Notes
+    -----
+    For Spearman, a rank correlation, we need to create an RDD[Double] for each column
+    and sort it in order to retrieve the ranks and then join the columns back into an RDD[Vector],
+    which is fairly costly. Cache the input Dataset before calling corr with `method = 'spearman'`
+    to avoid recomputing the common lineage.
     """
     @staticmethod
-    @since("2.2.0")
     def corr(dataset, column, method="pearson"):
         """
         Compute the correlation matrix with specified method using dataset.
 
-        :param dataset:
-          A Dataset or a DataFrame.
-        :param column:
-          The name of the column of vectors for which the correlation coefficient needs
-          to be computed. This must be a column of the dataset, and it must contain
-          Vector objects.
-        :param method:
-          String specifying the method to use for computing correlation.
-          Supported: `pearson` (default), `spearman`.
-        :return:
-          A DataFrame that contains the correlation matrix of the column of vectors. This
-          DataFrame contains a single row and a single column of name
-          '$METHODNAME($COLUMN)'.
-
-        >>> from pyspark.ml.linalg import Vectors
+        .. versionadded:: 2.2.0
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            A DataFrame.
+        column : str
+            The name of the column of vectors for which the correlation coefficient needs
+            to be computed. This must be a column of the dataset, and it must contain
+            Vector objects.
+        method : str, optional
+            String specifying the method to use for computing correlation.
+            Supported: `pearson` (default), `spearman`.
+
+        Returns
+        -------
+        A DataFrame that contains the correlation matrix of the column of vectors. This
+        DataFrame contains a single row and a single column of name `METHODNAME(COLUMN)`.
+
+        Examples
+        --------
+        >>> from pyspark.ml.linalg import DenseMatrix, Vectors
         >>> from pyspark.ml.stat import Correlation
         >>> dataset = [[Vectors.dense([1, 0, 0, -2])],
         ...            [Vectors.dense([4, 5, 0, 3])],
@@ -160,28 +178,36 @@ class KolmogorovSmirnovTest(object):
 
     """
     @staticmethod
-    @since("2.4.0")
     def test(dataset, sampleCol, distName, *params):
         """
         Conduct a one-sample, two-sided Kolmogorov-Smirnov test for probability distribution
         equality. Currently supports the normal distribution, taking as parameters the mean and
         standard deviation.
 
-        :param dataset:
-          a Dataset or a DataFrame containing the sample of data to test.
-        :param sampleCol:
-          Name of sample column in dataset, of any numerical type.
-        :param distName:
-          a `string` name for a theoretical distribution, currently only support "norm".
-        :param params:
-          a list of `Double` values specifying the parameters to be used for the theoretical
-          distribution. For "norm" distribution, the parameters includes mean and variance.
-        :return:
-          A DataFrame that contains the Kolmogorov-Smirnov test result for the input sampled data.
-          This DataFrame will contain a single Row with the following fields:
-          - `pValue: Double`
-          - `statistic: Double`
-
+        .. versionadded:: 2.4.0
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            a Dataset or a DataFrame containing the sample of data to test.
+        sampleCol : str
+            Name of sample column in dataset, of any numerical type.
+        distName : str
+            a `string` name for a theoretical distribution, currently only support "norm".
+        params : float
+            a list of `float` values specifying the parameters to be used for the theoretical
+            distribution. For "norm" distribution, the parameters includes mean and variance.
+
+        Returns
+        -------
+        A DataFrame that contains the Kolmogorov-Smirnov test result for the input sampled data.
+        This DataFrame will contain a single Row with the following fields:
+
+        - `pValue: Double`
+        - `statistic: Double`
+
+        Examples
+        --------
         >>> from pyspark.ml.stat import KolmogorovSmirnovTest
         >>> dataset = [[-1.0], [0.0], [1.0]]
         >>> dataset = spark.createDataFrame(dataset, ['sample'])
@@ -212,6 +238,10 @@ class Summarizer(object):
     The methods in this package provide various statistics for Vectors contained inside DataFrames.
     This class lets users pick the statistics they would like to extract for a given column.
 
+    .. versionadded:: 2.4.0
+
+    Examples
+    --------
     >>> from pyspark.ml.stat import Summarizer
     >>> from pyspark.sql import Row
     >>> from pyspark.ml.linalg import Vectors
@@ -222,14 +252,14 @@ class Summarizer(object):
     +-----------------------------------+
     |aggregate_metrics(features, weight)|
     +-----------------------------------+
-    |[[1.0,1.0,1.0], 1]                 |
+    |{[1.0,1.0,1.0], 1}                 |
     +-----------------------------------+
     <BLANKLINE>
     >>> df.select(summarizer.summary(df.features)).show(truncate=False)
     +--------------------------------+
     |aggregate_metrics(features, 1.0)|
     +--------------------------------+
-    |[[1.0,1.5,2.0], 2]              |
+    |{[1.0,1.5,2.0], 2}              |
     +--------------------------------+
     <BLANKLINE>
     >>> df.select(Summarizer.mean(df.features, df.weight)).show(truncate=False)
@@ -246,9 +276,6 @@ class Summarizer(object):
     |[1.0,1.5,2.0] |
     +--------------+
     <BLANKLINE>
-
-    .. versionadded:: 2.4.0
-
     """
     @staticmethod
     @since("2.4.0")
@@ -345,12 +372,11 @@ def _get_single_metric(col, weightCol, metric):
                                                 col._jc, weightCol._jc))
 
     @staticmethod
-    @since("2.4.0")
     def metrics(*metrics):
         """
         Given a list of metrics, provides a builder that it turns computes metrics from a column.
 
-        See the documentation of [[Summarizer]] for an example.
+        See the documentation of :py:class:`Summarizer` for an example.
 
         The following metrics are accepted (case sensitive):
          - mean: a vector that contains the coefficient-wise mean.
@@ -364,13 +390,21 @@ def metrics(*metrics):
          - normL2: the Euclidean norm for each coefficient.
          - normL1: the L1 norm of each coefficient (sum of the absolute values).
 
-        :param metrics:
-         metrics that can be provided.
-        :return:
-         an object of :py:class:`pyspark.ml.stat.SummaryBuilder`
+        .. versionadded:: 2.4.0
 
-        Note: Currently, the performance of this interface is about 2x~3x slower then using the RDD
+        Notes
+        -----
+        Currently, the performance of this interface is about 2x~3x slower than using the RDD
         interface.
+
+        Examples
+        --------
+        metrics : str
+            metrics that can be provided.
+
+        Returns
+        -------
+        :py:class:`pyspark.ml.stat.SummaryBuilder`
         """
         sc = SparkContext._active_spark_context
         js = JavaWrapper._new_java_obj("org.apache.spark.ml.stat.Summarizer.metrics",
@@ -391,19 +425,25 @@ class SummaryBuilder(JavaWrapper):
     def __init__(self, jSummaryBuilder):
         super(SummaryBuilder, self).__init__(jSummaryBuilder)
 
-    @since("2.4.0")
     def summary(self, featuresCol, weightCol=None):
         """
         Returns an aggregate object that contains the summary of the column with the requested
         metrics.
 
-        :param featuresCol:
-         a column that contains features Vector object.
-        :param weightCol:
-         a column that contains weight value. Default weight is 1.0.
-        :return:
-         an aggregate column that contains the statistics. The exact content of this
-         structure is determined during the creation of the builder.
+        .. versionadded:: 2.4.0
+
+        Parameters
+        ----------
+        featuresCol : str
+            a column that contains features Vector object.
+        weightCol : str, optional
+            a column that contains weight value. Default weight is 1.0.
+
+        Returns
+        -------
+        :py:class:`pyspark.sql.Column`
+            an aggregate column that contains the statistics. The exact content of this
+            structure is determined during the creation of the builder.
         """
         featuresCol, weightCol = Summarizer._check_param(featuresCol, weightCol)
         return Column(self._java_obj.summary(featuresCol._jc, weightCol._jc))
@@ -412,141 +452,21 @@ def summary(self, featuresCol, weightCol=None):
 class MultivariateGaussian(object):
     """Represents a (mean, cov) tuple
 
+    .. versionadded:: 3.0.0
+
+    Examples
+    --------
+    >>> from pyspark.ml.linalg import DenseMatrix, Vectors
     >>> m = MultivariateGaussian(Vectors.dense([11,12]), DenseMatrix(2, 2, (1.0, 3.0, 5.0, 2.0)))
     >>> (m.mean, m.cov.toArray())
     (DenseVector([11.0, 12.0]), array([[ 1.,  5.],
            [ 3.,  2.]]))
-
-    .. versionadded:: 3.0.0
-
     """
     def __init__(self, mean, cov):
         self.mean = mean
         self.cov = cov
 
 
-class ANOVATest(object):
-    """
-    Conduct ANOVA Classification Test for continuous features against categorical labels.
-
-    .. versionadded:: 3.1.0
-    """
-    @staticmethod
-    @since("3.1.0")
-    def test(dataset, featuresCol, labelCol, flatten=False):
-        """
-        Perform an ANOVA test using dataset.
-
-        :param dataset:
-          DataFrame of categorical labels and continuous features.
-        :param featuresCol:
-          Name of features column in dataset, of type `Vector` (`VectorUDT`).
-        :param labelCol:
-          Name of label column in dataset, of any numerical type.
-        :param flatten: if True, flattens the returned dataframe.
-        :return:
-          DataFrame containing the test result for every feature against the label.
-          If flatten is True, this DataFrame will contain one row per feature with the following
-          fields:
-          - `featureIndex: int`
-          - `pValue: float`
-          - `degreesOfFreedom: int`
-          - `fValue: float`
-          If flatten is False, this DataFrame will contain a single Row with the following fields:
-          - `pValues: Vector`
-          - `degreesOfFreedom: Array[int]`
-          - `fValues: Vector`
-          Each of these fields has one value per feature.
-
-        >>> from pyspark.ml.linalg import Vectors
-        >>> from pyspark.ml.stat import ANOVATest
-        >>> dataset = [[2.0, Vectors.dense([0.43486404, 0.57153633, 0.43175686,
-        ...                                 0.51418671, 0.61632374, 0.96565515])],
-        ...            [1.0, Vectors.dense([0.49162732, 0.6785187, 0.85460572,
-        ...                                 0.59784822, 0.12394819, 0.53783355])],
-        ...            [2.0, Vectors.dense([0.30879653, 0.54904515, 0.17103889,
-        ...                                 0.40492506, 0.18957493, 0.5440016])],
-        ...            [3.0, Vectors.dense([0.68114391, 0.60549825, 0.69094651,
-        ...                                 0.62102109, 0.05471483, 0.96449167])]]
-        >>> dataset = spark.createDataFrame(dataset, ["label", "features"])
-        >>> anovaResult = ANOVATest.test(dataset, 'features', 'label')
-        >>> row = anovaResult.select("fValues", "pValues").collect()
-        >>> row[0].fValues
-        DenseVector([4.0264, 18.4713, 3.4659, 1.9042, 0.5532, 0.512])
-        >>> row[0].pValues
-        DenseVector([0.3324, 0.1623, 0.3551, 0.456, 0.689, 0.7029])
-        >>> anovaResult = ANOVATest.test(dataset, 'features', 'label', True)
-        >>> row = anovaResult.orderBy("featureIndex").collect()
-        >>> row[0].fValue
-        4.026438671875297
-        """
-        sc = SparkContext._active_spark_context
-        javaTestObj = _jvm().org.apache.spark.ml.stat.ANOVATest
-        args = [_py2java(sc, arg) for arg in (dataset, featuresCol, labelCol, flatten)]
-        return _java2py(sc, javaTestObj.test(*args))
-
-
-class FValueTest(object):
-    """
-    Conduct F Regression test for continuous features against continuous labels.
-
-    .. versionadded:: 3.1.0
-    """
-    @staticmethod
-    @since("3.1.0")
-    def test(dataset, featuresCol, labelCol, flatten=False):
-        """
-        Perform a F Regression test using dataset.
-
-        :param dataset:
-          DataFrame of continuous labels and continuous features.
-        :param featuresCol:
-          Name of features column in dataset, of type `Vector` (`VectorUDT`).
-        :param labelCol:
-          Name of label column in dataset, of any numerical type.
-        :param flatten: if True, flattens the returned dataframe.
-        :return:
-          DataFrame containing the test result for every feature against the label.
-          If flatten is True, this DataFrame will contain one row per feature with the following
-          fields:
-          - `featureIndex: int`
-          - `pValue: float`
-          - `degreesOfFreedom: int`
-          - `fValue: float`
-          If flatten is False, this DataFrame will contain a single Row with the following fields:
-          - `pValues: Vector`
-          - `degreesOfFreedom: Array[int]`
-          - `fValues: Vector`
-          Each of these fields has one value per feature.
-
-        >>> from pyspark.ml.linalg import Vectors
-        >>> from pyspark.ml.stat import FValueTest
-        >>> dataset = [[0.57495218, Vectors.dense([0.43486404, 0.57153633, 0.43175686,
-        ...                                        0.51418671, 0.61632374, 0.96565515])],
-        ...            [0.84619853, Vectors.dense([0.49162732, 0.6785187, 0.85460572,
-        ...                                        0.59784822, 0.12394819, 0.53783355])],
-        ...            [0.39777647, Vectors.dense([0.30879653, 0.54904515, 0.17103889,
-        ...                                        0.40492506, 0.18957493, 0.5440016])],
-        ...            [0.79201573, Vectors.dense([0.68114391, 0.60549825, 0.69094651,
-        ...                                        0.62102109, 0.05471483, 0.96449167])]]
-        >>> dataset = spark.createDataFrame(dataset, ["label", "features"])
-        >>> fValueResult = FValueTest.test(dataset, 'features', 'label')
-        >>> row = fValueResult.select("fValues", "pValues").collect()
-        >>> row[0].fValues
-        DenseVector([3.741, 7.5807, 142.0684, 34.9849, 0.4112, 0.0539])
-        >>> row[0].pValues
-        DenseVector([0.1928, 0.1105, 0.007, 0.0274, 0.5871, 0.838])
-        >>> fValueResult = FValueTest.test(dataset, 'features', 'label', True)
-        >>> row = fValueResult.orderBy("featureIndex").collect()
-        >>> row[0].fValue
-        3.7409548308350593
-        """
-        sc = SparkContext._active_spark_context
-        javaTestObj = _jvm().org.apache.spark.ml.stat.FValueTest
-        args = [_py2java(sc, arg) for arg in (dataset, featuresCol, labelCol, flatten)]
-        return _java2py(sc, javaTestObj.test(*args))
-
-
 if __name__ == "__main__":
     import doctest
     import numpy
diff --git a/python/pyspark/ml/stat.pyi b/python/pyspark/ml/stat.pyi
new file mode 100644
index 0000000000000..30485a7f63d2d
--- /dev/null
+++ b/python/pyspark/ml/stat.pyi
@@ -0,0 +1,77 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Optional
+
+from pyspark.ml.linalg import Matrix, Vector
+from pyspark.ml.wrapper import JavaWrapper
+from pyspark.sql.column import Column
+from pyspark.sql.dataframe import DataFrame
+
+from py4j.java_gateway import JavaObject  # type: ignore[import]
+
+class ChiSquareTest:
+    @staticmethod
+    def test(
+        dataset: DataFrame, featuresCol: str, labelCol: str, flatten: bool = ...
+    ) -> DataFrame: ...
+
+class Correlation:
+    @staticmethod
+    def corr(dataset: DataFrame, column: str, method: str = ...) -> DataFrame: ...
+
+class KolmogorovSmirnovTest:
+    @staticmethod
+    def test(
+        dataset: DataFrame, sampleCol: str, distName: str, *params: float
+    ) -> DataFrame: ...
+
+class Summarizer:
+    @staticmethod
+    def mean(col: Column, weightCol: Optional[Column] = ...) -> Column: ...
+    @staticmethod
+    def sum(col: Column, weightCol: Optional[Column] = ...) -> Column: ...
+    @staticmethod
+    def variance(col: Column, weightCol: Optional[Column] = ...) -> Column: ...
+    @staticmethod
+    def std(col: Column, weightCol: Optional[Column] = ...) -> Column: ...
+    @staticmethod
+    def count(col: Column, weightCol: Optional[Column] = ...) -> Column: ...
+    @staticmethod
+    def numNonZeros(col: Column, weightCol: Optional[Column] = ...) -> Column: ...
+    @staticmethod
+    def max(col: Column, weightCol: Optional[Column] = ...) -> Column: ...
+    @staticmethod
+    def min(col: Column, weightCol: Optional[Column] = ...) -> Column: ...
+    @staticmethod
+    def normL1(col: Column, weightCol: Optional[Column] = ...) -> Column: ...
+    @staticmethod
+    def normL2(col: Column, weightCol: Optional[Column] = ...) -> Column: ...
+    @staticmethod
+    def metrics(*metrics: str) -> SummaryBuilder: ...
+
+class SummaryBuilder(JavaWrapper):
+    def __init__(self, jSummaryBuilder: JavaObject) -> None: ...
+    def summary(
+        self, featuresCol: Column, weightCol: Optional[Column] = ...
+    ) -> Column: ...
+
+class MultivariateGaussian:
+    mean: Vector
+    cov: Matrix
+    def __init__(self, mean: Vector, cov: Matrix) -> None: ...
diff --git a/python/pyspark/ml/tests/test_algorithms.py b/python/pyspark/ml/tests/test_algorithms.py
index 2faf2d98f0271..35ce48b926663 100644
--- a/python/pyspark/ml/tests/test_algorithms.py
+++ b/python/pyspark/ml/tests/test_algorithms.py
@@ -25,7 +25,7 @@
     MultilayerPerceptronClassifier, OneVsRest
 from pyspark.ml.clustering import DistributedLDAModel, KMeans, LocalLDAModel, LDA, LDAModel
 from pyspark.ml.fpm import FPGrowth
-from pyspark.ml.linalg import Matrices, Vectors
+from pyspark.ml.linalg import Matrices, Vectors, DenseVector
 from pyspark.ml.recommendation import ALS
 from pyspark.ml.regression import GeneralizedLinearRegression, LinearRegression
 from pyspark.sql import Row
@@ -86,7 +86,7 @@ def test_raw_and_probability_prediction(self):
         expected_rawPrediction = [-11.6081922998, -8.15827998691, 22.17757045]
         self.assertTrue(result.prediction, expected_prediction)
         self.assertTrue(np.allclose(result.probability, expected_probability, atol=1E-4))
-        self.assertTrue(np.allclose(result.rawPrediction, expected_rawPrediction, atol=1))
+        self.assertTrue(np.allclose(result.rawPrediction, expected_rawPrediction, rtol=0.1))
 
 
 class OneVsRestTests(SparkSessionTestCase):
@@ -116,7 +116,19 @@ def test_output_columns(self):
         output = model.transform(df)
         self.assertEqual(output.columns, ["label", "features", "rawPrediction", "prediction"])
 
-    def test_parallelism_doesnt_change_output(self):
+    def test_raw_prediction_column_is_of_vector_type(self):
+        # SPARK-35142: `OneVsRestModel` outputs raw prediction as a string column
+        df = self.spark.createDataFrame([(0.0, Vectors.dense(1.0, 0.8)),
+                                         (1.0, Vectors.sparse(2, [], [])),
+                                         (2.0, Vectors.dense(0.5, 0.5))],
+                                        ["label", "features"])
+        lr = LogisticRegression(maxIter=5, regParam=0.01)
+        ovr = OneVsRest(classifier=lr, parallelism=1)
+        model = ovr.fit(df)
+        row = model.transform(df).head()
+        self.assertIsInstance(row["rawPrediction"], DenseVector)
+
+    def test_parallelism_does_not_change_output(self):
         df = self.spark.createDataFrame([(0.0, Vectors.dense(1.0, 0.8)),
                                          (1.0, Vectors.sparse(2, [], [])),
                                          (2.0, Vectors.dense(0.5, 0.5))],
@@ -226,8 +238,8 @@ def test_association_rules(self):
         fpm = fp.fit(self.data)
 
         expected_association_rules = self.spark.createDataFrame(
-            [([3], [1], 1.0, 1.0), ([2], [1], 1.0, 1.0)],
-            ["antecedent", "consequent", "confidence", "lift"]
+            [([3], [1], 1.0, 1.0, 0.5), ([2], [1], 1.0, 1.0, 0.75)],
+            ["antecedent", "consequent", "confidence", "lift", "support"]
         )
         actual_association_rules = fpm.associationRules
 
@@ -330,10 +342,10 @@ def test_linear_regression_with_huber_loss(self):
 
 
 if __name__ == "__main__":
-    from pyspark.ml.tests.test_algorithms import *
+    from pyspark.ml.tests.test_algorithms import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/ml/tests/test_base.py b/python/pyspark/ml/tests/test_base.py
index 1b7d1c75854af..28166e2d5cec2 100644
--- a/python/pyspark/ml/tests/test_base.py
+++ b/python/pyspark/ml/tests/test_base.py
@@ -19,7 +19,15 @@
 
 from pyspark.sql.types import DoubleType, IntegerType
 from pyspark.testing.mlutils import MockDataset, MockEstimator, MockUnaryTransformer, \
-    SparkSessionTestCase
+    MockTransformer, SparkSessionTestCase
+
+
+class TransformerTests(unittest.TestCase):
+
+    def test_transform_invalid_type(self):
+        transformer = MockTransformer()
+        data = MockDataset()
+        self.assertRaises(TypeError, transformer.transform, data, "")
 
 
 class UnaryTransformerTests(SparkSessionTestCase):
@@ -52,13 +60,18 @@ def test_unary_transformer_transform(self):
 
 
 class EstimatorTest(unittest.TestCase):
+    def setUp(self):
+        self.estimator = MockEstimator()
+        self.data = MockDataset()
+
+    def test_fit_invalid_params(self):
+        invalid_type_parms = ""
+        self.assertRaises(TypeError, self.estimator.fit, self.data, invalid_type_parms)
 
     def testDefaultFitMultiple(self):
         N = 4
-        data = MockDataset()
-        estimator = MockEstimator()
-        params = [{estimator.fake: i} for i in range(N)]
-        modelIter = estimator.fitMultiple(data, params)
+        params = [{self.estimator.fake: i} for i in range(N)]
+        modelIter = self.estimator.fitMultiple(self.data, params)
         indexList = []
         for index, model in modelIter:
             self.assertEqual(model.getFake(), index)
@@ -67,10 +80,10 @@ def testDefaultFitMultiple(self):
 
 
 if __name__ == "__main__":
-    from pyspark.ml.tests.test_base import *
+    from pyspark.ml.tests.test_base import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/ml/tests/test_evaluation.py b/python/pyspark/ml/tests/test_evaluation.py
index fdd6ee7a5366f..28df1b26dcd6b 100644
--- a/python/pyspark/ml/tests/test_evaluation.py
+++ b/python/pyspark/ml/tests/test_evaluation.py
@@ -27,6 +27,12 @@
 
 class EvaluatorTests(SparkSessionTestCase):
 
+    def test_evaluate_invalid_type(self):
+        evaluator = RegressionEvaluator(metricName="r2")
+        df = self.spark.createDataFrame([Row(label=1.0, prediction=1.1)])
+        invalid_type = ""
+        self.assertRaises(TypeError, evaluator.evaluate, df, invalid_type)
+
     def test_java_params(self):
         """
         This tests a bug fixed by SPARK-18274 which causes multiple copies
@@ -53,10 +59,10 @@ def test_clustering_evaluator_with_cosine_distance(self):
 
 
 if __name__ == "__main__":
-    from pyspark.ml.tests.test_evaluation import *
+    from pyspark.ml.tests.test_evaluation import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/ml/tests/test_feature.py b/python/pyspark/ml/tests/test_feature.py
index 4c6bfa696b110..2cceb04338806 100644
--- a/python/pyspark/ml/tests/test_feature.py
+++ b/python/pyspark/ml/tests/test_feature.py
@@ -16,12 +16,8 @@
 # limitations under the License.
 #
 
-import sys
 import unittest
 
-if sys.version > '3':
-    basestring = str
-
 from pyspark.ml.feature import Binarizer, CountVectorizer, CountVectorizerModel, HashingTF, IDF, \
     NGram, RFormula, StopWordsRemover, StringIndexer, StringIndexerModel, VectorSizeHint
 from pyspark.ml.linalg import DenseVector, SparseVector, Vectors
@@ -91,7 +87,7 @@ def test_stopwordsremover(self):
         transformedDF = stopWordRemover.transform(dataset)
         self.assertEqual(transformedDF.head().output, ["panda"])
         self.assertEqual(type(stopWordRemover.getStopWords()), list)
-        self.assertTrue(isinstance(stopWordRemover.getStopWords()[0], basestring))
+        self.assertTrue(isinstance(stopWordRemover.getStopWords()[0], str))
         # Custom
         stopwords = ["panda"]
         stopWordRemover.setStopWords(stopwords)
@@ -173,7 +169,7 @@ def test_count_vectorizer_from_vocab(self):
 
         # Test an empty vocabulary
         with QuietTest(self.sc):
-            with self.assertRaisesRegexp(Exception, "vocabSize.*invalid.*0"):
+            with self.assertRaisesRegex(Exception, "vocabSize.*invalid.*0"):
                 CountVectorizerModel.from_vocabulary([], inputCol="words")
 
         # Test model with default settings can transform
@@ -236,6 +232,7 @@ def test_string_indexer_from_labels(self):
         model = StringIndexerModel.from_labels(["a", "b", "c"], inputCol="label",
                                                outputCol="indexed", handleInvalid="keep")
         self.assertEqual(model.labels, ["a", "b", "c"])
+        self.assertEqual(model.labelsArray, [("a", "b", "c")])
 
         df1 = self.spark.createDataFrame([
             (0, "a"),
@@ -304,10 +301,10 @@ def test_apply_binary_term_freqs(self):
 
 
 if __name__ == "__main__":
-    from pyspark.ml.tests.test_feature import *
+    from pyspark.ml.tests.test_feature import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/ml/tests/test_image.py b/python/pyspark/ml/tests/test_image.py
index 5cc2a815eaa14..00e4c95a84355 100644
--- a/python/pyspark/ml/tests/test_image.py
+++ b/python/pyspark/ml/tests/test_image.py
@@ -16,10 +16,8 @@
 #
 import unittest
 
-import py4j
-
 from pyspark.ml.image import ImageSchema
-from pyspark.testing.mlutils import PySparkTestCase, SparkSessionTestCase
+from pyspark.testing.mlutils import SparkSessionTestCase
 from pyspark.sql import Row
 from pyspark.testing.utils import QuietTest
 
@@ -35,7 +33,7 @@ def test_read_images(self):
         self.assertEqual(df.count(), 4)
         first_row = df.take(1)[0][0]
         # compare `schema.simpleString()` instead of directly compare schema,
-        # because the df loaded from datasouce may change schema column nullability.
+        # because the df loaded from datasource may change schema column nullability.
         self.assertEqual(df.schema.simpleString(), ImageSchema.imageSchema.simpleString())
         self.assertEqual(df.schema["image"].dataType.simpleString(),
                          ImageSchema.columnSchema.simpleString())
@@ -49,29 +47,29 @@ def test_read_images(self):
         self.assertEqual(ImageSchema.undefinedImageType, "Undefined")
 
         with QuietTest(self.sc):
-            self.assertRaisesRegexp(
+            self.assertRaisesRegex(
                 TypeError,
                 "image argument should be pyspark.sql.types.Row; however",
                 lambda: ImageSchema.toNDArray("a"))
 
         with QuietTest(self.sc):
-            self.assertRaisesRegexp(
+            self.assertRaisesRegex(
                 ValueError,
                 "image argument should have attributes specified in",
                 lambda: ImageSchema.toNDArray(Row(a=1)))
 
         with QuietTest(self.sc):
-            self.assertRaisesRegexp(
+            self.assertRaisesRegex(
                 TypeError,
                 "array argument should be numpy.ndarray; however, it got",
                 lambda: ImageSchema.toImage("a"))
 
 
 if __name__ == "__main__":
-    from pyspark.ml.tests.test_image import *
+    from pyspark.ml.tests.test_image import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/ml/tests/test_linalg.py b/python/pyspark/ml/tests/test_linalg.py
index 2cba5396f275d..18c01ddf88e67 100644
--- a/python/pyspark/ml/tests/test_linalg.py
+++ b/python/pyspark/ml/tests/test_linalg.py
@@ -378,10 +378,10 @@ def test_infer_schema(self):
 
 
 if __name__ == "__main__":
-    from pyspark.ml.tests.test_linalg import *
+    from pyspark.ml.tests.test_linalg import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/ml/tests/test_param.py b/python/pyspark/ml/tests/test_param.py
index 1b2b1914cc036..da875588cfe18 100644
--- a/python/pyspark/ml/tests/test_param.py
+++ b/python/pyspark/ml/tests/test_param.py
@@ -17,7 +17,6 @@
 #
 
 import inspect
-import sys
 import array as pyarray
 import unittest
 
@@ -31,14 +30,11 @@
 from pyspark.ml.linalg import DenseVector, SparseVector, Vectors
 from pyspark.ml.param import Param, Params, TypeConverters
 from pyspark.ml.param.shared import HasInputCol, HasMaxIter, HasSeed
+from pyspark.ml.regression import LinearRegressionModel, GeneralizedLinearRegressionModel
 from pyspark.ml.wrapper import JavaParams
 from pyspark.testing.mlutils import check_params, PySparkTestCase, SparkSessionTestCase
 
 
-if sys.version > '3':
-    xrange = range
-
-
 class ParamTypeConversionTests(PySparkTestCase):
     """
     Test that param type conversion happens.
@@ -67,14 +63,14 @@ def test_vector(self):
     def test_list(self):
         l = [0, 1]
         for lst_like in [l, np.array(l), DenseVector(l), SparseVector(len(l), range(len(l)), l),
-                         pyarray.array('l', l), xrange(2), tuple(l)]:
+                         pyarray.array('l', l), range(2), tuple(l)]:
             converted = TypeConverters.toList(lst_like)
             self.assertEqual(type(converted), list)
             self.assertListEqual(converted, l)
 
     def test_list_int(self):
         for indices in [[1.0, 2.0], np.array([1.0, 2.0]), DenseVector([1.0, 2.0]),
-                        SparseVector(2, {0: 1.0, 1: 2.0}), xrange(1, 3), (1.0, 2.0),
+                        SparseVector(2, {0: 1.0, 1: 2.0}), range(1, 3), (1.0, 2.0),
                         pyarray.array('d', [1.0, 2.0])]:
             vs = VectorSlicer(indices=indices)
             self.assertListEqual(vs.getIndices(), [1, 2])
@@ -200,12 +196,11 @@ def test_resolveparam(self):
         self.assertEqual(testParams._resolveParam("maxIter"), testParams.maxIter)
 
         self.assertEqual(testParams._resolveParam(u"maxIter"), testParams.maxIter)
-        if sys.version_info[0] >= 3:
-            # In Python 3, it is allowed to get/set attributes with non-ascii characters.
-            e_cls = AttributeError
-        else:
-            e_cls = UnicodeEncodeError
-        self.assertRaises(e_cls, lambda: testParams._resolveParam(u"아"))
+        self.assertRaises(AttributeError, lambda: testParams._resolveParam(u"아"))
+
+        # Invalid type
+        invalid_type = 1
+        self.assertRaises(TypeError, testParams._resolveParam, invalid_type)
 
     def test_params(self):
         testParams = TestParams()
@@ -318,7 +313,7 @@ def test_logistic_regression_check_thresholds(self):
             LogisticRegression
         )
 
-        self.assertRaisesRegexp(
+        self.assertRaisesRegex(
             ValueError,
             "Logistic Regression getThreshold found inconsistent.*$",
             LogisticRegression, threshold=0.42, thresholds=[0.5, 0.5]
@@ -342,6 +337,16 @@ def test_default_params_transferred(self):
         self.assertFalse(binarizer.isSet(binarizer.outputCol))
         self.assertEqual(result[0][0], 1.0)
 
+    def test_lr_evaluate_invaild_type(self):
+        lr = LinearRegressionModel()
+        invalid_type = ""
+        self.assertRaises(TypeError, lr.evaluate, invalid_type)
+
+    def test_glr_evaluate_invaild_type(self):
+        glr = GeneralizedLinearRegressionModel()
+        invalid_type = ""
+        self.assertRaises(TypeError, glr.evaluate, invalid_type)
+
 
 class DefaultValuesTests(PySparkTestCase):
     """
@@ -368,23 +373,21 @@ def test_java_params(self):
                         and issubclass(cls, JavaParams) and not inspect.isabstract(cls) \
                         and not re.match("_?Java", name) and name != '_LSH' \
                         and name != '_Selector':
-                    # NOTE: disable check_params_exist until there is parity with Scala API
-
-                    check_params(self, cls(), check_params_exist=False)
+                    check_params(self, cls(), check_params_exist=True)
 
         # Additional classes that need explicit construction
         from pyspark.ml.feature import CountVectorizerModel, StringIndexerModel
         check_params(self, CountVectorizerModel.from_vocabulary(['a'], 'input'),
-                     check_params_exist=False)
+                     check_params_exist=True)
         check_params(self, StringIndexerModel.from_labels(['a', 'b'], 'input'),
-                     check_params_exist=False)
+                     check_params_exist=True)
 
 
 if __name__ == "__main__":
-    from pyspark.ml.tests.test_param import *
+    from pyspark.ml.tests.test_param import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/ml/tests/test_persistence.py b/python/pyspark/ml/tests/test_persistence.py
index d4edcc26e17ac..77a6c0309628a 100644
--- a/python/pyspark/ml/tests/test_persistence.py
+++ b/python/pyspark/ml/tests/test_persistence.py
@@ -21,19 +21,78 @@
 import unittest
 
 from pyspark.ml import Transformer
-from pyspark.ml.classification import DecisionTreeClassifier, LogisticRegression, OneVsRest, \
-    OneVsRestModel
+from pyspark.ml.classification import DecisionTreeClassifier, FMClassifier, \
+    FMClassificationModel, LogisticRegression, MultilayerPerceptronClassifier, \
+    MultilayerPerceptronClassificationModel, OneVsRest, OneVsRestModel
 from pyspark.ml.clustering import KMeans
 from pyspark.ml.feature import Binarizer, HashingTF, PCA
 from pyspark.ml.linalg import Vectors
 from pyspark.ml.param import Params
 from pyspark.ml.pipeline import Pipeline, PipelineModel
-from pyspark.ml.regression import DecisionTreeRegressor, LinearRegression
+from pyspark.ml.regression import DecisionTreeRegressor, GeneralizedLinearRegression, \
+    GeneralizedLinearRegressionModel, \
+    LinearRegression
 from pyspark.ml.util import DefaultParamsReadable, DefaultParamsWriter
 from pyspark.ml.wrapper import JavaParams
 from pyspark.testing.mlutils import MockUnaryTransformer, SparkSessionTestCase
 
 
+class TestDefaultSolver(SparkSessionTestCase):
+
+    def test_multilayer_load(self):
+        df = self.spark.createDataFrame([(0.0, Vectors.dense([0.0, 0.0])),
+                                         (1.0, Vectors.dense([0.0, 1.0])),
+                                         (1.0, Vectors.dense([1.0, 0.0])),
+                                         (0.0, Vectors.dense([1.0, 1.0]))],
+                                        ["label",  "features"])
+
+        mlp = MultilayerPerceptronClassifier(layers=[2, 2, 2], seed=123)
+        model = mlp.fit(df)
+        self.assertEqual(model.getSolver(), "l-bfgs")
+        transformed1 = model.transform(df)
+        path = tempfile.mkdtemp()
+        model_path = path + "/mlp"
+        model.save(model_path)
+        model2 = MultilayerPerceptronClassificationModel.load(model_path)
+        self.assertEqual(model2.getSolver(), "l-bfgs")
+        transformed2 = model2.transform(df)
+        self.assertEqual(transformed1.take(4), transformed2.take(4))
+
+    def test_fm_load(self):
+        df = self.spark.createDataFrame([(1.0, Vectors.dense(1.0)),
+                                         (0.0, Vectors.sparse(1, [], []))],
+                                        ["label",  "features"])
+        fm = FMClassifier(factorSize=2, maxIter=50, stepSize=2.0)
+        model = fm.fit(df)
+        self.assertEqual(model.getSolver(), "adamW")
+        transformed1 = model.transform(df)
+        path = tempfile.mkdtemp()
+        model_path = path + "/fm"
+        model.save(model_path)
+        model2 = FMClassificationModel.load(model_path)
+        self.assertEqual(model2.getSolver(), "adamW")
+        transformed2 = model2.transform(df)
+        self.assertEqual(transformed1.take(2), transformed2.take(2))
+
+    def test_glr_load(self):
+        df = self.spark.createDataFrame([(1.0, Vectors.dense(0.0, 0.0)),
+                                         (1.0, Vectors.dense(1.0, 2.0)),
+                                         (2.0, Vectors.dense(0.0, 0.0)),
+                                         (2.0, Vectors.dense(1.0, 1.0))],
+                                        ["label",  "features"])
+        glr = GeneralizedLinearRegression(family="gaussian", link="identity", linkPredictionCol="p")
+        model = glr.fit(df)
+        self.assertEqual(model.getSolver(), "irls")
+        transformed1 = model.transform(df)
+        path = tempfile.mkdtemp()
+        model_path = path + "/glr"
+        model.save(model_path)
+        model2 = GeneralizedLinearRegressionModel.load(model_path)
+        self.assertEqual(model2.getSolver(), "irls")
+        transformed2 = model2.transform(df)
+        self.assertEqual(transformed1.take(4), transformed2.take(4))
+
+
 class PersistenceTest(SparkSessionTestCase):
 
     def test_linear_regression(self):
@@ -178,6 +237,11 @@ def _compare_pipelines(self, m1, m2):
                 self.assertEqual(len(m1.models), len(m2.models))
                 for x, y in zip(m1.models, m2.models):
                     self._compare_pipelines(x, y)
+        elif isinstance(m1, Params):
+            # Test on python backend Estimator/Transformer/Model/Evaluator
+            self.assertEqual(len(m1.params), len(m2.params))
+            for p in m1.params:
+                self._compare_params(m1, m2, p)
         else:
             raise RuntimeError("_compare_pipelines does not yet support type: %s" % type(m1))
 
@@ -267,14 +331,14 @@ def test_python_transformer_pipeline_persistence(self):
             except OSError:
                 pass
 
-    def test_onevsrest(self):
+    def _run_test_onevsrest(self, LogisticRegressionCls):
         temp_path = tempfile.mkdtemp()
         df = self.spark.createDataFrame([(0.0, 0.5, Vectors.dense(1.0, 0.8)),
                                          (1.0, 0.5, Vectors.sparse(2, [], [])),
                                          (2.0, 1.0, Vectors.dense(0.5, 0.5))] * 10,
                                         ["label", "wt", "features"])
 
-        lr = LogisticRegression(maxIter=5, regParam=0.01)
+        lr = LogisticRegressionCls(maxIter=5, regParam=0.01)
         ovr = OneVsRest(classifier=lr)
 
         def reload_and_compare(ovr, suffix):
@@ -291,6 +355,11 @@ def reload_and_compare(ovr, suffix):
         reload_and_compare(OneVsRest(classifier=lr), "ovr")
         reload_and_compare(OneVsRest(classifier=lr).setWeightCol("wt"), "ovrw")
 
+    def test_onevsrest(self):
+        from pyspark.testing.mlutils import DummyLogisticRegression
+        self._run_test_onevsrest(LogisticRegression)
+        self._run_test_onevsrest(DummyLogisticRegression)
+
     def test_decisiontree_classifier(self):
         dt = DecisionTreeClassifier(maxDepth=1)
         path = tempfile.mkdtemp()
@@ -383,7 +452,7 @@ def test_default_read_write_default_params(self):
         del metadata['defaultParamMap']
         metadataStr = json.dumps(metadata, separators=[',',  ':'])
         loadedMetadata = reader._parseMetaData(metadataStr, )
-        with self.assertRaisesRegexp(AssertionError, "`defaultParamMap` section not found"):
+        with self.assertRaisesRegex(AssertionError, "`defaultParamMap` section not found"):
             reader.getAndSetParams(lr, loadedMetadata)
 
         # Prior to 2.4.0, metadata doesn't have `defaultParamMap`.
@@ -394,10 +463,10 @@ def test_default_read_write_default_params(self):
 
 
 if __name__ == "__main__":
-    from pyspark.ml.tests.test_persistence import *
+    from pyspark.ml.tests.test_persistence import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/ml/tests/test_pipeline.py b/python/pyspark/ml/tests/test_pipeline.py
index 18a8b7d3b0d4a..c29b2d3f44679 100644
--- a/python/pyspark/ml/tests/test_pipeline.py
+++ b/python/pyspark/ml/tests/test_pipeline.py
@@ -59,10 +59,10 @@ def doTransform(pipeline):
 
 
 if __name__ == "__main__":
-    from pyspark.ml.tests.test_pipeline import *
+    from pyspark.ml.tests.test_pipeline import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/ml/tests/test_stat.py b/python/pyspark/ml/tests/test_stat.py
index d583da2e8a408..a2403b38873db 100644
--- a/python/pyspark/ml/tests/test_stat.py
+++ b/python/pyspark/ml/tests/test_stat.py
@@ -40,10 +40,10 @@ def test_chisquaretest(self):
 
 
 if __name__ == "__main__":
-    from pyspark.ml.tests.test_stat import *
+    from pyspark.ml.tests.test_stat import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/ml/tests/test_training_summary.py b/python/pyspark/ml/tests/test_training_summary.py
index b5054095d190b..5b31c871fb271 100644
--- a/python/pyspark/ml/tests/test_training_summary.py
+++ b/python/pyspark/ml/tests/test_training_summary.py
@@ -15,14 +15,14 @@
 # limitations under the License.
 #
 
-import sys
 import unittest
 
-if sys.version > '3':
-    basestring = str
-
-from pyspark.ml.classification import BinaryLogisticRegressionSummary, LogisticRegression, \
-    LogisticRegressionSummary
+from pyspark.ml.classification import BinaryLogisticRegressionSummary, \
+    BinaryRandomForestClassificationSummary, FMClassifier, \
+    FMClassificationSummary, LinearSVC, LinearSVCSummary,  \
+    LogisticRegression, LogisticRegressionSummary, \
+    MultilayerPerceptronClassifier, MultilayerPerceptronClassificationSummary, \
+    RandomForestClassificationSummary, RandomForestClassifier
 from pyspark.ml.clustering import BisectingKMeans, GaussianMixture, KMeans
 from pyspark.ml.linalg import Vectors
 from pyspark.ml.regression import GeneralizedLinearRegression, LinearRegression
@@ -42,7 +42,7 @@ def test_linear_regression_summary(self):
         self.assertTrue(model.hasSummary)
         s = model.summary
         # test that api is callable and returns expected types
-        self.assertGreater(s.totalIterations, 0)
+        self.assertEqual(s.totalIterations, 0)
         self.assertTrue(isinstance(s.predictions, DataFrame))
         self.assertEqual(s.predictionCol, "prediction")
         self.assertEqual(s.labelCol, "label")
@@ -99,7 +99,7 @@ def test_glr_summary(self):
         self.assertEqual(s.residualDegreeOfFreedom, 1)
         self.assertEqual(s.residualDegreeOfFreedomNull, 2)
         self.assertEqual(s.rank, 1)
-        self.assertTrue(isinstance(s.solver, basestring))
+        self.assertTrue(isinstance(s.solver, str))
         self.assertTrue(isinstance(s.aic, float))
         self.assertTrue(isinstance(s.deviance, float))
         self.assertTrue(isinstance(s.nullDeviance, float))
@@ -193,6 +193,203 @@ def test_multiclass_logistic_regression_summary(self):
         self.assertFalse(isinstance(sameSummary, BinaryLogisticRegressionSummary))
         self.assertAlmostEqual(sameSummary.accuracy, s.accuracy)
 
+    def test_linear_svc_summary(self):
+        df = self.spark.createDataFrame([(1.0, 2.0, Vectors.dense(1.0, 1.0, 1.0)),
+                                         (0.0, 2.0, Vectors.dense(1.0, 2.0, 3.0))],
+                                        ["label", "weight", "features"])
+        svc = LinearSVC(maxIter=5, weightCol="weight")
+        model = svc.fit(df)
+        self.assertTrue(model.hasSummary)
+        s = model.summary()
+        # test that api is callable and returns expected types
+        self.assertTrue(isinstance(s.predictions, DataFrame))
+        self.assertEqual(s.scoreCol, "rawPrediction")
+        self.assertEqual(s.labelCol, "label")
+        self.assertEqual(s.predictionCol, "prediction")
+        objHist = s.objectiveHistory
+        self.assertTrue(isinstance(objHist, list) and isinstance(objHist[0], float))
+        self.assertGreater(s.totalIterations, 0)
+        self.assertTrue(isinstance(s.labels, list))
+        self.assertTrue(isinstance(s.truePositiveRateByLabel, list))
+        self.assertTrue(isinstance(s.falsePositiveRateByLabel, list))
+        self.assertTrue(isinstance(s.precisionByLabel, list))
+        self.assertTrue(isinstance(s.recallByLabel, list))
+        self.assertTrue(isinstance(s.fMeasureByLabel(), list))
+        self.assertTrue(isinstance(s.fMeasureByLabel(1.0), list))
+        self.assertTrue(isinstance(s.roc, DataFrame))
+        self.assertAlmostEqual(s.areaUnderROC, 1.0, 2)
+        self.assertTrue(isinstance(s.pr, DataFrame))
+        self.assertTrue(isinstance(s.fMeasureByThreshold, DataFrame))
+        self.assertTrue(isinstance(s.precisionByThreshold, DataFrame))
+        self.assertTrue(isinstance(s.recallByThreshold, DataFrame))
+        print(s.weightedTruePositiveRate)
+        self.assertAlmostEqual(s.weightedTruePositiveRate, 1.0, 2)
+        self.assertAlmostEqual(s.weightedFalsePositiveRate, 0.0, 2)
+        self.assertAlmostEqual(s.weightedRecall, 1.0, 2)
+        self.assertAlmostEqual(s.weightedPrecision, 1.0, 2)
+        self.assertAlmostEqual(s.weightedFMeasure(), 1.0, 2)
+        self.assertAlmostEqual(s.weightedFMeasure(1.0), 1.0, 2)
+        # test evaluation (with training dataset) produces a summary with same values
+        # one check is enough to verify a summary is returned, Scala version runs full test
+        sameSummary = model.evaluate(df)
+        self.assertTrue(isinstance(sameSummary, LinearSVCSummary))
+        self.assertAlmostEqual(sameSummary.areaUnderROC, s.areaUnderROC)
+
+    def test_binary_randomforest_classification_summary(self):
+        df = self.spark.createDataFrame([(1.0, 2.0, Vectors.dense(1.0)),
+                                         (0.0, 2.0, Vectors.sparse(1, [], []))],
+                                        ["label", "weight", "features"])
+        rf = RandomForestClassifier(weightCol="weight")
+        model = rf.fit(df)
+        self.assertTrue(model.hasSummary)
+        s = model.summary
+        # test that api is callable and returns expected types
+        self.assertTrue(isinstance(s.predictions, DataFrame))
+        self.assertEqual(s.labelCol, "label")
+        self.assertEqual(s.predictionCol, "prediction")
+        self.assertEqual(s.totalIterations, 0)
+        self.assertTrue(isinstance(s.labels, list))
+        self.assertTrue(isinstance(s.truePositiveRateByLabel, list))
+        self.assertTrue(isinstance(s.falsePositiveRateByLabel, list))
+        self.assertTrue(isinstance(s.precisionByLabel, list))
+        self.assertTrue(isinstance(s.recallByLabel, list))
+        self.assertTrue(isinstance(s.fMeasureByLabel(), list))
+        self.assertTrue(isinstance(s.fMeasureByLabel(1.0), list))
+        self.assertTrue(isinstance(s.roc, DataFrame))
+        self.assertAlmostEqual(s.areaUnderROC, 1.0, 2)
+        self.assertTrue(isinstance(s.pr, DataFrame))
+        self.assertTrue(isinstance(s.fMeasureByThreshold, DataFrame))
+        self.assertTrue(isinstance(s.precisionByThreshold, DataFrame))
+        self.assertTrue(isinstance(s.recallByThreshold, DataFrame))
+        self.assertAlmostEqual(s.accuracy, 1.0, 2)
+        self.assertAlmostEqual(s.weightedTruePositiveRate, 1.0, 2)
+        self.assertAlmostEqual(s.weightedFalsePositiveRate, 0.0, 2)
+        self.assertAlmostEqual(s.weightedRecall, 1.0, 2)
+        self.assertAlmostEqual(s.weightedPrecision, 1.0, 2)
+        self.assertAlmostEqual(s.weightedFMeasure(), 1.0, 2)
+        self.assertAlmostEqual(s.weightedFMeasure(1.0), 1.0, 2)
+        # test evaluation (with training dataset) produces a summary with same values
+        # one check is enough to verify a summary is returned, Scala version runs full test
+        sameSummary = model.evaluate(df)
+        self.assertTrue(isinstance(sameSummary, BinaryRandomForestClassificationSummary))
+        self.assertAlmostEqual(sameSummary.areaUnderROC, s.areaUnderROC)
+
+    def test_multiclass_randomforest_classification_summary(self):
+        df = self.spark.createDataFrame([(1.0, 2.0, Vectors.dense(1.0)),
+                                         (0.0, 2.0, Vectors.sparse(1, [], [])),
+                                         (2.0, 2.0, Vectors.dense(2.0)),
+                                         (2.0, 2.0, Vectors.dense(1.9))],
+                                        ["label", "weight", "features"])
+        rf = RandomForestClassifier(weightCol="weight")
+        model = rf.fit(df)
+        self.assertTrue(model.hasSummary)
+        s = model.summary
+        # test that api is callable and returns expected types
+        self.assertTrue(isinstance(s.predictions, DataFrame))
+        self.assertEqual(s.labelCol, "label")
+        self.assertEqual(s.predictionCol, "prediction")
+        self.assertEqual(s.totalIterations, 0)
+        self.assertTrue(isinstance(s.labels, list))
+        self.assertTrue(isinstance(s.truePositiveRateByLabel, list))
+        self.assertTrue(isinstance(s.falsePositiveRateByLabel, list))
+        self.assertTrue(isinstance(s.precisionByLabel, list))
+        self.assertTrue(isinstance(s.recallByLabel, list))
+        self.assertTrue(isinstance(s.fMeasureByLabel(), list))
+        self.assertTrue(isinstance(s.fMeasureByLabel(1.0), list))
+        self.assertAlmostEqual(s.accuracy, 1.0, 2)
+        self.assertAlmostEqual(s.weightedTruePositiveRate, 1.0, 2)
+        self.assertAlmostEqual(s.weightedFalsePositiveRate, 0.0, 2)
+        self.assertAlmostEqual(s.weightedRecall, 1.0, 2)
+        self.assertAlmostEqual(s.weightedPrecision, 1.0, 2)
+        self.assertAlmostEqual(s.weightedFMeasure(), 1.0, 2)
+        self.assertAlmostEqual(s.weightedFMeasure(1.0), 1.0, 2)
+        # test evaluation (with training dataset) produces a summary with same values
+        # one check is enough to verify a summary is returned, Scala version runs full test
+        sameSummary = model.evaluate(df)
+        self.assertTrue(isinstance(sameSummary, RandomForestClassificationSummary))
+        self.assertFalse(isinstance(sameSummary, BinaryRandomForestClassificationSummary))
+        self.assertAlmostEqual(sameSummary.accuracy, s.accuracy)
+
+    def test_fm_classification_summary(self):
+        df = self.spark.createDataFrame([(1.0, Vectors.dense(2.0)),
+                                         (0.0, Vectors.dense(2.0)),
+                                         (0.0, Vectors.dense(6.0)),
+                                         (1.0, Vectors.dense(3.0))
+                                         ],
+                                        ["label", "features"])
+        fm = FMClassifier(maxIter=5)
+        model = fm.fit(df)
+        self.assertTrue(model.hasSummary)
+        s = model.summary()
+        # test that api is callable and returns expected types
+        self.assertTrue(isinstance(s.predictions, DataFrame))
+        self.assertEqual(s.scoreCol, "probability")
+        self.assertEqual(s.labelCol, "label")
+        self.assertEqual(s.predictionCol, "prediction")
+        objHist = s.objectiveHistory
+        self.assertTrue(isinstance(objHist, list) and isinstance(objHist[0], float))
+        self.assertGreater(s.totalIterations, 0)
+        self.assertTrue(isinstance(s.labels, list))
+        self.assertTrue(isinstance(s.truePositiveRateByLabel, list))
+        self.assertTrue(isinstance(s.falsePositiveRateByLabel, list))
+        self.assertTrue(isinstance(s.precisionByLabel, list))
+        self.assertTrue(isinstance(s.recallByLabel, list))
+        self.assertTrue(isinstance(s.fMeasureByLabel(), list))
+        self.assertTrue(isinstance(s.fMeasureByLabel(1.0), list))
+        self.assertTrue(isinstance(s.roc, DataFrame))
+        self.assertAlmostEqual(s.areaUnderROC, 0.625, 2)
+        self.assertTrue(isinstance(s.pr, DataFrame))
+        self.assertTrue(isinstance(s.fMeasureByThreshold, DataFrame))
+        self.assertTrue(isinstance(s.precisionByThreshold, DataFrame))
+        self.assertTrue(isinstance(s.recallByThreshold, DataFrame))
+        self.assertAlmostEqual(s.weightedTruePositiveRate, 0.75, 2)
+        self.assertAlmostEqual(s.weightedFalsePositiveRate, 0.25, 2)
+        self.assertAlmostEqual(s.weightedRecall, 0.75, 2)
+        self.assertAlmostEqual(s.weightedPrecision, 0.8333333333333333, 2)
+        self.assertAlmostEqual(s.weightedFMeasure(), 0.7333333333333334, 2)
+        self.assertAlmostEqual(s.weightedFMeasure(1.0), 0.7333333333333334, 2)
+        # test evaluation (with training dataset) produces a summary with same values
+        # one check is enough to verify a summary is returned, Scala version runs full test
+        sameSummary = model.evaluate(df)
+        self.assertTrue(isinstance(sameSummary, FMClassificationSummary))
+        self.assertAlmostEqual(sameSummary.areaUnderROC, s.areaUnderROC)
+
+    def test_mlp_classification_summary(self):
+        df = self.spark.createDataFrame([(0.0, Vectors.dense([0.0, 0.0])),
+                                         (1.0, Vectors.dense([0.0, 1.0])),
+                                         (1.0, Vectors.dense([1.0, 0.0])),
+                                         (0.0, Vectors.dense([1.0, 1.0]))
+                                         ],
+                                        ["label", "features"])
+        mlp = MultilayerPerceptronClassifier(layers=[2, 2, 2], seed=123)
+        model = mlp.fit(df)
+        self.assertTrue(model.hasSummary)
+        s = model.summary()
+        # test that api is callable and returns expected types
+        self.assertTrue(isinstance(s.predictions, DataFrame))
+        self.assertEqual(s.labelCol, "label")
+        self.assertEqual(s.predictionCol, "prediction")
+        self.assertGreater(s.totalIterations, 0)
+        self.assertTrue(isinstance(s.labels, list))
+        self.assertTrue(isinstance(s.truePositiveRateByLabel, list))
+        self.assertTrue(isinstance(s.falsePositiveRateByLabel, list))
+        self.assertTrue(isinstance(s.precisionByLabel, list))
+        self.assertTrue(isinstance(s.recallByLabel, list))
+        self.assertTrue(isinstance(s.fMeasureByLabel(), list))
+        self.assertTrue(isinstance(s.fMeasureByLabel(1.0), list))
+        self.assertAlmostEqual(s.accuracy, 1.0, 2)
+        self.assertAlmostEqual(s.weightedTruePositiveRate, 1.0, 2)
+        self.assertAlmostEqual(s.weightedFalsePositiveRate, 0.0, 2)
+        self.assertAlmostEqual(s.weightedRecall, 1.0, 2)
+        self.assertAlmostEqual(s.weightedPrecision, 1.0, 2)
+        self.assertAlmostEqual(s.weightedFMeasure(), 1.0, 2)
+        self.assertAlmostEqual(s.weightedFMeasure(1.0), 1.0, 2)
+        # test evaluation (with training dataset) produces a summary with same values
+        # one check is enough to verify a summary is returned, Scala version runs full test
+        sameSummary = model.evaluate(df)
+        self.assertTrue(isinstance(sameSummary, MultilayerPerceptronClassificationSummary))
+        self.assertAlmostEqual(sameSummary.accuracy, s.accuracy)
+
     def test_gaussian_mixture_summary(self):
         data = [(Vectors.dense(1.0),), (Vectors.dense(5.0),), (Vectors.dense(10.0),),
                 (Vectors.sparse(1, [], []),)]
@@ -245,10 +442,10 @@ def test_kmeans_summary(self):
 
 
 if __name__ == "__main__":
-    from pyspark.ml.tests.test_training_summary import *
+    from pyspark.ml.tests.test_training_summary import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/ml/tests/test_tuning.py b/python/pyspark/ml/tests/test_tuning.py
index 6bcc3f93e1b3b..9f6c8192e0ff3 100644
--- a/python/pyspark/ml/tests/test_tuning.py
+++ b/python/pyspark/ml/tests/test_tuning.py
@@ -16,8 +16,10 @@
 #
 
 import tempfile
+import math
 import unittest
 
+import numpy as np
 from pyspark.ml.feature import HashingTF, Tokenizer
 from pyspark.ml import Estimator, Pipeline, Model
 from pyspark.ml.classification import LogisticRegression, LogisticRegressionModel, OneVsRest
@@ -26,9 +28,10 @@
 from pyspark.ml.linalg import Vectors
 from pyspark.ml.param import Param, Params
 from pyspark.ml.tuning import CrossValidator, CrossValidatorModel, ParamGridBuilder, \
-    TrainValidationSplit, TrainValidationSplitModel
+    TrainValidationSplit, TrainValidationSplitModel, ParamRandomBuilder
 from pyspark.sql.functions import rand
-from pyspark.testing.mlutils import SparkSessionTestCase
+from pyspark.testing.mlutils import DummyEvaluator, DummyLogisticRegression, \
+    DummyLogisticRegressionModel, SparkSessionTestCase
 
 
 class HasInducedError(Params):
@@ -64,6 +67,108 @@ def _fit(self, dataset):
         return model
 
 
+class DummyParams(Params):
+
+    def __init__(self):
+        super(DummyParams, self).__init__()
+        self.test_param = Param(self, "test_param", "dummy parameter for testing")
+        self.another_test_param = Param(self, "another_test_param", "second parameter for testing")
+
+
+class ParamRandomBuilderTests(unittest.TestCase):
+
+    def __init__(self, methodName):
+        super(ParamRandomBuilderTests, self).__init__(methodName=methodName)
+        self.dummy_params = DummyParams()
+        self.to_test = ParamRandomBuilder()
+        self.n = 100
+
+    def check_ranges(self, params, lowest, highest, expected_type):
+        self.assertEqual(self.n, len(params))
+        for param in params:
+            for v in param.values():
+                self.assertGreaterEqual(v, lowest)
+                self.assertLessEqual(v, highest)
+                self.assertEqual(type(v), expected_type)
+
+    def check_addRandom_ranges(self, x, y, expected_type):
+        params = self.to_test.addRandom(self.dummy_params.test_param, x, y, self.n).build()
+        self.check_ranges(params, x, y, expected_type)
+
+    def check_addLog10Random_ranges(self, x, y, expected_type):
+        params = self.to_test.addLog10Random(self.dummy_params.test_param, x, y, self.n).build()
+        self.check_ranges(params, x, y, expected_type)
+
+    @staticmethod
+    def counts(xs):
+        key_to_count = {}
+        for v in xs:
+            k = int(v)
+            if key_to_count.get(k) is None:
+                key_to_count[k] = 1
+            else:
+                key_to_count[k] = key_to_count[k] + 1
+        return key_to_count
+
+    @staticmethod
+    def raw_values_of(params):
+        values = []
+        for param in params:
+            for v in param.values():
+                values.append(v)
+        return values
+
+    def check_even_distribution(self, vs, bin_function):
+        binned = map(lambda x: bin_function(x), vs)
+        histogram = self.counts(binned)
+        values = list(histogram.values())
+        sd = np.std(values)
+        mu = np.mean(values)
+        for k, v in histogram.items():
+            self.assertLess(abs(v - mu), 5 * sd, "{} values for bucket {} is unlikely "
+                                                 "when the mean is {} and standard deviation {}"
+                            .format(v, k, mu, sd))
+
+    def test_distribution(self):
+        params = self.to_test.addRandom(self.dummy_params.test_param, 0, 20000, 10000).build()
+        values = self.raw_values_of(params)
+        self.check_even_distribution(values, lambda x: x // 1000)
+
+    def test_logarithmic_distribution(self):
+        params = self.to_test.addLog10Random(self.dummy_params.test_param, 1, 1e10, 10000).build()
+        values = self.raw_values_of(params)
+        self.check_even_distribution(values, lambda x: math.log10(x))
+
+    def test_param_cardinality(self):
+        num_random_params = 7
+        values = [1, 2, 3]
+        self.to_test.addRandom(self.dummy_params.test_param, 1, 10, num_random_params)
+        self.to_test.addGrid(self.dummy_params.another_test_param, values)
+        self.assertEqual(len(self.to_test.build()), num_random_params * len(values))
+
+    def test_add_random_integer_logarithmic_range(self):
+        self.check_addLog10Random_ranges(100, 200, int)
+
+    def test_add_logarithmic_random_float_and_integer_yields_floats(self):
+        self.check_addLog10Random_ranges(100, 200., float)
+
+    def test_add_random_float_logarithmic_range(self):
+        self.check_addLog10Random_ranges(100., 200., float)
+
+    def test_add_random_integer_range(self):
+        self.check_addRandom_ranges(100, 200, int)
+
+    def test_add_random_float_and_integer_yields_floats(self):
+        self.check_addRandom_ranges(100, 200., float)
+
+    def test_add_random_float_range(self):
+        self.check_addRandom_ranges(100., 200., float)
+
+    def test_unexpected_type(self):
+        with self.assertRaises(TypeError):
+            self.to_test.addRandom(self.dummy_params.test_param, 1, "wrong type", 1).build()
+
+
 class ParamGridBuilderTests(SparkSessionTestCase):
 
     def test_addGrid(self):
@@ -73,7 +178,21 @@ def test_addGrid(self):
                     .build())
 
 
-class CrossValidatorTests(SparkSessionTestCase):
+class ValidatorTestUtilsMixin:
+    def assert_param_maps_equal(self, paramMaps1, paramMaps2):
+        self.assertEqual(len(paramMaps1), len(paramMaps2))
+        for paramMap1, paramMap2 in zip(paramMaps1, paramMaps2):
+            self.assertEqual(set(paramMap1.keys()), set(paramMap2.keys()))
+            for param in paramMap1.keys():
+                v1 = paramMap1[param]
+                v2 = paramMap2[param]
+                if isinstance(v1, Params):
+                    self.assertEqual(v1.uid, v2.uid)
+                else:
+                    self.assertEqual(v1, v2)
+
+
+class CrossValidatorTests(SparkSessionTestCase, ValidatorTestUtilsMixin):
 
     def test_copy(self):
         dataset = self.spark.createDataFrame([
@@ -89,15 +208,50 @@ def test_copy(self):
         grid = (ParamGridBuilder()
                 .addGrid(iee.inducedError, [100.0, 0.0, 10000.0])
                 .build())
-        cv = CrossValidator(estimator=iee, estimatorParamMaps=grid, evaluator=evaluator)
+        cv = CrossValidator(
+            estimator=iee,
+            estimatorParamMaps=grid,
+            evaluator=evaluator,
+            collectSubModels=True,
+            numFolds=2
+        )
         cvCopied = cv.copy()
-        self.assertEqual(cv.getEstimator().uid, cvCopied.getEstimator().uid)
+        for param in [
+            lambda x: x.getEstimator().uid,
+            # SPARK-32092: CrossValidator.copy() needs to copy all existing params
+            lambda x: x.getNumFolds(),
+            lambda x: x.getFoldCol(),
+            lambda x: x.getCollectSubModels(),
+            lambda x: x.getParallelism(),
+            lambda x: x.getSeed()
+        ]:
+            self.assertEqual(param(cv), param(cvCopied))
 
         cvModel = cv.fit(dataset)
         cvModelCopied = cvModel.copy()
         for index in range(len(cvModel.avgMetrics)):
             self.assertTrue(abs(cvModel.avgMetrics[index] - cvModelCopied.avgMetrics[index])
                             < 0.0001)
+        # SPARK-32092: CrossValidatorModel.copy() needs to copy all existing params
+        for param in [
+            lambda x: x.getNumFolds(),
+            lambda x: x.getFoldCol(),
+            lambda x: x.getSeed()
+        ]:
+            self.assertEqual(param(cvModel), param(cvModelCopied))
+
+        cvModel.avgMetrics[0] = 'foo'
+        self.assertNotEqual(
+            cvModelCopied.avgMetrics[0],
+            'foo',
+            "Changing the original avgMetrics should not affect the copied model"
+        )
+        cvModel.subModels[0][0].getInducedError = lambda: 'foo'
+        self.assertNotEqual(
+            cvModelCopied.subModels[0][0].getInducedError(),
+            'foo',
+            "Changing the original subModels should not affect the copied model"
+        )
 
     def test_fit_minimize_metric(self):
         dataset = self.spark.createDataFrame([
@@ -152,7 +306,7 @@ def test_param_grid_type_coercion(self):
             for v in param.values():
                 assert(type(v) == float)
 
-    def test_save_load_trained_model(self):
+    def _run_test_save_load_trained_model(self, LogisticRegressionCls, LogisticRegressionModelCls):
         # This tests saving and loading the trained model only.
         # Save/load for CrossValidator will be added later: SPARK-13786
         temp_path = tempfile.mkdtemp()
@@ -163,20 +317,48 @@ def test_save_load_trained_model(self):
              (Vectors.dense([0.6]), 1.0),
              (Vectors.dense([1.0]), 1.0)] * 10,
             ["features", "label"])
-        lr = LogisticRegression()
+        lr = LogisticRegressionCls()
         grid = ParamGridBuilder().addGrid(lr.maxIter, [0, 1]).build()
         evaluator = BinaryClassificationEvaluator()
-        cv = CrossValidator(estimator=lr, estimatorParamMaps=grid, evaluator=evaluator)
+        cv = CrossValidator(
+            estimator=lr,
+            estimatorParamMaps=grid,
+            evaluator=evaluator,
+            collectSubModels=True,
+            numFolds=4,
+            seed=42
+        )
         cvModel = cv.fit(dataset)
         lrModel = cvModel.bestModel
 
-        cvModelPath = temp_path + "/cvModel"
-        lrModel.save(cvModelPath)
-        loadedLrModel = LogisticRegressionModel.load(cvModelPath)
+        lrModelPath = temp_path + "/lrModel"
+        lrModel.save(lrModelPath)
+        loadedLrModel = LogisticRegressionModelCls.load(lrModelPath)
         self.assertEqual(loadedLrModel.uid, lrModel.uid)
         self.assertEqual(loadedLrModel.intercept, lrModel.intercept)
 
-    def test_save_load_simple_estimator(self):
+        # SPARK-32092: Saving and then loading CrossValidatorModel should not change the params
+        cvModelPath = temp_path + "/cvModel"
+        cvModel.save(cvModelPath)
+        loadedCvModel = CrossValidatorModel.load(cvModelPath)
+        for param in [
+            lambda x: x.getNumFolds(),
+            lambda x: x.getFoldCol(),
+            lambda x: x.getSeed(),
+            lambda x: len(x.subModels)
+        ]:
+            self.assertEqual(param(cvModel), param(loadedCvModel))
+
+        self.assertTrue(all(
+            loadedCvModel.isSet(param) for param in loadedCvModel.params
+        ))
+
+    def test_save_load_trained_model(self):
+        self._run_test_save_load_trained_model(LogisticRegression, LogisticRegressionModel)
+        self._run_test_save_load_trained_model(DummyLogisticRegression,
+                                               DummyLogisticRegressionModel)
+
+    def _run_test_save_load_simple_estimator(self, LogisticRegressionCls, evaluatorCls):
         temp_path = tempfile.mkdtemp()
         dataset = self.spark.createDataFrame(
             [(Vectors.dense([0.0]), 0.0),
@@ -186,9 +368,9 @@ def test_save_load_simple_estimator(self):
              (Vectors.dense([1.0]), 1.0)] * 10,
             ["features", "label"])
 
-        lr = LogisticRegression()
+        lr = LogisticRegressionCls()
         grid = ParamGridBuilder().addGrid(lr.maxIter, [0, 1]).build()
-        evaluator = BinaryClassificationEvaluator()
+        evaluator = evaluatorCls()
 
         # test save/load of CrossValidator
         cv = CrossValidator(estimator=lr, estimatorParamMaps=grid, evaluator=evaluator)
@@ -198,7 +380,7 @@ def test_save_load_simple_estimator(self):
         loadedCV = CrossValidator.load(cvPath)
         self.assertEqual(loadedCV.getEstimator().uid, cv.getEstimator().uid)
         self.assertEqual(loadedCV.getEvaluator().uid, cv.getEvaluator().uid)
-        self.assertEqual(loadedCV.getEstimatorParamMaps(), cv.getEstimatorParamMaps())
+        self.assert_param_maps_equal(loadedCV.getEstimatorParamMaps(), cv.getEstimatorParamMaps())
 
         # test save/load of CrossValidatorModel
         cvModelPath = temp_path + "/cvModel"
@@ -206,6 +388,12 @@ def test_save_load_simple_estimator(self):
         loadedModel = CrossValidatorModel.load(cvModelPath)
         self.assertEqual(loadedModel.bestModel.uid, cvModel.bestModel.uid)
 
+    def test_save_load_simple_estimator(self):
+        self._run_test_save_load_simple_estimator(
+            LogisticRegression, BinaryClassificationEvaluator)
+        self._run_test_save_load_simple_estimator(
+            DummyLogisticRegression, DummyEvaluator)
+
     def test_parallel_evaluation(self):
         dataset = self.spark.createDataFrame(
             [(Vectors.dense([0.0]), 0.0),
@@ -271,7 +459,7 @@ def checkSubModels(subModels):
             for j in range(len(grid)):
                 self.assertEqual(cvModel.subModels[i][j].uid, cvModel3.subModels[i][j].uid)
 
-    def test_save_load_nested_estimator(self):
+    def _run_test_save_load_nested_estimator(self, LogisticRegressionCls):
         temp_path = tempfile.mkdtemp()
         dataset = self.spark.createDataFrame(
             [(Vectors.dense([0.0]), 0.0),
@@ -281,9 +469,9 @@ def test_save_load_nested_estimator(self):
              (Vectors.dense([1.0]), 1.0)] * 10,
             ["features", "label"])
 
-        ova = OneVsRest(classifier=LogisticRegression())
-        lr1 = LogisticRegression().setMaxIter(100)
-        lr2 = LogisticRegression().setMaxIter(150)
+        ova = OneVsRest(classifier=LogisticRegressionCls())
+        lr1 = LogisticRegressionCls().setMaxIter(100)
+        lr2 = LogisticRegressionCls().setMaxIter(150)
         grid = ParamGridBuilder().addGrid(ova.classifier, [lr1, lr2]).build()
         evaluator = MulticlassClassificationEvaluator()
 
@@ -293,6 +481,7 @@ def test_save_load_nested_estimator(self):
         cvPath = temp_path + "/cv"
         cv.save(cvPath)
         loadedCV = CrossValidator.load(cvPath)
+        self.assert_param_maps_equal(loadedCV.getEstimatorParamMaps(), grid)
         self.assertEqual(loadedCV.getEstimator().uid, cv.getEstimator().uid)
         self.assertEqual(loadedCV.getEvaluator().uid, cv.getEvaluator().uid)
 
@@ -309,9 +498,14 @@ def test_save_load_nested_estimator(self):
         cvModelPath = temp_path + "/cvModel"
         cvModel.save(cvModelPath)
         loadedModel = CrossValidatorModel.load(cvModelPath)
+        self.assert_param_maps_equal(loadedModel.getEstimatorParamMaps(), grid)
         self.assertEqual(loadedModel.bestModel.uid, cvModel.bestModel.uid)
 
-    def test_save_load_pipeline_estimator(self):
+    def test_save_load_nested_estimator(self):
+        self._run_test_save_load_nested_estimator(LogisticRegression)
+        self._run_test_save_load_nested_estimator(DummyLogisticRegression)
+
+    def _run_test_save_load_pipeline_estimator(self, LogisticRegressionCls):
         temp_path = tempfile.mkdtemp()
         training = self.spark.createDataFrame([
             (0, "a b c d e spark", 1.0),
@@ -328,9 +522,9 @@ def test_save_load_pipeline_estimator(self):
         tokenizer = Tokenizer(inputCol="text", outputCol="words")
         hashingTF = HashingTF(inputCol=tokenizer.getOutputCol(), outputCol="features")
 
-        ova = OneVsRest(classifier=LogisticRegression())
-        lr1 = LogisticRegression().setMaxIter(5)
-        lr2 = LogisticRegression().setMaxIter(10)
+        ova = OneVsRest(classifier=LogisticRegressionCls())
+        lr1 = LogisticRegressionCls().setMaxIter(5)
+        lr2 = LogisticRegressionCls().setMaxIter(10)
 
         pipeline = Pipeline(stages=[tokenizer, hashingTF, ova])
 
@@ -343,6 +537,11 @@ def test_save_load_pipeline_estimator(self):
                                   estimatorParamMaps=paramGrid,
                                   evaluator=MulticlassClassificationEvaluator(),
                                   numFolds=2)  # use 3+ folds in practice
+        cvPath = temp_path + "/cv"
+        crossval.save(cvPath)
+        loadedCV = CrossValidator.load(cvPath)
+        self.assert_param_maps_equal(loadedCV.getEstimatorParamMaps(), paramGrid)
+        self.assertEqual(loadedCV.getEstimator().uid, crossval.getEstimator().uid)
 
         # Run cross-validation, and choose the best set of parameters.
         cvModel = crossval.fit(training)
@@ -363,6 +562,11 @@ def test_save_load_pipeline_estimator(self):
                                    estimatorParamMaps=paramGrid,
                                    evaluator=MulticlassClassificationEvaluator(),
                                    numFolds=2)  # use 3+ folds in practice
+        cv2Path = temp_path + "/cv2"
+        crossval2.save(cv2Path)
+        loadedCV2 = CrossValidator.load(cv2Path)
+        self.assert_param_maps_equal(loadedCV2.getEstimatorParamMaps(), paramGrid)
+        self.assertEqual(loadedCV2.getEstimator().uid, crossval2.getEstimator().uid)
 
         # Run cross-validation, and choose the best set of parameters.
         cvModel2 = crossval2.fit(training)
@@ -380,8 +584,84 @@ def test_save_load_pipeline_estimator(self):
                                               original_nested_pipeline_model.stages):
             self.assertEqual(loadedStage.uid, originalStage.uid)
 
+    def test_save_load_pipeline_estimator(self):
+        self._run_test_save_load_pipeline_estimator(LogisticRegression)
+        self._run_test_save_load_pipeline_estimator(DummyLogisticRegression)
+
+    def test_user_specified_folds(self):
+        from pyspark.sql import functions as F
+
+        dataset = self.spark.createDataFrame(
+            [(Vectors.dense([0.0]), 0.0),
+             (Vectors.dense([0.4]), 1.0),
+             (Vectors.dense([0.5]), 0.0),
+             (Vectors.dense([0.6]), 1.0),
+             (Vectors.dense([1.0]), 1.0)] * 10,
+            ["features", "label"]).repartition(2, "features")
+
+        dataset_with_folds = dataset.repartition(1).withColumn("random", rand(100)) \
+            .withColumn("fold", F.when(F.col("random") < 0.33, 0)
+                        .when(F.col("random") < 0.66, 1)
+                        .otherwise(2)).repartition(2, "features")
+
+        lr = LogisticRegression()
+        grid = ParamGridBuilder().addGrid(lr.maxIter, [20]).build()
+        evaluator = BinaryClassificationEvaluator()
+
+        cv = CrossValidator(estimator=lr, estimatorParamMaps=grid, evaluator=evaluator, numFolds=3)
+        cv_with_user_folds = CrossValidator(estimator=lr,
+                                            estimatorParamMaps=grid,
+                                            evaluator=evaluator,
+                                            numFolds=3,
+                                            foldCol="fold")
+
+        self.assertEqual(cv.getEstimator().uid, cv_with_user_folds.getEstimator().uid)
+
+        cvModel1 = cv.fit(dataset)
+        cvModel2 = cv_with_user_folds.fit(dataset_with_folds)
+        for index in range(len(cvModel1.avgMetrics)):
+            print(abs(cvModel1.avgMetrics[index] - cvModel2.avgMetrics[index]))
+            self.assertTrue(abs(cvModel1.avgMetrics[index] - cvModel2.avgMetrics[index])
+                            < 0.1)
+
+        # test save/load of CrossValidator
+        temp_path = tempfile.mkdtemp()
+        cvPath = temp_path + "/cv"
+        cv_with_user_folds.save(cvPath)
+        loadedCV = CrossValidator.load(cvPath)
+        self.assertEqual(loadedCV.getFoldCol(), cv_with_user_folds.getFoldCol())
 
-class TrainValidationSplitTests(SparkSessionTestCase):
+    def test_invalid_user_specified_folds(self):
+        dataset_with_folds = self.spark.createDataFrame(
+            [(Vectors.dense([0.0]), 0.0, 0),
+             (Vectors.dense([0.4]), 1.0, 1),
+             (Vectors.dense([0.5]), 0.0, 2),
+             (Vectors.dense([0.6]), 1.0, 0),
+             (Vectors.dense([1.0]), 1.0, 1)] * 10,
+            ["features", "label", "fold"])
+
+        lr = LogisticRegression()
+        grid = ParamGridBuilder().addGrid(lr.maxIter, [20]).build()
+        evaluator = BinaryClassificationEvaluator()
+
+        cv = CrossValidator(estimator=lr,
+                            estimatorParamMaps=grid,
+                            evaluator=evaluator,
+                            numFolds=2,
+                            foldCol="fold")
+        with self.assertRaisesRegex(Exception, "Fold number must be in range"):
+            cv.fit(dataset_with_folds)
+
+        cv = CrossValidator(estimator=lr,
+                            estimatorParamMaps=grid,
+                            evaluator=evaluator,
+                            numFolds=4,
+                            foldCol="fold")
+        with self.assertRaisesRegex(Exception, "The validation data at fold 3 is empty"):
+            cv.fit(dataset_with_folds)
+
+
+class TrainValidationSplitTests(SparkSessionTestCase, ValidatorTestUtilsMixin):
 
     def test_fit_minimize_metric(self):
         dataset = self.spark.createDataFrame([
@@ -437,7 +717,7 @@ def test_fit_maximize_metric(self):
                          "validationMetrics has the same size of grid parameter")
         self.assertEqual(1.0, max(validationMetrics))
 
-    def test_save_load_trained_model(self):
+    def _run_test_save_load_trained_model(self, LogisticRegressionCls, LogisticRegressionModelCls):
         # This tests saving and loading the trained model only.
         # Save/load for TrainValidationSplit will be added later: SPARK-13786
         temp_path = tempfile.mkdtemp()
@@ -448,20 +728,44 @@ def test_save_load_trained_model(self):
              (Vectors.dense([0.6]), 1.0),
              (Vectors.dense([1.0]), 1.0)] * 10,
             ["features", "label"])
-        lr = LogisticRegression()
+        lr = LogisticRegressionCls()
         grid = ParamGridBuilder().addGrid(lr.maxIter, [0, 1]).build()
         evaluator = BinaryClassificationEvaluator()
-        tvs = TrainValidationSplit(estimator=lr, estimatorParamMaps=grid, evaluator=evaluator)
+        tvs = TrainValidationSplit(
+            estimator=lr,
+            estimatorParamMaps=grid,
+            evaluator=evaluator,
+            collectSubModels=True,
+            seed=42
+        )
         tvsModel = tvs.fit(dataset)
         lrModel = tvsModel.bestModel
 
-        tvsModelPath = temp_path + "/tvsModel"
-        lrModel.save(tvsModelPath)
-        loadedLrModel = LogisticRegressionModel.load(tvsModelPath)
+        lrModelPath = temp_path + "/lrModel"
+        lrModel.save(lrModelPath)
+        loadedLrModel = LogisticRegressionModelCls.load(lrModelPath)
         self.assertEqual(loadedLrModel.uid, lrModel.uid)
         self.assertEqual(loadedLrModel.intercept, lrModel.intercept)
 
-    def test_save_load_simple_estimator(self):
+        tvsModelPath = temp_path + "/tvsModel"
+        tvsModel.save(tvsModelPath)
+        loadedTvsModel = TrainValidationSplitModel.load(tvsModelPath)
+        for param in [
+            lambda x: x.getSeed(),
+            lambda x: x.getTrainRatio(),
+        ]:
+            self.assertEqual(param(tvsModel), param(loadedTvsModel))
+
+        self.assertTrue(all(
+            loadedTvsModel.isSet(param) for param in loadedTvsModel.params
+        ))
+
+    def test_save_load_trained_model(self):
+        self._run_test_save_load_trained_model(LogisticRegression, LogisticRegressionModel)
+        self._run_test_save_load_trained_model(DummyLogisticRegression,
+                                               DummyLogisticRegressionModel)
+
+    def _run_test_save_load_simple_estimator(self, LogisticRegressionCls, evaluatorCls):
         # This tests saving and loading the trained model only.
         # Save/load for TrainValidationSplit will be added later: SPARK-13786
         temp_path = tempfile.mkdtemp()
@@ -472,9 +776,9 @@ def test_save_load_simple_estimator(self):
              (Vectors.dense([0.6]), 1.0),
              (Vectors.dense([1.0]), 1.0)] * 10,
             ["features", "label"])
-        lr = LogisticRegression()
+        lr = LogisticRegressionCls()
         grid = ParamGridBuilder().addGrid(lr.maxIter, [0, 1]).build()
-        evaluator = BinaryClassificationEvaluator()
+        evaluator = evaluatorCls()
         tvs = TrainValidationSplit(estimator=lr, estimatorParamMaps=grid, evaluator=evaluator)
         tvsModel = tvs.fit(dataset)
 
@@ -483,13 +787,20 @@ def test_save_load_simple_estimator(self):
         loadedTvs = TrainValidationSplit.load(tvsPath)
         self.assertEqual(loadedTvs.getEstimator().uid, tvs.getEstimator().uid)
         self.assertEqual(loadedTvs.getEvaluator().uid, tvs.getEvaluator().uid)
-        self.assertEqual(loadedTvs.getEstimatorParamMaps(), tvs.getEstimatorParamMaps())
+        self.assert_param_maps_equal(
+            loadedTvs.getEstimatorParamMaps(), tvs.getEstimatorParamMaps())
 
         tvsModelPath = temp_path + "/tvsModel"
         tvsModel.save(tvsModelPath)
         loadedModel = TrainValidationSplitModel.load(tvsModelPath)
         self.assertEqual(loadedModel.bestModel.uid, tvsModel.bestModel.uid)
 
+    def test_save_load_simple_estimator(self):
+        self._run_test_save_load_simple_estimator(
+            LogisticRegression, BinaryClassificationEvaluator)
+        self._run_test_save_load_simple_estimator(
+            DummyLogisticRegression, DummyEvaluator)
+
     def test_parallel_evaluation(self):
         dataset = self.spark.createDataFrame(
             [(Vectors.dense([0.0]), 0.0),
@@ -542,7 +853,7 @@ def test_expose_sub_models(self):
         for i in range(len(grid)):
             self.assertEqual(tvsModel.subModels[i].uid, tvsModel3.subModels[i].uid)
 
-    def test_save_load_nested_estimator(self):
+    def _run_test_save_load_nested_estimator(self, LogisticRegressionCls):
         # This tests saving and loading the trained model only.
         # Save/load for TrainValidationSplit will be added later: SPARK-13786
         temp_path = tempfile.mkdtemp()
@@ -553,9 +864,9 @@ def test_save_load_nested_estimator(self):
              (Vectors.dense([0.6]), 1.0),
              (Vectors.dense([1.0]), 1.0)] * 10,
             ["features", "label"])
-        ova = OneVsRest(classifier=LogisticRegression())
-        lr1 = LogisticRegression().setMaxIter(100)
-        lr2 = LogisticRegression().setMaxIter(150)
+        ova = OneVsRest(classifier=LogisticRegressionCls())
+        lr1 = LogisticRegressionCls().setMaxIter(100)
+        lr2 = LogisticRegressionCls().setMaxIter(150)
         grid = ParamGridBuilder().addGrid(ova.classifier, [lr1, lr2]).build()
         evaluator = MulticlassClassificationEvaluator()
 
@@ -564,6 +875,7 @@ def test_save_load_nested_estimator(self):
         tvsPath = temp_path + "/tvs"
         tvs.save(tvsPath)
         loadedTvs = TrainValidationSplit.load(tvsPath)
+        self.assert_param_maps_equal(loadedTvs.getEstimatorParamMaps(), grid)
         self.assertEqual(loadedTvs.getEstimator().uid, tvs.getEstimator().uid)
         self.assertEqual(loadedTvs.getEvaluator().uid, tvs.getEvaluator().uid)
 
@@ -579,9 +891,14 @@ def test_save_load_nested_estimator(self):
         tvsModelPath = temp_path + "/tvsModel"
         tvsModel.save(tvsModelPath)
         loadedModel = TrainValidationSplitModel.load(tvsModelPath)
+        self.assert_param_maps_equal(loadedModel.getEstimatorParamMaps(), grid)
         self.assertEqual(loadedModel.bestModel.uid, tvsModel.bestModel.uid)
 
-    def test_save_load_pipeline_estimator(self):
+    def test_save_load_nested_estimator(self):
+        self._run_test_save_load_nested_estimator(LogisticRegression)
+        self._run_test_save_load_nested_estimator(DummyLogisticRegression)
+
+    def _run_test_save_load_pipeline_estimator(self, LogisticRegressionCls):
         temp_path = tempfile.mkdtemp()
         training = self.spark.createDataFrame([
             (0, "a b c d e spark", 1.0),
@@ -598,9 +915,9 @@ def test_save_load_pipeline_estimator(self):
         tokenizer = Tokenizer(inputCol="text", outputCol="words")
         hashingTF = HashingTF(inputCol=tokenizer.getOutputCol(), outputCol="features")
 
-        ova = OneVsRest(classifier=LogisticRegression())
-        lr1 = LogisticRegression().setMaxIter(5)
-        lr2 = LogisticRegression().setMaxIter(10)
+        ova = OneVsRest(classifier=LogisticRegressionCls())
+        lr1 = LogisticRegressionCls().setMaxIter(5)
+        lr2 = LogisticRegressionCls().setMaxIter(10)
 
         pipeline = Pipeline(stages=[tokenizer, hashingTF, ova])
 
@@ -612,6 +929,11 @@ def test_save_load_pipeline_estimator(self):
         tvs = TrainValidationSplit(estimator=pipeline,
                                    estimatorParamMaps=paramGrid,
                                    evaluator=MulticlassClassificationEvaluator())
+        tvsPath = temp_path + "/tvs"
+        tvs.save(tvsPath)
+        loadedTvs = TrainValidationSplit.load(tvsPath)
+        self.assert_param_maps_equal(loadedTvs.getEstimatorParamMaps(), paramGrid)
+        self.assertEqual(loadedTvs.getEstimator().uid, tvs.getEstimator().uid)
 
         # Run train validation split, and choose the best set of parameters.
         tvsModel = tvs.fit(training)
@@ -631,6 +953,11 @@ def test_save_load_pipeline_estimator(self):
         tvs2 = TrainValidationSplit(estimator=nested_pipeline,
                                     estimatorParamMaps=paramGrid,
                                     evaluator=MulticlassClassificationEvaluator())
+        tvs2Path = temp_path + "/tvs2"
+        tvs2.save(tvs2Path)
+        loadedTvs2 = TrainValidationSplit.load(tvs2Path)
+        self.assert_param_maps_equal(loadedTvs2.getEstimatorParamMaps(), paramGrid)
+        self.assertEqual(loadedTvs2.getEstimator().uid, tvs2.getEstimator().uid)
 
         # Run train validation split, and choose the best set of parameters.
         tvsModel2 = tvs2.fit(training)
@@ -648,6 +975,10 @@ def test_save_load_pipeline_estimator(self):
                                               original_nested_pipeline_model.stages):
             self.assertEqual(loadedStage.uid, originalStage.uid)
 
+    def test_save_load_pipeline_estimator(self):
+        self._run_test_save_load_pipeline_estimator(LogisticRegression)
+        self._run_test_save_load_pipeline_estimator(DummyLogisticRegression)
+
     def test_copy(self):
         dataset = self.spark.createDataFrame([
             (10, 10.0),
@@ -662,11 +993,30 @@ def test_copy(self):
         grid = ParamGridBuilder() \
             .addGrid(iee.inducedError, [100.0, 0.0, 10000.0]) \
             .build()
-        tvs = TrainValidationSplit(estimator=iee, estimatorParamMaps=grid, evaluator=evaluator)
+        tvs = TrainValidationSplit(
+            estimator=iee,
+            estimatorParamMaps=grid,
+            evaluator=evaluator,
+            collectSubModels=True
+        )
         tvsModel = tvs.fit(dataset)
         tvsCopied = tvs.copy()
         tvsModelCopied = tvsModel.copy()
 
+        for param in [
+            lambda x: x.getCollectSubModels(),
+            lambda x: x.getParallelism(),
+            lambda x: x.getSeed(),
+            lambda x: x.getTrainRatio(),
+        ]:
+            self.assertEqual(param(tvs), param(tvsCopied))
+
+        for param in [
+            lambda x: x.getSeed(),
+            lambda x: x.getTrainRatio(),
+        ]:
+            self.assertEqual(param(tvsModel), param(tvsModelCopied))
+
         self.assertEqual(tvs.getEstimator().uid, tvsCopied.getEstimator().uid,
                          "Copied TrainValidationSplit has the same uid of Estimator")
 
@@ -678,12 +1028,25 @@ def test_copy(self):
             self.assertEqual(tvsModel.validationMetrics[index],
                              tvsModelCopied.validationMetrics[index])
 
+        tvsModel.validationMetrics[0] = 'foo'
+        self.assertNotEqual(
+            tvsModelCopied.validationMetrics[0],
+            'foo',
+            "Changing the original validationMetrics should not affect the copied model"
+        )
+        tvsModel.subModels[0].getInducedError = lambda: 'foo'
+        self.assertNotEqual(
+            tvsModelCopied.subModels[0].getInducedError(),
+            'foo',
+            "Changing the original subModels should not affect the copied model"
+        )
+
 
 if __name__ == "__main__":
-    from pyspark.ml.tests.test_tuning import *
+    from pyspark.ml.tests.test_tuning import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/ml/tests/test_util.py b/python/pyspark/ml/tests/test_util.py
new file mode 100644
index 0000000000000..498a649e480a8
--- /dev/null
+++ b/python/pyspark/ml/tests/test_util.py
@@ -0,0 +1,84 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import unittest
+
+from pyspark.ml import Pipeline
+from pyspark.ml.classification import LogisticRegression, OneVsRest
+from pyspark.ml.feature import VectorAssembler
+from pyspark.ml.linalg import Vectors
+from pyspark.ml.util import MetaAlgorithmReadWrite
+from pyspark.testing.mlutils import SparkSessionTestCase
+
+
+class MetaAlgorithmReadWriteTests(SparkSessionTestCase):
+
+    def test_getAllNestedStages(self):
+        def _check_uid_set_equal(stages, expected_stages):
+            uids = set(map(lambda x: x.uid, stages))
+            expected_uids = set(map(lambda x: x.uid, expected_stages))
+            self.assertEqual(uids, expected_uids)
+
+        df1 = self.spark.createDataFrame([
+            (Vectors.dense([1., 2.]), 1.0),
+            (Vectors.dense([-1., -2.]), 0.0),
+        ], ['features', 'label'])
+        df2 = self.spark.createDataFrame([
+            (1., 2., 1.0),
+            (1., 2., 0.0),
+        ], ['a', 'b', 'label'])
+        vs = VectorAssembler(inputCols=['a', 'b'], outputCol='features')
+        lr = LogisticRegression()
+        pipeline = Pipeline(stages=[vs, lr])
+        pipelineModel = pipeline.fit(df2)
+        ova = OneVsRest(classifier=lr)
+        ovaModel = ova.fit(df1)
+
+        ova_pipeline = Pipeline(stages=[vs, ova])
+        nested_pipeline = Pipeline(stages=[ova_pipeline])
+
+        _check_uid_set_equal(
+            MetaAlgorithmReadWrite.getAllNestedStages(pipeline),
+            [pipeline, vs, lr]
+        )
+        _check_uid_set_equal(
+            MetaAlgorithmReadWrite.getAllNestedStages(pipelineModel),
+            [pipelineModel] + pipelineModel.stages
+        )
+        _check_uid_set_equal(
+            MetaAlgorithmReadWrite.getAllNestedStages(ova),
+            [ova, lr]
+        )
+        _check_uid_set_equal(
+            MetaAlgorithmReadWrite.getAllNestedStages(ovaModel),
+            [ovaModel, lr] + ovaModel.models
+        )
+        _check_uid_set_equal(
+            MetaAlgorithmReadWrite.getAllNestedStages(nested_pipeline),
+            [nested_pipeline, ova_pipeline, vs, ova, lr]
+        )
+
+
+if __name__ == "__main__":
+    from pyspark.ml.tests.test_util import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/ml/tests/test_wrapper.py b/python/pyspark/ml/tests/test_wrapper.py
index c0747155cb72e..8ed6a6bad95ed 100644
--- a/python/pyspark/ml/tests/test_wrapper.py
+++ b/python/pyspark/ml/tests/test_wrapper.py
@@ -21,7 +21,9 @@
 
 from pyspark.ml.linalg import DenseVector, Vectors
 from pyspark.ml.regression import LinearRegression
-from pyspark.ml.wrapper import _java2py, _py2java, JavaParams, JavaWrapper
+from pyspark.ml.wrapper import (  # type: ignore[attr-defined]
+    _java2py, _py2java, JavaParams, JavaWrapper
+)
 from pyspark.testing.mllibutils import MLlibTestCase
 from pyspark.testing.mlutils import SparkSessionTestCase
 from pyspark.testing.utils import eventually
@@ -52,7 +54,7 @@ def test_java_object_gets_detached(self):
         model.__del__()
 
         def condition():
-            with self.assertRaisesRegexp(py4j.protocol.Py4JError, error_no_object):
+            with self.assertRaisesRegex(py4j.protocol.Py4JError, error_no_object):
                 model._java_obj.toString()
             self.assertIn("LinearRegressionTrainingSummary", summary._java_obj.toString())
             return True
@@ -65,9 +67,9 @@ def condition():
             pass
 
         def condition():
-            with self.assertRaisesRegexp(py4j.protocol.Py4JError, error_no_object):
+            with self.assertRaisesRegex(py4j.protocol.Py4JError, error_no_object):
                 model._java_obj.toString()
-            with self.assertRaisesRegexp(py4j.protocol.Py4JError, error_no_object):
+            with self.assertRaisesRegex(py4j.protocol.Py4JError, error_no_object):
                 summary._java_obj.toString()
             return True
 
@@ -117,10 +119,10 @@ def test_new_java_array(self):
         self.assertEqual(_java2py(self.sc, java_array), expected_str_list)
 
 if __name__ == "__main__":
-    from pyspark.ml.tests.test_wrapper import *
+    from pyspark.ml.tests.test_wrapper import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/ml/tree.py b/python/pyspark/ml/tree.py
index a13b27ec8a79c..7ddeb097c4676 100644
--- a/python/pyspark/ml/tree.py
+++ b/python/pyspark/ml/tree.py
@@ -15,12 +15,12 @@
 # limitations under the License.
 #
 
-from pyspark import since, keyword_only
-from pyspark.ml.param.shared import *
-from pyspark.ml.util import *
-from pyspark.ml.wrapper import JavaEstimator, JavaModel, JavaParams, \
-    JavaPredictor, JavaPredictionModel
-from pyspark.ml.common import inherit_doc, _java2py, _py2java
+from pyspark import since
+from pyspark.ml.param import Params
+from pyspark.ml.param.shared import HasCheckpointInterval, HasSeed, HasWeightCol, Param, \
+    TypeConverters, HasMaxIter, HasStepSize, HasValidationIndicatorCol
+from pyspark.ml.wrapper import JavaPredictionModel
+from pyspark.ml.common import inherit_doc
 
 
 @inherit_doc
@@ -67,7 +67,8 @@ class _DecisionTreeParams(HasCheckpointInterval, HasSeed, HasWeightCol):
                     typeConverter=TypeConverters.toString)
 
     maxDepth = Param(Params._dummy(), "maxDepth", "Maximum depth of the tree. (>= 0) E.g., " +
-                     "depth 0 means 1 leaf node; depth 1 means 1 internal node + 2 leaf nodes.",
+                     "depth 0 means 1 leaf node; depth 1 means 1 internal node + 2 leaf nodes. " +
+                     "Must be in range [0, 30].",
                      typeConverter=TypeConverters.toInt)
 
     maxBins = Param(Params._dummy(), "maxBins", "Max number of bins for discretizing continuous " +
diff --git a/python/pyspark/ml/tree.pyi b/python/pyspark/ml/tree.pyi
new file mode 100644
index 0000000000000..ff6307654c569
--- /dev/null
+++ b/python/pyspark/ml/tree.pyi
@@ -0,0 +1,112 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import List, Sequence
+from pyspark.ml._typing import P, T
+
+from pyspark.ml.linalg import Vector
+from pyspark import since as since  # noqa: F401
+from pyspark.ml.common import inherit_doc as inherit_doc  # noqa: F401
+from pyspark.ml.param import Param, Params as Params
+from pyspark.ml.param.shared import (  # noqa: F401
+    HasCheckpointInterval as HasCheckpointInterval,
+    HasMaxIter as HasMaxIter,
+    HasSeed as HasSeed,
+    HasStepSize as HasStepSize,
+    HasValidationIndicatorCol as HasValidationIndicatorCol,
+    HasWeightCol as HasWeightCol,
+    Param as Param,
+    TypeConverters as TypeConverters,
+)
+from pyspark.ml.wrapper import JavaPredictionModel as JavaPredictionModel
+
+class _DecisionTreeModel(JavaPredictionModel[T]):
+    @property
+    def numNodes(self) -> int: ...
+    @property
+    def depth(self) -> int: ...
+    @property
+    def toDebugString(self) -> str: ...
+    def predictLeaf(self, value: Vector) -> float: ...
+
+class _DecisionTreeParams(HasCheckpointInterval, HasSeed, HasWeightCol):
+    leafCol: Param[str]
+    maxDepth: Param[int]
+    maxBins: Param[int]
+    minInstancesPerNode: Param[int]
+    minWeightFractionPerNode: Param[float]
+    minInfoGain: Param[float]
+    maxMemoryInMB: Param[int]
+    cacheNodeIds: Param[bool]
+    def __init__(self) -> None: ...
+    def setLeafCol(self: P, value: str) -> P: ...
+    def getLeafCol(self) -> str: ...
+    def getMaxDepth(self) -> int: ...
+    def getMaxBins(self) -> int: ...
+    def getMinInstancesPerNode(self) -> int: ...
+    def getMinInfoGain(self) -> float: ...
+    def getMaxMemoryInMB(self) -> int: ...
+    def getCacheNodeIds(self) -> bool: ...
+
+class _TreeEnsembleModel(JavaPredictionModel[T]):
+    @property
+    def trees(self) -> Sequence[_DecisionTreeModel]: ...
+    @property
+    def getNumTrees(self) -> int: ...
+    @property
+    def treeWeights(self) -> List[float]: ...
+    @property
+    def totalNumNodes(self) -> int: ...
+    @property
+    def toDebugString(self) -> str: ...
+
+class _TreeEnsembleParams(_DecisionTreeParams):
+    subsamplingRate: Param[float]
+    supportedFeatureSubsetStrategies: List[str]
+    featureSubsetStrategy: Param[str]
+    def __init__(self) -> None: ...
+    def getSubsamplingRate(self) -> float: ...
+    def getFeatureSubsetStrategy(self) -> str: ...
+
+class _RandomForestParams(_TreeEnsembleParams):
+    numTrees: Param[int]
+    bootstrap: Param[bool]
+    def __init__(self) -> None: ...
+    def getNumTrees(self) -> int: ...
+    def getBootstrap(self) -> bool: ...
+
+class _GBTParams(
+    _TreeEnsembleParams, HasMaxIter, HasStepSize, HasValidationIndicatorCol
+):
+    stepSize: Param[float]
+    validationTol: Param[float]
+    def getValidationTol(self) -> float: ...
+
+class _HasVarianceImpurity(Params):
+    supportedImpurities: List[str]
+    impurity: Param[str]
+    def __init__(self) -> None: ...
+    def getImpurity(self) -> str: ...
+
+class _TreeClassifierParams(Params):
+    supportedImpurities: List[str]
+    impurity: Param[str]
+    def __init__(self) -> None: ...
+    def getImpurity(self) -> str: ...
+
+class _TreeRegressorParams(_HasVarianceImpurity): ...
diff --git a/python/pyspark/ml/tuning.py b/python/pyspark/ml/tuning.py
index cb4542c5d025f..85174c8cd02f2 100644
--- a/python/pyspark/ml/tuning.py
+++ b/python/pyspark/ml/tuning.py
@@ -14,23 +14,30 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-import itertools
+
+import os
 import sys
+import itertools
+import random
+import math
 from multiprocessing.pool import ThreadPool
 
 import numpy as np
 
-from pyspark import since, keyword_only
-from pyspark.ml import Estimator, Model
-from pyspark.ml.common import _py2java, _java2py
+from pyspark import keyword_only, since, SparkContext
+from pyspark.ml import Estimator, Transformer, Model
+from pyspark.ml.common import inherit_doc, _py2java, _java2py
+from pyspark.ml.evaluation import Evaluator
 from pyspark.ml.param import Params, Param, TypeConverters
 from pyspark.ml.param.shared import HasCollectSubModels, HasParallelism, HasSeed
-from pyspark.ml.util import *
-from pyspark.ml.wrapper import JavaParams
-from pyspark.sql.functions import rand
+from pyspark.ml.util import DefaultParamsReader, DefaultParamsWriter, MetaAlgorithmReadWrite, \
+    MLReadable, MLReader, MLWritable, MLWriter, JavaMLReader, JavaMLWriter
+from pyspark.ml.wrapper import JavaParams, JavaEstimator, JavaWrapper
+from pyspark.sql.functions import col, lit, rand, UserDefinedFunction
+from pyspark.sql.types import BooleanType
 
 __all__ = ['ParamGridBuilder', 'CrossValidator', 'CrossValidatorModel', 'TrainValidationSplit',
-           'TrainValidationSplitModel']
+           'TrainValidationSplitModel', 'ParamRandomBuilder']
 
 
 def _parallelFitTasks(est, train, eva, validation, epm, collectSubModel):
@@ -38,18 +45,34 @@ def _parallelFitTasks(est, train, eva, validation, epm, collectSubModel):
     Creates a list of callables which can be called from different threads to fit and evaluate
     an estimator in parallel. Each callable returns an `(index, metric)` pair.
 
-    :param est: Estimator, the estimator to be fit.
-    :param train: DataFrame, training data set, used for fitting.
-    :param eva: Evaluator, used to compute `metric`
-    :param validation: DataFrame, validation data set, used for evaluation.
-    :param epm: Sequence of ParamMap, params maps to be used during fitting & evaluation.
-    :param collectSubModel: Whether to collect sub model.
-    :return: (int, float, subModel), an index into `epm` and the associated metric value.
+    Parameters
+    ----------
+    est : :py:class:`pyspark.ml.baseEstimator`
+        he estimator to be fit.
+    train : :py:class:`pyspark.sql.DataFrame`
+        DataFrame, training data set, used for fitting.
+    eva : :py:class:`pyspark.ml.evaluation.Evaluator`
+        used to compute `metric`
+    validation : :py:class:`pyspark.sql.DataFrame`
+        DataFrame, validation data set, used for evaluation.
+    epm : :py:class:`collections.abc.Sequence`
+        Sequence of ParamMap, params maps to be used during fitting & evaluation.
+    collectSubModel : bool
+        Whether to collect sub model.
+
+    Returns
+    -------
+    tuple
+        (int, float, subModel), an index into `epm` and the associated metric value.
     """
     modelIter = est.fitMultiple(train, epm)
 
     def singleTask():
         index, model = next(modelIter)
+        # TODO: duplicate evaluator to take extra params from input
+        #  Note: Supporting tuning params in evaluator need update method
+        #  `MetaAlgorithmReadWrite.getAllNestedStages`, make it return
+        #  all nested stages and evaluators
         metric = eva.evaluate(model.transform(validation, epm[index]))
         return index, metric, model if collectSubModel else None
 
@@ -60,6 +83,11 @@ class ParamGridBuilder(object):
     r"""
     Builder for a param grid used in grid search-based model selection.
 
+
+    .. versionadded:: 1.4.0
+
+    Examples
+    --------
     >>> from pyspark.ml.classification import LogisticRegression
     >>> lr = LogisticRegression()
     >>> output = ParamGridBuilder() \
@@ -77,8 +105,6 @@ class ParamGridBuilder(object):
     True
     >>> all([m in expected for m in output])
     True
-
-    .. versionadded:: 1.4.0
     """
 
     def __init__(self):
@@ -128,6 +154,50 @@ def to_key_value_pairs(keys, values):
         return [dict(to_key_value_pairs(keys, prod)) for prod in itertools.product(*grid_values)]
 
 
+class ParamRandomBuilder(ParamGridBuilder):
+    r"""
+    Builder for random value parameters used in search-based model selection.
+
+
+    .. versionadded:: 3.2.0
+    """
+
+    @since("3.2.0")
+    def addRandom(self, param, x, y, n):
+        """
+        Adds n random values between x and y.
+        The arguments x and y can be integers, floats or a combination of the two. If either
+        x or y is a float, the domain of the random value will be float.
+        """
+        if type(x) == int and type(y) == int:
+            values = map(lambda _: random.randrange(x, y), range(n))
+        elif type(x) == float or type(y) == float:
+            values = map(lambda _: random.uniform(x, y), range(n))
+        else:
+            raise TypeError("unable to make range for types %s and %s" % type(x) % type(y))
+        self.addGrid(param, values)
+        return self
+
+    @since("3.2.0")
+    def addLog10Random(self, param, x, y, n):
+        """
+        Adds n random values scaled logarithmically (base 10) between x and y.
+        For instance, a distribution for x=1.0, y=10000.0 and n=5 might reasonably look like
+        [1.6, 65.3, 221.9, 1024.3, 8997.5]
+        """
+        def logarithmic_random():
+            rand = random.uniform(math.log10(x), math.log10(y))
+            value = 10 ** rand
+            if type(x) == int and type(y) == int:
+                value = int(value)
+            return value
+
+        values = map(lambda _: logarithmic_random(), range(n))
+        self.addGrid(param, values)
+
+        return self
+
+
 class _ValidatorParams(HasSeed):
     """
     Common params for TrainValidationSplit and CrossValidator.
@@ -169,8 +239,16 @@ def _from_java_impl(cls, java_stage):
         # Load information from java_stage to the instance.
         estimator = JavaParams._from_java(java_stage.getEstimator())
         evaluator = JavaParams._from_java(java_stage.getEvaluator())
-        epms = [estimator._transfer_param_map_from_java(epm)
-                for epm in java_stage.getEstimatorParamMaps()]
+        if isinstance(estimator, JavaEstimator):
+            epms = [estimator._transfer_param_map_from_java(epm)
+                    for epm in java_stage.getEstimatorParamMaps()]
+        elif MetaAlgorithmReadWrite.isMetaEstimator(estimator):
+            # Meta estimator such as Pipeline, OneVsRest
+            epms = _ValidatorSharedReadWrite.meta_estimator_transfer_param_maps_from_java(
+                estimator, java_stage.getEstimatorParamMaps())
+        else:
+            raise ValueError('Unsupported estimator used in tuning: ' + str(estimator))
+
         return estimator, epms, evaluator
 
     def _to_java_impl(self):
@@ -181,15 +259,297 @@ def _to_java_impl(self):
         gateway = SparkContext._gateway
         cls = SparkContext._jvm.org.apache.spark.ml.param.ParamMap
 
-        java_epms = gateway.new_array(cls, len(self.getEstimatorParamMaps()))
-        for idx, epm in enumerate(self.getEstimatorParamMaps()):
-            java_epms[idx] = self.getEstimator()._transfer_param_map_to_java(epm)
+        estimator = self.getEstimator()
+        if isinstance(estimator, JavaEstimator):
+            java_epms = gateway.new_array(cls, len(self.getEstimatorParamMaps()))
+            for idx, epm in enumerate(self.getEstimatorParamMaps()):
+                java_epms[idx] = self.getEstimator()._transfer_param_map_to_java(epm)
+        elif MetaAlgorithmReadWrite.isMetaEstimator(estimator):
+            # Meta estimator such as Pipeline, OneVsRest
+            java_epms = _ValidatorSharedReadWrite.meta_estimator_transfer_param_maps_to_java(
+                estimator, self.getEstimatorParamMaps())
+        else:
+            raise ValueError('Unsupported estimator used in tuning: ' + str(estimator))
 
         java_estimator = self.getEstimator()._to_java()
         java_evaluator = self.getEvaluator()._to_java()
         return java_estimator, java_epms, java_evaluator
 
 
+class _ValidatorSharedReadWrite:
+
+    @staticmethod
+    def meta_estimator_transfer_param_maps_to_java(pyEstimator, pyParamMaps):
+        pyStages = MetaAlgorithmReadWrite.getAllNestedStages(pyEstimator)
+        stagePairs = list(map(lambda stage: (stage, stage._to_java()), pyStages))
+        sc = SparkContext._active_spark_context
+
+        paramMapCls = SparkContext._jvm.org.apache.spark.ml.param.ParamMap
+        javaParamMaps = SparkContext._gateway.new_array(paramMapCls, len(pyParamMaps))
+
+        for idx, pyParamMap in enumerate(pyParamMaps):
+            javaParamMap = JavaWrapper._new_java_obj("org.apache.spark.ml.param.ParamMap")
+            for pyParam, pyValue in pyParamMap.items():
+                javaParam = None
+                for pyStage, javaStage in stagePairs:
+                    if pyStage._testOwnParam(pyParam.parent, pyParam.name):
+                        javaParam = javaStage.getParam(pyParam.name)
+                        break
+                if javaParam is None:
+                    raise ValueError('Resolve param in estimatorParamMaps failed: ' + str(pyParam))
+                if isinstance(pyValue, Params) and hasattr(pyValue, "_to_java"):
+                    javaValue = pyValue._to_java()
+                else:
+                    javaValue = _py2java(sc, pyValue)
+                pair = javaParam.w(javaValue)
+                javaParamMap.put([pair])
+            javaParamMaps[idx] = javaParamMap
+        return javaParamMaps
+
+    @staticmethod
+    def meta_estimator_transfer_param_maps_from_java(pyEstimator, javaParamMaps):
+        pyStages = MetaAlgorithmReadWrite.getAllNestedStages(pyEstimator)
+        stagePairs = list(map(lambda stage: (stage, stage._to_java()), pyStages))
+        sc = SparkContext._active_spark_context
+        pyParamMaps = []
+        for javaParamMap in javaParamMaps:
+            pyParamMap = dict()
+            for javaPair in javaParamMap.toList():
+                javaParam = javaPair.param()
+                pyParam = None
+                for pyStage, javaStage in stagePairs:
+                    if pyStage._testOwnParam(javaParam.parent(), javaParam.name()):
+                        pyParam = pyStage.getParam(javaParam.name())
+                if pyParam is None:
+                    raise ValueError('Resolve param in estimatorParamMaps failed: ' +
+                                     javaParam.parent() + '.' + javaParam.name())
+                javaValue = javaPair.value()
+                if sc._jvm.Class.forName("org.apache.spark.ml.util.DefaultParamsWritable") \
+                        .isInstance(javaValue):
+                    pyValue = JavaParams._from_java(javaValue)
+                else:
+                    pyValue = _java2py(sc, javaValue)
+                pyParamMap[pyParam] = pyValue
+            pyParamMaps.append(pyParamMap)
+        return pyParamMaps
+
+    @staticmethod
+    def is_java_convertible(instance):
+        allNestedStages = MetaAlgorithmReadWrite.getAllNestedStages(instance.getEstimator())
+        evaluator_convertible = isinstance(instance.getEvaluator(), JavaParams)
+        estimator_convertible = all(map(lambda stage: hasattr(stage, '_to_java'), allNestedStages))
+        return estimator_convertible and evaluator_convertible
+
+    @staticmethod
+    def saveImpl(path, instance, sc, extraMetadata=None):
+        numParamsNotJson = 0
+        jsonEstimatorParamMaps = []
+        for paramMap in instance.getEstimatorParamMaps():
+            jsonParamMap = []
+            for p, v in paramMap.items():
+                jsonParam = {'parent': p.parent, 'name': p.name}
+                if (isinstance(v, Estimator) and not MetaAlgorithmReadWrite.isMetaEstimator(v)) \
+                        or isinstance(v, Transformer) or isinstance(v, Evaluator):
+                    relative_path = f'epm_{p.name}{numParamsNotJson}'
+                    param_path = os.path.join(path, relative_path)
+                    numParamsNotJson += 1
+                    v.save(param_path)
+                    jsonParam['value'] = relative_path
+                    jsonParam['isJson'] = False
+                elif isinstance(v, MLWritable):
+                    raise RuntimeError(
+                        "ValidatorSharedReadWrite.saveImpl does not handle parameters of type: "
+                        "MLWritable that are not Estimaor/Evaluator/Transformer, and if parameter "
+                        "is estimator, it cannot be meta estimator such as Validator or OneVsRest")
+                else:
+                    jsonParam['value'] = v
+                    jsonParam['isJson'] = True
+                jsonParamMap.append(jsonParam)
+            jsonEstimatorParamMaps.append(jsonParamMap)
+
+        skipParams = ['estimator', 'evaluator', 'estimatorParamMaps']
+        jsonParams = DefaultParamsWriter.extractJsonParams(instance, skipParams)
+        jsonParams['estimatorParamMaps'] = jsonEstimatorParamMaps
+
+        DefaultParamsWriter.saveMetadata(instance, path, sc, extraMetadata, jsonParams)
+        evaluatorPath = os.path.join(path, 'evaluator')
+        instance.getEvaluator().save(evaluatorPath)
+        estimatorPath = os.path.join(path, 'estimator')
+        instance.getEstimator().save(estimatorPath)
+
+    @staticmethod
+    def load(path, sc, metadata):
+        evaluatorPath = os.path.join(path, 'evaluator')
+        evaluator = DefaultParamsReader.loadParamsInstance(evaluatorPath, sc)
+        estimatorPath = os.path.join(path, 'estimator')
+        estimator = DefaultParamsReader.loadParamsInstance(estimatorPath, sc)
+
+        uidToParams = MetaAlgorithmReadWrite.getUidMap(estimator)
+        uidToParams[evaluator.uid] = evaluator
+
+        jsonEstimatorParamMaps = metadata['paramMap']['estimatorParamMaps']
+
+        estimatorParamMaps = []
+        for jsonParamMap in jsonEstimatorParamMaps:
+            paramMap = {}
+            for jsonParam in jsonParamMap:
+                est = uidToParams[jsonParam['parent']]
+                param = getattr(est, jsonParam['name'])
+                if 'isJson' not in jsonParam or ('isJson' in jsonParam and jsonParam['isJson']):
+                    value = jsonParam['value']
+                else:
+                    relativePath = jsonParam['value']
+                    valueSavedPath = os.path.join(path, relativePath)
+                    value = DefaultParamsReader.loadParamsInstance(valueSavedPath, sc)
+                paramMap[param] = value
+            estimatorParamMaps.append(paramMap)
+
+        return metadata, estimator, evaluator, estimatorParamMaps
+
+    @staticmethod
+    def validateParams(instance):
+        estiamtor = instance.getEstimator()
+        evaluator = instance.getEvaluator()
+        uidMap = MetaAlgorithmReadWrite.getUidMap(estiamtor)
+
+        for elem in [evaluator] + list(uidMap.values()):
+            if not isinstance(elem, MLWritable):
+                raise ValueError(f'Validator write will fail because it contains {elem.uid} '
+                                 f'which is not writable.')
+
+        estimatorParamMaps = instance.getEstimatorParamMaps()
+        paramErr = 'Validator save requires all Params in estimatorParamMaps to apply to ' \
+                   f'its Estimator, An extraneous Param was found: '
+        for paramMap in estimatorParamMaps:
+            for param in paramMap:
+                if param.parent not in uidMap:
+                    raise ValueError(paramErr + repr(param))
+
+    @staticmethod
+    def getValidatorModelWriterPersistSubModelsParam(writer):
+        if 'persistsubmodels' in writer.optionMap:
+            persistSubModelsParam = writer.optionMap['persistsubmodels'].lower()
+            if persistSubModelsParam == 'true':
+                return True
+            elif persistSubModelsParam == 'false':
+                return False
+            else:
+                raise ValueError(
+                    f'persistSubModels option value {persistSubModelsParam} is invalid, '
+                    f"the possible values are True, 'True' or False, 'False'")
+        else:
+            return writer.instance.subModels is not None
+
+
+_save_with_persist_submodels_no_submodels_found_err = \
+    'When persisting tuning models, you can only set persistSubModels to true if the tuning ' \
+    'was done with collectSubModels set to true. To save the sub-models, try rerunning fitting ' \
+    'with collectSubModels set to true.'
+
+
+@inherit_doc
+class CrossValidatorReader(MLReader):
+
+    def __init__(self, cls):
+        super(CrossValidatorReader, self).__init__()
+        self.cls = cls
+
+    def load(self, path):
+        metadata = DefaultParamsReader.loadMetadata(path, self.sc)
+        if not DefaultParamsReader.isPythonParamsInstance(metadata):
+            return JavaMLReader(self.cls).load(path)
+        else:
+            metadata, estimator, evaluator, estimatorParamMaps = \
+                _ValidatorSharedReadWrite.load(path, self.sc, metadata)
+            cv = CrossValidator(estimator=estimator,
+                                estimatorParamMaps=estimatorParamMaps,
+                                evaluator=evaluator)
+            cv = cv._resetUid(metadata['uid'])
+            DefaultParamsReader.getAndSetParams(cv, metadata, skipParams=['estimatorParamMaps'])
+            return cv
+
+
+@inherit_doc
+class CrossValidatorWriter(MLWriter):
+
+    def __init__(self, instance):
+        super(CrossValidatorWriter, self).__init__()
+        self.instance = instance
+
+    def saveImpl(self, path):
+        _ValidatorSharedReadWrite.validateParams(self.instance)
+        _ValidatorSharedReadWrite.saveImpl(path, self.instance, self.sc)
+
+
+@inherit_doc
+class CrossValidatorModelReader(MLReader):
+
+    def __init__(self, cls):
+        super(CrossValidatorModelReader, self).__init__()
+        self.cls = cls
+
+    def load(self, path):
+        metadata = DefaultParamsReader.loadMetadata(path, self.sc)
+        if not DefaultParamsReader.isPythonParamsInstance(metadata):
+            return JavaMLReader(self.cls).load(path)
+        else:
+            metadata, estimator, evaluator, estimatorParamMaps = \
+                _ValidatorSharedReadWrite.load(path, self.sc, metadata)
+            numFolds = metadata['paramMap']['numFolds']
+            bestModelPath = os.path.join(path, 'bestModel')
+            bestModel = DefaultParamsReader.loadParamsInstance(bestModelPath, self.sc)
+            avgMetrics = metadata['avgMetrics']
+            persistSubModels = ('persistSubModels' in metadata) and metadata['persistSubModels']
+
+            if persistSubModels:
+                subModels = [[None] * len(estimatorParamMaps)] * numFolds
+                for splitIndex in range(numFolds):
+                    for paramIndex in range(len(estimatorParamMaps)):
+                        modelPath = os.path.join(
+                            path, 'subModels', f'fold{splitIndex}', f'{paramIndex}')
+                        subModels[splitIndex][paramIndex] = \
+                            DefaultParamsReader.loadParamsInstance(modelPath, self.sc)
+            else:
+                subModels = None
+
+            cvModel = CrossValidatorModel(bestModel, avgMetrics=avgMetrics, subModels=subModels)
+            cvModel = cvModel._resetUid(metadata['uid'])
+            cvModel.set(cvModel.estimator, estimator)
+            cvModel.set(cvModel.estimatorParamMaps, estimatorParamMaps)
+            cvModel.set(cvModel.evaluator, evaluator)
+            DefaultParamsReader.getAndSetParams(
+                cvModel, metadata, skipParams=['estimatorParamMaps'])
+            return cvModel
+
+
+@inherit_doc
+class CrossValidatorModelWriter(MLWriter):
+
+    def __init__(self, instance):
+        super(CrossValidatorModelWriter, self).__init__()
+        self.instance = instance
+
+    def saveImpl(self, path):
+        _ValidatorSharedReadWrite.validateParams(self.instance)
+        instance = self.instance
+        persistSubModels = _ValidatorSharedReadWrite \
+            .getValidatorModelWriterPersistSubModelsParam(self)
+        extraMetadata = {'avgMetrics': instance.avgMetrics,
+                         'persistSubModels': persistSubModels}
+        _ValidatorSharedReadWrite.saveImpl(path, instance, self.sc, extraMetadata=extraMetadata)
+        bestModelPath = os.path.join(path, 'bestModel')
+        instance.bestModel.save(bestModelPath)
+        if persistSubModels:
+            if instance.subModels is None:
+                raise ValueError(_save_with_persist_submodels_no_submodels_found_err)
+            subModelsPath = os.path.join(path, 'subModels')
+            for splitIndex in range(instance.getNumFolds()):
+                splitPath = os.path.join(subModelsPath, f'fold{splitIndex}')
+                for paramIndex in range(len(instance.getEstimatorParamMaps())):
+                    modelPath = os.path.join(splitPath, f'{paramIndex}')
+                    instance.subModels[splitIndex][paramIndex].save(modelPath)
+
+
 class _CrossValidatorParams(_ValidatorParams):
     """
     Params for :py:class:`CrossValidator` and :py:class:`CrossValidatorModel`.
@@ -200,6 +560,16 @@ class _CrossValidatorParams(_ValidatorParams):
     numFolds = Param(Params._dummy(), "numFolds", "number of folds for cross validation",
                      typeConverter=TypeConverters.toInt)
 
+    foldCol = Param(Params._dummy(), "foldCol", "Param for the column name of user " +
+                    "specified fold number. Once this is specified, :py:class:`CrossValidator` " +
+                    "won't do random k-fold split. Note that this column should be integer type " +
+                    "with range [0, numFolds) and Spark will throw exception on out-of-range " +
+                    "fold numbers.", typeConverter=TypeConverters.toString)
+
+    def __init__(self, *args):
+        super(_CrossValidatorParams, self).__init__(*args)
+        self._setDefault(numFolds=3, foldCol="")
+
     @since("1.4.0")
     def getNumFolds(self):
         """
@@ -207,6 +577,13 @@ def getNumFolds(self):
         """
         return self.getOrDefault(self.numFolds)
 
+    @since("3.1.0")
+    def getFoldCol(self):
+        """
+        Gets the value of foldCol or its default value.
+        """
+        return self.getOrDefault(self.foldCol)
+
 
 class CrossValidator(Estimator, _CrossValidatorParams, HasParallelism, HasCollectSubModels,
                      MLReadable, MLWritable):
@@ -218,7 +595,10 @@ class CrossValidator(Estimator, _CrossValidatorParams, HasParallelism, HasCollec
     each of which uses 2/3 of the data for training and 1/3 for testing. Each fold is used as the
     test set exactly once.
 
+    .. versionadded:: 1.4.0
 
+    Examples
+    --------
     >>> from pyspark.ml.classification import LogisticRegression
     >>> from pyspark.ml.evaluation import BinaryClassificationEvaluator
     >>> from pyspark.ml.linalg import Vectors
@@ -249,29 +629,29 @@ class CrossValidator(Estimator, _CrossValidatorParams, HasParallelism, HasCollec
     [0.5, ...
     >>> evaluator.evaluate(cvModel.transform(dataset))
     0.8333...
-
-    .. versionadded:: 1.4.0
+    >>> evaluator.evaluate(cvModelRead.transform(dataset))
+    0.8333...
     """
 
     @keyword_only
-    def __init__(self, estimator=None, estimatorParamMaps=None, evaluator=None, numFolds=3,
-                 seed=None, parallelism=1, collectSubModels=False):
+    def __init__(self, *, estimator=None, estimatorParamMaps=None, evaluator=None, numFolds=3,
+                 seed=None, parallelism=1, collectSubModels=False, foldCol=""):
         """
-        __init__(self, estimator=None, estimatorParamMaps=None, evaluator=None, numFolds=3,\
-                 seed=None, parallelism=1, collectSubModels=False)
+        __init__(self, \\*, estimator=None, estimatorParamMaps=None, evaluator=None, numFolds=3,\
+                 seed=None, parallelism=1, collectSubModels=False, foldCol="")
         """
         super(CrossValidator, self).__init__()
-        self._setDefault(numFolds=3, parallelism=1)
+        self._setDefault(parallelism=1)
         kwargs = self._input_kwargs
         self._set(**kwargs)
 
     @keyword_only
     @since("1.4.0")
-    def setParams(self, estimator=None, estimatorParamMaps=None, evaluator=None, numFolds=3,
-                  seed=None, parallelism=1, collectSubModels=False):
+    def setParams(self, *, estimator=None, estimatorParamMaps=None, evaluator=None, numFolds=3,
+                  seed=None, parallelism=1, collectSubModels=False, foldCol=""):
         """
-        setParams(self, estimator=None, estimatorParamMaps=None, evaluator=None, numFolds=3,\
-                  seed=None, parallelism=1, collectSubModels=False):
+        setParams(self, \\*, estimator=None, estimatorParamMaps=None, evaluator=None, numFolds=3,\
+                  seed=None, parallelism=1, collectSubModels=False, foldCol=""):
         Sets params for cross validator.
         """
         kwargs = self._input_kwargs
@@ -305,6 +685,13 @@ def setNumFolds(self, value):
         """
         return self._set(numFolds=value)
 
+    @since("3.1.0")
+    def setFoldCol(self, value):
+        """
+        Sets the value of :py:attr:`foldCol`.
+        """
+        return self._set(foldCol=value)
+
     def setSeed(self, value):
         """
         Sets the value of :py:attr:`seed`.
@@ -329,10 +716,6 @@ def _fit(self, dataset):
         numModels = len(epm)
         eva = self.getOrDefault(self.evaluator)
         nFolds = self.getOrDefault(self.numFolds)
-        seed = self.getOrDefault(self.seed)
-        h = 1.0 / nFolds
-        randCol = self.uid + "_rand"
-        df = dataset.select("*", rand(seed).alias(randCol))
         metrics = [0.0] * numModels
 
         pool = ThreadPool(processes=min(self.getParallelism(), numModels))
@@ -341,12 +724,10 @@ def _fit(self, dataset):
         if collectSubModelsParam:
             subModels = [[None for j in range(numModels)] for i in range(nFolds)]
 
+        datasets = self._kFold(dataset)
         for i in range(nFolds):
-            validateLB = i * h
-            validateUB = (i + 1) * h
-            condition = (df[randCol] >= validateLB) & (df[randCol] < validateUB)
-            validation = df.filter(condition).cache()
-            train = df.filter(~condition).cache()
+            validation = datasets[i][1].cache()
+            train = datasets[i][0].cache()
 
             tasks = _parallelFitTasks(est, train, eva, validation, epm, collectSubModelsParam)
             for j, metric, subModel in pool.imap_unordered(lambda f: f(), tasks):
@@ -364,15 +745,63 @@ def _fit(self, dataset):
         bestModel = est.fit(dataset, epm[bestIndex])
         return self._copyValues(CrossValidatorModel(bestModel, metrics, subModels))
 
-    @since("1.4.0")
+    def _kFold(self, dataset):
+        nFolds = self.getOrDefault(self.numFolds)
+        foldCol = self.getOrDefault(self.foldCol)
+
+        datasets = []
+        if not foldCol:
+            # Do random k-fold split.
+            seed = self.getOrDefault(self.seed)
+            h = 1.0 / nFolds
+            randCol = self.uid + "_rand"
+            df = dataset.select("*", rand(seed).alias(randCol))
+            for i in range(nFolds):
+                validateLB = i * h
+                validateUB = (i + 1) * h
+                condition = (df[randCol] >= validateLB) & (df[randCol] < validateUB)
+                validation = df.filter(condition)
+                train = df.filter(~condition)
+                datasets.append((train, validation))
+        else:
+            # Use user-specified fold numbers.
+            def checker(foldNum):
+                if foldNum < 0 or foldNum >= nFolds:
+                    raise ValueError(
+                        "Fold number must be in range [0, %s), but got %s." % (nFolds, foldNum))
+                return True
+
+            checker_udf = UserDefinedFunction(checker, BooleanType())
+            for i in range(nFolds):
+                training = dataset.filter(checker_udf(dataset[foldCol]) & (col(foldCol) != lit(i)))
+                validation = dataset.filter(
+                    checker_udf(dataset[foldCol]) & (col(foldCol) == lit(i)))
+                if training.rdd.getNumPartitions() == 0 or len(training.take(1)) == 0:
+                    raise ValueError("The training data at fold %s is empty." % i)
+                if validation.rdd.getNumPartitions() == 0 or len(validation.take(1)) == 0:
+                    raise ValueError("The validation data at fold %s is empty." % i)
+                datasets.append((training, validation))
+
+        return datasets
+
     def copy(self, extra=None):
         """
         Creates a copy of this instance with a randomly generated uid
         and some extra params. This copies creates a deep copy of
         the embedded paramMap, and copies the embedded and extra parameters over.
 
-        :param extra: Extra parameters to copy to the new instance
-        :return: Copy of this instance
+
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        extra : dict, optional
+            Extra parameters to copy to the new instance
+
+        Returns
+        -------
+        :py:class:`CrossValidator`
+            Copy of this instance
         """
         if extra is None:
             extra = dict()
@@ -387,13 +816,15 @@ def copy(self, extra=None):
     @since("2.3.0")
     def write(self):
         """Returns an MLWriter instance for this ML instance."""
-        return JavaMLWriter(self)
+        if _ValidatorSharedReadWrite.is_java_convertible(self):
+            return JavaMLWriter(self)
+        return CrossValidatorWriter(self)
 
     @classmethod
     @since("2.3.0")
     def read(cls):
         """Returns an MLReader instance for this class."""
-        return JavaMLReader(cls)
+        return CrossValidatorReader(cls)
 
     @classmethod
     def _from_java(cls, java_stage):
@@ -407,10 +838,11 @@ def _from_java(cls, java_stage):
         seed = java_stage.getSeed()
         parallelism = java_stage.getParallelism()
         collectSubModels = java_stage.getCollectSubModels()
+        foldCol = java_stage.getFoldCol()
         # Create a new instance of this stage.
         py_stage = cls(estimator=estimator, estimatorParamMaps=epms, evaluator=evaluator,
                        numFolds=numFolds, seed=seed, parallelism=parallelism,
-                       collectSubModels=collectSubModels)
+                       collectSubModels=collectSubModels, foldCol=foldCol)
         py_stage._resetUid(java_stage.uid())
         return py_stage
 
@@ -418,7 +850,10 @@ def _to_java(self):
         """
         Transfer this instance to a Java CrossValidator. Used for ML persistence.
 
-        :return: Java object equivalent to this instance.
+        Returns
+        -------
+        py4j.java_gateway.JavaObject
+            Java object equivalent to this instance.
         """
 
         estimator, epms, evaluator = super(CrossValidator, self)._to_java_impl()
@@ -431,6 +866,7 @@ def _to_java(self):
         _java_obj.setNumFolds(self.getNumFolds())
         _java_obj.setParallelism(self.getParallelism())
         _java_obj.setCollectSubModels(self.getCollectSubModels())
+        _java_obj.setFoldCol(self.getFoldCol())
 
         return _java_obj
 
@@ -445,20 +881,19 @@ class CrossValidatorModel(Model, _CrossValidatorParams, MLReadable, MLWritable):
     .. versionadded:: 1.4.0
     """
 
-    def __init__(self, bestModel, avgMetrics=[], subModels=None):
+    def __init__(self, bestModel, avgMetrics=None, subModels=None):
         super(CrossValidatorModel, self).__init__()
         #: best model from cross validation
         self.bestModel = bestModel
         #: Average cross-validation metrics for each paramMap in
         #: CrossValidator.estimatorParamMaps, in the corresponding order.
-        self.avgMetrics = avgMetrics
+        self.avgMetrics = avgMetrics or []
         #: sub model list from cross validation
         self.subModels = subModels
 
     def _transform(self, dataset):
         return self.bestModel.transform(dataset)
 
-    @since("1.4.0")
     def copy(self, extra=None):
         """
         Creates a copy of this instance with a randomly generated uid
@@ -467,26 +902,40 @@ def copy(self, extra=None):
         copies the embedded and extra parameters over.
         It does not copy the extra Params into the subModels.
 
-        :param extra: Extra parameters to copy to the new instance
-        :return: Copy of this instance
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        extra : dict, optional
+            Extra parameters to copy to the new instance
+
+        Returns
+        -------
+        :py:class:`CrossValidatorModel`
+            Copy of this instance
         """
         if extra is None:
             extra = dict()
         bestModel = self.bestModel.copy(extra)
-        avgMetrics = self.avgMetrics
-        subModels = self.subModels
-        return CrossValidatorModel(bestModel, avgMetrics, subModels)
+        avgMetrics = list(self.avgMetrics)
+        subModels = [
+            [sub_model.copy() for sub_model in fold_sub_models]
+            for fold_sub_models in self.subModels
+        ]
+        return self._copyValues(CrossValidatorModel(bestModel, avgMetrics, subModels), extra=extra)
 
     @since("2.3.0")
     def write(self):
         """Returns an MLWriter instance for this ML instance."""
-        return JavaMLWriter(self)
+        if _ValidatorSharedReadWrite.is_java_convertible(self):
+            return JavaMLWriter(self)
+        return CrossValidatorModelWriter(self)
 
     @classmethod
     @since("2.3.0")
     def read(cls):
         """Returns an MLReader instance for this class."""
-        return JavaMLReader(cls)
+        return CrossValidatorModelReader(cls)
 
     @classmethod
     def _from_java(cls, java_stage):
@@ -499,8 +948,17 @@ def _from_java(cls, java_stage):
         avgMetrics = _java2py(sc, java_stage.avgMetrics())
         estimator, epms, evaluator = super(CrossValidatorModel, cls)._from_java_impl(java_stage)
 
-        py_stage = cls(bestModel=bestModel, avgMetrics=avgMetrics)._set(estimator=estimator)
-        py_stage = py_stage._set(estimatorParamMaps=epms)._set(evaluator=evaluator)
+        py_stage = cls(bestModel=bestModel, avgMetrics=avgMetrics)
+        params = {
+            "evaluator": evaluator,
+            "estimator": estimator,
+            "estimatorParamMaps": epms,
+            "numFolds": java_stage.getNumFolds(),
+            "foldCol": java_stage.getFoldCol(),
+            "seed": java_stage.getSeed(),
+        }
+        for param_name, param_val in params.items():
+            py_stage = py_stage._set(**{param_name: param_val})
 
         if java_stage.hasSubModels():
             py_stage.subModels = [[JavaParams._from_java(sub_model)
@@ -514,7 +972,10 @@ def _to_java(self):
         """
         Transfer this instance to a Java CrossValidatorModel. Used for ML persistence.
 
-        :return: Java object equivalent to this instance.
+        Returns
+        -------
+        py4j.java_gateway.JavaObject
+            Java object equivalent to this instance.
         """
 
         sc = SparkContext._active_spark_context
@@ -524,9 +985,18 @@ def _to_java(self):
                                              _py2java(sc, self.avgMetrics))
         estimator, epms, evaluator = super(CrossValidatorModel, self)._to_java_impl()
 
-        _java_obj.set("evaluator", evaluator)
-        _java_obj.set("estimator", estimator)
-        _java_obj.set("estimatorParamMaps", epms)
+        params = {
+            "evaluator": evaluator,
+            "estimator": estimator,
+            "estimatorParamMaps": epms,
+            "numFolds": self.getNumFolds(),
+            "foldCol": self.getFoldCol(),
+            "seed": self.getSeed(),
+        }
+        for param_name, param_val in params.items():
+            java_param = _java_obj.getParam(param_name)
+            pair = java_param.w(param_val)
+            _java_obj.set(pair)
 
         if self.subModels is not None:
             java_sub_models = [[sub_model._to_java() for sub_model in fold_sub_models]
@@ -535,6 +1005,106 @@ def _to_java(self):
         return _java_obj
 
 
+@inherit_doc
+class TrainValidationSplitReader(MLReader):
+
+    def __init__(self, cls):
+        super(TrainValidationSplitReader, self).__init__()
+        self.cls = cls
+
+    def load(self, path):
+        metadata = DefaultParamsReader.loadMetadata(path, self.sc)
+        if not DefaultParamsReader.isPythonParamsInstance(metadata):
+            return JavaMLReader(self.cls).load(path)
+        else:
+            metadata, estimator, evaluator, estimatorParamMaps = \
+                _ValidatorSharedReadWrite.load(path, self.sc, metadata)
+            tvs = TrainValidationSplit(estimator=estimator,
+                                       estimatorParamMaps=estimatorParamMaps,
+                                       evaluator=evaluator)
+            tvs = tvs._resetUid(metadata['uid'])
+            DefaultParamsReader.getAndSetParams(tvs, metadata, skipParams=['estimatorParamMaps'])
+            return tvs
+
+
+@inherit_doc
+class TrainValidationSplitWriter(MLWriter):
+
+    def __init__(self, instance):
+        super(TrainValidationSplitWriter, self).__init__()
+        self.instance = instance
+
+    def saveImpl(self, path):
+        _ValidatorSharedReadWrite.validateParams(self.instance)
+        _ValidatorSharedReadWrite.saveImpl(path, self.instance, self.sc)
+
+
+@inherit_doc
+class TrainValidationSplitModelReader(MLReader):
+
+    def __init__(self, cls):
+        super(TrainValidationSplitModelReader, self).__init__()
+        self.cls = cls
+
+    def load(self, path):
+        metadata = DefaultParamsReader.loadMetadata(path, self.sc)
+        if not DefaultParamsReader.isPythonParamsInstance(metadata):
+            return JavaMLReader(self.cls).load(path)
+        else:
+            metadata, estimator, evaluator, estimatorParamMaps = \
+                _ValidatorSharedReadWrite.load(path, self.sc, metadata)
+            bestModelPath = os.path.join(path, 'bestModel')
+            bestModel = DefaultParamsReader.loadParamsInstance(bestModelPath, self.sc)
+            validationMetrics = metadata['validationMetrics']
+            persistSubModels = ('persistSubModels' in metadata) and metadata['persistSubModels']
+
+            if persistSubModels:
+                subModels = [None] * len(estimatorParamMaps)
+                for paramIndex in range(len(estimatorParamMaps)):
+                    modelPath = os.path.join(path, 'subModels', f'{paramIndex}')
+                    subModels[paramIndex] = \
+                        DefaultParamsReader.loadParamsInstance(modelPath, self.sc)
+            else:
+                subModels = None
+
+            tvsModel = TrainValidationSplitModel(
+                bestModel, validationMetrics=validationMetrics, subModels=subModels)
+            tvsModel = tvsModel._resetUid(metadata['uid'])
+            tvsModel.set(tvsModel.estimator, estimator)
+            tvsModel.set(tvsModel.estimatorParamMaps, estimatorParamMaps)
+            tvsModel.set(tvsModel.evaluator, evaluator)
+            DefaultParamsReader.getAndSetParams(
+                tvsModel, metadata, skipParams=['estimatorParamMaps'])
+            return tvsModel
+
+
+@inherit_doc
+class TrainValidationSplitModelWriter(MLWriter):
+
+    def __init__(self, instance):
+        super(TrainValidationSplitModelWriter, self).__init__()
+        self.instance = instance
+
+    def saveImpl(self, path):
+        _ValidatorSharedReadWrite.validateParams(self.instance)
+        instance = self.instance
+        persistSubModels = _ValidatorSharedReadWrite \
+            .getValidatorModelWriterPersistSubModelsParam(self)
+
+        extraMetadata = {'validationMetrics': instance.validationMetrics,
+                         'persistSubModels': persistSubModels}
+        _ValidatorSharedReadWrite.saveImpl(path, instance, self.sc, extraMetadata=extraMetadata)
+        bestModelPath = os.path.join(path, 'bestModel')
+        instance.bestModel.save(bestModelPath)
+        if persistSubModels:
+            if instance.subModels is None:
+                raise ValueError(_save_with_persist_submodels_no_submodels_found_err)
+            subModelsPath = os.path.join(path, 'subModels')
+            for paramIndex in range(len(instance.getEstimatorParamMaps())):
+                modelPath = os.path.join(subModelsPath, f'{paramIndex}')
+                instance.subModels[paramIndex].save(modelPath)
+
+
 class _TrainValidationSplitParams(_ValidatorParams):
     """
     Params for :py:class:`TrainValidationSplit` and :py:class:`TrainValidationSplitModel`.
@@ -545,6 +1115,10 @@ class _TrainValidationSplitParams(_ValidatorParams):
     trainRatio = Param(Params._dummy(), "trainRatio", "Param for ratio between train and\
      validation data. Must be between 0 and 1.", typeConverter=TypeConverters.toFloat)
 
+    def __init__(self, *args):
+        super(_TrainValidationSplitParams, self).__init__(*args)
+        self._setDefault(trainRatio=0.75)
+
     @since("2.0.0")
     def getTrainRatio(self):
         """
@@ -560,6 +1134,10 @@ class TrainValidationSplit(Estimator, _TrainValidationSplitParams, HasParallelis
     validation sets, and uses evaluation metric on the validation set to select the best model.
     Similar to :class:`CrossValidator`, but only splits the set once.
 
+    .. versionadded:: 2.0.0
+
+    Examples
+    --------
     >>> from pyspark.ml.classification import LogisticRegression
     >>> from pyspark.ml.evaluation import BinaryClassificationEvaluator
     >>> from pyspark.ml.linalg import Vectors
@@ -590,29 +1168,29 @@ class TrainValidationSplit(Estimator, _TrainValidationSplitParams, HasParallelis
     [0.5, ...
     >>> evaluator.evaluate(tvsModel.transform(dataset))
     0.833...
-
-    .. versionadded:: 2.0.0
+    >>> evaluator.evaluate(tvsModelRead.transform(dataset))
+    0.833...
     """
 
     @keyword_only
-    def __init__(self, estimator=None, estimatorParamMaps=None, evaluator=None, trainRatio=0.75,
-                 parallelism=1, collectSubModels=False, seed=None):
+    def __init__(self, *, estimator=None, estimatorParamMaps=None, evaluator=None,
+                 trainRatio=0.75, parallelism=1, collectSubModels=False, seed=None):
         """
-        __init__(self, estimator=None, estimatorParamMaps=None, evaluator=None, trainRatio=0.75,\
-                 parallelism=1, collectSubModels=False, seed=None)
+        __init__(self, \\*, estimator=None, estimatorParamMaps=None, evaluator=None, \
+                 trainRatio=0.75, parallelism=1, collectSubModels=False, seed=None)
         """
         super(TrainValidationSplit, self).__init__()
-        self._setDefault(trainRatio=0.75, parallelism=1)
+        self._setDefault(parallelism=1)
         kwargs = self._input_kwargs
         self._set(**kwargs)
 
     @since("2.0.0")
     @keyword_only
-    def setParams(self, estimator=None, estimatorParamMaps=None, evaluator=None, trainRatio=0.75,
-                  parallelism=1, collectSubModels=False, seed=None):
+    def setParams(self, *, estimator=None, estimatorParamMaps=None, evaluator=None,
+                  trainRatio=0.75, parallelism=1, collectSubModels=False, seed=None):
         """
-        setParams(self, estimator=None, estimatorParamMaps=None, evaluator=None, trainRatio=0.75,\
-                  parallelism=1, collectSubModels=False, seed=None):
+        setParams(self, \\*, estimator=None, estimatorParamMaps=None, evaluator=None, \
+                  trainRatio=0.75, parallelism=1, collectSubModels=False, seed=None):
         Sets params for the train validation split.
         """
         kwargs = self._input_kwargs
@@ -700,15 +1278,23 @@ def _fit(self, dataset):
         bestModel = est.fit(dataset, epm[bestIndex])
         return self._copyValues(TrainValidationSplitModel(bestModel, metrics, subModels))
 
-    @since("2.0.0")
     def copy(self, extra=None):
         """
         Creates a copy of this instance with a randomly generated uid
         and some extra params. This copies creates a deep copy of
         the embedded paramMap, and copies the embedded and extra parameters over.
 
-        :param extra: Extra parameters to copy to the new instance
-        :return: Copy of this instance
+        .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        extra : dict, optional
+            Extra parameters to copy to the new instance
+
+        Returns
+        -------
+        :py:class:`TrainValidationSplit`
+            Copy of this instance
         """
         if extra is None:
             extra = dict()
@@ -723,13 +1309,15 @@ def copy(self, extra=None):
     @since("2.3.0")
     def write(self):
         """Returns an MLWriter instance for this ML instance."""
-        return JavaMLWriter(self)
+        if _ValidatorSharedReadWrite.is_java_convertible(self):
+            return JavaMLWriter(self)
+        return TrainValidationSplitWriter(self)
 
     @classmethod
     @since("2.3.0")
     def read(cls):
         """Returns an MLReader instance for this class."""
-        return JavaMLReader(cls)
+        return TrainValidationSplitReader(cls)
 
     @classmethod
     def _from_java(cls, java_stage):
@@ -753,7 +1341,11 @@ def _from_java(cls, java_stage):
     def _to_java(self):
         """
         Transfer this instance to a Java TrainValidationSplit. Used for ML persistence.
-        :return: Java object equivalent to this instance.
+
+        Returns
+        -------
+        py4j.java_gateway.JavaObject
+            Java object equivalent to this instance.
         """
 
         estimator, epms, evaluator = super(TrainValidationSplit, self)._to_java_impl()
@@ -777,19 +1369,18 @@ class TrainValidationSplitModel(Model, _TrainValidationSplitParams, MLReadable,
     .. versionadded:: 2.0.0
     """
 
-    def __init__(self, bestModel, validationMetrics=[], subModels=None):
+    def __init__(self, bestModel, validationMetrics=None, subModels=None):
         super(TrainValidationSplitModel, self).__init__()
         #: best model from train validation split
         self.bestModel = bestModel
         #: evaluated validation metrics
-        self.validationMetrics = validationMetrics
+        self.validationMetrics = validationMetrics or []
         #: sub models from train validation split
         self.subModels = subModels
 
     def _transform(self, dataset):
         return self.bestModel.transform(dataset)
 
-    @since("2.0.0")
     def copy(self, extra=None):
         """
         Creates a copy of this instance with a randomly generated uid
@@ -799,26 +1390,40 @@ def copy(self, extra=None):
         And, this creates a shallow copy of the validationMetrics.
         It does not copy the extra Params into the subModels.
 
-        :param extra: Extra parameters to copy to the new instance
-        :return: Copy of this instance
+        .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        extra : dict, optional
+            Extra parameters to copy to the new instance
+
+        Returns
+        -------
+        :py:class:`TrainValidationSplitModel`
+            Copy of this instance
         """
         if extra is None:
             extra = dict()
         bestModel = self.bestModel.copy(extra)
         validationMetrics = list(self.validationMetrics)
-        subModels = self.subModels
-        return TrainValidationSplitModel(bestModel, validationMetrics, subModels)
+        subModels = [model.copy() for model in self.subModels]
+        return self._copyValues(
+            TrainValidationSplitModel(bestModel, validationMetrics, subModels),
+            extra=extra
+        )
 
     @since("2.3.0")
     def write(self):
         """Returns an MLWriter instance for this ML instance."""
-        return JavaMLWriter(self)
+        if _ValidatorSharedReadWrite.is_java_convertible(self):
+            return JavaMLWriter(self)
+        return TrainValidationSplitModelWriter(self)
 
     @classmethod
     @since("2.3.0")
     def read(cls):
         """Returns an MLReader instance for this class."""
-        return JavaMLReader(cls)
+        return TrainValidationSplitModelReader(cls)
 
     @classmethod
     def _from_java(cls, java_stage):
@@ -835,8 +1440,16 @@ def _from_java(cls, java_stage):
                                            cls)._from_java_impl(java_stage)
         # Create a new instance of this stage.
         py_stage = cls(bestModel=bestModel,
-                       validationMetrics=validationMetrics)._set(estimator=estimator)
-        py_stage = py_stage._set(estimatorParamMaps=epms)._set(evaluator=evaluator)
+                       validationMetrics=validationMetrics)
+        params = {
+            "evaluator": evaluator,
+            "estimator": estimator,
+            "estimatorParamMaps": epms,
+            "trainRatio": java_stage.getTrainRatio(),
+            "seed": java_stage.getSeed(),
+        }
+        for param_name, param_val in params.items():
+            py_stage = py_stage._set(**{param_name: param_val})
 
         if java_stage.hasSubModels():
             py_stage.subModels = [JavaParams._from_java(sub_model)
@@ -848,7 +1461,11 @@ def _from_java(cls, java_stage):
     def _to_java(self):
         """
         Transfer this instance to a Java TrainValidationSplitModel. Used for ML persistence.
-        :return: Java object equivalent to this instance.
+
+        Returns
+        -------
+        py4j.java_gateway.JavaObject
+            Java object equivalent to this instance.
         """
 
         sc = SparkContext._active_spark_context
@@ -859,9 +1476,17 @@ def _to_java(self):
             _py2java(sc, self.validationMetrics))
         estimator, epms, evaluator = super(TrainValidationSplitModel, self)._to_java_impl()
 
-        _java_obj.set("evaluator", evaluator)
-        _java_obj.set("estimator", estimator)
-        _java_obj.set("estimatorParamMaps", epms)
+        params = {
+            "evaluator": evaluator,
+            "estimator": estimator,
+            "estimatorParamMaps": epms,
+            "trainRatio": self.getTrainRatio(),
+            "seed": self.getSeed(),
+        }
+        for param_name, param_val in params.items():
+            java_param = _java_obj.getParam(param_name)
+            pair = java_param.w(param_val)
+            _java_obj.set(pair)
 
         if self.subModels is not None:
             java_sub_models = [sub_model._to_java() for sub_model in self.subModels]
diff --git a/python/pyspark/ml/tuning.pyi b/python/pyspark/ml/tuning.pyi
new file mode 100644
index 0000000000000..028cebdccac92
--- /dev/null
+++ b/python/pyspark/ml/tuning.pyi
@@ -0,0 +1,230 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import overload
+from typing import Any, List, Optional, Tuple, Type
+from pyspark.ml._typing import ParamMap
+
+from pyspark.ml import Estimator, Model
+from pyspark.ml.evaluation import Evaluator
+from pyspark.ml.param import Param
+from pyspark.ml.param.shared import HasCollectSubModels, HasParallelism, HasSeed
+from pyspark.ml.util import MLReader, MLReadable, MLWriter, MLWritable
+
+class ParamGridBuilder:
+    def __init__(self) -> None: ...
+    def addGrid(self, param: Param, values: List[Any]) -> ParamGridBuilder: ...
+    @overload
+    def baseOn(self, __args: ParamMap) -> ParamGridBuilder: ...
+    @overload
+    def baseOn(self, *args: Tuple[Param, Any]) -> ParamGridBuilder: ...
+    def build(self) -> List[ParamMap]: ...
+
+class ParamRandomBuilder(ParamGridBuilder):
+    def __init__(self) -> None: ...
+    def addRandom(self, param: Param, x: Any, y: Any, n: int) -> ParamRandomBuilder: ...
+    def addLog10Random(self, param: Param, x: Any, y: Any, n: int) -> ParamRandomBuilder: ...
+
+class _ValidatorParams(HasSeed):
+    estimator: Param[Estimator]
+    estimatorParamMaps: Param[List[ParamMap]]
+    evaluator: Param[Evaluator]
+    def getEstimator(self) -> Estimator: ...
+    def getEstimatorParamMaps(self) -> List[ParamMap]: ...
+    def getEvaluator(self) -> Evaluator: ...
+
+class _CrossValidatorParams(_ValidatorParams):
+    numFolds: Param[int]
+    foldCol: Param[str]
+    def __init__(self, *args: Any): ...
+    def getNumFolds(self) -> int: ...
+    def getFoldCol(self) -> str: ...
+
+class CrossValidator(
+    Estimator[CrossValidatorModel],
+    _CrossValidatorParams,
+    HasParallelism,
+    HasCollectSubModels,
+    MLReadable[CrossValidator],
+    MLWritable,
+):
+    def __init__(
+        self,
+        *,
+        estimator: Optional[Estimator] = ...,
+        estimatorParamMaps: Optional[List[ParamMap]] = ...,
+        evaluator: Optional[Evaluator] = ...,
+        numFolds: int = ...,
+        seed: Optional[int] = ...,
+        parallelism: int = ...,
+        collectSubModels: bool = ...,
+        foldCol: str = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        estimator: Optional[Estimator] = ...,
+        estimatorParamMaps: Optional[List[ParamMap]] = ...,
+        evaluator: Optional[Evaluator] = ...,
+        numFolds: int = ...,
+        seed: Optional[int] = ...,
+        parallelism: int = ...,
+        collectSubModels: bool = ...,
+        foldCol: str = ...
+    ) -> CrossValidator: ...
+    def setEstimator(self, value: Estimator) -> CrossValidator: ...
+    def setEstimatorParamMaps(self, value: List[ParamMap]) -> CrossValidator: ...
+    def setEvaluator(self, value: Evaluator) -> CrossValidator: ...
+    def setNumFolds(self, value: int) -> CrossValidator: ...
+    def setFoldCol(self, value: str) -> CrossValidator: ...
+    def setSeed(self, value: int) -> CrossValidator: ...
+    def setParallelism(self, value: int) -> CrossValidator: ...
+    def setCollectSubModels(self, value: bool) -> CrossValidator: ...
+    def copy(self, extra: Optional[ParamMap] = ...) -> CrossValidator: ...
+    def write(self) -> MLWriter: ...
+    @classmethod
+    def read(cls: Type[CrossValidator]) -> MLReader: ...
+
+class CrossValidatorModel(
+    Model, _CrossValidatorParams, MLReadable[CrossValidatorModel], MLWritable
+):
+    bestModel: Model
+    avgMetrics: List[float]
+    subModels: List[List[Model]]
+    def __init__(
+        self,
+        bestModel: Model,
+        avgMetrics: Optional[List[float]] = ...,
+        subModels: Optional[List[List[Model]]] = ...,
+    ) -> None: ...
+    def copy(self, extra: Optional[ParamMap] = ...) -> CrossValidatorModel: ...
+    def write(self) -> MLWriter: ...
+    @classmethod
+    def read(cls: Type[CrossValidatorModel]) -> MLReader: ...
+
+class _TrainValidationSplitParams(_ValidatorParams):
+    trainRatio: Param[float]
+    def __init__(self, *args: Any): ...
+    def getTrainRatio(self) -> float: ...
+
+class TrainValidationSplit(
+    Estimator[TrainValidationSplitModel],
+    _TrainValidationSplitParams,
+    HasParallelism,
+    HasCollectSubModels,
+    MLReadable[TrainValidationSplit],
+    MLWritable,
+):
+    def __init__(
+        self,
+        *,
+        estimator: Optional[Estimator] = ...,
+        estimatorParamMaps: Optional[List[ParamMap]] = ...,
+        evaluator: Optional[Evaluator] = ...,
+        trainRatio: float = ...,
+        parallelism: int = ...,
+        collectSubModels: bool = ...,
+        seed: Optional[int] = ...
+    ) -> None: ...
+    def setParams(
+        self,
+        *,
+        estimator: Optional[Estimator] = ...,
+        estimatorParamMaps: Optional[List[ParamMap]] = ...,
+        evaluator: Optional[Evaluator] = ...,
+        trainRatio: float = ...,
+        parallelism: int = ...,
+        collectSubModels: bool = ...,
+        seed: Optional[int] = ...
+    ) -> TrainValidationSplit: ...
+    def setEstimator(self, value: Estimator) -> TrainValidationSplit: ...
+    def setEstimatorParamMaps(self, value: List[ParamMap]) -> TrainValidationSplit: ...
+    def setEvaluator(self, value: Evaluator) -> TrainValidationSplit: ...
+    def setTrainRatio(self, value: float) -> TrainValidationSplit: ...
+    def setSeed(self, value: int) -> TrainValidationSplit: ...
+    def setParallelism(self, value: int) -> TrainValidationSplit: ...
+    def setCollectSubModels(self, value: bool) -> TrainValidationSplit: ...
+    def copy(self, extra: Optional[ParamMap] = ...) -> TrainValidationSplit: ...
+    def write(self) -> MLWriter: ...
+    @classmethod
+    def read(cls: Type[TrainValidationSplit]) -> MLReader: ...
+
+class TrainValidationSplitModel(
+    Model,
+    _TrainValidationSplitParams,
+    MLReadable[TrainValidationSplitModel],
+    MLWritable,
+):
+    bestModel: Model
+    validationMetrics: List[float]
+    subModels: List[Model]
+    def __init__(
+        self,
+        bestModel: Model,
+        validationMetrics: Optional[List[float]] = ...,
+        subModels: Optional[List[Model]] = ...,
+    ) -> None: ...
+    def setEstimator(self, value: Estimator) -> TrainValidationSplitModel: ...
+    def setEstimatorParamMaps(
+        self, value: List[ParamMap]
+    ) -> TrainValidationSplitModel: ...
+    def setEvaluator(self, value: Evaluator) -> TrainValidationSplitModel: ...
+    def copy(self, extra: Optional[ParamMap] = ...) -> TrainValidationSplitModel: ...
+    def write(self) -> MLWriter: ...
+    @classmethod
+    def read(cls: Type[TrainValidationSplitModel]) -> MLReader: ...
+
+class CrossValidatorWriter(MLWriter):
+    instance: CrossValidator
+    def __init__(self, instance: CrossValidator) -> None: ...
+    def saveImpl(self, path: str) -> None: ...
+
+class CrossValidatorReader(MLReader[CrossValidator]):
+    cls: Type[CrossValidator]
+    def __init__(self, cls: Type[CrossValidator]) -> None: ...
+    def load(self, path: str) -> CrossValidator: ...
+
+class CrossValidatorModelWriter(MLWriter):
+    instance: CrossValidatorModel
+    def __init__(self, instance: CrossValidatorModel) -> None: ...
+    def saveImpl(self, path: str) -> None: ...
+
+class CrossValidatorModelReader(MLReader[CrossValidatorModel]):
+    cls: Type[CrossValidatorModel]
+    def __init__(self, cls: Type[CrossValidatorModel]) -> None: ...
+    def load(self, path: str) -> CrossValidatorModel: ...
+
+class TrainValidationSplitWriter(MLWriter):
+    instance: TrainValidationSplit
+    def __init__(self, instance: TrainValidationSplit) -> None: ...
+    def saveImpl(self, path: str) -> None: ...
+
+class TrainValidationSplitReader(MLReader[TrainValidationSplit]):
+    cls: Type[TrainValidationSplit]
+    def __init__(self, cls: Type[TrainValidationSplit]) -> None: ...
+    def load(self, path: str) -> TrainValidationSplit: ...
+
+class TrainValidationSplitModelWriter(MLWriter):
+    instance: TrainValidationSplitModel
+    def __init__(self, instance: TrainValidationSplitModel) -> None: ...
+    def saveImpl(self, path: str) -> None: ...
+
+class TrainValidationSplitModelReader(MLReader[TrainValidationSplitModel]):
+    cls: Type[TrainValidationSplitModel]
+    def __init__(self, cls: Type[TrainValidationSplitModel]) -> None: ...
+    def load(self, path: str) -> TrainValidationSplitModel: ...
diff --git a/python/pyspark/ml/util.py b/python/pyspark/ml/util.py
index aac2b38d3f57d..21d2725791407 100644
--- a/python/pyspark/ml/util.py
+++ b/python/pyspark/ml/util.py
@@ -16,16 +16,9 @@
 #
 
 import json
-import sys
 import os
 import time
 import uuid
-import warnings
-
-if sys.version > '3':
-    basestring = str
-    unicode = str
-    long = int
 
 from pyspark import SparkContext, since
 from pyspark.ml.common import inherit_doc
@@ -60,10 +53,10 @@ def __repr__(self):
     @classmethod
     def _randomUID(cls):
         """
-        Generate a unique unicode id for the object. The default implementation
+        Generate a unique string id for the object. The default implementation
         concatenates the class name, "_", and 12 random hex chars.
         """
-        return unicode(cls.__name__ + "_" + uuid.uuid4().hex[-12:])
+        return str(cls.__name__ + "_" + uuid.uuid4().hex[-12:])
 
 
 @inherit_doc
@@ -113,6 +106,7 @@ class MLWriter(BaseReadWrite):
     def __init__(self):
         super(MLWriter, self).__init__()
         self.shouldOverwrite = False
+        self.optionMap = {}
 
     def _handleOverwrite(self, path):
         from pyspark.ml.wrapper import JavaWrapper
@@ -139,6 +133,14 @@ def overwrite(self):
         self.shouldOverwrite = True
         return self
 
+    def option(self, key, value):
+        """
+        Adds an option to the underlying MLWriter. See the documentation for the specific model's
+        writer for possible options. The option name (key) is case-insensitive.
+        """
+        self.optionMap[key.lower()] = str(value)
+        return self
+
 
 @inherit_doc
 class GeneralMLWriter(MLWriter):
@@ -150,7 +152,7 @@ class GeneralMLWriter(MLWriter):
 
     def format(self, source):
         """
-        Specifies the format of ML export (e.g. "pmml", "internal", or the fully qualified class
+        Specifies the format of ML export ("pmml", "internal", or the fully qualified class
         name for export).
         """
         self.source = source
@@ -170,8 +172,8 @@ def __init__(self, instance):
 
     def save(self, path):
         """Save the ML instance to the input path."""
-        if not isinstance(path, basestring):
-            raise TypeError("path should be a basestring, got type %s" % type(path))
+        if not isinstance(path, str):
+            raise TypeError("path should be a string, got type %s" % type(path))
         self._jwrite.save(path)
 
     def overwrite(self):
@@ -200,7 +202,7 @@ def __init__(self, instance):
 
     def format(self, source):
         """
-        Specifies the format of ML export (e.g. "pmml", "internal", or the fully qualified class
+        Specifies the format of ML export ("pmml", "internal", or the fully qualified class
         name for export).
         """
         self._jwrite.format(source)
@@ -275,8 +277,8 @@ def __init__(self, clazz):
 
     def load(self, path):
         """Load the ML instance from the input path."""
-        if not isinstance(path, basestring):
-            raise TypeError("path should be a basestring, got type %s" % type(path))
+        if not isinstance(path, str):
+            raise TypeError("path should be a string, got type %s" % type(path))
         java_obj = self._jread.load(path)
         if not hasattr(self._clazz, "_from_java"):
             raise NotImplementedError("This Java ML type cannot be loaded into Python currently: %r"
@@ -382,6 +384,13 @@ def __init__(self, instance):
     def saveImpl(self, path):
         DefaultParamsWriter.saveMetadata(self.instance, path, self.sc)
 
+    @staticmethod
+    def extractJsonParams(instance, skipParams):
+        paramMap = instance.extractParamMap()
+        jsonParams = {param.name: value for param, value in paramMap.items()
+                      if param.name not in skipParams}
+        return jsonParams
+
     @staticmethod
     def saveMetadata(instance, path, sc, extraMetadata=None, paramMap=None):
         """
@@ -395,8 +404,12 @@ def saveMetadata(instance, path, sc, extraMetadata=None, paramMap=None):
         - defaultParamMap (since 2.4.0)
         - (optionally, extra metadata)
 
-        :param extraMetadata:  Extra metadata to be saved at same level as uid, paramMap, etc.
-        :param paramMap:  If given, this is saved in the "paramMap" field.
+        Parameters
+        ----------
+        extraMetadata : dict, optional
+            Extra metadata to be saved at same level as uid, paramMap, etc.
+        paramMap : dict, optional
+            If given, this is saved in the "paramMap" field.
         """
         metadataPath = os.path.join(path, "metadata")
         metadataJson = DefaultParamsWriter._get_metadata_to_save(instance,
@@ -411,7 +424,9 @@ def _get_metadata_to_save(instance, sc, extraMetadata=None, paramMap=None):
         Helper for :py:meth:`DefaultParamsWriter.saveMetadata` which extracts the JSON to save.
         This is useful for ensemble models which need to save metadata for many sub-models.
 
-        .. note:: :py:meth:`DefaultParamsWriter.saveMetadata` for details on what this includes.
+        Notes
+        -----
+        See :py:meth:`DefaultParamsWriter.saveMetadata` for details on what this includes.
         """
         uid = instance.uid
         cls = instance.__module__ + '.' + instance.__class__.__name__
@@ -430,7 +445,7 @@ def _get_metadata_to_save(instance, sc, extraMetadata=None, paramMap=None):
         for p in instance._defaultParamMap:
             jsonDefaultParams[p.name] = instance._defaultParamMap[p]
 
-        basicMetadata = {"class": cls, "timestamp": long(round(time.time() * 1000)),
+        basicMetadata = {"class": cls, "timestamp": int(round(time.time() * 1000)),
                          "sparkVersion": sc.version, "uid": uid, "paramMap": jsonParams,
                          "defaultParamMap": jsonDefaultParams}
         if extraMetadata is not None:
@@ -498,7 +513,12 @@ def loadMetadata(path, sc, expectedClassName=""):
         """
         Load metadata saved using :py:meth:`DefaultParamsWriter.saveMetadata`
 
-        :param expectedClassName:  If non empty, this is checked against the loaded metadata.
+        Parameters
+        ----------
+        path : str
+        sc : :py:class:`pyspark.SparkContext`
+        expectedClassName : str, optional
+            If non empty, this is checked against the loaded metadata.
         """
         metadataPath = os.path.join(path, "metadata")
         metadataStr = sc.textFile(metadataPath, 1).first()
@@ -511,8 +531,12 @@ def _parseMetaData(metadataStr, expectedClassName=""):
         Parse metadata JSON string produced by :py:meth`DefaultParamsWriter._get_metadata_to_save`.
         This is a helper function for :py:meth:`DefaultParamsReader.loadMetadata`.
 
-        :param metadataStr:  JSON string of metadata
-        :param expectedClassName:  If non empty, this is checked against the loaded metadata.
+        Parameters
+        ----------
+        metadataStr : str
+            JSON string of metadata
+        expectedClassName : str, optional
+            If non empty, this is checked against the loaded metadata.
         """
         metadata = json.loads(metadataStr)
         className = metadata['class']
@@ -522,15 +546,16 @@ def _parseMetaData(metadataStr, expectedClassName=""):
         return metadata
 
     @staticmethod
-    def getAndSetParams(instance, metadata):
+    def getAndSetParams(instance, metadata, skipParams=None):
         """
         Extract Params from metadata, and set them in the instance.
         """
         # Set user-supplied param values
         for paramName in metadata['paramMap']:
             param = instance.getParam(paramName)
-            paramValue = metadata['paramMap'][paramName]
-            instance.set(param, paramValue)
+            if skipParams is None or paramName not in skipParams:
+                paramValue = metadata['paramMap'][paramName]
+                instance.set(param, paramValue)
 
         # Set default param values
         majorAndMinorVersions = VersionUtils.majorMinorVersion(metadata['sparkVersion'])
@@ -546,6 +571,10 @@ def getAndSetParams(instance, metadata):
                 paramValue = metadata['defaultParamMap'][paramName]
                 instance._setDefault(**{paramName: paramValue})
 
+    @staticmethod
+    def isPythonParamsInstance(metadata):
+        return metadata['class'].startswith('pyspark.ml.')
+
     @staticmethod
     def loadParamsInstance(path, sc):
         """
@@ -553,7 +582,10 @@ def loadParamsInstance(path, sc):
         This assumes the instance inherits from :py:class:`MLReadable`.
         """
         metadata = DefaultParamsReader.loadMetadata(path, sc)
-        pythonClassName = metadata['class'].replace("org.apache.spark", "pyspark")
+        if DefaultParamsReader.isPythonParamsInstance(metadata):
+            pythonClassName = metadata['class']
+        else:
+            pythonClassName = metadata['class'].replace("org.apache.spark", "pyspark")
         py_type = DefaultParamsReader.__get_class(pythonClassName)
         instance = py_type.load(path)
         return instance
@@ -584,3 +616,51 @@ def summary(self):
         no summary exists.
         """
         return (self._call_java("summary"))
+
+
+class MetaAlgorithmReadWrite:
+
+    @staticmethod
+    def isMetaEstimator(pyInstance):
+        from pyspark.ml import Estimator, Pipeline
+        from pyspark.ml.tuning import _ValidatorParams
+        from pyspark.ml.classification import OneVsRest
+        return isinstance(pyInstance, Pipeline) or isinstance(pyInstance, OneVsRest) or \
+            (isinstance(pyInstance, Estimator) and isinstance(pyInstance, _ValidatorParams))
+
+    @staticmethod
+    def getAllNestedStages(pyInstance):
+        from pyspark.ml import Pipeline, PipelineModel
+        from pyspark.ml.tuning import _ValidatorParams
+        from pyspark.ml.classification import OneVsRest, OneVsRestModel
+
+        # TODO: We need to handle `RFormulaModel.pipelineModel` here after Pyspark RFormulaModel
+        #  support pipelineModel property.
+        if isinstance(pyInstance, Pipeline):
+            pySubStages = pyInstance.getStages()
+        elif isinstance(pyInstance, PipelineModel):
+            pySubStages = pyInstance.stages
+        elif isinstance(pyInstance, _ValidatorParams):
+            raise ValueError('PySpark does not support nested validator.')
+        elif isinstance(pyInstance, OneVsRest):
+            pySubStages = [pyInstance.getClassifier()]
+        elif isinstance(pyInstance, OneVsRestModel):
+            pySubStages = [pyInstance.getClassifier()] + pyInstance.models
+        else:
+            pySubStages = []
+
+        nestedStages = []
+        for pySubStage in pySubStages:
+            nestedStages.extend(MetaAlgorithmReadWrite.getAllNestedStages(pySubStage))
+
+        return [pyInstance] + nestedStages
+
+    @staticmethod
+    def getUidMap(instance):
+        nestedStages = MetaAlgorithmReadWrite.getAllNestedStages(instance)
+        uidMap = {stage.uid: stage for stage in nestedStages}
+        if len(nestedStages) != len(uidMap):
+            raise RuntimeError(f'{instance.__class__.__module__}.{instance.__class__.__name__}'
+                               f'.load found a compound estimator with stages with duplicate '
+                               f'UIDs. List of UIDs: {list(uidMap.keys())}.')
+        return uidMap
diff --git a/python/pyspark/ml/util.pyi b/python/pyspark/ml/util.pyi
new file mode 100644
index 0000000000000..db28c095a5568
--- /dev/null
+++ b/python/pyspark/ml/util.pyi
@@ -0,0 +1,136 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Any, Dict, Generic, Optional, Type, TypeVar, Union
+
+from pyspark import SparkContext as SparkContext, since as since  # noqa: F401
+from pyspark.ml.common import inherit_doc as inherit_doc  # noqa: F401
+from pyspark.sql import SparkSession as SparkSession
+from pyspark.util import VersionUtils as VersionUtils  # noqa: F401
+
+S = TypeVar("S")
+R = TypeVar("R", bound=MLReadable)
+
+class Identifiable:
+    uid: str
+    def __init__(self) -> None: ...
+
+class BaseReadWrite:
+    def __init__(self) -> None: ...
+    def session(self, sparkSession: SparkSession) -> Union[MLWriter, MLReader]: ...
+    @property
+    def sparkSession(self) -> SparkSession: ...
+    @property
+    def sc(self) -> SparkContext: ...
+
+class MLWriter(BaseReadWrite):
+    shouldOverwrite: bool = ...
+    def __init__(self) -> None: ...
+    def save(self, path: str) -> None: ...
+    def saveImpl(self, path: str) -> None: ...
+    def overwrite(self) -> MLWriter: ...
+
+class GeneralMLWriter(MLWriter):
+    source: str
+    def format(self, source: str) -> MLWriter: ...
+
+class JavaMLWriter(MLWriter):
+    def __init__(self, instance: JavaMLWritable) -> None: ...
+    def save(self, path: str) -> None: ...
+    def overwrite(self) -> JavaMLWriter: ...
+    def option(self, key: str, value: Any) -> JavaMLWriter: ...
+    def session(self, sparkSession: SparkSession) -> JavaMLWriter: ...
+
+class GeneralJavaMLWriter(JavaMLWriter):
+    def __init__(self, instance: MLWritable) -> None: ...
+    def format(self, source: str) -> GeneralJavaMLWriter: ...
+
+class MLWritable:
+    def write(self) -> MLWriter: ...
+    def save(self, path: str) -> None: ...
+
+class JavaMLWritable(MLWritable):
+    def write(self) -> JavaMLWriter: ...
+
+class GeneralJavaMLWritable(JavaMLWritable):
+    def write(self) -> GeneralJavaMLWriter: ...
+
+class MLReader(BaseReadWrite, Generic[R]):
+    def load(self, path: str) -> R: ...
+
+class JavaMLReader(MLReader[R]):
+    def __init__(self, clazz: Type[JavaMLReadable]) -> None: ...
+    def load(self, path: str) -> R: ...
+    def session(self, sparkSession: SparkSession) -> JavaMLReader[R]: ...
+
+class MLReadable(Generic[R]):
+    @classmethod
+    def read(cls: Type[R]) -> MLReader[R]: ...
+    @classmethod
+    def load(cls: Type[R], path: str) -> R: ...
+
+class JavaMLReadable(MLReadable[R]):
+    @classmethod
+    def read(cls: Type[R]) -> JavaMLReader[R]: ...
+
+class DefaultParamsWritable(MLWritable):
+    def write(self) -> MLWriter: ...
+
+class DefaultParamsWriter(MLWriter):
+    instance: DefaultParamsWritable
+    def __init__(self, instance: DefaultParamsWritable) -> None: ...
+    def saveImpl(self, path: str) -> None: ...
+    @staticmethod
+    def saveMetadata(
+        instance: DefaultParamsWritable,
+        path: str,
+        sc: SparkContext,
+        extraMetadata: Optional[Dict[str, Any]] = ...,
+        paramMap: Optional[Dict[str, Any]] = ...,
+    ) -> None: ...
+
+class DefaultParamsReadable(MLReadable[R]):
+    @classmethod
+    def read(cls: Type[R]) -> MLReader[R]: ...
+
+class DefaultParamsReader(MLReader[R]):
+    cls: Type[R]
+    def __init__(self, cls: Type[MLReadable]) -> None: ...
+    def load(self, path: str) -> R: ...
+    @staticmethod
+    def loadMetadata(
+        path: str, sc: SparkContext, expectedClassName: str = ...
+    ) -> Dict[str, Any]: ...
+    @staticmethod
+    def getAndSetParams(instance: R, metadata: Dict[str, Any]) -> None: ...
+    @staticmethod
+    def loadParamsInstance(path: str, sc: SparkContext) -> R: ...
+
+class HasTrainingSummary(Generic[S]):
+    @property
+    def hasSummary(self) -> bool: ...
+    @property
+    def summary(self) -> S: ...
+
+class MetaAlgorithmReadWrite:
+    @staticmethod
+    def isMetaEstimator(pyInstance: Any) -> bool: ...
+    @staticmethod
+    def getAllNestedStages(pyInstance: Any) -> list: ...
+    @staticmethod
+    def getUidMap(instance: Any) -> dict: ...
diff --git a/python/pyspark/ml/wrapper.py b/python/pyspark/ml/wrapper.py
index e59c6c7b250a8..3c8dc03a82971 100644
--- a/python/pyspark/ml/wrapper.py
+++ b/python/pyspark/ml/wrapper.py
@@ -16,9 +16,6 @@
 #
 
 from abc import ABCMeta, abstractmethod
-import sys
-if sys.version >= '3':
-    xrange = range
 
 from pyspark import since
 from pyspark import SparkContext
@@ -26,7 +23,6 @@
 from pyspark.ml import Estimator, Predictor, PredictionModel, Transformer, Model
 from pyspark.ml.base import _PredictorParams
 from pyspark.ml.param import Params
-from pyspark.ml.param.shared import HasFeaturesCol, HasLabelCol, HasPredictionCol
 from pyspark.ml.util import _jvm
 from pyspark.ml.common import inherit_doc, _java2py, _py2java
 
@@ -79,19 +75,25 @@ def _new_java_array(pylist, java_class):
         enough for all elements. The empty slots in the inner Java arrays will
         be filled with null to make the non-jagged 2D array.
 
-        :param pylist:
-          Python list to convert to a Java Array.
-        :param java_class:
-          Java class to specify the type of Array. Should be in the
-          form of sc._gateway.jvm.* (sc is a valid Spark Context).
-        :return:
-          Java Array of converted pylist.
+        Parameters
+        ----------
+        pylist : list
+            Python list to convert to a Java Array.
+        java_class : :py:class:`py4j.java_gateway.JavaClass`
+            Java class to specify the type of Array. Should be in the
+            form of sc._gateway.jvm.* (sc is a valid Spark Context).
+
+            Example primitive Java classes:
+
+            - basestring -> sc._gateway.jvm.java.lang.String
+            - int -> sc._gateway.jvm.java.lang.Integer
+            - float -> sc._gateway.jvm.java.lang.Double
+            - bool -> sc._gateway.jvm.java.lang.Boolean
 
-        Example primitive Java classes:
-          - basestring -> sc._gateway.jvm.java.lang.String
-          - int -> sc._gateway.jvm.java.lang.Integer
-          - float -> sc._gateway.jvm.java.lang.Double
-          - bool -> sc._gateway.jvm.java.lang.Boolean
+        Returns
+        -------
+        :py:class:`py4j.java_collections.JavaArray`
+          Java Array of converted pylist.
         """
         sc = SparkContext._active_spark_context
         java_array = None
@@ -99,21 +101,21 @@ def _new_java_array(pylist, java_class):
             # If pylist is a 2D array, then a 2D java array will be created.
             # The 2D array is a square, non-jagged 2D array that is big enough for all elements.
             inner_array_length = 0
-            for i in xrange(len(pylist)):
+            for i in range(len(pylist)):
                 inner_array_length = max(inner_array_length, len(pylist[i]))
             java_array = sc._gateway.new_array(java_class, len(pylist), inner_array_length)
-            for i in xrange(len(pylist)):
-                for j in xrange(len(pylist[i])):
+            for i in range(len(pylist)):
+                for j in range(len(pylist[i])):
                     java_array[i][j] = pylist[i][j]
         else:
             java_array = sc._gateway.new_array(java_class, len(pylist))
-            for i in xrange(len(pylist)):
+            for i in range(len(pylist)):
                 java_array[i] = pylist[i]
         return java_array
 
 
 @inherit_doc
-class JavaParams(JavaWrapper, Params):
+class JavaParams(JavaWrapper, Params, metaclass=ABCMeta):
     """
     Utility class to help create wrapper classes from Java/Scala
     implementations of pipeline components.
@@ -121,8 +123,6 @@ class JavaParams(JavaWrapper, Params):
     #: The param values in the Java object should be
     #: synced with the Python wrapper in fit/transform/evaluate/copy.
 
-    __metaclass__ = ABCMeta
-
     def _make_java_param_pair(self, param, value):
         """
         Makes a Java param pair.
@@ -218,7 +218,10 @@ def _to_java(self):
 
         Meta-algorithms such as Pipeline should override this method.
 
-        :return: Java object equivalent to this instance.
+        Returns
+        -------
+        py4j.java_gateway.JavaObject
+            Java object equivalent to this instance.
         """
         self._transfer_params_to_java()
         return self._java_obj
@@ -270,8 +273,15 @@ def copy(self, extra=None):
         extra params. So both the Python wrapper and the Java pipeline
         component get copied.
 
-        :param extra: Extra parameters to copy to the new instance
-        :return: Copy of this instance
+        Parameters
+        ----------
+        extra : dict, optional
+            Extra parameters to copy to the new instance
+
+        Returns
+        -------
+        :py:class:`JavaParams`
+            Copy of this instance
         """
         if extra is None:
             extra = dict()
@@ -291,14 +301,12 @@ def clear(self, param):
 
 
 @inherit_doc
-class JavaEstimator(JavaParams, Estimator):
+class JavaEstimator(JavaParams, Estimator, metaclass=ABCMeta):
     """
     Base class for :py:class:`Estimator`s that wrap Java/Scala
     implementations.
     """
 
-    __metaclass__ = ABCMeta
-
     @abstractmethod
     def _create_model(self, java_model):
         """
@@ -310,10 +318,15 @@ def _fit_java(self, dataset):
         """
         Fits a Java model to the input dataset.
 
-        :param dataset: input dataset, which is an instance of
-                        :py:class:`pyspark.sql.DataFrame`
-        :param params: additional params (overwriting embedded values)
-        :return: fitted Java model
+        Examples
+        --------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            input dataset
+
+        Returns
+        -------
+        py4j.java_gateway.JavaObject
+            fitted Java model
         """
         self._transfer_params_to_java()
         return self._java_obj.fit(dataset._jdf)
@@ -325,30 +338,26 @@ def _fit(self, dataset):
 
 
 @inherit_doc
-class JavaTransformer(JavaParams, Transformer):
+class JavaTransformer(JavaParams, Transformer, metaclass=ABCMeta):
     """
     Base class for :py:class:`Transformer`s that wrap Java/Scala
     implementations. Subclasses should ensure they have the transformer Java object
     available as _java_obj.
     """
 
-    __metaclass__ = ABCMeta
-
     def _transform(self, dataset):
         self._transfer_params_to_java()
         return DataFrame(self._java_obj.transform(dataset._jdf), dataset.sql_ctx)
 
 
 @inherit_doc
-class JavaModel(JavaTransformer, Model):
+class JavaModel(JavaTransformer, Model, metaclass=ABCMeta):
     """
     Base class for :py:class:`Model`s that wrap Java/Scala
     implementations. Subclasses should inherit this class before
     param mix-ins, because this sets the UID from the Java model.
     """
 
-    __metaclass__ = ABCMeta
-
     def __init__(self, java_model=None):
         """
         Initialize this instance with a Java model object.
@@ -378,12 +387,11 @@ def __repr__(self):
 
 
 @inherit_doc
-class JavaPredictor(Predictor, JavaEstimator, _PredictorParams):
+class JavaPredictor(Predictor, JavaEstimator, _PredictorParams, metaclass=ABCMeta):
     """
     (Private) Java Estimator for prediction tasks (regression and classification).
     """
-
-    __metaclass__ = ABCMeta
+    pass
 
 
 @inherit_doc
diff --git a/python/pyspark/ml/wrapper.pyi b/python/pyspark/ml/wrapper.pyi
new file mode 100644
index 0000000000000..830224c177d1e
--- /dev/null
+++ b/python/pyspark/ml/wrapper.pyi
@@ -0,0 +1,48 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import abc
+from typing import Any, Optional
+from pyspark.ml._typing import P, T, JM, ParamMap
+
+from pyspark.ml import Estimator, Predictor, PredictionModel, Transformer, Model
+from pyspark.ml.base import _PredictorParams
+from pyspark.ml.param import Param, Params
+
+class JavaWrapper:
+    def __init__(self, java_obj: Optional[Any] = ...) -> None: ...
+    def __del__(self) -> None: ...
+
+class JavaParams(JavaWrapper, Params, metaclass=abc.ABCMeta):
+    def copy(self: P, extra: Optional[ParamMap] = ...) -> P: ...
+    def clear(self, param: Param) -> None: ...
+
+class JavaEstimator(JavaParams, Estimator[JM], metaclass=abc.ABCMeta): ...
+class JavaTransformer(JavaParams, Transformer, metaclass=abc.ABCMeta): ...
+
+class JavaModel(JavaTransformer, Model, metaclass=abc.ABCMeta):
+    def __init__(self, java_model: Optional[Any] = ...) -> None: ...
+
+class JavaPredictor(
+    Predictor[JM], JavaEstimator, _PredictorParams, metaclass=abc.ABCMeta
+): ...
+
+class JavaPredictionModel(PredictionModel[T], JavaModel, _PredictorParams):
+    @property
+    def numFeatures(self) -> int: ...
+    def predict(self, value: T) -> float: ...
diff --git a/python/pyspark/mllib/__init__.py b/python/pyspark/mllib/__init__.py
index ae26521ea96bf..6067693111547 100644
--- a/python/pyspark/mllib/__init__.py
+++ b/python/pyspark/mllib/__init__.py
@@ -21,8 +21,6 @@
 The `pyspark.mllib` package is in maintenance mode as of the Spark 2.0.0 release to encourage
 migration to the DataFrame-based APIs under the `pyspark.ml` package.
 """
-from __future__ import absolute_import
-
 # MLlib currently needs NumPy 1.4+, so complain if lower
 
 import numpy
diff --git a/python/pyspark/mllib/_typing.pyi b/python/pyspark/mllib/_typing.pyi
new file mode 100644
index 0000000000000..213a69996b0ad
--- /dev/null
+++ b/python/pyspark/mllib/_typing.pyi
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import List, Tuple, Union
+from pyspark.mllib.linalg import Vector
+from numpy import ndarray  # noqa: F401
+
+VectorLike = Union[Vector, List[float], Tuple[float, ...]]
diff --git a/python/pyspark/mllib/classification.py b/python/pyspark/mllib/classification.py
index c52da2ad633b2..5705401ae3360 100644
--- a/python/pyspark/mllib/classification.py
+++ b/python/pyspark/mllib/classification.py
@@ -23,7 +23,7 @@
 
 from pyspark import RDD, since
 from pyspark.mllib.common import callMLlibFunc, _py2java, _java2py
-from pyspark.mllib.linalg import SparseVector, _convert_to_vector
+from pyspark.mllib.linalg import _convert_to_vector
 from pyspark.mllib.regression import (
     LabeledPoint, LinearModel, _regression_train_wrapper,
     StreamingLinearAlgorithm)
@@ -88,20 +88,27 @@ class LogisticRegressionModel(LinearClassificationModel):
     Classification model trained using Multinomial/Binary Logistic
     Regression.
 
-    :param weights:
-      Weights computed for every feature.
-    :param intercept:
-      Intercept computed for this model. (Only used in Binary Logistic
-      Regression. In Multinomial Logistic Regression, the intercepts will
-      not bea single value, so the intercepts will be part of the
-      weights.)
-    :param numFeatures:
-      The dimension of the features.
-    :param numClasses:
-      The number of possible outcomes for k classes classification problem
-      in Multinomial Logistic Regression. By default, it is binary
-      logistic regression so numClasses will be set to 2.
+    .. versionadded:: 0.9.0
 
+    Parameters
+    ----------
+    weights : :py:class:`pyspark.mllib.linalg.Vector`
+        Weights computed for every feature.
+    intercept : float
+        Intercept computed for this model. (Only used in Binary Logistic
+        Regression. In Multinomial Logistic Regression, the intercepts will
+        not be a single value, so the intercepts will be part of the
+        weights.)
+    numFeatures : int
+        The dimension of the features.
+    numClasses : int
+        The number of possible outcomes for k classes classification problem
+        in Multinomial Logistic Regression. By default, it is binary
+        logistic regression so numClasses will be set to 2.
+
+    Examples
+    --------
+    >>> from pyspark.mllib.linalg import SparseVector
     >>> data = [
     ...     LabeledPoint(0.0, [0.0, 1.0]),
     ...     LabeledPoint(1.0, [1.0, 0.0]),
@@ -158,8 +165,6 @@ class LogisticRegressionModel(LinearClassificationModel):
     1
     >>> mcm.predict([0.0, 0.0, 0.3])
     2
-
-    .. versionadded:: 0.9.0
     """
     def __init__(self, weights, intercept, numFeatures, numClasses):
         super(LogisticRegressionModel, self).__init__(weights, intercept)
@@ -262,58 +267,64 @@ def __repr__(self):
 
 class LogisticRegressionWithSGD(object):
     """
+    Train a classification model for Binary Logistic Regression using Stochastic Gradient Descent.
+
     .. versionadded:: 0.9.0
-    .. note:: Deprecated in 2.0.0. Use ml.classification.LogisticRegression or
-            LogisticRegressionWithLBFGS.
+    .. deprecated:: 2.0.0
+        Use ml.classification.LogisticRegression or LogisticRegressionWithLBFGS.
     """
     @classmethod
-    @since('0.9.0')
     def train(cls, data, iterations=100, step=1.0, miniBatchFraction=1.0,
               initialWeights=None, regParam=0.01, regType="l2", intercept=False,
               validateData=True, convergenceTol=0.001):
         """
         Train a logistic regression model on the given data.
 
-        :param data:
-          The training data, an RDD of LabeledPoint.
-        :param iterations:
-          The number of iterations.
-          (default: 100)
-        :param step:
-          The step parameter used in SGD.
-          (default: 1.0)
-        :param miniBatchFraction:
-          Fraction of data to be used for each SGD iteration.
-          (default: 1.0)
-        :param initialWeights:
-          The initial weights.
-          (default: None)
-        :param regParam:
-          The regularizer parameter.
-          (default: 0.01)
-        :param regType:
-          The type of regularizer used for training our model.
-          Supported values:
+        .. versionadded:: 0.9.0
+
+        Parameters
+        ----------
+        data : :py:class:`pyspark.RDD`
+            The training data, an RDD of :py:class:`pyspark.mllib.regression.LabeledPoint`.
+        iterations : int, optional
+            The number of iterations.
+            (default: 100)
+        step : float, optional
+            The step parameter used in SGD.
+            (default: 1.0)
+        miniBatchFraction : float, optional
+            Fraction of data to be used for each SGD iteration.
+            (default: 1.0)
+        initialWeights : :py:class:`pyspark.mllib.linalg.Vector` or convertible, optional
+            The initial weights.
+            (default: None)
+        regParam : float, optional
+            The regularizer parameter.
+            (default: 0.01)
+        regType : str, optional
+            The type of regularizer used for training our model.
+            Supported values:
 
             - "l1" for using L1 regularization
             - "l2" for using L2 regularization (default)
             - None for no regularization
-        :param intercept:
-          Boolean parameter which indicates the use or not of the
-          augmented representation for training data (i.e., whether bias
-          features are activated or not).
-          (default: False)
-        :param validateData:
-          Boolean parameter which indicates if the algorithm should
-          validate data before training.
-          (default: True)
-        :param convergenceTol:
-          A condition which decides iteration termination.
-          (default: 0.001)
+
+        intercept : bool, optional
+            Boolean parameter which indicates the use or not of the
+            augmented representation for training data (i.e., whether bias
+            features are activated or not).
+            (default: False)
+        validateData : bool, optional
+            Boolean parameter which indicates if the algorithm should
+            validate data before training.
+            (default: True)
+        convergenceTol : float, optional
+            A condition which decides iteration termination.
+            (default: 0.001)
         """
         warnings.warn(
             "Deprecated in 2.0.0. Use ml.classification.LogisticRegression or "
-            "LogisticRegressionWithLBFGS.", DeprecationWarning)
+            "LogisticRegressionWithLBFGS.", FutureWarning)
 
         def train(rdd, i):
             return callMLlibFunc("trainLogisticRegressionModelWithSGD", rdd, int(iterations),
@@ -325,55 +336,65 @@ def train(rdd, i):
 
 class LogisticRegressionWithLBFGS(object):
     """
+    Train a classification model for Multinomial/Binary Logistic Regression
+    using Limited-memory BFGS.
+
+    Standard feature scaling and L2 regularization are used by default.
     .. versionadded:: 1.2.0
     """
     @classmethod
-    @since('1.2.0')
     def train(cls, data, iterations=100, initialWeights=None, regParam=0.0, regType="l2",
               intercept=False, corrections=10, tolerance=1e-6, validateData=True, numClasses=2):
         """
         Train a logistic regression model on the given data.
 
-        :param data:
-          The training data, an RDD of LabeledPoint.
-        :param iterations:
-          The number of iterations.
-          (default: 100)
-        :param initialWeights:
-          The initial weights.
-          (default: None)
-        :param regParam:
-          The regularizer parameter.
-          (default: 0.0)
-        :param regType:
-          The type of regularizer used for training our model.
-          Supported values:
+        .. versionadded:: 1.2.0
+
+        Parameters
+        ----------
+        data : :py:class:`pyspark.RDD`
+            The training data, an RDD of :py:class:`pyspark.mllib.regression.LabeledPoint`.
+        iterations : int, optional
+            The number of iterations.
+            (default: 100)
+        initialWeights : :py:class:`pyspark.mllib.linalg.Vector` or convertible, optional
+            The initial weights.
+            (default: None)
+        regParam : float, optional
+            The regularizer parameter.
+            (default: 0.01)
+        regType : str, optional
+            The type of regularizer used for training our model.
+            Supported values:
 
             - "l1" for using L1 regularization
             - "l2" for using L2 regularization (default)
             - None for no regularization
-        :param intercept:
-          Boolean parameter which indicates the use or not of the
-          augmented representation for training data (i.e., whether bias
-          features are activated or not).
-          (default: False)
-        :param corrections:
-          The number of corrections used in the LBFGS update.
-          If a known updater is used for binary classification,
-          it calls the ml implementation and this parameter will
-          have no effect. (default: 10)
-        :param tolerance:
-          The convergence tolerance of iterations for L-BFGS.
-          (default: 1e-6)
-        :param validateData:
-          Boolean parameter which indicates if the algorithm should
-          validate data before training.
-          (default: True)
-        :param numClasses:
-          The number of classes (i.e., outcomes) a label can take in
-          Multinomial Logistic Regression.
-          (default: 2)
 
+        intercept : bool, optional
+            Boolean parameter which indicates the use or not of the
+            augmented representation for training data (i.e., whether bias
+            features are activated or not).
+            (default: False)
+        corrections : int, optional
+            The number of corrections used in the LBFGS update.
+            If a known updater is used for binary classification,
+            it calls the ml implementation and this parameter will
+            have no effect. (default: 10)
+        tolerance : float, optional
+            The convergence tolerance of iterations for L-BFGS.
+            (default: 1e-6)
+        validateData : bool, optional
+            Boolean parameter which indicates if the algorithm should
+            validate data before training.
+            (default: True)
+        numClasses : int, optional
+            The number of classes (i.e., outcomes) a label can take in
+            Multinomial Logistic Regression.
+            (default: 2)
+
+        Examples
+        --------
         >>> data = [
         ...     LabeledPoint(0.0, [0.0, 1.0]),
         ...     LabeledPoint(1.0, [1.0, 0.0]),
@@ -405,11 +426,18 @@ class SVMModel(LinearClassificationModel):
     """
     Model for Support Vector Machines (SVMs).
 
-    :param weights:
-      Weights computed for every feature.
-    :param intercept:
-      Intercept computed for this model.
+    .. versionadded:: 0.9.0
+
+    Parameters
+    ----------
+    weights : :py:class:`pyspark.mllib.linalg.Vector`
+        Weights computed for every feature.
+    intercept : float
+        Intercept computed for this model.
 
+    Examples
+    --------
+    >>> from pyspark.mllib.linalg import SparseVector
     >>> data = [
     ...     LabeledPoint(0.0, [0.0]),
     ...     LabeledPoint(1.0, [1.0]),
@@ -449,8 +477,6 @@ class SVMModel(LinearClassificationModel):
     ...    rmtree(path)
     ... except:
     ...    pass
-
-    .. versionadded:: 0.9.0
     """
     def __init__(self, weights, intercept):
         super(SVMModel, self).__init__(weights, intercept)
@@ -499,53 +525,59 @@ def load(cls, sc, path):
 
 class SVMWithSGD(object):
     """
+    Train a Support Vector Machine (SVM) using Stochastic Gradient Descent.
+
     .. versionadded:: 0.9.0
     """
 
     @classmethod
-    @since('0.9.0')
     def train(cls, data, iterations=100, step=1.0, regParam=0.01,
               miniBatchFraction=1.0, initialWeights=None, regType="l2",
               intercept=False, validateData=True, convergenceTol=0.001):
         """
         Train a support vector machine on the given data.
 
-        :param data:
-          The training data, an RDD of LabeledPoint.
-        :param iterations:
-          The number of iterations.
-          (default: 100)
-        :param step:
-          The step parameter used in SGD.
-          (default: 1.0)
-        :param regParam:
-          The regularizer parameter.
-          (default: 0.01)
-        :param miniBatchFraction:
-          Fraction of data to be used for each SGD iteration.
-          (default: 1.0)
-        :param initialWeights:
-          The initial weights.
-          (default: None)
-        :param regType:
-          The type of regularizer used for training our model.
-          Allowed values:
+        .. versionadded:: 0.9.0
+
+        Parameters
+        ----------
+        data : :py:class:`pyspark.RDD`
+            The training data, an RDD of :py:class:`pyspark.mllib.regression.LabeledPoint`.
+        iterations : int, optional
+            The number of iterations.
+            (default: 100)
+        step : float, optional
+            The step parameter used in SGD.
+            (default: 1.0)
+        regParam : float, optional
+            The regularizer parameter.
+            (default: 0.01)
+        miniBatchFraction : float, optional
+            Fraction of data to be used for each SGD iteration.
+            (default: 1.0)
+        initialWeights : :py:class:`pyspark.mllib.linalg.Vector` or convertible, optional
+            The initial weights.
+            (default: None)
+        regType : str, optional
+            The type of regularizer used for training our model.
+            Allowed values:
 
             - "l1" for using L1 regularization
             - "l2" for using L2 regularization (default)
             - None for no regularization
-        :param intercept:
-          Boolean parameter which indicates the use or not of the
-          augmented representation for training data (i.e. whether bias
-          features are activated or not).
-          (default: False)
-        :param validateData:
-          Boolean parameter which indicates if the algorithm should
-          validate data before training.
-          (default: True)
-        :param convergenceTol:
-          A condition which decides iteration termination.
-          (default: 0.001)
+
+        intercept : bool, optional
+            Boolean parameter which indicates the use or not of the
+            augmented representation for training data (i.e. whether bias
+            features are activated or not).
+            (default: False)
+        validateData : bool, optional
+            Boolean parameter which indicates if the algorithm should
+            validate data before training.
+            (default: True)
+        convergenceTol : float, optional
+            A condition which decides iteration termination.
+            (default: 0.001)
         """
         def train(rdd, i):
             return callMLlibFunc("trainSVMModelWithSGD", rdd, int(iterations), float(step),
@@ -561,14 +593,21 @@ class NaiveBayesModel(Saveable, Loader):
     """
     Model for Naive Bayes classifiers.
 
-    :param labels:
-      List of labels.
-    :param pi:
-      Log of class priors, whose dimension is C, number of labels.
-    :param theta:
-      Log of class conditional probabilities, whose dimension is C-by-D,
-      where D is number of features.
+    .. versionadded:: 0.9.0
 
+    Parameters
+    ----------
+    labels : :py:class:`numpy.ndarray`
+        List of labels.
+    pi : :py:class:`numpy.ndarray`
+        Log of class priors, whose dimension is C, number of labels.
+    theta : :py:class:`numpy.ndarray`
+        Log of class conditional probabilities, whose dimension is C-by-D,
+        where D is number of features.
+
+    Examples
+    --------
+    >>> from pyspark.mllib.linalg import SparseVector
     >>> data = [
     ...     LabeledPoint(0.0, [0.0, 0.0]),
     ...     LabeledPoint(0.0, [0.0, 1.0]),
@@ -602,8 +641,6 @@ class NaiveBayesModel(Saveable, Loader):
     ...     rmtree(path)
     ... except OSError:
     ...     pass
-
-    .. versionadded:: 0.9.0
     """
     def __init__(self, labels, pi, theta):
         self.labels = labels
@@ -649,11 +686,12 @@ def load(cls, sc, path):
 
 class NaiveBayes(object):
     """
+    Train a Multinomial Naive Bayes model.
+
     .. versionadded:: 0.9.0
     """
 
     @classmethod
-    @since('0.9.0')
     def train(cls, data, lambda_=1.0):
         """
         Train a Naive Bayes model given an RDD of (label, features)
@@ -666,11 +704,15 @@ def train(cls, data, lambda_=1.0):
         it can also be used as `Bernoulli NB <http://tinyurl.com/p7c96j6>`_.
         The input feature values must be nonnegative.
 
-        :param data:
-          RDD of LabeledPoint.
-        :param lambda_:
-          The smoothing parameter.
-          (default: 1.0)
+        .. versionadded:: 0.9.0
+
+        Parameters
+        ----------
+        data : :py:class:`pyspark.RDD`
+            The training data, an RDD of :py:class:`pyspark.mllib.regression.LabeledPoint`.
+        lambda\\_ : float, optional
+            The smoothing parameter.
+            (default: 1.0)
         """
         first = data.first()
         if not isinstance(first, LabeledPoint):
@@ -691,23 +733,25 @@ class StreamingLogisticRegressionWithSGD(StreamingLinearAlgorithm):
     of features must be constant. An initial weight
     vector must be provided.
 
-    :param stepSize:
-      Step size for each iteration of gradient descent.
-      (default: 0.1)
-    :param numIterations:
-      Number of iterations run for each batch of data.
-      (default: 50)
-    :param miniBatchFraction:
-      Fraction of each batch of data to use for updates.
-      (default: 1.0)
-    :param regParam:
-      L2 Regularization parameter.
-      (default: 0.0)
-    :param convergenceTol:
-      Value used to determine when to terminate iterations.
-      (default: 0.001)
-
     .. versionadded:: 1.5.0
+
+    Parameters
+    ----------
+    stepSize : float, optional
+        Step size for each iteration of gradient descent.
+        (default: 0.1)
+    numIterations : int, optional
+        Number of iterations run for each batch of data.
+        (default: 50)
+    miniBatchFraction : float, optional
+        Fraction of each batch of data to use for updates.
+        (default: 1.0)
+    regParam : float, optional
+        L2 Regularization parameter.
+        (default: 0.0)
+    convergenceTol : float, optional
+        Value used to determine when to terminate iterations.
+        (default: 0.001)
     """
     def __init__(self, stepSize=0.1, numIterations=50, miniBatchFraction=1.0, regParam=0.0,
                  convergenceTol=0.001):
diff --git a/python/pyspark/mllib/classification.pyi b/python/pyspark/mllib/classification.pyi
new file mode 100644
index 0000000000000..967b0a9f289dd
--- /dev/null
+++ b/python/pyspark/mllib/classification.pyi
@@ -0,0 +1,151 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import overload
+from typing import Optional, Union
+
+from pyspark.context import SparkContext
+from pyspark.rdd import RDD
+from pyspark.mllib._typing import VectorLike
+from pyspark.mllib.linalg import Vector
+from pyspark.mllib.regression import LabeledPoint, LinearModel, StreamingLinearAlgorithm
+from pyspark.mllib.util import Saveable, Loader
+from pyspark.streaming.dstream import DStream
+
+from numpy import float64, ndarray  # type: ignore[import]
+
+class LinearClassificationModel(LinearModel):
+    def __init__(self, weights: Vector, intercept: float) -> None: ...
+    def setThreshold(self, value: float) -> None: ...
+    @property
+    def threshold(self) -> Optional[float]: ...
+    def clearThreshold(self) -> None: ...
+    @overload
+    def predict(self, test: VectorLike) -> Union[int, float, float64]: ...
+    @overload
+    def predict(self, test: RDD[VectorLike]) -> RDD[Union[int, float]]: ...
+
+class LogisticRegressionModel(LinearClassificationModel):
+    def __init__(
+        self, weights: Vector, intercept: float, numFeatures: int, numClasses: int
+    ) -> None: ...
+    @property
+    def numFeatures(self) -> int: ...
+    @property
+    def numClasses(self) -> int: ...
+    @overload
+    def predict(self, x: VectorLike) -> Union[int, float]: ...
+    @overload
+    def predict(self, x: RDD[VectorLike]) -> RDD[Union[int, float]]: ...
+    def save(self, sc: SparkContext, path: str) -> None: ...
+    @classmethod
+    def load(cls, sc: SparkContext, path: str) -> LogisticRegressionModel: ...
+
+class LogisticRegressionWithSGD:
+    @classmethod
+    def train(
+        cls,
+        data: RDD[LabeledPoint],
+        iterations: int = ...,
+        step: float = ...,
+        miniBatchFraction: float = ...,
+        initialWeights: Optional[VectorLike] = ...,
+        regParam: float = ...,
+        regType: str = ...,
+        intercept: bool = ...,
+        validateData: bool = ...,
+        convergenceTol: float = ...,
+    ) -> LogisticRegressionModel: ...
+
+class LogisticRegressionWithLBFGS:
+    @classmethod
+    def train(
+        cls,
+        data: RDD[LabeledPoint],
+        iterations: int = ...,
+        initialWeights: Optional[VectorLike] = ...,
+        regParam: float = ...,
+        regType: str = ...,
+        intercept: bool = ...,
+        corrections: int = ...,
+        tolerance: float = ...,
+        validateData: bool = ...,
+        numClasses: int = ...,
+    ) -> LogisticRegressionModel: ...
+
+class SVMModel(LinearClassificationModel):
+    def __init__(self, weights: Vector, intercept: float) -> None: ...
+    @overload
+    def predict(self, x: VectorLike) -> float64: ...
+    @overload
+    def predict(self, x: RDD[VectorLike]) -> RDD[float64]: ...
+    def save(self, sc: SparkContext, path: str) -> None: ...
+    @classmethod
+    def load(cls, sc: SparkContext, path: str) -> SVMModel: ...
+
+class SVMWithSGD:
+    @classmethod
+    def train(
+        cls,
+        data: RDD[LabeledPoint],
+        iterations: int = ...,
+        step: float = ...,
+        regParam: float = ...,
+        miniBatchFraction: float = ...,
+        initialWeights: Optional[VectorLike] = ...,
+        regType: str = ...,
+        intercept: bool = ...,
+        validateData: bool = ...,
+        convergenceTol: float = ...,
+    ) -> SVMModel: ...
+
+class NaiveBayesModel(Saveable, Loader[NaiveBayesModel]):
+    labels: ndarray
+    pi: ndarray
+    theta: ndarray
+    def __init__(self, labels: ndarray, pi: ndarray, theta: ndarray) -> None: ...
+    @overload
+    def predict(self, x: VectorLike) -> float64: ...
+    @overload
+    def predict(self, x: RDD[VectorLike]) -> RDD[float64]: ...
+    def save(self, sc: SparkContext, path: str) -> None: ...
+    @classmethod
+    def load(cls, sc: SparkContext, path: str) -> NaiveBayesModel: ...
+
+class NaiveBayes:
+    @classmethod
+    def train(cls, data: RDD[VectorLike], lambda_: float = ...) -> NaiveBayesModel: ...
+
+class StreamingLogisticRegressionWithSGD(StreamingLinearAlgorithm):
+    stepSize: float
+    numIterations: int
+    regParam: float
+    miniBatchFraction: float
+    convergenceTol: float
+    def __init__(
+        self,
+        stepSize: float = ...,
+        numIterations: int = ...,
+        miniBatchFraction: float = ...,
+        regParam: float = ...,
+        convergenceTol: float = ...,
+    ) -> None: ...
+    def setInitialWeights(
+        self, initialWeights: VectorLike
+    ) -> StreamingLogisticRegressionWithSGD: ...
+    def trainOn(self, dstream: DStream[LabeledPoint]) -> None: ...
diff --git a/python/pyspark/mllib/clustering.py b/python/pyspark/mllib/clustering.py
index e41e5c9cc8e89..3922f1baada1c 100644
--- a/python/pyspark/mllib/clustering.py
+++ b/python/pyspark/mllib/clustering.py
@@ -17,22 +17,15 @@
 
 import sys
 import array as pyarray
-import warnings
-
-if sys.version > '3':
-    xrange = range
-    basestring = str
-
 from math import exp, log
+from collections import namedtuple
 
 from numpy import array, random, tile
 
-from collections import namedtuple
-
 from pyspark import SparkContext, since
-from pyspark.rdd import RDD, ignore_unicode_prefix
+from pyspark.rdd import RDD
 from pyspark.mllib.common import JavaModelWrapper, callMLlibFunc, callJavaFunc, _py2java, _java2py
-from pyspark.mllib.linalg import SparseVector, _convert_to_vector, DenseVector
+from pyspark.mllib.linalg import SparseVector, _convert_to_vector, DenseVector  # noqa: F401
 from pyspark.mllib.stat.distribution import MultivariateGaussian
 from pyspark.mllib.util import Saveable, Loader, inherit_doc, JavaLoader, JavaSaveable
 from pyspark.streaming import DStream
@@ -48,6 +41,10 @@ class BisectingKMeansModel(JavaModelWrapper):
     """
     A clustering model derived from the bisecting k-means method.
 
+    .. versionadded:: 2.0.0
+
+    Examples
+    --------
     >>> data = array([0.0,0.0, 1.0,1.0, 9.0,8.0, 8.0,9.0]).reshape(4, 2)
     >>> bskm = BisectingKMeans()
     >>> model = bskm.train(sc.parallelize(data, 2), k=4)
@@ -58,8 +55,6 @@ class BisectingKMeansModel(JavaModelWrapper):
     4
     >>> model.computeCost(p)
     0.0
-
-    .. versionadded:: 2.0.0
     """
 
     def __init__(self, java_model):
@@ -79,17 +74,25 @@ def k(self):
         """Get the number of clusters"""
         return self.call("k")
 
-    @since('2.0.0')
     def predict(self, x):
         """
         Find the cluster that each of the points belongs to in this
         model.
 
-        :param x:
-          A data point (or RDD of points) to determine cluster index.
-        :return:
-          Predicted cluster index or an RDD of predicted cluster indices
-          if the input is an RDD.
+        .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        x : :py:class:`pyspark.mllib.linalg.Vector` or :py:class:`pyspark.RDD`
+            A data point (or RDD of points) to determine cluster index.
+            :py:class:`pyspark.mllib.linalg.Vector` can be replaced with equivalent
+            objects (list, tuple, numpy.ndarray).
+
+        Returns
+        -------
+        int or :py:class:`pyspark.RDD` of int
+            Predicted cluster index or an RDD of predicted cluster indices
+            if the input is an RDD.
         """
         if isinstance(x, RDD):
             vecs = x.map(_convert_to_vector)
@@ -98,15 +101,20 @@ def predict(self, x):
         x = _convert_to_vector(x)
         return self.call("predict", x)
 
-    @since('2.0.0')
     def computeCost(self, x):
         """
         Return the Bisecting K-means cost (sum of squared distances of
         points to their nearest center) for this model on the given
         data. If provided with an RDD of points returns the sum.
 
-        :param point:
-          A data point (or RDD of points) to compute the cost(s).
+        .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        point : :py:class:`pyspark.mllib.linalg.Vector` or :py:class:`pyspark.RDD`
+            A data point (or RDD of points) to compute the cost(s).
+            :py:class:`pyspark.mllib.linalg.Vector` can be replaced with equivalent
+            objects (list, tuple, numpy.ndarray).
         """
         if isinstance(x, RDD):
             vecs = x.map(_convert_to_vector)
@@ -129,37 +137,43 @@ class BisectingKMeans(object):
     clusters on the bottom level would result more than `k` leaf
     clusters, larger clusters get higher priority.
 
-    Based on
-    `Steinbach, Karypis, and Kumar, A comparison of document clustering
-    techniques, KDD Workshop on Text Mining, 2000
-    <http://glaros.dtc.umn.edu/gkhome/fetch/papers/docclusterKDDTMW00.pdf>`_.
-
     .. versionadded:: 2.0.0
+
+    Notes
+    -----
+    See the original paper [1]_
+
+    .. [1] Steinbach, M. et al. "A Comparison of Document Clustering Techniques." (2000).
+        KDD Workshop on Text Mining, 2000
+        http://glaros.dtc.umn.edu/gkhome/fetch/papers/docclusterKDDTMW00.pdf
     """
 
     @classmethod
-    @since('2.0.0')
     def train(self, rdd, k=4, maxIterations=20, minDivisibleClusterSize=1.0, seed=-1888008604):
         """
         Runs the bisecting k-means algorithm return the model.
 
-        :param rdd:
-          Training points as an `RDD` of `Vector` or convertible
-          sequence types.
-        :param k:
-          The desired number of leaf clusters. The actual number could
-          be smaller if there are no divisible leaf clusters.
-          (default: 4)
-        :param maxIterations:
-          Maximum number of iterations allowed to split clusters.
-          (default: 20)
-        :param minDivisibleClusterSize:
-          Minimum number of points (if >= 1.0) or the minimum proportion
-          of points (if < 1.0) of a divisible cluster.
-          (default: 1)
-        :param seed:
-          Random seed value for cluster initialization.
-          (default: -1888008604 from classOf[BisectingKMeans].getName.##)
+        .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        rdd : :py:class:`pyspark.RDD`
+            Training points as an `RDD` of `Vector` or convertible
+            sequence types.
+        k : int, optional
+            The desired number of leaf clusters. The actual number could
+            be smaller if there are no divisible leaf clusters.
+            (default: 4)
+        maxIterations : int, optional
+            Maximum number of iterations allowed to split clusters.
+            (default: 20)
+        minDivisibleClusterSize : float, optional
+            Minimum number of points (if >= 1.0) or the minimum proportion
+            of points (if < 1.0) of a divisible cluster.
+            (default: 1)
+        seed : int, optional
+            Random seed value for cluster initialization.
+            (default: -1888008604 from classOf[BisectingKMeans].getName.##)
         """
         java_model = callMLlibFunc(
             "trainBisectingKMeans", rdd.map(_convert_to_vector),
@@ -172,6 +186,10 @@ class KMeansModel(Saveable, Loader):
 
     """A clustering model derived from the k-means method.
 
+    .. versionadded:: 0.9.0
+
+    Examples
+    --------
     >>> data = array([0.0,0.0, 1.0,1.0, 9.0,8.0, 8.0,9.0]).reshape(4, 2)
     >>> model = KMeans.train(
     ...     sc.parallelize(data), 2, maxIterations=10, initializationMode="random",
@@ -220,8 +238,6 @@ class KMeansModel(Saveable, Loader):
     ...     initialModel = KMeansModel([(-1000.0,-1000.0),(5.0,5.0),(1000.0,1000.0)]))
     >>> model.clusterCenters
     [array([-1000., -1000.]), array([ 5.,  5.]), array([ 1000.,  1000.])]
-
-    .. versionadded:: 0.9.0
     """
 
     def __init__(self, centers):
@@ -239,17 +255,25 @@ def k(self):
         """Total number of clusters."""
         return len(self.centers)
 
-    @since('0.9.0')
     def predict(self, x):
         """
         Find the cluster that each of the points belongs to in this
         model.
 
-        :param x:
-          A data point (or RDD of points) to determine cluster index.
-        :return:
-          Predicted cluster index or an RDD of predicted cluster indices
-          if the input is an RDD.
+        .. versionadded:: 0.9.0
+
+        Parameters
+        ----------
+        x : :py:class:`pyspark.mllib.linalg.Vector` or :py:class:`pyspark.RDD`
+            A data point (or RDD of points) to determine cluster index.
+            :py:class:`pyspark.mllib.linalg.Vector` can be replaced with equivalent
+            objects (list, tuple, numpy.ndarray).
+
+        Returns
+        -------
+        int or :py:class:`pyspark.RDD` of int
+            Predicted cluster index or an RDD of predicted cluster indices
+            if the input is an RDD.
         """
         best = 0
         best_distance = float("inf")
@@ -257,22 +281,25 @@ def predict(self, x):
             return x.map(self.predict)
 
         x = _convert_to_vector(x)
-        for i in xrange(len(self.centers)):
+        for i in range(len(self.centers)):
             distance = x.squared_distance(self.centers[i])
             if distance < best_distance:
                 best = i
                 best_distance = distance
         return best
 
-    @since('1.4.0')
     def computeCost(self, rdd):
         """
         Return the K-means cost (sum of squared distances of points to
         their nearest center) for this model on the given
         data.
 
-        :param rdd:
-          The RDD of points to compute the cost on.
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        rdd : ::py:class:`pyspark.RDD`
+            The RDD of points to compute the cost on.
         """
         cost = callMLlibFunc("computeCostKmeansModel", rdd.map(_convert_to_vector),
                              [_convert_to_vector(c) for c in self.centers])
@@ -299,46 +326,51 @@ def load(cls, sc, path):
 
 class KMeans(object):
     """
+    K-means clustering.
+
     .. versionadded:: 0.9.0
     """
 
     @classmethod
-    @since('0.9.0')
     def train(cls, rdd, k, maxIterations=100, initializationMode="k-means||",
               seed=None, initializationSteps=2, epsilon=1e-4, initialModel=None):
         """
         Train a k-means clustering model.
 
-        :param rdd:
-          Training points as an `RDD` of `Vector` or convertible
-          sequence types.
-        :param k:
-          Number of clusters to create.
-        :param maxIterations:
-          Maximum number of iterations allowed.
-          (default: 100)
-        :param initializationMode:
-          The initialization algorithm. This can be either "random" or
-          "k-means||".
-          (default: "k-means||")
-        :param seed:
-          Random seed value for cluster initialization. Set as None to
-          generate seed based on system time.
-          (default: None)
-        :param initializationSteps:
-          Number of steps for the k-means|| initialization mode.
-          This is an advanced setting -- the default of 2 is almost
-          always enough.
-          (default: 2)
-        :param epsilon:
-          Distance threshold within which a center will be considered to
-          have converged. If all centers move less than this Euclidean
-          distance, iterations are stopped.
-          (default: 1e-4)
-        :param initialModel:
-          Initial cluster centers can be provided as a KMeansModel object
-          rather than using the random or k-means|| initializationModel.
-          (default: None)
+        .. versionadded:: 0.9.0
+
+        Parameters
+        ----------
+        rdd : ::py:class:`pyspark.RDD`
+            Training points as an `RDD` of :py:class:`pyspark.mllib.linalg.Vector`
+            or convertible sequence types.
+        k : int
+            Number of clusters to create.
+        maxIterations : int, optional
+            Maximum number of iterations allowed.
+            (default: 100)
+        initializationMode : str, optional
+            The initialization algorithm. This can be either "random" or
+            "k-means||".
+            (default: "k-means||")
+        seed : int, optional
+            Random seed value for cluster initialization. Set as None to
+            generate seed based on system time.
+            (default: None)
+        initializationSteps :
+            Number of steps for the k-means|| initialization mode.
+            This is an advanced setting -- the default of 2 is almost
+            always enough.
+            (default: 2)
+        epsilon : float, optional
+            Distance threshold within which a center will be considered to
+            have converged. If all centers move less than this Euclidean
+            distance, iterations are stopped.
+            (default: 1e-4)
+        initialModel : :py:class:`KMeansModel`, optional
+            Initial cluster centers can be provided as a KMeansModel object
+            rather than using the random or k-means|| initializationModel.
+            (default: None)
         """
         clusterInitialModel = []
         if initialModel is not None:
@@ -359,6 +391,10 @@ class GaussianMixtureModel(JavaModelWrapper, JavaSaveable, JavaLoader):
     """
     A clustering model derived from the Gaussian Mixture Model method.
 
+    .. versionadded:: 1.3.0
+
+    Examples
+    --------
     >>> from pyspark.mllib.linalg import Vectors, DenseMatrix
     >>> from numpy.testing import assert_equal
     >>> from shutil import rmtree
@@ -417,8 +453,6 @@ class GaussianMixtureModel(JavaModelWrapper, JavaSaveable, JavaLoader):
     True
     >>> labels[2]==labels[3]==labels[4]
     True
-
-    .. versionadded:: 1.3.0
     """
 
     @property
@@ -447,17 +481,23 @@ def k(self):
         """Number of gaussians in mixture."""
         return len(self.weights)
 
-    @since('1.3.0')
     def predict(self, x):
         """
         Find the cluster to which the point 'x' or each point in RDD 'x'
         has maximum membership in this model.
 
-        :param x:
-          A feature vector or an RDD of vectors representing data points.
-        :return:
-          Predicted cluster label or an RDD of predicted cluster labels
-          if the input is an RDD.
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        x : :py:class:`pyspark.mllib.linalg.Vector` or :py:class:`pyspark.RDD`
+            A feature vector or an RDD of vectors representing data points.
+
+        Returns
+        -------
+        numpy.float64 or :py:class:`pyspark.RDD` of int
+            Predicted cluster label or an RDD of predicted cluster labels
+            if the input is an RDD.
         """
         if isinstance(x, RDD):
             cluster_labels = self.predictSoft(x).map(lambda z: z.index(max(z)))
@@ -466,16 +506,22 @@ def predict(self, x):
             z = self.predictSoft(x)
             return z.argmax()
 
-    @since('1.3.0')
     def predictSoft(self, x):
         """
         Find the membership of point 'x' or each point in RDD 'x' to all mixture components.
 
-        :param x:
-          A feature vector or an RDD of vectors representing data points.
-        :return:
-          The membership value to all mixture components for vector 'x'
-          or each vector in RDD 'x'.
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        x : :py:class:`pyspark.mllib.linalg.Vector` or :py:class:`pyspark.RDD`
+            A feature vector or an RDD of vectors representing data points.
+
+        Returns
+        -------
+        numpy.ndarray or :py:class:`pyspark.RDD`
+            The membership value to all mixture components for vector 'x'
+            or each vector in RDD 'x'.
         """
         if isinstance(x, RDD):
             means, sigmas = zip(*[(g.mu, g.sigma) for g in self.gaussians])
@@ -486,14 +532,16 @@ def predictSoft(self, x):
             return self.call("predictSoft", _convert_to_vector(x)).toArray()
 
     @classmethod
-    @since('1.5.0')
     def load(cls, sc, path):
         """Load the GaussianMixtureModel from disk.
 
-        :param sc:
-          SparkContext.
-        :param path:
-          Path to where the model is stored.
+        .. versionadded:: 1.5.0
+
+        Parameters
+        ----------
+        sc : :py:class:`SparkContext`
+        path : str
+            Path to where the model is stored.
         """
         model = cls._load_java(sc, path)
         wrapper = sc._jvm.org.apache.spark.mllib.api.python.GaussianMixtureModelWrapper(model)
@@ -506,32 +554,36 @@ class GaussianMixture(object):
 
     .. versionadded:: 1.3.0
     """
+
     @classmethod
-    @since('1.3.0')
     def train(cls, rdd, k, convergenceTol=1e-3, maxIterations=100, seed=None, initialModel=None):
         """
         Train a Gaussian Mixture clustering model.
 
-        :param rdd:
-          Training points as an `RDD` of `Vector` or convertible
-          sequence types.
-        :param k:
-          Number of independent Gaussians in the mixture model.
-        :param convergenceTol:
-          Maximum change in log-likelihood at which convergence is
-          considered to have occurred.
-          (default: 1e-3)
-        :param maxIterations:
-          Maximum number of iterations allowed.
-          (default: 100)
-        :param seed:
-          Random seed for initial Gaussian distribution. Set as None to
-          generate seed based on system time.
-          (default: None)
-        :param initialModel:
-          Initial GMM starting point, bypassing the random
-          initialization.
-          (default: None)
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        rdd : ::py:class:`pyspark.RDD`
+            Training points as an `RDD` of :py:class:`pyspark.mllib.linalg.Vector`
+            or convertible sequence types.
+        k : int
+            Number of independent Gaussians in the mixture model.
+        convergenceTol : float, optional
+            Maximum change in log-likelihood at which convergence is
+            considered to have occurred.
+            (default: 1e-3)
+        maxIterations : int, optional
+            Maximum number of iterations allowed.
+            (default: 100)
+        seed : int, optional
+            Random seed for initial Gaussian distribution. Set as None to
+            generate seed based on system time.
+            (default: None)
+        initialModel : GaussianMixtureModel, optional
+            Initial GMM starting point, bypassing the random
+            initialization.
+            (default: None)
         """
         initialModelWeights = None
         initialModelMu = None
@@ -552,8 +604,12 @@ def train(cls, rdd, k, convergenceTol=1e-3, maxIterations=100, seed=None, initia
 class PowerIterationClusteringModel(JavaModelWrapper, JavaSaveable, JavaLoader):
 
     """
-    Model produced by [[PowerIterationClustering]].
+    Model produced by :py:class:`PowerIterationClustering`.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> import math
     >>> def genCircle(r, n):
     ...     points = []
@@ -596,8 +652,6 @@ class PowerIterationClusteringModel(JavaModelWrapper, JavaSaveable, JavaLoader):
     ...     rmtree(path)
     ... except OSError:
     ...     pass
-
-    .. versionadded:: 1.5.0
     """
 
     @property
@@ -630,37 +684,48 @@ def load(cls, sc, path):
 
 class PowerIterationClustering(object):
     """
-    Power Iteration Clustering (PIC), a scalable graph clustering algorithm
-    developed by [[http://www.cs.cmu.edu/~frank/papers/icml2010-pic-final.pdf Lin and Cohen]].
-    From the abstract: PIC finds a very low-dimensional embedding of a
-    dataset using truncated power iteration on a normalized pair-wise
-    similarity matrix of the data.
+    Power Iteration Clustering (PIC), a scalable graph clustering algorithm.
+
+
+    Developed by Lin and Cohen [1]_. From the abstract:
+
+        "PIC finds a very low-dimensional embedding of a
+        dataset using truncated power iteration on a normalized pair-wise
+        similarity matrix of the data."
 
     .. versionadded:: 1.5.0
+
+    .. [1] Lin, Frank & Cohen, William. (2010). Power Iteration Clustering.
+        http://www.cs.cmu.edu/~frank/papers/icml2010-pic-final.pdf
     """
 
     @classmethod
-    @since('1.5.0')
     def train(cls, rdd, k, maxIterations=100, initMode="random"):
         r"""
-        :param rdd:
-          An RDD of (i, j, s\ :sub:`ij`\) tuples representing the
-          affinity matrix, which is the matrix A in the PIC paper.  The
-          similarity s\ :sub:`ij`\ must be nonnegative.  This is a symmetric
-          matrix and hence s\ :sub:`ij`\ = s\ :sub:`ji`\  For any (i, j) with
-          nonzero similarity, there should be either (i, j, s\ :sub:`ij`\) or
-          (j, i, s\ :sub:`ji`\) in the input.  Tuples with i = j are ignored,
-          because it is assumed s\ :sub:`ij`\ = 0.0.
-        :param k:
-          Number of clusters.
-        :param maxIterations:
-          Maximum number of iterations of the PIC algorithm.
-          (default: 100)
-        :param initMode:
-          Initialization mode. This can be either "random" to use
-          a random vector as vertex properties, or "degree" to use
-          normalized sum similarities.
-          (default: "random")
+        Train PowerIterationClusteringModel
+
+        .. versionadded:: 1.5.0
+
+        Parameters
+        ----------
+        rdd : :py:class:`pyspark.RDD`
+            An RDD of (i, j, s\ :sub:`ij`\) tuples representing the
+            affinity matrix, which is the matrix A in the PIC paper.  The
+            similarity s\ :sub:`ij`\ must be nonnegative.  This is a symmetric
+            matrix and hence s\ :sub:`ij`\ = s\ :sub:`ji`\  For any (i, j) with
+            nonzero similarity, there should be either (i, j, s\ :sub:`ij`\) or
+            (j, i, s\ :sub:`ji`\) in the input.  Tuples with i = j are ignored,
+            because it is assumed s\ :sub:`ij`\ = 0.0.
+        k : int
+            Number of clusters.
+        maxIterations : int, optional
+            Maximum number of iterations of the PIC algorithm.
+            (default: 100)
+        initMode : str, optional
+            Initialization mode. This can be either "random" to use
+            a random vector as vertex properties, or "degree" to use
+            normalized sum similarities.
+            (default: "random")
         """
         model = callMLlibFunc("trainPowerIterationClusteringModel",
                               rdd.map(_convert_to_vector), int(k), int(maxIterations), initMode)
@@ -680,35 +745,43 @@ class StreamingKMeansModel(KMeansModel):
 
     The update formula for each centroid is given by
 
-    * c_t+1 = ((c_t * n_t * a) + (x_t * m_t)) / (n_t + m_t)
-    * n_t+1 = n_t * a + m_t
+    - c_t+1 = ((c_t * n_t * a) + (x_t * m_t)) / (n_t + m_t)
+    - n_t+1 = n_t * a + m_t
 
     where
 
-    * c_t: Centroid at the n_th iteration.
-    * n_t: Number of samples (or) weights associated with the centroid
-           at the n_th iteration.
-    * x_t: Centroid of the new data closest to c_t.
-    * m_t: Number of samples (or) weights of the new data closest to c_t
-    * c_t+1: New centroid.
-    * n_t+1: New number of weights.
-    * a: Decay Factor, which gives the forgetfulness.
-
-    .. note:: If a is set to 1, it is the weighted mean of the previous
-        and new data. If it set to zero, the old centroids are completely
-        forgotten.
+    - c_t: Centroid at the n_th iteration.
+    - n_t: Number of samples (or) weights associated with the centroid
+      at the n_th iteration.
+    - x_t: Centroid of the new data closest to c_t.
+    - m_t: Number of samples (or) weights of the new data closest to c_t
+    - c_t+1: New centroid.
+    - n_t+1: New number of weights.
+    - a: Decay Factor, which gives the forgetfulness.
 
-    :param clusterCenters:
-      Initial cluster centers.
-    :param clusterWeights:
-      List of weights assigned to each cluster.
+    .. versionadded:: 1.5.0
 
+    Parameters
+    ----------
+    clusterCenters : list of :py:class:`pyspark.mllib.linalg.Vector` or covertible
+        Initial cluster centers.
+    clusterWeights : :py:class:`pyspark.mllib.linalg.Vector` or covertible
+        List of weights assigned to each cluster.
+
+    Notes
+    -----
+    If a is set to 1, it is the weighted mean of the previous
+    and new data. If it set to zero, the old centroids are completely
+    forgotten.
+
+    Examples
+    --------
     >>> initCenters = [[0.0, 0.0], [1.0, 1.0]]
     >>> initWeights = [1.0, 1.0]
     >>> stkm = StreamingKMeansModel(initCenters, initWeights)
     >>> data = sc.parallelize([[-0.1, -0.1], [0.1, 0.1],
     ...                        [0.9, 0.9], [1.1, 1.1]])
-    >>> stkm = stkm.update(data, 1.0, u"batches")
+    >>> stkm = stkm.update(data, 1.0, "batches")
     >>> stkm.centers
     array([[ 0.,  0.],
            [ 1.,  1.]])
@@ -720,7 +793,7 @@ class StreamingKMeansModel(KMeansModel):
     [3.0, 3.0]
     >>> decayFactor = 0.0
     >>> data = sc.parallelize([DenseVector([1.5, 1.5]), DenseVector([0.2, 0.2])])
-    >>> stkm = stkm.update(data, 0.0, u"batches")
+    >>> stkm = stkm.update(data, 0.0, "batches")
     >>> stkm.centers
     array([[ 0.2,  0.2],
            [ 1.5,  1.5]])
@@ -730,8 +803,6 @@ class StreamingKMeansModel(KMeansModel):
     0
     >>> stkm.predict([1.5, 1.5])
     1
-
-    .. versionadded:: 1.5.0
     """
     def __init__(self, clusterCenters, clusterWeights):
         super(StreamingKMeansModel, self).__init__(centers=clusterCenters)
@@ -743,19 +814,22 @@ def clusterWeights(self):
         """Return the cluster weights."""
         return self._clusterWeights
 
-    @ignore_unicode_prefix
     @since('1.5.0')
     def update(self, data, decayFactor, timeUnit):
         """Update the centroids, according to data
 
-        :param data:
-          RDD with new data for the model update.
-        :param decayFactor:
-          Forgetfulness of the previous centroids.
-        :param timeUnit:
-          Can be "batches" or "points". If points, then the decay factor
-          is raised to the power of number of new points and if batches,
-          then decay factor will be used as is.
+        .. versionadded:: 1.5.0
+
+        Parameters
+        ----------
+        data : :py:class:`pyspark.RDD`
+            RDD with new data for the model update.
+        decayFactor : float
+            Forgetfulness of the previous centroids.
+        timeUnit :  str
+            Can be "batches" or "points". If points, then the decay factor
+            is raised to the power of number of new points and if batches,
+            then decay factor will be used as is.
         """
         if not isinstance(data, RDD):
             raise TypeError("Data should be of an RDD, got %s." % type(data))
@@ -780,19 +854,21 @@ class StreamingKMeans(object):
     More details on how the centroids are updated are provided under the
     docs of StreamingKMeansModel.
 
-    :param k:
-      Number of clusters.
-      (default: 2)
-    :param decayFactor:
-      Forgetfulness of the previous centroids.
-      (default: 1.0)
-    :param timeUnit:
-      Can be "batches" or "points". If points, then the decay factor is
-      raised to the power of number of new points and if batches, then
-      decay factor will be used as is.
-      (default: "batches")
-
     .. versionadded:: 1.5.0
+
+    Parameters
+    ----------
+    k : int, optional
+        Number of clusters.
+        (default: 2)
+    decayFactor : float, optional
+        Forgetfulness of the previous centroids.
+        (default: 1.0)
+    timeUnit : str, optional
+        Can be "batches" or "points". If points, then the decay factor is
+        raised to the power of number of new points and if batches, then
+        decay factor will be used as is.
+        (default: "batches")
     """
     def __init__(self, k=2, decayFactor=1.0, timeUnit="batches"):
         self._k = k
@@ -851,7 +927,7 @@ def setInitialCenters(self, centers, weights):
     @since('1.5.0')
     def setRandomCenters(self, dim, weight, seed):
         """
-        Set the initial centres to be random samples from
+        Set the initial centers to be random samples from
         a gaussian population with constant weights.
         """
         rng = random.RandomState(seed)
@@ -895,13 +971,23 @@ class LDAModel(JavaModelWrapper, JavaSaveable, Loader):
 
     Latent Dirichlet Allocation (LDA), a topic model designed for text documents.
     Terminology
+
     - "word" = "term": an element of the vocabulary
     - "token": instance of a term appearing in a document
     - "topic": multinomial distribution over words representing some concept
-    References:
-    - Original LDA paper (journal version):
-    Blei, Ng, and Jordan.  "Latent Dirichlet Allocation."  JMLR, 2003.
 
+    .. versionadded:: 1.5.0
+
+    Notes
+    -----
+    See the original LDA paper (journal version) [1]_
+
+    .. [1] Blei, D. et al. "Latent Dirichlet Allocation."
+        J. Mach. Learn. Res. 3 (2003): 993-1022.
+        https://www.jmlr.org/papers/v3/blei03a
+
+    Examples
+    --------
     >>> from pyspark.mllib.linalg import Vectors
     >>> from numpy.testing import assert_almost_equal, assert_equal
     >>> data = [
@@ -933,8 +1019,6 @@ class LDAModel(JavaModelWrapper, JavaSaveable, Loader):
     ...     rmtree(path)
     ... except OSError:
     ...     pass
-
-    .. versionadded:: 1.5.0
     """
 
     @since('1.5.0')
@@ -947,19 +1031,24 @@ def vocabSize(self):
         """Vocabulary size (number of terms or terms in the vocabulary)"""
         return self.call("vocabSize")
 
-    @since('1.6.0')
     def describeTopics(self, maxTermsPerTopic=None):
         """Return the topics described by weighted terms.
 
-        WARNING: If vocabSize and k are large, this can return a large object!
-
-        :param maxTermsPerTopic:
-          Maximum number of terms to collect for each topic.
-          (default: vocabulary size)
-        :return:
-          Array over topics. Each topic is represented as a pair of
-          matching arrays: (term indices, term weights in topic).
-          Each topic's terms are sorted in order of decreasing weight.
+        .. versionadded:: 1.6.0
+        .. warning:: If vocabSize and k are large, this can return a large object!
+
+        Parameters
+        ----------
+        maxTermsPerTopic : int, optional
+            Maximum number of terms to collect for each topic.
+            (default: vocabulary size)
+
+        Returns
+        -------
+        list
+            Array over topics. Each topic is represented as a pair of
+            matching arrays: (term indices, term weights in topic).
+            Each topic's terms are sorted in order of decreasing weight.
         """
         if maxTermsPerTopic is None:
             topics = self.call("describeTopics")
@@ -968,66 +1057,73 @@ def describeTopics(self, maxTermsPerTopic=None):
         return topics
 
     @classmethod
-    @since('1.5.0')
     def load(cls, sc, path):
         """Load the LDAModel from disk.
 
-        :param sc:
-          SparkContext.
-        :param path:
-          Path to where the model is stored.
+        .. versionadded:: 1.5.0
+
+        Parameters
+        ----------
+        sc : :py:class:`pyspark.SparkContext`
+        path : str
+            Path to where the model is stored.
         """
         if not isinstance(sc, SparkContext):
             raise TypeError("sc should be a SparkContext, got type %s" % type(sc))
-        if not isinstance(path, basestring):
-            raise TypeError("path should be a basestring, got type %s" % type(path))
+        if not isinstance(path, str):
+            raise TypeError("path should be a string, got type %s" % type(path))
         model = callMLlibFunc("loadLDAModel", sc, path)
         return LDAModel(model)
 
 
 class LDA(object):
     """
+    Train Latent Dirichlet Allocation (LDA) model.
+
     .. versionadded:: 1.5.0
     """
 
     @classmethod
-    @since('1.5.0')
     def train(cls, rdd, k=10, maxIterations=20, docConcentration=-1.0,
               topicConcentration=-1.0, seed=None, checkpointInterval=10, optimizer="em"):
         """Train a LDA model.
 
-        :param rdd:
-          RDD of documents, which are tuples of document IDs and term
-          (word) count vectors. The term count vectors are "bags of
-          words" with a fixed-size vocabulary (where the vocabulary size
-          is the length of the vector). Document IDs must be unique
-          and >= 0.
-        :param k:
-          Number of topics to infer, i.e., the number of soft cluster
-          centers.
-          (default: 10)
-        :param maxIterations:
-          Maximum number of iterations allowed.
-          (default: 20)
-        :param docConcentration:
-          Concentration parameter (commonly named "alpha") for the prior
-          placed on documents' distributions over topics ("theta").
-          (default: -1.0)
-        :param topicConcentration:
-          Concentration parameter (commonly named "beta" or "eta") for
-          the prior placed on topics' distributions over terms.
-          (default: -1.0)
-        :param seed:
-          Random seed for cluster initialization. Set as None to generate
-          seed based on system time.
-          (default: None)
-        :param checkpointInterval:
-          Period (in iterations) between checkpoints.
-          (default: 10)
-        :param optimizer:
-          LDAOptimizer used to perform the actual calculation. Currently
-          "em", "online" are supported.
-          (default: "em")
+        .. versionadded:: 1.5.0
+
+        Parameters
+        ----------
+        rdd : :py:class:`pyspark.RDD`
+            RDD of documents, which are tuples of document IDs and term
+            (word) count vectors. The term count vectors are "bags of
+            words" with a fixed-size vocabulary (where the vocabulary size
+            is the length of the vector). Document IDs must be unique
+            and >= 0.
+        k : int, optional
+            Number of topics to infer, i.e., the number of soft cluster
+            centers.
+            (default: 10)
+        maxIterations : int, optional
+            Maximum number of iterations allowed.
+            (default: 20)
+        docConcentration : float, optional
+            Concentration parameter (commonly named "alpha") for the prior
+            placed on documents' distributions over topics ("theta").
+            (default: -1.0)
+        topicConcentration : float, optional
+            Concentration parameter (commonly named "beta" or "eta") for
+            the prior placed on topics' distributions over terms.
+            (default: -1.0)
+        seed : int, optional
+            Random seed for cluster initialization. Set as None to generate
+            seed based on system time.
+            (default: None)
+        checkpointInterval : int, optional
+            Period (in iterations) between checkpoints.
+            (default: 10)
+        optimizer : str, optional
+            LDAOptimizer used to perform the actual calculation. Currently
+            "em", "online" are supported.
+            (default: "em")
         """
         model = callMLlibFunc("trainLDAModel", rdd, k, maxIterations,
                               docConcentration, topicConcentration, seed,
diff --git a/python/pyspark/mllib/clustering.pyi b/python/pyspark/mllib/clustering.pyi
new file mode 100644
index 0000000000000..b4f349612f0fe
--- /dev/null
+++ b/python/pyspark/mllib/clustering.pyi
@@ -0,0 +1,198 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import overload
+from typing import List, NamedTuple, Optional, Tuple, TypeVar
+
+import array
+
+from numpy import float64, int64, ndarray  # type: ignore[import]
+from py4j.java_gateway import JavaObject  # type: ignore[import]
+
+from pyspark.mllib._typing import VectorLike
+from pyspark.context import SparkContext
+from pyspark.rdd import RDD
+from pyspark.mllib.common import JavaModelWrapper
+from pyspark.mllib.stat.distribution import MultivariateGaussian
+from pyspark.mllib.util import Saveable, Loader, JavaLoader, JavaSaveable
+from pyspark.streaming.dstream import DStream
+
+T = TypeVar("T")
+
+class BisectingKMeansModel(JavaModelWrapper):
+    centers: List[ndarray]
+    def __init__(self, java_model: JavaObject) -> None: ...
+    @property
+    def clusterCenters(self) -> List[ndarray]: ...
+    @property
+    def k(self) -> int: ...
+    @overload
+    def predict(self, x: VectorLike) -> int: ...
+    @overload
+    def predict(self, x: RDD[VectorLike]) -> RDD[int]: ...
+    @overload
+    def computeCost(self, x: VectorLike) -> float: ...
+    @overload
+    def computeCost(self, x: RDD[VectorLike]) -> float: ...
+
+class BisectingKMeans:
+    @classmethod
+    def train(
+        self,
+        rdd: RDD[VectorLike],
+        k: int = ...,
+        maxIterations: int = ...,
+        minDivisibleClusterSize: float = ...,
+        seed: int = ...,
+    ) -> BisectingKMeansModel: ...
+
+class KMeansModel(Saveable, Loader[KMeansModel]):
+    centers: List[ndarray]
+    def __init__(self, centers: List[VectorLike]) -> None: ...
+    @property
+    def clusterCenters(self) -> List[ndarray]: ...
+    @property
+    def k(self) -> int: ...
+    @overload
+    def predict(self, x: VectorLike) -> int: ...
+    @overload
+    def predict(self, x: RDD[VectorLike]) -> RDD[int]: ...
+    def computeCost(self, rdd: RDD[VectorLike]) -> float: ...
+    def save(self, sc: SparkContext, path: str) -> None: ...
+    @classmethod
+    def load(cls, sc: SparkContext, path: str) -> KMeansModel: ...
+
+class KMeans:
+    @classmethod
+    def train(
+        cls,
+        rdd: RDD[VectorLike],
+        k: int,
+        maxIterations: int = ...,
+        initializationMode: str = ...,
+        seed: Optional[int] = ...,
+        initializationSteps: int = ...,
+        epsilon: float = ...,
+        initialModel: Optional[KMeansModel] = ...,
+    ) -> KMeansModel: ...
+
+class GaussianMixtureModel(
+    JavaModelWrapper, JavaSaveable, JavaLoader[GaussianMixtureModel]
+):
+    @property
+    def weights(self) -> ndarray: ...
+    @property
+    def gaussians(self) -> List[MultivariateGaussian]: ...
+    @property
+    def k(self) -> int: ...
+    @overload
+    def predict(self, x: VectorLike) -> int64: ...
+    @overload
+    def predict(self, x: RDD[VectorLike]) -> RDD[int]: ...
+    @overload
+    def predictSoft(self, x: VectorLike) -> ndarray: ...
+    @overload
+    def predictSoft(self, x: RDD[VectorLike]) -> RDD[array.array]: ...
+    @classmethod
+    def load(cls, sc: SparkContext, path: str) -> GaussianMixtureModel: ...
+
+class GaussianMixture:
+    @classmethod
+    def train(
+        cls,
+        rdd: RDD[VectorLike],
+        k: int,
+        convergenceTol: float = ...,
+        maxIterations: int = ...,
+        seed: Optional[int] = ...,
+        initialModel: Optional[GaussianMixtureModel] = ...,
+    ) -> GaussianMixtureModel: ...
+
+class PowerIterationClusteringModel(
+    JavaModelWrapper, JavaSaveable, JavaLoader[PowerIterationClusteringModel]
+):
+    @property
+    def k(self) -> int: ...
+    def assignments(self) -> RDD[PowerIterationClustering.Assignment]: ...
+    @classmethod
+    def load(cls, sc: SparkContext, path: str) -> PowerIterationClusteringModel: ...
+
+class PowerIterationClustering:
+    @classmethod
+    def train(
+        cls,
+        rdd: RDD[Tuple[int, int, float]],
+        k: int,
+        maxIterations: int = ...,
+        initMode: str = ...,
+    ) -> PowerIterationClusteringModel: ...
+    class Assignment(NamedTuple("Assignment", [("id", int), ("cluster", int)])): ...
+
+class StreamingKMeansModel(KMeansModel):
+    def __init__(
+        self, clusterCenters: List[VectorLike], clusterWeights: VectorLike
+    ) -> None: ...
+    @property
+    def clusterWeights(self) -> List[float64]: ...
+    centers: ndarray
+    def update(
+        self, data: RDD[VectorLike], decayFactor: float, timeUnit: str
+    ) -> StreamingKMeansModel: ...
+
+class StreamingKMeans:
+    def __init__(
+        self, k: int = ..., decayFactor: float = ..., timeUnit: str = ...
+    ) -> None: ...
+    def latestModel(self) -> StreamingKMeansModel: ...
+    def setK(self, k: int) -> StreamingKMeans: ...
+    def setDecayFactor(self, decayFactor: float) -> StreamingKMeans: ...
+    def setHalfLife(self, halfLife: float, timeUnit: str) -> StreamingKMeans: ...
+    def setInitialCenters(
+        self, centers: List[VectorLike], weights: List[float]
+    ) -> StreamingKMeans: ...
+    def setRandomCenters(
+        self, dim: int, weight: float, seed: int
+    ) -> StreamingKMeans: ...
+    def trainOn(self, dstream: DStream[VectorLike]) -> None: ...
+    def predictOn(self, dstream: DStream[VectorLike]) -> DStream[int]: ...
+    def predictOnValues(
+        self, dstream: DStream[Tuple[T, VectorLike]]
+    ) -> DStream[Tuple[T, int]]: ...
+
+class LDAModel(JavaModelWrapper, JavaSaveable, Loader[LDAModel]):
+    def topicsMatrix(self) -> ndarray: ...
+    def vocabSize(self) -> int: ...
+    def describeTopics(
+        self, maxTermsPerTopic: Optional[int] = ...
+    ) -> List[Tuple[List[int], List[float]]]: ...
+    @classmethod
+    def load(cls, sc: SparkContext, path: str) -> LDAModel: ...
+
+class LDA:
+    @classmethod
+    def train(
+        cls,
+        rdd: RDD[Tuple[int, VectorLike]],
+        k: int = ...,
+        maxIterations: int = ...,
+        docConcentration: float = ...,
+        topicConcentration: float = ...,
+        seed: Optional[int] = ...,
+        checkpointInterval: int = ...,
+        optimizer: str = ...,
+    ) -> LDAModel: ...
diff --git a/python/pyspark/mllib/common.py b/python/pyspark/mllib/common.py
index bac8f350563ec..24e2f198251ad 100644
--- a/python/pyspark/mllib/common.py
+++ b/python/pyspark/mllib/common.py
@@ -15,11 +15,6 @@
 # limitations under the License.
 #
 
-import sys
-if sys.version >= '3':
-    long = int
-    unicode = str
-
 import py4j.protocol
 from py4j.protocol import Py4JJavaError
 from py4j.java_gateway import JavaObject
@@ -81,7 +76,7 @@ def _py2java(sc, obj):
         obj = [_py2java(sc, x) for x in obj]
     elif isinstance(obj, JavaObject):
         pass
-    elif isinstance(obj, (int, long, float, bool, bytes, unicode)):
+    elif isinstance(obj, (int, float, bool, bytes, str)):
         pass
     else:
         data = bytearray(PickleSerializer().dumps(obj))
diff --git a/python/pyspark/mllib/common.pyi b/python/pyspark/mllib/common.pyi
new file mode 100644
index 0000000000000..daba212d93633
--- /dev/null
+++ b/python/pyspark/mllib/common.pyi
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Any, TypeVar
+
+import pyspark.context
+
+from py4j.java_gateway import JavaObject
+
+C = TypeVar("C", bound=type)
+
+def callJavaFunc(sc: pyspark.context.SparkContext, func: Any, *args: Any) -> Any: ...
+def callMLlibFunc(name: str, *args: Any) -> Any: ...
+
+class JavaModelWrapper:
+    def __init__(self, java_model: JavaObject) -> None: ...
+    def __del__(self) -> None: ...
+    def call(self, name: str, *a: Any) -> Any: ...
+
+def inherit_doc(cls: C) -> C: ...
diff --git a/python/pyspark/mllib/evaluation.py b/python/pyspark/mllib/evaluation.py
index f3be827fb6e4f..2f25c7672a93a 100644
--- a/python/pyspark/mllib/evaluation.py
+++ b/python/pyspark/mllib/evaluation.py
@@ -30,8 +30,15 @@ class BinaryClassificationMetrics(JavaModelWrapper):
     """
     Evaluator for binary classification.
 
-    :param scoreAndLabels: an RDD of score, label and optional weight.
+    .. versionadded:: 1.4.0
+
+    Parameters
+    ----------
+    scoreAndLabels : :py:class:`pyspark.RDD`
+        an RDD of score, label and optional weight.
 
+    Examples
+    --------
     >>> scoreAndLabels = sc.parallelize([
     ...     (0.1, 0.0), (0.1, 1.0), (0.4, 0.0), (0.6, 0.0), (0.6, 1.0), (0.6, 1.0), (0.8, 1.0)], 2)
     >>> metrics = BinaryClassificationMetrics(scoreAndLabels)
@@ -48,8 +55,6 @@ class BinaryClassificationMetrics(JavaModelWrapper):
     0.79...
     >>> metrics.areaUnderPR
     0.88...
-
-    .. versionadded:: 1.4.0
     """
 
     def __init__(self, scoreAndLabels):
@@ -95,8 +100,15 @@ class RegressionMetrics(JavaModelWrapper):
     """
     Evaluator for regression.
 
-    :param predictionAndObservations: an RDD of prediction, observation and optional weight.
+    .. versionadded:: 1.4.0
+
+    Parameters
+    ----------
+    predictionAndObservations : :py:class:`pyspark.RDD`
+        an RDD of prediction, observation and optional weight.
 
+    Examples
+    --------
     >>> predictionAndObservations = sc.parallelize([
     ...     (2.5, 3.0), (0.0, -0.5), (2.0, 2.0), (8.0, 7.0)])
     >>> metrics = RegressionMetrics(predictionAndObservations)
@@ -115,8 +127,6 @@ class RegressionMetrics(JavaModelWrapper):
     >>> metrics = RegressionMetrics(predictionAndObservationsWithOptWeight)
     >>> metrics.rootMeanSquaredError
     0.68...
-
-    .. versionadded:: 1.4.0
     """
 
     def __init__(self, predictionAndObservations):
@@ -182,9 +192,15 @@ class MulticlassMetrics(JavaModelWrapper):
     """
     Evaluator for multiclass classification.
 
-    :param predictionAndLabels: an RDD of prediction, label, optional weight
-     and optional probability.
+    .. versionadded:: 1.4.0
+
+    Parameters
+    ----------
+    predictionAndLabels : :py:class:`pyspark.RDD`
+        an RDD of prediction, label, optional weight and optional probability.
 
+    Examples
+    --------
     >>> predictionAndLabels = sc.parallelize([(0.0, 0.0), (0.0, 1.0), (0.0, 0.0),
     ...     (1.0, 0.0), (1.0, 1.0), (1.0, 1.0), (1.0, 1.0), (2.0, 2.0), (2.0, 0.0)])
     >>> metrics = MulticlassMetrics(predictionAndLabels)
@@ -246,8 +262,6 @@ class MulticlassMetrics(JavaModelWrapper):
     >>> metrics = MulticlassMetrics(predictionAndLabelsWithProbabilities)
     >>> metrics.logLoss()
     0.9682...
-
-    .. versionadded:: 1.4.0
     """
 
     def __init__(self, predictionAndLabels):
@@ -377,9 +391,15 @@ class RankingMetrics(JavaModelWrapper):
     """
     Evaluator for ranking algorithms.
 
-    :param predictionAndLabels: an RDD of (predicted ranking,
-                                ground truth set) pairs.
+    .. versionadded:: 1.4.0
 
+    Parameters
+    ----------
+    predictionAndLabels : :py:class:`pyspark.RDD`
+        an RDD of (predicted ranking, ground truth set) pairs.
+
+    Examples
+    --------
     >>> predictionAndLabels = sc.parallelize([
     ...     ([1, 6, 2, 7, 8, 3, 9, 10, 4, 5], [1, 2, 3, 4, 5]),
     ...     ([4, 1, 5, 6, 2, 7, 3, 8, 9, 10], [1, 2, 3]),
@@ -407,8 +427,6 @@ class RankingMetrics(JavaModelWrapper):
     0.35...
     >>> metrics.recallAt(15)
     0.66...
-
-    .. versionadded:: 1.4.0
     """
 
     def __init__(self, predictionAndLabels):
@@ -439,7 +457,7 @@ def meanAveragePrecision(self):
         """
         Returns the mean average precision (MAP) of all the queries.
         If a query has an empty ground truth set, the average precision will be zero and
-        a log warining is generated.
+        a log warning is generated.
         """
         return self.call("meanAveragePrecision")
 
@@ -448,7 +466,7 @@ def meanAveragePrecisionAt(self, k):
         """
         Returns the mean average precision (MAP) at first k ranking of all the queries.
         If a query has an empty ground truth set, the average precision will be zero and
-        a log warining is generated.
+        a log warning is generated.
         """
         return self.call("meanAveragePrecisionAt", int(k))
 
@@ -484,10 +502,16 @@ class MultilabelMetrics(JavaModelWrapper):
     """
     Evaluator for multilabel classification.
 
-    :param predictionAndLabels: an RDD of (predictions, labels) pairs,
-                                both are non-null Arrays, each with
-                                unique elements.
+    .. versionadded:: 1.4.0
+
+    Parameters
+    ----------
+    predictionAndLabels : :py:class:`pyspark.RDD`
+        an RDD of (predictions, labels) pairs,
+        both are non-null Arrays, each with unique elements.
 
+    Examples
+    --------
     >>> predictionAndLabels = sc.parallelize([([0.0, 1.0], [0.0, 2.0]), ([0.0, 2.0], [0.0, 1.0]),
     ...     ([], [0.0]), ([2.0], [2.0]), ([2.0, 0.0], [2.0, 0.0]),
     ...     ([0.0, 1.0, 2.0], [0.0, 1.0]), ([1.0], [1.0, 2.0])])
@@ -516,8 +540,6 @@ class MultilabelMetrics(JavaModelWrapper):
     0.28...
     >>> metrics.accuracy
     0.54...
-
-    .. versionadded:: 1.4.0
     """
 
     def __init__(self, predictionAndLabels):
diff --git a/python/pyspark/mllib/evaluation.pyi b/python/pyspark/mllib/evaluation.pyi
new file mode 100644
index 0000000000000..03583784f0c3b
--- /dev/null
+++ b/python/pyspark/mllib/evaluation.pyi
@@ -0,0 +1,94 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import List, Optional, Tuple, TypeVar
+from pyspark.rdd import RDD
+from pyspark.mllib.common import JavaModelWrapper
+from pyspark.mllib.linalg import Matrix
+
+T = TypeVar("T")
+
+class BinaryClassificationMetrics(JavaModelWrapper):
+    def __init__(self, scoreAndLabels: RDD[Tuple[float, float]]) -> None: ...
+    @property
+    def areaUnderROC(self) -> float: ...
+    @property
+    def areaUnderPR(self) -> float: ...
+    def unpersist(self) -> None: ...
+
+class RegressionMetrics(JavaModelWrapper):
+    def __init__(self, predictionAndObservations: RDD[Tuple[float, float]]) -> None: ...
+    @property
+    def explainedVariance(self) -> float: ...
+    @property
+    def meanAbsoluteError(self) -> float: ...
+    @property
+    def meanSquaredError(self) -> float: ...
+    @property
+    def rootMeanSquaredError(self) -> float: ...
+    @property
+    def r2(self) -> float: ...
+
+class MulticlassMetrics(JavaModelWrapper):
+    def __init__(self, predictionAndLabels: RDD[Tuple[float, float]]) -> None: ...
+    def confusionMatrix(self) -> Matrix: ...
+    def truePositiveRate(self, label: float) -> float: ...
+    def falsePositiveRate(self, label: float) -> float: ...
+    def precision(self, label: float = ...) -> float: ...
+    def recall(self, label: float = ...) -> float: ...
+    def fMeasure(self, label: float = ..., beta: Optional[float] = ...) -> float: ...
+    @property
+    def accuracy(self) -> float: ...
+    @property
+    def weightedTruePositiveRate(self) -> float: ...
+    @property
+    def weightedFalsePositiveRate(self) -> float: ...
+    @property
+    def weightedRecall(self) -> float: ...
+    @property
+    def weightedPrecision(self) -> float: ...
+    def weightedFMeasure(self, beta: Optional[float] = ...) -> float: ...
+
+class RankingMetrics(JavaModelWrapper):
+    def __init__(self, predictionAndLabels: RDD[Tuple[List[T], List[T]]]) -> None: ...
+    def precisionAt(self, k: int) -> float: ...
+    @property
+    def meanAveragePrecision(self) -> float: ...
+    def meanAveragePrecisionAt(self, k: int) -> float: ...
+    def ndcgAt(self, k: int) -> float: ...
+    def recallAt(self, k: int) -> float: ...
+
+class MultilabelMetrics(JavaModelWrapper):
+    def __init__(
+        self, predictionAndLabels: RDD[Tuple[List[float], List[float]]]
+    ) -> None: ...
+    def precision(self, label: Optional[float] = ...) -> float: ...
+    def recall(self, label: Optional[float] = ...) -> float: ...
+    def f1Measure(self, label: Optional[float] = ...) -> float: ...
+    @property
+    def microPrecision(self) -> float: ...
+    @property
+    def microRecall(self) -> float: ...
+    @property
+    def microF1Measure(self) -> float: ...
+    @property
+    def hammingLoss(self) -> float: ...
+    @property
+    def subsetAccuracy(self) -> float: ...
+    @property
+    def accuracy(self) -> float: ...
diff --git a/python/pyspark/mllib/feature.py b/python/pyspark/mllib/feature.py
index 3efae6ff0ecc3..1d37ab815655b 100644
--- a/python/pyspark/mllib/feature.py
+++ b/python/pyspark/mllib/feature.py
@@ -18,22 +18,14 @@
 """
 Python package for feature in MLlib.
 """
-from __future__ import absolute_import
-
 import sys
 import warnings
-if sys.version >= '3':
-    basestring = str
-    unicode = str
-
 from py4j.protocol import Py4JJavaError
 
 from pyspark import since
-from pyspark.rdd import RDD, ignore_unicode_prefix
+from pyspark.rdd import RDD
 from pyspark.mllib.common import callMLlibFunc, JavaModelWrapper
-from pyspark.mllib.linalg import (
-    Vector, Vectors, DenseVector, SparseVector, _convert_to_vector)
-from pyspark.mllib.regression import LabeledPoint
+from pyspark.mllib.linalg import Vectors, _convert_to_vector
 from pyspark.mllib.util import JavaLoader, JavaSaveable
 
 __all__ = ['Normalizer', 'StandardScalerModel', 'StandardScaler',
@@ -49,7 +41,10 @@ def transform(self, vector):
         """
         Applies transformation on a vector.
 
-        :param vector: vector to be transformed.
+        Parameters
+        ----------
+        vector : :py:class:`pyspark.mllib.linalg.Vector` or :py:class:`pyspark.RDD`
+            vector or convertible or RDD to be transformed.
         """
         raise NotImplementedError
 
@@ -64,8 +59,16 @@ class Normalizer(VectorTransformer):
     For `p` = float('inf'), max(abs(vector)) will be used as norm for
     normalization.
 
-    :param p: Normalization in L^p^ space, p = 2 by default.
+    .. versionadded:: 1.2.0
 
+    Parameters
+    ----------
+    p : float, optional
+        Normalization in L^p^ space, p = 2 by default.
+
+    Examples
+    --------
+    >>> from pyspark.mllib.linalg import Vectors
     >>> v = Vectors.dense(range(3))
     >>> nor = Normalizer(1)
     >>> nor.transform(v)
@@ -78,21 +81,27 @@ class Normalizer(VectorTransformer):
     >>> nor2 = Normalizer(float("inf"))
     >>> nor2.transform(v)
     DenseVector([0.0, 0.5, 1.0])
-
-    .. versionadded:: 1.2.0
     """
     def __init__(self, p=2.0):
         assert p >= 1.0, "p should be greater than 1.0"
         self.p = float(p)
 
-    @since('1.2.0')
     def transform(self, vector):
         """
         Applies unit length normalization on a vector.
 
-        :param vector: vector or RDD of vector to be normalized.
-        :return: normalized vector. If the norm of the input is zero, it
-                 will return the input vector.
+        .. versionadded:: 1.2.0
+
+        Parameters
+        ----------
+        vector : :py:class:`pyspark.mllib.linalg.Vector` or :py:class:`pyspark.RDD`
+            vector or RDD of vector to be normalized.
+
+        Returns
+        -------
+        :py:class:`pyspark.mllib.linalg.Vector` or :py:class:`pyspark.RDD`
+            normalized vector(s). If the norm of the input is zero, it
+            will return the input vector.
         """
         if isinstance(vector, RDD):
             vector = vector.map(_convert_to_vector)
@@ -110,11 +119,16 @@ def transform(self, vector):
         """
         Applies transformation on a vector or an RDD[Vector].
 
-        .. note:: In Python, transform cannot currently be used within
-            an RDD transformation or action.
-            Call transform directly on the RDD instead.
+        Parameters
+        ----------
+        vector : :py:class:`pyspark.mllib.linalg.Vector` or :py:class:`pyspark.RDD`
+            Input vector(s) to be transformed.
 
-        :param vector: Vector or RDD of Vector to be transformed.
+        Notes
+        -----
+        In Python, transform cannot currently be used within
+        an RDD transformation or action.
+        Call transform directly on the RDD instead.
         """
         if isinstance(vector, RDD):
             vector = vector.map(_convert_to_vector)
@@ -130,19 +144,29 @@ class StandardScalerModel(JavaVectorTransformer):
     .. versionadded:: 1.2.0
     """
 
-    @since('1.2.0')
     def transform(self, vector):
         """
         Applies standardization transformation on a vector.
 
-        .. note:: In Python, transform cannot currently be used within
-            an RDD transformation or action.
-            Call transform directly on the RDD instead.
+        .. versionadded:: 1.2.0
+
+        Parameters
+        ----------
+        vector : :py:class:`pyspark.mllib.linalg.Vector` or :py:class:`pyspark.RDD`
+            Input vector(s) to be standardized.
+
+        Returns
+        -------
+        :py:class:`pyspark.mllib.linalg.Vector` or :py:class:`pyspark.RDD`
+            Standardized vector(s). If the variance of a column is
+            zero, it will return default `0.0` for the column with
+            zero variance.
 
-        :param vector: Vector or RDD of Vector to be standardized.
-        :return: Standardized vector. If the variance of a column is
-                 zero, it will return default `0.0` for the column with
-                 zero variance.
+        Notes
+        -----
+        In Python, transform cannot currently be used within
+        an RDD transformation or action.
+        Call transform directly on the RDD instead.
         """
         return JavaVectorTransformer.transform(self, vector)
 
@@ -203,12 +227,20 @@ class StandardScaler(object):
     variance using column summary statistics on the samples in the
     training set.
 
-    :param withMean: False by default. Centers the data with mean
-                     before scaling. It will build a dense output, so take
-                     care when applying to sparse input.
-    :param withStd: True by default. Scales the data to unit
-                    standard deviation.
+    .. versionadded:: 1.2.0
 
+    Parameters
+    ----------
+    withMean : bool, optional
+        False by default. Centers the data with mean
+        before scaling. It will build a dense output, so take
+        care when applying to sparse input.
+    withStd : bool, optional
+        True by default. Scales the data to unit
+        standard deviation.
+
+    Examples
+    --------
     >>> vs = [Vectors.dense([-2.0, 2.3, 0]), Vectors.dense([3.8, 0.0, 1.9])]
     >>> dataset = sc.parallelize(vs)
     >>> standardizer = StandardScaler(True, True)
@@ -225,8 +257,6 @@ class StandardScaler(object):
     True
     >>> model.withMean
     True
-
-    .. versionadded:: 1.2.0
     """
     def __init__(self, withMean=False, withStd=True):
         if not (withMean or withStd):
@@ -234,15 +264,22 @@ def __init__(self, withMean=False, withStd=True):
         self.withMean = withMean
         self.withStd = withStd
 
-    @since('1.2.0')
     def fit(self, dataset):
         """
         Computes the mean and variance and stores as a model to be used
         for later scaling.
 
-        :param dataset: The data used to compute the mean and variance
-                     to build the transformation model.
-        :return: a StandardScalarModel
+        .. versionadded:: 1.2.0
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.RDD`
+            The data used to compute the mean and variance
+            to build the transformation model.
+
+        Returns
+        -------
+        :py:class:`StandardScalerModel`
         """
         dataset = dataset.map(_convert_to_vector)
         jmodel = callMLlibFunc("fitStandardScaler", self.withMean, self.withStd, dataset)
@@ -256,13 +293,21 @@ class ChiSqSelectorModel(JavaVectorTransformer):
     .. versionadded:: 1.4.0
     """
 
-    @since('1.4.0')
     def transform(self, vector):
         """
         Applies transformation on a vector.
 
-        :param vector: Vector or RDD of Vector to be transformed.
-        :return: transformed vector.
+        .. versionadded:: 1.4.0
+
+        Examples
+        --------
+        vector : :py:class:`pyspark.mllib.linalg.Vector` or :py:class:`pyspark.RDD`
+            Input vector(s) to be transformed.
+
+        Returns
+        -------
+        :py:class:`pyspark.mllib.linalg.Vector` or :py:class:`pyspark.RDD`
+            transformed vector(s).
         """
         return JavaVectorTransformer.transform(self, vector)
 
@@ -291,6 +336,12 @@ class ChiSqSelector(object):
     By default, the selection method is `numTopFeatures`, with the default number of top features
     set to 50.
 
+    .. versionadded:: 1.4.0
+
+    Examples
+    --------
+    >>> from pyspark.mllib.linalg import SparseVector, DenseVector
+    >>> from pyspark.mllib.regression import LabeledPoint
     >>> data = sc.parallelize([
     ...     LabeledPoint(0.0, SparseVector(3, {0: 8.0, 1: 7.0})),
     ...     LabeledPoint(1.0, SparseVector(3, {1: 9.0, 2: 6.0})),
@@ -311,8 +362,6 @@ class ChiSqSelector(object):
     >>> model = ChiSqSelector(selectorType="percentile", percentile=0.34).fit(data)
     >>> model.transform(DenseVector([7.0, 9.0, 5.0]))
     DenseVector([7.0])
-
-    .. versionadded:: 1.4.0
     """
     def __init__(self, numTopFeatures=50, selectorType="numTopFeatures", percentile=0.1, fpr=0.05,
                  fdr=0.05, fwe=0.05):
@@ -377,15 +426,18 @@ def setSelectorType(self, selectorType):
         self.selectorType = str(selectorType)
         return self
 
-    @since('1.4.0')
     def fit(self, data):
         """
         Returns a ChiSquared feature selector.
 
-        :param data: an `RDD[LabeledPoint]` containing the labeled dataset
-                     with categorical features. Real-valued features will be
-                     treated as categorical for each distinct value.
-                     Apply feature discretizer before using this function.
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        data : :py:class:`pyspark.RDD` of :py:class:`pyspark.mllib.regression.LabeledPoint`
+            containing the labeled dataset with categorical features.
+            Real-valued features will be treated as categorical for each
+            distinct value. Apply feature discretizer before using this function.
         """
         jmodel = callMLlibFunc("fitChiSqSelector", self.selectorType, self.numTopFeatures,
                                self.percentile, self.fpr, self.fdr, self.fwe, data)
@@ -404,6 +456,10 @@ class PCA(object):
     """
     A feature transformer that projects vectors to a low-dimensional space using PCA.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> data = [Vectors.sparse(5, [(1, 1.0), (3, 7.0)]),
     ...     Vectors.dense([2.0, 0.0, 3.0, 4.0, 5.0]),
     ...     Vectors.dense([4.0, 0.0, 0.0, 6.0, 7.0])]
@@ -413,20 +469,26 @@ class PCA(object):
     1.648...
     >>> pcArray[1]
     -4.013...
-
-    .. versionadded:: 1.5.0
     """
     def __init__(self, k):
         """
-        :param k: number of principal components.
+        Parameters
+        ----------
+        k : int
+            number of principal components.
         """
         self.k = int(k)
 
-    @since('1.5.0')
     def fit(self, data):
         """
         Computes a [[PCAModel]] that contains the principal components of the input vectors.
-        :param data: source vectors
+
+        .. versionadded:: 1.5.0
+
+        Parameters
+        ----------
+        data : :py:class:`pyspark.RDD`
+            source vectors
         """
         jmodel = callMLlibFunc("fitPCA", self.k, data)
         return PCAModel(jmodel)
@@ -437,16 +499,23 @@ class HashingTF(object):
     Maps a sequence of terms to their term frequencies using the hashing
     trick.
 
-    .. note:: The terms must be hashable (can not be dict/set/list...).
+    .. versionadded:: 1.2.0
+
+    Parameters
+    ----------
+    numFeatures : int, optional
+        number of features (default: 2^20)
 
-    :param numFeatures: number of features (default: 2^20)
+    Notes
+    -----
+    The terms must be hashable (can not be dict/set/list...).
 
+    Examples
+    --------
     >>> htf = HashingTF(100)
     >>> doc = "a a b b c d".split(" ")
     >>> htf.transform(doc)
     SparseVector(100, {...})
-
-    .. versionadded:: 1.2.0
     """
     def __init__(self, numFeatures=1 << 20):
         self.numFeatures = numFeatures
@@ -490,7 +559,7 @@ class IDFModel(JavaVectorTransformer):
 
     .. versionadded:: 1.2.0
     """
-    @since('1.2.0')
+
     def transform(self, x):
         """
         Transforms term frequency (TF) vectors to TF-IDF vectors.
@@ -499,13 +568,24 @@ def transform(self, x):
         the terms which occur in fewer than `minDocFreq`
         documents will have an entry of 0.
 
-        .. note:: In Python, transform cannot currently be used within
-            an RDD transformation or action.
-            Call transform directly on the RDD instead.
+        .. versionadded:: 1.2.0
 
-        :param x: an RDD of term frequency vectors or a term frequency
-                  vector
-        :return: an RDD of TF-IDF vectors or a TF-IDF vector
+        Parameters
+        ----------
+        x : :py:class:`pyspark.mllib.linalg.Vector` or :py:class:`pyspark.RDD`
+            an RDD of term frequency vectors or a term frequency
+            vector
+
+        Returns
+        -------
+        :py:class:`pyspark.mllib.linalg.Vector` or :py:class:`pyspark.RDD`
+            an RDD of TF-IDF vectors or a TF-IDF vector
+
+        Notes
+        -----
+        In Python, transform cannot currently be used within
+        an RDD transformation or action.
+        Call transform directly on the RDD instead.
         """
         return JavaVectorTransformer.transform(self, x)
 
@@ -544,9 +624,15 @@ class IDF(object):
     `minDocFreq`). For terms that are not in at least `minDocFreq`
     documents, the IDF is found as 0, resulting in TF-IDFs of 0.
 
-    :param minDocFreq: minimum of documents in which a term
-                       should appear for filtering
+    .. versionadded:: 1.2.0
+
+    Parameters
+    ----------
+    minDocFreq : int
+        minimum of documents in which a term should appear for filtering
 
+    Examples
+    --------
     >>> n = 4
     >>> freqs = [Vectors.sparse(n, (1, 3), (1.0, 2.0)),
     ...          Vectors.dense([0.0, 1.0, 2.0, 3.0]),
@@ -565,18 +651,20 @@ class IDF(object):
     DenseVector([0.0, 0.0, 1.3863, 0.863])
     >>> model.transform(Vectors.sparse(n, (1, 3), (1.0, 2.0)))
     SparseVector(4, {1: 0.0, 3: 0.5754})
-
-    .. versionadded:: 1.2.0
     """
     def __init__(self, minDocFreq=0):
         self.minDocFreq = minDocFreq
 
-    @since('1.2.0')
     def fit(self, dataset):
         """
         Computes the inverse document frequency.
 
-        :param dataset: an RDD of term frequency vectors
+        .. versionadded:: 1.2.0
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.RDD`
+            an RDD of term frequency vectors
         """
         if not isinstance(dataset, RDD):
             raise TypeError("dataset should be an RDD of term frequency vectors")
@@ -587,36 +675,57 @@ def fit(self, dataset):
 class Word2VecModel(JavaVectorTransformer, JavaSaveable, JavaLoader):
     """
     class for Word2Vec model
-
-    .. versionadded:: 1.2.0
     """
-    @since('1.2.0')
+
     def transform(self, word):
         """
         Transforms a word to its vector representation
 
-        .. note:: Local use only
+        .. versionadded:: 1.2.0
+
+        Parameters
+        ----------
+        word : str
+            a word
 
-        :param word: a word
-        :return: vector representation of word(s)
+        Returns
+        -------
+        :py:class:`pyspark.mllib.linalg.Vector`
+            vector representation of word(s)
+
+        Notes
+        -----
+        Local use only
         """
         try:
             return self.call("transform", word)
         except Py4JJavaError:
             raise ValueError("%s not found" % word)
 
-    @since('1.2.0')
     def findSynonyms(self, word, num):
         """
         Find synonyms of a word
 
-        :param word: a word or a vector representation of word
-        :param num: number of synonyms to find
-        :return: array of (word, cosineSimilarity)
+        .. versionadded:: 1.2.0
+
+        Parameters
+        ----------
+
+        word : str or  :py:class:`pyspark.mllib.linalg.Vector`
+            a word or a vector representation of word
+        num : int
+            number of synonyms to find
+
+        Returns
+        -------
+        :py:class:`collections.abc.Iterable`
+            array of (word, cosineSimilarity)
 
-        .. note:: Local use only
+        Notes
+        -----
+        Local use only
         """
-        if not isinstance(word, basestring):
+        if not isinstance(word, str):
             word = _convert_to_vector(word)
         words, similarity = self.call("findSynonyms", word, num)
         return zip(words, similarity)
@@ -640,7 +749,6 @@ def load(cls, sc, path):
         return Word2VecModel(model)
 
 
-@ignore_unicode_prefix
 class Word2Vec(object):
     """Word2Vec creates vector representation of words in a text corpus.
     The algorithm first constructs a vocabulary from the corpus
@@ -659,6 +767,10 @@ class Word2Vec(object):
     and Distributed Representations of Words and Phrases and their
     Compositionality.
 
+    .. versionadded:: 1.2.0
+
+    Examples
+    --------
     >>> sentence = "a b " * 100 + "a c " * 10
     >>> localDoc = [sentence, sentence]
     >>> doc = sc.parallelize(localDoc).map(lambda line: line.split(" "))
@@ -668,7 +780,7 @@ class Word2Vec(object):
 
     >>> syms = model.findSynonyms("a", 2)
     >>> [s[0] for s in syms]
-    [u'b', u'c']
+    ['b', 'c']
 
     But querying for synonyms of a vector may return the word whose
     representation is that vector:
@@ -676,7 +788,7 @@ class Word2Vec(object):
     >>> vec = model.transform("a")
     >>> syms = model.findSynonyms(vec, 2)
     >>> [s[0] for s in syms]
-    [u'a', u'b']
+    ['a', 'b']
 
     >>> import os, tempfile
     >>> path = tempfile.mkdtemp()
@@ -686,15 +798,12 @@ class Word2Vec(object):
     True
     >>> syms = sameModel.findSynonyms("a", 2)
     >>> [s[0] for s in syms]
-    [u'b', u'c']
+    ['b', 'c']
     >>> from shutil import rmtree
     >>> try:
     ...     rmtree(path)
     ... except OSError:
     ...     pass
-
-    .. versionadded:: 1.2.0
-
     """
     def __init__(self):
         """
@@ -767,13 +876,20 @@ def setWindowSize(self, windowSize):
         self.windowSize = windowSize
         return self
 
-    @since('1.2.0')
     def fit(self, data):
         """
         Computes the vector representation of each word in vocabulary.
 
-        :param data: training data. RDD of list of string
-        :return: Word2VecModel instance
+        .. versionadded:: 1.2.0
+
+        Parameters
+        ----------
+        data : :py:class:`pyspark.RDD`
+            training data. RDD of list of string
+
+        Returns
+        -------
+        :py:class:`Word2VecModel`
         """
         if not isinstance(data, RDD):
             raise TypeError("data should be an RDD of list of string")
@@ -789,6 +905,10 @@ class ElementwiseProduct(VectorTransformer):
     Scales each column of the vector, with the supplied weight vector.
     i.e the elementwise product.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> weight = Vectors.dense([1.0, 2.0, 3.0])
     >>> eprod = ElementwiseProduct(weight)
     >>> a = Vectors.dense([2.0, 1.0, 3.0])
@@ -798,8 +918,6 @@ class ElementwiseProduct(VectorTransformer):
     >>> rdd = sc.parallelize([a, b])
     >>> eprod.transform(rdd).collect()
     [DenseVector([2.0, 2.0, 9.0]), DenseVector([9.0, 6.0, 12.0])]
-
-    .. versionadded:: 1.5.0
     """
     def __init__(self, scalingVector):
         self.scalingVector = _convert_to_vector(scalingVector)
diff --git a/python/pyspark/mllib/feature.pyi b/python/pyspark/mllib/feature.pyi
new file mode 100644
index 0000000000000..24a46f6bee798
--- /dev/null
+++ b/python/pyspark/mllib/feature.pyi
@@ -0,0 +1,167 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import overload
+from typing import Iterable, Hashable, List, Tuple, Union
+
+from pyspark.mllib._typing import VectorLike
+from pyspark.context import SparkContext
+from pyspark.rdd import RDD
+from pyspark.mllib.common import JavaModelWrapper
+from pyspark.mllib.linalg import Vector
+from pyspark.mllib.regression import LabeledPoint
+from pyspark.mllib.util import JavaLoader, JavaSaveable
+
+from py4j.java_collections import JavaMap  # type: ignore[import]
+
+class VectorTransformer:
+    @overload
+    def transform(self, vector: VectorLike) -> Vector: ...
+    @overload
+    def transform(self, vector: RDD[VectorLike]) -> RDD[Vector]: ...
+
+class Normalizer(VectorTransformer):
+    p: float
+    def __init__(self, p: float = ...) -> None: ...
+    @overload
+    def transform(self, vector: VectorLike) -> Vector: ...
+    @overload
+    def transform(self, vector: RDD[VectorLike]) -> RDD[Vector]: ...
+
+class JavaVectorTransformer(JavaModelWrapper, VectorTransformer):
+    @overload
+    def transform(self, vector: VectorLike) -> Vector: ...
+    @overload
+    def transform(self, vector: RDD[VectorLike]) -> RDD[Vector]: ...
+
+class StandardScalerModel(JavaVectorTransformer):
+    @overload
+    def transform(self, vector: VectorLike) -> Vector: ...
+    @overload
+    def transform(self, vector: RDD[VectorLike]) -> RDD[Vector]: ...
+    def setWithMean(self, withMean: bool) -> StandardScalerModel: ...
+    def setWithStd(self, withStd: bool) -> StandardScalerModel: ...
+    @property
+    def withStd(self) -> bool: ...
+    @property
+    def withMean(self) -> bool: ...
+    @property
+    def std(self) -> Vector: ...
+    @property
+    def mean(self) -> Vector: ...
+
+class StandardScaler:
+    withMean: bool
+    withStd: bool
+    def __init__(self, withMean: bool = ..., withStd: bool = ...) -> None: ...
+    def fit(self, dataset: RDD[VectorLike]) -> StandardScalerModel: ...
+
+class ChiSqSelectorModel(JavaVectorTransformer):
+    @overload
+    def transform(self, vector: VectorLike) -> Vector: ...
+    @overload
+    def transform(self, vector: RDD[VectorLike]) -> RDD[Vector]: ...
+
+class ChiSqSelector:
+    numTopFeatures: int
+    selectorType: str
+    percentile: float
+    fpr: float
+    fdr: float
+    fwe: float
+    def __init__(
+        self,
+        numTopFeatures: int = ...,
+        selectorType: str = ...,
+        percentile: float = ...,
+        fpr: float = ...,
+        fdr: float = ...,
+        fwe: float = ...,
+    ) -> None: ...
+    def setNumTopFeatures(self, numTopFeatures: int) -> ChiSqSelector: ...
+    def setPercentile(self, percentile: float) -> ChiSqSelector: ...
+    def setFpr(self, fpr: float) -> ChiSqSelector: ...
+    def setFdr(self, fdr: float) -> ChiSqSelector: ...
+    def setFwe(self, fwe: float) -> ChiSqSelector: ...
+    def setSelectorType(self, selectorType: str) -> ChiSqSelector: ...
+    def fit(self, data: RDD[LabeledPoint]) -> ChiSqSelectorModel: ...
+
+class PCAModel(JavaVectorTransformer): ...
+
+class PCA:
+    k: int
+    def __init__(self, k: int) -> None: ...
+    def fit(self, data: RDD[VectorLike]) -> PCAModel: ...
+
+class HashingTF:
+    numFeatures: int
+    binary: bool
+    def __init__(self, numFeatures: int = ...) -> None: ...
+    def setBinary(self, value: bool) -> HashingTF: ...
+    def indexOf(self, term: Hashable) -> int: ...
+    @overload
+    def transform(self, document: Iterable[Hashable]) -> Vector: ...
+    @overload
+    def transform(self, document: RDD[Iterable[Hashable]]) -> RDD[Vector]: ...
+
+class IDFModel(JavaVectorTransformer):
+    @overload
+    def transform(self, x: VectorLike) -> Vector: ...
+    @overload
+    def transform(self, x: RDD[VectorLike]) -> RDD[Vector]: ...
+    def idf(self) -> Vector: ...
+    def docFreq(self) -> List[int]: ...
+    def numDocs(self) -> int: ...
+
+class IDF:
+    minDocFreq: int
+    def __init__(self, minDocFreq: int = ...) -> None: ...
+    def fit(self, dataset: RDD[VectorLike]) -> IDFModel: ...
+
+class Word2VecModel(JavaVectorTransformer, JavaSaveable, JavaLoader[Word2VecModel]):
+    def transform(self, word: str) -> Vector: ...  # type: ignore
+    def findSynonyms(self, word: Union[str, VectorLike], num: int) -> Iterable[Tuple[str, float]]: ...
+    def getVectors(self) -> JavaMap: ...
+    @classmethod
+    def load(cls, sc: SparkContext, path: str) -> Word2VecModel: ...
+
+class Word2Vec:
+    vectorSize: int
+    learningRate: float
+    numPartitions: int
+    numIterations: int
+    seed: int
+    minCount: int
+    windowSize: int
+    def __init__(self) -> None: ...
+    def setVectorSize(self, vectorSize: int) -> Word2Vec: ...
+    def setLearningRate(self, learningRate: float) -> Word2Vec: ...
+    def setNumPartitions(self, numPartitions: int) -> Word2Vec: ...
+    def setNumIterations(self, numIterations: int) -> Word2Vec: ...
+    def setSeed(self, seed: int) -> Word2Vec: ...
+    def setMinCount(self, minCount: int) -> Word2Vec: ...
+    def setWindowSize(self, windowSize: int) -> Word2Vec: ...
+    def fit(self, data: RDD[List[str]]) -> Word2VecModel: ...
+
+class ElementwiseProduct(VectorTransformer):
+    scalingVector: Vector
+    def __init__(self, scalingVector: Vector) -> None: ...
+    @overload
+    def transform(self, vector: VectorLike) -> Vector: ...
+    @overload
+    def transform(self, vector: RDD[VectorLike]) -> RDD[Vector]: ...
diff --git a/python/pyspark/mllib/fpm.py b/python/pyspark/mllib/fpm.py
index 373a141456b2f..1f87a15cb11c9 100644
--- a/python/pyspark/mllib/fpm.py
+++ b/python/pyspark/mllib/fpm.py
@@ -20,7 +20,6 @@
 from collections import namedtuple
 
 from pyspark import since
-from pyspark.rdd import ignore_unicode_prefix
 from pyspark.mllib.common import JavaModelWrapper, callMLlibFunc
 from pyspark.mllib.util import JavaSaveable, JavaLoader, inherit_doc
 
@@ -28,24 +27,25 @@
 
 
 @inherit_doc
-@ignore_unicode_prefix
 class FPGrowthModel(JavaModelWrapper, JavaSaveable, JavaLoader):
     """
     A FP-Growth model for mining frequent itemsets
     using the Parallel FP-Growth algorithm.
 
+    .. versionadded:: 1.4.0
+
+    Examples
+    --------
     >>> data = [["a", "b", "c"], ["a", "b", "d", "e"], ["a", "c", "e"], ["a", "c", "f"]]
     >>> rdd = sc.parallelize(data, 2)
     >>> model = FPGrowth.train(rdd, 0.6, 2)
     >>> sorted(model.freqItemsets().collect())
-    [FreqItemset(items=[u'a'], freq=4), FreqItemset(items=[u'c'], freq=3), ...
+    [FreqItemset(items=['a'], freq=4), FreqItemset(items=['c'], freq=3), ...
     >>> model_path = temp_path + "/fpm"
     >>> model.save(sc, model_path)
     >>> sameModel = FPGrowthModel.load(sc, model_path)
     >>> sorted(model.freqItemsets().collect()) == sorted(sameModel.freqItemsets().collect())
     True
-
-    .. versionadded:: 1.4.0
     """
 
     @since("1.4.0")
@@ -74,20 +74,23 @@ class FPGrowth(object):
     """
 
     @classmethod
-    @since("1.4.0")
     def train(cls, data, minSupport=0.3, numPartitions=-1):
         """
         Computes an FP-Growth model that contains frequent itemsets.
 
-        :param data:
-          The input data set, each element contains a transaction.
-        :param minSupport:
-          The minimal support level.
-          (default: 0.3)
-        :param numPartitions:
-          The number of partitions used by parallel FP-growth. A value
-          of -1 will use the same number as input data.
-          (default: -1)
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        data : :py:class:`pyspark.RDD`
+            The input data set, each element contains a transaction.
+        minSupport : float, optional
+            The minimal support level.
+            (default: 0.3)
+        numPartitions : int, optional
+            The number of partitions used by parallel FP-growth. A value
+            of -1 will use the same number as input data.
+            (default: -1)
         """
         model = callMLlibFunc("trainFPGrowthModel", data, float(minSupport), int(numPartitions))
         return FPGrowthModel(model)
@@ -101,11 +104,14 @@ class FreqItemset(namedtuple("FreqItemset", ["items", "freq"])):
 
 
 @inherit_doc
-@ignore_unicode_prefix
 class PrefixSpanModel(JavaModelWrapper):
     """
     Model fitted by PrefixSpan
 
+    .. versionadded:: 1.6.0
+
+    Examples
+    --------
     >>> data = [
     ...    [["a", "b"], ["c"]],
     ...    [["a"], ["c", "b"], ["a", "b"]],
@@ -114,9 +120,7 @@ class PrefixSpanModel(JavaModelWrapper):
     >>> rdd = sc.parallelize(data, 2)
     >>> model = PrefixSpan.train(rdd)
     >>> sorted(model.freqSequences().collect())
-    [FreqSequence(sequence=[[u'a']], freq=3), FreqSequence(sequence=[[u'a'], [u'a']], freq=1), ...
-
-    .. versionadded:: 1.6.0
+    [FreqSequence(sequence=[['a']], freq=3), FreqSequence(sequence=[['a'], ['a']], freq=1), ...
     """
 
     @since("1.6.0")
@@ -128,38 +132,45 @@ def freqSequences(self):
 class PrefixSpan(object):
     """
     A parallel PrefixSpan algorithm to mine frequent sequential patterns.
-    The PrefixSpan algorithm is described in J. Pei, et al., PrefixSpan:
-    Mining Sequential Patterns Efficiently by Prefix-Projected Pattern Growth
-    ([[https://doi.org/10.1109/ICDE.2001.914830]]).
+    The PrefixSpan algorithm is described in Jian Pei et al (2001) [1]_
 
     .. versionadded:: 1.6.0
+
+    .. [1] Jian Pei et al.,
+        "PrefixSpan,: mining sequential patterns efficiently by prefix-projected pattern growth,"
+        Proceedings 17th International Conference on Data Engineering, Heidelberg,
+        Germany, 2001, pp. 215-224,
+        doi: https://doi.org/10.1109/ICDE.2001.914830
     """
 
     @classmethod
-    @since("1.6.0")
     def train(cls, data, minSupport=0.1, maxPatternLength=10, maxLocalProjDBSize=32000000):
         """
         Finds the complete set of frequent sequential patterns in the
         input sequences of itemsets.
 
-        :param data:
-          The input data set, each element contains a sequence of
-          itemsets.
-        :param minSupport:
-          The minimal support level of the sequential pattern, any
-          pattern that appears more than (minSupport *
-          size-of-the-dataset) times will be output.
-          (default: 0.1)
-        :param maxPatternLength:
-          The maximal length of the sequential pattern, any pattern
-          that appears less than maxPatternLength will be output.
-          (default: 10)
-        :param maxLocalProjDBSize:
-          The maximum number of items (including delimiters used in the
-          internal storage format) allowed in a projected database before
-          local processing. If a projected database exceeds this size,
-          another iteration of distributed prefix growth is run.
-          (default: 32000000)
+        .. versionadded:: 1.6.0
+
+        Parameters
+        ----------
+        data : :py:class:`pyspark.RDD`
+            The input data set, each element contains a sequence of
+            itemsets.
+        minSupport : float, optional
+            The minimal support level of the sequential pattern, any
+            pattern that appears more than (minSupport *
+            size-of-the-dataset) times will be output.
+            (default: 0.1)
+        maxPatternLength : int, optional
+            The maximal length of the sequential pattern, any pattern
+            that appears less than maxPatternLength will be output.
+            (default: 10)
+        maxLocalProjDBSize : int, optional
+            The maximum number of items (including delimiters used in the
+            internal storage format) allowed in a projected database before
+            local processing. If a projected database exceeds this size,
+            another iteration of distributed prefix growth is run.
+            (default: 32000000)
         """
         model = callMLlibFunc("trainPrefixSpanModel",
                               data, minSupport, maxPatternLength, maxLocalProjDBSize)
diff --git a/python/pyspark/mllib/fpm.pyi b/python/pyspark/mllib/fpm.pyi
new file mode 100644
index 0000000000000..c5a6b5f6806c0
--- /dev/null
+++ b/python/pyspark/mllib/fpm.pyi
@@ -0,0 +1,57 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Generic, List, TypeVar
+from pyspark.context import SparkContext
+from pyspark.rdd import RDD
+from pyspark.mllib.common import JavaModelWrapper
+from pyspark.mllib.util import JavaSaveable, JavaLoader
+
+T = TypeVar("T")
+
+class FPGrowthModel(
+    JavaModelWrapper, JavaSaveable, JavaLoader[FPGrowthModel], Generic[T]
+):
+    def freqItemsets(self) -> RDD[FPGrowth.FreqItemset[T]]: ...
+    @classmethod
+    def load(cls, sc: SparkContext, path: str) -> FPGrowthModel: ...
+
+class FPGrowth:
+    @classmethod
+    def train(
+        cls, data: RDD[List[T]], minSupport: float = ..., numPartitions: int = ...
+    ) -> FPGrowthModel[T]: ...
+    class FreqItemset(Generic[T]):
+        items: List[T]
+        freq: int
+
+class PrefixSpanModel(JavaModelWrapper, Generic[T]):
+    def freqSequences(self) -> RDD[PrefixSpan.FreqSequence[T]]: ...
+
+class PrefixSpan:
+    @classmethod
+    def train(
+        cls,
+        data: RDD[List[List[T]]],
+        minSupport: float = ...,
+        maxPatternLength: int = ...,
+        maxLocalProjDBSize: int = ...,
+    ) -> PrefixSpanModel[T]: ...
+    class FreqSequence(tuple, Generic[T]):
+        sequence: List[T]
+        freq: int
diff --git a/python/pyspark/mllib/linalg/__init__.py b/python/pyspark/mllib/linalg/__init__.py
index cd09621b13b56..f20004ab70ab3 100644
--- a/python/pyspark/mllib/linalg/__init__.py
+++ b/python/pyspark/mllib/linalg/__init__.py
@@ -27,15 +27,6 @@
 import array
 import struct
 
-if sys.version >= '3':
-    basestring = str
-    xrange = range
-    import copyreg as copy_reg
-    long = int
-else:
-    from itertools import izip as zip
-    import copy_reg
-
 import numpy as np
 
 from pyspark import since
@@ -49,13 +40,6 @@
            'QRDecomposition']
 
 
-if sys.version_info[:2] == (2, 7):
-    # speed up pickling array in Python 2.7
-    def fast_pickle_array(ar):
-        return array.array, (ar.typecode, ar.tostring())
-    copy_reg.pickle(array.array, fast_pickle_array)
-
-
 # Check whether we have SciPy. MLlib works without it too, but if we have it, some methods,
 # such as _dot and _serialize_double_vector, start to support scipy.sparse matrices.
 
@@ -70,7 +54,7 @@ def fast_pickle_array(ar):
 def _convert_to_vector(l):
     if isinstance(l, Vector):
         return l
-    elif type(l) in (array.array, np.array, np.ndarray, list, tuple, xrange):
+    elif type(l) in (array.array, np.array, np.ndarray, list, tuple, range):
         return DenseVector(l)
     elif _have_scipy and scipy.sparse.issparse(l):
         assert l.shape[1] == 1, "Expected column vector"
@@ -87,6 +71,8 @@ def _vector_size(v):
     """
     Returns the size of the vector.
 
+    Examples
+    --------
     >>> _vector_size([1., 2., 3.])
     3
     >>> _vector_size((1., 2., 3.))
@@ -104,7 +90,7 @@ def _vector_size(v):
     """
     if isinstance(v, Vector):
         return len(v)
-    elif type(v) in (array.array, list, tuple, xrange):
+    elif type(v) in (array.array, list, tuple, range):
         return len(v)
     elif type(v) == np.ndarray:
         if v.ndim == 1 or (v.ndim == 2 and v.shape[1] == 1):
@@ -247,7 +233,9 @@ def toArray(self):
         """
         Convert the vector into an numpy.ndarray
 
-        :return: numpy.ndarray
+        Returns
+        -------
+        :py:class:`numpy.ndarray`
         """
         raise NotImplementedError
 
@@ -256,7 +244,9 @@ def asML(self):
         Convert this vector to the new mllib-local representation.
         This does NOT copy the data; it copies references.
 
-        :return: :py:class:`pyspark.ml.linalg.Vector`
+        Returns
+        -------
+        :py:class:`pyspark.ml.linalg.Vector`
         """
         raise NotImplementedError
 
@@ -267,6 +257,8 @@ class DenseVector(Vector):
     storage and arithmetics will be delegated to the underlying numpy
     array.
 
+    Examples
+    --------
     >>> v = Vectors.dense([1.0, 2.0])
     >>> u = Vectors.dense([3.0, 4.0])
     >>> v + u
@@ -298,6 +290,8 @@ def parse(s):
         """
         Parse string representation back into the DenseVector.
 
+        Examples
+        --------
         >>> DenseVector.parse(' [ 0.0,1.0,2.0,  3.0]')
         DenseVector([0.0, 1.0, 2.0, 3.0])
         """
@@ -328,6 +322,8 @@ def norm(self, p):
         """
         Calculates the norm of a DenseVector.
 
+        Examples
+        --------
         >>> a = DenseVector([0, -1, 2, -3])
         >>> a.norm(2)
         3.7...
@@ -343,6 +339,8 @@ def dot(self, other):
         and a target NumPy array that is either 1- or 2-dimensional.
         Equivalent to calling numpy.dot of the two vectors.
 
+        Examples
+        --------
         >>> dense = DenseVector(array.array('d', [1., 2.]))
         >>> dense.dot(dense)
         5.0
@@ -383,6 +381,8 @@ def squared_distance(self, other):
         """
         Squared distance of two Vectors.
 
+        Examples
+        --------
         >>> dense1 = DenseVector(array.array('d', [1., 2.]))
         >>> dense1.squared_distance(dense1)
         0.0
@@ -428,9 +428,11 @@ def asML(self):
         Convert this vector to the new mllib-local representation.
         This does NOT copy the data; it copies references.
 
-        :return: :py:class:`pyspark.ml.linalg.DenseVector`
-
         .. versionadded:: 2.0.0
+
+        Returns
+        -------
+        :py:class:`pyspark.ml.linalg.DenseVector`
         """
         return newlinalg.DenseVector(self.array)
 
@@ -459,7 +461,7 @@ def __eq__(self, other):
         elif isinstance(other, SparseVector):
             if len(self) != other.size:
                 return False
-            return Vectors._equals(list(xrange(len(self))), self.array, other.indices, other.values)
+            return Vectors._equals(list(range(len(self))), self.array, other.indices, other.values)
         return False
 
     def __ne__(self, other):
@@ -517,12 +519,18 @@ def __init__(self, size, *args):
         (index, value) pairs, or two separate arrays of indices and
         values (sorted by index).
 
-        :param size: Size of the vector.
-        :param args: Active entries, as a dictionary {index: value, ...},
-          a list of tuples [(index, value), ...], or a list of strictly
-          increasing indices and a list of corresponding values [index, ...],
-          [value, ...]. Inactive entries are treated as zeros.
-
+        Parameters
+        ----------
+        size : int
+            Size of the vector.
+        args
+            Active entries, as a dictionary {index: value, ...},
+            a list of tuples [(index, value), ...], or a list of strictly
+            increasing indices and a list of corresponding values [index, ...],
+            [value, ...]. Inactive entries are treated as zeros.
+
+        Examples
+        --------
         >>> SparseVector(4, {1: 1.0, 3: 5.5})
         SparseVector(4, {1: 1.0, 3: 5.5})
         >>> SparseVector(4, [(1, 1.0), (3, 5.5)])
@@ -556,7 +564,7 @@ def __init__(self, size, *args):
                 self.indices = np.array(args[0], dtype=np.int32)
                 self.values = np.array(args[1], dtype=np.float64)
             assert len(self.indices) == len(self.values), "index and value arrays not same length"
-            for i in xrange(len(self.indices) - 1):
+            for i in range(len(self.indices) - 1):
                 if self.indices[i] >= self.indices[i + 1]:
                     raise TypeError(
                         "Indices %s and %s are not strictly increasing"
@@ -572,6 +580,8 @@ def norm(self, p):
         """
         Calculates the norm of a SparseVector.
 
+        Examples
+        --------
         >>> a = SparseVector(4, [0, 1], [3., -4.])
         >>> a.norm(1)
         7.0
@@ -590,6 +600,8 @@ def parse(s):
         """
         Parse string representation back into the SparseVector.
 
+        Examples
+        --------
         >>> SparseVector.parse(' (4, [0,1 ],[ 4.0,5.0] )')
         SparseVector(4, {0: 4.0, 1: 5.0})
         """
@@ -638,6 +650,8 @@ def dot(self, other):
         """
         Dot product with a SparseVector or 1- or 2-dimensional Numpy array.
 
+        Examples
+        --------
         >>> a = SparseVector(4, [1, 3], [3.0, 4.0])
         >>> a.dot(a)
         25.0
@@ -694,6 +708,8 @@ def squared_distance(self, other):
         """
         Squared distance from a SparseVector or 1-dimensional NumPy array.
 
+        Examples
+        --------
         >>> a = SparseVector(4, [1, 3], [3.0, 4.0])
         >>> a.squared_distance(a)
         0.0
@@ -770,9 +786,11 @@ def asML(self):
         Convert this vector to the new mllib-local representation.
         This does NOT copy the data; it copies references.
 
-        :return: :py:class:`pyspark.ml.linalg.SparseVector`
-
         .. versionadded:: 2.0.0
+
+        Returns
+        -------
+        :py:class:`pyspark.ml.linalg.SparseVector`
         """
         return newlinalg.SparseVector(self.size, self.indices, self.values)
 
@@ -788,7 +806,7 @@ def __repr__(self):
         inds = self.indices
         vals = self.values
         entries = ", ".join(["{0}: {1}".format(inds[i], _format_float(vals[i]))
-                             for i in xrange(len(inds))])
+                             for i in range(len(inds))])
         return "SparseVector({0}, {{{1}}})".format(self.size, entries)
 
     def __eq__(self, other):
@@ -798,7 +816,7 @@ def __eq__(self, other):
         elif isinstance(other, DenseVector):
             if self.size != len(other):
                 return False
-            return Vectors._equals(self.indices, self.values, list(xrange(len(other))), other.array)
+            return Vectors._equals(self.indices, self.values, list(range(len(other))), other.array)
         return False
 
     def __getitem__(self, index):
@@ -844,10 +862,12 @@ class Vectors(object):
     """
     Factory methods for working with vectors.
 
-    .. note:: Dense vectors are simply represented as NumPy array objects,
-        so there is no need to covert them for use in MLlib. For sparse vectors,
-        the factory methods in this class create an MLlib-compatible type, or users
-        can pass in SciPy's `scipy.sparse` column vectors.
+    Notes
+    -----
+    Dense vectors are simply represented as NumPy array objects,
+    so there is no need to covert them for use in MLlib. For sparse vectors,
+    the factory methods in this class create an MLlib-compatible type, or users
+    can pass in SciPy's `scipy.sparse` column vectors.
     """
 
     @staticmethod
@@ -857,10 +877,16 @@ def sparse(size, *args):
         (index, value) pairs, or two separate arrays of indices and
         values (sorted by index).
 
-        :param size: Size of the vector.
-        :param args: Non-zero entries, as a dictionary, list of tuples,
-                     or two sorted lists containing indices and values.
+        Parameters
+        ----------
+        size : int
+            Size of the vector.
+        args
+            Non-zero entries, as a dictionary, list of tuples,
+            or two sorted lists containing indices and values.
 
+        Examples
+        --------
         >>> Vectors.sparse(4, {1: 1.0, 3: 5.5})
         SparseVector(4, {1: 1.0, 3: 5.5})
         >>> Vectors.sparse(4, [(1, 1.0), (3, 5.5)])
@@ -875,12 +901,14 @@ def dense(*elements):
         """
         Create a dense vector of 64-bit floats from a Python list or numbers.
 
+        Examples
+        --------
         >>> Vectors.dense([1, 2, 3])
         DenseVector([1.0, 2.0, 3.0])
         >>> Vectors.dense(1.0, 2.0)
         DenseVector([1.0, 2.0])
         """
-        if len(elements) == 1 and not isinstance(elements[0], (float, int, long)):
+        if len(elements) == 1 and not isinstance(elements[0], (float, int)):
             # it's list, numpy.array or other iterable object.
             elements = elements[0]
         return DenseVector(elements)
@@ -891,10 +919,15 @@ def fromML(vec):
         Convert a vector from the new mllib-local representation.
         This does NOT copy the data; it copies references.
 
-        :param vec: a :py:class:`pyspark.ml.linalg.Vector`
-        :return: a :py:class:`pyspark.mllib.linalg.Vector`
-
         .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        vec : :py:class:`pyspark.ml.linalg.Vector`
+
+        Returns
+        -------
+        :py:class:`pyspark.mllib.linalg.Vector`
         """
         if isinstance(vec, newlinalg.DenseVector):
             return DenseVector(vec.array)
@@ -909,6 +942,8 @@ def stringify(vector):
         Converts a vector into a string, which can be recognized by
         Vectors.parse().
 
+        Examples
+        --------
         >>> Vectors.stringify(Vectors.sparse(2, [1], [1.0]))
         '(2,[1],[1.0])'
         >>> Vectors.stringify(Vectors.dense([0.0, 1.0]))
@@ -923,6 +958,8 @@ def squared_distance(v1, v2):
         a and b can be of type SparseVector, DenseVector, np.ndarray
         or array.array.
 
+        Examples
+        --------
         >>> a = Vectors.sparse(4, [(0, 1), (3, 4)])
         >>> b = Vectors.dense([2, 5, 4, 1])
         >>> a.squared_distance(b)
@@ -942,6 +979,8 @@ def norm(vector, p):
     def parse(s):
         """Parse a string representation back into the Vector.
 
+        Examples
+        --------
         >>> Vectors.parse('[2,1,2 ]')
         DenseVector([2.0, 1.0, 2.0])
         >>> Vectors.parse(' ( 100,  [0],  [2])')
@@ -1039,6 +1078,8 @@ def __str__(self):
         """
         Pretty printing of a DenseMatrix
 
+        Examples
+        --------
         >>> dm = DenseMatrix(2, 2, range(4))
         >>> print(dm)
         DenseMatrix([[ 0.,  2.],
@@ -1060,6 +1101,8 @@ def __repr__(self):
         """
         Representation of a DenseMatrix
 
+        Examples
+        --------
         >>> dm = DenseMatrix(2, 2, range(4))
         >>> dm
         DenseMatrix(2, 2, [0.0, 1.0, 2.0, 3.0], False)
@@ -1083,6 +1126,8 @@ def toArray(self):
         """
         Return an numpy.ndarray
 
+        Examples
+        --------
         >>> m = DenseMatrix(2, 2, range(4))
         >>> m.toArray()
         array([[ 0.,  2.],
@@ -1114,9 +1159,11 @@ def asML(self):
         Convert this matrix to the new mllib-local representation.
         This does NOT copy the data; it copies references.
 
-        :return: :py:class:`pyspark.ml.linalg.DenseMatrix`
-
         .. versionadded:: 2.0.0
+
+        Returns
+        -------
+        :py:class:`pyspark.ml.linalg.DenseMatrix`
         """
         return newlinalg.DenseMatrix(self.numRows, self.numCols, self.values, self.isTransposed)
 
@@ -1170,6 +1217,8 @@ def __str__(self):
         """
         Pretty printing of a SparseMatrix
 
+        Examples
+        --------
         >>> sm1 = SparseMatrix(2, 2, [0, 2, 3], [0, 1, 1], [2, 3, 4])
         >>> print(sm1)
         2 X 2 CSCMatrix
@@ -1216,6 +1265,8 @@ def __repr__(self):
         """
         Representation of a SparseMatrix
 
+        Examples
+        --------
         >>> sm1 = SparseMatrix(2, 2, [0, 2, 3], [0, 1, 1], [2, 3, 4])
         >>> sm1
         SparseMatrix(2, 2, [0, 2, 3], [0, 1, 1], [2.0, 3.0, 4.0], False)
@@ -1279,7 +1330,7 @@ def toArray(self):
         Return an numpy.ndarray
         """
         A = np.zeros((self.numRows, self.numCols), dtype=np.float64, order='F')
-        for k in xrange(self.colPtrs.size - 1):
+        for k in range(self.colPtrs.size - 1):
             startptr = self.colPtrs[k]
             endptr = self.colPtrs[k + 1]
             if self.isTransposed:
@@ -1297,9 +1348,11 @@ def asML(self):
         Convert this matrix to the new mllib-local representation.
         This does NOT copy the data; it copies references.
 
-        :return: :py:class:`pyspark.ml.linalg.SparseMatrix`
-
         .. versionadded:: 2.0.0
+
+        Returns
+        -------
+        :py:class:`pyspark.ml.linalg.SparseMatrix`
         """
         return newlinalg.SparseMatrix(self.numRows, self.numCols, self.colPtrs, self.rowIndices,
                                       self.values, self.isTransposed)
@@ -1330,10 +1383,15 @@ def fromML(mat):
         Convert a matrix from the new mllib-local representation.
         This does NOT copy the data; it copies references.
 
-        :param mat: a :py:class:`pyspark.ml.linalg.Matrix`
-        :return: a :py:class:`pyspark.mllib.linalg.Matrix`
-
         .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        mat : :py:class:`pyspark.ml.linalg.Matrix`
+
+        Returns
+        -------
+        :py:class:`pyspark.mllib.linalg.Matrix`
         """
         if isinstance(mat, newlinalg.DenseMatrix):
             return DenseMatrix(mat.numRows, mat.numCols, mat.values, mat.isTransposed)
diff --git a/python/pyspark/mllib/linalg/__init__.pyi b/python/pyspark/mllib/linalg/__init__.pyi
new file mode 100644
index 0000000000000..60d16b26f3590
--- /dev/null
+++ b/python/pyspark/mllib/linalg/__init__.pyi
@@ -0,0 +1,286 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import overload
+from typing import (
+    Any,
+    Dict,
+    Generic,
+    Iterable,
+    List,
+    Optional,
+    Tuple,
+    Type,
+    TypeVar,
+    Union,
+)
+from pyspark.ml import linalg as newlinalg
+from pyspark.sql.types import StructType, UserDefinedType
+from numpy import float64, ndarray  # type: ignore[import]
+
+QT = TypeVar("QT")
+RT = TypeVar("RT")
+
+class VectorUDT(UserDefinedType):
+    @classmethod
+    def sqlType(cls) -> StructType: ...
+    @classmethod
+    def module(cls) -> str: ...
+    @classmethod
+    def scalaUDT(cls) -> str: ...
+    def serialize(
+        self, obj: Vector
+    ) -> Tuple[int, Optional[int], Optional[List[int]], List[float]]: ...
+    def deserialize(self, datum: Any) -> Vector: ...
+    def simpleString(self) -> str: ...
+
+class MatrixUDT(UserDefinedType):
+    @classmethod
+    def sqlType(cls) -> StructType: ...
+    @classmethod
+    def module(cls) -> str: ...
+    @classmethod
+    def scalaUDT(cls) -> str: ...
+    def serialize(
+        self, obj: Matrix
+    ) -> Tuple[
+        int, int, int, Optional[List[int]], Optional[List[int]], List[float], bool
+    ]: ...
+    def deserialize(self, datum: Any) -> Matrix: ...
+    def simpleString(self) -> str: ...
+
+class Vector:
+    __UDT__: VectorUDT
+    def toArray(self) -> ndarray: ...
+    def asML(self) -> newlinalg.Vector: ...
+
+class DenseVector(Vector):
+    array: ndarray
+    @overload
+    def __init__(self, *elements: float) -> None: ...
+    @overload
+    def __init__(self, __arr: bytes) -> None: ...
+    @overload
+    def __init__(self, __arr: Iterable[float]) -> None: ...
+    @staticmethod
+    def parse(s: str) -> DenseVector: ...
+    def __reduce__(self) -> Tuple[Type[DenseVector], bytes]: ...
+    def numNonzeros(self) -> int: ...
+    def norm(self, p: Union[float, str]) -> float64: ...
+    def dot(self, other: Iterable[float]) -> float64: ...
+    def squared_distance(self, other: Iterable[float]) -> float64: ...
+    def toArray(self) -> ndarray: ...
+    def asML(self) -> newlinalg.DenseVector: ...
+    @property
+    def values(self) -> ndarray: ...
+    def __getitem__(self, item: int) -> float64: ...
+    def __len__(self) -> int: ...
+    def __eq__(self, other: Any) -> bool: ...
+    def __ne__(self, other: Any) -> bool: ...
+    def __hash__(self) -> int: ...
+    def __getattr__(self, item: str) -> Any: ...
+    def __neg__(self) -> DenseVector: ...
+    def __add__(self, other: Union[float, Iterable[float]]) -> DenseVector: ...
+    def __sub__(self, other: Union[float, Iterable[float]]) -> DenseVector: ...
+    def __mul__(self, other: Union[float, Iterable[float]]) -> DenseVector: ...
+    def __div__(self, other: Union[float, Iterable[float]]) -> DenseVector: ...
+    def __truediv__(self, other: Union[float, Iterable[float]]) -> DenseVector: ...
+    def __mod__(self, other: Union[float, Iterable[float]]) -> DenseVector: ...
+    def __radd__(self, other: Union[float, Iterable[float]]) -> DenseVector: ...
+    def __rsub__(self, other: Union[float, Iterable[float]]) -> DenseVector: ...
+    def __rmul__(self, other: Union[float, Iterable[float]]) -> DenseVector: ...
+    def __rdiv__(self, other: Union[float, Iterable[float]]) -> DenseVector: ...
+    def __rtruediv__(self, other: Union[float, Iterable[float]]) -> DenseVector: ...
+    def __rmod__(self, other: Union[float, Iterable[float]]) -> DenseVector: ...
+
+class SparseVector(Vector):
+    size: int
+    indices: ndarray
+    values: ndarray
+    @overload
+    def __init__(self, size: int, *args: Tuple[int, float]) -> None: ...
+    @overload
+    def __init__(self, size: int, __indices: bytes, __values: bytes) -> None: ...
+    @overload
+    def __init__(
+        self, size: int, __indices: Iterable[int], __values: Iterable[float]
+    ) -> None: ...
+    @overload
+    def __init__(self, size: int, __pairs: Iterable[Tuple[int, float]]) -> None: ...
+    @overload
+    def __init__(self, size: int, __map: Dict[int, float]) -> None: ...
+    def numNonzeros(self) -> int: ...
+    def norm(self, p: Union[float, str]) -> float64: ...
+    def __reduce__(self) -> Tuple[Type[SparseVector], Tuple[int, bytes, bytes]]: ...
+    @staticmethod
+    def parse(s: str) -> SparseVector: ...
+    def dot(self, other: Iterable[float]) -> float64: ...
+    def squared_distance(self, other: Iterable[float]) -> float64: ...
+    def toArray(self) -> ndarray: ...
+    def asML(self) -> newlinalg.SparseVector: ...
+    def __len__(self) -> int: ...
+    def __eq__(self, other: Any) -> bool: ...
+    def __getitem__(self, index: int) -> float64: ...
+    def __ne__(self, other: Any) -> bool: ...
+    def __hash__(self) -> int: ...
+
+class Vectors:
+    @overload
+    @staticmethod
+    def sparse(size: int, *args: Tuple[int, float]) -> SparseVector: ...
+    @overload
+    @staticmethod
+    def sparse(size: int, __indices: bytes, __values: bytes) -> SparseVector: ...
+    @overload
+    @staticmethod
+    def sparse(
+        size: int, __indices: Iterable[int], __values: Iterable[float]
+    ) -> SparseVector: ...
+    @overload
+    @staticmethod
+    def sparse(size: int, __pairs: Iterable[Tuple[int, float]]) -> SparseVector: ...
+    @overload
+    @staticmethod
+    def sparse(size: int, __map: Dict[int, float]) -> SparseVector: ...
+    @overload
+    @staticmethod
+    def dense(*elements: float) -> DenseVector: ...
+    @overload
+    @staticmethod
+    def dense(__arr: bytes) -> DenseVector: ...
+    @overload
+    @staticmethod
+    def dense(__arr: Iterable[float]) -> DenseVector: ...
+    @staticmethod
+    def fromML(vec: newlinalg.DenseVector) -> DenseVector: ...
+    @staticmethod
+    def stringify(vector: Vector) -> str: ...
+    @staticmethod
+    def squared_distance(v1: Vector, v2: Vector) -> float64: ...
+    @staticmethod
+    def norm(vector: Vector, p: Union[float, str]) -> float64: ...
+    @staticmethod
+    def parse(s: str) -> Vector: ...
+    @staticmethod
+    def zeros(size: int) -> DenseVector: ...
+
+class Matrix:
+    __UDT__: MatrixUDT
+    numRows: int
+    numCols: int
+    isTransposed: bool
+    def __init__(
+        self, numRows: int, numCols: int, isTransposed: bool = ...
+    ) -> None: ...
+    def toArray(self) -> ndarray: ...
+    def asML(self) -> newlinalg.Matrix: ...
+
+class DenseMatrix(Matrix):
+    values: Any
+    @overload
+    def __init__(
+        self, numRows: int, numCols: int, values: bytes, isTransposed: bool = ...
+    ) -> None: ...
+    @overload
+    def __init__(
+        self,
+        numRows: int,
+        numCols: int,
+        values: Iterable[float],
+        isTransposed: bool = ...,
+    ) -> None: ...
+    def __reduce__(self) -> Tuple[Type[DenseMatrix], Tuple[int, int, bytes, int]]: ...
+    def toArray(self) -> ndarray: ...
+    def toSparse(self) -> SparseMatrix: ...
+    def asML(self) -> newlinalg.DenseMatrix: ...
+    def __getitem__(self, indices: Tuple[int, int]) -> float64: ...
+    def __eq__(self, other: Any) -> bool: ...
+
+class SparseMatrix(Matrix):
+    colPtrs: ndarray
+    rowIndices: ndarray
+    values: ndarray
+    @overload
+    def __init__(
+        self,
+        numRows: int,
+        numCols: int,
+        colPtrs: bytes,
+        rowIndices: bytes,
+        values: bytes,
+        isTransposed: bool = ...,
+    ) -> None: ...
+    @overload
+    def __init__(
+        self,
+        numRows: int,
+        numCols: int,
+        colPtrs: Iterable[int],
+        rowIndices: Iterable[int],
+        values: Iterable[float],
+        isTransposed: bool = ...,
+    ) -> None: ...
+    def __reduce__(
+        self,
+    ) -> Tuple[Type[SparseMatrix], Tuple[int, int, bytes, bytes, bytes, int]]: ...
+    def __getitem__(self, indices: Tuple[int, int]) -> float64: ...
+    def toArray(self) -> ndarray: ...
+    def toDense(self) -> DenseMatrix: ...
+    def asML(self) -> newlinalg.SparseMatrix: ...
+    def __eq__(self, other: Any) -> bool: ...
+
+class Matrices:
+    @overload
+    @staticmethod
+    def dense(
+        numRows: int, numCols: int, values: bytes, isTransposed: bool = ...
+    ) -> DenseMatrix: ...
+    @overload
+    @staticmethod
+    def dense(
+        numRows: int, numCols: int, values: Iterable[float], isTransposed: bool = ...
+    ) -> DenseMatrix: ...
+    @overload
+    @staticmethod
+    def sparse(
+        numRows: int,
+        numCols: int,
+        colPtrs: bytes,
+        rowIndices: bytes,
+        values: bytes,
+        isTransposed: bool = ...,
+    ) -> SparseMatrix: ...
+    @overload
+    @staticmethod
+    def sparse(
+        numRows: int,
+        numCols: int,
+        colPtrs: Iterable[int],
+        rowIndices: Iterable[int],
+        values: Iterable[float],
+        isTransposed: bool = ...,
+    ) -> SparseMatrix: ...
+    @staticmethod
+    def fromML(mat: newlinalg.Matrix) -> Matrix: ...
+
+class QRDecomposition(Generic[QT, RT]):
+    def __init__(self, Q: QT, R: RT) -> None: ...
+    @property
+    def Q(self) -> QT: ...
+    @property
+    def R(self) -> RT: ...
diff --git a/python/pyspark/mllib/linalg/distributed.py b/python/pyspark/mllib/linalg/distributed.py
index 56701758c89c9..0128c204d97d3 100644
--- a/python/pyspark/mllib/linalg/distributed.py
+++ b/python/pyspark/mllib/linalg/distributed.py
@@ -21,9 +21,6 @@
 
 import sys
 
-if sys.version >= '3':
-    long = int
-
 from py4j.java_gateway import JavaObject
 
 from pyspark import RDD, since
@@ -58,16 +55,22 @@ class RowMatrix(DistributedMatrix):
     Represents a row-oriented distributed Matrix with no meaningful
     row indices.
 
-    :param rows: An RDD or DataFrame of vectors. If a DataFrame is provided, it must have a single
-                 vector typed column.
-    :param numRows: Number of rows in the matrix. A non-positive
-                    value means unknown, at which point the number
-                    of rows will be determined by the number of
-                    records in the `rows` RDD.
-    :param numCols: Number of columns in the matrix. A non-positive
-                    value means unknown, at which point the number
-                    of columns will be determined by the size of
-                    the first row.
+
+    Parameters
+    ----------
+    rows : :py:class:`pyspark.RDD` or :py:class:`pyspark.sql.DataFrame`
+        An RDD or DataFrame of vectors. If a DataFrame is provided, it must have a single
+        vector typed column.
+    numRows : int, optional
+        Number of rows in the matrix. A non-positive
+        value means unknown, at which point the number
+        of rows will be determined by the number of
+        records in the `rows` RDD.
+    numCols : int, optional
+        Number of columns in the matrix. A non-positive
+        value means unknown, at which point the number
+        of columns will be determined by the size of
+        the first row.
     """
     def __init__(self, rows, numRows=0, numCols=0):
         """
@@ -80,6 +83,8 @@ def __init__(self, rows, numRows=0, numCols=0):
         object, in which case we can wrap it directly.  This
         assists in clean matrix conversions.
 
+        Examples
+        --------
         >>> rows = sc.parallelize([[1, 2, 3], [4, 5, 6]])
         >>> mat = RowMatrix(rows)
 
@@ -95,9 +100,9 @@ def __init__(self, rows, numRows=0, numCols=0):
         """
         if isinstance(rows, RDD):
             rows = rows.map(_convert_to_vector)
-            java_matrix = callMLlibFunc("createRowMatrix", rows, long(numRows), int(numCols))
+            java_matrix = callMLlibFunc("createRowMatrix", rows, int(numRows), int(numCols))
         elif isinstance(rows, DataFrame):
-            java_matrix = callMLlibFunc("createRowMatrix", rows, long(numRows), int(numCols))
+            java_matrix = callMLlibFunc("createRowMatrix", rows, int(numRows), int(numCols))
         elif (isinstance(rows, JavaObject)
               and rows.getClass().getSimpleName() == "RowMatrix"):
             java_matrix = rows
@@ -111,6 +116,8 @@ def rows(self):
         """
         Rows of the RowMatrix stored as an RDD of vectors.
 
+        Examples
+        --------
         >>> mat = RowMatrix(sc.parallelize([[1, 2, 3], [4, 5, 6]]))
         >>> rows = mat.rows
         >>> rows.first()
@@ -122,6 +129,8 @@ def numRows(self):
         """
         Get or compute the number of rows.
 
+        Examples
+        --------
         >>> rows = sc.parallelize([[1, 2, 3], [4, 5, 6],
         ...                        [7, 8, 9], [10, 11, 12]])
 
@@ -139,6 +148,8 @@ def numCols(self):
         """
         Get or compute the number of cols.
 
+        Examples
+        --------
         >>> rows = sc.parallelize([[1, 2, 3], [4, 5, 6],
         ...                        [7, 8, 9], [10, 11, 12]])
 
@@ -152,14 +163,19 @@ def numCols(self):
         """
         return self._java_matrix_wrapper.call("numCols")
 
-    @since('2.0.0')
     def computeColumnSummaryStatistics(self):
         """
         Computes column-wise summary statistics.
 
-        :return: :class:`MultivariateStatisticalSummary` object
-                 containing column-wise summary statistics.
+        .. versionadded:: 2.0.0
 
+        Returns
+        -------
+        :py:class:`MultivariateStatisticalSummary`
+            object containing column-wise summary statistics.
+
+        Examples
+        --------
         >>> rows = sc.parallelize([[1, 2, 3], [4, 5, 6]])
         >>> mat = RowMatrix(rows)
 
@@ -170,14 +186,19 @@ def computeColumnSummaryStatistics(self):
         java_col_stats = self._java_matrix_wrapper.call("computeColumnSummaryStatistics")
         return MultivariateStatisticalSummary(java_col_stats)
 
-    @since('2.0.0')
     def computeCovariance(self):
         """
         Computes the covariance matrix, treating each row as an
         observation.
 
-        .. note:: This cannot be computed on matrices with more than 65535 columns.
+        .. versionadded:: 2.0.0
+
+        Notes
+        -----
+        This cannot be computed on matrices with more than 65535 columns.
 
+        Examples
+        --------
         >>> rows = sc.parallelize([[1, 2], [2, 1]])
         >>> mat = RowMatrix(rows)
 
@@ -186,13 +207,18 @@ def computeCovariance(self):
         """
         return self._java_matrix_wrapper.call("computeCovariance")
 
-    @since('2.0.0')
     def computeGramianMatrix(self):
         """
         Computes the Gramian matrix `A^T A`.
 
-        .. note:: This cannot be computed on matrices with more than 65535 columns.
+        .. versionadded:: 2.0.0
+
+        Notes
+        -----
+        This cannot be computed on matrices with more than 65535 columns.
 
+        Examples
+        --------
         >>> rows = sc.parallelize([[1, 2, 3], [4, 5, 6]])
         >>> mat = RowMatrix(rows)
 
@@ -223,11 +249,12 @@ def columnSimilarities(self, threshold=0.0):
         similarity threshold.
 
         To describe the guarantee, we set some notation:
-            * Let A be the smallest in magnitude non-zero element of
-              this matrix.
-            * Let B be the largest in magnitude non-zero element of
-              this matrix.
-            * Let L be the maximum number of non-zeros per row.
+
+        - Let A be the smallest in magnitude non-zero element of
+          this matrix.
+        - Let B be the largest in magnitude non-zero element of
+          this matrix.
+        - Let L be the maximum number of non-zeros per row.
 
         For example, for {0,1} matrices: A=B=1.
         Another example, for the Netflix matrix: A=1, B=5
@@ -239,20 +266,31 @@ def columnSimilarities(self, threshold=0.0):
         The shuffle size is bounded by the *smaller* of the following
         two expressions:
 
-            * O(n log(n) L / (threshold * A))
-            * O(m L^2^)
+        - O(n log(n) L / (threshold * A))
+        - O(m L^2^)
 
         The latter is the cost of the brute-force approach, so for
         non-zero thresholds, the cost is always cheaper than the
         brute-force approach.
 
-        :param: threshold: Set to 0 for deterministic guaranteed
-                           correctness. Similarities above this
-                           threshold are estimated with the cost vs
-                           estimate quality trade-off described above.
-        :return: An n x n sparse upper-triangular CoordinateMatrix of
-                 cosine similarities between columns of this matrix.
+        .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        threshold : float, optional
+            Set to 0 for deterministic guaranteed
+            correctness. Similarities above this
+            threshold are estimated with the cost vs
+            estimate quality trade-off described above.
+
+        Returns
+        -------
+        :py:class:`CoordinateMatrix`
+            An n x n sparse upper-triangular CoordinateMatrix of
+            cosine similarities between columns of this matrix.
 
+        Examples
+        --------
         >>> rows = sc.parallelize([[1, 2], [1, 5]])
         >>> mat = RowMatrix(rows)
 
@@ -263,23 +301,32 @@ def columnSimilarities(self, threshold=0.0):
         java_sims_mat = self._java_matrix_wrapper.call("columnSimilarities", float(threshold))
         return CoordinateMatrix(java_sims_mat)
 
-    @since('2.0.0')
     def tallSkinnyQR(self, computeQ=False):
         """
         Compute the QR decomposition of this RowMatrix.
 
         The implementation is designed to optimize the QR decomposition
-        (factorization) for the RowMatrix of a tall and skinny shape.
+        (factorization) for the RowMatrix of a tall and skinny shape [1]_.
+
+        .. [1] Paul G. Constantine, David F. Gleich. "Tall and skinny QR
+            factorizations in MapReduce architectures"
+            https://doi.org/10.1145/1996092.1996103
+
+        .. versionadded:: 2.0.0
 
-        Reference:
-         Paul G. Constantine, David F. Gleich. "Tall and skinny QR
-         factorizations in MapReduce architectures"
-         ([[https://doi.org/10.1145/1996092.1996103]])
+        Parameters
+        ----------
+        computeQ : bool, optional
+            whether to computeQ
 
-        :param: computeQ: whether to computeQ
-        :return: QRDecomposition(Q: RowMatrix, R: Matrix), where
-                 Q = None if computeQ = false.
+        Returns
+        -------
+        :py:class:`pyspark.mllib.linalg.QRDecomposition`
+            QRDecomposition(Q: RowMatrix, R: Matrix), where
+            Q = None if computeQ = false.
 
+        Examples
+        --------
         >>> rows = sc.parallelize([[3, -6], [4, -8], [0, 1]])
         >>> mat = RowMatrix(rows)
         >>> decomp = mat.tallSkinnyQR(True)
@@ -304,7 +351,6 @@ def tallSkinnyQR(self, computeQ=False):
         R = decomp.call("R")
         return QRDecomposition(Q, R)
 
-    @since('2.2.0')
     def computeSVD(self, k, computeU=False, rCond=1e-9):
         """
         Computes the singular value decomposition of the RowMatrix.
@@ -312,27 +358,39 @@ def computeSVD(self, k, computeU=False, rCond=1e-9):
         The given row matrix A of dimension (m X n) is decomposed into
         U * s * V'T where
 
-        * U: (m X k) (left singular vectors) is a RowMatrix whose
-             columns are the eigenvectors of (A X A')
-        * s: DenseVector consisting of square root of the eigenvalues
-             (singular values) in descending order.
-        * v: (n X k) (right singular vectors) is a Matrix whose columns
-             are the eigenvectors of (A' X A)
+        - U: (m X k) (left singular vectors) is a RowMatrix whose
+          columns are the eigenvectors of (A X A')
+        - s: DenseVector consisting of square root of the eigenvalues
+          (singular values) in descending order.
+        - v: (n X k) (right singular vectors) is a Matrix whose columns
+          are the eigenvectors of (A' X A)
 
         For more specific details on implementation, please refer
         the Scala documentation.
 
-        :param k: Number of leading singular values to keep (`0 < k <= n`).
-                  It might return less than k if there are numerically zero singular values
-                  or there are not enough Ritz values converged before the maximum number of
-                  Arnoldi update iterations is reached (in case that matrix A is ill-conditioned).
-        :param computeU: Whether or not to compute U. If set to be
-                         True, then U is computed by A * V * s^-1
-        :param rCond: Reciprocal condition number. All singular values
-                      smaller than rCond * s[0] are treated as zero
-                      where s[0] is the largest singular value.
-        :returns: :py:class:`SingularValueDecomposition`
-
+        .. versionadded:: 2.2.0
+
+        Parameters
+        ----------
+        k : int
+            Number of leading singular values to keep (`0 < k <= n`).
+            It might return less than k if there are numerically zero singular values
+            or there are not enough Ritz values converged before the maximum number of
+            Arnoldi update iterations is reached (in case that matrix A is ill-conditioned).
+        computeU : bool, optional
+            Whether or not to compute U. If set to be
+            True, then U is computed by A * V * s^-1
+        rCond : float, optional
+            Reciprocal condition number. All singular values
+            smaller than rCond * s[0] are treated as zero
+            where s[0] is the largest singular value.
+
+        Returns
+        -------
+        :py:class:`SingularValueDecomposition`
+
+        Examples
+        --------
         >>> rows = sc.parallelize([[3, 1, 1], [-1, 3, 1]])
         >>> rm = RowMatrix(rows)
 
@@ -348,16 +406,27 @@ def computeSVD(self, k, computeU=False, rCond=1e-9):
             "computeSVD", int(k), bool(computeU), float(rCond))
         return SingularValueDecomposition(j_model)
 
-    @since('2.2.0')
     def computePrincipalComponents(self, k):
         """
         Computes the k principal components of the given row matrix
 
-        .. note:: This cannot be computed on matrices with more than 65535 columns.
+        .. versionadded:: 2.2.0
+
+        Notes
+        -----
+        This cannot be computed on matrices with more than 65535 columns.
 
-        :param k: Number of principal components to keep.
-        :returns: :py:class:`pyspark.mllib.linalg.DenseMatrix`
+        Parameters
+        ----------
+        k : int
+            Number of principal components to keep.
 
+        Returns
+        -------
+        :py:class:`pyspark.mllib.linalg.DenseMatrix`
+
+        Examples
+        --------
         >>> rows = sc.parallelize([[1, 2, 3], [2, 4, 5], [3, 6, 1]])
         >>> rm = RowMatrix(rows)
 
@@ -373,22 +442,30 @@ def computePrincipalComponents(self, k):
         """
         return self._java_matrix_wrapper.call("computePrincipalComponents", k)
 
-    @since('2.2.0')
     def multiply(self, matrix):
         """
         Multiply this matrix by a local dense matrix on the right.
 
-        :param matrix: a local dense matrix whose number of rows must match the number of columns
-                       of this matrix
-        :returns: :py:class:`RowMatrix`
+        .. versionadded:: 2.2.0
+
+        Parameters
+        ----------
+        matrix : :py:class:`pyspark.mllib.linalg.Matrix`
+            a local dense matrix whose number of rows must match the number of columns
+            of this matrix
 
+        Returns
+        -------
+        :py:class:`RowMatrix`
+
+        Examples
+        --------
         >>> rm = RowMatrix(sc.parallelize([[0, 1], [2, 3]]))
         >>> rm.multiply(DenseMatrix(2, 2, [0, 2, 1, 3])).rows.collect()
         [DenseVector([2.0, 3.0]), DenseVector([6.0, 11.0])]
         """
         if not isinstance(matrix, DenseMatrix):
-            raise ValueError("Only multiplication with DenseMatrix "
-                             "is supported.")
+            raise TypeError("Only multiplication with DenseMatrix is supported.")
         j_model = self._java_matrix_wrapper.call("multiply", matrix)
         return RowMatrix(j_model)
 
@@ -439,13 +516,17 @@ class IndexedRow(object):
     """
     Represents a row of an IndexedRowMatrix.
 
-    Just a wrapper over a (long, vector) tuple.
+    Just a wrapper over a (int, vector) tuple.
 
-    :param index: The index for the given row.
-    :param vector: The row in the matrix at the given index.
+    Parameters
+    ----------
+    index : int
+        The index for the given row.
+    vector : :py:class:`pyspark.mllib.linalg.Vector` or convertible
+        The row in the matrix at the given index.
     """
     def __init__(self, index, vector):
-        self.index = long(index)
+        self.index = int(index)
         self.vector = _convert_to_vector(vector)
 
     def __repr__(self):
@@ -465,16 +546,21 @@ class IndexedRowMatrix(DistributedMatrix):
     """
     Represents a row-oriented distributed Matrix with indexed rows.
 
-    :param rows: An RDD of IndexedRows or (long, vector) tuples or a DataFrame consisting of a
-                 long typed column of indices and a vector typed column.
-    :param numRows: Number of rows in the matrix. A non-positive
-                    value means unknown, at which point the number
-                    of rows will be determined by the max row
-                    index plus one.
-    :param numCols: Number of columns in the matrix. A non-positive
-                    value means unknown, at which point the number
-                    of columns will be determined by the size of
-                    the first row.
+    Parameters
+    ----------
+    rows : :py:class:`pyspark.RDD`
+        An RDD of IndexedRows or (int, vector) tuples or a DataFrame consisting of a
+        int typed column of indices and a vector typed column.
+    numRows : int, optional
+        Number of rows in the matrix. A non-positive
+        value means unknown, at which point the number
+        of rows will be determined by the max row
+        index plus one.
+    numCols : int, optional
+        Number of columns in the matrix. A non-positive
+        value means unknown, at which point the number
+        of columns will be determined by the size of
+        the first row.
     """
     def __init__(self, rows, numRows=0, numCols=0):
         """
@@ -487,6 +573,8 @@ def __init__(self, rows, numRows=0, numCols=0):
         object, in which case we can wrap it directly.  This
         assists in clean matrix conversions.
 
+        Examples
+        --------
         >>> rows = sc.parallelize([IndexedRow(0, [1, 2, 3]),
         ...                        IndexedRow(1, [4, 5, 6])])
         >>> mat = IndexedRowMatrix(rows)
@@ -510,14 +598,14 @@ def __init__(self, rows, numRows=0, numCols=0):
             # both be easily serialized.  We will convert back to
             # IndexedRows on the Scala side.
             java_matrix = callMLlibFunc("createIndexedRowMatrix", rows.toDF(),
-                                        long(numRows), int(numCols))
+                                        int(numRows), int(numCols))
         elif isinstance(rows, DataFrame):
-            java_matrix = callMLlibFunc("createIndexedRowMatrix", rows, long(numRows), int(numCols))
+            java_matrix = callMLlibFunc("createIndexedRowMatrix", rows, int(numRows), int(numCols))
         elif (isinstance(rows, JavaObject)
               and rows.getClass().getSimpleName() == "IndexedRowMatrix"):
             java_matrix = rows
         else:
-            raise TypeError("rows should be an RDD of IndexedRows or (long, vector) tuples, "
+            raise TypeError("rows should be an RDD of IndexedRows or (int, vector) tuples, "
                             "got %s" % type(rows))
 
         self._java_matrix_wrapper = JavaModelWrapper(java_matrix)
@@ -527,6 +615,8 @@ def rows(self):
         """
         Rows of the IndexedRowMatrix stored as an RDD of IndexedRows.
 
+        Examples
+        --------
         >>> mat = IndexedRowMatrix(sc.parallelize([IndexedRow(0, [1, 2, 3]),
         ...                                        IndexedRow(1, [4, 5, 6])]))
         >>> rows = mat.rows
@@ -545,6 +635,8 @@ def numRows(self):
         """
         Get or compute the number of rows.
 
+        Examples
+        --------
         >>> rows = sc.parallelize([IndexedRow(0, [1, 2, 3]),
         ...                        IndexedRow(1, [4, 5, 6]),
         ...                        IndexedRow(2, [7, 8, 9]),
@@ -564,6 +656,8 @@ def numCols(self):
         """
         Get or compute the number of cols.
 
+        Examples
+        --------
         >>> rows = sc.parallelize([IndexedRow(0, [1, 2, 3]),
         ...                        IndexedRow(1, [4, 5, 6]),
         ...                        IndexedRow(2, [7, 8, 9]),
@@ -583,6 +677,8 @@ def columnSimilarities(self):
         """
         Compute all cosine similarities between columns.
 
+        Examples
+        --------
         >>> rows = sc.parallelize([IndexedRow(0, [1, 2, 3]),
         ...                        IndexedRow(6, [4, 5, 6])])
         >>> mat = IndexedRowMatrix(rows)
@@ -593,13 +689,18 @@ def columnSimilarities(self):
         java_coordinate_matrix = self._java_matrix_wrapper.call("columnSimilarities")
         return CoordinateMatrix(java_coordinate_matrix)
 
-    @since('2.0.0')
     def computeGramianMatrix(self):
         """
         Computes the Gramian matrix `A^T A`.
 
-        .. note:: This cannot be computed on matrices with more than 65535 columns.
+        .. versionadded:: 2.0.0
+
+        Notes
+        -----
+        This cannot be computed on matrices with more than 65535 columns.
 
+        Examples
+        --------
         >>> rows = sc.parallelize([IndexedRow(0, [1, 2, 3]),
         ...                        IndexedRow(1, [4, 5, 6])])
         >>> mat = IndexedRowMatrix(rows)
@@ -613,6 +714,8 @@ def toRowMatrix(self):
         """
         Convert this matrix to a RowMatrix.
 
+        Examples
+        --------
         >>> rows = sc.parallelize([IndexedRow(0, [1, 2, 3]),
         ...                        IndexedRow(6, [4, 5, 6])])
         >>> mat = IndexedRowMatrix(rows).toRowMatrix()
@@ -626,6 +729,8 @@ def toCoordinateMatrix(self):
         """
         Convert this matrix to a CoordinateMatrix.
 
+        Examples
+        --------
         >>> rows = sc.parallelize([IndexedRow(0, [1, 0]),
         ...                        IndexedRow(6, [0, 5])])
         >>> mat = IndexedRowMatrix(rows).toCoordinateMatrix()
@@ -639,13 +744,19 @@ def toBlockMatrix(self, rowsPerBlock=1024, colsPerBlock=1024):
         """
         Convert this matrix to a BlockMatrix.
 
-        :param rowsPerBlock: Number of rows that make up each block.
-                             The blocks forming the final rows are not
-                             required to have the given number of rows.
-        :param colsPerBlock: Number of columns that make up each block.
-                             The blocks forming the final columns are not
-                             required to have the given number of columns.
-
+        Parameters
+        ----------
+        rowsPerBlock : int, optional
+            Number of rows that make up each block.
+            The blocks forming the final rows are not
+            required to have the given number of rows.
+        colsPerBlock : int, optional
+            Number of columns that make up each block.
+            The blocks forming the final columns are not
+            required to have the given number of columns.
+
+        Examples
+        --------
         >>> rows = sc.parallelize([IndexedRow(0, [1, 2, 3]),
         ...                        IndexedRow(6, [4, 5, 6])])
         >>> mat = IndexedRowMatrix(rows).toBlockMatrix()
@@ -664,7 +775,6 @@ def toBlockMatrix(self, rowsPerBlock=1024, colsPerBlock=1024):
                                                            colsPerBlock)
         return BlockMatrix(java_block_matrix, rowsPerBlock, colsPerBlock)
 
-    @since('2.2.0')
     def computeSVD(self, k, computeU=False, rCond=1e-9):
         """
         Computes the singular value decomposition of the IndexedRowMatrix.
@@ -682,17 +792,29 @@ def computeSVD(self, k, computeU=False, rCond=1e-9):
         For more specific details on implementation, please refer
         the scala documentation.
 
-        :param k: Number of leading singular values to keep (`0 < k <= n`).
-                  It might return less than k if there are numerically zero singular values
-                  or there are not enough Ritz values converged before the maximum number of
-                  Arnoldi update iterations is reached (in case that matrix A is ill-conditioned).
-        :param computeU: Whether or not to compute U. If set to be
-                         True, then U is computed by A * V * s^-1
-        :param rCond: Reciprocal condition number. All singular values
-                      smaller than rCond * s[0] are treated as zero
-                      where s[0] is the largest singular value.
-        :returns: SingularValueDecomposition object
-
+        .. versionadded:: 2.2.0
+
+        Parameters
+        ----------
+        k : int
+            Number of leading singular values to keep (`0 < k <= n`).
+            It might return less than k if there are numerically zero singular values
+            or there are not enough Ritz values converged before the maximum number of
+            Arnoldi update iterations is reached (in case that matrix A is ill-conditioned).
+        computeU : bool, optional
+            Whether or not to compute U. If set to be
+            True, then U is computed by A * V * s^-1
+        rCond : float, optional
+            Reciprocal condition number. All singular values
+            smaller than rCond * s[0] are treated as zero
+            where s[0] is the largest singular value.
+
+        Returns
+        -------
+        :py:class:`SingularValueDecomposition`
+
+        Examples
+        --------
         >>> rows = [(0, (3, 1, 1)), (1, (-1, 3, 1))]
         >>> irm = IndexedRowMatrix(sc.parallelize(rows))
         >>> svd_model = irm.computeSVD(2, True)
@@ -708,22 +830,30 @@ def computeSVD(self, k, computeU=False, rCond=1e-9):
             "computeSVD", int(k), bool(computeU), float(rCond))
         return SingularValueDecomposition(j_model)
 
-    @since('2.2.0')
     def multiply(self, matrix):
         """
         Multiply this matrix by a local dense matrix on the right.
 
-        :param matrix: a local dense matrix whose number of rows must match the number of columns
-                       of this matrix
-        :returns: :py:class:`IndexedRowMatrix`
+        .. versionadded:: 2.2.0
+
+        Parameters
+        ----------
+        matrix : :py:class:`pyspark.mllib.linalg.Matrix`
+            a local dense matrix whose number of rows must match the number of columns
+            of this matrix
 
+        Returns
+        -------
+        :py:class:`IndexedRowMatrix`
+
+        Examples
+        --------
         >>> mat = IndexedRowMatrix(sc.parallelize([(0, (0, 1)), (1, (2, 3))]))
         >>> mat.multiply(DenseMatrix(2, 2, [0, 2, 1, 3])).rows.collect()
         [IndexedRow(0, [2.0,3.0]), IndexedRow(1, [6.0,11.0])]
         """
         if not isinstance(matrix, DenseMatrix):
-            raise ValueError("Only multiplication with DenseMatrix "
-                             "is supported.")
+            raise TypeError("Only multiplication with DenseMatrix is supported.")
         return IndexedRowMatrix(self._java_matrix_wrapper.call("multiply", matrix))
 
 
@@ -731,15 +861,20 @@ class MatrixEntry(object):
     """
     Represents an entry of a CoordinateMatrix.
 
-    Just a wrapper over a (long, long, float) tuple.
+    Just a wrapper over a (int, int, float) tuple.
 
-    :param i: The row index of the matrix.
-    :param j: The column index of the matrix.
-    :param value: The (i, j)th entry of the matrix, as a float.
+    Parameters
+    ----------
+    i : int
+        The row index of the matrix.
+    j : int
+        The column index of the matrix.
+    value : float
+        The (i, j)th entry of the matrix, as a float.
     """
     def __init__(self, i, j, value):
-        self.i = long(i)
-        self.j = long(j)
+        self.i = int(i)
+        self.j = int(j)
         self.value = float(value)
 
     def __repr__(self):
@@ -759,16 +894,21 @@ class CoordinateMatrix(DistributedMatrix):
     """
     Represents a matrix in coordinate format.
 
-    :param entries: An RDD of MatrixEntry inputs or
-                    (long, long, float) tuples.
-    :param numRows: Number of rows in the matrix. A non-positive
-                    value means unknown, at which point the number
-                    of rows will be determined by the max row
-                    index plus one.
-    :param numCols: Number of columns in the matrix. A non-positive
-                    value means unknown, at which point the number
-                    of columns will be determined by the max row
-                    index plus one.
+    Parameters
+    ----------
+    entries : :py:class:`pyspark.RDD`
+        An RDD of MatrixEntry inputs or
+        (int, int, float) tuples.
+    numRows : int, optional
+        Number of rows in the matrix. A non-positive
+        value means unknown, at which point the number
+        of rows will be determined by the max row
+        index plus one.
+    numCols : int, optional
+        Number of columns in the matrix. A non-positive
+        value means unknown, at which point the number
+        of columns will be determined by the max row
+        index plus one.
     """
     def __init__(self, entries, numRows=0, numCols=0):
         """
@@ -781,6 +921,8 @@ def __init__(self, entries, numRows=0, numCols=0):
         object, in which case we can wrap it directly.  This
         assists in clean matrix conversions.
 
+        Examples
+        --------
         >>> entries = sc.parallelize([MatrixEntry(0, 0, 1.2),
         ...                           MatrixEntry(6, 4, 2.1)])
         >>> mat = CoordinateMatrix(entries)
@@ -804,13 +946,13 @@ def __init__(self, entries, numRows=0, numCols=0):
             # each be easily serialized. We will convert back to
             # MatrixEntry inputs on the Scala side.
             java_matrix = callMLlibFunc("createCoordinateMatrix", entries.toDF(),
-                                        long(numRows), long(numCols))
+                                        int(numRows), int(numCols))
         elif (isinstance(entries, JavaObject)
               and entries.getClass().getSimpleName() == "CoordinateMatrix"):
             java_matrix = entries
         else:
             raise TypeError("entries should be an RDD of MatrixEntry entries or "
-                            "(long, long, float) tuples, got %s" % type(entries))
+                            "(int, int, float) tuples, got %s" % type(entries))
 
         self._java_matrix_wrapper = JavaModelWrapper(java_matrix)
 
@@ -820,6 +962,8 @@ def entries(self):
         Entries of the CoordinateMatrix stored as an RDD of
         MatrixEntries.
 
+        Examples
+        --------
         >>> mat = CoordinateMatrix(sc.parallelize([MatrixEntry(0, 0, 1.2),
         ...                                        MatrixEntry(6, 4, 2.1)]))
         >>> entries = mat.entries
@@ -838,6 +982,8 @@ def numRows(self):
         """
         Get or compute the number of rows.
 
+        Examples
+        --------
         >>> entries = sc.parallelize([MatrixEntry(0, 0, 1.2),
         ...                           MatrixEntry(1, 0, 2),
         ...                           MatrixEntry(2, 1, 3.7)])
@@ -856,6 +1002,8 @@ def numCols(self):
         """
         Get or compute the number of cols.
 
+        Examples
+        --------
         >>> entries = sc.parallelize([MatrixEntry(0, 0, 1.2),
         ...                           MatrixEntry(1, 0, 2),
         ...                           MatrixEntry(2, 1, 3.7)])
@@ -870,11 +1018,14 @@ def numCols(self):
         """
         return self._java_matrix_wrapper.call("numCols")
 
-    @since('2.0.0')
     def transpose(self):
         """
         Transpose this CoordinateMatrix.
 
+        .. versionadded:: 2.0.0
+
+        Examples
+        --------
         >>> entries = sc.parallelize([MatrixEntry(0, 0, 1.2),
         ...                           MatrixEntry(1, 0, 2),
         ...                           MatrixEntry(2, 1, 3.7)])
@@ -894,6 +1045,8 @@ def toRowMatrix(self):
         """
         Convert this matrix to a RowMatrix.
 
+        Examples
+        --------
         >>> entries = sc.parallelize([MatrixEntry(0, 0, 1.2),
         ...                           MatrixEntry(6, 4, 2.1)])
         >>> mat = CoordinateMatrix(entries).toRowMatrix()
@@ -918,6 +1071,8 @@ def toIndexedRowMatrix(self):
         """
         Convert this matrix to an IndexedRowMatrix.
 
+        Examples
+        --------
         >>> entries = sc.parallelize([MatrixEntry(0, 0, 1.2),
         ...                           MatrixEntry(6, 4, 2.1)])
         >>> mat = CoordinateMatrix(entries).toIndexedRowMatrix()
@@ -941,13 +1096,19 @@ def toBlockMatrix(self, rowsPerBlock=1024, colsPerBlock=1024):
         """
         Convert this matrix to a BlockMatrix.
 
-        :param rowsPerBlock: Number of rows that make up each block.
-                             The blocks forming the final rows are not
-                             required to have the given number of rows.
-        :param colsPerBlock: Number of columns that make up each block.
-                             The blocks forming the final columns are not
-                             required to have the given number of columns.
-
+        Parameters
+        ----------
+        rowsPerBlock : int, optional
+            Number of rows that make up each block.
+            The blocks forming the final rows are not
+            required to have the given number of rows.
+        colsPerBlock : int, optional
+            Number of columns that make up each block.
+            The blocks forming the final columns are not
+            required to have the given number of columns.
+
+        Examples
+        --------
         >>> entries = sc.parallelize([MatrixEntry(0, 0, 1.2),
         ...                           MatrixEntry(6, 4, 2.1)])
         >>> mat = CoordinateMatrix(entries).toBlockMatrix()
@@ -986,26 +1147,33 @@ class BlockMatrix(DistributedMatrix):
     """
     Represents a distributed matrix in blocks of local matrices.
 
-    :param blocks: An RDD of sub-matrix blocks
-                   ((blockRowIndex, blockColIndex), sub-matrix) that
-                   form this distributed matrix. If multiple blocks
-                   with the same index exist, the results for
-                   operations like add and multiply will be
-                   unpredictable.
-    :param rowsPerBlock: Number of rows that make up each block.
-                         The blocks forming the final rows are not
-                         required to have the given number of rows.
-    :param colsPerBlock: Number of columns that make up each block.
-                         The blocks forming the final columns are not
-                         required to have the given number of columns.
-    :param numRows: Number of rows of this matrix. If the supplied
-                    value is less than or equal to zero, the number
-                    of rows will be calculated when `numRows` is
-                    invoked.
-    :param numCols: Number of columns of this matrix. If the supplied
-                    value is less than or equal to zero, the number
-                    of columns will be calculated when `numCols` is
-                    invoked.
+    Parameters
+    ----------
+    blocks : :py:class:`pyspark.RDD`
+        An RDD of sub-matrix blocks
+        ((blockRowIndex, blockColIndex), sub-matrix) that
+        form this distributed matrix. If multiple blocks
+        with the same index exist, the results for
+        operations like add and multiply will be
+        unpredictable.
+    rowsPerBlock : int
+        Number of rows that make up each block.
+        The blocks forming the final rows are not
+        required to have the given number of rows.
+    colsPerBlock : int
+        Number of columns that make up each block.
+        The blocks forming the final columns are not
+        required to have the given number of columns.
+    numRows : int, optional
+        Number of rows of this matrix. If the supplied
+        value is less than or equal to zero, the number
+        of rows will be calculated when `numRows` is
+        invoked.
+    numCols : int, optional
+        Number of columns of this matrix. If the supplied
+        value is less than or equal to zero, the number
+        of columns will be calculated when `numCols` is
+        invoked.
     """
     def __init__(self, blocks, rowsPerBlock, colsPerBlock, numRows=0, numCols=0):
         """
@@ -1018,6 +1186,8 @@ def __init__(self, blocks, rowsPerBlock, colsPerBlock, numRows=0, numCols=0):
         object, in which case we can wrap it directly.  This
         assists in clean matrix conversions.
 
+        Examples
+        --------
         >>> blocks = sc.parallelize([((0, 0), Matrices.dense(3, 2, [1, 2, 3, 4, 5, 6])),
         ...                          ((1, 0), Matrices.dense(3, 2, [7, 8, 9, 10, 11, 12]))])
         >>> mat = BlockMatrix(blocks, 3, 2)
@@ -1044,7 +1214,7 @@ def __init__(self, blocks, rowsPerBlock, colsPerBlock, numRows=0, numCols=0):
             # the Scala side.
             java_matrix = callMLlibFunc("createBlockMatrix", blocks.toDF(),
                                         int(rowsPerBlock), int(colsPerBlock),
-                                        long(numRows), long(numCols))
+                                        int(numRows), int(numCols))
         elif (isinstance(blocks, JavaObject)
               and blocks.getClass().getSimpleName() == "BlockMatrix"):
             java_matrix = blocks
@@ -1061,6 +1231,8 @@ def blocks(self):
         ((blockRowIndex, blockColIndex), sub-matrix) that form this
         distributed matrix.
 
+        Examples
+        --------
         >>> mat = BlockMatrix(
         ...     sc.parallelize([((0, 0), Matrices.dense(3, 2, [1, 2, 3, 4, 5, 6])),
         ...                     ((1, 0), Matrices.dense(3, 2, [7, 8, 9, 10, 11, 12]))]), 3, 2)
@@ -1082,6 +1254,8 @@ def rowsPerBlock(self):
         """
         Number of rows that make up each block.
 
+        Examples
+        --------
         >>> blocks = sc.parallelize([((0, 0), Matrices.dense(3, 2, [1, 2, 3, 4, 5, 6])),
         ...                          ((1, 0), Matrices.dense(3, 2, [7, 8, 9, 10, 11, 12]))])
         >>> mat = BlockMatrix(blocks, 3, 2)
@@ -1095,6 +1269,8 @@ def colsPerBlock(self):
         """
         Number of columns that make up each block.
 
+        Examples
+        --------
         >>> blocks = sc.parallelize([((0, 0), Matrices.dense(3, 2, [1, 2, 3, 4, 5, 6])),
         ...                          ((1, 0), Matrices.dense(3, 2, [7, 8, 9, 10, 11, 12]))])
         >>> mat = BlockMatrix(blocks, 3, 2)
@@ -1108,6 +1284,8 @@ def numRowBlocks(self):
         """
         Number of rows of blocks in the BlockMatrix.
 
+        Examples
+        --------
         >>> blocks = sc.parallelize([((0, 0), Matrices.dense(3, 2, [1, 2, 3, 4, 5, 6])),
         ...                          ((1, 0), Matrices.dense(3, 2, [7, 8, 9, 10, 11, 12]))])
         >>> mat = BlockMatrix(blocks, 3, 2)
@@ -1121,6 +1299,8 @@ def numColBlocks(self):
         """
         Number of columns of blocks in the BlockMatrix.
 
+        Examples
+        --------
         >>> blocks = sc.parallelize([((0, 0), Matrices.dense(3, 2, [1, 2, 3, 4, 5, 6])),
         ...                          ((1, 0), Matrices.dense(3, 2, [7, 8, 9, 10, 11, 12]))])
         >>> mat = BlockMatrix(blocks, 3, 2)
@@ -1133,6 +1313,8 @@ def numRows(self):
         """
         Get or compute the number of rows.
 
+        Examples
+        --------
         >>> blocks = sc.parallelize([((0, 0), Matrices.dense(3, 2, [1, 2, 3, 4, 5, 6])),
         ...                          ((1, 0), Matrices.dense(3, 2, [7, 8, 9, 10, 11, 12]))])
 
@@ -1150,6 +1332,8 @@ def numCols(self):
         """
         Get or compute the number of cols.
 
+        Examples
+        --------
         >>> blocks = sc.parallelize([((0, 0), Matrices.dense(3, 2, [1, 2, 3, 4, 5, 6])),
         ...                          ((1, 0), Matrices.dense(3, 2, [7, 8, 9, 10, 11, 12]))])
 
@@ -1200,6 +1384,8 @@ def add(self, other):
         two dense sub matrix blocks are added, the output block will
         also be a DenseMatrix.
 
+        Examples
+        --------
         >>> dm1 = Matrices.dense(3, 2, [1, 2, 3, 4, 5, 6])
         >>> dm2 = Matrices.dense(3, 2, [7, 8, 9, 10, 11, 12])
         >>> sm = Matrices.sparse(3, 2, [0, 1, 3], [0, 1, 2], [7, 11, 12])
@@ -1223,7 +1409,6 @@ def add(self, other):
         java_block_matrix = self._java_matrix_wrapper.call("add", other_java_block_matrix)
         return BlockMatrix(java_block_matrix, self.rowsPerBlock, self.colsPerBlock)
 
-    @since('2.0.0')
     def subtract(self, other):
         """
         Subtracts the given block matrix `other` from this block matrix:
@@ -1235,6 +1420,10 @@ def subtract(self, other):
         If two dense sub matrix blocks are subtracted, the output block
         will also be a DenseMatrix.
 
+        .. versionadded:: 2.0.0
+
+        Examples
+        --------
         >>> dm1 = Matrices.dense(3, 2, [3, 1, 5, 4, 6, 2])
         >>> dm2 = Matrices.dense(3, 2, [7, 8, 9, 10, 11, 12])
         >>> sm = Matrices.sparse(3, 2, [0, 1, 3], [0, 1, 2], [1, 2, 3])
@@ -1268,6 +1457,8 @@ def multiply(self, other):
         This may cause some performance issues until support for
         multiplying two sparse matrices is added.
 
+        Examples
+        --------
         >>> dm1 = Matrices.dense(2, 3, [1, 2, 3, 4, 5, 6])
         >>> dm2 = Matrices.dense(2, 3, [7, 8, 9, 10, 11, 12])
         >>> dm3 = Matrices.dense(3, 2, [1, 2, 3, 4, 5, 6])
@@ -1293,12 +1484,15 @@ def multiply(self, other):
         java_block_matrix = self._java_matrix_wrapper.call("multiply", other_java_block_matrix)
         return BlockMatrix(java_block_matrix, self.rowsPerBlock, self.colsPerBlock)
 
-    @since('2.0.0')
     def transpose(self):
         """
         Transpose this BlockMatrix. Returns a new BlockMatrix
         instance sharing the same underlying data. Is a lazy operation.
 
+        .. versionadded:: 2.0.0
+
+        Examples
+        --------
         >>> blocks = sc.parallelize([((0, 0), Matrices.dense(3, 2, [1, 2, 3, 4, 5, 6])),
         ...                          ((1, 0), Matrices.dense(3, 2, [7, 8, 9, 10, 11, 12]))])
         >>> mat = BlockMatrix(blocks, 3, 2)
@@ -1314,6 +1508,8 @@ def toLocalMatrix(self):
         """
         Collect the distributed matrix on the driver as a DenseMatrix.
 
+        Examples
+        --------
         >>> blocks = sc.parallelize([((0, 0), Matrices.dense(3, 2, [1, 2, 3, 4, 5, 6])),
         ...                          ((1, 0), Matrices.dense(3, 2, [7, 8, 9, 10, 11, 12]))])
         >>> mat = BlockMatrix(blocks, 3, 2).toLocalMatrix()
@@ -1336,6 +1532,8 @@ def toIndexedRowMatrix(self):
         """
         Convert this matrix to an IndexedRowMatrix.
 
+        Examples
+        --------
         >>> blocks = sc.parallelize([((0, 0), Matrices.dense(3, 2, [1, 2, 3, 4, 5, 6])),
         ...                          ((1, 0), Matrices.dense(3, 2, [7, 8, 9, 10, 11, 12]))])
         >>> mat = BlockMatrix(blocks, 3, 2).toIndexedRowMatrix()
@@ -1359,6 +1557,8 @@ def toCoordinateMatrix(self):
         """
         Convert this matrix to a CoordinateMatrix.
 
+        Examples
+        --------
         >>> blocks = sc.parallelize([((0, 0), Matrices.dense(1, 2, [1, 2])),
         ...                          ((1, 0), Matrices.dense(1, 2, [7, 8]))])
         >>> mat = BlockMatrix(blocks, 1, 2).toCoordinateMatrix()
diff --git a/python/pyspark/mllib/linalg/distributed.pyi b/python/pyspark/mllib/linalg/distributed.pyi
new file mode 100644
index 0000000000000..7ec2d60c5a947
--- /dev/null
+++ b/python/pyspark/mllib/linalg/distributed.pyi
@@ -0,0 +1,151 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Generic, Sequence, Optional, Tuple, TypeVar, Union
+from pyspark.rdd import RDD
+from pyspark.storagelevel import StorageLevel
+from pyspark.mllib.common import JavaModelWrapper
+from pyspark.mllib.linalg import Vector, Matrix, QRDecomposition
+from pyspark.mllib.stat import MultivariateStatisticalSummary
+import pyspark.sql.dataframe
+from numpy import ndarray  # noqa: F401
+
+VectorLike = Union[Vector, Sequence[Union[float, int]]]
+
+UT = TypeVar("UT")
+VT = TypeVar("VT")
+
+class DistributedMatrix:
+    def numRows(self) -> int: ...
+    def numCols(self) -> int: ...
+
+class RowMatrix(DistributedMatrix):
+    def __init__(
+        self,
+        rows: Union[RDD[Vector], pyspark.sql.dataframe.DataFrame],
+        numRows: int = ...,
+        numCols: int = ...,
+    ) -> None: ...
+    @property
+    def rows(self) -> RDD[Vector]: ...
+    def numRows(self) -> int: ...
+    def numCols(self) -> int: ...
+    def computeColumnSummaryStatistics(self) -> MultivariateStatisticalSummary: ...
+    def computeCovariance(self) -> Matrix: ...
+    def computeGramianMatrix(self) -> Matrix: ...
+    def columnSimilarities(self, threshold: float = ...) -> CoordinateMatrix: ...
+    def tallSkinnyQR(
+        self, computeQ: bool = ...
+    ) -> QRDecomposition[RowMatrix, Matrix]: ...
+    def computeSVD(
+        self, k: int, computeU: bool = ..., rCond: float = ...
+    ) -> SingularValueDecomposition[RowMatrix, Matrix]: ...
+    def computePrincipalComponents(self, k: int) -> Matrix: ...
+    def multiply(self, matrix: Matrix) -> RowMatrix: ...
+
+class SingularValueDecomposition(JavaModelWrapper, Generic[UT, VT]):
+    @property
+    def U(self) -> Optional[UT]: ...
+    @property
+    def s(self) -> Vector: ...
+    @property
+    def V(self) -> VT: ...
+
+class IndexedRow:
+    index: int
+    vector: VectorLike
+    def __init__(self, index: int, vector: VectorLike) -> None: ...
+
+class IndexedRowMatrix(DistributedMatrix):
+    def __init__(
+        self,
+        rows: RDD[Union[Tuple[int, VectorLike], IndexedRow]],
+        numRows: int = ...,
+        numCols: int = ...,
+    ) -> None: ...
+    @property
+    def rows(self) -> RDD[IndexedRow]: ...
+    def numRows(self) -> int: ...
+    def numCols(self) -> int: ...
+    def columnSimilarities(self) -> CoordinateMatrix: ...
+    def computeGramianMatrix(self) -> Matrix: ...
+    def toRowMatrix(self) -> RowMatrix: ...
+    def toCoordinateMatrix(self) -> CoordinateMatrix: ...
+    def toBlockMatrix(
+        self, rowsPerBlock: int = ..., colsPerBlock: int = ...
+    ) -> BlockMatrix: ...
+    def computeSVD(
+        self, k: int, computeU: bool = ..., rCond: float = ...
+    ) -> SingularValueDecomposition[IndexedRowMatrix, Matrix]: ...
+    def multiply(self, matrix: Matrix) -> IndexedRowMatrix: ...
+
+class MatrixEntry:
+    i: int
+    j: int
+    value: float
+    def __init__(self, i: int, j: int, value: float) -> None: ...
+
+class CoordinateMatrix(DistributedMatrix):
+    def __init__(
+        self,
+        entries: RDD[Union[Tuple[int, int, float], MatrixEntry]],
+        numRows: int = ...,
+        numCols: int = ...,
+    ) -> None: ...
+    @property
+    def entries(self) -> RDD[MatrixEntry]: ...
+    def numRows(self) -> int: ...
+    def numCols(self) -> int: ...
+    def transpose(self) -> CoordinateMatrix: ...
+    def toRowMatrix(self) -> RowMatrix: ...
+    def toIndexedRowMatrix(self) -> IndexedRowMatrix: ...
+    def toBlockMatrix(
+        self, rowsPerBlock: int = ..., colsPerBlock: int = ...
+    ) -> BlockMatrix: ...
+
+class BlockMatrix(DistributedMatrix):
+    def __init__(
+        self,
+        blocks: RDD[Tuple[Tuple[int, int], Matrix]],
+        rowsPerBlock: int,
+        colsPerBlock: int,
+        numRows: int = ...,
+        numCols: int = ...,
+    ) -> None: ...
+    @property
+    def blocks(self) -> RDD[Tuple[Tuple[int, int], Matrix]]: ...
+    @property
+    def rowsPerBlock(self) -> int: ...
+    @property
+    def colsPerBlock(self) -> int: ...
+    @property
+    def numRowBlocks(self) -> int: ...
+    @property
+    def numColBlocks(self) -> int: ...
+    def numRows(self) -> int: ...
+    def numCols(self) -> int: ...
+    def cache(self) -> BlockMatrix: ...
+    def persist(self, storageLevel: StorageLevel) -> BlockMatrix: ...
+    def validate(self) -> None: ...
+    def add(self, other: BlockMatrix) -> BlockMatrix: ...
+    def subtract(self, other: BlockMatrix) -> BlockMatrix: ...
+    def multiply(self, other: BlockMatrix) -> BlockMatrix: ...
+    def transpose(self) -> BlockMatrix: ...
+    def toLocalMatrix(self) -> Matrix: ...
+    def toIndexedRowMatrix(self) -> IndexedRowMatrix: ...
+    def toCoordinateMatrix(self) -> CoordinateMatrix: ...
diff --git a/python/pyspark/mllib/random.py b/python/pyspark/mllib/random.py
index 6106c58584882..a33dfe26fbad9 100644
--- a/python/pyspark/mllib/random.py
+++ b/python/pyspark/mllib/random.py
@@ -22,7 +22,6 @@
 import sys
 from functools import wraps
 
-from pyspark import since
 from pyspark.mllib.common import callMLlibFunc
 
 
@@ -46,7 +45,6 @@ class RandomRDDs(object):
     """
 
     @staticmethod
-    @since("1.1.0")
     def uniformRDD(sc, size, numPartitions=None, seed=None):
         """
         Generates an RDD comprised of i.i.d. samples from the
@@ -56,12 +54,26 @@ def uniformRDD(sc, size, numPartitions=None, seed=None):
         to U(a, b), use
         ``RandomRDDs.uniformRDD(sc, n, p, seed).map(lambda v: a + (b - a) * v)``
 
-        :param sc: SparkContext used to create the RDD.
-        :param size: Size of the RDD.
-        :param numPartitions: Number of partitions in the RDD (default: `sc.defaultParallelism`).
-        :param seed: Random seed (default: a random long integer).
-        :return: RDD of float comprised of i.i.d. samples ~ `U(0.0, 1.0)`.
-
+        .. versionadded:: 1.1.0
+
+        Parameters
+        ----------
+        sc : :py:class:`pyspark.SparkContext`
+            used to create the RDD.
+        size : int
+            Size of the RDD.
+        numPartitions : int, optional
+            Number of partitions in the RDD (default: `sc.defaultParallelism`).
+        seed : int, optional
+            Random seed (default: a random long integer).
+
+        Returns
+        -------
+        :py:class:`pyspark.RDD`
+            RDD of float comprised of i.i.d. samples ~ `U(0.0, 1.0)`.
+
+        Examples
+        --------
         >>> x = RandomRDDs.uniformRDD(sc, 100).collect()
         >>> len(x)
         100
@@ -76,7 +88,6 @@ def uniformRDD(sc, size, numPartitions=None, seed=None):
         return callMLlibFunc("uniformRDD", sc._jsc, size, numPartitions, seed)
 
     @staticmethod
-    @since("1.1.0")
     def normalRDD(sc, size, numPartitions=None, seed=None):
         """
         Generates an RDD comprised of i.i.d. samples from the standard normal
@@ -86,12 +97,26 @@ def normalRDD(sc, size, numPartitions=None, seed=None):
         to some other normal N(mean, sigma^2), use
         ``RandomRDDs.normal(sc, n, p, seed).map(lambda v: mean + sigma * v)``
 
-        :param sc: SparkContext used to create the RDD.
-        :param size: Size of the RDD.
-        :param numPartitions: Number of partitions in the RDD (default: `sc.defaultParallelism`).
-        :param seed: Random seed (default: a random long integer).
-        :return: RDD of float comprised of i.i.d. samples ~ N(0.0, 1.0).
-
+        .. versionadded:: 1.1.0
+
+        Parameters
+        ----------
+        sc : :py:class:`pyspark.SparkContext`
+            used to create the RDD.
+        size : int
+            Size of the RDD.
+        numPartitions : int, optional
+            Number of partitions in the RDD (default: `sc.defaultParallelism`).
+        seed : int, optional
+            Random seed (default: a random long integer).
+
+        Returns
+        -------
+        :py:class:`pyspark.RDD`
+            RDD of float comprised of i.i.d. samples ~ N(0.0, 1.0).
+
+        Examples
+        --------
         >>> x = RandomRDDs.normalRDD(sc, 1000, seed=1)
         >>> stats = x.stats()
         >>> stats.count()
@@ -104,20 +129,34 @@ def normalRDD(sc, size, numPartitions=None, seed=None):
         return callMLlibFunc("normalRDD", sc._jsc, size, numPartitions, seed)
 
     @staticmethod
-    @since("1.3.0")
     def logNormalRDD(sc, mean, std, size, numPartitions=None, seed=None):
         """
         Generates an RDD comprised of i.i.d. samples from the log normal
         distribution with the input mean and standard distribution.
 
-        :param sc: SparkContext used to create the RDD.
-        :param mean: mean for the log Normal distribution
-        :param std: std for the log Normal distribution
-        :param size: Size of the RDD.
-        :param numPartitions: Number of partitions in the RDD (default: `sc.defaultParallelism`).
-        :param seed: Random seed (default: a random long integer).
-        :return: RDD of float comprised of i.i.d. samples ~ log N(mean, std).
-
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        sc : :py:class:`pyspark.SparkContext`
+            used to create the RDD.
+        mean : float
+            mean for the log Normal distribution
+        std : float
+            std for the log Normal distribution
+        size : int
+            Size of the RDD.
+        numPartitions : int, optional
+            Number of partitions in the RDD (default: `sc.defaultParallelism`).
+        seed : int, optional
+            Random seed (default: a random long integer).
+
+        Returns
+        -------
+        RDD of float comprised of i.i.d. samples ~ log N(mean, std).
+
+        Examples
+        --------
         >>> from math import sqrt, exp
         >>> mean = 0.0
         >>> std = 1.0
@@ -137,19 +176,33 @@ def logNormalRDD(sc, mean, std, size, numPartitions=None, seed=None):
                              size, numPartitions, seed)
 
     @staticmethod
-    @since("1.1.0")
     def poissonRDD(sc, mean, size, numPartitions=None, seed=None):
         """
         Generates an RDD comprised of i.i.d. samples from the Poisson
         distribution with the input mean.
 
-        :param sc: SparkContext used to create the RDD.
-        :param mean: Mean, or lambda, for the Poisson distribution.
-        :param size: Size of the RDD.
-        :param numPartitions: Number of partitions in the RDD (default: `sc.defaultParallelism`).
-        :param seed: Random seed (default: a random long integer).
-        :return: RDD of float comprised of i.i.d. samples ~ Pois(mean).
-
+        .. versionadded:: 1.1.0
+
+        Parameters
+        ----------
+        sc : :py:class:`pyspark.SparkContext`
+            SparkContext used to create the RDD.
+        mean : float
+            Mean, or lambda, for the Poisson distribution.
+        size : int
+            Size of the RDD.
+        numPartitions : int, optional
+            Number of partitions in the RDD (default: `sc.defaultParallelism`).
+        seed : int, optional
+            Random seed (default: a random long integer).
+
+        Returns
+        -------
+        :py:class:`pyspark.RDD`
+            RDD of float comprised of i.i.d. samples ~ Pois(mean).
+
+        Examples
+        --------
         >>> mean = 100.0
         >>> x = RandomRDDs.poissonRDD(sc, mean, 1000, seed=2)
         >>> stats = x.stats()
@@ -164,19 +217,33 @@ def poissonRDD(sc, mean, size, numPartitions=None, seed=None):
         return callMLlibFunc("poissonRDD", sc._jsc, float(mean), size, numPartitions, seed)
 
     @staticmethod
-    @since("1.3.0")
     def exponentialRDD(sc, mean, size, numPartitions=None, seed=None):
         """
         Generates an RDD comprised of i.i.d. samples from the Exponential
         distribution with the input mean.
 
-        :param sc: SparkContext used to create the RDD.
-        :param mean: Mean, or 1 / lambda, for the Exponential distribution.
-        :param size: Size of the RDD.
-        :param numPartitions: Number of partitions in the RDD (default: `sc.defaultParallelism`).
-        :param seed: Random seed (default: a random long integer).
-        :return: RDD of float comprised of i.i.d. samples ~ Exp(mean).
-
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        sc : :py:class:`pyspark.SparkContext`
+            SparkContext used to create the RDD.
+        mean : float
+            Mean, or 1 / lambda, for the Exponential distribution.
+        size : int
+            Size of the RDD.
+        numPartitions : int, optional
+            Number of partitions in the RDD (default: `sc.defaultParallelism`).
+        seed : int, optional
+            Random seed (default: a random long integer).
+
+        Returns
+        -------
+        :py:class:`pyspark.RDD`
+            RDD of float comprised of i.i.d. samples ~ Exp(mean).
+
+        Examples
+        --------
         >>> mean = 2.0
         >>> x = RandomRDDs.exponentialRDD(sc, mean, 1000, seed=2)
         >>> stats = x.stats()
@@ -191,20 +258,35 @@ def exponentialRDD(sc, mean, size, numPartitions=None, seed=None):
         return callMLlibFunc("exponentialRDD", sc._jsc, float(mean), size, numPartitions, seed)
 
     @staticmethod
-    @since("1.3.0")
     def gammaRDD(sc, shape, scale, size, numPartitions=None, seed=None):
         """
         Generates an RDD comprised of i.i.d. samples from the Gamma
         distribution with the input shape and scale.
 
-        :param sc: SparkContext used to create the RDD.
-        :param shape: shape (> 0) parameter for the Gamma distribution
-        :param scale: scale (> 0) parameter for the Gamma distribution
-        :param size: Size of the RDD.
-        :param numPartitions: Number of partitions in the RDD (default: `sc.defaultParallelism`).
-        :param seed: Random seed (default: a random long integer).
-        :return: RDD of float comprised of i.i.d. samples ~ Gamma(shape, scale).
-
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        sc : :py:class:`pyspark.SparkContext`
+            SparkContext used to create the RDD.
+        shape : float
+            shape (> 0) parameter for the Gamma distribution
+        scale : float
+            scale (> 0) parameter for the Gamma distribution
+        size : int
+            Size of the RDD.
+        numPartitions : int, optional
+            Number of partitions in the RDD (default: `sc.defaultParallelism`).
+        seed : int, optional
+            Random seed (default: a random long integer).
+
+        Returns
+        -------
+        :py:class:`pyspark.RDD`
+            RDD of float comprised of i.i.d. samples ~ Gamma(shape, scale).
+
+        Examples
+        --------
         >>> from math import sqrt
         >>> shape = 1.0
         >>> scale = 2.0
@@ -224,19 +306,33 @@ def gammaRDD(sc, shape, scale, size, numPartitions=None, seed=None):
 
     @staticmethod
     @toArray
-    @since("1.1.0")
     def uniformVectorRDD(sc, numRows, numCols, numPartitions=None, seed=None):
         """
         Generates an RDD comprised of vectors containing i.i.d. samples drawn
         from the uniform distribution U(0.0, 1.0).
 
-        :param sc: SparkContext used to create the RDD.
-        :param numRows: Number of Vectors in the RDD.
-        :param numCols: Number of elements in each Vector.
-        :param numPartitions: Number of partitions in the RDD.
-        :param seed: Seed for the RNG that generates the seed for the generator in each partition.
-        :return: RDD of Vector with vectors containing i.i.d samples ~ `U(0.0, 1.0)`.
-
+        .. versionadded:: 1.1.0
+
+        Parameters
+        ----------
+        sc : :py:class:`pyspark.SparkContext`
+            SparkContext used to create the RDD.
+        numRows : int
+            Number of Vectors in the RDD.
+        numCols : int
+            Number of elements in each Vector.
+        numPartitions : int, optional
+            Number of partitions in the RDD.
+        seed : int, optional
+            Seed for the RNG that generates the seed for the generator in each partition.
+
+        Returns
+        -------
+        :py:class:`pyspark.RDD`
+            RDD of Vector with vectors containing i.i.d samples ~ `U(0.0, 1.0)`.
+
+        Examples
+        --------
         >>> import numpy as np
         >>> mat = np.matrix(RandomRDDs.uniformVectorRDD(sc, 10, 10).collect())
         >>> mat.shape
@@ -250,19 +346,33 @@ def uniformVectorRDD(sc, numRows, numCols, numPartitions=None, seed=None):
 
     @staticmethod
     @toArray
-    @since("1.1.0")
     def normalVectorRDD(sc, numRows, numCols, numPartitions=None, seed=None):
         """
         Generates an RDD comprised of vectors containing i.i.d. samples drawn
         from the standard normal distribution.
 
-        :param sc: SparkContext used to create the RDD.
-        :param numRows: Number of Vectors in the RDD.
-        :param numCols: Number of elements in each Vector.
-        :param numPartitions: Number of partitions in the RDD (default: `sc.defaultParallelism`).
-        :param seed: Random seed (default: a random long integer).
-        :return: RDD of Vector with vectors containing i.i.d. samples ~ `N(0.0, 1.0)`.
-
+        .. versionadded:: 1.1.0
+
+        Parameters
+        ----------
+        sc : :py:class:`pyspark.SparkContext`
+            SparkContext used to create the RDD.
+        numRows : int
+            Number of Vectors in the RDD.
+        numCols : int
+            Number of elements in each Vector.
+        numPartitions : int, optional
+            Number of partitions in the RDD (default: `sc.defaultParallelism`).
+        seed : int, optional
+            Random seed (default: a random long integer).
+
+        Returns
+        -------
+        :py:class:`pyspark.RDD`
+            RDD of Vector with vectors containing i.i.d. samples ~ `N(0.0, 1.0)`.
+
+        Examples
+        --------
         >>> import numpy as np
         >>> mat = np.matrix(RandomRDDs.normalVectorRDD(sc, 100, 100, seed=1).collect())
         >>> mat.shape
@@ -276,21 +386,37 @@ def normalVectorRDD(sc, numRows, numCols, numPartitions=None, seed=None):
 
     @staticmethod
     @toArray
-    @since("1.3.0")
     def logNormalVectorRDD(sc, mean, std, numRows, numCols, numPartitions=None, seed=None):
         """
         Generates an RDD comprised of vectors containing i.i.d. samples drawn
         from the log normal distribution.
 
-        :param sc: SparkContext used to create the RDD.
-        :param mean: Mean of the log normal distribution
-        :param std: Standard Deviation of the log normal distribution
-        :param numRows: Number of Vectors in the RDD.
-        :param numCols: Number of elements in each Vector.
-        :param numPartitions: Number of partitions in the RDD (default: `sc.defaultParallelism`).
-        :param seed: Random seed (default: a random long integer).
-        :return: RDD of Vector with vectors containing i.i.d. samples ~ log `N(mean, std)`.
-
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        sc : :py:class:`pyspark.SparkContext`
+            SparkContext used to create the RDD.
+        mean : float
+            Mean of the log normal distribution
+        std : float
+            Standard Deviation of the log normal distribution
+        numRows : int
+            Number of Vectors in the RDD.
+        numCols : int
+            Number of elements in each Vector.
+        numPartitions : int, optional
+            Number of partitions in the RDD (default: `sc.defaultParallelism`).
+        seed : int, optional
+            Random seed (default: a random long integer).
+
+        Returns
+        -------
+        :py:class:`pyspark.RDD`
+            RDD of Vector with vectors containing i.i.d. samples ~ log `N(mean, std)`.
+
+        Examples
+        --------
         >>> import numpy as np
         >>> from math import sqrt, exp
         >>> mean = 0.0
@@ -311,20 +437,35 @@ def logNormalVectorRDD(sc, mean, std, numRows, numCols, numPartitions=None, seed
 
     @staticmethod
     @toArray
-    @since("1.1.0")
     def poissonVectorRDD(sc, mean, numRows, numCols, numPartitions=None, seed=None):
         """
         Generates an RDD comprised of vectors containing i.i.d. samples drawn
         from the Poisson distribution with the input mean.
 
-        :param sc: SparkContext used to create the RDD.
-        :param mean: Mean, or lambda, for the Poisson distribution.
-        :param numRows: Number of Vectors in the RDD.
-        :param numCols: Number of elements in each Vector.
-        :param numPartitions: Number of partitions in the RDD (default: `sc.defaultParallelism`)
-        :param seed: Random seed (default: a random long integer).
-        :return: RDD of Vector with vectors containing i.i.d. samples ~ Pois(mean).
-
+        .. versionadded:: 1.1.0
+
+        Parameters
+        ----------
+        sc : :py:class:`pyspark.SparkContext`
+            SparkContext used to create the RDD.
+        mean : float
+            Mean, or lambda, for the Poisson distribution.
+        numRows : float
+            Number of Vectors in the RDD.
+        numCols : int
+            Number of elements in each Vector.
+        numPartitions : int, optional
+            Number of partitions in the RDD (default: `sc.defaultParallelism`)
+        seed : int, optional
+            Random seed (default: a random long integer).
+
+        Returns
+        -------
+        :py:class:`pyspark.RDD`
+            RDD of Vector with vectors containing i.i.d. samples ~ Pois(mean).
+
+        Examples
+        --------
         >>> import numpy as np
         >>> mean = 100.0
         >>> rdd = RandomRDDs.poissonVectorRDD(sc, mean, 100, 100, seed=1)
@@ -342,20 +483,35 @@ def poissonVectorRDD(sc, mean, numRows, numCols, numPartitions=None, seed=None):
 
     @staticmethod
     @toArray
-    @since("1.3.0")
     def exponentialVectorRDD(sc, mean, numRows, numCols, numPartitions=None, seed=None):
         """
         Generates an RDD comprised of vectors containing i.i.d. samples drawn
         from the Exponential distribution with the input mean.
 
-        :param sc: SparkContext used to create the RDD.
-        :param mean: Mean, or 1 / lambda, for the Exponential distribution.
-        :param numRows: Number of Vectors in the RDD.
-        :param numCols: Number of elements in each Vector.
-        :param numPartitions: Number of partitions in the RDD (default: `sc.defaultParallelism`)
-        :param seed: Random seed (default: a random long integer).
-        :return: RDD of Vector with vectors containing i.i.d. samples ~ Exp(mean).
-
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        sc : :py:class:`pyspark.SparkContext`
+            SparkContext used to create the RDD.
+        mean : float
+            Mean, or 1 / lambda, for the Exponential distribution.
+        numRows : int
+            Number of Vectors in the RDD.
+        numCols : int
+            Number of elements in each Vector.
+        numPartitions : int, optional
+            Number of partitions in the RDD (default: `sc.defaultParallelism`)
+        seed : int, optional
+            Random seed (default: a random long integer).
+
+        Returns
+        -------
+        :py:class:`pyspark.RDD`
+            RDD of Vector with vectors containing i.i.d. samples ~ Exp(mean).
+
+        Examples
+        --------
         >>> import numpy as np
         >>> mean = 0.5
         >>> rdd = RandomRDDs.exponentialVectorRDD(sc, mean, 100, 100, seed=1)
@@ -373,21 +529,37 @@ def exponentialVectorRDD(sc, mean, numRows, numCols, numPartitions=None, seed=No
 
     @staticmethod
     @toArray
-    @since("1.3.0")
     def gammaVectorRDD(sc, shape, scale, numRows, numCols, numPartitions=None, seed=None):
         """
         Generates an RDD comprised of vectors containing i.i.d. samples drawn
         from the Gamma distribution.
 
-        :param sc: SparkContext used to create the RDD.
-        :param shape: Shape (> 0) of the Gamma distribution
-        :param scale: Scale (> 0) of the Gamma distribution
-        :param numRows: Number of Vectors in the RDD.
-        :param numCols: Number of elements in each Vector.
-        :param numPartitions: Number of partitions in the RDD (default: `sc.defaultParallelism`).
-        :param seed: Random seed (default: a random long integer).
-        :return: RDD of Vector with vectors containing i.i.d. samples ~ Gamma(shape, scale).
-
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        sc : :py:class:`pyspark.SparkContext`
+            SparkContext used to create the RDD.
+        shape : float
+            Shape (> 0) of the Gamma distribution
+        scale : float
+            Scale (> 0) of the Gamma distribution
+        numRows : int
+            Number of Vectors in the RDD.
+        numCols : int
+            Number of elements in each Vector.
+        numPartitions : int, optional
+            Number of partitions in the RDD (default: `sc.defaultParallelism`).
+        seed : int, optional,
+            Random seed (default: a random long integer).
+
+        Returns
+        -------
+        :py:class:`pyspark.RDD`
+            RDD of Vector with vectors containing i.i.d. samples ~ Gamma(shape, scale).
+
+        Examples
+        --------
         >>> import numpy as np
         >>> from math import sqrt
         >>> shape = 1.0
diff --git a/python/pyspark/mllib/random.pyi b/python/pyspark/mllib/random.pyi
new file mode 100644
index 0000000000000..ec83170625c74
--- /dev/null
+++ b/python/pyspark/mllib/random.pyi
@@ -0,0 +1,126 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Optional
+from pyspark.context import SparkContext
+from pyspark.rdd import RDD
+from pyspark.mllib.linalg import Vector
+
+class RandomRDDs:
+    @staticmethod
+    def uniformRDD(
+        sc: SparkContext,
+        size: int,
+        numPartitions: Optional[int] = ...,
+        seed: Optional[int] = ...,
+    ) -> RDD[float]: ...
+    @staticmethod
+    def normalRDD(
+        sc: SparkContext,
+        size: int,
+        numPartitions: Optional[int] = ...,
+        seed: Optional[int] = ...,
+    ) -> RDD[float]: ...
+    @staticmethod
+    def logNormalRDD(
+        sc: SparkContext,
+        mean: float,
+        std: float,
+        size: int,
+        numPartitions: Optional[int] = ...,
+        seed: Optional[int] = ...,
+    ) -> RDD[float]: ...
+    @staticmethod
+    def poissonRDD(
+        sc: SparkContext,
+        mean: float,
+        size: int,
+        numPartitions: Optional[int] = ...,
+        seed: Optional[int] = ...,
+    ) -> RDD[float]: ...
+    @staticmethod
+    def exponentialRDD(
+        sc: SparkContext,
+        mean: float,
+        size: int,
+        numPartitions: Optional[int] = ...,
+        seed: Optional[int] = ...,
+    ) -> RDD[float]: ...
+    @staticmethod
+    def gammaRDD(
+        sc: SparkContext,
+        shape: float,
+        scale: float,
+        size: int,
+        numPartitions: Optional[int] = ...,
+        seed: Optional[int] = ...,
+    ) -> RDD[float]: ...
+    @staticmethod
+    def uniformVectorRDD(
+        sc: SparkContext,
+        numRows: int,
+        numCols: int,
+        numPartitions: Optional[int] = ...,
+        seed: Optional[int] = ...,
+    ) -> RDD[Vector]: ...
+    @staticmethod
+    def normalVectorRDD(
+        sc: SparkContext,
+        numRows: int,
+        numCols: int,
+        numPartitions: Optional[int] = ...,
+        seed: Optional[int] = ...,
+    ) -> RDD[Vector]: ...
+    @staticmethod
+    def logNormalVectorRDD(
+        sc: SparkContext,
+        mean: float,
+        std: float,
+        numRows: int,
+        numCols: int,
+        numPartitions: Optional[int] = ...,
+        seed: Optional[int] = ...,
+    ) -> RDD[Vector]: ...
+    @staticmethod
+    def poissonVectorRDD(
+        sc: SparkContext,
+        mean: float,
+        numRows: int,
+        numCols: int,
+        numPartitions: Optional[int] = ...,
+        seed: Optional[int] = ...,
+    ) -> RDD[Vector]: ...
+    @staticmethod
+    def exponentialVectorRDD(
+        sc: SparkContext,
+        mean: float,
+        numRows: int,
+        numCols: int,
+        numPartitions: Optional[int] = ...,
+        seed: Optional[int] = ...,
+    ) -> RDD[Vector]: ...
+    @staticmethod
+    def gammaVectorRDD(
+        sc: SparkContext,
+        shape: float,
+        scale: float,
+        numRows: int,
+        numCols: int,
+        numPartitions: Optional[int] = ...,
+        seed: Optional[int] = ...,
+    ) -> RDD[Vector]: ...
diff --git a/python/pyspark/mllib/recommendation.py b/python/pyspark/mllib/recommendation.py
index 3dd7cb200c280..7a5fb6e6eea9e 100644
--- a/python/pyspark/mllib/recommendation.py
+++ b/python/pyspark/mllib/recommendation.py
@@ -32,13 +32,15 @@ class Rating(namedtuple("Rating", ["user", "product", "rating"])):
     """
     Represents a (user, product, rating) tuple.
 
+    .. versionadded:: 1.2.0
+
+    Examples
+    --------
     >>> r = Rating(1, 2, 5.0)
     >>> (r.user, r.product, r.rating)
     (1, 2, 5.0)
     >>> (r[0], r[1], r[2])
     (1, 2, 5.0)
-
-    .. versionadded:: 1.2.0
     """
 
     def __reduce__(self):
@@ -51,6 +53,10 @@ class MatrixFactorizationModel(JavaModelWrapper, JavaSaveable, JavaLoader):
     """A matrix factorisation model trained by regularized alternating
     least-squares.
 
+    .. versionadded:: 0.9.0
+
+    Examples
+    --------
     >>> r1 = (1, 1, 1.0)
     >>> r2 = (1, 2, 2.0)
     >>> r3 = (2, 1, 2.0)
@@ -126,8 +132,6 @@ class MatrixFactorizationModel(JavaModelWrapper, JavaSaveable, JavaLoader):
     ...     rmtree(path)
     ... except OSError:
     ...     pass
-
-    .. versionadded:: 0.9.0
     """
     @since("0.9.0")
     def predict(self, user, product):
@@ -237,7 +241,6 @@ def _prepare(cls, ratings):
         return ratings
 
     @classmethod
-    @since("0.9.0")
     def train(cls, ratings, rank, iterations=5, lambda_=0.01, blocks=-1, nonnegative=False,
               seed=None):
         """
@@ -247,35 +250,38 @@ def train(cls, ratings, rank, iterations=5, lambda_=0.01, blocks=-1, nonnegative
         features). To solve for these features, ALS is run iteratively with
         a configurable level of parallelism.
 
-        :param ratings:
-          RDD of `Rating` or (userID, productID, rating) tuple.
-        :param rank:
-          Number of features to use (also referred to as the number of latent factors).
-        :param iterations:
-          Number of iterations of ALS.
-          (default: 5)
-        :param lambda_:
-          Regularization parameter.
-          (default: 0.01)
-        :param blocks:
-          Number of blocks used to parallelize the computation. A value
-          of -1 will use an auto-configured number of blocks.
-          (default: -1)
-        :param nonnegative:
-          A value of True will solve least-squares with nonnegativity
-          constraints.
-          (default: False)
-        :param seed:
-          Random seed for initial matrix factorization model. A value
-          of None will use system time as the seed.
-          (default: None)
+        .. versionadded:: 0.9.0
+
+        Parameters
+        ----------
+        ratings : :py:class:`pyspark.RDD`
+            RDD of `Rating` or (userID, productID, rating) tuple.
+        rank : int
+            Number of features to use (also referred to as the number of latent factors).
+        iterations : int, optional
+            Number of iterations of ALS.
+            (default: 5)
+        lambda\\_ : float, optional
+            Regularization parameter.
+            (default: 0.01)
+        blocks : int, optional
+            Number of blocks used to parallelize the computation. A value
+            of -1 will use an auto-configured number of blocks.
+            (default: -1)
+        nonnegative : bool, optional
+            A value of True will solve least-squares with nonnegativity
+            constraints.
+            (default: False)
+        seed : bool, optional
+            Random seed for initial matrix factorization model. A value
+            of None will use system time as the seed.
+            (default: None)
         """
         model = callMLlibFunc("trainALSModel", cls._prepare(ratings), rank, iterations,
                               lambda_, blocks, nonnegative, seed)
         return MatrixFactorizationModel(model)
 
     @classmethod
-    @since("0.9.0")
     def trainImplicit(cls, ratings, rank, iterations=5, lambda_=0.01, blocks=-1, alpha=0.01,
                       nonnegative=False, seed=None):
         """
@@ -285,31 +291,35 @@ def trainImplicit(cls, ratings, rank, iterations=5, lambda_=0.01, blocks=-1, alp
         given rank (number of features). To solve for these features, ALS
         is run iteratively with a configurable level of parallelism.
 
-        :param ratings:
-          RDD of `Rating` or (userID, productID, rating) tuple.
-        :param rank:
-          Number of features to use (also referred to as the number of latent factors).
-        :param iterations:
-          Number of iterations of ALS.
-          (default: 5)
-        :param lambda_:
-          Regularization parameter.
-          (default: 0.01)
-        :param blocks:
-          Number of blocks used to parallelize the computation. A value
-          of -1 will use an auto-configured number of blocks.
-          (default: -1)
-        :param alpha:
-          A constant used in computing confidence.
-          (default: 0.01)
-        :param nonnegative:
-          A value of True will solve least-squares with nonnegativity
-          constraints.
-          (default: False)
-        :param seed:
-          Random seed for initial matrix factorization model. A value
-          of None will use system time as the seed.
-          (default: None)
+        .. versionadded:: 0.9.0
+
+        Parameters
+        ----------
+        ratings : :py:class:`pyspark.RDD`
+            RDD of `Rating` or (userID, productID, rating) tuple.
+        rank : int
+            Number of features to use (also referred to as the number of latent factors).
+        iterations : int, optional
+            Number of iterations of ALS.
+            (default: 5)
+        lambda\\_ : float, optional
+            Regularization parameter.
+            (default: 0.01)
+        blocks : int, optional
+            Number of blocks used to parallelize the computation. A value
+            of -1 will use an auto-configured number of blocks.
+            (default: -1)
+        alpha : float, optional
+            A constant used in computing confidence.
+            (default: 0.01)
+        nonnegative : bool, optional
+            A value of True will solve least-squares with nonnegativity
+            constraints.
+            (default: False)
+        seed : int, optional
+            Random seed for initial matrix factorization model. A value
+            of None will use system time as the seed.
+            (default: None)
         """
         model = callMLlibFunc("trainImplicitALSModel", cls._prepare(ratings), rank,
                               iterations, lambda_, blocks, alpha, nonnegative, seed)
diff --git a/python/pyspark/mllib/recommendation.pyi b/python/pyspark/mllib/recommendation.pyi
new file mode 100644
index 0000000000000..4fea0acf3c1f9
--- /dev/null
+++ b/python/pyspark/mllib/recommendation.pyi
@@ -0,0 +1,75 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import List, Optional, Tuple, Type, Union
+
+import array
+from collections import namedtuple
+
+from pyspark.context import SparkContext
+from pyspark.rdd import RDD
+from pyspark.mllib.common import JavaModelWrapper
+from pyspark.mllib.util import JavaLoader, JavaSaveable
+
+class Rating(namedtuple("Rating", ["user", "product", "rating"])):
+    def __reduce__(self) -> Tuple[Type[Rating], Tuple[int, int, float]]: ...
+
+class MatrixFactorizationModel(
+    JavaModelWrapper, JavaSaveable, JavaLoader[MatrixFactorizationModel]
+):
+    def predict(self, user: int, product: int) -> float: ...
+    def predictAll(self, user_product: RDD[Tuple[int, int]]) -> RDD[Rating]: ...
+    def userFeatures(self) -> RDD[Tuple[int, array.array]]: ...
+    def productFeatures(self) -> RDD[Tuple[int, array.array]]: ...
+    def recommendUsers(self, product: int, num: int) -> List[Rating]: ...
+    def recommendProducts(self, user: int, num: int) -> List[Rating]: ...
+    def recommendProductsForUsers(
+        self, num: int
+    ) -> RDD[Tuple[int, Tuple[Rating, ...]]]: ...
+    def recommendUsersForProducts(
+        self, num: int
+    ) -> RDD[Tuple[int, Tuple[Rating, ...]]]: ...
+    @property
+    def rank(self) -> int: ...
+    @classmethod
+    def load(cls, sc: SparkContext, path: str) -> MatrixFactorizationModel: ...
+
+class ALS:
+    @classmethod
+    def train(
+        cls,
+        ratings: Union[RDD[Rating], RDD[Tuple[int, int, float]]],
+        rank: int,
+        iterations: int = ...,
+        lambda_: float = ...,
+        blocks: int = ...,
+        nonnegative: bool = ...,
+        seed: Optional[int] = ...,
+    ) -> MatrixFactorizationModel: ...
+    @classmethod
+    def trainImplicit(
+        cls,
+        ratings: Union[RDD[Rating], RDD[Tuple[int, int, float]]],
+        rank: int,
+        iterations: int = ...,
+        lambda_: float = ...,
+        blocks: int = ...,
+        alpha: float = ...,
+        nonnegative: bool = ...,
+        seed: Optional[int] = ...,
+    ) -> MatrixFactorizationModel: ...
diff --git a/python/pyspark/mllib/regression.py b/python/pyspark/mllib/regression.py
index 56ee0083abca4..3908e4abfaed5 100644
--- a/python/pyspark/mllib/regression.py
+++ b/python/pyspark/mllib/regression.py
@@ -23,7 +23,7 @@
 from pyspark import RDD, since
 from pyspark.streaming.dstream import DStream
 from pyspark.mllib.common import callMLlibFunc, _py2java, _java2py, inherit_doc
-from pyspark.mllib.linalg import SparseVector, _convert_to_vector
+from pyspark.mllib.linalg import _convert_to_vector
 from pyspark.mllib.util import Saveable, Loader
 
 __all__ = ['LabeledPoint', 'LinearModel',
@@ -39,15 +39,19 @@ class LabeledPoint(object):
     """
     Class that represents the features and labels of a data point.
 
-    :param label:
-      Label for this data point.
-    :param features:
-      Vector of features for this point (NumPy array, list,
-      pyspark.mllib.linalg.SparseVector, or scipy.sparse column matrix).
-
-    .. note:: 'label' and 'features' are accessible as class attributes.
-
     .. versionadded:: 1.0.0
+
+    Parameters
+    ----------
+    label : int
+        Label for this data point.
+    features : :py:class:`pyspark.mllib.linalg.Vector` or convertible
+        Vector of features for this point (NumPy array, list,
+        pyspark.mllib.linalg.SparseVector, or scipy.sparse column matrix).
+
+    Notes
+    -----
+    'label' and 'features' are accessible as class attributes.
     """
 
     def __init__(self, label, features):
@@ -69,12 +73,14 @@ class LinearModel(object):
     """
     A linear model that has a vector of coefficients and an intercept.
 
-    :param weights:
-      Weights computed for every feature.
-    :param intercept:
-      Intercept computed for this model.
-
     .. versionadded:: 0.9.0
+
+    Parameters
+    ----------
+    weights : :py:class:`pyspark.mllib.linalg.Vector`
+        Weights computed for every feature.
+    intercept : float
+      Intercept computed for this model.
     """
 
     def __init__(self, weights, intercept):
@@ -102,13 +108,16 @@ class LinearRegressionModelBase(LinearModel):
 
     """A linear regression model.
 
+    .. versionadded:: 0.9.0
+
+    Examples
+    --------
+    >>> from pyspark.mllib.linalg import SparseVector
     >>> lrmb = LinearRegressionModelBase(np.array([1.0, 2.0]), 0.1)
     >>> abs(lrmb.predict(np.array([-1.03, 7.777])) - 14.624) < 1e-6
     True
     >>> abs(lrmb.predict(SparseVector(2, {0: -1.03, 1: 7.777})) - 14.624) < 1e-6
     True
-
-    .. versionadded:: 0.9.0
     """
 
     @since("0.9.0")
@@ -128,6 +137,11 @@ class LinearRegressionModel(LinearRegressionModelBase):
 
     """A linear regression model derived from a least-squares fit.
 
+    .. versionadded:: 0.9.0
+
+    Examples
+    --------
+    >>> from pyspark.mllib.linalg import SparseVector
     >>> from pyspark.mllib.regression import LabeledPoint
     >>> data = [
     ...     LabeledPoint(0.0, [0.0]),
@@ -179,8 +193,6 @@ class LinearRegressionModel(LinearRegressionModelBase):
     True
     >>> abs(lrm.predict(SparseVector(1, {0: 1.0})) - 1) < 0.5
     True
-
-    .. versionadded:: 0.9.0
     """
     @since("1.4.0")
     def save(self, sc, path):
@@ -222,11 +234,13 @@ def _regression_train_wrapper(train_func, modelClass, data, initial_weights):
 
 class LinearRegressionWithSGD(object):
     """
+    Train a linear regression model with no regularization using Stochastic Gradient Descent.
+
     .. versionadded:: 0.9.0
-    .. note:: Deprecated in 2.0.0. Use ml.regression.LinearRegression.
+    .. deprecated:: 2.0.0
+        Use :py:class:`pyspark.ml.regression.LinearRegression`.
     """
     @classmethod
-    @since("0.9.0")
     def train(cls, data, iterations=100, step=1.0, miniBatchFraction=1.0,
               initialWeights=None, regParam=0.0, regType=None, intercept=False,
               validateData=True, convergenceTol=0.001):
@@ -242,45 +256,50 @@ def train(cls, data, iterations=100, step=1.0, miniBatchFraction=1.0,
         corresponding right hand side label y.
         See also the documentation for the precise formulation.
 
-        :param data:
-          The training data, an RDD of LabeledPoint.
-        :param iterations:
-          The number of iterations.
-          (default: 100)
-        :param step:
-          The step parameter used in SGD.
-          (default: 1.0)
-        :param miniBatchFraction:
-          Fraction of data to be used for each SGD iteration.
-          (default: 1.0)
-        :param initialWeights:
-          The initial weights.
-          (default: None)
-        :param regParam:
-          The regularizer parameter.
-          (default: 0.0)
-        :param regType:
-          The type of regularizer used for training our model.
-          Supported values:
+        .. versionadded:: 0.9.0
+
+        Parameters
+        ----------
+        data : :py:class:`pyspark.RDD`
+            The training data, an RDD of LabeledPoint.
+        iterations : int, optional
+            The number of iterations.
+            (default: 100)
+        step : float, optional
+            The step parameter used in SGD.
+            (default: 1.0)
+        miniBatchFraction : float, optional
+            Fraction of data to be used for each SGD iteration.
+            (default: 1.0)
+        initialWeights : :py:class:`pyspark.mllib.linalg.Vector` or convertible, optional
+            The initial weights.
+            (default: None)
+        regParam : float, optional
+            The regularizer parameter.
+            (default: 0.0)
+        regType : str, optional
+            The type of regularizer used for training our model.
+            Supported values:
 
             - "l1" for using L1 regularization
             - "l2" for using L2 regularization
             - None for no regularization (default)
-        :param intercept:
-          Boolean parameter which indicates the use or not of the
-          augmented representation for training data (i.e., whether bias
-          features are activated or not).
-          (default: False)
-        :param validateData:
-          Boolean parameter which indicates if the algorithm should
-          validate data before training.
-          (default: True)
-        :param convergenceTol:
-          A condition which decides iteration termination.
-          (default: 0.001)
+
+        intercept : bool, optional
+            Boolean parameter which indicates the use or not of the
+            augmented representation for training data (i.e., whether bias
+            features are activated or not).
+            (default: False)
+        validateData : bool, optional
+            Boolean parameter which indicates if the algorithm should
+            validate data before training.
+            (default: True)
+        convergenceTol : float, optional
+            A condition which decides iteration termination.
+            (default: 0.001)
         """
         warnings.warn(
-            "Deprecated in 2.0.0. Use ml.regression.LinearRegression.", DeprecationWarning)
+            "Deprecated in 2.0.0. Use ml.regression.LinearRegression.", FutureWarning)
 
         def train(rdd, i):
             return callMLlibFunc("trainLinearRegressionModelWithSGD", rdd, int(iterations),
@@ -297,6 +316,11 @@ class LassoModel(LinearRegressionModelBase):
     """A linear regression model derived from a least-squares fit with
     an l_1 penalty term.
 
+    .. versionadded:: 0.9.0
+
+    Examples
+    --------
+    >>> from pyspark.mllib.linalg import SparseVector
     >>> from pyspark.mllib.regression import LabeledPoint
     >>> data = [
     ...     LabeledPoint(0.0, [0.0]),
@@ -348,8 +372,6 @@ class LassoModel(LinearRegressionModelBase):
     True
     >>> abs(lrm.predict(SparseVector(1, {0: 1.0})) - 1) < 0.5
     True
-
-    .. versionadded:: 0.9.0
     """
     @since("1.4.0")
     def save(self, sc, path):
@@ -372,12 +394,14 @@ def load(cls, sc, path):
 
 class LassoWithSGD(object):
     """
+    Train a regression model with L1-regularization using Stochastic Gradient Descent.
+
     .. versionadded:: 0.9.0
-    .. note:: Deprecated in 2.0.0. Use ml.regression.LinearRegression with elasticNetParam = 1.0.
-            Note the default regParam is 0.01 for LassoWithSGD, but is 0.0 for LinearRegression.
+    .. deprecated:: 2.0.0
+        Use :py:class:`pyspark.ml.regression.LinearRegression` with elasticNetParam = 1.0.
+        Note the default regParam is 0.01 for LassoWithSGD, but is 0.0 for LinearRegression.
     """
     @classmethod
-    @since("0.9.0")
     def train(cls, data, iterations=100, step=1.0, regParam=0.01,
               miniBatchFraction=1.0, initialWeights=None, intercept=False,
               validateData=True, convergenceTol=0.001):
@@ -392,40 +416,45 @@ def train(cls, data, iterations=100, step=1.0, regParam=0.01,
         of rows of A, each with its corresponding right hand side label y.
         See also the documentation for the precise formulation.
 
-        :param data:
-          The training data, an RDD of LabeledPoint.
-        :param iterations:
-          The number of iterations.
-          (default: 100)
-        :param step:
-          The step parameter used in SGD.
-          (default: 1.0)
-        :param regParam:
-          The regularizer parameter.
-          (default: 0.01)
-        :param miniBatchFraction:
-          Fraction of data to be used for each SGD iteration.
-          (default: 1.0)
-        :param initialWeights:
-          The initial weights.
-          (default: None)
-        :param intercept:
-          Boolean parameter which indicates the use or not of the
-          augmented representation for training data (i.e. whether bias
-          features are activated or not).
-          (default: False)
-        :param validateData:
-          Boolean parameter which indicates if the algorithm should
-          validate data before training.
-          (default: True)
-        :param convergenceTol:
-          A condition which decides iteration termination.
-          (default: 0.001)
+        .. versionadded:: 0.9.0
+
+        Parameters
+        ----------
+        data : :py:class:`pyspark.RDD`
+            The training data, an RDD of LabeledPoint.
+        iterations : int, optional
+            The number of iterations.
+            (default: 100)
+        step : float, optional
+            The step parameter used in SGD.
+            (default: 1.0)
+        regParam : float, optional
+            The regularizer parameter.
+            (default: 0.01)
+        miniBatchFraction : float, optional
+            Fraction of data to be used for each SGD iteration.
+            (default: 1.0)
+        initialWeights : :py:class:`pyspark.mllib.linalg.Vector` or convertible, optional
+            The initial weights.
+            (default: None)
+        intercept : bool, optional
+            Boolean parameter which indicates the use or not of the
+            augmented representation for training data (i.e. whether bias
+            features are activated or not).
+            (default: False)
+        validateData : bool, optional
+            Boolean parameter which indicates if the algorithm should
+            validate data before training.
+            (default: True)
+        convergenceTol : float, optional
+            A condition which decides iteration termination.
+            (default: 0.001)
         """
         warnings.warn(
             "Deprecated in 2.0.0. Use ml.regression.LinearRegression with elasticNetParam = 1.0. "
             "Note the default regParam is 0.01 for LassoWithSGD, but is 0.0 for LinearRegression.",
-            DeprecationWarning)
+            FutureWarning
+        )
 
         def train(rdd, i):
             return callMLlibFunc("trainLassoModelWithSGD", rdd, int(iterations), float(step),
@@ -441,6 +470,11 @@ class RidgeRegressionModel(LinearRegressionModelBase):
     """A linear regression model derived from a least-squares fit with
     an l_2 penalty term.
 
+    .. versionadded:: 0.9.0
+
+    Examples
+    --------
+    >>> from pyspark.mllib.linalg import SparseVector
     >>> from pyspark.mllib.regression import LabeledPoint
     >>> data = [
     ...     LabeledPoint(0.0, [0.0]),
@@ -492,8 +526,6 @@ class RidgeRegressionModel(LinearRegressionModelBase):
     True
     >>> abs(lrm.predict(SparseVector(1, {0: 1.0})) - 1) < 0.5
     True
-
-    .. versionadded:: 0.9.0
     """
     @since("1.4.0")
     def save(self, sc, path):
@@ -516,13 +548,15 @@ def load(cls, sc, path):
 
 class RidgeRegressionWithSGD(object):
     """
+    Train a regression model with L2-regularization using Stochastic Gradient Descent.
+
     .. versionadded:: 0.9.0
-    .. note:: Deprecated in 2.0.0. Use ml.regression.LinearRegression with elasticNetParam = 0.0.
-            Note the default regParam is 0.01 for RidgeRegressionWithSGD, but is 0.0 for
-            LinearRegression.
+    .. deprecated:: 2.0.0
+        Use :py:class:`pyspark.ml.regression.LinearRegression` with elasticNetParam = 0.0.
+        Note the default regParam is 0.01 for RidgeRegressionWithSGD, but is 0.0 for
+        LinearRegression.
     """
     @classmethod
-    @since("0.9.0")
     def train(cls, data, iterations=100, step=1.0, regParam=0.01,
               miniBatchFraction=1.0, initialWeights=None, intercept=False,
               validateData=True, convergenceTol=0.001):
@@ -537,40 +571,44 @@ def train(cls, data, iterations=100, step=1.0, regParam=0.01,
         of rows of A, each with its corresponding right hand side label y.
         See also the documentation for the precise formulation.
 
-        :param data:
-          The training data, an RDD of LabeledPoint.
-        :param iterations:
-          The number of iterations.
-          (default: 100)
-        :param step:
-          The step parameter used in SGD.
-          (default: 1.0)
-        :param regParam:
-          The regularizer parameter.
-          (default: 0.01)
-        :param miniBatchFraction:
-          Fraction of data to be used for each SGD iteration.
-          (default: 1.0)
-        :param initialWeights:
-          The initial weights.
-          (default: None)
-        :param intercept:
-          Boolean parameter which indicates the use or not of the
-          augmented representation for training data (i.e. whether bias
-          features are activated or not).
-          (default: False)
-        :param validateData:
-          Boolean parameter which indicates if the algorithm should
-          validate data before training.
-          (default: True)
-        :param convergenceTol:
-          A condition which decides iteration termination.
-          (default: 0.001)
+        .. versionadded:: 0.9.0
+
+        Parameters
+        ----------
+        data : :py:class:`pyspark.RDD`
+            The training data, an RDD of LabeledPoint.
+        iterations : int, optional
+            The number of iterations.
+            (default: 100)
+        step : float, optional
+            The step parameter used in SGD.
+            (default: 1.0)
+        regParam : float, optional
+            The regularizer parameter.
+            (default: 0.01)
+        miniBatchFraction : float, optional
+            Fraction of data to be used for each SGD iteration.
+            (default: 1.0)
+        initialWeights : :py:class:`pyspark.mllib.linalg.Vector` or convertible, optional
+            The initial weights.
+            (default: None)
+        intercept : bool, optional
+            Boolean parameter which indicates the use or not of the
+            augmented representation for training data (i.e. whether bias
+            features are activated or not).
+            (default: False)
+        validateData : bool, optional
+            Boolean parameter which indicates if the algorithm should
+            validate data before training.
+            (default: True)
+        convergenceTol : float, optional
+            A condition which decides iteration termination.
+            (default: 0.001)
         """
         warnings.warn(
             "Deprecated in 2.0.0. Use ml.regression.LinearRegression with elasticNetParam = 0.0. "
             "Note the default regParam is 0.01 for RidgeRegressionWithSGD, but is 0.0 for "
-            "LinearRegression.", DeprecationWarning)
+            "LinearRegression.", FutureWarning)
 
         def train(rdd, i):
             return callMLlibFunc("trainRidgeModelWithSGD", rdd, int(iterations), float(step),
@@ -585,15 +623,21 @@ class IsotonicRegressionModel(Saveable, Loader):
     """
     Regression model for isotonic regression.
 
-    :param boundaries:
-      Array of boundaries for which predictions are known. Boundaries
-      must be sorted in increasing order.
-    :param predictions:
-      Array of predictions associated to the boundaries at the same
-      index. Results of isotonic regression and therefore monotone.
-    :param isotonic:
-      Indicates whether this is isotonic or antitonic.
+    .. versionadded:: 1.4.0
 
+    Parameters
+    ----------
+    boundaries : ndarray
+        Array of boundaries for which predictions are known. Boundaries
+        must be sorted in increasing order.
+    predictions : ndarray
+        Array of predictions associated to the boundaries at the same
+        index. Results of isotonic regression and therefore monotone.
+    isotonic : true
+        Indicates whether this is isotonic or antitonic.
+
+    Examples
+    --------
     >>> data = [(1, 0, 1), (2, 1, 1), (3, 2, 1), (1, 3, 1), (6, 4, 1), (17, 5, 1), (16, 6, 1)]
     >>> irm = IsotonicRegression.train(sc.parallelize(data))
     >>> irm.predict(3)
@@ -615,8 +659,6 @@ class IsotonicRegressionModel(Saveable, Loader):
     ...     rmtree(path)
     ... except OSError:
     ...     pass
-
-    .. versionadded:: 1.4.0
     """
 
     def __init__(self, boundaries, predictions, isotonic):
@@ -624,7 +666,6 @@ def __init__(self, boundaries, predictions, isotonic):
         self.predictions = predictions
         self.isotonic = isotonic
 
-    @since("1.4.0")
     def predict(self, x):
         """
         Predict labels for provided features.
@@ -643,8 +684,13 @@ def predict(self, x):
         values with the same boundary then the same rules as in 2)
         are used.
 
-        :param x:
-          Feature or RDD of Features to be labeled.
+
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        x : :py:class:`pyspark.mllib.linalg.Vector` or :py:class:`pyspark.RDD`
+            Feature or RDD of Features to be labeled.
         """
         if isinstance(x, RDD):
             return x.map(lambda v: self.predict(v))
@@ -676,35 +722,42 @@ class IsotonicRegression(object):
     Currently implemented using parallelized pool adjacent violators
     algorithm. Only univariate (single feature) algorithm supported.
 
-    Sequential PAV implementation based on:
+    .. versionadded:: 1.4.0
+
+    Notes
+    -----
+    Sequential PAV implementation based on
+    Tibshirani, Ryan J., Holger Hoefling, and Robert Tibshirani (2011) [1]_
 
-      Tibshirani, Ryan J., Holger Hoefling, and Robert Tibshirani.
-      "Nearly-isotonic regression." Technometrics 53.1 (2011): 54-61.
-      Available from http://www.stat.cmu.edu/~ryantibs/papers/neariso.pdf
+    Sequential PAV parallelization based on
+    Kearsley, Anthony J., Richard A. Tapia, and Michael W. Trosset (1996) [2]_
 
-    Sequential PAV parallelization based on:
+    See also
+    `Isotonic regression (Wikipedia) <http://en.wikipedia.org/wiki/Isotonic_regression>`_.
 
-        Kearsley, Anthony J., Richard A. Tapia, and Michael W. Trosset.
+    .. [1] Tibshirani, Ryan J., Holger Hoefling, and Robert Tibshirani.
+        "Nearly-isotonic regression." Technometrics 53.1 (2011): 54-61.
+        Available from http://www.stat.cmu.edu/~ryantibs/papers/neariso.pdf
+    .. [2] Kearsley, Anthony J., Richard A. Tapia, and Michael W. Trosset
         "An approach to parallelizing isotonic regression."
         Applied Mathematics and Parallel Computing. Physica-Verlag HD, 1996. 141-147.
         Available from http://softlib.rice.edu/pub/CRPC-TRs/reports/CRPC-TR96640.pdf
-
-    See `Isotonic regression (Wikipedia) <http://en.wikipedia.org/wiki/Isotonic_regression>`_.
-
-    .. versionadded:: 1.4.0
     """
 
     @classmethod
-    @since("1.4.0")
     def train(cls, data, isotonic=True):
         """
         Train an isotonic regression model on the given data.
 
-        :param data:
-          RDD of (label, feature, weight) tuples.
-        :param isotonic:
-          Whether this is isotonic (which is default) or antitonic.
-          (default: True)
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        data : :py:class:`pyspark.RDD`
+            RDD of (label, feature, weight) tuples.
+        isotonic : bool, optional
+            Whether this is isotonic (which is default) or antitonic.
+            (default: True)
         """
         boundaries, predictions = callMLlibFunc("trainIsotonicRegressionModel",
                                                 data.map(_convert_to_vector), bool(isotonic))
@@ -735,28 +788,34 @@ def _validate(self, dstream):
                 "dstream should be a DStream object, got %s" % type(dstream))
         if not self._model:
             raise ValueError(
-                "Model must be intialized using setInitialWeights")
+                "Model must be initialized using setInitialWeights")
 
-    @since("1.5.0")
     def predictOn(self, dstream):
         """
         Use the model to make predictions on batches of data from a
         DStream.
 
-        :return:
-          DStream containing predictions.
+        .. versionadded:: 1.5.0
+
+        Returns
+        -------
+        :py:class:`pyspark.streaming.DStream`
+            DStream containing predictions.
         """
         self._validate(dstream)
         return dstream.map(lambda x: self._model.predict(x))
 
-    @since("1.5.0")
     def predictOnValues(self, dstream):
         """
         Use the model to make predictions on the values of a DStream and
         carry over its keys.
 
-        :return:
-          DStream containing the input keys and the predictions as values.
+        .. versionadded:: 1.5.0
+
+        Returns
+        -------
+        :py:class:`pyspark.streaming.DStream`
+            DStream containing predictions.
         """
         self._validate(dstream)
         return dstream.mapValues(lambda x: self._model.predict(x))
@@ -775,20 +834,22 @@ class StreamingLinearRegressionWithSGD(StreamingLinearAlgorithm):
     of features must be constant. An initial weight vector must
     be provided.
 
-    :param stepSize:
-      Step size for each iteration of gradient descent.
-      (default: 0.1)
-    :param numIterations:
-      Number of iterations run for each batch of data.
-      (default: 50)
-    :param miniBatchFraction:
-      Fraction of each batch of data to use for updates.
-      (default: 1.0)
-    :param convergenceTol:
-      Value used to determine when to terminate iterations.
-      (default: 0.001)
-
     .. versionadded:: 1.5.0
+
+    Parameters
+    ----------
+    stepSize : float, optional
+        Step size for each iteration of gradient descent.
+        (default: 0.1)
+    numIterations : int, optional
+        Number of iterations run for each batch of data.
+        (default: 50)
+    miniBatchFraction : float, optional
+        Fraction of each batch of data to use for updates.
+        (default: 1.0)
+    convergenceTol : float, optional
+        Value used to determine when to terminate iterations.
+        (default: 0.001)
     """
     def __init__(self, stepSize=0.1, numIterations=50, miniBatchFraction=1.0, convergenceTol=0.001):
         self.stepSize = stepSize
diff --git a/python/pyspark/mllib/regression.pyi b/python/pyspark/mllib/regression.pyi
new file mode 100644
index 0000000000000..0283378b98cf3
--- /dev/null
+++ b/python/pyspark/mllib/regression.pyi
@@ -0,0 +1,155 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import overload
+from typing import Iterable, Optional, Tuple, TypeVar
+from pyspark.rdd import RDD
+from pyspark.mllib._typing import VectorLike
+from pyspark.context import SparkContext
+from pyspark.mllib.linalg import Vector
+from pyspark.mllib.util import Saveable, Loader
+from pyspark.streaming.dstream import DStream
+from numpy import ndarray  # type: ignore[import]
+
+K = TypeVar("K")
+
+class LabeledPoint:
+    label: int
+    features: Vector
+    def __init__(self, label: float, features: Iterable[float]) -> None: ...
+    def __reduce__(self) -> Tuple[type, Tuple[bytes]]: ...
+
+class LinearModel:
+    def __init__(self, weights: Vector, intercept: float) -> None: ...
+    @property
+    def weights(self) -> Vector: ...
+    @property
+    def intercept(self) -> float: ...
+
+class LinearRegressionModelBase(LinearModel):
+    @overload
+    def predict(self, x: Vector) -> float: ...
+    @overload
+    def predict(self, x: RDD[Vector]) -> RDD[float]: ...
+
+class LinearRegressionModel(LinearRegressionModelBase):
+    def save(self, sc: SparkContext, path: str) -> None: ...
+    @classmethod
+    def load(cls, sc: SparkContext, path: str) -> LinearRegressionModel: ...
+
+class LinearRegressionWithSGD:
+    @classmethod
+    def train(
+        cls,
+        data: RDD[LabeledPoint],
+        iterations: int = ...,
+        step: float = ...,
+        miniBatchFraction: float = ...,
+        initialWeights: Optional[VectorLike] = ...,
+        regParam: float = ...,
+        regType: Optional[str] = ...,
+        intercept: bool = ...,
+        validateData: bool = ...,
+        convergenceTol: float = ...,
+    ) -> LinearRegressionModel: ...
+
+class LassoModel(LinearRegressionModelBase):
+    def save(self, sc: SparkContext, path: str) -> None: ...
+    @classmethod
+    def load(cls, sc: SparkContext, path: str) -> LassoModel: ...
+
+class LassoWithSGD:
+    @classmethod
+    def train(
+        cls,
+        data: RDD[LabeledPoint],
+        iterations: int = ...,
+        step: float = ...,
+        regParam: float = ...,
+        miniBatchFraction: float = ...,
+        initialWeights: Optional[VectorLike] = ...,
+        intercept: bool = ...,
+        validateData: bool = ...,
+        convergenceTol: float = ...,
+    ) -> LassoModel: ...
+
+class RidgeRegressionModel(LinearRegressionModelBase):
+    def save(self, sc: SparkContext, path: str) -> None: ...
+    @classmethod
+    def load(cls, sc: SparkContext, path: str) -> RidgeRegressionModel: ...
+
+class RidgeRegressionWithSGD:
+    @classmethod
+    def train(
+        cls,
+        data: RDD[LabeledPoint],
+        iterations: int = ...,
+        step: float = ...,
+        regParam: float = ...,
+        miniBatchFraction: float = ...,
+        initialWeights: Optional[VectorLike] = ...,
+        intercept: bool = ...,
+        validateData: bool = ...,
+        convergenceTol: float = ...,
+    ) -> RidgeRegressionModel: ...
+
+class IsotonicRegressionModel(Saveable, Loader[IsotonicRegressionModel]):
+    boundaries: ndarray
+    predictions: ndarray
+    isotonic: bool
+    def __init__(
+        self, boundaries: ndarray, predictions: ndarray, isotonic: bool
+    ) -> None: ...
+    @overload
+    def predict(self, x: Vector) -> ndarray: ...
+    @overload
+    def predict(self, x: RDD[Vector]) -> RDD[ndarray]: ...
+    def save(self, sc: SparkContext, path: str) -> None: ...
+    @classmethod
+    def load(cls, sc: SparkContext, path: str) -> IsotonicRegressionModel: ...
+
+class IsotonicRegression:
+    @classmethod
+    def train(
+        cls, data: RDD[VectorLike], isotonic: bool = ...
+    ) -> IsotonicRegressionModel: ...
+
+class StreamingLinearAlgorithm:
+    def __init__(self, model: LinearModel) -> None: ...
+    def latestModel(self) -> LinearModel: ...
+    def predictOn(self, dstream: DStream[VectorLike]) -> DStream[float]: ...
+    def predictOnValues(
+        self, dstream: DStream[Tuple[K, VectorLike]]
+    ) -> DStream[Tuple[K, float]]: ...
+
+class StreamingLinearRegressionWithSGD(StreamingLinearAlgorithm):
+    stepSize: float
+    numIterations: int
+    miniBatchFraction: float
+    convergenceTol: float
+    def __init__(
+        self,
+        stepSize: float = ...,
+        numIterations: int = ...,
+        miniBatchFraction: float = ...,
+        convergenceTol: float = ...,
+    ) -> None: ...
+    def setInitialWeights(
+        self, initialWeights: VectorLike
+    ) -> StreamingLinearRegressionWithSGD: ...
+    def trainOn(self, dstream: DStream[LabeledPoint]) -> None: ...
diff --git a/python/pyspark/mllib/stat/KernelDensity.py b/python/pyspark/mllib/stat/KernelDensity.py
index 7250eab6705a7..1d4d43e53519c 100644
--- a/python/pyspark/mllib/stat/KernelDensity.py
+++ b/python/pyspark/mllib/stat/KernelDensity.py
@@ -15,11 +15,6 @@
 # limitations under the License.
 #
 
-import sys
-
-if sys.version > '3':
-    xrange = range
-
 import numpy as np
 
 from pyspark.mllib.common import callMLlibFunc
@@ -31,6 +26,8 @@ class KernelDensity(object):
     Estimate probability density at required points given an RDD of samples
     from the population.
 
+    Examples
+    --------
     >>> kd = KernelDensity()
     >>> sample = sc.parallelize([0.0, 1.0])
     >>> kd.setSample(sample)
diff --git a/python/pyspark/mllib/stat/KernelDensity.pyi b/python/pyspark/mllib/stat/KernelDensity.pyi
new file mode 100644
index 0000000000000..efc70c9470dbe
--- /dev/null
+++ b/python/pyspark/mllib/stat/KernelDensity.pyi
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Iterable
+from pyspark.rdd import RDD
+from numpy import ndarray  # type: ignore[import]
+
+class KernelDensity:
+    def __init__(self) -> None: ...
+    def setBandwidth(self, bandwidth: float) -> None: ...
+    def setSample(self, sample: RDD[float]) -> None: ...
+    def estimate(self, points: Iterable[float]) -> ndarray: ...
diff --git a/python/pyspark/mllib/stat/__init__.py b/python/pyspark/mllib/stat/__init__.py
index c8a721d3fe41c..d3b4ddf7e4c68 100644
--- a/python/pyspark/mllib/stat/__init__.py
+++ b/python/pyspark/mllib/stat/__init__.py
@@ -19,10 +19,11 @@
 Python package for statistical functions in MLlib.
 """
 
-from pyspark.mllib.stat._statistics import *
+from pyspark.mllib.stat._statistics import Statistics, MultivariateStatisticalSummary
 from pyspark.mllib.stat.distribution import MultivariateGaussian
-from pyspark.mllib.stat.test import ChiSqTestResult
+from pyspark.mllib.stat.test import ChiSqTestResult, KolmogorovSmirnovTestResult
 from pyspark.mllib.stat.KernelDensity import KernelDensity
 
-__all__ = ["Statistics", "MultivariateStatisticalSummary", "ChiSqTestResult",
+__all__ = ["Statistics", "MultivariateStatisticalSummary",
+           "ChiSqTestResult", "KolmogorovSmirnovTestResult",
            "MultivariateGaussian", "KernelDensity"]
diff --git a/python/pyspark/mllib/stat/__init__.pyi b/python/pyspark/mllib/stat/__init__.pyi
new file mode 100644
index 0000000000000..bdd080a08cd56
--- /dev/null
+++ b/python/pyspark/mllib/stat/__init__.pyi
@@ -0,0 +1,29 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from pyspark.mllib.stat.KernelDensity import (  # noqa: F401
+    KernelDensity as KernelDensity,
+)
+from pyspark.mllib.stat._statistics import (  # noqa: F401
+    MultivariateStatisticalSummary as MultivariateStatisticalSummary,
+    Statistics as Statistics,
+)
+from pyspark.mllib.stat.distribution import (  # noqa: F401
+    MultivariateGaussian as MultivariateGaussian,
+)
+from pyspark.mllib.stat.test import ChiSqTestResult as ChiSqTestResult  # noqa: F401
diff --git a/python/pyspark/mllib/stat/_statistics.py b/python/pyspark/mllib/stat/_statistics.py
index d49f741a2f44a..d8f3cb840e45c 100644
--- a/python/pyspark/mllib/stat/_statistics.py
+++ b/python/pyspark/mllib/stat/_statistics.py
@@ -16,10 +16,8 @@
 #
 
 import sys
-if sys.version >= '3':
-    basestring = str
 
-from pyspark.rdd import RDD, ignore_unicode_prefix
+from pyspark.rdd import RDD
 from pyspark.mllib.common import callMLlibFunc, JavaModelWrapper
 from pyspark.mllib.linalg import Matrix, _convert_to_vector
 from pyspark.mllib.regression import LabeledPoint
@@ -67,11 +65,19 @@ def colStats(rdd):
         """
         Computes column-wise summary statistics for the input RDD[Vector].
 
-        :param rdd: an RDD[Vector] for which column-wise summary statistics
-                    are to be computed.
-        :return: :class:`MultivariateStatisticalSummary` object containing
-                 column-wise summary statistics.
+        Parameters
+        ----------
+        rdd : :py:class:`pyspark.RDD`
+            an RDD[Vector] for which column-wise summary statistics
+            are to be computed.
 
+        Returns
+        -------
+        :class:`MultivariateStatisticalSummary`
+            object containing column-wise summary statistics.
+
+        Examples
+        --------
         >>> from pyspark.mllib.linalg import Vectors
         >>> rdd = sc.parallelize([Vectors.dense([2, 0, 0, -2]),
         ...                       Vectors.dense([4, 5, 0,  3]),
@@ -105,13 +111,24 @@ def corr(x, y=None, method=None):
         to specify the method to be used for single RDD inout.
         If two RDDs of floats are passed in, a single float is returned.
 
-        :param x: an RDD of vector for which the correlation matrix is to be computed,
-                  or an RDD of float of the same cardinality as y when y is specified.
-        :param y: an RDD of float of the same cardinality as x.
-        :param method: String specifying the method to use for computing correlation.
-                       Supported: `pearson` (default), `spearman`
-        :return: Correlation matrix comparing columns in x.
-
+        Parameters
+        ----------
+        x : :py:class:`pyspark.RDD`
+            an RDD of vector for which the correlation matrix is to be computed,
+            or an RDD of float of the same cardinality as y when y is specified.
+        y : :py:class:`pyspark.RDD`, optional
+            an RDD of float of the same cardinality as x.
+        method : str, optional
+            String specifying the method to use for computing correlation.
+            Supported: `pearson` (default), `spearman`
+
+        Returns
+        -------
+        :py:class:`pyspark.mllib.linalg.Matrix`
+            Correlation matrix comparing columns in x.
+
+        Examples
+        --------
         >>> x = sc.parallelize([1.0, 0.0, -2.0], 2)
         >>> y = sc.parallelize([4.0, 5.0, 3.0], 2)
         >>> zeros = sc.parallelize([0.0, 0.0, 0.0], 2)
@@ -157,12 +174,11 @@ def corr(x, y=None, method=None):
             return callMLlibFunc("corr", x.map(float), y.map(float), method)
 
     @staticmethod
-    @ignore_unicode_prefix
     def chiSqTest(observed, expected=None):
         """
         If `observed` is Vector, conduct Pearson's chi-squared goodness
         of fit test of the observed data against the expected distribution,
-        or againt the uniform distribution (by default), with each category
+        or against the uniform distribution (by default), with each category
         having an expected frequency of `1 / len(observed)`.
 
         If `observed` is matrix, conduct Pearson's independence test on the
@@ -175,20 +191,33 @@ def chiSqTest(observed, expected=None):
         contingency matrix for which the chi-squared statistic is computed.
         All label and feature values must be categorical.
 
-        .. note:: `observed` cannot contain negative values
-
-        :param observed: it could be a vector containing the observed categorical
-                         counts/relative frequencies, or the contingency matrix
-                         (containing either counts or relative frequencies),
-                         or an RDD of LabeledPoint containing the labeled dataset
-                         with categorical features. Real-valued features will be
-                         treated as categorical for each distinct value.
-        :param expected: Vector containing the expected categorical counts/relative
-                         frequencies. `expected` is rescaled if the `expected` sum
-                         differs from the `observed` sum.
-        :return: ChiSquaredTest object containing the test statistic, degrees
-                 of freedom, p-value, the method used, and the null hypothesis.
-
+        Parameters
+        ----------
+        observed : :py:class:`pyspark.mllib.linalg.Vector` or \
+            :py:class:`pyspark.mllib.linalg.Matrix`
+            it could be a vector containing the observed categorical
+            counts/relative frequencies, or the contingency matrix
+            (containing either counts or relative frequencies),
+            or an RDD of LabeledPoint containing the labeled dataset
+            with categorical features. Real-valued features will be
+            treated as categorical for each distinct value.
+        expected : :py:class:`pyspark.mllib.linalg.Vector`
+            Vector containing the expected categorical counts/relative
+            frequencies. `expected` is rescaled if the `expected` sum
+            differs from the `observed` sum.
+
+        Returns
+        -------
+        :py:class:`pyspark.mllib.stat.ChiSqTestResult`
+            object containing the test statistic, degrees
+            of freedom, p-value, the method used, and the null hypothesis.
+
+        Notes
+        -----
+        `observed` cannot contain negative values
+
+        Examples
+        --------
         >>> from pyspark.mllib.linalg import Vectors, Matrices
         >>> observed = Vectors.dense([4, 6, 5])
         >>> pearson = Statistics.chiSqTest(observed)
@@ -199,9 +228,9 @@ def chiSqTest(observed, expected=None):
         >>> print(round(pearson.pValue, 4))
         0.8187
         >>> pearson.method
-        u'pearson'
+        'pearson'
         >>> pearson.nullHypothesis
-        u'observed follows the same distribution as expected.'
+        'observed follows the same distribution as expected.'
 
         >>> observed = Vectors.dense([21, 38, 43, 80])
         >>> expected = Vectors.dense([3, 5, 7, 20])
@@ -242,7 +271,6 @@ def chiSqTest(observed, expected=None):
         return ChiSqTestResult(jmodel)
 
     @staticmethod
-    @ignore_unicode_prefix
     def kolmogorovSmirnovTest(data, distName="norm", *params):
         """
         Performs the Kolmogorov-Smirnov (KS) test for data sampled from
@@ -263,17 +291,28 @@ def kolmogorovSmirnovTest(data, distName="norm", *params):
         For specific details of the implementation, please have a look
         at the Scala documentation.
 
-        :param data: RDD, samples from the data
-        :param distName: string, currently only "norm" is supported.
-                         (Normal distribution) to calculate the
-                         theoretical distribution of the data.
-        :param params: additional values which need to be provided for
-                       a certain distribution.
-                       If not provided, the default values are used.
-        :return: KolmogorovSmirnovTestResult object containing the test
-                 statistic, degrees of freedom, p-value,
-                 the method used, and the null hypothesis.
 
+        Parameters
+        ----------
+        data : :py:class:`pyspark.RDD`
+            RDD, samples from the data
+        distName : str, optional
+            string, currently only "norm" is supported.
+            (Normal distribution) to calculate the
+            theoretical distribution of the data.
+        params
+            additional values which need to be provided for
+            a certain distribution.
+            If not provided, the default values are used.
+
+        Returns
+        -------
+        :py:class:`pyspark.mllib.stat.KolmogorovSmirnovTestResult`
+            object containing the test statistic, degrees of freedom, p-value,
+            the method used, and the null hypothesis.
+
+        Examples
+        --------
         >>> kstest = Statistics.kolmogorovSmirnovTest
         >>> data = sc.parallelize([-1.0, 0.0, 1.0])
         >>> ksmodel = kstest(data, "norm")
@@ -282,7 +321,7 @@ def kolmogorovSmirnovTest(data, distName="norm", *params):
         >>> print(round(ksmodel.statistic, 3))
         0.175
         >>> ksmodel.nullHypothesis
-        u'Sample follows theoretical distribution'
+        'Sample follows theoretical distribution'
 
         >>> data = sc.parallelize([2.0, 3.0, 4.0])
         >>> ksmodel = kstest(data, "norm", 3.0, 1.0)
@@ -293,7 +332,7 @@ def kolmogorovSmirnovTest(data, distName="norm", *params):
         """
         if not isinstance(data, RDD):
             raise TypeError("data should be an RDD, got %s." % type(data))
-        if not isinstance(distName, basestring):
+        if not isinstance(distName, str):
             raise TypeError("distName should be a string, got %s." % type(distName))
 
         params = [float(param) for param in params]
diff --git a/python/pyspark/mllib/stat/_statistics.pyi b/python/pyspark/mllib/stat/_statistics.pyi
new file mode 100644
index 0000000000000..3834d51639eb2
--- /dev/null
+++ b/python/pyspark/mllib/stat/_statistics.pyi
@@ -0,0 +1,69 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import List, Optional, overload, Union
+from typing_extensions import Literal
+
+from numpy import ndarray  # type: ignore[import]
+
+from pyspark.mllib.common import JavaModelWrapper
+from pyspark.mllib.linalg import Vector, Matrix
+from pyspark.mllib.regression import LabeledPoint
+from pyspark.mllib.stat.test import ChiSqTestResult, KolmogorovSmirnovTestResult
+from pyspark.rdd import RDD
+
+CorrelationMethod = Union[Literal["spearman"], Literal["pearson"]]
+
+class MultivariateStatisticalSummary(JavaModelWrapper):
+    def mean(self) -> ndarray: ...
+    def variance(self) -> ndarray: ...
+    def count(self) -> int: ...
+    def numNonzeros(self) -> ndarray: ...
+    def max(self) -> ndarray: ...
+    def min(self) -> ndarray: ...
+    def normL1(self) -> ndarray: ...
+    def normL2(self) -> ndarray: ...
+
+class Statistics:
+    @staticmethod
+    def colStats(rdd: RDD[Vector]) -> MultivariateStatisticalSummary: ...
+    @overload
+    @staticmethod
+    def corr(
+        x: RDD[Vector], *, method: Optional[CorrelationMethod] = ...
+    ) -> Matrix: ...
+    @overload
+    @staticmethod
+    def corr(
+        x: RDD[float], y: RDD[float], method: Optional[CorrelationMethod] = ...
+    ) -> float: ...
+    @overload
+    @staticmethod
+    def chiSqTest(observed: Matrix) -> ChiSqTestResult: ...
+    @overload
+    @staticmethod
+    def chiSqTest(
+        observed: Vector, expected: Optional[Vector] = ...
+    ) -> ChiSqTestResult: ...
+    @overload
+    @staticmethod
+    def chiSqTest(observed: RDD[LabeledPoint]) -> List[ChiSqTestResult]: ...
+    @staticmethod
+    def kolmogorovSmirnovTest(
+        data: RDD[float], distName: Literal["norm"] = ..., *params: float
+    ) -> KolmogorovSmirnovTestResult: ...
diff --git a/python/pyspark/mllib/stat/distribution.py b/python/pyspark/mllib/stat/distribution.py
index 46f7a1d2f277a..aa35ac6dfdae1 100644
--- a/python/pyspark/mllib/stat/distribution.py
+++ b/python/pyspark/mllib/stat/distribution.py
@@ -24,6 +24,8 @@ class MultivariateGaussian(namedtuple('MultivariateGaussian', ['mu', 'sigma'])):
 
     """Represents a (mu, sigma) tuple
 
+    Examples
+    --------
     >>> m = MultivariateGaussian(Vectors.dense([11,12]),DenseMatrix(2, 2, (1.0, 3.0, 5.0, 2.0)))
     >>> (m.mu, m.sigma.toArray())
     (DenseVector([11.0, 12.0]), array([[ 1., 5.],[ 3., 2.]]))
diff --git a/python/pyspark/mllib/stat/distribution.pyi b/python/pyspark/mllib/stat/distribution.pyi
new file mode 100644
index 0000000000000..8bb93f91b07b5
--- /dev/null
+++ b/python/pyspark/mllib/stat/distribution.pyi
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import NamedTuple
+
+from pyspark.mllib.linalg import Vector, Matrix
+
+class MultivariateGaussian(NamedTuple):
+    mu: Vector
+    sigma: Matrix
diff --git a/python/pyspark/mllib/stat/test.pyi b/python/pyspark/mllib/stat/test.pyi
new file mode 100644
index 0000000000000..a65f8e40e87d8
--- /dev/null
+++ b/python/pyspark/mllib/stat/test.pyi
@@ -0,0 +1,39 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Generic, Tuple, TypeVar
+
+from pyspark.mllib.common import JavaModelWrapper
+
+DF = TypeVar("DF", int, float, Tuple[int, ...], Tuple[float, ...])
+
+class TestResult(JavaModelWrapper, Generic[DF]):
+    @property
+    def pValue(self) -> float: ...
+    @property
+    def degreesOfFreedom(self) -> DF: ...
+    @property
+    def statistic(self) -> float: ...
+    @property
+    def nullHypothesis(self) -> str: ...
+
+class ChiSqTestResult(TestResult[int]):
+    @property
+    def method(self) -> str: ...
+
+class KolmogorovSmirnovTestResult(TestResult[int]): ...
diff --git a/python/pyspark/mllib/tests/test_algorithms.py b/python/pyspark/mllib/tests/test_algorithms.py
index 4e9dd6b3baa0e..89d09fae5cfbc 100644
--- a/python/pyspark/mllib/tests/test_algorithms.py
+++ b/python/pyspark/mllib/tests/test_algorithms.py
@@ -292,10 +292,10 @@ def test_fpgrowth(self):
 
 
 if __name__ == "__main__":
-    from pyspark.mllib.tests.test_algorithms import *
+    from pyspark.mllib.tests.test_algorithms import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/mllib/tests/test_feature.py b/python/pyspark/mllib/tests/test_feature.py
index 9e1da0f860310..7fba83b3ea35f 100644
--- a/python/pyspark/mllib/tests/test_feature.py
+++ b/python/pyspark/mllib/tests/test_feature.py
@@ -18,9 +18,9 @@
 from math import sqrt
 import unittest
 
-from numpy import array, random, exp, abs, tile
+from numpy import array, abs, tile
 
-from pyspark.mllib.linalg import Vector, SparseVector, DenseVector, VectorUDT, Vectors
+from pyspark.mllib.linalg import SparseVector, DenseVector, Vectors
 from pyspark.mllib.linalg.distributed import RowMatrix
 from pyspark.mllib.feature import HashingTF, IDF, StandardScaler, ElementwiseProduct, Word2Vec
 from pyspark.testing.mllibutils import MLlibTestCase
@@ -182,10 +182,10 @@ def test_pca(self):
 
 
 if __name__ == "__main__":
-    from pyspark.mllib.tests.test_feature import *
+    from pyspark.mllib.tests.test_feature import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/mllib/tests/test_linalg.py b/python/pyspark/mllib/tests/test_linalg.py
index 312730e8aff8b..680076ed94190 100644
--- a/python/pyspark/mllib/tests/test_linalg.py
+++ b/python/pyspark/mllib/tests/test_linalg.py
@@ -15,7 +15,6 @@
 # limitations under the License.
 #
 
-import sys
 import array as pyarray
 import unittest
 
@@ -23,17 +22,16 @@
 
 import pyspark.ml.linalg as newlinalg
 from pyspark.serializers import PickleSerializer
-from pyspark.mllib.linalg import Vector, SparseVector, DenseVector, VectorUDT, _convert_to_vector, \
+from pyspark.mllib.linalg import (  # type: ignore[attr-defined]
+    Vector, SparseVector, DenseVector, VectorUDT, _convert_to_vector,
     DenseMatrix, SparseMatrix, Vectors, Matrices, MatrixUDT
-from pyspark.mllib.linalg.distributed import RowMatrix, IndexedRowMatrix
+)
+from pyspark.mllib.linalg.distributed import RowMatrix, IndexedRowMatrix, IndexedRow
 from pyspark.mllib.regression import LabeledPoint
 from pyspark.sql import Row
 from pyspark.testing.mllibutils import MLlibTestCase
 from pyspark.testing.utils import have_scipy
 
-if sys.version >= '3':
-    long = int
-
 
 class VectorTests(MLlibTestCase):
 
@@ -447,13 +445,24 @@ def test_row_matrix_from_dataframe(self):
 
     def test_indexed_row_matrix_from_dataframe(self):
         from pyspark.sql.utils import IllegalArgumentException
-        df = self.spark.createDataFrame([Row(long(0), Vectors.dense(1))])
+        df = self.spark.createDataFrame([Row(int(0), Vectors.dense(1))])
         matrix = IndexedRowMatrix(df)
         self.assertEqual(matrix.numRows(), 1)
         self.assertEqual(matrix.numCols(), 1)
         with self.assertRaises(IllegalArgumentException):
             IndexedRowMatrix(df.drop("_1"))
 
+    def test_row_matrix_invalid_type(self):
+        rows = self.sc.parallelize([[1, 2, 3], [4, 5, 6]])
+        invalid_type = ""
+        matrix = RowMatrix(rows)
+        self.assertRaises(TypeError, matrix.multiply, invalid_type)
+
+        irows = self.sc.parallelize([IndexedRow(0, [1, 2, 3]),
+                                     IndexedRow(1, [4, 5, 6])])
+        imatrix = IndexedRowMatrix(irows)
+        self.assertRaises(TypeError, imatrix.multiply, invalid_type)
+
 
 class MatrixUDTTests(MLlibTestCase):
 
@@ -642,10 +651,10 @@ def test_regression(self):
 
 
 if __name__ == "__main__":
-    from pyspark.mllib.tests.test_linalg import *
+    from pyspark.mllib.tests.test_linalg import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/mllib/tests/test_stat.py b/python/pyspark/mllib/tests/test_stat.py
index c222a33efb01e..414106fe51cc8 100644
--- a/python/pyspark/mllib/tests/test_stat.py
+++ b/python/pyspark/mllib/tests/test_stat.py
@@ -20,8 +20,7 @@
 
 from numpy import array
 
-from pyspark.mllib.linalg import Vector, SparseVector, DenseVector, VectorUDT, _convert_to_vector, \
-    DenseMatrix, SparseMatrix, Vectors, Matrices, MatrixUDT
+from pyspark.mllib.linalg import Vectors, Matrices
 from pyspark.mllib.random import RandomRDDs
 from pyspark.mllib.regression import LabeledPoint
 from pyspark.mllib.stat import Statistics
@@ -178,10 +177,10 @@ def test_R_implementation_equivalence(self):
 
 
 if __name__ == "__main__":
-    from pyspark.mllib.tests.test_stat import *
+    from pyspark.mllib.tests.test_stat import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/mllib/tests/test_streaming_algorithms.py b/python/pyspark/mllib/tests/test_streaming_algorithms.py
index 2077809a043f1..f6c6779e83f13 100644
--- a/python/pyspark/mllib/tests/test_streaming_algorithms.py
+++ b/python/pyspark/mllib/tests/test_streaming_algorithms.py
@@ -15,7 +15,6 @@
 # limitations under the License.
 #
 
-from time import time, sleep
 import unittest
 
 from numpy import array, random, exp, dot, all, mean, abs
@@ -190,7 +189,7 @@ def generateLogisticInput(offset, scale, nPoints, seed):
         Generate 1 / (1 + exp(-x * scale + offset))
 
         where,
-        x is randomnly distributed and the threshold
+        x is randomly distributed and the threshold
         and labels for each sample in x is obtained from a random uniform
         distribution.
         """
@@ -327,7 +326,7 @@ def condition():
                 return True
             return "Latest errors: " + ", ".join(map(lambda x: str(x), errors))
 
-        eventually(condition, timeout=60.0)
+        eventually(condition, timeout=180.0)
 
 
 class StreamingLinearRegressionWithTests(MLLibStreamingTestCase):
@@ -434,9 +433,9 @@ def test_train_prediction(self):
         slr = StreamingLinearRegressionWithSGD(stepSize=0.2, numIterations=25)
         slr.setInitialWeights([0.0])
 
-        # Create ten batches with 100 sample points in each.
+        # Create fifteen batches with 100 sample points in each.
         batches = []
-        for i in range(10):
+        for i in range(15):
             batch = LinearDataGenerator.generateLinearInput(
                 0.0, [10.0], [0.0], [1.0 / 3.0], 100, 42 + i, 0.1)
             batches.append(self.sc.parallelize(batch))
@@ -463,14 +462,14 @@ def condition():
                 return True
             return "Latest errors: " + ", ".join(map(lambda x: str(x), errors))
 
-        eventually(condition)
+        eventually(condition, timeout=180.0)
 
 
 if __name__ == "__main__":
-    from pyspark.mllib.tests.test_streaming_algorithms import *
+    from pyspark.mllib.tests.test_streaming_algorithms import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/mllib/tests/test_util.py b/python/pyspark/mllib/tests/test_util.py
index 76bac6c5c0581..2be3f17069fd4 100644
--- a/python/pyspark/mllib/tests/test_util.py
+++ b/python/pyspark/mllib/tests/test_util.py
@@ -19,7 +19,7 @@
 import tempfile
 import unittest
 
-from pyspark.mllib.common import _to_java_object_rdd
+from pyspark.mllib.common import _to_java_object_rdd  # type: ignore[attr-defined]
 from pyspark.mllib.util import LinearDataGenerator
 from pyspark.mllib.util import MLUtils
 from pyspark.mllib.linalg import SparseVector, DenseVector, Vectors
@@ -94,10 +94,10 @@ def test_to_java_object_rdd(self):  # SPARK-6660
 
 
 if __name__ == "__main__":
-    from pyspark.mllib.tests.test_util import *
+    from pyspark.mllib.tests.test_util import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/mllib/tree.py b/python/pyspark/mllib/tree.py
index 2d8df461acf9f..493dcf8db6fd2 100644
--- a/python/pyspark/mllib/tree.py
+++ b/python/pyspark/mllib/tree.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import absolute_import
-
 import sys
 import random
 
@@ -35,15 +33,18 @@ class TreeEnsembleModel(JavaModelWrapper, JavaSaveable):
 
     .. versionadded:: 1.3.0
     """
-    @since("1.3.0")
     def predict(self, x):
         """
         Predict values for a single data point or an RDD of points using
         the model trained.
 
-        .. note:: In Python, predict cannot currently be used within an RDD
-            transformation or action.
-            Call predict directly on the RDD instead.
+        .. versionadded:: 1.3.0
+
+        Notes
+        -----
+        In Python, predict cannot currently be used within an RDD
+        transformation or action.
+        Call predict directly on the RDD instead.
         """
         if isinstance(x, RDD):
             return self.call("predict", x.map(_convert_to_vector))
@@ -81,18 +82,23 @@ class DecisionTreeModel(JavaModelWrapper, JavaSaveable, JavaLoader):
 
     .. versionadded:: 1.1.0
     """
-    @since("1.1.0")
     def predict(self, x):
         """
         Predict the label of one or more examples.
 
-        .. note:: In Python, predict cannot currently be used within an RDD
-            transformation or action.
-            Call predict directly on the RDD instead.
+        .. versionadded:: 1.1.0
 
-        :param x:
-          Data point (feature vector), or an RDD of data points (feature
-          vectors).
+        Parameters
+        ----------
+        x : :py:class:`pyspark.mllib.linalg.Vector` or :py:class:`pyspark.RDD`
+            Data point (feature vector), or an RDD of data points (feature
+            vectors).
+
+        Notes
+        -----
+        In Python, predict cannot currently be used within an RDD
+        transformation or action.
+        Call predict directly on the RDD instead.
         """
         if isinstance(x, RDD):
             return self.call("predict", x.map(_convert_to_vector))
@@ -145,45 +151,50 @@ def _train(cls, data, type, numClasses, features, impurity="gini", maxDepth=5, m
         return DecisionTreeModel(model)
 
     @classmethod
-    @since("1.1.0")
     def trainClassifier(cls, data, numClasses, categoricalFeaturesInfo,
                         impurity="gini", maxDepth=5, maxBins=32, minInstancesPerNode=1,
                         minInfoGain=0.0):
         """
         Train a decision tree model for classification.
 
-        :param data:
-          Training data: RDD of LabeledPoint. Labels should take values
-          {0, 1, ..., numClasses-1}.
-        :param numClasses:
-          Number of classes for classification.
-        :param categoricalFeaturesInfo:
-          Map storing arity of categorical features. An entry (n -> k)
-          indicates that feature n is categorical with k categories
-          indexed from 0: {0, 1, ..., k-1}.
-        :param impurity:
-          Criterion used for information gain calculation.
-          Supported values: "gini" or "entropy".
-          (default: "gini")
-        :param maxDepth:
-          Maximum depth of tree (e.g. depth 0 means 1 leaf node, depth 1
-          means 1 internal node + 2 leaf nodes).
-          (default: 5)
-        :param maxBins:
-          Number of bins used for finding splits at each node.
-          (default: 32)
-        :param minInstancesPerNode:
-          Minimum number of instances required at child nodes to create
-          the parent split.
-          (default: 1)
-        :param minInfoGain:
-          Minimum info gain required to create a split.
-          (default: 0.0)
-        :return:
-          DecisionTreeModel.
-
-        Example usage:
-
+        .. versionadded:: 1.1.0
+
+        Parameters
+        ----------
+        data :  :py:class:`pyspark.RDD`
+            Training data: RDD of LabeledPoint. Labels should take values
+            {0, 1, ..., numClasses-1}.
+        numClasses : int
+            Number of classes for classification.
+        categoricalFeaturesInfo : dict
+            Map storing arity of categorical features. An entry (n -> k)
+            indicates that feature n is categorical with k categories
+            indexed from 0: {0, 1, ..., k-1}.
+        impurity : str, optional
+            Criterion used for information gain calculation.
+            Supported values: "gini" or "entropy".
+            (default: "gini")
+        maxDepth : int, optional
+            Maximum depth of tree (e.g. depth 0 means 1 leaf node, depth 1
+            means 1 internal node + 2 leaf nodes).
+            (default: 5)
+        maxBins : int, optional
+            Number of bins used for finding splits at each node.
+            (default: 32)
+        minInstancesPerNode : int, optional
+            Minimum number of instances required at child nodes to create
+            the parent split.
+            (default: 1)
+        minInfoGain : float, optional
+            Minimum info gain required to create a split.
+            (default: 0.0)
+
+        Returns
+        -------
+        :py:class:`DecisionTreeModel`
+
+        Examples
+        --------
         >>> from numpy import array
         >>> from pyspark.mllib.regression import LabeledPoint
         >>> from pyspark.mllib.tree import DecisionTree
@@ -224,35 +235,39 @@ def trainRegressor(cls, data, categoricalFeaturesInfo,
         """
         Train a decision tree model for regression.
 
-        :param data:
-          Training data: RDD of LabeledPoint. Labels are real numbers.
-        :param categoricalFeaturesInfo:
-          Map storing arity of categorical features. An entry (n -> k)
-          indicates that feature n is categorical with k categories
-          indexed from 0: {0, 1, ..., k-1}.
-        :param impurity:
-          Criterion used for information gain calculation.
-          The only supported value for regression is "variance".
-          (default: "variance")
-        :param maxDepth:
-          Maximum depth of tree (e.g. depth 0 means 1 leaf node, depth 1
-          means 1 internal node + 2 leaf nodes).
-          (default: 5)
-        :param maxBins:
-          Number of bins used for finding splits at each node.
-          (default: 32)
-        :param minInstancesPerNode:
-          Minimum number of instances required at child nodes to create
-          the parent split.
-          (default: 1)
-        :param minInfoGain:
-          Minimum info gain required to create a split.
-          (default: 0.0)
-        :return:
-          DecisionTreeModel.
-
-        Example usage:
-
+        Parameters
+        ----------
+        data : :py:class:`pyspark.RDD`
+            Training data: RDD of LabeledPoint. Labels are real numbers.
+        categoricalFeaturesInfo : dict
+            Map storing arity of categorical features. An entry (n -> k)
+            indicates that feature n is categorical with k categories
+            indexed from 0: {0, 1, ..., k-1}.
+        impurity : str, optional
+            Criterion used for information gain calculation.
+            The only supported value for regression is "variance".
+            (default: "variance")
+        maxDepth : int, optional
+            Maximum depth of tree (e.g. depth 0 means 1 leaf node, depth 1
+            means 1 internal node + 2 leaf nodes).
+            (default: 5)
+        maxBins : int, optional
+            Number of bins used for finding splits at each node.
+            (default: 32)
+        minInstancesPerNode : int, optional
+            Minimum number of instances required at child nodes to create
+            the parent split.
+            (default: 1)
+        minInfoGain : float, optional
+            Minimum info gain required to create a split.
+            (default: 0.0)
+
+        Returns
+        -------
+        :py:class:`DecisionTreeModel`
+
+        Examples
+        --------
         >>> from pyspark.mllib.regression import LabeledPoint
         >>> from pyspark.mllib.tree import DecisionTree
         >>> from pyspark.mllib.linalg import SparseVector
@@ -315,7 +330,6 @@ def _train(cls, data, algo, numClasses, categoricalFeaturesInfo, numTrees,
         return RandomForestModel(model)
 
     @classmethod
-    @since("1.2.0")
     def trainClassifier(cls, data, numClasses, categoricalFeaturesInfo, numTrees,
                         featureSubsetStrategy="auto", impurity="gini", maxDepth=4, maxBins=32,
                         seed=None):
@@ -323,44 +337,51 @@ def trainClassifier(cls, data, numClasses, categoricalFeaturesInfo, numTrees,
         Train a random forest model for binary or multiclass
         classification.
 
-        :param data:
-          Training dataset: RDD of LabeledPoint. Labels should take values
-          {0, 1, ..., numClasses-1}.
-        :param numClasses:
-          Number of classes for classification.
-        :param categoricalFeaturesInfo:
-          Map storing arity of categorical features. An entry (n -> k)
-          indicates that feature n is categorical with k categories
-          indexed from 0: {0, 1, ..., k-1}.
-        :param numTrees:
-          Number of trees in the random forest.
-        :param featureSubsetStrategy:
-          Number of features to consider for splits at each node.
-          Supported values: "auto", "all", "sqrt", "log2", "onethird".
-          If "auto" is set, this parameter is set based on numTrees:
-          if numTrees == 1, set to "all";
-          if numTrees > 1 (forest) set to "sqrt".
-          (default: "auto")
-        :param impurity:
-          Criterion used for information gain calculation.
-          Supported values: "gini" or "entropy".
-          (default: "gini")
-        :param maxDepth:
-          Maximum depth of tree (e.g. depth 0 means 1 leaf node, depth 1
-          means 1 internal node + 2 leaf nodes).
-          (default: 4)
-        :param maxBins:
-          Maximum number of bins used for splitting features.
-          (default: 32)
-        :param seed:
-          Random seed for bootstrapping and choosing feature subsets.
-          Set as None to generate seed based on system time.
-          (default: None)
-        :return:
-          RandomForestModel that can be used for prediction.
-
-        Example usage:
-
+        .. versionadded:: 1.2.0
+
+        Parameters
+        ----------
+        data : :py:class:`pyspark.RDD`
+            Training dataset: RDD of LabeledPoint. Labels should take values
+            {0, 1, ..., numClasses-1}.
+        numClasses : int
+            Number of classes for classification.
+        categoricalFeaturesInfo : dict
+            Map storing arity of categorical features. An entry (n -> k)
+            indicates that feature n is categorical with k categories
+            indexed from 0: {0, 1, ..., k-1}.
+        numTrees : int
+            Number of trees in the random forest.
+        featureSubsetStrategy : str, optional
+            Number of features to consider for splits at each node.
+            Supported values: "auto", "all", "sqrt", "log2", "onethird".
+            If "auto" is set, this parameter is set based on numTrees:
+            if numTrees == 1, set to "all";
+            if numTrees > 1 (forest) set to "sqrt".
+            (default: "auto")
+        impurity : str, optional
+            Criterion used for information gain calculation.
+            Supported values: "gini" or "entropy".
+            (default: "gini")
+        maxDepth : int, optional
+            Maximum depth of tree (e.g. depth 0 means 1 leaf node, depth 1
+            means 1 internal node + 2 leaf nodes).
+            (default: 4)
+        maxBins : int, optional
+            Maximum number of bins used for splitting features.
+            (default: 32)
+        seed : int, Optional
+            Random seed for bootstrapping and choosing feature subsets.
+            Set as None to generate seed based on system time.
+            (default: None)
+
+        Returns
+        -------
+        :py:class:`RandomForestModel`
+            that can be used for prediction.
+
+        Examples
+        --------
         >>> from pyspark.mllib.regression import LabeledPoint
         >>> from pyspark.mllib.tree import RandomForest
         >>>
@@ -407,47 +428,55 @@ def trainClassifier(cls, data, numClasses, categoricalFeaturesInfo, numTrees,
                           maxDepth, maxBins, seed)
 
     @classmethod
-    @since("1.2.0")
     def trainRegressor(cls, data, categoricalFeaturesInfo, numTrees, featureSubsetStrategy="auto",
                        impurity="variance", maxDepth=4, maxBins=32, seed=None):
         """
         Train a random forest model for regression.
 
-        :param data:
-          Training dataset: RDD of LabeledPoint. Labels are real numbers.
-        :param categoricalFeaturesInfo:
-          Map storing arity of categorical features. An entry (n -> k)
-          indicates that feature n is categorical with k categories
-          indexed from 0: {0, 1, ..., k-1}.
-        :param numTrees:
-          Number of trees in the random forest.
-        :param featureSubsetStrategy:
-          Number of features to consider for splits at each node.
-          Supported values: "auto", "all", "sqrt", "log2", "onethird".
-          If "auto" is set, this parameter is set based on numTrees:
-          if numTrees == 1, set to "all";
-          if numTrees > 1 (forest) set to "onethird" for regression.
-          (default: "auto")
-        :param impurity:
-          Criterion used for information gain calculation.
-          The only supported value for regression is "variance".
-          (default: "variance")
-        :param maxDepth:
-          Maximum depth of tree (e.g. depth 0 means 1 leaf node, depth 1
-          means 1 internal node + 2 leaf nodes).
-          (default: 4)
-        :param maxBins:
-          Maximum number of bins used for splitting features.
-          (default: 32)
-        :param seed:
-          Random seed for bootstrapping and choosing feature subsets.
-          Set as None to generate seed based on system time.
-          (default: None)
-        :return:
-          RandomForestModel that can be used for prediction.
-
-        Example usage:
-
+        .. versionadded:: 1.2.0
+
+        Parameters
+        ----------
+        data : :py:class:`pyspark.RDD`
+            Training dataset: RDD of LabeledPoint. Labels are real numbers.
+        categoricalFeaturesInfo : dict
+            Map storing arity of categorical features. An entry (n -> k)
+            indicates that feature n is categorical with k categories
+            indexed from 0: {0, 1, ..., k-1}.
+        numTrees : int
+            Number of trees in the random forest.
+        featureSubsetStrategy : str, optional
+            Number of features to consider for splits at each node.
+            Supported values: "auto", "all", "sqrt", "log2", "onethird".
+            If "auto" is set, this parameter is set based on numTrees:
+
+            - if numTrees == 1, set to "all";
+            - if numTrees > 1 (forest) set to "onethird" for regression.
+
+            (default: "auto")
+        impurity : str, optional
+            Criterion used for information gain calculation.
+            The only supported value for regression is "variance".
+            (default: "variance")
+        maxDepth : int, optional
+            Maximum depth of tree (e.g. depth 0 means 1 leaf node, depth 1
+            means 1 internal node + 2 leaf nodes).
+            (default: 4)
+        maxBins : int, optional
+            Maximum number of bins used for splitting features.
+            (default: 32)
+        seed : int, optional
+            Random seed for bootstrapping and choosing feature subsets.
+            Set as None to generate seed based on system time.
+            (default: None)
+
+        Returns
+        -------
+        :py:class:`RandomForestModel`
+            that can be used for prediction.
+
+        Examples
+        --------
         >>> from pyspark.mllib.regression import LabeledPoint
         >>> from pyspark.mllib.tree import RandomForest
         >>> from pyspark.mllib.linalg import SparseVector
@@ -507,45 +536,51 @@ def _train(cls, data, algo, categoricalFeaturesInfo,
         return GradientBoostedTreesModel(model)
 
     @classmethod
-    @since("1.3.0")
     def trainClassifier(cls, data, categoricalFeaturesInfo,
                         loss="logLoss", numIterations=100, learningRate=0.1, maxDepth=3,
                         maxBins=32):
         """
         Train a gradient-boosted trees model for classification.
 
-        :param data:
-          Training dataset: RDD of LabeledPoint. Labels should take values
-          {0, 1}.
-        :param categoricalFeaturesInfo:
-          Map storing arity of categorical features. An entry (n -> k)
-          indicates that feature n is categorical with k categories
-          indexed from 0: {0, 1, ..., k-1}.
-        :param loss:
-          Loss function used for minimization during gradient boosting.
-          Supported values: "logLoss", "leastSquaresError",
-          "leastAbsoluteError".
-          (default: "logLoss")
-        :param numIterations:
-          Number of iterations of boosting.
-          (default: 100)
-        :param learningRate:
-          Learning rate for shrinking the contribution of each estimator.
-          The learning rate should be between in the interval (0, 1].
-          (default: 0.1)
-        :param maxDepth:
-          Maximum depth of tree (e.g. depth 0 means 1 leaf node, depth 1
-          means 1 internal node + 2 leaf nodes).
-          (default: 3)
-        :param maxBins:
-          Maximum number of bins used for splitting features. DecisionTree
-          requires maxBins >= max categories.
-          (default: 32)
-        :return:
-          GradientBoostedTreesModel that can be used for prediction.
-
-        Example usage:
-
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        data : :py:class:`pyspark.RDD`
+            Training dataset: RDD of LabeledPoint. Labels should take values
+            {0, 1}.
+        categoricalFeaturesInfo : dict
+            Map storing arity of categorical features. An entry (n -> k)
+            indicates that feature n is categorical with k categories
+            indexed from 0: {0, 1, ..., k-1}.
+        loss : str, optional
+            Loss function used for minimization during gradient boosting.
+            Supported values: "logLoss", "leastSquaresError",
+            "leastAbsoluteError".
+            (default: "logLoss")
+        numIterations : int, optional
+            Number of iterations of boosting.
+            (default: 100)
+        learningRate : float, optional
+            Learning rate for shrinking the contribution of each estimator.
+            The learning rate should be between in the interval (0, 1].
+            (default: 0.1)
+        maxDepth : int, optional
+            Maximum depth of tree (e.g. depth 0 means 1 leaf node, depth 1
+            means 1 internal node + 2 leaf nodes).
+            (default: 3)
+        maxBins : int, optional
+            Maximum number of bins used for splitting features. DecisionTree
+            requires maxBins >= max categories.
+            (default: 32)
+
+        Returns
+        -------
+        :py:class:`GradientBoostedTreesModel`
+            that can be used for prediction.
+
+        Examples
+        --------
         >>> from pyspark.mllib.regression import LabeledPoint
         >>> from pyspark.mllib.tree import GradientBoostedTrees
         >>>
@@ -576,44 +611,50 @@ def trainClassifier(cls, data, categoricalFeaturesInfo,
                           loss, numIterations, learningRate, maxDepth, maxBins)
 
     @classmethod
-    @since("1.3.0")
     def trainRegressor(cls, data, categoricalFeaturesInfo,
                        loss="leastSquaresError", numIterations=100, learningRate=0.1, maxDepth=3,
                        maxBins=32):
         """
         Train a gradient-boosted trees model for regression.
 
-        :param data:
-          Training dataset: RDD of LabeledPoint. Labels are real numbers.
-        :param categoricalFeaturesInfo:
-          Map storing arity of categorical features. An entry (n -> k)
-          indicates that feature n is categorical with k categories
-          indexed from 0: {0, 1, ..., k-1}.
-        :param loss:
-          Loss function used for minimization during gradient boosting.
-          Supported values: "logLoss", "leastSquaresError",
-          "leastAbsoluteError".
-          (default: "leastSquaresError")
-        :param numIterations:
-          Number of iterations of boosting.
-          (default: 100)
-        :param learningRate:
-          Learning rate for shrinking the contribution of each estimator.
-          The learning rate should be between in the interval (0, 1].
-          (default: 0.1)
-        :param maxDepth:
-          Maximum depth of tree (e.g. depth 0 means 1 leaf node, depth 1
-          means 1 internal node + 2 leaf nodes).
-          (default: 3)
-        :param maxBins:
-          Maximum number of bins used for splitting features. DecisionTree
-          requires maxBins >= max categories.
-          (default: 32)
-        :return:
-          GradientBoostedTreesModel that can be used for prediction.
-
-        Example usage:
-
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        data :
+            Training dataset: RDD of LabeledPoint. Labels are real numbers.
+        categoricalFeaturesInfo : dict
+            Map storing arity of categorical features. An entry (n -> k)
+            indicates that feature n is categorical with k categories
+            indexed from 0: {0, 1, ..., k-1}.
+        loss : str, optional
+            Loss function used for minimization during gradient boosting.
+            Supported values: "logLoss", "leastSquaresError",
+            "leastAbsoluteError".
+            (default: "leastSquaresError")
+        numIterations : int, optional
+            Number of iterations of boosting.
+            (default: 100)
+        learningRate : float, optional
+            Learning rate for shrinking the contribution of each estimator.
+            The learning rate should be between in the interval (0, 1].
+            (default: 0.1)
+        maxDepth : int, optional
+            Maximum depth of tree (e.g. depth 0 means 1 leaf node, depth 1
+            means 1 internal node + 2 leaf nodes).
+            (default: 3)
+        maxBins : int, optional
+            Maximum number of bins used for splitting features. DecisionTree
+            requires maxBins >= max categories.
+            (default: 32)
+
+        Returns
+        -------
+        :py:class:`GradientBoostedTreesModel`
+            that can be used for prediction.
+
+        Examples
+        --------
         >>> from pyspark.mllib.regression import LabeledPoint
         >>> from pyspark.mllib.tree import GradientBoostedTrees
         >>> from pyspark.mllib.linalg import SparseVector
diff --git a/python/pyspark/mllib/tree.pyi b/python/pyspark/mllib/tree.pyi
new file mode 100644
index 0000000000000..511afdeb063d9
--- /dev/null
+++ b/python/pyspark/mllib/tree.pyi
@@ -0,0 +1,126 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import overload
+from typing import Dict, Optional, Tuple
+from pyspark.mllib._typing import VectorLike
+from pyspark.rdd import RDD
+from pyspark.mllib.common import JavaModelWrapper
+from pyspark.mllib.regression import LabeledPoint
+from pyspark.mllib.util import JavaLoader, JavaSaveable
+
+class TreeEnsembleModel(JavaModelWrapper, JavaSaveable):
+    @overload
+    def predict(self, x: VectorLike) -> float: ...
+    @overload
+    def predict(self, x: RDD[VectorLike]) -> RDD[VectorLike]: ...
+    def numTrees(self) -> int: ...
+    def totalNumNodes(self) -> int: ...
+    def toDebugString(self) -> str: ...
+
+class DecisionTreeModel(JavaModelWrapper, JavaSaveable, JavaLoader[DecisionTreeModel]):
+    @overload
+    def predict(self, x: VectorLike) -> float: ...
+    @overload
+    def predict(self, x: RDD[VectorLike]) -> RDD[VectorLike]: ...
+    def numNodes(self) -> int: ...
+    def depth(self) -> int: ...
+    def toDebugString(self) -> str: ...
+
+class DecisionTree:
+    @classmethod
+    def trainClassifier(
+        cls,
+        data: RDD[LabeledPoint],
+        numClasses: int,
+        categoricalFeaturesInfo: Dict[int, int],
+        impurity: str = ...,
+        maxDepth: int = ...,
+        maxBins: int = ...,
+        minInstancesPerNode: int = ...,
+        minInfoGain: float = ...,
+    ) -> DecisionTreeModel: ...
+    @classmethod
+    def trainRegressor(
+        cls,
+        data: RDD[LabeledPoint],
+        categoricalFeaturesInfo: Dict[int, int],
+        impurity: str = ...,
+        maxDepth: int = ...,
+        maxBins: int = ...,
+        minInstancesPerNode: int = ...,
+        minInfoGain: float = ...,
+    ) -> DecisionTreeModel: ...
+
+class RandomForestModel(TreeEnsembleModel, JavaLoader[RandomForestModel]): ...
+
+class RandomForest:
+    supportedFeatureSubsetStrategies: Tuple[str, ...]
+    @classmethod
+    def trainClassifier(
+        cls,
+        data: RDD[LabeledPoint],
+        numClasses: int,
+        categoricalFeaturesInfo: Dict[int, int],
+        numTrees: int,
+        featureSubsetStrategy: str = ...,
+        impurity: str = ...,
+        maxDepth: int = ...,
+        maxBins: int = ...,
+        seed: Optional[int] = ...,
+    ) -> RandomForestModel: ...
+    @classmethod
+    def trainRegressor(
+        cls,
+        data: RDD[LabeledPoint],
+        categoricalFeaturesInfo: Dict[int, int],
+        numTrees: int,
+        featureSubsetStrategy: str = ...,
+        impurity: str = ...,
+        maxDepth: int = ...,
+        maxBins: int = ...,
+        seed: Optional[int] = ...,
+    ) -> RandomForestModel: ...
+
+class GradientBoostedTreesModel(
+    TreeEnsembleModel, JavaLoader[GradientBoostedTreesModel]
+): ...
+
+class GradientBoostedTrees:
+    @classmethod
+    def trainClassifier(
+        cls,
+        data: RDD[LabeledPoint],
+        categoricalFeaturesInfo: Dict[int, int],
+        loss: str = ...,
+        numIterations: int = ...,
+        learningRate: float = ...,
+        maxDepth: int = ...,
+        maxBins: int = ...,
+    ) -> GradientBoostedTreesModel: ...
+    @classmethod
+    def trainRegressor(
+        cls,
+        data: RDD[LabeledPoint],
+        categoricalFeaturesInfo: Dict[int, int],
+        loss: str = ...,
+        numIterations: int = ...,
+        learningRate: float = ...,
+        maxDepth: int = ...,
+        maxBins: int = ...,
+    ) -> GradientBoostedTreesModel: ...
diff --git a/python/pyspark/mllib/util.py b/python/pyspark/mllib/util.py
index f0f9cda4672b1..68feb9563852c 100644
--- a/python/pyspark/mllib/util.py
+++ b/python/pyspark/mllib/util.py
@@ -18,10 +18,6 @@
 import sys
 import numpy as np
 
-if sys.version > '3':
-    xrange = range
-    basestring = str
-
 from pyspark import SparkContext, since
 from pyspark.mllib.common import callMLlibFunc, inherit_doc
 from pyspark.mllib.linalg import Vectors, SparseVector, _convert_to_vector
@@ -46,7 +42,7 @@ def _parse_libsvm_line(line):
         nnz = len(items) - 1
         indices = np.zeros(nnz, dtype=np.int32)
         values = np.zeros(nnz)
-        for i in xrange(nnz):
+        for i in range(nnz):
             index, value = items[1 + i].split(":")
             indices[i] = int(index) - 1
             values[i] = float(value)
@@ -61,15 +57,14 @@ def _convert_labeled_point_to_libsvm(p):
         v = _convert_to_vector(p.features)
         if isinstance(v, SparseVector):
             nnz = len(v.indices)
-            for i in xrange(nnz):
+            for i in range(nnz):
                 items.append(str(v.indices[i] + 1) + ":" + str(v.values[i]))
         else:
-            for i in xrange(len(v)):
+            for i in range(len(v)):
                 items.append(str(i + 1) + ":" + str(v[i]))
         return " ".join(items)
 
     @staticmethod
-    @since("1.0.0")
     def loadLibSVMFile(sc, path, numFeatures=-1, minPartitions=None):
         """
         Loads labeled data in the LIBSVM format into an RDD of
@@ -83,20 +78,33 @@ def loadLibSVMFile(sc, path, numFeatures=-1, minPartitions=None):
         method parses each line into a LabeledPoint, where the feature
         indices are converted to zero-based.
 
-        :param sc: Spark context
-        :param path: file or directory path in any Hadoop-supported file
-                     system URI
-        :param numFeatures: number of features, which will be determined
-                            from the input data if a nonpositive value
-                            is given. This is useful when the dataset is
-                            already split into multiple files and you
-                            want to load them separately, because some
-                            features may not present in certain files,
-                            which leads to inconsistent feature
-                            dimensions.
-        :param minPartitions: min number of partitions
-        :return: labeled data stored as an RDD of LabeledPoint
-
+        .. versionadded:: 1.0.0
+
+        Parameters
+        ----------
+        sc : :py:class:`pyspark.SparkContext`
+            Spark context
+        path : str
+            file or directory path in any Hadoop-supported file system URI
+        numFeatures : int, optional
+            number of features, which will be determined
+            from the input data if a nonpositive value
+            is given. This is useful when the dataset is
+            already split into multiple files and you
+            want to load them separately, because some
+            features may not present in certain files,
+            which leads to inconsistent feature
+            dimensions.
+        minPartitions : int, optional
+            min number of partitions
+
+        Returns
+        -------
+        :py:class:`pyspark.RDD`
+            labeled data stored as an RDD of LabeledPoint
+
+        Examples
+        --------
         >>> from tempfile import NamedTemporaryFile
         >>> from pyspark.mllib.util import MLUtils
         >>> from pyspark.mllib.regression import LabeledPoint
@@ -122,14 +130,21 @@ def loadLibSVMFile(sc, path, numFeatures=-1, minPartitions=None):
         return parsed.map(lambda x: LabeledPoint(x[0], Vectors.sparse(numFeatures, x[1], x[2])))
 
     @staticmethod
-    @since("1.0.0")
     def saveAsLibSVMFile(data, dir):
         """
         Save labeled data in LIBSVM format.
 
-        :param data: an RDD of LabeledPoint to be saved
-        :param dir: directory to save the data
+        .. versionadded:: 1.0.0
+
+        Parameters
+        ----------
+        data : :py:class:`pyspark.RDD`
+            an RDD of LabeledPoint to be saved
+        dir : str
+            directory to save the data
 
+        Examples
+        --------
         >>> from tempfile import NamedTemporaryFile
         >>> from fileinput import input
         >>> from pyspark.mllib.regression import LabeledPoint
@@ -147,17 +162,28 @@ def saveAsLibSVMFile(data, dir):
         lines.saveAsTextFile(dir)
 
     @staticmethod
-    @since("1.1.0")
     def loadLabeledPoints(sc, path, minPartitions=None):
         """
         Load labeled points saved using RDD.saveAsTextFile.
 
-        :param sc: Spark context
-        :param path: file or directory path in any Hadoop-supported file
-                     system URI
-        :param minPartitions: min number of partitions
-        :return: labeled data stored as an RDD of LabeledPoint
+        .. versionadded:: 1.0.0
+
+        Parameters
+        ----------
+        sc : :py:class:`pyspark.SparkContext`
+            Spark context
+        path : str
+            file or directory path in any Hadoop-supported file system URI
+        minPartitions : int, optional
+            min number of partitions
+
+        Returns
+        -------
+        :py:class:`pyspark.RDD`
+            labeled data stored as an RDD of LabeledPoint
 
+        Examples
+        --------
         >>> from tempfile import NamedTemporaryFile
         >>> from pyspark.mllib.util import MLUtils
         >>> from pyspark.mllib.regression import LabeledPoint
@@ -197,7 +223,6 @@ def loadVectors(sc, path):
         return callMLlibFunc("loadVectors", sc, path)
 
     @staticmethod
-    @since("2.0.0")
     def convertVectorColumnsToML(dataset, *cols):
         """
         Converts vector columns in an input DataFrame from the
@@ -205,16 +230,26 @@ def convertVectorColumnsToML(dataset, *cols):
         :py:class:`pyspark.ml.linalg.Vector` type under the `spark.ml`
         package.
 
-        :param dataset:
-          input dataset
-        :param cols:
-          a list of vector columns to be converted.
-          New vector columns will be ignored. If unspecified, all old
-          vector columns will be converted excepted nested ones.
-        :return:
-          the input dataset with old vector columns converted to the
-          new vector type
+        .. versionadded:: 2.0.0
 
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            input dataset
+        \\*cols : str
+            Vector columns to be converted.
+
+            New vector columns will be ignored. If unspecified, all old
+            vector columns will be converted excepted nested ones.
+
+        Returns
+        -------
+        :py:class:`pyspark.sql.DataFrame`
+            the input dataset with old vector columns converted to the
+            new vector type
+
+        Examples
+        --------
         >>> import pyspark
         >>> from pyspark.mllib.linalg import Vectors
         >>> from pyspark.mllib.util import MLUtils
@@ -237,7 +272,6 @@ def convertVectorColumnsToML(dataset, *cols):
         return callMLlibFunc("convertVectorColumnsToML", dataset, list(cols))
 
     @staticmethod
-    @since("2.0.0")
     def convertVectorColumnsFromML(dataset, *cols):
         """
         Converts vector columns in an input DataFrame to the
@@ -245,16 +279,26 @@ def convertVectorColumnsFromML(dataset, *cols):
         :py:class:`pyspark.ml.linalg.Vector` type under the `spark.ml`
         package.
 
-        :param dataset:
-          input dataset
-        :param cols:
-          a list of vector columns to be converted.
-          Old vector columns will be ignored. If unspecified, all new
-          vector columns will be converted except nested ones.
-        :return:
-          the input dataset with new vector columns converted to the
-          old vector type
+        .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            input dataset
+        \\*cols : str
+            Vector columns to be converted.
+
+            Old vector columns will be ignored. If unspecified, all new
+            vector columns will be converted except nested ones.
 
+        Returns
+        -------
+        :py:class:`pyspark.sql.DataFrame`
+            the input dataset with new vector columns converted to the
+            old vector type
+
+        Examples
+        --------
         >>> import pyspark
         >>> from pyspark.ml.linalg import Vectors
         >>> from pyspark.mllib.util import MLUtils
@@ -277,7 +321,6 @@ def convertVectorColumnsFromML(dataset, *cols):
         return callMLlibFunc("convertVectorColumnsFromML", dataset, list(cols))
 
     @staticmethod
-    @since("2.0.0")
     def convertMatrixColumnsToML(dataset, *cols):
         """
         Converts matrix columns in an input DataFrame from the
@@ -285,16 +328,26 @@ def convertMatrixColumnsToML(dataset, *cols):
         :py:class:`pyspark.ml.linalg.Matrix` type under the `spark.ml`
         package.
 
-        :param dataset:
-          input dataset
-        :param cols:
-          a list of matrix columns to be converted.
-          New matrix columns will be ignored. If unspecified, all old
-          matrix columns will be converted excepted nested ones.
-        :return:
-          the input dataset with old matrix columns converted to the
-          new matrix type
+        .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            input dataset
+        \\*cols : str
+            Matrix columns to be converted.
+
+            New matrix columns will be ignored. If unspecified, all old
+            matrix columns will be converted excepted nested ones.
 
+        Returns
+        -------
+        :py:class:`pyspark.sql.DataFrame`
+            the input dataset with old matrix columns converted to the
+            new matrix type
+
+        Examples
+        --------
         >>> import pyspark
         >>> from pyspark.mllib.linalg import Matrices
         >>> from pyspark.mllib.util import MLUtils
@@ -317,7 +370,6 @@ def convertMatrixColumnsToML(dataset, *cols):
         return callMLlibFunc("convertMatrixColumnsToML", dataset, list(cols))
 
     @staticmethod
-    @since("2.0.0")
     def convertMatrixColumnsFromML(dataset, *cols):
         """
         Converts matrix columns in an input DataFrame to the
@@ -325,16 +377,26 @@ def convertMatrixColumnsFromML(dataset, *cols):
         :py:class:`pyspark.ml.linalg.Matrix` type under the `spark.ml`
         package.
 
-        :param dataset:
-          input dataset
-        :param cols:
-          a list of matrix columns to be converted.
-          Old matrix columns will be ignored. If unspecified, all new
-          matrix columns will be converted except nested ones.
-        :return:
-          the input dataset with new matrix columns converted to the
-          old matrix type
+        .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        dataset : :py:class:`pyspark.sql.DataFrame`
+            input dataset
+        \\*cols : str
+            Matrix columns to be converted.
 
+            Old matrix columns will be ignored. If unspecified, all new
+            matrix columns will be converted except nested ones.
+
+        Returns
+        -------
+        :py:class:`pyspark.sql.DataFrame`
+            the input dataset with new matrix columns converted to the
+            old matrix type
+
+        Examples
+        --------
         >>> import pyspark
         >>> from pyspark.ml.linalg import Matrices
         >>> from pyspark.mllib.util import MLUtils
@@ -374,10 +436,14 @@ def save(self, sc, path):
 
         The model may be loaded using :py:meth:`Loader.load`.
 
-        :param sc: Spark context used to save model data.
-        :param path: Path specifying the directory in which to save
-                     this model. If the directory already exists,
-                     this method throws an exception.
+        Parameters
+        ----------
+        sc : :py:class:`pyspark.SparkContext`
+            Spark context used to save model data.
+        path : str
+            Path specifying the directory in which to save
+            this model. If the directory already exists,
+            this method throws an exception.
         """
         raise NotImplementedError
 
@@ -396,8 +462,8 @@ def save(self, sc, path):
         """Save this model to the given path."""
         if not isinstance(sc, SparkContext):
             raise TypeError("sc should be a SparkContext, got type %s" % type(sc))
-        if not isinstance(path, basestring):
-            raise TypeError("path should be a basestring, got type %s" % type(path))
+        if not isinstance(path, str):
+            raise TypeError("path should be a string, got type %s" % type(path))
         self._java_model.save(sc._jsc.sc(), path)
 
 
@@ -414,10 +480,17 @@ def load(cls, sc, path):
         Load a model from the given path. The model should have been
         saved using :py:meth:`Saveable.save`.
 
-        :param sc: Spark context used for loading model files.
-        :param path: Path specifying the directory to which the model
-                     was saved.
-        :return: model instance
+        Parameters
+        ----------
+        sc : :py:class:`pyspark.SparkContext`
+            Spark context used for loading model files.
+        path : str
+            Path specifying the directory to which the model was saved.
+
+        Returns
+        -------
+        object
+            model instance
         """
         raise NotImplementedError
 
@@ -467,20 +540,33 @@ class LinearDataGenerator(object):
     """
 
     @staticmethod
-    @since("1.5.0")
     def generateLinearInput(intercept, weights, xMean, xVariance,
                             nPoints, seed, eps):
         """
-        :param: intercept bias factor, the term c in X'w + c
-        :param: weights   feature vector, the term w in X'w + c
-        :param: xMean     Point around which the data X is centered.
-        :param: xVariance Variance of the given data
-        :param: nPoints   Number of points to be generated
-        :param: seed      Random Seed
-        :param: eps       Used to scale the noise. If eps is set high,
-                          the amount of gaussian noise added is more.
-
-        Returns a list of LabeledPoints of length nPoints
+        .. versionadded:: 1.5.0
+
+        Parameters
+        ----------
+        intercept : float
+            bias factor, the term c in X'w + c
+        weights : :py:class:`pyspark.mllib.linalg.Vector` or convertible
+            feature vector, the term w in X'w + c
+        xMean : :py:class:`pyspark.mllib.linalg.Vector` or convertible
+            Point around which the data X is centered.
+        xVariance : :py:class:`pyspark.mllib.linalg.Vector` or convertible
+            Variance of the given data
+        nPoints : int
+            Number of points to be generated
+        seed : int
+            Random Seed
+        eps : float
+            Used to scale the noise. If eps is set high,
+            the amount of gaussian noise added is more.
+
+        Returns
+        -------
+        list
+            of :py:class:`pyspark.mllib.regression.LabeledPoints` of length nPoints
         """
         weights = [float(weight) for weight in weights]
         xMean = [float(mean) for mean in xMean]
diff --git a/python/pyspark/mllib/util.pyi b/python/pyspark/mllib/util.pyi
new file mode 100644
index 0000000000000..265f765ee263a
--- /dev/null
+++ b/python/pyspark/mllib/util.pyi
@@ -0,0 +1,90 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Generic, List, Optional, TypeVar
+
+from pyspark.mllib._typing import VectorLike
+from pyspark.context import SparkContext
+from pyspark.mllib.linalg import Vector
+from pyspark.mllib.regression import LabeledPoint
+from pyspark.rdd import RDD
+from pyspark.sql.dataframe import DataFrame
+
+T = TypeVar("T")
+
+class MLUtils:
+    @staticmethod
+    def loadLibSVMFile(
+        sc: SparkContext,
+        path: str,
+        numFeatures: int = ...,
+        minPartitions: Optional[int] = ...,
+    ) -> RDD[LabeledPoint]: ...
+    @staticmethod
+    def saveAsLibSVMFile(data: RDD[LabeledPoint], dir: str) -> None: ...
+    @staticmethod
+    def loadLabeledPoints(
+        sc: SparkContext, path: str, minPartitions: Optional[int] = ...
+    ) -> RDD[LabeledPoint]: ...
+    @staticmethod
+    def appendBias(data: Vector) -> Vector: ...
+    @staticmethod
+    def loadVectors(sc: SparkContext, path: str) -> RDD[Vector]: ...
+    @staticmethod
+    def convertVectorColumnsToML(dataset: DataFrame, *cols: str) -> DataFrame: ...
+    @staticmethod
+    def convertVectorColumnsFromML(dataset: DataFrame, *cols: str) -> DataFrame: ...
+    @staticmethod
+    def convertMatrixColumnsToML(dataset: DataFrame, *cols: str) -> DataFrame: ...
+    @staticmethod
+    def convertMatrixColumnsFromML(dataset: DataFrame, *cols: str) -> DataFrame: ...
+
+class Saveable:
+    def save(self, sc: SparkContext, path: str) -> None: ...
+
+class JavaSaveable(Saveable):
+    def save(self, sc: SparkContext, path: str) -> None: ...
+
+class Loader(Generic[T]):
+    @classmethod
+    def load(cls, sc: SparkContext, path: str) -> T: ...
+
+class JavaLoader(Loader[T]):
+    @classmethod
+    def load(cls, sc: SparkContext, path: str) -> T: ...
+
+class LinearDataGenerator:
+    @staticmethod
+    def generateLinearInput(
+        intercept: float,
+        weights: VectorLike,
+        xMean: VectorLike,
+        xVariance: VectorLike,
+        nPoints: int,
+        seed: int,
+        eps: float,
+    ) -> List[LabeledPoint]: ...
+    @staticmethod
+    def generateLinearRDD(
+        sc: SparkContext,
+        nexamples: int,
+        nfeatures: int,
+        eps: float,
+        nParts: int = ...,
+        intercept: float = ...,
+    ) -> RDD[LabeledPoint]: ...
diff --git a/python/pyspark/pandas/__init__.py b/python/pyspark/pandas/__init__.py
new file mode 100644
index 0000000000000..422a92915b28d
--- /dev/null
+++ b/python/pyspark/pandas/__init__.py
@@ -0,0 +1,175 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import os
+import sys
+from distutils.version import LooseVersion
+import warnings
+
+from pyspark.sql.pandas.utils import require_minimum_pandas_version, require_minimum_pyarrow_version
+
+try:
+    require_minimum_pandas_version()
+    require_minimum_pyarrow_version()
+except ImportError as e:
+    if os.environ.get("SPARK_TESTING"):
+        warnings.warn(str(e))
+        sys.exit()
+    else:
+        raise
+
+from pyspark.pandas.version import __version__  # noqa: F401
+
+
+def assert_python_version():
+    major = 3
+    minor = 5
+    deprecated_version = (major, minor)
+    min_supported_version = (major, minor + 1)
+
+    if sys.version_info[:2] <= deprecated_version:
+        warnings.warn(
+            "pandas-on-Spark support for Python {dep_ver} is deprecated and will be dropped in "
+            "the future release. At that point, existing Python {dep_ver} workflows "
+            "that use pandas-on-Spark will continue to work without modification, but "
+            "Python {dep_ver} users will no longer get access to the latest pandas-on-Spark "
+            "features and bugfixes. We recommend that you upgrade to Python {min_ver} or "
+            "newer.".format(
+                dep_ver=".".join(map(str, deprecated_version)),
+                min_ver=".".join(map(str, min_supported_version)),
+            ),
+            FutureWarning,
+        )
+
+
+assert_python_version()
+
+import pyarrow
+
+if (
+    LooseVersion(pyarrow.__version__) >= LooseVersion("2.0.0")
+    and "PYARROW_IGNORE_TIMEZONE" not in os.environ
+):
+    import logging
+
+    logging.warning(
+        "'PYARROW_IGNORE_TIMEZONE' environment variable was not set. It is required to "
+        "set this environment variable to '1' in both driver and executor sides if you use "
+        "pyarrow>=2.0.0. "
+        "pandas-on-Spark will set it for you but it does not work if there is a Spark context "
+        "already launched."
+    )
+    os.environ["PYARROW_IGNORE_TIMEZONE"] = "1"
+
+from pyspark.pandas.frame import DataFrame
+from pyspark.pandas.indexes.base import Index
+from pyspark.pandas.indexes.category import CategoricalIndex
+from pyspark.pandas.indexes.datetimes import DatetimeIndex
+from pyspark.pandas.indexes.multi import MultiIndex
+from pyspark.pandas.indexes.numeric import Float64Index, Int64Index
+from pyspark.pandas.series import Series
+from pyspark.pandas.groupby import NamedAgg
+
+__all__ = [  # noqa: F405
+    "read_csv",
+    "read_parquet",
+    "to_datetime",
+    "date_range",
+    "from_pandas",
+    "get_dummies",
+    "DataFrame",
+    "Series",
+    "Index",
+    "MultiIndex",
+    "Int64Index",
+    "Float64Index",
+    "CategoricalIndex",
+    "DatetimeIndex",
+    "sql",
+    "range",
+    "concat",
+    "melt",
+    "get_option",
+    "set_option",
+    "reset_option",
+    "read_sql_table",
+    "read_sql_query",
+    "read_sql",
+    "options",
+    "option_context",
+    "NamedAgg",
+]
+
+
+def _auto_patch_spark():
+    import os
+    import logging
+
+    # Attach a usage logger.
+    logger_module = os.getenv("KOALAS_USAGE_LOGGER", "")
+    if logger_module != "":
+        try:
+            from pyspark.pandas import usage_logging
+
+            usage_logging.attach(logger_module)
+        except Exception as e:
+            logger = logging.getLogger("pyspark.pandas.usage_logger")
+            logger.warning(
+                "Tried to attach usage logger `{}`, but an exception was raised: {}".format(
+                    logger_module, str(e)
+                )
+            )
+
+    # Autopatching is on by default.
+    x = os.getenv("SPARK_KOALAS_AUTOPATCH", "true")
+    if x.lower() in ("true", "1", "enabled"):
+        logger = logging.getLogger("spark")
+        logger.info(
+            "Patching spark automatically. You can disable it by setting "
+            "SPARK_KOALAS_AUTOPATCH=false in your environment"
+        )
+
+        from pyspark.sql import dataframe as df
+
+        df.DataFrame.to_koalas = DataFrame.to_koalas
+
+
+def _auto_patch_pandas():
+    import pandas as pd
+
+    # In order to use it in test cases.
+    global _frame_has_class_getitem
+    global _series_has_class_getitem
+
+    _frame_has_class_getitem = hasattr(pd.DataFrame, "__class_getitem__")
+    _series_has_class_getitem = hasattr(pd.Series, "__class_getitem__")
+
+    if sys.version_info >= (3, 7):
+        # Just in case pandas implements '__class_getitem__' later.
+        if not _frame_has_class_getitem:
+            pd.DataFrame.__class_getitem__ = lambda params: DataFrame.__class_getitem__(params)
+
+        if not _series_has_class_getitem:
+            pd.Series.__class_getitem__ = lambda params: Series.__class_getitem__(params)
+
+
+_auto_patch_spark()
+_auto_patch_pandas()
+
+# Import after the usage logger is attached.
+from pyspark.pandas.config import get_option, options, option_context, reset_option, set_option
+from pyspark.pandas.namespace import *  # F405
+from pyspark.pandas.sql_processor import sql
diff --git a/python/pyspark/pandas/accessors.py b/python/pyspark/pandas/accessors.py
new file mode 100644
index 0000000000000..40ca22bc561c4
--- /dev/null
+++ b/python/pyspark/pandas/accessors.py
@@ -0,0 +1,917 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+pandas-on-Spark specific features.
+"""
+import inspect
+from typing import Any, Optional, Tuple, Union, TYPE_CHECKING, cast
+import types
+
+import numpy as np  # noqa: F401
+import pandas as pd
+from pyspark.sql import functions as F
+from pyspark.sql.functions import pandas_udf, PandasUDFType
+from pyspark.sql.types import StructField, StructType
+
+from pyspark.pandas.internal import (
+    InternalFrame,
+    SPARK_INDEX_NAME_FORMAT,
+    SPARK_DEFAULT_SERIES_NAME,
+)
+from pyspark.pandas.typedef import infer_return_type, DataFrameType, ScalarType, SeriesType
+from pyspark.pandas.spark.utils import as_nullable_spark_type, force_decimal_precision_scale
+from pyspark.pandas.utils import (
+    is_name_like_value,
+    is_name_like_tuple,
+    name_like_string,
+    scol_for,
+    verify_temp_column_name,
+)
+
+if TYPE_CHECKING:
+    from pyspark.pandas.frame import DataFrame  # noqa: F401 (SPARK-34943)
+    from pyspark.pandas.series import Series  # noqa: F401 (SPARK-34943)
+
+
+class PandasOnSparkFrameMethods(object):
+    """ pandas-on-Spark specific features for DataFrame. """
+
+    def __init__(self, frame: "DataFrame"):
+        self._kdf = frame
+
+    def attach_id_column(self, id_type: str, column: Union[Any, Tuple]) -> "DataFrame":
+        """
+        Attach a column to be used as identifier of rows similar to the default index.
+
+        See also `Default Index type
+        <https://koalas.readthedocs.io/en/latest/user_guide/options.html#default-index-type>`_.
+
+        Parameters
+        ----------
+        id_type : string
+            The id type.
+
+            - 'sequence' : a sequence that increases one by one.
+
+              .. note:: this uses Spark's Window without specifying partition specification.
+                  This leads to move all data into single partition in single machine and
+                  could cause serious performance degradation.
+                  Avoid this method against very large dataset.
+
+            - 'distributed-sequence' : a sequence that increases one by one,
+              by group-by and group-map approach in a distributed manner.
+            - 'distributed' : a monotonically increasing sequence simply by using PySpark’s
+              monotonically_increasing_id function in a fully distributed manner.
+
+        column : string or tuple of string
+            The column name.
+
+        Returns
+        -------
+        DataFrame
+            The DataFrame attached the column.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({"x": ['a', 'b', 'c']})
+        >>> df.koalas.attach_id_column(id_type="sequence", column="id")
+           x  id
+        0  a   0
+        1  b   1
+        2  c   2
+
+        >>> df.koalas.attach_id_column(id_type="distributed-sequence", column=0)
+           x  0
+        0  a  0
+        1  b  1
+        2  c  2
+
+        >>> df.koalas.attach_id_column(id_type="distributed", column=0.0)
+        ... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+           x  0.0
+        0  a  ...
+        1  b  ...
+        2  c  ...
+
+        For multi-index columns:
+
+        >>> df = ps.DataFrame({("x", "y"): ['a', 'b', 'c']})
+        >>> df.koalas.attach_id_column(id_type="sequence", column=("id-x", "id-y"))
+           x id-x
+           y id-y
+        0  a    0
+        1  b    1
+        2  c    2
+
+        >>> df.koalas.attach_id_column(id_type="distributed-sequence", column=(0, 1.0))
+           x   0
+           y 1.0
+        0  a   0
+        1  b   1
+        2  c   2
+        """
+        from pyspark.pandas.frame import DataFrame
+
+        if id_type == "sequence":
+            attach_func = InternalFrame.attach_sequence_column
+        elif id_type == "distributed-sequence":
+            attach_func = InternalFrame.attach_distributed_sequence_column
+        elif id_type == "distributed":
+            attach_func = InternalFrame.attach_distributed_column
+        else:
+            raise ValueError(
+                "id_type should be one of 'sequence', 'distributed-sequence' and 'distributed'"
+            )
+
+        assert is_name_like_value(column, allow_none=False), column
+        if not is_name_like_tuple(column):
+            column = (column,)
+
+        internal = self._kdf._internal
+
+        if len(column) != internal.column_labels_level:
+            raise ValueError(
+                "The given column `{}` must be the same length as the existing columns.".format(
+                    column
+                )
+            )
+        elif column in internal.column_labels:
+            raise ValueError(
+                "The given column `{}` already exists.".format(name_like_string(column))
+            )
+
+        # Make sure the underlying Spark column names are the form of
+        # `name_like_string(column_label)`.
+        sdf = internal.spark_frame.select(
+            [
+                scol.alias(SPARK_INDEX_NAME_FORMAT(i))
+                for i, scol in enumerate(internal.index_spark_columns)
+            ]
+            + [
+                scol.alias(name_like_string(label))
+                for scol, label in zip(internal.data_spark_columns, internal.column_labels)
+            ]
+        )
+        sdf = attach_func(sdf, name_like_string(column))
+
+        return DataFrame(
+            InternalFrame(
+                spark_frame=sdf,
+                index_spark_columns=[
+                    scol_for(sdf, SPARK_INDEX_NAME_FORMAT(i)) for i in range(internal.index_level)
+                ],
+                index_names=internal.index_names,
+                index_dtypes=internal.index_dtypes,
+                column_labels=internal.column_labels + [column],
+                data_spark_columns=(
+                    [scol_for(sdf, name_like_string(label)) for label in internal.column_labels]
+                    + [scol_for(sdf, name_like_string(column))]
+                ),
+                data_dtypes=(internal.data_dtypes + [None]),
+                column_label_names=internal.column_label_names,
+            ).resolved_copy
+        )
+
+    def apply_batch(self, func, args=(), **kwds) -> "DataFrame":
+        """
+        Apply a function that takes pandas DataFrame and outputs pandas DataFrame. The pandas
+        DataFrame given to the function is of a batch used internally.
+
+        See also `Transform and apply a function
+        <https://koalas.readthedocs.io/en/latest/user_guide/transform_apply.html>`_.
+
+        .. note:: the `func` is unable to access to the whole input frame. pandas-on-Spark
+            internally splits the input series into multiple batches and calls `func` with each
+            batch multiple times. Therefore, operations such as global aggregations are impossible.
+            See the example below.
+
+            >>> # This case does not return the length of whole frame but of the batch internally
+            ... # used.
+            ... def length(pdf) -> ps.DataFrame[int]:
+            ...     return pd.DataFrame([len(pdf)])
+            ...
+            >>> df = ps.DataFrame({'A': range(1000)})
+            >>> df.koalas.apply_batch(length)  # doctest: +SKIP
+                c0
+            0   83
+            1   83
+            2   83
+            ...
+            10  83
+            11  83
+
+        .. note:: this API executes the function once to infer the type which is
+            potentially expensive, for instance, when the dataset is created after
+            aggregations or sorting.
+
+            To avoid this, specify return type in ``func``, for instance, as below:
+
+            >>> def plus_one(x) -> ps.DataFrame[float, float]:
+            ...     return x + 1
+
+            If the return type is specified, the output column names become
+            `c0, c1, c2 ... cn`. These names are positionally mapped to the returned
+            DataFrame in ``func``.
+
+            To specify the column names, you can assign them in a pandas friendly style as below:
+
+            >>> def plus_one(x) -> ps.DataFrame["a": float, "b": float]:
+            ...     return x + 1
+
+            >>> pdf = pd.DataFrame({'a': [1, 2, 3], 'b': [3, 4, 5]})
+            >>> def plus_one(x) -> ps.DataFrame[zip(pdf.dtypes, pdf.columns)]:
+            ...     return x + 1
+
+            When the given function has the return type annotated, the original index of the
+            DataFrame will be lost and a default index will be attached to the result DataFrame.
+            Please be careful about configuring the default index. See also `Default Index Type
+            <https://koalas.readthedocs.io/en/latest/user_guide/options.html#default-index-type>`_.
+
+
+        Parameters
+        ----------
+        func : function
+            Function to apply to each pandas frame.
+        args : tuple
+            Positional arguments to pass to `func` in addition to the
+            array/series.
+        **kwds
+            Additional keyword arguments to pass as keywords arguments to
+            `func`.
+
+        Returns
+        -------
+        DataFrame
+
+        See Also
+        --------
+        DataFrame.apply: For row/columnwise operations.
+        DataFrame.applymap: For elementwise operations.
+        DataFrame.aggregate: Only perform aggregating type operations.
+        DataFrame.transform: Only perform transforming type operations.
+        Series.koalas.transform_batch: transform the search as each pandas chunks.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([(1, 2), (3, 4), (5, 6)], columns=['A', 'B'])
+        >>> df
+           A  B
+        0  1  2
+        1  3  4
+        2  5  6
+
+        >>> def query_func(pdf) -> ps.DataFrame[int, int]:
+        ...     return pdf.query('A == 1')
+        >>> df.koalas.apply_batch(query_func)
+           c0  c1
+        0   1   2
+
+        >>> def query_func(pdf) -> ps.DataFrame["A": int, "B": int]:
+        ...     return pdf.query('A == 1')
+        >>> df.koalas.apply_batch(query_func)
+           A  B
+        0  1  2
+
+        You can also omit the type hints so pandas-on-Spark infers the return schema as below:
+
+        >>> df.koalas.apply_batch(lambda pdf: pdf.query('A == 1'))
+           A  B
+        0  1  2
+
+        You can also specify extra arguments.
+
+        >>> def calculation(pdf, y, z) -> ps.DataFrame[int, int]:
+        ...     return pdf ** y + z
+        >>> df.koalas.apply_batch(calculation, args=(10,), z=20)
+                c0        c1
+        0       21      1044
+        1    59069   1048596
+        2  9765645  60466196
+
+        You can also use ``np.ufunc`` and built-in functions as input.
+
+        >>> df.koalas.apply_batch(np.add, args=(10,))
+            A   B
+        0  11  12
+        1  13  14
+        2  15  16
+
+        >>> (df * -1).koalas.apply_batch(abs)
+           A  B
+        0  1  2
+        1  3  4
+        2  5  6
+
+        """
+        # TODO: codes here partially duplicate `DataFrame.apply`. Can we deduplicate?
+
+        from pyspark.pandas.groupby import GroupBy
+        from pyspark.pandas.frame import DataFrame
+        from pyspark import pandas as ps
+
+        if not isinstance(func, types.FunctionType):
+            assert callable(func), "the first argument should be a callable function."
+            f = func
+            func = lambda *args, **kwargs: f(*args, **kwargs)
+
+        spec = inspect.getfullargspec(func)
+        return_sig = spec.annotations.get("return", None)
+        should_infer_schema = return_sig is None
+
+        original_func = func
+        func = lambda o: original_func(o, *args, **kwds)
+
+        self_applied = DataFrame(self._kdf._internal.resolved_copy)  # type: DataFrame
+
+        if should_infer_schema:
+            # Here we execute with the first 1000 to get the return type.
+            # If the records were less than 1000, it uses pandas API directly for a shortcut.
+            limit = ps.get_option("compute.shortcut_limit")
+            pdf = self_applied.head(limit + 1)._to_internal_pandas()
+            applied = func(pdf)
+            if not isinstance(applied, pd.DataFrame):
+                raise ValueError(
+                    "The given function should return a frame; however, "
+                    "the return type was %s." % type(applied)
+                )
+            kdf = ps.DataFrame(applied)  # type: DataFrame
+            if len(pdf) <= limit:
+                return kdf
+
+            return_schema = force_decimal_precision_scale(
+                as_nullable_spark_type(kdf._internal.to_internal_spark_frame.schema)
+            )
+
+            output_func = GroupBy._make_pandas_df_builder_func(
+                self_applied, func, return_schema, retain_index=True
+            )
+            sdf = self_applied._internal.to_internal_spark_frame.mapInPandas(
+                lambda iterator: map(output_func, iterator), schema=return_schema
+            )
+
+            # If schema is inferred, we can restore indexes too.
+            internal = kdf._internal.with_new_sdf(sdf)
+        else:
+            return_type = infer_return_type(original_func)
+            is_return_dataframe = isinstance(return_type, DataFrameType)
+            if not is_return_dataframe:
+                raise TypeError(
+                    "The given function should specify a frame as its type "
+                    "hints; however, the return type was %s." % return_sig
+                )
+            return_schema = cast(DataFrameType, return_type).spark_type
+
+            output_func = GroupBy._make_pandas_df_builder_func(
+                self_applied, func, return_schema, retain_index=False
+            )
+            sdf = self_applied._internal.to_internal_spark_frame.mapInPandas(
+                lambda iterator: map(output_func, iterator), schema=return_schema
+            )
+
+            # Otherwise, it loses index.
+            internal = InternalFrame(
+                spark_frame=sdf,
+                index_spark_columns=None,
+                data_dtypes=cast(DataFrameType, return_type).dtypes,
+            )
+
+        return DataFrame(internal)
+
+    def transform_batch(self, func, *args, **kwargs) -> Union["DataFrame", "Series"]:
+        """
+        Transform chunks with a function that takes pandas DataFrame and outputs pandas DataFrame.
+        The pandas DataFrame given to the function is of a batch used internally. The length of
+        each input and output should be the same.
+
+        See also `Transform and apply a function
+        <https://koalas.readthedocs.io/en/latest/user_guide/transform_apply.html>`_.
+
+        .. note:: the `func` is unable to access to the whole input frame. pandas-on-Spark
+            internally splits the input series into multiple batches and calls `func` with each
+            batch multiple times. Therefore, operations such as global aggregations are impossible.
+            See the example below.
+
+            >>> # This case does not return the length of whole frame but of the batch internally
+            ... # used.
+            ... def length(pdf) -> ps.DataFrame[int]:
+            ...     return pd.DataFrame([len(pdf)] * len(pdf))
+            ...
+            >>> df = ps.DataFrame({'A': range(1000)})
+            >>> df.koalas.transform_batch(length)  # doctest: +SKIP
+                c0
+            0   83
+            1   83
+            2   83
+            ...
+
+        .. note:: this API executes the function once to infer the type which is
+            potentially expensive, for instance, when the dataset is created after
+            aggregations or sorting.
+
+            To avoid this, specify return type in ``func``, for instance, as below:
+
+            >>> def plus_one(x) -> ps.DataFrame[float, float]:
+            ...     return x + 1
+
+            If the return type is specified, the output column names become
+            `c0, c1, c2 ... cn`. These names are positionally mapped to the returned
+            DataFrame in ``func``.
+
+            To specify the column names, you can assign them in a pandas friendly style as below:
+
+            >>> def plus_one(x) -> ps.DataFrame['a': float, 'b': float]:
+            ...     return x + 1
+
+            >>> pdf = pd.DataFrame({'a': [1, 2, 3], 'b': [3, 4, 5]})
+            >>> def plus_one(x) -> ps.DataFrame[zip(pdf.dtypes, pdf.columns)]:
+            ...     return x + 1
+
+            When the given function returns DataFrame and has the return type annotated, the
+            original index of the DataFrame will be lost and then a default index will be attached
+            to the result. Please be careful about configuring the default index. See also
+            `Default Index Type
+            <https://koalas.readthedocs.io/en/latest/user_guide/options.html#default-index-type>`_.
+
+        Parameters
+        ----------
+        func : function
+            Function to transform each pandas frame.
+        *args
+            Positional arguments to pass to func.
+        **kwargs
+            Keyword arguments to pass to func.
+
+        Returns
+        -------
+        DataFrame or Series
+
+        See Also
+        --------
+        DataFrame.koalas.apply_batch: For row/columnwise operations.
+        Series.koalas.transform_batch: transform the search as each pandas chunks.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([(1, 2), (3, 4), (5, 6)], columns=['A', 'B'])
+        >>> df
+           A  B
+        0  1  2
+        1  3  4
+        2  5  6
+
+        >>> def plus_one_func(pdf) -> ps.DataFrame[int, int]:
+        ...     return pdf + 1
+        >>> df.koalas.transform_batch(plus_one_func)
+           c0  c1
+        0   2   3
+        1   4   5
+        2   6   7
+
+        >>> def plus_one_func(pdf) -> ps.DataFrame['A': int, 'B': int]:
+        ...     return pdf + 1
+        >>> df.koalas.transform_batch(plus_one_func)
+           A  B
+        0  2  3
+        1  4  5
+        2  6  7
+
+        >>> def plus_one_func(pdf) -> ps.Series[int]:
+        ...     return pdf.B + 1
+        >>> df.koalas.transform_batch(plus_one_func)
+        0    3
+        1    5
+        2    7
+        dtype: int64
+
+        You can also omit the type hints so pandas-on-Spark infers the return schema as below:
+
+        >>> df.koalas.transform_batch(lambda pdf: pdf + 1)
+           A  B
+        0  2  3
+        1  4  5
+        2  6  7
+
+        >>> (df * -1).koalas.transform_batch(abs)
+           A  B
+        0  1  2
+        1  3  4
+        2  5  6
+
+        Note that you should not transform the index. The index information will not change.
+
+        >>> df.koalas.transform_batch(lambda pdf: pdf.B + 1)
+        0    3
+        1    5
+        2    7
+        Name: B, dtype: int64
+
+        You can also specify extra arguments as below.
+
+        >>> df.koalas.transform_batch(lambda pdf, a, b, c: pdf.B + a + b + c, 1, 2, c=3)
+        0     8
+        1    10
+        2    12
+        Name: B, dtype: int64
+        """
+        from pyspark.pandas.groupby import GroupBy
+        from pyspark.pandas.frame import DataFrame
+        from pyspark.pandas.series import first_series
+        from pyspark import pandas as ps
+
+        assert callable(func), "the first argument should be a callable function."
+        spec = inspect.getfullargspec(func)
+        return_sig = spec.annotations.get("return", None)
+        should_infer_schema = return_sig is None
+        original_func = func
+        func = lambda o: original_func(o, *args, **kwargs)
+
+        names = self._kdf._internal.to_internal_spark_frame.schema.names
+
+        def pandas_concat(series):
+            # The input can only be a DataFrame for struct from Spark 3.0.
+            # This works around to make the input as a frame. See SPARK-27240
+            pdf = pd.concat(series, axis=1)
+            pdf.columns = names
+            return pdf
+
+        def apply_func(pdf):
+            return func(pdf).to_frame()
+
+        def pandas_extract(pdf, name):
+            # This is for output to work around a DataFrame for struct
+            # from Spark 3.0.  See SPARK-23836
+            return pdf[name]
+
+        def pandas_series_func(f):
+            ff = f
+            return lambda *series: first_series(ff(*series))
+
+        def pandas_frame_func(f, field_name):
+            ff = f
+            return lambda *series: pandas_extract(ff(pandas_concat(series)), field_name)
+
+        if should_infer_schema:
+            # Here we execute with the first 1000 to get the return type.
+            # If the records were less than 1000, it uses pandas API directly for a shortcut.
+            limit = ps.get_option("compute.shortcut_limit")
+            pdf = self._kdf.head(limit + 1)._to_internal_pandas()
+            transformed = func(pdf)
+            if not isinstance(transformed, (pd.DataFrame, pd.Series)):
+                raise ValueError(
+                    "The given function should return a frame; however, "
+                    "the return type was %s." % type(transformed)
+                )
+            if len(transformed) != len(pdf):
+                raise ValueError("transform_batch cannot produce aggregated results")
+            kdf_or_kser = ps.from_pandas(transformed)
+
+            if isinstance(kdf_or_kser, ps.Series):
+                kser = cast(ps.Series, kdf_or_kser)
+
+                spark_return_type = force_decimal_precision_scale(
+                    as_nullable_spark_type(kser.spark.data_type)
+                )
+                return_schema = StructType(
+                    [StructField(SPARK_DEFAULT_SERIES_NAME, spark_return_type)]
+                )
+                output_func = GroupBy._make_pandas_df_builder_func(
+                    self._kdf, apply_func, return_schema, retain_index=False
+                )
+
+                pudf = pandas_udf(
+                    pandas_series_func(output_func),
+                    returnType=spark_return_type,
+                    functionType=PandasUDFType.SCALAR,
+                )
+                columns = self._kdf._internal.spark_columns
+                # TODO: Index will be lost in this case.
+                internal = self._kdf._internal.copy(
+                    column_labels=kser._internal.column_labels,
+                    data_spark_columns=[
+                        pudf(F.struct(*columns)).alias(kser._internal.data_spark_column_names[0])
+                    ],
+                    data_dtypes=kser._internal.data_dtypes,
+                    column_label_names=kser._internal.column_label_names,
+                )
+                return first_series(DataFrame(internal))
+            else:
+                kdf = cast(DataFrame, kdf_or_kser)
+                if len(pdf) <= limit:
+                    # only do the short cut when it returns a frame to avoid
+                    # operations on different dataframes in case of series.
+                    return kdf
+
+                # Force nullability.
+                return_schema = force_decimal_precision_scale(
+                    as_nullable_spark_type(kdf._internal.to_internal_spark_frame.schema)
+                )
+
+                self_applied = DataFrame(self._kdf._internal.resolved_copy)  # type: DataFrame
+
+                output_func = GroupBy._make_pandas_df_builder_func(
+                    self_applied, func, return_schema, retain_index=True
+                )
+                columns = self_applied._internal.spark_columns
+
+                pudf = pandas_udf(
+                    output_func, returnType=return_schema, functionType=PandasUDFType.SCALAR
+                )
+                temp_struct_column = verify_temp_column_name(
+                    self_applied._internal.spark_frame, "__temp_struct__"
+                )
+                applied = pudf(F.struct(*columns)).alias(temp_struct_column)
+                sdf = self_applied._internal.spark_frame.select(applied)
+                sdf = sdf.selectExpr("%s.*" % temp_struct_column)
+
+                return DataFrame(kdf._internal.with_new_sdf(sdf))
+        else:
+            return_type = infer_return_type(original_func)
+            is_return_series = isinstance(return_type, SeriesType)
+            is_return_dataframe = isinstance(return_type, DataFrameType)
+            if not is_return_dataframe and not is_return_series:
+                raise TypeError(
+                    "The given function should specify a frame or series as its type "
+                    "hints; however, the return type was %s." % return_sig
+                )
+            if is_return_series:
+                spark_return_type = force_decimal_precision_scale(
+                    as_nullable_spark_type(cast(SeriesType, return_type).spark_type)
+                )
+                return_schema = StructType(
+                    [StructField(SPARK_DEFAULT_SERIES_NAME, spark_return_type)]
+                )
+                output_func = GroupBy._make_pandas_df_builder_func(
+                    self._kdf, apply_func, return_schema, retain_index=False
+                )
+
+                pudf = pandas_udf(
+                    pandas_series_func(output_func),
+                    returnType=spark_return_type,
+                    functionType=PandasUDFType.SCALAR,
+                )
+                columns = self._kdf._internal.spark_columns
+                internal = self._kdf._internal.copy(
+                    column_labels=[None],
+                    data_spark_columns=[pudf(F.struct(*columns)).alias(SPARK_DEFAULT_SERIES_NAME)],
+                    data_dtypes=[cast(SeriesType, return_type).dtype],
+                    column_label_names=None,
+                )
+                return first_series(DataFrame(internal))
+            else:
+                return_schema = cast(DataFrameType, return_type).spark_type
+
+                self_applied = DataFrame(self._kdf._internal.resolved_copy)
+
+                output_func = GroupBy._make_pandas_df_builder_func(
+                    self_applied, func, return_schema, retain_index=False
+                )
+                columns = self_applied._internal.spark_columns
+
+                pudf = pandas_udf(
+                    output_func, returnType=return_schema, functionType=PandasUDFType.SCALAR
+                )
+                temp_struct_column = verify_temp_column_name(
+                    self_applied._internal.spark_frame, "__temp_struct__"
+                )
+                applied = pudf(F.struct(*columns)).alias(temp_struct_column)
+                sdf = self_applied._internal.spark_frame.select(applied)
+                sdf = sdf.selectExpr("%s.*" % temp_struct_column)
+
+                internal = InternalFrame(
+                    spark_frame=sdf,
+                    index_spark_columns=None,
+                    data_dtypes=cast(DataFrameType, return_type).dtypes,
+                )
+                return DataFrame(internal)
+
+
+class PandasOnSparkSeriesMethods(object):
+    """ pandas-on-Spark specific features for Series. """
+
+    def __init__(self, series: "Series"):
+        self._kser = series
+
+    def transform_batch(self, func, *args, **kwargs) -> "Series":
+        """
+        Transform the data with the function that takes pandas Series and outputs pandas Series.
+        The pandas Series given to the function is of a batch used internally.
+
+        See also `Transform and apply a function
+        <https://koalas.readthedocs.io/en/latest/user_guide/transform_apply.html>`_.
+
+        .. note:: the `func` is unable to access to the whole input series. pandas-on-Spark
+            internally splits the input series into multiple batches and calls `func` with each
+            batch multiple times. Therefore, operations such as global aggregations are impossible.
+            See the example below.
+
+            >>> # This case does not return the length of whole frame but of the batch internally
+            ... # used.
+            ... def length(pser) -> ps.Series[int]:
+            ...     return pd.Series([len(pser)] * len(pser))
+            ...
+            >>> df = ps.DataFrame({'A': range(1000)})
+            >>> df.A.koalas.transform_batch(length)  # doctest: +SKIP
+                c0
+            0   83
+            1   83
+            2   83
+            ...
+
+        .. note:: this API executes the function once to infer the type which is
+            potentially expensive, for instance, when the dataset is created after
+            aggregations or sorting.
+
+            To avoid this, specify return type in ``func``, for instance, as below:
+
+            >>> def plus_one(x) -> ps.Series[int]:
+            ...     return x + 1
+
+        Parameters
+        ----------
+        func : function
+            Function to apply to each pandas frame.
+        *args
+            Positional arguments to pass to func.
+        **kwargs
+            Keyword arguments to pass to func.
+
+        Returns
+        -------
+        DataFrame
+
+        See Also
+        --------
+        DataFrame.koalas.apply_batch : Similar but it takes pandas DataFrame as its internal batch.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([(1, 2), (3, 4), (5, 6)], columns=['A', 'B'])
+        >>> df
+           A  B
+        0  1  2
+        1  3  4
+        2  5  6
+
+        >>> def plus_one_func(pser) -> ps.Series[np.int64]:
+        ...     return pser + 1
+        >>> df.A.koalas.transform_batch(plus_one_func)
+        0    2
+        1    4
+        2    6
+        Name: A, dtype: int64
+
+        You can also omit the type hints so pandas-on-Spark infers the return schema as below:
+
+        >>> df.A.koalas.transform_batch(lambda pser: pser + 1)
+        0    2
+        1    4
+        2    6
+        Name: A, dtype: int64
+
+        You can also specify extra arguments.
+
+        >>> def plus_one_func(pser, a, b, c=3) -> ps.Series[np.int64]:
+        ...     return pser + a + b + c
+        >>> df.A.koalas.transform_batch(plus_one_func, 1, b=2)
+        0     7
+        1     9
+        2    11
+        Name: A, dtype: int64
+
+        You can also use ``np.ufunc`` and built-in functions as input.
+
+        >>> df.A.koalas.transform_batch(np.add, 10)
+        0    11
+        1    13
+        2    15
+        Name: A, dtype: int64
+
+        >>> (df * -1).A.koalas.transform_batch(abs)
+        0    1
+        1    3
+        2    5
+        Name: A, dtype: int64
+        """
+        assert callable(func), "the first argument should be a callable function."
+
+        return_sig = None
+        try:
+            spec = inspect.getfullargspec(func)
+            return_sig = spec.annotations.get("return", None)
+        except TypeError:
+            # Falls back to schema inference if it fails to get signature.
+            pass
+
+        return_type = None
+        if return_sig is not None:
+            # Extract the signature arguments from this function.
+            sig_return = infer_return_type(func)
+            if not isinstance(sig_return, SeriesType):
+                raise ValueError(
+                    "Expected the return type of this function to be of type column,"
+                    " but found type {}".format(sig_return)
+                )
+            return_type = cast(SeriesType, sig_return)
+
+        return self._transform_batch(lambda c: func(c, *args, **kwargs), return_type)
+
+    def _transform_batch(self, func, return_type: Optional[Union[SeriesType, ScalarType]]):
+        from pyspark.pandas.groupby import GroupBy
+        from pyspark.pandas.series import Series, first_series
+        from pyspark import pandas as ps
+
+        if not isinstance(func, types.FunctionType):
+            f = func
+            func = lambda *args, **kwargs: f(*args, **kwargs)
+
+        if return_type is None:
+            # TODO: In this case, it avoids the shortcut for now (but only infers schema)
+            #  because it returns a series from a different DataFrame and it has a different
+            #  anchor. We should fix this to allow the shortcut or only allow to infer
+            #  schema.
+            limit = ps.get_option("compute.shortcut_limit")
+            pser = self._kser.head(limit + 1)._to_internal_pandas()
+            transformed = pser.transform(func)
+            kser = Series(transformed)  # type: Series
+            spark_return_type = force_decimal_precision_scale(
+                as_nullable_spark_type(kser.spark.data_type)
+            )
+            dtype = kser.dtype
+        else:
+            spark_return_type = return_type.spark_type
+            dtype = return_type.dtype
+
+        kdf = self._kser.to_frame()
+        columns = kdf._internal.spark_column_names
+
+        def pandas_concat(series):
+            # The input can only be a DataFrame for struct from Spark 3.0.
+            # This works around to make the input as a frame. See SPARK-27240
+            pdf = pd.concat(series, axis=1)
+            pdf.columns = columns
+            return pdf
+
+        def apply_func(pdf):
+            return func(first_series(pdf)).to_frame()
+
+        return_schema = StructType([StructField(SPARK_DEFAULT_SERIES_NAME, spark_return_type)])
+        output_func = GroupBy._make_pandas_df_builder_func(
+            kdf, apply_func, return_schema, retain_index=False
+        )
+
+        pudf = pandas_udf(
+            lambda *series: first_series(output_func(pandas_concat(series))),
+            returnType=spark_return_type,
+            functionType=PandasUDFType.SCALAR,
+        )
+
+        return self._kser._with_new_scol(
+            scol=pudf(*kdf._internal.spark_columns).alias(
+                self._kser._internal.spark_column_names[0]
+            ),
+            dtype=dtype,
+        )
+
+
+def _test():
+    import os
+    import doctest
+    import sys
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.accessors
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.accessors.__dict__.copy()
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]")
+        .appName("pyspark.pandas.accessors tests")
+        .getOrCreate()
+    )
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.accessors,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/base.py b/python/pyspark/pandas/base.py
new file mode 100644
index 0000000000000..9eff56d814376
--- /dev/null
+++ b/python/pyspark/pandas/base.py
@@ -0,0 +1,2024 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Base and utility classes for pandas-on-Spark objects.
+"""
+from abc import ABCMeta, abstractmethod
+import datetime
+from functools import wraps, partial
+from itertools import chain
+from typing import Any, Callable, Optional, Tuple, Union, cast, TYPE_CHECKING
+import warnings
+
+import numpy as np
+import pandas as pd  # noqa: F401
+from pandas.api.types import is_list_like, CategoricalDtype
+from pyspark import sql as spark
+from pyspark.sql import functions as F, Window, Column
+from pyspark.sql.types import (
+    BooleanType,
+    DateType,
+    DoubleType,
+    FloatType,
+    IntegralType,
+    LongType,
+    NumericType,
+    StringType,
+    TimestampType,
+)
+
+from pyspark import pandas as ps  # For running doctests and reference resolution in PyCharm.
+from pyspark.pandas import numpy_compat
+from pyspark.pandas.config import get_option, option_context
+from pyspark.pandas.internal import (
+    InternalFrame,
+    NATURAL_ORDER_COLUMN_NAME,
+    SPARK_DEFAULT_INDEX_NAME,
+)
+from pyspark.pandas.spark import functions as SF
+from pyspark.pandas.spark.accessors import SparkIndexOpsMethods
+from pyspark.pandas.typedef import (
+    Dtype,
+    as_spark_type,
+    extension_dtypes,
+    pandas_on_spark_type,
+    spark_type_to_pandas_dtype,
+)
+from pyspark.pandas.utils import (
+    combine_frames,
+    same_anchor,
+    scol_for,
+    validate_axis,
+    ERROR_MESSAGE_CANNOT_COMBINE,
+)
+from pyspark.pandas.frame import DataFrame
+
+if TYPE_CHECKING:
+    from pyspark.pandas.indexes import Index  # noqa: F401 (SPARK-34943)
+    from pyspark.pandas.series import Series  # noqa: F401 (SPARK-34943)
+
+
+def should_alignment_for_column_op(self: "IndexOpsMixin", other: "IndexOpsMixin") -> bool:
+    from pyspark.pandas.series import Series
+
+    if isinstance(self, Series) and isinstance(other, Series):
+        return not same_anchor(self, other)
+    else:
+        return self._internal.spark_frame is not other._internal.spark_frame
+
+
+def align_diff_index_ops(func, this_index_ops: "IndexOpsMixin", *args) -> "IndexOpsMixin":
+    """
+    Align the `IndexOpsMixin` objects and apply the function.
+
+    Parameters
+    ----------
+    func : The function to apply
+    this_index_ops : IndexOpsMixin
+        A base `IndexOpsMixin` object
+    args : list of other arguments including other `IndexOpsMixin` objects
+
+    Returns
+    -------
+    `Index` if all `this_index_ops` and arguments are `Index`; otherwise `Series`
+    """
+    from pyspark.pandas.indexes import Index
+    from pyspark.pandas.series import Series, first_series
+
+    cols = [arg for arg in args if isinstance(arg, IndexOpsMixin)]
+
+    if isinstance(this_index_ops, Series) and all(isinstance(col, Series) for col in cols):
+        combined = combine_frames(
+            this_index_ops.to_frame(),
+            *[cast(Series, col).rename(i) for i, col in enumerate(cols)],
+            how="full"
+        )
+
+        return column_op(func)(
+            combined["this"]._kser_for(combined["this"]._internal.column_labels[0]),
+            *[
+                combined["that"]._kser_for(label)
+                for label in combined["that"]._internal.column_labels
+            ]
+        ).rename(this_index_ops.name)
+    else:
+        # This could cause as many counts, reset_index calls, joins for combining
+        # as the number of `Index`s in `args`. So far it's fine since we can assume the ops
+        # only work between at most two `Index`s. We might need to fix it in the future.
+
+        self_len = len(this_index_ops)
+        if any(len(col) != self_len for col in args if isinstance(col, IndexOpsMixin)):
+            raise ValueError("operands could not be broadcast together with shapes")
+
+        with option_context("compute.default_index_type", "distributed-sequence"):
+            if isinstance(this_index_ops, Index) and all(isinstance(col, Index) for col in cols):
+                return Index(
+                    column_op(func)(
+                        this_index_ops.to_series().reset_index(drop=True),
+                        *[
+                            arg.to_series().reset_index(drop=True)
+                            if isinstance(arg, Index)
+                            else arg
+                            for arg in args
+                        ]
+                    ).sort_index(),
+                    name=this_index_ops.name,
+                )
+            elif isinstance(this_index_ops, Series):
+                this = this_index_ops.reset_index()
+                that = [
+                    cast(Series, col.to_series() if isinstance(col, Index) else col)
+                    .rename(i)
+                    .reset_index(drop=True)
+                    for i, col in enumerate(cols)
+                ]
+
+                combined = combine_frames(this, *that, how="full").sort_index()
+                combined = combined.set_index(
+                    combined._internal.column_labels[: this_index_ops._internal.index_level]
+                )
+                combined.index.names = this_index_ops._internal.index_names
+
+                return column_op(func)(
+                    first_series(combined["this"]),
+                    *[
+                        combined["that"]._kser_for(label)
+                        for label in combined["that"]._internal.column_labels
+                    ]
+                ).rename(this_index_ops.name)
+            else:
+                this = cast(Index, this_index_ops).to_frame().reset_index(drop=True)
+
+                that_series = next(col for col in cols if isinstance(col, Series))
+                that_frame = that_series._kdf[
+                    [
+                        cast(Series, col.to_series() if isinstance(col, Index) else col).rename(i)
+                        for i, col in enumerate(cols)
+                    ]
+                ]
+
+                combined = combine_frames(this, that_frame.reset_index()).sort_index()
+
+                self_index = (
+                    combined["this"].set_index(combined["this"]._internal.column_labels).index
+                )
+
+                other = combined["that"].set_index(
+                    combined["that"]._internal.column_labels[: that_series._internal.index_level]
+                )
+                other.index.names = that_series._internal.index_names
+
+                return column_op(func)(
+                    self_index,
+                    *[
+                        other._kser_for(label)
+                        for label, col in zip(other._internal.column_labels, cols)
+                    ]
+                ).rename(that_series.name)
+
+
+def booleanize_null(scol, f) -> Column:
+    """
+    Booleanize Null in Spark Column
+    """
+    comp_ops = [
+        getattr(Column, "__{}__".format(comp_op))
+        for comp_op in ["eq", "ne", "lt", "le", "ge", "gt"]
+    ]
+
+    if f in comp_ops:
+        # if `f` is "!=", fill null with True otherwise False
+        filler = f == Column.__ne__
+        scol = F.when(scol.isNull(), filler).otherwise(scol)
+
+    return scol
+
+
+def column_op(f):
+    """
+    A decorator that wraps APIs taking/returning Spark Column so that pandas-on-Spark Series can be
+    supported too. If this decorator is used for the `f` function that takes Spark Column and
+    returns Spark Column, decorated `f` takes pandas-on-Spark Series as well and returns
+    pandas-on-Spark Series.
+
+    :param f: a function that takes Spark Column and returns Spark Column.
+    :param self: pandas-on-Spark Series
+    :param args: arguments that the function `f` takes.
+    """
+
+    @wraps(f)
+    def wrapper(self, *args):
+        from pyspark.pandas.series import Series
+
+        # It is possible for the function `f` takes other arguments than Spark Column.
+        # To cover this case, explicitly check if the argument is pandas-on-Spark Series and
+        # extract Spark Column. For other arguments, they are used as are.
+        cols = [arg for arg in args if isinstance(arg, IndexOpsMixin)]
+
+        if all(not should_alignment_for_column_op(self, col) for col in cols):
+            # Same DataFrame anchors
+            args = [arg.spark.column if isinstance(arg, IndexOpsMixin) else arg for arg in args]
+            scol = f(self.spark.column, *args)
+
+            spark_type = self._internal.spark_frame.select(scol).schema[0].dataType
+            use_extension_dtypes = any(
+                isinstance(col.dtype, extension_dtypes) for col in [self] + cols
+            )
+            dtype = spark_type_to_pandas_dtype(
+                spark_type, use_extension_dtypes=use_extension_dtypes
+            )
+
+            if not isinstance(dtype, extension_dtypes):
+                scol = booleanize_null(scol, f)
+
+            if isinstance(self, Series) or not any(isinstance(col, Series) for col in cols):
+                index_ops = self._with_new_scol(scol, dtype=dtype)
+            else:
+                kser = next(col for col in cols if isinstance(col, Series))
+                index_ops = kser._with_new_scol(scol, dtype=dtype)
+        elif get_option("compute.ops_on_diff_frames"):
+            index_ops = align_diff_index_ops(f, self, *args)
+        else:
+            raise ValueError(ERROR_MESSAGE_CANNOT_COMBINE)
+
+        if not all(self.name == col.name for col in cols):
+            index_ops = index_ops.rename(None)
+
+        return index_ops
+
+    return wrapper
+
+
+def numpy_column_op(f):
+    @wraps(f)
+    def wrapper(self, *args):
+        # PySpark does not support NumPy type out of the box. For now, we convert NumPy types
+        # into some primitive types understandable in PySpark.
+        new_args = []
+        for arg in args:
+            # TODO: This is a quick hack to support NumPy type. We should revisit this.
+            if isinstance(self.spark.data_type, LongType) and isinstance(arg, np.timedelta64):
+                new_args.append(float(arg / np.timedelta64(1, "s")))
+            else:
+                new_args.append(arg)
+        return column_op(f)(self, *new_args)
+
+    return wrapper
+
+
+class IndexOpsMixin(object, metaclass=ABCMeta):
+    """common ops mixin to support a unified interface / docs for Series / Index
+
+    Assuming there are following attributes or properties and function.
+    """
+
+    @property
+    @abstractmethod
+    def _internal(self) -> InternalFrame:
+        pass
+
+    @property
+    @abstractmethod
+    def _kdf(self) -> DataFrame:
+        pass
+
+    @abstractmethod
+    def _with_new_scol(self, scol: spark.Column, *, dtype=None):
+        pass
+
+    @property
+    @abstractmethod
+    def _column_label(self) -> Tuple:
+        pass
+
+    @property
+    @abstractmethod
+    def spark(self) -> SparkIndexOpsMethods:
+        pass
+
+    @property
+    def spark_column(self) -> Column:
+        warnings.warn(
+            "Series.spark_column is deprecated as of Series.spark.column. "
+            "Please use the API instead.",
+            FutureWarning,
+        )
+        return self.spark.column
+
+    spark_column.__doc__ = SparkIndexOpsMethods.column.__doc__
+
+    # arithmetic operators
+    __neg__ = column_op(Column.__neg__)
+
+    def __add__(self, other) -> Union["Series", "Index"]:
+        if not isinstance(self.spark.data_type, StringType) and (
+            (isinstance(other, IndexOpsMixin) and isinstance(other.spark.data_type, StringType))
+            or isinstance(other, str)
+        ):
+            raise TypeError("string addition can only be applied to string series or literals.")
+
+        if isinstance(self.spark.data_type, TimestampType):
+            raise TypeError("addition can not be applied to date times.")
+
+        if isinstance(self.spark.data_type, StringType):
+            # Concatenate string columns
+            if isinstance(other, IndexOpsMixin) and isinstance(other.spark.data_type, StringType):
+                return column_op(F.concat)(self, other)
+            # Handle df['col'] + 'literal'
+            elif isinstance(other, str):
+                return column_op(F.concat)(self, F.lit(other))
+            else:
+                raise TypeError("string addition can only be applied to string series or literals.")
+        else:
+            return column_op(Column.__add__)(self, other)
+
+    def __sub__(self, other) -> Union["Series", "Index"]:
+        if (
+            isinstance(self.spark.data_type, StringType)
+            or (isinstance(other, IndexOpsMixin) and isinstance(other.spark.data_type, StringType))
+            or isinstance(other, str)
+        ):
+            raise TypeError("substraction can not be applied to string series or literals.")
+
+        if isinstance(self.spark.data_type, TimestampType):
+            # Note that timestamp subtraction casts arguments to integer. This is to mimic pandas's
+            # behaviors. pandas returns 'timedelta64[ns]' from 'datetime64[ns]'s subtraction.
+            msg = (
+                "Note that there is a behavior difference of timestamp subtraction. "
+                "The timestamp subtraction returns an integer in seconds, "
+                "whereas pandas returns 'timedelta64[ns]'."
+            )
+            if isinstance(other, IndexOpsMixin) and isinstance(
+                other.spark.data_type, TimestampType
+            ):
+                warnings.warn(msg, UserWarning)
+                return self.astype("long") - other.astype("long")
+            elif isinstance(other, datetime.datetime):
+                warnings.warn(msg, UserWarning)
+                return self.astype("long") - F.lit(other).cast(as_spark_type("long"))
+            else:
+                raise TypeError("datetime subtraction can only be applied to datetime series.")
+        elif isinstance(self.spark.data_type, DateType):
+            # Note that date subtraction casts arguments to integer. This is to mimic pandas's
+            # behaviors. pandas returns 'timedelta64[ns]' in days from date's subtraction.
+            msg = (
+                "Note that there is a behavior difference of date subtraction. "
+                "The date subtraction returns an integer in days, "
+                "whereas pandas returns 'timedelta64[ns]'."
+            )
+            if isinstance(other, IndexOpsMixin) and isinstance(other.spark.data_type, DateType):
+                warnings.warn(msg, UserWarning)
+                return column_op(F.datediff)(self, other).astype("long")
+            elif isinstance(other, datetime.date) and not isinstance(other, datetime.datetime):
+                warnings.warn(msg, UserWarning)
+                return column_op(F.datediff)(self, F.lit(other)).astype("long")
+            else:
+                raise TypeError("date subtraction can only be applied to date series.")
+        return column_op(Column.__sub__)(self, other)
+
+    def __mul__(self, other) -> Union["Series", "Index"]:
+        if isinstance(other, str):
+            raise TypeError("multiplication can not be applied to a string literal.")
+
+        if isinstance(self.spark.data_type, TimestampType):
+            raise TypeError("multiplication can not be applied to date times.")
+
+        if (
+            isinstance(self.spark.data_type, IntegralType)
+            and isinstance(other, IndexOpsMixin)
+            and isinstance(other.spark.data_type, StringType)
+        ):
+            return column_op(SF.repeat)(other, self)
+
+        if isinstance(self.spark.data_type, StringType):
+            if (
+                isinstance(other, IndexOpsMixin) and isinstance(other.spark.data_type, IntegralType)
+            ) or isinstance(other, int):
+                return column_op(SF.repeat)(self, other)
+            else:
+                raise TypeError(
+                    "a string series can only be multiplied to an int series or literal"
+                )
+
+        return column_op(Column.__mul__)(self, other)
+
+    def __truediv__(self, other) -> Union["Series", "Index"]:
+        """
+        __truediv__ has different behaviour between pandas and PySpark for several cases.
+        1. When divide np.inf by zero, PySpark returns null whereas pandas returns np.inf
+        2. When divide positive number by zero, PySpark returns null whereas pandas returns np.inf
+        3. When divide -np.inf by zero, PySpark returns null whereas pandas returns -np.inf
+        4. When divide negative number by zero, PySpark returns null whereas pandas returns -np.inf
+
+        +-------------------------------------------+
+        | dividend (divisor: 0) | PySpark |  pandas |
+        |-----------------------|---------|---------|
+        |         np.inf        |   null  |  np.inf |
+        |        -np.inf        |   null  | -np.inf |
+        |           10          |   null  |  np.inf |
+        |          -10          |   null  | -np.inf |
+        +-----------------------|---------|---------+
+        """
+
+        if (
+            isinstance(self.spark.data_type, StringType)
+            or (isinstance(other, IndexOpsMixin) and isinstance(other.spark.data_type, StringType))
+            or isinstance(other, str)
+        ):
+            raise TypeError("division can not be applied on string series or literals.")
+
+        if isinstance(self.spark.data_type, TimestampType):
+            raise TypeError("division can not be applied to date times.")
+
+        def truediv(left, right):
+            return F.when(F.lit(right != 0) | F.lit(right).isNull(), left.__div__(right)).otherwise(
+                F.when(F.lit(left == np.inf) | F.lit(left == -np.inf), left).otherwise(
+                    F.lit(np.inf).__div__(left)
+                )
+            )
+
+        return numpy_column_op(truediv)(self, other)
+
+    def __mod__(self, other) -> Union["Series", "Index"]:
+        if (
+            isinstance(self.spark.data_type, StringType)
+            or (isinstance(other, IndexOpsMixin) and isinstance(other.spark.data_type, StringType))
+            or isinstance(other, str)
+        ):
+            raise TypeError("modulo can not be applied on string series or literals.")
+
+        if isinstance(self.spark.data_type, TimestampType):
+            raise TypeError("modulo can not be applied to date times.")
+
+        def mod(left, right):
+            return ((left % right) + right) % right
+
+        return column_op(mod)(self, other)
+
+    def __radd__(self, other) -> Union["Series", "Index"]:
+        # Handle 'literal' + df['col']
+        if not isinstance(self.spark.data_type, StringType) and isinstance(other, str):
+            raise TypeError("string addition can only be applied to string series or literals.")
+
+        if isinstance(self.spark.data_type, TimestampType):
+            raise TypeError("addition can not be applied to date times.")
+
+        if isinstance(self.spark.data_type, StringType):
+            if isinstance(other, str):
+                return self._with_new_scol(
+                    F.concat(F.lit(other), self.spark.column)
+                )  # TODO: dtype?
+            else:
+                raise TypeError("string addition can only be applied to string series or literals.")
+        else:
+            return column_op(Column.__radd__)(self, other)
+
+    def __rsub__(self, other) -> Union["Series", "Index"]:
+        if isinstance(self.spark.data_type, StringType) or isinstance(other, str):
+            raise TypeError("substraction can not be applied to string series or literals.")
+
+        if isinstance(self.spark.data_type, TimestampType):
+            # Note that timestamp subtraction casts arguments to integer. This is to mimic pandas's
+            # behaviors. pandas returns 'timedelta64[ns]' from 'datetime64[ns]'s subtraction.
+            msg = (
+                "Note that there is a behavior difference of timestamp subtraction. "
+                "The timestamp subtraction returns an integer in seconds, "
+                "whereas pandas returns 'timedelta64[ns]'."
+            )
+            if isinstance(other, datetime.datetime):
+                warnings.warn(msg, UserWarning)
+                return -(self.astype("long") - F.lit(other).cast(as_spark_type("long")))
+            else:
+                raise TypeError("datetime subtraction can only be applied to datetime series.")
+        elif isinstance(self.spark.data_type, DateType):
+            # Note that date subtraction casts arguments to integer. This is to mimic pandas's
+            # behaviors. pandas returns 'timedelta64[ns]' in days from date's subtraction.
+            msg = (
+                "Note that there is a behavior difference of date subtraction. "
+                "The date subtraction returns an integer in days, "
+                "whereas pandas returns 'timedelta64[ns]'."
+            )
+            if isinstance(other, datetime.date) and not isinstance(other, datetime.datetime):
+                warnings.warn(msg, UserWarning)
+                return -column_op(F.datediff)(self, F.lit(other)).astype("long")
+            else:
+                raise TypeError("date subtraction can only be applied to date series.")
+        return column_op(Column.__rsub__)(self, other)
+
+    def __rmul__(self, other) -> Union["Series", "Index"]:
+        if isinstance(other, str):
+            raise TypeError("multiplication can not be applied to a string literal.")
+
+        if isinstance(self.spark.data_type, TimestampType):
+            raise TypeError("multiplication can not be applied to date times.")
+
+        if isinstance(self.spark.data_type, StringType):
+            if isinstance(other, int):
+                return column_op(SF.repeat)(self, other)
+            else:
+                raise TypeError(
+                    "a string series can only be multiplied to an int series or literal"
+                )
+
+        return column_op(Column.__rmul__)(self, other)
+
+    def __rtruediv__(self, other) -> Union["Series", "Index"]:
+        if isinstance(self.spark.data_type, StringType) or isinstance(other, str):
+            raise TypeError("division can not be applied on string series or literals.")
+
+        if isinstance(self.spark.data_type, TimestampType):
+            raise TypeError("division can not be applied to date times.")
+
+        def rtruediv(left, right):
+            return F.when(left == 0, F.lit(np.inf).__div__(right)).otherwise(
+                F.lit(right).__truediv__(left)
+            )
+
+        return numpy_column_op(rtruediv)(self, other)
+
+    def __floordiv__(self, other) -> Union["Series", "Index"]:
+        """
+        __floordiv__ has different behaviour between pandas and PySpark for several cases.
+        1. When divide np.inf by zero, PySpark returns null whereas pandas returns np.inf
+        2. When divide positive number by zero, PySpark returns null whereas pandas returns np.inf
+        3. When divide -np.inf by zero, PySpark returns null whereas pandas returns -np.inf
+        4. When divide negative number by zero, PySpark returns null whereas pandas returns -np.inf
+
+        +-------------------------------------------+
+        | dividend (divisor: 0) | PySpark |  pandas |
+        |-----------------------|---------|---------|
+        |         np.inf        |   null  |  np.inf |
+        |        -np.inf        |   null  | -np.inf |
+        |           10          |   null  |  np.inf |
+        |          -10          |   null  | -np.inf |
+        +-----------------------|---------|---------+
+        """
+        if (
+            isinstance(self.spark.data_type, StringType)
+            or (isinstance(other, IndexOpsMixin) and isinstance(other.spark.data_type, StringType))
+            or isinstance(other, str)
+        ):
+            raise TypeError("division can not be applied on string series or literals.")
+
+        if isinstance(self.spark.data_type, TimestampType):
+            raise TypeError("division can not be applied to date times.")
+
+        def floordiv(left, right):
+            return F.when(F.lit(right is np.nan), np.nan).otherwise(
+                F.when(
+                    F.lit(right != 0) | F.lit(right).isNull(), F.floor(left.__div__(right))
+                ).otherwise(
+                    F.when(F.lit(left == np.inf) | F.lit(left == -np.inf), left).otherwise(
+                        F.lit(np.inf).__div__(left)
+                    )
+                )
+            )
+
+        return numpy_column_op(floordiv)(self, other)
+
+    def __rfloordiv__(self, other) -> Union["Series", "Index"]:
+        if isinstance(self.spark.data_type, StringType) or isinstance(other, str):
+            raise TypeError("division can not be applied on string series or literals.")
+
+        if isinstance(self.spark.data_type, TimestampType):
+            raise TypeError("division can not be applied to date times.")
+
+        def rfloordiv(left, right):
+            return F.when(F.lit(left == 0), F.lit(np.inf).__div__(right)).otherwise(
+                F.when(F.lit(left) == np.nan, np.nan).otherwise(F.floor(F.lit(right).__div__(left)))
+            )
+
+        return numpy_column_op(rfloordiv)(self, other)
+
+    def __rmod__(self, other) -> Union["Series", "Index"]:
+        if isinstance(self.spark.data_type, StringType) or isinstance(other, str):
+            raise TypeError("modulo can not be applied on string series or literals.")
+
+        if isinstance(self.spark.data_type, TimestampType):
+            raise TypeError("modulo can not be applied to date times.")
+
+        def rmod(left, right):
+            return ((right % left) + left) % left
+
+        return column_op(rmod)(self, other)
+
+    def __pow__(self, other) -> Union["Series", "Index"]:
+        def pow_func(left, right):
+            return F.when(left == 1, left).otherwise(Column.__pow__(left, right))
+
+        return column_op(pow_func)(self, other)
+
+    def __rpow__(self, other) -> Union["Series", "Index"]:
+        def rpow_func(left, right):
+            return F.when(F.lit(right == 1), right).otherwise(Column.__rpow__(left, right))
+
+        return column_op(rpow_func)(self, other)
+
+    __abs__ = column_op(F.abs)
+
+    # comparison operators
+    __eq__ = column_op(Column.__eq__)
+    __ne__ = column_op(Column.__ne__)
+    __lt__ = column_op(Column.__lt__)
+    __le__ = column_op(Column.__le__)
+    __ge__ = column_op(Column.__ge__)
+    __gt__ = column_op(Column.__gt__)
+
+    # `and`, `or`, `not` cannot be overloaded in Python,
+    # so use bitwise operators as boolean operators
+    def __and__(self, other) -> Union["Series", "Index"]:
+        if isinstance(self.dtype, extension_dtypes) or (
+            isinstance(other, IndexOpsMixin) and isinstance(other.dtype, extension_dtypes)
+        ):
+
+            def and_func(left, right):
+                if not isinstance(right, spark.Column):
+                    if pd.isna(right):
+                        right = F.lit(None)
+                    else:
+                        right = F.lit(right)
+                return left & right
+
+        else:
+
+            def and_func(left, right):
+                if not isinstance(right, spark.Column):
+                    if pd.isna(right):
+                        right = F.lit(None)
+                    else:
+                        right = F.lit(right)
+                scol = left & right
+                return F.when(scol.isNull(), False).otherwise(scol)
+
+        return column_op(and_func)(self, other)
+
+    def __or__(self, other) -> Union["Series", "Index"]:
+        if isinstance(self.dtype, extension_dtypes) or (
+            isinstance(other, IndexOpsMixin) and isinstance(other.dtype, extension_dtypes)
+        ):
+
+            def or_func(left, right):
+                if not isinstance(right, spark.Column):
+                    if pd.isna(right):
+                        right = F.lit(None)
+                    else:
+                        right = F.lit(right)
+                return left | right
+
+        else:
+
+            def or_func(left, right):
+                if not isinstance(right, spark.Column) and pd.isna(right):
+                    return F.lit(False)
+                else:
+                    scol = left | F.lit(right)
+                    return F.when(left.isNull() | scol.isNull(), False).otherwise(scol)
+
+        return column_op(or_func)(self, other)
+
+    __invert__ = column_op(Column.__invert__)
+
+    def __rand__(self, other) -> Union["Series", "Index"]:
+        return self.__and__(other)
+
+    def __ror__(self, other) -> Union["Series", "Index"]:
+        return self.__or__(other)
+
+    def __len__(self):
+        return len(self._kdf)
+
+    # NDArray Compat
+    def __array_ufunc__(self, ufunc: Callable, method: str, *inputs: Any, **kwargs: Any):
+        # Try dunder methods first.
+        result = numpy_compat.maybe_dispatch_ufunc_to_dunder_op(
+            self, ufunc, method, *inputs, **kwargs
+        )
+
+        # After that, we try with PySpark APIs.
+        if result is NotImplemented:
+            result = numpy_compat.maybe_dispatch_ufunc_to_spark_func(
+                self, ufunc, method, *inputs, **kwargs
+            )
+
+        if result is not NotImplemented:
+            return result
+        else:
+            # TODO: support more APIs?
+            raise NotImplementedError(
+                "pandas-on-Spark objects currently do not support %s." % ufunc)
+
+    @property
+    def dtype(self) -> Dtype:
+        """Return the dtype object of the underlying data.
+
+        Examples
+        --------
+        >>> s = ps.Series([1, 2, 3])
+        >>> s.dtype
+        dtype('int64')
+
+        >>> s = ps.Series(list('abc'))
+        >>> s.dtype
+        dtype('O')
+
+        >>> s = ps.Series(pd.date_range('20130101', periods=3))
+        >>> s.dtype
+        dtype('<M8[ns]')
+
+        >>> s.rename("a").to_frame().set_index("a").index.dtype
+        dtype('<M8[ns]')
+        """
+        return self._internal.data_dtypes[0]
+
+    @property
+    def empty(self) -> bool:
+        """
+        Returns true if the current object is empty. Otherwise, returns false.
+
+        >>> ps.range(10).id.empty
+        False
+
+        >>> ps.range(0).id.empty
+        True
+
+        >>> ps.DataFrame({}, index=list('abc')).index.empty
+        False
+        """
+        return self._internal.resolved_copy.spark_frame.rdd.isEmpty()
+
+    @property
+    def hasnans(self) -> bool:
+        """
+        Return True if it has any missing values. Otherwise, it returns False.
+
+        >>> ps.DataFrame({}, index=list('abc')).index.hasnans
+        False
+
+        >>> ps.Series(['a', None]).hasnans
+        True
+
+        >>> ps.Series([1.0, 2.0, np.nan]).hasnans
+        True
+
+        >>> ps.Series([1, 2, 3]).hasnans
+        False
+
+        >>> (ps.Series([1.0, 2.0, np.nan]) + 1).hasnans
+        True
+
+        >>> ps.Series([1, 2, 3]).rename("a").to_frame().set_index("a").index.hasnans
+        False
+        """
+        sdf = self._internal.spark_frame
+        scol = self.spark.column
+
+        if isinstance(self.spark.data_type, (DoubleType, FloatType)):
+            return sdf.select(F.max(scol.isNull() | F.isnan(scol))).collect()[0][0]
+        else:
+            return sdf.select(F.max(scol.isNull())).collect()[0][0]
+
+    @property
+    def is_monotonic(self) -> bool:
+        """
+        Return boolean if values in the object are monotonically increasing.
+
+        .. note:: the current implementation of is_monotonic requires to shuffle
+            and aggregate multiple times to check the order locally and globally,
+            which is potentially expensive. In case of multi-index, all data are
+            transferred to single node which can easily cause out-of-memory error currently.
+
+        .. note:: Disable the Spark config `spark.sql.optimizer.nestedSchemaPruning.enabled`
+            for multi-index if you're using pandas-on-Spark < 1.7.0 with PySpark 3.1.1.
+
+        Returns
+        -------
+        is_monotonic : bool
+
+        Examples
+        --------
+        >>> ser = ps.Series(['1/1/2018', '3/1/2018', '4/1/2018'])
+        >>> ser.is_monotonic
+        True
+
+        >>> df = ps.DataFrame({'dates': [None, '1/1/2018', '2/1/2018', '3/1/2018']})
+        >>> df.dates.is_monotonic
+        False
+
+        >>> df.index.is_monotonic
+        True
+
+        >>> ser = ps.Series([1])
+        >>> ser.is_monotonic
+        True
+
+        >>> ser = ps.Series([])
+        >>> ser.is_monotonic
+        True
+
+        >>> ser.rename("a").to_frame().set_index("a").index.is_monotonic
+        True
+
+        >>> ser = ps.Series([5, 4, 3, 2, 1], index=[1, 2, 3, 4, 5])
+        >>> ser.is_monotonic
+        False
+
+        >>> ser.index.is_monotonic
+        True
+
+        Support for MultiIndex
+
+        >>> midx = ps.MultiIndex.from_tuples(
+        ... [('x', 'a'), ('x', 'b'), ('y', 'c'), ('y', 'd'), ('z', 'e')])
+        >>> midx  # doctest: +SKIP
+        MultiIndex([('x', 'a'),
+                    ('x', 'b'),
+                    ('y', 'c'),
+                    ('y', 'd'),
+                    ('z', 'e')],
+                   )
+        >>> midx.is_monotonic
+        True
+
+        >>> midx = ps.MultiIndex.from_tuples(
+        ... [('z', 'a'), ('z', 'b'), ('y', 'c'), ('y', 'd'), ('x', 'e')])
+        >>> midx  # doctest: +SKIP
+        MultiIndex([('z', 'a'),
+                    ('z', 'b'),
+                    ('y', 'c'),
+                    ('y', 'd'),
+                    ('x', 'e')],
+                   )
+        >>> midx.is_monotonic
+        False
+        """
+        return self._is_monotonic("increasing")
+
+    is_monotonic_increasing = is_monotonic
+
+    @property
+    def is_monotonic_decreasing(self) -> bool:
+        """
+        Return boolean if values in the object are monotonically decreasing.
+
+        .. note:: the current implementation of is_monotonic_decreasing requires to shuffle
+            and aggregate multiple times to check the order locally and globally,
+            which is potentially expensive. In case of multi-index, all data are transferred
+            to single node which can easily cause out-of-memory error currently.
+
+        .. note:: Disable the Spark config `spark.sql.optimizer.nestedSchemaPruning.enabled`
+            for multi-index if you're using pandas-on-Spark < 1.7.0 with PySpark 3.1.1.
+
+        Returns
+        -------
+        is_monotonic : bool
+
+        Examples
+        --------
+        >>> ser = ps.Series(['4/1/2018', '3/1/2018', '1/1/2018'])
+        >>> ser.is_monotonic_decreasing
+        True
+
+        >>> df = ps.DataFrame({'dates': [None, '3/1/2018', '2/1/2018', '1/1/2018']})
+        >>> df.dates.is_monotonic_decreasing
+        False
+
+        >>> df.index.is_monotonic_decreasing
+        False
+
+        >>> ser = ps.Series([1])
+        >>> ser.is_monotonic_decreasing
+        True
+
+        >>> ser = ps.Series([])
+        >>> ser.is_monotonic_decreasing
+        True
+
+        >>> ser.rename("a").to_frame().set_index("a").index.is_monotonic_decreasing
+        True
+
+        >>> ser = ps.Series([5, 4, 3, 2, 1], index=[1, 2, 3, 4, 5])
+        >>> ser.is_monotonic_decreasing
+        True
+
+        >>> ser.index.is_monotonic_decreasing
+        False
+
+        Support for MultiIndex
+
+        >>> midx = ps.MultiIndex.from_tuples(
+        ... [('x', 'a'), ('x', 'b'), ('y', 'c'), ('y', 'd'), ('z', 'e')])
+        >>> midx  # doctest: +SKIP
+        MultiIndex([('x', 'a'),
+                    ('x', 'b'),
+                    ('y', 'c'),
+                    ('y', 'd'),
+                    ('z', 'e')],
+                   )
+        >>> midx.is_monotonic_decreasing
+        False
+
+        >>> midx = ps.MultiIndex.from_tuples(
+        ... [('z', 'e'), ('z', 'd'), ('y', 'c'), ('y', 'b'), ('x', 'a')])
+        >>> midx  # doctest: +SKIP
+        MultiIndex([('z', 'a'),
+                    ('z', 'b'),
+                    ('y', 'c'),
+                    ('y', 'd'),
+                    ('x', 'e')],
+                   )
+        >>> midx.is_monotonic_decreasing
+        True
+        """
+        return self._is_monotonic("decreasing")
+
+    def _is_locally_monotonic_spark_column(self, order):
+        window = (
+            Window.partitionBy(F.col("__partition_id"))
+            .orderBy(NATURAL_ORDER_COLUMN_NAME)
+            .rowsBetween(-1, -1)
+        )
+
+        if order == "increasing":
+            return (F.col("__origin") >= F.lag(F.col("__origin"), 1).over(window)) & F.col(
+                "__origin"
+            ).isNotNull()
+        else:
+            return (F.col("__origin") <= F.lag(F.col("__origin"), 1).over(window)) & F.col(
+                "__origin"
+            ).isNotNull()
+
+    def _is_monotonic(self, order):
+        assert order in ("increasing", "decreasing")
+
+        sdf = self._internal.spark_frame
+
+        sdf = (
+            sdf.select(
+                F.spark_partition_id().alias(
+                    "__partition_id"
+                ),  # Make sure we use the same partition id in the whole job.
+                F.col(NATURAL_ORDER_COLUMN_NAME),
+                self.spark.column.alias("__origin"),
+            )
+            .select(
+                F.col("__partition_id"),
+                F.col("__origin"),
+                self._is_locally_monotonic_spark_column(order).alias(
+                    "__comparison_within_partition"
+                ),
+            )
+            .groupby(F.col("__partition_id"))
+            .agg(
+                F.min(F.col("__origin")).alias("__partition_min"),
+                F.max(F.col("__origin")).alias("__partition_max"),
+                F.min(F.coalesce(F.col("__comparison_within_partition"), F.lit(True))).alias(
+                    "__comparison_within_partition"
+                ),
+            )
+        )
+
+        # Now we're windowing the aggregation results without partition specification.
+        # The number of rows here will be as the same of partitions, which is expected
+        # to be small.
+        window = Window.orderBy(F.col("__partition_id")).rowsBetween(-1, -1)
+        if order == "increasing":
+            comparison_col = F.col("__partition_min") >= F.lag(F.col("__partition_max"), 1).over(
+                window
+            )
+        else:
+            comparison_col = F.col("__partition_min") <= F.lag(F.col("__partition_max"), 1).over(
+                window
+            )
+
+        sdf = sdf.select(
+            comparison_col.alias("__comparison_between_partitions"),
+            F.col("__comparison_within_partition"),
+        )
+
+        ret = sdf.select(
+            F.min(F.coalesce(F.col("__comparison_between_partitions"), F.lit(True)))
+            & F.min(F.coalesce(F.col("__comparison_within_partition"), F.lit(True)))
+        ).collect()[0][0]
+        if ret is None:
+            return True
+        else:
+            return ret
+
+    @property
+    def ndim(self) -> int:
+        """
+        Return an int representing the number of array dimensions.
+
+        Return 1 for Series / Index / MultiIndex.
+
+        Examples
+        --------
+
+        For Series
+
+        >>> s = ps.Series([None, 1, 2, 3, 4], index=[4, 5, 2, 1, 8])
+        >>> s.ndim
+        1
+
+        For Index
+
+        >>> s.index.ndim
+        1
+
+        For MultiIndex
+
+        >>> midx = pd.MultiIndex([['lama', 'cow', 'falcon'],
+        ...                       ['speed', 'weight', 'length']],
+        ...                      [[0, 0, 0, 1, 1, 1, 2, 2, 2],
+        ...                       [1, 1, 1, 1, 1, 2, 1, 2, 2]])
+        >>> s = ps.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], index=midx)
+        >>> s.index.ndim
+        1
+        """
+        return 1
+
+    def astype(self, dtype: Union[str, type, Dtype]) -> Union["Index", "Series"]:
+        """
+        Cast a pandas-on-Spark object to a specified dtype ``dtype``.
+
+        Parameters
+        ----------
+        dtype : data type
+            Use a numpy.dtype or Python type to cast entire pandas object to
+            the same type.
+
+        Returns
+        -------
+        casted : same type as caller
+
+        See Also
+        --------
+        to_datetime : Convert argument to datetime.
+
+        Examples
+        --------
+        >>> ser = ps.Series([1, 2], dtype='int32')
+        >>> ser
+        0    1
+        1    2
+        dtype: int32
+
+        >>> ser.astype('int64')
+        0    1
+        1    2
+        dtype: int64
+
+        >>> ser.rename("a").to_frame().set_index("a").index.astype('int64')
+        Int64Index([1, 2], dtype='int64', name='a')
+        """
+        dtype, spark_type = pandas_on_spark_type(dtype)
+        if not spark_type:
+            raise ValueError("Type {} not understood".format(dtype))
+
+        if isinstance(self.dtype, CategoricalDtype):
+            if isinstance(dtype, CategoricalDtype) and dtype.categories is None:
+                return cast(Union[ps.Index, ps.Series], self).copy()
+
+            categories = self.dtype.categories
+            if len(categories) == 0:
+                scol = F.lit(None)
+            else:
+                kvs = list(
+                    chain(
+                        *[
+                            (F.lit(code), F.lit(category))
+                            for code, category in enumerate(categories)
+                        ]
+                    )
+                )
+                map_scol = F.create_map(kvs)
+                scol = map_scol.getItem(self.spark.column)
+            return self._with_new_scol(
+                scol.alias(self._internal.data_spark_column_names[0])
+            ).astype(dtype)
+        elif isinstance(dtype, CategoricalDtype):
+            if dtype.categories is None:
+                codes, uniques = self.factorize()
+                return codes._with_new_scol(
+                    codes.spark.column, dtype=CategoricalDtype(categories=uniques)
+                )
+            else:
+                categories = dtype.categories
+                if len(categories) == 0:
+                    scol = F.lit(-1)
+                else:
+                    kvs = list(
+                        chain(
+                            *[
+                                (F.lit(category), F.lit(code))
+                                for code, category in enumerate(categories)
+                            ]
+                        )
+                    )
+                    map_scol = F.create_map(kvs)
+
+                    scol = F.coalesce(map_scol.getItem(self.spark.column), F.lit(-1))
+                return self._with_new_scol(
+                    scol.alias(self._internal.data_spark_column_names[0]), dtype=dtype
+                )
+
+        if isinstance(spark_type, BooleanType):
+            if isinstance(dtype, extension_dtypes):
+                scol = self.spark.column.cast(spark_type)
+            else:
+                if isinstance(self.spark.data_type, StringType):
+                    scol = F.when(self.spark.column.isNull(), F.lit(False)).otherwise(
+                        F.length(self.spark.column) > 0
+                    )
+                elif isinstance(self.spark.data_type, (FloatType, DoubleType)):
+                    scol = F.when(
+                        self.spark.column.isNull() | F.isnan(self.spark.column), F.lit(True)
+                    ).otherwise(self.spark.column.cast(spark_type))
+                else:
+                    scol = F.when(self.spark.column.isNull(), F.lit(False)).otherwise(
+                        self.spark.column.cast(spark_type)
+                    )
+        elif isinstance(spark_type, StringType):
+            if isinstance(dtype, extension_dtypes):
+                if isinstance(self.spark.data_type, BooleanType):
+                    scol = F.when(
+                        self.spark.column.isNotNull(),
+                        F.when(self.spark.column, "True").otherwise("False"),
+                    )
+                elif isinstance(self.spark.data_type, TimestampType):
+                    # seems like a pandas' bug?
+                    scol = F.when(self.spark.column.isNull(), str(pd.NaT)).otherwise(
+                        self.spark.column.cast(spark_type)
+                    )
+                else:
+                    scol = self.spark.column.cast(spark_type)
+            else:
+                if isinstance(self.spark.data_type, NumericType):
+                    null_str = str(np.nan)
+                elif isinstance(self.spark.data_type, (DateType, TimestampType)):
+                    null_str = str(pd.NaT)
+                else:
+                    null_str = str(None)
+                if isinstance(self.spark.data_type, BooleanType):
+                    casted = F.when(self.spark.column, "True").otherwise("False")
+                else:
+                    casted = self.spark.column.cast(spark_type)
+                scol = F.when(self.spark.column.isNull(), null_str).otherwise(casted)
+        else:
+            scol = self.spark.column.cast(spark_type)
+        return self._with_new_scol(
+            scol.alias(self._internal.data_spark_column_names[0]), dtype=dtype
+        )
+
+    def isin(self, values) -> Union["Series", "Index"]:
+        """
+        Check whether `values` are contained in Series or Index.
+
+        Return a boolean Series or Index showing whether each element in the Series
+        matches an element in the passed sequence of `values` exactly.
+
+        Parameters
+        ----------
+        values : set or list-like
+            The sequence of values to test.
+
+        Returns
+        -------
+        isin : Series (bool dtype) or Index (bool dtype)
+
+        Examples
+        --------
+        >>> s = ps.Series(['lama', 'cow', 'lama', 'beetle', 'lama',
+        ...                'hippo'], name='animal')
+        >>> s.isin(['cow', 'lama'])
+        0     True
+        1     True
+        2     True
+        3    False
+        4     True
+        5    False
+        Name: animal, dtype: bool
+
+        Passing a single string as ``s.isin('lama')`` will raise an error. Use
+        a list of one element instead:
+
+        >>> s.isin(['lama'])
+        0     True
+        1    False
+        2     True
+        3    False
+        4     True
+        5    False
+        Name: animal, dtype: bool
+
+        >>> s.rename("a").to_frame().set_index("a").index.isin(['lama'])
+        Index([True, False, True, False, True, False], dtype='object', name='a')
+        """
+        if not is_list_like(values):
+            raise TypeError(
+                "only list-like objects are allowed to be passed"
+                " to isin(), you passed a [{values_type}]".format(values_type=type(values).__name__)
+            )
+
+        values = values.tolist() if isinstance(values, np.ndarray) else list(values)
+        return self._with_new_scol(self.spark.column.isin(values))
+
+    def isnull(self) -> Union["Series", "Index"]:
+        """
+        Detect existing (non-missing) values.
+
+        Return a boolean same-sized object indicating if the values are NA.
+        NA values, such as None or numpy.NaN, gets mapped to True values.
+        Everything else gets mapped to False values. Characters such as empty strings '' or
+        numpy.inf are not considered NA values
+        (unless you set pandas.options.mode.use_inf_as_na = True).
+
+        Returns
+        -------
+        Series or Index : Mask of bool values for each element in Series
+            that indicates whether an element is not an NA value.
+
+        Examples
+        --------
+        >>> ser = ps.Series([5, 6, np.NaN])
+        >>> ser.isna()  # doctest: +NORMALIZE_WHITESPACE
+        0    False
+        1    False
+        2     True
+        dtype: bool
+
+        >>> ser.rename("a").to_frame().set_index("a").index.isna()
+        Index([False, False, True], dtype='object', name='a')
+        """
+        from pyspark.pandas.indexes import MultiIndex
+
+        if isinstance(self, MultiIndex):
+            raise NotImplementedError("isna is not defined for MultiIndex")
+        if isinstance(self.spark.data_type, (FloatType, DoubleType)):
+            return self._with_new_scol(self.spark.column.isNull() | F.isnan(self.spark.column))
+        else:
+            return self._with_new_scol(self.spark.column.isNull())
+
+    isna = isnull
+
+    def notnull(self) -> Union["Series", "Index"]:
+        """
+        Detect existing (non-missing) values.
+        Return a boolean same-sized object indicating if the values are not NA.
+        Non-missing values get mapped to True.
+        Characters such as empty strings '' or numpy.inf are not considered NA values
+        (unless you set pandas.options.mode.use_inf_as_na = True).
+        NA values, such as None or numpy.NaN, get mapped to False values.
+
+        Returns
+        -------
+        Series or Index : Mask of bool values for each element in Series
+            that indicates whether an element is not an NA value.
+
+        Examples
+        --------
+        Show which entries in a Series are not NA.
+
+        >>> ser = ps.Series([5, 6, np.NaN])
+        >>> ser
+        0    5.0
+        1    6.0
+        2    NaN
+        dtype: float64
+
+        >>> ser.notna()
+        0     True
+        1     True
+        2    False
+        dtype: bool
+
+        >>> ser.rename("a").to_frame().set_index("a").index.notna()
+        Index([True, True, False], dtype='object', name='a')
+        """
+        from pyspark.pandas.indexes import MultiIndex
+
+        if isinstance(self, MultiIndex):
+            raise NotImplementedError("notna is not defined for MultiIndex")
+        return (~self.isnull()).rename(
+            self.name  # type: ignore
+        )
+
+    notna = notnull
+
+    # TODO: axis, skipna, and many arguments should be implemented.
+    def all(self, axis: Union[int, str] = 0) -> bool:
+        """
+        Return whether all elements are True.
+
+        Returns True unless there at least one element within a series that is
+        False or equivalent (e.g. zero or empty)
+
+        Parameters
+        ----------
+        axis : {0 or 'index'}, default 0
+            Indicate which axis or axes should be reduced.
+
+            * 0 / 'index' : reduce the index, return a Series whose index is the
+              original column labels.
+
+        Examples
+        --------
+        >>> ps.Series([True, True]).all()
+        True
+
+        >>> ps.Series([True, False]).all()
+        False
+
+        >>> ps.Series([0, 1]).all()
+        False
+
+        >>> ps.Series([1, 2, 3]).all()
+        True
+
+        >>> ps.Series([True, True, None]).all()
+        True
+
+        >>> ps.Series([True, False, None]).all()
+        False
+
+        >>> ps.Series([]).all()
+        True
+
+        >>> ps.Series([np.nan]).all()
+        True
+
+        >>> df = ps.Series([True, False, None]).rename("a").to_frame()
+        >>> df.set_index("a").index.all()
+        False
+        """
+        axis = validate_axis(axis)
+        if axis != 0:
+            raise NotImplementedError('axis should be either 0 or "index" currently.')
+
+        sdf = self._internal.spark_frame.select(self.spark.column)
+        col = scol_for(sdf, sdf.columns[0])
+
+        # Note that we're ignoring `None`s here for now.
+        # any and every was added as of Spark 3.0
+        # ret = sdf.select(F.expr("every(CAST(`%s` AS BOOLEAN))" % sdf.columns[0])).collect()[0][0]
+        # Here we use min as its alternative:
+        ret = sdf.select(F.min(F.coalesce(col.cast("boolean"), F.lit(True)))).collect()[0][0]
+        if ret is None:
+            return True
+        else:
+            return ret
+
+    # TODO: axis, skipna, and many arguments should be implemented.
+    def any(self, axis: Union[int, str] = 0) -> bool:
+        """
+        Return whether any element is True.
+
+        Returns False unless there at least one element within a series that is
+        True or equivalent (e.g. non-zero or non-empty).
+
+        Parameters
+        ----------
+        axis : {0 or 'index'}, default 0
+            Indicate which axis or axes should be reduced.
+
+            * 0 / 'index' : reduce the index, return a Series whose index is the
+              original column labels.
+
+        Examples
+        --------
+        >>> ps.Series([False, False]).any()
+        False
+
+        >>> ps.Series([True, False]).any()
+        True
+
+        >>> ps.Series([0, 0]).any()
+        False
+
+        >>> ps.Series([0, 1, 2]).any()
+        True
+
+        >>> ps.Series([False, False, None]).any()
+        False
+
+        >>> ps.Series([True, False, None]).any()
+        True
+
+        >>> ps.Series([]).any()
+        False
+
+        >>> ps.Series([np.nan]).any()
+        False
+
+        >>> df = ps.Series([True, False, None]).rename("a").to_frame()
+        >>> df.set_index("a").index.any()
+        True
+        """
+        axis = validate_axis(axis)
+        if axis != 0:
+            raise NotImplementedError('axis should be either 0 or "index" currently.')
+
+        sdf = self._internal.spark_frame.select(self.spark.column)
+        col = scol_for(sdf, sdf.columns[0])
+
+        # Note that we're ignoring `None`s here for now.
+        # any and every was added as of Spark 3.0
+        # ret = sdf.select(F.expr("any(CAST(`%s` AS BOOLEAN))" % sdf.columns[0])).collect()[0][0]
+        # Here we use max as its alternative:
+        ret = sdf.select(F.max(F.coalesce(col.cast("boolean"), F.lit(False)))).collect()[0][0]
+        if ret is None:
+            return False
+        else:
+            return ret
+
+    # TODO: add frep and axis parameter
+    def shift(self, periods=1, fill_value=None) -> Union["Series", "Index"]:
+        """
+        Shift Series/Index by desired number of periods.
+
+        .. note:: the current implementation of shift uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Parameters
+        ----------
+        periods : int
+            Number of periods to shift. Can be positive or negative.
+        fill_value : object, optional
+            The scalar value to use for newly introduced missing values.
+            The default depends on the dtype of self. For numeric data, np.nan is used.
+
+        Returns
+        -------
+        Copy of input Series/Index, shifted.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'Col1': [10, 20, 15, 30, 45],
+        ...                    'Col2': [13, 23, 18, 33, 48],
+        ...                    'Col3': [17, 27, 22, 37, 52]},
+        ...                   columns=['Col1', 'Col2', 'Col3'])
+
+        >>> df.Col1.shift(periods=3)
+        0     NaN
+        1     NaN
+        2     NaN
+        3    10.0
+        4    20.0
+        Name: Col1, dtype: float64
+
+        >>> df.Col2.shift(periods=3, fill_value=0)
+        0     0
+        1     0
+        2     0
+        3    13
+        4    23
+        Name: Col2, dtype: int64
+
+        >>> df.index.shift(periods=3, fill_value=0)
+        Int64Index([0, 0, 0, 0, 1], dtype='int64')
+        """
+        return self._shift(periods, fill_value).spark.analyzed
+
+    def _shift(self, periods, fill_value, *, part_cols=()):
+        if not isinstance(periods, int):
+            raise TypeError("periods should be an int; however, got [%s]" % type(periods).__name__)
+
+        col = self.spark.column
+        window = (
+            Window.partitionBy(*part_cols)
+            .orderBy(NATURAL_ORDER_COLUMN_NAME)
+            .rowsBetween(-periods, -periods)
+        )
+        lag_col = F.lag(col, periods).over(window)
+        col = F.when(lag_col.isNull() | F.isnan(lag_col), fill_value).otherwise(lag_col)
+        return self._with_new_scol(col, dtype=self.dtype)
+
+    # TODO: Update Documentation for Bins Parameter when its supported
+    def value_counts(
+        self, normalize=False, sort=True, ascending=False, bins=None, dropna=True
+    ) -> "Series":
+        """
+        Return a Series containing counts of unique values.
+        The resulting object will be in descending order so that the
+        first element is the most frequently-occurring element.
+        Excludes NA values by default.
+
+        Parameters
+        ----------
+        normalize : boolean, default False
+            If True then the object returned will contain the relative
+            frequencies of the unique values.
+        sort : boolean, default True
+            Sort by values.
+        ascending : boolean, default False
+            Sort in ascending order.
+        bins : Not Yet Supported
+        dropna : boolean, default True
+            Don't include counts of NaN.
+
+        Returns
+        -------
+        counts : Series
+
+        See Also
+        --------
+        Series.count: Number of non-NA elements in a Series.
+
+        Examples
+        --------
+        For Series
+
+        >>> df = ps.DataFrame({'x':[0, 0, 1, 1, 1, np.nan]})
+        >>> df.x.value_counts()  # doctest: +NORMALIZE_WHITESPACE
+        1.0    3
+        0.0    2
+        Name: x, dtype: int64
+
+        With `normalize` set to `True`, returns the relative frequency by
+        dividing all values by the sum of values.
+
+        >>> df.x.value_counts(normalize=True)  # doctest: +NORMALIZE_WHITESPACE
+        1.0    0.6
+        0.0    0.4
+        Name: x, dtype: float64
+
+        **dropna**
+        With `dropna` set to `False` we can also see NaN index values.
+
+        >>> df.x.value_counts(dropna=False)  # doctest: +NORMALIZE_WHITESPACE
+        1.0    3
+        0.0    2
+        NaN    1
+        Name: x, dtype: int64
+
+        For Index
+
+        >>> idx = ps.Index([3, 1, 2, 3, 4, np.nan])
+        >>> idx
+        Float64Index([3.0, 1.0, 2.0, 3.0, 4.0, nan], dtype='float64')
+
+        >>> idx.value_counts().sort_index()
+        1.0    1
+        2.0    1
+        3.0    2
+        4.0    1
+        dtype: int64
+
+        **sort**
+
+        With `sort` set to `False`, the result wouldn't be sorted by number of count.
+
+        >>> idx.value_counts(sort=True).sort_index()
+        1.0    1
+        2.0    1
+        3.0    2
+        4.0    1
+        dtype: int64
+
+        **normalize**
+
+        With `normalize` set to `True`, returns the relative frequency by
+        dividing all values by the sum of values.
+
+        >>> idx.value_counts(normalize=True).sort_index()
+        1.0    0.2
+        2.0    0.2
+        3.0    0.4
+        4.0    0.2
+        dtype: float64
+
+        **dropna**
+
+        With `dropna` set to `False` we can also see NaN index values.
+
+        >>> idx.value_counts(dropna=False).sort_index()  # doctest: +SKIP
+        1.0    1
+        2.0    1
+        3.0    2
+        4.0    1
+        NaN    1
+        dtype: int64
+
+        For MultiIndex.
+
+        >>> midx = pd.MultiIndex([['lama', 'cow', 'falcon'],
+        ...                       ['speed', 'weight', 'length']],
+        ...                      [[0, 0, 0, 1, 1, 1, 2, 2, 2],
+        ...                       [1, 1, 1, 1, 1, 2, 1, 2, 2]])
+        >>> s = ps.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], index=midx)
+        >>> s.index  # doctest: +SKIP
+        MultiIndex([(  'lama', 'weight'),
+                    (  'lama', 'weight'),
+                    (  'lama', 'weight'),
+                    (   'cow', 'weight'),
+                    (   'cow', 'weight'),
+                    (   'cow', 'length'),
+                    ('falcon', 'weight'),
+                    ('falcon', 'length'),
+                    ('falcon', 'length')],
+                   )
+
+        >>> s.index.value_counts().sort_index()
+        (cow, length)       1
+        (cow, weight)       2
+        (falcon, length)    2
+        (falcon, weight)    1
+        (lama, weight)      3
+        dtype: int64
+
+        >>> s.index.value_counts(normalize=True).sort_index()
+        (cow, length)       0.111111
+        (cow, weight)       0.222222
+        (falcon, length)    0.222222
+        (falcon, weight)    0.111111
+        (lama, weight)      0.333333
+        dtype: float64
+
+        If Index has name, keep the name up.
+
+        >>> idx = ps.Index([0, 0, 0, 1, 1, 2, 3], name='pandas-on-Spark')
+        >>> idx.value_counts().sort_index()
+        0    3
+        1    2
+        2    1
+        3    1
+        Name: pandas-on-Spark, dtype: int64
+        """
+        from pyspark.pandas.series import first_series
+
+        if bins is not None:
+            raise NotImplementedError("value_counts currently does not support bins")
+
+        if dropna:
+            sdf_dropna = self._internal.spark_frame.select(self.spark.column).dropna()
+        else:
+            sdf_dropna = self._internal.spark_frame.select(self.spark.column)
+        index_name = SPARK_DEFAULT_INDEX_NAME
+        column_name = self._internal.data_spark_column_names[0]
+        sdf = sdf_dropna.groupby(scol_for(sdf_dropna, column_name).alias(index_name)).count()
+        if sort:
+            if ascending:
+                sdf = sdf.orderBy(F.col("count"))
+            else:
+                sdf = sdf.orderBy(F.col("count").desc())
+
+        if normalize:
+            sum = sdf_dropna.count()
+            sdf = sdf.withColumn("count", F.col("count") / F.lit(sum))
+
+        internal = InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=[scol_for(sdf, index_name)],
+            column_labels=self._internal.column_labels,
+            data_spark_columns=[scol_for(sdf, "count")],
+            column_label_names=self._internal.column_label_names,
+        )
+
+        return first_series(DataFrame(internal))
+
+    def nunique(self, dropna: bool = True, approx: bool = False, rsd: float = 0.05) -> int:
+        """
+        Return number of unique elements in the object.
+        Excludes NA values by default.
+
+        Parameters
+        ----------
+        dropna : bool, default True
+            Don’t include NaN in the count.
+        approx: bool, default False
+            If False, will use the exact algorithm and return the exact number of unique.
+            If True, it uses the HyperLogLog approximate algorithm, which is significantly faster
+            for large amount of data.
+            Note: This parameter is specific to pandas-on-Spark and is not found in pandas.
+        rsd: float, default 0.05
+            Maximum estimation error allowed in the HyperLogLog algorithm.
+            Note: Just like ``approx`` this parameter is specific to pandas-on-Spark.
+
+        Returns
+        -------
+        int
+
+        See Also
+        --------
+        DataFrame.nunique: Method nunique for DataFrame.
+        Series.count: Count non-NA/null observations in the Series.
+
+        Examples
+        --------
+        >>> ps.Series([1, 2, 3, np.nan]).nunique()
+        3
+
+        >>> ps.Series([1, 2, 3, np.nan]).nunique(dropna=False)
+        4
+
+        On big data, we recommend using the approximate algorithm to speed up this function.
+        The result will be very close to the exact unique count.
+
+        >>> ps.Series([1, 2, 3, np.nan]).nunique(approx=True)
+        3
+
+        >>> idx = ps.Index([1, 1, 2, None])
+        >>> idx
+        Float64Index([1.0, 1.0, 2.0, nan], dtype='float64')
+
+        >>> idx.nunique()
+        2
+
+        >>> idx.nunique(dropna=False)
+        3
+        """
+        res = self._internal.spark_frame.select([self._nunique(dropna, approx, rsd)])
+        return res.collect()[0][0]
+
+    def _nunique(self, dropna=True, approx=False, rsd=0.05):
+        colname = self._internal.data_spark_column_names[0]
+        count_fn = partial(F.approx_count_distinct, rsd=rsd) if approx else F.countDistinct
+        if dropna:
+            return count_fn(self.spark.column).alias(colname)
+        else:
+            return (
+                count_fn(self.spark.column)
+                + F.when(
+                    F.count(F.when(self.spark.column.isNull(), 1).otherwise(None)) >= 1, 1
+                ).otherwise(0)
+            ).alias(colname)
+
+    def take(self, indices) -> Union["Series", "Index"]:
+        """
+        Return the elements in the given *positional* indices along an axis.
+
+        This means that we are not indexing according to actual values in
+        the index attribute of the object. We are indexing according to the
+        actual position of the element in the object.
+
+        Parameters
+        ----------
+        indices : array-like
+            An array of ints indicating which positions to take.
+
+        Returns
+        -------
+        taken : same type as caller
+            An array-like containing the elements taken from the object.
+
+        See Also
+        --------
+        DataFrame.loc : Select a subset of a DataFrame by labels.
+        DataFrame.iloc : Select a subset of a DataFrame by positions.
+        numpy.take : Take elements from an array along an axis.
+
+        Examples
+        --------
+
+        Series
+
+        >>> kser = ps.Series([100, 200, 300, 400, 500])
+        >>> kser
+        0    100
+        1    200
+        2    300
+        3    400
+        4    500
+        dtype: int64
+
+        >>> kser.take([0, 2, 4]).sort_index()
+        0    100
+        2    300
+        4    500
+        dtype: int64
+
+        Index
+
+        >>> kidx = ps.Index([100, 200, 300, 400, 500])
+        >>> kidx
+        Int64Index([100, 200, 300, 400, 500], dtype='int64')
+
+        >>> kidx.take([0, 2, 4]).sort_values()
+        Int64Index([100, 300, 500], dtype='int64')
+
+        MultiIndex
+
+        >>> kmidx = ps.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("x", "c")])
+        >>> kmidx  # doctest: +SKIP
+        MultiIndex([('x', 'a'),
+                    ('x', 'b'),
+                    ('x', 'c')],
+                   )
+
+        >>> kmidx.take([0, 2])  # doctest: +SKIP
+        MultiIndex([('x', 'a'),
+                    ('x', 'c')],
+                   )
+        """
+        if not is_list_like(indices) or isinstance(indices, (dict, set)):
+            raise TypeError("`indices` must be a list-like except dict or set")
+        if isinstance(self, ps.Series):
+            return cast(ps.Series, self.iloc[indices])
+        else:
+            return self._kdf.iloc[indices].index
+
+    def factorize(
+        self, sort: bool = True, na_sentinel: Optional[int] = -1
+    ) -> Tuple[Union["Series", "Index"], pd.Index]:
+        """
+        Encode the object as an enumerated type or categorical variable.
+
+        This method is useful for obtaining a numeric representation of an
+        array when all that matters is identifying distinct values.
+
+        Parameters
+        ----------
+        sort : bool, default True
+        na_sentinel : int or None, default -1
+            Value to mark "not found". If None, will not drop the NaN
+            from the uniques of the values.
+
+        Returns
+        -------
+        codes : Series or Index
+            A Series or Index that's an indexer into `uniques`.
+            ``uniques.take(codes)`` will have the same values as `values`.
+        uniques : pd.Index
+            The unique valid values.
+
+            .. note ::
+
+               Even if there's a missing value in `values`, `uniques` will
+               *not* contain an entry for it.
+
+        Examples
+        --------
+        >>> kser = ps.Series(['b', None, 'a', 'c', 'b'])
+        >>> codes, uniques = kser.factorize()
+        >>> codes
+        0    1
+        1   -1
+        2    0
+        3    2
+        4    1
+        dtype: int32
+        >>> uniques
+        Index(['a', 'b', 'c'], dtype='object')
+
+        >>> codes, uniques = kser.factorize(na_sentinel=None)
+        >>> codes
+        0    1
+        1    3
+        2    0
+        3    2
+        4    1
+        dtype: int32
+        >>> uniques
+        Index(['a', 'b', 'c', None], dtype='object')
+
+        >>> codes, uniques = kser.factorize(na_sentinel=-2)
+        >>> codes
+        0    1
+        1   -2
+        2    0
+        3    2
+        4    1
+        dtype: int32
+        >>> uniques
+        Index(['a', 'b', 'c'], dtype='object')
+
+        For Index:
+
+        >>> kidx = ps.Index(['b', None, 'a', 'c', 'b'])
+        >>> codes, uniques = kidx.factorize()
+        >>> codes
+        Int64Index([1, -1, 0, 2, 1], dtype='int64')
+        >>> uniques
+        Index(['a', 'b', 'c'], dtype='object')
+        """
+        from pyspark.pandas.series import first_series
+
+        assert (na_sentinel is None) or isinstance(na_sentinel, int)
+        assert sort is True
+
+        if isinstance(self.dtype, CategoricalDtype):
+            categories = self.dtype.categories
+            if len(categories) == 0:
+                scol = F.lit(None)
+            else:
+                kvs = list(
+                    chain(
+                        *[
+                            (F.lit(code), F.lit(category))
+                            for code, category in enumerate(categories)
+                        ]
+                    )
+                )
+                map_scol = F.create_map(kvs)
+                scol = map_scol.getItem(self.spark.column)
+            codes, uniques = self._with_new_scol(
+                scol.alias(self._internal.data_spark_column_names[0])
+            ).factorize(na_sentinel=na_sentinel)
+            return codes, uniques.astype(self.dtype)
+
+        uniq_sdf = self._internal.spark_frame.select(self.spark.column).distinct()
+
+        # Check number of uniques and constructs sorted `uniques_list`
+        max_compute_count = get_option("compute.max_rows")
+        if max_compute_count is not None:
+            uniq_pdf = uniq_sdf.limit(max_compute_count + 1).toPandas()
+            if len(uniq_pdf) > max_compute_count:
+                raise ValueError(
+                    "Current Series has more then {0} unique values. "
+                    "Please set 'compute.max_rows' by using 'pyspark.pandas.config.set_option' "
+                    "to more than {0} rows. Note that, before changing the "
+                    "'compute.max_rows', this operation is considerably expensive.".format(
+                        max_compute_count
+                    )
+                )
+        else:
+            uniq_pdf = uniq_sdf.toPandas()
+        # pandas takes both NaN and null in Spark to np.nan, so de-duplication is required
+        uniq_series = first_series(uniq_pdf).drop_duplicates()
+        uniques_list = uniq_series.tolist()
+        uniques_list = sorted(uniques_list, key=lambda x: (pd.isna(x), x))
+
+        # Constructs `unique_to_code` mapping non-na unique to code
+        unique_to_code = {}
+        if na_sentinel is not None:
+            na_sentinel_code = na_sentinel
+        code = 0
+        for unique in uniques_list:
+            if pd.isna(unique):
+                if na_sentinel is None:
+                    na_sentinel_code = code
+            else:
+                unique_to_code[unique] = code
+            code += 1
+
+        kvs = list(
+            chain(*([(F.lit(unique), F.lit(code)) for unique, code in unique_to_code.items()]))
+        )
+
+        if len(kvs) == 0:  # uniques are all missing values
+            new_scol = F.lit(na_sentinel_code)
+        else:
+            scol = self.spark.column
+            if isinstance(self.spark.data_type, (FloatType, DoubleType)):
+                cond = scol.isNull() | F.isnan(scol)
+            else:
+                cond = scol.isNull()
+            map_scol = F.create_map(kvs)
+
+            null_scol = F.when(cond, F.lit(na_sentinel_code))
+            new_scol = null_scol.otherwise(map_scol.getItem(scol))
+
+        codes = self._with_new_scol(new_scol.alias(self._internal.data_spark_column_names[0]))
+
+        if na_sentinel is not None:
+            # Drops the NaN from the uniques of the values
+            uniques_list = [x for x in uniques_list if not pd.isna(x)]
+
+        uniques = pd.Index(uniques_list)
+
+        return codes, uniques
+
+
+def _test():
+    import os
+    import doctest
+    import sys
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.base
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.base.__dict__.copy()
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]").appName("pyspark.pandas.base tests").getOrCreate()
+    )
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.base,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/categorical.py b/python/pyspark/pandas/categorical.py
new file mode 100644
index 0000000000000..a9321d9540bad
--- /dev/null
+++ b/python/pyspark/pandas/categorical.py
@@ -0,0 +1,194 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from typing import TYPE_CHECKING
+
+import pandas as pd
+from pandas.api.types import CategoricalDtype
+
+if TYPE_CHECKING:
+    import pyspark.pandas as ps  # noqa: F401 (SPARK-34943)
+
+
+class CategoricalAccessor(object):
+    """
+    Accessor object for categorical properties of the Series values.
+
+    Examples
+    --------
+    >>> s = ps.Series(list("abbccc"), dtype="category")
+    >>> s  # doctest: +SKIP
+    0    a
+    1    b
+    2    b
+    3    c
+    4    c
+    5    c
+    dtype: category
+    Categories (3, object): ['a', 'b', 'c']
+
+    >>> s.cat.categories
+    Index(['a', 'b', 'c'], dtype='object')
+
+    >>> s.cat.codes
+    0    0
+    1    1
+    2    1
+    3    2
+    4    2
+    5    2
+    dtype: int8
+    """
+
+    def __init__(self, series: "ps.Series"):
+        if not isinstance(series.dtype, CategoricalDtype):
+            raise ValueError("Cannot call CategoricalAccessor on type {}".format(series.dtype))
+        self._data = series
+
+    @property
+    def categories(self) -> pd.Index:
+        """
+        The categories of this categorical.
+
+        Examples
+        --------
+        >>> s = ps.Series(list("abbccc"), dtype="category")
+        >>> s  # doctest: +SKIP
+        0    a
+        1    b
+        2    b
+        3    c
+        4    c
+        5    c
+        dtype: category
+        Categories (3, object): ['a', 'b', 'c']
+
+        >>> s.cat.categories
+        Index(['a', 'b', 'c'], dtype='object')
+        """
+        return self._data.dtype.categories
+
+    @categories.setter
+    def categories(self, categories) -> None:
+        raise NotImplementedError()
+
+    @property
+    def ordered(self) -> bool:
+        """
+        Whether the categories have an ordered relationship.
+
+        Examples
+        --------
+        >>> s = ps.Series(list("abbccc"), dtype="category")
+        >>> s  # doctest: +SKIP
+        0    a
+        1    b
+        2    b
+        3    c
+        4    c
+        5    c
+        dtype: category
+        Categories (3, object): ['a', 'b', 'c']
+
+        >>> s.cat.ordered
+        False
+        """
+        return self._data.dtype.ordered
+
+    @property
+    def codes(self) -> "ps.Series":
+        """
+        Return Series of codes as well as the index.
+
+        Examples
+        --------
+        >>> s = ps.Series(list("abbccc"), dtype="category")
+        >>> s  # doctest: +SKIP
+        0    a
+        1    b
+        2    b
+        3    c
+        4    c
+        5    c
+        dtype: category
+        Categories (3, object): ['a', 'b', 'c']
+
+        >>> s.cat.codes
+        0    0
+        1    1
+        2    1
+        3    2
+        4    2
+        5    2
+        dtype: int8
+        """
+        return self._data._with_new_scol(self._data.spark.column).rename()
+
+    def add_categories(self, new_categories, inplace: bool = False):
+        raise NotImplementedError()
+
+    def as_ordered(self, inplace: bool = False):
+        raise NotImplementedError()
+
+    def as_unordered(self, inplace: bool = False):
+        raise NotImplementedError()
+
+    def remove_categories(self, removals, inplace: bool = False):
+        raise NotImplementedError()
+
+    def remove_unused_categories(self):
+        raise NotImplementedError()
+
+    def rename_categories(self, new_categories, inplace: bool = False):
+        raise NotImplementedError()
+
+    def reorder_categories(self, new_categories, ordered: bool = None, inplace: bool = False):
+        raise NotImplementedError()
+
+    def set_categories(
+        self, new_categories, ordered: bool = None, rename: bool = False, inplace: bool = False
+    ):
+        raise NotImplementedError()
+
+
+def _test():
+    import os
+    import doctest
+    import sys
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.categorical
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.categorical.__dict__.copy()
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]")
+        .appName("pyspark.pandas.categorical tests")
+        .getOrCreate()
+    )
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.categorical,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/config.py b/python/pyspark/pandas/config.py
new file mode 100644
index 0000000000000..93c6c8419d019
--- /dev/null
+++ b/python/pyspark/pandas/config.py
@@ -0,0 +1,471 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Infrastructure of options for pandas-on-Spark.
+"""
+from contextlib import contextmanager
+import json
+from typing import Union, Any, Tuple, Callable, List, Dict  # noqa: F401 (SPARK-34943)
+
+from pyspark._globals import _NoValue, _NoValueType
+
+from pyspark.pandas.utils import default_session
+
+
+__all__ = ["get_option", "set_option", "reset_option", "options", "option_context"]
+
+
+class Option:
+    """
+    Option class that defines an option with related properties.
+
+    This class holds all information relevant to the one option. Also,
+    Its instance can validate if the given value is acceptable or not.
+
+    It is currently for internal usage only.
+
+    Parameters
+    ----------
+    key: str, keyword-only argument
+        the option name to use.
+    doc: str, keyword-only argument
+        the documentation for the current option.
+    default: Any, keyword-only argument
+        default value for this option.
+    types: Union[Tuple[type, ...], type], keyword-only argument
+        default is str. It defines the expected types for this option. It is
+        used with `isinstance` to validate the given value to this option.
+    check_func: Tuple[Callable[[Any], bool], str], keyword-only argument
+        default is a function that always returns `True` with a empty string.
+        It defines:
+          - a function to check the given value to this option
+          - the error message to show when this check is failed
+        When new value is set to this option, this function is called to check
+        if the given value is valid.
+
+    Examples
+    --------
+    >>> option = Option(
+    ...     key='option.name',
+    ...     doc="this is a test option",
+    ...     default="default",
+    ...     types=(float, int),
+    ...     check_func=(lambda v: v > 0, "should be a positive float"))
+
+    >>> option.validate('abc')  # doctest: +NORMALIZE_WHITESPACE
+    Traceback (most recent call last):
+      ...
+    TypeError: The value for option 'option.name' was <class 'str'>;
+    however, expected types are [(<class 'float'>, <class 'int'>)].
+
+    >>> option.validate(-1.1)
+    Traceback (most recent call last):
+      ...
+    ValueError: should be a positive float
+
+    >>> option.validate(1.1)
+    """
+
+    def __init__(
+        self,
+        *,
+        key: str,
+        doc: str,
+        default: Any,
+        types: Union[Tuple[type, ...], type] = str,
+        check_func: Tuple[Callable[[Any], bool], str] = (lambda v: True, "")
+    ):
+        self.key = key
+        self.doc = doc
+        self.default = default
+        self.types = types
+        self.check_func = check_func
+
+    def validate(self, v: Any) -> None:
+        """
+        Validate the given value and throw an exception with related information such as key.
+        """
+        if not isinstance(v, self.types):
+            raise TypeError(
+                "The value for option '%s' was %s; however, expected types are "
+                "[%s]." % (self.key, type(v), str(self.types))
+            )
+        if not self.check_func[0](v):
+            raise ValueError(self.check_func[1])
+
+
+# Available options.
+#
+# NOTE: if you are fixing or adding an option here, make sure you execute `show_options()` and
+#     copy & paste the results into show_options 'docs/source/user_guide/options.rst' as well.
+#     See the examples below:
+#     >>> from pyspark.pandas.config import show_options
+#     >>> show_options()
+_options = [
+    Option(
+        key="display.max_rows",
+        doc=(
+            "This sets the maximum number of rows pandas-on-Spark should output when printing out "
+            "various output. For example, this value determines the number of rows to be "
+            "shown at the repr() in a dataframe. Set `None` to unlimit the input length. "
+            "Default is 1000."
+        ),
+        default=1000,
+        types=(int, type(None)),
+        check_func=(
+            lambda v: v is None or v >= 0,
+            "'display.max_rows' should be greater than or equal to 0.",
+        ),
+    ),
+    Option(
+        key="compute.max_rows",
+        doc=(
+            "'compute.max_rows' sets the limit of the current pandas-on-Spark DataFrame. "
+            "Set `None` to unlimit the input length. When the limit is set, it is executed "
+            "by the shortcut by collecting the data into the driver, and then using the pandas "
+            "API. If the limit is unset, the operation is executed by PySpark. Default is 1000."
+        ),
+        default=1000,
+        types=(int, type(None)),
+        check_func=(
+            lambda v: v is None or v >= 0,
+            "'compute.max_rows' should be greater than or equal to 0.",
+        ),
+    ),
+    Option(
+        key="compute.shortcut_limit",
+        doc=(
+            "'compute.shortcut_limit' sets the limit for a shortcut. "
+            "It computes specified number of rows and use its schema. When the dataframe "
+            "length is larger than this limit, pandas-on-Spark uses PySpark to compute."
+        ),
+        default=1000,
+        types=int,
+        check_func=(
+            lambda v: v >= 0,
+            "'compute.shortcut_limit' should be greater than or equal to 0.",
+        ),
+    ),
+    Option(
+        key="compute.ops_on_diff_frames",
+        doc=(
+            "This determines whether or not to operate between two different dataframes. "
+            "For example, 'combine_frames' function internally performs a join operation which "
+            "can be expensive in general. So, if `compute.ops_on_diff_frames` variable is not "
+            "True, that method throws an exception."
+        ),
+        default=False,
+        types=bool,
+    ),
+    Option(
+        key="compute.default_index_type",
+        doc=("This sets the default index type: sequence, distributed and distributed-sequence."),
+        default="sequence",
+        types=str,
+        check_func=(
+            lambda v: v in ("sequence", "distributed", "distributed-sequence"),
+            "Index type should be one of 'sequence', 'distributed', 'distributed-sequence'.",
+        ),
+    ),
+    Option(
+        key="compute.ordered_head",
+        doc=(
+            "'compute.ordered_head' sets whether or not to operate head with natural ordering. "
+            "pandas-on-Spark does not guarantee the row ordering so `head` could return some "
+            "rows from distributed partitions. If 'compute.ordered_head' is set to True, "
+            "pandas-on-Spark performs natural ordering beforehand, but it will cause a "
+            "performance overhead."
+        ),
+        default=False,
+        types=bool,
+    ),
+    Option(
+        key="plotting.max_rows",
+        doc=(
+            "'plotting.max_rows' sets the visual limit on top-n-based plots such as `plot.bar` "
+            "and `plot.pie`. If it is set to 1000, the first 1000 data points will be used "
+            "for plotting. Default is 1000."
+        ),
+        default=1000,
+        types=int,
+        check_func=(
+            lambda v: v is v >= 0,
+            "'plotting.max_rows' should be greater than or equal to 0.",
+        ),
+    ),
+    Option(
+        key="plotting.sample_ratio",
+        doc=(
+            "'plotting.sample_ratio' sets the proportion of data that will be plotted for sample-"
+            "based plots such as `plot.line` and `plot.area`. "
+            "This option defaults to 'plotting.max_rows' option."
+        ),
+        default=None,
+        types=(float, type(None)),
+        check_func=(
+            lambda v: v is None or 1 >= v >= 0,
+            "'plotting.sample_ratio' should be 1.0 >= value >= 0.0.",
+        ),
+    ),
+    Option(
+        key="plotting.backend",
+        doc=(
+            "Backend to use for plotting. Default is plotly. "
+            "Supports any package that has a top-level `.plot` method. "
+            "Known options are: [matplotlib, plotly]."
+        ),
+        default="plotly",
+        types=str,
+    ),
+]  # type: List[Option]
+
+_options_dict = dict(zip((option.key for option in _options), _options))  # type: Dict[str, Option]
+
+_key_format = "pandas_on_Spark.{}".format
+
+
+class OptionError(AttributeError, KeyError):
+    pass
+
+
+def show_options():
+    """
+    Make a pretty table that can be copied and pasted into public documentation.
+    This is currently for an internal purpose.
+
+    Examples
+    --------
+    >>> show_options()  # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
+    ================... =======... =====================...
+    Option              Default    Description
+    ================... =======... =====================...
+    display.max_rows    1000       This sets the maximum...
+    ...
+    ================... =======... =====================...
+    """
+
+    import textwrap
+
+    header = ["Option", "Default", "Description"]
+    row_format = "{:<31} {:<14} {:<53}"
+
+    print(row_format.format("=" * 31, "=" * 14, "=" * 53))
+    print(row_format.format(*header))
+    print(row_format.format("=" * 31, "=" * 14, "=" * 53))
+
+    for option in _options:
+        doc = textwrap.fill(option.doc, 53)
+        formatted = "".join([line + "\n" + (" " * 47) for line in doc.split("\n")]).rstrip()
+        print(row_format.format(option.key, repr(option.default), formatted))
+
+    print(row_format.format("=" * 31, "=" * 14, "=" * 53))
+
+
+def get_option(key: str, default: Union[Any, _NoValueType] = _NoValue) -> Any:
+    """
+    Retrieves the value of the specified option.
+
+    Parameters
+    ----------
+    key : str
+        The key which should match a single option.
+    default : object
+        The default value if the option is not set yet. The value should be JSON serializable.
+
+    Returns
+    -------
+    result : the value of the option
+
+    Raises
+    ------
+    OptionError : if no such option exists and the default is not provided
+    """
+    _check_option(key)
+    if default is _NoValue:
+        default = _options_dict[key].default
+    _options_dict[key].validate(default)
+
+    return json.loads(default_session().conf.get(_key_format(key), default=json.dumps(default)))
+
+
+def set_option(key: str, value: Any) -> None:
+    """
+    Sets the value of the specified option.
+
+    Parameters
+    ----------
+    key : str
+        The key which should match a single option.
+    value : object
+        New value of option. The value should be JSON serializable.
+
+    Returns
+    -------
+    None
+    """
+    _check_option(key)
+    _options_dict[key].validate(value)
+
+    default_session().conf.set(_key_format(key), json.dumps(value))
+
+
+def reset_option(key: str) -> None:
+    """
+    Reset one option to their default value.
+
+    Pass "all" as argument to reset all options.
+
+    Parameters
+    ----------
+    key : str
+        If specified only option will be reset.
+
+    Returns
+    -------
+    None
+    """
+    _check_option(key)
+    default_session().conf.unset(_key_format(key))
+
+
+@contextmanager
+def option_context(*args):
+    """
+    Context manager to temporarily set options in the `with` statement context.
+
+    You need to invoke as ``option_context(pat, val, [(pat, val), ...])``.
+
+    Examples
+    --------
+    >>> with option_context('display.max_rows', 10, 'compute.max_rows', 5):
+    ...     print(get_option('display.max_rows'), get_option('compute.max_rows'))
+    10 5
+    >>> print(get_option('display.max_rows'), get_option('compute.max_rows'))
+    1000 1000
+    """
+    if len(args) == 0 or len(args) % 2 != 0:
+        raise ValueError("Need to invoke as option_context(pat, val, [(pat, val), ...]).")
+    opts = dict(zip(args[::2], args[1::2]))
+    orig_opts = {key: get_option(key) for key in opts}
+    try:
+        for key, value in opts.items():
+            set_option(key, value)
+        yield
+    finally:
+        for key, value in orig_opts.items():
+            set_option(key, value)
+
+
+def _check_option(key: str) -> None:
+    if key not in _options_dict:
+        raise OptionError(
+            "No such option: '{}'. Available options are [{}]".format(
+                key, ", ".join(list(_options_dict.keys()))
+            )
+        )
+
+
+class DictWrapper:
+    """ provide attribute-style access to a nested dict"""
+
+    def __init__(self, d, prefix=""):
+        object.__setattr__(self, "d", d)
+        object.__setattr__(self, "prefix", prefix)
+
+    def __setattr__(self, key, val):
+        prefix = object.__getattribute__(self, "prefix")
+        d = object.__getattribute__(self, "d")
+        if prefix:
+            prefix += "."
+        canonical_key = prefix + key
+
+        candidates = [
+            k for k in d.keys() if all(x in k.split(".") for x in canonical_key.split("."))
+        ]
+        if len(candidates) == 1 and candidates[0] == canonical_key:
+            return set_option(canonical_key, val)
+        else:
+            raise OptionError(
+                "No such option: '{}'. Available options are [{}]".format(
+                    key, ", ".join(list(_options_dict.keys()))
+                )
+            )
+
+    def __getattr__(self, key):
+        prefix = object.__getattribute__(self, "prefix")
+        d = object.__getattribute__(self, "d")
+        if prefix:
+            prefix += "."
+        canonical_key = prefix + key
+
+        candidates = [
+            k for k in d.keys() if all(x in k.split(".") for x in canonical_key.split("."))
+        ]
+        if len(candidates) == 1 and candidates[0] == canonical_key:
+            return get_option(canonical_key)
+        elif len(candidates) == 0:
+            raise OptionError(
+                "No such option: '{}'. Available options are [{}]".format(
+                    key, ", ".join(list(_options_dict.keys()))
+                )
+            )
+        else:
+            return DictWrapper(d, canonical_key)
+
+    def __dir__(self):
+        prefix = object.__getattribute__(self, "prefix")
+        d = object.__getattribute__(self, "d")
+
+        if prefix == "":
+            candidates = d.keys()
+            offset = 0
+        else:
+            candidates = [k for k in d.keys() if all(x in k.split(".") for x in prefix.split("."))]
+            offset = len(prefix) + 1  # prefix (e.g. "compute.") to trim.
+        return [c[offset:] for c in candidates]
+
+
+options = DictWrapper(_options_dict)
+
+
+def _test():
+    import os
+    import doctest
+    import sys
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.config
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.config.__dict__.copy()
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]").appName("pyspark.pandas.config tests").getOrCreate()
+    )
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.config,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/datetimes.py b/python/pyspark/pandas/datetimes.py
new file mode 100644
index 0000000000000..b18302a46cb93
--- /dev/null
+++ b/python/pyspark/pandas/datetimes.py
@@ -0,0 +1,880 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Date/Time related functions on pandas-on-Spark Series
+"""
+from typing import TYPE_CHECKING
+
+import numpy as np  # noqa: F401 (SPARK-34943)
+import pandas as pd  # noqa: F401
+import pyspark.sql.functions as F
+from pyspark.sql.types import DateType, TimestampType, LongType
+
+if TYPE_CHECKING:
+    import pyspark.pandas as ps  # noqa: F401 (SPARK-34943)
+
+
+class DatetimeMethods(object):
+    """Date/Time methods for pandas-on-Spark Series"""
+
+    def __init__(self, series: "ps.Series"):
+        if not isinstance(series.spark.data_type, (DateType, TimestampType)):
+            raise ValueError(
+                "Cannot call DatetimeMethods on type {}".format(series.spark.data_type)
+            )
+        self._data = series
+
+    # Properties
+    @property
+    def date(self) -> "ps.Series":
+        """
+        Returns a Series of python datetime.date objects (namely, the date
+        part of Timestamps without timezone information).
+        """
+        # TODO: Hit a weird exception
+        # syntax error in attribute name: `to_date(`start_date`)` with alias
+        return self._data.spark.transform(F.to_date)
+
+    @property
+    def time(self) -> "ps.Series":
+        raise NotImplementedError()
+
+    @property
+    def timetz(self) -> "ps.Series":
+        raise NotImplementedError()
+
+    @property
+    def year(self) -> "ps.Series":
+        """
+        The year of the datetime.
+        """
+        return self._data.spark.transform(lambda c: F.year(c).cast(LongType()))
+
+    @property
+    def month(self) -> "ps.Series":
+        """
+        The month of the timestamp as January = 1 December = 12.
+        """
+        return self._data.spark.transform(lambda c: F.month(c).cast(LongType()))
+
+    @property
+    def day(self) -> "ps.Series":
+        """
+        The days of the datetime.
+        """
+        return self._data.spark.transform(lambda c: F.dayofmonth(c).cast(LongType()))
+
+    @property
+    def hour(self) -> "ps.Series":
+        """
+        The hours of the datetime.
+        """
+        return self._data.spark.transform(lambda c: F.hour(c).cast(LongType()))
+
+    @property
+    def minute(self) -> "ps.Series":
+        """
+        The minutes of the datetime.
+        """
+        return self._data.spark.transform(lambda c: F.minute(c).cast(LongType()))
+
+    @property
+    def second(self) -> "ps.Series":
+        """
+        The seconds of the datetime.
+        """
+        return self._data.spark.transform(lambda c: F.second(c).cast(LongType()))
+
+    @property
+    def microsecond(self) -> "ps.Series":
+        """
+        The microseconds of the datetime.
+        """
+
+        def pandas_microsecond(s) -> "ps.Series[np.int64]":
+            return s.dt.microsecond
+
+        return self._data.koalas.transform_batch(pandas_microsecond)
+
+    @property
+    def nanosecond(self) -> "ps.Series":
+        raise NotImplementedError()
+
+    @property
+    def week(self) -> "ps.Series":
+        """
+        The week ordinal of the year.
+        """
+        return self._data.spark.transform(lambda c: F.weekofyear(c).cast(LongType()))
+
+    @property
+    def weekofyear(self) -> "ps.Series":
+        return self.week
+
+    weekofyear.__doc__ = week.__doc__
+
+    @property
+    def dayofweek(self) -> "ps.Series":
+        """
+        The day of the week with Monday=0, Sunday=6.
+
+        Return the day of the week. It is assumed the week starts on
+        Monday, which is denoted by 0 and ends on Sunday which is denoted
+        by 6. This method is available on both Series with datetime
+        values (using the `dt` accessor).
+
+        Returns
+        -------
+        Series
+            Containing integers indicating the day number.
+
+        See Also
+        --------
+        Series.dt.dayofweek : Alias.
+        Series.dt.weekday : Alias.
+        Series.dt.day_name : Returns the name of the day of the week.
+
+        Examples
+        --------
+        >>> s = ps.from_pandas(pd.date_range('2016-12-31', '2017-01-08', freq='D').to_series())
+        >>> s.dt.dayofweek
+        2016-12-31    5
+        2017-01-01    6
+        2017-01-02    0
+        2017-01-03    1
+        2017-01-04    2
+        2017-01-05    3
+        2017-01-06    4
+        2017-01-07    5
+        2017-01-08    6
+        dtype: int64
+        """
+
+        def pandas_dayofweek(s) -> "ps.Series[np.int64]":
+            return s.dt.dayofweek
+
+        return self._data.koalas.transform_batch(pandas_dayofweek)
+
+    @property
+    def weekday(self) -> "ps.Series":
+        return self.dayofweek
+
+    weekday.__doc__ = dayofweek.__doc__
+
+    @property
+    def dayofyear(self) -> "ps.Series":
+        """
+        The ordinal day of the year.
+        """
+
+        def pandas_dayofyear(s) -> "ps.Series[np.int64]":
+            return s.dt.dayofyear
+
+        return self._data.koalas.transform_batch(pandas_dayofyear)
+
+    @property
+    def quarter(self) -> "ps.Series":
+        """
+        The quarter of the date.
+        """
+
+        def pandas_quarter(s) -> "ps.Series[np.int64]":
+            return s.dt.quarter
+
+        return self._data.koalas.transform_batch(pandas_quarter)
+
+    @property
+    def is_month_start(self) -> "ps.Series":
+        """
+        Indicates whether the date is the first day of the month.
+
+        Returns
+        -------
+        Series
+            For Series, returns a Series with boolean values.
+
+        See Also
+        --------
+        is_month_end : Return a boolean indicating whether the date
+            is the last day of the month.
+
+        Examples
+        --------
+        This method is available on Series with datetime values under
+        the ``.dt`` accessor.
+
+        >>> s = ps.Series(pd.date_range("2018-02-27", periods=3))
+        >>> s
+        0   2018-02-27
+        1   2018-02-28
+        2   2018-03-01
+        dtype: datetime64[ns]
+
+        >>> s.dt.is_month_start
+        0    False
+        1    False
+        2     True
+        dtype: bool
+        """
+
+        def pandas_is_month_start(s) -> "ps.Series[bool]":
+            return s.dt.is_month_start
+
+        return self._data.koalas.transform_batch(pandas_is_month_start)
+
+    @property
+    def is_month_end(self) -> "ps.Series":
+        """
+        Indicates whether the date is the last day of the month.
+
+        Returns
+        -------
+        Series
+            For Series, returns a Series with boolean values.
+
+        See Also
+        --------
+        is_month_start : Return a boolean indicating whether the date
+            is the first day of the month.
+
+        Examples
+        --------
+        This method is available on Series with datetime values under
+        the ``.dt`` accessor.
+
+        >>> s = ps.Series(pd.date_range("2018-02-27", periods=3))
+        >>> s
+        0   2018-02-27
+        1   2018-02-28
+        2   2018-03-01
+        dtype: datetime64[ns]
+
+        >>> s.dt.is_month_end
+        0    False
+        1     True
+        2    False
+        dtype: bool
+        """
+
+        def pandas_is_month_end(s) -> "ps.Series[bool]":
+            return s.dt.is_month_end
+
+        return self._data.koalas.transform_batch(pandas_is_month_end)
+
+    @property
+    def is_quarter_start(self) -> "ps.Series":
+        """
+        Indicator for whether the date is the first day of a quarter.
+
+        Returns
+        -------
+        is_quarter_start : Series
+            The same type as the original data with boolean values. Series will
+            have the same name and index.
+
+        See Also
+        --------
+        quarter : Return the quarter of the date.
+        is_quarter_end : Similar property for indicating the quarter start.
+
+        Examples
+        --------
+        This method is available on Series with datetime values under
+        the ``.dt`` accessor.
+
+        >>> df = ps.DataFrame({'dates': pd.date_range("2017-03-30",
+        ...                   periods=4)})
+        >>> df
+               dates
+        0 2017-03-30
+        1 2017-03-31
+        2 2017-04-01
+        3 2017-04-02
+
+        >>> df.dates.dt.quarter
+        0    1
+        1    1
+        2    2
+        3    2
+        Name: dates, dtype: int64
+
+        >>> df.dates.dt.is_quarter_start
+        0    False
+        1    False
+        2     True
+        3    False
+        Name: dates, dtype: bool
+        """
+
+        def pandas_is_quarter_start(s) -> "ps.Series[bool]":
+            return s.dt.is_quarter_start
+
+        return self._data.koalas.transform_batch(pandas_is_quarter_start)
+
+    @property
+    def is_quarter_end(self) -> "ps.Series":
+        """
+        Indicator for whether the date is the last day of a quarter.
+
+        Returns
+        -------
+        is_quarter_end : Series
+            The same type as the original data with boolean values. Series will
+            have the same name and index.
+
+        See Also
+        --------
+        quarter : Return the quarter of the date.
+        is_quarter_start : Similar property indicating the quarter start.
+
+        Examples
+        --------
+        This method is available on Series with datetime values under
+        the ``.dt`` accessor.
+
+        >>> df = ps.DataFrame({'dates': pd.date_range("2017-03-30",
+        ...                   periods=4)})
+        >>> df
+               dates
+        0 2017-03-30
+        1 2017-03-31
+        2 2017-04-01
+        3 2017-04-02
+
+        >>> df.dates.dt.quarter
+        0    1
+        1    1
+        2    2
+        3    2
+        Name: dates, dtype: int64
+
+        >>> df.dates.dt.is_quarter_start
+        0    False
+        1    False
+        2     True
+        3    False
+        Name: dates, dtype: bool
+        """
+
+        def pandas_is_quarter_end(s) -> "ps.Series[bool]":
+            return s.dt.is_quarter_end
+
+        return self._data.koalas.transform_batch(pandas_is_quarter_end)
+
+    @property
+    def is_year_start(self) -> "ps.Series":
+        """
+        Indicate whether the date is the first day of a year.
+
+        Returns
+        -------
+        Series
+            The same type as the original data with boolean values. Series will
+            have the same name and index.
+
+        See Also
+        --------
+        is_year_end : Similar property indicating the last day of the year.
+
+        Examples
+        --------
+        This method is available on Series with datetime values under
+        the ``.dt`` accessor.
+
+        >>> dates = ps.Series(pd.date_range("2017-12-30", periods=3))
+        >>> dates
+        0   2017-12-30
+        1   2017-12-31
+        2   2018-01-01
+        dtype: datetime64[ns]
+
+        >>> dates.dt.is_year_start
+        0    False
+        1    False
+        2     True
+        dtype: bool
+        """
+
+        def pandas_is_year_start(s) -> "ps.Series[bool]":
+            return s.dt.is_year_start
+
+        return self._data.koalas.transform_batch(pandas_is_year_start)
+
+    @property
+    def is_year_end(self) -> "ps.Series":
+        """
+        Indicate whether the date is the last day of the year.
+
+        Returns
+        -------
+        Series
+            The same type as the original data with boolean values. Series will
+            have the same name and index.
+
+        See Also
+        --------
+        is_year_start : Similar property indicating the start of the year.
+
+        Examples
+        --------
+        This method is available on Series with datetime values under
+        the ``.dt`` accessor.
+
+        >>> dates = ps.Series(pd.date_range("2017-12-30", periods=3))
+        >>> dates
+        0   2017-12-30
+        1   2017-12-31
+        2   2018-01-01
+        dtype: datetime64[ns]
+
+        >>> dates.dt.is_year_end
+        0    False
+        1     True
+        2    False
+        dtype: bool
+        """
+
+        def pandas_is_year_end(s) -> "ps.Series[bool]":
+            return s.dt.is_year_end
+
+        return self._data.koalas.transform_batch(pandas_is_year_end)
+
+    @property
+    def is_leap_year(self) -> "ps.Series":
+        """
+        Boolean indicator if the date belongs to a leap year.
+
+        A leap year is a year, which has 366 days (instead of 365) including
+        29th of February as an intercalary day.
+        Leap years are years which are multiples of four with the exception
+        of years divisible by 100 but not by 400.
+
+        Returns
+        -------
+        Series
+             Booleans indicating if dates belong to a leap year.
+
+        Examples
+        --------
+        This method is available on Series with datetime values under
+        the ``.dt`` accessor.
+
+        >>> dates_series = ps.Series(pd.date_range("2012-01-01", "2015-01-01", freq="Y"))
+        >>> dates_series
+        0   2012-12-31
+        1   2013-12-31
+        2   2014-12-31
+        dtype: datetime64[ns]
+
+        >>> dates_series.dt.is_leap_year
+        0     True
+        1    False
+        2    False
+        dtype: bool
+        """
+
+        def pandas_is_leap_year(s) -> "ps.Series[bool]":
+            return s.dt.is_leap_year
+
+        return self._data.koalas.transform_batch(pandas_is_leap_year)
+
+    @property
+    def daysinmonth(self) -> "ps.Series":
+        """
+        The number of days in the month.
+        """
+
+        def pandas_daysinmonth(s) -> "ps.Series[np.int64]":
+            return s.dt.daysinmonth
+
+        return self._data.koalas.transform_batch(pandas_daysinmonth)
+
+    @property
+    def days_in_month(self) -> "ps.Series":
+        return self.daysinmonth
+
+    days_in_month.__doc__ = daysinmonth.__doc__
+
+    # Methods
+
+    def tz_localize(self, tz) -> "ps.Series":
+        """
+        Localize tz-naive Datetime column to tz-aware Datetime column.
+        """
+        # Neither tz-naive or tz-aware datetime exists in Spark
+        raise NotImplementedError()
+
+    def tz_convert(self, tz) -> "ps.Series":
+        """
+        Convert tz-aware Datetime column from one time zone to another.
+        """
+        # tz-aware datetime doesn't exist in Spark
+        raise NotImplementedError()
+
+    def normalize(self) -> "ps.Series":
+        """
+        Convert times to midnight.
+
+        The time component of the date-time is converted to midnight i.e.
+        00:00:00. This is useful in cases, when the time does not matter.
+        Length is unaltered. The timezones are unaffected.
+
+        This method is available on Series with datetime values under
+        the ``.dt`` accessor, and directly on Datetime Array.
+
+        Returns
+        -------
+        Series
+            The same type as the original data. Series will have the same
+            name and index.
+
+        See Also
+        --------
+        floor : Floor the series to the specified freq.
+        ceil : Ceil the series to the specified freq.
+        round : Round the series to the specified freq.
+
+        Examples
+        --------
+        >>> series = ps.Series(pd.Series(pd.date_range('2012-1-1 12:45:31', periods=3, freq='M')))
+        >>> series.dt.normalize()
+        0   2012-01-31
+        1   2012-02-29
+        2   2012-03-31
+        dtype: datetime64[ns]
+        """
+
+        def pandas_normalize(s) -> "ps.Series[np.datetime64]":
+            return s.dt.normalize()
+
+        return self._data.koalas.transform_batch(pandas_normalize)
+
+    def strftime(self, date_format) -> "ps.Series":
+        """
+        Convert to a string Series using specified date_format.
+
+        Return an series of formatted strings specified by date_format, which
+        supports the same string format as the python standard library. Details
+        of the string format can be found in python string format
+        doc.
+
+        Parameters
+        ----------
+        date_format : str
+            Date format string (example: "%%Y-%%m-%%d").
+
+        Returns
+        -------
+        Series
+            Series of formatted strings.
+
+        See Also
+        --------
+        to_datetime : Convert the given argument to datetime.
+        normalize : Return series with times to midnight.
+        round : Round the series to the specified freq.
+        floor : Floor the series to the specified freq.
+
+        Examples
+        --------
+        >>> series = ps.Series(pd.date_range(pd.Timestamp("2018-03-10 09:00"),
+        ...                                  periods=3, freq='s'))
+        >>> series
+        0   2018-03-10 09:00:00
+        1   2018-03-10 09:00:01
+        2   2018-03-10 09:00:02
+        dtype: datetime64[ns]
+
+        >>> series.dt.strftime('%B %d, %Y, %r')
+        0    March 10, 2018, 09:00:00 AM
+        1    March 10, 2018, 09:00:01 AM
+        2    March 10, 2018, 09:00:02 AM
+        dtype: object
+        """
+
+        def pandas_strftime(s) -> "ps.Series[str]":
+            return s.dt.strftime(date_format)
+
+        return self._data.koalas.transform_batch(pandas_strftime)
+
+    def round(self, freq, *args, **kwargs) -> "ps.Series":
+        """
+        Perform round operation on the data to the specified freq.
+
+        Parameters
+        ----------
+        freq : str or Offset
+            The frequency level to round the index to. Must be a fixed
+            frequency like 'S' (second) not 'ME' (month end).
+
+        nonexistent : 'shift_forward', 'shift_backward, 'NaT', timedelta, default 'raise'
+            A nonexistent time does not exist in a particular timezone
+            where clocks moved forward due to DST.
+
+            - 'shift_forward' will shift the nonexistent time forward to the
+              closest existing time
+            - 'shift_backward' will shift the nonexistent time backward to the
+              closest existing time
+            - 'NaT' will return NaT where there are nonexistent times
+            - timedelta objects will shift nonexistent times by the timedelta
+            - 'raise' will raise an NonExistentTimeError if there are
+              nonexistent times
+
+            .. note:: this option only works with pandas 0.24.0+
+
+        Returns
+        -------
+        Series
+            a Series with the same index for a Series.
+
+        Raises
+        ------
+        ValueError if the `freq` cannot be converted.
+
+        Examples
+        --------
+        >>> series = ps.Series(pd.date_range('1/1/2018 11:59:00', periods=3, freq='min'))
+        >>> series
+        0   2018-01-01 11:59:00
+        1   2018-01-01 12:00:00
+        2   2018-01-01 12:01:00
+        dtype: datetime64[ns]
+
+        >>> series.dt.round("H")
+        0   2018-01-01 12:00:00
+        1   2018-01-01 12:00:00
+        2   2018-01-01 12:00:00
+        dtype: datetime64[ns]
+        """
+
+        def pandas_round(s) -> "ps.Series[np.datetime64]":
+            return s.dt.round(freq, *args, **kwargs)
+
+        return self._data.koalas.transform_batch(pandas_round)
+
+    def floor(self, freq, *args, **kwargs) -> "ps.Series":
+        """
+        Perform floor operation on the data to the specified freq.
+
+        Parameters
+        ----------
+        freq : str or Offset
+            The frequency level to floor the index to. Must be a fixed
+            frequency like 'S' (second) not 'ME' (month end).
+
+        nonexistent : 'shift_forward', 'shift_backward, 'NaT', timedelta, default 'raise'
+            A nonexistent time does not exist in a particular timezone
+            where clocks moved forward due to DST.
+
+            - 'shift_forward' will shift the nonexistent time forward to the
+              closest existing time
+            - 'shift_backward' will shift the nonexistent time backward to the
+              closest existing time
+            - 'NaT' will return NaT where there are nonexistent times
+            - timedelta objects will shift nonexistent times by the timedelta
+            - 'raise' will raise an NonExistentTimeError if there are
+              nonexistent times
+
+            .. note:: this option only works with pandas 0.24.0+
+
+        Returns
+        -------
+        Series
+            a Series with the same index for a Series.
+
+        Raises
+        ------
+        ValueError if the `freq` cannot be converted.
+
+        Examples
+        --------
+        >>> series = ps.Series(pd.date_range('1/1/2018 11:59:00', periods=3, freq='min'))
+        >>> series
+        0   2018-01-01 11:59:00
+        1   2018-01-01 12:00:00
+        2   2018-01-01 12:01:00
+        dtype: datetime64[ns]
+
+        >>> series.dt.floor("H")
+        0   2018-01-01 11:00:00
+        1   2018-01-01 12:00:00
+        2   2018-01-01 12:00:00
+        dtype: datetime64[ns]
+        """
+
+        def pandas_floor(s) -> "ps.Series[np.datetime64]":
+            return s.dt.floor(freq, *args, **kwargs)
+
+        return self._data.koalas.transform_batch(pandas_floor)
+
+    def ceil(self, freq, *args, **kwargs) -> "ps.Series":
+        """
+        Perform ceil operation on the data to the specified freq.
+
+        Parameters
+        ----------
+        freq : str or Offset
+            The frequency level to round the index to. Must be a fixed
+            frequency like 'S' (second) not 'ME' (month end).
+
+        nonexistent : 'shift_forward', 'shift_backward, 'NaT', timedelta, default 'raise'
+            A nonexistent time does not exist in a particular timezone
+            where clocks moved forward due to DST.
+
+            - 'shift_forward' will shift the nonexistent time forward to the
+              closest existing time
+            - 'shift_backward' will shift the nonexistent time backward to the
+              closest existing time
+            - 'NaT' will return NaT where there are nonexistent times
+            - timedelta objects will shift nonexistent times by the timedelta
+            - 'raise' will raise an NonExistentTimeError if there are
+              nonexistent times
+
+            .. note:: this option only works with pandas 0.24.0+
+
+        Returns
+        -------
+        Series
+            a Series with the same index for a Series.
+
+        Raises
+        ------
+        ValueError if the `freq` cannot be converted.
+
+        Examples
+        --------
+        >>> series = ps.Series(pd.date_range('1/1/2018 11:59:00', periods=3, freq='min'))
+        >>> series
+        0   2018-01-01 11:59:00
+        1   2018-01-01 12:00:00
+        2   2018-01-01 12:01:00
+        dtype: datetime64[ns]
+
+        >>> series.dt.ceil("H")
+        0   2018-01-01 12:00:00
+        1   2018-01-01 12:00:00
+        2   2018-01-01 13:00:00
+        dtype: datetime64[ns]
+        """
+
+        def pandas_ceil(s) -> "ps.Series[np.datetime64]":
+            return s.dt.ceil(freq, *args, **kwargs)
+
+        return self._data.koalas.transform_batch(pandas_ceil)
+
+    def month_name(self, locale=None) -> "ps.Series":
+        """
+        Return the month names of the series with specified locale.
+
+        Parameters
+        ----------
+        locale : str, optional
+            Locale determining the language in which to return the month name.
+            Default is English locale.
+
+        Returns
+        -------
+        Series
+            Series of month names.
+
+        Examples
+        --------
+        >>> series = ps.Series(pd.date_range(start='2018-01', freq='M', periods=3))
+        >>> series
+        0   2018-01-31
+        1   2018-02-28
+        2   2018-03-31
+        dtype: datetime64[ns]
+
+        >>> series.dt.month_name()
+        0     January
+        1    February
+        2       March
+        dtype: object
+        """
+
+        def pandas_month_name(s) -> "ps.Series[str]":
+            return s.dt.month_name(locale=locale)
+
+        return self._data.koalas.transform_batch(pandas_month_name)
+
+    def day_name(self, locale=None) -> "ps.Series":
+        """
+        Return the day names of the series with specified locale.
+
+        Parameters
+        ----------
+        locale : str, optional
+            Locale determining the language in which to return the day name.
+            Default is English locale.
+
+        Returns
+        -------
+        Series
+            Series of day names.
+
+        Examples
+        --------
+        >>> series = ps.Series(pd.date_range(start='2018-01-01', freq='D', periods=3))
+        >>> series
+        0   2018-01-01
+        1   2018-01-02
+        2   2018-01-03
+        dtype: datetime64[ns]
+
+        >>> series.dt.day_name()
+        0       Monday
+        1      Tuesday
+        2    Wednesday
+        dtype: object
+        """
+
+        def pandas_day_name(s) -> "ps.Series[str]":
+            return s.dt.day_name(locale=locale)
+
+        return self._data.koalas.transform_batch(pandas_day_name)
+
+
+def _test():
+    import os
+    import doctest
+    import sys
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.datetimes
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.datetimes.__dict__.copy()
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]")
+        .appName("pyspark.pandas.datetimes tests")
+        .getOrCreate()
+    )
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.datetimes,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/exceptions.py b/python/pyspark/pandas/exceptions.py
new file mode 100644
index 0000000000000..5e4b1864f4738
--- /dev/null
+++ b/python/pyspark/pandas/exceptions.py
@@ -0,0 +1,136 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Exceptions/Errors used in pandas-on-Spark.
+"""
+
+
+class DataError(Exception):
+    pass
+
+
+class SparkPandasIndexingError(Exception):
+    pass
+
+
+def code_change_hint(pandas_function, spark_target_function):
+    if pandas_function is not None and spark_target_function is not None:
+        return "You are trying to use pandas function {}, use spark function {}".format(
+            pandas_function, spark_target_function
+        )
+    elif pandas_function is not None and spark_target_function is None:
+        return (
+            "You are trying to use pandas function {}, checkout the spark "
+            "user guide to find a relevant function"
+        ).format(pandas_function)
+    elif pandas_function is None and spark_target_function is not None:
+        return "Use spark function {}".format(spark_target_function)
+    else:  # both none
+        return "Checkout the spark user guide to find a relevant function"
+
+
+class SparkPandasNotImplementedError(NotImplementedError):
+    def __init__(self, pandas_function=None, spark_target_function=None, description=""):
+        self.pandas_source = pandas_function
+        self.spark_target = spark_target_function
+        hint = code_change_hint(pandas_function, spark_target_function)
+        if len(description) > 0:
+            description += " " + hint
+        else:
+            description = hint
+        super().__init__(description)
+
+
+class PandasNotImplementedError(NotImplementedError):
+    def __init__(
+        self,
+        class_name,
+        method_name=None,
+        arg_name=None,
+        property_name=None,
+        deprecated=False,
+        reason="",
+    ):
+        assert (method_name is None) != (property_name is None)
+        self.class_name = class_name
+        self.method_name = method_name
+        self.arg_name = arg_name
+        if method_name is not None:
+            if arg_name is not None:
+                msg = "The method `{0}.{1}()` does not support `{2}` parameter. {3}".format(
+                    class_name, method_name, arg_name, reason
+                )
+            else:
+                if deprecated:
+                    msg = (
+                        "The method `{0}.{1}()` is deprecated in pandas and will therefore "
+                        + "not be supported in pandas-on-Spark. {2}"
+                    ).format(class_name, method_name, reason)
+                else:
+                    if reason == "":
+                        reason = " yet."
+                    else:
+                        reason = ". " + reason
+                    msg = "The method `{0}.{1}()` is not implemented{2}".format(
+                        class_name, method_name, reason
+                    )
+        else:
+            if deprecated:
+                msg = (
+                    "The property `{0}.{1}()` is deprecated in pandas and will therefore "
+                    + "not be supported in pandas-on-Spark. {2}"
+                ).format(class_name, property_name, reason)
+            else:
+                if reason == "":
+                    reason = " yet."
+                else:
+                    reason = ". " + reason
+                msg = "The property `{0}.{1}()` is not implemented{2}".format(
+                    class_name, property_name, reason
+                )
+        super().__init__(msg)
+
+
+def _test():
+    import os
+    import doctest
+    import sys
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.exceptions
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.exceptions.__dict__.copy()
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]")
+        .appName("pyspark.pandas.exceptions tests")
+        .getOrCreate()
+    )
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.exceptions,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/extensions.py b/python/pyspark/pandas/extensions.py
new file mode 100644
index 0000000000000..59b3343f06895
--- /dev/null
+++ b/python/pyspark/pandas/extensions.py
@@ -0,0 +1,374 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import warnings
+
+
+class CachedAccessor:
+    """
+    Custom property-like object.
+
+    A descriptor for caching accessors:
+
+    Parameters
+    ----------
+    name : str
+        Namespace that accessor's methods, properties, etc will be accessed under, e.g. "foo" for a
+        dataframe accessor yields the accessor ``df.foo``
+    accessor: cls
+        Class with the extension methods.
+
+    Notes
+    -----
+    For accessor, the class's __init__ method assumes that you are registering an accessor for one
+    of ``Series``, ``DataFrame``, or ``Index``.
+
+    This object is not meant to be instantiated directly. Instead, use register_dataframe_accessor,
+    register_series_accessor, or register_index_accessor.
+
+    The pandas-on-Spark accessor is modified based on pandas.core.accessor.
+    """
+
+    def __init__(self, name, accessor):
+        self._name = name
+        self._accessor = accessor
+
+    def __get__(self, obj, cls):
+        if obj is None:
+            return self._accessor
+        accessor_obj = self._accessor(obj)
+        object.__setattr__(obj, self._name, accessor_obj)
+        return accessor_obj
+
+
+def _register_accessor(name, cls):
+    """
+    Register a custom accessor on {klass} objects.
+
+    Parameters
+    ----------
+    name : str
+        Name under which the accessor should be registered. A warning is issued if this name
+        conflicts with a preexisting attribute.
+
+    Returns
+    -------
+    callable
+        A class decorator.
+
+    See Also
+    --------
+    register_dataframe_accessor: Register a custom accessor on DataFrame objects
+    register_series_accessor: Register a custom accessor on Series objects
+    register_index_accessor: Register a custom accessor on Index objects
+
+    Notes
+    -----
+    When accessed, your accessor will be initialiazed with the pandas-on-Spark object the user
+    is interacting with. The code signature must be:
+
+    .. code-block:: python
+
+        def __init__(self, pandas_on_spark_obj):
+            # constructor logic
+        ...
+
+    In the pandas API, if data passed to your accessor has an incorrect dtype, it's recommended to
+    raise an ``AttributeError`` for consistency purposes. In pandas-on-Spark, ``ValueError`` is more
+    frequently used to annotate when a value's datatype is unexpected for a given method/function.
+
+    Ultimately, you can structure this however you like, but pandas-on-Spark would likely do
+    something like this:
+
+    >>> ps.Series(['a', 'b']).dt
+    ...
+    Traceback (most recent call last):
+        ...
+    ValueError: Cannot call DatetimeMethods on type StringType
+
+    Note: This function is not meant to be used directly - instead, use register_dataframe_accessor,
+    register_series_accessor, or register_index_accessor.
+    """
+
+    def decorator(accessor):
+        if hasattr(cls, name):
+            msg = (
+                "registration of accessor {0} under name '{1}' for type {2} is overriding "
+                "a preexisting attribute with the same name.".format(accessor, name, cls.__name__)
+            )
+
+            warnings.warn(
+                msg, UserWarning, stacklevel=2,
+            )
+        setattr(cls, name, CachedAccessor(name, accessor))
+        return accessor
+
+    return decorator
+
+
+def register_dataframe_accessor(name):
+    """
+    Register a custom accessor with a DataFrame
+
+    Parameters
+    ----------
+    name : str
+        name used when calling the accessor after its registered
+
+    Returns
+    -------
+    callable
+        A class decorator.
+
+    See Also
+    --------
+    register_series_accessor: Register a custom accessor on Series objects
+    register_index_accessor: Register a custom accessor on Index objects
+
+    Notes
+    -----
+    When accessed, your accessor will be initialiazed with the pandas-on-Spark object the user
+    is interacting with. The accessor's init method should always ingest the object being accessed.
+    See the examples for the init signature.
+
+    In the pandas API, if data passed to your accessor has an incorrect dtype, it's recommended to
+    raise an ``AttributeError`` for consistency purposes. In pandas-on-Spark, ``ValueError`` is more
+    frequently used to annotate when a value's datatype is unexpected for a given method/function.
+
+    Ultimately, you can structure this however you like, but pandas-on-Spark would likely do
+    something like this:
+
+    >>> ps.Series(['a', 'b']).dt
+    ...
+    Traceback (most recent call last):
+        ...
+    ValueError: Cannot call DatetimeMethods on type StringType
+
+    Examples
+    --------
+    In your library code::
+
+        from pyspark.pandas.extensions import register_dataframe_accessor
+
+        @register_dataframe_accessor("geo")
+        class GeoAccessor:
+
+            def __init__(self, pandas_on_spark_obj):
+                self._obj = pandas_on_spark_obj
+                # other constructor logic
+
+            @property
+            def center(self):
+                # return the geographic center point of this DataFrame
+                lat = self._obj.latitude
+                lon = self._obj.longitude
+                return (float(lon.mean()), float(lat.mean()))
+
+            def plot(self):
+                # plot this array's data on a map
+                pass
+
+    Then, in an ipython session::
+
+        >>> ## Import if the accessor is in the other file.
+        >>> # from my_ext_lib import GeoAccessor
+        >>> kdf = ps.DataFrame({"longitude": np.linspace(0,10),
+        ...                     "latitude": np.linspace(0, 20)})
+        >>> kdf.geo.center  # doctest: +SKIP
+        (5.0, 10.0)
+
+        >>> kdf.geo.plot()  # doctest: +SKIP
+    """
+    from pyspark.pandas import DataFrame
+
+    return _register_accessor(name, DataFrame)
+
+
+def register_series_accessor(name):
+    """
+    Register a custom accessor with a Series object
+
+    Parameters
+    ----------
+    name : str
+        name used when calling the accessor after its registered
+
+    Returns
+    -------
+    callable
+        A class decorator.
+
+    See Also
+    --------
+    register_dataframe_accessor: Register a custom accessor on DataFrame objects
+    register_index_accessor: Register a custom accessor on Index objects
+
+    Notes
+    -----
+    When accessed, your accessor will be initialiazed with the pandas-on-Spark object the user is
+    interacting with. The code signature must be::
+
+        def __init__(self, pandas_on_spark_obj):
+            # constructor logic
+        ...
+
+    In the pandas API, if data passed to your accessor has an incorrect dtype, it's recommended to
+    raise an ``AttributeError`` for consistency purposes. In pandas-on-Spark, ``ValueError`` is more
+    frequently used to annotate when a value's datatype is unexpected for a given method/function.
+
+    Ultimately, you can structure this however you like, but pandas-on-Spark would likely do
+    something like this:
+
+    >>> ps.Series(['a', 'b']).dt
+    ...
+    Traceback (most recent call last):
+        ...
+    ValueError: Cannot call DatetimeMethods on type StringType
+
+    Examples
+    --------
+    In your library code::
+
+        from pyspark.pandas.extensions import register_series_accessor
+
+        @register_series_accessor("geo")
+        class GeoAccessor:
+
+            def __init__(self, pandas_on_spark_obj):
+                self._obj = pandas_on_spark_obj
+
+            @property
+            def is_valid(self):
+                # boolean check to see if series contains valid geometry
+                return True
+
+    Then, in an ipython session::
+
+        >>> ## Import if the accessor is in the other file.
+        >>> # from my_ext_lib import GeoAccessor
+        >>> kdf = ps.DataFrame({"longitude": np.linspace(0,10),
+        ...                     "latitude": np.linspace(0, 20)})
+        >>> kdf.longitude.geo.is_valid  # doctest: +SKIP
+        True
+    """
+    from pyspark.pandas import Series
+
+    return _register_accessor(name, Series)
+
+
+def register_index_accessor(name):
+    """
+    Register a custom accessor with an Index
+
+    Parameters
+    ----------
+    name : str
+        name used when calling the accessor after its registered
+
+    Returns
+    -------
+    callable
+        A class decorator.
+
+    See Also
+    --------
+    register_dataframe_accessor: Register a custom accessor on DataFrame objects
+    register_series_accessor: Register a custom accessor on Series objects
+
+    Notes
+    -----
+    When accessed, your accessor will be initialiazed with the pandas-on-Spark object the user is
+    interacting with. The code signature must be::
+
+        def __init__(self, pandas_on_spark_obj):
+            # constructor logic
+        ...
+
+    In the pandas API, if data passed to your accessor has an incorrect dtype, it's recommended to
+    raise an ``AttributeError`` for consistency purposes. In pandas-on-Spark, ``ValueError`` is more
+    frequently used to annotate when a value's datatype is unexpected for a given method/function.
+
+    Ultimately, you can structure this however you like, but pandas-on-Spark would likely do
+    something like this:
+
+    >>> ps.Series(['a', 'b']).dt
+    ...
+    Traceback (most recent call last):
+        ...
+    ValueError: Cannot call DatetimeMethods on type StringType
+
+    Examples
+    --------
+    In your library code::
+
+        from pyspark.pandas.extensions import register_index_accessor
+
+        @register_index_accessor("foo")
+        class CustomAccessor:
+
+            def __init__(self, pandas_on_spark_obj):
+                self._obj = pandas_on_spark_obj
+                self.item = "baz"
+
+            @property
+            def bar(self):
+                # return item value
+                return self.item
+
+    Then, in an ipython session::
+
+        >>> ## Import if the accessor is in the other file.
+        >>> # from my_ext_lib import CustomAccessor
+        >>> kdf = ps.DataFrame({"longitude": np.linspace(0,10),
+        ...                     "latitude": np.linspace(0, 20)})
+        >>> kdf.index.foo.bar  # doctest: +SKIP
+        'baz'
+    """
+    from pyspark.pandas import Index
+
+    return _register_accessor(name, Index)
+
+
+def _test():
+    import os
+    import doctest
+    import sys
+    import numpy
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.extensions
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.extensions.__dict__.copy()
+    globs["np"] = numpy
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]")
+        .appName("pyspark.pandas.extensions tests")
+        .getOrCreate()
+    )
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.extensions,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/frame.py b/python/pyspark/pandas/frame.py
new file mode 100644
index 0000000000000..22cd013f274c2
--- /dev/null
+++ b/python/pyspark/pandas/frame.py
@@ -0,0 +1,11995 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+A wrapper class for Spark DataFrame to behave similar to pandas DataFrame.
+"""
+from collections import OrderedDict, defaultdict, namedtuple
+from collections.abc import Mapping
+from distutils.version import LooseVersion
+import re
+import warnings
+import inspect
+import json
+import types
+from functools import partial, reduce
+import sys
+from itertools import zip_longest
+from typing import (
+    Any,
+    Optional,
+    List,
+    Tuple,
+    Union,
+    Generic,
+    TypeVar,
+    Iterable,
+    Iterator,
+    Dict,
+    Callable,
+    cast,
+    TYPE_CHECKING,
+)
+import datetime
+
+import numpy as np
+import pandas as pd
+from pandas.api.types import is_list_like, is_dict_like, is_scalar
+from pandas.api.extensions import ExtensionDtype
+from pandas.tseries.frequencies import DateOffset, to_offset
+
+if TYPE_CHECKING:
+    from pandas.io.formats.style import Styler  # noqa: F401 (SPARK-34943)
+
+if LooseVersion(pd.__version__) >= LooseVersion("0.24"):
+    from pandas.core.dtypes.common import infer_dtype_from_object
+else:
+    from pandas.core.dtypes.common import _get_dtype_from_object as infer_dtype_from_object
+from pandas.core.accessor import CachedAccessor
+from pandas.core.dtypes.inference import is_sequence
+from pyspark import StorageLevel
+from pyspark import sql as spark
+from pyspark.sql import Column, DataFrame as SparkDataFrame, functions as F
+from pyspark.sql.functions import pandas_udf
+from pyspark.sql.types import (
+    BooleanType,
+    DoubleType,
+    FloatType,
+    NumericType,
+    StringType,
+    StructType,
+    StructField,
+    ArrayType,
+)
+from pyspark.sql.window import Window
+
+from pyspark import pandas as ps  # For running doctests and reference resolution in PyCharm.
+from pyspark.pandas.accessors import PandasOnSparkFrameMethods
+from pyspark.pandas.config import option_context, get_option
+from pyspark.pandas.spark.accessors import SparkFrameMethods, CachedSparkFrameMethods
+from pyspark.pandas.utils import (
+    align_diff_frames,
+    column_labels_level,
+    combine_frames,
+    default_session,
+    is_name_like_tuple,
+    is_name_like_value,
+    is_testing,
+    name_like_string,
+    same_anchor,
+    scol_for,
+    validate_arguments_and_invoke_function,
+    validate_axis,
+    validate_bool_kwarg,
+    validate_how,
+    verify_temp_column_name,
+)
+from pyspark.pandas.spark.utils import as_nullable_spark_type, force_decimal_precision_scale
+from pyspark.pandas.generic import Frame
+from pyspark.pandas.internal import (
+    InternalFrame,
+    HIDDEN_COLUMNS,
+    NATURAL_ORDER_COLUMN_NAME,
+    SPARK_INDEX_NAME_FORMAT,
+    SPARK_DEFAULT_INDEX_NAME,
+    SPARK_DEFAULT_SERIES_NAME,
+)
+from pyspark.pandas.missing.frame import _MissingPandasLikeDataFrame
+from pyspark.pandas.ml import corr
+from pyspark.pandas.typedef import (
+    as_spark_type,
+    infer_return_type,
+    spark_type_to_pandas_dtype,
+    DataFrameType,
+    SeriesType,
+    Scalar,
+    ScalarType,
+)
+from pyspark.pandas.plot import PandasOnSparkPlotAccessor
+
+if TYPE_CHECKING:
+    from pyspark.pandas.indexes import Index  # noqa: F401 (SPARK-34943)
+    from pyspark.pandas.series import Series  # noqa: F401 (SPARK-34943)
+
+
+# These regular expression patterns are complied and defined here to avoid to compile the same
+# pattern every time it is used in _repr_ and _repr_html_ in DataFrame.
+# Two patterns basically seek the footer string from Pandas'
+REPR_PATTERN = re.compile(r"\n\n\[(?P<rows>[0-9]+) rows x (?P<columns>[0-9]+) columns\]$")
+REPR_HTML_PATTERN = re.compile(
+    r"\n\<p\>(?P<rows>[0-9]+) rows × (?P<columns>[0-9]+) columns\<\/p\>\n\<\/div\>$"
+)
+
+
+_flex_doc_FRAME = """
+Get {desc} of dataframe and other, element-wise (binary operator `{op_name}`).
+
+Equivalent to ``{equiv}``. With reverse version, `{reverse}`.
+
+Among flexible wrappers (`add`, `sub`, `mul`, `div`) to
+arithmetic operators: `+`, `-`, `*`, `/`, `//`.
+
+Parameters
+----------
+other : scalar
+    Any single data
+
+Returns
+-------
+DataFrame
+    Result of the arithmetic operation.
+
+Examples
+--------
+>>> df = ps.DataFrame({{'angles': [0, 3, 4],
+...                    'degrees': [360, 180, 360]}},
+...                   index=['circle', 'triangle', 'rectangle'],
+...                   columns=['angles', 'degrees'])
+>>> df
+           angles  degrees
+circle          0      360
+triangle        3      180
+rectangle       4      360
+
+Add a scalar with operator version which return the same
+results. Also reverse version.
+
+>>> df + 1
+           angles  degrees
+circle          1      361
+triangle        4      181
+rectangle       5      361
+
+>>> df.add(1)
+           angles  degrees
+circle          1      361
+triangle        4      181
+rectangle       5      361
+
+>>> df.add(df)
+           angles  degrees
+circle          0      720
+triangle        6      360
+rectangle       8      720
+
+>>> df + df + df
+           angles  degrees
+circle          0     1080
+triangle        9      540
+rectangle      12     1080
+
+>>> df.radd(1)
+           angles  degrees
+circle          1      361
+triangle        4      181
+rectangle       5      361
+
+Divide and true divide by constant with reverse version.
+
+>>> df / 10
+           angles  degrees
+circle        0.0     36.0
+triangle      0.3     18.0
+rectangle     0.4     36.0
+
+>>> df.div(10)
+           angles  degrees
+circle        0.0     36.0
+triangle      0.3     18.0
+rectangle     0.4     36.0
+
+>>> df.rdiv(10)
+             angles   degrees
+circle          inf  0.027778
+triangle   3.333333  0.055556
+rectangle  2.500000  0.027778
+
+>>> df.truediv(10)
+           angles  degrees
+circle        0.0     36.0
+triangle      0.3     18.0
+rectangle     0.4     36.0
+
+>>> df.rtruediv(10)
+             angles   degrees
+circle          inf  0.027778
+triangle   3.333333  0.055556
+rectangle  2.500000  0.027778
+
+Subtract by constant with reverse version.
+
+>>> df - 1
+           angles  degrees
+circle         -1      359
+triangle        2      179
+rectangle       3      359
+
+>>> df.sub(1)
+           angles  degrees
+circle         -1      359
+triangle        2      179
+rectangle       3      359
+
+>>> df.rsub(1)
+           angles  degrees
+circle          1     -359
+triangle       -2     -179
+rectangle      -3     -359
+
+Multiply by constant with reverse version.
+
+>>> df * 1
+           angles  degrees
+circle          0      360
+triangle        3      180
+rectangle       4      360
+
+>>> df.mul(1)
+           angles  degrees
+circle          0      360
+triangle        3      180
+rectangle       4      360
+
+>>> df.rmul(1)
+           angles  degrees
+circle          0      360
+triangle        3      180
+rectangle       4      360
+
+Floor Divide by constant with reverse version.
+
+>>> df // 10
+           angles  degrees
+circle        0.0     36.0
+triangle      0.0     18.0
+rectangle     0.0     36.0
+
+>>> df.floordiv(10)
+           angles  degrees
+circle        0.0     36.0
+triangle      0.0     18.0
+rectangle     0.0     36.0
+
+>>> df.rfloordiv(10)  # doctest: +SKIP
+           angles  degrees
+circle        inf      0.0
+triangle      3.0      0.0
+rectangle     2.0      0.0
+
+Mod by constant with reverse version.
+
+>>> df % 2
+           angles  degrees
+circle          0        0
+triangle        1        0
+rectangle       0        0
+
+>>> df.mod(2)
+           angles  degrees
+circle          0        0
+triangle        1        0
+rectangle       0        0
+
+>>> df.rmod(2)
+           angles  degrees
+circle        NaN        2
+triangle      2.0        2
+rectangle     2.0        2
+
+Power by constant with reverse version.
+
+>>> df ** 2
+           angles   degrees
+circle        0.0  129600.0
+triangle      9.0   32400.0
+rectangle    16.0  129600.0
+
+>>> df.pow(2)
+           angles   degrees
+circle        0.0  129600.0
+triangle      9.0   32400.0
+rectangle    16.0  129600.0
+
+>>> df.rpow(2)
+           angles        degrees
+circle        1.0  2.348543e+108
+triangle      8.0   1.532496e+54
+rectangle    16.0  2.348543e+108
+"""
+
+T = TypeVar("T")
+
+
+def _create_tuple_for_frame_type(params):
+    """
+    This is a workaround to support variadic generic in DataFrame.
+
+    See https://github.com/python/typing/issues/193
+    we always wraps the given type hints by a tuple to mimic the variadic generic.
+    """
+    from pyspark.pandas.typedef import NameTypeHolder
+
+    if isinstance(params, zip):
+        params = [slice(name, tpe) for name, tpe in params]
+
+    if isinstance(params, slice):
+        params = (params,)
+
+    if (
+        hasattr(params, "__len__")
+        and isinstance(params, Iterable)
+        and all(isinstance(param, slice) for param in params)
+    ):
+        for param in params:
+            if isinstance(param.start, str) and param.step is not None:
+                raise TypeError(
+                    "Type hints should be specified as "
+                    "DataFrame['name': type]; however, got %s" % param
+                )
+
+        name_classes = []
+        for param in params:
+            new_class = type("NameType", (NameTypeHolder,), {})
+            new_class.name = param.start
+            # When the given argument is a numpy's dtype instance.
+            new_class.tpe = param.stop.type if isinstance(param.stop, np.dtype) else param.stop
+            name_classes.append(new_class)
+
+        return Tuple[tuple(name_classes)]
+
+    if not isinstance(params, Iterable):
+        params = [params]
+
+    new_params = []
+    for param in params:
+        if isinstance(param, ExtensionDtype):
+            new_class = type("NameType", (NameTypeHolder,), {})
+            new_class.tpe = param
+            new_params.append(new_class)
+        else:
+            new_params.append(param.type if isinstance(param, np.dtype) else param)
+    return Tuple[tuple(new_params)]
+
+
+if (3, 5) <= sys.version_info < (3, 7) and __name__ != "__main__":
+    from typing import GenericMeta  # type: ignore
+
+    # This is a workaround to support variadic generic in DataFrame in Python 3.5+.
+    # See https://github.com/python/typing/issues/193
+    # We wrap the input params by a tuple to mimic variadic generic.
+    old_getitem = GenericMeta.__getitem__  # type: ignore
+
+    def new_getitem(self, params):
+        if hasattr(self, "is_dataframe"):
+            return old_getitem(self, _create_tuple_for_frame_type(params))
+        else:
+            return old_getitem(self, params)
+
+    GenericMeta.__getitem__ = new_getitem  # type: ignore
+
+
+class DataFrame(Frame, Generic[T]):
+    """
+    pandas-on-Spark DataFrame that corresponds to pandas DataFrame logically. This holds Spark
+    DataFrame internally.
+
+    :ivar _internal: an internal immutable Frame to manage metadata.
+    :type _internal: InternalFrame
+
+    Parameters
+    ----------
+    data : numpy ndarray (structured or homogeneous), dict, pandas DataFrame, Spark DataFrame \
+        or pandas-on-Spark Series
+        Dict can contain Series, arrays, constants, or list-like objects
+        If data is a dict, argument order is maintained for Python 3.6
+        and later.
+        Note that if `data` is a pandas DataFrame, a Spark DataFrame, and a pandas-on-Spark Series,
+        other arguments should not be used.
+    index : Index or array-like
+        Index to use for resulting frame. Will default to RangeIndex if
+        no indexing information part of input data and no index provided
+    columns : Index or array-like
+        Column labels to use for resulting frame. Will default to
+        RangeIndex (0, 1, 2, ..., n) if no column labels are provided
+    dtype : dtype, default None
+        Data type to force. Only a single dtype is allowed. If None, infer
+    copy : boolean, default False
+        Copy data from inputs. Only affects DataFrame / 2d ndarray input
+
+    Examples
+    --------
+    Constructing DataFrame from a dictionary.
+
+    >>> d = {'col1': [1, 2], 'col2': [3, 4]}
+    >>> df = ps.DataFrame(data=d, columns=['col1', 'col2'])
+    >>> df
+       col1  col2
+    0     1     3
+    1     2     4
+
+    Constructing DataFrame from pandas DataFrame
+
+    >>> df = ps.DataFrame(pd.DataFrame(data=d, columns=['col1', 'col2']))
+    >>> df
+       col1  col2
+    0     1     3
+    1     2     4
+
+    Notice that the inferred dtype is int64.
+
+    >>> df.dtypes
+    col1    int64
+    col2    int64
+    dtype: object
+
+    To enforce a single dtype:
+
+    >>> df = ps.DataFrame(data=d, dtype=np.int8)
+    >>> df.dtypes
+    col1    int8
+    col2    int8
+    dtype: object
+
+    Constructing DataFrame from numpy ndarray:
+
+    >>> df2 = ps.DataFrame(np.random.randint(low=0, high=10, size=(5, 5)),
+    ...                    columns=['a', 'b', 'c', 'd', 'e'])
+    >>> df2  # doctest: +SKIP
+       a  b  c  d  e
+    0  3  1  4  9  8
+    1  4  8  4  8  4
+    2  7  6  5  6  7
+    3  8  7  9  1  0
+    4  2  5  4  3  9
+    """
+
+    def __init__(self, data=None, index=None, columns=None, dtype=None, copy=False):
+        if isinstance(data, InternalFrame):
+            assert index is None
+            assert columns is None
+            assert dtype is None
+            assert not copy
+            internal = data
+        elif isinstance(data, spark.DataFrame):
+            assert index is None
+            assert columns is None
+            assert dtype is None
+            assert not copy
+            internal = InternalFrame(spark_frame=data, index_spark_columns=None)
+        elif isinstance(data, ps.Series):
+            assert index is None
+            assert columns is None
+            assert dtype is None
+            assert not copy
+            data = data.to_frame()
+            internal = data._internal
+        else:
+            if isinstance(data, pd.DataFrame):
+                assert index is None
+                assert columns is None
+                assert dtype is None
+                assert not copy
+                pdf = data
+            else:
+                pdf = pd.DataFrame(data=data, index=index, columns=columns, dtype=dtype, copy=copy)
+            internal = InternalFrame.from_pandas(pdf)
+
+        object.__setattr__(self, "_internal_frame", internal)
+
+    @property
+    def _ksers(self):
+        """ Return a dict of column label -> Series which anchors `self`. """
+        from pyspark.pandas.series import Series
+
+        if not hasattr(self, "_kseries"):
+            object.__setattr__(
+                self,
+                "_kseries",
+                {label: Series(data=self, index=label) for label in self._internal.column_labels},
+            )
+        else:
+            kseries = self._kseries
+            assert len(self._internal.column_labels) == len(kseries), (
+                len(self._internal.column_labels),
+                len(kseries),
+            )
+            if any(self is not kser._kdf for kser in kseries.values()):
+                # Refresh the dict to contain only Series anchoring `self`.
+                self._kseries = {
+                    label: kseries[label]
+                    if self is kseries[label]._kdf
+                    else Series(data=self, index=label)
+                    for label in self._internal.column_labels
+                }
+        return self._kseries
+
+    @property
+    def _internal(self) -> InternalFrame:
+        return self._internal_frame
+
+    def _update_internal_frame(self, internal: InternalFrame, requires_same_anchor: bool = True):
+        """
+        Update InternalFrame with the given one.
+
+        If the column_label is changed or the new InternalFrame is not the same `anchor`,
+        disconnect the link to the Series and create a new one.
+
+        If `requires_same_anchor` is `False`, checking whether or not the same anchor is ignored
+        and force to update the InternalFrame, e.g., replacing the internal with the resolved_copy,
+        updating the underlying Spark DataFrame which need to combine a different Spark DataFrame.
+
+        :param internal: the new InternalFrame
+        :param requires_same_anchor: whether checking the same anchor
+        """
+        from pyspark.pandas.series import Series
+
+        if hasattr(self, "_kseries"):
+            kseries = {}
+
+            for old_label, new_label in zip_longest(
+                self._internal.column_labels, internal.column_labels
+            ):
+                if old_label is not None:
+                    kser = self._ksers[old_label]
+
+                    renamed = old_label != new_label
+                    not_same_anchor = requires_same_anchor and not same_anchor(internal, kser)
+
+                    if renamed or not_same_anchor:
+                        kdf = DataFrame(self._internal.select_column(old_label))  # type: DataFrame
+                        kser._update_anchor(kdf)
+                        kser = None
+                else:
+                    kser = None
+                if new_label is not None:
+                    if kser is None:
+                        kser = Series(data=self, index=new_label)
+                    kseries[new_label] = kser
+
+            self._kseries = kseries
+
+        self._internal_frame = internal
+
+        if hasattr(self, "_repr_pandas_cache"):
+            del self._repr_pandas_cache
+
+    @property
+    def ndim(self) -> int:
+        """
+        Return an int representing the number of array dimensions.
+
+        return 2 for DataFrame.
+
+        Examples
+        --------
+
+        >>> df = ps.DataFrame([[1, 2], [4, 5], [7, 8]],
+        ...                   index=['cobra', 'viper', None],
+        ...                   columns=['max_speed', 'shield'])
+        >>> df
+               max_speed  shield
+        cobra          1       2
+        viper          4       5
+        NaN            7       8
+        >>> df.ndim
+        2
+        """
+        return 2
+
+    @property
+    def axes(self) -> List:
+        """
+        Return a list representing the axes of the DataFrame.
+
+        It has the row axis labels and column axis labels as the only members.
+        They are returned in that order.
+
+        Examples
+        --------
+
+        >>> df = ps.DataFrame({'col1': [1, 2], 'col2': [3, 4]})
+        >>> df.axes
+        [Int64Index([0, 1], dtype='int64'), Index(['col1', 'col2'], dtype='object')]
+        """
+        return [self.index, self.columns]
+
+    def _reduce_for_stat_function(self, sfun, name, axis=None, numeric_only=True, **kwargs):
+        """
+        Applies sfun to each column and returns a pd.Series where the number of rows equal the
+        number of columns.
+
+        Parameters
+        ----------
+        sfun : either an 1-arg function that takes a Column and returns a Column, or
+            a 2-arg function that takes a Column and its DataType and returns a Column.
+            axis: used only for sanity check because series only support index axis.
+        name : original pandas API name.
+        axis : axis to apply. 0 or 1, or 'index' or 'columns.
+        numeric_only : bool, default True
+            Include only float, int, boolean columns. False is not supported. This parameter
+            is mainly for pandas compatibility. Only 'DataFrame.count' uses this parameter
+            currently.
+        """
+        from inspect import signature
+        from pyspark.pandas.series import Series, first_series
+
+        axis = validate_axis(axis)
+        if axis == 0:
+            min_count = kwargs.get("min_count", 0)
+
+            exprs = [F.lit(None).cast(StringType()).alias(SPARK_DEFAULT_INDEX_NAME)]
+            new_column_labels = []
+            num_args = len(signature(sfun).parameters)
+            for label in self._internal.column_labels:
+                spark_column = self._internal.spark_column_for(label)
+                spark_type = self._internal.spark_type_for(label)
+
+                is_numeric_or_boolean = isinstance(spark_type, (NumericType, BooleanType))
+                keep_column = not numeric_only or is_numeric_or_boolean
+
+                if keep_column:
+                    if num_args == 1:
+                        # Only pass in the column if sfun accepts only one arg
+                        scol = sfun(spark_column)
+                    else:  # must be 2
+                        assert num_args == 2
+                        # Pass in both the column and its data type if sfun accepts two args
+                        scol = sfun(spark_column, spark_type)
+
+                    if min_count > 0:
+                        scol = F.when(
+                            Frame._count_expr(spark_column, spark_type) >= min_count, scol
+                        )
+
+                    exprs.append(scol.alias(name_like_string(label)))
+                    new_column_labels.append(label)
+
+            if len(exprs) == 1:
+                return Series([])
+
+            sdf = self._internal.spark_frame.select(*exprs)
+
+            # The data is expected to be small so it's fine to transpose/use default index.
+            with ps.option_context("compute.max_rows", 1):
+                internal = InternalFrame(
+                    spark_frame=sdf,
+                    index_spark_columns=[scol_for(sdf, SPARK_DEFAULT_INDEX_NAME)],
+                    column_labels=new_column_labels,
+                    column_label_names=self._internal.column_label_names,
+                )
+                return first_series(DataFrame(internal).transpose())
+
+        else:
+            # Here we execute with the first 1000 to get the return type.
+            # If the records were less than 1000, it uses pandas API directly for a shortcut.
+            limit = get_option("compute.shortcut_limit")
+            pdf = self.head(limit + 1)._to_internal_pandas()
+            pser = getattr(pdf, name)(axis=axis, numeric_only=numeric_only, **kwargs)
+            if len(pdf) <= limit:
+                return Series(pser)
+
+            @pandas_udf(returnType=as_spark_type(pser.dtype.type))
+            def calculate_columns_axis(*cols):
+                return getattr(pd.concat(cols, axis=1), name)(
+                    axis=axis, numeric_only=numeric_only, **kwargs
+                )
+
+            column_name = verify_temp_column_name(
+                self._internal.spark_frame.select(self._internal.index_spark_columns),
+                "__calculate_columns_axis__",
+            )
+            sdf = self._internal.spark_frame.select(
+                self._internal.index_spark_columns
+                + [calculate_columns_axis(*self._internal.data_spark_columns).alias(column_name)]
+            )
+            internal = InternalFrame(
+                spark_frame=sdf,
+                index_spark_columns=[
+                    scol_for(sdf, col) for col in self._internal.index_spark_column_names
+                ],
+                index_names=self._internal.index_names,
+                index_dtypes=self._internal.index_dtypes,
+            )
+            return first_series(DataFrame(internal)).rename(pser.name)
+
+    def _kser_for(self, label):
+        """
+        Create Series with a proper column label.
+
+        The given label must be verified to exist in `InternalFrame.column_labels`.
+
+        For example, in some method, self is like:
+
+        >>> self = ps.range(3)
+
+        `self._kser_for(label)` can be used with `InternalFrame.column_labels`:
+
+        >>> self._kser_for(self._internal.column_labels[0])
+        0    0
+        1    1
+        2    2
+        Name: id, dtype: int64
+
+        `self._kser_for(label)` must not be used directly with user inputs.
+        In that case, `self[label]` should be used instead, which checks the label exists or not:
+
+        >>> self['id']
+        0    0
+        1    1
+        2    2
+        Name: id, dtype: int64
+        """
+        return self._ksers[label]
+
+    def _apply_series_op(self, op, should_resolve: bool = False):
+        applied = []
+        for label in self._internal.column_labels:
+            applied.append(op(self._kser_for(label)))
+        internal = self._internal.with_new_columns(applied)
+        if should_resolve:
+            internal = internal.resolved_copy
+        return DataFrame(internal)
+
+    # Arithmetic Operators
+    def _map_series_op(self, op, other):
+        from pyspark.pandas.base import IndexOpsMixin
+
+        if not isinstance(other, DataFrame) and (
+            isinstance(other, IndexOpsMixin) or is_sequence(other)
+        ):
+            raise TypeError(
+                "%s with a sequence is currently not supported; "
+                "however, got %s." % (op, type(other).__name__)
+            )
+
+        if isinstance(other, DataFrame):
+            if self._internal.column_labels_level != other._internal.column_labels_level:
+                raise ValueError("cannot join with no overlapping index names")
+
+            if not same_anchor(self, other):
+                # Different DataFrames
+                def apply_op(kdf, this_column_labels, that_column_labels):
+                    for this_label, that_label in zip(this_column_labels, that_column_labels):
+                        yield (
+                            getattr(kdf._kser_for(this_label), op)(
+                                kdf._kser_for(that_label)
+                            ).rename(this_label),
+                            this_label,
+                        )
+
+                return align_diff_frames(apply_op, self, other, fillna=True, how="full")
+            else:
+                applied = []
+                column_labels = []
+                for label in self._internal.column_labels:
+                    if label in other._internal.column_labels:
+                        applied.append(getattr(self._kser_for(label), op)(other._kser_for(label)))
+                    else:
+                        applied.append(
+                            F.lit(None)
+                            .cast(self._internal.spark_type_for(label))
+                            .alias(name_like_string(label))
+                        )
+                    column_labels.append(label)
+                for label in other._internal.column_labels:
+                    if label not in column_labels:
+                        applied.append(
+                            F.lit(None)
+                            .cast(other._internal.spark_type_for(label))
+                            .alias(name_like_string(label))
+                        )
+                        column_labels.append(label)
+                internal = self._internal.with_new_columns(applied, column_labels=column_labels)
+                return DataFrame(internal)
+        else:
+            return self._apply_series_op(lambda kser: getattr(kser, op)(other))
+
+    def __add__(self, other) -> "DataFrame":
+        return self._map_series_op("add", other)
+
+    def __radd__(self, other) -> "DataFrame":
+        return self._map_series_op("radd", other)
+
+    def __div__(self, other) -> "DataFrame":
+        return self._map_series_op("div", other)
+
+    def __rdiv__(self, other) -> "DataFrame":
+        return self._map_series_op("rdiv", other)
+
+    def __truediv__(self, other) -> "DataFrame":
+        return self._map_series_op("truediv", other)
+
+    def __rtruediv__(self, other) -> "DataFrame":
+        return self._map_series_op("rtruediv", other)
+
+    def __mul__(self, other) -> "DataFrame":
+        return self._map_series_op("mul", other)
+
+    def __rmul__(self, other) -> "DataFrame":
+        return self._map_series_op("rmul", other)
+
+    def __sub__(self, other) -> "DataFrame":
+        return self._map_series_op("sub", other)
+
+    def __rsub__(self, other) -> "DataFrame":
+        return self._map_series_op("rsub", other)
+
+    def __pow__(self, other) -> "DataFrame":
+        return self._map_series_op("pow", other)
+
+    def __rpow__(self, other) -> "DataFrame":
+        return self._map_series_op("rpow", other)
+
+    def __mod__(self, other) -> "DataFrame":
+        return self._map_series_op("mod", other)
+
+    def __rmod__(self, other) -> "DataFrame":
+        return self._map_series_op("rmod", other)
+
+    def __floordiv__(self, other) -> "DataFrame":
+        return self._map_series_op("floordiv", other)
+
+    def __rfloordiv__(self, other) -> "DataFrame":
+        return self._map_series_op("rfloordiv", other)
+
+    def __abs__(self) -> "DataFrame":
+        return self._apply_series_op(lambda kser: abs(kser))
+
+    def __neg__(self) -> "DataFrame":
+        return self._apply_series_op(lambda kser: -kser)
+
+    def add(self, other) -> "DataFrame":
+        return self + other
+
+    # create accessor for plot
+    plot = CachedAccessor("plot", PandasOnSparkPlotAccessor)
+
+    # create accessor for Spark related methods.
+    spark = CachedAccessor("spark", SparkFrameMethods)
+
+    # create accessor for pandas-on-Spark specific methods.
+    koalas = CachedAccessor("koalas", PandasOnSparkFrameMethods)
+
+    def hist(self, bins=10, **kwds):
+        return self.plot.hist(bins, **kwds)
+
+    hist.__doc__ = PandasOnSparkPlotAccessor.hist.__doc__
+
+    def kde(self, bw_method=None, ind=None, **kwds):
+        return self.plot.kde(bw_method, ind, **kwds)
+
+    kde.__doc__ = PandasOnSparkPlotAccessor.kde.__doc__
+
+    add.__doc__ = _flex_doc_FRAME.format(
+        desc="Addition", op_name="+", equiv="dataframe + other", reverse="radd"
+    )
+
+    def radd(self, other) -> "DataFrame":
+        return other + self
+
+    radd.__doc__ = _flex_doc_FRAME.format(
+        desc="Addition", op_name="+", equiv="other + dataframe", reverse="add"
+    )
+
+    def div(self, other) -> "DataFrame":
+        return self / other
+
+    div.__doc__ = _flex_doc_FRAME.format(
+        desc="Floating division", op_name="/", equiv="dataframe / other", reverse="rdiv"
+    )
+
+    divide = div
+
+    def rdiv(self, other) -> "DataFrame":
+        return other / self
+
+    rdiv.__doc__ = _flex_doc_FRAME.format(
+        desc="Floating division", op_name="/", equiv="other / dataframe", reverse="div"
+    )
+
+    def truediv(self, other) -> "DataFrame":
+        return self / other
+
+    truediv.__doc__ = _flex_doc_FRAME.format(
+        desc="Floating division", op_name="/", equiv="dataframe / other", reverse="rtruediv"
+    )
+
+    def rtruediv(self, other) -> "DataFrame":
+        return other / self
+
+    rtruediv.__doc__ = _flex_doc_FRAME.format(
+        desc="Floating division", op_name="/", equiv="other / dataframe", reverse="truediv"
+    )
+
+    def mul(self, other) -> "DataFrame":
+        return self * other
+
+    mul.__doc__ = _flex_doc_FRAME.format(
+        desc="Multiplication", op_name="*", equiv="dataframe * other", reverse="rmul"
+    )
+
+    multiply = mul
+
+    def rmul(self, other) -> "DataFrame":
+        return other * self
+
+    rmul.__doc__ = _flex_doc_FRAME.format(
+        desc="Multiplication", op_name="*", equiv="other * dataframe", reverse="mul"
+    )
+
+    def sub(self, other) -> "DataFrame":
+        return self - other
+
+    sub.__doc__ = _flex_doc_FRAME.format(
+        desc="Subtraction", op_name="-", equiv="dataframe - other", reverse="rsub"
+    )
+
+    subtract = sub
+
+    def rsub(self, other) -> "DataFrame":
+        return other - self
+
+    rsub.__doc__ = _flex_doc_FRAME.format(
+        desc="Subtraction", op_name="-", equiv="other - dataframe", reverse="sub"
+    )
+
+    def mod(self, other) -> "DataFrame":
+        return self % other
+
+    mod.__doc__ = _flex_doc_FRAME.format(
+        desc="Modulo", op_name="%", equiv="dataframe % other", reverse="rmod"
+    )
+
+    def rmod(self, other) -> "DataFrame":
+        return other % self
+
+    rmod.__doc__ = _flex_doc_FRAME.format(
+        desc="Modulo", op_name="%", equiv="other % dataframe", reverse="mod"
+    )
+
+    def pow(self, other) -> "DataFrame":
+        return self ** other
+
+    pow.__doc__ = _flex_doc_FRAME.format(
+        desc="Exponential power of series", op_name="**", equiv="dataframe ** other", reverse="rpow"
+    )
+
+    def rpow(self, other) -> "DataFrame":
+        return other ** self
+
+    rpow.__doc__ = _flex_doc_FRAME.format(
+        desc="Exponential power", op_name="**", equiv="other ** dataframe", reverse="pow"
+    )
+
+    def floordiv(self, other) -> "DataFrame":
+        return self // other
+
+    floordiv.__doc__ = _flex_doc_FRAME.format(
+        desc="Integer division", op_name="//", equiv="dataframe // other", reverse="rfloordiv"
+    )
+
+    def rfloordiv(self, other) -> "DataFrame":
+        return other // self
+
+    rfloordiv.__doc__ = _flex_doc_FRAME.format(
+        desc="Integer division", op_name="//", equiv="other // dataframe", reverse="floordiv"
+    )
+
+    # Comparison Operators
+    def __eq__(self, other) -> "DataFrame":  # type: ignore
+        return self._map_series_op("eq", other)
+
+    def __ne__(self, other) -> "DataFrame":  # type: ignore
+        return self._map_series_op("ne", other)
+
+    def __lt__(self, other) -> "DataFrame":
+        return self._map_series_op("lt", other)
+
+    def __le__(self, other) -> "DataFrame":
+        return self._map_series_op("le", other)
+
+    def __ge__(self, other) -> "DataFrame":
+        return self._map_series_op("ge", other)
+
+    def __gt__(self, other) -> "DataFrame":
+        return self._map_series_op("gt", other)
+
+    def eq(self, other) -> "DataFrame":
+        """
+        Compare if the current value is equal to the other.
+
+        >>> df = ps.DataFrame({'a': [1, 2, 3, 4],
+        ...                    'b': [1, np.nan, 1, np.nan]},
+        ...                   index=['a', 'b', 'c', 'd'], columns=['a', 'b'])
+
+        >>> df.eq(1)
+               a      b
+        a   True   True
+        b  False  False
+        c  False   True
+        d  False  False
+        """
+        return self == other
+
+    equals = eq
+
+    def gt(self, other) -> "DataFrame":
+        """
+        Compare if the current value is greater than the other.
+
+        >>> df = ps.DataFrame({'a': [1, 2, 3, 4],
+        ...                    'b': [1, np.nan, 1, np.nan]},
+        ...                   index=['a', 'b', 'c', 'd'], columns=['a', 'b'])
+
+        >>> df.gt(2)
+               a      b
+        a  False  False
+        b  False  False
+        c   True  False
+        d   True  False
+        """
+        return self > other
+
+    def ge(self, other) -> "DataFrame":
+        """
+        Compare if the current value is greater than or equal to the other.
+
+        >>> df = ps.DataFrame({'a': [1, 2, 3, 4],
+        ...                    'b': [1, np.nan, 1, np.nan]},
+        ...                   index=['a', 'b', 'c', 'd'], columns=['a', 'b'])
+
+        >>> df.ge(1)
+              a      b
+        a  True   True
+        b  True  False
+        c  True   True
+        d  True  False
+        """
+        return self >= other
+
+    def lt(self, other) -> "DataFrame":
+        """
+        Compare if the current value is less than the other.
+
+        >>> df = ps.DataFrame({'a': [1, 2, 3, 4],
+        ...                    'b': [1, np.nan, 1, np.nan]},
+        ...                   index=['a', 'b', 'c', 'd'], columns=['a', 'b'])
+
+        >>> df.lt(1)
+               a      b
+        a  False  False
+        b  False  False
+        c  False  False
+        d  False  False
+        """
+        return self < other
+
+    def le(self, other) -> "DataFrame":
+        """
+        Compare if the current value is less than or equal to the other.
+
+        >>> df = ps.DataFrame({'a': [1, 2, 3, 4],
+        ...                    'b': [1, np.nan, 1, np.nan]},
+        ...                   index=['a', 'b', 'c', 'd'], columns=['a', 'b'])
+
+        >>> df.le(2)
+               a      b
+        a   True   True
+        b   True  False
+        c  False   True
+        d  False  False
+        """
+        return self <= other
+
+    def ne(self, other) -> "DataFrame":
+        """
+        Compare if the current value is not equal to the other.
+
+        >>> df = ps.DataFrame({'a': [1, 2, 3, 4],
+        ...                    'b': [1, np.nan, 1, np.nan]},
+        ...                   index=['a', 'b', 'c', 'd'], columns=['a', 'b'])
+
+        >>> df.ne(1)
+               a      b
+        a  False  False
+        b   True   True
+        c   True  False
+        d   True   True
+        """
+        return self != other
+
+    def applymap(self, func) -> "DataFrame":
+        """
+        Apply a function to a Dataframe elementwise.
+
+        This method applies a function that accepts and returns a scalar
+        to every element of a DataFrame.
+
+        .. note:: this API executes the function once to infer the type which is
+             potentially expensive, for instance, when the dataset is created after
+             aggregations or sorting.
+
+             To avoid this, specify return type in ``func``, for instance, as below:
+
+             >>> def square(x) -> np.int32:
+             ...     return x ** 2
+
+             pandas-on-Spark uses return type hint and does not try to infer the type.
+
+        Parameters
+        ----------
+        func : callable
+            Python function, returns a single value from a single value.
+
+        Returns
+        -------
+        DataFrame
+            Transformed DataFrame.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([[1, 2.12], [3.356, 4.567]])
+        >>> df
+               0      1
+        0  1.000  2.120
+        1  3.356  4.567
+
+        >>> def str_len(x) -> int:
+        ...     return len(str(x))
+        >>> df.applymap(str_len)
+           0  1
+        0  3  4
+        1  5  5
+
+        >>> def power(x) -> float:
+        ...     return x ** 2
+        >>> df.applymap(power)
+                   0          1
+        0   1.000000   4.494400
+        1  11.262736  20.857489
+
+        You can omit the type hint and let pandas-on-Spark infer its type.
+
+        >>> df.applymap(lambda x: x ** 2)
+                   0          1
+        0   1.000000   4.494400
+        1  11.262736  20.857489
+        """
+
+        # TODO: We can implement shortcut theoretically since it creates new DataFrame
+        #  anyway and we don't have to worry about operations on different DataFrames.
+        return self._apply_series_op(lambda kser: kser.apply(func))
+
+    # TODO: not all arguments are implemented comparing to pandas' for now.
+    def aggregate(
+        self, func: Union[List[str], Dict[Any, List[str]]]
+    ) -> Union["Series", "DataFrame", "Index"]:
+        """Aggregate using one or more operations over the specified axis.
+
+        Parameters
+        ----------
+        func : dict or a list
+             a dict mapping from column name (string) to
+             aggregate functions (list of strings).
+             If a list is given, the aggregation is performed against
+             all columns.
+
+        Returns
+        -------
+        DataFrame
+
+        Notes
+        -----
+        `agg` is an alias for `aggregate`. Use the alias.
+
+        See Also
+        --------
+        DataFrame.apply : Invoke function on DataFrame.
+        DataFrame.transform : Only perform transforming type operations.
+        DataFrame.groupby : Perform operations over groups.
+        Series.aggregate : The equivalent function for Series.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([[1, 2, 3],
+        ...                    [4, 5, 6],
+        ...                    [7, 8, 9],
+        ...                    [np.nan, np.nan, np.nan]],
+        ...                   columns=['A', 'B', 'C'])
+
+        >>> df
+             A    B    C
+        0  1.0  2.0  3.0
+        1  4.0  5.0  6.0
+        2  7.0  8.0  9.0
+        3  NaN  NaN  NaN
+
+        Aggregate these functions over the rows.
+
+        >>> df.agg(['sum', 'min'])[['A', 'B', 'C']].sort_index()
+                A     B     C
+        min   1.0   2.0   3.0
+        sum  12.0  15.0  18.0
+
+        Different aggregations per column.
+
+        >>> df.agg({'A' : ['sum', 'min'], 'B' : ['min', 'max']})[['A', 'B']].sort_index()
+                A    B
+        max   NaN  8.0
+        min   1.0  2.0
+        sum  12.0  NaN
+
+        For multi-index columns:
+
+        >>> df.columns = pd.MultiIndex.from_tuples([("X", "A"), ("X", "B"), ("Y", "C")])
+        >>> df.agg(['sum', 'min'])[[("X", "A"), ("X", "B"), ("Y", "C")]].sort_index()
+                X           Y
+                A     B     C
+        min   1.0   2.0   3.0
+        sum  12.0  15.0  18.0
+
+        >>> aggregated = df.agg({("X", "A") : ['sum', 'min'], ("X", "B") : ['min', 'max']})
+        >>> aggregated[[("X", "A"), ("X", "B")]].sort_index()  # doctest: +NORMALIZE_WHITESPACE
+                X
+                A    B
+        max   NaN  8.0
+        min   1.0  2.0
+        sum  12.0  NaN
+        """
+        from pyspark.pandas.groupby import GroupBy
+
+        if isinstance(func, list):
+            if all((isinstance(f, str) for f in func)):
+                func = dict([(column, func) for column in self.columns])
+            else:
+                raise ValueError(
+                    "If the given function is a list, it "
+                    "should only contains function names as strings."
+                )
+
+        if not isinstance(func, dict) or not all(
+            is_name_like_value(key)
+            and (
+                isinstance(value, str)
+                or (isinstance(value, list) and all(isinstance(v, str) for v in value))
+            )
+            for key, value in func.items()
+        ):
+            raise ValueError(
+                "aggs must be a dict mapping from column name to aggregate "
+                "functions (string or list of strings)."
+            )
+
+        with option_context("compute.default_index_type", "distributed"):
+            kdf = DataFrame(GroupBy._spark_groupby(self, func))  # type: DataFrame
+
+            # The codes below basically converts:
+            #
+            #           A         B
+            #         sum  min  min  max
+            #     0  12.0  1.0  2.0  8.0
+            #
+            # to:
+            #             A    B
+            #     max   NaN  8.0
+            #     min   1.0  2.0
+            #     sum  12.0  NaN
+            #
+            # Aggregated output is usually pretty much small.
+
+            return kdf.stack().droplevel(0)[list(func.keys())]
+
+    agg = aggregate
+
+    def corr(self, method="pearson") -> Union["Series", "DataFrame", "Index"]:
+        """
+        Compute pairwise correlation of columns, excluding NA/null values.
+
+        Parameters
+        ----------
+        method : {'pearson', 'spearman'}
+            * pearson : standard correlation coefficient
+            * spearman : Spearman rank correlation
+
+        Returns
+        -------
+        y : DataFrame
+
+        See Also
+        --------
+        Series.corr
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([(.2, .3), (.0, .6), (.6, .0), (.2, .1)],
+        ...                   columns=['dogs', 'cats'])
+        >>> df.corr('pearson')
+                  dogs      cats
+        dogs  1.000000 -0.851064
+        cats -0.851064  1.000000
+
+        >>> df.corr('spearman')
+                  dogs      cats
+        dogs  1.000000 -0.948683
+        cats -0.948683  1.000000
+
+        Notes
+        -----
+        There are behavior differences between pandas-on-Spark and pandas.
+
+        * the `method` argument only accepts 'pearson', 'spearman'
+        * the data should not contain NaNs. pandas-on-Spark will return an error.
+        * pandas-on-Spark doesn't support the following argument(s).
+
+          * `min_periods` argument is not supported
+        """
+        return ps.from_pandas(corr(self, method))
+
+    def iteritems(self) -> Iterator:
+        """
+        Iterator over (column name, Series) pairs.
+
+        Iterates over the DataFrame columns, returning a tuple with
+        the column name and the content as a Series.
+
+        Returns
+        -------
+        label : object
+            The column names for the DataFrame being iterated over.
+        content : Series
+            The column entries belonging to each label, as a Series.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'species': ['bear', 'bear', 'marsupial'],
+        ...                    'population': [1864, 22000, 80000]},
+        ...                   index=['panda', 'polar', 'koala'],
+        ...                   columns=['species', 'population'])
+        >>> df
+                 species  population
+        panda       bear        1864
+        polar       bear       22000
+        koala  marsupial       80000
+
+        >>> for label, content in df.iteritems():
+        ...    print('label:', label)
+        ...    print('content:', content.to_string())
+        ...
+        label: species
+        content: panda         bear
+        polar         bear
+        koala    marsupial
+        label: population
+        content: panda     1864
+        polar    22000
+        koala    80000
+        """
+        return (
+            (label if len(label) > 1 else label[0], self._kser_for(label))
+            for label in self._internal.column_labels
+        )
+
+    def iterrows(self) -> Iterator:
+        """
+        Iterate over DataFrame rows as (index, Series) pairs.
+
+        Yields
+        ------
+        index : label or tuple of label
+            The index of the row. A tuple for a `MultiIndex`.
+        data : pandas.Series
+            The data of the row as a Series.
+
+        it : generator
+            A generator that iterates over the rows of the frame.
+
+        Notes
+        -----
+
+        1. Because ``iterrows`` returns a Series for each row,
+           it does **not** preserve dtypes across the rows (dtypes are
+           preserved across columns for DataFrames). For example,
+
+           >>> df = ps.DataFrame([[1, 1.5]], columns=['int', 'float'])
+           >>> row = next(df.iterrows())[1]
+           >>> row
+           int      1.0
+           float    1.5
+           Name: 0, dtype: float64
+           >>> print(row['int'].dtype)
+           float64
+           >>> print(df['int'].dtype)
+           int64
+
+           To preserve dtypes while iterating over the rows, it is better
+           to use :meth:`itertuples` which returns namedtuples of the values
+           and which is generally faster than ``iterrows``.
+
+        2. You should **never modify** something you are iterating over.
+           This is not guaranteed to work in all cases. Depending on the
+           data types, the iterator returns a copy and not a view, and writing
+           to it will have no effect.
+        """
+
+        columns = self.columns
+        internal_index_columns = self._internal.index_spark_column_names
+        internal_data_columns = self._internal.data_spark_column_names
+
+        def extract_kv_from_spark_row(row):
+            k = (
+                row[internal_index_columns[0]]
+                if len(internal_index_columns) == 1
+                else tuple(row[c] for c in internal_index_columns)
+            )
+            v = [row[c] for c in internal_data_columns]
+            return k, v
+
+        for k, v in map(
+            extract_kv_from_spark_row, self._internal.resolved_copy.spark_frame.toLocalIterator()
+        ):
+            s = pd.Series(v, index=columns, name=k)
+            yield k, s
+
+    def itertuples(self, index: bool = True, name: Optional[str] = "PandasOnSpark") -> Iterator:
+        """
+        Iterate over DataFrame rows as namedtuples.
+
+        Parameters
+        ----------
+        index : bool, default True
+            If True, return the index as the first element of the tuple.
+        name : str or None, default "PandasOnSpark"
+            The name of the returned namedtuples or None to return regular
+            tuples.
+
+        Returns
+        -------
+        iterator
+            An object to iterate over namedtuples for each row in the
+            DataFrame with the first field possibly being the index and
+            following fields being the column values.
+
+        See Also
+        --------
+        DataFrame.iterrows : Iterate over DataFrame rows as (index, Series)
+            pairs.
+        DataFrame.items : Iterate over (column name, Series) pairs.
+
+        Notes
+        -----
+        The column names will be renamed to positional names if they are
+        invalid Python identifiers, repeated, or start with an underscore.
+        On python versions < 3.7 regular tuples are returned for DataFrames
+        with a large number of columns (>254).
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'num_legs': [4, 2], 'num_wings': [0, 2]},
+        ...                   index=['dog', 'hawk'])
+        >>> df
+              num_legs  num_wings
+        dog          4          0
+        hawk         2          2
+
+        >>> for row in df.itertuples():
+        ...     print(row)
+        ...
+        PandasOnSpark(Index='dog', num_legs=4, num_wings=0)
+        PandasOnSpark(Index='hawk', num_legs=2, num_wings=2)
+
+        By setting the `index` parameter to False we can remove the index
+        as the first element of the tuple:
+
+        >>> for row in df.itertuples(index=False):
+        ...     print(row)
+        ...
+        PandasOnSpark(num_legs=4, num_wings=0)
+        PandasOnSpark(num_legs=2, num_wings=2)
+
+        With the `name` parameter set we set a custom name for the yielded
+        namedtuples:
+
+        >>> for row in df.itertuples(name='Animal'):
+        ...     print(row)
+        ...
+        Animal(Index='dog', num_legs=4, num_wings=0)
+        Animal(Index='hawk', num_legs=2, num_wings=2)
+        """
+        fields = list(self.columns)
+        if index:
+            fields.insert(0, "Index")
+
+        index_spark_column_names = self._internal.index_spark_column_names
+        data_spark_column_names = self._internal.data_spark_column_names
+
+        def extract_kv_from_spark_row(row):
+            k = (
+                row[index_spark_column_names[0]]
+                if len(index_spark_column_names) == 1
+                else tuple(row[c] for c in index_spark_column_names)
+            )
+            v = [row[c] for c in data_spark_column_names]
+            return k, v
+
+        can_return_named_tuples = sys.version_info >= (3, 7) or len(self.columns) + index < 255
+
+        if name is not None and can_return_named_tuples:
+            itertuple = namedtuple(name, fields, rename=True)  # type: ignore
+            for k, v in map(
+                extract_kv_from_spark_row,
+                self._internal.resolved_copy.spark_frame.toLocalIterator(),
+            ):
+                yield itertuple._make(([k] if index else []) + list(v))
+        else:
+            for k, v in map(
+                extract_kv_from_spark_row,
+                self._internal.resolved_copy.spark_frame.toLocalIterator(),
+            ):
+                yield tuple(([k] if index else []) + list(v))
+
+    def items(self) -> Iterator:
+        """This is an alias of ``iteritems``."""
+        return self.iteritems()
+
+    def to_clipboard(self, excel=True, sep=None, **kwargs) -> None:
+        """
+        Copy object to the system clipboard.
+
+        Write a text representation of object to the system clipboard.
+        This can be pasted into Excel, for example.
+
+        .. note:: This method should only be used if the resulting DataFrame is expected
+            to be small, as all the data is loaded into the driver's memory.
+
+        Parameters
+        ----------
+        excel : bool, default True
+            - True, use the provided separator, writing in a csv format for
+              allowing easy pasting into excel.
+            - False, write a string representation of the object to the
+              clipboard.
+
+        sep : str, default ``'\\t'``
+            Field delimiter.
+        **kwargs
+            These parameters will be passed to DataFrame.to_csv.
+
+        Notes
+        -----
+        Requirements for your platform.
+
+          - Linux : `xclip`, or `xsel` (with `gtk` or `PyQt4` modules)
+          - Windows : none
+          - OS X : none
+
+        See Also
+        --------
+        read_clipboard : Read text from clipboard.
+
+        Examples
+        --------
+        Copy the contents of a DataFrame to the clipboard.
+
+        >>> df = ps.DataFrame([[1, 2, 3], [4, 5, 6]], columns=['A', 'B', 'C'])  # doctest: +SKIP
+        >>> df.to_clipboard(sep=',')  # doctest: +SKIP
+        ... # Wrote the following to the system clipboard:
+        ... # ,A,B,C
+        ... # 0,1,2,3
+        ... # 1,4,5,6
+
+        We can omit the index by passing the keyword `index` and setting
+        it to false.
+
+        >>> df.to_clipboard(sep=',', index=False)  # doctest: +SKIP
+        ... # Wrote the following to the system clipboard:
+        ... # A,B,C
+        ... # 1,2,3
+        ... # 4,5,6
+
+        This function also works for Series:
+
+        >>> df = ps.Series([1, 2, 3, 4, 5, 6, 7], name='x')  # doctest: +SKIP
+        >>> df.to_clipboard(sep=',')  # doctest: +SKIP
+        ... # Wrote the following to the system clipboard:
+        ... # 0, 1
+        ... # 1, 2
+        ... # 2, 3
+        ... # 3, 4
+        ... # 4, 5
+        ... # 5, 6
+        ... # 6, 7
+        """
+
+        args = locals()
+        kdf = self
+        return validate_arguments_and_invoke_function(
+            kdf._to_internal_pandas(), self.to_clipboard, pd.DataFrame.to_clipboard, args
+        )
+
+    def to_html(
+        self,
+        buf=None,
+        columns=None,
+        col_space=None,
+        header=True,
+        index=True,
+        na_rep="NaN",
+        formatters=None,
+        float_format=None,
+        sparsify=None,
+        index_names=True,
+        justify=None,
+        max_rows=None,
+        max_cols=None,
+        show_dimensions=False,
+        decimal=".",
+        bold_rows=True,
+        classes=None,
+        escape=True,
+        notebook=False,
+        border=None,
+        table_id=None,
+        render_links=False,
+    ) -> Optional[str]:
+        """
+        Render a DataFrame as an HTML table.
+
+        .. note:: This method should only be used if the resulting pandas object is expected
+                  to be small, as all the data is loaded into the driver's memory. If the input
+                  is large, set max_rows parameter.
+
+        Parameters
+        ----------
+        buf : StringIO-like, optional
+            Buffer to write to.
+        columns : sequence, optional, default None
+            The subset of columns to write. Writes all columns by default.
+        col_space : int, optional
+            The minimum width of each column.
+        header : bool, optional
+            Write out the column names. If a list of strings is given, it
+            is assumed to be aliases for the column names
+        index : bool, optional, default True
+            Whether to print index (row) labels.
+        na_rep : str, optional, default 'NaN'
+            String representation of NAN to use.
+        formatters : list or dict of one-param. functions, optional
+            Formatter functions to apply to columns' elements by position or
+            name.
+            The result of each function must be a unicode string.
+            List must be of length equal to the number of columns.
+        float_format : one-parameter function, optional, default None
+            Formatter function to apply to columns' elements if they are
+            floats. The result of this function must be a unicode string.
+        sparsify : bool, optional, default True
+            Set to False for a DataFrame with a hierarchical index to print
+            every multiindex key at each row.
+        index_names : bool, optional, default True
+            Prints the names of the indexes.
+        justify : str, default None
+            How to justify the column labels. If None uses the option from
+            the print configuration (controlled by set_option), 'right' out
+            of the box. Valid values are
+
+            * left
+            * right
+            * center
+            * justify
+            * justify-all
+            * start
+            * end
+            * inherit
+            * match-parent
+            * initial
+            * unset.
+        max_rows : int, optional
+            Maximum number of rows to display in the console.
+        max_cols : int, optional
+            Maximum number of columns to display in the console.
+        show_dimensions : bool, default False
+            Display DataFrame dimensions (number of rows by number of columns).
+        decimal : str, default '.'
+            Character recognized as decimal separator, e.g. ',' in Europe.
+        bold_rows : bool, default True
+            Make the row labels bold in the output.
+        classes : str or list or tuple, default None
+            CSS class(es) to apply to the resulting html table.
+        escape : bool, default True
+            Convert the characters <, >, and & to HTML-safe sequences.
+        notebook : {True, False}, default False
+            Whether the generated HTML is for IPython Notebook.
+        border : int
+            A ``border=border`` attribute is included in the opening
+            `<table>` tag. Default ``pd.options.html.border``.
+        table_id : str, optional
+            A css id is included in the opening `<table>` tag if specified.
+        render_links : bool, default False
+            Convert URLs to HTML links (only works with pandas 0.24+).
+
+        Returns
+        -------
+        str (or unicode, depending on data and options)
+            String representation of the dataframe.
+
+        See Also
+        --------
+        to_string : Convert DataFrame to a string.
+        """
+        # Make sure locals() call is at the top of the function so we don't capture local variables.
+        args = locals()
+        if max_rows is not None:
+            kdf = self.head(max_rows)
+        else:
+            kdf = self
+
+        return validate_arguments_and_invoke_function(
+            kdf._to_internal_pandas(), self.to_html, pd.DataFrame.to_html, args
+        )
+
+    def to_string(
+        self,
+        buf=None,
+        columns=None,
+        col_space=None,
+        header=True,
+        index=True,
+        na_rep="NaN",
+        formatters=None,
+        float_format=None,
+        sparsify=None,
+        index_names=True,
+        justify=None,
+        max_rows=None,
+        max_cols=None,
+        show_dimensions=False,
+        decimal=".",
+        line_width=None,
+    ) -> Optional[str]:
+        """
+        Render a DataFrame to a console-friendly tabular output.
+
+        .. note:: This method should only be used if the resulting pandas object is expected
+                  to be small, as all the data is loaded into the driver's memory. If the input
+                  is large, set max_rows parameter.
+
+        Parameters
+        ----------
+        buf : StringIO-like, optional
+            Buffer to write to.
+        columns : sequence, optional, default None
+            The subset of columns to write. Writes all columns by default.
+        col_space : int, optional
+            The minimum width of each column.
+        header : bool, optional
+            Write out the column names. If a list of strings is given, it
+            is assumed to be aliases for the column names
+        index : bool, optional, default True
+            Whether to print index (row) labels.
+        na_rep : str, optional, default 'NaN'
+            String representation of NAN to use.
+        formatters : list or dict of one-param. functions, optional
+            Formatter functions to apply to columns' elements by position or
+            name.
+            The result of each function must be a unicode string.
+            List must be of length equal to the number of columns.
+        float_format : one-parameter function, optional, default None
+            Formatter function to apply to columns' elements if they are
+            floats. The result of this function must be a unicode string.
+        sparsify : bool, optional, default True
+            Set to False for a DataFrame with a hierarchical index to print
+            every multiindex key at each row.
+        index_names : bool, optional, default True
+            Prints the names of the indexes.
+        justify : str, default None
+            How to justify the column labels. If None uses the option from
+            the print configuration (controlled by set_option), 'right' out
+            of the box. Valid values are
+
+            * left
+            * right
+            * center
+            * justify
+            * justify-all
+            * start
+            * end
+            * inherit
+            * match-parent
+            * initial
+            * unset.
+        max_rows : int, optional
+            Maximum number of rows to display in the console.
+        max_cols : int, optional
+            Maximum number of columns to display in the console.
+        show_dimensions : bool, default False
+            Display DataFrame dimensions (number of rows by number of columns).
+        decimal : str, default '.'
+            Character recognized as decimal separator, e.g. ',' in Europe.
+        line_width : int, optional
+            Width to wrap a line in characters.
+
+        Returns
+        -------
+        str (or unicode, depending on data and options)
+            String representation of the dataframe.
+
+        See Also
+        --------
+        to_html : Convert DataFrame to HTML.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'col1': [1, 2, 3], 'col2': [4, 5, 6]}, columns=['col1', 'col2'])
+        >>> print(df.to_string())
+           col1  col2
+        0     1     4
+        1     2     5
+        2     3     6
+
+        >>> print(df.to_string(max_rows=2))
+           col1  col2
+        0     1     4
+        1     2     5
+        """
+        # Make sure locals() call is at the top of the function so we don't capture local variables.
+        args = locals()
+        if max_rows is not None:
+            kdf = self.head(max_rows)
+        else:
+            kdf = self
+
+        return validate_arguments_and_invoke_function(
+            kdf._to_internal_pandas(), self.to_string, pd.DataFrame.to_string, args
+        )
+
+    def to_dict(self, orient="dict", into=dict) -> Union[List, Mapping]:
+        """
+        Convert the DataFrame to a dictionary.
+
+        The type of the key-value pairs can be customized with the parameters
+        (see below).
+
+        .. note:: This method should only be used if the resulting pandas DataFrame is expected
+            to be small, as all the data is loaded into the driver's memory.
+
+        Parameters
+        ----------
+        orient : str {'dict', 'list', 'series', 'split', 'records', 'index'}
+            Determines the type of the values of the dictionary.
+
+            - 'dict' (default) : dict like {column -> {index -> value}}
+            - 'list' : dict like {column -> [values]}
+            - 'series' : dict like {column -> Series(values)}
+            - 'split' : dict like
+              {'index' -> [index], 'columns' -> [columns], 'data' -> [values]}
+            - 'records' : list like
+              [{column -> value}, ... , {column -> value}]
+            - 'index' : dict like {index -> {column -> value}}
+
+            Abbreviations are allowed. `s` indicates `series` and `sp`
+            indicates `split`.
+
+        into : class, default dict
+            The collections.abc.Mapping subclass used for all Mappings
+            in the return value.  Can be the actual class or an empty
+            instance of the mapping type you want.  If you want a
+            collections.defaultdict, you must pass it initialized.
+
+        Returns
+        -------
+        dict, list or collections.abc.Mapping
+            Return a collections.abc.Mapping object representing the DataFrame.
+            The resulting transformation depends on the `orient` parameter.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'col1': [1, 2],
+        ...                    'col2': [0.5, 0.75]},
+        ...                   index=['row1', 'row2'],
+        ...                   columns=['col1', 'col2'])
+        >>> df
+              col1  col2
+        row1     1  0.50
+        row2     2  0.75
+
+        >>> df_dict = df.to_dict()
+        >>> sorted([(key, sorted(values.items())) for key, values in df_dict.items()])
+        [('col1', [('row1', 1), ('row2', 2)]), ('col2', [('row1', 0.5), ('row2', 0.75)])]
+
+        You can specify the return orientation.
+
+        >>> df_dict = df.to_dict('series')
+        >>> sorted(df_dict.items())
+        [('col1', row1    1
+        row2    2
+        Name: col1, dtype: int64), ('col2', row1    0.50
+        row2    0.75
+        Name: col2, dtype: float64)]
+
+        >>> df_dict = df.to_dict('split')
+        >>> sorted(df_dict.items())  # doctest: +ELLIPSIS
+        [('columns', ['col1', 'col2']), ('data', [[1..., 0.75]]), ('index', ['row1', 'row2'])]
+
+        >>> df_dict = df.to_dict('records')
+        >>> [sorted(values.items()) for values in df_dict]  # doctest: +ELLIPSIS
+        [[('col1', 1...), ('col2', 0.5)], [('col1', 2...), ('col2', 0.75)]]
+
+        >>> df_dict = df.to_dict('index')
+        >>> sorted([(key, sorted(values.items())) for key, values in df_dict.items()])
+        [('row1', [('col1', 1), ('col2', 0.5)]), ('row2', [('col1', 2), ('col2', 0.75)])]
+
+        You can also specify the mapping type.
+
+        >>> from collections import OrderedDict, defaultdict
+        >>> df.to_dict(into=OrderedDict)
+        OrderedDict([('col1', OrderedDict([('row1', 1), ('row2', 2)])), \
+('col2', OrderedDict([('row1', 0.5), ('row2', 0.75)]))])
+
+        If you want a `defaultdict`, you need to initialize it:
+
+        >>> dd = defaultdict(list)
+        >>> df.to_dict('records', into=dd)  # doctest: +ELLIPSIS
+        [defaultdict(<class 'list'>, {'col..., 'col...}), \
+defaultdict(<class 'list'>, {'col..., 'col...})]
+        """
+        # Make sure locals() call is at the top of the function so we don't capture local variables.
+        args = locals()
+        kdf = self
+        return validate_arguments_and_invoke_function(
+            kdf._to_internal_pandas(), self.to_dict, pd.DataFrame.to_dict, args
+        )
+
+    def to_latex(
+        self,
+        buf=None,
+        columns=None,
+        col_space=None,
+        header=True,
+        index=True,
+        na_rep="NaN",
+        formatters=None,
+        float_format=None,
+        sparsify=None,
+        index_names=True,
+        bold_rows=False,
+        column_format=None,
+        longtable=None,
+        escape=None,
+        encoding=None,
+        decimal=".",
+        multicolumn=None,
+        multicolumn_format=None,
+        multirow=None,
+    ) -> Optional[str]:
+        r"""
+        Render an object to a LaTeX tabular environment table.
+
+        Render an object to a tabular environment table. You can splice this into a LaTeX
+        document. Requires usepackage{booktabs}.
+
+        .. note:: This method should only be used if the resulting pandas object is expected
+                  to be small, as all the data is loaded into the driver's memory. If the input
+                  is large, consider alternative formats.
+
+        Parameters
+        ----------
+        buf : file descriptor or None
+            Buffer to write to. If None, the output is returned as a string.
+        columns : list of label, optional
+            The subset of columns to write. Writes all columns by default.
+        col_space : int, optional
+            The minimum width of each column.
+        header : bool or list of str, default True
+            Write out the column names. If a list of strings is given, it is assumed to be aliases
+            for the column names.
+        index : bool, default True
+            Write row names (index).
+        na_rep : str, default ‘NaN’
+            Missing data representation.
+        formatters : list of functions or dict of {str: function}, optional
+            Formatter functions to apply to columns’ elements by position or name. The result of
+            each function must be a unicode string. List must be of length equal to the number of
+            columns.
+        float_format : str, optional
+            Format string for floating point numbers.
+        sparsify : bool, optional
+            Set to False for a DataFrame with a hierarchical index to print every multiindex key at
+            each row. By default, the value will be read from the config module.
+        index_names : bool, default True
+            Prints the names of the indexes.
+        bold_rows : bool, default False
+            Make the row labels bold in the output.
+        column_format : str, optional
+            The columns format as specified in LaTeX table format e.g. ‘rcl’ for 3 columns. By
+            default, ‘l’ will be used for all columns except columns of numbers, which default
+            to ‘r’.
+        longtable : bool, optional
+            By default, the value will be read from the pandas config module. Use a longtable
+            environment instead of tabular. Requires adding a usepackage{longtable} to your LaTeX
+            preamble.
+        escape : bool, optional
+            By default, the value will be read from the pandas config module. When set to False
+            prevents from escaping latex special characters in column names.
+        encoding : str, optional
+            A string representing the encoding to use in the output file, defaults to ‘ascii’ on
+            Python 2 and ‘utf-8’ on Python 3.
+        decimal : str, default ‘.’
+            Character recognized as decimal separator, e.g. ‘,’ in Europe.
+        multicolumn : bool, default True
+            Use multicolumn to enhance MultiIndex columns. The default will be read from the config
+            module.
+        multicolumn_format : str, default ‘l’
+            The alignment for multicolumns, similar to column_format The default will be read from
+            the config module.
+        multirow : bool, default False
+            Use multirow to enhance MultiIndex rows. Requires adding a usepackage{multirow} to your
+            LaTeX preamble. Will print centered labels (instead of top-aligned) across the contained
+            rows, separating groups via clines. The default will be read from the pandas config
+            module.
+
+        Returns
+        -------
+        str or None
+            If buf is None, returns the resulting LateX format as a string. Otherwise returns None.
+
+        See Also
+        --------
+        DataFrame.to_string : Render a DataFrame to a console-friendly
+            tabular output.
+        DataFrame.to_html : Render a DataFrame as an HTML table.
+
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'name': ['Raphael', 'Donatello'],
+        ...                    'mask': ['red', 'purple'],
+        ...                    'weapon': ['sai', 'bo staff']},
+        ...                   columns=['name', 'mask', 'weapon'])
+        >>> print(df.to_latex(index=False)) # doctest: +NORMALIZE_WHITESPACE
+        \begin{tabular}{lll}
+        \toprule
+              name &    mask &    weapon \\
+        \midrule
+           Raphael &     red &       sai \\
+         Donatello &  purple &  bo staff \\
+        \bottomrule
+        \end{tabular}
+        <BLANKLINE>
+        """
+
+        args = locals()
+        kdf = self
+        return validate_arguments_and_invoke_function(
+            kdf._to_internal_pandas(), self.to_latex, pd.DataFrame.to_latex, args
+        )
+
+    # TODO: enable doctests once we drop Spark 2.3.x (due to type coercion logic
+    #  when creating arrays)
+    def transpose(self) -> "DataFrame":
+        """
+        Transpose index and columns.
+
+        Reflect the DataFrame over its main diagonal by writing rows as columns
+        and vice-versa. The property :attr:`.T` is an accessor to the method
+        :meth:`transpose`.
+
+        .. note:: This method is based on an expensive operation due to the nature
+            of big data. Internally it needs to generate each row for each value, and
+            then group twice - it is a huge operation. To prevent misusage, this method
+            has the 'compute.max_rows' default limit of input length, and raises a ValueError.
+
+                >>> from pyspark.pandas.config import option_context
+                >>> with option_context('compute.max_rows', 1000):  # doctest: +NORMALIZE_WHITESPACE
+                ...     ps.DataFrame({'a': range(1001)}).transpose()
+                Traceback (most recent call last):
+                  ...
+                ValueError: Current DataFrame has more then the given limit 1000 rows.
+                Please set 'compute.max_rows' by using 'pyspark.pandas.config.set_option'
+                to retrieve to retrieve more than 1000 rows. Note that, before changing the
+                'compute.max_rows', this operation is considerably expensive.
+
+        Returns
+        -------
+        DataFrame
+            The transposed DataFrame.
+
+        Notes
+        -----
+        Transposing a DataFrame with mixed dtypes will result in a homogeneous
+        DataFrame with the coerced dtype. For instance, if int and float have
+        to be placed in same column, it becomes float. If type coercion is not
+        possible, it fails.
+
+        Also, note that the values in index should be unique because they become
+        unique column names.
+
+        In addition, if Spark 2.3 is used, the types should always be exactly same.
+
+        Examples
+        --------
+        **Square DataFrame with homogeneous dtype**
+
+        >>> d1 = {'col1': [1, 2], 'col2': [3, 4]}
+        >>> df1 = ps.DataFrame(data=d1, columns=['col1', 'col2'])
+        >>> df1
+           col1  col2
+        0     1     3
+        1     2     4
+
+        >>> df1_transposed = df1.T.sort_index()  # doctest: +SKIP
+        >>> df1_transposed  # doctest: +SKIP
+              0  1
+        col1  1  2
+        col2  3  4
+
+        When the dtype is homogeneous in the original DataFrame, we get a
+        transposed DataFrame with the same dtype:
+
+        >>> df1.dtypes
+        col1    int64
+        col2    int64
+        dtype: object
+        >>> df1_transposed.dtypes  # doctest: +SKIP
+        0    int64
+        1    int64
+        dtype: object
+
+        **Non-square DataFrame with mixed dtypes**
+
+        >>> d2 = {'score': [9.5, 8],
+        ...       'kids': [0, 0],
+        ...       'age': [12, 22]}
+        >>> df2 = ps.DataFrame(data=d2, columns=['score', 'kids', 'age'])
+        >>> df2
+           score  kids  age
+        0    9.5     0   12
+        1    8.0     0   22
+
+        >>> df2_transposed = df2.T.sort_index()  # doctest: +SKIP
+        >>> df2_transposed  # doctest: +SKIP
+                  0     1
+        age    12.0  22.0
+        kids    0.0   0.0
+        score   9.5   8.0
+
+        When the DataFrame has mixed dtypes, we get a transposed DataFrame with
+        the coerced dtype:
+
+        >>> df2.dtypes
+        score    float64
+        kids       int64
+        age        int64
+        dtype: object
+
+        >>> df2_transposed.dtypes  # doctest: +SKIP
+        0    float64
+        1    float64
+        dtype: object
+        """
+        max_compute_count = get_option("compute.max_rows")
+        if max_compute_count is not None:
+            pdf = self.head(max_compute_count + 1)._to_internal_pandas()
+            if len(pdf) > max_compute_count:
+                raise ValueError(
+                    "Current DataFrame has more then the given limit {0} rows. "
+                    "Please set 'compute.max_rows' by using 'pyspark.pandas.config.set_option' "
+                    "to retrieve to retrieve more than {0} rows. Note that, before changing the "
+                    "'compute.max_rows', this operation is considerably expensive.".format(
+                        max_compute_count
+                    )
+                )
+            return DataFrame(pdf.transpose())
+
+        # Explode the data to be pairs.
+        #
+        # For instance, if the current input DataFrame is as below:
+        #
+        # +------+------+------+------+------+
+        # |index1|index2|(a,x1)|(a,x2)|(b,x3)|
+        # +------+------+------+------+------+
+        # |    y1|    z1|     1|     0|     0|
+        # |    y2|    z2|     0|    50|     0|
+        # |    y3|    z3|     3|     2|     1|
+        # +------+------+------+------+------+
+        #
+        # Output of `exploded_df` becomes as below:
+        #
+        # +-----------------+-----------------+-----------------+-----+
+        # |            index|__index_level_0__|__index_level_1__|value|
+        # +-----------------+-----------------+-----------------+-----+
+        # |{"a":["y1","z1"]}|                a|               x1|    1|
+        # |{"a":["y1","z1"]}|                a|               x2|    0|
+        # |{"a":["y1","z1"]}|                b|               x3|    0|
+        # |{"a":["y2","z2"]}|                a|               x1|    0|
+        # |{"a":["y2","z2"]}|                a|               x2|   50|
+        # |{"a":["y2","z2"]}|                b|               x3|    0|
+        # |{"a":["y3","z3"]}|                a|               x1|    3|
+        # |{"a":["y3","z3"]}|                a|               x2|    2|
+        # |{"a":["y3","z3"]}|                b|               x3|    1|
+        # +-----------------+-----------------+-----------------+-----+
+        pairs = F.explode(
+            F.array(
+                *[
+                    F.struct(
+                        [
+                            F.lit(col).alias(SPARK_INDEX_NAME_FORMAT(i))
+                            for i, col in enumerate(label)
+                        ]
+                        + [self._internal.spark_column_for(label).alias("value")]
+                    )
+                    for label in self._internal.column_labels
+                ]
+            )
+        )
+
+        exploded_df = self._internal.spark_frame.withColumn("pairs", pairs).select(
+            [
+                F.to_json(
+                    F.struct(
+                        F.array([scol for scol in self._internal.index_spark_columns]).alias("a")
+                    )
+                ).alias("index"),
+                F.col("pairs.*"),
+            ]
+        )
+
+        # After that, executes pivot with key and its index column.
+        # Note that index column should contain unique values since column names
+        # should be unique.
+        internal_index_columns = [
+            SPARK_INDEX_NAME_FORMAT(i) for i in range(self._internal.column_labels_level)
+        ]
+        pivoted_df = exploded_df.groupBy(internal_index_columns).pivot("index")
+
+        transposed_df = pivoted_df.agg(F.first(F.col("value")))
+
+        new_data_columns = list(
+            filter(lambda x: x not in internal_index_columns, transposed_df.columns)
+        )
+
+        column_labels = [
+            None if len(label) == 1 and label[0] is None else label
+            for label in (tuple(json.loads(col)["a"]) for col in new_data_columns)
+        ]
+
+        internal = InternalFrame(
+            spark_frame=transposed_df,
+            index_spark_columns=[scol_for(transposed_df, col) for col in internal_index_columns],
+            index_names=self._internal.column_label_names,
+            column_labels=column_labels,
+            data_spark_columns=[scol_for(transposed_df, col) for col in new_data_columns],
+            column_label_names=self._internal.index_names,
+        )
+
+        return DataFrame(internal)
+
+    T = property(transpose)
+
+    def apply_batch(self, func, args=(), **kwds) -> "DataFrame":
+        warnings.warn(
+            "DataFrame.apply_batch is deprecated as of DataFrame.koalas.apply_batch. "
+            "Please use the API instead.",
+            FutureWarning,
+        )
+        return self.koalas.apply_batch(func, args=args, **kwds)
+
+    apply_batch.__doc__ = PandasOnSparkFrameMethods.apply_batch.__doc__
+
+    # TODO: Remove this API.
+    def map_in_pandas(self, func) -> "DataFrame":
+        warnings.warn(
+            "DataFrame.map_in_pandas is deprecated as of DataFrame.koalas.apply_batch. "
+            "Please use the API instead.",
+            FutureWarning,
+        )
+        return self.koalas.apply_batch(func)
+
+    map_in_pandas.__doc__ = PandasOnSparkFrameMethods.apply_batch.__doc__
+
+    def apply(self, func, axis=0, args=(), **kwds) -> Union["Series", "DataFrame", "Index"]:
+        """
+        Apply a function along an axis of the DataFrame.
+
+        Objects passed to the function are Series objects whose index is
+        either the DataFrame's index (``axis=0``) or the DataFrame's columns
+        (``axis=1``).
+
+        See also `Transform and apply a function
+        <https://koalas.readthedocs.io/en/latest/user_guide/transform_apply.html>`_.
+
+        .. note:: when `axis` is 0 or 'index', the `func` is unable to access
+            to the whole input series. pandas-on-Spark internally splits the input series into
+            multiple batches and calls `func` with each batch multiple times. Therefore, operations
+            such as global aggregations are impossible. See the example below.
+
+            >>> # This case does not return the length of whole series but of the batch internally
+            ... # used.
+            ... def length(s) -> int:
+            ...     return len(s)
+            ...
+            >>> df = ps.DataFrame({'A': range(1000)})
+            >>> df.apply(length, axis=0)  # doctest: +SKIP
+            0     83
+            1     83
+            2     83
+            ...
+            10    83
+            11    83
+            dtype: int32
+
+        .. note:: this API executes the function once to infer the type which is
+            potentially expensive, for instance, when the dataset is created after
+            aggregations or sorting.
+
+            To avoid this, specify the return type as `Series` or scalar value in ``func``,
+            for instance, as below:
+
+            >>> def square(s) -> ps.Series[np.int32]:
+            ...     return s ** 2
+
+            pandas-on-Spark uses return type hint and does not try to infer the type.
+
+            In case when axis is 1, it requires to specify `DataFrame` or scalar value
+            with type hints as below:
+
+            >>> def plus_one(x) -> ps.DataFrame[float, float]:
+            ...     return x + 1
+
+            If the return type is specified as `DataFrame`, the output column names become
+            `c0, c1, c2 ... cn`. These names are positionally mapped to the returned
+            DataFrame in ``func``.
+
+            To specify the column names, you can assign them in a pandas friendly style as below:
+
+            >>> def plus_one(x) -> ps.DataFrame["a": float, "b": float]:
+            ...     return x + 1
+
+            >>> pdf = pd.DataFrame({'a': [1, 2, 3], 'b': [3, 4, 5]})
+            >>> def plus_one(x) -> ps.DataFrame[zip(pdf.dtypes, pdf.columns)]:
+            ...     return x + 1
+
+            However, this way switches the index type to default index type in the output
+            because the type hint cannot express the index type at this moment. Use
+            `reset_index()` to keep index as a workaround.
+
+            When the given function has the return type annotated, the original index of the
+            DataFrame will be lost and then a default index will be attached to the result.
+            Please be careful about configuring the default index. See also `Default Index Type
+            <https://koalas.readthedocs.io/en/latest/user_guide/options.html#default-index-type>`_.
+
+        Parameters
+        ----------
+        func : function
+            Function to apply to each column or row.
+        axis : {0 or 'index', 1 or 'columns'}, default 0
+            Axis along which the function is applied:
+
+            * 0 or 'index': apply function to each column.
+            * 1 or 'columns': apply function to each row.
+        args : tuple
+            Positional arguments to pass to `func` in addition to the
+            array/series.
+        **kwds
+            Additional keyword arguments to pass as keywords arguments to
+            `func`.
+
+        Returns
+        -------
+        Series or DataFrame
+            Result of applying ``func`` along the given axis of the
+            DataFrame.
+
+        See Also
+        --------
+        DataFrame.applymap : For elementwise operations.
+        DataFrame.aggregate : Only perform aggregating type operations.
+        DataFrame.transform : Only perform transforming type operations.
+        Series.apply : The equivalent function for Series.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([[4, 9]] * 3, columns=['A', 'B'])
+        >>> df
+           A  B
+        0  4  9
+        1  4  9
+        2  4  9
+
+        Using a numpy universal function (in this case the same as
+        ``np.sqrt(df)``):
+
+        >>> def sqrt(x) -> ps.Series[float]:
+        ...     return np.sqrt(x)
+        ...
+        >>> df.apply(sqrt, axis=0)
+             A    B
+        0  2.0  3.0
+        1  2.0  3.0
+        2  2.0  3.0
+
+        You can omit the type hint and let pandas-on-Spark infer its type.
+
+        >>> df.apply(np.sqrt, axis=0)
+             A    B
+        0  2.0  3.0
+        1  2.0  3.0
+        2  2.0  3.0
+
+        When `axis` is 1 or 'columns', it applies the function for each row.
+
+        >>> def summation(x) -> np.int64:
+        ...     return np.sum(x)
+        ...
+        >>> df.apply(summation, axis=1)
+        0    13
+        1    13
+        2    13
+        dtype: int64
+
+        Likewise, you can omit the type hint and let pandas-on-Spark infer its type.
+
+        >>> df.apply(np.sum, axis=1)
+        0    13
+        1    13
+        2    13
+        dtype: int64
+
+        >>> df.apply(max, axis=1)
+        0    9
+        1    9
+        2    9
+        dtype: int64
+
+        Returning a list-like will result in a Series
+
+        >>> df.apply(lambda x: [1, 2], axis=1)
+        0    [1, 2]
+        1    [1, 2]
+        2    [1, 2]
+        dtype: object
+
+        In order to specify the types when `axis` is '1', it should use DataFrame[...]
+        annotation. In this case, the column names are automatically generated.
+
+        >>> def identify(x) -> ps.DataFrame['A': np.int64, 'B': np.int64]:
+        ...     return x
+        ...
+        >>> df.apply(identify, axis=1)
+           A  B
+        0  4  9
+        1  4  9
+        2  4  9
+
+        You can also specify extra arguments.
+
+        >>> def plus_two(a, b, c) -> ps.DataFrame[np.int64, np.int64]:
+        ...     return a + b + c
+        ...
+        >>> df.apply(plus_two, axis=1, args=(1,), c=3)
+           c0  c1
+        0   8  13
+        1   8  13
+        2   8  13
+        """
+        from pyspark.pandas.groupby import GroupBy
+        from pyspark.pandas.series import first_series
+
+        if not isinstance(func, types.FunctionType):
+            assert callable(func), "the first argument should be a callable function."
+            f = func
+            func = lambda *args, **kwargs: f(*args, **kwargs)
+
+        axis = validate_axis(axis)
+        should_return_series = False
+        spec = inspect.getfullargspec(func)
+        return_sig = spec.annotations.get("return", None)
+        should_infer_schema = return_sig is None
+
+        def apply_func(pdf):
+            pdf_or_pser = pdf.apply(func, axis=axis, args=args, **kwds)
+            if isinstance(pdf_or_pser, pd.Series):
+                return pdf_or_pser.to_frame()
+            else:
+                return pdf_or_pser
+
+        self_applied = DataFrame(self._internal.resolved_copy)  # type: "DataFrame"
+
+        column_labels = None  # type: Optional[List[Tuple]]
+        if should_infer_schema:
+            # Here we execute with the first 1000 to get the return type.
+            # If the records were less than 1000, it uses pandas API directly for a shortcut.
+            limit = get_option("compute.shortcut_limit")
+            pdf = self_applied.head(limit + 1)._to_internal_pandas()
+            applied = pdf.apply(func, axis=axis, args=args, **kwds)
+            kser_or_kdf = ps.from_pandas(applied)
+            if len(pdf) <= limit:
+                return kser_or_kdf
+
+            kdf = kser_or_kdf
+            if isinstance(kser_or_kdf, ps.Series):
+                should_return_series = True
+                kdf = kser_or_kdf._kdf
+
+            return_schema = force_decimal_precision_scale(
+                as_nullable_spark_type(kdf._internal.to_internal_spark_frame.schema)
+            )
+
+            output_func = GroupBy._make_pandas_df_builder_func(
+                self_applied, apply_func, return_schema, retain_index=True
+            )
+            sdf = self_applied._internal.to_internal_spark_frame.mapInPandas(
+                lambda iterator: map(output_func, iterator), schema=return_schema
+            )
+
+            # If schema is inferred, we can restore indexes too.
+            internal = kdf._internal.with_new_sdf(sdf)
+        else:
+            return_type = infer_return_type(func)
+            require_index_axis = isinstance(return_type, SeriesType)
+            require_column_axis = isinstance(return_type, DataFrameType)
+
+            if require_index_axis:
+                if axis != 0:
+                    raise TypeError(
+                        "The given function should specify a scalar or a series as its type "
+                        "hints when axis is 0 or 'index'; however, the return type "
+                        "was %s" % return_sig
+                    )
+                return_schema = cast(SeriesType, return_type).spark_type
+                fields_types = zip(
+                    self_applied.columns, [return_schema] * len(self_applied.columns)
+                )
+                return_schema = StructType([StructField(c, t) for c, t in fields_types])
+                data_dtypes = [cast(SeriesType, return_type).dtype] * len(self_applied.columns)
+            elif require_column_axis:
+                if axis != 1:
+                    raise TypeError(
+                        "The given function should specify a scalar or a frame as its type "
+                        "hints when axis is 1 or 'column'; however, the return type "
+                        "was %s" % return_sig
+                    )
+                return_schema = cast(DataFrameType, return_type).spark_type
+                data_dtypes = cast(DataFrameType, return_type).dtypes
+            else:
+                # any axis is fine.
+                should_return_series = True
+                return_schema = cast(ScalarType, return_type).spark_type
+                return_schema = StructType([StructField(SPARK_DEFAULT_SERIES_NAME, return_schema)])
+                data_dtypes = [cast(ScalarType, return_type).dtype]
+                column_labels = [None]
+
+            output_func = GroupBy._make_pandas_df_builder_func(
+                self_applied, apply_func, return_schema, retain_index=False
+            )
+            sdf = self_applied._internal.to_internal_spark_frame.mapInPandas(
+                lambda iterator: map(output_func, iterator), schema=return_schema
+            )
+
+            # Otherwise, it loses index.
+            internal = InternalFrame(
+                spark_frame=sdf,
+                index_spark_columns=None,
+                column_labels=column_labels,
+                data_dtypes=data_dtypes,
+            )
+
+        result = DataFrame(internal)  # type: "DataFrame"
+        if should_return_series:
+            return first_series(result)
+        else:
+            return result
+
+    def transform(self, func, axis=0, *args, **kwargs) -> "DataFrame":
+        """
+        Call ``func`` on self producing a Series with transformed values
+        and that has the same length as its input.
+
+        See also `Transform and apply a function
+        <https://koalas.readthedocs.io/en/latest/user_guide/transform_apply.html>`_.
+
+        .. note:: this API executes the function once to infer the type which is
+             potentially expensive, for instance, when the dataset is created after
+             aggregations or sorting.
+
+             To avoid this, specify return type in ``func``, for instance, as below:
+
+             >>> def square(x) -> ps.Series[np.int32]:
+             ...     return x ** 2
+
+             pandas-on-Spark uses return type hint and does not try to infer the type.
+
+        .. note:: the series within ``func`` is actually multiple pandas series as the
+            segments of the whole pandas-on-Spark series; therefore, the length of each series
+            is not guaranteed. As an example, an aggregation against each series
+            does work as a global aggregation but an aggregation of each segment. See
+            below:
+
+            >>> def func(x) -> ps.Series[np.int32]:
+            ...     return x + sum(x)
+
+        Parameters
+        ----------
+        func : function
+            Function to use for transforming the data. It must work when pandas Series
+            is passed.
+        axis : int, default 0 or 'index'
+            Can only be set to 0 at the moment.
+        *args
+            Positional arguments to pass to func.
+        **kwargs
+            Keyword arguments to pass to func.
+
+        Returns
+        -------
+        DataFrame
+            A DataFrame that must have the same length as self.
+
+        Raises
+        ------
+        Exception : If the returned DataFrame has a different length than self.
+
+        See Also
+        --------
+        DataFrame.aggregate : Only perform aggregating type operations.
+        DataFrame.apply : Invoke function on DataFrame.
+        Series.transform : The equivalent function for Series.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'A': range(3), 'B': range(1, 4)}, columns=['A', 'B'])
+        >>> df
+           A  B
+        0  0  1
+        1  1  2
+        2  2  3
+
+        >>> def square(x) -> ps.Series[np.int32]:
+        ...     return x ** 2
+        >>> df.transform(square)
+           A  B
+        0  0  1
+        1  1  4
+        2  4  9
+
+        You can omit the type hint and let pandas-on-Spark infer its type.
+
+        >>> df.transform(lambda x: x ** 2)
+           A  B
+        0  0  1
+        1  1  4
+        2  4  9
+
+        For multi-index columns:
+
+        >>> df.columns = [('X', 'A'), ('X', 'B')]
+        >>> df.transform(square)  # doctest: +NORMALIZE_WHITESPACE
+           X
+           A  B
+        0  0  1
+        1  1  4
+        2  4  9
+
+        >>> (df * -1).transform(abs)  # doctest: +NORMALIZE_WHITESPACE
+           X
+           A  B
+        0  0  1
+        1  1  2
+        2  2  3
+
+        You can also specify extra arguments.
+
+        >>> def calculation(x, y, z) -> ps.Series[int]:
+        ...     return x ** y + z
+        >>> df.transform(calculation, y=10, z=20)  # doctest: +NORMALIZE_WHITESPACE
+              X
+              A      B
+        0    20     21
+        1    21   1044
+        2  1044  59069
+        """
+        if not isinstance(func, types.FunctionType):
+            assert callable(func), "the first argument should be a callable function."
+            f = func
+            func = lambda *args, **kwargs: f(*args, **kwargs)
+
+        axis = validate_axis(axis)
+        if axis != 0:
+            raise NotImplementedError('axis should be either 0 or "index" currently.')
+
+        spec = inspect.getfullargspec(func)
+        return_sig = spec.annotations.get("return", None)
+        should_infer_schema = return_sig is None
+
+        if should_infer_schema:
+            # Here we execute with the first 1000 to get the return type.
+            # If the records were less than 1000, it uses pandas API directly for a shortcut.
+            limit = get_option("compute.shortcut_limit")
+            pdf = self.head(limit + 1)._to_internal_pandas()
+            transformed = pdf.transform(func, axis, *args, **kwargs)
+            kdf = DataFrame(transformed)  # type: "DataFrame"
+            if len(pdf) <= limit:
+                return kdf
+
+            applied = []
+            for input_label, output_label in zip(
+                self._internal.column_labels, kdf._internal.column_labels
+            ):
+                kser = self._kser_for(input_label)
+                dtype = kdf._internal.dtype_for(output_label)
+                return_schema = force_decimal_precision_scale(
+                    as_nullable_spark_type(kdf._internal.spark_type_for(output_label))
+                )
+                applied.append(
+                    kser.koalas._transform_batch(
+                        func=lambda c: func(c, *args, **kwargs),
+                        return_type=SeriesType(dtype, return_schema),
+                    )
+                )
+
+            internal = self._internal.with_new_columns(
+                applied, data_dtypes=kdf._internal.data_dtypes
+            )
+            return DataFrame(internal)
+        else:
+            return self._apply_series_op(
+                lambda kser: kser.koalas.transform_batch(func, *args, **kwargs)
+            )
+
+    def transform_batch(self, func, *args, **kwargs) -> "DataFrame":
+        warnings.warn(
+            "DataFrame.transform_batch is deprecated as of DataFrame.koalas.transform_batch. "
+            "Please use the API instead.",
+            FutureWarning,
+        )
+        return self.koalas.transform_batch(func, *args, **kwargs)
+
+    transform_batch.__doc__ = PandasOnSparkFrameMethods.transform_batch.__doc__
+
+    def pop(self, item) -> "DataFrame":
+        """
+        Return item and drop from frame. Raise KeyError if not found.
+
+        Parameters
+        ----------
+        item : str
+            Label of column to be popped.
+
+        Returns
+        -------
+        Series
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([('falcon', 'bird', 389.0),
+        ...                    ('parrot', 'bird', 24.0),
+        ...                    ('lion', 'mammal', 80.5),
+        ...                    ('monkey','mammal', np.nan)],
+        ...                   columns=('name', 'class', 'max_speed'))
+
+        >>> df
+             name   class  max_speed
+        0  falcon    bird      389.0
+        1  parrot    bird       24.0
+        2    lion  mammal       80.5
+        3  monkey  mammal        NaN
+
+        >>> df.pop('class')
+        0      bird
+        1      bird
+        2    mammal
+        3    mammal
+        Name: class, dtype: object
+
+        >>> df
+             name  max_speed
+        0  falcon      389.0
+        1  parrot       24.0
+        2    lion       80.5
+        3  monkey        NaN
+
+        Also support for MultiIndex
+
+        >>> df = ps.DataFrame([('falcon', 'bird', 389.0),
+        ...                    ('parrot', 'bird', 24.0),
+        ...                    ('lion', 'mammal', 80.5),
+        ...                    ('monkey','mammal', np.nan)],
+        ...                   columns=('name', 'class', 'max_speed'))
+        >>> columns = [('a', 'name'), ('a', 'class'), ('b', 'max_speed')]
+        >>> df.columns = pd.MultiIndex.from_tuples(columns)
+        >>> df
+                a                 b
+             name   class max_speed
+        0  falcon    bird     389.0
+        1  parrot    bird      24.0
+        2    lion  mammal      80.5
+        3  monkey  mammal       NaN
+
+        >>> df.pop('a')
+             name   class
+        0  falcon    bird
+        1  parrot    bird
+        2    lion  mammal
+        3  monkey  mammal
+
+        >>> df
+                  b
+          max_speed
+        0     389.0
+        1      24.0
+        2      80.5
+        3       NaN
+        """
+        result = self[item]
+        self._update_internal_frame(self.drop(item)._internal)
+        return result
+
+    # TODO: add axis parameter can work when '1' or 'columns'
+    def xs(self, key, axis=0, level=None) -> Union["DataFrame", "Series"]:
+        """
+        Return cross-section from the DataFrame.
+
+        This method takes a `key` argument to select data at a particular
+        level of a MultiIndex.
+
+        Parameters
+        ----------
+        key : label or tuple of label
+            Label contained in the index, or partially in a MultiIndex.
+        axis : 0 or 'index', default 0
+            Axis to retrieve cross-section on.
+            currently only support 0 or 'index'
+        level : object, defaults to first n levels (n=1 or len(key))
+            In case of a key partially contained in a MultiIndex, indicate
+            which levels are used. Levels can be referred by label or position.
+
+        Returns
+        -------
+        DataFrame or Series
+            Cross-section from the original DataFrame
+            corresponding to the selected index levels.
+
+        See Also
+        --------
+        DataFrame.loc : Access a group of rows and columns
+            by label(s) or a boolean array.
+        DataFrame.iloc : Purely integer-location based indexing
+            for selection by position.
+
+        Examples
+        --------
+        >>> d = {'num_legs': [4, 4, 2, 2],
+        ...      'num_wings': [0, 0, 2, 2],
+        ...      'class': ['mammal', 'mammal', 'mammal', 'bird'],
+        ...      'animal': ['cat', 'dog', 'bat', 'penguin'],
+        ...      'locomotion': ['walks', 'walks', 'flies', 'walks']}
+        >>> df = ps.DataFrame(data=d)
+        >>> df = df.set_index(['class', 'animal', 'locomotion'])
+        >>> df  # doctest: +NORMALIZE_WHITESPACE
+                                   num_legs  num_wings
+        class  animal  locomotion
+        mammal cat     walks              4          0
+               dog     walks              4          0
+               bat     flies              2          2
+        bird   penguin walks              2          2
+
+        Get values at specified index
+
+        >>> df.xs('mammal')  # doctest: +NORMALIZE_WHITESPACE
+                           num_legs  num_wings
+        animal locomotion
+        cat    walks              4          0
+        dog    walks              4          0
+        bat    flies              2          2
+
+        Get values at several indexes
+
+        >>> df.xs(('mammal', 'dog'))  # doctest: +NORMALIZE_WHITESPACE
+                    num_legs  num_wings
+        locomotion
+        walks              4          0
+
+        >>> df.xs(('mammal', 'dog', 'walks'))  # doctest: +NORMALIZE_WHITESPACE
+        num_legs     4
+        num_wings    0
+        Name: (mammal, dog, walks), dtype: int64
+
+        Get values at specified index and level
+
+        >>> df.xs('cat', level=1)  # doctest: +NORMALIZE_WHITESPACE
+                           num_legs  num_wings
+        class  locomotion
+        mammal walks              4          0
+        """
+        from pyspark.pandas.series import first_series
+
+        if not is_name_like_value(key):
+            raise TypeError("'key' should be a scalar value or tuple that contains scalar values")
+
+        if level is not None and is_name_like_tuple(key):
+            raise KeyError(key)
+
+        axis = validate_axis(axis)
+        if axis != 0:
+            raise NotImplementedError('axis should be either 0 or "index" currently.')
+
+        if not is_name_like_tuple(key):
+            key = (key,)
+        if len(key) > self._internal.index_level:
+            raise KeyError(
+                "Key length ({}) exceeds index depth ({})".format(
+                    len(key), self._internal.index_level
+                )
+            )
+        if level is None:
+            level = 0
+
+        rows = [
+            self._internal.index_spark_columns[lvl] == index for lvl, index in enumerate(key, level)
+        ]
+        internal = self._internal.with_filter(reduce(lambda x, y: x & y, rows))
+
+        if len(key) == self._internal.index_level:
+            kdf = DataFrame(internal)  # type: DataFrame
+            pdf = kdf.head(2)._to_internal_pandas()
+            if len(pdf) == 0:
+                raise KeyError(key)
+            elif len(pdf) > 1:
+                return kdf
+            else:
+                return first_series(DataFrame(pdf.transpose()))
+        else:
+            index_spark_columns = (
+                internal.index_spark_columns[:level]
+                + internal.index_spark_columns[level + len(key):]
+            )
+            index_names = internal.index_names[:level] + internal.index_names[level + len(key):]
+            index_dtypes = internal.index_dtypes[:level] + internal.index_dtypes[level + len(key):]
+
+            internal = internal.copy(
+                index_spark_columns=index_spark_columns,
+                index_names=index_names,
+                index_dtypes=index_dtypes,
+            ).resolved_copy
+            return DataFrame(internal)
+
+    def between_time(
+        self,
+        start_time: Union[datetime.time, str],
+        end_time: Union[datetime.time, str],
+        include_start: bool = True,
+        include_end: bool = True,
+        axis: Union[int, str] = 0,
+    ) -> "DataFrame":
+        """
+        Select values between particular times of the day (example: 9:00-9:30 AM).
+
+        By setting ``start_time`` to be later than ``end_time``,
+        you can get the times that are *not* between the two times.
+
+        Parameters
+        ----------
+        start_time : datetime.time or str
+            Initial time as a time filter limit.
+        end_time : datetime.time or str
+            End time as a time filter limit.
+        include_start : bool, default True
+            Whether the start time needs to be included in the result.
+        include_end : bool, default True
+            Whether the end time needs to be included in the result.
+        axis : {0 or 'index', 1 or 'columns'}, default 0
+            Determine range time on index or columns value.
+
+        Returns
+        -------
+        DataFrame
+            Data from the original object filtered to the specified dates range.
+
+        Raises
+        ------
+        TypeError
+            If the index is not  a :class:`DatetimeIndex`
+
+        See Also
+        --------
+        at_time : Select values at a particular time of the day.
+        first : Select initial periods of time series based on a date offset.
+        last : Select final periods of time series based on a date offset.
+        DatetimeIndex.indexer_between_time : Get just the index locations for
+            values between particular times of the day.
+
+        Examples
+        --------
+        >>> idx = pd.date_range('2018-04-09', periods=4, freq='1D20min')
+        >>> kdf = ps.DataFrame({'A': [1, 2, 3, 4]}, index=idx)
+        >>> kdf
+                             A
+        2018-04-09 00:00:00  1
+        2018-04-10 00:20:00  2
+        2018-04-11 00:40:00  3
+        2018-04-12 01:00:00  4
+
+        >>> kdf.between_time('0:15', '0:45')
+                             A
+        2018-04-10 00:20:00  2
+        2018-04-11 00:40:00  3
+
+        You get the times that are *not* between two times by setting
+        ``start_time`` later than ``end_time``:
+
+        >>> kdf.between_time('0:45', '0:15')
+                             A
+        2018-04-09 00:00:00  1
+        2018-04-12 01:00:00  4
+        """
+        axis = validate_axis(axis)
+
+        if axis != 0:
+            raise NotImplementedError("between_time currently only works for axis=0")
+
+        if not isinstance(self.index, ps.DatetimeIndex):
+            raise TypeError("Index must be DatetimeIndex")
+
+        kdf = self.copy()
+        kdf.index.name = verify_temp_column_name(kdf, "__index_name__")
+        return_types = [kdf.index.dtype] + list(kdf.dtypes)
+
+        def pandas_between_time(pdf) -> ps.DataFrame[return_types]:  # type: ignore
+            return pdf.between_time(start_time, end_time, include_start, include_end).reset_index()
+
+        # apply_batch will remove the index of the pandas-on-Spark DataFrame and attach a
+        # default index, which will never be used. So use "distributed" index as a dummy to
+        # avoid overhead.
+        with option_context("compute.default_index_type", "distributed"):
+            kdf = kdf.koalas.apply_batch(pandas_between_time)
+
+        return DataFrame(
+            self._internal.copy(
+                spark_frame=kdf._internal.spark_frame,
+                index_spark_columns=kdf._internal.data_spark_columns[:1],
+                data_spark_columns=kdf._internal.data_spark_columns[1:],
+            )
+        )
+
+    # TODO: implement axis=1
+    def at_time(
+        self, time: Union[datetime.time, str], asof: bool = False, axis: Union[int, str] = 0
+    ) -> "DataFrame":
+        """
+        Select values at particular time of day (example: 9:30AM).
+
+        Parameters
+        ----------
+        time : datetime.time or str
+        axis : {0 or 'index', 1 or 'columns'}, default 0
+
+        Returns
+        -------
+        DataFrame
+
+        Raises
+        ------
+        TypeError
+            If the index is not  a :class:`DatetimeIndex`
+
+        See Also
+        --------
+        between_time : Select values between particular times of the day.
+        DatetimeIndex.indexer_at_time : Get just the index locations for
+            values at particular time of the day.
+
+        Examples
+        --------
+        >>> idx = pd.date_range('2018-04-09', periods=4, freq='12H')
+        >>> kdf = ps.DataFrame({'A': [1, 2, 3, 4]}, index=idx)
+        >>> kdf
+                             A
+        2018-04-09 00:00:00  1
+        2018-04-09 12:00:00  2
+        2018-04-10 00:00:00  3
+        2018-04-10 12:00:00  4
+
+        >>> kdf.at_time('12:00')
+                             A
+        2018-04-09 12:00:00  2
+        2018-04-10 12:00:00  4
+        """
+        if asof:
+            raise NotImplementedError("'asof' argument is not supported")
+
+        axis = validate_axis(axis)
+
+        if axis != 0:
+            raise NotImplementedError("at_time currently only works for axis=0")
+
+        if not isinstance(self.index, ps.DatetimeIndex):
+            raise TypeError("Index must be DatetimeIndex")
+
+        kdf = self.copy()
+        kdf.index.name = verify_temp_column_name(kdf, "__index_name__")
+        return_types = [kdf.index.dtype] + list(kdf.dtypes)
+
+        if LooseVersion(pd.__version__) < LooseVersion("0.24"):
+
+            def pandas_at_time(pdf) -> ps.DataFrame[return_types]:  # type: ignore
+                return pdf.at_time(time, asof).reset_index()
+
+        else:
+
+            def pandas_at_time(pdf) -> ps.DataFrame[return_types]:  # type: ignore
+                return pdf.at_time(time, asof, axis).reset_index()
+
+        # apply_batch will remove the index of the pandas-on-Spark DataFrame and attach
+        # a default index, which will never be used. So use "distributed" index as a dummy
+        # to avoid overhead.
+        with option_context("compute.default_index_type", "distributed"):
+            kdf = kdf.koalas.apply_batch(pandas_at_time)
+
+        return DataFrame(
+            self._internal.copy(
+                spark_frame=kdf._internal.spark_frame,
+                index_spark_columns=kdf._internal.data_spark_columns[:1],
+                data_spark_columns=kdf._internal.data_spark_columns[1:],
+            )
+        )
+
+    def where(self, cond, other=np.nan) -> "DataFrame":
+        """
+        Replace values where the condition is False.
+
+        Parameters
+        ----------
+        cond : boolean DataFrame
+            Where cond is True, keep the original value. Where False,
+            replace with corresponding value from other.
+        other : scalar, DataFrame
+            Entries where cond is False are replaced with corresponding value from other.
+
+        Returns
+        -------
+        DataFrame
+
+        Examples
+        --------
+
+        >>> from pyspark.pandas.config import set_option, reset_option
+        >>> set_option("compute.ops_on_diff_frames", True)
+        >>> df1 = ps.DataFrame({'A': [0, 1, 2, 3, 4], 'B':[100, 200, 300, 400, 500]})
+        >>> df2 = ps.DataFrame({'A': [0, -1, -2, -3, -4], 'B':[-100, -200, -300, -400, -500]})
+        >>> df1
+           A    B
+        0  0  100
+        1  1  200
+        2  2  300
+        3  3  400
+        4  4  500
+        >>> df2
+           A    B
+        0  0 -100
+        1 -1 -200
+        2 -2 -300
+        3 -3 -400
+        4 -4 -500
+
+        >>> df1.where(df1 > 0).sort_index()
+             A      B
+        0  NaN  100.0
+        1  1.0  200.0
+        2  2.0  300.0
+        3  3.0  400.0
+        4  4.0  500.0
+
+        >>> df1.where(df1 > 1, 10).sort_index()
+            A    B
+        0  10  100
+        1  10  200
+        2   2  300
+        3   3  400
+        4   4  500
+
+        >>> df1.where(df1 > 1, df1 + 100).sort_index()
+             A    B
+        0  100  100
+        1  101  200
+        2    2  300
+        3    3  400
+        4    4  500
+
+        >>> df1.where(df1 > 1, df2).sort_index()
+           A    B
+        0  0  100
+        1 -1  200
+        2  2  300
+        3  3  400
+        4  4  500
+
+        When the column name of cond is different from self, it treats all values are False
+
+        >>> cond = ps.DataFrame({'C': [0, -1, -2, -3, -4], 'D':[4, 3, 2, 1, 0]}) % 3 == 0
+        >>> cond
+               C      D
+        0   True  False
+        1  False   True
+        2  False  False
+        3   True  False
+        4  False   True
+
+        >>> df1.where(cond).sort_index()
+            A   B
+        0 NaN NaN
+        1 NaN NaN
+        2 NaN NaN
+        3 NaN NaN
+        4 NaN NaN
+
+        When the type of cond is Series, it just check boolean regardless of column name
+
+        >>> cond = ps.Series([1, 2]) > 1
+        >>> cond
+        0    False
+        1     True
+        dtype: bool
+
+        >>> df1.where(cond).sort_index()
+             A      B
+        0  NaN    NaN
+        1  1.0  200.0
+        2  NaN    NaN
+        3  NaN    NaN
+        4  NaN    NaN
+
+        >>> reset_option("compute.ops_on_diff_frames")
+        """
+        from pyspark.pandas.series import Series
+
+        tmp_cond_col_name = "__tmp_cond_col_{}__".format
+        tmp_other_col_name = "__tmp_other_col_{}__".format
+
+        kdf = self.copy()
+
+        tmp_cond_col_names = [
+            tmp_cond_col_name(name_like_string(label)) for label in self._internal.column_labels
+        ]
+        if isinstance(cond, DataFrame):
+            cond = cond[
+                [
+                    (
+                        cond._internal.spark_column_for(label)
+                        if label in cond._internal.column_labels
+                        else F.lit(False)
+                    ).alias(name)
+                    for label, name in zip(self._internal.column_labels, tmp_cond_col_names)
+                ]
+            ]
+            kdf[tmp_cond_col_names] = cond
+        elif isinstance(cond, Series):
+            cond = cond.to_frame()
+            cond = cond[
+                [cond._internal.data_spark_columns[0].alias(name) for name in tmp_cond_col_names]
+            ]
+            kdf[tmp_cond_col_names] = cond
+        else:
+            raise TypeError("type of cond must be a DataFrame or Series")
+
+        tmp_other_col_names = [
+            tmp_other_col_name(name_like_string(label)) for label in self._internal.column_labels
+        ]
+        if isinstance(other, DataFrame):
+            other = other[
+                [
+                    (
+                        other._internal.spark_column_for(label)
+                        if label in other._internal.column_labels
+                        else F.lit(np.nan)
+                    ).alias(name)
+                    for label, name in zip(self._internal.column_labels, tmp_other_col_names)
+                ]
+            ]
+            kdf[tmp_other_col_names] = other
+        elif isinstance(other, Series):
+            other = other.to_frame()
+            other = other[
+                [other._internal.data_spark_columns[0].alias(name) for name in tmp_other_col_names]
+            ]
+            kdf[tmp_other_col_names] = other
+        else:
+            for label in self._internal.column_labels:
+                kdf[tmp_other_col_name(name_like_string(label))] = other
+
+        # above logic make spark dataframe looks like below:
+        # +-----------------+---+---+------------------+-------------------+------------------+--...
+        # |__index_level_0__|  A|  B|__tmp_cond_col_A__|__tmp_other_col_A__|__tmp_cond_col_B__|__...
+        # +-----------------+---+---+------------------+-------------------+------------------+--...
+        # |                0|  0|100|              true|                  0|             false|  ...
+        # |                1|  1|200|             false|                 -1|             false|  ...
+        # |                3|  3|400|              true|                 -3|             false|  ...
+        # |                2|  2|300|             false|                 -2|              true|  ...
+        # |                4|  4|500|             false|                 -4|             false|  ...
+        # +-----------------+---+---+------------------+-------------------+------------------+--...
+
+        data_spark_columns = []
+        for label in self._internal.column_labels:
+            data_spark_columns.append(
+                F.when(
+                    kdf[tmp_cond_col_name(name_like_string(label))].spark.column,
+                    kdf._internal.spark_column_for(label),
+                )
+                .otherwise(kdf[tmp_other_col_name(name_like_string(label))].spark.column)
+                .alias(kdf._internal.spark_column_name_for(label))
+            )
+
+        return DataFrame(
+            kdf._internal.with_new_columns(
+                data_spark_columns, column_labels=self._internal.column_labels  # TODO: dtypes?
+            )
+        )
+
+    def mask(self, cond, other=np.nan) -> "DataFrame":
+        """
+        Replace values where the condition is True.
+
+        Parameters
+        ----------
+        cond : boolean DataFrame
+            Where cond is False, keep the original value. Where True,
+            replace with corresponding value from other.
+        other : scalar, DataFrame
+            Entries where cond is True are replaced with corresponding value from other.
+
+        Returns
+        -------
+        DataFrame
+
+        Examples
+        --------
+
+        >>> from pyspark.pandas.config import set_option, reset_option
+        >>> set_option("compute.ops_on_diff_frames", True)
+        >>> df1 = ps.DataFrame({'A': [0, 1, 2, 3, 4], 'B':[100, 200, 300, 400, 500]})
+        >>> df2 = ps.DataFrame({'A': [0, -1, -2, -3, -4], 'B':[-100, -200, -300, -400, -500]})
+        >>> df1
+           A    B
+        0  0  100
+        1  1  200
+        2  2  300
+        3  3  400
+        4  4  500
+        >>> df2
+           A    B
+        0  0 -100
+        1 -1 -200
+        2 -2 -300
+        3 -3 -400
+        4 -4 -500
+
+        >>> df1.mask(df1 > 0).sort_index()
+             A   B
+        0  0.0 NaN
+        1  NaN NaN
+        2  NaN NaN
+        3  NaN NaN
+        4  NaN NaN
+
+        >>> df1.mask(df1 > 1, 10).sort_index()
+            A   B
+        0   0  10
+        1   1  10
+        2  10  10
+        3  10  10
+        4  10  10
+
+        >>> df1.mask(df1 > 1, df1 + 100).sort_index()
+             A    B
+        0    0  200
+        1    1  300
+        2  102  400
+        3  103  500
+        4  104  600
+
+        >>> df1.mask(df1 > 1, df2).sort_index()
+           A    B
+        0  0 -100
+        1  1 -200
+        2 -2 -300
+        3 -3 -400
+        4 -4 -500
+
+        >>> reset_option("compute.ops_on_diff_frames")
+        """
+        from pyspark.pandas.series import Series
+
+        if not isinstance(cond, (DataFrame, Series)):
+            raise TypeError("type of cond must be a DataFrame or Series")
+
+        cond_inversed = cond._apply_series_op(lambda kser: ~kser)
+        return self.where(cond_inversed, other)
+
+    @property
+    def index(self) -> "Index":
+        """The index (row labels) Column of the DataFrame.
+
+        Currently not supported when the DataFrame has no index.
+
+        See Also
+        --------
+        Index
+        """
+        from pyspark.pandas.indexes.base import Index
+
+        return Index._new_instance(self)
+
+    @property
+    def empty(self) -> bool:
+        """
+        Returns true if the current DataFrame is empty. Otherwise, returns false.
+
+        Examples
+        --------
+        >>> ps.range(10).empty
+        False
+
+        >>> ps.range(0).empty
+        True
+
+        >>> ps.DataFrame({}, index=list('abc')).empty
+        True
+        """
+        return (
+            len(self._internal.column_labels) == 0
+            or self._internal.resolved_copy.spark_frame.rdd.isEmpty()
+        )
+
+    @property
+    def style(self) -> "Styler":
+        """
+        Property returning a Styler object containing methods for
+        building a styled HTML representation for the DataFrame.
+
+        .. note:: currently it collects top 1000 rows and return its
+            pandas `pandas.io.formats.style.Styler` instance.
+
+        Examples
+        --------
+        >>> ps.range(1001).style  # doctest: +SKIP
+        <pandas.io.formats.style.Styler object at ...>
+        """
+        max_results = get_option("compute.max_rows")
+        pdf = self.head(max_results + 1)._to_internal_pandas()
+        if len(pdf) > max_results:
+            warnings.warn("'style' property will only use top %s rows." % max_results, UserWarning)
+        return pdf.head(max_results).style
+
+    def set_index(self, keys, drop=True, append=False, inplace=False) -> Optional["DataFrame"]:
+        """Set the DataFrame index (row labels) using one or more existing columns.
+
+        Set the DataFrame index (row labels) using one or more existing
+        columns or arrays (of the correct length). The index can replace the
+        existing index or expand on it.
+
+        Parameters
+        ----------
+        keys : label or array-like or list of labels/arrays
+            This parameter can be either a single column key, a single array of
+            the same length as the calling DataFrame, or a list containing an
+            arbitrary combination of column keys and arrays. Here, "array"
+            encompasses :class:`Series`, :class:`Index` and ``np.ndarray``.
+        drop : bool, default True
+            Delete columns to be used as the new index.
+        append : bool, default False
+            Whether to append columns to existing index.
+        inplace : bool, default False
+            Modify the DataFrame in place (do not create a new object).
+
+        Returns
+        -------
+        DataFrame
+            Changed row labels.
+
+        See Also
+        --------
+        DataFrame.reset_index : Opposite of set_index.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'month': [1, 4, 7, 10],
+        ...                    'year': [2012, 2014, 2013, 2014],
+        ...                    'sale': [55, 40, 84, 31]},
+        ...                   columns=['month', 'year', 'sale'])
+        >>> df
+           month  year  sale
+        0      1  2012    55
+        1      4  2014    40
+        2      7  2013    84
+        3     10  2014    31
+
+        Set the index to become the 'month' column:
+
+        >>> df.set_index('month')  # doctest: +NORMALIZE_WHITESPACE
+               year  sale
+        month
+        1      2012    55
+        4      2014    40
+        7      2013    84
+        10     2014    31
+
+        Create a MultiIndex using columns 'year' and 'month':
+
+        >>> df.set_index(['year', 'month'])  # doctest: +NORMALIZE_WHITESPACE
+                    sale
+        year  month
+        2012  1     55
+        2014  4     40
+        2013  7     84
+        2014  10    31
+        """
+        inplace = validate_bool_kwarg(inplace, "inplace")
+        if is_name_like_tuple(keys):
+            keys = [keys]
+        elif is_name_like_value(keys):
+            keys = [(keys,)]
+        else:
+            keys = [key if is_name_like_tuple(key) else (key,) for key in keys]
+        columns = set(self._internal.column_labels)
+        for key in keys:
+            if key not in columns:
+                raise KeyError(name_like_string(key))
+
+        if drop:
+            column_labels = [label for label in self._internal.column_labels if label not in keys]
+        else:
+            column_labels = self._internal.column_labels
+        if append:
+            index_spark_columns = self._internal.index_spark_columns + [
+                self._internal.spark_column_for(label) for label in keys
+            ]
+            index_names = self._internal.index_names + keys
+            index_dtypes = self._internal.index_dtypes + [
+                self._internal.dtype_for(label) for label in keys
+            ]
+        else:
+            index_spark_columns = [self._internal.spark_column_for(label) for label in keys]
+            index_names = keys
+            index_dtypes = [self._internal.dtype_for(label) for label in keys]
+
+        internal = self._internal.copy(
+            index_spark_columns=index_spark_columns,
+            index_names=index_names,
+            index_dtypes=index_dtypes,
+            column_labels=column_labels,
+            data_spark_columns=[self._internal.spark_column_for(label) for label in column_labels],
+            data_dtypes=[self._internal.dtype_for(label) for label in column_labels],
+        )
+
+        if inplace:
+            self._update_internal_frame(internal)
+            return None
+        else:
+            return DataFrame(internal)
+
+    def reset_index(
+        self, level=None, drop=False, inplace=False, col_level=0, col_fill=""
+    ) -> Optional["DataFrame"]:
+        """Reset the index, or a level of it.
+
+        For DataFrame with multi-level index, return new DataFrame with labeling information in
+        the columns under the index names, defaulting to 'level_0', 'level_1', etc. if any are None.
+        For a standard index, the index name will be used (if set), otherwise a default 'index' or
+        'level_0' (if 'index' is already taken) will be used.
+
+        Parameters
+        ----------
+        level : int, str, tuple, or list, default None
+            Only remove the given levels from the index. Removes all levels by
+            default.
+        drop : bool, default False
+            Do not try to insert index into dataframe columns. This resets
+            the index to the default integer index.
+        inplace : bool, default False
+            Modify the DataFrame in place (do not create a new object).
+        col_level : int or str, default 0
+            If the columns have multiple levels, determines which level the
+            labels are inserted into. By default it is inserted into the first
+            level.
+        col_fill : object, default ''
+            If the columns have multiple levels, determines how the other
+            levels are named. If None then the index name is repeated.
+
+        Returns
+        -------
+        DataFrame
+            DataFrame with the new index.
+
+        See Also
+        --------
+        DataFrame.set_index : Opposite of reset_index.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([('bird', 389.0),
+        ...                    ('bird', 24.0),
+        ...                    ('mammal', 80.5),
+        ...                    ('mammal', np.nan)],
+        ...                   index=['falcon', 'parrot', 'lion', 'monkey'],
+        ...                   columns=('class', 'max_speed'))
+        >>> df
+                 class  max_speed
+        falcon    bird      389.0
+        parrot    bird       24.0
+        lion    mammal       80.5
+        monkey  mammal        NaN
+
+        When we reset the index, the old index is added as a column. Unlike pandas, pandas-on-Spark
+        does not automatically add a sequential index. The following 0, 1, 2, 3 are only
+        there when we display the DataFrame.
+
+        >>> df.reset_index()
+            index   class  max_speed
+        0  falcon    bird      389.0
+        1  parrot    bird       24.0
+        2    lion  mammal       80.5
+        3  monkey  mammal        NaN
+
+        We can use the `drop` parameter to avoid the old index being added as
+        a column:
+
+        >>> df.reset_index(drop=True)
+            class  max_speed
+        0    bird      389.0
+        1    bird       24.0
+        2  mammal       80.5
+        3  mammal        NaN
+
+        You can also use `reset_index` with `MultiIndex`.
+
+        >>> index = pd.MultiIndex.from_tuples([('bird', 'falcon'),
+        ...                                    ('bird', 'parrot'),
+        ...                                    ('mammal', 'lion'),
+        ...                                    ('mammal', 'monkey')],
+        ...                                   names=['class', 'name'])
+        >>> columns = pd.MultiIndex.from_tuples([('speed', 'max'),
+        ...                                      ('species', 'type')])
+        >>> df = ps.DataFrame([(389.0, 'fly'),
+        ...                    ( 24.0, 'fly'),
+        ...                    ( 80.5, 'run'),
+        ...                    (np.nan, 'jump')],
+        ...                   index=index,
+        ...                   columns=columns)
+        >>> df  # doctest: +NORMALIZE_WHITESPACE
+                       speed species
+                         max    type
+        class  name
+        bird   falcon  389.0     fly
+               parrot   24.0     fly
+        mammal lion     80.5     run
+               monkey    NaN    jump
+
+        If the index has multiple levels, we can reset a subset of them:
+
+        >>> df.reset_index(level='class')  # doctest: +NORMALIZE_WHITESPACE
+                 class  speed species
+                          max    type
+        name
+        falcon    bird  389.0     fly
+        parrot    bird   24.0     fly
+        lion    mammal   80.5     run
+        monkey  mammal    NaN    jump
+
+        If we are not dropping the index, by default, it is placed in the top
+        level. We can place it in another level:
+
+        >>> df.reset_index(level='class', col_level=1)  # doctest: +NORMALIZE_WHITESPACE
+                        speed species
+                 class    max    type
+        name
+        falcon    bird  389.0     fly
+        parrot    bird   24.0     fly
+        lion    mammal   80.5     run
+        monkey  mammal    NaN    jump
+
+        When the index is inserted under another level, we can specify under
+        which one with the parameter `col_fill`:
+
+        >>> df.reset_index(level='class', col_level=1,
+        ...                col_fill='species')  # doctest: +NORMALIZE_WHITESPACE
+                      species  speed species
+                        class    max    type
+        name
+        falcon           bird  389.0     fly
+        parrot           bird   24.0     fly
+        lion           mammal   80.5     run
+        monkey         mammal    NaN    jump
+
+        If we specify a nonexistent level for `col_fill`, it is created:
+
+        >>> df.reset_index(level='class', col_level=1,
+        ...                col_fill='genus')  # doctest: +NORMALIZE_WHITESPACE
+                        genus  speed species
+                        class    max    type
+        name
+        falcon           bird  389.0     fly
+        parrot           bird   24.0     fly
+        lion           mammal   80.5     run
+        monkey         mammal    NaN    jump
+        """
+        inplace = validate_bool_kwarg(inplace, "inplace")
+        multi_index = self._internal.index_level > 1
+
+        def rename(index):
+            if multi_index:
+                return ("level_{}".format(index),)
+            else:
+                if ("index",) not in self._internal.column_labels:
+                    return ("index",)
+                else:
+                    return ("level_{}".format(index),)
+
+        if level is None:
+            new_column_labels = [
+                name if name is not None else rename(i)
+                for i, name in enumerate(self._internal.index_names)
+            ]
+            new_data_spark_columns = [
+                scol.alias(name_like_string(label))
+                for scol, label in zip(self._internal.index_spark_columns, new_column_labels)
+            ]
+            new_data_dtypes = self._internal.index_dtypes
+
+            index_spark_columns = []
+            index_names = []
+            index_dtypes = []
+        else:
+            if is_list_like(level):
+                level = list(level)
+            if isinstance(level, int) or is_name_like_tuple(level):
+                level = [level]
+            elif is_name_like_value(level):
+                level = [(level,)]
+            else:
+                level = [
+                    lvl if isinstance(lvl, int) or is_name_like_tuple(lvl) else (lvl,)
+                    for lvl in level
+                ]
+
+            if all(isinstance(l, int) for l in level):
+                for lev in level:
+                    if lev >= self._internal.index_level:
+                        raise IndexError(
+                            "Too many levels: Index has only {} level, not {}".format(
+                                self._internal.index_level, lev + 1
+                            )
+                        )
+                idx = level
+            elif all(is_name_like_tuple(lev) for lev in level):
+                idx = []
+                for l in level:
+                    try:
+                        i = self._internal.index_names.index(l)
+                        idx.append(i)
+                    except ValueError:
+                        if multi_index:
+                            raise KeyError("Level unknown not found")
+                        else:
+                            raise KeyError(
+                                "Level unknown must be same as name ({})".format(
+                                    name_like_string(self._internal.index_names[0])
+                                )
+                            )
+            else:
+                raise ValueError("Level should be all int or all string.")
+            idx.sort()
+
+            new_column_labels = []
+            new_data_spark_columns = []
+            new_data_dtypes = []
+
+            index_spark_columns = self._internal.index_spark_columns.copy()
+            index_names = self._internal.index_names.copy()
+            index_dtypes = self._internal.index_dtypes.copy()
+
+            for i in idx[::-1]:
+                name = index_names.pop(i)
+                new_column_labels.insert(0, name if name is not None else rename(i))
+
+                scol = index_spark_columns.pop(i)
+                new_data_spark_columns.insert(0, scol.alias(name_like_string(name)))
+
+                new_data_dtypes.insert(0, index_dtypes.pop(i))
+
+        if drop:
+            new_data_spark_columns = []
+            new_column_labels = []
+            new_data_dtypes = []
+
+        for label in new_column_labels:
+            if label in self._internal.column_labels:
+                raise ValueError("cannot insert {}, already exists".format(name_like_string(label)))
+
+        if self._internal.column_labels_level > 1:
+            column_depth = len(self._internal.column_labels[0])
+            if col_level >= column_depth:
+                raise IndexError(
+                    "Too many levels: Index has only {} levels, not {}".format(
+                        column_depth, col_level + 1
+                    )
+                )
+            if any(col_level + len(label) > column_depth for label in new_column_labels):
+                raise ValueError("Item must have length equal to number of levels.")
+            new_column_labels = [
+                tuple(
+                    ([col_fill] * col_level)
+                    + list(label)
+                    + ([col_fill] * (column_depth - (len(label) + col_level)))
+                )
+                for label in new_column_labels
+            ]
+
+        internal = self._internal.copy(
+            index_spark_columns=index_spark_columns,
+            index_names=index_names,
+            index_dtypes=index_dtypes,
+            column_labels=new_column_labels + self._internal.column_labels,
+            data_spark_columns=new_data_spark_columns + self._internal.data_spark_columns,
+            data_dtypes=new_data_dtypes + self._internal.data_dtypes,
+        )
+
+        if inplace:
+            self._update_internal_frame(internal)
+            return None
+        else:
+            return DataFrame(internal)
+
+    def isnull(self) -> "DataFrame":
+        """
+        Detects missing values for items in the current Dataframe.
+
+        Return a boolean same-sized Dataframe indicating if the values are NA.
+        NA values, such as None or numpy.NaN, gets mapped to True values.
+        Everything else gets mapped to False values.
+
+        See Also
+        --------
+        DataFrame.notnull
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([(.2, .3), (.0, None), (.6, None), (.2, .1)])
+        >>> df.isnull()
+               0      1
+        0  False  False
+        1  False   True
+        2  False   True
+        3  False  False
+
+        >>> df = ps.DataFrame([[None, 'bee', None], ['dog', None, 'fly']])
+        >>> df.isnull()
+               0      1      2
+        0   True  False   True
+        1  False   True  False
+        """
+        return self._apply_series_op(lambda kser: kser.isnull())
+
+    isna = isnull
+
+    def notnull(self) -> "DataFrame":
+        """
+        Detects non-missing values for items in the current Dataframe.
+
+        This function takes a dataframe and indicates whether it's
+        values are valid (not missing, which is ``NaN`` in numeric
+        datatypes, ``None`` or ``NaN`` in objects and ``NaT`` in datetimelike).
+
+        See Also
+        --------
+        DataFrame.isnull
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([(.2, .3), (.0, None), (.6, None), (.2, .1)])
+        >>> df.notnull()
+              0      1
+        0  True   True
+        1  True  False
+        2  True  False
+        3  True   True
+
+        >>> df = ps.DataFrame([['ant', 'bee', 'cat'], ['dog', None, 'fly']])
+        >>> df.notnull()
+              0      1     2
+        0  True   True  True
+        1  True  False  True
+        """
+        return self._apply_series_op(lambda kser: kser.notnull())
+
+    notna = notnull
+
+    def insert(
+        self,
+        loc: int,
+        column,
+        value: Union[Scalar, "Series", Iterable],
+        allow_duplicates: bool = False,
+    ) -> None:
+        """
+        Insert column into DataFrame at specified location.
+
+        Raises a ValueError if `column` is already contained in the DataFrame,
+        unless `allow_duplicates` is set to True.
+
+        Parameters
+        ----------
+        loc : int
+            Insertion index. Must verify 0 <= loc <= len(columns).
+        column : str, number, or hashable object
+            Label of the inserted column.
+        value : int, Series, or array-like
+        allow_duplicates : bool, optional
+
+        Examples
+        --------
+        >>> kdf = ps.DataFrame([1, 2, 3])
+        >>> kdf.sort_index()
+           0
+        0  1
+        1  2
+        2  3
+        >>> kdf.insert(0, 'x', 4)
+        >>> kdf.sort_index()
+           x  0
+        0  4  1
+        1  4  2
+        2  4  3
+
+        >>> from pyspark.pandas.config import set_option, reset_option
+        >>> set_option("compute.ops_on_diff_frames", True)
+
+        >>> kdf.insert(1, 'y', [5, 6, 7])
+        >>> kdf.sort_index()
+           x  y  0
+        0  4  5  1
+        1  4  6  2
+        2  4  7  3
+
+        >>> kdf.insert(2, 'z', ps.Series([8, 9, 10]))
+        >>> kdf.sort_index()
+           x  y   z  0
+        0  4  5   8  1
+        1  4  6   9  2
+        2  4  7  10  3
+
+        >>> reset_option("compute.ops_on_diff_frames")
+        """
+        if not isinstance(loc, int):
+            raise TypeError("loc must be int")
+
+        assert 0 <= loc <= len(self.columns)
+        assert allow_duplicates is False
+
+        if not is_name_like_value(column):
+            raise TypeError(
+                '"column" should be a scalar value or tuple that contains scalar values'
+            )
+
+        if is_name_like_tuple(column):
+            if len(column) != len(self.columns.levels):
+                # To be consistent with pandas
+                raise ValueError('"column" must have length equal to number of column levels.')
+
+        if column in self.columns:
+            raise ValueError("cannot insert %s, already exists" % column)
+
+        kdf = self.copy()
+        kdf[column] = value
+        columns = kdf.columns[:-1].insert(loc, kdf.columns[-1])
+        kdf = kdf[columns]
+        self._update_internal_frame(kdf._internal)
+
+    # TODO: add frep and axis parameter
+    def shift(self, periods=1, fill_value=None) -> "DataFrame":
+        """
+        Shift DataFrame by desired number of periods.
+
+        .. note:: the current implementation of shift uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Parameters
+        ----------
+        periods : int
+            Number of periods to shift. Can be positive or negative.
+        fill_value : object, optional
+            The scalar value to use for newly introduced missing values.
+            The default depends on the dtype of self. For numeric data, np.nan is used.
+
+        Returns
+        -------
+        Copy of input DataFrame, shifted.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'Col1': [10, 20, 15, 30, 45],
+        ...                    'Col2': [13, 23, 18, 33, 48],
+        ...                    'Col3': [17, 27, 22, 37, 52]},
+        ...                   columns=['Col1', 'Col2', 'Col3'])
+
+        >>> df.shift(periods=3)
+           Col1  Col2  Col3
+        0   NaN   NaN   NaN
+        1   NaN   NaN   NaN
+        2   NaN   NaN   NaN
+        3  10.0  13.0  17.0
+        4  20.0  23.0  27.0
+
+        >>> df.shift(periods=3, fill_value=0)
+           Col1  Col2  Col3
+        0     0     0     0
+        1     0     0     0
+        2     0     0     0
+        3    10    13    17
+        4    20    23    27
+
+        """
+        return self._apply_series_op(
+            lambda kser: kser._shift(periods, fill_value), should_resolve=True
+        )
+
+    # TODO: axis should support 1 or 'columns' either at this moment
+    def diff(self, periods: int = 1, axis: Union[int, str] = 0) -> "DataFrame":
+        """
+        First discrete difference of element.
+
+        Calculates the difference of a DataFrame element compared with another element in the
+        DataFrame (default is the element in the same column of the previous row).
+
+        .. note:: the current implementation of diff uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Parameters
+        ----------
+        periods : int, default 1
+            Periods to shift for calculating difference, accepts negative values.
+        axis : int, default 0 or 'index'
+            Can only be set to 0 at the moment.
+
+        Returns
+        -------
+        diffed : DataFrame
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'a': [1, 2, 3, 4, 5, 6],
+        ...                    'b': [1, 1, 2, 3, 5, 8],
+        ...                    'c': [1, 4, 9, 16, 25, 36]}, columns=['a', 'b', 'c'])
+        >>> df
+           a  b   c
+        0  1  1   1
+        1  2  1   4
+        2  3  2   9
+        3  4  3  16
+        4  5  5  25
+        5  6  8  36
+
+        >>> df.diff()
+             a    b     c
+        0  NaN  NaN   NaN
+        1  1.0  0.0   3.0
+        2  1.0  1.0   5.0
+        3  1.0  1.0   7.0
+        4  1.0  2.0   9.0
+        5  1.0  3.0  11.0
+
+        Difference with previous column
+
+        >>> df.diff(periods=3)
+             a    b     c
+        0  NaN  NaN   NaN
+        1  NaN  NaN   NaN
+        2  NaN  NaN   NaN
+        3  3.0  2.0  15.0
+        4  3.0  4.0  21.0
+        5  3.0  6.0  27.0
+
+        Difference with following row
+
+        >>> df.diff(periods=-1)
+             a    b     c
+        0 -1.0  0.0  -3.0
+        1 -1.0 -1.0  -5.0
+        2 -1.0 -1.0  -7.0
+        3 -1.0 -2.0  -9.0
+        4 -1.0 -3.0 -11.0
+        5  NaN  NaN   NaN
+        """
+        axis = validate_axis(axis)
+        if axis != 0:
+            raise NotImplementedError('axis should be either 0 or "index" currently.')
+
+        return self._apply_series_op(lambda kser: kser._diff(periods), should_resolve=True)
+
+    # TODO: axis should support 1 or 'columns' either at this moment
+    def nunique(
+        self,
+        axis: Union[int, str] = 0,
+        dropna: bool = True,
+        approx: bool = False,
+        rsd: float = 0.05,
+    ) -> "Series":
+        """
+        Return number of unique elements in the object.
+
+        Excludes NA values by default.
+
+        Parameters
+        ----------
+        axis : int, default 0 or 'index'
+            Can only be set to 0 at the moment.
+        dropna : bool, default True
+            Don’t include NaN in the count.
+        approx: bool, default False
+            If False, will use the exact algorithm and return the exact number of unique.
+            If True, it uses the HyperLogLog approximate algorithm, which is significantly faster
+            for large amount of data.
+            Note: This parameter is specific to pandas-on-Spark and is not found in pandas.
+        rsd: float, default 0.05
+            Maximum estimation error allowed in the HyperLogLog algorithm.
+            Note: Just like ``approx`` this parameter is specific to pandas-on-Spark.
+
+        Returns
+        -------
+        The number of unique values per column as a pandas-on-Spark Series.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'A': [1, 2, 3], 'B': [np.nan, 3, np.nan]})
+        >>> df.nunique()
+        A    3
+        B    1
+        dtype: int64
+
+        >>> df.nunique(dropna=False)
+        A    3
+        B    2
+        dtype: int64
+
+        On big data, we recommend using the approximate algorithm to speed up this function.
+        The result will be very close to the exact unique count.
+
+        >>> df.nunique(approx=True)
+        A    3
+        B    1
+        dtype: int64
+        """
+        from pyspark.pandas.series import first_series
+
+        axis = validate_axis(axis)
+        if axis != 0:
+            raise NotImplementedError('axis should be either 0 or "index" currently.')
+        sdf = self._internal.spark_frame.select(
+            [F.lit(None).cast(StringType()).alias(SPARK_DEFAULT_INDEX_NAME)]
+            + [
+                self._kser_for(label)._nunique(dropna, approx, rsd)
+                for label in self._internal.column_labels
+            ]
+        )
+
+        # The data is expected to be small so it's fine to transpose/use default index.
+        with ps.option_context("compute.max_rows", 1):
+            internal = self._internal.copy(
+                spark_frame=sdf,
+                index_spark_columns=[scol_for(sdf, SPARK_DEFAULT_INDEX_NAME)],
+                index_names=[None],
+                index_dtypes=[None],
+                data_spark_columns=[
+                    scol_for(sdf, col) for col in self._internal.data_spark_column_names
+                ],
+                data_dtypes=None,
+            )
+            return first_series(DataFrame(internal).transpose())
+
+    def round(self, decimals=0) -> "DataFrame":
+        """
+        Round a DataFrame to a variable number of decimal places.
+
+        Parameters
+        ----------
+        decimals : int, dict, Series
+            Number of decimal places to round each column to. If an int is
+            given, round each column to the same number of places.
+            Otherwise dict and Series round to variable numbers of places.
+            Column names should be in the keys if `decimals` is a
+            dict-like, or in the index if `decimals` is a Series. Any
+            columns not included in `decimals` will be left as is. Elements
+            of `decimals` which are not columns of the input will be
+            ignored.
+
+            .. note:: If `decimals` is a Series, it is expected to be small,
+                as all the data is loaded into the driver's memory.
+
+        Returns
+        -------
+        DataFrame
+
+        See Also
+        --------
+        Series.round
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'A':[0.028208, 0.038683, 0.877076],
+        ...                    'B':[0.992815, 0.645646, 0.149370],
+        ...                    'C':[0.173891, 0.577595, 0.491027]},
+        ...                    columns=['A', 'B', 'C'],
+        ...                    index=['first', 'second', 'third'])
+        >>> df
+                       A         B         C
+        first   0.028208  0.992815  0.173891
+        second  0.038683  0.645646  0.577595
+        third   0.877076  0.149370  0.491027
+
+        >>> df.round(2)
+                   A     B     C
+        first   0.03  0.99  0.17
+        second  0.04  0.65  0.58
+        third   0.88  0.15  0.49
+
+        >>> df.round({'A': 1, 'C': 2})
+                  A         B     C
+        first   0.0  0.992815  0.17
+        second  0.0  0.645646  0.58
+        third   0.9  0.149370  0.49
+
+        >>> decimals = ps.Series([1, 0, 2], index=['A', 'B', 'C'])
+        >>> df.round(decimals)
+                  A    B     C
+        first   0.0  1.0  0.17
+        second  0.0  1.0  0.58
+        third   0.9  0.0  0.49
+        """
+        if isinstance(decimals, ps.Series):
+            decimals = {
+                k if isinstance(k, tuple) else (k,): v
+                for k, v in decimals._to_internal_pandas().items()
+            }
+        elif isinstance(decimals, dict):
+            decimals = {k if is_name_like_tuple(k) else (k,): v for k, v in decimals.items()}
+        elif isinstance(decimals, int):
+            decimals = {k: decimals for k in self._internal.column_labels}
+        else:
+            raise TypeError("decimals must be an integer, a dict-like or a Series")
+
+        def op(kser):
+            label = kser._column_label
+            if label in decimals:
+                return F.round(kser.spark.column, decimals[label]).alias(
+                    kser._internal.data_spark_column_names[0]
+                )
+            else:
+                return kser
+
+        return self._apply_series_op(op)
+
+    def _mark_duplicates(self, subset=None, keep="first"):
+        if subset is None:
+            subset = self._internal.column_labels
+        else:
+            if is_name_like_tuple(subset):
+                subset = [subset]
+            elif is_name_like_value(subset):
+                subset = [(subset,)]
+            else:
+                subset = [sub if is_name_like_tuple(sub) else (sub,) for sub in subset]
+            diff = set(subset).difference(set(self._internal.column_labels))
+            if len(diff) > 0:
+                raise KeyError(", ".join([name_like_string(d) for d in diff]))
+        group_cols = [self._internal.spark_column_name_for(label) for label in subset]
+
+        sdf = self._internal.resolved_copy.spark_frame
+
+        column = verify_temp_column_name(sdf, "__duplicated__")
+
+        if keep == "first" or keep == "last":
+            if keep == "first":
+                ord_func = spark.functions.asc
+            else:
+                ord_func = spark.functions.desc
+            window = (
+                Window.partitionBy(group_cols)
+                .orderBy(ord_func(NATURAL_ORDER_COLUMN_NAME))
+                .rowsBetween(Window.unboundedPreceding, Window.currentRow)
+            )
+            sdf = sdf.withColumn(column, F.row_number().over(window) > 1)
+        elif not keep:
+            window = Window.partitionBy(group_cols).rowsBetween(
+                Window.unboundedPreceding, Window.unboundedFollowing
+            )
+            sdf = sdf.withColumn(column, F.count("*").over(window) > 1)
+        else:
+            raise ValueError("'keep' only supports 'first', 'last' and False")
+        return sdf, column
+
+    def duplicated(self, subset=None, keep="first") -> "Series":
+        """
+        Return boolean Series denoting duplicate rows, optionally only considering certain columns.
+
+        Parameters
+        ----------
+        subset : column label or sequence of labels, optional
+            Only consider certain columns for identifying duplicates,
+            by default use all of the columns
+        keep : {'first', 'last', False}, default 'first'
+           - ``first`` : Mark duplicates as ``True`` except for the first occurrence.
+           - ``last`` : Mark duplicates as ``True`` except for the last occurrence.
+           - False : Mark all duplicates as ``True``.
+
+        Returns
+        -------
+        duplicated : Series
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'a': [1, 1, 1, 3], 'b': [1, 1, 1, 4], 'c': [1, 1, 1, 5]},
+        ...                   columns = ['a', 'b', 'c'])
+        >>> df
+           a  b  c
+        0  1  1  1
+        1  1  1  1
+        2  1  1  1
+        3  3  4  5
+
+        >>> df.duplicated().sort_index()
+        0    False
+        1     True
+        2     True
+        3    False
+        dtype: bool
+
+        Mark duplicates as ``True`` except for the last occurrence.
+
+        >>> df.duplicated(keep='last').sort_index()
+        0     True
+        1     True
+        2    False
+        3    False
+        dtype: bool
+
+        Mark all duplicates as ``True``.
+
+        >>> df.duplicated(keep=False).sort_index()
+        0     True
+        1     True
+        2     True
+        3    False
+        dtype: bool
+        """
+        from pyspark.pandas.series import first_series
+
+        sdf, column = self._mark_duplicates(subset, keep)
+
+        sdf = sdf.select(
+            self._internal.index_spark_columns
+            + [scol_for(sdf, column).alias(SPARK_DEFAULT_SERIES_NAME)]
+        )
+        return first_series(
+            DataFrame(
+                InternalFrame(
+                    spark_frame=sdf,
+                    index_spark_columns=[
+                        scol_for(sdf, col) for col in self._internal.index_spark_column_names
+                    ],
+                    index_names=self._internal.index_names,
+                    index_dtypes=self._internal.index_dtypes,
+                    column_labels=[None],  # type: ignore
+                    data_spark_columns=[scol_for(sdf, SPARK_DEFAULT_SERIES_NAME)],
+                )
+            )
+        )
+
+    # TODO: support other as DataFrame or array-like
+    def dot(self, other: "Series") -> "Series":
+        """
+        Compute the matrix multiplication between the DataFrame and other.
+
+        This method computes the matrix product between the DataFrame and the
+        values of an other Series
+
+        It can also be called using ``self @ other`` in Python >= 3.5.
+
+        .. note:: This method is based on an expensive operation due to the nature
+            of big data. Internally it needs to generate each row for each value, and
+            then group twice - it is a huge operation. To prevent misusage, this method
+            has the 'compute.max_rows' default limit of input length, and raises a ValueError.
+
+                >>> from pyspark.pandas.config import option_context
+                >>> with option_context(
+                ...     'compute.max_rows', 1000, "compute.ops_on_diff_frames", True
+                ... ):  # doctest: +NORMALIZE_WHITESPACE
+                ...     kdf = ps.DataFrame({'a': range(1001)})
+                ...     kser = ps.Series([2], index=['a'])
+                ...     kdf.dot(kser)
+                Traceback (most recent call last):
+                  ...
+                ValueError: Current DataFrame has more then the given limit 1000 rows.
+                Please set 'compute.max_rows' by using 'pyspark.pandas.config.set_option'
+                to retrieve to retrieve more than 1000 rows. Note that, before changing the
+                'compute.max_rows', this operation is considerably expensive.
+
+        Parameters
+        ----------
+        other : Series
+            The other object to compute the matrix product with.
+
+        Returns
+        -------
+        Series
+            Return the matrix product between self and other as a Series.
+
+        See Also
+        --------
+        Series.dot: Similar method for Series.
+
+        Notes
+        -----
+        The dimensions of DataFrame and other must be compatible in order to
+        compute the matrix multiplication. In addition, the column names of
+        DataFrame and the index of other must contain the same values, as they
+        will be aligned prior to the multiplication.
+
+        The dot method for Series computes the inner product, instead of the
+        matrix product here.
+
+        Examples
+        --------
+        >>> from pyspark.pandas.config import set_option, reset_option
+        >>> set_option("compute.ops_on_diff_frames", True)
+        >>> kdf = ps.DataFrame([[0, 1, -2, -1], [1, 1, 1, 1]])
+        >>> kser = ps.Series([1, 1, 2, 1])
+        >>> kdf.dot(kser)
+        0   -4
+        1    5
+        dtype: int64
+
+        Note how shuffling of the objects does not change the result.
+
+        >>> kser2 = kser.reindex([1, 0, 2, 3])
+        >>> kdf.dot(kser2)
+        0   -4
+        1    5
+        dtype: int64
+        >>> kdf @ kser2
+        0   -4
+        1    5
+        dtype: int64
+        >>> reset_option("compute.ops_on_diff_frames")
+        """
+        if not isinstance(other, ps.Series):
+            raise TypeError("Unsupported type {}".format(type(other).__name__))
+        else:
+            return cast(ps.Series, other.dot(self.transpose())).rename(None)
+
+    def __matmul__(self, other):
+        """
+        Matrix multiplication using binary `@` operator in Python>=3.5.
+        """
+        return self.dot(other)
+
+    def to_koalas(self, index_col: Optional[Union[str, List[str]]] = None) -> "DataFrame":
+        """
+        Converts the existing DataFrame into a pandas-on-Spark DataFrame.
+
+        This method is monkey-patched into Spark's DataFrame and can be used
+        to convert a Spark DataFrame into a pandas-on-Spark DataFrame. If running on
+        an existing pandas-on-Spark DataFrame, the method returns itself.
+
+        If a pandas-on-Spark DataFrame is converted to a Spark DataFrame and then back
+        to pandas-on-Spark, it will lose the index information and the original index
+        will be turned into a normal column.
+
+        Parameters
+        ----------
+        index_col: str or list of str, optional, default: None
+            Index column of table in Spark.
+
+        See Also
+        --------
+        DataFrame.to_spark
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'col1': [1, 2], 'col2': [3, 4]}, columns=['col1', 'col2'])
+        >>> df
+           col1  col2
+        0     1     3
+        1     2     4
+
+        >>> spark_df = df.to_spark()
+        >>> spark_df
+        DataFrame[col1: bigint, col2: bigint]
+
+        >>> kdf = spark_df.to_koalas()
+        >>> kdf
+           col1  col2
+        0     1     3
+        1     2     4
+
+        We can specify the index columns.
+
+        >>> kdf = spark_df.to_koalas(index_col='col1')
+        >>> kdf  # doctest: +NORMALIZE_WHITESPACE
+              col2
+        col1
+        1        3
+        2        4
+
+        Calling to_koalas on a pandas-on-Spark DataFrame simply returns itself.
+
+        >>> df.to_koalas()
+           col1  col2
+        0     1     3
+        1     2     4
+        """
+        if isinstance(self, DataFrame):
+            return self
+        else:
+            assert isinstance(self, spark.DataFrame), type(self)
+            from pyspark.pandas.namespace import _get_index_map
+
+            index_spark_columns, index_names = _get_index_map(self, index_col)
+            internal = InternalFrame(
+                spark_frame=self, index_spark_columns=index_spark_columns, index_names=index_names
+            )
+            return DataFrame(internal)
+
+    def cache(self) -> "CachedDataFrame":
+        warnings.warn(
+            "DataFrame.cache is deprecated as of DataFrame.spark.cache. "
+            "Please use the API instead.",
+            FutureWarning,
+        )
+        return self.spark.cache()
+
+    cache.__doc__ = SparkFrameMethods.cache.__doc__
+
+    def persist(self, storage_level=StorageLevel.MEMORY_AND_DISK) -> "CachedDataFrame":
+        warnings.warn(
+            "DataFrame.persist is deprecated as of DataFrame.spark.persist. "
+            "Please use the API instead.",
+            FutureWarning,
+        )
+        return self.spark.persist(storage_level)
+
+    persist.__doc__ = SparkFrameMethods.persist.__doc__
+
+    def hint(self, name: str, *parameters) -> "DataFrame":
+        warnings.warn(
+            "DataFrame.hint is deprecated as of DataFrame.spark.hint. "
+            "Please use the API instead.",
+            FutureWarning,
+        )
+        return self.spark.hint(name, *parameters)
+
+    hint.__doc__ = SparkFrameMethods.hint.__doc__
+
+    def to_table(
+        self,
+        name: str,
+        format: Optional[str] = None,
+        mode: str = "overwrite",
+        partition_cols: Optional[Union[str, List[str]]] = None,
+        index_col: Optional[Union[str, List[str]]] = None,
+        **options
+    ) -> None:
+        return self.spark.to_table(name, format, mode, partition_cols, index_col, **options)
+
+    to_table.__doc__ = SparkFrameMethods.to_table.__doc__
+
+    def to_delta(
+        self,
+        path: str,
+        mode: str = "overwrite",
+        partition_cols: Optional[Union[str, List[str]]] = None,
+        index_col: Optional[Union[str, List[str]]] = None,
+        **options
+    ) -> None:
+        """
+        Write the DataFrame out as a Delta Lake table.
+
+        Parameters
+        ----------
+        path : str, required
+            Path to write to.
+        mode : str {'append', 'overwrite', 'ignore', 'error', 'errorifexists'}, default
+            'overwrite'. Specifies the behavior of the save operation when the destination
+            exists already.
+
+            - 'append': Append the new data to existing data.
+            - 'overwrite': Overwrite existing data.
+            - 'ignore': Silently ignore this operation if data already exists.
+            - 'error' or 'errorifexists': Throw an exception if data already exists.
+
+        partition_cols : str or list of str, optional, default None
+            Names of partitioning columns
+        index_col: str or list of str, optional, default: None
+            Column names to be used in Spark to represent pandas-on-Spark's index. The index name
+            in pandas-on-Spark is ignored. By default, the index is always lost.
+        options : dict
+            All other options passed directly into Delta Lake.
+
+        See Also
+        --------
+        read_delta
+        DataFrame.to_parquet
+        DataFrame.to_table
+        DataFrame.to_spark_io
+
+        Examples
+        --------
+
+        >>> df = ps.DataFrame(dict(
+        ...    date=list(pd.date_range('2012-1-1 12:00:00', periods=3, freq='M')),
+        ...    country=['KR', 'US', 'JP'],
+        ...    code=[1, 2 ,3]), columns=['date', 'country', 'code'])
+        >>> df
+                         date country  code
+        0 2012-01-31 12:00:00      KR     1
+        1 2012-02-29 12:00:00      US     2
+        2 2012-03-31 12:00:00      JP     3
+
+        Create a new Delta Lake table, partitioned by one column:
+
+        >>> df.to_delta('%s/to_delta/foo' % path, partition_cols='date')  # doctest: +SKIP
+
+        Partitioned by two columns:
+
+        >>> df.to_delta('%s/to_delta/bar' % path,
+        ...             partition_cols=['date', 'country'])  # doctest: +SKIP
+
+        Overwrite an existing table's partitions, using the 'replaceWhere' capability in Delta:
+
+        >>> df.to_delta('%s/to_delta/bar' % path,
+        ...             mode='overwrite', replaceWhere='date >= "2012-01-01"')  # doctest: +SKIP
+        """
+        if "options" in options and isinstance(options.get("options"), dict) and len(options) == 1:
+            options = options.get("options")  # type: ignore
+
+        self.spark.to_spark_io(
+            path=path,
+            mode=mode,
+            format="delta",
+            partition_cols=partition_cols,
+            index_col=index_col,
+            **options,
+        )
+
+    def to_parquet(
+        self,
+        path: str,
+        mode: str = "overwrite",
+        partition_cols: Optional[Union[str, List[str]]] = None,
+        compression: Optional[str] = None,
+        index_col: Optional[Union[str, List[str]]] = None,
+        **options
+    ) -> None:
+        """
+        Write the DataFrame out as a Parquet file or directory.
+
+        Parameters
+        ----------
+        path : str, required
+            Path to write to.
+        mode : str {'append', 'overwrite', 'ignore', 'error', 'errorifexists'},
+            default 'overwrite'. Specifies the behavior of the save operation when the
+            destination exists already.
+
+            - 'append': Append the new data to existing data.
+            - 'overwrite': Overwrite existing data.
+            - 'ignore': Silently ignore this operation if data already exists.
+            - 'error' or 'errorifexists': Throw an exception if data already exists.
+
+        partition_cols : str or list of str, optional, default None
+            Names of partitioning columns
+        compression : str {'none', 'uncompressed', 'snappy', 'gzip', 'lzo', 'brotli', 'lz4', 'zstd'}
+            Compression codec to use when saving to file. If None is set, it uses the
+            value specified in `spark.sql.parquet.compression.codec`.
+        index_col: str or list of str, optional, default: None
+            Column names to be used in Spark to represent pandas-on-Spark's index. The index name
+            in pandas-on-Spark is ignored. By default, the index is always lost.
+        options : dict
+            All other options passed directly into Spark's data source.
+
+        See Also
+        --------
+        read_parquet
+        DataFrame.to_delta
+        DataFrame.to_table
+        DataFrame.to_spark_io
+
+        Examples
+        --------
+        >>> df = ps.DataFrame(dict(
+        ...    date=list(pd.date_range('2012-1-1 12:00:00', periods=3, freq='M')),
+        ...    country=['KR', 'US', 'JP'],
+        ...    code=[1, 2 ,3]), columns=['date', 'country', 'code'])
+        >>> df
+                         date country  code
+        0 2012-01-31 12:00:00      KR     1
+        1 2012-02-29 12:00:00      US     2
+        2 2012-03-31 12:00:00      JP     3
+
+        >>> df.to_parquet('%s/to_parquet/foo.parquet' % path, partition_cols='date')
+
+        >>> df.to_parquet(
+        ...     '%s/to_parquet/foo.parquet' % path,
+        ...     mode = 'overwrite',
+        ...     partition_cols=['date', 'country'])
+        """
+        if "options" in options and isinstance(options.get("options"), dict) and len(options) == 1:
+            options = options.get("options")  # type: ignore
+
+        builder = self.to_spark(index_col=index_col).write.mode(mode)
+        if partition_cols is not None:
+            builder.partitionBy(partition_cols)
+        builder._set_opts(compression=compression)
+        builder.options(**options).format("parquet").save(path)
+
+    def to_orc(
+        self,
+        path: str,
+        mode: str = "overwrite",
+        partition_cols: Optional[Union[str, List[str]]] = None,
+        index_col: Optional[Union[str, List[str]]] = None,
+        **options
+    ) -> None:
+        """
+        Write the DataFrame out as a ORC file or directory.
+
+        Parameters
+        ----------
+        path : str, required
+            Path to write to.
+        mode : str {'append', 'overwrite', 'ignore', 'error', 'errorifexists'},
+            default 'overwrite'. Specifies the behavior of the save operation when the
+            destination exists already.
+
+            - 'append': Append the new data to existing data.
+            - 'overwrite': Overwrite existing data.
+            - 'ignore': Silently ignore this operation if data already exists.
+            - 'error' or 'errorifexists': Throw an exception if data already exists.
+
+        partition_cols : str or list of str, optional, default None
+            Names of partitioning columns
+        index_col: str or list of str, optional, default: None
+            Column names to be used in Spark to represent pandas-on-Spark's index. The index name
+            in pandas-on-Spark is ignored. By default, the index is always lost.
+        options : dict
+            All other options passed directly into Spark's data source.
+
+        See Also
+        --------
+        read_orc
+        DataFrame.to_delta
+        DataFrame.to_parquet
+        DataFrame.to_table
+        DataFrame.to_spark_io
+
+        Examples
+        --------
+        >>> df = ps.DataFrame(dict(
+        ...    date=list(pd.date_range('2012-1-1 12:00:00', periods=3, freq='M')),
+        ...    country=['KR', 'US', 'JP'],
+        ...    code=[1, 2 ,3]), columns=['date', 'country', 'code'])
+        >>> df
+                         date country  code
+        0 2012-01-31 12:00:00      KR     1
+        1 2012-02-29 12:00:00      US     2
+        2 2012-03-31 12:00:00      JP     3
+
+        >>> df.to_orc('%s/to_orc/foo.orc' % path, partition_cols='date')
+
+        >>> df.to_orc(
+        ...     '%s/to_orc/foo.orc' % path,
+        ...     mode = 'overwrite',
+        ...     partition_cols=['date', 'country'])
+        """
+        if "options" in options and isinstance(options.get("options"), dict) and len(options) == 1:
+            options = options.get("options")  # type: ignore
+
+        self.spark.to_spark_io(
+            path=path,
+            mode=mode,
+            format="orc",
+            partition_cols=partition_cols,
+            index_col=index_col,
+            **options,
+        )
+
+    def to_spark_io(
+        self,
+        path: Optional[str] = None,
+        format: Optional[str] = None,
+        mode: str = "overwrite",
+        partition_cols: Optional[Union[str, List[str]]] = None,
+        index_col: Optional[Union[str, List[str]]] = None,
+        **options
+    ) -> None:
+        return self.spark.to_spark_io(path, format, mode, partition_cols, index_col, **options)
+
+    to_spark_io.__doc__ = SparkFrameMethods.to_spark_io.__doc__
+
+    def to_spark(self, index_col: Optional[Union[str, List[str]]] = None) -> SparkDataFrame:
+        return self.spark.frame(index_col)
+
+    to_spark.__doc__ = SparkFrameMethods.__doc__
+
+    def to_pandas(self) -> pd.DataFrame:
+        """
+        Return a pandas DataFrame.
+
+        .. note:: This method should only be used if the resulting pandas DataFrame is expected
+            to be small, as all the data is loaded into the driver's memory.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([(.2, .3), (.0, .6), (.6, .0), (.2, .1)],
+        ...                   columns=['dogs', 'cats'])
+        >>> df.to_pandas()
+           dogs  cats
+        0   0.2   0.3
+        1   0.0   0.6
+        2   0.6   0.0
+        3   0.2   0.1
+        """
+        return self._internal.to_pandas_frame.copy()
+
+    # Alias to maintain backward compatibility with Spark
+    def toPandas(self) -> pd.DataFrame:
+        warnings.warn(
+            "DataFrame.toPandas is deprecated as of DataFrame.to_pandas. "
+            "Please use the API instead.",
+            FutureWarning,
+        )
+        return self.to_pandas()
+
+    toPandas.__doc__ = to_pandas.__doc__
+
+    def assign(self, **kwargs) -> "DataFrame":
+        """
+        Assign new columns to a DataFrame.
+
+        Returns a new object with all original columns in addition to new ones.
+        Existing columns that are re-assigned will be overwritten.
+
+        Parameters
+        ----------
+        **kwargs : dict of {str: callable, Series or Index}
+            The column names are keywords. If the values are
+            callable, they are computed on the DataFrame and
+            assigned to the new columns. The callable must not
+            change input DataFrame (though pandas-on-Spark doesn't check it).
+            If the values are not callable, (e.g. a Series or a literal),
+            they are simply assigned.
+
+        Returns
+        -------
+        DataFrame
+            A new DataFrame with the new columns in addition to
+            all the existing columns.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'temp_c': [17.0, 25.0]},
+        ...                   index=['Portland', 'Berkeley'])
+        >>> df
+                  temp_c
+        Portland    17.0
+        Berkeley    25.0
+
+        Where the value is a callable, evaluated on `df`:
+
+        >>> df.assign(temp_f=lambda x: x.temp_c * 9 / 5 + 32)
+                  temp_c  temp_f
+        Portland    17.0    62.6
+        Berkeley    25.0    77.0
+
+        Alternatively, the same behavior can be achieved by directly
+        referencing an existing Series or sequence and you can also
+        create multiple columns within the same assign.
+
+        >>> assigned = df.assign(temp_f=df['temp_c'] * 9 / 5 + 32,
+        ...                      temp_k=df['temp_c'] + 273.15,
+        ...                      temp_idx=df.index)
+        >>> assigned[['temp_c', 'temp_f', 'temp_k', 'temp_idx']]
+                  temp_c  temp_f  temp_k  temp_idx
+        Portland    17.0    62.6  290.15  Portland
+        Berkeley    25.0    77.0  298.15  Berkeley
+
+        Notes
+        -----
+        Assigning multiple columns within the same ``assign`` is possible
+        but you cannot refer to newly created or modified columns. This
+        feature is supported in pandas for Python 3.6 and later but not in
+        pandas-on-Spark. In pandas-on-Spark, all items are computed first,
+        and then assigned.
+        """
+        return self._assign(kwargs)
+
+    def _assign(self, kwargs):
+        assert isinstance(kwargs, dict)
+        from pyspark.pandas.indexes import MultiIndex
+        from pyspark.pandas.series import IndexOpsMixin
+
+        for k, v in kwargs.items():
+            is_invalid_assignee = (
+                not (isinstance(v, (IndexOpsMixin, spark.Column)) or callable(v) or is_scalar(v))
+            ) or isinstance(v, MultiIndex)
+            if is_invalid_assignee:
+                raise TypeError(
+                    "Column assignment doesn't support type " "{0}".format(type(v).__name__)
+                )
+            if callable(v):
+                kwargs[k] = v(self)
+
+        pairs = {
+            (k if is_name_like_tuple(k) else (k,)): (
+                (v.spark.column, v.dtype)
+                if isinstance(v, IndexOpsMixin) and not isinstance(v, MultiIndex)
+                else (v, None)
+                if isinstance(v, spark.Column)
+                else (F.lit(v), None)
+            )
+            for k, v in kwargs.items()
+        }
+
+        scols = []
+        data_dtypes = []
+        for label in self._internal.column_labels:
+            for i in range(len(label)):
+                if label[: len(label) - i] in pairs:
+                    scol, dtype = pairs[label[: len(label) - i]]
+                    scol = scol.alias(self._internal.spark_column_name_for(label))
+                    break
+            else:
+                scol = self._internal.spark_column_for(label)
+                dtype = self._internal.dtype_for(label)
+            scols.append(scol)
+            data_dtypes.append(dtype)
+
+        column_labels = self._internal.column_labels.copy()
+        for label, (scol, dtype) in pairs.items():
+            if label not in set(i[: len(label)] for i in self._internal.column_labels):
+                scols.append(scol.alias(name_like_string(label)))
+                column_labels.append(label)
+                data_dtypes.append(dtype)
+
+        level = self._internal.column_labels_level
+        column_labels = [
+            tuple(list(label) + ([""] * (level - len(label)))) for label in column_labels
+        ]
+
+        internal = self._internal.with_new_columns(
+            scols, column_labels=column_labels, data_dtypes=data_dtypes
+        )
+        return DataFrame(internal)
+
+    @staticmethod
+    def from_records(
+        data: Union[np.array, List[tuple], dict, pd.DataFrame],
+        index: Union[str, list, np.array] = None,
+        exclude: list = None,
+        columns: list = None,
+        coerce_float: bool = False,
+        nrows: int = None,
+    ) -> "DataFrame":
+        """
+        Convert structured or record ndarray to DataFrame.
+
+        Parameters
+        ----------
+        data : ndarray (structured dtype), list of tuples, dict, or DataFrame
+        index : string, list of fields, array-like
+            Field of array to use as the index, alternately a specific set of input labels to use
+        exclude : sequence, default None
+            Columns or fields to exclude
+        columns : sequence, default None
+            Column names to use. If the passed data do not have names associated with them, this
+            argument provides names for the columns. Otherwise this argument indicates the order of
+            the columns in the result (any names not found in the data will become all-NA columns)
+        coerce_float : boolean, default False
+            Attempt to convert values of non-string, non-numeric objects (like decimal.Decimal) to
+            floating point, useful for SQL result sets
+        nrows : int, default None
+            Number of rows to read if data is an iterator
+
+        Returns
+        -------
+        df : DataFrame
+
+        Examples
+        --------
+        Use dict as input
+
+        >>> ps.DataFrame.from_records({'A': [1, 2, 3]})
+           A
+        0  1
+        1  2
+        2  3
+
+        Use list of tuples as input
+
+        >>> ps.DataFrame.from_records([(1, 2), (3, 4)])
+           0  1
+        0  1  2
+        1  3  4
+
+        Use NumPy array as input
+
+        >>> ps.DataFrame.from_records(np.eye(3))
+             0    1    2
+        0  1.0  0.0  0.0
+        1  0.0  1.0  0.0
+        2  0.0  0.0  1.0
+        """
+        return DataFrame(
+            pd.DataFrame.from_records(data, index, exclude, columns, coerce_float, nrows)
+        )
+
+    def to_records(self, index=True, column_dtypes=None, index_dtypes=None) -> np.recarray:
+        """
+        Convert DataFrame to a NumPy record array.
+
+        Index will be included as the first field of the record array if
+        requested.
+
+        .. note:: This method should only be used if the resulting NumPy ndarray is
+            expected to be small, as all the data is loaded into the driver's memory.
+
+        Parameters
+        ----------
+        index : bool, default True
+            Include index in resulting record array, stored in 'index'
+            field or using the index label, if set.
+        column_dtypes : str, type, dict, default None
+            If a string or type, the data type to store all columns. If
+            a dictionary, a mapping of column names and indices (zero-indexed)
+            to specific data types.
+        index_dtypes : str, type, dict, default None
+            If a string or type, the data type to store all index levels. If
+            a dictionary, a mapping of index level names and indices
+            (zero-indexed) to specific data types.
+            This mapping is applied only if `index=True`.
+
+        Returns
+        -------
+        numpy.recarray
+            NumPy ndarray with the DataFrame labels as fields and each row
+            of the DataFrame as entries.
+
+        See Also
+        --------
+        DataFrame.from_records: Convert structured or record ndarray
+            to DataFrame.
+        numpy.recarray: An ndarray that allows field access using
+            attributes, analogous to typed columns in a
+            spreadsheet.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'A': [1, 2], 'B': [0.5, 0.75]},
+        ...                   index=['a', 'b'])
+        >>> df
+           A     B
+        a  1  0.50
+        b  2  0.75
+
+        >>> df.to_records() # doctest: +SKIP
+        rec.array([('a', 1, 0.5 ), ('b', 2, 0.75)],
+                  dtype=[('index', 'O'), ('A', '<i8'), ('B', '<f8')])
+
+        The index can be excluded from the record array:
+
+        >>> df.to_records(index=False) # doctest: +SKIP
+        rec.array([(1, 0.5 ), (2, 0.75)],
+                  dtype=[('A', '<i8'), ('B', '<f8')])
+
+        Specification of dtype for columns is new in pandas 0.24.0.
+        Data types can be specified for the columns:
+
+        >>> df.to_records(column_dtypes={"A": "int32"}) # doctest: +SKIP
+        rec.array([('a', 1, 0.5 ), ('b', 2, 0.75)],
+                  dtype=[('index', 'O'), ('A', '<i4'), ('B', '<f8')])
+
+        Specification of dtype for index is new in pandas 0.24.0.
+        Data types can also be specified for the index:
+
+        >>> df.to_records(index_dtypes="<S2") # doctest: +SKIP
+        rec.array([(b'a', 1, 0.5 ), (b'b', 2, 0.75)],
+                  dtype=[('index', 'S2'), ('A', '<i8'), ('B', '<f8')])
+        """
+        args = locals()
+        kdf = self
+
+        return validate_arguments_and_invoke_function(
+            kdf._to_internal_pandas(), self.to_records, pd.DataFrame.to_records, args
+        )
+
+    def copy(self, deep=None) -> "DataFrame":
+        """
+        Make a copy of this object's indices and data.
+
+        Parameters
+        ----------
+        deep : None
+            this parameter is not supported but just dummy parameter to match pandas.
+
+        Returns
+        -------
+        copy : DataFrame
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'x': [1, 2], 'y': [3, 4], 'z': [5, 6], 'w': [7, 8]},
+        ...                   columns=['x', 'y', 'z', 'w'])
+        >>> df
+           x  y  z  w
+        0  1  3  5  7
+        1  2  4  6  8
+        >>> df_copy = df.copy()
+        >>> df_copy
+           x  y  z  w
+        0  1  3  5  7
+        1  2  4  6  8
+        """
+        return DataFrame(self._internal)
+
+    def dropna(
+        self, axis=0, how="any", thresh=None, subset=None, inplace=False
+    ) -> Optional["DataFrame"]:
+        """
+        Remove missing values.
+
+        Parameters
+        ----------
+        axis : {0 or 'index'}, default 0
+            Determine if rows or columns which contain missing values are
+            removed.
+
+            * 0, or 'index' : Drop rows which contain missing values.
+        how : {'any', 'all'}, default 'any'
+            Determine if row or column is removed from DataFrame, when we have
+            at least one NA or all NA.
+
+            * 'any' : If any NA values are present, drop that row or column.
+            * 'all' : If all values are NA, drop that row or column.
+
+        thresh : int, optional
+            Require that many non-NA values.
+        subset : array-like, optional
+            Labels along other axis to consider, e.g. if you are dropping rows
+            these would be a list of columns to include.
+        inplace : bool, default False
+            If True, do operation inplace and return None.
+
+        Returns
+        -------
+        DataFrame
+            DataFrame with NA entries dropped from it.
+
+        See Also
+        --------
+        DataFrame.drop : Drop specified labels from columns.
+        DataFrame.isnull: Indicate missing values.
+        DataFrame.notnull : Indicate existing (non-missing) values.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({"name": ['Alfred', 'Batman', 'Catwoman'],
+        ...                    "toy": [None, 'Batmobile', 'Bullwhip'],
+        ...                    "born": [None, "1940-04-25", None]},
+        ...                   columns=['name', 'toy', 'born'])
+        >>> df
+               name        toy        born
+        0    Alfred       None        None
+        1    Batman  Batmobile  1940-04-25
+        2  Catwoman   Bullwhip        None
+
+        Drop the rows where at least one element is missing.
+
+        >>> df.dropna()
+             name        toy        born
+        1  Batman  Batmobile  1940-04-25
+
+        Drop the columns where at least one element is missing.
+
+        >>> df.dropna(axis='columns')
+               name
+        0    Alfred
+        1    Batman
+        2  Catwoman
+
+        Drop the rows where all elements are missing.
+
+        >>> df.dropna(how='all')
+               name        toy        born
+        0    Alfred       None        None
+        1    Batman  Batmobile  1940-04-25
+        2  Catwoman   Bullwhip        None
+
+        Keep only the rows with at least 2 non-NA values.
+
+        >>> df.dropna(thresh=2)
+               name        toy        born
+        1    Batman  Batmobile  1940-04-25
+        2  Catwoman   Bullwhip        None
+
+        Define in which columns to look for missing values.
+
+        >>> df.dropna(subset=['name', 'born'])
+             name        toy        born
+        1  Batman  Batmobile  1940-04-25
+
+        Keep the DataFrame with valid entries in the same variable.
+
+        >>> df.dropna(inplace=True)
+        >>> df
+             name        toy        born
+        1  Batman  Batmobile  1940-04-25
+        """
+        axis = validate_axis(axis)
+        inplace = validate_bool_kwarg(inplace, "inplace")
+
+        if thresh is None:
+            if how is None:
+                raise TypeError("must specify how or thresh")
+            elif how not in ("any", "all"):
+                raise ValueError("invalid how option: {h}".format(h=how))
+
+        if subset is not None:
+            if isinstance(subset, str):
+                labels = [(subset,)]  # type: Optional[List[Tuple]]
+            elif isinstance(subset, tuple):
+                labels = [subset]
+            else:
+                labels = [sub if isinstance(sub, tuple) else (sub,) for sub in subset]
+        else:
+            labels = None
+
+        if axis == 0:
+            if labels is not None:
+                invalids = [label for label in labels if label not in self._internal.column_labels]
+                if len(invalids) > 0:
+                    raise KeyError(invalids)
+            else:
+                labels = self._internal.column_labels
+
+            cnt = reduce(
+                lambda x, y: x + y,
+                [
+                    F.when(self._kser_for(label).notna().spark.column, 1).otherwise(0)
+                    for label in labels
+                ],
+                F.lit(0),
+            )
+            if thresh is not None:
+                pred = cnt >= F.lit(int(thresh))
+            elif how == "any":
+                pred = cnt == F.lit(len(labels))
+            elif how == "all":
+                pred = cnt > F.lit(0)
+
+            internal = self._internal.with_filter(pred)
+            if inplace:
+                self._update_internal_frame(internal)
+                return None
+            else:
+                return DataFrame(internal)
+        else:
+            assert axis == 1
+
+            internal = self._internal.resolved_copy
+
+            if labels is not None:
+                if any(len(lbl) != internal.index_level for lbl in labels):
+                    raise ValueError(
+                        "The length of each subset must be the same as the index size."
+                    )
+
+                cond = reduce(
+                    lambda x, y: x | y,
+                    [
+                        reduce(
+                            lambda x, y: x & y,
+                            [
+                                scol == F.lit(l)
+                                for l, scol in zip(lbl, internal.index_spark_columns)
+                            ],
+                        )
+                        for lbl in labels
+                    ],
+                )
+
+                internal = internal.with_filter(cond)
+
+            null_counts = []
+            for label in internal.column_labels:
+                scol = internal.spark_column_for(label)
+                if isinstance(internal.spark_type_for(label), (FloatType, DoubleType)):
+                    cond = scol.isNull() | F.isnan(scol)
+                else:
+                    cond = scol.isNull()
+                null_counts.append(
+                    F.sum(F.when(~cond, 1).otherwise(0)).alias(name_like_string(label))
+                )
+
+            counts = internal.spark_frame.select(null_counts + [F.count("*")]).head()
+
+            if thresh is not None:
+                column_labels = [
+                    label
+                    for label, cnt in zip(internal.column_labels, counts)
+                    if (cnt or 0) >= int(thresh)
+                ]
+            elif how == "any":
+                column_labels = [
+                    label
+                    for label, cnt in zip(internal.column_labels, counts)
+                    if (cnt or 0) == counts[-1]
+                ]
+            elif how == "all":
+                column_labels = [
+                    label for label, cnt in zip(internal.column_labels, counts) if (cnt or 0) > 0
+                ]
+
+            kdf = self[column_labels]
+            if inplace:
+                self._update_internal_frame(kdf._internal)
+                return None
+            else:
+                return kdf
+
+    # TODO: add 'limit' when value parameter exists
+    def fillna(
+        self, value=None, method=None, axis=None, inplace=False, limit=None
+    ) -> Optional["DataFrame"]:
+        """Fill NA/NaN values.
+
+        .. note:: the current implementation of 'method' parameter in fillna uses Spark's Window
+            without specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Parameters
+        ----------
+        value : scalar, dict, Series
+            Value to use to fill holes. alternately a dict/Series of values
+            specifying which value to use for each column.
+            DataFrame is not supported.
+        method : {'backfill', 'bfill', 'pad', 'ffill', None}, default None
+            Method to use for filling holes in reindexed Series pad / ffill: propagate last valid
+            observation forward to next valid backfill / bfill:
+            use NEXT valid observation to fill gap
+        axis : {0 or `index`}
+            1 and `columns` are not supported.
+        inplace : boolean, default False
+            Fill in place (do not create a new object)
+        limit : int, default None
+            If method is specified, this is the maximum number of consecutive NaN values to
+            forward/backward fill. In other words, if there is a gap with more than this number of
+            consecutive NaNs, it will only be partially filled. If method is not specified,
+            this is the maximum number of entries along the entire axis where NaNs will be filled.
+            Must be greater than 0 if not None
+
+        Returns
+        -------
+        DataFrame
+            DataFrame with NA entries filled.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({
+        ...     'A': [None, 3, None, None],
+        ...     'B': [2, 4, None, 3],
+        ...     'C': [None, None, None, 1],
+        ...     'D': [0, 1, 5, 4]
+        ...     },
+        ...     columns=['A', 'B', 'C', 'D'])
+        >>> df
+             A    B    C  D
+        0  NaN  2.0  NaN  0
+        1  3.0  4.0  NaN  1
+        2  NaN  NaN  NaN  5
+        3  NaN  3.0  1.0  4
+
+        Replace all NaN elements with 0s.
+
+        >>> df.fillna(0)
+             A    B    C  D
+        0  0.0  2.0  0.0  0
+        1  3.0  4.0  0.0  1
+        2  0.0  0.0  0.0  5
+        3  0.0  3.0  1.0  4
+
+        We can also propagate non-null values forward or backward.
+
+        >>> df.fillna(method='ffill')
+             A    B    C  D
+        0  NaN  2.0  NaN  0
+        1  3.0  4.0  NaN  1
+        2  3.0  4.0  NaN  5
+        3  3.0  3.0  1.0  4
+
+        Replace all NaN elements in column 'A', 'B', 'C', and 'D', with 0, 1,
+        2, and 3 respectively.
+
+        >>> values = {'A': 0, 'B': 1, 'C': 2, 'D': 3}
+        >>> df.fillna(value=values)
+             A    B    C  D
+        0  0.0  2.0  2.0  0
+        1  3.0  4.0  2.0  1
+        2  0.0  1.0  2.0  5
+        3  0.0  3.0  1.0  4
+        """
+        axis = validate_axis(axis)
+        if axis != 0:
+            raise NotImplementedError("fillna currently only works for axis=0 or axis='index'")
+
+        if value is not None:
+            if not isinstance(value, (float, int, str, bool, dict, pd.Series)):
+                raise TypeError("Unsupported type %s" % type(value).__name__)
+            if limit is not None:
+                raise ValueError("limit parameter for value is not support now")
+            if isinstance(value, pd.Series):
+                value = value.to_dict()
+            if isinstance(value, dict):
+                for v in value.values():
+                    if not isinstance(v, (float, int, str, bool)):
+                        raise TypeError("Unsupported type %s" % type(v).__name__)
+                value = {k if is_name_like_tuple(k) else (k,): v for k, v in value.items()}
+
+                def op(kser):
+                    label = kser._column_label
+                    for k, v in value.items():
+                        if k == label[: len(k)]:
+                            return kser._fillna(
+                                value=value[k], method=method, axis=axis, limit=limit
+                            )
+                    else:
+                        return kser
+
+            else:
+                op = lambda kser: kser._fillna(value=value, method=method, axis=axis, limit=limit)
+        elif method is not None:
+            op = lambda kser: kser._fillna(value=value, method=method, axis=axis, limit=limit)
+        else:
+            raise ValueError("Must specify a fillna 'value' or 'method' parameter.")
+
+        kdf = self._apply_series_op(op, should_resolve=(method is not None))
+
+        inplace = validate_bool_kwarg(inplace, "inplace")
+        if inplace:
+            self._update_internal_frame(kdf._internal, requires_same_anchor=False)
+            return None
+        else:
+            return kdf
+
+    def replace(
+        self, to_replace=None, value=None, inplace=False, limit=None, regex=False, method="pad",
+    ) -> Optional["DataFrame"]:
+        """
+        Returns a new DataFrame replacing a value with another value.
+
+        Parameters
+        ----------
+        to_replace : int, float, string, list, tuple or dict
+            Value to be replaced.
+        value : int, float, string, list or tuple
+            Value to use to replace holes. The replacement value must be an int, float,
+            or string.
+            If value is a list or tuple, value should be of the same length with to_replace.
+        inplace : boolean, default False
+            Fill in place (do not create a new object)
+
+        Returns
+        -------
+        DataFrame
+            Object after replacement.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({"name": ['Ironman', 'Captain America', 'Thor', 'Hulk'],
+        ...                    "weapon": ['Mark-45', 'Shield', 'Mjolnir', 'Smash']},
+        ...                   columns=['name', 'weapon'])
+        >>> df
+                      name   weapon
+        0          Ironman  Mark-45
+        1  Captain America   Shield
+        2             Thor  Mjolnir
+        3             Hulk    Smash
+
+        Scalar `to_replace` and `value`
+
+        >>> df.replace('Ironman', 'War-Machine')
+                      name   weapon
+        0      War-Machine  Mark-45
+        1  Captain America   Shield
+        2             Thor  Mjolnir
+        3             Hulk    Smash
+
+        List like `to_replace` and `value`
+
+        >>> df.replace(['Ironman', 'Captain America'], ['Rescue', 'Hawkeye'], inplace=True)
+        >>> df
+              name   weapon
+        0   Rescue  Mark-45
+        1  Hawkeye   Shield
+        2     Thor  Mjolnir
+        3     Hulk    Smash
+
+        Dicts can be used to specify different replacement values for different existing values
+        To use a dict in this way the value parameter should be None
+
+        >>> df.replace({'Mjolnir': 'Stormbuster'})
+              name       weapon
+        0   Rescue      Mark-45
+        1  Hawkeye       Shield
+        2     Thor  Stormbuster
+        3     Hulk        Smash
+
+        Dict can specify that different values should be replaced in different columns
+        The value parameter should not be None in this case
+
+        >>> df.replace({'weapon': 'Mjolnir'}, 'Stormbuster')
+              name       weapon
+        0   Rescue      Mark-45
+        1  Hawkeye       Shield
+        2     Thor  Stormbuster
+        3     Hulk        Smash
+
+        Nested dictionaries
+        The value parameter should be None to use a nested dict in this way
+
+        >>> df.replace({'weapon': {'Mjolnir': 'Stormbuster'}})
+              name       weapon
+        0   Rescue      Mark-45
+        1  Hawkeye       Shield
+        2     Thor  Stormbuster
+        3     Hulk        Smash
+        """
+        if method != "pad":
+            raise NotImplementedError("replace currently works only for method='pad")
+        if limit is not None:
+            raise NotImplementedError("replace currently works only when limit=None")
+        if regex is not False:
+            raise NotImplementedError("replace currently doesn't supports regex")
+        inplace = validate_bool_kwarg(inplace, "inplace")
+
+        if value is not None and not isinstance(value, (int, float, str, list, tuple, dict)):
+            raise TypeError("Unsupported type {}".format(type(value).__name__))
+        if to_replace is not None and not isinstance(
+            to_replace, (int, float, str, list, tuple, dict)
+        ):
+            raise TypeError("Unsupported type {}".format(type(to_replace).__name__))
+
+        if isinstance(value, (list, tuple)) and isinstance(to_replace, (list, tuple)):
+            if len(value) != len(to_replace):
+                raise ValueError("Length of to_replace and value must be same")
+
+        if isinstance(to_replace, dict) and (
+            value is not None or all(isinstance(i, dict) for i in to_replace.values())
+        ):
+
+            def op(kser):
+                if kser.name in to_replace:
+                    return kser.replace(to_replace=to_replace[kser.name], value=value, regex=regex)
+                else:
+                    return kser
+
+        else:
+            op = lambda kser: kser.replace(to_replace=to_replace, value=value, regex=regex)
+
+        kdf = self._apply_series_op(op)
+        if inplace:
+            self._update_internal_frame(kdf._internal)
+            return None
+        else:
+            return kdf
+
+    def clip(self, lower: Union[float, int] = None, upper: Union[float, int] = None) -> "DataFrame":
+        """
+        Trim values at input threshold(s).
+
+        Assigns values outside boundary to boundary values.
+
+        Parameters
+        ----------
+        lower : float or int, default None
+            Minimum threshold value. All values below this threshold will be set to it.
+        upper : float or int, default None
+            Maximum threshold value. All values above this threshold will be set to it.
+
+        Returns
+        -------
+        DataFrame
+            DataFrame with the values outside the clip boundaries replaced.
+
+        Examples
+        --------
+        >>> ps.DataFrame({'A': [0, 2, 4]}).clip(1, 3)
+           A
+        0  1
+        1  2
+        2  3
+
+        Notes
+        -----
+        One difference between this implementation and pandas is that running
+        pd.DataFrame({'A': ['a', 'b']}).clip(0, 1) will crash with "TypeError: '<=' not supported
+        between instances of 'str' and 'int'" while ps.DataFrame({'A': ['a', 'b']}).clip(0, 1)
+        will output the original DataFrame, simply ignoring the incompatible types.
+        """
+        if is_list_like(lower) or is_list_like(upper):
+            raise TypeError(
+                "List-like value are not supported for 'lower' and 'upper' at the " + "moment"
+            )
+
+        if lower is None and upper is None:
+            return self
+
+        return self._apply_series_op(lambda kser: kser.clip(lower=lower, upper=upper))
+
+    def head(self, n: int = 5) -> "DataFrame":
+        """
+        Return the first `n` rows.
+
+        This function returns the first `n` rows for the object based
+        on position. It is useful for quickly testing if your object
+        has the right type of data in it.
+
+        Parameters
+        ----------
+        n : int, default 5
+            Number of rows to select.
+
+        Returns
+        -------
+        obj_head : same type as caller
+            The first `n` rows of the caller object.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'animal':['alligator', 'bee', 'falcon', 'lion',
+        ...                    'monkey', 'parrot', 'shark', 'whale', 'zebra']})
+        >>> df
+              animal
+        0  alligator
+        1        bee
+        2     falcon
+        3       lion
+        4     monkey
+        5     parrot
+        6      shark
+        7      whale
+        8      zebra
+
+        Viewing the first 5 lines
+
+        >>> df.head()
+              animal
+        0  alligator
+        1        bee
+        2     falcon
+        3       lion
+        4     monkey
+
+        Viewing the first `n` lines (three in this case)
+
+        >>> df.head(3)
+              animal
+        0  alligator
+        1        bee
+        2     falcon
+        """
+        if n < 0:
+            n = len(self) + n
+        if n <= 0:
+            return DataFrame(self._internal.with_filter(F.lit(False)))
+        else:
+            sdf = self._internal.resolved_copy.spark_frame
+            if get_option("compute.ordered_head"):
+                sdf = sdf.orderBy(NATURAL_ORDER_COLUMN_NAME)
+            return DataFrame(self._internal.with_new_sdf(sdf.limit(n)))
+
+    def last(self, offset: Union[str, DateOffset]) -> "DataFrame":
+        """
+        Select final periods of time series data based on a date offset.
+
+        When having a DataFrame with dates as index, this function can
+        select the last few rows based on a date offset.
+
+        Parameters
+        ----------
+        offset : str or DateOffset
+            The offset length of the data that will be selected. For instance,
+            '3D' will display all the rows having their index within the last 3 days.
+
+        Returns
+        -------
+        DataFrame
+            A subset of the caller.
+
+        Raises
+        ------
+        TypeError
+            If the index is not a :class:`DatetimeIndex`
+
+        Examples
+        --------
+
+        >>> index = pd.date_range('2018-04-09', periods=4, freq='2D')
+        >>> kdf = ps.DataFrame({'A': [1, 2, 3, 4]}, index=index)
+        >>> kdf
+                    A
+        2018-04-09  1
+        2018-04-11  2
+        2018-04-13  3
+        2018-04-15  4
+
+        Get the rows for the last 3 days:
+
+        >>> kdf.last('3D')
+                    A
+        2018-04-13  3
+        2018-04-15  4
+
+        Notice the data for 3 last calendar days were returned, not the last
+        3 observed days in the dataset, and therefore data for 2018-04-11 was
+        not returned.
+        """
+        # Check index type should be format DateTime
+        if not isinstance(self.index, ps.DatetimeIndex):
+            raise TypeError("'last' only supports a DatetimeIndex")
+
+        offset = to_offset(offset)
+        from_date = self.index.max() - offset
+
+        return cast(DataFrame, self.loc[from_date:])
+
+    def first(self, offset: Union[str, DateOffset]) -> "DataFrame":
+        """
+        Select first periods of time series data based on a date offset.
+
+        When having a DataFrame with dates as index, this function can
+        select the first few rows based on a date offset.
+
+        Parameters
+        ----------
+        offset : str or DateOffset
+            The offset length of the data that will be selected. For instance,
+            '3D' will display all the rows having their index within the first 3 days.
+
+        Returns
+        -------
+        DataFrame
+            A subset of the caller.
+
+        Raises
+        ------
+        TypeError
+            If the index is not a :class:`DatetimeIndex`
+
+        Examples
+        --------
+
+        >>> index = pd.date_range('2018-04-09', periods=4, freq='2D')
+        >>> kdf = ps.DataFrame({'A': [1, 2, 3, 4]}, index=index)
+        >>> kdf
+                    A
+        2018-04-09  1
+        2018-04-11  2
+        2018-04-13  3
+        2018-04-15  4
+
+        Get the rows for the last 3 days:
+
+        >>> kdf.first('3D')
+                    A
+        2018-04-09  1
+        2018-04-11  2
+
+        Notice the data for 3 first calendar days were returned, not the first
+        3 observed days in the dataset, and therefore data for 2018-04-13 was
+        not returned.
+        """
+        # Check index type should be format DatetimeIndex
+        if not isinstance(self.index, ps.DatetimeIndex):
+            raise TypeError("'first' only supports a DatetimeIndex")
+
+        offset = to_offset(offset)
+        to_date = self.index.min() + offset
+
+        return cast(DataFrame, self.loc[:to_date])
+
+    def pivot_table(
+        self, values=None, index=None, columns=None, aggfunc="mean", fill_value=None
+    ) -> "DataFrame":
+        """
+        Create a spreadsheet-style pivot table as a DataFrame. The levels in
+        the pivot table will be stored in MultiIndex objects (hierarchical
+        indexes) on the index and columns of the result DataFrame.
+
+        Parameters
+        ----------
+        values : column to aggregate.
+            They should be either a list less than three or a string.
+        index : column (string) or list of columns
+            If an array is passed, it must be the same length as the data.
+            The list should contain string.
+        columns : column
+            Columns used in the pivot operation. Only one column is supported and
+            it should be a string.
+        aggfunc : function (string), dict, default mean
+            If dict is passed, the key is column to aggregate and value
+            is function or list of functions.
+        fill_value : scalar, default None
+            Value to replace missing values with.
+
+        Returns
+        -------
+        table : DataFrame
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({"A": ["foo", "foo", "foo", "foo", "foo",
+        ...                          "bar", "bar", "bar", "bar"],
+        ...                    "B": ["one", "one", "one", "two", "two",
+        ...                          "one", "one", "two", "two"],
+        ...                    "C": ["small", "large", "large", "small",
+        ...                          "small", "large", "small", "small",
+        ...                          "large"],
+        ...                    "D": [1, 2, 2, 3, 3, 4, 5, 6, 7],
+        ...                    "E": [2, 4, 5, 5, 6, 6, 8, 9, 9]},
+        ...                   columns=['A', 'B', 'C', 'D', 'E'])
+        >>> df
+             A    B      C  D  E
+        0  foo  one  small  1  2
+        1  foo  one  large  2  4
+        2  foo  one  large  2  5
+        3  foo  two  small  3  5
+        4  foo  two  small  3  6
+        5  bar  one  large  4  6
+        6  bar  one  small  5  8
+        7  bar  two  small  6  9
+        8  bar  two  large  7  9
+
+        This first example aggregates values by taking the sum.
+
+        >>> table = df.pivot_table(values='D', index=['A', 'B'],
+        ...                        columns='C', aggfunc='sum')
+        >>> table.sort_index()  # doctest: +NORMALIZE_WHITESPACE
+        C        large  small
+        A   B
+        bar one    4.0      5
+            two    7.0      6
+        foo one    4.0      1
+            two    NaN      6
+
+        We can also fill missing values using the `fill_value` parameter.
+
+        >>> table = df.pivot_table(values='D', index=['A', 'B'],
+        ...                        columns='C', aggfunc='sum', fill_value=0)
+        >>> table.sort_index()  # doctest: +NORMALIZE_WHITESPACE
+        C        large  small
+        A   B
+        bar one      4      5
+            two      7      6
+        foo one      4      1
+            two      0      6
+
+        We can also calculate multiple types of aggregations for any given
+        value column.
+
+        >>> table = df.pivot_table(values=['D'], index =['C'],
+        ...                        columns="A", aggfunc={'D': 'mean'})
+        >>> table.sort_index()  # doctest: +NORMALIZE_WHITESPACE
+                 D
+        A      bar       foo
+        C
+        large  5.5  2.000000
+        small  5.5  2.333333
+
+        The next example aggregates on multiple values.
+
+        >>> table = df.pivot_table(index=['C'], columns="A", values=['D', 'E'],
+        ...                         aggfunc={'D': 'mean', 'E': 'sum'})
+        >>> table.sort_index() # doctest: +NORMALIZE_WHITESPACE
+                 D             E
+        A      bar       foo bar foo
+        C
+        large  5.5  2.000000  15   9
+        small  5.5  2.333333  17  13
+        """
+        if not is_name_like_value(columns):
+            raise TypeError("columns should be one column name.")
+
+        if not is_name_like_value(values) and not (
+            isinstance(values, list) and all(is_name_like_value(v) for v in values)
+        ):
+            raise TypeError("values should be one column or list of columns.")
+
+        if not isinstance(aggfunc, str) and (
+            not isinstance(aggfunc, dict)
+            or not all(
+                is_name_like_value(key) and isinstance(value, str) for key, value in aggfunc.items()
+            )
+        ):
+            raise TypeError(
+                "aggfunc must be a dict mapping from column name "
+                "to aggregate functions (string)."
+            )
+
+        if isinstance(aggfunc, dict) and index is None:
+            raise NotImplementedError(
+                "pivot_table doesn't support aggfunc" " as dict and without index."
+            )
+        if isinstance(values, list) and index is None:
+            raise NotImplementedError("values can't be a list without index.")
+
+        if columns not in self.columns:
+            raise ValueError("Wrong columns {}.".format(name_like_string(columns)))
+        if not is_name_like_tuple(columns):
+            columns = (columns,)
+
+        if isinstance(values, list):
+            values = [col if is_name_like_tuple(col) else (col,) for col in values]
+            if not all(
+                isinstance(self._internal.spark_type_for(col), NumericType) for col in values
+            ):
+                raise TypeError("values should be a numeric type.")
+        else:
+            values = values if is_name_like_tuple(values) else (values,)
+            if not isinstance(self._internal.spark_type_for(values), NumericType):
+                raise TypeError("values should be a numeric type.")
+
+        if isinstance(aggfunc, str):
+            if isinstance(values, list):
+                agg_cols = [
+                    F.expr(
+                        "{1}(`{0}`) as `{0}`".format(
+                            self._internal.spark_column_name_for(value), aggfunc
+                        )
+                    )
+                    for value in values
+                ]
+            else:
+                agg_cols = [
+                    F.expr(
+                        "{1}(`{0}`) as `{0}`".format(
+                            self._internal.spark_column_name_for(values), aggfunc
+                        )
+                    )
+                ]
+        elif isinstance(aggfunc, dict):
+            aggfunc = {
+                key if is_name_like_tuple(key) else (key,): value for key, value in aggfunc.items()
+            }
+            agg_cols = [
+                F.expr(
+                    "{1}(`{0}`) as `{0}`".format(self._internal.spark_column_name_for(key), value)
+                )
+                for key, value in aggfunc.items()
+            ]
+            agg_columns = [key for key, _ in aggfunc.items()]
+
+            if set(agg_columns) != set(values):
+                raise ValueError("Columns in aggfunc must be the same as values.")
+
+        sdf = self._internal.resolved_copy.spark_frame
+        if index is None:
+            sdf = (
+                sdf.groupBy()
+                .pivot(pivot_col=self._internal.spark_column_name_for(columns))
+                .agg(*agg_cols)
+            )
+
+        elif isinstance(index, list):
+            index = [label if is_name_like_tuple(label) else (label,) for label in index]
+            sdf = (
+                sdf.groupBy([self._internal.spark_column_name_for(label) for label in index])
+                .pivot(pivot_col=self._internal.spark_column_name_for(columns))
+                .agg(*agg_cols)
+            )
+        else:
+            raise TypeError("index should be a None or a list of columns.")
+
+        if fill_value is not None and isinstance(fill_value, (int, float)):
+            sdf = sdf.fillna(fill_value)
+
+        if index is not None:
+            index_columns = [self._internal.spark_column_name_for(label) for label in index]
+            index_dtypes = [self._internal.dtype_for(label) for label in index]
+
+            if isinstance(values, list):
+                data_columns = [column for column in sdf.columns if column not in index_columns]
+
+                if len(values) > 1:
+                    # If we have two values, Spark will return column's name
+                    # in this format: column_values, where column contains
+                    # their values in the DataFrame and values is
+                    # the column list passed to the pivot_table().
+                    # E.g. if column is b and values is ['b','e'],
+                    # then ['2_b', '2_e', '3_b', '3_e'].
+
+                    # We sort the columns of Spark DataFrame by values.
+                    data_columns.sort(key=lambda x: x.split("_", 1)[1])
+                    sdf = sdf.select(index_columns + data_columns)
+
+                    column_name_to_index = dict(
+                        zip(self._internal.data_spark_column_names, self._internal.column_labels)
+                    )
+                    column_labels = [
+                        tuple(list(column_name_to_index[name.split("_")[1]]) + [name.split("_")[0]])
+                        for name in data_columns
+                    ]
+                    column_label_names = ([None] * column_labels_level(values)) + [columns]
+                    internal = InternalFrame(
+                        spark_frame=sdf,
+                        index_spark_columns=[scol_for(sdf, col) for col in index_columns],
+                        index_names=index,
+                        index_dtypes=index_dtypes,
+                        column_labels=column_labels,
+                        data_spark_columns=[scol_for(sdf, col) for col in data_columns],
+                        column_label_names=column_label_names,  # type: ignore
+                    )
+                    kdf = DataFrame(internal)  # type: "DataFrame"
+                else:
+                    column_labels = [tuple(list(values[0]) + [column]) for column in data_columns]
+                    column_label_names = ([None] * len(values[0])) + [columns]
+                    internal = InternalFrame(
+                        spark_frame=sdf,
+                        index_spark_columns=[scol_for(sdf, col) for col in index_columns],
+                        index_names=index,
+                        index_dtypes=index_dtypes,
+                        column_labels=column_labels,
+                        data_spark_columns=[scol_for(sdf, col) for col in data_columns],
+                        column_label_names=column_label_names,  # type: ignore
+                    )
+                    kdf = DataFrame(internal)
+            else:
+                internal = InternalFrame(
+                    spark_frame=sdf,
+                    index_spark_columns=[scol_for(sdf, col) for col in index_columns],
+                    index_names=index,
+                    index_dtypes=index_dtypes,
+                    column_label_names=[columns],
+                )
+                kdf = DataFrame(internal)
+        else:
+            if isinstance(values, list):
+                index_values = values[-1]
+            else:
+                index_values = values
+            index_map = OrderedDict()  # type: Dict[str, Optional[Tuple]]
+            for i, index_value in enumerate(index_values):
+                colname = SPARK_INDEX_NAME_FORMAT(i)
+                sdf = sdf.withColumn(colname, F.lit(index_value))
+                index_map[colname] = None
+            internal = InternalFrame(
+                spark_frame=sdf,
+                index_spark_columns=[scol_for(sdf, col) for col in index_map.keys()],
+                index_names=list(index_map.values()),
+                column_label_names=[columns],
+            )
+            kdf = DataFrame(internal)
+
+        kdf_columns = kdf.columns
+        if isinstance(kdf_columns, pd.MultiIndex):
+            kdf.columns = kdf_columns.set_levels(
+                kdf_columns.levels[-1].astype(
+                    spark_type_to_pandas_dtype(self._kser_for(columns).spark.data_type)
+                ),
+                level=-1,
+            )
+        else:
+            kdf.columns = kdf_columns.astype(
+                spark_type_to_pandas_dtype(self._kser_for(columns).spark.data_type)
+            )
+
+        return kdf
+
+    def pivot(self, index=None, columns=None, values=None) -> "DataFrame":
+        """
+        Return reshaped DataFrame organized by given index / column values.
+
+        Reshape data (produce a "pivot" table) based on column values. Uses
+        unique values from specified `index` / `columns` to form axes of the
+        resulting DataFrame. This function does not support data
+        aggregation.
+
+        Parameters
+        ----------
+        index : string, optional
+            Column to use to make new frame's index. If None, uses
+            existing index.
+        columns : string
+            Column to use to make new frame's columns.
+        values : string, object or a list of the previous
+            Column(s) to use for populating new frame's values.
+
+        Returns
+        -------
+        DataFrame
+            Returns reshaped DataFrame.
+
+        See Also
+        --------
+        DataFrame.pivot_table : Generalization of pivot that can handle
+            duplicate values for one index/column pair.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'foo': ['one', 'one', 'one', 'two', 'two',
+        ...                            'two'],
+        ...                    'bar': ['A', 'B', 'C', 'A', 'B', 'C'],
+        ...                    'baz': [1, 2, 3, 4, 5, 6],
+        ...                    'zoo': ['x', 'y', 'z', 'q', 'w', 't']},
+        ...                   columns=['foo', 'bar', 'baz', 'zoo'])
+        >>> df
+           foo bar  baz zoo
+        0  one   A    1   x
+        1  one   B    2   y
+        2  one   C    3   z
+        3  two   A    4   q
+        4  two   B    5   w
+        5  two   C    6   t
+
+        >>> df.pivot(index='foo', columns='bar', values='baz').sort_index()
+        ... # doctest: +NORMALIZE_WHITESPACE
+        bar  A  B  C
+        foo
+        one  1  2  3
+        two  4  5  6
+
+        >>> df.pivot(columns='bar', values='baz').sort_index()  # doctest: +NORMALIZE_WHITESPACE
+        bar  A    B    C
+        0  1.0  NaN  NaN
+        1  NaN  2.0  NaN
+        2  NaN  NaN  3.0
+        3  4.0  NaN  NaN
+        4  NaN  5.0  NaN
+        5  NaN  NaN  6.0
+
+        Notice that, unlike pandas raises an ValueError when duplicated values are found,
+        pandas-on-Spark's pivot still works with its first value it meets during operation because
+        pivot is an expensive operation and it is preferred to permissively execute over failing
+        fast when processing large data.
+
+        >>> df = ps.DataFrame({"foo": ['one', 'one', 'two', 'two'],
+        ...                    "bar": ['A', 'A', 'B', 'C'],
+        ...                    "baz": [1, 2, 3, 4]}, columns=['foo', 'bar', 'baz'])
+        >>> df
+           foo bar  baz
+        0  one   A    1
+        1  one   A    2
+        2  two   B    3
+        3  two   C    4
+
+        >>> df.pivot(index='foo', columns='bar', values='baz').sort_index()
+        ... # doctest: +NORMALIZE_WHITESPACE
+        bar    A    B    C
+        foo
+        one  1.0  NaN  NaN
+        two  NaN  3.0  4.0
+
+        It also support multi-index and multi-index column.
+        >>> df.columns = pd.MultiIndex.from_tuples([('a', 'foo'), ('a', 'bar'), ('b', 'baz')])
+
+        >>> df = df.set_index(('a', 'bar'), append=True)
+        >>> df  # doctest: +NORMALIZE_WHITESPACE
+                      a   b
+                    foo baz
+          (a, bar)
+        0 A         one   1
+        1 A         one   2
+        2 B         two   3
+        3 C         two   4
+
+        >>> df.pivot(columns=('a', 'foo'), values=('b', 'baz')).sort_index()
+        ... # doctest: +NORMALIZE_WHITESPACE
+        ('a', 'foo')  one  two
+          (a, bar)
+        0 A           1.0  NaN
+        1 A           2.0  NaN
+        2 B           NaN  3.0
+        3 C           NaN  4.0
+
+        """
+        if columns is None:
+            raise ValueError("columns should be set.")
+
+        if values is None:
+            raise ValueError("values should be set.")
+
+        should_use_existing_index = index is not None
+        if should_use_existing_index:
+            df = self
+            index = [index]
+        else:
+            # The index after `reset_index()` will never be used, so use "distributed" index
+            # as a dummy to avoid overhead.
+            with option_context("compute.default_index_type", "distributed"):
+                df = self.reset_index()
+            index = df._internal.column_labels[: self._internal.index_level]
+
+        df = df.pivot_table(index=index, columns=columns, values=values, aggfunc="first")
+
+        if should_use_existing_index:
+            return df
+        else:
+            internal = df._internal.copy(index_names=self._internal.index_names)
+            return DataFrame(internal)
+
+    @property
+    def columns(self) -> pd.Index:
+        """The column labels of the DataFrame."""
+        names = [
+            name if name is None or len(name) > 1 else name[0]
+            for name in self._internal.column_label_names
+        ]
+        if self._internal.column_labels_level > 1:
+            columns = pd.MultiIndex.from_tuples(self._internal.column_labels, names=names)
+        else:
+            columns = pd.Index([label[0] for label in self._internal.column_labels], name=names[0])
+        return columns
+
+    @columns.setter
+    def columns(self, columns) -> None:
+        if isinstance(columns, pd.MultiIndex):
+            column_labels = columns.tolist()
+        else:
+            column_labels = [
+                col if is_name_like_tuple(col, allow_none=False) else (col,) for col in columns
+            ]
+
+        if len(self._internal.column_labels) != len(column_labels):
+            raise ValueError(
+                "Length mismatch: Expected axis has {} elements, "
+                "new values have {} elements".format(
+                    len(self._internal.column_labels), len(column_labels)
+                )
+            )
+
+        if isinstance(columns, pd.Index):
+            column_label_names = [
+                name if is_name_like_tuple(name) else (name,) for name in columns.names
+            ]  # type: Optional[List]
+        else:
+            column_label_names = None
+
+        ksers = [
+            self._kser_for(label).rename(name)
+            for label, name in zip(self._internal.column_labels, column_labels)
+        ]
+        self._update_internal_frame(
+            self._internal.with_new_columns(ksers, column_label_names=column_label_names)
+        )
+
+    @property
+    def dtypes(self) -> pd.Series:
+        """Return the dtypes in the DataFrame.
+
+        This returns a Series with the data type of each column. The result's index is the original
+        DataFrame's columns. Columns with mixed types are stored with the object dtype.
+
+        Returns
+        -------
+        pd.Series
+            The data type of each column.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'a': list('abc'),
+        ...                    'b': list(range(1, 4)),
+        ...                    'c': np.arange(3, 6).astype('i1'),
+        ...                    'd': np.arange(4.0, 7.0, dtype='float64'),
+        ...                    'e': [True, False, True],
+        ...                    'f': pd.date_range('20130101', periods=3)},
+        ...                   columns=['a', 'b', 'c', 'd', 'e', 'f'])
+        >>> df.dtypes
+        a            object
+        b             int64
+        c              int8
+        d           float64
+        e              bool
+        f    datetime64[ns]
+        dtype: object
+        """
+        return pd.Series(
+            [self._kser_for(label).dtype for label in self._internal.column_labels],
+            index=pd.Index(
+                [label if len(label) > 1 else label[0] for label in self._internal.column_labels]
+            ),
+        )
+
+    def spark_schema(self, index_col: Optional[Union[str, List[str]]] = None) -> StructType:
+        warnings.warn(
+            "DataFrame.spark_schema is deprecated as of DataFrame.spark.schema. "
+            "Please use the API instead.",
+            FutureWarning,
+        )
+        return self.spark.schema(index_col)
+
+    spark_schema.__doc__ = SparkFrameMethods.schema.__doc__
+
+    def print_schema(self, index_col: Optional[Union[str, List[str]]] = None) -> None:
+        warnings.warn(
+            "DataFrame.print_schema is deprecated as of DataFrame.spark.print_schema. "
+            "Please use the API instead.",
+            FutureWarning,
+        )
+        return self.spark.print_schema(index_col)
+
+    print_schema.__doc__ = SparkFrameMethods.print_schema.__doc__
+
+    def select_dtypes(self, include=None, exclude=None) -> "DataFrame":
+        """
+        Return a subset of the DataFrame's columns based on the column dtypes.
+
+        Parameters
+        ----------
+        include, exclude : scalar or list-like
+            A selection of dtypes or strings to be included/excluded. At least
+            one of these parameters must be supplied. It also takes Spark SQL
+            DDL type strings, for instance, 'string' and 'date'.
+
+        Returns
+        -------
+        DataFrame
+            The subset of the frame including the dtypes in ``include`` and
+            excluding the dtypes in ``exclude``.
+
+        Raises
+        ------
+        ValueError
+            * If both of ``include`` and ``exclude`` are empty
+
+                >>> df = ps.DataFrame({'a': [1, 2] * 3,
+                ...                    'b': [True, False] * 3,
+                ...                    'c': [1.0, 2.0] * 3})
+                >>> df.select_dtypes()
+                Traceback (most recent call last):
+                ...
+                ValueError: at least one of include or exclude must be nonempty
+
+            * If ``include`` and ``exclude`` have overlapping elements
+
+                >>> df = ps.DataFrame({'a': [1, 2] * 3,
+                ...                    'b': [True, False] * 3,
+                ...                    'c': [1.0, 2.0] * 3})
+                >>> df.select_dtypes(include='a', exclude='a')
+                Traceback (most recent call last):
+                ...
+                ValueError: include and exclude overlap on {'a'}
+
+        Notes
+        -----
+        * To select datetimes, use ``np.datetime64``, ``'datetime'`` or
+          ``'datetime64'``
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'a': [1, 2] * 3,
+        ...                    'b': [True, False] * 3,
+        ...                    'c': [1.0, 2.0] * 3,
+        ...                    'd': ['a', 'b'] * 3}, columns=['a', 'b', 'c', 'd'])
+        >>> df
+           a      b    c  d
+        0  1   True  1.0  a
+        1  2  False  2.0  b
+        2  1   True  1.0  a
+        3  2  False  2.0  b
+        4  1   True  1.0  a
+        5  2  False  2.0  b
+
+        >>> df.select_dtypes(include='bool')
+               b
+        0   True
+        1  False
+        2   True
+        3  False
+        4   True
+        5  False
+
+        >>> df.select_dtypes(include=['float64'], exclude=['int'])
+             c
+        0  1.0
+        1  2.0
+        2  1.0
+        3  2.0
+        4  1.0
+        5  2.0
+
+        >>> df.select_dtypes(exclude=['int'])
+               b    c  d
+        0   True  1.0  a
+        1  False  2.0  b
+        2   True  1.0  a
+        3  False  2.0  b
+        4   True  1.0  a
+        5  False  2.0  b
+
+        Spark SQL DDL type strings can be used as well.
+
+        >>> df.select_dtypes(exclude=['string'])
+           a      b    c
+        0  1   True  1.0
+        1  2  False  2.0
+        2  1   True  1.0
+        3  2  False  2.0
+        4  1   True  1.0
+        5  2  False  2.0
+        """
+        from pyspark.sql.types import _parse_datatype_string
+
+        if not is_list_like(include):
+            include = (include,) if include is not None else ()
+        if not is_list_like(exclude):
+            exclude = (exclude,) if exclude is not None else ()
+
+        if not any((include, exclude)):
+            raise ValueError("at least one of include or exclude must be " "nonempty")
+
+        # can't both include AND exclude!
+        if set(include).intersection(set(exclude)):
+            raise ValueError(
+                "include and exclude overlap on {inc_ex}".format(
+                    inc_ex=set(include).intersection(set(exclude))
+                )
+            )
+
+        # Handle Spark types
+        include_spark_type = []
+        for inc in include:
+            try:
+                include_spark_type.append(_parse_datatype_string(inc))
+            except:
+                pass
+
+        exclude_spark_type = []
+        for exc in exclude:
+            try:
+                exclude_spark_type.append(_parse_datatype_string(exc))
+            except:
+                pass
+
+        # Handle pandas types
+        include_numpy_type = []
+        for inc in include:
+            try:
+                include_numpy_type.append(infer_dtype_from_object(inc))
+            except:
+                pass
+
+        exclude_numpy_type = []
+        for exc in exclude:
+            try:
+                exclude_numpy_type.append(infer_dtype_from_object(exc))
+            except:
+                pass
+
+        column_labels = []
+        for label in self._internal.column_labels:
+            if len(include) > 0:
+                should_include = (
+                    infer_dtype_from_object(self._kser_for(label).dtype.name) in include_numpy_type
+                    or self._internal.spark_type_for(label) in include_spark_type
+                )
+            else:
+                should_include = not (
+                    infer_dtype_from_object(self._kser_for(label).dtype.name) in exclude_numpy_type
+                    or self._internal.spark_type_for(label) in exclude_spark_type
+                )
+
+            if should_include:
+                column_labels.append(label)
+
+        return DataFrame(
+            self._internal.with_new_columns([self._kser_for(label) for label in column_labels])
+        )
+
+    def droplevel(self, level, axis=0) -> "DataFrame":
+        """
+        Return DataFrame with requested index / column level(s) removed.
+
+        Parameters
+        ----------
+        level: int, str, or list-like
+            If a string is given, must be the name of a level If list-like, elements must
+            be names or positional indexes of levels.
+
+        axis: {0 or ‘index’, 1 or ‘columns’}, default 0
+
+        Returns
+        -------
+        DataFrame with requested index / column level(s) removed.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame(
+        ...     [[3, 4], [7, 8], [11, 12]],
+        ...     index=pd.MultiIndex.from_tuples([(1, 2), (5, 6), (9, 10)], names=["a", "b"]),
+        ... )
+
+        >>> df.columns = pd.MultiIndex.from_tuples([
+        ...   ('c', 'e'), ('d', 'f')
+        ... ], names=['level_1', 'level_2'])
+
+        >>> df  # doctest: +NORMALIZE_WHITESPACE
+        level_1   c   d
+        level_2   e   f
+        a b
+        1 2      3   4
+        5 6      7   8
+        9 10    11  12
+
+        >>> df.droplevel('a')  # doctest: +NORMALIZE_WHITESPACE
+        level_1   c   d
+        level_2   e   f
+        b
+        2        3   4
+        6        7   8
+        10      11  12
+
+        >>> df.droplevel('level_2', axis=1)  # doctest: +NORMALIZE_WHITESPACE
+        level_1   c   d
+        a b
+        1 2      3   4
+        5 6      7   8
+        9 10    11  12
+        """
+        axis = validate_axis(axis)
+        if axis == 0:
+            if not isinstance(level, (tuple, list)):  # huh?
+                level = [level]
+
+            index_names = self.index.names
+            nlevels = self._internal.index_level
+
+            int_level = set()
+            for n in level:
+                if isinstance(n, int):
+                    if n < 0:
+                        n = n + nlevels
+                        if n < 0:
+                            raise IndexError(
+                                "Too many levels: Index has only {} levels, "
+                                "{} is not a valid level number".format(nlevels, (n - nlevels))
+                            )
+                    if n >= nlevels:
+                        raise IndexError(
+                            "Too many levels: Index has only {} levels, not {}".format(
+                                nlevels, (n + 1)
+                            )
+                        )
+                else:
+                    if n not in index_names:
+                        raise KeyError("Level {} not found".format(n))
+                    n = index_names.index(n)
+                int_level.add(n)
+
+            if len(level) >= nlevels:
+                raise ValueError(
+                    "Cannot remove {} levels from an index with {} levels: "
+                    "at least one level must be left.".format(len(level), nlevels)
+                )
+
+            index_spark_columns, index_names, index_dtypes = zip(
+                *[
+                    item
+                    for i, item in enumerate(
+                        zip(
+                            self._internal.index_spark_columns,
+                            self._internal.index_names,
+                            self._internal.index_dtypes,
+                        )
+                    )
+                    if i not in int_level
+                ]
+            )
+
+            internal = self._internal.copy(
+                index_spark_columns=list(index_spark_columns),
+                index_names=list(index_names),
+                index_dtypes=list(index_dtypes),
+            )
+            return DataFrame(internal)
+        else:
+            kdf = self.copy()
+            kdf.columns = kdf.columns.droplevel(level)
+            return kdf
+
+    def drop(
+        self, labels=None, axis=1, columns: Union[Any, Tuple, List[Any], List[Tuple]] = None
+    ) -> "DataFrame":
+        """
+        Drop specified labels from columns.
+
+        Remove columns by specifying label names and axis=1 or columns.
+        When specifying both labels and columns, only labels will be dropped.
+        Removing rows is yet to be implemented.
+
+        Parameters
+        ----------
+        labels : single label or list-like
+            Column labels to drop.
+        axis : {1 or 'columns'}, default 1
+            .. dropna currently only works for axis=1 'columns'
+               axis=0 is yet to be implemented.
+        columns : single label or list-like
+            Alternative to specifying axis (``labels, axis=1``
+            is equivalent to ``columns=labels``).
+
+        Returns
+        -------
+        dropped : DataFrame
+
+        See Also
+        --------
+        Series.dropna
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'x': [1, 2], 'y': [3, 4], 'z': [5, 6], 'w': [7, 8]},
+        ...                   columns=['x', 'y', 'z', 'w'])
+        >>> df
+           x  y  z  w
+        0  1  3  5  7
+        1  2  4  6  8
+
+        >>> df.drop('x', axis=1)
+           y  z  w
+        0  3  5  7
+        1  4  6  8
+
+        >>> df.drop(['y', 'z'], axis=1)
+           x  w
+        0  1  7
+        1  2  8
+
+        >>> df.drop(columns=['y', 'z'])
+           x  w
+        0  1  7
+        1  2  8
+
+        Also support for MultiIndex
+
+        >>> df = ps.DataFrame({'x': [1, 2], 'y': [3, 4], 'z': [5, 6], 'w': [7, 8]},
+        ...                   columns=['x', 'y', 'z', 'w'])
+        >>> columns = [('a', 'x'), ('a', 'y'), ('b', 'z'), ('b', 'w')]
+        >>> df.columns = pd.MultiIndex.from_tuples(columns)
+        >>> df  # doctest: +NORMALIZE_WHITESPACE
+           a     b
+           x  y  z  w
+        0  1  3  5  7
+        1  2  4  6  8
+        >>> df.drop('a')  # doctest: +NORMALIZE_WHITESPACE
+           b
+           z  w
+        0  5  7
+        1  6  8
+
+        Notes
+        -----
+        Currently only axis = 1 is supported in this function,
+        axis = 0 is yet to be implemented.
+        """
+        if labels is not None:
+            axis = validate_axis(axis)
+            if axis == 1:
+                return self.drop(columns=labels)
+            raise NotImplementedError("Drop currently only works for axis=1")
+        elif columns is not None:
+            if is_name_like_tuple(columns):
+                columns = [columns]
+            elif is_name_like_value(columns):
+                columns = [(columns,)]
+            else:
+                columns = [col if is_name_like_tuple(col) else (col,) for col in columns]
+            drop_column_labels = set(
+                label
+                for label in self._internal.column_labels
+                for col in columns
+                if label[: len(col)] == col
+            )
+            if len(drop_column_labels) == 0:
+                raise KeyError(columns)
+            cols, labels = zip(
+                *(
+                    (column, label)
+                    for column, label in zip(
+                        self._internal.data_spark_column_names, self._internal.column_labels
+                    )
+                    if label not in drop_column_labels
+                )
+            )
+            internal = self._internal.with_new_columns([self._kser_for(label) for label in labels])
+            return DataFrame(internal)
+        else:
+            raise ValueError("Need to specify at least one of 'labels' or 'columns'")
+
+    def _sort(
+        self, by: List[Column], ascending: Union[bool, List[bool]], inplace: bool, na_position: str
+    ):
+        if isinstance(ascending, bool):
+            ascending = [ascending] * len(by)
+        if len(ascending) != len(by):
+            raise ValueError(
+                "Length of ascending ({}) != length of by ({})".format(len(ascending), len(by))
+            )
+        if na_position not in ("first", "last"):
+            raise ValueError("invalid na_position: '{}'".format(na_position))
+
+        # Mapper: Get a spark column function for (ascending, na_position) combination
+        # Note that 'asc_nulls_first' and friends were added as of Spark 2.4, see SPARK-23847.
+        mapper = {
+            (True, "first"): lambda x: Column(getattr(x._jc, "asc_nulls_first")()),
+            (True, "last"): lambda x: Column(getattr(x._jc, "asc_nulls_last")()),
+            (False, "first"): lambda x: Column(getattr(x._jc, "desc_nulls_first")()),
+            (False, "last"): lambda x: Column(getattr(x._jc, "desc_nulls_last")()),
+        }
+        by = [mapper[(asc, na_position)](scol) for scol, asc in zip(by, ascending)]
+        sdf = self._internal.resolved_copy.spark_frame.sort(*(by + [NATURAL_ORDER_COLUMN_NAME]))
+        kdf = DataFrame(self._internal.with_new_sdf(sdf))  # type: DataFrame
+        if inplace:
+            self._update_internal_frame(kdf._internal)
+            return None
+        else:
+            return kdf
+
+    def sort_values(
+        self,
+        by: Union[Any, List[Any], Tuple, List[Tuple]],
+        ascending: Union[bool, List[bool]] = True,
+        inplace: bool = False,
+        na_position: str = "last",
+    ) -> Optional["DataFrame"]:
+        """
+        Sort by the values along either axis.
+
+        Parameters
+        ----------
+        by : str or list of str
+        ascending : bool or list of bool, default True
+             Sort ascending vs. descending. Specify list for multiple sort
+             orders.  If this is a list of bools, must match the length of
+             the by.
+        inplace : bool, default False
+             if True, perform operation in-place
+        na_position : {'first', 'last'}, default 'last'
+             `first` puts NaNs at the beginning, `last` puts NaNs at the end
+
+        Returns
+        -------
+        sorted_obj : DataFrame
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({
+        ...     'col1': ['A', 'B', None, 'D', 'C'],
+        ...     'col2': [2, 9, 8, 7, 4],
+        ...     'col3': [0, 9, 4, 2, 3],
+        ...   },
+        ...   columns=['col1', 'col2', 'col3'])
+        >>> df
+           col1  col2  col3
+        0     A     2     0
+        1     B     9     9
+        2  None     8     4
+        3     D     7     2
+        4     C     4     3
+
+        Sort by col1
+
+        >>> df.sort_values(by=['col1'])
+           col1  col2  col3
+        0     A     2     0
+        1     B     9     9
+        4     C     4     3
+        3     D     7     2
+        2  None     8     4
+
+        Sort Descending
+
+        >>> df.sort_values(by='col1', ascending=False)
+           col1  col2  col3
+        3     D     7     2
+        4     C     4     3
+        1     B     9     9
+        0     A     2     0
+        2  None     8     4
+
+        Sort by multiple columns
+
+        >>> df = ps.DataFrame({
+        ...     'col1': ['A', 'A', 'B', None, 'D', 'C'],
+        ...     'col2': [2, 1, 9, 8, 7, 4],
+        ...     'col3': [0, 1, 9, 4, 2, 3],
+        ...   },
+        ...   columns=['col1', 'col2', 'col3'])
+        >>> df.sort_values(by=['col1', 'col2'])
+           col1  col2  col3
+        1     A     1     1
+        0     A     2     0
+        2     B     9     9
+        5     C     4     3
+        4     D     7     2
+        3  None     8     4
+        """
+        inplace = validate_bool_kwarg(inplace, "inplace")
+        if is_name_like_value(by):
+            by = [by]
+        else:
+            assert is_list_like(by), type(by)
+
+        new_by = []
+        for colname in by:
+            ser = self[colname]
+            if not isinstance(ser, ps.Series):
+                raise ValueError(
+                    "The column %s is not unique. For a multi-index, the label must be a tuple "
+                    "with elements corresponding to each level." % name_like_string(colname)
+                )
+            new_by.append(ser.spark.column)
+
+        return self._sort(by=new_by, ascending=ascending, inplace=inplace, na_position=na_position)
+
+    def sort_index(
+        self,
+        axis: int = 0,
+        level: Optional[Union[int, List[int]]] = None,
+        ascending: bool = True,
+        inplace: bool = False,
+        kind: str = None,
+        na_position: str = "last",
+    ) -> Optional["DataFrame"]:
+        """
+        Sort object by labels (along an axis)
+
+        Parameters
+        ----------
+        axis : index, columns to direct sorting. Currently, only axis = 0 is supported.
+        level : int or level name or list of ints or list of level names
+            if not None, sort on values in specified index level(s)
+        ascending : boolean, default True
+            Sort ascending vs. descending
+        inplace : bool, default False
+            if True, perform operation in-place
+        kind : str, default None
+            pandas-on-Spark does not allow specifying the sorting algorithm at the moment,
+            default None
+        na_position : {‘first’, ‘last’}, default ‘last’
+            first puts NaNs at the beginning, last puts NaNs at the end. Not implemented for
+            MultiIndex.
+
+        Returns
+        -------
+        sorted_obj : DataFrame
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'A': [2, 1, np.nan]}, index=['b', 'a', np.nan])
+
+        >>> df.sort_index()
+               A
+        a    1.0
+        b    2.0
+        NaN  NaN
+
+        >>> df.sort_index(ascending=False)
+               A
+        b    2.0
+        a    1.0
+        NaN  NaN
+
+        >>> df.sort_index(na_position='first')
+               A
+        NaN  NaN
+        a    1.0
+        b    2.0
+
+        >>> df.sort_index(inplace=True)
+        >>> df
+               A
+        a    1.0
+        b    2.0
+        NaN  NaN
+
+        >>> df = ps.DataFrame({'A': range(4), 'B': range(4)[::-1]},
+        ...                   index=[['b', 'b', 'a', 'a'], [1, 0, 1, 0]],
+        ...                   columns=['A', 'B'])
+
+        >>> df.sort_index()
+             A  B
+        a 0  3  0
+          1  2  1
+        b 0  1  2
+          1  0  3
+
+        >>> df.sort_index(level=1)  # doctest: +SKIP
+             A  B
+        a 0  3  0
+        b 0  1  2
+        a 1  2  1
+        b 1  0  3
+
+        >>> df.sort_index(level=[1, 0])
+             A  B
+        a 0  3  0
+        b 0  1  2
+        a 1  2  1
+        b 1  0  3
+        """
+        inplace = validate_bool_kwarg(inplace, "inplace")
+        axis = validate_axis(axis)
+        if axis != 0:
+            raise NotImplementedError("No other axis than 0 are supported at the moment")
+        if kind is not None:
+            raise NotImplementedError(
+                "Specifying the sorting algorithm is not supported at the moment."
+            )
+
+        if level is None or (is_list_like(level) and len(level) == 0):  # type: ignore
+            by = self._internal.index_spark_columns
+        elif is_list_like(level):
+            by = [self._internal.index_spark_columns[l] for l in level]  # type: ignore
+        else:
+            by = [self._internal.index_spark_columns[level]]  # type: ignore
+
+        return self._sort(by=by, ascending=ascending, inplace=inplace, na_position=na_position)
+
+    def swaplevel(self, i=-2, j=-1, axis=0) -> "DataFrame":
+        """
+        Swap levels i and j in a MultiIndex on a particular axis.
+
+        Parameters
+        ----------
+        i, j : int or str
+            Levels of the indices to be swapped. Can pass level name as string.
+        axis : {0 or 'index', 1 or 'columns'}, default 0
+            The axis to swap levels on. 0 or 'index' for row-wise, 1 or
+            'columns' for column-wise.
+
+        Returns
+        -------
+        DataFrame
+            DataFrame with levels swapped in MultiIndex.
+
+        Examples
+        --------
+        >>> midx = pd.MultiIndex.from_arrays(
+        ...     [['red', 'blue'], [1, 2], ['s', 'm']], names = ['color', 'number', 'size'])
+        >>> midx  # doctest: +SKIP
+        MultiIndex([( 'red', 1, 's'),
+                    ('blue', 2, 'm')],
+                   names=['color', 'number', 'size'])
+
+        Swap levels in a MultiIndex on index.
+
+        >>> kdf = ps.DataFrame({'x': [5, 6], 'y':[5, 6]}, index=midx)
+        >>> kdf  # doctest: +NORMALIZE_WHITESPACE
+                           x  y
+        color number size
+        red   1      s     5  5
+        blue  2      m     6  6
+
+        >>> kdf.swaplevel()  # doctest: +NORMALIZE_WHITESPACE
+                           x  y
+        color size number
+        red   s    1       5  5
+        blue  m    2       6  6
+
+        >>> kdf.swaplevel(0, 1)  # doctest: +NORMALIZE_WHITESPACE
+                           x  y
+        number color size
+        1      red   s     5  5
+        2      blue  m     6  6
+
+        >>> kdf.swaplevel('number', 'size')  # doctest: +NORMALIZE_WHITESPACE
+                           x  y
+        color size number
+        red   s    1       5  5
+        blue  m    2       6  6
+
+        Swap levels in a MultiIndex on columns.
+
+        >>> kdf = ps.DataFrame({'x': [5, 6], 'y':[5, 6]})
+        >>> kdf.columns = midx
+        >>> kdf
+        color  red blue
+        number   1    2
+        size     s    m
+        0        5    5
+        1        6    6
+
+        >>> kdf.swaplevel(axis=1)
+        color  red blue
+        size     s    m
+        number   1    2
+        0        5    5
+        1        6    6
+
+        >>> kdf.swaplevel(axis=1)
+        color  red blue
+        size     s    m
+        number   1    2
+        0        5    5
+        1        6    6
+
+        >>> kdf.swaplevel(0, 1, axis=1)
+        number   1    2
+        color  red blue
+        size     s    m
+        0        5    5
+        1        6    6
+
+        >>> kdf.swaplevel('number', 'color', axis=1)
+        number   1    2
+        color  red blue
+        size     s    m
+        0        5    5
+        1        6    6
+        """
+        axis = validate_axis(axis)
+        if axis == 0:
+            internal = self._swaplevel_index(i, j)
+        else:
+            assert axis == 1
+            internal = self._swaplevel_columns(i, j)
+
+        return DataFrame(internal)
+
+    def swapaxes(self, i: Union[str, int], j: Union[str, int], copy: bool = True) -> "DataFrame":
+        """
+        Interchange axes and swap values axes appropriately.
+
+        .. note:: This method is based on an expensive operation due to the nature
+            of big data. Internally it needs to generate each row for each value, and
+            then group twice - it is a huge operation. To prevent misusage, this method
+            has the 'compute.max_rows' default limit of input length, and raises a ValueError.
+
+                >>> from pyspark.pandas.config import option_context
+                >>> with option_context('compute.max_rows', 1000):  # doctest: +NORMALIZE_WHITESPACE
+                ...     ps.DataFrame({'a': range(1001)}).swapaxes(i=0, j=1)
+                Traceback (most recent call last):
+                  ...
+                ValueError: Current DataFrame has more then the given limit 1000 rows.
+                Please set 'compute.max_rows' by using 'pyspark.pandas.config.set_option'
+                to retrieve to retrieve more than 1000 rows. Note that, before changing the
+                'compute.max_rows', this operation is considerably expensive.
+
+        Parameters
+        ----------
+        i: {0 or 'index', 1 or 'columns'}. The axis to swap.
+        j: {0 or 'index', 1 or 'columns'}. The axis to swap.
+        copy : bool, default True.
+
+        Returns
+        -------
+        DataFrame
+
+        Examples
+        --------
+        >>> kdf = ps.DataFrame(
+        ...     [[1, 2, 3], [4, 5, 6], [7, 8, 9]], index=['x', 'y', 'z'], columns=['a', 'b', 'c']
+        ... )
+        >>> kdf
+           a  b  c
+        x  1  2  3
+        y  4  5  6
+        z  7  8  9
+        >>> kdf.swapaxes(i=1, j=0)
+           x  y  z
+        a  1  4  7
+        b  2  5  8
+        c  3  6  9
+        >>> kdf.swapaxes(i=1, j=1)
+           a  b  c
+        x  1  2  3
+        y  4  5  6
+        z  7  8  9
+        """
+        assert copy is True
+
+        i = validate_axis(i)
+        j = validate_axis(j)
+
+        return self.copy() if i == j else self.transpose()
+
+    def _swaplevel_columns(self, i, j) -> InternalFrame:
+        assert isinstance(self.columns, pd.MultiIndex)
+        for index in (i, j):
+            if not isinstance(index, int) and index not in self.columns.names:
+                raise KeyError("Level %s not found" % index)
+
+        i = i if isinstance(i, int) else self.columns.names.index(i)
+        j = j if isinstance(j, int) else self.columns.names.index(j)
+        for index in (i, j):
+            if index >= len(self.columns) or index < -len(self.columns):
+                raise IndexError(
+                    "Too many levels: Columns have only %s levels, "
+                    "%s is not a valid level number" % (self._internal.index_level, index)
+                )
+
+        column_label_names = self._internal.column_label_names.copy()
+        column_label_names[i], column_label_names[j], = (
+            column_label_names[j],
+            column_label_names[i],
+        )
+        column_labels = self._internal._column_labels
+        column_label_list = [list(label) for label in column_labels]
+        for label_list in column_label_list:
+            label_list[i], label_list[j] = label_list[j], label_list[i]
+        column_labels = [tuple(x) for x in column_label_list]
+        internal = self._internal.copy(
+            column_label_names=list(column_label_names), column_labels=list(column_labels)
+        )
+        return internal
+
+    def _swaplevel_index(self, i, j) -> InternalFrame:
+        assert isinstance(self.index, ps.MultiIndex)
+        for index in (i, j):
+            if not isinstance(index, int) and index not in self.index.names:
+                raise KeyError("Level %s not found" % index)
+
+        i = i if isinstance(i, int) else self.index.names.index(i)
+        j = j if isinstance(j, int) else self.index.names.index(j)
+        for index in (i, j):
+            if index >= self._internal.index_level or index < -self._internal.index_level:
+                raise IndexError(
+                    "Too many levels: Index has only %s levels, "
+                    "%s is not a valid level number" % (self._internal.index_level, index)
+                )
+
+        index_map = list(
+            zip(
+                self._internal.index_spark_columns,
+                self._internal.index_names,
+                self._internal.index_dtypes,
+            )
+        )
+        index_map[i], index_map[j], = index_map[j], index_map[i]
+        index_spark_columns, index_names, index_dtypes = zip(*index_map)
+        internal = self._internal.copy(
+            index_spark_columns=list(index_spark_columns),
+            index_names=list(index_names),
+            index_dtypes=list(index_dtypes),
+        )
+        return internal
+
+    # TODO:  add keep = First
+    def nlargest(self, n: int, columns: "Any") -> "DataFrame":
+        """
+        Return the first `n` rows ordered by `columns` in descending order.
+
+        Return the first `n` rows with the largest values in `columns`, in
+        descending order. The columns that are not specified are returned as
+        well, but not used for ordering.
+
+        This method is equivalent to
+        ``df.sort_values(columns, ascending=False).head(n)``, but more
+        performant in pandas.
+        In pandas-on-Spark, thanks to Spark's lazy execution and query optimizer,
+        the two would have same performance.
+
+        Parameters
+        ----------
+        n : int
+            Number of rows to return.
+        columns : label or list of labels
+            Column label(s) to order by.
+
+        Returns
+        -------
+        DataFrame
+            The first `n` rows ordered by the given columns in descending
+            order.
+
+        See Also
+        --------
+        DataFrame.nsmallest : Return the first `n` rows ordered by `columns` in
+            ascending order.
+        DataFrame.sort_values : Sort DataFrame by the values.
+        DataFrame.head : Return the first `n` rows without re-ordering.
+
+        Notes
+        -----
+
+        This function cannot be used with all column types. For example, when
+        specifying columns with `object` or `category` dtypes, ``TypeError`` is
+        raised.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'X': [1, 2, 3, 5, 6, 7, np.nan],
+        ...                    'Y': [6, 7, 8, 9, 10, 11, 12]})
+        >>> df
+             X   Y
+        0  1.0   6
+        1  2.0   7
+        2  3.0   8
+        3  5.0   9
+        4  6.0  10
+        5  7.0  11
+        6  NaN  12
+
+        In the following example, we will use ``nlargest`` to select the three
+        rows having the largest values in column "population".
+
+        >>> df.nlargest(n=3, columns='X')
+             X   Y
+        5  7.0  11
+        4  6.0  10
+        3  5.0   9
+
+        >>> df.nlargest(n=3, columns=['Y', 'X'])
+             X   Y
+        6  NaN  12
+        5  7.0  11
+        4  6.0  10
+
+        """
+        return self.sort_values(by=columns, ascending=False).head(n=n)
+
+    # TODO: add keep = First
+    def nsmallest(self, n: int, columns: "Any") -> "DataFrame":
+        """
+        Return the first `n` rows ordered by `columns` in ascending order.
+
+        Return the first `n` rows with the smallest values in `columns`, in
+        ascending order. The columns that are not specified are returned as
+        well, but not used for ordering.
+
+        This method is equivalent to ``df.sort_values(columns, ascending=True).head(n)``,
+        but more performant. In pandas-on-Spark, thanks to Spark's lazy execution and query
+        optimizer, the two would have same performance.
+
+        Parameters
+        ----------
+        n : int
+            Number of items to retrieve.
+        columns : list or str
+            Column name or names to order by.
+
+        Returns
+        -------
+        DataFrame
+
+        See Also
+        --------
+        DataFrame.nlargest : Return the first `n` rows ordered by `columns` in
+            descending order.
+        DataFrame.sort_values : Sort DataFrame by the values.
+        DataFrame.head : Return the first `n` rows without re-ordering.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'X': [1, 2, 3, 5, 6, 7, np.nan],
+        ...                    'Y': [6, 7, 8, 9, 10, 11, 12]})
+        >>> df
+             X   Y
+        0  1.0   6
+        1  2.0   7
+        2  3.0   8
+        3  5.0   9
+        4  6.0  10
+        5  7.0  11
+        6  NaN  12
+
+        In the following example, we will use ``nsmallest`` to select the
+        three rows having the smallest values in column "a".
+
+        >>> df.nsmallest(n=3, columns='X') # doctest: +NORMALIZE_WHITESPACE
+             X   Y
+        0  1.0   6
+        1  2.0   7
+        2  3.0   8
+
+        To order by the largest values in column "a" and then "c", we can
+        specify multiple columns like in the next example.
+
+        >>> df.nsmallest(n=3, columns=['Y', 'X']) # doctest: +NORMALIZE_WHITESPACE
+             X   Y
+        0  1.0   6
+        1  2.0   7
+        2  3.0   8
+        """
+        return self.sort_values(by=columns, ascending=True).head(n=n)
+
+    def isin(self, values) -> "DataFrame":
+        """
+        Whether each element in the DataFrame is contained in values.
+
+        Parameters
+        ----------
+        values : iterable or dict
+           The sequence of values to test. If values is a dict,
+           the keys must be the column names, which must match.
+           Series and DataFrame are not supported.
+
+        Returns
+        -------
+        DataFrame
+            DataFrame of booleans showing whether each element in the DataFrame
+            is contained in values.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'num_legs': [2, 4], 'num_wings': [2, 0]},
+        ...                   index=['falcon', 'dog'],
+        ...                   columns=['num_legs', 'num_wings'])
+        >>> df
+                num_legs  num_wings
+        falcon         2          2
+        dog            4          0
+
+        When ``values`` is a list check whether every value in the DataFrame
+        is present in the list (which animals have 0 or 2 legs or wings)
+
+        >>> df.isin([0, 2])
+                num_legs  num_wings
+        falcon      True       True
+        dog        False       True
+
+        When ``values`` is a dict, we can pass values to check for each
+        column separately:
+
+        >>> df.isin({'num_wings': [0, 3]})
+                num_legs  num_wings
+        falcon     False      False
+        dog        False       True
+        """
+        if isinstance(values, (pd.DataFrame, pd.Series)):
+            raise NotImplementedError("DataFrame and Series are not supported")
+        if isinstance(values, dict) and not set(values.keys()).issubset(self.columns):
+            raise AttributeError(
+                "'DataFrame' object has no attribute %s"
+                % (set(values.keys()).difference(self.columns))
+            )
+
+        data_spark_columns = []
+        if isinstance(values, dict):
+            for i, col in enumerate(self.columns):
+                if col in values:
+                    item = values[col]
+                    item = item.tolist() if isinstance(item, np.ndarray) else list(item)
+                    data_spark_columns.append(
+                        self._internal.spark_column_for(self._internal.column_labels[i])
+                        .isin(item)
+                        .alias(self._internal.data_spark_column_names[i])
+                    )
+                else:
+                    data_spark_columns.append(
+                        F.lit(False).alias(self._internal.data_spark_column_names[i])
+                    )
+        elif is_list_like(values):
+            values = values.tolist() if isinstance(values, np.ndarray) else list(values)
+            data_spark_columns += [
+                self._internal.spark_column_for(label)
+                .isin(values)
+                .alias(self._internal.spark_column_name_for(label))
+                for label in self._internal.column_labels
+            ]
+        else:
+            raise TypeError("Values should be iterable, Series, DataFrame or dict.")
+
+        return DataFrame(self._internal.with_new_columns(data_spark_columns))
+
+    @property
+    def shape(self) -> Tuple[int, int]:
+        """
+        Return a tuple representing the dimensionality of the DataFrame.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'col1': [1, 2], 'col2': [3, 4]})
+        >>> df.shape
+        (2, 2)
+
+        >>> df = ps.DataFrame({'col1': [1, 2], 'col2': [3, 4],
+        ...                    'col3': [5, 6]})
+        >>> df.shape
+        (2, 3)
+        """
+        return len(self), len(self.columns)
+
+    def merge(
+        self,
+        right: "DataFrame",
+        how: str = "inner",
+        on: Optional[Union[Any, List[Any], Tuple, List[Tuple]]] = None,
+        left_on: Optional[Union[Any, List[Any], Tuple, List[Tuple]]] = None,
+        right_on: Optional[Union[Any, List[Any], Tuple, List[Tuple]]] = None,
+        left_index: bool = False,
+        right_index: bool = False,
+        suffixes: Tuple[str, str] = ("_x", "_y"),
+    ) -> "DataFrame":
+        """
+        Merge DataFrame objects with a database-style join.
+
+        The index of the resulting DataFrame will be one of the following:
+            - 0...n if no index is used for merging
+            - Index of the left DataFrame if merged only on the index of the right DataFrame
+            - Index of the right DataFrame if merged only on the index of the left DataFrame
+            - All involved indices if merged using the indices of both DataFrames
+                e.g. if `left` with indices (a, x) and `right` with indices (b, x), the result will
+                be an index (x, a, b)
+
+        Parameters
+        ----------
+        right: Object to merge with.
+        how: Type of merge to be performed.
+            {'left', 'right', 'outer', 'inner'}, default 'inner'
+
+            left: use only keys from left frame, similar to a SQL left outer join; not preserve
+                key order unlike pandas.
+            right: use only keys from right frame, similar to a SQL right outer join; not preserve
+                key order unlike pandas.
+            outer: use union of keys from both frames, similar to a SQL full outer join; sort keys
+                lexicographically.
+            inner: use intersection of keys from both frames, similar to a SQL inner join;
+                not preserve the order of the left keys unlike pandas.
+        on: Column or index level names to join on. These must be found in both DataFrames. If on
+            is None and not merging on indexes then this defaults to the intersection of the
+            columns in both DataFrames.
+        left_on: Column or index level names to join on in the left DataFrame. Can also
+            be an array or list of arrays of the length of the left DataFrame.
+            These arrays are treated as if they are columns.
+        right_on: Column or index level names to join on in the right DataFrame. Can also
+            be an array or list of arrays of the length of the right DataFrame.
+            These arrays are treated as if they are columns.
+        left_index: Use the index from the left DataFrame as the join key(s). If it is a
+            MultiIndex, the number of keys in the other DataFrame (either the index or a number of
+            columns) must match the number of levels.
+        right_index: Use the index from the right DataFrame as the join key. Same caveats as
+            left_index.
+        suffixes: Suffix to apply to overlapping column names in the left and right side,
+            respectively.
+
+        Returns
+        -------
+        DataFrame
+            A DataFrame of the two merged objects.
+
+        See Also
+        --------
+        DataFrame.join : Join columns of another DataFrame.
+        DataFrame.update : Modify in place using non-NA values from another DataFrame.
+        DataFrame.hint : Specifies some hint on the current DataFrame.
+        broadcast : Marks a DataFrame as small enough for use in broadcast joins.
+
+        Examples
+        --------
+        >>> df1 = ps.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'],
+        ...                     'value': [1, 2, 3, 5]},
+        ...                    columns=['lkey', 'value'])
+        >>> df2 = ps.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'],
+        ...                     'value': [5, 6, 7, 8]},
+        ...                    columns=['rkey', 'value'])
+        >>> df1
+          lkey  value
+        0  foo      1
+        1  bar      2
+        2  baz      3
+        3  foo      5
+        >>> df2
+          rkey  value
+        0  foo      5
+        1  bar      6
+        2  baz      7
+        3  foo      8
+
+        Merge df1 and df2 on the lkey and rkey columns. The value columns have
+        the default suffixes, _x and _y, appended.
+
+        >>> merged = df1.merge(df2, left_on='lkey', right_on='rkey')
+        >>> merged.sort_values(by=['lkey', 'value_x', 'rkey', 'value_y'])  # doctest: +ELLIPSIS
+          lkey  value_x rkey  value_y
+        ...bar        2  bar        6
+        ...baz        3  baz        7
+        ...foo        1  foo        5
+        ...foo        1  foo        8
+        ...foo        5  foo        5
+        ...foo        5  foo        8
+
+        >>> left_kdf = ps.DataFrame({'A': [1, 2]})
+        >>> right_kdf = ps.DataFrame({'B': ['x', 'y']}, index=[1, 2])
+
+        >>> left_kdf.merge(right_kdf, left_index=True, right_index=True).sort_index()
+           A  B
+        1  2  x
+
+        >>> left_kdf.merge(right_kdf, left_index=True, right_index=True, how='left').sort_index()
+           A     B
+        0  1  None
+        1  2     x
+
+        >>> left_kdf.merge(right_kdf, left_index=True, right_index=True, how='right').sort_index()
+             A  B
+        1  2.0  x
+        2  NaN  y
+
+        >>> left_kdf.merge(right_kdf, left_index=True, right_index=True, how='outer').sort_index()
+             A     B
+        0  1.0  None
+        1  2.0     x
+        2  NaN     y
+
+        Notes
+        -----
+        As described in #263, joining string columns currently returns None for missing values
+            instead of NaN.
+        """
+
+        def to_list(os: Optional[Union[Any, List[Any], Tuple, List[Tuple]]]) -> List[Tuple]:
+            if os is None:
+                return []
+            elif is_name_like_tuple(os):
+                return [os]  # type: ignore
+            elif is_name_like_value(os):
+                return [(os,)]
+            else:
+                return [o if is_name_like_tuple(o) else (o,) for o in os]
+
+        if isinstance(right, ps.Series):
+            right = right.to_frame()
+
+        if on:
+            if left_on or right_on:
+                raise ValueError(
+                    'Can only pass argument "on" OR "left_on" and "right_on", '
+                    "not a combination of both."
+                )
+            left_key_names = list(map(self._internal.spark_column_name_for, to_list(on)))
+            right_key_names = list(map(right._internal.spark_column_name_for, to_list(on)))
+        else:
+            # TODO: need special handling for multi-index.
+            if left_index:
+                left_key_names = self._internal.index_spark_column_names
+            else:
+                left_key_names = list(map(self._internal.spark_column_name_for, to_list(left_on)))
+            if right_index:
+                right_key_names = right._internal.index_spark_column_names
+            else:
+                right_key_names = list(
+                    map(right._internal.spark_column_name_for, to_list(right_on))
+                )
+
+            if left_key_names and not right_key_names:
+                raise ValueError("Must pass right_on or right_index=True")
+            if right_key_names and not left_key_names:
+                raise ValueError("Must pass left_on or left_index=True")
+            if not left_key_names and not right_key_names:
+                common = list(self.columns.intersection(right.columns))
+                if len(common) == 0:
+                    raise ValueError(
+                        "No common columns to perform merge on. Merge options: "
+                        "left_on=None, right_on=None, left_index=False, right_index=False"
+                    )
+                left_key_names = list(map(self._internal.spark_column_name_for, to_list(common)))
+                right_key_names = list(map(right._internal.spark_column_name_for, to_list(common)))
+            if len(left_key_names) != len(right_key_names):
+                raise ValueError("len(left_keys) must equal len(right_keys)")
+
+        # We should distinguish the name to avoid ambiguous column name after merging.
+        right_prefix = "__right_"
+        right_key_names = [right_prefix + right_key_name for right_key_name in right_key_names]
+
+        how = validate_how(how)
+
+        def resolve(internal, side):
+            rename = lambda col: "__{}_{}".format(side, col)
+            internal = internal.resolved_copy
+            sdf = internal.spark_frame
+            sdf = sdf.select(
+                [
+                    scol_for(sdf, col).alias(rename(col))
+                    for col in sdf.columns
+                    if col not in HIDDEN_COLUMNS
+                ]
+                + list(HIDDEN_COLUMNS)
+            )
+            return internal.copy(
+                spark_frame=sdf,
+                index_spark_columns=[
+                    scol_for(sdf, rename(col)) for col in internal.index_spark_column_names
+                ],
+                data_spark_columns=[
+                    scol_for(sdf, rename(col)) for col in internal.data_spark_column_names
+                ],
+            )
+
+        left_internal = self._internal.resolved_copy
+        right_internal = resolve(right._internal, "right")
+
+        left_table = left_internal.spark_frame.alias("left_table")
+        right_table = right_internal.spark_frame.alias("right_table")
+
+        left_key_columns = [scol_for(left_table, label) for label in left_key_names]
+        right_key_columns = [scol_for(right_table, label) for label in right_key_names]
+
+        join_condition = reduce(
+            lambda x, y: x & y,
+            [lkey == rkey for lkey, rkey in zip(left_key_columns, right_key_columns)],
+        )
+
+        joined_table = left_table.join(right_table, join_condition, how=how)
+
+        # Unpack suffixes tuple for convenience
+        left_suffix = suffixes[0]
+        right_suffix = suffixes[1]
+
+        # Append suffixes to columns with the same name to avoid conflicts later
+        duplicate_columns = set(left_internal.column_labels) & set(right_internal.column_labels)
+
+        exprs = []
+        data_columns = []
+        column_labels = []
+
+        left_scol_for = lambda label: scol_for(
+            left_table, left_internal.spark_column_name_for(label)
+        )
+        right_scol_for = lambda label: scol_for(
+            right_table, right_internal.spark_column_name_for(label)
+        )
+
+        for label in left_internal.column_labels:
+            col = left_internal.spark_column_name_for(label)
+            scol = left_scol_for(label)
+            if label in duplicate_columns:
+                spark_column_name = left_internal.spark_column_name_for(label)
+                if (
+                    spark_column_name in left_key_names
+                    and (right_prefix + spark_column_name) in right_key_names
+                ):
+                    right_scol = right_scol_for(label)
+                    if how == "right":
+                        scol = right_scol.alias(col)
+                    elif how == "full":
+                        scol = F.when(scol.isNotNull(), scol).otherwise(right_scol).alias(col)
+                    else:
+                        pass
+                else:
+                    col = col + left_suffix
+                    scol = scol.alias(col)
+                    label = tuple([str(label[0]) + left_suffix] + list(label[1:]))
+            exprs.append(scol)
+            data_columns.append(col)
+            column_labels.append(label)
+        for label in right_internal.column_labels:
+            # recover `right_prefix` here.
+            col = right_internal.spark_column_name_for(label)[len(right_prefix):]
+            scol = right_scol_for(label).alias(col)
+            if label in duplicate_columns:
+                spark_column_name = left_internal.spark_column_name_for(label)
+                if (
+                    spark_column_name in left_key_names
+                    and (right_prefix + spark_column_name) in right_key_names
+                ):
+                    continue
+                else:
+                    col = col + right_suffix
+                    scol = scol.alias(col)
+                    label = tuple([str(label[0]) + right_suffix] + list(label[1:]))
+            exprs.append(scol)
+            data_columns.append(col)
+            column_labels.append(label)
+
+        left_index_scols = left_internal.index_spark_columns
+        right_index_scols = right_internal.index_spark_columns
+
+        # Retain indices if they are used for joining
+        if left_index:
+            if right_index:
+                if how in ("inner", "left"):
+                    exprs.extend(left_index_scols)
+                    index_spark_column_names = left_internal.index_spark_column_names
+                    index_names = left_internal.index_names
+                elif how == "right":
+                    exprs.extend(right_index_scols)
+                    index_spark_column_names = right_internal.index_spark_column_names
+                    index_names = right_internal.index_names
+                else:
+                    index_spark_column_names = left_internal.index_spark_column_names
+                    index_names = left_internal.index_names
+                    for col, left_scol, right_scol in zip(
+                        index_spark_column_names, left_index_scols, right_index_scols
+                    ):
+                        scol = F.when(left_scol.isNotNull(), left_scol).otherwise(right_scol)
+                        exprs.append(scol.alias(col))
+            else:
+                exprs.extend(right_index_scols)
+                index_spark_column_names = right_internal.index_spark_column_names
+                index_names = right_internal.index_names
+        elif right_index:
+            exprs.extend(left_index_scols)
+            index_spark_column_names = left_internal.index_spark_column_names
+            index_names = left_internal.index_names
+        else:
+            index_spark_column_names = []
+            index_names = []
+
+        selected_columns = joined_table.select(*exprs)
+
+        internal = InternalFrame(
+            spark_frame=selected_columns,
+            index_spark_columns=[
+                scol_for(selected_columns, col) for col in index_spark_column_names
+            ],
+            index_names=index_names,
+            column_labels=column_labels,
+            data_spark_columns=[scol_for(selected_columns, col) for col in data_columns],
+        )
+        return DataFrame(internal)
+
+    def join(
+        self,
+        right: "DataFrame",
+        on: Optional[Union[Any, List[Any], Tuple, List[Tuple]]] = None,
+        how: str = "left",
+        lsuffix: str = "",
+        rsuffix: str = "",
+    ) -> "DataFrame":
+        """
+        Join columns of another DataFrame.
+
+        Join columns with `right` DataFrame either on index or on a key column. Efficiently join
+        multiple DataFrame objects by index at once by passing a list.
+
+        Parameters
+        ----------
+        right: DataFrame, Series
+        on: str, list of str, or array-like, optional
+            Column or index level name(s) in the caller to join on the index in `right`, otherwise
+            joins index-on-index. If multiple values given, the `right` DataFrame must have a
+            MultiIndex. Can pass an array as the join key if it is not already contained in the
+            calling DataFrame. Like an Excel VLOOKUP operation.
+        how: {'left', 'right', 'outer', 'inner'}, default 'left'
+            How to handle the operation of the two objects.
+
+            * left: use `left` frame’s index (or column if on is specified).
+            * right: use `right`’s index.
+            * outer: form union of `left` frame’s index (or column if on is specified) with
+              right’s index, and sort it. lexicographically.
+            * inner: form intersection of `left` frame’s index (or column if on is specified)
+              with `right`’s index, preserving the order of the `left`’s one.
+        lsuffix : str, default ''
+            Suffix to use from left frame's overlapping columns.
+        rsuffix : str, default ''
+            Suffix to use from `right` frame's overlapping columns.
+
+        Returns
+        -------
+        DataFrame
+            A dataframe containing columns from both the `left` and `right`.
+
+        See Also
+        --------
+        DataFrame.merge: For column(s)-on-columns(s) operations.
+        DataFrame.update : Modify in place using non-NA values from another DataFrame.
+        DataFrame.hint : Specifies some hint on the current DataFrame.
+        broadcast : Marks a DataFrame as small enough for use in broadcast joins.
+
+        Notes
+        -----
+        Parameters on, lsuffix, and rsuffix are not supported when passing a list of DataFrame
+        objects.
+
+        Examples
+        --------
+        >>> kdf1 = ps.DataFrame({'key': ['K0', 'K1', 'K2', 'K3'],
+        ...                      'A': ['A0', 'A1', 'A2', 'A3']},
+        ...                     columns=['key', 'A'])
+        >>> kdf2 = ps.DataFrame({'key': ['K0', 'K1', 'K2'],
+        ...                      'B': ['B0', 'B1', 'B2']},
+        ...                     columns=['key', 'B'])
+        >>> kdf1
+          key   A
+        0  K0  A0
+        1  K1  A1
+        2  K2  A2
+        3  K3  A3
+        >>> kdf2
+          key   B
+        0  K0  B0
+        1  K1  B1
+        2  K2  B2
+
+        Join DataFrames using their indexes.
+
+        >>> join_kdf = kdf1.join(kdf2, lsuffix='_left', rsuffix='_right')
+        >>> join_kdf.sort_values(by=join_kdf.columns)
+          key_left   A key_right     B
+        0       K0  A0        K0    B0
+        1       K1  A1        K1    B1
+        2       K2  A2        K2    B2
+        3       K3  A3      None  None
+
+        If we want to join using the key columns, we need to set key to be the index in both df and
+        right. The joined DataFrame will have key as its index.
+
+        >>> join_kdf = kdf1.set_index('key').join(kdf2.set_index('key'))
+        >>> join_kdf.sort_values(by=join_kdf.columns) # doctest: +NORMALIZE_WHITESPACE
+              A     B
+        key
+        K0   A0    B0
+        K1   A1    B1
+        K2   A2    B2
+        K3   A3  None
+
+        Another option to join using the key columns is to use the on parameter. DataFrame.join
+        always uses right’s index but we can use any column in df. This method not preserve the
+        original DataFrame’s index in the result unlike pandas.
+
+        >>> join_kdf = kdf1.join(kdf2.set_index('key'), on='key')
+        >>> join_kdf.index
+        Int64Index([0, 1, 2, 3], dtype='int64')
+        """
+        if isinstance(right, ps.Series):
+            common = list(self.columns.intersection([right.name]))
+        else:
+            common = list(self.columns.intersection(right.columns))
+        if len(common) > 0 and not lsuffix and not rsuffix:
+            raise ValueError(
+                "columns overlap but no suffix specified: " "{rename}".format(rename=common)
+            )
+
+        need_set_index = False
+        if on:
+            if not is_list_like(on):
+                on = [on]  # type: ignore
+            if len(on) != right._internal.index_level:
+                raise ValueError(
+                    'len(left_on) must equal the number of levels in the index of "right"'
+                )
+
+            need_set_index = len(set(on) & set(self.index.names)) == 0
+        if need_set_index:
+            self = self.set_index(on)
+        join_kdf = self.merge(
+            right, left_index=True, right_index=True, how=how, suffixes=(lsuffix, rsuffix)
+        )
+        return join_kdf.reset_index() if need_set_index else join_kdf
+
+    def append(
+        self,
+        other: "DataFrame",
+        ignore_index: bool = False,
+        verify_integrity: bool = False,
+        sort: bool = False,
+    ) -> "DataFrame":
+        """
+        Append rows of other to the end of caller, returning a new object.
+
+        Columns in other that are not in the caller are added as new columns.
+
+        Parameters
+        ----------
+        other : DataFrame or Series/dict-like object, or list of these
+            The data to append.
+
+        ignore_index : boolean, default False
+            If True, do not use the index labels.
+
+        verify_integrity : boolean, default False
+            If True, raise ValueError on creating index with duplicates.
+
+        sort : boolean, default False
+            Currently not supported.
+
+        Returns
+        -------
+        appended : DataFrame
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([[1, 2], [3, 4]], columns=list('AB'))
+
+        >>> df.append(df)
+           A  B
+        0  1  2
+        1  3  4
+        0  1  2
+        1  3  4
+
+        >>> df.append(df, ignore_index=True)
+           A  B
+        0  1  2
+        1  3  4
+        2  1  2
+        3  3  4
+        """
+        if isinstance(other, ps.Series):
+            raise TypeError("DataFrames.append() does not support appending Series to DataFrames")
+        if sort:
+            raise NotImplementedError("The 'sort' parameter is currently not supported")
+
+        if not ignore_index:
+            index_scols = self._internal.index_spark_columns
+            if len(index_scols) != other._internal.index_level:
+                raise ValueError("Both DataFrames have to have the same number of index levels")
+
+            if verify_integrity and len(index_scols) > 0:
+                if (
+                    self._internal.spark_frame.select(index_scols)
+                    .intersect(
+                        other._internal.spark_frame.select(other._internal.index_spark_columns)
+                    )
+                    .count()
+                ) > 0:
+                    raise ValueError("Indices have overlapping values")
+
+        # Lazy import to avoid circular dependency issues
+        from pyspark.pandas.namespace import concat
+
+        return cast(DataFrame, concat([self, other], ignore_index=ignore_index))
+
+    # TODO: add 'filter_func' and 'errors' parameter
+    def update(self, other: "DataFrame", join: str = "left", overwrite: bool = True) -> None:
+        """
+        Modify in place using non-NA values from another DataFrame.
+        Aligns on indices. There is no return value.
+
+        Parameters
+        ----------
+        other : DataFrame, or Series
+        join : 'left', default 'left'
+            Only left join is implemented, keeping the index and columns of the original object.
+        overwrite : bool, default True
+            How to handle non-NA values for overlapping keys:
+
+            * True: overwrite original DataFrame's values with values from `other`.
+            * False: only update values that are NA in the original DataFrame.
+
+        Returns
+        -------
+        None : method directly changes calling object
+
+        See Also
+        --------
+        DataFrame.merge : For column(s)-on-columns(s) operations.
+        DataFrame.join : Join columns of another DataFrame.
+        DataFrame.hint : Specifies some hint on the current DataFrame.
+        broadcast : Marks a DataFrame as small enough for use in broadcast joins.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'A': [1, 2, 3], 'B': [400, 500, 600]}, columns=['A', 'B'])
+        >>> new_df = ps.DataFrame({'B': [4, 5, 6], 'C': [7, 8, 9]}, columns=['B', 'C'])
+        >>> df.update(new_df)
+        >>> df.sort_index()
+           A  B
+        0  1  4
+        1  2  5
+        2  3  6
+
+        The DataFrame's length does not increase as a result of the update,
+        only values at matching index/column labels are updated.
+
+        >>> df = ps.DataFrame({'A': ['a', 'b', 'c'], 'B': ['x', 'y', 'z']}, columns=['A', 'B'])
+        >>> new_df = ps.DataFrame({'B': ['d', 'e', 'f', 'g', 'h', 'i']}, columns=['B'])
+        >>> df.update(new_df)
+        >>> df.sort_index()
+           A  B
+        0  a  d
+        1  b  e
+        2  c  f
+
+        For Series, it's name attribute must be set.
+
+        >>> df = ps.DataFrame({'A': ['a', 'b', 'c'], 'B': ['x', 'y', 'z']}, columns=['A', 'B'])
+        >>> new_column = ps.Series(['d', 'e'], name='B', index=[0, 2])
+        >>> df.update(new_column)
+        >>> df.sort_index()
+           A  B
+        0  a  d
+        1  b  y
+        2  c  e
+
+        If `other` contains None the corresponding values are not updated in the original dataframe.
+
+        >>> df = ps.DataFrame({'A': [1, 2, 3], 'B': [400, 500, 600]}, columns=['A', 'B'])
+        >>> new_df = ps.DataFrame({'B': [4, None, 6]}, columns=['B'])
+        >>> df.update(new_df)
+        >>> df.sort_index()
+           A      B
+        0  1    4.0
+        1  2  500.0
+        2  3    6.0
+        """
+        if join != "left":
+            raise NotImplementedError("Only left join is supported")
+
+        if isinstance(other, ps.Series):
+            other = other.to_frame()
+
+        update_columns = list(
+            set(self._internal.column_labels).intersection(set(other._internal.column_labels))
+        )
+        update_sdf = self.join(
+            other[update_columns], rsuffix="_new"
+        )._internal.resolved_copy.spark_frame
+
+        data_dtypes = self._internal.data_dtypes.copy()
+        for column_labels in update_columns:
+            column_name = self._internal.spark_column_name_for(column_labels)
+            old_col = scol_for(update_sdf, column_name)
+            new_col = scol_for(
+                update_sdf, other._internal.spark_column_name_for(column_labels) + "_new"
+            )
+            if overwrite:
+                update_sdf = update_sdf.withColumn(
+                    column_name, F.when(new_col.isNull(), old_col).otherwise(new_col)
+                )
+            else:
+                update_sdf = update_sdf.withColumn(
+                    column_name, F.when(old_col.isNull(), new_col).otherwise(old_col)
+                )
+            data_dtypes[self._internal.column_labels.index(column_labels)] = None  # TODO: dtype?
+        sdf = update_sdf.select(
+            [scol_for(update_sdf, col) for col in self._internal.spark_column_names]
+            + list(HIDDEN_COLUMNS)
+        )
+        internal = self._internal.with_new_sdf(sdf, data_dtypes=data_dtypes)
+        self._update_internal_frame(internal, requires_same_anchor=False)
+
+    def sample(
+        self,
+        n: Optional[int] = None,
+        frac: Optional[float] = None,
+        replace: bool = False,
+        random_state: Optional[int] = None,
+    ) -> "DataFrame":
+        """
+        Return a random sample of items from an axis of object.
+
+        Please call this function using named argument by specifying the ``frac`` argument.
+
+        You can use `random_state` for reproducibility. However, note that different from pandas,
+        specifying a seed in pandas-on-Spark/Spark does not guarantee the sampled rows will
+        be fixed. The result set depends on not only the seed, but also how the data is distributed
+        across machines and to some extent network randomness when shuffle operations are involved.
+        Even in the simplest case, the result set will depend on the system's CPU core count.
+
+        Parameters
+        ----------
+        n : int, optional
+            Number of items to return. This is currently NOT supported. Use frac instead.
+        frac : float, optional
+            Fraction of axis items to return.
+        replace : bool, default False
+            Sample with or without replacement.
+        random_state : int, optional
+            Seed for the random number generator (if int).
+
+        Returns
+        -------
+        Series or DataFrame
+            A new object of same type as caller containing the sampled items.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'num_legs': [2, 4, 8, 0],
+        ...                    'num_wings': [2, 0, 0, 0],
+        ...                    'num_specimen_seen': [10, 2, 1, 8]},
+        ...                   index=['falcon', 'dog', 'spider', 'fish'],
+        ...                   columns=['num_legs', 'num_wings', 'num_specimen_seen'])
+        >>> df  # doctest: +SKIP
+                num_legs  num_wings  num_specimen_seen
+        falcon         2          2                 10
+        dog            4          0                  2
+        spider         8          0                  1
+        fish           0          0                  8
+
+        A random 25% sample of the ``DataFrame``.
+        Note that we use `random_state` to ensure the reproducibility of
+        the examples.
+
+        >>> df.sample(frac=0.25, random_state=1)  # doctest: +SKIP
+                num_legs  num_wings  num_specimen_seen
+        falcon         2          2                 10
+        fish           0          0                  8
+
+        Extract 25% random elements from the ``Series`` ``df['num_legs']``, with replacement,
+        so the same items could appear more than once.
+
+        >>> df['num_legs'].sample(frac=0.4, replace=True, random_state=1)  # doctest: +SKIP
+        falcon    2
+        spider    8
+        spider    8
+        Name: num_legs, dtype: int64
+
+        Specifying the exact number of items to return is not supported at the moment.
+
+        >>> df.sample(n=5)  # doctest: +ELLIPSIS
+        Traceback (most recent call last):
+            ...
+        NotImplementedError: Function sample currently does not support specifying ...
+        """
+        # Note: we don't run any of the doctests because the result can change depending on the
+        # system's core count.
+        if n is not None:
+            raise NotImplementedError(
+                "Function sample currently does not support specifying "
+                "exact number of items to return. Use frac instead."
+            )
+
+        if frac is None:
+            raise ValueError("frac must be specified.")
+
+        sdf = self._internal.resolved_copy.spark_frame.sample(
+            withReplacement=replace, fraction=frac, seed=random_state
+        )
+        return DataFrame(self._internal.with_new_sdf(sdf))
+
+    def astype(self, dtype) -> "DataFrame":
+        """
+        Cast a pandas-on-Spark object to a specified dtype ``dtype``.
+
+        Parameters
+        ----------
+        dtype : data type, or dict of column name -> data type
+            Use a numpy.dtype or Python type to cast entire pandas-on-Spark object to
+            the same type. Alternatively, use {col: dtype, ...}, where col is a
+            column label and dtype is a numpy.dtype or Python type to cast one
+            or more of the DataFrame's columns to column-specific types.
+
+        Returns
+        -------
+        casted : same type as caller
+
+        See Also
+        --------
+        to_datetime : Convert argument to datetime.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'a': [1, 2, 3], 'b': [1, 2, 3]}, dtype='int64')
+        >>> df
+           a  b
+        0  1  1
+        1  2  2
+        2  3  3
+
+        Convert to float type:
+
+        >>> df.astype('float')
+             a    b
+        0  1.0  1.0
+        1  2.0  2.0
+        2  3.0  3.0
+
+        Convert to int64 type back:
+
+        >>> df.astype('int64')
+           a  b
+        0  1  1
+        1  2  2
+        2  3  3
+
+        Convert column a to float type:
+
+        >>> df.astype({'a': float})
+             a  b
+        0  1.0  1
+        1  2.0  2
+        2  3.0  3
+
+        """
+        applied = []
+        if is_dict_like(dtype):
+            for col_name in dtype.keys():
+                if col_name not in self.columns:
+                    raise KeyError(
+                        "Only a column name can be used for the "
+                        "key in a dtype mappings argument."
+                    )
+            for col_name, col in self.items():
+                if col_name in dtype:
+                    applied.append(col.astype(dtype=dtype[col_name]))
+                else:
+                    applied.append(col)
+        else:
+            for col_name, col in self.items():
+                applied.append(col.astype(dtype=dtype))
+        return DataFrame(self._internal.with_new_columns(applied))
+
+    def add_prefix(self, prefix) -> "DataFrame":
+        """
+        Prefix labels with string `prefix`.
+
+        For Series, the row labels are prefixed.
+        For DataFrame, the column labels are prefixed.
+
+        Parameters
+        ----------
+        prefix : str
+           The string to add before each label.
+
+        Returns
+        -------
+        DataFrame
+           New DataFrame with updated labels.
+
+        See Also
+        --------
+        Series.add_prefix: Prefix row labels with string `prefix`.
+        Series.add_suffix: Suffix row labels with string `suffix`.
+        DataFrame.add_suffix: Suffix column labels with string `suffix`.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'A': [1, 2, 3, 4], 'B': [3, 4, 5, 6]}, columns=['A', 'B'])
+        >>> df
+           A  B
+        0  1  3
+        1  2  4
+        2  3  5
+        3  4  6
+
+        >>> df.add_prefix('col_')
+           col_A  col_B
+        0      1      3
+        1      2      4
+        2      3      5
+        3      4      6
+        """
+        assert isinstance(prefix, str)
+        return self._apply_series_op(
+            lambda kser: kser.rename(tuple([prefix + i for i in kser._column_label]))
+        )
+
+    def add_suffix(self, suffix) -> "DataFrame":
+        """
+        Suffix labels with string `suffix`.
+
+        For Series, the row labels are suffixed.
+        For DataFrame, the column labels are suffixed.
+
+        Parameters
+        ----------
+        suffix : str
+           The string to add before each label.
+
+        Returns
+        -------
+        DataFrame
+           New DataFrame with updated labels.
+
+        See Also
+        --------
+        Series.add_prefix: Prefix row labels with string `prefix`.
+        Series.add_suffix: Suffix row labels with string `suffix`.
+        DataFrame.add_prefix: Prefix column labels with string `prefix`.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'A': [1, 2, 3, 4], 'B': [3, 4, 5, 6]}, columns=['A', 'B'])
+        >>> df
+           A  B
+        0  1  3
+        1  2  4
+        2  3  5
+        3  4  6
+
+        >>> df.add_suffix('_col')
+           A_col  B_col
+        0      1      3
+        1      2      4
+        2      3      5
+        3      4      6
+        """
+        assert isinstance(suffix, str)
+        return self._apply_series_op(
+            lambda kser: kser.rename(tuple([i + suffix for i in kser._column_label]))
+        )
+
+    # TODO: include, and exclude should be implemented.
+    def describe(self, percentiles: Optional[List[float]] = None) -> "DataFrame":
+        """
+        Generate descriptive statistics that summarize the central tendency,
+        dispersion and shape of a dataset's distribution, excluding
+        ``NaN`` values.
+
+        Analyzes both numeric and object series, as well
+        as ``DataFrame`` column sets of mixed data types. The output
+        will vary depending on what is provided. Refer to the notes
+        below for more detail.
+
+        Parameters
+        ----------
+        percentiles : list of ``float`` in range [0.0, 1.0], default [0.25, 0.5, 0.75]
+            A list of percentiles to be computed.
+
+        Returns
+        -------
+        DataFrame
+            Summary statistics of the Dataframe provided.
+
+        See Also
+        --------
+        DataFrame.count: Count number of non-NA/null observations.
+        DataFrame.max: Maximum of the values in the object.
+        DataFrame.min: Minimum of the values in the object.
+        DataFrame.mean: Mean of the values.
+        DataFrame.std: Standard deviation of the observations.
+
+        Notes
+        -----
+        For numeric data, the result's index will include ``count``,
+        ``mean``, ``std``, ``min``, ``25%``, ``50%``, ``75%``, ``max``.
+
+        Currently only numeric data is supported.
+
+        Examples
+        --------
+        Describing a numeric ``Series``.
+
+        >>> s = ps.Series([1, 2, 3])
+        >>> s.describe()
+        count    3.0
+        mean     2.0
+        std      1.0
+        min      1.0
+        25%      1.0
+        50%      2.0
+        75%      3.0
+        max      3.0
+        dtype: float64
+
+        Describing a ``DataFrame``. Only numeric fields are returned.
+
+        >>> df = ps.DataFrame({'numeric1': [1, 2, 3],
+        ...                    'numeric2': [4.0, 5.0, 6.0],
+        ...                    'object': ['a', 'b', 'c']
+        ...                   },
+        ...                   columns=['numeric1', 'numeric2', 'object'])
+        >>> df.describe()
+               numeric1  numeric2
+        count       3.0       3.0
+        mean        2.0       5.0
+        std         1.0       1.0
+        min         1.0       4.0
+        25%         1.0       4.0
+        50%         2.0       5.0
+        75%         3.0       6.0
+        max         3.0       6.0
+
+        For multi-index columns:
+
+        >>> df.columns = [('num', 'a'), ('num', 'b'), ('obj', 'c')]
+        >>> df.describe()  # doctest: +NORMALIZE_WHITESPACE
+               num
+                 a    b
+        count  3.0  3.0
+        mean   2.0  5.0
+        std    1.0  1.0
+        min    1.0  4.0
+        25%    1.0  4.0
+        50%    2.0  5.0
+        75%    3.0  6.0
+        max    3.0  6.0
+
+        >>> df[('num', 'b')].describe()
+        count    3.0
+        mean     5.0
+        std      1.0
+        min      4.0
+        25%      4.0
+        50%      5.0
+        75%      6.0
+        max      6.0
+        Name: (num, b), dtype: float64
+
+        Describing a ``DataFrame`` and selecting custom percentiles.
+
+        >>> df = ps.DataFrame({'numeric1': [1, 2, 3],
+        ...                    'numeric2': [4.0, 5.0, 6.0]
+        ...                   },
+        ...                   columns=['numeric1', 'numeric2'])
+        >>> df.describe(percentiles = [0.85, 0.15])
+               numeric1  numeric2
+        count       3.0       3.0
+        mean        2.0       5.0
+        std         1.0       1.0
+        min         1.0       4.0
+        15%         1.0       4.0
+        50%         2.0       5.0
+        85%         3.0       6.0
+        max         3.0       6.0
+
+        Describing a column from a ``DataFrame`` by accessing it as
+        an attribute.
+
+        >>> df.numeric1.describe()
+        count    3.0
+        mean     2.0
+        std      1.0
+        min      1.0
+        25%      1.0
+        50%      2.0
+        75%      3.0
+        max      3.0
+        Name: numeric1, dtype: float64
+
+        Describing a column from a ``DataFrame`` by accessing it as
+        an attribute and selecting custom percentiles.
+
+        >>> df.numeric1.describe(percentiles = [0.85, 0.15])
+        count    3.0
+        mean     2.0
+        std      1.0
+        min      1.0
+        15%      1.0
+        50%      2.0
+        85%      3.0
+        max      3.0
+        Name: numeric1, dtype: float64
+        """
+        exprs = []
+        column_labels = []
+        for label in self._internal.column_labels:
+            scol = self._internal.spark_column_for(label)
+            spark_type = self._internal.spark_type_for(label)
+            if isinstance(spark_type, DoubleType) or isinstance(spark_type, FloatType):
+                exprs.append(
+                    F.nanvl(scol, F.lit(None)).alias(self._internal.spark_column_name_for(label))
+                )
+                column_labels.append(label)
+            elif isinstance(spark_type, NumericType):
+                exprs.append(scol)
+                column_labels.append(label)
+
+        if len(exprs) == 0:
+            raise ValueError("Cannot describe a DataFrame without columns")
+
+        if percentiles is not None:
+            if any((p < 0.0) or (p > 1.0) for p in percentiles):
+                raise ValueError("Percentiles should all be in the interval [0, 1]")
+            # appending 50% if not in percentiles already
+            percentiles = (percentiles + [0.5]) if 0.5 not in percentiles else percentiles
+        else:
+            percentiles = [0.25, 0.5, 0.75]
+
+        formatted_perc = ["{:.0%}".format(p) for p in sorted(percentiles)]
+        stats = ["count", "mean", "stddev", "min", *formatted_perc, "max"]
+
+        sdf = self._internal.spark_frame.select(*exprs).summary(stats)
+        sdf = sdf.replace("stddev", "std", subset="summary")
+
+        internal = InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=[scol_for(sdf, "summary")],
+            column_labels=column_labels,
+            data_spark_columns=[
+                scol_for(sdf, self._internal.spark_column_name_for(label))
+                for label in column_labels
+            ],
+        )
+        return DataFrame(internal).astype("float64")
+
+    def drop_duplicates(self, subset=None, keep="first", inplace=False) -> Optional["DataFrame"]:
+        """
+        Return DataFrame with duplicate rows removed, optionally only
+        considering certain columns.
+
+        Parameters
+        ----------
+        subset : column label or sequence of labels, optional
+            Only consider certain columns for identifying duplicates, by
+            default use all of the columns.
+        keep : {'first', 'last', False}, default 'first'
+            Determines which duplicates (if any) to keep.
+            - ``first`` : Drop duplicates except for the first occurrence.
+            - ``last`` : Drop duplicates except for the last occurrence.
+            - False : Drop all duplicates.
+        inplace : boolean, default False
+            Whether to drop duplicates in place or to return a copy.
+
+        Returns
+        -------
+        DataFrame
+            DataFrame with duplicates removed or None if ``inplace=True``.
+
+        >>> df = ps.DataFrame(
+        ...     {'a': [1, 2, 2, 2, 3], 'b': ['a', 'a', 'a', 'c', 'd']}, columns = ['a', 'b'])
+        >>> df
+           a  b
+        0  1  a
+        1  2  a
+        2  2  a
+        3  2  c
+        4  3  d
+
+        >>> df.drop_duplicates().sort_index()
+           a  b
+        0  1  a
+        1  2  a
+        3  2  c
+        4  3  d
+
+        >>> df.drop_duplicates('a').sort_index()
+           a  b
+        0  1  a
+        1  2  a
+        4  3  d
+
+        >>> df.drop_duplicates(['a', 'b']).sort_index()
+           a  b
+        0  1  a
+        1  2  a
+        3  2  c
+        4  3  d
+
+        >>> df.drop_duplicates(keep='last').sort_index()
+           a  b
+        0  1  a
+        2  2  a
+        3  2  c
+        4  3  d
+
+        >>> df.drop_duplicates(keep=False).sort_index()
+           a  b
+        0  1  a
+        3  2  c
+        4  3  d
+        """
+        inplace = validate_bool_kwarg(inplace, "inplace")
+
+        sdf, column = self._mark_duplicates(subset, keep)
+
+        sdf = sdf.where(~scol_for(sdf, column)).drop(column)
+        internal = self._internal.with_new_sdf(sdf)
+        if inplace:
+            self._update_internal_frame(internal)
+            return None
+        else:
+            return DataFrame(internal)
+
+    def reindex(
+        self,
+        labels: Optional[Any] = None,
+        index: Optional[Any] = None,
+        columns: Optional[Any] = None,
+        axis: Optional[Union[int, str]] = None,
+        copy: Optional[bool] = True,
+        fill_value: Optional[Any] = None,
+    ) -> "DataFrame":
+        """
+        Conform DataFrame to new index with optional filling logic, placing
+        NA/NaN in locations having no value in the previous index. A new object
+        is produced unless the new index is equivalent to the current one and
+        ``copy=False``.
+
+        Parameters
+        ----------
+        labels: array-like, optional
+            New labels / index to conform the axis specified by ‘axis’ to.
+        index, columns: array-like, optional
+            New labels / index to conform to, should be specified using keywords.
+            Preferably an Index object to avoid duplicating data
+        axis: int or str, optional
+            Axis to target. Can be either the axis name (‘index’, ‘columns’) or
+            number (0, 1).
+        copy : bool, default True
+            Return a new object, even if the passed indexes are the same.
+        fill_value : scalar, default np.NaN
+            Value to use for missing values. Defaults to NaN, but can be any
+            "compatible" value.
+
+        Returns
+        -------
+        DataFrame with changed index.
+
+        See Also
+        --------
+        DataFrame.set_index : Set row labels.
+        DataFrame.reset_index : Remove row labels or move them to new columns.
+
+        Examples
+        --------
+
+        ``DataFrame.reindex`` supports two calling conventions
+
+        * ``(index=index_labels, columns=column_labels, ...)``
+        * ``(labels, axis={'index', 'columns'}, ...)``
+
+        We *highly* recommend using keyword arguments to clarify your
+        intent.
+
+        Create a dataframe with some fictional data.
+
+        >>> index = ['Firefox', 'Chrome', 'Safari', 'IE10', 'Konqueror']
+        >>> df = ps.DataFrame({
+        ...      'http_status': [200, 200, 404, 404, 301],
+        ...      'response_time': [0.04, 0.02, 0.07, 0.08, 1.0]},
+        ...       index=index,
+        ...       columns=['http_status', 'response_time'])
+        >>> df
+                   http_status  response_time
+        Firefox            200           0.04
+        Chrome             200           0.02
+        Safari             404           0.07
+        IE10               404           0.08
+        Konqueror          301           1.00
+
+        Create a new index and reindex the dataframe. By default
+        values in the new index that do not have corresponding
+        records in the dataframe are assigned ``NaN``.
+
+        >>> new_index= ['Safari', 'Iceweasel', 'Comodo Dragon', 'IE10',
+        ...             'Chrome']
+        >>> df.reindex(new_index).sort_index()
+                       http_status  response_time
+        Chrome               200.0           0.02
+        Comodo Dragon          NaN            NaN
+        IE10                 404.0           0.08
+        Iceweasel              NaN            NaN
+        Safari               404.0           0.07
+
+        We can fill in the missing values by passing a value to
+        the keyword ``fill_value``.
+
+        >>> df.reindex(new_index, fill_value=0, copy=False).sort_index()
+                       http_status  response_time
+        Chrome                 200           0.02
+        Comodo Dragon            0           0.00
+        IE10                   404           0.08
+        Iceweasel                0           0.00
+        Safari                 404           0.07
+
+        We can also reindex the columns.
+
+        >>> df.reindex(columns=['http_status', 'user_agent']).sort_index()
+                   http_status  user_agent
+        Chrome             200         NaN
+        Firefox            200         NaN
+        IE10               404         NaN
+        Konqueror          301         NaN
+        Safari             404         NaN
+
+        Or we can use "axis-style" keyword arguments
+
+        >>> df.reindex(['http_status', 'user_agent'], axis="columns").sort_index()
+                   http_status  user_agent
+        Chrome             200         NaN
+        Firefox            200         NaN
+        IE10               404         NaN
+        Konqueror          301         NaN
+        Safari             404         NaN
+
+        To further illustrate the filling functionality in
+        ``reindex``, we will create a dataframe with a
+        monotonically increasing index (for example, a sequence
+        of dates).
+
+        >>> date_index = pd.date_range('1/1/2010', periods=6, freq='D')
+        >>> df2 = ps.DataFrame({"prices": [100, 101, np.nan, 100, 89, 88]},
+        ...                    index=date_index)
+        >>> df2.sort_index()
+                    prices
+        2010-01-01   100.0
+        2010-01-02   101.0
+        2010-01-03     NaN
+        2010-01-04   100.0
+        2010-01-05    89.0
+        2010-01-06    88.0
+
+        Suppose we decide to expand the dataframe to cover a wider
+        date range.
+
+        >>> date_index2 = pd.date_range('12/29/2009', periods=10, freq='D')
+        >>> df2.reindex(date_index2).sort_index()
+                    prices
+        2009-12-29     NaN
+        2009-12-30     NaN
+        2009-12-31     NaN
+        2010-01-01   100.0
+        2010-01-02   101.0
+        2010-01-03     NaN
+        2010-01-04   100.0
+        2010-01-05    89.0
+        2010-01-06    88.0
+        2010-01-07     NaN
+        """
+        if axis is not None and (index is not None or columns is not None):
+            raise TypeError("Cannot specify both 'axis' and any of 'index' or 'columns'.")
+
+        if labels is not None:
+            axis = validate_axis(axis)
+            if axis == 0:
+                index = labels
+            elif axis == 1:
+                columns = labels
+            else:
+                raise ValueError(
+                    "No axis named %s for object type %s." % (axis, type(axis).__name__)
+                )
+
+        if index is not None and not is_list_like(index):
+            raise TypeError(
+                "Index must be called with a collection of some kind, "
+                "%s was passed" % type(index)
+            )
+
+        if columns is not None and not is_list_like(columns):
+            raise TypeError(
+                "Columns must be called with a collection of some kind, "
+                "%s was passed" % type(columns)
+            )
+
+        df = self
+
+        if index is not None:
+            df = df._reindex_index(index, fill_value)
+
+        if columns is not None:
+            df = df._reindex_columns(columns, fill_value)
+
+        # Copy
+        if copy and df is self:
+            return df.copy()
+        else:
+            return df
+
+    def _reindex_index(self, index, fill_value):
+        # When axis is index, we can mimic pandas' by a right outer join.
+        nlevels = self._internal.index_level
+        assert nlevels <= 1 or (
+            isinstance(index, ps.MultiIndex) and nlevels == index.nlevels
+        ), "MultiIndex DataFrame can only be reindexed with a similar pandas-on-Spark MultiIndex."
+
+        index_columns = self._internal.index_spark_column_names
+        frame = self._internal.resolved_copy.spark_frame.drop(NATURAL_ORDER_COLUMN_NAME)
+
+        if isinstance(index, ps.Index):
+            if nlevels != index.nlevels:
+                return DataFrame(index._internal.with_new_columns([])).reindex(
+                    columns=self.columns, fill_value=fill_value
+                )
+
+            index_names = index._internal.index_names
+            scols = index._internal.index_spark_columns
+            labels = index._internal.spark_frame.select(
+                [scol.alias(index_column) for scol, index_column in zip(scols, index_columns)]
+            )
+        else:
+            kser = ps.Series(list(index))
+            labels = kser._internal.spark_frame.select(kser.spark.column.alias(index_columns[0]))
+            index_names = self._internal.index_names
+
+        if fill_value is not None:
+            frame_index_columns = [
+                verify_temp_column_name(frame, "__frame_index_column_{}__".format(i))
+                for i in range(nlevels)
+            ]
+            index_scols = [
+                scol_for(frame, index_col).alias(frame_index_col)
+                for index_col, frame_index_col in zip(index_columns, frame_index_columns)
+            ]
+            scols = self._internal.resolved_copy.data_spark_columns
+            frame = frame.select(index_scols + scols)
+
+            temp_fill_value = verify_temp_column_name(frame, "__fill_value__")
+            labels = labels.withColumn(temp_fill_value, F.lit(fill_value))
+
+            frame_index_scols = [scol_for(frame, col) for col in frame_index_columns]
+            labels_index_scols = [scol_for(labels, col) for col in index_columns]
+
+            joined_df = frame.join(
+                labels,
+                on=[fcol == lcol for fcol, lcol in zip(frame_index_scols, labels_index_scols)],
+                how="right",
+            )
+
+            joined_df = joined_df.select(
+                *labels_index_scols,
+                *[
+                    F.when(
+                        reduce(
+                            lambda c1, c2: c1 & c2,
+                            [
+                                fcol.isNull() & lcol.isNotNull()
+                                for fcol, lcol in zip(frame_index_scols, labels_index_scols)
+                            ],
+                        ),
+                        scol_for(joined_df, temp_fill_value),
+                    )
+                    .otherwise(scol_for(joined_df, col))
+                    .alias(col)
+                    for col in self._internal.data_spark_column_names
+                ],
+            )
+        else:
+            joined_df = frame.join(labels, on=index_columns, how="right")
+
+        sdf = joined_df.drop(NATURAL_ORDER_COLUMN_NAME)
+        internal = self._internal.copy(
+            spark_frame=sdf,
+            index_spark_columns=[
+                scol_for(sdf, col) for col in self._internal.index_spark_column_names
+            ],
+            index_names=index_names,
+            index_dtypes=None,  # TODO: dtypes?
+            data_spark_columns=[
+                scol_for(sdf, col) for col in self._internal.data_spark_column_names
+            ],
+        )
+        return DataFrame(internal)
+
+    def _reindex_columns(self, columns, fill_value):
+        level = self._internal.column_labels_level
+        if level > 1:
+            label_columns = list(columns)
+            for col in label_columns:
+                if not isinstance(col, tuple):
+                    raise TypeError("Expected tuple, got {}".format(type(col).__name__))
+        else:
+            label_columns = [(col,) for col in columns]
+        for col in label_columns:
+            if len(col) != level:
+                raise ValueError(
+                    "shape (1,{}) doesn't match the shape (1,{})".format(len(col), level)
+                )
+        fill_value = np.nan if fill_value is None else fill_value
+        scols_or_ksers, labels = [], []
+        for label in label_columns:
+            if label in self._internal.column_labels:
+                scols_or_ksers.append(self._kser_for(label))
+            else:
+                scols_or_ksers.append(F.lit(fill_value).alias(name_like_string(label)))
+            labels.append(label)
+
+        if isinstance(columns, pd.Index):
+            column_label_names = [
+                name if is_name_like_tuple(name) else (name,) for name in columns.names
+            ]
+            internal = self._internal.with_new_columns(
+                scols_or_ksers, column_labels=labels, column_label_names=column_label_names
+            )
+        else:
+            internal = self._internal.with_new_columns(scols_or_ksers, column_labels=labels)
+
+        return DataFrame(internal)
+
+    def reindex_like(self, other: "DataFrame", copy: bool = True) -> "DataFrame":
+        """
+        Return a DataFrame with matching indices as other object.
+
+        Conform the object to the same index on all axes. Places NA/NaN in locations
+        having no value in the previous index. A new object is produced unless the
+        new index is equivalent to the current one and copy=False.
+
+        Parameters
+        ----------
+        other : DataFrame
+            Its row and column indices are used to define the new indices
+            of this object.
+        copy : bool, default True
+            Return a new object, even if the passed indexes are the same.
+
+        Returns
+        -------
+        DataFrame
+            DataFrame with changed indices on each axis.
+
+        See Also
+        --------
+        DataFrame.set_index : Set row labels.
+        DataFrame.reset_index : Remove row labels or move them to new columns.
+        DataFrame.reindex : Change to new indices or expand indices.
+
+        Notes
+        -----
+        Same as calling
+        ``.reindex(index=other.index, columns=other.columns,...)``.
+
+        Examples
+        --------
+
+        >>> df1 = ps.DataFrame([[24.3, 75.7, 'high'],
+        ...                     [31, 87.8, 'high'],
+        ...                     [22, 71.6, 'medium'],
+        ...                     [35, 95, 'medium']],
+        ...                    columns=['temp_celsius', 'temp_fahrenheit',
+        ...                             'windspeed'],
+        ...                    index=pd.date_range(start='2014-02-12',
+        ...                                        end='2014-02-15', freq='D'))
+        >>> df1
+                    temp_celsius  temp_fahrenheit windspeed
+        2014-02-12          24.3             75.7      high
+        2014-02-13          31.0             87.8      high
+        2014-02-14          22.0             71.6    medium
+        2014-02-15          35.0             95.0    medium
+
+        >>> df2 = ps.DataFrame([[28, 'low'],
+        ...                     [30, 'low'],
+        ...                     [35.1, 'medium']],
+        ...                    columns=['temp_celsius', 'windspeed'],
+        ...                    index=pd.DatetimeIndex(['2014-02-12', '2014-02-13',
+        ...                                            '2014-02-15']))
+        >>> df2
+                    temp_celsius windspeed
+        2014-02-12          28.0       low
+        2014-02-13          30.0       low
+        2014-02-15          35.1    medium
+
+        >>> df2.reindex_like(df1).sort_index() # doctest: +NORMALIZE_WHITESPACE
+                    temp_celsius  temp_fahrenheit windspeed
+        2014-02-12          28.0              NaN       low
+        2014-02-13          30.0              NaN       low
+        2014-02-14           NaN              NaN       None
+        2014-02-15          35.1              NaN    medium
+        """
+
+        if isinstance(other, DataFrame):
+            return self.reindex(index=other.index, columns=other.columns, copy=copy)
+        else:
+            raise TypeError("other must be a pandas-on-Spark DataFrame")
+
+    def melt(self, id_vars=None, value_vars=None, var_name=None, value_name="value") -> "DataFrame":
+        """
+        Unpivot a DataFrame from wide format to long format, optionally
+        leaving identifier variables set.
+
+        This function is useful to massage a DataFrame into a format where one
+        or more columns are identifier variables (`id_vars`), while all other
+        columns, considered measured variables (`value_vars`), are "unpivoted" to
+        the row axis, leaving just two non-identifier columns, 'variable' and
+        'value'.
+
+        Parameters
+        ----------
+        frame : DataFrame
+        id_vars : tuple, list, or ndarray, optional
+            Column(s) to use as identifier variables.
+        value_vars : tuple, list, or ndarray, optional
+            Column(s) to unpivot. If not specified, uses all columns that
+            are not set as `id_vars`.
+        var_name : scalar, default 'variable'
+            Name to use for the 'variable' column. If None it uses `frame.columns.name` or
+            ‘variable’.
+        value_name : scalar, default 'value'
+            Name to use for the 'value' column.
+
+        Returns
+        -------
+        DataFrame
+            Unpivoted DataFrame.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'A': {0: 'a', 1: 'b', 2: 'c'},
+        ...                    'B': {0: 1, 1: 3, 2: 5},
+        ...                    'C': {0: 2, 1: 4, 2: 6}},
+        ...                   columns=['A', 'B', 'C'])
+        >>> df
+           A  B  C
+        0  a  1  2
+        1  b  3  4
+        2  c  5  6
+
+        >>> ps.melt(df)
+          variable value
+        0        A     a
+        1        B     1
+        2        C     2
+        3        A     b
+        4        B     3
+        5        C     4
+        6        A     c
+        7        B     5
+        8        C     6
+
+        >>> df.melt(id_vars='A')
+           A variable  value
+        0  a        B      1
+        1  a        C      2
+        2  b        B      3
+        3  b        C      4
+        4  c        B      5
+        5  c        C      6
+
+        >>> df.melt(value_vars='A')
+          variable value
+        0        A     a
+        1        A     b
+        2        A     c
+
+        >>> ps.melt(df, id_vars=['A', 'B'])
+           A  B variable  value
+        0  a  1        C      2
+        1  b  3        C      4
+        2  c  5        C      6
+
+        >>> df.melt(id_vars=['A'], value_vars=['C'])
+           A variable  value
+        0  a        C      2
+        1  b        C      4
+        2  c        C      6
+
+        The names of 'variable' and 'value' columns can be customized:
+
+        >>> ps.melt(df, id_vars=['A'], value_vars=['B'],
+        ...         var_name='myVarname', value_name='myValname')
+           A myVarname  myValname
+        0  a         B          1
+        1  b         B          3
+        2  c         B          5
+        """
+        column_labels = self._internal.column_labels
+
+        if id_vars is None:
+            id_vars = []
+        else:
+            if isinstance(id_vars, tuple):
+                if self._internal.column_labels_level == 1:
+                    id_vars = [idv if is_name_like_tuple(idv) else (idv,) for idv in id_vars]
+                else:
+                    raise ValueError(
+                        "id_vars must be a list of tuples" " when columns are a MultiIndex"
+                    )
+            elif is_name_like_value(id_vars):
+                id_vars = [(id_vars,)]
+            else:
+                id_vars = [idv if is_name_like_tuple(idv) else (idv,) for idv in id_vars]
+
+            non_existence_col = [idv for idv in id_vars if idv not in column_labels]
+            if len(non_existence_col) != 0:
+                raveled_column_labels = np.ravel(column_labels)
+                missing = [
+                    nec for nec in np.ravel(non_existence_col) if nec not in raveled_column_labels
+                ]
+                if len(missing) != 0:
+                    raise KeyError(
+                        "The following 'id_vars' are not present"
+                        " in the DataFrame: {}".format(missing)
+                    )
+                else:
+                    raise KeyError(
+                        "None of {} are in the {}".format(non_existence_col, column_labels)
+                    )
+
+        if value_vars is None:
+            value_vars = []
+        else:
+            if isinstance(value_vars, tuple):
+                if self._internal.column_labels_level == 1:
+                    value_vars = [
+                        valv if is_name_like_tuple(valv) else (valv,) for valv in value_vars
+                    ]
+                else:
+                    raise ValueError(
+                        "value_vars must be a list of tuples" " when columns are a MultiIndex"
+                    )
+            elif is_name_like_value(value_vars):
+                value_vars = [(value_vars,)]
+            else:
+                value_vars = [valv if is_name_like_tuple(valv) else (valv,) for valv in value_vars]
+
+            non_existence_col = [valv for valv in value_vars if valv not in column_labels]
+            if len(non_existence_col) != 0:
+                raveled_column_labels = np.ravel(column_labels)
+                missing = [
+                    nec for nec in np.ravel(non_existence_col) if nec not in raveled_column_labels
+                ]
+                if len(missing) != 0:
+                    raise KeyError(
+                        "The following 'value_vars' are not present"
+                        " in the DataFrame: {}".format(missing)
+                    )
+                else:
+                    raise KeyError(
+                        "None of {} are in the {}".format(non_existence_col, column_labels)
+                    )
+
+        if len(value_vars) == 0:
+            value_vars = column_labels
+
+        column_labels = [label for label in column_labels if label not in id_vars]
+
+        sdf = self._internal.spark_frame
+
+        if var_name is None:
+            if (
+                self._internal.column_labels_level == 1
+                and self._internal.column_label_names[0] is None
+            ):
+                var_name = ["variable"]
+            else:
+                var_name = [
+                    name_like_string(name) if name is not None else "variable_{}".format(i)
+                    for i, name in enumerate(self._internal.column_label_names)
+                ]
+        elif isinstance(var_name, str):
+            var_name = [var_name]
+
+        pairs = F.explode(
+            F.array(
+                *[
+                    F.struct(
+                        *(
+                            [F.lit(c).alias(name) for c, name in zip(label, var_name)]
+                            + [self._internal.spark_column_for(label).alias(value_name)]
+                        )
+                    )
+                    for label in column_labels
+                    if label in value_vars
+                ]
+            )
+        )
+
+        columns = (
+            [
+                self._internal.spark_column_for(label).alias(name_like_string(label))
+                for label in id_vars
+            ]
+            + [F.col("pairs.`%s`" % name) for name in var_name]
+            + [F.col("pairs.`%s`" % value_name)]
+        )
+        exploded_df = sdf.withColumn("pairs", pairs).select(columns)
+
+        return DataFrame(
+            InternalFrame(
+                spark_frame=exploded_df,
+                index_spark_columns=None,
+                column_labels=(
+                    [label if len(label) == 1 else (name_like_string(label),) for label in id_vars]
+                    + [(name,) for name in var_name]
+                    + [(value_name,)]
+                ),
+            )
+        )
+
+    def stack(self) -> Union["DataFrame", "Series"]:
+        """
+        Stack the prescribed level(s) from columns to index.
+
+        Return a reshaped DataFrame or Series having a multi-level
+        index with one or more new inner-most levels compared to the current
+        DataFrame. The new inner-most levels are created by pivoting the
+        columns of the current dataframe:
+
+          - if the columns have a single level, the output is a Series;
+          - if the columns have multiple levels, the new index
+            level(s) is (are) taken from the prescribed level(s) and
+            the output is a DataFrame.
+
+        The new index levels are sorted.
+
+        Returns
+        -------
+        DataFrame or Series
+            Stacked dataframe or series.
+
+        See Also
+        --------
+        DataFrame.unstack : Unstack prescribed level(s) from index axis
+            onto column axis.
+        DataFrame.pivot : Reshape dataframe from long format to wide
+            format.
+        DataFrame.pivot_table : Create a spreadsheet-style pivot table
+            as a DataFrame.
+
+        Notes
+        -----
+        The function is named by analogy with a collection of books
+        being reorganized from being side by side on a horizontal
+        position (the columns of the dataframe) to being stacked
+        vertically on top of each other (in the index of the
+        dataframe).
+
+        Examples
+        --------
+        **Single level columns**
+
+        >>> df_single_level_cols = ps.DataFrame([[0, 1], [2, 3]],
+        ...                                     index=['cat', 'dog'],
+        ...                                     columns=['weight', 'height'])
+
+        Stacking a dataframe with a single level column axis returns a Series:
+
+        >>> df_single_level_cols
+             weight  height
+        cat       0       1
+        dog       2       3
+        >>> df_single_level_cols.stack().sort_index()
+        cat  height    1
+             weight    0
+        dog  height    3
+             weight    2
+        dtype: int64
+
+        **Multi level columns: simple case**
+
+        >>> multicol1 = pd.MultiIndex.from_tuples([('weight', 'kg'),
+        ...                                        ('weight', 'pounds')])
+        >>> df_multi_level_cols1 = ps.DataFrame([[1, 2], [2, 4]],
+        ...                                     index=['cat', 'dog'],
+        ...                                     columns=multicol1)
+
+        Stacking a dataframe with a multi-level column axis:
+
+        >>> df_multi_level_cols1  # doctest: +NORMALIZE_WHITESPACE
+            weight
+                kg pounds
+        cat      1      2
+        dog      2      4
+        >>> df_multi_level_cols1.stack().sort_index()
+                    weight
+        cat kg           1
+            pounds       2
+        dog kg           2
+            pounds       4
+
+        **Missing values**
+
+        >>> multicol2 = pd.MultiIndex.from_tuples([('weight', 'kg'),
+        ...                                        ('height', 'm')])
+        >>> df_multi_level_cols2 = ps.DataFrame([[1.0, 2.0], [3.0, 4.0]],
+        ...                                     index=['cat', 'dog'],
+        ...                                     columns=multicol2)
+
+        It is common to have missing values when stacking a dataframe
+        with multi-level columns, as the stacked dataframe typically
+        has more values than the original dataframe. Missing values
+        are filled with NaNs:
+
+        >>> df_multi_level_cols2
+            weight height
+                kg      m
+        cat    1.0    2.0
+        dog    3.0    4.0
+        >>> df_multi_level_cols2.stack().sort_index()  # doctest: +SKIP
+                height  weight
+        cat kg     NaN     1.0
+            m      2.0     NaN
+        dog kg     NaN     3.0
+            m      4.0     NaN
+        """
+        from pyspark.pandas.series import first_series
+
+        if len(self._internal.column_labels) == 0:
+            return DataFrame(
+                self._internal.copy(
+                    column_label_names=self._internal.column_label_names[:-1]
+                ).with_filter(F.lit(False))
+            )
+
+        column_labels = defaultdict(dict)  # type: Union[defaultdict, OrderedDict]
+        index_values = set()
+        should_returns_series = False
+        for label in self._internal.column_labels:
+            new_label = label[:-1]
+            if len(new_label) == 0:
+                new_label = None
+                should_returns_series = True
+            value = label[-1]
+
+            scol = self._internal.spark_column_for(label)
+            column_labels[new_label][value] = scol
+
+            index_values.add(value)
+
+        column_labels = OrderedDict(sorted(column_labels.items(), key=lambda x: x[0]))
+
+        index_name = self._internal.column_label_names[-1]
+        column_label_names = self._internal.column_label_names[:-1]
+        if len(column_label_names) == 0:
+            column_label_names = [None]
+
+        index_column = SPARK_INDEX_NAME_FORMAT(self._internal.index_level)
+        data_columns = [name_like_string(label) for label in column_labels]
+
+        structs = [
+            F.struct(
+                [F.lit(value).alias(index_column)]
+                + [
+                    (
+                        column_labels[label][value]
+                        if value in column_labels[label]
+                        else F.lit(None)
+                    ).alias(name)
+                    for label, name in zip(column_labels, data_columns)
+                ]
+            ).alias(value)
+            for value in index_values
+        ]
+
+        pairs = F.explode(F.array(structs))
+
+        sdf = self._internal.spark_frame.withColumn("pairs", pairs)
+        sdf = sdf.select(
+            self._internal.index_spark_columns
+            + [sdf["pairs"][index_column].alias(index_column)]
+            + [sdf["pairs"][name].alias(name) for name in data_columns]
+        )
+
+        internal = InternalFrame(  # TODO: dtypes?
+            spark_frame=sdf,
+            index_spark_columns=[
+                scol_for(sdf, col)
+                for col in (self._internal.index_spark_column_names + [index_column])
+            ],
+            index_names=self._internal.index_names + [index_name],
+            index_dtypes=self._internal.index_dtypes + [None],
+            column_labels=list(column_labels),
+            data_spark_columns=[scol_for(sdf, col) for col in data_columns],
+            column_label_names=column_label_names,  # type: ignore
+        )
+        kdf = DataFrame(internal)  # type: "DataFrame"
+
+        if should_returns_series:
+            return first_series(kdf)
+        else:
+            return kdf
+
+    def unstack(self) -> Union["DataFrame", "Series"]:
+        """
+        Pivot the (necessarily hierarchical) index labels.
+
+        Returns a DataFrame having a new level of column labels whose inner-most level
+        consists of the pivoted index labels.
+
+        If the index is not a MultiIndex, the output will be a Series.
+
+        .. note:: If the index is a MultiIndex, the output DataFrame could be very wide, and
+            it could cause a serious performance degradation since Spark partitions it row based.
+
+        Returns
+        -------
+        Series or DataFrame
+
+        See Also
+        --------
+        DataFrame.pivot : Pivot a table based on column values.
+        DataFrame.stack : Pivot a level of the column labels (inverse operation from unstack).
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({"A": {"0": "a", "1": "b", "2": "c"},
+        ...                    "B": {"0": "1", "1": "3", "2": "5"},
+        ...                    "C": {"0": "2", "1": "4", "2": "6"}},
+        ...                   columns=["A", "B", "C"])
+        >>> df
+           A  B  C
+        0  a  1  2
+        1  b  3  4
+        2  c  5  6
+
+        >>> df.unstack().sort_index()
+        A  0    a
+           1    b
+           2    c
+        B  0    1
+           1    3
+           2    5
+        C  0    2
+           1    4
+           2    6
+        dtype: object
+
+        >>> df.columns = pd.MultiIndex.from_tuples([('X', 'A'), ('X', 'B'), ('Y', 'C')])
+        >>> df.unstack().sort_index()
+        X  A  0    a
+              1    b
+              2    c
+           B  0    1
+              1    3
+              2    5
+        Y  C  0    2
+              1    4
+              2    6
+        dtype: object
+
+        For MultiIndex case:
+
+        >>> df = ps.DataFrame({"A": ["a", "b", "c"],
+        ...                    "B": [1, 3, 5],
+        ...                    "C": [2, 4, 6]},
+        ...                   columns=["A", "B", "C"])
+        >>> df = df.set_index('A', append=True)
+        >>> df  # doctest: +NORMALIZE_WHITESPACE
+             B  C
+          A
+        0 a  1  2
+        1 b  3  4
+        2 c  5  6
+        >>> df.unstack().sort_index()  # doctest: +NORMALIZE_WHITESPACE
+             B              C
+        A    a    b    c    a    b    c
+        0  1.0  NaN  NaN  2.0  NaN  NaN
+        1  NaN  3.0  NaN  NaN  4.0  NaN
+        2  NaN  NaN  5.0  NaN  NaN  6.0
+        """
+        from pyspark.pandas.series import first_series
+
+        if self._internal.index_level > 1:
+            # The index after `reset_index()` will never be used, so use "distributed" index
+            # as a dummy to avoid overhead.
+            with option_context("compute.default_index_type", "distributed"):
+                df = self.reset_index()
+            index = df._internal.column_labels[: self._internal.index_level - 1]
+            columns = df.columns[self._internal.index_level - 1]
+            df = df.pivot_table(
+                index=index, columns=columns, values=self._internal.column_labels, aggfunc="first"
+            )
+            internal = df._internal.copy(
+                index_names=self._internal.index_names[:-1],
+                index_dtypes=self._internal.index_dtypes[:-1],
+                column_label_names=(
+                    df._internal.column_label_names[:-1]
+                    + [
+                        None
+                        if self._internal.index_names[-1] is None
+                        else df._internal.column_label_names[-1]
+                    ]
+                ),
+            )
+            return DataFrame(internal)
+
+        # TODO: Codes here are similar with melt. Should we deduplicate?
+        column_labels = self._internal.column_labels
+        ser_name = SPARK_DEFAULT_SERIES_NAME
+        sdf = self._internal.spark_frame
+        new_index_columns = [
+            SPARK_INDEX_NAME_FORMAT(i) for i in range(self._internal.column_labels_level)
+        ]
+
+        new_index_map = list(zip(new_index_columns, self._internal.column_label_names))
+
+        pairs = F.explode(
+            F.array(
+                *[
+                    F.struct(
+                        *(
+                            [F.lit(c).alias(name) for c, name in zip(idx, new_index_columns)]
+                            + [self._internal.spark_column_for(idx).alias(ser_name)]
+                        )
+                    )
+                    for idx in column_labels
+                ]
+            )
+        )
+
+        columns = [
+            F.col("pairs.%s" % name)
+            for name in new_index_columns[: self._internal.column_labels_level]
+        ] + [F.col("pairs.%s" % ser_name)]
+
+        new_index_len = len(new_index_columns)
+        existing_index_columns = []
+        for i, index_name in enumerate(self._internal.index_names):
+            new_index_map.append((SPARK_INDEX_NAME_FORMAT(i + new_index_len), index_name))
+            existing_index_columns.append(
+                self._internal.index_spark_columns[i].alias(
+                    SPARK_INDEX_NAME_FORMAT(i + new_index_len)
+                )
+            )
+
+        exploded_df = sdf.withColumn("pairs", pairs).select(existing_index_columns + columns)
+
+        index_spark_column_names, index_names = zip(*new_index_map)
+        return first_series(
+            DataFrame(
+                InternalFrame(  # TODO: dtypes?
+                    exploded_df,
+                    index_spark_columns=[
+                        scol_for(exploded_df, col) for col in index_spark_column_names
+                    ],
+                    index_names=list(index_names),
+                    column_labels=[None],
+                )
+            )
+        )
+
+    # TODO: axis, skipna, and many arguments should be implemented.
+    def all(self, axis: Union[int, str] = 0) -> "Series":
+        """
+        Return whether all elements are True.
+
+        Returns True unless there is at least one element within a series that is
+        False or equivalent (e.g. zero or empty)
+
+        Parameters
+        ----------
+        axis : {0 or 'index'}, default 0
+            Indicate which axis or axes should be reduced.
+
+            * 0 / 'index' : reduce the index, return a Series whose index is the
+              original column labels.
+
+        Returns
+        -------
+        Series
+
+        Examples
+        --------
+        Create a dataframe from a dictionary.
+
+        >>> df = ps.DataFrame({
+        ...    'col1': [True, True, True],
+        ...    'col2': [True, False, False],
+        ...    'col3': [0, 0, 0],
+        ...    'col4': [1, 2, 3],
+        ...    'col5': [True, True, None],
+        ...    'col6': [True, False, None]},
+        ...    columns=['col1', 'col2', 'col3', 'col4', 'col5', 'col6'])
+
+        Default behaviour checks if column-wise values all return a boolean.
+
+        >>> df.all()
+        col1     True
+        col2    False
+        col3    False
+        col4     True
+        col5     True
+        col6    False
+        dtype: bool
+        """
+        from pyspark.pandas.series import first_series
+
+        axis = validate_axis(axis)
+        if axis != 0:
+            raise NotImplementedError('axis should be either 0 or "index" currently.')
+
+        applied = []
+        column_labels = self._internal.column_labels
+        for label in column_labels:
+            scol = self._internal.spark_column_for(label)
+            all_col = F.min(F.coalesce(scol.cast("boolean"), F.lit(True)))
+            applied.append(F.when(all_col.isNull(), True).otherwise(all_col))
+
+        # TODO: there is a similar logic to transpose in, for instance,
+        #  DataFrame.any, Series.quantile. Maybe we should deduplicate it.
+        value_column = "value"
+        cols = []
+        for label, applied_col in zip(column_labels, applied):
+            cols.append(
+                F.struct(
+                    [F.lit(col).alias(SPARK_INDEX_NAME_FORMAT(i)) for i, col in enumerate(label)]
+                    + [applied_col.alias(value_column)]
+                )
+            )
+
+        sdf = self._internal.spark_frame.select(F.array(*cols).alias("arrays")).select(
+            F.explode(F.col("arrays"))
+        )
+        sdf = sdf.selectExpr("col.*")
+
+        internal = InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=[
+                scol_for(sdf, SPARK_INDEX_NAME_FORMAT(i))
+                for i in range(self._internal.column_labels_level)
+            ],
+            index_names=self._internal.column_label_names,
+            column_labels=[None],
+            data_spark_columns=[scol_for(sdf, value_column)],
+        )
+
+        return first_series(DataFrame(internal))
+
+    # TODO: axis, skipna, and many arguments should be implemented.
+    def any(self, axis: Union[int, str] = 0) -> "Series":
+        """
+        Return whether any element is True.
+
+        Returns False unless there is at least one element within a series that is
+        True or equivalent (e.g. non-zero or non-empty).
+
+        Parameters
+        ----------
+        axis : {0 or 'index'}, default 0
+            Indicate which axis or axes should be reduced.
+
+            * 0 / 'index' : reduce the index, return a Series whose index is the
+              original column labels.
+
+        Returns
+        -------
+        Series
+
+        Examples
+        --------
+        Create a dataframe from a dictionary.
+
+        >>> df = ps.DataFrame({
+        ...    'col1': [False, False, False],
+        ...    'col2': [True, False, False],
+        ...    'col3': [0, 0, 1],
+        ...    'col4': [0, 1, 2],
+        ...    'col5': [False, False, None],
+        ...    'col6': [True, False, None]},
+        ...    columns=['col1', 'col2', 'col3', 'col4', 'col5', 'col6'])
+
+        Default behaviour checks if column-wise values all return a boolean.
+
+        >>> df.any()
+        col1    False
+        col2     True
+        col3     True
+        col4     True
+        col5    False
+        col6     True
+        dtype: bool
+        """
+        from pyspark.pandas.series import first_series
+
+        axis = validate_axis(axis)
+        if axis != 0:
+            raise NotImplementedError('axis should be either 0 or "index" currently.')
+
+        applied = []
+        column_labels = self._internal.column_labels
+        for label in column_labels:
+            scol = self._internal.spark_column_for(label)
+            all_col = F.max(F.coalesce(scol.cast("boolean"), F.lit(False)))
+            applied.append(F.when(all_col.isNull(), False).otherwise(all_col))
+
+        # TODO: there is a similar logic to transpose in, for instance,
+        #  DataFrame.all, Series.quantile. Maybe we should deduplicate it.
+        value_column = "value"
+        cols = []
+        for label, applied_col in zip(column_labels, applied):
+            cols.append(
+                F.struct(
+                    [F.lit(col).alias(SPARK_INDEX_NAME_FORMAT(i)) for i, col in enumerate(label)]
+                    + [applied_col.alias(value_column)]
+                )
+            )
+
+        sdf = self._internal.spark_frame.select(F.array(*cols).alias("arrays")).select(
+            F.explode(F.col("arrays"))
+        )
+        sdf = sdf.selectExpr("col.*")
+
+        internal = InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=[
+                scol_for(sdf, SPARK_INDEX_NAME_FORMAT(i))
+                for i in range(self._internal.column_labels_level)
+            ],
+            index_names=self._internal.column_label_names,
+            column_labels=[None],
+            data_spark_columns=[scol_for(sdf, value_column)],
+        )
+
+        return first_series(DataFrame(internal))
+
+    # TODO: add axis, numeric_only, pct, na_option parameter
+    def rank(self, method="average", ascending=True) -> "DataFrame":
+        """
+        Compute numerical data ranks (1 through n) along axis. Equal values are
+        assigned a rank that is the average of the ranks of those values.
+
+        .. note:: the current implementation of rank uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Parameters
+        ----------
+        method : {'average', 'min', 'max', 'first', 'dense'}
+            * average: average rank of group
+            * min: lowest rank in group
+            * max: highest rank in group
+            * first: ranks assigned in order they appear in the array
+            * dense: like 'min', but rank always increases by 1 between groups
+        ascending : boolean, default True
+            False for ranks by high (1) to low (N)
+
+        Returns
+        -------
+        ranks : same type as caller
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'A': [1, 2, 2, 3], 'B': [4, 3, 2, 1]}, columns= ['A', 'B'])
+        >>> df
+           A  B
+        0  1  4
+        1  2  3
+        2  2  2
+        3  3  1
+
+        >>> df.rank().sort_index()
+             A    B
+        0  1.0  4.0
+        1  2.5  3.0
+        2  2.5  2.0
+        3  4.0  1.0
+
+        If method is set to 'min', it use lowest rank in group.
+
+        >>> df.rank(method='min').sort_index()
+             A    B
+        0  1.0  4.0
+        1  2.0  3.0
+        2  2.0  2.0
+        3  4.0  1.0
+
+        If method is set to 'max', it use highest rank in group.
+
+        >>> df.rank(method='max').sort_index()
+             A    B
+        0  1.0  4.0
+        1  3.0  3.0
+        2  3.0  2.0
+        3  4.0  1.0
+
+        If method is set to 'dense', it leaves no gaps in group.
+
+        >>> df.rank(method='dense').sort_index()
+             A    B
+        0  1.0  4.0
+        1  2.0  3.0
+        2  2.0  2.0
+        3  3.0  1.0
+        """
+        return self._apply_series_op(
+            lambda kser: kser._rank(method=method, ascending=ascending), should_resolve=True
+        )
+
+    def filter(self, items=None, like=None, regex=None, axis=None) -> "DataFrame":
+        """
+        Subset rows or columns of dataframe according to labels in
+        the specified index.
+
+        Note that this routine does not filter a dataframe on its
+        contents. The filter is applied to the labels of the index.
+
+        Parameters
+        ----------
+        items : list-like
+            Keep labels from axis which are in items.
+        like : string
+            Keep labels from axis for which "like in label == True".
+        regex : string (regular expression)
+            Keep labels from axis for which re.search(regex, label) == True.
+        axis : int or string axis name
+            The axis to filter on.  By default this is the info axis,
+            'index' for Series, 'columns' for DataFrame.
+
+        Returns
+        -------
+        same type as input object
+
+        See Also
+        --------
+        DataFrame.loc
+
+        Notes
+        -----
+        The ``items``, ``like``, and ``regex`` parameters are
+        enforced to be mutually exclusive.
+
+        ``axis`` defaults to the info axis that is used when indexing
+        with ``[]``.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame(np.array(([1, 2, 3], [4, 5, 6])),
+        ...                   index=['mouse', 'rabbit'],
+        ...                   columns=['one', 'two', 'three'])
+
+        >>> # select columns by name
+        >>> df.filter(items=['one', 'three'])
+                one  three
+        mouse     1      3
+        rabbit    4      6
+
+        >>> # select columns by regular expression
+        >>> df.filter(regex='e$', axis=1)
+                one  three
+        mouse     1      3
+        rabbit    4      6
+
+        >>> # select rows containing 'bbi'
+        >>> df.filter(like='bbi', axis=0)
+                one  two  three
+        rabbit    4    5      6
+
+        For a Series,
+
+        >>> # select rows by name
+        >>> df.one.filter(items=['rabbit'])
+        rabbit    4
+        Name: one, dtype: int64
+
+        >>> # select rows by regular expression
+        >>> df.one.filter(regex='e$')
+        mouse    1
+        Name: one, dtype: int64
+
+        >>> # select rows containing 'bbi'
+        >>> df.one.filter(like='bbi')
+        rabbit    4
+        Name: one, dtype: int64
+        """
+        if sum(x is not None for x in (items, like, regex)) > 1:
+            raise TypeError(
+                "Keyword arguments `items`, `like`, or `regex` " "are mutually exclusive"
+            )
+
+        axis = validate_axis(axis, none_axis=1)
+
+        index_scols = self._internal.index_spark_columns
+
+        if items is not None:
+            if is_list_like(items):
+                items = list(items)
+            else:
+                raise ValueError("items should be a list-like object.")
+            if axis == 0:
+                if len(index_scols) == 1:
+                    col = None
+                    for item in items:
+                        if col is None:
+                            col = index_scols[0] == F.lit(item)
+                        else:
+                            col = col | (index_scols[0] == F.lit(item))
+                elif len(index_scols) > 1:
+                    # for multi-index
+                    col = None
+                    for item in items:
+                        if not isinstance(item, tuple):
+                            raise TypeError("Unsupported type {}".format(type(item).__name__))
+                        if not item:
+                            raise ValueError("The item should not be empty.")
+                        midx_col = None
+                        for i, element in enumerate(item):
+                            if midx_col is None:
+                                midx_col = index_scols[i] == F.lit(element)
+                            else:
+                                midx_col = midx_col & (index_scols[i] == F.lit(element))
+                        if col is None:
+                            col = midx_col
+                        else:
+                            col = col | midx_col
+                else:
+                    raise ValueError("Single or multi index must be specified.")
+                return DataFrame(self._internal.with_filter(col))
+            else:
+                return self[items]
+        elif like is not None:
+            if axis == 0:
+                col = None
+                for index_scol in index_scols:
+                    if col is None:
+                        col = index_scol.contains(like)
+                    else:
+                        col = col | index_scol.contains(like)
+                return DataFrame(self._internal.with_filter(col))
+            else:
+                column_labels = self._internal.column_labels
+                output_labels = [label for label in column_labels if any(like in i for i in label)]
+                return self[output_labels]
+        elif regex is not None:
+            if axis == 0:
+                col = None
+                for index_scol in index_scols:
+                    if col is None:
+                        col = index_scol.rlike(regex)
+                    else:
+                        col = col | index_scol.rlike(regex)
+                return DataFrame(self._internal.with_filter(col))
+            else:
+                column_labels = self._internal.column_labels
+                matcher = re.compile(regex)
+                output_labels = [
+                    label
+                    for label in column_labels
+                    if any(matcher.search(i) is not None for i in label)
+                ]
+                return self[output_labels]
+        else:
+            raise TypeError("Must pass either `items`, `like`, or `regex`")
+
+    def rename(
+        self,
+        mapper=None,
+        index=None,
+        columns=None,
+        axis="index",
+        inplace=False,
+        level=None,
+        errors="ignore",
+    ) -> Optional["DataFrame"]:
+
+        """
+        Alter axes labels.
+        Function / dict values must be unique (1-to-1). Labels not contained in a dict / Series
+        will be left as-is. Extra labels listed don’t throw an error.
+
+        Parameters
+        ----------
+        mapper : dict-like or function
+            Dict-like or functions transformations to apply to that axis’ values.
+            Use either `mapper` and `axis` to specify the axis to target with `mapper`, or `index`
+            and `columns`.
+        index : dict-like or function
+            Alternative to specifying axis ("mapper, axis=0" is equivalent to "index=mapper").
+        columns : dict-like or function
+            Alternative to specifying axis ("mapper, axis=1" is equivalent to "columns=mapper").
+        axis : int or str, default 'index'
+            Axis to target with mapper. Can be either the axis name ('index', 'columns') or
+            number (0, 1).
+        inplace : bool, default False
+            Whether to return a new DataFrame.
+        level : int or level name, default None
+            In case of a MultiIndex, only rename labels in the specified level.
+        errors : {'ignore', 'raise}, default 'ignore'
+            If 'raise', raise a `KeyError` when a dict-like `mapper`, `index`, or `columns`
+            contains labels that are not present in the Index being transformed. If 'ignore',
+            existing keys will be renamed and extra keys will be ignored.
+
+        Returns
+        -------
+        DataFrame with the renamed axis labels.
+
+        Raises
+        ------
+        `KeyError`
+            If any of the labels is not found in the selected axis and "errors='raise'".
+
+        Examples
+        --------
+        >>> kdf1 = ps.DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]})
+        >>> kdf1.rename(columns={"A": "a", "B": "c"})  # doctest: +NORMALIZE_WHITESPACE
+           a  c
+        0  1  4
+        1  2  5
+        2  3  6
+
+        >>> kdf1.rename(index={1: 10, 2: 20})  # doctest: +NORMALIZE_WHITESPACE
+            A  B
+        0   1  4
+        10  2  5
+        20  3  6
+
+        >>> def str_lower(s) -> str:
+        ...     return str.lower(s)
+        >>> kdf1.rename(str_lower, axis='columns')  # doctest: +NORMALIZE_WHITESPACE
+           a  b
+        0  1  4
+        1  2  5
+        2  3  6
+
+        >>> def mul10(x) -> int:
+        ...     return x * 10
+        >>> kdf1.rename(mul10, axis='index')  # doctest: +NORMALIZE_WHITESPACE
+            A  B
+        0   1  4
+        10  2  5
+        20  3  6
+
+        >>> idx = pd.MultiIndex.from_tuples([('X', 'A'), ('X', 'B'), ('Y', 'C'), ('Y', 'D')])
+        >>> kdf2 = ps.DataFrame([[1, 2, 3, 4], [5, 6, 7, 8]], columns=idx)
+        >>> kdf2.rename(columns=str_lower, level=0)  # doctest: +NORMALIZE_WHITESPACE
+           x     y
+           A  B  C  D
+        0  1  2  3  4
+        1  5  6  7  8
+
+        >>> kdf3 = ps.DataFrame([[1, 2], [3, 4], [5, 6], [7, 8]], index=idx, columns=list('ab'))
+        >>> kdf3.rename(index=str_lower)  # doctest: +NORMALIZE_WHITESPACE
+             a  b
+        x a  1  2
+          b  3  4
+        y c  5  6
+          d  7  8
+        """
+
+        def gen_mapper_fn(mapper):
+            if isinstance(mapper, dict):
+                if len(mapper) == 0:
+                    if errors == "raise":
+                        raise KeyError("Index include label which is not in the `mapper`.")
+                    else:
+                        return DataFrame(self._internal)
+
+                type_set = set(map(lambda x: type(x), mapper.values()))
+                if len(type_set) > 1:
+                    raise ValueError("Mapper dict should have the same value type.")
+                spark_return_type = as_spark_type(list(type_set)[0])
+
+                def mapper_fn(x):
+                    if x in mapper:
+                        return mapper[x]
+                    else:
+                        if errors == "raise":
+                            raise KeyError("Index include value which is not in the `mapper`")
+                        return x
+
+            elif callable(mapper):
+                spark_return_type = cast(ScalarType, infer_return_type(mapper)).spark_type
+
+                def mapper_fn(x):
+                    return mapper(x)
+
+            else:
+                raise ValueError(
+                    "`mapper` or `index` or `columns` should be "
+                    "either dict-like or function type."
+                )
+            return mapper_fn, spark_return_type
+
+        index_mapper_fn = None
+        index_mapper_ret_stype = None
+        columns_mapper_fn = None
+
+        inplace = validate_bool_kwarg(inplace, "inplace")
+        if mapper:
+            axis = validate_axis(axis)
+            if axis == 0:
+                index_mapper_fn, index_mapper_ret_stype = gen_mapper_fn(mapper)
+            elif axis == 1:
+                columns_mapper_fn, columns_mapper_ret_stype = gen_mapper_fn(mapper)
+            else:
+                raise ValueError(
+                    "argument axis should be either the axis name "
+                    "(‘index’, ‘columns’) or number (0, 1)"
+                )
+        else:
+            if index:
+                index_mapper_fn, index_mapper_ret_stype = gen_mapper_fn(index)
+            if columns:
+                columns_mapper_fn, _ = gen_mapper_fn(columns)
+
+            if not index and not columns:
+                raise ValueError("Either `index` or `columns` should be provided.")
+
+        kdf = self.copy()
+        if index_mapper_fn:
+            # rename index labels, if `level` is None, rename all index columns, otherwise only
+            # rename the corresponding level index.
+            # implement this by transform the underlying spark dataframe,
+            # Example:
+            # suppose the kdf index column in underlying spark dataframe is "index_0", "index_1",
+            # if rename level 0 index labels, will do:
+            #   ``kdf._sdf.withColumn("index_0", mapper_fn_udf(col("index_0"))``
+            # if rename all index labels (`level` is None), then will do:
+            #   ```
+            #   kdf._sdf.withColumn("index_0", mapper_fn_udf(col("index_0"))
+            #           .withColumn("index_1", mapper_fn_udf(col("index_1"))
+            #   ```
+
+            index_columns = kdf._internal.index_spark_column_names
+            num_indices = len(index_columns)
+            if level:
+                if level < 0 or level >= num_indices:
+                    raise ValueError("level should be an integer between [0, num_indices)")
+
+            def gen_new_index_column(level):
+                index_col_name = index_columns[level]
+
+                index_mapper_udf = pandas_udf(
+                    lambda s: s.map(index_mapper_fn), returnType=index_mapper_ret_stype
+                )
+                return index_mapper_udf(scol_for(kdf._internal.spark_frame, index_col_name))
+
+            sdf = kdf._internal.resolved_copy.spark_frame
+            index_dtypes = self._internal.index_dtypes.copy()
+            if level is None:
+                for i in range(num_indices):
+                    sdf = sdf.withColumn(index_columns[i], gen_new_index_column(i))
+                    index_dtypes[i] = None  # TODO: dtype?
+            else:
+                sdf = sdf.withColumn(index_columns[level], gen_new_index_column(level))
+                index_dtypes[level] = None  # TODO: dtype?
+            kdf = DataFrame(kdf._internal.with_new_sdf(sdf, index_dtypes=index_dtypes))
+        if columns_mapper_fn:
+            # rename column name.
+            # Will modify the `_internal._column_labels` and transform underlying spark dataframe
+            # to the same column name with `_internal._column_labels`.
+            if level:
+                if level < 0 or level >= kdf._internal.column_labels_level:
+                    raise ValueError("level should be an integer between [0, column_labels_level)")
+
+            def gen_new_column_labels_entry(column_labels_entry):
+                if isinstance(column_labels_entry, tuple):
+                    if level is None:
+                        # rename all level columns
+                        return tuple(map(columns_mapper_fn, column_labels_entry))
+                    else:
+                        # only rename specified level column
+                        entry_list = list(column_labels_entry)
+                        entry_list[level] = columns_mapper_fn(entry_list[level])
+                        return tuple(entry_list)
+                else:
+                    return columns_mapper_fn(column_labels_entry)
+
+            new_column_labels = list(map(gen_new_column_labels_entry, kdf._internal.column_labels))
+
+            new_data_scols = [
+                kdf._kser_for(old_label).rename(new_label)
+                for old_label, new_label in zip(kdf._internal.column_labels, new_column_labels)
+            ]
+            kdf = DataFrame(kdf._internal.with_new_columns(new_data_scols))
+        if inplace:
+            self._update_internal_frame(kdf._internal)
+            return None
+        else:
+            return kdf
+
+    def rename_axis(
+        self,
+        mapper: Optional[Any] = None,
+        index: Optional[Any] = None,
+        columns: Optional[Any] = None,
+        axis: Optional[Union[int, str]] = 0,
+        inplace: Optional[bool] = False,
+    ) -> Optional["DataFrame"]:
+        """
+        Set the name of the axis for the index or columns.
+
+        Parameters
+        ----------
+        mapper : scalar, list-like, optional
+            A scalar, list-like, dict-like or functions transformations to
+            apply to the axis name attribute.
+        index, columns : scalar, list-like, dict-like or function, optional
+            A scalar, list-like, dict-like or functions transformations to
+            apply to that axis' values.
+
+            Use either ``mapper`` and ``axis`` to
+            specify the axis to target with ``mapper``, or ``index``
+            and/or ``columns``.
+        axis : {0 or 'index', 1 or 'columns'}, default 0
+            The axis to rename.
+        inplace : bool, default False
+            Modifies the object directly, instead of creating a new DataFrame.
+
+        Returns
+        -------
+        DataFrame, or None if `inplace` is True.
+
+        See Also
+        --------
+        Series.rename : Alter Series index labels or name.
+        DataFrame.rename : Alter DataFrame index labels or name.
+        Index.rename : Set new names on index.
+
+        Notes
+        -----
+        ``DataFrame.rename_axis`` supports two calling conventions
+
+        * ``(index=index_mapper, columns=columns_mapper, ...)``
+        * ``(mapper, axis={'index', 'columns'}, ...)``
+
+        The first calling convention will only modify the names of
+        the index and/or the names of the Index object that is the columns.
+
+        The second calling convention will modify the names of the
+        corresponding index specified by axis.
+
+        We *highly* recommend using keyword arguments to clarify your
+        intent.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({"num_legs": [4, 4, 2],
+        ...                    "num_arms": [0, 0, 2]},
+        ...                   index=["dog", "cat", "monkey"],
+        ...                   columns=["num_legs", "num_arms"])
+        >>> df
+                num_legs  num_arms
+        dog            4         0
+        cat            4         0
+        monkey         2         2
+
+        >>> df = df.rename_axis("animal").sort_index()
+        >>> df  # doctest: +NORMALIZE_WHITESPACE
+                num_legs  num_arms
+        animal
+        cat            4         0
+        dog            4         0
+        monkey         2         2
+
+        >>> df = df.rename_axis("limbs", axis="columns").sort_index()
+        >>> df # doctest: +NORMALIZE_WHITESPACE
+        limbs   num_legs  num_arms
+        animal
+        cat            4         0
+        dog            4         0
+        monkey         2         2
+
+        **MultiIndex**
+
+        >>> index = pd.MultiIndex.from_product([['mammal'],
+        ...                                     ['dog', 'cat', 'monkey']],
+        ...                                    names=['type', 'name'])
+        >>> df = ps.DataFrame({"num_legs": [4, 4, 2],
+        ...                    "num_arms": [0, 0, 2]},
+        ...                   index=index,
+        ...                   columns=["num_legs", "num_arms"])
+        >>> df  # doctest: +NORMALIZE_WHITESPACE
+                       num_legs  num_arms
+        type   name
+        mammal dog            4         0
+               cat            4         0
+               monkey         2         2
+
+        >>> df.rename_axis(index={'type': 'class'}).sort_index()  # doctest: +NORMALIZE_WHITESPACE
+                       num_legs  num_arms
+        class  name
+        mammal cat            4         0
+               dog            4         0
+               monkey         2         2
+
+        >>> df.rename_axis(index=str.upper).sort_index()  # doctest: +NORMALIZE_WHITESPACE
+                       num_legs  num_arms
+        TYPE   NAME
+        mammal cat            4         0
+               dog            4         0
+               monkey         2         2
+        """
+
+        def gen_names(v, curnames):
+            if is_scalar(v):
+                newnames = [v]
+            elif is_list_like(v) and not is_dict_like(v):
+                newnames = list(v)
+            elif is_dict_like(v):
+                newnames = [v[name] if name in v else name for name in curnames]
+            elif callable(v):
+                newnames = [v(name) for name in curnames]
+            else:
+                raise ValueError(
+                    "`mapper` or `index` or `columns` should be "
+                    "either dict-like or function type."
+                )
+
+            if len(newnames) != len(curnames):
+                raise ValueError(
+                    "Length of new names must be {}, got {}".format(len(curnames), len(newnames))
+                )
+
+            return [name if is_name_like_tuple(name) else (name,) for name in newnames]
+
+        if mapper is not None and (index is not None or columns is not None):
+            raise TypeError("Cannot specify both 'mapper' and any of 'index' or 'columns'.")
+
+        if mapper is not None:
+            axis = validate_axis(axis)
+            if axis == 0:
+                index = mapper
+            elif axis == 1:
+                columns = mapper
+
+        column_label_names = (
+            gen_names(columns, self.columns.names)
+            if columns is not None
+            else self._internal.column_label_names
+        )
+        index_names = (
+            gen_names(index, self.index.names) if index is not None else self._internal.index_names
+        )
+
+        internal = self._internal.copy(
+            index_names=index_names, column_label_names=column_label_names
+        )
+        if inplace:
+            self._update_internal_frame(internal)
+            return None
+        else:
+            return DataFrame(internal)
+
+    def keys(self) -> pd.Index:
+        """
+        Return alias for columns.
+
+        Returns
+        -------
+        Index
+            Columns of the DataFrame.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([[1, 2], [4, 5], [7, 8]],
+        ...                   index=['cobra', 'viper', 'sidewinder'],
+        ...                   columns=['max_speed', 'shield'])
+        >>> df
+                    max_speed  shield
+        cobra               1       2
+        viper               4       5
+        sidewinder          7       8
+
+        >>> df.keys()
+        Index(['max_speed', 'shield'], dtype='object')
+        """
+        return self.columns
+
+    def pct_change(self, periods=1) -> "DataFrame":
+        """
+        Percentage change between the current and a prior element.
+
+        .. note:: the current implementation of this API uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Parameters
+        ----------
+        periods : int, default 1
+            Periods to shift for forming percent change.
+
+        Returns
+        -------
+        DataFrame
+
+        Examples
+        --------
+        Percentage change in French franc, Deutsche Mark, and Italian lira
+        from 1980-01-01 to 1980-03-01.
+
+        >>> df = ps.DataFrame({
+        ...     'FR': [4.0405, 4.0963, 4.3149],
+        ...     'GR': [1.7246, 1.7482, 1.8519],
+        ...     'IT': [804.74, 810.01, 860.13]},
+        ...     index=['1980-01-01', '1980-02-01', '1980-03-01'])
+        >>> df
+                        FR      GR      IT
+        1980-01-01  4.0405  1.7246  804.74
+        1980-02-01  4.0963  1.7482  810.01
+        1980-03-01  4.3149  1.8519  860.13
+
+        >>> df.pct_change()
+                          FR        GR        IT
+        1980-01-01       NaN       NaN       NaN
+        1980-02-01  0.013810  0.013684  0.006549
+        1980-03-01  0.053365  0.059318  0.061876
+
+        You can set periods to shift for forming percent change
+
+        >>> df.pct_change(2)
+                          FR        GR       IT
+        1980-01-01       NaN       NaN      NaN
+        1980-02-01       NaN       NaN      NaN
+        1980-03-01  0.067912  0.073814  0.06883
+        """
+        window = Window.orderBy(NATURAL_ORDER_COLUMN_NAME).rowsBetween(-periods, -periods)
+
+        def op(kser):
+            prev_row = F.lag(kser.spark.column, periods).over(window)
+            return ((kser.spark.column - prev_row) / prev_row).alias(
+                kser._internal.data_spark_column_names[0]
+            )
+
+        return self._apply_series_op(op, should_resolve=True)
+
+    # TODO: axis = 1
+    def idxmax(self, axis=0) -> "Series":
+        """
+        Return index of first occurrence of maximum over requested axis.
+        NA/null values are excluded.
+
+        .. note:: This API collect all rows with maximum value using `to_pandas()`
+            because we suppose the number of rows with max values are usually small in general.
+
+        Parameters
+        ----------
+        axis : 0 or 'index'
+            Can only be set to 0 at the moment.
+
+        Returns
+        -------
+        Series
+
+        See Also
+        --------
+        Series.idxmax
+
+        Examples
+        --------
+        >>> kdf = ps.DataFrame({'a': [1, 2, 3, 2],
+        ...                     'b': [4.0, 2.0, 3.0, 1.0],
+        ...                     'c': [300, 200, 400, 200]})
+        >>> kdf
+           a    b    c
+        0  1  4.0  300
+        1  2  2.0  200
+        2  3  3.0  400
+        3  2  1.0  200
+
+        >>> kdf.idxmax()
+        a    2
+        b    0
+        c    2
+        dtype: int64
+
+        For Multi-column Index
+
+        >>> kdf = ps.DataFrame({'a': [1, 2, 3, 2],
+        ...                     'b': [4.0, 2.0, 3.0, 1.0],
+        ...                     'c': [300, 200, 400, 200]})
+        >>> kdf.columns = pd.MultiIndex.from_tuples([('a', 'x'), ('b', 'y'), ('c', 'z')])
+        >>> kdf
+           a    b    c
+           x    y    z
+        0  1  4.0  300
+        1  2  2.0  200
+        2  3  3.0  400
+        3  2  1.0  200
+
+        >>> kdf.idxmax()
+        a  x    2
+        b  y    0
+        c  z    2
+        dtype: int64
+        """
+        max_cols = map(lambda scol: F.max(scol), self._internal.data_spark_columns)
+        sdf_max = self._internal.spark_frame.select(*max_cols).head()
+        # `sdf_max` looks like below
+        # +------+------+------+
+        # |(a, x)|(b, y)|(c, z)|
+        # +------+------+------+
+        # |     3|   4.0|   400|
+        # +------+------+------+
+
+        conds = (
+            scol == max_val for scol, max_val in zip(self._internal.data_spark_columns, sdf_max)
+        )
+        cond = reduce(lambda x, y: x | y, conds)
+
+        kdf = DataFrame(self._internal.with_filter(cond))  # type: "DataFrame"
+
+        return cast(ps.Series, ps.from_pandas(kdf._to_internal_pandas().idxmax()))
+
+    # TODO: axis = 1
+    def idxmin(self, axis=0) -> "Series":
+        """
+        Return index of first occurrence of minimum over requested axis.
+        NA/null values are excluded.
+
+        .. note:: This API collect all rows with minimum value using `to_pandas()`
+            because we suppose the number of rows with min values are usually small in general.
+
+        Parameters
+        ----------
+        axis : 0 or 'index'
+            Can only be set to 0 at the moment.
+
+        Returns
+        -------
+        Series
+
+        See Also
+        --------
+        Series.idxmin
+
+        Examples
+        --------
+        >>> kdf = ps.DataFrame({'a': [1, 2, 3, 2],
+        ...                     'b': [4.0, 2.0, 3.0, 1.0],
+        ...                     'c': [300, 200, 400, 200]})
+        >>> kdf
+           a    b    c
+        0  1  4.0  300
+        1  2  2.0  200
+        2  3  3.0  400
+        3  2  1.0  200
+
+        >>> kdf.idxmin()
+        a    0
+        b    3
+        c    1
+        dtype: int64
+
+        For Multi-column Index
+
+        >>> kdf = ps.DataFrame({'a': [1, 2, 3, 2],
+        ...                     'b': [4.0, 2.0, 3.0, 1.0],
+        ...                     'c': [300, 200, 400, 200]})
+        >>> kdf.columns = pd.MultiIndex.from_tuples([('a', 'x'), ('b', 'y'), ('c', 'z')])
+        >>> kdf
+           a    b    c
+           x    y    z
+        0  1  4.0  300
+        1  2  2.0  200
+        2  3  3.0  400
+        3  2  1.0  200
+
+        >>> kdf.idxmin()
+        a  x    0
+        b  y    3
+        c  z    1
+        dtype: int64
+        """
+        min_cols = map(lambda scol: F.min(scol), self._internal.data_spark_columns)
+        sdf_min = self._internal.spark_frame.select(*min_cols).head()
+
+        conds = (
+            scol == min_val for scol, min_val in zip(self._internal.data_spark_columns, sdf_min)
+        )
+        cond = reduce(lambda x, y: x | y, conds)
+
+        kdf = DataFrame(self._internal.with_filter(cond))  # type: "DataFrame"
+
+        return cast(ps.Series, ps.from_pandas(kdf._to_internal_pandas().idxmin()))
+
+    def info(self, verbose=None, buf=None, max_cols=None, null_counts=None) -> None:
+        """
+        Print a concise summary of a DataFrame.
+
+        This method prints information about a DataFrame including
+        the index dtype and column dtypes, non-null values and memory usage.
+
+        Parameters
+        ----------
+        verbose : bool, optional
+            Whether to print the full summary.
+        buf : writable buffer, defaults to sys.stdout
+            Where to send the output. By default, the output is printed to
+            sys.stdout. Pass a writable buffer if you need to further process
+            the output.
+        max_cols : int, optional
+            When to switch from the verbose to the truncated output. If the
+            DataFrame has more than `max_cols` columns, the truncated output
+            is used.
+        null_counts : bool, optional
+            Whether to show the non-null counts.
+
+        Returns
+        -------
+        None
+            This method prints a summary of a DataFrame and returns None.
+
+        See Also
+        --------
+        DataFrame.describe: Generate descriptive statistics of DataFrame
+            columns.
+
+        Examples
+        --------
+        >>> int_values = [1, 2, 3, 4, 5]
+        >>> text_values = ['alpha', 'beta', 'gamma', 'delta', 'epsilon']
+        >>> float_values = [0.0, 0.25, 0.5, 0.75, 1.0]
+        >>> df = ps.DataFrame(
+        ...     {"int_col": int_values, "text_col": text_values, "float_col": float_values},
+        ...     columns=['int_col', 'text_col', 'float_col'])
+        >>> df
+           int_col text_col  float_col
+        0        1    alpha       0.00
+        1        2     beta       0.25
+        2        3    gamma       0.50
+        3        4    delta       0.75
+        4        5  epsilon       1.00
+
+        Prints information of all columns:
+
+        >>> df.info(verbose=True)  # doctest: +SKIP
+        <class 'pyspark.pandas.frame.DataFrame'>
+        Index: 5 entries, 0 to 4
+        Data columns (total 3 columns):
+         #   Column     Non-Null Count  Dtype
+        ---  ------     --------------  -----
+         0   int_col    5 non-null      int64
+         1   text_col   5 non-null      object
+         2   float_col  5 non-null      float64
+        dtypes: float64(1), int64(1), object(1)
+
+        Prints a summary of columns count and its dtypes but not per column
+        information:
+
+        >>> df.info(verbose=False)  # doctest: +SKIP
+        <class 'pyspark.pandas.frame.DataFrame'>
+        Index: 5 entries, 0 to 4
+        Columns: 3 entries, int_col to float_col
+        dtypes: float64(1), int64(1), object(1)
+
+        Pipe output of DataFrame.info to buffer instead of sys.stdout, get
+        buffer content and writes to a text file:
+
+        >>> import io
+        >>> buffer = io.StringIO()
+        >>> df.info(buf=buffer)
+        >>> s = buffer.getvalue()
+        >>> with open('%s/info.txt' % path, "w",
+        ...           encoding="utf-8") as f:
+        ...     _ = f.write(s)
+        >>> with open('%s/info.txt' % path) as f:
+        ...     f.readlines()  # doctest: +SKIP
+        ["<class 'pyspark.pandas.frame.DataFrame'>\\n",
+        'Index: 5 entries, 0 to 4\\n',
+        'Data columns (total 3 columns):\\n',
+        ' #   Column     Non-Null Count  Dtype  \\n',
+        '---  ------     --------------  -----  \\n',
+        ' 0   int_col    5 non-null      int64  \\n',
+        ' 1   text_col   5 non-null      object \\n',
+        ' 2   float_col  5 non-null      float64\\n',
+        'dtypes: float64(1), int64(1), object(1)']
+        """
+        # To avoid pandas' existing config affects pandas-on-Spark.
+        # TODO: should we have corresponding pandas-on-Spark configs?
+        with pd.option_context(
+            "display.max_info_columns", sys.maxsize, "display.max_info_rows", sys.maxsize
+        ):
+            try:
+                # hack to use pandas' info as is.
+                object.__setattr__(self, "_data", self)
+                count_func = self.count
+                self.count = lambda: count_func().to_pandas()  # type: ignore
+                return pd.DataFrame.info(
+                    self,
+                    verbose=verbose,
+                    buf=buf,
+                    max_cols=max_cols,
+                    memory_usage=False,
+                    null_counts=null_counts,
+                )
+            finally:
+                del self._data
+                self.count = count_func  # type: ignore
+
+    # TODO: fix parameter 'axis' and 'numeric_only' to work same as pandas'
+    def quantile(
+        self,
+        q: Union[float, Iterable[float]] = 0.5,
+        axis: Union[int, str] = 0,
+        numeric_only: bool = True,
+        accuracy: int = 10000,
+    ) -> Union["DataFrame", "Series"]:
+        """
+        Return value at the given quantile.
+
+        .. note:: Unlike pandas', the quantile in pandas-on-Spark is an approximated quantile
+            based upon approximate percentile computation because computing quantile across a
+            large dataset is extremely expensive.
+
+        Parameters
+        ----------
+        q : float or array-like, default 0.5 (50% quantile)
+            0 <= q <= 1, the quantile(s) to compute.
+        axis : int or str, default 0 or 'index'
+            Can only be set to 0 at the moment.
+        numeric_only : bool, default True
+            If False, the quantile of datetime and timedelta data will be computed as well.
+            Can only be set to True at the moment.
+        accuracy : int, optional
+            Default accuracy of approximation. Larger value means better accuracy.
+            The relative error can be deduced by 1.0 / accuracy.
+
+        Returns
+        -------
+        Series or DataFrame
+            If q is an array, a DataFrame will be returned where the
+            index is q, the columns are the columns of self, and the values are the quantiles.
+            If q is a float, a Series will be returned where the
+            index is the columns of self and the values are the quantiles.
+
+        Examples
+        --------
+        >>> kdf = ps.DataFrame({'a': [1, 2, 3, 4, 5], 'b': [6, 7, 8, 9, 0]})
+        >>> kdf
+           a  b
+        0  1  6
+        1  2  7
+        2  3  8
+        3  4  9
+        4  5  0
+
+        >>> kdf.quantile(.5)
+        a    3.0
+        b    7.0
+        Name: 0.5, dtype: float64
+
+        >>> kdf.quantile([.25, .5, .75])
+                a    b
+        0.25  2.0  6.0
+        0.50  3.0  7.0
+        0.75  4.0  8.0
+        """
+        axis = validate_axis(axis)
+        if axis != 0:
+            raise NotImplementedError('axis should be either 0 or "index" currently.')
+
+        if not isinstance(accuracy, int):
+            raise TypeError(
+                "accuracy must be an integer; however, got [%s]" % type(accuracy).__name__
+            )
+
+        if isinstance(q, Iterable):
+            q = list(q)
+
+        for v in q if isinstance(q, list) else [q]:
+            if not isinstance(v, float):
+                raise TypeError(
+                    "q must be a float or an array of floats; however, [%s] found." % type(v)
+                )
+            if v < 0.0 or v > 1.0:
+                raise ValueError("percentiles should all be in the interval [0, 1].")
+
+        def quantile(spark_column, spark_type):
+            if isinstance(spark_type, (BooleanType, NumericType)):
+                return F.percentile_approx(spark_column.cast(DoubleType()), q, accuracy)
+            else:
+                raise TypeError(
+                    "Could not convert {} ({}) to numeric".format(
+                        spark_type_to_pandas_dtype(spark_type), spark_type.simpleString()
+                    )
+                )
+
+        if isinstance(q, list):
+            # First calculate the percentiles from all columns and map it to each `quantiles`
+            # by creating each entry as a struct. So, it becomes an array of structs as below:
+            #
+            # +-----------------------------------------+
+            # |                                   arrays|
+            # +-----------------------------------------+
+            # |[[0.25, 2, 6], [0.5, 3, 7], [0.75, 4, 8]]|
+            # +-----------------------------------------+
+
+            percentile_cols = []
+            percentile_col_names = []
+            column_labels = []
+            for label, column in zip(
+                self._internal.column_labels, self._internal.data_spark_column_names
+            ):
+                spark_type = self._internal.spark_type_for(label)
+
+                is_numeric_or_boolean = isinstance(spark_type, (NumericType, BooleanType))
+                keep_column = not numeric_only or is_numeric_or_boolean
+
+                if keep_column:
+                    percentile_col = quantile(self._internal.spark_column_for(label), spark_type)
+                    percentile_cols.append(percentile_col.alias(column))
+                    percentile_col_names.append(column)
+                    column_labels.append(label)
+
+            if len(percentile_cols) == 0:
+                return DataFrame(index=q)
+
+            sdf = self._internal.spark_frame.select(percentile_cols)
+            # Here, after select percentile cols, a spark_frame looks like below:
+            # +---------+---------+
+            # |        a|        b|
+            # +---------+---------+
+            # |[2, 3, 4]|[6, 7, 8]|
+            # +---------+---------+
+
+            cols_dict = OrderedDict()  # type: OrderedDict
+            for column in percentile_col_names:
+                cols_dict[column] = list()
+                for i in range(len(q)):
+                    cols_dict[column].append(scol_for(sdf, column).getItem(i).alias(column))
+
+            internal_index_column = SPARK_DEFAULT_INDEX_NAME
+            cols = []
+            for i, col in enumerate(zip(*cols_dict.values())):
+                cols.append(F.struct(F.lit(q[i]).alias(internal_index_column), *col))
+            sdf = sdf.select(F.array(*cols).alias("arrays"))
+
+            # And then, explode it and manually set the index.
+            # +-----------------+---+---+
+            # |__index_level_0__|  a|  b|
+            # +-----------------+---+---+
+            # |             0.25|  2|  6|
+            # |              0.5|  3|  7|
+            # |             0.75|  4|  8|
+            # +-----------------+---+---+
+            sdf = sdf.select(F.explode(F.col("arrays"))).selectExpr("col.*")
+
+            internal = InternalFrame(
+                spark_frame=sdf,
+                index_spark_columns=[scol_for(sdf, internal_index_column)],
+                column_labels=column_labels,
+                data_spark_columns=[scol_for(sdf, col) for col in percentile_col_names],
+            )
+            return DataFrame(internal)
+        else:
+            return self._reduce_for_stat_function(
+                quantile, name="quantile", numeric_only=numeric_only
+            ).rename(q)
+
+    def query(self, expr, inplace=False) -> Optional["DataFrame"]:
+        """
+        Query the columns of a DataFrame with a boolean expression.
+
+        .. note:: Internal columns that starting with a '__' prefix are able to access, however,
+            they are not supposed to be accessed.
+
+        .. note:: This API delegates to Spark SQL so the syntax follows Spark SQL. Therefore, the
+            pandas specific syntax such as `@` is not supported. If you want the pandas syntax,
+            you can work around with :meth:`DataFrame.koalas.apply_batch`, but you should
+            be aware that `query_func` will be executed at different nodes in a distributed manner.
+            So, for example, to use `@` syntax, make sure the variable is serialized by, for
+            example, putting it within the closure as below.
+
+            >>> df = ps.DataFrame({'A': range(2000), 'B': range(2000)})
+            >>> def query_func(pdf):
+            ...     num = 1995
+            ...     return pdf.query('A > @num')
+            >>> df.koalas.apply_batch(query_func)
+                     A     B
+            1996  1996  1996
+            1997  1997  1997
+            1998  1998  1998
+            1999  1999  1999
+
+        Parameters
+        ----------
+        expr : str
+            The query string to evaluate.
+
+            You can refer to column names that contain spaces by surrounding
+            them in backticks.
+
+            For example, if one of your columns is called ``a a`` and you want
+            to sum it with ``b``, your query should be ```a a` + b``.
+
+        inplace : bool
+            Whether the query should modify the data in place or return
+            a modified copy.
+
+        Returns
+        -------
+        DataFrame
+            DataFrame resulting from the provided query expression.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'A': range(1, 6),
+        ...                    'B': range(10, 0, -2),
+        ...                    'C C': range(10, 5, -1)})
+        >>> df
+           A   B  C C
+        0  1  10   10
+        1  2   8    9
+        2  3   6    8
+        3  4   4    7
+        4  5   2    6
+
+        >>> df.query('A > B')
+           A  B  C C
+        4  5  2    6
+
+        The previous expression is equivalent to
+
+        >>> df[df.A > df.B]
+           A  B  C C
+        4  5  2    6
+
+        For columns with spaces in their name, you can use backtick quoting.
+
+        >>> df.query('B == `C C`')
+           A   B  C C
+        0  1  10   10
+
+        The previous expression is equivalent to
+
+        >>> df[df.B == df['C C']]
+           A   B  C C
+        0  1  10   10
+        """
+        if isinstance(self.columns, pd.MultiIndex):
+            raise TypeError("Doesn't support for MultiIndex columns")
+        if not isinstance(expr, str):
+            raise TypeError(
+                "expr must be a string to be evaluated, {} given".format(type(expr).__name__)
+            )
+        inplace = validate_bool_kwarg(inplace, "inplace")
+
+        data_columns = [label[0] for label in self._internal.column_labels]
+        sdf = self._internal.spark_frame.select(
+            self._internal.index_spark_columns
+            + [
+                scol.alias(col)
+                for scol, col in zip(self._internal.data_spark_columns, data_columns)
+            ]
+        ).filter(expr)
+        internal = self._internal.with_new_sdf(sdf, data_columns=data_columns)
+
+        if inplace:
+            self._update_internal_frame(internal)
+            return None
+        else:
+            return DataFrame(internal)
+
+    def explain(self, extended: Optional[bool] = None, mode: Optional[str] = None) -> None:
+        warnings.warn(
+            "DataFrame.explain is deprecated as of DataFrame.spark.explain. "
+            "Please use the API instead.",
+            FutureWarning,
+        )
+        return self.spark.explain(extended, mode)
+
+    explain.__doc__ = SparkFrameMethods.explain.__doc__
+
+    def take(self, indices, axis=0, **kwargs) -> "DataFrame":
+        """
+        Return the elements in the given *positional* indices along an axis.
+
+        This means that we are not indexing according to actual values in
+        the index attribute of the object. We are indexing according to the
+        actual position of the element in the object.
+
+        Parameters
+        ----------
+        indices : array-like
+            An array of ints indicating which positions to take.
+        axis : {0 or 'index', 1 or 'columns', None}, default 0
+            The axis on which to select elements. ``0`` means that we are
+            selecting rows, ``1`` means that we are selecting columns.
+        **kwargs
+            For compatibility with :meth:`numpy.take`. Has no effect on the
+            output.
+
+        Returns
+        -------
+        taken : same type as caller
+            An array-like containing the elements taken from the object.
+
+        See Also
+        --------
+        DataFrame.loc : Select a subset of a DataFrame by labels.
+        DataFrame.iloc : Select a subset of a DataFrame by positions.
+        numpy.take : Take elements from an array along an axis.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([('falcon', 'bird', 389.0),
+        ...                    ('parrot', 'bird', 24.0),
+        ...                    ('lion', 'mammal', 80.5),
+        ...                    ('monkey', 'mammal', np.nan)],
+        ...                   columns=['name', 'class', 'max_speed'],
+        ...                   index=[0, 2, 3, 1])
+        >>> df
+             name   class  max_speed
+        0  falcon    bird      389.0
+        2  parrot    bird       24.0
+        3    lion  mammal       80.5
+        1  monkey  mammal        NaN
+
+        Take elements at positions 0 and 3 along the axis 0 (default).
+
+        Note how the actual indices selected (0 and 1) do not correspond to
+        our selected indices 0 and 3. That's because we are selecting the 0th
+        and 3rd rows, not rows whose indices equal 0 and 3.
+
+        >>> df.take([0, 3]).sort_index()
+             name   class  max_speed
+        0  falcon    bird      389.0
+        1  monkey  mammal        NaN
+
+        Take elements at indices 1 and 2 along the axis 1 (column selection).
+
+        >>> df.take([1, 2], axis=1)
+            class  max_speed
+        0    bird      389.0
+        2    bird       24.0
+        3  mammal       80.5
+        1  mammal        NaN
+
+        We may take elements using negative integers for positive indices,
+        starting from the end of the object, just like with Python lists.
+
+        >>> df.take([-1, -2]).sort_index()
+             name   class  max_speed
+        1  monkey  mammal        NaN
+        3    lion  mammal       80.5
+        """
+        axis = validate_axis(axis)
+        if not is_list_like(indices) or isinstance(indices, (dict, set)):
+            raise TypeError("`indices` must be a list-like except dict or set")
+        if axis == 0:
+            return cast(DataFrame, self.iloc[indices, :])
+        else:
+            return cast(DataFrame, self.iloc[:, indices])
+
+    def eval(self, expr, inplace=False) -> Optional[Union["DataFrame", "Series"]]:
+        """
+        Evaluate a string describing operations on DataFrame columns.
+
+        Operates on columns only, not specific rows or elements. This allows
+        `eval` to run arbitrary code, which can make you vulnerable to code
+        injection if you pass user input to this function.
+
+        Parameters
+        ----------
+        expr : str
+            The expression string to evaluate.
+        inplace : bool, default False
+            If the expression contains an assignment, whether to perform the
+            operation inplace and mutate the existing DataFrame. Otherwise,
+            a new DataFrame is returned.
+
+        Returns
+        -------
+        The result of the evaluation.
+
+        See Also
+        --------
+        DataFrame.query : Evaluates a boolean expression to query the columns
+            of a frame.
+        DataFrame.assign : Can evaluate an expression or function to create new
+            values for a column.
+        eval : Evaluate a Python expression as a string using various
+            backends.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'A': range(1, 6), 'B': range(10, 0, -2)})
+        >>> df
+           A   B
+        0  1  10
+        1  2   8
+        2  3   6
+        3  4   4
+        4  5   2
+        >>> df.eval('A + B')
+        0    11
+        1    10
+        2     9
+        3     8
+        4     7
+        dtype: int64
+
+        Assignment is allowed though by default the original DataFrame is not
+        modified.
+
+        >>> df.eval('C = A + B')
+           A   B   C
+        0  1  10  11
+        1  2   8  10
+        2  3   6   9
+        3  4   4   8
+        4  5   2   7
+        >>> df
+           A   B
+        0  1  10
+        1  2   8
+        2  3   6
+        3  4   4
+        4  5   2
+
+        Use ``inplace=True`` to modify the original DataFrame.
+
+        >>> df.eval('C = A + B', inplace=True)
+        >>> df
+           A   B   C
+        0  1  10  11
+        1  2   8  10
+        2  3   6   9
+        3  4   4   8
+        4  5   2   7
+        """
+        from pyspark.pandas.series import first_series
+
+        if isinstance(self.columns, pd.MultiIndex):
+            raise TypeError("`eval` is not supported for multi-index columns")
+        inplace = validate_bool_kwarg(inplace, "inplace")
+        should_return_series = False
+        series_name = None
+        should_return_scalar = False
+
+        # Since `eval_func` doesn't have a type hint, inferring the schema is always preformed
+        # in the `apply_batch`. Hence, the variables `should_return_series`, `series_name`,
+        # and `should_return_scalar` can be updated.
+        def eval_func(pdf):
+            nonlocal should_return_series
+            nonlocal series_name
+            nonlocal should_return_scalar
+            result_inner = pdf.eval(expr, inplace=inplace)
+            if inplace:
+                result_inner = pdf
+            if isinstance(result_inner, pd.Series):
+                should_return_series = True
+                series_name = result_inner.name
+                result_inner = result_inner.to_frame()
+            elif is_scalar(result_inner):
+                should_return_scalar = True
+                result_inner = pd.Series(result_inner).to_frame()
+            return result_inner
+
+        result = self.koalas.apply_batch(eval_func)
+        if inplace:
+            # Here, the result is always a frame because the error is thrown during schema inference
+            # from pandas.
+            self._update_internal_frame(result._internal, requires_same_anchor=False)
+            return None
+        elif should_return_series:
+            return first_series(result).rename(series_name)
+        elif should_return_scalar:
+            return first_series(result)[0]
+        else:
+            # Returns a frame
+            return result
+
+    def explode(self, column) -> "DataFrame":
+        """
+        Transform each element of a list-like to a row, replicating index values.
+
+        Parameters
+        ----------
+        column : str or tuple
+            Column to explode.
+
+        Returns
+        -------
+        DataFrame
+            Exploded lists to rows of the subset columns;
+            index will be duplicated for these rows.
+
+        See Also
+        --------
+        DataFrame.unstack : Pivot a level of the (necessarily hierarchical)
+            index labels.
+        DataFrame.melt : Unpivot a DataFrame from wide format to long format.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'A': [[1, 2, 3], [], [3, 4]], 'B': 1})
+        >>> df
+                   A  B
+        0  [1, 2, 3]  1
+        1         []  1
+        2     [3, 4]  1
+
+        >>> df.explode('A')
+             A  B
+        0  1.0  1
+        0  2.0  1
+        0  3.0  1
+        1  NaN  1
+        2  3.0  1
+        2  4.0  1
+        """
+        from pyspark.pandas.series import Series
+
+        if not is_name_like_value(column):
+            raise TypeError("column must be a scalar")
+
+        kdf = DataFrame(self._internal.resolved_copy)  # type: "DataFrame"
+        kser = kdf[column]
+        if not isinstance(kser, Series):
+            raise ValueError(
+                "The column %s is not unique. For a multi-index, the label must be a tuple "
+                "with elements corresponding to each level." % name_like_string(column)
+            )
+        if not isinstance(kser.spark.data_type, ArrayType):
+            return self.copy()
+
+        sdf = kdf._internal.spark_frame.withColumn(
+            kser._internal.data_spark_column_names[0], F.explode_outer(kser.spark.column)
+        )
+
+        data_dtypes = kdf._internal.data_dtypes.copy()
+        data_dtypes[kdf._internal.column_labels.index(kser._column_label)] = None  # TODO: dtype?
+
+        internal = kdf._internal.with_new_sdf(sdf, data_dtypes=data_dtypes)
+        return DataFrame(internal)
+
+    def mad(self, axis=0) -> "Series":
+        """
+        Return the mean absolute deviation of values.
+
+        Parameters
+        ----------
+        axis : {index (0), columns (1)}
+            Axis for the function to be applied on.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'a': [1, 2, 3, np.nan], 'b': [0.1, 0.2, 0.3, np.nan]},
+        ...                   columns=['a', 'b'])
+
+        >>> df.mad()
+        a    0.666667
+        b    0.066667
+        dtype: float64
+
+        >>> df.mad(axis=1)
+        0    0.45
+        1    0.90
+        2    1.35
+        3     NaN
+        dtype: float64
+        """
+        from pyspark.pandas.series import first_series
+
+        axis = validate_axis(axis)
+
+        if axis == 0:
+
+            def get_spark_column(kdf, label):
+                scol = kdf._internal.spark_column_for(label)
+                col_type = kdf._internal.spark_type_for(label)
+
+                if isinstance(col_type, BooleanType):
+                    scol = scol.cast("integer")
+
+                return scol
+
+            new_column_labels = []
+            for label in self._internal.column_labels:
+                # Filtering out only columns of numeric and boolean type column.
+                dtype = self._kser_for(label).spark.data_type
+                if isinstance(dtype, (NumericType, BooleanType)):
+                    new_column_labels.append(label)
+
+            new_columns = [
+                F.avg(get_spark_column(self, label)).alias(name_like_string(label))
+                for label in new_column_labels
+            ]
+
+            mean_data = self._internal.spark_frame.select(new_columns).first()
+
+            new_columns = [
+                F.avg(
+                    F.abs(get_spark_column(self, label) - mean_data[name_like_string(label)])
+                ).alias(name_like_string(label))
+                for label in new_column_labels
+            ]
+
+            sdf = self._internal.spark_frame.select(
+                [F.lit(None).cast(StringType()).alias(SPARK_DEFAULT_INDEX_NAME)] + new_columns
+            )
+
+            # The data is expected to be small so it's fine to transpose/use default index.
+            with ps.option_context("compute.max_rows", 1):
+                internal = InternalFrame(
+                    spark_frame=sdf,
+                    index_spark_columns=[scol_for(sdf, SPARK_DEFAULT_INDEX_NAME)],
+                    column_labels=new_column_labels,
+                    column_label_names=self._internal.column_label_names,
+                )
+                return first_series(DataFrame(internal).transpose())
+
+        else:
+
+            @pandas_udf(returnType=DoubleType())
+            def calculate_columns_axis(*cols):
+                return pd.concat(cols, axis=1).mad(axis=1)
+
+            internal = self._internal.copy(
+                column_labels=[None],
+                data_spark_columns=[
+                    calculate_columns_axis(*self._internal.data_spark_columns).alias(
+                        SPARK_DEFAULT_SERIES_NAME
+                    )
+                ],
+                data_dtypes=[None],
+                column_label_names=None,
+            )
+            return first_series(DataFrame(internal))
+
+    def tail(self, n=5) -> "DataFrame":
+        """
+        Return the last `n` rows.
+
+        This function returns last `n` rows from the object based on
+        position. It is useful for quickly verifying data, for example,
+        after sorting or appending rows.
+
+        For negative values of `n`, this function returns all rows except
+        the first `n` rows, equivalent to ``df[n:]``.
+
+        Parameters
+        ----------
+        n : int, default 5
+            Number of rows to select.
+
+        Returns
+        -------
+        type of caller
+            The last `n` rows of the caller object.
+
+        See Also
+        --------
+        DataFrame.head : The first `n` rows of the caller object.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'animal': ['alligator', 'bee', 'falcon', 'lion',
+        ...                    'monkey', 'parrot', 'shark', 'whale', 'zebra']})
+        >>> df
+              animal
+        0  alligator
+        1        bee
+        2     falcon
+        3       lion
+        4     monkey
+        5     parrot
+        6      shark
+        7      whale
+        8      zebra
+
+        Viewing the last 5 lines
+
+        >>> df.tail()  # doctest: +SKIP
+           animal
+        4  monkey
+        5  parrot
+        6   shark
+        7   whale
+        8   zebra
+
+        Viewing the last `n` lines (three in this case)
+
+        >>> df.tail(3)  # doctest: +SKIP
+          animal
+        6  shark
+        7  whale
+        8  zebra
+
+        For negative values of `n`
+
+        >>> df.tail(-3)  # doctest: +SKIP
+           animal
+        3    lion
+        4  monkey
+        5  parrot
+        6   shark
+        7   whale
+        8   zebra
+        """
+        if not isinstance(n, int):
+            raise TypeError("bad operand type for unary -: '{}'".format(type(n).__name__))
+        if n < 0:
+            n = len(self) + n
+        if n <= 0:
+            return ps.DataFrame(self._internal.with_filter(F.lit(False)))
+        # Should use `resolved_copy` here for the case like `(kdf + 1).tail()`
+        sdf = self._internal.resolved_copy.spark_frame
+        rows = sdf.tail(n)
+        new_sdf = default_session().createDataFrame(rows, sdf.schema)
+
+        return DataFrame(self._internal.with_new_sdf(new_sdf))
+
+    def align(
+        self,
+        other: Union["DataFrame", "Series"],
+        join: str = "outer",
+        axis: Optional[Union[int, str]] = None,
+        copy: bool = True,
+    ) -> Tuple["DataFrame", Union["DataFrame", "Series"]]:
+        """
+        Align two objects on their axes with the specified join method.
+
+        Join method is specified for each axis Index.
+
+        Parameters
+        ----------
+        other : DataFrame or Series
+        join : {{'outer', 'inner', 'left', 'right'}}, default 'outer'
+        axis : allowed axis of the other object, default None
+            Align on index (0), columns (1), or both (None).
+        copy : bool, default True
+            Always returns new objects. If copy=False and no reindexing is
+            required then original objects are returned.
+
+        Returns
+        -------
+        (left, right) : (DataFrame, type of other)
+            Aligned objects.
+
+        Examples
+        --------
+        >>> ps.set_option("compute.ops_on_diff_frames", True)
+        >>> df1 = ps.DataFrame({"a": [1, 2, 3], "b": ["a", "b", "c"]}, index=[10, 20, 30])
+        >>> df2 = ps.DataFrame({"a": [4, 5, 6], "c": ["d", "e", "f"]}, index=[10, 11, 12])
+
+        Align both axis:
+
+        >>> aligned_l, aligned_r = df1.align(df2)
+        >>> aligned_l.sort_index()
+              a     b   c
+        10  1.0     a NaN
+        11  NaN  None NaN
+        12  NaN  None NaN
+        20  2.0     b NaN
+        30  3.0     c NaN
+        >>> aligned_r.sort_index()
+              a   b     c
+        10  4.0 NaN     d
+        11  5.0 NaN     e
+        12  6.0 NaN     f
+        20  NaN NaN  None
+        30  NaN NaN  None
+
+        Align only axis=0 (index):
+
+        >>> aligned_l, aligned_r = df1.align(df2, axis=0)
+        >>> aligned_l.sort_index()
+              a     b
+        10  1.0     a
+        11  NaN  None
+        12  NaN  None
+        20  2.0     b
+        30  3.0     c
+        >>> aligned_r.sort_index()
+              a     c
+        10  4.0     d
+        11  5.0     e
+        12  6.0     f
+        20  NaN  None
+        30  NaN  None
+
+        Align only axis=1 (column):
+
+        >>> aligned_l, aligned_r = df1.align(df2, axis=1)
+        >>> aligned_l.sort_index()
+            a  b   c
+        10  1  a NaN
+        20  2  b NaN
+        30  3  c NaN
+        >>> aligned_r.sort_index()
+            a   b  c
+        10  4 NaN  d
+        11  5 NaN  e
+        12  6 NaN  f
+
+        Align with the join type "inner":
+
+        >>> aligned_l, aligned_r = df1.align(df2, join="inner")
+        >>> aligned_l.sort_index()
+            a
+        10  1
+        >>> aligned_r.sort_index()
+            a
+        10  4
+
+        Align with a Series:
+
+        >>> s = ps.Series([7, 8, 9], index=[10, 11, 12])
+        >>> aligned_l, aligned_r = df1.align(s, axis=0)
+        >>> aligned_l.sort_index()
+              a     b
+        10  1.0     a
+        11  NaN  None
+        12  NaN  None
+        20  2.0     b
+        30  3.0     c
+        >>> aligned_r.sort_index()
+        10    7.0
+        11    8.0
+        12    9.0
+        20    NaN
+        30    NaN
+        dtype: float64
+
+        >>> ps.reset_option("compute.ops_on_diff_frames")
+        """
+        from pyspark.pandas.series import Series, first_series
+
+        if not isinstance(other, (DataFrame, Series)):
+            raise TypeError("unsupported type: {}".format(type(other).__name__))
+
+        how = validate_how(join)
+        axis = validate_axis(axis, None)
+
+        right_is_series = isinstance(other, Series)
+        if right_is_series:
+            if axis is None:
+                raise ValueError("Must specify axis=0 or 1")
+            elif axis != 0:
+                raise NotImplementedError(
+                    "align currently only works for axis=0 when right is Series"
+                )
+
+        left = self
+        right = other
+
+        if (axis is None or axis == 0) and not same_anchor(left, right):
+            combined = combine_frames(left, right, how=how)
+            left = combined["this"]
+            right = combined["that"]
+
+            if right_is_series:
+                right = first_series(right).rename(other.name)
+
+        if (
+            axis is None or axis == 1
+        ) and left._internal.column_labels != right._internal.column_labels:
+
+            if left._internal.column_labels_level != right._internal.column_labels_level:
+                raise ValueError("cannot join with no overlapping index names")
+
+            left = left.copy()
+            right = right.copy()
+
+            if how == "full":
+                column_labels = sorted(
+                    list(set(left._internal.column_labels) | set(right._internal.column_labels))
+                )
+            elif how == "inner":
+                column_labels = sorted(
+                    list(set(left._internal.column_labels) & set(right._internal.column_labels))
+                )
+            elif how == "left":
+                column_labels = left._internal.column_labels
+            else:
+                column_labels = right._internal.column_labels
+
+            for label in column_labels:
+                if label not in left._internal.column_labels:
+                    left[label] = F.lit(None).cast(DoubleType())
+            left = left[column_labels]
+            for label in column_labels:
+                if label not in right._internal.column_labels:
+                    right[label] = F.lit(None).cast(DoubleType())
+            right = right[column_labels]
+
+        return (left.copy(), right.copy()) if copy else (left, right)
+
+    @staticmethod
+    def from_dict(data, orient="columns", dtype=None, columns=None) -> "DataFrame":
+        """
+        Construct DataFrame from dict of array-like or dicts.
+
+        Creates DataFrame object from dictionary by columns or by index
+        allowing dtype specification.
+
+        Parameters
+        ----------
+        data : dict
+            Of the form {field : array-like} or {field : dict}.
+        orient : {'columns', 'index'}, default 'columns'
+            The "orientation" of the data. If the keys of the passed dict
+            should be the columns of the resulting DataFrame, pass 'columns'
+            (default). Otherwise if the keys should be rows, pass 'index'.
+        dtype : dtype, default None
+            Data type to force, otherwise infer.
+        columns : list, default None
+            Column labels to use when ``orient='index'``. Raises a ValueError
+            if used with ``orient='columns'``.
+
+        Returns
+        -------
+        DataFrame
+
+        See Also
+        --------
+        DataFrame.from_records : DataFrame from structured ndarray, sequence
+            of tuples or dicts, or DataFrame.
+        DataFrame : DataFrame object creation using constructor.
+
+        Examples
+        --------
+        By default the keys of the dict become the DataFrame columns:
+
+        >>> data = {'col_1': [3, 2, 1, 0], 'col_2': [10, 20, 30, 40]}
+        >>> ps.DataFrame.from_dict(data)
+           col_1  col_2
+        0      3     10
+        1      2     20
+        2      1     30
+        3      0     40
+
+        Specify ``orient='index'`` to create the DataFrame using dictionary
+        keys as rows:
+
+        >>> data = {'row_1': [3, 2, 1, 0], 'row_2': [10, 20, 30, 40]}
+        >>> ps.DataFrame.from_dict(data, orient='index').sort_index()
+                0   1   2   3
+        row_1   3   2   1   0
+        row_2  10  20  30  40
+
+        When using the 'index' orientation, the column names can be
+        specified manually:
+
+        >>> ps.DataFrame.from_dict(data, orient='index',
+        ...                        columns=['A', 'B', 'C', 'D']).sort_index()
+                A   B   C   D
+        row_1   3   2   1   0
+        row_2  10  20  30  40
+        """
+        return DataFrame(pd.DataFrame.from_dict(data, orient=orient, dtype=dtype, columns=columns))
+
+    def _to_internal_pandas(self):
+        """
+        Return a pandas DataFrame directly from _internal to avoid overhead of copy.
+
+        This method is for internal use only.
+        """
+        return self._internal.to_pandas_frame
+
+    def _get_or_create_repr_pandas_cache(self, n):
+        if not hasattr(self, "_repr_pandas_cache") or n not in self._repr_pandas_cache:
+            object.__setattr__(
+                self, "_repr_pandas_cache", {n: self.head(n + 1)._to_internal_pandas()}
+            )
+        return self._repr_pandas_cache[n]
+
+    def __repr__(self):
+        max_display_count = get_option("display.max_rows")
+        if max_display_count is None:
+            return self._to_internal_pandas().to_string()
+
+        pdf = self._get_or_create_repr_pandas_cache(max_display_count)
+        pdf_length = len(pdf)
+        pdf = pdf.iloc[:max_display_count]
+        if pdf_length > max_display_count:
+            repr_string = pdf.to_string(show_dimensions=True)
+            match = REPR_PATTERN.search(repr_string)
+            if match is not None:
+                nrows = match.group("rows")
+                ncols = match.group("columns")
+                footer = "\n\n[Showing only the first {nrows} rows x {ncols} columns]".format(
+                    nrows=nrows, ncols=ncols
+                )
+                return REPR_PATTERN.sub(footer, repr_string)
+        return pdf.to_string()
+
+    def _repr_html_(self):
+        max_display_count = get_option("display.max_rows")
+        # pandas 0.25.1 has a regression about HTML representation so 'bold_rows'
+        # has to be set as False explicitly. See https://github.com/pandas-dev/pandas/issues/28204
+        bold_rows = not (LooseVersion("0.25.1") == LooseVersion(pd.__version__))
+        if max_display_count is None:
+            return self._to_internal_pandas().to_html(notebook=True, bold_rows=bold_rows)
+
+        pdf = self._get_or_create_repr_pandas_cache(max_display_count)
+        pdf_length = len(pdf)
+        pdf = pdf.iloc[:max_display_count]
+        if pdf_length > max_display_count:
+            repr_html = pdf.to_html(show_dimensions=True, notebook=True, bold_rows=bold_rows)
+            match = REPR_HTML_PATTERN.search(repr_html)
+            if match is not None:
+                nrows = match.group("rows")
+                ncols = match.group("columns")
+                by = chr(215)
+                footer = (
+                    "\n<p>Showing only the first {rows} rows "
+                    "{by} {cols} columns</p>\n</div>".format(rows=nrows, by=by, cols=ncols)
+                )
+                return REPR_HTML_PATTERN.sub(footer, repr_html)
+        return pdf.to_html(notebook=True, bold_rows=bold_rows)
+
+    def __getitem__(self, key):
+        from pyspark.pandas.series import Series
+
+        if key is None:
+            raise KeyError("none key")
+        elif isinstance(key, Series):
+            return self.loc[key.astype(bool)]
+        elif isinstance(key, slice):
+            if any(type(n) == int or None for n in [key.start, key.stop]):
+                # Seems like pandas Frame always uses int as positional search when slicing
+                # with ints.
+                return self.iloc[key]
+            return self.loc[key]
+        elif is_name_like_value(key):
+            return self.loc[:, key]
+        elif is_list_like(key):
+            return self.loc[:, list(key)]
+        raise NotImplementedError(key)
+
+    def __setitem__(self, key, value):
+        from pyspark.pandas.series import Series
+
+        if isinstance(value, (DataFrame, Series)) and not same_anchor(value, self):
+            # Different Series or DataFrames
+            level = self._internal.column_labels_level
+            key = DataFrame._index_normalized_label(level, key)
+            value = DataFrame._index_normalized_frame(level, value)
+
+            def assign_columns(kdf, this_column_labels, that_column_labels):
+                assert len(key) == len(that_column_labels)
+                # Note that here intentionally uses `zip_longest` that combine
+                # that_columns.
+                for k, this_label, that_label in zip_longest(
+                    key, this_column_labels, that_column_labels
+                ):
+                    yield (kdf._kser_for(that_label), tuple(["that", *k]))
+                    if this_label is not None and this_label[1:] != k:
+                        yield (kdf._kser_for(this_label), this_label)
+
+            kdf = align_diff_frames(assign_columns, self, value, fillna=False, how="left")
+        elif isinstance(value, list):
+            if len(self) != len(value):
+                raise ValueError("Length of values does not match length of index")
+
+            # TODO: avoid using default index?
+            with option_context(
+                "compute.default_index_type",
+                "distributed-sequence",
+                "compute.ops_on_diff_frames",
+                True,
+            ):
+                kdf = self.reset_index()
+                kdf[key] = ps.DataFrame(value)
+                kdf = kdf.set_index(kdf.columns[: self._internal.index_level])
+                kdf.index.names = self.index.names
+
+        elif isinstance(key, list):
+            assert isinstance(value, DataFrame)
+            # Same DataFrames.
+            field_names = value.columns
+            kdf = self._assign({k: value[c] for k, c in zip(key, field_names)})
+        else:
+            # Same Series.
+            kdf = self._assign({key: value})
+
+        self._update_internal_frame(kdf._internal)
+
+    @staticmethod
+    def _index_normalized_label(level, labels):
+        """
+        Returns a label that is normalized against the current column index level.
+        For example, the key "abc" can be ("abc", "", "") if the current Frame has
+        a multi-index for its column
+        """
+        if is_name_like_tuple(labels):
+            labels = [labels]
+        elif is_name_like_value(labels):
+            labels = [(labels,)]
+        else:
+            labels = [k if is_name_like_tuple(k) else (k,) for k in labels]
+
+        if any(len(label) > level for label in labels):
+            raise KeyError(
+                "Key length ({}) exceeds index depth ({})".format(
+                    max(len(label) for label in labels), level
+                )
+            )
+        return [tuple(list(label) + ([""] * (level - len(label)))) for label in labels]
+
+    @staticmethod
+    def _index_normalized_frame(level, kser_or_kdf):
+        """
+        Returns a frame that is normalized against the current column index level.
+        For example, the name in `pd.Series([...], name="abc")` can be can be
+        ("abc", "", "") if the current DataFrame has a multi-index for its column
+        """
+        from pyspark.pandas.series import Series
+
+        if isinstance(kser_or_kdf, Series):
+            kdf = kser_or_kdf.to_frame()
+        else:
+            assert isinstance(kser_or_kdf, DataFrame), type(kser_or_kdf)
+            kdf = kser_or_kdf.copy()
+
+        kdf.columns = pd.MultiIndex.from_tuples(
+            [
+                tuple([name_like_string(label)] + ([""] * (level - 1)))
+                for label in kdf._internal.column_labels
+            ],
+        )
+
+        return kdf
+
+    def __getattr__(self, key: str) -> Any:
+        if key.startswith("__"):
+            raise AttributeError(key)
+        if hasattr(_MissingPandasLikeDataFrame, key):
+            property_or_func = getattr(_MissingPandasLikeDataFrame, key)
+            if isinstance(property_or_func, property):
+                return property_or_func.fget(self)  # type: ignore
+            else:
+                return partial(property_or_func, self)
+
+        try:
+            return self.loc[:, key]
+        except KeyError:
+            raise AttributeError(
+                "'%s' object has no attribute '%s'" % (self.__class__.__name__, key)
+            )
+
+    def __setattr__(self, key: str, value) -> None:
+        try:
+            object.__getattribute__(self, key)
+            return object.__setattr__(self, key, value)
+        except AttributeError:
+            pass
+
+        if (key,) in self._internal.column_labels:
+            self[key] = value
+        else:
+            msg = "pandas-on-Spark doesn't allow columns to be created via a new attribute name"
+            if is_testing():
+                raise AssertionError(msg)
+            else:
+                warnings.warn(msg, UserWarning)
+
+    def __len__(self):
+        return self._internal.resolved_copy.spark_frame.count()
+
+    def __dir__(self):
+        fields = [
+            f for f in self._internal.resolved_copy.spark_frame.schema.fieldNames() if " " not in f
+        ]
+        return super().__dir__() + fields
+
+    def __iter__(self):
+        return iter(self.columns)
+
+    # NDArray Compat
+    def __array_ufunc__(self, ufunc: Callable, method: str, *inputs: Any, **kwargs: Any):
+        # TODO: is it possible to deduplicate it with '_map_series_op'?
+        if all(isinstance(inp, DataFrame) for inp in inputs) and any(
+            not same_anchor(inp, inputs[0]) for inp in inputs
+        ):
+            # binary only
+            assert len(inputs) == 2
+            this = inputs[0]
+            that = inputs[1]
+            if this._internal.column_labels_level != that._internal.column_labels_level:
+                raise ValueError("cannot join with no overlapping index names")
+
+            # Different DataFrames
+            def apply_op(kdf, this_column_labels, that_column_labels):
+                for this_label, that_label in zip(this_column_labels, that_column_labels):
+                    yield (
+                        ufunc(
+                            kdf._kser_for(this_label), kdf._kser_for(that_label), **kwargs
+                        ).rename(this_label),
+                        this_label,
+                    )
+
+            return align_diff_frames(apply_op, this, that, fillna=True, how="full")
+        else:
+            # DataFrame and Series
+            applied = []
+            this = inputs[0]
+            assert all(inp is this for inp in inputs if isinstance(inp, DataFrame))
+
+            for label in this._internal.column_labels:
+                arguments = []
+                for inp in inputs:
+                    arguments.append(inp[label] if isinstance(inp, DataFrame) else inp)
+                # both binary and unary.
+                applied.append(ufunc(*arguments, **kwargs).rename(label))
+
+            internal = this._internal.with_new_columns(applied)
+            return DataFrame(internal)
+
+    if sys.version_info >= (3, 7):
+
+        def __class_getitem__(cls, params):
+            # This is a workaround to support variadic generic in DataFrame in Python 3.7.
+            # See https://github.com/python/typing/issues/193
+            # we always wraps the given type hints by a tuple to mimic the variadic generic.
+            return _create_tuple_for_frame_type(params)
+
+    elif (3, 5) <= sys.version_info < (3, 7):
+        # This is a workaround to support variadic generic in DataFrame in Python 3.5+
+        # The implementation is in its metaclass so this flag is needed to distinguish
+        # pandas-on-Spark DataFrame.
+        is_dataframe = None
+
+
+def _reduce_spark_multi(sdf, aggs):
+    """
+    Performs a reduction on a spark DataFrame, the functions being known sql aggregate functions.
+    """
+    assert isinstance(sdf, spark.DataFrame)
+    sdf0 = sdf.agg(*aggs)
+    l = sdf0.limit(2).toPandas()
+    assert len(l) == 1, (sdf, l)
+    row = l.iloc[0]
+    l2 = list(row)
+    assert len(l2) == len(aggs), (row, l2)
+    return l2
+
+
+class CachedDataFrame(DataFrame):
+    """
+    Cached pandas-on-Spark DataFrame, which corresponds to pandas DataFrame logically, but
+    internally it caches the corresponding Spark DataFrame.
+    """
+
+    def __init__(self, internal, storage_level=None):
+        if storage_level is None:
+            object.__setattr__(self, "_cached", internal.spark_frame.cache())
+        elif isinstance(storage_level, StorageLevel):
+            object.__setattr__(self, "_cached", internal.spark_frame.persist(storage_level))
+        else:
+            raise TypeError(
+                "Only a valid pyspark.StorageLevel type is acceptable for the `storage_level`"
+            )
+        super().__init__(internal)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exception_type, exception_value, traceback):
+        self.spark.unpersist()
+
+    # create accessor for Spark related methods.
+    spark = CachedAccessor("spark", CachedSparkFrameMethods)
+
+    @property
+    def storage_level(self) -> StorageLevel:
+        warnings.warn(
+            "DataFrame.storage_level is deprecated as of DataFrame.spark.storage_level. "
+            "Please use the API instead.",
+            FutureWarning,
+        )
+        return self.spark.storage_level
+
+    storage_level.__doc__ = CachedSparkFrameMethods.storage_level.__doc__
+
+    def unpersist(self) -> None:
+        warnings.warn(
+            "DataFrame.unpersist is deprecated as of DataFrame.spark.unpersist. "
+            "Please use the API instead.",
+            FutureWarning,
+        )
+        return self.spark.unpersist()
+
+    unpersist.__doc__ = CachedSparkFrameMethods.unpersist.__doc__
+
+
+def _test():
+    import os
+    import doctest
+    import shutil
+    import sys
+    import tempfile
+    import uuid
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.frame
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.frame.__dict__.copy()
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]").appName("pyspark.pandas.frame tests").getOrCreate()
+    )
+
+    db_name = "db%s" % str(uuid.uuid4()).replace("-", "")
+    spark.sql("CREATE DATABASE %s" % db_name)
+    globs["db"] = db_name
+
+    path = tempfile.mkdtemp()
+    globs["path"] = path
+
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.frame,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+
+    shutil.rmtree(path, ignore_errors=True)
+    spark.sql("DROP DATABASE IF EXISTS %s CASCADE" % db_name)
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/generic.py b/python/pyspark/pandas/generic.py
new file mode 100644
index 0000000000000..9eede11a508d2
--- /dev/null
+++ b/python/pyspark/pandas/generic.py
@@ -0,0 +1,3139 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+A base class of DataFrame/Column to behave similar to pandas DataFrame/Series.
+"""
+from abc import ABCMeta, abstractmethod
+from collections import Counter
+from collections.abc import Iterable
+from distutils.version import LooseVersion
+from functools import reduce
+from typing import Any, List, Optional, Tuple, Union, TYPE_CHECKING, cast
+import warnings
+
+import numpy as np  # noqa: F401
+import pandas as pd
+from pandas.api.types import is_list_like
+
+from pyspark.sql import functions as F
+from pyspark.sql.types import (
+    BooleanType,
+    DoubleType,
+    FloatType,
+    IntegralType,
+    LongType,
+    NumericType,
+)
+
+from pyspark import pandas as ps  # For running doctests and reference resolution in PyCharm.
+from pyspark.pandas.indexing import AtIndexer, iAtIndexer, iLocIndexer, LocIndexer
+from pyspark.pandas.internal import InternalFrame
+from pyspark.pandas.typedef import Scalar, spark_type_to_pandas_dtype
+from pyspark.pandas.utils import (
+    is_name_like_tuple,
+    is_name_like_value,
+    name_like_string,
+    scol_for,
+    sql_conf,
+    validate_arguments_and_invoke_function,
+    validate_axis,
+    SPARK_CONF_ARROW_ENABLED,
+)
+from pyspark.pandas.window import Rolling, Expanding
+
+if TYPE_CHECKING:
+    from pyspark.pandas.frame import DataFrame  # noqa: F401 (SPARK-34943)
+    from pyspark.pandas.groupby import DataFrameGroupBy, SeriesGroupBy  # noqa: F401 (SPARK-34943)
+    from pyspark.pandas.series import Series  # noqa: F401 (SPARK-34943)
+
+
+class Frame(object, metaclass=ABCMeta):
+    """
+    The base class for both DataFrame and Series.
+    """
+
+    @abstractmethod
+    def __getitem__(self, key):
+        pass
+
+    @property
+    @abstractmethod
+    def _internal(self) -> InternalFrame:
+        pass
+
+    @abstractmethod
+    def _apply_series_op(self, op, should_resolve: bool = False):
+        pass
+
+    @abstractmethod
+    def _reduce_for_stat_function(self, sfun, name, axis=None, numeric_only=True, **kwargs):
+        pass
+
+    @property
+    @abstractmethod
+    def dtypes(self):
+        pass
+
+    @abstractmethod
+    def to_pandas(self):
+        pass
+
+    @property
+    @abstractmethod
+    def index(self):
+        pass
+
+    @abstractmethod
+    def copy(self):
+        pass
+
+    @abstractmethod
+    def _to_internal_pandas(self):
+        pass
+
+    @abstractmethod
+    def head(self, n: int = 5):
+        pass
+
+    # TODO: add 'axis' parameter
+    def cummin(self, skipna: bool = True) -> Union["Series", "DataFrame"]:
+        """
+        Return cumulative minimum over a DataFrame or Series axis.
+
+        Returns a DataFrame or Series of the same size containing the cumulative minimum.
+
+        .. note:: the current implementation of cummin uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Parameters
+        ----------
+        skipna : boolean, default True
+            Exclude NA/null values. If an entire row/column is NA, the result will be NA.
+
+        Returns
+        -------
+        DataFrame or Series
+
+        See Also
+        --------
+        DataFrame.min : Return the minimum over DataFrame axis.
+        DataFrame.cummax : Return cumulative maximum over DataFrame axis.
+        DataFrame.cummin : Return cumulative minimum over DataFrame axis.
+        DataFrame.cumsum : Return cumulative sum over DataFrame axis.
+        Series.min : Return the minimum over Series axis.
+        Series.cummax : Return cumulative maximum over Series axis.
+        Series.cummin : Return cumulative minimum over Series axis.
+        Series.cumsum : Return cumulative sum over Series axis.
+        Series.cumprod : Return cumulative product over Series axis.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([[2.0, 1.0], [3.0, None], [1.0, 0.0]], columns=list('AB'))
+        >>> df
+             A    B
+        0  2.0  1.0
+        1  3.0  NaN
+        2  1.0  0.0
+
+        By default, iterates over rows and finds the minimum in each column.
+
+        >>> df.cummin()
+             A    B
+        0  2.0  1.0
+        1  2.0  NaN
+        2  1.0  0.0
+
+        It works identically in Series.
+
+        >>> df.A.cummin()
+        0    2.0
+        1    2.0
+        2    1.0
+        Name: A, dtype: float64
+        """
+        return self._apply_series_op(lambda kser: kser._cum(F.min, skipna), should_resolve=True)
+
+    # TODO: add 'axis' parameter
+    def cummax(self, skipna: bool = True) -> Union["Series", "DataFrame"]:
+        """
+        Return cumulative maximum over a DataFrame or Series axis.
+
+        Returns a DataFrame or Series of the same size containing the cumulative maximum.
+
+        .. note:: the current implementation of cummax uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Parameters
+        ----------
+        skipna : boolean, default True
+            Exclude NA/null values. If an entire row/column is NA, the result will be NA.
+
+        Returns
+        -------
+        DataFrame or Series
+
+        See Also
+        --------
+        DataFrame.max : Return the maximum over DataFrame axis.
+        DataFrame.cummax : Return cumulative maximum over DataFrame axis.
+        DataFrame.cummin : Return cumulative minimum over DataFrame axis.
+        DataFrame.cumsum : Return cumulative sum over DataFrame axis.
+        DataFrame.cumprod : Return cumulative product over DataFrame axis.
+        Series.max : Return the maximum over Series axis.
+        Series.cummax : Return cumulative maximum over Series axis.
+        Series.cummin : Return cumulative minimum over Series axis.
+        Series.cumsum : Return cumulative sum over Series axis.
+        Series.cumprod : Return cumulative product over Series axis.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([[2.0, 1.0], [3.0, None], [1.0, 0.0]], columns=list('AB'))
+        >>> df
+             A    B
+        0  2.0  1.0
+        1  3.0  NaN
+        2  1.0  0.0
+
+        By default, iterates over rows and finds the maximum in each column.
+
+        >>> df.cummax()
+             A    B
+        0  2.0  1.0
+        1  3.0  NaN
+        2  3.0  1.0
+
+        It works identically in Series.
+
+        >>> df.B.cummax()
+        0    1.0
+        1    NaN
+        2    1.0
+        Name: B, dtype: float64
+        """
+        return self._apply_series_op(lambda kser: kser._cum(F.max, skipna), should_resolve=True)
+
+    # TODO: add 'axis' parameter
+    def cumsum(self, skipna: bool = True) -> Union["Series", "DataFrame"]:
+        """
+        Return cumulative sum over a DataFrame or Series axis.
+
+        Returns a DataFrame or Series of the same size containing the cumulative sum.
+
+        .. note:: the current implementation of cumsum uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Parameters
+        ----------
+        skipna : boolean, default True
+            Exclude NA/null values. If an entire row/column is NA, the result will be NA.
+
+        Returns
+        -------
+        DataFrame or Series
+
+        See Also
+        --------
+        DataFrame.sum : Return the sum over DataFrame axis.
+        DataFrame.cummax : Return cumulative maximum over DataFrame axis.
+        DataFrame.cummin : Return cumulative minimum over DataFrame axis.
+        DataFrame.cumsum : Return cumulative sum over DataFrame axis.
+        DataFrame.cumprod : Return cumulative product over DataFrame axis.
+        Series.sum : Return the sum over Series axis.
+        Series.cummax : Return cumulative maximum over Series axis.
+        Series.cummin : Return cumulative minimum over Series axis.
+        Series.cumsum : Return cumulative sum over Series axis.
+        Series.cumprod : Return cumulative product over Series axis.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([[2.0, 1.0], [3.0, None], [1.0, 0.0]], columns=list('AB'))
+        >>> df
+             A    B
+        0  2.0  1.0
+        1  3.0  NaN
+        2  1.0  0.0
+
+        By default, iterates over rows and finds the sum in each column.
+
+        >>> df.cumsum()
+             A    B
+        0  2.0  1.0
+        1  5.0  NaN
+        2  6.0  1.0
+
+        It works identically in Series.
+
+        >>> df.A.cumsum()
+        0    2.0
+        1    5.0
+        2    6.0
+        Name: A, dtype: float64
+        """
+        return self._apply_series_op(lambda kser: kser._cumsum(skipna), should_resolve=True)
+
+    # TODO: add 'axis' parameter
+    # TODO: use pandas_udf to support negative values and other options later
+    #  other window except unbounded ones is supported as of Spark 3.0.
+    def cumprod(self, skipna: bool = True) -> Union["Series", "DataFrame"]:
+        """
+        Return cumulative product over a DataFrame or Series axis.
+
+        Returns a DataFrame or Series of the same size containing the cumulative product.
+
+        .. note:: the current implementation of cumprod uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        .. note:: unlike pandas', pandas-on-Spark's emulates cumulative product by
+            ``exp(sum(log(...)))`` trick. Therefore, it only works for positive numbers.
+
+        Parameters
+        ----------
+        skipna : boolean, default True
+            Exclude NA/null values. If an entire row/column is NA, the result will be NA.
+
+        Returns
+        -------
+        DataFrame or Series
+
+        See Also
+        --------
+        DataFrame.cummax : Return cumulative maximum over DataFrame axis.
+        DataFrame.cummin : Return cumulative minimum over DataFrame axis.
+        DataFrame.cumsum : Return cumulative sum over DataFrame axis.
+        DataFrame.cumprod : Return cumulative product over DataFrame axis.
+        Series.cummax : Return cumulative maximum over Series axis.
+        Series.cummin : Return cumulative minimum over Series axis.
+        Series.cumsum : Return cumulative sum over Series axis.
+        Series.cumprod : Return cumulative product over Series axis.
+
+        Raises
+        ------
+        Exception : If the values is equal to or lower than 0.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([[2.0, 1.0], [3.0, None], [4.0, 10.0]], columns=list('AB'))
+        >>> df
+             A     B
+        0  2.0   1.0
+        1  3.0   NaN
+        2  4.0  10.0
+
+        By default, iterates over rows and finds the sum in each column.
+
+        >>> df.cumprod()
+              A     B
+        0   2.0   1.0
+        1   6.0   NaN
+        2  24.0  10.0
+
+        It works identically in Series.
+
+        >>> df.A.cumprod()
+        0     2.0
+        1     6.0
+        2    24.0
+        Name: A, dtype: float64
+        """
+        return self._apply_series_op(lambda kser: kser._cumprod(skipna), should_resolve=True)
+
+    # TODO: Although this has removed pandas >= 1.0.0, but we're keeping this as deprecated
+    # since we're using this for `DataFrame.info` internally.
+    # We can drop it once our minimal pandas version becomes 1.0.0.
+    def get_dtype_counts(self) -> pd.Series:
+        """
+        Return counts of unique dtypes in this object.
+
+        .. deprecated:: 0.14.0
+
+        Returns
+        -------
+        dtype : pd.Series
+            Series with the count of columns with each dtype.
+
+        See Also
+        --------
+        dtypes : Return the dtypes in this object.
+
+        Examples
+        --------
+        >>> a = [['a', 1, 1], ['b', 2, 2], ['c', 3, 3]]
+        >>> df = ps.DataFrame(a, columns=['str', 'int1', 'int2'])
+        >>> df
+          str  int1  int2
+        0   a     1     1
+        1   b     2     2
+        2   c     3     3
+
+        >>> df.get_dtype_counts().sort_values()
+        object    1
+        int64     2
+        dtype: int64
+
+        >>> df.str.get_dtype_counts().sort_values()
+        object    1
+        dtype: int64
+        """
+        warnings.warn(
+            "`get_dtype_counts` has been deprecated and will be "
+            "removed in a future version. For DataFrames use "
+            "`.dtypes.value_counts()",
+            FutureWarning,
+        )
+        if not isinstance(self.dtypes, Iterable):
+            dtypes = [self.dtypes]
+        else:
+            dtypes = list(self.dtypes)
+        return pd.Series(dict(Counter([d.name for d in dtypes])))
+
+    def pipe(self, func, *args, **kwargs) -> Any:
+        r"""
+        Apply func(self, \*args, \*\*kwargs).
+
+        Parameters
+        ----------
+        func : function
+            function to apply to the DataFrame.
+            ``args``, and ``kwargs`` are passed into ``func``.
+            Alternatively a ``(callable, data_keyword)`` tuple where
+            ``data_keyword`` is a string indicating the keyword of
+            ``callable`` that expects the DataFrames.
+        args : iterable, optional
+            positional arguments passed into ``func``.
+        kwargs : mapping, optional
+            a dictionary of keyword arguments passed into ``func``.
+
+        Returns
+        -------
+        object : the return type of ``func``.
+
+        Notes
+        -----
+        Use ``.pipe`` when chaining together functions that expect
+        Series, DataFrames or GroupBy objects. For example, given
+
+        >>> df = ps.DataFrame({'category': ['A', 'A', 'B'],
+        ...                    'col1': [1, 2, 3],
+        ...                    'col2': [4, 5, 6]},
+        ...                   columns=['category', 'col1', 'col2'])
+        >>> def keep_category_a(df):
+        ...     return df[df['category'] == 'A']
+        >>> def add_one(df, column):
+        ...     return df.assign(col3=df[column] + 1)
+        >>> def multiply(df, column1, column2):
+        ...     return df.assign(col4=df[column1] * df[column2])
+
+
+        instead of writing
+
+        >>> multiply(add_one(keep_category_a(df), column="col1"), column1="col2", column2="col3")
+          category  col1  col2  col3  col4
+        0        A     1     4     2     8
+        1        A     2     5     3    15
+
+
+        You can write
+
+        >>> (df.pipe(keep_category_a)
+        ...    .pipe(add_one, column="col1")
+        ...    .pipe(multiply, column1="col2", column2="col3")
+        ... )
+          category  col1  col2  col3  col4
+        0        A     1     4     2     8
+        1        A     2     5     3    15
+
+
+        If you have a function that takes the data as (say) the second
+        argument, pass a tuple indicating which keyword expects the
+        data. For example, suppose ``f`` takes its data as ``df``:
+
+        >>> def multiply_2(column1, df, column2):
+        ...     return df.assign(col4=df[column1] * df[column2])
+
+
+        Then you can write
+
+        >>> (df.pipe(keep_category_a)
+        ...    .pipe(add_one, column="col1")
+        ...    .pipe((multiply_2, 'df'), column1="col2", column2="col3")
+        ... )
+          category  col1  col2  col3  col4
+        0        A     1     4     2     8
+        1        A     2     5     3    15
+
+        You can use lambda as wel
+
+        >>> ps.Series([1, 2, 3]).pipe(lambda x: (x + 1).rename("value"))
+        0    2
+        1    3
+        2    4
+        Name: value, dtype: int64
+        """
+
+        if isinstance(func, tuple):
+            func, target = func
+            if target in kwargs:
+                raise ValueError("%s is both the pipe target and a keyword " "argument" % target)
+            kwargs[target] = self
+            return func(*args, **kwargs)
+        else:
+            return func(self, *args, **kwargs)
+
+    def to_numpy(self) -> np.ndarray:
+        """
+        A NumPy ndarray representing the values in this DataFrame or Series.
+
+        .. note:: This method should only be used if the resulting NumPy ndarray is expected
+            to be small, as all the data is loaded into the driver's memory.
+
+        Returns
+        -------
+        numpy.ndarray
+
+        Examples
+        --------
+        >>> ps.DataFrame({"A": [1, 2], "B": [3, 4]}).to_numpy()
+        array([[1, 3],
+               [2, 4]])
+
+        With heterogeneous data, the lowest common type will have to be used.
+
+        >>> ps.DataFrame({"A": [1, 2], "B": [3.0, 4.5]}).to_numpy()
+        array([[1. , 3. ],
+               [2. , 4.5]])
+
+        For a mix of numeric and non-numeric types, the output array will have object dtype.
+
+        >>> df = ps.DataFrame({"A": [1, 2], "B": [3.0, 4.5], "C": pd.date_range('2000', periods=2)})
+        >>> df.to_numpy()
+        array([[1, 3.0, Timestamp('2000-01-01 00:00:00')],
+               [2, 4.5, Timestamp('2000-01-02 00:00:00')]], dtype=object)
+
+        For Series,
+
+        >>> ps.Series(['a', 'b', 'a']).to_numpy()
+        array(['a', 'b', 'a'], dtype=object)
+        """
+        return self.to_pandas().values
+
+    @property
+    def values(self) -> np.ndarray:
+        """
+        Return a Numpy representation of the DataFrame or the Series.
+
+        .. warning:: We recommend using `DataFrame.to_numpy()` or `Series.to_numpy()` instead.
+
+        .. note:: This method should only be used if the resulting NumPy ndarray is expected
+            to be small, as all the data is loaded into the driver's memory.
+
+        Returns
+        -------
+        numpy.ndarray
+
+        Examples
+        --------
+        A DataFrame where all columns are the same type (e.g., int64) results in an array of
+        the same type.
+
+        >>> df = ps.DataFrame({'age':    [ 3,  29],
+        ...                    'height': [94, 170],
+        ...                    'weight': [31, 115]})
+        >>> df
+           age  height  weight
+        0    3      94      31
+        1   29     170     115
+        >>> df.dtypes
+        age       int64
+        height    int64
+        weight    int64
+        dtype: object
+        >>> df.values
+        array([[  3,  94,  31],
+               [ 29, 170, 115]])
+
+        A DataFrame with mixed type columns(e.g., str/object, int64, float32) results in an ndarray
+        of the broadest type that accommodates these mixed types (e.g., object).
+
+        >>> df2 = ps.DataFrame([('parrot',   24.0, 'second'),
+        ...                     ('lion',     80.5, 'first'),
+        ...                     ('monkey', np.nan, None)],
+        ...                   columns=('name', 'max_speed', 'rank'))
+        >>> df2.dtypes
+        name          object
+        max_speed    float64
+        rank          object
+        dtype: object
+        >>> df2.values
+        array([['parrot', 24.0, 'second'],
+               ['lion', 80.5, 'first'],
+               ['monkey', nan, None]], dtype=object)
+
+        For Series,
+
+        >>> ps.Series([1, 2, 3]).values
+        array([1, 2, 3])
+
+        >>> ps.Series(list('aabc')).values
+        array(['a', 'a', 'b', 'c'], dtype=object)
+        """
+        warnings.warn("We recommend using `{}.to_numpy()` instead.".format(type(self).__name__))
+        return self.to_numpy()
+
+    def to_csv(
+        self,
+        path=None,
+        sep=",",
+        na_rep="",
+        columns=None,
+        header=True,
+        quotechar='"',
+        date_format=None,
+        escapechar=None,
+        num_files=None,
+        mode: str = "overwrite",
+        partition_cols: Optional[Union[str, List[str]]] = None,
+        index_col: Optional[Union[str, List[str]]] = None,
+        **options
+    ) -> Optional[str]:
+        r"""
+        Write object to a comma-separated values (csv) file.
+
+        .. note:: pandas-on-Spark `to_csv` writes files to a path or URI. Unlike pandas',
+            pandas-on-Spark respects HDFS's property such as 'fs.default.name'.
+
+        .. note:: pandas-on-Spark writes CSV files into the directory, `path`, and writes
+            multiple `part-...` files in the directory when `path` is specified.
+            This behaviour was inherited from Apache Spark. The number of files can
+            be controlled by `num_files`.
+
+        Parameters
+        ----------
+        path : str, default None
+            File path. If None is provided the result is returned as a string.
+        sep : str, default ','
+            String of length 1. Field delimiter for the output file.
+        na_rep : str, default ''
+            Missing data representation.
+        columns : sequence, optional
+            Columns to write.
+        header : bool or list of str, default True
+            Write out the column names. If a list of strings is given it is
+            assumed to be aliases for the column names.
+        quotechar : str, default '\"'
+            String of length 1. Character used to quote fields.
+        date_format : str, default None
+            Format string for datetime objects.
+        escapechar : str, default None
+            String of length 1. Character used to escape `sep` and `quotechar`
+            when appropriate.
+        num_files : the number of files to be written in `path` directory when
+            this is a path.
+        mode : str {'append', 'overwrite', 'ignore', 'error', 'errorifexists'},
+            default 'overwrite'. Specifies the behavior of the save operation when the
+            destination exists already.
+
+            - 'append': Append the new data to existing data.
+            - 'overwrite': Overwrite existing data.
+            - 'ignore': Silently ignore this operation if data already exists.
+            - 'error' or 'errorifexists': Throw an exception if data already exists.
+
+        partition_cols : str or list of str, optional, default None
+            Names of partitioning columns
+        index_col: str or list of str, optional, default: None
+            Column names to be used in Spark to represent pandas-on-Spark's index. The index name
+            in pandas-on-Spark is ignored. By default, the index is always lost.
+        options: keyword arguments for additional options specific to PySpark.
+            This kwargs are specific to PySpark's CSV options to pass. Check
+            the options in PySpark's API documentation for spark.write.csv(...).
+            It has higher priority and overwrites all other options.
+            This parameter only works when `path` is specified.
+
+        Returns
+        -------
+        str or None
+
+        See Also
+        --------
+        read_csv
+        DataFrame.to_delta
+        DataFrame.to_table
+        DataFrame.to_parquet
+        DataFrame.to_spark_io
+
+        Examples
+        --------
+        >>> df = ps.DataFrame(dict(
+        ...    date=list(pd.date_range('2012-1-1 12:00:00', periods=3, freq='M')),
+        ...    country=['KR', 'US', 'JP'],
+        ...    code=[1, 2 ,3]), columns=['date', 'country', 'code'])
+        >>> df.sort_values(by="date")  # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
+                           date country  code
+        ... 2012-01-31 12:00:00      KR     1
+        ... 2012-02-29 12:00:00      US     2
+        ... 2012-03-31 12:00:00      JP     3
+
+        >>> print(df.to_csv())  # doctest: +NORMALIZE_WHITESPACE
+        date,country,code
+        2012-01-31 12:00:00,KR,1
+        2012-02-29 12:00:00,US,2
+        2012-03-31 12:00:00,JP,3
+
+        >>> df.cummax().to_csv(path=r'%s/to_csv/foo.csv' % path, num_files=1)
+        >>> ps.read_csv(
+        ...    path=r'%s/to_csv/foo.csv' % path
+        ... ).sort_values(by="date")  # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
+                           date country  code
+        ... 2012-01-31 12:00:00      KR     1
+        ... 2012-02-29 12:00:00      US     2
+        ... 2012-03-31 12:00:00      US     3
+
+        In case of Series,
+
+        >>> print(df.date.to_csv())  # doctest: +NORMALIZE_WHITESPACE
+        date
+        2012-01-31 12:00:00
+        2012-02-29 12:00:00
+        2012-03-31 12:00:00
+
+        >>> df.date.to_csv(path=r'%s/to_csv/foo.csv' % path, num_files=1)
+        >>> ps.read_csv(
+        ...     path=r'%s/to_csv/foo.csv' % path
+        ... ).sort_values(by="date")  # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
+                           date
+        ... 2012-01-31 12:00:00
+        ... 2012-02-29 12:00:00
+        ... 2012-03-31 12:00:00
+
+        You can preserve the index in the roundtrip as below.
+
+        >>> df.set_index("country", append=True, inplace=True)
+        >>> df.date.to_csv(
+        ...     path=r'%s/to_csv/bar.csv' % path,
+        ...     num_files=1,
+        ...     index_col=["index1", "index2"])
+        >>> ps.read_csv(
+        ...     path=r'%s/to_csv/bar.csv' % path, index_col=["index1", "index2"]
+        ... ).sort_values(by="date")  # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
+                                     date
+        index1 index2
+        ...    ...    2012-01-31 12:00:00
+        ...    ...    2012-02-29 12:00:00
+        ...    ...    2012-03-31 12:00:00
+        """
+        if "options" in options and isinstance(options.get("options"), dict) and len(options) == 1:
+            options = options.get("options")  # type: ignore
+
+        if path is None:
+            # If path is none, just collect and use pandas's to_csv.
+            kdf_or_ser = self
+            if (LooseVersion("0.24") > LooseVersion(pd.__version__)) and isinstance(
+                self, ps.Series
+            ):
+                # 0.23 seems not having 'columns' parameter in Series' to_csv.
+                return kdf_or_ser.to_pandas().to_csv(  # type: ignore
+                    None,
+                    sep=sep,
+                    na_rep=na_rep,
+                    header=header,
+                    date_format=date_format,
+                    index=False,
+                )
+            else:
+                return kdf_or_ser.to_pandas().to_csv(  # type: ignore
+                    None,
+                    sep=sep,
+                    na_rep=na_rep,
+                    columns=columns,
+                    header=header,
+                    quotechar=quotechar,
+                    date_format=date_format,
+                    escapechar=escapechar,
+                    index=False,
+                )
+
+        kdf = self
+        if isinstance(self, ps.Series):
+            kdf = self.to_frame()
+
+        if columns is None:
+            column_labels = kdf._internal.column_labels
+        else:
+            column_labels = []
+            for label in columns:
+                if not is_name_like_tuple(label):
+                    label = (label,)
+                if label not in kdf._internal.column_labels:
+                    raise KeyError(name_like_string(label))
+                column_labels.append(label)
+
+        if isinstance(index_col, str):
+            index_cols = [index_col]
+        elif index_col is None:
+            index_cols = []
+        else:
+            index_cols = index_col
+
+        if header is True and kdf._internal.column_labels_level > 1:
+            raise ValueError("to_csv only support one-level index column now")
+        elif isinstance(header, list):
+            sdf = kdf.to_spark(index_col)  # type: ignore
+            sdf = sdf.select(
+                [scol_for(sdf, name_like_string(label)) for label in index_cols]
+                + [
+                    scol_for(sdf, str(i) if label is None else name_like_string(label)).alias(
+                        new_name
+                    )
+                    for i, (label, new_name) in enumerate(zip(column_labels, header))
+                ]
+            )
+            header = True
+        else:
+            sdf = kdf.to_spark(index_col)  # type: ignore
+            sdf = sdf.select(
+                [scol_for(sdf, name_like_string(label)) for label in index_cols]
+                + [
+                    scol_for(sdf, str(i) if label is None else name_like_string(label))
+                    for i, label in enumerate(column_labels)
+                ]
+            )
+
+        if num_files is not None:
+            sdf = sdf.repartition(num_files)
+
+        builder = sdf.write.mode(mode)
+        if partition_cols is not None:
+            builder.partitionBy(partition_cols)
+        builder._set_opts(
+            sep=sep,
+            nullValue=na_rep,
+            header=header,
+            quote=quotechar,
+            dateFormat=date_format,
+            charToEscapeQuoteEscaping=escapechar,
+        )
+        builder.options(**options).format("csv").save(path)
+        return None
+
+    def to_json(
+        self,
+        path=None,
+        compression="uncompressed",
+        num_files=None,
+        mode: str = "overwrite",
+        orient="records",
+        lines=True,
+        partition_cols: Optional[Union[str, List[str]]] = None,
+        index_col: Optional[Union[str, List[str]]] = None,
+        **options
+    ) -> Optional[str]:
+        """
+        Convert the object to a JSON string.
+
+        .. note:: pandas-on-Spark `to_json` writes files to a path or URI. Unlike pandas',
+            pandas-on-Spark respects HDFS's property such as 'fs.default.name'.
+
+        .. note:: pandas-on-Spark writes JSON files into the directory, `path`, and writes
+            multiple `part-...` files in the directory when `path` is specified.
+            This behaviour was inherited from Apache Spark. The number of files can
+            be controlled by `num_files`.
+
+        .. note:: output JSON format is different from pandas'. It always use `orient='records'`
+            for its output. This behaviour might have to change in the near future.
+
+        Note NaN's and None will be converted to null and datetime objects
+        will be converted to UNIX timestamps.
+
+        Parameters
+        ----------
+        path : string, optional
+            File path. If not specified, the result is returned as
+            a string.
+        lines : bool, default True
+            If ‘orient’ is ‘records’ write out line delimited json format.
+            Will throw ValueError if incorrect ‘orient’ since others are not
+            list like. It should be always True for now.
+        orient : str, default 'records'
+             It should be always 'records' for now.
+        compression : {'gzip', 'bz2', 'xz', None}
+            A string representing the compression to use in the output file,
+            only used when the first argument is a filename. By default, the
+            compression is inferred from the filename.
+        num_files : the number of files to be written in `path` directory when
+            this is a path.
+        mode : str {'append', 'overwrite', 'ignore', 'error', 'errorifexists'},
+            default 'overwrite'. Specifies the behavior of the save operation when the
+            destination exists already.
+
+            - 'append': Append the new data to existing data.
+            - 'overwrite': Overwrite existing data.
+            - 'ignore': Silently ignore this operation if data already exists.
+            - 'error' or 'errorifexists': Throw an exception if data already exists.
+
+        partition_cols : str or list of str, optional, default None
+            Names of partitioning columns
+        index_col: str or list of str, optional, default: None
+            Column names to be used in Spark to represent pandas-on-Spark's index. The index name
+            in pandas-on-Spark is ignored. By default, the index is always lost.
+        options: keyword arguments for additional options specific to PySpark.
+            It is specific to PySpark's JSON options to pass. Check
+            the options in PySpark's API documentation for `spark.write.json(...)`.
+            It has a higher priority and overwrites all other options.
+            This parameter only works when `path` is specified.
+
+        Returns
+        --------
+        str or None
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([['a', 'b'], ['c', 'd']],
+        ...                   columns=['col 1', 'col 2'])
+        >>> df.to_json()
+        '[{"col 1":"a","col 2":"b"},{"col 1":"c","col 2":"d"}]'
+
+        >>> df['col 1'].to_json()
+        '[{"col 1":"a"},{"col 1":"c"}]'
+
+        >>> df.to_json(path=r'%s/to_json/foo.json' % path, num_files=1)
+        >>> ps.read_json(
+        ...     path=r'%s/to_json/foo.json' % path
+        ... ).sort_values(by="col 1")
+          col 1 col 2
+        0     a     b
+        1     c     d
+
+        >>> df['col 1'].to_json(path=r'%s/to_json/foo.json' % path, num_files=1, index_col="index")
+        >>> ps.read_json(
+        ...     path=r'%s/to_json/foo.json' % path, index_col="index"
+        ... ).sort_values(by="col 1")  # doctest: +NORMALIZE_WHITESPACE
+              col 1
+        index
+        0         a
+        1         c
+        """
+        if "options" in options and isinstance(options.get("options"), dict) and len(options) == 1:
+            options = options.get("options")  # type: ignore
+
+        if not lines:
+            raise NotImplementedError("lines=False is not implemented yet.")
+
+        if orient != "records":
+            raise NotImplementedError("orient='records' is supported only for now.")
+
+        if path is None:
+            # If path is none, just collect and use pandas's to_json.
+            kdf_or_ser = self
+            pdf = kdf_or_ser.to_pandas()  # type: ignore
+            if isinstance(self, ps.Series):
+                pdf = pdf.to_frame()
+            # To make the format consistent and readable by `read_json`, convert it to pandas' and
+            # use 'records' orient for now.
+            return pdf.to_json(orient="records")
+
+        kdf = self
+        if isinstance(self, ps.Series):
+            kdf = self.to_frame()
+        sdf = kdf.to_spark(index_col=index_col)  # type: ignore
+
+        if num_files is not None:
+            sdf = sdf.repartition(num_files)
+
+        builder = sdf.write.mode(mode)
+        if partition_cols is not None:
+            builder.partitionBy(partition_cols)
+        builder._set_opts(compression=compression)
+        builder.options(**options).format("json").save(path)
+        return None
+
+    def to_excel(
+        self,
+        excel_writer,
+        sheet_name="Sheet1",
+        na_rep="",
+        float_format=None,
+        columns=None,
+        header=True,
+        index=True,
+        index_label=None,
+        startrow=0,
+        startcol=0,
+        engine=None,
+        merge_cells=True,
+        encoding=None,
+        inf_rep="inf",
+        verbose=True,
+        freeze_panes=None,
+    ) -> None:
+        """
+        Write object to an Excel sheet.
+
+        .. note:: This method should only be used if the resulting DataFrame is expected
+                  to be small, as all the data is loaded into the driver's memory.
+
+        To write a single object to an Excel .xlsx file it is only necessary to
+        specify a target file name. To write to multiple sheets it is necessary to
+        create an `ExcelWriter` object with a target file name, and specify a sheet
+        in the file to write to.
+
+        Multiple sheets may be written to by specifying unique `sheet_name`.
+        With all data written to the file it is necessary to save the changes.
+        Note that creating an `ExcelWriter` object with a file name that already
+        exists will result in the contents of the existing file being erased.
+
+        Parameters
+        ----------
+        excel_writer : str or ExcelWriter object
+            File path or existing ExcelWriter.
+        sheet_name : str, default 'Sheet1'
+            Name of sheet which will contain DataFrame.
+        na_rep : str, default ''
+            Missing data representation.
+        float_format : str, optional
+            Format string for floating point numbers. For example
+            ``float_format="%%.2f"`` will format 0.1234 to 0.12.
+        columns : sequence or list of str, optional
+            Columns to write.
+        header : bool or list of str, default True
+            Write out the column names. If a list of string is given it is
+            assumed to be aliases for the column names.
+        index : bool, default True
+            Write row names (index).
+        index_label : str or sequence, optional
+            Column label for index column(s) if desired. If not specified, and
+            `header` and `index` are True, then the index names are used. A
+            sequence should be given if the DataFrame uses MultiIndex.
+        startrow : int, default 0
+            Upper left cell row to dump data frame.
+        startcol : int, default 0
+            Upper left cell column to dump data frame.
+        engine : str, optional
+            Write engine to use, 'openpyxl' or 'xlsxwriter'. You can also set this
+            via the options ``io.excel.xlsx.writer``, ``io.excel.xls.writer``, and
+            ``io.excel.xlsm.writer``.
+        merge_cells : bool, default True
+            Write MultiIndex and Hierarchical Rows as merged cells.
+        encoding : str, optional
+            Encoding of the resulting excel file. Only necessary for xlwt,
+            other writers support unicode natively.
+        inf_rep : str, default 'inf'
+            Representation for infinity (there is no native representation for
+            infinity in Excel).
+        verbose : bool, default True
+            Display more information in the error logs.
+        freeze_panes : tuple of int (length 2), optional
+            Specifies the one-based bottommost row and rightmost column that
+            is to be frozen.
+
+        Notes
+        -----
+        Once a workbook has been saved it is not possible write further data
+        without rewriting the whole workbook.
+
+        See Also
+        --------
+        read_excel : Read Excel file.
+
+        Examples
+        --------
+        Create, write to and save a workbook:
+
+        >>> df1 = ps.DataFrame([['a', 'b'], ['c', 'd']],
+        ...                    index=['row 1', 'row 2'],
+        ...                    columns=['col 1', 'col 2'])
+        >>> df1.to_excel("output.xlsx")  # doctest: +SKIP
+
+        To specify the sheet name:
+
+        >>> df1.to_excel("output.xlsx")  # doctest: +SKIP
+        >>> df1.to_excel("output.xlsx",
+        ...              sheet_name='Sheet_name_1')  # doctest: +SKIP
+
+        If you wish to write to more than one sheet in the workbook, it is
+        necessary to specify an ExcelWriter object:
+
+        >>> with pd.ExcelWriter('output.xlsx') as writer:  # doctest: +SKIP
+        ...      df1.to_excel(writer, sheet_name='Sheet_name_1')
+        ...      df2.to_excel(writer, sheet_name='Sheet_name_2')
+
+        To set the library that is used to write the Excel file,
+        you can pass the `engine` keyword (the default engine is
+        automatically chosen depending on the file extension):
+
+        >>> df1.to_excel('output1.xlsx', engine='xlsxwriter')  # doctest: +SKIP
+        """
+        # Make sure locals() call is at the top of the function so we don't capture local variables.
+        args = locals()
+        kdf = self
+
+        if isinstance(self, ps.DataFrame):
+            f = pd.DataFrame.to_excel
+        elif isinstance(self, ps.Series):
+            f = pd.Series.to_excel
+        else:
+            raise TypeError(
+                "Constructor expects DataFrame or Series; however, " "got [%s]" % (self,)
+            )
+        return validate_arguments_and_invoke_function(
+            kdf._to_internal_pandas(), self.to_excel, f, args
+        )
+
+    def mean(
+        self, axis: Union[int, str] = None, numeric_only: bool = None
+    ) -> Union[Scalar, "Series"]:
+        """
+        Return the mean of the values.
+
+        Parameters
+        ----------
+        axis : {index (0), columns (1)}
+            Axis for the function to be applied on.
+        numeric_only : bool, default None
+            Include only float, int, boolean columns. False is not supported. This parameter
+            is mainly for pandas compatibility.
+
+        Returns
+        -------
+        mean : scalar for a Series, and a Series for a DataFrame.
+
+        Examples
+        --------
+
+        >>> df = ps.DataFrame({'a': [1, 2, 3, np.nan], 'b': [0.1, 0.2, 0.3, np.nan]},
+        ...                   columns=['a', 'b'])
+
+        On a DataFrame:
+
+        >>> df.mean()
+        a    2.0
+        b    0.2
+        dtype: float64
+
+        >>> df.mean(axis=1)
+        0    0.55
+        1    1.10
+        2    1.65
+        3     NaN
+        dtype: float64
+
+        On a Series:
+
+        >>> df['a'].mean()
+        2.0
+        """
+        axis = validate_axis(axis)
+
+        if numeric_only is None and axis == 0:
+            numeric_only = True
+
+        def mean(spark_column, spark_type):
+            if isinstance(spark_type, BooleanType):
+                spark_column = spark_column.cast(LongType())
+            elif not isinstance(spark_type, NumericType):
+                raise TypeError(
+                    "Could not convert {} ({}) to numeric".format(
+                        spark_type_to_pandas_dtype(spark_type), spark_type.simpleString()
+                    )
+                )
+            return F.mean(spark_column)
+
+        return self._reduce_for_stat_function(
+            mean, name="mean", axis=axis, numeric_only=numeric_only
+        )
+
+    def sum(
+        self, axis: Union[int, str] = None, numeric_only: bool = None, min_count: int = 0
+    ) -> Union[Scalar, "Series"]:
+        """
+        Return the sum of the values.
+
+        Parameters
+        ----------
+        axis : {index (0), columns (1)}
+            Axis for the function to be applied on.
+        numeric_only : bool, default None
+            Include only float, int, boolean columns. False is not supported. This parameter
+            is mainly for pandas compatibility.
+        min_count : int, default 0
+            The required number of valid values to perform the operation. If fewer than
+             ``min_count`` non-NA values are present the result will be NA.
+
+        Returns
+        -------
+        sum : scalar for a Series, and a Series for a DataFrame.
+
+        Examples
+        --------
+
+        >>> df = ps.DataFrame({'a': [1, 2, 3, np.nan], 'b': [0.1, np.nan, 0.3, np.nan]},
+        ...                   columns=['a', 'b'])
+
+        On a DataFrame:
+
+        >>> df.sum()
+        a    6.0
+        b    0.4
+        dtype: float64
+
+        >>> df.sum(axis=1)
+        0    1.1
+        1    2.0
+        2    3.3
+        3    0.0
+        dtype: float64
+
+        >>> df.sum(min_count=3)
+        a    6.0
+        b    NaN
+        dtype: float64
+
+        >>> df.sum(axis=1, min_count=1)
+        0    1.1
+        1    2.0
+        2    3.3
+        3    NaN
+        dtype: float64
+
+        On a Series:
+
+        >>> df['a'].sum()
+        6.0
+
+        >>> df['a'].sum(min_count=3)
+        6.0
+        >>> df['b'].sum(min_count=3)
+        nan
+        """
+        axis = validate_axis(axis)
+
+        if numeric_only is None and axis == 0:
+            numeric_only = True
+        elif numeric_only is True and axis == 1:
+            numeric_only = None
+
+        def sum(spark_column, spark_type):
+            if isinstance(spark_type, BooleanType):
+                spark_column = spark_column.cast(LongType())
+            elif not isinstance(spark_type, NumericType):
+                raise TypeError(
+                    "Could not convert {} ({}) to numeric".format(
+                        spark_type_to_pandas_dtype(spark_type), spark_type.simpleString()
+                    )
+                )
+            return F.coalesce(F.sum(spark_column), F.lit(0))
+
+        return self._reduce_for_stat_function(
+            sum, name="sum", axis=axis, numeric_only=numeric_only, min_count=min_count
+        )
+
+    def product(
+        self, axis: Union[int, str] = None, numeric_only: bool = None, min_count: int = 0
+    ) -> Union[Scalar, "Series"]:
+        """
+        Return the product of the values.
+
+        .. note:: unlike pandas', pandas-on-Spark's emulates product by ``exp(sum(log(...)))``
+            trick. Therefore, it only works for positive numbers.
+
+        Parameters
+        ----------
+        axis : {index (0), columns (1)}
+            Axis for the function to be applied on.
+        numeric_only : bool, default None
+            Include only float, int, boolean columns. False is not supported. This parameter
+            is mainly for pandas compatibility.
+        min_count : int, default 0
+            The required number of valid values to perform the operation. If fewer than
+            ``min_count`` non-NA values are present the result will be NA.
+
+        Examples
+        --------
+        On a DataFrame:
+
+        Non-numeric type column is not included to the result.
+
+        >>> kdf = ps.DataFrame({'A': [1, 2, 3, 4, 5],
+        ...                     'B': [10, 20, 30, 40, 50],
+        ...                     'C': ['a', 'b', 'c', 'd', 'e']})
+        >>> kdf
+           A   B  C
+        0  1  10  a
+        1  2  20  b
+        2  3  30  c
+        3  4  40  d
+        4  5  50  e
+
+        >>> kdf.prod()
+        A         120
+        B    12000000
+        dtype: int64
+
+        If there is no numeric type columns, returns empty Series.
+
+        >>> ps.DataFrame({"key": ['a', 'b', 'c'], "val": ['x', 'y', 'z']}).prod()
+        Series([], dtype: float64)
+
+        On a Series:
+
+        >>> ps.Series([1, 2, 3, 4, 5]).prod()
+        120
+
+        By default, the product of an empty or all-NA Series is ``1``
+
+        >>> ps.Series([]).prod()
+        1.0
+
+        This can be controlled with the ``min_count`` parameter
+
+        >>> ps.Series([]).prod(min_count=1)
+        nan
+        """
+        axis = validate_axis(axis)
+
+        if numeric_only is None and axis == 0:
+            numeric_only = True
+        elif numeric_only is True and axis == 1:
+            numeric_only = None
+
+        def prod(spark_column, spark_type):
+            if isinstance(spark_type, BooleanType):
+                scol = F.min(F.coalesce(spark_column, F.lit(True))).cast(LongType())
+            elif isinstance(spark_type, NumericType):
+                num_zeros = F.sum(F.when(spark_column == 0, 1).otherwise(0))
+                sign = F.when(
+                    F.sum(F.when(spark_column < 0, 1).otherwise(0)) % 2 == 0, 1
+                ).otherwise(-1)
+
+                scol = F.when(num_zeros > 0, 0).otherwise(
+                    sign * F.exp(F.sum(F.log(F.abs(spark_column))))
+                )
+
+                if isinstance(spark_type, IntegralType):
+                    scol = F.round(scol).cast(LongType())
+            else:
+                raise TypeError(
+                    "Could not convert {} ({}) to numeric".format(
+                        spark_type_to_pandas_dtype(spark_type), spark_type.simpleString()
+                    )
+                )
+
+            return F.coalesce(scol, F.lit(1))
+
+        return self._reduce_for_stat_function(
+            prod, name="prod", axis=axis, numeric_only=numeric_only, min_count=min_count
+        )
+
+    prod = product
+
+    def skew(
+        self, axis: Union[int, str] = None, numeric_only: bool = None
+    ) -> Union[Scalar, "Series"]:
+        """
+        Return unbiased skew normalized by N-1.
+
+        Parameters
+        ----------
+        axis : {index (0), columns (1)}
+            Axis for the function to be applied on.
+        numeric_only : bool, default None
+            Include only float, int, boolean columns. False is not supported. This parameter
+            is mainly for pandas compatibility.
+
+        Returns
+        -------
+        skew : scalar for a Series, and a Series for a DataFrame.
+
+        Examples
+        --------
+
+        >>> df = ps.DataFrame({'a': [1, 2, 3, np.nan], 'b': [0.1, 0.2, 0.3, np.nan]},
+        ...                   columns=['a', 'b'])
+
+        On a DataFrame:
+
+        >>> df.skew()  # doctest: +SKIP
+        a    0.000000e+00
+        b   -3.319678e-16
+        dtype: float64
+
+        On a Series:
+
+        >>> df['a'].skew()
+        0.0
+        """
+        axis = validate_axis(axis)
+
+        if numeric_only is None and axis == 0:
+            numeric_only = True
+
+        def skew(spark_column, spark_type):
+            if isinstance(spark_type, BooleanType):
+                spark_column = spark_column.cast(LongType())
+            elif not isinstance(spark_type, NumericType):
+                raise TypeError(
+                    "Could not convert {} ({}) to numeric".format(
+                        spark_type_to_pandas_dtype(spark_type), spark_type.simpleString()
+                    )
+                )
+            return F.skewness(spark_column)
+
+        return self._reduce_for_stat_function(
+            skew, name="skew", axis=axis, numeric_only=numeric_only
+        )
+
+    def kurtosis(
+        self, axis: Union[int, str] = None, numeric_only: bool = None
+    ) -> Union[Scalar, "Series"]:
+        """
+        Return unbiased kurtosis using Fisher’s definition of kurtosis (kurtosis of normal == 0.0).
+        Normalized by N-1.
+
+        Parameters
+        ----------
+        axis : {index (0), columns (1)}
+            Axis for the function to be applied on.
+        numeric_only : bool, default None
+            Include only float, int, boolean columns. False is not supported. This parameter
+            is mainly for pandas compatibility.
+
+        Returns
+        -------
+        kurt : scalar for a Series, and a Series for a DataFrame.
+
+        Examples
+        --------
+
+        >>> df = ps.DataFrame({'a': [1, 2, 3, np.nan], 'b': [0.1, 0.2, 0.3, np.nan]},
+        ...                   columns=['a', 'b'])
+
+        On a DataFrame:
+
+        >>> df.kurtosis()
+        a   -1.5
+        b   -1.5
+        dtype: float64
+
+        On a Series:
+
+        >>> df['a'].kurtosis()
+        -1.5
+        """
+        axis = validate_axis(axis)
+
+        if numeric_only is None and axis == 0:
+            numeric_only = True
+
+        def kurtosis(spark_column, spark_type):
+            if isinstance(spark_type, BooleanType):
+                spark_column = spark_column.cast(LongType())
+            elif not isinstance(spark_type, NumericType):
+                raise TypeError(
+                    "Could not convert {} ({}) to numeric".format(
+                        spark_type_to_pandas_dtype(spark_type), spark_type.simpleString()
+                    )
+                )
+            return F.kurtosis(spark_column)
+
+        return self._reduce_for_stat_function(
+            kurtosis, name="kurtosis", axis=axis, numeric_only=numeric_only
+        )
+
+    kurt = kurtosis
+
+    def min(
+        self, axis: Union[int, str] = None, numeric_only: bool = None
+    ) -> Union[Scalar, "Series"]:
+        """
+        Return the minimum of the values.
+
+        Parameters
+        ----------
+        axis : {index (0), columns (1)}
+            Axis for the function to be applied on.
+        numeric_only : bool, default None
+            If True, include only float, int, boolean columns. This parameter is mainly for
+            pandas compatibility. False is supported; however, the columns should
+            be all numeric or all non-numeric.
+
+        Returns
+        -------
+        min : scalar for a Series, and a Series for a DataFrame.
+
+        Examples
+        --------
+
+        >>> df = ps.DataFrame({'a': [1, 2, 3, np.nan], 'b': [0.1, 0.2, 0.3, np.nan]},
+        ...                   columns=['a', 'b'])
+
+        On a DataFrame:
+
+        >>> df.min()
+        a    1.0
+        b    0.1
+        dtype: float64
+
+        >>> df.min(axis=1)
+        0    0.1
+        1    0.2
+        2    0.3
+        3    NaN
+        dtype: float64
+
+        On a Series:
+
+        >>> df['a'].min()
+        1.0
+        """
+        axis = validate_axis(axis)
+
+        if numeric_only is None and axis == 0:
+            numeric_only = True
+        elif numeric_only is True and axis == 1:
+            numeric_only = None
+
+        return self._reduce_for_stat_function(
+            F.min, name="min", axis=axis, numeric_only=numeric_only
+        )
+
+    def max(
+        self, axis: Union[int, str] = None, numeric_only: bool = None
+    ) -> Union[Scalar, "Series"]:
+        """
+        Return the maximum of the values.
+
+        Parameters
+        ----------
+        axis : {index (0), columns (1)}
+            Axis for the function to be applied on.
+        numeric_only : bool, default None
+            If True, include only float, int, boolean columns. This parameter is mainly for
+            pandas compatibility. False is supported; however, the columns should
+            be all numeric or all non-numeric.
+
+        Returns
+        -------
+        max : scalar for a Series, and a Series for a DataFrame.
+
+        Examples
+        --------
+
+        >>> df = ps.DataFrame({'a': [1, 2, 3, np.nan], 'b': [0.1, 0.2, 0.3, np.nan]},
+        ...                   columns=['a', 'b'])
+
+        On a DataFrame:
+
+        >>> df.max()
+        a    3.0
+        b    0.3
+        dtype: float64
+
+        >>> df.max(axis=1)
+        0    1.0
+        1    2.0
+        2    3.0
+        3    NaN
+        dtype: float64
+
+        On a Series:
+
+        >>> df['a'].max()
+        3.0
+        """
+        axis = validate_axis(axis)
+
+        if numeric_only is None and axis == 0:
+            numeric_only = True
+        elif numeric_only is True and axis == 1:
+            numeric_only = None
+
+        return self._reduce_for_stat_function(
+            F.max, name="max", axis=axis, numeric_only=numeric_only
+        )
+
+    def count(
+        self, axis: Union[int, str] = None, numeric_only: bool = False
+    ) -> Union[Scalar, "Series"]:
+        """
+        Count non-NA cells for each column.
+
+        The values `None`, `NaN` are considered NA.
+
+        Parameters
+        ----------
+        axis : {0 or ‘index’, 1 or ‘columns’}, default 0
+            If 0 or ‘index’ counts are generated for each column. If 1 or ‘columns’ counts are
+            generated for each row.
+        numeric_only : bool, default False
+            If True, include only float, int, boolean columns. This parameter is mainly for
+            pandas compatibility.
+
+        Returns
+        -------
+        max : scalar for a Series, and a Series for a DataFrame.
+
+        See Also
+        --------
+        DataFrame.shape: Number of DataFrame rows and columns (including NA
+            elements).
+        DataFrame.isna: Boolean same-sized DataFrame showing places of NA
+            elements.
+
+        Examples
+        --------
+        Constructing DataFrame from a dictionary:
+
+        >>> df = ps.DataFrame({"Person":
+        ...                    ["John", "Myla", "Lewis", "John", "Myla"],
+        ...                    "Age": [24., np.nan, 21., 33, 26],
+        ...                    "Single": [False, True, True, True, False]},
+        ...                   columns=["Person", "Age", "Single"])
+        >>> df
+          Person   Age  Single
+        0   John  24.0   False
+        1   Myla   NaN    True
+        2  Lewis  21.0    True
+        3   John  33.0    True
+        4   Myla  26.0   False
+
+        Notice the uncounted NA values:
+
+        >>> df.count()
+        Person    5
+        Age       4
+        Single    5
+        dtype: int64
+
+        >>> df.count(axis=1)
+        0    3
+        1    2
+        2    3
+        3    3
+        4    3
+        dtype: int64
+
+        On a Series:
+
+        >>> df['Person'].count()
+        5
+
+        >>> df['Age'].count()
+        4
+        """
+
+        return self._reduce_for_stat_function(
+            Frame._count_expr, name="count", axis=axis, numeric_only=numeric_only
+        )
+
+    def std(
+        self, axis: Union[int, str] = None, ddof: int = 1, numeric_only: bool = None
+    ) -> Union[Scalar, "Series"]:
+        """
+        Return sample standard deviation.
+
+        Parameters
+        ----------
+        axis : {index (0), columns (1)}
+            Axis for the function to be applied on.
+        ddof : int, default 1
+            Delta Degrees of Freedom. The divisor used in calculations is N - ddof,
+            where N represents the number of elements.
+        numeric_only : bool, default None
+            Include only float, int, boolean columns. False is not supported. This parameter
+            is mainly for pandas compatibility.
+
+        Returns
+        -------
+        std : scalar for a Series, and a Series for a DataFrame.
+
+        Examples
+        --------
+
+        >>> df = ps.DataFrame({'a': [1, 2, 3, np.nan], 'b': [0.1, 0.2, 0.3, np.nan]},
+        ...                   columns=['a', 'b'])
+
+        On a DataFrame:
+
+        >>> df.std()
+        a    1.0
+        b    0.1
+        dtype: float64
+
+        >>> df.std(axis=1)
+        0    0.636396
+        1    1.272792
+        2    1.909188
+        3         NaN
+        dtype: float64
+
+        >>> df.std(ddof=0)
+        a    0.816497
+        b    0.081650
+        dtype: float64
+
+        On a Series:
+
+        >>> df['a'].std()
+        1.0
+
+        >>> df['a'].std(ddof=0)
+        0.816496580927726
+        """
+        assert ddof in (0, 1)
+
+        axis = validate_axis(axis)
+
+        if numeric_only is None and axis == 0:
+            numeric_only = True
+
+        def std(spark_column, spark_type):
+            if isinstance(spark_type, BooleanType):
+                spark_column = spark_column.cast(LongType())
+            elif not isinstance(spark_type, NumericType):
+                raise TypeError(
+                    "Could not convert {} ({}) to numeric".format(
+                        spark_type_to_pandas_dtype(spark_type), spark_type.simpleString()
+                    )
+                )
+            if ddof == 0:
+                return F.stddev_pop(spark_column)
+            else:
+                return F.stddev_samp(spark_column)
+
+        return self._reduce_for_stat_function(
+            std, name="std", axis=axis, numeric_only=numeric_only, ddof=ddof
+        )
+
+    def var(
+        self, axis: Union[int, str] = None, ddof: int = 1, numeric_only: bool = None
+    ) -> Union[Scalar, "Series"]:
+        """
+        Return unbiased variance.
+
+        Parameters
+        ----------
+        axis : {index (0), columns (1)}
+            Axis for the function to be applied on.
+        ddof : int, default 1
+            Delta Degrees of Freedom. The divisor used in calculations is N - ddof,
+            where N represents the number of elements.
+        numeric_only : bool, default None
+            Include only float, int, boolean columns. False is not supported. This parameter
+            is mainly for pandas compatibility.
+
+        Returns
+        -------
+        var : scalar for a Series, and a Series for a DataFrame.
+
+        Examples
+        --------
+
+        >>> df = ps.DataFrame({'a': [1, 2, 3, np.nan], 'b': [0.1, 0.2, 0.3, np.nan]},
+        ...                   columns=['a', 'b'])
+
+        On a DataFrame:
+
+        >>> df.var()
+        a    1.00
+        b    0.01
+        dtype: float64
+
+        >>> df.var(axis=1)
+        0    0.405
+        1    1.620
+        2    3.645
+        3      NaN
+        dtype: float64
+
+        >>> df.var(ddof=0)
+        a    0.666667
+        b    0.006667
+        dtype: float64
+
+        On a Series:
+
+        >>> df['a'].var()
+        1.0
+
+        >>> df['a'].var(ddof=0)
+        0.6666666666666666
+        """
+        assert ddof in (0, 1)
+
+        axis = validate_axis(axis)
+
+        if numeric_only is None and axis == 0:
+            numeric_only = True
+
+        def var(spark_column, spark_type):
+            if isinstance(spark_type, BooleanType):
+                spark_column = spark_column.cast(LongType())
+            elif not isinstance(spark_type, NumericType):
+                raise TypeError(
+                    "Could not convert {} ({}) to numeric".format(
+                        spark_type_to_pandas_dtype(spark_type), spark_type.simpleString()
+                    )
+                )
+            if ddof == 0:
+                return F.var_pop(spark_column)
+            else:
+                return F.var_samp(spark_column)
+
+        return self._reduce_for_stat_function(
+            var, name="var", axis=axis, numeric_only=numeric_only, ddof=ddof
+        )
+
+    def median(
+        self, axis: Union[int, str] = None, numeric_only: bool = None, accuracy: int = 10000
+    ) -> Union[Scalar, "Series"]:
+        """
+        Return the median of the values for the requested axis.
+
+        .. note:: Unlike pandas', the median in pandas-on-Spark is an approximated median based upon
+            approximate percentile computation because computing median across a large dataset
+            is extremely expensive.
+
+        Parameters
+        ----------
+        axis : {index (0), columns (1)}
+            Axis for the function to be applied on.
+        numeric_only : bool, default None
+            Include only float, int, boolean columns. False is not supported. This parameter
+            is mainly for pandas compatibility.
+        accuracy : int, optional
+            Default accuracy of approximation. Larger value means better accuracy.
+            The relative error can be deduced by 1.0 / accuracy.
+
+        Returns
+        -------
+        median : scalar or Series
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({
+        ...     'a': [24., 21., 25., 33., 26.], 'b': [1, 2, 3, 4, 5]}, columns=['a', 'b'])
+        >>> df
+              a  b
+        0  24.0  1
+        1  21.0  2
+        2  25.0  3
+        3  33.0  4
+        4  26.0  5
+
+        On a DataFrame:
+
+        >>> df.median()
+        a    25.0
+        b     3.0
+        dtype: float64
+
+        On a Series:
+
+        >>> df['a'].median()
+        25.0
+        >>> (df['b'] + 100).median()
+        103.0
+
+        For multi-index columns,
+
+        >>> df.columns = pd.MultiIndex.from_tuples([('x', 'a'), ('y', 'b')])
+        >>> df
+              x  y
+              a  b
+        0  24.0  1
+        1  21.0  2
+        2  25.0  3
+        3  33.0  4
+        4  26.0  5
+
+        On a DataFrame:
+
+        >>> df.median()
+        x  a    25.0
+        y  b     3.0
+        dtype: float64
+
+        >>> df.median(axis=1)
+        0    12.5
+        1    11.5
+        2    14.0
+        3    18.5
+        4    15.5
+        dtype: float64
+
+        On a Series:
+
+        >>> df[('x', 'a')].median()
+        25.0
+        >>> (df[('y', 'b')] + 100).median()
+        103.0
+        """
+        axis = validate_axis(axis)
+
+        if numeric_only is None and axis == 0:
+            numeric_only = True
+
+        if not isinstance(accuracy, int):
+            raise TypeError(
+                "accuracy must be an integer; however, got [%s]" % type(accuracy).__name__
+            )
+
+        def median(spark_column, spark_type):
+            if isinstance(spark_type, (BooleanType, NumericType)):
+                return F.percentile_approx(spark_column.cast(DoubleType()), 0.5, accuracy)
+            else:
+                raise TypeError(
+                    "Could not convert {} ({}) to numeric".format(
+                        spark_type_to_pandas_dtype(spark_type), spark_type.simpleString()
+                    )
+                )
+
+        return self._reduce_for_stat_function(
+            median, name="median", numeric_only=numeric_only, axis=axis
+        )
+
+    def sem(
+        self, axis: Union[int, str] = None, ddof: int = 1, numeric_only: bool = None
+    ) -> Union[Scalar, "Series"]:
+        """
+        Return unbiased standard error of the mean over requested axis.
+
+        Parameters
+        ----------
+        axis : {index (0), columns (1)}
+            Axis for the function to be applied on.
+        ddof : int, default 1
+            Delta Degrees of Freedom. The divisor used in calculations is N - ddof,
+            where N represents the number of elements.
+        numeric_only : bool, default None
+            Include only float, int, boolean columns. False is not supported. This parameter
+            is mainly for pandas compatibility.
+
+        Returns
+        -------
+        scalar(for Series) or Series(for DataFrame)
+
+        Examples
+        --------
+        >>> kdf = ps.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
+        >>> kdf
+           a  b
+        0  1  4
+        1  2  5
+        2  3  6
+
+        >>> kdf.sem()
+        a    0.57735
+        b    0.57735
+        dtype: float64
+
+        >>> kdf.sem(ddof=0)
+        a    0.471405
+        b    0.471405
+        dtype: float64
+
+        >>> kdf.sem(axis=1)
+        0    1.5
+        1    1.5
+        2    1.5
+        dtype: float64
+
+        Support for Series
+
+        >>> kser = kdf.a
+        >>> kser
+        0    1
+        1    2
+        2    3
+        Name: a, dtype: int64
+
+        >>> kser.sem()
+        0.5773502691896258
+
+        >>> kser.sem(ddof=0)
+        0.47140452079103173
+        """
+        assert ddof in (0, 1)
+
+        axis = validate_axis(axis)
+
+        if numeric_only is None and axis == 0:
+            numeric_only = True
+
+        def std(spark_column, spark_type):
+            if isinstance(spark_type, BooleanType):
+                spark_column = spark_column.cast(LongType())
+            elif not isinstance(spark_type, NumericType):
+                raise TypeError(
+                    "Could not convert {} ({}) to numeric".format(
+                        spark_type_to_pandas_dtype(spark_type), spark_type.simpleString()
+                    )
+                )
+            if ddof == 0:
+                return F.stddev_pop(spark_column)
+            else:
+                return F.stddev_samp(spark_column)
+
+        def sem(spark_column, spark_type):
+            return std(spark_column, spark_type) / pow(
+                Frame._count_expr(spark_column, spark_type), 0.5
+            )
+
+        return self._reduce_for_stat_function(
+            sem, name="sem", numeric_only=numeric_only, axis=axis, ddof=ddof
+        )
+
+    @property
+    def size(self) -> int:
+        """
+        Return an int representing the number of elements in this object.
+
+        Return the number of rows if Series. Otherwise return the number of
+        rows times number of columns if DataFrame.
+
+        Examples
+        --------
+        >>> s = ps.Series({'a': 1, 'b': 2, 'c': None})
+        >>> s.size
+        3
+
+        >>> df = ps.DataFrame({'col1': [1, 2, None], 'col2': [3, 4, None]})
+        >>> df.size
+        6
+
+        >>> df = ps.DataFrame(index=[1, 2, None])
+        >>> df.size
+        0
+        """
+        num_columns = len(self._internal.data_spark_columns)
+        if num_columns == 0:
+            return 0
+        else:
+            return len(self) * num_columns  # type: ignore
+
+    def abs(self) -> Union["DataFrame", "Series"]:
+        """
+        Return a Series/DataFrame with absolute numeric value of each element.
+
+        Returns
+        -------
+        abs : Series/DataFrame containing the absolute value of each element.
+
+        Examples
+        --------
+
+        Absolute numeric values in a Series.
+
+        >>> s = ps.Series([-1.10, 2, -3.33, 4])
+        >>> s.abs()
+        0    1.10
+        1    2.00
+        2    3.33
+        3    4.00
+        dtype: float64
+
+        Absolute numeric values in a DataFrame.
+
+        >>> df = ps.DataFrame({
+        ...     'a': [4, 5, 6, 7],
+        ...     'b': [10, 20, 30, 40],
+        ...     'c': [100, 50, -30, -50]
+        ...   },
+        ...   columns=['a', 'b', 'c'])
+        >>> df.abs()
+           a   b    c
+        0  4  10  100
+        1  5  20   50
+        2  6  30   30
+        3  7  40   50
+        """
+
+        def abs(kser):
+            if isinstance(kser.spark.data_type, BooleanType):
+                return kser
+            elif isinstance(kser.spark.data_type, NumericType):
+                return kser.spark.transform(F.abs)
+            else:
+                raise TypeError(
+                    "bad operand type for abs(): {} ({})".format(
+                        spark_type_to_pandas_dtype(kser.spark.data_type),
+                        kser.spark.data_type.simpleString(),
+                    )
+                )
+
+        return self._apply_series_op(abs)
+
+    # TODO: by argument only support the grouping name and as_index only for now. Documentation
+    # should be updated when it's supported.
+    def groupby(
+        self, by, axis=0, as_index: bool = True, dropna: bool = True
+    ) -> Union["DataFrameGroupBy", "SeriesGroupBy"]:
+        """
+        Group DataFrame or Series using a Series of columns.
+
+        A groupby operation involves some combination of splitting the
+        object, applying a function, and combining the results. This can be
+        used to group large amounts of data and compute operations on these
+        groups.
+
+        Parameters
+        ----------
+        by : Series, label, or list of labels
+            Used to determine the groups for the groupby.
+            If Series is passed, the Series or dict VALUES
+            will be used to determine the groups. A label or list of
+            labels may be passed to group by the columns in ``self``.
+        axis : int, default 0 or 'index'
+            Can only be set to 0 at the moment.
+        as_index : bool, default True
+            For aggregated output, return object with group labels as the
+            index. Only relevant for DataFrame input. as_index=False is
+            effectively "SQL-style" grouped output.
+        dropna : bool, default True
+            If True, and if group keys contain NA values,
+            NA values together with row/column will be dropped.
+            If False, NA values will also be treated as the key in groups.
+
+        Returns
+        -------
+        DataFrameGroupBy or SeriesGroupBy
+            Depends on the calling object and returns groupby object that
+            contains information about the groups.
+
+        See Also
+        --------
+        koalas.groupby.GroupBy
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'Animal': ['Falcon', 'Falcon',
+        ...                               'Parrot', 'Parrot'],
+        ...                    'Max Speed': [380., 370., 24., 26.]},
+        ...                   columns=['Animal', 'Max Speed'])
+        >>> df
+           Animal  Max Speed
+        0  Falcon      380.0
+        1  Falcon      370.0
+        2  Parrot       24.0
+        3  Parrot       26.0
+
+        >>> df.groupby(['Animal']).mean().sort_index()  # doctest: +NORMALIZE_WHITESPACE
+                Max Speed
+        Animal
+        Falcon      375.0
+        Parrot       25.0
+
+        >>> df.groupby(['Animal'], as_index=False).mean().sort_values('Animal')
+        ... # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
+           Animal  Max Speed
+        ...Falcon      375.0
+        ...Parrot       25.0
+
+        We can also choose to include NA in group keys or not by setting dropna parameter,
+        the default setting is True:
+
+        >>> l = [[1, 2, 3], [1, None, 4], [2, 1, 3], [1, 2, 2]]
+        >>> df = ps.DataFrame(l, columns=["a", "b", "c"])
+        >>> df.groupby(by=["b"]).sum().sort_index()  # doctest: +NORMALIZE_WHITESPACE
+             a  c
+        b
+        1.0  2  3
+        2.0  2  5
+
+        >>> df.groupby(by=["b"], dropna=False).sum().sort_index()  # doctest: +NORMALIZE_WHITESPACE
+             a  c
+        b
+        1.0  2  3
+        2.0  2  5
+        NaN  1  4
+        """
+        from pyspark.pandas.groupby import DataFrameGroupBy, SeriesGroupBy
+
+        if isinstance(by, ps.DataFrame):
+            raise ValueError("Grouper for '{}' not 1-dimensional".format(type(by).__name__))
+        elif isinstance(by, ps.Series):
+            by = [by]
+        elif is_name_like_tuple(by):
+            if isinstance(self, ps.Series):
+                raise KeyError(by)
+            by = [by]
+        elif is_name_like_value(by):
+            if isinstance(self, ps.Series):
+                raise KeyError(by)
+            by = [(by,)]
+        elif is_list_like(by):
+            new_by = []  # type: List[Union[Tuple, ps.Series]]
+            for key in by:
+                if isinstance(key, ps.DataFrame):
+                    raise ValueError(
+                        "Grouper for '{}' not 1-dimensional".format(type(key).__name__)
+                    )
+                elif isinstance(key, ps.Series):
+                    new_by.append(key)
+                elif is_name_like_tuple(key):
+                    if isinstance(self, ps.Series):
+                        raise KeyError(key)
+                    new_by.append(key)
+                elif is_name_like_value(key):
+                    if isinstance(self, ps.Series):
+                        raise KeyError(key)
+                    new_by.append((key,))
+                else:
+                    raise ValueError(
+                        "Grouper for '{}' not 1-dimensional".format(type(key).__name__)
+                    )
+            by = new_by
+        else:
+            raise ValueError("Grouper for '{}' not 1-dimensional".format(type(by).__name__))
+        if not len(by):
+            raise ValueError("No group keys passed!")
+        axis = validate_axis(axis)
+        if axis != 0:
+            raise NotImplementedError('axis should be either 0 or "index" currently.')
+
+        if isinstance(self, ps.DataFrame):
+            return DataFrameGroupBy._build(self, by, as_index=as_index, dropna=dropna)
+        elif isinstance(self, ps.Series):
+            return SeriesGroupBy._build(self, by, as_index=as_index, dropna=dropna)
+        else:
+            raise TypeError(
+                "Constructor expects DataFrame or Series; however, " "got [%s]" % (self,)
+            )
+
+    def bool(self) -> bool:
+        """
+        Return the bool of a single element in the current object.
+
+        This must be a boolean scalar value, either True or False. Raise a ValueError if
+        the object does not have exactly 1 element, or that element is not boolean
+
+        Returns
+        --------
+        bool
+
+        Examples
+        --------
+        >>> ps.DataFrame({'a': [True]}).bool()
+        True
+
+        >>> ps.Series([False]).bool()
+        False
+
+        If there are non-boolean or multiple values exist, it raises an exception in all
+        cases as below.
+
+        >>> ps.DataFrame({'a': ['a']}).bool()
+        Traceback (most recent call last):
+          ...
+        ValueError: bool cannot act on a non-boolean single element DataFrame
+
+        >>> ps.DataFrame({'a': [True], 'b': [False]}).bool()  # doctest: +NORMALIZE_WHITESPACE
+        Traceback (most recent call last):
+          ...
+        ValueError: The truth value of a DataFrame is ambiguous. Use a.empty, a.bool(),
+        a.item(), a.any() or a.all().
+
+        >>> ps.Series([1]).bool()
+        Traceback (most recent call last):
+          ...
+        ValueError: bool cannot act on a non-boolean single element DataFrame
+        """
+        if isinstance(self, ps.DataFrame):
+            df = self
+        elif isinstance(self, ps.Series):
+            df = self.to_dataframe()
+        else:
+            raise TypeError("bool() expects DataFrame or Series; however, " "got [%s]" % (self,))
+        return df.head(2)._to_internal_pandas().bool()
+
+    def first_valid_index(self) -> Optional[Union[Scalar, Tuple[Scalar, ...]]]:
+        """
+        Retrieves the index of the first valid value.
+
+        Returns
+        -------
+        scalar, tuple, or None
+
+        Examples
+        --------
+
+        Support for DataFrame
+
+        >>> kdf = ps.DataFrame({'a': [None, 2, 3, 2],
+        ...                     'b': [None, 2.0, 3.0, 1.0],
+        ...                     'c': [None, 200, 400, 200]},
+        ...                     index=['Q', 'W', 'E', 'R'])
+        >>> kdf
+             a    b      c
+        Q  NaN  NaN    NaN
+        W  2.0  2.0  200.0
+        E  3.0  3.0  400.0
+        R  2.0  1.0  200.0
+
+        >>> kdf.first_valid_index()
+        'W'
+
+        Support for MultiIndex columns
+
+        >>> kdf.columns = pd.MultiIndex.from_tuples([('a', 'x'), ('b', 'y'), ('c', 'z')])
+        >>> kdf
+             a    b      c
+             x    y      z
+        Q  NaN  NaN    NaN
+        W  2.0  2.0  200.0
+        E  3.0  3.0  400.0
+        R  2.0  1.0  200.0
+
+        >>> kdf.first_valid_index()
+        'W'
+
+        Support for Series.
+
+        >>> s = ps.Series([None, None, 3, 4, 5], index=[100, 200, 300, 400, 500])
+        >>> s
+        100    NaN
+        200    NaN
+        300    3.0
+        400    4.0
+        500    5.0
+        dtype: float64
+
+        >>> s.first_valid_index()
+        300
+
+        Support for MultiIndex
+
+        >>> midx = pd.MultiIndex([['lama', 'cow', 'falcon'],
+        ...                       ['speed', 'weight', 'length']],
+        ...                      [[0, 0, 0, 1, 1, 1, 2, 2, 2],
+        ...                       [0, 1, 2, 0, 1, 2, 0, 1, 2]])
+        >>> s = ps.Series([None, None, None, None, 250, 1.5, 320, 1, 0.3], index=midx)
+        >>> s
+        lama    speed       NaN
+                weight      NaN
+                length      NaN
+        cow     speed       NaN
+                weight    250.0
+                length      1.5
+        falcon  speed     320.0
+                weight      1.0
+                length      0.3
+        dtype: float64
+
+        >>> s.first_valid_index()
+        ('cow', 'weight')
+        """
+        data_spark_columns = self._internal.data_spark_columns
+
+        if len(data_spark_columns) == 0:
+            return None
+
+        cond = reduce(lambda x, y: x & y, map(lambda x: x.isNotNull(), data_spark_columns))
+
+        with sql_conf({SPARK_CONF_ARROW_ENABLED: False}):
+            # Disable Arrow to keep row ordering.
+            first_valid_row = (
+                self._internal.spark_frame.filter(cond)
+                .select(self._internal.index_spark_columns)
+                .limit(1)
+                .toPandas()
+            )
+
+        # For Empty Series or DataFrame, returns None.
+        if len(first_valid_row) == 0:
+            return None
+
+        first_valid_row = first_valid_row.iloc[0]
+        if len(first_valid_row) == 1:
+            return first_valid_row.iloc[0]
+        else:
+            return tuple(first_valid_row)
+
+    def last_valid_index(self) -> Optional[Union[Scalar, Tuple[Scalar, ...]]]:
+        """
+        Return index for last non-NA/null value.
+
+        Returns
+        -------
+        scalar, tuple, or None
+
+        Notes
+        -----
+        This API only works with PySpark >= 3.0.
+
+        Examples
+        --------
+
+        Support for DataFrame
+
+        >>> kdf = ps.DataFrame({'a': [1, 2, 3, None],
+        ...                     'b': [1.0, 2.0, 3.0, None],
+        ...                     'c': [100, 200, 400, None]},
+        ...                     index=['Q', 'W', 'E', 'R'])
+        >>> kdf
+             a    b      c
+        Q  1.0  1.0  100.0
+        W  2.0  2.0  200.0
+        E  3.0  3.0  400.0
+        R  NaN  NaN    NaN
+
+        >>> kdf.last_valid_index()  # doctest: +SKIP
+        'E'
+
+        Support for MultiIndex columns
+
+        >>> kdf.columns = pd.MultiIndex.from_tuples([('a', 'x'), ('b', 'y'), ('c', 'z')])
+        >>> kdf
+             a    b      c
+             x    y      z
+        Q  1.0  1.0  100.0
+        W  2.0  2.0  200.0
+        E  3.0  3.0  400.0
+        R  NaN  NaN    NaN
+
+        >>> kdf.last_valid_index()  # doctest: +SKIP
+        'E'
+
+        Support for Series.
+
+        >>> s = ps.Series([1, 2, 3, None, None], index=[100, 200, 300, 400, 500])
+        >>> s
+        100    1.0
+        200    2.0
+        300    3.0
+        400    NaN
+        500    NaN
+        dtype: float64
+
+        >>> s.last_valid_index()  # doctest: +SKIP
+        300
+
+        Support for MultiIndex
+
+        >>> midx = pd.MultiIndex([['lama', 'cow', 'falcon'],
+        ...                       ['speed', 'weight', 'length']],
+        ...                      [[0, 0, 0, 1, 1, 1, 2, 2, 2],
+        ...                       [0, 1, 2, 0, 1, 2, 0, 1, 2]])
+        >>> s = ps.Series([250, 1.5, 320, 1, 0.3, None, None, None, None], index=midx)
+        >>> s
+        lama    speed     250.0
+                weight      1.5
+                length    320.0
+        cow     speed       1.0
+                weight      0.3
+                length      NaN
+        falcon  speed       NaN
+                weight      NaN
+                length      NaN
+        dtype: float64
+
+        >>> s.last_valid_index()  # doctest: +SKIP
+        ('cow', 'weight')
+        """
+        data_spark_columns = self._internal.data_spark_columns
+
+        if len(data_spark_columns) == 0:
+            return None
+
+        cond = reduce(lambda x, y: x & y, map(lambda x: x.isNotNull(), data_spark_columns))
+
+        last_valid_rows = (
+            self._internal.spark_frame.filter(cond)
+            .select(self._internal.index_spark_columns)
+            .tail(1)
+        )
+
+        # For Empty Series or DataFrame, returns None.
+        if len(last_valid_rows) == 0:
+            return None
+
+        last_valid_row = last_valid_rows[0]
+
+        if len(last_valid_row) == 1:
+            return last_valid_row[0]
+        else:
+            return tuple(last_valid_row)
+
+    # TODO: 'center', 'win_type', 'on', 'axis' parameter should be implemented.
+    def rolling(self, window, min_periods=None) -> Rolling:
+        """
+        Provide rolling transformations.
+
+        .. note:: 'min_periods' in pandas-on-Spark works as a fixed window size unlike pandas.
+            Unlike pandas, NA is also counted as the period. This might be changed
+            in the near future.
+
+        Parameters
+        ----------
+        window : int, or offset
+            Size of the moving window.
+            This is the number of observations used for calculating the statistic.
+            Each window will be a fixed size.
+
+        min_periods : int, default None
+            Minimum number of observations in window required to have a value
+            (otherwise result is NA).
+            For a window that is specified by an offset, min_periods will default to 1.
+            Otherwise, min_periods will default to the size of the window.
+
+        Returns
+        -------
+        a Window sub-classed for the particular operation
+        """
+        return Rolling(self, window=window, min_periods=min_periods)
+
+    # TODO: 'center' and 'axis' parameter should be implemented.
+    #   'axis' implementation, refer https://github.com/pyspark.pandas/pull/607
+    def expanding(self, min_periods=1) -> Expanding:
+        """
+        Provide expanding transformations.
+
+        .. note:: 'min_periods' in pandas-on-Spark works as a fixed window size unlike pandas.
+            Unlike pandas, NA is also counted as the period. This might be changed
+            in the near future.
+
+        Parameters
+        ----------
+        min_periods : int, default 1
+            Minimum number of observations in window required to have a value
+            (otherwise result is NA).
+
+        Returns
+        -------
+        a Window sub-classed for the particular operation
+        """
+        return Expanding(self, min_periods=min_periods)
+
+    def get(self, key, default=None) -> Any:
+        """
+        Get item from object for given key (DataFrame column, Panel slice,
+        etc.). Returns default value if not found.
+
+        Parameters
+        ----------
+        key : object
+
+        Returns
+        -------
+        value : same type as items contained in object
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'x':range(3), 'y':['a','b','b'], 'z':['a','b','b']},
+        ...                   columns=['x', 'y', 'z'], index=[10, 20, 20])
+        >>> df
+            x  y  z
+        10  0  a  a
+        20  1  b  b
+        20  2  b  b
+
+        >>> df.get('x')
+        10    0
+        20    1
+        20    2
+        Name: x, dtype: int64
+
+        >>> df.get(['x', 'y'])
+            x  y
+        10  0  a
+        20  1  b
+        20  2  b
+
+        >>> df.x.get(10)
+        0
+
+        >>> df.x.get(20)
+        20    1
+        20    2
+        Name: x, dtype: int64
+
+        >>> df.x.get(15, -1)
+        -1
+        """
+        try:
+            return self[key]
+        except (KeyError, ValueError, IndexError):
+            return default
+
+    def squeeze(self, axis=None) -> Union[Scalar, "DataFrame", "Series"]:
+        """
+        Squeeze 1 dimensional axis objects into scalars.
+
+        Series or DataFrames with a single element are squeezed to a scalar.
+        DataFrames with a single column or a single row are squeezed to a
+        Series. Otherwise the object is unchanged.
+
+        This method is most useful when you don't know if your
+        object is a Series or DataFrame, but you do know it has just a single
+        column. In that case you can safely call `squeeze` to ensure you have a
+        Series.
+
+        Parameters
+        ----------
+        axis : {0 or 'index', 1 or 'columns', None}, default None
+            A specific axis to squeeze. By default, all length-1 axes are
+            squeezed.
+
+        Returns
+        -------
+        DataFrame, Series, or scalar
+            The projection after squeezing `axis` or all the axes.
+
+        See Also
+        --------
+        Series.iloc : Integer-location based indexing for selecting scalars.
+        DataFrame.iloc : Integer-location based indexing for selecting Series.
+        Series.to_frame : Inverse of DataFrame.squeeze for a
+            single-column DataFrame.
+
+        Examples
+        --------
+        >>> primes = ps.Series([2, 3, 5, 7])
+
+        Slicing might produce a Series with a single value:
+
+        >>> even_primes = primes[primes % 2 == 0]
+        >>> even_primes
+        0    2
+        dtype: int64
+
+        >>> even_primes.squeeze()
+        2
+
+        Squeezing objects with more than one value in every axis does nothing:
+
+        >>> odd_primes = primes[primes % 2 == 1]
+        >>> odd_primes
+        1    3
+        2    5
+        3    7
+        dtype: int64
+
+        >>> odd_primes.squeeze()
+        1    3
+        2    5
+        3    7
+        dtype: int64
+
+        Squeezing is even more effective when used with DataFrames.
+
+        >>> df = ps.DataFrame([[1, 2], [3, 4]], columns=['a', 'b'])
+        >>> df
+           a  b
+        0  1  2
+        1  3  4
+
+        Slicing a single column will produce a DataFrame with the columns
+        having only one value:
+
+        >>> df_a = df[['a']]
+        >>> df_a
+           a
+        0  1
+        1  3
+
+        So the columns can be squeezed down, resulting in a Series:
+
+        >>> df_a.squeeze('columns')
+        0    1
+        1    3
+        Name: a, dtype: int64
+
+        Slicing a single row from a single column will produce a single
+        scalar DataFrame:
+
+        >>> df_1a = df.loc[[1], ['a']]
+        >>> df_1a
+           a
+        1  3
+
+        Squeezing the rows produces a single scalar Series:
+
+        >>> df_1a.squeeze('rows')
+        a    3
+        Name: 1, dtype: int64
+
+        Squeezing all axes will project directly into a scalar:
+
+        >>> df_1a.squeeze()
+        3
+        """
+        if axis is not None:
+            axis = "index" if axis == "rows" else axis
+            axis = validate_axis(axis)
+
+        if isinstance(self, ps.DataFrame):
+            from pyspark.pandas.series import first_series
+
+            is_squeezable = len(self.columns[:2]) == 1
+            # If DataFrame has multiple columns, there is no change.
+            if not is_squeezable:
+                return self
+            series_from_column = first_series(self)
+            has_single_value = len(series_from_column.head(2)) == 1
+            # If DataFrame has only a single value, use pandas API directly.
+            if has_single_value:
+                result = self._to_internal_pandas().squeeze(axis)
+                return ps.Series(result) if isinstance(result, pd.Series) else result
+            elif axis == 0:
+                return self
+            else:
+                return series_from_column
+        else:
+            # The case of Series is simple.
+            # If Series has only a single value, just return it as a scalar.
+            # Otherwise, there is no change.
+            self_top_two = self.head(2)
+            has_single_value = len(self_top_two) == 1
+            return cast(Union[Scalar, ps.Series], self_top_two[0] if has_single_value else self)
+
+    def truncate(
+        self, before=None, after=None, axis=None, copy=True
+    ) -> Union["DataFrame", "Series"]:
+        """
+        Truncate a Series or DataFrame before and after some index value.
+
+        This is a useful shorthand for boolean indexing based on index
+        values above or below certain thresholds.
+
+        .. note:: This API is dependent on :meth:`Index.is_monotonic_increasing`
+            which can be expensive.
+
+        Parameters
+        ----------
+        before : date, str, int
+            Truncate all rows before this index value.
+        after : date, str, int
+            Truncate all rows after this index value.
+        axis : {0 or 'index', 1 or 'columns'}, optional
+            Axis to truncate. Truncates the index (rows) by default.
+        copy : bool, default is True,
+            Return a copy of the truncated section.
+
+        Returns
+        -------
+        type of caller
+            The truncated Series or DataFrame.
+
+        See Also
+        --------
+        DataFrame.loc : Select a subset of a DataFrame by label.
+        DataFrame.iloc : Select a subset of a DataFrame by position.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'A': ['a', 'b', 'c', 'd', 'e'],
+        ...                    'B': ['f', 'g', 'h', 'i', 'j'],
+        ...                    'C': ['k', 'l', 'm', 'n', 'o']},
+        ...                   index=[1, 2, 3, 4, 5])
+        >>> df
+           A  B  C
+        1  a  f  k
+        2  b  g  l
+        3  c  h  m
+        4  d  i  n
+        5  e  j  o
+
+        >>> df.truncate(before=2, after=4)
+           A  B  C
+        2  b  g  l
+        3  c  h  m
+        4  d  i  n
+
+        The columns of a DataFrame can be truncated.
+
+        >>> df.truncate(before="A", after="B", axis="columns")
+           A  B
+        1  a  f
+        2  b  g
+        3  c  h
+        4  d  i
+        5  e  j
+
+        For Series, only rows can be truncated.
+
+        >>> df['A'].truncate(before=2, after=4)
+        2    b
+        3    c
+        4    d
+        Name: A, dtype: object
+
+        A Series has index that sorted integers.
+
+        >>> s = ps.Series([10, 20, 30, 40, 50, 60, 70],
+        ...               index=[1, 2, 3, 4, 5, 6, 7])
+        >>> s
+        1    10
+        2    20
+        3    30
+        4    40
+        5    50
+        6    60
+        7    70
+        dtype: int64
+
+        >>> s.truncate(2, 5)
+        2    20
+        3    30
+        4    40
+        5    50
+        dtype: int64
+
+        A Series has index that sorted strings.
+
+        >>> s = ps.Series([10, 20, 30, 40, 50, 60, 70],
+        ...               index=['a', 'b', 'c', 'd', 'e', 'f', 'g'])
+        >>> s
+        a    10
+        b    20
+        c    30
+        d    40
+        e    50
+        f    60
+        g    70
+        dtype: int64
+
+        >>> s.truncate('b', 'e')
+        b    20
+        c    30
+        d    40
+        e    50
+        dtype: int64
+        """
+        from pyspark.pandas.series import first_series
+
+        axis = validate_axis(axis)
+        indexes = self.index
+        indexes_increasing = indexes.is_monotonic_increasing
+        if not indexes_increasing and not indexes.is_monotonic_decreasing:
+            raise ValueError("truncate requires a sorted index")
+        if (before is None) and (after is None):
+            return cast(Union[ps.DataFrame, ps.Series], self.copy() if copy else self)
+        if (before is not None and after is not None) and before > after:
+            raise ValueError("Truncate: %s must be after %s" % (after, before))
+
+        if isinstance(self, ps.Series):
+            if indexes_increasing:
+                result = first_series(self.to_frame().loc[before:after]).rename(self.name)
+            else:
+                result = first_series(self.to_frame().loc[after:before]).rename(self.name)
+        elif isinstance(self, ps.DataFrame):
+            if axis == 0:
+                if indexes_increasing:
+                    result = self.loc[before:after]
+                else:
+                    result = self.loc[after:before]
+            elif axis == 1:
+                result = self.loc[:, before:after]
+
+        return cast(Union[ps.DataFrame, ps.Series], result.copy() if copy else result)
+
+    def to_markdown(self, buf=None, mode=None) -> str:
+        """
+        Print Series or DataFrame in Markdown-friendly format.
+
+        .. note:: This method should only be used if the resulting pandas object is expected
+                  to be small, as all the data is loaded into the driver's memory.
+
+        Parameters
+        ----------
+        buf : writable buffer, defaults to sys.stdout
+            Where to send the output. By default, the output is printed to
+            sys.stdout. Pass a writable buffer if you need to further process
+            the output.
+        mode : str, optional
+            Mode in which file is opened.
+        **kwargs
+            These parameters will be passed to `tabulate`.
+
+        Returns
+        -------
+        str
+            Series or DataFrame in Markdown-friendly format.
+
+        Notes
+        -----
+        Requires the `tabulate <https://pypi.org/project/tabulate>`_ package.
+
+        Examples
+        --------
+        >>> kser = ps.Series(["elk", "pig", "dog", "quetzal"], name="animal")
+        >>> print(kser.to_markdown())  # doctest: +SKIP
+        |    | animal   |
+        |---:|:---------|
+        |  0 | elk      |
+        |  1 | pig      |
+        |  2 | dog      |
+        |  3 | quetzal  |
+
+        >>> kdf = ps.DataFrame(
+        ...     data={"animal_1": ["elk", "pig"], "animal_2": ["dog", "quetzal"]}
+        ... )
+        >>> print(kdf.to_markdown())  # doctest: +SKIP
+        |    | animal_1   | animal_2   |
+        |---:|:-----------|:-----------|
+        |  0 | elk        | dog        |
+        |  1 | pig        | quetzal    |
+        """
+        # `to_markdown` is supported in pandas >= 1.0.0 since it's newly added in pandas 1.0.0.
+        if LooseVersion(pd.__version__) < LooseVersion("1.0.0"):
+            raise NotImplementedError(
+                "`to_markdown()` only supported in pandas-on-Spark with pandas >= 1.0.0"
+            )
+        # Make sure locals() call is at the top of the function so we don't capture local variables.
+        args = locals()
+        kser_or_kdf = self
+        internal_pandas = kser_or_kdf._to_internal_pandas()
+        return validate_arguments_and_invoke_function(
+            internal_pandas, self.to_markdown, type(internal_pandas).to_markdown, args
+        )
+
+    @abstractmethod
+    def fillna(self, value=None, method=None, axis=None, inplace=False, limit=None):
+        pass
+
+    # TODO: add 'downcast' when value parameter exists
+    def bfill(self, axis=None, inplace=False, limit=None) -> Union["DataFrame", "Series"]:
+        """
+        Synonym for `DataFrame.fillna()` or `Series.fillna()` with ``method=`bfill```.
+
+        .. note:: the current implementation of 'bfill' uses Spark's Window
+            without specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Parameters
+        ----------
+        axis : {0 or `index`}
+            1 and `columns` are not supported.
+        inplace : boolean, default False
+            Fill in place (do not create a new object)
+        limit : int, default None
+            If method is specified, this is the maximum number of consecutive NaN values to
+            forward/backward fill. In other words, if there is a gap with more than this number of
+            consecutive NaNs, it will only be partially filled. If method is not specified,
+            this is the maximum number of entries along the entire axis where NaNs will be filled.
+            Must be greater than 0 if not None
+
+        Returns
+        -------
+        DataFrame or Series
+            DataFrame or Series with NA entries filled.
+
+        Examples
+        --------
+        >>> kdf = ps.DataFrame({
+        ...     'A': [None, 3, None, None],
+        ...     'B': [2, 4, None, 3],
+        ...     'C': [None, None, None, 1],
+        ...     'D': [0, 1, 5, 4]
+        ...     },
+        ...     columns=['A', 'B', 'C', 'D'])
+        >>> kdf
+             A    B    C  D
+        0  NaN  2.0  NaN  0
+        1  3.0  4.0  NaN  1
+        2  NaN  NaN  NaN  5
+        3  NaN  3.0  1.0  4
+
+        Propagate non-null values backward.
+
+        >>> kdf.bfill()
+             A    B    C  D
+        0  3.0  2.0  1.0  0
+        1  3.0  4.0  1.0  1
+        2  NaN  3.0  1.0  5
+        3  NaN  3.0  1.0  4
+
+        For Series
+
+        >>> kser = ps.Series([None, None, None, 1])
+        >>> kser
+        0    NaN
+        1    NaN
+        2    NaN
+        3    1.0
+        dtype: float64
+
+        >>> kser.bfill()
+        0    1.0
+        1    1.0
+        2    1.0
+        3    1.0
+        dtype: float64
+        """
+        return self.fillna(method="bfill", axis=axis, inplace=inplace, limit=limit)
+
+    backfill = bfill
+
+    # TODO: add 'downcast' when value parameter exists
+    def ffill(self, axis=None, inplace=False, limit=None) -> Union["DataFrame", "Series"]:
+        """
+        Synonym for `DataFrame.fillna()` or `Series.fillna()` with ``method=`ffill```.
+
+        .. note:: the current implementation of 'ffill' uses Spark's Window
+            without specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Parameters
+        ----------
+        axis : {0 or `index`}
+            1 and `columns` are not supported.
+        inplace : boolean, default False
+            Fill in place (do not create a new object)
+        limit : int, default None
+            If method is specified, this is the maximum number of consecutive NaN values to
+            forward/backward fill. In other words, if there is a gap with more than this number of
+            consecutive NaNs, it will only be partially filled. If method is not specified,
+            this is the maximum number of entries along the entire axis where NaNs will be filled.
+            Must be greater than 0 if not None
+
+        Returns
+        -------
+        DataFrame or Series
+            DataFrame or Series with NA entries filled.
+
+        Examples
+        --------
+        >>> kdf = ps.DataFrame({
+        ...     'A': [None, 3, None, None],
+        ...     'B': [2, 4, None, 3],
+        ...     'C': [None, None, None, 1],
+        ...     'D': [0, 1, 5, 4]
+        ...     },
+        ...     columns=['A', 'B', 'C', 'D'])
+        >>> kdf
+             A    B    C  D
+        0  NaN  2.0  NaN  0
+        1  3.0  4.0  NaN  1
+        2  NaN  NaN  NaN  5
+        3  NaN  3.0  1.0  4
+
+        Propagate non-null values forward.
+
+        >>> kdf.ffill()
+             A    B    C  D
+        0  NaN  2.0  NaN  0
+        1  3.0  4.0  NaN  1
+        2  3.0  4.0  NaN  5
+        3  3.0  3.0  1.0  4
+
+        For Series
+
+        >>> kser = ps.Series([2, 4, None, 3])
+        >>> kser
+        0    2.0
+        1    4.0
+        2    NaN
+        3    3.0
+        dtype: float64
+
+        >>> kser.ffill()
+        0    2.0
+        1    4.0
+        2    4.0
+        3    3.0
+        dtype: float64
+        """
+        return self.fillna(method="ffill", axis=axis, inplace=inplace, limit=limit)
+
+    pad = ffill
+
+    @property
+    def at(self) -> AtIndexer:
+        return AtIndexer(self)
+
+    at.__doc__ = AtIndexer.__doc__
+
+    @property
+    def iat(self) -> iAtIndexer:
+        return iAtIndexer(self)
+
+    iat.__doc__ = iAtIndexer.__doc__
+
+    @property
+    def iloc(self) -> iLocIndexer:
+        return iLocIndexer(self)
+
+    iloc.__doc__ = iLocIndexer.__doc__
+
+    @property
+    def loc(self) -> LocIndexer:
+        return LocIndexer(self)
+
+    loc.__doc__ = LocIndexer.__doc__
+
+    def __bool__(self):
+        raise ValueError(
+            "The truth value of a {0} is ambiguous. "
+            "Use a.empty, a.bool(), a.item(), a.any() or a.all().".format(self.__class__.__name__)
+        )
+
+    @staticmethod
+    def _count_expr(spark_column, spark_type):
+        # Special handle floating point types because Spark's count treats nan as a valid value,
+        # whereas pandas count doesn't include nan.
+        if isinstance(spark_type, (FloatType, DoubleType)):
+            return F.count(F.nanvl(spark_column, F.lit(None)))
+        else:
+            return F.count(spark_column)
+
+
+def _test():
+    import os
+    import doctest
+    import shutil
+    import sys
+    import tempfile
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.generic
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.generic.__dict__.copy()
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]")
+        .appName("pyspark.pandas.generic tests")
+        .getOrCreate()
+    )
+
+    path = tempfile.mkdtemp()
+    globs["path"] = path
+
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.generic,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+
+    shutil.rmtree(path, ignore_errors=True)
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/groupby.py b/python/pyspark/pandas/groupby.py
new file mode 100644
index 0000000000000..4bb43394f874a
--- /dev/null
+++ b/python/pyspark/pandas/groupby.py
@@ -0,0 +1,3217 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+A wrapper for GroupedData to behave similar to pandas GroupBy.
+"""
+
+from abc import ABCMeta, abstractmethod
+import sys
+import inspect
+from collections import OrderedDict, namedtuple
+from collections.abc import Callable
+from distutils.version import LooseVersion
+from functools import partial
+from itertools import product
+from typing import Any, List, Set, Tuple, Union, cast
+
+import pandas as pd
+from pandas.api.types import is_hashable, is_list_like
+
+from pyspark.sql import Window, functions as F
+from pyspark.sql.types import (
+    FloatType,
+    DoubleType,
+    NumericType,
+    StructField,
+    StructType,
+    StringType,
+)
+from pyspark.sql.functions import PandasUDFType, pandas_udf, Column
+
+from pyspark import pandas as ps  # For running doctests and reference resolution in PyCharm.
+from pyspark.pandas.typedef import infer_return_type, DataFrameType, ScalarType, SeriesType
+from pyspark.pandas.frame import DataFrame
+from pyspark.pandas.internal import (
+    InternalFrame,
+    HIDDEN_COLUMNS,
+    NATURAL_ORDER_COLUMN_NAME,
+    SPARK_INDEX_NAME_FORMAT,
+    SPARK_DEFAULT_SERIES_NAME,
+)
+from pyspark.pandas.missing.groupby import (
+    MissingPandasLikeDataFrameGroupBy,
+    MissingPandasLikeSeriesGroupBy,
+)
+from pyspark.pandas.series import Series, first_series
+from pyspark.pandas.config import get_option
+from pyspark.pandas.utils import (
+    align_diff_frames,
+    is_name_like_tuple,
+    is_name_like_value,
+    name_like_string,
+    same_anchor,
+    scol_for,
+    verify_temp_column_name,
+)
+from pyspark.pandas.spark.utils import as_nullable_spark_type, force_decimal_precision_scale
+from pyspark.pandas.window import RollingGroupby, ExpandingGroupby
+from pyspark.pandas.exceptions import DataError
+
+# to keep it the same as pandas
+NamedAgg = namedtuple("NamedAgg", ["column", "aggfunc"])
+
+
+class GroupBy(object, metaclass=ABCMeta):
+    """
+    :ivar _kdf: The parent dataframe that is used to perform the groupby
+    :type _kdf: DataFrame
+    :ivar _groupkeys: The list of keys that will be used to perform the grouping
+    :type _groupkeys: List[Series]
+    """
+
+    def __init__(
+        self,
+        kdf: DataFrame,
+        groupkeys: List[Series],
+        as_index: bool,
+        dropna: bool,
+        column_labels_to_exlcude: Set[Tuple],
+        agg_columns_selected: bool,
+        agg_columns: List[Series],
+    ):
+        self._kdf = kdf
+        self._groupkeys = groupkeys
+        self._as_index = as_index
+        self._dropna = dropna
+        self._column_labels_to_exlcude = column_labels_to_exlcude
+        self._agg_columns_selected = agg_columns_selected
+        self._agg_columns = agg_columns
+
+    @property
+    def _groupkeys_scols(self):
+        return [s.spark.column for s in self._groupkeys]
+
+    @property
+    def _agg_columns_scols(self):
+        return [s.spark.column for s in self._agg_columns]
+
+    @abstractmethod
+    def _apply_series_op(self, op, should_resolve: bool = False, numeric_only: bool = False):
+        pass
+
+    # TODO: Series support is not implemented yet.
+    # TODO: not all arguments are implemented comparing to pandas' for now.
+    def aggregate(self, func_or_funcs=None, *args, **kwargs) -> DataFrame:
+        """Aggregate using one or more operations over the specified axis.
+
+        Parameters
+        ----------
+        func_or_funcs : dict, str or list
+             a dict mapping from column name (string) to
+             aggregate functions (string or list of strings).
+
+        Returns
+        -------
+        Series or DataFrame
+
+            The return can be:
+
+            * Series : when DataFrame.agg is called with a single function
+            * DataFrame : when DataFrame.agg is called with several functions
+
+            Return Series or DataFrame.
+
+        Notes
+        -----
+        `agg` is an alias for `aggregate`. Use the alias.
+
+        See Also
+        --------
+        pyspark.pandas.Series.groupby
+        pyspark.pandas.DataFrame.groupby
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'A': [1, 1, 2, 2],
+        ...                    'B': [1, 2, 3, 4],
+        ...                    'C': [0.362, 0.227, 1.267, -0.562]},
+        ...                   columns=['A', 'B', 'C'])
+
+        >>> df
+           A  B      C
+        0  1  1  0.362
+        1  1  2  0.227
+        2  2  3  1.267
+        3  2  4 -0.562
+
+        Different aggregations per column
+
+        >>> aggregated = df.groupby('A').agg({'B': 'min', 'C': 'sum'})
+        >>> aggregated[['B', 'C']].sort_index()  # doctest: +NORMALIZE_WHITESPACE
+           B      C
+        A
+        1  1  0.589
+        2  3  0.705
+
+        >>> aggregated = df.groupby('A').agg({'B': ['min', 'max']})
+        >>> aggregated.sort_index()  # doctest: +NORMALIZE_WHITESPACE
+             B
+           min  max
+        A
+        1    1    2
+        2    3    4
+
+        >>> aggregated = df.groupby('A').agg('min')
+        >>> aggregated.sort_index()  # doctest: +NORMALIZE_WHITESPACE
+             B      C
+        A
+        1    1  0.227
+        2    3 -0.562
+
+        >>> aggregated = df.groupby('A').agg(['min', 'max'])
+        >>> aggregated.sort_index()  # doctest: +NORMALIZE_WHITESPACE
+             B           C
+           min  max    min    max
+        A
+        1    1    2  0.227  0.362
+        2    3    4 -0.562  1.267
+
+        To control the output names with different aggregations per column, pandas-on-Spark
+        also supports 'named aggregation' or nested renaming in .agg. It can also be
+        used when applying multiple aggregation functions to specific columns.
+
+        >>> aggregated = df.groupby('A').agg(b_max=ps.NamedAgg(column='B', aggfunc='max'))
+        >>> aggregated.sort_index()  # doctest: +NORMALIZE_WHITESPACE
+             b_max
+        A
+        1        2
+        2        4
+
+        >>> aggregated = df.groupby('A').agg(b_max=('B', 'max'), b_min=('B', 'min'))
+        >>> aggregated.sort_index()  # doctest: +NORMALIZE_WHITESPACE
+             b_max   b_min
+        A
+        1        2       1
+        2        4       3
+
+        >>> aggregated = df.groupby('A').agg(b_max=('B', 'max'), c_min=('C', 'min'))
+        >>> aggregated.sort_index()  # doctest: +NORMALIZE_WHITESPACE
+             b_max   c_min
+        A
+        1        2   0.227
+        2        4  -0.562
+        """
+        # I think current implementation of func and arguments in pandas-on-Spark for aggregate
+        # is different than pandas, later once arguments are added, this could be removed.
+        if func_or_funcs is None and kwargs is None:
+            raise ValueError("No aggregation argument or function specified.")
+
+        relabeling = func_or_funcs is None and is_multi_agg_with_relabel(**kwargs)
+        if relabeling:
+            func_or_funcs, columns, order = normalize_keyword_aggregation(kwargs)
+
+        if not isinstance(func_or_funcs, (str, list)):
+            if not isinstance(func_or_funcs, dict) or not all(
+                is_name_like_value(key)
+                and (
+                    isinstance(value, str)
+                    or isinstance(value, list)
+                    and all(isinstance(v, str) for v in value)
+                )
+                for key, value in func_or_funcs.items()
+            ):
+                raise ValueError(
+                    "aggs must be a dict mapping from column name "
+                    "to aggregate functions (string or list of strings)."
+                )
+
+        else:
+            agg_cols = [col.name for col in self._agg_columns]
+            func_or_funcs = OrderedDict([(col, func_or_funcs) for col in agg_cols])
+
+        kdf = DataFrame(
+            GroupBy._spark_groupby(self._kdf, func_or_funcs, self._groupkeys)
+        )  # type: DataFrame
+
+        if self._dropna:
+            kdf = DataFrame(
+                kdf._internal.with_new_sdf(
+                    kdf._internal.spark_frame.dropna(subset=kdf._internal.index_spark_column_names)
+                )
+            )
+
+        if not self._as_index:
+            should_drop_index = set(
+                i for i, gkey in enumerate(self._groupkeys) if gkey._kdf is not self._kdf
+            )
+            if len(should_drop_index) > 0:
+                kdf = kdf.reset_index(level=should_drop_index, drop=True)
+            if len(should_drop_index) < len(self._groupkeys):
+                kdf = kdf.reset_index()
+
+        if relabeling:
+            kdf = kdf[order]
+            kdf.columns = columns
+        return kdf
+
+    agg = aggregate
+
+    @staticmethod
+    def _spark_groupby(kdf, func, groupkeys=()):
+        groupkey_names = [SPARK_INDEX_NAME_FORMAT(i) for i in range(len(groupkeys))]
+        groupkey_scols = [s.spark.column.alias(name) for s, name in zip(groupkeys, groupkey_names)]
+
+        multi_aggs = any(isinstance(v, list) for v in func.values())
+        reordered = []
+        data_columns = []
+        column_labels = []
+        for key, value in func.items():
+            label = key if is_name_like_tuple(key) else (key,)
+            if len(label) != kdf._internal.column_labels_level:
+                raise TypeError("The length of the key must be the same as the column label level.")
+            for aggfunc in [value] if isinstance(value, str) else value:
+                column_label = tuple(list(label) + [aggfunc]) if multi_aggs else label
+                column_labels.append(column_label)
+
+                data_col = name_like_string(column_label)
+                data_columns.append(data_col)
+
+                col_name = kdf._internal.spark_column_name_for(label)
+                if aggfunc == "nunique":
+                    reordered.append(
+                        F.expr("count(DISTINCT `{0}`) as `{1}`".format(col_name, data_col))
+                    )
+
+                # Implement "quartiles" aggregate function for ``describe``.
+                elif aggfunc == "quartiles":
+                    reordered.append(
+                        F.expr(
+                            "percentile_approx(`{0}`, array(0.25, 0.5, 0.75)) as `{1}`".format(
+                                col_name, data_col
+                            )
+                        )
+                    )
+
+                else:
+                    reordered.append(
+                        F.expr("{1}(`{0}`) as `{2}`".format(col_name, aggfunc, data_col))
+                    )
+
+        sdf = kdf._internal.spark_frame.select(groupkey_scols + kdf._internal.data_spark_columns)
+        sdf = sdf.groupby(*groupkey_names).agg(*reordered)
+        if len(groupkeys) > 0:
+            index_spark_column_names = groupkey_names
+            index_names = [kser._column_label for kser in groupkeys]
+            index_dtypes = [kser.dtype for kser in groupkeys]
+        else:
+            index_spark_column_names = []
+            index_names = []
+            index_dtypes = []
+
+        return InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=[scol_for(sdf, col) for col in index_spark_column_names],
+            index_names=index_names,
+            index_dtypes=index_dtypes,
+            column_labels=column_labels,
+            data_spark_columns=[scol_for(sdf, col) for col in data_columns],
+        )
+
+    def count(self) -> Union[DataFrame, Series]:
+        """
+        Compute count of group, excluding missing values.
+
+        See Also
+        --------
+        pyspark.pandas.Series.groupby
+        pyspark.pandas.DataFrame.groupby
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'A': [1, 1, 2, 1, 2],
+        ...                    'B': [np.nan, 2, 3, 4, 5],
+        ...                    'C': [1, 2, 1, 1, 2]}, columns=['A', 'B', 'C'])
+        >>> df.groupby('A').count().sort_index()  # doctest: +NORMALIZE_WHITESPACE
+            B  C
+        A
+        1  2  3
+        2  2  2
+        """
+        return self._reduce_for_stat_function(F.count, only_numeric=False)
+
+    # TODO: We should fix See Also when Series implementation is finished.
+    def first(self) -> Union[DataFrame, Series]:
+        """
+        Compute first of group values.
+
+        See Also
+        --------
+        pyspark.pandas.Series.groupby
+        pyspark.pandas.DataFrame.groupby
+        """
+        return self._reduce_for_stat_function(F.first, only_numeric=False)
+
+    def last(self) -> Union[DataFrame, Series]:
+        """
+        Compute last of group values.
+
+        See Also
+        --------
+        pyspark.pandas.Series.groupby
+        pyspark.pandas.DataFrame.groupby
+        """
+        return self._reduce_for_stat_function(
+            lambda col: F.last(col, ignorenulls=True), only_numeric=False
+        )
+
+    def max(self) -> Union[DataFrame, Series]:
+        """
+        Compute max of group values.
+
+        See Also
+        --------
+        pyspark.pandas.Series.groupby
+        pyspark.pandas.DataFrame.groupby
+        """
+        return self._reduce_for_stat_function(F.max, only_numeric=False)
+
+    # TODO: examples should be updated.
+    def mean(self) -> Union[DataFrame, Series]:
+        """
+        Compute mean of groups, excluding missing values.
+
+        Returns
+        -------
+        koalas.Series or koalas.DataFrame
+
+        See Also
+        --------
+        pyspark.pandas.Series.groupby
+        pyspark.pandas.DataFrame.groupby
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'A': [1, 1, 2, 1, 2],
+        ...                    'B': [np.nan, 2, 3, 4, 5],
+        ...                    'C': [1, 2, 1, 1, 2]}, columns=['A', 'B', 'C'])
+
+        Groupby one column and return the mean of the remaining columns in
+        each group.
+
+        >>> df.groupby('A').mean().sort_index()  # doctest: +NORMALIZE_WHITESPACE
+             B         C
+        A
+        1  3.0  1.333333
+        2  4.0  1.500000
+        """
+
+        return self._reduce_for_stat_function(F.mean, only_numeric=True)
+
+    def min(self) -> Union[DataFrame, Series]:
+        """
+        Compute min of group values.
+
+        See Also
+        --------
+        pyspark.pandas.Series.groupby
+        pyspark.pandas.DataFrame.groupby
+        """
+        return self._reduce_for_stat_function(F.min, only_numeric=False)
+
+    # TODO: sync the doc.
+    def std(self, ddof: int = 1) -> Union[DataFrame, Series]:
+        """
+        Compute standard deviation of groups, excluding missing values.
+
+        Parameters
+        ----------
+        ddof : int, default 1
+            Delta Degrees of Freedom. The divisor used in calculations is N - ddof,
+            where N represents the number of elements.
+
+        See Also
+        --------
+        pyspark.pandas.Series.groupby
+        pyspark.pandas.DataFrame.groupby
+        """
+        assert ddof in (0, 1)
+
+        return self._reduce_for_stat_function(
+            F.stddev_pop if ddof == 0 else F.stddev_samp, only_numeric=True
+        )
+
+    def sum(self) -> Union[DataFrame, Series]:
+        """
+        Compute sum of group values
+
+        See Also
+        --------
+        pyspark.pandas.Series.groupby
+        pyspark.pandas.DataFrame.groupby
+        """
+        return self._reduce_for_stat_function(F.sum, only_numeric=True)
+
+    # TODO: sync the doc.
+    def var(self, ddof: int = 1) -> Union[DataFrame, Series]:
+        """
+        Compute variance of groups, excluding missing values.
+
+        Parameters
+        ----------
+        ddof : int, default 1
+            Delta Degrees of Freedom. The divisor used in calculations is N - ddof,
+            where N represents the number of elements.
+
+        See Also
+        --------
+        pyspark.pandas.Series.groupby
+        pyspark.pandas.DataFrame.groupby
+        """
+        assert ddof in (0, 1)
+
+        return self._reduce_for_stat_function(
+            F.var_pop if ddof == 0 else F.var_samp, only_numeric=True
+        )
+
+    # TODO: skipna should be implemented.
+    def all(self) -> Union[DataFrame, Series]:
+        """
+        Returns True if all values in the group are truthful, else False.
+
+        See Also
+        --------
+        pyspark.pandas.Series.groupby
+        pyspark.pandas.DataFrame.groupby
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'A': [1, 1, 2, 2, 3, 3, 4, 4, 5, 5],
+        ...                    'B': [True, True, True, False, False,
+        ...                          False, None, True, None, False]},
+        ...                   columns=['A', 'B'])
+        >>> df
+           A      B
+        0  1   True
+        1  1   True
+        2  2   True
+        3  2  False
+        4  3  False
+        5  3  False
+        6  4   None
+        7  4   True
+        8  5   None
+        9  5  False
+
+        >>> df.groupby('A').all().sort_index()  # doctest: +NORMALIZE_WHITESPACE
+               B
+        A
+        1   True
+        2  False
+        3  False
+        4   True
+        5  False
+        """
+        return self._reduce_for_stat_function(
+            lambda col: F.min(F.coalesce(col.cast("boolean"), F.lit(True))), only_numeric=False
+        )
+
+    # TODO: skipna should be implemented.
+    def any(self) -> Union[DataFrame, Series]:
+        """
+        Returns True if any value in the group is truthful, else False.
+
+        See Also
+        --------
+        pyspark.pandas.Series.groupby
+        pyspark.pandas.DataFrame.groupby
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'A': [1, 1, 2, 2, 3, 3, 4, 4, 5, 5],
+        ...                    'B': [True, True, True, False, False,
+        ...                          False, None, True, None, False]},
+        ...                   columns=['A', 'B'])
+        >>> df
+           A      B
+        0  1   True
+        1  1   True
+        2  2   True
+        3  2  False
+        4  3  False
+        5  3  False
+        6  4   None
+        7  4   True
+        8  5   None
+        9  5  False
+
+        >>> df.groupby('A').any().sort_index()  # doctest: +NORMALIZE_WHITESPACE
+               B
+        A
+        1   True
+        2   True
+        3  False
+        4   True
+        5  False
+        """
+        return self._reduce_for_stat_function(
+            lambda col: F.max(F.coalesce(col.cast("boolean"), F.lit(False))), only_numeric=False
+        )
+
+    # TODO: groupby multiply columns should be implemented.
+    def size(self) -> Series:
+        """
+        Compute group sizes.
+
+        See Also
+        --------
+        pyspark.pandas.Series.groupby
+        pyspark.pandas.DataFrame.groupby
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'A': [1, 2, 2, 3, 3, 3],
+        ...                    'B': [1, 1, 2, 3, 3, 3]},
+        ...                   columns=['A', 'B'])
+        >>> df
+           A  B
+        0  1  1
+        1  2  1
+        2  2  2
+        3  3  3
+        4  3  3
+        5  3  3
+
+        >>> df.groupby('A').size().sort_index()
+        A
+        1    1
+        2    2
+        3    3
+        dtype: int64
+
+        >>> df.groupby(['A', 'B']).size().sort_index()
+        A  B
+        1  1    1
+        2  1    1
+           2    1
+        3  3    3
+        dtype: int64
+
+        For Series,
+
+        >>> df.B.groupby(df.A).size().sort_index()
+        A
+        1    1
+        2    2
+        3    3
+        Name: B, dtype: int64
+
+        >>> df.groupby(df.A).B.size().sort_index()
+        A
+        1    1
+        2    2
+        3    3
+        Name: B, dtype: int64
+        """
+        groupkeys = self._groupkeys
+        groupkey_names = [SPARK_INDEX_NAME_FORMAT(i) for i in range(len(groupkeys))]
+        groupkey_scols = [s.spark.column.alias(name) for s, name in zip(groupkeys, groupkey_names)]
+        sdf = self._kdf._internal.spark_frame.select(
+            groupkey_scols + self._kdf._internal.data_spark_columns
+        )
+        sdf = sdf.groupby(*groupkey_names).count()
+        internal = InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=[scol_for(sdf, col) for col in groupkey_names],
+            index_names=[kser._column_label for kser in groupkeys],
+            index_dtypes=[kser.dtype for kser in groupkeys],
+            column_labels=[None],
+            data_spark_columns=[scol_for(sdf, "count")],
+        )
+        return first_series(DataFrame(internal))
+
+    def diff(self, periods=1) -> Union[DataFrame, Series]:
+        """
+        First discrete difference of element.
+
+        Calculates the difference of a DataFrame element compared with another element in the
+        DataFrame group (default is the element in the same column of the previous row).
+
+        Parameters
+        ----------
+        periods : int, default 1
+            Periods to shift for calculating difference, accepts negative values.
+
+        Returns
+        -------
+        diffed : DataFrame or Series
+
+        See Also
+        --------
+        pyspark.pandas.Series.groupby
+        pyspark.pandas.DataFrame.groupby
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'a': [1, 2, 3, 4, 5, 6],
+        ...                    'b': [1, 1, 2, 3, 5, 8],
+        ...                    'c': [1, 4, 9, 16, 25, 36]}, columns=['a', 'b', 'c'])
+        >>> df
+           a  b   c
+        0  1  1   1
+        1  2  1   4
+        2  3  2   9
+        3  4  3  16
+        4  5  5  25
+        5  6  8  36
+
+        >>> df.groupby(['b']).diff().sort_index()
+             a    c
+        0  NaN  NaN
+        1  1.0  3.0
+        2  NaN  NaN
+        3  NaN  NaN
+        4  NaN  NaN
+        5  NaN  NaN
+
+        Difference with previous column in a group.
+
+        >>> df.groupby(['b'])['a'].diff().sort_index()
+        0    NaN
+        1    1.0
+        2    NaN
+        3    NaN
+        4    NaN
+        5    NaN
+        Name: a, dtype: float64
+        """
+        return self._apply_series_op(
+            lambda sg: sg._kser._diff(periods, part_cols=sg._groupkeys_scols), should_resolve=True
+        )
+
+    def cumcount(self, ascending=True) -> Series:
+        """
+        Number each item in each group from 0 to the length of that group - 1.
+
+        Essentially this is equivalent to
+
+        .. code-block:: python
+
+            self.apply(lambda x: pd.Series(np.arange(len(x)), x.index))
+
+        Parameters
+        ----------
+        ascending : bool, default True
+            If False, number in reverse, from length of group - 1 to 0.
+
+        Returns
+        -------
+        Series
+            Sequence number of each element within each group.
+
+        Examples
+        --------
+
+        >>> df = ps.DataFrame([['a'], ['a'], ['a'], ['b'], ['b'], ['a']],
+        ...                   columns=['A'])
+        >>> df
+           A
+        0  a
+        1  a
+        2  a
+        3  b
+        4  b
+        5  a
+        >>> df.groupby('A').cumcount().sort_index()
+        0    0
+        1    1
+        2    2
+        3    0
+        4    1
+        5    3
+        dtype: int64
+        >>> df.groupby('A').cumcount(ascending=False).sort_index()
+        0    3
+        1    2
+        2    1
+        3    1
+        4    0
+        5    0
+        dtype: int64
+        """
+        ret = (
+            self._groupkeys[0]
+            .rename()
+            .spark.transform(lambda _: F.lit(0))
+            ._cum(F.count, True, part_cols=self._groupkeys_scols, ascending=ascending)
+            - 1
+        )
+        internal = ret._internal.resolved_copy
+        return first_series(DataFrame(internal))
+
+    def cummax(self) -> Union[DataFrame, Series]:
+        """
+        Cumulative max for each group.
+
+        Returns
+        -------
+        Series or DataFrame
+
+        See Also
+        --------
+        Series.cummax
+        DataFrame.cummax
+
+        Examples
+        --------
+        >>> df = ps.DataFrame(
+        ...     [[1, None, 4], [1, 0.1, 3], [1, 20.0, 2], [4, 10.0, 1]],
+        ...     columns=list('ABC'))
+        >>> df
+           A     B  C
+        0  1   NaN  4
+        1  1   0.1  3
+        2  1  20.0  2
+        3  4  10.0  1
+
+        By default, iterates over rows and finds the sum in each column.
+
+        >>> df.groupby("A").cummax().sort_index()
+              B  C
+        0   NaN  4
+        1   0.1  4
+        2  20.0  4
+        3  10.0  1
+
+        It works as below in Series.
+
+        >>> df.C.groupby(df.A).cummax().sort_index()
+        0    4
+        1    4
+        2    4
+        3    1
+        Name: C, dtype: int64
+        """
+        return self._apply_series_op(
+            lambda sg: sg._kser._cum(F.max, True, part_cols=sg._groupkeys_scols),
+            should_resolve=True,
+            numeric_only=True,
+        )
+
+    def cummin(self) -> Union[DataFrame, Series]:
+        """
+        Cumulative min for each group.
+
+        Returns
+        -------
+        Series or DataFrame
+
+        See Also
+        --------
+        Series.cummin
+        DataFrame.cummin
+
+        Examples
+        --------
+        >>> df = ps.DataFrame(
+        ...     [[1, None, 4], [1, 0.1, 3], [1, 20.0, 2], [4, 10.0, 1]],
+        ...     columns=list('ABC'))
+        >>> df
+           A     B  C
+        0  1   NaN  4
+        1  1   0.1  3
+        2  1  20.0  2
+        3  4  10.0  1
+
+        By default, iterates over rows and finds the sum in each column.
+
+        >>> df.groupby("A").cummin().sort_index()
+              B  C
+        0   NaN  4
+        1   0.1  3
+        2   0.1  2
+        3  10.0  1
+
+        It works as below in Series.
+
+        >>> df.B.groupby(df.A).cummin().sort_index()
+        0     NaN
+        1     0.1
+        2     0.1
+        3    10.0
+        Name: B, dtype: float64
+        """
+        return self._apply_series_op(
+            lambda sg: sg._kser._cum(F.min, True, part_cols=sg._groupkeys_scols),
+            should_resolve=True,
+            numeric_only=True,
+        )
+
+    def cumprod(self) -> Union[DataFrame, Series]:
+        """
+        Cumulative product for each group.
+
+        Returns
+        -------
+        Series or DataFrame
+
+        See Also
+        --------
+        Series.cumprod
+        DataFrame.cumprod
+
+        Examples
+        --------
+        >>> df = ps.DataFrame(
+        ...     [[1, None, 4], [1, 0.1, 3], [1, 20.0, 2], [4, 10.0, 1]],
+        ...     columns=list('ABC'))
+        >>> df
+           A     B  C
+        0  1   NaN  4
+        1  1   0.1  3
+        2  1  20.0  2
+        3  4  10.0  1
+
+        By default, iterates over rows and finds the sum in each column.
+
+        >>> df.groupby("A").cumprod().sort_index()
+              B   C
+        0   NaN   4
+        1   0.1  12
+        2   2.0  24
+        3  10.0   1
+
+        It works as below in Series.
+
+        >>> df.B.groupby(df.A).cumprod().sort_index()
+        0     NaN
+        1     0.1
+        2     2.0
+        3    10.0
+        Name: B, dtype: float64
+        """
+        return self._apply_series_op(
+            lambda sg: sg._kser._cumprod(True, part_cols=sg._groupkeys_scols),
+            should_resolve=True,
+            numeric_only=True,
+        )
+
+    def cumsum(self) -> Union[DataFrame, Series]:
+        """
+        Cumulative sum for each group.
+
+        Returns
+        -------
+        Series or DataFrame
+
+        See Also
+        --------
+        Series.cumsum
+        DataFrame.cumsum
+
+        Examples
+        --------
+        >>> df = ps.DataFrame(
+        ...     [[1, None, 4], [1, 0.1, 3], [1, 20.0, 2], [4, 10.0, 1]],
+        ...     columns=list('ABC'))
+        >>> df
+           A     B  C
+        0  1   NaN  4
+        1  1   0.1  3
+        2  1  20.0  2
+        3  4  10.0  1
+
+        By default, iterates over rows and finds the sum in each column.
+
+        >>> df.groupby("A").cumsum().sort_index()
+              B  C
+        0   NaN  4
+        1   0.1  7
+        2  20.1  9
+        3  10.0  1
+
+        It works as below in Series.
+
+        >>> df.B.groupby(df.A).cumsum().sort_index()
+        0     NaN
+        1     0.1
+        2    20.1
+        3    10.0
+        Name: B, dtype: float64
+        """
+        return self._apply_series_op(
+            lambda sg: sg._kser._cumsum(True, part_cols=sg._groupkeys_scols),
+            should_resolve=True,
+            numeric_only=True,
+        )
+
+    def apply(self, func, *args, **kwargs) -> Union[DataFrame, Series]:
+        """
+        Apply function `func` group-wise and combine the results together.
+
+        The function passed to `apply` must take a DataFrame as its first
+        argument and return a DataFrame. `apply` will
+        then take care of combining the results back together into a single
+        dataframe. `apply` is therefore a highly flexible
+        grouping method.
+
+        While `apply` is a very flexible method, its downside is that
+        using it can be quite a bit slower than using more specific methods
+        like `agg` or `transform`. pandas-on-Spark offers a wide range of method that will
+        be much faster than using `apply` for their specific purposes, so try to
+        use them before reaching for `apply`.
+
+        .. note:: this API executes the function once to infer the type which is
+            potentially expensive, for instance, when the dataset is created after
+            aggregations or sorting.
+
+            To avoid this, specify return type in ``func``, for instance, as below:
+
+            >>> def pandas_div(x) -> ps.DataFrame[float, float]:
+            ...     return x[['B', 'C']] / x[['B', 'C']]
+
+            If the return type is specified, the output column names become
+            `c0, c1, c2 ... cn`. These names are positionally mapped to the returned
+            DataFrame in ``func``.
+
+            To specify the column names, you can assign them in a pandas friendly style as below:
+
+            >>> def pandas_div(x) -> ps.DataFrame["a": float, "b": float]:
+            ...     return x[['B', 'C']] / x[['B', 'C']]
+
+            >>> pdf = pd.DataFrame({'B': [1.], 'C': [3.]})
+            >>> def plus_one(x) -> ps.DataFrame[zip(pdf.columns, pdf.dtypes)]:
+            ...     return x[['B', 'C']] / x[['B', 'C']]
+
+            When the given function has the return type annotated, the original index of the
+            GroupBy object will be lost and a default index will be attached to the result.
+            Please be careful about configuring the default index. See also `Default Index Type
+            <https://koalas.readthedocs.io/en/latest/user_guide/options.html#default-index-type>`_.
+
+        .. note:: the dataframe within ``func`` is actually a pandas dataframe. Therefore,
+            any pandas APIs within this function is allowed.
+
+        Parameters
+        ----------
+        func : callable
+            A callable that takes a DataFrame as its first argument, and
+            returns a dataframe.
+        *args
+            Positional arguments to pass to func.
+        **kwargs
+            Keyword arguments to pass to func.
+
+        Returns
+        -------
+        applied : DataFrame or Series
+
+        See Also
+        --------
+        aggregate : Apply aggregate function to the GroupBy object.
+        DataFrame.apply : Apply a function to a DataFrame.
+        Series.apply : Apply a function to a Series.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'A': 'a a b'.split(),
+        ...                    'B': [1, 2, 3],
+        ...                    'C': [4, 6, 5]}, columns=['A', 'B', 'C'])
+        >>> g = df.groupby('A')
+
+        Notice that ``g`` has two groups, ``a`` and ``b``.
+        Calling `apply` in various ways, we can get different grouping results:
+
+        Below the functions passed to `apply` takes a DataFrame as
+        its argument and returns a DataFrame. `apply` combines the result for
+        each group together into a new DataFrame:
+
+        >>> def plus_min(x):
+        ...     return x + x.min()
+        >>> g.apply(plus_min).sort_index()  # doctest: +NORMALIZE_WHITESPACE
+            A  B   C
+        0  aa  2   8
+        1  aa  3  10
+        2  bb  6  10
+
+        >>> g.apply(sum).sort_index()  # doctest: +NORMALIZE_WHITESPACE
+            A  B   C
+        A
+        a  aa  3  10
+        b   b  3   5
+
+        >>> g.apply(len).sort_index()  # doctest: +NORMALIZE_WHITESPACE
+        A
+        a    2
+        b    1
+        dtype: int64
+
+        You can specify the type hint and prevent schema inference for better performance.
+
+        >>> def pandas_div(x) -> ps.DataFrame[float, float]:
+        ...     return x[['B', 'C']] / x[['B', 'C']]
+        >>> g.apply(pandas_div).sort_index()  # doctest: +NORMALIZE_WHITESPACE
+            c0   c1
+        0  1.0  1.0
+        1  1.0  1.0
+        2  1.0  1.0
+
+        In case of Series, it works as below.
+
+        >>> def plus_max(x) -> ps.Series[np.int]:
+        ...     return x + x.max()
+        >>> df.B.groupby(df.A).apply(plus_max).sort_index()  # doctest: +SKIP
+        0    6
+        1    3
+        2    4
+        Name: B, dtype: int64
+
+        >>> def plus_min(x):
+        ...     return x + x.min()
+        >>> df.B.groupby(df.A).apply(plus_min).sort_index()
+        0    2
+        1    3
+        2    6
+        Name: B, dtype: int64
+
+        You can also return a scalar value as a aggregated value of the group:
+
+        >>> def plus_length(x) -> np.int:
+        ...     return len(x)
+        >>> df.B.groupby(df.A).apply(plus_length).sort_index()  # doctest: +SKIP
+        0    1
+        1    2
+        Name: B, dtype: int64
+
+        The extra arguments to the function can be passed as below.
+
+        >>> def calculation(x, y, z) -> np.int:
+        ...     return len(x) + y * z
+        >>> df.B.groupby(df.A).apply(calculation, 5, z=10).sort_index()  # doctest: +SKIP
+        0    51
+        1    52
+        Name: B, dtype: int64
+        """
+        from pandas.core.base import SelectionMixin
+
+        if not isinstance(func, Callable):  # type: ignore
+            raise TypeError("%s object is not callable" % type(func).__name__)
+
+        spec = inspect.getfullargspec(func)
+        return_sig = spec.annotations.get("return", None)
+        should_infer_schema = return_sig is None
+
+        is_series_groupby = isinstance(self, SeriesGroupBy)
+
+        kdf = self._kdf
+
+        if self._agg_columns_selected:
+            agg_columns = self._agg_columns
+        else:
+            agg_columns = [
+                kdf._kser_for(label)
+                for label in kdf._internal.column_labels
+                if label not in self._column_labels_to_exlcude
+            ]
+
+        kdf, groupkey_labels, groupkey_names = GroupBy._prepare_group_map_apply(
+            kdf, self._groupkeys, agg_columns
+        )
+
+        if is_series_groupby:
+            name = kdf.columns[-1]
+            pandas_apply = SelectionMixin._builtin_table.get(func, func)
+        else:
+            f = SelectionMixin._builtin_table.get(func, func)
+
+            def pandas_apply(pdf, *a, **k):
+                return f(pdf.drop(groupkey_names, axis=1), *a, **k)
+
+        should_return_series = False
+
+        if should_infer_schema:
+            # Here we execute with the first 1000 to get the return type.
+            limit = get_option("compute.shortcut_limit")
+            pdf = kdf.head(limit + 1)._to_internal_pandas()
+            groupkeys = [
+                pdf[groupkey_name].rename(kser.name)
+                for groupkey_name, kser in zip(groupkey_names, self._groupkeys)
+            ]
+            if is_series_groupby:
+                pser_or_pdf = pdf.groupby(groupkeys)[name].apply(pandas_apply, *args, **kwargs)
+            else:
+                pser_or_pdf = pdf.groupby(groupkeys).apply(pandas_apply, *args, **kwargs)
+            kser_or_kdf = ps.from_pandas(pser_or_pdf)
+
+            if len(pdf) <= limit:
+                if isinstance(kser_or_kdf, ps.Series) and is_series_groupby:
+                    kser_or_kdf = kser_or_kdf.rename(cast(SeriesGroupBy, self)._kser.name)
+                return cast(Union[Series, DataFrame], kser_or_kdf)
+
+            if isinstance(kser_or_kdf, Series):
+                should_return_series = True
+                kdf_from_pandas = kser_or_kdf._kdf
+            else:
+                kdf_from_pandas = cast(DataFrame, kser_or_kdf)
+
+            return_schema = force_decimal_precision_scale(
+                as_nullable_spark_type(
+                    kdf_from_pandas._internal.spark_frame.drop(*HIDDEN_COLUMNS).schema
+                )
+            )
+        else:
+            return_type = infer_return_type(func)
+            if not is_series_groupby and isinstance(return_type, SeriesType):
+                raise TypeError(
+                    "Series as a return type hint at frame groupby is not supported "
+                    "currently; however got [%s]. Use DataFrame type hint instead." % return_sig
+                )
+
+            if isinstance(return_type, DataFrameType):
+                return_schema = cast(DataFrameType, return_type).spark_type
+                data_dtypes = cast(DataFrameType, return_type).dtypes
+            else:
+                should_return_series = True
+                return_schema = cast(Union[SeriesType, ScalarType], return_type).spark_type
+                if is_series_groupby:
+                    return_schema = StructType([StructField(name, return_schema)])
+                else:
+                    return_schema = StructType(
+                        [StructField(SPARK_DEFAULT_SERIES_NAME, return_schema)]
+                    )
+                data_dtypes = [cast(Union[SeriesType, ScalarType], return_type).dtype]
+
+        def pandas_groupby_apply(pdf):
+
+            if not is_series_groupby and LooseVersion(pd.__version__) < LooseVersion("0.25"):
+                # `groupby.apply` in pandas<0.25 runs the functions twice for the first group.
+                # https://github.com/pandas-dev/pandas/pull/24748
+
+                should_skip_first_call = True
+
+                def wrapped_func(df, *a, **k):
+                    nonlocal should_skip_first_call
+                    if should_skip_first_call:
+                        should_skip_first_call = False
+                        if should_return_series:
+                            return pd.Series()
+                        else:
+                            return pd.DataFrame()
+                    else:
+                        return pandas_apply(df, *a, **k)
+
+            else:
+                wrapped_func = pandas_apply
+
+            if is_series_groupby:
+                pdf_or_ser = pdf.groupby(groupkey_names)[name].apply(wrapped_func, *args, **kwargs)
+            else:
+                pdf_or_ser = pdf.groupby(groupkey_names).apply(wrapped_func, *args, **kwargs)
+
+            if not isinstance(pdf_or_ser, pd.DataFrame):
+                return pd.DataFrame(pdf_or_ser)
+            else:
+                return pdf_or_ser
+
+        sdf = GroupBy._spark_group_map_apply(
+            kdf,
+            pandas_groupby_apply,
+            [kdf._internal.spark_column_for(label) for label in groupkey_labels],
+            return_schema,
+            retain_index=should_infer_schema,
+        )
+
+        if should_infer_schema:
+            # If schema is inferred, we can restore indexes too.
+            internal = kdf_from_pandas._internal.with_new_sdf(sdf)
+        else:
+            # Otherwise, it loses index.
+            internal = InternalFrame(
+                spark_frame=sdf, index_spark_columns=None, data_dtypes=data_dtypes
+            )
+
+        if should_return_series:
+            kser = first_series(DataFrame(internal))
+            if is_series_groupby:
+                kser = kser.rename(cast(SeriesGroupBy, self)._kser.name)
+            return kser
+        else:
+            return DataFrame(internal)
+
+    # TODO: implement 'dropna' parameter
+    def filter(self, func) -> Union[DataFrame, Series]:
+        """
+        Return a copy of a DataFrame excluding elements from groups that
+        do not satisfy the boolean criterion specified by func.
+
+        Parameters
+        ----------
+        f : function
+            Function to apply to each subframe. Should return True or False.
+        dropna : Drop groups that do not pass the filter. True by default;
+            if False, groups that evaluate False are filled with NaNs.
+
+        Returns
+        -------
+        filtered : DataFrame or Series
+
+        Notes
+        -----
+        Each subframe is endowed the attribute 'name' in case you need to know
+        which group you are working on.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'A' : ['foo', 'bar', 'foo', 'bar',
+        ...                           'foo', 'bar'],
+        ...                    'B' : [1, 2, 3, 4, 5, 6],
+        ...                    'C' : [2.0, 5., 8., 1., 2., 9.]}, columns=['A', 'B', 'C'])
+        >>> grouped = df.groupby('A')
+        >>> grouped.filter(lambda x: x['B'].mean() > 3.)
+             A  B    C
+        1  bar  2  5.0
+        3  bar  4  1.0
+        5  bar  6  9.0
+
+        >>> df.B.groupby(df.A).filter(lambda x: x.mean() > 3.)
+        1    2
+        3    4
+        5    6
+        Name: B, dtype: int64
+        """
+        from pandas.core.base import SelectionMixin
+
+        if not isinstance(func, Callable):  # type: ignore
+            raise TypeError("%s object is not callable" % type(func).__name__)
+
+        is_series_groupby = isinstance(self, SeriesGroupBy)
+
+        kdf = self._kdf
+
+        if self._agg_columns_selected:
+            agg_columns = self._agg_columns
+        else:
+            agg_columns = [
+                kdf._kser_for(label)
+                for label in kdf._internal.column_labels
+                if label not in self._column_labels_to_exlcude
+            ]
+
+        data_schema = (
+            kdf[agg_columns]._internal.resolved_copy.spark_frame.drop(*HIDDEN_COLUMNS).schema
+        )
+
+        kdf, groupkey_labels, groupkey_names = GroupBy._prepare_group_map_apply(
+            kdf, self._groupkeys, agg_columns
+        )
+
+        if is_series_groupby:
+
+            def pandas_filter(pdf):
+                return pd.DataFrame(pdf.groupby(groupkey_names)[pdf.columns[-1]].filter(func))
+
+        else:
+            f = SelectionMixin._builtin_table.get(func, func)
+
+            def wrapped_func(pdf):
+                return f(pdf.drop(groupkey_names, axis=1))
+
+            def pandas_filter(pdf):
+                return pdf.groupby(groupkey_names).filter(wrapped_func).drop(groupkey_names, axis=1)
+
+        sdf = GroupBy._spark_group_map_apply(
+            kdf,
+            pandas_filter,
+            [kdf._internal.spark_column_for(label) for label in groupkey_labels],
+            data_schema,
+            retain_index=True,
+        )
+
+        kdf = DataFrame(self._kdf[agg_columns]._internal.with_new_sdf(sdf))
+        if is_series_groupby:
+            return first_series(kdf)
+        else:
+            return kdf
+
+    @staticmethod
+    def _prepare_group_map_apply(kdf, groupkeys, agg_columns):
+        groupkey_labels = [
+            verify_temp_column_name(kdf, "__groupkey_{}__".format(i)) for i in range(len(groupkeys))
+        ]
+        kdf = kdf[[s.rename(label) for s, label in zip(groupkeys, groupkey_labels)] + agg_columns]
+        groupkey_names = [label if len(label) > 1 else label[0] for label in groupkey_labels]
+        return DataFrame(kdf._internal.resolved_copy), groupkey_labels, groupkey_names
+
+    @staticmethod
+    def _spark_group_map_apply(kdf, func, groupkeys_scols, return_schema, retain_index):
+        output_func = GroupBy._make_pandas_df_builder_func(kdf, func, return_schema, retain_index)
+        grouped_map_func = pandas_udf(return_schema, PandasUDFType.GROUPED_MAP)(output_func)
+        sdf = kdf._internal.spark_frame.drop(*HIDDEN_COLUMNS)
+        return sdf.groupby(*groupkeys_scols).apply(grouped_map_func)
+
+    @staticmethod
+    def _make_pandas_df_builder_func(kdf, func, return_schema, retain_index):
+        """
+        Creates a function that can be used inside the pandas UDF. This function can construct
+        the same pandas DataFrame as if the pandas-on-Spark DataFrame is collected to driver side.
+        The index, column labels, etc. are re-constructed within the function.
+        """
+        arguments_for_restore_index = kdf._internal.arguments_for_restore_index
+
+        def rename_output(pdf):
+            pdf = InternalFrame.restore_index(pdf.copy(), **arguments_for_restore_index)
+
+            pdf = func(pdf)
+
+            # If schema should be inferred, we don't restore index. pandas seems restoring
+            # the index in some cases.
+            # When Spark output type is specified, without executing it, we don't know
+            # if we should restore the index or not. For instance, see the example in
+            # https://github.com/pyspark.pandas/issues/628.
+            pdf, _, _, _, _ = InternalFrame.prepare_pandas_frame(pdf, retain_index=retain_index)
+
+            # Just positionally map the column names to given schema's.
+            pdf.columns = return_schema.names
+
+            return pdf
+
+        return rename_output
+
+    def rank(self, method="average", ascending=True) -> Union[DataFrame, Series]:
+        """
+        Provide the rank of values within each group.
+
+        Parameters
+        ----------
+        method : {'average', 'min', 'max', 'first', 'dense'}, default 'average'
+            * average: average rank of group
+            * min: lowest rank in group
+            * max: highest rank in group
+            * first: ranks assigned in order they appear in the array
+            * dense: like 'min', but rank always increases by 1 between groups
+        ascending : boolean, default True
+            False for ranks by high (1) to low (N)
+
+        Returns
+        -------
+        DataFrame with ranking of values within each group
+
+        Examples
+        --------
+
+        >>> df = ps.DataFrame({
+        ...     'a': [1, 1, 1, 2, 2, 2, 3, 3, 3],
+        ...     'b': [1, 2, 2, 2, 3, 3, 3, 4, 4]}, columns=['a', 'b'])
+        >>> df
+           a  b
+        0  1  1
+        1  1  2
+        2  1  2
+        3  2  2
+        4  2  3
+        5  2  3
+        6  3  3
+        7  3  4
+        8  3  4
+
+        >>> df.groupby("a").rank().sort_index()
+             b
+        0  1.0
+        1  2.5
+        2  2.5
+        3  1.0
+        4  2.5
+        5  2.5
+        6  1.0
+        7  2.5
+        8  2.5
+
+        >>> df.b.groupby(df.a).rank(method='max').sort_index()
+        0    1.0
+        1    3.0
+        2    3.0
+        3    1.0
+        4    3.0
+        5    3.0
+        6    1.0
+        7    3.0
+        8    3.0
+        Name: b, dtype: float64
+
+        """
+        return self._apply_series_op(
+            lambda sg: sg._kser._rank(method, ascending, part_cols=sg._groupkeys_scols),
+            should_resolve=True,
+        )
+
+    # TODO: add axis parameter
+    def idxmax(self, skipna=True) -> Union[DataFrame, Series]:
+        """
+        Return index of first occurrence of maximum over requested axis in group.
+        NA/null values are excluded.
+
+        Parameters
+        ----------
+        skipna : boolean, default True
+            Exclude NA/null values. If an entire row/column is NA, the result will be NA.
+
+        See Also
+        --------
+        Series.idxmax
+        DataFrame.idxmax
+        pyspark.pandas.Series.groupby
+        pyspark.pandas.DataFrame.groupby
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'a': [1, 1, 2, 2, 3],
+        ...                    'b': [1, 2, 3, 4, 5],
+        ...                    'c': [5, 4, 3, 2, 1]}, columns=['a', 'b', 'c'])
+
+        >>> df.groupby(['a'])['b'].idxmax().sort_index() # doctest: +NORMALIZE_WHITESPACE
+        a
+        1  1
+        2  3
+        3  4
+        Name: b, dtype: int64
+
+        >>> df.groupby(['a']).idxmax().sort_index() # doctest: +NORMALIZE_WHITESPACE
+           b  c
+        a
+        1  1  0
+        2  3  2
+        3  4  4
+        """
+        if self._kdf._internal.index_level != 1:
+            raise ValueError("idxmax only support one-level index now")
+
+        groupkey_names = ["__groupkey_{}__".format(i) for i in range(len(self._groupkeys))]
+
+        sdf = self._kdf._internal.spark_frame
+        for s, name in zip(self._groupkeys, groupkey_names):
+            sdf = sdf.withColumn(name, s.spark.column)
+        index = self._kdf._internal.index_spark_column_names[0]
+
+        stat_exprs = []
+        for kser, c in zip(self._agg_columns, self._agg_columns_scols):
+            name = kser._internal.data_spark_column_names[0]
+
+            if skipna:
+                order_column = Column(c._jc.desc_nulls_last())
+            else:
+                order_column = Column(c._jc.desc_nulls_first())
+            window = Window.partitionBy(groupkey_names).orderBy(
+                order_column, NATURAL_ORDER_COLUMN_NAME
+            )
+            sdf = sdf.withColumn(
+                name, F.when(F.row_number().over(window) == 1, scol_for(sdf, index)).otherwise(None)
+            )
+            stat_exprs.append(F.max(scol_for(sdf, name)).alias(name))
+
+        sdf = sdf.groupby(*groupkey_names).agg(*stat_exprs)
+
+        internal = InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=[scol_for(sdf, col) for col in groupkey_names],
+            index_names=[kser._column_label for kser in self._groupkeys],
+            index_dtypes=[kser.dtype for kser in self._groupkeys],
+            column_labels=[kser._column_label for kser in self._agg_columns],
+            data_spark_columns=[
+                scol_for(sdf, kser._internal.data_spark_column_names[0])
+                for kser in self._agg_columns
+            ],
+        )
+        return DataFrame(internal)
+
+    # TODO: add axis parameter
+    def idxmin(self, skipna=True) -> Union[DataFrame, Series]:
+        """
+        Return index of first occurrence of minimum over requested axis in group.
+        NA/null values are excluded.
+
+        Parameters
+        ----------
+        skipna : boolean, default True
+            Exclude NA/null values. If an entire row/column is NA, the result will be NA.
+
+        See Also
+        --------
+        Series.idxmin
+        DataFrame.idxmin
+        pyspark.pandas.Series.groupby
+        pyspark.pandas.DataFrame.groupby
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'a': [1, 1, 2, 2, 3],
+        ...                    'b': [1, 2, 3, 4, 5],
+        ...                    'c': [5, 4, 3, 2, 1]}, columns=['a', 'b', 'c'])
+
+        >>> df.groupby(['a'])['b'].idxmin().sort_index() # doctest: +NORMALIZE_WHITESPACE
+        a
+        1    0
+        2    2
+        3    4
+        Name: b, dtype: int64
+
+        >>> df.groupby(['a']).idxmin().sort_index() # doctest: +NORMALIZE_WHITESPACE
+           b  c
+        a
+        1  0  1
+        2  2  3
+        3  4  4
+        """
+        if self._kdf._internal.index_level != 1:
+            raise ValueError("idxmin only support one-level index now")
+
+        groupkey_names = ["__groupkey_{}__".format(i) for i in range(len(self._groupkeys))]
+
+        sdf = self._kdf._internal.spark_frame
+        for s, name in zip(self._groupkeys, groupkey_names):
+            sdf = sdf.withColumn(name, s.spark.column)
+        index = self._kdf._internal.index_spark_column_names[0]
+
+        stat_exprs = []
+        for kser, c in zip(self._agg_columns, self._agg_columns_scols):
+            name = kser._internal.data_spark_column_names[0]
+
+            if skipna:
+                order_column = Column(c._jc.asc_nulls_last())
+            else:
+                order_column = Column(c._jc.asc_nulls_first())
+            window = Window.partitionBy(groupkey_names).orderBy(
+                order_column, NATURAL_ORDER_COLUMN_NAME
+            )
+            sdf = sdf.withColumn(
+                name, F.when(F.row_number().over(window) == 1, scol_for(sdf, index)).otherwise(None)
+            )
+            stat_exprs.append(F.max(scol_for(sdf, name)).alias(name))
+
+        sdf = sdf.groupby(*groupkey_names).agg(*stat_exprs)
+
+        internal = InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=[scol_for(sdf, col) for col in groupkey_names],
+            index_names=[kser._column_label for kser in self._groupkeys],
+            column_labels=[kser._column_label for kser in self._agg_columns],
+            data_spark_columns=[
+                scol_for(sdf, kser._internal.data_spark_column_names[0])
+                for kser in self._agg_columns
+            ],
+        )
+        return DataFrame(internal)
+
+    def fillna(
+        self, value=None, method=None, axis=None, inplace=False, limit=None
+    ) -> Union[DataFrame, Series]:
+        """Fill NA/NaN values in group.
+
+        Parameters
+        ----------
+        value : scalar, dict, Series
+            Value to use to fill holes. alternately a dict/Series of values
+            specifying which value to use for each column.
+            DataFrame is not supported.
+        method : {'backfill', 'bfill', 'pad', 'ffill', None}, default None
+            Method to use for filling holes in reindexed Series pad / ffill: propagate last valid
+            observation forward to next valid backfill / bfill:
+            use NEXT valid observation to fill gap
+        axis : {0 or `index`}
+            1 and `columns` are not supported.
+        inplace : boolean, default False
+            Fill in place (do not create a new object)
+        limit : int, default None
+            If method is specified, this is the maximum number of consecutive NaN values to
+            forward/backward fill. In other words, if there is a gap with more than this number of
+            consecutive NaNs, it will only be partially filled. If method is not specified,
+            this is the maximum number of entries along the entire axis where NaNs will be filled.
+            Must be greater than 0 if not None
+
+        Returns
+        -------
+        DataFrame
+            DataFrame with NA entries filled.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({
+        ...     'A': [1, 1, 2, 2],
+        ...     'B': [2, 4, None, 3],
+        ...     'C': [None, None, None, 1],
+        ...     'D': [0, 1, 5, 4]
+        ...     },
+        ...     columns=['A', 'B', 'C', 'D'])
+        >>> df
+           A    B    C  D
+        0  1  2.0  NaN  0
+        1  1  4.0  NaN  1
+        2  2  NaN  NaN  5
+        3  2  3.0  1.0  4
+
+        We can also propagate non-null values forward or backward in group.
+
+        >>> df.groupby(['A'])['B'].fillna(method='ffill').sort_index()
+        0    2.0
+        1    4.0
+        2    NaN
+        3    3.0
+        Name: B, dtype: float64
+
+        >>> df.groupby(['A']).fillna(method='bfill').sort_index()
+             B    C  D
+        0  2.0  NaN  0
+        1  4.0  NaN  1
+        2  3.0  1.0  5
+        3  3.0  1.0  4
+        """
+        return self._apply_series_op(
+            lambda sg: sg._kser._fillna(
+                value=value, method=method, axis=axis, limit=limit, part_cols=sg._groupkeys_scols
+            ),
+            should_resolve=(method is not None),
+        )
+
+    def bfill(self, limit=None) -> Union[DataFrame, Series]:
+        """
+        Synonym for `DataFrame.fillna()` with ``method=`bfill```.
+
+        Parameters
+        ----------
+        axis : {0 or `index`}
+            1 and `columns` are not supported.
+        inplace : boolean, default False
+            Fill in place (do not create a new object)
+        limit : int, default None
+            If method is specified, this is the maximum number of consecutive NaN values to
+            forward/backward fill. In other words, if there is a gap with more than this number of
+            consecutive NaNs, it will only be partially filled. If method is not specified,
+            this is the maximum number of entries along the entire axis where NaNs will be filled.
+            Must be greater than 0 if not None
+
+        Returns
+        -------
+        DataFrame
+            DataFrame with NA entries filled.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({
+        ...     'A': [1, 1, 2, 2],
+        ...     'B': [2, 4, None, 3],
+        ...     'C': [None, None, None, 1],
+        ...     'D': [0, 1, 5, 4]
+        ...     },
+        ...     columns=['A', 'B', 'C', 'D'])
+        >>> df
+           A    B    C  D
+        0  1  2.0  NaN  0
+        1  1  4.0  NaN  1
+        2  2  NaN  NaN  5
+        3  2  3.0  1.0  4
+
+        Propagate non-null values backward.
+
+        >>> df.groupby(['A']).bfill().sort_index()
+             B    C  D
+        0  2.0  NaN  0
+        1  4.0  NaN  1
+        2  3.0  1.0  5
+        3  3.0  1.0  4
+        """
+        return self.fillna(method="bfill", limit=limit)
+
+    backfill = bfill
+
+    def ffill(self, limit=None) -> Union[DataFrame, Series]:
+        """
+        Synonym for `DataFrame.fillna()` with ``method=`ffill```.
+
+        Parameters
+        ----------
+        axis : {0 or `index`}
+            1 and `columns` are not supported.
+        inplace : boolean, default False
+            Fill in place (do not create a new object)
+        limit : int, default None
+            If method is specified, this is the maximum number of consecutive NaN values to
+            forward/backward fill. In other words, if there is a gap with more than this number of
+            consecutive NaNs, it will only be partially filled. If method is not specified,
+            this is the maximum number of entries along the entire axis where NaNs will be filled.
+            Must be greater than 0 if not None
+
+        Returns
+        -------
+        DataFrame
+            DataFrame with NA entries filled.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({
+        ...     'A': [1, 1, 2, 2],
+        ...     'B': [2, 4, None, 3],
+        ...     'C': [None, None, None, 1],
+        ...     'D': [0, 1, 5, 4]
+        ...     },
+        ...     columns=['A', 'B', 'C', 'D'])
+        >>> df
+           A    B    C  D
+        0  1  2.0  NaN  0
+        1  1  4.0  NaN  1
+        2  2  NaN  NaN  5
+        3  2  3.0  1.0  4
+
+        Propagate non-null values forward.
+
+        >>> df.groupby(['A']).ffill().sort_index()
+             B    C  D
+        0  2.0  NaN  0
+        1  4.0  NaN  1
+        2  NaN  NaN  5
+        3  3.0  1.0  4
+        """
+        return self.fillna(method="ffill", limit=limit)
+
+    pad = ffill
+
+    def _limit(self, n: int, asc: bool):
+        """
+        Private function for tail and head.
+        """
+        kdf = self._kdf
+
+        if self._agg_columns_selected:
+            agg_columns = self._agg_columns
+        else:
+            agg_columns = [
+                kdf._kser_for(label)
+                for label in kdf._internal.column_labels
+                if label not in self._column_labels_to_exlcude
+            ]
+
+        kdf, groupkey_labels, _ = GroupBy._prepare_group_map_apply(
+            kdf, self._groupkeys, agg_columns,
+        )
+
+        groupkey_scols = [kdf._internal.spark_column_for(label) for label in groupkey_labels]
+
+        sdf = kdf._internal.spark_frame
+        tmp_col = verify_temp_column_name(sdf, "__row_number__")
+
+        # This part is handled differently depending on whether it is a tail or a head.
+        window = (
+            Window.partitionBy(groupkey_scols).orderBy(F.col(NATURAL_ORDER_COLUMN_NAME).asc())
+            if asc
+            else Window.partitionBy(groupkey_scols).orderBy(F.col(NATURAL_ORDER_COLUMN_NAME).desc())
+        )
+
+        sdf = (
+            sdf.withColumn(tmp_col, F.row_number().over(window))
+            .filter(F.col(tmp_col) <= n)
+            .drop(tmp_col)
+        )
+
+        internal = kdf._internal.with_new_sdf(sdf)
+        return DataFrame(internal).drop(groupkey_labels, axis=1)
+
+    def head(self, n=5) -> Union[DataFrame, Series]:
+        """
+        Return first n rows of each group.
+
+        Returns
+        -------
+        DataFrame or Series
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'a': [1, 1, 1, 1, 2, 2, 2, 3, 3, 3],
+        ...                    'b': [2, 3, 1, 4, 6, 9, 8, 10, 7, 5],
+        ...                    'c': [3, 5, 2, 5, 1, 2, 6, 4, 3, 6]},
+        ...                   columns=['a', 'b', 'c'],
+        ...                   index=[7, 2, 4, 1, 3, 4, 9, 10, 5, 6])
+        >>> df
+            a   b  c
+        7   1   2  3
+        2   1   3  5
+        4   1   1  2
+        1   1   4  5
+        3   2   6  1
+        4   2   9  2
+        9   2   8  6
+        10  3  10  4
+        5   3   7  3
+        6   3   5  6
+
+        >>> df.groupby('a').head(2).sort_index()
+            a   b  c
+        2   1   3  5
+        3   2   6  1
+        4   2   9  2
+        5   3   7  3
+        7   1   2  3
+        10  3  10  4
+
+        >>> df.groupby('a')['b'].head(2).sort_index()
+        2      3
+        3      6
+        4      9
+        5      7
+        7      2
+        10    10
+        Name: b, dtype: int64
+        """
+        return self._limit(n, asc=True)
+
+    def tail(self, n=5) -> Union[DataFrame, Series]:
+        """
+        Return last n rows of each group.
+
+        Similar to `.apply(lambda x: x.tail(n))`, but it returns a subset of rows from
+        the original DataFrame with original index and order preserved (`as_index` flag is ignored).
+
+        Does not work for negative values of n.
+
+        Returns
+        -------
+        DataFrame or Series
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'a': [1, 1, 1, 1, 2, 2, 2, 3, 3, 3],
+        ...                    'b': [2, 3, 1, 4, 6, 9, 8, 10, 7, 5],
+        ...                    'c': [3, 5, 2, 5, 1, 2, 6, 4, 3, 6]},
+        ...                   columns=['a', 'b', 'c'],
+        ...                   index=[7, 2, 3, 1, 3, 4, 9, 10, 5, 6])
+        >>> df
+            a   b  c
+        7   1   2  3
+        2   1   3  5
+        3   1   1  2
+        1   1   4  5
+        3   2   6  1
+        4   2   9  2
+        9   2   8  6
+        10  3  10  4
+        5   3   7  3
+        6   3   5  6
+
+        >>> df.groupby('a').tail(2).sort_index()
+           a  b  c
+        1  1  4  5
+        3  1  1  2
+        4  2  9  2
+        5  3  7  3
+        6  3  5  6
+        9  2  8  6
+
+        >>> df.groupby('a')['b'].tail(2).sort_index()
+        1    4
+        3    1
+        4    9
+        5    7
+        6    5
+        9    8
+        Name: b, dtype: int64
+        """
+        return self._limit(n, asc=False)
+
+    def shift(self, periods=1, fill_value=None) -> Union[DataFrame, Series]:
+        """
+        Shift each group by periods observations.
+
+        Parameters
+        ----------
+        periods : integer, default 1
+            number of periods to shift
+        fill_value : optional
+
+        Returns
+        -------
+        Series or DataFrame
+            Object shifted within each group.
+
+        Examples
+        --------
+
+        >>> df = ps.DataFrame({
+        ...     'a': [1, 1, 1, 2, 2, 2, 3, 3, 3],
+        ...     'b': [1, 2, 2, 2, 3, 3, 3, 4, 4]}, columns=['a', 'b'])
+        >>> df
+           a  b
+        0  1  1
+        1  1  2
+        2  1  2
+        3  2  2
+        4  2  3
+        5  2  3
+        6  3  3
+        7  3  4
+        8  3  4
+
+        >>> df.groupby('a').shift().sort_index()  # doctest: +SKIP
+             b
+        0  NaN
+        1  1.0
+        2  2.0
+        3  NaN
+        4  2.0
+        5  3.0
+        6  NaN
+        7  3.0
+        8  4.0
+
+        >>> df.groupby('a').shift(periods=-1, fill_value=0).sort_index()  # doctest: +SKIP
+           b
+        0  2
+        1  2
+        2  0
+        3  3
+        4  3
+        5  0
+        6  4
+        7  4
+        8  0
+        """
+        return self._apply_series_op(
+            lambda sg: sg._kser._shift(periods, fill_value, part_cols=sg._groupkeys_scols),
+            should_resolve=True,
+        )
+
+    def transform(self, func, *args, **kwargs) -> Union[DataFrame, Series]:
+        """
+        Apply function column-by-column to the GroupBy object.
+
+        The function passed to `transform` must take a Series as its first
+        argument and return a Series. The given function is executed for
+        each series in each grouped data.
+
+        While `transform` is a very flexible method, its downside is that
+        using it can be quite a bit slower than using more specific methods
+        like `agg` or `transform`. pandas-on-Spark offers a wide range of method that will
+        be much faster than using `transform` for their specific purposes, so try to
+        use them before reaching for `transform`.
+
+        .. note:: this API executes the function once to infer the type which is
+             potentially expensive, for instance, when the dataset is created after
+             aggregations or sorting.
+
+             To avoid this, specify return type in ``func``, for instance, as below:
+
+             >>> def convert_to_string(x) -> ps.Series[str]:
+             ...     return x.apply("a string {}".format)
+
+            When the given function has the return type annotated, the original index of the
+            GroupBy object will be lost and a default index will be attached to the result.
+            Please be careful about configuring the default index. See also `Default Index Type
+            <https://koalas.readthedocs.io/en/latest/user_guide/options.html#default-index-type>`_.
+
+        .. note:: the series within ``func`` is actually a pandas series. Therefore,
+            any pandas APIs within this function is allowed.
+
+
+        Parameters
+        ----------
+        func : callable
+            A callable that takes a Series as its first argument, and
+            returns a Series.
+        *args
+            Positional arguments to pass to func.
+        **kwargs
+            Keyword arguments to pass to func.
+
+        Returns
+        -------
+        applied : DataFrame
+
+        See Also
+        --------
+        aggregate : Apply aggregate function to the GroupBy object.
+        Series.apply : Apply a function to a Series.
+
+        Examples
+        --------
+
+        >>> df = ps.DataFrame({'A': [0, 0, 1],
+        ...                    'B': [1, 2, 3],
+        ...                    'C': [4, 6, 5]}, columns=['A', 'B', 'C'])
+
+        >>> g = df.groupby('A')
+
+        Notice that ``g`` has two groups, ``0`` and ``1``.
+        Calling `transform` in various ways, we can get different grouping results:
+        Below the functions passed to `transform` takes a Series as
+        its argument and returns a Series. `transform` applies the function on each series
+        in each grouped data, and combine them into a new DataFrame:
+
+        >>> def convert_to_string(x) -> ps.Series[str]:
+        ...     return x.apply("a string {}".format)
+        >>> g.transform(convert_to_string)  # doctest: +NORMALIZE_WHITESPACE
+                    B           C
+        0  a string 1  a string 4
+        1  a string 2  a string 6
+        2  a string 3  a string 5
+
+        >>> def plus_max(x) -> ps.Series[np.int]:
+        ...     return x + x.max()
+        >>> g.transform(plus_max)  # doctest: +NORMALIZE_WHITESPACE
+           B   C
+        0  3  10
+        1  4  12
+        2  6  10
+
+        You can omit the type hint and let pandas-on-Spark infer its type.
+
+        >>> def plus_min(x):
+        ...     return x + x.min()
+        >>> g.transform(plus_min)  # doctest: +NORMALIZE_WHITESPACE
+           B   C
+        0  2   8
+        1  3  10
+        2  6  10
+
+        In case of Series, it works as below.
+
+        >>> df.B.groupby(df.A).transform(plus_max)
+        0    3
+        1    4
+        2    6
+        Name: B, dtype: int64
+
+        >>> (df * -1).B.groupby(df.A).transform(abs)
+        0    1
+        1    2
+        2    3
+        Name: B, dtype: int64
+
+        You can also specify extra arguments to pass to the function.
+
+        >>> def calculation(x, y, z) -> ps.Series[np.int]:
+        ...     return x + x.min() + y + z
+        >>> g.transform(calculation, 5, z=20)  # doctest: +NORMALIZE_WHITESPACE
+            B   C
+        0  27  33
+        1  28  35
+        2  31  35
+        """
+        if not isinstance(func, Callable):  # type: ignore
+            raise TypeError("%s object is not callable" % type(func).__name__)
+
+        spec = inspect.getfullargspec(func)
+        return_sig = spec.annotations.get("return", None)
+
+        kdf, groupkey_labels, groupkey_names = GroupBy._prepare_group_map_apply(
+            self._kdf, self._groupkeys, agg_columns=self._agg_columns
+        )
+
+        def pandas_transform(pdf):
+            return pdf.groupby(groupkey_names).transform(func, *args, **kwargs)
+
+        should_infer_schema = return_sig is None
+
+        if should_infer_schema:
+            # Here we execute with the first 1000 to get the return type.
+            # If the records were less than 1000, it uses pandas API directly for a shortcut.
+            limit = get_option("compute.shortcut_limit")
+            pdf = kdf.head(limit + 1)._to_internal_pandas()
+            pdf = pdf.groupby(groupkey_names).transform(func, *args, **kwargs)
+            kdf_from_pandas = DataFrame(pdf)  # type: DataFrame
+            return_schema = force_decimal_precision_scale(
+                as_nullable_spark_type(
+                    kdf_from_pandas._internal.spark_frame.drop(*HIDDEN_COLUMNS).schema
+                )
+            )
+            if len(pdf) <= limit:
+                return kdf_from_pandas
+
+            sdf = GroupBy._spark_group_map_apply(
+                kdf,
+                pandas_transform,
+                [kdf._internal.spark_column_for(label) for label in groupkey_labels],
+                return_schema,
+                retain_index=True,
+            )
+            # If schema is inferred, we can restore indexes too.
+            internal = kdf_from_pandas._internal.with_new_sdf(sdf)
+        else:
+            return_type = infer_return_type(func)
+            if not isinstance(return_type, SeriesType):
+                raise TypeError(
+                    "Expected the return type of this function to be of Series type, "
+                    "but found type {}".format(return_type)
+                )
+
+            return_schema = cast(SeriesType, return_type).spark_type
+            data_columns = kdf._internal.data_spark_column_names
+            return_schema = StructType(
+                [StructField(c, return_schema) for c in data_columns if c not in groupkey_names]
+            )
+            data_dtypes = [
+                cast(SeriesType, return_type).dtype for c in data_columns if c not in groupkey_names
+            ]
+
+            sdf = GroupBy._spark_group_map_apply(
+                kdf,
+                pandas_transform,
+                [kdf._internal.spark_column_for(label) for label in groupkey_labels],
+                return_schema,
+                retain_index=False,
+            )
+            # Otherwise, it loses index.
+            internal = InternalFrame(
+                spark_frame=sdf, index_spark_columns=None, data_dtypes=data_dtypes
+            )
+
+        return DataFrame(internal)
+
+    def nunique(self, dropna=True) -> Union[DataFrame, Series]:
+        """
+        Return DataFrame with number of distinct observations per group for each column.
+
+        Parameters
+        ----------
+        dropna : boolean, default True
+            Don’t include NaN in the counts.
+
+        Returns
+        -------
+        nunique : DataFrame or Series
+
+        Examples
+        --------
+
+        >>> df = ps.DataFrame({'id': ['spam', 'egg', 'egg', 'spam',
+        ...                           'ham', 'ham'],
+        ...                    'value1': [1, 5, 5, 2, 5, 5],
+        ...                    'value2': list('abbaxy')}, columns=['id', 'value1', 'value2'])
+        >>> df
+             id  value1 value2
+        0  spam       1      a
+        1   egg       5      b
+        2   egg       5      b
+        3  spam       2      a
+        4   ham       5      x
+        5   ham       5      y
+
+        >>> df.groupby('id').nunique().sort_index() # doctest: +SKIP
+              value1  value2
+        id
+        egg        1       1
+        ham        1       2
+        spam       2       1
+
+        >>> df.groupby('id')['value1'].nunique().sort_index() # doctest: +NORMALIZE_WHITESPACE
+        id
+        egg     1
+        ham     1
+        spam    2
+        Name: value1, dtype: int64
+        """
+        if dropna:
+            stat_function = lambda col: F.countDistinct(col)
+        else:
+            stat_function = lambda col: (
+                F.countDistinct(col)
+                + F.when(F.count(F.when(col.isNull(), 1).otherwise(None)) >= 1, 1).otherwise(0)
+            )
+
+        return self._reduce_for_stat_function(stat_function, only_numeric=False)
+
+    def rolling(self, window, min_periods=None) -> RollingGroupby:
+        """
+        Return an rolling grouper, providing rolling
+        functionality per group.
+
+        .. note:: 'min_periods' in pandas-on-Spark works as a fixed window size unlike pandas.
+        Unlike pandas, NA is also counted as the period. This might be changed
+        in the near future.
+
+        Parameters
+        ----------
+        window : int, or offset
+            Size of the moving window.
+            This is the number of observations used for calculating the statistic.
+            Each window will be a fixed size.
+
+        min_periods : int, default 1
+            Minimum number of observations in window required to have a value
+            (otherwise result is NA).
+
+        See Also
+        --------
+        Series.groupby
+        DataFrame.groupby
+        """
+        return RollingGroupby(self, window, min_periods=min_periods)
+
+    def expanding(self, min_periods=1) -> ExpandingGroupby:
+        """
+        Return an expanding grouper, providing expanding
+        functionality per group.
+
+        .. note:: 'min_periods' in pandas-on-Spark works as a fixed window size unlike pandas.
+        Unlike pandas, NA is also counted as the period. This might be changed
+        in the near future.
+
+        Parameters
+        ----------
+        min_periods : int, default 1
+            Minimum number of observations in window required to have a value
+            (otherwise result is NA).
+
+        See Also
+        --------
+        Series.groupby
+        DataFrame.groupby
+        """
+        return ExpandingGroupby(self, min_periods=min_periods)
+
+    def get_group(self, name) -> Union[DataFrame, Series]:
+        """
+        Construct DataFrame from group with provided name.
+
+        Parameters
+        ----------
+        name : object
+            The name of the group to get as a DataFrame.
+
+        Returns
+        -------
+        group : same type as obj
+
+        Examples
+        --------
+        >>> kdf = ps.DataFrame([('falcon', 'bird', 389.0),
+        ...                     ('parrot', 'bird', 24.0),
+        ...                     ('lion', 'mammal', 80.5),
+        ...                     ('monkey', 'mammal', np.nan)],
+        ...                    columns=['name', 'class', 'max_speed'],
+        ...                    index=[0, 2, 3, 1])
+        >>> kdf
+             name   class  max_speed
+        0  falcon    bird      389.0
+        2  parrot    bird       24.0
+        3    lion  mammal       80.5
+        1  monkey  mammal        NaN
+
+        >>> kdf.groupby("class").get_group("bird").sort_index()
+             name class  max_speed
+        0  falcon  bird      389.0
+        2  parrot  bird       24.0
+
+        >>> kdf.groupby("class").get_group("mammal").sort_index()
+             name   class  max_speed
+        1  monkey  mammal        NaN
+        3    lion  mammal       80.5
+        """
+        groupkeys = self._groupkeys
+        if not is_hashable(name):
+            raise TypeError("unhashable type: '{}'".format(type(name).__name__))
+        elif len(groupkeys) > 1:
+            if not isinstance(name, tuple):
+                raise ValueError("must supply a tuple to get_group with multiple grouping keys")
+            if len(groupkeys) != len(name):
+                raise ValueError(
+                    "must supply a same-length tuple to get_group with multiple grouping keys"
+                )
+        if not is_list_like(name):
+            name = [name]
+        cond = F.lit(True)
+        for groupkey, item in zip(groupkeys, name):
+            scol = groupkey.spark.column
+            cond = cond & (scol == item)
+        if self._agg_columns_selected:
+            internal = self._kdf._internal
+            spark_frame = internal.spark_frame.select(
+                internal.index_spark_columns + self._agg_columns_scols
+            ).filter(cond)
+
+            internal = internal.copy(
+                spark_frame=spark_frame,
+                index_spark_columns=[
+                    scol_for(spark_frame, col) for col in internal.index_spark_column_names
+                ],
+                column_labels=[s._column_label for s in self._agg_columns],
+                data_spark_columns=[
+                    scol_for(spark_frame, s._internal.data_spark_column_names[0])
+                    for s in self._agg_columns
+                ],
+                data_dtypes=[s.dtype for s in self._agg_columns],
+            )
+        else:
+            internal = self._kdf._internal.with_filter(cond)
+        if internal.spark_frame.head() is None:
+            raise KeyError(name)
+
+        return DataFrame(internal)
+
+    def median(self, numeric_only=True, accuracy=10000) -> Union[DataFrame, Series]:
+        """
+        Compute median of groups, excluding missing values.
+
+        For multiple groupings, the result index will be a MultiIndex
+
+        .. note:: Unlike pandas', the median in pandas-on-Spark is an approximated median based upon
+            approximate percentile computation because computing median across a large dataset
+            is extremely expensive.
+
+        Parameters
+        ----------
+        numeric_only : bool, default True
+            Include only float, int, boolean columns. False is not supported. This parameter
+            is mainly for pandas compatibility.
+
+        Returns
+        -------
+        Series or DataFrame
+            Median of values within each group.
+
+        Examples
+        --------
+        >>> kdf = ps.DataFrame({'a': [1., 1., 1., 1., 2., 2., 2., 3., 3., 3.],
+        ...                     'b': [2., 3., 1., 4., 6., 9., 8., 10., 7., 5.],
+        ...                     'c': [3., 5., 2., 5., 1., 2., 6., 4., 3., 6.]},
+        ...                    columns=['a', 'b', 'c'],
+        ...                    index=[7, 2, 4, 1, 3, 4, 9, 10, 5, 6])
+        >>> kdf
+              a     b    c
+        7   1.0   2.0  3.0
+        2   1.0   3.0  5.0
+        4   1.0   1.0  2.0
+        1   1.0   4.0  5.0
+        3   2.0   6.0  1.0
+        4   2.0   9.0  2.0
+        9   2.0   8.0  6.0
+        10  3.0  10.0  4.0
+        5   3.0   7.0  3.0
+        6   3.0   5.0  6.0
+
+        DataFrameGroupBy
+
+        >>> kdf.groupby('a').median().sort_index()  # doctest: +NORMALIZE_WHITESPACE
+               b    c
+        a
+        1.0  2.0  3.0
+        2.0  8.0  2.0
+        3.0  7.0  4.0
+
+        SeriesGroupBy
+
+        >>> kdf.groupby('a')['b'].median().sort_index()
+        a
+        1.0    2.0
+        2.0    8.0
+        3.0    7.0
+        Name: b, dtype: float64
+        """
+        if not isinstance(accuracy, int):
+            raise TypeError(
+                "accuracy must be an integer; however, got [%s]" % type(accuracy).__name__
+            )
+
+        stat_function = lambda col: F.percentile_approx(col, 0.5, accuracy)
+        return self._reduce_for_stat_function(stat_function, only_numeric=numeric_only)
+
+    def _reduce_for_stat_function(self, sfun, only_numeric):
+        agg_columns = self._agg_columns
+        agg_columns_scols = self._agg_columns_scols
+
+        groupkey_names = [SPARK_INDEX_NAME_FORMAT(i) for i in range(len(self._groupkeys))]
+        groupkey_scols = [s.alias(name) for s, name in zip(self._groupkeys_scols, groupkey_names)]
+
+        sdf = self._kdf._internal.spark_frame.select(groupkey_scols + agg_columns_scols)
+
+        data_columns = []
+        column_labels = []
+        if len(agg_columns) > 0:
+            stat_exprs = []
+            for kser in agg_columns:
+                spark_type = kser.spark.data_type
+                name = kser._internal.data_spark_column_names[0]
+                label = kser._column_label
+                scol = scol_for(sdf, name)
+                # TODO: we should have a function that takes dataframes and converts the numeric
+                # types. Converting the NaNs is used in a few places, it should be in utils.
+                # Special handle floating point types because Spark's count treats nan as a valid
+                # value, whereas pandas count doesn't include nan.
+                if isinstance(spark_type, DoubleType) or isinstance(spark_type, FloatType):
+                    stat_exprs.append(sfun(F.nanvl(scol, F.lit(None))).alias(name))
+                    data_columns.append(name)
+                    column_labels.append(label)
+                elif isinstance(spark_type, NumericType) or not only_numeric:
+                    stat_exprs.append(sfun(scol).alias(name))
+                    data_columns.append(name)
+                    column_labels.append(label)
+            sdf = sdf.groupby(*groupkey_names).agg(*stat_exprs)
+        else:
+            sdf = sdf.select(*groupkey_names).distinct()
+
+        internal = InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=[scol_for(sdf, col) for col in groupkey_names],
+            index_names=[kser._column_label for kser in self._groupkeys],
+            index_dtypes=[kser.dtype for kser in self._groupkeys],
+            column_labels=column_labels,
+            data_spark_columns=[scol_for(sdf, col) for col in data_columns],
+            column_label_names=self._kdf._internal.column_label_names,
+        )
+        kdf = DataFrame(internal)
+
+        if self._dropna:
+            kdf = DataFrame(
+                kdf._internal.with_new_sdf(
+                    kdf._internal.spark_frame.dropna(subset=kdf._internal.index_spark_column_names)
+                )
+            )
+
+        if not self._as_index:
+            should_drop_index = set(
+                i for i, gkey in enumerate(self._groupkeys) if gkey._kdf is not self._kdf
+            )
+            if len(should_drop_index) > 0:
+                kdf = kdf.reset_index(level=should_drop_index, drop=True)
+            if len(should_drop_index) < len(self._groupkeys):
+                kdf = kdf.reset_index()
+        return kdf
+
+    @staticmethod
+    def _resolve_grouping_from_diff_dataframes(
+        kdf: DataFrame, by: List[Union[Series, Tuple]]
+    ) -> Tuple[DataFrame, List[Series], Set[Tuple]]:
+        column_labels_level = kdf._internal.column_labels_level
+
+        column_labels = []
+        additional_ksers = []
+        additional_column_labels = []
+        tmp_column_labels = set()
+        for i, col_or_s in enumerate(by):
+            if isinstance(col_or_s, Series):
+                if col_or_s._kdf is kdf:
+                    column_labels.append(col_or_s._column_label)
+                elif same_anchor(col_or_s, kdf):
+                    temp_label = verify_temp_column_name(kdf, "__tmp_groupkey_{}__".format(i))
+                    column_labels.append(temp_label)
+                    additional_ksers.append(col_or_s.rename(temp_label))
+                    additional_column_labels.append(temp_label)
+                else:
+                    temp_label = verify_temp_column_name(
+                        kdf,
+                        tuple(
+                            ([""] * (column_labels_level - 1)) + ["__tmp_groupkey_{}__".format(i)]
+                        ),
+                    )
+                    column_labels.append(temp_label)
+                    tmp_column_labels.add(temp_label)
+            elif isinstance(col_or_s, tuple):
+                kser = kdf[col_or_s]
+                if not isinstance(kser, Series):
+                    raise ValueError(name_like_string(col_or_s))
+                column_labels.append(col_or_s)
+            else:
+                raise ValueError(col_or_s)
+
+        kdf = DataFrame(
+            kdf._internal.with_new_columns(
+                [kdf._kser_for(label) for label in kdf._internal.column_labels] + additional_ksers
+            )
+        )
+
+        def assign_columns(kdf, this_column_labels, that_column_labels):
+            raise NotImplementedError(
+                "Duplicated labels with groupby() and "
+                "'compute.ops_on_diff_frames' option are not supported currently "
+                "Please use unique labels in series and frames."
+            )
+
+        for col_or_s, label in zip(by, column_labels):
+            if label in tmp_column_labels:
+                kser = col_or_s
+                kdf = align_diff_frames(
+                    assign_columns,
+                    kdf,
+                    kser.rename(label),
+                    fillna=False,
+                    how="inner",
+                    preserve_order_column=True,
+                )
+
+        tmp_column_labels |= set(additional_column_labels)
+
+        new_by_series = []
+        for col_or_s, label in zip(by, column_labels):
+            if label in tmp_column_labels:
+                kser = col_or_s
+                new_by_series.append(kdf._kser_for(label).rename(kser.name))
+            else:
+                new_by_series.append(kdf._kser_for(label))
+
+        return kdf, new_by_series, tmp_column_labels
+
+    @staticmethod
+    def _resolve_grouping(kdf: DataFrame, by: List[Union[Series, Tuple]]) -> List[Series]:
+        new_by_series = []
+        for col_or_s in by:
+            if isinstance(col_or_s, Series):
+                new_by_series.append(col_or_s)
+            elif isinstance(col_or_s, tuple):
+                kser = kdf[col_or_s]
+                if not isinstance(kser, Series):
+                    raise ValueError(name_like_string(col_or_s))
+                new_by_series.append(kser)
+            else:
+                raise ValueError(col_or_s)
+        return new_by_series
+
+
+class DataFrameGroupBy(GroupBy):
+    @staticmethod
+    def _build(
+        kdf: DataFrame, by: List[Union[Series, Tuple]], as_index: bool, dropna: bool
+    ) -> "DataFrameGroupBy":
+        if any(isinstance(col_or_s, Series) and not same_anchor(kdf, col_or_s) for col_or_s in by):
+            (
+                kdf,
+                new_by_series,
+                column_labels_to_exlcude,
+            ) = GroupBy._resolve_grouping_from_diff_dataframes(kdf, by)
+        else:
+            new_by_series = GroupBy._resolve_grouping(kdf, by)
+            column_labels_to_exlcude = set()
+        return DataFrameGroupBy(
+            kdf,
+            new_by_series,
+            as_index=as_index,
+            dropna=dropna,
+            column_labels_to_exlcude=column_labels_to_exlcude,
+        )
+
+    def __init__(
+        self,
+        kdf: DataFrame,
+        by: List[Series],
+        as_index: bool,
+        dropna: bool,
+        column_labels_to_exlcude: Set[Tuple],
+        agg_columns: List[Tuple] = None,
+    ):
+
+        agg_columns_selected = agg_columns is not None
+        if agg_columns_selected:
+            for label in agg_columns:
+                if label in column_labels_to_exlcude:
+                    raise KeyError(label)
+        else:
+            agg_columns = [
+                label
+                for label in kdf._internal.column_labels
+                if not any(label == key._column_label and key._kdf is kdf for key in by)
+                and label not in column_labels_to_exlcude
+            ]
+
+        super().__init__(
+            kdf=kdf,
+            groupkeys=by,
+            as_index=as_index,
+            dropna=dropna,
+            column_labels_to_exlcude=column_labels_to_exlcude,
+            agg_columns_selected=agg_columns_selected,
+            agg_columns=[kdf[label] for label in agg_columns],
+        )
+
+    def __getattr__(self, item: str) -> Any:
+        if hasattr(MissingPandasLikeDataFrameGroupBy, item):
+            property_or_func = getattr(MissingPandasLikeDataFrameGroupBy, item)
+            if isinstance(property_or_func, property):
+                return property_or_func.fget(self)  # type: ignore
+            else:
+                return partial(property_or_func, self)
+        return self.__getitem__(item)
+
+    def __getitem__(self, item):
+        if self._as_index and is_name_like_value(item):
+            return SeriesGroupBy(
+                self._kdf._kser_for(item if is_name_like_tuple(item) else (item,)),
+                self._groupkeys,
+                dropna=self._dropna,
+            )
+        else:
+            if is_name_like_tuple(item):
+                item = [item]
+            elif is_name_like_value(item):
+                item = [(item,)]
+            else:
+                item = [i if is_name_like_tuple(i) else (i,) for i in item]
+            if not self._as_index:
+                groupkey_names = set(key._column_label for key in self._groupkeys)
+                for name in item:
+                    if name in groupkey_names:
+                        raise ValueError(
+                            "cannot insert {}, already exists".format(name_like_string(name))
+                        )
+            return DataFrameGroupBy(
+                self._kdf,
+                self._groupkeys,
+                as_index=self._as_index,
+                dropna=self._dropna,
+                column_labels_to_exlcude=self._column_labels_to_exlcude,
+                agg_columns=item,
+            )
+
+    def _apply_series_op(self, op, should_resolve: bool = False, numeric_only: bool = False):
+        applied = []
+        for column in self._agg_columns:
+            applied.append(op(column.groupby(self._groupkeys)))
+        if numeric_only:
+            applied = [col for col in applied if isinstance(col.spark.data_type, NumericType)]
+            if not applied:
+                raise DataError("No numeric types to aggregate")
+        internal = self._kdf._internal.with_new_columns(applied, keep_order=False)
+        if should_resolve:
+            internal = internal.resolved_copy
+        return DataFrame(internal)
+
+    # TODO: Implement 'percentiles', 'include', and 'exclude' arguments.
+    # TODO: Add ``DataFrame.select_dtypes`` to See Also when 'include'
+    #   and 'exclude' arguments are implemented.
+    def describe(self) -> DataFrame:
+        """
+        Generate descriptive statistics that summarize the central tendency,
+        dispersion and shape of a dataset's distribution, excluding
+        ``NaN`` values.
+
+        Analyzes both numeric and object series, as well
+        as ``DataFrame`` column sets of mixed data types. The output
+        will vary depending on what is provided. Refer to the notes
+        below for more detail.
+
+        .. note:: Unlike pandas, the percentiles in pandas-on-Spark are based upon
+            approximate percentile computation because computing percentiles
+            across a large dataset is extremely expensive.
+
+        Returns
+        -------
+        DataFrame
+            Summary statistics of the DataFrame provided.
+
+        See Also
+        --------
+        DataFrame.count
+        DataFrame.max
+        DataFrame.min
+        DataFrame.mean
+        DataFrame.std
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'a': [1, 1, 3], 'b': [4, 5, 6], 'c': [7, 8, 9]})
+        >>> df
+           a  b  c
+        0  1  4  7
+        1  1  5  8
+        2  3  6  9
+
+        Describing a ``DataFrame``. By default only numeric fields
+        are returned.
+
+        >>> described = df.groupby('a').describe()
+        >>> described.sort_index()  # doctest: +NORMALIZE_WHITESPACE
+              b                                        c
+          count mean       std min 25% 50% 75% max count mean       std min 25% 50% 75% max
+        a
+        1   2.0  4.5  0.707107 4.0 4.0 4.0 5.0 5.0   2.0  7.5  0.707107 7.0 7.0 7.0 8.0 8.0
+        3   1.0  6.0       NaN 6.0 6.0 6.0 6.0 6.0   1.0  9.0       NaN 9.0 9.0 9.0 9.0 9.0
+
+        """
+        for col in self._agg_columns:
+            if isinstance(col.spark.data_type, StringType):
+                raise NotImplementedError(
+                    "DataFrameGroupBy.describe() doesn't support for string type for now"
+                )
+
+        kdf = self.aggregate(["count", "mean", "std", "min", "quartiles", "max"])
+        sdf = kdf._internal.spark_frame
+        agg_column_labels = [col._column_label for col in self._agg_columns]
+        formatted_percentiles = ["25%", "50%", "75%"]
+
+        # Split "quartiles" columns into first, second, and third quartiles.
+        for label in agg_column_labels:
+            quartiles_col = name_like_string(tuple(list(label) + ["quartiles"]))
+            for i, percentile in enumerate(formatted_percentiles):
+                sdf = sdf.withColumn(
+                    name_like_string(tuple(list(label) + [percentile])),
+                    scol_for(sdf, quartiles_col)[i],
+                )
+            sdf = sdf.drop(quartiles_col)
+
+        # Reorder columns lexicographically by agg column followed by stats.
+        stats = ["count", "mean", "std", "min"] + formatted_percentiles + ["max"]
+        column_labels = [tuple(list(label) + [s]) for label, s in product(agg_column_labels, stats)]
+        data_columns = map(name_like_string, column_labels)
+
+        # Reindex the DataFrame to reflect initial grouping and agg columns.
+        internal = kdf._internal.copy(
+            spark_frame=sdf,
+            column_labels=column_labels,
+            data_spark_columns=[scol_for(sdf, col) for col in data_columns],
+            data_dtypes=None,
+        )
+
+        # Cast columns to ``"float64"`` to match `pandas.DataFrame.groupby`.
+        return DataFrame(internal).astype("float64")
+
+
+class SeriesGroupBy(GroupBy):
+    @staticmethod
+    def _build(
+        kser: Series, by: List[Union[Series, Tuple]], as_index: bool, dropna: bool
+    ) -> "SeriesGroupBy":
+        if any(isinstance(col_or_s, Series) and not same_anchor(kser, col_or_s) for col_or_s in by):
+            kdf, new_by_series, _ = GroupBy._resolve_grouping_from_diff_dataframes(
+                kser.to_frame(), by
+            )
+            return SeriesGroupBy(
+                first_series(kdf).rename(kser.name), new_by_series, as_index=as_index, dropna=dropna
+            )
+        else:
+            new_by_series = GroupBy._resolve_grouping(kser._kdf, by)
+            return SeriesGroupBy(kser, new_by_series, as_index=as_index, dropna=dropna)
+
+    def __init__(self, kser: Series, by: List[Series], as_index: bool = True, dropna: bool = True):
+        if not as_index:
+            raise TypeError("as_index=False only valid with DataFrame")
+        super().__init__(
+            kdf=kser._kdf,
+            groupkeys=by,
+            as_index=True,
+            dropna=dropna,
+            column_labels_to_exlcude=set(),
+            agg_columns_selected=True,
+            agg_columns=[kser],
+        )
+        self._kser = kser
+
+    def __getattr__(self, item: str) -> Any:
+        if hasattr(MissingPandasLikeSeriesGroupBy, item):
+            property_or_func = getattr(MissingPandasLikeSeriesGroupBy, item)
+            if isinstance(property_or_func, property):
+                return property_or_func.fget(self)  # type: ignore
+            else:
+                return partial(property_or_func, self)
+        raise AttributeError(item)
+
+    def _apply_series_op(self, op, should_resolve: bool = False, numeric_only: bool = False):
+        if numeric_only and not isinstance(self._agg_columns[0].spark.data_type, NumericType):
+            raise DataError("No numeric types to aggregate")
+        kser = op(self)
+        if should_resolve:
+            internal = kser._internal.resolved_copy
+            return first_series(DataFrame(internal))
+        else:
+            return kser
+
+    def _reduce_for_stat_function(self, sfun, only_numeric):
+        return first_series(super()._reduce_for_stat_function(sfun, only_numeric))
+
+    def agg(self, *args, **kwargs) -> None:
+        return MissingPandasLikeSeriesGroupBy.agg(self, *args, **kwargs)
+
+    def aggregate(self, *args, **kwargs) -> None:
+        return MissingPandasLikeSeriesGroupBy.aggregate(self, *args, **kwargs)
+
+    def transform(self, func, *args, **kwargs) -> Series:
+        return first_series(super().transform(func, *args, **kwargs)).rename(self._kser.name)
+
+    transform.__doc__ = GroupBy.transform.__doc__
+
+    def idxmin(self, skipna=True) -> Series:
+        return first_series(super().idxmin(skipna))
+
+    idxmin.__doc__ = GroupBy.idxmin.__doc__
+
+    def idxmax(self, skipna=True) -> Series:
+        return first_series(super().idxmax(skipna))
+
+    idxmax.__doc__ = GroupBy.idxmax.__doc__
+
+    def head(self, n=5) -> Series:
+        return first_series(super().head(n)).rename(self._kser.name)
+
+    head.__doc__ = GroupBy.head.__doc__
+
+    def tail(self, n=5) -> Series:
+        return first_series(super().tail(n)).rename(self._kser.name)
+
+    tail.__doc__ = GroupBy.tail.__doc__
+
+    def size(self) -> Series:
+        return super().size().rename(self._kser.name)
+
+    size.__doc__ = GroupBy.size.__doc__
+
+    def get_group(self, name) -> Series:
+        return first_series(super().get_group(name))
+
+    get_group.__doc__ = GroupBy.get_group.__doc__
+
+    # TODO: add keep parameter
+    def nsmallest(self, n=5) -> Series:
+        """
+        Return the first n rows ordered by columns in ascending order in group.
+
+        Return the first n rows with the smallest values in columns, in ascending order.
+        The columns that are not specified are returned as well, but not used for ordering.
+
+        Parameters
+        ----------
+        n : int
+            Number of items to retrieve.
+
+        See Also
+        --------
+        pyspark.pandas.Series.nsmallest
+        pyspark.pandas.DataFrame.nsmallest
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'a': [1, 1, 1, 2, 2, 2, 3, 3, 3],
+        ...                    'b': [1, 2, 2, 2, 3, 3, 3, 4, 4]}, columns=['a', 'b'])
+
+        >>> df.groupby(['a'])['b'].nsmallest(1).sort_index()  # doctest: +NORMALIZE_WHITESPACE
+        a
+        1  0    1
+        2  3    2
+        3  6    3
+        Name: b, dtype: int64
+        """
+        if self._kser._internal.index_level > 1:
+            raise ValueError("nsmallest do not support multi-index now")
+
+        groupkey_col_names = [SPARK_INDEX_NAME_FORMAT(i) for i in range(len(self._groupkeys))]
+        sdf = self._kser._internal.spark_frame.select(
+            [scol.alias(name) for scol, name in zip(self._groupkeys_scols, groupkey_col_names)]
+            + [
+                scol.alias(SPARK_INDEX_NAME_FORMAT(i + len(self._groupkeys)))
+                for i, scol in enumerate(self._kser._internal.index_spark_columns)
+            ]
+            + [self._kser.spark.column]
+            + [NATURAL_ORDER_COLUMN_NAME]
+        )
+
+        window = Window.partitionBy(groupkey_col_names).orderBy(
+            scol_for(sdf, self._kser._internal.data_spark_column_names[0]).asc(),
+            NATURAL_ORDER_COLUMN_NAME,
+        )
+
+        temp_rank_column = verify_temp_column_name(sdf, "__rank__")
+        sdf = (
+            sdf.withColumn(temp_rank_column, F.row_number().over(window))
+            .filter(F.col(temp_rank_column) <= n)
+            .drop(temp_rank_column)
+        ).drop(NATURAL_ORDER_COLUMN_NAME)
+
+        internal = InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=(
+                [scol_for(sdf, col) for col in groupkey_col_names]
+                + [
+                    scol_for(sdf, SPARK_INDEX_NAME_FORMAT(i + len(self._groupkeys)))
+                    for i in range(self._kdf._internal.index_level)
+                ]
+            ),
+            index_names=(
+                [kser._column_label for kser in self._groupkeys] + self._kdf._internal.index_names
+            ),
+            index_dtypes=(
+                [kser.dtype for kser in self._groupkeys] + self._kdf._internal.index_dtypes
+            ),
+            column_labels=[self._kser._column_label],
+            data_spark_columns=[scol_for(sdf, self._kser._internal.data_spark_column_names[0])],
+            data_dtypes=[self._kser.dtype],
+        )
+        return first_series(DataFrame(internal))
+
+    # TODO: add keep parameter
+    def nlargest(self, n=5) -> Series:
+        """
+        Return the first n rows ordered by columns in descending order in group.
+
+        Return the first n rows with the smallest values in columns, in descending order.
+        The columns that are not specified are returned as well, but not used for ordering.
+
+        Parameters
+        ----------
+        n : int
+            Number of items to retrieve.
+
+        See Also
+        --------
+        pyspark.pandas.Series.nlargest
+        pyspark.pandas.DataFrame.nlargest
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'a': [1, 1, 1, 2, 2, 2, 3, 3, 3],
+        ...                    'b': [1, 2, 2, 2, 3, 3, 3, 4, 4]}, columns=['a', 'b'])
+
+        >>> df.groupby(['a'])['b'].nlargest(1).sort_index()  # doctest: +NORMALIZE_WHITESPACE
+        a
+        1  1    2
+        2  4    3
+        3  7    4
+        Name: b, dtype: int64
+        """
+        if self._kser._internal.index_level > 1:
+            raise ValueError("nlargest do not support multi-index now")
+
+        groupkey_col_names = [SPARK_INDEX_NAME_FORMAT(i) for i in range(len(self._groupkeys))]
+        sdf = self._kser._internal.spark_frame.select(
+            [scol.alias(name) for scol, name in zip(self._groupkeys_scols, groupkey_col_names)]
+            + [
+                scol.alias(SPARK_INDEX_NAME_FORMAT(i + len(self._groupkeys)))
+                for i, scol in enumerate(self._kser._internal.index_spark_columns)
+            ]
+            + [self._kser.spark.column]
+            + [NATURAL_ORDER_COLUMN_NAME]
+        )
+
+        window = Window.partitionBy(groupkey_col_names).orderBy(
+            scol_for(sdf, self._kser._internal.data_spark_column_names[0]).desc(),
+            NATURAL_ORDER_COLUMN_NAME,
+        )
+
+        temp_rank_column = verify_temp_column_name(sdf, "__rank__")
+        sdf = (
+            sdf.withColumn(temp_rank_column, F.row_number().over(window))
+            .filter(F.col(temp_rank_column) <= n)
+            .drop(temp_rank_column)
+        ).drop(NATURAL_ORDER_COLUMN_NAME)
+
+        internal = InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=(
+                [scol_for(sdf, col) for col in groupkey_col_names]
+                + [
+                    scol_for(sdf, SPARK_INDEX_NAME_FORMAT(i + len(self._groupkeys)))
+                    for i in range(self._kdf._internal.index_level)
+                ]
+            ),
+            index_names=(
+                [kser._column_label for kser in self._groupkeys] + self._kdf._internal.index_names
+            ),
+            index_dtypes=(
+                [kser.dtype for kser in self._groupkeys] + self._kdf._internal.index_dtypes
+            ),
+            column_labels=[self._kser._column_label],
+            data_spark_columns=[scol_for(sdf, self._kser._internal.data_spark_column_names[0])],
+            data_dtypes=[self._kser.dtype],
+        )
+        return first_series(DataFrame(internal))
+
+    # TODO: add bins, normalize parameter
+    def value_counts(self, sort=None, ascending=None, dropna=True) -> Series:
+        """
+        Compute group sizes.
+
+        Parameters
+        ----------
+        sort : boolean, default None
+            Sort by frequencies.
+        ascending : boolean, default False
+            Sort in ascending order.
+        dropna : boolean, default True
+            Don't include counts of NaN.
+
+        See Also
+        --------
+        pyspark.pandas.Series.groupby
+        pyspark.pandas.DataFrame.groupby
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'A': [1, 2, 2, 3, 3, 3],
+        ...                    'B': [1, 1, 2, 3, 3, 3]},
+        ...                   columns=['A', 'B'])
+        >>> df
+           A  B
+        0  1  1
+        1  2  1
+        2  2  2
+        3  3  3
+        4  3  3
+        5  3  3
+
+        >>> df.groupby('A')['B'].value_counts().sort_index()  # doctest: +NORMALIZE_WHITESPACE
+        A  B
+        1  1    1
+        2  1    1
+           2    1
+        3  3    3
+        Name: B, dtype: int64
+        """
+        groupkeys = self._groupkeys + self._agg_columns
+        groupkey_names = [SPARK_INDEX_NAME_FORMAT(i) for i in range(len(groupkeys))]
+        groupkey_cols = [s.spark.column.alias(name) for s, name in zip(groupkeys, groupkey_names)]
+
+        sdf = self._kdf._internal.spark_frame
+        agg_column = self._agg_columns[0]._internal.data_spark_column_names[0]
+        sdf = sdf.groupby(*groupkey_cols).count().withColumnRenamed("count", agg_column)
+
+        if sort:
+            if ascending:
+                sdf = sdf.orderBy(scol_for(sdf, agg_column).asc())
+            else:
+                sdf = sdf.orderBy(scol_for(sdf, agg_column).desc())
+
+        internal = InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=[scol_for(sdf, col) for col in groupkey_names],
+            index_names=[kser._column_label for kser in groupkeys],
+            index_dtypes=[kser.dtype for kser in groupkeys],
+            column_labels=[self._agg_columns[0]._column_label],
+            data_spark_columns=[scol_for(sdf, agg_column)],
+        )
+        return first_series(DataFrame(internal))
+
+    def unique(self) -> Series:
+        """
+        Return unique values in group.
+
+        Uniques are returned in order of unknown. It does NOT sort.
+
+        See Also
+        --------
+        pyspark.pandas.Series.unique
+        pyspark.pandas.Index.unique
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'a': [1, 1, 1, 2, 2, 2, 3, 3, 3],
+        ...                    'b': [1, 2, 2, 2, 3, 3, 3, 4, 4]}, columns=['a', 'b'])
+
+        >>> df.groupby(['a'])['b'].unique().sort_index()  # doctest: +SKIP
+        a
+        1    [1, 2]
+        2    [2, 3]
+        3    [3, 4]
+        Name: b, dtype: object
+        """
+        return self._reduce_for_stat_function(F.collect_set, only_numeric=False)
+
+
+def is_multi_agg_with_relabel(**kwargs):
+    """
+    Check whether the kwargs pass to .agg look like multi-agg with relabling.
+
+    Parameters
+    ----------
+    **kwargs : dict
+
+    Returns
+    -------
+    bool
+
+    Examples
+    --------
+    >>> is_multi_agg_with_relabel(a='max')
+    False
+    >>> is_multi_agg_with_relabel(a_max=('a', 'max'),
+    ...                            a_min=('a', 'min'))
+    True
+    >>> is_multi_agg_with_relabel()
+    False
+    """
+    if not kwargs:
+        return False
+    return all(isinstance(v, tuple) and len(v) == 2 for v in kwargs.values())
+
+
+def normalize_keyword_aggregation(kwargs):
+    """
+    Normalize user-provided kwargs.
+
+    Transforms from the new ``Dict[str, NamedAgg]`` style kwargs
+    to the old OrderedDict[str, List[scalar]]].
+
+    Parameters
+    ----------
+    kwargs : dict
+
+    Returns
+    -------
+    aggspec : dict
+        The transformed kwargs.
+    columns : List[str]
+        The user-provided keys.
+    order : List[Tuple[str, str]]
+        Pairs of the input and output column names.
+
+    Examples
+    --------
+    >>> normalize_keyword_aggregation({'output': ('input', 'sum')})
+    (OrderedDict([('input', ['sum'])]), ('output',), [('input', 'sum')])
+    """
+    # this is due to python version issue, not sure the impact on pandas-on-Spark
+    PY36 = sys.version_info >= (3, 6)
+    if not PY36:
+        kwargs = OrderedDict(sorted(kwargs.items()))
+
+    # TODO(Py35): When we drop python 3.5, change this to defaultdict(list)
+    aggspec = OrderedDict()
+    order = []
+    columns, pairs = list(zip(*kwargs.items()))
+
+    for column, aggfunc in pairs:
+        if column in aggspec:
+            aggspec[column].append(aggfunc)
+        else:
+            aggspec[column] = [aggfunc]
+
+        order.append((column, aggfunc))
+    # For MultiIndex, we need to flatten the tuple, e.g. (('y', 'A'), 'max') needs to be
+    # flattened to ('y', 'A', 'max'), it won't do anything on normal Index.
+    if isinstance(order[0][0], tuple):
+        order = [(*levs, method) for levs, method in order]
+    return aggspec, columns, order
+
+
+def _test():
+    import os
+    import doctest
+    import sys
+    import numpy
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.groupby
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.groupby.__dict__.copy()
+    globs["np"] = numpy
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]")
+        .appName("pyspark.pandas.groupby tests")
+        .getOrCreate()
+    )
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.groupby,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/indexes/__init__.py b/python/pyspark/pandas/indexes/__init__.py
new file mode 100644
index 0000000000000..cd2adbaf9fd63
--- /dev/null
+++ b/python/pyspark/pandas/indexes/__init__.py
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from pyspark.pandas.indexes.base import Index  # noqa: F401
+from pyspark.pandas.indexes.datetimes import DatetimeIndex  # noqa: F401
+from pyspark.pandas.indexes.multi import MultiIndex  # noqa: F401
+from pyspark.pandas.indexes.numeric import Float64Index, Int64Index  # noqa: F401
diff --git a/python/pyspark/pandas/indexes/base.py b/python/pyspark/pandas/indexes/base.py
new file mode 100644
index 0000000000000..95cc35737be33
--- /dev/null
+++ b/python/pyspark/pandas/indexes/base.py
@@ -0,0 +1,2501 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from functools import partial
+from typing import Any, List, Optional, Tuple, Union
+import warnings
+
+import pandas as pd
+import numpy as np
+from pandas.api.types import (
+    is_list_like,
+    is_interval_dtype,
+    is_bool_dtype,
+    is_categorical_dtype,
+    is_integer_dtype,
+    is_float_dtype,
+    is_numeric_dtype,
+    is_object_dtype,
+)
+from pandas.core.accessor import CachedAccessor
+from pandas.io.formats.printing import pprint_thing
+from pandas.api.types import CategoricalDtype, is_hashable
+from pandas._libs import lib
+
+from pyspark import sql as spark
+from pyspark.sql import functions as F
+from pyspark.sql.types import DataType, FractionalType, IntegralType, TimestampType
+
+from pyspark import pandas as ps  # For running doctests and reference resolution in PyCharm.
+from pyspark.pandas.config import get_option, option_context
+from pyspark.pandas.base import IndexOpsMixin
+from pyspark.pandas.frame import DataFrame
+from pyspark.pandas.missing.indexes import MissingPandasLikeIndex
+from pyspark.pandas.series import Series, first_series
+from pyspark.pandas.spark.accessors import SparkIndexMethods
+from pyspark.pandas.utils import (
+    is_name_like_tuple,
+    is_name_like_value,
+    name_like_string,
+    same_anchor,
+    scol_for,
+    verify_temp_column_name,
+    validate_bool_kwarg,
+    ERROR_MESSAGE_CANNOT_COMBINE,
+)
+from pyspark.pandas.internal import (
+    InternalFrame,
+    DEFAULT_SERIES_NAME,
+    SPARK_DEFAULT_INDEX_NAME,
+    SPARK_INDEX_NAME_FORMAT,
+)
+from pyspark.pandas.typedef import Scalar
+
+
+class Index(IndexOpsMixin):
+    """
+    pandas-on-Spark Index that corresponds to pandas Index logically. This might hold Spark Column
+    internally.
+
+    Parameters
+    ----------
+    data : array-like (1-dimensional)
+    dtype : dtype, default None
+        If dtype is None, we find the dtype that best fits the data.
+        If an actual dtype is provided, we coerce to that dtype if it's safe.
+        Otherwise, an error will be raised.
+    copy : bool
+        Make a copy of input ndarray.
+    name : object
+        Name to be stored in the index.
+    tupleize_cols : bool (default: True)
+        When True, attempt to create a MultiIndex if possible.
+
+    See Also
+    --------
+    MultiIndex : A multi-level, or hierarchical, Index.
+    DatetimeIndex : Index of datetime64 data.
+    Int64Index : A special case of :class:`Index` with purely integer labels.
+    Float64Index : A special case of :class:`Index` with purely float labels.
+
+    Examples
+    --------
+    >>> ps.DataFrame({'a': ['a', 'b', 'c']}, index=[1, 2, 3]).index
+    Int64Index([1, 2, 3], dtype='int64')
+
+    >>> ps.DataFrame({'a': [1, 2, 3]}, index=list('abc')).index
+    Index(['a', 'b', 'c'], dtype='object')
+
+    >>> ps.Index([1, 2, 3])
+    Int64Index([1, 2, 3], dtype='int64')
+
+    >>> ps.Index(list('abc'))
+    Index(['a', 'b', 'c'], dtype='object')
+
+    From a Series:
+
+    >>> s = ps.Series([1, 2, 3], index=[10, 20, 30])
+    >>> ps.Index(s)
+    Int64Index([1, 2, 3], dtype='int64')
+
+    From an Index:
+
+    >>> idx = ps.Index([1, 2, 3])
+    >>> ps.Index(idx)
+    Int64Index([1, 2, 3], dtype='int64')
+    """
+
+    def __new__(cls, data=None, dtype=None, copy=False, name=None, tupleize_cols=True, **kwargs):
+        if not is_hashable(name):
+            raise TypeError("Index.name must be a hashable type")
+
+        if isinstance(data, Series):
+            if dtype is not None:
+                data = data.astype(dtype)
+            if name is not None:
+                data = data.rename(name)
+
+            internal = InternalFrame(
+                spark_frame=data._internal.spark_frame,
+                index_spark_columns=data._internal.data_spark_columns,
+                index_names=data._internal.column_labels,
+                index_dtypes=data._internal.data_dtypes,
+                column_labels=[],
+                data_spark_columns=[],
+                data_dtypes=[],
+            )
+            return DataFrame(internal).index
+        elif isinstance(data, Index):
+            if copy:
+                data = data.copy()
+            if dtype is not None:
+                data = data.astype(dtype)
+            if name is not None:
+                data = data.rename(name)
+            return data
+
+        return ps.from_pandas(
+            pd.Index(
+                data=data, dtype=dtype, copy=copy, name=name, tupleize_cols=tupleize_cols, **kwargs
+            )
+        )
+
+    @staticmethod
+    def _new_instance(anchor: DataFrame) -> "Index":
+        from pyspark.pandas.indexes.category import CategoricalIndex
+        from pyspark.pandas.indexes.datetimes import DatetimeIndex
+        from pyspark.pandas.indexes.multi import MultiIndex
+        from pyspark.pandas.indexes.numeric import Float64Index, Int64Index
+
+        if anchor._internal.index_level > 1:
+            instance = object.__new__(MultiIndex)
+        elif isinstance(anchor._internal.index_dtypes[0], CategoricalDtype):
+            instance = object.__new__(CategoricalIndex)
+        elif isinstance(
+            anchor._internal.spark_type_for(anchor._internal.index_spark_columns[0]), IntegralType
+        ):
+            instance = object.__new__(Int64Index)
+        elif isinstance(
+            anchor._internal.spark_type_for(anchor._internal.index_spark_columns[0]), FractionalType
+        ):
+            instance = object.__new__(Float64Index)
+        elif isinstance(
+            anchor._internal.spark_type_for(anchor._internal.index_spark_columns[0]), TimestampType
+        ):
+            instance = object.__new__(DatetimeIndex)
+        else:
+            instance = object.__new__(Index)
+
+        instance._anchor = anchor
+        return instance
+
+    @property
+    def _kdf(self) -> DataFrame:
+        return self._anchor
+
+    @property
+    def _internal(self) -> InternalFrame:
+        internal = self._kdf._internal
+        return internal.copy(
+            column_labels=internal.index_names,
+            data_spark_columns=internal.index_spark_columns,
+            data_dtypes=internal.index_dtypes,
+            column_label_names=None,
+        )
+
+    @property
+    def _column_label(self):
+        return self._kdf._internal.index_names[0]
+
+    def _with_new_scol(self, scol: spark.Column, *, dtype=None) -> "Index":
+        """
+        Copy pandas-on-Spark Index with the new Spark Column.
+
+        :param scol: the new Spark Column
+        :return: the copied Index
+        """
+        internal = self._internal.copy(
+            index_spark_columns=[scol.alias(SPARK_DEFAULT_INDEX_NAME)],
+            index_dtypes=[dtype],
+            column_labels=[],
+            data_spark_columns=[],
+            data_dtypes=[],
+        )
+        return DataFrame(internal).index
+
+    spark = CachedAccessor("spark", SparkIndexMethods)
+
+    # This method is used via `DataFrame.info` API internally.
+    def _summary(self, name=None) -> str:
+        """
+        Return a summarized representation.
+
+        Parameters
+        ----------
+        name : str
+            name to use in the summary representation
+
+        Returns
+        -------
+        String with a summarized representation of the index
+        """
+        head, tail, total_count = tuple(
+            self._internal.spark_frame.select(
+                F.first(self.spark.column), F.last(self.spark.column), F.count(F.expr("*"))
+            )
+            .toPandas()
+            .iloc[0]
+        )
+
+        if total_count > 0:
+            index_summary = ", %s to %s" % (pprint_thing(head), pprint_thing(tail))
+        else:
+            index_summary = ""
+
+        if name is None:
+            name = type(self).__name__
+        return "%s: %s entries%s" % (name, total_count, index_summary)
+
+    @property
+    def size(self) -> int:
+        """
+        Return an int representing the number of elements in this object.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([(.2, .3), (.0, .6), (.6, .0), (.2, .1)],
+        ...                   columns=['dogs', 'cats'],
+        ...                   index=list('abcd'))
+        >>> df.index.size
+        4
+
+        >>> df.set_index('dogs', append=True).index.size
+        4
+        """
+        return len(self)
+
+    @property
+    def shape(self) -> tuple:
+        """
+        Return a tuple of the shape of the underlying data.
+
+        Examples
+        --------
+        >>> idx = ps.Index(['a', 'b', 'c'])
+        >>> idx
+        Index(['a', 'b', 'c'], dtype='object')
+        >>> idx.shape
+        (3,)
+
+        >>> midx = ps.MultiIndex.from_tuples([('a', 'x'), ('b', 'y'), ('c', 'z')])
+        >>> midx  # doctest: +SKIP
+        MultiIndex([('a', 'x'),
+                    ('b', 'y'),
+                    ('c', 'z')],
+                   )
+        >>> midx.shape
+        (3,)
+        """
+        return (len(self._kdf),)
+
+    def identical(self, other) -> bool:
+        """
+        Similar to equals, but check that other comparable attributes are
+        also equal.
+
+        Returns
+        -------
+        bool
+            If two Index objects have equal elements and same type True,
+            otherwise False.
+
+        Examples
+        --------
+
+        >>> from pyspark.pandas.config import option_context
+        >>> idx = ps.Index(['a', 'b', 'c'])
+        >>> midx = ps.MultiIndex.from_tuples([('a', 'x'), ('b', 'y'), ('c', 'z')])
+
+        For Index
+
+        >>> idx.identical(idx)
+        True
+        >>> with option_context('compute.ops_on_diff_frames', True):
+        ...     idx.identical(ps.Index(['a', 'b', 'c']))
+        True
+        >>> with option_context('compute.ops_on_diff_frames', True):
+        ...     idx.identical(ps.Index(['b', 'b', 'a']))
+        False
+        >>> idx.identical(midx)
+        False
+
+        For MultiIndex
+
+        >>> midx.identical(midx)
+        True
+        >>> with option_context('compute.ops_on_diff_frames', True):
+        ...     midx.identical(ps.MultiIndex.from_tuples([('a', 'x'), ('b', 'y'), ('c', 'z')]))
+        True
+        >>> with option_context('compute.ops_on_diff_frames', True):
+        ...     midx.identical(ps.MultiIndex.from_tuples([('c', 'z'), ('b', 'y'), ('a', 'x')]))
+        False
+        >>> midx.identical(idx)
+        False
+        """
+        from pyspark.pandas.indexes.multi import MultiIndex
+
+        self_name = self.names if isinstance(self, MultiIndex) else self.name
+        other_name = other.names if isinstance(other, MultiIndex) else other.name
+
+        return (
+            self_name == other_name  # to support non-index comparison by short-circuiting.
+            and self.equals(other)
+        )
+
+    def equals(self, other) -> bool:
+        """
+        Determine if two Index objects contain the same elements.
+
+        Returns
+        -------
+        bool
+            True if "other" is an Index and it has the same elements as calling
+            index; False otherwise.
+
+        Examples
+        --------
+
+        >>> from pyspark.pandas.config import option_context
+        >>> idx = ps.Index(['a', 'b', 'c'])
+        >>> idx.name = "name"
+        >>> midx = ps.MultiIndex.from_tuples([('a', 'x'), ('b', 'y'), ('c', 'z')])
+        >>> midx.names = ("nameA", "nameB")
+
+        For Index
+
+        >>> idx.equals(idx)
+        True
+        >>> with option_context('compute.ops_on_diff_frames', True):
+        ...     idx.equals(ps.Index(['a', 'b', 'c']))
+        True
+        >>> with option_context('compute.ops_on_diff_frames', True):
+        ...     idx.equals(ps.Index(['b', 'b', 'a']))
+        False
+        >>> idx.equals(midx)
+        False
+
+        For MultiIndex
+
+        >>> midx.equals(midx)
+        True
+        >>> with option_context('compute.ops_on_diff_frames', True):
+        ...     midx.equals(ps.MultiIndex.from_tuples([('a', 'x'), ('b', 'y'), ('c', 'z')]))
+        True
+        >>> with option_context('compute.ops_on_diff_frames', True):
+        ...     midx.equals(ps.MultiIndex.from_tuples([('c', 'z'), ('b', 'y'), ('a', 'x')]))
+        False
+        >>> midx.equals(idx)
+        False
+        """
+        if same_anchor(self, other):
+            return True
+        elif type(self) == type(other):
+            if get_option("compute.ops_on_diff_frames"):
+                # TODO: avoid using default index?
+                with option_context("compute.default_index_type", "distributed-sequence"):
+                    # Directly using Series from both self and other seems causing
+                    # some exceptions when 'compute.ops_on_diff_frames' is enabled.
+                    # Working around for now via using frame.
+                    return (
+                        self.to_series("self").reset_index(drop=True)
+                        == other.to_series("other").reset_index(drop=True)
+                    ).all()
+            else:
+                raise ValueError(ERROR_MESSAGE_CANNOT_COMBINE)
+        else:
+            return False
+
+    def transpose(self) -> "Index":
+        """
+        Return the transpose, For index, It will be index itself.
+
+        Examples
+        --------
+        >>> idx = ps.Index(['a', 'b', 'c'])
+        >>> idx
+        Index(['a', 'b', 'c'], dtype='object')
+
+        >>> idx.transpose()
+        Index(['a', 'b', 'c'], dtype='object')
+
+        For MultiIndex
+
+        >>> midx = ps.MultiIndex.from_tuples([('a', 'x'), ('b', 'y'), ('c', 'z')])
+        >>> midx  # doctest: +SKIP
+        MultiIndex([('a', 'x'),
+                    ('b', 'y'),
+                    ('c', 'z')],
+                   )
+
+        >>> midx.transpose()  # doctest: +SKIP
+        MultiIndex([('a', 'x'),
+                    ('b', 'y'),
+                    ('c', 'z')],
+                   )
+        """
+        return self
+
+    T = property(transpose)
+
+    def _to_internal_pandas(self) -> pd.Index:
+        """
+        Return a pandas Index directly from _internal to avoid overhead of copy.
+
+        This method is for internal use only.
+        """
+        return self._kdf._internal.to_pandas_frame.index
+
+    def to_pandas(self) -> pd.Index:
+        """
+        Return a pandas Index.
+
+        .. note:: This method should only be used if the resulting pandas object is expected
+                  to be small, as all the data is loaded into the driver's memory.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([(.2, .3), (.0, .6), (.6, .0), (.2, .1)],
+        ...                   columns=['dogs', 'cats'],
+        ...                   index=list('abcd'))
+        >>> df['dogs'].index.to_pandas()
+        Index(['a', 'b', 'c', 'd'], dtype='object')
+        """
+        return self._to_internal_pandas().copy()
+
+    def toPandas(self) -> pd.Index:
+        warnings.warn(
+            "Index.toPandas is deprecated as of Index.to_pandas. Please use the API instead.",
+            FutureWarning,
+        )
+        return self.to_pandas()
+
+    toPandas.__doc__ = to_pandas.__doc__
+
+    def to_numpy(self, dtype=None, copy=False) -> np.ndarray:
+        """
+        A NumPy ndarray representing the values in this Index or MultiIndex.
+
+        .. note:: This method should only be used if the resulting NumPy ndarray is expected
+            to be small, as all the data is loaded into the driver's memory.
+
+        Parameters
+        ----------
+        dtype : str or numpy.dtype, optional
+            The dtype to pass to :meth:`numpy.asarray`
+        copy : bool, default False
+            Whether to ensure that the returned value is a not a view on
+            another array. Note that ``copy=False`` does not *ensure* that
+            ``to_numpy()`` is no-copy. Rather, ``copy=True`` ensure that
+            a copy is made, even if not strictly necessary.
+
+        Returns
+        -------
+        numpy.ndarray
+
+        Examples
+        --------
+        >>> ps.Series([1, 2, 3, 4]).index.to_numpy()
+        array([0, 1, 2, 3])
+        >>> ps.DataFrame({'a': ['a', 'b', 'c']}, index=[[1, 2, 3], [4, 5, 6]]).index.to_numpy()
+        array([(1, 4), (2, 5), (3, 6)], dtype=object)
+        """
+        result = np.asarray(self._to_internal_pandas()._values, dtype=dtype)
+        if copy:
+            result = result.copy()
+        return result
+
+    @property
+    def values(self) -> np.ndarray:
+        """
+        Return an array representing the data in the Index.
+
+        .. warning:: We recommend using `Index.to_numpy()` instead.
+
+        .. note:: This method should only be used if the resulting NumPy ndarray is expected
+            to be small, as all the data is loaded into the driver's memory.
+
+        Returns
+        -------
+        numpy.ndarray
+
+        Examples
+        --------
+        >>> ps.Series([1, 2, 3, 4]).index.values
+        array([0, 1, 2, 3])
+        >>> ps.DataFrame({'a': ['a', 'b', 'c']}, index=[[1, 2, 3], [4, 5, 6]]).index.values
+        array([(1, 4), (2, 5), (3, 6)], dtype=object)
+        """
+        warnings.warn("We recommend using `{}.to_numpy()` instead.".format(type(self).__name__))
+        return self.to_numpy()
+
+    @property
+    def asi8(self) -> np.ndarray:
+        """
+        Integer representation of the values.
+
+        .. warning:: We recommend using `Index.to_numpy()` instead.
+
+        .. note:: This method should only be used if the resulting NumPy ndarray is expected
+            to be small, as all the data is loaded into the driver's memory.
+
+        Returns
+        -------
+        numpy.ndarray
+            An ndarray with int64 dtype.
+
+        Examples
+        --------
+        >>> ps.Index([1, 2, 3]).asi8
+        array([1, 2, 3])
+
+        Returns None for non-int64 dtype
+
+        >>> ps.Index(['a', 'b', 'c']).asi8 is None
+        True
+        """
+        warnings.warn("We recommend using `{}.to_numpy()` instead.".format(type(self).__name__))
+        if isinstance(self.spark.data_type, IntegralType):
+            return self.to_numpy()
+        elif isinstance(self.spark.data_type, TimestampType):
+            return np.array(list(map(lambda x: x.astype(np.int64), self.to_numpy())))
+        else:
+            return None
+
+    @property
+    def spark_type(self) -> DataType:
+        """ Returns the data type as defined by Spark, as a Spark DataType object."""
+        warnings.warn(
+            "Index.spark_type is deprecated as of Index.spark.data_type. "
+            "Please use the API instead.",
+            FutureWarning,
+        )
+        return self.spark.data_type
+
+    @property
+    def has_duplicates(self) -> bool:
+        """
+        If index has duplicates, return True, otherwise False.
+
+        Examples
+        --------
+        >>> idx = ps.Index([1, 5, 7, 7])
+        >>> idx.has_duplicates
+        True
+
+        >>> idx = ps.Index([1, 5, 7])
+        >>> idx.has_duplicates
+        False
+
+        >>> idx = ps.Index(["Watermelon", "Orange", "Apple",
+        ...                 "Watermelon"])
+        >>> idx.has_duplicates
+        True
+
+        >>> idx = ps.Index(["Orange", "Apple",
+        ...                 "Watermelon"])
+        >>> idx.has_duplicates
+        False
+        """
+        sdf = self._internal.spark_frame.select(self.spark.column)
+        scol = scol_for(sdf, sdf.columns[0])
+
+        return sdf.select(F.count(scol) != F.countDistinct(scol)).first()[0]
+
+    @property
+    def is_unique(self) -> bool:
+        """
+        Return if the index has unique values.
+
+        Examples
+        --------
+        >>> idx = ps.Index([1, 5, 7, 7])
+        >>> idx.is_unique
+        False
+
+        >>> idx = ps.Index([1, 5, 7])
+        >>> idx.is_unique
+        True
+
+        >>> idx = ps.Index(["Watermelon", "Orange", "Apple",
+        ...                 "Watermelon"])
+        >>> idx.is_unique
+        False
+
+        >>> idx = ps.Index(["Orange", "Apple",
+        ...                 "Watermelon"])
+        >>> idx.is_unique
+        True
+        """
+        return not self.has_duplicates
+
+    @property
+    def name(self) -> Union[Any, Tuple]:
+        """Return name of the Index."""
+        return self.names[0]
+
+    @name.setter
+    def name(self, name: Union[Any, Tuple]) -> None:
+        self.names = [name]
+
+    @property
+    def names(self) -> List[Union[Any, Tuple]]:
+        """Return names of the Index."""
+        return [
+            name if name is None or len(name) > 1 else name[0]
+            for name in self._internal.index_names  # type: ignore
+        ]
+
+    @names.setter
+    def names(self, names: List[Union[Any, Tuple]]) -> None:
+        if not is_list_like(names):
+            raise ValueError("Names must be a list-like")
+        if self._internal.index_level != len(names):
+            raise ValueError(
+                "Length of new names must be {}, got {}".format(
+                    self._internal.index_level, len(names)
+                )
+            )
+        if self._internal.index_level == 1:
+            self.rename(names[0], inplace=True)
+        else:
+            self.rename(names, inplace=True)
+
+    @property
+    def nlevels(self) -> int:
+        """
+        Number of levels in Index & MultiIndex.
+
+        Examples
+        --------
+        >>> kdf = ps.DataFrame({"a": [1, 2, 3]}, index=pd.Index(['a', 'b', 'c'], name="idx"))
+        >>> kdf.index.nlevels
+        1
+
+        >>> kdf = ps.DataFrame({'a': [1, 2, 3]}, index=[list('abc'), list('def')])
+        >>> kdf.index.nlevels
+        2
+        """
+        return self._internal.index_level
+
+    def rename(
+        self, name: Union[Any, Tuple, List[Union[Any, Tuple]]], inplace: bool = False
+    ) -> Optional["Index"]:
+        """
+        Alter Index or MultiIndex name.
+        Able to set new names without level. Defaults to returning new index.
+
+        Parameters
+        ----------
+        name : label or list of labels
+            Name(s) to set.
+        inplace : boolean, default False
+            Modifies the object directly, instead of creating a new Index or MultiIndex.
+
+        Returns
+        -------
+        Index or MultiIndex
+            The same type as the caller or None if inplace is True.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'a': ['A', 'C'], 'b': ['A', 'B']}, columns=['a', 'b'])
+        >>> df.index.rename("c")
+        Int64Index([0, 1], dtype='int64', name='c')
+
+        >>> df.set_index("a", inplace=True)
+        >>> df.index.rename("d")
+        Index(['A', 'C'], dtype='object', name='d')
+
+        You can also change the index name in place.
+
+        >>> df.index.rename("e", inplace=True)
+        >>> df.index
+        Index(['A', 'C'], dtype='object', name='e')
+
+        >>> df  # doctest: +NORMALIZE_WHITESPACE
+           b
+        e
+        A  A
+        C  B
+
+        Support for MultiIndex
+
+        >>> kidx = ps.MultiIndex.from_tuples([('a', 'x'), ('b', 'y')])
+        >>> kidx.names = ['hello', 'pandas-on-Spark']
+        >>> kidx  # doctest: +SKIP
+        MultiIndex([('a', 'x'),
+                    ('b', 'y')],
+                   names=['hello', 'pandas-on-Spark'])
+
+        >>> kidx.rename(['aloha', 'databricks'])  # doctest: +SKIP
+        MultiIndex([('a', 'x'),
+                    ('b', 'y')],
+                   names=['aloha', 'databricks'])
+        """
+        names = self._verify_for_rename(name)
+
+        internal = self._kdf._internal.copy(index_names=names)
+
+        if inplace:
+            self._kdf._update_internal_frame(internal)
+            return None
+        else:
+            return DataFrame(internal).index
+
+    def _verify_for_rename(self, name):
+        if is_hashable(name):
+            if is_name_like_tuple(name):
+                return [name]
+            elif is_name_like_value(name):
+                return [(name,)]
+        raise TypeError("Index.name must be a hashable type")
+
+    # TODO: add downcast parameter for fillna function
+    def fillna(self, value: Scalar) -> "Index":
+        """
+        Fill NA/NaN values with the specified value.
+
+        Parameters
+        ----------
+        value : scalar
+            Scalar value to use to fill holes (example: 0). This value cannot be a list-likes.
+
+        Returns
+        -------
+        Index :
+            filled with value
+
+        Examples
+        --------
+        >>> ki = ps.DataFrame({'a': ['a', 'b', 'c']}, index=[1, 2, None]).index
+        >>> ki
+        Float64Index([1.0, 2.0, nan], dtype='float64')
+
+        >>> ki.fillna(0)
+        Float64Index([1.0, 2.0, 0.0], dtype='float64')
+        """
+        if not isinstance(value, (float, int, str, bool)):
+            raise TypeError("Unsupported type %s" % type(value).__name__)
+        sdf = self._internal.spark_frame.fillna(value)
+        result = DataFrame(self._kdf._internal.with_new_sdf(sdf)).index  # TODO: dtype?
+        return result
+
+    # TODO: ADD keep parameter
+    def drop_duplicates(self) -> "Index":
+        """
+        Return Index with duplicate values removed.
+
+        Returns
+        -------
+        deduplicated : Index
+
+        See Also
+        --------
+        Series.drop_duplicates : Equivalent method on Series.
+        DataFrame.drop_duplicates : Equivalent method on DataFrame.
+
+        Examples
+        --------
+        Generate an pandas.Index with duplicate values.
+
+        >>> idx = ps.Index(['lama', 'cow', 'lama', 'beetle', 'lama', 'hippo'])
+
+        >>> idx.drop_duplicates().sort_values()
+        Index(['beetle', 'cow', 'hippo', 'lama'], dtype='object')
+        """
+        sdf = self._internal.spark_frame.select(
+            self._internal.index_spark_columns
+        ).drop_duplicates()
+        internal = InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=[
+                scol_for(sdf, col) for col in self._internal.index_spark_column_names
+            ],
+            index_names=self._internal.index_names,
+            index_dtypes=self._internal.index_dtypes,
+        )
+        return DataFrame(internal).index
+
+    def to_series(self, name: Union[Any, Tuple] = None) -> Series:
+        """
+        Create a Series with both index and values equal to the index keys
+        useful with map for returning an indexer based on an index.
+
+        Parameters
+        ----------
+        name : string, optional
+            name of resulting Series. If None, defaults to name of original
+            index
+
+        Returns
+        -------
+        Series : dtype will be based on the type of the Index values.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([(.2, .3), (.0, .6), (.6, .0), (.2, .1)],
+        ...                   columns=['dogs', 'cats'],
+        ...                   index=list('abcd'))
+        >>> df['dogs'].index.to_series()
+        a    a
+        b    b
+        c    c
+        d    d
+        dtype: object
+        """
+        if not is_hashable(name):
+            raise TypeError("Series.name must be a hashable type")
+        scol = self.spark.column
+        if name is not None:
+            scol = scol.alias(name_like_string(name))
+        elif self._internal.index_level == 1:
+            name = self.name
+        column_labels = [
+            name if is_name_like_tuple(name) else (name,)
+        ]  # type: List[Optional[Tuple]]
+        internal = self._internal.copy(
+            column_labels=column_labels, data_spark_columns=[scol], column_label_names=None
+        )
+        return first_series(DataFrame(internal))
+
+    def to_frame(self, index=True, name=None) -> DataFrame:
+        """
+        Create a DataFrame with a column containing the Index.
+
+        Parameters
+        ----------
+        index : boolean, default True
+            Set the index of the returned DataFrame as the original Index.
+        name : object, default None
+            The passed name should substitute for the index name (if it has
+            one).
+
+        Returns
+        -------
+        DataFrame
+            DataFrame containing the original Index data.
+
+        See Also
+        --------
+        Index.to_series : Convert an Index to a Series.
+        Series.to_frame : Convert Series to DataFrame.
+
+        Examples
+        --------
+        >>> idx = ps.Index(['Ant', 'Bear', 'Cow'], name='animal')
+        >>> idx.to_frame()  # doctest: +NORMALIZE_WHITESPACE
+               animal
+        animal
+        Ant       Ant
+        Bear     Bear
+        Cow       Cow
+
+        By default, the original Index is reused. To enforce a new Index:
+
+        >>> idx.to_frame(index=False)
+          animal
+        0    Ant
+        1   Bear
+        2    Cow
+
+        To override the name of the resulting column, specify `name`:
+
+        >>> idx.to_frame(name='zoo')  # doctest: +NORMALIZE_WHITESPACE
+                 zoo
+        animal
+        Ant      Ant
+        Bear    Bear
+        Cow      Cow
+        """
+        if name is None:
+            if self._internal.index_names[0] is None:
+                name = (DEFAULT_SERIES_NAME,)
+            else:
+                name = self._internal.index_names[0]
+        elif not is_name_like_tuple(name):
+            if is_name_like_value(name):
+                name = (name,)
+            else:
+                raise TypeError("unhashable type: '{}'".format(type(name).__name__))
+
+        return self._to_frame(index=index, names=[name])
+
+    def _to_frame(self, index, names):
+        if index:
+            index_spark_columns = self._internal.index_spark_columns
+            index_names = self._internal.index_names
+            index_dtypes = self._internal.index_dtypes
+        else:
+            index_spark_columns = []
+            index_names = []
+            index_dtypes = []
+
+        internal = InternalFrame(
+            spark_frame=self._internal.spark_frame,
+            index_spark_columns=index_spark_columns,
+            index_names=index_names,
+            index_dtypes=index_dtypes,
+            column_labels=names,
+            data_spark_columns=self._internal.index_spark_columns,
+            data_dtypes=self._internal.index_dtypes,
+        )
+        return DataFrame(internal)
+
+    def is_boolean(self) -> bool:
+        """
+        Return if the current index type is a boolean type.
+
+        Examples
+        --------
+        >>> ps.DataFrame({'a': [1]}, index=[True]).index.is_boolean()
+        True
+        """
+        return is_bool_dtype(self.dtype)
+
+    def is_categorical(self) -> bool:
+        """
+        Return if the current index type is a categorical type.
+
+        Examples
+        --------
+        >>> ps.DataFrame({'a': [1]}, index=[1]).index.is_categorical()
+        False
+        """
+        return is_categorical_dtype(self.dtype)
+
+    def is_floating(self) -> bool:
+        """
+        Return if the current index type is a floating type.
+
+        Examples
+        --------
+        >>> ps.DataFrame({'a': [1]}, index=[1]).index.is_floating()
+        False
+        """
+        return is_float_dtype(self.dtype)
+
+    def is_integer(self) -> bool:
+        """
+        Return if the current index type is a integer type.
+
+        Examples
+        --------
+        >>> ps.DataFrame({'a': [1]}, index=[1]).index.is_integer()
+        True
+        """
+        return is_integer_dtype(self.dtype)
+
+    def is_interval(self) -> bool:
+        """
+        Return if the current index type is an interval type.
+
+        Examples
+        --------
+        >>> ps.DataFrame({'a': [1]}, index=[1]).index.is_interval()
+        False
+        """
+        return is_interval_dtype(self.dtype)
+
+    def is_numeric(self) -> bool:
+        """
+        Return if the current index type is a numeric type.
+
+        Examples
+        --------
+        >>> ps.DataFrame({'a': [1]}, index=[1]).index.is_numeric()
+        True
+        """
+        return is_numeric_dtype(self.dtype)
+
+    def is_object(self) -> bool:
+        """
+        Return if the current index type is a object type.
+
+        Examples
+        --------
+        >>> ps.DataFrame({'a': [1]}, index=["a"]).index.is_object()
+        True
+        """
+        return is_object_dtype(self.dtype)
+
+    def is_type_compatible(self, kind) -> bool:
+        """
+        Whether the index type is compatible with the provided type.
+
+        Examples
+        --------
+        >>> kidx = ps.Index([1, 2, 3])
+        >>> kidx.is_type_compatible('integer')
+        True
+
+        >>> kidx = ps.Index([1.0, 2.0, 3.0])
+        >>> kidx.is_type_compatible('integer')
+        False
+        >>> kidx.is_type_compatible('floating')
+        True
+        """
+        return kind == self.inferred_type
+
+    def dropna(self) -> "Index":
+        """
+        Return Index or MultiIndex without NA/NaN values
+
+        Examples
+        --------
+
+        >>> df = ps.DataFrame([[1, 2], [4, 5], [7, 8]],
+        ...                   index=['cobra', 'viper', None],
+        ...                   columns=['max_speed', 'shield'])
+        >>> df
+               max_speed  shield
+        cobra          1       2
+        viper          4       5
+        NaN            7       8
+
+        >>> df.index.dropna()
+        Index(['cobra', 'viper'], dtype='object')
+
+        Also support for MultiIndex
+
+        >>> midx = pd.MultiIndex([['lama', 'cow', 'falcon'],
+        ...                       [None, 'weight', 'length']],
+        ...                      [[0, 1, 1, 1, 1, 1, 2, 2, 2],
+        ...                       [0, 1, 1, 0, 1, 2, 1, 1, 2]])
+        >>> s = ps.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, None],
+        ...               index=midx)
+        >>> s
+        lama    NaN        45.0
+        cow     weight    200.0
+                weight      1.2
+                NaN        30.0
+                weight    250.0
+                length      1.5
+        falcon  weight    320.0
+                weight      1.0
+                length      NaN
+        dtype: float64
+
+        >>> s.index.dropna()  # doctest: +SKIP
+        MultiIndex([(   'cow', 'weight'),
+                    (   'cow', 'weight'),
+                    (   'cow', 'weight'),
+                    (   'cow', 'length'),
+                    ('falcon', 'weight'),
+                    ('falcon', 'weight'),
+                    ('falcon', 'length')],
+                   )
+        """
+        sdf = self._internal.spark_frame.select(self._internal.index_spark_columns).dropna()
+        internal = InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=[
+                scol_for(sdf, col) for col in self._internal.index_spark_column_names
+            ],
+            index_names=self._internal.index_names,
+            index_dtypes=self._internal.index_dtypes,
+        )
+        return DataFrame(internal).index
+
+    def unique(self, level=None) -> "Index":
+        """
+        Return unique values in the index.
+
+        Be aware the order of unique values might be different than pandas.Index.unique
+
+        Parameters
+        ----------
+        level : int or str, optional, default is None
+
+        Returns
+        -------
+        Index without duplicates
+
+        See Also
+        --------
+        Series.unique
+        groupby.SeriesGroupBy.unique
+
+        Examples
+        --------
+        >>> ps.DataFrame({'a': ['a', 'b', 'c']}, index=[1, 1, 3]).index.unique().sort_values()
+        Int64Index([1, 3], dtype='int64')
+
+        >>> ps.DataFrame({'a': ['a', 'b', 'c']}, index=['d', 'e', 'e']).index.unique().sort_values()
+        Index(['d', 'e'], dtype='object')
+
+        MultiIndex
+
+        >>> ps.MultiIndex.from_tuples([("A", "X"), ("A", "Y"), ("A", "X")]).unique()
+        ... # doctest: +SKIP
+        MultiIndex([('A', 'X'),
+                    ('A', 'Y')],
+                   )
+        """
+        if level is not None:
+            self._validate_index_level(level)
+        scols = self._internal.index_spark_columns
+        sdf = self._kdf._internal.spark_frame.select(scols).distinct()
+        return DataFrame(
+            InternalFrame(
+                spark_frame=sdf,
+                index_spark_columns=[
+                    scol_for(sdf, col) for col in self._internal.index_spark_column_names
+                ],
+                index_names=self._internal.index_names,
+                index_dtypes=self._internal.index_dtypes,
+            )
+        ).index
+
+    # TODO: add error parameter
+    def drop(self, labels) -> "Index":
+        """
+        Make new Index with passed list of labels deleted.
+
+        Parameters
+        ----------
+        labels : array-like
+
+        Returns
+        -------
+        dropped : Index
+
+        Examples
+        --------
+        >>> index = ps.Index([1, 2, 3])
+        >>> index
+        Int64Index([1, 2, 3], dtype='int64')
+
+        >>> index.drop([1])
+        Int64Index([2, 3], dtype='int64')
+        """
+        internal = self._internal.resolved_copy
+        sdf = internal.spark_frame[~internal.index_spark_columns[0].isin(labels)]
+
+        internal = InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=[
+                scol_for(sdf, col) for col in self._internal.index_spark_column_names
+            ],
+            index_names=self._internal.index_names,
+            index_dtypes=self._internal.index_dtypes,
+            column_labels=[],
+            data_spark_columns=[],
+            data_dtypes=[],
+        )
+        return DataFrame(internal).index
+
+    def _validate_index_level(self, level):
+        """
+        Validate index level.
+        For single-level Index getting level number is a no-op, but some
+        verification must be done like in MultiIndex.
+        """
+        if isinstance(level, int):
+            if level < 0 and level != -1:
+                raise IndexError(
+                    "Too many levels: Index has only 1 level,"
+                    " %d is not a valid level number" % (level,)
+                )
+            elif level > 0:
+                raise IndexError("Too many levels:" " Index has only 1 level, not %d" % (level + 1))
+        elif level != self.name:
+            raise KeyError(
+                "Requested level ({}) does not match index name ({})".format(level, self.name)
+            )
+
+    def get_level_values(self, level) -> "Index":
+        """
+        Return Index if a valid level is given.
+
+        Examples:
+        --------
+        >>> kidx = ps.Index(['a', 'b', 'c'], name='ks')
+        >>> kidx.get_level_values(0)
+        Index(['a', 'b', 'c'], dtype='object', name='ks')
+
+        >>> kidx.get_level_values('ks')
+        Index(['a', 'b', 'c'], dtype='object', name='ks')
+        """
+        self._validate_index_level(level)
+        return self
+
+    def copy(self, name=None, deep=None) -> "Index":
+        """
+        Make a copy of this object. name sets those attributes on the new object.
+
+        Parameters
+        ----------
+        name : string, optional
+            to set name of index
+        deep : None
+            this parameter is not supported but just dummy parameter to match pandas.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([[1, 2], [4, 5], [7, 8]],
+        ...                   index=['cobra', 'viper', 'sidewinder'],
+        ...                   columns=['max_speed', 'shield'])
+        >>> df
+                    max_speed  shield
+        cobra               1       2
+        viper               4       5
+        sidewinder          7       8
+        >>> df.index
+        Index(['cobra', 'viper', 'sidewinder'], dtype='object')
+
+        Copy index
+
+        >>> df.index.copy()
+        Index(['cobra', 'viper', 'sidewinder'], dtype='object')
+
+        Copy index with name
+
+        >>> df.index.copy(name='snake')
+        Index(['cobra', 'viper', 'sidewinder'], dtype='object', name='snake')
+        """
+        result = self._kdf.copy().index
+        if name:
+            result.name = name
+        return result
+
+    def droplevel(self, level) -> "Index":
+        """
+        Return index with requested level(s) removed.
+        If resulting index has only 1 level left, the result will be
+        of Index type, not MultiIndex.
+
+        Parameters
+        ----------
+        level : int, str, tuple, or list-like, default 0
+            If a string is given, must be the name of a level
+            If list-like, elements must be names or indexes of levels.
+
+        Returns
+        -------
+        Index or MultiIndex
+
+        Examples
+        --------
+        >>> midx = ps.DataFrame({'a': ['a', 'b']}, index=[['a', 'x'], ['b', 'y'], [1, 2]]).index
+        >>> midx  # doctest: +SKIP
+        MultiIndex([('a', 'b', 1),
+                    ('x', 'y', 2)],
+                   )
+        >>> midx.droplevel([0, 1])  # doctest: +SKIP
+        Int64Index([1, 2], dtype='int64')
+        >>> midx.droplevel(0)  # doctest: +SKIP
+        MultiIndex([('b', 1),
+                    ('y', 2)],
+                   )
+        >>> midx.names = [("a", "b"), "b", "c"]
+        >>> midx.droplevel([('a', 'b')])  # doctest: +SKIP
+        MultiIndex([('b', 1),
+                    ('y', 2)],
+                   names=['b', 'c'])
+        """
+        names = self.names
+        nlevels = self.nlevels
+        if not is_list_like(level):
+            level = [level]
+
+        int_level = set()
+        for n in level:
+            if isinstance(n, int):
+                if n < 0:
+                    n = n + nlevels
+                    if n < 0:
+                        raise IndexError(
+                            "Too many levels: Index has only {} levels, "
+                            "{} is not a valid level number".format(nlevels, (n - nlevels))
+                        )
+                if n >= nlevels:
+                    raise IndexError(
+                        "Too many levels: Index has only {} levels, not {}".format(nlevels, n + 1)
+                    )
+            else:
+                if n not in names:
+                    raise KeyError("Level {} not found".format(n))
+                n = names.index(n)
+            int_level.add(n)
+
+        if len(level) >= nlevels:
+            raise ValueError(
+                "Cannot remove {} levels from an index with {} "
+                "levels: at least one level must be "
+                "left.".format(len(level), nlevels)
+            )
+
+        index_spark_columns, index_names, index_dtypes = zip(
+            *[
+                item
+                for i, item in enumerate(
+                    zip(
+                        self._internal.index_spark_columns,
+                        self._internal.index_names,
+                        self._internal.index_dtypes,
+                    )
+                )
+                if i not in int_level
+            ]
+        )
+
+        internal = self._internal.copy(
+            index_spark_columns=list(index_spark_columns),
+            index_names=list(index_names),
+            index_dtypes=list(index_dtypes),
+            column_labels=[],
+            data_spark_columns=[],
+            data_dtypes=[],
+        )
+        return DataFrame(internal).index
+
+    def symmetric_difference(self, other, result_name=None, sort=None) -> "Index":
+        """
+        Compute the symmetric difference of two Index objects.
+
+        Parameters
+        ----------
+        other : Index or array-like
+        result_name : str
+        sort : True or None, default None
+            Whether to sort the resulting index.
+            * True : Attempt to sort the result.
+            * None : Do not sort the result.
+
+        Returns
+        -------
+        symmetric_difference : Index
+
+        Notes
+        -----
+        ``symmetric_difference`` contains elements that appear in either
+        ``idx1`` or ``idx2`` but not both. Equivalent to the Index created by
+        ``idx1.difference(idx2) | idx2.difference(idx1)`` with duplicates
+        dropped.
+
+        Examples
+        --------
+        >>> s1 = ps.Series([1, 2, 3, 4], index=[1, 2, 3, 4])
+        >>> s2 = ps.Series([1, 2, 3, 4], index=[2, 3, 4, 5])
+
+        >>> s1.index.symmetric_difference(s2.index)  # doctest: +SKIP
+        Int64Index([5, 1], dtype='int64')
+
+        You can set name of result Index.
+
+        >>> s1.index.symmetric_difference(s2.index, result_name='pandas-on-Spark')  # doctest: +SKIP
+        Int64Index([5, 1], dtype='int64', name='pandas-on-Spark')
+
+        You can set sort to `True`, if you want to sort the resulting index.
+
+        >>> s1.index.symmetric_difference(s2.index, sort=True)
+        Int64Index([1, 5], dtype='int64')
+
+        You can also use the ``^`` operator:
+
+        >>> s1.index ^ s2.index  # doctest: +SKIP
+        Int64Index([5, 1], dtype='int64')
+        """
+        if type(self) != type(other):
+            raise NotImplementedError(
+                "Doesn't support symmetric_difference between Index & MultiIndex for now"
+            )
+
+        sdf_self = self._kdf._internal.spark_frame.select(self._internal.index_spark_columns)
+        sdf_other = other._kdf._internal.spark_frame.select(other._internal.index_spark_columns)
+
+        sdf_symdiff = sdf_self.union(sdf_other).subtract(sdf_self.intersect(sdf_other))
+
+        if sort:
+            sdf_symdiff = sdf_symdiff.sort(self._internal.index_spark_column_names)
+
+        internal = InternalFrame(
+            spark_frame=sdf_symdiff,
+            index_spark_columns=[
+                scol_for(sdf_symdiff, col) for col in self._internal.index_spark_column_names
+            ],
+            index_names=self._internal.index_names,
+            index_dtypes=self._internal.index_dtypes,
+        )
+        result = DataFrame(internal).index
+
+        if result_name:
+            result.name = result_name
+
+        return result
+
+    # TODO: return_indexer
+    def sort_values(self, ascending=True) -> "Index":
+        """
+        Return a sorted copy of the index.
+
+        .. note:: This method is not supported for pandas when index has NaN value.
+                  pandas raises unexpected TypeError, but we support treating NaN
+                  as the smallest value.
+
+        Parameters
+        ----------
+        ascending : bool, default True
+            Should the index values be sorted in an ascending order.
+
+        Returns
+        -------
+        sorted_index : ps.Index or ps.MultiIndex
+            Sorted copy of the index.
+
+        See Also
+        --------
+        Series.sort_values : Sort values of a Series.
+        DataFrame.sort_values : Sort values in a DataFrame.
+
+        Examples
+        --------
+        >>> idx = ps.Index([10, 100, 1, 1000])
+        >>> idx
+        Int64Index([10, 100, 1, 1000], dtype='int64')
+
+        Sort values in ascending order (default behavior).
+
+        >>> idx.sort_values()
+        Int64Index([1, 10, 100, 1000], dtype='int64')
+
+        Sort values in descending order.
+
+        >>> idx.sort_values(ascending=False)
+        Int64Index([1000, 100, 10, 1], dtype='int64')
+
+        Support for MultiIndex.
+
+        >>> kidx = ps.MultiIndex.from_tuples([('a', 'x', 1), ('c', 'y', 2), ('b', 'z', 3)])
+        >>> kidx  # doctest: +SKIP
+        MultiIndex([('a', 'x', 1),
+                    ('c', 'y', 2),
+                    ('b', 'z', 3)],
+                   )
+
+        >>> kidx.sort_values()  # doctest: +SKIP
+        MultiIndex([('a', 'x', 1),
+                    ('b', 'z', 3),
+                    ('c', 'y', 2)],
+                   )
+
+        >>> kidx.sort_values(ascending=False)  # doctest: +SKIP
+        MultiIndex([('c', 'y', 2),
+                    ('b', 'z', 3),
+                    ('a', 'x', 1)],
+                   )
+        """
+        sdf = self._internal.spark_frame
+        sdf = sdf.orderBy(self._internal.index_spark_columns, ascending=ascending).select(
+            self._internal.index_spark_columns
+        )
+
+        internal = InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=[
+                scol_for(sdf, col) for col in self._internal.index_spark_column_names
+            ],
+            index_names=self._internal.index_names,
+            index_dtypes=self._internal.index_dtypes,
+        )
+        return DataFrame(internal).index
+
+    def sort(self, *args, **kwargs) -> None:
+        """
+        Use sort_values instead.
+        """
+        raise TypeError("cannot sort an Index object in-place, use sort_values instead")
+
+    def min(self) -> Union[Scalar, Tuple[Scalar, ...]]:
+        """
+        Return the minimum value of the Index.
+
+        Returns
+        -------
+        scalar
+            Minimum value.
+
+        See Also
+        --------
+        Index.max : Return the maximum value of the object.
+        Series.min : Return the minimum value in a Series.
+        DataFrame.min : Return the minimum values in a DataFrame.
+
+        Examples
+        --------
+        >>> idx = ps.Index([3, 2, 1])
+        >>> idx.min()
+        1
+
+        >>> idx = ps.Index(['c', 'b', 'a'])
+        >>> idx.min()
+        'a'
+
+        For a MultiIndex, the maximum is determined lexicographically.
+
+        >>> idx = ps.MultiIndex.from_tuples([('a', 'x', 1), ('b', 'y', 2)])
+        >>> idx.min()
+        ('a', 'x', 1)
+        """
+        sdf = self._internal.spark_frame
+        min_row = (
+            sdf.select(F.min(F.struct(self._internal.index_spark_columns)).alias("min_row"))
+            .select("min_row.*")
+            .toPandas()
+        )
+        result = tuple(min_row.iloc[0])
+
+        return result if len(result) > 1 else result[0]
+
+    def max(self) -> Union[Scalar, Tuple[Scalar, ...]]:
+        """
+        Return the maximum value of the Index.
+
+        Returns
+        -------
+        scalar
+            Maximum value.
+
+        See Also
+        --------
+        Index.min : Return the minimum value in an Index.
+        Series.max : Return the maximum value in a Series.
+        DataFrame.max : Return the maximum values in a DataFrame.
+
+        Examples
+        --------
+        >>> idx = ps.Index([3, 2, 1])
+        >>> idx.max()
+        3
+
+        >>> idx = ps.Index(['c', 'b', 'a'])
+        >>> idx.max()
+        'c'
+
+        For a MultiIndex, the maximum is determined lexicographically.
+
+        >>> idx = ps.MultiIndex.from_tuples([('a', 'x', 1), ('b', 'y', 2)])
+        >>> idx.max()
+        ('b', 'y', 2)
+        """
+        sdf = self._internal.spark_frame
+        max_row = (
+            sdf.select(F.max(F.struct(self._internal.index_spark_columns)).alias("max_row"))
+            .select("max_row.*")
+            .toPandas()
+        )
+        result = tuple(max_row.iloc[0])
+
+        return result if len(result) > 1 else result[0]
+
+    def delete(self, loc) -> "Index":
+        """
+        Make new Index with passed location(-s) deleted.
+
+        .. note:: this API can be pretty expensive since it is based on
+             a global sequence internally.
+
+        Returns
+        -------
+        new_index : Index
+
+        Examples
+        --------
+        >>> kidx = ps.Index([10, 10, 9, 8, 4, 2, 4, 4, 2, 2, 10, 10])
+        >>> kidx
+        Int64Index([10, 10, 9, 8, 4, 2, 4, 4, 2, 2, 10, 10], dtype='int64')
+
+        >>> kidx.delete(0).sort_values()
+        Int64Index([2, 2, 2, 4, 4, 4, 8, 9, 10, 10, 10], dtype='int64')
+
+        >>> kidx.delete([0, 1, 2, 3, 10, 11]).sort_values()
+        Int64Index([2, 2, 2, 4, 4, 4], dtype='int64')
+
+        MultiIndex
+
+        >>> kidx = ps.MultiIndex.from_tuples([('a', 'x', 1), ('b', 'y', 2), ('c', 'z', 3)])
+        >>> kidx  # doctest: +SKIP
+        MultiIndex([('a', 'x', 1),
+                    ('b', 'y', 2),
+                    ('c', 'z', 3)],
+                   )
+
+        >>> kidx.delete([0, 2]).sort_values()  # doctest: +SKIP
+        MultiIndex([('b', 'y', 2)],
+                   )
+        """
+        length = len(self)
+
+        def is_len_exceeded(index):
+            """Check if the given index is exceeded the length or not"""
+            return index >= length if index >= 0 else abs(index) > length
+
+        if not is_list_like(loc):
+            if is_len_exceeded(loc):
+                raise IndexError(
+                    "index {} is out of bounds for axis 0 with size {}".format(loc, length)
+                )
+            loc = [loc]
+        else:
+            for index in loc:
+                if is_len_exceeded(index):
+                    raise IndexError(
+                        "index {} is out of bounds for axis 0 with size {}".format(index, length)
+                    )
+
+        loc = [int(item) for item in loc]
+        loc = [item if item >= 0 else length + item for item in loc]
+
+        # we need a temporary column such as '__index_value_0__'
+        # since 'InternalFrame.attach_default_index' will be failed
+        # when self._scol has name of '__index_level_0__'
+        index_value_column_format = "__index_value_{}__"
+
+        sdf = self._internal._sdf
+        index_value_column_names = [
+            verify_temp_column_name(sdf, index_value_column_format.format(i))
+            for i in range(self._internal.index_level)
+        ]
+        index_value_columns = [
+            index_scol.alias(index_vcol_name)
+            for index_scol, index_vcol_name in zip(
+                self._internal.index_spark_columns, index_value_column_names
+            )
+        ]
+        sdf = sdf.select(index_value_columns)
+
+        sdf = InternalFrame.attach_default_index(sdf, default_index_type="distributed-sequence")
+        # sdf here looks as below
+        # +-----------------+-----------------+-----------------+-----------------+
+        # |__index_level_0__|__index_value_0__|__index_value_1__|__index_value_2__|
+        # +-----------------+-----------------+-----------------+-----------------+
+        # |                0|                a|                x|                1|
+        # |                1|                b|                y|                2|
+        # |                2|                c|                z|                3|
+        # +-----------------+-----------------+-----------------+-----------------+
+
+        # delete rows which are matched with given `loc`
+        sdf = sdf.where(~F.col(SPARK_INDEX_NAME_FORMAT(0)).isin(loc))
+        sdf = sdf.select(index_value_column_names)
+        # sdf here looks as below, we should alias them back to origin spark column names
+        # +-----------------+-----------------+-----------------+
+        # |__index_value_0__|__index_value_1__|__index_value_2__|
+        # +-----------------+-----------------+-----------------+
+        # |                c|                z|                3|
+        # +-----------------+-----------------+-----------------+
+        index_origin_columns = [
+            F.col(index_vcol_name).alias(index_scol_name)
+            for index_vcol_name, index_scol_name in zip(
+                index_value_column_names, self._internal.index_spark_column_names
+            )
+        ]
+        sdf = sdf.select(index_origin_columns)
+
+        internal = InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=[
+                scol_for(sdf, col) for col in self._internal.index_spark_column_names
+            ],
+            index_names=self._internal.index_names,
+            index_dtypes=self._internal.index_dtypes,
+        )
+
+        return DataFrame(internal).index
+
+    def append(self, other: "Index") -> "Index":
+        """
+        Append a collection of Index options together.
+
+        Parameters
+        ----------
+        other : Index
+
+        Returns
+        -------
+        appended : Index
+
+        Examples
+        --------
+        >>> kidx = ps.Index([10, 5, 0, 5, 10, 5, 0, 10])
+        >>> kidx
+        Int64Index([10, 5, 0, 5, 10, 5, 0, 10], dtype='int64')
+
+        >>> kidx.append(kidx)
+        Int64Index([10, 5, 0, 5, 10, 5, 0, 10, 10, 5, 0, 5, 10, 5, 0, 10], dtype='int64')
+
+        Support for MiltiIndex
+
+        >>> kidx = ps.MultiIndex.from_tuples([('a', 'x'), ('b', 'y')])
+        >>> kidx  # doctest: +SKIP
+        MultiIndex([('a', 'x'),
+                    ('b', 'y')],
+                   )
+
+        >>> kidx.append(kidx)  # doctest: +SKIP
+        MultiIndex([('a', 'x'),
+                    ('b', 'y'),
+                    ('a', 'x'),
+                    ('b', 'y')],
+                   )
+        """
+        from pyspark.pandas.indexes.multi import MultiIndex
+
+        if type(self) is not type(other):
+            raise NotImplementedError(
+                "append() between Index & MultiIndex currently is not supported"
+            )
+
+        sdf_self = self._internal.spark_frame.select(self._internal.index_spark_columns)
+        sdf_other = other._internal.spark_frame.select(other._internal.index_spark_columns)
+        sdf_appended = sdf_self.union(sdf_other)
+
+        # names should be kept when MultiIndex, but Index wouldn't keep its name.
+        if isinstance(self, MultiIndex):
+            index_names = self._internal.index_names
+        else:
+            index_names = None
+
+        internal = InternalFrame(  # TODO: dtypes?
+            spark_frame=sdf_appended,
+            index_spark_columns=[
+                scol_for(sdf_appended, col) for col in self._internal.index_spark_column_names
+            ],
+            index_names=index_names,
+        )
+
+        return DataFrame(internal).index
+
+    def argmax(self) -> int:
+        """
+        Return a maximum argument indexer.
+
+        Parameters
+        ----------
+        skipna : bool, default True
+
+        Returns
+        -------
+        maximum argument indexer
+
+        Examples
+        --------
+        >>> kidx = ps.Index([10, 9, 8, 7, 100, 5, 4, 3, 100, 3])
+        >>> kidx
+        Int64Index([10, 9, 8, 7, 100, 5, 4, 3, 100, 3], dtype='int64')
+
+        >>> kidx.argmax()
+        4
+        """
+        sdf = self._internal.spark_frame.select(self.spark.column)
+        sequence_col = verify_temp_column_name(sdf, "__distributed_sequence_column__")
+        sdf = InternalFrame.attach_distributed_sequence_column(sdf, column_name=sequence_col)
+        # spark_frame here looks like below
+        # +-----------------+---------------+
+        # |__index_level_0__|__index_value__|
+        # +-----------------+---------------+
+        # |                0|             10|
+        # |                4|            100|
+        # |                2|              8|
+        # |                3|              7|
+        # |                6|              4|
+        # |                5|              5|
+        # |                7|              3|
+        # |                8|            100|
+        # |                1|              9|
+        # +-----------------+---------------+
+
+        return (
+            sdf.orderBy(
+                scol_for(sdf, self._internal.data_spark_column_names[0]).desc(),
+                F.col(sequence_col).asc(),
+            )
+            .select(sequence_col)
+            .first()[0]
+        )
+
+    def argmin(self) -> int:
+        """
+        Return a minimum argument indexer.
+
+        Parameters
+        ----------
+        skipna : bool, default True
+
+        Returns
+        -------
+        minimum argument indexer
+
+        Examples
+        --------
+        >>> kidx = ps.Index([10, 9, 8, 7, 100, 5, 4, 3, 100, 3])
+        >>> kidx
+        Int64Index([10, 9, 8, 7, 100, 5, 4, 3, 100, 3], dtype='int64')
+
+        >>> kidx.argmin()
+        7
+        """
+        sdf = self._internal.spark_frame.select(self.spark.column)
+        sequence_col = verify_temp_column_name(sdf, "__distributed_sequence_column__")
+        sdf = InternalFrame.attach_distributed_sequence_column(sdf, column_name=sequence_col)
+
+        return (
+            sdf.orderBy(
+                scol_for(sdf, self._internal.data_spark_column_names[0]).asc(),
+                F.col(sequence_col).asc(),
+            )
+            .select(sequence_col)
+            .first()[0]
+        )
+
+    def set_names(self, names, level=None, inplace=False) -> Optional["Index"]:
+        """
+        Set Index or MultiIndex name.
+        Able to set new names partially and by level.
+
+        Parameters
+        ----------
+        names : label or list of label
+            Name(s) to set.
+        level : int, label or list of int or label, optional
+            If the index is a MultiIndex, level(s) to set (None for all
+            levels). Otherwise level must be None.
+        inplace : bool, default False
+            Modifies the object directly, instead of creating a new Index or
+            MultiIndex.
+
+        Returns
+        -------
+        Index
+            The same type as the caller or None if inplace is True.
+
+        See Also
+        --------
+        Index.rename : Able to set new names without level.
+
+        Examples
+        --------
+        >>> idx = ps.Index([1, 2, 3, 4])
+        >>> idx
+        Int64Index([1, 2, 3, 4], dtype='int64')
+
+        >>> idx.set_names('quarter')
+        Int64Index([1, 2, 3, 4], dtype='int64', name='quarter')
+
+        For MultiIndex
+
+        >>> idx = ps.MultiIndex.from_tuples([('a', 'x'), ('b', 'y')])
+        >>> idx  # doctest: +SKIP
+        MultiIndex([('a', 'x'),
+                    ('b', 'y')],
+                   )
+
+        >>> idx.set_names(['kind', 'year'], inplace=True)
+        >>> idx  # doctest: +SKIP
+        MultiIndex([('a', 'x'),
+                    ('b', 'y')],
+                   names=['kind', 'year'])
+
+        >>> idx.set_names('species', level=0)  # doctest: +SKIP
+        MultiIndex([('a', 'x'),
+                    ('b', 'y')],
+                   names=['species', 'year'])
+        """
+        from pyspark.pandas.indexes.multi import MultiIndex
+
+        if isinstance(self, MultiIndex):
+            if level is not None:
+                self_names = self.names
+                self_names[level] = names
+                names = self_names
+        return self.rename(name=names, inplace=inplace)
+
+    def difference(self, other, sort=None) -> "Index":
+        """
+        Return a new Index with elements from the index that are not in
+        `other`.
+
+        This is the set difference of two Index objects.
+
+        Parameters
+        ----------
+        other : Index or array-like
+        sort : True or None, default None
+            Whether to sort the resulting index.
+            * True : Attempt to sort the result.
+            * None : Do not sort the result.
+
+        Returns
+        -------
+        difference : Index
+
+        Examples
+        --------
+
+        >>> idx1 = ps.Index([2, 1, 3, 4])
+        >>> idx2 = ps.Index([3, 4, 5, 6])
+        >>> idx1.difference(idx2, sort=True)
+        Int64Index([1, 2], dtype='int64')
+
+        MultiIndex
+
+        >>> midx1 = ps.MultiIndex.from_tuples([('a', 'x', 1), ('b', 'y', 2), ('c', 'z', 3)])
+        >>> midx2 = ps.MultiIndex.from_tuples([('a', 'x', 1), ('b', 'z', 2), ('k', 'z', 3)])
+        >>> midx1.difference(midx2)  # doctest: +SKIP
+        MultiIndex([('b', 'y', 2),
+                    ('c', 'z', 3)],
+                   )
+        """
+        from pyspark.pandas.indexes.multi import MultiIndex
+
+        if not is_list_like(other):
+            raise TypeError("Input must be Index or array-like")
+        if not isinstance(sort, (type(None), type(True))):
+            raise ValueError(
+                "The 'sort' keyword only takes the values of None or True; {} was passed.".format(
+                    sort
+                )
+            )
+        # Handling MultiIndex
+        if isinstance(self, MultiIndex) and not isinstance(other, MultiIndex):
+            if not all([isinstance(item, tuple) for item in other]):
+                raise TypeError("other must be a MultiIndex or a list of tuples")
+            other = MultiIndex.from_tuples(other)
+
+        if not isinstance(other, Index):
+            other = Index(other)
+
+        sdf_self = self._internal.spark_frame
+        sdf_other = other._internal.spark_frame
+        idx_self = self._internal.index_spark_columns
+        idx_other = other._internal.index_spark_columns
+        sdf_diff = sdf_self.select(idx_self).subtract(sdf_other.select(idx_other))
+        internal = InternalFrame(
+            spark_frame=sdf_diff,
+            index_spark_columns=[
+                scol_for(sdf_diff, col) for col in self._internal.index_spark_column_names
+            ],
+            index_names=self._internal.index_names,
+            index_dtypes=self._internal.index_dtypes,
+        )
+        result = DataFrame(internal).index
+        # Name(s) will be kept when only name(s) of (Multi)Index are the same.
+        if isinstance(self, type(other)) and isinstance(self, MultiIndex):
+            if self.names == other.names:
+                result.names = self.names
+        elif isinstance(self, type(other)) and not isinstance(self, MultiIndex):
+            if self.name == other.name:
+                result.name = self.name
+        return result if sort is None else result.sort_values()
+
+    @property
+    def is_all_dates(self) -> bool:
+        """
+        Return if all data types of the index are datetime.
+        remember that since pandas-on-Spark does not support multiple data types in an index,
+        so it returns True if any type of data is datetime.
+
+        Examples
+        --------
+        >>> from datetime import datetime
+
+        >>> idx = ps.Index([datetime(2019, 1, 1, 0, 0, 0), datetime(2019, 2, 3, 0, 0, 0)])
+        >>> idx
+        DatetimeIndex(['2019-01-01', '2019-02-03'], dtype='datetime64[ns]', freq=None)
+
+        >>> idx.is_all_dates
+        True
+
+        >>> idx = ps.Index([datetime(2019, 1, 1, 0, 0, 0), None])
+        >>> idx
+        DatetimeIndex(['2019-01-01', 'NaT'], dtype='datetime64[ns]', freq=None)
+
+        >>> idx.is_all_dates
+        True
+
+        >>> idx = ps.Index([0, 1, 2])
+        >>> idx
+        Int64Index([0, 1, 2], dtype='int64')
+
+        >>> idx.is_all_dates
+        False
+        """
+        return isinstance(self.spark.data_type, TimestampType)
+
+    def repeat(self, repeats: int) -> "Index":
+        """
+        Repeat elements of a Index/MultiIndex.
+
+        Returns a new Index/MultiIndex where each element of the current Index/MultiIndex
+        is repeated consecutively a given number of times.
+
+        Parameters
+        ----------
+        repeats : int
+            The number of repetitions for each element. This should be a
+            non-negative integer. Repeating 0 times will return an empty
+            Index.
+
+        Returns
+        -------
+        repeated_index : Index/MultiIndex
+            Newly created Index/MultiIndex with repeated elements.
+
+        See Also
+        --------
+        Series.repeat : Equivalent function for Series.
+
+        Examples
+        --------
+        >>> idx = ps.Index(['a', 'b', 'c'])
+        >>> idx
+        Index(['a', 'b', 'c'], dtype='object')
+        >>> idx.repeat(2)
+        Index(['a', 'b', 'c', 'a', 'b', 'c'], dtype='object')
+
+        For MultiIndex,
+
+        >>> midx = ps.MultiIndex.from_tuples([('x', 'a'), ('x', 'b'), ('y', 'c')])
+        >>> midx  # doctest: +SKIP
+        MultiIndex([('x', 'a'),
+                    ('x', 'b'),
+                    ('y', 'c')],
+                   )
+        >>> midx.repeat(2)  # doctest: +SKIP
+        MultiIndex([('x', 'a'),
+                    ('x', 'b'),
+                    ('y', 'c'),
+                    ('x', 'a'),
+                    ('x', 'b'),
+                    ('y', 'c')],
+                   )
+        >>> midx.repeat(0)  # doctest: +SKIP
+        MultiIndex([], )
+        """
+        if not isinstance(repeats, int):
+            raise TypeError(
+                "`repeats` argument must be integer, but got {}".format(type(repeats).__name__)
+            )
+        elif repeats < 0:
+            raise ValueError("negative dimensions are not allowed")
+
+        kdf = DataFrame(self._internal.resolved_copy)  # type: DataFrame
+        if repeats == 0:
+            return DataFrame(kdf._internal.with_filter(F.lit(False))).index
+        else:
+            return ps.concat([kdf] * repeats).index
+
+    def asof(self, label) -> Scalar:
+        """
+        Return the label from the index, or, if not present, the previous one.
+
+        Assuming that the index is sorted, return the passed index label if it
+        is in the index, or return the previous index label if the passed one
+        is not in the index.
+
+        .. note:: This API is dependent on :meth:`Index.is_monotonic_increasing`
+            which can be expensive.
+
+        Parameters
+        ----------
+        label : object
+            The label up to which the method returns the latest index label.
+
+        Returns
+        -------
+        object
+            The passed label if it is in the index. The previous label if the
+            passed label is not in the sorted index or `NaN` if there is no
+            such label.
+
+        Examples
+        --------
+        `Index.asof` returns the latest index label up to the passed label.
+
+        >>> idx = ps.Index(['2013-12-31', '2014-01-02', '2014-01-03'])
+        >>> idx.asof('2014-01-01')
+        '2013-12-31'
+
+        If the label is in the index, the method returns the passed label.
+
+        >>> idx.asof('2014-01-02')
+        '2014-01-02'
+
+        If all of the labels in the index are later than the passed label,
+        NaN is returned.
+
+        >>> idx.asof('1999-01-02')
+        nan
+        """
+        sdf = self._internal.spark_frame
+        if self.is_monotonic_increasing:
+            sdf = sdf.where(self.spark.column <= F.lit(label).cast(self.spark.data_type)).select(
+                F.max(self.spark.column)
+            )
+        elif self.is_monotonic_decreasing:
+            sdf = sdf.where(self.spark.column >= F.lit(label).cast(self.spark.data_type)).select(
+                F.min(self.spark.column)
+            )
+        else:
+            raise ValueError("index must be monotonic increasing or decreasing")
+
+        result = sdf.toPandas().iloc[0, 0]
+        return result if result is not None else np.nan
+
+    def union(self, other, sort=None) -> "Index":
+        """
+        Form the union of two Index objects.
+
+        Parameters
+        ----------
+        other : Index or array-like
+        sort : bool or None, default None
+            Whether to sort the resulting Index.
+
+        Returns
+        -------
+        union : Index
+
+        Examples
+        --------
+
+        Index
+
+        >>> idx1 = ps.Index([1, 2, 3, 4])
+        >>> idx2 = ps.Index([3, 4, 5, 6])
+        >>> idx1.union(idx2).sort_values()
+        Int64Index([1, 2, 3, 4, 5, 6], dtype='int64')
+
+        MultiIndex
+
+        >>> midx1 = ps.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("x", "c"), ("x", "d")])
+        >>> midx2 = ps.MultiIndex.from_tuples([("x", "c"), ("x", "d"), ("x", "e"), ("x", "f")])
+        >>> midx1.union(midx2).sort_values()  # doctest: +SKIP
+        MultiIndex([('x', 'a'),
+                    ('x', 'b'),
+                    ('x', 'c'),
+                    ('x', 'd'),
+                    ('x', 'e'),
+                    ('x', 'f')],
+                   )
+        """
+        from pyspark.pandas.indexes.multi import MultiIndex
+
+        sort = True if sort is None else sort
+        sort = validate_bool_kwarg(sort, "sort")
+        if type(self) is not type(other):
+            if isinstance(self, MultiIndex):
+                if not isinstance(other, list) or not all(
+                    [isinstance(item, tuple) for item in other]
+                ):
+                    raise TypeError("other must be a MultiIndex or a list of tuples")
+                other = MultiIndex.from_tuples(other)
+            else:
+                if isinstance(other, MultiIndex):
+                    # TODO: We can't support different type of values in a single column for now.
+                    raise NotImplementedError(
+                        "Union between Index and MultiIndex is not yet supported"
+                    )
+                elif isinstance(other, Series):
+                    other = other.to_frame()
+                    other = other.set_index(other.columns[0]).index
+                elif isinstance(other, DataFrame):
+                    raise ValueError("Index data must be 1-dimensional")
+                else:
+                    other = Index(other)
+        sdf_self = self._internal.spark_frame.select(self._internal.index_spark_columns)
+        sdf_other = other._internal.spark_frame.select(other._internal.index_spark_columns)
+        sdf = sdf_self.union(sdf_other.subtract(sdf_self))
+        if isinstance(self, MultiIndex):
+            sdf = sdf.drop_duplicates()
+        if sort:
+            sdf = sdf.sort(self._internal.index_spark_column_names)
+        internal = InternalFrame(  # TODO: dtypes?
+            spark_frame=sdf,
+            index_spark_columns=[
+                scol_for(sdf, col) for col in self._internal.index_spark_column_names
+            ],
+            index_names=self._internal.index_names,
+        )
+
+        return DataFrame(internal).index
+
+    def holds_integer(self) -> bool:
+        """
+        Whether the type is an integer type.
+        Always return False for MultiIndex.
+
+        Notes
+        -----
+        When Index contains null values the result can be different with pandas
+        since pandas-on-Spark cast integer to float when Index contains null values.
+
+        >>> ps.Index([1, 2, 3, None])
+        Float64Index([1.0, 2.0, 3.0, nan], dtype='float64')
+
+        Examples
+        --------
+        >>> kidx = ps.Index([1, 2, 3, 4])
+        >>> kidx.holds_integer()
+        True
+
+        Returns False for string type.
+
+        >>> kidx = ps.Index(["A", "B", "C", "D"])
+        >>> kidx.holds_integer()
+        False
+
+        Returns False for float type.
+
+        >>> kidx = ps.Index([1.1, 2.2, 3.3, 4.4])
+        >>> kidx.holds_integer()
+        False
+        """
+        return isinstance(self.spark.data_type, IntegralType)
+
+    def intersection(self, other) -> "Index":
+        """
+        Form the intersection of two Index objects.
+
+        This returns a new Index with elements common to the index and `other`.
+
+        Parameters
+        ----------
+        other : Index or array-like
+
+        Returns
+        -------
+        intersection : Index
+
+        Examples
+        --------
+        >>> idx1 = ps.Index([1, 2, 3, 4])
+        >>> idx2 = ps.Index([3, 4, 5, 6])
+        >>> idx1.intersection(idx2).sort_values()
+        Int64Index([3, 4], dtype='int64')
+        """
+        from pyspark.pandas.indexes.multi import MultiIndex
+
+        if isinstance(other, DataFrame):
+            raise ValueError("Index data must be 1-dimensional")
+        elif isinstance(other, MultiIndex):
+            # Always returns a no-named empty Index if `other` is MultiIndex.
+            return self._kdf.head(0).index.rename(None)
+        elif isinstance(other, Index):
+            spark_frame_other = other.to_frame().to_spark()
+            keep_name = self.name == other.name
+        elif isinstance(other, Series):
+            spark_frame_other = other.to_frame().to_spark()
+            keep_name = True
+        elif is_list_like(other):
+            other = Index(other)
+            if isinstance(other, MultiIndex):
+                return other.to_frame().head(0).index
+            spark_frame_other = other.to_frame().to_spark()
+            keep_name = True
+        else:
+            raise TypeError("Input must be Index or array-like")
+
+        spark_frame_self = self.to_frame(name=SPARK_DEFAULT_INDEX_NAME).to_spark()
+        spark_frame_intersected = spark_frame_self.intersect(spark_frame_other)
+        if keep_name:
+            index_names = self._internal.index_names
+        else:
+            index_names = None
+        internal = InternalFrame(  # TODO: dtypes?
+            spark_frame=spark_frame_intersected,
+            index_spark_columns=[scol_for(spark_frame_intersected, SPARK_DEFAULT_INDEX_NAME)],
+            index_names=index_names,
+        )
+
+        return DataFrame(internal).index
+
+    def item(self) -> Union[Scalar, Tuple[Scalar, ...]]:
+        """
+        Return the first element of the underlying data as a python scalar.
+
+        Returns
+        -------
+        scalar
+            The first element of Index.
+
+        Raises
+        ------
+        ValueError
+            If the data is not length-1.
+
+        Examples
+        --------
+        >>> kidx = ps.Index([10])
+        >>> kidx.item()
+        10
+        """
+        return self.to_series().item()
+
+    def insert(self, loc: int, item) -> "Index":
+        """
+        Make new Index inserting new item at location.
+
+        Follows Python list.append semantics for negative values.
+
+        Parameters
+        ----------
+        loc : int
+        item : object
+
+        Returns
+        -------
+        new_index : Index
+
+        Examples
+        --------
+        >>> kidx = ps.Index([1, 2, 3, 4, 5])
+        >>> kidx.insert(3, 100)
+        Int64Index([1, 2, 3, 100, 4, 5], dtype='int64')
+
+        For negative values
+
+        >>> kidx = ps.Index([1, 2, 3, 4, 5])
+        >>> kidx.insert(-3, 100)
+        Int64Index([1, 2, 100, 3, 4, 5], dtype='int64')
+        """
+        if loc < 0:
+            length = len(self)
+            loc = loc + length
+            loc = 0 if loc < 0 else loc
+
+        index_name = self._internal.index_spark_column_names[0]
+        sdf_before = self.to_frame(name=index_name)[:loc].to_spark()
+        sdf_middle = Index([item]).to_frame(name=index_name).to_spark()
+        sdf_after = self.to_frame(name=index_name)[loc:].to_spark()
+        sdf = sdf_before.union(sdf_middle).union(sdf_after)
+
+        internal = self._internal.with_new_sdf(sdf)  # TODO: dtype?
+        return DataFrame(internal).index
+
+    def view(self) -> "Index":
+        """
+        this is defined as a copy with the same identity
+        """
+        return self.copy()
+
+    def to_list(self) -> List:
+        """
+        Return a list of the values.
+
+        These are each a scalar type, which is a Python scalar
+        (for str, int, float) or a pandas scalar
+        (for Timestamp/Timedelta/Interval/Period)
+
+        .. note:: This method should only be used if the resulting list is expected
+            to be small, as all the data is loaded into the driver's memory.
+
+        Examples
+        --------
+        Index
+
+        >>> idx = ps.Index([1, 2, 3, 4, 5])
+        >>> idx.to_list()
+        [1, 2, 3, 4, 5]
+
+        MultiIndex
+
+        >>> tuples = [(1, 'red'), (1, 'blue'), (2, 'red'), (2, 'green')]
+        >>> midx = ps.MultiIndex.from_tuples(tuples)
+        >>> midx.to_list()
+        [(1, 'red'), (1, 'blue'), (2, 'red'), (2, 'green')]
+        """
+        return self._to_internal_pandas().tolist()
+
+    tolist = to_list
+
+    @property
+    def inferred_type(self) -> str:
+        """
+        Return a string of the type inferred from the values.
+
+        Examples
+        --------
+        >>> from datetime import datetime
+        >>> ps.Index([1, 2, 3]).inferred_type
+        'integer'
+
+        >>> ps.Index([1.0, 2.0, 3.0]).inferred_type
+        'floating'
+
+        >>> ps.Index(['a', 'b', 'c']).inferred_type
+        'string'
+
+        >>> ps.Index([True, False, True, False]).inferred_type
+        'boolean'
+        """
+        return lib.infer_dtype([self.to_series().head(1).item()])
+
+    def __getattr__(self, item: str) -> Any:
+        if hasattr(MissingPandasLikeIndex, item):
+            property_or_func = getattr(MissingPandasLikeIndex, item)
+            if isinstance(property_or_func, property):
+                return property_or_func.fget(self)  # type: ignore
+            else:
+                return partial(property_or_func, self)
+        raise AttributeError("'{}' object has no attribute '{}'".format(type(self).__name__, item))
+
+    def __repr__(self):
+        max_display_count = get_option("display.max_rows")
+        if max_display_count is None:
+            return repr(self._to_internal_pandas())
+
+        pindex = self._kdf._get_or_create_repr_pandas_cache(max_display_count).index
+
+        pindex_length = len(pindex)
+        repr_string = repr(pindex[:max_display_count])
+
+        if pindex_length > max_display_count:
+            footer = "\nShowing only the first {}".format(max_display_count)
+            return repr_string + footer
+        return repr_string
+
+    def __iter__(self):
+        return MissingPandasLikeIndex.__iter__(self)
+
+    def __xor__(self, other):
+        return self.symmetric_difference(other)
+
+    def __bool__(self):
+        raise ValueError(
+            "The truth value of a {0} is ambiguous. "
+            "Use a.empty, a.bool(), a.item(), a.any() or a.all().".format(self.__class__.__name__)
+        )
+
+
+def _test():
+    import os
+    import doctest
+    import sys
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.indexes.base
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.indexes.base.__dict__.copy()
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]")
+        .appName("pyspark.pandas.indexes.base tests")
+        .getOrCreate()
+    )
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.indexes.base,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/indexes/category.py b/python/pyspark/pandas/indexes/category.py
new file mode 100644
index 0000000000000..5a63bc8ae72b5
--- /dev/null
+++ b/python/pyspark/pandas/indexes/category.py
@@ -0,0 +1,218 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from functools import partial
+from typing import Any
+
+import pandas as pd
+from pandas.api.types import is_hashable
+
+from pyspark import pandas as ps
+from pyspark.pandas.indexes.base import Index
+from pyspark.pandas.missing.indexes import MissingPandasLikeCategoricalIndex
+from pyspark.pandas.series import Series
+
+
+class CategoricalIndex(Index):
+    """
+    Index based on an underlying `Categorical`.
+
+    CategoricalIndex can only take on a limited,
+    and usually fixed, number of possible values (`categories`). Also,
+    it might have an order, but numerical operations
+    (additions, divisions, ...) are not possible.
+
+    Parameters
+    ----------
+    data : array-like (1-dimensional)
+        The values of the categorical. If `categories` are given, values not in
+        `categories` will be replaced with NaN.
+    categories : index-like, optional
+        The categories for the categorical. Items need to be unique.
+        If the categories are not given here (and also not in `dtype`), they
+        will be inferred from the `data`.
+    ordered : bool, optional
+        Whether or not this categorical is treated as an ordered
+        categorical. If not given here or in `dtype`, the resulting
+        categorical will be unordered.
+    dtype : CategoricalDtype or "category", optional
+        If :class:`CategoricalDtype`, cannot be used together with
+        `categories` or `ordered`.
+    copy : bool, default False
+        Make a copy of input ndarray.
+    name : object, optional
+        Name to be stored in the index.
+
+    See Also
+    --------
+    Index : The base pandas-on-Spark Index type.
+
+    Examples
+    --------
+    >>> ps.CategoricalIndex(["a", "b", "c", "a", "b", "c"])  # doctest: +NORMALIZE_WHITESPACE
+    CategoricalIndex(['a', 'b', 'c', 'a', 'b', 'c'],
+                     categories=['a', 'b', 'c'], ordered=False, dtype='category')
+
+    ``CategoricalIndex`` can also be instantiated from a ``Categorical``:
+
+    >>> c = pd.Categorical(["a", "b", "c", "a", "b", "c"])
+    >>> ps.CategoricalIndex(c)  # doctest: +NORMALIZE_WHITESPACE
+    CategoricalIndex(['a', 'b', 'c', 'a', 'b', 'c'],
+                     categories=['a', 'b', 'c'], ordered=False, dtype='category')
+
+    Ordered ``CategoricalIndex`` can have a min and max value.
+
+    >>> ci = ps.CategoricalIndex(
+    ...     ["a", "b", "c", "a", "b", "c"], ordered=True, categories=["c", "b", "a"]
+    ... )
+    >>> ci  # doctest: +NORMALIZE_WHITESPACE
+    CategoricalIndex(['a', 'b', 'c', 'a', 'b', 'c'],
+                     categories=['c', 'b', 'a'], ordered=True, dtype='category')
+
+    From a Series:
+
+    >>> s = ps.Series(["a", "b", "c", "a", "b", "c"], index=[10, 20, 30, 40, 50, 60])
+    >>> ps.CategoricalIndex(s)  # doctest: +NORMALIZE_WHITESPACE
+    CategoricalIndex(['a', 'b', 'c', 'a', 'b', 'c'],
+                     categories=['a', 'b', 'c'], ordered=False, dtype='category')
+
+    From an Index:
+
+    >>> idx = ps.Index(["a", "b", "c", "a", "b", "c"])
+    >>> ps.CategoricalIndex(idx)  # doctest: +NORMALIZE_WHITESPACE
+    CategoricalIndex(['a', 'b', 'c', 'a', 'b', 'c'],
+                     categories=['a', 'b', 'c'], ordered=False, dtype='category')
+    """
+
+    def __new__(cls, data=None, categories=None, ordered=None, dtype=None, copy=False, name=None):
+        if not is_hashable(name):
+            raise TypeError("Index.name must be a hashable type")
+
+        if isinstance(data, (Series, Index)):
+            if dtype is None:
+                dtype = "category"
+            return Index(data, dtype=dtype, copy=copy, name=name)
+
+        return ps.from_pandas(
+            pd.CategoricalIndex(
+                data=data, categories=categories, ordered=ordered, dtype=dtype, name=name
+            )
+        )
+
+    @property
+    def codes(self) -> Index:
+        """
+        The category codes of this categorical.
+
+        Codes are an Index of integers which are the positions of the actual
+        values in the categories Index.
+
+        There is no setter, use the other categorical methods and the normal item
+        setter to change values in the categorical.
+
+        Returns
+        -------
+        Index
+            A non-writable view of the `codes` Index.
+
+        Examples
+        --------
+        >>> idx = ps.CategoricalIndex(list("abbccc"))
+        >>> idx  # doctest: +NORMALIZE_WHITESPACE
+        CategoricalIndex(['a', 'b', 'b', 'c', 'c', 'c'],
+                         categories=['a', 'b', 'c'], ordered=False, dtype='category')
+
+        >>> idx.codes
+        Int64Index([0, 1, 1, 2, 2, 2], dtype='int64')
+        """
+        return self._with_new_scol(self.spark.column).rename(None)
+
+    @property
+    def categories(self) -> pd.Index:
+        """
+        The categories of this categorical.
+
+        Examples
+        --------
+        >>> idx = ps.CategoricalIndex(list("abbccc"))
+        >>> idx  # doctest: +NORMALIZE_WHITESPACE
+        CategoricalIndex(['a', 'b', 'b', 'c', 'c', 'c'],
+                         categories=['a', 'b', 'c'], ordered=False, dtype='category')
+
+        >>> idx.categories
+        Index(['a', 'b', 'c'], dtype='object')
+        """
+        return self.dtype.categories
+
+    @categories.setter
+    def categories(self, categories):
+        raise NotImplementedError()
+
+    @property
+    def ordered(self) -> bool:
+        """
+        Whether the categories have an ordered relationship.
+
+        Examples
+        --------
+        >>> idx = ps.CategoricalIndex(list("abbccc"))
+        >>> idx  # doctest: +NORMALIZE_WHITESPACE
+        CategoricalIndex(['a', 'b', 'b', 'c', 'c', 'c'],
+                         categories=['a', 'b', 'c'], ordered=False, dtype='category')
+
+        >>> idx.ordered
+        False
+        """
+        return self.dtype.ordered
+
+    def __getattr__(self, item: str) -> Any:
+        if hasattr(MissingPandasLikeCategoricalIndex, item):
+            property_or_func = getattr(MissingPandasLikeCategoricalIndex, item)
+            if isinstance(property_or_func, property):
+                return property_or_func.fget(self)  # type: ignore
+            else:
+                return partial(property_or_func, self)
+        raise AttributeError("'CategoricalIndex' object has no attribute '{}'".format(item))
+
+
+def _test():
+    import os
+    import doctest
+    import sys
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.indexes.category
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.indexes.category.__dict__.copy()
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]")
+        .appName("pyspark.pandas.indexes.category tests")
+        .getOrCreate()
+    )
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.indexes.category,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/indexes/datetimes.py b/python/pyspark/pandas/indexes/datetimes.py
new file mode 100644
index 0000000000000..6236f69352dfb
--- /dev/null
+++ b/python/pyspark/pandas/indexes/datetimes.py
@@ -0,0 +1,772 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import datetime
+from functools import partial
+from typing import Any, Optional, Union
+
+import pandas as pd
+from pandas.api.types import is_hashable
+from pyspark._globals import _NoValue
+
+from pyspark import pandas as ps
+from pyspark.pandas.indexes.base import Index
+from pyspark.pandas.missing.indexes import MissingPandasLikeDatetimeIndex
+from pyspark.pandas.series import Series, first_series
+from pyspark.pandas.utils import verify_temp_column_name
+
+
+class DatetimeIndex(Index):
+    """
+    Immutable ndarray-like of datetime64 data.
+
+    Parameters
+    ----------
+    data : array-like (1-dimensional), optional
+        Optional datetime-like data to construct index with.
+    freq : str or pandas offset object, optional
+        One of pandas date offset strings or corresponding objects. The string
+        'infer' can be passed in order to set the frequency of the index as the
+        inferred frequency upon creation.
+    normalize : bool, default False
+        Normalize start/end dates to midnight before generating date range.
+    closed : {'left', 'right'}, optional
+        Set whether to include `start` and `end` that are on the
+        boundary. The default includes boundary points on either end.
+    ambiguous : 'infer', bool-ndarray, 'NaT', default 'raise'
+        When clocks moved backward due to DST, ambiguous times may arise.
+        For example in Central European Time (UTC+01), when going from 03:00
+        DST to 02:00 non-DST, 02:30:00 local time occurs both at 00:30:00 UTC
+        and at 01:30:00 UTC. In such a situation, the `ambiguous` parameter
+        dictates how ambiguous times should be handled.
+
+        - 'infer' will attempt to infer fall dst-transition hours based on
+          order
+        - bool-ndarray where True signifies a DST time, False signifies a
+          non-DST time (note that this flag is only applicable for ambiguous
+          times)
+        - 'NaT' will return NaT where there are ambiguous times
+        - 'raise' will raise an AmbiguousTimeError if there are ambiguous times.
+    dayfirst : bool, default False
+        If True, parse dates in `data` with the day first order.
+    yearfirst : bool, default False
+        If True parse dates in `data` with the year first order.
+    dtype : numpy.dtype or str, default None
+        Note that the only NumPy dtype allowed is ‘datetime64[ns]’.
+    copy : bool, default False
+        Make a copy of input ndarray.
+    name : label, default None
+        Name to be stored in the index.
+
+    See Also
+    --------
+    Index : The base pandas Index type.
+    to_datetime : Convert argument to datetime.
+
+    Examples
+    --------
+    >>> ps.DatetimeIndex(['1970-01-01', '1970-01-01', '1970-01-01'])
+    DatetimeIndex(['1970-01-01', '1970-01-01', '1970-01-01'], dtype='datetime64[ns]', freq=None)
+
+    From a Series:
+
+    >>> from datetime import datetime
+    >>> s = ps.Series([datetime(2021, 3, 1), datetime(2021, 3, 2)], index=[10, 20])
+    >>> ps.DatetimeIndex(s)
+    DatetimeIndex(['2021-03-01', '2021-03-02'], dtype='datetime64[ns]', freq=None)
+
+    From an Index:
+
+    >>> idx = ps.DatetimeIndex(['1970-01-01', '1970-01-01', '1970-01-01'])
+    >>> ps.DatetimeIndex(idx)
+    DatetimeIndex(['1970-01-01', '1970-01-01', '1970-01-01'], dtype='datetime64[ns]', freq=None)
+    """
+
+    def __new__(
+        cls,
+        data=None,
+        freq=_NoValue,
+        normalize=False,
+        closed=None,
+        ambiguous="raise",
+        dayfirst=False,
+        yearfirst=False,
+        dtype=None,
+        copy=False,
+        name=None,
+    ):
+        if not is_hashable(name):
+            raise TypeError("Index.name must be a hashable type")
+
+        if isinstance(data, (Series, Index)):
+            if dtype is None:
+                dtype = "datetime64[ns]"
+            return Index(data, dtype=dtype, copy=copy, name=name)
+
+        kwargs = dict(
+            data=data,
+            normalize=normalize,
+            closed=closed,
+            ambiguous=ambiguous,
+            dayfirst=dayfirst,
+            yearfirst=yearfirst,
+            dtype=dtype,
+            copy=copy,
+            name=name,
+        )
+        if freq is not _NoValue:
+            kwargs["freq"] = freq
+        return ps.from_pandas(pd.DatetimeIndex(**kwargs))
+
+    def __getattr__(self, item: str) -> Any:
+        if hasattr(MissingPandasLikeDatetimeIndex, item):
+            property_or_func = getattr(MissingPandasLikeDatetimeIndex, item)
+            if isinstance(property_or_func, property):
+                return property_or_func.fget(self)  # type: ignore
+            else:
+                return partial(property_or_func, self)
+        raise AttributeError("'DatetimeIndex' object has no attribute '{}'".format(item))
+
+    # Properties
+    @property
+    def year(self) -> Index:
+        """
+        The year of the datetime.
+        """
+        return Index(self.to_series().dt.year)
+
+    @property
+    def month(self) -> Index:
+        """
+        The month of the timestamp as January = 1 December = 12.
+        """
+        return Index(self.to_series().dt.month)
+
+    @property
+    def day(self) -> Index:
+        """
+        The days of the datetime.
+        """
+        return Index(self.to_series().dt.day)
+
+    @property
+    def hour(self) -> Index:
+        """
+        The hours of the datetime.
+        """
+        return Index(self.to_series().dt.hour)
+
+    @property
+    def minute(self) -> Index:
+        """
+        The minutes of the datetime.
+        """
+        return Index(self.to_series().dt.minute)
+
+    @property
+    def second(self) -> Index:
+        """
+        The seconds of the datetime.
+        """
+        return Index(self.to_series().dt.second)
+
+    @property
+    def microsecond(self) -> Index:
+        """
+        The microseconds of the datetime.
+        """
+        return Index(self.to_series().dt.microsecond)
+
+    @property
+    def week(self) -> Index:
+        """
+        The week ordinal of the year.
+        """
+        return Index(self.to_series().dt.week)
+
+    @property
+    def weekofyear(self) -> Index:
+        return Index(self.to_series().dt.weekofyear)
+
+    weekofyear.__doc__ = week.__doc__
+
+    @property
+    def dayofweek(self) -> Index:
+        """
+        The day of the week with Monday=0, Sunday=6.
+        Return the day of the week. It is assumed the week starts on
+        Monday, which is denoted by 0 and ends on Sunday which is denoted
+        by 6. This method is available on both Series with datetime
+        values (using the `dt` accessor) or DatetimeIndex.
+
+        Returns
+        -------
+        Series or Index
+            Containing integers indicating the day number.
+
+        See Also
+        --------
+        Series.dt.dayofweek : Alias.
+        Series.dt.weekday : Alias.
+        Series.dt.day_name : Returns the name of the day of the week.
+
+        Examples
+        --------
+        >>> idx = ps.date_range('2016-12-31', '2017-01-08', freq='D')
+        >>> idx.dayofweek
+        Int64Index([5, 6, 0, 1, 2, 3, 4, 5, 6], dtype='int64')
+        """
+        return Index(self.to_series().dt.dayofweek)
+
+    @property
+    def day_of_week(self) -> Index:
+        return self.dayofweek
+
+    day_of_week.__doc__ = dayofweek.__doc__
+
+    @property
+    def weekday(self) -> Index:
+        return Index(self.to_series().dt.weekday)
+
+    weekday.__doc__ = dayofweek.__doc__
+
+    @property
+    def dayofyear(self) -> Index:
+        """
+        The ordinal day of the year.
+        """
+        return Index(self.to_series().dt.dayofyear)
+
+    @property
+    def day_of_year(self) -> Index:
+        return self.dayofyear
+
+    day_of_year.__doc__ = dayofyear.__doc__
+
+    @property
+    def quarter(self) -> Index:
+        """
+        The quarter of the date.
+        """
+        return Index(self.to_series().dt.quarter)
+
+    @property
+    def is_month_start(self) -> Index:
+        """
+        Indicates whether the date is the first day of the month.
+
+        Returns
+        -------
+        Index
+            Returns a Index with boolean values
+
+        See Also
+        --------
+        is_month_end : Return a boolean indicating whether the date
+            is the last day of the month.
+
+        Examples
+        --------
+        >>> idx = ps.date_range("2018-02-27", periods=3)
+        >>> idx.is_month_start
+        Index([False, False, True], dtype='object')
+        """
+        return Index(self.to_series().dt.is_month_start)
+
+    @property
+    def is_month_end(self) -> Index:
+        """
+        Indicates whether the date is the last day of the month.
+
+        Returns
+        -------
+        Index
+            Returns a Index with boolean values.
+
+        See Also
+        --------
+        is_month_start : Return a boolean indicating whether the date
+            is the first day of the month.
+
+        Examples
+        --------
+        >>> idx = ps.date_range("2018-02-27", periods=3)
+        >>> idx.is_month_end
+        Index([False, True, False], dtype='object')
+        """
+        return Index(self.to_series().dt.is_month_end)
+
+    @property
+    def is_quarter_start(self) -> Index:
+        """
+        Indicator for whether the date is the first day of a quarter.
+
+        Returns
+        -------
+        is_quarter_start : Index
+            Returns an Index with boolean values.
+
+        See Also
+        --------
+        quarter : Return the quarter of the date.
+        is_quarter_end : Similar property for indicating the quarter start.
+
+        Examples
+        --------
+        >>> idx = ps.date_range('2017-03-30', periods=4)
+        >>> idx.is_quarter_start
+        Index([False, False, True, False], dtype='object')
+        """
+        return Index(self.to_series().dt.is_quarter_start)
+
+    @property
+    def is_quarter_end(self) -> Index:
+        """
+        Indicator for whether the date is the last day of a quarter.
+
+        Returns
+        -------
+        is_quarter_end : Index
+            Returns an Index with boolean values.
+
+        See Also
+        --------
+        quarter : Return the quarter of the date.
+        is_quarter_start : Similar property indicating the quarter start.
+
+        Examples
+        --------
+        >>> idx = ps.date_range('2017-03-30', periods=4)
+        >>> idx.is_quarter_end
+        Index([False, True, False, False], dtype='object')
+        """
+        return Index(self.to_series().dt.is_quarter_end)
+
+    @property
+    def is_year_start(self) -> Index:
+        """
+        Indicate whether the date is the first day of a year.
+
+        Returns
+        -------
+        Index
+            Returns an Index with boolean values.
+
+        See Also
+        --------
+        is_year_end : Similar property indicating the last day of the year.
+
+        Examples
+        --------
+        >>> idx = ps.date_range("2017-12-30", periods=3)
+        >>> idx.is_year_start
+        Index([False, False, True], dtype='object')
+        """
+        return Index(self.to_series().dt.is_year_start)
+
+    @property
+    def is_year_end(self) -> Index:
+        """
+        Indicate whether the date is the last day of the year.
+
+        Returns
+        -------
+        Index
+            Returns an Index with boolean values.
+
+        See Also
+        --------
+        is_year_start : Similar property indicating the start of the year.
+
+        Examples
+        --------
+        >>> idx = ps.date_range("2017-12-30", periods=3)
+        >>> idx.is_year_end
+        Index([False, True, False], dtype='object')
+        """
+        return Index(self.to_series().dt.is_year_end)
+
+    @property
+    def is_leap_year(self) -> Index:
+        """
+        Boolean indicator if the date belongs to a leap year.
+
+        A leap year is a year, which has 366 days (instead of 365) including
+        29th of February as an intercalary day.
+        Leap years are years which are multiples of four with the exception
+        of years divisible by 100 but not by 400.
+
+        Returns
+        -------
+        Index
+             Booleans indicating if dates belong to a leap year.
+
+        Examples
+        --------
+        >>> idx = ps.date_range("2012-01-01", "2015-01-01", freq="Y")
+        >>> idx.is_leap_year
+        Index([True, False, False], dtype='object')
+        """
+        return Index(self.to_series().dt.is_leap_year)
+
+    @property
+    def daysinmonth(self) -> Index:
+        """
+        The number of days in the month.
+        """
+        return Index(self.to_series().dt.daysinmonth)
+
+    @property
+    def days_in_month(self) -> Index:
+        return Index(self.to_series().dt.days_in_month)
+
+    days_in_month.__doc__ = daysinmonth.__doc__
+
+    # Methods
+    def ceil(self, freq, *args, **kwargs) -> "DatetimeIndex":
+        """
+        Perform ceil operation on the data to the specified freq.
+
+        Parameters
+        ----------
+        freq : str or Offset
+            The frequency level to ceil the index to. Must be a fixed
+            frequency like 'S' (second) not 'ME' (month end).
+
+        Returns
+        -------
+        DatetimeIndex
+
+        Raises
+        ------
+        ValueError if the `freq` cannot be converted.
+
+        Examples
+        --------
+        >>> rng = ps.date_range('1/1/2018 11:59:00', periods=3, freq='min')
+        >>> rng.ceil('H')  # doctest: +NORMALIZE_WHITESPACE
+        DatetimeIndex(['2018-01-01 12:00:00', '2018-01-01 12:00:00',
+                       '2018-01-01 13:00:00'],
+                      dtype='datetime64[ns]', freq=None)
+        """
+        disallow_nanoseconds(freq)
+
+        return DatetimeIndex(self.to_series().dt.ceil(freq, *args, **kwargs))
+
+    def floor(self, freq, *args, **kwargs) -> "DatetimeIndex":
+        """
+        Perform floor operation on the data to the specified freq.
+
+        Parameters
+        ----------
+        freq : str or Offset
+            The frequency level to floor the index to. Must be a fixed
+            frequency like 'S' (second) not 'ME' (month end).
+
+        Returns
+        -------
+        DatetimeIndex
+
+        Raises
+        ------
+        ValueError if the `freq` cannot be converted.
+
+        Examples
+        --------
+        >>> rng = ps.date_range('1/1/2018 11:59:00', periods=3, freq='min')
+        >>> rng.floor("H")  # doctest: +NORMALIZE_WHITESPACE
+        DatetimeIndex(['2018-01-01 11:00:00', '2018-01-01 12:00:00',
+                       '2018-01-01 12:00:00'],
+                      dtype='datetime64[ns]', freq=None)
+        """
+        disallow_nanoseconds(freq)
+
+        return DatetimeIndex(self.to_series().dt.floor(freq, *args, **kwargs))
+
+    def round(self, freq, *args, **kwargs) -> "DatetimeIndex":
+        """
+        Perform round operation on the data to the specified freq.
+
+        Parameters
+        ----------
+        freq : str or Offset
+            The frequency level to round the index to. Must be a fixed
+            frequency like 'S' (second) not 'ME' (month end).
+
+        Returns
+        -------
+        DatetimeIndex
+
+        Raises
+        ------
+        ValueError if the `freq` cannot be converted.
+
+        Examples
+        --------
+        >>> rng = ps.date_range('1/1/2018 11:59:00', periods=3, freq='min')
+        >>> rng.round("H")  # doctest: +NORMALIZE_WHITESPACE
+        DatetimeIndex(['2018-01-01 12:00:00', '2018-01-01 12:00:00',
+                       '2018-01-01 12:00:00'],
+                      dtype='datetime64[ns]', freq=None)
+        """
+        disallow_nanoseconds(freq)
+
+        return DatetimeIndex(self.to_series().dt.round(freq, *args, **kwargs))
+
+    def month_name(self, locale: Optional[str] = None) -> Index:
+        """
+        Return the month names of the DatetimeIndex with specified locale.
+
+        Parameters
+        ----------
+        locale : str, optional
+            Locale determining the language in which to return the month name.
+            Default is English locale.
+
+        Returns
+        -------
+        Index
+            Index of month names.
+
+        Examples
+        --------
+        >>> idx = ps.date_range(start='2018-01', freq='M', periods=3)
+        >>> idx.month_name()
+        Index(['January', 'February', 'March'], dtype='object')
+        """
+        return Index(self.to_series().dt.month_name(locale))
+
+    def day_name(self, locale: Optional[str] = None) -> Index:
+        """
+        Return the day names of the series with specified locale.
+
+        Parameters
+        ----------
+        locale : str, optional
+            Locale determining the language in which to return the day name.
+            Default is English locale.
+
+        Returns
+        -------
+        Index
+            Index of day names.
+
+        Examples
+        --------
+        >>> idx = ps.date_range(start='2018-01-01', freq='D', periods=3)
+        >>> idx.day_name()
+        Index(['Monday', 'Tuesday', 'Wednesday'], dtype='object')
+        """
+        return Index(self.to_series().dt.day_name(locale))
+
+    def normalize(self) -> "DatetimeIndex":
+        """
+        Convert times to midnight.
+
+        The time component of the date-time is converted to midnight i.e.
+        00:00:00. This is useful in cases, when the time does not matter.
+        Length is unaltered. The timezones are unaffected.
+
+        This method is available on Series with datetime values under
+        the ``.dt`` accessor.
+
+        Returns
+        -------
+        DatetimeIndex
+            The same type as the original data.
+
+        See Also
+        --------
+        floor : Floor the series to the specified freq.
+        ceil : Ceil the series to the specified freq.
+        round : Round the series to the specified freq.
+
+        Examples
+        --------
+        >>> idx = ps.date_range(start='2014-08-01 10:00', freq='H', periods=3)
+        >>> idx.normalize()
+        DatetimeIndex(['2014-08-01', '2014-08-01', '2014-08-01'], dtype='datetime64[ns]', freq=None)
+        """
+        return DatetimeIndex(self.to_series().dt.normalize())
+
+    def strftime(self, date_format: str) -> Index:
+        """
+        Convert to a string Index using specified date_format.
+
+        Return an Index of formatted strings specified by date_format, which
+        supports the same string format as the python standard library. Details
+        of the string format can be found in python string format
+        doc.
+
+        Parameters
+        ----------
+        date_format : str
+            Date format string (example: "%%Y-%%m-%%d").
+
+        Returns
+        -------
+        Index
+            Index of formatted strings.
+
+        See Also
+        --------
+        normalize : Return series with times to midnight.
+        round : Round the series to the specified freq.
+        floor : Floor the series to the specified freq.
+
+        Examples
+        --------
+        >>> idx = ps.date_range(pd.Timestamp("2018-03-10 09:00"), periods=3, freq='s')
+        >>> idx.strftime('%B %d, %Y, %r')  # doctest: +NORMALIZE_WHITESPACE
+        Index(['March 10, 2018, 09:00:00 AM', 'March 10, 2018, 09:00:01 AM',
+               'March 10, 2018, 09:00:02 AM'],
+              dtype='object')
+        """
+        return Index(self.to_series().dt.strftime(date_format))
+
+    def indexer_between_time(
+        self,
+        start_time: Union[datetime.time, str],
+        end_time: Union[datetime.time, str],
+        include_start: bool = True,
+        include_end: bool = True,
+    ) -> Index:
+        """
+        Return index locations of values between particular times of day
+        (example: 9:00-9:30AM).
+
+        Parameters
+        ----------
+        start_time, end_time : datetime.time, str
+            Time passed either as object (datetime.time) or as string in
+            appropriate format ("%H:%M", "%H%M", "%I:%M%p", "%I%M%p",
+            "%H:%M:%S", "%H%M%S", "%I:%M:%S%p","%I%M%S%p").
+        include_start : bool, default True
+        include_end : bool, default True
+
+        Returns
+        -------
+        values_between_time : Index of integers
+
+        Examples
+        --------
+        >>> kidx = ps.date_range("2000-01-01", periods=3, freq="T")
+        >>> kidx  # doctest: +NORMALIZE_WHITESPACE
+        DatetimeIndex(['2000-01-01 00:00:00', '2000-01-01 00:01:00',
+                       '2000-01-01 00:02:00'],
+                      dtype='datetime64[ns]', freq=None)
+
+        >>> kidx.indexer_between_time("00:01", "00:02").sort_values()
+        Int64Index([1, 2], dtype='int64')
+
+        >>> kidx.indexer_between_time("00:01", "00:02", include_end=False)
+        Int64Index([1], dtype='int64')
+
+        >>> kidx.indexer_between_time("00:01", "00:02", include_start=False)
+        Int64Index([2], dtype='int64')
+        """
+
+        def pandas_between_time(pdf) -> ps.DataFrame[int]:
+            return pdf.between_time(start_time, end_time, include_start, include_end)
+
+        kdf = self.to_frame()[[]]
+        id_column_name = verify_temp_column_name(kdf, "__id_column__")
+        kdf = kdf.koalas.attach_id_column("distributed-sequence", id_column_name)
+        with ps.option_context("compute.default_index_type", "distributed"):
+            # The attached index in the statement below will be dropped soon,
+            # so we enforce “distributed” default index type
+            kdf = kdf.koalas.apply_batch(pandas_between_time)
+        return ps.Index(first_series(kdf).rename(self.name))
+
+    def indexer_at_time(self, time: Union[datetime.time, str], asof: bool = False) -> Index:
+        """
+        Return index locations of values at particular time of day
+        (example: 9:30AM).
+
+        Parameters
+        ----------
+        time : datetime.time or str
+            Time passed in either as object (datetime.time) or as string in
+            appropriate format ("%H:%M", "%H%M", "%I:%M%p", "%I%M%p",
+            "%H:%M:%S", "%H%M%S", "%I:%M:%S%p", "%I%M%S%p").
+
+        Returns
+        -------
+        values_at_time : Index of integers
+
+        Examples
+        --------
+        >>> kidx = ps.date_range("2000-01-01", periods=3, freq="T")
+        >>> kidx  # doctest: +NORMALIZE_WHITESPACE
+        DatetimeIndex(['2000-01-01 00:00:00', '2000-01-01 00:01:00',
+                       '2000-01-01 00:02:00'],
+                      dtype='datetime64[ns]', freq=None)
+
+        >>> kidx.indexer_at_time("00:00")
+        Int64Index([0], dtype='int64')
+
+        >>> kidx.indexer_at_time("00:01")
+        Int64Index([1], dtype='int64')
+        """
+        if asof:
+            raise NotImplementedError("'asof' argument is not supported")
+
+        def pandas_at_time(pdf) -> ps.DataFrame[int]:
+            return pdf.at_time(time, asof)
+
+        kdf = self.to_frame()[[]]
+        id_column_name = verify_temp_column_name(kdf, "__id_column__")
+        kdf = kdf.koalas.attach_id_column("distributed-sequence", id_column_name)
+        with ps.option_context("compute.default_index_type", "distributed"):
+            # The attached index in the statement below will be dropped soon,
+            # so we enforce “distributed” default index type
+            kdf = kdf.koalas.apply_batch(pandas_at_time)
+        return ps.Index(first_series(kdf).rename(self.name))
+
+
+def disallow_nanoseconds(freq):
+    if freq in ["N", "ns"]:
+        raise ValueError("nanoseconds is not supported")
+
+
+def _test():
+    import os
+    import doctest
+    import sys
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.indexes.datetimes
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.indexes.datetimes.__dict__.copy()
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]")
+        .appName("pyspark.pandas.indexes.datetimes tests")
+        .getOrCreate()
+    )
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.indexes.datetimes,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/indexes/multi.py b/python/pyspark/pandas/indexes/multi.py
new file mode 100644
index 0000000000000..04717982e6288
--- /dev/null
+++ b/python/pyspark/pandas/indexes/multi.py
@@ -0,0 +1,1180 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from distutils.version import LooseVersion
+from functools import partial
+from typing import Any, Optional, Tuple, Union, cast
+import warnings
+
+import pandas as pd
+from pandas.api.types import is_list_like
+from pandas.api.types import is_hashable
+
+from pyspark import sql as spark
+from pyspark.sql import functions as F, Window
+
+# For running doctests and reference resolution in PyCharm.
+from pyspark import pandas as ps  # noqa: F401
+from pyspark.pandas.exceptions import PandasNotImplementedError
+from pyspark.pandas.frame import DataFrame
+from pyspark.pandas.indexes.base import Index
+from pyspark.pandas.missing.indexes import MissingPandasLikeMultiIndex
+from pyspark.pandas.series import Series, first_series
+from pyspark.pandas.utils import (
+    compare_disallow_null,
+    is_name_like_tuple,
+    name_like_string,
+    scol_for,
+    verify_temp_column_name,
+)
+from pyspark.pandas.internal import (
+    InternalFrame,
+    NATURAL_ORDER_COLUMN_NAME,
+    SPARK_INDEX_NAME_FORMAT,
+)
+from pyspark.pandas.typedef import Scalar
+
+
+class MultiIndex(Index):
+    """
+    pandas-on-Spark MultiIndex that corresponds to pandas MultiIndex logically. This might hold
+    Spark Column internally.
+
+    Parameters
+    ----------
+    levels : sequence of arrays
+        The unique labels for each level.
+    codes : sequence of arrays
+        Integers for each level designating which label at each location.
+    sortorder : optional int
+        Level of sortedness (must be lexicographically sorted by that
+        level).
+    names : optional sequence of objects
+        Names for each of the index levels. (name is accepted for compat).
+    copy : bool, default False
+        Copy the meta-data.
+    verify_integrity : bool, default True
+        Check that the levels/codes are consistent and valid.
+
+    See Also
+    --------
+    MultiIndex.from_arrays  : Convert list of arrays to MultiIndex.
+    MultiIndex.from_product : Create a MultiIndex from the cartesian product
+                              of iterables.
+    MultiIndex.from_tuples  : Convert list of tuples to a MultiIndex.
+    MultiIndex.from_frame   : Make a MultiIndex from a DataFrame.
+    Index : A single-level Index.
+
+    Examples
+    --------
+    >>> ps.DataFrame({'a': ['a', 'b', 'c']}, index=[[1, 2, 3], [4, 5, 6]]).index  # doctest: +SKIP
+    MultiIndex([(1, 4),
+                (2, 5),
+                (3, 6)],
+               )
+
+    >>> ps.DataFrame({'a': [1, 2, 3]}, index=[list('abc'), list('def')]).index  # doctest: +SKIP
+    MultiIndex([('a', 'd'),
+                ('b', 'e'),
+                ('c', 'f')],
+               )
+    """
+
+    def __new__(
+        cls,
+        levels=None,
+        codes=None,
+        sortorder=None,
+        names=None,
+        dtype=None,
+        copy=False,
+        name=None,
+        verify_integrity: bool = True,
+    ):
+        if LooseVersion(pd.__version__) < LooseVersion("0.24"):
+            if levels is None or codes is None:
+                raise TypeError("Must pass both levels and codes")
+
+            pidx = pd.MultiIndex(
+                levels=levels,
+                labels=codes,
+                sortorder=sortorder,
+                names=names,
+                dtype=dtype,
+                copy=copy,
+                name=name,
+                verify_integrity=verify_integrity,
+            )
+        else:
+            pidx = pd.MultiIndex(
+                levels=levels,
+                codes=codes,
+                sortorder=sortorder,
+                names=names,
+                dtype=dtype,
+                copy=copy,
+                name=name,
+                verify_integrity=verify_integrity,
+            )
+        return ps.from_pandas(pidx)
+
+    @property
+    def _internal(self):
+        internal = self._kdf._internal
+        scol = F.struct(internal.index_spark_columns)
+        return internal.copy(
+            column_labels=[None],
+            data_spark_columns=[scol],
+            data_dtypes=[None],
+            column_label_names=None,
+        )
+
+    @property
+    def _column_label(self):
+        return None
+
+    def __abs__(self):
+        raise TypeError("TypeError: cannot perform __abs__ with this index type: MultiIndex")
+
+    def _with_new_scol(self, scol: spark.Column, *, dtype=None):
+        raise NotImplementedError("Not supported for type MultiIndex")
+
+    def _align_and_column_op(self, f, *args) -> Index:
+        raise NotImplementedError("Not supported for type MultiIndex")
+
+    def any(self, *args, **kwargs) -> None:
+        raise TypeError("cannot perform any with this index type: MultiIndex")
+
+    def all(self, *args, **kwargs) -> None:
+        raise TypeError("cannot perform all with this index type: MultiIndex")
+
+    @staticmethod
+    def from_tuples(tuples, sortorder=None, names=None) -> "MultiIndex":
+        """
+        Convert list of tuples to MultiIndex.
+
+        Parameters
+        ----------
+        tuples : list / sequence of tuple-likes
+            Each tuple is the index of one row/column.
+        sortorder : int or None
+            Level of sortedness (must be lexicographically sorted by that level).
+        names : list / sequence of str, optional
+            Names for the levels in the index.
+
+        Returns
+        -------
+        index : MultiIndex
+
+        Examples
+        --------
+
+        >>> tuples = [(1, 'red'), (1, 'blue'),
+        ...           (2, 'red'), (2, 'blue')]
+        >>> ps.MultiIndex.from_tuples(tuples, names=('number', 'color'))  # doctest: +SKIP
+        MultiIndex([(1,  'red'),
+                    (1, 'blue'),
+                    (2,  'red'),
+                    (2, 'blue')],
+                   names=['number', 'color'])
+        """
+        return cast(
+            MultiIndex,
+            ps.from_pandas(
+                pd.MultiIndex.from_tuples(tuples=tuples, sortorder=sortorder, names=names)
+            ),
+        )
+
+    @staticmethod
+    def from_arrays(arrays, sortorder=None, names=None) -> "MultiIndex":
+        """
+        Convert arrays to MultiIndex.
+
+        Parameters
+        ----------
+        arrays: list / sequence of array-likes
+            Each array-like gives one level’s value for each data point. len(arrays)
+            is the number of levels.
+        sortorder: int or None
+            Level of sortedness (must be lexicographically sorted by that level).
+        names: list / sequence of str, optional
+            Names for the levels in the index.
+
+        Returns
+        -------
+        index: MultiIndex
+
+        Examples
+        --------
+
+        >>> arrays = [[1, 1, 2, 2], ['red', 'blue', 'red', 'blue']]
+        >>> ps.MultiIndex.from_arrays(arrays, names=('number', 'color'))  # doctest: +SKIP
+        MultiIndex([(1,  'red'),
+                    (1, 'blue'),
+                    (2,  'red'),
+                    (2, 'blue')],
+                   names=['number', 'color'])
+        """
+        return cast(
+            MultiIndex,
+            ps.from_pandas(
+                pd.MultiIndex.from_arrays(arrays=arrays, sortorder=sortorder, names=names)
+            ),
+        )
+
+    @staticmethod
+    def from_product(iterables, sortorder=None, names=None) -> "MultiIndex":
+        """
+        Make a MultiIndex from the cartesian product of multiple iterables.
+
+        Parameters
+        ----------
+        iterables : list / sequence of iterables
+            Each iterable has unique labels for each level of the index.
+        sortorder : int or None
+            Level of sortedness (must be lexicographically sorted by that
+            level).
+        names : list / sequence of str, optional
+            Names for the levels in the index.
+
+        Returns
+        -------
+        index : MultiIndex
+
+        See Also
+        --------
+        MultiIndex.from_arrays : Convert list of arrays to MultiIndex.
+        MultiIndex.from_tuples : Convert list of tuples to MultiIndex.
+
+        Examples
+        --------
+        >>> numbers = [0, 1, 2]
+        >>> colors = ['green', 'purple']
+        >>> ps.MultiIndex.from_product([numbers, colors],
+        ...                            names=['number', 'color'])  # doctest: +SKIP
+        MultiIndex([(0,  'green'),
+                    (0, 'purple'),
+                    (1,  'green'),
+                    (1, 'purple'),
+                    (2,  'green'),
+                    (2, 'purple')],
+                   names=['number', 'color'])
+        """
+        return cast(
+            MultiIndex,
+            ps.from_pandas(
+                pd.MultiIndex.from_product(iterables=iterables, sortorder=sortorder, names=names)
+            ),
+        )
+
+    @staticmethod
+    def from_frame(df, names=None) -> "MultiIndex":
+        """
+        Make a MultiIndex from a DataFrame.
+
+        Parameters
+        ----------
+        df : DataFrame
+            DataFrame to be converted to MultiIndex.
+        names : list-like, optional
+            If no names are provided, use the column names, or tuple of column
+            names if the columns is a MultiIndex. If a sequence, overwrite
+            names with the given sequence.
+
+        Returns
+        -------
+        MultiIndex
+            The MultiIndex representation of the given DataFrame.
+
+        See Also
+        --------
+        MultiIndex.from_arrays : Convert list of arrays to MultiIndex.
+        MultiIndex.from_tuples : Convert list of tuples to MultiIndex.
+        MultiIndex.from_product : Make a MultiIndex from cartesian product
+                                  of iterables.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([['HI', 'Temp'], ['HI', 'Precip'],
+        ...                    ['NJ', 'Temp'], ['NJ', 'Precip']],
+        ...                   columns=['a', 'b'])
+        >>> df  # doctest: +SKIP
+              a       b
+        0    HI    Temp
+        1    HI  Precip
+        2    NJ    Temp
+        3    NJ  Precip
+
+        >>> ps.MultiIndex.from_frame(df)  # doctest: +SKIP
+        MultiIndex([('HI',   'Temp'),
+                    ('HI', 'Precip'),
+                    ('NJ',   'Temp'),
+                    ('NJ', 'Precip')],
+                   names=['a', 'b'])
+
+        Using explicit names, instead of the column names
+
+        >>> ps.MultiIndex.from_frame(df, names=['state', 'observation'])  # doctest: +SKIP
+        MultiIndex([('HI',   'Temp'),
+                    ('HI', 'Precip'),
+                    ('NJ',   'Temp'),
+                    ('NJ', 'Precip')],
+                   names=['state', 'observation'])
+        """
+        if not isinstance(df, DataFrame):
+            raise TypeError("Input must be a DataFrame")
+        sdf = df.to_spark()
+
+        if names is None:
+            names = df._internal.column_labels
+        elif not is_list_like(names):
+            raise TypeError("Names should be list-like for a MultiIndex")
+        else:
+            names = [name if is_name_like_tuple(name) else (name,) for name in names]
+
+        internal = InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=[scol_for(sdf, col) for col in sdf.columns],
+            index_names=names,
+        )
+        return cast(MultiIndex, DataFrame(internal).index)
+
+    @property
+    def name(self) -> str:
+        raise PandasNotImplementedError(class_name="pd.MultiIndex", property_name="name")
+
+    @name.setter
+    def name(self, name: str) -> None:
+        raise PandasNotImplementedError(class_name="pd.MultiIndex", property_name="name")
+
+    def _verify_for_rename(self, name):
+        if is_list_like(name):
+            if self._internal.index_level != len(name):
+                raise ValueError(
+                    "Length of new names must be {}, got {}".format(
+                        self._internal.index_level, len(name)
+                    )
+                )
+            if any(not is_hashable(n) for n in name):
+                raise TypeError("MultiIndex.name must be a hashable type")
+            return [n if is_name_like_tuple(n) else (n,) for n in name]
+        else:
+            raise TypeError("Must pass list-like as `names`.")
+
+    def swaplevel(self, i=-2, j=-1) -> "MultiIndex":
+        """
+        Swap level i with level j.
+        Calling this method does not change the ordering of the values.
+
+        Parameters
+        ----------
+        i : int, str, default -2
+            First level of index to be swapped. Can pass level name as string.
+            Type of parameters can be mixed.
+        j : int, str, default -1
+            Second level of index to be swapped. Can pass level name as string.
+            Type of parameters can be mixed.
+
+        Returns
+        -------
+        MultiIndex
+            A new MultiIndex.
+
+        Examples
+        --------
+        >>> midx = ps.MultiIndex.from_arrays([['a', 'b'], [1, 2]], names = ['word', 'number'])
+        >>> midx  # doctest: +SKIP
+        MultiIndex([('a', 1),
+                    ('b', 2)],
+                   names=['word', 'number'])
+
+        >>> midx.swaplevel(0, 1)  # doctest: +SKIP
+        MultiIndex([(1, 'a'),
+                    (2, 'b')],
+                   names=['number', 'word'])
+
+        >>> midx.swaplevel('number', 'word')  # doctest: +SKIP
+        MultiIndex([(1, 'a'),
+                    (2, 'b')],
+                   names=['number', 'word'])
+        """
+        for index in (i, j):
+            if not isinstance(index, int) and index not in self.names:
+                raise KeyError("Level %s not found" % index)
+
+        i = i if isinstance(i, int) else self.names.index(i)
+        j = j if isinstance(j, int) else self.names.index(j)
+
+        for index in (i, j):
+            if index >= len(self.names) or index < -len(self.names):
+                raise IndexError(
+                    "Too many levels: Index has only %s levels, "
+                    "%s is not a valid level number" % (len(self.names), index)
+                )
+
+        index_map = list(
+            zip(
+                self._internal.index_spark_columns,
+                self._internal.index_names,
+                self._internal.index_dtypes,
+            )
+        )
+        index_map[i], index_map[j], = index_map[j], index_map[i]
+        index_spark_columns, index_names, index_dtypes = zip(*index_map)
+        internal = self._internal.copy(
+            index_spark_columns=list(index_spark_columns),
+            index_names=list(index_names),
+            index_dtypes=list(index_dtypes),
+            column_labels=[],
+            data_spark_columns=[],
+            data_dtypes=[],
+        )
+        return cast(MultiIndex, DataFrame(internal).index)
+
+    @property
+    def levshape(self) -> Tuple[int, ...]:
+        """
+        A tuple with the length of each level.
+
+        Examples
+        --------
+        >>> midx = ps.MultiIndex.from_tuples([('a', 'x'), ('b', 'y'), ('c', 'z')])
+        >>> midx  # doctest: +SKIP
+        MultiIndex([('a', 'x'),
+                    ('b', 'y'),
+                    ('c', 'z')],
+                   )
+
+        >>> midx.levshape
+        (3, 3)
+        """
+        result = self._internal.spark_frame.agg(
+            *(F.countDistinct(c) for c in self._internal.index_spark_columns)
+        ).collect()[0]
+        return tuple(result)
+
+    @staticmethod
+    def _comparator_for_monotonic_increasing(data_type):
+        return compare_disallow_null
+
+    def _is_monotonic(self, order):
+        if order == "increasing":
+            return self._is_monotonic_increasing().all()
+        else:
+            return self._is_monotonic_decreasing().all()
+
+    def _is_monotonic_increasing(self):
+        window = Window.orderBy(NATURAL_ORDER_COLUMN_NAME).rowsBetween(-1, -1)
+
+        cond = F.lit(True)
+        has_not_null = F.lit(True)
+        for scol in self._internal.index_spark_columns[::-1]:
+            data_type = self._internal.spark_type_for(scol)
+            prev = F.lag(scol, 1).over(window)
+            compare = MultiIndex._comparator_for_monotonic_increasing(data_type)
+            # Since pandas 1.1.4, null value is not allowed at any levels of MultiIndex.
+            # Therefore, we should check `has_not_null` over the all levels.
+            has_not_null = has_not_null & scol.isNotNull()
+            cond = F.when(scol.eqNullSafe(prev), cond).otherwise(
+                compare(scol, prev, spark.Column.__gt__)
+            )
+
+        cond = has_not_null & (prev.isNull() | cond)
+
+        cond_name = verify_temp_column_name(
+            self._internal.spark_frame.select(self._internal.index_spark_columns),
+            "__is_monotonic_increasing_cond__",
+        )
+
+        sdf = self._internal.spark_frame.select(
+            self._internal.index_spark_columns + [cond.alias(cond_name)]
+        )
+
+        internal = InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=[
+                scol_for(sdf, col) for col in self._internal.index_spark_column_names
+            ],
+            index_names=self._internal.index_names,
+            index_dtypes=self._internal.index_dtypes,
+        )
+
+        return first_series(DataFrame(internal))
+
+    @staticmethod
+    def _comparator_for_monotonic_decreasing(data_type):
+        return compare_disallow_null
+
+    def _is_monotonic_decreasing(self):
+        window = Window.orderBy(NATURAL_ORDER_COLUMN_NAME).rowsBetween(-1, -1)
+
+        cond = F.lit(True)
+        has_not_null = F.lit(True)
+        for scol in self._internal.index_spark_columns[::-1]:
+            data_type = self._internal.spark_type_for(scol)
+            prev = F.lag(scol, 1).over(window)
+            compare = MultiIndex._comparator_for_monotonic_increasing(data_type)
+            # Since pandas 1.1.4, null value is not allowed at any levels of MultiIndex.
+            # Therefore, we should check `has_not_null` over the all levels.
+            has_not_null = has_not_null & scol.isNotNull()
+            cond = F.when(scol.eqNullSafe(prev), cond).otherwise(
+                compare(scol, prev, spark.Column.__lt__)
+            )
+
+        cond = has_not_null & (prev.isNull() | cond)
+
+        cond_name = verify_temp_column_name(
+            self._internal.spark_frame.select(self._internal.index_spark_columns),
+            "__is_monotonic_decreasing_cond__",
+        )
+
+        sdf = self._internal.spark_frame.select(
+            self._internal.index_spark_columns + [cond.alias(cond_name)]
+        )
+
+        internal = InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=[
+                scol_for(sdf, col) for col in self._internal.index_spark_column_names
+            ],
+            index_names=self._internal.index_names,
+            index_dtypes=self._internal.index_dtypes,
+        )
+
+        return first_series(DataFrame(internal))
+
+    def to_frame(self, index=True, name=None) -> DataFrame:
+        """
+        Create a DataFrame with the levels of the MultiIndex as columns.
+        Column ordering is determined by the DataFrame constructor with data as
+        a dict.
+
+        Parameters
+        ----------
+        index : boolean, default True
+            Set the index of the returned DataFrame as the original MultiIndex.
+        name : list / sequence of strings, optional
+            The passed names should substitute index level names.
+
+        Returns
+        -------
+        DataFrame : a DataFrame containing the original MultiIndex data.
+
+        See Also
+        --------
+        DataFrame
+
+        Examples
+        --------
+        >>> tuples = [(1, 'red'), (1, 'blue'),
+        ...           (2, 'red'), (2, 'blue')]
+        >>> idx = ps.MultiIndex.from_tuples(tuples, names=('number', 'color'))
+        >>> idx  # doctest: +SKIP
+        MultiIndex([(1,  'red'),
+                    (1, 'blue'),
+                    (2,  'red'),
+                    (2, 'blue')],
+                   names=['number', 'color'])
+        >>> idx.to_frame()  # doctest: +NORMALIZE_WHITESPACE
+                      number color
+        number color
+        1      red         1   red
+               blue        1  blue
+        2      red         2   red
+               blue        2  blue
+
+        By default, the original Index is reused. To enforce a new Index:
+
+        >>> idx.to_frame(index=False)
+           number color
+        0       1   red
+        1       1  blue
+        2       2   red
+        3       2  blue
+
+        To override the name of the resulting column, specify `name`:
+
+        >>> idx.to_frame(name=['n', 'c'])  # doctest: +NORMALIZE_WHITESPACE
+                      n     c
+        number color
+        1      red    1   red
+               blue   1  blue
+        2      red    2   red
+               blue   2  blue
+        """
+        if name is None:
+            name = [
+                name if name is not None else (i,)
+                for i, name in enumerate(self._internal.index_names)
+            ]
+        elif is_list_like(name):
+            if len(name) != self._internal.index_level:
+                raise ValueError("'name' should have same length as number of levels on index.")
+            name = [n if is_name_like_tuple(n) else (n,) for n in name]
+        else:
+            raise TypeError("'name' must be a list / sequence of column names.")
+
+        return self._to_frame(index=index, names=name)
+
+    def to_pandas(self) -> pd.MultiIndex:
+        """
+        Return a pandas MultiIndex.
+
+        .. note:: This method should only be used if the resulting pandas object is expected
+                  to be small, as all the data is loaded into the driver's memory.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([(.2, .3), (.0, .6), (.6, .0), (.2, .1)],
+        ...                   columns=['dogs', 'cats'],
+        ...                   index=[list('abcd'), list('efgh')])
+        >>> df['dogs'].index.to_pandas()  # doctest: +SKIP
+        MultiIndex([('a', 'e'),
+                    ('b', 'f'),
+                    ('c', 'g'),
+                    ('d', 'h')],
+                   )
+        """
+        # TODO: We might need to handle internal state change.
+        # So far, we don't have any functions to change the internal state of MultiIndex except for
+        # series-like operations. In that case, it creates new Index object instead of MultiIndex.
+        return super().to_pandas()
+
+    def toPandas(self) -> pd.MultiIndex:
+        warnings.warn(
+            "MultiIndex.toPandas is deprecated as of MultiIndex.to_pandas. "
+            "Please use the API instead.",
+            FutureWarning,
+        )
+        return self.to_pandas()
+
+    toPandas.__doc__ = to_pandas.__doc__
+
+    def nunique(self, dropna=True) -> None:  # type: ignore
+        raise NotImplementedError("nunique is not defined for MultiIndex")
+
+    # TODO: add 'name' parameter after pd.MultiIndex.name is implemented
+    def copy(self, deep=None) -> "MultiIndex":  # type: ignore
+        """
+        Make a copy of this object.
+
+        Parameters
+        ----------
+        deep : None
+            this parameter is not supported but just dummy parameter to match pandas.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([(.2, .3), (.0, .6), (.6, .0), (.2, .1)],
+        ...                   columns=['dogs', 'cats'],
+        ...                   index=[list('abcd'), list('efgh')])
+        >>> df['dogs'].index  # doctest: +SKIP
+        MultiIndex([('a', 'e'),
+                    ('b', 'f'),
+                    ('c', 'g'),
+                    ('d', 'h')],
+                   )
+
+        Copy index
+
+        >>> df.index.copy()  # doctest: +SKIP
+        MultiIndex([('a', 'e'),
+                    ('b', 'f'),
+                    ('c', 'g'),
+                    ('d', 'h')],
+                   )
+        """
+        return super().copy(deep=deep)  # type: ignore
+
+    def symmetric_difference(self, other, result_name=None, sort=None) -> "MultiIndex":
+        """
+        Compute the symmetric difference of two MultiIndex objects.
+
+        Parameters
+        ----------
+        other : Index or array-like
+        result_name : list
+        sort : True or None, default None
+            Whether to sort the resulting index.
+            * True : Attempt to sort the result.
+            * None : Do not sort the result.
+
+        Returns
+        -------
+        symmetric_difference : MiltiIndex
+
+        Notes
+        -----
+        ``symmetric_difference`` contains elements that appear in either
+        ``idx1`` or ``idx2`` but not both. Equivalent to the Index created by
+        ``idx1.difference(idx2) | idx2.difference(idx1)`` with duplicates
+        dropped.
+
+        Examples
+        --------
+        >>> midx1 = pd.MultiIndex([['lama', 'cow', 'falcon'],
+        ...                        ['speed', 'weight', 'length']],
+        ...                       [[0, 0, 0, 1, 1, 1, 2, 2, 2],
+        ...                        [0, 0, 0, 0, 1, 2, 0, 1, 2]])
+        >>> midx2 = pd.MultiIndex([['pandas-on-Spark', 'cow', 'falcon'],
+        ...                        ['speed', 'weight', 'length']],
+        ...                       [[0, 0, 0, 1, 1, 1, 2, 2, 2],
+        ...                        [0, 0, 0, 0, 1, 2, 0, 1, 2]])
+        >>> s1 = ps.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3],
+        ...                index=midx1)
+        >>> s2 = ps.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3],
+        ...              index=midx2)
+
+        >>> s1.index.symmetric_difference(s2.index)  # doctest: +SKIP
+        MultiIndex([('pandas-on-Spark', 'speed'),
+                    (  'lama', 'speed')],
+                   )
+
+        You can set names of result Index.
+
+        >>> s1.index.symmetric_difference(s2.index, result_name=['a', 'b'])  # doctest: +SKIP
+        MultiIndex([('pandas-on-Spark', 'speed'),
+                    (  'lama', 'speed')],
+                   names=['a', 'b'])
+
+        You can set sort to `True`, if you want to sort the resulting index.
+
+        >>> s1.index.symmetric_difference(s2.index, sort=True)  # doctest: +SKIP
+        MultiIndex([('pandas-on-Spark', 'speed'),
+                    (  'lama', 'speed')],
+                   )
+
+        You can also use the ``^`` operator:
+
+        >>> s1.index ^ s2.index  # doctest: +SKIP
+        MultiIndex([('pandas-on-Spark', 'speed'),
+                    (  'lama', 'speed')],
+                   )
+        """
+        if type(self) != type(other):
+            raise NotImplementedError(
+                "Doesn't support symmetric_difference between Index & MultiIndex for now"
+            )
+
+        sdf_self = self._kdf._internal.spark_frame.select(self._internal.index_spark_columns)
+        sdf_other = other._kdf._internal.spark_frame.select(other._internal.index_spark_columns)
+
+        sdf_symdiff = sdf_self.union(sdf_other).subtract(sdf_self.intersect(sdf_other))
+
+        if sort:
+            sdf_symdiff = sdf_symdiff.sort(self._internal.index_spark_columns)
+
+        internal = InternalFrame(  # TODO: dtypes?
+            spark_frame=sdf_symdiff,
+            index_spark_columns=[
+                scol_for(sdf_symdiff, col) for col in self._internal.index_spark_column_names
+            ],
+            index_names=self._internal.index_names,
+        )
+        result = cast(MultiIndex, DataFrame(internal).index)
+
+        if result_name:
+            result.names = result_name
+
+        return result
+
+    # TODO: ADD error parameter
+    def drop(self, codes, level=None) -> "MultiIndex":
+        """
+        Make new MultiIndex with passed list of labels deleted
+
+        Parameters
+        ----------
+        codes : array-like
+            Must be a list of tuples
+        level : int or level name, default None
+
+        Returns
+        -------
+        dropped : MultiIndex
+
+        Examples
+        --------
+        >>> index = ps.MultiIndex.from_tuples([('a', 'x'), ('b', 'y'), ('c', 'z')])
+        >>> index # doctest: +SKIP
+        MultiIndex([('a', 'x'),
+                    ('b', 'y'),
+                    ('c', 'z')],
+                   )
+
+        >>> index.drop(['a']) # doctest: +SKIP
+        MultiIndex([('b', 'y'),
+                    ('c', 'z')],
+                   )
+
+        >>> index.drop(['x', 'y'], level=1) # doctest: +SKIP
+        MultiIndex([('c', 'z')],
+                   )
+        """
+        internal = self._internal.resolved_copy
+        sdf = internal.spark_frame
+        index_scols = internal.index_spark_columns
+        if level is None:
+            scol = index_scols[0]
+        elif isinstance(level, int):
+            scol = index_scols[level]
+        else:
+            scol = None
+            for index_spark_column, index_name in zip(
+                internal.index_spark_columns, internal.index_names
+            ):
+                if not isinstance(level, tuple):
+                    level = (level,)
+                if level == index_name:
+                    if scol is not None:
+                        raise ValueError(
+                            "The name {} occurs multiple times, use a level number".format(
+                                name_like_string(level)
+                            )
+                        )
+                    scol = index_spark_column
+            if scol is None:
+                raise KeyError("Level {} not found".format(name_like_string(level)))
+        sdf = sdf[~scol.isin(codes)]
+
+        internal = InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=[scol_for(sdf, col) for col in internal.index_spark_column_names],
+            index_names=internal.index_names,
+            index_dtypes=internal.index_dtypes,
+            column_labels=[],
+            data_spark_columns=[],
+            data_dtypes=[],
+        )
+        return cast(MultiIndex, DataFrame(internal).index)
+
+    def argmax(self) -> None:
+        raise TypeError("reduction operation 'argmax' not allowed for this dtype")
+
+    def argmin(self) -> None:
+        raise TypeError("reduction operation 'argmin' not allowed for this dtype")
+
+    def asof(self, label) -> None:
+        raise NotImplementedError(
+            "only the default get_loc method is currently supported for MultiIndex"
+        )
+
+    @property
+    def is_all_dates(self) -> bool:
+        """
+        is_all_dates always returns False for MultiIndex
+
+        Examples
+        --------
+        >>> from datetime import datetime
+
+        >>> idx = ps.MultiIndex.from_tuples(
+        ...     [(datetime(2019, 1, 1, 0, 0, 0), datetime(2019, 1, 1, 0, 0, 0)),
+        ...      (datetime(2019, 1, 1, 0, 0, 0), datetime(2019, 1, 1, 0, 0, 0))])
+        >>> idx  # doctest: +SKIP
+        MultiIndex([('2019-01-01', '2019-01-01'),
+                    ('2019-01-01', '2019-01-01')],
+                   )
+
+        >>> idx.is_all_dates
+        False
+        """
+        return False
+
+    def __getattr__(self, item: str) -> Any:
+        if hasattr(MissingPandasLikeMultiIndex, item):
+            property_or_func = getattr(MissingPandasLikeMultiIndex, item)
+            if isinstance(property_or_func, property):
+                return property_or_func.fget(self)  # type: ignore
+            else:
+                return partial(property_or_func, self)
+        raise AttributeError("'MultiIndex' object has no attribute '{}'".format(item))
+
+    def _get_level_number(self, level) -> Optional[int]:
+        """
+        Return the level number if a valid level is given.
+        """
+        count = self.names.count(level)
+        if (count > 1) and not isinstance(level, int):
+            raise ValueError("The name %s occurs multiple times, use a level number" % level)
+        if level in self.names:
+            level = self.names.index(level)
+        elif isinstance(level, int):
+            nlevels = self.nlevels
+            if level >= nlevels:
+                raise IndexError(
+                    "Too many levels: Index has only %d "
+                    "levels, %d is not a valid level number" % (nlevels, level)
+                )
+            if level < 0:
+                if (level + nlevels) < 0:
+                    raise IndexError(
+                        "Too many levels: Index has only %d levels, "
+                        "not %d" % (nlevels, level + 1)
+                    )
+                level = level + nlevels
+        else:
+            raise KeyError("Level %s not found" % str(level))
+            return None
+
+        return level
+
+    def get_level_values(self, level) -> Index:
+        """
+        Return vector of label values for requested level,
+        equal to the length of the index.
+
+        Parameters
+        ----------
+        level : int or str
+            ``level`` is either the integer position of the level in the
+            MultiIndex, or the name of the level.
+
+        Returns
+        -------
+        values : Index
+            Values is a level of this MultiIndex converted to
+            a single :class:`Index` (or subclass thereof).
+
+        Examples
+        --------
+
+        Create a MultiIndex:
+
+        >>> mi = ps.MultiIndex.from_tuples([('x', 'a'), ('x', 'b'), ('y', 'a')])
+        >>> mi.names = ['level_1', 'level_2']
+
+        Get level values by supplying level as either integer or name:
+
+        >>> mi.get_level_values(0)
+        Index(['x', 'x', 'y'], dtype='object', name='level_1')
+
+        >>> mi.get_level_values('level_2')
+        Index(['a', 'b', 'a'], dtype='object', name='level_2')
+        """
+        level = self._get_level_number(level)
+        index_scol = self._internal.index_spark_columns[level]
+        index_name = self._internal.index_names[level]
+        index_dtype = self._internal.index_dtypes[level]
+        internal = self._internal.copy(
+            index_spark_columns=[index_scol],
+            index_names=[index_name],
+            index_dtypes=[index_dtype],
+            column_labels=[],
+            data_spark_columns=[],
+            data_dtypes=[],
+        )
+        return DataFrame(internal).index
+
+    def insert(self, loc: int, item) -> Index:
+        """
+        Make new MultiIndex inserting new item at location.
+
+        Follows Python list.append semantics for negative values.
+
+        Parameters
+        ----------
+        loc : int
+        item : object
+
+        Returns
+        -------
+        new_index : MultiIndex
+
+        Examples
+        --------
+        >>> kmidx = ps.MultiIndex.from_tuples([("a", "x"), ("b", "y"), ("c", "z")])
+        >>> kmidx.insert(3, ("h", "j"))  # doctest: +SKIP
+        MultiIndex([('a', 'x'),
+                    ('b', 'y'),
+                    ('c', 'z'),
+                    ('h', 'j')],
+                   )
+
+        For negative values
+
+        >>> kmidx.insert(-2, ("h", "j"))  # doctest: +SKIP
+        MultiIndex([('a', 'x'),
+                    ('h', 'j'),
+                    ('b', 'y'),
+                    ('c', 'z')],
+                   )
+        """
+        length = len(self)
+        if loc < 0:
+            loc = loc + length
+            if loc < 0:
+                raise IndexError(
+                    "index {} is out of bounds for axis 0 with size {}".format(
+                        (loc - length), length
+                    )
+                )
+        else:
+            if loc > length:
+                raise IndexError(
+                    "index {} is out of bounds for axis 0 with size {}".format(loc, length)
+                )
+
+        index_name = self._internal.index_spark_column_names
+        sdf_before = self.to_frame(name=index_name)[:loc].to_spark()
+        sdf_middle = Index([item]).to_frame(name=index_name).to_spark()
+        sdf_after = self.to_frame(name=index_name)[loc:].to_spark()
+        sdf = sdf_before.union(sdf_middle).union(sdf_after)
+
+        internal = InternalFrame(  # TODO: dtypes?
+            spark_frame=sdf,
+            index_spark_columns=[
+                scol_for(sdf, col) for col in self._internal.index_spark_column_names
+            ],
+            index_names=self._internal.index_names,
+        )
+        return DataFrame(internal).index
+
+    def item(self) -> Tuple[Scalar, ...]:
+        """
+        Return the first element of the underlying data as a python tuple.
+
+        Returns
+        -------
+        tuple
+            The first element of MultiIndex.
+
+        Raises
+        ------
+        ValueError
+            If the data is not length-1.
+
+        Examples
+        --------
+        >>> kmidx = ps.MultiIndex.from_tuples([('a', 'x')])
+        >>> kmidx.item()
+        ('a', 'x')
+        """
+        return self._kdf.head(2)._to_internal_pandas().index.item()
+
+    def intersection(self, other) -> "MultiIndex":
+        """
+        Form the intersection of two Index objects.
+
+        This returns a new Index with elements common to the index and `other`.
+
+        Parameters
+        ----------
+        other : Index or array-like
+
+        Returns
+        -------
+        intersection : MultiIndex
+
+        Examples
+        --------
+        >>> midx1 = ps.MultiIndex.from_tuples([("a", "x"), ("b", "y"), ("c", "z")])
+        >>> midx2 = ps.MultiIndex.from_tuples([("c", "z"), ("d", "w")])
+        >>> midx1.intersection(midx2).sort_values()  # doctest: +SKIP
+        MultiIndex([('c', 'z')],
+                   )
+        """
+        if isinstance(other, Series) or not is_list_like(other):
+            raise TypeError("other must be a MultiIndex or a list of tuples")
+        elif isinstance(other, DataFrame):
+            raise ValueError("Index data must be 1-dimensional")
+        elif isinstance(other, MultiIndex):
+            spark_frame_other = other.to_frame().to_spark()
+            keep_name = self.names == other.names
+        elif isinstance(other, Index):
+            # Always returns an empty MultiIndex if `other` is Index.
+            return self.to_frame().head(0).index  # type: ignore
+        elif not all(isinstance(item, tuple) for item in other):
+            raise TypeError("other must be a MultiIndex or a list of tuples")
+        else:
+            other = MultiIndex.from_tuples(list(other))
+            spark_frame_other = other.to_frame().to_spark()
+            keep_name = True
+
+        default_name = [SPARK_INDEX_NAME_FORMAT(i) for i in range(self.nlevels)]
+        spark_frame_self = self.to_frame(name=default_name).to_spark()
+        spark_frame_intersected = spark_frame_self.intersect(spark_frame_other)
+        if keep_name:
+            index_names = self._internal.index_names
+        else:
+            index_names = None
+        internal = InternalFrame(  # TODO: dtypes?
+            spark_frame=spark_frame_intersected,
+            index_spark_columns=[scol_for(spark_frame_intersected, col) for col in default_name],
+            index_names=index_names,
+        )
+        return cast(MultiIndex, DataFrame(internal).index)
+
+    @property
+    def hasnans(self):
+        raise NotImplementedError("hasnans is not defined for MultiIndex")
+
+    @property
+    def inferred_type(self) -> str:
+        """
+        Return a string of the type inferred from the values.
+        """
+        # Always returns "mixed" for MultiIndex
+        return "mixed"
+
+    @property
+    def asi8(self) -> None:
+        """
+        Integer representation of the values.
+        """
+        # Always returns None for MultiIndex
+        return None
+
+    def factorize(
+        self, sort: bool = True, na_sentinel: Optional[int] = -1
+    ) -> Tuple[Union["Series", "Index"], pd.Index]:
+        return MissingPandasLikeMultiIndex.factorize(self, sort=sort, na_sentinel=na_sentinel)
+
+    def __iter__(self):
+        return MissingPandasLikeMultiIndex.__iter__(self)
+
+
+def _test():
+    import os
+    import doctest
+    import sys
+    import numpy
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.indexes.multi
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.indexes.multi.__dict__.copy()
+    globs["np"] = numpy
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]")
+        .appName("pyspark.pandas.indexes.multi tests")
+        .getOrCreate()
+    )
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.indexes.multi,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/indexes/numeric.py b/python/pyspark/pandas/indexes/numeric.py
new file mode 100644
index 0000000000000..08dedf6161691
--- /dev/null
+++ b/python/pyspark/pandas/indexes/numeric.py
@@ -0,0 +1,177 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import pandas as pd
+from pandas.api.types import is_hashable
+
+from pyspark import pandas as ps
+from pyspark.pandas.indexes.base import Index
+from pyspark.pandas.series import Series
+
+
+class NumericIndex(Index):
+    """
+    Provide numeric type operations.
+    This is an abstract class.
+    """
+
+    pass
+
+
+class IntegerIndex(NumericIndex):
+    """
+    This is an abstract class for Int64Index.
+    """
+
+    pass
+
+
+class Int64Index(IntegerIndex):
+    """
+    Immutable sequence used for indexing and alignment. The basic object
+    storing axis labels for all pandas objects. Int64Index is a special case
+    of `Index` with purely integer labels.
+
+    Parameters
+    ----------
+    data : array-like (1-dimensional)
+    dtype : NumPy dtype (default: int64)
+    copy : bool
+        Make a copy of input ndarray.
+    name : object
+        Name to be stored in the index.
+
+    See Also
+    --------
+    Index : The base pandas-on-Spark Index type.
+    Float64Index : A special case of :class:`Index` with purely float labels.
+
+    Notes
+    -----
+    An Index instance can **only** contain hashable objects.
+
+    Examples
+    --------
+    >>> ps.Int64Index([1, 2, 3])
+    Int64Index([1, 2, 3], dtype='int64')
+
+    From a Series:
+
+    >>> s = ps.Series([1, 2, 3], index=[10, 20, 30])
+    >>> ps.Int64Index(s)
+    Int64Index([1, 2, 3], dtype='int64')
+
+    From an Index:
+
+    >>> idx = ps.Index([1, 2, 3])
+    >>> ps.Int64Index(idx)
+    Int64Index([1, 2, 3], dtype='int64')
+    """
+
+    def __new__(cls, data=None, dtype=None, copy=False, name=None):
+        if not is_hashable(name):
+            raise TypeError("Index.name must be a hashable type")
+
+        if isinstance(data, (Series, Index)):
+            if dtype is None:
+                dtype = "int64"
+            return Index(data, dtype=dtype, copy=copy, name=name)
+
+        return ps.from_pandas(pd.Int64Index(data=data, dtype=dtype, copy=copy, name=name))
+
+
+class Float64Index(NumericIndex):
+    """
+    Immutable sequence used for indexing and alignment. The basic object
+    storing axis labels for all pandas objects. Float64Index is a special case
+    of `Index` with purely float labels.
+
+    Parameters
+    ----------
+    data : array-like (1-dimensional)
+    dtype : NumPy dtype (default: float64)
+    copy : bool
+        Make a copy of input ndarray.
+    name : object
+        Name to be stored in the index.
+
+    See Also
+    --------
+    Index : The base pandas-on-Spark Index type.
+    Int64Index : A special case of :class:`Index` with purely integer labels.
+
+    Notes
+    -----
+    An Index instance can **only** contain hashable objects.
+
+    Examples
+    --------
+    >>> ps.Float64Index([1.0, 2.0, 3.0])
+    Float64Index([1.0, 2.0, 3.0], dtype='float64')
+
+    From a Series:
+
+    >>> s = ps.Series([1, 2, 3], index=[10, 20, 30])
+    >>> ps.Float64Index(s)
+    Float64Index([1.0, 2.0, 3.0], dtype='float64')
+
+    From an Index:
+
+    >>> idx = ps.Index([1, 2, 3])
+    >>> ps.Float64Index(idx)
+    Float64Index([1.0, 2.0, 3.0], dtype='float64')
+    """
+
+    def __new__(cls, data=None, dtype=None, copy=False, name=None):
+        if not is_hashable(name):
+            raise TypeError("Index.name must be a hashable type")
+
+        if isinstance(data, (Series, Index)):
+            if dtype is None:
+                dtype = "float64"
+            return Index(data, dtype=dtype, copy=copy, name=name)
+
+        return ps.from_pandas(pd.Float64Index(data=data, dtype=dtype, copy=copy, name=name))
+
+
+def _test():
+    import os
+    import doctest
+    import sys
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.indexes.numeric
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.indexes.numeric.__dict__.copy()
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]")
+        .appName("pyspark.pandas.indexes.numeric tests")
+        .getOrCreate()
+    )
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.indexes.numeric,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/indexing.py b/python/pyspark/pandas/indexing.py
new file mode 100644
index 0000000000000..0b7c0b2a105d6
--- /dev/null
+++ b/python/pyspark/pandas/indexing.py
@@ -0,0 +1,1758 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+A loc indexer for pandas-on-Spark DataFrame/Series.
+"""
+from abc import ABCMeta, abstractmethod
+from collections.abc import Iterable
+from functools import reduce
+from typing import Any, Optional, List, Tuple, TYPE_CHECKING, Union, cast, Sized
+
+import pandas as pd
+from pandas.api.types import is_list_like
+from pyspark import sql as spark
+from pyspark.sql import functions as F
+from pyspark.sql.types import BooleanType, LongType
+from pyspark.sql.utils import AnalysisException
+import numpy as np
+
+from pyspark import pandas as ps  # noqa: F401
+from pyspark.pandas.internal import (
+    InternalFrame,
+    NATURAL_ORDER_COLUMN_NAME,
+    SPARK_DEFAULT_SERIES_NAME,
+)
+from pyspark.pandas.exceptions import SparkPandasIndexingError, SparkPandasNotImplementedError
+from pyspark.pandas.typedef.typehints import (
+    Dtype,
+    Scalar,
+    extension_dtypes,
+    spark_type_to_pandas_dtype,
+)
+from pyspark.pandas.utils import (
+    is_name_like_tuple,
+    is_name_like_value,
+    lazy_property,
+    name_like_string,
+    same_anchor,
+    scol_for,
+    verify_temp_column_name,
+)
+
+if TYPE_CHECKING:
+    from pyspark.pandas.frame import DataFrame  # noqa: F401 (SPARK-34943)
+    from pyspark.pandas.series import Series  # noqa: F401 (SPARK-34943)
+
+
+class IndexerLike(object):
+    def __init__(self, kdf_or_kser):
+        from pyspark.pandas.frame import DataFrame
+        from pyspark.pandas.series import Series
+
+        assert isinstance(kdf_or_kser, (DataFrame, Series)), "unexpected argument type: {}".format(
+            type(kdf_or_kser)
+        )
+        self._kdf_or_kser = kdf_or_kser
+
+    @property
+    def _is_df(self):
+        from pyspark.pandas.frame import DataFrame
+
+        return isinstance(self._kdf_or_kser, DataFrame)
+
+    @property
+    def _is_series(self):
+        from pyspark.pandas.series import Series
+
+        return isinstance(self._kdf_or_kser, Series)
+
+    @property
+    def _kdf(self):
+        if self._is_df:
+            return self._kdf_or_kser
+        else:
+            assert self._is_series
+            return self._kdf_or_kser._kdf
+
+    @property
+    def _internal(self):
+        return self._kdf._internal
+
+
+class AtIndexer(IndexerLike):
+    """
+    Access a single value for a row/column label pair.
+    If the index is not unique, all matching pairs are returned as an array.
+    Similar to ``loc``, in that both provide label-based lookups. Use ``at`` if you only need to
+    get a single value in a DataFrame or Series.
+
+    .. note:: Unlike pandas, pandas-on-Spark only allows using ``at`` to get values but not to
+        set them.
+
+    .. note:: Warning: If ``row_index`` matches a lot of rows, large amounts of data will be
+        fetched, potentially causing your machine to run out of memory.
+
+    Raises
+    ------
+    KeyError
+        When label does not exist in DataFrame
+
+    Examples
+    --------
+    >>> kdf = ps.DataFrame([[0, 2, 3], [0, 4, 1], [10, 20, 30]],
+    ...                    index=[4, 5, 5], columns=['A', 'B', 'C'])
+    >>> kdf
+        A   B   C
+    4   0   2   3
+    5   0   4   1
+    5  10  20  30
+
+    Get value at specified row/column pair
+
+    >>> kdf.at[4, 'B']
+    2
+
+    Get array if an index occurs multiple times
+
+    >>> kdf.at[5, 'B']
+    array([ 4, 20])
+    """
+
+    def __getitem__(self, key) -> Union["Series", "DataFrame", Scalar]:
+        if self._is_df:
+            if not isinstance(key, tuple) or len(key) != 2:
+                raise TypeError("Use DataFrame.at like .at[row_index, column_name]")
+            row_sel, col_sel = key
+        else:
+            assert self._is_series, type(self._kdf_or_kser)
+            if isinstance(key, tuple) and len(key) != 1:
+                raise TypeError("Use Series.at like .at[row_index]")
+            row_sel = key
+            col_sel = self._kdf_or_kser._column_label
+
+        if self._internal.index_level == 1:
+            if not is_name_like_value(row_sel, allow_none=False, allow_tuple=False):
+                raise ValueError("At based indexing on a single index can only have a single value")
+            row_sel = (row_sel,)
+        else:
+            if not is_name_like_tuple(row_sel, allow_none=False):
+                raise ValueError("At based indexing on multi-index can only have tuple values")
+
+        if col_sel is not None:
+            if not is_name_like_value(col_sel, allow_none=False):
+                raise ValueError("At based indexing on multi-index can only have tuple values")
+            if not is_name_like_tuple(col_sel):
+                col_sel = (col_sel,)
+
+        cond = reduce(
+            lambda x, y: x & y,
+            [scol == row for scol, row in zip(self._internal.index_spark_columns, row_sel)],
+        )
+        pdf = (
+            self._internal.spark_frame.drop(NATURAL_ORDER_COLUMN_NAME)
+            .filter(cond)
+            .select(self._internal.spark_column_for(col_sel))
+            .toPandas()
+        )
+
+        if len(pdf) < 1:
+            raise KeyError(name_like_string(row_sel))
+
+        values = pdf.iloc[:, 0].values
+        return (
+            values if (len(row_sel) < self._internal.index_level or len(values) > 1) else values[0]
+        )
+
+
+class iAtIndexer(IndexerLike):
+    """
+    Access a single value for a row/column pair by integer position.
+
+    Similar to ``iloc``, in that both provide integer-based lookups. Use
+    ``iat`` if you only need to get or set a single value in a DataFrame
+    or Series.
+
+    Raises
+    ------
+    KeyError
+        When label does not exist in DataFrame
+
+    Examples
+    --------
+    >>> df = ps.DataFrame([[0, 2, 3], [0, 4, 1], [10, 20, 30]],
+    ...                   columns=['A', 'B', 'C'])
+    >>> df
+        A   B   C
+    0   0   2   3
+    1   0   4   1
+    2  10  20  30
+
+    Get value at specified row/column pair
+
+    >>> df.iat[1, 2]
+    1
+
+    Get value within a series
+
+    >>> kser = ps.Series([1, 2, 3], index=[10, 20, 30])
+    >>> kser
+    10    1
+    20    2
+    30    3
+    dtype: int64
+
+    >>> kser.iat[1]
+    2
+    """
+
+    def __getitem__(self, key) -> Union["Series", "DataFrame", Scalar]:
+        if self._is_df:
+            if not isinstance(key, tuple) or len(key) != 2:
+                raise TypeError(
+                    "Use DataFrame.iat like .iat[row_integer_position, column_integer_position]"
+                )
+            row_sel, col_sel = key
+            if not isinstance(row_sel, int) or not isinstance(col_sel, int):
+                raise ValueError("iAt based indexing can only have integer indexers")
+            return self._kdf_or_kser.iloc[row_sel, col_sel]
+        else:
+            assert self._is_series, type(self._kdf_or_kser)
+            if not isinstance(key, int) and len(key) != 1:
+                raise TypeError("Use Series.iat like .iat[row_integer_position]")
+            if not isinstance(key, int):
+                raise ValueError("iAt based indexing can only have integer indexers")
+            return self._kdf_or_kser.iloc[key]
+
+
+class LocIndexerLike(IndexerLike, metaclass=ABCMeta):
+    def _select_rows(
+        self, rows_sel: Any
+    ) -> Tuple[Optional[spark.Column], Optional[int], Optional[int]]:
+        """
+        Dispatch the logic for select rows to more specific methods by `rows_sel` argument types.
+
+        Parameters
+        ----------
+        rows_sel : the key specified to select rows.
+
+        Returns
+        -------
+        Tuple of Spark column, int, int:
+
+            * The Spark column for the condition to filter the rows.
+            * The number of rows when the selection can be simplified by limit.
+            * The remaining index rows if the result index size is shrunk.
+        """
+        from pyspark.pandas.series import Series
+
+        if rows_sel is None:
+            return None, None, None
+        elif isinstance(rows_sel, Series):
+            return self._select_rows_by_series(rows_sel)
+        elif isinstance(rows_sel, spark.Column):
+            return self._select_rows_by_spark_column(rows_sel)
+        elif isinstance(rows_sel, slice):
+            if rows_sel == slice(None):
+                # If slice is None - select everything, so nothing to do
+                return None, None, None
+            return self._select_rows_by_slice(rows_sel)
+        elif isinstance(rows_sel, tuple):
+            return self._select_rows_else(rows_sel)
+        elif is_list_like(rows_sel):
+            return self._select_rows_by_iterable(rows_sel)
+        else:
+            return self._select_rows_else(rows_sel)
+
+    def _select_cols(
+        self, cols_sel: Any, missing_keys: Optional[List[Tuple]] = None
+    ) -> Tuple[
+        List[Tuple], Optional[List[spark.Column]], Optional[List[Dtype]], bool, Optional[Tuple]
+    ]:
+        """
+        Dispatch the logic for select columns to more specific methods by `cols_sel` argument types.
+
+        Parameters
+        ----------
+        cols_sel : the key specified to select columns.
+
+        Returns
+        -------
+        Tuple of list of column label, list of Spark columns, list of dtypes, bool:
+
+            * The column labels selected.
+            * The Spark columns selected.
+            * The dtypes selected.
+            * The boolean value whether Series should be returned or not.
+            * The Series name if needed.
+        """
+        from pyspark.pandas.series import Series
+
+        if cols_sel is None:
+            column_labels = self._internal.column_labels
+            data_spark_columns = self._internal.data_spark_columns
+            data_dtypes = self._internal.data_dtypes
+            return column_labels, data_spark_columns, data_dtypes, False, None
+        elif isinstance(cols_sel, Series):
+            return self._select_cols_by_series(cols_sel, missing_keys)
+        elif isinstance(cols_sel, spark.Column):
+            return self._select_cols_by_spark_column(cols_sel, missing_keys)
+        elif isinstance(cols_sel, slice):
+            if cols_sel == slice(None):
+                # If slice is None - select everything, so nothing to do
+                column_labels = self._internal.column_labels
+                data_spark_columns = self._internal.data_spark_columns
+                data_dtypes = self._internal.data_dtypes
+                return column_labels, data_spark_columns, data_dtypes, False, None
+            return self._select_cols_by_slice(cols_sel, missing_keys)
+        elif isinstance(cols_sel, tuple):
+            return self._select_cols_else(cols_sel, missing_keys)
+        elif is_list_like(cols_sel):
+            return self._select_cols_by_iterable(cols_sel, missing_keys)
+        else:
+            return self._select_cols_else(cols_sel, missing_keys)
+
+    # Methods for row selection
+
+    @abstractmethod
+    def _select_rows_by_series(
+        self, rows_sel: "Series"
+    ) -> Tuple[Optional[spark.Column], Optional[int], Optional[int]]:
+        """ Select rows by `Series` type key. """
+        pass
+
+    @abstractmethod
+    def _select_rows_by_spark_column(
+        self, rows_sel: spark.column
+    ) -> Tuple[Optional[spark.Column], Optional[int], Optional[int]]:
+        """ Select rows by Spark `Column` type key. """
+        pass
+
+    @abstractmethod
+    def _select_rows_by_slice(
+        self, rows_sel: slice
+    ) -> Tuple[Optional[spark.Column], Optional[int], Optional[int]]:
+        """ Select rows by `slice` type key. """
+        pass
+
+    @abstractmethod
+    def _select_rows_by_iterable(
+        self, rows_sel: Iterable
+    ) -> Tuple[Optional[spark.Column], Optional[int], Optional[int]]:
+        """ Select rows by `Iterable` type key. """
+        pass
+
+    @abstractmethod
+    def _select_rows_else(
+        self, rows_sel: Any
+    ) -> Tuple[Optional[spark.Column], Optional[int], Optional[int]]:
+        """ Select rows by other type key. """
+        pass
+
+    # Methods for col selection
+
+    @abstractmethod
+    def _select_cols_by_series(
+        self, cols_sel: "Series", missing_keys: Optional[List[Tuple]]
+    ) -> Tuple[
+        List[Tuple], Optional[List[spark.Column]], Optional[List[Dtype]], bool, Optional[Tuple]
+    ]:
+        """ Select columns by `Series` type key. """
+        pass
+
+    @abstractmethod
+    def _select_cols_by_spark_column(
+        self, cols_sel: spark.Column, missing_keys: Optional[List[Tuple]]
+    ) -> Tuple[
+        List[Tuple], Optional[List[spark.Column]], Optional[List[Dtype]], bool, Optional[Tuple]
+    ]:
+        """ Select columns by Spark `Column` type key. """
+        pass
+
+    @abstractmethod
+    def _select_cols_by_slice(
+        self, cols_sel: slice, missing_keys: Optional[List[Tuple]]
+    ) -> Tuple[
+        List[Tuple], Optional[List[spark.Column]], Optional[List[Dtype]], bool, Optional[Tuple]
+    ]:
+        """ Select columns by `slice` type key. """
+        pass
+
+    @abstractmethod
+    def _select_cols_by_iterable(
+        self, cols_sel: Iterable, missing_keys: Optional[List[Tuple]]
+    ) -> Tuple[
+        List[Tuple], Optional[List[spark.Column]], Optional[List[Dtype]], bool, Optional[Tuple]
+    ]:
+        """ Select columns by `Iterable` type key. """
+        pass
+
+    @abstractmethod
+    def _select_cols_else(
+        self, cols_sel: Any, missing_keys: Optional[List[Tuple]]
+    ) -> Tuple[
+        List[Tuple], Optional[List[spark.Column]], Optional[List[Dtype]], bool, Optional[Tuple]
+    ]:
+        """ Select columns by other type key. """
+        pass
+
+    def __getitem__(self, key) -> Union["Series", "DataFrame"]:
+        from pyspark.pandas.frame import DataFrame
+        from pyspark.pandas.series import Series, first_series
+
+        if self._is_series:
+            if isinstance(key, Series) and not same_anchor(key, self._kdf_or_kser):
+                kdf = self._kdf_or_kser.to_frame()
+                temp_col = verify_temp_column_name(kdf, "__temp_col__")
+
+                kdf[temp_col] = key
+                return type(self)(kdf[self._kdf_or_kser.name])[kdf[temp_col]]
+
+            cond, limit, remaining_index = self._select_rows(key)
+            if cond is None and limit is None:
+                return self._kdf_or_kser
+
+            column_label = self._kdf_or_kser._column_label
+            column_labels = [column_label]
+            data_spark_columns = [self._internal.spark_column_for(column_label)]
+            data_dtypes = [self._internal.dtype_for(column_label)]
+            returns_series = True
+            series_name = self._kdf_or_kser.name
+        else:
+            assert self._is_df
+            if isinstance(key, tuple):
+                if len(key) != 2:
+                    raise SparkPandasIndexingError("Only accepts pairs of candidates")
+                rows_sel, cols_sel = key
+            else:
+                rows_sel = key
+                cols_sel = None
+
+            if isinstance(rows_sel, Series) and not same_anchor(rows_sel, self._kdf_or_kser):
+                kdf = self._kdf_or_kser.copy()
+                temp_col = verify_temp_column_name(kdf, "__temp_col__")
+
+                kdf[temp_col] = rows_sel
+                return type(self)(kdf)[kdf[temp_col], cols_sel][list(self._kdf_or_kser.columns)]
+
+            cond, limit, remaining_index = self._select_rows(rows_sel)
+            (
+                column_labels,
+                data_spark_columns,
+                data_dtypes,
+                returns_series,
+                series_name,
+            ) = self._select_cols(cols_sel)
+
+            if cond is None and limit is None and returns_series:
+                kser = self._kdf_or_kser._kser_for(column_labels[0])
+                if series_name is not None and series_name != kser.name:
+                    kser = kser.rename(series_name)
+                return kser
+
+        if remaining_index is not None:
+            index_spark_columns = self._internal.index_spark_columns[-remaining_index:]
+            index_names = self._internal.index_names[-remaining_index:]
+            index_dtypes = self._internal.index_dtypes[-remaining_index:]
+        else:
+            index_spark_columns = self._internal.index_spark_columns
+            index_names = self._internal.index_names
+            index_dtypes = self._internal.index_dtypes
+
+        if len(column_labels) > 0:
+            column_labels = column_labels.copy()
+            column_labels_level = max(
+                len(label) if label is not None else 1 for label in column_labels
+            )
+            none_column = 0
+            for i, label in enumerate(column_labels):
+                if label is None:
+                    label = (none_column,)
+                    none_column += 1
+                if len(label) < column_labels_level:
+                    label = tuple(list(label) + ([""]) * (column_labels_level - len(label)))
+                column_labels[i] = label
+
+            if i == 0 and none_column == 1:
+                column_labels = [None]
+
+            column_label_names = self._internal.column_label_names[-column_labels_level:]
+        else:
+            column_label_names = self._internal.column_label_names
+
+        try:
+            sdf = self._internal.spark_frame
+
+            if cond is not None:
+                index_columns = sdf.select(index_spark_columns).columns
+                data_columns = sdf.select(data_spark_columns).columns
+                sdf = sdf.filter(cond).select(index_spark_columns + data_spark_columns)
+                index_spark_columns = [scol_for(sdf, col) for col in index_columns]
+                data_spark_columns = [scol_for(sdf, col) for col in data_columns]
+
+            if limit is not None:
+                if limit >= 0:
+                    sdf = sdf.limit(limit)
+                else:
+                    sdf = sdf.limit(sdf.count() + limit)
+                sdf = sdf.drop(NATURAL_ORDER_COLUMN_NAME)
+        except AnalysisException:
+            raise KeyError(
+                "[{}] don't exist in columns".format(
+                    [col._jc.toString() for col in data_spark_columns]
+                )
+            )
+
+        internal = InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=index_spark_columns,
+            index_names=index_names,
+            index_dtypes=index_dtypes,
+            column_labels=column_labels,
+            data_spark_columns=data_spark_columns,
+            data_dtypes=data_dtypes,
+            column_label_names=column_label_names,
+        )
+        kdf = DataFrame(internal)
+
+        if returns_series:
+            kdf_or_kser = first_series(kdf)
+            if series_name is not None and series_name != kdf_or_kser.name:
+                kdf_or_kser = kdf_or_kser.rename(series_name)
+        else:
+            kdf_or_kser = kdf
+
+        if remaining_index is not None and remaining_index == 0:
+            pdf_or_pser = kdf_or_kser.head(2).to_pandas()
+            length = len(pdf_or_pser)
+            if length == 0:
+                raise KeyError(name_like_string(key))
+            elif length == 1:
+                return pdf_or_pser.iloc[0]
+            else:
+                return kdf_or_kser
+        else:
+            return kdf_or_kser
+
+    def __setitem__(self, key, value):
+        from pyspark.pandas.frame import DataFrame
+        from pyspark.pandas.series import Series, first_series
+
+        if self._is_series:
+            if (
+                isinstance(key, Series)
+                and (isinstance(self, iLocIndexer) or not same_anchor(key, self._kdf_or_kser))
+            ) or (
+                isinstance(value, Series)
+                and (isinstance(self, iLocIndexer) or not same_anchor(value, self._kdf_or_kser))
+            ):
+                if self._kdf_or_kser.name is None:
+                    kdf = self._kdf_or_kser.to_frame()
+                    column_label = kdf._internal.column_labels[0]
+                else:
+                    kdf = self._kdf_or_kser._kdf.copy()
+                    column_label = self._kdf_or_kser._column_label
+                temp_natural_order = verify_temp_column_name(kdf, "__temp_natural_order__")
+                temp_key_col = verify_temp_column_name(kdf, "__temp_key_col__")
+                temp_value_col = verify_temp_column_name(kdf, "__temp_value_col__")
+
+                kdf[temp_natural_order] = F.monotonically_increasing_id()
+                if isinstance(key, Series):
+                    kdf[temp_key_col] = key
+                if isinstance(value, Series):
+                    kdf[temp_value_col] = value
+                kdf = kdf.sort_values(temp_natural_order).drop(temp_natural_order)
+
+                kser = kdf._kser_for(column_label)
+                if isinstance(key, Series):
+                    key = F.col(
+                        "`{}`".format(kdf[temp_key_col]._internal.data_spark_column_names[0])
+                    )
+                if isinstance(value, Series):
+                    value = F.col(
+                        "`{}`".format(kdf[temp_value_col]._internal.data_spark_column_names[0])
+                    )
+
+                type(self)(kser)[key] = value
+
+                if self._kdf_or_kser.name is None:
+                    kser = kser.rename()
+
+                self._kdf_or_kser._kdf._update_internal_frame(
+                    kser._kdf[
+                        self._kdf_or_kser._kdf._internal.column_labels
+                    ]._internal.resolved_copy,
+                    requires_same_anchor=False,
+                )
+                return
+
+            if isinstance(value, DataFrame):
+                raise ValueError("Incompatible indexer with DataFrame")
+
+            cond, limit, remaining_index = self._select_rows(key)
+            if cond is None:
+                cond = F.lit(True)
+            if limit is not None:
+                cond = cond & (self._internal.spark_frame[self._sequence_col] < F.lit(limit))
+
+            if isinstance(value, (Series, spark.Column)):
+                if remaining_index is not None and remaining_index == 0:
+                    raise ValueError(
+                        "No axis named {} for object type {}".format(key, type(value).__name__)
+                    )
+                if isinstance(value, Series):
+                    value = value.spark.column
+            else:
+                value = F.lit(value)
+            scol = (
+                F.when(cond, value)
+                .otherwise(self._internal.spark_column_for(self._kdf_or_kser._column_label))
+                .alias(name_like_string(self._kdf_or_kser.name or SPARK_DEFAULT_SERIES_NAME))
+            )
+
+            internal = self._internal.with_new_spark_column(
+                self._kdf_or_kser._column_label, scol  # TODO: dtype?
+            )
+            self._kdf_or_kser._kdf._update_internal_frame(internal, requires_same_anchor=False)
+        else:
+            assert self._is_df
+
+            if isinstance(key, tuple):
+                if len(key) != 2:
+                    raise SparkPandasIndexingError("Only accepts pairs of candidates")
+                rows_sel, cols_sel = key
+            else:
+                rows_sel = key
+                cols_sel = None
+
+            if isinstance(value, DataFrame):
+                if len(value.columns) == 1:
+                    value = first_series(value)
+                else:
+                    raise ValueError("Only a dataframe with one column can be assigned")
+
+            if (
+                isinstance(rows_sel, Series)
+                and (isinstance(self, iLocIndexer) or not same_anchor(rows_sel, self._kdf_or_kser))
+            ) or (
+                isinstance(value, Series)
+                and (isinstance(self, iLocIndexer) or not same_anchor(value, self._kdf_or_kser))
+            ):
+                kdf = self._kdf_or_kser.copy()
+                temp_natural_order = verify_temp_column_name(kdf, "__temp_natural_order__")
+                temp_key_col = verify_temp_column_name(kdf, "__temp_key_col__")
+                temp_value_col = verify_temp_column_name(kdf, "__temp_value_col__")
+
+                kdf[temp_natural_order] = F.monotonically_increasing_id()
+                if isinstance(rows_sel, Series):
+                    kdf[temp_key_col] = rows_sel
+                if isinstance(value, Series):
+                    kdf[temp_value_col] = value
+                kdf = kdf.sort_values(temp_natural_order).drop(temp_natural_order)
+
+                if isinstance(rows_sel, Series):
+                    rows_sel = F.col(
+                        "`{}`".format(kdf[temp_key_col]._internal.data_spark_column_names[0])
+                    )
+                if isinstance(value, Series):
+                    value = F.col(
+                        "`{}`".format(kdf[temp_value_col]._internal.data_spark_column_names[0])
+                    )
+
+                type(self)(kdf)[rows_sel, cols_sel] = value
+
+                self._kdf_or_kser._update_internal_frame(
+                    kdf[list(self._kdf_or_kser.columns)]._internal.resolved_copy,
+                    requires_same_anchor=False,
+                )
+                return
+
+            cond, limit, remaining_index = self._select_rows(rows_sel)
+            missing_keys = []
+            _, data_spark_columns, _, _, _ = self._select_cols(cols_sel, missing_keys=missing_keys)
+
+            if cond is None:
+                cond = F.lit(True)
+            if limit is not None:
+                cond = cond & (self._internal.spark_frame[self._sequence_col] < F.lit(limit))
+
+            if isinstance(value, (Series, spark.Column)):
+                if remaining_index is not None and remaining_index == 0:
+                    raise ValueError("Incompatible indexer with Series")
+                if len(data_spark_columns) > 1:
+                    raise ValueError("shape mismatch")
+                if isinstance(value, Series):
+                    value = value.spark.column
+            else:
+                value = F.lit(value)
+
+            new_data_spark_columns = []
+            new_dtypes = []
+            for new_scol, spark_column_name, new_dtype in zip(
+                self._internal.data_spark_columns,
+                self._internal.data_spark_column_names,
+                self._internal.data_dtypes,
+            ):
+                for scol in data_spark_columns:
+                    if new_scol._jc.equals(scol._jc):
+                        new_scol = F.when(cond, value).otherwise(scol).alias(spark_column_name)
+                        new_dtype = spark_type_to_pandas_dtype(
+                            self._internal.spark_frame.select(new_scol).schema[0].dataType,
+                            use_extension_dtypes=isinstance(new_dtype, extension_dtypes),
+                        )
+                        break
+                new_data_spark_columns.append(new_scol)
+                new_dtypes.append(new_dtype)
+
+            column_labels = self._internal.column_labels.copy()
+            for label in missing_keys:
+                if not is_name_like_tuple(label):
+                    label = (label,)
+                if len(label) < self._internal.column_labels_level:
+                    label = tuple(
+                        list(label) + ([""] * (self._internal.column_labels_level - len(label)))
+                    )
+                elif len(label) > self._internal.column_labels_level:
+                    raise KeyError(
+                        "Key length ({}) exceeds index depth ({})".format(
+                            len(label), self._internal.column_labels_level
+                        )
+                    )
+                column_labels.append(label)
+                new_data_spark_columns.append(F.when(cond, value).alias(name_like_string(label)))
+                new_dtypes.append(None)
+
+            internal = self._internal.with_new_columns(
+                new_data_spark_columns, column_labels=column_labels, data_dtypes=new_dtypes
+            )
+            self._kdf_or_kser._update_internal_frame(internal, requires_same_anchor=False)
+
+
+class LocIndexer(LocIndexerLike):
+    """
+    Access a group of rows and columns by label(s) or a boolean Series.
+
+    ``.loc[]`` is primarily label based, but may also be used with a
+    conditional boolean Series derived from the DataFrame or Series.
+
+    Allowed inputs are:
+
+    - A single label, e.g. ``5`` or ``'a'``, (note that ``5`` is
+      interpreted as a *label* of the index, and **never** as an
+      integer position along the index) for column selection.
+
+    - A list or array of labels, e.g. ``['a', 'b', 'c']``.
+
+    - A slice object with labels, e.g. ``'a':'f'``.
+
+    - A conditional boolean Series derived from the DataFrame or Series
+
+    - A boolean array of the same length as the column axis being sliced,
+      e.g. ``[True, False, True]``.
+
+    - An alignable boolean pandas Series to the column axis being sliced.
+      The index of the key will be aligned before masking.
+
+    Not allowed inputs which pandas allows are:
+
+    - A boolean array of the same length as the row axis being sliced,
+      e.g. ``[True, False, True]``.
+    - A ``callable`` function with one argument (the calling Series, DataFrame
+      or Panel) and that returns valid output for indexing (one of the above)
+
+    .. note:: MultiIndex is not supported yet.
+
+    .. note:: Note that contrary to usual python slices, **both** the
+        start and the stop are included, and the step of the slice is not allowed.
+
+    .. note:: With a list or array of labels for row selection,
+        pandas-on-Spark behaves as a filter without reordering by the labels.
+
+    See Also
+    --------
+    Series.loc : Access group of values using labels.
+
+    Examples
+    --------
+    **Getting values**
+
+    >>> df = ps.DataFrame([[1, 2], [4, 5], [7, 8]],
+    ...                   index=['cobra', 'viper', 'sidewinder'],
+    ...                   columns=['max_speed', 'shield'])
+    >>> df
+                max_speed  shield
+    cobra               1       2
+    viper               4       5
+    sidewinder          7       8
+
+    Single label. Note this returns the row as a Series.
+
+    >>> df.loc['viper']
+    max_speed    4
+    shield       5
+    Name: viper, dtype: int64
+
+    List of labels. Note using ``[[]]`` returns a DataFrame.
+    Also note that pandas-on-Spark behaves just a filter without reordering by the labels.
+
+    >>> df.loc[['viper', 'sidewinder']]
+                max_speed  shield
+    viper               4       5
+    sidewinder          7       8
+
+    >>> df.loc[['sidewinder', 'viper']]
+                max_speed  shield
+    viper               4       5
+    sidewinder          7       8
+
+    Single label for column.
+
+    >>> df.loc['cobra', 'shield']
+    2
+
+    List of labels for row.
+
+    >>> df.loc[['cobra'], 'shield']
+    cobra    2
+    Name: shield, dtype: int64
+
+    List of labels for column.
+
+    >>> df.loc['cobra', ['shield']]
+    shield    2
+    Name: cobra, dtype: int64
+
+    List of labels for both row and column.
+
+    >>> df.loc[['cobra'], ['shield']]
+           shield
+    cobra       2
+
+    Slice with labels for row and single label for column. As mentioned
+    above, note that both the start and stop of the slice are included.
+
+    >>> df.loc['cobra':'viper', 'max_speed']
+    cobra    1
+    viper    4
+    Name: max_speed, dtype: int64
+
+    Conditional that returns a boolean Series
+
+    >>> df.loc[df['shield'] > 6]
+                max_speed  shield
+    sidewinder          7       8
+
+    Conditional that returns a boolean Series with column labels specified
+
+    >>> df.loc[df['shield'] > 6, ['max_speed']]
+                max_speed
+    sidewinder          7
+
+    A boolean array of the same length as the column axis being sliced.
+
+    >>> df.loc[:, [False, True]]
+                shield
+    cobra            2
+    viper            5
+    sidewinder       8
+
+    An alignable boolean Series to the column axis being sliced.
+
+    >>> df.loc[:, pd.Series([False, True], index=['max_speed', 'shield'])]
+                shield
+    cobra            2
+    viper            5
+    sidewinder       8
+
+    **Setting values**
+
+    Setting value for all items matching the list of labels.
+
+    >>> df.loc[['viper', 'sidewinder'], ['shield']] = 50
+    >>> df
+                max_speed  shield
+    cobra               1       2
+    viper               4      50
+    sidewinder          7      50
+
+    Setting value for an entire row
+
+    >>> df.loc['cobra'] = 10
+    >>> df
+                max_speed  shield
+    cobra              10      10
+    viper               4      50
+    sidewinder          7      50
+
+    Set value for an entire column
+
+    >>> df.loc[:, 'max_speed'] = 30
+    >>> df
+                max_speed  shield
+    cobra              30      10
+    viper              30      50
+    sidewinder         30      50
+
+    Set value for an entire list of columns
+
+    >>> df.loc[:, ['max_speed', 'shield']] = 100
+    >>> df
+                max_speed  shield
+    cobra             100     100
+    viper             100     100
+    sidewinder        100     100
+
+    Set value with Series
+
+    >>> df.loc[:, 'shield'] = df['shield'] * 2
+    >>> df
+                max_speed  shield
+    cobra             100     200
+    viper             100     200
+    sidewinder        100     200
+
+    **Getting values on a DataFrame with an index that has integer labels**
+
+    Another example using integers for the index
+
+    >>> df = ps.DataFrame([[1, 2], [4, 5], [7, 8]],
+    ...                   index=[7, 8, 9],
+    ...                   columns=['max_speed', 'shield'])
+    >>> df
+       max_speed  shield
+    7          1       2
+    8          4       5
+    9          7       8
+
+    Slice with integer labels for rows. As mentioned above, note that both
+    the start and stop of the slice are included.
+
+    >>> df.loc[7:9]
+       max_speed  shield
+    7          1       2
+    8          4       5
+    9          7       8
+    """
+
+    @staticmethod
+    def _NotImplemented(description):
+        return SparkPandasNotImplementedError(
+            description=description,
+            pandas_function=".loc[..., ...]",
+            spark_target_function="select, where",
+        )
+
+    def _select_rows_by_series(
+        self, rows_sel: "Series"
+    ) -> Tuple[Optional[spark.Column], Optional[int], Optional[int]]:
+        assert isinstance(rows_sel.spark.data_type, BooleanType), rows_sel.spark.data_type
+        return rows_sel.spark.column, None, None
+
+    def _select_rows_by_spark_column(
+        self, rows_sel: spark.Column
+    ) -> Tuple[Optional[spark.Column], Optional[int], Optional[int]]:
+        spark_type = self._internal.spark_frame.select(rows_sel).schema[0].dataType
+        assert isinstance(spark_type, BooleanType), spark_type
+        return rows_sel, None, None
+
+    def _select_rows_by_slice(
+        self, rows_sel: slice
+    ) -> Tuple[Optional[spark.Column], Optional[int], Optional[int]]:
+        from pyspark.pandas.indexes import MultiIndex
+
+        if rows_sel.step is not None:
+            raise LocIndexer._NotImplemented("Cannot use step with Spark.")
+        elif self._internal.index_level == 1:
+            sdf = self._internal.spark_frame
+            index = self._kdf_or_kser.index
+            index_column = index.to_series()
+            index_data_type = index_column.spark.data_type
+            start = rows_sel.start
+            stop = rows_sel.stop
+
+            # get natural order from '__natural_order__' from start to stop
+            # to keep natural order.
+            start_and_stop = (
+                sdf.select(index_column.spark.column, NATURAL_ORDER_COLUMN_NAME)
+                .where(
+                    (index_column.spark.column == F.lit(start).cast(index_data_type))
+                    | (index_column.spark.column == F.lit(stop).cast(index_data_type))
+                )
+                .collect()
+            )
+
+            start = [row[1] for row in start_and_stop if row[0] == start]
+            start = start[0] if len(start) > 0 else None
+
+            stop = [row[1] for row in start_and_stop if row[0] == stop]
+            stop = stop[-1] if len(stop) > 0 else None
+
+            cond = []
+            if start is not None:
+                cond.append(F.col(NATURAL_ORDER_COLUMN_NAME) >= F.lit(start).cast(LongType()))
+            if stop is not None:
+                cond.append(F.col(NATURAL_ORDER_COLUMN_NAME) <= F.lit(stop).cast(LongType()))
+
+            # if index order is not monotonic increasing or decreasing
+            # and specified values don't exist in index, raise KeyError
+            if (start is None and rows_sel.start is not None) or (
+                stop is None and rows_sel.stop is not None
+            ):
+
+                inc = index_column.is_monotonic_increasing
+                if inc is False:
+                    dec = index_column.is_monotonic_decreasing
+
+                if start is None and rows_sel.start is not None:
+                    start = rows_sel.start
+                    if inc is not False:
+                        cond.append(index_column.spark.column >= F.lit(start).cast(index_data_type))
+                    elif dec is not False:
+                        cond.append(index_column.spark.column <= F.lit(start).cast(index_data_type))
+                    else:
+                        raise KeyError(rows_sel.start)
+                if stop is None and rows_sel.stop is not None:
+                    stop = rows_sel.stop
+                    if inc is not False:
+                        cond.append(index_column.spark.column <= F.lit(stop).cast(index_data_type))
+                    elif dec is not False:
+                        cond.append(index_column.spark.column >= F.lit(stop).cast(index_data_type))
+                    else:
+                        raise KeyError(rows_sel.stop)
+
+            return reduce(lambda x, y: x & y, cond), None, None
+        else:
+            index = self._kdf_or_kser.index
+            index_data_type = [f.dataType for f in index.to_series().spark.data_type]
+
+            start = rows_sel.start
+            if start is not None:
+                if not isinstance(start, tuple):
+                    start = (start,)
+                if len(start) == 0:
+                    start = None
+            stop = rows_sel.stop
+            if stop is not None:
+                if not isinstance(stop, tuple):
+                    stop = (stop,)
+                if len(stop) == 0:
+                    stop = None
+
+            depth = max(
+                len(start) if start is not None else 0, len(stop) if stop is not None else 0
+            )
+            if depth == 0:
+                return None, None, None
+            elif (
+                depth > self._internal.index_level
+                or not index.droplevel(list(range(self._internal.index_level)[depth:])).is_monotonic
+            ):
+                raise KeyError(
+                    "Key length ({}) was greater than MultiIndex sort depth".format(depth)
+                )
+
+            conds = []  # type: List[spark.Column]
+            if start is not None:
+                cond = F.lit(True)
+                for scol, value, dt in list(
+                    zip(self._internal.index_spark_columns, start, index_data_type)
+                )[::-1]:
+                    compare = MultiIndex._comparator_for_monotonic_increasing(dt)
+                    cond = F.when(scol.eqNullSafe(F.lit(value).cast(dt)), cond).otherwise(
+                        compare(scol, F.lit(value).cast(dt), spark.Column.__gt__)
+                    )
+                conds.append(cond)
+            if stop is not None:
+                cond = F.lit(True)
+                for scol, value, dt in list(
+                    zip(self._internal.index_spark_columns, stop, index_data_type)
+                )[::-1]:
+                    compare = MultiIndex._comparator_for_monotonic_increasing(dt)
+                    cond = F.when(scol.eqNullSafe(F.lit(value).cast(dt)), cond).otherwise(
+                        compare(scol, F.lit(value).cast(dt), spark.Column.__lt__)
+                    )
+                conds.append(cond)
+
+            return reduce(lambda x, y: x & y, conds), None, None
+
+    def _select_rows_by_iterable(
+        self, rows_sel: Iterable
+    ) -> Tuple[Optional[spark.Column], Optional[int], Optional[int]]:
+        rows_sel = list(rows_sel)
+        if len(rows_sel) == 0:
+            return F.lit(False), None, None
+        elif self._internal.index_level == 1:
+            index_column = self._kdf_or_kser.index.to_series()
+            index_data_type = index_column.spark.data_type
+            if len(rows_sel) == 1:
+                return (
+                    index_column.spark.column == F.lit(rows_sel[0]).cast(index_data_type),
+                    None,
+                    None,
+                )
+            else:
+                return (
+                    index_column.spark.column.isin(
+                        [F.lit(r).cast(index_data_type) for r in rows_sel]
+                    ),
+                    None,
+                    None,
+                )
+        else:
+            raise LocIndexer._NotImplemented("Cannot select with MultiIndex with Spark.")
+
+    def _select_rows_else(
+        self, rows_sel: Any
+    ) -> Tuple[Optional[spark.Column], Optional[int], Optional[int]]:
+        if not isinstance(rows_sel, tuple):
+            rows_sel = (rows_sel,)
+        if len(rows_sel) > self._internal.index_level:
+            raise SparkPandasIndexingError("Too many indexers")
+
+        rows = [scol == value for scol, value in zip(self._internal.index_spark_columns, rows_sel)]
+        return (
+            reduce(lambda x, y: x & y, rows),
+            None,
+            self._internal.index_level - len(rows_sel),
+        )
+
+    def _get_from_multiindex_column(
+        self, key, missing_keys, labels=None, recursed=0
+    ) -> Tuple[List[Tuple], Optional[List[spark.Column]], Any, bool, Optional[Tuple]]:
+        """ Select columns from multi-index columns. """
+        assert isinstance(key, tuple)
+        if labels is None:
+            labels = [(label, label) for label in self._internal.column_labels]
+        for k in key:
+            labels = [
+                (label, None if lbl is None else lbl[1:])
+                for label, lbl in labels
+                if (lbl is None and k is None) or (lbl is not None and lbl[0] == k)
+            ]
+            if len(labels) == 0:
+                if missing_keys is None:
+                    raise KeyError(k)
+                else:
+                    missing_keys.append(key)
+                    return [], [], [], False, None
+
+        if all(lbl is not None and len(lbl) > 0 and lbl[0] == "" for _, lbl in labels):
+            # If the head is '', drill down recursively.
+            labels = [(label, tuple([str(key), *lbl[1:]])) for i, (label, lbl) in enumerate(labels)]
+            return self._get_from_multiindex_column((str(key),), missing_keys, labels, recursed + 1)
+        else:
+            returns_series = all(lbl is None or len(lbl) == 0 for _, lbl in labels)
+            if returns_series:
+                labels = set(label for label, _ in labels)
+                assert len(labels) == 1
+                label = list(labels)[0]
+                column_labels = [label]
+                data_spark_columns = [self._internal.spark_column_for(label)]
+                data_dtypes = [self._internal.dtype_for(label)]
+                if label is None:
+                    series_name = None
+                else:
+                    if recursed > 0:
+                        label = label[:-recursed]
+                    series_name = label if len(label) > 1 else label[0]
+            else:
+                column_labels = [
+                    None if lbl is None or lbl == (None,) else lbl for _, lbl in labels
+                ]
+                data_spark_columns = [self._internal.spark_column_for(label) for label, _ in labels]
+                data_dtypes = [self._internal.dtype_for(label) for label, _ in labels]
+                series_name = None
+
+            return column_labels, data_spark_columns, data_dtypes, returns_series, series_name
+
+    def _select_cols_by_series(
+        self, cols_sel: "Series", missing_keys: Optional[List[Tuple]]
+    ) -> Tuple[
+        List[Tuple], Optional[List[spark.Column]], Optional[List[Dtype]], bool, Optional[Tuple]
+    ]:
+        column_labels = [cols_sel._column_label]
+        data_spark_columns = [cols_sel.spark.column]
+        data_dtypes = [cols_sel.dtype]
+        return column_labels, data_spark_columns, data_dtypes, True, None
+
+    def _select_cols_by_spark_column(
+        self, cols_sel: spark.Column, missing_keys: Optional[List[Tuple]]
+    ) -> Tuple[
+        List[Tuple], Optional[List[spark.Column]], Optional[List[Dtype]], bool, Optional[Tuple]
+    ]:
+        column_labels = [
+            (self._internal.spark_frame.select(cols_sel).columns[0],)
+        ]  # type: List[Tuple]
+        data_spark_columns = [cols_sel]
+        return column_labels, data_spark_columns, None, True, None
+
+    def _select_cols_by_slice(
+        self, cols_sel: slice, missing_keys: Optional[List[Tuple]]
+    ) -> Tuple[
+        List[Tuple], Optional[List[spark.Column]], Optional[List[Dtype]], bool, Optional[Tuple]
+    ]:
+        start, stop = self._kdf_or_kser.columns.slice_locs(start=cols_sel.start, end=cols_sel.stop)
+        column_labels = self._internal.column_labels[start:stop]
+        data_spark_columns = self._internal.data_spark_columns[start:stop]
+        data_dtypes = self._internal.data_dtypes[start:stop]
+        return column_labels, data_spark_columns, data_dtypes, False, None
+
+    def _select_cols_by_iterable(
+        self, cols_sel: Iterable, missing_keys: Optional[List[Tuple]]
+    ) -> Tuple[
+        List[Tuple], Optional[List[spark.Column]], Optional[List[Dtype]], bool, Optional[Tuple]
+    ]:
+        from pyspark.pandas.series import Series
+
+        if all(isinstance(key, Series) for key in cols_sel):
+            column_labels = [key._column_label for key in cols_sel]
+            data_spark_columns = [key.spark.column for key in cols_sel]
+            data_dtypes = [key.dtype for key in cols_sel]
+        elif all(isinstance(key, spark.Column) for key in cols_sel):
+            column_labels = [
+                (self._internal.spark_frame.select(col).columns[0],) for col in cols_sel
+            ]
+            data_spark_columns = list(cols_sel)
+            data_dtypes = None
+        elif all(isinstance(key, bool) for key in cols_sel) or all(
+            isinstance(key, np.bool_) for key in cols_sel
+        ):
+            if len(cast(Sized, cols_sel)) != len(self._internal.column_labels):
+                raise IndexError(
+                    "Boolean index has wrong length: %s instead of %s"
+                    % (len(cast(Sized, cols_sel)), len(self._internal.column_labels))
+                )
+            if isinstance(cols_sel, pd.Series):
+                if not cols_sel.index.sort_values().equals(self._kdf.columns.sort_values()):
+                    raise SparkPandasIndexingError(
+                        "Unalignable boolean Series provided as indexer "
+                        "(index of the boolean Series and of the indexed object do not match)"
+                    )
+                else:
+                    column_labels = [
+                        column_label
+                        for column_label in self._internal.column_labels
+                        if cols_sel[column_label if len(column_label) > 1 else column_label[0]]
+                    ]
+                    data_spark_columns = [
+                        self._internal.spark_column_for(column_label)
+                        for column_label in column_labels
+                    ]
+                    data_dtypes = [
+                        self._internal.dtype_for(column_label) for column_label in column_labels
+                    ]
+            else:
+                column_labels = [
+                    self._internal.column_labels[i] for i, col in enumerate(cols_sel) if col
+                ]
+                data_spark_columns = [
+                    self._internal.data_spark_columns[i] for i, col in enumerate(cols_sel) if col
+                ]
+                data_dtypes = [
+                    self._internal.data_dtypes[i] for i, col in enumerate(cols_sel) if col
+                ]
+        elif any(isinstance(key, tuple) for key in cols_sel) and any(
+            not is_name_like_tuple(key) for key in cols_sel
+        ):
+            raise TypeError(
+                "Expected tuple, got {}".format(
+                    type(set(key for key in cols_sel if not is_name_like_tuple(key)).pop())
+                )
+            )
+        else:
+            if missing_keys is None and all(isinstance(key, tuple) for key in cols_sel):
+                level = self._internal.column_labels_level
+                if any(len(key) != level for key in cols_sel):
+                    raise ValueError("All the key level should be the same as column index level.")
+
+            column_labels = []
+            data_spark_columns = []
+            data_dtypes = []
+            for key in cols_sel:
+                found = False
+                for label in self._internal.column_labels:
+                    if label == key or label[0] == key:
+                        column_labels.append(label)
+                        data_spark_columns.append(self._internal.spark_column_for(label))
+                        data_dtypes.append(self._internal.dtype_for(label))
+                        found = True
+                if not found:
+                    if missing_keys is None:
+                        raise KeyError("['{}'] not in index".format(name_like_string(key)))
+                    else:
+                        missing_keys.append(key)
+
+        return column_labels, data_spark_columns, data_dtypes, False, None
+
+    def _select_cols_else(
+        self, cols_sel: Any, missing_keys: Optional[List[Tuple]]
+    ) -> Tuple[
+        List[Tuple], Optional[List[spark.Column]], Optional[List[Dtype]], bool, Optional[Tuple]
+    ]:
+        if not is_name_like_tuple(cols_sel):
+            cols_sel = (cols_sel,)
+        return self._get_from_multiindex_column(cols_sel, missing_keys)
+
+
+class iLocIndexer(LocIndexerLike):
+    """
+    Purely integer-location based indexing for selection by position.
+
+    ``.iloc[]`` is primarily integer position based (from ``0`` to
+    ``length-1`` of the axis), but may also be used with a conditional boolean Series.
+
+    Allowed inputs are:
+
+    - An integer for column selection, e.g. ``5``.
+    - A list or array of integers for row selection with distinct index values,
+      e.g. ``[3, 4, 0]``
+    - A list or array of integers for column selection, e.g. ``[4, 3, 0]``.
+    - A boolean array for column selection.
+    - A slice object with ints for row and column selection, e.g. ``1:7``.
+
+    Not allowed inputs which pandas allows are:
+
+    - A list or array of integers for row selection with duplicated indexes,
+      e.g. ``[4, 4, 0]``.
+    - A boolean array for row selection.
+    - A ``callable`` function with one argument (the calling Series, DataFrame
+      or Panel) and that returns valid output for indexing (one of the above).
+      This is useful in method chains, when you don't have a reference to the
+      calling object, but would like to base your selection on some value.
+
+    ``.iloc`` will raise ``IndexError`` if a requested indexer is
+    out-of-bounds, except *slice* indexers which allow out-of-bounds
+    indexing (this conforms with python/numpy *slice* semantics).
+
+    See Also
+    --------
+    DataFrame.loc : Purely label-location based indexer for selection by label.
+    Series.iloc : Purely integer-location based indexing for
+                   selection by position.
+
+    Examples
+    --------
+
+    >>> mydict = [{'a': 1, 'b': 2, 'c': 3, 'd': 4},
+    ...           {'a': 100, 'b': 200, 'c': 300, 'd': 400},
+    ...           {'a': 1000, 'b': 2000, 'c': 3000, 'd': 4000 }]
+    >>> df = ps.DataFrame(mydict, columns=['a', 'b', 'c', 'd'])
+    >>> df
+          a     b     c     d
+    0     1     2     3     4
+    1   100   200   300   400
+    2  1000  2000  3000  4000
+
+    **Indexing just the rows**
+
+    A scalar integer for row selection.
+
+    >>> df.iloc[1]
+    a    100
+    b    200
+    c    300
+    d    400
+    Name: 1, dtype: int64
+
+    >>> df.iloc[[0]]
+       a  b  c  d
+    0  1  2  3  4
+
+    With a `slice` object.
+
+    >>> df.iloc[:3]
+          a     b     c     d
+    0     1     2     3     4
+    1   100   200   300   400
+    2  1000  2000  3000  4000
+
+    **Indexing both axes**
+
+    You can mix the indexer types for the index and columns. Use ``:`` to
+    select the entire axis.
+
+    With scalar integers.
+
+    >>> df.iloc[:1, 1]
+    0    2
+    Name: b, dtype: int64
+
+    With lists of integers.
+
+    >>> df.iloc[:2, [1, 3]]
+         b    d
+    0    2    4
+    1  200  400
+
+    With `slice` objects.
+
+    >>> df.iloc[:2, 0:3]
+         a    b    c
+    0    1    2    3
+    1  100  200  300
+
+    With a boolean array whose length matches the columns.
+
+    >>> df.iloc[:, [True, False, True, False]]
+          a     c
+    0     1     3
+    1   100   300
+    2  1000  3000
+
+    **Setting values**
+
+    Setting value for all items matching the list of labels.
+
+    >>> df.iloc[[1, 2], [1]] = 50
+    >>> df
+          a   b     c     d
+    0     1   2     3     4
+    1   100  50   300   400
+    2  1000  50  3000  4000
+
+    Setting value for an entire row
+
+    >>> df.iloc[0] = 10
+    >>> df
+          a   b     c     d
+    0    10  10    10    10
+    1   100  50   300   400
+    2  1000  50  3000  4000
+
+    Set value for an entire column
+
+    >>> df.iloc[:, 2] = 30
+    >>> df
+          a   b   c     d
+    0    10  10  30    10
+    1   100  50  30   400
+    2  1000  50  30  4000
+
+    Set value for an entire list of columns
+
+    >>> df.iloc[:, [2, 3]] = 100
+    >>> df
+          a   b    c    d
+    0    10  10  100  100
+    1   100  50  100  100
+    2  1000  50  100  100
+
+    Set value with Series
+
+    >>> df.iloc[:, 3] = df.iloc[:, 3] * 2
+    >>> df
+          a   b    c    d
+    0    10  10  100  200
+    1   100  50  100  200
+    2  1000  50  100  200
+    """
+
+    @staticmethod
+    def _NotImplemented(description):
+        return SparkPandasNotImplementedError(
+            description=description,
+            pandas_function=".iloc[..., ...]",
+            spark_target_function="select, where",
+        )
+
+    @lazy_property
+    def _internal(self):
+        # Use resolved_copy to fix the natural order.
+        internal = super()._internal.resolved_copy
+        sdf = InternalFrame.attach_distributed_sequence_column(
+            internal.spark_frame, column_name=self._sequence_col
+        )
+        return internal.with_new_sdf(spark_frame=sdf.orderBy(NATURAL_ORDER_COLUMN_NAME))
+
+    @lazy_property
+    def _sequence_col(self):
+        # Use resolved_copy to fix the natural order.
+        internal = super()._internal.resolved_copy
+        return verify_temp_column_name(internal.spark_frame, "__distributed_sequence_column__")
+
+    def _select_rows_by_series(
+        self, rows_sel: "Series"
+    ) -> Tuple[Optional[spark.Column], Optional[int], Optional[int]]:
+        raise iLocIndexer._NotImplemented(
+            ".iloc requires numeric slice, conditional "
+            "boolean Index or a sequence of positions as int, "
+            "got {}".format(type(rows_sel))
+        )
+
+    def _select_rows_by_spark_column(
+        self, rows_sel: spark.column
+    ) -> Tuple[Optional[spark.Column], Optional[int], Optional[int]]:
+        raise iLocIndexer._NotImplemented(
+            ".iloc requires numeric slice, conditional "
+            "boolean Index or a sequence of positions as int, "
+            "got {}".format(type(rows_sel))
+        )
+
+    def _select_rows_by_slice(
+        self, rows_sel: slice
+    ) -> Tuple[Optional[spark.Column], Optional[int], Optional[int]]:
+        def verify_type(i):
+            if not isinstance(i, int):
+                raise TypeError(
+                    "cannot do slice indexing with these indexers [{}] of {}".format(i, type(i))
+                )
+
+        has_negative = False
+        start = rows_sel.start
+        if start is not None:
+            verify_type(start)
+            if start == 0:
+                start = None
+            elif start < 0:
+                has_negative = True
+        stop = rows_sel.stop
+        if stop is not None:
+            verify_type(stop)
+            if stop < 0:
+                has_negative = True
+
+        step = rows_sel.step
+        if step is not None:
+            verify_type(step)
+            if step == 0:
+                raise ValueError("slice step cannot be zero")
+        else:
+            step = 1
+
+        if start is None and step == 1:
+            return None, stop, None
+
+        sdf = self._internal.spark_frame
+        sequence_scol = sdf[self._sequence_col]
+
+        if has_negative or (step < 0 and start is None):
+            cnt = sdf.count()
+
+        cond = []
+        if start is not None:
+            if start < 0:
+                start = start + cnt
+            if step >= 0:
+                cond.append(sequence_scol >= F.lit(start).cast(LongType()))
+            else:
+                cond.append(sequence_scol <= F.lit(start).cast(LongType()))
+        if stop is not None:
+            if stop < 0:
+                stop = stop + cnt
+            if step >= 0:
+                cond.append(sequence_scol < F.lit(stop).cast(LongType()))
+            else:
+                cond.append(sequence_scol > F.lit(stop).cast(LongType()))
+        if step != 1:
+            if step > 0:
+                start = start or 0
+            else:
+                start = start or (cnt - 1)
+            cond.append(((sequence_scol - start) % F.lit(step).cast(LongType())) == F.lit(0))
+
+        return reduce(lambda x, y: x & y, cond), None, None
+
+    def _select_rows_by_iterable(
+        self, rows_sel: Iterable
+    ) -> Tuple[Optional[spark.Column], Optional[int], Optional[int]]:
+        sdf = self._internal.spark_frame
+
+        if any(isinstance(key, (int, np.int, np.int64, np.int32)) and key < 0 for key in rows_sel):
+            offset = sdf.count()
+        else:
+            offset = 0
+
+        new_rows_sel = []
+        for key in list(rows_sel):
+            if not isinstance(key, (int, np.int, np.int64, np.int32)):
+                raise TypeError(
+                    "cannot do positional indexing with these indexers [{}] of {}".format(
+                        key, type(key)
+                    )
+                )
+            if key < 0:
+                key = key + offset
+            new_rows_sel.append(key)
+
+        if len(new_rows_sel) != len(set(new_rows_sel)):
+            raise NotImplementedError(
+                "Duplicated row selection is not currently supported; "
+                "however, normalised index was [%s]" % new_rows_sel
+            )
+
+        sequence_scol = sdf[self._sequence_col]
+        cond = []
+        for key in new_rows_sel:
+            cond.append(sequence_scol == F.lit(int(key)).cast(LongType()))
+
+        if len(cond) == 0:
+            cond = [F.lit(False)]
+        return reduce(lambda x, y: x | y, cond), None, None
+
+    def _select_rows_else(
+        self, rows_sel: Any
+    ) -> Tuple[Optional[spark.Column], Optional[int], Optional[int]]:
+        if isinstance(rows_sel, int):
+            sdf = self._internal.spark_frame
+            return (sdf[self._sequence_col] == rows_sel), None, 0
+        elif isinstance(rows_sel, tuple):
+            raise SparkPandasIndexingError("Too many indexers")
+        else:
+            raise iLocIndexer._NotImplemented(
+                ".iloc requires numeric slice, conditional "
+                "boolean Index or a sequence of positions as int, "
+                "got {}".format(type(rows_sel))
+            )
+
+    def _select_cols_by_series(
+        self, cols_sel: "Series", missing_keys: Optional[List[Tuple]]
+    ) -> Tuple[
+        List[Tuple], Optional[List[spark.Column]], Optional[List[Dtype]], bool, Optional[Tuple]
+    ]:
+        raise ValueError(
+            "Location based indexing can only have [integer, integer slice, "
+            "listlike of integers, boolean array] types, got {}".format(cols_sel)
+        )
+
+    def _select_cols_by_spark_column(
+        self, cols_sel: spark.Column, missing_keys: Optional[List[Tuple]]
+    ) -> Tuple[
+        List[Tuple], Optional[List[spark.Column]], Optional[List[Dtype]], bool, Optional[Tuple]
+    ]:
+        raise ValueError(
+            "Location based indexing can only have [integer, integer slice, "
+            "listlike of integers, boolean array] types, got {}".format(cols_sel)
+        )
+
+    def _select_cols_by_slice(
+        self, cols_sel: slice, missing_keys: Optional[List[Tuple]]
+    ) -> Tuple[
+        List[Tuple], Optional[List[spark.Column]], Optional[List[Dtype]], bool, Optional[Tuple]
+    ]:
+        if all(
+            s is None or isinstance(s, int) for s in (cols_sel.start, cols_sel.stop, cols_sel.step)
+        ):
+            column_labels = self._internal.column_labels[cols_sel]
+            data_spark_columns = self._internal.data_spark_columns[cols_sel]
+            data_dtypes = self._internal.data_dtypes[cols_sel]
+            return column_labels, data_spark_columns, data_dtypes, False, None
+        else:
+            not_none = (
+                cols_sel.start
+                if cols_sel.start is not None
+                else cols_sel.stop
+                if cols_sel.stop is not None
+                else cols_sel.step
+            )
+            raise TypeError(
+                "cannot do slice indexing with these indexers {} of {}".format(
+                    not_none, type(not_none)
+                )
+            )
+
+    def _select_cols_by_iterable(
+        self, cols_sel: Iterable, missing_keys: Optional[List[Tuple]]
+    ) -> Tuple[
+        List[Tuple], Optional[List[spark.Column]], Optional[List[Dtype]], bool, Optional[Tuple]
+    ]:
+        if all(isinstance(s, bool) for s in cols_sel):
+            cols_sel = [i for i, s in enumerate(cols_sel) if s]
+        if all(isinstance(s, int) for s in cols_sel):
+            column_labels = [self._internal.column_labels[s] for s in cols_sel]
+            data_spark_columns = [self._internal.data_spark_columns[s] for s in cols_sel]
+            data_dtypes = [self._internal.data_dtypes[s] for s in cols_sel]
+            return column_labels, data_spark_columns, data_dtypes, False, None
+        else:
+            raise TypeError("cannot perform reduce with flexible type")
+
+    def _select_cols_else(
+        self, cols_sel: Any, missing_keys: Optional[List[Tuple]]
+    ) -> Tuple[
+        List[Tuple], Optional[List[spark.Column]], Optional[List[Dtype]], bool, Optional[Tuple]
+    ]:
+        if isinstance(cols_sel, int):
+            if cols_sel > len(self._internal.column_labels):
+                raise KeyError(cols_sel)
+            column_labels = [self._internal.column_labels[cols_sel]]
+            data_spark_columns = [self._internal.data_spark_columns[cols_sel]]
+            data_dtypes = [self._internal.data_dtypes[cols_sel]]
+            return column_labels, data_spark_columns, data_dtypes, True, None
+        else:
+            raise ValueError(
+                "Location based indexing can only have [integer, integer slice, "
+                "listlike of integers, boolean array] types, got {}".format(cols_sel)
+            )
+
+    def __setitem__(self, key, value):
+        if is_list_like(value) and not isinstance(value, spark.Column):
+            iloc_item = self[key]
+            if not is_list_like(key) or not is_list_like(iloc_item):
+                raise ValueError("setting an array element with a sequence.")
+            else:
+                shape_iloc_item = iloc_item.shape
+                len_iloc_item = shape_iloc_item[0]
+                len_value = len(value)
+                if len_iloc_item != len_value:
+                    if self._is_series:
+                        raise ValueError(
+                            "cannot set using a list-like indexer with a different length than "
+                            "the value"
+                        )
+                    else:
+                        raise ValueError(
+                            "shape mismatch: value array of shape ({},) could not be broadcast "
+                            "to indexing result of shape {}".format(len_value, shape_iloc_item)
+                        )
+        super().__setitem__(key, value)
+        # Update again with resolved_copy to drop extra columns.
+        self._kdf._update_internal_frame(
+            self._kdf._internal.resolved_copy, requires_same_anchor=False
+        )
+
+        # Clean up implicitly cached properties to be able to reuse the indexer.
+        del self._internal
+        del self._sequence_col
+
+
+def _test():
+    import os
+    import doctest
+    import sys
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.indexing
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.indexing.__dict__.copy()
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]")
+        .appName("pyspark.pandas.indexing tests")
+        .getOrCreate()
+    )
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.indexing,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/internal.py b/python/pyspark/pandas/internal.py
new file mode 100644
index 0000000000000..f8b165e6c5857
--- /dev/null
+++ b/python/pyspark/pandas/internal.py
@@ -0,0 +1,1455 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+An internal immutable DataFrame with some metadata to manage indexes.
+"""
+import re
+from typing import Dict, List, Optional, Tuple, Union, TYPE_CHECKING
+from itertools import accumulate
+import py4j
+
+import numpy as np
+import pandas as pd
+from pandas.api.types import CategoricalDtype, is_datetime64_dtype, is_datetime64tz_dtype
+from pyspark import sql as spark
+from pyspark._globals import _NoValue, _NoValueType
+from pyspark.sql import functions as F, Window
+from pyspark.sql.functions import PandasUDFType, pandas_udf
+from pyspark.sql.types import BooleanType, DataType, StructField, StructType, LongType
+
+try:
+    from pyspark.sql.types import to_arrow_type
+except ImportError:
+    from pyspark.sql.pandas.types import to_arrow_type  # noqa: F401
+
+# For running doctests and reference resolution in PyCharm.
+from pyspark import pandas as ps  # noqa: F401
+
+if TYPE_CHECKING:
+    # This is required in old Python 3.5 to prevent circular reference.
+    from pyspark.pandas.series import Series  # noqa: F401 (SPARK-34943)
+from pyspark.pandas.config import get_option
+from pyspark.pandas.typedef import (
+    Dtype,
+    as_spark_type,
+    extension_dtypes,
+    infer_pd_series_spark_type,
+    spark_type_to_pandas_dtype,
+)
+from pyspark.pandas.utils import (
+    column_labels_level,
+    default_session,
+    is_name_like_tuple,
+    is_testing,
+    lazy_property,
+    name_like_string,
+    scol_for,
+    verify_temp_column_name,
+)
+
+
+# A function to turn given numbers to Spark columns that represent pandas-on-Spark index.
+SPARK_INDEX_NAME_FORMAT = "__index_level_{}__".format
+SPARK_DEFAULT_INDEX_NAME = SPARK_INDEX_NAME_FORMAT(0)
+# A pattern to check if the name of a Spark column is a pandas-on-Spark index name or not.
+SPARK_INDEX_NAME_PATTERN = re.compile(r"__index_level_[0-9]+__")
+
+NATURAL_ORDER_COLUMN_NAME = "__natural_order__"
+
+HIDDEN_COLUMNS = {NATURAL_ORDER_COLUMN_NAME}
+
+DEFAULT_SERIES_NAME = 0
+SPARK_DEFAULT_SERIES_NAME = str(DEFAULT_SERIES_NAME)
+
+
+class InternalFrame(object):
+    """
+    The internal immutable DataFrame which manages Spark DataFrame and column names and index
+    information.
+
+    .. note:: this is an internal class. It is not supposed to be exposed to users and users
+        should not directly access to it.
+
+    The internal immutable DataFrame represents the index information for a DataFrame it belongs to.
+    For instance, if we have a pandas-on-Spark DataFrame as below, pandas DataFrame does not
+    store the index as columns.
+
+    >>> kdf = ps.DataFrame({
+    ...     'A': [1, 2, 3, 4],
+    ...     'B': [5, 6, 7, 8],
+    ...     'C': [9, 10, 11, 12],
+    ...     'D': [13, 14, 15, 16],
+    ...     'E': [17, 18, 19, 20]}, columns = ['A', 'B', 'C', 'D', 'E'])
+    >>> kdf  # doctest: +NORMALIZE_WHITESPACE
+       A  B   C   D   E
+    0  1  5   9  13  17
+    1  2  6  10  14  18
+    2  3  7  11  15  19
+    3  4  8  12  16  20
+
+    However, all columns including index column are also stored in Spark DataFrame internally
+    as below.
+
+    >>> kdf._internal.to_internal_spark_frame.show()  # doctest: +NORMALIZE_WHITESPACE
+    +-----------------+---+---+---+---+---+
+    |__index_level_0__|  A|  B|  C|  D|  E|
+    +-----------------+---+---+---+---+---+
+    |                0|  1|  5|  9| 13| 17|
+    |                1|  2|  6| 10| 14| 18|
+    |                2|  3|  7| 11| 15| 19|
+    |                3|  4|  8| 12| 16| 20|
+    +-----------------+---+---+---+---+---+
+
+    In order to fill this gap, the current metadata is used by mapping Spark's internal column
+    to pandas-on-Spark's index. See the method below:
+
+    * `spark_frame` represents the internal Spark DataFrame
+
+    * `data_spark_column_names` represents non-indexing Spark column names
+
+    * `data_spark_columns` represents non-indexing Spark columns
+
+    * `data_dtypes` represents external non-indexing dtypes
+
+    * `index_spark_column_names` represents internal index Spark column names
+
+    * `index_spark_columns` represents internal index Spark columns
+
+    * `index_dtypes` represents external index dtypes
+
+    * `spark_column_names` represents all columns
+
+    * `index_names` represents the external index name as a label
+
+    * `to_internal_spark_frame` represents Spark DataFrame derived by the metadata. Includes index.
+
+    * `to_pandas_frame` represents pandas DataFrame derived by the metadata
+
+    >>> internal = kdf._internal
+    >>> internal.spark_frame.show()  # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
+    +-----------------+---+---+---+---+---+-----------------+
+    |__index_level_0__|  A|  B|  C|  D|  E|__natural_order__|
+    +-----------------+---+---+---+---+---+-----------------+
+    |                0|  1|  5|  9| 13| 17|              ...|
+    |                1|  2|  6| 10| 14| 18|              ...|
+    |                2|  3|  7| 11| 15| 19|              ...|
+    |                3|  4|  8| 12| 16| 20|              ...|
+    +-----------------+---+---+---+---+---+-----------------+
+    >>> internal.data_spark_column_names
+    ['A', 'B', 'C', 'D', 'E']
+    >>> internal.index_spark_column_names
+    ['__index_level_0__']
+    >>> internal.spark_column_names
+    ['__index_level_0__', 'A', 'B', 'C', 'D', 'E']
+    >>> internal.index_names
+    [None]
+    >>> internal.data_dtypes
+    [dtype('int64'), dtype('int64'), dtype('int64'), dtype('int64'), dtype('int64')]
+    >>> internal.index_dtypes
+    [dtype('int64')]
+    >>> internal.to_internal_spark_frame.show()  # doctest: +NORMALIZE_WHITESPACE
+    +-----------------+---+---+---+---+---+
+    |__index_level_0__|  A|  B|  C|  D|  E|
+    +-----------------+---+---+---+---+---+
+    |                0|  1|  5|  9| 13| 17|
+    |                1|  2|  6| 10| 14| 18|
+    |                2|  3|  7| 11| 15| 19|
+    |                3|  4|  8| 12| 16| 20|
+    +-----------------+---+---+---+---+---+
+    >>> internal.to_pandas_frame
+       A  B   C   D   E
+    0  1  5   9  13  17
+    1  2  6  10  14  18
+    2  3  7  11  15  19
+    3  4  8  12  16  20
+
+    In case that index is set to one of the existing column as below:
+
+    >>> kdf1 = kdf.set_index("A")
+    >>> kdf1  # doctest: +NORMALIZE_WHITESPACE
+       B   C   D   E
+    A
+    1  5   9  13  17
+    2  6  10  14  18
+    3  7  11  15  19
+    4  8  12  16  20
+
+    >>> kdf1._internal.to_internal_spark_frame.show()  # doctest: +NORMALIZE_WHITESPACE
+    +---+---+---+---+---+
+    |  A|  B|  C|  D|  E|
+    +---+---+---+---+---+
+    |  1|  5|  9| 13| 17|
+    |  2|  6| 10| 14| 18|
+    |  3|  7| 11| 15| 19|
+    |  4|  8| 12| 16| 20|
+    +---+---+---+---+---+
+
+    >>> internal = kdf1._internal
+    >>> internal.spark_frame.show()  # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
+    +-----------------+---+---+---+---+---+-----------------+
+    |__index_level_0__|  A|  B|  C|  D|  E|__natural_order__|
+    +-----------------+---+---+---+---+---+-----------------+
+    |                0|  1|  5|  9| 13| 17|              ...|
+    |                1|  2|  6| 10| 14| 18|              ...|
+    |                2|  3|  7| 11| 15| 19|              ...|
+    |                3|  4|  8| 12| 16| 20|              ...|
+    +-----------------+---+---+---+---+---+-----------------+
+    >>> internal.data_spark_column_names
+    ['B', 'C', 'D', 'E']
+    >>> internal.index_spark_column_names
+    ['A']
+    >>> internal.spark_column_names
+    ['A', 'B', 'C', 'D', 'E']
+    >>> internal.index_names
+    [('A',)]
+    >>> internal.data_dtypes
+    [dtype('int64'), dtype('int64'), dtype('int64'), dtype('int64')]
+    >>> internal.index_dtypes
+    [dtype('int64')]
+    >>> internal.to_internal_spark_frame.show()  # doctest: +NORMALIZE_WHITESPACE
+    +---+---+---+---+---+
+    |  A|  B|  C|  D|  E|
+    +---+---+---+---+---+
+    |  1|  5|  9| 13| 17|
+    |  2|  6| 10| 14| 18|
+    |  3|  7| 11| 15| 19|
+    |  4|  8| 12| 16| 20|
+    +---+---+---+---+---+
+    >>> internal.to_pandas_frame  # doctest: +NORMALIZE_WHITESPACE
+       B   C   D   E
+    A
+    1  5   9  13  17
+    2  6  10  14  18
+    3  7  11  15  19
+    4  8  12  16  20
+
+    In case that index becomes a multi index as below:
+
+    >>> kdf2 = kdf.set_index("A", append=True)
+    >>> kdf2  # doctest: +NORMALIZE_WHITESPACE
+         B   C   D   E
+      A
+    0 1  5   9  13  17
+    1 2  6  10  14  18
+    2 3  7  11  15  19
+    3 4  8  12  16  20
+
+    >>> kdf2._internal.to_internal_spark_frame.show()  # doctest: +NORMALIZE_WHITESPACE
+    +-----------------+---+---+---+---+---+
+    |__index_level_0__|  A|  B|  C|  D|  E|
+    +-----------------+---+---+---+---+---+
+    |                0|  1|  5|  9| 13| 17|
+    |                1|  2|  6| 10| 14| 18|
+    |                2|  3|  7| 11| 15| 19|
+    |                3|  4|  8| 12| 16| 20|
+    +-----------------+---+---+---+---+---+
+
+    >>> internal = kdf2._internal
+    >>> internal.spark_frame.show()  # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
+    +-----------------+---+---+---+---+---+-----------------+
+    |__index_level_0__|  A|  B|  C|  D|  E|__natural_order__|
+    +-----------------+---+---+---+---+---+-----------------+
+    |                0|  1|  5|  9| 13| 17|              ...|
+    |                1|  2|  6| 10| 14| 18|              ...|
+    |                2|  3|  7| 11| 15| 19|              ...|
+    |                3|  4|  8| 12| 16| 20|              ...|
+    +-----------------+---+---+---+---+---+-----------------+
+    >>> internal.data_spark_column_names
+    ['B', 'C', 'D', 'E']
+    >>> internal.index_spark_column_names
+    ['__index_level_0__', 'A']
+    >>> internal.spark_column_names
+    ['__index_level_0__', 'A', 'B', 'C', 'D', 'E']
+    >>> internal.index_names
+    [None, ('A',)]
+    >>> internal.data_dtypes
+    [dtype('int64'), dtype('int64'), dtype('int64'), dtype('int64')]
+    >>> internal.index_dtypes
+    [dtype('int64'), dtype('int64')]
+    >>> internal.to_internal_spark_frame.show()  # doctest: +NORMALIZE_WHITESPACE
+    +-----------------+---+---+---+---+---+
+    |__index_level_0__|  A|  B|  C|  D|  E|
+    +-----------------+---+---+---+---+---+
+    |                0|  1|  5|  9| 13| 17|
+    |                1|  2|  6| 10| 14| 18|
+    |                2|  3|  7| 11| 15| 19|
+    |                3|  4|  8| 12| 16| 20|
+    +-----------------+---+---+---+---+---+
+    >>> internal.to_pandas_frame  # doctest: +NORMALIZE_WHITESPACE
+         B   C   D   E
+      A
+    0 1  5   9  13  17
+    1 2  6  10  14  18
+    2 3  7  11  15  19
+    3 4  8  12  16  20
+
+    For multi-level columns, it also holds column_labels
+
+    >>> columns = pd.MultiIndex.from_tuples([('X', 'A'), ('X', 'B'),
+    ...                                      ('Y', 'C'), ('Y', 'D')])
+    >>> kdf3 = ps.DataFrame([
+    ...     [1, 2, 3, 4],
+    ...     [5, 6, 7, 8],
+    ...     [9, 10, 11, 12],
+    ...     [13, 14, 15, 16],
+    ...     [17, 18, 19, 20]], columns = columns)
+    >>> kdf3  # doctest: +NORMALIZE_WHITESPACE
+        X       Y
+        A   B   C   D
+    0   1   2   3   4
+    1   5   6   7   8
+    2   9  10  11  12
+    3  13  14  15  16
+    4  17  18  19  20
+
+    >>> internal = kdf3._internal
+    >>> internal.spark_frame.show()  # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
+    +-----------------+------+------+------+------+-----------------+
+    |__index_level_0__|(X, A)|(X, B)|(Y, C)|(Y, D)|__natural_order__|
+    +-----------------+------+------+------+------+-----------------+
+    |                0|     1|     2|     3|     4|              ...|
+    |                1|     5|     6|     7|     8|              ...|
+    |                2|     9|    10|    11|    12|              ...|
+    |                3|    13|    14|    15|    16|              ...|
+    |                4|    17|    18|    19|    20|              ...|
+    +-----------------+------+------+------+------+-----------------+
+    >>> internal.data_spark_column_names
+    ['(X, A)', '(X, B)', '(Y, C)', '(Y, D)']
+    >>> internal.column_labels
+    [('X', 'A'), ('X', 'B'), ('Y', 'C'), ('Y', 'D')]
+
+    For Series, it also holds scol to represent the column.
+
+    >>> kseries = kdf1.B
+    >>> kseries
+    A
+    1    5
+    2    6
+    3    7
+    4    8
+    Name: B, dtype: int64
+
+    >>> internal = kseries._internal
+    >>> internal.spark_frame.show()  # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
+    +-----------------+---+---+---+---+---+-----------------+
+    |__index_level_0__|  A|  B|  C|  D|  E|__natural_order__|
+    +-----------------+---+---+---+---+---+-----------------+
+    |                0|  1|  5|  9| 13| 17|              ...|
+    |                1|  2|  6| 10| 14| 18|              ...|
+    |                2|  3|  7| 11| 15| 19|              ...|
+    |                3|  4|  8| 12| 16| 20|              ...|
+    +-----------------+---+---+---+---+---+-----------------+
+    >>> internal.data_spark_column_names
+    ['B']
+    >>> internal.index_spark_column_names
+    ['A']
+    >>> internal.spark_column_names
+    ['A', 'B']
+    >>> internal.index_names
+    [('A',)]
+    >>> internal.data_dtypes
+    [dtype('int64')]
+    >>> internal.index_dtypes
+    [dtype('int64')]
+    >>> internal.to_internal_spark_frame.show()  # doctest: +NORMALIZE_WHITESPACE
+    +---+---+
+    |  A|  B|
+    +---+---+
+    |  1|  5|
+    |  2|  6|
+    |  3|  7|
+    |  4|  8|
+    +---+---+
+    >>> internal.to_pandas_frame  # doctest: +NORMALIZE_WHITESPACE
+       B
+    A
+    1  5
+    2  6
+    3  7
+    4  8
+    """
+
+    def __init__(
+        self,
+        spark_frame: spark.DataFrame,
+        index_spark_columns: Optional[List[spark.Column]],
+        index_names: Optional[List[Optional[Tuple]]] = None,
+        index_dtypes: Optional[List[Dtype]] = None,
+        column_labels: Optional[List[Tuple]] = None,
+        data_spark_columns: Optional[List[spark.Column]] = None,
+        data_dtypes: Optional[List[Dtype]] = None,
+        column_label_names: Optional[List[Optional[Tuple]]] = None,
+    ) -> None:
+        """
+        Create a new internal immutable DataFrame to manage Spark DataFrame, column fields and
+        index fields and names.
+
+        :param spark_frame: Spark DataFrame to be managed.
+        :param index_spark_columns: list of Spark Column
+                                    Spark Columns for the index.
+        :param index_names: list of tuples
+                            the index names.
+        :param index_dtypes: list of dtypes
+                             the index dtypes.
+        :param column_labels: list of tuples with the same length
+                              The multi-level values in the tuples.
+        :param data_spark_columns: list of Spark Column
+                                   Spark Columns to appear as columns. If this is None, calculated
+                                   from spark_frame.
+        :param data_dtypes: list of dtypes.
+                            the data dtypes.
+        :param column_label_names: Names for each of the column index levels.
+
+        See the examples below to refer what each parameter means.
+
+        >>> column_labels = pd.MultiIndex.from_tuples(
+        ...     [('a', 'x'), ('a', 'y'), ('b', 'z')], names=["column_labels_a", "column_labels_b"])
+        >>> row_index = pd.MultiIndex.from_tuples(
+        ...     [('foo', 'bar'), ('foo', 'bar'), ('zoo', 'bar')],
+        ...     names=["row_index_a", "row_index_b"])
+        >>> kdf = ps.DataFrame(
+        ...     [[1, 2, 3], [4, 5, 6], [7, 8, 9]], index=row_index, columns=column_labels)
+        >>> kdf.set_index(('a', 'x'), append=True, inplace=True)
+        >>> kdf  # doctest: +NORMALIZE_WHITESPACE
+        column_labels_a                  a  b
+        column_labels_b                  y  z
+        row_index_a row_index_b (a, x)
+        foo         bar         1       2  3
+                                4       5  6
+        zoo         bar         7       8  9
+
+        >>> internal = kdf._internal
+
+        >>> internal.spark_frame.show()  # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
+        +-----------------+-----------------+------+------+------+...
+        |__index_level_0__|__index_level_1__|(a, x)|(a, y)|(b, z)|...
+        +-----------------+-----------------+------+------+------+...
+        |              foo|              bar|     1|     2|     3|...
+        |              foo|              bar|     4|     5|     6|...
+        |              zoo|              bar|     7|     8|     9|...
+        +-----------------+-----------------+------+------+------+...
+
+        >>> internal.index_spark_columns  # doctest: +SKIP
+        [Column<'__index_level_0__'>, Column<'__index_level_1__'>, Column<'(a, x)'>]
+
+        >>> internal.index_names
+        [('row_index_a',), ('row_index_b',), ('a', 'x')]
+
+        >>> internal.index_dtypes
+        [dtype('O'), dtype('O'), dtype('int64')]
+
+        >>> internal.column_labels
+        [('a', 'y'), ('b', 'z')]
+
+        >>> internal.data_spark_columns  # doctest: +SKIP
+        [Column<'(a, y)'>, Column<'(b, z)'>]
+
+        >>> internal.data_dtypes
+        [dtype('int64'), dtype('int64')]
+
+        >>> internal.column_label_names
+        [('column_labels_a',), ('column_labels_b',)]
+        """
+
+        assert isinstance(spark_frame, spark.DataFrame)
+        assert not spark_frame.isStreaming, "pandas-on-Spark does not support Structured Streaming."
+
+        if not index_spark_columns:
+            if data_spark_columns is not None:
+                if column_labels is not None:
+                    data_spark_columns = [
+                        scol.alias(name_like_string(label))
+                        for scol, label in zip(data_spark_columns, column_labels)
+                    ]
+                spark_frame = spark_frame.select(data_spark_columns)
+
+            assert not any(SPARK_INDEX_NAME_PATTERN.match(name) for name in spark_frame.columns), (
+                "Index columns should not appear in columns of the Spark DataFrame. Avoid "
+                "index column names [%s]." % SPARK_INDEX_NAME_PATTERN
+            )
+
+            # Create default index.
+            spark_frame = InternalFrame.attach_default_index(spark_frame)
+            index_spark_columns = [scol_for(spark_frame, SPARK_DEFAULT_INDEX_NAME)]
+
+            if data_spark_columns is not None:
+                data_spark_columns = [
+                    scol_for(spark_frame, col)
+                    for col in spark_frame.columns
+                    if col != SPARK_DEFAULT_INDEX_NAME
+                ]
+
+        if NATURAL_ORDER_COLUMN_NAME not in spark_frame.columns:
+            spark_frame = spark_frame.withColumn(
+                NATURAL_ORDER_COLUMN_NAME, F.monotonically_increasing_id()
+            )
+
+        self._sdf = spark_frame  # type: spark.DataFrame
+
+        # index_spark_columns
+        assert all(
+            isinstance(index_scol, spark.Column) for index_scol in index_spark_columns
+        ), index_spark_columns
+
+        self._index_spark_columns = index_spark_columns  # type: List[spark.Column]
+
+        # index_names
+        if not index_names:
+            index_names = [None] * len(index_spark_columns)
+
+        assert len(index_spark_columns) == len(index_names), (
+            len(index_spark_columns),
+            len(index_names),
+        )
+        assert all(
+            is_name_like_tuple(index_name, check_type=True) for index_name in index_names
+        ), index_names
+
+        self._index_names = index_names  # type: List[Optional[Tuple]]
+
+        # index_dtypes
+        if not index_dtypes:
+            index_dtypes = [None] * len(index_spark_columns)
+
+        assert len(index_spark_columns) == len(index_dtypes), (
+            len(index_spark_columns),
+            len(index_dtypes),
+        )
+
+        index_dtypes = [
+            spark_type_to_pandas_dtype(spark_frame.select(scol).schema[0].dataType)
+            if dtype is None or dtype == np.dtype("object")
+            else dtype
+            for dtype, scol in zip(index_dtypes, index_spark_columns)
+        ]
+
+        assert all(
+            isinstance(dtype, Dtype.__args__)  # type: ignore
+            and (dtype == np.dtype("object") or as_spark_type(dtype, raise_error=False) is not None)
+            for dtype in index_dtypes
+        ), index_dtypes
+
+        self._index_dtypes = index_dtypes
+
+        # data_spark-columns
+        if data_spark_columns is None:
+            data_spark_columns = [
+                scol_for(spark_frame, col)
+                for col in spark_frame.columns
+                if all(
+                    not scol_for(spark_frame, col)._jc.equals(index_scol._jc)
+                    for index_scol in index_spark_columns
+                )
+                and col not in HIDDEN_COLUMNS
+            ]
+            self._data_spark_columns = data_spark_columns
+        else:
+            assert all(isinstance(scol, spark.Column) for scol in data_spark_columns)
+            self._data_spark_columns = data_spark_columns
+
+        # column_labels
+        if column_labels is None:
+            self._column_labels = [
+                (col,) for col in spark_frame.select(self._data_spark_columns).columns
+            ]  # type: List[Tuple]
+        else:
+            assert len(column_labels) == len(self._data_spark_columns), (
+                len(column_labels),
+                len(self._data_spark_columns),
+            )
+            if len(column_labels) == 1:
+                column_label = column_labels[0]
+                assert is_name_like_tuple(column_label, check_type=True), column_label
+            else:
+                assert all(
+                    is_name_like_tuple(column_label, check_type=True)
+                    for column_label in column_labels
+                ), column_labels
+                assert len(set(len(label) for label in column_labels)) <= 1, column_labels
+            self._column_labels = column_labels
+
+        # data_dtypes
+        if not data_dtypes:
+            data_dtypes = [None] * len(data_spark_columns)
+
+        assert len(data_spark_columns) == len(data_dtypes), (
+            len(data_spark_columns),
+            len(data_dtypes),
+        )
+
+        data_dtypes = [
+            spark_type_to_pandas_dtype(spark_frame.select(scol).schema[0].dataType)
+            if dtype is None or dtype == np.dtype("object")
+            else dtype
+            for dtype, scol in zip(data_dtypes, data_spark_columns)
+        ]
+
+        assert all(
+            isinstance(dtype, Dtype.__args__)  # type: ignore
+            and (dtype == np.dtype("object") or as_spark_type(dtype, raise_error=False) is not None)
+            for dtype in data_dtypes
+        ), data_dtypes
+
+        self._data_dtypes = data_dtypes
+
+        # column_label_names
+        if column_label_names is None:
+            self._column_label_names = [None] * column_labels_level(
+                self._column_labels
+            )  # type: List[Optional[Tuple]]
+        else:
+            if len(self._column_labels) > 0:
+                assert len(column_label_names) == column_labels_level(self._column_labels), (
+                    len(column_label_names),
+                    column_labels_level(self._column_labels),
+                )
+            else:
+                assert len(column_label_names) > 0, len(column_label_names)
+            assert all(
+                is_name_like_tuple(column_label_name, check_type=True)
+                for column_label_name in column_label_names
+            ), column_label_names
+            self._column_label_names = column_label_names
+
+    @staticmethod
+    def attach_default_index(sdf, default_index_type=None):
+        """
+        This method attaches a default index to Spark DataFrame. Spark does not have the index
+        notion so corresponding column should be generated.
+        There are several types of default index can be configured by `compute.default_index_type`.
+
+        >>> spark_frame = ps.range(10).to_spark()
+        >>> spark_frame
+        DataFrame[id: bigint]
+
+        It adds the default index column '__index_level_0__'.
+
+        >>> spark_frame = InternalFrame.attach_default_index(spark_frame)
+        >>> spark_frame
+        DataFrame[__index_level_0__: bigint, id: bigint]
+
+        It throws an exception if the given column name already exists.
+
+        >>> InternalFrame.attach_default_index(spark_frame)
+        ... # doctest: +ELLIPSIS
+        Traceback (most recent call last):
+          ...
+        AssertionError: '__index_level_0__' already exists...
+        """
+        index_column = SPARK_DEFAULT_INDEX_NAME
+        assert (
+            index_column not in sdf.columns
+        ), "'%s' already exists in the Spark column names '%s'" % (index_column, sdf.columns)
+
+        if default_index_type is None:
+            default_index_type = get_option("compute.default_index_type")
+
+        if default_index_type == "sequence":
+            return InternalFrame.attach_sequence_column(sdf, column_name=index_column)
+        elif default_index_type == "distributed-sequence":
+            return InternalFrame.attach_distributed_sequence_column(sdf, column_name=index_column)
+        elif default_index_type == "distributed":
+            return InternalFrame.attach_distributed_column(sdf, column_name=index_column)
+        else:
+            raise ValueError(
+                "'compute.default_index_type' should be one of 'sequence',"
+                " 'distributed-sequence' and 'distributed'"
+            )
+
+    @staticmethod
+    def attach_sequence_column(sdf, column_name):
+        scols = [scol_for(sdf, column) for column in sdf.columns]
+        sequential_index = (
+            F.row_number().over(Window.orderBy(F.monotonically_increasing_id())).cast("long") - 1
+        )
+        return sdf.select(sequential_index.alias(column_name), *scols)
+
+    @staticmethod
+    def attach_distributed_column(sdf, column_name):
+        scols = [scol_for(sdf, column) for column in sdf.columns]
+        return sdf.select(F.monotonically_increasing_id().alias(column_name), *scols)
+
+    @staticmethod
+    def attach_distributed_sequence_column(sdf, column_name):
+        """
+        This method attaches a Spark column that has a sequence in a distributed manner.
+        This is equivalent to the column assigned when default index type 'distributed-sequence'.
+
+        >>> sdf = ps.DataFrame(['a', 'b', 'c']).to_spark()
+        >>> sdf = InternalFrame.attach_distributed_sequence_column(sdf, column_name="sequence")
+        >>> sdf.show()  # doctest: +NORMALIZE_WHITESPACE
+        +--------+---+
+        |sequence|  0|
+        +--------+---+
+        |       0|  a|
+        |       1|  b|
+        |       2|  c|
+        +--------+---+
+        """
+        if len(sdf.columns) > 0:
+            try:
+                jdf = sdf._jdf.toDF()
+
+                sql_ctx = sdf.sql_ctx
+                encoders = sql_ctx._jvm.org.apache.spark.sql.Encoders
+                encoder = encoders.tuple(jdf.exprEnc(), encoders.scalaLong())
+
+                jrdd = jdf.localCheckpoint(False).rdd().zipWithIndex()
+
+                df = spark.DataFrame(
+                    sql_ctx.sparkSession._jsparkSession.createDataset(jrdd, encoder).toDF(), sql_ctx
+                )
+                columns = df.columns
+                return df.selectExpr(
+                    "`{}` as `{}`".format(columns[1], column_name), "`{}`.*".format(columns[0])
+                )
+            except py4j.protocol.Py4JError:
+                if is_testing():
+                    raise
+                return InternalFrame._attach_distributed_sequence_column(sdf, column_name)
+        else:
+            cnt = sdf.count()
+            if cnt > 0:
+                return default_session().range(cnt).toDF(column_name)
+            else:
+                return default_session().createDataFrame(
+                    [], schema=StructType().add(column_name, data_type=LongType(), nullable=False)
+                )
+
+    @staticmethod
+    def _attach_distributed_sequence_column(sdf, column_name):
+        """
+        >>> sdf = ps.DataFrame(['a', 'b', 'c']).to_spark()
+        >>> sdf = InternalFrame._attach_distributed_sequence_column(sdf, column_name="sequence")
+        >>> sdf.sort("sequence").show()  # doctest: +NORMALIZE_WHITESPACE
+        +--------+---+
+        |sequence|  0|
+        +--------+---+
+        |       0|  a|
+        |       1|  b|
+        |       2|  c|
+        +--------+---+
+        """
+        scols = [scol_for(sdf, column) for column in sdf.columns]
+
+        spark_partition_column = verify_temp_column_name(sdf, "__spark_partition_id__")
+        offset_column = verify_temp_column_name(sdf, "__offset__")
+        row_number_column = verify_temp_column_name(sdf, "__row_number__")
+
+        # 1. Calculates counts per each partition ID. `counts` here is, for instance,
+        #     {
+        #         1: 83,
+        #         6: 83,
+        #         3: 83,
+        #         ...
+        #     }
+        sdf = sdf.withColumn(spark_partition_column, F.spark_partition_id())
+
+        # Checkpoint the DataFrame to fix the partition ID.
+        sdf = sdf.localCheckpoint(eager=False)
+
+        counts = map(
+            lambda x: (x["key"], x["count"]),
+            sdf.groupby(sdf[spark_partition_column].alias("key")).count().collect(),
+        )
+
+        # 2. Calculates cumulative sum in an order of partition id.
+        #     Note that it does not matter if partition id guarantees its order or not.
+        #     We just need a one-by-one sequential id.
+
+        # sort by partition key.
+        sorted_counts = sorted(counts, key=lambda x: x[0])
+        # get cumulative sum in an order of partition key.
+        cumulative_counts = [0] + list(accumulate(map(lambda count: count[1], sorted_counts)))
+        # zip it with partition key.
+        sums = dict(zip(map(lambda count: count[0], sorted_counts), cumulative_counts))
+
+        # 3. Attach offset for each partition.
+        @pandas_udf(LongType(), PandasUDFType.SCALAR)
+        def offset(id):
+            current_partition_offset = sums[id.iloc[0]]
+            return pd.Series(current_partition_offset).repeat(len(id))
+
+        sdf = sdf.withColumn(offset_column, offset(spark_partition_column))
+
+        # 4. Calculate row_number in each partition.
+        w = Window.partitionBy(spark_partition_column).orderBy(F.monotonically_increasing_id())
+        row_number = F.row_number().over(w)
+        sdf = sdf.withColumn(row_number_column, row_number)
+
+        # 5. Calculate the index.
+        return sdf.select(
+            (sdf[offset_column] + sdf[row_number_column] - 1).alias(column_name), *scols
+        )
+
+    def spark_column_for(self, label: Tuple) -> spark.Column:
+        """ Return Spark Column for the given column label. """
+        column_labels_to_scol = dict(zip(self.column_labels, self.data_spark_columns))
+        if label in column_labels_to_scol:
+            return column_labels_to_scol[label]
+        else:
+            raise KeyError(name_like_string(label))
+
+    def spark_column_name_for(self, label_or_scol: Union[Tuple, spark.Column]) -> str:
+        """ Return the actual Spark column name for the given column label. """
+        if isinstance(label_or_scol, spark.Column):
+            scol = label_or_scol
+        else:
+            scol = self.spark_column_for(label_or_scol)
+        return self.spark_frame.select(scol).columns[0]
+
+    def spark_type_for(self, label_or_scol: Union[Tuple, spark.Column]) -> DataType:
+        """ Return DataType for the given column label. """
+        if isinstance(label_or_scol, spark.Column):
+            scol = label_or_scol
+        else:
+            scol = self.spark_column_for(label_or_scol)
+        return self.spark_frame.select(scol).schema[0].dataType
+
+    def spark_column_nullable_for(self, label_or_scol: Union[Tuple, spark.Column]) -> bool:
+        """ Return nullability for the given column label. """
+        if isinstance(label_or_scol, spark.Column):
+            scol = label_or_scol
+        else:
+            scol = self.spark_column_for(label_or_scol)
+        return self.spark_frame.select(scol).schema[0].nullable
+
+    def dtype_for(self, label: Tuple) -> Dtype:
+        """ Return dtype for the given column label. """
+        column_labels_to_dtype = dict(zip(self.column_labels, self.data_dtypes))
+        if label in column_labels_to_dtype:
+            return column_labels_to_dtype[label]
+        else:
+            raise KeyError(name_like_string(label))
+
+    @property
+    def spark_frame(self) -> spark.DataFrame:
+        """ Return the managed Spark DataFrame. """
+        return self._sdf
+
+    @lazy_property
+    def data_spark_column_names(self) -> List[str]:
+        """ Return the managed column field names. """
+        return self.spark_frame.select(self.data_spark_columns).columns
+
+    @property
+    def data_spark_columns(self) -> List[spark.Column]:
+        """ Return Spark Columns for the managed data columns. """
+        return self._data_spark_columns
+
+    @property
+    def index_spark_column_names(self) -> List[str]:
+        """ Return the managed index field names. """
+        return self.spark_frame.select(self.index_spark_columns).columns
+
+    @property
+    def index_spark_columns(self) -> List[spark.Column]:
+        """ Return Spark Columns for the managed index columns. """
+        return self._index_spark_columns
+
+    @lazy_property
+    def spark_column_names(self) -> List[str]:
+        """ Return all the field names including index field names. """
+        return self.spark_frame.select(self.spark_columns).columns
+
+    @lazy_property
+    def spark_columns(self) -> List[spark.Column]:
+        """ Return Spark Columns for the managed columns including index columns. """
+        index_spark_columns = self.index_spark_columns
+        return index_spark_columns + [
+            spark_column
+            for spark_column in self.data_spark_columns
+            if all(not spark_column._jc.equals(scol._jc) for scol in index_spark_columns)
+        ]
+
+    @property
+    def index_names(self) -> List[Optional[Tuple]]:
+        """ Return the managed index names. """
+        return self._index_names
+
+    @lazy_property
+    def index_level(self) -> int:
+        """ Return the level of the index. """
+        return len(self._index_names)
+
+    @property
+    def column_labels(self) -> List[Tuple]:
+        """ Return the managed column index. """
+        return self._column_labels
+
+    @lazy_property
+    def column_labels_level(self) -> int:
+        """ Return the level of the column index. """
+        return len(self._column_label_names)
+
+    @property
+    def column_label_names(self) -> List[Optional[Tuple]]:
+        """ Return names of the index levels. """
+        return self._column_label_names
+
+    @property
+    def index_dtypes(self) -> List[Dtype]:
+        """ Return dtypes for the managed index columns. """
+        return self._index_dtypes
+
+    @property
+    def data_dtypes(self) -> List[Dtype]:
+        """ Return dtypes for the managed columns. """
+        return self._data_dtypes
+
+    @lazy_property
+    def to_internal_spark_frame(self) -> spark.DataFrame:
+        """
+        Return as Spark DataFrame. This contains index columns as well
+        and should be only used for internal purposes.
+        """
+        index_spark_columns = self.index_spark_columns
+        data_columns = []
+        for spark_column in self.data_spark_columns:
+            if all(not spark_column._jc.equals(scol._jc) for scol in index_spark_columns):
+                data_columns.append(spark_column)
+        return self.spark_frame.select(index_spark_columns + data_columns)
+
+    @lazy_property
+    def to_pandas_frame(self) -> pd.DataFrame:
+        """ Return as pandas DataFrame. """
+        sdf = self.to_internal_spark_frame
+        pdf = sdf.toPandas()
+        if len(pdf) == 0 and len(sdf.schema) > 0:
+            pdf = pdf.astype(
+                {field.name: spark_type_to_pandas_dtype(field.dataType) for field in sdf.schema}
+            )
+
+        return InternalFrame.restore_index(pdf, **self.arguments_for_restore_index)
+
+    @lazy_property
+    def arguments_for_restore_index(self) -> Dict:
+        """ Create arguments for `restore_index`. """
+        column_names = []
+        ext_dtypes = {
+            col: dtype
+            for col, dtype in zip(self.index_spark_column_names, self.index_dtypes)
+            if isinstance(dtype, extension_dtypes)
+        }
+        categorical_dtypes = {
+            col: dtype
+            for col, dtype in zip(self.index_spark_column_names, self.index_dtypes)
+            if isinstance(dtype, CategoricalDtype)
+        }
+        for spark_column, column_name, dtype in zip(
+            self.data_spark_columns, self.data_spark_column_names, self.data_dtypes
+        ):
+            for index_spark_column_name, index_spark_column in zip(
+                self.index_spark_column_names, self.index_spark_columns
+            ):
+                if spark_column._jc.equals(index_spark_column._jc):
+                    column_names.append(index_spark_column_name)
+                    break
+            else:
+                column_names.append(column_name)
+                if isinstance(dtype, extension_dtypes):
+                    ext_dtypes[column_name] = dtype
+                elif isinstance(dtype, CategoricalDtype):
+                    categorical_dtypes[column_name] = dtype
+
+        return dict(
+            index_columns=self.index_spark_column_names,
+            index_names=self.index_names,
+            data_columns=column_names,
+            column_labels=self.column_labels,
+            column_label_names=self.column_label_names,
+            ext_dtypes=ext_dtypes,
+            categorical_dtypes=categorical_dtypes,
+        )
+
+    @staticmethod
+    def restore_index(
+        pdf: pd.DataFrame,
+        *,
+        index_columns: List[str],
+        index_names: List[Tuple],
+        data_columns: List[str],
+        column_labels: List[Tuple],
+        column_label_names: List[Tuple],
+        ext_dtypes: Dict[str, Dtype] = None,
+        categorical_dtypes: Dict[str, CategoricalDtype] = None
+    ) -> pd.DataFrame:
+        """
+        Restore pandas DataFrame indices using the metadata.
+
+        :param pdf: the pandas DataFrame to be processed.
+        :param index_columns: the original column names for index columns.
+        :param index_names: the index names after restored.
+        :param data_columns: the original column names for data columns.
+        :param column_labels: the column labels after restored.
+        :param column_label_names: the column label names after restored.
+        :param ext_dtypes: the map from the original column names to extension data types.
+        :param categorical_dtypes: the map from the original column names to categorical types.
+        :return: the restored pandas DataFrame
+
+        >>> pdf = pd.DataFrame({"index": [10, 20, 30], "a": ['a', 'b', 'c'], "b": [0, 2, 1]})
+        >>> InternalFrame.restore_index(
+        ...     pdf,
+        ...     index_columns=["index"],
+        ...     index_names=[("idx",)],
+        ...     data_columns=["a", "b", "index"],
+        ...     column_labels=[("x",), ("y",), ("z",)],
+        ...     column_label_names=[("lv1",)],
+        ...     ext_dtypes=None,
+        ...     categorical_dtypes={"b": CategoricalDtype(categories=["i", "j", "k"])}
+        ... )  # doctest: +NORMALIZE_WHITESPACE
+        lv1  x  y   z
+        idx
+        10   a  i  10
+        20   b  k  20
+        30   c  j  30
+        """
+        if ext_dtypes is not None and len(ext_dtypes) > 0:
+            pdf = pdf.astype(ext_dtypes, copy=True)
+
+        if categorical_dtypes is not None:
+            for col, dtype in categorical_dtypes.items():
+                pdf[col] = pd.Categorical.from_codes(
+                    pdf[col], categories=dtype.categories, ordered=dtype.ordered
+                )
+
+        append = False
+        for index_field in index_columns:
+            drop = index_field not in data_columns
+            pdf = pdf.set_index(index_field, drop=drop, append=append)
+            append = True
+        pdf = pdf[data_columns]
+
+        pdf.index.names = [
+            name if name is None or len(name) > 1 else name[0] for name in index_names
+        ]
+
+        names = [name if name is None or len(name) > 1 else name[0] for name in column_label_names]
+        if len(column_label_names) > 1:
+            pdf.columns = pd.MultiIndex.from_tuples(column_labels, names=names)
+        else:
+            pdf.columns = pd.Index(
+                [None if label is None else label[0] for label in column_labels], name=names[0],
+            )
+
+        return pdf
+
+    @lazy_property
+    def resolved_copy(self) -> "InternalFrame":
+        """ Copy the immutable InternalFrame with the updates resolved. """
+        sdf = self.spark_frame.select(self.spark_columns + list(HIDDEN_COLUMNS))
+        return self.copy(
+            spark_frame=sdf,
+            index_spark_columns=[scol_for(sdf, col) for col in self.index_spark_column_names],
+            data_spark_columns=[scol_for(sdf, col) for col in self.data_spark_column_names],
+        )
+
+    def with_new_sdf(
+        self,
+        spark_frame: spark.DataFrame,
+        *,
+        index_dtypes: Optional[List[Dtype]] = None,
+        data_columns: Optional[List[str]] = None,
+        data_dtypes: Optional[List[Dtype]] = None
+    ) -> "InternalFrame":
+        """ Copy the immutable InternalFrame with the updates by the specified Spark DataFrame.
+
+        :param spark_frame: the new Spark DataFrame
+        :param index_dtypes: the index dtypes. If None, the original dtyeps are used.
+        :param data_columns: the new column names. If None, the original one is used.
+        :param data_dtypes: the data dtypes. If None, the original dtyeps are used.
+        :return: the copied InternalFrame.
+        """
+        if index_dtypes is None:
+            index_dtypes = self.index_dtypes
+        else:
+            assert len(index_dtypes) == len(self.index_dtypes), (
+                len(index_dtypes),
+                len(self.index_dtypes),
+            )
+
+        if data_columns is None:
+            data_columns = self.data_spark_column_names
+        else:
+            assert len(data_columns) == len(self.column_labels), (
+                len(data_columns),
+                len(self.column_labels),
+            )
+
+        if data_dtypes is None:
+            data_dtypes = self.data_dtypes
+        else:
+            assert len(data_dtypes) == len(self.column_labels), (
+                len(data_dtypes),
+                len(self.column_labels),
+            )
+
+        sdf = spark_frame.drop(NATURAL_ORDER_COLUMN_NAME)
+        return self.copy(
+            spark_frame=sdf,
+            index_spark_columns=[scol_for(sdf, col) for col in self.index_spark_column_names],
+            index_dtypes=index_dtypes,
+            data_spark_columns=[scol_for(sdf, col) for col in data_columns],
+            data_dtypes=data_dtypes,
+        )
+
+    def with_new_columns(
+        self,
+        scols_or_ksers: List[Union[spark.Column, "Series"]],
+        *,
+        column_labels: Optional[List[Tuple]] = None,
+        data_dtypes: Optional[List[Dtype]] = None,
+        column_label_names: Optional[Union[List[Optional[Tuple]], _NoValueType]] = _NoValue,
+        keep_order: bool = True
+    ) -> "InternalFrame":
+        """
+        Copy the immutable InternalFrame with the updates by the specified Spark Columns or Series.
+
+        :param scols_or_ksers: the new Spark Columns or Series.
+        :param column_labels: the new column index.
+            If None, the column_labels of the corresponding `scols_or_ksers` is used if it is
+            Series; otherwise the original one is used.
+        :param data_dtypes: the new dtypes.
+            If None, the dtypes of the corresponding `scols_or_ksers` is used if it is Series;
+            otherwise the dtypes will be inferred from the corresponding `scols_or_ksers`.
+        :param column_label_names: the new names of the column index levels.
+        :return: the copied InternalFrame.
+        """
+        from pyspark.pandas.series import Series
+
+        if column_labels is None:
+            if all(isinstance(scol_or_kser, Series) for scol_or_kser in scols_or_ksers):
+                column_labels = [kser._column_label for kser in scols_or_ksers]
+            else:
+                assert len(scols_or_ksers) == len(self.column_labels), (
+                    len(scols_or_ksers),
+                    len(self.column_labels),
+                )
+                column_labels = []
+                for scol_or_kser, label in zip(scols_or_ksers, self.column_labels):
+                    if isinstance(scol_or_kser, Series):
+                        column_labels.append(scol_or_kser._column_label)
+                    else:
+                        column_labels.append(label)
+        else:
+            assert len(scols_or_ksers) == len(column_labels), (
+                len(scols_or_ksers),
+                len(column_labels),
+            )
+
+        data_spark_columns = []
+        for scol_or_kser in scols_or_ksers:
+            if isinstance(scol_or_kser, Series):
+                scol = scol_or_kser.spark.column
+            else:
+                scol = scol_or_kser
+            data_spark_columns.append(scol)
+
+        if data_dtypes is None:
+            data_dtypes = []
+            for scol_or_kser in scols_or_ksers:
+                if isinstance(scol_or_kser, Series):
+                    data_dtypes.append(scol_or_kser.dtype)
+                else:
+                    data_dtypes.append(None)
+        else:
+            assert len(scols_or_ksers) == len(data_dtypes), (
+                len(scols_or_ksers),
+                len(data_dtypes),
+            )
+
+        sdf = self.spark_frame
+        if not keep_order:
+            sdf = self.spark_frame.select(self.index_spark_columns + data_spark_columns)
+            index_spark_columns = [scol_for(sdf, col) for col in self.index_spark_column_names]
+            data_spark_columns = [
+                scol_for(sdf, col) for col in self.spark_frame.select(data_spark_columns).columns
+            ]
+        else:
+            index_spark_columns = self.index_spark_columns
+
+        if column_label_names is _NoValue:
+            column_label_names = self._column_label_names
+
+        return self.copy(
+            spark_frame=sdf,
+            index_spark_columns=index_spark_columns,
+            column_labels=column_labels,
+            data_spark_columns=data_spark_columns,
+            data_dtypes=data_dtypes,
+            column_label_names=column_label_names,
+        )
+
+    def with_filter(self, pred: Union[spark.Column, "Series"]) -> "InternalFrame":
+        """ Copy the immutable InternalFrame with the updates by the predicate.
+
+        :param pred: the predicate to filter.
+        :return: the copied InternalFrame.
+        """
+        from pyspark.pandas.series import Series
+
+        if isinstance(pred, Series):
+            assert isinstance(pred.spark.data_type, BooleanType), pred.spark.data_type
+            pred = pred.spark.column
+        else:
+            spark_type = self.spark_frame.select(pred).schema[0].dataType
+            assert isinstance(spark_type, BooleanType), spark_type
+
+        return self.with_new_sdf(self.spark_frame.filter(pred).select(self.spark_columns))
+
+    def with_new_spark_column(
+        self,
+        column_label: Tuple,
+        scol: spark.Column,
+        *,
+        dtype: Optional[Dtype] = None,
+        keep_order: bool = True
+    ) -> "InternalFrame":
+        """
+        Copy the immutable InternalFrame with the updates by the specified Spark Column.
+
+        :param column_label: the column label to be updated.
+        :param scol: the new Spark Column
+        :param dtype: the new dtype.
+            If not specified, the dtypes will be inferred from the spark Column.
+        :return: the copied InternalFrame.
+        """
+        assert column_label in self.column_labels, column_label
+
+        idx = self.column_labels.index(column_label)
+        data_spark_columns = self.data_spark_columns.copy()
+        data_spark_columns[idx] = scol
+        data_dtypes = self.data_dtypes.copy()
+        data_dtypes[idx] = dtype
+        return self.with_new_columns(
+            data_spark_columns, data_dtypes=data_dtypes, keep_order=keep_order
+        )
+
+    def select_column(self, column_label: Tuple) -> "InternalFrame":
+        """
+        Copy the immutable InternalFrame with the specified column.
+
+        :param column_label: the column label to use.
+        :return: the copied InternalFrame.
+        """
+        assert column_label in self.column_labels, column_label
+
+        return self.copy(
+            column_labels=[column_label],
+            data_spark_columns=[self.spark_column_for(column_label)],
+            data_dtypes=[self.dtype_for(column_label)],
+            column_label_names=None,
+        )
+
+    def copy(
+        self,
+        *,
+        spark_frame: Union[spark.DataFrame, _NoValueType] = _NoValue,
+        index_spark_columns: Union[List[spark.Column], _NoValueType] = _NoValue,
+        index_names: Union[List[Optional[Tuple]], _NoValueType] = _NoValue,
+        index_dtypes: Optional[Union[List[Dtype], _NoValueType]] = _NoValue,
+        column_labels: Optional[Union[List[Tuple], _NoValueType]] = _NoValue,
+        data_spark_columns: Optional[Union[List[spark.Column], _NoValueType]] = _NoValue,
+        data_dtypes: Optional[Union[List[Dtype], _NoValueType]] = _NoValue,
+        column_label_names: Optional[Union[List[Optional[Tuple]], _NoValueType]] = _NoValue
+    ) -> "InternalFrame":
+        """ Copy the immutable InternalFrame.
+
+        :param spark_frame: the new Spark DataFrame. If not specified, the original one is used.
+        :param index_spark_columns: the list of Spark Column.
+                                    If not specified, the original ones are used.
+        :param index_names: the index names. If not specified, the original ones are used.
+        :param index_dtypes: the index dtypes. If not specified, the original dtyeps are used.
+        :param column_labels: the new column labels. If not specified, the original ones are used.
+        :param data_spark_columns: the new Spark Columns.
+                                   If not specified, the original ones are used.
+        :param data_dtypes: the data dtypes. If not specified, the original dtyeps are used.
+        :param column_label_names: the new names of the column index levels.
+                                   If not specified, the original ones are used.
+        :return: the copied immutable InternalFrame.
+        """
+        if spark_frame is _NoValue:
+            spark_frame = self.spark_frame
+        if index_spark_columns is _NoValue:
+            index_spark_columns = self.index_spark_columns
+        if index_names is _NoValue:
+            index_names = self.index_names
+        if index_dtypes is _NoValue:
+            index_dtypes = self.index_dtypes
+        if column_labels is _NoValue:
+            column_labels = self.column_labels
+        if data_spark_columns is _NoValue:
+            data_spark_columns = self.data_spark_columns
+        if data_dtypes is _NoValue:
+            data_dtypes = self.data_dtypes
+        if column_label_names is _NoValue:
+            column_label_names = self.column_label_names
+        return InternalFrame(
+            spark_frame=spark_frame,
+            index_spark_columns=index_spark_columns,
+            index_names=index_names,
+            index_dtypes=index_dtypes,
+            column_labels=column_labels,
+            data_spark_columns=data_spark_columns,
+            data_dtypes=data_dtypes,
+            column_label_names=column_label_names,
+        )
+
+    @staticmethod
+    def from_pandas(pdf: pd.DataFrame) -> "InternalFrame":
+        """ Create an immutable DataFrame from pandas DataFrame.
+
+        :param pdf: :class:`pd.DataFrame`
+        :return: the created immutable DataFrame
+        """
+
+        index_names = [
+            name if name is None or isinstance(name, tuple) else (name,) for name in pdf.index.names
+        ]
+
+        columns = pdf.columns
+        if isinstance(columns, pd.MultiIndex):
+            column_labels = columns.tolist()
+        else:
+            column_labels = [(col,) for col in columns]
+        column_label_names = [
+            name if name is None or isinstance(name, tuple) else (name,) for name in columns.names
+        ]
+
+        (
+            pdf,
+            index_columns,
+            index_dtypes,
+            data_columns,
+            data_dtypes,
+        ) = InternalFrame.prepare_pandas_frame(pdf)
+
+        schema = StructType(
+            [
+                StructField(
+                    name, infer_pd_series_spark_type(col, dtype), nullable=bool(col.isnull().any()),
+                )
+                for (name, col), dtype in zip(pdf.iteritems(), index_dtypes + data_dtypes)
+            ]
+        )
+
+        sdf = default_session().createDataFrame(pdf, schema=schema)
+        return InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=[scol_for(sdf, col) for col in index_columns],
+            index_names=index_names,
+            index_dtypes=index_dtypes,
+            column_labels=column_labels,
+            data_spark_columns=[scol_for(sdf, col) for col in data_columns],
+            data_dtypes=data_dtypes,
+            column_label_names=column_label_names,
+        )
+
+    @staticmethod
+    def prepare_pandas_frame(
+        pdf: pd.DataFrame, *, retain_index: bool = True
+    ) -> Tuple[pd.DataFrame, List[str], List[Dtype], List[str], List[Dtype]]:
+        """
+        Prepare pandas DataFrame for creating Spark DataFrame.
+
+        :param pdf: the pandas DataFrame to be prepared.
+        :param retain_index: whether the indices should be retained.
+        :return: the tuple of
+            - the prepared pandas dataFrame
+            - index column names for Spark DataFrame
+            - index dtypes of the given pandas DataFrame
+            - data column names for Spark DataFrame
+            - data dtypes of the given pandas DataFrame
+
+        >>> pdf = pd.DataFrame(
+        ...    {("x", "a"): ['a', 'b', 'c'],
+        ...     ("y", "b"): pd.Categorical(["i", "k", "j"], categories=["i", "j", "k"])},
+        ...    index=[10, 20, 30])
+        >>> prepared, index_columns, index_dtypes, data_columns, data_dtypes = (
+        ...     InternalFrame.prepare_pandas_frame(pdf))
+        >>> prepared
+           __index_level_0__ (x, a)  (y, b)
+        0                 10      a       0
+        1                 20      b       2
+        2                 30      c       1
+        >>> index_columns
+        ['__index_level_0__']
+        >>> index_dtypes
+        [dtype('int64')]
+        >>> data_columns
+        ['(x, a)', '(y, b)']
+        >>> data_dtypes
+        [dtype('O'), CategoricalDtype(categories=['i', 'j', 'k'], ordered=False)]
+        """
+        pdf = pdf.copy()
+
+        data_columns = [name_like_string(col) for col in pdf.columns]
+        pdf.columns = data_columns
+
+        if retain_index:
+            index_nlevels = pdf.index.nlevels
+            index_columns = [SPARK_INDEX_NAME_FORMAT(i) for i in range(index_nlevels)]
+            pdf.index.names = index_columns
+            reset_index = pdf.reset_index()
+        else:
+            index_nlevels = 0
+            index_columns = []
+            reset_index = pdf
+
+        index_dtypes = list(reset_index.dtypes)[:index_nlevels]
+        data_dtypes = list(reset_index.dtypes)[index_nlevels:]
+
+        for name, col in reset_index.iteritems():
+            dt = col.dtype
+            if is_datetime64_dtype(dt) or is_datetime64tz_dtype(dt):
+                continue
+            elif isinstance(dt, CategoricalDtype):
+                col = col.cat.codes
+            reset_index[name] = col.replace({np.nan: None})
+
+        return reset_index, index_columns, index_dtypes, data_columns, data_dtypes
+
+
+def _test():
+    import os
+    import doctest
+    import sys
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.internal
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.internal.__dict__.copy()
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]")
+        .appName("pyspark.pandas.internal tests")
+        .getOrCreate()
+    )
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.internal,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/missing/__init__.py b/python/pyspark/pandas/missing/__init__.py
new file mode 100644
index 0000000000000..bfe1597064acf
--- /dev/null
+++ b/python/pyspark/pandas/missing/__init__.py
@@ -0,0 +1,48 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from pyspark.pandas.exceptions import PandasNotImplementedError
+
+
+def unsupported_function(class_name, method_name, deprecated=False, reason=""):
+    def unsupported_function(*args, **kwargs):
+        raise PandasNotImplementedError(
+            class_name=class_name, method_name=method_name, reason=reason
+        )
+
+    def deprecated_function(*args, **kwargs):
+        raise PandasNotImplementedError(
+            class_name=class_name, method_name=method_name, deprecated=deprecated, reason=reason
+        )
+
+    return deprecated_function if deprecated else unsupported_function
+
+
+def unsupported_property(class_name, property_name, deprecated=False, reason=""):
+    @property
+    def unsupported_property(self):
+        raise PandasNotImplementedError(
+            class_name=class_name, property_name=property_name, reason=reason
+        )
+
+    @property
+    def deprecated_property(self):
+        raise PandasNotImplementedError(
+            class_name=class_name, property_name=property_name, deprecated=deprecated, reason=reason
+        )
+
+    return deprecated_property if deprecated else unsupported_property
diff --git a/python/pyspark/pandas/missing/common.py b/python/pyspark/pandas/missing/common.py
new file mode 100644
index 0000000000000..1ebf28bb0bbf5
--- /dev/null
+++ b/python/pyspark/pandas/missing/common.py
@@ -0,0 +1,59 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+memory_usage = lambda f: f(
+    "memory_usage",
+    reason="Unlike pandas, most DataFrames are not materialized in memory in Spark "
+    "(and pandas-on-Spark), and as a result memory_usage() does not do what you intend it "
+    "to do. Use Spark's web UI to monitor disk and memory usage of your application.",
+)
+
+array = lambda f: f(
+    "array", reason="If you want to collect your data as an NumPy array, use 'to_numpy()' instead."
+)
+
+to_pickle = lambda f: f(
+    "to_pickle",
+    reason="For storage, we encourage you to use Delta or Parquet, instead of Python pickle "
+    "format.",
+)
+
+to_xarray = lambda f: f(
+    "to_xarray",
+    reason="If you want to collect your data as an NumPy array, use 'to_numpy()' instead.",
+)
+
+to_list = lambda f: f(
+    "to_list",
+    reason="If you want to collect your data as an NumPy array, use 'to_numpy()' instead.",
+)
+
+tolist = lambda f: f(
+    "tolist", reason="If you want to collect your data as an NumPy array, use 'to_numpy()' instead."
+)
+
+__iter__ = lambda f: f(
+    "__iter__",
+    reason="If you want to collect your data as an NumPy array, use 'to_numpy()' instead.",
+)
+
+duplicated = lambda f: f(
+    "duplicated",
+    reason="'duplicated' API returns np.ndarray and the data size is too large."
+    "You can just use DataFrame.deduplicated instead",
+)
diff --git a/python/pyspark/pandas/missing/frame.py b/python/pyspark/pandas/missing/frame.py
new file mode 100644
index 0000000000000..3082a2a362bb3
--- /dev/null
+++ b/python/pyspark/pandas/missing/frame.py
@@ -0,0 +1,98 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from distutils.version import LooseVersion
+
+import pandas as pd
+
+from pyspark.pandas.missing import unsupported_function, unsupported_property, common
+
+
+def _unsupported_function(method_name, deprecated=False, reason=""):
+    return unsupported_function(
+        class_name="pd.DataFrame", method_name=method_name, deprecated=deprecated, reason=reason
+    )
+
+
+def _unsupported_property(property_name, deprecated=False, reason=""):
+    return unsupported_property(
+        class_name="pd.DataFrame", property_name=property_name, deprecated=deprecated, reason=reason
+    )
+
+
+class _MissingPandasLikeDataFrame(object):
+
+    # Functions
+    asfreq = _unsupported_function("asfreq")
+    asof = _unsupported_function("asof")
+    boxplot = _unsupported_function("boxplot")
+    combine = _unsupported_function("combine")
+    combine_first = _unsupported_function("combine_first")
+    compare = _unsupported_function("compare")
+    convert_dtypes = _unsupported_function("convert_dtypes")
+    corrwith = _unsupported_function("corrwith")
+    cov = _unsupported_function("cov")
+    ewm = _unsupported_function("ewm")
+    infer_objects = _unsupported_function("infer_objects")
+    interpolate = _unsupported_function("interpolate")
+    lookup = _unsupported_function("lookup")
+    mode = _unsupported_function("mode")
+    reorder_levels = _unsupported_function("reorder_levels")
+    resample = _unsupported_function("resample")
+    set_axis = _unsupported_function("set_axis")
+    slice_shift = _unsupported_function("slice_shift")
+    to_feather = _unsupported_function("to_feather")
+    to_gbq = _unsupported_function("to_gbq")
+    to_hdf = _unsupported_function("to_hdf")
+    to_period = _unsupported_function("to_period")
+    to_sql = _unsupported_function("to_sql")
+    to_stata = _unsupported_function("to_stata")
+    to_timestamp = _unsupported_function("to_timestamp")
+    tshift = _unsupported_function("tshift")
+    tz_convert = _unsupported_function("tz_convert")
+    tz_localize = _unsupported_function("tz_localize")
+
+    # Deprecated functions
+    convert_objects = _unsupported_function("convert_objects", deprecated=True)
+    select = _unsupported_function("select", deprecated=True)
+    to_panel = _unsupported_function("to_panel", deprecated=True)
+    get_values = _unsupported_function("get_values", deprecated=True)
+    compound = _unsupported_function("compound", deprecated=True)
+    reindex_axis = _unsupported_function("reindex_axis", deprecated=True)
+
+    # Functions we won't support.
+    to_pickle = common.to_pickle(_unsupported_function)
+    memory_usage = common.memory_usage(_unsupported_function)
+    to_xarray = common.to_xarray(_unsupported_function)
+
+    if LooseVersion(pd.__version__) < LooseVersion("1.0"):
+        # Deprecated properties
+        blocks = _unsupported_property("blocks", deprecated=True)
+        ftypes = _unsupported_property("ftypes", deprecated=True)
+        is_copy = _unsupported_property("is_copy", deprecated=True)
+        ix = _unsupported_property("ix", deprecated=True)
+
+        # Deprecated functions
+        as_blocks = _unsupported_function("as_blocks", deprecated=True)
+        as_matrix = _unsupported_function("as_matrix", deprecated=True)
+        clip_lower = _unsupported_function("clip_lower", deprecated=True)
+        clip_upper = _unsupported_function("clip_upper", deprecated=True)
+        get_ftype_counts = _unsupported_function("get_ftype_counts", deprecated=True)
+        get_value = _unsupported_function("get_value", deprecated=True)
+        set_value = _unsupported_function("set_value", deprecated=True)
+        to_dense = _unsupported_function("to_dense", deprecated=True)
+        to_sparse = _unsupported_function("to_sparse", deprecated=True)
+        to_msgpack = _unsupported_function("to_msgpack", deprecated=True)
diff --git a/python/pyspark/pandas/missing/groupby.py b/python/pyspark/pandas/missing/groupby.py
new file mode 100644
index 0000000000000..9a1837492a22e
--- /dev/null
+++ b/python/pyspark/pandas/missing/groupby.py
@@ -0,0 +1,103 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from pyspark.pandas.missing import unsupported_function, unsupported_property
+
+
+def _unsupported_function(method_name, deprecated=False, reason=""):
+    return unsupported_function(
+        class_name="pd.groupby.GroupBy",
+        method_name=method_name,
+        deprecated=deprecated,
+        reason=reason,
+    )
+
+
+def _unsupported_property(property_name, deprecated=False, reason=""):
+    return unsupported_property(
+        class_name="pd.groupby.GroupBy",
+        property_name=property_name,
+        deprecated=deprecated,
+        reason=reason,
+    )
+
+
+class MissingPandasLikeDataFrameGroupBy(object):
+
+    # Properties
+    corr = _unsupported_property("corr")
+    corrwith = _unsupported_property("corrwith")
+    cov = _unsupported_property("cov")
+    dtypes = _unsupported_property("dtypes")
+    groups = _unsupported_property("groups")
+    hist = _unsupported_property("hist")
+    indices = _unsupported_property("indices")
+    mad = _unsupported_property("mad")
+    ngroups = _unsupported_property("ngroups")
+    plot = _unsupported_property("plot")
+    quantile = _unsupported_property("quantile")
+    skew = _unsupported_property("skew")
+    tshift = _unsupported_property("tshift")
+
+    # Deprecated properties
+    take = _unsupported_property("take", deprecated=True)
+
+    # Functions
+    boxplot = _unsupported_function("boxplot")
+    ngroup = _unsupported_function("ngroup")
+    nth = _unsupported_function("nth")
+    ohlc = _unsupported_function("ohlc")
+    pct_change = _unsupported_function("pct_change")
+    pipe = _unsupported_function("pipe")
+    prod = _unsupported_function("prod")
+    resample = _unsupported_function("resample")
+    sem = _unsupported_function("sem")
+
+
+class MissingPandasLikeSeriesGroupBy(object):
+
+    # Properties
+    corr = _unsupported_property("corr")
+    cov = _unsupported_property("cov")
+    dtype = _unsupported_property("dtype")
+    groups = _unsupported_property("groups")
+    hist = _unsupported_property("hist")
+    indices = _unsupported_property("indices")
+    is_monotonic_decreasing = _unsupported_property("is_monotonic_decreasing")
+    is_monotonic_increasing = _unsupported_property("is_monotonic_increasing")
+    mad = _unsupported_property("mad")
+    ngroups = _unsupported_property("ngroups")
+    plot = _unsupported_property("plot")
+    quantile = _unsupported_property("quantile")
+    skew = _unsupported_property("skew")
+    tshift = _unsupported_property("tshift")
+
+    # Deprecated properties
+    take = _unsupported_property("take", deprecated=True)
+
+    # Functions
+    agg = _unsupported_function("agg")
+    aggregate = _unsupported_function("aggregate")
+    describe = _unsupported_function("describe")
+    ngroup = _unsupported_function("ngroup")
+    nth = _unsupported_function("nth")
+    ohlc = _unsupported_function("ohlc")
+    pct_change = _unsupported_function("pct_change")
+    pipe = _unsupported_function("pipe")
+    prod = _unsupported_function("prod")
+    resample = _unsupported_function("resample")
+    sem = _unsupported_function("sem")
diff --git a/python/pyspark/pandas/missing/indexes.py b/python/pyspark/pandas/missing/indexes.py
new file mode 100644
index 0000000000000..0b8a975fa8e0b
--- /dev/null
+++ b/python/pyspark/pandas/missing/indexes.py
@@ -0,0 +1,218 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from distutils.version import LooseVersion
+
+import pandas as pd
+
+from pyspark.pandas.missing import unsupported_function, unsupported_property, common
+
+
+def _unsupported_function(method_name, deprecated=False, reason="", cls="Index"):
+    return unsupported_function(
+        class_name="pd.{}".format(cls),
+        method_name=method_name,
+        deprecated=deprecated,
+        reason=reason,
+    )
+
+
+def _unsupported_property(property_name, deprecated=False, reason="", cls="Index"):
+    return unsupported_property(
+        class_name="pd.{}".format(cls),
+        property_name=property_name,
+        deprecated=deprecated,
+        reason=reason,
+    )
+
+
+class MissingPandasLikeIndex(object):
+
+    # Properties
+    nbytes = _unsupported_property("nbytes")
+
+    # Functions
+    argsort = _unsupported_function("argsort")
+    asof_locs = _unsupported_function("asof_locs")
+    format = _unsupported_function("format")
+    get_indexer = _unsupported_function("get_indexer")
+    get_indexer_for = _unsupported_function("get_indexer_for")
+    get_indexer_non_unique = _unsupported_function("get_indexer_non_unique")
+    get_loc = _unsupported_function("get_loc")
+    get_slice_bound = _unsupported_function("get_slice_bound")
+    get_value = _unsupported_function("get_value")
+    groupby = _unsupported_function("groupby")
+    is_ = _unsupported_function("is_")
+    is_lexsorted_for_tuple = _unsupported_function("is_lexsorted_for_tuple")
+    join = _unsupported_function("join")
+    map = _unsupported_function("map")
+    putmask = _unsupported_function("putmask")
+    ravel = _unsupported_function("ravel")
+    reindex = _unsupported_function("reindex")
+    searchsorted = _unsupported_function("searchsorted")
+    slice_indexer = _unsupported_function("slice_indexer")
+    slice_locs = _unsupported_function("slice_locs")
+    sortlevel = _unsupported_function("sortlevel")
+    to_flat_index = _unsupported_function("to_flat_index")
+    to_native_types = _unsupported_function("to_native_types")
+    where = _unsupported_function("where")
+
+    # Deprecated functions
+    is_mixed = _unsupported_function("is_mixed")
+    get_values = _unsupported_function("get_values", deprecated=True)
+    set_value = _unsupported_function("set_value")
+
+    # Properties we won't support.
+    array = common.array(_unsupported_property)
+    duplicated = common.duplicated(_unsupported_property)
+
+    # Functions we won't support.
+    memory_usage = common.memory_usage(_unsupported_function)
+    __iter__ = common.__iter__(_unsupported_function)
+
+    if LooseVersion(pd.__version__) < LooseVersion("1.0"):
+        # Deprecated properties
+        strides = _unsupported_property("strides", deprecated=True)
+        data = _unsupported_property("data", deprecated=True)
+        itemsize = _unsupported_property("itemsize", deprecated=True)
+        base = _unsupported_property("base", deprecated=True)
+        flags = _unsupported_property("flags", deprecated=True)
+
+        # Deprecated functions
+        get_duplicates = _unsupported_function("get_duplicates", deprecated=True)
+        summary = _unsupported_function("summary", deprecated=True)
+        contains = _unsupported_function("contains", deprecated=True)
+
+
+class MissingPandasLikeDatetimeIndex(MissingPandasLikeIndex):
+
+    # Properties
+    nanosecond = _unsupported_property("nanosecond", cls="DatetimeIndex")
+    date = _unsupported_property("date", cls="DatetimeIndex")
+    time = _unsupported_property("time", cls="DatetimeIndex")
+    timetz = _unsupported_property("timetz", cls="DatetimeIndex")
+    tz = _unsupported_property("tz", cls="DatetimeIndex")
+    freq = _unsupported_property("freq", cls="DatetimeIndex")
+    freqstr = _unsupported_property("freqstr", cls="DatetimeIndex")
+    inferred_freq = _unsupported_property("inferred_freq", cls="DatetimeIndex")
+
+    # Functions
+    snap = _unsupported_function("snap", cls="DatetimeIndex")
+    tz_convert = _unsupported_function("tz_convert", cls="DatetimeIndex")
+    tz_localize = _unsupported_function("tz_localize", cls="DatetimeIndex")
+    to_period = _unsupported_function("to_period", cls="DatetimeIndex")
+    to_perioddelta = _unsupported_function("to_perioddelta", cls="DatetimeIndex")
+    to_pydatetime = _unsupported_function("to_pydatetime", cls="DatetimeIndex")
+    mean = _unsupported_function("mean", cls="DatetimeIndex")
+    std = _unsupported_function("std", cls="DatetimeIndex")
+
+
+class MissingPandasLikeCategoricalIndex(MissingPandasLikeIndex):
+
+    # Functions
+    rename_categories = _unsupported_function("rename_categories", cls="CategoricalIndex")
+    reorder_categories = _unsupported_function("reorder_categories", cls="CategoricalIndex")
+    add_categories = _unsupported_function("add_categories", cls="CategoricalIndex")
+    remove_categories = _unsupported_function("remove_categories", cls="CategoricalIndex")
+    remove_unused_categories = _unsupported_function(
+        "remove_unused_categories", cls="CategoricalIndex"
+    )
+    set_categories = _unsupported_function("set_categories", cls="CategoricalIndex")
+    as_ordered = _unsupported_function("as_ordered", cls="CategoricalIndex")
+    as_unordered = _unsupported_function("as_unordered", cls="CategoricalIndex")
+    map = _unsupported_function("map", cls="CategoricalIndex")
+
+
+class MissingPandasLikeMultiIndex(object):
+
+    # Deprecated properties
+    strides = _unsupported_property("strides", deprecated=True)
+    data = _unsupported_property("data", deprecated=True)
+    itemsize = _unsupported_property("itemsize", deprecated=True)
+
+    # Functions
+    argsort = _unsupported_function("argsort")
+    asof_locs = _unsupported_function("asof_locs")
+    equal_levels = _unsupported_function("equal_levels")
+    factorize = _unsupported_function("factorize")
+    format = _unsupported_function("format")
+    get_indexer = _unsupported_function("get_indexer")
+    get_indexer_for = _unsupported_function("get_indexer_for")
+    get_indexer_non_unique = _unsupported_function("get_indexer_non_unique")
+    get_loc = _unsupported_function("get_loc")
+    get_loc_level = _unsupported_function("get_loc_level")
+    get_locs = _unsupported_function("get_locs")
+    get_slice_bound = _unsupported_function("get_slice_bound")
+    get_value = _unsupported_function("get_value")
+    groupby = _unsupported_function("groupby")
+    is_ = _unsupported_function("is_")
+    is_lexsorted = _unsupported_function("is_lexsorted")
+    is_lexsorted_for_tuple = _unsupported_function("is_lexsorted_for_tuple")
+    join = _unsupported_function("join")
+    map = _unsupported_function("map")
+    putmask = _unsupported_function("putmask")
+    ravel = _unsupported_function("ravel")
+    reindex = _unsupported_function("reindex")
+    remove_unused_levels = _unsupported_function("remove_unused_levels")
+    reorder_levels = _unsupported_function("reorder_levels")
+    searchsorted = _unsupported_function("searchsorted")
+    set_codes = _unsupported_function("set_codes")
+    set_levels = _unsupported_function("set_levels")
+    slice_indexer = _unsupported_function("slice_indexer")
+    slice_locs = _unsupported_function("slice_locs")
+    sortlevel = _unsupported_function("sortlevel")
+    to_flat_index = _unsupported_function("to_flat_index")
+    to_native_types = _unsupported_function("to_native_types")
+    truncate = _unsupported_function("truncate")
+    where = _unsupported_function("where")
+
+    # Deprecated functions
+    is_mixed = _unsupported_function("is_mixed")
+    get_duplicates = _unsupported_function("get_duplicates", deprecated=True)
+    get_values = _unsupported_function("get_values", deprecated=True)
+    set_value = _unsupported_function("set_value", deprecated=True)
+
+    # Functions we won't support.
+    array = common.array(_unsupported_property)
+    duplicated = common.duplicated(_unsupported_property)
+    codes = _unsupported_property(
+        "codes",
+        reason="'codes' requires to collect all data into the driver which is against the "
+        "design principle of pandas-on-Spark. Alternatively, you could call 'to_pandas()' and"
+        " use 'codes' property in pandas.",
+    )
+    levels = _unsupported_property(
+        "levels",
+        reason="'levels' requires to collect all data into the driver which is against the "
+        "design principle of pandas-on-Spark. Alternatively, you could call 'to_pandas()' and"
+        " use 'levels' property in pandas.",
+    )
+    __iter__ = common.__iter__(_unsupported_function)
+
+    # Properties we won't support.
+    memory_usage = common.memory_usage(_unsupported_function)
+
+    if LooseVersion(pd.__version__) < LooseVersion("1.0"):
+        # Deprecated properties
+        base = _unsupported_property("base", deprecated=True)
+        labels = _unsupported_property("labels", deprecated=True)
+        flags = _unsupported_property("flags", deprecated=True)
+
+        # Deprecated functions
+        set_labels = _unsupported_function("set_labels")
+        summary = _unsupported_function("summary", deprecated=True)
+        to_hierarchical = _unsupported_function("to_hierarchical", deprecated=True)
+        contains = _unsupported_function("contains", deprecated=True)
diff --git a/python/pyspark/pandas/missing/series.py b/python/pyspark/pandas/missing/series.py
new file mode 100644
index 0000000000000..ef3f38b2277ad
--- /dev/null
+++ b/python/pyspark/pandas/missing/series.py
@@ -0,0 +1,125 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from distutils.version import LooseVersion
+
+import pandas as pd
+
+from pyspark.pandas.missing import unsupported_function, unsupported_property, common
+
+
+def _unsupported_function(method_name, deprecated=False, reason=""):
+    return unsupported_function(
+        class_name="pd.Series", method_name=method_name, deprecated=deprecated, reason=reason
+    )
+
+
+def _unsupported_property(property_name, deprecated=False, reason=""):
+    return unsupported_property(
+        class_name="pd.Series", property_name=property_name, deprecated=deprecated, reason=reason
+    )
+
+
+class MissingPandasLikeSeries(object):
+
+    # Functions
+    asfreq = _unsupported_function("asfreq")
+    autocorr = _unsupported_function("autocorr")
+    combine = _unsupported_function("combine")
+    convert_dtypes = _unsupported_function("convert_dtypes")
+    cov = _unsupported_function("cov")
+    ewm = _unsupported_function("ewm")
+    infer_objects = _unsupported_function("infer_objects")
+    interpolate = _unsupported_function("interpolate")
+    reorder_levels = _unsupported_function("reorder_levels")
+    resample = _unsupported_function("resample")
+    searchsorted = _unsupported_function("searchsorted")
+    set_axis = _unsupported_function("set_axis")
+    slice_shift = _unsupported_function("slice_shift")
+    to_hdf = _unsupported_function("to_hdf")
+    to_period = _unsupported_function("to_period")
+    to_sql = _unsupported_function("to_sql")
+    to_timestamp = _unsupported_function("to_timestamp")
+    tshift = _unsupported_function("tshift")
+    tz_convert = _unsupported_function("tz_convert")
+    tz_localize = _unsupported_function("tz_localize")
+    view = _unsupported_function("view")
+
+    # Deprecated functions
+    convert_objects = _unsupported_function("convert_objects", deprecated=True)
+    nonzero = _unsupported_function("nonzero", deprecated=True)
+    reindex_axis = _unsupported_function("reindex_axis", deprecated=True)
+    select = _unsupported_function("select", deprecated=True)
+    get_values = _unsupported_function("get_values", deprecated=True)
+
+    # Properties we won't support.
+    array = common.array(_unsupported_property)
+    duplicated = common.duplicated(_unsupported_property)
+    nbytes = _unsupported_property(
+        "nbytes",
+        reason="'nbytes' requires to compute whole dataset. You can calculate manually it, "
+        "with its 'itemsize', by explicitly executing its count. Use Spark's web UI "
+        "to monitor disk and memory usage of your application in general.",
+    )
+
+    # Functions we won't support.
+    memory_usage = common.memory_usage(_unsupported_function)
+    to_pickle = common.to_pickle(_unsupported_function)
+    to_xarray = common.to_xarray(_unsupported_function)
+    __iter__ = common.__iter__(_unsupported_function)
+    ravel = _unsupported_function(
+        "ravel",
+        reason="If you want to collect your flattened underlying data as an NumPy array, "
+        "use 'to_numpy().ravel()' instead.",
+    )
+
+    if LooseVersion(pd.__version__) < LooseVersion("1.0"):
+        # Deprecated properties
+        blocks = _unsupported_property("blocks", deprecated=True)
+        ftypes = _unsupported_property("ftypes", deprecated=True)
+        ftype = _unsupported_property("ftype", deprecated=True)
+        is_copy = _unsupported_property("is_copy", deprecated=True)
+        ix = _unsupported_property("ix", deprecated=True)
+        asobject = _unsupported_property("asobject", deprecated=True)
+        strides = _unsupported_property("strides", deprecated=True)
+        imag = _unsupported_property("imag", deprecated=True)
+        itemsize = _unsupported_property("itemsize", deprecated=True)
+        data = _unsupported_property("data", deprecated=True)
+        base = _unsupported_property("base", deprecated=True)
+        flags = _unsupported_property("flags", deprecated=True)
+
+        # Deprecated functions
+        as_blocks = _unsupported_function("as_blocks", deprecated=True)
+        as_matrix = _unsupported_function("as_matrix", deprecated=True)
+        clip_lower = _unsupported_function("clip_lower", deprecated=True)
+        clip_upper = _unsupported_function("clip_upper", deprecated=True)
+        compress = _unsupported_function("compress", deprecated=True)
+        get_ftype_counts = _unsupported_function("get_ftype_counts", deprecated=True)
+        get_value = _unsupported_function("get_value", deprecated=True)
+        set_value = _unsupported_function("set_value", deprecated=True)
+        valid = _unsupported_function("valid", deprecated=True)
+        to_dense = _unsupported_function("to_dense", deprecated=True)
+        to_sparse = _unsupported_function("to_sparse", deprecated=True)
+        to_msgpack = _unsupported_function("to_msgpack", deprecated=True)
+        compound = _unsupported_function("compound", deprecated=True)
+        put = _unsupported_function("put", deprecated=True)
+        ptp = _unsupported_function("ptp", deprecated=True)
+
+        # Functions we won't support.
+        real = _unsupported_property(
+            "real",
+            reason="If you want to collect your data as an NumPy array, use 'to_numpy()' instead.",
+        )
diff --git a/python/pyspark/pandas/missing/window.py b/python/pyspark/pandas/missing/window.py
new file mode 100644
index 0000000000000..a29887e9e1bd4
--- /dev/null
+++ b/python/pyspark/pandas/missing/window.py
@@ -0,0 +1,126 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from pyspark.pandas.missing import unsupported_function, unsupported_property
+
+
+def _unsupported_function_expanding(method_name, deprecated=False, reason=""):
+    return unsupported_function(
+        class_name="pandas.core.window.Expanding",
+        method_name=method_name,
+        deprecated=deprecated,
+        reason=reason,
+    )
+
+
+def _unsupported_property_expanding(property_name, deprecated=False, reason=""):
+    return unsupported_property(
+        class_name="pandas.core.window.Expanding",
+        property_name=property_name,
+        deprecated=deprecated,
+        reason=reason,
+    )
+
+
+def _unsupported_function_rolling(method_name, deprecated=False, reason=""):
+    return unsupported_function(
+        class_name="pandas.core.window.Rolling",
+        method_name=method_name,
+        deprecated=deprecated,
+        reason=reason,
+    )
+
+
+def _unsupported_property_rolling(property_name, deprecated=False, reason=""):
+    return unsupported_property(
+        class_name="pandas.core.window.Rolling",
+        property_name=property_name,
+        deprecated=deprecated,
+        reason=reason,
+    )
+
+
+class MissingPandasLikeExpanding(object):
+    agg = _unsupported_function_expanding("agg")
+    aggregate = _unsupported_function_expanding("aggregate")
+    apply = _unsupported_function_expanding("apply")
+    corr = _unsupported_function_expanding("corr")
+    cov = _unsupported_function_expanding("cov")
+    kurt = _unsupported_function_expanding("kurt")
+    median = _unsupported_function_expanding("median")
+    quantile = _unsupported_function_expanding("quantile")
+    skew = _unsupported_function_expanding("skew")
+    validate = _unsupported_function_expanding("validate")
+
+    exclusions = _unsupported_property_expanding("exclusions")
+    is_datetimelike = _unsupported_property_expanding("is_datetimelike")
+    is_freq_type = _unsupported_property_expanding("is_freq_type")
+    ndim = _unsupported_property_expanding("ndim")
+
+
+class MissingPandasLikeRolling(object):
+    agg = _unsupported_function_rolling("agg")
+    aggregate = _unsupported_function_rolling("aggregate")
+    apply = _unsupported_function_rolling("apply")
+    corr = _unsupported_function_rolling("corr")
+    cov = _unsupported_function_rolling("cov")
+    kurt = _unsupported_function_rolling("kurt")
+    median = _unsupported_function_rolling("median")
+    quantile = _unsupported_function_rolling("quantile")
+    skew = _unsupported_function_rolling("skew")
+    validate = _unsupported_function_rolling("validate")
+
+    exclusions = _unsupported_property_rolling("exclusions")
+    is_datetimelike = _unsupported_property_rolling("is_datetimelike")
+    is_freq_type = _unsupported_property_rolling("is_freq_type")
+    ndim = _unsupported_property_rolling("ndim")
+
+
+class MissingPandasLikeExpandingGroupby(object):
+    agg = _unsupported_function_expanding("agg")
+    aggregate = _unsupported_function_expanding("aggregate")
+    apply = _unsupported_function_expanding("apply")
+    corr = _unsupported_function_expanding("corr")
+    cov = _unsupported_function_expanding("cov")
+    kurt = _unsupported_function_expanding("kurt")
+    median = _unsupported_function_expanding("median")
+    quantile = _unsupported_function_expanding("quantile")
+    skew = _unsupported_function_expanding("skew")
+    validate = _unsupported_function_expanding("validate")
+
+    exclusions = _unsupported_property_expanding("exclusions")
+    is_datetimelike = _unsupported_property_expanding("is_datetimelike")
+    is_freq_type = _unsupported_property_expanding("is_freq_type")
+    ndim = _unsupported_property_expanding("ndim")
+
+
+class MissingPandasLikeRollingGroupby(object):
+    agg = _unsupported_function_rolling("agg")
+    aggregate = _unsupported_function_rolling("aggregate")
+    apply = _unsupported_function_rolling("apply")
+    corr = _unsupported_function_rolling("corr")
+    cov = _unsupported_function_rolling("cov")
+    kurt = _unsupported_function_rolling("kurt")
+    median = _unsupported_function_rolling("median")
+    quantile = _unsupported_function_rolling("quantile")
+    skew = _unsupported_function_rolling("skew")
+    validate = _unsupported_function_rolling("validate")
+
+    exclusions = _unsupported_property_rolling("exclusions")
+    is_datetimelike = _unsupported_property_rolling("is_datetimelike")
+    is_freq_type = _unsupported_property_rolling("is_freq_type")
+    ndim = _unsupported_property_rolling("ndim")
diff --git a/python/pyspark/pandas/ml.py b/python/pyspark/pandas/ml.py
new file mode 100644
index 0000000000000..af43e802bed51
--- /dev/null
+++ b/python/pyspark/pandas/ml.py
@@ -0,0 +1,115 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from typing import List, Tuple, TYPE_CHECKING
+
+import numpy as np
+import pandas as pd
+import pyspark
+
+from pyspark.ml.feature import VectorAssembler
+from pyspark.ml.stat import Correlation
+
+from pyspark.pandas.utils import column_labels_level
+
+if TYPE_CHECKING:
+    import pyspark.pandas as ps  # noqa: F401 (SPARK-34943)
+
+
+CORRELATION_OUTPUT_COLUMN = "__correlation_output__"
+
+
+def corr(kdf: "ps.DataFrame", method: str = "pearson") -> pd.DataFrame:
+    """
+    The correlation matrix of all the numerical columns of this dataframe.
+
+    Only accepts scalar numerical values for now.
+
+    :param kdf: the pandas-on-Spark dataframe.
+    :param method: {'pearson', 'spearman'}
+                   * pearson : standard correlation coefficient
+                   * spearman : Spearman rank correlation
+    :return: :class:`pandas.DataFrame`
+
+    >>> ps.DataFrame({'A': [0, 1], 'B': [1, 0], 'C': ['x', 'y']}).corr()
+         A    B
+    A  1.0 -1.0
+    B -1.0  1.0
+    """
+    assert method in ("pearson", "spearman")
+    ndf, column_labels = to_numeric_df(kdf)
+    corr = Correlation.corr(ndf, CORRELATION_OUTPUT_COLUMN, method)
+    pcorr = corr.toPandas()
+    arr = pcorr.iloc[0, 0].toArray()
+    if column_labels_level(column_labels) > 1:
+        idx = pd.MultiIndex.from_tuples(column_labels)
+    else:
+        idx = pd.Index([label[0] for label in column_labels])
+    return pd.DataFrame(arr, columns=idx, index=idx)
+
+
+def to_numeric_df(kdf: "ps.DataFrame") -> Tuple[pyspark.sql.DataFrame, List[Tuple]]:
+    """
+    Takes a dataframe and turns it into a dataframe containing a single numerical
+    vector of doubles. This dataframe has a single field called '_1'.
+
+    TODO: index is not preserved currently
+    :param kdf: the pandas-on-Spark dataframe.
+    :return: a pair of dataframe, list of strings (the name of the columns
+             that were converted to numerical types)
+
+    >>> to_numeric_df(ps.DataFrame({'A': [0, 1], 'B': [1, 0], 'C': ['x', 'y']}))
+    (DataFrame[__correlation_output__: vector], [('A',), ('B',)])
+    """
+    # TODO, it should be more robust.
+    accepted_types = {
+        np.dtype(dt)
+        for dt in [np.int8, np.int16, np.int32, np.int64, np.float32, np.float64, np.bool_]
+    }
+    numeric_column_labels = [
+        label for label in kdf._internal.column_labels if kdf[label].dtype in accepted_types
+    ]
+    numeric_df = kdf._internal.spark_frame.select(
+        *[kdf._internal.spark_column_for(idx) for idx in numeric_column_labels]
+    )
+    va = VectorAssembler(inputCols=numeric_df.columns, outputCol=CORRELATION_OUTPUT_COLUMN)
+    v = va.transform(numeric_df).select(CORRELATION_OUTPUT_COLUMN)
+    return v, numeric_column_labels
+
+
+def _test():
+    import os
+    import doctest
+    import sys
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.ml
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.ml.__dict__.copy()
+    globs["ps"] = pyspark.pandas
+    spark = SparkSession.builder.master("local[4]").appName("pyspark.pandas.ml tests").getOrCreate()
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.ml, globs=globs, optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE
+    )
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/mlflow.py b/python/pyspark/pandas/mlflow.py
new file mode 100644
index 0000000000000..5b81319a580ce
--- /dev/null
+++ b/python/pyspark/pandas/mlflow.py
@@ -0,0 +1,231 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+MLflow-related functions to load models and apply them to pandas-on-Spark dataframes.
+"""
+from pyspark.sql.types import DataType
+import pandas as pd
+import numpy as np
+from typing import Any
+
+from pyspark.pandas.utils import lazy_property, default_session
+from pyspark.pandas.frame import DataFrame
+from pyspark.pandas.series import first_series
+from pyspark.pandas.typedef import as_spark_type
+
+__all__ = ["PythonModelWrapper", "load_model"]
+
+
+class PythonModelWrapper(object):
+    """
+    A wrapper around MLflow's Python object model.
+
+    This wrapper acts as a predictor on pandas-on-Spark
+
+    """
+
+    def __init__(self, model_uri, return_type_hint):
+        self._model_uri = model_uri  # type: str
+        self._return_type_hint = return_type_hint
+
+    @lazy_property
+    def _return_type(self) -> DataType:
+        hint = self._return_type_hint
+        # The logic is simple for now, because it corresponds to the default
+        # case: continuous predictions
+        # TODO: do something smarter, for example when there is a sklearn.Classifier (it should
+        # return an integer or a categorical)
+        # We can do the same for pytorch/tensorflow/keras models by looking at the output types.
+        # However, this is probably better done in mlflow than here.
+        if hint == "infer" or not hint:
+            hint = np.float64
+        return as_spark_type(hint)
+
+    @lazy_property
+    def _model(self) -> Any:
+        """
+        The return object has to follow the API of mlflow.pyfunc.PythonModel.
+        """
+        from mlflow import pyfunc
+
+        return pyfunc.load_model(model_uri=self._model_uri)
+
+    @lazy_property
+    def _model_udf(self):
+        from mlflow import pyfunc
+
+        spark = default_session()
+        return pyfunc.spark_udf(spark, model_uri=self._model_uri, result_type=self._return_type)
+
+    def __str__(self):
+        return "PythonModelWrapper({})".format(str(self._model))
+
+    def __repr__(self):
+        return "PythonModelWrapper({})".format(repr(self._model))
+
+    def predict(self, data):
+        """
+        Returns a prediction on the data.
+
+        If the data is a pandas-on-Spark DataFrame, the return is a pandas-on-Spark Series.
+
+        If the data is a pandas Dataframe, the return is the expected output of the underlying
+        pyfunc object (typically a pandas Series or a numpy array).
+        """
+        if isinstance(data, pd.DataFrame):
+            return self._model.predict(data)
+        if isinstance(data, DataFrame):
+            return_col = self._model_udf(*data._internal.data_spark_columns)
+            # TODO: the columns should be named according to the mlflow spec
+            # However, this is only possible with spark >= 3.0
+            # s = F.struct(*data.columns)
+            # return_col = self._model_udf(s)
+            column_labels = [
+                (col,) for col in data._internal.spark_frame.select(return_col).columns
+            ]
+            internal = data._internal.copy(
+                column_labels=column_labels, data_spark_columns=[return_col], data_dtypes=None
+            )
+            return first_series(DataFrame(internal))
+
+
+def load_model(model_uri, predict_type="infer") -> PythonModelWrapper:
+    """
+    Loads an MLflow model into an wrapper that can be used both for pandas and pandas-on-Spark
+    DataFrame.
+
+    Parameters
+    ----------
+    model_uri : str
+        URI pointing to the model. See MLflow documentation for more details.
+    predict_type : a python basic type, a numpy basic type, a Spark type or 'infer'.
+       This is the return type that is expected when calling the predict function of the model.
+       If 'infer' is specified, the wrapper will attempt to determine automatically the return type
+       based on the model type.
+
+    Returns
+    -------
+    PythonModelWrapper
+        A wrapper around MLflow PythonModel objects. This wrapper is expected to adhere to the
+        interface of mlflow.pyfunc.PythonModel.
+
+    Examples
+    --------
+    Here is a full example that creates a model with scikit-learn and saves the model with
+     MLflow. The model is then loaded as a predictor that can be applied on a pandas-on-Spark
+     Dataframe.
+
+    We first initialize our MLflow environment:
+
+    >>> from mlflow.tracking import MlflowClient, set_tracking_uri
+    >>> import mlflow.sklearn
+    >>> from tempfile import mkdtemp
+    >>> d = mkdtemp("pandas_on_spark_mlflow")
+    >>> set_tracking_uri("file:%s"%d)
+    >>> client = MlflowClient()
+    >>> exp = mlflow.create_experiment("my_experiment")
+    >>> mlflow.set_experiment("my_experiment")
+
+    We aim at learning this numerical function using a simple linear regressor.
+
+    >>> from sklearn.linear_model import LinearRegression
+    >>> train = pd.DataFrame({"x1": np.arange(8), "x2": np.arange(8)**2,
+    ...                       "y": np.log(2 + np.arange(8))})
+    >>> train_x = train[["x1", "x2"]]
+    >>> train_y = train[["y"]]
+    >>> with mlflow.start_run():
+    ...     lr = LinearRegression()
+    ...     lr.fit(train_x, train_y)
+    ...     mlflow.sklearn.log_model(lr, "model")
+    LinearRegression(...)
+
+    Now that our model is logged using MLflow, we load it back and apply it on a pandas-on-Spark
+    dataframe:
+
+    >>> from pyspark.pandas.mlflow import load_model
+    >>> run_info = client.list_run_infos(exp)[-1]
+    >>> model = load_model("runs:/{run_id}/model".format(run_id=run_info.run_uuid))
+    >>> prediction_df = ps.DataFrame({"x1": [2.0], "x2": [4.0]})
+    >>> prediction_df["prediction"] = model.predict(prediction_df)
+    >>> prediction_df
+        x1   x2  prediction
+    0  2.0  4.0    1.355551
+
+    The model also works on pandas DataFrames as expected:
+
+    >>> model.predict(prediction_df[["x1", "x2"]].to_pandas())
+    array([[1.35555142]])
+
+    Notes
+    -----
+    Currently, the model prediction can only be merged back with the existing dataframe.
+    Other columns have to be manually joined.
+    For example, this code will not work:
+
+    >>> df = ps.DataFrame({"x1": [2.0], "x2": [3.0], "z": [-1]})
+    >>> features = df[["x1", "x2"]]
+    >>> y = model.predict(features)
+    >>> # Works:
+    >>> features["y"] = y   # doctest: +SKIP
+    >>> # Will fail with a message about dataframes not aligned.
+    >>> df["y"] = y   # doctest: +SKIP
+
+    A current workaround is to use the .merge() function, using the feature values
+    as merging keys.
+
+    >>> features['y'] = y
+    >>> everything = df.merge(features, on=['x1', 'x2'])
+    >>> everything
+        x1   x2  z         y
+    0  2.0  3.0 -1  1.376932
+    """
+    return PythonModelWrapper(model_uri, predict_type)
+
+
+def _test():
+    import os
+    import doctest
+    import sys
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.mlflow
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.mlflow.__dict__.copy()
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]").appName("pyspark.pandas.mlflow tests").getOrCreate()
+    )
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.mlflow,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    try:
+        import mlflow  # noqa: F401
+        import sklearn  # noqa: F401
+
+        _test()
+    except ImportError:
+        pass
diff --git a/python/pyspark/pandas/namespace.py b/python/pyspark/pandas/namespace.py
new file mode 100644
index 0000000000000..adfe718038601
--- /dev/null
+++ b/python/pyspark/pandas/namespace.py
@@ -0,0 +1,2912 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Wrappers around spark that correspond to common pandas functions.
+"""
+from typing import Any, Optional, Union, List, Tuple, Sized, cast
+from collections import OrderedDict
+from collections.abc import Iterable
+from distutils.version import LooseVersion
+from functools import reduce
+from io import BytesIO
+import json
+
+import numpy as np
+import pandas as pd
+from pandas.api.types import is_datetime64_dtype, is_datetime64tz_dtype, is_list_like
+import pyarrow as pa
+import pyarrow.parquet as pq
+from pyspark import sql as spark
+from pyspark.sql import functions as F
+from pyspark.sql.functions import pandas_udf, PandasUDFType
+from pyspark.sql.types import (
+    ByteType,
+    ShortType,
+    IntegerType,
+    LongType,
+    FloatType,
+    DoubleType,
+    BooleanType,
+    TimestampType,
+    DecimalType,
+    StringType,
+    DateType,
+    StructType,
+)
+
+from pyspark import pandas as ps  # noqa: F401
+from pyspark.pandas.base import IndexOpsMixin
+from pyspark.pandas.utils import (
+    align_diff_frames,
+    default_session,
+    is_name_like_tuple,
+    name_like_string,
+    same_anchor,
+    scol_for,
+    validate_axis,
+)
+from pyspark.pandas.frame import DataFrame, _reduce_spark_multi
+from pyspark.pandas.internal import (
+    InternalFrame,
+    DEFAULT_SERIES_NAME,
+    HIDDEN_COLUMNS,
+)
+from pyspark.pandas.series import Series, first_series
+from pyspark.pandas.spark.utils import as_nullable_spark_type, force_decimal_precision_scale
+from pyspark.pandas.indexes import Index, DatetimeIndex
+
+
+__all__ = [
+    "from_pandas",
+    "range",
+    "read_csv",
+    "read_delta",
+    "read_table",
+    "read_spark_io",
+    "read_parquet",
+    "read_clipboard",
+    "read_excel",
+    "read_html",
+    "to_datetime",
+    "date_range",
+    "get_dummies",
+    "concat",
+    "melt",
+    "isna",
+    "isnull",
+    "notna",
+    "notnull",
+    "read_sql_table",
+    "read_sql_query",
+    "read_sql",
+    "read_json",
+    "merge",
+    "to_numeric",
+    "broadcast",
+    "read_orc",
+]
+
+
+def from_pandas(pobj: Union[pd.DataFrame, pd.Series, pd.Index]) -> Union[Series, DataFrame, Index]:
+    """Create a pandas-on-Spark DataFrame, Series or Index from a pandas DataFrame, Series or Index.
+
+    This is similar to Spark's `SparkSession.createDataFrame()` with pandas DataFrame,
+    but this also works with pandas Series and picks the index.
+
+    Parameters
+    ----------
+    pobj : pandas.DataFrame or pandas.Series
+        pandas DataFrame or Series to read.
+
+    Returns
+    -------
+    Series or DataFrame
+        If a pandas Series is passed in, this function returns a pandas-on-Spark Series.
+        If a pandas DataFrame is passed in, this function returns a pandas-on-Spark DataFrame.
+    """
+    if isinstance(pobj, pd.Series):
+        return Series(pobj)
+    elif isinstance(pobj, pd.DataFrame):
+        return DataFrame(pobj)
+    elif isinstance(pobj, pd.Index):
+        return DataFrame(pd.DataFrame(index=pobj)).index
+    else:
+        raise TypeError("Unknown data type: {}".format(type(pobj).__name__))
+
+
+_range = range  # built-in range
+
+
+def range(
+    start: int, end: Optional[int] = None, step: int = 1, num_partitions: Optional[int] = None
+) -> DataFrame:
+    """
+    Create a DataFrame with some range of numbers.
+
+    The resulting DataFrame has a single int64 column named `id`, containing elements in a range
+    from ``start`` to ``end`` (exclusive) with step value ``step``. If only the first parameter
+    (i.e. start) is specified, we treat it as the end value with the start value being 0.
+
+    This is similar to the range function in SparkSession and is used primarily for testing.
+
+    Parameters
+    ----------
+    start : int
+        the start value (inclusive)
+    end : int, optional
+        the end value (exclusive)
+    step : int, optional, default 1
+        the incremental step
+    num_partitions : int, optional
+        the number of partitions of the DataFrame
+
+    Returns
+    -------
+    DataFrame
+
+    Examples
+    --------
+    When the first parameter is specified, we generate a range of values up till that number.
+
+    >>> ps.range(5)
+       id
+    0   0
+    1   1
+    2   2
+    3   3
+    4   4
+
+    When start, end, and step are specified:
+
+    >>> ps.range(start = 100, end = 200, step = 20)
+        id
+    0  100
+    1  120
+    2  140
+    3  160
+    4  180
+    """
+    sdf = default_session().range(start=start, end=end, step=step, numPartitions=num_partitions)
+    return DataFrame(sdf)
+
+
+def read_csv(
+    path,
+    sep=",",
+    header="infer",
+    names=None,
+    index_col=None,
+    usecols=None,
+    squeeze=False,
+    mangle_dupe_cols=True,
+    dtype=None,
+    nrows=None,
+    parse_dates=False,
+    quotechar=None,
+    escapechar=None,
+    comment=None,
+    **options
+) -> Union[DataFrame, Series]:
+    """Read CSV (comma-separated) file into DataFrame or Series.
+
+    Parameters
+    ----------
+    path : str
+        The path string storing the CSV file to be read.
+    sep : str, default ‘,’
+        Delimiter to use. Must be a single character.
+    header : int, list of int, default ‘infer’
+        Whether to to use as the column names, and the start of the data.
+        Default behavior is to infer the column names: if no names are passed
+        the behavior is identical to `header=0` and column names are inferred from
+        the first line of the file, if column names are passed explicitly then
+        the behavior is identical to `header=None`. Explicitly pass `header=0` to be
+        able to replace existing names
+    names : str or array-like, optional
+        List of column names to use. If file contains no header row, then you should
+        explicitly pass `header=None`. Duplicates in this list will cause an error to be issued.
+        If a string is given, it should be a DDL-formatted string in Spark SQL, which is
+        preferred to avoid schema inference for better performance.
+    index_col: str or list of str, optional, default: None
+        Index column of table in Spark.
+    usecols : list-like or callable, optional
+        Return a subset of the columns. If list-like, all elements must either be
+        positional (i.e. integer indices into the document columns) or strings that
+        correspond to column names provided either by the user in names or inferred
+        from the document header row(s).
+        If callable, the callable function will be evaluated against the column names,
+        returning names where the callable function evaluates to `True`.
+    squeeze : bool, default False
+        If the parsed data only contains one column then return a Series.
+    mangle_dupe_cols : bool, default True
+        Duplicate columns will be specified as 'X0', 'X1', ... 'XN', rather
+        than 'X' ... 'X'. Passing in False will cause data to be overwritten if
+        there are duplicate names in the columns.
+        Currently only `True` is allowed.
+    dtype : Type name or dict of column -> type, default None
+        Data type for data or columns. E.g. {‘a’: np.float64, ‘b’: np.int32} Use str or object
+        together with suitable na_values settings to preserve and not interpret dtype.
+    nrows : int, default None
+        Number of rows to read from the CSV file.
+    parse_dates : boolean or list of ints or names or list of lists or dict, default `False`.
+        Currently only `False` is allowed.
+    quotechar : str (length 1), optional
+        The character used to denote the start and end of a quoted item. Quoted items can include
+        the delimiter and it will be ignored.
+    escapechar : str (length 1), default None
+        One-character string used to escape delimiter
+    comment: str, optional
+        Indicates the line should not be parsed.
+    options : dict
+        All other options passed directly into Spark's data source.
+
+    Returns
+    -------
+    DataFrame or Series
+
+    See Also
+    --------
+    DataFrame.to_csv : Write DataFrame to a comma-separated values (csv) file.
+
+    Examples
+    --------
+    >>> ps.read_csv('data.csv')  # doctest: +SKIP
+    """
+    if "options" in options and isinstance(options.get("options"), dict) and len(options) == 1:
+        options = options.get("options")  # type: ignore
+
+    if mangle_dupe_cols is not True:
+        raise ValueError("mangle_dupe_cols can only be `True`: %s" % mangle_dupe_cols)
+    if parse_dates is not False:
+        raise ValueError("parse_dates can only be `False`: %s" % parse_dates)
+
+    if usecols is not None and not callable(usecols):
+        usecols = list(usecols)
+
+    if usecols is None or callable(usecols) or len(usecols) > 0:
+        reader = default_session().read
+        reader.option("inferSchema", True)
+        reader.option("sep", sep)
+
+        if header == "infer":
+            header = 0 if names is None else None
+        if header == 0:
+            reader.option("header", True)
+        elif header is None:
+            reader.option("header", False)
+        else:
+            raise ValueError("Unknown header argument {}".format(header))
+
+        if quotechar is not None:
+            reader.option("quote", quotechar)
+        if escapechar is not None:
+            reader.option("escape", escapechar)
+
+        if comment is not None:
+            if not isinstance(comment, str) or len(comment) != 1:
+                raise ValueError("Only length-1 comment characters supported")
+            reader.option("comment", comment)
+
+        reader.options(**options)
+
+        if isinstance(names, str):
+            sdf = reader.schema(names).csv(path)
+            column_labels = OrderedDict((col, col) for col in sdf.columns)
+        else:
+            sdf = reader.csv(path)
+            if is_list_like(names):
+                names = list(names)
+                if len(set(names)) != len(names):
+                    raise ValueError("Found non-unique column index")
+                if len(names) != len(sdf.columns):
+                    raise ValueError(
+                        "The number of names [%s] does not match the number "
+                        "of columns [%d]. Try names by a Spark SQL DDL-formatted "
+                        "string." % (len(sdf.schema), len(names))
+                    )
+                column_labels = OrderedDict(zip(names, sdf.columns))
+            elif header is None:
+                column_labels = OrderedDict(enumerate(sdf.columns))
+            else:
+                column_labels = OrderedDict((col, col) for col in sdf.columns)
+
+        if usecols is not None:
+            if callable(usecols):
+                column_labels = OrderedDict(
+                    (label, col) for label, col in column_labels.items() if usecols(label)
+                )
+                missing = []
+            elif all(isinstance(col, int) for col in usecols):
+                new_column_labels = OrderedDict(
+                    (label, col)
+                    for i, (label, col) in enumerate(column_labels.items())
+                    if i in usecols
+                )
+                missing = [
+                    col
+                    for col in usecols
+                    if col >= len(column_labels)
+                    or list(column_labels)[col] not in new_column_labels
+                ]
+                column_labels = new_column_labels
+            elif all(isinstance(col, str) for col in usecols):
+                new_column_labels = OrderedDict(
+                    (label, col) for label, col in column_labels.items() if label in usecols
+                )
+                missing = [col for col in usecols if col not in new_column_labels]
+                column_labels = new_column_labels
+            else:
+                raise ValueError(
+                    "'usecols' must either be list-like of all strings, "
+                    "all unicode, all integers or a callable."
+                )
+            if len(missing) > 0:
+                raise ValueError(
+                    "Usecols do not match columns, columns expected but not " "found: %s" % missing
+                )
+
+            if len(column_labels) > 0:
+                sdf = sdf.select([scol_for(sdf, col) for col in column_labels.values()])
+            else:
+                sdf = default_session().createDataFrame([], schema=StructType())
+    else:
+        sdf = default_session().createDataFrame([], schema=StructType())
+        column_labels = OrderedDict()
+
+    if nrows is not None:
+        sdf = sdf.limit(nrows)
+
+    if index_col is not None:
+        if isinstance(index_col, (str, int)):
+            index_col = [index_col]
+        for col in index_col:
+            if col not in column_labels:
+                raise KeyError(col)
+        index_spark_column_names = [column_labels[col] for col in index_col]
+        index_names = [(col,) for col in index_col]  # type: List[Tuple]
+        column_labels = OrderedDict(
+            (label, col) for label, col in column_labels.items() if label not in index_col
+        )
+    else:
+        index_spark_column_names = []
+        index_names = []
+
+    kdf = DataFrame(
+        InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=[scol_for(sdf, col) for col in index_spark_column_names],
+            index_names=index_names,
+            column_labels=[
+                label if is_name_like_tuple(label) else (label,) for label in column_labels
+            ],
+            data_spark_columns=[scol_for(sdf, col) for col in column_labels.values()],
+        )
+    )  # type: DataFrame
+
+    if dtype is not None:
+        if isinstance(dtype, dict):
+            for col, tpe in dtype.items():
+                kdf[col] = kdf[col].astype(tpe)
+        else:
+            for col in kdf.columns:
+                kdf[col] = kdf[col].astype(dtype)
+
+    if squeeze and len(kdf.columns) == 1:
+        return first_series(kdf)
+    else:
+        return kdf
+
+
+def read_json(
+    path: str, lines: bool = True, index_col: Optional[Union[str, List[str]]] = None, **options
+) -> DataFrame:
+    """
+    Convert a JSON string to DataFrame.
+
+    Parameters
+    ----------
+    path : string
+        File path
+    lines : bool, default True
+        Read the file as a json object per line. It should be always True for now.
+    index_col : str or list of str, optional, default: None
+        Index column of table in Spark.
+    options : dict
+        All other options passed directly into Spark's data source.
+
+    Examples
+    --------
+    >>> df = ps.DataFrame([['a', 'b'], ['c', 'd']],
+    ...                   columns=['col 1', 'col 2'])
+
+    >>> df.to_json(path=r'%s/read_json/foo.json' % path, num_files=1)
+    >>> ps.read_json(
+    ...     path=r'%s/read_json/foo.json' % path
+    ... ).sort_values(by="col 1")
+      col 1 col 2
+    0     a     b
+    1     c     d
+
+    >>> df.to_json(path=r'%s/read_json/foo.json' % path, num_files=1, lineSep='___')
+    >>> ps.read_json(
+    ...     path=r'%s/read_json/foo.json' % path, lineSep='___'
+    ... ).sort_values(by="col 1")
+      col 1 col 2
+    0     a     b
+    1     c     d
+
+    You can preserve the index in the roundtrip as below.
+
+    >>> df.to_json(path=r'%s/read_json/bar.json' % path, num_files=1, index_col="index")
+    >>> ps.read_json(
+    ...     path=r'%s/read_json/bar.json' % path, index_col="index"
+    ... ).sort_values(by="col 1")  # doctest: +NORMALIZE_WHITESPACE
+          col 1 col 2
+    index
+    0         a     b
+    1         c     d
+    """
+    if "options" in options and isinstance(options.get("options"), dict) and len(options) == 1:
+        options = options.get("options")  # type: ignore
+
+    if not lines:
+        raise NotImplementedError("lines=False is not implemented yet.")
+
+    return read_spark_io(path, format="json", index_col=index_col, **options)
+
+
+def read_delta(
+    path: str,
+    version: Optional[str] = None,
+    timestamp: Optional[str] = None,
+    index_col: Optional[Union[str, List[str]]] = None,
+    **options
+) -> DataFrame:
+    """
+    Read a Delta Lake table on some file system and return a DataFrame.
+
+    If the Delta Lake table is already stored in the catalog (aka the metastore), use 'read_table'.
+
+    Parameters
+    ----------
+    path : string
+        Path to the Delta Lake table.
+    version : string, optional
+        Specifies the table version (based on Delta's internal transaction version) to read from,
+        using Delta's time travel feature. This sets Delta's 'versionAsOf' option.
+    timestamp : string, optional
+        Specifies the table version (based on timestamp) to read from,
+        using Delta's time travel feature. This must be a valid date or timestamp string in Spark,
+        and sets Delta's 'timestampAsOf' option.
+    index_col : str or list of str, optional, default: None
+        Index column of table in Spark.
+    options
+        Additional options that can be passed onto Delta.
+
+    Returns
+    -------
+    DataFrame
+
+    See Also
+    --------
+    DataFrame.to_delta
+    read_table
+    read_spark_io
+    read_parquet
+
+    Examples
+    --------
+    >>> ps.range(1).to_delta('%s/read_delta/foo' % path)  # doctest: +SKIP
+    >>> ps.read_delta('%s/read_delta/foo' % path)  # doctest: +SKIP
+       id
+    0   0
+
+    >>> ps.range(10, 15, num_partitions=1).to_delta('%s/read_delta/foo' % path,
+    ...                                             mode='overwrite')  # doctest: +SKIP
+    >>> ps.read_delta('%s/read_delta/foo' % path)  # doctest: +SKIP
+       id
+    0  10
+    1  11
+    2  12
+    3  13
+    4  14
+
+    >>> ps.read_delta('%s/read_delta/foo' % path, version=0)  # doctest: +SKIP
+       id
+    0   0
+
+    You can preserve the index in the roundtrip as below.
+
+    >>> ps.range(10, 15, num_partitions=1).to_delta(
+    ...     '%s/read_delta/bar' % path, index_col="index")  # doctest: +SKIP
+    >>> ps.read_delta('%s/read_delta/bar' % path, index_col="index")  # doctest: +SKIP
+           id
+    index
+    0      10
+    1      11
+    2      12
+    3      13
+    4      14
+    """
+    if "options" in options and isinstance(options.get("options"), dict) and len(options) == 1:
+        options = options.get("options")  # type: ignore
+
+    if version is not None:
+        options["versionAsOf"] = version
+    if timestamp is not None:
+        options["timestampAsOf"] = timestamp
+    return read_spark_io(path, format="delta", index_col=index_col, **options)
+
+
+def read_table(name: str, index_col: Optional[Union[str, List[str]]] = None) -> DataFrame:
+    """
+    Read a Spark table and return a DataFrame.
+
+    Parameters
+    ----------
+    name : string
+        Table name in Spark.
+
+    index_col : str or list of str, optional, default: None
+        Index column of table in Spark.
+
+    Returns
+    -------
+    DataFrame
+
+    See Also
+    --------
+    DataFrame.to_table
+    read_delta
+    read_parquet
+    read_spark_io
+
+    Examples
+    --------
+    >>> ps.range(1).to_table('%s.my_table' % db)
+    >>> ps.read_table('%s.my_table' % db)
+       id
+    0   0
+
+    >>> ps.range(1).to_table('%s.my_table' % db, index_col="index")
+    >>> ps.read_table('%s.my_table' % db, index_col="index")  # doctest: +NORMALIZE_WHITESPACE
+           id
+    index
+    0       0
+    """
+    sdf = default_session().read.table(name)
+    index_spark_columns, index_names = _get_index_map(sdf, index_col)
+
+    return DataFrame(
+        InternalFrame(
+            spark_frame=sdf, index_spark_columns=index_spark_columns, index_names=index_names
+        )
+    )
+
+
+def read_spark_io(
+    path: Optional[str] = None,
+    format: Optional[str] = None,
+    schema: Union[str, "StructType"] = None,
+    index_col: Optional[Union[str, List[str]]] = None,
+    **options
+) -> DataFrame:
+    """Load a DataFrame from a Spark data source.
+
+    Parameters
+    ----------
+    path : string, optional
+        Path to the data source.
+    format : string, optional
+        Specifies the output data source format. Some common ones are:
+
+        - 'delta'
+        - 'parquet'
+        - 'orc'
+        - 'json'
+        - 'csv'
+    schema : string or StructType, optional
+        Input schema. If none, Spark tries to infer the schema automatically.
+        The schema can either be a Spark StructType, or a DDL-formatted string like
+        `col0 INT, col1 DOUBLE`.
+    index_col : str or list of str, optional, default: None
+        Index column of table in Spark.
+    options : dict
+        All other options passed directly into Spark's data source.
+
+    See Also
+    --------
+    DataFrame.to_spark_io
+    DataFrame.read_table
+    DataFrame.read_delta
+    DataFrame.read_parquet
+
+    Examples
+    --------
+    >>> ps.range(1).to_spark_io('%s/read_spark_io/data.parquet' % path)
+    >>> ps.read_spark_io(
+    ...     '%s/read_spark_io/data.parquet' % path, format='parquet', schema='id long')
+       id
+    0   0
+
+    >>> ps.range(10, 15, num_partitions=1).to_spark_io('%s/read_spark_io/data.json' % path,
+    ...                                                format='json', lineSep='__')
+    >>> ps.read_spark_io(
+    ...     '%s/read_spark_io/data.json' % path, format='json', schema='id long', lineSep='__')
+       id
+    0  10
+    1  11
+    2  12
+    3  13
+    4  14
+
+    You can preserve the index in the roundtrip as below.
+
+    >>> ps.range(10, 15, num_partitions=1).to_spark_io('%s/read_spark_io/data.orc' % path,
+    ...                                                format='orc', index_col="index")
+    >>> ps.read_spark_io(
+    ...     path=r'%s/read_spark_io/data.orc' % path, format="orc", index_col="index")
+    ... # doctest: +NORMALIZE_WHITESPACE
+           id
+    index
+    0      10
+    1      11
+    2      12
+    3      13
+    4      14
+    """
+    if "options" in options and isinstance(options.get("options"), dict) and len(options) == 1:
+        options = options.get("options")  # type: ignore
+
+    sdf = default_session().read.load(path=path, format=format, schema=schema, **options)
+    index_spark_columns, index_names = _get_index_map(sdf, index_col)
+
+    return DataFrame(
+        InternalFrame(
+            spark_frame=sdf, index_spark_columns=index_spark_columns, index_names=index_names
+        )
+    )
+
+
+def read_parquet(path, columns=None, index_col=None, pandas_metadata=False, **options) -> DataFrame:
+    """Load a parquet object from the file path, returning a DataFrame.
+
+    Parameters
+    ----------
+    path : string
+        File path
+    columns : list, default=None
+        If not None, only these columns will be read from the file.
+    index_col : str or list of str, optional, default: None
+        Index column of table in Spark.
+    pandas_metadata : bool, default: False
+        If True, try to respect the metadata if the Parquet file is written from pandas.
+    options : dict
+        All other options passed directly into Spark's data source.
+
+    Returns
+    -------
+    DataFrame
+
+    See Also
+    --------
+    DataFrame.to_parquet
+    DataFrame.read_table
+    DataFrame.read_delta
+    DataFrame.read_spark_io
+
+    Examples
+    --------
+    >>> ps.range(1).to_parquet('%s/read_spark_io/data.parquet' % path)
+    >>> ps.read_parquet('%s/read_spark_io/data.parquet' % path, columns=['id'])
+       id
+    0   0
+
+    You can preserve the index in the roundtrip as below.
+
+    >>> ps.range(1).to_parquet('%s/read_spark_io/data.parquet' % path, index_col="index")
+    >>> ps.read_parquet('%s/read_spark_io/data.parquet' % path, columns=['id'], index_col="index")
+    ... # doctest: +NORMALIZE_WHITESPACE
+           id
+    index
+    0       0
+    """
+    if "options" in options and isinstance(options.get("options"), dict) and len(options) == 1:
+        options = options.get("options")  # type: ignore
+
+    if columns is not None:
+        columns = list(columns)
+
+    index_names = None
+
+    if index_col is None and pandas_metadata:
+        # Try to read pandas metadata
+
+        @pandas_udf("index_col array<string>, index_names array<string>", PandasUDFType.SCALAR)
+        def read_index_metadata(pser):
+            binary = pser.iloc[0]
+            metadata = pq.ParquetFile(pa.BufferReader(binary)).metadata.metadata
+            if b"pandas" in metadata:
+                pandas_metadata = json.loads(metadata[b"pandas"].decode("utf8"))
+                if all(isinstance(col, str) for col in pandas_metadata["index_columns"]):
+                    index_col = []
+                    index_names = []
+                    for col in pandas_metadata["index_columns"]:
+                        index_col.append(col)
+                        for column in pandas_metadata["columns"]:
+                            if column["field_name"] == col:
+                                index_names.append(column["name"])
+                                break
+                        else:
+                            index_names.append(None)
+                    return pd.DataFrame({"index_col": [index_col], "index_names": [index_names]})
+            return pd.DataFrame({"index_col": [None], "index_names": [None]})
+
+        index_col, index_names = (
+            default_session()
+            .read.format("binaryFile")
+            .load(path)
+            .limit(1)
+            .select(read_index_metadata("content").alias("index_metadata"))
+            .select("index_metadata.*")
+            .head()
+        )
+
+    kdf = read_spark_io(path=path, format="parquet", options=options, index_col=index_col)
+
+    if columns is not None:
+        new_columns = [c for c in columns if c in kdf.columns]
+        if len(new_columns) > 0:
+            kdf = kdf[new_columns]
+        else:
+            sdf = default_session().createDataFrame([], schema=StructType())
+            index_spark_columns, index_names = _get_index_map(sdf, index_col)
+            kdf = DataFrame(
+                InternalFrame(
+                    spark_frame=sdf,
+                    index_spark_columns=index_spark_columns,
+                    index_names=index_names,
+                )
+            )
+
+    if index_names is not None:
+        kdf.index.names = index_names
+
+    return kdf
+
+
+def read_clipboard(sep=r"\s+", **kwargs) -> DataFrame:
+    r"""
+    Read text from clipboard and pass to read_csv. See read_csv for the
+    full argument list
+
+    Parameters
+    ----------
+    sep : str, default '\s+'
+        A string or regex delimiter. The default of '\s+' denotes
+        one or more whitespace characters.
+
+    See Also
+    --------
+    DataFrame.to_clipboard : Write text out to clipboard.
+
+    Returns
+    -------
+    parsed : DataFrame
+    """
+    return cast(DataFrame, from_pandas(pd.read_clipboard(sep, **kwargs)))
+
+
+def read_excel(
+    io,
+    sheet_name=0,
+    header=0,
+    names=None,
+    index_col=None,
+    usecols=None,
+    squeeze=False,
+    dtype=None,
+    engine=None,
+    converters=None,
+    true_values=None,
+    false_values=None,
+    skiprows=None,
+    nrows=None,
+    na_values=None,
+    keep_default_na=True,
+    verbose=False,
+    parse_dates=False,
+    date_parser=None,
+    thousands=None,
+    comment=None,
+    skipfooter=0,
+    convert_float=True,
+    mangle_dupe_cols=True,
+    **kwds
+) -> Union[DataFrame, Series, OrderedDict]:
+    """
+    Read an Excel file into a pandas-on-Spark DataFrame or Series.
+
+    Support both `xls` and `xlsx` file extensions from a local filesystem or URL.
+    Support an option to read a single sheet or a list of sheets.
+
+    Parameters
+    ----------
+    io : str, file descriptor, pathlib.Path, ExcelFile or xlrd.Book
+        The string could be a URL. The value URL must be available in Spark's DataFrameReader.
+
+        .. note::
+            If the underlying Spark is below 3.0, the parameter as a string is not supported.
+            You can use `ps.from_pandas(pd.read_excel(...))` as a workaround.
+
+    sheet_name : str, int, list, or None, default 0
+        Strings are used for sheet names. Integers are used in zero-indexed
+        sheet positions. Lists of strings/integers are used to request
+        multiple sheets. Specify None to get all sheets.
+
+        Available cases:
+
+        * Defaults to ``0``: 1st sheet as a `DataFrame`
+        * ``1``: 2nd sheet as a `DataFrame`
+        * ``"Sheet1"``: Load sheet with name "Sheet1"
+        * ``[0, 1, "Sheet5"]``: Load first, second and sheet named "Sheet5"
+          as a dict of `DataFrame`
+        * None: All sheets.
+
+    header : int, list of int, default 0
+        Row (0-indexed) to use for the column labels of the parsed
+        DataFrame. If a list of integers is passed those row positions will
+        be combined into a ``MultiIndex``. Use None if there is no header.
+    names : array-like, default None
+        List of column names to use. If file contains no header row,
+        then you should explicitly pass header=None.
+    index_col : int, list of int, default None
+        Column (0-indexed) to use as the row labels of the DataFrame.
+        Pass None if there is no such column.  If a list is passed,
+        those columns will be combined into a ``MultiIndex``.  If a
+        subset of data is selected with ``usecols``, index_col
+        is based on the subset.
+    usecols : int, str, list-like, or callable default None
+        Return a subset of the columns.
+
+        * If None, then parse all columns.
+        * If str, then indicates comma separated list of Excel column letters
+          and column ranges (e.g. "A:E" or "A,C,E:F"). Ranges are inclusive of
+          both sides.
+        * If list of int, then indicates list of column numbers to be parsed.
+        * If list of string, then indicates list of column names to be parsed.
+        * If callable, then evaluate each column name against it and parse the
+          column if the callable returns ``True``.
+    squeeze : bool, default False
+        If the parsed data only contains one column then return a Series.
+    dtype : Type name or dict of column -> type, default None
+        Data type for data or columns. E.g. {'a': np.float64, 'b': np.int32}
+        Use `object` to preserve data as stored in Excel and not interpret dtype.
+        If converters are specified, they will be applied INSTEAD
+        of dtype conversion.
+    engine : str, default None
+        If io is not a buffer or path, this must be set to identify io.
+        Acceptable values are None or xlrd.
+    converters : dict, default None
+        Dict of functions for converting values in certain columns. Keys can
+        either be integers or column labels, values are functions that take one
+        input argument, the Excel cell content, and return the transformed
+        content.
+    true_values : list, default None
+        Values to consider as True.
+    false_values : list, default None
+        Values to consider as False.
+    skiprows : list-like
+        Rows to skip at the beginning (0-indexed).
+    nrows : int, default None
+        Number of rows to parse.
+    na_values : scalar, str, list-like, or dict, default None
+        Additional strings to recognize as NA/NaN. If dict passed, specific
+        per-column NA values. By default the following values are interpreted
+        as NaN.
+    keep_default_na : bool, default True
+        If na_values are specified and keep_default_na is False the default NaN
+        values are overridden, otherwise they're appended to.
+    verbose : bool, default False
+        Indicate number of NA values placed in non-numeric columns.
+    parse_dates : bool, list-like, or dict, default False
+        The behavior is as follows:
+
+        * bool. If True -> try parsing the index.
+        * list of int or names. e.g. If [1, 2, 3] -> try parsing columns 1, 2, 3
+          each as a separate date column.
+        * list of lists. e.g.  If [[1, 3]] -> combine columns 1 and 3 and parse as
+          a single date column.
+        * dict, e.g. {{'foo' : [1, 3]}} -> parse columns 1, 3 as date and call
+          result 'foo'
+
+        If a column or index contains an unparseable date, the entire column or
+        index will be returned unaltered as an object data type. For non-standard
+        datetime parsing, use ``pd.to_datetime`` after ``pd.read_csv``
+
+        Note: A fast-path exists for iso8601-formatted dates.
+    date_parser : function, optional
+        Function to use for converting a sequence of string columns to an array of
+        datetime instances. The default uses ``dateutil.parser.parser`` to do the
+        conversion. pandas-on-Spark will try to call `date_parser` in three different ways,
+        advancing to the next if an exception occurs: 1) Pass one or more arrays
+        (as defined by `parse_dates`) as arguments; 2) concatenate (row-wise) the
+        string values from the columns defined by `parse_dates` into a single array
+        and pass that; and 3) call `date_parser` once for each row using one or
+        more strings (corresponding to the columns defined by `parse_dates`) as
+        arguments.
+    thousands : str, default None
+        Thousands separator for parsing string columns to numeric.  Note that
+        this parameter is only necessary for columns stored as TEXT in Excel,
+        any numeric columns will automatically be parsed, regardless of display
+        format.
+    comment : str, default None
+        Comments out remainder of line. Pass a character or characters to this
+        argument to indicate comments in the input file. Any data between the
+        comment string and the end of the current line is ignored.
+    skipfooter : int, default 0
+        Rows at the end to skip (0-indexed).
+    convert_float : bool, default True
+        Convert integral floats to int (i.e., 1.0 --> 1). If False, all numeric
+        data will be read in as floats: Excel stores all numbers as floats
+        internally.
+    mangle_dupe_cols : bool, default True
+        Duplicate columns will be specified as 'X', 'X.1', ...'X.N', rather than
+        'X'...'X'. Passing in False will cause data to be overwritten if there
+        are duplicate names in the columns.
+    **kwds : optional
+        Optional keyword arguments can be passed to ``TextFileReader``.
+
+    Returns
+    -------
+    DataFrame or dict of DataFrames
+        DataFrame from the passed in Excel file. See notes in sheet_name
+        argument for more information on when a dict of DataFrames is returned.
+
+    See Also
+    --------
+    DataFrame.to_excel : Write DataFrame to an Excel file.
+    DataFrame.to_csv : Write DataFrame to a comma-separated values (csv) file.
+    read_csv : Read a comma-separated values (csv) file into DataFrame.
+
+    Examples
+    --------
+    The file can be read using the file name as string or an open file object:
+
+    >>> ps.read_excel('tmp.xlsx', index_col=0)  # doctest: +SKIP
+           Name  Value
+    0   string1      1
+    1   string2      2
+    2  #Comment      3
+
+    >>> ps.read_excel(open('tmp.xlsx', 'rb'),
+    ...               sheet_name='Sheet3')  # doctest: +SKIP
+       Unnamed: 0      Name  Value
+    0           0   string1      1
+    1           1   string2      2
+    2           2  #Comment      3
+
+    Index and header can be specified via the `index_col` and `header` arguments
+
+    >>> ps.read_excel('tmp.xlsx', index_col=None, header=None)  # doctest: +SKIP
+         0         1      2
+    0  NaN      Name  Value
+    1  0.0   string1      1
+    2  1.0   string2      2
+    3  2.0  #Comment      3
+
+    Column types are inferred but can be explicitly specified
+
+    >>> ps.read_excel('tmp.xlsx', index_col=0,
+    ...               dtype={'Name': str, 'Value': float})  # doctest: +SKIP
+           Name  Value
+    0   string1    1.0
+    1   string2    2.0
+    2  #Comment    3.0
+
+    True, False, and NA values, and thousands separators have defaults,
+    but can be explicitly specified, too. Supply the values you would like
+    as strings or lists of strings!
+
+    >>> ps.read_excel('tmp.xlsx', index_col=0,
+    ...               na_values=['string1', 'string2'])  # doctest: +SKIP
+           Name  Value
+    0      None      1
+    1      None      2
+    2  #Comment      3
+
+    Comment lines in the excel input file can be skipped using the `comment` kwarg
+
+    >>> ps.read_excel('tmp.xlsx', index_col=0, comment='#')  # doctest: +SKIP
+          Name  Value
+    0  string1    1.0
+    1  string2    2.0
+    2     None    NaN
+    """
+
+    def pd_read_excel(io_or_bin, sn, sq):
+        return pd.read_excel(
+            io=BytesIO(io_or_bin) if isinstance(io_or_bin, (bytes, bytearray)) else io_or_bin,
+            sheet_name=sn,
+            header=header,
+            names=names,
+            index_col=index_col,
+            usecols=usecols,
+            squeeze=sq,
+            dtype=dtype,
+            engine=engine,
+            converters=converters,
+            true_values=true_values,
+            false_values=false_values,
+            skiprows=skiprows,
+            nrows=nrows,
+            na_values=na_values,
+            keep_default_na=keep_default_na,
+            verbose=verbose,
+            parse_dates=parse_dates,
+            date_parser=date_parser,
+            thousands=thousands,
+            comment=comment,
+            skipfooter=skipfooter,
+            convert_float=convert_float,
+            mangle_dupe_cols=mangle_dupe_cols,
+            **kwds
+        )
+
+    if isinstance(io, str):
+        # 'binaryFile' format is available since Spark 3.0.0.
+        binaries = default_session().read.format("binaryFile").load(io).select("content").head(2)
+        io_or_bin = binaries[0][0]
+        single_file = len(binaries) == 1
+    else:
+        io_or_bin = io
+        single_file = True
+
+    pdf_or_psers = pd_read_excel(io_or_bin, sn=sheet_name, sq=squeeze)
+
+    if single_file:
+        if isinstance(pdf_or_psers, dict):
+            return OrderedDict(
+                [(sn, from_pandas(pdf_or_pser)) for sn, pdf_or_pser in pdf_or_psers.items()]
+            )
+        else:
+            return cast(Union[DataFrame, Series], from_pandas(pdf_or_psers))
+    else:
+
+        def read_excel_on_spark(pdf_or_pser, sn):
+
+            if isinstance(pdf_or_pser, pd.Series):
+                pdf = pdf_or_pser.to_frame()
+            else:
+                pdf = pdf_or_pser
+
+            kdf = from_pandas(pdf)
+            return_schema = force_decimal_precision_scale(
+                as_nullable_spark_type(kdf._internal.spark_frame.drop(*HIDDEN_COLUMNS).schema)
+            )
+
+            def output_func(pdf):
+                pdf = pd.concat(
+                    [pd_read_excel(bin, sn=sn, sq=False) for bin in pdf[pdf.columns[0]]]
+                )
+
+                reset_index = pdf.reset_index()
+                for name, col in reset_index.iteritems():
+                    dt = col.dtype
+                    if is_datetime64_dtype(dt) or is_datetime64tz_dtype(dt):
+                        continue
+                    reset_index[name] = col.replace({np.nan: None})
+                pdf = reset_index
+
+                # Just positionally map the column names to given schema's.
+                return pdf.rename(columns=dict(zip(pdf.columns, return_schema.names)))
+
+            sdf = (
+                default_session()
+                .read.format("binaryFile")
+                .load(io)
+                .select("content")
+                .mapInPandas(lambda iterator: map(output_func, iterator), schema=return_schema)
+            )
+
+            kdf = DataFrame(kdf._internal.with_new_sdf(sdf))
+            if squeeze and len(kdf.columns) == 1:
+                return first_series(kdf)
+            else:
+                return kdf
+
+        if isinstance(pdf_or_psers, dict):
+            return OrderedDict(
+                [
+                    (sn, read_excel_on_spark(pdf_or_pser, sn))
+                    for sn, pdf_or_pser in pdf_or_psers.items()
+                ]
+            )
+        else:
+            return read_excel_on_spark(pdf_or_psers, sheet_name)
+
+
+def read_html(
+    io,
+    match=".+",
+    flavor=None,
+    header=None,
+    index_col=None,
+    skiprows=None,
+    attrs=None,
+    parse_dates=False,
+    thousands=",",
+    encoding=None,
+    decimal=".",
+    converters=None,
+    na_values=None,
+    keep_default_na=True,
+    displayed_only=True,
+) -> List[DataFrame]:
+    r"""Read HTML tables into a ``list`` of ``DataFrame`` objects.
+
+    Parameters
+    ----------
+    io : str or file-like
+        A URL, a file-like object, or a raw string containing HTML. Note that
+        lxml only accepts the http, ftp and file url protocols. If you have a
+        URL that starts with ``'https'`` you might try removing the ``'s'``.
+
+    match : str or compiled regular expression, optional
+        The set of tables containing text matching this regex or string will be
+        returned. Unless the HTML is extremely simple you will probably need to
+        pass a non-empty string here. Defaults to '.+' (match any non-empty
+        string). The default value will return all tables contained on a page.
+        This value is converted to a regular expression so that there is
+        consistent behavior between Beautiful Soup and lxml.
+
+    flavor : str or None, container of strings
+        The parsing engine to use. 'bs4' and 'html5lib' are synonymous with
+        each other, they are both there for backwards compatibility. The
+        default of ``None`` tries to use ``lxml`` to parse and if that fails it
+        falls back on ``bs4`` + ``html5lib``.
+
+    header : int or list-like or None, optional
+        The row (or list of rows for a :class:`~ps.MultiIndex`) to use to
+        make the columns headers.
+
+    index_col : int or list-like or None, optional
+        The column (or list of columns) to use to create the index.
+
+    skiprows : int or list-like or slice or None, optional
+        0-based. Number of rows to skip after parsing the column integer. If a
+        sequence of integers or a slice is given, will skip the rows indexed by
+        that sequence.  Note that a single element sequence means 'skip the nth
+        row' whereas an integer means 'skip n rows'.
+
+    attrs : dict or None, optional
+        This is a dictionary of attributes that you can pass to use to identify
+        the table in the HTML. These are not checked for validity before being
+        passed to lxml or Beautiful Soup. However, these attributes must be
+        valid HTML table attributes to work correctly. For example, ::
+
+            attrs = {'id': 'table'}
+
+        is a valid attribute dictionary because the 'id' HTML tag attribute is
+        a valid HTML attribute for *any* HTML tag as per `this document
+        <http://www.w3.org/TR/html-markup/global-attributes.html>`__. ::
+
+            attrs = {'asdf': 'table'}
+
+        is *not* a valid attribute dictionary because 'asdf' is not a valid
+        HTML attribute even if it is a valid XML attribute.  Valid HTML 4.01
+        table attributes can be found `here
+        <http://www.w3.org/TR/REC-html40/struct/tables.html#h-11.2>`__. A
+        working draft of the HTML 5 spec can be found `here
+        <http://www.w3.org/TR/html-markup/table.html>`__. It contains the
+        latest information on table attributes for the modern web.
+
+    parse_dates : bool, optional
+        See :func:`~ps.read_csv` for more details.
+
+    thousands : str, optional
+        Separator to use to parse thousands. Defaults to ``','``.
+
+    encoding : str or None, optional
+        The encoding used to decode the web page. Defaults to ``None``.``None``
+        preserves the previous encoding behavior, which depends on the
+        underlying parser library (e.g., the parser library will try to use
+        the encoding provided by the document).
+
+    decimal : str, default '.'
+        Character to recognize as decimal point (example: use ',' for European
+        data).
+
+    converters : dict, default None
+        Dict of functions for converting values in certain columns. Keys can
+        either be integers or column labels, values are functions that take one
+        input argument, the cell (not column) content, and return the
+        transformed content.
+
+    na_values : iterable, default None
+        Custom NA values
+
+    keep_default_na : bool, default True
+        If na_values are specified and keep_default_na is False the default NaN
+        values are overridden, otherwise they're appended to
+
+    displayed_only : bool, default True
+        Whether elements with "display: none" should be parsed
+
+    Returns
+    -------
+    dfs : list of DataFrames
+
+    See Also
+    --------
+    read_csv
+    DataFrame.to_html
+    """
+    pdfs = pd.read_html(
+        io=io,
+        match=match,
+        flavor=flavor,
+        header=header,
+        index_col=index_col,
+        skiprows=skiprows,
+        attrs=attrs,
+        parse_dates=parse_dates,
+        thousands=thousands,
+        encoding=encoding,
+        decimal=decimal,
+        converters=converters,
+        na_values=na_values,
+        keep_default_na=keep_default_na,
+        displayed_only=displayed_only,
+    )
+    return cast(List[DataFrame], [from_pandas(pdf) for pdf in pdfs])
+
+
+# TODO: add `coerce_float` and 'parse_dates' parameters
+def read_sql_table(
+    table_name, con, schema=None, index_col=None, columns=None, **options
+) -> DataFrame:
+    """
+    Read SQL database table into a DataFrame.
+
+    Given a table name and a JDBC URI, returns a DataFrame.
+
+    Parameters
+    ----------
+    table_name : str
+        Name of SQL table in database.
+    con : str
+        A JDBC URI could be provided as as str.
+
+        .. note:: The URI must be JDBC URI instead of Python's database URI.
+
+    schema : str, default None
+        Name of SQL schema in database to query (if database flavor
+        supports this). Uses default schema if None (default).
+    index_col : str or list of str, optional, default: None
+        Column(s) to set as index(MultiIndex).
+    columns : list, default None
+        List of column names to select from SQL table.
+    options : dict
+        All other options passed directly into Spark's JDBC data source.
+
+    Returns
+    -------
+    DataFrame
+        A SQL table is returned as two-dimensional data structure with labeled
+        axes.
+
+    See Also
+    --------
+    read_sql_query : Read SQL query into a DataFrame.
+    read_sql : Read SQL query or database table into a DataFrame.
+
+    Examples
+    --------
+    >>> ps.read_sql_table('table_name', 'jdbc:postgresql:db_name')  # doctest: +SKIP
+    """
+    if "options" in options and isinstance(options.get("options"), dict) and len(options) == 1:
+        options = options.get("options")  # type: ignore
+
+    reader = default_session().read
+    reader.option("dbtable", table_name)
+    reader.option("url", con)
+    if schema is not None:
+        reader.schema(schema)
+    reader.options(**options)
+    sdf = reader.format("jdbc").load()
+    index_spark_columns, index_names = _get_index_map(sdf, index_col)
+    kdf = DataFrame(
+        InternalFrame(
+            spark_frame=sdf, index_spark_columns=index_spark_columns, index_names=index_names
+        )
+    )  # type: DataFrame
+    if columns is not None:
+        if isinstance(columns, str):
+            columns = [columns]
+        kdf = kdf[columns]
+    return kdf
+
+
+# TODO: add `coerce_float`, `params`, and 'parse_dates' parameters
+def read_sql_query(sql, con, index_col=None, **options) -> DataFrame:
+    """Read SQL query into a DataFrame.
+
+    Returns a DataFrame corresponding to the result set of the query
+    string. Optionally provide an `index_col` parameter to use one of the
+    columns as the index, otherwise default index will be used.
+
+    .. note:: Some database might hit the issue of Spark: SPARK-27596
+
+    Parameters
+    ----------
+    sql : string SQL query
+        SQL query to be executed.
+    con : str
+        A JDBC URI could be provided as as str.
+
+        .. note:: The URI must be JDBC URI instead of Python's database URI.
+
+    index_col : string or list of strings, optional, default: None
+        Column(s) to set as index(MultiIndex).
+    options : dict
+        All other options passed directly into Spark's JDBC data source.
+
+    Returns
+    -------
+    DataFrame
+
+    See Also
+    --------
+    read_sql_table : Read SQL database table into a DataFrame.
+    read_sql
+
+    Examples
+    --------
+    >>> ps.read_sql_query('SELECT * FROM table_name', 'jdbc:postgresql:db_name')  # doctest: +SKIP
+    """
+    if "options" in options and isinstance(options.get("options"), dict) and len(options) == 1:
+        options = options.get("options")  # type: ignore
+
+    reader = default_session().read
+    reader.option("query", sql)
+    reader.option("url", con)
+    reader.options(**options)
+    sdf = reader.format("jdbc").load()
+    index_spark_columns, index_names = _get_index_map(sdf, index_col)
+    return DataFrame(
+        InternalFrame(
+            spark_frame=sdf, index_spark_columns=index_spark_columns, index_names=index_names
+        )
+    )
+
+
+# TODO: add `coerce_float`, `params`, and 'parse_dates' parameters
+def read_sql(sql, con, index_col=None, columns=None, **options) -> DataFrame:
+    """
+    Read SQL query or database table into a DataFrame.
+
+    This function is a convenience wrapper around ``read_sql_table`` and
+    ``read_sql_query`` (for backward compatibility). It will delegate
+    to the specific function depending on the provided input. A SQL query
+    will be routed to ``read_sql_query``, while a database table name will
+    be routed to ``read_sql_table``. Note that the delegated function might
+    have more specific notes about their functionality not listed here.
+
+    .. note:: Some database might hit the issue of Spark: SPARK-27596
+
+    Parameters
+    ----------
+    sql : string
+        SQL query to be executed or a table name.
+    con : str
+        A JDBC URI could be provided as as str.
+
+        .. note:: The URI must be JDBC URI instead of Python's database URI.
+
+    index_col : string or list of strings, optional, default: None
+        Column(s) to set as index(MultiIndex).
+    columns : list, default: None
+        List of column names to select from SQL table (only used when reading
+        a table).
+    options : dict
+        All other options passed directly into Spark's JDBC data source.
+
+    Returns
+    -------
+    DataFrame
+
+    See Also
+    --------
+    read_sql_table : Read SQL database table into a DataFrame.
+    read_sql_query : Read SQL query into a DataFrame.
+
+    Examples
+    --------
+    >>> ps.read_sql('table_name', 'jdbc:postgresql:db_name')  # doctest: +SKIP
+    >>> ps.read_sql('SELECT * FROM table_name', 'jdbc:postgresql:db_name')  # doctest: +SKIP
+    """
+    if "options" in options and isinstance(options.get("options"), dict) and len(options) == 1:
+        options = options.get("options")  # type: ignore
+
+    striped = sql.strip()
+    if " " not in striped:  # TODO: identify the table name or not more precisely.
+        return read_sql_table(sql, con, index_col=index_col, columns=columns, **options)
+    else:
+        return read_sql_query(sql, con, index_col=index_col, **options)
+
+
+def to_datetime(
+    arg, errors="raise", format=None, unit=None, infer_datetime_format=False, origin="unix"
+):
+    """
+    Convert argument to datetime.
+
+    Parameters
+    ----------
+    arg : integer, float, string, datetime, list, tuple, 1-d array, Series
+           or DataFrame/dict-like
+
+    errors : {'ignore', 'raise', 'coerce'}, default 'raise'
+
+        - If 'raise', then invalid parsing will raise an exception
+        - If 'coerce', then invalid parsing will be set as NaT
+        - If 'ignore', then invalid parsing will return the input
+    format : string, default None
+        strftime to parse time, eg "%d/%m/%Y", note that "%f" will parse
+        all the way up to nanoseconds.
+    unit : string, default None
+        unit of the arg (D,s,ms,us,ns) denote the unit, which is an
+        integer or float number. This will be based off the origin.
+        Example, with unit='ms' and origin='unix' (the default), this
+        would calculate the number of milliseconds to the unix epoch start.
+    infer_datetime_format : boolean, default False
+        If True and no `format` is given, attempt to infer the format of the
+        datetime strings, and if it can be inferred, switch to a faster
+        method of parsing them. In some cases this can increase the parsing
+        speed by ~5-10x.
+    origin : scalar, default 'unix'
+        Define the reference date. The numeric values would be parsed as number
+        of units (defined by `unit`) since this reference date.
+
+        - If 'unix' (or POSIX) time; origin is set to 1970-01-01.
+        - If 'julian', unit must be 'D', and origin is set to beginning of
+          Julian Calendar. Julian day number 0 is assigned to the day starting
+          at noon on January 1, 4713 BC.
+        - If Timestamp convertible, origin is set to Timestamp identified by
+          origin.
+
+    Returns
+    -------
+    ret : datetime if parsing succeeded.
+        Return type depends on input:
+
+        - list-like: DatetimeIndex
+        - Series: Series of datetime64 dtype
+        - scalar: Timestamp
+
+        In case when it is not possible to return designated types (e.g. when
+        any element of input is before Timestamp.min or after Timestamp.max)
+        return will have datetime.datetime type (or corresponding
+        array/Series).
+
+    Examples
+    --------
+    Assembling a datetime from multiple columns of a DataFrame. The keys can be
+    common abbreviations like ['year', 'month', 'day', 'minute', 'second',
+    'ms', 'us', 'ns']) or plurals of the same
+
+    >>> df = ps.DataFrame({'year': [2015, 2016],
+    ...                    'month': [2, 3],
+    ...                    'day': [4, 5]})
+    >>> ps.to_datetime(df)
+    0   2015-02-04
+    1   2016-03-05
+    dtype: datetime64[ns]
+
+    If a date does not meet the `timestamp limitations
+    <http://pandas.pydata.org/pandas-docs/stable/timeseries.html
+    #timeseries-timestamp-limits>`_, passing errors='ignore'
+    will return the original input instead of raising any exception.
+
+    Passing errors='coerce' will force an out-of-bounds date to NaT,
+    in addition to forcing non-dates (or non-parseable dates) to NaT.
+
+    >>> ps.to_datetime('13000101', format='%Y%m%d', errors='ignore')
+    datetime.datetime(1300, 1, 1, 0, 0)
+    >>> ps.to_datetime('13000101', format='%Y%m%d', errors='coerce')
+    NaT
+
+    Passing infer_datetime_format=True can often-times speedup a parsing
+    if its not an ISO8601 format exactly, but in a regular format.
+
+    >>> s = ps.Series(['3/11/2000', '3/12/2000', '3/13/2000'] * 1000)
+    >>> s.head()
+    0    3/11/2000
+    1    3/12/2000
+    2    3/13/2000
+    3    3/11/2000
+    4    3/12/2000
+    dtype: object
+
+    >>> import timeit
+    >>> timeit.timeit(
+    ...    lambda: repr(ps.to_datetime(s, infer_datetime_format=True)),
+    ...    number = 1)  # doctest: +SKIP
+    0.35832712500000063
+
+    >>> timeit.timeit(
+    ...    lambda: repr(ps.to_datetime(s, infer_datetime_format=False)),
+    ...    number = 1)  # doctest: +SKIP
+    0.8895321660000004
+
+    Using a unix epoch time
+
+    >>> ps.to_datetime(1490195805, unit='s')
+    Timestamp('2017-03-22 15:16:45')
+    >>> ps.to_datetime(1490195805433502912, unit='ns')
+    Timestamp('2017-03-22 15:16:45.433502912')
+
+    Using a non-unix epoch origin
+
+    >>> ps.to_datetime([1, 2, 3], unit='D', origin=pd.Timestamp('1960-01-01'))
+    DatetimeIndex(['1960-01-02', '1960-01-03', '1960-01-04'], dtype='datetime64[ns]', freq=None)
+    """
+
+    def pandas_to_datetime(pser_or_pdf) -> Series[np.datetime64]:
+        if isinstance(pser_or_pdf, pd.DataFrame):
+            pser_or_pdf = pser_or_pdf[["year", "month", "day"]]
+        return pd.to_datetime(
+            pser_or_pdf,
+            errors=errors,
+            format=format,
+            unit=unit,
+            infer_datetime_format=infer_datetime_format,
+            origin=origin,
+        )
+
+    if isinstance(arg, Series):
+        return arg.koalas.transform_batch(pandas_to_datetime)
+    if isinstance(arg, DataFrame):
+        kdf = arg[["year", "month", "day"]]
+        return kdf.koalas.transform_batch(pandas_to_datetime)
+    return pd.to_datetime(
+        arg,
+        errors=errors,
+        format=format,
+        unit=unit,
+        infer_datetime_format=infer_datetime_format,
+        origin=origin,
+    )
+
+
+def date_range(
+    start=None,
+    end=None,
+    periods=None,
+    freq=None,
+    tz=None,
+    normalize=False,
+    name=None,
+    closed=None,
+    **kwargs
+) -> DatetimeIndex:
+    """
+    Return a fixed frequency DatetimeIndex.
+
+    Parameters
+    ----------
+    start : str or datetime-like, optional
+        Left bound for generating dates.
+    end : str or datetime-like, optional
+        Right bound for generating dates.
+    periods : int, optional
+        Number of periods to generate.
+    freq : str or DateOffset, default 'D'
+        Frequency strings can have multiples, e.g. '5H'.
+    tz : str or tzinfo, optional
+        Time zone name for returning localized DatetimeIndex, for example
+        'Asia/Hong_Kong'. By default, the resulting DatetimeIndex is
+        timezone-naive.
+    normalize : bool, default False
+        Normalize start/end dates to midnight before generating date range.
+    name : str, default None
+        Name of the resulting DatetimeIndex.
+    closed : {None, 'left', 'right'}, optional
+        Make the interval closed with respect to the given frequency to
+        the 'left', 'right', or both sides (None, the default).
+    **kwargs
+        For compatibility. Has no effect on the result.
+
+    Returns
+    -------
+    rng : DatetimeIndex
+
+    See Also
+    --------
+    DatetimeIndex : An immutable container for datetimes.
+
+    Notes
+    -----
+    Of the four parameters ``start``, ``end``, ``periods``, and ``freq``,
+    exactly three must be specified. If ``freq`` is omitted, the resulting
+    ``DatetimeIndex`` will have ``periods`` linearly spaced elements between
+    ``start`` and ``end`` (closed on both sides).
+
+    To learn more about the frequency strings, please see `this link
+    <https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#offset-aliases>`__.
+
+    Examples
+    --------
+    **Specifying the values**
+
+    The next four examples generate the same `DatetimeIndex`, but vary
+    the combination of `start`, `end` and `periods`.
+
+    Specify `start` and `end`, with the default daily frequency.
+
+    >>> ps.date_range(start='1/1/2018', end='1/08/2018')  # doctest: +NORMALIZE_WHITESPACE
+    DatetimeIndex(['2018-01-01', '2018-01-02', '2018-01-03', '2018-01-04',
+                   '2018-01-05', '2018-01-06', '2018-01-07', '2018-01-08'],
+                  dtype='datetime64[ns]', freq=None)
+
+    Specify `start` and `periods`, the number of periods (days).
+
+    >>> ps.date_range(start='1/1/2018', periods=8)  # doctest: +NORMALIZE_WHITESPACE
+    DatetimeIndex(['2018-01-01', '2018-01-02', '2018-01-03', '2018-01-04',
+                   '2018-01-05', '2018-01-06', '2018-01-07', '2018-01-08'],
+                  dtype='datetime64[ns]', freq=None)
+
+    Specify `end` and `periods`, the number of periods (days).
+
+    >>> ps.date_range(end='1/1/2018', periods=8)  # doctest: +NORMALIZE_WHITESPACE
+    DatetimeIndex(['2017-12-25', '2017-12-26', '2017-12-27', '2017-12-28',
+                   '2017-12-29', '2017-12-30', '2017-12-31', '2018-01-01'],
+                  dtype='datetime64[ns]', freq=None)
+
+    Specify `start`, `end`, and `periods`; the frequency is generated
+    automatically (linearly spaced).
+
+    >>> ps.date_range(
+    ...     start='2018-04-24', end='2018-04-27', periods=3
+    ... )  # doctest: +NORMALIZE_WHITESPACE
+    DatetimeIndex(['2018-04-24 00:00:00', '2018-04-25 12:00:00',
+                   '2018-04-27 00:00:00'],
+                  dtype='datetime64[ns]', freq=None)
+
+    **Other Parameters**
+
+    Changed the `freq` (frequency) to ``'M'`` (month end frequency).
+
+    >>> ps.date_range(start='1/1/2018', periods=5, freq='M')  # doctest: +NORMALIZE_WHITESPACE
+    DatetimeIndex(['2018-01-31', '2018-02-28', '2018-03-31', '2018-04-30',
+                   '2018-05-31'],
+                  dtype='datetime64[ns]', freq=None)
+
+    Multiples are allowed
+
+    >>> ps.date_range(start='1/1/2018', periods=5, freq='3M')  # doctest: +NORMALIZE_WHITESPACE
+    DatetimeIndex(['2018-01-31', '2018-04-30', '2018-07-31', '2018-10-31',
+                   '2019-01-31'],
+                  dtype='datetime64[ns]', freq=None)
+
+    `freq` can also be specified as an Offset object.
+
+    >>> ps.date_range(
+    ...     start='1/1/2018', periods=5, freq=pd.offsets.MonthEnd(3)
+    ... )  # doctest: +NORMALIZE_WHITESPACE
+    DatetimeIndex(['2018-01-31', '2018-04-30', '2018-07-31', '2018-10-31',
+                   '2019-01-31'],
+                  dtype='datetime64[ns]', freq=None)
+
+    `closed` controls whether to include `start` and `end` that are on the
+    boundary. The default includes boundary points on either end.
+
+    >>> ps.date_range(
+    ...     start='2017-01-01', end='2017-01-04', closed=None
+    ... )  # doctest: +NORMALIZE_WHITESPACE
+    DatetimeIndex(['2017-01-01', '2017-01-02', '2017-01-03', '2017-01-04'],
+                   dtype='datetime64[ns]', freq=None)
+
+    Use ``closed='left'`` to exclude `end` if it falls on the boundary.
+
+    >>> ps.date_range(
+    ...     start='2017-01-01', end='2017-01-04', closed='left'
+    ... )  # doctest: +NORMALIZE_WHITESPACE
+    DatetimeIndex(['2017-01-01', '2017-01-02', '2017-01-03'], dtype='datetime64[ns]', freq=None)
+
+    Use ``closed='right'`` to exclude `start` if it falls on the boundary.
+
+    >>> ps.date_range(
+    ...     start='2017-01-01', end='2017-01-04', closed='right'
+    ... )  # doctest: +NORMALIZE_WHITESPACE
+    DatetimeIndex(['2017-01-02', '2017-01-03', '2017-01-04'], dtype='datetime64[ns]', freq=None)
+    """
+    assert freq not in ["N", "ns"], "nanoseconds is not supported"
+    assert tz is None, "Localized DatetimeIndex is not supported"
+
+    return cast(
+        DatetimeIndex,
+        ps.from_pandas(
+            pd.date_range(
+                start=start,
+                end=end,
+                periods=periods,
+                freq=freq,
+                tz=tz,
+                normalize=normalize,
+                name=name,
+                closed=closed,
+                **kwargs
+            )
+        ),
+    )
+
+
+def get_dummies(
+    data,
+    prefix=None,
+    prefix_sep="_",
+    dummy_na=False,
+    columns=None,
+    sparse=False,
+    drop_first=False,
+    dtype=None,
+) -> DataFrame:
+    """
+    Convert categorical variable into dummy/indicator variables, also
+    known as one hot encoding.
+
+    Parameters
+    ----------
+    data : array-like, Series, or DataFrame
+    prefix : string, list of strings, or dict of strings, default None
+        String to append DataFrame column names.
+        Pass a list with length equal to the number of columns
+        when calling get_dummies on a DataFrame. Alternatively, `prefix`
+        can be a dictionary mapping column names to prefixes.
+    prefix_sep : string, default '_'
+        If appending prefix, separator/delimiter to use. Or pass a
+        list or dictionary as with `prefix.`
+    dummy_na : bool, default False
+        Add a column to indicate NaNs, if False NaNs are ignored.
+    columns : list-like, default None
+        Column names in the DataFrame to be encoded.
+        If `columns` is None then all the columns with
+        `object` or `category` dtype will be converted.
+    sparse : bool, default False
+        Whether the dummy-encoded columns should be be backed by
+        a :class:`SparseArray` (True) or a regular NumPy array (False).
+        In pandas-on-Spark, this value must be "False".
+    drop_first : bool, default False
+        Whether to get k-1 dummies out of k categorical levels by removing the
+        first level.
+    dtype : dtype, default np.uint8
+        Data type for new columns. Only a single dtype is allowed.
+
+    Returns
+    -------
+    dummies : DataFrame
+
+    See Also
+    --------
+    Series.str.get_dummies
+
+    Examples
+    --------
+    >>> s = ps.Series(list('abca'))
+
+    >>> ps.get_dummies(s)
+       a  b  c
+    0  1  0  0
+    1  0  1  0
+    2  0  0  1
+    3  1  0  0
+
+    >>> df = ps.DataFrame({'A': ['a', 'b', 'a'], 'B': ['b', 'a', 'c'],
+    ...                    'C': [1, 2, 3]},
+    ...                   columns=['A', 'B', 'C'])
+
+    >>> ps.get_dummies(df, prefix=['col1', 'col2'])
+       C  col1_a  col1_b  col2_a  col2_b  col2_c
+    0  1       1       0       0       1       0
+    1  2       0       1       1       0       0
+    2  3       1       0       0       0       1
+
+    >>> ps.get_dummies(ps.Series(list('abcaa')))
+       a  b  c
+    0  1  0  0
+    1  0  1  0
+    2  0  0  1
+    3  1  0  0
+    4  1  0  0
+
+    >>> ps.get_dummies(ps.Series(list('abcaa')), drop_first=True)
+       b  c
+    0  0  0
+    1  1  0
+    2  0  1
+    3  0  0
+    4  0  0
+
+    >>> ps.get_dummies(ps.Series(list('abc')), dtype=float)
+         a    b    c
+    0  1.0  0.0  0.0
+    1  0.0  1.0  0.0
+    2  0.0  0.0  1.0
+    """
+    if sparse is not False:
+        raise NotImplementedError("get_dummies currently does not support sparse")
+
+    if columns is not None:
+        if not is_list_like(columns):
+            raise TypeError("Input must be a list-like for parameter `columns`")
+
+    if dtype is None:
+        dtype = "byte"
+
+    if isinstance(data, Series):
+        if prefix is not None:
+            prefix = [str(prefix)]
+        kdf = data.to_frame()
+        column_labels = kdf._internal.column_labels
+        remaining_columns = []
+    else:
+        if isinstance(prefix, str):
+            raise NotImplementedError(
+                "get_dummies currently does not support prefix as string types"
+            )
+        kdf = data.copy()
+
+        if columns is None:
+            column_labels = [
+                label
+                for label in kdf._internal.column_labels
+                if isinstance(
+                    kdf._internal.spark_type_for(label), _get_dummies_default_accept_types
+                )
+            ]
+        else:
+            if is_name_like_tuple(columns):
+                column_labels = [
+                    label
+                    for label in kdf._internal.column_labels
+                    if label[: len(columns)] == columns
+                ]
+                if len(column_labels) == 0:
+                    raise KeyError(name_like_string(columns))
+                if prefix is None:
+                    prefix = [
+                        str(label[len(columns):])
+                        if len(label) > len(columns) + 1
+                        else label[len(columns)]
+                        if len(label) == len(columns) + 1
+                        else ""
+                        for label in column_labels
+                    ]
+            elif any(isinstance(col, tuple) for col in columns) and any(
+                not is_name_like_tuple(col) for col in columns
+            ):
+                raise ValueError(
+                    "Expected tuple, got {}".format(
+                        type(set(col for col in columns if not is_name_like_tuple(col)).pop())
+                    )
+                )
+            else:
+                column_labels = [
+                    label
+                    for key in columns
+                    for label in kdf._internal.column_labels
+                    if label == key or label[0] == key
+                ]
+        if len(column_labels) == 0:
+            if columns is None:
+                return kdf
+            raise KeyError("{} not in index".format(columns))
+
+        if prefix is None:
+            prefix = [str(label) if len(label) > 1 else label[0] for label in column_labels]
+
+        column_labels_set = set(column_labels)
+        remaining_columns = [
+            (
+                kdf[label]
+                if kdf._internal.column_labels_level == 1
+                else kdf[label].rename(name_like_string(label))
+            )
+            for label in kdf._internal.column_labels
+            if label not in column_labels_set
+        ]
+
+    if any(
+        not isinstance(kdf._internal.spark_type_for(label), _get_dummies_acceptable_types)
+        for label in column_labels
+    ):
+        raise NotImplementedError(
+            "get_dummies currently only accept {} values".format(
+                ", ".join([t.typeName() for t in _get_dummies_acceptable_types])
+            )
+        )
+
+    if prefix is not None and len(column_labels) != len(prefix):
+        raise ValueError(
+            "Length of 'prefix' ({}) did not match the length of "
+            "the columns being encoded ({}).".format(len(prefix), len(column_labels))
+        )
+    elif isinstance(prefix, dict):
+        prefix = [prefix[column_label[0]] for column_label in column_labels]
+
+    all_values = _reduce_spark_multi(
+        kdf._internal.spark_frame,
+        [F.collect_set(kdf._internal.spark_column_for(label)) for label in column_labels],
+    )
+    for i, label in enumerate(column_labels):
+        values = all_values[i]
+        if isinstance(values, np.ndarray):
+            values = values.tolist()
+        values = sorted(values)
+        if drop_first:
+            values = values[1:]
+
+        def column_name(value):
+            if prefix is None or prefix[i] == "":
+                return value
+            else:
+                return "{}{}{}".format(prefix[i], prefix_sep, value)
+
+        for value in values:
+            remaining_columns.append(
+                (kdf[label].notnull() & (kdf[label] == value))
+                .astype(dtype)
+                .rename(column_name(value))
+            )
+        if dummy_na:
+            remaining_columns.append(kdf[label].isnull().astype(dtype).rename(column_name(np.nan)))
+
+    return kdf[remaining_columns]
+
+
+# TODO: there are many parameters to implement and support. See pandas's pd.concat.
+def concat(objs, axis=0, join="outer", ignore_index=False, sort=False) -> Union[Series, DataFrame]:
+    """
+    Concatenate pandas-on-Spark objects along a particular axis with optional set logic
+    along the other axes.
+
+    Parameters
+    ----------
+    objs : a sequence of Series or DataFrame
+        Any None objects will be dropped silently unless
+        they are all None in which case a ValueError will be raised
+    axis : {0/'index', 1/'columns'}, default 0
+        The axis to concatenate along.
+    join : {'inner', 'outer'}, default 'outer'
+        How to handle indexes on other axis (or axes).
+    ignore_index : bool, default False
+        If True, do not use the index values along the concatenation axis. The
+        resulting axis will be labeled 0, ..., n - 1. This is useful if you are
+        concatenating objects where the concatenation axis does not have
+        meaningful indexing information. Note the index values on the other
+        axes are still respected in the join.
+    sort : bool, default False
+        Sort non-concatenation axis if it is not already aligned.
+
+    Returns
+    -------
+    object, type of objs
+        When concatenating all ``Series`` along the index (axis=0), a
+        ``Series`` is returned. When ``objs`` contains at least one
+        ``DataFrame``, a ``DataFrame`` is returned. When concatenating along
+        the columns (axis=1), a ``DataFrame`` is returned.
+
+    See Also
+    --------
+    Series.append : Concatenate Series.
+    DataFrame.join : Join DataFrames using indexes.
+    DataFrame.merge : Merge DataFrames by indexes or columns.
+
+    Examples
+    --------
+    >>> from pyspark.pandas.config import set_option, reset_option
+    >>> set_option("compute.ops_on_diff_frames", True)
+
+    Combine two ``Series``.
+
+    >>> s1 = ps.Series(['a', 'b'])
+    >>> s2 = ps.Series(['c', 'd'])
+    >>> ps.concat([s1, s2])
+    0    a
+    1    b
+    0    c
+    1    d
+    dtype: object
+
+    Clear the existing index and reset it in the result
+    by setting the ``ignore_index`` option to ``True``.
+
+    >>> ps.concat([s1, s2], ignore_index=True)
+    0    a
+    1    b
+    2    c
+    3    d
+    dtype: object
+
+    Combine two ``DataFrame`` objects with identical columns.
+
+    >>> df1 = ps.DataFrame([['a', 1], ['b', 2]],
+    ...                    columns=['letter', 'number'])
+    >>> df1
+      letter  number
+    0      a       1
+    1      b       2
+    >>> df2 = ps.DataFrame([['c', 3], ['d', 4]],
+    ...                    columns=['letter', 'number'])
+    >>> df2
+      letter  number
+    0      c       3
+    1      d       4
+
+    >>> ps.concat([df1, df2])
+      letter  number
+    0      a       1
+    1      b       2
+    0      c       3
+    1      d       4
+
+    Combine ``DataFrame`` and ``Series`` objects with different columns.
+
+    >>> ps.concat([df2, s1])
+      letter  number     0
+    0      c     3.0  None
+    1      d     4.0  None
+    0   None     NaN     a
+    1   None     NaN     b
+
+    Combine ``DataFrame`` objects with overlapping columns
+    and return everything. Columns outside the intersection will
+    be filled with ``None`` values.
+
+    >>> df3 = ps.DataFrame([['c', 3, 'cat'], ['d', 4, 'dog']],
+    ...                    columns=['letter', 'number', 'animal'])
+    >>> df3
+      letter  number animal
+    0      c       3    cat
+    1      d       4    dog
+
+    >>> ps.concat([df1, df3])
+      letter  number animal
+    0      a       1   None
+    1      b       2   None
+    0      c       3    cat
+    1      d       4    dog
+
+    Sort the columns.
+
+    >>> ps.concat([df1, df3], sort=True)
+      animal letter  number
+    0   None      a       1
+    1   None      b       2
+    0    cat      c       3
+    1    dog      d       4
+
+    Combine ``DataFrame`` objects with overlapping columns
+    and return only those that are shared by passing ``inner`` to
+    the ``join`` keyword argument.
+
+    >>> ps.concat([df1, df3], join="inner")
+      letter  number
+    0      a       1
+    1      b       2
+    0      c       3
+    1      d       4
+
+    >>> df4 = ps.DataFrame([['bird', 'polly'], ['monkey', 'george']],
+    ...                    columns=['animal', 'name'])
+
+    Combine with column axis.
+
+    >>> ps.concat([df1, df4], axis=1)
+      letter  number  animal    name
+    0      a       1    bird   polly
+    1      b       2  monkey  george
+
+    >>> reset_option("compute.ops_on_diff_frames")
+    """
+    if isinstance(objs, (DataFrame, IndexOpsMixin)) or not isinstance(
+        objs, Iterable
+    ):  # TODO: support dict
+        raise TypeError(
+            "first argument must be an iterable of pandas-on-Spark "
+            "objects, you passed an object of type "
+            '"{name}"'.format(name=type(objs).__name__)
+        )
+
+    if len(cast(Sized, objs)) == 0:
+        raise ValueError("No objects to concatenate")
+    objs = list(filter(lambda obj: obj is not None, objs))
+    if len(objs) == 0:
+        raise ValueError("All objects passed were None")
+
+    for obj in objs:
+        if not isinstance(obj, (Series, DataFrame)):
+            raise TypeError(
+                "cannot concatenate object of type "
+                "'{name}"
+                "; only ps.Series "
+                "and ps.DataFrame are valid".format(name=type(objs).__name__)
+            )
+
+    if join not in ["inner", "outer"]:
+        raise ValueError("Only can inner (intersect) or outer (union) join the other axis.")
+
+    axis = validate_axis(axis)
+    if axis == 1:
+        kdfs = [obj.to_frame() if isinstance(obj, Series) else obj for obj in objs]
+
+        level = min(kdf._internal.column_labels_level for kdf in kdfs)
+        kdfs = [
+            DataFrame._index_normalized_frame(level, kdf)
+            if kdf._internal.column_labels_level > level
+            else kdf
+            for kdf in kdfs
+        ]
+
+        concat_kdf = kdfs[0]
+        column_labels = concat_kdf._internal.column_labels.copy()
+
+        kdfs_not_same_anchor = []
+        for kdf in kdfs[1:]:
+            duplicated = [label for label in kdf._internal.column_labels if label in column_labels]
+            if len(duplicated) > 0:
+                pretty_names = [name_like_string(label) for label in duplicated]
+                raise ValueError(
+                    "Labels have to be unique; however, got duplicated labels %s." % pretty_names
+                )
+            column_labels.extend(kdf._internal.column_labels)
+
+            if same_anchor(concat_kdf, kdf):
+                concat_kdf = DataFrame(
+                    concat_kdf._internal.with_new_columns(
+                        [
+                            concat_kdf._kser_for(label)
+                            for label in concat_kdf._internal.column_labels
+                        ]
+                        + [kdf._kser_for(label) for label in kdf._internal.column_labels]
+                    )
+                )
+            else:
+                kdfs_not_same_anchor.append(kdf)
+
+        if len(kdfs_not_same_anchor) > 0:
+
+            def resolve_func(kdf, this_column_labels, that_column_labels):
+                raise AssertionError("This should not happen.")
+
+            for kdf in kdfs_not_same_anchor:
+                if join == "inner":
+                    concat_kdf = align_diff_frames(
+                        resolve_func, concat_kdf, kdf, fillna=False, how="inner",
+                    )
+                elif join == "outer":
+                    concat_kdf = align_diff_frames(
+                        resolve_func, concat_kdf, kdf, fillna=False, how="full",
+                    )
+
+            concat_kdf = concat_kdf[column_labels]
+
+        if ignore_index:
+            concat_kdf.columns = list(map(str, _range(len(concat_kdf.columns))))
+
+        if sort:
+            concat_kdf = concat_kdf.sort_index()
+
+        return concat_kdf
+
+    # Series, Series ...
+    # We should return Series if objects are all Series.
+    should_return_series = all(map(lambda obj: isinstance(obj, Series), objs))
+
+    # DataFrame, Series ... & Series, Series ...
+    # In this case, we should return DataFrame.
+    new_objs = []
+    num_series = 0
+    series_names = set()
+    for obj in objs:
+        if isinstance(obj, Series):
+            num_series += 1
+            series_names.add(obj.name)
+            obj = obj.to_frame(DEFAULT_SERIES_NAME)
+        new_objs.append(obj)
+    objs = new_objs
+
+    column_labels_levels = set(obj._internal.column_labels_level for obj in objs)
+    if len(column_labels_levels) != 1:
+        raise ValueError("MultiIndex columns should have the same levels")
+
+    # DataFrame, DataFrame, ...
+    # All Series are converted into DataFrame and then compute concat.
+    if not ignore_index:
+        indices_of_kdfs = [kdf.index for kdf in objs]
+        index_of_first_kdf = indices_of_kdfs[0]
+        for index_of_kdf in indices_of_kdfs:
+            if index_of_first_kdf.names != index_of_kdf.names:
+                raise ValueError(
+                    "Index type and names should be same in the objects to concatenate. "
+                    "You passed different indices "
+                    "{index_of_first_kdf} and {index_of_kdf}".format(
+                        index_of_first_kdf=index_of_first_kdf.names, index_of_kdf=index_of_kdf.names
+                    )
+                )
+
+    column_labels_of_kdfs = [kdf._internal.column_labels for kdf in objs]
+    if ignore_index:
+        index_names_of_kdfs = [[] for _ in objs]  # type: List
+    else:
+        index_names_of_kdfs = [kdf._internal.index_names for kdf in objs]
+
+    if all(name == index_names_of_kdfs[0] for name in index_names_of_kdfs) and all(
+        idx == column_labels_of_kdfs[0] for idx in column_labels_of_kdfs
+    ):
+        # If all columns are in the same order and values, use it.
+        kdfs = objs
+    else:
+        if join == "inner":
+            interested_columns = set.intersection(*map(set, column_labels_of_kdfs))
+            # Keep the column order with its firsts DataFrame.
+            merged_columns = [
+                label for label in column_labels_of_kdfs[0] if label in interested_columns
+            ]
+
+            # When multi-index column, although pandas is flaky if `join="inner" and sort=False`,
+            # always sort to follow the `join="outer"` case behavior.
+            if (len(merged_columns) > 0 and len(merged_columns[0]) > 1) or sort:
+                # FIXME: better ordering
+                merged_columns = sorted(merged_columns, key=name_like_string)
+
+            kdfs = [kdf[merged_columns] for kdf in objs]
+        elif join == "outer":
+            merged_columns = []
+            for labels in column_labels_of_kdfs:
+                merged_columns.extend(label for label in labels if label not in merged_columns)
+
+            assert len(merged_columns) > 0
+
+            if LooseVersion(pd.__version__) < LooseVersion("0.24"):
+                # Always sort when multi-index columns, and if there are Series, never sort.
+                sort = len(merged_columns[0]) > 1 or (num_series == 0 and sort)
+            else:
+                # Always sort when multi-index columns or there are more than two Series,
+                # and if there is only one Series, never sort.
+                sort = len(merged_columns[0]) > 1 or num_series > 1 or (num_series != 1 and sort)
+
+            if sort:
+                # FIXME: better ordering
+                merged_columns = sorted(merged_columns, key=name_like_string)
+
+            kdfs = []
+            for kdf in objs:
+                columns_to_add = list(set(merged_columns) - set(kdf._internal.column_labels))
+
+                # TODO: NaN and None difference for missing values. pandas seems filling NaN.
+                sdf = kdf._internal.resolved_copy.spark_frame
+                for label in columns_to_add:
+                    sdf = sdf.withColumn(name_like_string(label), F.lit(None))
+
+                data_columns = kdf._internal.data_spark_column_names + [
+                    name_like_string(label) for label in columns_to_add
+                ]
+                kdf = DataFrame(
+                    kdf._internal.copy(
+                        spark_frame=sdf,
+                        index_spark_columns=[
+                            scol_for(sdf, col) for col in kdf._internal.index_spark_column_names
+                        ],
+                        column_labels=(kdf._internal.column_labels + columns_to_add),
+                        data_spark_columns=[scol_for(sdf, col) for col in data_columns],
+                        data_dtypes=(kdf._internal.data_dtypes + ([None] * len(columns_to_add))),
+                    )
+                )
+
+                kdfs.append(kdf[merged_columns])
+
+    if ignore_index:
+        sdfs = [kdf._internal.spark_frame.select(kdf._internal.data_spark_columns) for kdf in kdfs]
+    else:
+        sdfs = [
+            kdf._internal.spark_frame.select(
+                kdf._internal.index_spark_columns + kdf._internal.data_spark_columns
+            )
+            for kdf in kdfs
+        ]
+    concatenated = reduce(lambda x, y: x.union(y), sdfs)
+
+    if ignore_index:
+        index_spark_column_names = []
+        index_names = []
+        index_dtypes = []
+    else:
+        index_spark_column_names = kdfs[0]._internal.index_spark_column_names
+        index_names = kdfs[0]._internal.index_names
+        index_dtypes = kdfs[0]._internal.index_dtypes
+
+    result_kdf = DataFrame(
+        kdfs[0]._internal.copy(
+            spark_frame=concatenated,
+            index_spark_columns=[scol_for(concatenated, col) for col in index_spark_column_names],
+            index_names=index_names,
+            index_dtypes=index_dtypes,
+            data_spark_columns=[
+                scol_for(concatenated, col) for col in kdfs[0]._internal.data_spark_column_names
+            ],
+            data_dtypes=None,  # TODO: dtypes?
+        )
+    )  # type: DataFrame
+
+    if should_return_series:
+        # If all input were Series, we should return Series.
+        if len(series_names) == 1:
+            name = series_names.pop()
+        else:
+            name = None
+        return first_series(result_kdf).rename(name)
+    else:
+        return result_kdf
+
+
+def melt(frame, id_vars=None, value_vars=None, var_name=None, value_name="value") -> DataFrame:
+    return DataFrame.melt(frame, id_vars, value_vars, var_name, value_name)
+
+
+melt.__doc__ = DataFrame.melt.__doc__
+
+
+def isna(obj):
+    """
+    Detect missing values for an array-like object.
+
+    This function takes a scalar or array-like object and indicates
+    whether values are missing (``NaN`` in numeric arrays, ``None`` or ``NaN``
+    in object arrays).
+
+    Parameters
+    ----------
+    obj : scalar or array-like
+        Object to check for null or missing values.
+
+    Returns
+    -------
+    bool or array-like of bool
+        For scalar input, returns a scalar boolean.
+        For array input, returns an array of boolean indicating whether each
+        corresponding element is missing.
+
+    See Also
+    --------
+    Series.isna : Detect missing values in a Series.
+    Series.isnull : Detect missing values in a Series.
+    DataFrame.isna : Detect missing values in a DataFrame.
+    DataFrame.isnull : Detect missing values in a DataFrame.
+    Index.isna : Detect missing values in an Index.
+    Index.isnull : Detect missing values in an Index.
+
+    Examples
+    --------
+    Scalar arguments (including strings) result in a scalar boolean.
+
+    >>> ps.isna('dog')
+    False
+
+    >>> ps.isna(np.nan)
+    True
+
+    ndarrays result in an ndarray of booleans.
+
+    >>> array = np.array([[1, np.nan, 3], [4, 5, np.nan]])
+    >>> array
+    array([[ 1., nan,  3.],
+           [ 4.,  5., nan]])
+    >>> ps.isna(array)
+    array([[False,  True, False],
+           [False, False,  True]])
+
+    For Series and DataFrame, the same type is returned, containing booleans.
+
+    >>> df = ps.DataFrame({'a': ['ant', 'bee', 'cat'], 'b': ['dog', None, 'fly']})
+    >>> df
+         a     b
+    0  ant   dog
+    1  bee  None
+    2  cat   fly
+
+    >>> ps.isna(df)
+           a      b
+    0  False  False
+    1  False   True
+    2  False  False
+
+    >>> ps.isnull(df.b)
+    0    False
+    1     True
+    2    False
+    Name: b, dtype: bool
+    """
+    # TODO: Add back:
+    #     notnull : Boolean inverse of pandas.isnull.
+    #   into the See Also in the docstring. It does not find the method in the latest numpydoc.
+    if isinstance(obj, (DataFrame, Series)):
+        return obj.isnull()
+    else:
+        return pd.isnull(obj)
+
+
+isnull = isna
+
+
+def notna(obj):
+    """
+    Detect existing (non-missing) values.
+
+    Return a boolean same-sized object indicating if the values are not NA.
+    Non-missing values get mapped to True. NA values, such as None or
+    :attr:`numpy.NaN`, get mapped to False values.
+
+    Returns
+    -------
+    bool or array-like of bool
+        Mask of bool values for each element that
+        indicates whether an element is not an NA value.
+
+    See Also
+    --------
+    isna : Detect missing values for an array-like object.
+    Series.notna : Boolean inverse of Series.isna.
+    DataFrame.notnull : Boolean inverse of DataFrame.isnull.
+    Index.notna : Boolean inverse of Index.isna.
+    Index.notnull : Boolean inverse of Index.isnull.
+
+    Examples
+    --------
+    Show which entries in a DataFrame are not NA.
+
+    >>> df = ps.DataFrame({'age': [5, 6, np.NaN],
+    ...                    'born': [pd.NaT, pd.Timestamp('1939-05-27'),
+    ...                             pd.Timestamp('1940-04-25')],
+    ...                    'name': ['Alfred', 'Batman', ''],
+    ...                    'toy': [None, 'Batmobile', 'Joker']})
+    >>> df
+       age       born    name        toy
+    0  5.0        NaT  Alfred       None
+    1  6.0 1939-05-27  Batman  Batmobile
+    2  NaN 1940-04-25              Joker
+
+    >>> df.notnull()
+         age   born  name    toy
+    0   True  False  True  False
+    1   True   True  True   True
+    2  False   True  True   True
+
+    Show which entries in a Series are not NA.
+
+    >>> ser = ps.Series([5, 6, np.NaN])
+    >>> ser
+    0    5.0
+    1    6.0
+    2    NaN
+    dtype: float64
+
+    >>> ps.notna(ser)
+    0     True
+    1     True
+    2    False
+    dtype: bool
+
+    >>> ps.notna(ser.index)
+    True
+    """
+    # TODO: Add back:
+    #     Series.notnull :Boolean inverse of Series.isnull.
+    #     DataFrame.notna :Boolean inverse of DataFrame.isna.
+    #   into the See Also in the docstring. It does not find the method in the latest numpydoc.
+    if isinstance(obj, (DataFrame, Series)):
+        return obj.notna()
+    else:
+        return pd.notna(obj)
+
+
+notnull = notna
+
+
+def merge(
+    obj,
+    right: "DataFrame",
+    how: str = "inner",
+    on: Union[Any, List[Any], Tuple, List[Tuple]] = None,
+    left_on: Union[Any, List[Any], Tuple, List[Tuple]] = None,
+    right_on: Union[Any, List[Any], Tuple, List[Tuple]] = None,
+    left_index: bool = False,
+    right_index: bool = False,
+    suffixes: Tuple[str, str] = ("_x", "_y"),
+) -> "DataFrame":
+    """
+    Merge DataFrame objects with a database-style join.
+
+    The index of the resulting DataFrame will be one of the following:
+        - 0...n if no index is used for merging
+        - Index of the left DataFrame if merged only on the index of the right DataFrame
+        - Index of the right DataFrame if merged only on the index of the left DataFrame
+        - All involved indices if merged using the indices of both DataFrames
+            e.g. if `left` with indices (a, x) and `right` with indices (b, x), the result will
+            be an index (x, a, b)
+
+    Parameters
+    ----------
+    right: Object to merge with.
+    how: Type of merge to be performed.
+        {'left', 'right', 'outer', 'inner'}, default 'inner'
+
+        left: use only keys from left frame, similar to a SQL left outer join; preserve key
+            order.
+        right: use only keys from right frame, similar to a SQL right outer join; preserve key
+            order.
+        outer: use union of keys from both frames, similar to a SQL full outer join; sort keys
+            lexicographically.
+        inner: use intersection of keys from both frames, similar to a SQL inner join;
+            preserve the order of the left keys.
+    on: Column or index level names to join on. These must be found in both DataFrames. If on
+        is None and not merging on indexes then this defaults to the intersection of the
+        columns in both DataFrames.
+    left_on: Column or index level names to join on in the left DataFrame. Can also
+        be an array or list of arrays of the length of the left DataFrame.
+        These arrays are treated as if they are columns.
+    right_on: Column or index level names to join on in the right DataFrame. Can also
+        be an array or list of arrays of the length of the right DataFrame.
+        These arrays are treated as if they are columns.
+    left_index: Use the index from the left DataFrame as the join key(s). If it is a
+        MultiIndex, the number of keys in the other DataFrame (either the index or a number of
+        columns) must match the number of levels.
+    right_index: Use the index from the right DataFrame as the join key. Same caveats as
+        left_index.
+    suffixes: Suffix to apply to overlapping column names in the left and right side,
+        respectively.
+
+    Returns
+    -------
+    DataFrame
+        A DataFrame of the two merged objects.
+
+    Examples
+    --------
+
+    >>> df1 = ps.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'],
+    ...                     'value': [1, 2, 3, 5]},
+    ...                    columns=['lkey', 'value'])
+    >>> df2 = ps.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'],
+    ...                     'value': [5, 6, 7, 8]},
+    ...                    columns=['rkey', 'value'])
+    >>> df1
+      lkey  value
+    0  foo      1
+    1  bar      2
+    2  baz      3
+    3  foo      5
+    >>> df2
+      rkey  value
+    0  foo      5
+    1  bar      6
+    2  baz      7
+    3  foo      8
+
+    Merge df1 and df2 on the lkey and rkey columns. The value columns have
+    the default suffixes, _x and _y, appended.
+
+    >>> merged = ps.merge(df1, df2, left_on='lkey', right_on='rkey')
+    >>> merged.sort_values(by=['lkey', 'value_x', 'rkey', 'value_y'])  # doctest: +ELLIPSIS
+      lkey  value_x rkey  value_y
+    ...bar        2  bar        6
+    ...baz        3  baz        7
+    ...foo        1  foo        5
+    ...foo        1  foo        8
+    ...foo        5  foo        5
+    ...foo        5  foo        8
+
+    >>> left_kdf = ps.DataFrame({'A': [1, 2]})
+    >>> right_kdf = ps.DataFrame({'B': ['x', 'y']}, index=[1, 2])
+
+    >>> ps.merge(left_kdf, right_kdf, left_index=True, right_index=True).sort_index()
+       A  B
+    1  2  x
+
+    >>> ps.merge(left_kdf, right_kdf, left_index=True, right_index=True, how='left').sort_index()
+       A     B
+    0  1  None
+    1  2     x
+
+    >>> ps.merge(left_kdf, right_kdf, left_index=True, right_index=True, how='right').sort_index()
+         A  B
+    1  2.0  x
+    2  NaN  y
+
+    >>> ps.merge(left_kdf, right_kdf, left_index=True, right_index=True, how='outer').sort_index()
+         A     B
+    0  1.0  None
+    1  2.0     x
+    2  NaN     y
+
+    Notes
+    -----
+    As described in #263, joining string columns currently returns None for missing values
+        instead of NaN.
+    """
+    return obj.merge(
+        right,
+        how=how,
+        on=on,
+        left_on=left_on,
+        right_on=right_on,
+        left_index=left_index,
+        right_index=right_index,
+        suffixes=suffixes,
+    )
+
+
+def to_numeric(arg):
+    """
+    Convert argument to a numeric type.
+
+    Parameters
+    ----------
+    arg : scalar, list, tuple, 1-d array, or Series
+
+    Returns
+    -------
+    ret : numeric if parsing succeeded.
+
+    See Also
+    --------
+    DataFrame.astype : Cast argument to a specified dtype.
+    to_datetime : Convert argument to datetime.
+    to_timedelta : Convert argument to timedelta.
+    numpy.ndarray.astype : Cast a numpy array to a specified type.
+
+    Examples
+    --------
+
+    >>> kser = ps.Series(['1.0', '2', '-3'])
+    >>> kser
+    0    1.0
+    1      2
+    2     -3
+    dtype: object
+
+    >>> ps.to_numeric(kser)
+    0    1.0
+    1    2.0
+    2   -3.0
+    dtype: float32
+
+    If given Series contains invalid value to cast float, just cast it to `np.nan`
+
+    >>> kser = ps.Series(['apple', '1.0', '2', '-3'])
+    >>> kser
+    0    apple
+    1      1.0
+    2        2
+    3       -3
+    dtype: object
+
+    >>> ps.to_numeric(kser)
+    0    NaN
+    1    1.0
+    2    2.0
+    3   -3.0
+    dtype: float32
+
+    Also support for list, tuple, np.array, or a scalar
+
+    >>> ps.to_numeric(['1.0', '2', '-3'])
+    array([ 1.,  2., -3.])
+
+    >>> ps.to_numeric(('1.0', '2', '-3'))
+    array([ 1.,  2., -3.])
+
+    >>> ps.to_numeric(np.array(['1.0', '2', '-3']))
+    array([ 1.,  2., -3.])
+
+    >>> ps.to_numeric('1.0')
+    1.0
+    """
+    if isinstance(arg, Series):
+        return arg._with_new_scol(arg.spark.column.cast("float"))
+    else:
+        return pd.to_numeric(arg)
+
+
+def broadcast(obj) -> DataFrame:
+    """
+    Marks a DataFrame as small enough for use in broadcast joins.
+
+    Parameters
+    ----------
+    obj : DataFrame
+
+    Returns
+    -------
+    ret : DataFrame with broadcast hint.
+
+    See Also
+    --------
+    DataFrame.merge : Merge DataFrame objects with a database-style join.
+    DataFrame.join : Join columns of another DataFrame.
+    DataFrame.update : Modify in place using non-NA values from another DataFrame.
+    DataFrame.hint : Specifies some hint on the current DataFrame.
+
+    Examples
+    --------
+    >>> df1 = ps.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'],
+    ...                     'value': [1, 2, 3, 5]},
+    ...                    columns=['lkey', 'value']).set_index('lkey')
+    >>> df2 = ps.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'],
+    ...                     'value': [5, 6, 7, 8]},
+    ...                    columns=['rkey', 'value']).set_index('rkey')
+    >>> merged = df1.merge(ps.broadcast(df2), left_index=True, right_index=True)
+    >>> merged.spark.explain()  # doctest: +ELLIPSIS
+    == Physical Plan ==
+    ...
+    ...BroadcastHashJoin...
+    ...
+    """
+    if not isinstance(obj, DataFrame):
+        raise TypeError("Invalid type : expected DataFrame got {}".format(type(obj).__name__))
+    return DataFrame(
+        obj._internal.with_new_sdf(F.broadcast(obj._internal.resolved_copy.spark_frame))
+    )
+
+
+def read_orc(
+    path,
+    columns: Optional[List[str]] = None,
+    index_col: Optional[Union[str, List[str]]] = None,
+    **options
+) -> "DataFrame":
+    """
+    Load an ORC object from the file path, returning a DataFrame.
+
+    Parameters
+    ----------
+    path : str
+        The path string storing the ORC file to be read.
+    columns : list, default None
+        If not None, only these columns will be read from the file.
+    index_col : str or list of str, optional, default: None
+        Index column of table in Spark.
+    options : dict
+        All other options passed directly into Spark's data source.
+
+    Returns
+    -------
+    DataFrame
+
+    Examples
+    --------
+    >>> ps.range(1).to_orc('%s/read_spark_io/data.orc' % path)
+    >>> ps.read_orc('%s/read_spark_io/data.orc' % path, columns=['id'])
+       id
+    0   0
+
+    You can preserve the index in the roundtrip as below.
+
+    >>> ps.range(1).to_orc('%s/read_spark_io/data.orc' % path, index_col="index")
+    >>> ps.read_orc('%s/read_spark_io/data.orc' % path, columns=['id'], index_col="index")
+    ... # doctest: +NORMALIZE_WHITESPACE
+           id
+    index
+    0       0
+    """
+    if "options" in options and isinstance(options.get("options"), dict) and len(options) == 1:
+        options = options.get("options")  # type: ignore
+
+    kdf = read_spark_io(path, format="orc", index_col=index_col, **options)
+
+    if columns is not None:
+        kdf_columns = kdf.columns
+        new_columns = list()
+        for column in list(columns):
+            if column in kdf_columns:
+                new_columns.append(column)
+            else:
+                raise ValueError("Unknown column name '{}'".format(column))
+        kdf = kdf[new_columns]
+
+    return kdf
+
+
+def _get_index_map(
+    sdf: spark.DataFrame, index_col: Optional[Union[str, List[str]]] = None
+) -> Tuple[Optional[List[spark.Column]], Optional[List[Tuple]]]:
+    if index_col is not None:
+        if isinstance(index_col, str):
+            index_col = [index_col]
+        sdf_columns = set(sdf.columns)
+        for col in index_col:
+            if col not in sdf_columns:
+                raise KeyError(col)
+        index_spark_columns = [
+            scol_for(sdf, col) for col in index_col
+        ]  # type: Optional[List[spark.Column]]
+        index_names = [(col,) for col in index_col]  # type: Optional[List[Tuple]]
+    else:
+        index_spark_columns = None
+        index_names = None
+
+    return index_spark_columns, index_names
+
+
+_get_dummies_default_accept_types = (DecimalType, StringType, DateType)
+_get_dummies_acceptable_types = _get_dummies_default_accept_types + (
+    ByteType,
+    ShortType,
+    IntegerType,
+    LongType,
+    FloatType,
+    DoubleType,
+    BooleanType,
+    TimestampType,
+)
+
+
+def _test():
+    import os
+    import doctest
+    import shutil
+    import sys
+    import tempfile
+    import uuid
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.namespace
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.namespace.__dict__.copy()
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]")
+        .appName("pyspark.pandas.namespace tests")
+        .getOrCreate()
+    )
+
+    db_name = "db%s" % str(uuid.uuid4()).replace("-", "")
+    spark.sql("CREATE DATABASE %s" % db_name)
+    globs["db"] = db_name
+
+    path = tempfile.mkdtemp()
+    globs["path"] = path
+
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.namespace,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+
+    shutil.rmtree(path, ignore_errors=True)
+    spark.sql("DROP DATABASE IF EXISTS %s CASCADE" % db_name)
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/numpy_compat.py b/python/pyspark/pandas/numpy_compat.py
new file mode 100644
index 0000000000000..576708d900908
--- /dev/null
+++ b/python/pyspark/pandas/numpy_compat.py
@@ -0,0 +1,240 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from collections import OrderedDict
+from typing import Callable, Any
+
+import numpy as np
+from pyspark.sql import functions as F, Column
+from pyspark.sql.types import DoubleType, LongType, BooleanType
+
+
+unary_np_spark_mappings = OrderedDict(
+    {
+        "abs": F.abs,
+        "absolute": F.abs,
+        "arccos": F.acos,
+        "arccosh": F.pandas_udf(lambda s: np.arccosh(s), DoubleType()),
+        "arcsin": F.asin,
+        "arcsinh": F.pandas_udf(lambda s: np.arcsinh(s), DoubleType()),
+        "arctan": F.atan,
+        "arctanh": F.pandas_udf(lambda s: np.arctanh(s), DoubleType()),
+        "bitwise_not": F.bitwiseNOT,
+        "cbrt": F.cbrt,
+        "ceil": F.ceil,
+        # It requires complex type which pandas-on-Spark does not support yet
+        "conj": lambda _: NotImplemented,
+        "conjugate": lambda _: NotImplemented,  # It requires complex type
+        "cos": F.cos,
+        "cosh": F.pandas_udf(lambda s: np.cosh(s), DoubleType()),
+        "deg2rad": F.pandas_udf(lambda s: np.deg2rad(s), DoubleType()),
+        "degrees": F.degrees,
+        "exp": F.exp,
+        "exp2": F.pandas_udf(lambda s: np.exp2(s), DoubleType()),
+        "expm1": F.expm1,
+        "fabs": F.pandas_udf(lambda s: np.fabs(s), DoubleType()),
+        "floor": F.floor,
+        "frexp": lambda _: NotImplemented,  # 'frexp' output lengths become different
+        # and it cannot be supported via pandas UDF.
+        "invert": F.pandas_udf(lambda s: np.invert(s), DoubleType()),
+        "isfinite": lambda c: c != float("inf"),
+        "isinf": lambda c: c == float("inf"),
+        "isnan": F.isnan,
+        "isnat": lambda c: NotImplemented,  # pandas-on-Spark and PySpark does not have Nat concept.
+        "log": F.log,
+        "log10": F.log10,
+        "log1p": F.log1p,
+        "log2": F.pandas_udf(lambda s: np.log2(s), DoubleType()),
+        "logical_not": lambda c: ~(c.cast(BooleanType())),
+        "matmul": lambda _: NotImplemented,  # Can return a NumPy array in pandas.
+        "negative": lambda c: c * -1,
+        "positive": lambda c: c,
+        "rad2deg": F.pandas_udf(lambda s: np.rad2deg(s), DoubleType()),
+        "radians": F.radians,
+        "reciprocal": F.pandas_udf(lambda s: np.reciprocal(s), DoubleType()),
+        "rint": F.pandas_udf(lambda s: np.rint(s), DoubleType()),
+        "sign": lambda c: F.when(c == 0, 0).when(c < 0, -1).otherwise(1),
+        "signbit": lambda c: F.when(c < 0, True).otherwise(False),
+        "sin": F.sin,
+        "sinh": F.pandas_udf(lambda s: np.sinh(s), DoubleType()),
+        "spacing": F.pandas_udf(lambda s: np.spacing(s), DoubleType()),
+        "sqrt": F.sqrt,
+        "square": F.pandas_udf(lambda s: np.square(s), DoubleType()),
+        "tan": F.tan,
+        "tanh": F.pandas_udf(lambda s: np.tanh(s), DoubleType()),
+        "trunc": F.pandas_udf(lambda s: np.trunc(s), DoubleType()),
+    }
+)
+
+binary_np_spark_mappings = OrderedDict(
+    {
+        "arctan2": F.atan2,
+        "bitwise_and": lambda c1, c2: c1.bitwiseAND(c2),
+        "bitwise_or": lambda c1, c2: c1.bitwiseOR(c2),
+        "bitwise_xor": lambda c1, c2: c1.bitwiseXOR(c2),
+        "copysign": F.pandas_udf(lambda s1, s2: np.copysign(s1, s2), DoubleType()),
+        "float_power": F.pandas_udf(lambda s1, s2: np.float_power(s1, s2), DoubleType()),
+        "floor_divide": F.pandas_udf(lambda s1, s2: np.floor_divide(s1, s2), DoubleType()),
+        "fmax": F.pandas_udf(lambda s1, s2: np.fmax(s1, s2), DoubleType()),
+        "fmin": F.pandas_udf(lambda s1, s2: np.fmin(s1, s2), DoubleType()),
+        "fmod": F.pandas_udf(lambda s1, s2: np.fmod(s1, s2), DoubleType()),
+        "gcd": F.pandas_udf(lambda s1, s2: np.gcd(s1, s2), DoubleType()),
+        "heaviside": F.pandas_udf(lambda s1, s2: np.heaviside(s1, s2), DoubleType()),
+        "hypot": F.hypot,
+        "lcm": F.pandas_udf(lambda s1, s2: np.lcm(s1, s2), DoubleType()),
+        "ldexp": F.pandas_udf(lambda s1, s2: np.ldexp(s1, s2), DoubleType()),
+        "left_shift": F.pandas_udf(lambda s1, s2: np.left_shift(s1, s2), LongType()),
+        "logaddexp": F.pandas_udf(lambda s1, s2: np.logaddexp(s1, s2), DoubleType()),
+        "logaddexp2": F.pandas_udf(lambda s1, s2: np.logaddexp2(s1, s2), DoubleType()),
+        "logical_and": lambda c1, c2: c1.cast(BooleanType()) & c2.cast(BooleanType()),
+        "logical_or": lambda c1, c2: c1.cast(BooleanType()) | c2.cast(BooleanType()),
+        "logical_xor": lambda c1, c2: (
+            # mimics xor by logical operators.
+            (c1.cast(BooleanType()) | c2.cast(BooleanType()))
+            & (~(c1.cast(BooleanType())) | ~(c2.cast(BooleanType())))
+        ),
+        "maximum": F.greatest,
+        "minimum": F.least,
+        "modf": F.pandas_udf(lambda s1, s2: np.modf(s1, s2), DoubleType()),
+        "nextafter": F.pandas_udf(lambda s1, s2: np.nextafter(s1, s2), DoubleType()),
+        "right_shift": F.pandas_udf(lambda s1, s2: np.right_shift(s1, s2), LongType()),
+    }
+)
+
+
+# Copied from pandas.
+# See also https://docs.scipy.org/doc/numpy/reference/arrays.classes.html#standard-array-subclasses
+def maybe_dispatch_ufunc_to_dunder_op(
+    ser_or_index, ufunc: Callable, method: str, *inputs, **kwargs: Any
+):
+    special = {
+        "add",
+        "sub",
+        "mul",
+        "pow",
+        "mod",
+        "floordiv",
+        "truediv",
+        "divmod",
+        "eq",
+        "ne",
+        "lt",
+        "gt",
+        "le",
+        "ge",
+        "remainder",
+        "matmul",
+    }
+    aliases = {
+        "absolute": "abs",
+        "multiply": "mul",
+        "floor_divide": "floordiv",
+        "true_divide": "truediv",
+        "power": "pow",
+        "remainder": "mod",
+        "divide": "div",
+        "equal": "eq",
+        "not_equal": "ne",
+        "less": "lt",
+        "less_equal": "le",
+        "greater": "gt",
+        "greater_equal": "ge",
+    }
+
+    # For op(., Array) -> Array.__r{op}__
+    flipped = {
+        "lt": "__gt__",
+        "le": "__ge__",
+        "gt": "__lt__",
+        "ge": "__le__",
+        "eq": "__eq__",
+        "ne": "__ne__",
+    }
+
+    op_name = ufunc.__name__
+    op_name = aliases.get(op_name, op_name)
+
+    def not_implemented(*args, **kwargs):
+        return NotImplemented
+
+    if method == "__call__" and op_name in special and kwargs.get("out") is None:
+        if isinstance(inputs[0], type(ser_or_index)):
+            name = "__{}__".format(op_name)
+            return getattr(ser_or_index, name, not_implemented)(inputs[1])
+        else:
+            name = flipped.get(op_name, "__r{}__".format(op_name))
+            return getattr(ser_or_index, name, not_implemented)(inputs[0])
+    else:
+        return NotImplemented
+
+
+# See also https://docs.scipy.org/doc/numpy/reference/arrays.classes.html#standard-array-subclasses
+def maybe_dispatch_ufunc_to_spark_func(
+    ser_or_index, ufunc: Callable, method: str, *inputs, **kwargs: Any
+):
+    from pyspark.pandas.base import column_op
+
+    op_name = ufunc.__name__
+
+    if (
+        method == "__call__"
+        and (op_name in unary_np_spark_mappings or op_name in binary_np_spark_mappings)
+        and kwargs.get("out") is None
+    ):
+
+        np_spark_map_func = unary_np_spark_mappings.get(op_name) or binary_np_spark_mappings.get(
+            op_name
+        )
+
+        def convert_arguments(*args):
+            args = [  # type: ignore
+                F.lit(inp) if not isinstance(inp, Column) else inp for inp in args
+            ]  # type: ignore
+            return np_spark_map_func(*args)
+
+        return column_op(convert_arguments)(*inputs)  # type: ignore
+    else:
+        return NotImplemented
+
+
+def _test():
+    import os
+    import doctest
+    import sys
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.numpy_compat
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.numpy_compat.__dict__.copy()
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]")
+        .appName("pyspark.pandas.numpy_compat tests")
+        .getOrCreate()
+    )
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.numpy_compat,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/plot/__init__.py b/python/pyspark/pandas/plot/__init__.py
new file mode 100644
index 0000000000000..8b3376e7b214f
--- /dev/null
+++ b/python/pyspark/pandas/plot/__init__.py
@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from pyspark.pandas.plot.core import *  # noqa: F401
diff --git a/python/pyspark/pandas/plot/core.py b/python/pyspark/pandas/plot/core.py
new file mode 100644
index 0000000000000..ac4d606267740
--- /dev/null
+++ b/python/pyspark/pandas/plot/core.py
@@ -0,0 +1,1135 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import importlib
+
+import pandas as pd
+import numpy as np
+from pyspark.ml.feature import Bucketizer
+from pyspark.mllib.stat import KernelDensity
+from pyspark.sql import functions as F
+from pandas.core.base import PandasObject
+from pandas.core.dtypes.inference import is_integer
+
+from pyspark.pandas.missing import unsupported_function
+from pyspark.pandas.config import get_option
+from pyspark.pandas.utils import name_like_string
+
+
+class TopNPlotBase:
+    def get_top_n(self, data):
+        from pyspark.pandas import DataFrame, Series
+
+        max_rows = get_option("plotting.max_rows")
+        # Simply use the first 1k elements and make it into a pandas dataframe
+        # For categorical variables, it is likely called from df.x.value_counts().plot.xxx().
+        if isinstance(data, (Series, DataFrame)):
+            data = data.head(max_rows + 1).to_pandas()
+        else:
+            raise TypeError("Only DataFrame and Series are supported for plotting.")
+
+        self.partial = False
+        if len(data) > max_rows:
+            self.partial = True
+            data = data.iloc[:max_rows]
+        return data
+
+    def set_result_text(self, ax):
+        max_rows = get_option("plotting.max_rows")
+        assert hasattr(self, "partial")
+
+        if self.partial:
+            ax.text(
+                1,
+                1,
+                "showing top {} elements only".format(max_rows),
+                size=6,
+                ha="right",
+                va="bottom",
+                transform=ax.transAxes,
+            )
+
+
+class SampledPlotBase:
+    def get_sampled(self, data):
+        from pyspark.pandas import DataFrame, Series
+
+        fraction = get_option("plotting.sample_ratio")
+        if fraction is None:
+            fraction = 1 / (len(data) / get_option("plotting.max_rows"))
+            fraction = min(1.0, fraction)
+        self.fraction = fraction
+
+        if isinstance(data, (DataFrame, Series)):
+            if isinstance(data, Series):
+                data = data.to_frame()
+            sampled = data._internal.resolved_copy.spark_frame.sample(fraction=self.fraction)
+            return DataFrame(data._internal.with_new_sdf(sampled)).to_pandas()
+        else:
+            raise TypeError("Only DataFrame and Series are supported for plotting.")
+
+    def set_result_text(self, ax):
+        assert hasattr(self, "fraction")
+
+        if self.fraction < 1:
+            ax.text(
+                1,
+                1,
+                "showing the sampled result by fraction %s" % self.fraction,
+                size=6,
+                ha="right",
+                va="bottom",
+                transform=ax.transAxes,
+            )
+
+
+class HistogramPlotBase:
+    @staticmethod
+    def prepare_hist_data(data, bins):
+        # TODO: this logic is similar with KdePlotBase. Might have to deduplicate it.
+        from pyspark.pandas.series import Series
+
+        if isinstance(data, Series):
+            data = data.to_frame()
+
+        numeric_data = data.select_dtypes(
+            include=["byte", "decimal", "integer", "float", "long", "double", np.datetime64]
+        )
+
+        # no empty frames or series allowed
+        if len(numeric_data.columns) == 0:
+            raise TypeError(
+                "Empty {0!r}: no numeric data to " "plot".format(numeric_data.__class__.__name__)
+            )
+
+        if is_integer(bins):
+            # computes boundaries for the column
+            bins = HistogramPlotBase.get_bins(data.to_spark(), bins)
+
+        return numeric_data, bins
+
+    @staticmethod
+    def get_bins(sdf, bins):
+        # 'data' is a Spark DataFrame that selects all columns.
+        if len(sdf.columns) > 1:
+            min_col = F.least(*map(F.min, sdf))
+            max_col = F.greatest(*map(F.max, sdf))
+        else:
+            min_col = F.min(sdf.columns[-1])
+            max_col = F.max(sdf.columns[-1])
+        boundaries = sdf.select(min_col, max_col).first()
+
+        # divides the boundaries into bins
+        if boundaries[0] == boundaries[1]:
+            boundaries = (boundaries[0] - 0.5, boundaries[1] + 0.5)
+
+        return np.linspace(boundaries[0], boundaries[1], bins + 1)
+
+    @staticmethod
+    def compute_hist(kdf, bins):
+        # 'data' is a Spark DataFrame that selects one column.
+        assert isinstance(bins, (np.ndarray, np.generic))
+
+        sdf = kdf._internal.spark_frame
+        scols = []
+        input_column_names = []
+        for label in kdf._internal.column_labels:
+            input_column_name = name_like_string(label)
+            input_column_names.append(input_column_name)
+            scols.append(kdf._internal.spark_column_for(label).alias(input_column_name))
+        sdf = sdf.select(*scols)
+
+        # 1. Make the bucket output flat to:
+        #     +----------+-------+
+        #     |__group_id|buckets|
+        #     +----------+-------+
+        #     |0         |0.0    |
+        #     |0         |0.0    |
+        #     |0         |1.0    |
+        #     |0         |2.0    |
+        #     |0         |3.0    |
+        #     |0         |3.0    |
+        #     |1         |0.0    |
+        #     |1         |1.0    |
+        #     |1         |1.0    |
+        #     |1         |2.0    |
+        #     |1         |1.0    |
+        #     |1         |0.0    |
+        #     +----------+-------+
+        colnames = sdf.columns
+        bucket_names = ["__{}_bucket".format(colname) for colname in colnames]
+
+        output_df = None
+        for group_id, (colname, bucket_name) in enumerate(zip(colnames, bucket_names)):
+            # creates a Bucketizer to get corresponding bin of each value
+            bucketizer = Bucketizer(
+                splits=bins, inputCol=colname, outputCol=bucket_name, handleInvalid="skip"
+            )
+
+            bucket_df = bucketizer.transform(sdf)
+
+            if output_df is None:
+                output_df = bucket_df.select(
+                    F.lit(group_id).alias("__group_id"), F.col(bucket_name).alias("__bucket")
+                )
+            else:
+                output_df = output_df.union(
+                    bucket_df.select(
+                        F.lit(group_id).alias("__group_id"), F.col(bucket_name).alias("__bucket")
+                    )
+                )
+
+        # 2. Calculate the count based on each group and bucket.
+        #     +----------+-------+------+
+        #     |__group_id|buckets| count|
+        #     +----------+-------+------+
+        #     |0         |0.0    |2     |
+        #     |0         |1.0    |1     |
+        #     |0         |2.0    |1     |
+        #     |0         |3.0    |2     |
+        #     |1         |0.0    |2     |
+        #     |1         |1.0    |3     |
+        #     |1         |2.0    |1     |
+        #     +----------+-------+------+
+        result = (
+            output_df.groupby("__group_id", "__bucket")
+            .agg(F.count("*").alias("count"))
+            .toPandas()
+            .sort_values(by=["__group_id", "__bucket"])
+        )
+
+        # 3. Fill empty bins and calculate based on each group id. From:
+        #     +----------+--------+------+
+        #     |__group_id|__bucket| count|
+        #     +----------+--------+------+
+        #     |0         |0.0     |2     |
+        #     |0         |1.0     |1     |
+        #     |0         |2.0     |1     |
+        #     |0         |3.0     |2     |
+        #     +----------+--------+------+
+        #     +----------+--------+------+
+        #     |__group_id|__bucket| count|
+        #     +----------+--------+------+
+        #     |1         |0.0     |2     |
+        #     |1         |1.0     |3     |
+        #     |1         |2.0     |1     |
+        #     +----------+--------+------+
+        #
+        # to:
+        #     +-----------------+
+        #     |__values1__bucket|
+        #     +-----------------+
+        #     |2                |
+        #     |1                |
+        #     |1                |
+        #     |2                |
+        #     |0                |
+        #     +-----------------+
+        #     +-----------------+
+        #     |__values2__bucket|
+        #     +-----------------+
+        #     |2                |
+        #     |3                |
+        #     |1                |
+        #     |0                |
+        #     |0                |
+        #     +-----------------+
+        output_series = []
+        for i, (input_column_name, bucket_name) in enumerate(zip(input_column_names, bucket_names)):
+            current_bucket_result = result[result["__group_id"] == i]
+            # generates a pandas DF with one row for each bin
+            # we need this as some of the bins may be empty
+            indexes = pd.DataFrame({"__bucket": np.arange(0, len(bins) - 1)})
+            # merges the bins with counts on it and fills remaining ones with zeros
+            pdf = indexes.merge(current_bucket_result, how="left", on=["__bucket"]).fillna(0)[
+                ["count"]
+            ]
+            pdf.columns = [input_column_name]
+            output_series.append(pdf[input_column_name])
+
+        return output_series
+
+
+class BoxPlotBase:
+    @staticmethod
+    def compute_stats(data, colname, whis, precision):
+        # Computes mean, median, Q1 and Q3 with approx_percentile and precision
+        pdf = data._kdf._internal.resolved_copy.spark_frame.agg(
+            *[
+                F.expr(
+                    "approx_percentile(`{}`, {}, {})".format(colname, q, int(1.0 / precision))
+                ).alias("{}_{}%".format(colname, int(q * 100)))
+                for q in [0.25, 0.50, 0.75]
+            ],
+            F.mean("`%s`" % colname).alias("{}_mean".format(colname)),
+        ).toPandas()
+
+        # Computes IQR and Tukey's fences
+        iqr = "{}_iqr".format(colname)
+        p75 = "{}_75%".format(colname)
+        p25 = "{}_25%".format(colname)
+        pdf.loc[:, iqr] = pdf.loc[:, p75] - pdf.loc[:, p25]
+        pdf.loc[:, "{}_lfence".format(colname)] = pdf.loc[:, p25] - whis * pdf.loc[:, iqr]
+        pdf.loc[:, "{}_ufence".format(colname)] = pdf.loc[:, p75] + whis * pdf.loc[:, iqr]
+
+        qnames = ["25%", "50%", "75%", "mean", "lfence", "ufence"]
+        col_summ = pdf[["{}_{}".format(colname, q) for q in qnames]]
+        col_summ.columns = qnames
+        lfence, ufence = col_summ["lfence"], col_summ["ufence"]
+
+        stats = {
+            "mean": col_summ["mean"].values[0],
+            "med": col_summ["50%"].values[0],
+            "q1": col_summ["25%"].values[0],
+            "q3": col_summ["75%"].values[0],
+        }
+
+        return stats, (lfence.values[0], ufence.values[0])
+
+    @staticmethod
+    def outliers(data, colname, lfence, ufence):
+        # Builds expression to identify outliers
+        expression = F.col("`%s`" % colname).between(lfence, ufence)
+        # Creates a column to flag rows as outliers or not
+        return data._kdf._internal.resolved_copy.spark_frame.withColumn(
+            "__{}_outlier".format(colname), ~expression
+        )
+
+    @staticmethod
+    def calc_whiskers(colname, outliers):
+        # Computes min and max values of non-outliers - the whiskers
+        minmax = (
+            outliers.filter("not `__{}_outlier`".format(colname))
+            .agg(F.min("`%s`" % colname).alias("min"), F.max(colname).alias("max"))
+            .toPandas()
+        )
+        return minmax.iloc[0][["min", "max"]].values
+
+    @staticmethod
+    def get_fliers(colname, outliers, min_val):
+        # Filters only the outliers, should "showfliers" be True
+        fliers_df = outliers.filter("`__{}_outlier`".format(colname))
+
+        # If shows fliers, takes the top 1k with highest absolute values
+        # Here we normalize the values by subtracting the minimum value from
+        # each, and use absolute values.
+        order_col = F.abs(F.col("`{}`".format(colname)) - min_val.item())
+        fliers = (
+            fliers_df.select(F.col("`{}`".format(colname)))
+            .orderBy(order_col)
+            .limit(1001)
+            .toPandas()[colname]
+            .values
+        )
+
+        return fliers
+
+
+class KdePlotBase:
+    @staticmethod
+    def prepare_kde_data(data):
+        # TODO: this logic is similar with HistogramPlotBase. Might have to deduplicate it.
+        from pyspark.pandas.series import Series
+
+        if isinstance(data, Series):
+            data = data.to_frame()
+
+        numeric_data = data.select_dtypes(
+            include=["byte", "decimal", "integer", "float", "long", "double", np.datetime64]
+        )
+
+        # no empty frames or series allowed
+        if len(numeric_data.columns) == 0:
+            raise TypeError(
+                "Empty {0!r}: no numeric data to " "plot".format(numeric_data.__class__.__name__)
+            )
+
+        return numeric_data
+
+    @staticmethod
+    def get_ind(sdf, ind):
+        def calc_min_max():
+            if len(sdf.columns) > 1:
+                min_col = F.least(*map(F.min, sdf))
+                max_col = F.greatest(*map(F.max, sdf))
+            else:
+                min_col = F.min(sdf.columns[-1])
+                max_col = F.max(sdf.columns[-1])
+            return sdf.select(min_col, max_col).first()
+
+        if ind is None:
+            min_val, max_val = calc_min_max()
+            sample_range = max_val - min_val
+            ind = np.linspace(min_val - 0.5 * sample_range, max_val + 0.5 * sample_range, 1000,)
+        elif is_integer(ind):
+            min_val, max_val = calc_min_max()
+            sample_range = max_val - min_val
+            ind = np.linspace(min_val - 0.5 * sample_range, max_val + 0.5 * sample_range, ind,)
+        return ind
+
+    @staticmethod
+    def compute_kde(sdf, bw_method=None, ind=None):
+        # 'sdf' is a Spark DataFrame that selects one column.
+
+        # Using RDD is slow so we might have to change it to Dataset based implementation
+        # once Spark has that implementation.
+        sample = sdf.rdd.map(lambda x: float(x[0]))
+        kd = KernelDensity()
+        kd.setSample(sample)
+
+        assert isinstance(bw_method, (int, float)), "'bw_method' must be set as a scalar number."
+
+        if bw_method is not None:
+            # Match the bandwidth with Spark.
+            kd.setBandwidth(float(bw_method))
+        return kd.estimate(list(map(float, ind)))
+
+
+class PandasOnSparkPlotAccessor(PandasObject):
+    """
+    Series/Frames plotting accessor and method.
+
+    Uses the backend specified by the
+    option ``plotting.backend``. By default, plotly is used.
+
+    Plotting methods can also be accessed by calling the accessor as a method
+    with the ``kind`` argument:
+    ``s.plot(kind='hist')`` is equivalent to ``s.plot.hist()``
+    """
+
+    pandas_plot_data_map = {
+        "pie": TopNPlotBase().get_top_n,
+        "bar": TopNPlotBase().get_top_n,
+        "barh": TopNPlotBase().get_top_n,
+        "scatter": TopNPlotBase().get_top_n,
+        "area": SampledPlotBase().get_sampled,
+        "line": SampledPlotBase().get_sampled,
+    }
+    _backends = {}  # type: ignore
+
+    def __init__(self, data):
+        self.data = data
+
+    @staticmethod
+    def _find_backend(backend):
+        """
+        Find a pandas-on-Spark plotting backend
+        """
+        try:
+            return PandasOnSparkPlotAccessor._backends[backend]
+        except KeyError:
+            try:
+                module = importlib.import_module(backend)
+            except ImportError:
+                # We re-raise later on.
+                pass
+            else:
+                if hasattr(module, "plot") or hasattr(module, "plot_pandas_on_spark"):
+                    # Validate that the interface is implemented when the option
+                    # is set, rather than at plot time.
+                    PandasOnSparkPlotAccessor._backends[backend] = module
+                    return module
+
+        raise ValueError(
+            "Could not find plotting backend '{backend}'. Ensure that you've installed "
+            "the package providing the '{backend}' entrypoint, or that the package has a "
+            "top-level `.plot` method.".format(backend=backend)
+        )
+
+    @staticmethod
+    def _get_plot_backend(backend=None):
+        backend = backend or get_option("plotting.backend")
+        # Shortcut
+        if backend in PandasOnSparkPlotAccessor._backends:
+            return PandasOnSparkPlotAccessor._backends[backend]
+
+        if backend == "matplotlib":
+            # Because matplotlib is an optional dependency,
+            # we need to attempt an import here to raise an ImportError if needed.
+            try:
+                # test if matplotlib can be imported
+                import matplotlib  # noqa: F401
+                from pyspark.pandas.plot import matplotlib as module
+            except ImportError:
+                raise ImportError(
+                    "matplotlib is required for plotting when the "
+                    "default backend 'matplotlib' is selected."
+                ) from None
+
+            PandasOnSparkPlotAccessor._backends["matplotlib"] = module
+        elif backend == "plotly":
+            try:
+                # test if plotly can be imported
+                import plotly  # noqa: F401
+                from pyspark.pandas.plot import plotly as module
+            except ImportError:
+                raise ImportError(
+                    "plotly is required for plotting when the "
+                    "default backend 'plotly' is selected."
+                ) from None
+
+            PandasOnSparkPlotAccessor._backends["plotly"] = module
+        else:
+            module = PandasOnSparkPlotAccessor._find_backend(backend)
+            PandasOnSparkPlotAccessor._backends[backend] = module
+        return module
+
+    def __call__(self, kind="line", backend=None, **kwargs):
+        plot_backend = PandasOnSparkPlotAccessor._get_plot_backend(backend)
+        plot_data = self.data
+
+        kind = {"density": "kde"}.get(kind, kind)
+        if hasattr(plot_backend, "plot_pandas_on_spark"):
+            # use if there's pandas-on-Spark specific method.
+            return plot_backend.plot_pandas_on_spark(plot_data, kind=kind, **kwargs)
+        else:
+            # fallback to use pandas'
+            if not PandasOnSparkPlotAccessor.pandas_plot_data_map[kind]:
+                raise NotImplementedError(
+                    "'%s' plot is not supported with '%s' plot "
+                    "backend yet." % (kind, plot_backend.__name__)
+                )
+            plot_data = PandasOnSparkPlotAccessor.pandas_plot_data_map[kind](plot_data)
+            return plot_backend.plot(plot_data, kind=kind, **kwargs)
+
+    def line(self, x=None, y=None, **kwargs):
+        """
+        Plot DataFrame/Series as lines.
+
+        This function is useful to plot lines using Series's values
+        as coordinates.
+
+        Parameters
+        ----------
+        x : int or str, optional
+            Columns to use for the horizontal axis.
+            Either the location or the label of the columns to be used.
+            By default, it will use the DataFrame indices.
+        y : int, str, or list of them, optional
+            The values to be plotted.
+            Either the location or the label of the columns to be used.
+            By default, it will use the remaining DataFrame numeric columns.
+        **kwds
+            Keyword arguments to pass on to :meth:`Series.plot` or :meth:`DataFrame.plot`.
+
+        Returns
+        -------
+        :class:`plotly.graph_objs.Figure`
+            Return an custom object when ``backend!=plotly``.
+            Return an ndarray when ``subplots=True`` (matplotlib-only).
+
+        See Also
+        --------
+        plotly.express.line : Plot y versus x as lines and/or markers (plotly).
+        matplotlib.pyplot.plot : Plot y versus x as lines and/or markers (matplotlib).
+
+        Examples
+        --------
+        Basic plot.
+
+        For Series:
+
+        .. plotly::
+
+            >>> s = ps.Series([1, 3, 2])
+            >>> s.plot.line()  # doctest: +SKIP
+
+        For DataFrame:
+
+        .. plotly::
+
+            The following example shows the populations for some animals
+            over the years.
+
+            >>> df = ps.DataFrame({'pig': [20, 18, 489, 675, 1776],
+            ...                    'horse': [4, 25, 281, 600, 1900]},
+            ...                   index=[1990, 1997, 2003, 2009, 2014])
+            >>> df.plot.line()  # doctest: +SKIP
+
+        .. plotly::
+
+            The following example shows the relationship between both
+            populations.
+
+            >>> df = ps.DataFrame({'pig': [20, 18, 489, 675, 1776],
+            ...                    'horse': [4, 25, 281, 600, 1900]},
+            ...                   index=[1990, 1997, 2003, 2009, 2014])
+            >>> df.plot.line(x='pig', y='horse')  # doctest: +SKIP
+        """
+        return self(kind="line", x=x, y=y, **kwargs)
+
+    def bar(self, x=None, y=None, **kwds):
+        """
+        Vertical bar plot.
+
+        Parameters
+        ----------
+        x : label or position, optional
+            Allows plotting of one column versus another.
+            If not specified, the index of the DataFrame is used.
+        y : label or position, optional
+            Allows plotting of one column versus another.
+            If not specified, all numerical columns are used.
+        **kwds : optional
+            Additional keyword arguments are documented in
+            :meth:`pyspark.pandas.Series.plot` or
+            :meth:`pyspark.pandas.DataFrame.plot`.
+
+        Returns
+        -------
+        :class:`plotly.graph_objs.Figure`
+            Return an custom object when ``backend!=plotly``.
+            Return an ndarray when ``subplots=True`` (matplotlib-only).
+
+        Examples
+        --------
+        Basic plot.
+
+        For Series:
+
+        .. plotly::
+
+            >>> s = ps.Series([1, 3, 2])
+            >>> s.plot.bar()  # doctest: +SKIP
+
+        For DataFrame:
+
+        .. plotly::
+
+            >>> df = ps.DataFrame({'lab': ['A', 'B', 'C'], 'val': [10, 30, 20]})
+            >>> df.plot.bar(x='lab', y='val')  # doctest: +SKIP
+
+        Plot a whole dataframe to a bar plot. Each column is stacked with a
+        distinct color along the horizontal axis.
+
+        .. plotly::
+
+            >>> speed = [0.1, 17.5, 40, 48, 52, 69, 88]
+            >>> lifespan = [2, 8, 70, 1.5, 25, 12, 28]
+            >>> index = ['snail', 'pig', 'elephant',
+            ...          'rabbit', 'giraffe', 'coyote', 'horse']
+            >>> df = ps.DataFrame({'speed': speed,
+            ...                    'lifespan': lifespan}, index=index)
+            >>> df.plot.bar()  # doctest: +SKIP
+
+        Instead of stacking, the figure can be split by column with plotly
+        APIs.
+
+        .. plotly::
+
+            >>> from plotly.subplots import make_subplots
+            >>> speed = [0.1, 17.5, 40, 48, 52, 69, 88]
+            >>> lifespan = [2, 8, 70, 1.5, 25, 12, 28]
+            >>> index = ['snail', 'pig', 'elephant',
+            ...          'rabbit', 'giraffe', 'coyote', 'horse']
+            >>> df = ps.DataFrame({'speed': speed,
+            ...                    'lifespan': lifespan}, index=index)
+            >>> fig = (make_subplots(rows=2, cols=1)
+            ...        .add_trace(df.plot.bar(y='speed').data[0], row=1, col=1)
+            ...        .add_trace(df.plot.bar(y='speed').data[0], row=1, col=1)
+            ...        .add_trace(df.plot.bar(y='lifespan').data[0], row=2, col=1))
+            >>> fig  # doctest: +SKIP
+
+        Plot a single column.
+
+        .. plotly::
+
+            >>> speed = [0.1, 17.5, 40, 48, 52, 69, 88]
+            >>> lifespan = [2, 8, 70, 1.5, 25, 12, 28]
+            >>> index = ['snail', 'pig', 'elephant',
+            ...          'rabbit', 'giraffe', 'coyote', 'horse']
+            >>> df = ps.DataFrame({'speed': speed,
+            ...                    'lifespan': lifespan}, index=index)
+            >>> df.plot.bar(y='speed')  # doctest: +SKIP
+
+        Plot only selected categories for the DataFrame.
+
+        .. plotly::
+
+            >>> speed = [0.1, 17.5, 40, 48, 52, 69, 88]
+            >>> lifespan = [2, 8, 70, 1.5, 25, 12, 28]
+            >>> index = ['snail', 'pig', 'elephant',
+            ...          'rabbit', 'giraffe', 'coyote', 'horse']
+            >>> df = ps.DataFrame({'speed': speed,
+            ...                    'lifespan': lifespan}, index=index)
+            >>> df.plot.bar(x='lifespan')  # doctest: +SKIP
+        """
+        from pyspark.pandas import DataFrame, Series
+
+        if isinstance(self.data, Series):
+            return self(kind="bar", **kwds)
+        elif isinstance(self.data, DataFrame):
+            return self(kind="bar", x=x, y=y, **kwds)
+
+    def barh(self, x=None, y=None, **kwargs):
+        """
+        Make a horizontal bar plot.
+
+        A horizontal bar plot is a plot that presents quantitative data with
+        rectangular bars with lengths proportional to the values that they
+        represent. A bar plot shows comparisons among discrete categories. One
+        axis of the plot shows the specific categories being compared, and the
+        other axis represents a measured value.
+
+        Parameters
+        ----------
+        x : label or position, default DataFrame.index
+            Column to be used for categories.
+        y : label or position, default All numeric columns in dataframe
+            Columns to be plotted from the DataFrame.
+        **kwds
+            Keyword arguments to pass on to
+            :meth:`pyspark.pandas.DataFrame.plot` or :meth:`pyspark.pandas.Series.plot`.
+
+        Returns
+        -------
+        :class:`plotly.graph_objs.Figure`
+            Return an custom object when ``backend!=plotly``.
+            Return an ndarray when ``subplots=True`` (matplotlib-only).
+
+        See Also
+        --------
+        plotly.express.bar : Plot a vertical bar plot using plotly.
+        matplotlib.axes.Axes.bar : Plot a vertical bar plot using matplotlib.
+
+        Examples
+        --------
+        For Series:
+
+        .. plotly::
+
+            >>> df = ps.DataFrame({'lab': ['A', 'B', 'C'], 'val': [10, 30, 20]})
+            >>> df.val.plot.barh()  # doctest: +SKIP
+
+        For DataFrame:
+
+        .. plotly::
+
+            >>> df = ps.DataFrame({'lab': ['A', 'B', 'C'], 'val': [10, 30, 20]})
+            >>> df.plot.barh(x='lab', y='val')  # doctest: +SKIP
+
+        Plot a whole DataFrame to a horizontal bar plot
+
+        .. plotly::
+
+            >>> speed = [0.1, 17.5, 40, 48, 52, 69, 88]
+            >>> lifespan = [2, 8, 70, 1.5, 25, 12, 28]
+            >>> index = ['snail', 'pig', 'elephant',
+            ...          'rabbit', 'giraffe', 'coyote', 'horse']
+            >>> df = ps.DataFrame({'speed': speed,
+            ...                    'lifespan': lifespan}, index=index)
+            >>> df.plot.barh()  # doctest: +SKIP
+
+        Plot a column of the DataFrame to a horizontal bar plot
+
+        .. plotly::
+
+            >>> speed = [0.1, 17.5, 40, 48, 52, 69, 88]
+            >>> lifespan = [2, 8, 70, 1.5, 25, 12, 28]
+            >>> index = ['snail', 'pig', 'elephant',
+            ...          'rabbit', 'giraffe', 'coyote', 'horse']
+            >>> df = ps.DataFrame({'speed': speed,
+            ...                    'lifespan': lifespan}, index=index)
+            >>> df.plot.barh(y='speed')  # doctest: +SKIP
+
+        Plot DataFrame versus the desired column
+
+        .. plotly::
+
+            >>> speed = [0.1, 17.5, 40, 48, 52, 69, 88]
+            >>> lifespan = [2, 8, 70, 1.5, 25, 12, 28]
+            >>> index = ['snail', 'pig', 'elephant',
+            ...          'rabbit', 'giraffe', 'coyote', 'horse']
+            >>> df = ps.DataFrame({'speed': speed,
+            ...                    'lifespan': lifespan}, index=index)
+            >>> df.plot.barh(x='lifespan')  # doctest: +SKIP
+        """
+        from pyspark.pandas import DataFrame, Series
+
+        if isinstance(self.data, Series):
+            return self(kind="barh", **kwargs)
+        elif isinstance(self.data, DataFrame):
+            return self(kind="barh", x=x, y=y, **kwargs)
+
+    def box(self, **kwds):
+        """
+        Make a box plot of the Series columns.
+
+        Parameters
+        ----------
+        **kwds : optional
+            Additional keyword arguments are documented in
+            :meth:`pyspark.pandas.Series.plot`.
+
+        precision: scalar, default = 0.01
+            This argument is used by pandas-on-Spark to compute approximate statistics
+            for building a boxplot. Use *smaller* values to get more precise
+            statistics (matplotlib-only).
+
+        Returns
+        -------
+        :class:`plotly.graph_objs.Figure`
+            Return an custom object when ``backend!=plotly``.
+            Return an ndarray when ``subplots=True`` (matplotlib-only).
+
+        Notes
+        -----
+        There are behavior differences between pandas-on-Spark and pandas.
+
+        * pandas-on-Spark computes approximate statistics - expect differences between
+          pandas and pandas-on-Spark boxplots, especially regarding 1st and 3rd quartiles.
+        * The `whis` argument is only supported as a single number.
+        * pandas-on-Spark doesn't support the following argument(s) (matplotlib-only).
+
+          * `bootstrap` argument is not supported
+          * `autorange` argument is not supported
+
+        Examples
+        --------
+        Draw a box plot from a DataFrame with four columns of randomly
+        generated data.
+
+        For Series:
+
+        .. plotly::
+
+            >>> data = np.random.randn(25, 4)
+            >>> df = ps.DataFrame(data, columns=list('ABCD'))
+            >>> df['A'].plot.box()  # doctest: +SKIP
+
+        This is an unsupported function for DataFrame type
+        """
+        from pyspark.pandas import DataFrame, Series
+
+        if isinstance(self.data, Series):
+            return self(kind="box", **kwds)
+        elif isinstance(self.data, DataFrame):
+            return unsupported_function(class_name="pd.DataFrame", method_name="box")()
+
+    def hist(self, bins=10, **kwds):
+        """
+        Draw one histogram of the DataFrame’s columns.
+        A `histogram`_ is a representation of the distribution of data.
+        This function calls :meth:`plotting.backend.plot`,
+        on each series in the DataFrame, resulting in one histogram per column.
+
+        .. _histogram: https://en.wikipedia.org/wiki/Histogram
+
+        Parameters
+        ----------
+        bins : integer or sequence, default 10
+            Number of histogram bins to be used. If an integer is given, bins + 1
+            bin edges are calculated and returned. If bins is a sequence, gives
+            bin edges, including left edge of first bin and right edge of last
+            bin. In this case, bins is returned unmodified.
+        **kwds
+            All other plotting keyword arguments to be passed to
+            plotting backend.
+
+        Returns
+        -------
+        :class:`plotly.graph_objs.Figure`
+            Return an custom object when ``backend!=plotly``.
+            Return an ndarray when ``subplots=True`` (matplotlib-only).
+
+        Examples
+        --------
+        Basic plot.
+
+        For Series:
+
+        .. plotly::
+
+            >>> s = ps.Series([1, 3, 2])
+            >>> s.plot.hist()  # doctest: +SKIP
+
+        For DataFrame:
+
+        .. plotly::
+
+            >>> df = pd.DataFrame(
+            ...     np.random.randint(1, 7, 6000),
+            ...     columns=['one'])
+            >>> df['two'] = df['one'] + np.random.randint(1, 7, 6000)
+            >>> df = ps.from_pandas(df)
+            >>> df.plot.hist(bins=12, alpha=0.5)  # doctest: +SKIP
+        """
+        return self(kind="hist", bins=bins, **kwds)
+
+    def kde(self, bw_method=None, ind=None, **kwargs):
+        """
+        Generate Kernel Density Estimate plot using Gaussian kernels.
+
+        Parameters
+        ----------
+        bw_method : scalar
+            The method used to calculate the estimator bandwidth.
+            See KernelDensity in PySpark for more information.
+        ind : NumPy array or integer, optional
+            Evaluation points for the estimated PDF. If None (default),
+            1000 equally spaced points are used. If `ind` is a NumPy array, the
+            KDE is evaluated at the points passed. If `ind` is an integer,
+            `ind` number of equally spaced points are used.
+        **kwargs : optional
+            Keyword arguments to pass on to :meth:`pandas-on-Spark.Series.plot`.
+
+        Returns
+        -------
+        :class:`plotly.graph_objs.Figure`
+            Return an custom object when ``backend!=plotly``.
+            Return an ndarray when ``subplots=True`` (matplotlib-only).
+
+        Examples
+        --------
+        A scalar bandwidth should be specified. Using a small bandwidth value can
+        lead to over-fitting, while using a large bandwidth value may result
+        in under-fitting:
+
+        .. plotly::
+
+            >>> s = ps.Series([1, 2, 2.5, 3, 3.5, 4, 5])
+            >>> s.plot.kde(bw_method=0.3)  # doctest: +SKIP
+
+        .. plotly::
+
+            >>> s = ps.Series([1, 2, 2.5, 3, 3.5, 4, 5])
+            >>> s.plot.kde(bw_method=3)  # doctest: +SKIP
+
+        The `ind` parameter determines the evaluation points for the
+        plot of the estimated KDF:
+
+        .. plotly::
+
+            >>> s = ps.Series([1, 2, 2.5, 3, 3.5, 4, 5])
+            >>> s.plot.kde(ind=[1, 2, 3, 4, 5], bw_method=0.3)  # doctest: +SKIP
+
+        For DataFrame, it works in the same way as Series:
+
+        .. plotly::
+
+            >>> df = ps.DataFrame({
+            ...     'x': [1, 2, 2.5, 3, 3.5, 4, 5],
+            ...     'y': [4, 4, 4.5, 5, 5.5, 6, 6],
+            ... })
+            >>> df.plot.kde(bw_method=0.3)  # doctest: +SKIP
+
+        .. plotly::
+
+            >>> df = ps.DataFrame({
+            ...     'x': [1, 2, 2.5, 3, 3.5, 4, 5],
+            ...     'y': [4, 4, 4.5, 5, 5.5, 6, 6],
+            ... })
+            >>> df.plot.kde(bw_method=3)  # doctest: +SKIP
+
+        .. plotly::
+
+            >>> df = ps.DataFrame({
+            ...     'x': [1, 2, 2.5, 3, 3.5, 4, 5],
+            ...     'y': [4, 4, 4.5, 5, 5.5, 6, 6],
+            ... })
+            >>> df.plot.kde(ind=[1, 2, 3, 4, 5, 6], bw_method=0.3)  # doctest: +SKIP
+        """
+        return self(kind="kde", bw_method=bw_method, ind=ind, **kwargs)
+
+    density = kde
+
+    def area(self, x=None, y=None, **kwds):
+        """
+        Draw a stacked area plot.
+
+        An area plot displays quantitative data visually.
+        This function wraps the plotly area function.
+
+        Parameters
+        ----------
+        x : label or position, optional
+            Coordinates for the X axis. By default uses the index.
+        y : label or position, optional
+            Column to plot. By default uses all columns.
+        stacked : bool, default True
+            Area plots are stacked by default. Set to False to create a
+            unstacked plot (matplotlib-only).
+        **kwds : optional
+            Additional keyword arguments are documented in
+            :meth:`DataFrame.plot`.
+
+        Returns
+        -------
+        :class:`plotly.graph_objs.Figure`
+            Return an custom object when ``backend!=plotly``.
+            Return an ndarray when ``subplots=True`` (matplotlib-only).
+
+        Examples
+        --------
+
+        For Series
+
+        .. plotly::
+
+            >>> df = ps.DataFrame({
+            ...     'sales': [3, 2, 3, 9, 10, 6],
+            ...     'signups': [5, 5, 6, 12, 14, 13],
+            ...     'visits': [20, 42, 28, 62, 81, 50],
+            ... }, index=pd.date_range(start='2018/01/01', end='2018/07/01',
+            ...                        freq='M'))
+            >>> df.sales.plot.area()  # doctest: +SKIP
+
+        For DataFrame
+
+        .. plotly::
+
+            >>> df = ps.DataFrame({
+            ...     'sales': [3, 2, 3, 9, 10, 6],
+            ...     'signups': [5, 5, 6, 12, 14, 13],
+            ...     'visits': [20, 42, 28, 62, 81, 50],
+            ... }, index=pd.date_range(start='2018/01/01', end='2018/07/01',
+            ...                        freq='M'))
+            >>> df.plot.area()  # doctest: +SKIP
+        """
+        from pyspark.pandas import DataFrame, Series
+
+        if isinstance(self.data, Series):
+            return self(kind="area", **kwds)
+        elif isinstance(self.data, DataFrame):
+            return self(kind="area", x=x, y=y, **kwds)
+
+    def pie(self, **kwds):
+        """
+        Generate a pie plot.
+
+        A pie plot is a proportional representation of the numerical data in a
+        column. This function wraps :meth:`plotly.express.pie` for the
+        specified column.
+
+        Parameters
+        ----------
+        y : int or label, optional
+            Label or position of the column to plot.
+            If not provided, ``subplots=True`` argument must be passed (matplotlib-only).
+        **kwds
+            Keyword arguments to pass on to :meth:`pandas-on-Spark.Series.plot`.
+
+        Returns
+        -------
+        :class:`plotly.graph_objs.Figure`
+            Return an custom object when ``backend!=plotly``.
+            Return an ndarray when ``subplots=True`` (matplotlib-only).
+
+        Examples
+        --------
+
+        For Series:
+
+        .. plotly::
+
+            >>> df = ps.DataFrame({'mass': [0.330, 4.87, 5.97],
+            ...                    'radius': [2439.7, 6051.8, 6378.1]},
+            ...                   index=['Mercury', 'Venus', 'Earth'])
+            >>> df.mass.plot.pie()  # doctest: +SKIP
+
+
+        For DataFrame:
+
+        .. plotly::
+
+            >>> df = ps.DataFrame({'mass': [0.330, 4.87, 5.97],
+            ...                    'radius': [2439.7, 6051.8, 6378.1]},
+            ...                   index=['Mercury', 'Venus', 'Earth'])
+            >>> df.plot.pie(y='mass')  # doctest: +SKIP
+        """
+        from pyspark.pandas import DataFrame, Series
+
+        if isinstance(self.data, Series):
+            return self(kind="pie", **kwds)
+        else:
+            # pandas will raise an error if y is None and subplots if not True
+            if (
+                isinstance(self.data, DataFrame)
+                and kwds.get("y", None) is None
+                and not kwds.get("subplots", False)
+            ):
+                raise ValueError(
+                    "pie requires either y column or 'subplots=True' (matplotlib-only)"
+                )
+            return self(kind="pie", **kwds)
+
+    def scatter(self, x, y, **kwds):
+        """
+        Create a scatter plot with varying marker point size and color.
+
+        The coordinates of each point are defined by two dataframe columns and
+        filled circles are used to represent each point. This kind of plot is
+        useful to see complex correlations between two variables. Points could
+        be for instance natural 2D coordinates like longitude and latitude in
+        a map or, in general, any pair of metrics that can be plotted against
+        each other.
+
+        Parameters
+        ----------
+        x : int or str
+            The column name or column position to be used as horizontal
+            coordinates for each point.
+        y : int or str
+            The column name or column position to be used as vertical
+            coordinates for each point.
+        s : scalar or array_like, optional
+            (matplotlib-only).
+        c : str, int or array_like, optional
+            (matplotlib-only).
+
+        **kwds: Optional
+            Keyword arguments to pass on to :meth:`pyspark.pandas.DataFrame.plot`.
+
+        Returns
+        -------
+        :class:`plotly.graph_objs.Figure`
+            Return an custom object when ``backend!=plotly``.
+            Return an ndarray when ``subplots=True`` (matplotlib-only).
+
+        See Also
+        --------
+        plotly.express.scatter : Scatter plot using multiple input data
+            formats (plotly).
+        matplotlib.pyplot.scatter : Scatter plot using multiple input data
+            formats (matplotlib).
+
+        Examples
+        --------
+        Let's see how to draw a scatter plot using coordinates from the values
+        in a DataFrame's columns.
+
+        .. plotly::
+
+            >>> df = ps.DataFrame([[5.1, 3.5, 0], [4.9, 3.0, 0], [7.0, 3.2, 1],
+            ...                    [6.4, 3.2, 1], [5.9, 3.0, 2]],
+            ...                   columns=['length', 'width', 'species'])
+            >>> df.plot.scatter(x='length', y='width')  # doctest: +SKIP
+
+        And now with dark scheme:
+
+        .. plotly::
+
+            >>> df = ps.DataFrame([[5.1, 3.5, 0], [4.9, 3.0, 0], [7.0, 3.2, 1],
+            ...                    [6.4, 3.2, 1], [5.9, 3.0, 2]],
+            ...                   columns=['length', 'width', 'species'])
+            >>> fig = df.plot.scatter(x='length', y='width')
+            >>> fig.update_layout(template="plotly_dark")  # doctest: +SKIP
+        """
+        return self(kind="scatter", x=x, y=y, **kwds)
+
+    def hexbin(self, **kwds):
+        return unsupported_function(class_name="pd.DataFrame", method_name="hexbin")()
diff --git a/python/pyspark/pandas/plot/matplotlib.py b/python/pyspark/pandas/plot/matplotlib.py
new file mode 100644
index 0000000000000..02d1bfc5dbed9
--- /dev/null
+++ b/python/pyspark/pandas/plot/matplotlib.py
@@ -0,0 +1,897 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from distutils.version import LooseVersion
+
+import matplotlib as mat
+import numpy as np
+import pandas as pd
+from matplotlib.axes._base import _process_plot_format
+from pandas.core.dtypes.inference import is_list_like
+from pandas.io.formats.printing import pprint_thing
+
+from pyspark.pandas.plot import (
+    TopNPlotBase,
+    SampledPlotBase,
+    HistogramPlotBase,
+    BoxPlotBase,
+    unsupported_function,
+    KdePlotBase,
+)
+
+
+if LooseVersion(pd.__version__) < LooseVersion("0.25"):
+    from pandas.plotting._core import (
+        _all_kinds,
+        BarPlot as PandasBarPlot,
+        BoxPlot as PandasBoxPlot,
+        HistPlot as PandasHistPlot,
+        MPLPlot as PandasMPLPlot,
+        PiePlot as PandasPiePlot,
+        AreaPlot as PandasAreaPlot,
+        LinePlot as PandasLinePlot,
+        BarhPlot as PandasBarhPlot,
+        ScatterPlot as PandasScatterPlot,
+        KdePlot as PandasKdePlot,
+    )
+else:
+    from pandas.plotting._matplotlib import (
+        BarPlot as PandasBarPlot,
+        BoxPlot as PandasBoxPlot,
+        HistPlot as PandasHistPlot,
+        PiePlot as PandasPiePlot,
+        AreaPlot as PandasAreaPlot,
+        LinePlot as PandasLinePlot,
+        BarhPlot as PandasBarhPlot,
+        ScatterPlot as PandasScatterPlot,
+        KdePlot as PandasKdePlot,
+    )
+    from pandas.plotting._core import PlotAccessor
+    from pandas.plotting._matplotlib.core import MPLPlot as PandasMPLPlot
+
+    _all_kinds = PlotAccessor._all_kinds
+
+
+class PandasOnSparkBarPlot(PandasBarPlot, TopNPlotBase):
+    def __init__(self, data, **kwargs):
+        super().__init__(self.get_top_n(data), **kwargs)
+
+    def _plot(self, ax, x, y, w, start=0, log=False, **kwds):
+        self.set_result_text(ax)
+        return ax.bar(x, y, w, bottom=start, log=log, **kwds)
+
+
+class PandasOnSparkBoxPlot(PandasBoxPlot, BoxPlotBase):
+    def boxplot(
+        self,
+        ax,
+        bxpstats,
+        notch=None,
+        sym=None,
+        vert=None,
+        whis=None,
+        positions=None,
+        widths=None,
+        patch_artist=None,
+        bootstrap=None,
+        usermedians=None,
+        conf_intervals=None,
+        meanline=None,
+        showmeans=None,
+        showcaps=None,
+        showbox=None,
+        showfliers=None,
+        boxprops=None,
+        labels=None,
+        flierprops=None,
+        medianprops=None,
+        meanprops=None,
+        capprops=None,
+        whiskerprops=None,
+        manage_ticks=None,
+        # manage_xticks is for compatibility of matplotlib < 3.1.0.
+        # Remove this when minimum version is 3.0.0
+        manage_xticks=None,
+        autorange=False,
+        zorder=None,
+        precision=None,
+    ):
+        def update_dict(dictionary, rc_name, properties):
+            """ Loads properties in the dictionary from rc file if not already
+            in the dictionary"""
+            rc_str = "boxplot.{0}.{1}"
+            if dictionary is None:
+                dictionary = dict()
+            for prop_dict in properties:
+                dictionary.setdefault(prop_dict, mat.rcParams[rc_str.format(rc_name, prop_dict)])
+            return dictionary
+
+        # Common property dictionaries loading from rc
+        flier_props = [
+            "color",
+            "marker",
+            "markerfacecolor",
+            "markeredgecolor",
+            "markersize",
+            "linestyle",
+            "linewidth",
+        ]
+        default_props = ["color", "linewidth", "linestyle"]
+
+        boxprops = update_dict(boxprops, "boxprops", default_props)
+        whiskerprops = update_dict(whiskerprops, "whiskerprops", default_props)
+        capprops = update_dict(capprops, "capprops", default_props)
+        medianprops = update_dict(medianprops, "medianprops", default_props)
+        meanprops = update_dict(meanprops, "meanprops", default_props)
+        flierprops = update_dict(flierprops, "flierprops", flier_props)
+
+        if patch_artist:
+            boxprops["linestyle"] = "solid"
+            boxprops["edgecolor"] = boxprops.pop("color")
+
+        # if non-default sym value, put it into the flier dictionary
+        # the logic for providing the default symbol ('b+') now lives
+        # in bxp in the initial value of final_flierprops
+        # handle all of the `sym` related logic here so we only have to pass
+        # on the flierprops dict.
+        if sym is not None:
+            # no-flier case, which should really be done with
+            # 'showfliers=False' but none-the-less deal with it to keep back
+            # compatibility
+            if sym == "":
+                # blow away existing dict and make one for invisible markers
+                flierprops = dict(linestyle="none", marker="", color="none")
+                # turn the fliers off just to be safe
+                showfliers = False
+            # now process the symbol string
+            else:
+                # process the symbol string
+                # discarded linestyle
+                _, marker, color = _process_plot_format(sym)
+                # if we have a marker, use it
+                if marker is not None:
+                    flierprops["marker"] = marker
+                # if we have a color, use it
+                if color is not None:
+                    # assume that if color is passed in the user want
+                    # filled symbol, if the users want more control use
+                    # flierprops
+                    flierprops["color"] = color
+                    flierprops["markerfacecolor"] = color
+                    flierprops["markeredgecolor"] = color
+
+        # replace medians if necessary:
+        if usermedians is not None:
+            if len(np.ravel(usermedians)) != len(bxpstats) or np.shape(usermedians)[0] != len(
+                bxpstats
+            ):
+                raise ValueError("usermedians length not compatible with x")
+            else:
+                # reassign medians as necessary
+                for stats, med in zip(bxpstats, usermedians):
+                    if med is not None:
+                        stats["med"] = med
+
+        if conf_intervals is not None:
+            if np.shape(conf_intervals)[0] != len(bxpstats):
+                err_mess = "conf_intervals length not compatible with x"
+                raise ValueError(err_mess)
+            else:
+                for stats, ci in zip(bxpstats, conf_intervals):
+                    if ci is not None:
+                        if len(ci) != 2:
+                            raise ValueError("each confidence interval must " "have two values")
+                        else:
+                            if ci[0] is not None:
+                                stats["cilo"] = ci[0]
+                            if ci[1] is not None:
+                                stats["cihi"] = ci[1]
+
+        should_manage_ticks = True
+        if manage_xticks is not None:
+            should_manage_ticks = manage_xticks
+        if manage_ticks is not None:
+            should_manage_ticks = manage_ticks
+
+        if LooseVersion(mat.__version__) < LooseVersion("3.1.0"):
+            extra_args = {"manage_xticks": should_manage_ticks}
+        else:
+            extra_args = {"manage_ticks": should_manage_ticks}
+
+        artists = ax.bxp(
+            bxpstats,
+            positions=positions,
+            widths=widths,
+            vert=vert,
+            patch_artist=patch_artist,
+            shownotches=notch,
+            showmeans=showmeans,
+            showcaps=showcaps,
+            showbox=showbox,
+            boxprops=boxprops,
+            flierprops=flierprops,
+            medianprops=medianprops,
+            meanprops=meanprops,
+            meanline=meanline,
+            showfliers=showfliers,
+            capprops=capprops,
+            whiskerprops=whiskerprops,
+            zorder=zorder,
+            **extra_args,
+        )
+        return artists
+
+    def _plot(self, ax, bxpstats, column_num=None, return_type="axes", **kwds):
+        bp = self.boxplot(ax, bxpstats, **kwds)
+
+        if return_type == "dict":
+            return bp, bp
+        elif return_type == "both":
+            return self.BP(ax=ax, lines=bp), bp
+        else:
+            return ax, bp
+
+    def _compute_plot_data(self):
+        colname = self.data.name
+        spark_column_name = self.data._internal.spark_column_name_for(self.data._column_label)
+        data = self.data
+
+        # Updates all props with the rc defaults from matplotlib
+        self.kwds.update(PandasOnSparkBoxPlot.rc_defaults(**self.kwds))
+
+        # Gets some important kwds
+        showfliers = self.kwds.get("showfliers", False)
+        whis = self.kwds.get("whis", 1.5)
+        labels = self.kwds.get("labels", [colname])
+
+        # This one is pandas-on-Spark specific to control precision for approx_percentile
+        precision = self.kwds.get("precision", 0.01)
+
+        # # Computes mean, median, Q1 and Q3 with approx_percentile and precision
+        col_stats, col_fences = BoxPlotBase.compute_stats(data, spark_column_name, whis, precision)
+
+        # # Creates a column to flag rows as outliers or not
+        outliers = BoxPlotBase.outliers(data, spark_column_name, *col_fences)
+
+        # # Computes min and max values of non-outliers - the whiskers
+        whiskers = BoxPlotBase.calc_whiskers(spark_column_name, outliers)
+
+        if showfliers:
+            fliers = BoxPlotBase.get_fliers(spark_column_name, outliers, whiskers[0])
+        else:
+            fliers = []
+
+        # Builds bxpstats dict
+        stats = []
+        item = {
+            "mean": col_stats["mean"],
+            "med": col_stats["med"],
+            "q1": col_stats["q1"],
+            "q3": col_stats["q3"],
+            "whislo": whiskers[0],
+            "whishi": whiskers[1],
+            "fliers": fliers,
+            "label": labels[0],
+        }
+        stats.append(item)
+
+        self.data = {labels[0]: stats}
+
+    def _make_plot(self):
+        bxpstats = list(self.data.values())[0]
+        ax = self._get_ax(0)
+        kwds = self.kwds.copy()
+
+        for stats in bxpstats:
+            if len(stats["fliers"]) > 1000:
+                stats["fliers"] = stats["fliers"][:1000]
+                ax.text(
+                    1,
+                    1,
+                    "showing top 1,000 fliers only",
+                    size=6,
+                    ha="right",
+                    va="bottom",
+                    transform=ax.transAxes,
+                )
+
+        ret, bp = self._plot(ax, bxpstats, column_num=0, return_type=self.return_type, **kwds)
+        self.maybe_color_bp(bp)
+        self._return_obj = ret
+
+        labels = [l for l, _ in self.data.items()]
+        labels = [pprint_thing(l) for l in labels]
+        if not self.use_index:
+            labels = [pprint_thing(key) for key in range(len(labels))]
+        self._set_ticklabels(ax, labels)
+
+    @staticmethod
+    def rc_defaults(
+        notch=None,
+        vert=None,
+        whis=None,
+        patch_artist=None,
+        bootstrap=None,
+        meanline=None,
+        showmeans=None,
+        showcaps=None,
+        showbox=None,
+        showfliers=None,
+        **kwargs
+    ):
+        # Missing arguments default to rcParams.
+        if whis is None:
+            whis = mat.rcParams["boxplot.whiskers"]
+        if bootstrap is None:
+            bootstrap = mat.rcParams["boxplot.bootstrap"]
+
+        if notch is None:
+            notch = mat.rcParams["boxplot.notch"]
+        if vert is None:
+            vert = mat.rcParams["boxplot.vertical"]
+        if patch_artist is None:
+            patch_artist = mat.rcParams["boxplot.patchartist"]
+        if meanline is None:
+            meanline = mat.rcParams["boxplot.meanline"]
+        if showmeans is None:
+            showmeans = mat.rcParams["boxplot.showmeans"]
+        if showcaps is None:
+            showcaps = mat.rcParams["boxplot.showcaps"]
+        if showbox is None:
+            showbox = mat.rcParams["boxplot.showbox"]
+        if showfliers is None:
+            showfliers = mat.rcParams["boxplot.showfliers"]
+
+        return dict(
+            whis=whis,
+            bootstrap=bootstrap,
+            notch=notch,
+            vert=vert,
+            patch_artist=patch_artist,
+            meanline=meanline,
+            showmeans=showmeans,
+            showcaps=showcaps,
+            showbox=showbox,
+            showfliers=showfliers,
+        )
+
+
+class PandasOnSparkHistPlot(PandasHistPlot, HistogramPlotBase):
+    def _args_adjust(self):
+        if is_list_like(self.bottom):
+            self.bottom = np.array(self.bottom)
+
+    def _compute_plot_data(self):
+        self.data, self.bins = HistogramPlotBase.prepare_hist_data(self.data, self.bins)
+
+    def _make_plot(self):
+        # TODO: this logic is similar with KdePlot. Might have to deduplicate it.
+        # 'num_colors' requires to calculate `shape` which has to count all.
+        # Use 1 for now to save the computation.
+        colors = self._get_colors(num_colors=1)
+        stacking_id = self._get_stacking_id()
+        output_series = HistogramPlotBase.compute_hist(self.data, self.bins)
+
+        for (i, label), y in zip(enumerate(self.data._internal.column_labels), output_series):
+            ax = self._get_ax(i)
+
+            kwds = self.kwds.copy()
+
+            label = pprint_thing(label if len(label) > 1 else label[0])
+            kwds["label"] = label
+
+            style, kwds = self._apply_style_colors(colors, kwds, i, label)
+            if style is not None:
+                kwds["style"] = style
+
+            kwds = self._make_plot_keywords(kwds, y)
+            artists = self._plot(ax, y, column_num=i, stacking_id=stacking_id, **kwds)
+            self._add_legend_handle(artists[0], label, index=i)
+
+    @classmethod
+    def _plot(cls, ax, y, style=None, bins=None, bottom=0, column_num=0, stacking_id=None, **kwds):
+        if column_num == 0:
+            cls._initialize_stacker(ax, stacking_id, len(bins) - 1)
+
+        base = np.zeros(len(bins) - 1)
+        bottom = bottom + cls._get_stacked_values(ax, stacking_id, base, kwds["label"])
+
+        # Since the counts were computed already, we use them as weights and just generate
+        # one entry for each bin
+        n, bins, patches = ax.hist(bins[:-1], bins=bins, bottom=bottom, weights=y, **kwds)
+
+        cls._update_stacker(ax, stacking_id, n)
+        return patches
+
+
+class PandasOnSparkPiePlot(PandasPiePlot, TopNPlotBase):
+    def __init__(self, data, **kwargs):
+        super().__init__(self.get_top_n(data), **kwargs)
+
+    def _make_plot(self):
+        self.set_result_text(self._get_ax(0))
+        super()._make_plot()
+
+
+class PandasOnSparkAreaPlot(PandasAreaPlot, SampledPlotBase):
+    def __init__(self, data, **kwargs):
+        super().__init__(self.get_sampled(data), **kwargs)
+
+    def _make_plot(self):
+        self.set_result_text(self._get_ax(0))
+        super()._make_plot()
+
+
+class PandasOnSparkLinePlot(PandasLinePlot, SampledPlotBase):
+    def __init__(self, data, **kwargs):
+        super().__init__(self.get_sampled(data), **kwargs)
+
+    def _make_plot(self):
+        self.set_result_text(self._get_ax(0))
+        super()._make_plot()
+
+
+class PandasOnSparkBarhPlot(PandasBarhPlot, TopNPlotBase):
+    def __init__(self, data, **kwargs):
+        super().__init__(self.get_top_n(data), **kwargs)
+
+    def _make_plot(self):
+        self.set_result_text(self._get_ax(0))
+        super()._make_plot()
+
+
+class PandasOnSparkScatterPlot(PandasScatterPlot, TopNPlotBase):
+    def __init__(self, data, x, y, **kwargs):
+        super().__init__(self.get_top_n(data), x, y, **kwargs)
+
+    def _make_plot(self):
+        self.set_result_text(self._get_ax(0))
+        super()._make_plot()
+
+
+class PandasOnSparkKdePlot(PandasKdePlot, KdePlotBase):
+    def _compute_plot_data(self):
+        self.data = KdePlotBase.prepare_kde_data(self.data)
+
+    def _make_plot(self):
+        # 'num_colors' requires to calculate `shape` which has to count all.
+        # Use 1 for now to save the computation.
+        colors = self._get_colors(num_colors=1)
+        stacking_id = self._get_stacking_id()
+
+        sdf = self.data._internal.spark_frame
+
+        for i, label in enumerate(self.data._internal.column_labels):
+            # 'y' is a Spark DataFrame that selects one column.
+            y = sdf.select(self.data._internal.spark_column_for(label))
+            ax = self._get_ax(i)
+
+            kwds = self.kwds.copy()
+
+            label = pprint_thing(label if len(label) > 1 else label[0])
+            kwds["label"] = label
+
+            style, kwds = self._apply_style_colors(colors, kwds, i, label)
+            if style is not None:
+                kwds["style"] = style
+
+            kwds = self._make_plot_keywords(kwds, y)
+            artists = self._plot(ax, y, column_num=i, stacking_id=stacking_id, **kwds)
+            self._add_legend_handle(artists[0], label, index=i)
+
+    def _get_ind(self, y):
+        return KdePlotBase.get_ind(y, self.ind)
+
+    @classmethod
+    def _plot(
+        cls, ax, y, style=None, bw_method=None, ind=None, column_num=None, stacking_id=None, **kwds
+    ):
+        y = KdePlotBase.compute_kde(y, bw_method=bw_method, ind=ind)
+        lines = PandasMPLPlot._plot(ax, ind, y, style=style, **kwds)
+        return lines
+
+
+_klasses = [
+    PandasOnSparkHistPlot,
+    PandasOnSparkBarPlot,
+    PandasOnSparkBoxPlot,
+    PandasOnSparkPiePlot,
+    PandasOnSparkAreaPlot,
+    PandasOnSparkLinePlot,
+    PandasOnSparkBarhPlot,
+    PandasOnSparkScatterPlot,
+    PandasOnSparkKdePlot,
+]
+_plot_klass = {getattr(klass, "_kind"): klass for klass in _klasses}
+_common_kinds = {"area", "bar", "barh", "box", "hist", "kde", "line", "pie"}
+_series_kinds = _common_kinds.union(set())
+_dataframe_kinds = _common_kinds.union({"scatter", "hexbin"})
+_pandas_on_spark_all_kinds = _common_kinds.union(_series_kinds).union(_dataframe_kinds)
+
+
+def plot_pandas_on_spark(data, kind, **kwargs):
+    if kind not in _pandas_on_spark_all_kinds:
+        raise ValueError("{} is not a valid plot kind".format(kind))
+
+    from pyspark.pandas import DataFrame, Series
+
+    if isinstance(data, Series):
+        if kind not in _series_kinds:
+            return unsupported_function(class_name="pd.Series", method_name=kind)()
+        return plot_series(data=data, kind=kind, **kwargs)
+    elif isinstance(data, DataFrame):
+        if kind not in _dataframe_kinds:
+            return unsupported_function(class_name="pd.DataFrame", method_name=kind)()
+        return plot_frame(data=data, kind=kind, **kwargs)
+
+
+def plot_series(
+    data,
+    kind="line",
+    ax=None,  # Series unique
+    figsize=None,
+    use_index=True,
+    title=None,
+    grid=None,
+    legend=False,
+    style=None,
+    logx=False,
+    logy=False,
+    loglog=False,
+    xticks=None,
+    yticks=None,
+    xlim=None,
+    ylim=None,
+    rot=None,
+    fontsize=None,
+    colormap=None,
+    table=False,
+    yerr=None,
+    xerr=None,
+    label=None,
+    secondary_y=False,  # Series unique
+    **kwds
+):
+    """
+    Make plots of Series using matplotlib / pylab.
+
+    Each plot kind has a corresponding method on the
+    ``Series.plot`` accessor:
+    ``s.plot(kind='line')`` is equivalent to
+    ``s.plot.line()``.
+
+    Parameters
+    ----------
+    data : Series
+
+    kind : str
+        - 'line' : line plot (default)
+        - 'bar' : vertical bar plot
+        - 'barh' : horizontal bar plot
+        - 'hist' : histogram
+        - 'box' : boxplot
+        - 'kde' : Kernel Density Estimation plot
+        - 'density' : same as 'kde'
+        - 'area' : area plot
+        - 'pie' : pie plot
+
+    ax : matplotlib axes object
+        If not passed, uses gca()
+    figsize : a tuple (width, height) in inches
+    use_index : boolean, default True
+        Use index as ticks for x axis
+    title : string or list
+        Title to use for the plot. If a string is passed, print the string at
+        the top of the figure. If a list is passed and `subplots` is True,
+        print each item in the list above the corresponding subplot.
+    grid : boolean, default None (matlab style default)
+        Axis grid lines
+    legend : False/True/'reverse'
+        Place legend on axis subplots
+    style : list or dict
+        matplotlib line style per column
+    logx : boolean, default False
+        Use log scaling on x axis
+    logy : boolean, default False
+        Use log scaling on y axis
+    loglog : boolean, default False
+        Use log scaling on both x and y axes
+    xticks : sequence
+        Values to use for the xticks
+    yticks : sequence
+        Values to use for the yticks
+    xlim : 2-tuple/list
+    ylim : 2-tuple/list
+    rot : int, default None
+        Rotation for ticks (xticks for vertical, yticks for horizontal plots)
+    fontsize : int, default None
+        Font size for xticks and yticks
+    colormap : str or matplotlib colormap object, default None
+        Colormap to select colors from. If string, load colormap with that name
+        from matplotlib.
+    colorbar : boolean, optional
+        If True, plot colorbar (only relevant for 'scatter' and 'hexbin' plots)
+    position : float
+        Specify relative alignments for bar plot layout.
+        From 0 (left/bottom-end) to 1 (right/top-end). Default is 0.5 (center)
+    table : boolean, Series or DataFrame, default False
+        If True, draw a table using the data in the DataFrame and the data will
+        be transposed to meet matplotlib's default layout.
+        If a Series or DataFrame is passed, use passed data to draw a table.
+    yerr : DataFrame, Series, array-like, dict and str
+        See :ref:`Plotting with Error Bars <visualization.errorbars>` for
+        detail.
+    xerr : same types as yerr.
+    label : label argument to provide to plot
+    secondary_y : boolean or sequence of ints, default False
+        If True then y-axis will be on the right
+    mark_right : boolean, default True
+        When using a secondary_y axis, automatically mark the column
+        labels with "(right)" in the legend
+    **kwds : keywords
+        Options to pass to matplotlib plotting method
+
+    Returns
+    -------
+    axes : :class:`matplotlib.axes.Axes` or numpy.ndarray of them
+
+    Notes
+    -----
+
+    - See matplotlib documentation online for more on this subject
+    - If `kind` = 'bar' or 'barh', you can specify relative alignments
+      for bar plot layout by `position` keyword.
+      From 0 (left/bottom-end) to 1 (right/top-end). Default is 0.5 (center)
+    """
+
+    # function copied from pandas.plotting._core
+    # so it calls modified _plot below
+
+    import matplotlib.pyplot as plt
+
+    if ax is None and len(plt.get_fignums()) > 0:
+        with plt.rc_context():
+            ax = plt.gca()
+        ax = PandasMPLPlot._get_ax_layer(ax)
+    return _plot(
+        data,
+        kind=kind,
+        ax=ax,
+        figsize=figsize,
+        use_index=use_index,
+        title=title,
+        grid=grid,
+        legend=legend,
+        style=style,
+        logx=logx,
+        logy=logy,
+        loglog=loglog,
+        xticks=xticks,
+        yticks=yticks,
+        xlim=xlim,
+        ylim=ylim,
+        rot=rot,
+        fontsize=fontsize,
+        colormap=colormap,
+        table=table,
+        yerr=yerr,
+        xerr=xerr,
+        label=label,
+        secondary_y=secondary_y,
+        **kwds,
+    )
+
+
+def plot_frame(
+    data,
+    x=None,
+    y=None,
+    kind="line",
+    ax=None,
+    subplots=None,
+    sharex=None,
+    sharey=False,
+    layout=None,
+    figsize=None,
+    use_index=True,
+    title=None,
+    grid=None,
+    legend=True,
+    style=None,
+    logx=False,
+    logy=False,
+    loglog=False,
+    xticks=None,
+    yticks=None,
+    xlim=None,
+    ylim=None,
+    rot=None,
+    fontsize=None,
+    colormap=None,
+    table=False,
+    yerr=None,
+    xerr=None,
+    secondary_y=False,
+    sort_columns=False,
+    **kwds
+):
+    """
+    Make plots of DataFrames using matplotlib / pylab.
+
+    Each plot kind has a corresponding method on the
+    ``DataFrame.plot`` accessor:
+    ``kdf.plot(kind='line')`` is equivalent to
+    ``kdf.plot.line()``.
+
+    Parameters
+    ----------
+    data : DataFrame
+
+    kind : str
+        - 'line' : line plot (default)
+        - 'bar' : vertical bar plot
+        - 'barh' : horizontal bar plot
+        - 'hist' : histogram
+        - 'box' : boxplot
+        - 'kde' : Kernel Density Estimation plot
+        - 'density' : same as 'kde'
+        - 'area' : area plot
+        - 'pie' : pie plot
+        - 'scatter' : scatter plot
+    ax : matplotlib axes object
+        If not passed, uses gca()
+    x : label or position, default None
+    y : label, position or list of label, positions, default None
+        Allows plotting of one column versus another.
+    figsize : a tuple (width, height) in inches
+    use_index : boolean, default True
+        Use index as ticks for x axis
+    title : string or list
+        Title to use for the plot. If a string is passed, print the string at
+        the top of the figure. If a list is passed and `subplots` is True,
+        print each item in the list above the corresponding subplot.
+    grid : boolean, default None (matlab style default)
+        Axis grid lines
+    legend : False/True/'reverse'
+        Place legend on axis subplots
+    style : list or dict
+        matplotlib line style per column
+    logx : boolean, default False
+        Use log scaling on x axis
+    logy : boolean, default False
+        Use log scaling on y axis
+    loglog : boolean, default False
+        Use log scaling on both x and y axes
+    xticks : sequence
+        Values to use for the xticks
+    yticks : sequence
+        Values to use for the yticks
+    xlim : 2-tuple/list
+    ylim : 2-tuple/list
+    sharex: bool or None, default is None
+        Whether to share x axis or not.
+    sharey: bool, default is False
+        Whether to share y axis or not.
+    rot : int, default None
+        Rotation for ticks (xticks for vertical, yticks for horizontal plots)
+    fontsize : int, default None
+        Font size for xticks and yticks
+    colormap : str or matplotlib colormap object, default None
+        Colormap to select colors from. If string, load colormap with that name
+        from matplotlib.
+    colorbar : boolean, optional
+        If True, plot colorbar (only relevant for 'scatter' and 'hexbin' plots)
+    position : float
+        Specify relative alignments for bar plot layout.
+        From 0 (left/bottom-end) to 1 (right/top-end). Default is 0.5 (center)
+    table : boolean, Series or DataFrame, default False
+        If True, draw a table using the data in the DataFrame and the data will
+        be transposed to meet matplotlib's default layout.
+        If a Series or DataFrame is passed, use passed data to draw a table.
+    yerr : DataFrame, Series, array-like, dict and str
+        See :ref:`Plotting with Error Bars <visualization.errorbars>` for
+        detail.
+    xerr : same types as yerr.
+    label : label argument to provide to plot
+    secondary_y : boolean or sequence of ints, default False
+        If True then y-axis will be on the right
+    mark_right : boolean, default True
+        When using a secondary_y axis, automatically mark the column
+        labels with "(right)" in the legend
+    sort_columns: bool, default is False
+        When True, will sort values on plots.
+    **kwds : keywords
+        Options to pass to matplotlib plotting method
+
+    Returns
+    -------
+    axes : :class:`matplotlib.axes.Axes` or numpy.ndarray of them
+
+    Notes
+    -----
+
+    - See matplotlib documentation online for more on this subject
+    - If `kind` = 'bar' or 'barh', you can specify relative alignments
+      for bar plot layout by `position` keyword.
+      From 0 (left/bottom-end) to 1 (right/top-end). Default is 0.5 (center)
+    """
+
+    return _plot(
+        data,
+        kind=kind,
+        x=x,
+        y=y,
+        ax=ax,
+        figsize=figsize,
+        use_index=use_index,
+        title=title,
+        grid=grid,
+        legend=legend,
+        subplots=subplots,
+        style=style,
+        logx=logx,
+        logy=logy,
+        loglog=loglog,
+        xticks=xticks,
+        yticks=yticks,
+        xlim=xlim,
+        ylim=ylim,
+        rot=rot,
+        fontsize=fontsize,
+        colormap=colormap,
+        table=table,
+        yerr=yerr,
+        xerr=xerr,
+        sharex=sharex,
+        sharey=sharey,
+        secondary_y=secondary_y,
+        layout=layout,
+        sort_columns=sort_columns,
+        **kwds,
+    )
+
+
+def _plot(data, x=None, y=None, subplots=False, ax=None, kind="line", **kwds):
+    from pyspark.pandas import DataFrame
+
+    # function copied from pandas.plotting._core
+    # and adapted to handle pandas-on-Spark DataFrame and Series
+
+    kind = kind.lower().strip()
+    kind = {"density": "kde"}.get(kind, kind)
+    if kind in _all_kinds:
+        klass = _plot_klass[kind]
+    else:
+        raise ValueError("%r is not a valid plot kind" % kind)
+
+    # scatter and hexbin are inherited from PlanePlot which require x and y
+    if kind in ("scatter", "hexbin"):
+        plot_obj = klass(data, x, y, subplots=subplots, ax=ax, kind=kind, **kwds)
+    else:
+
+        # check data type and do preprocess before applying plot
+        if isinstance(data, DataFrame):
+            if x is not None:
+                data = data.set_index(x)
+            # TODO: check if value of y is plottable
+            if y is not None:
+                data = data[y]
+
+        plot_obj = klass(data, subplots=subplots, ax=ax, kind=kind, **kwds)
+    plot_obj.generate()
+    plot_obj.draw()
+    return plot_obj.result
diff --git a/python/pyspark/pandas/plot/plotly.py b/python/pyspark/pandas/plot/plotly.py
new file mode 100644
index 0000000000000..c27b3b3f9a39b
--- /dev/null
+++ b/python/pyspark/pandas/plot/plotly.py
@@ -0,0 +1,213 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from typing import TYPE_CHECKING, Union
+
+import pandas as pd
+
+from pyspark.pandas.plot import (
+    HistogramPlotBase,
+    name_like_string,
+    PandasOnSparkPlotAccessor,
+    BoxPlotBase,
+    KdePlotBase,
+)
+
+if TYPE_CHECKING:
+    import pyspark.pandas as ps  # noqa: F401 (SPARK-34943)
+
+
+def plot_pandas_on_spark(
+        data: Union["ps.DataFrame", "ps.Series"], kind: str, **kwargs):
+    import plotly
+
+    # pandas-on-Spark specific plots
+    if kind == "pie":
+        return plot_pie(data, **kwargs)
+    if kind == "hist":
+        return plot_histogram(data, **kwargs)
+    if kind == "box":
+        return plot_box(data, **kwargs)
+    if kind == "kde" or kind == "density":
+        return plot_kde(data, **kwargs)
+
+    # Other plots.
+    return plotly.plot(PandasOnSparkPlotAccessor.pandas_plot_data_map[kind](data), kind, **kwargs)
+
+
+def plot_pie(data: Union["ps.DataFrame", "ps.Series"], **kwargs):
+    from plotly import express
+
+    data = PandasOnSparkPlotAccessor.pandas_plot_data_map["pie"](data)
+
+    if isinstance(data, pd.Series):
+        pdf = data.to_frame()
+        return express.pie(pdf, values=pdf.columns[0], names=pdf.index, **kwargs)
+    elif isinstance(data, pd.DataFrame):
+        values = kwargs.pop("y", None)
+        default_names = None
+        if values is not None:
+            default_names = data.index
+
+        return express.pie(
+            data,
+            values=kwargs.pop("values", values),
+            names=kwargs.pop("names", default_names),
+            **kwargs,
+        )
+    else:
+        raise RuntimeError("Unexpected type: [%s]" % type(data))
+
+
+def plot_histogram(data: Union["ps.DataFrame", "ps.Series"], **kwargs):
+    import plotly.graph_objs as go
+
+    bins = kwargs.get("bins", 10)
+    kdf, bins = HistogramPlotBase.prepare_hist_data(data, bins)
+    assert len(bins) > 2, "the number of buckets must be higher than 2."
+    output_series = HistogramPlotBase.compute_hist(kdf, bins)
+    prev = float("%.9f" % bins[0])  # to make it prettier, truncate.
+    text_bins = []
+    for b in bins[1:]:
+        norm_b = float("%.9f" % b)
+        text_bins.append("[%s, %s)" % (prev, norm_b))
+        prev = norm_b
+    text_bins[-1] = text_bins[-1][:-1] + "]"  # replace ) to ] for the last bucket.
+
+    bins = 0.5 * (bins[:-1] + bins[1:])
+
+    output_series = list(output_series)
+    bars = []
+    for series in output_series:
+        bars.append(
+            go.Bar(
+                x=bins,
+                y=series,
+                name=name_like_string(series.name),
+                text=text_bins,
+                hovertemplate=(
+                    "variable=" + name_like_string(series.name) + "<br>value=%{text}<br>count=%{y}"
+                ),
+            )
+        )
+
+    fig = go.Figure(data=bars, layout=go.Layout(barmode="stack"))
+    fig["layout"]["xaxis"]["title"] = "value"
+    fig["layout"]["yaxis"]["title"] = "count"
+    return fig
+
+
+def plot_box(data: Union["ps.DataFrame", "ps.Series"], **kwargs):
+    import plotly.graph_objs as go
+    import pyspark.pandas as ps
+
+    if isinstance(data, ps.DataFrame):
+        raise RuntimeError(
+            "plotly does not support a box plot with pandas-on-Spark DataFrame. Use Series instead."
+        )
+
+    # 'whis' isn't actually an argument in plotly (but in matplotlib). But seems like
+    # plotly doesn't expose the reach of the whiskers to the beyond the first and
+    # third quartiles (?). Looks they use default 1.5.
+    whis = kwargs.pop("whis", 1.5)
+    # 'precision' is pandas-on-Spark specific to control precision for approx_percentile
+    precision = kwargs.pop("precision", 0.01)
+
+    # Plotly options
+    boxpoints = kwargs.pop("boxpoints", "suspectedoutliers")
+    notched = kwargs.pop("notched", False)
+    if boxpoints not in ["suspectedoutliers", False]:
+        raise ValueError(
+            "plotly plotting backend does not support 'boxpoints' set to '%s'. "
+            "Set to 'suspectedoutliers' or False." % boxpoints
+        )
+    if notched:
+        raise ValueError(
+            "plotly plotting backend does not support 'notched' set to '%s'. "
+            "Set to False." % notched
+        )
+
+    colname = name_like_string(data.name)
+    spark_column_name = data._internal.spark_column_name_for(data._column_label)
+
+    # Computes mean, median, Q1 and Q3 with approx_percentile and precision
+    col_stats, col_fences = BoxPlotBase.compute_stats(data, spark_column_name, whis, precision)
+
+    # Creates a column to flag rows as outliers or not
+    outliers = BoxPlotBase.outliers(data, spark_column_name, *col_fences)
+
+    # Computes min and max values of non-outliers - the whiskers
+    whiskers = BoxPlotBase.calc_whiskers(spark_column_name, outliers)
+
+    fliers = None
+    if boxpoints:
+        fliers = BoxPlotBase.get_fliers(spark_column_name, outliers, whiskers[0])
+        fliers = [fliers] if len(fliers) > 0 else None
+
+    fig = go.Figure()
+    fig.add_trace(
+        go.Box(
+            name=colname,
+            q1=[col_stats["q1"]],
+            median=[col_stats["med"]],
+            q3=[col_stats["q3"]],
+            mean=[col_stats["mean"]],
+            lowerfence=[whiskers[0]],
+            upperfence=[whiskers[1]],
+            y=fliers,
+            boxpoints=boxpoints,
+            notched=notched,
+            **kwargs,  # this is for workarounds. Box takes different options from express.box.
+        )
+    )
+    fig["layout"]["xaxis"]["title"] = colname
+    fig["layout"]["yaxis"]["title"] = "value"
+    return fig
+
+
+def plot_kde(data: Union["ps.DataFrame", "ps.Series"], **kwargs):
+    from plotly import express
+    import pyspark.pandas as ps
+
+    if isinstance(data, ps.DataFrame) and "color" not in kwargs:
+        kwargs["color"] = "names"
+
+    kdf = KdePlotBase.prepare_kde_data(data)
+    sdf = kdf._internal.spark_frame
+    data_columns = kdf._internal.data_spark_columns
+    ind = KdePlotBase.get_ind(sdf.select(*data_columns), kwargs.pop("ind", None))
+    bw_method = kwargs.pop("bw_method", None)
+
+    pdfs = []
+    for label in kdf._internal.column_labels:
+        pdfs.append(
+            pd.DataFrame(
+                {
+                    "Density": KdePlotBase.compute_kde(
+                        sdf.select(kdf._internal.spark_column_for(label)),
+                        ind=ind,
+                        bw_method=bw_method,
+                    ),
+                    "names": name_like_string(label),
+                    "index": ind,
+                }
+            )
+        )
+    pdf = pd.concat(pdfs)
+
+    fig = express.line(pdf, x="index", y="Density", **kwargs)
+    fig["layout"]["xaxis"]["title"] = None
+    return fig
diff --git a/python/pyspark/pandas/series.py b/python/pyspark/pandas/series.py
new file mode 100644
index 0000000000000..47c012cd55c33
--- /dev/null
+++ b/python/pyspark/pandas/series.py
@@ -0,0 +1,6256 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+A wrapper class for Spark Column to behave similar to pandas Series.
+"""
+import datetime
+import re
+import inspect
+import sys
+import warnings
+from collections.abc import Mapping
+from functools import partial, wraps, reduce
+from typing import Any, Generic, Iterable, List, Optional, Tuple, TypeVar, Union, cast
+
+import numpy as np
+import pandas as pd
+from pandas.core.accessor import CachedAccessor
+from pandas.io.formats.printing import pprint_thing
+from pandas.api.types import is_list_like, is_hashable
+from pandas.api.extensions import ExtensionDtype
+from pandas.tseries.frequencies import DateOffset
+from pyspark import sql as spark
+from pyspark.sql import functions as F, Column
+from pyspark.sql.types import (
+    BooleanType,
+    DoubleType,
+    FloatType,
+    IntegerType,
+    LongType,
+    NumericType,
+    StructType,
+    IntegralType,
+    ArrayType,
+)
+from pyspark.sql.window import Window
+
+from pyspark import pandas as ps  # For running doctests and reference resolution in PyCharm.
+from pyspark.pandas.accessors import PandasOnSparkSeriesMethods
+from pyspark.pandas.categorical import CategoricalAccessor
+from pyspark.pandas.config import get_option
+from pyspark.pandas.base import IndexOpsMixin
+from pyspark.pandas.exceptions import SparkPandasIndexingError
+from pyspark.pandas.frame import DataFrame
+from pyspark.pandas.generic import Frame
+from pyspark.pandas.internal import (
+    InternalFrame,
+    DEFAULT_SERIES_NAME,
+    NATURAL_ORDER_COLUMN_NAME,
+    SPARK_DEFAULT_INDEX_NAME,
+    SPARK_DEFAULT_SERIES_NAME,
+)
+from pyspark.pandas.missing.series import MissingPandasLikeSeries
+from pyspark.pandas.plot import PandasOnSparkPlotAccessor
+from pyspark.pandas.ml import corr
+from pyspark.pandas.utils import (
+    combine_frames,
+    is_name_like_tuple,
+    is_name_like_value,
+    name_like_string,
+    same_anchor,
+    scol_for,
+    sql_conf,
+    validate_arguments_and_invoke_function,
+    validate_axis,
+    validate_bool_kwarg,
+    verify_temp_column_name,
+    SPARK_CONF_ARROW_ENABLED,
+)
+from pyspark.pandas.datetimes import DatetimeMethods
+from pyspark.pandas.spark.accessors import SparkSeriesMethods
+from pyspark.pandas.strings import StringMethods
+from pyspark.pandas.typedef import (
+    infer_return_type,
+    spark_type_to_pandas_dtype,
+    ScalarType,
+    Scalar,
+    SeriesType,
+)
+
+
+# This regular expression pattern is complied and defined here to avoid to compile the same
+# pattern every time it is used in _repr_ in Series.
+# This pattern basically seeks the footer string from pandas'
+REPR_PATTERN = re.compile(r"Length: (?P<length>[0-9]+)")
+
+_flex_doc_SERIES = """
+Return {desc} of series and other, element-wise (binary operator `{op_name}`).
+
+Equivalent to ``{equiv}``
+
+Parameters
+----------
+other : Series or scalar value
+
+Returns
+-------
+Series
+    The result of the operation.
+
+See Also
+--------
+Series.{reverse}
+
+{series_examples}
+"""
+
+_add_example_SERIES = """
+Examples
+--------
+>>> df = ps.DataFrame({'a': [2, 2, 4, np.nan],
+...                    'b': [2, np.nan, 2, np.nan]},
+...                   index=['a', 'b', 'c', 'd'], columns=['a', 'b'])
+>>> df
+     a    b
+a  2.0  2.0
+b  2.0  NaN
+c  4.0  2.0
+d  NaN  NaN
+
+>>> df.a.add(df.b)
+a    4.0
+b    NaN
+c    6.0
+d    NaN
+dtype: float64
+
+>>> df.a.radd(df.b)
+a    4.0
+b    NaN
+c    6.0
+d    NaN
+dtype: float64
+"""
+
+_sub_example_SERIES = """
+Examples
+--------
+>>> df = ps.DataFrame({'a': [2, 2, 4, np.nan],
+...                    'b': [2, np.nan, 2, np.nan]},
+...                   index=['a', 'b', 'c', 'd'], columns=['a', 'b'])
+>>> df
+     a    b
+a  2.0  2.0
+b  2.0  NaN
+c  4.0  2.0
+d  NaN  NaN
+
+>>> df.a.subtract(df.b)
+a    0.0
+b    NaN
+c    2.0
+d    NaN
+dtype: float64
+
+>>> df.a.rsub(df.b)
+a    0.0
+b    NaN
+c   -2.0
+d    NaN
+dtype: float64
+"""
+
+_mul_example_SERIES = """
+Examples
+--------
+>>> df = ps.DataFrame({'a': [2, 2, 4, np.nan],
+...                    'b': [2, np.nan, 2, np.nan]},
+...                   index=['a', 'b', 'c', 'd'], columns=['a', 'b'])
+>>> df
+     a    b
+a  2.0  2.0
+b  2.0  NaN
+c  4.0  2.0
+d  NaN  NaN
+
+>>> df.a.multiply(df.b)
+a    4.0
+b    NaN
+c    8.0
+d    NaN
+dtype: float64
+
+>>> df.a.rmul(df.b)
+a    4.0
+b    NaN
+c    8.0
+d    NaN
+dtype: float64
+"""
+
+_div_example_SERIES = """
+Examples
+--------
+>>> df = ps.DataFrame({'a': [2, 2, 4, np.nan],
+...                    'b': [2, np.nan, 2, np.nan]},
+...                   index=['a', 'b', 'c', 'd'], columns=['a', 'b'])
+>>> df
+     a    b
+a  2.0  2.0
+b  2.0  NaN
+c  4.0  2.0
+d  NaN  NaN
+
+>>> df.a.divide(df.b)
+a    1.0
+b    NaN
+c    2.0
+d    NaN
+dtype: float64
+
+>>> df.a.rdiv(df.b)
+a    1.0
+b    NaN
+c    0.5
+d    NaN
+dtype: float64
+"""
+
+_pow_example_SERIES = """
+Examples
+--------
+>>> df = ps.DataFrame({'a': [2, 2, 4, np.nan],
+...                    'b': [2, np.nan, 2, np.nan]},
+...                   index=['a', 'b', 'c', 'd'], columns=['a', 'b'])
+>>> df
+     a    b
+a  2.0  2.0
+b  2.0  NaN
+c  4.0  2.0
+d  NaN  NaN
+
+>>> df.a.pow(df.b)
+a     4.0
+b     NaN
+c    16.0
+d     NaN
+dtype: float64
+
+>>> df.a.rpow(df.b)
+a     4.0
+b     NaN
+c    16.0
+d     NaN
+dtype: float64
+"""
+
+_mod_example_SERIES = """
+Examples
+--------
+>>> df = ps.DataFrame({'a': [2, 2, 4, np.nan],
+...                    'b': [2, np.nan, 2, np.nan]},
+...                   index=['a', 'b', 'c', 'd'], columns=['a', 'b'])
+>>> df
+     a    b
+a  2.0  2.0
+b  2.0  NaN
+c  4.0  2.0
+d  NaN  NaN
+
+>>> df.a.mod(df.b)
+a    0.0
+b    NaN
+c    0.0
+d    NaN
+dtype: float64
+
+>>> df.a.rmod(df.b)
+a    0.0
+b    NaN
+c    2.0
+d    NaN
+dtype: float64
+"""
+
+_floordiv_example_SERIES = """
+Examples
+--------
+>>> df = ps.DataFrame({'a': [2, 2, 4, np.nan],
+...                    'b': [2, np.nan, 2, np.nan]},
+...                   index=['a', 'b', 'c', 'd'], columns=['a', 'b'])
+>>> df
+     a    b
+a  2.0  2.0
+b  2.0  NaN
+c  4.0  2.0
+d  NaN  NaN
+
+>>> df.a.floordiv(df.b)
+a    1.0
+b    NaN
+c    2.0
+d    NaN
+dtype: float64
+
+>>> df.a.rfloordiv(df.b)
+a    1.0
+b    NaN
+c    0.0
+d    NaN
+dtype: float64
+"""
+
+T = TypeVar("T")
+
+# Needed to disambiguate Series.str and str type
+str_type = str
+
+
+def _create_type_for_series_type(param):
+    from pyspark.pandas.typedef import NameTypeHolder
+
+    if isinstance(param, ExtensionDtype):
+        new_class = type("NameType", (NameTypeHolder,), {})
+        new_class.tpe = param
+    else:
+        new_class = param.type if isinstance(param, np.dtype) else param
+
+    return SeriesType[new_class]
+
+
+if (3, 5) <= sys.version_info < (3, 7) and __name__ != "__main__":
+    from typing import GenericMeta  # type: ignore
+
+    old_getitem = GenericMeta.__getitem__  # type: ignore
+
+    def new_getitem(self, params):
+        if hasattr(self, "is_series"):
+            return old_getitem(self, _create_type_for_series_type(params))
+        else:
+            return old_getitem(self, params)
+
+    GenericMeta.__getitem__ = new_getitem  # type: ignore
+
+
+class Series(Frame, IndexOpsMixin, Generic[T]):
+    """
+    pandas-on-Spark Series that corresponds to pandas Series logically. This holds Spark Column
+    internally.
+
+    :ivar _internal: an internal immutable Frame to manage metadata.
+    :type _internal: InternalFrame
+    :ivar _kdf: Parent's pandas-on-Spark DataFrame
+    :type _kdf: ps.DataFrame
+
+    Parameters
+    ----------
+    data : array-like, dict, or scalar value, pandas Series
+        Contains data stored in Series
+        If data is a dict, argument order is maintained for Python 3.6
+        and later.
+        Note that if `data` is a pandas Series, other arguments should not be used.
+    index : array-like or Index (1d)
+        Values must be hashable and have the same length as `data`.
+        Non-unique index values are allowed. Will default to
+        RangeIndex (0, 1, 2, ..., n) if not provided. If both a dict and index
+        sequence are used, the index will override the keys found in the
+        dict.
+    dtype : numpy.dtype or None
+        If None, dtype will be inferred
+    copy : boolean, default False
+        Copy input data
+    """
+
+    def __init__(self, data=None, index=None, dtype=None, name=None, copy=False, fastpath=False):
+        assert data is not None
+
+        if isinstance(data, DataFrame):
+            assert dtype is None
+            assert name is None
+            assert not copy
+            assert not fastpath
+
+            self._anchor = data
+            self._col_label = index
+        else:
+            if isinstance(data, pd.Series):
+                assert index is None
+                assert dtype is None
+                assert name is None
+                assert not copy
+                assert not fastpath
+                s = data
+            else:
+                s = pd.Series(
+                    data=data, index=index, dtype=dtype, name=name, copy=copy, fastpath=fastpath
+                )
+            internal = InternalFrame.from_pandas(pd.DataFrame(s))
+            if s.name is None:
+                internal = internal.copy(column_labels=[None])
+            anchor = DataFrame(internal)
+
+            self._anchor = anchor
+            self._col_label = anchor._internal.column_labels[0]
+            object.__setattr__(anchor, "_kseries", {self._column_label: self})
+
+    @property
+    def _kdf(self) -> DataFrame:
+        return self._anchor
+
+    @property
+    def _internal(self) -> InternalFrame:
+        return self._kdf._internal.select_column(self._column_label)
+
+    @property
+    def _column_label(self) -> Tuple:
+        return self._col_label
+
+    def _update_anchor(self, kdf: DataFrame):
+        assert kdf._internal.column_labels == [self._column_label], (
+            kdf._internal.column_labels,
+            [self._column_label],
+        )
+        self._anchor = kdf
+        object.__setattr__(kdf, "_kseries", {self._column_label: self})
+
+    def _with_new_scol(self, scol: spark.Column, *, dtype=None) -> "Series":
+        """
+        Copy pandas-on-Spark Series with the new Spark Column.
+
+        :param scol: the new Spark Column
+        :return: the copied Series
+        """
+        internal = self._internal.copy(
+            data_spark_columns=[scol.alias(name_like_string(self._column_label))],
+            data_dtypes=[dtype],
+        )
+        return first_series(DataFrame(internal))
+
+    spark = CachedAccessor("spark", SparkSeriesMethods)
+
+    @property
+    def dtypes(self) -> np.dtype:
+        """Return the dtype object of the underlying data.
+
+        >>> s = ps.Series(list('abc'))
+        >>> s.dtype == s.dtypes
+        True
+        """
+        return self.dtype
+
+    @property
+    def axes(self) -> List:
+        """
+        Return a list of the row axis labels.
+
+        Examples
+        --------
+
+        >>> kser = ps.Series([1, 2, 3])
+        >>> kser.axes
+        [Int64Index([0, 1, 2], dtype='int64')]
+        """
+        return [self.index]
+
+    @property
+    def spark_type(self):
+        warnings.warn(
+            "Series.spark_type is deprecated as of Series.spark.data_type. "
+            "Please use the API instead.",
+            FutureWarning,
+        )
+        return self.spark.data_type
+
+    spark_type.__doc__ = SparkSeriesMethods.data_type.__doc__
+
+    # Arithmetic Operators
+    def add(self, other) -> "Series":
+        return self + other
+
+    add.__doc__ = _flex_doc_SERIES.format(
+        desc="Addition",
+        op_name="+",
+        equiv="series + other",
+        reverse="radd",
+        series_examples=_add_example_SERIES,
+    )
+
+    def radd(self, other) -> "Series":
+        return other + self
+
+    radd.__doc__ = _flex_doc_SERIES.format(
+        desc="Reverse Addition",
+        op_name="+",
+        equiv="other + series",
+        reverse="add",
+        series_examples=_add_example_SERIES,
+    )
+
+    def div(self, other) -> "Series":
+        return self / other
+
+    div.__doc__ = _flex_doc_SERIES.format(
+        desc="Floating division",
+        op_name="/",
+        equiv="series / other",
+        reverse="rdiv",
+        series_examples=_div_example_SERIES,
+    )
+
+    divide = div
+
+    def rdiv(self, other) -> "Series":
+        return other / self
+
+    rdiv.__doc__ = _flex_doc_SERIES.format(
+        desc="Reverse Floating division",
+        op_name="/",
+        equiv="other / series",
+        reverse="div",
+        series_examples=_div_example_SERIES,
+    )
+
+    def truediv(self, other) -> "Series":
+        return self / other
+
+    truediv.__doc__ = _flex_doc_SERIES.format(
+        desc="Floating division",
+        op_name="/",
+        equiv="series / other",
+        reverse="rtruediv",
+        series_examples=_div_example_SERIES,
+    )
+
+    def rtruediv(self, other) -> "Series":
+        return other / self
+
+    rtruediv.__doc__ = _flex_doc_SERIES.format(
+        desc="Reverse Floating division",
+        op_name="/",
+        equiv="other / series",
+        reverse="truediv",
+        series_examples=_div_example_SERIES,
+    )
+
+    def mul(self, other) -> "Series":
+        return self * other
+
+    mul.__doc__ = _flex_doc_SERIES.format(
+        desc="Multiplication",
+        op_name="*",
+        equiv="series * other",
+        reverse="rmul",
+        series_examples=_mul_example_SERIES,
+    )
+
+    multiply = mul
+
+    def rmul(self, other) -> "Series":
+        return other * self
+
+    rmul.__doc__ = _flex_doc_SERIES.format(
+        desc="Reverse Multiplication",
+        op_name="*",
+        equiv="other * series",
+        reverse="mul",
+        series_examples=_mul_example_SERIES,
+    )
+
+    def sub(self, other) -> "Series":
+        return self - other
+
+    sub.__doc__ = _flex_doc_SERIES.format(
+        desc="Subtraction",
+        op_name="-",
+        equiv="series - other",
+        reverse="rsub",
+        series_examples=_sub_example_SERIES,
+    )
+
+    subtract = sub
+
+    def rsub(self, other) -> "Series":
+        return other - self
+
+    rsub.__doc__ = _flex_doc_SERIES.format(
+        desc="Reverse Subtraction",
+        op_name="-",
+        equiv="other - series",
+        reverse="sub",
+        series_examples=_sub_example_SERIES,
+    )
+
+    def mod(self, other) -> "Series":
+        return self % other
+
+    mod.__doc__ = _flex_doc_SERIES.format(
+        desc="Modulo",
+        op_name="%",
+        equiv="series % other",
+        reverse="rmod",
+        series_examples=_mod_example_SERIES,
+    )
+
+    def rmod(self, other) -> "Series":
+        return other % self
+
+    rmod.__doc__ = _flex_doc_SERIES.format(
+        desc="Reverse Modulo",
+        op_name="%",
+        equiv="other % series",
+        reverse="mod",
+        series_examples=_mod_example_SERIES,
+    )
+
+    def pow(self, other) -> "Series":
+        return self ** other
+
+    pow.__doc__ = _flex_doc_SERIES.format(
+        desc="Exponential power of series",
+        op_name="**",
+        equiv="series ** other",
+        reverse="rpow",
+        series_examples=_pow_example_SERIES,
+    )
+
+    def rpow(self, other) -> "Series":
+        return other ** self
+
+    rpow.__doc__ = _flex_doc_SERIES.format(
+        desc="Reverse Exponential power",
+        op_name="**",
+        equiv="other ** series",
+        reverse="pow",
+        series_examples=_pow_example_SERIES,
+    )
+
+    def floordiv(self, other) -> "Series":
+        return self // other
+
+    floordiv.__doc__ = _flex_doc_SERIES.format(
+        desc="Integer division",
+        op_name="//",
+        equiv="series // other",
+        reverse="rfloordiv",
+        series_examples=_floordiv_example_SERIES,
+    )
+
+    def rfloordiv(self, other) -> "Series":
+        return other // self
+
+    rfloordiv.__doc__ = _flex_doc_SERIES.format(
+        desc="Reverse Integer division",
+        op_name="//",
+        equiv="other // series",
+        reverse="floordiv",
+        series_examples=_floordiv_example_SERIES,
+    )
+
+    # create accessor for pandas-on-Spark specific methods.
+    koalas = CachedAccessor("koalas", PandasOnSparkSeriesMethods)
+
+    # Comparison Operators
+    def eq(self, other) -> bool:
+        """
+        Compare if the current value is equal to the other.
+
+        >>> df = ps.DataFrame({'a': [1, 2, 3, 4],
+        ...                    'b': [1, np.nan, 1, np.nan]},
+        ...                   index=['a', 'b', 'c', 'd'], columns=['a', 'b'])
+
+        >>> df.a == 1
+        a     True
+        b    False
+        c    False
+        d    False
+        Name: a, dtype: bool
+
+        >>> df.b.eq(1)
+        a     True
+        b    False
+        c     True
+        d    False
+        Name: b, dtype: bool
+        """
+        return self == other
+
+    equals = eq
+
+    def gt(self, other) -> "Series":
+        """
+        Compare if the current value is greater than the other.
+
+        >>> df = ps.DataFrame({'a': [1, 2, 3, 4],
+        ...                    'b': [1, np.nan, 1, np.nan]},
+        ...                   index=['a', 'b', 'c', 'd'], columns=['a', 'b'])
+
+        >>> df.a > 1
+        a    False
+        b     True
+        c     True
+        d     True
+        Name: a, dtype: bool
+
+        >>> df.b.gt(1)
+        a    False
+        b    False
+        c    False
+        d    False
+        Name: b, dtype: bool
+        """
+        return self > other
+
+    def ge(self, other) -> "Series":
+        """
+        Compare if the current value is greater than or equal to the other.
+
+        >>> df = ps.DataFrame({'a': [1, 2, 3, 4],
+        ...                    'b': [1, np.nan, 1, np.nan]},
+        ...                   index=['a', 'b', 'c', 'd'], columns=['a', 'b'])
+
+        >>> df.a >= 2
+        a    False
+        b     True
+        c     True
+        d     True
+        Name: a, dtype: bool
+
+        >>> df.b.ge(2)
+        a    False
+        b    False
+        c    False
+        d    False
+        Name: b, dtype: bool
+        """
+        return self >= other
+
+    def lt(self, other) -> "Series":
+        """
+        Compare if the current value is less than the other.
+
+        >>> df = ps.DataFrame({'a': [1, 2, 3, 4],
+        ...                    'b': [1, np.nan, 1, np.nan]},
+        ...                   index=['a', 'b', 'c', 'd'], columns=['a', 'b'])
+
+        >>> df.a < 1
+        a    False
+        b    False
+        c    False
+        d    False
+        Name: a, dtype: bool
+
+        >>> df.b.lt(2)
+        a     True
+        b    False
+        c     True
+        d    False
+        Name: b, dtype: bool
+        """
+        return self < other
+
+    def le(self, other) -> "Series":
+        """
+        Compare if the current value is less than or equal to the other.
+
+        >>> df = ps.DataFrame({'a': [1, 2, 3, 4],
+        ...                    'b': [1, np.nan, 1, np.nan]},
+        ...                   index=['a', 'b', 'c', 'd'], columns=['a', 'b'])
+
+        >>> df.a <= 2
+        a     True
+        b     True
+        c    False
+        d    False
+        Name: a, dtype: bool
+
+        >>> df.b.le(2)
+        a     True
+        b    False
+        c     True
+        d    False
+        Name: b, dtype: bool
+        """
+        return self <= other
+
+    def ne(self, other) -> "Series":
+        """
+        Compare if the current value is not equal to the other.
+
+        >>> df = ps.DataFrame({'a': [1, 2, 3, 4],
+        ...                    'b': [1, np.nan, 1, np.nan]},
+        ...                   index=['a', 'b', 'c', 'd'], columns=['a', 'b'])
+
+        >>> df.a != 1
+        a    False
+        b     True
+        c     True
+        d     True
+        Name: a, dtype: bool
+
+        >>> df.b.ne(1)
+        a    False
+        b     True
+        c    False
+        d     True
+        Name: b, dtype: bool
+        """
+        return self != other
+
+    def divmod(self, other) -> Tuple["Series", "Series"]:
+        """
+        Return Integer division and modulo of series and other, element-wise
+        (binary operator `divmod`).
+
+        Parameters
+        ----------
+        other : Series or scalar value
+
+        Returns
+        -------
+        2-Tuple of Series
+            The result of the operation.
+
+        See Also
+        --------
+        Series.rdivmod
+        """
+        return (self.floordiv(other), self.mod(other))
+
+    def rdivmod(self, other) -> Tuple["Series", "Series"]:
+        """
+        Return Integer division and modulo of series and other, element-wise
+        (binary operator `rdivmod`).
+
+        Parameters
+        ----------
+        other : Series or scalar value
+
+        Returns
+        -------
+        2-Tuple of Series
+            The result of the operation.
+
+        See Also
+        --------
+        Series.divmod
+        """
+        return (self.rfloordiv(other), self.rmod(other))
+
+    def between(self, left, right, inclusive=True) -> "Series":
+        """
+        Return boolean Series equivalent to left <= series <= right.
+        This function returns a boolean vector containing `True` wherever the
+        corresponding Series element is between the boundary values `left` and
+        `right`. NA values are treated as `False`.
+
+        Parameters
+        ----------
+        left : scalar or list-like
+            Left boundary.
+        right : scalar or list-like
+            Right boundary.
+        inclusive : bool, default True
+            Include boundaries.
+
+        Returns
+        -------
+        Series
+            Series representing whether each element is between left and
+            right (inclusive).
+
+        See Also
+        --------
+        Series.gt : Greater than of series and other.
+        Series.lt : Less than of series and other.
+
+        Notes
+        -----
+        This function is equivalent to ``(left <= ser) & (ser <= right)``
+
+        Examples
+        --------
+        >>> s = ps.Series([2, 0, 4, 8, np.nan])
+
+        Boundary values are included by default:
+
+        >>> s.between(1, 4)
+        0     True
+        1    False
+        2     True
+        3    False
+        4    False
+        dtype: bool
+
+        With `inclusive` set to ``False`` boundary values are excluded:
+
+        >>> s.between(1, 4, inclusive=False)
+        0     True
+        1    False
+        2    False
+        3    False
+        4    False
+        dtype: bool
+
+        `left` and `right` can be any scalar value:
+
+        >>> s = ps.Series(['Alice', 'Bob', 'Carol', 'Eve'])
+        >>> s.between('Anna', 'Daniel')
+        0    False
+        1     True
+        2     True
+        3    False
+        dtype: bool
+        """
+        if inclusive:
+            lmask = self >= left
+            rmask = self <= right
+        else:
+            lmask = self > left
+            rmask = self < right
+
+        return lmask & rmask
+
+    # TODO: arg should support Series
+    # TODO: NaN and None
+    def map(self, arg) -> "Series":
+        """
+        Map values of Series according to input correspondence.
+
+        Used for substituting each value in a Series with another value,
+        that may be derived from a function, a ``dict``.
+
+        .. note:: make sure the size of the dictionary is not huge because it could
+            downgrade the performance or throw OutOfMemoryError due to a huge
+            expression within Spark. Consider the input as a functions as an
+            alternative instead in this case.
+
+        Parameters
+        ----------
+        arg : function or dict
+            Mapping correspondence.
+
+        Returns
+        -------
+        Series
+            Same index as caller.
+
+        See Also
+        --------
+        Series.apply : For applying more complex functions on a Series.
+        DataFrame.applymap : Apply a function elementwise on a whole DataFrame.
+
+        Notes
+        -----
+        When ``arg`` is a dictionary, values in Series that are not in the
+        dictionary (as keys) are converted to ``None``. However, if the
+        dictionary is a ``dict`` subclass that defines ``__missing__`` (i.e.
+        provides a method for default values), then this default is used
+        rather than ``None``.
+
+        Examples
+        --------
+        >>> s = ps.Series(['cat', 'dog', None, 'rabbit'])
+        >>> s
+        0       cat
+        1       dog
+        2      None
+        3    rabbit
+        dtype: object
+
+        ``map`` accepts a ``dict``. Values that are not found
+        in the ``dict`` are converted to ``None``, unless the dict has a default
+        value (e.g. ``defaultdict``):
+
+        >>> s.map({'cat': 'kitten', 'dog': 'puppy'})
+        0    kitten
+        1     puppy
+        2      None
+        3      None
+        dtype: object
+
+        It also accepts a function:
+
+        >>> def format(x) -> str:
+        ...     return 'I am a {}'.format(x)
+
+        >>> s.map(format)
+        0       I am a cat
+        1       I am a dog
+        2      I am a None
+        3    I am a rabbit
+        dtype: object
+        """
+        if isinstance(arg, dict):
+            is_start = True
+            # In case dictionary is empty.
+            current = F.when(F.lit(False), F.lit(None).cast(self.spark.data_type))
+
+            for to_replace, value in arg.items():
+                if is_start:
+                    current = F.when(self.spark.column == F.lit(to_replace), value)
+                    is_start = False
+                else:
+                    current = current.when(self.spark.column == F.lit(to_replace), value)
+
+            if hasattr(arg, "__missing__"):
+                tmp_val = arg[np._NoValue]
+                del arg[np._NoValue]  # Remove in case it's set in defaultdict.
+                current = current.otherwise(F.lit(tmp_val))
+            else:
+                current = current.otherwise(F.lit(None).cast(self.spark.data_type))
+            return self._with_new_scol(current)
+        else:
+            return self.apply(arg)
+
+    def alias(self, name) -> "Series":
+        """An alias for :meth:`Series.rename`."""
+        warnings.warn(
+            "Series.alias is deprecated as of Series.rename. Please use the API instead.",
+            FutureWarning,
+        )
+        return self.rename(name)
+
+    @property
+    def shape(self):
+        """Return a tuple of the shape of the underlying data."""
+        return (len(self),)
+
+    @property
+    def name(self) -> Union[Any, Tuple]:
+        """Return name of the Series."""
+        name = self._column_label
+        if name is not None and len(name) == 1:
+            return name[0]
+        else:
+            return name
+
+    @name.setter
+    def name(self, name: Union[Any, Tuple]):
+        self.rename(name, inplace=True)
+
+    # TODO: Functionality and documentation should be matched. Currently, changing index labels
+    # taking dictionary and function to change index are not supported.
+    def rename(self, index=None, **kwargs) -> "Series":
+        """
+        Alter Series name.
+
+        Parameters
+        ----------
+        index : scalar
+            Scalar will alter the ``Series.name`` attribute.
+
+        inplace : bool, default False
+            Whether to return a new Series. If True then value of copy is
+            ignored.
+
+        Returns
+        -------
+        Series
+            Series with name altered.
+
+        Examples
+        --------
+
+        >>> s = ps.Series([1, 2, 3])
+        >>> s
+        0    1
+        1    2
+        2    3
+        dtype: int64
+
+        >>> s.rename("my_name")  # scalar, changes Series.name
+        0    1
+        1    2
+        2    3
+        Name: my_name, dtype: int64
+        """
+        if index is None:
+            pass
+        elif not is_hashable(index):
+            raise TypeError("Series.name must be a hashable type")
+        elif not isinstance(index, tuple):
+            index = (index,)
+        scol = self.spark.column.alias(name_like_string(index))
+
+        internal = self._internal.copy(
+            column_labels=[index], data_spark_columns=[scol], column_label_names=None
+        )
+        kdf = DataFrame(internal)  # type: DataFrame
+
+        if kwargs.get("inplace", False):
+            self._col_label = index
+            self._update_anchor(kdf)
+            return self
+        else:
+            return first_series(kdf)
+
+    def rename_axis(
+        self, mapper: Optional[Any] = None, index: Optional[Any] = None, inplace: bool = False
+    ) -> Optional["Series"]:
+        """
+        Set the name of the axis for the index or columns.
+
+        Parameters
+        ----------
+        mapper, index :  scalar, list-like, dict-like or function, optional
+            A scalar, list-like, dict-like or functions transformations to
+            apply to the index values.
+        inplace : bool, default False
+            Modifies the object directly, instead of creating a new Series.
+
+        Returns
+        -------
+        Series, or None if `inplace` is True.
+
+        See Also
+        --------
+        Series.rename : Alter Series index labels or name.
+        DataFrame.rename : Alter DataFrame index labels or name.
+        Index.rename : Set new names on index.
+
+        Examples
+        --------
+        >>> s = ps.Series(["dog", "cat", "monkey"], name="animal")
+        >>> s  # doctest: +NORMALIZE_WHITESPACE
+        0       dog
+        1       cat
+        2    monkey
+        Name: animal, dtype: object
+        >>> s.rename_axis("index").sort_index()  # doctest: +NORMALIZE_WHITESPACE
+        index
+        0       dog
+        1       cat
+        2    monkey
+        Name: animal, dtype: object
+
+        **MultiIndex**
+
+        >>> index = pd.MultiIndex.from_product([['mammal'],
+        ...                                        ['dog', 'cat', 'monkey']],
+        ...                                       names=['type', 'name'])
+        >>> s = ps.Series([4, 4, 2], index=index, name='num_legs')
+        >>> s  # doctest: +NORMALIZE_WHITESPACE
+        type    name
+        mammal  dog       4
+                cat       4
+                monkey    2
+        Name: num_legs, dtype: int64
+        >>> s.rename_axis(index={'type': 'class'}).sort_index()  # doctest: +NORMALIZE_WHITESPACE
+        class   name
+        mammal  cat       4
+                dog       4
+                monkey    2
+        Name: num_legs, dtype: int64
+        >>> s.rename_axis(index=str.upper).sort_index()  # doctest: +NORMALIZE_WHITESPACE
+        TYPE    NAME
+        mammal  cat       4
+                dog       4
+                monkey    2
+        Name: num_legs, dtype: int64
+        """
+        kdf = self.to_frame().rename_axis(mapper=mapper, index=index, inplace=False)
+        if inplace:
+            self._update_anchor(kdf)
+            return None
+        else:
+            return first_series(kdf)
+
+    @property
+    def index(self) -> "ps.Index":
+        """The index (axis labels) Column of the Series.
+
+        See Also
+        --------
+        Index
+        """
+        return self._kdf.index
+
+    @property
+    def is_unique(self) -> bool:
+        """
+        Return boolean if values in the object are unique
+
+        Returns
+        -------
+        is_unique : boolean
+
+        >>> ps.Series([1, 2, 3]).is_unique
+        True
+        >>> ps.Series([1, 2, 2]).is_unique
+        False
+        >>> ps.Series([1, 2, 3, None]).is_unique
+        True
+        """
+        scol = self.spark.column
+
+        # Here we check:
+        #   1. the distinct count without nulls and count without nulls for non-null values
+        #   2. count null values and see if null is a distinct value.
+        #
+        # This workaround is in order to calculate the distinct count including nulls in
+        # single pass. Note that COUNT(DISTINCT expr) in Spark is designed to ignore nulls.
+        return self._internal.spark_frame.select(
+            (F.count(scol) == F.countDistinct(scol))
+            & (F.count(F.when(scol.isNull(), 1).otherwise(None)) <= 1)
+        ).collect()[0][0]
+
+    def reset_index(
+        self, level=None, drop=False, name=None, inplace=False
+    ) -> Optional[Union["Series", DataFrame]]:
+        """
+        Generate a new DataFrame or Series with the index reset.
+
+        This is useful when the index needs to be treated as a column,
+        or when the index is meaningless and needs to be reset
+        to the default before another operation.
+
+        Parameters
+        ----------
+        level : int, str, tuple, or list, default optional
+            For a Series with a MultiIndex, only remove the specified levels from the index.
+            Removes all levels by default.
+        drop : bool, default False
+            Just reset the index, without inserting it as a column in the new DataFrame.
+        name : object, optional
+            The name to use for the column containing the original Series values.
+            Uses self.name by default. This argument is ignored when drop is True.
+        inplace : bool, default False
+            Modify the Series in place (do not create a new object).
+
+        Returns
+        -------
+        Series or DataFrame
+            When `drop` is False (the default), a DataFrame is returned.
+            The newly created columns will come first in the DataFrame,
+            followed by the original Series values.
+            When `drop` is True, a `Series` is returned.
+            In either case, if ``inplace=True``, no value is returned.
+
+        Examples
+        --------
+        >>> s = ps.Series([1, 2, 3, 4], index=pd.Index(['a', 'b', 'c', 'd'], name='idx'))
+
+        Generate a DataFrame with default index.
+
+        >>> s.reset_index()
+          idx  0
+        0   a  1
+        1   b  2
+        2   c  3
+        3   d  4
+
+        To specify the name of the new column use `name`.
+
+        >>> s.reset_index(name='values')
+          idx  values
+        0   a       1
+        1   b       2
+        2   c       3
+        3   d       4
+
+        To generate a new Series with the default set `drop` to True.
+
+        >>> s.reset_index(drop=True)
+        0    1
+        1    2
+        2    3
+        3    4
+        dtype: int64
+
+        To update the Series in place, without generating a new one
+        set `inplace` to True. Note that it also requires ``drop=True``.
+
+        >>> s.reset_index(inplace=True, drop=True)
+        >>> s
+        0    1
+        1    2
+        2    3
+        3    4
+        dtype: int64
+        """
+        inplace = validate_bool_kwarg(inplace, "inplace")
+        if inplace and not drop:
+            raise TypeError("Cannot reset_index inplace on a Series to create a DataFrame")
+
+        if drop:
+            kdf = self._kdf[[self.name]]
+        else:
+            kser = self
+            if name is not None:
+                kser = kser.rename(name)
+            kdf = kser.to_frame()
+        kdf = kdf.reset_index(level=level, drop=drop)
+        if drop:
+            if inplace:
+                self._update_anchor(kdf)
+                return None
+            else:
+                return first_series(kdf)
+        else:
+            return kdf
+
+    def to_frame(self, name: Union[Any, Tuple] = None) -> DataFrame:
+        """
+        Convert Series to DataFrame.
+
+        Parameters
+        ----------
+        name : object, default None
+            The passed name should substitute for the series name (if it has
+            one).
+
+        Returns
+        -------
+        DataFrame
+            DataFrame representation of Series.
+
+        Examples
+        --------
+        >>> s = ps.Series(["a", "b", "c"])
+        >>> s.to_frame()
+           0
+        0  a
+        1  b
+        2  c
+
+        >>> s = ps.Series(["a", "b", "c"], name="vals")
+        >>> s.to_frame()
+          vals
+        0    a
+        1    b
+        2    c
+        """
+        if name is not None:
+            renamed = self.rename(name)
+        elif self._column_label is None:
+            renamed = self.rename(DEFAULT_SERIES_NAME)
+        else:
+            renamed = self
+        return DataFrame(renamed._internal)
+
+    to_dataframe = to_frame
+
+    def to_string(
+        self,
+        buf=None,
+        na_rep="NaN",
+        float_format=None,
+        header=True,
+        index=True,
+        length=False,
+        dtype=False,
+        name=False,
+        max_rows=None,
+    ) -> str:
+        """
+        Render a string representation of the Series.
+
+        .. note:: This method should only be used if the resulting pandas object is expected
+                  to be small, as all the data is loaded into the driver's memory. If the input
+                  is large, set max_rows parameter.
+
+        Parameters
+        ----------
+        buf : StringIO-like, optional
+            buffer to write to
+        na_rep : string, optional
+            string representation of NAN to use, default 'NaN'
+        float_format : one-parameter function, optional
+            formatter function to apply to columns' elements if they are floats
+            default None
+        header : boolean, default True
+            Add the Series header (index name)
+        index : bool, optional
+            Add index (row) labels, default True
+        length : boolean, default False
+            Add the Series length
+        dtype : boolean, default False
+            Add the Series dtype
+        name : boolean, default False
+            Add the Series name if not None
+        max_rows : int, optional
+            Maximum number of rows to show before truncating. If None, show
+            all.
+
+        Returns
+        -------
+        formatted : string (if not buffer passed)
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([(.2, .3), (.0, .6), (.6, .0), (.2, .1)], columns=['dogs', 'cats'])
+        >>> print(df['dogs'].to_string())
+        0    0.2
+        1    0.0
+        2    0.6
+        3    0.2
+
+        >>> print(df['dogs'].to_string(max_rows=2))
+        0    0.2
+        1    0.0
+        """
+        # Make sure locals() call is at the top of the function so we don't capture local variables.
+        args = locals()
+        if max_rows is not None:
+            kseries = self.head(max_rows)
+        else:
+            kseries = self
+
+        return validate_arguments_and_invoke_function(
+            kseries._to_internal_pandas(), self.to_string, pd.Series.to_string, args
+        )
+
+    def to_clipboard(self, excel=True, sep=None, **kwargs) -> None:
+        # Docstring defined below by reusing DataFrame.to_clipboard's.
+        args = locals()
+        kseries = self
+
+        return validate_arguments_and_invoke_function(
+            kseries._to_internal_pandas(), self.to_clipboard, pd.Series.to_clipboard, args
+        )
+
+    to_clipboard.__doc__ = DataFrame.to_clipboard.__doc__
+
+    def to_dict(self, into=dict) -> Mapping:
+        """
+        Convert Series to {label -> value} dict or dict-like object.
+
+        .. note:: This method should only be used if the resulting pandas DataFrame is expected
+            to be small, as all the data is loaded into the driver's memory.
+
+        Parameters
+        ----------
+        into : class, default dict
+            The collections.abc.Mapping subclass to use as the return
+            object. Can be the actual class or an empty
+            instance of the mapping type you want.  If you want a
+            collections.defaultdict, you must pass it initialized.
+
+        Returns
+        -------
+        collections.abc.Mapping
+            Key-value representation of Series.
+
+        Examples
+        --------
+        >>> s = ps.Series([1, 2, 3, 4])
+        >>> s_dict = s.to_dict()
+        >>> sorted(s_dict.items())
+        [(0, 1), (1, 2), (2, 3), (3, 4)]
+
+        >>> from collections import OrderedDict, defaultdict
+        >>> s.to_dict(OrderedDict)
+        OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)])
+
+        >>> dd = defaultdict(list)
+        >>> s.to_dict(dd)  # doctest: +ELLIPSIS
+        defaultdict(<class 'list'>, {...})
+        """
+        # Make sure locals() call is at the top of the function so we don't capture local variables.
+        args = locals()
+        kseries = self
+        return validate_arguments_and_invoke_function(
+            kseries._to_internal_pandas(), self.to_dict, pd.Series.to_dict, args
+        )
+
+    def to_latex(
+        self,
+        buf=None,
+        columns=None,
+        col_space=None,
+        header=True,
+        index=True,
+        na_rep="NaN",
+        formatters=None,
+        float_format=None,
+        sparsify=None,
+        index_names=True,
+        bold_rows=False,
+        column_format=None,
+        longtable=None,
+        escape=None,
+        encoding=None,
+        decimal=".",
+        multicolumn=None,
+        multicolumn_format=None,
+        multirow=None,
+    ) -> Optional[str]:
+
+        args = locals()
+        kseries = self
+        return validate_arguments_and_invoke_function(
+            kseries._to_internal_pandas(), self.to_latex, pd.Series.to_latex, args
+        )
+
+    to_latex.__doc__ = DataFrame.to_latex.__doc__
+
+    def to_pandas(self) -> pd.Series:
+        """
+        Return a pandas Series.
+
+        .. note:: This method should only be used if the resulting pandas object is expected
+                  to be small, as all the data is loaded into the driver's memory.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([(.2, .3), (.0, .6), (.6, .0), (.2, .1)], columns=['dogs', 'cats'])
+        >>> df['dogs'].to_pandas()
+        0    0.2
+        1    0.0
+        2    0.6
+        3    0.2
+        Name: dogs, dtype: float64
+        """
+        return self._to_internal_pandas().copy()
+
+    # Alias to maintain backward compatibility with Spark
+    def toPandas(self) -> pd.Series:
+        warnings.warn(
+            "Series.toPandas is deprecated as of Series.to_pandas. Please use the API instead.",
+            FutureWarning,
+        )
+        return self.to_pandas()
+
+    toPandas.__doc__ = to_pandas.__doc__
+
+    def to_list(self) -> List:
+        """
+        Return a list of the values.
+
+        These are each a scalar type, which is a Python scalar
+        (for str, int, float) or a pandas scalar
+        (for Timestamp/Timedelta/Interval/Period)
+
+        .. note:: This method should only be used if the resulting list is expected
+            to be small, as all the data is loaded into the driver's memory.
+
+        """
+        return self._to_internal_pandas().tolist()
+
+    tolist = to_list
+
+    def drop_duplicates(self, keep="first", inplace=False) -> Optional["Series"]:
+        """
+        Return Series with duplicate values removed.
+
+        Parameters
+        ----------
+        keep : {'first', 'last', ``False``}, default 'first'
+            Method to handle dropping duplicates:
+            - 'first' : Drop duplicates except for the first occurrence.
+            - 'last' : Drop duplicates except for the last occurrence.
+            - ``False`` : Drop all duplicates.
+        inplace : bool, default ``False``
+            If ``True``, performs operation inplace and returns None.
+
+        Returns
+        -------
+        Series
+            Series with duplicates dropped.
+
+        Examples
+        --------
+        Generate a Series with duplicated entries.
+
+        >>> s = ps.Series(['lama', 'cow', 'lama', 'beetle', 'lama', 'hippo'],
+        ...               name='animal')
+        >>> s.sort_index()
+        0      lama
+        1       cow
+        2      lama
+        3    beetle
+        4      lama
+        5     hippo
+        Name: animal, dtype: object
+
+        With the 'keep' parameter, the selection behaviour of duplicated values
+        can be changed. The value 'first' keeps the first occurrence for each
+        set of duplicated entries. The default value of keep is 'first'.
+
+        >>> s.drop_duplicates().sort_index()
+        0      lama
+        1       cow
+        3    beetle
+        5     hippo
+        Name: animal, dtype: object
+
+        The value 'last' for parameter 'keep' keeps the last occurrence for
+        each set of duplicated entries.
+
+        >>> s.drop_duplicates(keep='last').sort_index()
+        1       cow
+        3    beetle
+        4      lama
+        5     hippo
+        Name: animal, dtype: object
+
+        The value ``False`` for parameter 'keep' discards all sets of
+        duplicated entries. Setting the value of 'inplace' to ``True`` performs
+        the operation inplace and returns ``None``.
+
+        >>> s.drop_duplicates(keep=False, inplace=True)
+        >>> s.sort_index()
+        1       cow
+        3    beetle
+        5     hippo
+        Name: animal, dtype: object
+        """
+        inplace = validate_bool_kwarg(inplace, "inplace")
+        kdf = self._kdf[[self.name]].drop_duplicates(keep=keep)
+
+        if inplace:
+            self._update_anchor(kdf)
+            return None
+        else:
+            return first_series(kdf)
+
+    def reindex(self, index: Optional[Any] = None, fill_value: Optional[Any] = None,) -> "Series":
+        """
+        Conform Series to new index with optional filling logic, placing
+        NA/NaN in locations having no value in the previous index. A new object
+        is produced.
+
+        Parameters
+        ----------
+        index: array-like, optional
+            New labels / index to conform to, should be specified using keywords.
+            Preferably an Index object to avoid duplicating data
+        fill_value : scalar, default np.NaN
+            Value to use for missing values. Defaults to NaN, but can be any
+            "compatible" value.
+
+        Returns
+        -------
+        Series with changed index.
+
+        See Also
+        --------
+        Series.reset_index : Remove row labels or move them to new columns.
+
+        Examples
+        --------
+
+        Create a series with some fictional data.
+
+        >>> index = ['Firefox', 'Chrome', 'Safari', 'IE10', 'Konqueror']
+        >>> ser = ps.Series([200, 200, 404, 404, 301],
+        ...                 index=index, name='http_status')
+        >>> ser
+        Firefox      200
+        Chrome       200
+        Safari       404
+        IE10         404
+        Konqueror    301
+        Name: http_status, dtype: int64
+
+        Create a new index and reindex the Series. By default
+        values in the new index that do not have corresponding
+        records in the Series are assigned ``NaN``.
+
+        >>> new_index= ['Safari', 'Iceweasel', 'Comodo Dragon', 'IE10',
+        ...             'Chrome']
+        >>> ser.reindex(new_index).sort_index()
+        Chrome           200.0
+        Comodo Dragon      NaN
+        IE10             404.0
+        Iceweasel          NaN
+        Safari           404.0
+        Name: http_status, dtype: float64
+
+        We can fill in the missing values by passing a value to
+        the keyword ``fill_value``.
+
+        >>> ser.reindex(new_index, fill_value=0).sort_index()
+        Chrome           200
+        Comodo Dragon      0
+        IE10             404
+        Iceweasel          0
+        Safari           404
+        Name: http_status, dtype: int64
+
+        To further illustrate the filling functionality in
+        ``reindex``, we will create a Series with a
+        monotonically increasing index (for example, a sequence
+        of dates).
+
+        >>> date_index = pd.date_range('1/1/2010', periods=6, freq='D')
+        >>> ser2 = ps.Series([100, 101, np.nan, 100, 89, 88],
+        ...                  name='prices', index=date_index)
+        >>> ser2.sort_index()
+        2010-01-01    100.0
+        2010-01-02    101.0
+        2010-01-03      NaN
+        2010-01-04    100.0
+        2010-01-05     89.0
+        2010-01-06     88.0
+        Name: prices, dtype: float64
+
+        Suppose we decide to expand the series to cover a wider
+        date range.
+
+        >>> date_index2 = pd.date_range('12/29/2009', periods=10, freq='D')
+        >>> ser2.reindex(date_index2).sort_index()
+        2009-12-29      NaN
+        2009-12-30      NaN
+        2009-12-31      NaN
+        2010-01-01    100.0
+        2010-01-02    101.0
+        2010-01-03      NaN
+        2010-01-04    100.0
+        2010-01-05     89.0
+        2010-01-06     88.0
+        2010-01-07      NaN
+        Name: prices, dtype: float64
+        """
+
+        return first_series(self.to_frame().reindex(index=index, fill_value=fill_value)).rename(
+            self.name
+        )
+
+    def reindex_like(self, other: Union["Series", "DataFrame"]) -> "Series":
+        """
+        Return a Series with matching indices as other object.
+
+        Conform the object to the same index on all axes. Places NA/NaN in locations
+        having no value in the previous index.
+
+        Parameters
+        ----------
+        other : Series or DataFrame
+            Its row and column indices are used to define the new indices
+            of this object.
+
+        Returns
+        -------
+        Series
+            Series with changed indices on each axis.
+
+        See Also
+        --------
+        DataFrame.set_index : Set row labels.
+        DataFrame.reset_index : Remove row labels or move them to new columns.
+        DataFrame.reindex : Change to new indices or expand indices.
+
+        Notes
+        -----
+        Same as calling
+        ``.reindex(index=other.index, ...)``.
+
+        Examples
+        --------
+
+        >>> s1 = ps.Series([24.3, 31.0, 22.0, 35.0],
+        ...                index=pd.date_range(start='2014-02-12',
+        ...                                    end='2014-02-15', freq='D'),
+        ...                name="temp_celsius")
+        >>> s1
+        2014-02-12    24.3
+        2014-02-13    31.0
+        2014-02-14    22.0
+        2014-02-15    35.0
+        Name: temp_celsius, dtype: float64
+
+        >>> s2 = ps.Series(["low", "low", "medium"],
+        ...                index=pd.DatetimeIndex(['2014-02-12', '2014-02-13',
+        ...                                        '2014-02-15']),
+        ...                name="winspeed")
+        >>> s2
+        2014-02-12       low
+        2014-02-13       low
+        2014-02-15    medium
+        Name: winspeed, dtype: object
+
+        >>> s2.reindex_like(s1).sort_index()
+        2014-02-12       low
+        2014-02-13       low
+        2014-02-14      None
+        2014-02-15    medium
+        Name: winspeed, dtype: object
+        """
+        if isinstance(other, (Series, DataFrame)):
+            return self.reindex(index=other.index)
+        else:
+            raise TypeError("other must be a pandas-on-Spark Series or DataFrame")
+
+    def fillna(
+        self, value=None, method=None, axis=None, inplace=False, limit=None
+    ) -> Optional["Series"]:
+        """Fill NA/NaN values.
+
+        .. note:: the current implementation of 'method' parameter in fillna uses Spark's Window
+            without specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Parameters
+        ----------
+        value : scalar, dict, Series
+            Value to use to fill holes. alternately a dict/Series of values
+            specifying which value to use for each column.
+            DataFrame is not supported.
+        method : {'backfill', 'bfill', 'pad', 'ffill', None}, default None
+            Method to use for filling holes in reindexed Series pad / ffill: propagate last valid
+            observation forward to next valid backfill / bfill:
+            use NEXT valid observation to fill gap
+        axis : {0 or `index`}
+            1 and `columns` are not supported.
+        inplace : boolean, default False
+            Fill in place (do not create a new object)
+        limit : int, default None
+            If method is specified, this is the maximum number of consecutive NaN values to
+            forward/backward fill. In other words, if there is a gap with more than this number of
+            consecutive NaNs, it will only be partially filled. If method is not specified,
+            this is the maximum number of entries along the entire axis where NaNs will be filled.
+            Must be greater than 0 if not None
+
+        Returns
+        -------
+        Series
+            Series with NA entries filled.
+
+        Examples
+        --------
+        >>> s = ps.Series([np.nan, 2, 3, 4, np.nan, 6], name='x')
+        >>> s
+        0    NaN
+        1    2.0
+        2    3.0
+        3    4.0
+        4    NaN
+        5    6.0
+        Name: x, dtype: float64
+
+        Replace all NaN elements with 0s.
+
+        >>> s.fillna(0)
+        0    0.0
+        1    2.0
+        2    3.0
+        3    4.0
+        4    0.0
+        5    6.0
+        Name: x, dtype: float64
+
+        We can also propagate non-null values forward or backward.
+
+        >>> s.fillna(method='ffill')
+        0    NaN
+        1    2.0
+        2    3.0
+        3    4.0
+        4    4.0
+        5    6.0
+        Name: x, dtype: float64
+
+        >>> s = ps.Series([np.nan, 'a', 'b', 'c', np.nan], name='x')
+        >>> s.fillna(method='ffill')
+        0    None
+        1       a
+        2       b
+        3       c
+        4       c
+        Name: x, dtype: object
+        """
+        kser = self._fillna(value=value, method=method, axis=axis, limit=limit)
+
+        if method is not None:
+            kser = DataFrame(kser._kdf._internal.resolved_copy)._kser_for(self._column_label)
+
+        inplace = validate_bool_kwarg(inplace, "inplace")
+        if inplace:
+            self._kdf._update_internal_frame(kser._kdf._internal, requires_same_anchor=False)
+            return None
+        else:
+            return kser._with_new_scol(kser.spark.column)  # TODO: dtype?
+
+    def _fillna(self, value=None, method=None, axis=None, limit=None, part_cols=()):
+        axis = validate_axis(axis)
+        if axis != 0:
+            raise NotImplementedError("fillna currently only works for axis=0 or axis='index'")
+        if (value is None) and (method is None):
+            raise ValueError("Must specify a fillna 'value' or 'method' parameter.")
+        if (method is not None) and (method not in ["ffill", "pad", "backfill", "bfill"]):
+            raise ValueError("Expecting 'pad', 'ffill', 'backfill' or 'bfill'.")
+
+        scol = self.spark.column
+
+        if isinstance(self.spark.data_type, (FloatType, DoubleType)):
+            cond = scol.isNull() | F.isnan(scol)
+        else:
+            if not self.spark.nullable:
+                return self.copy()
+            cond = scol.isNull()
+
+        if value is not None:
+            if not isinstance(value, (float, int, str, bool)):
+                raise TypeError("Unsupported type %s" % type(value).__name__)
+            if limit is not None:
+                raise ValueError("limit parameter for value is not support now")
+            scol = F.when(cond, value).otherwise(scol)
+        else:
+            if method in ["ffill", "pad"]:
+                func = F.last
+                end = Window.currentRow - 1
+                if limit is not None:
+                    begin = Window.currentRow - limit
+                else:
+                    begin = Window.unboundedPreceding
+            elif method in ["bfill", "backfill"]:
+                func = F.first
+                begin = Window.currentRow + 1
+                if limit is not None:
+                    end = Window.currentRow + limit
+                else:
+                    end = Window.unboundedFollowing
+
+            window = (
+                Window.partitionBy(*part_cols)
+                .orderBy(NATURAL_ORDER_COLUMN_NAME)
+                .rowsBetween(begin, end)
+            )
+            scol = F.when(cond, func(scol, True).over(window)).otherwise(scol)
+
+        return DataFrame(
+            self._kdf._internal.with_new_spark_column(
+                self._column_label, scol.alias(name_like_string(self.name))  # TODO: dtype?
+            )
+        )._kser_for(self._column_label)
+
+    def dropna(self, axis=0, inplace=False, **kwargs) -> Optional["Series"]:
+        """
+        Return a new Series with missing values removed.
+
+        Parameters
+        ----------
+        axis : {0 or 'index'}, default 0
+            There is only one axis to drop values from.
+        inplace : bool, default False
+            If True, do operation inplace and return None.
+        **kwargs
+            Not in use.
+
+        Returns
+        -------
+        Series
+            Series with NA entries dropped from it.
+
+        Examples
+        --------
+        >>> ser = ps.Series([1., 2., np.nan])
+        >>> ser
+        0    1.0
+        1    2.0
+        2    NaN
+        dtype: float64
+
+        Drop NA values from a Series.
+
+        >>> ser.dropna()
+        0    1.0
+        1    2.0
+        dtype: float64
+
+        Keep the Series with valid entries in the same variable.
+
+        >>> ser.dropna(inplace=True)
+        >>> ser
+        0    1.0
+        1    2.0
+        dtype: float64
+        """
+        inplace = validate_bool_kwarg(inplace, "inplace")
+        # TODO: last two examples from pandas produce different results.
+        kdf = self._kdf[[self.name]].dropna(axis=axis, inplace=False)
+        if inplace:
+            self._update_anchor(kdf)
+            return None
+        else:
+            return first_series(kdf)
+
+    def clip(self, lower: Union[float, int] = None, upper: Union[float, int] = None) -> "Series":
+        """
+        Trim values at input threshold(s).
+
+        Assigns values outside boundary to boundary values.
+
+        Parameters
+        ----------
+        lower : float or int, default None
+            Minimum threshold value. All values below this threshold will be set to it.
+        upper : float or int, default None
+            Maximum threshold value. All values above this threshold will be set to it.
+
+        Returns
+        -------
+        Series
+            Series with the values outside the clip boundaries replaced
+
+        Examples
+        --------
+        >>> ps.Series([0, 2, 4]).clip(1, 3)
+        0    1
+        1    2
+        2    3
+        dtype: int64
+
+        Notes
+        -----
+        One difference between this implementation and pandas is that running
+        `pd.Series(['a', 'b']).clip(0, 1)` will crash with "TypeError: '<=' not supported between
+        instances of 'str' and 'int'" while `ps.Series(['a', 'b']).clip(0, 1)` will output the
+        original Series, simply ignoring the incompatible types.
+        """
+        if is_list_like(lower) or is_list_like(upper):
+            raise TypeError(
+                "List-like value are not supported for 'lower' and 'upper' at the " + "moment"
+            )
+
+        if lower is None and upper is None:
+            return self
+
+        if isinstance(self.spark.data_type, NumericType):
+            scol = self.spark.column
+            if lower is not None:
+                scol = F.when(scol < lower, lower).otherwise(scol)
+            if upper is not None:
+                scol = F.when(scol > upper, upper).otherwise(scol)
+            return self._with_new_scol(scol, dtype=self.dtype)
+        else:
+            return self
+
+    def drop(
+        self, labels=None, index: Union[Any, Tuple, List[Any], List[Tuple]] = None, level=None
+    ) -> "Series":
+        """
+        Return Series with specified index labels removed.
+
+        Remove elements of a Series based on specifying the index labels.
+        When using a multi-index, labels on different levels can be removed by specifying the level.
+
+        Parameters
+        ----------
+        labels : single label or list-like
+            Index labels to drop.
+        index : None
+            Redundant for application on Series, but index can be used instead of labels.
+        level : int or level name, optional
+            For MultiIndex, level for which the labels will be removed.
+
+        Returns
+        -------
+        Series
+            Series with specified index labels removed.
+
+        See Also
+        --------
+        Series.dropna
+
+        Examples
+        --------
+        >>> s = ps.Series(data=np.arange(3), index=['A', 'B', 'C'])
+        >>> s
+        A    0
+        B    1
+        C    2
+        dtype: int64
+
+        Drop single label A
+
+        >>> s.drop('A')
+        B    1
+        C    2
+        dtype: int64
+
+        Drop labels B and C
+
+        >>> s.drop(labels=['B', 'C'])
+        A    0
+        dtype: int64
+
+        With 'index' rather than 'labels' returns exactly same result.
+
+        >>> s.drop(index='A')
+        B    1
+        C    2
+        dtype: int64
+
+        >>> s.drop(index=['B', 'C'])
+        A    0
+        dtype: int64
+
+        Also support for MultiIndex
+
+        >>> midx = pd.MultiIndex([['lama', 'cow', 'falcon'],
+        ...                       ['speed', 'weight', 'length']],
+        ...                      [[0, 0, 0, 1, 1, 1, 2, 2, 2],
+        ...                       [0, 1, 2, 0, 1, 2, 0, 1, 2]])
+        >>> s = ps.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3],
+        ...               index=midx)
+        >>> s
+        lama    speed      45.0
+                weight    200.0
+                length      1.2
+        cow     speed      30.0
+                weight    250.0
+                length      1.5
+        falcon  speed     320.0
+                weight      1.0
+                length      0.3
+        dtype: float64
+
+        >>> s.drop(labels='weight', level=1)
+        lama    speed      45.0
+                length      1.2
+        cow     speed      30.0
+                length      1.5
+        falcon  speed     320.0
+                length      0.3
+        dtype: float64
+
+        >>> s.drop(('lama', 'weight'))
+        lama    speed      45.0
+                length      1.2
+        cow     speed      30.0
+                weight    250.0
+                length      1.5
+        falcon  speed     320.0
+                weight      1.0
+                length      0.3
+        dtype: float64
+
+        >>> s.drop([('lama', 'speed'), ('falcon', 'weight')])
+        lama    weight    200.0
+                length      1.2
+        cow     speed      30.0
+                weight    250.0
+                length      1.5
+        falcon  speed     320.0
+                length      0.3
+        dtype: float64
+        """
+        return first_series(self._drop(labels=labels, index=index, level=level))
+
+    def _drop(
+        self, labels=None, index: Union[Any, Tuple, List[Any], List[Tuple]] = None, level=None
+    ):
+        if labels is not None:
+            if index is not None:
+                raise ValueError("Cannot specify both 'labels' and 'index'")
+            return self._drop(index=labels, level=level)
+        if index is not None:
+            internal = self._internal
+            if level is None:
+                level = 0
+            if level >= internal.index_level:
+                raise ValueError("'level' should be less than the number of indexes")
+
+            if is_name_like_tuple(index):  # type: ignore
+                index = [index]
+            elif is_name_like_value(index):
+                index = [(index,)]
+            elif all(is_name_like_value(idxes, allow_tuple=False) for idxes in index):
+                index = [(idex,) for idex in index]
+            elif not all(is_name_like_tuple(idxes) for idxes in index):
+                raise ValueError(
+                    "If the given index is a list, it "
+                    "should only contains names as all tuples or all non tuples "
+                    "that contain index names"
+                )
+
+            drop_index_scols = []
+            for idxes in index:
+                try:
+                    index_scols = [
+                        internal.index_spark_columns[lvl] == idx
+                        for lvl, idx in enumerate(idxes, level)
+                    ]
+                except IndexError:
+                    raise KeyError(
+                        "Key length ({}) exceeds index depth ({})".format(
+                            internal.index_level, len(idxes)
+                        )
+                    )
+                drop_index_scols.append(reduce(lambda x, y: x & y, index_scols))
+
+            cond = ~reduce(lambda x, y: x | y, drop_index_scols)
+
+            return DataFrame(internal.with_filter(cond))
+        else:
+            raise ValueError("Need to specify at least one of 'labels' or 'index'")
+
+    def head(self, n: int = 5) -> "Series":
+        """
+        Return the first n rows.
+
+        This function returns the first n rows for the object based on position.
+        It is useful for quickly testing if your object has the right type of data in it.
+
+        Parameters
+        ----------
+        n : Integer, default =  5
+
+        Returns
+        -------
+        The first n rows of the caller object.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'animal':['alligator', 'bee', 'falcon', 'lion']})
+        >>> df.animal.head(2)  # doctest: +NORMALIZE_WHITESPACE
+        0     alligator
+        1     bee
+        Name: animal, dtype: object
+        """
+        return first_series(self.to_frame().head(n)).rename(self.name)
+
+    def last(self, offset: Union[str, DateOffset]) -> "Series":
+        """
+        Select final periods of time series data based on a date offset.
+
+        When having a Series with dates as index, this function can
+        select the last few elements based on a date offset.
+
+        Parameters
+        ----------
+        offset : str or DateOffset
+            The offset length of the data that will be selected. For instance,
+            '3D' will display all the rows having their index within the last 3 days.
+
+        Returns
+        -------
+        Series
+            A subset of the caller.
+
+        Raises
+        ------
+        TypeError
+            If the index is not a :class:`DatetimeIndex`
+
+        Examples
+        --------
+        >>> index = pd.date_range('2018-04-09', periods=4, freq='2D')
+        >>> kser = ps.Series([1, 2, 3, 4], index=index)
+        >>> kser
+        2018-04-09    1
+        2018-04-11    2
+        2018-04-13    3
+        2018-04-15    4
+        dtype: int64
+
+        Get the rows for the last 3 days:
+
+        >>> kser.last('3D')
+        2018-04-13    3
+        2018-04-15    4
+        dtype: int64
+
+        Notice the data for 3 last calendar days were returned, not the last
+        3 observed days in the dataset, and therefore data for 2018-04-11 was
+        not returned.
+        """
+        return first_series(self.to_frame().last(offset)).rename(self.name)
+
+    def first(self, offset: Union[str, DateOffset]) -> "Series":
+        """
+        Select first periods of time series data based on a date offset.
+
+        When having a Series with dates as index, this function can
+        select the first few elements based on a date offset.
+
+        Parameters
+        ----------
+        offset : str or DateOffset
+            The offset length of the data that will be selected. For instance,
+            '3D' will display all the rows having their index within the first 3 days.
+
+        Returns
+        -------
+        Series
+            A subset of the caller.
+
+        Raises
+        ------
+        TypeError
+            If the index is not a :class:`DatetimeIndex`
+
+        Examples
+        --------
+        >>> index = pd.date_range('2018-04-09', periods=4, freq='2D')
+        >>> kser = ps.Series([1, 2, 3, 4], index=index)
+        >>> kser
+        2018-04-09    1
+        2018-04-11    2
+        2018-04-13    3
+        2018-04-15    4
+        dtype: int64
+
+        Get the rows for the first 3 days:
+
+        >>> kser.first('3D')
+        2018-04-09    1
+        2018-04-11    2
+        dtype: int64
+
+        Notice the data for 3 first calendar days were returned, not the first
+        3 observed days in the dataset, and therefore data for 2018-04-13 was
+        not returned.
+        """
+        return first_series(self.to_frame().first(offset)).rename(self.name)
+
+    # TODO: Categorical type isn't supported (due to PySpark's limitation) and
+    # some doctests related with timestamps were not added.
+    def unique(self) -> "Series":
+        """
+        Return unique values of Series object.
+
+        Uniques are returned in order of appearance. Hash table-based unique,
+        therefore does NOT sort.
+
+        .. note:: This method returns newly created Series whereas pandas returns
+                  the unique values as a NumPy array.
+
+        Returns
+        -------
+        Returns the unique values as a Series.
+
+        See Also
+        --------
+        Index.unique
+        groupby.SeriesGroupBy.unique
+
+        Examples
+        --------
+        >>> kser = ps.Series([2, 1, 3, 3], name='A')
+        >>> kser.unique().sort_values()  # doctest: +NORMALIZE_WHITESPACE, +ELLIPSIS
+        <BLANKLINE>
+        ...  1
+        ...  2
+        ...  3
+        Name: A, dtype: int64
+
+        >>> ps.Series([pd.Timestamp('2016-01-01') for _ in range(3)]).unique()
+        0   2016-01-01
+        dtype: datetime64[ns]
+
+        >>> kser.name = ('x', 'a')
+        >>> kser.unique().sort_values()  # doctest: +NORMALIZE_WHITESPACE, +ELLIPSIS
+        <BLANKLINE>
+        ...  1
+        ...  2
+        ...  3
+        Name: (x, a), dtype: int64
+        """
+        sdf = self._internal.spark_frame.select(self.spark.column).distinct()
+        internal = InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=None,
+            column_labels=[self._column_label],
+            data_spark_columns=[scol_for(sdf, self._internal.data_spark_column_names[0])],
+            data_dtypes=[self.dtype],
+            column_label_names=self._internal.column_label_names,
+        )
+        return first_series(DataFrame(internal))
+
+    def sort_values(
+        self, ascending: bool = True, inplace: bool = False, na_position: str = "last"
+    ) -> Optional["Series"]:
+        """
+        Sort by the values.
+
+        Sort a Series in ascending or descending order by some criterion.
+
+        Parameters
+        ----------
+        ascending : bool or list of bool, default True
+             Sort ascending vs. descending. Specify list for multiple sort
+             orders.  If this is a list of bools, must match the length of
+             the by.
+        inplace : bool, default False
+             if True, perform operation in-place
+        na_position : {'first', 'last'}, default 'last'
+             `first` puts NaNs at the beginning, `last` puts NaNs at the end
+
+        Returns
+        -------
+        sorted_obj : Series ordered by values.
+
+        Examples
+        --------
+        >>> s = ps.Series([np.nan, 1, 3, 10, 5])
+        >>> s
+        0     NaN
+        1     1.0
+        2     3.0
+        3    10.0
+        4     5.0
+        dtype: float64
+
+        Sort values ascending order (default behaviour)
+
+        >>> s.sort_values(ascending=True)
+        1     1.0
+        2     3.0
+        4     5.0
+        3    10.0
+        0     NaN
+        dtype: float64
+
+        Sort values descending order
+
+        >>> s.sort_values(ascending=False)
+        3    10.0
+        4     5.0
+        2     3.0
+        1     1.0
+        0     NaN
+        dtype: float64
+
+        Sort values inplace
+
+        >>> s.sort_values(ascending=False, inplace=True)
+        >>> s
+        3    10.0
+        4     5.0
+        2     3.0
+        1     1.0
+        0     NaN
+        dtype: float64
+
+        Sort values putting NAs first
+
+        >>> s.sort_values(na_position='first')
+        0     NaN
+        1     1.0
+        2     3.0
+        4     5.0
+        3    10.0
+        dtype: float64
+
+        Sort a series of strings
+
+        >>> s = ps.Series(['z', 'b', 'd', 'a', 'c'])
+        >>> s
+        0    z
+        1    b
+        2    d
+        3    a
+        4    c
+        dtype: object
+
+        >>> s.sort_values()
+        3    a
+        1    b
+        4    c
+        2    d
+        0    z
+        dtype: object
+        """
+        inplace = validate_bool_kwarg(inplace, "inplace")
+        kdf = self._kdf[[self.name]]._sort(
+            by=[self.spark.column], ascending=ascending, inplace=False, na_position=na_position
+        )
+
+        if inplace:
+            self._update_anchor(kdf)
+            return None
+        else:
+            return first_series(kdf)
+
+    def sort_index(
+        self,
+        axis: int = 0,
+        level: Optional[Union[int, List[int]]] = None,
+        ascending: bool = True,
+        inplace: bool = False,
+        kind: str = None,
+        na_position: str = "last",
+    ) -> Optional["Series"]:
+        """
+        Sort object by labels (along an axis)
+
+        Parameters
+        ----------
+        axis : index, columns to direct sorting. Currently, only axis = 0 is supported.
+        level : int or level name or list of ints or list of level names
+            if not None, sort on values in specified index level(s)
+        ascending : boolean, default True
+            Sort ascending vs. descending
+        inplace : bool, default False
+            if True, perform operation in-place
+        kind : str, default None
+            pandas-on-Spark does not allow specifying the sorting algorithm at the moment,
+            default None
+        na_position : {‘first’, ‘last’}, default ‘last’
+            first puts NaNs at the beginning, last puts NaNs at the end. Not implemented for
+            MultiIndex.
+
+        Returns
+        -------
+        sorted_obj : Series
+
+        Examples
+        --------
+        >>> df = ps.Series([2, 1, np.nan], index=['b', 'a', np.nan])
+
+        >>> df.sort_index()
+        a      1.0
+        b      2.0
+        NaN    NaN
+        dtype: float64
+
+        >>> df.sort_index(ascending=False)
+        b      2.0
+        a      1.0
+        NaN    NaN
+        dtype: float64
+
+        >>> df.sort_index(na_position='first')
+        NaN    NaN
+        a      1.0
+        b      2.0
+        dtype: float64
+
+        >>> df.sort_index(inplace=True)
+        >>> df
+        a      1.0
+        b      2.0
+        NaN    NaN
+        dtype: float64
+
+        >>> df = ps.Series(range(4), index=[['b', 'b', 'a', 'a'], [1, 0, 1, 0]], name='0')
+
+        >>> df.sort_index()
+        a  0    3
+           1    2
+        b  0    1
+           1    0
+        Name: 0, dtype: int64
+
+        >>> df.sort_index(level=1)  # doctest: +SKIP
+        a  0    3
+        b  0    1
+        a  1    2
+        b  1    0
+        Name: 0, dtype: int64
+
+        >>> df.sort_index(level=[1, 0])
+        a  0    3
+        b  0    1
+        a  1    2
+        b  1    0
+        Name: 0, dtype: int64
+        """
+        inplace = validate_bool_kwarg(inplace, "inplace")
+        kdf = self._kdf[[self.name]].sort_index(
+            axis=axis, level=level, ascending=ascending, kind=kind, na_position=na_position
+        )
+
+        if inplace:
+            self._update_anchor(kdf)
+            return None
+        else:
+            return first_series(kdf)
+
+    def swaplevel(self, i=-2, j=-1, copy: bool = True) -> "Series":
+        """
+        Swap levels i and j in a MultiIndex.
+        Default is to swap the two innermost levels of the index.
+
+        Parameters
+        ----------
+        i, j : int, str
+            Level of the indices to be swapped. Can pass level name as string.
+        copy : bool, default True
+            Whether to copy underlying data. Must be True.
+
+        Returns
+        -------
+        Series
+            Series with levels swapped in MultiIndex.
+
+        Examples
+        --------
+        >>> midx = pd.MultiIndex.from_arrays([['a', 'b'], [1, 2]], names = ['word', 'number'])
+        >>> midx  # doctest: +SKIP
+        MultiIndex([('a', 1),
+                    ('b', 2)],
+                   names=['word', 'number'])
+        >>> kser = ps.Series(['x', 'y'], index=midx)
+        >>> kser
+        word  number
+        a     1         x
+        b     2         y
+        dtype: object
+        >>> kser.swaplevel()
+        number  word
+        1       a       x
+        2       b       y
+        dtype: object
+        >>> kser.swaplevel(0, 1)
+        number  word
+        1       a       x
+        2       b       y
+        dtype: object
+        >>> kser.swaplevel('number', 'word')
+        number  word
+        1       a       x
+        2       b       y
+        dtype: object
+        """
+        assert copy is True
+
+        return first_series(self.to_frame().swaplevel(i, j, axis=0)).rename(self.name)
+
+    def swapaxes(self, i: Union[str, int], j: Union[str, int], copy: bool = True) -> "Series":
+        """
+        Interchange axes and swap values axes appropriately.
+
+        Parameters
+        ----------
+        i: {0 or 'index', 1 or 'columns'}. The axis to swap.
+        j: {0 or 'index', 1 or 'columns'}. The axis to swap.
+        copy : bool, default True.
+
+        Returns
+        -------
+        Series
+
+        Examples
+        --------
+        >>> kser = ps.Series([1, 2, 3], index=["x", "y", "z"])
+        >>> kser
+        x    1
+        y    2
+        z    3
+        dtype: int64
+        >>>
+        >>> kser.swapaxes(0, 0)
+        x    1
+        y    2
+        z    3
+        dtype: int64
+        """
+        assert copy is True
+
+        i = validate_axis(i)
+        j = validate_axis(j)
+        if not i == j == 0:
+            raise ValueError("Axis must be 0 for Series")
+
+        return self.copy()
+
+    def add_prefix(self, prefix) -> "Series":
+        """
+        Prefix labels with string `prefix`.
+
+        For Series, the row labels are prefixed.
+        For DataFrame, the column labels are prefixed.
+
+        Parameters
+        ----------
+        prefix : str
+           The string to add before each label.
+
+        Returns
+        -------
+        Series
+           New Series with updated labels.
+
+        See Also
+        --------
+        Series.add_suffix: Suffix column labels with string `suffix`.
+        DataFrame.add_suffix: Suffix column labels with string `suffix`.
+        DataFrame.add_prefix: Prefix column labels with string `prefix`.
+
+        Examples
+        --------
+        >>> s = ps.Series([1, 2, 3, 4])
+        >>> s
+        0    1
+        1    2
+        2    3
+        3    4
+        dtype: int64
+
+        >>> s.add_prefix('item_')
+        item_0    1
+        item_1    2
+        item_2    3
+        item_3    4
+        dtype: int64
+        """
+        assert isinstance(prefix, str)
+        internal = self._internal.resolved_copy
+        sdf = internal.spark_frame.select(
+            [
+                F.concat(F.lit(prefix), index_spark_column).alias(index_spark_column_name)
+                for index_spark_column, index_spark_column_name in zip(
+                    internal.index_spark_columns, internal.index_spark_column_names
+                )
+            ]
+            + internal.data_spark_columns
+        )
+        return first_series(
+            DataFrame(internal.with_new_sdf(sdf, index_dtypes=([None] * internal.index_level)))
+        )
+
+    def add_suffix(self, suffix) -> "Series":
+        """
+        Suffix labels with string suffix.
+
+        For Series, the row labels are suffixed.
+        For DataFrame, the column labels are suffixed.
+
+        Parameters
+        ----------
+        suffix : str
+           The string to add after each label.
+
+        Returns
+        -------
+        Series
+           New Series with updated labels.
+
+        See Also
+        --------
+        Series.add_prefix: Prefix row labels with string `prefix`.
+        DataFrame.add_prefix: Prefix column labels with string `prefix`.
+        DataFrame.add_suffix: Suffix column labels with string `suffix`.
+
+        Examples
+        --------
+        >>> s = ps.Series([1, 2, 3, 4])
+        >>> s
+        0    1
+        1    2
+        2    3
+        3    4
+        dtype: int64
+
+        >>> s.add_suffix('_item')
+        0_item    1
+        1_item    2
+        2_item    3
+        3_item    4
+        dtype: int64
+        """
+        assert isinstance(suffix, str)
+        internal = self._internal.resolved_copy
+        sdf = internal.spark_frame.select(
+            [
+                F.concat(index_spark_column, F.lit(suffix)).alias(index_spark_column_name)
+                for index_spark_column, index_spark_column_name in zip(
+                    internal.index_spark_columns, internal.index_spark_column_names
+                )
+            ]
+            + internal.data_spark_columns
+        )
+        return first_series(
+            DataFrame(internal.with_new_sdf(sdf, index_dtypes=([None] * internal.index_level)))
+        )
+
+    def corr(self, other, method="pearson") -> float:
+        """
+        Compute correlation with `other` Series, excluding missing values.
+
+        Parameters
+        ----------
+        other : Series
+        method : {'pearson', 'spearman'}
+            * pearson : standard correlation coefficient
+            * spearman : Spearman rank correlation
+
+        Returns
+        -------
+        correlation : float
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'s1': [.2, .0, .6, .2],
+        ...                    's2': [.3, .6, .0, .1]})
+        >>> s1 = df.s1
+        >>> s2 = df.s2
+        >>> s1.corr(s2, method='pearson')  # doctest: +ELLIPSIS
+        -0.851064...
+
+        >>> s1.corr(s2, method='spearman')  # doctest: +ELLIPSIS
+        -0.948683...
+
+        Notes
+        -----
+        There are behavior differences between pandas-on-Spark and pandas.
+
+        * the `method` argument only accepts 'pearson', 'spearman'
+        * the data should not contain NaNs. pandas-on-Spark will return an error.
+        * pandas-on-Spark doesn't support the following argument(s).
+
+          * `min_periods` argument is not supported
+        """
+        # This implementation is suboptimal because it computes more than necessary,
+        # but it should be a start
+        columns = ["__corr_arg1__", "__corr_arg2__"]
+        kdf = self._kdf.assign(__corr_arg1__=self, __corr_arg2__=other)[columns]
+        kdf.columns = columns
+        c = corr(kdf, method=method)
+        return c.loc[tuple(columns)]
+
+    def nsmallest(self, n: int = 5) -> "Series":
+        """
+        Return the smallest `n` elements.
+
+        Parameters
+        ----------
+        n : int, default 5
+            Return this many ascending sorted values.
+
+        Returns
+        -------
+        Series
+            The `n` smallest values in the Series, sorted in increasing order.
+
+        See Also
+        --------
+        Series.nlargest: Get the `n` largest elements.
+        Series.sort_values: Sort Series by values.
+        Series.head: Return the first `n` rows.
+
+        Notes
+        -----
+        Faster than ``.sort_values().head(n)`` for small `n` relative to
+        the size of the ``Series`` object.
+        In pandas-on-Spark, thanks to Spark's lazy execution and query optimizer,
+        the two would have same performance.
+
+        Examples
+        --------
+        >>> data = [1, 2, 3, 4, np.nan ,6, 7, 8]
+        >>> s = ps.Series(data)
+        >>> s
+        0    1.0
+        1    2.0
+        2    3.0
+        3    4.0
+        4    NaN
+        5    6.0
+        6    7.0
+        7    8.0
+        dtype: float64
+
+        The `n` largest elements where ``n=5`` by default.
+
+        >>> s.nsmallest()
+        0    1.0
+        1    2.0
+        2    3.0
+        3    4.0
+        5    6.0
+        dtype: float64
+
+        >>> s.nsmallest(3)
+        0    1.0
+        1    2.0
+        2    3.0
+        dtype: float64
+        """
+        return self.sort_values(ascending=True).head(n)
+
+    def nlargest(self, n: int = 5) -> "Series":
+        """
+        Return the largest `n` elements.
+
+        Parameters
+        ----------
+        n : int, default 5
+
+        Returns
+        -------
+        Series
+            The `n` largest values in the Series, sorted in decreasing order.
+
+        See Also
+        --------
+        Series.nsmallest: Get the `n` smallest elements.
+        Series.sort_values: Sort Series by values.
+        Series.head: Return the first `n` rows.
+
+        Notes
+        -----
+        Faster than ``.sort_values(ascending=False).head(n)`` for small `n`
+        relative to the size of the ``Series`` object.
+
+        In pandas-on-Spark, thanks to Spark's lazy execution and query optimizer,
+        the two would have same performance.
+
+        Examples
+        --------
+        >>> data = [1, 2, 3, 4, np.nan ,6, 7, 8]
+        >>> s = ps.Series(data)
+        >>> s
+        0    1.0
+        1    2.0
+        2    3.0
+        3    4.0
+        4    NaN
+        5    6.0
+        6    7.0
+        7    8.0
+        dtype: float64
+
+        The `n` largest elements where ``n=5`` by default.
+
+        >>> s.nlargest()
+        7    8.0
+        6    7.0
+        5    6.0
+        3    4.0
+        2    3.0
+        dtype: float64
+
+        >>> s.nlargest(n=3)
+        7    8.0
+        6    7.0
+        5    6.0
+        dtype: float64
+
+
+        """
+        return self.sort_values(ascending=False).head(n)
+
+    def append(
+        self, to_append: "Series", ignore_index: bool = False, verify_integrity: bool = False
+    ) -> "Series":
+        """
+        Concatenate two or more Series.
+
+        Parameters
+        ----------
+        to_append : Series or list/tuple of Series
+        ignore_index : boolean, default False
+            If True, do not use the index labels.
+        verify_integrity : boolean, default False
+            If True, raise Exception on creating index with duplicates
+
+        Returns
+        -------
+        appended : Series
+
+        Examples
+        --------
+        >>> s1 = ps.Series([1, 2, 3])
+        >>> s2 = ps.Series([4, 5, 6])
+        >>> s3 = ps.Series([4, 5, 6], index=[3,4,5])
+
+        >>> s1.append(s2)
+        0    1
+        1    2
+        2    3
+        0    4
+        1    5
+        2    6
+        dtype: int64
+
+        >>> s1.append(s3)
+        0    1
+        1    2
+        2    3
+        3    4
+        4    5
+        5    6
+        dtype: int64
+
+        With ignore_index set to True:
+
+        >>> s1.append(s2, ignore_index=True)
+        0    1
+        1    2
+        2    3
+        3    4
+        4    5
+        5    6
+        dtype: int64
+        """
+        return first_series(
+            self.to_frame().append(to_append.to_frame(), ignore_index, verify_integrity)
+        ).rename(self.name)
+
+    def sample(
+        self,
+        n: Optional[int] = None,
+        frac: Optional[float] = None,
+        replace: bool = False,
+        random_state: Optional[int] = None,
+    ) -> "Series":
+        return first_series(
+            self.to_frame().sample(n=n, frac=frac, replace=replace, random_state=random_state)
+        ).rename(self.name)
+
+    sample.__doc__ = DataFrame.sample.__doc__
+
+    def hist(self, bins=10, **kwds):
+        return self.plot.hist(bins, **kwds)
+
+    hist.__doc__ = PandasOnSparkPlotAccessor.hist.__doc__
+
+    def apply(self, func, args=(), **kwds) -> "Series":
+        """
+        Invoke function on values of Series.
+
+        Can be a Python function that only works on the Series.
+
+        .. note:: this API executes the function once to infer the type which is
+             potentially expensive, for instance, when the dataset is created after
+             aggregations or sorting.
+
+             To avoid this, specify return type in ``func``, for instance, as below:
+
+             >>> def square(x) -> np.int32:
+             ...     return x ** 2
+
+             pandas-on-Spark uses return type hint and does not try to infer the type.
+
+        Parameters
+        ----------
+        func : function
+            Python function to apply. Note that type hint for return type is required.
+        args : tuple
+            Positional arguments passed to func after the series value.
+        **kwds
+            Additional keyword arguments passed to func.
+
+        Returns
+        -------
+        Series
+
+        See Also
+        --------
+        Series.aggregate : Only perform aggregating type operations.
+        Series.transform : Only perform transforming type operations.
+        DataFrame.apply : The equivalent function for DataFrame.
+
+        Examples
+        --------
+        Create a Series with typical summer temperatures for each city.
+
+        >>> s = ps.Series([20, 21, 12],
+        ...               index=['London', 'New York', 'Helsinki'])
+        >>> s
+        London      20
+        New York    21
+        Helsinki    12
+        dtype: int64
+
+
+        Square the values by defining a function and passing it as an
+        argument to ``apply()``.
+
+        >>> def square(x) -> np.int64:
+        ...     return x ** 2
+        >>> s.apply(square)
+        London      400
+        New York    441
+        Helsinki    144
+        dtype: int64
+
+
+        Define a custom function that needs additional positional
+        arguments and pass these additional arguments using the
+        ``args`` keyword
+
+        >>> def subtract_custom_value(x, custom_value) -> np.int64:
+        ...     return x - custom_value
+
+        >>> s.apply(subtract_custom_value, args=(5,))
+        London      15
+        New York    16
+        Helsinki     7
+        dtype: int64
+
+
+        Define a custom function that takes keyword arguments
+        and pass these arguments to ``apply``
+
+        >>> def add_custom_values(x, **kwargs) -> np.int64:
+        ...     for month in kwargs:
+        ...         x += kwargs[month]
+        ...     return x
+
+        >>> s.apply(add_custom_values, june=30, july=20, august=25)
+        London      95
+        New York    96
+        Helsinki    87
+        dtype: int64
+
+
+        Use a function from the Numpy library
+
+        >>> def numpy_log(col) -> np.float64:
+        ...     return np.log(col)
+        >>> s.apply(numpy_log)
+        London      2.995732
+        New York    3.044522
+        Helsinki    2.484907
+        dtype: float64
+
+
+        You can omit the type hint and let pandas-on-Spark infer its type.
+
+        >>> s.apply(np.log)
+        London      2.995732
+        New York    3.044522
+        Helsinki    2.484907
+        dtype: float64
+
+        """
+        assert callable(func), "the first argument should be a callable function."
+        try:
+            spec = inspect.getfullargspec(func)
+            return_sig = spec.annotations.get("return", None)
+            should_infer_schema = return_sig is None
+        except TypeError:
+            # Falls back to schema inference if it fails to get signature.
+            should_infer_schema = True
+
+        apply_each = wraps(func)(lambda s: s.apply(func, args=args, **kwds))
+
+        if should_infer_schema:
+            return self.koalas._transform_batch(apply_each, None)
+        else:
+            sig_return = infer_return_type(func)
+            if not isinstance(sig_return, ScalarType):
+                raise ValueError(
+                    "Expected the return type of this function to be of scalar type, "
+                    "but found type {}".format(sig_return)
+                )
+            return_type = cast(ScalarType, sig_return)
+            return self.koalas._transform_batch(apply_each, return_type)
+
+    # TODO: not all arguments are implemented comparing to pandas' for now.
+    def aggregate(self, func: Union[str, List[str]]) -> Union[Scalar, "Series"]:
+        """Aggregate using one or more operations over the specified axis.
+
+        Parameters
+        ----------
+        func : str or a list of str
+            function name(s) as string apply to series.
+
+        Returns
+        -------
+        scalar, Series
+            The return can be:
+            - scalar : when Series.agg is called with single function
+            - Series : when Series.agg is called with several functions
+
+        Notes
+        -----
+        `agg` is an alias for `aggregate`. Use the alias.
+
+        See Also
+        --------
+        Series.apply : Invoke function on a Series.
+        Series.transform : Only perform transforming type operations.
+        Series.groupby : Perform operations over groups.
+        DataFrame.aggregate : The equivalent function for DataFrame.
+
+        Examples
+        --------
+        >>> s = ps.Series([1, 2, 3, 4])
+        >>> s.agg('min')
+        1
+
+        >>> s.agg(['min', 'max']).sort_index()
+        max    4
+        min    1
+        dtype: int64
+        """
+        if isinstance(func, list):
+            return first_series(self.to_frame().aggregate(func)).rename(self.name)
+        elif isinstance(func, str):
+            return getattr(self, func)()
+        else:
+            raise TypeError("func must be a string or list of strings")
+
+    agg = aggregate
+
+    def transpose(self, *args, **kwargs) -> "Series":
+        """
+        Return the transpose, which is by definition self.
+
+        Examples
+        --------
+        It returns the same object as the transpose of the given series object, which is by
+        definition self.
+
+        >>> s = ps.Series([1, 2, 3])
+        >>> s
+        0    1
+        1    2
+        2    3
+        dtype: int64
+
+        >>> s.transpose()
+        0    1
+        1    2
+        2    3
+        dtype: int64
+        """
+        return self.copy()
+
+    T = property(transpose)
+
+    def transform(self, func, axis=0, *args, **kwargs) -> Union["Series", DataFrame]:
+        """
+        Call ``func`` producing the same type as `self` with transformed values
+        and that has the same axis length as input.
+
+        .. note:: this API executes the function once to infer the type which is
+             potentially expensive, for instance, when the dataset is created after
+             aggregations or sorting.
+
+             To avoid this, specify return type in ``func``, for instance, as below:
+
+             >>> def square(x) -> np.int32:
+             ...     return x ** 2
+
+             pandas-on-Spark uses return type hint and does not try to infer the type.
+
+        Parameters
+        ----------
+        func : function or list
+            A function or a list of functions to use for transforming the data.
+        axis : int, default 0 or 'index'
+            Can only be set to 0 at the moment.
+        *args
+            Positional arguments to pass to `func`.
+        **kwargs
+            Keyword arguments to pass to `func`.
+
+        Returns
+        -------
+        An instance of the same type with `self` that must have the same length as input.
+
+        See Also
+        --------
+        Series.aggregate : Only perform aggregating type operations.
+        Series.apply : Invoke function on Series.
+        DataFrame.transform : The equivalent function for DataFrame.
+
+        Examples
+        --------
+
+        >>> s = ps.Series(range(3))
+        >>> s
+        0    0
+        1    1
+        2    2
+        dtype: int64
+
+        >>> def sqrt(x) -> float:
+        ...     return np.sqrt(x)
+        >>> s.transform(sqrt)
+        0    0.000000
+        1    1.000000
+        2    1.414214
+        dtype: float64
+
+        Even though the resulting instance must have the same length as the
+        input, it is possible to provide several input functions:
+
+        >>> def exp(x) -> float:
+        ...     return np.exp(x)
+        >>> s.transform([sqrt, exp])
+               sqrt       exp
+        0  0.000000  1.000000
+        1  1.000000  2.718282
+        2  1.414214  7.389056
+
+        You can omit the type hint and let pandas-on-Spark infer its type.
+
+        >>> s.transform([np.sqrt, np.exp])
+               sqrt       exp
+        0  0.000000  1.000000
+        1  1.000000  2.718282
+        2  1.414214  7.389056
+        """
+        axis = validate_axis(axis)
+        if axis != 0:
+            raise NotImplementedError('axis should be either 0 or "index" currently.')
+
+        if isinstance(func, list):
+            applied = []
+            for f in func:
+                applied.append(self.apply(f, args=args, **kwargs).rename(f.__name__))
+
+            internal = self._internal.with_new_columns(applied)
+            return DataFrame(internal)
+        else:
+            return self.apply(func, args=args, **kwargs)
+
+    def transform_batch(self, func, *args, **kwargs) -> "ps.Series":
+        warnings.warn(
+            "Series.transform_batch is deprecated as of Series.koalas.transform_batch. "
+            "Please use the API instead.",
+            FutureWarning,
+        )
+        return self.koalas.transform_batch(func, *args, **kwargs)
+
+    transform_batch.__doc__ = PandasOnSparkSeriesMethods.transform_batch.__doc__
+
+    def round(self, decimals=0) -> "Series":
+        """
+        Round each value in a Series to the given number of decimals.
+
+        Parameters
+        ----------
+        decimals : int
+            Number of decimal places to round to (default: 0).
+            If decimals is negative, it specifies the number of
+            positions to the left of the decimal point.
+
+        Returns
+        -------
+        Series object
+
+        See Also
+        --------
+        DataFrame.round
+
+        Examples
+        --------
+        >>> df = ps.Series([0.028208, 0.038683, 0.877076], name='x')
+        >>> df
+        0    0.028208
+        1    0.038683
+        2    0.877076
+        Name: x, dtype: float64
+
+        >>> df.round(2)
+        0    0.03
+        1    0.04
+        2    0.88
+        Name: x, dtype: float64
+        """
+        if not isinstance(decimals, int):
+            raise TypeError("decimals must be an integer")
+        scol = F.round(self.spark.column, decimals)
+        return self._with_new_scol(scol)
+
+    # TODO: add 'interpolation' parameter.
+    def quantile(
+        self, q: Union[float, Iterable[float]] = 0.5, accuracy: int = 10000
+    ) -> Union[Scalar, "Series"]:
+        """
+        Return value at the given quantile.
+
+        .. note:: Unlike pandas', the quantile in pandas-on-Spark is an approximated quantile
+            based upon approximate percentile computation because computing quantile across
+            a large dataset is extremely expensive.
+
+        Parameters
+        ----------
+        q : float or array-like, default 0.5 (50% quantile)
+            0 <= q <= 1, the quantile(s) to compute.
+        accuracy : int, optional
+            Default accuracy of approximation. Larger value means better accuracy.
+            The relative error can be deduced by 1.0 / accuracy.
+
+        Returns
+        -------
+        float or Series
+            If the current object is a Series and ``q`` is an array, a Series will be
+            returned where the index is ``q`` and the values are the quantiles, otherwise
+            a float will be returned.
+
+        Examples
+        --------
+        >>> s = ps.Series([1, 2, 3, 4, 5])
+        >>> s.quantile(.5)
+        3.0
+
+        >>> (s + 1).quantile(.5)
+        4.0
+
+        >>> s.quantile([.25, .5, .75])
+        0.25    2.0
+        0.50    3.0
+        0.75    4.0
+        dtype: float64
+
+        >>> (s + 1).quantile([.25, .5, .75])
+        0.25    3.0
+        0.50    4.0
+        0.75    5.0
+        dtype: float64
+        """
+        if isinstance(q, Iterable):
+            return first_series(
+                self.to_frame().quantile(q=q, axis=0, numeric_only=False, accuracy=accuracy)
+            ).rename(self.name)
+        else:
+            if not isinstance(accuracy, int):
+                raise TypeError(
+                    "accuracy must be an integer; however, got [%s]" % type(accuracy).__name__
+                )
+
+            if not isinstance(q, float):
+                raise TypeError(
+                    "q must be a float or an array of floats; however, [%s] found." % type(q)
+                )
+            if q < 0.0 or q > 1.0:
+                raise ValueError("percentiles should all be in the interval [0, 1].")
+
+            def quantile(spark_column, spark_type):
+                if isinstance(spark_type, (BooleanType, NumericType)):
+                    return F.percentile_approx(spark_column.cast(DoubleType()), q, accuracy)
+                else:
+                    raise TypeError(
+                        "Could not convert {} ({}) to numeric".format(
+                            spark_type_to_pandas_dtype(spark_type), spark_type.simpleString()
+                        )
+                    )
+
+            return self._reduce_for_stat_function(quantile, name="quantile")
+
+    # TODO: add axis, numeric_only, pct, na_option parameter
+    def rank(self, method="average", ascending=True) -> "Series":
+        """
+        Compute numerical data ranks (1 through n) along axis. Equal values are
+        assigned a rank that is the average of the ranks of those values.
+
+        .. note:: the current implementation of rank uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Parameters
+        ----------
+        method : {'average', 'min', 'max', 'first', 'dense'}
+            * average: average rank of group
+            * min: lowest rank in group
+            * max: highest rank in group
+            * first: ranks assigned in order they appear in the array
+            * dense: like 'min', but rank always increases by 1 between groups
+        ascending : boolean, default True
+            False for ranks by high (1) to low (N)
+
+        Returns
+        -------
+        ranks : same type as caller
+
+        Examples
+        --------
+        >>> s = ps.Series([1, 2, 2, 3], name='A')
+        >>> s
+        0    1
+        1    2
+        2    2
+        3    3
+        Name: A, dtype: int64
+
+        >>> s.rank()
+        0    1.0
+        1    2.5
+        2    2.5
+        3    4.0
+        Name: A, dtype: float64
+
+        If method is set to 'min', it use lowest rank in group.
+
+        >>> s.rank(method='min')
+        0    1.0
+        1    2.0
+        2    2.0
+        3    4.0
+        Name: A, dtype: float64
+
+        If method is set to 'max', it use highest rank in group.
+
+        >>> s.rank(method='max')
+        0    1.0
+        1    3.0
+        2    3.0
+        3    4.0
+        Name: A, dtype: float64
+
+        If method is set to 'first', it is assigned rank in order without groups.
+
+        >>> s.rank(method='first')
+        0    1.0
+        1    2.0
+        2    3.0
+        3    4.0
+        Name: A, dtype: float64
+
+        If method is set to 'dense', it leaves no gaps in group.
+
+        >>> s.rank(method='dense')
+        0    1.0
+        1    2.0
+        2    2.0
+        3    3.0
+        Name: A, dtype: float64
+        """
+        return self._rank(method, ascending).spark.analyzed
+
+    def _rank(self, method="average", ascending=True, *, part_cols=()):
+        if method not in ["average", "min", "max", "first", "dense"]:
+            msg = "method must be one of 'average', 'min', 'max', 'first', 'dense'"
+            raise ValueError(msg)
+
+        if self._internal.index_level > 1:
+            raise ValueError("rank do not support index now")
+
+        if ascending:
+            asc_func = lambda scol: scol.asc()
+        else:
+            asc_func = lambda scol: scol.desc()
+
+        if method == "first":
+            window = (
+                Window.orderBy(
+                    asc_func(self.spark.column), asc_func(F.col(NATURAL_ORDER_COLUMN_NAME)),
+                )
+                .partitionBy(*part_cols)
+                .rowsBetween(Window.unboundedPreceding, Window.currentRow)
+            )
+            scol = F.row_number().over(window)
+        elif method == "dense":
+            window = (
+                Window.orderBy(asc_func(self.spark.column))
+                .partitionBy(*part_cols)
+                .rowsBetween(Window.unboundedPreceding, Window.currentRow)
+            )
+            scol = F.dense_rank().over(window)
+        else:
+            if method == "average":
+                stat_func = F.mean
+            elif method == "min":
+                stat_func = F.min
+            elif method == "max":
+                stat_func = F.max
+            window1 = (
+                Window.orderBy(asc_func(self.spark.column))
+                .partitionBy(*part_cols)
+                .rowsBetween(Window.unboundedPreceding, Window.currentRow)
+            )
+            window2 = Window.partitionBy([self.spark.column] + list(part_cols)).rowsBetween(
+                Window.unboundedPreceding, Window.unboundedFollowing
+            )
+            scol = stat_func(F.row_number().over(window1)).over(window2)
+        kser = self._with_new_scol(scol)
+        return kser.astype(np.float64)
+
+    def filter(self, items=None, like=None, regex=None, axis=None) -> "Series":
+        axis = validate_axis(axis)
+        if axis == 1:
+            raise ValueError("Series does not support columns axis.")
+        return first_series(
+            self.to_frame().filter(items=items, like=like, regex=regex, axis=axis)
+        ).rename(self.name)
+
+    filter.__doc__ = DataFrame.filter.__doc__
+
+    def describe(self, percentiles: Optional[List[float]] = None) -> "Series":
+        return first_series(self.to_frame().describe(percentiles)).rename(self.name)
+
+    describe.__doc__ = DataFrame.describe.__doc__
+
+    def diff(self, periods=1) -> "Series":
+        """
+        First discrete difference of element.
+
+        Calculates the difference of a Series element compared with another element in the
+        DataFrame (default is the element in the same column of the previous row).
+
+        .. note:: the current implementation of diff uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Parameters
+        ----------
+        periods : int, default 1
+            Periods to shift for calculating difference, accepts negative values.
+
+        Returns
+        -------
+        diffed : Series
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'a': [1, 2, 3, 4, 5, 6],
+        ...                    'b': [1, 1, 2, 3, 5, 8],
+        ...                    'c': [1, 4, 9, 16, 25, 36]}, columns=['a', 'b', 'c'])
+        >>> df
+           a  b   c
+        0  1  1   1
+        1  2  1   4
+        2  3  2   9
+        3  4  3  16
+        4  5  5  25
+        5  6  8  36
+
+        >>> df.b.diff()
+        0    NaN
+        1    0.0
+        2    1.0
+        3    1.0
+        4    2.0
+        5    3.0
+        Name: b, dtype: float64
+
+        Difference with previous value
+
+        >>> df.c.diff(periods=3)
+        0     NaN
+        1     NaN
+        2     NaN
+        3    15.0
+        4    21.0
+        5    27.0
+        Name: c, dtype: float64
+
+        Difference with following value
+
+        >>> df.c.diff(periods=-1)
+        0    -3.0
+        1    -5.0
+        2    -7.0
+        3    -9.0
+        4   -11.0
+        5     NaN
+        Name: c, dtype: float64
+        """
+        return self._diff(periods).spark.analyzed
+
+    def _diff(self, periods, *, part_cols=()):
+        if not isinstance(periods, int):
+            raise TypeError("periods should be an int; however, got [%s]" % type(periods).__name__)
+        window = (
+            Window.partitionBy(*part_cols)
+            .orderBy(NATURAL_ORDER_COLUMN_NAME)
+            .rowsBetween(-periods, -periods)
+        )
+        scol = self.spark.column - F.lag(self.spark.column, periods).over(window)
+        return self._with_new_scol(scol, dtype=self.dtype)
+
+    def idxmax(self, skipna=True) -> Union[Tuple, Any]:
+        """
+        Return the row label of the maximum value.
+
+        If multiple values equal the maximum, the first row label with that
+        value is returned.
+
+        Parameters
+        ----------
+        skipna : bool, default True
+            Exclude NA/null values. If the entire Series is NA, the result
+            will be NA.
+
+        Returns
+        -------
+        Index
+            Label of the maximum value.
+
+        Raises
+        ------
+        ValueError
+            If the Series is empty.
+
+        See Also
+        --------
+        Series.idxmin : Return index *label* of the first occurrence
+            of minimum of values.
+
+        Examples
+        --------
+        >>> s = ps.Series(data=[1, None, 4, 3, 5],
+        ...               index=['A', 'B', 'C', 'D', 'E'])
+        >>> s
+        A    1.0
+        B    NaN
+        C    4.0
+        D    3.0
+        E    5.0
+        dtype: float64
+
+        >>> s.idxmax()
+        'E'
+
+        If `skipna` is False and there is an NA value in the data,
+        the function returns ``nan``.
+
+        >>> s.idxmax(skipna=False)
+        nan
+
+        In case of multi-index, you get a tuple:
+
+        >>> index = pd.MultiIndex.from_arrays([
+        ...     ['a', 'a', 'b', 'b'], ['c', 'd', 'e', 'f']], names=('first', 'second'))
+        >>> s = ps.Series(data=[1, None, 4, 5], index=index)
+        >>> s
+        first  second
+        a      c         1.0
+               d         NaN
+        b      e         4.0
+               f         5.0
+        dtype: float64
+
+        >>> s.idxmax()
+        ('b', 'f')
+
+        If multiple values equal the maximum, the first row label with that
+        value is returned.
+
+        >>> s = ps.Series([1, 100, 1, 100, 1, 100], index=[10, 3, 5, 2, 1, 8])
+        >>> s
+        10      1
+        3     100
+        5       1
+        2     100
+        1       1
+        8     100
+        dtype: int64
+
+        >>> s.idxmax()
+        3
+        """
+        sdf = self._internal.spark_frame
+        scol = self.spark.column
+        index_scols = self._internal.index_spark_columns
+        # desc_nulls_(last|first) is used via Py4J directly because
+        # it's not supported in Spark 2.3.
+        if skipna:
+            sdf = sdf.orderBy(Column(scol._jc.desc_nulls_last()), NATURAL_ORDER_COLUMN_NAME)
+        else:
+            sdf = sdf.orderBy(Column(scol._jc.desc_nulls_first()), NATURAL_ORDER_COLUMN_NAME)
+        results = sdf.select([scol] + index_scols).take(1)
+        if len(results) == 0:
+            raise ValueError("attempt to get idxmin of an empty sequence")
+        if results[0][0] is None:
+            # This will only happens when skipna is False because we will
+            # place nulls first.
+            return np.nan
+        values = list(results[0][1:])
+        if len(values) == 1:
+            return values[0]
+        else:
+            return tuple(values)
+
+    def idxmin(self, skipna=True) -> Union[Tuple, Any]:
+        """
+        Return the row label of the minimum value.
+
+        If multiple values equal the minimum, the first row label with that
+        value is returned.
+
+        Parameters
+        ----------
+        skipna : bool, default True
+            Exclude NA/null values. If the entire Series is NA, the result
+            will be NA.
+
+        Returns
+        -------
+        Index
+            Label of the minimum value.
+
+        Raises
+        ------
+        ValueError
+            If the Series is empty.
+
+        See Also
+        --------
+        Series.idxmax : Return index *label* of the first occurrence
+            of maximum of values.
+
+        Notes
+        -----
+        This method is the Series version of ``ndarray.argmin``. This method
+        returns the label of the minimum, while ``ndarray.argmin`` returns
+        the position. To get the position, use ``series.values.argmin()``.
+
+        Examples
+        --------
+        >>> s = ps.Series(data=[1, None, 4, 0],
+        ...               index=['A', 'B', 'C', 'D'])
+        >>> s
+        A    1.0
+        B    NaN
+        C    4.0
+        D    0.0
+        dtype: float64
+
+        >>> s.idxmin()
+        'D'
+
+        If `skipna` is False and there is an NA value in the data,
+        the function returns ``nan``.
+
+        >>> s.idxmin(skipna=False)
+        nan
+
+        In case of multi-index, you get a tuple:
+
+        >>> index = pd.MultiIndex.from_arrays([
+        ...     ['a', 'a', 'b', 'b'], ['c', 'd', 'e', 'f']], names=('first', 'second'))
+        >>> s = ps.Series(data=[1, None, 4, 0], index=index)
+        >>> s
+        first  second
+        a      c         1.0
+               d         NaN
+        b      e         4.0
+               f         0.0
+        dtype: float64
+
+        >>> s.idxmin()
+        ('b', 'f')
+
+        If multiple values equal the minimum, the first row label with that
+        value is returned.
+
+        >>> s = ps.Series([1, 100, 1, 100, 1, 100], index=[10, 3, 5, 2, 1, 8])
+        >>> s
+        10      1
+        3     100
+        5       1
+        2     100
+        1       1
+        8     100
+        dtype: int64
+
+        >>> s.idxmin()
+        10
+        """
+        sdf = self._internal.spark_frame
+        scol = self.spark.column
+        index_scols = self._internal.index_spark_columns
+        # asc_nulls_(last|first)is used via Py4J directly because
+        # it's not supported in Spark 2.3.
+        if skipna:
+            sdf = sdf.orderBy(Column(scol._jc.asc_nulls_last()), NATURAL_ORDER_COLUMN_NAME)
+        else:
+            sdf = sdf.orderBy(Column(scol._jc.asc_nulls_first()), NATURAL_ORDER_COLUMN_NAME)
+        results = sdf.select([scol] + index_scols).take(1)
+        if len(results) == 0:
+            raise ValueError("attempt to get idxmin of an empty sequence")
+        if results[0][0] is None:
+            # This will only happens when skipna is False because we will
+            # place nulls first.
+            return np.nan
+        values = list(results[0][1:])
+        if len(values) == 1:
+            return values[0]
+        else:
+            return tuple(values)
+
+    def pop(self, item) -> Union["Series", Scalar]:
+        """
+        Return item and drop from series.
+
+        Parameters
+        ----------
+        item : str
+            Label of index to be popped.
+
+        Returns
+        -------
+        Value that is popped from series.
+
+        Examples
+        --------
+        >>> s = ps.Series(data=np.arange(3), index=['A', 'B', 'C'])
+        >>> s
+        A    0
+        B    1
+        C    2
+        dtype: int64
+
+        >>> s.pop('A')
+        0
+
+        >>> s
+        B    1
+        C    2
+        dtype: int64
+
+        >>> s = ps.Series(data=np.arange(3), index=['A', 'A', 'C'])
+        >>> s
+        A    0
+        A    1
+        C    2
+        dtype: int64
+
+        >>> s.pop('A')
+        A    0
+        A    1
+        dtype: int64
+
+        >>> s
+        C    2
+        dtype: int64
+
+        Also support for MultiIndex
+
+        >>> midx = pd.MultiIndex([['lama', 'cow', 'falcon'],
+        ...                       ['speed', 'weight', 'length']],
+        ...                      [[0, 0, 0, 1, 1, 1, 2, 2, 2],
+        ...                       [0, 1, 2, 0, 1, 2, 0, 1, 2]])
+        >>> s = ps.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3],
+        ...               index=midx)
+        >>> s
+        lama    speed      45.0
+                weight    200.0
+                length      1.2
+        cow     speed      30.0
+                weight    250.0
+                length      1.5
+        falcon  speed     320.0
+                weight      1.0
+                length      0.3
+        dtype: float64
+
+        >>> s.pop('lama')
+        speed      45.0
+        weight    200.0
+        length      1.2
+        dtype: float64
+
+        >>> s
+        cow     speed      30.0
+                weight    250.0
+                length      1.5
+        falcon  speed     320.0
+                weight      1.0
+                length      0.3
+        dtype: float64
+
+        Also support for MultiIndex with several indexs.
+
+        >>> midx = pd.MultiIndex([['a', 'b', 'c'],
+        ...                       ['lama', 'cow', 'falcon'],
+        ...                       ['speed', 'weight', 'length']],
+        ...                      [[0, 0, 0, 0, 0, 0, 1, 1, 1],
+        ...                       [0, 0, 0, 1, 1, 1, 2, 2, 2],
+        ...                       [0, 1, 2, 0, 1, 2, 0, 0, 2]]
+        ...  )
+        >>> s = ps.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3],
+        ...              index=midx)
+        >>> s
+        a  lama    speed      45.0
+                   weight    200.0
+                   length      1.2
+           cow     speed      30.0
+                   weight    250.0
+                   length      1.5
+        b  falcon  speed     320.0
+                   speed       1.0
+                   length      0.3
+        dtype: float64
+
+        >>> s.pop(('a', 'lama'))
+        speed      45.0
+        weight    200.0
+        length      1.2
+        dtype: float64
+
+        >>> s
+        a  cow     speed      30.0
+                   weight    250.0
+                   length      1.5
+        b  falcon  speed     320.0
+                   speed       1.0
+                   length      0.3
+        dtype: float64
+
+        >>> s.pop(('b', 'falcon', 'speed'))
+        (b, falcon, speed)    320.0
+        (b, falcon, speed)      1.0
+        dtype: float64
+        """
+        if not is_name_like_value(item):
+            raise TypeError("'key' should be string or tuple that contains strings")
+        if not is_name_like_tuple(item):
+            item = (item,)
+        if self._internal.index_level < len(item):
+            raise KeyError(
+                "Key length ({}) exceeds index depth ({})".format(
+                    len(item), self._internal.index_level
+                )
+            )
+
+        internal = self._internal
+        scols = internal.index_spark_columns[len(item):] + [self.spark.column]
+        rows = [internal.spark_columns[level] == index for level, index in enumerate(item)]
+        sdf = internal.spark_frame.filter(reduce(lambda x, y: x & y, rows)).select(scols)
+
+        kdf = self._drop(item)
+        self._update_anchor(kdf)
+
+        if self._internal.index_level == len(item):
+            # if spark_frame has one column and one data, return data only without frame
+            pdf = sdf.limit(2).toPandas()
+            length = len(pdf)
+            if length == 1:
+                return pdf[internal.data_spark_column_names[0]].iloc[0]
+
+            item_string = name_like_string(item)
+            sdf = sdf.withColumn(SPARK_DEFAULT_INDEX_NAME, F.lit(str(item_string)))
+            internal = InternalFrame(
+                spark_frame=sdf,
+                index_spark_columns=[scol_for(sdf, SPARK_DEFAULT_INDEX_NAME)],
+                column_labels=[self._column_label],
+                data_dtypes=[self.dtype],
+            )
+            return first_series(DataFrame(internal))
+        else:
+            internal = internal.copy(
+                spark_frame=sdf,
+                index_spark_columns=[
+                    scol_for(sdf, col) for col in internal.index_spark_column_names[len(item):]
+                ],
+                index_dtypes=internal.index_dtypes[len(item):],
+                index_names=self._internal.index_names[len(item):],
+                data_spark_columns=[scol_for(sdf, internal.data_spark_column_names[0])],
+            )
+            return first_series(DataFrame(internal))
+
+    def copy(self, deep=None) -> "Series":
+        """
+        Make a copy of this object's indices and data.
+
+        Parameters
+        ----------
+        deep : None
+            this parameter is not supported but just dummy parameter to match pandas.
+
+        Returns
+        -------
+        copy : Series
+
+        Examples
+        --------
+        >>> s = ps.Series([1, 2], index=["a", "b"])
+        >>> s
+        a    1
+        b    2
+        dtype: int64
+        >>> s_copy = s.copy()
+        >>> s_copy
+        a    1
+        b    2
+        dtype: int64
+        """
+        return self._kdf.copy()._kser_for(self._column_label)
+
+    def mode(self, dropna=True) -> "Series":
+        """
+        Return the mode(s) of the dataset.
+
+        Always returns Series even if only one value is returned.
+
+        Parameters
+        ----------
+        dropna : bool, default True
+            Don't consider counts of NaN/NaT.
+
+        Returns
+        -------
+        Series
+            Modes of the Series.
+
+        Examples
+        --------
+        >>> s = ps.Series([0, 0, 1, 1, 1, np.nan, np.nan, np.nan])
+        >>> s
+        0    0.0
+        1    0.0
+        2    1.0
+        3    1.0
+        4    1.0
+        5    NaN
+        6    NaN
+        7    NaN
+        dtype: float64
+
+        >>> s.mode()
+        0    1.0
+        dtype: float64
+
+        If there are several same modes, all items are shown
+
+        >>> s = ps.Series([0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3,
+        ...                np.nan, np.nan, np.nan])
+        >>> s
+        0     0.0
+        1     0.0
+        2     1.0
+        3     1.0
+        4     1.0
+        5     2.0
+        6     2.0
+        7     2.0
+        8     3.0
+        9     3.0
+        10    3.0
+        11    NaN
+        12    NaN
+        13    NaN
+        dtype: float64
+
+        >>> s.mode().sort_values()  # doctest: +NORMALIZE_WHITESPACE, +ELLIPSIS
+        <BLANKLINE>
+        ...  1.0
+        ...  2.0
+        ...  3.0
+        dtype: float64
+
+        With 'dropna' set to 'False', we can also see NaN in the result
+
+        >>> s.mode(False).sort_values()  # doctest: +NORMALIZE_WHITESPACE, +ELLIPSIS
+        <BLANKLINE>
+        ...  1.0
+        ...  2.0
+        ...  3.0
+        ...  NaN
+        dtype: float64
+        """
+        ser_count = self.value_counts(dropna=dropna, sort=False)
+        sdf_count = ser_count._internal.spark_frame
+        most_value = ser_count.max()
+        sdf_most_value = sdf_count.filter("count == {}".format(most_value))
+        sdf = sdf_most_value.select(
+            F.col(SPARK_DEFAULT_INDEX_NAME).alias(SPARK_DEFAULT_SERIES_NAME)
+        )
+        internal = InternalFrame(spark_frame=sdf, index_spark_columns=None, column_labels=[None])
+
+        return first_series(DataFrame(internal))
+
+    def keys(self) -> "ps.Index":
+        """
+        Return alias for index.
+
+        Returns
+        -------
+        Index
+            Index of the Series.
+
+        Examples
+        --------
+        >>> midx = pd.MultiIndex([['lama', 'cow', 'falcon'],
+        ...                       ['speed', 'weight', 'length']],
+        ...                      [[0, 0, 0, 1, 1, 1, 2, 2, 2],
+        ...                       [0, 1, 2, 0, 1, 2, 0, 1, 2]])
+        >>> kser = ps.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], index=midx)
+
+        >>> kser.keys()  # doctest: +SKIP
+        MultiIndex([(  'lama',  'speed'),
+                    (  'lama', 'weight'),
+                    (  'lama', 'length'),
+                    (   'cow',  'speed'),
+                    (   'cow', 'weight'),
+                    (   'cow', 'length'),
+                    ('falcon',  'speed'),
+                    ('falcon', 'weight'),
+                    ('falcon', 'length')],
+                   )
+        """
+        return self.index
+
+    # TODO: 'regex', 'method' parameter
+    def replace(self, to_replace=None, value=None, regex=False) -> "Series":
+        """
+        Replace values given in to_replace with value.
+        Values of the Series are replaced with other values dynamically.
+
+        Parameters
+        ----------
+        to_replace : str, list, tuple, dict, Series, int, float, or None
+            How to find the values that will be replaced.
+            * numeric, str:
+
+                - numeric: numeric values equal to to_replace will be replaced with value
+                - str: string exactly matching to_replace will be replaced with value
+
+            * list of str or numeric:
+
+                - if to_replace and value are both lists or tuples, they must be the same length.
+                - str and numeric rules apply as above.
+
+            * dict:
+
+                - Dicts can be used to specify different replacement values for different
+                  existing values.
+                  For example, {'a': 'b', 'y': 'z'} replaces the value ‘a’ with ‘b’ and ‘y’
+                  with ‘z’. To use a dict in this way the value parameter should be None.
+                - For a DataFrame a dict can specify that different values should be replaced
+                  in different columns. For example, {'a': 1, 'b': 'z'} looks for the value 1
+                  in column ‘a’ and the value ‘z’ in column ‘b’ and replaces these values with
+                  whatever is specified in value.
+                  The value parameter should not be None in this case.
+                  You can treat this as a special case of passing two lists except that you are
+                  specifying the column to search in.
+
+            See the examples section for examples of each of these.
+
+        value : scalar, dict, list, tuple, str default None
+            Value to replace any values matching to_replace with.
+            For a DataFrame a dict of values can be used to specify which value to use
+            for each column (columns not in the dict will not be filled).
+            Regular expressions, strings and lists or dicts of such objects are also allowed.
+
+        Returns
+        -------
+        Series
+            Object after replacement.
+
+        Examples
+        --------
+
+        Scalar `to_replace` and `value`
+
+        >>> s = ps.Series([0, 1, 2, 3, 4])
+        >>> s
+        0    0
+        1    1
+        2    2
+        3    3
+        4    4
+        dtype: int64
+
+        >>> s.replace(0, 5)
+        0    5
+        1    1
+        2    2
+        3    3
+        4    4
+        dtype: int64
+
+        List-like `to_replace`
+
+        >>> s.replace([0, 4], 5000)
+        0    5000
+        1       1
+        2       2
+        3       3
+        4    5000
+        dtype: int64
+
+        >>> s.replace([1, 2, 3], [10, 20, 30])
+        0     0
+        1    10
+        2    20
+        3    30
+        4     4
+        dtype: int64
+
+        Dict-like `to_replace`
+
+        >>> s.replace({1: 1000, 2: 2000, 3: 3000, 4: 4000})
+        0       0
+        1    1000
+        2    2000
+        3    3000
+        4    4000
+        dtype: int64
+
+        Also support for MultiIndex
+
+        >>> midx = pd.MultiIndex([['lama', 'cow', 'falcon'],
+        ...                       ['speed', 'weight', 'length']],
+        ...                      [[0, 0, 0, 1, 1, 1, 2, 2, 2],
+        ...                       [0, 1, 2, 0, 1, 2, 0, 1, 2]])
+        >>> s = ps.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3],
+        ...               index=midx)
+        >>> s
+        lama    speed      45.0
+                weight    200.0
+                length      1.2
+        cow     speed      30.0
+                weight    250.0
+                length      1.5
+        falcon  speed     320.0
+                weight      1.0
+                length      0.3
+        dtype: float64
+
+        >>> s.replace(45, 450)
+        lama    speed     450.0
+                weight    200.0
+                length      1.2
+        cow     speed      30.0
+                weight    250.0
+                length      1.5
+        falcon  speed     320.0
+                weight      1.0
+                length      0.3
+        dtype: float64
+
+        >>> s.replace([45, 30, 320], 500)
+        lama    speed     500.0
+                weight    200.0
+                length      1.2
+        cow     speed     500.0
+                weight    250.0
+                length      1.5
+        falcon  speed     500.0
+                weight      1.0
+                length      0.3
+        dtype: float64
+
+        >>> s.replace({45: 450, 30: 300})
+        lama    speed     450.0
+                weight    200.0
+                length      1.2
+        cow     speed     300.0
+                weight    250.0
+                length      1.5
+        falcon  speed     320.0
+                weight      1.0
+                length      0.3
+        dtype: float64
+        """
+        if to_replace is None:
+            return self.fillna(method="ffill")
+        if not isinstance(to_replace, (str, list, tuple, dict, int, float)):
+            raise TypeError("'to_replace' should be one of str, list, tuple, dict, int, float")
+        if regex:
+            raise NotImplementedError("replace currently not support for regex")
+        to_replace = list(to_replace) if isinstance(to_replace, tuple) else to_replace
+        value = list(value) if isinstance(value, tuple) else value
+        if isinstance(to_replace, list) and isinstance(value, list):
+            if not len(to_replace) == len(value):
+                raise ValueError(
+                    "Replacement lists must match in length. Expecting {} got {}".format(
+                        len(to_replace), len(value)
+                    )
+                )
+            to_replace = {k: v for k, v in zip(to_replace, value)}
+        if isinstance(to_replace, dict):
+            is_start = True
+            if len(to_replace) == 0:
+                current = self.spark.column
+            else:
+                for to_replace_, value in to_replace.items():
+                    cond = (
+                        (F.isnan(self.spark.column) | self.spark.column.isNull())
+                        if pd.isna(to_replace_)
+                        else (self.spark.column == F.lit(to_replace_))
+                    )
+                    if is_start:
+                        current = F.when(cond, value)
+                        is_start = False
+                    else:
+                        current = current.when(cond, value)
+                current = current.otherwise(self.spark.column)
+        else:
+            cond = self.spark.column.isin(to_replace)
+            # to_replace may be a scalar
+            if np.array(pd.isna(to_replace)).any():
+                cond = cond | F.isnan(self.spark.column) | self.spark.column.isNull()
+            current = F.when(cond, value).otherwise(self.spark.column)
+
+        return self._with_new_scol(current)  # TODO: dtype?
+
+    def update(self, other) -> None:
+        """
+        Modify Series in place using non-NA values from passed Series. Aligns on index.
+
+        Parameters
+        ----------
+        other : Series
+
+        Examples
+        --------
+        >>> from pyspark.pandas.config import set_option, reset_option
+        >>> set_option("compute.ops_on_diff_frames", True)
+        >>> s = ps.Series([1, 2, 3])
+        >>> s.update(ps.Series([4, 5, 6]))
+        >>> s.sort_index()
+        0    4
+        1    5
+        2    6
+        dtype: int64
+
+        >>> s = ps.Series(['a', 'b', 'c'])
+        >>> s.update(ps.Series(['d', 'e'], index=[0, 2]))
+        >>> s.sort_index()
+        0    d
+        1    b
+        2    e
+        dtype: object
+
+        >>> s = ps.Series([1, 2, 3])
+        >>> s.update(ps.Series([4, 5, 6, 7, 8]))
+        >>> s.sort_index()
+        0    4
+        1    5
+        2    6
+        dtype: int64
+
+        >>> s = ps.Series([1, 2, 3], index=[10, 11, 12])
+        >>> s
+        10    1
+        11    2
+        12    3
+        dtype: int64
+
+        >>> s.update(ps.Series([4, 5, 6]))
+        >>> s.sort_index()
+        10    1
+        11    2
+        12    3
+        dtype: int64
+
+        >>> s.update(ps.Series([4, 5, 6], index=[11, 12, 13]))
+        >>> s.sort_index()
+        10    1
+        11    4
+        12    5
+        dtype: int64
+
+        If ``other`` contains NaNs the corresponding values are not updated
+        in the original Series.
+
+        >>> s = ps.Series([1, 2, 3])
+        >>> s.update(ps.Series([4, np.nan, 6]))
+        >>> s.sort_index()
+        0    4.0
+        1    2.0
+        2    6.0
+        dtype: float64
+
+        >>> reset_option("compute.ops_on_diff_frames")
+        """
+        if not isinstance(other, Series):
+            raise TypeError("'other' must be a Series")
+
+        combined = combine_frames(self._kdf, other._kdf, how="leftouter")
+
+        this_scol = combined["this"]._internal.spark_column_for(self._column_label)
+        that_scol = combined["that"]._internal.spark_column_for(other._column_label)
+
+        scol = (
+            F.when(that_scol.isNotNull(), that_scol)
+            .otherwise(this_scol)
+            .alias(self._kdf._internal.spark_column_name_for(self._column_label))
+        )
+
+        internal = combined["this"]._internal.with_new_spark_column(
+            self._column_label, scol  # TODO: dtype?
+        )
+
+        self._kdf._update_internal_frame(internal.resolved_copy, requires_same_anchor=False)
+
+    def where(self, cond, other=np.nan) -> "Series":
+        """
+        Replace values where the condition is False.
+
+        Parameters
+        ----------
+        cond : boolean Series
+            Where cond is True, keep the original value. Where False,
+            replace with corresponding value from other.
+        other : scalar, Series
+            Entries where cond is False are replaced with corresponding value from other.
+
+        Returns
+        -------
+        Series
+
+        Examples
+        --------
+
+        >>> from pyspark.pandas.config import set_option, reset_option
+        >>> set_option("compute.ops_on_diff_frames", True)
+        >>> s1 = ps.Series([0, 1, 2, 3, 4])
+        >>> s2 = ps.Series([100, 200, 300, 400, 500])
+        >>> s1.where(s1 > 0).sort_index()
+        0    NaN
+        1    1.0
+        2    2.0
+        3    3.0
+        4    4.0
+        dtype: float64
+
+        >>> s1.where(s1 > 1, 10).sort_index()
+        0    10
+        1    10
+        2     2
+        3     3
+        4     4
+        dtype: int64
+
+        >>> s1.where(s1 > 1, s1 + 100).sort_index()
+        0    100
+        1    101
+        2      2
+        3      3
+        4      4
+        dtype: int64
+
+        >>> s1.where(s1 > 1, s2).sort_index()
+        0    100
+        1    200
+        2      2
+        3      3
+        4      4
+        dtype: int64
+
+        >>> reset_option("compute.ops_on_diff_frames")
+        """
+        assert isinstance(cond, Series)
+
+        # We should check the DataFrame from both `cond` and `other`.
+        should_try_ops_on_diff_frame = not same_anchor(cond, self) or (
+            isinstance(other, Series) and not same_anchor(other, self)
+        )
+
+        if should_try_ops_on_diff_frame:
+            # Try to perform it with 'compute.ops_on_diff_frame' option.
+            kdf = self.to_frame()
+            tmp_cond_col = verify_temp_column_name(kdf, "__tmp_cond_col__")
+            tmp_other_col = verify_temp_column_name(kdf, "__tmp_other_col__")
+
+            kdf[tmp_cond_col] = cond
+            kdf[tmp_other_col] = other
+
+            # above logic makes a Spark DataFrame looks like below:
+            # +-----------------+---+----------------+-----------------+
+            # |__index_level_0__|  0|__tmp_cond_col__|__tmp_other_col__|
+            # +-----------------+---+----------------+-----------------+
+            # |                0|  0|           false|              100|
+            # |                1|  1|           false|              200|
+            # |                3|  3|            true|              400|
+            # |                2|  2|            true|              300|
+            # |                4|  4|            true|              500|
+            # +-----------------+---+----------------+-----------------+
+            condition = (
+                F.when(
+                    kdf[tmp_cond_col].spark.column,
+                    kdf._kser_for(kdf._internal.column_labels[0]).spark.column,
+                )
+                .otherwise(kdf[tmp_other_col].spark.column)
+                .alias(kdf._internal.data_spark_column_names[0])
+            )
+
+            internal = kdf._internal.with_new_columns(
+                [condition], column_labels=self._internal.column_labels
+            )
+            return first_series(DataFrame(internal))
+        else:
+            if isinstance(other, Series):
+                other = other.spark.column
+            condition = (
+                F.when(cond.spark.column, self.spark.column)
+                .otherwise(other)
+                .alias(self._internal.data_spark_column_names[0])
+            )
+            return self._with_new_scol(condition)
+
+    def mask(self, cond, other=np.nan) -> "Series":
+        """
+        Replace values where the condition is True.
+
+        Parameters
+        ----------
+        cond : boolean Series
+            Where cond is False, keep the original value. Where True,
+            replace with corresponding value from other.
+        other : scalar, Series
+            Entries where cond is True are replaced with corresponding value from other.
+
+        Returns
+        -------
+        Series
+
+        Examples
+        --------
+
+        >>> from pyspark.pandas.config import set_option, reset_option
+        >>> set_option("compute.ops_on_diff_frames", True)
+        >>> s1 = ps.Series([0, 1, 2, 3, 4])
+        >>> s2 = ps.Series([100, 200, 300, 400, 500])
+        >>> s1.mask(s1 > 0).sort_index()
+        0    0.0
+        1    NaN
+        2    NaN
+        3    NaN
+        4    NaN
+        dtype: float64
+
+        >>> s1.mask(s1 > 1, 10).sort_index()
+        0     0
+        1     1
+        2    10
+        3    10
+        4    10
+        dtype: int64
+
+        >>> s1.mask(s1 > 1, s1 + 100).sort_index()
+        0      0
+        1      1
+        2    102
+        3    103
+        4    104
+        dtype: int64
+
+        >>> s1.mask(s1 > 1, s2).sort_index()
+        0      0
+        1      1
+        2    300
+        3    400
+        4    500
+        dtype: int64
+
+        >>> reset_option("compute.ops_on_diff_frames")
+        """
+        return self.where(~cond, other)
+
+    def xs(self, key, level=None) -> "Series":
+        """
+        Return cross-section from the Series.
+
+        This method takes a `key` argument to select data at a particular
+        level of a MultiIndex.
+
+        Parameters
+        ----------
+        key : label or tuple of label
+            Label contained in the index, or partially in a MultiIndex.
+        level : object, defaults to first n levels (n=1 or len(key))
+            In case of a key partially contained in a MultiIndex, indicate
+            which levels are used. Levels can be referred by label or position.
+
+        Returns
+        -------
+        Series
+            Cross-section from the original Series
+            corresponding to the selected index levels.
+
+        Examples
+        --------
+        >>> midx = pd.MultiIndex([['a', 'b', 'c'],
+        ...                       ['lama', 'cow', 'falcon'],
+        ...                       ['speed', 'weight', 'length']],
+        ...                      [[0, 0, 0, 1, 1, 1, 2, 2, 2],
+        ...                       [0, 0, 0, 1, 1, 1, 2, 2, 2],
+        ...                       [0, 1, 2, 0, 1, 2, 0, 1, 2]])
+        >>> s = ps.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3],
+        ...               index=midx)
+        >>> s
+        a  lama    speed      45.0
+                   weight    200.0
+                   length      1.2
+        b  cow     speed      30.0
+                   weight    250.0
+                   length      1.5
+        c  falcon  speed     320.0
+                   weight      1.0
+                   length      0.3
+        dtype: float64
+
+        Get values at specified index
+
+        >>> s.xs('a')
+        lama  speed      45.0
+              weight    200.0
+              length      1.2
+        dtype: float64
+
+        Get values at several indexes
+
+        >>> s.xs(('a', 'lama'))
+        speed      45.0
+        weight    200.0
+        length      1.2
+        dtype: float64
+
+        Get values at specified index and level
+
+        >>> s.xs('lama', level=1)
+        a  speed      45.0
+           weight    200.0
+           length      1.2
+        dtype: float64
+        """
+        if not isinstance(key, tuple):
+            key = (key,)
+        if level is None:
+            level = 0
+
+        internal = self._internal
+        scols = (
+            internal.index_spark_columns[:level]
+            + internal.index_spark_columns[level + len(key):]
+            + [self.spark.column]
+        )
+        rows = [internal.spark_columns[lvl] == index for lvl, index in enumerate(key, level)]
+        sdf = internal.spark_frame.filter(reduce(lambda x, y: x & y, rows)).select(scols)
+
+        if internal.index_level == len(key):
+            # if spark_frame has one column and one data, return data only without frame
+            pdf = sdf.limit(2).toPandas()
+            length = len(pdf)
+            if length == 1:
+                return pdf[self._internal.data_spark_column_names[0]].iloc[0]
+
+        index_spark_column_names = (
+            internal.index_spark_column_names[:level]
+            + internal.index_spark_column_names[level + len(key):]
+        )
+        index_names = internal.index_names[:level] + internal.index_names[level + len(key):]
+        index_dtypes = internal.index_dtypes[:level] + internal.index_dtypes[level + len(key):]
+
+        internal = internal.copy(
+            spark_frame=sdf,
+            index_spark_columns=[scol_for(sdf, col) for col in index_spark_column_names],
+            index_names=index_names,
+            index_dtypes=index_dtypes,
+            data_spark_columns=[scol_for(sdf, internal.data_spark_column_names[0])],
+        )
+        return first_series(DataFrame(internal))
+
+    def pct_change(self, periods=1) -> "Series":
+        """
+        Percentage change between the current and a prior element.
+
+        .. note:: the current implementation of this API uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Parameters
+        ----------
+        periods : int, default 1
+            Periods to shift for forming percent change.
+
+        Returns
+        -------
+        Series
+
+        Examples
+        --------
+
+        >>> kser = ps.Series([90, 91, 85], index=[2, 4, 1])
+        >>> kser
+        2    90
+        4    91
+        1    85
+        dtype: int64
+
+        >>> kser.pct_change()
+        2         NaN
+        4    0.011111
+        1   -0.065934
+        dtype: float64
+
+        >>> kser.sort_index().pct_change()
+        1         NaN
+        2    0.058824
+        4    0.011111
+        dtype: float64
+
+        >>> kser.pct_change(periods=2)
+        2         NaN
+        4         NaN
+        1   -0.055556
+        dtype: float64
+        """
+        scol = self.spark.column
+
+        window = Window.orderBy(NATURAL_ORDER_COLUMN_NAME).rowsBetween(-periods, -periods)
+        prev_row = F.lag(scol, periods).over(window)
+
+        return self._with_new_scol((scol - prev_row) / prev_row).spark.analyzed
+
+    def combine_first(self, other) -> "Series":
+        """
+        Combine Series values, choosing the calling Series's values first.
+
+        Parameters
+        ----------
+        other : Series
+            The value(s) to be combined with the `Series`.
+
+        Returns
+        -------
+        Series
+            The result of combining the Series with the other object.
+
+        See Also
+        --------
+        Series.combine : Perform elementwise operation on two Series
+            using a given function.
+
+        Notes
+        -----
+        Result index will be the union of the two indexes.
+
+        Examples
+        --------
+        >>> s1 = ps.Series([1, np.nan])
+        >>> s2 = ps.Series([3, 4])
+        >>> with ps.option_context("compute.ops_on_diff_frames", True):
+        ...     s1.combine_first(s2)
+        0    1.0
+        1    4.0
+        dtype: float64
+        """
+        if not isinstance(other, ps.Series):
+            raise TypeError("`combine_first` only allows `Series` for parameter `other`")
+        if same_anchor(self, other):
+            this = self.spark.column
+            that = other.spark.column
+            combined = self._kdf
+        else:
+            combined = combine_frames(self._kdf, other._kdf)
+            this = combined["this"]._internal.spark_column_for(self._column_label)
+            that = combined["that"]._internal.spark_column_for(other._column_label)
+        # If `self` has missing value, use value of `other`
+        cond = F.when(this.isNull(), that).otherwise(this)
+        # If `self` and `other` come from same frame, the anchor should be kept
+        if same_anchor(self, other):
+            return self._with_new_scol(cond)  # TODO: dtype?
+        index_scols = combined._internal.index_spark_columns
+        sdf = combined._internal.spark_frame.select(
+            *index_scols, cond.alias(self._internal.data_spark_column_names[0])
+        ).distinct()
+        internal = self._internal.with_new_sdf(sdf, data_dtypes=[None])  # TODO: dtype?
+        return first_series(DataFrame(internal))
+
+    def dot(self, other: Union["Series", DataFrame]) -> Union[Scalar, "Series"]:
+        """
+        Compute the dot product between the Series and the columns of other.
+
+        This method computes the dot product between the Series and another
+        one, or the Series and each columns of a DataFrame.
+
+        It can also be called using `self @ other` in Python >= 3.5.
+
+        .. note:: This API is slightly different from pandas when indexes from both Series
+            are not aligned. To match with pandas', it requires to read the whole data for,
+            for example, counting. pandas raises an exception; however, pandas-on-Spark
+            just proceeds and performs by ignoring mismatches with NaN permissively.
+
+            >>> pdf1 = pd.Series([1, 2, 3], index=[0, 1, 2])
+            >>> pdf2 = pd.Series([1, 2, 3], index=[0, 1, 3])
+            >>> pdf1.dot(pdf2)  # doctest: +SKIP
+            ...
+            ValueError: matrices are not aligned
+
+            >>> kdf1 = ps.Series([1, 2, 3], index=[0, 1, 2])
+            >>> kdf2 = ps.Series([1, 2, 3], index=[0, 1, 3])
+            >>> kdf1.dot(kdf2)  # doctest: +SKIP
+            5
+
+        Parameters
+        ----------
+        other : Series, DataFrame.
+            The other object to compute the dot product with its columns.
+
+        Returns
+        -------
+        scalar, Series
+            Return the dot product of the Series and other if other is a
+            Series, the Series of the dot product of Series and each rows of
+            other if other is a DataFrame.
+
+        Notes
+        -----
+        The Series and other has to share the same index if other is a Series
+        or a DataFrame.
+
+        Examples
+        --------
+        >>> s = ps.Series([0, 1, 2, 3])
+
+        >>> s.dot(s)
+        14
+
+        >>> s @ s
+        14
+
+        >>> kdf = ps.DataFrame({'x': [0, 1, 2, 3], 'y': [0, -1, -2, -3]})
+        >>> kdf
+           x  y
+        0  0  0
+        1  1 -1
+        2  2 -2
+        3  3 -3
+
+        >>> with ps.option_context("compute.ops_on_diff_frames", True):
+        ...     s.dot(kdf)
+        ...
+        x    14
+        y   -14
+        dtype: int64
+        """
+        if isinstance(other, DataFrame):
+            if not same_anchor(self, other):
+                if not self.index.sort_values().equals(other.index.sort_values()):
+                    raise ValueError("matrices are not aligned")
+
+            other = other.copy()
+            column_labels = other._internal.column_labels
+
+            self_column_label = verify_temp_column_name(other, "__self_column__")
+            other[self_column_label] = self
+            self_kser = other._kser_for(self_column_label)
+
+            product_ksers = [other._kser_for(label) * self_kser for label in column_labels]
+
+            dot_product_kser = DataFrame(
+                other._internal.with_new_columns(product_ksers, column_labels=column_labels)
+            ).sum()
+
+            return cast(Series, dot_product_kser).rename(self.name)
+
+        else:
+            assert isinstance(other, Series)
+            if not same_anchor(self, other):
+                if len(self.index) != len(other.index):
+                    raise ValueError("matrices are not aligned")
+            return (self * other).sum()
+
+    def __matmul__(self, other):
+        """
+        Matrix multiplication using binary `@` operator in Python>=3.5.
+        """
+        return self.dot(other)
+
+    def repeat(self, repeats: Union[int, "Series"]) -> "Series":
+        """
+        Repeat elements of a Series.
+
+        Returns a new Series where each element of the current Series
+        is repeated consecutively a given number of times.
+
+        Parameters
+        ----------
+        repeats : int or Series
+            The number of repetitions for each element. This should be a
+            non-negative integer. Repeating 0 times will return an empty
+            Series.
+
+        Returns
+        -------
+        Series
+            Newly created Series with repeated elements.
+
+        See Also
+        --------
+        Index.repeat : Equivalent function for Index.
+
+        Examples
+        --------
+        >>> s = ps.Series(['a', 'b', 'c'])
+        >>> s
+        0    a
+        1    b
+        2    c
+        dtype: object
+        >>> s.repeat(2)
+        0    a
+        1    b
+        2    c
+        0    a
+        1    b
+        2    c
+        dtype: object
+        >>> ps.Series([1, 2, 3]).repeat(0)
+        Series([], dtype: int64)
+        """
+        if not isinstance(repeats, (int, Series)):
+            raise TypeError(
+                "`repeats` argument must be integer or Series, but got {}".format(type(repeats))
+            )
+
+        if isinstance(repeats, Series):
+            if not same_anchor(self, repeats):
+                kdf = self.to_frame()
+                temp_repeats = verify_temp_column_name(kdf, "__temp_repeats__")
+                kdf[temp_repeats] = repeats
+                return (
+                    kdf._kser_for(kdf._internal.column_labels[0])
+                    .repeat(kdf[temp_repeats])
+                    .rename(self.name)
+                )
+            else:
+                scol = F.explode(
+                    F.array_repeat(self.spark.column, repeats.astype("int32").spark.column)
+                ).alias(name_like_string(self.name))
+                sdf = self._internal.spark_frame.select(self._internal.index_spark_columns + [scol])
+                internal = self._internal.copy(
+                    spark_frame=sdf,
+                    index_spark_columns=[
+                        scol_for(sdf, col) for col in self._internal.index_spark_column_names
+                    ],
+                    data_spark_columns=[scol_for(sdf, name_like_string(self.name))],
+                )
+                return first_series(DataFrame(internal))
+        else:
+            if repeats < 0:
+                raise ValueError("negative dimensions are not allowed")
+
+            kdf = self._kdf[[self.name]]
+            if repeats == 0:
+                return first_series(DataFrame(kdf._internal.with_filter(F.lit(False))))
+            else:
+                return first_series(ps.concat([kdf] * repeats))
+
+    def asof(self, where) -> Union[Scalar, "Series"]:
+        """
+        Return the last row(s) without any NaNs before `where`.
+
+        The last row (for each element in `where`, if list) without any
+        NaN is taken.
+
+        If there is no good value, NaN is returned.
+
+        .. note:: This API is dependent on :meth:`Index.is_monotonic_increasing`
+            which can be expensive.
+
+        Parameters
+        ----------
+        where : index or array-like of indices
+
+        Returns
+        -------
+        scalar or Series
+
+            The return can be:
+
+            * scalar : when `self` is a Series and `where` is a scalar
+            * Series: when `self` is a Series and `where` is an array-like
+
+            Return scalar or Series
+
+        Notes
+        -----
+        Indices are assumed to be sorted. Raises if this is not the case.
+
+        Examples
+        --------
+        >>> s = ps.Series([1, 2, np.nan, 4], index=[10, 20, 30, 40])
+        >>> s
+        10    1.0
+        20    2.0
+        30    NaN
+        40    4.0
+        dtype: float64
+
+        A scalar `where`.
+
+        >>> s.asof(20)
+        2.0
+
+        For a sequence `where`, a Series is returned. The first value is
+        NaN, because the first element of `where` is before the first
+        index value.
+
+        >>> s.asof([5, 20]).sort_index()
+        5     NaN
+        20    2.0
+        dtype: float64
+
+        Missing values are not considered. The following is ``2.0``, not
+        NaN, even though NaN is at the index location for ``30``.
+
+        >>> s.asof(30)
+        2.0
+        """
+        should_return_series = True
+        if isinstance(self.index, ps.MultiIndex):
+            raise ValueError("asof is not supported for a MultiIndex")
+        if isinstance(where, (ps.Index, ps.Series, DataFrame)):
+            raise ValueError("where cannot be an Index, Series or a DataFrame")
+        if not self.index.is_monotonic_increasing:
+            raise ValueError("asof requires a sorted index")
+        if not is_list_like(where):
+            should_return_series = False
+            where = [where]
+        index_scol = self._internal.index_spark_columns[0]
+        index_type = self._internal.spark_type_for(index_scol)
+        cond = [
+            F.max(F.when(index_scol <= F.lit(index).cast(index_type), self.spark.column))
+            for index in where
+        ]
+        sdf = self._internal.spark_frame.select(cond)
+        if not should_return_series:
+            with sql_conf({SPARK_CONF_ARROW_ENABLED: False}):
+                # Disable Arrow to keep row ordering.
+                result = sdf.limit(1).toPandas().iloc[0, 0]
+            return result if result is not None else np.nan
+
+        # The data is expected to be small so it's fine to transpose/use default index.
+        with ps.option_context("compute.default_index_type", "distributed", "compute.max_rows", 1):
+            kdf = ps.DataFrame(sdf)  # type: DataFrame
+            kdf.columns = pd.Index(where)
+            return first_series(kdf.transpose()).rename(self.name)
+
+    def mad(self) -> float:
+        """
+        Return the mean absolute deviation of values.
+
+        Examples
+        --------
+        >>> s = ps.Series([1, 2, 3, 4])
+        >>> s
+        0    1
+        1    2
+        2    3
+        3    4
+        dtype: int64
+
+        >>> s.mad()
+        1.0
+        """
+
+        sdf = self._internal.spark_frame
+        spark_column = self.spark.column
+        avg = unpack_scalar(sdf.select(F.avg(spark_column)))
+        mad = unpack_scalar(sdf.select(F.avg(F.abs(spark_column - avg))))
+
+        return mad
+
+    def unstack(self, level=-1) -> DataFrame:
+        """
+        Unstack, a.k.a. pivot, Series with MultiIndex to produce DataFrame.
+        The level involved will automatically get sorted.
+
+        Notes
+        -----
+        Unlike pandas, pandas-on-Spark doesn't check whether an index is duplicated or not
+        because the checking of duplicated index requires scanning whole data which
+        can be quite expensive.
+
+        Parameters
+        ----------
+        level : int, str, or list of these, default last level
+            Level(s) to unstack, can pass level name.
+
+        Returns
+        -------
+        DataFrame
+            Unstacked Series.
+
+        Examples
+        --------
+        >>> s = ps.Series([1, 2, 3, 4],
+        ...               index=pd.MultiIndex.from_product([['one', 'two'],
+        ...                                                 ['a', 'b']]))
+        >>> s
+        one  a    1
+             b    2
+        two  a    3
+             b    4
+        dtype: int64
+
+        >>> s.unstack(level=-1).sort_index()
+             a  b
+        one  1  2
+        two  3  4
+
+        >>> s.unstack(level=0).sort_index()
+           one  two
+        a    1    3
+        b    2    4
+        """
+        if not isinstance(self.index, ps.MultiIndex):
+            raise ValueError("Series.unstack only support for a MultiIndex")
+        index_nlevels = self.index.nlevels
+        if level > 0 and (level > index_nlevels - 1):
+            raise IndexError(
+                "Too many levels: Index has only {} levels, not {}".format(index_nlevels, level + 1)
+            )
+        elif level < 0 and (level < -index_nlevels):
+            raise IndexError(
+                "Too many levels: Index has only {} levels, {} is not a valid level number".format(
+                    index_nlevels, level
+                )
+            )
+
+        internal = self._internal.resolved_copy
+
+        index_map = list(zip(internal.index_spark_column_names, internal.index_names))
+        pivot_col, column_label_names = index_map.pop(level)
+        index_scol_names, index_names = zip(*index_map)
+        col = internal.data_spark_column_names[0]
+
+        sdf = internal.spark_frame
+        sdf = sdf.groupby(list(index_scol_names)).pivot(pivot_col).agg(F.first(scol_for(sdf, col)))
+        internal = InternalFrame(  # TODO: dtypes?
+            spark_frame=sdf,
+            index_spark_columns=[scol_for(sdf, col) for col in index_scol_names],
+            index_names=list(index_names),
+            column_label_names=[column_label_names],
+        )
+        return DataFrame(internal)
+
+    def item(self) -> Scalar:
+        """
+        Return the first element of the underlying data as a Python scalar.
+
+        Returns
+        -------
+        scalar
+            The first element of Series.
+
+        Raises
+        ------
+        ValueError
+            If the data is not length-1.
+
+        Examples
+        --------
+        >>> kser = ps.Series([10])
+        >>> kser.item()
+        10
+        """
+        return self.head(2)._to_internal_pandas().item()
+
+    def iteritems(self) -> Iterable:
+        """
+        Lazily iterate over (index, value) tuples.
+
+        This method returns an iterable tuple (index, value). This is
+        convenient if you want to create a lazy iterator.
+
+        .. note:: Unlike pandas', the iteritems in pandas-on-Spark returns generator rather
+            zip object
+
+        Returns
+        -------
+        iterable
+            Iterable of tuples containing the (index, value) pairs from a
+            Series.
+
+        See Also
+        --------
+        DataFrame.items : Iterate over (column name, Series) pairs.
+        DataFrame.iterrows : Iterate over DataFrame rows as (index, Series) pairs.
+
+        Examples
+        --------
+        >>> s = ps.Series(['A', 'B', 'C'])
+        >>> for index, value in s.items():
+        ...     print("Index : {}, Value : {}".format(index, value))
+        Index : 0, Value : A
+        Index : 1, Value : B
+        Index : 2, Value : C
+        """
+        internal_index_columns = self._internal.index_spark_column_names
+        internal_data_column = self._internal.data_spark_column_names[0]
+
+        def extract_kv_from_spark_row(row):
+            k = (
+                row[internal_index_columns[0]]
+                if len(internal_index_columns) == 1
+                else tuple(row[c] for c in internal_index_columns)
+            )
+            v = row[internal_data_column]
+            return k, v
+
+        for k, v in map(
+            extract_kv_from_spark_row, self._internal.resolved_copy.spark_frame.toLocalIterator()
+        ):
+            yield k, v
+
+    def items(self) -> Iterable:
+        """This is an alias of ``iteritems``."""
+        return self.iteritems()
+
+    def droplevel(self, level) -> "Series":
+        """
+        Return Series with requested index level(s) removed.
+
+        Parameters
+        ----------
+        level : int, str, or list-like
+            If a string is given, must be the name of a level
+            If list-like, elements must be names or positional indexes
+            of levels.
+
+        Returns
+        -------
+        Series
+            Series with requested index level(s) removed.
+
+        Examples
+        --------
+        >>> kser = ps.Series(
+        ...     [1, 2, 3],
+        ...     index=pd.MultiIndex.from_tuples(
+        ...         [("x", "a"), ("x", "b"), ("y", "c")], names=["level_1", "level_2"]
+        ...     ),
+        ... )
+        >>> kser
+        level_1  level_2
+        x        a          1
+                 b          2
+        y        c          3
+        dtype: int64
+
+        Removing specific index level by level
+
+        >>> kser.droplevel(0)
+        level_2
+        a    1
+        b    2
+        c    3
+        dtype: int64
+
+        Removing specific index level by name
+
+        >>> kser.droplevel("level_2")
+        level_1
+        x    1
+        x    2
+        y    3
+        dtype: int64
+        """
+        return first_series(self.to_frame().droplevel(level=level, axis=0)).rename(self.name)
+
+    def tail(self, n=5) -> "Series":
+        """
+        Return the last `n` rows.
+
+        This function returns last `n` rows from the object based on
+        position. It is useful for quickly verifying data, for example,
+        after sorting or appending rows.
+
+        For negative values of `n`, this function returns all rows except
+        the first `n` rows, equivalent to ``df[n:]``.
+
+        Parameters
+        ----------
+        n : int, default 5
+            Number of rows to select.
+
+        Returns
+        -------
+        type of caller
+            The last `n` rows of the caller object.
+
+        See Also
+        --------
+        DataFrame.head : The first `n` rows of the caller object.
+
+        Examples
+        --------
+        >>> kser = ps.Series([1, 2, 3, 4, 5])
+        >>> kser
+        0    1
+        1    2
+        2    3
+        3    4
+        4    5
+        dtype: int64
+
+        >>> kser.tail(3)  # doctest: +SKIP
+        2    3
+        3    4
+        4    5
+        dtype: int64
+        """
+        return first_series(self.to_frame().tail(n=n)).rename(self.name)
+
+    def explode(self) -> "Series":
+        """
+        Transform each element of a list-like to a row.
+
+        Returns
+        -------
+        Series
+            Exploded lists to rows; index will be duplicated for these rows.
+
+        See Also
+        --------
+        Series.str.split : Split string values on specified separator.
+        Series.unstack : Unstack, a.k.a. pivot, Series with MultiIndex
+            to produce DataFrame.
+        DataFrame.melt : Unpivot a DataFrame from wide format to long format.
+        DataFrame.explode : Explode a DataFrame from list-like
+            columns to long format.
+
+        Examples
+        --------
+        >>> kser = ps.Series([[1, 2, 3], [], [3, 4]])
+        >>> kser
+        0    [1, 2, 3]
+        1           []
+        2       [3, 4]
+        dtype: object
+
+        >>> kser.explode()  # doctest: +SKIP
+        0    1.0
+        0    2.0
+        0    3.0
+        1    NaN
+        2    3.0
+        2    4.0
+        dtype: float64
+        """
+        if not isinstance(self.spark.data_type, ArrayType):
+            return self.copy()
+
+        scol = F.explode_outer(self.spark.column).alias(name_like_string(self._column_label))
+
+        internal = self._internal.with_new_columns([scol], keep_order=False)
+        return first_series(DataFrame(internal))
+
+    def argsort(self) -> "Series":
+        """
+        Return the integer indices that would sort the Series values.
+        Unlike pandas, the index order is not preserved in the result.
+
+        Returns
+        -------
+        Series
+            Positions of values within the sort order with -1 indicating
+            nan values.
+
+        Examples
+        --------
+        >>> kser = ps.Series([3, 3, 4, 1, 6, 2, 3, 7, 8, 7, 10])
+        >>> kser
+        0      3
+        1      3
+        2      4
+        3      1
+        4      6
+        5      2
+        6      3
+        7      7
+        8      8
+        9      7
+        10    10
+        dtype: int64
+
+        >>> kser.argsort().sort_index()
+        0      3
+        1      5
+        2      0
+        3      1
+        4      6
+        5      2
+        6      4
+        7      7
+        8      9
+        9      8
+        10    10
+        dtype: int64
+        """
+        notnull = self.loc[self.notnull()]
+
+        sdf_for_index = notnull._internal.spark_frame.select(notnull._internal.index_spark_columns)
+
+        tmp_join_key = verify_temp_column_name(sdf_for_index, "__tmp_join_key__")
+        sdf_for_index = InternalFrame.attach_distributed_sequence_column(
+            sdf_for_index, tmp_join_key
+        )
+        # sdf_for_index:
+        # +----------------+-----------------+
+        # |__tmp_join_key__|__index_level_0__|
+        # +----------------+-----------------+
+        # |               0|                0|
+        # |               1|                1|
+        # |               2|                2|
+        # |               3|                3|
+        # |               4|                4|
+        # +----------------+-----------------+
+
+        sdf_for_data = notnull._internal.spark_frame.select(
+            notnull.spark.column.alias("values"), NATURAL_ORDER_COLUMN_NAME
+        )
+        sdf_for_data = InternalFrame.attach_distributed_sequence_column(
+            sdf_for_data, SPARK_DEFAULT_SERIES_NAME
+        )
+        # sdf_for_data:
+        # +---+------+-----------------+
+        # |  0|values|__natural_order__|
+        # +---+------+-----------------+
+        # |  0|     3|      25769803776|
+        # |  1|     3|      51539607552|
+        # |  2|     4|      77309411328|
+        # |  3|     1|     103079215104|
+        # |  4|     2|     128849018880|
+        # +---+------+-----------------+
+
+        sdf_for_data = sdf_for_data.sort(
+            scol_for(sdf_for_data, "values"), NATURAL_ORDER_COLUMN_NAME
+        ).drop("values", NATURAL_ORDER_COLUMN_NAME)
+
+        tmp_join_key = verify_temp_column_name(sdf_for_data, "__tmp_join_key__")
+        sdf_for_data = InternalFrame.attach_distributed_sequence_column(sdf_for_data, tmp_join_key)
+        # sdf_for_index:                         sdf_for_data:
+        # +----------------+-----------------+   +----------------+---+
+        # |__tmp_join_key__|__index_level_0__|   |__tmp_join_key__|  0|
+        # +----------------+-----------------+   +----------------+---+
+        # |               0|                0|   |               0|  3|
+        # |               1|                1|   |               1|  4|
+        # |               2|                2|   |               2|  0|
+        # |               3|                3|   |               3|  1|
+        # |               4|                4|   |               4|  2|
+        # +----------------+-----------------+   +----------------+---+
+
+        sdf = sdf_for_index.join(sdf_for_data, on=tmp_join_key).drop(tmp_join_key)
+
+        internal = self._internal.with_new_sdf(
+            spark_frame=sdf, data_columns=[SPARK_DEFAULT_SERIES_NAME], data_dtypes=[None]
+        )
+        kser = first_series(DataFrame(internal))
+
+        return cast(
+            Series, ps.concat([kser, self.loc[self.isnull()].spark.transform(lambda _: F.lit(-1))])
+        )
+
+    def argmax(self) -> int:
+        """
+        Return int position of the largest value in the Series.
+
+        If the maximum is achieved in multiple locations,
+        the first row position is returned.
+
+        Returns
+        -------
+        int
+            Row position of the maximum value.
+
+        Examples
+        --------
+        Consider dataset containing cereal calories
+
+        >>> s = ps.Series({'Corn Flakes': 100.0, 'Almond Delight': 110.0,
+        ...                'Cinnamon Toast Crunch': 120.0, 'Cocoa Puff': 110.0})
+        >>> s  # doctest: +SKIP
+        Corn Flakes              100.0
+        Almond Delight           110.0
+        Cinnamon Toast Crunch    120.0
+        Cocoa Puff               110.0
+        dtype: float64
+
+        >>> s.argmax()  # doctest: +SKIP
+        2
+        """
+        sdf = self._internal.spark_frame.select(self.spark.column, NATURAL_ORDER_COLUMN_NAME)
+        max_value = sdf.select(
+            F.max(scol_for(sdf, self._internal.data_spark_column_names[0])),
+            F.first(NATURAL_ORDER_COLUMN_NAME),
+        ).head()
+        if max_value[1] is None:
+            raise ValueError("attempt to get argmax of an empty sequence")
+        elif max_value[0] is None:
+            return -1
+        # We should remember the natural sequence started from 0
+        seq_col_name = verify_temp_column_name(sdf, "__distributed_sequence_column__")
+        sdf = InternalFrame.attach_distributed_sequence_column(
+            sdf.drop(NATURAL_ORDER_COLUMN_NAME), seq_col_name
+        )
+        # If the maximum is achieved in multiple locations, the first row position is returned.
+        return sdf.filter(
+            scol_for(sdf, self._internal.data_spark_column_names[0]) == max_value[0]
+        ).head()[0]
+
+    def argmin(self) -> int:
+        """
+        Return int position of the smallest value in the Series.
+
+        If the minimum is achieved in multiple locations,
+        the first row position is returned.
+
+        Returns
+        -------
+        int
+            Row position of the minimum value.
+
+        Examples
+        --------
+        Consider dataset containing cereal calories
+
+        >>> s = ps.Series({'Corn Flakes': 100.0, 'Almond Delight': 110.0,
+        ...                'Cinnamon Toast Crunch': 120.0, 'Cocoa Puff': 110.0})
+        >>> s  # doctest: +SKIP
+        Corn Flakes              100.0
+        Almond Delight           110.0
+        Cinnamon Toast Crunch    120.0
+        Cocoa Puff               110.0
+        dtype: float64
+
+        >>> s.argmin()  # doctest: +SKIP
+        0
+        """
+        sdf = self._internal.spark_frame.select(self.spark.column, NATURAL_ORDER_COLUMN_NAME)
+        min_value = sdf.select(
+            F.min(scol_for(sdf, self._internal.data_spark_column_names[0])),
+            F.first(NATURAL_ORDER_COLUMN_NAME),
+        ).head()
+        if min_value[1] is None:
+            raise ValueError("attempt to get argmin of an empty sequence")
+        elif min_value[0] is None:
+            return -1
+        # We should remember the natural sequence started from 0
+        seq_col_name = verify_temp_column_name(sdf, "__distributed_sequence_column__")
+        sdf = InternalFrame.attach_distributed_sequence_column(
+            sdf.drop(NATURAL_ORDER_COLUMN_NAME), seq_col_name
+        )
+        # If the minimum is achieved in multiple locations, the first row position is returned.
+        return sdf.filter(
+            scol_for(sdf, self._internal.data_spark_column_names[0]) == min_value[0]
+        ).head()[0]
+
+    def compare(
+        self, other: "Series", keep_shape: bool = False, keep_equal: bool = False
+    ) -> DataFrame:
+        """
+        Compare to another Series and show the differences.
+
+        Parameters
+        ----------
+        other : Series
+            Object to compare with.
+        keep_shape : bool, default False
+            If true, all rows and columns are kept.
+            Otherwise, only the ones with different values are kept.
+        keep_equal : bool, default False
+            If true, the result keeps values that are equal.
+            Otherwise, equal values are shown as NaNs.
+
+        Returns
+        -------
+        DataFrame
+
+        Notes
+        -----
+        Matching NaNs will not appear as a difference.
+
+        Examples
+        --------
+
+        >>> from pyspark.pandas.config import set_option, reset_option
+        >>> set_option("compute.ops_on_diff_frames", True)
+        >>> s1 = ps.Series(["a", "b", "c", "d", "e"])
+        >>> s2 = ps.Series(["a", "a", "c", "b", "e"])
+
+        Align the differences on columns
+
+        >>> s1.compare(s2).sort_index()
+          self other
+        1    b     a
+        3    d     b
+
+        Keep all original rows
+
+        >>> s1.compare(s2, keep_shape=True).sort_index()
+           self other
+        0  None  None
+        1     b     a
+        2  None  None
+        3     d     b
+        4  None  None
+
+        Keep all original rows and also all original values
+
+        >>> s1.compare(s2, keep_shape=True, keep_equal=True).sort_index()
+          self other
+        0    a     a
+        1    b     a
+        2    c     c
+        3    d     b
+        4    e     e
+
+        >>> reset_option("compute.ops_on_diff_frames")
+        """
+        if same_anchor(self, other):
+            self_column_label = verify_temp_column_name(other.to_frame(), "__self_column__")
+            other_column_label = verify_temp_column_name(self.to_frame(), "__other_column__")
+            combined = DataFrame(
+                self._internal.with_new_columns(
+                    [self.rename(self_column_label), other.rename(other_column_label)]
+                )
+            )  # type: DataFrame
+        else:
+            if not self.index.equals(other.index):
+                raise ValueError("Can only compare identically-labeled Series objects")
+
+            combined = combine_frames(self.to_frame(), other.to_frame())
+
+        this_column_label = "self"
+        that_column_label = "other"
+        if keep_equal and keep_shape:
+            combined.columns = pd.Index([this_column_label, that_column_label])
+            return combined
+
+        this_data_scol = combined._internal.data_spark_columns[0]
+        that_data_scol = combined._internal.data_spark_columns[1]
+        index_scols = combined._internal.index_spark_columns
+        sdf = combined._internal.spark_frame
+        if keep_shape:
+            this_scol = (
+                F.when(this_data_scol == that_data_scol, None)
+                .otherwise(this_data_scol)
+                .alias(this_column_label)
+            )
+            that_scol = (
+                F.when(this_data_scol == that_data_scol, None)
+                .otherwise(that_data_scol)
+                .alias(that_column_label)
+            )
+        else:
+            sdf = sdf.filter(~this_data_scol.eqNullSafe(that_data_scol))
+            this_scol = this_data_scol.alias(this_column_label)
+            that_scol = that_data_scol.alias(that_column_label)
+
+        sdf = sdf.select(index_scols + [this_scol, that_scol, NATURAL_ORDER_COLUMN_NAME])
+        internal = InternalFrame(
+            spark_frame=sdf,
+            index_spark_columns=[
+                scol_for(sdf, col) for col in self._internal.index_spark_column_names
+            ],
+            index_names=self._internal.index_names,
+            index_dtypes=self._internal.index_dtypes,
+            column_labels=[(this_column_label,), (that_column_label,)],
+            data_spark_columns=[scol_for(sdf, this_column_label), scol_for(sdf, that_column_label)],
+            column_label_names=[None],
+        )
+        return DataFrame(internal)
+
+    def align(
+        self,
+        other: Union[DataFrame, "Series"],
+        join: str = "outer",
+        axis: Optional[Union[int, str]] = None,
+        copy: bool = True,
+    ) -> Tuple["Series", Union[DataFrame, "Series"]]:
+        """
+        Align two objects on their axes with the specified join method.
+
+        Join method is specified for each axis Index.
+
+        Parameters
+        ----------
+        other : DataFrame or Series
+        join : {{'outer', 'inner', 'left', 'right'}}, default 'outer'
+        axis : allowed axis of the other object, default None
+            Align on index (0), columns (1), or both (None).
+        copy : bool, default True
+            Always returns new objects. If copy=False and no reindexing is
+            required then original objects are returned.
+
+        Returns
+        -------
+        (left, right) : (Series, type of other)
+            Aligned objects.
+
+        Examples
+        --------
+        >>> ps.set_option("compute.ops_on_diff_frames", True)
+        >>> s1 = ps.Series([7, 8, 9], index=[10, 11, 12])
+        >>> s2 = ps.Series(["g", "h", "i"], index=[10, 20, 30])
+
+        >>> aligned_l, aligned_r = s1.align(s2)
+        >>> aligned_l.sort_index()
+        10    7.0
+        11    8.0
+        12    9.0
+        20    NaN
+        30    NaN
+        dtype: float64
+        >>> aligned_r.sort_index()
+        10       g
+        11    None
+        12    None
+        20       h
+        30       i
+        dtype: object
+
+        Align with the join type "inner":
+
+        >>> aligned_l, aligned_r = s1.align(s2, join="inner")
+        >>> aligned_l.sort_index()
+        10    7
+        dtype: int64
+        >>> aligned_r.sort_index()
+        10    g
+        dtype: object
+
+        Align with a DataFrame:
+
+        >>> df = ps.DataFrame({"a": [1, 2, 3], "b": ["a", "b", "c"]}, index=[10, 20, 30])
+        >>> aligned_l, aligned_r = s1.align(df)
+        >>> aligned_l.sort_index()
+        10    7.0
+        11    8.0
+        12    9.0
+        20    NaN
+        30    NaN
+        dtype: float64
+        >>> aligned_r.sort_index()
+              a     b
+        10  1.0     a
+        11  NaN  None
+        12  NaN  None
+        20  2.0     b
+        30  3.0     c
+
+        >>> ps.reset_option("compute.ops_on_diff_frames")
+        """
+        axis = validate_axis(axis)
+        if axis == 1:
+            raise ValueError("Series does not support columns axis.")
+
+        self_df = self.to_frame()
+        left, right = self_df.align(other, join=join, axis=axis, copy=False)
+
+        if left is self_df:
+            left_ser = self
+        else:
+            left_ser = first_series(left).rename(self.name)
+
+        return (left_ser.copy(), right.copy()) if copy else (left_ser, right)
+
+    def between_time(
+        self,
+        start_time: Union[datetime.time, str],
+        end_time: Union[datetime.time, str],
+        include_start: bool = True,
+        include_end: bool = True,
+        axis: Union[int, str] = 0,
+    ) -> "Series":
+        """
+        Select values between particular times of the day (example: 9:00-9:30 AM).
+
+        By setting ``start_time`` to be later than ``end_time``,
+        you can get the times that are *not* between the two times.
+
+        Parameters
+        ----------
+        start_time : datetime.time or str
+            Initial time as a time filter limit.
+        end_time : datetime.time or str
+            End time as a time filter limit.
+        include_start : bool, default True
+            Whether the start time needs to be included in the result.
+        include_end : bool, default True
+            Whether the end time needs to be included in the result.
+        axis : {0 or 'index', 1 or 'columns'}, default 0
+            Determine range time on index or columns value.
+
+        Returns
+        -------
+        Series
+            Data from the original object filtered to the specified dates range.
+
+        Raises
+        ------
+        TypeError
+            If the index is not  a :class:`DatetimeIndex`
+
+        See Also
+        --------
+        at_time : Select values at a particular time of the day.
+        last : Select final periods of time series based on a date offset.
+        DatetimeIndex.indexer_between_time : Get just the index locations for
+            values between particular times of the day.
+
+        Examples
+        --------
+        >>> idx = pd.date_range('2018-04-09', periods=4, freq='1D20min')
+        >>> kser = ps.Series([1, 2, 3, 4], index=idx)
+        >>> kser
+        2018-04-09 00:00:00    1
+        2018-04-10 00:20:00    2
+        2018-04-11 00:40:00    3
+        2018-04-12 01:00:00    4
+        dtype: int64
+
+        >>> kser.between_time('0:15', '0:45')
+        2018-04-10 00:20:00    2
+        2018-04-11 00:40:00    3
+        dtype: int64
+        """
+        return first_series(
+            self.to_frame().between_time(start_time, end_time, include_start, include_end, axis)
+        ).rename(self.name)
+
+    def at_time(
+        self, time: Union[datetime.time, str], asof: bool = False, axis: Union[int, str] = 0
+    ) -> "Series":
+        """
+        Select values at particular time of day (example: 9:30AM).
+
+        Parameters
+        ----------
+        time : datetime.time or str
+        axis : {0 or 'index', 1 or 'columns'}, default 0
+
+        Returns
+        -------
+        Series
+
+        Raises
+        ------
+        TypeError
+            If the index is not  a :class:`DatetimeIndex`
+
+        See Also
+        --------
+        between_time : Select values between particular times of the day.
+        DatetimeIndex.indexer_at_time : Get just the index locations for
+            values at particular time of the day.
+
+        Examples
+        --------
+        >>> idx = pd.date_range('2018-04-09', periods=4, freq='12H')
+        >>> kser = ps.Series([1, 2, 3, 4], index=idx)
+        >>> kser
+        2018-04-09 00:00:00    1
+        2018-04-09 12:00:00    2
+        2018-04-10 00:00:00    3
+        2018-04-10 12:00:00    4
+        dtype: int64
+
+        >>> kser.at_time('12:00')
+        2018-04-09 12:00:00    2
+        2018-04-10 12:00:00    4
+        dtype: int64
+        """
+        return first_series(self.to_frame().at_time(time, asof, axis)).rename(self.name)
+
+    def _cum(self, func, skipna, part_cols=(), ascending=True):
+        # This is used to cummin, cummax, cumsum, etc.
+
+        if ascending:
+            window = (
+                Window.orderBy(F.asc(NATURAL_ORDER_COLUMN_NAME))
+                .partitionBy(*part_cols)
+                .rowsBetween(Window.unboundedPreceding, Window.currentRow)
+            )
+        else:
+            window = (
+                Window.orderBy(F.desc(NATURAL_ORDER_COLUMN_NAME))
+                .partitionBy(*part_cols)
+                .rowsBetween(Window.unboundedPreceding, Window.currentRow)
+            )
+
+        if skipna:
+            # There is a behavior difference between pandas and PySpark. In case of cummax,
+            #
+            # Input:
+            #      A    B
+            # 0  2.0  1.0
+            # 1  5.0  NaN
+            # 2  1.0  0.0
+            # 3  2.0  4.0
+            # 4  4.0  9.0
+            #
+            # pandas:
+            #      A    B
+            # 0  2.0  1.0
+            # 1  5.0  NaN
+            # 2  5.0  1.0
+            # 3  5.0  4.0
+            # 4  5.0  9.0
+            #
+            # PySpark:
+            #      A    B
+            # 0  2.0  1.0
+            # 1  5.0  1.0
+            # 2  5.0  1.0
+            # 3  5.0  4.0
+            # 4  5.0  9.0
+
+            scol = F.when(
+                # Manually sets nulls given the column defined above.
+                self.spark.column.isNull(),
+                F.lit(None),
+            ).otherwise(func(self.spark.column).over(window))
+        else:
+            # Here, we use two Windows.
+            # One for real data.
+            # The other one for setting nulls after the first null it meets.
+            #
+            # There is a behavior difference between pandas and PySpark. In case of cummax,
+            #
+            # Input:
+            #      A    B
+            # 0  2.0  1.0
+            # 1  5.0  NaN
+            # 2  1.0  0.0
+            # 3  2.0  4.0
+            # 4  4.0  9.0
+            #
+            # pandas:
+            #      A    B
+            # 0  2.0  1.0
+            # 1  5.0  NaN
+            # 2  5.0  NaN
+            # 3  5.0  NaN
+            # 4  5.0  NaN
+            #
+            # PySpark:
+            #      A    B
+            # 0  2.0  1.0
+            # 1  5.0  1.0
+            # 2  5.0  1.0
+            # 3  5.0  4.0
+            # 4  5.0  9.0
+            scol = F.when(
+                # By going through with max, it sets True after the first time it meets null.
+                F.max(self.spark.column.isNull()).over(window),
+                # Manually sets nulls given the column defined above.
+                F.lit(None),
+            ).otherwise(func(self.spark.column).over(window))
+
+        return self._with_new_scol(scol)
+
+    def _cumsum(self, skipna, part_cols=()):
+        kser = self
+        if isinstance(kser.spark.data_type, BooleanType):
+            kser = kser.spark.transform(lambda scol: scol.cast(LongType()))
+        elif not isinstance(kser.spark.data_type, NumericType):
+            raise TypeError(
+                "Could not convert {} ({}) to numeric".format(
+                    spark_type_to_pandas_dtype(kser.spark.data_type),
+                    kser.spark.data_type.simpleString(),
+                )
+            )
+        return kser._cum(F.sum, skipna, part_cols)
+
+    def _cumprod(self, skipna, part_cols=()):
+        if isinstance(self.spark.data_type, BooleanType):
+            scol = self._cum(
+                lambda scol: F.min(F.coalesce(scol, F.lit(True))), skipna, part_cols
+            ).spark.column.cast(LongType())
+        elif isinstance(self.spark.data_type, NumericType):
+            num_zeros = self._cum(
+                lambda scol: F.sum(F.when(scol == 0, 1).otherwise(0)), skipna, part_cols
+            ).spark.column
+            num_negatives = self._cum(
+                lambda scol: F.sum(F.when(scol < 0, 1).otherwise(0)), skipna, part_cols
+            ).spark.column
+            sign = F.when(num_negatives % 2 == 0, 1).otherwise(-1)
+
+            abs_prod = F.exp(
+                self._cum(lambda scol: F.sum(F.log(F.abs(scol))), skipna, part_cols).spark.column
+            )
+
+            scol = F.when(num_zeros > 0, 0).otherwise(sign * abs_prod)
+
+            if isinstance(self.spark.data_type, IntegralType):
+                scol = F.round(scol).cast(LongType())
+        else:
+            raise TypeError(
+                "Could not convert {} ({}) to numeric".format(
+                    spark_type_to_pandas_dtype(self.spark.data_type),
+                    self.spark.data_type.simpleString(),
+                )
+            )
+
+        return self._with_new_scol(scol)
+
+    # ----------------------------------------------------------------------
+    # Accessor Methods
+    # ----------------------------------------------------------------------
+    dt = CachedAccessor("dt", DatetimeMethods)
+    str = CachedAccessor("str", StringMethods)
+    cat = CachedAccessor("cat", CategoricalAccessor)
+    plot = CachedAccessor("plot", PandasOnSparkPlotAccessor)
+
+    # ----------------------------------------------------------------------
+
+    def _apply_series_op(self, op, should_resolve: bool = False):
+        kser = op(self)
+        if should_resolve:
+            internal = kser._internal.resolved_copy
+            return first_series(DataFrame(internal))
+        else:
+            return kser
+
+    def _reduce_for_stat_function(self, sfun, name, axis=None, numeric_only=None, **kwargs):
+        """
+        Applies sfun to the column and returns a scalar
+
+        Parameters
+        ----------
+        sfun : the stats function to be used for aggregation
+        name : original pandas API name.
+        axis : used only for sanity check because series only support index axis.
+        numeric_only : not used by this implementation, but passed down by stats functions
+        """
+        from inspect import signature
+
+        axis = validate_axis(axis)
+        if axis == 1:
+            raise ValueError("Series does not support columns axis.")
+        num_args = len(signature(sfun).parameters)
+        spark_column = self.spark.column
+        spark_type = self.spark.data_type
+
+        if num_args == 1:
+            # Only pass in the column if sfun accepts only one arg
+            scol = sfun(spark_column)
+        else:  # must be 2
+            assert num_args == 2
+            # Pass in both the column and its data type if sfun accepts two args
+            scol = sfun(spark_column, spark_type)
+
+        min_count = kwargs.get("min_count", 0)
+        if min_count > 0:
+            scol = F.when(Frame._count_expr(spark_column, spark_type) >= min_count, scol)
+
+        result = unpack_scalar(self._internal.spark_frame.select(scol))
+        return result if result is not None else np.nan
+
+    def __getitem__(self, key):
+        try:
+            if (isinstance(key, slice) and any(type(n) == int for n in [key.start, key.stop])) or (
+                type(key) == int
+                and not isinstance(self.index.spark.data_type, (IntegerType, LongType))
+            ):
+                # Seems like pandas Series always uses int as positional search when slicing
+                # with ints, searches based on index values when the value is int.
+                return self.iloc[key]
+            return self.loc[key]
+        except SparkPandasIndexingError:
+            raise KeyError(
+                "Key length ({}) exceeds index depth ({})".format(
+                    len(key), self._internal.index_level
+                )
+            )
+
+    def __getattr__(self, item: str_type) -> Any:
+        if item.startswith("__"):
+            raise AttributeError(item)
+        if hasattr(MissingPandasLikeSeries, item):
+            property_or_func = getattr(MissingPandasLikeSeries, item)
+            if isinstance(property_or_func, property):
+                return property_or_func.fget(self)  # type: ignore
+            else:
+                return partial(property_or_func, self)
+        raise AttributeError("'Series' object has no attribute '{}'".format(item))
+
+    def _to_internal_pandas(self):
+        """
+        Return a pandas Series directly from _internal to avoid overhead of copy.
+
+        This method is for internal use only.
+        """
+        return self._kdf._internal.to_pandas_frame[self.name]
+
+    def __repr__(self):
+        max_display_count = get_option("display.max_rows")
+        if max_display_count is None:
+            return self._to_internal_pandas().to_string(name=self.name, dtype=self.dtype)
+
+        pser = self._kdf._get_or_create_repr_pandas_cache(max_display_count)[self.name]
+        pser_length = len(pser)
+        pser = pser.iloc[:max_display_count]
+        if pser_length > max_display_count:
+            repr_string = pser.to_string(length=True)
+            rest, prev_footer = repr_string.rsplit("\n", 1)
+            match = REPR_PATTERN.search(prev_footer)
+            if match is not None:
+                length = match.group("length")
+                dtype_name = str(self.dtype.name)
+                if self.name is None:
+                    footer = "\ndtype: {dtype}\nShowing only the first {length}".format(
+                        length=length, dtype=pprint_thing(dtype_name)
+                    )
+                else:
+                    footer = (
+                        "\nName: {name}, dtype: {dtype}"
+                        "\nShowing only the first {length}".format(
+                            length=length, name=self.name, dtype=pprint_thing(dtype_name)
+                        )
+                    )
+                return rest + footer
+        return pser.to_string(name=self.name, dtype=self.dtype)
+
+    def __dir__(self):
+        if not isinstance(self.spark.data_type, StructType):
+            fields = []
+        else:
+            fields = [f for f in self.spark.data_type.fieldNames() if " " not in f]
+        return super().__dir__() + fields
+
+    def __iter__(self):
+        return MissingPandasLikeSeries.__iter__(self)
+
+    if sys.version_info >= (3, 7):
+        # In order to support the type hints such as Series[...]. See DataFrame.__class_getitem__.
+        def __class_getitem__(cls, params):
+            return _create_type_for_series_type(params)
+
+    elif (3, 5) <= sys.version_info < (3, 7):
+        # The implementation is in its metaclass so this flag is needed to distinguish
+        # pandas-on-Spark Series.
+        is_series = None
+
+
+def unpack_scalar(sdf):
+    """
+    Takes a dataframe that is supposed to contain a single row with a single scalar value,
+    and returns this value.
+    """
+    l = sdf.limit(2).toPandas()
+    assert len(l) == 1, (sdf, l)
+    row = l.iloc[0]
+    l2 = list(row)
+    assert len(l2) == 1, (row, l2)
+    return l2[0]
+
+
+def first_series(df) -> Union["Series", pd.Series]:
+    """
+    Takes a DataFrame and returns the first column of the DataFrame as a Series
+    """
+    assert isinstance(df, (DataFrame, pd.DataFrame)), type(df)
+    if isinstance(df, DataFrame):
+        return df._kser_for(df._internal.column_labels[0])
+    else:
+        return df[df.columns[0]]
+
+
+def _test():
+    import os
+    import doctest
+    import sys
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.series
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.series.__dict__.copy()
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]").appName("pyspark.pandas.series tests").getOrCreate()
+    )
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.series,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/spark/__init__.py b/python/pyspark/pandas/spark/__init__.py
new file mode 100644
index 0000000000000..cce3acad34a49
--- /dev/null
+++ b/python/pyspark/pandas/spark/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
diff --git a/python/pyspark/pandas/spark/accessors.py b/python/pyspark/pandas/spark/accessors.py
new file mode 100644
index 0000000000000..575d5f5f8bf09
--- /dev/null
+++ b/python/pyspark/pandas/spark/accessors.py
@@ -0,0 +1,1274 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Spark related features. Usually, the features here are missing in pandas
+but Spark has it.
+"""
+from abc import ABCMeta, abstractmethod
+from typing import TYPE_CHECKING, Optional, Union, List, cast
+
+from pyspark import StorageLevel
+from pyspark.sql import Column, DataFrame as SparkDataFrame
+from pyspark.sql.types import DataType, StructType
+
+if TYPE_CHECKING:
+    import pyspark.pandas as ps  # noqa: F401 (SPARK-34943)
+    from pyspark.pandas.base import IndexOpsMixin  # noqa: F401 (SPARK-34943)
+    from pyspark.pandas.frame import CachedDataFrame  # noqa: F401 (SPARK-34943)
+
+
+class SparkIndexOpsMethods(object, metaclass=ABCMeta):
+    """Spark related features. Usually, the features here are missing in pandas
+    but Spark has it."""
+
+    def __init__(self, data: Union["IndexOpsMixin"]):
+        self._data = data
+
+    @property
+    def data_type(self) -> DataType:
+        """ Returns the data type as defined by Spark, as a Spark DataType object."""
+        return self._data._internal.spark_type_for(self._data._column_label)
+
+    @property
+    def nullable(self) -> bool:
+        """ Returns the nullability as defined by Spark. """
+        return self._data._internal.spark_column_nullable_for(self._data._column_label)
+
+    @property
+    def column(self) -> Column:
+        """
+        Spark Column object representing the Series/Index.
+
+        .. note:: This Spark Column object is strictly stick to its base DataFrame the Series/Index
+            was derived from.
+        """
+        return self._data._internal.spark_column_for(self._data._column_label)
+
+    def transform(self, func) -> Union["ps.Series", "ps.Index"]:
+        """
+        Applies a function that takes and returns a Spark column. It allows to natively
+        apply a Spark function and column APIs with the Spark column internally used
+        in Series or Index. The output length of the Spark column should be same as input's.
+
+        .. note:: It requires to have the same input and output length; therefore,
+            the aggregate Spark functions such as count does not work.
+
+        Parameters
+        ----------
+        func : function
+            Function to use for transforming the data by using Spark columns.
+
+        Returns
+        -------
+        Series or Index
+
+        Raises
+        ------
+        ValueError : If the output from the function is not a Spark column.
+
+        Examples
+        --------
+        >>> from pyspark.sql.functions import log
+        >>> df = ps.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}, columns=["a", "b"])
+        >>> df
+           a  b
+        0  1  4
+        1  2  5
+        2  3  6
+
+        >>> df.a.spark.transform(lambda c: log(c))
+        0    0.000000
+        1    0.693147
+        2    1.098612
+        Name: a, dtype: float64
+
+        >>> df.index.spark.transform(lambda c: c + 10)
+        Int64Index([10, 11, 12], dtype='int64')
+
+        >>> df.a.spark.transform(lambda c: c + df.b.spark.column)
+        0    5
+        1    7
+        2    9
+        Name: a, dtype: int64
+        """
+        from pyspark.pandas import MultiIndex
+
+        if isinstance(self._data, MultiIndex):
+            raise NotImplementedError("MultiIndex does not support spark.transform yet.")
+        output = func(self._data.spark.column)
+        if not isinstance(output, Column):
+            raise ValueError(
+                "The output of the function [%s] should be of a "
+                "pyspark.sql.Column; however, got [%s]." % (func, type(output))
+            )
+        new_ser = self._data._with_new_scol(scol=output)
+        # Trigger the resolution so it throws an exception if anything does wrong
+        # within the function, for example,
+        # `df1.a.spark.transform(lambda _: F.col("non-existent"))`.
+        new_ser._internal.to_internal_spark_frame
+        return new_ser
+
+    @property
+    @abstractmethod
+    def analyzed(self) -> Union["ps.Series", "ps.Index"]:
+        pass
+
+
+class SparkSeriesMethods(SparkIndexOpsMethods):
+    def transform(self, func) -> "ps.Series":
+        return cast("ps.Series", super().transform(func))
+
+    transform.__doc__ = SparkIndexOpsMethods.transform.__doc__
+
+    def apply(self, func) -> "ps.Series":
+        """
+        Applies a function that takes and returns a Spark column. It allows to natively
+        apply a Spark function and column APIs with the Spark column internally used
+        in Series or Index.
+
+        .. note:: It forces to lose the index and end up with using default index. It is
+            preferred to use :meth:`Series.spark.transform` or `:meth:`DataFrame.spark.apply`
+            with specifying the `inedx_col`.
+
+        .. note:: It does not require to have the same length of the input and output.
+            However, it requires to create a new DataFrame internally which will require
+            to set `compute.ops_on_diff_frames` to compute even with the same origin
+            DataFrame that is expensive, whereas :meth:`Series.spark.transform` does not
+            require it.
+
+        Parameters
+        ----------
+        func : function
+            Function to apply the function against the data by using Spark columns.
+
+        Returns
+        -------
+        Series
+
+        Raises
+        ------
+        ValueError : If the output from the function is not a Spark column.
+
+        Examples
+        --------
+        >>> from pyspark import pandas as ps
+        >>> from pyspark.sql.functions import count, lit
+        >>> df = ps.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}, columns=["a", "b"])
+        >>> df
+           a  b
+        0  1  4
+        1  2  5
+        2  3  6
+
+        >>> df.a.spark.apply(lambda c: count(c))
+        0    3
+        Name: a, dtype: int64
+
+        >>> df.a.spark.apply(lambda c: c + df.b.spark.column)
+        0    5
+        1    7
+        2    9
+        Name: a, dtype: int64
+        """
+        from pyspark.pandas.frame import DataFrame
+        from pyspark.pandas.series import Series, first_series
+        from pyspark.pandas.internal import HIDDEN_COLUMNS
+
+        output = func(self._data.spark.column)
+        if not isinstance(output, Column):
+            raise ValueError(
+                "The output of the function [%s] should be of a "
+                "pyspark.sql.Column; however, got [%s]." % (func, type(output))
+            )
+        assert isinstance(self._data, Series)
+
+        sdf = self._data._internal.spark_frame.drop(*HIDDEN_COLUMNS).select(output)
+        # Lose index.
+        return first_series(DataFrame(sdf)).rename(self._data.name)
+
+    @property
+    def analyzed(self) -> "ps.Series":
+        """
+        Returns a new Series with the analyzed Spark DataFrame.
+
+        After multiple operations, the underlying Spark plan could grow huge
+        and make the Spark planner take a long time to finish the planning.
+
+        This function is for the workaround to avoid it.
+
+        .. note:: After analyzed, operations between the analyzed Series and the original one
+            will **NOT** work without setting a config `compute.ops_on_diff_frames` to `True`.
+
+        Returns
+        -------
+        Series
+
+        Examples
+        --------
+        >>> ser = ps.Series([1, 2, 3])
+        >>> ser
+        0    1
+        1    2
+        2    3
+        dtype: int64
+
+        The analyzed one should return the same value.
+
+        >>> ser.spark.analyzed
+        0    1
+        1    2
+        2    3
+        dtype: int64
+
+        However, it won't work with the same anchor Series.
+
+        >>> ser + ser.spark.analyzed
+        Traceback (most recent call last):
+        ...
+        ValueError: ... enable 'compute.ops_on_diff_frames' option.
+
+        >>> with ps.option_context('compute.ops_on_diff_frames', True):
+        ...     (ser + ser.spark.analyzed).sort_index()
+        0    2
+        1    4
+        2    6
+        dtype: int64
+        """
+        from pyspark.pandas.frame import DataFrame
+        from pyspark.pandas.series import first_series
+
+        return first_series(DataFrame(self._data._internal.resolved_copy))
+
+
+class SparkIndexMethods(SparkIndexOpsMethods):
+    def transform(self, func) -> "ps.Index":
+        return cast("ps.Index", super().transform(func))
+
+    transform.__doc__ = SparkIndexOpsMethods.transform.__doc__
+
+    @property
+    def analyzed(self) -> "ps.Index":
+        """
+        Returns a new Index with the analyzed Spark DataFrame.
+
+        After multiple operations, the underlying Spark plan could grow huge
+        and make the Spark planner take a long time to finish the planning.
+
+        This function is for the workaround to avoid it.
+
+        .. note:: After analyzed, operations between the analyzed Series and the original one
+            will **NOT** work without setting a config `compute.ops_on_diff_frames` to `True`.
+
+        Returns
+        -------
+        Index
+
+        Examples
+        --------
+        >>> idx = ps.Index([1, 2, 3])
+        >>> idx
+        Int64Index([1, 2, 3], dtype='int64')
+
+        The analyzed one should return the same value.
+
+        >>> idx.spark.analyzed
+        Int64Index([1, 2, 3], dtype='int64')
+
+        However, it won't work with the same anchor Index.
+
+        >>> idx + idx.spark.analyzed
+        Traceback (most recent call last):
+        ...
+        ValueError: ... enable 'compute.ops_on_diff_frames' option.
+
+        >>> with ps.option_context('compute.ops_on_diff_frames', True):
+        ...     (idx + idx.spark.analyzed).sort_values()
+        Int64Index([2, 4, 6], dtype='int64')
+        """
+        from pyspark.pandas.frame import DataFrame
+
+        return DataFrame(self._data._internal.resolved_copy).index
+
+
+class SparkFrameMethods(object):
+    """Spark related features. Usually, the features here are missing in pandas
+    but Spark has it."""
+
+    def __init__(self, frame: "ps.DataFrame"):
+        self._kdf = frame
+
+    def schema(self, index_col: Optional[Union[str, List[str]]] = None) -> StructType:
+        """
+        Returns the underlying Spark schema.
+
+        Returns
+        -------
+        pyspark.sql.types.StructType
+            The underlying Spark schema.
+
+        Parameters
+        ----------
+        index_col: str or list of str, optional, default: None
+            Column names to be used in Spark to represent pandas-on-Spark's index. The index name
+            in pandas-on-Spark is ignored. By default, the index is always lost.
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'a': list('abc'),
+        ...                    'b': list(range(1, 4)),
+        ...                    'c': np.arange(3, 6).astype('i1'),
+        ...                    'd': np.arange(4.0, 7.0, dtype='float64'),
+        ...                    'e': [True, False, True],
+        ...                    'f': pd.date_range('20130101', periods=3)},
+        ...                   columns=['a', 'b', 'c', 'd', 'e', 'f'])
+        >>> df.spark.schema().simpleString()
+        'struct<a:string,b:bigint,c:tinyint,d:double,e:boolean,f:timestamp>'
+        >>> df.spark.schema(index_col='index').simpleString()
+        'struct<index:bigint,a:string,b:bigint,c:tinyint,d:double,e:boolean,f:timestamp>'
+        """
+        return self.frame(index_col).schema
+
+    def print_schema(self, index_col: Optional[Union[str, List[str]]] = None) -> None:
+        """
+        Prints out the underlying Spark schema in the tree format.
+
+        Parameters
+        ----------
+        index_col: str or list of str, optional, default: None
+            Column names to be used in Spark to represent pandas-on-Spark's index. The index name
+            in pandas-on-Spark is ignored. By default, the index is always lost.
+
+        Returns
+        -------
+        None
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'a': list('abc'),
+        ...                    'b': list(range(1, 4)),
+        ...                    'c': np.arange(3, 6).astype('i1'),
+        ...                    'd': np.arange(4.0, 7.0, dtype='float64'),
+        ...                    'e': [True, False, True],
+        ...                    'f': pd.date_range('20130101', periods=3)},
+        ...                   columns=['a', 'b', 'c', 'd', 'e', 'f'])
+        >>> df.spark.print_schema()  # doctest: +NORMALIZE_WHITESPACE
+        root
+         |-- a: string (nullable = false)
+         |-- b: long (nullable = false)
+         |-- c: byte (nullable = false)
+         |-- d: double (nullable = false)
+         |-- e: boolean (nullable = false)
+         |-- f: timestamp (nullable = false)
+        >>> df.spark.print_schema(index_col='index')  # doctest: +NORMALIZE_WHITESPACE
+        root
+         |-- index: long (nullable = false)
+         |-- a: string (nullable = false)
+         |-- b: long (nullable = false)
+         |-- c: byte (nullable = false)
+         |-- d: double (nullable = false)
+         |-- e: boolean (nullable = false)
+         |-- f: timestamp (nullable = false)
+        """
+        self.frame(index_col).printSchema()
+
+    def frame(self, index_col: Optional[Union[str, List[str]]] = None) -> SparkDataFrame:
+        """
+        Return the current DataFrame as a Spark DataFrame.  :meth:`DataFrame.spark.frame` is an
+        alias of  :meth:`DataFrame.to_spark`.
+
+        Parameters
+        ----------
+        index_col: str or list of str, optional, default: None
+            Column names to be used in Spark to represent pandas-on-Spark's index. The index name
+            in pandas-on-Spark is ignored. By default, the index is always lost.
+
+        See Also
+        --------
+        DataFrame.to_spark
+        DataFrame.to_koalas
+        DataFrame.spark.frame
+
+        Examples
+        --------
+        By default, this method loses the index as below.
+
+        >>> df = ps.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6], 'c': [7, 8, 9]})
+        >>> df.to_spark().show()  # doctest: +NORMALIZE_WHITESPACE
+        +---+---+---+
+        |  a|  b|  c|
+        +---+---+---+
+        |  1|  4|  7|
+        |  2|  5|  8|
+        |  3|  6|  9|
+        +---+---+---+
+
+        >>> df = ps.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6], 'c': [7, 8, 9]})
+        >>> df.spark.frame().show()  # doctest: +NORMALIZE_WHITESPACE
+        +---+---+---+
+        |  a|  b|  c|
+        +---+---+---+
+        |  1|  4|  7|
+        |  2|  5|  8|
+        |  3|  6|  9|
+        +---+---+---+
+
+        If `index_col` is set, it keeps the index column as specified.
+
+        >>> df.to_spark(index_col="index").show()  # doctest: +NORMALIZE_WHITESPACE
+        +-----+---+---+---+
+        |index|  a|  b|  c|
+        +-----+---+---+---+
+        |    0|  1|  4|  7|
+        |    1|  2|  5|  8|
+        |    2|  3|  6|  9|
+        +-----+---+---+---+
+
+        Keeping index column is useful when you want to call some Spark APIs and
+        convert it back to pandas-on-Spark DataFrame without creating a default index, which
+        can affect performance.
+
+        >>> spark_df = df.to_spark(index_col="index")
+        >>> spark_df = spark_df.filter("a == 2")
+        >>> spark_df.to_koalas(index_col="index")  # doctest: +NORMALIZE_WHITESPACE
+               a  b  c
+        index
+        1      2  5  8
+
+        In case of multi-index, specify a list to `index_col`.
+
+        >>> new_df = df.set_index("a", append=True)
+        >>> new_spark_df = new_df.to_spark(index_col=["index_1", "index_2"])
+        >>> new_spark_df.show()  # doctest: +NORMALIZE_WHITESPACE
+        +-------+-------+---+---+
+        |index_1|index_2|  b|  c|
+        +-------+-------+---+---+
+        |      0|      1|  4|  7|
+        |      1|      2|  5|  8|
+        |      2|      3|  6|  9|
+        +-------+-------+---+---+
+
+        Likewise, can be converted to back to pandas-on-Spark DataFrame.
+
+        >>> new_spark_df.to_koalas(
+        ...     index_col=["index_1", "index_2"])  # doctest: +NORMALIZE_WHITESPACE
+                         b  c
+        index_1 index_2
+        0       1        4  7
+        1       2        5  8
+        2       3        6  9
+        """
+        from pyspark.pandas.utils import name_like_string
+
+        kdf = self._kdf
+
+        data_column_names = []
+        data_columns = []
+        for i, (label, spark_column, column_name) in enumerate(
+            zip(
+                kdf._internal.column_labels,
+                kdf._internal.data_spark_columns,
+                kdf._internal.data_spark_column_names,
+            )
+        ):
+            name = str(i) if label is None else name_like_string(label)
+            data_column_names.append(name)
+            if column_name != name:
+                spark_column = spark_column.alias(name)
+            data_columns.append(spark_column)
+
+        if index_col is None:
+            return kdf._internal.spark_frame.select(data_columns)
+        else:
+            if isinstance(index_col, str):
+                index_col = [index_col]
+
+            old_index_scols = kdf._internal.index_spark_columns
+
+            if len(index_col) != len(old_index_scols):
+                raise ValueError(
+                    "length of index columns is %s; however, the length of the given "
+                    "'index_col' is %s." % (len(old_index_scols), len(index_col))
+                )
+
+            if any(col in data_column_names for col in index_col):
+                raise ValueError("'index_col' cannot be overlapped with other columns.")
+
+            new_index_scols = [
+                index_scol.alias(col) for index_scol, col in zip(old_index_scols, index_col)
+            ]
+            return kdf._internal.spark_frame.select(new_index_scols + data_columns)
+
+    def cache(self) -> "CachedDataFrame":
+        """
+        Yields and caches the current DataFrame.
+
+        The pandas-on-Spark DataFrame is yielded as a protected resource and its corresponding
+        data is cached which gets uncached after execution goes of the context.
+
+        If you want to specify the StorageLevel manually, use :meth:`DataFrame.spark.persist`
+
+        See Also
+        --------
+        DataFrame.spark.persist
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([(.2, .3), (.0, .6), (.6, .0), (.2, .1)],
+        ...                   columns=['dogs', 'cats'])
+        >>> df
+           dogs  cats
+        0   0.2   0.3
+        1   0.0   0.6
+        2   0.6   0.0
+        3   0.2   0.1
+
+        >>> with df.spark.cache() as cached_df:
+        ...     print(cached_df.count())
+        ...
+        dogs    4
+        cats    4
+        dtype: int64
+
+        >>> df = df.spark.cache()
+        >>> df.to_pandas().mean(axis=1)
+        0    0.25
+        1    0.30
+        2    0.30
+        3    0.15
+        dtype: float64
+
+        To uncache the dataframe, use `unpersist` function
+
+        >>> df.spark.unpersist()
+        """
+        from pyspark.pandas.frame import CachedDataFrame
+
+        self._kdf._update_internal_frame(
+            self._kdf._internal.resolved_copy, requires_same_anchor=False
+        )
+        return CachedDataFrame(self._kdf._internal)
+
+    def persist(
+        self, storage_level: StorageLevel = StorageLevel.MEMORY_AND_DISK
+    ) -> "CachedDataFrame":
+        """
+        Yields and caches the current DataFrame with a specific StorageLevel.
+        If a StogeLevel is not given, the `MEMORY_AND_DISK` level is used by default like PySpark.
+
+        The pandas-on-Spark DataFrame is yielded as a protected resource and its corresponding
+        data is cached which gets uncached after execution goes of the context.
+
+        See Also
+        --------
+        DataFrame.spark.cache
+
+        Examples
+        --------
+        >>> import pyspark
+        >>> df = ps.DataFrame([(.2, .3), (.0, .6), (.6, .0), (.2, .1)],
+        ...                   columns=['dogs', 'cats'])
+        >>> df
+           dogs  cats
+        0   0.2   0.3
+        1   0.0   0.6
+        2   0.6   0.0
+        3   0.2   0.1
+
+        Set the StorageLevel to `MEMORY_ONLY`.
+
+        >>> with df.spark.persist(pyspark.StorageLevel.MEMORY_ONLY) as cached_df:
+        ...     print(cached_df.spark.storage_level)
+        ...     print(cached_df.count())
+        ...
+        Memory Serialized 1x Replicated
+        dogs    4
+        cats    4
+        dtype: int64
+
+        Set the StorageLevel to `DISK_ONLY`.
+
+        >>> with df.spark.persist(pyspark.StorageLevel.DISK_ONLY) as cached_df:
+        ...     print(cached_df.spark.storage_level)
+        ...     print(cached_df.count())
+        ...
+        Disk Serialized 1x Replicated
+        dogs    4
+        cats    4
+        dtype: int64
+
+        If a StorageLevel is not given, it uses `MEMORY_AND_DISK` by default.
+
+        >>> with df.spark.persist() as cached_df:
+        ...     print(cached_df.spark.storage_level)
+        ...     print(cached_df.count())
+        ...
+        Disk Memory Serialized 1x Replicated
+        dogs    4
+        cats    4
+        dtype: int64
+
+        >>> df = df.spark.persist()
+        >>> df.to_pandas().mean(axis=1)
+        0    0.25
+        1    0.30
+        2    0.30
+        3    0.15
+        dtype: float64
+
+        To uncache the dataframe, use `unpersist` function
+
+        >>> df.spark.unpersist()
+        """
+        from pyspark.pandas.frame import CachedDataFrame
+
+        self._kdf._update_internal_frame(
+            self._kdf._internal.resolved_copy, requires_same_anchor=False
+        )
+        return CachedDataFrame(self._kdf._internal, storage_level=storage_level)
+
+    def hint(self, name: str, *parameters) -> "ps.DataFrame":
+        """
+        Specifies some hint on the current DataFrame.
+
+        Parameters
+        ----------
+        name : A name of the hint.
+        parameters : Optional parameters.
+
+        Returns
+        -------
+        ret : DataFrame with the hint.
+
+        See Also
+        --------
+        broadcast : Marks a DataFrame as small enough for use in broadcast joins.
+
+        Examples
+        --------
+        >>> df1 = ps.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'],
+        ...                     'value': [1, 2, 3, 5]},
+        ...                    columns=['lkey', 'value']).set_index('lkey')
+        >>> df2 = ps.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'],
+        ...                     'value': [5, 6, 7, 8]},
+        ...                    columns=['rkey', 'value']).set_index('rkey')
+        >>> merged = df1.merge(df2.spark.hint("broadcast"), left_index=True, right_index=True)
+        >>> merged.spark.explain()  # doctest: +ELLIPSIS
+        == Physical Plan ==
+        ...
+        ...BroadcastHashJoin...
+        ...
+        """
+        from pyspark.pandas.frame import DataFrame
+
+        internal = self._kdf._internal.resolved_copy
+        return DataFrame(internal.with_new_sdf(internal.spark_frame.hint(name, *parameters)))
+
+    def to_table(
+        self,
+        name: str,
+        format: Optional[str] = None,
+        mode: str = "overwrite",
+        partition_cols: Optional[Union[str, List[str]]] = None,
+        index_col: Optional[Union[str, List[str]]] = None,
+        **options
+    ) -> None:
+        """
+        Write the DataFrame into a Spark table. :meth:`DataFrame.spark.to_table`
+        is an alias of :meth:`DataFrame.to_table`.
+
+        Parameters
+        ----------
+        name : str, required
+            Table name in Spark.
+        format : string, optional
+            Specifies the output data source format. Some common ones are:
+
+            - 'delta'
+            - 'parquet'
+            - 'orc'
+            - 'json'
+            - 'csv'
+
+        mode : str {'append', 'overwrite', 'ignore', 'error', 'errorifexists'}, default
+            'overwrite'. Specifies the behavior of the save operation when the table exists
+            already.
+
+            - 'append': Append the new data to existing data.
+            - 'overwrite': Overwrite existing data.
+            - 'ignore': Silently ignore this operation if data already exists.
+            - 'error' or 'errorifexists': Throw an exception if data already exists.
+
+        partition_cols : str or list of str, optional, default None
+            Names of partitioning columns
+        index_col: str or list of str, optional, default: None
+            Column names to be used in Spark to represent pandas-on-Spark's index. The index name
+            in pandas-on-Spark is ignored. By default, the index is always lost.
+        options
+            Additional options passed directly to Spark.
+
+        Returns
+        -------
+        None
+
+        See Also
+        --------
+        read_table
+        DataFrame.to_spark_io
+        DataFrame.spark.to_spark_io
+        DataFrame.to_parquet
+
+        Examples
+        --------
+        >>> df = ps.DataFrame(dict(
+        ...    date=list(pd.date_range('2012-1-1 12:00:00', periods=3, freq='M')),
+        ...    country=['KR', 'US', 'JP'],
+        ...    code=[1, 2 ,3]), columns=['date', 'country', 'code'])
+        >>> df
+                         date country  code
+        0 2012-01-31 12:00:00      KR     1
+        1 2012-02-29 12:00:00      US     2
+        2 2012-03-31 12:00:00      JP     3
+
+        >>> df.to_table('%s.my_table' % db, partition_cols='date')
+        """
+        if "options" in options and isinstance(options.get("options"), dict) and len(options) == 1:
+            options = options.get("options")  # type: ignore
+
+        self._kdf.spark.frame(index_col=index_col).write.saveAsTable(
+            name=name, format=format, mode=mode, partitionBy=partition_cols, **options
+        )
+
+    def to_spark_io(
+        self,
+        path: Optional[str] = None,
+        format: Optional[str] = None,
+        mode: str = "overwrite",
+        partition_cols: Optional[Union[str, List[str]]] = None,
+        index_col: Optional[Union[str, List[str]]] = None,
+        **options
+    ) -> None:
+        """Write the DataFrame out to a Spark data source. :meth:`DataFrame.spark.to_spark_io`
+        is an alias of :meth:`DataFrame.to_spark_io`.
+
+        Parameters
+        ----------
+        path : string, optional
+            Path to the data source.
+        format : string, optional
+            Specifies the output data source format. Some common ones are:
+
+            - 'delta'
+            - 'parquet'
+            - 'orc'
+            - 'json'
+            - 'csv'
+        mode : str {'append', 'overwrite', 'ignore', 'error', 'errorifexists'}, default
+            'overwrite'. Specifies the behavior of the save operation when data already.
+
+            - 'append': Append the new data to existing data.
+            - 'overwrite': Overwrite existing data.
+            - 'ignore': Silently ignore this operation if data already exists.
+            - 'error' or 'errorifexists': Throw an exception if data already exists.
+        partition_cols : str or list of str, optional
+            Names of partitioning columns
+        index_col: str or list of str, optional, default: None
+            Column names to be used in Spark to represent pandas-on-Spark's index. The index name
+            in pandas-on-Spark is ignored. By default, the index is always lost.
+        options : dict
+            All other options passed directly into Spark's data source.
+
+        Returns
+        -------
+        None
+
+        See Also
+        --------
+        read_spark_io
+        DataFrame.to_delta
+        DataFrame.to_parquet
+        DataFrame.to_table
+        DataFrame.to_spark_io
+        DataFrame.spark.to_spark_io
+
+        Examples
+        --------
+        >>> df = ps.DataFrame(dict(
+        ...    date=list(pd.date_range('2012-1-1 12:00:00', periods=3, freq='M')),
+        ...    country=['KR', 'US', 'JP'],
+        ...    code=[1, 2 ,3]), columns=['date', 'country', 'code'])
+        >>> df
+                         date country  code
+        0 2012-01-31 12:00:00      KR     1
+        1 2012-02-29 12:00:00      US     2
+        2 2012-03-31 12:00:00      JP     3
+
+        >>> df.to_spark_io(path='%s/to_spark_io/foo.json' % path, format='json')
+        """
+        if "options" in options and isinstance(options.get("options"), dict) and len(options) == 1:
+            options = options.get("options")  # type: ignore
+
+        self._kdf.spark.frame(index_col=index_col).write.save(
+            path=path, format=format, mode=mode, partitionBy=partition_cols, **options
+        )
+
+    def explain(self, extended: Optional[bool] = None, mode: Optional[str] = None) -> None:
+        """
+        Prints the underlying (logical and physical) Spark plans to the console for debugging
+        purpose.
+
+        Parameters
+        ----------
+        extended : boolean, default ``False``.
+            If ``False``, prints only the physical plan.
+        mode : string, default ``None``.
+            The expected output format of plans.
+
+        Returns
+        -------
+        None
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({'id': range(10)})
+        >>> df.spark.explain()  # doctest: +ELLIPSIS
+        == Physical Plan ==
+        ...
+
+        >>> df.spark.explain(True)  # doctest: +ELLIPSIS
+        == Parsed Logical Plan ==
+        ...
+        == Analyzed Logical Plan ==
+        ...
+        == Optimized Logical Plan ==
+        ...
+        == Physical Plan ==
+        ...
+
+        >>> df.spark.explain("extended")  # doctest: +ELLIPSIS
+        == Parsed Logical Plan ==
+        ...
+        == Analyzed Logical Plan ==
+        ...
+        == Optimized Logical Plan ==
+        ...
+        == Physical Plan ==
+        ...
+
+        >>> df.spark.explain(mode="extended")  # doctest: +ELLIPSIS
+        == Parsed Logical Plan ==
+        ...
+        == Analyzed Logical Plan ==
+        ...
+        == Optimized Logical Plan ==
+        ...
+        == Physical Plan ==
+        ...
+        """
+        self._kdf._internal.to_internal_spark_frame.explain(extended, mode)
+
+    def apply(self, func, index_col: Optional[Union[str, List[str]]] = None) -> "ps.DataFrame":
+        """
+        Applies a function that takes and returns a Spark DataFrame. It allows natively
+        apply a Spark function and column APIs with the Spark column internally used
+        in Series or Index.
+
+        .. note:: set `index_col` and keep the column named as so in the output Spark
+            DataFrame to avoid using the default index to prevent performance penalty.
+            If you omit `index_col`, it will use default index which is potentially
+            expensive in general.
+
+        .. note:: it will lose column labels. This is a synonym of
+            ``func(kdf.to_spark(index_col)).to_koalas(index_col)``.
+
+        Parameters
+        ----------
+        func : function
+            Function to apply the function against the data by using Spark DataFrame.
+
+        Returns
+        -------
+        DataFrame
+
+        Raises
+        ------
+        ValueError : If the output from the function is not a Spark DataFrame.
+
+        Examples
+        --------
+        >>> kdf = ps.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}, columns=["a", "b"])
+        >>> kdf
+           a  b
+        0  1  4
+        1  2  5
+        2  3  6
+
+        >>> kdf.spark.apply(
+        ...     lambda sdf: sdf.selectExpr("a + b as c", "index"), index_col="index")
+        ... # doctest: +NORMALIZE_WHITESPACE
+               c
+        index
+        0      5
+        1      7
+        2      9
+
+        The case below ends up with using the default index, which should be avoided
+        if possible.
+
+        >>> kdf.spark.apply(lambda sdf: sdf.groupby("a").count().sort("a"))
+           a  count
+        0  1      1
+        1  2      1
+        2  3      1
+        """
+        output = func(self.frame(index_col))
+        if not isinstance(output, SparkDataFrame):
+            raise ValueError(
+                "The output of the function [%s] should be of a "
+                "pyspark.sql.DataFrame; however, got [%s]." % (func, type(output))
+            )
+        return output.to_koalas(index_col)
+
+    def repartition(self, num_partitions: int) -> "ps.DataFrame":
+        """
+        Returns a new DataFrame partitioned by the given partitioning expressions. The
+        resulting DataFrame is hash partitioned.
+
+        Parameters
+        ----------
+        num_partitions : int
+            The target number of partitions.
+
+        Returns
+        -------
+        DataFrame
+
+        Examples
+        --------
+        >>> kdf = ps.DataFrame({"age": [5, 5, 2, 2],
+        ...         "name": ["Bob", "Bob", "Alice", "Alice"]}).set_index("age")
+        >>> kdf.sort_index()  # doctest: +NORMALIZE_WHITESPACE
+              name
+        age
+        2    Alice
+        2    Alice
+        5      Bob
+        5      Bob
+        >>> new_kdf = kdf.spark.repartition(7)
+        >>> new_kdf.to_spark().rdd.getNumPartitions()
+        7
+        >>> new_kdf.sort_index()   # doctest: +NORMALIZE_WHITESPACE
+              name
+        age
+        2    Alice
+        2    Alice
+        5      Bob
+        5      Bob
+        """
+        from pyspark.pandas.frame import DataFrame
+
+        internal = self._kdf._internal.resolved_copy
+        repartitioned_sdf = internal.spark_frame.repartition(num_partitions)
+        return DataFrame(internal.with_new_sdf(repartitioned_sdf))
+
+    def coalesce(self, num_partitions: int) -> "ps.DataFrame":
+        """
+        Returns a new DataFrame that has exactly `num_partitions` partitions.
+
+        .. note:: This operation results in a narrow dependency, e.g. if you go from 1000
+            partitions to 100 partitions, there will not be a shuffle, instead each of the 100 new
+            partitions will claim 10 of the current partitions. If a larger number of partitions is
+            requested, it will stay at the current number of partitions. However, if you're doing a
+            drastic coalesce, e.g. to num_partitions = 1, this may result in your computation taking
+            place on fewer nodes than you like (e.g. one node in the case of num_partitions = 1). To
+            avoid this, you can call repartition(). This will add a shuffle step, but means the
+            current upstream partitions will be executed in parallel (per whatever the current
+            partitioning is).
+
+        Parameters
+        ----------
+        num_partitions : int
+            The target number of partitions.
+
+        Returns
+        -------
+        DataFrame
+
+        Examples
+        --------
+        >>> kdf = ps.DataFrame({"age": [5, 5, 2, 2],
+        ...         "name": ["Bob", "Bob", "Alice", "Alice"]}).set_index("age")
+        >>> kdf.sort_index()  # doctest: +NORMALIZE_WHITESPACE
+              name
+        age
+        2    Alice
+        2    Alice
+        5      Bob
+        5      Bob
+        >>> new_kdf = kdf.spark.coalesce(1)
+        >>> new_kdf.to_spark().rdd.getNumPartitions()
+        1
+        >>> new_kdf.sort_index()   # doctest: +NORMALIZE_WHITESPACE
+              name
+        age
+        2    Alice
+        2    Alice
+        5      Bob
+        5      Bob
+        """
+        from pyspark.pandas.frame import DataFrame
+
+        internal = self._kdf._internal.resolved_copy
+        coalesced_sdf = internal.spark_frame.coalesce(num_partitions)
+        return DataFrame(internal.with_new_sdf(coalesced_sdf))
+
+    def checkpoint(self, eager: bool = True) -> "ps.DataFrame":
+        """Returns a checkpointed version of this DataFrame.
+
+        Checkpointing can be used to truncate the logical plan of this DataFrame, which is
+        especially useful in iterative algorithms where the plan may grow exponentially. It will be
+        saved to files inside the checkpoint directory set with `SparkContext.setCheckpointDir`.
+
+        Parameters
+        ----------
+        eager : bool
+            Whether to checkpoint this DataFrame immediately
+
+        Returns
+        -------
+        DataFrame
+
+        Examples
+        --------
+        >>> kdf = ps.DataFrame({"a": ["a", "b", "c"]})
+        >>> kdf
+           a
+        0  a
+        1  b
+        2  c
+        >>> new_kdf = kdf.spark.checkpoint()  # doctest: +SKIP
+        >>> new_kdf  # doctest: +SKIP
+           a
+        0  a
+        1  b
+        2  c
+        """
+        from pyspark.pandas.frame import DataFrame
+
+        internal = self._kdf._internal.resolved_copy
+        checkpointed_sdf = internal.spark_frame.checkpoint(eager)
+        return DataFrame(internal.with_new_sdf(checkpointed_sdf))
+
+    def local_checkpoint(self, eager: bool = True) -> "ps.DataFrame":
+        """Returns a locally checkpointed version of this DataFrame.
+
+        Checkpointing can be used to truncate the logical plan of this DataFrame, which is
+        especially useful in iterative algorithms where the plan may grow exponentially. Local
+        checkpoints are stored in the executors using the caching subsystem and therefore they are
+        not reliable.
+
+        Parameters
+        ----------
+        eager : bool
+            Whether to locally checkpoint this DataFrame immediately
+
+        Returns
+        -------
+        DataFrame
+
+        Examples
+        --------
+        >>> kdf = ps.DataFrame({"a": ["a", "b", "c"]})
+        >>> kdf
+           a
+        0  a
+        1  b
+        2  c
+        >>> new_kdf = kdf.spark.local_checkpoint()
+        >>> new_kdf
+           a
+        0  a
+        1  b
+        2  c
+        """
+        from pyspark.pandas.frame import DataFrame
+
+        internal = self._kdf._internal.resolved_copy
+        checkpointed_sdf = internal.spark_frame.localCheckpoint(eager)
+        return DataFrame(internal.with_new_sdf(checkpointed_sdf))
+
+    @property
+    def analyzed(self) -> "ps.DataFrame":
+        """
+        Returns a new DataFrame with the analyzed Spark DataFrame.
+
+        After multiple operations, the underlying Spark plan could grow huge
+        and make the Spark planner take a long time to finish the planning.
+
+        This function is for the workaround to avoid it.
+
+        .. note:: After analyzed, operations between the analyzed DataFrame and the original one
+            will **NOT** work without setting a config `compute.ops_on_diff_frames` to `True`.
+
+        Returns
+        -------
+        DataFrame
+
+        Examples
+        --------
+        >>> df = ps.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}, columns=["a", "b"])
+        >>> df
+           a  b
+        0  1  4
+        1  2  5
+        2  3  6
+
+        The analyzed one should return the same value.
+
+        >>> df.spark.analyzed
+           a  b
+        0  1  4
+        1  2  5
+        2  3  6
+
+        However, it won't work with the same anchor Series.
+
+        >>> df + df.spark.analyzed
+        Traceback (most recent call last):
+        ...
+        ValueError: ... enable 'compute.ops_on_diff_frames' option.
+
+        >>> with ps.option_context('compute.ops_on_diff_frames', True):
+        ...     (df + df.spark.analyzed).sort_index()
+           a   b
+        0  2   8
+        1  4  10
+        2  6  12
+        """
+        from pyspark.pandas.frame import DataFrame
+
+        return DataFrame(self._kdf._internal.resolved_copy)
+
+
+class CachedSparkFrameMethods(SparkFrameMethods):
+    """Spark related features for cached DataFrame. This is usually created via
+    `df.spark.cache()`."""
+
+    def __init__(self, frame: "CachedDataFrame"):
+        super().__init__(frame)
+
+    @property
+    def storage_level(self) -> StorageLevel:
+        """
+        Return the storage level of this cache.
+
+        Examples
+        --------
+        >>> import pyspark.pandas as ps
+        >>> import pyspark
+        >>> df = ps.DataFrame([(.2, .3), (.0, .6), (.6, .0), (.2, .1)],
+        ...                   columns=['dogs', 'cats'])
+        >>> df
+           dogs  cats
+        0   0.2   0.3
+        1   0.0   0.6
+        2   0.6   0.0
+        3   0.2   0.1
+
+        >>> with df.spark.cache() as cached_df:
+        ...     print(cached_df.spark.storage_level)
+        ...
+        Disk Memory Deserialized 1x Replicated
+
+        Set the StorageLevel to `MEMORY_ONLY`.
+
+        >>> with df.spark.persist(pyspark.StorageLevel.MEMORY_ONLY) as cached_df:
+        ...     print(cached_df.spark.storage_level)
+        ...
+        Memory Serialized 1x Replicated
+        """
+        return self._kdf._cached.storageLevel
+
+    def unpersist(self) -> None:
+        """
+        The `unpersist` function is used to uncache the pandas-on-Spark DataFrame when it
+        is not used with `with` statement.
+
+        Returns
+        -------
+        None
+
+        Examples
+        --------
+        >>> df = ps.DataFrame([(.2, .3), (.0, .6), (.6, .0), (.2, .1)],
+        ...                   columns=['dogs', 'cats'])
+        >>> df = df.spark.cache()
+
+        To uncache the dataframe, use `unpersist` function
+
+        >>> df.spark.unpersist()
+        """
+        if self._kdf._cached.is_cached:
+            self._kdf._cached.unpersist()
+
+
+def _test():
+    import os
+    import doctest
+    import shutil
+    import sys
+    import tempfile
+    import uuid
+    import numpy
+    import pandas
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.spark.accessors
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.spark.accessors.__dict__.copy()
+    globs["np"] = numpy
+    globs["pd"] = pandas
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]")
+        .appName("pyspark.pandas.spark.accessors tests")
+        .getOrCreate()
+    )
+
+    db_name = "db%s" % str(uuid.uuid4()).replace("-", "")
+    spark.sql("CREATE DATABASE %s" % db_name)
+    globs["db"] = db_name
+
+    path = tempfile.mkdtemp()
+    globs["path"] = path
+
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.spark.accessors,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+
+    shutil.rmtree(path, ignore_errors=True)
+    spark.sql("DROP DATABASE IF EXISTS %s CASCADE" % db_name)
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/spark/functions.py b/python/pyspark/pandas/spark/functions.py
new file mode 100644
index 0000000000000..cb060bd7d39dc
--- /dev/null
+++ b/python/pyspark/pandas/spark/functions.py
@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+Additional Spark functions used in pandas-on-Spark.
+"""
+
+from pyspark import SparkContext
+from pyspark.sql.column import Column, _to_java_column, _create_column_from_literal
+
+
+def repeat(col, n):
+    """
+    Repeats a string column n times, and returns it as a new string column.
+    """
+    sc = SparkContext._active_spark_context
+    n = _to_java_column(n) if isinstance(n, Column) else _create_column_from_literal(n)
+    return _call_udf(sc, "repeat", _to_java_column(col), n)
+
+
+def _call_udf(sc, name, *cols):
+    return Column(sc._jvm.functions.callUDF(name, _make_arguments(sc, *cols)))
+
+
+def _make_arguments(sc, *cols):
+    java_arr = sc._gateway.new_array(sc._jvm.Column, len(cols))
+    for i, col in enumerate(cols):
+        java_arr[i] = col
+    return java_arr
diff --git a/python/pyspark/pandas/spark/utils.py b/python/pyspark/pandas/spark/utils.py
new file mode 100644
index 0000000000000..09aed454c3313
--- /dev/null
+++ b/python/pyspark/pandas/spark/utils.py
@@ -0,0 +1,143 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+Helpers and utilities to deal with PySpark instances
+"""
+from pyspark.sql.types import DecimalType, StructType, MapType, ArrayType, StructField, DataType
+
+
+def as_nullable_spark_type(dt: DataType) -> DataType:
+    """
+    Returns a nullable schema or data types.
+
+    Examples
+    --------
+    >>> from pyspark.sql.types import *
+    >>> as_nullable_spark_type(StructType([
+    ...     StructField("A", IntegerType(), True),
+    ...     StructField("B", FloatType(), False)]))  # doctest: +NORMALIZE_WHITESPACE
+    StructType(List(StructField(A,IntegerType,true),StructField(B,FloatType,true)))
+
+    >>> as_nullable_spark_type(StructType([
+    ...     StructField("A",
+    ...         StructType([
+    ...             StructField('a',
+    ...                 MapType(IntegerType(),
+    ...                 ArrayType(IntegerType(), False), False), False),
+    ...             StructField('b', StringType(), True)])),
+    ...     StructField("B", FloatType(), False)]))  # doctest: +NORMALIZE_WHITESPACE
+    StructType(List(StructField(A,StructType(List(StructField(a,MapType(IntegerType,ArrayType\
+(IntegerType,true),true),true),StructField(b,StringType,true))),true),\
+StructField(B,FloatType,true)))
+    """
+    if isinstance(dt, StructType):
+        new_fields = []
+        for field in dt.fields:
+            new_fields.append(
+                StructField(
+                    field.name,
+                    as_nullable_spark_type(field.dataType),
+                    nullable=True,
+                    metadata=field.metadata,
+                )
+            )
+        return StructType(new_fields)
+    elif isinstance(dt, ArrayType):
+        return ArrayType(as_nullable_spark_type(dt.elementType), containsNull=True)
+    elif isinstance(dt, MapType):
+        return MapType(
+            as_nullable_spark_type(dt.keyType),
+            as_nullable_spark_type(dt.valueType),
+            valueContainsNull=True,
+        )
+    else:
+        return dt
+
+
+def force_decimal_precision_scale(dt: DataType, precision: int = 38, scale: int = 18) -> DataType:
+    """
+    Returns a data type with a fixed decimal type.
+
+    The precision and scale of the decimal type are fixed with the given values.
+
+    Examples
+    --------
+    >>> from pyspark.sql.types import *
+    >>> force_decimal_precision_scale(StructType([
+    ...     StructField("A", DecimalType(10, 0), True),
+    ...     StructField("B", DecimalType(14, 7), False)]))  # doctest: +NORMALIZE_WHITESPACE
+    StructType(List(StructField(A,DecimalType(38,18),true),StructField(B,DecimalType(38,18),false)))
+
+    >>> force_decimal_precision_scale(StructType([
+    ...     StructField("A",
+    ...         StructType([
+    ...             StructField('a',
+    ...                 MapType(DecimalType(5, 0),
+    ...                 ArrayType(DecimalType(20, 0), False), False), False),
+    ...             StructField('b', StringType(), True)])),
+    ...     StructField("B", DecimalType(30, 15), False)]),
+    ...     precision=30, scale=15)  # doctest: +NORMALIZE_WHITESPACE
+    StructType(List(StructField(A,StructType(List(StructField(a,MapType(DecimalType(30,15),\
+ArrayType(DecimalType(30,15),false),false),false),StructField(b,StringType,true))),true),\
+StructField(B,DecimalType(30,15),false)))
+    """
+    if isinstance(dt, StructType):
+        new_fields = []
+        for field in dt.fields:
+            new_fields.append(
+                StructField(
+                    field.name,
+                    force_decimal_precision_scale(field.dataType, precision, scale),
+                    nullable=field.nullable,
+                    metadata=field.metadata,
+                )
+            )
+        return StructType(new_fields)
+    elif isinstance(dt, ArrayType):
+        return ArrayType(
+            force_decimal_precision_scale(dt.elementType, precision, scale),
+            containsNull=dt.containsNull,
+        )
+    elif isinstance(dt, MapType):
+        return MapType(
+            force_decimal_precision_scale(dt.keyType, precision, scale),
+            force_decimal_precision_scale(dt.valueType, precision, scale),
+            valueContainsNull=dt.valueContainsNull,
+        )
+    elif isinstance(dt, DecimalType):
+        return DecimalType(precision=precision, scale=scale)
+    else:
+        return dt
+
+
+def _test():
+    import doctest
+    import sys
+    import pyspark.pandas.spark.utils
+
+    globs = pyspark.pandas.spark.utils.__dict__.copy()
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.spark.utils,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/sql_processor.py b/python/pyspark/pandas/sql_processor.py
new file mode 100644
index 0000000000000..beebfa1ef436f
--- /dev/null
+++ b/python/pyspark/pandas/sql_processor.py
@@ -0,0 +1,333 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import _string
+from typing import Dict, Any, Optional  # noqa: F401 (SPARK-34943)
+import inspect
+import pandas as pd
+
+from pyspark.sql import SparkSession, DataFrame as SDataFrame  # noqa: F401 (SPARK-34943)
+
+from pyspark import pandas as ps  # For running doctests and reference resolution in PyCharm.
+from pyspark.pandas.utils import default_session
+from pyspark.pandas.frame import DataFrame
+from pyspark.pandas.series import Series
+
+
+__all__ = ["sql"]
+
+from builtins import globals as builtin_globals
+from builtins import locals as builtin_locals
+
+
+def sql(query: str, globals=None, locals=None, **kwargs) -> DataFrame:
+    """
+    Execute a SQL query and return the result as a pandas-on-Spark DataFrame.
+
+    This function also supports embedding Python variables (locals, globals, and parameters)
+    in the SQL statement by wrapping them in curly braces. See examples section for details.
+
+    In addition to the locals, globals and parameters, the function will also attempt
+    to determine if the program currently runs in an IPython (or Jupyter) environment
+    and to import the variables from this environment. The variables have the same
+    precedence as globals.
+
+    The following variable types are supported:
+
+        * string
+        * int
+        * float
+        * list, tuple, range of above types
+        * pandas-on-Spark DataFrame
+        * pandas-on-Spark Series
+        * pandas DataFrame
+
+    Parameters
+    ----------
+    query : str
+        the SQL query
+    globals : dict, optional
+        the dictionary of global variables, if explicitly set by the user
+    locals : dict, optional
+        the dictionary of local variables, if explicitly set by the user
+    kwargs
+        other variables that the user may want to set manually that can be referenced in the query
+
+    Returns
+    -------
+    pandas-on-Spark DataFrame
+
+    Examples
+    --------
+
+    Calling a built-in SQL function.
+
+    >>> ps.sql("select * from range(10) where id > 7")
+       id
+    0   8
+    1   9
+
+    A query can also reference a local variable or parameter by wrapping them in curly braces:
+
+    >>> bound1 = 7
+    >>> ps.sql("select * from range(10) where id > {bound1} and id < {bound2}", bound2=9)
+       id
+    0   8
+
+    You can also wrap a DataFrame with curly braces to query it directly. Note that when you do
+    that, the indexes, if any, automatically become top level columns.
+
+    >>> mydf = ps.range(10)
+    >>> x = range(4)
+    >>> ps.sql("SELECT * from {mydf} WHERE id IN {x}")
+       id
+    0   0
+    1   1
+    2   2
+    3   3
+
+    Queries can also be arbitrarily nested in functions:
+
+    >>> def statement():
+    ...     mydf2 = ps.DataFrame({"x": range(2)})
+    ...     return ps.sql("SELECT * from {mydf2}")
+    >>> statement()
+       x
+    0  0
+    1  1
+
+    Mixing pandas-on-Spark and pandas DataFrames in a join operation. Note that the index is
+    dropped.
+
+    >>> ps.sql('''
+    ...   SELECT m1.a, m2.b
+    ...   FROM {table1} m1 INNER JOIN {table2} m2
+    ...   ON m1.key = m2.key
+    ...   ORDER BY m1.a, m2.b''',
+    ...   table1=ps.DataFrame({"a": [1,2], "key": ["a", "b"]}),
+    ...   table2=pd.DataFrame({"b": [3,4,5], "key": ["a", "b", "b"]}))
+       a  b
+    0  1  3
+    1  2  4
+    2  2  5
+
+    Also, it is possible to query using Series.
+
+    >>> myser = ps.Series({'a': [1.0, 2.0, 3.0], 'b': [15.0, 30.0, 45.0]})
+    >>> ps.sql("SELECT * from {myser}")
+                        0
+    0     [1.0, 2.0, 3.0]
+    1  [15.0, 30.0, 45.0]
+    """
+    if globals is None:
+        globals = _get_ipython_scope()
+    _globals = builtin_globals() if globals is None else dict(globals)
+    _locals = builtin_locals() if locals is None else dict(locals)
+    # The default choice is the globals
+    _dict = dict(_globals)
+    # The vars:
+    _scope = _get_local_scope()
+    _dict.update(_scope)
+    # Then the locals
+    _dict.update(_locals)
+    # Highest order of precedence is the locals
+    _dict.update(kwargs)
+    return SQLProcessor(_dict, query, default_session()).execute()
+
+
+_CAPTURE_SCOPES = 2
+
+
+def _get_local_scope():
+    # Get 2 scopes above (_get_local_scope -> sql -> ...) to capture the vars there.
+    try:
+        return inspect.stack()[_CAPTURE_SCOPES][0].f_locals
+    except Exception as e:
+        # TODO (rxin, thunterdb): use a more narrow scope exception.
+        # See https://github.com/pyspark.pandas/pull/448
+        return {}
+
+
+def _get_ipython_scope():
+    """
+    Tries to extract the dictionary of variables if the program is running
+    in an IPython notebook environment.
+    """
+    try:
+        from IPython import get_ipython
+
+        shell = get_ipython()
+        return shell.user_ns
+    except Exception as e:
+        # TODO (rxin, thunterdb): use a more narrow scope exception.
+        # See https://github.com/pyspark.pandas/pull/448
+        return None
+
+
+# Originally from pymysql package
+_escape_table = [chr(x) for x in range(128)]
+_escape_table[0] = "\\0"
+_escape_table[ord("\\")] = "\\\\"
+_escape_table[ord("\n")] = "\\n"
+_escape_table[ord("\r")] = "\\r"
+_escape_table[ord("\032")] = "\\Z"
+_escape_table[ord('"')] = '\\"'
+_escape_table[ord("'")] = "\\'"
+
+
+def escape_sql_string(value: str) -> str:
+    """Escapes value without adding quotes.
+
+    >>> escape_sql_string("foo\\nbar")
+    'foo\\\\nbar'
+
+    >>> escape_sql_string("'abc'de")
+    "\\\\'abc\\\\'de"
+
+    >>> escape_sql_string('"abc"de')
+    '\\\\"abc\\\\"de'
+    """
+    return value.translate(_escape_table)
+
+
+class SQLProcessor(object):
+    def __init__(self, scope: Dict[str, Any], statement: str, session: SparkSession):
+        self._scope = scope
+        self._statement = statement
+        # All the temporary views created when executing this statement
+        # The key is the name of the variable in {}
+        # The value is the cached Spark Dataframe.
+        self._temp_views = {}  # type: Dict[str, SDataFrame]
+        # All the other variables, converted to a normalized form.
+        # The normalized form is typically a string
+        self._cached_vars = {}  # type: Dict[str, Any]
+        # The SQL statement after:
+        # - all the dataframes have been have been registered as temporary views
+        # - all the values have been converted normalized to equivalent SQL representations
+        self._normalized_statement = None  # type: Optional[str]
+        self._session = session
+
+    def execute(self) -> DataFrame:
+        """
+        Returns a DataFrame for which the SQL statement has been executed by
+        the underlying SQL engine.
+
+        >>> str0 = 'abc'
+        >>> ps.sql("select {str0}")
+           abc
+        0  abc
+
+        >>> str1 = 'abc"abc'
+        >>> str2 = "abc'abc"
+        >>> ps.sql("select {str0}, {str1}, {str2}")
+           abc  abc"abc  abc'abc
+        0  abc  abc"abc  abc'abc
+
+        >>> strs = ['a', 'b']
+        >>> ps.sql("select 'a' in {strs} as cond1, 'c' in {strs} as cond2")
+           cond1  cond2
+        0   True  False
+        """
+        blocks = _string.formatter_parser(self._statement)
+        # TODO: use a string builder
+        res = ""
+        try:
+            for (pre, inner, _, _) in blocks:
+                var_next = "" if inner is None else self._convert(inner)
+                res = res + pre + var_next
+            self._normalized_statement = res
+
+            sdf = self._session.sql(self._normalized_statement)
+        finally:
+            for v in self._temp_views:
+                self._session.catalog.dropTempView(v)
+        return DataFrame(sdf)
+
+    def _convert(self, key) -> Any:
+        """
+        Given a {} key, returns an equivalent SQL representation.
+        This conversion performs all the necessary escaping so that the string
+        returned can be directly injected into the SQL statement.
+        """
+        # Already cached?
+        if key in self._cached_vars:
+            return self._cached_vars[key]
+        # Analyze:
+        if key not in self._scope:
+            raise ValueError(
+                "The key {} in the SQL statement was not found in global,"
+                " local or parameters variables".format(key)
+            )
+        var = self._scope[key]
+        fillin = self._convert_var(var)
+        self._cached_vars[key] = fillin
+        return fillin
+
+    def _convert_var(self, var) -> Any:
+        """
+        Converts a python object into a string that is legal SQL.
+        """
+        if isinstance(var, (int, float)):
+            return str(var)
+        if isinstance(var, Series):
+            return self._convert_var(var.to_dataframe())
+        if isinstance(var, pd.DataFrame):
+            return self._convert_var(ps.DataFrame(var))
+        if isinstance(var, DataFrame):
+            df_id = "pandas_on_spark_" + str(id(var))
+            if df_id not in self._temp_views:
+                sdf = var.to_spark()
+                sdf.createOrReplaceTempView(df_id)
+                self._temp_views[df_id] = sdf
+            return df_id
+        if isinstance(var, str):
+            return '"' + escape_sql_string(var) + '"'
+        if isinstance(var, list):
+            return "(" + ", ".join([self._convert_var(v) for v in var]) + ")"
+        if isinstance(var, (tuple, range)):
+            return self._convert_var(list(var))
+        raise ValueError("Unsupported variable type {}: {}".format(type(var).__name__, str(var)))
+
+
+def _test():
+    import os
+    import doctest
+    import sys
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.sql_processor
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.sql_processor.__dict__.copy()
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]")
+        .appName("pyspark.pandas.sql_processor tests")
+        .getOrCreate()
+    )
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.sql_processor,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/strings.py b/python/pyspark/pandas/strings.py
new file mode 100644
index 0000000000000..2ae294008120e
--- /dev/null
+++ b/python/pyspark/pandas/strings.py
@@ -0,0 +1,2319 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+String functions on pandas-on-Spark Series
+"""
+from typing import Union, TYPE_CHECKING, cast, Optional, List
+
+import numpy as np
+
+from pyspark.sql.types import StringType, BinaryType, ArrayType, LongType, MapType
+from pyspark.sql import functions as F
+from pyspark.sql.functions import pandas_udf, PandasUDFType
+
+from pyspark.pandas.spark import functions as SF
+
+if TYPE_CHECKING:
+    import pyspark.pandas as ps  # noqa: F401 (SPARK-34943)
+
+
+class StringMethods(object):
+    """String methods for pandas-on-Spark Series"""
+
+    def __init__(self, series: "ps.Series"):
+        if not isinstance(series.spark.data_type, (StringType, BinaryType, ArrayType)):
+            raise ValueError("Cannot call StringMethods on type {}".format(series.spark.data_type))
+        self._data = series
+
+    # Methods
+    def capitalize(self) -> "ps.Series":
+        """
+        Convert Strings in the series to be capitalized.
+
+        Examples
+        --------
+        >>> s = ps.Series(['lower', 'CAPITALS', 'this is a sentence', 'SwApCaSe'])
+        >>> s
+        0                 lower
+        1              CAPITALS
+        2    this is a sentence
+        3              SwApCaSe
+        dtype: object
+
+        >>> s.str.capitalize()
+        0                 Lower
+        1              Capitals
+        2    This is a sentence
+        3              Swapcase
+        dtype: object
+        """
+
+        def pandas_capitalize(s) -> "ps.Series[str]":
+            return s.str.capitalize()
+
+        return self._data.koalas.transform_batch(pandas_capitalize)
+
+    def title(self) -> "ps.Series":
+        """
+        Convert Strings in the series to be titlecase.
+
+        Examples
+        --------
+        >>> s = ps.Series(['lower', 'CAPITALS', 'this is a sentence', 'SwApCaSe'])
+        >>> s
+        0                 lower
+        1              CAPITALS
+        2    this is a sentence
+        3              SwApCaSe
+        dtype: object
+
+        >>> s.str.title()
+        0                 Lower
+        1              Capitals
+        2    This Is A Sentence
+        3              Swapcase
+        dtype: object
+        """
+
+        def pandas_title(s) -> "ps.Series[str]":
+            return s.str.title()
+
+        return self._data.koalas.transform_batch(pandas_title)
+
+    def lower(self) -> "ps.Series":
+        """
+        Convert strings in the Series/Index to all lowercase.
+
+        Examples
+        --------
+        >>> s = ps.Series(['lower', 'CAPITALS', 'this is a sentence', 'SwApCaSe'])
+        >>> s
+        0                 lower
+        1              CAPITALS
+        2    this is a sentence
+        3              SwApCaSe
+        dtype: object
+
+        >>> s.str.lower()
+        0                 lower
+        1              capitals
+        2    this is a sentence
+        3              swapcase
+        dtype: object
+        """
+        return self._data.spark.transform(F.lower)
+
+    def upper(self) -> "ps.Series":
+        """
+        Convert strings in the Series/Index to all uppercase.
+
+        Examples
+        --------
+        >>> s = ps.Series(['lower', 'CAPITALS', 'this is a sentence', 'SwApCaSe'])
+        >>> s
+        0                 lower
+        1              CAPITALS
+        2    this is a sentence
+        3              SwApCaSe
+        dtype: object
+
+        >>> s.str.upper()
+        0                 LOWER
+        1              CAPITALS
+        2    THIS IS A SENTENCE
+        3              SWAPCASE
+        dtype: object
+        """
+        return self._data.spark.transform(F.upper)
+
+    def swapcase(self) -> "ps.Series":
+        """
+        Convert strings in the Series/Index to be swapcased.
+
+        Examples
+        --------
+        >>> s = ps.Series(['lower', 'CAPITALS', 'this is a sentence', 'SwApCaSe'])
+        >>> s
+        0                 lower
+        1              CAPITALS
+        2    this is a sentence
+        3              SwApCaSe
+        dtype: object
+
+        >>> s.str.swapcase()
+        0                 LOWER
+        1              capitals
+        2    THIS IS A SENTENCE
+        3              sWaPcAsE
+        dtype: object
+        """
+
+        def pandas_swapcase(s) -> "ps.Series[str]":
+            return s.str.swapcase()
+
+        return self._data.koalas.transform_batch(pandas_swapcase)
+
+    def startswith(self, pattern, na=None) -> "ps.Series":
+        """
+        Test if the start of each string element matches a pattern.
+
+        Equivalent to :func:`str.startswith`.
+
+        Parameters
+        ----------
+        pattern : str
+            Character sequence. Regular expressions are not accepted.
+        na : object, default None
+            Object shown if element is not a string. NaN converted to None.
+
+        Returns
+        -------
+        Series of bool or object
+            pandas-on-Spark Series of booleans indicating whether the given pattern
+            matches the start of each string element.
+
+        Examples
+        --------
+        >>> s = ps.Series(['bat', 'Bear', 'cat', np.nan])
+        >>> s
+        0     bat
+        1    Bear
+        2     cat
+        3    None
+        dtype: object
+
+        >>> s.str.startswith('b')
+        0     True
+        1    False
+        2    False
+        3     None
+        dtype: object
+
+        Specifying na to be False instead of None.
+
+        >>> s.str.startswith('b', na=False)
+        0     True
+        1    False
+        2    False
+        3    False
+        dtype: bool
+        """
+
+        def pandas_startswith(s) -> "ps.Series[bool]":
+            return s.str.startswith(pattern, na)
+
+        return self._data.koalas.transform_batch(pandas_startswith)
+
+    def endswith(self, pattern, na=None) -> "ps.Series":
+        """
+        Test if the end of each string element matches a pattern.
+
+        Equivalent to :func:`str.endswith`.
+
+        Parameters
+        ----------
+        pattern : str
+            Character sequence. Regular expressions are not accepted.
+        na : object, default None
+            Object shown if element is not a string. NaN converted to None.
+
+        Returns
+        -------
+        Series of bool or object
+            pandas-on-Spark Series of booleans indicating whether the given pattern
+            matches the end of each string element.
+
+        Examples
+        --------
+        >>> s = ps.Series(['bat', 'Bear', 'cat', np.nan])
+        >>> s
+        0     bat
+        1    Bear
+        2     cat
+        3    None
+        dtype: object
+
+        >>> s.str.endswith('t')
+        0     True
+        1    False
+        2     True
+        3     None
+        dtype: object
+
+        Specifying na to be False instead of None.
+
+        >>> s.str.endswith('t', na=False)
+        0     True
+        1    False
+        2     True
+        3    False
+        dtype: bool
+        """
+
+        def pandas_endswith(s) -> "ps.Series[bool]":
+            return s.str.endswith(pattern, na)
+
+        return self._data.koalas.transform_batch(pandas_endswith)
+
+    def strip(self, to_strip=None) -> "ps.Series":
+        """
+        Remove leading and trailing characters.
+
+        Strip whitespaces (including newlines) or a set of specified
+        characters from each string in the Series/Index from left and
+        right sides. Equivalent to :func:`str.strip`.
+
+        Parameters
+        ----------
+        to_strip : str
+            Specifying the set of characters to be removed. All combinations
+            of this set of characters will be stripped. If None then
+            whitespaces are removed.
+
+        Returns
+        -------
+        Series of objects
+
+        Examples
+        --------
+        >>> s = ps.Series(['1. Ant.', '2. Bee!\\t', None])
+        >>> s
+        0      1. Ant.
+        1    2. Bee!\\t
+        2         None
+        dtype: object
+
+        >>> s.str.strip()
+        0    1. Ant.
+        1    2. Bee!
+        2       None
+        dtype: object
+
+        >>> s.str.strip('12.')
+        0        Ant
+        1     Bee!\\t
+        2       None
+        dtype: object
+
+        >>> s.str.strip('.!\\t')
+        0    1. Ant
+        1    2. Bee
+        2      None
+        dtype: object
+        """
+
+        def pandas_strip(s) -> "ps.Series[str]":
+            return s.str.strip(to_strip)
+
+        return self._data.koalas.transform_batch(pandas_strip)
+
+    def lstrip(self, to_strip=None) -> "ps.Series":
+        """
+        Remove leading characters.
+
+        Strip whitespaces (including newlines) or a set of specified
+        characters from each string in the Series/Index from left side.
+        Equivalent to :func:`str.lstrip`.
+
+        Parameters
+        ----------
+        to_strip : str
+            Specifying the set of characters to be removed. All combinations
+            of this set of characters will be stripped. If None then
+            whitespaces are removed.
+
+        Returns
+        -------
+        Series of object
+
+        Examples
+        --------
+        >>> s = ps.Series(['1. Ant.', '2. Bee!\\t', None])
+        >>> s
+        0      1. Ant.
+        1    2. Bee!\\t
+        2         None
+        dtype: object
+
+        >>> s.str.lstrip('12.')
+        0       Ant.
+        1     Bee!\\t
+        2       None
+        dtype: object
+        """
+
+        def pandas_lstrip(s) -> "ps.Series[str]":
+            return s.str.lstrip(to_strip)
+
+        return self._data.koalas.transform_batch(pandas_lstrip)
+
+    def rstrip(self, to_strip=None) -> "ps.Series":
+        """
+        Remove trailing characters.
+
+        Strip whitespaces (including newlines) or a set of specified
+        characters from each string in the Series/Index from right side.
+        Equivalent to :func:`str.rstrip`.
+
+        Parameters
+        ----------
+        to_strip : str
+            Specifying the set of characters to be removed. All combinations
+            of this set of characters will be stripped. If None then
+            whitespaces are removed.
+
+        Returns
+        -------
+        Series of object
+
+        Examples
+        --------
+        >>> s = ps.Series(['1. Ant.', '2. Bee!\\t', None])
+        >>> s
+        0      1. Ant.
+        1    2. Bee!\\t
+        2         None
+        dtype: object
+
+        >>> s.str.rstrip('.!\\t')
+        0    1. Ant
+        1    2. Bee
+        2      None
+        dtype: object
+        """
+
+        def pandas_rstrip(s) -> "ps.Series[str]":
+            return s.str.rstrip(to_strip)
+
+        return self._data.koalas.transform_batch(pandas_rstrip)
+
+    def get(self, i) -> "ps.Series":
+        """
+        Extract element from each string or string list/tuple in the Series
+        at the specified position.
+
+        Parameters
+        ----------
+        i : int
+            Position of element to extract.
+
+        Returns
+        -------
+        Series of objects
+
+        Examples
+        --------
+        >>> s1 = ps.Series(["String", "123"])
+        >>> s1
+        0    String
+        1       123
+        dtype: object
+
+        >>> s1.str.get(1)
+        0    t
+        1    2
+        dtype: object
+
+        >>> s1.str.get(-1)
+        0    g
+        1    3
+        dtype: object
+
+        >>> s2 = ps.Series([["a", "b", "c"], ["x", "y"]])
+        >>> s2
+        0    [a, b, c]
+        1       [x, y]
+        dtype: object
+
+        >>> s2.str.get(0)
+        0    a
+        1    x
+        dtype: object
+
+        >>> s2.str.get(2)
+        0       c
+        1    None
+        dtype: object
+        """
+
+        def pandas_get(s) -> "ps.Series[str]":
+            return s.str.get(i)
+
+        return self._data.koalas.transform_batch(pandas_get)
+
+    def isalnum(self) -> "ps.Series":
+        """
+        Check whether all characters in each string are alphanumeric.
+
+        This is equivalent to running the Python string method
+        :func:`str.isalnum` for each element of the Series/Index.
+        If a string has zero characters, False is returned for that check.
+
+        Examples
+        --------
+        >>> s1 = ps.Series(['one', 'one1', '1', ''])
+
+        >>> s1.str.isalnum()
+        0     True
+        1     True
+        2     True
+        3    False
+        dtype: bool
+
+        Note that checks against characters mixed with any additional
+        punctuation or whitespace will evaluate to false for an alphanumeric
+        check.
+
+        >>> s2 = ps.Series(['A B', '1.5', '3,000'])
+        >>> s2.str.isalnum()
+        0    False
+        1    False
+        2    False
+        dtype: bool
+        """
+
+        def pandas_isalnum(s) -> "ps.Series[bool]":
+            return s.str.isalnum()
+
+        return self._data.koalas.transform_batch(pandas_isalnum)
+
+    def isalpha(self) -> "ps.Series":
+        """
+        Check whether all characters in each string are alphabetic.
+
+        This is equivalent to running the Python string method
+        :func:`str.isalpha` for each element of the Series/Index.
+        If a string has zero characters, False is returned for that check.
+
+        Examples
+        --------
+        >>> s1 = ps.Series(['one', 'one1', '1', ''])
+
+        >>> s1.str.isalpha()
+        0     True
+        1    False
+        2    False
+        3    False
+        dtype: bool
+        """
+
+        def pandas_isalpha(s) -> "ps.Series[bool]":
+            return s.str.isalpha()
+
+        return self._data.koalas.transform_batch(pandas_isalpha)
+
+    def isdigit(self) -> "ps.Series":
+        """
+        Check whether all characters in each string are digits.
+
+        This is equivalent to running the Python string method
+        :func:`str.isdigit` for each element of the Series/Index.
+        If a string has zero characters, False is returned for that check.
+
+        Examples
+        --------
+        >>> s = ps.Series(['23', '³', '⅕', ''])
+
+        The s.str.isdecimal method checks for characters used to form numbers
+        in base 10.
+
+        >>> s.str.isdecimal()
+        0     True
+        1    False
+        2    False
+        3    False
+        dtype: bool
+
+        The s.str.isdigit method is the same as s.str.isdecimal but also
+        includes special digits, like superscripted and subscripted digits in
+        unicode.
+
+        >>> s.str.isdigit()
+        0     True
+        1     True
+        2    False
+        3    False
+        dtype: bool
+
+        The s.str.isnumeric method is the same as s.str.isdigit but also
+        includes other characters that can represent quantities such as unicode
+        fractions.
+
+        >>> s.str.isnumeric()
+        0     True
+        1     True
+        2     True
+        3    False
+        dtype: bool
+        """
+
+        def pandas_isdigit(s) -> "ps.Series[bool]":
+            return s.str.isdigit()
+
+        return self._data.koalas.transform_batch(pandas_isdigit)
+
+    def isspace(self) -> "ps.Series":
+        """
+        Check whether all characters in each string are whitespaces.
+
+        This is equivalent to running the Python string method
+        :func:`str.isspace` for each element of the Series/Index.
+        If a string has zero characters, False is returned for that check.
+
+        Examples
+        --------
+        >>> s = ps.Series([' ', '\\t\\r\\n ', ''])
+        >>> s.str.isspace()
+        0     True
+        1     True
+        2    False
+        dtype: bool
+        """
+
+        def pandas_isspace(s) -> "ps.Series[bool]":
+            return s.str.isspace()
+
+        return self._data.koalas.transform_batch(pandas_isspace)
+
+    def islower(self) -> "ps.Series":
+        """
+        Check whether all characters in each string are lowercase.
+
+        This is equivalent to running the Python string method
+        :func:`str.islower` for each element of the Series/Index.
+        If a string has zero characters, False is returned for that check.
+
+        Examples
+        --------
+        >>> s = ps.Series(['leopard', 'Golden Eagle', 'SNAKE', ''])
+        >>> s.str.islower()
+        0     True
+        1    False
+        2    False
+        3    False
+        dtype: bool
+        """
+
+        def pandas_isspace(s) -> "ps.Series[bool]":
+            return s.str.islower()
+
+        return self._data.koalas.transform_batch(pandas_isspace)
+
+    def isupper(self) -> "ps.Series":
+        """
+        Check whether all characters in each string are uppercase.
+
+        This is equivalent to running the Python string method
+        :func:`str.isupper` for each element of the Series/Index.
+        If a string has zero characters, False is returned for that check.
+
+        Examples
+        --------
+        >>> s = ps.Series(['leopard', 'Golden Eagle', 'SNAKE', ''])
+        >>> s.str.isupper()
+        0    False
+        1    False
+        2     True
+        3    False
+        dtype: bool
+        """
+
+        def pandas_isspace(s) -> "ps.Series[bool]":
+            return s.str.isupper()
+
+        return self._data.koalas.transform_batch(pandas_isspace)
+
+    def istitle(self) -> "ps.Series":
+        """
+        Check whether all characters in each string are titlecase.
+
+        This is equivalent to running the Python string method
+        :func:`str.istitle` for each element of the Series/Index.
+        If a string has zero characters, False is returned for that check.
+
+        Examples
+        --------
+        >>> s = ps.Series(['leopard', 'Golden Eagle', 'SNAKE', ''])
+
+        The s.str.istitle method checks for whether all words are in title
+        case (whether only the first letter of each word is capitalized).
+        Words are assumed to be as any sequence of non-numeric characters
+        separated by whitespace characters.
+
+        >>> s.str.istitle()
+        0    False
+        1     True
+        2    False
+        3    False
+        dtype: bool
+        """
+
+        def pandas_istitle(s) -> "ps.Series[bool]":
+            return s.str.istitle()
+
+        return self._data.koalas.transform_batch(pandas_istitle)
+
+    def isnumeric(self) -> "ps.Series":
+        """
+        Check whether all characters in each string are numeric.
+
+        This is equivalent to running the Python string method
+        :func:`str.isnumeric` for each element of the Series/Index.
+        If a string has zero characters, False is returned for that check.
+
+        Examples
+        --------
+        >>> s1 = ps.Series(['one', 'one1', '1', ''])
+        >>> s1.str.isnumeric()
+        0    False
+        1    False
+        2     True
+        3    False
+        dtype: bool
+
+        >>> s2 = ps.Series(['23', '³', '⅕', ''])
+
+        The s2.str.isdecimal method checks for characters used to form numbers
+        in base 10.
+
+        >>> s2.str.isdecimal()
+        0     True
+        1    False
+        2    False
+        3    False
+        dtype: bool
+
+        The s2.str.isdigit method is the same as s2.str.isdecimal but also
+        includes special digits, like superscripted and subscripted digits in
+        unicode.
+
+        >>> s2.str.isdigit()
+        0     True
+        1     True
+        2    False
+        3    False
+        dtype: bool
+
+        The s2.str.isnumeric method is the same as s2.str.isdigit but also
+        includes other characters that can represent quantities such as unicode
+        fractions.
+
+        >>> s2.str.isnumeric()
+        0     True
+        1     True
+        2     True
+        3    False
+        dtype: bool
+        """
+
+        def pandas_isnumeric(s) -> "ps.Series[bool]":
+            return s.str.isnumeric()
+
+        return self._data.koalas.transform_batch(pandas_isnumeric)
+
+    def isdecimal(self) -> "ps.Series":
+        """
+        Check whether all characters in each string are decimals.
+
+        This is equivalent to running the Python string method
+        :func:`str.isdecimal` for each element of the Series/Index.
+        If a string has zero characters, False is returned for that check.
+
+        Examples
+        --------
+        >>> s = ps.Series(['23', '³', '⅕', ''])
+
+        The s.str.isdecimal method checks for characters used to form numbers
+        in base 10.
+
+        >>> s.str.isdecimal()
+        0     True
+        1    False
+        2    False
+        3    False
+        dtype: bool
+
+        The s.str.isdigit method is the same as s.str.isdecimal but also
+        includes special digits, like superscripted and subscripted digits in
+        unicode.
+
+        >>> s.str.isdigit()
+        0     True
+        1     True
+        2    False
+        3    False
+        dtype: bool
+
+        The s.str.isnumeric method is the same as s.str.isdigit but also
+        includes other characters that can represent quantities such as unicode
+        fractions.
+
+        >>> s.str.isnumeric()
+        0     True
+        1     True
+        2     True
+        3    False
+        dtype: bool
+        """
+
+        def pandas_isdecimal(s) -> "ps.Series[bool]":
+            return s.str.isdecimal()
+
+        return self._data.koalas.transform_batch(pandas_isdecimal)
+
+    def cat(self, others=None, sep=None, na_rep=None, join=None) -> "ps.Series":
+        """
+        Not supported.
+        """
+        raise NotImplementedError()
+
+    def center(self, width, fillchar=" ") -> "ps.Series":
+        """
+        Filling left and right side of strings in the Series/Index with an
+        additional character. Equivalent to :func:`str.center`.
+
+        Parameters
+        ----------
+        width : int
+            Minimum width of resulting string; additional characters will be
+            filled with fillchar.
+        fillchar : str
+            Additional character for filling, default is whitespace.
+
+        Returns
+        -------
+        Series of objects
+
+        Examples
+        --------
+        >>> s = ps.Series(["caribou", "tiger"])
+        >>> s
+        0    caribou
+        1      tiger
+        dtype: object
+
+        >>> s.str.center(width=10, fillchar='-')
+        0    -caribou--
+        1    --tiger---
+        dtype: object
+        """
+
+        def pandas_center(s) -> "ps.Series[str]":
+            return s.str.center(width, fillchar)
+
+        return self._data.koalas.transform_batch(pandas_center)
+
+    def contains(self, pat, case=True, flags=0, na=None, regex=True) -> "ps.Series":
+        """
+        Test if pattern or regex is contained within a string of a Series.
+
+        Return boolean Series based on whether a given pattern or regex is
+        contained within a string of a Series.
+
+        Analogous to :func:`match`, but less strict, relying on
+        :func:`re.search` instead of :func:`re.match`.
+
+        Parameters
+        ----------
+        pat : str
+            Character sequence or regular expression.
+        case : bool, default True
+            If True, case sensitive.
+        flags : int, default 0 (no flags)
+            Flags to pass through to the re module, e.g. re.IGNORECASE.
+        na : default None
+            Fill value for missing values. NaN converted to None.
+        regex : bool, default True
+            If True, assumes the pat is a regular expression.
+            If False, treats the pat as a literal string.
+
+
+        Returns
+        -------
+        Series of boolean values or object
+            A Series of boolean values indicating whether the given pattern is
+            contained within the string of each element of the Series.
+
+        Examples
+        --------
+        Returning a Series of booleans using only a literal pattern.
+
+        >>> s1 = ps.Series(['Mouse', 'dog', 'house and parrot', '23', np.NaN])
+        >>> s1.str.contains('og', regex=False)
+        0    False
+        1     True
+        2    False
+        3    False
+        4     None
+        dtype: object
+
+        Specifying case sensitivity using case.
+
+        >>> s1.str.contains('oG', case=True, regex=True)
+        0    False
+        1    False
+        2    False
+        3    False
+        4     None
+        dtype: object
+
+        Specifying na to be False instead of NaN replaces NaN values with
+        False. If Series does not contain NaN values the resultant dtype will
+        be bool, otherwise, an object dtype.
+
+        >>> s1.str.contains('og', na=False, regex=True)
+        0    False
+        1     True
+        2    False
+        3    False
+        4    False
+        dtype: bool
+
+        Returning ‘house’ or ‘dog’ when either expression occurs in a string.
+
+        >>> s1.str.contains('house|dog', regex=True)
+        0    False
+        1     True
+        2     True
+        3    False
+        4     None
+        dtype: object
+
+        Ignoring case sensitivity using flags with regex.
+
+        >>> import re
+        >>> s1.str.contains('PARROT', flags=re.IGNORECASE, regex=True)
+        0    False
+        1    False
+        2     True
+        3    False
+        4     None
+        dtype: object
+
+        Returning any digit using regular expression.
+
+        >>> s1.str.contains('[0-9]', regex=True)
+        0    False
+        1    False
+        2    False
+        3     True
+        4     None
+        dtype: object
+
+        Ensure pat is a not a literal pattern when regex is set to True.
+        Note in the following example one might expect only s2[1] and s2[3]
+        to return True. However, ‘.0’ as a regex matches any character followed
+        by a 0.
+
+        >>> s2 = ps.Series(['40','40.0','41','41.0','35'])
+        >>> s2.str.contains('.0', regex=True)
+        0     True
+        1     True
+        2    False
+        3     True
+        4    False
+        dtype: bool
+        """
+
+        def pandas_contains(s) -> "ps.Series[bool]":
+            return s.str.contains(pat, case, flags, na, regex)
+
+        return self._data.koalas.transform_batch(pandas_contains)
+
+    def count(self, pat, flags=0) -> "ps.Series":
+        """
+        Count occurrences of pattern in each string of the Series.
+
+        This function is used to count the number of times a particular regex
+        pattern is repeated in each of the string elements of the Series.
+
+        Parameters
+        ----------
+        pat : str
+            Valid regular expression.
+        flags : int, default 0 (no flags)
+            Flags for the re module.
+
+        Returns
+        -------
+        Series of int
+            A Series containing the integer counts of pattern matches.
+
+        Examples
+        --------
+        >>> s = ps.Series(['A', 'B', 'Aaba', 'Baca', np.NaN, 'CABA', 'cat'])
+        >>> s.str.count('a')
+        0    0.0
+        1    0.0
+        2    2.0
+        3    2.0
+        4    NaN
+        5    0.0
+        6    1.0
+        dtype: float64
+
+        Escape '$' to find the literal dollar sign.
+
+        >>> s = ps.Series(['$', 'B', 'Aab$', '$$ca', 'C$B$', 'cat'])
+        >>> s.str.count('\\$')
+        0    1
+        1    0
+        2    1
+        3    2
+        4    2
+        5    0
+        dtype: int64
+        """
+
+        def pandas_count(s) -> "ps.Series[int]":
+            return s.str.count(pat, flags)
+
+        return self._data.koalas.transform_batch(pandas_count)
+
+    def decode(self, encoding, errors="strict") -> "ps.Series":
+        """
+        Not supported.
+        """
+        raise NotImplementedError()
+
+    def encode(self, encoding, errors="strict") -> "ps.Series":
+        """
+        Not supported.
+        """
+        raise NotImplementedError()
+
+    def extract(self, pat, flags=0, expand=True) -> "ps.Series":
+        """
+        Not supported.
+        """
+        raise NotImplementedError()
+
+    def extractall(self, pat, flags=0) -> "ps.Series":
+        """
+        Not supported.
+        """
+        raise NotImplementedError()
+
+    def find(self, sub, start=0, end=None) -> "ps.Series":
+        """
+        Return lowest indexes in each strings in the Series where the
+        substring is fully contained between [start:end].
+
+        Return -1 on failure. Equivalent to standard :func:`str.find`.
+
+        Parameters
+        ----------
+        sub : str
+            Substring being searched.
+        start : int
+            Left edge index.
+        end : int
+            Right edge index.
+
+        Returns
+        -------
+        Series of int
+            Series of lowest matching indexes.
+
+        Examples
+        --------
+        >>> s = ps.Series(['apple', 'oranges', 'bananas'])
+
+        >>> s.str.find('a')
+        0    0
+        1    2
+        2    1
+        dtype: int64
+
+        >>> s.str.find('a', start=2)
+        0   -1
+        1    2
+        2    3
+        dtype: int64
+
+        >>> s.str.find('a', end=1)
+        0    0
+        1   -1
+        2   -1
+        dtype: int64
+
+        >>> s.str.find('a', start=2, end=2)
+        0   -1
+        1   -1
+        2   -1
+        dtype: int64
+        """
+
+        def pandas_find(s) -> "ps.Series[int]":
+            return s.str.find(sub, start, end)
+
+        return self._data.koalas.transform_batch(pandas_find)
+
+    def findall(self, pat, flags=0) -> "ps.Series":
+        """
+        Find all occurrences of pattern or regular expression in the Series.
+
+        Equivalent to applying :func:`re.findall` to all the elements in
+        the Series.
+
+        Parameters
+        ----------
+        pat : str
+            Pattern or regular expression.
+        flags : int, default 0 (no flags)
+            `re` module flags, e.g. `re.IGNORECASE`.
+
+        Returns
+        -------
+        Series of object
+            All non-overlapping matches of pattern or regular expression in
+            each string of this Series.
+
+        Examples
+        --------
+        >>> s = ps.Series(['Lion', 'Monkey', 'Rabbit'])
+
+        The search for the pattern ‘Monkey’ returns one match:
+
+        >>> s.str.findall('Monkey')
+        0          []
+        1    [Monkey]
+        2          []
+        dtype: object
+
+        On the other hand, the search for the pattern ‘MONKEY’ doesn’t return
+        any match:
+
+        >>> s.str.findall('MONKEY')
+        0    []
+        1    []
+        2    []
+        dtype: object
+
+        Flags can be added to the pattern or regular expression. For instance,
+        to find the pattern ‘MONKEY’ ignoring the case:
+
+        >>> import re
+        >>> s.str.findall('MONKEY', flags=re.IGNORECASE)
+        0          []
+        1    [Monkey]
+        2          []
+        dtype: object
+
+        When the pattern matches more than one string in the Series, all
+        matches are returned:
+
+        >>> s.str.findall('on')
+        0    [on]
+        1    [on]
+        2      []
+        dtype: object
+
+        Regular expressions are supported too. For instance, the search for all
+        the strings ending with the word ‘on’ is shown next:
+
+        >>> s.str.findall('on$')
+        0    [on]
+        1      []
+        2      []
+        dtype: object
+
+        If the pattern is found more than once in the same string, then a list
+        of multiple strings is returned:
+
+        >>> s.str.findall('b')
+        0        []
+        1        []
+        2    [b, b]
+        dtype: object
+        """
+        # type hint does not support to specify array type yet.
+        pudf = pandas_udf(
+            lambda s: s.str.findall(pat, flags),
+            returnType=ArrayType(StringType(), containsNull=True),
+            functionType=PandasUDFType.SCALAR,
+        )
+        return self._data._with_new_scol(scol=pudf(self._data.spark.column))
+
+    def index(self, sub, start=0, end=None) -> "ps.Series":
+        """
+        Return lowest indexes in each strings where the substring is fully
+        contained between [start:end].
+
+        This is the same as :func:`str.find` except instead of returning -1,
+        it raises a ValueError when the substring is not found. Equivalent to
+        standard :func:`str.index`.
+
+        Parameters
+        ----------
+        sub : str
+            Substring being searched.
+        start : int
+            Left edge index.
+        end : int
+            Right edge index.
+
+        Returns
+        -------
+        Series of int
+            Series of lowest matching indexes.
+
+        Examples
+        --------
+        >>> s = ps.Series(['apple', 'oranges', 'bananas'])
+
+        >>> s.str.index('a')
+        0    0
+        1    2
+        2    1
+        dtype: int64
+
+        The following expression throws an exception:
+
+        >>> s.str.index('a', start=2) # doctest: +SKIP
+        """
+
+        def pandas_index(s) -> "ps.Series[np.int64]":
+            return s.str.index(sub, start, end)
+
+        return self._data.koalas.transform_batch(pandas_index)
+
+    def join(self, sep) -> "ps.Series":
+        """
+        Join lists contained as elements in the Series with passed delimiter.
+
+        If the elements of a Series are lists themselves, join the content of
+        these lists using the delimiter passed to the function. This function
+        is an equivalent to calling :func:`str.join` on the lists.
+
+        Parameters
+        ----------
+        sep : str
+            Delimiter to use between list entries.
+
+        Returns
+        -------
+        Series of object
+            Series with list entries concatenated by intervening occurrences of
+            the delimiter.
+
+        See Also
+        --------
+        str.split : Split strings around given separator/delimiter.
+        str.rsplit : Splits string around given separator/delimiter,
+            starting from the right.
+
+        Examples
+        --------
+        Example with a list that contains a None element.
+
+        >>> s = ps.Series([['lion', 'elephant', 'zebra'],
+        ...                ['cat', None, 'dog']])
+        >>> s
+        0    [lion, elephant, zebra]
+        1           [cat, None, dog]
+        dtype: object
+
+        Join all lists using a ‘-‘. The list containing None will produce None.
+
+        >>> s.str.join('-')
+        0    lion-elephant-zebra
+        1                   None
+        dtype: object
+        """
+
+        def pandas_join(s) -> "ps.Series[str]":
+            return s.str.join(sep)
+
+        return self._data.koalas.transform_batch(pandas_join)
+
+    def len(self) -> "ps.Series":
+        """
+        Computes the length of each element in the Series.
+
+        The element may be a sequence (such as a string, tuple or list).
+
+        Returns
+        -------
+        Series of int
+            A Series of integer values indicating the length of each element in
+            the Series.
+
+        Examples
+        --------
+        Returns the length (number of characters) in a string. Returns the
+        number of entries for lists or tuples.
+
+        >>> s1 = ps.Series(['dog', 'monkey'])
+        >>> s1.str.len()
+        0    3
+        1    6
+        dtype: int64
+
+        >>> s2 = ps.Series([["a", "b", "c"], []])
+        >>> s2.str.len()
+        0    3
+        1    0
+        dtype: int64
+        """
+        if isinstance(self._data.spark.data_type, (ArrayType, MapType)):
+            return self._data.spark.transform(lambda c: F.size(c).cast(LongType()))
+        else:
+            return self._data.spark.transform(lambda c: F.length(c).cast(LongType()))
+
+    def ljust(self, width, fillchar=" ") -> "ps.Series":
+        """
+        Filling right side of strings in the Series with an additional
+        character. Equivalent to :func:`str.ljust`.
+
+        Parameters
+        ----------
+        width : int
+            Minimum width of resulting string; additional characters will be
+            filled with `fillchar`.
+        fillchar : str
+            Additional character for filling, default is whitespace.
+
+        Returns
+        -------
+        Series of object
+
+        Examples
+        --------
+        >>> s = ps.Series(["caribou", "tiger"])
+        >>> s
+        0    caribou
+        1      tiger
+        dtype: object
+
+        >>> s.str.ljust(width=10, fillchar='-')
+        0    caribou---
+        1    tiger-----
+        dtype: object
+        """
+
+        def pandas_ljust(s) -> "ps.Series[str]":
+            return s.str.ljust(width, fillchar)
+
+        return self._data.koalas.transform_batch(pandas_ljust)
+
+    def match(self, pat, case=True, flags=0, na=np.NaN) -> "ps.Series":
+        """
+        Determine if each string matches a regular expression.
+
+        Analogous to :func:`contains`, but more strict, relying on
+        :func:`re.match` instead of :func:`re.search`.
+
+        Parameters
+        ----------
+        pat : str
+            Character sequence or regular expression.
+        case : bool, default True
+            If True, case sensitive.
+        flags : int, default 0 (no flags)
+            Flags to pass through to the re module, e.g. re.IGNORECASE.
+        na : default NaN
+            Fill value for missing values.
+
+        Returns
+        -------
+        Series of boolean values or object
+            A Series of boolean values indicating whether the given pattern can
+            be matched in the string of each element of the Series.
+
+        Examples
+        --------
+        >>> s = ps.Series(['Mouse', 'dog', 'house and parrot', '23', np.NaN])
+        >>> s.str.match('dog')
+        0    False
+        1     True
+        2    False
+        3    False
+        4     None
+        dtype: object
+
+        >>> s.str.match('mouse|dog', case=False)
+        0     True
+        1     True
+        2    False
+        3    False
+        4     None
+        dtype: object
+
+        >>> s.str.match('.+and.+', na=True)
+        0    False
+        1    False
+        2     True
+        3    False
+        4     True
+        dtype: bool
+
+        >>> import re
+        >>> s.str.match('MOUSE', flags=re.IGNORECASE)
+        0     True
+        1    False
+        2    False
+        3    False
+        4     None
+        dtype: object
+        """
+
+        def pandas_match(s) -> "ps.Series[bool]":
+            return s.str.match(pat, case, flags, na)
+
+        return self._data.koalas.transform_batch(pandas_match)
+
+    def normalize(self, form) -> "ps.Series":
+        """
+        Return the Unicode normal form for the strings in the Series.
+
+        For more information on the forms, see the
+        :func:`unicodedata.normalize`.
+
+        Parameters
+        ----------
+        form : {‘NFC’, ‘NFKC’, ‘NFD’, ‘NFKD’}
+            Unicode form.
+
+        Returns
+        -------
+        Series of objects
+            A Series of normalized strings.
+        """
+
+        def pandas_normalize(s) -> "ps.Series[str]":
+            return s.str.normalize(form)
+
+        return self._data.koalas.transform_batch(pandas_normalize)
+
+    def pad(self, width, side="left", fillchar=" ") -> "ps.Series":
+        """
+        Pad strings in the Series up to width.
+
+        Parameters
+        ----------
+        width : int
+            Minimum width of resulting string; additional characters will be
+            filled with character defined in `fillchar`.
+        side : {‘left’, ‘right’, ‘both’}, default ‘left’
+            Side from which to fill resulting string.
+        fillchar : str, default ' '
+            Additional character for filling, default is whitespace.
+
+        Returns
+        -------
+        Series of object
+            Returns Series with minimum number of char in object.
+
+        Examples
+        --------
+        >>> s = ps.Series(["caribou", "tiger"])
+        >>> s
+        0    caribou
+        1      tiger
+        dtype: object
+
+        >>> s.str.pad(width=10)
+        0       caribou
+        1         tiger
+        dtype: object
+
+        >>> s.str.pad(width=10, side='right', fillchar='-')
+        0    caribou---
+        1    tiger-----
+        dtype: object
+
+        >>> s.str.pad(width=10, side='both', fillchar='-')
+        0    -caribou--
+        1    --tiger---
+        dtype: object
+        """
+
+        def pandas_pad(s) -> "ps.Series[str]":
+            return s.str.pad(width, side, fillchar)
+
+        return self._data.koalas.transform_batch(pandas_pad)
+
+    def partition(self, sep=" ", expand=True) -> "ps.Series":
+        """
+        Not supported.
+        """
+        raise NotImplementedError()
+
+    def repeat(self, repeats) -> "ps.Series":
+        """
+        Duplicate each string in the Series.
+
+        Parameters
+        ----------
+        repeats : int
+            Repeat the string given number of times (int). Sequence of int
+            is not supported.
+
+        Returns
+        -------
+        Series of object
+            Series or Index of repeated string objects specified by input
+            parameter repeats.
+
+        Examples
+        --------
+        >>> s = ps.Series(['a', 'b', 'c'])
+        >>> s
+        0    a
+        1    b
+        2    c
+        dtype: object
+
+        Single int repeats string in Series
+
+        >>> s.str.repeat(repeats=2)
+        0    aa
+        1    bb
+        2    cc
+        dtype: object
+        """
+        if not isinstance(repeats, int):
+            raise TypeError("repeats expects an int parameter")
+        return self._data.spark.transform(lambda c: SF.repeat(col=c, n=repeats))
+
+    def replace(self, pat, repl, n=-1, case=None, flags=0, regex=True) -> "ps.Series":
+        """
+        Replace occurrences of pattern/regex in the Series with some other
+        string. Equivalent to :func:`str.replace` or :func:`re.sub`.
+
+        Parameters
+        ----------
+        pat : str or compiled regex
+            String can be a character sequence or regular expression.
+        repl : str or callable
+            Replacement string or a callable. The callable is passed the regex
+            match object and must return a replacement string to be used. See
+            :func:`re.sub`.
+        n : int, default -1 (all)
+            Number of replacements to make from start.
+        case : boolean, default None
+            If True, case sensitive (the default if pat is a string).
+            Set to False for case insensitive.
+            Cannot be set if pat is a compiled regex.
+        flags: int, default 0 (no flags)
+            re module flags, e.g. re.IGNORECASE.
+            Cannot be set if pat is a compiled regex.
+        regex : boolean, default True
+            If True, assumes the passed-in pattern is a regular expression.
+            If False, treats the pattern as a literal string.
+            Cannot be set to False if pat is a compile regex or repl is a
+            callable.
+
+        Returns
+        -------
+        Series of object
+            A copy of the string with all matching occurrences of pat replaced
+            by repl.
+
+        Examples
+        --------
+        When pat is a string and regex is True (the default), the given pat is
+        compiled as a regex. When repl is a string, it replaces matching regex
+        patterns as with :func:`re.sub`. NaN value(s) in the Series are changed
+        to None:
+
+        >>> ps.Series(['foo', 'fuz', np.nan]).str.replace('f.', 'ba', regex=True)
+        0     bao
+        1     baz
+        2    None
+        dtype: object
+
+        When pat is a string and regex is False, every pat is replaced with
+        repl as with :func:`str.replace`:
+
+        >>> ps.Series(['f.o', 'fuz', np.nan]).str.replace('f.', 'ba', regex=False)
+        0     bao
+        1     fuz
+        2    None
+        dtype: object
+
+        When repl is a callable, it is called on every pat using
+        :func:`re.sub`. The callable should expect one positional argument (a
+        regex object) and return a string.
+
+        Reverse every lowercase alphabetic word:
+
+        >>> repl = lambda m: m.group(0)[::-1]
+        >>> ps.Series(['foo 123', 'bar baz', np.nan]).str.replace(r'[a-z]+', repl)
+        0    oof 123
+        1    rab zab
+        2       None
+        dtype: object
+
+        Using regex groups (extract second group and swap case):
+
+        >>> pat = r"(?P<one>\\w+) (?P<two>\\w+) (?P<three>\\w+)"
+        >>> repl = lambda m: m.group('two').swapcase()
+        >>> ps.Series(['One Two Three', 'Foo Bar Baz']).str.replace(pat, repl)
+        0    tWO
+        1    bAR
+        dtype: object
+
+        Using a compiled regex with flags:
+
+        >>> import re
+        >>> regex_pat = re.compile(r'FUZ', flags=re.IGNORECASE)
+        >>> ps.Series(['foo', 'fuz', np.nan]).str.replace(regex_pat, 'bar')
+        0     foo
+        1     bar
+        2    None
+        dtype: object
+        """
+
+        def pandas_replace(s) -> "ps.Series[str]":
+            return s.str.replace(pat, repl, n=n, case=case, flags=flags, regex=regex)
+
+        return self._data.koalas.transform_batch(pandas_replace)
+
+    def rfind(self, sub, start=0, end=None) -> "ps.Series":
+        """
+        Return highest indexes in each strings in the Series where the
+        substring is fully contained between [start:end].
+
+        Return -1 on failure. Equivalent to standard :func:`str.rfind`.
+
+        Parameters
+        ----------
+        sub : str
+            Substring being searched.
+        start : int
+            Left edge index.
+        end : int
+            Right edge index.
+
+        Returns
+        -------
+        Series of int
+            Series of highest matching indexes.
+
+        Examples
+        --------
+        >>> s = ps.Series(['apple', 'oranges', 'bananas'])
+
+        >>> s.str.rfind('a')
+        0    0
+        1    2
+        2    5
+        dtype: int64
+
+        >>> s.str.rfind('a', start=2)
+        0   -1
+        1    2
+        2    5
+        dtype: int64
+
+        >>> s.str.rfind('a', end=1)
+        0    0
+        1   -1
+        2   -1
+        dtype: int64
+
+        >>> s.str.rfind('a', start=2, end=2)
+        0   -1
+        1   -1
+        2   -1
+        dtype: int64
+        """
+
+        def pandas_rfind(s) -> "ps.Series[int]":
+            return s.str.rfind(sub, start, end)
+
+        return self._data.koalas.transform_batch(pandas_rfind)
+
+    def rindex(self, sub, start=0, end=None) -> "ps.Series":
+        """
+        Return highest indexes in each strings where the substring is fully
+        contained between [start:end].
+
+        This is the same as :func:`str.rfind` except instead of returning -1,
+        it raises a ValueError when the substring is not found. Equivalent to
+        standard :func:`str.rindex`.
+
+        Parameters
+        ----------
+        sub : str
+            Substring being searched.
+        start : int
+            Left edge index.
+        end : int
+            Right edge index.
+
+        Returns
+        -------
+        Series of int
+            Series of highest matching indexes.
+
+        Examples
+        --------
+        >>> s = ps.Series(['apple', 'oranges', 'bananas'])
+
+        >>> s.str.rindex('a')
+        0    0
+        1    2
+        2    5
+        dtype: int64
+
+        The following expression throws an exception:
+
+        >>> s.str.rindex('a', start=2) # doctest: +SKIP
+        """
+
+        def pandas_rindex(s) -> "ps.Series[np.int64]":
+            return s.str.rindex(sub, start, end)
+
+        return self._data.koalas.transform_batch(pandas_rindex)
+
+    def rjust(self, width, fillchar=" ") -> "ps.Series":
+        """
+        Filling left side of strings in the Series with an additional
+        character. Equivalent to :func:`str.rjust`.
+
+        Parameters
+        ----------
+        width : int
+            Minimum width of resulting string; additional characters will be
+            filled with `fillchar`.
+        fillchar : str
+            Additional character for filling, default is whitespace.
+
+        Returns
+        -------
+        Series of object
+
+        Examples
+        --------
+        >>> s = ps.Series(["caribou", "tiger"])
+        >>> s
+        0    caribou
+        1      tiger
+        dtype: object
+
+        >>> s.str.rjust(width=10)
+        0       caribou
+        1         tiger
+        dtype: object
+
+        >>> s.str.rjust(width=10, fillchar='-')
+        0    ---caribou
+        1    -----tiger
+        dtype: object
+        """
+
+        def pandas_rjust(s) -> "ps.Series[str]":
+            return s.str.rjust(width, fillchar)
+
+        return self._data.koalas.transform_batch(pandas_rjust)
+
+    def rpartition(self, sep=" ", expand=True) -> "ps.Series":
+        """
+        Not supported.
+        """
+        raise NotImplementedError()
+
+    def slice(self, start=None, stop=None, step=None) -> "ps.Series":
+        """
+        Slice substrings from each element in the Series.
+
+        Parameters
+        ----------
+        start : int, optional
+            Start position for slice operation.
+        stop : int, optional
+            Stop position for slice operation.
+        step : int, optional
+            Step size for slice operation.
+
+        Returns
+        -------
+        Series of object
+            Series from sliced substrings from original string objects.
+
+        Examples
+        --------
+        >>> s = ps.Series(["koala", "fox", "chameleon"])
+        >>> s
+        0        koala
+        1          fox
+        2    chameleon
+        dtype: object
+
+        >>> s.str.slice(start=1)
+        0        oala
+        1          ox
+        2    hameleon
+        dtype: object
+
+        >>> s.str.slice(stop=2)
+        0    ko
+        1    fo
+        2    ch
+        dtype: object
+
+        >>> s.str.slice(step=2)
+        0      kaa
+        1       fx
+        2    caeen
+        dtype: object
+
+        >>> s.str.slice(start=0, stop=5, step=3)
+        0    kl
+        1     f
+        2    cm
+        dtype: object
+        """
+
+        def pandas_slice(s) -> "ps.Series[str]":
+            return s.str.slice(start, stop, step)
+
+        return self._data.koalas.transform_batch(pandas_slice)
+
+    def slice_replace(self, start=None, stop=None, repl=None) -> "ps.Series":
+        """
+        Slice substrings from each element in the Series.
+
+        Parameters
+        ----------
+        start : int, optional
+            Start position for slice operation. If not specified (None), the
+            slice is unbounded on the left, i.e. slice from the start of the
+            string.
+        stop : int, optional
+            Stop position for slice operation. If not specified (None), the
+            slice is unbounded on the right, i.e. slice until the end of the
+            string.
+        repl : str, optional
+            String for replacement. If not specified (None), the sliced region
+            is replaced with an empty string.
+
+        Returns
+        -------
+        Series of object
+            Series from sliced substrings from original string objects.
+
+        Examples
+        --------
+        >>> s = ps.Series(['a', 'ab', 'abc', 'abdc', 'abcde'])
+        >>> s
+        0        a
+        1       ab
+        2      abc
+        3     abdc
+        4    abcde
+        dtype: object
+
+        Specify just start, meaning replace start until the end of the string
+        with repl.
+
+        >>> s.str.slice_replace(1, repl='X')
+        0    aX
+        1    aX
+        2    aX
+        3    aX
+        4    aX
+        dtype: object
+
+        Specify just stop, meaning the start of the string to stop is replaced
+        with repl, and the rest of the string is included.
+
+        >>> s.str.slice_replace(stop=2, repl='X')
+        0       X
+        1       X
+        2      Xc
+        3     Xdc
+        4    Xcde
+        dtype: object
+
+        Specify start and stop, meaning the slice from start to stop is
+        replaced with repl. Everything before or after start and stop is
+        included as is.
+
+        >>> s.str.slice_replace(start=1, stop=3, repl='X')
+        0      aX
+        1      aX
+        2      aX
+        3     aXc
+        4    aXde
+        dtype: object
+        """
+
+        def pandas_slice_replace(s) -> "ps.Series[str]":
+            return s.str.slice_replace(start, stop, repl)
+
+        return self._data.koalas.transform_batch(pandas_slice_replace)
+
+    def split(self, pat=None, n=-1, expand=False) -> Union["ps.Series", "ps.DataFrame"]:
+        """
+        Split strings around given separator/delimiter.
+
+        Splits the string in the Series from the beginning, at the specified
+        delimiter string. Equivalent to :func:`str.split`.
+
+        Parameters
+        ----------
+        pat : str, optional
+            String or regular expression to split on. If not specified, split
+            on whitespace.
+        n : int, default -1 (all)
+            Limit number of splits in output. None, 0 and -1 will be
+            interpreted as return all splits.
+        expand : bool, default False
+            Expand the splitted strings into separate columns.
+
+            * If ``True``, `n` must be a positive integer, and return DataFrame expanding
+              dimensionality.
+            * If ``False``, return Series, containing lists of strings.
+
+        Returns
+        -------
+        Series, DataFrame
+            Type matches caller unless `expand=True` (see Notes).
+
+        See Also
+        --------
+        str.rsplit : Splits string around given separator/delimiter,
+            starting from the right.
+        str.join : Join lists contained as elements in the Series/Index
+            with passed delimiter.
+
+        Notes
+        -----
+        The handling of the `n` keyword depends on the number of found splits:
+
+        - If found splits > `n`,  make first `n` splits only
+        - If found splits <= `n`, make all splits
+        - If for a certain row the number of found splits < `n`,
+          append `None` for padding up to `n` if ``expand=True``
+
+        If using ``expand=True``, Series callers return DataFrame objects with `n + 1` columns.
+
+        .. note:: Even if `n` is much larger than found splits, the number of columns does NOT
+            shrink unlike pandas.
+
+        Examples
+        --------
+        >>> s = ps.Series(["this is a regular sentence",
+        ...                "https://docs.python.org/3/tutorial/index.html",
+        ...                np.nan])
+
+        In the default setting, the string is split by whitespace.
+
+        >>> s.str.split()
+        0                   [this, is, a, regular, sentence]
+        1    [https://docs.python.org/3/tutorial/index.html]
+        2                                               None
+        dtype: object
+
+        Without the n parameter, the outputs of rsplit and split are identical.
+
+        >>> s.str.rsplit()
+        0                   [this, is, a, regular, sentence]
+        1    [https://docs.python.org/3/tutorial/index.html]
+        2                                               None
+        dtype: object
+
+        The n parameter can be used to limit the number of splits on the
+        delimiter. The outputs of split and rsplit are different.
+
+        >>> s.str.split(n=2)
+        0                     [this, is, a regular sentence]
+        1    [https://docs.python.org/3/tutorial/index.html]
+        2                                               None
+        dtype: object
+
+        >>> s.str.rsplit(n=2)
+        0                     [this is a, regular, sentence]
+        1    [https://docs.python.org/3/tutorial/index.html]
+        2                                               None
+        dtype: object
+
+        The pat parameter can be used to split by other characters.
+
+        >>> s.str.split(pat = "/")
+        0                         [this is a regular sentence]
+        1    [https:, , docs.python.org, 3, tutorial, index...
+        2                                                 None
+        dtype: object
+
+        When using ``expand=True``, the split elements will expand out into
+        separate columns. If NaN is present, it is propagated throughout
+        the columns during the split.
+
+        >>> s.str.split(n=4, expand=True)
+                                                       0     1     2        3         4
+        0                                           this    is     a  regular  sentence
+        1  https://docs.python.org/3/tutorial/index.html  None  None     None      None
+        2                                           None  None  None     None      None
+
+        For slightly more complex use cases like splitting the html document name
+        from a url, a combination of parameter settings can be used.
+
+        >>> s.str.rsplit("/", n=1, expand=True)
+                                            0           1
+        0          this is a regular sentence        None
+        1  https://docs.python.org/3/tutorial  index.html
+        2                                None        None
+
+        Remember to escape special characters when explicitly using regular
+        expressions.
+
+        >>> s = ps.Series(["1+1=2"])
+        >>> s.str.split(r"\\+|=", n=2, expand=True)
+           0  1  2
+        0  1  1  2
+        """
+        from pyspark.pandas.frame import DataFrame
+
+        if expand and n <= 0:
+            raise NotImplementedError("expand=True is currently only supported with n > 0.")
+
+        # type hint does not support to specify array type yet.
+        pudf = pandas_udf(
+            lambda s: s.str.split(pat, n),
+            returnType=ArrayType(StringType(), containsNull=True),
+            functionType=PandasUDFType.SCALAR,
+        )
+        kser = self._data._with_new_scol(pudf(self._data.spark.column), dtype=self._data.dtype)
+
+        if expand:
+            kdf = kser.to_frame()
+            scol = kdf._internal.data_spark_columns[0]
+            spark_columns = [scol[i].alias(str(i)) for i in range(n + 1)]
+            column_labels = [(i,) for i in range(n + 1)]
+            internal = kdf._internal.with_new_columns(
+                spark_columns,
+                column_labels=cast(Optional[List], column_labels),
+                data_dtypes=([self._data.dtype] * len(column_labels)),
+            )
+            return DataFrame(internal)
+        else:
+            return kser
+
+    def rsplit(self, pat=None, n=-1, expand=False) -> Union["ps.Series", "ps.DataFrame"]:
+        """
+        Split strings around given separator/delimiter.
+
+        Splits the string in the Series from the end, at the specified
+        delimiter string. Equivalent to :func:`str.rsplit`.
+
+        Parameters
+        ----------
+        pat : str, optional
+            String or regular expression to split on. If not specified, split
+            on whitespace.
+        n : int, default -1 (all)
+            Limit number of splits in output. None, 0 and -1 will be
+            interpreted as return all splits.
+        expand : bool, default False
+            Expand the splitted strings into separate columns.
+
+            * If ``True``, `n` must be a positive integer, and return DataFrame expanding
+              dimensionality.
+            * If ``False``, return Series, containing lists of strings.
+
+        Returns
+        -------
+        Series, DataFrame
+            Type matches caller unless `expand=True` (see Notes).
+
+        See Also
+        --------
+        str.split : Split strings around given separator/delimiter.
+        str.join : Join lists contained as elements in the Series/Index
+            with passed delimiter.
+
+        Notes
+        -----
+        The handling of the `n` keyword depends on the number of found splits:
+
+        - If found splits > `n`,  make first `n` splits only
+        - If found splits <= `n`, make all splits
+        - If for a certain row the number of found splits < `n`,
+          append `None` for padding up to `n` if ``expand=True``
+
+        If using ``expand=True``, Series callers return DataFrame objects with `n + 1` columns.
+
+        .. note:: Even if `n` is much larger than found splits, the number of columns does NOT
+            shrink unlike pandas.
+
+        Examples
+        --------
+        >>> s = ps.Series(["this is a regular sentence",
+        ...                "https://docs.python.org/3/tutorial/index.html",
+        ...                np.nan])
+
+        In the default setting, the string is split by whitespace.
+
+        >>> s.str.split()
+        0                   [this, is, a, regular, sentence]
+        1    [https://docs.python.org/3/tutorial/index.html]
+        2                                               None
+        dtype: object
+
+        Without the n parameter, the outputs of rsplit and split are identical.
+
+        >>> s.str.rsplit()
+        0                   [this, is, a, regular, sentence]
+        1    [https://docs.python.org/3/tutorial/index.html]
+        2                                               None
+        dtype: object
+
+        The n parameter can be used to limit the number of splits on the
+        delimiter. The outputs of split and rsplit are different.
+
+        >>> s.str.split(n=2)
+        0                     [this, is, a regular sentence]
+        1    [https://docs.python.org/3/tutorial/index.html]
+        2                                               None
+        dtype: object
+
+        >>> s.str.rsplit(n=2)
+        0                     [this is a, regular, sentence]
+        1    [https://docs.python.org/3/tutorial/index.html]
+        2                                               None
+        dtype: object
+
+        When using ``expand=True``, the split elements will expand out into
+        separate columns. If NaN is present, it is propagated throughout
+        the columns during the split.
+
+        >>> s.str.split(n=4, expand=True)
+                                                       0     1     2        3         4
+        0                                           this    is     a  regular  sentence
+        1  https://docs.python.org/3/tutorial/index.html  None  None     None      None
+        2                                           None  None  None     None      None
+
+        For slightly more complex use cases like splitting the html document name
+        from a url, a combination of parameter settings can be used.
+
+        >>> s.str.rsplit("/", n=1, expand=True)
+                                            0           1
+        0          this is a regular sentence        None
+        1  https://docs.python.org/3/tutorial  index.html
+        2                                None        None
+
+        Remember to escape special characters when explicitly using regular
+        expressions.
+
+        >>> s = ps.Series(["1+1=2"])
+        >>> s.str.split(r"\\+|=", n=2, expand=True)
+           0  1  2
+        0  1  1  2
+        """
+        from pyspark.pandas.frame import DataFrame
+
+        if expand and n <= 0:
+            raise NotImplementedError("expand=True is currently only supported with n > 0.")
+
+        # type hint does not support to specify array type yet.
+        pudf = pandas_udf(
+            lambda s: s.str.rsplit(pat, n),
+            returnType=ArrayType(StringType(), containsNull=True),
+            functionType=PandasUDFType.SCALAR,
+        )
+        kser = self._data._with_new_scol(pudf(self._data.spark.column), dtype=self._data.dtype)
+
+        if expand:
+            kdf = kser.to_frame()
+            scol = kdf._internal.data_spark_columns[0]
+            spark_columns = [scol[i].alias(str(i)) for i in range(n + 1)]
+            column_labels = [(i,) for i in range(n + 1)]
+            internal = kdf._internal.with_new_columns(
+                spark_columns,
+                column_labels=cast(Optional[List], column_labels),
+                data_dtypes=([self._data.dtype] * len(column_labels)),
+            )
+            return DataFrame(internal)
+        else:
+            return kser
+
+    def translate(self, table) -> "ps.Series":
+        """
+        Map all characters in the string through the given mapping table.
+        Equivalent to standard :func:`str.translate`.
+
+        Parameters
+        ----------
+        table : dict
+            Table is a mapping of Unicode ordinals to Unicode ordinals,
+            strings, or None. Unmapped characters are left untouched.
+            Characters mapped to None are deleted. :func:`str.maketrans` is a
+            helper function for making translation tables.
+
+        Returns
+        -------
+        Series of object
+            Series with translated strings.
+
+        Examples
+        --------
+        >>> s = ps.Series(["dog", "cat", "bird"])
+        >>> m = str.maketrans({'a': 'X', 'i': 'Y', 'o': None})
+        >>> s.str.translate(m)
+        0      dg
+        1     cXt
+        2    bYrd
+        dtype: object
+        """
+
+        def pandas_translate(s) -> "ps.Series[str]":
+            return s.str.translate(table)
+
+        return self._data.koalas.transform_batch(pandas_translate)
+
+    def wrap(self, width, **kwargs) -> "ps.Series":
+        """
+        Wrap long strings in the Series to be formatted in paragraphs with
+        length less than a given width.
+
+        This method has the same keyword parameters and defaults as
+        :class:`textwrap.TextWrapper`.
+
+        Parameters
+        ----------
+        width : int
+            Maximum line-width. Lines separated with newline char.
+        expand_tabs : bool, optional
+            If true, tab characters will be expanded to spaces (default: True).
+        replace_whitespace : bool, optional
+            If true, each whitespace character remaining after tab expansion
+            will be replaced by a single space (default: True).
+        drop_whitespace : bool, optional
+            If true, whitespace that, after wrapping, happens to end up at the
+            beginning or end of a line is dropped (default: True).
+        break_long_words : bool, optional
+            If true, then words longer than width will be broken in order to
+            ensure that no lines are longer than width. If it is false, long
+            words will not be broken, and some lines may be longer than width
+            (default: True).
+        break_on_hyphens : bool, optional
+            If true, wrapping will occur preferably on whitespace and right
+            after hyphens in compound words, as it is customary in English.
+            If false, only whitespaces will be considered as potentially good
+            places for line breaks, but you need to set break_long_words to
+            false if you want truly insecable words (default: True).
+
+        Returns
+        -------
+        Series of object
+            Series with wrapped strings.
+
+        Examples
+        --------
+        >>> s = ps.Series(['line to be wrapped', 'another line to be wrapped'])
+        >>> s.str.wrap(12)
+        0             line to be\\nwrapped
+        1    another line\\nto be\\nwrapped
+        dtype: object
+        """
+
+        def pandas_wrap(s) -> "ps.Series[str]":
+            return s.str.wrap(width, **kwargs)
+
+        return self._data.koalas.transform_batch(pandas_wrap)
+
+    def zfill(self, width) -> "ps.Series":
+        """
+        Pad strings in the Series by prepending ‘0’ characters.
+
+        Strings in the Series are padded with ‘0’ characters on the left of the
+        string to reach a total string length width. Strings in the Series with
+        length greater or equal to width are unchanged.
+
+        Differs from :func:`str.zfill` which has special handling for ‘+’/’-‘
+        in the string.
+
+        Parameters
+        ----------
+        width : int
+            Minimum length of resulting string; strings with length less than
+            width be prepended with ‘0’ characters.
+
+        Returns
+        -------
+        Series of object
+            Series with '0' left-padded strings.
+
+        Examples
+        --------
+        >>> s = ps.Series(['-1', '1', '1000', np.nan])
+        >>> s
+        0      -1
+        1       1
+        2    1000
+        3    None
+        dtype: object
+
+        Note that NaN is not a string, therefore it is converted to NaN. The
+        minus sign in '-1' is treated as a regular character and the zero is
+        added to the left of it (:func:`str.zfill` would have moved it to the
+        left). 1000 remains unchanged as it is longer than width.
+
+        >>> s.str.zfill(3)
+        0     0-1
+        1     001
+        2    1000
+        3    None
+        dtype: object
+        """
+
+        def pandas_zfill(s) -> "ps.Series[str]":
+            return s.str.zfill(width)
+
+        return self._data.koalas.transform_batch(pandas_zfill)
+
+    def get_dummies(self, sep="|"):
+        """
+        Not supported.
+        """
+        raise NotImplementedError()
+
+
+def _test():
+    import os
+    import doctest
+    import sys
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.strings
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.strings.__dict__.copy()
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]")
+        .appName("pyspark.pandas.strings tests")
+        .getOrCreate()
+    )
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.strings,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/tests/__init__.py b/python/pyspark/pandas/tests/__init__.py
new file mode 100644
index 0000000000000..cce3acad34a49
--- /dev/null
+++ b/python/pyspark/pandas/tests/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
diff --git a/python/pyspark/pandas/tests/indexes/__init__.py b/python/pyspark/pandas/tests/indexes/__init__.py
new file mode 100644
index 0000000000000..cce3acad34a49
--- /dev/null
+++ b/python/pyspark/pandas/tests/indexes/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
diff --git a/python/pyspark/pandas/tests/indexes/test_base.py b/python/pyspark/pandas/tests/indexes/test_base.py
new file mode 100644
index 0000000000000..87656c923924f
--- /dev/null
+++ b/python/pyspark/pandas/tests/indexes/test_base.py
@@ -0,0 +1,2288 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import inspect
+import unittest
+from distutils.version import LooseVersion
+from datetime import datetime
+
+import numpy as np
+import pandas as pd
+
+import pyspark.pandas as ps
+from pyspark.pandas.exceptions import PandasNotImplementedError
+from pyspark.pandas.missing.indexes import (
+    MissingPandasLikeCategoricalIndex,
+    MissingPandasLikeDatetimeIndex,
+    MissingPandasLikeIndex,
+    MissingPandasLikeMultiIndex,
+)
+from pyspark.testing.pandasutils import PandasOnSparkTestCase, TestUtils, SPARK_CONF_ARROW_ENABLED
+
+
+class IndexesTest(PandasOnSparkTestCase, TestUtils):
+    @property
+    def pdf(self):
+        return pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5, 6, 7, 8, 9], "b": [4, 5, 6, 3, 2, 1, 0, 0, 0]},
+            index=[0, 1, 3, 5, 6, 8, 9, 9, 9],
+        )
+
+    @property
+    def kdf(self):
+        return ps.from_pandas(self.pdf)
+
+    def test_index_basic(self):
+        for pdf in [
+            pd.DataFrame(np.random.randn(10, 5), index=np.random.randint(100, size=10)),
+            pd.DataFrame(
+                np.random.randn(10, 5), index=np.random.randint(100, size=10).astype(np.int32)
+            ),
+            pd.DataFrame(np.random.randn(10, 5), index=np.random.randn(10)),
+            pd.DataFrame(np.random.randn(10, 5), index=np.random.randn(10).astype(np.float32)),
+            pd.DataFrame(np.random.randn(10, 5), index=list("abcdefghij")),
+            pd.DataFrame(
+                np.random.randn(10, 5), index=pd.date_range("2011-01-01", freq="D", periods=10)
+            ),
+            pd.DataFrame(np.random.randn(10, 5), index=pd.Categorical(list("abcdefghij"))),
+            pd.DataFrame(np.random.randn(10, 5), columns=list("abcde")).set_index(["a", "b"]),
+        ]:
+            kdf = ps.from_pandas(pdf)
+            self.assert_eq(kdf.index, pdf.index)
+            self.assert_eq(type(kdf.index).__name__, type(pdf.index).__name__)
+
+    def test_index_from_series(self):
+        pser = pd.Series([1, 2, 3], name="a", index=[10, 20, 30])
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(ps.Index(kser), pd.Index(pser))
+        self.assert_eq(ps.Index(kser, dtype="float"), pd.Index(pser, dtype="float"))
+        self.assert_eq(ps.Index(kser, name="x"), pd.Index(pser, name="x"))
+
+        if LooseVersion(pd.__version__) >= LooseVersion("1.1"):
+            self.assert_eq(ps.Int64Index(kser), pd.Int64Index(pser))
+            self.assert_eq(ps.Float64Index(kser), pd.Float64Index(pser))
+        else:
+            self.assert_eq(ps.Int64Index(kser), pd.Int64Index(pser).rename("a"))
+            self.assert_eq(ps.Float64Index(kser), pd.Float64Index(pser).rename("a"))
+
+        pser = pd.Series([datetime(2021, 3, 1), datetime(2021, 3, 2)], name="x", index=[10, 20])
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(ps.Index(kser), pd.Index(pser))
+        self.assert_eq(ps.DatetimeIndex(kser), pd.DatetimeIndex(pser))
+
+    def test_index_from_index(self):
+        pidx = pd.Index([1, 2, 3], name="a")
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(ps.Index(kidx), pd.Index(pidx))
+        self.assert_eq(ps.Index(kidx, dtype="float"), pd.Index(pidx, dtype="float"))
+        self.assert_eq(ps.Index(kidx, name="x"), pd.Index(pidx, name="x"))
+
+        self.assert_eq(ps.Int64Index(kidx), pd.Int64Index(pidx))
+        self.assert_eq(ps.Float64Index(kidx), pd.Float64Index(pidx))
+
+        pidx = pd.DatetimeIndex(["2021-03-01", "2021-03-02"])
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(ps.Index(kidx), pd.Index(pidx))
+        self.assert_eq(ps.DatetimeIndex(kidx), pd.DatetimeIndex(pidx))
+
+    def test_index_getattr(self):
+        kidx = self.kdf.index
+        item = "databricks"
+
+        expected_error_message = "'.*Index' object has no attribute '{}'".format(item)
+        with self.assertRaisesRegex(AttributeError, expected_error_message):
+            kidx.__getattr__(item)
+        with self.assertRaisesRegex(AttributeError, expected_error_message):
+            ps.from_pandas(pd.date_range("2011-01-01", freq="D", periods=10)).__getattr__(item)
+
+    def test_multi_index_getattr(self):
+        arrays = [[1, 1, 2, 2], ["red", "blue", "red", "blue"]]
+        idx = pd.MultiIndex.from_arrays(arrays, names=("number", "color"))
+        pdf = pd.DataFrame(np.random.randn(4, 5), idx)
+        kdf = ps.from_pandas(pdf)
+        kidx = kdf.index
+        item = "databricks"
+
+        expected_error_message = "'MultiIndex' object has no attribute '{}'".format(item)
+        with self.assertRaisesRegex(AttributeError, expected_error_message):
+            kidx.__getattr__(item)
+
+    def test_to_series(self):
+        pidx = self.pdf.index
+        kidx = self.kdf.index
+
+        self.assert_eq(kidx.to_series(), pidx.to_series())
+        self.assert_eq(kidx.to_series(name="a"), pidx.to_series(name="a"))
+
+        # With name
+        pidx.name = "Koalas"
+        kidx.name = "Koalas"
+        self.assert_eq(kidx.to_series(), pidx.to_series())
+        self.assert_eq(kidx.to_series(name=("x", "a")), pidx.to_series(name=("x", "a")))
+
+        # With tupled name
+        pidx.name = ("x", "a")
+        kidx.name = ("x", "a")
+        self.assert_eq(kidx.to_series(), pidx.to_series())
+        self.assert_eq(kidx.to_series(name="a"), pidx.to_series(name="a"))
+
+        self.assert_eq((kidx + 1).to_series(), (pidx + 1).to_series())
+
+        pidx = self.pdf.set_index("b", append=True).index
+        kidx = self.kdf.set_index("b", append=True).index
+
+        with self.sql_conf({SPARK_CONF_ARROW_ENABLED: False}):
+            self.assert_eq(kidx.to_series(), pidx.to_series())
+            self.assert_eq(kidx.to_series(name="a"), pidx.to_series(name="a"))
+
+        expected_error_message = "Series.name must be a hashable type"
+        with self.assertRaisesRegex(TypeError, expected_error_message):
+            kidx.to_series(name=["x", "a"])
+
+    def test_to_frame(self):
+        pidx = self.pdf.index
+        kidx = self.kdf.index
+
+        self.assert_eq(kidx.to_frame(), pidx.to_frame())
+        self.assert_eq(kidx.to_frame(index=False), pidx.to_frame(index=False))
+
+        pidx.name = "a"
+        kidx.name = "a"
+
+        self.assert_eq(kidx.to_frame(), pidx.to_frame())
+        self.assert_eq(kidx.to_frame(index=False), pidx.to_frame(index=False))
+
+        if LooseVersion(pd.__version__) >= LooseVersion("0.24"):
+            # The `name` argument is added in pandas 0.24.
+            self.assert_eq(kidx.to_frame(name="x"), pidx.to_frame(name="x"))
+            self.assert_eq(
+                kidx.to_frame(index=False, name="x"), pidx.to_frame(index=False, name="x"),
+            )
+
+            self.assertRaises(TypeError, lambda: kidx.to_frame(name=["x"]))
+
+            # non-string name
+            self.assert_eq(kidx.to_frame(name=10), pidx.to_frame(name=10))
+            self.assert_eq(kidx.to_frame(name=("x", 10)), pidx.to_frame(name=("x", 10)))
+
+        pidx = self.pdf.set_index("b", append=True).index
+        kidx = self.kdf.set_index("b", append=True).index
+
+        self.assert_eq(kidx.to_frame(), pidx.to_frame())
+        self.assert_eq(kidx.to_frame(index=False), pidx.to_frame(index=False))
+
+        if LooseVersion(pd.__version__) >= LooseVersion("0.24"):
+            # The `name` argument is added in pandas 0.24.
+            self.assert_eq(kidx.to_frame(name=["x", "y"]), pidx.to_frame(name=["x", "y"]))
+            self.assert_eq(kidx.to_frame(name=("x", "y")), pidx.to_frame(name=("x", "y")))
+            self.assert_eq(
+                kidx.to_frame(index=False, name=["x", "y"]),
+                pidx.to_frame(index=False, name=["x", "y"]),
+            )
+
+            self.assertRaises(TypeError, lambda: kidx.to_frame(name="x"))
+            self.assertRaises(ValueError, lambda: kidx.to_frame(name=["x"]))
+
+            # non-string names
+            self.assert_eq(kidx.to_frame(name=[10, 20]), pidx.to_frame(name=[10, 20]))
+            self.assert_eq(kidx.to_frame(name=("x", 10)), pidx.to_frame(name=("x", 10)))
+            self.assert_eq(
+                kidx.to_frame(name=[("x", 10), ("y", 20)]),
+                pidx.to_frame(name=[("x", 10), ("y", 20)]),
+            )
+
+    def test_index_names(self):
+        kdf = self.kdf
+        self.assertIsNone(kdf.index.name)
+
+        idx = pd.Index([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], name="x")
+        pdf = pd.DataFrame(np.random.randn(10, 5), index=idx, columns=list("abcde"))
+        kdf = ps.from_pandas(pdf)
+
+        pser = pdf.a
+        kser = kdf.a
+
+        self.assertEqual(kdf.index.name, pdf.index.name)
+        self.assertEqual(kdf.index.names, pdf.index.names)
+
+        pidx = pdf.index
+        kidx = kdf.index
+        pidx.name = "renamed"
+        kidx.name = "renamed"
+        self.assertEqual(kidx.name, pidx.name)
+        self.assertEqual(kidx.names, pidx.names)
+        self.assert_eq(kidx, pidx)
+        self.assertEqual(kdf.index.name, pdf.index.name)
+        self.assertEqual(kdf.index.names, pdf.index.names)
+        self.assertEqual(kser.index.names, pser.index.names)
+
+        pidx.name = None
+        kidx.name = None
+        self.assertEqual(kidx.name, pidx.name)
+        self.assertEqual(kidx.names, pidx.names)
+        self.assert_eq(kidx, pidx)
+        self.assertEqual(kdf.index.name, pdf.index.name)
+        self.assertEqual(kdf.index.names, pdf.index.names)
+        self.assertEqual(kser.index.names, pser.index.names)
+
+        with self.assertRaisesRegex(ValueError, "Names must be a list-like"):
+            kidx.names = "hi"
+
+        expected_error_message = "Length of new names must be {}, got {}".format(
+            kdf._internal.index_level, len(["0", "1"])
+        )
+        with self.assertRaisesRegex(ValueError, expected_error_message):
+            kidx.names = ["0", "1"]
+
+        expected_error_message = "Index.name must be a hashable type"
+        with self.assertRaisesRegex(TypeError, expected_error_message):
+            ps.Index([1, 2, 3], name=["0", "1"])
+        with self.assertRaisesRegex(TypeError, expected_error_message):
+            kidx.name = ["renamed"]
+        with self.assertRaisesRegex(TypeError, expected_error_message):
+            kidx.name = ["0", "1"]
+        with self.assertRaisesRegex(TypeError, expected_error_message):
+            ps.Index([(1, 2), (3, 4)], names=["a", ["b"]])
+
+    def test_multi_index_names(self):
+        arrays = [[1, 1, 2, 2], ["red", "blue", "red", "blue"]]
+        idx = pd.MultiIndex.from_arrays(arrays, names=("number", "color"))
+        pdf = pd.DataFrame(np.random.randn(4, 5), idx)
+        kdf = ps.from_pandas(pdf)
+
+        self.assertEqual(kdf.index.names, pdf.index.names)
+
+        pidx = pdf.index
+        kidx = kdf.index
+        pidx.names = ["renamed_number", "renamed_color"]
+        kidx.names = ["renamed_number", "renamed_color"]
+        self.assertEqual(kidx.names, pidx.names)
+
+        pidx.names = ["renamed_number", None]
+        kidx.names = ["renamed_number", None]
+        self.assertEqual(kidx.names, pidx.names)
+        self.assert_eq(kidx, pidx)
+
+        with self.assertRaises(PandasNotImplementedError):
+            kidx.name
+        with self.assertRaises(PandasNotImplementedError):
+            kidx.name = "renamed"
+
+    def test_index_rename(self):
+        pdf = pd.DataFrame(
+            np.random.randn(10, 5), index=pd.Index([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], name="x")
+        )
+        kdf = ps.from_pandas(pdf)
+
+        pidx = pdf.index
+        kidx = kdf.index
+
+        self.assert_eq(kidx.rename("y"), pidx.rename("y"))
+        self.assert_eq(kdf.index.names, pdf.index.names)
+
+        # non-string names
+        self.assert_eq(kidx.rename(0), pidx.rename(0))
+        self.assert_eq(kidx.rename(("y", 0)), pidx.rename(("y", 0)))
+
+        kidx.rename("z", inplace=True)
+        pidx.rename("z", inplace=True)
+
+        self.assert_eq(kidx, pidx)
+        self.assert_eq(kdf.index.names, pdf.index.names)
+
+        self.assert_eq(kidx.rename(None), pidx.rename(None))
+        self.assert_eq(kdf.index.names, pdf.index.names)
+
+        self.assertRaises(TypeError, lambda: kidx.rename(["x", "y"]))
+
+    def test_multi_index_rename(self):
+        arrays = [[1, 1, 2, 2], ["red", "blue", "red", "blue"]]
+        idx = pd.MultiIndex.from_arrays(arrays, names=("number", "color"))
+        pdf = pd.DataFrame(np.random.randn(4, 5), idx)
+        kdf = ps.from_pandas(pdf)
+
+        pmidx = pdf.index
+        kmidx = kdf.index
+
+        self.assert_eq(kmidx.rename(["n", "c"]), pmidx.rename(["n", "c"]))
+        self.assert_eq(kdf.index.names, pdf.index.names)
+
+        # non-string names
+        self.assert_eq(kmidx.rename([0, 1]), pmidx.rename([0, 1]))
+        self.assert_eq(
+            kmidx.rename([("x", "a"), ("y", "b")]), pmidx.rename([("x", "a"), ("y", "b")])
+        )
+
+        kmidx.rename(["num", "col"], inplace=True)
+        pmidx.rename(["num", "col"], inplace=True)
+
+        self.assert_eq(kmidx, pmidx)
+        self.assert_eq(kdf.index.names, pdf.index.names)
+
+        self.assert_eq(kmidx.rename([None, None]), pmidx.rename([None, None]))
+        self.assert_eq(kdf.index.names, pdf.index.names)
+
+        self.assertRaises(TypeError, lambda: kmidx.rename("number"))
+        self.assertRaises(TypeError, lambda: kmidx.rename(None))
+        self.assertRaises(ValueError, lambda: kmidx.rename(["number"]))
+
+    def test_multi_index_levshape(self):
+        pidx = pd.MultiIndex.from_tuples([("a", "x", 1), ("b", "y", 2)])
+        kidx = ps.from_pandas(pidx)
+        self.assertEqual(pidx.levshape, kidx.levshape)
+
+    def test_index_unique(self):
+        kidx = self.kdf.index
+
+        # here the output is different than pandas in terms of order
+        expected = [0, 1, 3, 5, 6, 8, 9]
+
+        self.assert_eq(expected, sorted(kidx.unique().to_pandas()))
+        self.assert_eq(expected, sorted(kidx.unique(level=0).to_pandas()))
+
+        expected = [1, 2, 4, 6, 7, 9, 10]
+        self.assert_eq(expected, sorted((kidx + 1).unique().to_pandas()))
+
+        with self.assertRaisesRegex(IndexError, "Too many levels*"):
+            kidx.unique(level=1)
+
+        with self.assertRaisesRegex(KeyError, "Requested level (hi)*"):
+            kidx.unique(level="hi")
+
+    def test_multi_index_copy(self):
+        arrays = [[1, 1, 2, 2], ["red", "blue", "red", "blue"]]
+        idx = pd.MultiIndex.from_arrays(arrays, names=("number", "color"))
+        pdf = pd.DataFrame(np.random.randn(4, 5), idx)
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.index.copy(), pdf.index.copy())
+
+    def test_drop_duplicates(self):
+        pidx = pd.Index([4, 2, 4, 1, 4, 3])
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(kidx.drop_duplicates().sort_values(), pidx.drop_duplicates().sort_values())
+        self.assert_eq(
+            (kidx + 1).drop_duplicates().sort_values(), (pidx + 1).drop_duplicates().sort_values()
+        )
+
+    def test_dropna(self):
+        pidx = pd.Index([np.nan, 2, 4, 1, np.nan, 3])
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(kidx.dropna(), pidx.dropna())
+        self.assert_eq((kidx + 1).dropna(), (pidx + 1).dropna())
+
+    def test_index_symmetric_difference(self):
+        pidx1 = pd.Index([1, 2, 3, 4])
+        pidx2 = pd.Index([2, 3, 4, 5])
+        kidx1 = ps.from_pandas(pidx1)
+        kidx2 = ps.from_pandas(pidx2)
+
+        self.assert_eq(
+            kidx1.symmetric_difference(kidx2).sort_values(),
+            pidx1.symmetric_difference(pidx2).sort_values(),
+        )
+        self.assert_eq(
+            (kidx1 + 1).symmetric_difference(kidx2).sort_values(),
+            (pidx1 + 1).symmetric_difference(pidx2).sort_values(),
+        )
+
+        pmidx1 = pd.MultiIndex(
+            [["lama", "cow", "falcon"], ["speed", "weight", "length"]],
+            [[0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 0, 0, 0, 1, 2, 0, 1, 2]],
+        )
+        pmidx2 = pd.MultiIndex(
+            [["koalas", "cow", "falcon"], ["speed", "weight", "length"]],
+            [[0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 0, 0, 0, 1, 2, 0, 1, 2]],
+        )
+        kmidx1 = ps.from_pandas(pmidx1)
+        kmidx2 = ps.from_pandas(pmidx2)
+
+        self.assert_eq(
+            kmidx1.symmetric_difference(kmidx2).sort_values(),
+            pmidx1.symmetric_difference(pmidx2).sort_values(),
+        )
+
+        idx = ps.Index(["a", "b", "c"])
+        midx = ps.MultiIndex.from_tuples([("a", "x"), ("b", "y"), ("c", "z")])
+
+        with self.assertRaisesRegex(NotImplementedError, "Doesn't support*"):
+            idx.symmetric_difference(midx)
+
+    def test_multi_index_symmetric_difference(self):
+        idx = ps.Index(["a", "b", "c"])
+        midx = ps.MultiIndex.from_tuples([("a", "x"), ("b", "y"), ("c", "z")])
+        midx_ = ps.MultiIndex.from_tuples([("a", "x"), ("b", "y"), ("c", "z")])
+
+        self.assert_eq(
+            midx.symmetric_difference(midx_),
+            midx.to_pandas().symmetric_difference(midx_.to_pandas()),
+        )
+
+        with self.assertRaisesRegex(NotImplementedError, "Doesn't support*"):
+            midx.symmetric_difference(idx)
+
+    def test_missing(self):
+        kdf = ps.DataFrame(
+            {
+                "a": [1, 2, 3],
+                "b": [4, 5, 6],
+                "c": pd.date_range("2011-01-01", freq="D", periods=3),
+                "d": pd.Categorical(["a", "b", "c"]),
+            }
+        )
+
+        # Index functions
+        missing_functions = inspect.getmembers(MissingPandasLikeIndex, inspect.isfunction)
+        unsupported_functions = [
+            name for (name, type_) in missing_functions if type_.__name__ == "unsupported_function"
+        ]
+        for name in unsupported_functions:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "method.*Index.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.set_index("a").index, name)()
+
+        deprecated_functions = [
+            name for (name, type_) in missing_functions if type_.__name__ == "deprecated_function"
+        ]
+        for name in deprecated_functions:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "method.*Index.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.set_index("a").index, name)()
+
+        # MultiIndex functions
+        missing_functions = inspect.getmembers(MissingPandasLikeMultiIndex, inspect.isfunction)
+        unsupported_functions = [
+            name for (name, type_) in missing_functions if type_.__name__ == "unsupported_function"
+        ]
+        for name in unsupported_functions:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "method.*Index.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.set_index(["a", "b"]).index, name)()
+
+        deprecated_functions = [
+            name for (name, type_) in missing_functions if type_.__name__ == "deprecated_function"
+        ]
+        for name in deprecated_functions:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "method.*Index.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.set_index(["a", "b"]).index, name)()
+
+        # DatetimeIndex functions
+        missing_functions = inspect.getmembers(MissingPandasLikeDatetimeIndex, inspect.isfunction)
+        unsupported_functions = [
+            name for (name, type_) in missing_functions if type_.__name__ == "unsupported_function"
+        ]
+        for name in unsupported_functions:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "method.*Index.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.set_index("c").index, name)()
+
+        deprecated_functions = [
+            name for (name, type_) in missing_functions if type_.__name__ == "deprecated_function"
+        ]
+        for name in deprecated_functions:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "method.*Index.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.set_index("c").index, name)()
+
+        # CategoricalIndex functions
+        missing_functions = inspect.getmembers(
+            MissingPandasLikeCategoricalIndex, inspect.isfunction
+        )
+        unsupported_functions = [
+            name for (name, type_) in missing_functions if type_.__name__ == "unsupported_function"
+        ]
+        for name in unsupported_functions:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "method.*Index.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.set_index("d").index, name)()
+
+        deprecated_functions = [
+            name for (name, type_) in missing_functions if type_.__name__ == "deprecated_function"
+        ]
+        for name in deprecated_functions:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "method.*Index.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.set_index("d").index, name)()
+
+        # Index properties
+        missing_properties = inspect.getmembers(
+            MissingPandasLikeIndex, lambda o: isinstance(o, property)
+        )
+        unsupported_properties = [
+            name
+            for (name, type_) in missing_properties
+            if type_.fget.__name__ == "unsupported_property"
+        ]
+        for name in unsupported_properties:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "property.*Index.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.set_index("a").index, name)
+
+        deprecated_properties = [
+            name
+            for (name, type_) in missing_properties
+            if type_.fget.__name__ == "deprecated_property"
+        ]
+        for name in deprecated_properties:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "property.*Index.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.set_index("a").index, name)
+
+        # MultiIndex properties
+        missing_properties = inspect.getmembers(
+            MissingPandasLikeMultiIndex, lambda o: isinstance(o, property)
+        )
+        unsupported_properties = [
+            name
+            for (name, type_) in missing_properties
+            if type_.fget.__name__ == "unsupported_property"
+        ]
+        for name in unsupported_properties:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "property.*Index.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.set_index(["a", "b"]).index, name)
+
+        deprecated_properties = [
+            name
+            for (name, type_) in missing_properties
+            if type_.fget.__name__ == "deprecated_property"
+        ]
+        for name in deprecated_properties:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "property.*Index.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.set_index(["a", "b"]).index, name)
+
+        # DatetimeIndex properties
+        missing_properties = inspect.getmembers(
+            MissingPandasLikeDatetimeIndex, lambda o: isinstance(o, property)
+        )
+        unsupported_properties = [
+            name
+            for (name, type_) in missing_properties
+            if type_.fget.__name__ == "unsupported_property"
+        ]
+        for name in unsupported_properties:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "property.*Index.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.set_index("c").index, name)
+
+        # CategoricalIndex properties
+        missing_properties = inspect.getmembers(
+            MissingPandasLikeCategoricalIndex, lambda o: isinstance(o, property)
+        )
+        unsupported_properties = [
+            name
+            for (name, type_) in missing_properties
+            if type_.fget.__name__ == "unsupported_property"
+        ]
+        for name in unsupported_properties:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "property.*Index.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.set_index("d").index, name)
+
+    def test_index_has_duplicates(self):
+        indexes = [("a", "b", "c"), ("a", "a", "c"), (1, 3, 3), (1, 2, 3)]
+        names = [None, "ks", "ks", None]
+        has_dup = [False, True, True, False]
+
+        for idx, name, expected in zip(indexes, names, has_dup):
+            pdf = pd.DataFrame({"a": [1, 2, 3]}, index=pd.Index(idx, name=name))
+            kdf = ps.from_pandas(pdf)
+
+            self.assertEqual(kdf.index.has_duplicates, expected)
+
+    def test_multiindex_has_duplicates(self):
+        indexes = [
+            [list("abc"), list("edf")],
+            [list("aac"), list("edf")],
+            [list("aac"), list("eef")],
+            [[1, 4, 4], [4, 6, 6]],
+        ]
+        has_dup = [False, False, True, True]
+
+        for idx, expected in zip(indexes, has_dup):
+            pdf = pd.DataFrame({"a": [1, 2, 3]}, index=idx)
+            kdf = ps.from_pandas(pdf)
+
+            self.assertEqual(kdf.index.has_duplicates, expected)
+
+    def test_multi_index_not_supported(self):
+        kdf = ps.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6], "c": [7, 8, 9]})
+
+        with self.assertRaisesRegex(TypeError, "cannot perform any with this index type"):
+            kdf.set_index(["a", "b"]).index.any()
+
+        with self.assertRaisesRegex(TypeError, "cannot perform all with this index type"):
+            kdf.set_index(["a", "b"]).index.all()
+
+    def test_index_nlevels(self):
+        pdf = pd.DataFrame({"a": [1, 2, 3]}, index=pd.Index(["a", "b", "c"]))
+        kdf = ps.from_pandas(pdf)
+
+        self.assertEqual(kdf.index.nlevels, 1)
+
+    def test_multiindex_nlevel(self):
+        pdf = pd.DataFrame({"a": [1, 2, 3]}, index=[list("abc"), list("def")])
+        kdf = ps.from_pandas(pdf)
+
+        self.assertEqual(kdf.index.nlevels, 2)
+
+    def test_multiindex_from_arrays(self):
+        arrays = [["a", "a", "b", "b"], ["red", "blue", "red", "blue"]]
+        pidx = pd.MultiIndex.from_arrays(arrays)
+        kidx = ps.MultiIndex.from_arrays(arrays)
+
+        self.assert_eq(pidx, kidx)
+
+    def test_multiindex_swaplevel(self):
+        pidx = pd.MultiIndex.from_arrays([["a", "b"], [1, 2]])
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.swaplevel(0, 1), kidx.swaplevel(0, 1))
+
+        pidx = pd.MultiIndex.from_arrays([["a", "b"], [1, 2]], names=["word", "number"])
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.swaplevel(0, 1), kidx.swaplevel(0, 1))
+
+        pidx = pd.MultiIndex.from_arrays([["a", "b"], [1, 2]], names=["word", None])
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.swaplevel(-2, -1), kidx.swaplevel(-2, -1))
+        self.assert_eq(pidx.swaplevel(0, 1), kidx.swaplevel(0, 1))
+        self.assert_eq(pidx.swaplevel("word", 1), kidx.swaplevel("word", 1))
+
+        with self.assertRaisesRegex(IndexError, "Too many levels: Index"):
+            kidx.swaplevel(-3, "word")
+        with self.assertRaisesRegex(IndexError, "Too many levels: Index"):
+            kidx.swaplevel(0, 2)
+        with self.assertRaisesRegex(IndexError, "Too many levels: Index"):
+            kidx.swaplevel(0, -3)
+        with self.assertRaisesRegex(KeyError, "Level work not found"):
+            kidx.swaplevel(0, "work")
+
+    def test_multiindex_droplevel(self):
+        pidx = pd.MultiIndex.from_tuples(
+            [("a", "x", 1), ("b", "y", 2)], names=["level1", "level2", "level3"]
+        )
+        kidx = ps.from_pandas(pidx)
+        with self.assertRaisesRegex(IndexError, "Too many levels: Index has only 3 levels, not 5"):
+            kidx.droplevel(4)
+
+        with self.assertRaisesRegex(KeyError, "Level level4 not found"):
+            kidx.droplevel("level4")
+
+        with self.assertRaisesRegex(KeyError, "Level.*level3.*level4.*not found"):
+            kidx.droplevel([("level3", "level4")])
+
+        with self.assertRaisesRegex(
+            ValueError,
+            "Cannot remove 4 levels from an index with 3 levels: at least one "
+            "level must be left.",
+        ):
+            kidx.droplevel([0, 0, 1, 2])
+
+        with self.assertRaisesRegex(
+            ValueError,
+            "Cannot remove 3 levels from an index with 3 levels: at least one "
+            "level must be left.",
+        ):
+            kidx.droplevel([0, 1, 2])
+
+        self.assert_eq(pidx.droplevel(0), kidx.droplevel(0))
+        self.assert_eq(pidx.droplevel([0, 1]), kidx.droplevel([0, 1]))
+        self.assert_eq(pidx.droplevel((0, 1)), kidx.droplevel((0, 1)))
+        self.assert_eq(pidx.droplevel([0, "level2"]), kidx.droplevel([0, "level2"]))
+        self.assert_eq(pidx.droplevel((0, "level2")), kidx.droplevel((0, "level2")))
+
+        # non-string names
+        pidx = pd.MultiIndex.from_tuples([("a", "x", 1), ("b", "y", 2)], names=[1.0, 2.0, 3.0])
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.droplevel(1.0), kidx.droplevel(1.0))
+        self.assert_eq(pidx.droplevel([0, 2.0]), kidx.droplevel([0, 2.0]))
+
+    def test_index_fillna(self):
+        pidx = pd.Index([1, 2, None])
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(pidx.fillna(0), kidx.fillna(0), almost=True)
+        self.assert_eq(pidx.rename("name").fillna(0), kidx.rename("name").fillna(0), almost=True)
+
+        with self.assertRaisesRegex(TypeError, "Unsupported type list"):
+            kidx.fillna([1, 2])
+
+    def test_index_drop(self):
+        pidx = pd.Index([1, 2, 3])
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(pidx.drop(1), kidx.drop(1))
+        self.assert_eq(pidx.drop([1, 2]), kidx.drop([1, 2]))
+        self.assert_eq((pidx + 1).drop([2, 3]), (kidx + 1).drop([2, 3]))
+
+    def test_multiindex_drop(self):
+        pidx = pd.MultiIndex.from_tuples(
+            [("a", "x"), ("b", "y"), ("c", "z")], names=["level1", "level2"]
+        )
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.drop("a"), kidx.drop("a"))
+        self.assert_eq(pidx.drop(["a", "b"]), kidx.drop(["a", "b"]))
+        self.assert_eq(pidx.drop(["x", "y"], level=1), kidx.drop(["x", "y"], level=1))
+        self.assert_eq(pidx.drop(["x", "y"], level="level2"), kidx.drop(["x", "y"], level="level2"))
+
+        pidx.names = ["lv1", "lv2"]
+        kidx.names = ["lv1", "lv2"]
+        self.assert_eq(pidx.drop(["x", "y"], level="lv2"), kidx.drop(["x", "y"], level="lv2"))
+
+        self.assertRaises(IndexError, lambda: kidx.drop(["a", "b"], level=2))
+        self.assertRaises(KeyError, lambda: kidx.drop(["a", "b"], level="level"))
+
+        kidx.names = ["lv", "lv"]
+        self.assertRaises(ValueError, lambda: kidx.drop(["x", "y"], level="lv"))
+
+    def test_sort_values(self):
+        pidx = pd.Index([-10, -100, 200, 100])
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(pidx.sort_values(), kidx.sort_values())
+        self.assert_eq(pidx.sort_values(ascending=False), kidx.sort_values(ascending=False))
+
+        pidx.name = "koalas"
+        kidx.name = "koalas"
+
+        self.assert_eq(pidx.sort_values(), kidx.sort_values())
+        self.assert_eq(pidx.sort_values(ascending=False), kidx.sort_values(ascending=False))
+
+        pidx = pd.MultiIndex.from_tuples([("a", "x", 1), ("b", "y", 2), ("c", "z", 3)])
+        kidx = ps.from_pandas(pidx)
+
+        pidx.names = ["hello", "koalas", "goodbye"]
+        kidx.names = ["hello", "koalas", "goodbye"]
+
+        self.assert_eq(pidx.sort_values(), kidx.sort_values())
+        self.assert_eq(pidx.sort_values(ascending=False), kidx.sort_values(ascending=False))
+
+    def test_index_drop_duplicates(self):
+        pidx = pd.Index([1, 1, 2])
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.drop_duplicates().sort_values(), kidx.drop_duplicates().sort_values())
+
+        pidx = pd.MultiIndex.from_tuples([(1, 1), (1, 1), (2, 2)], names=["level1", "level2"])
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.drop_duplicates().sort_values(), kidx.drop_duplicates().sort_values())
+
+    def test_index_sort(self):
+        idx = ps.Index([1, 2, 3, 4, 5])
+        midx = ps.MultiIndex.from_tuples([("a", "x", 1), ("b", "y", 2)])
+
+        with self.assertRaisesRegex(
+            TypeError, "cannot sort an Index object in-place, use sort_values instead"
+        ):
+            idx.sort()
+        with self.assertRaisesRegex(
+            TypeError, "cannot sort an Index object in-place, use sort_values instead"
+        ):
+            midx.sort()
+
+    def test_multiindex_isna(self):
+        kidx = ps.MultiIndex.from_tuples([("a", "x", 1), ("b", "y", 2), ("c", "z", 3)])
+
+        with self.assertRaisesRegex(NotImplementedError, "isna is not defined for MultiIndex"):
+            kidx.isna()
+
+        with self.assertRaisesRegex(NotImplementedError, "isna is not defined for MultiIndex"):
+            kidx.isnull()
+
+        with self.assertRaisesRegex(NotImplementedError, "notna is not defined for MultiIndex"):
+            kidx.notna()
+
+        with self.assertRaisesRegex(NotImplementedError, "notna is not defined for MultiIndex"):
+            kidx.notnull()
+
+    def test_index_nunique(self):
+        pidx = pd.Index([1, 1, 2, None])
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(pidx.nunique(), kidx.nunique())
+        self.assert_eq(pidx.nunique(dropna=True), kidx.nunique(dropna=True))
+
+    def test_multiindex_nunique(self):
+        kidx = ps.MultiIndex.from_tuples([("a", "x", 1), ("b", "y", 2), ("c", "z", 3)])
+        with self.assertRaisesRegex(NotImplementedError, "notna is not defined for MultiIndex"):
+            kidx.notnull()
+
+    def test_multiindex_rename(self):
+        pidx = pd.MultiIndex.from_tuples([("a", "x", 1), ("b", "y", 2), ("c", "z", 3)])
+        kidx = ps.from_pandas(pidx)
+
+        pidx = pidx.rename(list("ABC"))
+        kidx = kidx.rename(list("ABC"))
+        self.assert_eq(pidx, kidx)
+
+        pidx = pidx.rename(["my", "name", "is"])
+        kidx = kidx.rename(["my", "name", "is"])
+        self.assert_eq(pidx, kidx)
+
+    def test_multiindex_set_names(self):
+        pidx = pd.MultiIndex.from_tuples([("a", "x", 1), ("b", "y", 2), ("c", "z", 3)])
+        kidx = ps.from_pandas(pidx)
+
+        pidx = pidx.set_names(["set", "new", "names"])
+        kidx = kidx.set_names(["set", "new", "names"])
+        self.assert_eq(pidx, kidx)
+
+        pidx.set_names(["set", "new", "names"], inplace=True)
+        kidx.set_names(["set", "new", "names"], inplace=True)
+        self.assert_eq(pidx, kidx)
+
+        pidx = pidx.set_names("first", level=0)
+        kidx = kidx.set_names("first", level=0)
+        self.assert_eq(pidx, kidx)
+
+        pidx = pidx.set_names("second", level=1)
+        kidx = kidx.set_names("second", level=1)
+        self.assert_eq(pidx, kidx)
+
+        pidx = pidx.set_names("third", level=2)
+        kidx = kidx.set_names("third", level=2)
+        self.assert_eq(pidx, kidx)
+
+        pidx.set_names("first", level=0, inplace=True)
+        kidx.set_names("first", level=0, inplace=True)
+        self.assert_eq(pidx, kidx)
+
+        pidx.set_names("second", level=1, inplace=True)
+        kidx.set_names("second", level=1, inplace=True)
+        self.assert_eq(pidx, kidx)
+
+        pidx.set_names("third", level=2, inplace=True)
+        kidx.set_names("third", level=2, inplace=True)
+        self.assert_eq(pidx, kidx)
+
+    def test_multiindex_from_tuples(self):
+        tuples = [(1, "red"), (1, "blue"), (2, "red"), (2, "blue")]
+        pidx = pd.MultiIndex.from_tuples(tuples)
+        kidx = ps.MultiIndex.from_tuples(tuples)
+
+        self.assert_eq(pidx, kidx)
+
+    def test_multiindex_from_product(self):
+        iterables = [[0, 1, 2], ["green", "purple"]]
+        pidx = pd.MultiIndex.from_product(iterables)
+        kidx = ps.MultiIndex.from_product(iterables)
+
+        self.assert_eq(pidx, kidx)
+
+    def test_multiindex_tuple_column_name(self):
+        column_labels = pd.MultiIndex.from_tuples([("a", "x"), ("a", "y"), ("b", "z")])
+        pdf = pd.DataFrame([[1, 2, 3], [4, 5, 6], [7, 8, 9]], columns=column_labels)
+        pdf.set_index(("a", "x"), append=True, inplace=True)
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf, kdf)
+
+    def test_len(self):
+        pidx = pd.Index(range(10000))
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(len(pidx), len(kidx))
+
+        pidx = pd.MultiIndex.from_tuples([("a", "x", 1), ("b", "y", 2), ("c", "z", 3)])
+        kidx = ps.MultiIndex.from_tuples([("a", "x", 1), ("b", "y", 2), ("c", "z", 3)])
+
+        self.assert_eq(len(pidx), len(kidx))
+
+    def test_delete(self):
+        pidx = pd.Index([10, 9, 8, 7, 6, 7, 8, 9, 10])
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(pidx.delete(8).sort_values(), kidx.delete(8).sort_values())
+        self.assert_eq(pidx.delete(-9).sort_values(), kidx.delete(-9).sort_values())
+        self.assert_eq(pidx.delete([-9, 0, 8]).sort_values(), kidx.delete([-9, 0, 8]).sort_values())
+
+        with self.assertRaisesRegex(IndexError, "index 9 is out of bounds for axis 0 with size 9"):
+            kidx.delete([0, 9])
+        with self.assertRaisesRegex(
+            IndexError, "index -10 is out of bounds for axis 0 with size 9"
+        ):
+            kidx.delete([-10, 0])
+        with self.assertRaisesRegex(IndexError, "index 9 is out of bounds for axis 0 with size 9"):
+            kidx.delete(9)
+        with self.assertRaisesRegex(
+            IndexError, "index -10 is out of bounds for axis 0 with size 9"
+        ):
+            kidx.delete(-10)
+
+        # MultiIndex
+        pidx = pd.MultiIndex.from_tuples([("a", "x", 1), ("b", "y", 2), ("c", "z", 3)])
+        kidx = ps.MultiIndex.from_tuples([("a", "x", 1), ("b", "y", 2), ("c", "z", 3)])
+
+        self.assert_eq(pidx.delete(2).sort_values(), kidx.delete(2).sort_values())
+        self.assert_eq(pidx.delete(-3).sort_values(), kidx.delete(-3).sort_values())
+        self.assert_eq(pidx.delete([-3, 0, 2]).sort_values(), kidx.delete([-3, 0, 2]).sort_values())
+
+        with self.assertRaisesRegex(IndexError, "index 3 is out of bounds for axis 0 with size 3"):
+            kidx.delete([0, 3])
+        with self.assertRaisesRegex(IndexError, "index -4 is out of bounds for axis 0 with size 3"):
+            kidx.delete([-4, 0])
+        with self.assertRaisesRegex(IndexError, "index 3 is out of bounds for axis 0 with size 3"):
+            kidx.delete(3)
+        with self.assertRaisesRegex(IndexError, "index -4 is out of bounds for axis 0 with size 3"):
+            kidx.delete(-4)
+
+    def test_append(self):
+        # Index
+        pidx = pd.Index(range(10000))
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(pidx.append(pidx), kidx.append(kidx))
+
+        # Index with name
+        pidx1 = pd.Index(range(10000), name="a")
+        pidx2 = pd.Index(range(10000), name="b")
+        kidx1 = ps.from_pandas(pidx1)
+        kidx2 = ps.from_pandas(pidx2)
+
+        self.assert_eq(pidx1.append(pidx2), kidx1.append(kidx2))
+
+        self.assert_eq(pidx2.append(pidx1), kidx2.append(kidx1))
+
+        # Index from DataFrame
+        pdf1 = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}, index=["a", "b", "c"])
+        pdf2 = pd.DataFrame({"a": [7, 8, 9], "d": [10, 11, 12]}, index=["x", "y", "z"])
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        pidx1 = pdf1.set_index("a").index
+        pidx2 = pdf2.set_index("d").index
+        kidx1 = kdf1.set_index("a").index
+        kidx2 = kdf2.set_index("d").index
+
+        self.assert_eq(pidx1.append(pidx2), kidx1.append(kidx2))
+
+        self.assert_eq(pidx2.append(pidx1), kidx2.append(kidx1))
+
+        # Index from DataFrame with MultiIndex columns
+        pdf1 = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
+        pdf2 = pd.DataFrame({"a": [7, 8, 9], "d": [10, 11, 12]})
+        pdf1.columns = pd.MultiIndex.from_tuples([("a", "x"), ("b", "y")])
+        pdf2.columns = pd.MultiIndex.from_tuples([("a", "x"), ("d", "y")])
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        pidx1 = pdf1.set_index(("a", "x")).index
+        pidx2 = pdf2.set_index(("d", "y")).index
+        kidx1 = kdf1.set_index(("a", "x")).index
+        kidx2 = kdf2.set_index(("d", "y")).index
+
+        self.assert_eq(pidx1.append(pidx2), kidx1.append(kidx2))
+
+        self.assert_eq(pidx2.append(pidx1), kidx2.append(kidx1))
+
+        # MultiIndex
+        pmidx = pd.MultiIndex.from_tuples([("a", "x", 1), ("b", "y", 2), ("c", "z", 3)])
+        kmidx = ps.from_pandas(pmidx)
+
+        self.assert_eq(pmidx.append(pmidx), kmidx.append(kmidx))
+
+        # MultiIndex with names
+        pmidx1 = pd.MultiIndex.from_tuples(
+            [("a", "x", 1), ("b", "y", 2), ("c", "z", 3)], names=["x", "y", "z"]
+        )
+        pmidx2 = pd.MultiIndex.from_tuples(
+            [("a", "x", 1), ("b", "y", 2), ("c", "z", 3)], names=["p", "q", "r"]
+        )
+        kmidx1 = ps.from_pandas(pmidx1)
+        kmidx2 = ps.from_pandas(pmidx2)
+
+        self.assert_eq(pmidx1.append(pmidx2), kmidx1.append(kmidx2))
+
+        self.assert_eq(pmidx2.append(pmidx1), kmidx2.append(kmidx1))
+
+        self.assert_eq(pmidx1.append(pmidx2).names, kmidx1.append(kmidx2).names)
+
+        self.assert_eq(pmidx1.append(pmidx2).names, kmidx1.append(kmidx2).names)
+
+        # Index & MultiIndex currently is not supported
+        expected_error_message = r"append\(\) between Index & MultiIndex currently is not supported"
+        with self.assertRaisesRegex(NotImplementedError, expected_error_message):
+            kidx.append(kmidx)
+        with self.assertRaisesRegex(NotImplementedError, expected_error_message):
+            kmidx.append(kidx)
+
+    def test_argmin(self):
+        pidx = pd.Index([100, 50, 10, 20, 30, 60, 0, 50, 0, 100, 100, 100, 20, 0, 0])
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(pidx.argmin(), kidx.argmin())
+
+        # MultiIndex
+        kidx = ps.MultiIndex.from_tuples([("a", "x", 1), ("b", "y", 2), ("c", "z", 3)])
+        with self.assertRaisesRegex(
+            TypeError, "reduction operation 'argmin' not allowed for this dtype"
+        ):
+            kidx.argmin()
+
+    def test_argmax(self):
+        pidx = pd.Index([100, 50, 10, 20, 30, 60, 0, 50, 0, 100, 100, 100, 20, 0, 0])
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(pidx.argmax(), kidx.argmax())
+
+        # MultiIndex
+        kidx = ps.MultiIndex.from_tuples([("a", "x", 1), ("b", "y", 2), ("c", "z", 3)])
+        with self.assertRaisesRegex(
+            TypeError, "reduction operation 'argmax' not allowed for this dtype"
+        ):
+            kidx.argmax()
+
+    def test_min(self):
+        pidx = pd.Index([3, 2, 1])
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(pidx.min(), kidx.min())
+
+        # MultiIndex
+        pmidx = pd.MultiIndex.from_tuples([("a", "x", 1), ("b", "y", 2)])
+        kmidx = ps.from_pandas(pmidx)
+
+        self.assert_eq(pmidx.min(), kmidx.min())
+
+        pidx = pd.DatetimeIndex(["2021-02-01", "2021-01-01", "2021-04-01", "2021-03-01"])
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(pidx.min(), kidx.min())
+
+    def test_max(self):
+        pidx = pd.Index([3, 2, 1])
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(pidx.max(), kidx.max())
+
+        # MultiIndex
+        pmidx = pd.MultiIndex.from_tuples([("a", "x", 1), ("b", "y", 2)])
+        kmidx = ps.from_pandas(pmidx)
+
+        self.assert_eq(pmidx.max(), kmidx.max())
+
+        pidx = pd.DatetimeIndex(["2021-02-01", "2021-01-01", "2021-04-01", "2021-03-01"])
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(pidx.max(), kidx.max())
+
+    def test_monotonic(self):
+        # test monotonic_increasing & monotonic_decreasing for MultiIndex.
+        # Since the Behavior for null value was changed in pandas >= 1.0.0,
+        # several cases are tested differently.
+        datas = []
+
+        # increasing / decreasing ordered each index level with string
+        datas.append([("w", "a"), ("x", "b"), ("y", "c"), ("z", "d")])
+        datas.append([("w", "d"), ("x", "c"), ("y", "b"), ("z", "a")])
+        datas.append([("z", "a"), ("y", "b"), ("x", "c"), ("w", "d")])
+        datas.append([("z", "d"), ("y", "c"), ("x", "b"), ("w", "a")])
+        # mixed order each index level with string
+        datas.append([("z", "a"), ("x", "b"), ("y", "c"), ("w", "d")])
+        datas.append([("z", "a"), ("y", "c"), ("x", "b"), ("w", "d")])
+
+        # increasing / decreasing ordered each index level with integer
+        datas.append([(1, 100), (2, 200), (3, 300), (4, 400), (5, 500)])
+        datas.append([(1, 500), (2, 400), (3, 300), (4, 200), (5, 100)])
+        datas.append([(5, 100), (4, 200), (3, 300), (2, 400), (1, 500)])
+        datas.append([(5, 500), (4, 400), (3, 300), (2, 200), (1, 100)])
+        # mixed order each index level with integer
+        datas.append([(1, 500), (3, 400), (2, 300), (4, 200), (5, 100)])
+        datas.append([(1, 100), (2, 300), (3, 200), (4, 400), (5, 500)])
+
+        # integer / negative mixed tests
+        datas.append([("a", -500), ("b", -400), ("c", -300), ("d", -200), ("e", -100)])
+        datas.append([("e", -500), ("d", -400), ("c", -300), ("b", -200), ("a", -100)])
+        datas.append([(-5, "a"), (-4, "b"), (-3, "c"), (-2, "d"), (-1, "e")])
+        datas.append([(-5, "e"), (-4, "d"), (-3, "c"), (-2, "b"), (-1, "a")])
+        datas.append([(-5, "e"), (-3, "d"), (-2, "c"), (-4, "b"), (-1, "a")])
+        datas.append([(-5, "e"), (-4, "c"), (-3, "b"), (-2, "d"), (-1, "a")])
+
+        # boolean type tests
+        datas.append([(True, True), (True, True)])
+        datas.append([(True, True), (True, False)])
+        datas.append([(True, False), (True, True)])
+        datas.append([(False, True), (False, True)])
+        datas.append([(False, True), (False, False)])
+        datas.append([(False, False), (False, True)])
+        datas.append([(True, True), (False, True)])
+        datas.append([(True, True), (False, False)])
+        datas.append([(True, False), (False, True)])
+        datas.append([(False, True), (True, True)])
+        datas.append([(False, True), (True, False)])
+        datas.append([(False, False), (True, True)])
+
+        # duplicated index value tests
+        datas.append([("x", "d"), ("y", "c"), ("y", "b"), ("z", "a")])
+        datas.append([("x", "d"), ("y", "b"), ("y", "c"), ("z", "a")])
+
+        # more depth tests
+        datas.append([("x", "d", "o"), ("y", "c", "p"), ("y", "c", "q"), ("z", "a", "r")])
+        datas.append([("x", "d", "o"), ("y", "c", "q"), ("y", "c", "p"), ("z", "a", "r")])
+
+        # None type tests (None type is treated as False from pandas >= 1.1.4)
+        # Refer https://github.com/pandas-dev/pandas/issues/37220
+        datas.append([(1, 100), (2, 200), (None, 300), (4, 400), (5, 500)])
+        datas.append([(1, 100), (2, 200), (None, None), (4, 400), (5, 500)])
+        datas.append([("x", "d"), ("y", "c"), ("y", None), ("z", "a")])
+        datas.append([("x", "d"), ("y", "c"), ("y", "b"), (None, "a")])
+        datas.append([("x", "d"), ("y", "b"), ("y", "c"), (None, "a")])
+        datas.append([("x", "d", "o"), ("y", "c", "p"), ("y", "c", None), ("z", "a", "r")])
+
+        for data in datas:
+            with self.subTest(data=data):
+                pmidx = pd.MultiIndex.from_tuples(data)
+                kmidx = ps.from_pandas(pmidx)
+                self.assert_eq(kmidx.is_monotonic_increasing, pmidx.is_monotonic_increasing)
+                self.assert_eq(kmidx.is_monotonic_decreasing, pmidx.is_monotonic_decreasing)
+
+        # datas below return different result depends on pandas version.
+        # Because the behavior of handling null values is changed in pandas >= 1.1.4.
+        # Since Koalas follows latest pandas, all of them should return `False`.
+        datas = []
+        datas.append([(1, 100), (2, 200), (3, None), (4, 400), (5, 500)])
+        datas.append([(1, None), (2, 200), (3, 300), (4, 400), (5, 500)])
+        datas.append([(1, 100), (2, 200), (3, 300), (4, 400), (5, None)])
+        datas.append([(False, None), (True, True)])
+        datas.append([(None, False), (True, True)])
+        datas.append([(False, False), (True, None)])
+        datas.append([(False, False), (None, True)])
+        datas.append([("x", "d"), ("y", None), ("y", None), ("z", "a")])
+        datas.append([("x", "d", "o"), ("y", "c", None), ("y", "c", None), ("z", "a", "r")])
+        datas.append([(1, 100), (2, 200), (3, 300), (4, 400), (None, 500)])
+        datas.append([(1, 100), (2, 200), (3, 300), (4, 400), (None, None)])
+        datas.append([(5, 100), (4, 200), (3, None), (2, 400), (1, 500)])
+        datas.append([(5, None), (4, 200), (3, 300), (2, 400), (1, 500)])
+        datas.append([(5, 100), (4, 200), (3, None), (2, 400), (1, 500)])
+        datas.append([(5, 100), (4, 200), (3, 300), (2, 400), (1, None)])
+        datas.append([(True, None), (True, True)])
+        datas.append([(None, True), (True, True)])
+        datas.append([(True, True), (None, True)])
+        datas.append([(True, True), (True, None)])
+        datas.append([(None, 100), (2, 200), (3, 300), (4, 400), (5, 500)])
+        datas.append([(None, None), (2, 200), (3, 300), (4, 400), (5, 500)])
+        datas.append([("x", "d"), ("y", None), ("y", "c"), ("z", "a")])
+        datas.append([("x", "d", "o"), ("y", "c", None), ("y", "c", "q"), ("z", "a", "r")])
+
+        for data in datas:
+            with self.subTest(data=data):
+                pmidx = pd.MultiIndex.from_tuples(data)
+                kmidx = ps.from_pandas(pmidx)
+                if LooseVersion(pd.__version__) < LooseVersion("1.1.4"):
+                    self.assert_eq(kmidx.is_monotonic_increasing, False)
+                    self.assert_eq(kmidx.is_monotonic_decreasing, False)
+                else:
+                    self.assert_eq(kmidx.is_monotonic_increasing, pmidx.is_monotonic_increasing)
+                    self.assert_eq(kmidx.is_monotonic_decreasing, pmidx.is_monotonic_decreasing)
+
+        # The datas below are tested another way since they cannot be an arguments for
+        # `MultiIndex.from_tuples` in pandas >= 1.1.0.
+        # Refer https://github.com/databricks/koalas/pull/1688#issuecomment-667156560 for detail.
+        if LooseVersion(pd.__version__) < LooseVersion("1.1.0"):
+            pmidx = pd.MultiIndex.from_tuples(
+                [(-5, None), (-4, None), (-3, None), (-2, None), (-1, None)]
+            )
+            kmidx = ps.from_pandas(pmidx)
+            self.assert_eq(kmidx.is_monotonic_increasing, False)
+            self.assert_eq(kmidx.is_monotonic_decreasing, False)
+
+            pmidx = pd.MultiIndex.from_tuples(
+                [(None, "e"), (None, "c"), (None, "b"), (None, "d"), (None, "a")]
+            )
+            kmidx = ps.from_pandas(pmidx)
+            self.assert_eq(kmidx.is_monotonic_increasing, False)
+            self.assert_eq(kmidx.is_monotonic_decreasing, False)
+
+            pmidx = pd.MultiIndex.from_tuples(
+                [(None, None), (None, None), (None, None), (None, None), (None, None)]
+            )
+            kmidx = ps.from_pandas(pmidx)
+            self.assert_eq(kmidx.is_monotonic_increasing, False)
+            self.assert_eq(kmidx.is_monotonic_decreasing, False)
+
+            pmidx = pd.MultiIndex.from_tuples([(None, None)])
+            kmidx = ps.from_pandas(pmidx)
+            self.assert_eq(kmidx.is_monotonic_increasing, False)
+            self.assert_eq(kmidx.is_monotonic_decreasing, False)
+
+        # Disable the test cases below because pandas returns `True` or `False` randomly.
+        # else:
+            # [(-5, None), (-4, None), (-3, None), (-2, None), (-1, None)]
+            # kdf = ps.DataFrame({"a": [-5, -4, -3, -2, -1], "b": [1, 1, 1, 1, 1]})
+            # kdf["b"] = None
+            # kmidx = kdf.set_index(["a", "b"]).index
+            # pmidx = kmidx.to_pandas()
+            # self.assert_eq(kmidx.is_monotonic_increasing, pmidx.is_monotonic_increasing)
+            # self.assert_eq(kmidx.is_monotonic_decreasing, pmidx.is_monotonic_decreasing)
+
+            # [(None, "e"), (None, "c"), (None, "b"), (None, "d"), (None, "a")]
+            # kdf = ps.DataFrame({"a": [1, 1, 1, 1, 1], "b": ["e", "c", "b", "d", "a"]})
+            # kdf["a"] = None
+            # kmidx = kdf.set_index(["a", "b"]).index
+            # pmidx = kmidx.to_pandas()
+            # self.assert_eq(kmidx.is_monotonic_increasing, pmidx.is_monotonic_increasing)
+            # self.assert_eq(kmidx.is_monotonic_decreasing, pmidx.is_monotonic_decreasing)
+
+            # [(None, None), (None, None), (None, None), (None, None), (None, None)]
+            # kdf = ps.DataFrame({"a": [1, 1, 1, 1, 1], "b": [1, 1, 1, 1, 1]})
+            # kdf["a"] = None
+            # kdf["b"] = None
+            # kmidx = kdf.set_index(["a", "b"]).index
+            # pmidx = kmidx.to_pandas()
+            # self.assert_eq(kmidx.is_monotonic_increasing, pmidx.is_monotonic_increasing)
+            # self.assert_eq(kmidx.is_monotonic_decreasing, pmidx.is_monotonic_decreasing)
+            # [(None, None)]
+            # kdf = ps.DataFrame({"a": [1], "b": [1]})
+            # kdf["a"] = None
+            # kdf["b"] = None
+            # kmidx = kdf.set_index(["a", "b"]).index
+            # pmidx = kmidx.to_pandas()
+            # self.assert_eq(kmidx.is_monotonic_increasing, pmidx.is_monotonic_increasing)
+            # self.assert_eq(kmidx.is_monotonic_decreasing, pmidx.is_monotonic_decreasing)
+
+    def test_difference(self):
+        # Index
+        pidx1 = pd.Index([1, 2, 3, 4], name="koalas")
+        pidx2 = pd.Index([3, 4, 5, 6], name="koalas")
+        kidx1 = ps.from_pandas(pidx1)
+        kidx2 = ps.from_pandas(pidx2)
+
+        self.assert_eq(kidx1.difference(kidx2).sort_values(), pidx1.difference(pidx2).sort_values())
+        self.assert_eq(
+            kidx1.difference([3, 4, 5, 6]).sort_values(),
+            pidx1.difference([3, 4, 5, 6]).sort_values(),
+        )
+        self.assert_eq(
+            kidx1.difference((3, 4, 5, 6)).sort_values(),
+            pidx1.difference((3, 4, 5, 6)).sort_values(),
+        )
+        self.assert_eq(
+            kidx1.difference({3, 4, 5, 6}).sort_values(),
+            pidx1.difference({3, 4, 5, 6}).sort_values(),
+        )
+        self.assert_eq(
+            kidx1.difference({3: 1, 4: 2, 5: 3, 6: 4}).sort_values(),
+            pidx1.difference({3: 1, 4: 2, 5: 3, 6: 4}).sort_values(),
+        )
+
+        # Exceptions for Index
+        with self.assertRaisesRegex(TypeError, "Input must be Index or array-like"):
+            kidx1.difference("1234")
+        with self.assertRaisesRegex(TypeError, "Input must be Index or array-like"):
+            kidx1.difference(1234)
+        with self.assertRaisesRegex(TypeError, "Input must be Index or array-like"):
+            kidx1.difference(12.34)
+        with self.assertRaisesRegex(TypeError, "Input must be Index or array-like"):
+            kidx1.difference(None)
+        with self.assertRaisesRegex(TypeError, "Input must be Index or array-like"):
+            kidx1.difference(np.nan)
+        with self.assertRaisesRegex(
+            ValueError, "The 'sort' keyword only takes the values of None or True; 1 was passed."
+        ):
+            kidx1.difference(kidx2, sort=1)
+
+        # MultiIndex
+        pidx1 = pd.MultiIndex.from_tuples(
+            [("a", "x", 1), ("b", "y", 2), ("c", "z", 3)], names=["hello", "koalas", "world"]
+        )
+        pidx2 = pd.MultiIndex.from_tuples(
+            [("a", "x", 1), ("b", "z", 2), ("k", "z", 3)], names=["hello", "koalas", "world"]
+        )
+        kidx1 = ps.from_pandas(pidx1)
+        kidx2 = ps.from_pandas(pidx2)
+
+        self.assert_eq(kidx1.difference(kidx2).sort_values(), pidx1.difference(pidx2).sort_values())
+        self.assert_eq(
+            kidx1.difference({("a", "x", 1)}).sort_values(),
+            pidx1.difference({("a", "x", 1)}).sort_values(),
+        )
+        self.assert_eq(
+            kidx1.difference({("a", "x", 1): [1, 2, 3]}).sort_values(),
+            pidx1.difference({("a", "x", 1): [1, 2, 3]}).sort_values(),
+        )
+
+        # Exceptions for MultiIndex
+        with self.assertRaisesRegex(TypeError, "other must be a MultiIndex or a list of tuples"):
+            kidx1.difference(["b", "z", "2"])
+
+    def test_repeat(self):
+        pidx = pd.Index(["a", "b", "c"])
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(kidx.repeat(3).sort_values(), pidx.repeat(3).sort_values())
+        self.assert_eq(kidx.repeat(0).sort_values(), pidx.repeat(0).sort_values())
+        self.assert_eq((kidx + "x").repeat(3).sort_values(), (pidx + "x").repeat(3).sort_values())
+
+        self.assertRaises(ValueError, lambda: kidx.repeat(-1))
+        self.assertRaises(TypeError, lambda: kidx.repeat("abc"))
+
+        pmidx = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "c")])
+        kmidx = ps.from_pandas(pmidx)
+
+        self.assert_eq(kmidx.repeat(3).sort_values(), pmidx.repeat(3).sort_values())
+        self.assert_eq(kmidx.repeat(0).sort_values(), pmidx.repeat(0).sort_values(), almost=True)
+
+        self.assertRaises(ValueError, lambda: kmidx.repeat(-1))
+        self.assertRaises(TypeError, lambda: kmidx.repeat("abc"))
+
+    def test_unique(self):
+        pidx = pd.Index(["a", "b", "a"])
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(kidx.unique().sort_values(), pidx.unique().sort_values())
+        self.assert_eq(kidx.unique().sort_values(), pidx.unique().sort_values())
+
+        pmidx = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("x", "a")])
+        kmidx = ps.from_pandas(pmidx)
+
+        self.assert_eq(kmidx.unique().sort_values(), pmidx.unique().sort_values())
+        self.assert_eq(kmidx.unique().sort_values(), pmidx.unique().sort_values())
+
+    def test_asof(self):
+        # Increasing values
+        pidx = pd.Index(["2013-12-31", "2014-01-02", "2014-01-03"])
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(kidx.asof("2014-01-01"), pidx.asof("2014-01-01"))
+        self.assert_eq(kidx.asof("2014-01-02"), pidx.asof("2014-01-02"))
+        self.assert_eq(repr(kidx.asof("1999-01-02")), repr(pidx.asof("1999-01-02")))
+        self.assert_eq(kidx.asof("2014-01-04"), pidx.asof("2014-01-04"))
+
+        pidx = pd.DatetimeIndex(["2013-12-31", "2014-01-02", "2014-01-03"])
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(kidx.asof("2014-01-01"), pidx.asof("2014-01-01"))
+        self.assert_eq(kidx.asof("2014-01-02"), pidx.asof("2014-01-02"))
+        self.assert_eq(repr(kidx.asof("1999-01-02")), repr(pidx.asof("1999-01-02")))
+
+        # Decreasing values
+        pidx = pd.Index(["2014-01-03", "2014-01-02", "2013-12-31"])
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(kidx.asof("2014-01-01"), pidx.asof("2014-01-01"))
+        self.assert_eq(kidx.asof("2014-01-02"), pidx.asof("2014-01-02"))
+        self.assert_eq(kidx.asof("1999-01-02"), pidx.asof("1999-01-02"))
+        self.assert_eq(repr(kidx.asof("2015-01-02")), repr(pidx.asof("2015-01-02")))
+
+        pidx = pd.DatetimeIndex(["2014-01-03", "2014-01-02", "2013-12-31"])
+        kidx = ps.from_pandas(pidx)
+
+        # TODO: a pandas bug?
+        # self.assert_eq(kidx.asof("2014-01-01"), pidx.asof("2014-01-01"))
+        # self.assert_eq(kidx.asof("2014-01-02"), pidx.asof("2014-01-02"))
+        # self.assert_eq(kidx.asof("1999-01-02"), pidx.asof("1999-01-02"))
+        # self.assert_eq(repr(kidx.asof("2015-01-02")), repr(pidx.asof("2015-01-02")))
+        self.assert_eq(kidx.asof("2014-01-01"), pd.Timestamp("2014-01-02 00:00:00"))
+        self.assert_eq(kidx.asof("2014-01-02"), pd.Timestamp("2014-01-02 00:00:00"))
+        self.assert_eq(kidx.asof("1999-01-02"), pd.Timestamp("2013-12-31 00:00:00"))
+        self.assert_eq(repr(kidx.asof("2015-01-02")), repr(pd.NaT))
+
+        # Not increasing, neither decreasing (ValueError)
+        kidx = ps.Index(["2013-12-31", "2015-01-02", "2014-01-03"])
+        self.assertRaises(ValueError, lambda: kidx.asof("2013-12-31"))
+
+        kmidx = ps.MultiIndex.from_tuples([("a", "a"), ("a", "b"), ("a", "c")])
+        self.assertRaises(NotImplementedError, lambda: kmidx.asof(("a", "b")))
+
+    def test_union(self):
+        # Index
+        pidx1 = pd.Index([1, 2, 3, 4])
+        pidx2 = pd.Index([3, 4, 5, 6])
+        kidx1 = ps.from_pandas(pidx1)
+        kidx2 = ps.from_pandas(pidx2)
+
+        self.assert_eq(kidx1.union(kidx2), pidx1.union(pidx2))
+        self.assert_eq(kidx2.union(kidx1), pidx2.union(pidx1))
+        self.assert_eq(kidx1.union([3, 4, 5, 6]), pidx1.union([3, 4, 5, 6]), almost=True)
+        self.assert_eq(kidx2.union([1, 2, 3, 4]), pidx2.union([1, 2, 3, 4]), almost=True)
+        self.assert_eq(
+            kidx1.union(ps.Series([3, 4, 5, 6])), pidx1.union(pd.Series([3, 4, 5, 6])), almost=True
+        )
+        self.assert_eq(
+            kidx2.union(ps.Series([1, 2, 3, 4])), pidx2.union(pd.Series([1, 2, 3, 4])), almost=True
+        )
+
+        # Testing if the result is correct after sort=False.
+        # The `sort` argument is added in pandas 0.24.
+        if LooseVersion(pd.__version__) >= LooseVersion("0.24"):
+            self.assert_eq(
+                kidx1.union(kidx2, sort=False).sort_values(),
+                pidx1.union(pidx2, sort=False).sort_values(),
+            )
+            self.assert_eq(
+                kidx2.union(kidx1, sort=False).sort_values(),
+                pidx2.union(pidx1, sort=False).sort_values(),
+            )
+            self.assert_eq(
+                kidx1.union([3, 4, 5, 6], sort=False).sort_values(),
+                pidx1.union([3, 4, 5, 6], sort=False).sort_values(),
+                almost=True,
+            )
+            self.assert_eq(
+                kidx2.union([1, 2, 3, 4], sort=False).sort_values(),
+                pidx2.union([1, 2, 3, 4], sort=False).sort_values(),
+                almost=True,
+            )
+            self.assert_eq(
+                kidx1.union(ps.Series([3, 4, 5, 6]), sort=False).sort_values(),
+                pidx1.union(pd.Series([3, 4, 5, 6]), sort=False).sort_values(),
+                almost=True,
+            )
+            self.assert_eq(
+                kidx2.union(ps.Series([1, 2, 3, 4]), sort=False).sort_values(),
+                pidx2.union(pd.Series([1, 2, 3, 4]), sort=False).sort_values(),
+                almost=True,
+            )
+
+        # Duplicated values for Index is supported in pandas >= 1.0.0
+        if LooseVersion(pd.__version__) >= LooseVersion("1.0.0"):
+            pidx1 = pd.Index([1, 2, 3, 4, 3, 4, 3, 4])
+            pidx2 = pd.Index([3, 4, 3, 4, 5, 6])
+            kidx1 = ps.from_pandas(pidx1)
+            kidx2 = ps.from_pandas(pidx2)
+
+            self.assert_eq(kidx1.union(kidx2), pidx1.union(pidx2))
+            self.assert_eq(kidx2.union(kidx1), pidx2.union(pidx1))
+            self.assert_eq(
+                kidx1.union([3, 4, 3, 3, 5, 6]), pidx1.union([3, 4, 3, 4, 5, 6]), almost=True
+            )
+            self.assert_eq(
+                kidx2.union([1, 2, 3, 4, 3, 4, 3, 4]),
+                pidx2.union([1, 2, 3, 4, 3, 4, 3, 4]),
+                almost=True,
+            )
+            self.assert_eq(
+                kidx1.union(ps.Series([3, 4, 3, 3, 5, 6])),
+                pidx1.union(pd.Series([3, 4, 3, 4, 5, 6])),
+                almost=True,
+            )
+            self.assert_eq(
+                kidx2.union(ps.Series([1, 2, 3, 4, 3, 4, 3, 4])),
+                pidx2.union(pd.Series([1, 2, 3, 4, 3, 4, 3, 4])),
+                almost=True,
+            )
+
+        # MultiIndex
+        pmidx1 = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("x", "a"), ("x", "b")])
+        pmidx2 = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("x", "c"), ("x", "d")])
+        pmidx3 = pd.MultiIndex.from_tuples([(1, 1), (1, 2), (1, 3), (1, 4), (1, 3), (1, 4)])
+        pmidx4 = pd.MultiIndex.from_tuples([(1, 3), (1, 4), (1, 5), (1, 6)])
+        kmidx1 = ps.from_pandas(pmidx1)
+        kmidx2 = ps.from_pandas(pmidx2)
+        kmidx3 = ps.from_pandas(pmidx3)
+        kmidx4 = ps.from_pandas(pmidx4)
+
+        self.assert_eq(kmidx1.union(kmidx2), pmidx1.union(pmidx2))
+        self.assert_eq(kmidx2.union(kmidx1), pmidx2.union(pmidx1))
+        self.assert_eq(kmidx3.union(kmidx4), pmidx3.union(pmidx4))
+        self.assert_eq(kmidx4.union(kmidx3), pmidx4.union(pmidx3))
+        self.assert_eq(
+            kmidx1.union([("x", "a"), ("x", "b"), ("x", "c"), ("x", "d")]),
+            pmidx1.union([("x", "a"), ("x", "b"), ("x", "c"), ("x", "d")]),
+        )
+        self.assert_eq(
+            kmidx2.union([("x", "a"), ("x", "b"), ("x", "a"), ("x", "b")]),
+            pmidx2.union([("x", "a"), ("x", "b"), ("x", "a"), ("x", "b")]),
+        )
+        self.assert_eq(
+            kmidx3.union([(1, 3), (1, 4), (1, 5), (1, 6)]),
+            pmidx3.union([(1, 3), (1, 4), (1, 5), (1, 6)]),
+        )
+        self.assert_eq(
+            kmidx4.union([(1, 1), (1, 2), (1, 3), (1, 4), (1, 3), (1, 4)]),
+            pmidx4.union([(1, 1), (1, 2), (1, 3), (1, 4), (1, 3), (1, 4)]),
+        )
+
+        # Testing if the result is correct after sort=False.
+        # The `sort` argument is added in pandas 0.24.
+        if LooseVersion(pd.__version__) >= LooseVersion("0.24"):
+            self.assert_eq(
+                kmidx1.union(kmidx2, sort=False).sort_values(),
+                pmidx1.union(pmidx2, sort=False).sort_values(),
+            )
+            self.assert_eq(
+                kmidx2.union(kmidx1, sort=False).sort_values(),
+                pmidx2.union(pmidx1, sort=False).sort_values(),
+            )
+            self.assert_eq(
+                kmidx3.union(kmidx4, sort=False).sort_values(),
+                pmidx3.union(pmidx4, sort=False).sort_values(),
+            )
+            self.assert_eq(
+                kmidx4.union(kmidx3, sort=False).sort_values(),
+                pmidx4.union(pmidx3, sort=False).sort_values(),
+            )
+            self.assert_eq(
+                kmidx1.union(
+                    [("x", "a"), ("x", "b"), ("x", "c"), ("x", "d")], sort=False
+                ).sort_values(),
+                pmidx1.union(
+                    [("x", "a"), ("x", "b"), ("x", "c"), ("x", "d")], sort=False
+                ).sort_values(),
+            )
+            self.assert_eq(
+                kmidx2.union(
+                    [("x", "a"), ("x", "b"), ("x", "a"), ("x", "b")], sort=False
+                ).sort_values(),
+                pmidx2.union(
+                    [("x", "a"), ("x", "b"), ("x", "a"), ("x", "b")], sort=False
+                ).sort_values(),
+            )
+            self.assert_eq(
+                kmidx3.union([(1, 3), (1, 4), (1, 5), (1, 6)], sort=False).sort_values(),
+                pmidx3.union([(1, 3), (1, 4), (1, 5), (1, 6)], sort=False).sort_values(),
+            )
+            self.assert_eq(
+                kmidx4.union(
+                    [(1, 1), (1, 2), (1, 3), (1, 4), (1, 3), (1, 4)], sort=False
+                ).sort_values(),
+                pmidx4.union(
+                    [(1, 1), (1, 2), (1, 3), (1, 4), (1, 3), (1, 4)], sort=False
+                ).sort_values(),
+            )
+
+        self.assertRaises(NotImplementedError, lambda: kidx1.union(kmidx1))
+        self.assertRaises(TypeError, lambda: kmidx1.union(kidx1))
+        self.assertRaises(TypeError, lambda: kmidx1.union(["x", "a"]))
+        self.assertRaises(ValueError, lambda: kidx1.union(ps.range(2)))
+
+    def test_take(self):
+        # Index
+        pidx = pd.Index([100, 200, 300, 400, 500], name="Koalas")
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(kidx.take([0, 2, 4]).sort_values(), pidx.take([0, 2, 4]).sort_values())
+        self.assert_eq(
+            kidx.take(range(0, 5, 2)).sort_values(), pidx.take(range(0, 5, 2)).sort_values()
+        )
+        self.assert_eq(kidx.take([-4, -2, 0]).sort_values(), pidx.take([-4, -2, 0]).sort_values())
+        self.assert_eq(
+            kidx.take(range(-4, 1, 2)).sort_values(), pidx.take(range(-4, 1, 2)).sort_values()
+        )
+
+        # MultiIndex
+        pmidx = pd.MultiIndex.from_tuples(
+            [("x", "a"), ("x", "b"), ("x", "c")], names=["hello", "Koalas"]
+        )
+        kmidx = ps.from_pandas(pmidx)
+
+        self.assert_eq(kmidx.take([0, 2]).sort_values(), pmidx.take([0, 2]).sort_values())
+        self.assert_eq(
+            kmidx.take(range(0, 4, 2)).sort_values(), pmidx.take(range(0, 4, 2)).sort_values()
+        )
+        self.assert_eq(kmidx.take([-2, 0]).sort_values(), pmidx.take([-2, 0]).sort_values())
+        self.assert_eq(
+            kmidx.take(range(-2, 1, 2)).sort_values(), pmidx.take(range(-2, 1, 2)).sort_values()
+        )
+
+        # Checking the type of indices.
+        self.assertRaises(TypeError, lambda: kidx.take(1))
+        self.assertRaises(TypeError, lambda: kidx.take("1"))
+        self.assertRaises(TypeError, lambda: kidx.take({1, 2}))
+        self.assertRaises(TypeError, lambda: kidx.take({1: None, 2: None}))
+        self.assertRaises(TypeError, lambda: kmidx.take(1))
+        self.assertRaises(TypeError, lambda: kmidx.take("1"))
+        self.assertRaises(TypeError, lambda: kmidx.take({1, 2}))
+        self.assertRaises(TypeError, lambda: kmidx.take({1: None, 2: None}))
+
+    def test_index_get_level_values(self):
+        pidx = pd.Index([1, 2, 3], name="ks")
+        kidx = ps.from_pandas(pidx)
+
+        for level in [0, "ks"]:
+            self.assert_eq(kidx.get_level_values(level), pidx.get_level_values(level))
+
+    def test_multiindex_get_level_values(self):
+        pmidx = pd.MultiIndex.from_tuples([("a", "d"), ("b", "e"), ("c", "f")])
+        pmidx.names = ["level_1", "level_2"]
+        kmidx = ps.from_pandas(pmidx)
+
+        for level in [0, 1, "level_1", "level_2"]:
+            self.assert_eq(kmidx.get_level_values(level), pmidx.get_level_values(level))
+
+    def test_index_get_level_number(self):
+        # name of two levels are the same, which is None
+        kdf = ps.DataFrame({"a": [1, 2, 3]}, index=[list("aac"), list("ddf")])
+        with self.assertRaisesRegex(
+            ValueError, "The name None occurs multiple times, use a level number"
+        ):
+            kdf.index._get_level_number(None)
+
+        mi = pd.MultiIndex.from_arrays((list("abc"), list("def")))
+        mi.names = ["level_1", "level_2"]
+        kdf = ps.DataFrame({"a": [1, 2, 3]}, index=mi)
+
+        # level is not int and not in the level name list
+        with self.assertRaisesRegex(KeyError, "Level lv_3 not found"):
+            kdf.index._get_level_number("lv_3")
+
+        # level is int, but an invalid negative number
+        with self.assertRaisesRegex(IndexError, "Too many levels: Index has only"):
+            kdf.index._get_level_number(-3)
+
+        # level is int, but an invalid positive number
+        with self.assertRaisesRegex(IndexError, "Too many levels: Index has only"):
+            kdf.index._get_level_number(3)
+
+        # Correct and valid inputs in numbers
+        level_number = [-2, -1, 0, 1]
+        outputs = [0, 1, 0, 1]
+
+        for lv, output in zip(level_number, outputs):
+            self.assertEqual(output, kdf.index._get_level_number(lv))
+
+        # Valid inputs as level names
+        level_names = ["level_1", "level_2"]
+        outputs = [0, 1]
+
+        for lv, output in zip(level_names, outputs):
+            self.assertEqual(output, kdf.index._get_level_number(lv))
+
+    def test_holds_integer(self):
+        pidx = pd.Index([1, 2, 3, 4])
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.holds_integer(), kidx.holds_integer())
+
+        pidx = pd.Index([1.1, 2.2, 3.3, 4.4])
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.holds_integer(), kidx.holds_integer())
+
+        pidx = pd.Index(["A", "B", "C", "D"])
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.holds_integer(), kidx.holds_integer())
+
+        # MultiIndex
+        pmidx = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "a")])
+        kmidx = ps.from_pandas(pmidx)
+        self.assert_eq(pmidx.holds_integer(), kmidx.holds_integer())
+
+        pmidx = pd.MultiIndex.from_tuples([(10, 1), (10, 2), (20, 1)])
+        kmidx = ps.from_pandas(pmidx)
+        self.assert_eq(pmidx.holds_integer(), kmidx.holds_integer())
+
+    def test_abs(self):
+        pidx = pd.Index([-2, -1, 0, 1])
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(abs(pidx), abs(kidx))
+        self.assert_eq(np.abs(pidx), np.abs(kidx))
+
+        kidx = ps.MultiIndex.from_tuples([(1, 2)], names=["level1", "level2"])
+        with self.assertRaisesRegex(TypeError, "perform __abs__ with this index"):
+            abs(kidx)
+
+    def test_hasnans(self):
+        # BooleanType
+        pidx = pd.Index([True, False, True, True])
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.hasnans, kidx.hasnans)
+
+        pidx = pd.Index([True, False, np.nan, True])
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.hasnans, kidx.hasnans)
+
+        # TimestampType
+        pser = pd.Series([pd.Timestamp("2020-07-30") for _ in range(3)])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.hasnans, kser.hasnans)
+
+        pser = pd.Series([pd.Timestamp("2020-07-30"), np.nan, pd.Timestamp("2020-07-30")])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.hasnans, kser.hasnans)
+
+        # Not supported for MultiIndex
+        kmidx = ps.Index([("a", 1), ("b", 2)])
+        self.assertRaises(NotImplementedError, lambda: kmidx.hasnans())
+
+    def test_intersection(self):
+        pidx = pd.Index([1, 2, 3, 4], name="Koalas")
+        kidx = ps.from_pandas(pidx)
+
+        # other = Index
+        pidx_other = pd.Index([3, 4, 5, 6], name="Koalas")
+        kidx_other = ps.from_pandas(pidx_other)
+        self.assert_eq(pidx.intersection(pidx_other), kidx.intersection(kidx_other).sort_values())
+        self.assert_eq(
+            (pidx + 1).intersection(pidx_other), (kidx + 1).intersection(kidx_other).sort_values()
+        )
+
+        pidx_other_different_name = pd.Index([3, 4, 5, 6], name="Databricks")
+        kidx_other_different_name = ps.from_pandas(pidx_other_different_name)
+        self.assert_eq(
+            pidx.intersection(pidx_other_different_name),
+            kidx.intersection(kidx_other_different_name).sort_values(),
+        )
+        self.assert_eq(
+            (pidx + 1).intersection(pidx_other_different_name),
+            (kidx + 1).intersection(kidx_other_different_name).sort_values(),
+        )
+
+        pidx_other_from_frame = pd.DataFrame({"a": [3, 4, 5, 6]}).set_index("a").index
+        kidx_other_from_frame = ps.from_pandas(pidx_other_from_frame)
+        self.assert_eq(
+            pidx.intersection(pidx_other_from_frame),
+            kidx.intersection(kidx_other_from_frame).sort_values(),
+        )
+        self.assert_eq(
+            (pidx + 1).intersection(pidx_other_from_frame),
+            (kidx + 1).intersection(kidx_other_from_frame).sort_values(),
+        )
+
+        # other = MultiIndex
+        pmidx = pd.MultiIndex.from_tuples([("a", "x"), ("b", "y"), ("c", "z")])
+        kmidx = ps.from_pandas(pmidx)
+        if LooseVersion(pd.__version__) < LooseVersion("1.2.0"):
+            self.assert_eq(
+                kidx.intersection(kmidx).sort_values(),
+                kidx._kdf.head(0).index.rename(None),
+                almost=True,
+            )
+            self.assert_eq(
+                (kidx + 1).intersection(kmidx).sort_values(),
+                kidx._kdf.head(0).index.rename(None),
+                almost=True,
+            )
+        else:
+            self.assert_eq(
+                pidx.intersection(pmidx), kidx.intersection(kmidx).sort_values(), almost=True
+            )
+            self.assert_eq(
+                (pidx + 1).intersection(pmidx),
+                (kidx + 1).intersection(kmidx).sort_values(),
+                almost=True,
+            )
+
+        # other = Series
+        pser = pd.Series([3, 4, 5, 6])
+        kser = ps.from_pandas(pser)
+        if LooseVersion(pd.__version__) < LooseVersion("1.2.0"):
+            self.assert_eq(kidx.intersection(kser).sort_values(), ps.Index([3, 4], name="Koalas"))
+            self.assert_eq(
+                (kidx + 1).intersection(kser).sort_values(), ps.Index([3, 4, 5], name="Koalas")
+            )
+        else:
+            self.assert_eq(pidx.intersection(pser), kidx.intersection(kser).sort_values())
+            self.assert_eq(
+                (pidx + 1).intersection(pser), (kidx + 1).intersection(kser).sort_values()
+            )
+
+        pser_different_name = pd.Series([3, 4, 5, 6], name="Databricks")
+        kser_different_name = ps.from_pandas(pser_different_name)
+        if LooseVersion(pd.__version__) < LooseVersion("1.2.0"):
+            self.assert_eq(
+                kidx.intersection(kser_different_name).sort_values(),
+                ps.Index([3, 4], name="Koalas"),
+            )
+            self.assert_eq(
+                (kidx + 1).intersection(kser_different_name).sort_values(),
+                ps.Index([3, 4, 5], name="Koalas"),
+            )
+        else:
+            self.assert_eq(
+                pidx.intersection(pser_different_name),
+                kidx.intersection(kser_different_name).sort_values(),
+            )
+            self.assert_eq(
+                (pidx + 1).intersection(pser_different_name),
+                (kidx + 1).intersection(kser_different_name).sort_values(),
+            )
+
+        others = ([3, 4, 5, 6], (3, 4, 5, 6), {3: None, 4: None, 5: None, 6: None})
+        for other in others:
+            if LooseVersion(pd.__version__) < LooseVersion("1.2.0"):
+                self.assert_eq(
+                    kidx.intersection(other).sort_values(), ps.Index([3, 4], name="Koalas")
+                )
+                self.assert_eq(
+                    (kidx + 1).intersection(other).sort_values(), ps.Index([3, 4, 5], name="Koalas")
+                )
+            else:
+                self.assert_eq(pidx.intersection(other), kidx.intersection(other).sort_values())
+                self.assert_eq(
+                    (pidx + 1).intersection(other), (kidx + 1).intersection(other).sort_values()
+                )
+
+        # MultiIndex / other = Index
+        self.assert_eq(
+            pmidx.intersection(pidx), kmidx.intersection(kidx).sort_values(), almost=True
+        )
+        self.assert_eq(
+            pmidx.intersection(pidx_other_from_frame),
+            kmidx.intersection(kidx_other_from_frame).sort_values(),
+            almost=True,
+        )
+
+        # MultiIndex / other = MultiIndex
+        pmidx_other = pd.MultiIndex.from_tuples([("c", "z"), ("d", "w")])
+        kmidx_other = ps.from_pandas(pmidx_other)
+        self.assert_eq(
+            pmidx.intersection(pmidx_other), kmidx.intersection(kmidx_other).sort_values()
+        )
+
+        # MultiIndex / other = list
+        other = [("c", "z"), ("d", "w")]
+        self.assert_eq(pmidx.intersection(other), kmidx.intersection(other).sort_values())
+
+        # MultiIndex / other = tuple
+        other = (("c", "z"), ("d", "w"))
+        self.assert_eq(pmidx.intersection(other), kmidx.intersection(other).sort_values())
+
+        # MultiIndex / other = dict
+        other = {("c", "z"): None, ("d", "w"): None}
+        self.assert_eq(pmidx.intersection(other), kmidx.intersection(other).sort_values())
+
+        with self.assertRaisesRegex(TypeError, "Input must be Index or array-like"):
+            kidx.intersection(4)
+        with self.assertRaisesRegex(TypeError, "other must be a MultiIndex or a list of tuples"):
+            kmidx.intersection(4)
+        with self.assertRaisesRegex(TypeError, "other must be a MultiIndex or a list of tuples"):
+            kmidx.intersection(ps.Series([3, 4, 5, 6]))
+        with self.assertRaisesRegex(ValueError, "Index data must be 1-dimensional"):
+            kidx.intersection(ps.DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]}))
+        with self.assertRaisesRegex(ValueError, "Index data must be 1-dimensional"):
+            kmidx.intersection(ps.DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]}))
+
+    def test_item(self):
+        pidx = pd.Index([10])
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(pidx.item(), kidx.item())
+
+        # with timestamp
+        pidx = pd.Index([datetime(1990, 3, 9)])
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(pidx.item(), kidx.item())
+
+        # MultiIndex
+        pmidx = pd.MultiIndex.from_tuples([("a", "x")])
+        kmidx = ps.from_pandas(pmidx)
+
+        self.assert_eq(pmidx.item(), kmidx.item())
+
+        # MultiIndex with timestamp
+        pmidx = pd.MultiIndex.from_tuples([(datetime(1990, 3, 9), datetime(2019, 8, 15))])
+        kmidx = ps.from_pandas(pmidx)
+
+        self.assert_eq(pidx.item(), kidx.item())
+
+        err_msg = "can only convert an array of size 1 to a Python scalar"
+        with self.assertRaisesRegex(ValueError, err_msg):
+            ps.Index([10, 20]).item()
+        with self.assertRaisesRegex(ValueError, err_msg):
+            ps.MultiIndex.from_tuples([("a", "x"), ("b", "y")]).item()
+
+    def test_inferred_type(self):
+        # Integer
+        pidx = pd.Index([1, 2, 3])
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.inferred_type, kidx.inferred_type)
+
+        # Floating
+        pidx = pd.Index([1.0, 2.0, 3.0])
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.inferred_type, kidx.inferred_type)
+
+        # String
+        pidx = pd.Index(["a", "b", "c"])
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.inferred_type, kidx.inferred_type)
+
+        # Boolean
+        pidx = pd.Index([True, False, True, False])
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.inferred_type, kidx.inferred_type)
+
+        # MultiIndex
+        pmidx = pd.MultiIndex.from_tuples([("a", "x")])
+        kmidx = ps.from_pandas(pmidx)
+        self.assert_eq(pmidx.inferred_type, kmidx.inferred_type)
+
+    def test_multi_index_from_index(self):
+        tuples = [(1, "red"), (1, "blue"), (2, "red"), (2, "blue")]
+        pmidx = pd.Index(tuples)
+        kmidx = ps.Index(tuples)
+
+        self.assertTrue(isinstance(kmidx, ps.MultiIndex))
+        self.assert_eq(pmidx, kmidx)
+
+        # Specify the `names`
+        pmidx = pd.Index(tuples, names=["Hello", "Koalas"])
+        kmidx = ps.Index(tuples, names=["Hello", "Koalas"])
+
+        self.assertTrue(isinstance(kmidx, ps.MultiIndex))
+        self.assert_eq(pmidx, kmidx)
+
+    @unittest.skipIf(
+        LooseVersion(pd.__version__) < LooseVersion("0.24"),
+        "MultiIndex.from_frame is new in pandas 0.24",
+    )
+    def test_multiindex_from_frame(self):
+        pdf = pd.DataFrame(
+            [["HI", "Temp"], ["HI", "Precip"], ["NJ", "Temp"], ["NJ", "Precip"]], columns=["a", "b"]
+        )
+        kdf = ps.from_pandas(pdf)
+        pidx = pd.MultiIndex.from_frame(pdf)
+        kidx = ps.MultiIndex.from_frame(kdf)
+
+        self.assert_eq(pidx, kidx)
+
+        # Specify `names`
+        pidx = pd.MultiIndex.from_frame(pdf, names=["state", "observation"])
+        kidx = ps.MultiIndex.from_frame(kdf, names=["state", "observation"])
+        self.assert_eq(pidx, kidx)
+
+        pidx = pd.MultiIndex.from_frame(pdf, names=("state", "observation"))
+        kidx = ps.MultiIndex.from_frame(kdf, names=("state", "observation"))
+        self.assert_eq(pidx, kidx)
+
+        # MultiIndex columns
+        pidx = pd.MultiIndex.from_tuples([("a", "w"), ("b", "x")])
+        pdf.columns = pidx
+        kdf = ps.from_pandas(pdf)
+
+        pidx = pd.MultiIndex.from_frame(pdf)
+        kidx = ps.MultiIndex.from_frame(kdf)
+
+        self.assert_eq(pidx, kidx)
+
+        # tuples for names
+        pidx = pd.MultiIndex.from_frame(pdf, names=[("a", "w"), ("b", "x")])
+        kidx = ps.MultiIndex.from_frame(kdf, names=[("a", "w"), ("b", "x")])
+
+        self.assert_eq(pidx, kidx)
+
+        err_msg = "Input must be a DataFrame"
+        with self.assertRaisesRegex(TypeError, err_msg):
+            ps.MultiIndex.from_frame({"a": [1, 2, 3], "b": [4, 5, 6]})
+
+        self.assertRaises(TypeError, lambda: ps.MultiIndex.from_frame(kdf, names="ab"))
+
+        # non-string names
+        self.assert_eq(
+            ps.MultiIndex.from_frame(kdf, names=[0, 1]), pd.MultiIndex.from_frame(pdf, names=[0, 1])
+        )
+        self.assert_eq(
+            ps.MultiIndex.from_frame(kdf, names=[("x", 0), ("y", 1)]),
+            pd.MultiIndex.from_frame(pdf, names=[("x", 0), ("y", 1)]),
+        )
+
+        pdf = pd.DataFrame([["HI", "Temp"], ["HI", "Precip"], ["NJ", "Temp"], ["NJ", "Precip"]])
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(ps.MultiIndex.from_frame(kdf), pd.MultiIndex.from_frame(pdf))
+
+    def test_is_type_compatible(self):
+        data_types = ["integer", "floating", "string", "boolean"]
+        # Integer
+        pidx = pd.Index([1, 2, 3])
+        kidx = ps.from_pandas(pidx)
+        for data_type in data_types:
+            self.assert_eq(pidx.is_type_compatible(data_type), kidx.is_type_compatible(data_type))
+
+        # Floating
+        pidx = pd.Index([1.0, 2.0, 3.0])
+        kidx = ps.from_pandas(pidx)
+        for data_type in data_types:
+            self.assert_eq(pidx.is_type_compatible(data_type), kidx.is_type_compatible(data_type))
+
+        # String
+        pidx = pd.Index(["a", "b", "c"])
+        kidx = ps.from_pandas(pidx)
+        for data_type in data_types:
+            self.assert_eq(pidx.is_type_compatible(data_type), kidx.is_type_compatible(data_type))
+
+        # Boolean
+        pidx = pd.Index([True, False, True, False])
+        kidx = ps.from_pandas(pidx)
+        for data_type in data_types:
+            self.assert_eq(pidx.is_type_compatible(data_type), kidx.is_type_compatible(data_type))
+
+        # MultiIndex
+        pmidx = pd.MultiIndex.from_tuples([("a", "x")])
+        kmidx = ps.from_pandas(pmidx)
+        for data_type in data_types:
+            self.assert_eq(pmidx.is_type_compatible(data_type), kmidx.is_type_compatible(data_type))
+
+    def test_asi8(self):
+        # Integer
+        pidx = pd.Index([1, 2, 3])
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.asi8, kidx.asi8)
+        self.assert_eq(pidx.astype("int").asi8, kidx.astype("int").asi8)
+        self.assert_eq(pidx.astype("int16").asi8, kidx.astype("int16").asi8)
+        self.assert_eq(pidx.astype("int8").asi8, kidx.astype("int8").asi8)
+
+        # Integer with missing value
+        pidx = pd.Index([1, 2, None, 4, 5])
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.asi8, kidx.asi8)
+
+        # Datetime
+        pidx = pd.date_range(end="1/1/2018", periods=3)
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.asi8, kidx.asi8)
+
+        # Floating
+        pidx = pd.Index([1.0, 2.0, 3.0])
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.asi8, kidx.asi8)
+
+        # String
+        pidx = pd.Index(["a", "b", "c"])
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.asi8, kidx.asi8)
+
+        # Boolean
+        pidx = pd.Index([True, False, True, False])
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.asi8, kidx.asi8)
+
+        # MultiIndex
+        pmidx = pd.MultiIndex.from_tuples([(1, 2)])
+        kmidx = ps.from_pandas(pmidx)
+        self.assert_eq(pmidx.asi8, kmidx.asi8)
+
+    def test_index_is_unique(self):
+        indexes = [("a", "b", "c"), ("a", "a", "c"), (1, 3, 3), (1, 2, 3)]
+        names = [None, "ks", "ks", None]
+        is_uniq = [True, False, False, True]
+
+        for idx, name, expected in zip(indexes, names, is_uniq):
+            pdf = pd.DataFrame({"a": [1, 2, 3]}, index=pd.Index(idx, name=name))
+            kdf = ps.from_pandas(pdf)
+
+            self.assertEqual(kdf.index.is_unique, expected)
+
+    def test_multiindex_is_unique(self):
+        indexes = [
+            [list("abc"), list("edf")],
+            [list("aac"), list("edf")],
+            [list("aac"), list("eef")],
+            [[1, 4, 4], [4, 6, 6]],
+        ]
+        is_uniq = [True, True, False, False]
+
+        for idx, expected in zip(indexes, is_uniq):
+            pdf = pd.DataFrame({"a": [1, 2, 3]}, index=idx)
+            kdf = ps.from_pandas(pdf)
+
+            self.assertEqual(kdf.index.is_unique, expected)
+
+    def test_view(self):
+        pidx = pd.Index([1, 2, 3, 4], name="Koalas")
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(pidx.view(), kidx.view())
+
+        # MultiIndex
+        pmidx = pd.MultiIndex.from_tuples([("a", "x"), ("b", "y"), ("c", "z")])
+        kmidx = ps.from_pandas(pmidx)
+
+        self.assert_eq(pmidx.view(), kmidx.view())
+
+    def test_insert(self):
+        # Integer
+        pidx = pd.Index([1, 2, 3], name="Koalas")
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.insert(1, 100), kidx.insert(1, 100))
+        self.assert_eq(pidx.insert(-1, 100), kidx.insert(-1, 100))
+        self.assert_eq(pidx.insert(100, 100), kidx.insert(100, 100))
+        self.assert_eq(pidx.insert(-100, 100), kidx.insert(-100, 100))
+
+        # Floating
+        pidx = pd.Index([1.0, 2.0, 3.0], name="Koalas")
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.insert(1, 100.0), kidx.insert(1, 100.0))
+        self.assert_eq(pidx.insert(-1, 100.0), kidx.insert(-1, 100.0))
+        self.assert_eq(pidx.insert(100, 100.0), kidx.insert(100, 100.0))
+        self.assert_eq(pidx.insert(-100, 100.0), kidx.insert(-100, 100.0))
+
+        # String
+        pidx = pd.Index(["a", "b", "c"], name="Koalas")
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.insert(1, "x"), kidx.insert(1, "x"))
+        self.assert_eq(pidx.insert(-1, "x"), kidx.insert(-1, "x"))
+        self.assert_eq(pidx.insert(100, "x"), kidx.insert(100, "x"))
+        self.assert_eq(pidx.insert(-100, "x"), kidx.insert(-100, "x"))
+
+        # Boolean
+        pidx = pd.Index([True, False, True, False], name="Koalas")
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(pidx.insert(1, True), kidx.insert(1, True))
+        self.assert_eq(pidx.insert(-1, True), kidx.insert(-1, True))
+        self.assert_eq(pidx.insert(100, True), kidx.insert(100, True))
+        self.assert_eq(pidx.insert(-100, True), kidx.insert(-100, True))
+
+        # MultiIndex
+        pmidx = pd.MultiIndex.from_tuples(
+            [("a", "x"), ("b", "y"), ("c", "z")], names=["Hello", "Koalas"]
+        )
+        kmidx = ps.from_pandas(pmidx)
+        self.assert_eq(pmidx.insert(2, ("h", "j")), kmidx.insert(2, ("h", "j")))
+        self.assert_eq(pmidx.insert(-1, ("h", "j")), kmidx.insert(-1, ("h", "j")))
+
+        err_msg = "index 4 is out of bounds for axis 0 with size 3"
+        with self.assertRaisesRegex(IndexError, err_msg):
+            kmidx.insert(4, ("b", "y"))
+
+    def test_astype(self):
+        pidx = pd.Index([10, 20, 15, 30, 45], name="x")
+        kidx = ps.Index(pidx)
+
+        self.assert_eq(kidx.astype(int), pidx.astype(int))
+        self.assert_eq(kidx.astype(np.int), pidx.astype(np.int))
+        self.assert_eq(kidx.astype(np.int8), pidx.astype(np.int8))
+        self.assert_eq(kidx.astype(np.int16), pidx.astype(np.int16))
+        self.assert_eq(kidx.astype(np.int32), pidx.astype(np.int32))
+        self.assert_eq(kidx.astype(np.int64), pidx.astype(np.int64))
+        self.assert_eq(kidx.astype(np.byte), pidx.astype(np.byte))
+        self.assert_eq(kidx.astype("int"), pidx.astype("int"))
+        self.assert_eq(kidx.astype("int8"), pidx.astype("int8"))
+        self.assert_eq(kidx.astype("int16"), pidx.astype("int16"))
+        self.assert_eq(kidx.astype("int32"), pidx.astype("int32"))
+        self.assert_eq(kidx.astype("int64"), pidx.astype("int64"))
+        self.assert_eq(kidx.astype("b"), pidx.astype("b"))
+        self.assert_eq(kidx.astype("byte"), pidx.astype("byte"))
+        self.assert_eq(kidx.astype("i"), pidx.astype("i"))
+        self.assert_eq(kidx.astype("long"), pidx.astype("long"))
+        self.assert_eq(kidx.astype("short"), pidx.astype("short"))
+        self.assert_eq(kidx.astype(np.float), pidx.astype(np.float))
+        self.assert_eq(kidx.astype(np.float32), pidx.astype(np.float32))
+        self.assert_eq(kidx.astype(np.float64), pidx.astype(np.float64))
+        self.assert_eq(kidx.astype("float"), pidx.astype("float"))
+        self.assert_eq(kidx.astype("float32"), pidx.astype("float32"))
+        self.assert_eq(kidx.astype("float64"), pidx.astype("float64"))
+        self.assert_eq(kidx.astype("double"), pidx.astype("double"))
+        self.assert_eq(kidx.astype("f"), pidx.astype("f"))
+        self.assert_eq(kidx.astype(bool), pidx.astype(bool))
+        self.assert_eq(kidx.astype("bool"), pidx.astype("bool"))
+        self.assert_eq(kidx.astype("?"), pidx.astype("?"))
+        self.assert_eq(kidx.astype(np.unicode_), pidx.astype(np.unicode_))
+        self.assert_eq(kidx.astype("str"), pidx.astype("str"))
+        self.assert_eq(kidx.astype("U"), pidx.astype("U"))
+
+        pidx = pd.Index([10, 20, 15, 30, 45, None], name="x")
+        kidx = ps.Index(pidx)
+
+        pidx = pd.Index(["hi", "hi ", " ", " \t", "", None], name="x")
+        kidx = ps.Index(pidx)
+
+        self.assert_eq(kidx.astype(bool), pidx.astype(bool))
+        self.assert_eq(kidx.astype(str).to_numpy(), ["hi", "hi ", " ", " \t", "", "None"])
+
+        pidx = pd.Index([True, False, None], name="x")
+        kidx = ps.Index(pidx)
+
+        self.assert_eq(kidx.astype(bool), pidx.astype(bool))
+
+        pidx = pd.Index(["2020-10-27"], name="x")
+        kidx = ps.Index(pidx)
+
+        self.assert_eq(kidx.astype("datetime64[ns]"), pidx.astype("datetime64[ns]"))
+
+        with self.assertRaisesRegex(TypeError, "not understood"):
+            kidx.astype("int63")
+
+    def test_to_list(self):
+        # Index
+        pidx = pd.Index([1, 2, 3, 4, 5])
+        kidx = ps.from_pandas(pidx)
+        # MultiIndex
+        tuples = [(1, "red"), (1, "blue"), (2, "red"), (2, "green")]
+        pmidx = pd.MultiIndex.from_tuples(tuples)
+        kmidx = ps.from_pandas(pmidx)
+
+        self.assert_eq(kidx.tolist(), pidx.tolist())
+        self.assert_eq(kmidx.tolist(), pmidx.tolist())
+
+    def test_index_ops(self):
+        pidx = pd.Index([1, 2, 3, 4, 5])
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(kidx * 100 + kidx * 10 + kidx, pidx * 100 + pidx * 10 + pidx)
+
+        pidx = pd.Index([1, 2, 3, 4, 5], name="a")
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(kidx * 100 + kidx * 10 + kidx, pidx * 100 + pidx * 10 + pidx)
+
+        pdf = pd.DataFrame(
+            index=pd.MultiIndex.from_tuples([(1, 2), (3, 4), (5, 6)], names=["a", "b"])
+        )
+        kdf = ps.from_pandas(pdf)
+
+        pidx1 = pdf.index.get_level_values(0)
+        pidx2 = pdf.index.get_level_values(1)
+        kidx1 = kdf.index.get_level_values(0)
+        kidx2 = kdf.index.get_level_values(1)
+
+        if LooseVersion(pd.__version__) >= LooseVersion("1.0"):
+            self.assert_eq(kidx1 * 10 + kidx2, pidx1 * 10 + pidx2)
+        else:
+            self.assert_eq(kidx1 * 10 + kidx2, (pidx1 * 10 + pidx2).rename(None))
+
+    def test_factorize(self):
+        pidx = pd.Index(["a", "b", "a", "b"])
+        kidx = ps.from_pandas(pidx)
+        pcodes, puniques = pidx.factorize(sort=True)
+        kcodes, kuniques = kidx.factorize()
+        self.assert_eq(pcodes.tolist(), kcodes.to_list())
+        self.assert_eq(puniques, kuniques)
+
+        pmidx = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "c")])
+        kmidx = ps.from_pandas(pmidx)
+
+        self.assertRaises(PandasNotImplementedError, lambda: kmidx.factorize())
+
+
+if __name__ == "__main__":
+    from pyspark.pandas.tests.indexes.test_base import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/indexes/test_category.py b/python/pyspark/pandas/tests/indexes/test_category.py
new file mode 100644
index 0000000000000..360e31863dfca
--- /dev/null
+++ b/python/pyspark/pandas/tests/indexes/test_category.py
@@ -0,0 +1,124 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from distutils.version import LooseVersion
+
+import pandas as pd
+from pandas.api.types import CategoricalDtype
+
+import pyspark.pandas as ps
+from pyspark.testing.pandasutils import PandasOnSparkTestCase, TestUtils
+
+
+class CategoricalIndexTest(PandasOnSparkTestCase, TestUtils):
+    def test_categorical_index(self):
+        pidx = pd.CategoricalIndex([1, 2, 3])
+        kidx = ps.CategoricalIndex([1, 2, 3])
+
+        self.assert_eq(kidx, pidx)
+        self.assert_eq(kidx.categories, pidx.categories)
+        self.assert_eq(kidx.codes, pd.Index(pidx.codes))
+        self.assert_eq(kidx.ordered, pidx.ordered)
+
+        pidx = pd.Index([1, 2, 3], dtype="category")
+        kidx = ps.Index([1, 2, 3], dtype="category")
+
+        self.assert_eq(kidx, pidx)
+        self.assert_eq(kidx.categories, pidx.categories)
+        self.assert_eq(kidx.codes, pd.Index(pidx.codes))
+        self.assert_eq(kidx.ordered, pidx.ordered)
+
+        pdf = pd.DataFrame(
+            {
+                "a": pd.Categorical([1, 2, 3, 1, 2, 3]),
+                "b": pd.Categorical(["a", "b", "c", "a", "b", "c"], categories=["c", "b", "a"]),
+            },
+            index=pd.Categorical([10, 20, 30, 20, 30, 10], categories=[30, 10, 20], ordered=True),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        pidx = pdf.set_index("b").index
+        kidx = kdf.set_index("b").index
+
+        self.assert_eq(kidx, pidx)
+        self.assert_eq(kidx.categories, pidx.categories)
+        self.assert_eq(kidx.codes, pd.Index(pidx.codes))
+        self.assert_eq(kidx.ordered, pidx.ordered)
+
+        pidx = pdf.set_index(["a", "b"]).index.get_level_values(0)
+        kidx = kdf.set_index(["a", "b"]).index.get_level_values(0)
+
+        self.assert_eq(kidx, pidx)
+        self.assert_eq(kidx.categories, pidx.categories)
+        self.assert_eq(kidx.codes, pd.Index(pidx.codes))
+        self.assert_eq(kidx.ordered, pidx.ordered)
+
+    def test_astype(self):
+        pidx = pd.Index(["a", "b", "c"])
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(kidx.astype("category"), pidx.astype("category"))
+        self.assert_eq(
+            kidx.astype(CategoricalDtype(["c", "a", "b"])),
+            pidx.astype(CategoricalDtype(["c", "a", "b"])),
+        )
+
+        pcidx = pidx.astype(CategoricalDtype(["c", "a", "b"]))
+        kcidx = kidx.astype(CategoricalDtype(["c", "a", "b"]))
+
+        self.assert_eq(kcidx.astype("category"), pcidx.astype("category"))
+
+        if LooseVersion(pd.__version__) >= LooseVersion("1.2"):
+            self.assert_eq(
+                kcidx.astype(CategoricalDtype(["b", "c", "a"])),
+                pcidx.astype(CategoricalDtype(["b", "c", "a"])),
+            )
+        else:
+            self.assert_eq(
+                kcidx.astype(CategoricalDtype(["b", "c", "a"])),
+                pidx.astype(CategoricalDtype(["b", "c", "a"])),
+            )
+
+        self.assert_eq(kcidx.astype(str), pcidx.astype(str))
+
+    def test_factorize(self):
+        pidx = pd.CategoricalIndex([1, 2, 3, None])
+        kidx = ps.from_pandas(pidx)
+
+        pcodes, puniques = pidx.factorize()
+        kcodes, kuniques = kidx.factorize()
+
+        self.assert_eq(kcodes.tolist(), pcodes.tolist())
+        self.assert_eq(kuniques, puniques)
+
+        pcodes, puniques = pidx.factorize(na_sentinel=-2)
+        kcodes, kuniques = kidx.factorize(na_sentinel=-2)
+
+        self.assert_eq(kcodes.tolist(), pcodes.tolist())
+        self.assert_eq(kuniques, puniques)
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.pandas.tests.indexes.test_category import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/indexes/test_datetime.py b/python/pyspark/pandas/tests/indexes/test_datetime.py
new file mode 100644
index 0000000000000..af511ed6c214d
--- /dev/null
+++ b/python/pyspark/pandas/tests/indexes/test_datetime.py
@@ -0,0 +1,232 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import datetime
+
+from distutils.version import LooseVersion
+
+import pandas as pd
+
+import pyspark.pandas as ps
+from pyspark.testing.pandasutils import PandasOnSparkTestCase, TestUtils
+
+
+class DatetimeIndexTest(PandasOnSparkTestCase, TestUtils):
+    @property
+    def fixed_freqs(self):
+        return [
+            "D",
+            "H",
+            "T",  # min
+            "S",
+            "L",  # ms
+            "U",  # us
+            # 'N' not supported
+        ]
+
+    @property
+    def non_fixed_freqs(self):
+        return ["W", "Q"]
+
+    @property
+    def pidxs(self):
+        return [
+            pd.DatetimeIndex([0]),
+            pd.DatetimeIndex(["2004-01-01", "2002-12-31", "2000-04-01"]),
+        ] + [
+            pd.date_range("2000-01-01", periods=3, freq=freq)
+            for freq in (self.fixed_freqs + self.non_fixed_freqs)
+        ]
+
+    @property
+    def kidxs(self):
+        return [ps.from_pandas(pidx) for pidx in self.pidxs]
+
+    @property
+    def idx_pairs(self):
+        return list(zip(self.kidxs, self.pidxs))
+
+    def _disallow_nanoseconds(self, f):
+        self.assertRaises(ValueError, lambda: f(freq="ns"))
+        self.assertRaises(ValueError, lambda: f(freq="N"))
+
+    def test_properties(self):
+        for kidx, pidx in self.idx_pairs:
+            self.assert_eq(kidx.year, pidx.year)
+            self.assert_eq(kidx.month, pidx.month)
+            self.assert_eq(kidx.day, pidx.day)
+            self.assert_eq(kidx.hour, pidx.hour)
+            self.assert_eq(kidx.minute, pidx.minute)
+            self.assert_eq(kidx.second, pidx.second)
+            self.assert_eq(kidx.microsecond, pidx.microsecond)
+            self.assert_eq(kidx.week, pidx.week)
+            self.assert_eq(kidx.weekofyear, pidx.weekofyear)
+            self.assert_eq(kidx.dayofweek, pidx.dayofweek)
+            self.assert_eq(kidx.weekday, pidx.weekday)
+            self.assert_eq(kidx.dayofyear, pidx.dayofyear)
+            self.assert_eq(kidx.quarter, pidx.quarter)
+            self.assert_eq(kidx.daysinmonth, pidx.daysinmonth)
+            self.assert_eq(kidx.days_in_month, pidx.days_in_month)
+            self.assert_eq(kidx.is_month_start, pd.Index(pidx.is_month_start))
+            self.assert_eq(kidx.is_month_end, pd.Index(pidx.is_month_end))
+            self.assert_eq(kidx.is_quarter_start, pd.Index(pidx.is_quarter_start))
+            self.assert_eq(kidx.is_quarter_end, pd.Index(pidx.is_quarter_end))
+            self.assert_eq(kidx.is_year_start, pd.Index(pidx.is_year_start))
+            self.assert_eq(kidx.is_year_end, pd.Index(pidx.is_year_end))
+            self.assert_eq(kidx.is_leap_year, pd.Index(pidx.is_leap_year))
+
+            if LooseVersion(pd.__version__) >= LooseVersion("1.2.0"):
+                self.assert_eq(kidx.day_of_year, pidx.day_of_year)
+                self.assert_eq(kidx.day_of_week, pidx.day_of_week)
+
+    def test_ceil(self):
+        for kidx, pidx in self.idx_pairs:
+            for freq in self.fixed_freqs:
+                self.assert_eq(kidx.ceil(freq), pidx.ceil(freq))
+
+        self._disallow_nanoseconds(self.kidxs[0].ceil)
+
+    def test_floor(self):
+        for kidx, pidx in self.idx_pairs:
+            for freq in self.fixed_freqs:
+                self.assert_eq(kidx.floor(freq), pidx.floor(freq))
+
+        self._disallow_nanoseconds(self.kidxs[0].floor)
+
+    def test_round(self):
+        for kidx, pidx in self.idx_pairs:
+            for freq in self.fixed_freqs:
+                self.assert_eq(kidx.round(freq), pidx.round(freq))
+
+        self._disallow_nanoseconds(self.kidxs[0].round)
+
+    def test_day_name(self):
+        for kidx, pidx in self.idx_pairs:
+            self.assert_eq(kidx.day_name(), pidx.day_name())
+
+    def test_month_name(self):
+        for kidx, pidx in self.idx_pairs:
+            self.assert_eq(kidx.day_name(), pidx.day_name())
+
+    def test_normalize(self):
+        for kidx, pidx in self.idx_pairs:
+            self.assert_eq(kidx.normalize(), pidx.normalize())
+
+    def test_strftime(self):
+        for kidx, pidx in self.idx_pairs:
+            self.assert_eq(
+                kidx.strftime(date_format="%B %d, %Y"), pidx.strftime(date_format="%B %d, %Y")
+            )
+
+    def test_indexer_between_time(self):
+        for kidx, pidx in self.idx_pairs:
+            self.assert_eq(
+                kidx.indexer_between_time("00:00:00", "00:01:00").sort_values(),
+                pd.Index(pidx.indexer_between_time("00:00:00", "00:01:00")),
+            )
+
+            self.assert_eq(
+                kidx.indexer_between_time(
+                    datetime.time(0, 0, 0), datetime.time(0, 1, 0)
+                ).sort_values(),
+                pd.Index(pidx.indexer_between_time(datetime.time(0, 0, 0), datetime.time(0, 1, 0))),
+            )
+
+            self.assert_eq(
+                kidx.indexer_between_time("00:00:00", "00:01:00", True, False).sort_values(),
+                pd.Index(pidx.indexer_between_time("00:00:00", "00:01:00", True, False)),
+            )
+
+            self.assert_eq(
+                kidx.indexer_between_time("00:00:00", "00:01:00", False, True).sort_values(),
+                pd.Index(pidx.indexer_between_time("00:00:00", "00:01:00", False, True)),
+            )
+
+            self.assert_eq(
+                kidx.indexer_between_time("00:00:00", "00:01:00", False, False).sort_values(),
+                pd.Index(pidx.indexer_between_time("00:00:00", "00:01:00", False, False)),
+            )
+
+            self.assert_eq(
+                kidx.indexer_between_time("00:00:00", "00:01:00", True, True).sort_values(),
+                pd.Index(pidx.indexer_between_time("00:00:00", "00:01:00", True, True)),
+            )
+
+    def test_indexer_at_time(self):
+        for kidx, pidx in self.idx_pairs:
+            self.assert_eq(
+                kidx.indexer_at_time("00:00:00").sort_values(),
+                pd.Index(pidx.indexer_at_time("00:00:00")),
+            )
+
+            self.assert_eq(
+                kidx.indexer_at_time(datetime.time(0, 1, 0)).sort_values(),
+                pd.Index(pidx.indexer_at_time(datetime.time(0, 1, 0))),
+            )
+
+            self.assert_eq(
+                kidx.indexer_at_time("00:00:01").sort_values(),
+                pd.Index(pidx.indexer_at_time("00:00:01")),
+            )
+
+        self.assertRaises(
+            NotImplementedError,
+            lambda: ps.DatetimeIndex([0]).indexer_at_time("00:00:00", asof=True),
+        )
+
+    def test_arithmetic_op_exceptions(self):
+        for kidx, pidx in self.idx_pairs:
+            py_datetime = pidx.to_pydatetime()
+            for other in [1, 0.1, kidx, kidx.to_series().reset_index(drop=True), py_datetime]:
+                expected_err_msg = "addition can not be applied to date times."
+                self.assertRaisesRegex(TypeError, expected_err_msg, lambda: kidx + other)
+                self.assertRaisesRegex(TypeError, expected_err_msg, lambda: other + kidx)
+
+                expected_err_msg = "multiplication can not be applied to date times."
+                self.assertRaisesRegex(TypeError, expected_err_msg, lambda: kidx * other)
+                self.assertRaisesRegex(TypeError, expected_err_msg, lambda: other * kidx)
+
+                expected_err_msg = "division can not be applied to date times."
+                self.assertRaisesRegex(TypeError, expected_err_msg, lambda: kidx / other)
+                self.assertRaisesRegex(TypeError, expected_err_msg, lambda: other / kidx)
+                self.assertRaisesRegex(TypeError, expected_err_msg, lambda: kidx // other)
+                self.assertRaisesRegex(TypeError, expected_err_msg, lambda: other // kidx)
+
+                expected_err_msg = "modulo can not be applied to date times."
+                self.assertRaisesRegex(TypeError, expected_err_msg, lambda: kidx % other)
+                self.assertRaisesRegex(TypeError, expected_err_msg, lambda: other % kidx)
+
+            expected_err_msg = "datetime subtraction can only be applied to datetime series."
+
+            for other in [1, 0.1]:
+                self.assertRaisesRegex(TypeError, expected_err_msg, lambda: kidx - other)
+                self.assertRaisesRegex(TypeError, expected_err_msg, lambda: other - kidx)
+
+            self.assertRaisesRegex(TypeError, expected_err_msg, lambda: kidx - other)
+            self.assertRaises(NotImplementedError, lambda: py_datetime - kidx)
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.pandas.tests.indexes.test_datetime import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/plot/__init__.py b/python/pyspark/pandas/tests/plot/__init__.py
new file mode 100644
index 0000000000000..cce3acad34a49
--- /dev/null
+++ b/python/pyspark/pandas/tests/plot/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
diff --git a/python/pyspark/pandas/tests/plot/test_frame_plot.py b/python/pyspark/pandas/tests/plot/test_frame_plot.py
new file mode 100644
index 0000000000000..b57acd495939e
--- /dev/null
+++ b/python/pyspark/pandas/tests/plot/test_frame_plot.py
@@ -0,0 +1,123 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import pandas as pd
+import numpy as np
+
+from pyspark import pandas as ps
+from pyspark.pandas.config import set_option, reset_option, option_context
+from pyspark.pandas.plot import TopNPlotBase, SampledPlotBase, HistogramPlotBase
+from pyspark.pandas.exceptions import PandasNotImplementedError
+from pyspark.testing.pandasutils import PandasOnSparkTestCase
+
+
+class DataFramePlotTest(PandasOnSparkTestCase):
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        set_option("plotting.max_rows", 2000)
+        set_option("plotting.sample_ratio", None)
+
+    @classmethod
+    def tearDownClass(cls):
+        reset_option("plotting.max_rows")
+        reset_option("plotting.sample_ratio")
+        super().tearDownClass()
+
+    def test_missing(self):
+        kdf = ps.DataFrame(np.random.rand(2500, 4), columns=["a", "b", "c", "d"])
+
+        unsupported_functions = ["box", "hexbin"]
+
+        for name in unsupported_functions:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "method.*DataFrame.*{}.*not implemented".format(name)
+            ):
+                getattr(kdf.plot, name)()
+
+    def test_topn_max_rows(self):
+
+        pdf = pd.DataFrame(np.random.rand(2500, 4), columns=["a", "b", "c", "d"])
+        kdf = ps.from_pandas(pdf)
+
+        data = TopNPlotBase().get_top_n(kdf)
+        self.assertEqual(len(data), 2000)
+
+    def test_sampled_plot_with_ratio(self):
+        with option_context("plotting.sample_ratio", 0.5):
+            pdf = pd.DataFrame(np.random.rand(2500, 4), columns=["a", "b", "c", "d"])
+            kdf = ps.from_pandas(pdf)
+            data = SampledPlotBase().get_sampled(kdf)
+            self.assertEqual(round(len(data) / 2500, 1), 0.5)
+
+    def test_sampled_plot_with_max_rows(self):
+        # 'plotting.max_rows' is 2000
+        pdf = pd.DataFrame(np.random.rand(2000, 4), columns=["a", "b", "c", "d"])
+        kdf = ps.from_pandas(pdf)
+        data = SampledPlotBase().get_sampled(kdf)
+        self.assertEqual(round(len(data) / 2000, 1), 1)
+
+    def test_compute_hist_single_column(self):
+        kdf = ps.DataFrame(
+            {"a": [1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 50]}, index=[0, 1, 3, 5, 6, 8, 9, 9, 9, 10, 10]
+        )
+
+        expected_bins = np.linspace(1, 50, 11)
+        bins = HistogramPlotBase.get_bins(kdf[["a"]].to_spark(), 10)
+
+        expected_histogram = np.array([5, 4, 1, 0, 0, 0, 0, 0, 0, 1])
+        histogram = HistogramPlotBase.compute_hist(kdf[["a"]], bins)[0]
+        self.assert_eq(pd.Series(expected_bins), pd.Series(bins))
+        self.assert_eq(pd.Series(expected_histogram, name="a"), histogram, almost=True)
+
+    def test_compute_hist_multi_columns(self):
+        expected_bins = np.linspace(1, 50, 11)
+        kdf = ps.DataFrame(
+            {
+                "a": [1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 50],
+                "b": [50, 50, 30, 30, 30, 24, 10, 5, 4, 3, 1],
+            }
+        )
+
+        bins = HistogramPlotBase.get_bins(kdf.to_spark(), 10)
+        self.assert_eq(pd.Series(expected_bins), pd.Series(bins))
+
+        expected_histograms = [
+            np.array([5, 4, 1, 0, 0, 0, 0, 0, 0, 1]),
+            np.array([4, 1, 0, 0, 1, 3, 0, 0, 0, 2]),
+        ]
+        histograms = HistogramPlotBase.compute_hist(kdf, bins)
+        expected_names = ["a", "b"]
+
+        for histogram, expected_histogram, expected_name in zip(
+            histograms, expected_histograms, expected_names
+        ):
+            self.assert_eq(
+                pd.Series(expected_histogram, name=expected_name), histogram, almost=True
+            )
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.pandas.tests.plot.test_frame_plot import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/plot/test_frame_plot_matplotlib.py b/python/pyspark/pandas/tests/plot/test_frame_plot_matplotlib.py
new file mode 100644
index 0000000000000..5de5c90856eef
--- /dev/null
+++ b/python/pyspark/pandas/tests/plot/test_frame_plot_matplotlib.py
@@ -0,0 +1,474 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import base64
+from distutils.version import LooseVersion
+from io import BytesIO
+import unittest
+
+import pandas as pd
+import numpy as np
+
+from pyspark import pandas as ps
+from pyspark.pandas.config import set_option, reset_option
+from pyspark.testing.pandasutils import (
+    have_matplotlib,
+    matplotlib_requirement_message,
+    PandasOnSparkTestCase,
+    TestUtils,
+)
+
+if have_matplotlib:
+    import matplotlib
+    from matplotlib import pyplot as plt
+
+    matplotlib.use("agg")
+
+
+@unittest.skipIf(not have_matplotlib, matplotlib_requirement_message)
+class DataFramePlotMatplotlibTest(PandasOnSparkTestCase, TestUtils):
+    sample_ratio_default = None
+
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        if LooseVersion(pd.__version__) >= LooseVersion("0.25"):
+            pd.set_option("plotting.backend", "matplotlib")
+        set_option("plotting.backend", "matplotlib")
+        set_option("plotting.max_rows", 2000)
+        set_option("plotting.sample_ratio", None)
+
+    @classmethod
+    def tearDownClass(cls):
+        if LooseVersion(pd.__version__) >= LooseVersion("0.25"):
+            pd.reset_option("plotting.backend")
+        reset_option("plotting.backend")
+        reset_option("plotting.max_rows")
+        reset_option("plotting.sample_ratio")
+        super().tearDownClass()
+
+    @property
+    def pdf1(self):
+        return pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 50], "b": [2, 3, 4, 5, 7, 9, 10, 15, 34, 45, 49]},
+            index=[0, 1, 3, 5, 6, 8, 9, 9, 9, 10, 10],
+        )
+
+    @property
+    def kdf1(self):
+        return ps.from_pandas(self.pdf1)
+
+    @staticmethod
+    def plot_to_base64(ax):
+        bytes_data = BytesIO()
+        ax.figure.savefig(bytes_data, format="png")
+        bytes_data.seek(0)
+        b64_data = base64.b64encode(bytes_data.read())
+        plt.close(ax.figure)
+        return b64_data
+
+    def test_line_plot(self):
+        def check_line_plot(pdf, kdf):
+            ax1 = pdf.plot(kind="line", colormap="Paired")
+            bin1 = self.plot_to_base64(ax1)
+            ax2 = kdf.plot(kind="line", colormap="Paired")
+            bin2 = self.plot_to_base64(ax2)
+            self.assertEqual(bin1, bin2)
+
+            ax3 = pdf.plot.line(colormap="Paired")
+            bin3 = self.plot_to_base64(ax3)
+            ax4 = kdf.plot.line(colormap="Paired")
+            bin4 = self.plot_to_base64(ax4)
+            self.assertEqual(bin3, bin4)
+
+        pdf1 = self.pdf1
+        kdf1 = self.kdf1
+        check_line_plot(pdf1, kdf1)
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("y", "b")])
+        pdf1.columns = columns
+        kdf1.columns = columns
+        check_line_plot(pdf1, kdf1)
+
+    def test_area_plot(self):
+        def check_area_plot(pdf, kdf):
+            ax1 = pdf.plot(kind="area", colormap="Paired")
+            bin1 = self.plot_to_base64(ax1)
+            ax2 = kdf.plot(kind="area", colormap="Paired")
+            bin2 = self.plot_to_base64(ax2)
+            self.assertEqual(bin1, bin2)
+
+            ax3 = pdf.plot.area(colormap="Paired")
+            bin3 = self.plot_to_base64(ax3)
+            ax4 = kdf.plot.area(colormap="Paired")
+            bin4 = self.plot_to_base64(ax4)
+            self.assertEqual(bin3, bin4)
+
+        pdf = self.pdf1
+        kdf = self.kdf1
+        check_area_plot(pdf, kdf)
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("y", "b")])
+        pdf.columns = columns
+        kdf.columns = columns
+        check_area_plot(pdf, kdf)
+
+    def test_area_plot_stacked_false(self):
+        def check_area_plot_stacked_false(pdf, kdf):
+            ax1 = pdf.plot.area(stacked=False)
+            bin1 = self.plot_to_base64(ax1)
+            ax2 = kdf.plot.area(stacked=False)
+            bin2 = self.plot_to_base64(ax2)
+            self.assertEqual(bin1, bin2)
+
+            # test if frame area plot is correct when stacked=False because default is True
+
+        pdf = pd.DataFrame(
+            {
+                "sales": [3, 2, 3, 9, 10, 6],
+                "signups": [5, 5, 6, 12, 14, 13],
+                "visits": [20, 42, 28, 62, 81, 50],
+            },
+            index=pd.date_range(start="2018/01/01", end="2018/07/01", freq="M"),
+        )
+        kdf = ps.from_pandas(pdf)
+        check_area_plot_stacked_false(pdf, kdf)
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "sales"), ("x", "signups"), ("y", "visits")])
+        pdf.columns = columns
+        kdf.columns = columns
+        check_area_plot_stacked_false(pdf, kdf)
+
+    def test_area_plot_y(self):
+        def check_area_plot_y(pdf, kdf, y):
+            ax1 = pdf.plot.area(y=y)
+            bin1 = self.plot_to_base64(ax1)
+            ax2 = kdf.plot.area(y=y)
+            bin2 = self.plot_to_base64(ax2)
+            self.assertEqual(bin1, bin2)
+
+        # test if frame area plot is correct when y is specified
+        pdf = pd.DataFrame(
+            {
+                "sales": [3, 2, 3, 9, 10, 6],
+                "signups": [5, 5, 6, 12, 14, 13],
+                "visits": [20, 42, 28, 62, 81, 50],
+            },
+            index=pd.date_range(start="2018/01/01", end="2018/07/01", freq="M"),
+        )
+        kdf = ps.from_pandas(pdf)
+        check_area_plot_y(pdf, kdf, y="sales")
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "sales"), ("x", "signups"), ("y", "visits")])
+        pdf.columns = columns
+        kdf.columns = columns
+        check_area_plot_y(pdf, kdf, y=("x", "sales"))
+
+    def test_barh_plot_with_x_y(self):
+        def check_barh_plot_with_x_y(pdf, kdf, x, y):
+            ax1 = pdf.plot(kind="barh", x=x, y=y, colormap="Paired")
+            bin1 = self.plot_to_base64(ax1)
+            ax2 = kdf.plot(kind="barh", x=x, y=y, colormap="Paired")
+            bin2 = self.plot_to_base64(ax2)
+            self.assertEqual(bin1, bin2)
+
+            ax3 = pdf.plot.barh(x=x, y=y, colormap="Paired")
+            bin3 = self.plot_to_base64(ax3)
+            ax4 = kdf.plot.barh(x=x, y=y, colormap="Paired")
+            bin4 = self.plot_to_base64(ax4)
+            self.assertEqual(bin3, bin4)
+
+        # this is testing plot with specified x and y
+        pdf1 = pd.DataFrame({"lab": ["A", "B", "C"], "val": [10, 30, 20]})
+        kdf1 = ps.from_pandas(pdf1)
+        check_barh_plot_with_x_y(pdf1, kdf1, x="lab", y="val")
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "lab"), ("y", "val")])
+        pdf1.columns = columns
+        kdf1.columns = columns
+        check_barh_plot_with_x_y(pdf1, kdf1, x=("x", "lab"), y=("y", "val"))
+
+    def test_barh_plot(self):
+        def check_barh_plot(pdf, kdf):
+            ax1 = pdf.plot(kind="barh", colormap="Paired")
+            bin1 = self.plot_to_base64(ax1)
+            ax2 = kdf.plot(kind="barh", colormap="Paired")
+            bin2 = self.plot_to_base64(ax2)
+            self.assertEqual(bin1, bin2)
+
+            ax3 = pdf.plot.barh(colormap="Paired")
+            bin3 = self.plot_to_base64(ax3)
+            ax4 = kdf.plot.barh(colormap="Paired")
+            bin4 = self.plot_to_base64(ax4)
+            self.assertEqual(bin3, bin4)
+
+        # this is testing when x or y is not assigned
+        pdf1 = pd.DataFrame({"lab": ["A", "B", "C"], "val": [10, 30, 20]})
+        kdf1 = ps.from_pandas(pdf1)
+        check_barh_plot(pdf1, kdf1)
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "lab"), ("y", "val")])
+        pdf1.columns = columns
+        kdf1.columns = columns
+        check_barh_plot(pdf1, kdf1)
+
+    def test_bar_plot(self):
+        def check_bar_plot(pdf, kdf):
+            ax1 = pdf.plot(kind="bar", colormap="Paired")
+            bin1 = self.plot_to_base64(ax1)
+            ax2 = kdf.plot(kind="bar", colormap="Paired")
+            bin2 = self.plot_to_base64(ax2)
+            self.assertEqual(bin1, bin2)
+
+            ax3 = pdf.plot.bar(colormap="Paired")
+            bin3 = self.plot_to_base64(ax3)
+            ax4 = kdf.plot.bar(colormap="Paired")
+            bin4 = self.plot_to_base64(ax4)
+            self.assertEqual(bin3, bin4)
+
+        pdf1 = self.pdf1
+        kdf1 = self.kdf1
+        check_bar_plot(pdf1, kdf1)
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "lab"), ("y", "val")])
+        pdf1.columns = columns
+        kdf1.columns = columns
+        check_bar_plot(pdf1, kdf1)
+
+    def test_bar_with_x_y(self):
+        # this is testing plot with specified x and y
+        pdf = pd.DataFrame({"lab": ["A", "B", "C"], "val": [10, 30, 20]})
+        kdf = ps.from_pandas(pdf)
+
+        ax1 = pdf.plot(kind="bar", x="lab", y="val", colormap="Paired")
+        bin1 = self.plot_to_base64(ax1)
+        ax2 = kdf.plot(kind="bar", x="lab", y="val", colormap="Paired")
+        bin2 = self.plot_to_base64(ax2)
+        self.assertEqual(bin1, bin2)
+
+        ax3 = pdf.plot.bar(x="lab", y="val", colormap="Paired")
+        bin3 = self.plot_to_base64(ax3)
+        ax4 = kdf.plot.bar(x="lab", y="val", colormap="Paired")
+        bin4 = self.plot_to_base64(ax4)
+        self.assertEqual(bin3, bin4)
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "lab"), ("y", "val")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        ax5 = pdf.plot(kind="bar", x=("x", "lab"), y=("y", "val"), colormap="Paired")
+        bin5 = self.plot_to_base64(ax5)
+        ax6 = kdf.plot(kind="bar", x=("x", "lab"), y=("y", "val"), colormap="Paired")
+        bin6 = self.plot_to_base64(ax6)
+        self.assertEqual(bin5, bin6)
+
+        ax7 = pdf.plot.bar(x=("x", "lab"), y=("y", "val"), colormap="Paired")
+        bin7 = self.plot_to_base64(ax7)
+        ax8 = kdf.plot.bar(x=("x", "lab"), y=("y", "val"), colormap="Paired")
+        bin8 = self.plot_to_base64(ax8)
+        self.assertEqual(bin7, bin8)
+
+    def test_pie_plot(self):
+        def check_pie_plot(pdf, kdf, y):
+            ax1 = pdf.plot.pie(y=y, figsize=(5, 5), colormap="Paired")
+            bin1 = self.plot_to_base64(ax1)
+            ax2 = kdf.plot.pie(y=y, figsize=(5, 5), colormap="Paired")
+            bin2 = self.plot_to_base64(ax2)
+            self.assertEqual(bin1, bin2)
+
+            ax1 = pdf.plot(kind="pie", y=y, figsize=(5, 5), colormap="Paired")
+            bin1 = self.plot_to_base64(ax1)
+            ax2 = kdf.plot(kind="pie", y=y, figsize=(5, 5), colormap="Paired")
+            bin2 = self.plot_to_base64(ax2)
+            self.assertEqual(bin1, bin2)
+
+            ax11, ax12 = pdf.plot.pie(figsize=(5, 5), subplots=True, colormap="Paired")
+            bin11 = self.plot_to_base64(ax11)
+            bin12 = self.plot_to_base64(ax12)
+            self.assertEqual(bin11, bin12)
+
+            ax21, ax22 = kdf.plot.pie(figsize=(5, 5), subplots=True, colormap="Paired")
+            bin21 = self.plot_to_base64(ax21)
+            bin22 = self.plot_to_base64(ax22)
+            self.assertEqual(bin21, bin22)
+
+            ax11, ax12 = pdf.plot(kind="pie", figsize=(5, 5), subplots=True, colormap="Paired")
+            bin11 = self.plot_to_base64(ax11)
+            bin12 = self.plot_to_base64(ax12)
+            self.assertEqual(bin11, bin12)
+
+            ax21, ax22 = kdf.plot(kind="pie", figsize=(5, 5), subplots=True, colormap="Paired")
+            bin21 = self.plot_to_base64(ax21)
+            bin22 = self.plot_to_base64(ax22)
+            self.assertEqual(bin21, bin22)
+
+        pdf1 = pd.DataFrame(
+            {"mass": [0.330, 4.87, 5.97], "radius": [2439.7, 6051.8, 6378.1]},
+            index=["Mercury", "Venus", "Earth"],
+        )
+        kdf1 = ps.from_pandas(pdf1)
+        check_pie_plot(pdf1, kdf1, y="mass")
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "mass"), ("y", "radius")])
+        pdf1.columns = columns
+        kdf1.columns = columns
+        check_pie_plot(pdf1, kdf1, y=("x", "mass"))
+
+    def test_pie_plot_error_message(self):
+        # this is to test if error is correctly raising when y is not specified
+        # and subplots is not set to True
+        pdf = pd.DataFrame(
+            {"mass": [0.330, 4.87, 5.97], "radius": [2439.7, 6051.8, 6378.1]},
+            index=["Mercury", "Venus", "Earth"],
+        )
+        kdf = ps.from_pandas(pdf)
+
+        with self.assertRaises(ValueError) as context:
+            kdf.plot.pie(figsize=(5, 5), colormap="Paired")
+        error_message = "pie requires either y column or 'subplots=True'"
+        self.assertTrue(error_message in str(context.exception))
+
+    def test_scatter_plot(self):
+        def check_scatter_plot(pdf, kdf, x, y, c):
+            ax1 = pdf.plot.scatter(x=x, y=y)
+            bin1 = self.plot_to_base64(ax1)
+            ax2 = kdf.plot.scatter(x=x, y=y)
+            bin2 = self.plot_to_base64(ax2)
+            self.assertEqual(bin1, bin2)
+
+            ax1 = pdf.plot(kind="scatter", x=x, y=y)
+            bin1 = self.plot_to_base64(ax1)
+            ax2 = kdf.plot(kind="scatter", x=x, y=y)
+            bin2 = self.plot_to_base64(ax2)
+            self.assertEqual(bin1, bin2)
+
+            # check when keyword c is given as name of a column
+            ax1 = pdf.plot.scatter(x=x, y=y, c=c, s=50)
+            bin1 = self.plot_to_base64(ax1)
+            ax2 = kdf.plot.scatter(x=x, y=y, c=c, s=50)
+            bin2 = self.plot_to_base64(ax2)
+            self.assertEqual(bin1, bin2)
+
+        # Use pandas scatter plot example
+        pdf1 = pd.DataFrame(np.random.rand(50, 4), columns=["a", "b", "c", "d"])
+        kdf1 = ps.from_pandas(pdf1)
+        check_scatter_plot(pdf1, kdf1, x="a", y="b", c="c")
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "c"), ("z", "d")])
+        pdf1.columns = columns
+        kdf1.columns = columns
+        check_scatter_plot(pdf1, kdf1, x=("x", "a"), y=("x", "b"), c=("y", "c"))
+
+    def test_hist_plot(self):
+        def check_hist_plot(pdf, kdf):
+            _, ax1 = plt.subplots(1, 1)
+            ax1 = pdf.plot.hist()
+            bin1 = self.plot_to_base64(ax1)
+            _, ax2 = plt.subplots(1, 1)
+            ax2 = kdf.plot.hist()
+            bin2 = self.plot_to_base64(ax2)
+            self.assertEqual(bin1, bin2)
+
+            ax1 = pdf.plot.hist(bins=15)
+            bin1 = self.plot_to_base64(ax1)
+            ax2 = kdf.plot.hist(bins=15)
+            bin2 = self.plot_to_base64(ax2)
+            self.assertEqual(bin1, bin2)
+
+            ax1 = pdf.plot(kind="hist", bins=15)
+            bin1 = self.plot_to_base64(ax1)
+            ax2 = kdf.plot(kind="hist", bins=15)
+            bin2 = self.plot_to_base64(ax2)
+            self.assertEqual(bin1, bin2)
+
+            ax1 = pdf.plot.hist(bins=3, bottom=[2, 1, 3])
+            bin1 = self.plot_to_base64(ax1)
+            ax2 = kdf.plot.hist(bins=3, bottom=[2, 1, 3])
+            bin2 = self.plot_to_base64(ax2)
+            self.assertEqual(bin1, bin2)
+
+        pdf1 = self.pdf1
+        kdf1 = self.kdf1
+        check_hist_plot(pdf1, kdf1)
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("y", "b")])
+        pdf1.columns = columns
+        kdf1.columns = columns
+        check_hist_plot(pdf1, kdf1)
+
+    def test_kde_plot(self):
+        def moving_average(a, n=10):
+            ret = np.cumsum(a, dtype=float)
+            ret[n:] = ret[n:] - ret[:-n]
+            return ret[n - 1:] / n
+
+        def check_kde_plot(pdf, kdf, *args, **kwargs):
+            _, ax1 = plt.subplots(1, 1)
+            ax1 = pdf.plot.kde(*args, **kwargs)
+            _, ax2 = plt.subplots(1, 1)
+            ax2 = kdf.plot.kde(*args, **kwargs)
+
+            try:
+                for i, (line1, line2) in enumerate(zip(ax1.get_lines(), ax2.get_lines())):
+                    expected = line1.get_xydata().ravel()
+                    actual = line2.get_xydata().ravel()
+                    # TODO: Due to implementation difference, the output is different comparing
+                    # to pandas'. We should identify the root cause of difference, and reduce
+                    # the diff.
+
+                    # Note: Data is from 1 to 50. So, it smooths them by moving average and compares
+                    # both.
+                    self.assertTrue(
+                        np.allclose(moving_average(actual), moving_average(expected), rtol=3.0)
+                    )
+            finally:
+                ax1.cla()
+                ax2.cla()
+
+        pdf1 = self.pdf1
+        kdf1 = self.kdf1
+        check_kde_plot(pdf1, kdf1, bw_method=0.3)
+        check_kde_plot(pdf1, kdf1, ind=[1, 2, 3], bw_method=3.0)
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("y", "b")])
+        pdf1.columns = columns
+        pdf1.columns = columns
+        check_kde_plot(pdf1, kdf1, bw_method=0.3)
+        check_kde_plot(pdf1, kdf1, ind=[1, 2, 3], bw_method=3.0)
+
+
+if __name__ == "__main__":
+    from pyspark.pandas.tests.plot.test_frame_plot_matplotlib import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/plot/test_frame_plot_plotly.py b/python/pyspark/pandas/tests/plot/test_frame_plot_plotly.py
new file mode 100644
index 0000000000000..33d6bef2c8c1a
--- /dev/null
+++ b/python/pyspark/pandas/tests/plot/test_frame_plot_plotly.py
@@ -0,0 +1,275 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import unittest
+from distutils.version import LooseVersion
+import pprint
+
+import pandas as pd
+import numpy as np
+
+from pyspark import pandas as ps
+from pyspark.pandas.config import set_option, reset_option
+from pyspark.testing.pandasutils import (
+    have_plotly,
+    plotly_requirement_message,
+    PandasOnSparkTestCase,
+    TestUtils,
+)
+from pyspark.pandas.utils import name_like_string
+
+if have_plotly:
+    from plotly import express
+    import plotly.graph_objs as go
+
+
+@unittest.skipIf(
+    not have_plotly or LooseVersion(pd.__version__) < "1.0.0",
+    plotly_requirement_message + " Or pandas<1.0; pandas<1.0 does not support latest plotly "
+    "and/or 'plotting.backend' option.",
+)
+class DataFramePlotPlotlyTest(PandasOnSparkTestCase, TestUtils):
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        pd.set_option("plotting.backend", "plotly")
+        set_option("plotting.backend", "plotly")
+        set_option("plotting.max_rows", 2000)
+        set_option("plotting.sample_ratio", None)
+
+    @classmethod
+    def tearDownClass(cls):
+        pd.reset_option("plotting.backend")
+        reset_option("plotting.backend")
+        reset_option("plotting.max_rows")
+        reset_option("plotting.sample_ratio")
+        super().tearDownClass()
+
+    @property
+    def pdf1(self):
+        return pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 50], "b": [2, 3, 4, 5, 7, 9, 10, 15, 34, 45, 49]},
+            index=[0, 1, 3, 5, 6, 8, 9, 9, 9, 10, 10],
+        )
+
+    @property
+    def kdf1(self):
+        return ps.from_pandas(self.pdf1)
+
+    def test_line_plot(self):
+        def check_line_plot(pdf, kdf):
+            self.assertEqual(pdf.plot(kind="line"), kdf.plot(kind="line"))
+            self.assertEqual(pdf.plot.line(), kdf.plot.line())
+
+        pdf1 = self.pdf1
+        kdf1 = self.kdf1
+        check_line_plot(pdf1, kdf1)
+
+    def test_area_plot(self):
+        def check_area_plot(pdf, kdf):
+            self.assertEqual(pdf.plot(kind="area"), kdf.plot(kind="area"))
+            self.assertEqual(pdf.plot.area(), kdf.plot.area())
+
+        pdf = self.pdf1
+        kdf = self.kdf1
+        check_area_plot(pdf, kdf)
+
+    def test_area_plot_y(self):
+        def check_area_plot_y(pdf, kdf, y):
+            self.assertEqual(pdf.plot.area(y=y), kdf.plot.area(y=y))
+
+        # test if frame area plot is correct when y is specified
+        pdf = pd.DataFrame(
+            {
+                "sales": [3, 2, 3, 9, 10, 6],
+                "signups": [5, 5, 6, 12, 14, 13],
+                "visits": [20, 42, 28, 62, 81, 50],
+            },
+            index=pd.date_range(start="2018/01/01", end="2018/07/01", freq="M"),
+        )
+        kdf = ps.from_pandas(pdf)
+        check_area_plot_y(pdf, kdf, y="sales")
+
+    def test_barh_plot_with_x_y(self):
+        def check_barh_plot_with_x_y(pdf, kdf, x, y):
+            self.assertEqual(pdf.plot(kind="barh", x=x, y=y), kdf.plot(kind="barh", x=x, y=y))
+            self.assertEqual(pdf.plot.barh(x=x, y=y), kdf.plot.barh(x=x, y=y))
+
+        # this is testing plot with specified x and y
+        pdf1 = pd.DataFrame({"lab": ["A", "B", "C"], "val": [10, 30, 20]})
+        kdf1 = ps.from_pandas(pdf1)
+        check_barh_plot_with_x_y(pdf1, kdf1, x="lab", y="val")
+
+    def test_barh_plot(self):
+        def check_barh_plot(pdf, kdf):
+            self.assertEqual(pdf.plot(kind="barh"), kdf.plot(kind="barh"))
+            self.assertEqual(pdf.plot.barh(), kdf.plot.barh())
+
+        # this is testing when x or y is not assigned
+        pdf1 = pd.DataFrame({"lab": [20.1, 40.5, 60.6], "val": [10, 30, 20]})
+        kdf1 = ps.from_pandas(pdf1)
+        check_barh_plot(pdf1, kdf1)
+
+    def test_bar_plot(self):
+        def check_bar_plot(pdf, kdf):
+            self.assertEqual(pdf.plot(kind="bar"), kdf.plot(kind="bar"))
+            self.assertEqual(pdf.plot.bar(), kdf.plot.bar())
+
+        pdf1 = self.pdf1
+        kdf1 = self.kdf1
+        check_bar_plot(pdf1, kdf1)
+
+    def test_bar_with_x_y(self):
+        # this is testing plot with specified x and y
+        pdf = pd.DataFrame({"lab": ["A", "B", "C"], "val": [10, 30, 20]})
+        kdf = ps.from_pandas(pdf)
+
+        self.assertEqual(
+            pdf.plot(kind="bar", x="lab", y="val"), kdf.plot(kind="bar", x="lab", y="val")
+        )
+        self.assertEqual(pdf.plot.bar(x="lab", y="val"), kdf.plot.bar(x="lab", y="val"))
+
+    def test_scatter_plot(self):
+        def check_scatter_plot(pdf, kdf, x, y, c):
+            self.assertEqual(pdf.plot.scatter(x=x, y=y), kdf.plot.scatter(x=x, y=y))
+            self.assertEqual(pdf.plot(kind="scatter", x=x, y=y), kdf.plot(kind="scatter", x=x, y=y))
+
+            # check when keyword c is given as name of a column
+            self.assertEqual(
+                pdf.plot.scatter(x=x, y=y, c=c, s=50), kdf.plot.scatter(x=x, y=y, c=c, s=50)
+            )
+
+        # Use pandas scatter plot example
+        pdf1 = pd.DataFrame(np.random.rand(50, 4), columns=["a", "b", "c", "d"])
+        kdf1 = ps.from_pandas(pdf1)
+        check_scatter_plot(pdf1, kdf1, x="a", y="b", c="c")
+
+    def test_pie_plot(self):
+        def check_pie_plot(kdf):
+            pdf = kdf.to_pandas()
+            self.assertEqual(
+                kdf.plot(kind="pie", y=kdf.columns[0]),
+                express.pie(pdf, values="a", names=pdf.index),
+            )
+
+            self.assertEqual(
+                kdf.plot(kind="pie", values="a"), express.pie(pdf, values="a"),
+            )
+
+        kdf1 = self.kdf1
+        check_pie_plot(kdf1)
+
+        # TODO: support multi-index columns
+        # columns = pd.MultiIndex.from_tuples([("x", "y"), ("y", "z")])
+        # kdf1.columns = columns
+        # check_pie_plot(kdf1)
+
+        # TODO: support multi-index
+        # kdf1 = ps.DataFrame(
+        #     {
+        #         "a": [1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 50],
+        #         "b": [2, 3, 4, 5, 7, 9, 10, 15, 34, 45, 49]
+        #     },
+        #     index=pd.MultiIndex.from_tuples([("x", "y")] * 11),
+        # )
+        # check_pie_plot(kdf1)
+
+    def test_hist_plot(self):
+        def check_hist_plot(kdf):
+            bins = np.array([1.0, 5.9, 10.8, 15.7, 20.6, 25.5, 30.4, 35.3, 40.2, 45.1, 50.0])
+            data = [
+                np.array([5.0, 4.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0]),
+                np.array([4.0, 3.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 1.0]),
+            ]
+            prev = bins[0]
+            text_bins = []
+            for b in bins[1:]:
+                text_bins.append("[%s, %s)" % (prev, b))
+                prev = b
+            text_bins[-1] = text_bins[-1][:-1] + "]"
+            bins = 0.5 * (bins[:-1] + bins[1:])
+            name_a = name_like_string(kdf.columns[0])
+            name_b = name_like_string(kdf.columns[1])
+            bars = [
+                go.Bar(
+                    x=bins,
+                    y=data[0],
+                    name=name_a,
+                    text=text_bins,
+                    hovertemplate=("variable=" + name_a + "<br>value=%{text}<br>count=%{y}"),
+                ),
+                go.Bar(
+                    x=bins,
+                    y=data[1],
+                    name=name_b,
+                    text=text_bins,
+                    hovertemplate=("variable=" + name_b + "<br>value=%{text}<br>count=%{y}"),
+                ),
+            ]
+            fig = go.Figure(data=bars, layout=go.Layout(barmode="stack"))
+            fig["layout"]["xaxis"]["title"] = "value"
+            fig["layout"]["yaxis"]["title"] = "count"
+
+            self.assertEqual(
+                pprint.pformat(kdf.plot(kind="hist").to_dict()), pprint.pformat(fig.to_dict())
+            )
+
+        kdf1 = self.kdf1
+        check_hist_plot(kdf1)
+
+        columns = pd.MultiIndex.from_tuples([("x", "y"), ("y", "z")])
+        kdf1.columns = columns
+        check_hist_plot(kdf1)
+
+    def test_kde_plot(self):
+        kdf = ps.DataFrame({"a": [1, 2, 3, 4, 5], "b": [1, 3, 5, 7, 9], "c": [2, 4, 6, 8, 10]})
+
+        pdf = pd.DataFrame(
+            {
+                "Density": [
+                    0.03515491,
+                    0.06834979,
+                    0.00663503,
+                    0.02372059,
+                    0.06834979,
+                    0.01806934,
+                    0.01806934,
+                    0.06834979,
+                    0.02372059,
+                ],
+                "names": ["a", "a", "a", "b", "b", "b", "c", "c", "c"],
+                "index": [-3.5, 5.5, 14.5, -3.5, 5.5, 14.5, -3.5, 5.5, 14.5],
+            }
+        )
+
+        actual = kdf.plot.kde(bw_method=5, ind=3)
+
+        expected = express.line(pdf, x="index", y="Density", color="names")
+        expected["layout"]["xaxis"]["title"] = None
+
+        self.assertEqual(pprint.pformat(actual.to_dict()), pprint.pformat(expected.to_dict()))
+
+
+if __name__ == "__main__":
+    from pyspark.pandas.tests.plot.test_frame_plot_plotly import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/plot/test_series_plot.py b/python/pyspark/pandas/tests/plot/test_series_plot.py
new file mode 100644
index 0000000000000..fbfda88648cbc
--- /dev/null
+++ b/python/pyspark/pandas/tests/plot/test_series_plot.py
@@ -0,0 +1,101 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import unittest
+
+import pandas as pd
+import numpy as np
+
+from pyspark import pandas as ps
+from pyspark.pandas.plot import PandasOnSparkPlotAccessor, BoxPlotBase
+from pyspark.testing.pandasutils import have_plotly, plotly_requirement_message
+
+
+class SeriesPlotTest(unittest.TestCase):
+    @property
+    def pdf1(self):
+        return pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 50]}, index=[0, 1, 3, 5, 6, 8, 9, 9, 9, 10, 10]
+        )
+
+    @property
+    def kdf1(self):
+        return ps.from_pandas(self.pdf1)
+
+    @unittest.skipIf(not have_plotly, plotly_requirement_message)
+    def test_plot_backends(self):
+        plot_backend = "plotly"
+
+        with ps.option_context("plotting.backend", plot_backend):
+            self.assertEqual(ps.options.plotting.backend, plot_backend)
+
+            module = PandasOnSparkPlotAccessor._get_plot_backend(plot_backend)
+            self.assertEqual(module.__name__, "pyspark.pandas.plot.plotly")
+
+    def test_plot_backends_incorrect(self):
+        fake_plot_backend = "none_plotting_module"
+
+        with ps.option_context("plotting.backend", fake_plot_backend):
+            self.assertEqual(ps.options.plotting.backend, fake_plot_backend)
+
+            with self.assertRaises(ValueError):
+                PandasOnSparkPlotAccessor._get_plot_backend(fake_plot_backend)
+
+    def test_box_summary(self):
+        def check_box_summary(kdf, pdf):
+            k = 1.5
+            stats, fences = BoxPlotBase.compute_stats(kdf["a"], "a", whis=k, precision=0.01)
+            outliers = BoxPlotBase.outliers(kdf["a"], "a", *fences)
+            whiskers = BoxPlotBase.calc_whiskers("a", outliers)
+            fliers = BoxPlotBase.get_fliers("a", outliers, whiskers[0])
+
+            expected_mean = pdf["a"].mean()
+            expected_median = pdf["a"].median()
+            expected_q1 = np.percentile(pdf["a"], 25)
+            expected_q3 = np.percentile(pdf["a"], 75)
+            iqr = expected_q3 - expected_q1
+            expected_fences = (expected_q1 - k * iqr, expected_q3 + k * iqr)
+            pdf["outlier"] = ~pdf["a"].between(fences[0], fences[1])
+            expected_whiskers = (
+                pdf.query("not outlier")["a"].min(),
+                pdf.query("not outlier")["a"].max(),
+            )
+            expected_fliers = pdf.query("outlier")["a"].values
+
+            self.assertEqual(expected_mean, stats["mean"])
+            self.assertEqual(expected_median, stats["med"])
+            self.assertEqual(expected_q1, stats["q1"] + 0.5)
+            self.assertEqual(expected_q3, stats["q3"] - 0.5)
+            self.assertEqual(expected_fences[0], fences[0] + 2.0)
+            self.assertEqual(expected_fences[1], fences[1] - 2.0)
+            self.assertEqual(expected_whiskers[0], whiskers[0])
+            self.assertEqual(expected_whiskers[1], whiskers[1])
+            self.assertEqual(expected_fliers, fliers)
+
+        check_box_summary(self.kdf1, self.pdf1)
+        check_box_summary(-self.kdf1, -self.pdf1)
+
+
+if __name__ == "__main__":
+    from pyspark.pandas.tests.plot.test_series_plot import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/plot/test_series_plot_matplotlib.py b/python/pyspark/pandas/tests/plot/test_series_plot_matplotlib.py
new file mode 100644
index 0000000000000..364a39bff8216
--- /dev/null
+++ b/python/pyspark/pandas/tests/plot/test_series_plot_matplotlib.py
@@ -0,0 +1,407 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import base64
+from distutils.version import LooseVersion
+from io import BytesIO
+import unittest
+
+import numpy as np
+import pandas as pd
+
+from pyspark import pandas as ps
+from pyspark.pandas.config import set_option, reset_option
+from pyspark.testing.pandasutils import (
+    have_matplotlib,
+    matplotlib_requirement_message,
+    PandasOnSparkTestCase,
+    TestUtils,
+)
+
+if have_matplotlib:
+    import matplotlib
+    from matplotlib import pyplot as plt
+
+    matplotlib.use("agg")
+
+
+@unittest.skipIf(not have_matplotlib, matplotlib_requirement_message)
+class SeriesPlotMatplotlibTest(PandasOnSparkTestCase, TestUtils):
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        if LooseVersion(pd.__version__) >= LooseVersion("0.25"):
+            pd.set_option("plotting.backend", "matplotlib")
+        set_option("plotting.backend", "matplotlib")
+        set_option("plotting.max_rows", 1000)
+
+    @classmethod
+    def tearDownClass(cls):
+        if LooseVersion(pd.__version__) >= LooseVersion("0.25"):
+            pd.reset_option("plotting.backend")
+        reset_option("plotting.backend")
+        reset_option("plotting.max_rows")
+        super().tearDownClass()
+
+    @property
+    def pdf1(self):
+        return pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 50]}, index=[0, 1, 3, 5, 6, 8, 9, 9, 9, 10, 10]
+        )
+
+    @property
+    def kdf1(self):
+        return ps.from_pandas(self.pdf1)
+
+    @property
+    def kdf2(self):
+        return ps.range(1002)
+
+    @property
+    def pdf2(self):
+        return self.kdf2.to_pandas()
+
+    @staticmethod
+    def plot_to_base64(ax):
+        bytes_data = BytesIO()
+        ax.figure.savefig(bytes_data, format="png")
+        bytes_data.seek(0)
+        b64_data = base64.b64encode(bytes_data.read())
+        plt.close(ax.figure)
+        return b64_data
+
+    def test_bar_plot(self):
+        pdf = self.pdf1
+        kdf = self.kdf1
+
+        ax1 = pdf["a"].plot(kind="bar", colormap="Paired")
+        bin1 = self.plot_to_base64(ax1)
+        ax2 = kdf["a"].plot(kind="bar", colormap="Paired")
+        bin2 = self.plot_to_base64(ax2)
+        self.assertEqual(bin1, bin2)
+
+        ax1 = pdf["a"].plot(kind="bar", colormap="Paired")
+        bin1 = self.plot_to_base64(ax1)
+        ax2 = kdf["a"].plot(kind="bar", colormap="Paired")
+        bin2 = self.plot_to_base64(ax2)
+        self.assertEqual(bin1, bin2)
+
+    def test_bar_plot_limited(self):
+        pdf = self.pdf2
+        kdf = self.kdf2
+
+        _, ax1 = plt.subplots(1, 1)
+        ax1 = pdf["id"][:1000].plot.bar(colormap="Paired")
+        ax1.text(
+            1,
+            1,
+            "showing top 1000 elements only",
+            size=6,
+            ha="right",
+            va="bottom",
+            transform=ax1.transAxes,
+        )
+        bin1 = self.plot_to_base64(ax1)
+
+        _, ax2 = plt.subplots(1, 1)
+        ax2 = kdf["id"].plot.bar(colormap="Paired")
+        bin2 = self.plot_to_base64(ax2)
+
+        self.assertEqual(bin1, bin2)
+
+    def test_pie_plot(self):
+        pdf = self.pdf1
+        kdf = self.kdf1
+
+        ax1 = pdf["a"].plot.pie(colormap="Paired")
+        bin1 = self.plot_to_base64(ax1)
+        ax2 = kdf["a"].plot.pie(colormap="Paired")
+        bin2 = self.plot_to_base64(ax2)
+        self.assertEqual(bin1, bin2)
+
+        ax1 = pdf["a"].plot(kind="pie", colormap="Paired")
+        bin1 = self.plot_to_base64(ax1)
+        ax2 = kdf["a"].plot(kind="pie", colormap="Paired")
+        bin2 = self.plot_to_base64(ax2)
+        self.assertEqual(bin1, bin2)
+
+    def test_pie_plot_limited(self):
+        pdf = self.pdf2
+        kdf = self.kdf2
+
+        _, ax1 = plt.subplots(1, 1)
+        ax1 = pdf["id"][:1000].plot.pie(colormap="Paired")
+        ax1.text(
+            1,
+            1,
+            "showing top 1000 elements only",
+            size=6,
+            ha="right",
+            va="bottom",
+            transform=ax1.transAxes,
+        )
+        bin1 = self.plot_to_base64(ax1)
+
+        _, ax2 = plt.subplots(1, 1)
+        ax2 = kdf["id"].plot.pie(colormap="Paired")
+        bin2 = self.plot_to_base64(ax2)
+
+        self.assertEqual(bin1, bin2)
+
+    def test_line_plot(self):
+        pdf = self.pdf1
+        kdf = self.kdf1
+
+        ax1 = pdf["a"].plot(kind="line", colormap="Paired")
+        bin1 = self.plot_to_base64(ax1)
+        ax2 = kdf["a"].plot(kind="line", colormap="Paired")
+        bin2 = self.plot_to_base64(ax2)
+        self.assertEqual(bin1, bin2)
+
+        ax1 = pdf["a"].plot.line(colormap="Paired")
+        bin1 = self.plot_to_base64(ax1)
+        ax2 = kdf["a"].plot.line(colormap="Paired")
+        bin2 = self.plot_to_base64(ax2)
+        self.assertEqual(bin1, bin2)
+
+    def test_barh_plot(self):
+        pdf = self.pdf1
+        kdf = self.kdf1
+
+        ax1 = pdf["a"].plot(kind="barh", colormap="Paired")
+        bin1 = self.plot_to_base64(ax1)
+        ax2 = kdf["a"].plot(kind="barh", colormap="Paired")
+        bin2 = self.plot_to_base64(ax2)
+        self.assertEqual(bin1, bin2)
+
+    def test_barh_plot_limited(self):
+        pdf = self.pdf2
+        kdf = self.kdf2
+
+        _, ax1 = plt.subplots(1, 1)
+        ax1 = pdf["id"][:1000].plot.barh(colormap="Paired")
+        ax1.text(
+            1,
+            1,
+            "showing top 1000 elements only",
+            size=6,
+            ha="right",
+            va="bottom",
+            transform=ax1.transAxes,
+        )
+        bin1 = self.plot_to_base64(ax1)
+
+        _, ax2 = plt.subplots(1, 1)
+        ax2 = kdf["id"].plot.barh(colormap="Paired")
+        bin2 = self.plot_to_base64(ax2)
+
+        self.assertEqual(bin1, bin2)
+
+    def test_hist(self):
+        pdf = pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 50]}, index=[0, 1, 3, 5, 6, 8, 9, 9, 9, 10, 10]
+        )
+
+        kdf = ps.from_pandas(pdf)
+
+        def plot_to_base64(ax):
+            bytes_data = BytesIO()
+            ax.figure.savefig(bytes_data, format="png")
+            bytes_data.seek(0)
+            b64_data = base64.b64encode(bytes_data.read())
+            plt.close(ax.figure)
+            return b64_data
+
+        _, ax1 = plt.subplots(1, 1)
+        # Using plot.hist() because pandas changes ticks props when called hist()
+        ax1 = pdf["a"].plot.hist()
+        _, ax2 = plt.subplots(1, 1)
+        ax2 = kdf["a"].hist()
+        self.assert_eq(plot_to_base64(ax1), plot_to_base64(ax2))
+
+    def test_hist_plot(self):
+        pdf = self.pdf1
+        kdf = self.kdf1
+
+        _, ax1 = plt.subplots(1, 1)
+        ax1 = pdf["a"].plot.hist()
+        bin1 = self.plot_to_base64(ax1)
+        _, ax2 = plt.subplots(1, 1)
+        ax2 = kdf["a"].plot.hist()
+        bin2 = self.plot_to_base64(ax2)
+        self.assertEqual(bin1, bin2)
+
+        ax1 = pdf["a"].plot.hist(bins=15)
+        bin1 = self.plot_to_base64(ax1)
+        ax2 = kdf["a"].plot.hist(bins=15)
+        bin2 = self.plot_to_base64(ax2)
+        self.assertEqual(bin1, bin2)
+
+        ax1 = pdf["a"].plot(kind="hist", bins=15)
+        bin1 = self.plot_to_base64(ax1)
+        ax2 = kdf["a"].plot(kind="hist", bins=15)
+        bin2 = self.plot_to_base64(ax2)
+        self.assertEqual(bin1, bin2)
+
+        ax1 = pdf["a"].plot.hist(bins=3, bottom=[2, 1, 3])
+        bin1 = self.plot_to_base64(ax1)
+        ax2 = kdf["a"].plot.hist(bins=3, bottom=[2, 1, 3])
+        bin2 = self.plot_to_base64(ax2)
+        self.assertEqual(bin1, bin2)
+
+    def test_area_plot(self):
+        pdf = pd.DataFrame(
+            {
+                "sales": [3, 2, 3, 9, 10, 6],
+                "signups": [5, 5, 6, 12, 14, 13],
+                "visits": [20, 42, 28, 62, 81, 50],
+            },
+            index=pd.date_range(start="2018/01/01", end="2018/07/01", freq="M"),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        ax1 = pdf["sales"].plot(kind="area", colormap="Paired")
+        bin1 = self.plot_to_base64(ax1)
+        ax2 = kdf["sales"].plot(kind="area", colormap="Paired")
+        bin2 = self.plot_to_base64(ax2)
+        self.assertEqual(bin1, bin2)
+
+        ax1 = pdf["sales"].plot.area(colormap="Paired")
+        bin1 = self.plot_to_base64(ax1)
+        ax2 = kdf["sales"].plot.area(colormap="Paired")
+        bin2 = self.plot_to_base64(ax2)
+        self.assertEqual(bin1, bin2)
+
+        # just a sanity check for df.col type
+        ax1 = pdf.sales.plot(kind="area", colormap="Paired")
+        bin1 = self.plot_to_base64(ax1)
+        ax2 = kdf.sales.plot(kind="area", colormap="Paired")
+        bin2 = self.plot_to_base64(ax2)
+        self.assertEqual(bin1, bin2)
+
+    def test_box_plot(self):
+        def check_box_plot(pser, kser, *args, **kwargs):
+            _, ax1 = plt.subplots(1, 1)
+            ax1 = pser.plot.box(*args, **kwargs)
+            _, ax2 = plt.subplots(1, 1)
+            ax2 = kser.plot.box(*args, **kwargs)
+
+            diffs = [
+                np.array([0, 0.5, 0, 0.5, 0, -0.5, 0, -0.5, 0, 0.5]),
+                np.array([0, 0.5, 0, 0]),
+                np.array([0, -0.5, 0, 0]),
+            ]
+
+            try:
+                for i, (line1, line2) in enumerate(zip(ax1.get_lines(), ax2.get_lines())):
+                    expected = line1.get_xydata().ravel()
+                    actual = line2.get_xydata().ravel()
+                    if i < 3:
+                        actual += diffs[i]
+                    self.assert_eq(pd.Series(expected), pd.Series(actual))
+            finally:
+                ax1.cla()
+                ax2.cla()
+
+        # Non-named Series
+        pser = pd.Series([1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 50], [0, 1, 3, 5, 6, 8, 9, 9, 9, 10, 10])
+        kser = ps.from_pandas(pser)
+
+        spec = [(self.pdf1.a, self.kdf1.a), (pser, kser)]
+
+        for p, k in spec:
+            check_box_plot(p, k)
+            check_box_plot(p, k, showfliers=True)
+            check_box_plot(p, k, sym="")
+            check_box_plot(p, k, sym=".", color="r")
+            check_box_plot(p, k, use_index=False, labels=["Test"])
+            check_box_plot(p, k, usermedians=[2.0])
+            check_box_plot(p, k, conf_intervals=[(1.0, 3.0)])
+
+        val = (1, 3)
+        self.assertRaises(
+            ValueError, lambda: check_box_plot(self.pdf1, self.kdf1, usermedians=[2.0, 3.0])
+        )
+        self.assertRaises(
+            ValueError, lambda: check_box_plot(self.pdf1, self.kdf1, conf_intervals=[val, val])
+        )
+        self.assertRaises(
+            ValueError, lambda: check_box_plot(self.pdf1, self.kdf1, conf_intervals=[(1,)])
+        )
+
+    def test_kde_plot(self):
+        def moving_average(a, n=10):
+            ret = np.cumsum(a, dtype=float)
+            ret[n:] = ret[n:] - ret[:-n]
+            return ret[n - 1:] / n
+
+        def check_kde_plot(pdf, kdf, *args, **kwargs):
+            _, ax1 = plt.subplots(1, 1)
+            ax1 = pdf["a"].plot.kde(*args, **kwargs)
+            _, ax2 = plt.subplots(1, 1)
+            ax2 = kdf["a"].plot.kde(*args, **kwargs)
+
+            try:
+                for i, (line1, line2) in enumerate(zip(ax1.get_lines(), ax2.get_lines())):
+                    expected = line1.get_xydata().ravel()
+                    actual = line2.get_xydata().ravel()
+                    # TODO: Due to implementation difference, the output is different comparing
+                    # to pandas'. We should identify the root cause of difference, and reduce
+                    # the diff.
+
+                    # Note: Data is from 1 to 50. So, it smooths them by moving average and compares
+                    # both.
+                    self.assertTrue(
+                        np.allclose(moving_average(actual), moving_average(expected), rtol=3)
+                    )
+            finally:
+                ax1.cla()
+                ax2.cla()
+
+        check_kde_plot(self.pdf1, self.kdf1, bw_method=0.3)
+        check_kde_plot(self.pdf1, self.kdf1, ind=[1, 2, 3, 4, 5], bw_method=3.0)
+
+    def test_empty_hist(self):
+        pdf = self.pdf1.assign(categorical="A")
+        kdf = ps.from_pandas(pdf)
+        kser = kdf["categorical"]
+
+        with self.assertRaisesRegex(TypeError, "Empty 'DataFrame': no numeric data to plot"):
+            kser.plot.hist()
+
+    def test_single_value_hist(self):
+        pdf = self.pdf1.assign(single=2)
+        kdf = ps.from_pandas(pdf)
+
+        _, ax1 = plt.subplots(1, 1)
+        ax1 = pdf["single"].plot.hist()
+        bin1 = self.plot_to_base64(ax1)
+        _, ax2 = plt.subplots(1, 1)
+        ax2 = kdf["single"].plot.hist()
+        bin2 = self.plot_to_base64(ax2)
+        self.assertEqual(bin1, bin2)
+
+
+if __name__ == "__main__":
+    from pyspark.pandas.tests.plot.test_series_plot_matplotlib import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/plot/test_series_plot_plotly.py b/python/pyspark/pandas/tests/plot/test_series_plot_plotly.py
new file mode 100644
index 0000000000000..2a14d373d2a5f
--- /dev/null
+++ b/python/pyspark/pandas/tests/plot/test_series_plot_plotly.py
@@ -0,0 +1,246 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import unittest
+from distutils.version import LooseVersion
+import pprint
+
+import pandas as pd
+import numpy as np
+
+from pyspark import pandas as ps
+from pyspark.pandas.config import set_option, reset_option
+from pyspark.pandas.utils import name_like_string
+from pyspark.testing.pandasutils import (
+    have_plotly,
+    plotly_requirement_message,
+    PandasOnSparkTestCase,
+    TestUtils,
+)
+
+if have_plotly:
+    from plotly import express
+    import plotly.graph_objs as go
+
+
+@unittest.skipIf(
+    not have_plotly or LooseVersion(pd.__version__) < "1.0.0",
+    plotly_requirement_message + " Or pandas<1.0; pandas<1.0 does not support latest plotly "
+    "and/or 'plotting.backend' option.",
+)
+class SeriesPlotPlotlyTest(PandasOnSparkTestCase, TestUtils):
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        pd.set_option("plotting.backend", "plotly")
+        set_option("plotting.backend", "plotly")
+        set_option("plotting.max_rows", 1000)
+        set_option("plotting.sample_ratio", None)
+
+    @classmethod
+    def tearDownClass(cls):
+        pd.reset_option("plotting.backend")
+        reset_option("plotting.backend")
+        reset_option("plotting.max_rows")
+        reset_option("plotting.sample_ratio")
+        super().tearDownClass()
+
+    @property
+    def pdf1(self):
+        return pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 50]}, index=[0, 1, 3, 5, 6, 8, 9, 9, 9, 10, 10]
+        )
+
+    @property
+    def kdf1(self):
+        return ps.from_pandas(self.pdf1)
+
+    @property
+    def kdf2(self):
+        return ps.range(1002)
+
+    @property
+    def pdf2(self):
+        return self.kdf2.to_pandas()
+
+    def test_bar_plot(self):
+        pdf = self.pdf1
+        kdf = self.kdf1
+
+        self.assertEqual(pdf["a"].plot(kind="bar"), kdf["a"].plot(kind="bar"))
+        self.assertEqual(pdf["a"].plot.bar(), kdf["a"].plot.bar())
+
+    def test_line_plot(self):
+        pdf = self.pdf1
+        kdf = self.kdf1
+
+        self.assertEqual(pdf["a"].plot(kind="line"), kdf["a"].plot(kind="line"))
+        self.assertEqual(pdf["a"].plot.line(), kdf["a"].plot.line())
+
+    def test_barh_plot(self):
+        pdf = self.pdf1
+        kdf = self.kdf1
+
+        self.assertEqual(pdf["a"].plot(kind="barh"), kdf["a"].plot(kind="barh"))
+
+    def test_area_plot(self):
+        pdf = pd.DataFrame(
+            {
+                "sales": [3, 2, 3, 9, 10, 6],
+                "signups": [5, 5, 6, 12, 14, 13],
+                "visits": [20, 42, 28, 62, 81, 50],
+            },
+            index=pd.date_range(start="2018/01/01", end="2018/07/01", freq="M"),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assertEqual(pdf["sales"].plot(kind="area"), kdf["sales"].plot(kind="area"))
+        self.assertEqual(pdf["sales"].plot.area(), kdf["sales"].plot.area())
+
+        # just a sanity check for df.col type
+        self.assertEqual(pdf.sales.plot(kind="area"), kdf.sales.plot(kind="area"))
+
+    def test_pie_plot(self):
+        kdf = self.kdf1
+        pdf = kdf.to_pandas()
+        self.assertEqual(
+            kdf["a"].plot(kind="pie"), express.pie(pdf, values=pdf.columns[0], names=pdf.index),
+        )
+
+        # TODO: support multi-index columns
+        # columns = pd.MultiIndex.from_tuples([("x", "y")])
+        # kdf.columns = columns
+        # pdf.columns = columns
+        # self.assertEqual(
+        #     kdf[("x", "y")].plot(kind="pie"),
+        #     express.pie(pdf, values=pdf.iloc[:, 0].to_numpy(), names=pdf.index.to_numpy()),
+        # )
+
+        # TODO: support multi-index
+        # kdf = ps.DataFrame(
+        #     {
+        #         "a": [1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 50],
+        #         "b": [2, 3, 4, 5, 7, 9, 10, 15, 34, 45, 49]
+        #     },
+        #     index=pd.MultiIndex.from_tuples([("x", "y")] * 11),
+        # )
+        # pdf = kdf.to_pandas()
+        # self.assertEqual(
+        #     kdf["a"].plot(kind="pie"), express.pie(pdf, values=pdf.columns[0], names=pdf.index),
+        # )
+
+    def test_hist_plot(self):
+        def check_hist_plot(kser):
+            bins = np.array([1.0, 5.9, 10.8, 15.7, 20.6, 25.5, 30.4, 35.3, 40.2, 45.1, 50.0])
+            data = np.array([5.0, 4.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0])
+            prev = bins[0]
+            text_bins = []
+            for b in bins[1:]:
+                text_bins.append("[%s, %s)" % (prev, b))
+                prev = b
+            text_bins[-1] = text_bins[-1][:-1] + "]"
+            bins = 0.5 * (bins[:-1] + bins[1:])
+            name_a = name_like_string(kser.name)
+            bars = [
+                go.Bar(
+                    x=bins,
+                    y=data,
+                    name=name_a,
+                    text=text_bins,
+                    hovertemplate=("variable=" + name_a + "<br>value=%{text}<br>count=%{y}"),
+                ),
+            ]
+            fig = go.Figure(data=bars, layout=go.Layout(barmode="stack"))
+            fig["layout"]["xaxis"]["title"] = "value"
+            fig["layout"]["yaxis"]["title"] = "count"
+
+            self.assertEqual(
+                pprint.pformat(kser.plot(kind="hist").to_dict()), pprint.pformat(fig.to_dict())
+            )
+
+        kdf1 = self.kdf1
+        check_hist_plot(kdf1["a"])
+
+        columns = pd.MultiIndex.from_tuples([("x", "y")])
+        kdf1.columns = columns
+        check_hist_plot(kdf1[("x", "y")])
+
+    def test_pox_plot(self):
+        def check_pox_plot(kser):
+            fig = go.Figure()
+            fig.add_trace(
+                go.Box(
+                    name=name_like_string(kser.name),
+                    q1=[3],
+                    median=[6],
+                    q3=[9],
+                    mean=[10.0],
+                    lowerfence=[1],
+                    upperfence=[15],
+                    y=[[50]],
+                    boxpoints="suspectedoutliers",
+                    notched=False,
+                )
+            )
+            fig["layout"]["xaxis"]["title"] = name_like_string(kser.name)
+            fig["layout"]["yaxis"]["title"] = "value"
+
+            self.assertEqual(
+                pprint.pformat(kser.plot(kind="box").to_dict()), pprint.pformat(fig.to_dict())
+            )
+
+        kdf1 = self.kdf1
+        check_pox_plot(kdf1["a"])
+
+        columns = pd.MultiIndex.from_tuples([("x", "y")])
+        kdf1.columns = columns
+        check_pox_plot(kdf1[("x", "y")])
+
+    def test_pox_plot_arguments(self):
+        with self.assertRaisesRegex(ValueError, "does not support"):
+            self.kdf1.a.plot.box(boxpoints="all")
+        with self.assertRaisesRegex(ValueError, "does not support"):
+            self.kdf1.a.plot.box(notched=True)
+        self.kdf1.a.plot.box(hovertext="abc")  # other arguments should not throw an exception
+
+    def test_kde_plot(self):
+        kdf = ps.DataFrame({"a": [1, 2, 3, 4, 5]})
+        pdf = pd.DataFrame(
+            {
+                "Density": [0.05709372, 0.07670272, 0.05709372],
+                "names": ["a", "a", "a"],
+                "index": [-1.0, 3.0, 7.0],
+            }
+        )
+
+        actual = kdf.a.plot.kde(bw_method=5, ind=3)
+
+        expected = express.line(pdf, x="index", y="Density")
+        expected["layout"]["xaxis"]["title"] = None
+
+        self.assertEqual(pprint.pformat(actual.to_dict()), pprint.pformat(expected.to_dict()))
+
+
+if __name__ == "__main__":
+    from pyspark.pandas.tests.plot.test_series_plot_plotly import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_categorical.py b/python/pyspark/pandas/tests/test_categorical.py
new file mode 100644
index 0000000000000..28de94bbcb849
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_categorical.py
@@ -0,0 +1,475 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from distutils.version import LooseVersion
+
+import numpy as np
+import pandas as pd
+from pandas.api.types import CategoricalDtype
+
+import pyspark.pandas as ps
+from pyspark.testing.pandasutils import PandasOnSparkTestCase, TestUtils
+
+
+class CategoricalTest(PandasOnSparkTestCase, TestUtils):
+    @property
+    def pdf(self):
+        return pd.DataFrame(
+            {
+                "a": pd.Categorical([1, 2, 3, 1, 2, 3]),
+                "b": pd.Categorical(
+                    ["b", "a", "c", "c", "b", "a"], categories=["c", "b", "d", "a"]
+                ),
+            },
+        )
+
+    @property
+    def kdf(self):
+        return ps.from_pandas(self.pdf)
+
+    @property
+    def df_pair(self):
+        return self.pdf, self.kdf
+
+    def test_categorical_frame(self):
+        pdf, kdf = self.df_pair
+
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kdf.a, pdf.a)
+        self.assert_eq(kdf.b, pdf.b)
+        self.assert_eq(kdf.index, pdf.index)
+
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+        self.assert_eq(kdf.sort_values("b"), pdf.sort_values("b"))
+
+    def test_categorical_series(self):
+        pser = pd.Series([1, 2, 3], dtype="category")
+        kser = ps.Series([1, 2, 3], dtype="category")
+
+        self.assert_eq(kser, pser)
+        self.assert_eq(kser.cat.categories, pser.cat.categories)
+        self.assert_eq(kser.cat.codes, pser.cat.codes)
+        self.assert_eq(kser.cat.ordered, pser.cat.ordered)
+
+    def test_astype(self):
+        pser = pd.Series(["a", "b", "c"])
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(kser.astype("category"), pser.astype("category"))
+        self.assert_eq(
+            kser.astype(CategoricalDtype(["c", "a", "b"])),
+            pser.astype(CategoricalDtype(["c", "a", "b"])),
+        )
+
+        pcser = pser.astype(CategoricalDtype(["c", "a", "b"]))
+        kcser = kser.astype(CategoricalDtype(["c", "a", "b"]))
+
+        self.assert_eq(kcser.astype("category"), pcser.astype("category"))
+
+        if LooseVersion(pd.__version__) >= LooseVersion("1.2"):
+            self.assert_eq(
+                kcser.astype(CategoricalDtype(["b", "c", "a"])),
+                pcser.astype(CategoricalDtype(["b", "c", "a"])),
+            )
+        else:
+            self.assert_eq(
+                kcser.astype(CategoricalDtype(["b", "c", "a"])),
+                pser.astype(CategoricalDtype(["b", "c", "a"])),
+            )
+
+        self.assert_eq(kcser.astype(str), pcser.astype(str))
+
+    def test_factorize(self):
+        pser = pd.Series(["a", "b", "c", None], dtype=CategoricalDtype(["c", "a", "d", "b"]))
+        kser = ps.from_pandas(pser)
+
+        pcodes, puniques = pser.factorize()
+        kcodes, kuniques = kser.factorize()
+
+        self.assert_eq(kcodes.tolist(), pcodes.tolist())
+        self.assert_eq(kuniques, puniques)
+
+        pcodes, puniques = pser.factorize(na_sentinel=-2)
+        kcodes, kuniques = kser.factorize(na_sentinel=-2)
+
+        self.assert_eq(kcodes.tolist(), pcodes.tolist())
+        self.assert_eq(kuniques, puniques)
+
+    def test_frame_apply(self):
+        pdf, kdf = self.df_pair
+
+        self.assert_eq(kdf.apply(lambda x: x).sort_index(), pdf.apply(lambda x: x).sort_index())
+        self.assert_eq(
+            kdf.apply(lambda x: x, axis=1).sort_index(), pdf.apply(lambda x: x, axis=1).sort_index()
+        )
+
+    def test_frame_apply_without_shortcut(self):
+        with ps.option_context("compute.shortcut_limit", 0):
+            self.test_frame_apply()
+
+        pdf = pd.DataFrame(
+            {"a": ["a", "b", "c", "a", "b", "c"], "b": ["b", "a", "c", "c", "b", "a"]}
+        )
+        kdf = ps.from_pandas(pdf)
+
+        dtype = CategoricalDtype(categories=["a", "b", "c"])
+
+        def categorize(ser) -> ps.Series[dtype]:
+            return ser.astype(dtype)
+
+        self.assert_eq(
+            kdf.apply(categorize).sort_values(["a", "b"]).reset_index(drop=True),
+            pdf.apply(categorize).sort_values(["a", "b"]).reset_index(drop=True),
+        )
+
+    def test_frame_transform(self):
+        pdf, kdf = self.df_pair
+
+        self.assert_eq(kdf.transform(lambda x: x), pdf.transform(lambda x: x))
+        self.assert_eq(kdf.transform(lambda x: x.cat.codes), pdf.transform(lambda x: x.cat.codes))
+
+        pdf = pd.DataFrame(
+            {"a": ["a", "b", "c", "a", "b", "c"], "b": ["b", "a", "c", "c", "b", "a"]}
+        )
+        kdf = ps.from_pandas(pdf)
+
+        dtype = CategoricalDtype(categories=["a", "b", "c", "d"])
+
+        self.assert_eq(
+            kdf.transform(lambda x: x.astype(dtype)).sort_index(),
+            pdf.transform(lambda x: x.astype(dtype)).sort_index(),
+        )
+
+    def test_frame_transform_without_shortcut(self):
+        with ps.option_context("compute.shortcut_limit", 0):
+            self.test_frame_transform()
+
+        pdf, kdf = self.df_pair
+
+        def codes(pser) -> ps.Series[np.int8]:
+            return pser.cat.codes
+
+        self.assert_eq(kdf.transform(codes), pdf.transform(codes))
+
+        pdf = pd.DataFrame(
+            {"a": ["a", "b", "c", "a", "b", "c"], "b": ["b", "a", "c", "c", "b", "a"]}
+        )
+        kdf = ps.from_pandas(pdf)
+
+        dtype = CategoricalDtype(categories=["a", "b", "c", "d"])
+
+        def to_category(pser) -> ps.Series[dtype]:
+            return pser.astype(dtype)
+
+        self.assert_eq(
+            kdf.transform(to_category).sort_index(), pdf.transform(to_category).sort_index()
+        )
+
+    def test_series_apply(self):
+        pdf, kdf = self.df_pair
+
+        self.assert_eq(kdf.a.apply(lambda x: x).sort_index(), pdf.a.apply(lambda x: x).sort_index())
+
+    def test_series_apply_without_shortcut(self):
+        with ps.option_context("compute.shortcut_limit", 0):
+            self.test_series_apply()
+
+        pdf, kdf = self.df_pair
+        ret = kdf.a.dtype
+
+        def identity(pser) -> ret:
+            return pser
+
+        self.assert_eq(kdf.a.apply(identity).sort_index(), pdf.a.apply(identity).sort_index())
+
+        # TODO: The return type is still category.
+        # def to_str(x) -> str:
+        #     return str(x)
+        #
+        # self.assert_eq(
+        #     kdf.a.apply(to_str).sort_index(), pdf.a.apply(to_str).sort_index()
+        # )
+
+    def test_groupby_apply(self):
+        pdf, kdf = self.df_pair
+
+        self.assert_eq(
+            kdf.groupby("a").apply(lambda df: df).sort_index(),
+            pdf.groupby("a").apply(lambda df: df).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby("b").apply(lambda df: df[["a"]]).sort_index(),
+            pdf.groupby("b").apply(lambda df: df[["a"]]).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(["a", "b"]).apply(lambda df: df).sort_index(),
+            pdf.groupby(["a", "b"]).apply(lambda df: df).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby("a").apply(lambda df: df.b.cat.codes).sort_index(),
+            pdf.groupby("a").apply(lambda df: df.b.cat.codes).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby("a")["b"].apply(lambda b: b.cat.codes).sort_index(),
+            pdf.groupby("a")["b"].apply(lambda b: b.cat.codes).sort_index(),
+        )
+
+        # TODO: grouping by a categorical type sometimes preserves unused categories.
+        # self.assert_eq(
+        #     kdf.groupby("a").apply(len).sort_index(), pdf.groupby("a").apply(len).sort_index(),
+        # )
+
+    def test_groupby_apply_without_shortcut(self):
+        with ps.option_context("compute.shortcut_limit", 0):
+            self.test_groupby_apply()
+
+        pdf, kdf = self.df_pair
+
+        def identity(df) -> ps.DataFrame[zip(kdf.columns, kdf.dtypes)]:
+            return df
+
+        self.assert_eq(
+            kdf.groupby("a").apply(identity).sort_values(["a", "b"]).reset_index(drop=True),
+            pdf.groupby("a").apply(identity).sort_values(["a", "b"]).reset_index(drop=True),
+        )
+
+    def test_groupby_transform(self):
+        pdf, kdf = self.df_pair
+
+        self.assert_eq(
+            kdf.groupby("a").transform(lambda x: x).sort_index(),
+            pdf.groupby("a").transform(lambda x: x).sort_index(),
+        )
+
+        dtype = CategoricalDtype(categories=["a", "b", "c", "d"])
+
+        self.assert_eq(
+            kdf.groupby("a").transform(lambda x: x.astype(dtype)).sort_index(),
+            pdf.groupby("a").transform(lambda x: x.astype(dtype)).sort_index(),
+        )
+
+    def test_groupby_transform_without_shortcut(self):
+        with ps.option_context("compute.shortcut_limit", 0):
+            self.test_groupby_transform()
+
+        pdf, kdf = self.df_pair
+
+        def identity(x) -> ps.Series[kdf.b.dtype]:  # type: ignore
+            return x
+
+        self.assert_eq(
+            kdf.groupby("a").transform(identity).sort_values("b").reset_index(drop=True),
+            pdf.groupby("a").transform(identity).sort_values("b").reset_index(drop=True),
+        )
+
+        dtype = CategoricalDtype(categories=["a", "b", "c", "d"])
+
+        def astype(x) -> ps.Series[dtype]:
+            return x.astype(dtype)
+
+        if LooseVersion(pd.__version__) >= LooseVersion("1.2"):
+            self.assert_eq(
+                kdf.groupby("a").transform(astype).sort_values("b").reset_index(drop=True),
+                pdf.groupby("a").transform(astype).sort_values("b").reset_index(drop=True),
+            )
+        else:
+            expected = pdf.groupby("a").transform(astype)
+            expected["b"] = dtype.categories.take(expected["b"].cat.codes).astype(dtype)
+            self.assert_eq(
+                kdf.groupby("a").transform(astype).sort_values("b").reset_index(drop=True),
+                expected.sort_values("b").reset_index(drop=True),
+            )
+
+    def test_frame_apply_batch(self):
+        pdf, kdf = self.df_pair
+
+        self.assert_eq(
+            kdf.koalas.apply_batch(lambda pdf: pdf.astype(str)).sort_index(),
+            pdf.astype(str).sort_index(),
+        )
+
+        pdf = pd.DataFrame(
+            {"a": ["a", "b", "c", "a", "b", "c"], "b": ["b", "a", "c", "c", "b", "a"]}
+        )
+        kdf = ps.from_pandas(pdf)
+
+        dtype = CategoricalDtype(categories=["a", "b", "c", "d"])
+
+        self.assert_eq(
+            kdf.koalas.apply_batch(lambda pdf: pdf.astype(dtype)).sort_index(),
+            pdf.astype(dtype).sort_index(),
+        )
+
+    def test_frame_apply_batch_without_shortcut(self):
+        with ps.option_context("compute.shortcut_limit", 0):
+            self.test_frame_apply_batch()
+
+        pdf, kdf = self.df_pair
+
+        def to_str(pdf) -> 'ps.DataFrame["a":str, "b":str]':  # noqa: F405
+            return pdf.astype(str)
+
+        self.assert_eq(
+            kdf.koalas.apply_batch(to_str).sort_values(["a", "b"]).reset_index(drop=True),
+            to_str(pdf).sort_values(["a", "b"]).reset_index(drop=True),
+        )
+
+        pdf = pd.DataFrame(
+            {"a": ["a", "b", "c", "a", "b", "c"], "b": ["b", "a", "c", "c", "b", "a"]}
+        )
+        kdf = ps.from_pandas(pdf)
+
+        dtype = CategoricalDtype(categories=["a", "b", "c", "d"])
+        ret = ps.DataFrame["a":dtype, "b":dtype]
+
+        def to_category(pdf) -> ret:
+            return pdf.astype(dtype)
+
+        self.assert_eq(
+            kdf.koalas.apply_batch(to_category).sort_values(["a", "b"]).reset_index(drop=True),
+            to_category(pdf).sort_values(["a", "b"]).reset_index(drop=True),
+        )
+
+    def test_frame_transform_batch(self):
+        pdf, kdf = self.df_pair
+
+        self.assert_eq(
+            kdf.koalas.transform_batch(lambda pdf: pdf.astype(str)).sort_index(),
+            pdf.astype(str).sort_index(),
+        )
+        self.assert_eq(
+            kdf.koalas.transform_batch(lambda pdf: pdf.b.cat.codes).sort_index(),
+            pdf.b.cat.codes.sort_index(),
+        )
+
+        pdf = pd.DataFrame(
+            {"a": ["a", "b", "c", "a", "b", "c"], "b": ["b", "a", "c", "c", "b", "a"]}
+        )
+        kdf = ps.from_pandas(pdf)
+
+        dtype = CategoricalDtype(categories=["a", "b", "c", "d"])
+
+        self.assert_eq(
+            kdf.koalas.transform_batch(lambda pdf: pdf.astype(dtype)).sort_index(),
+            pdf.astype(dtype).sort_index(),
+        )
+        self.assert_eq(
+            kdf.koalas.transform_batch(lambda pdf: pdf.b.astype(dtype)).sort_index(),
+            pdf.b.astype(dtype).sort_index(),
+        )
+
+    def test_frame_transform_batch_without_shortcut(self):
+        with ps.option_context("compute.shortcut_limit", 0):
+            self.test_frame_transform_batch()
+
+        pdf, kdf = self.df_pair
+
+        def to_str(pdf) -> 'ps.DataFrame["a":str, "b":str]':  # noqa: F405
+            return pdf.astype(str)
+
+        self.assert_eq(
+            kdf.koalas.transform_batch(to_str).sort_index(), to_str(pdf).sort_index(),
+        )
+
+        def to_codes(pdf) -> ps.Series[np.int8]:
+            return pdf.b.cat.codes
+
+        self.assert_eq(
+            kdf.koalas.transform_batch(to_codes).sort_index(), to_codes(pdf).sort_index(),
+        )
+
+        pdf = pd.DataFrame(
+            {"a": ["a", "b", "c", "a", "b", "c"], "b": ["b", "a", "c", "c", "b", "a"]}
+        )
+        kdf = ps.from_pandas(pdf)
+
+        dtype = CategoricalDtype(categories=["a", "b", "c", "d"])
+        ret = ps.DataFrame["a":dtype, "b":dtype]
+
+        def to_category(pdf) -> ret:
+            return pdf.astype(dtype)
+
+        self.assert_eq(
+            kdf.koalas.transform_batch(to_category).sort_index(), to_category(pdf).sort_index(),
+        )
+
+        def to_category(pdf) -> ps.Series[dtype]:
+            return pdf.b.astype(dtype)
+
+        self.assert_eq(
+            kdf.koalas.transform_batch(to_category).sort_index(),
+            to_category(pdf).rename().sort_index(),
+        )
+
+    def test_series_transform_batch(self):
+        pdf, kdf = self.df_pair
+
+        self.assert_eq(
+            kdf.a.koalas.transform_batch(lambda pser: pser.astype(str)).sort_index(),
+            pdf.a.astype(str).sort_index(),
+        )
+
+        pdf = pd.DataFrame(
+            {"a": ["a", "b", "c", "a", "b", "c"], "b": ["b", "a", "c", "c", "b", "a"]}
+        )
+        kdf = ps.from_pandas(pdf)
+
+        dtype = CategoricalDtype(categories=["a", "b", "c", "d"])
+
+        self.assert_eq(
+            kdf.a.koalas.transform_batch(lambda pser: pser.astype(dtype)).sort_index(),
+            pdf.a.astype(dtype).sort_index(),
+        )
+
+    def test_series_transform_batch_without_shortcut(self):
+        with ps.option_context("compute.shortcut_limit", 0):
+            self.test_series_transform_batch()
+
+        pdf, kdf = self.df_pair
+
+        def to_str(pser) -> ps.Series[str]:
+            return pser.astype(str)
+
+        self.assert_eq(
+            kdf.a.koalas.transform_batch(to_str).sort_index(), to_str(pdf.a).sort_index()
+        )
+
+        pdf = pd.DataFrame(
+            {"a": ["a", "b", "c", "a", "b", "c"], "b": ["b", "a", "c", "c", "b", "a"]}
+        )
+        kdf = ps.from_pandas(pdf)
+
+        dtype = CategoricalDtype(categories=["a", "b", "c", "d"])
+
+        def to_category(pser) -> ps.Series[dtype]:
+            return pser.astype(dtype)
+
+        self.assert_eq(
+            kdf.a.koalas.transform_batch(to_category).sort_index(), to_category(pdf.a).sort_index()
+        )
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.pandas.tests.test_categorical import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_config.py b/python/pyspark/pandas/tests/test_config.py
new file mode 100644
index 0000000000000..0709caddf8f33
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_config.py
@@ -0,0 +1,155 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from pyspark import pandas as ps
+from pyspark.pandas import config
+from pyspark.pandas.config import Option, DictWrapper
+from pyspark.testing.pandasutils import PandasOnSparkTestCase
+
+
+class ConfigTest(PandasOnSparkTestCase):
+    def setUp(self):
+        config._options_dict["test.config"] = Option(key="test.config", doc="", default="default")
+
+        config._options_dict["test.config.list"] = Option(
+            key="test.config.list", doc="", default=[], types=list
+        )
+        config._options_dict["test.config.float"] = Option(
+            key="test.config.float", doc="", default=1.2, types=float
+        )
+
+        config._options_dict["test.config.int"] = Option(
+            key="test.config.int",
+            doc="",
+            default=1,
+            types=int,
+            check_func=(lambda v: v > 0, "bigger then 0"),
+        )
+        config._options_dict["test.config.int.none"] = Option(
+            key="test.config.int", doc="", default=None, types=(int, type(None))
+        )
+
+    def tearDown(self):
+        ps.reset_option("test.config")
+        del config._options_dict["test.config"]
+        del config._options_dict["test.config.list"]
+        del config._options_dict["test.config.float"]
+        del config._options_dict["test.config.int"]
+        del config._options_dict["test.config.int.none"]
+
+    def test_get_set_reset_option(self):
+        self.assertEqual(ps.get_option("test.config"), "default")
+
+        ps.set_option("test.config", "value")
+        self.assertEqual(ps.get_option("test.config"), "value")
+
+        ps.reset_option("test.config")
+        self.assertEqual(ps.get_option("test.config"), "default")
+
+    def test_get_set_reset_option_different_types(self):
+        ps.set_option("test.config.list", [1, 2, 3, 4])
+        self.assertEqual(ps.get_option("test.config.list"), [1, 2, 3, 4])
+
+        ps.set_option("test.config.float", 5.0)
+        self.assertEqual(ps.get_option("test.config.float"), 5.0)
+
+        ps.set_option("test.config.int", 123)
+        self.assertEqual(ps.get_option("test.config.int"), 123)
+
+        self.assertEqual(ps.get_option("test.config.int.none"), None)  # default None
+        ps.set_option("test.config.int.none", 123)
+        self.assertEqual(ps.get_option("test.config.int.none"), 123)
+        ps.set_option("test.config.int.none", None)
+        self.assertEqual(ps.get_option("test.config.int.none"), None)
+
+    def test_different_types(self):
+        with self.assertRaisesRegex(TypeError, "was <class 'int'>"):
+            ps.set_option("test.config.list", 1)
+
+        with self.assertRaisesRegex(TypeError, "however, expected types are"):
+            ps.set_option("test.config.float", "abc")
+
+        with self.assertRaisesRegex(TypeError, "[<class 'int'>]"):
+            ps.set_option("test.config.int", "abc")
+
+        with self.assertRaisesRegex(TypeError, "(<class 'int'>, <class 'NoneType'>)"):
+            ps.set_option("test.config.int.none", "abc")
+
+    def test_check_func(self):
+        with self.assertRaisesRegex(ValueError, "bigger then 0"):
+            ps.set_option("test.config.int", -1)
+
+    def test_unknown_option(self):
+        with self.assertRaisesRegex(config.OptionError, "No such option"):
+            ps.get_option("unknown")
+
+        with self.assertRaisesRegex(config.OptionError, "Available options"):
+            ps.set_option("unknown", "value")
+
+        with self.assertRaisesRegex(config.OptionError, "test.config"):
+            ps.reset_option("unknown")
+
+    def test_namespace_access(self):
+        try:
+            self.assertEqual(ps.options.compute.max_rows, ps.get_option("compute.max_rows"))
+            ps.options.compute.max_rows = 0
+            self.assertEqual(ps.options.compute.max_rows, 0)
+            self.assertTrue(isinstance(ps.options.compute, DictWrapper))
+
+            wrapper = ps.options.compute
+            self.assertEqual(wrapper.max_rows, ps.get_option("compute.max_rows"))
+            wrapper.max_rows = 1000
+            self.assertEqual(ps.options.compute.max_rows, 1000)
+
+            self.assertRaisesRegex(config.OptionError, "No such option", lambda: ps.options.compu)
+            self.assertRaisesRegex(
+                config.OptionError, "No such option", lambda: ps.options.compute.max
+            )
+            self.assertRaisesRegex(
+                config.OptionError, "No such option", lambda: ps.options.max_rows1
+            )
+
+            with self.assertRaisesRegex(config.OptionError, "No such option"):
+                ps.options.compute.max = 0
+            with self.assertRaisesRegex(config.OptionError, "No such option"):
+                ps.options.compute = 0
+            with self.assertRaisesRegex(config.OptionError, "No such option"):
+                ps.options.com = 0
+        finally:
+            ps.reset_option("compute.max_rows")
+
+    def test_dir_options(self):
+        self.assertTrue("compute.default_index_type" in dir(ps.options))
+        self.assertTrue("plotting.sample_ratio" in dir(ps.options))
+
+        self.assertTrue("default_index_type" in dir(ps.options.compute))
+        self.assertTrue("sample_ratio" not in dir(ps.options.compute))
+
+        self.assertTrue("default_index_type" not in dir(ps.options.plotting))
+        self.assertTrue("sample_ratio" in dir(ps.options.plotting))
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.pandas.tests.test_config import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_csv.py b/python/pyspark/pandas/tests/test_csv.py
new file mode 100644
index 0000000000000..17b3060c9221b
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_csv.py
@@ -0,0 +1,434 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import shutil
+import tempfile
+from contextlib import contextmanager
+
+import pandas as pd
+import numpy as np
+
+from pyspark import pandas as ps
+from pyspark.testing.pandasutils import PandasOnSparkTestCase, TestUtils
+
+
+def normalize_text(s):
+    return "\n".join(map(str.strip, s.strip().split("\n")))
+
+
+class CsvTest(PandasOnSparkTestCase, TestUtils):
+    def setUp(self):
+        self.tmp_dir = tempfile.mkdtemp(prefix=CsvTest.__name__)
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp_dir, ignore_errors=True)
+
+    @property
+    def csv_text(self):
+        return normalize_text(
+            """
+            name,amount
+            Alice,100
+            Bob,-200
+            Charlie,300
+            Dennis,400
+            Edith,-500
+            Frank,600
+            Alice,200
+            Frank,-200
+            Bob,600
+            Alice,400
+            Frank,200
+            Alice,300
+            Edith,600
+            """
+        )
+
+    @property
+    def csv_text_2(self):
+        return normalize_text(
+            """
+            A,B
+            item1,1
+            item2,1,2
+            item3,1,2,3,4
+            item4,1
+            """
+        )
+
+    @property
+    def csv_text_with_comments(self):
+        return normalize_text(
+            """
+            # header
+            %s
+            # comment
+            Alice,400
+            Edith,600
+            # footer
+            """
+            % self.csv_text
+        )
+
+    @property
+    def tab_delimited_csv_text(self):
+        return normalize_text(
+            """
+            name\tamount
+            Alice\t100
+            Bob\t-200
+            Charlie\t300
+            """
+        )
+
+    @property
+    def q_quoted_csv_text(self):
+        return normalize_text(
+            """
+            QnameQ,QamountQ
+            QA,liceQ,Q100Q
+            QB,obQ,Q-200Q
+            QC,harlieQ,Q300Q
+            """
+        )
+
+    @property
+    def e_escapeted_csv_text(self):
+        return normalize_text(
+            """
+            name,amount
+            "AE"lice",100
+            "BE"ob",-200
+            "CE"harlie",300
+            """
+        )
+
+    @contextmanager
+    def csv_file(self, csv):
+        with self.temp_file() as tmp:
+            with open(tmp, "w") as f:
+                f.write(csv)
+            yield tmp
+
+    def test_read_csv(self):
+        with self.csv_file(self.csv_text) as fn:
+
+            def check(header="infer", names=None, usecols=None, index_col=None):
+                expected = pd.read_csv(
+                    fn, header=header, names=names, usecols=usecols, index_col=index_col
+                )
+                actual = ps.read_csv(
+                    fn, header=header, names=names, usecols=usecols, index_col=index_col
+                )
+                self.assert_eq(expected, actual, almost=True)
+
+            check()
+            check(header=0)
+            check(header=None)
+            check(names=["n", "a"])
+            check(names=[("x", "n"), ("y", "a")])
+            check(names=[10, 20])
+            check(header=0, names=["n", "a"])
+            check(usecols=[1])
+            check(usecols=[1, 0])
+            check(usecols=["amount"])
+            check(usecols=["amount", "name"])
+            check(usecols=[])
+            check(usecols=[1, 1])
+            check(usecols=["amount", "amount"])
+            check(header=None, usecols=[1])
+            check(names=["n", "a"], usecols=["a"])
+            check(header=None, names=["n", "a"], usecols=["a"])
+            check(index_col=["amount"])
+            check(header=None, index_col=[1])
+            check(names=["n", "a"], index_col=["a"])
+
+            # check with pyspark patch.
+            expected = pd.read_csv(fn)
+            actual = ps.read_csv(fn)
+            self.assert_eq(expected, actual, almost=True)
+
+            self.assertRaisesRegex(
+                ValueError, "non-unique", lambda: ps.read_csv(fn, names=["n", "n"])
+            )
+            self.assertRaisesRegex(
+                ValueError,
+                "does not match the number.*3",
+                lambda: ps.read_csv(fn, names=["n", "a", "b"]),
+            )
+            self.assertRaisesRegex(
+                ValueError,
+                "does not match the number.*3",
+                lambda: ps.read_csv(fn, header=0, names=["n", "a", "b"]),
+            )
+            self.assertRaisesRegex(
+                ValueError, "Usecols do not match.*3", lambda: ps.read_csv(fn, usecols=[1, 3])
+            )
+            self.assertRaisesRegex(
+                ValueError,
+                "Usecols do not match.*col",
+                lambda: ps.read_csv(fn, usecols=["amount", "col"]),
+            )
+            self.assertRaisesRegex(
+                ValueError, "Unknown header argument 1", lambda: ps.read_csv(fn, header="1")
+            )
+            expected_error_message = (
+                "'usecols' must either be list-like of all strings, "
+                "all unicode, all integers or a callable."
+            )
+            self.assertRaisesRegex(
+                ValueError, expected_error_message, lambda: ps.read_csv(fn, usecols=[1, "amount"])
+            )
+
+            # check with index_col
+            expected = pd.read_csv(fn).set_index("name")
+            actual = ps.read_csv(fn, index_col="name")
+            self.assert_eq(expected, actual, almost=True)
+
+    def test_read_with_spark_schema(self):
+        with self.csv_file(self.csv_text_2) as fn:
+            actual = ps.read_csv(fn, names="A string, B string, C long, D long, E long")
+            expected = pd.read_csv(fn, names=["A", "B", "C", "D", "E"])
+            self.assert_eq(expected, actual)
+
+    def test_read_csv_with_comment(self):
+        with self.csv_file(self.csv_text_with_comments) as fn:
+            expected = pd.read_csv(fn, comment="#")
+            actual = ps.read_csv(fn, comment="#")
+            self.assert_eq(expected, actual, almost=True)
+
+            self.assertRaisesRegex(
+                ValueError,
+                "Only length-1 comment characters supported",
+                lambda: ps.read_csv(fn, comment="").show(),
+            )
+            self.assertRaisesRegex(
+                ValueError,
+                "Only length-1 comment characters supported",
+                lambda: ps.read_csv(fn, comment="##").show(),
+            )
+            self.assertRaisesRegex(
+                ValueError,
+                "Only length-1 comment characters supported",
+                lambda: ps.read_csv(fn, comment=1),
+            )
+            self.assertRaisesRegex(
+                ValueError,
+                "Only length-1 comment characters supported",
+                lambda: ps.read_csv(fn, comment=[1]),
+            )
+
+    def test_read_csv_with_limit(self):
+        with self.csv_file(self.csv_text_with_comments) as fn:
+            expected = pd.read_csv(fn, comment="#", nrows=2)
+            actual = ps.read_csv(fn, comment="#", nrows=2)
+            self.assert_eq(expected, actual, almost=True)
+
+    def test_read_csv_with_sep(self):
+        with self.csv_file(self.tab_delimited_csv_text) as fn:
+            expected = pd.read_csv(fn, sep="\t")
+            actual = ps.read_csv(fn, sep="\t")
+            self.assert_eq(expected, actual, almost=True)
+
+    def test_read_csv_with_squeeze(self):
+        with self.csv_file(self.csv_text) as fn:
+            expected = pd.read_csv(fn, squeeze=True, usecols=["name"])
+            actual = ps.read_csv(fn, squeeze=True, usecols=["name"])
+            self.assert_eq(expected, actual, almost=True)
+
+            expected = pd.read_csv(fn, squeeze=True, usecols=["name", "amount"])
+            actual = ps.read_csv(fn, squeeze=True, usecols=["name", "amount"])
+            self.assert_eq(expected, actual, almost=True)
+
+            expected = pd.read_csv(fn, squeeze=True, usecols=["name", "amount"], index_col=["name"])
+            actual = ps.read_csv(fn, squeeze=True, usecols=["name", "amount"], index_col=["name"])
+            self.assert_eq(expected, actual, almost=True)
+
+    def test_read_csv_with_mangle_dupe_cols(self):
+        self.assertRaisesRegex(
+            ValueError, "mangle_dupe_cols", lambda: ps.read_csv("path", mangle_dupe_cols=False)
+        )
+
+    def test_read_csv_with_parse_dates(self):
+        self.assertRaisesRegex(
+            ValueError, "parse_dates", lambda: ps.read_csv("path", parse_dates=True)
+        )
+
+    def test_read_csv_with_dtype(self):
+        with self.csv_file(self.csv_text) as fn:
+            self.assert_eq(ps.read_csv(fn), pd.read_csv(fn), almost=True)
+            self.assert_eq(ps.read_csv(fn, dtype=str), pd.read_csv(fn, dtype=str))
+            self.assert_eq(
+                ps.read_csv(fn, dtype={"amount": "int64"}),
+                pd.read_csv(fn, dtype={"amount": "int64"}),
+            )
+
+    def test_read_csv_with_quotechar(self):
+        with self.csv_file(self.q_quoted_csv_text) as fn:
+            self.assert_eq(
+                ps.read_csv(fn, quotechar="Q"), pd.read_csv(fn, quotechar="Q"), almost=True
+            )
+
+    def test_read_csv_with_escapechar(self):
+        with self.csv_file(self.e_escapeted_csv_text) as fn:
+            self.assert_eq(
+                ps.read_csv(fn, escapechar="E"), pd.read_csv(fn, escapechar="E"), almost=True
+            )
+
+            self.assert_eq(
+                ps.read_csv(fn, escapechar="ABC", escape="E"),
+                pd.read_csv(fn, escapechar="E"),
+                almost=True,
+            )
+
+    def test_to_csv(self):
+        pdf = pd.DataFrame({"aa": [1, 2, 3], "bb": [4, 5, 6]}, index=[0, 1, 3])
+        kdf = ps.DataFrame(pdf)
+
+        self.assert_eq(kdf.to_csv(), pdf.to_csv(index=False))
+        self.assert_eq(kdf.to_csv(columns=["aa"]), pdf.to_csv(columns=["aa"], index=False))
+        self.assert_eq(kdf.aa.to_csv(), pdf.aa.to_csv(index=False, header=True))
+
+        pdf = pd.DataFrame({"a": [1, np.nan, 3], "b": ["one", "two", None]}, index=[0, 1, 3])
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.to_csv(na_rep="null"), pdf.to_csv(na_rep="null", index=False))
+        self.assert_eq(
+            kdf.a.to_csv(na_rep="null"), pdf.a.to_csv(na_rep="null", index=False, header=True)
+        )
+
+        self.assertRaises(KeyError, lambda: kdf.to_csv(columns=["ab"]))
+
+        pdf = pd.DataFrame({"a": [1.0, 2.0, 3.0], "b": [4.0, 5.0, 6.0]}, index=[0, 1, 3])
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.to_csv(), pdf.to_csv(index=False))
+        self.assert_eq(kdf.to_csv(header=False), pdf.to_csv(header=False, index=False))
+        self.assert_eq(kdf.to_csv(), pdf.to_csv(index=False))
+
+        # non-string names
+        pdf = pd.DataFrame({10: [1, 2, 3], 20: [4, 5, 6]}, index=[0, 1, 3])
+        kdf = ps.DataFrame(pdf)
+
+        self.assert_eq(kdf.to_csv(), pdf.to_csv(index=False))
+        self.assert_eq(kdf.to_csv(columns=[10]), pdf.to_csv(columns=[10], index=False))
+
+        self.assertRaises(TypeError, lambda: kdf.to_csv(columns=10))
+
+    def _check_output(self, dir, expected):
+        output_paths = [path for path in os.listdir(dir) if path.startswith("part-")]
+        assert len(output_paths) > 0
+        output_path = "%s/%s" % (dir, output_paths[0])
+        with open(output_path) as f:
+            self.assertEqual(f.read(), expected)
+
+    def test_to_csv_with_path(self):
+        pdf = pd.DataFrame({"a": [1, 2, 3], "b": ["a", "b", "c"]})
+        kdf = ps.DataFrame(pdf)
+
+        tmp_dir = "{}/tmp1".format(self.tmp_dir)
+
+        kdf.to_csv(tmp_dir, num_files=1)
+        self._check_output(tmp_dir, pdf.to_csv(index=False))
+
+        tmp_dir = "{}/tmp2".format(self.tmp_dir)
+
+        self.assertRaises(KeyError, lambda: kdf.to_csv(tmp_dir, columns=["c"], num_files=1))
+
+        # non-string names
+        pdf = pd.DataFrame({10: [1, 2, 3], 20: ["a", "b", "c"]})
+        kdf = ps.DataFrame(pdf)
+
+        tmp_dir = "{}/tmp3".format(self.tmp_dir)
+
+        kdf.to_csv(tmp_dir, num_files=1)
+        self._check_output(tmp_dir, pdf.to_csv(index=False))
+
+        tmp_dir = "{}/tmp4".format(self.tmp_dir)
+
+        kdf.to_csv(tmp_dir, columns=[10], num_files=1)
+        self._check_output(tmp_dir, pdf.to_csv(columns=[10], index=False))
+
+        tmp_dir = "{}/tmp5".format(self.tmp_dir)
+
+        self.assertRaises(TypeError, lambda: kdf.to_csv(tmp_dir, columns=10, num_files=1))
+
+    def test_to_csv_with_path_and_basic_options(self):
+        pdf = pd.DataFrame({"aa": [1, 2, 3], "bb": ["a", "b", "c"]})
+        kdf = ps.DataFrame(pdf)
+
+        kdf.to_csv(self.tmp_dir, num_files=1, sep="|", header=False, columns=["aa"])
+        expected = pdf.to_csv(index=False, sep="|", header=False, columns=["aa"])
+
+        self._check_output(self.tmp_dir, expected)
+
+    def test_to_csv_with_path_and_basic_options_multiindex_columns(self):
+        pdf = pd.DataFrame({("x", "a"): [1, 2, 3], ("y", "b"): ["a", "b", "c"]})
+        kdf = ps.DataFrame(pdf)
+
+        with self.assertRaises(ValueError):
+            kdf.to_csv(self.tmp_dir, num_files=1, sep="|", columns=[("x", "a")])
+
+        kdf.to_csv(self.tmp_dir, num_files=1, sep="|", header=["a"], columns=[("x", "a")])
+        pdf.columns = ["a", "b"]
+        expected = pdf.to_csv(index=False, sep="|", columns=["a"])
+
+        self._check_output(self.tmp_dir, expected)
+
+    def test_to_csv_with_path_and_pyspark_options(self):
+        pdf = pd.DataFrame({"a": [1, 2, 3, None], "b": ["a", "b", "c", None]})
+        kdf = ps.DataFrame(pdf)
+
+        kdf.to_csv(self.tmp_dir, nullValue="null", num_files=1)
+        expected = pdf.to_csv(index=False, na_rep="null")
+
+        self._check_output(self.tmp_dir, expected)
+
+    def test_to_csv_with_partition_cols(self):
+        pdf = pd.DataFrame({"a": [1, 2, 3], "b": ["a", "b", "c"]})
+        kdf = ps.DataFrame(pdf)
+
+        kdf.to_csv(self.tmp_dir, partition_cols="b", num_files=1)
+
+        partition_paths = [path for path in os.listdir(self.tmp_dir) if path.startswith("b=")]
+        assert len(partition_paths) > 0
+        for partition_path in partition_paths:
+            column, value = partition_path.split("=")
+            expected = pdf[pdf[column] == value].drop("b", axis=1).to_csv(index=False)
+
+            output_paths = [
+                path
+                for path in os.listdir("%s/%s" % (self.tmp_dir, partition_path))
+                if path.startswith("part-")
+            ]
+            assert len(output_paths) > 0
+            output_path = "%s/%s/%s" % (self.tmp_dir, partition_path, output_paths[0])
+            with open(output_path) as f:
+                self.assertEqual(f.read(), expected)
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.pandas.tests.test_csv import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_dataframe.py b/python/pyspark/pandas/tests/test_dataframe.py
new file mode 100644
index 0000000000000..7577f012e622b
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_dataframe.py
@@ -0,0 +1,5526 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from datetime import datetime
+from distutils.version import LooseVersion
+import inspect
+import sys
+import unittest
+from io import StringIO
+
+import numpy as np
+import pandas as pd
+from pandas.tseries.offsets import DateOffset
+from pyspark import StorageLevel
+from pyspark.ml.linalg import SparseVector
+from pyspark.sql import functions as F
+
+from pyspark import pandas as ps
+from pyspark.pandas.config import option_context
+from pyspark.pandas.exceptions import PandasNotImplementedError
+from pyspark.pandas.frame import CachedDataFrame
+from pyspark.pandas.missing.frame import _MissingPandasLikeDataFrame
+from pyspark.pandas.typedef.typehints import (
+    extension_dtypes,
+    extension_dtypes_available,
+    extension_float_dtypes_available,
+    extension_object_dtypes_available,
+)
+from pyspark.testing.pandasutils import (
+    have_tabulate,
+    PandasOnSparkTestCase,
+    SPARK_CONF_ARROW_ENABLED,
+    tabulate_requirement_message,
+)
+from pyspark.testing.sqlutils import SQLTestUtils
+from pyspark.pandas.utils import name_like_string
+
+
+class DataFrameTest(PandasOnSparkTestCase, SQLTestUtils):
+    @property
+    def pdf(self):
+        return pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5, 6, 7, 8, 9], "b": [4, 5, 6, 3, 2, 1, 0, 0, 0]},
+            index=np.random.rand(9),
+        )
+
+    @property
+    def kdf(self):
+        return ps.from_pandas(self.pdf)
+
+    @property
+    def df_pair(self):
+        pdf = self.pdf
+        kdf = ps.from_pandas(pdf)
+        return pdf, kdf
+
+    def test_dataframe(self):
+        pdf, kdf = self.df_pair
+
+        self.assert_eq(kdf["a"] + 1, pdf["a"] + 1)
+
+        self.assert_eq(kdf.columns, pd.Index(["a", "b"]))
+
+        self.assert_eq(kdf[kdf["b"] > 2], pdf[pdf["b"] > 2])
+        self.assert_eq(-kdf[kdf["b"] > 2], -pdf[pdf["b"] > 2])
+        self.assert_eq(kdf[["a", "b"]], pdf[["a", "b"]])
+        self.assert_eq(kdf.a, pdf.a)
+        self.assert_eq(kdf.b.mean(), pdf.b.mean())
+        self.assert_eq(kdf.b.var(), pdf.b.var())
+        self.assert_eq(kdf.b.std(), pdf.b.std())
+
+        pdf, kdf = self.df_pair
+        self.assert_eq(kdf[["a", "b"]], pdf[["a", "b"]])
+
+        self.assertEqual(kdf.a.notnull().rename("x").name, "x")
+
+        # check ps.DataFrame(ps.Series)
+        pser = pd.Series([1, 2, 3], name="x", index=np.random.rand(3))
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pd.DataFrame(pser), ps.DataFrame(kser))
+
+        # check kdf[pd.Index]
+        pdf, kdf = self.df_pair
+        column_mask = pdf.columns.isin(["a", "b"])
+        index_cols = pdf.columns[column_mask]
+        self.assert_eq(kdf[index_cols], pdf[index_cols])
+
+    def _check_extension(self, kdf, pdf):
+        if LooseVersion("1.1") <= LooseVersion(pd.__version__) < LooseVersion("1.2.2"):
+            self.assert_eq(kdf, pdf, check_exact=False)
+            for dtype in kdf.dtypes:
+                self.assertTrue(isinstance(dtype, extension_dtypes))
+        else:
+            self.assert_eq(kdf, pdf)
+
+    @unittest.skipIf(not extension_dtypes_available, "pandas extension dtypes are not available")
+    def test_extension_dtypes(self):
+        pdf = pd.DataFrame(
+            {
+                "a": pd.Series([1, 2, None, 4], dtype="Int8"),
+                "b": pd.Series([1, None, None, 4], dtype="Int16"),
+                "c": pd.Series([1, 2, None, None], dtype="Int32"),
+                "d": pd.Series([None, 2, None, 4], dtype="Int64"),
+            }
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self._check_extension(kdf, pdf)
+        self._check_extension(kdf + F.lit(1).cast("byte"), pdf + 1)
+        self._check_extension(kdf + kdf, pdf + pdf)
+
+    @unittest.skipIf(not extension_dtypes_available, "pandas extension dtypes are not available")
+    def test_astype_extension_dtypes(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 2, None, 4],
+                "b": [1, None, None, 4],
+                "c": [1, 2, None, None],
+                "d": [None, 2, None, 4],
+            }
+        )
+        kdf = ps.from_pandas(pdf)
+
+        astype = {"a": "Int8", "b": "Int16", "c": "Int32", "d": "Int64"}
+
+        self._check_extension(kdf.astype(astype), pdf.astype(astype))
+
+    @unittest.skipIf(
+        not extension_object_dtypes_available, "pandas extension object dtypes are not available"
+    )
+    def test_extension_object_dtypes(self):
+        pdf = pd.DataFrame(
+            {
+                "a": pd.Series(["a", "b", None, "c"], dtype="string"),
+                "b": pd.Series([True, None, False, True], dtype="boolean"),
+            }
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self._check_extension(kdf, pdf)
+
+    @unittest.skipIf(
+        not extension_object_dtypes_available, "pandas extension object dtypes are not available"
+    )
+    def test_astype_extension_object_dtypes(self):
+        pdf = pd.DataFrame({"a": ["a", "b", None, "c"], "b": [True, None, False, True]})
+        kdf = ps.from_pandas(pdf)
+
+        astype = {"a": "string", "b": "boolean"}
+
+        self._check_extension(kdf.astype(astype), pdf.astype(astype))
+
+    @unittest.skipIf(
+        not extension_float_dtypes_available, "pandas extension float dtypes are not available"
+    )
+    def test_extension_float_dtypes(self):
+        pdf = pd.DataFrame(
+            {
+                "a": pd.Series([1.0, 2.0, None, 4.0], dtype="Float32"),
+                "b": pd.Series([1.0, None, 3.0, 4.0], dtype="Float64"),
+            }
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self._check_extension(kdf, pdf)
+        self._check_extension(kdf + 1, pdf + 1)
+        self._check_extension(kdf + kdf, pdf + pdf)
+
+    @unittest.skipIf(
+        not extension_float_dtypes_available, "pandas extension float dtypes are not available"
+    )
+    def test_astype_extension_float_dtypes(self):
+        pdf = pd.DataFrame({"a": [1.0, 2.0, None, 4.0], "b": [1.0, None, 3.0, 4.0]})
+        kdf = ps.from_pandas(pdf)
+
+        astype = {"a": "Float32", "b": "Float64"}
+
+        self._check_extension(kdf.astype(astype), pdf.astype(astype))
+
+    def test_insert(self):
+        #
+        # Basic DataFrame
+        #
+        pdf = pd.DataFrame([1, 2, 3])
+        kdf = ps.from_pandas(pdf)
+
+        kdf.insert(1, "b", 10)
+        pdf.insert(1, "b", 10)
+        self.assert_eq(kdf.sort_index(), pdf.sort_index(), almost=True)
+        kdf.insert(2, "c", 0.1)
+        pdf.insert(2, "c", 0.1)
+        self.assert_eq(kdf.sort_index(), pdf.sort_index(), almost=True)
+        kdf.insert(3, "d", kdf.b + 1)
+        pdf.insert(3, "d", pdf.b + 1)
+        self.assert_eq(kdf.sort_index(), pdf.sort_index(), almost=True)
+
+        kser = ps.Series([4, 5, 6])
+        self.assertRaises(ValueError, lambda: kdf.insert(0, "y", kser))
+        self.assertRaisesRegex(
+            ValueError, "cannot insert b, already exists", lambda: kdf.insert(1, "b", 10)
+        )
+        self.assertRaisesRegex(
+            TypeError,
+            '"column" should be a scalar value or tuple that contains scalar values',
+            lambda: kdf.insert(0, list("abc"), kser),
+        )
+        self.assertRaises(ValueError, lambda: kdf.insert(0, "e", [7, 8, 9, 10]))
+        self.assertRaises(ValueError, lambda: kdf.insert(0, "f", ps.Series([7, 8])))
+        self.assertRaises(AssertionError, lambda: kdf.insert(100, "y", kser))
+        self.assertRaises(AssertionError, lambda: kdf.insert(1, "y", kser, allow_duplicates=True))
+
+        #
+        # DataFrame with MultiIndex as columns
+        #
+        pdf = pd.DataFrame({("x", "a", "b"): [1, 2, 3]})
+        kdf = ps.from_pandas(pdf)
+
+        kdf.insert(1, "b", 10)
+        pdf.insert(1, "b", 10)
+        self.assert_eq(kdf.sort_index(), pdf.sort_index(), almost=True)
+        kdf.insert(2, "c", 0.1)
+        pdf.insert(2, "c", 0.1)
+        self.assert_eq(kdf.sort_index(), pdf.sort_index(), almost=True)
+        kdf.insert(3, "d", kdf.b + 1)
+        pdf.insert(3, "d", pdf.b + 1)
+        self.assert_eq(kdf.sort_index(), pdf.sort_index(), almost=True)
+
+        self.assertRaisesRegex(
+            ValueError, "cannot insert d, already exists", lambda: kdf.insert(4, "d", 11)
+        )
+        self.assertRaisesRegex(
+            ValueError,
+            '"column" must have length equal to number of column levels.',
+            lambda: kdf.insert(4, ("e",), 11),
+        )
+
+    def test_inplace(self):
+        pdf, kdf = self.df_pair
+
+        pser = pdf.a
+        kser = kdf.a
+
+        pdf["a"] = pdf["a"] + 10
+        kdf["a"] = kdf["a"] + 10
+
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kser, pser)
+
+    def test_assign_list(self):
+        pdf, kdf = self.df_pair
+
+        pser = pdf.a
+        kser = kdf.a
+
+        pdf["x"] = [10, 20, 30, 40, 50, 60, 70, 80, 90]
+        kdf["x"] = [10, 20, 30, 40, 50, 60, 70, 80, 90]
+
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+        self.assert_eq(kser, pser)
+
+        with self.assertRaisesRegex(ValueError, "Length of values does not match length of index"):
+            kdf["z"] = [10, 20, 30, 40, 50, 60, 70, 80]
+
+    def test_dataframe_multiindex_columns(self):
+        pdf = pd.DataFrame(
+            {
+                ("x", "a", "1"): [1, 2, 3],
+                ("x", "b", "2"): [4, 5, 6],
+                ("y.z", "c.d", "3"): [7, 8, 9],
+                ("x", "b", "4"): [10, 11, 12],
+            },
+            index=np.random.rand(3),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kdf["x"], pdf["x"])
+        self.assert_eq(kdf["y.z"], pdf["y.z"])
+        self.assert_eq(kdf["x"]["b"], pdf["x"]["b"])
+        self.assert_eq(kdf["x"]["b"]["2"], pdf["x"]["b"]["2"])
+
+        self.assert_eq(kdf.x, pdf.x)
+        self.assert_eq(kdf.x.b, pdf.x.b)
+        self.assert_eq(kdf.x.b["2"], pdf.x.b["2"])
+
+        self.assertRaises(KeyError, lambda: kdf["z"])
+        self.assertRaises(AttributeError, lambda: kdf.z)
+
+        self.assert_eq(kdf[("x",)], pdf[("x",)])
+        self.assert_eq(kdf[("x", "a")], pdf[("x", "a")])
+        self.assert_eq(kdf[("x", "a", "1")], pdf[("x", "a", "1")])
+
+    def test_dataframe_column_level_name(self):
+        column = pd.Index(["A", "B", "C"], name="X")
+        pdf = pd.DataFrame([[1, 2, 3], [4, 5, 6]], columns=column, index=np.random.rand(2))
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kdf.columns.names, pdf.columns.names)
+        self.assert_eq(kdf.to_pandas().columns.names, pdf.columns.names)
+
+    def test_dataframe_multiindex_names_level(self):
+        columns = pd.MultiIndex.from_tuples(
+            [("X", "A", "Z"), ("X", "B", "Z"), ("Y", "C", "Z"), ("Y", "D", "Z")],
+            names=["lvl_1", "lvl_2", "lv_3"],
+        )
+        pdf = pd.DataFrame(
+            [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16], [17, 18, 19, 20]],
+            columns=columns,
+            index=np.random.rand(5),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.columns.names, pdf.columns.names)
+        self.assert_eq(kdf.to_pandas().columns.names, pdf.columns.names)
+
+        kdf1 = ps.from_pandas(pdf)
+        self.assert_eq(kdf1.columns.names, pdf.columns.names)
+
+        self.assertRaises(
+            AssertionError, lambda: ps.DataFrame(kdf1._internal.copy(column_label_names=("level",)))
+        )
+
+        self.assert_eq(kdf["X"], pdf["X"])
+        self.assert_eq(kdf["X"].columns.names, pdf["X"].columns.names)
+        self.assert_eq(kdf["X"].to_pandas().columns.names, pdf["X"].columns.names)
+        self.assert_eq(kdf["X"]["A"], pdf["X"]["A"])
+        self.assert_eq(kdf["X"]["A"].columns.names, pdf["X"]["A"].columns.names)
+        self.assert_eq(kdf["X"]["A"].to_pandas().columns.names, pdf["X"]["A"].columns.names)
+        self.assert_eq(kdf[("X", "A")], pdf[("X", "A")])
+        self.assert_eq(kdf[("X", "A")].columns.names, pdf[("X", "A")].columns.names)
+        self.assert_eq(kdf[("X", "A")].to_pandas().columns.names, pdf[("X", "A")].columns.names)
+        self.assert_eq(kdf[("X", "A", "Z")], pdf[("X", "A", "Z")])
+
+    def test_itertuples(self):
+        pdf = pd.DataFrame({"num_legs": [4, 2], "num_wings": [0, 2]}, index=["dog", "hawk"])
+        kdf = ps.from_pandas(pdf)
+
+        for ptuple, ktuple in zip(
+            pdf.itertuples(index=False, name="Animal"), kdf.itertuples(index=False, name="Animal")
+        ):
+            self.assert_eq(ptuple, ktuple)
+        for ptuple, ktuple in zip(pdf.itertuples(name=None), kdf.itertuples(name=None)):
+            self.assert_eq(ptuple, ktuple)
+
+        pdf.index = pd.MultiIndex.from_arrays(
+            [[1, 2], ["black", "brown"]], names=("count", "color")
+        )
+        kdf = ps.from_pandas(pdf)
+        for ptuple, ktuple in zip(pdf.itertuples(name="Animal"), kdf.itertuples(name="Animal")):
+            self.assert_eq(ptuple, ktuple)
+
+        pdf.columns = pd.MultiIndex.from_arrays(
+            [["CA", "WA"], ["age", "children"]], names=("origin", "info")
+        )
+        kdf = ps.from_pandas(pdf)
+        for ptuple, ktuple in zip(pdf.itertuples(name="Animal"), kdf.itertuples(name="Animal")):
+            self.assert_eq(ptuple, ktuple)
+
+        pdf = pd.DataFrame([1, 2, 3])
+        kdf = ps.from_pandas(pdf)
+        for ptuple, ktuple in zip(
+            (pdf + 1).itertuples(name="num"), (kdf + 1).itertuples(name="num")
+        ):
+            self.assert_eq(ptuple, ktuple)
+
+        # DataFrames with a large number of columns (>254)
+        pdf = pd.DataFrame(np.random.random((1, 255)))
+        kdf = ps.from_pandas(pdf)
+        for ptuple, ktuple in zip(pdf.itertuples(name="num"), kdf.itertuples(name="num")):
+            self.assert_eq(ptuple, ktuple)
+
+    def test_iterrows(self):
+        pdf = pd.DataFrame(
+            {
+                ("x", "a", "1"): [1, 2, 3],
+                ("x", "b", "2"): [4, 5, 6],
+                ("y.z", "c.d", "3"): [7, 8, 9],
+                ("x", "b", "4"): [10, 11, 12],
+            },
+            index=np.random.rand(3),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        for (pdf_k, pdf_v), (kdf_k, kdf_v) in zip(pdf.iterrows(), kdf.iterrows()):
+            self.assert_eq(pdf_k, kdf_k)
+            self.assert_eq(pdf_v, kdf_v)
+
+    def test_reset_index(self):
+        pdf = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}, index=np.random.rand(3))
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.reset_index(), pdf.reset_index())
+        self.assert_eq(kdf.reset_index().index, pdf.reset_index().index)
+        self.assert_eq(kdf.reset_index(drop=True), pdf.reset_index(drop=True))
+
+        pdf.index.name = "a"
+        kdf.index.name = "a"
+
+        with self.assertRaisesRegex(ValueError, "cannot insert a, already exists"):
+            kdf.reset_index()
+
+        self.assert_eq(kdf.reset_index(drop=True), pdf.reset_index(drop=True))
+
+        # inplace
+        pser = pdf.a
+        kser = kdf.a
+        pdf.reset_index(drop=True, inplace=True)
+        kdf.reset_index(drop=True, inplace=True)
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kser, pser)
+
+    def test_reset_index_with_default_index_types(self):
+        pdf = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}, index=np.random.rand(3))
+        kdf = ps.from_pandas(pdf)
+
+        with ps.option_context("compute.default_index_type", "sequence"):
+            self.assert_eq(kdf.reset_index(), pdf.reset_index())
+
+        with ps.option_context("compute.default_index_type", "distributed-sequence"):
+            self.assert_eq(kdf.reset_index(), pdf.reset_index())
+
+        with ps.option_context("compute.default_index_type", "distributed"):
+            # the index is different.
+            self.assert_eq(kdf.reset_index().to_pandas().reset_index(drop=True), pdf.reset_index())
+
+    def test_reset_index_with_multiindex_columns(self):
+        index = pd.MultiIndex.from_tuples(
+            [("bird", "falcon"), ("bird", "parrot"), ("mammal", "lion"), ("mammal", "monkey")],
+            names=["class", "name"],
+        )
+        columns = pd.MultiIndex.from_tuples([("speed", "max"), ("species", "type")])
+        pdf = pd.DataFrame(
+            [(389.0, "fly"), (24.0, "fly"), (80.5, "run"), (np.nan, "jump")],
+            index=index,
+            columns=columns,
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kdf.reset_index(), pdf.reset_index())
+        self.assert_eq(kdf.reset_index(level="class"), pdf.reset_index(level="class"))
+        self.assert_eq(
+            kdf.reset_index(level="class", col_level=1), pdf.reset_index(level="class", col_level=1)
+        )
+        self.assert_eq(
+            kdf.reset_index(level="class", col_level=1, col_fill="species"),
+            pdf.reset_index(level="class", col_level=1, col_fill="species"),
+        )
+        self.assert_eq(
+            kdf.reset_index(level="class", col_level=1, col_fill="genus"),
+            pdf.reset_index(level="class", col_level=1, col_fill="genus"),
+        )
+
+        with self.assertRaisesRegex(IndexError, "Index has only 2 levels, not 3"):
+            kdf.reset_index(col_level=2)
+
+        pdf.index.names = [("x", "class"), ("y", "name")]
+        kdf.index.names = [("x", "class"), ("y", "name")]
+
+        self.assert_eq(kdf.reset_index(), pdf.reset_index())
+
+        with self.assertRaisesRegex(ValueError, "Item must have length equal to number of levels."):
+            kdf.reset_index(col_level=1)
+
+    def test_index_to_frame_reset_index(self):
+        def check(kdf, pdf):
+            self.assert_eq(kdf.reset_index(), pdf.reset_index())
+            self.assert_eq(kdf.reset_index(drop=True), pdf.reset_index(drop=True))
+
+            pdf.reset_index(drop=True, inplace=True)
+            kdf.reset_index(drop=True, inplace=True)
+            self.assert_eq(kdf, pdf)
+
+        pdf, kdf = self.df_pair
+        check(kdf.index.to_frame(), pdf.index.to_frame())
+        check(kdf.index.to_frame(index=False), pdf.index.to_frame(index=False))
+
+        if LooseVersion(pd.__version__) >= LooseVersion("0.24"):
+            # The `name` argument is added in pandas 0.24.
+            check(kdf.index.to_frame(name="a"), pdf.index.to_frame(name="a"))
+            check(
+                kdf.index.to_frame(index=False, name="a"), pdf.index.to_frame(index=False, name="a")
+            )
+            check(kdf.index.to_frame(name=("x", "a")), pdf.index.to_frame(name=("x", "a")))
+            check(
+                kdf.index.to_frame(index=False, name=("x", "a")),
+                pdf.index.to_frame(index=False, name=("x", "a")),
+            )
+
+    def test_multiindex_column_access(self):
+        columns = pd.MultiIndex.from_tuples(
+            [
+                ("a", "", "", "b"),
+                ("c", "", "d", ""),
+                ("e", "", "f", ""),
+                ("e", "g", "", ""),
+                ("", "", "", "h"),
+                ("i", "", "", ""),
+            ]
+        )
+
+        pdf = pd.DataFrame(
+            [
+                (1, "a", "x", 10, 100, 1000),
+                (2, "b", "y", 20, 200, 2000),
+                (3, "c", "z", 30, 300, 3000),
+            ],
+            columns=columns,
+            index=np.random.rand(3),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kdf["a"], pdf["a"])
+        self.assert_eq(kdf["a"]["b"], pdf["a"]["b"])
+        self.assert_eq(kdf["c"], pdf["c"])
+        self.assert_eq(kdf["c"]["d"], pdf["c"]["d"])
+        self.assert_eq(kdf["e"], pdf["e"])
+        self.assert_eq(kdf["e"][""]["f"], pdf["e"][""]["f"])
+        self.assert_eq(kdf["e"]["g"], pdf["e"]["g"])
+        self.assert_eq(kdf[""], pdf[""])
+        self.assert_eq(kdf[""]["h"], pdf[""]["h"])
+        self.assert_eq(kdf["i"], pdf["i"])
+
+        self.assert_eq(kdf[["a", "e"]], pdf[["a", "e"]])
+        self.assert_eq(kdf[["e", "a"]], pdf[["e", "a"]])
+
+        self.assert_eq(kdf[("a",)], pdf[("a",)])
+        self.assert_eq(kdf[("e", "g")], pdf[("e", "g")])
+        # self.assert_eq(kdf[("i",)], pdf[("i",)])
+        self.assert_eq(kdf[("i", "")], pdf[("i", "")])
+
+        self.assertRaises(KeyError, lambda: kdf[("a", "b")])
+
+    def test_repr_cache_invalidation(self):
+        # If there is any cache, inplace operations should invalidate it.
+        df = ps.range(10)
+        df.__repr__()
+        df["a"] = df["id"]
+        self.assertEqual(df.__repr__(), df.to_pandas().__repr__())
+
+    def test_repr_html_cache_invalidation(self):
+        # If there is any cache, inplace operations should invalidate it.
+        df = ps.range(10)
+        df._repr_html_()
+        df["a"] = df["id"]
+        self.assertEqual(df._repr_html_(), df.to_pandas()._repr_html_())
+
+    def test_empty_dataframe(self):
+        pdf = pd.DataFrame({"a": pd.Series([], dtype="i1"), "b": pd.Series([], dtype="str")})
+
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kdf, pdf)
+
+        with self.sql_conf({SPARK_CONF_ARROW_ENABLED: False}):
+            kdf = ps.from_pandas(pdf)
+            self.assert_eq(kdf, pdf)
+
+    def test_all_null_dataframe(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [None, None, None, "a"],
+                "b": [None, None, None, 1],
+                "c": [None, None, None] + list(np.arange(1, 2).astype("i1")),
+                "d": [None, None, None, 1.0],
+                "e": [None, None, None, True],
+                "f": [None, None, None] + list(pd.date_range("20130101", periods=1)),
+            },
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.iloc[:-1], pdf.iloc[:-1])
+
+        with self.sql_conf({SPARK_CONF_ARROW_ENABLED: False}):
+            self.assert_eq(kdf.iloc[:-1], pdf.iloc[:-1])
+
+        pdf = pd.DataFrame(
+            {
+                "a": pd.Series([None, None, None], dtype="float64"),
+                "b": pd.Series([None, None, None], dtype="str"),
+            },
+        )
+
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kdf, pdf)
+
+        with self.sql_conf({SPARK_CONF_ARROW_ENABLED: False}):
+            kdf = ps.from_pandas(pdf)
+            self.assert_eq(kdf, pdf)
+
+    def test_nullable_object(self):
+        pdf = pd.DataFrame(
+            {
+                "a": list("abc") + [np.nan, None],
+                "b": list(range(1, 4)) + [np.nan, None],
+                "c": list(np.arange(3, 6).astype("i1")) + [np.nan, None],
+                "d": list(np.arange(4.0, 7.0, dtype="float64")) + [np.nan, None],
+                "e": [True, False, True, np.nan, None],
+                "f": list(pd.date_range("20130101", periods=3)) + [np.nan, None],
+            },
+            index=np.random.rand(5),
+        )
+
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kdf, pdf)
+
+        with self.sql_conf({SPARK_CONF_ARROW_ENABLED: False}):
+            kdf = ps.from_pandas(pdf)
+            self.assert_eq(kdf, pdf)
+
+    def test_assign(self):
+        pdf, kdf = self.df_pair
+
+        kdf["w"] = 1.0
+        pdf["w"] = 1.0
+
+        self.assert_eq(kdf, pdf)
+
+        kdf.w = 10.0
+        pdf.w = 10.0
+
+        self.assert_eq(kdf, pdf)
+
+        kdf[1] = 1.0
+        pdf[1] = 1.0
+
+        self.assert_eq(kdf, pdf)
+
+        kdf = kdf.assign(a=kdf["a"] * 2)
+        pdf = pdf.assign(a=pdf["a"] * 2)
+
+        self.assert_eq(kdf, pdf)
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "w"), ("y", "v")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        kdf[("a", "c")] = "def"
+        pdf[("a", "c")] = "def"
+
+        self.assert_eq(kdf, pdf)
+
+        kdf = kdf.assign(Z="ZZ")
+        pdf = pdf.assign(Z="ZZ")
+
+        self.assert_eq(kdf, pdf)
+
+        kdf["x"] = "ghi"
+        pdf["x"] = "ghi"
+
+        self.assert_eq(kdf, pdf)
+
+    def test_head(self):
+        pdf, kdf = self.df_pair
+
+        self.assert_eq(kdf.head(2), pdf.head(2))
+        self.assert_eq(kdf.head(3), pdf.head(3))
+        self.assert_eq(kdf.head(0), pdf.head(0))
+        self.assert_eq(kdf.head(-3), pdf.head(-3))
+        self.assert_eq(kdf.head(-10), pdf.head(-10))
+
+    def test_attributes(self):
+        kdf = self.kdf
+
+        self.assertIn("a", dir(kdf))
+        self.assertNotIn("foo", dir(kdf))
+        self.assertRaises(AttributeError, lambda: kdf.foo)
+
+        kdf = ps.DataFrame({"a b c": [1, 2, 3]})
+        self.assertNotIn("a b c", dir(kdf))
+        kdf = ps.DataFrame({"a": [1, 2], 5: [1, 2]})
+        self.assertIn("a", dir(kdf))
+        self.assertNotIn(5, dir(kdf))
+
+    def test_column_names(self):
+        pdf, kdf = self.df_pair
+
+        self.assert_eq(kdf.columns, pdf.columns)
+        self.assert_eq(kdf[["b", "a"]].columns, pdf[["b", "a"]].columns)
+        self.assert_eq(kdf["a"].name, pdf["a"].name)
+        self.assert_eq((kdf["a"] + 1).name, (pdf["a"] + 1).name)
+
+        self.assert_eq((kdf.a + kdf.b).name, (pdf.a + pdf.b).name)
+        self.assert_eq((kdf.a + kdf.b.rename("a")).name, (pdf.a + pdf.b.rename("a")).name)
+        self.assert_eq((kdf.a + kdf.b.rename()).name, (pdf.a + pdf.b.rename()).name)
+        self.assert_eq((kdf.a.rename() + kdf.b).name, (pdf.a.rename() + pdf.b).name)
+        self.assert_eq(
+            (kdf.a.rename() + kdf.b.rename()).name, (pdf.a.rename() + pdf.b.rename()).name
+        )
+
+    def test_rename_columns(self):
+        pdf = pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5, 6, 7], "b": [7, 6, 5, 4, 3, 2, 1]}, index=np.random.rand(7)
+        )
+        kdf = ps.from_pandas(pdf)
+
+        kdf.columns = ["x", "y"]
+        pdf.columns = ["x", "y"]
+        self.assert_eq(kdf.columns, pd.Index(["x", "y"]))
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kdf._internal.data_spark_column_names, ["x", "y"])
+        self.assert_eq(kdf.to_spark().columns, ["x", "y"])
+        self.assert_eq(kdf.to_spark(index_col="index").columns, ["index", "x", "y"])
+
+        columns = pdf.columns
+        columns.name = "lvl_1"
+
+        kdf.columns = columns
+        self.assert_eq(kdf.columns.names, ["lvl_1"])
+        self.assert_eq(kdf, pdf)
+
+        msg = "Length mismatch: Expected axis has 2 elements, new values have 4 elements"
+        with self.assertRaisesRegex(ValueError, msg):
+            kdf.columns = [1, 2, 3, 4]
+
+        # Multi-index columns
+        pdf = pd.DataFrame(
+            {("A", "0"): [1, 2, 2, 3], ("B", "1"): [1, 2, 3, 4]}, index=np.random.rand(4)
+        )
+        kdf = ps.from_pandas(pdf)
+
+        columns = pdf.columns
+        self.assert_eq(kdf.columns, columns)
+        self.assert_eq(kdf, pdf)
+
+        pdf.columns = ["x", "y"]
+        kdf.columns = ["x", "y"]
+        self.assert_eq(kdf.columns, pd.Index(["x", "y"]))
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kdf._internal.data_spark_column_names, ["x", "y"])
+        self.assert_eq(kdf.to_spark().columns, ["x", "y"])
+        self.assert_eq(kdf.to_spark(index_col="index").columns, ["index", "x", "y"])
+
+        pdf.columns = columns
+        kdf.columns = columns
+        self.assert_eq(kdf.columns, columns)
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kdf._internal.data_spark_column_names, ["(A, 0)", "(B, 1)"])
+        self.assert_eq(kdf.to_spark().columns, ["(A, 0)", "(B, 1)"])
+        self.assert_eq(kdf.to_spark(index_col="index").columns, ["index", "(A, 0)", "(B, 1)"])
+
+        columns.names = ["lvl_1", "lvl_2"]
+
+        kdf.columns = columns
+        self.assert_eq(kdf.columns.names, ["lvl_1", "lvl_2"])
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kdf._internal.data_spark_column_names, ["(A, 0)", "(B, 1)"])
+        self.assert_eq(kdf.to_spark().columns, ["(A, 0)", "(B, 1)"])
+        self.assert_eq(kdf.to_spark(index_col="index").columns, ["index", "(A, 0)", "(B, 1)"])
+
+    def test_rename_dataframe(self):
+        pdf1 = pd.DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]})
+        kdf1 = ps.from_pandas(pdf1)
+
+        self.assert_eq(
+            kdf1.rename(columns={"A": "a", "B": "b"}), pdf1.rename(columns={"A": "a", "B": "b"})
+        )
+
+        result_kdf = kdf1.rename(index={1: 10, 2: 20})
+        result_pdf = pdf1.rename(index={1: 10, 2: 20})
+        self.assert_eq(result_kdf, result_pdf)
+
+        # inplace
+        pser = result_pdf.A
+        kser = result_kdf.A
+        result_kdf.rename(index={10: 100, 20: 200}, inplace=True)
+        result_pdf.rename(index={10: 100, 20: 200}, inplace=True)
+        self.assert_eq(result_kdf, result_pdf)
+        self.assert_eq(kser, pser)
+
+        def str_lower(s) -> str:
+            return str.lower(s)
+
+        self.assert_eq(
+            kdf1.rename(str_lower, axis="columns"), pdf1.rename(str_lower, axis="columns")
+        )
+
+        def mul10(x) -> int:
+            return x * 10
+
+        self.assert_eq(kdf1.rename(mul10, axis="index"), pdf1.rename(mul10, axis="index"))
+
+        self.assert_eq(
+            kdf1.rename(columns=str_lower, index={1: 10, 2: 20}),
+            pdf1.rename(columns=str_lower, index={1: 10, 2: 20}),
+        )
+
+        idx = pd.MultiIndex.from_tuples([("X", "A"), ("X", "B"), ("Y", "C"), ("Y", "D")])
+        pdf2 = pd.DataFrame([[1, 2, 3, 4], [5, 6, 7, 8]], columns=idx)
+        kdf2 = ps.from_pandas(pdf2)
+
+        self.assert_eq(kdf2.rename(columns=str_lower), pdf2.rename(columns=str_lower))
+
+        self.assert_eq(
+            kdf2.rename(columns=str_lower, level=0), pdf2.rename(columns=str_lower, level=0)
+        )
+        self.assert_eq(
+            kdf2.rename(columns=str_lower, level=1), pdf2.rename(columns=str_lower, level=1)
+        )
+
+        pdf3 = pd.DataFrame([[1, 2], [3, 4], [5, 6], [7, 8]], index=idx, columns=list("ab"))
+        kdf3 = ps.from_pandas(pdf3)
+
+        self.assert_eq(kdf3.rename(index=str_lower), pdf3.rename(index=str_lower))
+        self.assert_eq(kdf3.rename(index=str_lower, level=0), pdf3.rename(index=str_lower, level=0))
+        self.assert_eq(kdf3.rename(index=str_lower, level=1), pdf3.rename(index=str_lower, level=1))
+
+        pdf4 = pdf2 + 1
+        kdf4 = kdf2 + 1
+        self.assert_eq(kdf4.rename(columns=str_lower), pdf4.rename(columns=str_lower))
+
+        pdf5 = pdf3 + 1
+        kdf5 = kdf3 + 1
+        self.assert_eq(kdf5.rename(index=str_lower), pdf5.rename(index=str_lower))
+
+    def test_rename_axis(self):
+        index = pd.Index(["A", "B", "C"], name="index")
+        columns = pd.Index(["numbers", "values"], name="cols")
+        pdf = pd.DataFrame([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], index=index, columns=columns)
+        kdf = ps.from_pandas(pdf)
+
+        for axis in [0, "index"]:
+            self.assert_eq(
+                pdf.rename_axis("index2", axis=axis).sort_index(),
+                kdf.rename_axis("index2", axis=axis).sort_index(),
+            )
+            self.assert_eq(
+                pdf.rename_axis(["index2"], axis=axis).sort_index(),
+                kdf.rename_axis(["index2"], axis=axis).sort_index(),
+            )
+
+        for axis in [1, "columns"]:
+            self.assert_eq(
+                pdf.rename_axis("cols2", axis=axis).sort_index(),
+                kdf.rename_axis("cols2", axis=axis).sort_index(),
+            )
+            self.assert_eq(
+                pdf.rename_axis(["cols2"], axis=axis).sort_index(),
+                kdf.rename_axis(["cols2"], axis=axis).sort_index(),
+            )
+
+        pdf2 = pdf.copy()
+        kdf2 = kdf.copy()
+        pdf2.rename_axis("index2", axis="index", inplace=True)
+        kdf2.rename_axis("index2", axis="index", inplace=True)
+        self.assert_eq(pdf2.sort_index(), kdf2.sort_index())
+
+        self.assertRaises(ValueError, lambda: kdf.rename_axis(["index2", "index3"], axis=0))
+        self.assertRaises(ValueError, lambda: kdf.rename_axis(["cols2", "cols3"], axis=1))
+        self.assertRaises(TypeError, lambda: kdf.rename_axis(mapper=["index2"], index=["index3"]))
+
+        # index/columns parameters and dict_like/functions mappers introduced in pandas 0.24.0
+        if LooseVersion(pd.__version__) >= LooseVersion("0.24.0"):
+            self.assert_eq(
+                pdf.rename_axis(index={"index": "index2"}, columns={"cols": "cols2"}).sort_index(),
+                kdf.rename_axis(index={"index": "index2"}, columns={"cols": "cols2"}).sort_index(),
+            )
+
+            self.assert_eq(
+                pdf.rename_axis(
+                    index={"missing": "index2"}, columns={"missing": "cols2"}
+                ).sort_index(),
+                kdf.rename_axis(
+                    index={"missing": "index2"}, columns={"missing": "cols2"}
+                ).sort_index(),
+            )
+
+            self.assert_eq(
+                pdf.rename_axis(index=str.upper, columns=str.upper).sort_index(),
+                kdf.rename_axis(index=str.upper, columns=str.upper).sort_index(),
+            )
+        else:
+            expected = pdf
+            expected.index.name = "index2"
+            expected.columns.name = "cols2"
+            result = kdf.rename_axis(
+                index={"index": "index2"}, columns={"cols": "cols2"}
+            ).sort_index()
+            self.assert_eq(expected, result)
+
+            expected.index.name = "index"
+            expected.columns.name = "cols"
+            result = kdf.rename_axis(
+                index={"missing": "index2"}, columns={"missing": "cols2"}
+            ).sort_index()
+            self.assert_eq(expected, result)
+
+            expected.index.name = "INDEX"
+            expected.columns.name = "COLS"
+            result = kdf.rename_axis(index=str.upper, columns=str.upper).sort_index()
+            self.assert_eq(expected, result)
+
+        index = pd.MultiIndex.from_tuples(
+            [("A", "B"), ("C", "D"), ("E", "F")], names=["index1", "index2"]
+        )
+        columns = pd.MultiIndex.from_tuples(
+            [("numbers", "first"), ("values", "second")], names=["cols1", "cols2"]
+        )
+        pdf = pd.DataFrame([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], index=index, columns=columns)
+        kdf = ps.from_pandas(pdf)
+
+        for axis in [0, "index"]:
+            self.assert_eq(
+                pdf.rename_axis(["index3", "index4"], axis=axis).sort_index(),
+                kdf.rename_axis(["index3", "index4"], axis=axis).sort_index(),
+            )
+
+        for axis in [1, "columns"]:
+            self.assert_eq(
+                pdf.rename_axis(["cols3", "cols4"], axis=axis).sort_index(),
+                kdf.rename_axis(["cols3", "cols4"], axis=axis).sort_index(),
+            )
+
+        self.assertRaises(
+            ValueError, lambda: kdf.rename_axis(["index3", "index4", "index5"], axis=0)
+        )
+        self.assertRaises(ValueError, lambda: kdf.rename_axis(["cols3", "cols4", "cols5"], axis=1))
+
+        # index/columns parameters and dict_like/functions mappers introduced in pandas 0.24.0
+        if LooseVersion(pd.__version__) >= LooseVersion("0.24.0"):
+            self.assert_eq(
+                pdf.rename_axis(
+                    index={"index1": "index3"}, columns={"cols1": "cols3"}
+                ).sort_index(),
+                kdf.rename_axis(
+                    index={"index1": "index3"}, columns={"cols1": "cols3"}
+                ).sort_index(),
+            )
+
+            self.assert_eq(
+                pdf.rename_axis(
+                    index={"missing": "index3"}, columns={"missing": "cols3"}
+                ).sort_index(),
+                kdf.rename_axis(
+                    index={"missing": "index3"}, columns={"missing": "cols3"}
+                ).sort_index(),
+            )
+
+            self.assert_eq(
+                pdf.rename_axis(
+                    index={"index1": "index3", "index2": "index4"},
+                    columns={"cols1": "cols3", "cols2": "cols4"},
+                ).sort_index(),
+                kdf.rename_axis(
+                    index={"index1": "index3", "index2": "index4"},
+                    columns={"cols1": "cols3", "cols2": "cols4"},
+                ).sort_index(),
+            )
+
+            self.assert_eq(
+                pdf.rename_axis(index=str.upper, columns=str.upper).sort_index(),
+                kdf.rename_axis(index=str.upper, columns=str.upper).sort_index(),
+            )
+        else:
+            expected = pdf
+            expected.index.names = ["index3", "index2"]
+            expected.columns.names = ["cols3", "cols2"]
+            result = kdf.rename_axis(
+                index={"index1": "index3"}, columns={"cols1": "cols3"}
+            ).sort_index()
+            self.assert_eq(expected, result)
+
+            expected.index.names = ["index1", "index2"]
+            expected.columns.names = ["cols1", "cols2"]
+            result = kdf.rename_axis(
+                index={"missing": "index2"}, columns={"missing": "cols2"}
+            ).sort_index()
+            self.assert_eq(expected, result)
+
+            expected.index.names = ["index3", "index4"]
+            expected.columns.names = ["cols3", "cols4"]
+            result = kdf.rename_axis(
+                index={"index1": "index3", "index2": "index4"},
+                columns={"cols1": "cols3", "cols2": "cols4"},
+            ).sort_index()
+            self.assert_eq(expected, result)
+
+            expected.index.names = ["INDEX1", "INDEX2"]
+            expected.columns.names = ["COLS1", "COLS2"]
+            result = kdf.rename_axis(index=str.upper, columns=str.upper).sort_index()
+            self.assert_eq(expected, result)
+
+    def test_dot_in_column_name(self):
+        self.assert_eq(
+            ps.DataFrame(ps.range(1)._internal.spark_frame.selectExpr("1L as `a.b`"))["a.b"],
+            ps.Series([1], name="a.b"),
+        )
+
+    def test_aggregate(self):
+        pdf = pd.DataFrame(
+            [[1, 2, 3], [4, 5, 6], [7, 8, 9], [np.nan, np.nan, np.nan]], columns=["A", "B", "C"]
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            kdf.agg(["sum", "min"])[["A", "B", "C"]].sort_index(),  # TODO?: fix column order
+            pdf.agg(["sum", "min"])[["A", "B", "C"]].sort_index(),
+        )
+        self.assert_eq(
+            kdf.agg({"A": ["sum", "min"], "B": ["min", "max"]})[["A", "B"]].sort_index(),
+            pdf.agg({"A": ["sum", "min"], "B": ["min", "max"]})[["A", "B"]].sort_index(),
+        )
+
+        self.assertRaises(KeyError, lambda: kdf.agg({"A": ["sum", "min"], "X": ["min", "max"]}))
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("X", "A"), ("X", "B"), ("Y", "C")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(
+            kdf.agg(["sum", "min"])[[("X", "A"), ("X", "B"), ("Y", "C")]].sort_index(),
+            pdf.agg(["sum", "min"])[[("X", "A"), ("X", "B"), ("Y", "C")]].sort_index(),
+        )
+        self.assert_eq(
+            kdf.agg({("X", "A"): ["sum", "min"], ("X", "B"): ["min", "max"]})[
+                [("X", "A"), ("X", "B")]
+            ].sort_index(),
+            pdf.agg({("X", "A"): ["sum", "min"], ("X", "B"): ["min", "max"]})[
+                [("X", "A"), ("X", "B")]
+            ].sort_index(),
+        )
+
+        self.assertRaises(TypeError, lambda: kdf.agg({"X": ["sum", "min"], "Y": ["min", "max"]}))
+
+        # non-string names
+        pdf = pd.DataFrame(
+            [[1, 2, 3], [4, 5, 6], [7, 8, 9], [np.nan, np.nan, np.nan]], columns=[10, 20, 30]
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            kdf.agg(["sum", "min"])[[10, 20, 30]].sort_index(),
+            pdf.agg(["sum", "min"])[[10, 20, 30]].sort_index(),
+        )
+        self.assert_eq(
+            kdf.agg({10: ["sum", "min"], 20: ["min", "max"]})[[10, 20]].sort_index(),
+            pdf.agg({10: ["sum", "min"], 20: ["min", "max"]})[[10, 20]].sort_index(),
+        )
+
+        columns = pd.MultiIndex.from_tuples([("X", 10), ("X", 20), ("Y", 30)])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(
+            kdf.agg(["sum", "min"])[[("X", 10), ("X", 20), ("Y", 30)]].sort_index(),
+            pdf.agg(["sum", "min"])[[("X", 10), ("X", 20), ("Y", 30)]].sort_index(),
+        )
+        self.assert_eq(
+            kdf.agg({("X", 10): ["sum", "min"], ("X", 20): ["min", "max"]})[
+                [("X", 10), ("X", 20)]
+            ].sort_index(),
+            pdf.agg({("X", 10): ["sum", "min"], ("X", 20): ["min", "max"]})[
+                [("X", 10), ("X", 20)]
+            ].sort_index(),
+        )
+
+        pdf = pd.DataFrame(
+            [datetime(2019, 2, 2, 0, 0, 0, 0), datetime(2019, 2, 3, 0, 0, 0, 0)],
+            columns=["timestamp"],
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.timestamp.min(), pdf.timestamp.min())
+        self.assert_eq(kdf.timestamp.max(), pdf.timestamp.max())
+
+    def test_droplevel(self):
+        pdf = (
+            pd.DataFrame([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]])
+            .set_index([0, 1])
+            .rename_axis(["a", "b"])
+        )
+        pdf.columns = pd.MultiIndex.from_tuples(
+            [("c", "e"), ("d", "f")], names=["level_1", "level_2"]
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assertRaises(ValueError, lambda: kdf.droplevel(["a", "b"]))
+        self.assertRaises(ValueError, lambda: kdf.droplevel([1, 1, 1, 1, 1]))
+        self.assertRaises(IndexError, lambda: kdf.droplevel(2))
+        self.assertRaises(IndexError, lambda: kdf.droplevel(-3))
+        self.assertRaises(KeyError, lambda: kdf.droplevel({"a"}))
+        self.assertRaises(KeyError, lambda: kdf.droplevel({"a": 1}))
+
+        self.assertRaises(ValueError, lambda: kdf.droplevel(["level_1", "level_2"], axis=1))
+        self.assertRaises(IndexError, lambda: kdf.droplevel(2, axis=1))
+        self.assertRaises(IndexError, lambda: kdf.droplevel(-3, axis=1))
+        self.assertRaises(KeyError, lambda: kdf.droplevel({"level_1"}, axis=1))
+        self.assertRaises(KeyError, lambda: kdf.droplevel({"level_1": 1}, axis=1))
+
+        # droplevel is new in pandas 0.24.0
+        if LooseVersion(pd.__version__) >= LooseVersion("0.24.0"):
+            self.assert_eq(pdf.droplevel("a"), kdf.droplevel("a"))
+            self.assert_eq(pdf.droplevel(["a"]), kdf.droplevel(["a"]))
+            self.assert_eq(pdf.droplevel(("a",)), kdf.droplevel(("a",)))
+            self.assert_eq(pdf.droplevel(0), kdf.droplevel(0))
+            self.assert_eq(pdf.droplevel(-1), kdf.droplevel(-1))
+
+            self.assert_eq(pdf.droplevel("level_1", axis=1), kdf.droplevel("level_1", axis=1))
+            self.assert_eq(pdf.droplevel(["level_1"], axis=1), kdf.droplevel(["level_1"], axis=1))
+            self.assert_eq(pdf.droplevel(("level_1",), axis=1), kdf.droplevel(("level_1",), axis=1))
+            self.assert_eq(pdf.droplevel(0, axis=1), kdf.droplevel(0, axis=1))
+            self.assert_eq(pdf.droplevel(-1, axis=1), kdf.droplevel(-1, axis=1))
+        else:
+            expected = pdf.copy()
+            expected.index = expected.index.droplevel("a")
+
+            self.assert_eq(expected, kdf.droplevel("a"))
+            self.assert_eq(expected, kdf.droplevel(["a"]))
+            self.assert_eq(expected, kdf.droplevel(("a",)))
+            self.assert_eq(expected, kdf.droplevel(0))
+
+            expected = pdf.copy()
+            expected.index = expected.index.droplevel(-1)
+
+            self.assert_eq(expected, kdf.droplevel(-1))
+
+            expected = pdf.copy()
+            expected.columns = expected.columns.droplevel("level_1")
+
+            self.assert_eq(expected, kdf.droplevel("level_1", axis=1))
+            self.assert_eq(expected, kdf.droplevel(["level_1"], axis=1))
+            self.assert_eq(expected, kdf.droplevel(("level_1",), axis=1))
+            self.assert_eq(expected, kdf.droplevel(0, axis=1))
+
+            expected = pdf.copy()
+            expected.columns = expected.columns.droplevel(-1)
+
+            self.assert_eq(expected, kdf.droplevel(-1, axis=1))
+
+        # Tupled names
+        pdf.columns.names = [("level", 1), ("level", 2)]
+        pdf.index.names = [("a", 10), ("x", 20)]
+        kdf = ps.from_pandas(pdf)
+
+        self.assertRaises(KeyError, lambda: kdf.droplevel("a"))
+        self.assertRaises(KeyError, lambda: kdf.droplevel(("a", 10)))
+
+        # droplevel is new in pandas 0.24.0
+        if LooseVersion(pd.__version__) >= LooseVersion("0.24.0"):
+            self.assert_eq(pdf.droplevel([("a", 10)]), kdf.droplevel([("a", 10)]))
+            self.assert_eq(
+                pdf.droplevel([("level", 1)], axis=1), kdf.droplevel([("level", 1)], axis=1)
+            )
+        else:
+            expected = pdf.copy()
+            expected.index = expected.index.droplevel([("a", 10)])
+
+            self.assert_eq(expected, kdf.droplevel([("a", 10)]))
+
+            expected = pdf.copy()
+            expected.columns = expected.columns.droplevel([("level", 1)])
+
+            self.assert_eq(expected, kdf.droplevel([("level", 1)], axis=1))
+
+        # non-string names
+        pdf = (
+            pd.DataFrame([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]])
+            .set_index([0, 1])
+            .rename_axis([10.0, 20.0])
+        )
+        pdf.columns = pd.MultiIndex.from_tuples([("c", "e"), ("d", "f")], names=[100.0, 200.0])
+        kdf = ps.from_pandas(pdf)
+
+        # droplevel is new in pandas 0.24.0
+        if LooseVersion(pd.__version__) >= LooseVersion("0.24.0"):
+            self.assert_eq(pdf.droplevel(10.0), kdf.droplevel(10.0))
+            self.assert_eq(pdf.droplevel([10.0]), kdf.droplevel([10.0]))
+            self.assert_eq(pdf.droplevel((10.0,)), kdf.droplevel((10.0,)))
+            self.assert_eq(pdf.droplevel(0), kdf.droplevel(0))
+            self.assert_eq(pdf.droplevel(-1), kdf.droplevel(-1))
+            self.assert_eq(pdf.droplevel(100.0, axis=1), kdf.droplevel(100.0, axis=1))
+            self.assert_eq(pdf.droplevel(0, axis=1), kdf.droplevel(0, axis=1))
+        else:
+            expected = pdf.copy()
+            expected.index = expected.index.droplevel(10.0)
+
+            self.assert_eq(expected, kdf.droplevel(10.0))
+            self.assert_eq(expected, kdf.droplevel([10.0]))
+            self.assert_eq(expected, kdf.droplevel((10.0,)))
+            self.assert_eq(expected, kdf.droplevel(0))
+
+            expected = pdf.copy()
+            expected.index = expected.index.droplevel(-1)
+            self.assert_eq(expected, kdf.droplevel(-1))
+
+            expected = pdf.copy()
+            expected.columns = expected.columns.droplevel(100.0)
+
+            self.assert_eq(expected, kdf.droplevel(100.0, axis=1))
+            self.assert_eq(expected, kdf.droplevel(0, axis=1))
+
+    def test_drop(self):
+        pdf = pd.DataFrame({"x": [1, 2], "y": [3, 4], "z": [5, 6]}, index=np.random.rand(2))
+        kdf = ps.from_pandas(pdf)
+
+        # Assert 'labels' or 'columns' parameter is set
+        expected_error_message = "Need to specify at least one of 'labels' or 'columns'"
+        with self.assertRaisesRegex(ValueError, expected_error_message):
+            kdf.drop()
+        # Assert axis cannot be 0
+        with self.assertRaisesRegex(NotImplementedError, "Drop currently only works for axis=1"):
+            kdf.drop("x", axis=0)
+        # Assert using a str for 'labels' works
+        self.assert_eq(kdf.drop("x", axis=1), pdf.drop("x", axis=1))
+        # Assert axis is 1 by default
+        self.assert_eq(kdf.drop("x"), pdf.drop("x", axis=1))
+        # Assert using a list for 'labels' works
+        self.assert_eq(kdf.drop(["y", "z"], axis=1), pdf.drop(["y", "z"], axis=1))
+        # Assert using 'columns' instead of 'labels' produces the same results
+        self.assert_eq(kdf.drop(columns="x"), pdf.drop(columns="x"))
+        self.assert_eq(kdf.drop(columns=["y", "z"]), pdf.drop(columns=["y", "z"]))
+
+        # Assert 'labels' being used when both 'labels' and 'columns' are specified
+        # TODO: should throw an error?
+        expected_output = pd.DataFrame({"y": [3, 4], "z": [5, 6]}, index=kdf.index.to_pandas())
+        self.assert_eq(kdf.drop(labels=["x"], columns=["y"]), expected_output)
+
+        columns = pd.MultiIndex.from_tuples([(1, "x"), (1, "y"), (2, "z")])
+        pdf.columns = columns
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.drop(columns=1), pdf.drop(columns=1))
+        self.assert_eq(kdf.drop(columns=(1, "x")), pdf.drop(columns=(1, "x")))
+        self.assert_eq(kdf.drop(columns=[(1, "x"), 2]), pdf.drop(columns=[(1, "x"), 2]))
+
+        self.assertRaises(KeyError, lambda: kdf.drop(columns=3))
+        self.assertRaises(KeyError, lambda: kdf.drop(columns=(1, "z")))
+
+        # non-string names
+        pdf = pd.DataFrame({10: [1, 2], 20: [3, 4], 30: [5, 6]}, index=np.random.rand(2))
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.drop(10), pdf.drop(10, axis=1))
+        self.assert_eq(kdf.drop([20, 30]), pdf.drop([20, 30], axis=1))
+
+    def _test_dropna(self, pdf, axis):
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.dropna(axis=axis), pdf.dropna(axis=axis))
+        self.assert_eq(kdf.dropna(axis=axis, how="all"), pdf.dropna(axis=axis, how="all"))
+        self.assert_eq(kdf.dropna(axis=axis, subset=["x"]), pdf.dropna(axis=axis, subset=["x"]))
+        self.assert_eq(kdf.dropna(axis=axis, subset="x"), pdf.dropna(axis=axis, subset=["x"]))
+        self.assert_eq(
+            kdf.dropna(axis=axis, subset=["y", "z"]), pdf.dropna(axis=axis, subset=["y", "z"])
+        )
+        self.assert_eq(
+            kdf.dropna(axis=axis, subset=["y", "z"], how="all"),
+            pdf.dropna(axis=axis, subset=["y", "z"], how="all"),
+        )
+
+        self.assert_eq(kdf.dropna(axis=axis, thresh=2), pdf.dropna(axis=axis, thresh=2))
+        self.assert_eq(
+            kdf.dropna(axis=axis, thresh=1, subset=["y", "z"]),
+            pdf.dropna(axis=axis, thresh=1, subset=["y", "z"]),
+        )
+
+        pdf2 = pdf.copy()
+        kdf2 = kdf.copy()
+        pser = pdf2[pdf2.columns[0]]
+        kser = kdf2[kdf2.columns[0]]
+        pdf2.dropna(inplace=True)
+        kdf2.dropna(inplace=True)
+        self.assert_eq(kdf2, pdf2)
+        self.assert_eq(kser, pser)
+
+        # multi-index
+        columns = pd.MultiIndex.from_tuples([("a", "x"), ("a", "y"), ("b", "z")])
+        if axis == 0:
+            pdf.columns = columns
+        else:
+            pdf.index = columns
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.dropna(axis=axis), pdf.dropna(axis=axis))
+        self.assert_eq(kdf.dropna(axis=axis, how="all"), pdf.dropna(axis=axis, how="all"))
+        self.assert_eq(
+            kdf.dropna(axis=axis, subset=[("a", "x")]), pdf.dropna(axis=axis, subset=[("a", "x")])
+        )
+        self.assert_eq(
+            kdf.dropna(axis=axis, subset=("a", "x")), pdf.dropna(axis=axis, subset=[("a", "x")])
+        )
+        self.assert_eq(
+            kdf.dropna(axis=axis, subset=[("a", "y"), ("b", "z")]),
+            pdf.dropna(axis=axis, subset=[("a", "y"), ("b", "z")]),
+        )
+        self.assert_eq(
+            kdf.dropna(axis=axis, subset=[("a", "y"), ("b", "z")], how="all"),
+            pdf.dropna(axis=axis, subset=[("a", "y"), ("b", "z")], how="all"),
+        )
+
+        self.assert_eq(kdf.dropna(axis=axis, thresh=2), pdf.dropna(axis=axis, thresh=2))
+        self.assert_eq(
+            kdf.dropna(axis=axis, thresh=1, subset=[("a", "y"), ("b", "z")]),
+            pdf.dropna(axis=axis, thresh=1, subset=[("a", "y"), ("b", "z")]),
+        )
+
+    def test_dropna_axis_index(self):
+        pdf = pd.DataFrame(
+            {
+                "x": [np.nan, 2, 3, 4, np.nan, 6],
+                "y": [1, 2, np.nan, 4, np.nan, np.nan],
+                "z": [1, 2, 3, 4, np.nan, np.nan],
+            },
+            index=np.random.rand(6),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self._test_dropna(pdf, axis=0)
+
+        # empty
+        pdf = pd.DataFrame(index=np.random.rand(6))
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.dropna(), pdf.dropna())
+        self.assert_eq(kdf.dropna(how="all"), pdf.dropna(how="all"))
+        self.assert_eq(kdf.dropna(thresh=0), pdf.dropna(thresh=0))
+        self.assert_eq(kdf.dropna(thresh=1), pdf.dropna(thresh=1))
+
+        with self.assertRaisesRegex(ValueError, "No axis named foo"):
+            kdf.dropna(axis="foo")
+
+        self.assertRaises(KeyError, lambda: kdf.dropna(subset="1"))
+        with self.assertRaisesRegex(ValueError, "invalid how option: 1"):
+            kdf.dropna(how=1)
+        with self.assertRaisesRegex(TypeError, "must specify how or thresh"):
+            kdf.dropna(how=None)
+
+    def test_dropna_axis_column(self):
+        pdf = pd.DataFrame(
+            {
+                "x": [np.nan, 2, 3, 4, np.nan, 6],
+                "y": [1, 2, np.nan, 4, np.nan, np.nan],
+                "z": [1, 2, 3, 4, np.nan, np.nan],
+            },
+            index=[str(r) for r in np.random.rand(6)],
+        ).T
+
+        self._test_dropna(pdf, axis=1)
+
+        # empty
+        pdf = pd.DataFrame({"x": [], "y": [], "z": []})
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.dropna(axis=1), pdf.dropna(axis=1))
+        self.assert_eq(kdf.dropna(axis=1, how="all"), pdf.dropna(axis=1, how="all"))
+        self.assert_eq(kdf.dropna(axis=1, thresh=0), pdf.dropna(axis=1, thresh=0))
+        self.assert_eq(kdf.dropna(axis=1, thresh=1), pdf.dropna(axis=1, thresh=1))
+
+    def test_dtype(self):
+        pdf = pd.DataFrame(
+            {
+                "a": list("abc"),
+                "b": list(range(1, 4)),
+                "c": np.arange(3, 6).astype("i1"),
+                "d": np.arange(4.0, 7.0, dtype="float64"),
+                "e": [True, False, True],
+                "f": pd.date_range("20130101", periods=3),
+            },
+            index=np.random.rand(3),
+        )
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kdf, pdf)
+        self.assertTrue((kdf.dtypes == pdf.dtypes).all())
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples(zip(list("xxxyyz"), list("abcdef")))
+        pdf.columns = columns
+        kdf.columns = columns
+        self.assertTrue((kdf.dtypes == pdf.dtypes).all())
+
+    def test_fillna(self):
+        pdf = pd.DataFrame(
+            {
+                "x": [np.nan, 2, 3, 4, np.nan, 6],
+                "y": [1, 2, np.nan, 4, np.nan, np.nan],
+                "z": [1, 2, 3, 4, np.nan, np.nan],
+            },
+            index=np.random.rand(6),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kdf.fillna(-1), pdf.fillna(-1))
+        self.assert_eq(
+            kdf.fillna({"x": -1, "y": -2, "z": -5}), pdf.fillna({"x": -1, "y": -2, "z": -5})
+        )
+        self.assert_eq(pdf.fillna(method="ffill"), kdf.fillna(method="ffill"))
+        self.assert_eq(pdf.fillna(method="ffill", limit=2), kdf.fillna(method="ffill", limit=2))
+        self.assert_eq(pdf.fillna(method="bfill"), kdf.fillna(method="bfill"))
+        self.assert_eq(pdf.fillna(method="bfill", limit=2), kdf.fillna(method="bfill", limit=2))
+
+        pdf = pdf.set_index(["x", "y"])
+        kdf = ps.from_pandas(pdf)
+        # check multi index
+        self.assert_eq(kdf.fillna(-1), pdf.fillna(-1))
+        self.assert_eq(pdf.fillna(method="bfill"), kdf.fillna(method="bfill"))
+        self.assert_eq(pdf.fillna(method="ffill"), kdf.fillna(method="ffill"))
+
+        pser = pdf.z
+        kser = kdf.z
+        pdf.fillna({"x": -1, "y": -2, "z": -5}, inplace=True)
+        kdf.fillna({"x": -1, "y": -2, "z": -5}, inplace=True)
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kser, pser)
+
+        s_nan = pd.Series([-1, -2, -5], index=["x", "y", "z"], dtype=int)
+        self.assert_eq(kdf.fillna(s_nan), pdf.fillna(s_nan))
+
+        with self.assertRaisesRegex(NotImplementedError, "fillna currently only"):
+            kdf.fillna(-1, axis=1)
+        with self.assertRaisesRegex(NotImplementedError, "fillna currently only"):
+            kdf.fillna(-1, axis="columns")
+        with self.assertRaisesRegex(ValueError, "limit parameter for value is not support now"):
+            kdf.fillna(-1, limit=1)
+        with self.assertRaisesRegex(TypeError, "Unsupported.*DataFrame"):
+            kdf.fillna(pd.DataFrame({"x": [-1], "y": [-1], "z": [-1]}))
+        with self.assertRaisesRegex(TypeError, "Unsupported.*int64"):
+            kdf.fillna({"x": np.int64(-6), "y": np.int64(-4), "z": -5})
+        with self.assertRaisesRegex(ValueError, "Expecting 'pad', 'ffill', 'backfill' or 'bfill'."):
+            kdf.fillna(method="xxx")
+        with self.assertRaisesRegex(
+            ValueError, "Must specify a fillna 'value' or 'method' parameter."
+        ):
+            kdf.fillna()
+
+        # multi-index columns
+        pdf = pd.DataFrame(
+            {
+                ("x", "a"): [np.nan, 2, 3, 4, np.nan, 6],
+                ("x", "b"): [1, 2, np.nan, 4, np.nan, np.nan],
+                ("y", "c"): [1, 2, 3, 4, np.nan, np.nan],
+            },
+            index=np.random.rand(6),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.fillna(-1), pdf.fillna(-1))
+        self.assert_eq(
+            kdf.fillna({("x", "a"): -1, ("x", "b"): -2, ("y", "c"): -5}),
+            pdf.fillna({("x", "a"): -1, ("x", "b"): -2, ("y", "c"): -5}),
+        )
+        self.assert_eq(pdf.fillna(method="ffill"), kdf.fillna(method="ffill"))
+        self.assert_eq(pdf.fillna(method="ffill", limit=2), kdf.fillna(method="ffill", limit=2))
+        self.assert_eq(pdf.fillna(method="bfill"), kdf.fillna(method="bfill"))
+        self.assert_eq(pdf.fillna(method="bfill", limit=2), kdf.fillna(method="bfill", limit=2))
+
+        self.assert_eq(kdf.fillna({"x": -1}), pdf.fillna({"x": -1}))
+
+        if sys.version_info >= (3, 6):
+            # flaky in Python 3.5.
+            self.assert_eq(
+                kdf.fillna({"x": -1, ("x", "b"): -2}), pdf.fillna({"x": -1, ("x", "b"): -2})
+            )
+            self.assert_eq(
+                kdf.fillna({("x", "b"): -2, "x": -1}), pdf.fillna({("x", "b"): -2, "x": -1})
+            )
+
+        # check multi index
+        pdf = pdf.set_index([("x", "a"), ("x", "b")])
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kdf.fillna(-1), pdf.fillna(-1))
+        self.assert_eq(
+            kdf.fillna({("x", "a"): -1, ("x", "b"): -2, ("y", "c"): -5}),
+            pdf.fillna({("x", "a"): -1, ("x", "b"): -2, ("y", "c"): -5}),
+        )
+
+    def test_isnull(self):
+        pdf = pd.DataFrame(
+            {"x": [1, 2, 3, 4, None, 6], "y": list("abdabd")}, index=np.random.rand(6)
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.notnull(), pdf.notnull())
+        self.assert_eq(kdf.isnull(), pdf.isnull())
+
+    def test_to_datetime(self):
+        pdf = pd.DataFrame(
+            {"year": [2015, 2016], "month": [2, 3], "day": [4, 5]}, index=np.random.rand(2)
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(pd.to_datetime(pdf), ps.to_datetime(kdf))
+
+    def test_nunique(self):
+        pdf = pd.DataFrame({"A": [1, 2, 3], "B": [np.nan, 3, np.nan]}, index=np.random.rand(3))
+        kdf = ps.from_pandas(pdf)
+
+        # Assert NaNs are dropped by default
+        self.assert_eq(kdf.nunique(), pdf.nunique())
+
+        # Assert including NaN values
+        self.assert_eq(kdf.nunique(dropna=False), pdf.nunique(dropna=False))
+
+        # Assert approximate counts
+        self.assert_eq(
+            ps.DataFrame({"A": range(100)}).nunique(approx=True), pd.Series([103], index=["A"]),
+        )
+        self.assert_eq(
+            ps.DataFrame({"A": range(100)}).nunique(approx=True, rsd=0.01),
+            pd.Series([100], index=["A"]),
+        )
+
+        # Assert unsupported axis value yet
+        msg = 'axis should be either 0 or "index" currently.'
+        with self.assertRaisesRegex(NotImplementedError, msg):
+            kdf.nunique(axis=1)
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("X", "A"), ("Y", "B")], names=["1", "2"])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(kdf.nunique(), pdf.nunique())
+        self.assert_eq(kdf.nunique(dropna=False), pdf.nunique(dropna=False))
+
+    def test_sort_values(self):
+        pdf = pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5, None, 7], "b": [7, 6, 5, 4, 3, 2, 1]}, index=np.random.rand(7)
+        )
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kdf.sort_values("b"), pdf.sort_values("b"))
+        self.assert_eq(kdf.sort_values(["b", "a"]), pdf.sort_values(["b", "a"]))
+        self.assert_eq(
+            kdf.sort_values(["b", "a"], ascending=[False, True]),
+            pdf.sort_values(["b", "a"], ascending=[False, True]),
+        )
+
+        self.assertRaises(ValueError, lambda: kdf.sort_values(["b", "a"], ascending=[False]))
+
+        self.assert_eq(
+            kdf.sort_values(["b", "a"], na_position="first"),
+            pdf.sort_values(["b", "a"], na_position="first"),
+        )
+
+        self.assertRaises(ValueError, lambda: kdf.sort_values(["b", "a"], na_position="invalid"))
+
+        pserA = pdf.a
+        kserA = kdf.a
+        self.assert_eq(kdf.sort_values("b", inplace=True), pdf.sort_values("b", inplace=True))
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kserA, pserA)
+
+        # multi-index columns
+        pdf = pd.DataFrame(
+            {("X", 10): [1, 2, 3, 4, 5, None, 7], ("X", 20): [7, 6, 5, 4, 3, 2, 1]},
+            index=np.random.rand(7),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.sort_values(("X", 20)), pdf.sort_values(("X", 20)))
+        self.assert_eq(
+            kdf.sort_values([("X", 20), ("X", 10)]), pdf.sort_values([("X", 20), ("X", 10)])
+        )
+
+        self.assertRaisesRegex(
+            ValueError,
+            "For a multi-index, the label must be a tuple with elements",
+            lambda: kdf.sort_values(["X"]),
+        )
+
+        # non-string names
+        pdf = pd.DataFrame(
+            {10: [1, 2, 3, 4, 5, None, 7], 20: [7, 6, 5, 4, 3, 2, 1]}, index=np.random.rand(7)
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.sort_values(20), pdf.sort_values(20))
+        self.assert_eq(kdf.sort_values([20, 10]), pdf.sort_values([20, 10]))
+
+    def test_sort_index(self):
+        pdf = pd.DataFrame(
+            {"A": [2, 1, np.nan], "B": [np.nan, 0, np.nan]}, index=["b", "a", np.nan]
+        )
+        kdf = ps.from_pandas(pdf)
+
+        # Assert invalid parameters
+        self.assertRaises(NotImplementedError, lambda: kdf.sort_index(axis=1))
+        self.assertRaises(NotImplementedError, lambda: kdf.sort_index(kind="mergesort"))
+        self.assertRaises(ValueError, lambda: kdf.sort_index(na_position="invalid"))
+
+        # Assert default behavior without parameters
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+        # Assert sorting descending
+        self.assert_eq(kdf.sort_index(ascending=False), pdf.sort_index(ascending=False))
+        # Assert sorting NA indices first
+        self.assert_eq(kdf.sort_index(na_position="first"), pdf.sort_index(na_position="first"))
+
+        # Assert sorting inplace
+        pserA = pdf.A
+        kserA = kdf.A
+        self.assertEqual(kdf.sort_index(inplace=True), pdf.sort_index(inplace=True))
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kserA, pserA)
+
+        # Assert multi-indices
+        pdf = pd.DataFrame(
+            {"A": range(4), "B": range(4)[::-1]}, index=[["b", "b", "a", "a"], [1, 0, 1, 0]]
+        )
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+        self.assert_eq(kdf.sort_index(level=[1, 0]), pdf.sort_index(level=[1, 0]))
+        self.assert_eq(kdf.reset_index().sort_index(), pdf.reset_index().sort_index())
+
+        # Assert with multi-index columns
+        columns = pd.MultiIndex.from_tuples([("X", "A"), ("X", "B")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+
+    def test_swaplevel(self):
+        # MultiIndex with two levels
+        arrays = [[1, 1, 2, 2], ["red", "blue", "red", "blue"]]
+        pidx = pd.MultiIndex.from_arrays(arrays, names=("number", "color"))
+        pdf = pd.DataFrame({"x1": ["a", "b", "c", "d"], "x2": ["a", "b", "c", "d"]}, index=pidx)
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.swaplevel(), kdf.swaplevel())
+        self.assert_eq(pdf.swaplevel(0, 1), kdf.swaplevel(0, 1))
+        self.assert_eq(pdf.swaplevel(1, 1), kdf.swaplevel(1, 1))
+        self.assert_eq(pdf.swaplevel("number", "color"), kdf.swaplevel("number", "color"))
+
+        # MultiIndex with more than two levels
+        arrays = [[1, 1, 2, 2], ["red", "blue", "red", "blue"], ["l", "m", "s", "xs"]]
+        pidx = pd.MultiIndex.from_arrays(arrays, names=("number", "color", "size"))
+        pdf = pd.DataFrame({"x1": ["a", "b", "c", "d"], "x2": ["a", "b", "c", "d"]}, index=pidx)
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.swaplevel(), kdf.swaplevel())
+        self.assert_eq(pdf.swaplevel(0, 1), kdf.swaplevel(0, 1))
+        self.assert_eq(pdf.swaplevel(0, 2), kdf.swaplevel(0, 2))
+        self.assert_eq(pdf.swaplevel(1, 2), kdf.swaplevel(1, 2))
+        self.assert_eq(pdf.swaplevel(1, 1), kdf.swaplevel(1, 1))
+        self.assert_eq(pdf.swaplevel(-1, -2), kdf.swaplevel(-1, -2))
+        self.assert_eq(pdf.swaplevel("number", "color"), kdf.swaplevel("number", "color"))
+        self.assert_eq(pdf.swaplevel("number", "size"), kdf.swaplevel("number", "size"))
+        self.assert_eq(pdf.swaplevel("color", "size"), kdf.swaplevel("color", "size"))
+        self.assert_eq(
+            pdf.swaplevel("color", "size", axis="index"),
+            kdf.swaplevel("color", "size", axis="index"),
+        )
+        self.assert_eq(
+            pdf.swaplevel("color", "size", axis=0), kdf.swaplevel("color", "size", axis=0)
+        )
+
+        pdf = pd.DataFrame(
+            {
+                "x1": ["a", "b", "c", "d"],
+                "x2": ["a", "b", "c", "d"],
+                "x3": ["a", "b", "c", "d"],
+                "x4": ["a", "b", "c", "d"],
+            }
+        )
+        pidx = pd.MultiIndex.from_arrays(arrays, names=("number", "color", "size"))
+        pdf.columns = pidx
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.swaplevel(axis=1), kdf.swaplevel(axis=1))
+        self.assert_eq(pdf.swaplevel(0, 1, axis=1), kdf.swaplevel(0, 1, axis=1))
+        self.assert_eq(pdf.swaplevel(0, 2, axis=1), kdf.swaplevel(0, 2, axis=1))
+        self.assert_eq(pdf.swaplevel(1, 2, axis=1), kdf.swaplevel(1, 2, axis=1))
+        self.assert_eq(pdf.swaplevel(1, 1, axis=1), kdf.swaplevel(1, 1, axis=1))
+        self.assert_eq(pdf.swaplevel(-1, -2, axis=1), kdf.swaplevel(-1, -2, axis=1))
+        self.assert_eq(
+            pdf.swaplevel("number", "color", axis=1), kdf.swaplevel("number", "color", axis=1)
+        )
+        self.assert_eq(
+            pdf.swaplevel("number", "size", axis=1), kdf.swaplevel("number", "size", axis=1)
+        )
+        self.assert_eq(
+            pdf.swaplevel("color", "size", axis=1), kdf.swaplevel("color", "size", axis=1)
+        )
+        self.assert_eq(
+            pdf.swaplevel("color", "size", axis="columns"),
+            kdf.swaplevel("color", "size", axis="columns"),
+        )
+
+        # Error conditions
+        self.assertRaises(AssertionError, lambda: ps.DataFrame([1, 2]).swaplevel())
+        self.assertRaises(IndexError, lambda: kdf.swaplevel(0, 9, axis=1))
+        self.assertRaises(KeyError, lambda: kdf.swaplevel("not_number", "color", axis=1))
+        self.assertRaises(ValueError, lambda: kdf.swaplevel(axis=2))
+
+    def test_swapaxes(self):
+        pdf = pd.DataFrame(
+            [[1, 2, 3], [4, 5, 6], [7, 8, 9]], index=["x", "y", "z"], columns=["a", "b", "c"]
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.swapaxes(0, 1), pdf.swapaxes(0, 1))
+        self.assert_eq(kdf.swapaxes(1, 0), pdf.swapaxes(1, 0))
+        self.assert_eq(kdf.swapaxes("index", "columns"), pdf.swapaxes("index", "columns"))
+        self.assert_eq(kdf.swapaxes("columns", "index"), pdf.swapaxes("columns", "index"))
+        self.assert_eq((kdf + 1).swapaxes(0, 1), (pdf + 1).swapaxes(0, 1))
+
+        self.assertRaises(AssertionError, lambda: kdf.swapaxes(0, 1, copy=False))
+        self.assertRaises(ValueError, lambda: kdf.swapaxes(0, -1))
+
+    def test_nlargest(self):
+        pdf = pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5, None, 7], "b": [7, 6, 5, 4, 3, 2, 1]}, index=np.random.rand(7)
+        )
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kdf.nlargest(n=5, columns="a"), pdf.nlargest(5, columns="a"))
+        self.assert_eq(kdf.nlargest(n=5, columns=["a", "b"]), pdf.nlargest(5, columns=["a", "b"]))
+
+    def test_nsmallest(self):
+        pdf = pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5, None, 7], "b": [7, 6, 5, 4, 3, 2, 1]}, index=np.random.rand(7)
+        )
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kdf.nsmallest(n=5, columns="a"), pdf.nsmallest(5, columns="a"))
+        self.assert_eq(kdf.nsmallest(n=5, columns=["a", "b"]), pdf.nsmallest(5, columns=["a", "b"]))
+
+    def test_xs(self):
+        d = {
+            "num_legs": [4, 4, 2, 2],
+            "num_wings": [0, 0, 2, 2],
+            "class": ["mammal", "mammal", "mammal", "bird"],
+            "animal": ["cat", "dog", "bat", "penguin"],
+            "locomotion": ["walks", "walks", "flies", "walks"],
+        }
+        pdf = pd.DataFrame(data=d)
+        pdf = pdf.set_index(["class", "animal", "locomotion"])
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.xs("mammal"), pdf.xs("mammal"))
+        self.assert_eq(kdf.xs(("mammal",)), pdf.xs(("mammal",)))
+        self.assert_eq(kdf.xs(("mammal", "dog", "walks")), pdf.xs(("mammal", "dog", "walks")))
+        self.assert_eq(
+            ps.concat([kdf, kdf]).xs(("mammal", "dog", "walks")),
+            pd.concat([pdf, pdf]).xs(("mammal", "dog", "walks")),
+        )
+        self.assert_eq(kdf.xs("cat", level=1), pdf.xs("cat", level=1))
+        self.assert_eq(kdf.xs("flies", level=2), pdf.xs("flies", level=2))
+        self.assert_eq(kdf.xs("mammal", level=-3), pdf.xs("mammal", level=-3))
+
+        msg = 'axis should be either 0 or "index" currently.'
+        with self.assertRaisesRegex(NotImplementedError, msg):
+            kdf.xs("num_wings", axis=1)
+        with self.assertRaises(KeyError):
+            kdf.xs(("mammal", "dog", "walk"))
+        msg = r"'Key length \(4\) exceeds index depth \(3\)'"
+        with self.assertRaisesRegex(KeyError, msg):
+            kdf.xs(("mammal", "dog", "walks", "foo"))
+
+        self.assertRaises(IndexError, lambda: kdf.xs("foo", level=-4))
+        self.assertRaises(IndexError, lambda: kdf.xs("foo", level=3))
+
+        self.assertRaises(KeyError, lambda: kdf.xs(("dog", "walks"), level=1))
+
+        # non-string names
+        pdf = pd.DataFrame(data=d)
+        pdf = pdf.set_index(["class", "animal", "num_legs", "num_wings"])
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.xs(("mammal", "dog", 4)), pdf.xs(("mammal", "dog", 4)))
+        self.assert_eq(kdf.xs(2, level=2), pdf.xs(2, level=2))
+
+        self.assert_eq((kdf + "a").xs(("mammal", "dog", 4)), (pdf + "a").xs(("mammal", "dog", 4)))
+        self.assert_eq((kdf + "a").xs(2, level=2), (pdf + "a").xs(2, level=2))
+
+    def test_missing(self):
+        kdf = self.kdf
+
+        missing_functions = inspect.getmembers(_MissingPandasLikeDataFrame, inspect.isfunction)
+        unsupported_functions = [
+            name for (name, type_) in missing_functions if type_.__name__ == "unsupported_function"
+        ]
+        for name in unsupported_functions:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "method.*DataFrame.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf, name)()
+
+        deprecated_functions = [
+            name for (name, type_) in missing_functions if type_.__name__ == "deprecated_function"
+        ]
+        for name in deprecated_functions:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "method.*DataFrame.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf, name)()
+
+        missing_properties = inspect.getmembers(
+            _MissingPandasLikeDataFrame, lambda o: isinstance(o, property)
+        )
+        unsupported_properties = [
+            name
+            for (name, type_) in missing_properties
+            if type_.fget.__name__ == "unsupported_property"
+        ]
+        for name in unsupported_properties:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "property.*DataFrame.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf, name)
+        deprecated_properties = [
+            name
+            for (name, type_) in missing_properties
+            if type_.fget.__name__ == "deprecated_property"
+        ]
+        for name in deprecated_properties:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "property.*DataFrame.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf, name)
+
+    def test_to_numpy(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [4, 2, 3, 4, 8, 6],
+                "b": [1, 2, 9, 4, 2, 4],
+                "c": ["one", "three", "six", "seven", "one", "5"],
+            },
+            index=np.random.rand(6),
+        )
+
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.to_numpy(), pdf.values)
+
+    def test_to_pandas(self):
+        pdf, kdf = self.df_pair
+        self.assert_eq(kdf.toPandas(), pdf)
+        self.assert_eq(kdf.to_pandas(), pdf)
+
+    def test_isin(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [4, 2, 3, 4, 8, 6],
+                "b": [1, 2, 9, 4, 2, 4],
+                "c": ["one", "three", "six", "seven", "one", "5"],
+            },
+            index=np.random.rand(6),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.isin([4, "six"]), pdf.isin([4, "six"]))
+        # Seems like pandas has a bug when passing `np.array` as parameter
+        self.assert_eq(kdf.isin(np.array([4, "six"])), pdf.isin([4, "six"]))
+        self.assert_eq(
+            kdf.isin({"a": [2, 8], "c": ["three", "one"]}),
+            pdf.isin({"a": [2, 8], "c": ["three", "one"]}),
+        )
+        self.assert_eq(
+            kdf.isin({"a": np.array([2, 8]), "c": ["three", "one"]}),
+            pdf.isin({"a": np.array([2, 8]), "c": ["three", "one"]}),
+        )
+
+        msg = "'DataFrame' object has no attribute {'e'}"
+        with self.assertRaisesRegex(AttributeError, msg):
+            kdf.isin({"e": [5, 7], "a": [1, 6]})
+
+        msg = "DataFrame and Series are not supported"
+        with self.assertRaisesRegex(NotImplementedError, msg):
+            kdf.isin(pdf)
+
+        msg = "Values should be iterable, Series, DataFrame or dict."
+        with self.assertRaisesRegex(TypeError, msg):
+            kdf.isin(1)
+
+    def test_merge(self):
+        left_pdf = pd.DataFrame(
+            {
+                "lkey": ["foo", "bar", "baz", "foo", "bar", "l"],
+                "value": [1, 2, 3, 5, 6, 7],
+                "x": list("abcdef"),
+            },
+            columns=["lkey", "value", "x"],
+        )
+        right_pdf = pd.DataFrame(
+            {
+                "rkey": ["baz", "foo", "bar", "baz", "foo", "r"],
+                "value": [4, 5, 6, 7, 8, 9],
+                "y": list("efghij"),
+            },
+            columns=["rkey", "value", "y"],
+        )
+        right_ps = pd.Series(list("defghi"), name="x", index=[5, 6, 7, 8, 9, 10])
+
+        left_kdf = ps.from_pandas(left_pdf)
+        right_kdf = ps.from_pandas(right_pdf)
+        right_kser = ps.from_pandas(right_ps)
+
+        def check(op, right_kdf=right_kdf, right_pdf=right_pdf):
+            k_res = op(left_kdf, right_kdf)
+            k_res = k_res.to_pandas()
+            k_res = k_res.sort_values(by=list(k_res.columns))
+            k_res = k_res.reset_index(drop=True)
+            p_res = op(left_pdf, right_pdf)
+            p_res = p_res.sort_values(by=list(p_res.columns))
+            p_res = p_res.reset_index(drop=True)
+            self.assert_eq(k_res, p_res)
+
+        check(lambda left, right: left.merge(right))
+        check(lambda left, right: left.merge(right, on="value"))
+        check(lambda left, right: left.merge(right, left_on="lkey", right_on="rkey"))
+        check(lambda left, right: left.set_index("lkey").merge(right.set_index("rkey")))
+        check(
+            lambda left, right: left.set_index("lkey").merge(
+                right, left_index=True, right_on="rkey"
+            )
+        )
+        check(
+            lambda left, right: left.merge(
+                right.set_index("rkey"), left_on="lkey", right_index=True
+            )
+        )
+        check(
+            lambda left, right: left.set_index("lkey").merge(
+                right.set_index("rkey"), left_index=True, right_index=True
+            )
+        )
+
+        # MultiIndex
+        check(
+            lambda left, right: left.merge(
+                right, left_on=["lkey", "value"], right_on=["rkey", "value"]
+            )
+        )
+        check(
+            lambda left, right: left.set_index(["lkey", "value"]).merge(
+                right, left_index=True, right_on=["rkey", "value"]
+            )
+        )
+        check(
+            lambda left, right: left.merge(
+                right.set_index(["rkey", "value"]), left_on=["lkey", "value"], right_index=True
+            )
+        )
+        # TODO: when both left_index=True and right_index=True with multi-index
+        # check(lambda left, right: left.set_index(['lkey', 'value']).merge(
+        #     right.set_index(['rkey', 'value']), left_index=True, right_index=True))
+
+        # join types
+        for how in ["inner", "left", "right", "outer"]:
+            check(lambda left, right: left.merge(right, on="value", how=how))
+            check(lambda left, right: left.merge(right, left_on="lkey", right_on="rkey", how=how))
+
+        # suffix
+        check(
+            lambda left, right: left.merge(
+                right, left_on="lkey", right_on="rkey", suffixes=["_left", "_right"]
+            )
+        )
+
+        # Test Series on the right
+        # pd.DataFrame.merge with Series is implemented since version 0.24.0
+        if LooseVersion(pd.__version__) >= LooseVersion("0.24.0"):
+            check(lambda left, right: left.merge(right), right_kser, right_ps)
+            check(
+                lambda left, right: left.merge(right, left_on="x", right_on="x"),
+                right_kser,
+                right_ps,
+            )
+            check(
+                lambda left, right: left.set_index("x").merge(right, left_index=True, right_on="x"),
+                right_kser,
+                right_ps,
+            )
+
+            # Test join types with Series
+            for how in ["inner", "left", "right", "outer"]:
+                check(lambda left, right: left.merge(right, how=how), right_kser, right_ps)
+                check(
+                    lambda left, right: left.merge(right, left_on="x", right_on="x", how=how),
+                    right_kser,
+                    right_ps,
+                )
+
+            # suffix with Series
+            check(
+                lambda left, right: left.merge(
+                    right,
+                    suffixes=["_left", "_right"],
+                    how="outer",
+                    left_index=True,
+                    right_index=True,
+                ),
+                right_kser,
+                right_ps,
+            )
+
+        # multi-index columns
+        left_columns = pd.MultiIndex.from_tuples([(10, "lkey"), (10, "value"), (20, "x")])
+        left_pdf.columns = left_columns
+        left_kdf.columns = left_columns
+
+        right_columns = pd.MultiIndex.from_tuples([(10, "rkey"), (10, "value"), (30, "y")])
+        right_pdf.columns = right_columns
+        right_kdf.columns = right_columns
+
+        check(lambda left, right: left.merge(right))
+        check(lambda left, right: left.merge(right, on=[(10, "value")]))
+        check(
+            lambda left, right: (left.set_index((10, "lkey")).merge(right.set_index((10, "rkey"))))
+        )
+        check(
+            lambda left, right: (
+                left.set_index((10, "lkey")).merge(
+                    right.set_index((10, "rkey")), left_index=True, right_index=True
+                )
+            )
+        )
+        # TODO: when both left_index=True and right_index=True with multi-index columns
+        # check(lambda left, right: left.merge(right,
+        #                                      left_on=[('a', 'lkey')], right_on=[('a', 'rkey')]))
+        # check(lambda left, right: (left.set_index(('a', 'lkey'))
+        #                            .merge(right, left_index=True, right_on=[('a', 'rkey')])))
+
+        # non-string names
+        left_pdf.columns = [10, 100, 1000]
+        left_kdf.columns = [10, 100, 1000]
+
+        right_pdf.columns = [20, 100, 2000]
+        right_kdf.columns = [20, 100, 2000]
+
+        check(lambda left, right: left.merge(right))
+        check(lambda left, right: left.merge(right, on=[100]))
+        check(lambda left, right: (left.set_index(10).merge(right.set_index(20))))
+        check(
+            lambda left, right: (
+                left.set_index(10).merge(right.set_index(20), left_index=True, right_index=True)
+            )
+        )
+
+    def test_merge_same_anchor(self):
+        pdf = pd.DataFrame(
+            {
+                "lkey": ["foo", "bar", "baz", "foo", "bar", "l"],
+                "rkey": ["baz", "foo", "bar", "baz", "foo", "r"],
+                "value": [1, 1, 3, 5, 6, 7],
+                "x": list("abcdef"),
+                "y": list("efghij"),
+            },
+            columns=["lkey", "rkey", "value", "x", "y"],
+        )
+        kdf = ps.from_pandas(pdf)
+
+        left_pdf = pdf[["lkey", "value", "x"]]
+        right_pdf = pdf[["rkey", "value", "y"]]
+        left_kdf = kdf[["lkey", "value", "x"]]
+        right_kdf = kdf[["rkey", "value", "y"]]
+
+        def check(op, right_kdf=right_kdf, right_pdf=right_pdf):
+            k_res = op(left_kdf, right_kdf)
+            k_res = k_res.to_pandas()
+            k_res = k_res.sort_values(by=list(k_res.columns))
+            k_res = k_res.reset_index(drop=True)
+            p_res = op(left_pdf, right_pdf)
+            p_res = p_res.sort_values(by=list(p_res.columns))
+            p_res = p_res.reset_index(drop=True)
+            self.assert_eq(k_res, p_res)
+
+        check(lambda left, right: left.merge(right))
+        check(lambda left, right: left.merge(right, on="value"))
+        check(lambda left, right: left.merge(right, left_on="lkey", right_on="rkey"))
+        check(lambda left, right: left.set_index("lkey").merge(right.set_index("rkey")))
+        check(
+            lambda left, right: left.set_index("lkey").merge(
+                right, left_index=True, right_on="rkey"
+            )
+        )
+        check(
+            lambda left, right: left.merge(
+                right.set_index("rkey"), left_on="lkey", right_index=True
+            )
+        )
+        check(
+            lambda left, right: left.set_index("lkey").merge(
+                right.set_index("rkey"), left_index=True, right_index=True
+            )
+        )
+
+    def test_merge_retains_indices(self):
+        left_pdf = pd.DataFrame({"A": [0, 1]})
+        right_pdf = pd.DataFrame({"B": [1, 2]}, index=[1, 2])
+        left_kdf = ps.from_pandas(left_pdf)
+        right_kdf = ps.from_pandas(right_pdf)
+
+        self.assert_eq(
+            left_kdf.merge(right_kdf, left_index=True, right_index=True),
+            left_pdf.merge(right_pdf, left_index=True, right_index=True),
+        )
+        self.assert_eq(
+            left_kdf.merge(right_kdf, left_on="A", right_index=True),
+            left_pdf.merge(right_pdf, left_on="A", right_index=True),
+        )
+        self.assert_eq(
+            left_kdf.merge(right_kdf, left_index=True, right_on="B"),
+            left_pdf.merge(right_pdf, left_index=True, right_on="B"),
+        )
+        self.assert_eq(
+            left_kdf.merge(right_kdf, left_on="A", right_on="B"),
+            left_pdf.merge(right_pdf, left_on="A", right_on="B"),
+        )
+
+    def test_merge_how_parameter(self):
+        left_pdf = pd.DataFrame({"A": [1, 2]})
+        right_pdf = pd.DataFrame({"B": ["x", "y"]}, index=[1, 2])
+        left_kdf = ps.from_pandas(left_pdf)
+        right_kdf = ps.from_pandas(right_pdf)
+
+        kdf = left_kdf.merge(right_kdf, left_index=True, right_index=True)
+        pdf = left_pdf.merge(right_pdf, left_index=True, right_index=True)
+        self.assert_eq(
+            kdf.sort_values(by=list(kdf.columns)).reset_index(drop=True),
+            pdf.sort_values(by=list(pdf.columns)).reset_index(drop=True),
+        )
+
+        kdf = left_kdf.merge(right_kdf, left_index=True, right_index=True, how="left")
+        pdf = left_pdf.merge(right_pdf, left_index=True, right_index=True, how="left")
+        self.assert_eq(
+            kdf.sort_values(by=list(kdf.columns)).reset_index(drop=True),
+            pdf.sort_values(by=list(pdf.columns)).reset_index(drop=True),
+        )
+
+        kdf = left_kdf.merge(right_kdf, left_index=True, right_index=True, how="right")
+        pdf = left_pdf.merge(right_pdf, left_index=True, right_index=True, how="right")
+        self.assert_eq(
+            kdf.sort_values(by=list(kdf.columns)).reset_index(drop=True),
+            pdf.sort_values(by=list(pdf.columns)).reset_index(drop=True),
+        )
+
+        kdf = left_kdf.merge(right_kdf, left_index=True, right_index=True, how="outer")
+        pdf = left_pdf.merge(right_pdf, left_index=True, right_index=True, how="outer")
+        self.assert_eq(
+            kdf.sort_values(by=list(kdf.columns)).reset_index(drop=True),
+            pdf.sort_values(by=list(pdf.columns)).reset_index(drop=True),
+        )
+
+    def test_merge_raises(self):
+        left = ps.DataFrame(
+            {"value": [1, 2, 3, 5, 6], "x": list("abcde")},
+            columns=["value", "x"],
+            index=["foo", "bar", "baz", "foo", "bar"],
+        )
+        right = ps.DataFrame(
+            {"value": [4, 5, 6, 7, 8], "y": list("fghij")},
+            columns=["value", "y"],
+            index=["baz", "foo", "bar", "baz", "foo"],
+        )
+
+        with self.assertRaisesRegex(ValueError, "No common columns to perform merge on"):
+            left[["x"]].merge(right[["y"]])
+
+        with self.assertRaisesRegex(ValueError, "not a combination of both"):
+            left.merge(right, on="value", left_on="x")
+
+        with self.assertRaisesRegex(ValueError, "Must pass right_on or right_index=True"):
+            left.merge(right, left_on="x")
+
+        with self.assertRaisesRegex(ValueError, "Must pass right_on or right_index=True"):
+            left.merge(right, left_index=True)
+
+        with self.assertRaisesRegex(ValueError, "Must pass left_on or left_index=True"):
+            left.merge(right, right_on="y")
+
+        with self.assertRaisesRegex(ValueError, "Must pass left_on or left_index=True"):
+            left.merge(right, right_index=True)
+
+        with self.assertRaisesRegex(
+            ValueError, "len\\(left_keys\\) must equal len\\(right_keys\\)"
+        ):
+            left.merge(right, left_on="value", right_on=["value", "y"])
+
+        with self.assertRaisesRegex(
+            ValueError, "len\\(left_keys\\) must equal len\\(right_keys\\)"
+        ):
+            left.merge(right, left_on=["value", "x"], right_on="value")
+
+        with self.assertRaisesRegex(ValueError, "['inner', 'left', 'right', 'full', 'outer']"):
+            left.merge(right, left_index=True, right_index=True, how="foo")
+
+        with self.assertRaisesRegex(KeyError, "id"):
+            left.merge(right, on="id")
+
+    def test_append(self):
+        pdf = pd.DataFrame([[1, 2], [3, 4]], columns=list("AB"))
+        kdf = ps.from_pandas(pdf)
+        other_pdf = pd.DataFrame([[3, 4], [5, 6]], columns=list("BC"), index=[2, 3])
+        other_kdf = ps.from_pandas(other_pdf)
+
+        self.assert_eq(kdf.append(kdf), pdf.append(pdf))
+        self.assert_eq(kdf.append(kdf, ignore_index=True), pdf.append(pdf, ignore_index=True))
+
+        # Assert DataFrames with non-matching columns
+        self.assert_eq(kdf.append(other_kdf), pdf.append(other_pdf))
+
+        # Assert appending a Series fails
+        msg = "DataFrames.append() does not support appending Series to DataFrames"
+        with self.assertRaises(TypeError, msg=msg):
+            kdf.append(kdf["A"])
+
+        # Assert using the sort parameter raises an exception
+        msg = "The 'sort' parameter is currently not supported"
+        with self.assertRaises(NotImplementedError, msg=msg):
+            kdf.append(kdf, sort=True)
+
+        # Assert using 'verify_integrity' only raises an exception for overlapping indices
+        self.assert_eq(
+            kdf.append(other_kdf, verify_integrity=True),
+            pdf.append(other_pdf, verify_integrity=True),
+        )
+        msg = "Indices have overlapping values"
+        with self.assertRaises(ValueError, msg=msg):
+            kdf.append(kdf, verify_integrity=True)
+
+        # Skip integrity verification when ignore_index=True
+        self.assert_eq(
+            kdf.append(kdf, ignore_index=True, verify_integrity=True),
+            pdf.append(pdf, ignore_index=True, verify_integrity=True),
+        )
+
+        # Assert appending multi-index DataFrames
+        multi_index_pdf = pd.DataFrame([[1, 2], [3, 4]], columns=list("AB"), index=[[2, 3], [4, 5]])
+        multi_index_kdf = ps.from_pandas(multi_index_pdf)
+        other_multi_index_pdf = pd.DataFrame(
+            [[5, 6], [7, 8]], columns=list("AB"), index=[[2, 3], [6, 7]]
+        )
+        other_multi_index_kdf = ps.from_pandas(other_multi_index_pdf)
+
+        self.assert_eq(
+            multi_index_kdf.append(multi_index_kdf), multi_index_pdf.append(multi_index_pdf)
+        )
+
+        # Assert DataFrames with non-matching columns
+        self.assert_eq(
+            multi_index_kdf.append(other_multi_index_kdf),
+            multi_index_pdf.append(other_multi_index_pdf),
+        )
+
+        # Assert using 'verify_integrity' only raises an exception for overlapping indices
+        self.assert_eq(
+            multi_index_kdf.append(other_multi_index_kdf, verify_integrity=True),
+            multi_index_pdf.append(other_multi_index_pdf, verify_integrity=True),
+        )
+        with self.assertRaises(ValueError, msg=msg):
+            multi_index_kdf.append(multi_index_kdf, verify_integrity=True)
+
+        # Skip integrity verification when ignore_index=True
+        self.assert_eq(
+            multi_index_kdf.append(multi_index_kdf, ignore_index=True, verify_integrity=True),
+            multi_index_pdf.append(multi_index_pdf, ignore_index=True, verify_integrity=True),
+        )
+
+        # Assert trying to append DataFrames with different index levels
+        msg = "Both DataFrames have to have the same number of index levels"
+        with self.assertRaises(ValueError, msg=msg):
+            kdf.append(multi_index_kdf)
+
+        # Skip index level check when ignore_index=True
+        self.assert_eq(
+            kdf.append(multi_index_kdf, ignore_index=True),
+            pdf.append(multi_index_pdf, ignore_index=True),
+        )
+
+        columns = pd.MultiIndex.from_tuples([("A", "X"), ("A", "Y")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(kdf.append(kdf), pdf.append(pdf))
+
+    def test_clip(self):
+        pdf = pd.DataFrame(
+            {"A": [0, 2, 4], "B": [4, 2, 0], "X": [-1, 10, 0]}, index=np.random.rand(3)
+        )
+        kdf = ps.from_pandas(pdf)
+
+        # Assert list-like values are not accepted for 'lower' and 'upper'
+        msg = "List-like value are not supported for 'lower' and 'upper' at the moment"
+        with self.assertRaises(TypeError, msg=msg):
+            kdf.clip(lower=[1])
+        with self.assertRaises(TypeError, msg=msg):
+            kdf.clip(upper=[1])
+
+        # Assert no lower or upper
+        self.assert_eq(kdf.clip(), pdf.clip())
+        # Assert lower only
+        self.assert_eq(kdf.clip(1), pdf.clip(1))
+        # Assert upper only
+        self.assert_eq(kdf.clip(upper=3), pdf.clip(upper=3))
+        # Assert lower and upper
+        self.assert_eq(kdf.clip(1, 3), pdf.clip(1, 3))
+
+        pdf["clip"] = pdf.A.clip(lower=1, upper=3)
+        kdf["clip"] = kdf.A.clip(lower=1, upper=3)
+        self.assert_eq(kdf, pdf)
+
+        # Assert behavior on string values
+        str_kdf = ps.DataFrame({"A": ["a", "b", "c"]}, index=np.random.rand(3))
+        self.assert_eq(str_kdf.clip(1, 3), str_kdf)
+
+    def test_binary_operators(self):
+        pdf = pd.DataFrame(
+            {"A": [0, 2, 4], "B": [4, 2, 0], "X": [-1, 10, 0]}, index=np.random.rand(3)
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf + kdf.copy(), pdf + pdf.copy())
+
+        self.assertRaisesRegex(
+            ValueError,
+            "it comes from a different dataframe",
+            lambda: ps.range(10).add(ps.range(10)),
+        )
+
+        self.assertRaisesRegex(
+            TypeError,
+            "add with a sequence is currently not supported",
+            lambda: ps.range(10).add(ps.range(10).id),
+        )
+
+    def test_binary_operator_add(self):
+        # Positive
+        pdf = pd.DataFrame({"a": ["x"], "b": ["y"], "c": [1], "d": [2]})
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf["a"] + kdf["b"], pdf["a"] + pdf["b"])
+        self.assert_eq(kdf["c"] + kdf["d"], pdf["c"] + pdf["d"])
+
+        # Negative
+        ks_err_msg = "string addition can only be applied to string series or literals"
+
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: kdf["a"] + kdf["c"])
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: kdf["c"] + kdf["a"])
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: kdf["c"] + "literal")
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: "literal" + kdf["c"])
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: 1 + kdf["a"])
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: kdf["a"] + 1)
+
+    def test_binary_operator_sub(self):
+        # Positive
+        pdf = pd.DataFrame({"a": [2], "b": [1]})
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf["a"] - kdf["b"], pdf["a"] - pdf["b"])
+
+        # Negative
+        kdf = ps.DataFrame({"a": ["x"], "b": [1]})
+        ks_err_msg = "substraction can not be applied to string series or literals"
+
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: kdf["a"] - kdf["b"])
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: kdf["b"] - kdf["a"])
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: kdf["b"] - "literal")
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: "literal" - kdf["b"])
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: 1 - kdf["a"])
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: kdf["a"] - 1)
+
+        kdf = ps.DataFrame({"a": ["x"], "b": ["y"]})
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: kdf["a"] - kdf["b"])
+
+    def test_binary_operator_truediv(self):
+        # Positive
+        pdf = pd.DataFrame({"a": [3], "b": [2]})
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf["a"] / kdf["b"], pdf["a"] / pdf["b"])
+
+        # Negative
+        kdf = ps.DataFrame({"a": ["x"], "b": [1]})
+        ks_err_msg = "division can not be applied on string series or literals"
+
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: kdf["a"] / kdf["b"])
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: kdf["b"] / kdf["a"])
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: kdf["b"] / "literal")
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: "literal" / kdf["b"])
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: 1 / kdf["a"])
+
+    def test_binary_operator_floordiv(self):
+        kdf = ps.DataFrame({"a": ["x"], "b": [1]})
+        ks_err_msg = "division can not be applied on string series or literals"
+
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: kdf["a"] // kdf["b"])
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: kdf["b"] // kdf["a"])
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: kdf["b"] // "literal")
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: "literal" // kdf["b"])
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: 1 // kdf["a"])
+
+    def test_binary_operator_mod(self):
+        # Positive
+        pdf = pd.DataFrame({"a": [3], "b": [2]})
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf["a"] % kdf["b"], pdf["a"] % pdf["b"])
+
+        # Negative
+        kdf = ps.DataFrame({"a": ["x"], "b": [1]})
+        ks_err_msg = "modulo can not be applied on string series or literals"
+
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: kdf["a"] % kdf["b"])
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: kdf["b"] % kdf["a"])
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: kdf["b"] % "literal")
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: 1 % kdf["a"])
+
+    def test_binary_operator_multiply(self):
+        # Positive
+        pdf = pd.DataFrame({"a": ["x", "y"], "b": [1, 2], "c": [3, 4]})
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf["b"] * kdf["c"], pdf["b"] * pdf["c"])
+        self.assert_eq(kdf["c"] * kdf["b"], pdf["c"] * pdf["b"])
+        self.assert_eq(kdf["a"] * kdf["b"], pdf["a"] * pdf["b"])
+        self.assert_eq(kdf["b"] * kdf["a"], pdf["b"] * pdf["a"])
+        self.assert_eq(kdf["a"] * 2, pdf["a"] * 2)
+        self.assert_eq(kdf["b"] * 2, pdf["b"] * 2)
+        self.assert_eq(2 * kdf["a"], 2 * pdf["a"])
+        self.assert_eq(2 * kdf["b"], 2 * pdf["b"])
+
+        # Negative
+        kdf = ps.DataFrame({"a": ["x"], "b": [2]})
+        ks_err_msg = "multiplication can not be applied to a string literal"
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: kdf["b"] * "literal")
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: "literal" * kdf["b"])
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: kdf["a"] * "literal")
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: "literal" * kdf["a"])
+
+        ks_err_msg = "a string series can only be multiplied to an int series or literal"
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: kdf["a"] * kdf["a"])
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: kdf["a"] * 0.1)
+        self.assertRaisesRegex(TypeError, ks_err_msg, lambda: 0.1 * kdf["a"])
+
+    def test_sample(self):
+        pdf = pd.DataFrame({"A": [0, 2, 4]})
+        kdf = ps.from_pandas(pdf)
+
+        # Make sure the tests run, but we can't check the result because they are non-deterministic.
+        kdf.sample(frac=0.1)
+        kdf.sample(frac=0.2, replace=True)
+        kdf.sample(frac=0.2, random_state=5)
+        kdf["A"].sample(frac=0.2)
+        kdf["A"].sample(frac=0.2, replace=True)
+        kdf["A"].sample(frac=0.2, random_state=5)
+
+        with self.assertRaises(ValueError):
+            kdf.sample()
+        with self.assertRaises(NotImplementedError):
+            kdf.sample(n=1)
+
+    def test_add_prefix(self):
+        pdf = pd.DataFrame({"A": [1, 2, 3, 4], "B": [3, 4, 5, 6]}, index=np.random.rand(4))
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.add_prefix("col_"), kdf.add_prefix("col_"))
+
+        columns = pd.MultiIndex.from_tuples([("X", "A"), ("X", "B")])
+        pdf.columns = columns
+        kdf.columns = columns
+        self.assert_eq(pdf.add_prefix("col_"), kdf.add_prefix("col_"))
+
+    def test_add_suffix(self):
+        pdf = pd.DataFrame({"A": [1, 2, 3, 4], "B": [3, 4, 5, 6]}, index=np.random.rand(4))
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.add_suffix("first_series"), kdf.add_suffix("first_series"))
+
+        columns = pd.MultiIndex.from_tuples([("X", "A"), ("X", "B")])
+        pdf.columns = columns
+        kdf.columns = columns
+        self.assert_eq(pdf.add_suffix("first_series"), kdf.add_suffix("first_series"))
+
+    def test_join(self):
+        # check basic function
+        pdf1 = pd.DataFrame(
+            {"key": ["K0", "K1", "K2", "K3"], "A": ["A0", "A1", "A2", "A3"]}, columns=["key", "A"]
+        )
+        pdf2 = pd.DataFrame(
+            {"key": ["K0", "K1", "K2"], "B": ["B0", "B1", "B2"]}, columns=["key", "B"]
+        )
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        join_pdf = pdf1.join(pdf2, lsuffix="_left", rsuffix="_right")
+        join_pdf.sort_values(by=list(join_pdf.columns), inplace=True)
+
+        join_kdf = kdf1.join(kdf2, lsuffix="_left", rsuffix="_right")
+        join_kdf.sort_values(by=list(join_kdf.columns), inplace=True)
+
+        self.assert_eq(join_pdf, join_kdf)
+
+        # join with duplicated columns in Series
+        with self.assertRaisesRegex(ValueError, "columns overlap but no suffix specified"):
+            ks1 = ps.Series(["A1", "A5"], index=[1, 2], name="A")
+            kdf1.join(ks1, how="outer")
+        # join with duplicated columns in DataFrame
+        with self.assertRaisesRegex(ValueError, "columns overlap but no suffix specified"):
+            kdf1.join(kdf2, how="outer")
+
+        # check `on` parameter
+        join_pdf = pdf1.join(pdf2.set_index("key"), on="key", lsuffix="_left", rsuffix="_right")
+        join_pdf.sort_values(by=list(join_pdf.columns), inplace=True)
+
+        join_kdf = kdf1.join(kdf2.set_index("key"), on="key", lsuffix="_left", rsuffix="_right")
+        join_kdf.sort_values(by=list(join_kdf.columns), inplace=True)
+        self.assert_eq(join_pdf.reset_index(drop=True), join_kdf.reset_index(drop=True))
+
+        join_pdf = pdf1.set_index("key").join(
+            pdf2.set_index("key"), on="key", lsuffix="_left", rsuffix="_right"
+        )
+        join_pdf.sort_values(by=list(join_pdf.columns), inplace=True)
+
+        join_kdf = kdf1.set_index("key").join(
+            kdf2.set_index("key"), on="key", lsuffix="_left", rsuffix="_right"
+        )
+        join_kdf.sort_values(by=list(join_kdf.columns), inplace=True)
+        self.assert_eq(join_pdf.reset_index(drop=True), join_kdf.reset_index(drop=True))
+
+        # multi-index columns
+        columns1 = pd.MultiIndex.from_tuples([("x", "key"), ("Y", "A")])
+        columns2 = pd.MultiIndex.from_tuples([("x", "key"), ("Y", "B")])
+        pdf1.columns = columns1
+        pdf2.columns = columns2
+        kdf1.columns = columns1
+        kdf2.columns = columns2
+
+        join_pdf = pdf1.join(pdf2, lsuffix="_left", rsuffix="_right")
+        join_pdf.sort_values(by=list(join_pdf.columns), inplace=True)
+
+        join_kdf = kdf1.join(kdf2, lsuffix="_left", rsuffix="_right")
+        join_kdf.sort_values(by=list(join_kdf.columns), inplace=True)
+
+        self.assert_eq(join_pdf, join_kdf)
+
+        # check `on` parameter
+        join_pdf = pdf1.join(
+            pdf2.set_index(("x", "key")), on=[("x", "key")], lsuffix="_left", rsuffix="_right"
+        )
+        join_pdf.sort_values(by=list(join_pdf.columns), inplace=True)
+
+        join_kdf = kdf1.join(
+            kdf2.set_index(("x", "key")), on=[("x", "key")], lsuffix="_left", rsuffix="_right"
+        )
+        join_kdf.sort_values(by=list(join_kdf.columns), inplace=True)
+
+        self.assert_eq(join_pdf.reset_index(drop=True), join_kdf.reset_index(drop=True))
+
+        join_pdf = pdf1.set_index(("x", "key")).join(
+            pdf2.set_index(("x", "key")), on=[("x", "key")], lsuffix="_left", rsuffix="_right"
+        )
+        join_pdf.sort_values(by=list(join_pdf.columns), inplace=True)
+
+        join_kdf = kdf1.set_index(("x", "key")).join(
+            kdf2.set_index(("x", "key")), on=[("x", "key")], lsuffix="_left", rsuffix="_right"
+        )
+        join_kdf.sort_values(by=list(join_kdf.columns), inplace=True)
+
+        self.assert_eq(join_pdf.reset_index(drop=True), join_kdf.reset_index(drop=True))
+
+        # multi-index
+        midx1 = pd.MultiIndex.from_tuples(
+            [("w", "a"), ("x", "b"), ("y", "c"), ("z", "d")], names=["index1", "index2"]
+        )
+        midx2 = pd.MultiIndex.from_tuples(
+            [("w", "a"), ("x", "b"), ("y", "c")], names=["index1", "index2"]
+        )
+        pdf1.index = midx1
+        pdf2.index = midx2
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        join_pdf = pdf1.join(pdf2, on=["index1", "index2"], rsuffix="_right")
+        join_pdf.sort_values(by=list(join_pdf.columns), inplace=True)
+
+        join_kdf = kdf1.join(kdf2, on=["index1", "index2"], rsuffix="_right")
+        join_kdf.sort_values(by=list(join_kdf.columns), inplace=True)
+
+        self.assert_eq(join_pdf, join_kdf)
+
+        with self.assertRaisesRegex(
+            ValueError, r'len\(left_on\) must equal the number of levels in the index of "right"'
+        ):
+            kdf1.join(kdf2, on=["index1"], rsuffix="_right")
+
+    def test_replace(self):
+        pdf = pd.DataFrame(
+            {
+                "name": ["Ironman", "Captain America", "Thor", "Hulk"],
+                "weapon": ["Mark-45", "Shield", "Mjolnir", "Smash"],
+            },
+            index=np.random.rand(4),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        with self.assertRaisesRegex(
+            NotImplementedError, "replace currently works only for method='pad"
+        ):
+            kdf.replace(method="bfill")
+        with self.assertRaisesRegex(
+            NotImplementedError, "replace currently works only when limit=None"
+        ):
+            kdf.replace(limit=10)
+        with self.assertRaisesRegex(
+            NotImplementedError, "replace currently doesn't supports regex"
+        ):
+            kdf.replace(regex="")
+
+        with self.assertRaisesRegex(ValueError, "Length of to_replace and value must be same"):
+            kdf.replace(to_replace=["Ironman"], value=["Spiderman", "Doctor Strange"])
+
+        self.assert_eq(kdf.replace("Ironman", "Spiderman"), pdf.replace("Ironman", "Spiderman"))
+        self.assert_eq(
+            kdf.replace(["Ironman", "Captain America"], ["Rescue", "Hawkeye"]),
+            pdf.replace(["Ironman", "Captain America"], ["Rescue", "Hawkeye"]),
+        )
+        self.assert_eq(
+            kdf.replace(("Ironman", "Captain America"), ("Rescue", "Hawkeye")),
+            pdf.replace(("Ironman", "Captain America"), ("Rescue", "Hawkeye")),
+        )
+
+        # inplace
+        pser = pdf.name
+        kser = kdf.name
+        pdf.replace("Ironman", "Spiderman", inplace=True)
+        kdf.replace("Ironman", "Spiderman", inplace=True)
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kser, pser)
+
+        pdf = pd.DataFrame(
+            {"A": [0, 1, 2, 3, np.nan], "B": [5, 6, 7, 8, np.nan], "C": ["a", "b", "c", "d", None]},
+            index=np.random.rand(5),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.replace([0, 1, 2, 3, 5, 6], 4), pdf.replace([0, 1, 2, 3, 5, 6], 4))
+
+        self.assert_eq(
+            kdf.replace([0, 1, 2, 3, 5, 6], [6, 5, 4, 3, 2, 1]),
+            pdf.replace([0, 1, 2, 3, 5, 6], [6, 5, 4, 3, 2, 1]),
+        )
+
+        self.assert_eq(kdf.replace({0: 10, 1: 100, 7: 200}), pdf.replace({0: 10, 1: 100, 7: 200}))
+
+        self.assert_eq(
+            kdf.replace({"A": [0, np.nan], "B": [5, np.nan]}, 100),
+            pdf.replace({"A": [0, np.nan], "B": [5, np.nan]}, 100),
+        )
+
+        self.assert_eq(
+            kdf.replace({"A": {0: 100, 4: 400, np.nan: 700}}),
+            pdf.replace({"A": {0: 100, 4: 400, np.nan: 700}}),
+        )
+        self.assert_eq(
+            kdf.replace({"X": {0: 100, 4: 400, np.nan: 700}}),
+            pdf.replace({"X": {0: 100, 4: 400, np.nan: 700}}),
+        )
+
+        self.assert_eq(kdf.replace({"C": ["a", None]}, "e"), pdf.replace({"C": ["a", None]}, "e"))
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("X", "A"), ("X", "B"), ("Y", "C")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(kdf.replace([0, 1, 2, 3, 5, 6], 4), pdf.replace([0, 1, 2, 3, 5, 6], 4))
+
+        self.assert_eq(
+            kdf.replace([0, 1, 2, 3, 5, 6], [6, 5, 4, 3, 2, 1]),
+            pdf.replace([0, 1, 2, 3, 5, 6], [6, 5, 4, 3, 2, 1]),
+        )
+
+        self.assert_eq(kdf.replace({0: 10, 1: 100, 7: 200}), pdf.replace({0: 10, 1: 100, 7: 200}))
+
+        self.assert_eq(
+            kdf.replace({("X", "A"): [0, np.nan], ("X", "B"): 5}, 100),
+            pdf.replace({("X", "A"): [0, np.nan], ("X", "B"): 5}, 100),
+        )
+
+        self.assert_eq(
+            kdf.replace({("X", "A"): {0: 100, 4: 400, np.nan: 700}}),
+            pdf.replace({("X", "A"): {0: 100, 4: 400, np.nan: 700}}),
+        )
+        self.assert_eq(
+            kdf.replace({("X", "B"): {0: 100, 4: 400, np.nan: 700}}),
+            pdf.replace({("X", "B"): {0: 100, 4: 400, np.nan: 700}}),
+        )
+
+        self.assert_eq(
+            kdf.replace({("Y", "C"): ["a", None]}, "e"), pdf.replace({("Y", "C"): ["a", None]}, "e")
+        )
+
+    def test_update(self):
+        # check base function
+        def get_data(left_columns=None, right_columns=None):
+            left_pdf = pd.DataFrame(
+                {"A": ["1", "2", "3", "4"], "B": ["100", "200", np.nan, np.nan]}, columns=["A", "B"]
+            )
+            right_pdf = pd.DataFrame(
+                {"B": ["x", np.nan, "y", np.nan], "C": ["100", "200", "300", "400"]},
+                columns=["B", "C"],
+            )
+
+            left_kdf = ps.DataFrame(
+                {"A": ["1", "2", "3", "4"], "B": ["100", "200", None, None]}, columns=["A", "B"]
+            )
+            right_kdf = ps.DataFrame(
+                {"B": ["x", None, "y", None], "C": ["100", "200", "300", "400"]}, columns=["B", "C"]
+            )
+            if left_columns is not None:
+                left_pdf.columns = left_columns
+                left_kdf.columns = left_columns
+            if right_columns is not None:
+                right_pdf.columns = right_columns
+                right_kdf.columns = right_columns
+            return left_kdf, left_pdf, right_kdf, right_pdf
+
+        left_kdf, left_pdf, right_kdf, right_pdf = get_data()
+        pser = left_pdf.B
+        kser = left_kdf.B
+        left_pdf.update(right_pdf)
+        left_kdf.update(right_kdf)
+        self.assert_eq(left_pdf.sort_values(by=["A", "B"]), left_kdf.sort_values(by=["A", "B"]))
+        self.assert_eq(kser.sort_index(), pser.sort_index())
+
+        left_kdf, left_pdf, right_kdf, right_pdf = get_data()
+        left_pdf.update(right_pdf, overwrite=False)
+        left_kdf.update(right_kdf, overwrite=False)
+        self.assert_eq(left_pdf.sort_values(by=["A", "B"]), left_kdf.sort_values(by=["A", "B"]))
+
+        with self.assertRaises(NotImplementedError):
+            left_kdf.update(right_kdf, join="right")
+
+        # multi-index columns
+        left_columns = pd.MultiIndex.from_tuples([("X", "A"), ("X", "B")])
+        right_columns = pd.MultiIndex.from_tuples([("X", "B"), ("Y", "C")])
+
+        left_kdf, left_pdf, right_kdf, right_pdf = get_data(
+            left_columns=left_columns, right_columns=right_columns
+        )
+        left_pdf.update(right_pdf)
+        left_kdf.update(right_kdf)
+        self.assert_eq(
+            left_pdf.sort_values(by=[("X", "A"), ("X", "B")]),
+            left_kdf.sort_values(by=[("X", "A"), ("X", "B")]),
+        )
+
+        left_kdf, left_pdf, right_kdf, right_pdf = get_data(
+            left_columns=left_columns, right_columns=right_columns
+        )
+        left_pdf.update(right_pdf, overwrite=False)
+        left_kdf.update(right_kdf, overwrite=False)
+        self.assert_eq(
+            left_pdf.sort_values(by=[("X", "A"), ("X", "B")]),
+            left_kdf.sort_values(by=[("X", "A"), ("X", "B")]),
+        )
+
+        right_columns = pd.MultiIndex.from_tuples([("Y", "B"), ("Y", "C")])
+        left_kdf, left_pdf, right_kdf, right_pdf = get_data(
+            left_columns=left_columns, right_columns=right_columns
+        )
+        left_pdf.update(right_pdf)
+        left_kdf.update(right_kdf)
+        self.assert_eq(
+            left_pdf.sort_values(by=[("X", "A"), ("X", "B")]),
+            left_kdf.sort_values(by=[("X", "A"), ("X", "B")]),
+        )
+
+    def test_pivot_table_dtypes(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [4, 2, 3, 4, 8, 6],
+                "b": [1, 2, 2, 4, 2, 4],
+                "e": [1, 2, 2, 4, 2, 4],
+                "c": [1, 2, 9, 4, 7, 4],
+            },
+            index=np.random.rand(6),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        # Skip columns comparison by reset_index
+        res_df = kdf.pivot_table(
+            index=["c"], columns="a", values=["b"], aggfunc={"b": "mean"}
+        ).dtypes.reset_index(drop=True)
+        exp_df = pdf.pivot_table(
+            index=["c"], columns="a", values=["b"], aggfunc={"b": "mean"}
+        ).dtypes.reset_index(drop=True)
+        self.assert_eq(res_df, exp_df)
+
+        # Results don't have the same column's name
+
+        # Todo: self.assert_eq(kdf.pivot_table(columns="a", values="b").dtypes,
+        #  pdf.pivot_table(columns="a", values="b").dtypes)
+
+        # Todo: self.assert_eq(kdf.pivot_table(index=['c'], columns="a", values="b").dtypes,
+        #  pdf.pivot_table(index=['c'], columns="a", values="b").dtypes)
+
+        # Todo: self.assert_eq(kdf.pivot_table(index=['e', 'c'], columns="a", values="b").dtypes,
+        #  pdf.pivot_table(index=['e', 'c'], columns="a", values="b").dtypes)
+
+        # Todo: self.assert_eq(kdf.pivot_table(index=['e', 'c'],
+        #  columns="a", values="b", fill_value=999).dtypes, pdf.pivot_table(index=['e', 'c'],
+        #  columns="a", values="b", fill_value=999).dtypes)
+
+    def test_pivot_table(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [4, 2, 3, 4, 8, 6],
+                "b": [1, 2, 2, 4, 2, 4],
+                "e": [10, 20, 20, 40, 20, 40],
+                "c": [1, 2, 9, 4, 7, 4],
+                "d": [-1, -2, -3, -4, -5, -6],
+            },
+            index=np.random.rand(6),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        # Checking if both DataFrames have the same results
+        self.assert_eq(
+            kdf.pivot_table(columns="a", values="b").sort_index(),
+            pdf.pivot_table(columns="a", values="b").sort_index(),
+            almost=True,
+        )
+
+        self.assert_eq(
+            kdf.pivot_table(index=["c"], columns="a", values="b").sort_index(),
+            pdf.pivot_table(index=["c"], columns="a", values="b").sort_index(),
+            almost=True,
+        )
+
+        self.assert_eq(
+            kdf.pivot_table(index=["c"], columns="a", values="b", aggfunc="sum").sort_index(),
+            pdf.pivot_table(index=["c"], columns="a", values="b", aggfunc="sum").sort_index(),
+            almost=True,
+        )
+
+        self.assert_eq(
+            kdf.pivot_table(index=["c"], columns="a", values=["b"], aggfunc="sum").sort_index(),
+            pdf.pivot_table(index=["c"], columns="a", values=["b"], aggfunc="sum").sort_index(),
+            almost=True,
+        )
+
+        self.assert_eq(
+            kdf.pivot_table(
+                index=["c"], columns="a", values=["b", "e"], aggfunc="sum"
+            ).sort_index(),
+            pdf.pivot_table(
+                index=["c"], columns="a", values=["b", "e"], aggfunc="sum"
+            ).sort_index(),
+            almost=True,
+        )
+
+        self.assert_eq(
+            kdf.pivot_table(
+                index=["c"], columns="a", values=["b", "e", "d"], aggfunc="sum"
+            ).sort_index(),
+            pdf.pivot_table(
+                index=["c"], columns="a", values=["b", "e", "d"], aggfunc="sum"
+            ).sort_index(),
+            almost=True,
+        )
+
+        self.assert_eq(
+            kdf.pivot_table(
+                index=["c"], columns="a", values=["b", "e"], aggfunc={"b": "mean", "e": "sum"}
+            ).sort_index(),
+            pdf.pivot_table(
+                index=["c"], columns="a", values=["b", "e"], aggfunc={"b": "mean", "e": "sum"}
+            ).sort_index(),
+            almost=True,
+        )
+
+        self.assert_eq(
+            kdf.pivot_table(index=["e", "c"], columns="a", values="b").sort_index(),
+            pdf.pivot_table(index=["e", "c"], columns="a", values="b").sort_index(),
+            almost=True,
+        )
+
+        self.assert_eq(
+            kdf.pivot_table(index=["e", "c"], columns="a", values="b", fill_value=999).sort_index(),
+            pdf.pivot_table(index=["e", "c"], columns="a", values="b", fill_value=999).sort_index(),
+            almost=True,
+        )
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples(
+            [("x", "a"), ("x", "b"), ("y", "e"), ("z", "c"), ("w", "d")]
+        )
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(
+            kdf.pivot_table(columns=("x", "a"), values=("x", "b")).sort_index(),
+            pdf.pivot_table(columns=[("x", "a")], values=[("x", "b")]).sort_index(),
+            almost=True,
+        )
+
+        self.assert_eq(
+            kdf.pivot_table(
+                index=[("z", "c")], columns=("x", "a"), values=[("x", "b")]
+            ).sort_index(),
+            pdf.pivot_table(
+                index=[("z", "c")], columns=[("x", "a")], values=[("x", "b")]
+            ).sort_index(),
+            almost=True,
+        )
+
+        self.assert_eq(
+            kdf.pivot_table(
+                index=[("z", "c")], columns=("x", "a"), values=[("x", "b"), ("y", "e")]
+            ).sort_index(),
+            pdf.pivot_table(
+                index=[("z", "c")], columns=[("x", "a")], values=[("x", "b"), ("y", "e")]
+            ).sort_index(),
+            almost=True,
+        )
+
+        self.assert_eq(
+            kdf.pivot_table(
+                index=[("z", "c")], columns=("x", "a"), values=[("x", "b"), ("y", "e"), ("w", "d")]
+            ).sort_index(),
+            pdf.pivot_table(
+                index=[("z", "c")],
+                columns=[("x", "a")],
+                values=[("x", "b"), ("y", "e"), ("w", "d")],
+            ).sort_index(),
+            almost=True,
+        )
+
+        self.assert_eq(
+            kdf.pivot_table(
+                index=[("z", "c")],
+                columns=("x", "a"),
+                values=[("x", "b"), ("y", "e")],
+                aggfunc={("x", "b"): "mean", ("y", "e"): "sum"},
+            ).sort_index(),
+            pdf.pivot_table(
+                index=[("z", "c")],
+                columns=[("x", "a")],
+                values=[("x", "b"), ("y", "e")],
+                aggfunc={("x", "b"): "mean", ("y", "e"): "sum"},
+            ).sort_index(),
+            almost=True,
+        )
+
+    def test_pivot_table_and_index(self):
+        # https://github.com/databricks/koalas/issues/805
+        pdf = pd.DataFrame(
+            {
+                "A": ["foo", "foo", "foo", "foo", "foo", "bar", "bar", "bar", "bar"],
+                "B": ["one", "one", "one", "two", "two", "one", "one", "two", "two"],
+                "C": [
+                    "small",
+                    "large",
+                    "large",
+                    "small",
+                    "small",
+                    "large",
+                    "small",
+                    "small",
+                    "large",
+                ],
+                "D": [1, 2, 2, 3, 3, 4, 5, 6, 7],
+                "E": [2, 4, 5, 5, 6, 6, 8, 9, 9],
+            },
+            columns=["A", "B", "C", "D", "E"],
+            index=np.random.rand(9),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        ptable = pdf.pivot_table(
+            values="D", index=["A", "B"], columns="C", aggfunc="sum", fill_value=0
+        ).sort_index()
+        ktable = kdf.pivot_table(
+            values="D", index=["A", "B"], columns="C", aggfunc="sum", fill_value=0
+        ).sort_index()
+
+        self.assert_eq(ktable, ptable)
+        self.assert_eq(ktable.index, ptable.index)
+        self.assert_eq(repr(ktable.index), repr(ptable.index))
+
+    def test_stack(self):
+        pdf_single_level_cols = pd.DataFrame(
+            [[0, 1], [2, 3]], index=["cat", "dog"], columns=["weight", "height"]
+        )
+        kdf_single_level_cols = ps.from_pandas(pdf_single_level_cols)
+
+        self.assert_eq(
+            kdf_single_level_cols.stack().sort_index(), pdf_single_level_cols.stack().sort_index()
+        )
+
+        multicol1 = pd.MultiIndex.from_tuples(
+            [("weight", "kg"), ("weight", "pounds")], names=["x", "y"]
+        )
+        pdf_multi_level_cols1 = pd.DataFrame(
+            [[1, 2], [2, 4]], index=["cat", "dog"], columns=multicol1
+        )
+        kdf_multi_level_cols1 = ps.from_pandas(pdf_multi_level_cols1)
+
+        self.assert_eq(
+            kdf_multi_level_cols1.stack().sort_index(), pdf_multi_level_cols1.stack().sort_index()
+        )
+
+        multicol2 = pd.MultiIndex.from_tuples([("weight", "kg"), ("height", "m")])
+        pdf_multi_level_cols2 = pd.DataFrame(
+            [[1.0, 2.0], [3.0, 4.0]], index=["cat", "dog"], columns=multicol2
+        )
+        kdf_multi_level_cols2 = ps.from_pandas(pdf_multi_level_cols2)
+
+        self.assert_eq(
+            kdf_multi_level_cols2.stack().sort_index(), pdf_multi_level_cols2.stack().sort_index()
+        )
+
+        pdf = pd.DataFrame(
+            {
+                ("y", "c"): [True, True],
+                ("x", "b"): [False, False],
+                ("x", "c"): [True, False],
+                ("y", "a"): [False, True],
+            }
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.stack().sort_index(), pdf.stack().sort_index())
+        self.assert_eq(kdf[[]].stack().sort_index(), pdf[[]].stack().sort_index(), almost=True)
+
+    def test_unstack(self):
+        pdf = pd.DataFrame(
+            np.random.randn(3, 3),
+            index=pd.MultiIndex.from_tuples([("rg1", "x"), ("rg1", "y"), ("rg2", "z")]),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.unstack().sort_index(), pdf.unstack().sort_index(), almost=True)
+
+    def test_pivot_errors(self):
+        kdf = ps.range(10)
+
+        with self.assertRaisesRegex(ValueError, "columns should be set"):
+            kdf.pivot(index="id")
+
+        with self.assertRaisesRegex(ValueError, "values should be set"):
+            kdf.pivot(index="id", columns="id")
+
+    def test_pivot_table_errors(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [4, 2, 3, 4, 8, 6],
+                "b": [1, 2, 2, 4, 2, 4],
+                "e": [1, 2, 2, 4, 2, 4],
+                "c": [1, 2, 9, 4, 7, 4],
+            },
+            index=np.random.rand(6),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assertRaises(KeyError, lambda: kdf.pivot_table(index=["c"], columns="a", values=5))
+
+        msg = "index should be a None or a list of columns."
+        with self.assertRaisesRegex(TypeError, msg):
+            kdf.pivot_table(index="c", columns="a", values="b")
+
+        msg = "pivot_table doesn't support aggfunc as dict and without index."
+        with self.assertRaisesRegex(NotImplementedError, msg):
+            kdf.pivot_table(columns="a", values=["b", "e"], aggfunc={"b": "mean", "e": "sum"})
+
+        msg = "columns should be one column name."
+        with self.assertRaisesRegex(TypeError, msg):
+            kdf.pivot_table(columns=["a"], values=["b"], aggfunc={"b": "mean", "e": "sum"})
+
+        msg = "Columns in aggfunc must be the same as values."
+        with self.assertRaisesRegex(ValueError, msg):
+            kdf.pivot_table(
+                index=["e", "c"], columns="a", values="b", aggfunc={"b": "mean", "e": "sum"}
+            )
+
+        msg = "values can't be a list without index."
+        with self.assertRaisesRegex(NotImplementedError, msg):
+            kdf.pivot_table(columns="a", values=["b", "e"])
+
+        msg = "Wrong columns A."
+        with self.assertRaisesRegex(ValueError, msg):
+            kdf.pivot_table(
+                index=["c"], columns="A", values=["b", "e"], aggfunc={"b": "mean", "e": "sum"}
+            )
+
+        kdf = ps.DataFrame(
+            {
+                "A": ["foo", "foo", "foo", "foo", "foo", "bar", "bar", "bar", "bar"],
+                "B": ["one", "one", "one", "two", "two", "one", "one", "two", "two"],
+                "C": [
+                    "small",
+                    "large",
+                    "large",
+                    "small",
+                    "small",
+                    "large",
+                    "small",
+                    "small",
+                    "large",
+                ],
+                "D": [1, 2, 2, 3, 3, 4, 5, 6, 7],
+                "E": [2, 4, 5, 5, 6, 6, 8, 9, 9],
+            },
+            columns=["A", "B", "C", "D", "E"],
+            index=np.random.rand(9),
+        )
+
+        msg = "values should be a numeric type."
+        with self.assertRaisesRegex(TypeError, msg):
+            kdf.pivot_table(
+                index=["C"], columns="A", values=["B", "E"], aggfunc={"B": "mean", "E": "sum"}
+            )
+
+        msg = "values should be a numeric type."
+        with self.assertRaisesRegex(TypeError, msg):
+            kdf.pivot_table(index=["C"], columns="A", values="B", aggfunc={"B": "mean"})
+
+    def test_transpose(self):
+        # TODO: what if with random index?
+        pdf1 = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}, columns=["col1", "col2"])
+        kdf1 = ps.from_pandas(pdf1)
+
+        pdf2 = pd.DataFrame(
+            data={"score": [9, 8], "kids": [0, 0], "age": [12, 22]},
+            columns=["score", "kids", "age"],
+        )
+        kdf2 = ps.from_pandas(pdf2)
+
+        self.assert_eq(pdf1.transpose().sort_index(), kdf1.transpose().sort_index())
+        self.assert_eq(pdf2.transpose().sort_index(), kdf2.transpose().sort_index())
+
+        with option_context("compute.max_rows", None):
+            self.assert_eq(pdf1.transpose().sort_index(), kdf1.transpose().sort_index())
+
+            self.assert_eq(pdf2.transpose().sort_index(), kdf2.transpose().sort_index())
+
+        pdf3 = pd.DataFrame(
+            {
+                ("cg1", "a"): [1, 2, 3],
+                ("cg1", "b"): [4, 5, 6],
+                ("cg2", "c"): [7, 8, 9],
+                ("cg3", "d"): [9, 9, 9],
+            },
+            index=pd.MultiIndex.from_tuples([("rg1", "x"), ("rg1", "y"), ("rg2", "z")]),
+        )
+        kdf3 = ps.from_pandas(pdf3)
+
+        self.assert_eq(pdf3.transpose().sort_index(), kdf3.transpose().sort_index())
+
+        with option_context("compute.max_rows", None):
+            self.assert_eq(pdf3.transpose().sort_index(), kdf3.transpose().sort_index())
+
+    def _test_cummin(self, pdf, kdf):
+        self.assert_eq(pdf.cummin(), kdf.cummin())
+        self.assert_eq(pdf.cummin(skipna=False), kdf.cummin(skipna=False))
+        self.assert_eq(pdf.cummin().sum(), kdf.cummin().sum())
+
+    def test_cummin(self):
+        pdf = pd.DataFrame(
+            [[2.0, 1.0], [5, None], [1.0, 0.0], [2.0, 4.0], [4.0, 9.0]],
+            columns=list("AB"),
+            index=np.random.rand(5),
+        )
+        kdf = ps.from_pandas(pdf)
+        self._test_cummin(pdf, kdf)
+
+    def test_cummin_multiindex_columns(self):
+        arrays = [np.array(["A", "A", "B", "B"]), np.array(["one", "two", "one", "two"])]
+        pdf = pd.DataFrame(np.random.randn(3, 4), index=["A", "C", "B"], columns=arrays)
+        pdf.at["C", ("A", "two")] = None
+        kdf = ps.from_pandas(pdf)
+        self._test_cummin(pdf, kdf)
+
+    def _test_cummax(self, pdf, kdf):
+        self.assert_eq(pdf.cummax(), kdf.cummax())
+        self.assert_eq(pdf.cummax(skipna=False), kdf.cummax(skipna=False))
+        self.assert_eq(pdf.cummax().sum(), kdf.cummax().sum())
+
+    def test_cummax(self):
+        pdf = pd.DataFrame(
+            [[2.0, 1.0], [5, None], [1.0, 0.0], [2.0, 4.0], [4.0, 9.0]],
+            columns=list("AB"),
+            index=np.random.rand(5),
+        )
+        kdf = ps.from_pandas(pdf)
+        self._test_cummax(pdf, kdf)
+
+    def test_cummax_multiindex_columns(self):
+        arrays = [np.array(["A", "A", "B", "B"]), np.array(["one", "two", "one", "two"])]
+        pdf = pd.DataFrame(np.random.randn(3, 4), index=["A", "C", "B"], columns=arrays)
+        pdf.at["C", ("A", "two")] = None
+        kdf = ps.from_pandas(pdf)
+        self._test_cummax(pdf, kdf)
+
+    def _test_cumsum(self, pdf, kdf):
+        self.assert_eq(pdf.cumsum(), kdf.cumsum())
+        self.assert_eq(pdf.cumsum(skipna=False), kdf.cumsum(skipna=False))
+        self.assert_eq(pdf.cumsum().sum(), kdf.cumsum().sum())
+
+    def test_cumsum(self):
+        pdf = pd.DataFrame(
+            [[2.0, 1.0], [5, None], [1.0, 0.0], [2.0, 4.0], [4.0, 9.0]],
+            columns=list("AB"),
+            index=np.random.rand(5),
+        )
+        kdf = ps.from_pandas(pdf)
+        self._test_cumsum(pdf, kdf)
+
+    def test_cumsum_multiindex_columns(self):
+        arrays = [np.array(["A", "A", "B", "B"]), np.array(["one", "two", "one", "two"])]
+        pdf = pd.DataFrame(np.random.randn(3, 4), index=["A", "C", "B"], columns=arrays)
+        pdf.at["C", ("A", "two")] = None
+        kdf = ps.from_pandas(pdf)
+        self._test_cumsum(pdf, kdf)
+
+    def _test_cumprod(self, pdf, kdf):
+        self.assert_eq(pdf.cumprod(), kdf.cumprod(), almost=True)
+        self.assert_eq(pdf.cumprod(skipna=False), kdf.cumprod(skipna=False), almost=True)
+        self.assert_eq(pdf.cumprod().sum(), kdf.cumprod().sum(), almost=True)
+
+    def test_cumprod(self):
+        pdf = pd.DataFrame(
+            [[2.0, 1.0, 1], [5, None, 2], [1.0, -1.0, -3], [2.0, 0, 4], [4.0, 9.0, 5]],
+            columns=list("ABC"),
+            index=np.random.rand(5),
+        )
+        kdf = ps.from_pandas(pdf)
+        self._test_cumprod(pdf, kdf)
+
+    def test_cumprod_multiindex_columns(self):
+        arrays = [np.array(["A", "A", "B", "B"]), np.array(["one", "two", "one", "two"])]
+        pdf = pd.DataFrame(np.random.rand(3, 4), index=["A", "C", "B"], columns=arrays)
+        pdf.at["C", ("A", "two")] = None
+        kdf = ps.from_pandas(pdf)
+        self._test_cumprod(pdf, kdf)
+
+    def test_drop_duplicates(self):
+        pdf = pd.DataFrame(
+            {"a": [1, 2, 2, 2, 3], "b": ["a", "a", "a", "c", "d"]}, index=np.random.rand(5)
+        )
+        kdf = ps.from_pandas(pdf)
+
+        # inplace is False
+        for keep in ["first", "last", False]:
+            with self.subTest(keep=keep):
+                self.assert_eq(
+                    pdf.drop_duplicates(keep=keep).sort_index(),
+                    kdf.drop_duplicates(keep=keep).sort_index(),
+                )
+                self.assert_eq(
+                    pdf.drop_duplicates("a", keep=keep).sort_index(),
+                    kdf.drop_duplicates("a", keep=keep).sort_index(),
+                )
+                self.assert_eq(
+                    pdf.drop_duplicates(["a", "b"], keep=keep).sort_index(),
+                    kdf.drop_duplicates(["a", "b"], keep=keep).sort_index(),
+                )
+                self.assert_eq(
+                    pdf.set_index("a", append=True).drop_duplicates(keep=keep).sort_index(),
+                    kdf.set_index("a", append=True).drop_duplicates(keep=keep).sort_index(),
+                )
+                self.assert_eq(
+                    pdf.set_index("a", append=True).drop_duplicates("b", keep=keep).sort_index(),
+                    kdf.set_index("a", append=True).drop_duplicates("b", keep=keep).sort_index(),
+                )
+
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("y", "b")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        # inplace is False
+        for keep in ["first", "last", False]:
+            with self.subTest("multi-index columns", keep=keep):
+                self.assert_eq(
+                    pdf.drop_duplicates(keep=keep).sort_index(),
+                    kdf.drop_duplicates(keep=keep).sort_index(),
+                )
+                self.assert_eq(
+                    pdf.drop_duplicates(("x", "a"), keep=keep).sort_index(),
+                    kdf.drop_duplicates(("x", "a"), keep=keep).sort_index(),
+                )
+                self.assert_eq(
+                    pdf.drop_duplicates([("x", "a"), ("y", "b")], keep=keep).sort_index(),
+                    kdf.drop_duplicates([("x", "a"), ("y", "b")], keep=keep).sort_index(),
+                )
+
+        # inplace is True
+        subset_list = [None, "a", ["a", "b"]]
+        for subset in subset_list:
+            pdf = pd.DataFrame(
+                {"a": [1, 2, 2, 2, 3], "b": ["a", "a", "a", "c", "d"]}, index=np.random.rand(5)
+            )
+            kdf = ps.from_pandas(pdf)
+            pser = pdf.a
+            kser = kdf.a
+            pdf.drop_duplicates(subset=subset, inplace=True)
+            kdf.drop_duplicates(subset=subset, inplace=True)
+            self.assert_eq(kdf.sort_index(), pdf.sort_index())
+            self.assert_eq(kser.sort_index(), pser.sort_index())
+
+        # multi-index columns, inplace is True
+        subset_list = [None, ("x", "a"), [("x", "a"), ("y", "b")]]
+        for subset in subset_list:
+            pdf = pd.DataFrame(
+                {"a": [1, 2, 2, 2, 3], "b": ["a", "a", "a", "c", "d"]}, index=np.random.rand(5)
+            )
+            kdf = ps.from_pandas(pdf)
+            columns = pd.MultiIndex.from_tuples([("x", "a"), ("y", "b")])
+            pdf.columns = columns
+            kdf.columns = columns
+            pser = pdf[("x", "a")]
+            kser = kdf[("x", "a")]
+            pdf.drop_duplicates(subset=subset, inplace=True)
+            kdf.drop_duplicates(subset=subset, inplace=True)
+            self.assert_eq(kdf.sort_index(), pdf.sort_index())
+            self.assert_eq(kser.sort_index(), pser.sort_index())
+
+        # non-string names
+        pdf = pd.DataFrame(
+            {10: [1, 2, 2, 2, 3], 20: ["a", "a", "a", "c", "d"]}, index=np.random.rand(5)
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            pdf.drop_duplicates(10, keep=keep).sort_index(),
+            kdf.drop_duplicates(10, keep=keep).sort_index(),
+        )
+        self.assert_eq(
+            pdf.drop_duplicates([10, 20], keep=keep).sort_index(),
+            kdf.drop_duplicates([10, 20], keep=keep).sort_index(),
+        )
+
+    def test_reindex(self):
+        index = pd.Index(["A", "B", "C", "D", "E"])
+        columns = pd.Index(["numbers"])
+        pdf = pd.DataFrame([1.0, 2.0, 3.0, 4.0, None], index=index, columns=columns)
+        kdf = ps.from_pandas(pdf)
+
+        columns2 = pd.Index(["numbers", "2", "3"], name="cols2")
+        self.assert_eq(
+            pdf.reindex(columns=columns2).sort_index(), kdf.reindex(columns=columns2).sort_index(),
+        )
+
+        columns = pd.Index(["numbers"], name="cols")
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(
+            pdf.reindex(["A", "B", "C"], columns=["numbers", "2", "3"]).sort_index(),
+            kdf.reindex(["A", "B", "C"], columns=["numbers", "2", "3"]).sort_index(),
+        )
+
+        self.assert_eq(
+            pdf.reindex(["A", "B", "C"], index=["numbers", "2", "3"]).sort_index(),
+            kdf.reindex(["A", "B", "C"], index=["numbers", "2", "3"]).sort_index(),
+        )
+
+        self.assert_eq(
+            pdf.reindex(index=["A", "B"]).sort_index(), kdf.reindex(index=["A", "B"]).sort_index()
+        )
+
+        self.assert_eq(
+            pdf.reindex(index=["A", "B", "2", "3"]).sort_index(),
+            kdf.reindex(index=["A", "B", "2", "3"]).sort_index(),
+        )
+
+        self.assert_eq(
+            pdf.reindex(index=["A", "E", "2", "3"], fill_value=0).sort_index(),
+            kdf.reindex(index=["A", "E", "2", "3"], fill_value=0).sort_index(),
+        )
+
+        self.assert_eq(
+            pdf.reindex(columns=["numbers"]).sort_index(),
+            kdf.reindex(columns=["numbers"]).sort_index(),
+        )
+
+        # Using float as fill_value to avoid int64/32 clash
+        self.assert_eq(
+            pdf.reindex(columns=["numbers", "2", "3"], fill_value=0.0).sort_index(),
+            kdf.reindex(columns=["numbers", "2", "3"], fill_value=0.0).sort_index(),
+        )
+
+        columns2 = pd.Index(["numbers", "2", "3"])
+        self.assert_eq(
+            pdf.reindex(columns=columns2).sort_index(), kdf.reindex(columns=columns2).sort_index(),
+        )
+
+        columns2 = pd.Index(["numbers", "2", "3"], name="cols2")
+        self.assert_eq(
+            pdf.reindex(columns=columns2).sort_index(), kdf.reindex(columns=columns2).sort_index(),
+        )
+
+        # Reindexing single Index on single Index
+        pindex2 = pd.Index(["A", "C", "D", "E", "0"], name="index2")
+        kindex2 = ps.from_pandas(pindex2)
+
+        for fill_value in [None, 0]:
+            self.assert_eq(
+                pdf.reindex(index=pindex2, fill_value=fill_value).sort_index(),
+                kdf.reindex(index=kindex2, fill_value=fill_value).sort_index(),
+            )
+
+        pindex2 = pd.DataFrame({"index2": ["A", "C", "D", "E", "0"]}).set_index("index2").index
+        kindex2 = ps.from_pandas(pindex2)
+
+        for fill_value in [None, 0]:
+            self.assert_eq(
+                pdf.reindex(index=pindex2, fill_value=fill_value).sort_index(),
+                kdf.reindex(index=kindex2, fill_value=fill_value).sort_index(),
+            )
+
+        # Reindexing MultiIndex on single Index
+        pindex = pd.MultiIndex.from_tuples(
+            [("A", "B"), ("C", "D"), ("F", "G")], names=["name1", "name2"]
+        )
+        kindex = ps.from_pandas(pindex)
+
+        self.assert_eq(
+            pdf.reindex(index=pindex, fill_value=0.0).sort_index(),
+            kdf.reindex(index=kindex, fill_value=0.0).sort_index(),
+        )
+
+        self.assertRaises(TypeError, lambda: kdf.reindex(columns=["numbers", "2", "3"], axis=1))
+        self.assertRaises(TypeError, lambda: kdf.reindex(columns=["numbers", "2", "3"], axis=2))
+        self.assertRaises(TypeError, lambda: kdf.reindex(index=["A", "B", "C"], axis=1))
+        self.assertRaises(TypeError, lambda: kdf.reindex(index=123))
+
+        # Reindexing MultiIndex on MultiIndex
+        pdf = pd.DataFrame({"numbers": [1.0, 2.0, None]}, index=pindex)
+        kdf = ps.from_pandas(pdf)
+        pindex2 = pd.MultiIndex.from_tuples(
+            [("A", "G"), ("C", "D"), ("I", "J")], names=["name1", "name2"]
+        )
+        kindex2 = ps.from_pandas(pindex2)
+
+        for fill_value in [None, 0.0]:
+            self.assert_eq(
+                pdf.reindex(index=pindex2, fill_value=fill_value).sort_index(),
+                kdf.reindex(index=kindex2, fill_value=fill_value).sort_index(),
+            )
+
+        pindex2 = (
+            pd.DataFrame({"index_level_1": ["A", "C", "I"], "index_level_2": ["G", "D", "J"]})
+            .set_index(["index_level_1", "index_level_2"])
+            .index
+        )
+        kindex2 = ps.from_pandas(pindex2)
+
+        for fill_value in [None, 0.0]:
+            self.assert_eq(
+                pdf.reindex(index=pindex2, fill_value=fill_value).sort_index(),
+                kdf.reindex(index=kindex2, fill_value=fill_value).sort_index(),
+            )
+
+        columns = pd.MultiIndex.from_tuples([("X", "numbers")], names=["cols1", "cols2"])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        # Reindexing MultiIndex index on MultiIndex columns and MultiIndex index
+        for fill_value in [None, 0.0]:
+            self.assert_eq(
+                pdf.reindex(index=pindex2, fill_value=fill_value).sort_index(),
+                kdf.reindex(index=kindex2, fill_value=fill_value).sort_index(),
+            )
+
+        index = pd.Index(["A", "B", "C", "D", "E"])
+        pdf = pd.DataFrame(data=[1.0, 2.0, 3.0, 4.0, None], index=index, columns=columns)
+        kdf = ps.from_pandas(pdf)
+        pindex2 = pd.Index(["A", "C", "D", "E", "0"], name="index2")
+        kindex2 = ps.from_pandas(pindex2)
+
+        # Reindexing single Index on MultiIndex columns and single Index
+        for fill_value in [None, 0.0]:
+            self.assert_eq(
+                pdf.reindex(index=pindex2, fill_value=fill_value).sort_index(),
+                kdf.reindex(index=kindex2, fill_value=fill_value).sort_index(),
+            )
+
+        for fill_value in [None, 0.0]:
+            self.assert_eq(
+                pdf.reindex(
+                    columns=[("X", "numbers"), ("Y", "2"), ("Y", "3")], fill_value=fill_value
+                ).sort_index(),
+                kdf.reindex(
+                    columns=[("X", "numbers"), ("Y", "2"), ("Y", "3")], fill_value=fill_value
+                ).sort_index(),
+            )
+
+        columns2 = pd.MultiIndex.from_tuples(
+            [("X", "numbers"), ("Y", "2"), ("Y", "3")], names=["cols3", "cols4"]
+        )
+        self.assert_eq(
+            pdf.reindex(columns=columns2).sort_index(), kdf.reindex(columns=columns2).sort_index(),
+        )
+
+        self.assertRaises(TypeError, lambda: kdf.reindex(columns=["X"]))
+        self.assertRaises(ValueError, lambda: kdf.reindex(columns=[("X",)]))
+
+    def test_reindex_like(self):
+        data = [[1.0, 2.0], [3.0, None], [None, 4.0]]
+        index = pd.Index(["A", "B", "C"], name="index")
+        columns = pd.Index(["numbers", "values"], name="cols")
+        pdf = pd.DataFrame(data=data, index=index, columns=columns)
+        kdf = ps.from_pandas(pdf)
+
+        # Reindexing single Index on single Index
+        data2 = [[5.0, None], [6.0, 7.0], [8.0, None]]
+        index2 = pd.Index(["A", "C", "D"], name="index2")
+        columns2 = pd.Index(["numbers", "F"], name="cols2")
+        pdf2 = pd.DataFrame(data=data2, index=index2, columns=columns2)
+        kdf2 = ps.from_pandas(pdf2)
+
+        self.assert_eq(
+            pdf.reindex_like(pdf2).sort_index(), kdf.reindex_like(kdf2).sort_index(),
+        )
+
+        pdf2 = pd.DataFrame({"index_level_1": ["A", "C", "I"]})
+        kdf2 = ps.from_pandas(pdf2)
+
+        self.assert_eq(
+            pdf.reindex_like(pdf2.set_index(["index_level_1"])).sort_index(),
+            kdf.reindex_like(kdf2.set_index(["index_level_1"])).sort_index(),
+        )
+
+        # Reindexing MultiIndex on single Index
+        index2 = pd.MultiIndex.from_tuples(
+            [("A", "G"), ("C", "D"), ("I", "J")], names=["name3", "name4"]
+        )
+        pdf2 = pd.DataFrame(data=data2, index=index2)
+        kdf2 = ps.from_pandas(pdf2)
+
+        self.assert_eq(
+            pdf.reindex_like(pdf2).sort_index(), kdf.reindex_like(kdf2).sort_index(),
+        )
+
+        self.assertRaises(TypeError, lambda: kdf.reindex_like(index2))
+        self.assertRaises(AssertionError, lambda: kdf2.reindex_like(kdf))
+
+        # Reindexing MultiIndex on MultiIndex
+        columns2 = pd.MultiIndex.from_tuples(
+            [("numbers", "third"), ("values", "second")], names=["cols3", "cols4"]
+        )
+        pdf2.columns = columns2
+        kdf2.columns = columns2
+
+        columns = pd.MultiIndex.from_tuples(
+            [("numbers", "first"), ("values", "second")], names=["cols1", "cols2"]
+        )
+        index = pd.MultiIndex.from_tuples(
+            [("A", "B"), ("C", "D"), ("E", "F")], names=["name1", "name2"]
+        )
+        pdf = pd.DataFrame(data=data, index=index, columns=columns)
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            pdf.reindex_like(pdf2).sort_index(), kdf.reindex_like(kdf2).sort_index(),
+        )
+
+    def test_melt(self):
+        pdf = pd.DataFrame(
+            {"A": [1, 3, 5], "B": [2, 4, 6], "C": [7, 8, 9]}, index=np.random.rand(3)
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            kdf.melt().sort_values(["variable", "value"]).reset_index(drop=True),
+            pdf.melt().sort_values(["variable", "value"]),
+        )
+        self.assert_eq(
+            kdf.melt(id_vars="A").sort_values(["variable", "value"]).reset_index(drop=True),
+            pdf.melt(id_vars="A").sort_values(["variable", "value"]),
+        )
+        self.assert_eq(
+            kdf.melt(id_vars=["A", "B"]).sort_values(["variable", "value"]).reset_index(drop=True),
+            pdf.melt(id_vars=["A", "B"]).sort_values(["variable", "value"]),
+        )
+        self.assert_eq(
+            kdf.melt(id_vars=("A", "B")).sort_values(["variable", "value"]).reset_index(drop=True),
+            pdf.melt(id_vars=("A", "B")).sort_values(["variable", "value"]),
+        )
+        self.assert_eq(
+            kdf.melt(id_vars=["A"], value_vars=["C"])
+            .sort_values(["variable", "value"])
+            .reset_index(drop=True),
+            pdf.melt(id_vars=["A"], value_vars=["C"]).sort_values(["variable", "value"]),
+        )
+        self.assert_eq(
+            kdf.melt(id_vars=["A"], value_vars=["B"], var_name="myVarname", value_name="myValname")
+            .sort_values(["myVarname", "myValname"])
+            .reset_index(drop=True),
+            pdf.melt(
+                id_vars=["A"], value_vars=["B"], var_name="myVarname", value_name="myValname"
+            ).sort_values(["myVarname", "myValname"]),
+        )
+        self.assert_eq(
+            kdf.melt(value_vars=("A", "B"))
+            .sort_values(["variable", "value"])
+            .reset_index(drop=True),
+            pdf.melt(value_vars=("A", "B")).sort_values(["variable", "value"]),
+        )
+
+        self.assertRaises(KeyError, lambda: kdf.melt(id_vars="Z"))
+        self.assertRaises(KeyError, lambda: kdf.melt(value_vars="Z"))
+
+        # multi-index columns
+        if LooseVersion("0.24") <= LooseVersion(pd.__version__) < LooseVersion("1.0.0"):
+            # pandas >=0.24,<1.0 doesn't support mixed int/str columns in melt.
+            # see: https://github.com/pandas-dev/pandas/pull/29792
+            TEN = "10"
+            TWELVE = "20"
+        else:
+            TEN = 10.0
+            TWELVE = 20.0
+
+        columns = pd.MultiIndex.from_tuples([(TEN, "A"), (TEN, "B"), (TWELVE, "C")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(
+            kdf.melt().sort_values(["variable_0", "variable_1", "value"]).reset_index(drop=True),
+            pdf.melt().sort_values(["variable_0", "variable_1", "value"]),
+        )
+        self.assert_eq(
+            kdf.melt(id_vars=[(TEN, "A")])
+            .sort_values(["variable_0", "variable_1", "value"])
+            .reset_index(drop=True),
+            pdf.melt(id_vars=[(TEN, "A")])
+            .sort_values(["variable_0", "variable_1", "value"])
+            .rename(columns=name_like_string),
+        )
+        self.assert_eq(
+            kdf.melt(id_vars=[(TEN, "A")], value_vars=[(TWELVE, "C")])
+            .sort_values(["variable_0", "variable_1", "value"])
+            .reset_index(drop=True),
+            pdf.melt(id_vars=[(TEN, "A")], value_vars=[(TWELVE, "C")])
+            .sort_values(["variable_0", "variable_1", "value"])
+            .rename(columns=name_like_string),
+        )
+        self.assert_eq(
+            kdf.melt(
+                id_vars=[(TEN, "A")],
+                value_vars=[(TEN, "B")],
+                var_name=["myV1", "myV2"],
+                value_name="myValname",
+            )
+            .sort_values(["myV1", "myV2", "myValname"])
+            .reset_index(drop=True),
+            pdf.melt(
+                id_vars=[(TEN, "A")],
+                value_vars=[(TEN, "B")],
+                var_name=["myV1", "myV2"],
+                value_name="myValname",
+            )
+            .sort_values(["myV1", "myV2", "myValname"])
+            .rename(columns=name_like_string),
+        )
+
+        columns.names = ["v0", "v1"]
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(
+            kdf.melt().sort_values(["v0", "v1", "value"]).reset_index(drop=True),
+            pdf.melt().sort_values(["v0", "v1", "value"]),
+        )
+
+        self.assertRaises(ValueError, lambda: kdf.melt(id_vars=(TEN, "A")))
+        self.assertRaises(ValueError, lambda: kdf.melt(value_vars=(TEN, "A")))
+        self.assertRaises(KeyError, lambda: kdf.melt(id_vars=[TEN]))
+        self.assertRaises(KeyError, lambda: kdf.melt(id_vars=[(TWELVE, "A")]))
+        self.assertRaises(KeyError, lambda: kdf.melt(value_vars=[TWELVE]))
+        self.assertRaises(KeyError, lambda: kdf.melt(value_vars=[(TWELVE, "A")]))
+
+        # non-string names
+        pdf.columns = [10.0, 20.0, 30.0]
+        kdf.columns = [10.0, 20.0, 30.0]
+
+        self.assert_eq(
+            kdf.melt().sort_values(["variable", "value"]).reset_index(drop=True),
+            pdf.melt().sort_values(["variable", "value"]),
+        )
+        self.assert_eq(
+            kdf.melt(id_vars=10.0).sort_values(["variable", "value"]).reset_index(drop=True),
+            pdf.melt(id_vars=10.0).sort_values(["variable", "value"]),
+        )
+        self.assert_eq(
+            kdf.melt(id_vars=[10.0, 20.0])
+            .sort_values(["variable", "value"])
+            .reset_index(drop=True),
+            pdf.melt(id_vars=[10.0, 20.0]).sort_values(["variable", "value"]),
+        )
+        self.assert_eq(
+            kdf.melt(id_vars=(10.0, 20.0))
+            .sort_values(["variable", "value"])
+            .reset_index(drop=True),
+            pdf.melt(id_vars=(10.0, 20.0)).sort_values(["variable", "value"]),
+        )
+        self.assert_eq(
+            kdf.melt(id_vars=[10.0], value_vars=[30.0])
+            .sort_values(["variable", "value"])
+            .reset_index(drop=True),
+            pdf.melt(id_vars=[10.0], value_vars=[30.0]).sort_values(["variable", "value"]),
+        )
+        self.assert_eq(
+            kdf.melt(value_vars=(10.0, 20.0))
+            .sort_values(["variable", "value"])
+            .reset_index(drop=True),
+            pdf.melt(value_vars=(10.0, 20.0)).sort_values(["variable", "value"]),
+        )
+
+    def test_all(self):
+        pdf = pd.DataFrame(
+            {
+                "col1": [False, False, False],
+                "col2": [True, False, False],
+                "col3": [0, 0, 1],
+                "col4": [0, 1, 2],
+                "col5": [False, False, None],
+                "col6": [True, False, None],
+            },
+            index=np.random.rand(3),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.all(), pdf.all())
+
+        columns = pd.MultiIndex.from_tuples(
+            [
+                ("a", "col1"),
+                ("a", "col2"),
+                ("a", "col3"),
+                ("b", "col4"),
+                ("b", "col5"),
+                ("c", "col6"),
+            ]
+        )
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(kdf.all(), pdf.all())
+
+        columns.names = ["X", "Y"]
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(kdf.all(), pdf.all())
+
+        with self.assertRaisesRegex(
+            NotImplementedError, 'axis should be either 0 or "index" currently.'
+        ):
+            kdf.all(axis=1)
+
+    def test_any(self):
+        pdf = pd.DataFrame(
+            {
+                "col1": [False, False, False],
+                "col2": [True, False, False],
+                "col3": [0, 0, 1],
+                "col4": [0, 1, 2],
+                "col5": [False, False, None],
+                "col6": [True, False, None],
+            },
+            index=np.random.rand(3),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.any(), pdf.any())
+
+        columns = pd.MultiIndex.from_tuples(
+            [
+                ("a", "col1"),
+                ("a", "col2"),
+                ("a", "col3"),
+                ("b", "col4"),
+                ("b", "col5"),
+                ("c", "col6"),
+            ]
+        )
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(kdf.any(), pdf.any())
+
+        columns.names = ["X", "Y"]
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(kdf.any(), pdf.any())
+
+        with self.assertRaisesRegex(
+            NotImplementedError, 'axis should be either 0 or "index" currently.'
+        ):
+            kdf.any(axis=1)
+
+    def test_rank(self):
+        pdf = pd.DataFrame(
+            data={"col1": [1, 2, 3, 1], "col2": [3, 4, 3, 1]},
+            columns=["col1", "col2"],
+            index=np.random.rand(4),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(pdf.rank().sort_index(), kdf.rank().sort_index())
+        self.assert_eq(pdf.rank().sum(), kdf.rank().sum())
+        self.assert_eq(
+            pdf.rank(ascending=False).sort_index(), kdf.rank(ascending=False).sort_index()
+        )
+        self.assert_eq(pdf.rank(method="min").sort_index(), kdf.rank(method="min").sort_index())
+        self.assert_eq(pdf.rank(method="max").sort_index(), kdf.rank(method="max").sort_index())
+        self.assert_eq(pdf.rank(method="first").sort_index(), kdf.rank(method="first").sort_index())
+        self.assert_eq(pdf.rank(method="dense").sort_index(), kdf.rank(method="dense").sort_index())
+
+        msg = "method must be one of 'average', 'min', 'max', 'first', 'dense'"
+        with self.assertRaisesRegex(ValueError, msg):
+            kdf.rank(method="nothing")
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "col1"), ("y", "col2")])
+        pdf.columns = columns
+        kdf.columns = columns
+        self.assert_eq(pdf.rank().sort_index(), kdf.rank().sort_index())
+
+    def test_round(self):
+        pdf = pd.DataFrame(
+            {
+                "A": [0.028208, 0.038683, 0.877076],
+                "B": [0.992815, 0.645646, 0.149370],
+                "C": [0.173891, 0.577595, 0.491027],
+            },
+            columns=["A", "B", "C"],
+            index=np.random.rand(3),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        pser = pd.Series([1, 0, 2], index=["A", "B", "C"])
+        kser = ps.Series([1, 0, 2], index=["A", "B", "C"])
+        self.assert_eq(pdf.round(2), kdf.round(2))
+        self.assert_eq(pdf.round({"A": 1, "C": 2}), kdf.round({"A": 1, "C": 2}))
+        self.assert_eq(pdf.round({"A": 1, "D": 2}), kdf.round({"A": 1, "D": 2}))
+        self.assert_eq(pdf.round(pser), kdf.round(kser))
+        msg = "decimals must be an integer, a dict-like or a Series"
+        with self.assertRaisesRegex(TypeError, msg):
+            kdf.round(1.5)
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("X", "A"), ("X", "B"), ("Y", "C")])
+        pdf.columns = columns
+        kdf.columns = columns
+        pser = pd.Series([1, 0, 2], index=columns)
+        kser = ps.Series([1, 0, 2], index=columns)
+        self.assert_eq(pdf.round(2), kdf.round(2))
+        self.assert_eq(
+            pdf.round({("X", "A"): 1, ("Y", "C"): 2}), kdf.round({("X", "A"): 1, ("Y", "C"): 2})
+        )
+        self.assert_eq(pdf.round({("X", "A"): 1, "Y": 2}), kdf.round({("X", "A"): 1, "Y": 2}))
+        self.assert_eq(pdf.round(pser), kdf.round(kser))
+
+        # non-string names
+        pdf = pd.DataFrame(
+            {
+                10: [0.028208, 0.038683, 0.877076],
+                20: [0.992815, 0.645646, 0.149370],
+                30: [0.173891, 0.577595, 0.491027],
+            },
+            index=np.random.rand(3),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(pdf.round({10: 1, 30: 2}), kdf.round({10: 1, 30: 2}))
+
+    def test_shift(self):
+        pdf = pd.DataFrame(
+            {
+                "Col1": [10, 20, 15, 30, 45],
+                "Col2": [13, 23, 18, 33, 48],
+                "Col3": [17, 27, 22, 37, 52],
+            },
+            index=np.random.rand(5),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(pdf.shift(3), kdf.shift(3))
+        self.assert_eq(pdf.shift().shift(-1), kdf.shift().shift(-1))
+        self.assert_eq(pdf.shift().sum().astype(int), kdf.shift().sum())
+
+        # Need the expected result since pandas 0.23 does not support `fill_value` argument.
+        pdf1 = pd.DataFrame(
+            {"Col1": [0, 0, 0, 10, 20], "Col2": [0, 0, 0, 13, 23], "Col3": [0, 0, 0, 17, 27]},
+            index=pdf.index,
+        )
+        self.assert_eq(pdf1, kdf.shift(periods=3, fill_value=0))
+        msg = "should be an int"
+        with self.assertRaisesRegex(TypeError, msg):
+            kdf.shift(1.5)
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "Col1"), ("x", "Col2"), ("y", "Col3")])
+        pdf.columns = columns
+        kdf.columns = columns
+        self.assert_eq(pdf.shift(3), kdf.shift(3))
+        self.assert_eq(pdf.shift().shift(-1), kdf.shift().shift(-1))
+
+    def test_diff(self):
+        pdf = pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5, 6], "b": [1, 1, 2, 3, 5, 8], "c": [1, 4, 9, 16, 25, 36]},
+            index=np.random.rand(6),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(pdf.diff(), kdf.diff())
+        self.assert_eq(pdf.diff().diff(-1), kdf.diff().diff(-1))
+        self.assert_eq(pdf.diff().sum().astype(int), kdf.diff().sum())
+
+        msg = "should be an int"
+        with self.assertRaisesRegex(TypeError, msg):
+            kdf.diff(1.5)
+        msg = 'axis should be either 0 or "index" currently.'
+        with self.assertRaisesRegex(NotImplementedError, msg):
+            kdf.diff(axis=1)
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "Col1"), ("x", "Col2"), ("y", "Col3")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(pdf.diff(), kdf.diff())
+
+    def test_duplicated(self):
+        pdf = pd.DataFrame(
+            {"a": [1, 1, 2, 3], "b": [1, 1, 1, 4], "c": [1, 1, 1, 5]}, index=np.random.rand(4)
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(pdf.duplicated().sort_index(), kdf.duplicated().sort_index())
+        self.assert_eq(
+            pdf.duplicated(keep="last").sort_index(), kdf.duplicated(keep="last").sort_index(),
+        )
+        self.assert_eq(
+            pdf.duplicated(keep=False).sort_index(), kdf.duplicated(keep=False).sort_index(),
+        )
+        self.assert_eq(
+            pdf.duplicated(subset="b").sort_index(), kdf.duplicated(subset="b").sort_index(),
+        )
+        self.assert_eq(
+            pdf.duplicated(subset=["b"]).sort_index(), kdf.duplicated(subset=["b"]).sort_index(),
+        )
+        with self.assertRaisesRegex(ValueError, "'keep' only supports 'first', 'last' and False"):
+            kdf.duplicated(keep="false")
+        with self.assertRaisesRegex(KeyError, "'d'"):
+            kdf.duplicated(subset=["d"])
+
+        pdf.index.name = "x"
+        kdf.index.name = "x"
+        self.assert_eq(pdf.duplicated().sort_index(), kdf.duplicated().sort_index())
+
+        # multi-index
+        self.assert_eq(
+            pdf.set_index("a", append=True).duplicated().sort_index(),
+            kdf.set_index("a", append=True).duplicated().sort_index(),
+        )
+        self.assert_eq(
+            pdf.set_index("a", append=True).duplicated(keep=False).sort_index(),
+            kdf.set_index("a", append=True).duplicated(keep=False).sort_index(),
+        )
+        self.assert_eq(
+            pdf.set_index("a", append=True).duplicated(subset=["b"]).sort_index(),
+            kdf.set_index("a", append=True).duplicated(subset=["b"]).sort_index(),
+        )
+
+        # mutli-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "c")])
+        pdf.columns = columns
+        kdf.columns = columns
+        self.assert_eq(pdf.duplicated().sort_index(), kdf.duplicated().sort_index())
+        self.assert_eq(
+            pdf.duplicated(subset=("x", "b")).sort_index(),
+            kdf.duplicated(subset=("x", "b")).sort_index(),
+        )
+        self.assert_eq(
+            pdf.duplicated(subset=[("x", "b")]).sort_index(),
+            kdf.duplicated(subset=[("x", "b")]).sort_index(),
+        )
+
+        # non-string names
+        pdf = pd.DataFrame(
+            {10: [1, 1, 2, 3], 20: [1, 1, 1, 4], 30: [1, 1, 1, 5]}, index=np.random.rand(4)
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(pdf.duplicated().sort_index(), kdf.duplicated().sort_index())
+        self.assert_eq(
+            pdf.duplicated(subset=10).sort_index(), kdf.duplicated(subset=10).sort_index(),
+        )
+
+    def test_ffill(self):
+        idx = np.random.rand(6)
+        pdf = pd.DataFrame(
+            {
+                "x": [np.nan, 2, 3, 4, np.nan, 6],
+                "y": [1, 2, np.nan, 4, np.nan, np.nan],
+                "z": [1, 2, 3, 4, np.nan, np.nan],
+            },
+            index=idx,
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.ffill(), pdf.ffill())
+        self.assert_eq(kdf.ffill(limit=1), pdf.ffill(limit=1))
+
+        pser = pdf.y
+        kser = kdf.y
+
+        kdf.ffill(inplace=True)
+        pdf.ffill(inplace=True)
+
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kser, pser)
+        self.assert_eq(kser[idx[2]], pser[idx[2]])
+
+    def test_bfill(self):
+        idx = np.random.rand(6)
+        pdf = pd.DataFrame(
+            {
+                "x": [np.nan, 2, 3, 4, np.nan, 6],
+                "y": [1, 2, np.nan, 4, np.nan, np.nan],
+                "z": [1, 2, 3, 4, np.nan, np.nan],
+            },
+            index=idx,
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.bfill(), pdf.bfill())
+        self.assert_eq(kdf.bfill(limit=1), pdf.bfill(limit=1))
+
+        pser = pdf.x
+        kser = kdf.x
+
+        kdf.bfill(inplace=True)
+        pdf.bfill(inplace=True)
+
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kser, pser)
+        self.assert_eq(kser[idx[0]], pser[idx[0]])
+
+    def test_filter(self):
+        pdf = pd.DataFrame(
+            {
+                "aa": ["aa", "bd", "bc", "ab", "ce"],
+                "ba": [1, 2, 3, 4, 5],
+                "cb": [1.0, 2.0, 3.0, 4.0, 5.0],
+                "db": [1.0, np.nan, 3.0, np.nan, 5.0],
+            }
+        )
+        pdf = pdf.set_index("aa")
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            kdf.filter(items=["ab", "aa"], axis=0).sort_index(),
+            pdf.filter(items=["ab", "aa"], axis=0).sort_index(),
+        )
+        self.assert_eq(
+            kdf.filter(items=["ba", "db"], axis=1).sort_index(),
+            pdf.filter(items=["ba", "db"], axis=1).sort_index(),
+        )
+
+        self.assert_eq(kdf.filter(like="b", axis="index"), pdf.filter(like="b", axis="index"))
+        self.assert_eq(kdf.filter(like="c", axis="columns"), pdf.filter(like="c", axis="columns"))
+
+        self.assert_eq(kdf.filter(regex="b.*", axis="index"), pdf.filter(regex="b.*", axis="index"))
+        self.assert_eq(
+            kdf.filter(regex="b.*", axis="columns"), pdf.filter(regex="b.*", axis="columns")
+        )
+
+        pdf = pdf.set_index("ba", append=True)
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            kdf.filter(items=[("aa", 1), ("bd", 2)], axis=0).sort_index(),
+            pdf.filter(items=[("aa", 1), ("bd", 2)], axis=0).sort_index(),
+        )
+
+        with self.assertRaisesRegex(TypeError, "Unsupported type list"):
+            kdf.filter(items=[["aa", 1], ("bd", 2)], axis=0)
+
+        with self.assertRaisesRegex(ValueError, "The item should not be empty."):
+            kdf.filter(items=[(), ("bd", 2)], axis=0)
+
+        self.assert_eq(kdf.filter(like="b", axis=0), pdf.filter(like="b", axis=0))
+
+        self.assert_eq(kdf.filter(regex="b.*", axis=0), pdf.filter(regex="b.*", axis=0))
+
+        with self.assertRaisesRegex(ValueError, "items should be a list-like object"):
+            kdf.filter(items="b")
+
+        with self.assertRaisesRegex(ValueError, "No axis named"):
+            kdf.filter(regex="b.*", axis=123)
+
+        with self.assertRaisesRegex(TypeError, "Must pass either `items`, `like`"):
+            kdf.filter()
+
+        with self.assertRaisesRegex(TypeError, "mutually exclusive"):
+            kdf.filter(regex="b.*", like="aaa")
+
+        # multi-index columns
+        pdf = pd.DataFrame(
+            {
+                ("x", "aa"): ["aa", "ab", "bc", "bd", "ce"],
+                ("x", "ba"): [1, 2, 3, 4, 5],
+                ("y", "cb"): [1.0, 2.0, 3.0, 4.0, 5.0],
+                ("z", "db"): [1.0, np.nan, 3.0, np.nan, 5.0],
+            }
+        )
+        pdf = pdf.set_index(("x", "aa"))
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            kdf.filter(items=["ab", "aa"], axis=0).sort_index(),
+            pdf.filter(items=["ab", "aa"], axis=0).sort_index(),
+        )
+        self.assert_eq(
+            kdf.filter(items=[("x", "ba"), ("z", "db")], axis=1).sort_index(),
+            pdf.filter(items=[("x", "ba"), ("z", "db")], axis=1).sort_index(),
+        )
+
+        self.assert_eq(kdf.filter(like="b", axis="index"), pdf.filter(like="b", axis="index"))
+        self.assert_eq(kdf.filter(like="c", axis="columns"), pdf.filter(like="c", axis="columns"))
+
+        self.assert_eq(kdf.filter(regex="b.*", axis="index"), pdf.filter(regex="b.*", axis="index"))
+        self.assert_eq(
+            kdf.filter(regex="b.*", axis="columns"), pdf.filter(regex="b.*", axis="columns")
+        )
+
+    def test_pipe(self):
+        kdf = ps.DataFrame(
+            {"category": ["A", "A", "B"], "col1": [1, 2, 3], "col2": [4, 5, 6]},
+            columns=["category", "col1", "col2"],
+        )
+
+        self.assertRaisesRegex(
+            ValueError,
+            "arg is both the pipe target and a keyword argument",
+            lambda: kdf.pipe((lambda x: x, "arg"), arg="1"),
+        )
+
+    def test_transform(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 2, 3, 4, 5, 6] * 100,
+                "b": [1.0, 1.0, 2.0, 3.0, 5.0, 8.0] * 100,
+                "c": [1, 4, 9, 16, 25, 36] * 100,
+            },
+            columns=["a", "b", "c"],
+            index=np.random.rand(600),
+        )
+        kdf = ps.DataFrame(pdf)
+        self.assert_eq(
+            kdf.transform(lambda x: x + 1).sort_index(), pdf.transform(lambda x: x + 1).sort_index()
+        )
+        self.assert_eq(
+            kdf.transform(lambda x, y: x + y, y=2).sort_index(),
+            pdf.transform(lambda x, y: x + y, y=2).sort_index(),
+        )
+        with option_context("compute.shortcut_limit", 500):
+            self.assert_eq(
+                kdf.transform(lambda x: x + 1).sort_index(),
+                pdf.transform(lambda x: x + 1).sort_index(),
+            )
+            self.assert_eq(
+                kdf.transform(lambda x, y: x + y, y=1).sort_index(),
+                pdf.transform(lambda x, y: x + y, y=1).sort_index(),
+            )
+
+        with self.assertRaisesRegex(AssertionError, "the first argument should be a callable"):
+            kdf.transform(1)
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "c")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(
+            kdf.transform(lambda x: x + 1).sort_index(), pdf.transform(lambda x: x + 1).sort_index()
+        )
+        with option_context("compute.shortcut_limit", 500):
+            self.assert_eq(
+                kdf.transform(lambda x: x + 1).sort_index(),
+                pdf.transform(lambda x: x + 1).sort_index(),
+            )
+
+    def test_apply(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 2, 3, 4, 5, 6] * 100,
+                "b": [1.0, 1.0, 2.0, 3.0, 5.0, 8.0] * 100,
+                "c": [1, 4, 9, 16, 25, 36] * 100,
+            },
+            columns=["a", "b", "c"],
+            index=np.random.rand(600),
+        )
+        kdf = ps.DataFrame(pdf)
+
+        self.assert_eq(
+            kdf.apply(lambda x: x + 1).sort_index(), pdf.apply(lambda x: x + 1).sort_index()
+        )
+        self.assert_eq(
+            kdf.apply(lambda x, b: x + b, args=(1,)).sort_index(),
+            pdf.apply(lambda x, b: x + b, args=(1,)).sort_index(),
+        )
+        self.assert_eq(
+            kdf.apply(lambda x, b: x + b, b=1).sort_index(),
+            pdf.apply(lambda x, b: x + b, b=1).sort_index(),
+        )
+
+        with option_context("compute.shortcut_limit", 500):
+            self.assert_eq(
+                kdf.apply(lambda x: x + 1).sort_index(), pdf.apply(lambda x: x + 1).sort_index()
+            )
+            self.assert_eq(
+                kdf.apply(lambda x, b: x + b, args=(1,)).sort_index(),
+                pdf.apply(lambda x, b: x + b, args=(1,)).sort_index(),
+            )
+            self.assert_eq(
+                kdf.apply(lambda x, b: x + b, b=1).sort_index(),
+                pdf.apply(lambda x, b: x + b, b=1).sort_index(),
+            )
+
+        # returning a Series
+        self.assert_eq(
+            kdf.apply(lambda x: len(x), axis=1).sort_index(),
+            pdf.apply(lambda x: len(x), axis=1).sort_index(),
+        )
+        self.assert_eq(
+            kdf.apply(lambda x, c: len(x) + c, axis=1, c=100).sort_index(),
+            pdf.apply(lambda x, c: len(x) + c, axis=1, c=100).sort_index(),
+        )
+        with option_context("compute.shortcut_limit", 500):
+            self.assert_eq(
+                kdf.apply(lambda x: len(x), axis=1).sort_index(),
+                pdf.apply(lambda x: len(x), axis=1).sort_index(),
+            )
+            self.assert_eq(
+                kdf.apply(lambda x, c: len(x) + c, axis=1, c=100).sort_index(),
+                pdf.apply(lambda x, c: len(x) + c, axis=1, c=100).sort_index(),
+            )
+
+        with self.assertRaisesRegex(AssertionError, "the first argument should be a callable"):
+            kdf.apply(1)
+
+        with self.assertRaisesRegex(TypeError, "The given function.*1 or 'column'; however"):
+
+            def f1(_) -> ps.DataFrame[int]:
+                pass
+
+            kdf.apply(f1, axis=0)
+
+        with self.assertRaisesRegex(TypeError, "The given function.*0 or 'index'; however"):
+
+            def f2(_) -> ps.Series[int]:
+                pass
+
+            kdf.apply(f2, axis=1)
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "c")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(
+            kdf.apply(lambda x: x + 1).sort_index(), pdf.apply(lambda x: x + 1).sort_index()
+        )
+        with option_context("compute.shortcut_limit", 500):
+            self.assert_eq(
+                kdf.apply(lambda x: x + 1).sort_index(), pdf.apply(lambda x: x + 1).sort_index()
+            )
+
+        # returning a Series
+        self.assert_eq(
+            kdf.apply(lambda x: len(x), axis=1).sort_index(),
+            pdf.apply(lambda x: len(x), axis=1).sort_index(),
+        )
+        with option_context("compute.shortcut_limit", 500):
+            self.assert_eq(
+                kdf.apply(lambda x: len(x), axis=1).sort_index(),
+                pdf.apply(lambda x: len(x), axis=1).sort_index(),
+            )
+
+    def test_apply_batch(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 2, 3, 4, 5, 6] * 100,
+                "b": [1.0, 1.0, 2.0, 3.0, 5.0, 8.0] * 100,
+                "c": [1, 4, 9, 16, 25, 36] * 100,
+            },
+            columns=["a", "b", "c"],
+            index=np.random.rand(600),
+        )
+        kdf = ps.DataFrame(pdf)
+
+        # One to test alias.
+        self.assert_eq(kdf.apply_batch(lambda pdf: pdf + 1).sort_index(), (pdf + 1).sort_index())
+        self.assert_eq(
+            kdf.koalas.apply_batch(lambda pdf, a: pdf + a, args=(1,)).sort_index(),
+            (pdf + 1).sort_index(),
+        )
+        with option_context("compute.shortcut_limit", 500):
+            self.assert_eq(
+                kdf.koalas.apply_batch(lambda pdf: pdf + 1).sort_index(), (pdf + 1).sort_index()
+            )
+            self.assert_eq(
+                kdf.koalas.apply_batch(lambda pdf, b: pdf + b, b=1).sort_index(),
+                (pdf + 1).sort_index(),
+            )
+
+        with self.assertRaisesRegex(AssertionError, "the first argument should be a callable"):
+            kdf.koalas.apply_batch(1)
+
+        with self.assertRaisesRegex(TypeError, "The given function.*frame as its type hints"):
+
+            def f2(_) -> ps.Series[int]:
+                pass
+
+            kdf.koalas.apply_batch(f2)
+
+        with self.assertRaisesRegex(ValueError, "The given function should return a frame"):
+            kdf.koalas.apply_batch(lambda pdf: 1)
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "c")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(kdf.koalas.apply_batch(lambda x: x + 1).sort_index(), (pdf + 1).sort_index())
+        with option_context("compute.shortcut_limit", 500):
+            self.assert_eq(
+                kdf.koalas.apply_batch(lambda x: x + 1).sort_index(), (pdf + 1).sort_index()
+            )
+
+    def test_transform_batch(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 2, 3, 4, 5, 6] * 100,
+                "b": [1.0, 1.0, 2.0, 3.0, 5.0, 8.0] * 100,
+                "c": [1, 4, 9, 16, 25, 36] * 100,
+            },
+            columns=["a", "b", "c"],
+            index=np.random.rand(600),
+        )
+        kdf = ps.DataFrame(pdf)
+
+        # One to test alias.
+        self.assert_eq(
+            kdf.transform_batch(lambda pdf: pdf + 1).sort_index(), (pdf + 1).sort_index()
+        )
+        self.assert_eq(
+            kdf.koalas.transform_batch(lambda pdf: pdf.c + 1).sort_index(), (pdf.c + 1).sort_index()
+        )
+        self.assert_eq(
+            kdf.koalas.transform_batch(lambda pdf, a: pdf + a, 1).sort_index(),
+            (pdf + 1).sort_index(),
+        )
+        self.assert_eq(
+            kdf.koalas.transform_batch(lambda pdf, a: pdf.c + a, a=1).sort_index(),
+            (pdf.c + 1).sort_index(),
+        )
+
+        with option_context("compute.shortcut_limit", 500):
+            self.assert_eq(
+                kdf.koalas.transform_batch(lambda pdf: pdf + 1).sort_index(), (pdf + 1).sort_index()
+            )
+            self.assert_eq(
+                kdf.koalas.transform_batch(lambda pdf: pdf.b + 1).sort_index(),
+                (pdf.b + 1).sort_index(),
+            )
+            self.assert_eq(
+                kdf.koalas.transform_batch(lambda pdf, a: pdf + a, 1).sort_index(),
+                (pdf + 1).sort_index(),
+            )
+            self.assert_eq(
+                kdf.koalas.transform_batch(lambda pdf, a: pdf.c + a, a=1).sort_index(),
+                (pdf.c + 1).sort_index(),
+            )
+
+        with self.assertRaisesRegex(AssertionError, "the first argument should be a callable"):
+            kdf.koalas.transform_batch(1)
+
+        with self.assertRaisesRegex(ValueError, "The given function should return a frame"):
+            kdf.koalas.transform_batch(lambda pdf: 1)
+
+        with self.assertRaisesRegex(
+            ValueError, "transform_batch cannot produce aggregated results"
+        ):
+            kdf.koalas.transform_batch(lambda pdf: pd.Series(1))
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "c")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(
+            kdf.koalas.transform_batch(lambda x: x + 1).sort_index(), (pdf + 1).sort_index()
+        )
+        with option_context("compute.shortcut_limit", 500):
+            self.assert_eq(
+                kdf.koalas.transform_batch(lambda x: x + 1).sort_index(), (pdf + 1).sort_index()
+            )
+
+    def test_transform_batch_same_anchor(self):
+        kdf = ps.range(10)
+        kdf["d"] = kdf.koalas.transform_batch(lambda pdf: pdf.id + 1)
+        self.assert_eq(
+            kdf, pd.DataFrame({"id": list(range(10)), "d": list(range(1, 11))}, columns=["id", "d"])
+        )
+
+        kdf = ps.range(10)
+        # One to test alias.
+        kdf["d"] = kdf.id.transform_batch(lambda ser: ser + 1)
+        self.assert_eq(
+            kdf, pd.DataFrame({"id": list(range(10)), "d": list(range(1, 11))}, columns=["id", "d"])
+        )
+
+        kdf = ps.range(10)
+
+        def plus_one(pdf) -> ps.Series[np.int64]:
+            return pdf.id + 1
+
+        kdf["d"] = kdf.koalas.transform_batch(plus_one)
+        self.assert_eq(
+            kdf, pd.DataFrame({"id": list(range(10)), "d": list(range(1, 11))}, columns=["id", "d"])
+        )
+
+        kdf = ps.range(10)
+
+        def plus_one(ser) -> ps.Series[np.int64]:
+            return ser + 1
+
+        kdf["d"] = kdf.id.koalas.transform_batch(plus_one)
+        self.assert_eq(
+            kdf, pd.DataFrame({"id": list(range(10)), "d": list(range(1, 11))}, columns=["id", "d"])
+        )
+
+    def test_empty_timestamp(self):
+        pdf = pd.DataFrame(
+            {
+                "t": [
+                    datetime(2019, 1, 1, 0, 0, 0),
+                    datetime(2019, 1, 2, 0, 0, 0),
+                    datetime(2019, 1, 3, 0, 0, 0),
+                ]
+            },
+            index=np.random.rand(3),
+        )
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kdf[kdf["t"] != kdf["t"]], pdf[pdf["t"] != pdf["t"]])
+        self.assert_eq(kdf[kdf["t"] != kdf["t"]].dtypes, pdf[pdf["t"] != pdf["t"]].dtypes)
+
+    def test_to_spark(self):
+        kdf = ps.from_pandas(self.pdf)
+
+        with self.assertRaisesRegex(ValueError, "'index_col' cannot be overlapped"):
+            kdf.to_spark(index_col="a")
+
+        with self.assertRaisesRegex(ValueError, "length of index columns.*1.*3"):
+            kdf.to_spark(index_col=["x", "y", "z"])
+
+    def test_keys(self):
+        pdf = pd.DataFrame(
+            [[1, 2], [4, 5], [7, 8]],
+            index=["cobra", "viper", "sidewinder"],
+            columns=["max_speed", "shield"],
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.keys(), pdf.keys())
+
+    def test_quantile(self):
+        pdf, kdf = self.df_pair
+
+        self.assert_eq(kdf.quantile(0.5), pdf.quantile(0.5))
+        self.assert_eq(kdf.quantile([0.25, 0.5, 0.75]), pdf.quantile([0.25, 0.5, 0.75]))
+
+        self.assert_eq(kdf.loc[[]].quantile(0.5), pdf.loc[[]].quantile(0.5))
+        self.assert_eq(
+            kdf.loc[[]].quantile([0.25, 0.5, 0.75]), pdf.loc[[]].quantile([0.25, 0.5, 0.75])
+        )
+
+        with self.assertRaisesRegex(
+            NotImplementedError, 'axis should be either 0 or "index" currently.'
+        ):
+            kdf.quantile(0.5, axis=1)
+        with self.assertRaisesRegex(TypeError, "accuracy must be an integer; however"):
+            kdf.quantile(accuracy="a")
+        with self.assertRaisesRegex(TypeError, "q must be a float or an array of floats;"):
+            kdf.quantile(q="a")
+        with self.assertRaisesRegex(TypeError, "q must be a float or an array of floats;"):
+            kdf.quantile(q=["a"])
+
+        self.assert_eq(kdf.quantile(0.5, numeric_only=False), pdf.quantile(0.5, numeric_only=False))
+        self.assert_eq(
+            kdf.quantile([0.25, 0.5, 0.75], numeric_only=False),
+            pdf.quantile([0.25, 0.5, 0.75], numeric_only=False),
+        )
+
+        # multi-index column
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("y", "b")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(kdf.quantile(0.5), pdf.quantile(0.5))
+        self.assert_eq(kdf.quantile([0.25, 0.5, 0.75]), pdf.quantile([0.25, 0.5, 0.75]))
+
+        pdf = pd.DataFrame({"x": ["a", "b", "c"]})
+        kdf = ps.from_pandas(pdf)
+
+        if LooseVersion(pd.__version__) >= LooseVersion("1.0.0"):
+            self.assert_eq(kdf.quantile(0.5), pdf.quantile(0.5))
+            self.assert_eq(kdf.quantile([0.25, 0.5, 0.75]), pdf.quantile([0.25, 0.5, 0.75]))
+        else:
+            self.assert_eq(kdf.quantile(0.5), pd.Series(name=0.5))
+            self.assert_eq(kdf.quantile([0.25, 0.5, 0.75]), pd.DataFrame(index=[0.25, 0.5, 0.75]))
+
+        with self.assertRaisesRegex(TypeError, "Could not convert object \\(string\\) to numeric"):
+            kdf.quantile(0.5, numeric_only=False)
+        with self.assertRaisesRegex(TypeError, "Could not convert object \\(string\\) to numeric"):
+            kdf.quantile([0.25, 0.5, 0.75], numeric_only=False)
+
+    def test_pct_change(self):
+        pdf = pd.DataFrame(
+            {"a": [1, 2, 3, 2], "b": [4.0, 2.0, 3.0, 1.0], "c": [300, 200, 400, 200]},
+            index=np.random.rand(4),
+        )
+        pdf.columns = pd.MultiIndex.from_tuples([("a", "x"), ("b", "y"), ("c", "z")])
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.pct_change(2), pdf.pct_change(2), check_exact=False)
+        self.assert_eq(kdf.pct_change().sum(), pdf.pct_change().sum(), check_exact=False)
+
+    def test_where(self):
+        kdf = ps.from_pandas(self.pdf)
+
+        with self.assertRaisesRegex(TypeError, "type of cond must be a DataFrame or Series"):
+            kdf.where(1)
+
+    def test_mask(self):
+        kdf = ps.from_pandas(self.pdf)
+
+        with self.assertRaisesRegex(TypeError, "type of cond must be a DataFrame or Series"):
+            kdf.mask(1)
+
+    def test_query(self):
+        pdf = pd.DataFrame({"A": range(1, 6), "B": range(10, 0, -2), "C": range(10, 5, -1)})
+        kdf = ps.from_pandas(pdf)
+
+        exprs = ("A > B", "A < C", "C == B")
+        for expr in exprs:
+            self.assert_eq(kdf.query(expr), pdf.query(expr))
+
+        # test `inplace=True`
+        for expr in exprs:
+            dummy_kdf = kdf.copy()
+            dummy_pdf = pdf.copy()
+
+            pser = dummy_pdf.A
+            kser = dummy_kdf.A
+            dummy_pdf.query(expr, inplace=True)
+            dummy_kdf.query(expr, inplace=True)
+
+            self.assert_eq(dummy_kdf, dummy_pdf)
+            self.assert_eq(kser, pser)
+
+        # invalid values for `expr`
+        invalid_exprs = (1, 1.0, (exprs[0],), [exprs[0]])
+        for expr in invalid_exprs:
+            with self.assertRaisesRegex(
+                TypeError,
+                "expr must be a string to be evaluated, {} given".format(type(expr).__name__),
+            ):
+                kdf.query(expr)
+
+        # invalid values for `inplace`
+        invalid_inplaces = (1, 0, "True", "False")
+        for inplace in invalid_inplaces:
+            with self.assertRaisesRegex(
+                TypeError,
+                'For argument "inplace" expected type bool, received type {}.'.format(
+                    type(inplace).__name__
+                ),
+            ):
+                kdf.query("a < b", inplace=inplace)
+
+        # doesn't support for MultiIndex columns
+        columns = pd.MultiIndex.from_tuples([("A", "Z"), ("B", "X"), ("C", "C")])
+        kdf.columns = columns
+        with self.assertRaisesRegex(TypeError, "Doesn't support for MultiIndex columns"):
+            kdf.query("('A', 'Z') > ('B', 'X')")
+
+    def test_take(self):
+        pdf = pd.DataFrame(
+            {"A": range(0, 50000), "B": range(100000, 0, -2), "C": range(100000, 50000, -1)}
+        )
+        kdf = ps.from_pandas(pdf)
+
+        # axis=0 (default)
+        self.assert_eq(kdf.take([1, 2]).sort_index(), pdf.take([1, 2]).sort_index())
+        self.assert_eq(kdf.take([-1, -2]).sort_index(), pdf.take([-1, -2]).sort_index())
+        self.assert_eq(
+            kdf.take(range(100, 110)).sort_index(), pdf.take(range(100, 110)).sort_index()
+        )
+        self.assert_eq(
+            kdf.take(range(-110, -100)).sort_index(), pdf.take(range(-110, -100)).sort_index()
+        )
+        self.assert_eq(
+            kdf.take([10, 100, 1000, 10000]).sort_index(),
+            pdf.take([10, 100, 1000, 10000]).sort_index(),
+        )
+        self.assert_eq(
+            kdf.take([-10, -100, -1000, -10000]).sort_index(),
+            pdf.take([-10, -100, -1000, -10000]).sort_index(),
+        )
+
+        # axis=1
+        self.assert_eq(kdf.take([1, 2], axis=1).sort_index(), pdf.take([1, 2], axis=1).sort_index())
+        self.assert_eq(
+            kdf.take([-1, -2], axis=1).sort_index(), pdf.take([-1, -2], axis=1).sort_index()
+        )
+        self.assert_eq(
+            kdf.take(range(1, 3), axis=1).sort_index(), pdf.take(range(1, 3), axis=1).sort_index(),
+        )
+        self.assert_eq(
+            kdf.take(range(-1, -3), axis=1).sort_index(),
+            pdf.take(range(-1, -3), axis=1).sort_index(),
+        )
+        self.assert_eq(
+            kdf.take([2, 1], axis=1).sort_index(), pdf.take([2, 1], axis=1).sort_index(),
+        )
+        self.assert_eq(
+            kdf.take([-1, -2], axis=1).sort_index(), pdf.take([-1, -2], axis=1).sort_index(),
+        )
+
+        # MultiIndex columns
+        columns = pd.MultiIndex.from_tuples([("A", "Z"), ("B", "X"), ("C", "C")])
+        kdf.columns = columns
+        pdf.columns = columns
+
+        # MultiIndex columns with axis=0 (default)
+        self.assert_eq(kdf.take([1, 2]).sort_index(), pdf.take([1, 2]).sort_index())
+        self.assert_eq(kdf.take([-1, -2]).sort_index(), pdf.take([-1, -2]).sort_index())
+        self.assert_eq(
+            kdf.take(range(100, 110)).sort_index(), pdf.take(range(100, 110)).sort_index()
+        )
+        self.assert_eq(
+            kdf.take(range(-110, -100)).sort_index(), pdf.take(range(-110, -100)).sort_index()
+        )
+        self.assert_eq(
+            kdf.take([10, 100, 1000, 10000]).sort_index(),
+            pdf.take([10, 100, 1000, 10000]).sort_index(),
+        )
+        self.assert_eq(
+            kdf.take([-10, -100, -1000, -10000]).sort_index(),
+            pdf.take([-10, -100, -1000, -10000]).sort_index(),
+        )
+
+        # axis=1
+        self.assert_eq(kdf.take([1, 2], axis=1).sort_index(), pdf.take([1, 2], axis=1).sort_index())
+        self.assert_eq(
+            kdf.take([-1, -2], axis=1).sort_index(), pdf.take([-1, -2], axis=1).sort_index()
+        )
+        self.assert_eq(
+            kdf.take(range(1, 3), axis=1).sort_index(), pdf.take(range(1, 3), axis=1).sort_index(),
+        )
+        self.assert_eq(
+            kdf.take(range(-1, -3), axis=1).sort_index(),
+            pdf.take(range(-1, -3), axis=1).sort_index(),
+        )
+        self.assert_eq(
+            kdf.take([2, 1], axis=1).sort_index(), pdf.take([2, 1], axis=1).sort_index(),
+        )
+        self.assert_eq(
+            kdf.take([-1, -2], axis=1).sort_index(), pdf.take([-1, -2], axis=1).sort_index(),
+        )
+
+        # Checking the type of indices.
+        self.assertRaises(TypeError, lambda: kdf.take(1))
+        self.assertRaises(TypeError, lambda: kdf.take("1"))
+        self.assertRaises(TypeError, lambda: kdf.take({1, 2}))
+        self.assertRaises(TypeError, lambda: kdf.take({1: None, 2: None}))
+
+    def test_axes(self):
+        pdf = self.pdf
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.axes, kdf.axes)
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("y", "b")])
+        pdf.columns = columns
+        kdf.columns = columns
+        self.assert_eq(pdf.axes, kdf.axes)
+
+    def test_udt(self):
+        sparse_values = {0: 0.1, 1: 1.1}
+        sparse_vector = SparseVector(len(sparse_values), sparse_values)
+        pdf = pd.DataFrame({"a": [sparse_vector], "b": [10]})
+
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kdf, pdf)
+
+    def test_eval(self):
+        pdf = pd.DataFrame({"A": range(1, 6), "B": range(10, 0, -2)})
+        kdf = ps.from_pandas(pdf)
+
+        # operation between columns (returns Series)
+        self.assert_eq(pdf.eval("A + B"), kdf.eval("A + B"))
+        self.assert_eq(pdf.eval("A + A"), kdf.eval("A + A"))
+        # assignment (returns DataFrame)
+        self.assert_eq(pdf.eval("C = A + B"), kdf.eval("C = A + B"))
+        self.assert_eq(pdf.eval("A = A + A"), kdf.eval("A = A + A"))
+        # operation between scalars (returns scalar)
+        self.assert_eq(pdf.eval("1 + 1"), kdf.eval("1 + 1"))
+        # complicated operations with assignment
+        self.assert_eq(
+            pdf.eval("B = A + B // (100 + 200) * (500 - B) - 10.5"),
+            kdf.eval("B = A + B // (100 + 200) * (500 - B) - 10.5"),
+        )
+
+        # inplace=True (only support for assignment)
+        pdf.eval("C = A + B", inplace=True)
+        kdf.eval("C = A + B", inplace=True)
+        self.assert_eq(pdf, kdf)
+        pser = pdf.A
+        kser = kdf.A
+        pdf.eval("A = B + C", inplace=True)
+        kdf.eval("A = B + C", inplace=True)
+        self.assert_eq(pdf, kdf)
+        self.assert_eq(pser, kser)
+
+        # doesn't support for multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("y", "b"), ("z", "c")])
+        kdf.columns = columns
+        self.assertRaises(TypeError, lambda: kdf.eval("x.a + y.b"))
+
+    @unittest.skipIf(not have_tabulate, tabulate_requirement_message)
+    def test_to_markdown(self):
+        pdf = pd.DataFrame(data={"animal_1": ["elk", "pig"], "animal_2": ["dog", "quetzal"]})
+        kdf = ps.from_pandas(pdf)
+
+        # `to_markdown()` is supported in pandas >= 1.0.0 since it's newly added in pandas 1.0.0.
+        if LooseVersion(pd.__version__) < LooseVersion("1.0.0"):
+            self.assertRaises(NotImplementedError, lambda: kdf.to_markdown())
+        else:
+            self.assert_eq(pdf.to_markdown(), kdf.to_markdown())
+
+    def test_cache(self):
+        pdf = pd.DataFrame(
+            [(0.2, 0.3), (0.0, 0.6), (0.6, 0.0), (0.2, 0.1)], columns=["dogs", "cats"]
+        )
+        kdf = ps.from_pandas(pdf)
+
+        with kdf.cache() as cached_df:
+            self.assert_eq(isinstance(cached_df, CachedDataFrame), True)
+            self.assert_eq(
+                repr(cached_df.storage_level), repr(StorageLevel(True, True, False, True))
+            )
+
+    def test_persist(self):
+        pdf = pd.DataFrame(
+            [(0.2, 0.3), (0.0, 0.6), (0.6, 0.0), (0.2, 0.1)], columns=["dogs", "cats"]
+        )
+        kdf = ps.from_pandas(pdf)
+        storage_levels = [
+            StorageLevel.DISK_ONLY,
+            StorageLevel.MEMORY_AND_DISK,
+            StorageLevel.MEMORY_ONLY,
+            StorageLevel.OFF_HEAP,
+        ]
+
+        for storage_level in storage_levels:
+            with kdf.persist(storage_level) as cached_df:
+                self.assert_eq(isinstance(cached_df, CachedDataFrame), True)
+                self.assert_eq(repr(cached_df.storage_level), repr(storage_level))
+
+        self.assertRaises(TypeError, lambda: kdf.persist("DISK_ONLY"))
+
+    def test_squeeze(self):
+        axises = [None, 0, 1, "rows", "index", "columns"]
+
+        # Multiple columns
+        pdf = pd.DataFrame([[1, 2], [3, 4]], columns=["a", "b"], index=["x", "y"])
+        kdf = ps.from_pandas(pdf)
+        for axis in axises:
+            self.assert_eq(pdf.squeeze(axis), kdf.squeeze(axis))
+        # Multiple columns with MultiIndex columns
+        columns = pd.MultiIndex.from_tuples([("A", "Z"), ("B", "X")])
+        pdf.columns = columns
+        kdf.columns = columns
+        for axis in axises:
+            self.assert_eq(pdf.squeeze(axis), kdf.squeeze(axis))
+
+        # Single column with single value
+        pdf = pd.DataFrame([[1]], columns=["a"], index=["x"])
+        kdf = ps.from_pandas(pdf)
+        for axis in axises:
+            self.assert_eq(pdf.squeeze(axis), kdf.squeeze(axis))
+        # Single column with single value with MultiIndex column
+        columns = pd.MultiIndex.from_tuples([("A", "Z")])
+        pdf.columns = columns
+        kdf.columns = columns
+        for axis in axises:
+            self.assert_eq(pdf.squeeze(axis), kdf.squeeze(axis))
+
+        # Single column with multiple values
+        pdf = pd.DataFrame([1, 2, 3, 4], columns=["a"])
+        kdf = ps.from_pandas(pdf)
+        for axis in axises:
+            self.assert_eq(pdf.squeeze(axis), kdf.squeeze(axis))
+        # Single column with multiple values with MultiIndex column
+        pdf.columns = columns
+        kdf.columns = columns
+        for axis in axises:
+            self.assert_eq(pdf.squeeze(axis), kdf.squeeze(axis))
+
+    def test_rfloordiv(self):
+        pdf = pd.DataFrame(
+            {"angles": [0, 3, 4], "degrees": [360, 180, 360]},
+            index=["circle", "triangle", "rectangle"],
+            columns=["angles", "degrees"],
+        )
+        kdf = ps.from_pandas(pdf)
+
+        if LooseVersion(pd.__version__) < LooseVersion("1.0.0") and LooseVersion(
+            pd.__version__
+        ) >= LooseVersion("0.24.0"):
+            expected_result = pd.DataFrame(
+                {"angles": [np.inf, 3.0, 2.0], "degrees": [0.0, 0.0, 0.0]},
+                index=["circle", "triangle", "rectangle"],
+                columns=["angles", "degrees"],
+            )
+        else:
+            expected_result = pdf.rfloordiv(10)
+
+        self.assert_eq(kdf.rfloordiv(10), expected_result)
+
+    def test_truncate(self):
+        pdf1 = pd.DataFrame(
+            {
+                "A": ["a", "b", "c", "d", "e", "f", "g"],
+                "B": ["h", "i", "j", "k", "l", "m", "n"],
+                "C": ["o", "p", "q", "r", "s", "t", "u"],
+            },
+            index=[-500, -20, -1, 0, 400, 550, 1000],
+        )
+        kdf1 = ps.from_pandas(pdf1)
+        pdf2 = pd.DataFrame(
+            {
+                "A": ["a", "b", "c", "d", "e", "f", "g"],
+                "B": ["h", "i", "j", "k", "l", "m", "n"],
+                "C": ["o", "p", "q", "r", "s", "t", "u"],
+            },
+            index=[1000, 550, 400, 0, -1, -20, -500],
+        )
+        kdf2 = ps.from_pandas(pdf2)
+
+        self.assert_eq(kdf1.truncate(), pdf1.truncate())
+        self.assert_eq(kdf1.truncate(before=-20), pdf1.truncate(before=-20))
+        self.assert_eq(kdf1.truncate(after=400), pdf1.truncate(after=400))
+        self.assert_eq(kdf1.truncate(copy=False), pdf1.truncate(copy=False))
+        self.assert_eq(kdf1.truncate(-20, 400, copy=False), pdf1.truncate(-20, 400, copy=False))
+        # The bug for these tests has been fixed in pandas 1.1.0.
+        if LooseVersion(pd.__version__) >= LooseVersion("1.1.0"):
+            self.assert_eq(kdf2.truncate(0, 550), pdf2.truncate(0, 550))
+            self.assert_eq(kdf2.truncate(0, 550, copy=False), pdf2.truncate(0, 550, copy=False))
+        else:
+            expected_kdf = ps.DataFrame(
+                {"A": ["b", "c", "d"], "B": ["i", "j", "k"], "C": ["p", "q", "r"]},
+                index=[550, 400, 0],
+            )
+            self.assert_eq(kdf2.truncate(0, 550), expected_kdf)
+            self.assert_eq(kdf2.truncate(0, 550, copy=False), expected_kdf)
+
+        # axis = 1
+        self.assert_eq(kdf1.truncate(axis=1), pdf1.truncate(axis=1))
+        self.assert_eq(kdf1.truncate(before="B", axis=1), pdf1.truncate(before="B", axis=1))
+        self.assert_eq(kdf1.truncate(after="A", axis=1), pdf1.truncate(after="A", axis=1))
+        self.assert_eq(kdf1.truncate(copy=False, axis=1), pdf1.truncate(copy=False, axis=1))
+        self.assert_eq(kdf2.truncate("B", "C", axis=1), pdf2.truncate("B", "C", axis=1))
+        self.assert_eq(
+            kdf1.truncate("B", "C", copy=False, axis=1),
+            pdf1.truncate("B", "C", copy=False, axis=1),
+        )
+
+        # MultiIndex columns
+        columns = pd.MultiIndex.from_tuples([("A", "Z"), ("B", "X"), ("C", "Z")])
+        pdf1.columns = columns
+        kdf1.columns = columns
+        pdf2.columns = columns
+        kdf2.columns = columns
+
+        self.assert_eq(kdf1.truncate(), pdf1.truncate())
+        self.assert_eq(kdf1.truncate(before=-20), pdf1.truncate(before=-20))
+        self.assert_eq(kdf1.truncate(after=400), pdf1.truncate(after=400))
+        self.assert_eq(kdf1.truncate(copy=False), pdf1.truncate(copy=False))
+        self.assert_eq(kdf1.truncate(-20, 400, copy=False), pdf1.truncate(-20, 400, copy=False))
+        # The bug for these tests has been fixed in pandas 1.1.0.
+        if LooseVersion(pd.__version__) >= LooseVersion("1.1.0"):
+            self.assert_eq(kdf2.truncate(0, 550), pdf2.truncate(0, 550))
+            self.assert_eq(kdf2.truncate(0, 550, copy=False), pdf2.truncate(0, 550, copy=False))
+        else:
+            expected_kdf.columns = columns
+            self.assert_eq(kdf2.truncate(0, 550), expected_kdf)
+            self.assert_eq(kdf2.truncate(0, 550, copy=False), expected_kdf)
+        # axis = 1
+        self.assert_eq(kdf1.truncate(axis=1), pdf1.truncate(axis=1))
+        self.assert_eq(kdf1.truncate(before="B", axis=1), pdf1.truncate(before="B", axis=1))
+        self.assert_eq(kdf1.truncate(after="A", axis=1), pdf1.truncate(after="A", axis=1))
+        self.assert_eq(kdf1.truncate(copy=False, axis=1), pdf1.truncate(copy=False, axis=1))
+        self.assert_eq(kdf2.truncate("B", "C", axis=1), pdf2.truncate("B", "C", axis=1))
+        self.assert_eq(
+            kdf1.truncate("B", "C", copy=False, axis=1),
+            pdf1.truncate("B", "C", copy=False, axis=1),
+        )
+
+        # Exceptions
+        kdf = ps.DataFrame(
+            {
+                "A": ["a", "b", "c", "d", "e", "f", "g"],
+                "B": ["h", "i", "j", "k", "l", "m", "n"],
+                "C": ["o", "p", "q", "r", "s", "t", "u"],
+            },
+            index=[-500, 100, 400, 0, -1, 550, -20],
+        )
+        msg = "truncate requires a sorted index"
+        with self.assertRaisesRegex(ValueError, msg):
+            kdf.truncate()
+
+        kdf = ps.DataFrame(
+            {
+                "A": ["a", "b", "c", "d", "e", "f", "g"],
+                "B": ["h", "i", "j", "k", "l", "m", "n"],
+                "C": ["o", "p", "q", "r", "s", "t", "u"],
+            },
+            index=[-500, -20, -1, 0, 400, 550, 1000],
+        )
+        msg = "Truncate: -20 must be after 400"
+        with self.assertRaisesRegex(ValueError, msg):
+            kdf.truncate(400, -20)
+        msg = "Truncate: B must be after C"
+        with self.assertRaisesRegex(ValueError, msg):
+            kdf.truncate("C", "B", axis=1)
+
+    def test_explode(self):
+        pdf = pd.DataFrame({"A": [[-1.0, np.nan], [0.0, np.inf], [1.0, -np.inf]], "B": 1})
+        pdf.index.name = "index"
+        pdf.columns.name = "columns"
+        kdf = ps.from_pandas(pdf)
+
+        if LooseVersion(pd.__version__) >= LooseVersion("0.25.0"):
+            expected_result1 = pdf.explode("A")
+            expected_result2 = pdf.explode("B")
+        else:
+            expected_result1 = pd.DataFrame(
+                {"A": [-1, np.nan, 0, np.inf, 1, -np.inf], "B": [1, 1, 1, 1, 1, 1]},
+                index=pd.Index([0, 0, 1, 1, 2, 2]),
+            )
+            expected_result1.index.name = "index"
+            expected_result1.columns.name = "columns"
+            expected_result2 = pdf
+
+        self.assert_eq(kdf.explode("A"), expected_result1, almost=True)
+        self.assert_eq(repr(kdf.explode("B")), repr(expected_result2))
+        self.assert_eq(kdf.explode("A").index.name, expected_result1.index.name)
+        self.assert_eq(kdf.explode("A").columns.name, expected_result1.columns.name)
+
+        self.assertRaises(TypeError, lambda: kdf.explode(["A", "B"]))
+
+        # MultiIndex
+        midx = pd.MultiIndex.from_tuples(
+            [("x", "a"), ("x", "b"), ("y", "c")], names=["index1", "index2"]
+        )
+        pdf.index = midx
+        kdf = ps.from_pandas(pdf)
+
+        if LooseVersion(pd.__version__) >= LooseVersion("0.25.0"):
+            expected_result1 = pdf.explode("A")
+            expected_result2 = pdf.explode("B")
+        else:
+            midx = pd.MultiIndex.from_tuples(
+                [("x", "a"), ("x", "a"), ("x", "b"), ("x", "b"), ("y", "c"), ("y", "c")],
+                names=["index1", "index2"],
+            )
+            expected_result1.index = midx
+            expected_result2 = pdf
+
+        self.assert_eq(kdf.explode("A"), expected_result1, almost=True)
+        self.assert_eq(repr(kdf.explode("B")), repr(expected_result2))
+        self.assert_eq(kdf.explode("A").index.names, expected_result1.index.names)
+        self.assert_eq(kdf.explode("A").columns.name, expected_result1.columns.name)
+
+        self.assertRaises(TypeError, lambda: kdf.explode(["A", "B"]))
+
+        # MultiIndex columns
+        columns = pd.MultiIndex.from_tuples([("A", "Z"), ("B", "X")], names=["column1", "column2"])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        if LooseVersion(pd.__version__) >= LooseVersion("0.25.0"):
+            expected_result1 = pdf.explode(("A", "Z"))
+            expected_result2 = pdf.explode(("B", "X"))
+            expected_result3 = pdf.A.explode("Z")
+        else:
+            expected_result1.columns = columns
+            expected_result2 = pdf
+            expected_result3 = pd.DataFrame({"Z": [-1, np.nan, 0, np.inf, 1, -np.inf]}, index=midx)
+            expected_result3.index.name = "index"
+            expected_result3.columns.name = "column2"
+
+        self.assert_eq(kdf.explode(("A", "Z")), expected_result1, almost=True)
+        self.assert_eq(repr(kdf.explode(("B", "X"))), repr(expected_result2))
+        self.assert_eq(kdf.explode(("A", "Z")).index.names, expected_result1.index.names)
+        self.assert_eq(kdf.explode(("A", "Z")).columns.names, expected_result1.columns.names)
+
+        self.assert_eq(kdf.A.explode("Z"), expected_result3, almost=True)
+
+        self.assertRaises(TypeError, lambda: kdf.explode(["A", "B"]))
+        self.assertRaises(ValueError, lambda: kdf.explode("A"))
+
+    def test_spark_schema(self):
+        kdf = ps.DataFrame(
+            {
+                "a": list("abc"),
+                "b": list(range(1, 4)),
+                "c": np.arange(3, 6).astype("i1"),
+                "d": np.arange(4.0, 7.0, dtype="float64"),
+                "e": [True, False, True],
+                "f": pd.date_range("20130101", periods=3),
+            },
+            columns=["a", "b", "c", "d", "e", "f"],
+        )
+        self.assertEqual(kdf.spark_schema(), kdf.spark.schema())
+        self.assertEqual(kdf.spark_schema("index"), kdf.spark.schema("index"))
+
+    def test_print_schema(self):
+        kdf = ps.DataFrame(
+            {"a": list("abc"), "b": list(range(1, 4)), "c": np.arange(3, 6).astype("i1")},
+            columns=["a", "b", "c"],
+        )
+
+        prev = sys.stdout
+        try:
+            out = StringIO()
+            sys.stdout = out
+            kdf.print_schema()
+            actual = out.getvalue().strip()
+
+            out = StringIO()
+            sys.stdout = out
+            kdf.spark.print_schema()
+            expected = out.getvalue().strip()
+
+            self.assertEqual(actual, expected)
+        finally:
+            sys.stdout = prev
+
+    def test_explain_hint(self):
+        kdf1 = ps.DataFrame(
+            {"lkey": ["foo", "bar", "baz", "foo"], "value": [1, 2, 3, 5]}, columns=["lkey", "value"]
+        )
+        kdf2 = ps.DataFrame(
+            {"rkey": ["foo", "bar", "baz", "foo"], "value": [5, 6, 7, 8]}, columns=["rkey", "value"]
+        )
+        merged = kdf1.merge(kdf2.hint("broadcast"), left_on="lkey", right_on="rkey")
+        prev = sys.stdout
+        try:
+            out = StringIO()
+            sys.stdout = out
+            merged.explain()
+            actual = out.getvalue().strip()
+
+            out = StringIO()
+            sys.stdout = out
+            merged.spark.explain()
+            expected = out.getvalue().strip()
+
+            self.assertEqual(actual, expected)
+        finally:
+            sys.stdout = prev
+
+    def test_mad(self):
+        pdf = pd.DataFrame(
+            {
+                "A": [1, 2, None, 4, np.nan],
+                "B": [-0.1, 0.2, -0.3, np.nan, 0.5],
+                "C": ["a", "b", "c", "d", "e"],
+            }
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.mad(), pdf.mad())
+        self.assert_eq(kdf.mad(axis=1), pdf.mad(axis=1))
+
+        with self.assertRaises(ValueError):
+            kdf.mad(axis=2)
+
+        # MultiIndex columns
+        columns = pd.MultiIndex.from_tuples([("A", "X"), ("A", "Y"), ("A", "Z")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(kdf.mad(), pdf.mad())
+        self.assert_eq(kdf.mad(axis=1), pdf.mad(axis=1))
+
+        pdf = pd.DataFrame({"A": [True, True, False, False], "B": [True, False, False, True]})
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.mad(), pdf.mad())
+        self.assert_eq(kdf.mad(axis=1), pdf.mad(axis=1))
+
+    def test_abs(self):
+        pdf = pd.DataFrame({"a": [-2, -1, 0, 1]})
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(abs(kdf), abs(pdf))
+        self.assert_eq(np.abs(kdf), np.abs(pdf))
+
+    def test_iteritems(self):
+        pdf = pd.DataFrame(
+            {"species": ["bear", "bear", "marsupial"], "population": [1864, 22000, 80000]},
+            index=["panda", "polar", "koala"],
+            columns=["species", "population"],
+        )
+        kdf = ps.from_pandas(pdf)
+
+        for (p_name, p_items), (k_name, k_items) in zip(pdf.iteritems(), kdf.iteritems()):
+            self.assert_eq(p_name, k_name)
+            self.assert_eq(p_items, k_items)
+
+    def test_tail(self):
+        pdf = pd.DataFrame({"x": range(1000)})
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(pdf.tail(), kdf.tail())
+        self.assert_eq(pdf.tail(10), kdf.tail(10))
+        self.assert_eq(pdf.tail(-990), kdf.tail(-990))
+        self.assert_eq(pdf.tail(0), kdf.tail(0))
+        self.assert_eq(pdf.tail(-1001), kdf.tail(-1001))
+        self.assert_eq(pdf.tail(1001), kdf.tail(1001))
+        self.assert_eq((pdf + 1).tail(), (kdf + 1).tail())
+        self.assert_eq((pdf + 1).tail(10), (kdf + 1).tail(10))
+        self.assert_eq((pdf + 1).tail(-990), (kdf + 1).tail(-990))
+        self.assert_eq((pdf + 1).tail(0), (kdf + 1).tail(0))
+        self.assert_eq((pdf + 1).tail(-1001), (kdf + 1).tail(-1001))
+        self.assert_eq((pdf + 1).tail(1001), (kdf + 1).tail(1001))
+        with self.assertRaisesRegex(TypeError, "bad operand type for unary -: 'str'"):
+            kdf.tail("10")
+
+    def test_last_valid_index(self):
+        pdf = pd.DataFrame(
+            {"a": [1, 2, 3, None], "b": [1.0, 2.0, 3.0, None], "c": [100, 200, 400, None]},
+            index=["Q", "W", "E", "R"],
+        )
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.last_valid_index(), kdf.last_valid_index())
+        self.assert_eq(pdf[[]].last_valid_index(), kdf[[]].last_valid_index())
+
+        # MultiIndex columns
+        pdf.columns = pd.MultiIndex.from_tuples([("a", "x"), ("b", "y"), ("c", "z")])
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.last_valid_index(), kdf.last_valid_index())
+
+        # Empty DataFrame
+        pdf = pd.Series([]).to_frame()
+        kdf = ps.Series([]).to_frame()
+        self.assert_eq(pdf.last_valid_index(), kdf.last_valid_index())
+
+    def test_last(self):
+        index = pd.date_range("2018-04-09", periods=4, freq="2D")
+        pdf = pd.DataFrame([1, 2, 3, 4], index=index)
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.last("1D"), kdf.last("1D"))
+        self.assert_eq(pdf.last(DateOffset(days=1)), kdf.last(DateOffset(days=1)))
+        with self.assertRaisesRegex(TypeError, "'last' only supports a DatetimeIndex"):
+            ps.DataFrame([1, 2, 3, 4]).last("1D")
+
+    def test_first(self):
+        index = pd.date_range("2018-04-09", periods=4, freq="2D")
+        pdf = pd.DataFrame([1, 2, 3, 4], index=index)
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.first("1D"), kdf.first("1D"))
+        self.assert_eq(pdf.first(DateOffset(days=1)), kdf.first(DateOffset(days=1)))
+        with self.assertRaisesRegex(TypeError, "'first' only supports a DatetimeIndex"):
+            ps.DataFrame([1, 2, 3, 4]).first("1D")
+
+    def test_first_valid_index(self):
+        pdf = pd.DataFrame(
+            {"a": [None, 2, 3, 2], "b": [None, 2.0, 3.0, 1.0], "c": [None, 200, 400, 200]},
+            index=["Q", "W", "E", "R"],
+        )
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.first_valid_index(), kdf.first_valid_index())
+        self.assert_eq(pdf[[]].first_valid_index(), kdf[[]].first_valid_index())
+
+        # MultiIndex columns
+        pdf.columns = pd.MultiIndex.from_tuples([("a", "x"), ("b", "y"), ("c", "z")])
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.first_valid_index(), kdf.first_valid_index())
+
+        # Empty DataFrame
+        pdf = pd.Series([]).to_frame()
+        kdf = ps.Series([]).to_frame()
+        self.assert_eq(pdf.first_valid_index(), kdf.first_valid_index())
+
+        pdf = pd.DataFrame(
+            {"a": [None, 2, 3, 2], "b": [None, 2.0, 3.0, 1.0], "c": [None, 200, 400, 200]},
+            index=[
+                datetime(2021, 1, 1),
+                datetime(2021, 2, 1),
+                datetime(2021, 3, 1),
+                datetime(2021, 4, 1),
+            ],
+        )
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.first_valid_index(), kdf.first_valid_index())
+
+    def test_product(self):
+        pdf = pd.DataFrame(
+            {"A": [1, 2, 3, 4, 5], "B": [10, 20, 30, 40, 50], "C": ["a", "b", "c", "d", "e"]}
+        )
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.prod(), kdf.prod().sort_index())
+
+        # Named columns
+        pdf.columns.name = "Koalas"
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.prod(), kdf.prod().sort_index())
+
+        # MultiIndex columns
+        pdf.columns = pd.MultiIndex.from_tuples([("a", "x"), ("b", "y"), ("c", "z")])
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.prod(), kdf.prod().sort_index())
+
+        # Named MultiIndex columns
+        pdf.columns.names = ["Hello", "Koalas"]
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.prod(), kdf.prod().sort_index())
+
+        # No numeric columns
+        pdf = pd.DataFrame({"key": ["a", "b", "c"], "val": ["x", "y", "z"]})
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.prod(), kdf.prod().sort_index())
+
+        # No numeric named columns
+        pdf.columns.name = "Koalas"
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.prod(), kdf.prod().sort_index(), almost=True)
+
+        # No numeric MultiIndex columns
+        pdf.columns = pd.MultiIndex.from_tuples([("a", "x"), ("b", "y")])
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.prod(), kdf.prod().sort_index(), almost=True)
+
+        # No numeric named MultiIndex columns
+        pdf.columns.names = ["Hello", "Koalas"]
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.prod(), kdf.prod().sort_index(), almost=True)
+
+        # All NaN columns
+        pdf = pd.DataFrame(
+            {
+                "A": [np.nan, np.nan, np.nan, np.nan, np.nan],
+                "B": [10, 20, 30, 40, 50],
+                "C": ["a", "b", "c", "d", "e"],
+            }
+        )
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.prod(), kdf.prod().sort_index(), check_exact=False)
+
+        # All NaN named columns
+        pdf.columns.name = "Koalas"
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.prod(), kdf.prod().sort_index(), check_exact=False)
+
+        # All NaN MultiIndex columns
+        pdf.columns = pd.MultiIndex.from_tuples([("a", "x"), ("b", "y"), ("c", "z")])
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.prod(), kdf.prod().sort_index(), check_exact=False)
+
+        # All NaN named MultiIndex columns
+        pdf.columns.names = ["Hello", "Koalas"]
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.prod(), kdf.prod().sort_index(), check_exact=False)
+
+    def test_from_dict(self):
+        data = {"row_1": [3, 2, 1, 0], "row_2": [10, 20, 30, 40]}
+        pdf = pd.DataFrame.from_dict(data)
+        kdf = ps.DataFrame.from_dict(data)
+        self.assert_eq(pdf, kdf)
+
+        pdf = pd.DataFrame.from_dict(data, dtype="int8")
+        kdf = ps.DataFrame.from_dict(data, dtype="int8")
+        self.assert_eq(pdf, kdf)
+
+        pdf = pd.DataFrame.from_dict(data, orient="index", columns=["A", "B", "C", "D"])
+        kdf = ps.DataFrame.from_dict(data, orient="index", columns=["A", "B", "C", "D"])
+        self.assert_eq(pdf, kdf)
+
+    def test_pad(self):
+        pdf = pd.DataFrame(
+            {
+                "A": [None, 3, None, None],
+                "B": [2, 4, None, 3],
+                "C": [None, None, None, 1],
+                "D": [0, 1, 5, 4],
+            },
+            columns=["A", "B", "C", "D"],
+        )
+        kdf = ps.from_pandas(pdf)
+
+        if LooseVersion(pd.__version__) >= LooseVersion("1.1"):
+            self.assert_eq(pdf.pad(), kdf.pad())
+
+            # Test `inplace=True`
+            pdf.pad(inplace=True)
+            kdf.pad(inplace=True)
+            self.assert_eq(pdf, kdf)
+        else:
+            expected = ps.DataFrame(
+                {
+                    "A": [None, 3, 3, 3],
+                    "B": [2.0, 4.0, 4.0, 3.0],
+                    "C": [None, None, None, 1],
+                    "D": [0, 1, 5, 4],
+                },
+                columns=["A", "B", "C", "D"],
+            )
+            self.assert_eq(expected, kdf.pad())
+
+            # Test `inplace=True`
+            kdf.pad(inplace=True)
+            self.assert_eq(expected, kdf)
+
+    def test_backfill(self):
+        pdf = pd.DataFrame(
+            {
+                "A": [None, 3, None, None],
+                "B": [2, 4, None, 3],
+                "C": [None, None, None, 1],
+                "D": [0, 1, 5, 4],
+            },
+            columns=["A", "B", "C", "D"],
+        )
+        kdf = ps.from_pandas(pdf)
+
+        if LooseVersion(pd.__version__) >= LooseVersion("1.1"):
+            self.assert_eq(pdf.backfill(), kdf.backfill())
+
+            # Test `inplace=True`
+            pdf.backfill(inplace=True)
+            kdf.backfill(inplace=True)
+            self.assert_eq(pdf, kdf)
+        else:
+            expected = ps.DataFrame(
+                {
+                    "A": [3.0, 3.0, None, None],
+                    "B": [2.0, 4.0, 3.0, 3.0],
+                    "C": [1.0, 1.0, 1.0, 1.0],
+                    "D": [0, 1, 5, 4],
+                },
+                columns=["A", "B", "C", "D"],
+            )
+            self.assert_eq(expected, kdf.backfill())
+
+            # Test `inplace=True`
+            kdf.backfill(inplace=True)
+            self.assert_eq(expected, kdf)
+
+    def test_align(self):
+        pdf1 = pd.DataFrame({"a": [1, 2, 3], "b": ["a", "b", "c"]}, index=[10, 20, 30])
+        kdf1 = ps.from_pandas(pdf1)
+
+        for join in ["outer", "inner", "left", "right"]:
+            for axis in [None, 0, 1]:
+                kdf_l, kdf_r = kdf1.align(kdf1[["b"]], join=join, axis=axis)
+                pdf_l, pdf_r = pdf1.align(pdf1[["b"]], join=join, axis=axis)
+                self.assert_eq(kdf_l, pdf_l)
+                self.assert_eq(kdf_r, pdf_r)
+
+                kdf_l, kdf_r = kdf1[["a"]].align(kdf1[["b", "a"]], join=join, axis=axis)
+                pdf_l, pdf_r = pdf1[["a"]].align(pdf1[["b", "a"]], join=join, axis=axis)
+                self.assert_eq(kdf_l, pdf_l)
+                self.assert_eq(kdf_r, pdf_r)
+
+                kdf_l, kdf_r = kdf1[["b", "a"]].align(kdf1[["a"]], join=join, axis=axis)
+                pdf_l, pdf_r = pdf1[["b", "a"]].align(pdf1[["a"]], join=join, axis=axis)
+                self.assert_eq(kdf_l, pdf_l)
+                self.assert_eq(kdf_r, pdf_r)
+
+        kdf_l, kdf_r = kdf1.align(kdf1["b"], axis=0)
+        pdf_l, pdf_r = pdf1.align(pdf1["b"], axis=0)
+        self.assert_eq(kdf_l, pdf_l)
+        self.assert_eq(kdf_r, pdf_r)
+
+        kdf_l, kser_b = kdf1[["a"]].align(kdf1["b"], axis=0)
+        pdf_l, pser_b = pdf1[["a"]].align(pdf1["b"], axis=0)
+        self.assert_eq(kdf_l, pdf_l)
+        self.assert_eq(kser_b, pser_b)
+
+        self.assertRaises(ValueError, lambda: kdf1.align(kdf1, join="unknown"))
+        self.assertRaises(ValueError, lambda: kdf1.align(kdf1["b"]))
+        self.assertRaises(NotImplementedError, lambda: kdf1.align(kdf1["b"], axis=1))
+
+        pdf2 = pd.DataFrame({"a": [4, 5, 6], "d": ["d", "e", "f"]}, index=[10, 11, 12])
+        kdf2 = ps.from_pandas(pdf2)
+
+        for join in ["outer", "inner", "left", "right"]:
+            kdf_l, kdf_r = kdf1.align(kdf2, join=join, axis=1)
+            pdf_l, pdf_r = pdf1.align(pdf2, join=join, axis=1)
+            self.assert_eq(kdf_l.sort_index(), pdf_l.sort_index())
+            self.assert_eq(kdf_r.sort_index(), pdf_r.sort_index())
+
+    def test_between_time(self):
+        idx = pd.date_range("2018-04-09", periods=4, freq="1D20min")
+        pdf = pd.DataFrame({"A": [1, 2, 3, 4]}, index=idx)
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(
+            pdf.between_time("0:15", "0:45").sort_index(),
+            kdf.between_time("0:15", "0:45").sort_index(),
+        )
+
+        pdf.index.name = "ts"
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(
+            pdf.between_time("0:15", "0:45").sort_index(),
+            kdf.between_time("0:15", "0:45").sort_index(),
+        )
+
+        # Column label is 'index'
+        pdf.columns = pd.Index(["index"])
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(
+            pdf.between_time("0:15", "0:45").sort_index(),
+            kdf.between_time("0:15", "0:45").sort_index(),
+        )
+
+        # Both index name and column label are 'index'
+        pdf.index.name = "index"
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(
+            pdf.between_time("0:15", "0:45").sort_index(),
+            kdf.between_time("0:15", "0:45").sort_index(),
+        )
+
+        # Index name is 'index', column label is ('X', 'A')
+        pdf.columns = pd.MultiIndex.from_arrays([["X"], ["A"]])
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(
+            pdf.between_time("0:15", "0:45").sort_index(),
+            kdf.between_time("0:15", "0:45").sort_index(),
+        )
+
+        with self.assertRaisesRegex(
+            NotImplementedError, "between_time currently only works for axis=0"
+        ):
+            kdf.between_time("0:15", "0:45", axis=1)
+
+        kdf = ps.DataFrame({"A": [1, 2, 3, 4]})
+        with self.assertRaisesRegex(TypeError, "Index must be DatetimeIndex"):
+            kdf.between_time("0:15", "0:45")
+
+    def test_at_time(self):
+        idx = pd.date_range("2018-04-09", periods=4, freq="1D20min")
+        pdf = pd.DataFrame({"A": [1, 2, 3, 4]}, index=idx)
+        kdf = ps.from_pandas(pdf)
+        kdf.at_time("0:20")
+        self.assert_eq(
+            pdf.at_time("0:20").sort_index(), kdf.at_time("0:20").sort_index(),
+        )
+
+        # Index name is 'ts'
+        pdf.index.name = "ts"
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(
+            pdf.at_time("0:20").sort_index(), kdf.at_time("0:20").sort_index(),
+        )
+
+        # Index name is 'ts', column label is 'index'
+        pdf.columns = pd.Index(["index"])
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(
+            pdf.at_time("0:40").sort_index(), kdf.at_time("0:40").sort_index(),
+        )
+
+        # Both index name and column label are 'index'
+        pdf.index.name = "index"
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(
+            pdf.at_time("0:40").sort_index(), kdf.at_time("0:40").sort_index(),
+        )
+
+        # Index name is 'index', column label is ('X', 'A')
+        pdf.columns = pd.MultiIndex.from_arrays([["X"], ["A"]])
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(
+            pdf.at_time("0:40").sort_index(), kdf.at_time("0:40").sort_index(),
+        )
+
+        with self.assertRaisesRegex(NotImplementedError, "'asof' argument is not supported"):
+            kdf.at_time("0:15", asof=True)
+
+        with self.assertRaisesRegex(NotImplementedError, "at_time currently only works for axis=0"):
+            kdf.at_time("0:15", axis=1)
+
+        kdf = ps.DataFrame({"A": [1, 2, 3, 4]})
+        with self.assertRaisesRegex(TypeError, "Index must be DatetimeIndex"):
+            kdf.at_time("0:15")
+
+
+if __name__ == "__main__":
+    from pyspark.pandas.tests.test_dataframe import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_dataframe_conversion.py b/python/pyspark/pandas/tests/test_dataframe_conversion.py
new file mode 100644
index 0000000000000..92ddef601412c
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_dataframe_conversion.py
@@ -0,0 +1,275 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import shutil
+import string
+import tempfile
+import unittest
+
+import numpy as np
+import pandas as pd
+
+from distutils.version import LooseVersion
+from pyspark import pandas as ps
+from pyspark.testing.pandasutils import PandasOnSparkTestCase, TestUtils
+from pyspark.testing.sqlutils import SQLTestUtils
+
+
+class DataFrameConversionTest(PandasOnSparkTestCase, SQLTestUtils, TestUtils):
+    """Test cases for "small data" conversion and I/O."""
+
+    def setUp(self):
+        self.tmp_dir = tempfile.mkdtemp(prefix=DataFrameConversionTest.__name__)
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp_dir, ignore_errors=True)
+
+    @property
+    def pdf(self):
+        return pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}, index=[0, 1, 3])
+
+    @property
+    def kdf(self):
+        return ps.from_pandas(self.pdf)
+
+    @staticmethod
+    def strip_all_whitespace(str):
+        """A helper function to remove all whitespace from a string."""
+        return str.translate({ord(c): None for c in string.whitespace})
+
+    def test_to_html(self):
+        expected = self.strip_all_whitespace(
+            """
+            <table border="1" class="dataframe">
+              <thead>
+                <tr style="text-align: right;"><th></th><th>a</th><th>b</th></tr>
+              </thead>
+              <tbody>
+                <tr><th>0</th><td>1</td><td>4</td></tr>
+                <tr><th>1</th><td>2</td><td>5</td></tr>
+                <tr><th>3</th><td>3</td><td>6</td></tr>
+              </tbody>
+            </table>
+            """
+        )
+        got = self.strip_all_whitespace(self.kdf.to_html())
+        self.assert_eq(got, expected)
+
+        # with max_rows set
+        expected = self.strip_all_whitespace(
+            """
+            <table border="1" class="dataframe">
+              <thead>
+                <tr style="text-align: right;"><th></th><th>a</th><th>b</th></tr>
+              </thead>
+              <tbody>
+                <tr><th>0</th><td>1</td><td>4</td></tr>
+                <tr><th>1</th><td>2</td><td>5</td></tr>
+              </tbody>
+            </table>
+            """
+        )
+        got = self.strip_all_whitespace(self.kdf.to_html(max_rows=2))
+        self.assert_eq(got, expected)
+
+    @staticmethod
+    def get_excel_dfs(koalas_location, pandas_location):
+        return {
+            "got": pd.read_excel(koalas_location, index_col=0),
+            "expected": pd.read_excel(pandas_location, index_col=0),
+        }
+
+    @unittest.skip("openpyxl")
+    def test_to_excel(self):
+        with self.temp_dir() as dirpath:
+            pandas_location = dirpath + "/" + "output1.xlsx"
+            koalas_location = dirpath + "/" + "output2.xlsx"
+
+            pdf = self.pdf
+            kdf = self.kdf
+            kdf.to_excel(koalas_location)
+            pdf.to_excel(pandas_location)
+            dataframes = self.get_excel_dfs(koalas_location, pandas_location)
+            self.assert_eq(dataframes["got"], dataframes["expected"])
+
+            kdf.a.to_excel(koalas_location)
+            pdf.a.to_excel(pandas_location)
+            dataframes = self.get_excel_dfs(koalas_location, pandas_location)
+            self.assert_eq(dataframes["got"], dataframes["expected"])
+
+            pdf = pd.DataFrame({"a": [1, None, 3], "b": ["one", "two", None]}, index=[0, 1, 3])
+
+            kdf = ps.from_pandas(pdf)
+
+            kdf.to_excel(koalas_location, na_rep="null")
+            pdf.to_excel(pandas_location, na_rep="null")
+            dataframes = self.get_excel_dfs(koalas_location, pandas_location)
+            self.assert_eq(dataframes["got"], dataframes["expected"])
+
+            pdf = pd.DataFrame({"a": [1.0, 2.0, 3.0], "b": [4.0, 5.0, 6.0]}, index=[0, 1, 3])
+
+            kdf = ps.from_pandas(pdf)
+
+            kdf.to_excel(koalas_location, float_format="%.1f")
+            pdf.to_excel(pandas_location, float_format="%.1f")
+            dataframes = self.get_excel_dfs(koalas_location, pandas_location)
+            self.assert_eq(dataframes["got"], dataframes["expected"])
+
+            kdf.to_excel(koalas_location, header=False)
+            pdf.to_excel(pandas_location, header=False)
+            dataframes = self.get_excel_dfs(koalas_location, pandas_location)
+            self.assert_eq(dataframes["got"], dataframes["expected"])
+
+            kdf.to_excel(koalas_location, index=False)
+            pdf.to_excel(pandas_location, index=False)
+            dataframes = self.get_excel_dfs(koalas_location, pandas_location)
+            self.assert_eq(dataframes["got"], dataframes["expected"])
+
+    def test_to_json(self):
+        pdf = self.pdf
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.to_json(orient="records"), pdf.to_json(orient="records"))
+
+    def test_to_json_negative(self):
+        kdf = ps.from_pandas(self.pdf)
+
+        with self.assertRaises(NotImplementedError):
+            kdf.to_json(orient="table")
+
+        with self.assertRaises(NotImplementedError):
+            kdf.to_json(lines=False)
+
+    def test_read_json_negative(self):
+        with self.assertRaises(NotImplementedError):
+            ps.read_json("invalid", lines=False)
+
+    def test_to_json_with_path(self):
+        pdf = pd.DataFrame({"a": [1], "b": ["a"]})
+        kdf = ps.DataFrame(pdf)
+
+        kdf.to_json(self.tmp_dir, num_files=1)
+        expected = pdf.to_json(orient="records")
+
+        output_paths = [path for path in os.listdir(self.tmp_dir) if path.startswith("part-")]
+        assert len(output_paths) > 0
+        output_path = "%s/%s" % (self.tmp_dir, output_paths[0])
+        self.assertEqual("[%s]" % open(output_path).read().strip(), expected)
+
+    def test_to_json_with_partition_cols(self):
+        pdf = pd.DataFrame({"a": [1, 2, 3], "b": ["a", "b", "c"]})
+        kdf = ps.DataFrame(pdf)
+
+        kdf.to_json(self.tmp_dir, partition_cols="b", num_files=1)
+
+        partition_paths = [path for path in os.listdir(self.tmp_dir) if path.startswith("b=")]
+        assert len(partition_paths) > 0
+        for partition_path in partition_paths:
+            column, value = partition_path.split("=")
+            expected = pdf[pdf[column] == value].drop("b", axis=1).to_json(orient="records")
+
+            output_paths = [
+                path
+                for path in os.listdir("%s/%s" % (self.tmp_dir, partition_path))
+                if path.startswith("part-")
+            ]
+            assert len(output_paths) > 0
+            output_path = "%s/%s/%s" % (self.tmp_dir, partition_path, output_paths[0])
+            with open(output_path) as f:
+                self.assertEqual("[%s]" % open(output_path).read().strip(), expected)
+
+    @unittest.skip("Pyperclip could not find a copy/paste mechanism for Linux.")
+    def test_to_clipboard(self):
+        pdf = self.pdf
+        kdf = self.kdf
+
+        self.assert_eq(kdf.to_clipboard(), pdf.to_clipboard())
+        self.assert_eq(kdf.to_clipboard(excel=False), pdf.to_clipboard(excel=False))
+        self.assert_eq(
+            kdf.to_clipboard(sep=";", index=False), pdf.to_clipboard(sep=";", index=False)
+        )
+
+    def test_to_latex(self):
+        pdf = self.pdf
+        kdf = self.kdf
+
+        self.assert_eq(kdf.to_latex(), pdf.to_latex())
+        self.assert_eq(kdf.to_latex(col_space=2), pdf.to_latex(col_space=2))
+        self.assert_eq(kdf.to_latex(header=True), pdf.to_latex(header=True))
+        self.assert_eq(kdf.to_latex(index=False), pdf.to_latex(index=False))
+        self.assert_eq(kdf.to_latex(na_rep="-"), pdf.to_latex(na_rep="-"))
+        self.assert_eq(kdf.to_latex(float_format="%.1f"), pdf.to_latex(float_format="%.1f"))
+        self.assert_eq(kdf.to_latex(sparsify=False), pdf.to_latex(sparsify=False))
+        self.assert_eq(kdf.to_latex(index_names=False), pdf.to_latex(index_names=False))
+        self.assert_eq(kdf.to_latex(bold_rows=True), pdf.to_latex(bold_rows=True))
+        self.assert_eq(kdf.to_latex(decimal=","), pdf.to_latex(decimal=","))
+        if LooseVersion(pd.__version__) < LooseVersion("1.0.0"):
+            self.assert_eq(kdf.to_latex(encoding="ascii"), pdf.to_latex(encoding="ascii"))
+
+    def test_to_records(self):
+        if LooseVersion(pd.__version__) >= LooseVersion("0.24.0"):
+            pdf = pd.DataFrame({"A": [1, 2], "B": [0.5, 0.75]}, index=["a", "b"])
+
+            kdf = ps.from_pandas(pdf)
+
+            self.assert_eq(kdf.to_records(), pdf.to_records())
+            self.assert_eq(kdf.to_records(index=False), pdf.to_records(index=False))
+            self.assert_eq(kdf.to_records(index_dtypes="<S2"), pdf.to_records(index_dtypes="<S2"))
+
+    def test_from_records(self):
+        # Assert using a dict as input
+        self.assert_eq(
+            ps.DataFrame.from_records({"A": [1, 2, 3]}), pd.DataFrame.from_records({"A": [1, 2, 3]})
+        )
+        # Assert using a list of tuples as input
+        self.assert_eq(
+            ps.DataFrame.from_records([(1, 2), (3, 4)]), pd.DataFrame.from_records([(1, 2), (3, 4)])
+        )
+        # Assert using a NumPy array as input
+        self.assert_eq(ps.DataFrame.from_records(np.eye(3)), pd.DataFrame.from_records(np.eye(3)))
+        # Asserting using a custom index
+        self.assert_eq(
+            ps.DataFrame.from_records([(1, 2), (3, 4)], index=[2, 3]),
+            pd.DataFrame.from_records([(1, 2), (3, 4)], index=[2, 3]),
+        )
+        # Assert excluding excluding column(s)
+        self.assert_eq(
+            ps.DataFrame.from_records({"A": [1, 2, 3], "B": [1, 2, 3]}, exclude=["B"]),
+            pd.DataFrame.from_records({"A": [1, 2, 3], "B": [1, 2, 3]}, exclude=["B"]),
+        )
+        # Assert limiting to certain column(s)
+        self.assert_eq(
+            ps.DataFrame.from_records({"A": [1, 2, 3], "B": [1, 2, 3]}, columns=["A"]),
+            pd.DataFrame.from_records({"A": [1, 2, 3], "B": [1, 2, 3]}, columns=["A"]),
+        )
+        # Assert limiting to a number of rows
+        self.assert_eq(
+            ps.DataFrame.from_records([(1, 2), (3, 4)], nrows=1),
+            pd.DataFrame.from_records([(1, 2), (3, 4)], nrows=1),
+        )
+
+
+if __name__ == "__main__":
+    from pyspark.pandas.tests.test_dataframe_conversion import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_dataframe_spark_io.py b/python/pyspark/pandas/tests/test_dataframe_spark_io.py
new file mode 100644
index 0000000000000..f0982bd4e2564
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_dataframe_spark_io.py
@@ -0,0 +1,457 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from distutils.version import LooseVersion
+import unittest
+import glob
+import os
+
+import numpy as np
+import pandas as pd
+import pyarrow as pa
+
+from pyspark import pandas as ps
+from pyspark.testing.pandasutils import PandasOnSparkTestCase, TestUtils
+
+
+class DataFrameSparkIOTest(PandasOnSparkTestCase, TestUtils):
+    """Test cases for big data I/O using Spark."""
+
+    @property
+    def test_column_order(self):
+        return ["i32", "i64", "f", "bhello"]
+
+    @property
+    def test_pdf(self):
+        pdf = pd.DataFrame(
+            {
+                "i32": np.arange(20, dtype=np.int32) % 3,
+                "i64": np.arange(20, dtype=np.int64) % 5,
+                "f": np.arange(20, dtype=np.float64),
+                "bhello": np.random.choice(["hello", "yo", "people"], size=20).astype("O"),
+            },
+            columns=self.test_column_order,
+            index=np.random.rand(20),
+        )
+        return pdf
+
+    def test_parquet_read(self):
+        with self.temp_dir() as tmp:
+            data = self.test_pdf
+            self.spark.createDataFrame(data, "i32 int, i64 long, f double, bhello string").coalesce(
+                1
+            ).write.parquet(tmp, mode="overwrite")
+
+            def check(columns, expected):
+                if LooseVersion("0.21.1") <= LooseVersion(pd.__version__):
+                    expected = pd.read_parquet(tmp, columns=columns)
+                actual = ps.read_parquet(tmp, columns=columns)
+                self.assertPandasEqual(expected, actual.to_pandas())
+
+            check(None, data)
+            check(["i32", "i64"], data[["i32", "i64"]])
+            check(["i64", "i32"], data[["i64", "i32"]])
+
+            if LooseVersion(pa.__version__) < LooseVersion("1.0.0"):
+                # TODO: `pd.read_parquet()` changed the behavior due to PyArrow 1.0.0.
+                #       We might want to adjust the behavior. Let's see how pandas handles it.
+                check(("i32", "i64"), data[["i32", "i64"]])
+                check(["a", "b", "i32", "i64"], data[["i32", "i64"]])
+                check([], pd.DataFrame([]))
+                check(["a"], pd.DataFrame([]))
+                check("i32", pd.DataFrame([]))
+                check("float", data[["f"]])
+
+            # check with pyspark patch.
+            if LooseVersion("0.21.1") <= LooseVersion(pd.__version__):
+                expected = pd.read_parquet(tmp)
+            else:
+                expected = data
+            actual = ps.read_parquet(tmp)
+            self.assertPandasEqual(expected, actual.to_pandas())
+
+            # When index columns are known
+            pdf = self.test_pdf
+            expected = ps.DataFrame(pdf)
+
+            expected_idx = expected.set_index("bhello")[["f", "i32", "i64"]]
+            actual_idx = ps.read_parquet(tmp, index_col="bhello")[["f", "i32", "i64"]]
+            self.assert_eq(
+                actual_idx.sort_values(by="f").to_spark().toPandas(),
+                expected_idx.sort_values(by="f").to_spark().toPandas(),
+            )
+
+    def test_parquet_read_with_pandas_metadata(self):
+        with self.temp_dir() as tmp:
+            expected1 = self.test_pdf
+
+            path1 = "{}/file1.parquet".format(tmp)
+            expected1.to_parquet(path1)
+
+            self.assert_eq(ps.read_parquet(path1, pandas_metadata=True), expected1)
+
+            expected2 = expected1.reset_index()
+
+            path2 = "{}/file2.parquet".format(tmp)
+            expected2.to_parquet(path2)
+
+            self.assert_eq(ps.read_parquet(path2, pandas_metadata=True), expected2)
+
+            expected3 = expected2.set_index("index", append=True)
+
+            path3 = "{}/file3.parquet".format(tmp)
+            expected3.to_parquet(path3)
+
+            self.assert_eq(ps.read_parquet(path3, pandas_metadata=True), expected3)
+
+    def test_parquet_write(self):
+        with self.temp_dir() as tmp:
+            pdf = self.test_pdf
+            expected = ps.DataFrame(pdf)
+
+            # Write out partitioned by one column
+            expected.to_parquet(tmp, mode="overwrite", partition_cols="i32")
+            # Reset column order, as once the data is written out, Spark rearranges partition
+            # columns to appear first.
+            actual = ps.read_parquet(tmp)
+            self.assertFalse((actual.columns == self.test_column_order).all())
+            actual = actual[self.test_column_order]
+            self.assert_eq(
+                actual.sort_values(by="f").to_spark().toPandas(),
+                expected.sort_values(by="f").to_spark().toPandas(),
+            )
+
+            # Write out partitioned by two columns
+            expected.to_parquet(tmp, mode="overwrite", partition_cols=["i32", "bhello"])
+            # Reset column order, as once the data is written out, Spark rearranges partition
+            # columns to appear first.
+            actual = ps.read_parquet(tmp)
+            self.assertFalse((actual.columns == self.test_column_order).all())
+            actual = actual[self.test_column_order]
+            self.assert_eq(
+                actual.sort_values(by="f").to_spark().toPandas(),
+                expected.sort_values(by="f").to_spark().toPandas(),
+            )
+
+    def test_table(self):
+        with self.table("test_table"):
+            pdf = self.test_pdf
+            expected = ps.DataFrame(pdf)
+
+            # Write out partitioned by one column
+            expected.spark.to_table("test_table", mode="overwrite", partition_cols="i32")
+            # Reset column order, as once the data is written out, Spark rearranges partition
+            # columns to appear first.
+            actual = ps.read_table("test_table")
+            self.assertFalse((actual.columns == self.test_column_order).all())
+            actual = actual[self.test_column_order]
+            self.assert_eq(
+                actual.sort_values(by="f").to_spark().toPandas(),
+                expected.sort_values(by="f").to_spark().toPandas(),
+            )
+
+            # Write out partitioned by two columns
+            expected.to_table("test_table", mode="overwrite", partition_cols=["i32", "bhello"])
+            # Reset column order, as once the data is written out, Spark rearranges partition
+            # columns to appear first.
+            actual = ps.read_table("test_table")
+            self.assertFalse((actual.columns == self.test_column_order).all())
+            actual = actual[self.test_column_order]
+            self.assert_eq(
+                actual.sort_values(by="f").to_spark().toPandas(),
+                expected.sort_values(by="f").to_spark().toPandas(),
+            )
+
+            # When index columns are known
+            expected_idx = expected.set_index("bhello")[["f", "i32", "i64"]]
+            actual_idx = ps.read_table("test_table", index_col="bhello")[["f", "i32", "i64"]]
+            self.assert_eq(
+                actual_idx.sort_values(by="f").to_spark().toPandas(),
+                expected_idx.sort_values(by="f").to_spark().toPandas(),
+            )
+
+            expected_idx = expected.set_index(["bhello"])[["f", "i32", "i64"]]
+            actual_idx = ps.read_table("test_table", index_col=["bhello"])[["f", "i32", "i64"]]
+            self.assert_eq(
+                actual_idx.sort_values(by="f").to_spark().toPandas(),
+                expected_idx.sort_values(by="f").to_spark().toPandas(),
+            )
+
+            expected_idx = expected.set_index(["i32", "bhello"])[["f", "i64"]]
+            actual_idx = ps.read_table("test_table", index_col=["i32", "bhello"])[["f", "i64"]]
+            self.assert_eq(
+                actual_idx.sort_values(by="f").to_spark().toPandas(),
+                expected_idx.sort_values(by="f").to_spark().toPandas(),
+            )
+
+    def test_spark_io(self):
+        with self.temp_dir() as tmp:
+            pdf = self.test_pdf
+            expected = ps.DataFrame(pdf)
+
+            # Write out partitioned by one column
+            expected.to_spark_io(tmp, format="json", mode="overwrite", partition_cols="i32")
+            # Reset column order, as once the data is written out, Spark rearranges partition
+            # columns to appear first.
+            actual = ps.read_spark_io(tmp, format="json")
+            self.assertFalse((actual.columns == self.test_column_order).all())
+            actual = actual[self.test_column_order]
+            self.assert_eq(
+                actual.sort_values(by="f").to_spark().toPandas(),
+                expected.sort_values(by="f").to_spark().toPandas(),
+            )
+
+            # Write out partitioned by two columns
+            expected.to_spark_io(
+                tmp, format="json", mode="overwrite", partition_cols=["i32", "bhello"]
+            )
+            # Reset column order, as once the data is written out, Spark rearranges partition
+            # columns to appear first.
+            actual = ps.read_spark_io(path=tmp, format="json")
+            self.assertFalse((actual.columns == self.test_column_order).all())
+            actual = actual[self.test_column_order]
+            self.assert_eq(
+                actual.sort_values(by="f").to_spark().toPandas(),
+                expected.sort_values(by="f").to_spark().toPandas(),
+            )
+
+            # When index columns are known
+            pdf = self.test_pdf
+            expected = ps.DataFrame(pdf)
+            col_order = ["f", "i32", "i64"]
+
+            expected_idx = expected.set_index("bhello")[col_order]
+            actual_idx = ps.read_spark_io(tmp, format="json", index_col="bhello")[col_order]
+            self.assert_eq(
+                actual_idx.sort_values(by="f").to_spark().toPandas(),
+                expected_idx.sort_values(by="f").to_spark().toPandas(),
+            )
+
+    @unittest.skip("openpyxl")
+    def test_read_excel(self):
+        with self.temp_dir() as tmp:
+
+            path1 = "{}/file1.xlsx".format(tmp)
+            self.test_pdf[["i32"]].to_excel(path1)
+
+            self.assert_eq(ps.read_excel(open(path1, "rb")), pd.read_excel(open(path1, "rb")))
+            self.assert_eq(
+                ps.read_excel(open(path1, "rb"), index_col=0),
+                pd.read_excel(open(path1, "rb"), index_col=0),
+            )
+            self.assert_eq(
+                ps.read_excel(open(path1, "rb"), index_col=0, squeeze=True),
+                pd.read_excel(open(path1, "rb"), index_col=0, squeeze=True),
+            )
+
+            self.assert_eq(ps.read_excel(path1), pd.read_excel(path1))
+            self.assert_eq(ps.read_excel(path1, index_col=0), pd.read_excel(path1, index_col=0))
+            self.assert_eq(
+                ps.read_excel(path1, index_col=0, squeeze=True),
+                pd.read_excel(path1, index_col=0, squeeze=True),
+            )
+
+            self.assert_eq(ps.read_excel(tmp), pd.read_excel(path1))
+
+            path2 = "{}/file2.xlsx".format(tmp)
+            self.test_pdf[["i32"]].to_excel(path2)
+            self.assert_eq(
+                ps.read_excel(tmp, index_col=0).sort_index(),
+                pd.concat(
+                    [pd.read_excel(path1, index_col=0), pd.read_excel(path2, index_col=0)]
+                ).sort_index(),
+            )
+            self.assert_eq(
+                ps.read_excel(tmp, index_col=0, squeeze=True).sort_index(),
+                pd.concat(
+                    [
+                        pd.read_excel(path1, index_col=0, squeeze=True),
+                        pd.read_excel(path2, index_col=0, squeeze=True),
+                    ]
+                ).sort_index(),
+            )
+
+        with self.temp_dir() as tmp:
+            path1 = "{}/file1.xlsx".format(tmp)
+            with pd.ExcelWriter(path1) as writer:
+                self.test_pdf.to_excel(writer, sheet_name="Sheet_name_1")
+                self.test_pdf[["i32"]].to_excel(writer, sheet_name="Sheet_name_2")
+
+            sheet_names = [["Sheet_name_1", "Sheet_name_2"], None]
+
+            pdfs1 = pd.read_excel(open(path1, "rb"), sheet_name=None, index_col=0)
+            pdfs1_squeezed = pd.read_excel(
+                open(path1, "rb"), sheet_name=None, index_col=0, squeeze=True
+            )
+
+            for sheet_name in sheet_names:
+                kdfs = ps.read_excel(open(path1, "rb"), sheet_name=sheet_name, index_col=0)
+                self.assert_eq(kdfs["Sheet_name_1"], pdfs1["Sheet_name_1"])
+                self.assert_eq(kdfs["Sheet_name_2"], pdfs1["Sheet_name_2"])
+
+                kdfs = ps.read_excel(
+                    open(path1, "rb"), sheet_name=sheet_name, index_col=0, squeeze=True
+                )
+                self.assert_eq(kdfs["Sheet_name_1"], pdfs1_squeezed["Sheet_name_1"])
+                self.assert_eq(kdfs["Sheet_name_2"], pdfs1_squeezed["Sheet_name_2"])
+
+            self.assert_eq(
+                ps.read_excel(tmp, index_col=0, sheet_name="Sheet_name_2"),
+                pdfs1["Sheet_name_2"],
+            )
+
+            for sheet_name in sheet_names:
+                kdfs = ps.read_excel(tmp, sheet_name=sheet_name, index_col=0)
+                self.assert_eq(kdfs["Sheet_name_1"], pdfs1["Sheet_name_1"])
+                self.assert_eq(kdfs["Sheet_name_2"], pdfs1["Sheet_name_2"])
+
+                kdfs = ps.read_excel(tmp, sheet_name=sheet_name, index_col=0, squeeze=True)
+                self.assert_eq(kdfs["Sheet_name_1"], pdfs1_squeezed["Sheet_name_1"])
+                self.assert_eq(kdfs["Sheet_name_2"], pdfs1_squeezed["Sheet_name_2"])
+
+            path2 = "{}/file2.xlsx".format(tmp)
+            with pd.ExcelWriter(path2) as writer:
+                self.test_pdf.to_excel(writer, sheet_name="Sheet_name_1")
+                self.test_pdf[["i32"]].to_excel(writer, sheet_name="Sheet_name_2")
+
+            pdfs2 = pd.read_excel(path2, sheet_name=None, index_col=0)
+            pdfs2_squeezed = pd.read_excel(path2, sheet_name=None, index_col=0, squeeze=True)
+
+            self.assert_eq(
+                ps.read_excel(tmp, sheet_name="Sheet_name_2", index_col=0).sort_index(),
+                pd.concat([pdfs1["Sheet_name_2"], pdfs2["Sheet_name_2"]]).sort_index(),
+            )
+            self.assert_eq(
+                ps.read_excel(
+                    tmp, sheet_name="Sheet_name_2", index_col=0, squeeze=True
+                ).sort_index(),
+                pd.concat(
+                    [pdfs1_squeezed["Sheet_name_2"], pdfs2_squeezed["Sheet_name_2"]]
+                ).sort_index(),
+            )
+
+            for sheet_name in sheet_names:
+                kdfs = ps.read_excel(tmp, sheet_name=sheet_name, index_col=0)
+                self.assert_eq(
+                    kdfs["Sheet_name_1"].sort_index(),
+                    pd.concat([pdfs1["Sheet_name_1"], pdfs2["Sheet_name_1"]]).sort_index(),
+                )
+                self.assert_eq(
+                    kdfs["Sheet_name_2"].sort_index(),
+                    pd.concat([pdfs1["Sheet_name_2"], pdfs2["Sheet_name_2"]]).sort_index(),
+                )
+
+                kdfs = ps.read_excel(tmp, sheet_name=sheet_name, index_col=0, squeeze=True)
+                self.assert_eq(
+                    kdfs["Sheet_name_1"].sort_index(),
+                    pd.concat(
+                        [pdfs1_squeezed["Sheet_name_1"], pdfs2_squeezed["Sheet_name_1"]]
+                    ).sort_index(),
+                )
+                self.assert_eq(
+                    kdfs["Sheet_name_2"].sort_index(),
+                    pd.concat(
+                        [pdfs1_squeezed["Sheet_name_2"], pdfs2_squeezed["Sheet_name_2"]]
+                    ).sort_index(),
+                )
+
+    def test_read_orc(self):
+        with self.temp_dir() as tmp:
+            path = "{}/file1.orc".format(tmp)
+            data = self.test_pdf
+            self.spark.createDataFrame(data, "i32 int, i64 long, f double, bhello string").coalesce(
+                1
+            ).write.orc(path, mode="overwrite")
+
+            # `spark.write.orc` create a directory contains distributed orc files.
+            # But pandas only can read from file, not directory. Therefore, we need orc file path.
+            orc_file_path = glob.glob(os.path.join(path, "*.orc"))[0]
+
+            expected = data.reset_index()[data.columns]
+            actual = ps.read_orc(path)
+            self.assertPandasEqual(expected, actual.to_pandas())
+
+            # columns
+            columns = ["i32", "i64"]
+            expected = data.reset_index()[columns]
+            actual = ps.read_orc(path, columns=columns)
+            self.assertPandasEqual(expected, actual.to_pandas())
+
+            # index_col
+            expected = data.set_index("i32")
+            actual = ps.read_orc(path, index_col="i32")
+            self.assert_eq(actual, expected)
+
+            expected = data.set_index(["i32", "f"])
+            actual = ps.read_orc(path, index_col=["i32", "f"])
+            self.assert_eq(actual, expected)
+
+            # index_col with columns
+            expected = data.set_index("i32")[["i64", "bhello"]]
+            actual = ps.read_orc(path, index_col=["i32"], columns=["i64", "bhello"])
+            self.assert_eq(actual, expected)
+
+            expected = data.set_index(["i32", "f"])[["bhello", "i64"]]
+            actual = ps.read_orc(path, index_col=["i32", "f"], columns=["bhello", "i64"])
+            self.assert_eq(actual, expected)
+
+            msg = "Unknown column name 'i'"
+            with self.assertRaises(ValueError, msg=msg):
+                ps.read_orc(path, columns="i32")
+            msg = "Unknown column name 'i34'"
+            with self.assertRaises(ValueError, msg=msg):
+                ps.read_orc(path, columns=["i34", "i64"])
+
+    def test_orc_write(self):
+        with self.temp_dir() as tmp:
+            pdf = self.test_pdf
+            expected = ps.DataFrame(pdf)
+
+            # Write out partitioned by one column
+            expected.to_orc(tmp, mode="overwrite", partition_cols="i32")
+            # Reset column order, as once the data is written out, Spark rearranges partition
+            # columns to appear first.
+            actual = ps.read_orc(tmp)
+            self.assertFalse((actual.columns == self.test_column_order).all())
+            actual = actual[self.test_column_order]
+            self.assert_eq(
+                actual.sort_values(by="f").to_spark().toPandas(),
+                expected.sort_values(by="f").to_spark().toPandas(),
+            )
+
+            # Write out partitioned by two columns
+            expected.to_orc(tmp, mode="overwrite", partition_cols=["i32", "bhello"])
+            # Reset column order, as once the data is written out, Spark rearranges partition
+            # columns to appear first.
+            actual = ps.read_orc(tmp)
+            self.assertFalse((actual.columns == self.test_column_order).all())
+            actual = actual[self.test_column_order]
+            self.assert_eq(
+                actual.sort_values(by="f").to_spark().toPandas(),
+                expected.sort_values(by="f").to_spark().toPandas(),
+            )
+
+
+if __name__ == "__main__":
+    from pyspark.pandas.tests.test_dataframe_spark_io import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_default_index.py b/python/pyspark/pandas/tests/test_default_index.py
new file mode 100644
index 0000000000000..838e04a9eb241
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_default_index.py
@@ -0,0 +1,51 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import pandas as pd
+
+from pyspark import pandas as ps
+from pyspark.testing.pandasutils import PandasOnSparkTestCase
+
+
+class DefaultIndexTest(PandasOnSparkTestCase):
+    def test_default_index_sequence(self):
+        with ps.option_context("compute.default_index_type", "sequence"):
+            sdf = self.spark.range(1000)
+            self.assert_eq(ps.DataFrame(sdf), pd.DataFrame({"id": list(range(1000))}))
+
+    def test_default_index_distributed_sequence(self):
+        with ps.option_context("compute.default_index_type", "distributed-sequence"):
+            sdf = self.spark.range(1000)
+            self.assert_eq(ps.DataFrame(sdf), pd.DataFrame({"id": list(range(1000))}))
+
+    def test_default_index_distributed(self):
+        with ps.option_context("compute.default_index_type", "distributed"):
+            sdf = self.spark.range(1000)
+            pdf = ps.DataFrame(sdf).to_pandas()
+            self.assertEqual(len(set(pdf.index)), len(pdf))
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.pandas.tests.test_default_index import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_expanding.py b/python/pyspark/pandas/tests/test_expanding.py
new file mode 100644
index 0000000000000..7198a1d5d0aa7
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_expanding.py
@@ -0,0 +1,306 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from distutils.version import LooseVersion
+
+import numpy as np
+import pandas as pd
+
+import pyspark.pandas as ps
+from pyspark.pandas.window import Expanding
+from pyspark.testing.pandasutils import PandasOnSparkTestCase, TestUtils
+
+
+class ExpandingTest(PandasOnSparkTestCase, TestUtils):
+    def _test_expanding_func(self, f):
+        pser = pd.Series([1, 2, 3], index=np.random.rand(3))
+        kser = ps.from_pandas(pser)
+        self.assert_eq(getattr(kser.expanding(2), f)(), getattr(pser.expanding(2), f)())
+        self.assert_eq(getattr(kser.expanding(2), f)().sum(), getattr(pser.expanding(2), f)().sum())
+
+        # Multiindex
+        pser = pd.Series(
+            [1, 2, 3], index=pd.MultiIndex.from_tuples([("a", "x"), ("a", "y"), ("b", "z")])
+        )
+        kser = ps.from_pandas(pser)
+        self.assert_eq(getattr(kser.expanding(2), f)(), getattr(pser.expanding(2), f)())
+
+        pdf = pd.DataFrame(
+            {"a": [1.0, 2.0, 3.0, 2.0], "b": [4.0, 2.0, 3.0, 1.0]}, index=np.random.rand(4)
+        )
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(getattr(kdf.expanding(2), f)(), getattr(pdf.expanding(2), f)())
+        self.assert_eq(getattr(kdf.expanding(2), f)().sum(), getattr(pdf.expanding(2), f)().sum())
+
+        # Multiindex column
+        columns = pd.MultiIndex.from_tuples([("a", "x"), ("a", "y")])
+        pdf.columns = columns
+        kdf.columns = columns
+        self.assert_eq(getattr(kdf.expanding(2), f)(), getattr(pdf.expanding(2), f)())
+
+    def test_expanding_error(self):
+        with self.assertRaisesRegex(ValueError, "min_periods must be >= 0"):
+            ps.range(10).expanding(-1)
+
+        with self.assertRaisesRegex(
+            TypeError, "kdf_or_kser must be a series or dataframe; however, got:.*int"
+        ):
+            Expanding(1, 2)
+
+    def test_expanding_repr(self):
+        self.assertEqual(repr(ps.range(10).expanding(5)), "Expanding [min_periods=5]")
+
+    def test_expanding_count(self):
+        # The behaviour of Expanding.count are different between pandas>=1.0.0 and lower,
+        # and we're following the behaviour of latest version of pandas.
+        if LooseVersion(pd.__version__) >= LooseVersion("1.0.0"):
+            self._test_expanding_func("count")
+        else:
+            # Series
+            idx = np.random.rand(3)
+            kser = ps.Series([1, 2, 3], index=idx, name="a")
+            expected_result = pd.Series([None, 2.0, 3.0], index=idx, name="a")
+            self.assert_eq(kser.expanding(2).count().sort_index(), expected_result.sort_index())
+            self.assert_eq(kser.expanding(2).count().sum(), expected_result.sum())
+
+            # MultiIndex
+            midx = pd.MultiIndex.from_tuples([("a", "x"), ("a", "y"), ("b", "z")])
+            kser = ps.Series([1, 2, 3], index=midx, name="a")
+            expected_result = pd.Series([None, 2.0, 3.0], index=midx, name="a")
+            self.assert_eq(kser.expanding(2).count().sort_index(), expected_result.sort_index())
+
+            # DataFrame
+            kdf = ps.DataFrame({"a": [1, 2, 3, 2], "b": [4.0, 2.0, 3.0, 1.0]})
+            expected_result = pd.DataFrame({"a": [None, 2.0, 3.0, 4.0], "b": [None, 2.0, 3.0, 4.0]})
+            self.assert_eq(kdf.expanding(2).count().sort_index(), expected_result.sort_index())
+            self.assert_eq(kdf.expanding(2).count().sum(), expected_result.sum())
+
+            # MultiIndex columns
+            idx = np.random.rand(4)
+            kdf = ps.DataFrame({"a": [1, 2, 3, 2], "b": [4.0, 2.0, 3.0, 1.0]}, index=idx)
+            kdf.columns = pd.MultiIndex.from_tuples([("a", "x"), ("a", "y")])
+            expected_result = pd.DataFrame(
+                {("a", "x"): [None, 2.0, 3.0, 4.0], ("a", "y"): [None, 2.0, 3.0, 4.0]}, index=idx,
+            )
+            self.assert_eq(kdf.expanding(2).count().sort_index(), expected_result.sort_index())
+
+    def test_expanding_min(self):
+        self._test_expanding_func("min")
+
+    def test_expanding_max(self):
+        self._test_expanding_func("max")
+
+    def test_expanding_mean(self):
+        self._test_expanding_func("mean")
+
+    def test_expanding_sum(self):
+        self._test_expanding_func("sum")
+
+    def test_expanding_std(self):
+        self._test_expanding_func("std")
+
+    def test_expanding_var(self):
+        self._test_expanding_func("var")
+
+    def _test_groupby_expanding_func(self, f):
+        pser = pd.Series([1, 2, 3, 2], index=np.random.rand(4), name="a")
+        kser = ps.from_pandas(pser)
+        self.assert_eq(
+            getattr(kser.groupby(kser).expanding(2), f)().sort_index(),
+            getattr(pser.groupby(pser).expanding(2), f)().sort_index(),
+        )
+        self.assert_eq(
+            getattr(kser.groupby(kser).expanding(2), f)().sum(),
+            getattr(pser.groupby(pser).expanding(2), f)().sum(),
+        )
+
+        # Multiindex
+        pser = pd.Series(
+            [1, 2, 3, 2],
+            index=pd.MultiIndex.from_tuples([("a", "x"), ("a", "y"), ("b", "z"), ("c", "z")]),
+            name="a",
+        )
+        kser = ps.from_pandas(pser)
+        self.assert_eq(
+            getattr(kser.groupby(kser).expanding(2), f)().sort_index(),
+            getattr(pser.groupby(pser).expanding(2), f)().sort_index(),
+        )
+
+        pdf = pd.DataFrame({"a": [1.0, 2.0, 3.0, 2.0], "b": [4.0, 2.0, 3.0, 1.0]})
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(
+            getattr(kdf.groupby(kdf.a).expanding(2), f)().sort_index(),
+            getattr(pdf.groupby(pdf.a).expanding(2), f)().sort_index(),
+        )
+        self.assert_eq(
+            getattr(kdf.groupby(kdf.a).expanding(2), f)().sum(),
+            getattr(pdf.groupby(pdf.a).expanding(2), f)().sum(),
+        )
+        self.assert_eq(
+            getattr(kdf.groupby(kdf.a + 1).expanding(2), f)().sort_index(),
+            getattr(pdf.groupby(pdf.a + 1).expanding(2), f)().sort_index(),
+        )
+        self.assert_eq(
+            getattr(kdf.b.groupby(kdf.a).expanding(2), f)().sort_index(),
+            getattr(pdf.b.groupby(pdf.a).expanding(2), f)().sort_index(),
+        )
+        self.assert_eq(
+            getattr(kdf.groupby(kdf.a)["b"].expanding(2), f)().sort_index(),
+            getattr(pdf.groupby(pdf.a)["b"].expanding(2), f)().sort_index(),
+        )
+        self.assert_eq(
+            getattr(kdf.groupby(kdf.a)[["b"]].expanding(2), f)().sort_index(),
+            getattr(pdf.groupby(pdf.a)[["b"]].expanding(2), f)().sort_index(),
+        )
+
+        # Multiindex column
+        columns = pd.MultiIndex.from_tuples([("a", "x"), ("a", "y")])
+        pdf.columns = columns
+        kdf.columns = columns
+        self.assert_eq(
+            getattr(kdf.groupby(("a", "x")).expanding(2), f)().sort_index(),
+            getattr(pdf.groupby(("a", "x")).expanding(2), f)().sort_index(),
+        )
+
+        self.assert_eq(
+            getattr(kdf.groupby([("a", "x"), ("a", "y")]).expanding(2), f)().sort_index(),
+            getattr(pdf.groupby([("a", "x"), ("a", "y")]).expanding(2), f)().sort_index(),
+        )
+
+    def test_groupby_expanding_count(self):
+        # The behaviour of ExpandingGroupby.count are different between pandas>=1.0.0 and lower,
+        # and we're following the behaviour of latest version of pandas.
+        if LooseVersion(pd.__version__) >= LooseVersion("1.0.0"):
+            self._test_groupby_expanding_func("count")
+        else:
+            # Series
+            kser = ps.Series([1, 2, 3, 2], index=np.random.rand(4))
+            midx = pd.MultiIndex.from_tuples(
+                list(zip(kser.to_pandas().values, kser.index.to_pandas().values))
+            )
+            expected_result = pd.Series([np.nan, np.nan, np.nan, 2], index=midx)
+            self.assert_eq(
+                kser.groupby(kser).expanding(2).count().sort_index(), expected_result.sort_index()
+            )
+            self.assert_eq(kser.groupby(kser).expanding(2).count().sum(), expected_result.sum())
+
+            # MultiIndex
+            kser = ps.Series(
+                [1, 2, 3, 2],
+                index=pd.MultiIndex.from_tuples([("a", "x"), ("a", "y"), ("b", "z"), ("a", "y")]),
+            )
+            midx = pd.MultiIndex.from_tuples(
+                [(1, "a", "x"), (2, "a", "y"), (3, "b", "z"), (2, "a", "y")]
+            )
+            expected_result = pd.Series([np.nan, np.nan, np.nan, 2], index=midx)
+            self.assert_eq(
+                kser.groupby(kser).expanding(2).count().sort_index(), expected_result.sort_index()
+            )
+
+            # DataFrame
+            kdf = ps.DataFrame({"a": [1, 2, 3, 2], "b": [4.0, 2.0, 3.0, 1.0]})
+            midx = pd.MultiIndex.from_tuples([(1, 0), (2, 1), (2, 3), (3, 2)], names=["a", None])
+            expected_result = pd.DataFrame(
+                {"a": [None, None, 2.0, None], "b": [None, None, 2.0, None]}, index=midx
+            )
+            self.assert_eq(
+                kdf.groupby(kdf.a).expanding(2).count().sort_index(), expected_result.sort_index()
+            )
+            self.assert_eq(kdf.groupby(kdf.a).expanding(2).count().sum(), expected_result.sum())
+            expected_result = pd.DataFrame(
+                {"a": [None, None, 2.0, None], "b": [None, None, 2.0, None]},
+                index=pd.MultiIndex.from_tuples(
+                    [(2, 0), (3, 1), (3, 3), (4, 2)], names=["a", None]
+                ),
+            )
+            self.assert_eq(
+                kdf.groupby(kdf.a + 1).expanding(2).count().sort_index(),
+                expected_result.sort_index(),
+            )
+            expected_result = pd.Series([None, None, 2.0, None], index=midx, name="b")
+            self.assert_eq(
+                kdf.b.groupby(kdf.a).expanding(2).count().sort_index(), expected_result.sort_index()
+            )
+            self.assert_eq(
+                kdf.groupby(kdf.a)["b"].expanding(2).count().sort_index(),
+                expected_result.sort_index(),
+            )
+            expected_result = pd.DataFrame({"b": [None, None, 2.0, None]}, index=midx)
+            self.assert_eq(
+                kdf.groupby(kdf.a)[["b"]].expanding(2).count().sort_index(),
+                expected_result.sort_index(),
+            )
+
+            # MultiIndex column
+            kdf = ps.DataFrame({"a": [1, 2, 3, 2], "b": [4.0, 2.0, 3.0, 1.0]})
+            kdf.columns = pd.MultiIndex.from_tuples([("a", "x"), ("a", "y")])
+            midx = pd.MultiIndex.from_tuples(
+                [(1, 0), (2, 1), (2, 3), (3, 2)], names=[("a", "x"), None]
+            )
+            expected_result = pd.DataFrame(
+                {"a": [None, None, 2.0, None], "b": [None, None, 2.0, None]}, index=midx
+            )
+            expected_result.columns = pd.MultiIndex.from_tuples([("a", "x"), ("a", "y")])
+            self.assert_eq(
+                kdf.groupby(("a", "x")).expanding(2).count().sort_index(),
+                expected_result.sort_index(),
+            )
+            midx = pd.MultiIndex.from_tuples(
+                [(1, 4.0, 0), (2, 1.0, 3), (2, 2.0, 1), (3, 3.0, 2)],
+                names=[("a", "x"), ("a", "y"), None],
+            )
+            expected_result = pd.DataFrame(
+                {
+                    ("a", "x"): [np.nan, np.nan, np.nan, np.nan],
+                    ("a", "y"): [np.nan, np.nan, np.nan, np.nan],
+                },
+                index=midx,
+            )
+            self.assert_eq(
+                kdf.groupby([("a", "x"), ("a", "y")]).expanding(2).count().sort_index(),
+                expected_result.sort_index(),
+            )
+
+    def test_groupby_expanding_min(self):
+        self._test_groupby_expanding_func("min")
+
+    def test_groupby_expanding_max(self):
+        self._test_groupby_expanding_func("max")
+
+    def test_groupby_expanding_mean(self):
+        self._test_groupby_expanding_func("mean")
+
+    def test_groupby_expanding_sum(self):
+        self._test_groupby_expanding_func("sum")
+
+    def test_groupby_expanding_std(self):
+        self._test_groupby_expanding_func("std")
+
+    def test_groupby_expanding_var(self):
+        self._test_groupby_expanding_func("var")
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.pandas.tests.test_expanding import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_extension.py b/python/pyspark/pandas/tests/test_extension.py
new file mode 100644
index 0000000000000..17dc2bcd8b2be
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_extension.py
@@ -0,0 +1,151 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import contextlib
+
+import numpy as np
+import pandas as pd
+
+from pyspark import pandas as ps
+from pyspark.testing.pandasutils import assert_produces_warning, PandasOnSparkTestCase
+from pyspark.pandas.extensions import (
+    register_dataframe_accessor,
+    register_series_accessor,
+    register_index_accessor,
+)
+
+
+@contextlib.contextmanager
+def ensure_removed(obj, attr):
+    """
+    Ensure attribute attached to 'obj' during testing is removed in the end
+    """
+    try:
+        yield
+
+    finally:
+        try:
+            delattr(obj, attr)
+        except AttributeError:
+            pass
+
+
+class CustomAccessor:
+    def __init__(self, obj):
+        self.obj = obj
+        self.item = "item"
+
+    @property
+    def prop(self):
+        return self.item
+
+    def method(self):
+        return self.item
+
+    def check_length(self, col=None):
+        if type(self.obj) == ps.DataFrame or col is not None:
+            return len(self.obj[col])
+        else:
+            try:
+                return len(self.obj)
+            except Exception as e:
+                raise ValueError(str(e))
+
+
+class ExtensionTest(PandasOnSparkTestCase):
+    @property
+    def pdf(self):
+        return pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5, 6, 7, 8, 9], "b": [4, 5, 6, 3, 2, 1, 0, 0, 0]},
+            index=np.random.rand(9),
+        )
+
+    @property
+    def kdf(self):
+        return ps.from_pandas(self.pdf)
+
+    @property
+    def accessor(self):
+        return CustomAccessor(self.kdf)
+
+    def test_setup(self):
+        self.assertEqual("item", self.accessor.item)
+
+    def test_dataframe_register(self):
+        with ensure_removed(ps.DataFrame, "test"):
+            register_dataframe_accessor("test")(CustomAccessor)
+            assert self.kdf.test.prop == "item"
+            assert self.kdf.test.method() == "item"
+            assert len(self.kdf["a"]) == self.kdf.test.check_length("a")
+
+    def test_series_register(self):
+        with ensure_removed(ps.Series, "test"):
+            register_series_accessor("test")(CustomAccessor)
+            assert self.kdf.a.test.prop == "item"
+            assert self.kdf.a.test.method() == "item"
+            assert self.kdf.a.test.check_length() == len(self.kdf["a"])
+
+    def test_index_register(self):
+        with ensure_removed(ps.Index, "test"):
+            register_index_accessor("test")(CustomAccessor)
+            assert self.kdf.index.test.prop == "item"
+            assert self.kdf.index.test.method() == "item"
+            assert self.kdf.index.test.check_length() == self.kdf.index.size
+
+    def test_accessor_works(self):
+        register_series_accessor("test")(CustomAccessor)
+
+        s = ps.Series([1, 2])
+        assert s.test.obj is s
+        assert s.test.prop == "item"
+        assert s.test.method() == "item"
+
+    def test_overwrite_warns(self):
+        mean = ps.Series.mean
+        try:
+            with assert_produces_warning(UserWarning, raise_on_extra_warnings=False) as w:
+                register_series_accessor("mean")(CustomAccessor)
+                s = ps.Series([1, 2])
+                assert s.mean.prop == "item"
+            msg = str(w[0].message)
+            assert "mean" in msg
+            assert "CustomAccessor" in msg
+            assert "Series" in msg
+        finally:
+            ps.Series.mean = mean
+
+    def test_raises_attr_error(self):
+        with ensure_removed(ps.Series, "bad"):
+
+            class Bad:
+                def __init__(self, data):
+                    raise AttributeError("whoops")
+
+            with self.assertRaises(AttributeError):
+                ps.Series([1, 2], dtype=object).bad
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.pandas.tests.test_extension import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_frame_spark.py b/python/pyspark/pandas/tests/test_frame_spark.py
new file mode 100644
index 0000000000000..6a226a740fede
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_frame_spark.py
@@ -0,0 +1,153 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+
+import pandas as pd
+
+from pyspark import pandas as ps
+from pyspark.testing.pandasutils import PandasOnSparkTestCase, TestUtils
+from pyspark.testing.sqlutils import SQLTestUtils
+
+
+class SparkFrameMethodsTest(PandasOnSparkTestCase, SQLTestUtils, TestUtils):
+    def test_frame_apply_negative(self):
+        with self.assertRaisesRegex(
+            ValueError, "The output of the function.* pyspark.sql.DataFrame.*int"
+        ):
+            ps.range(10).spark.apply(lambda scol: 1)
+
+    def test_hint(self):
+        pdf1 = pd.DataFrame(
+            {"lkey": ["foo", "bar", "baz", "foo"], "value": [1, 2, 3, 5]}
+        ).set_index("lkey")
+        pdf2 = pd.DataFrame(
+            {"rkey": ["foo", "bar", "baz", "foo"], "value": [5, 6, 7, 8]}
+        ).set_index("rkey")
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        hints = ["broadcast", "merge", "shuffle_hash", "shuffle_replicate_nl"]
+
+        for hint in hints:
+            self.assert_eq(
+                pdf1.merge(pdf2, left_index=True, right_index=True).sort_values(
+                    ["value_x", "value_y"]
+                ),
+                kdf1.merge(kdf2.spark.hint(hint), left_index=True, right_index=True).sort_values(
+                    ["value_x", "value_y"]
+                ),
+                almost=True,
+            )
+            self.assert_eq(
+                pdf1.merge(pdf2 + 1, left_index=True, right_index=True).sort_values(
+                    ["value_x", "value_y"]
+                ),
+                kdf1.merge(
+                    (kdf2 + 1).spark.hint(hint), left_index=True, right_index=True
+                ).sort_values(["value_x", "value_y"]),
+                almost=True,
+            )
+
+    def test_repartition(self):
+        kdf = ps.DataFrame({"age": [5, 5, 2, 2], "name": ["Bob", "Bob", "Alice", "Alice"]})
+        num_partitions = kdf.to_spark().rdd.getNumPartitions() + 1
+
+        num_partitions += 1
+        new_kdf = kdf.spark.repartition(num_partitions)
+        self.assertEqual(new_kdf.to_spark().rdd.getNumPartitions(), num_partitions)
+        self.assert_eq(kdf.sort_index(), new_kdf.sort_index())
+
+        # Reserves Index
+        kdf = kdf.set_index("age")
+        num_partitions += 1
+        new_kdf = kdf.spark.repartition(num_partitions)
+        self.assertEqual(new_kdf.to_spark().rdd.getNumPartitions(), num_partitions)
+        self.assert_eq(kdf.sort_index(), new_kdf.sort_index())
+
+        # Reflects internal changes
+        kdf = kdf.reset_index()
+        kdf = kdf.set_index("name")
+        kdf2 = kdf + 1
+        num_partitions += 1
+        self.assert_eq(kdf2.sort_index(), (kdf + 1).spark.repartition(num_partitions).sort_index())
+
+        # Reserves MultiIndex
+        kdf = ps.DataFrame({"a": ["a", "b", "c"]}, index=[[1, 2, 3], [4, 5, 6]])
+        num_partitions = kdf.to_spark().rdd.getNumPartitions() + 1
+        new_kdf = kdf.spark.repartition(num_partitions)
+        self.assertEqual(new_kdf.to_spark().rdd.getNumPartitions(), num_partitions)
+        self.assert_eq(kdf.sort_index(), new_kdf.sort_index())
+
+    def test_coalesce(self):
+        num_partitions = 10
+        kdf = ps.DataFrame({"age": [5, 5, 2, 2], "name": ["Bob", "Bob", "Alice", "Alice"]})
+        kdf = kdf.spark.repartition(num_partitions)
+
+        num_partitions -= 1
+        new_kdf = kdf.spark.coalesce(num_partitions)
+        self.assertEqual(new_kdf.to_spark().rdd.getNumPartitions(), num_partitions)
+        self.assert_eq(kdf.sort_index(), new_kdf.sort_index())
+
+        # Reserves Index
+        kdf = kdf.set_index("age")
+        num_partitions -= 1
+        new_kdf = kdf.spark.coalesce(num_partitions)
+        self.assertEqual(new_kdf.to_spark().rdd.getNumPartitions(), num_partitions)
+        self.assert_eq(kdf.sort_index(), new_kdf.sort_index())
+
+        # Reflects internal changes
+        kdf = kdf.reset_index()
+        kdf = kdf.set_index("name")
+        kdf2 = kdf + 1
+        num_partitions -= 1
+        self.assert_eq(kdf2.sort_index(), (kdf + 1).spark.coalesce(num_partitions).sort_index())
+
+        # Reserves MultiIndex
+        kdf = ps.DataFrame({"a": ["a", "b", "c"]}, index=[[1, 2, 3], [4, 5, 6]])
+        num_partitions -= 1
+        kdf = kdf.spark.repartition(num_partitions)
+
+        num_partitions -= 1
+        new_kdf = kdf.spark.coalesce(num_partitions)
+        self.assertEqual(new_kdf.to_spark().rdd.getNumPartitions(), num_partitions)
+        self.assert_eq(kdf.sort_index(), new_kdf.sort_index())
+
+    def test_checkpoint(self):
+        with self.temp_dir() as tmp:
+            self.spark.sparkContext.setCheckpointDir(tmp)
+            kdf = ps.DataFrame({"a": ["a", "b", "c"]})
+            new_kdf = kdf.spark.checkpoint()
+            self.assertIsNotNone(os.listdir(tmp))
+            self.assert_eq(kdf, new_kdf)
+
+    def test_local_checkpoint(self):
+        kdf = ps.DataFrame({"a": ["a", "b", "c"]})
+        new_kdf = kdf.spark.local_checkpoint()
+        self.assert_eq(kdf, new_kdf)
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.pandas.tests.test_frame_spark import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_groupby.py b/python/pyspark/pandas/tests/test_groupby.py
new file mode 100644
index 0000000000000..dde3162604784
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_groupby.py
@@ -0,0 +1,2815 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import unittest
+import inspect
+from distutils.version import LooseVersion
+from itertools import product
+
+import numpy as np
+import pandas as pd
+
+from pyspark import pandas as ps
+from pyspark.pandas.config import option_context
+from pyspark.pandas.exceptions import PandasNotImplementedError, DataError
+from pyspark.pandas.missing.groupby import (
+    MissingPandasLikeDataFrameGroupBy,
+    MissingPandasLikeSeriesGroupBy,
+)
+from pyspark.pandas.groupby import is_multi_agg_with_relabel
+from pyspark.testing.pandasutils import PandasOnSparkTestCase, TestUtils
+
+
+class GroupByTest(PandasOnSparkTestCase, TestUtils):
+    def test_groupby_simple(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 2, 6, 4, 4, 6, 4, 3, 7],
+                "b": [4, 2, 7, 3, 3, 1, 1, 1, 2],
+                "c": [4, 2, 7, 3, None, 1, 1, 1, 2],
+                "d": list("abcdefght"),
+            },
+            index=[0, 1, 3, 5, 6, 8, 9, 9, 9],
+        )
+        kdf = ps.from_pandas(pdf)
+
+        for as_index in [True, False]:
+            if as_index:
+                sort = lambda df: df.sort_index()
+            else:
+                sort = lambda df: df.sort_values("a").reset_index(drop=True)
+            self.assert_eq(
+                sort(kdf.groupby("a", as_index=as_index).sum()),
+                sort(pdf.groupby("a", as_index=as_index).sum()),
+            )
+            self.assert_eq(
+                sort(kdf.groupby("a", as_index=as_index).b.sum()),
+                sort(pdf.groupby("a", as_index=as_index).b.sum()),
+            )
+            self.assert_eq(
+                sort(kdf.groupby("a", as_index=as_index)["b"].sum()),
+                sort(pdf.groupby("a", as_index=as_index)["b"].sum()),
+            )
+            self.assert_eq(
+                sort(kdf.groupby("a", as_index=as_index)[["b", "c"]].sum()),
+                sort(pdf.groupby("a", as_index=as_index)[["b", "c"]].sum()),
+            )
+            self.assert_eq(
+                sort(kdf.groupby("a", as_index=as_index)[[]].sum()),
+                sort(pdf.groupby("a", as_index=as_index)[[]].sum()),
+            )
+            self.assert_eq(
+                sort(kdf.groupby("a", as_index=as_index)["c"].sum()),
+                sort(pdf.groupby("a", as_index=as_index)["c"].sum()),
+            )
+
+        self.assert_eq(kdf.groupby("a").a.sum().sort_index(), pdf.groupby("a").a.sum().sort_index())
+        self.assert_eq(
+            kdf.groupby("a")["a"].sum().sort_index(), pdf.groupby("a")["a"].sum().sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby("a")[["a"]].sum().sort_index(), pdf.groupby("a")[["a"]].sum().sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby("a")[["a", "c"]].sum().sort_index(),
+            pdf.groupby("a")[["a", "c"]].sum().sort_index(),
+        )
+
+        self.assert_eq(
+            kdf.a.groupby(kdf.b).sum().sort_index(), pdf.a.groupby(pdf.b).sum().sort_index()
+        )
+
+        for axis in [0, "index"]:
+            self.assert_eq(
+                kdf.groupby("a", axis=axis).a.sum().sort_index(),
+                pdf.groupby("a", axis=axis).a.sum().sort_index(),
+            )
+            self.assert_eq(
+                kdf.groupby("a", axis=axis)["a"].sum().sort_index(),
+                pdf.groupby("a", axis=axis)["a"].sum().sort_index(),
+            )
+            self.assert_eq(
+                kdf.groupby("a", axis=axis)[["a"]].sum().sort_index(),
+                pdf.groupby("a", axis=axis)[["a"]].sum().sort_index(),
+            )
+            self.assert_eq(
+                kdf.groupby("a", axis=axis)[["a", "c"]].sum().sort_index(),
+                pdf.groupby("a", axis=axis)[["a", "c"]].sum().sort_index(),
+            )
+
+            self.assert_eq(
+                kdf.a.groupby(kdf.b, axis=axis).sum().sort_index(),
+                pdf.a.groupby(pdf.b, axis=axis).sum().sort_index(),
+            )
+
+        self.assertRaises(ValueError, lambda: kdf.groupby("a", as_index=False).a)
+        self.assertRaises(ValueError, lambda: kdf.groupby("a", as_index=False)["a"])
+        self.assertRaises(ValueError, lambda: kdf.groupby("a", as_index=False)[["a"]])
+        self.assertRaises(ValueError, lambda: kdf.groupby("a", as_index=False)[["a", "c"]])
+        self.assertRaises(KeyError, lambda: kdf.groupby("z", as_index=False)[["a", "c"]])
+        self.assertRaises(KeyError, lambda: kdf.groupby(["z"], as_index=False)[["a", "c"]])
+
+        self.assertRaises(TypeError, lambda: kdf.a.groupby(kdf.b, as_index=False))
+
+        self.assertRaises(NotImplementedError, lambda: kdf.groupby("a", axis=1))
+        self.assertRaises(NotImplementedError, lambda: kdf.groupby("a", axis="columns"))
+        self.assertRaises(ValueError, lambda: kdf.groupby("a", "b"))
+        self.assertRaises(TypeError, lambda: kdf.a.groupby(kdf.a, kdf.b))
+
+        # we can't use column name/names as a parameter `by` for `SeriesGroupBy`.
+        self.assertRaises(KeyError, lambda: kdf.a.groupby(by="a"))
+        self.assertRaises(KeyError, lambda: kdf.a.groupby(by=["a", "b"]))
+        self.assertRaises(KeyError, lambda: kdf.a.groupby(by=("a", "b")))
+
+        # we can't use DataFrame as a parameter `by` for `DataFrameGroupBy`/`SeriesGroupBy`.
+        self.assertRaises(ValueError, lambda: kdf.groupby(kdf))
+        self.assertRaises(ValueError, lambda: kdf.a.groupby(kdf))
+        self.assertRaises(ValueError, lambda: kdf.a.groupby((kdf,)))
+
+        # non-string names
+        pdf = pd.DataFrame(
+            {
+                10: [1, 2, 6, 4, 4, 6, 4, 3, 7],
+                20: [4, 2, 7, 3, 3, 1, 1, 1, 2],
+                30: [4, 2, 7, 3, None, 1, 1, 1, 2],
+                40: list("abcdefght"),
+            },
+            index=[0, 1, 3, 5, 6, 8, 9, 9, 9],
+        )
+        kdf = ps.from_pandas(pdf)
+
+        for as_index in [True, False]:
+            if as_index:
+                sort = lambda df: df.sort_index()
+            else:
+                sort = lambda df: df.sort_values(10).reset_index(drop=True)
+            self.assert_eq(
+                sort(kdf.groupby(10, as_index=as_index).sum()),
+                sort(pdf.groupby(10, as_index=as_index).sum()),
+            )
+            self.assert_eq(
+                sort(kdf.groupby(10, as_index=as_index)[20].sum()),
+                sort(pdf.groupby(10, as_index=as_index)[20].sum()),
+            )
+            self.assert_eq(
+                sort(kdf.groupby(10, as_index=as_index)[[20, 30]].sum()),
+                sort(pdf.groupby(10, as_index=as_index)[[20, 30]].sum()),
+            )
+
+    def test_groupby_multiindex_columns(self):
+        pdf = pd.DataFrame(
+            {
+                (10, "a"): [1, 2, 6, 4, 4, 6, 4, 3, 7],
+                (10, "b"): [4, 2, 7, 3, 3, 1, 1, 1, 2],
+                (20, "c"): [4, 2, 7, 3, None, 1, 1, 1, 2],
+                (30, "d"): list("abcdefght"),
+            },
+            index=[0, 1, 3, 5, 6, 8, 9, 9, 9],
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            kdf.groupby((10, "a")).sum().sort_index(), pdf.groupby((10, "a")).sum().sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby((10, "a"), as_index=False)
+            .sum()
+            .sort_values((10, "a"))
+            .reset_index(drop=True),
+            pdf.groupby((10, "a"), as_index=False)
+            .sum()
+            .sort_values((10, "a"))
+            .reset_index(drop=True),
+        )
+        self.assert_eq(
+            kdf.groupby((10, "a"))[[(20, "c")]].sum().sort_index(),
+            pdf.groupby((10, "a"))[[(20, "c")]].sum().sort_index(),
+        )
+
+        # TODO: a pandas bug?
+        #  expected = pdf.groupby((10, "a"))[(20, "c")].sum().sort_index()
+        expected = pd.Series(
+            [4.0, 2.0, 1.0, 4.0, 8.0, 2.0],
+            name=(20, "c"),
+            index=pd.Index([1, 2, 3, 4, 6, 7], name=(10, "a")),
+        )
+
+        self.assert_eq(kdf.groupby((10, "a"))[(20, "c")].sum().sort_index(), expected)
+
+        if LooseVersion(pd.__version__) < LooseVersion("1.1.3"):
+            self.assert_eq(
+                kdf[(20, "c")].groupby(kdf[(10, "a")]).sum().sort_index(),
+                pdf[(20, "c")].groupby(pdf[(10, "a")]).sum().sort_index(),
+            )
+        else:
+            # seems like a pandas bug introduced in pandas 1.1.3.
+            self.assert_eq(kdf[(20, "c")].groupby(kdf[(10, "a")]).sum().sort_index(), expected)
+
+    def test_split_apply_combine_on_series(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 2, 6, 4, 4, 6, 4, 3, 7],
+                "b": [4, 2, 7, 3, 3, 1, 1, 1, 2],
+                "c": [4, 2, 7, 3, None, 1, 1, 1, 2],
+                "d": list("abcdefght"),
+            },
+            index=[0, 1, 3, 5, 6, 8, 9, 9, 9],
+        )
+        kdf = ps.from_pandas(pdf)
+
+        funcs = [
+            ((True, False), ["sum", "min", "max", "count", "first", "last"]),
+            ((True, True), ["mean"]),
+            ((False, False), ["var", "std"]),
+        ]
+        funcs = [(check_exact, almost, f) for (check_exact, almost), fs in funcs for f in fs]
+
+        for as_index in [True, False]:
+            if as_index:
+                sort = lambda df: df.sort_index()
+            else:
+                sort = lambda df: df.sort_values(list(df.columns)).reset_index(drop=True)
+
+            for check_exact, almost, func in funcs:
+                for kkey, pkey in [("b", "b"), (kdf.b, pdf.b)]:
+                    with self.subTest(as_index=as_index, func=func, key=pkey):
+                        if as_index is True or func != "std":
+                            self.assert_eq(
+                                sort(getattr(kdf.groupby(kkey, as_index=as_index).a, func)()),
+                                sort(getattr(pdf.groupby(pkey, as_index=as_index).a, func)()),
+                                check_exact=check_exact,
+                                almost=almost,
+                            )
+                            self.assert_eq(
+                                sort(getattr(kdf.groupby(kkey, as_index=as_index), func)()),
+                                sort(getattr(pdf.groupby(pkey, as_index=as_index), func)()),
+                                check_exact=check_exact,
+                                almost=almost,
+                            )
+                        else:
+                            # seems like a pandas' bug for as_index=False and func == "std"?
+                            self.assert_eq(
+                                sort(getattr(kdf.groupby(kkey, as_index=as_index).a, func)()),
+                                sort(pdf.groupby(pkey, as_index=True).a.std().reset_index()),
+                                check_exact=check_exact,
+                                almost=almost,
+                            )
+                            self.assert_eq(
+                                sort(getattr(kdf.groupby(kkey, as_index=as_index), func)()),
+                                sort(pdf.groupby(pkey, as_index=True).std().reset_index()),
+                                check_exact=check_exact,
+                                almost=almost,
+                            )
+
+                for kkey, pkey in [(kdf.b + 1, pdf.b + 1), (kdf.copy().b, pdf.copy().b)]:
+                    with self.subTest(as_index=as_index, func=func, key=pkey):
+                        self.assert_eq(
+                            sort(getattr(kdf.groupby(kkey, as_index=as_index).a, func)()),
+                            sort(getattr(pdf.groupby(pkey, as_index=as_index).a, func)()),
+                            check_exact=check_exact,
+                            almost=almost,
+                        )
+                        self.assert_eq(
+                            sort(getattr(kdf.groupby(kkey, as_index=as_index), func)()),
+                            sort(getattr(pdf.groupby(pkey, as_index=as_index), func)()),
+                            check_exact=check_exact,
+                            almost=almost,
+                        )
+
+            for check_exact, almost, func in funcs:
+                for i in [0, 4, 7]:
+                    with self.subTest(as_index=as_index, func=func, i=i):
+                        self.assert_eq(
+                            sort(getattr(kdf.groupby(kdf.b > i, as_index=as_index).a, func)()),
+                            sort(getattr(pdf.groupby(pdf.b > i, as_index=as_index).a, func)()),
+                            check_exact=check_exact,
+                            almost=almost,
+                        )
+                        self.assert_eq(
+                            sort(getattr(kdf.groupby(kdf.b > i, as_index=as_index), func)()),
+                            sort(getattr(pdf.groupby(pdf.b > i, as_index=as_index), func)()),
+                            check_exact=check_exact,
+                            almost=almost,
+                        )
+
+        for check_exact, almost, func in funcs:
+            for kkey, pkey in [
+                (kdf.b, pdf.b),
+                (kdf.b + 1, pdf.b + 1),
+                (kdf.copy().b, pdf.copy().b),
+                (kdf.b.rename(), pdf.b.rename()),
+            ]:
+                with self.subTest(func=func, key=pkey):
+                    self.assert_eq(
+                        getattr(kdf.a.groupby(kkey), func)().sort_index(),
+                        getattr(pdf.a.groupby(pkey), func)().sort_index(),
+                        check_exact=check_exact,
+                        almost=almost,
+                    )
+                    self.assert_eq(
+                        getattr((kdf.a + 1).groupby(kkey), func)().sort_index(),
+                        getattr((pdf.a + 1).groupby(pkey), func)().sort_index(),
+                        check_exact=check_exact,
+                        almost=almost,
+                    )
+                    self.assert_eq(
+                        getattr((kdf.b + 1).groupby(kkey), func)().sort_index(),
+                        getattr((pdf.b + 1).groupby(pkey), func)().sort_index(),
+                        check_exact=check_exact,
+                        almost=almost,
+                    )
+                    self.assert_eq(
+                        getattr(kdf.a.rename().groupby(kkey), func)().sort_index(),
+                        getattr(pdf.a.rename().groupby(pkey), func)().sort_index(),
+                        check_exact=check_exact,
+                        almost=almost,
+                    )
+
+    def test_aggregate(self):
+        pdf = pd.DataFrame(
+            {"A": [1, 1, 2, 2], "B": [1, 2, 3, 4], "C": [0.362, 0.227, 1.267, -0.562]}
+        )
+        kdf = ps.from_pandas(pdf)
+
+        for as_index in [True, False]:
+            if as_index:
+                sort = lambda df: df.sort_index()
+            else:
+                sort = lambda df: df.sort_values(list(df.columns)).reset_index(drop=True)
+
+            for kkey, pkey in [("A", "A"), (kdf.A, pdf.A)]:
+                with self.subTest(as_index=as_index, key=pkey):
+                    self.assert_eq(
+                        sort(kdf.groupby(kkey, as_index=as_index).agg("sum")),
+                        sort(pdf.groupby(pkey, as_index=as_index).agg("sum")),
+                    )
+                    self.assert_eq(
+                        sort(kdf.groupby(kkey, as_index=as_index).agg({"B": "min", "C": "sum"})),
+                        sort(pdf.groupby(pkey, as_index=as_index).agg({"B": "min", "C": "sum"})),
+                    )
+                    self.assert_eq(
+                        sort(
+                            kdf.groupby(kkey, as_index=as_index).agg(
+                                {"B": ["min", "max"], "C": "sum"}
+                            )
+                        ),
+                        sort(
+                            pdf.groupby(pkey, as_index=as_index).agg(
+                                {"B": ["min", "max"], "C": "sum"}
+                            )
+                        ),
+                    )
+
+                    if as_index:
+                        self.assert_eq(
+                            sort(kdf.groupby(kkey, as_index=as_index).agg(["sum"])),
+                            sort(pdf.groupby(pkey, as_index=as_index).agg(["sum"])),
+                        )
+                    else:
+                        # seems like a pandas' bug for as_index=False and func_or_funcs is list?
+                        self.assert_eq(
+                            sort(kdf.groupby(kkey, as_index=as_index).agg(["sum"])),
+                            sort(pdf.groupby(pkey, as_index=True).agg(["sum"]).reset_index()),
+                        )
+
+            for kkey, pkey in [(kdf.A + 1, pdf.A + 1), (kdf.copy().A, pdf.copy().A)]:
+                with self.subTest(as_index=as_index, key=pkey):
+                    self.assert_eq(
+                        sort(kdf.groupby(kkey, as_index=as_index).agg("sum")),
+                        sort(pdf.groupby(pkey, as_index=as_index).agg("sum")),
+                    )
+                    self.assert_eq(
+                        sort(kdf.groupby(kkey, as_index=as_index).agg({"B": "min", "C": "sum"})),
+                        sort(pdf.groupby(pkey, as_index=as_index).agg({"B": "min", "C": "sum"})),
+                    )
+                    self.assert_eq(
+                        sort(
+                            kdf.groupby(kkey, as_index=as_index).agg(
+                                {"B": ["min", "max"], "C": "sum"}
+                            )
+                        ),
+                        sort(
+                            pdf.groupby(pkey, as_index=as_index).agg(
+                                {"B": ["min", "max"], "C": "sum"}
+                            )
+                        ),
+                    )
+                    self.assert_eq(
+                        sort(kdf.groupby(kkey, as_index=as_index).agg(["sum"])),
+                        sort(pdf.groupby(pkey, as_index=as_index).agg(["sum"])),
+                    )
+
+        expected_error_message = (
+            r"aggs must be a dict mapping from column name to aggregate functions "
+            r"\(string or list of strings\)."
+        )
+        with self.assertRaisesRegex(ValueError, expected_error_message):
+            kdf.groupby("A", as_index=as_index).agg(0)
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([(10, "A"), (10, "B"), (20, "C")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        for as_index in [True, False]:
+            stats_kdf = kdf.groupby((10, "A"), as_index=as_index).agg(
+                {(10, "B"): "min", (20, "C"): "sum"}
+            )
+            stats_pdf = pdf.groupby((10, "A"), as_index=as_index).agg(
+                {(10, "B"): "min", (20, "C"): "sum"}
+            )
+            self.assert_eq(
+                stats_kdf.sort_values(by=[(10, "B"), (20, "C")]).reset_index(drop=True),
+                stats_pdf.sort_values(by=[(10, "B"), (20, "C")]).reset_index(drop=True),
+            )
+
+        stats_kdf = kdf.groupby((10, "A")).agg({(10, "B"): ["min", "max"], (20, "C"): "sum"})
+        stats_pdf = pdf.groupby((10, "A")).agg({(10, "B"): ["min", "max"], (20, "C"): "sum"})
+        self.assert_eq(
+            stats_kdf.sort_values(
+                by=[(10, "B", "min"), (10, "B", "max"), (20, "C", "sum")]
+            ).reset_index(drop=True),
+            stats_pdf.sort_values(
+                by=[(10, "B", "min"), (10, "B", "max"), (20, "C", "sum")]
+            ).reset_index(drop=True),
+        )
+
+        # non-string names
+        pdf.columns = [10, 20, 30]
+        kdf.columns = [10, 20, 30]
+
+        for as_index in [True, False]:
+            stats_kdf = kdf.groupby(10, as_index=as_index).agg({20: "min", 30: "sum"})
+            stats_pdf = pdf.groupby(10, as_index=as_index).agg({20: "min", 30: "sum"})
+            self.assert_eq(
+                stats_kdf.sort_values(by=[20, 30]).reset_index(drop=True),
+                stats_pdf.sort_values(by=[20, 30]).reset_index(drop=True),
+            )
+
+        stats_kdf = kdf.groupby(10).agg({20: ["min", "max"], 30: "sum"})
+        stats_pdf = pdf.groupby(10).agg({20: ["min", "max"], 30: "sum"})
+        self.assert_eq(
+            stats_kdf.sort_values(by=[(20, "min"), (20, "max"), (30, "sum")]).reset_index(
+                drop=True
+            ),
+            stats_pdf.sort_values(by=[(20, "min"), (20, "max"), (30, "sum")]).reset_index(
+                drop=True
+            ),
+        )
+
+    def test_aggregate_func_str_list(self):
+        # this is test for cases where only string or list is assigned
+        pdf = pd.DataFrame(
+            {
+                "kind": ["cat", "dog", "cat", "dog"],
+                "height": [9.1, 6.0, 9.5, 34.0],
+                "weight": [7.9, 7.5, 9.9, 198.0],
+            }
+        )
+        kdf = ps.from_pandas(pdf)
+
+        agg_funcs = ["max", "min", ["min", "max"]]
+        for aggfunc in agg_funcs:
+
+            # Since in Koalas groupby, the order of rows might be different
+            # so sort on index to ensure they have same output
+            sorted_agg_kdf = kdf.groupby("kind").agg(aggfunc).sort_index()
+            sorted_agg_pdf = pdf.groupby("kind").agg(aggfunc).sort_index()
+            self.assert_eq(sorted_agg_kdf, sorted_agg_pdf)
+
+        # test on multi index column case
+        pdf = pd.DataFrame(
+            {"A": [1, 1, 2, 2], "B": [1, 2, 3, 4], "C": [0.362, 0.227, 1.267, -0.562]}
+        )
+        kdf = ps.from_pandas(pdf)
+
+        columns = pd.MultiIndex.from_tuples([("X", "A"), ("X", "B"), ("Y", "C")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        for aggfunc in agg_funcs:
+            sorted_agg_kdf = kdf.groupby(("X", "A")).agg(aggfunc).sort_index()
+            sorted_agg_pdf = pdf.groupby(("X", "A")).agg(aggfunc).sort_index()
+            self.assert_eq(sorted_agg_kdf, sorted_agg_pdf)
+
+    @unittest.skipIf(pd.__version__ < "0.25.0", "not supported before pandas 0.25.0")
+    def test_aggregate_relabel(self):
+        # this is to test named aggregation in groupby
+        pdf = pd.DataFrame({"group": ["a", "a", "b", "b"], "A": [0, 1, 2, 3], "B": [5, 6, 7, 8]})
+        kdf = ps.from_pandas(pdf)
+
+        # different agg column, same function
+        agg_pdf = pdf.groupby("group").agg(a_max=("A", "max"), b_max=("B", "max")).sort_index()
+        agg_kdf = kdf.groupby("group").agg(a_max=("A", "max"), b_max=("B", "max")).sort_index()
+        self.assert_eq(agg_pdf, agg_kdf)
+
+        # same agg column, different functions
+        agg_pdf = pdf.groupby("group").agg(b_max=("B", "max"), b_min=("B", "min")).sort_index()
+        agg_kdf = kdf.groupby("group").agg(b_max=("B", "max"), b_min=("B", "min")).sort_index()
+        self.assert_eq(agg_pdf, agg_kdf)
+
+        # test on NamedAgg
+        agg_pdf = (
+            pdf.groupby("group").agg(b_max=pd.NamedAgg(column="B", aggfunc="max")).sort_index()
+        )
+        agg_kdf = (
+            kdf.groupby("group").agg(b_max=ps.NamedAgg(column="B", aggfunc="max")).sort_index()
+        )
+        self.assert_eq(agg_kdf, agg_pdf)
+
+        # test on NamedAgg multi columns aggregation
+        agg_pdf = (
+            pdf.groupby("group")
+            .agg(
+                b_max=pd.NamedAgg(column="B", aggfunc="max"),
+                b_min=pd.NamedAgg(column="B", aggfunc="min"),
+            )
+            .sort_index()
+        )
+        agg_kdf = (
+            kdf.groupby("group")
+            .agg(
+                b_max=ps.NamedAgg(column="B", aggfunc="max"),
+                b_min=ps.NamedAgg(column="B", aggfunc="min"),
+            )
+            .sort_index()
+        )
+        self.assert_eq(agg_kdf, agg_pdf)
+
+    def test_dropna(self):
+        pdf = pd.DataFrame(
+            {"A": [None, 1, None, 1, 2], "B": [1, 2, 3, None, None], "C": [4, 5, 6, 7, None]}
+        )
+        kdf = ps.from_pandas(pdf)
+
+        # pd.DataFrame.groupby with dropna parameter is implemented since pandas 1.1.0
+        if LooseVersion(pd.__version__) >= LooseVersion("1.1.0"):
+            for dropna in [True, False]:
+                for as_index in [True, False]:
+                    if as_index:
+                        sort = lambda df: df.sort_index()
+                    else:
+                        sort = lambda df: df.sort_values("A").reset_index(drop=True)
+
+                    self.assert_eq(
+                        sort(kdf.groupby("A", as_index=as_index, dropna=dropna).std()),
+                        sort(pdf.groupby("A", as_index=as_index, dropna=dropna).std()),
+                    )
+
+                    self.assert_eq(
+                        sort(kdf.groupby("A", as_index=as_index, dropna=dropna).B.std()),
+                        sort(pdf.groupby("A", as_index=as_index, dropna=dropna).B.std()),
+                    )
+                    self.assert_eq(
+                        sort(kdf.groupby("A", as_index=as_index, dropna=dropna)["B"].std()),
+                        sort(pdf.groupby("A", as_index=as_index, dropna=dropna)["B"].std()),
+                    )
+
+                    self.assert_eq(
+                        sort(
+                            kdf.groupby("A", as_index=as_index, dropna=dropna).agg(
+                                {"B": "min", "C": "std"}
+                            )
+                        ),
+                        sort(
+                            pdf.groupby("A", as_index=as_index, dropna=dropna).agg(
+                                {"B": "min", "C": "std"}
+                            )
+                        ),
+                    )
+
+            for dropna in [True, False]:
+                for as_index in [True, False]:
+                    if as_index:
+                        sort = lambda df: df.sort_index()
+                    else:
+                        sort = lambda df: df.sort_values(["A", "B"]).reset_index(drop=True)
+
+                    self.assert_eq(
+                        sort(
+                            kdf.groupby(["A", "B"], as_index=as_index, dropna=dropna).agg(
+                                {"C": ["min", "std"]}
+                            )
+                        ),
+                        sort(
+                            pdf.groupby(["A", "B"], as_index=as_index, dropna=dropna).agg(
+                                {"C": ["min", "std"]}
+                            )
+                        ),
+                        almost=True,
+                    )
+
+            # multi-index columns
+            columns = pd.MultiIndex.from_tuples([("X", "A"), ("X", "B"), ("Y", "C")])
+            pdf.columns = columns
+            kdf.columns = columns
+
+            for dropna in [True, False]:
+                for as_index in [True, False]:
+                    if as_index:
+                        sort = lambda df: df.sort_index()
+                    else:
+                        sort = lambda df: df.sort_values(("X", "A")).reset_index(drop=True)
+                    sorted_stats_kdf = sort(
+                        kdf.groupby(("X", "A"), as_index=as_index, dropna=dropna).agg(
+                            {("X", "B"): "min", ("Y", "C"): "std"}
+                        )
+                    )
+                    sorted_stats_pdf = sort(
+                        pdf.groupby(("X", "A"), as_index=as_index, dropna=dropna).agg(
+                            {("X", "B"): "min", ("Y", "C"): "std"}
+                        )
+                    )
+                    self.assert_eq(sorted_stats_kdf, sorted_stats_pdf)
+        else:
+            # Testing dropna=True (pandas default behavior)
+            for as_index in [True, False]:
+                if as_index:
+                    sort = lambda df: df.sort_index()
+                else:
+                    sort = lambda df: df.sort_values("A").reset_index(drop=True)
+
+                self.assert_eq(
+                    sort(kdf.groupby("A", as_index=as_index, dropna=True)["B"].min()),
+                    sort(pdf.groupby("A", as_index=as_index)["B"].min()),
+                )
+
+                if as_index:
+                    sort = lambda df: df.sort_index()
+                else:
+                    sort = lambda df: df.sort_values(["A", "B"]).reset_index(drop=True)
+
+                self.assert_eq(
+                    sort(
+                        kdf.groupby(["A", "B"], as_index=as_index, dropna=True).agg(
+                            {"C": ["min", "std"]}
+                        )
+                    ),
+                    sort(pdf.groupby(["A", "B"], as_index=as_index).agg({"C": ["min", "std"]})),
+                    almost=True,
+                )
+
+            # Testing dropna=False
+            index = pd.Index([1.0, 2.0, np.nan], name="A")
+            expected = pd.Series([2.0, np.nan, 1.0], index=index, name="B")
+            result = kdf.groupby("A", as_index=True, dropna=False)["B"].min().sort_index()
+            self.assert_eq(expected, result)
+
+            expected = pd.DataFrame({"A": [1.0, 2.0, np.nan], "B": [2.0, np.nan, 1.0]})
+            result = (
+                kdf.groupby("A", as_index=False, dropna=False)["B"]
+                .min()
+                .sort_values("A")
+                .reset_index(drop=True)
+            )
+            self.assert_eq(expected, result)
+
+            index = pd.MultiIndex.from_tuples(
+                [(1.0, 2.0), (1.0, None), (2.0, None), (None, 1.0), (None, 3.0)], names=["A", "B"]
+            )
+            expected = pd.DataFrame(
+                {
+                    ("C", "min"): [5.0, 7.0, np.nan, 4.0, 6.0],
+                    ("C", "std"): [np.nan, np.nan, np.nan, np.nan, np.nan],
+                },
+                index=index,
+            )
+            result = (
+                kdf.groupby(["A", "B"], as_index=True, dropna=False)
+                .agg({"C": ["min", "std"]})
+                .sort_index()
+            )
+            self.assert_eq(expected, result)
+
+            expected = pd.DataFrame(
+                {
+                    ("A", ""): [1.0, 1.0, 2.0, np.nan, np.nan],
+                    ("B", ""): [2.0, np.nan, np.nan, 1.0, 3.0],
+                    ("C", "min"): [5.0, 7.0, np.nan, 4.0, 6.0],
+                    ("C", "std"): [np.nan, np.nan, np.nan, np.nan, np.nan],
+                }
+            )
+            result = (
+                kdf.groupby(["A", "B"], as_index=False, dropna=False)
+                .agg({"C": ["min", "std"]})
+                .sort_values(["A", "B"])
+                .reset_index(drop=True)
+            )
+            self.assert_eq(expected, result)
+
+    def test_describe(self):
+        # support for numeric type, not support for string type yet
+        datas = []
+        datas.append({"a": [1, 1, 3], "b": [4, 5, 6], "c": [7, 8, 9]})
+        datas.append({"a": [-1, -1, -3], "b": [-4, -5, -6], "c": [-7, -8, -9]})
+        datas.append({"a": [0, 0, 0], "b": [0, 0, 0], "c": [0, 8, 0]})
+        # it is okay if string type column as a group key
+        datas.append({"a": ["a", "a", "c"], "b": [4, 5, 6], "c": [7, 8, 9]})
+
+        percentiles = [0.25, 0.5, 0.75]
+        formatted_percentiles = ["25%", "50%", "75%"]
+        non_percentile_stats = ["count", "mean", "std", "min", "max"]
+
+        for data in datas:
+            pdf = pd.DataFrame(data)
+            kdf = ps.from_pandas(pdf)
+
+            describe_pdf = pdf.groupby("a").describe().sort_index()
+            describe_kdf = kdf.groupby("a").describe().sort_index()
+
+            # since the result of percentile columns are slightly difference from pandas,
+            # we should check them separately: non-percentile columns & percentile columns
+
+            # 1. Check that non-percentile columns are equal.
+            agg_cols = [col.name for col in kdf.groupby("a")._agg_columns]
+            self.assert_eq(
+                describe_kdf.drop(list(product(agg_cols, formatted_percentiles))),
+                describe_pdf.drop(columns=formatted_percentiles, level=1),
+                check_exact=False,
+            )
+
+            # 2. Check that percentile columns are equal.
+            # The interpolation argument is yet to be implemented in Koalas.
+            quantile_pdf = pdf.groupby("a").quantile(percentiles, interpolation="nearest")
+            quantile_pdf = quantile_pdf.unstack(level=1).astype(float)
+            self.assert_eq(
+                describe_kdf.drop(list(product(agg_cols, non_percentile_stats))),
+                quantile_pdf.rename(columns="{:.0%}".format, level=1),
+            )
+
+        # not support for string type yet
+        datas = []
+        datas.append({"a": ["a", "a", "c"], "b": ["d", "e", "f"], "c": ["g", "h", "i"]})
+        datas.append({"a": ["a", "a", "c"], "b": [4, 0, 1], "c": ["g", "h", "i"]})
+        for data in datas:
+            pdf = pd.DataFrame(data)
+            kdf = ps.from_pandas(pdf)
+
+            self.assertRaises(NotImplementedError, lambda: kdf.groupby("a").describe().sort_index())
+
+        # multi-index columns
+        pdf = pd.DataFrame({("x", "a"): [1, 1, 3], ("x", "b"): [4, 5, 6], ("y", "c"): [7, 8, 9]})
+        kdf = ps.from_pandas(pdf)
+
+        describe_pdf = pdf.groupby(("x", "a")).describe().sort_index()
+        describe_kdf = kdf.groupby(("x", "a")).describe().sort_index()
+
+        # 1. Check that non-percentile columns are equal.
+        agg_column_labels = [col._column_label for col in kdf.groupby(("x", "a"))._agg_columns]
+        self.assert_eq(
+            describe_kdf.drop(
+                [
+                    tuple(list(label) + [s])
+                    for label, s in product(agg_column_labels, formatted_percentiles)
+                ]
+            ),
+            describe_pdf.drop(columns=formatted_percentiles, level=2),
+            check_exact=False,
+        )
+
+        # 2. Check that percentile columns are equal.
+        # The interpolation argument is yet to be implemented in Koalas.
+        quantile_pdf = pdf.groupby(("x", "a")).quantile(percentiles, interpolation="nearest")
+        quantile_pdf = quantile_pdf.unstack(level=1).astype(float)
+
+        self.assert_eq(
+            describe_kdf.drop(
+                [
+                    tuple(list(label) + [s])
+                    for label, s in product(agg_column_labels, non_percentile_stats)
+                ]
+            ),
+            quantile_pdf.rename(columns="{:.0%}".format, level=2),
+        )
+
+    def test_aggregate_relabel_multiindex(self):
+        pdf = pd.DataFrame({"A": [0, 1, 2, 3], "B": [5, 6, 7, 8], "group": ["a", "a", "b", "b"]})
+        pdf.columns = pd.MultiIndex.from_tuples([("y", "A"), ("y", "B"), ("x", "group")])
+        kdf = ps.from_pandas(pdf)
+
+        if LooseVersion(pd.__version__) < LooseVersion("1.0.0"):
+            agg_pdf = pd.DataFrame(
+                {"a_max": [1, 3]}, index=pd.Index(["a", "b"], name=("x", "group"))
+            )
+        elif LooseVersion(pd.__version__) >= LooseVersion("1.0.0"):
+            agg_pdf = pdf.groupby(("x", "group")).agg(a_max=(("y", "A"), "max")).sort_index()
+        agg_kdf = kdf.groupby(("x", "group")).agg(a_max=(("y", "A"), "max")).sort_index()
+        self.assert_eq(agg_pdf, agg_kdf)
+
+        # same column, different methods
+        if LooseVersion(pd.__version__) < LooseVersion("1.0.0"):
+            agg_pdf = pd.DataFrame(
+                {"a_max": [1, 3], "a_min": [0, 2]}, index=pd.Index(["a", "b"], name=("x", "group"))
+            )
+        elif LooseVersion(pd.__version__) >= LooseVersion("1.0.0"):
+            agg_pdf = (
+                pdf.groupby(("x", "group"))
+                .agg(a_max=(("y", "A"), "max"), a_min=(("y", "A"), "min"))
+                .sort_index()
+            )
+        agg_kdf = (
+            kdf.groupby(("x", "group"))
+            .agg(a_max=(("y", "A"), "max"), a_min=(("y", "A"), "min"))
+            .sort_index()
+        )
+        self.assert_eq(agg_pdf, agg_kdf)
+
+        # different column, different methods
+        if LooseVersion(pd.__version__) < LooseVersion("1.0.0"):
+            agg_pdf = pd.DataFrame(
+                {"a_max": [6, 8], "a_min": [0, 2]}, index=pd.Index(["a", "b"], name=("x", "group"))
+            )
+        elif LooseVersion(pd.__version__) >= LooseVersion("1.0.0"):
+            agg_pdf = (
+                pdf.groupby(("x", "group"))
+                .agg(a_max=(("y", "B"), "max"), a_min=(("y", "A"), "min"))
+                .sort_index()
+            )
+        agg_kdf = (
+            kdf.groupby(("x", "group"))
+            .agg(a_max=(("y", "B"), "max"), a_min=(("y", "A"), "min"))
+            .sort_index()
+        )
+        self.assert_eq(agg_pdf, agg_kdf)
+
+    def test_all_any(self):
+        pdf = pd.DataFrame(
+            {
+                "A": [1, 1, 2, 2, 3, 3, 4, 4, 5, 5],
+                "B": [True, True, True, False, False, False, None, True, None, False],
+            }
+        )
+        kdf = ps.from_pandas(pdf)
+
+        for as_index in [True, False]:
+            if as_index:
+                sort = lambda df: df.sort_index()
+            else:
+                sort = lambda df: df.sort_values("A").reset_index(drop=True)
+            self.assert_eq(
+                sort(kdf.groupby("A", as_index=as_index).all()),
+                sort(pdf.groupby("A", as_index=as_index).all()),
+            )
+            self.assert_eq(
+                sort(kdf.groupby("A", as_index=as_index).any()),
+                sort(pdf.groupby("A", as_index=as_index).any()),
+            )
+
+            self.assert_eq(
+                sort(kdf.groupby("A", as_index=as_index).all()).B,
+                sort(pdf.groupby("A", as_index=as_index).all()).B,
+            )
+            self.assert_eq(
+                sort(kdf.groupby("A", as_index=as_index).any()).B,
+                sort(pdf.groupby("A", as_index=as_index).any()).B,
+            )
+
+        self.assert_eq(
+            kdf.B.groupby(kdf.A).all().sort_index(), pdf.B.groupby(pdf.A).all().sort_index()
+        )
+        self.assert_eq(
+            kdf.B.groupby(kdf.A).any().sort_index(), pdf.B.groupby(pdf.A).any().sort_index()
+        )
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("X", "A"), ("Y", "B")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        for as_index in [True, False]:
+            if as_index:
+                sort = lambda df: df.sort_index()
+            else:
+                sort = lambda df: df.sort_values(("X", "A")).reset_index(drop=True)
+            self.assert_eq(
+                sort(kdf.groupby(("X", "A"), as_index=as_index).all()),
+                sort(pdf.groupby(("X", "A"), as_index=as_index).all()),
+            )
+            self.assert_eq(
+                sort(kdf.groupby(("X", "A"), as_index=as_index).any()),
+                sort(pdf.groupby(("X", "A"), as_index=as_index).any()),
+            )
+
+    def test_raises(self):
+        kdf = ps.DataFrame(
+            {"a": [1, 2, 6, 4, 4, 6, 4, 3, 7], "b": [4, 2, 7, 3, 3, 1, 1, 1, 2]},
+            index=[0, 1, 3, 5, 6, 8, 9, 9, 9],
+        )
+        # test raises with incorrect key
+        self.assertRaises(ValueError, lambda: kdf.groupby([]))
+        self.assertRaises(KeyError, lambda: kdf.groupby("x"))
+        self.assertRaises(KeyError, lambda: kdf.groupby(["a", "x"]))
+        self.assertRaises(KeyError, lambda: kdf.groupby("a")["x"])
+        self.assertRaises(KeyError, lambda: kdf.groupby("a")["b", "x"])
+        self.assertRaises(KeyError, lambda: kdf.groupby("a")[["b", "x"]])
+
+    def test_nunique(self):
+        pdf = pd.DataFrame(
+            {"a": [1, 1, 1, 1, 1, 0, 0, 0, 0, 0], "b": [2, 2, 2, 3, 3, 4, 4, 5, 5, 5]}
+        )
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(
+            kdf.groupby("a").agg({"b": "nunique"}).sort_index(),
+            pdf.groupby("a").agg({"b": "nunique"}).sort_index(),
+        )
+        if LooseVersion(pd.__version__) < LooseVersion("1.1.0"):
+            expected = ps.DataFrame({"b": [2, 2]}, index=pd.Index([0, 1], name="a"))
+            self.assert_eq(kdf.groupby("a").nunique().sort_index(), expected)
+            self.assert_eq(
+                kdf.groupby("a").nunique(dropna=False).sort_index(), expected,
+            )
+        else:
+            self.assert_eq(
+                kdf.groupby("a").nunique().sort_index(), pdf.groupby("a").nunique().sort_index()
+            )
+            self.assert_eq(
+                kdf.groupby("a").nunique(dropna=False).sort_index(),
+                pdf.groupby("a").nunique(dropna=False).sort_index(),
+            )
+        self.assert_eq(
+            kdf.groupby("a")["b"].nunique().sort_index(),
+            pdf.groupby("a")["b"].nunique().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby("a")["b"].nunique(dropna=False).sort_index(),
+            pdf.groupby("a")["b"].nunique(dropna=False).sort_index(),
+        )
+
+        nunique_kdf = kdf.groupby("a", as_index=False).agg({"b": "nunique"})
+        nunique_pdf = pdf.groupby("a", as_index=False).agg({"b": "nunique"})
+        self.assert_eq(
+            nunique_kdf.sort_values(["a", "b"]).reset_index(drop=True),
+            nunique_pdf.sort_values(["a", "b"]).reset_index(drop=True),
+        )
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("y", "b")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        if LooseVersion(pd.__version__) < LooseVersion("1.1.0"):
+            expected = ps.DataFrame({("y", "b"): [2, 2]}, index=pd.Index([0, 1], name=("x", "a")))
+            self.assert_eq(
+                kdf.groupby(("x", "a")).nunique().sort_index(), expected,
+            )
+            self.assert_eq(
+                kdf.groupby(("x", "a")).nunique(dropna=False).sort_index(), expected,
+            )
+        else:
+            self.assert_eq(
+                kdf.groupby(("x", "a")).nunique().sort_index(),
+                pdf.groupby(("x", "a")).nunique().sort_index(),
+            )
+            self.assert_eq(
+                kdf.groupby(("x", "a")).nunique(dropna=False).sort_index(),
+                pdf.groupby(("x", "a")).nunique(dropna=False).sort_index(),
+            )
+
+    def test_unique(self):
+        for pdf in [
+            pd.DataFrame(
+                {"a": [1, 1, 1, 1, 1, 0, 0, 0, 0, 0], "b": [2, 2, 2, 3, 3, 4, 4, 5, 5, 5]}
+            ),
+            pd.DataFrame(
+                {
+                    "a": [1, 1, 1, 1, 1, 0, 0, 0, 0, 0],
+                    "b": ["w", "w", "w", "x", "x", "y", "y", "z", "z", "z"],
+                }
+            ),
+        ]:
+            with self.subTest(pdf=pdf):
+                kdf = ps.from_pandas(pdf)
+
+                actual = kdf.groupby("a")["b"].unique().sort_index().to_pandas()
+                expect = pdf.groupby("a")["b"].unique().sort_index()
+                self.assert_eq(len(actual), len(expect))
+                for act, exp in zip(actual, expect):
+                    self.assertTrue(sorted(act) == sorted(exp))
+
+    def test_value_counts(self):
+        pdf = pd.DataFrame({"A": [1, 2, 2, 3, 3, 3], "B": [1, 1, 2, 3, 3, 3]}, columns=["A", "B"])
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(
+            kdf.groupby("A")["B"].value_counts().sort_index(),
+            pdf.groupby("A")["B"].value_counts().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby("A")["B"].value_counts(sort=True, ascending=False).sort_index(),
+            pdf.groupby("A")["B"].value_counts(sort=True, ascending=False).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby("A")["B"].value_counts(sort=True, ascending=True).sort_index(),
+            pdf.groupby("A")["B"].value_counts(sort=True, ascending=True).sort_index(),
+        )
+        self.assert_eq(
+            kdf.B.rename().groupby(kdf.A).value_counts().sort_index(),
+            pdf.B.rename().groupby(pdf.A).value_counts().sort_index(),
+        )
+        self.assert_eq(
+            kdf.B.groupby(kdf.A.rename()).value_counts().sort_index(),
+            pdf.B.groupby(pdf.A.rename()).value_counts().sort_index(),
+        )
+        self.assert_eq(
+            kdf.B.rename().groupby(kdf.A.rename()).value_counts().sort_index(),
+            pdf.B.rename().groupby(pdf.A.rename()).value_counts().sort_index(),
+        )
+
+    def test_size(self):
+        pdf = pd.DataFrame({"A": [1, 2, 2, 3, 3, 3], "B": [1, 1, 2, 3, 3, 3]})
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kdf.groupby("A").size().sort_index(), pdf.groupby("A").size().sort_index())
+        self.assert_eq(
+            kdf.groupby("A")["B"].size().sort_index(), pdf.groupby("A")["B"].size().sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby("A")[["B"]].size().sort_index(), pdf.groupby("A")[["B"]].size().sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby(["A", "B"]).size().sort_index(), pdf.groupby(["A", "B"]).size().sort_index()
+        )
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("X", "A"), ("Y", "B")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(
+            kdf.groupby(("X", "A")).size().sort_index(), pdf.groupby(("X", "A")).size().sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby([("X", "A"), ("Y", "B")]).size().sort_index(),
+            pdf.groupby([("X", "A"), ("Y", "B")]).size().sort_index(),
+        )
+
+    def test_diff(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 2, 3, 4, 5, 6] * 3,
+                "b": [1, 1, 2, 3, 5, 8] * 3,
+                "c": [1, 4, 9, 16, 25, 36] * 3,
+            }
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.groupby("b").diff().sort_index(), pdf.groupby("b").diff().sort_index())
+        self.assert_eq(
+            kdf.groupby(["a", "b"]).diff().sort_index(), pdf.groupby(["a", "b"]).diff().sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby(["b"])["a"].diff().sort_index(), pdf.groupby(["b"])["a"].diff().sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby(["b"])[["a", "b"]].diff().sort_index(),
+            pdf.groupby(["b"])[["a", "b"]].diff().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.b // 5).diff().sort_index(), pdf.groupby(pdf.b // 5).diff().sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.b // 5)["a"].diff().sort_index(),
+            pdf.groupby(pdf.b // 5)["a"].diff().sort_index(),
+        )
+
+        self.assert_eq(kdf.groupby("b").diff().sum(), pdf.groupby("b").diff().sum().astype(int))
+        self.assert_eq(kdf.groupby(["b"])["a"].diff().sum(), pdf.groupby(["b"])["a"].diff().sum())
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "c")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(
+            kdf.groupby(("x", "b")).diff().sort_index(), pdf.groupby(("x", "b")).diff().sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby([("x", "a"), ("x", "b")]).diff().sort_index(),
+            pdf.groupby([("x", "a"), ("x", "b")]).diff().sort_index(),
+        )
+
+    def test_rank(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 2, 3, 4, 5, 6] * 3,
+                "b": [1, 1, 2, 3, 5, 8] * 3,
+                "c": [1, 4, 9, 16, 25, 36] * 3,
+            },
+            index=np.random.rand(6 * 3),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.groupby("b").rank().sort_index(), pdf.groupby("b").rank().sort_index())
+        self.assert_eq(
+            kdf.groupby(["a", "b"]).rank().sort_index(), pdf.groupby(["a", "b"]).rank().sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby(["b"])["a"].rank().sort_index(), pdf.groupby(["b"])["a"].rank().sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby(["b"])[["a", "c"]].rank().sort_index(),
+            pdf.groupby(["b"])[["a", "c"]].rank().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.b // 5).rank().sort_index(), pdf.groupby(pdf.b // 5).rank().sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.b // 5)["a"].rank().sort_index(),
+            pdf.groupby(pdf.b // 5)["a"].rank().sort_index(),
+        )
+
+        self.assert_eq(kdf.groupby("b").rank().sum(), pdf.groupby("b").rank().sum())
+        self.assert_eq(kdf.groupby(["b"])["a"].rank().sum(), pdf.groupby(["b"])["a"].rank().sum())
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "c")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(
+            kdf.groupby(("x", "b")).rank().sort_index(), pdf.groupby(("x", "b")).rank().sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby([("x", "a"), ("x", "b")]).rank().sort_index(),
+            pdf.groupby([("x", "a"), ("x", "b")]).rank().sort_index(),
+        )
+
+    def test_cumcount(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 2, 3, 4, 5, 6] * 3,
+                "b": [1, 1, 2, 3, 5, 8] * 3,
+                "c": [1, 4, 9, 16, 25, 36] * 3,
+            },
+            index=np.random.rand(6 * 3),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        for ascending in [True, False]:
+            self.assert_eq(
+                kdf.groupby("b").cumcount(ascending=ascending).sort_index(),
+                pdf.groupby("b").cumcount(ascending=ascending).sort_index(),
+            )
+            self.assert_eq(
+                kdf.groupby(["a", "b"]).cumcount(ascending=ascending).sort_index(),
+                pdf.groupby(["a", "b"]).cumcount(ascending=ascending).sort_index(),
+            )
+            self.assert_eq(
+                kdf.groupby(["b"])["a"].cumcount(ascending=ascending).sort_index(),
+                pdf.groupby(["b"])["a"].cumcount(ascending=ascending).sort_index(),
+            )
+            self.assert_eq(
+                kdf.groupby(["b"])[["a", "c"]].cumcount(ascending=ascending).sort_index(),
+                pdf.groupby(["b"])[["a", "c"]].cumcount(ascending=ascending).sort_index(),
+            )
+            self.assert_eq(
+                kdf.groupby(kdf.b // 5).cumcount(ascending=ascending).sort_index(),
+                pdf.groupby(pdf.b // 5).cumcount(ascending=ascending).sort_index(),
+            )
+            self.assert_eq(
+                kdf.groupby(kdf.b // 5)["a"].cumcount(ascending=ascending).sort_index(),
+                pdf.groupby(pdf.b // 5)["a"].cumcount(ascending=ascending).sort_index(),
+            )
+            self.assert_eq(
+                kdf.groupby("b").cumcount(ascending=ascending).sum(),
+                pdf.groupby("b").cumcount(ascending=ascending).sum(),
+            )
+            self.assert_eq(
+                kdf.a.rename().groupby(kdf.b).cumcount(ascending=ascending).sort_index(),
+                pdf.a.rename().groupby(pdf.b).cumcount(ascending=ascending).sort_index(),
+            )
+            self.assert_eq(
+                kdf.a.groupby(kdf.b.rename()).cumcount(ascending=ascending).sort_index(),
+                pdf.a.groupby(pdf.b.rename()).cumcount(ascending=ascending).sort_index(),
+            )
+            self.assert_eq(
+                kdf.a.rename().groupby(kdf.b.rename()).cumcount(ascending=ascending).sort_index(),
+                pdf.a.rename().groupby(pdf.b.rename()).cumcount(ascending=ascending).sort_index(),
+            )
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "c")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        for ascending in [True, False]:
+            self.assert_eq(
+                kdf.groupby(("x", "b")).cumcount(ascending=ascending).sort_index(),
+                pdf.groupby(("x", "b")).cumcount(ascending=ascending).sort_index(),
+            )
+            self.assert_eq(
+                kdf.groupby([("x", "a"), ("x", "b")]).cumcount(ascending=ascending).sort_index(),
+                pdf.groupby([("x", "a"), ("x", "b")]).cumcount(ascending=ascending).sort_index(),
+            )
+
+    def test_cummin(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 2, 3, 4, 5, 6] * 3,
+                "b": [1, 1, 2, 3, 5, 8] * 3,
+                "c": [1, 4, 9, 16, 25, 36] * 3,
+            },
+            index=np.random.rand(6 * 3),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            kdf.groupby("b").cummin().sort_index(), pdf.groupby("b").cummin().sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby(["a", "b"]).cummin().sort_index(),
+            pdf.groupby(["a", "b"]).cummin().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(["b"])["a"].cummin().sort_index(),
+            pdf.groupby(["b"])["a"].cummin().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(["b"])[["a", "c"]].cummin().sort_index(),
+            pdf.groupby(["b"])[["a", "c"]].cummin().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.b // 5).cummin().sort_index(),
+            pdf.groupby(pdf.b // 5).cummin().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.b // 5)["a"].cummin().sort_index(),
+            pdf.groupby(pdf.b // 5)["a"].cummin().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby("b").cummin().sum().sort_index(),
+            pdf.groupby("b").cummin().sum().sort_index(),
+        )
+        self.assert_eq(
+            kdf.a.rename().groupby(kdf.b).cummin().sort_index(),
+            pdf.a.rename().groupby(pdf.b).cummin().sort_index(),
+        )
+        self.assert_eq(
+            kdf.a.groupby(kdf.b.rename()).cummin().sort_index(),
+            pdf.a.groupby(pdf.b.rename()).cummin().sort_index(),
+        )
+        self.assert_eq(
+            kdf.a.rename().groupby(kdf.b.rename()).cummin().sort_index(),
+            pdf.a.rename().groupby(pdf.b.rename()).cummin().sort_index(),
+        )
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "c")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(
+            kdf.groupby(("x", "b")).cummin().sort_index(),
+            pdf.groupby(("x", "b")).cummin().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby([("x", "a"), ("x", "b")]).cummin().sort_index(),
+            pdf.groupby([("x", "a"), ("x", "b")]).cummin().sort_index(),
+        )
+
+        kdf = ps.DataFrame([["a"], ["b"], ["c"]], columns=["A"])
+        self.assertRaises(DataError, lambda: kdf.groupby(["A"]).cummin())
+        kdf = ps.DataFrame([[1, "a"], [2, "b"], [3, "c"]], columns=["A", "B"])
+        self.assertRaises(DataError, lambda: kdf.groupby(["A"])["B"].cummin())
+
+    def test_cummax(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 2, 3, 4, 5, 6] * 3,
+                "b": [1, 1, 2, 3, 5, 8] * 3,
+                "c": [1, 4, 9, 16, 25, 36] * 3,
+            },
+            index=np.random.rand(6 * 3),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            kdf.groupby("b").cummax().sort_index(), pdf.groupby("b").cummax().sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby(["a", "b"]).cummax().sort_index(),
+            pdf.groupby(["a", "b"]).cummax().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(["b"])["a"].cummax().sort_index(),
+            pdf.groupby(["b"])["a"].cummax().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(["b"])[["a", "c"]].cummax().sort_index(),
+            pdf.groupby(["b"])[["a", "c"]].cummax().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.b // 5).cummax().sort_index(),
+            pdf.groupby(pdf.b // 5).cummax().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.b // 5)["a"].cummax().sort_index(),
+            pdf.groupby(pdf.b // 5)["a"].cummax().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby("b").cummax().sum().sort_index(),
+            pdf.groupby("b").cummax().sum().sort_index(),
+        )
+        self.assert_eq(
+            kdf.a.rename().groupby(kdf.b).cummax().sort_index(),
+            pdf.a.rename().groupby(pdf.b).cummax().sort_index(),
+        )
+        self.assert_eq(
+            kdf.a.groupby(kdf.b.rename()).cummax().sort_index(),
+            pdf.a.groupby(pdf.b.rename()).cummax().sort_index(),
+        )
+        self.assert_eq(
+            kdf.a.rename().groupby(kdf.b.rename()).cummax().sort_index(),
+            pdf.a.rename().groupby(pdf.b.rename()).cummax().sort_index(),
+        )
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "c")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(
+            kdf.groupby(("x", "b")).cummax().sort_index(),
+            pdf.groupby(("x", "b")).cummax().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby([("x", "a"), ("x", "b")]).cummax().sort_index(),
+            pdf.groupby([("x", "a"), ("x", "b")]).cummax().sort_index(),
+        )
+
+        kdf = ps.DataFrame([["a"], ["b"], ["c"]], columns=["A"])
+        self.assertRaises(DataError, lambda: kdf.groupby(["A"]).cummax())
+        kdf = ps.DataFrame([[1, "a"], [2, "b"], [3, "c"]], columns=["A", "B"])
+        self.assertRaises(DataError, lambda: kdf.groupby(["A"])["B"].cummax())
+
+    def test_cumsum(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 2, 3, 4, 5, 6] * 3,
+                "b": [1, 1, 2, 3, 5, 8] * 3,
+                "c": [1, 4, 9, 16, 25, 36] * 3,
+            },
+            index=np.random.rand(6 * 3),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            kdf.groupby("b").cumsum().sort_index(), pdf.groupby("b").cumsum().sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby(["a", "b"]).cumsum().sort_index(),
+            pdf.groupby(["a", "b"]).cumsum().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(["b"])["a"].cumsum().sort_index(),
+            pdf.groupby(["b"])["a"].cumsum().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(["b"])[["a", "c"]].cumsum().sort_index(),
+            pdf.groupby(["b"])[["a", "c"]].cumsum().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.b // 5).cumsum().sort_index(),
+            pdf.groupby(pdf.b // 5).cumsum().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.b // 5)["a"].cumsum().sort_index(),
+            pdf.groupby(pdf.b // 5)["a"].cumsum().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby("b").cumsum().sum().sort_index(),
+            pdf.groupby("b").cumsum().sum().sort_index(),
+        )
+        self.assert_eq(
+            kdf.a.rename().groupby(kdf.b).cumsum().sort_index(),
+            pdf.a.rename().groupby(pdf.b).cumsum().sort_index(),
+        )
+        self.assert_eq(
+            kdf.a.groupby(kdf.b.rename()).cumsum().sort_index(),
+            pdf.a.groupby(pdf.b.rename()).cumsum().sort_index(),
+        )
+        self.assert_eq(
+            kdf.a.rename().groupby(kdf.b.rename()).cumsum().sort_index(),
+            pdf.a.rename().groupby(pdf.b.rename()).cumsum().sort_index(),
+        )
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "c")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(
+            kdf.groupby(("x", "b")).cumsum().sort_index(),
+            pdf.groupby(("x", "b")).cumsum().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby([("x", "a"), ("x", "b")]).cumsum().sort_index(),
+            pdf.groupby([("x", "a"), ("x", "b")]).cumsum().sort_index(),
+        )
+
+        kdf = ps.DataFrame([["a"], ["b"], ["c"]], columns=["A"])
+        self.assertRaises(DataError, lambda: kdf.groupby(["A"]).cumsum())
+        kdf = ps.DataFrame([[1, "a"], [2, "b"], [3, "c"]], columns=["A", "B"])
+        self.assertRaises(DataError, lambda: kdf.groupby(["A"])["B"].cumsum())
+
+    def test_cumprod(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 2, -3, 4, -5, 6] * 3,
+                "b": [1, 1, 2, 3, 5, 8] * 3,
+                "c": [1, 0, 9, 16, 25, 36] * 3,
+            },
+            index=np.random.rand(6 * 3),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            kdf.groupby("b").cumprod().sort_index(),
+            pdf.groupby("b").cumprod().sort_index(),
+            check_exact=False,
+        )
+        self.assert_eq(
+            kdf.groupby(["a", "b"]).cumprod().sort_index(),
+            pdf.groupby(["a", "b"]).cumprod().sort_index(),
+            check_exact=False,
+        )
+        self.assert_eq(
+            kdf.groupby(["b"])["a"].cumprod().sort_index(),
+            pdf.groupby(["b"])["a"].cumprod().sort_index(),
+            check_exact=False,
+        )
+        self.assert_eq(
+            kdf.groupby(["b"])[["a", "c"]].cumprod().sort_index(),
+            pdf.groupby(["b"])[["a", "c"]].cumprod().sort_index(),
+            check_exact=False,
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.b // 3).cumprod().sort_index(),
+            pdf.groupby(pdf.b // 3).cumprod().sort_index(),
+            check_exact=False,
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.b // 3)["a"].cumprod().sort_index(),
+            pdf.groupby(pdf.b // 3)["a"].cumprod().sort_index(),
+            check_exact=False,
+        )
+        self.assert_eq(
+            kdf.groupby("b").cumprod().sum().sort_index(),
+            pdf.groupby("b").cumprod().sum().sort_index(),
+            check_exact=False,
+        )
+        self.assert_eq(
+            kdf.a.rename().groupby(kdf.b).cumprod().sort_index(),
+            pdf.a.rename().groupby(pdf.b).cumprod().sort_index(),
+            check_exact=False,
+        )
+        self.assert_eq(
+            kdf.a.groupby(kdf.b.rename()).cumprod().sort_index(),
+            pdf.a.groupby(pdf.b.rename()).cumprod().sort_index(),
+            check_exact=False,
+        )
+        self.assert_eq(
+            kdf.a.rename().groupby(kdf.b.rename()).cumprod().sort_index(),
+            pdf.a.rename().groupby(pdf.b.rename()).cumprod().sort_index(),
+            check_exact=False,
+        )
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "c")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(
+            kdf.groupby(("x", "b")).cumprod().sort_index(),
+            pdf.groupby(("x", "b")).cumprod().sort_index(),
+            check_exact=False,
+        )
+        self.assert_eq(
+            kdf.groupby([("x", "a"), ("x", "b")]).cumprod().sort_index(),
+            pdf.groupby([("x", "a"), ("x", "b")]).cumprod().sort_index(),
+            check_exact=False,
+        )
+
+        kdf = ps.DataFrame([["a"], ["b"], ["c"]], columns=["A"])
+        self.assertRaises(DataError, lambda: kdf.groupby(["A"]).cumprod())
+        kdf = ps.DataFrame([[1, "a"], [2, "b"], [3, "c"]], columns=["A", "B"])
+        self.assertRaises(DataError, lambda: kdf.groupby(["A"])["B"].cumprod())
+
+    def test_nsmallest(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 1, 1, 2, 2, 2, 3, 3, 3] * 3,
+                "b": [1, 2, 2, 2, 3, 3, 3, 4, 4] * 3,
+                "c": [1, 2, 2, 2, 3, 3, 3, 4, 4] * 3,
+                "d": [1, 2, 2, 2, 3, 3, 3, 4, 4] * 3,
+            },
+            index=np.random.rand(9 * 3),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            kdf.groupby(["a"])["b"].nsmallest(1).sort_values(),
+            pdf.groupby(["a"])["b"].nsmallest(1).sort_values(),
+        )
+        self.assert_eq(
+            kdf.groupby(["a"])["b"].nsmallest(2).sort_index(),
+            pdf.groupby(["a"])["b"].nsmallest(2).sort_index(),
+        )
+        self.assert_eq(
+            (kdf.b * 10).groupby(kdf.a).nsmallest(2).sort_index(),
+            (pdf.b * 10).groupby(pdf.a).nsmallest(2).sort_index(),
+        )
+        self.assert_eq(
+            kdf.b.rename().groupby(kdf.a).nsmallest(2).sort_index(),
+            pdf.b.rename().groupby(pdf.a).nsmallest(2).sort_index(),
+        )
+        self.assert_eq(
+            kdf.b.groupby(kdf.a.rename()).nsmallest(2).sort_index(),
+            pdf.b.groupby(pdf.a.rename()).nsmallest(2).sort_index(),
+        )
+        self.assert_eq(
+            kdf.b.rename().groupby(kdf.a.rename()).nsmallest(2).sort_index(),
+            pdf.b.rename().groupby(pdf.a.rename()).nsmallest(2).sort_index(),
+        )
+        with self.assertRaisesRegex(ValueError, "nsmallest do not support multi-index now"):
+            kdf.set_index(["a", "b"]).groupby(["c"])["d"].nsmallest(1)
+
+    def test_nlargest(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 1, 1, 2, 2, 2, 3, 3, 3] * 3,
+                "b": [1, 2, 2, 2, 3, 3, 3, 4, 4] * 3,
+                "c": [1, 2, 2, 2, 3, 3, 3, 4, 4] * 3,
+                "d": [1, 2, 2, 2, 3, 3, 3, 4, 4] * 3,
+            },
+            index=np.random.rand(9 * 3),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            kdf.groupby(["a"])["b"].nlargest(1).sort_values(),
+            pdf.groupby(["a"])["b"].nlargest(1).sort_values(),
+        )
+        self.assert_eq(
+            kdf.groupby(["a"])["b"].nlargest(2).sort_index(),
+            pdf.groupby(["a"])["b"].nlargest(2).sort_index(),
+        )
+        self.assert_eq(
+            (kdf.b * 10).groupby(kdf.a).nlargest(2).sort_index(),
+            (pdf.b * 10).groupby(pdf.a).nlargest(2).sort_index(),
+        )
+        self.assert_eq(
+            kdf.b.rename().groupby(kdf.a).nlargest(2).sort_index(),
+            pdf.b.rename().groupby(pdf.a).nlargest(2).sort_index(),
+        )
+        self.assert_eq(
+            kdf.b.groupby(kdf.a.rename()).nlargest(2).sort_index(),
+            pdf.b.groupby(pdf.a.rename()).nlargest(2).sort_index(),
+        )
+        self.assert_eq(
+            kdf.b.rename().groupby(kdf.a.rename()).nlargest(2).sort_index(),
+            pdf.b.rename().groupby(pdf.a.rename()).nlargest(2).sort_index(),
+        )
+        with self.assertRaisesRegex(ValueError, "nlargest do not support multi-index now"):
+            kdf.set_index(["a", "b"]).groupby(["c"])["d"].nlargest(1)
+
+    def test_fillna(self):
+        pdf = pd.DataFrame(
+            {
+                "A": [1, 1, 2, 2] * 3,
+                "B": [2, 4, None, 3] * 3,
+                "C": [None, None, None, 1] * 3,
+                "D": [0, 1, 5, 4] * 3,
+            }
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            kdf.groupby("A").fillna(0).sort_index(), pdf.groupby("A").fillna(0).sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby("A")["C"].fillna(0).sort_index(),
+            pdf.groupby("A")["C"].fillna(0).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby("A")[["C"]].fillna(0).sort_index(),
+            pdf.groupby("A")[["C"]].fillna(0).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby("A").fillna(method="bfill").sort_index(),
+            pdf.groupby("A").fillna(method="bfill").sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby("A")["C"].fillna(method="bfill").sort_index(),
+            pdf.groupby("A")["C"].fillna(method="bfill").sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby("A")[["C"]].fillna(method="bfill").sort_index(),
+            pdf.groupby("A")[["C"]].fillna(method="bfill").sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby("A").fillna(method="ffill").sort_index(),
+            pdf.groupby("A").fillna(method="ffill").sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby("A")["C"].fillna(method="ffill").sort_index(),
+            pdf.groupby("A")["C"].fillna(method="ffill").sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby("A")[["C"]].fillna(method="ffill").sort_index(),
+            pdf.groupby("A")[["C"]].fillna(method="ffill").sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.A // 5).fillna(method="bfill").sort_index(),
+            pdf.groupby(pdf.A // 5).fillna(method="bfill").sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.A // 5)["C"].fillna(method="bfill").sort_index(),
+            pdf.groupby(pdf.A // 5)["C"].fillna(method="bfill").sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.A // 5)[["C"]].fillna(method="bfill").sort_index(),
+            pdf.groupby(pdf.A // 5)[["C"]].fillna(method="bfill").sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.A // 5).fillna(method="ffill").sort_index(),
+            pdf.groupby(pdf.A // 5).fillna(method="ffill").sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.A // 5)["C"].fillna(method="ffill").sort_index(),
+            pdf.groupby(pdf.A // 5)["C"].fillna(method="ffill").sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.A // 5)[["C"]].fillna(method="ffill").sort_index(),
+            pdf.groupby(pdf.A // 5)[["C"]].fillna(method="ffill").sort_index(),
+        )
+        self.assert_eq(
+            kdf.C.rename().groupby(kdf.A).fillna(0).sort_index(),
+            pdf.C.rename().groupby(pdf.A).fillna(0).sort_index(),
+        )
+        self.assert_eq(
+            kdf.C.groupby(kdf.A.rename()).fillna(0).sort_index(),
+            pdf.C.groupby(pdf.A.rename()).fillna(0).sort_index(),
+        )
+        self.assert_eq(
+            kdf.C.rename().groupby(kdf.A.rename()).fillna(0).sort_index(),
+            pdf.C.rename().groupby(pdf.A.rename()).fillna(0).sort_index(),
+        )
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("X", "A"), ("X", "B"), ("Y", "C"), ("Z", "D")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(
+            kdf.groupby(("X", "A")).fillna(0).sort_index(),
+            pdf.groupby(("X", "A")).fillna(0).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(("X", "A")).fillna(method="bfill").sort_index(),
+            pdf.groupby(("X", "A")).fillna(method="bfill").sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(("X", "A")).fillna(method="ffill").sort_index(),
+            pdf.groupby(("X", "A")).fillna(method="ffill").sort_index(),
+        )
+
+    def test_ffill(self):
+        idx = np.random.rand(4 * 3)
+        pdf = pd.DataFrame(
+            {
+                "A": [1, 1, 2, 2] * 3,
+                "B": [2, 4, None, 3] * 3,
+                "C": [None, None, None, 1] * 3,
+                "D": [0, 1, 5, 4] * 3,
+            },
+            index=idx,
+        )
+        kdf = ps.from_pandas(pdf)
+
+        if LooseVersion(pd.__version__) <= LooseVersion("0.24.2"):
+            self.assert_eq(
+                kdf.groupby("A").ffill().sort_index(),
+                pdf.groupby("A").ffill().sort_index().drop("A", 1),
+            )
+            self.assert_eq(
+                kdf.groupby("A")[["B"]].ffill().sort_index(),
+                pdf.groupby("A")[["B"]].ffill().sort_index().drop("A", 1),
+            )
+        else:
+            self.assert_eq(
+                kdf.groupby("A").ffill().sort_index(), pdf.groupby("A").ffill().sort_index()
+            )
+            self.assert_eq(
+                kdf.groupby("A")[["B"]].ffill().sort_index(),
+                pdf.groupby("A")[["B"]].ffill().sort_index(),
+            )
+        self.assert_eq(
+            kdf.groupby("A")["B"].ffill().sort_index(), pdf.groupby("A")["B"].ffill().sort_index()
+        )
+        self.assert_eq(kdf.groupby("A")["B"].ffill()[idx[6]], pdf.groupby("A")["B"].ffill()[idx[6]])
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("X", "A"), ("X", "B"), ("Y", "C"), ("Z", "D")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        if LooseVersion(pd.__version__) <= LooseVersion("0.24.2"):
+            self.assert_eq(
+                kdf.groupby(("X", "A")).ffill().sort_index(),
+                pdf.groupby(("X", "A")).ffill().sort_index().drop(("X", "A"), 1),
+            )
+        else:
+            self.assert_eq(
+                kdf.groupby(("X", "A")).ffill().sort_index(),
+                pdf.groupby(("X", "A")).ffill().sort_index(),
+            )
+
+    def test_bfill(self):
+        idx = np.random.rand(4 * 3)
+        pdf = pd.DataFrame(
+            {
+                "A": [1, 1, 2, 2] * 3,
+                "B": [2, 4, None, 3] * 3,
+                "C": [None, None, None, 1] * 3,
+                "D": [0, 1, 5, 4] * 3,
+            },
+            index=idx,
+        )
+        kdf = ps.from_pandas(pdf)
+
+        if LooseVersion(pd.__version__) <= LooseVersion("0.24.2"):
+            self.assert_eq(
+                kdf.groupby("A").bfill().sort_index(),
+                pdf.groupby("A").bfill().sort_index().drop("A", 1),
+            )
+            self.assert_eq(
+                kdf.groupby("A")[["B"]].bfill().sort_index(),
+                pdf.groupby("A")[["B"]].bfill().sort_index().drop("A", 1),
+            )
+        else:
+            self.assert_eq(
+                kdf.groupby("A").bfill().sort_index(), pdf.groupby("A").bfill().sort_index()
+            )
+            self.assert_eq(
+                kdf.groupby("A")[["B"]].bfill().sort_index(),
+                pdf.groupby("A")[["B"]].bfill().sort_index(),
+            )
+        self.assert_eq(
+            kdf.groupby("A")["B"].bfill().sort_index(), pdf.groupby("A")["B"].bfill().sort_index(),
+        )
+        self.assert_eq(kdf.groupby("A")["B"].bfill()[idx[6]], pdf.groupby("A")["B"].bfill()[idx[6]])
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("X", "A"), ("X", "B"), ("Y", "C"), ("Z", "D")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        if LooseVersion(pd.__version__) <= LooseVersion("0.24.2"):
+            self.assert_eq(
+                kdf.groupby(("X", "A")).bfill().sort_index(),
+                pdf.groupby(("X", "A")).bfill().sort_index().drop(("X", "A"), 1),
+            )
+        else:
+            self.assert_eq(
+                kdf.groupby(("X", "A")).bfill().sort_index(),
+                pdf.groupby(("X", "A")).bfill().sort_index(),
+            )
+
+    @unittest.skipIf(pd.__version__ < "0.24.0", "not supported before pandas 0.24.0")
+    def test_shift(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 1, 2, 2, 3, 3] * 3,
+                "b": [1, 1, 2, 2, 3, 4] * 3,
+                "c": [1, 4, 9, 16, 25, 36] * 3,
+            },
+            index=np.random.rand(6 * 3),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.groupby("a").shift().sort_index(), pdf.groupby("a").shift().sort_index())
+        # TODO: seems like a pandas' bug when fill_value is not None?
+        # self.assert_eq(kdf.groupby(['a', 'b']).shift(periods=-1, fill_value=0).sort_index(),
+        #                pdf.groupby(['a', 'b']).shift(periods=-1, fill_value=0).sort_index())
+        self.assert_eq(
+            kdf.groupby(["b"])["a"].shift().sort_index(),
+            pdf.groupby(["b"])["a"].shift().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(["a", "b"])["c"].shift().sort_index(),
+            pdf.groupby(["a", "b"])["c"].shift().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.b // 5).shift().sort_index(),
+            pdf.groupby(pdf.b // 5).shift().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.b // 5)["a"].shift().sort_index(),
+            pdf.groupby(pdf.b // 5)["a"].shift().sort_index(),
+        )
+        # TODO: known pandas' bug when fill_value is not None pandas>=1.0.0
+        # https://github.com/pandas-dev/pandas/issues/31971#issue-565171762
+        if LooseVersion(pd.__version__) < LooseVersion("1.0.0"):
+            self.assert_eq(
+                kdf.groupby(["b"])[["a", "c"]].shift(periods=-1, fill_value=0).sort_index(),
+                pdf.groupby(["b"])[["a", "c"]].shift(periods=-1, fill_value=0).sort_index(),
+            )
+        self.assert_eq(
+            kdf.a.rename().groupby(kdf.b).shift().sort_index(),
+            pdf.a.rename().groupby(pdf.b).shift().sort_index(),
+        )
+        self.assert_eq(
+            kdf.a.groupby(kdf.b.rename()).shift().sort_index(),
+            pdf.a.groupby(pdf.b.rename()).shift().sort_index(),
+        )
+        self.assert_eq(
+            kdf.a.rename().groupby(kdf.b.rename()).shift().sort_index(),
+            pdf.a.rename().groupby(pdf.b.rename()).shift().sort_index(),
+        )
+
+        self.assert_eq(kdf.groupby("a").shift().sum(), pdf.groupby("a").shift().sum().astype(int))
+        self.assert_eq(
+            kdf.a.rename().groupby(kdf.b).shift().sum(),
+            pdf.a.rename().groupby(pdf.b).shift().sum(),
+        )
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "c")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(
+            kdf.groupby(("x", "a")).shift().sort_index(),
+            pdf.groupby(("x", "a")).shift().sort_index(),
+        )
+        # TODO: seems like a pandas' bug when fill_value is not None?
+        # self.assert_eq(kdf.groupby([('x', 'a'), ('x', 'b')]).shift(periods=-1,
+        #                                                            fill_value=0).sort_index(),
+        #                pdf.groupby([('x', 'a'), ('x', 'b')]).shift(periods=-1,
+        #                                                            fill_value=0).sort_index())
+
+    def test_apply(self):
+        pdf = pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5, 6], "b": [1, 1, 2, 3, 5, 8], "c": [1, 4, 9, 16, 25, 36]},
+            columns=["a", "b", "c"],
+        )
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(
+            kdf.groupby("b").apply(lambda x: x + x.min()).sort_index(),
+            pdf.groupby("b").apply(lambda x: x + x.min()).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby("b").apply(len).sort_index(), pdf.groupby("b").apply(len).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby("b")["a"].apply(lambda x, y, z: x + x.min() + y * z, 10, z=20).sort_index(),
+            pdf.groupby("b")["a"].apply(lambda x, y, z: x + x.min() + y * z, 10, z=20).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby("b")[["a"]].apply(lambda x: x + x.min()).sort_index(),
+            pdf.groupby("b")[["a"]].apply(lambda x: x + x.min()).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(["a", "b"]).apply(lambda x, y, z: x + x.min() + y + z, 1, z=2).sort_index(),
+            pdf.groupby(["a", "b"]).apply(lambda x, y, z: x + x.min() + y + z, 1, z=2).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(["b"])["c"].apply(lambda x: 1).sort_index(),
+            pdf.groupby(["b"])["c"].apply(lambda x: 1).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(["b"])["c"].apply(len).sort_index(),
+            pdf.groupby(["b"])["c"].apply(len).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.b // 5).apply(lambda x: x + x.min()).sort_index(),
+            pdf.groupby(pdf.b // 5).apply(lambda x: x + x.min()).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.b // 5)["a"].apply(lambda x: x + x.min()).sort_index(),
+            pdf.groupby(pdf.b // 5)["a"].apply(lambda x: x + x.min()).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.b // 5)[["a"]].apply(lambda x: x + x.min()).sort_index(),
+            pdf.groupby(pdf.b // 5)[["a"]].apply(lambda x: x + x.min()).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.b // 5)[["a"]].apply(len).sort_index(),
+            pdf.groupby(pdf.b // 5)[["a"]].apply(len).sort_index(),
+            almost=True,
+        )
+        self.assert_eq(
+            kdf.a.rename().groupby(kdf.b).apply(lambda x: x + x.min()).sort_index(),
+            pdf.a.rename().groupby(pdf.b).apply(lambda x: x + x.min()).sort_index(),
+        )
+        self.assert_eq(
+            kdf.a.groupby(kdf.b.rename()).apply(lambda x: x + x.min()).sort_index(),
+            pdf.a.groupby(pdf.b.rename()).apply(lambda x: x + x.min()).sort_index(),
+        )
+        self.assert_eq(
+            kdf.a.rename().groupby(kdf.b.rename()).apply(lambda x: x + x.min()).sort_index(),
+            pdf.a.rename().groupby(pdf.b.rename()).apply(lambda x: x + x.min()).sort_index(),
+        )
+
+        with self.assertRaisesRegex(TypeError, "int object is not callable"):
+            kdf.groupby("b").apply(1)
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "c")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(
+            kdf.groupby(("x", "b")).apply(lambda x: 1).sort_index(),
+            pdf.groupby(("x", "b")).apply(lambda x: 1).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby([("x", "a"), ("x", "b")]).apply(lambda x: x + x.min()).sort_index(),
+            pdf.groupby([("x", "a"), ("x", "b")]).apply(lambda x: x + x.min()).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(("x", "b")).apply(len).sort_index(),
+            pdf.groupby(("x", "b")).apply(len).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby([("x", "a"), ("x", "b")]).apply(len).sort_index(),
+            pdf.groupby([("x", "a"), ("x", "b")]).apply(len).sort_index(),
+        )
+
+    def test_apply_without_shortcut(self):
+        with option_context("compute.shortcut_limit", 0):
+            self.test_apply()
+
+    def test_apply_negative(self):
+        def func(_) -> ps.Series[int]:
+            return pd.Series([1])
+
+        with self.assertRaisesRegex(TypeError, "Series as a return type hint at frame groupby"):
+            ps.range(10).groupby("id").apply(func)
+
+    def test_apply_with_new_dataframe(self):
+        pdf = pd.DataFrame(
+            {"timestamp": [0.0, 0.5, 1.0, 0.0, 0.5], "car_id": ["A", "A", "A", "B", "B"]}
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            kdf.groupby("car_id").apply(lambda _: pd.DataFrame({"column": [0.0]})).sort_index(),
+            pdf.groupby("car_id").apply(lambda _: pd.DataFrame({"column": [0.0]})).sort_index(),
+        )
+
+        self.assert_eq(
+            kdf.groupby("car_id")
+            .apply(lambda df: pd.DataFrame({"mean": [df["timestamp"].mean()]}))
+            .sort_index(),
+            pdf.groupby("car_id")
+            .apply(lambda df: pd.DataFrame({"mean": [df["timestamp"].mean()]}))
+            .sort_index(),
+        )
+
+        # dataframe with 1000+ records
+        pdf = pd.DataFrame(
+            {
+                "timestamp": [0.0, 0.5, 1.0, 0.0, 0.5] * 300,
+                "car_id": ["A", "A", "A", "B", "B"] * 300,
+            }
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            kdf.groupby("car_id").apply(lambda _: pd.DataFrame({"column": [0.0]})).sort_index(),
+            pdf.groupby("car_id").apply(lambda _: pd.DataFrame({"column": [0.0]})).sort_index(),
+        )
+
+        self.assert_eq(
+            kdf.groupby("car_id")
+            .apply(lambda df: pd.DataFrame({"mean": [df["timestamp"].mean()]}))
+            .sort_index(),
+            pdf.groupby("car_id")
+            .apply(lambda df: pd.DataFrame({"mean": [df["timestamp"].mean()]}))
+            .sort_index(),
+        )
+
+    def test_apply_with_new_dataframe_without_shortcut(self):
+        with option_context("compute.shortcut_limit", 0):
+            self.test_apply_with_new_dataframe()
+
+    def test_apply_key_handling(self):
+        pdf = pd.DataFrame(
+            {"d": [1.0, 1.0, 1.0, 2.0, 2.0, 2.0], "v": [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]}
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            kdf.groupby("d").apply(sum).sort_index(), pdf.groupby("d").apply(sum).sort_index()
+        )
+
+        with ps.option_context("compute.shortcut_limit", 1):
+            self.assert_eq(
+                kdf.groupby("d").apply(sum).sort_index(), pdf.groupby("d").apply(sum).sort_index()
+            )
+
+    def test_apply_with_side_effect(self):
+        pdf = pd.DataFrame(
+            {"d": [1.0, 1.0, 1.0, 2.0, 2.0, 2.0], "v": [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]}
+        )
+        kdf = ps.from_pandas(pdf)
+
+        acc = ps.utils.default_session().sparkContext.accumulator(0)
+
+        def sum_with_acc_frame(x) -> ps.DataFrame[np.float64, np.float64]:
+            nonlocal acc
+            acc += 1
+            return np.sum(x)
+
+        actual = kdf.groupby("d").apply(sum_with_acc_frame).sort_index()
+        actual.columns = ["d", "v"]
+        self.assert_eq(actual, pdf.groupby("d").apply(sum).sort_index().reset_index(drop=True))
+        self.assert_eq(acc.value, 2)
+
+        def sum_with_acc_series(x) -> np.float64:
+            nonlocal acc
+            acc += 1
+            return np.sum(x)
+
+        self.assert_eq(
+            kdf.groupby("d")["v"].apply(sum_with_acc_series).sort_index(),
+            pdf.groupby("d")["v"].apply(sum).sort_index().reset_index(drop=True),
+        )
+        self.assert_eq(acc.value, 4)
+
+    def test_transform(self):
+        pdf = pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5, 6], "b": [1, 1, 2, 3, 5, 8], "c": [1, 4, 9, 16, 25, 36]},
+            columns=["a", "b", "c"],
+        )
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(
+            kdf.groupby("b").transform(lambda x: x + x.min()).sort_index(),
+            pdf.groupby("b").transform(lambda x: x + x.min()).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby("b")["a"].transform(lambda x: x + x.min()).sort_index(),
+            pdf.groupby("b")["a"].transform(lambda x: x + x.min()).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby("b")[["a"]].transform(lambda x: x + x.min()).sort_index(),
+            pdf.groupby("b")[["a"]].transform(lambda x: x + x.min()).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(["a", "b"]).transform(lambda x: x + x.min()).sort_index(),
+            pdf.groupby(["a", "b"]).transform(lambda x: x + x.min()).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(["b"])["c"].transform(lambda x: x + x.min()).sort_index(),
+            pdf.groupby(["b"])["c"].transform(lambda x: x + x.min()).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.b // 5).transform(lambda x: x + x.min()).sort_index(),
+            pdf.groupby(pdf.b // 5).transform(lambda x: x + x.min()).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.b // 5)["a"].transform(lambda x: x + x.min()).sort_index(),
+            pdf.groupby(pdf.b // 5)["a"].transform(lambda x: x + x.min()).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf.b // 5)[["a"]].transform(lambda x: x + x.min()).sort_index(),
+            pdf.groupby(pdf.b // 5)[["a"]].transform(lambda x: x + x.min()).sort_index(),
+        )
+        self.assert_eq(
+            kdf.a.rename().groupby(kdf.b).transform(lambda x: x + x.min()).sort_index(),
+            pdf.a.rename().groupby(pdf.b).transform(lambda x: x + x.min()).sort_index(),
+        )
+        self.assert_eq(
+            kdf.a.groupby(kdf.b.rename()).transform(lambda x: x + x.min()).sort_index(),
+            pdf.a.groupby(pdf.b.rename()).transform(lambda x: x + x.min()).sort_index(),
+        )
+        self.assert_eq(
+            kdf.a.rename().groupby(kdf.b.rename()).transform(lambda x: x + x.min()).sort_index(),
+            pdf.a.rename().groupby(pdf.b.rename()).transform(lambda x: x + x.min()).sort_index(),
+        )
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "c")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(
+            kdf.groupby(("x", "b")).transform(lambda x: x + x.min()).sort_index(),
+            pdf.groupby(("x", "b")).transform(lambda x: x + x.min()).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby([("x", "a"), ("x", "b")]).transform(lambda x: x + x.min()).sort_index(),
+            pdf.groupby([("x", "a"), ("x", "b")]).transform(lambda x: x + x.min()).sort_index(),
+        )
+
+    def test_transform_without_shortcut(self):
+        with option_context("compute.shortcut_limit", 0):
+            self.test_transform()
+
+    def test_filter(self):
+        pdf = pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5, 6], "b": [1, 1, 2, 3, 5, 8], "c": [1, 4, 9, 16, 25, 36]},
+            columns=["a", "b", "c"],
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            kdf.groupby("b").filter(lambda x: any(x.a == 2)).sort_index(),
+            pdf.groupby("b").filter(lambda x: any(x.a == 2)).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby("b")["a"].filter(lambda x: any(x == 2)).sort_index(),
+            pdf.groupby("b")["a"].filter(lambda x: any(x == 2)).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby("b")[["a"]].filter(lambda x: any(x.a == 2)).sort_index(),
+            pdf.groupby("b")[["a"]].filter(lambda x: any(x.a == 2)).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(["a", "b"]).filter(lambda x: any(x.a == 2)).sort_index(),
+            pdf.groupby(["a", "b"]).filter(lambda x: any(x.a == 2)).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf["b"] // 5).filter(lambda x: any(x.a == 2)).sort_index(),
+            pdf.groupby(pdf["b"] // 5).filter(lambda x: any(x.a == 2)).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf["b"] // 5)["a"].filter(lambda x: any(x == 2)).sort_index(),
+            pdf.groupby(pdf["b"] // 5)["a"].filter(lambda x: any(x == 2)).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kdf["b"] // 5)[["a"]].filter(lambda x: any(x.a == 2)).sort_index(),
+            pdf.groupby(pdf["b"] // 5)[["a"]].filter(lambda x: any(x.a == 2)).sort_index(),
+        )
+        self.assert_eq(
+            kdf.a.rename().groupby(kdf.b).filter(lambda x: any(x == 2)).sort_index(),
+            pdf.a.rename().groupby(pdf.b).filter(lambda x: any(x == 2)).sort_index(),
+        )
+        self.assert_eq(
+            kdf.a.groupby(kdf.b.rename()).filter(lambda x: any(x == 2)).sort_index(),
+            pdf.a.groupby(pdf.b.rename()).filter(lambda x: any(x == 2)).sort_index(),
+        )
+        self.assert_eq(
+            kdf.a.rename().groupby(kdf.b.rename()).filter(lambda x: any(x == 2)).sort_index(),
+            pdf.a.rename().groupby(pdf.b.rename()).filter(lambda x: any(x == 2)).sort_index(),
+        )
+
+        with self.assertRaisesRegex(TypeError, "int object is not callable"):
+            kdf.groupby("b").filter(1)
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "c")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(
+            kdf.groupby(("x", "b")).filter(lambda x: any(x[("x", "a")] == 2)).sort_index(),
+            pdf.groupby(("x", "b")).filter(lambda x: any(x[("x", "a")] == 2)).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby([("x", "a"), ("x", "b")])
+            .filter(lambda x: any(x[("x", "a")] == 2))
+            .sort_index(),
+            pdf.groupby([("x", "a"), ("x", "b")])
+            .filter(lambda x: any(x[("x", "a")] == 2))
+            .sort_index(),
+        )
+
+    def test_idxmax(self):
+        pdf = pd.DataFrame(
+            {"a": [1, 1, 2, 2, 3] * 3, "b": [1, 2, 3, 4, 5] * 3, "c": [5, 4, 3, 2, 1] * 3}
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            pdf.groupby(["a"]).idxmax().sort_index(), kdf.groupby(["a"]).idxmax().sort_index()
+        )
+        self.assert_eq(
+            pdf.groupby(["a"]).idxmax(skipna=False).sort_index(),
+            kdf.groupby(["a"]).idxmax(skipna=False).sort_index(),
+        )
+        self.assert_eq(
+            pdf.groupby(["a"])["b"].idxmax().sort_index(),
+            kdf.groupby(["a"])["b"].idxmax().sort_index(),
+        )
+        self.assert_eq(
+            pdf.b.rename().groupby(pdf.a).idxmax().sort_index(),
+            kdf.b.rename().groupby(kdf.a).idxmax().sort_index(),
+        )
+        self.assert_eq(
+            pdf.b.groupby(pdf.a.rename()).idxmax().sort_index(),
+            kdf.b.groupby(kdf.a.rename()).idxmax().sort_index(),
+        )
+        self.assert_eq(
+            pdf.b.rename().groupby(pdf.a.rename()).idxmax().sort_index(),
+            kdf.b.rename().groupby(kdf.a.rename()).idxmax().sort_index(),
+        )
+
+        with self.assertRaisesRegex(ValueError, "idxmax only support one-level index now"):
+            kdf.set_index(["a", "b"]).groupby(["c"]).idxmax()
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "c")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(
+            pdf.groupby(("x", "a")).idxmax().sort_index(),
+            kdf.groupby(("x", "a")).idxmax().sort_index(),
+        )
+        self.assert_eq(
+            pdf.groupby(("x", "a")).idxmax(skipna=False).sort_index(),
+            kdf.groupby(("x", "a")).idxmax(skipna=False).sort_index(),
+        )
+
+    def test_idxmin(self):
+        pdf = pd.DataFrame(
+            {"a": [1, 1, 2, 2, 3] * 3, "b": [1, 2, 3, 4, 5] * 3, "c": [5, 4, 3, 2, 1] * 3}
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            pdf.groupby(["a"]).idxmin().sort_index(), kdf.groupby(["a"]).idxmin().sort_index()
+        )
+        self.assert_eq(
+            pdf.groupby(["a"]).idxmin(skipna=False).sort_index(),
+            kdf.groupby(["a"]).idxmin(skipna=False).sort_index(),
+        )
+        self.assert_eq(
+            pdf.groupby(["a"])["b"].idxmin().sort_index(),
+            kdf.groupby(["a"])["b"].idxmin().sort_index(),
+        )
+        self.assert_eq(
+            pdf.b.rename().groupby(pdf.a).idxmin().sort_index(),
+            kdf.b.rename().groupby(kdf.a).idxmin().sort_index(),
+        )
+        self.assert_eq(
+            pdf.b.groupby(pdf.a.rename()).idxmin().sort_index(),
+            kdf.b.groupby(kdf.a.rename()).idxmin().sort_index(),
+        )
+        self.assert_eq(
+            pdf.b.rename().groupby(pdf.a.rename()).idxmin().sort_index(),
+            kdf.b.rename().groupby(kdf.a.rename()).idxmin().sort_index(),
+        )
+
+        with self.assertRaisesRegex(ValueError, "idxmin only support one-level index now"):
+            kdf.set_index(["a", "b"]).groupby(["c"]).idxmin()
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "c")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(
+            pdf.groupby(("x", "a")).idxmin().sort_index(),
+            kdf.groupby(("x", "a")).idxmin().sort_index(),
+        )
+        self.assert_eq(
+            pdf.groupby(("x", "a")).idxmin(skipna=False).sort_index(),
+            kdf.groupby(("x", "a")).idxmin(skipna=False).sort_index(),
+        )
+
+    def test_head(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 1, 1, 1, 2, 2, 2, 3, 3, 3] * 3,
+                "b": [2, 3, 1, 4, 6, 9, 8, 10, 7, 5] * 3,
+                "c": [3, 5, 2, 5, 1, 2, 6, 4, 3, 6] * 3,
+            },
+            index=np.random.rand(10 * 3),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(pdf.groupby("a").head(2).sort_index(), kdf.groupby("a").head(2).sort_index())
+        self.assert_eq(
+            pdf.groupby("a").head(-2).sort_index(), kdf.groupby("a").head(-2).sort_index()
+        )
+        self.assert_eq(
+            pdf.groupby("a").head(100000).sort_index(), kdf.groupby("a").head(100000).sort_index()
+        )
+
+        self.assert_eq(
+            pdf.groupby("a")["b"].head(2).sort_index(), kdf.groupby("a")["b"].head(2).sort_index()
+        )
+        self.assert_eq(
+            pdf.groupby("a")["b"].head(-2).sort_index(), kdf.groupby("a")["b"].head(-2).sort_index()
+        )
+        self.assert_eq(
+            pdf.groupby("a")["b"].head(100000).sort_index(),
+            kdf.groupby("a")["b"].head(100000).sort_index(),
+        )
+
+        self.assert_eq(
+            pdf.groupby("a")[["b"]].head(2).sort_index(),
+            kdf.groupby("a")[["b"]].head(2).sort_index(),
+        )
+        self.assert_eq(
+            pdf.groupby("a")[["b"]].head(-2).sort_index(),
+            kdf.groupby("a")[["b"]].head(-2).sort_index(),
+        )
+        self.assert_eq(
+            pdf.groupby("a")[["b"]].head(100000).sort_index(),
+            kdf.groupby("a")[["b"]].head(100000).sort_index(),
+        )
+
+        self.assert_eq(
+            pdf.groupby(pdf.a // 2).head(2).sort_index(),
+            kdf.groupby(kdf.a // 2).head(2).sort_index(),
+        )
+        self.assert_eq(
+            pdf.groupby(pdf.a // 2)["b"].head(2).sort_index(),
+            kdf.groupby(kdf.a // 2)["b"].head(2).sort_index(),
+        )
+        self.assert_eq(
+            pdf.groupby(pdf.a // 2)[["b"]].head(2).sort_index(),
+            kdf.groupby(kdf.a // 2)[["b"]].head(2).sort_index(),
+        )
+
+        self.assert_eq(
+            pdf.b.rename().groupby(pdf.a).head(2).sort_index(),
+            kdf.b.rename().groupby(kdf.a).head(2).sort_index(),
+        )
+        self.assert_eq(
+            pdf.b.groupby(pdf.a.rename()).head(2).sort_index(),
+            kdf.b.groupby(kdf.a.rename()).head(2).sort_index(),
+        )
+        self.assert_eq(
+            pdf.b.rename().groupby(pdf.a.rename()).head(2).sort_index(),
+            kdf.b.rename().groupby(kdf.a.rename()).head(2).sort_index(),
+        )
+
+        # multi-index
+        midx = pd.MultiIndex(
+            [["x", "y"], ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"]],
+            [[0, 0, 0, 0, 0, 1, 1, 1, 1, 1], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]],
+        )
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 1, 1, 1, 2, 2, 2, 3, 3, 3],
+                "b": [2, 3, 1, 4, 6, 9, 8, 10, 7, 5],
+                "c": [3, 5, 2, 5, 1, 2, 6, 4, 3, 6],
+            },
+            columns=["a", "b", "c"],
+            index=midx,
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(pdf.groupby("a").head(2).sort_index(), kdf.groupby("a").head(2).sort_index())
+        self.assert_eq(
+            pdf.groupby("a").head(-2).sort_index(), kdf.groupby("a").head(-2).sort_index()
+        )
+        self.assert_eq(
+            pdf.groupby("a").head(100000).sort_index(), kdf.groupby("a").head(100000).sort_index()
+        )
+
+        self.assert_eq(
+            pdf.groupby("a")["b"].head(2).sort_index(), kdf.groupby("a")["b"].head(2).sort_index()
+        )
+        self.assert_eq(
+            pdf.groupby("a")["b"].head(-2).sort_index(), kdf.groupby("a")["b"].head(-2).sort_index()
+        )
+        self.assert_eq(
+            pdf.groupby("a")["b"].head(100000).sort_index(),
+            kdf.groupby("a")["b"].head(100000).sort_index(),
+        )
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "c")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(
+            pdf.groupby(("x", "a")).head(2).sort_index(),
+            kdf.groupby(("x", "a")).head(2).sort_index(),
+        )
+        self.assert_eq(
+            pdf.groupby(("x", "a")).head(-2).sort_index(),
+            kdf.groupby(("x", "a")).head(-2).sort_index(),
+        )
+        self.assert_eq(
+            pdf.groupby(("x", "a")).head(100000).sort_index(),
+            kdf.groupby(("x", "a")).head(100000).sort_index(),
+        )
+
+    def test_missing(self):
+        kdf = ps.DataFrame({"a": [1, 2, 3, 4, 5, 6, 7, 8, 9]})
+
+        # DataFrameGroupBy functions
+        missing_functions = inspect.getmembers(
+            MissingPandasLikeDataFrameGroupBy, inspect.isfunction
+        )
+        unsupported_functions = [
+            name for (name, type_) in missing_functions if type_.__name__ == "unsupported_function"
+        ]
+        for name in unsupported_functions:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "method.*GroupBy.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.groupby("a"), name)()
+
+        deprecated_functions = [
+            name for (name, type_) in missing_functions if type_.__name__ == "deprecated_function"
+        ]
+        for name in deprecated_functions:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "method.*GroupBy.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.groupby("a"), name)()
+
+        # SeriesGroupBy functions
+        missing_functions = inspect.getmembers(MissingPandasLikeSeriesGroupBy, inspect.isfunction)
+        unsupported_functions = [
+            name for (name, type_) in missing_functions if type_.__name__ == "unsupported_function"
+        ]
+        for name in unsupported_functions:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "method.*GroupBy.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.a.groupby(kdf.a), name)()
+
+        deprecated_functions = [
+            name for (name, type_) in missing_functions if type_.__name__ == "deprecated_function"
+        ]
+        for name in deprecated_functions:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "method.*GroupBy.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.a.groupby(kdf.a), name)()
+
+        # DataFrameGroupBy properties
+        missing_properties = inspect.getmembers(
+            MissingPandasLikeDataFrameGroupBy, lambda o: isinstance(o, property)
+        )
+        unsupported_properties = [
+            name
+            for (name, type_) in missing_properties
+            if type_.fget.__name__ == "unsupported_property"
+        ]
+        for name in unsupported_properties:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "property.*GroupBy.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.groupby("a"), name)
+        deprecated_properties = [
+            name
+            for (name, type_) in missing_properties
+            if type_.fget.__name__ == "deprecated_property"
+        ]
+        for name in deprecated_properties:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "property.*GroupBy.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.groupby("a"), name)
+
+        # SeriesGroupBy properties
+        missing_properties = inspect.getmembers(
+            MissingPandasLikeSeriesGroupBy, lambda o: isinstance(o, property)
+        )
+        unsupported_properties = [
+            name
+            for (name, type_) in missing_properties
+            if type_.fget.__name__ == "unsupported_property"
+        ]
+        for name in unsupported_properties:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "property.*GroupBy.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.a.groupby(kdf.a), name)
+        deprecated_properties = [
+            name
+            for (name, type_) in missing_properties
+            if type_.fget.__name__ == "deprecated_property"
+        ]
+        for name in deprecated_properties:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "property.*GroupBy.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.a.groupby(kdf.a), name)
+
+    @staticmethod
+    def test_is_multi_agg_with_relabel():
+
+        assert is_multi_agg_with_relabel(a="max") is False
+        assert is_multi_agg_with_relabel(a_min=("a", "max"), a_max=("a", "min")) is True
+
+    def test_get_group(self):
+        pdf = pd.DataFrame(
+            [
+                ("falcon", "bird", 389.0),
+                ("parrot", "bird", 24.0),
+                ("lion", "mammal", 80.5),
+                ("monkey", "mammal", np.nan),
+            ],
+            columns=["name", "class", "max_speed"],
+            index=[0, 2, 3, 1],
+        )
+        pdf.columns.name = "Koalas"
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            kdf.groupby("class").get_group("bird"), pdf.groupby("class").get_group("bird"),
+        )
+        self.assert_eq(
+            kdf.groupby("class")["name"].get_group("mammal"),
+            pdf.groupby("class")["name"].get_group("mammal"),
+        )
+        self.assert_eq(
+            kdf.groupby("class")[["name"]].get_group("mammal"),
+            pdf.groupby("class")[["name"]].get_group("mammal"),
+        )
+        self.assert_eq(
+            kdf.groupby(["class", "name"]).get_group(("mammal", "lion")),
+            pdf.groupby(["class", "name"]).get_group(("mammal", "lion")),
+        )
+        self.assert_eq(
+            kdf.groupby(["class", "name"])["max_speed"].get_group(("mammal", "lion")),
+            pdf.groupby(["class", "name"])["max_speed"].get_group(("mammal", "lion")),
+        )
+        self.assert_eq(
+            kdf.groupby(["class", "name"])[["max_speed"]].get_group(("mammal", "lion")),
+            pdf.groupby(["class", "name"])[["max_speed"]].get_group(("mammal", "lion")),
+        )
+        self.assert_eq(
+            (kdf.max_speed + 1).groupby(kdf["class"]).get_group("mammal"),
+            (pdf.max_speed + 1).groupby(pdf["class"]).get_group("mammal"),
+        )
+        self.assert_eq(
+            kdf.groupby("max_speed").get_group(80.5), pdf.groupby("max_speed").get_group(80.5),
+        )
+
+        self.assertRaises(KeyError, lambda: kdf.groupby("class").get_group("fish"))
+        self.assertRaises(TypeError, lambda: kdf.groupby("class").get_group(["bird", "mammal"]))
+        self.assertRaises(KeyError, lambda: kdf.groupby("class")["name"].get_group("fish"))
+        self.assertRaises(
+            TypeError, lambda: kdf.groupby("class")["name"].get_group(["bird", "mammal"])
+        )
+        self.assertRaises(
+            KeyError, lambda: kdf.groupby(["class", "name"]).get_group(("lion", "mammal"))
+        )
+        self.assertRaises(ValueError, lambda: kdf.groupby(["class", "name"]).get_group(("lion",)))
+        self.assertRaises(ValueError, lambda: kdf.groupby(["class", "name"]).get_group(("mammal",)))
+        self.assertRaises(ValueError, lambda: kdf.groupby(["class", "name"]).get_group("mammal"))
+
+        # MultiIndex columns
+        pdf.columns = pd.MultiIndex.from_tuples([("A", "name"), ("B", "class"), ("C", "max_speed")])
+        pdf.columns.names = ["Hello", "Koalas"]
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(
+            kdf.groupby(("B", "class")).get_group("bird"),
+            pdf.groupby(("B", "class")).get_group("bird"),
+        )
+        self.assert_eq(
+            kdf.groupby(("B", "class"))[[("A", "name")]].get_group("mammal"),
+            pdf.groupby(("B", "class"))[[("A", "name")]].get_group("mammal"),
+        )
+        self.assert_eq(
+            kdf.groupby([("B", "class"), ("A", "name")]).get_group(("mammal", "lion")),
+            pdf.groupby([("B", "class"), ("A", "name")]).get_group(("mammal", "lion")),
+        )
+        self.assert_eq(
+            kdf.groupby([("B", "class"), ("A", "name")])[[("C", "max_speed")]].get_group(
+                ("mammal", "lion")
+            ),
+            pdf.groupby([("B", "class"), ("A", "name")])[[("C", "max_speed")]].get_group(
+                ("mammal", "lion")
+            ),
+        )
+        self.assert_eq(
+            (kdf[("C", "max_speed")] + 1).groupby(kdf[("B", "class")]).get_group("mammal"),
+            (pdf[("C", "max_speed")] + 1).groupby(pdf[("B", "class")]).get_group("mammal"),
+        )
+        self.assert_eq(
+            kdf.groupby(("C", "max_speed")).get_group(80.5),
+            pdf.groupby(("C", "max_speed")).get_group(80.5),
+        )
+
+        self.assertRaises(KeyError, lambda: kdf.groupby(("B", "class")).get_group("fish"))
+        self.assertRaises(
+            TypeError, lambda: kdf.groupby(("B", "class")).get_group(["bird", "mammal"])
+        )
+        self.assertRaises(
+            KeyError, lambda: kdf.groupby(("B", "class"))[("A", "name")].get_group("fish")
+        )
+        self.assertRaises(
+            KeyError,
+            lambda: kdf.groupby([("B", "class"), ("A", "name")]).get_group(("lion", "mammal")),
+        )
+        self.assertRaises(
+            ValueError, lambda: kdf.groupby([("B", "class"), ("A", "name")]).get_group(("lion",)),
+        )
+        self.assertRaises(
+            ValueError, lambda: kdf.groupby([("B", "class"), ("A", "name")]).get_group(("mammal",))
+        )
+        self.assertRaises(
+            ValueError, lambda: kdf.groupby([("B", "class"), ("A", "name")]).get_group("mammal")
+        )
+
+    def test_median(self):
+        kdf = ps.DataFrame(
+            {
+                "a": [1.0, 1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 3.0, 3.0, 3.0],
+                "b": [2.0, 3.0, 1.0, 4.0, 6.0, 9.0, 8.0, 10.0, 7.0, 5.0],
+                "c": [3.0, 5.0, 2.0, 5.0, 1.0, 2.0, 6.0, 4.0, 3.0, 6.0],
+            },
+            columns=["a", "b", "c"],
+            index=[7, 2, 4, 1, 3, 4, 9, 10, 5, 6],
+        )
+        # DataFrame
+        expected_result = ps.DataFrame(
+            {"b": [2.0, 8.0, 7.0], "c": [3.0, 2.0, 4.0]}, index=pd.Index([1.0, 2.0, 3.0], name="a")
+        )
+        self.assert_eq(expected_result, kdf.groupby("a").median().sort_index())
+        # Series
+        expected_result = ps.Series(
+            [2.0, 8.0, 7.0], name="b", index=pd.Index([1.0, 2.0, 3.0], name="a")
+        )
+        self.assert_eq(expected_result, kdf.groupby("a")["b"].median().sort_index())
+
+        with self.assertRaisesRegex(TypeError, "accuracy must be an integer; however"):
+            kdf.groupby("a").median(accuracy="a")
+
+    def test_tail(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 1, 1, 1, 2, 2, 2, 3, 3, 3] * 3,
+                "b": [2, 3, 1, 4, 6, 9, 8, 10, 7, 5] * 3,
+                "c": [3, 5, 2, 5, 1, 2, 6, 4, 3, 6] * 3,
+            },
+            index=np.random.rand(10 * 3),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(pdf.groupby("a").tail(2).sort_index(), kdf.groupby("a").tail(2).sort_index())
+        self.assert_eq(
+            pdf.groupby("a").tail(-2).sort_index(), kdf.groupby("a").tail(-2).sort_index()
+        )
+        self.assert_eq(
+            pdf.groupby("a").tail(100000).sort_index(), kdf.groupby("a").tail(100000).sort_index()
+        )
+
+        self.assert_eq(
+            pdf.groupby("a")["b"].tail(2).sort_index(), kdf.groupby("a")["b"].tail(2).sort_index()
+        )
+        self.assert_eq(
+            pdf.groupby("a")["b"].tail(-2).sort_index(), kdf.groupby("a")["b"].tail(-2).sort_index()
+        )
+        self.assert_eq(
+            pdf.groupby("a")["b"].tail(100000).sort_index(),
+            kdf.groupby("a")["b"].tail(100000).sort_index(),
+        )
+
+        self.assert_eq(
+            pdf.groupby("a")[["b"]].tail(2).sort_index(),
+            kdf.groupby("a")[["b"]].tail(2).sort_index(),
+        )
+        self.assert_eq(
+            pdf.groupby("a")[["b"]].tail(-2).sort_index(),
+            kdf.groupby("a")[["b"]].tail(-2).sort_index(),
+        )
+        self.assert_eq(
+            pdf.groupby("a")[["b"]].tail(100000).sort_index(),
+            kdf.groupby("a")[["b"]].tail(100000).sort_index(),
+        )
+
+        self.assert_eq(
+            pdf.groupby(pdf.a // 2).tail(2).sort_index(),
+            kdf.groupby(kdf.a // 2).tail(2).sort_index(),
+        )
+        self.assert_eq(
+            pdf.groupby(pdf.a // 2)["b"].tail(2).sort_index(),
+            kdf.groupby(kdf.a // 2)["b"].tail(2).sort_index(),
+        )
+        self.assert_eq(
+            pdf.groupby(pdf.a // 2)[["b"]].tail(2).sort_index(),
+            kdf.groupby(kdf.a // 2)[["b"]].tail(2).sort_index(),
+        )
+
+        self.assert_eq(
+            pdf.b.rename().groupby(pdf.a).tail(2).sort_index(),
+            kdf.b.rename().groupby(kdf.a).tail(2).sort_index(),
+        )
+        self.assert_eq(
+            pdf.b.groupby(pdf.a.rename()).tail(2).sort_index(),
+            kdf.b.groupby(kdf.a.rename()).tail(2).sort_index(),
+        )
+        self.assert_eq(
+            pdf.b.rename().groupby(pdf.a.rename()).tail(2).sort_index(),
+            kdf.b.rename().groupby(kdf.a.rename()).tail(2).sort_index(),
+        )
+
+        # multi-index
+        midx = pd.MultiIndex(
+            [["x", "y"], ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"]],
+            [[0, 0, 0, 0, 0, 1, 1, 1, 1, 1], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]],
+        )
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 1, 1, 1, 2, 2, 2, 3, 3, 3],
+                "b": [2, 3, 1, 4, 6, 9, 8, 10, 7, 5],
+                "c": [3, 5, 2, 5, 1, 2, 6, 4, 3, 6],
+            },
+            columns=["a", "b", "c"],
+            index=midx,
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(pdf.groupby("a").tail(2).sort_index(), kdf.groupby("a").tail(2).sort_index())
+        self.assert_eq(
+            pdf.groupby("a").tail(-2).sort_index(), kdf.groupby("a").tail(-2).sort_index()
+        )
+        self.assert_eq(
+            pdf.groupby("a").tail(100000).sort_index(), kdf.groupby("a").tail(100000).sort_index()
+        )
+
+        self.assert_eq(
+            pdf.groupby("a")["b"].tail(2).sort_index(), kdf.groupby("a")["b"].tail(2).sort_index()
+        )
+        self.assert_eq(
+            pdf.groupby("a")["b"].tail(-2).sort_index(), kdf.groupby("a")["b"].tail(-2).sort_index()
+        )
+        self.assert_eq(
+            pdf.groupby("a")["b"].tail(100000).sort_index(),
+            kdf.groupby("a")["b"].tail(100000).sort_index(),
+        )
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "c")])
+        pdf.columns = columns
+        kdf.columns = columns
+
+        self.assert_eq(
+            pdf.groupby(("x", "a")).tail(2).sort_index(),
+            kdf.groupby(("x", "a")).tail(2).sort_index(),
+        )
+        self.assert_eq(
+            pdf.groupby(("x", "a")).tail(-2).sort_index(),
+            kdf.groupby(("x", "a")).tail(-2).sort_index(),
+        )
+        self.assert_eq(
+            pdf.groupby(("x", "a")).tail(100000).sort_index(),
+            kdf.groupby(("x", "a")).tail(100000).sort_index(),
+        )
+
+    def test_ddof(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 1, 1, 1, 2, 2, 2, 3, 3, 3] * 3,
+                "b": [2, 3, 1, 4, 6, 9, 8, 10, 7, 5] * 3,
+                "c": [3, 5, 2, 5, 1, 2, 6, 4, 3, 6] * 3,
+            },
+            index=np.random.rand(10 * 3),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        for ddof in (0, 1):
+            # std
+            self.assert_eq(
+                pdf.groupby("a").std(ddof=ddof).sort_index(),
+                kdf.groupby("a").std(ddof=ddof).sort_index(),
+                check_exact=False,
+            )
+            self.assert_eq(
+                pdf.groupby("a")["b"].std(ddof=ddof).sort_index(),
+                kdf.groupby("a")["b"].std(ddof=ddof).sort_index(),
+                check_exact=False,
+            )
+            # var
+            self.assert_eq(
+                pdf.groupby("a").var(ddof=ddof).sort_index(),
+                kdf.groupby("a").var(ddof=ddof).sort_index(),
+                check_exact=False,
+            )
+            self.assert_eq(
+                pdf.groupby("a")["b"].var(ddof=ddof).sort_index(),
+                kdf.groupby("a")["b"].var(ddof=ddof).sort_index(),
+                check_exact=False,
+            )
+
+
+if __name__ == "__main__":
+    from pyspark.pandas.tests.test_groupby import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_indexing.py b/python/pyspark/pandas/tests/test_indexing.py
new file mode 100644
index 0000000000000..0d02c46b0e2d1
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_indexing.py
@@ -0,0 +1,1327 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import datetime
+from distutils.version import LooseVersion
+import unittest
+
+import numpy as np
+import pandas as pd
+
+from pyspark import pandas as ps
+from pyspark.pandas.exceptions import SparkPandasIndexingError
+from pyspark.testing.pandasutils import ComparisonTestBase, PandasOnSparkTestCase, compare_both
+
+
+class BasicIndexingTest(ComparisonTestBase):
+    @property
+    def pdf(self):
+        return pd.DataFrame(
+            {"month": [1, 4, 7, 10], "year": [2012, 2014, 2013, 2014], "sale": [55, 40, 84, 31]}
+        )
+
+    @compare_both(almost=False)
+    def test_indexing(self, df):
+        df1 = df.set_index("month")
+        yield df1
+
+        yield df.set_index("month", drop=False)
+        yield df.set_index("month", append=True)
+        yield df.set_index(["year", "month"])
+        yield df.set_index(["year", "month"], drop=False)
+        yield df.set_index(["year", "month"], append=True)
+
+        yield df1.set_index("year", drop=False, append=True)
+
+        df2 = df1.copy()
+        df2.set_index("year", append=True, inplace=True)
+        yield df2
+
+        self.assertRaisesRegex(KeyError, "unknown", lambda: df.set_index("unknown"))
+        self.assertRaisesRegex(KeyError, "unknown", lambda: df.set_index(["month", "unknown"]))
+
+        for d in [df, df1, df2]:
+            yield d.reset_index()
+            yield d.reset_index(drop=True)
+
+        yield df1.reset_index(level=0)
+        yield df2.reset_index(level=1)
+        yield df2.reset_index(level=[1, 0])
+        yield df1.reset_index(level="month")
+        yield df2.reset_index(level="year")
+        yield df2.reset_index(level=["month", "year"])
+        yield df2.reset_index(level="month", drop=True)
+        yield df2.reset_index(level=["month", "year"], drop=True)
+
+        self.assertRaisesRegex(
+            IndexError,
+            "Too many levels: Index has only 1 level, not 3",
+            lambda: df1.reset_index(level=2),
+        )
+        self.assertRaisesRegex(
+            IndexError,
+            "Too many levels: Index has only 1 level, not 4",
+            lambda: df1.reset_index(level=[3, 2]),
+        )
+        self.assertRaisesRegex(KeyError, "unknown.*month", lambda: df1.reset_index(level="unknown"))
+        self.assertRaisesRegex(
+            KeyError, "Level unknown not found", lambda: df2.reset_index(level="unknown")
+        )
+
+        df3 = df2.copy()
+        df3.reset_index(inplace=True)
+        yield df3
+
+        yield df1.sale.reset_index()
+        yield df1.sale.reset_index(level=0)
+        yield df2.sale.reset_index(level=[1, 0])
+        yield df1.sale.reset_index(drop=True)
+        yield df1.sale.reset_index(name="s")
+        yield df1.sale.reset_index(name="s", drop=True)
+
+        s = df1.sale
+        self.assertRaisesRegex(
+            TypeError,
+            "Cannot reset_index inplace on a Series to create a DataFrame",
+            lambda: s.reset_index(inplace=True),
+        )
+        s.reset_index(drop=True, inplace=True)
+        yield s
+        yield df1
+
+        # multi-index columns
+        df4 = df.copy()
+        df4.columns = pd.MultiIndex.from_tuples(
+            [("cal", "month"), ("cal", "year"), ("num", "sale")]
+        )
+        df5 = df4.set_index(("cal", "month"))
+        yield df5
+        yield df4.set_index([("cal", "month"), ("num", "sale")])
+
+        self.assertRaises(KeyError, lambda: df5.reset_index(level=("cal", "month")))
+
+        yield df5.reset_index(level=[("cal", "month")])
+
+        # non-string names
+        df6 = df.copy()
+        df6.columns = [10.0, 20.0, 30.0]
+        df7 = df6.set_index(10.0)
+        yield df7
+        yield df6.set_index([10.0, 30.0])
+
+        yield df7.reset_index(level=10.0)
+        yield df7.reset_index(level=[10.0])
+
+        df8 = df.copy()
+        df8.columns = pd.MultiIndex.from_tuples([(10, "month"), (10, "year"), (20, "sale")])
+        df9 = df8.set_index((10, "month"))
+        yield df9
+        yield df8.set_index([(10, "month"), (20, "sale")])
+
+        yield df9.reset_index(level=[(10, "month")])
+
+    def test_from_pandas_with_explicit_index(self):
+        pdf = self.pdf
+
+        df1 = ps.from_pandas(pdf.set_index("month"))
+        self.assertPandasEqual(df1.to_pandas(), pdf.set_index("month"))
+
+        df2 = ps.from_pandas(pdf.set_index(["year", "month"]))
+        self.assertPandasEqual(df2.to_pandas(), pdf.set_index(["year", "month"]))
+
+    def test_limitations(self):
+        df = self.kdf.set_index("month")
+
+        self.assertRaisesRegex(
+            ValueError,
+            "Level should be all int or all string.",
+            lambda: df.reset_index([1, "month"]),
+        )
+
+
+class IndexingTest(PandasOnSparkTestCase):
+    @property
+    def pdf(self):
+        return pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5, 6, 7, 8, 9], "b": [4, 5, 6, 3, 2, 1, 0, 0, 0]},
+            index=[0, 1, 3, 5, 6, 8, 9, 9, 9],
+        )
+
+    @property
+    def kdf(self):
+        return ps.from_pandas(self.pdf)
+
+    @property
+    def pdf2(self):
+        return pd.DataFrame(
+            {0: [1, 2, 3, 4, 5, 6, 7, 8, 9], 1: [4, 5, 6, 3, 2, 1, 0, 0, 0]},
+            index=[0, 1, 3, 5, 6, 8, 9, 9, 9],
+        )
+
+    @property
+    def kdf2(self):
+        return ps.from_pandas(self.pdf2)
+
+    def test_at(self):
+        pdf = self.pdf
+        kdf = self.kdf
+        # Create the equivalent of pdf.loc[3] as a Koalas Series
+        # This is necessary because .loc[n] does not currently work with Koalas DataFrames (#383)
+        test_series = ps.Series([3, 6], index=["a", "b"], name="3")
+
+        # Assert invalided signatures raise TypeError
+        with self.assertRaises(TypeError, msg="Use DataFrame.at like .at[row_index, column_name]"):
+            kdf.at[3]
+        with self.assertRaises(TypeError, msg="Use DataFrame.at like .at[row_index, column_name]"):
+            kdf.at["ab"]  # 'ab' is of length 2 but str type instead of tuple
+        with self.assertRaises(TypeError, msg="Use Series.at like .at[column_name]"):
+            test_series.at[3, "b"]
+
+        # Assert .at for DataFrames
+        self.assertEqual(kdf.at[3, "b"], 6)
+        self.assertEqual(kdf.at[3, "b"], pdf.at[3, "b"])
+        self.assert_eq(kdf.at[9, "b"], np.array([0, 0, 0]))
+        self.assert_eq(kdf.at[9, "b"], pdf.at[9, "b"])
+
+        # Assert .at for Series
+        self.assertEqual(test_series.at["b"], 6)
+        self.assertEqual(test_series.at["b"], pdf.loc[3].at["b"])
+
+        # Assert multi-character indices
+        self.assertEqual(
+            ps.Series([0, 1], index=["ab", "cd"]).at["ab"],
+            pd.Series([0, 1], index=["ab", "cd"]).at["ab"],
+        )
+
+        # Assert invalid column or index names result in a KeyError like with pandas
+        with self.assertRaises(KeyError, msg="x"):
+            kdf.at[3, "x"]
+        with self.assertRaises(KeyError, msg=99):
+            kdf.at[99, "b"]
+
+        with self.assertRaises(ValueError):
+            kdf.at[(3, 6), "b"]
+        with self.assertRaises(KeyError):
+            kdf.at[3, ("x", "b")]
+
+        # Assert setting values fails
+        with self.assertRaises(TypeError):
+            kdf.at[3, "b"] = 10
+
+        # non-string column names
+        pdf = self.pdf2
+        kdf = self.kdf2
+
+        # Assert .at for DataFrames
+        self.assertEqual(kdf.at[3, 1], 6)
+        self.assertEqual(kdf.at[3, 1], pdf.at[3, 1])
+        self.assert_eq(kdf.at[9, 1], np.array([0, 0, 0]))
+        self.assert_eq(kdf.at[9, 1], pdf.at[9, 1])
+
+    def test_at_multiindex(self):
+        pdf = self.pdf.set_index("b", append=True)
+        kdf = self.kdf.set_index("b", append=True)
+
+        # TODO: seems like a pandas' bug in pandas>=1.1.0
+        if LooseVersion(pd.__version__) < LooseVersion("1.1.0"):
+            self.assert_eq(kdf.at[(3, 6), "a"], pdf.at[(3, 6), "a"])
+            self.assert_eq(kdf.at[(3,), "a"], pdf.at[(3,), "a"])
+            self.assert_eq(list(kdf.at[(9, 0), "a"]), list(pdf.at[(9, 0), "a"]))
+            self.assert_eq(list(kdf.at[(9,), "a"]), list(pdf.at[(9,), "a"]))
+        else:
+            self.assert_eq(kdf.at[(3, 6), "a"], 3)
+            self.assert_eq(kdf.at[(3,), "a"], np.array([3]))
+            self.assert_eq(list(kdf.at[(9, 0), "a"]), [7, 8, 9])
+            self.assert_eq(list(kdf.at[(9,), "a"]), [7, 8, 9])
+
+        with self.assertRaises(ValueError):
+            kdf.at[3, "a"]
+
+    def test_at_multiindex_columns(self):
+        arrays = [np.array(["bar", "bar", "baz", "baz"]), np.array(["one", "two", "one", "two"])]
+
+        pdf = pd.DataFrame(np.random.randn(3, 4), index=["A", "B", "C"], columns=arrays)
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.at["B", ("bar", "one")], pdf.at["B", ("bar", "one")])
+
+        with self.assertRaises(KeyError):
+            kdf.at["B", "bar"]
+
+        # non-string column names
+        arrays = [np.array([0, 0, 1, 1]), np.array([1, 2, 1, 2])]
+
+        pdf = pd.DataFrame(np.random.randn(3, 4), index=["A", "B", "C"], columns=arrays)
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.at["B", (0, 1)], pdf.at["B", (0, 1)])
+
+    def test_iat(self):
+        pdf = self.pdf
+        kdf = self.kdf
+        # Create the equivalent of pdf.loc[3] as a Koalas Series
+        # This is necessary because .loc[n] does not currently work with Koalas DataFrames (#383)
+        test_series = ps.Series([3, 6], index=["a", "b"], name="3")
+
+        # Assert invalided signatures raise TypeError
+        with self.assertRaises(
+            TypeError,
+            msg="Use DataFrame.at like .iat[row_interget_position, column_integer_position]",
+        ):
+            kdf.iat[3]
+        with self.assertRaises(
+            ValueError, msg="iAt based indexing on multi-index can only have tuple values"
+        ):
+            kdf.iat[3, "b"]  # 'ab' is of length 2 but str type instead of tuple
+        with self.assertRaises(TypeError, msg="Use Series.iat like .iat[row_integer_position]"):
+            test_series.iat[3, "b"]
+
+        # Assert .iat for DataFrames
+        self.assertEqual(kdf.iat[7, 0], 8)
+        self.assertEqual(kdf.iat[7, 0], pdf.iat[7, 0])
+
+        # Assert .iat for Series
+        self.assertEqual(test_series.iat[1], 6)
+        self.assertEqual(test_series.iat[1], pdf.loc[3].iat[1])
+
+        # Assert invalid column or integer position result in a KeyError like with pandas
+        with self.assertRaises(KeyError, msg=99):
+            kdf.iat[0, 99]
+        with self.assertRaises(KeyError, msg=99):
+            kdf.iat[99, 0]
+
+        with self.assertRaises(ValueError):
+            kdf.iat[(1, 1), 1]
+        with self.assertRaises(ValueError):
+            kdf.iat[1, (1, 1)]
+
+        # Assert setting values fails
+        with self.assertRaises(TypeError):
+            kdf.iat[4, 1] = 10
+
+    def test_iat_multiindex(self):
+        pdf = self.pdf.set_index("b", append=True)
+        kdf = self.kdf.set_index("b", append=True)
+
+        self.assert_eq(kdf.iat[7, 0], pdf.iat[7, 0])
+
+        with self.assertRaises(ValueError):
+            kdf.iat[3, "a"]
+
+    def test_iat_multiindex_columns(self):
+        arrays = [np.array(["bar", "bar", "baz", "baz"]), np.array(["one", "two", "one", "two"])]
+
+        pdf = pd.DataFrame(np.random.randn(3, 4), index=["A", "B", "C"], columns=arrays)
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.iat[1, 3], pdf.iat[1, 3])
+
+        with self.assertRaises(KeyError):
+            kdf.iat[0, 99]
+        with self.assertRaises(KeyError):
+            kdf.iat[99, 0]
+
+    def test_loc(self):
+        kdf = self.kdf
+        pdf = self.pdf
+
+        self.assert_eq(kdf.loc[5:5], pdf.loc[5:5])
+        self.assert_eq(kdf.loc[3:8], pdf.loc[3:8])
+        self.assert_eq(kdf.loc[:8], pdf.loc[:8])
+        self.assert_eq(kdf.loc[3:], pdf.loc[3:])
+        self.assert_eq(kdf.loc[[5]], pdf.loc[[5]])
+        self.assert_eq(kdf.loc[:], pdf.loc[:])
+
+        # TODO?: self.assert_eq(kdf.loc[[3, 4, 1, 8]], pdf.loc[[3, 4, 1, 8]])
+        # TODO?: self.assert_eq(kdf.loc[[3, 4, 1, 9]], pdf.loc[[3, 4, 1, 9]])
+        # TODO?: self.assert_eq(kdf.loc[np.array([3, 4, 1, 9])], pdf.loc[np.array([3, 4, 1, 9])])
+
+        self.assert_eq(kdf.a.loc[5:5], pdf.a.loc[5:5])
+        self.assert_eq(kdf.a.loc[3:8], pdf.a.loc[3:8])
+        self.assert_eq(kdf.a.loc[:8], pdf.a.loc[:8])
+        self.assert_eq(kdf.a.loc[3:], pdf.a.loc[3:])
+        self.assert_eq(kdf.a.loc[[5]], pdf.a.loc[[5]])
+
+        # TODO?: self.assert_eq(kdf.a.loc[[3, 4, 1, 8]], pdf.a.loc[[3, 4, 1, 8]])
+        # TODO?: self.assert_eq(kdf.a.loc[[3, 4, 1, 9]], pdf.a.loc[[3, 4, 1, 9]])
+        # TODO?: self.assert_eq(kdf.a.loc[np.array([3, 4, 1, 9])],
+        #                       pdf.a.loc[np.array([3, 4, 1, 9])])
+
+        self.assert_eq(kdf.a.loc[[]], pdf.a.loc[[]])
+        self.assert_eq(kdf.a.loc[np.array([])], pdf.a.loc[np.array([])])
+
+        self.assert_eq(kdf.loc[1000:], pdf.loc[1000:])
+        self.assert_eq(kdf.loc[-2000:-1000], pdf.loc[-2000:-1000])
+
+        self.assert_eq(kdf.loc[5], pdf.loc[5])
+        self.assert_eq(kdf.loc[9], pdf.loc[9])
+        self.assert_eq(kdf.a.loc[5], pdf.a.loc[5])
+        self.assert_eq(kdf.a.loc[9], pdf.a.loc[9])
+
+        self.assertRaises(KeyError, lambda: kdf.loc[10])
+        self.assertRaises(KeyError, lambda: kdf.a.loc[10])
+
+        # monotonically increasing index test
+        pdf = pd.DataFrame({"a": [1, 2, 3, 4, 5, 6, 7, 8, 9]}, index=[0, 1, 1, 2, 2, 2, 4, 5, 6])
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.loc[:2], pdf.loc[:2])
+        self.assert_eq(kdf.loc[:3], pdf.loc[:3])
+        self.assert_eq(kdf.loc[3:], pdf.loc[3:])
+        self.assert_eq(kdf.loc[4:], pdf.loc[4:])
+        self.assert_eq(kdf.loc[3:2], pdf.loc[3:2])
+        self.assert_eq(kdf.loc[-1:2], pdf.loc[-1:2])
+        self.assert_eq(kdf.loc[3:10], pdf.loc[3:10])
+
+        # monotonically decreasing index test
+        pdf = pd.DataFrame({"a": [1, 2, 3, 4, 5, 6, 7, 8, 9]}, index=[6, 5, 5, 4, 4, 4, 2, 1, 0])
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.loc[:4], pdf.loc[:4])
+        self.assert_eq(kdf.loc[:3], pdf.loc[:3])
+        self.assert_eq(kdf.loc[3:], pdf.loc[3:])
+        self.assert_eq(kdf.loc[2:], pdf.loc[2:])
+        self.assert_eq(kdf.loc[2:3], pdf.loc[2:3])
+        self.assert_eq(kdf.loc[2:-1], pdf.loc[2:-1])
+        self.assert_eq(kdf.loc[10:3], pdf.loc[10:3])
+
+        # test when type of key is string and given value is not included in key
+        pdf = pd.DataFrame({"a": [1, 2, 3]}, index=["a", "b", "d"])
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.loc["a":"z"], pdf.loc["a":"z"])
+
+        # KeyError when index is not monotonic increasing or decreasing
+        # and specified values don't exist in index
+        kdf = ps.DataFrame([[1, 2], [4, 5], [7, 8]], index=["cobra", "viper", "sidewinder"])
+
+        self.assertRaises(KeyError, lambda: kdf.loc["cobra":"koalas"])
+        self.assertRaises(KeyError, lambda: kdf.loc["koalas":"viper"])
+
+        kdf = ps.DataFrame([[1, 2], [4, 5], [7, 8]], index=[10, 30, 20])
+
+        self.assertRaises(KeyError, lambda: kdf.loc[0:30])
+        self.assertRaises(KeyError, lambda: kdf.loc[10:100])
+
+    def test_loc_non_informative_index(self):
+        pdf = pd.DataFrame({"x": [1, 2, 3, 4]}, index=[10, 20, 30, 40])
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.loc[20:30], pdf.loc[20:30])
+
+        pdf = pd.DataFrame({"x": [1, 2, 3, 4]}, index=[10, 20, 20, 40])
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kdf.loc[20:20], pdf.loc[20:20])
+
+    def test_loc_with_series(self):
+        kdf = self.kdf
+        pdf = self.pdf
+
+        self.assert_eq(kdf.loc[kdf.a % 2 == 0], pdf.loc[pdf.a % 2 == 0])
+        self.assert_eq(kdf.loc[kdf.a % 2 == 0, "a"], pdf.loc[pdf.a % 2 == 0, "a"])
+        self.assert_eq(kdf.loc[kdf.a % 2 == 0, ["a"]], pdf.loc[pdf.a % 2 == 0, ["a"]])
+        self.assert_eq(kdf.a.loc[kdf.a % 2 == 0], pdf.a.loc[pdf.a % 2 == 0])
+
+        self.assert_eq(kdf.loc[kdf.copy().a % 2 == 0], pdf.loc[pdf.copy().a % 2 == 0])
+        self.assert_eq(kdf.loc[kdf.copy().a % 2 == 0, "a"], pdf.loc[pdf.copy().a % 2 == 0, "a"])
+        self.assert_eq(kdf.loc[kdf.copy().a % 2 == 0, ["a"]], pdf.loc[pdf.copy().a % 2 == 0, ["a"]])
+        self.assert_eq(kdf.a.loc[kdf.copy().a % 2 == 0], pdf.a.loc[pdf.copy().a % 2 == 0])
+
+    def test_loc_noindex(self):
+        kdf = self.kdf
+        kdf = kdf.reset_index()
+        pdf = self.pdf
+        pdf = pdf.reset_index()
+
+        self.assert_eq(kdf[["a"]], pdf[["a"]])
+
+        self.assert_eq(kdf.loc[:], pdf.loc[:])
+        self.assert_eq(kdf.loc[5:5], pdf.loc[5:5])
+
+    def test_loc_multiindex(self):
+        kdf = self.kdf
+        kdf = kdf.set_index("b", append=True)
+        pdf = self.pdf
+        pdf = pdf.set_index("b", append=True)
+
+        self.assert_eq(kdf.loc[:], pdf.loc[:])
+        self.assert_eq(kdf.loc[5:5], pdf.loc[5:5])
+        self.assert_eq(kdf.loc[5:9], pdf.loc[5:9])
+
+        self.assert_eq(kdf.loc[5], pdf.loc[5])
+        self.assert_eq(kdf.loc[9], pdf.loc[9])
+        # TODO: self.assert_eq(kdf.loc[(5, 3)], pdf.loc[(5, 3)])
+        # TODO: self.assert_eq(kdf.loc[(9, 0)], pdf.loc[(9, 0)])
+        self.assert_eq(kdf.a.loc[5], pdf.a.loc[5])
+        self.assert_eq(kdf.a.loc[9], pdf.a.loc[9])
+        self.assertTrue((kdf.a.loc[(5, 3)] == pdf.a.loc[(5, 3)]).all())
+        self.assert_eq(kdf.a.loc[(9, 0)], pdf.a.loc[(9, 0)])
+
+        # monotonically increasing index test
+        pdf = pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5]},
+            index=pd.MultiIndex.from_tuples(
+                [("x", "a"), ("x", "b"), ("y", "c"), ("y", "d"), ("z", "e")]
+            ),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        for rows_sel in [
+            slice(None),
+            slice("y", None),
+            slice(None, "y"),
+            slice(("x", "b"), None),
+            slice(None, ("y", "c")),
+            slice(("x", "b"), ("y", "c")),
+            slice("x", ("y", "c")),
+            slice(("x", "b"), "y"),
+        ]:
+            with self.subTest("monotonically increasing", rows_sel=rows_sel):
+                self.assert_eq(kdf.loc[rows_sel], pdf.loc[rows_sel])
+                self.assert_eq(kdf.a.loc[rows_sel], pdf.a.loc[rows_sel])
+
+        # monotonically increasing first index test
+        pdf = pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5]},
+            index=pd.MultiIndex.from_tuples(
+                [("x", "a"), ("x", "b"), ("y", "c"), ("y", "a"), ("z", "e")]
+            ),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        for rows_sel in [
+            slice(None),
+            slice("y", None),
+            slice(None, "y"),
+        ]:
+            with self.subTest("monotonically increasing first index", rows_sel=rows_sel):
+                self.assert_eq(kdf.loc[rows_sel], pdf.loc[rows_sel])
+                self.assert_eq(kdf.a.loc[rows_sel], pdf.a.loc[rows_sel])
+
+        for rows_sel in [
+            slice(("x", "b"), None),
+            slice(None, ("y", "c")),
+            slice(("x", "b"), ("y", "c")),
+            slice("x", ("y", "c")),
+            slice(("x", "b"), "y"),
+        ]:
+            with self.subTest("monotonically increasing first index", rows_sel=rows_sel):
+                self.assertRaises(KeyError, lambda: kdf.loc[rows_sel])
+                self.assertRaises(KeyError, lambda: kdf.a.loc[rows_sel])
+
+        # not monotonically increasing index test
+        pdf = pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5]},
+            index=pd.MultiIndex.from_tuples(
+                [("z", "e"), ("y", "d"), ("y", "c"), ("x", "b"), ("x", "a")]
+            ),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        for rows_sel in [
+            slice("y", None),
+            slice(None, "y"),
+            slice(("x", "b"), None),
+            slice(None, ("y", "c")),
+            slice(("x", "b"), ("y", "c")),
+            slice("x", ("y", "c")),
+            slice(("x", "b"), "y"),
+        ]:
+            with self.subTest("monotonically decreasing", rows_sel=rows_sel):
+                self.assertRaises(KeyError, lambda: kdf.loc[rows_sel])
+                self.assertRaises(KeyError, lambda: kdf.a.loc[rows_sel])
+
+    def test_loc2d_multiindex(self):
+        kdf = self.kdf
+        kdf = kdf.set_index("b", append=True)
+        pdf = self.pdf
+        pdf = pdf.set_index("b", append=True)
+
+        self.assert_eq(kdf.loc[:, :], pdf.loc[:, :])
+        self.assert_eq(kdf.loc[:, "a"], pdf.loc[:, "a"])
+        self.assert_eq(kdf.loc[5:5, "a"], pdf.loc[5:5, "a"])
+
+        self.assert_eq(kdf.loc[:, "a":"a"], pdf.loc[:, "a":"a"])
+        self.assert_eq(kdf.loc[:, "a":"c"], pdf.loc[:, "a":"c"])
+        self.assert_eq(kdf.loc[:, "b":"c"], pdf.loc[:, "b":"c"])
+
+    def test_loc2d(self):
+        kdf = self.kdf
+        pdf = self.pdf
+
+        # index indexer is always regarded as slice for duplicated values
+        self.assert_eq(kdf.loc[5:5, "a"], pdf.loc[5:5, "a"])
+        self.assert_eq(kdf.loc[[5], "a"], pdf.loc[[5], "a"])
+        self.assert_eq(kdf.loc[5:5, ["a"]], pdf.loc[5:5, ["a"]])
+        self.assert_eq(kdf.loc[[5], ["a"]], pdf.loc[[5], ["a"]])
+        self.assert_eq(kdf.loc[:, :], pdf.loc[:, :])
+
+        self.assert_eq(kdf.loc[3:8, "a"], pdf.loc[3:8, "a"])
+        self.assert_eq(kdf.loc[:8, "a"], pdf.loc[:8, "a"])
+        self.assert_eq(kdf.loc[3:, "a"], pdf.loc[3:, "a"])
+        self.assert_eq(kdf.loc[[8], "a"], pdf.loc[[8], "a"])
+
+        self.assert_eq(kdf.loc[3:8, ["a"]], pdf.loc[3:8, ["a"]])
+        self.assert_eq(kdf.loc[:8, ["a"]], pdf.loc[:8, ["a"]])
+        self.assert_eq(kdf.loc[3:, ["a"]], pdf.loc[3:, ["a"]])
+        # TODO?: self.assert_eq(kdf.loc[[3, 4, 3], ['a']], pdf.loc[[3, 4, 3], ['a']])
+
+        self.assertRaises(SparkPandasIndexingError, lambda: kdf.loc[3, 3, 3])
+        self.assertRaises(SparkPandasIndexingError, lambda: kdf.a.loc[3, 3])
+        self.assertRaises(SparkPandasIndexingError, lambda: kdf.a.loc[3:, 3])
+        self.assertRaises(SparkPandasIndexingError, lambda: kdf.a.loc[kdf.a % 2 == 0, 3])
+
+        self.assert_eq(kdf.loc[5, "a"], pdf.loc[5, "a"])
+        self.assert_eq(kdf.loc[9, "a"], pdf.loc[9, "a"])
+        self.assert_eq(kdf.loc[5, ["a"]], pdf.loc[5, ["a"]])
+        self.assert_eq(kdf.loc[9, ["a"]], pdf.loc[9, ["a"]])
+
+        self.assert_eq(kdf.loc[:, "a":"a"], pdf.loc[:, "a":"a"])
+        self.assert_eq(kdf.loc[:, "a":"d"], pdf.loc[:, "a":"d"])
+        self.assert_eq(kdf.loc[:, "c":"d"], pdf.loc[:, "c":"d"])
+
+        # bool list-like column select
+        bool_list = [True, False]
+        self.assert_eq(kdf.loc[:, bool_list], pdf.loc[:, bool_list])
+        self.assert_eq(kdf.loc[:, np.array(bool_list)], pdf.loc[:, np.array(bool_list)])
+
+        pser = pd.Series(bool_list, index=pdf.columns)
+        self.assert_eq(kdf.loc[:, pser], pdf.loc[:, pser])
+        pser = pd.Series(list(reversed(bool_list)), index=list(reversed(pdf.columns)))
+        self.assert_eq(kdf.loc[:, pser], pdf.loc[:, pser])
+
+        self.assertRaises(IndexError, lambda: kdf.loc[:, bool_list[:-1]])
+        self.assertRaises(IndexError, lambda: kdf.loc[:, np.array(bool_list + [True])])
+        self.assertRaises(SparkPandasIndexingError, lambda: kdf.loc[:, pd.Series(bool_list)])
+
+        # non-string column names
+        kdf = self.kdf2
+        pdf = self.pdf2
+
+        self.assert_eq(kdf.loc[5:5, 0], pdf.loc[5:5, 0])
+        self.assert_eq(kdf.loc[5:5, [0]], pdf.loc[5:5, [0]])
+        self.assert_eq(kdf.loc[3:8, 0], pdf.loc[3:8, 0])
+        self.assert_eq(kdf.loc[3:8, [0]], pdf.loc[3:8, [0]])
+
+        self.assert_eq(kdf.loc[:, 0:0], pdf.loc[:, 0:0])
+        self.assert_eq(kdf.loc[:, 0:3], pdf.loc[:, 0:3])
+        self.assert_eq(kdf.loc[:, 2:3], pdf.loc[:, 2:3])
+
+    def test_loc2d_multiindex_columns(self):
+        arrays = [np.array(["bar", "bar", "baz", "baz"]), np.array(["one", "two", "one", "two"])]
+
+        pdf = pd.DataFrame(np.random.randn(3, 4), index=["A", "B", "C"], columns=arrays)
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.loc["B":"B", "bar"], pdf.loc["B":"B", "bar"])
+        self.assert_eq(kdf.loc["B":"B", ["bar"]], pdf.loc["B":"B", ["bar"]])
+
+        self.assert_eq(kdf.loc[:, "bar":"bar"], pdf.loc[:, "bar":"bar"])
+        self.assert_eq(kdf.loc[:, "bar":("baz", "one")], pdf.loc[:, "bar":("baz", "one")])
+        self.assert_eq(
+            kdf.loc[:, ("bar", "two"):("baz", "one")], pdf.loc[:, ("bar", "two"):("baz", "one")]
+        )
+        self.assert_eq(kdf.loc[:, ("bar", "two"):"bar"], pdf.loc[:, ("bar", "two"):"bar"])
+        self.assert_eq(kdf.loc[:, "a":"bax"], pdf.loc[:, "a":"bax"])
+        self.assert_eq(
+            kdf.loc[:, ("bar", "x"):("baz", "a")],
+            pdf.loc[:, ("bar", "x"):("baz", "a")],
+            almost=True,
+        )
+
+        pdf = pd.DataFrame(
+            np.random.randn(3, 4),
+            index=["A", "B", "C"],
+            columns=pd.MultiIndex.from_tuples(
+                [("bar", "two"), ("bar", "one"), ("baz", "one"), ("baz", "two")]
+            ),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.loc[:, "bar":"baz"], pdf.loc[:, "bar":"baz"])
+
+        self.assertRaises(KeyError, lambda: kdf.loc[:, "bar":("baz", "one")])
+        self.assertRaises(KeyError, lambda: kdf.loc[:, ("bar", "two"):"bar"])
+
+        # bool list-like column select
+        bool_list = [True, False, True, False]
+        self.assert_eq(kdf.loc[:, bool_list], pdf.loc[:, bool_list])
+        self.assert_eq(kdf.loc[:, np.array(bool_list)], pdf.loc[:, np.array(bool_list)])
+
+        pser = pd.Series(bool_list, index=pdf.columns)
+        self.assert_eq(kdf.loc[:, pser], pdf.loc[:, pser])
+
+        pser = pd.Series(list(reversed(bool_list)), index=list(reversed(pdf.columns)))
+        self.assert_eq(kdf.loc[:, pser], pdf.loc[:, pser])
+
+        # non-string column names
+        arrays = [np.array([0, 0, 1, 1]), np.array([1, 2, 1, 2])]
+
+        pdf = pd.DataFrame(np.random.randn(3, 4), index=["A", "B", "C"], columns=arrays)
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.loc["B":"B", 0], pdf.loc["B":"B", 0])
+        self.assert_eq(kdf.loc["B":"B", [0]], pdf.loc["B":"B", [0]])
+        self.assert_eq(kdf.loc[:, 0:0], pdf.loc[:, 0:0])
+        self.assert_eq(kdf.loc[:, 0:(1, 1)], pdf.loc[:, 0:(1, 1)])
+        self.assert_eq(kdf.loc[:, (0, 2):(1, 1)], pdf.loc[:, (0, 2):(1, 1)])
+        self.assert_eq(kdf.loc[:, (0, 2):0], pdf.loc[:, (0, 2):0])
+        self.assert_eq(kdf.loc[:, -1:2], pdf.loc[:, -1:2])
+
+    def test_loc2d_with_known_divisions(self):
+        pdf = pd.DataFrame(
+            np.random.randn(20, 5), index=list("abcdefghijklmnopqrst"), columns=list("ABCDE")
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.loc[["a"], "A"], pdf.loc[["a"], "A"])
+        self.assert_eq(kdf.loc[["a"], ["A"]], pdf.loc[["a"], ["A"]])
+        self.assert_eq(kdf.loc["a":"o", "A"], pdf.loc["a":"o", "A"])
+        self.assert_eq(kdf.loc["a":"o", ["A"]], pdf.loc["a":"o", ["A"]])
+        self.assert_eq(kdf.loc[["n"], ["A"]], pdf.loc[["n"], ["A"]])
+        self.assert_eq(kdf.loc[["a", "c", "n"], ["A"]], pdf.loc[["a", "c", "n"], ["A"]])
+        # TODO?: self.assert_eq(kdf.loc[['t', 'b'], ['A']], pdf.loc[['t', 'b'], ['A']])
+        # TODO?: self.assert_eq(kdf.loc[['r', 'r', 'c', 'g', 'h'], ['A']],
+        # TODO?:                pdf.loc[['r', 'r', 'c', 'g', 'h'], ['A']])
+
+    @unittest.skip("TODO: should handle duplicated columns properly")
+    def test_loc2d_duplicated_columns(self):
+        pdf = pd.DataFrame(
+            np.random.randn(20, 5), index=list("abcdefghijklmnopqrst"), columns=list("AABCD")
+        )
+        kdf = ps.from_pandas(pdf)
+
+        # TODO?: self.assert_eq(kdf.loc[['a'], 'A'], pdf.loc[['a'], 'A'])
+        # TODO?: self.assert_eq(kdf.loc[['a'], ['A']], pdf.loc[['a'], ['A']])
+        self.assert_eq(kdf.loc[["j"], "B"], pdf.loc[["j"], "B"])
+        self.assert_eq(kdf.loc[["j"], ["B"]], pdf.loc[["j"], ["B"]])
+
+        # TODO?: self.assert_eq(kdf.loc['a':'o', 'A'], pdf.loc['a':'o', 'A'])
+        # TODO?: self.assert_eq(kdf.loc['a':'o', ['A']], pdf.loc['a':'o', ['A']])
+        self.assert_eq(kdf.loc["j":"q", "B"], pdf.loc["j":"q", "B"])
+        self.assert_eq(kdf.loc["j":"q", ["B"]], pdf.loc["j":"q", ["B"]])
+
+        # TODO?: self.assert_eq(kdf.loc['a':'o', 'B':'D'], pdf.loc['a':'o', 'B':'D'])
+        # TODO?: self.assert_eq(kdf.loc['a':'o', 'B':'D'], pdf.loc['a':'o', 'B':'D'])
+        # TODO?: self.assert_eq(kdf.loc['j':'q', 'B':'A'], pdf.loc['j':'q', 'B':'A'])
+        # TODO?: self.assert_eq(kdf.loc['j':'q', 'B':'A'], pdf.loc['j':'q', 'B':'A'])
+
+        self.assert_eq(kdf.loc[kdf.B > 0, "B"], pdf.loc[pdf.B > 0, "B"])
+        # TODO?: self.assert_eq(kdf.loc[kdf.B > 0, ['A', 'C']], pdf.loc[pdf.B > 0, ['A', 'C']])
+
+    def test_getitem(self):
+        pdf = pd.DataFrame(
+            {
+                "A": [1, 2, 3, 4, 5, 6, 7, 8, 9],
+                "B": [9, 8, 7, 6, 5, 4, 3, 2, 1],
+                "C": [True, False, True] * 3,
+            },
+            columns=list("ABC"),
+        )
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kdf["A"], pdf["A"])
+
+        self.assert_eq(kdf[["A", "B"]], pdf[["A", "B"]])
+
+        self.assert_eq(kdf[kdf.C], pdf[pdf.C])
+
+        self.assertRaises(KeyError, lambda: kdf["X"])
+        self.assertRaises(KeyError, lambda: kdf[["A", "X"]])
+        self.assertRaises(AttributeError, lambda: kdf.X)
+
+        # not str/unicode
+        # TODO?: pdf = pd.DataFrame(np.random.randn(10, 5))
+        # TODO?: kdf = ps.from_pandas(pdf)
+        # TODO?: self.assert_eq(kdf[0], pdf[0])
+        # TODO?: self.assert_eq(kdf[[1, 2]], pdf[[1, 2]])
+
+        # TODO?: self.assertRaises(KeyError, lambda: pdf[8])
+        # TODO?: self.assertRaises(KeyError, lambda: pdf[[1, 8]])
+
+        # non-string column names
+        pdf = pd.DataFrame(
+            {
+                10: [1, 2, 3, 4, 5, 6, 7, 8, 9],
+                20: [9, 8, 7, 6, 5, 4, 3, 2, 1],
+                30: [True, False, True] * 3,
+            }
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf[10], pdf[10])
+        self.assert_eq(kdf[[10, 20]], pdf[[10, 20]])
+
+    def test_getitem_slice(self):
+        pdf = pd.DataFrame(
+            {
+                "A": [1, 2, 3, 4, 5, 6, 7, 8, 9],
+                "B": [9, 8, 7, 6, 5, 4, 3, 2, 1],
+                "C": [True, False, True] * 3,
+            },
+            index=list("abcdefghi"),
+        )
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kdf["a":"e"], pdf["a":"e"])
+        self.assert_eq(kdf["a":"b"], pdf["a":"b"])
+        self.assert_eq(kdf["f":], pdf["f":])
+
+    def test_loc_on_numpy_datetimes(self):
+        pdf = pd.DataFrame(
+            {"x": [1, 2, 3]}, index=list(map(np.datetime64, ["2014", "2015", "2016"]))
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.loc["2014":"2015"], pdf.loc["2014":"2015"])
+
+    def test_loc_on_pandas_datetimes(self):
+        pdf = pd.DataFrame(
+            {"x": [1, 2, 3]}, index=list(map(pd.Timestamp, ["2014", "2015", "2016"]))
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.loc["2014":"2015"], pdf.loc["2014":"2015"])
+
+    @unittest.skip("TODO?: the behavior of slice for datetime")
+    def test_loc_datetime_no_freq(self):
+        datetime_index = pd.date_range("2016-01-01", "2016-01-31", freq="12h")
+        datetime_index.freq = None  # FORGET FREQUENCY
+        pdf = pd.DataFrame({"num": range(len(datetime_index))}, index=datetime_index)
+        kdf = ps.from_pandas(pdf)
+
+        slice_ = slice("2016-01-03", "2016-01-05")
+        result = kdf.loc[slice_, :]
+        expected = pdf.loc[slice_, :]
+        self.assert_eq(result, expected)
+
+    @unittest.skip("TODO?: the behavior of slice for datetime")
+    def test_loc_timestamp_str(self):
+        pdf = pd.DataFrame(
+            {"A": np.random.randn(100), "B": np.random.randn(100)},
+            index=pd.date_range("2011-01-01", freq="H", periods=100),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        # partial string slice
+        # TODO?: self.assert_eq(pdf.loc['2011-01-02'],
+        # TODO?:                kdf.loc['2011-01-02'])
+        self.assert_eq(pdf.loc["2011-01-02":"2011-01-05"], kdf.loc["2011-01-02":"2011-01-05"])
+
+        # series
+        # TODO?: self.assert_eq(pdf.A.loc['2011-01-02'],
+        # TODO?:                kdf.A.loc['2011-01-02'])
+        self.assert_eq(pdf.A.loc["2011-01-02":"2011-01-05"], kdf.A.loc["2011-01-02":"2011-01-05"])
+
+        pdf = pd.DataFrame(
+            {"A": np.random.randn(100), "B": np.random.randn(100)},
+            index=pd.date_range("2011-01-01", freq="M", periods=100),
+        )
+        kdf = ps.from_pandas(pdf)
+        # TODO?: self.assert_eq(pdf.loc['2011-01'], kdf.loc['2011-01'])
+        # TODO?: self.assert_eq(pdf.loc['2011'], kdf.loc['2011'])
+
+        self.assert_eq(pdf.loc["2011-01":"2012-05"], kdf.loc["2011-01":"2012-05"])
+        self.assert_eq(pdf.loc["2011":"2015"], kdf.loc["2011":"2015"])
+
+        # series
+        # TODO?: self.assert_eq(pdf.B.loc['2011-01'], kdf.B.loc['2011-01'])
+        # TODO?: self.assert_eq(pdf.B.loc['2011'], kdf.B.loc['2011'])
+
+        self.assert_eq(pdf.B.loc["2011-01":"2012-05"], kdf.B.loc["2011-01":"2012-05"])
+        self.assert_eq(pdf.B.loc["2011":"2015"], kdf.B.loc["2011":"2015"])
+
+    @unittest.skip("TODO?: the behavior of slice for datetime")
+    def test_getitem_timestamp_str(self):
+        pdf = pd.DataFrame(
+            {"A": np.random.randn(100), "B": np.random.randn(100)},
+            index=pd.date_range("2011-01-01", freq="H", periods=100),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        # partial string slice
+        # TODO?: self.assert_eq(pdf['2011-01-02'],
+        # TODO?:                kdf['2011-01-02'])
+        self.assert_eq(pdf["2011-01-02":"2011-01-05"], kdf["2011-01-02":"2011-01-05"])
+
+        pdf = pd.DataFrame(
+            {"A": np.random.randn(100), "B": np.random.randn(100)},
+            index=pd.date_range("2011-01-01", freq="M", periods=100),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        # TODO?: self.assert_eq(pdf['2011-01'], kdf['2011-01'])
+        # TODO?: self.assert_eq(pdf['2011'], kdf['2011'])
+
+        self.assert_eq(pdf["2011-01":"2012-05"], kdf["2011-01":"2012-05"])
+        self.assert_eq(pdf["2011":"2015"], kdf["2011":"2015"])
+
+    @unittest.skip("TODO?: period index can't convert to DataFrame correctly")
+    def test_getitem_period_str(self):
+        pdf = pd.DataFrame(
+            {"A": np.random.randn(100), "B": np.random.randn(100)},
+            index=pd.period_range("2011-01-01", freq="H", periods=100),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        # partial string slice
+        # TODO?: self.assert_eq(pdf['2011-01-02'],
+        # TODO?:                kdf['2011-01-02'])
+        self.assert_eq(pdf["2011-01-02":"2011-01-05"], kdf["2011-01-02":"2011-01-05"])
+
+        pdf = pd.DataFrame(
+            {"A": np.random.randn(100), "B": np.random.randn(100)},
+            index=pd.period_range("2011-01-01", freq="M", periods=100),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        # TODO?: self.assert_eq(pdf['2011-01'], kdf['2011-01'])
+        # TODO?: self.assert_eq(pdf['2011'], kdf['2011'])
+
+        self.assert_eq(pdf["2011-01":"2012-05"], kdf["2011-01":"2012-05"])
+        self.assert_eq(pdf["2011":"2015"], kdf["2011":"2015"])
+
+    def test_iloc(self):
+        pdf = pd.DataFrame({"A": [1, 2], "B": [3, 4], "C": [5, 6]})
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.iloc[0, 0], pdf.iloc[0, 0])
+        for indexer in [0, [0], [0, 1], [1, 0], [False, True, True], slice(0, 1)]:
+            self.assert_eq(kdf.iloc[:, indexer], pdf.iloc[:, indexer])
+            self.assert_eq(kdf.iloc[:1, indexer], pdf.iloc[:1, indexer])
+            self.assert_eq(kdf.iloc[:-1, indexer], pdf.iloc[:-1, indexer])
+            # self.assert_eq(kdf.iloc[kdf.index == 2, indexer], pdf.iloc[pdf.index == 2, indexer])
+
+    def test_iloc_multiindex_columns(self):
+        arrays = [np.array(["bar", "bar", "baz", "baz"]), np.array(["one", "two", "one", "two"])]
+
+        pdf = pd.DataFrame(np.random.randn(3, 4), index=["A", "B", "C"], columns=arrays)
+        kdf = ps.from_pandas(pdf)
+
+        for indexer in [0, [0], [0, 1], [1, 0], [False, True, True, True], slice(0, 1)]:
+            self.assert_eq(kdf.iloc[:, indexer], pdf.iloc[:, indexer])
+            self.assert_eq(kdf.iloc[:1, indexer], pdf.iloc[:1, indexer])
+            self.assert_eq(kdf.iloc[:-1, indexer], pdf.iloc[:-1, indexer])
+            # self.assert_eq(kdf.iloc[kdf.index == "B", indexer],
+            #                pdf.iloc[pdf.index == "B", indexer])
+
+    def test_iloc_series(self):
+        pser = pd.Series([1, 2, 3])
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(kser.iloc[0], pser.iloc[0])
+        self.assert_eq(kser.iloc[:], pser.iloc[:])
+        self.assert_eq(kser.iloc[:1], pser.iloc[:1])
+        self.assert_eq(kser.iloc[:-1], pser.iloc[:-1])
+
+        self.assert_eq((kser + 1).iloc[0], (pser + 1).iloc[0])
+        self.assert_eq((kser + 1).iloc[:], (pser + 1).iloc[:])
+        self.assert_eq((kser + 1).iloc[:1], (pser + 1).iloc[:1])
+        self.assert_eq((kser + 1).iloc[:-1], (pser + 1).iloc[:-1])
+
+    def test_iloc_slice_rows_sel(self):
+        pdf = pd.DataFrame({"A": [1, 2] * 5, "B": [3, 4] * 5, "C": [5, 6] * 5})
+        kdf = ps.from_pandas(pdf)
+
+        for rows_sel in [
+            slice(None),
+            slice(0, 1),
+            slice(1, 2),
+            slice(-3, None),
+            slice(None, -3),
+            slice(None, 0),
+            slice(None, None, 3),
+            slice(3, 8, 2),
+            slice(None, None, -2),
+            slice(8, 3, -2),
+            slice(8, None, -2),
+            slice(None, 3, -2),
+        ]:
+            with self.subTest(rows_sel=rows_sel):
+                self.assert_eq(kdf.iloc[rows_sel].sort_index(), pdf.iloc[rows_sel].sort_index())
+                self.assert_eq(kdf.A.iloc[rows_sel].sort_index(), pdf.A.iloc[rows_sel].sort_index())
+                self.assert_eq(
+                    (kdf.A + 1).iloc[rows_sel].sort_index(), (pdf.A + 1).iloc[rows_sel].sort_index()
+                )
+
+    def test_iloc_iterable_rows_sel(self):
+        pdf = pd.DataFrame({"A": [1, 2] * 5, "B": [3, 4] * 5, "C": [5, 6] * 5})
+        kdf = ps.from_pandas(pdf)
+
+        for rows_sel in [
+            [],
+            np.array([0, 1]),
+            [1, 2],
+            np.array([-3]),
+            [3],
+            np.array([-2]),
+            [8, 3, -5],
+        ]:
+            with self.subTest(rows_sel=rows_sel):
+                self.assert_eq(kdf.iloc[rows_sel].sort_index(), pdf.iloc[rows_sel].sort_index())
+                self.assert_eq(kdf.A.iloc[rows_sel].sort_index(), pdf.A.iloc[rows_sel].sort_index())
+                self.assert_eq(
+                    (kdf.A + 1).iloc[rows_sel].sort_index(), (pdf.A + 1).iloc[rows_sel].sort_index()
+                )
+
+            with self.subTest(rows_sel=rows_sel):
+                self.assert_eq(
+                    kdf.iloc[rows_sel, :].sort_index(), pdf.iloc[rows_sel, :].sort_index()
+                )
+
+            with self.subTest(rows_sel=rows_sel):
+                self.assert_eq(
+                    kdf.iloc[rows_sel, :1].sort_index(), pdf.iloc[rows_sel, :1].sort_index()
+                )
+
+    def test_frame_loc_setitem(self):
+        pdf = pd.DataFrame(
+            [[1, 2], [4, 5], [7, 8]],
+            index=["cobra", "viper", "sidewinder"],
+            columns=["max_speed", "shield"],
+        )
+        kdf = ps.from_pandas(pdf)
+
+        pser1 = pdf.max_speed
+        pser2 = pdf.shield
+        kser1 = kdf.max_speed
+        kser2 = kdf.shield
+
+        pdf.loc[["viper", "sidewinder"], ["shield", "max_speed"]] = 10
+        kdf.loc[["viper", "sidewinder"], ["shield", "max_speed"]] = 10
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kser1, pser1)
+        self.assert_eq(kser2, pser2)
+
+        pdf.loc[["viper", "sidewinder"], "shield"] = 50
+        kdf.loc[["viper", "sidewinder"], "shield"] = 50
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kser1, pser1)
+        self.assert_eq(kser2, pser2)
+
+        pdf.loc["cobra", "max_speed"] = 30
+        kdf.loc["cobra", "max_speed"] = 30
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kser1, pser1)
+        self.assert_eq(kser2, pser2)
+
+        pdf.loc[pdf.max_speed < 5, "max_speed"] = -pdf.max_speed
+        kdf.loc[kdf.max_speed < 5, "max_speed"] = -kdf.max_speed
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kser1, pser1)
+        self.assert_eq(kser2, pser2)
+
+        pdf.loc[pdf.max_speed < 2, "max_speed"] = -pdf.max_speed
+        kdf.loc[kdf.max_speed < 2, "max_speed"] = -kdf.max_speed
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kser1, pser1)
+        self.assert_eq(kser2, pser2)
+
+        pdf.loc[:, "min_speed"] = 0
+        kdf.loc[:, "min_speed"] = 0
+        self.assert_eq(kdf, pdf, almost=True)
+        self.assert_eq(kser1, pser1)
+        self.assert_eq(kser2, pser2)
+
+        with self.assertRaisesRegex(ValueError, "Incompatible indexer with Series"):
+            kdf.loc["cobra", "max_speed"] = -kdf.max_speed
+        with self.assertRaisesRegex(ValueError, "shape mismatch"):
+            kdf.loc[:, ["shield", "max_speed"]] = -kdf.max_speed
+        with self.assertRaisesRegex(ValueError, "Only a dataframe with one column can be assigned"):
+            kdf.loc[:, "max_speed"] = kdf
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples(
+            [("x", "max_speed"), ("x", "shield"), ("y", "min_speed")]
+        )
+        pdf.columns = columns
+        kdf.columns = columns
+
+        pdf.loc[:, ("y", "shield")] = -pdf[("x", "shield")]
+        kdf.loc[:, ("y", "shield")] = -kdf[("x", "shield")]
+        self.assert_eq(kdf, pdf, almost=True)
+        self.assert_eq(kser1, pser1)
+        self.assert_eq(kser2, pser2)
+
+        pdf.loc[:, "z"] = 100
+        kdf.loc[:, "z"] = 100
+        self.assert_eq(kdf, pdf, almost=True)
+        self.assert_eq(kser1, pser1)
+        self.assert_eq(kser2, pser2)
+
+        with self.assertRaisesRegex(KeyError, "Key length \\(3\\) exceeds index depth \\(2\\)"):
+            kdf.loc[:, [("x", "max_speed", "foo")]] = -kdf[("x", "shield")]
+
+        pdf = pd.DataFrame(
+            [[1], [4], [7]], index=["cobra", "viper", "sidewinder"], columns=["max_speed"]
+        )
+        kdf = ps.from_pandas(pdf)
+
+        pdf.loc[:, "max_speed"] = pdf
+        kdf.loc[:, "max_speed"] = kdf
+        self.assert_eq(kdf, pdf)
+
+    def test_frame_iloc_setitem(self):
+        pdf = pd.DataFrame(
+            [[1, 2], [4, 5], [7, 8]],
+            index=["cobra", "viper", "sidewinder"],
+            columns=["max_speed", "shield"],
+        )
+        kdf = ps.from_pandas(pdf)
+
+        pdf.iloc[[1, 2], [1, 0]] = 10
+        kdf.iloc[[1, 2], [1, 0]] = 10
+        self.assert_eq(kdf, pdf)
+
+        pdf.iloc[0, 1] = 50
+        kdf.iloc[0, 1] = 50
+        self.assert_eq(kdf, pdf)
+
+        with self.assertRaisesRegex(ValueError, "setting an array element with a sequence."):
+            kdf.iloc[0, 0] = -kdf.max_speed
+        with self.assertRaisesRegex(ValueError, "shape mismatch"):
+            kdf.iloc[:, [1, 0]] = -kdf.max_speed
+        with self.assertRaisesRegex(ValueError, "Only a dataframe with one column can be assigned"):
+            kdf.iloc[:, 0] = kdf
+
+        pdf = pd.DataFrame(
+            [[1], [4], [7]], index=["cobra", "viper", "sidewinder"], columns=["max_speed"]
+        )
+        kdf = ps.from_pandas(pdf)
+
+        pdf.iloc[:, 0] = pdf
+        kdf.iloc[:, 0] = kdf
+        self.assert_eq(kdf, pdf)
+
+    def test_series_loc_setitem(self):
+        pdf = pd.DataFrame({"x": [1, 2, 3], "y": [4, 5, 6]}, index=["cobra", "viper", "sidewinder"])
+        kdf = ps.from_pandas(pdf)
+
+        pser = pdf.x
+        psery = pdf.y
+        kser = kdf.x
+        ksery = kdf.y
+
+        pser.loc[pser % 2 == 1] = -pser
+        kser.loc[kser % 2 == 1] = -kser
+        self.assert_eq(kser, pser)
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(ksery, psery)
+
+        for key, value in [
+            (["viper", "sidewinder"], 10),
+            ("viper", 50),
+            (slice(None), 10),
+            (slice(None, "viper"), 20),
+            (slice("viper", None), 30),
+        ]:
+            with self.subTest(key=key, value=value):
+                pser.loc[key] = value
+                kser.loc[key] = value
+                self.assert_eq(kser, pser)
+                self.assert_eq(kdf, pdf)
+                self.assert_eq(ksery, psery)
+
+        with self.assertRaises(ValueError):
+            kser.loc["viper"] = -kser
+
+        # multiindex
+        pser = pd.Series(
+            [1, 2, 3],
+            index=pd.MultiIndex.from_tuples([("x", "cobra"), ("x", "viper"), ("y", "sidewinder")]),
+        )
+        kser = ps.from_pandas(pser)
+
+        pser.loc["x"] = pser * 10
+        kser.loc["x"] = kser * 10
+        self.assert_eq(kser, pser)
+
+        pser.loc["y"] = pser * 10
+        kser.loc["y"] = kser * 10
+        self.assert_eq(kser, pser)
+
+        if LooseVersion(pd.__version__) < LooseVersion("1.0"):
+            # TODO: seems like a pandas' bug in pandas>=1.0.0?
+            pser.loc[("x", "viper"):"y"] = pser * 20
+            kser.loc[("x", "viper"):"y"] = kser * 20
+            self.assert_eq(kser, pser)
+
+    def test_series_iloc_setitem(self):
+        pdf = pd.DataFrame({"x": [1, 2, 3], "y": [4, 5, 6]}, index=["cobra", "viper", "sidewinder"])
+        kdf = ps.from_pandas(pdf)
+
+        pser = pdf.x
+        psery = pdf.y
+        kser = kdf.x
+        ksery = kdf.y
+
+        piloc = pser.iloc
+        kiloc = kser.iloc
+
+        pser1 = pser + 1
+        kser1 = kser + 1
+
+        for key, value in [
+            ([1, 2], 10),
+            (1, 50),
+            (slice(None), 10),
+            (slice(None, 1), 20),
+            (slice(1, None), 30),
+        ]:
+            with self.subTest(key=key, value=value):
+                pser.iloc[key] = value
+                kser.iloc[key] = value
+                self.assert_eq(kser, pser)
+                self.assert_eq(kdf, pdf)
+                self.assert_eq(ksery, psery)
+
+                piloc[key] = -value
+                kiloc[key] = -value
+                self.assert_eq(kser, pser)
+                self.assert_eq(kdf, pdf)
+                self.assert_eq(ksery, psery)
+
+                pser1.iloc[key] = value
+                kser1.iloc[key] = value
+                self.assert_eq(kser1, pser1)
+                self.assert_eq(kdf, pdf)
+                self.assert_eq(ksery, psery)
+
+        with self.assertRaises(ValueError):
+            kser.iloc[1] = -kser
+
+        pser = pd.Index([1, 2, 3]).to_series()
+        kser = ps.Index([1, 2, 3]).to_series()
+
+        pser1 = pser + 1
+        kser1 = kser + 1
+
+        pser.iloc[0] = 10
+        kser.iloc[0] = 10
+        self.assert_eq(kser, pser)
+
+        pser1.iloc[0] = 20
+        kser1.iloc[0] = 20
+        self.assert_eq(kser1, pser1)
+
+        pdf = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
+        kdf = ps.from_pandas(pdf)
+
+        pser = pdf.a
+        kser = kdf.a
+
+        pser.iloc[[0, 1, 2]] = -pdf.b
+        kser.iloc[[0, 1, 2]] = -kdf.b
+        self.assert_eq(kser, pser)
+        self.assert_eq(kdf, pdf)
+
+        with self.assertRaisesRegex(ValueError, "setting an array element with a sequence."):
+            kser.iloc[1] = kdf[["b"]]
+
+    def test_iloc_raises(self):
+        pdf = pd.DataFrame({"A": [1, 2], "B": [3, 4], "C": [5, 6]})
+        kdf = ps.from_pandas(pdf)
+
+        with self.assertRaisesRegex(SparkPandasIndexingError, "Only accepts pairs of candidates"):
+            kdf.iloc[[0, 1], [0, 1], [1, 2]]
+
+        with self.assertRaisesRegex(SparkPandasIndexingError, "Too many indexers"):
+            kdf.A.iloc[[0, 1], [0, 1]]
+
+        with self.assertRaisesRegex(TypeError, "cannot do slice indexing with these indexers"):
+            kdf.iloc[:"b", :]
+
+        with self.assertRaisesRegex(TypeError, "cannot do slice indexing with these indexers"):
+            kdf.iloc[:, :"b"]
+
+        with self.assertRaisesRegex(TypeError, "cannot perform reduce with flexible type"):
+            kdf.iloc[:, ["A"]]
+
+        with self.assertRaisesRegex(ValueError, "Location based indexing can only have"):
+            kdf.iloc[:, "A"]
+
+        with self.assertRaisesRegex(IndexError, "out of range"):
+            kdf.iloc[:, [5, 6]]
+
+    def test_index_operator_datetime(self):
+        dates = pd.date_range("20130101", periods=6)
+        pdf = pd.DataFrame(np.random.randn(6, 4), index=dates, columns=list("ABCD"))
+        kdf = ps.from_pandas(pdf)
+
+        # Positional iloc search
+        self.assert_eq(kdf[:4], pdf[:4], almost=True)
+        self.assert_eq(kdf[:3], pdf[:3], almost=True)
+        self.assert_eq(kdf[3:], pdf[3:], almost=True)
+        self.assert_eq(kdf[2:], pdf[2:], almost=True)
+        self.assert_eq(kdf[2:3], pdf[2:3], almost=True)
+        self.assert_eq(kdf[2:-1], pdf[2:-1], almost=True)
+        self.assert_eq(kdf[10:3], pdf[10:3], almost=True)
+
+        # Index loc search
+        self.assert_eq(kdf.A[4], pdf.A[4])
+        self.assert_eq(kdf.A[3], pdf.A[3])
+
+        # Positional iloc search
+        self.assert_eq(kdf.A[:4], pdf.A[:4], almost=True)
+        self.assert_eq(kdf.A[:3], pdf.A[:3], almost=True)
+        self.assert_eq(kdf.A[3:], pdf.A[3:], almost=True)
+        self.assert_eq(kdf.A[2:], pdf.A[2:], almost=True)
+        self.assert_eq(kdf.A[2:3], pdf.A[2:3], almost=True)
+        self.assert_eq(kdf.A[2:-1], pdf.A[2:-1], almost=True)
+        self.assert_eq(kdf.A[10:3], pdf.A[10:3], almost=True)
+
+        dt1 = datetime.datetime.strptime("2013-01-02", "%Y-%m-%d")
+        dt2 = datetime.datetime.strptime("2013-01-04", "%Y-%m-%d")
+
+        # Index loc search
+        self.assert_eq(kdf[:dt2], pdf[:dt2], almost=True)
+        self.assert_eq(kdf[dt1:], pdf[dt1:], almost=True)
+        self.assert_eq(kdf[dt1:dt2], pdf[dt1:dt2], almost=True)
+        self.assert_eq(kdf.A[dt2], pdf.A[dt2], almost=True)
+        self.assert_eq(kdf.A[:dt2], pdf.A[:dt2], almost=True)
+        self.assert_eq(kdf.A[dt1:], pdf.A[dt1:], almost=True)
+        self.assert_eq(kdf.A[dt1:dt2], pdf.A[dt1:dt2], almost=True)
+
+    def test_index_operator_int(self):
+        pdf = pd.DataFrame(np.random.randn(6, 4), index=[1, 3, 5, 7, 9, 11], columns=list("ABCD"))
+        kdf = ps.from_pandas(pdf)
+
+        # Positional iloc search
+        self.assert_eq(kdf[:4], pdf[:4])
+        self.assert_eq(kdf[:3], pdf[:3])
+        self.assert_eq(kdf[3:], pdf[3:])
+        self.assert_eq(kdf[2:], pdf[2:])
+        self.assert_eq(kdf[2:3], pdf[2:3])
+        self.assert_eq(kdf[2:-1], pdf[2:-1])
+        self.assert_eq(kdf[10:3], pdf[10:3])
+
+        # Index loc search
+        self.assert_eq(kdf.A[5], pdf.A[5])
+        self.assert_eq(kdf.A[3], pdf.A[3])
+        with self.assertRaisesRegex(
+            NotImplementedError, "Duplicated row selection is not currently supported"
+        ):
+            kdf.iloc[[1, 1]]
+
+
+if __name__ == "__main__":
+    from pyspark.pandas.tests.test_indexing import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_indexops_spark.py b/python/pyspark/pandas/tests/test_indexops_spark.py
new file mode 100644
index 0000000000000..831b764271753
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_indexops_spark.py
@@ -0,0 +1,75 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import pandas as pd
+from pyspark.sql.utils import AnalysisException
+from pyspark.sql import functions as F
+
+from pyspark import pandas as ps
+from pyspark.testing.pandasutils import PandasOnSparkTestCase
+from pyspark.testing.sqlutils import SQLTestUtils
+
+
+class SparkIndexOpsMethodsTest(PandasOnSparkTestCase, SQLTestUtils):
+    @property
+    def pser(self):
+        return pd.Series([1, 2, 3, 4, 5, 6, 7], name="x")
+
+    @property
+    def kser(self):
+        return ps.from_pandas(self.pser)
+
+    def test_series_transform_negative(self):
+        with self.assertRaisesRegex(
+            ValueError, "The output of the function.* pyspark.sql.Column.*int"
+        ):
+            self.kser.spark.transform(lambda scol: 1)
+
+        with self.assertRaisesRegex(AnalysisException, "cannot resolve.*non-existent.*"):
+            self.kser.spark.transform(lambda scol: F.col("non-existent"))
+
+    def test_multiindex_transform_negative(self):
+        with self.assertRaisesRegex(
+            NotImplementedError, "MultiIndex does not support spark.transform yet"
+        ):
+            midx = pd.MultiIndex(
+                [["lama", "cow", "falcon"], ["speed", "weight", "length"]],
+                [[0, 0, 0, 1, 1, 1, 2, 2, 2], [1, 1, 1, 1, 1, 2, 1, 2, 2]],
+            )
+            s = ps.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], index=midx)
+            s.index.spark.transform(lambda scol: scol)
+
+    def test_series_apply_negative(self):
+        with self.assertRaisesRegex(
+            ValueError, "The output of the function.* pyspark.sql.Column.*int"
+        ):
+            self.kser.spark.apply(lambda scol: 1)
+
+        with self.assertRaisesRegex(AnalysisException, "cannot resolve.*non-existent.*"):
+            self.kser.spark.transform(lambda scol: F.col("non-existent"))
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.pandas.tests.test_indexops_spark import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_internal.py b/python/pyspark/pandas/tests/test_internal.py
new file mode 100644
index 0000000000000..f9e96cd995cdc
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_internal.py
@@ -0,0 +1,104 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import pandas as pd
+
+from pyspark.pandas.internal import (
+    InternalFrame,
+    SPARK_DEFAULT_INDEX_NAME,
+    SPARK_INDEX_NAME_FORMAT,
+)
+from pyspark.testing.pandasutils import PandasOnSparkTestCase
+from pyspark.testing.sqlutils import SQLTestUtils
+
+
+class InternalFrameTest(PandasOnSparkTestCase, SQLTestUtils):
+    def test_from_pandas(self):
+        pdf = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
+
+        internal = InternalFrame.from_pandas(pdf)
+        sdf = internal.spark_frame
+
+        self.assert_eq(internal.index_spark_column_names, [SPARK_DEFAULT_INDEX_NAME])
+        self.assert_eq(internal.index_names, [None])
+        self.assert_eq(internal.column_labels, [("a",), ("b",)])
+        self.assert_eq(internal.data_spark_column_names, ["a", "b"])
+        self.assertTrue(internal.spark_column_for(("a",))._jc.equals(sdf["a"]._jc))
+        self.assertTrue(internal.spark_column_for(("b",))._jc.equals(sdf["b"]._jc))
+
+        self.assert_eq(internal.to_pandas_frame, pdf)
+
+        # non-string column name
+        pdf1 = pd.DataFrame({0: [1, 2, 3], 1: [4, 5, 6]})
+
+        internal = InternalFrame.from_pandas(pdf1)
+        sdf = internal.spark_frame
+
+        self.assert_eq(internal.index_spark_column_names, [SPARK_DEFAULT_INDEX_NAME])
+        self.assert_eq(internal.index_names, [None])
+        self.assert_eq(internal.column_labels, [(0,), (1,)])
+        self.assert_eq(internal.data_spark_column_names, ["0", "1"])
+        self.assertTrue(internal.spark_column_for((0,))._jc.equals(sdf["0"]._jc))
+        self.assertTrue(internal.spark_column_for((1,))._jc.equals(sdf["1"]._jc))
+
+        self.assert_eq(internal.to_pandas_frame, pdf1)
+
+        # multi-index
+        pdf.set_index("a", append=True, inplace=True)
+
+        internal = InternalFrame.from_pandas(pdf)
+        sdf = internal.spark_frame
+
+        self.assert_eq(
+            internal.index_spark_column_names,
+            [SPARK_INDEX_NAME_FORMAT(0), SPARK_INDEX_NAME_FORMAT(1)],
+        )
+        self.assert_eq(internal.index_names, [None, ("a",)])
+        self.assert_eq(internal.column_labels, [("b",)])
+        self.assert_eq(internal.data_spark_column_names, ["b"])
+        self.assertTrue(internal.spark_column_for(("b",))._jc.equals(sdf["b"]._jc))
+
+        self.assert_eq(internal.to_pandas_frame, pdf)
+
+        # multi-index columns
+        pdf.columns = pd.MultiIndex.from_tuples([("x", "b")])
+
+        internal = InternalFrame.from_pandas(pdf)
+        sdf = internal.spark_frame
+
+        self.assert_eq(
+            internal.index_spark_column_names,
+            [SPARK_INDEX_NAME_FORMAT(0), SPARK_INDEX_NAME_FORMAT(1)],
+        )
+        self.assert_eq(internal.index_names, [None, ("a",)])
+        self.assert_eq(internal.column_labels, [("x", "b")])
+        self.assert_eq(internal.data_spark_column_names, ["(x, b)"])
+        self.assertTrue(internal.spark_column_for(("x", "b"))._jc.equals(sdf["(x, b)"]._jc))
+
+        self.assert_eq(internal.to_pandas_frame, pdf)
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.pandas.tests.test_internal import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_namespace.py b/python/pyspark/pandas/tests/test_namespace.py
new file mode 100644
index 0000000000000..c45c65316b5d5
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_namespace.py
@@ -0,0 +1,338 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import itertools
+
+import pandas as pd
+
+from pyspark import pandas as ps
+from pyspark.pandas.namespace import _get_index_map
+from pyspark.testing.pandasutils import PandasOnSparkTestCase
+from pyspark.testing.sqlutils import SQLTestUtils
+
+
+class NamespaceTest(PandasOnSparkTestCase, SQLTestUtils):
+    def test_from_pandas(self):
+        pdf = pd.DataFrame({"year": [2015, 2016], "month": [2, 3], "day": [4, 5]})
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf, pdf)
+
+        pser = pdf.year
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(kser, pser)
+
+        pidx = pdf.index
+        kidx = ps.from_pandas(pidx)
+
+        self.assert_eq(kidx, pidx)
+
+        pmidx = pdf.set_index("year", append=True).index
+        kmidx = ps.from_pandas(pmidx)
+
+        self.assert_eq(kmidx, pmidx)
+
+        expected_error_message = "Unknown data type: {}".format(type(kidx).__name__)
+        with self.assertRaisesRegex(TypeError, expected_error_message):
+            ps.from_pandas(kidx)
+
+    def test_to_datetime(self):
+        pdf = pd.DataFrame({"year": [2015, 2016], "month": [2, 3], "day": [4, 5]})
+        kdf = ps.from_pandas(pdf)
+        dict_from_pdf = pdf.to_dict()
+
+        self.assert_eq(pd.to_datetime(pdf), ps.to_datetime(kdf))
+        self.assert_eq(pd.to_datetime(dict_from_pdf), ps.to_datetime(dict_from_pdf))
+
+        self.assert_eq(pd.to_datetime(1490195805, unit="s"), ps.to_datetime(1490195805, unit="s"))
+        self.assert_eq(
+            pd.to_datetime(1490195805433502912, unit="ns"),
+            ps.to_datetime(1490195805433502912, unit="ns"),
+        )
+
+        self.assert_eq(
+            pd.to_datetime([1, 2, 3], unit="D", origin=pd.Timestamp("1960-01-01")),
+            ps.to_datetime([1, 2, 3], unit="D", origin=pd.Timestamp("1960-01-01")),
+        )
+
+    def test_date_range(self):
+        self.assert_eq(
+            ps.date_range(start="1/1/2018", end="1/08/2018"),
+            pd.date_range(start="1/1/2018", end="1/08/2018"),
+        )
+        self.assert_eq(
+            ps.date_range(start="1/1/2018", periods=8), pd.date_range(start="1/1/2018", periods=8)
+        )
+        self.assert_eq(
+            ps.date_range(end="1/1/2018", periods=8), pd.date_range(end="1/1/2018", periods=8)
+        )
+        self.assert_eq(
+            ps.date_range(start="2018-04-24", end="2018-04-27", periods=3),
+            pd.date_range(start="2018-04-24", end="2018-04-27", periods=3),
+        )
+
+        self.assert_eq(
+            ps.date_range(start="1/1/2018", periods=5, freq="M"),
+            pd.date_range(start="1/1/2018", periods=5, freq="M"),
+        )
+
+        self.assert_eq(
+            ps.date_range(start="1/1/2018", periods=5, freq="3M"),
+            pd.date_range(start="1/1/2018", periods=5, freq="3M"),
+        )
+
+        self.assert_eq(
+            ps.date_range(start="1/1/2018", periods=5, freq=pd.offsets.MonthEnd(3)),
+            pd.date_range(start="1/1/2018", periods=5, freq=pd.offsets.MonthEnd(3)),
+        )
+
+        self.assert_eq(
+            ps.date_range(start="2017-01-01", end="2017-01-04", closed="left"),
+            pd.date_range(start="2017-01-01", end="2017-01-04", closed="left"),
+        )
+
+        self.assert_eq(
+            ps.date_range(start="2017-01-01", end="2017-01-04", closed="right"),
+            pd.date_range(start="2017-01-01", end="2017-01-04", closed="right"),
+        )
+
+        self.assertRaises(
+            AssertionError, lambda: ps.date_range(start="1/1/2018", periods=5, tz="Asia/Tokyo")
+        )
+        self.assertRaises(
+            AssertionError, lambda: ps.date_range(start="1/1/2018", periods=5, freq="ns")
+        )
+        self.assertRaises(
+            AssertionError, lambda: ps.date_range(start="1/1/2018", periods=5, freq="N")
+        )
+
+    def test_concat_index_axis(self):
+        pdf = pd.DataFrame({"A": [0, 2, 4], "B": [1, 3, 5], "C": [6, 7, 8]})
+        # TODO: pdf.columns.names = ["ABC"]
+        kdf = ps.from_pandas(pdf)
+
+        ignore_indexes = [True, False]
+        joins = ["inner", "outer"]
+        sorts = [True, False]
+
+        objs = [
+            ([kdf, kdf], [pdf, pdf]),
+            ([kdf, kdf.reset_index()], [pdf, pdf.reset_index()]),
+            ([kdf.reset_index(), kdf], [pdf.reset_index(), pdf]),
+            ([kdf, kdf[["C", "A"]]], [pdf, pdf[["C", "A"]]]),
+            ([kdf[["C", "A"]], kdf], [pdf[["C", "A"]], pdf]),
+            ([kdf, kdf["C"]], [pdf, pdf["C"]]),
+            ([kdf["C"], kdf], [pdf["C"], pdf]),
+            ([kdf["C"], kdf, kdf["A"]], [pdf["C"], pdf, pdf["A"]]),
+            ([kdf, kdf["C"], kdf["A"]], [pdf, pdf["C"], pdf["A"]]),
+        ]
+
+        for ignore_index, join, sort in itertools.product(ignore_indexes, joins, sorts):
+            for i, (kdfs, pdfs) in enumerate(objs):
+                with self.subTest(
+                    ignore_index=ignore_index, join=join, sort=sort, pdfs=pdfs, pair=i
+                ):
+                    self.assert_eq(
+                        ps.concat(kdfs, ignore_index=ignore_index, join=join, sort=sort),
+                        pd.concat(pdfs, ignore_index=ignore_index, join=join, sort=sort),
+                        almost=(join == "outer"),
+                    )
+
+        self.assertRaisesRegex(TypeError, "first argument must be", lambda: ps.concat(kdf))
+        self.assertRaisesRegex(TypeError, "cannot concatenate object", lambda: ps.concat([kdf, 1]))
+
+        kdf2 = kdf.set_index("B", append=True)
+        self.assertRaisesRegex(
+            ValueError, "Index type and names should be same", lambda: ps.concat([kdf, kdf2])
+        )
+
+        self.assertRaisesRegex(ValueError, "No objects to concatenate", lambda: ps.concat([]))
+
+        self.assertRaisesRegex(ValueError, "All objects passed", lambda: ps.concat([None, None]))
+
+        pdf3 = pdf.copy()
+        kdf3 = kdf.copy()
+
+        columns = pd.MultiIndex.from_tuples([("X", "A"), ("X", "B"), ("Y", "C")])
+        # TODO: colums.names = ["XYZ", "ABC"]
+        pdf3.columns = columns
+        kdf3.columns = columns
+
+        objs = [
+            ([kdf3, kdf3], [pdf3, pdf3]),
+            ([kdf3, kdf3.reset_index()], [pdf3, pdf3.reset_index()]),
+            ([kdf3.reset_index(), kdf3], [pdf3.reset_index(), pdf3]),
+            ([kdf3, kdf3[[("Y", "C"), ("X", "A")]]], [pdf3, pdf3[[("Y", "C"), ("X", "A")]]]),
+            ([kdf3[[("Y", "C"), ("X", "A")]], kdf3], [pdf3[[("Y", "C"), ("X", "A")]], pdf3]),
+        ]
+
+        for ignore_index, sort in itertools.product(ignore_indexes, sorts):
+            for i, (kdfs, pdfs) in enumerate(objs):
+                with self.subTest(
+                    ignore_index=ignore_index, join="outer", sort=sort, pdfs=pdfs, pair=i
+                ):
+                    self.assert_eq(
+                        ps.concat(kdfs, ignore_index=ignore_index, join="outer", sort=sort),
+                        pd.concat(pdfs, ignore_index=ignore_index, join="outer", sort=sort),
+                    )
+
+        # Skip tests for `join="inner" and sort=False` since pandas is flaky.
+        for ignore_index in ignore_indexes:
+            for i, (kdfs, pdfs) in enumerate(objs):
+                with self.subTest(
+                    ignore_index=ignore_index, join="inner", sort=True, pdfs=pdfs, pair=i
+                ):
+                    self.assert_eq(
+                        ps.concat(kdfs, ignore_index=ignore_index, join="inner", sort=True),
+                        pd.concat(pdfs, ignore_index=ignore_index, join="inner", sort=True),
+                    )
+
+        self.assertRaisesRegex(
+            ValueError,
+            "MultiIndex columns should have the same levels",
+            lambda: ps.concat([kdf, kdf3]),
+        )
+        self.assertRaisesRegex(
+            ValueError,
+            "MultiIndex columns should have the same levels",
+            lambda: ps.concat([kdf3[("Y", "C")], kdf3]),
+        )
+
+        pdf4 = pd.DataFrame({"A": [0, 2, 4], "B": [1, 3, 5], "C": [10, 20, 30]})
+        kdf4 = ps.from_pandas(pdf4)
+        self.assertRaisesRegex(
+            ValueError,
+            r"Only can inner \(intersect\) or outer \(union\) join the other axis.",
+            lambda: ps.concat([kdf, kdf4], join=""),
+        )
+
+        self.assertRaisesRegex(
+            ValueError,
+            r"Only can inner \(intersect\) or outer \(union\) join the other axis.",
+            lambda: ps.concat([kdf, kdf4], join="", axis=1),
+        )
+
+        self.assertRaisesRegex(
+            ValueError,
+            r"Only can inner \(intersect\) or outer \(union\) join the other axis.",
+            lambda: ps.concat([kdf.A, kdf4.B], join="", axis=1),
+        )
+
+        self.assertRaisesRegex(
+            ValueError,
+            r"Labels have to be unique; however, got duplicated labels \['A'\].",
+            lambda: ps.concat([kdf.A, kdf4.A], join="inner", axis=1),
+        )
+
+    def test_concat_column_axis(self):
+        pdf1 = pd.DataFrame({"A": [0, 2, 4], "B": [1, 3, 5]}, index=[1, 2, 3])
+        pdf1.columns.names = ["AB"]
+        pdf2 = pd.DataFrame({"C": [1, 2, 3], "D": [4, 5, 6]}, index=[1, 3, 5])
+        pdf2.columns.names = ["CD"]
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        kdf3 = kdf1.copy()
+        kdf4 = kdf2.copy()
+        pdf3 = pdf1.copy()
+        pdf4 = pdf2.copy()
+
+        columns = pd.MultiIndex.from_tuples([("X", "A"), ("X", "B")], names=["X", "AB"])
+        pdf3.columns = columns
+        kdf3.columns = columns
+
+        columns = pd.MultiIndex.from_tuples([("X", "C"), ("X", "D")], names=["Y", "CD"])
+        pdf4.columns = columns
+        kdf4.columns = columns
+
+        ignore_indexes = [True, False]
+        joins = ["inner", "outer"]
+
+        objs = [
+            ([kdf1.A, kdf1.A.rename("B")], [pdf1.A, pdf1.A.rename("B")]),
+            ([kdf3[("X", "A")], kdf3[("X", "B")]], [pdf3[("X", "A")], pdf3[("X", "B")]],),
+            (
+                [kdf3[("X", "A")], kdf3[("X", "B")].rename("ABC")],
+                [pdf3[("X", "A")], pdf3[("X", "B")].rename("ABC")],
+            ),
+            (
+                [kdf3[("X", "A")].rename("ABC"), kdf3[("X", "B")]],
+                [pdf3[("X", "A")].rename("ABC"), pdf3[("X", "B")]],
+            ),
+        ]
+
+        for ignore_index, join in itertools.product(ignore_indexes, joins):
+            for i, (kdfs, pdfs) in enumerate(objs):
+                with self.subTest(ignore_index=ignore_index, join=join, pdfs=pdfs, pair=i):
+                    actual = ps.concat(kdfs, axis=1, ignore_index=ignore_index, join=join)
+                    expected = pd.concat(pdfs, axis=1, ignore_index=ignore_index, join=join)
+                    self.assert_eq(
+                        repr(actual.sort_values(list(actual.columns)).reset_index(drop=True)),
+                        repr(expected.sort_values(list(expected.columns)).reset_index(drop=True)),
+                    )
+
+    # test dataframes equality with broadcast hint.
+    def test_broadcast(self):
+        kdf = ps.DataFrame(
+            {"key": ["K0", "K1", "K2", "K3"], "A": ["A0", "A1", "A2", "A3"]}, columns=["key", "A"]
+        )
+        self.assert_eq(kdf, ps.broadcast(kdf))
+
+        kdf.columns = ["x", "y"]
+        self.assert_eq(kdf, ps.broadcast(kdf))
+
+        kdf.columns = [("a", "c"), ("b", "d")]
+        self.assert_eq(kdf, ps.broadcast(kdf))
+
+        kser = ps.Series([1, 2, 3])
+        expected_error_message = "Invalid type : expected DataFrame got {}".format(
+            type(kser).__name__
+        )
+        with self.assertRaisesRegex(TypeError, expected_error_message):
+            ps.broadcast(kser)
+
+    def test_get_index_map(self):
+        kdf = ps.DataFrame({"year": [2015, 2016], "month": [2, 3], "day": [4, 5]})
+        sdf = kdf.to_spark()
+        self.assertEqual(_get_index_map(sdf), (None, None))
+
+        def check(actual, expected):
+            actual_scols, actual_labels = actual
+            expected_column_names, expected_labels = expected
+            self.assertEqual(len(actual_scols), len(expected_column_names))
+            for actual_scol, expected_column_name in zip(actual_scols, expected_column_names):
+                expected_scol = sdf[expected_column_name]
+                self.assertTrue(actual_scol._jc.equals(expected_scol._jc))
+            self.assertEqual(actual_labels, expected_labels)
+
+        check(_get_index_map(sdf, "year"), (["year"], [("year",)]))
+        check(_get_index_map(sdf, ["year", "month"]), (["year", "month"], [("year",), ("month",)]))
+
+        self.assertRaises(KeyError, lambda: _get_index_map(sdf, ["year", "hour"]))
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.pandas.tests.test_namespace import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_numpy_compat.py b/python/pyspark/pandas/tests/test_numpy_compat.py
new file mode 100644
index 0000000000000..ce2bbe1702130
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_numpy_compat.py
@@ -0,0 +1,211 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from distutils.version import LooseVersion
+
+import numpy as np
+import pandas as pd
+
+from pyspark import pandas as ps
+from pyspark.pandas import set_option, reset_option
+from pyspark.pandas.numpy_compat import unary_np_spark_mappings, binary_np_spark_mappings
+from pyspark.testing.pandasutils import PandasOnSparkTestCase
+from pyspark.testing.sqlutils import SQLTestUtils
+
+
+class NumPyCompatTest(PandasOnSparkTestCase, SQLTestUtils):
+    blacklist = [
+        # Koalas does not currently support
+        "conj",
+        "conjugate",
+        "isnat",
+        "matmul",
+        "frexp",
+        # Values are close enough but tests failed.
+        "arccos",
+        "exp",
+        "expm1",
+        "log",  # flaky
+        "log10",  # flaky
+        "log1p",  # flaky
+        "modf",
+        "floor_divide",  # flaky
+        # Results seem inconsistent in a different version of, I (Hyukjin) suspect, PyArrow.
+        # From PyArrow 0.15, seems it returns the correct results via PySpark. Probably we
+        # can enable it later when Koalas switches to PyArrow 0.15 completely.
+        "left_shift",
+    ]
+
+    @property
+    def pdf(self):
+        return pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5, 6, 7, 8, 9], "b": [4, 5, 6, 3, 2, 1, 0, 0, 0]},
+            index=[0, 1, 3, 5, 6, 8, 9, 9, 9],
+        )
+
+    @property
+    def kdf(self):
+        return ps.from_pandas(self.pdf)
+
+    def test_np_add_series(self):
+        kdf = self.kdf
+        pdf = self.pdf
+
+        if LooseVersion(pd.__version__) < LooseVersion("0.25"):
+            self.assert_eq(np.add(kdf.a, kdf.b), np.add(pdf.a, pdf.b).rename())
+        else:
+            self.assert_eq(np.add(kdf.a, kdf.b), np.add(pdf.a, pdf.b))
+
+        kdf = self.kdf
+        pdf = self.pdf
+        self.assert_eq(np.add(kdf.a, 1), np.add(pdf.a, 1))
+
+    def test_np_add_index(self):
+        k_index = self.kdf.index
+        p_index = self.pdf.index
+        self.assert_eq(np.add(k_index, k_index), np.add(p_index, p_index))
+
+    def test_np_unsupported_series(self):
+        kdf = self.kdf
+        with self.assertRaisesRegex(NotImplementedError, "pandas.*not.*support.*sqrt.*"):
+            np.sqrt(kdf.a, kdf.b)
+
+    def test_np_unsupported_frame(self):
+        kdf = self.kdf
+        with self.assertRaisesRegex(NotImplementedError, "on-Spark.*not.*support.*sqrt.*"):
+            np.sqrt(kdf, kdf)
+
+    def test_np_spark_compat_series(self):
+        # Use randomly generated dataFrame
+        pdf = pd.DataFrame(
+            np.random.randint(-100, 100, size=(np.random.randint(100), 2)), columns=["a", "b"]
+        )
+        pdf2 = pd.DataFrame(
+            np.random.randint(-100, 100, size=(len(pdf), len(pdf.columns))), columns=["a", "b"]
+        )
+        kdf = ps.from_pandas(pdf)
+        kdf2 = ps.from_pandas(pdf2)
+
+        for np_name, spark_func in unary_np_spark_mappings.items():
+            np_func = getattr(np, np_name)
+            if np_name not in self.blacklist:
+                try:
+                    # unary ufunc
+                    self.assert_eq(np_func(pdf.a), np_func(kdf.a), almost=True)
+                except Exception as e:
+                    raise AssertionError("Test in '%s' function was failed." % np_name) from e
+
+        for np_name, spark_func in binary_np_spark_mappings.items():
+            np_func = getattr(np, np_name)
+            if np_name not in self.blacklist:
+                try:
+                    # binary ufunc
+                    if LooseVersion(pd.__version__) < LooseVersion("0.25"):
+                        self.assert_eq(
+                            np_func(pdf.a, pdf.b).rename(), np_func(kdf.a, kdf.b), almost=True
+                        )
+                    else:
+                        self.assert_eq(np_func(pdf.a, pdf.b), np_func(kdf.a, kdf.b), almost=True)
+                    self.assert_eq(np_func(pdf.a, 1), np_func(kdf.a, 1), almost=True)
+                except Exception as e:
+                    raise AssertionError("Test in '%s' function was failed." % np_name) from e
+
+        # Test only top 5 for now. 'compute.ops_on_diff_frames' option increases too much time.
+        try:
+            set_option("compute.ops_on_diff_frames", True)
+            for np_name, spark_func in list(binary_np_spark_mappings.items())[:5]:
+                np_func = getattr(np, np_name)
+                if np_name not in self.blacklist:
+                    try:
+                        # binary ufunc
+                        if LooseVersion(pd.__version__) < LooseVersion("0.25"):
+                            self.assert_eq(
+                                np_func(pdf.a, pdf2.b).sort_index().rename(),
+                                np_func(kdf.a, kdf2.b).sort_index(),
+                                almost=True,
+                            )
+                        else:
+                            self.assert_eq(
+                                np_func(pdf.a, pdf2.b).sort_index(),
+                                np_func(kdf.a, kdf2.b).sort_index(),
+                                almost=True,
+                            )
+                    except Exception as e:
+                        raise AssertionError("Test in '%s' function was failed." % np_name) from e
+        finally:
+            reset_option("compute.ops_on_diff_frames")
+
+    def test_np_spark_compat_frame(self):
+        # Use randomly generated dataFrame
+        pdf = pd.DataFrame(
+            np.random.randint(-100, 100, size=(np.random.randint(100), 2)), columns=["a", "b"]
+        )
+        pdf2 = pd.DataFrame(
+            np.random.randint(-100, 100, size=(len(pdf), len(pdf.columns))), columns=["a", "b"]
+        )
+        kdf = ps.from_pandas(pdf)
+        kdf2 = ps.from_pandas(pdf2)
+
+        for np_name, spark_func in unary_np_spark_mappings.items():
+            np_func = getattr(np, np_name)
+            if np_name not in self.blacklist:
+                try:
+                    # unary ufunc
+                    self.assert_eq(np_func(pdf), np_func(kdf), almost=True)
+                except Exception as e:
+                    raise AssertionError("Test in '%s' function was failed." % np_name) from e
+
+        for np_name, spark_func in binary_np_spark_mappings.items():
+            np_func = getattr(np, np_name)
+            if np_name not in self.blacklist:
+                try:
+                    # binary ufunc
+                    self.assert_eq(np_func(pdf, pdf), np_func(kdf, kdf), almost=True)
+                    self.assert_eq(np_func(pdf, 1), np_func(kdf, 1), almost=True)
+                except Exception as e:
+                    raise AssertionError("Test in '%s' function was failed." % np_name) from e
+
+        # Test only top 5 for now. 'compute.ops_on_diff_frames' option increases too much time.
+        try:
+            set_option("compute.ops_on_diff_frames", True)
+            for np_name, spark_func in list(binary_np_spark_mappings.items())[:5]:
+                np_func = getattr(np, np_name)
+                if np_name not in self.blacklist:
+                    try:
+                        # binary ufunc
+                        self.assert_eq(
+                            np_func(pdf, pdf2).sort_index(),
+                            np_func(kdf, kdf2).sort_index(),
+                            almost=True,
+                        )
+
+                    except Exception as e:
+                        raise AssertionError("Test in '%s' function was failed." % np_name) from e
+        finally:
+            reset_option("compute.ops_on_diff_frames")
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.pandas.tests.test_numpy_compat import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_ops_on_diff_frames.py b/python/pyspark/pandas/tests/test_ops_on_diff_frames.py
new file mode 100644
index 0000000000000..31a296f45f69a
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_ops_on_diff_frames.py
@@ -0,0 +1,1948 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from distutils.version import LooseVersion
+from itertools import product
+import unittest
+
+import pandas as pd
+import numpy as np
+
+from pyspark import pandas as ps
+from pyspark.pandas.config import set_option, reset_option
+from pyspark.pandas.frame import DataFrame
+from pyspark.testing.pandasutils import PandasOnSparkTestCase
+from pyspark.testing.sqlutils import SQLTestUtils
+from pyspark.pandas.typedef.typehints import (
+    extension_dtypes,
+    extension_dtypes_available,
+    extension_float_dtypes_available,
+    extension_object_dtypes_available,
+)
+
+
+class OpsOnDiffFramesEnabledTest(PandasOnSparkTestCase, SQLTestUtils):
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        set_option("compute.ops_on_diff_frames", True)
+
+    @classmethod
+    def tearDownClass(cls):
+        reset_option("compute.ops_on_diff_frames")
+        super().tearDownClass()
+
+    @property
+    def pdf1(self):
+        return pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5, 6, 7, 8, 9], "b": [4, 5, 6, 3, 2, 1, 0, 0, 0]},
+            index=[0, 1, 3, 5, 6, 8, 9, 10, 11],
+        )
+
+    @property
+    def pdf2(self):
+        return pd.DataFrame(
+            {"a": [9, 8, 7, 6, 5, 4, 3, 2, 1], "b": [0, 0, 0, 4, 5, 6, 1, 2, 3]},
+            index=list(range(9)),
+        )
+
+    @property
+    def pdf3(self):
+        return pd.DataFrame(
+            {"b": [1, 1, 1, 1, 1, 1, 1, 1, 1], "c": [1, 1, 1, 1, 1, 1, 1, 1, 1]},
+            index=list(range(9)),
+        )
+
+    @property
+    def pdf4(self):
+        return pd.DataFrame(
+            {"e": [2, 2, 2, 2, 2, 2, 2, 2, 2], "f": [2, 2, 2, 2, 2, 2, 2, 2, 2]},
+            index=list(range(9)),
+        )
+
+    @property
+    def pdf5(self):
+        return pd.DataFrame(
+            {
+                "a": [1, 2, 3, 4, 5, 6, 7, 8, 9],
+                "b": [4, 5, 6, 3, 2, 1, 0, 0, 0],
+                "c": [4, 5, 6, 3, 2, 1, 0, 0, 0],
+            },
+            index=[0, 1, 3, 5, 6, 8, 9, 10, 11],
+        ).set_index(["a", "b"])
+
+    @property
+    def pdf6(self):
+        return pd.DataFrame(
+            {
+                "a": [9, 8, 7, 6, 5, 4, 3, 2, 1],
+                "b": [0, 0, 0, 4, 5, 6, 1, 2, 3],
+                "c": [9, 8, 7, 6, 5, 4, 3, 2, 1],
+                "e": [4, 5, 6, 3, 2, 1, 0, 0, 0],
+            },
+            index=list(range(9)),
+        ).set_index(["a", "b"])
+
+    @property
+    def pser1(self):
+        midx = pd.MultiIndex(
+            [["lama", "cow", "falcon", "koala"], ["speed", "weight", "length", "power"]],
+            [[0, 3, 1, 1, 1, 2, 2, 2], [0, 2, 0, 3, 2, 0, 1, 3]],
+        )
+        return pd.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1], index=midx)
+
+    @property
+    def pser2(self):
+        midx = pd.MultiIndex(
+            [["lama", "cow", "falcon"], ["speed", "weight", "length"]],
+            [[0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 1, 2, 0, 1, 2, 0, 1, 2]],
+        )
+        return pd.Series([-45, 200, -1.2, 30, -250, 1.5, 320, 1, -0.3], index=midx)
+
+    @property
+    def pser3(self):
+        midx = pd.MultiIndex(
+            [["koalas", "cow", "falcon"], ["speed", "weight", "length"]],
+            [[0, 0, 0, 1, 1, 1, 2, 2, 2], [1, 1, 2, 0, 0, 2, 2, 2, 1]],
+        )
+        return pd.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], index=midx)
+
+    @property
+    def kdf1(self):
+        return ps.from_pandas(self.pdf1)
+
+    @property
+    def kdf2(self):
+        return ps.from_pandas(self.pdf2)
+
+    @property
+    def kdf3(self):
+        return ps.from_pandas(self.pdf3)
+
+    @property
+    def kdf4(self):
+        return ps.from_pandas(self.pdf4)
+
+    @property
+    def kdf5(self):
+        return ps.from_pandas(self.pdf5)
+
+    @property
+    def kdf6(self):
+        return ps.from_pandas(self.pdf6)
+
+    @property
+    def kser1(self):
+        return ps.from_pandas(self.pser1)
+
+    @property
+    def kser2(self):
+        return ps.from_pandas(self.pser2)
+
+    @property
+    def kser3(self):
+        return ps.from_pandas(self.pser3)
+
+    def test_ranges(self):
+        self.assert_eq(
+            (ps.range(10) + ps.range(10)).sort_index(),
+            (
+                ps.DataFrame({"id": list(range(10))}) + ps.DataFrame({"id": list(range(10))})
+            ).sort_index(),
+        )
+
+    def test_no_matched_index(self):
+        with self.assertRaisesRegex(ValueError, "Index names must be exactly matched"):
+            ps.DataFrame({"a": [1, 2, 3]}).set_index("a") + ps.DataFrame(
+                {"b": [1, 2, 3]}
+            ).set_index("b")
+
+    def test_arithmetic(self):
+        self._test_arithmetic_frame(self.pdf1, self.pdf2, check_extension=False)
+        self._test_arithmetic_series(self.pser1, self.pser2, check_extension=False)
+
+    @unittest.skipIf(not extension_dtypes_available, "pandas extension dtypes are not available")
+    def test_arithmetic_extension_dtypes(self):
+        self._test_arithmetic_frame(
+            self.pdf1.astype("Int64"), self.pdf2.astype("Int64"), check_extension=True
+        )
+        self._test_arithmetic_series(
+            self.pser1.astype(int).astype("Int64"),
+            self.pser2.astype(int).astype("Int64"),
+            check_extension=True,
+        )
+
+    @unittest.skipIf(
+        not extension_float_dtypes_available, "pandas extension float dtypes are not available"
+    )
+    def test_arithmetic_extension_float_dtypes(self):
+        self._test_arithmetic_frame(
+            self.pdf1.astype("Float64"), self.pdf2.astype("Float64"), check_extension=True
+        )
+        self._test_arithmetic_series(
+            self.pser1.astype("Float64"), self.pser2.astype("Float64"), check_extension=True
+        )
+
+    def _test_arithmetic_frame(self, pdf1, pdf2, *, check_extension):
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        def assert_eq(actual, expected):
+            if LooseVersion("1.1") <= LooseVersion(pd.__version__) < LooseVersion("1.2.2"):
+                self.assert_eq(actual, expected, check_exact=not check_extension)
+                if check_extension:
+                    if isinstance(actual, DataFrame):
+                        for dtype in actual.dtypes:
+                            self.assertTrue(isinstance(dtype, extension_dtypes))
+                    else:
+                        self.assertTrue(isinstance(actual.dtype, extension_dtypes))
+            else:
+                self.assert_eq(actual, expected)
+
+        # Series
+        assert_eq((kdf1.a - kdf2.b).sort_index(), (pdf1.a - pdf2.b).sort_index())
+
+        assert_eq((kdf1.a * kdf2.a).sort_index(), (pdf1.a * pdf2.a).sort_index())
+
+        if check_extension and not extension_float_dtypes_available:
+            self.assert_eq(
+                (kdf1["a"] / kdf2["a"]).sort_index(), (pdf1["a"] / pdf2["a"]).sort_index()
+            )
+        else:
+            assert_eq((kdf1["a"] / kdf2["a"]).sort_index(), (pdf1["a"] / pdf2["a"]).sort_index())
+
+        # DataFrame
+        assert_eq((kdf1 + kdf2).sort_index(), (pdf1 + pdf2).sort_index())
+
+        # Multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b")])
+        kdf1.columns = columns
+        kdf2.columns = columns
+        pdf1.columns = columns
+        pdf2.columns = columns
+
+        # Series
+        assert_eq(
+            (kdf1[("x", "a")] - kdf2[("x", "b")]).sort_index(),
+            (pdf1[("x", "a")] - pdf2[("x", "b")]).sort_index(),
+        )
+
+        assert_eq(
+            (kdf1[("x", "a")] - kdf2["x"]["b"]).sort_index(),
+            (pdf1[("x", "a")] - pdf2["x"]["b"]).sort_index(),
+        )
+
+        assert_eq(
+            (kdf1["x"]["a"] - kdf2[("x", "b")]).sort_index(),
+            (pdf1["x"]["a"] - pdf2[("x", "b")]).sort_index(),
+        )
+
+        # DataFrame
+        assert_eq((kdf1 + kdf2).sort_index(), (pdf1 + pdf2).sort_index())
+
+    def _test_arithmetic_series(self, pser1, pser2, *, check_extension):
+        kser1 = ps.from_pandas(pser1)
+        kser2 = ps.from_pandas(pser2)
+
+        def assert_eq(actual, expected):
+            if LooseVersion("1.1") <= LooseVersion(pd.__version__) < LooseVersion("1.2.2"):
+                self.assert_eq(actual, expected, check_exact=not check_extension)
+                if check_extension:
+                    self.assertTrue(isinstance(actual.dtype, extension_dtypes))
+            else:
+                self.assert_eq(actual, expected)
+
+        # MultiIndex Series
+        assert_eq((kser1 + kser2).sort_index(), (pser1 + pser2).sort_index())
+
+        assert_eq((kser1 - kser2).sort_index(), (pser1 - pser2).sort_index())
+
+        assert_eq((kser1 * kser2).sort_index(), (pser1 * pser2).sort_index())
+
+        if check_extension and not extension_float_dtypes_available:
+            self.assert_eq((kser1 / kser2).sort_index(), (pser1 / pser2).sort_index())
+        else:
+            assert_eq((kser1 / kser2).sort_index(), (pser1 / pser2).sort_index())
+
+    def test_arithmetic_chain(self):
+        self._test_arithmetic_chain_frame(self.pdf1, self.pdf2, self.pdf3, check_extension=False)
+        self._test_arithmetic_chain_series(
+            self.pser1, self.pser2, self.pser3, check_extension=False
+        )
+
+    @unittest.skipIf(not extension_dtypes_available, "pandas extension dtypes are not available")
+    def test_arithmetic_chain_extension_dtypes(self):
+        self._test_arithmetic_chain_frame(
+            self.pdf1.astype("Int64"),
+            self.pdf2.astype("Int64"),
+            self.pdf3.astype("Int64"),
+            check_extension=True,
+        )
+        self._test_arithmetic_chain_series(
+            self.pser1.astype(int).astype("Int64"),
+            self.pser2.astype(int).astype("Int64"),
+            self.pser3.astype(int).astype("Int64"),
+            check_extension=True,
+        )
+
+    @unittest.skipIf(
+        not extension_float_dtypes_available, "pandas extension float dtypes are not available"
+    )
+    def test_arithmetic_chain_extension_float_dtypes(self):
+        self._test_arithmetic_chain_frame(
+            self.pdf1.astype("Float64"),
+            self.pdf2.astype("Float64"),
+            self.pdf3.astype("Float64"),
+            check_extension=True,
+        )
+        self._test_arithmetic_chain_series(
+            self.pser1.astype("Float64"),
+            self.pser2.astype("Float64"),
+            self.pser3.astype("Float64"),
+            check_extension=True,
+        )
+
+    def _test_arithmetic_chain_frame(self, pdf1, pdf2, pdf3, *, check_extension):
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+        kdf3 = ps.from_pandas(pdf3)
+
+        common_columns = set(kdf1.columns).intersection(kdf2.columns).intersection(kdf3.columns)
+
+        def assert_eq(actual, expected):
+            if LooseVersion("1.1") <= LooseVersion(pd.__version__) < LooseVersion("1.2.2"):
+                self.assert_eq(actual, expected, check_exact=not check_extension)
+                if check_extension:
+                    if isinstance(actual, DataFrame):
+                        for column, dtype in zip(actual.columns, actual.dtypes):
+                            if column in common_columns:
+                                self.assertTrue(isinstance(dtype, extension_dtypes))
+                            else:
+                                self.assertFalse(isinstance(dtype, extension_dtypes))
+                    else:
+                        self.assertTrue(isinstance(actual.dtype, extension_dtypes))
+            else:
+                self.assert_eq(actual, expected)
+
+        # Series
+        assert_eq((kdf1.a - kdf2.b - kdf3.c).sort_index(), (pdf1.a - pdf2.b - pdf3.c).sort_index())
+
+        assert_eq(
+            (kdf1.a * (kdf2.a * kdf3.c)).sort_index(), (pdf1.a * (pdf2.a * pdf3.c)).sort_index()
+        )
+
+        if check_extension and not extension_float_dtypes_available:
+            self.assert_eq(
+                (kdf1["a"] / kdf2["a"] / kdf3["c"]).sort_index(),
+                (pdf1["a"] / pdf2["a"] / pdf3["c"]).sort_index(),
+            )
+        else:
+            assert_eq(
+                (kdf1["a"] / kdf2["a"] / kdf3["c"]).sort_index(),
+                (pdf1["a"] / pdf2["a"] / pdf3["c"]).sort_index(),
+            )
+
+        # DataFrame
+        if check_extension and (
+            LooseVersion("1.0") <= LooseVersion(pd.__version__) < LooseVersion("1.1")
+        ):
+            self.assert_eq(
+                (kdf1 + kdf2 - kdf3).sort_index(), (pdf1 + pdf2 - pdf3).sort_index(), almost=True
+            )
+        else:
+            assert_eq((kdf1 + kdf2 - kdf3).sort_index(), (pdf1 + pdf2 - pdf3).sort_index())
+
+        # Multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b")])
+        kdf1.columns = columns
+        kdf2.columns = columns
+        pdf1.columns = columns
+        pdf2.columns = columns
+        columns = pd.MultiIndex.from_tuples([("x", "b"), ("y", "c")])
+        kdf3.columns = columns
+        pdf3.columns = columns
+
+        common_columns = set(kdf1.columns).intersection(kdf2.columns).intersection(kdf3.columns)
+
+        # Series
+        assert_eq(
+            (kdf1[("x", "a")] - kdf2[("x", "b")] - kdf3[("y", "c")]).sort_index(),
+            (pdf1[("x", "a")] - pdf2[("x", "b")] - pdf3[("y", "c")]).sort_index(),
+        )
+
+        assert_eq(
+            (kdf1[("x", "a")] * (kdf2[("x", "b")] * kdf3[("y", "c")])).sort_index(),
+            (pdf1[("x", "a")] * (pdf2[("x", "b")] * pdf3[("y", "c")])).sort_index(),
+        )
+
+        # DataFrame
+        if check_extension and (
+            LooseVersion("1.0") <= LooseVersion(pd.__version__) < LooseVersion("1.1")
+        ):
+            self.assert_eq(
+                (kdf1 + kdf2 - kdf3).sort_index(), (pdf1 + pdf2 - pdf3).sort_index(), almost=True
+            )
+        else:
+            assert_eq((kdf1 + kdf2 - kdf3).sort_index(), (pdf1 + pdf2 - pdf3).sort_index())
+
+    def _test_arithmetic_chain_series(self, pser1, pser2, pser3, *, check_extension):
+        kser1 = ps.from_pandas(pser1)
+        kser2 = ps.from_pandas(pser2)
+        kser3 = ps.from_pandas(pser3)
+
+        def assert_eq(actual, expected):
+            if LooseVersion("1.1") <= LooseVersion(pd.__version__) < LooseVersion("1.2.2"):
+                self.assert_eq(actual, expected, check_exact=not check_extension)
+                if check_extension:
+                    self.assertTrue(isinstance(actual.dtype, extension_dtypes))
+            else:
+                self.assert_eq(actual, expected)
+
+        # MultiIndex Series
+        assert_eq((kser1 + kser2 - kser3).sort_index(), (pser1 + pser2 - pser3).sort_index())
+
+        assert_eq((kser1 * kser2 * kser3).sort_index(), (pser1 * pser2 * pser3).sort_index())
+
+        if check_extension and not extension_float_dtypes_available:
+            if LooseVersion(pd.__version__) >= LooseVersion("1.0"):
+                self.assert_eq(
+                    (kser1 - kser2 / kser3).sort_index(), (pser1 - pser2 / pser3).sort_index()
+                )
+            else:
+                expected = pd.Series(
+                    [249.0, np.nan, 0.0, 0.88, np.nan, np.nan, np.nan, np.nan, np.nan, -np.inf]
+                    + [np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan],
+                    index=pd.MultiIndex(
+                        [
+                            ["cow", "falcon", "koala", "koalas", "lama"],
+                            ["length", "power", "speed", "weight"],
+                        ],
+                        [
+                            [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 3, 3, 3, 4, 4, 4],
+                            [0, 1, 2, 2, 3, 0, 0, 1, 2, 3, 0, 0, 3, 3, 0, 2, 3],
+                        ],
+                    ),
+                )
+                self.assert_eq((kser1 - kser2 / kser3).sort_index(), expected)
+        else:
+            assert_eq((kser1 - kser2 / kser3).sort_index(), (pser1 - pser2 / pser3).sort_index())
+
+        assert_eq((kser1 + kser2 * kser3).sort_index(), (pser1 + pser2 * pser3).sort_index())
+
+    def test_mod(self):
+        pser = pd.Series([100, None, -300, None, 500, -700])
+        pser_other = pd.Series([-150] * 6)
+        kser = ps.from_pandas(pser)
+        kser_other = ps.from_pandas(pser_other)
+
+        self.assert_eq(kser.mod(kser_other).sort_index(), pser.mod(pser_other))
+        self.assert_eq(kser.mod(kser_other).sort_index(), pser.mod(pser_other))
+        self.assert_eq(kser.mod(kser_other).sort_index(), pser.mod(pser_other))
+
+    def test_rmod(self):
+        pser = pd.Series([100, None, -300, None, 500, -700])
+        pser_other = pd.Series([-150] * 6)
+        kser = ps.from_pandas(pser)
+        kser_other = ps.from_pandas(pser_other)
+
+        self.assert_eq(kser.rmod(kser_other).sort_index(), pser.rmod(pser_other))
+        self.assert_eq(kser.rmod(kser_other).sort_index(), pser.rmod(pser_other))
+        self.assert_eq(kser.rmod(kser_other).sort_index(), pser.rmod(pser_other))
+
+    def test_getitem_boolean_series(self):
+        pdf1 = pd.DataFrame(
+            {"A": [0, 1, 2, 3, 4], "B": [100, 200, 300, 400, 500]}, index=[20, 10, 30, 0, 50]
+        )
+        pdf2 = pd.DataFrame(
+            {"A": [0, -1, -2, -3, -4], "B": [-100, -200, -300, -400, -500]},
+            index=[0, 30, 10, 20, 50],
+        )
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        self.assert_eq(pdf1[pdf2.A > -3].sort_index(), kdf1[kdf2.A > -3].sort_index())
+
+        self.assert_eq(pdf1.A[pdf2.A > -3].sort_index(), kdf1.A[kdf2.A > -3].sort_index())
+
+        self.assert_eq(
+            (pdf1.A + 1)[pdf2.A > -3].sort_index(), (kdf1.A + 1)[kdf2.A > -3].sort_index()
+        )
+
+    def test_loc_getitem_boolean_series(self):
+        pdf1 = pd.DataFrame(
+            {"A": [0, 1, 2, 3, 4], "B": [100, 200, 300, 400, 500]}, index=[20, 10, 30, 0, 50]
+        )
+        pdf2 = pd.DataFrame(
+            {"A": [0, -1, -2, -3, -4], "B": [-100, -200, -300, -400, -500]},
+            index=[20, 10, 30, 0, 50],
+        )
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        self.assert_eq(pdf1.loc[pdf2.A > -3].sort_index(), kdf1.loc[kdf2.A > -3].sort_index())
+
+        self.assert_eq(pdf1.A.loc[pdf2.A > -3].sort_index(), kdf1.A.loc[kdf2.A > -3].sort_index())
+
+        self.assert_eq(
+            (pdf1.A + 1).loc[pdf2.A > -3].sort_index(), (kdf1.A + 1).loc[kdf2.A > -3].sort_index()
+        )
+
+    def test_bitwise(self):
+        pser1 = pd.Series([True, False, True, False, np.nan, np.nan, True, False, np.nan])
+        pser2 = pd.Series([True, False, False, True, True, False, np.nan, np.nan, np.nan])
+        kser1 = ps.from_pandas(pser1)
+        kser2 = ps.from_pandas(pser2)
+
+        self.assert_eq(pser1 | pser2, (kser1 | kser2).sort_index())
+        self.assert_eq(pser1 & pser2, (kser1 & kser2).sort_index())
+
+        pser1 = pd.Series([True, False, np.nan], index=list("ABC"))
+        pser2 = pd.Series([False, True, np.nan], index=list("DEF"))
+        kser1 = ps.from_pandas(pser1)
+        kser2 = ps.from_pandas(pser2)
+
+        self.assert_eq(pser1 | pser2, (kser1 | kser2).sort_index())
+        self.assert_eq(pser1 & pser2, (kser1 & kser2).sort_index())
+
+    @unittest.skipIf(
+        not extension_object_dtypes_available, "pandas extension object dtypes are not available"
+    )
+    def test_bitwise_extension_dtype(self):
+        def assert_eq(actual, expected):
+            if LooseVersion("1.1") <= LooseVersion(pd.__version__) < LooseVersion("1.2.2"):
+                self.assert_eq(actual, expected, check_exact=False)
+                self.assertTrue(isinstance(actual.dtype, extension_dtypes))
+            else:
+                self.assert_eq(actual, expected)
+
+        pser1 = pd.Series(
+            [True, False, True, False, np.nan, np.nan, True, False, np.nan], dtype="boolean"
+        )
+        pser2 = pd.Series(
+            [True, False, False, True, True, False, np.nan, np.nan, np.nan], dtype="boolean"
+        )
+        kser1 = ps.from_pandas(pser1)
+        kser2 = ps.from_pandas(pser2)
+
+        assert_eq((kser1 | kser2).sort_index(), pser1 | pser2)
+        assert_eq((kser1 & kser2).sort_index(), pser1 & pser2)
+
+        pser1 = pd.Series([True, False, np.nan], index=list("ABC"), dtype="boolean")
+        pser2 = pd.Series([False, True, np.nan], index=list("DEF"), dtype="boolean")
+        kser1 = ps.from_pandas(pser1)
+        kser2 = ps.from_pandas(pser2)
+
+        # a pandas bug?
+        # assert_eq((kser1 | kser2).sort_index(), pser1 | pser2)
+        # assert_eq((kser1 & kser2).sort_index(), pser1 & pser2)
+        assert_eq(
+            (kser1 | kser2).sort_index(),
+            pd.Series([True, None, None, None, True, None], index=list("ABCDEF"), dtype="boolean"),
+        )
+        assert_eq(
+            (kser1 & kser2).sort_index(),
+            pd.Series(
+                [None, False, None, False, None, None], index=list("ABCDEF"), dtype="boolean"
+            ),
+        )
+
+    def test_concat_column_axis(self):
+        pdf1 = pd.DataFrame({"A": [0, 2, 4], "B": [1, 3, 5]}, index=[1, 2, 3])
+        pdf1.columns.names = ["AB"]
+        pdf2 = pd.DataFrame({"C": [1, 2, 3], "D": [4, 5, 6]}, index=[1, 3, 5])
+        pdf2.columns.names = ["CD"]
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        kdf3 = kdf1.copy()
+        kdf4 = kdf2.copy()
+        pdf3 = pdf1.copy()
+        pdf4 = pdf2.copy()
+
+        columns = pd.MultiIndex.from_tuples([("X", "A"), ("X", "B")], names=["X", "AB"])
+        pdf3.columns = columns
+        kdf3.columns = columns
+
+        columns = pd.MultiIndex.from_tuples([("X", "C"), ("X", "D")], names=["Y", "CD"])
+        pdf4.columns = columns
+        kdf4.columns = columns
+
+        pdf5 = pd.DataFrame({"A": [0, 2, 4], "B": [1, 3, 5]}, index=[1, 2, 3])
+        pdf6 = pd.DataFrame({"C": [1, 2, 3]}, index=[1, 3, 5])
+        kdf5 = ps.from_pandas(pdf5)
+        kdf6 = ps.from_pandas(pdf6)
+
+        ignore_indexes = [True, False]
+        joins = ["inner", "outer"]
+
+        objs = [
+            ([kdf1.A, kdf2.C], [pdf1.A, pdf2.C]),
+            # TODO: ([kdf1, kdf2.C], [pdf1, pdf2.C]),
+            ([kdf1.A, kdf2], [pdf1.A, pdf2]),
+            ([kdf1.A, kdf2.C], [pdf1.A, pdf2.C]),
+            ([kdf3[("X", "A")], kdf4[("X", "C")]], [pdf3[("X", "A")], pdf4[("X", "C")]]),
+            ([kdf3, kdf4[("X", "C")]], [pdf3, pdf4[("X", "C")]]),
+            ([kdf3[("X", "A")], kdf4], [pdf3[("X", "A")], pdf4]),
+            ([kdf3, kdf4], [pdf3, pdf4]),
+            ([kdf5, kdf6], [pdf5, pdf6]),
+            ([kdf6, kdf5], [pdf6, pdf5]),
+        ]
+
+        for ignore_index, join in product(ignore_indexes, joins):
+            for i, (kdfs, pdfs) in enumerate(objs):
+                with self.subTest(ignore_index=ignore_index, join=join, pdfs=pdfs, pair=i):
+                    actual = ps.concat(kdfs, axis=1, ignore_index=ignore_index, join=join)
+                    expected = pd.concat(pdfs, axis=1, ignore_index=ignore_index, join=join)
+                    self.assert_eq(
+                        repr(actual.sort_values(list(actual.columns)).reset_index(drop=True)),
+                        repr(expected.sort_values(list(expected.columns)).reset_index(drop=True)),
+                    )
+
+    def test_combine_first(self):
+        pser1 = pd.Series({"falcon": 330.0, "eagle": 160.0})
+        pser2 = pd.Series({"falcon": 345.0, "eagle": 200.0, "duck": 30.0})
+        kser1 = ps.from_pandas(pser1)
+        kser2 = ps.from_pandas(pser2)
+
+        self.assert_eq(
+            kser1.combine_first(kser2).sort_index(), pser1.combine_first(pser2).sort_index()
+        )
+        with self.assertRaisesRegex(
+            TypeError, "`combine_first` only allows `Series` for parameter `other`"
+        ):
+            kser1.combine_first(50)
+
+        kser1.name = ("X", "A")
+        kser2.name = ("Y", "B")
+        pser1.name = ("X", "A")
+        pser2.name = ("Y", "B")
+        self.assert_eq(
+            kser1.combine_first(kser2).sort_index(), pser1.combine_first(pser2).sort_index()
+        )
+
+        # MultiIndex
+        midx1 = pd.MultiIndex(
+            [["lama", "cow", "falcon", "koala"], ["speed", "weight", "length", "power"]],
+            [[0, 3, 1, 1, 1, 2, 2, 2], [0, 2, 0, 3, 2, 0, 1, 3]],
+        )
+        midx2 = pd.MultiIndex(
+            [["lama", "cow", "falcon"], ["speed", "weight", "length"]],
+            [[0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 1, 2, 0, 1, 2, 0, 1, 2]],
+        )
+        pser1 = pd.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1], index=midx1)
+        pser2 = pd.Series([-45, 200, -1.2, 30, -250, 1.5, 320, 1, -0.3], index=midx2)
+        kser1 = ps.from_pandas(pser1)
+        kser2 = ps.from_pandas(pser2)
+
+        self.assert_eq(
+            kser1.combine_first(kser2).sort_index(), pser1.combine_first(pser2).sort_index()
+        )
+
+        # Series come from same DataFrame
+        pdf = pd.DataFrame(
+            {
+                "A": {"falcon": 330.0, "eagle": 160.0},
+                "B": {"falcon": 345.0, "eagle": 200.0, "duck": 30.0},
+            }
+        )
+        pser1 = pdf.A
+        pser2 = pdf.B
+        kser1 = ps.from_pandas(pser1)
+        kser2 = ps.from_pandas(pser2)
+
+        self.assert_eq(
+            kser1.combine_first(kser2).sort_index(), pser1.combine_first(pser2).sort_index()
+        )
+
+        kser1.name = ("X", "A")
+        kser2.name = ("Y", "B")
+        pser1.name = ("X", "A")
+        pser2.name = ("Y", "B")
+
+        self.assert_eq(
+            kser1.combine_first(kser2).sort_index(), pser1.combine_first(pser2).sort_index()
+        )
+
+    def test_insert(self):
+        #
+        # Basic DataFrame
+        #
+        pdf = pd.DataFrame([1, 2, 3])
+        kdf = ps.from_pandas(pdf)
+
+        pser = pd.Series([4, 5, 6])
+        kser = ps.from_pandas(pser)
+        kdf.insert(1, "y", kser)
+        pdf.insert(1, "y", pser)
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+
+        #
+        # DataFrame with Index different from inserting Series'
+        #
+        pdf = pd.DataFrame([1, 2, 3], index=[10, 20, 30])
+        kdf = ps.from_pandas(pdf)
+
+        pser = pd.Series([4, 5, 6])
+        kser = ps.from_pandas(pser)
+        kdf.insert(1, "y", kser)
+        pdf.insert(1, "y", pser)
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+
+        #
+        # DataFrame with Multi-index columns
+        #
+        pdf = pd.DataFrame({("x", "a"): [1, 2, 3]})
+        kdf = ps.from_pandas(pdf)
+
+        pser = pd.Series([4, 5, 6])
+        kser = ps.from_pandas(pser)
+        pdf = pd.DataFrame({("x", "a", "b"): [1, 2, 3]})
+        kdf = ps.from_pandas(pdf)
+        kdf.insert(0, "a", kser)
+        pdf.insert(0, "a", pser)
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+        kdf.insert(0, ("b", "c", ""), kser)
+        pdf.insert(0, ("b", "c", ""), pser)
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+
+    def test_compare(self):
+        if LooseVersion(pd.__version__) >= LooseVersion("1.1"):
+            pser1 = pd.Series(["b", "c", np.nan, "g", np.nan])
+            pser2 = pd.Series(["a", "c", np.nan, np.nan, "h"])
+            kser1 = ps.from_pandas(pser1)
+            kser2 = ps.from_pandas(pser2)
+            self.assert_eq(
+                pser1.compare(pser2).sort_index(), kser1.compare(kser2).sort_index(),
+            )
+
+            # `keep_shape=True`
+            self.assert_eq(
+                pser1.compare(pser2, keep_shape=True).sort_index(),
+                kser1.compare(kser2, keep_shape=True).sort_index(),
+            )
+            # `keep_equal=True`
+            self.assert_eq(
+                pser1.compare(pser2, keep_equal=True).sort_index(),
+                kser1.compare(kser2, keep_equal=True).sort_index(),
+            )
+            # `keep_shape=True` and `keep_equal=True`
+            self.assert_eq(
+                pser1.compare(pser2, keep_shape=True, keep_equal=True).sort_index(),
+                kser1.compare(kser2, keep_shape=True, keep_equal=True).sort_index(),
+            )
+
+            # MultiIndex
+            pser1.index = pd.MultiIndex.from_tuples(
+                [("a", "x"), ("b", "y"), ("c", "z"), ("x", "k"), ("q", "l")]
+            )
+            pser2.index = pd.MultiIndex.from_tuples(
+                [("a", "x"), ("b", "y"), ("c", "z"), ("x", "k"), ("q", "l")]
+            )
+            kser1 = ps.from_pandas(pser1)
+            kser2 = ps.from_pandas(pser2)
+            self.assert_eq(
+                pser1.compare(pser2).sort_index(), kser1.compare(kser2).sort_index(),
+            )
+
+            # `keep_shape=True` with MultiIndex
+            self.assert_eq(
+                pser1.compare(pser2, keep_shape=True).sort_index(),
+                kser1.compare(kser2, keep_shape=True).sort_index(),
+            )
+            # `keep_equal=True` with MultiIndex
+            self.assert_eq(
+                pser1.compare(pser2, keep_equal=True).sort_index(),
+                kser1.compare(kser2, keep_equal=True).sort_index(),
+            )
+            # `keep_shape=True` and `keep_equal=True` with MultiIndex
+            self.assert_eq(
+                pser1.compare(pser2, keep_shape=True, keep_equal=True).sort_index(),
+                kser1.compare(kser2, keep_shape=True, keep_equal=True).sort_index(),
+            )
+        else:
+            kser1 = ps.Series(["b", "c", np.nan, "g", np.nan])
+            kser2 = ps.Series(["a", "c", np.nan, np.nan, "h"])
+            expected = ps.DataFrame(
+                [["b", "a"], ["g", None], [None, "h"]], index=[0, 3, 4], columns=["self", "other"]
+            )
+            self.assert_eq(expected, kser1.compare(kser2).sort_index())
+
+            # `keep_shape=True`
+            expected = ps.DataFrame(
+                [["b", "a"], [None, None], [None, None], ["g", None], [None, "h"]],
+                index=[0, 1, 2, 3, 4],
+                columns=["self", "other"],
+            )
+            self.assert_eq(
+                expected, kser1.compare(kser2, keep_shape=True).sort_index(),
+            )
+            # `keep_equal=True`
+            expected = ps.DataFrame(
+                [["b", "a"], ["g", None], [None, "h"]], index=[0, 3, 4], columns=["self", "other"]
+            )
+            self.assert_eq(
+                expected, kser1.compare(kser2, keep_equal=True).sort_index(),
+            )
+            # `keep_shape=True` and `keep_equal=True`
+            expected = ps.DataFrame(
+                [["b", "a"], ["c", "c"], [None, None], ["g", None], [None, "h"]],
+                index=[0, 1, 2, 3, 4],
+                columns=["self", "other"],
+            )
+            self.assert_eq(
+                expected, kser1.compare(kser2, keep_shape=True, keep_equal=True).sort_index(),
+            )
+
+            # MultiIndex
+            kser1 = ps.Series(
+                ["b", "c", np.nan, "g", np.nan],
+                index=pd.MultiIndex.from_tuples(
+                    [("a", "x"), ("b", "y"), ("c", "z"), ("x", "k"), ("q", "l")]
+                ),
+            )
+            kser2 = ps.Series(
+                ["a", "c", np.nan, np.nan, "h"],
+                index=pd.MultiIndex.from_tuples(
+                    [("a", "x"), ("b", "y"), ("c", "z"), ("x", "k"), ("q", "l")]
+                ),
+            )
+            expected = ps.DataFrame(
+                [["b", "a"], [None, "h"], ["g", None]],
+                index=pd.MultiIndex.from_tuples([("a", "x"), ("q", "l"), ("x", "k")]),
+                columns=["self", "other"],
+            )
+            self.assert_eq(expected, kser1.compare(kser2).sort_index())
+
+            # `keep_shape=True`
+            expected = ps.DataFrame(
+                [["b", "a"], [None, None], [None, None], [None, "h"], ["g", None]],
+                index=pd.MultiIndex.from_tuples(
+                    [("a", "x"), ("b", "y"), ("c", "z"), ("q", "l"), ("x", "k")]
+                ),
+                columns=["self", "other"],
+            )
+            self.assert_eq(
+                expected, kser1.compare(kser2, keep_shape=True).sort_index(),
+            )
+            # `keep_equal=True`
+            expected = ps.DataFrame(
+                [["b", "a"], [None, "h"], ["g", None]],
+                index=pd.MultiIndex.from_tuples([("a", "x"), ("q", "l"), ("x", "k")]),
+                columns=["self", "other"],
+            )
+            self.assert_eq(
+                expected, kser1.compare(kser2, keep_equal=True).sort_index(),
+            )
+            # `keep_shape=True` and `keep_equal=True`
+            expected = ps.DataFrame(
+                [["b", "a"], ["c", "c"], [None, None], [None, "h"], ["g", None]],
+                index=pd.MultiIndex.from_tuples(
+                    [("a", "x"), ("b", "y"), ("c", "z"), ("q", "l"), ("x", "k")]
+                ),
+                columns=["self", "other"],
+            )
+            self.assert_eq(
+                expected, kser1.compare(kser2, keep_shape=True, keep_equal=True).sort_index(),
+            )
+
+        # Different Index
+        with self.assertRaisesRegex(
+            ValueError, "Can only compare identically-labeled Series objects"
+        ):
+            kser1 = ps.Series([1, 2, 3, 4, 5], index=pd.Index([1, 2, 3, 4, 5]),)
+            kser2 = ps.Series([2, 2, 3, 4, 1], index=pd.Index([5, 4, 3, 2, 1]),)
+            kser1.compare(kser2)
+        # Different MultiIndex
+        with self.assertRaisesRegex(
+            ValueError, "Can only compare identically-labeled Series objects"
+        ):
+            kser1 = ps.Series(
+                [1, 2, 3, 4, 5],
+                index=pd.MultiIndex.from_tuples(
+                    [("a", "x"), ("b", "y"), ("c", "z"), ("x", "k"), ("q", "l")]
+                ),
+            )
+            kser2 = ps.Series(
+                [2, 2, 3, 4, 1],
+                index=pd.MultiIndex.from_tuples(
+                    [("a", "x"), ("b", "y"), ("c", "a"), ("x", "k"), ("q", "l")]
+                ),
+            )
+            kser1.compare(kser2)
+
+    def test_different_columns(self):
+        kdf1 = self.kdf1
+        kdf4 = self.kdf4
+        pdf1 = self.pdf1
+        pdf4 = self.pdf4
+
+        self.assert_eq((kdf1 + kdf4).sort_index(), (pdf1 + pdf4).sort_index(), almost=True)
+
+        # Multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b")])
+        kdf1.columns = columns
+        pdf1.columns = columns
+        columns = pd.MultiIndex.from_tuples([("z", "e"), ("z", "f")])
+        kdf4.columns = columns
+        pdf4.columns = columns
+
+        self.assert_eq((kdf1 + kdf4).sort_index(), (pdf1 + pdf4).sort_index(), almost=True)
+
+    def test_assignment_series(self):
+        kdf = ps.from_pandas(self.pdf1)
+        pdf = self.pdf1
+        kser = kdf.a
+        pser = pdf.a
+        kdf["a"] = self.kdf2.a
+        pdf["a"] = self.pdf2.a
+
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+        self.assert_eq(kser, pser)
+
+        kdf = ps.from_pandas(self.pdf1)
+        pdf = self.pdf1
+        kser = kdf.a
+        pser = pdf.a
+        kdf["a"] = self.kdf2.b
+        pdf["a"] = self.pdf2.b
+
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+        self.assert_eq(kser, pser)
+
+        kdf = ps.from_pandas(self.pdf1)
+        pdf = self.pdf1
+        kdf["c"] = self.kdf2.a
+        pdf["c"] = self.pdf2.a
+
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+
+        # Multi-index columns
+        kdf = ps.from_pandas(self.pdf1)
+        pdf = self.pdf1
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b")])
+        kdf.columns = columns
+        pdf.columns = columns
+        kdf[("y", "c")] = self.kdf2.a
+        pdf[("y", "c")] = self.pdf2.a
+
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+
+        pdf = pd.DataFrame({"a": [1, 2, 3], "Koalas": [0, 1, 2]}).set_index("Koalas", drop=False)
+        kdf = ps.from_pandas(pdf)
+
+        kdf.index.name = None
+        kdf["NEW"] = ps.Series([100, 200, 300])
+
+        pdf.index.name = None
+        pdf["NEW"] = pd.Series([100, 200, 300])
+
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+
+    def test_assignment_frame(self):
+        kdf = ps.from_pandas(self.pdf1)
+        pdf = self.pdf1
+        kser = kdf.a
+        pser = pdf.a
+        kdf[["a", "b"]] = self.kdf1
+        pdf[["a", "b"]] = self.pdf1
+
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+        self.assert_eq(kser, pser)
+
+        # 'c' does not exist in `kdf`.
+        kdf = ps.from_pandas(self.pdf1)
+        pdf = self.pdf1
+        kser = kdf.a
+        pser = pdf.a
+        kdf[["b", "c"]] = self.kdf1
+        pdf[["b", "c"]] = self.pdf1
+
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+        self.assert_eq(kser, pser)
+
+        # 'c' and 'd' do not exist in `kdf`.
+        kdf = ps.from_pandas(self.pdf1)
+        pdf = self.pdf1
+        kdf[["c", "d"]] = self.kdf1
+        pdf[["c", "d"]] = self.pdf1
+
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+
+        # Multi-index columns
+        columns = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b")])
+        kdf = ps.from_pandas(self.pdf1)
+        pdf = self.pdf1
+        kdf.columns = columns
+        pdf.columns = columns
+        kdf[[("y", "c"), ("z", "d")]] = self.kdf1
+        pdf[[("y", "c"), ("z", "d")]] = self.pdf1
+
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+
+        kdf = ps.from_pandas(self.pdf1)
+        pdf = self.pdf1
+        kdf1 = ps.from_pandas(self.pdf1)
+        pdf1 = self.pdf1
+        kdf1.columns = columns
+        pdf1.columns = columns
+        kdf[["c", "d"]] = kdf1
+        pdf[["c", "d"]] = pdf1
+
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+
+    def test_assignment_series_chain(self):
+        kdf = ps.from_pandas(self.pdf1)
+        pdf = self.pdf1
+        kdf["a"] = self.kdf1.a
+        pdf["a"] = self.pdf1.a
+
+        kdf["a"] = self.kdf2.b
+        pdf["a"] = self.pdf2.b
+
+        kdf["d"] = self.kdf3.c
+        pdf["d"] = self.pdf3.c
+
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+
+    def test_assignment_frame_chain(self):
+        kdf = ps.from_pandas(self.pdf1)
+        pdf = self.pdf1
+        kdf[["a", "b"]] = self.kdf1
+        pdf[["a", "b"]] = self.pdf1
+
+        kdf[["e", "f"]] = self.kdf3
+        pdf[["e", "f"]] = self.pdf3
+
+        kdf[["b", "c"]] = self.kdf2
+        pdf[["b", "c"]] = self.pdf2
+
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+
+    def test_multi_index_arithmetic(self):
+        kdf5 = self.kdf5
+        kdf6 = self.kdf6
+        pdf5 = self.pdf5
+        pdf6 = self.pdf6
+
+        # Series
+        self.assert_eq((kdf5.c - kdf6.e).sort_index(), (pdf5.c - pdf6.e).sort_index())
+
+        self.assert_eq((kdf5["c"] / kdf6["e"]).sort_index(), (pdf5["c"] / pdf6["e"]).sort_index())
+
+        # DataFrame
+        self.assert_eq((kdf5 + kdf6).sort_index(), (pdf5 + pdf6).sort_index(), almost=True)
+
+    def test_multi_index_assignment_series(self):
+        kdf = ps.from_pandas(self.pdf5)
+        pdf = self.pdf5
+        kdf["x"] = self.kdf6.e
+        pdf["x"] = self.pdf6.e
+
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+
+        kdf = ps.from_pandas(self.pdf5)
+        pdf = self.pdf5
+        kdf["e"] = self.kdf6.e
+        pdf["e"] = self.pdf6.e
+
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+
+        kdf = ps.from_pandas(self.pdf5)
+        pdf = self.pdf5
+        kdf["c"] = self.kdf6.e
+        pdf["c"] = self.pdf6.e
+
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+
+    def test_multi_index_assignment_frame(self):
+        kdf = ps.from_pandas(self.pdf5)
+        pdf = self.pdf5
+        kdf[["c"]] = self.kdf5
+        pdf[["c"]] = self.pdf5
+
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+
+        kdf = ps.from_pandas(self.pdf5)
+        pdf = self.pdf5
+        kdf[["x"]] = self.kdf5
+        pdf[["x"]] = self.pdf5
+
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+
+        kdf = ps.from_pandas(self.pdf6)
+        pdf = self.pdf6
+        kdf[["x", "y"]] = self.kdf6
+        pdf[["x", "y"]] = self.pdf6
+
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+
+    def test_frame_loc_setitem(self):
+        pdf_orig = pd.DataFrame(
+            [[1, 2], [4, 5], [7, 8]],
+            index=["cobra", "viper", "sidewinder"],
+            columns=["max_speed", "shield"],
+        )
+        kdf_orig = ps.DataFrame(pdf_orig)
+
+        pdf = pdf_orig.copy()
+        kdf = kdf_orig.copy()
+        pser1 = pdf.max_speed
+        pser2 = pdf.shield
+        kser1 = kdf.max_speed
+        kser2 = kdf.shield
+
+        another_kdf = ps.DataFrame(pdf_orig)
+
+        kdf.loc[["viper", "sidewinder"], ["shield"]] = -another_kdf.max_speed
+        pdf.loc[["viper", "sidewinder"], ["shield"]] = -pdf.max_speed
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kser1, pser1)
+        self.assert_eq(kser2, pser2)
+
+        pdf = pdf_orig.copy()
+        kdf = kdf_orig.copy()
+        pser1 = pdf.max_speed
+        pser2 = pdf.shield
+        kser1 = kdf.max_speed
+        kser2 = kdf.shield
+        kdf.loc[another_kdf.max_speed < 5, ["shield"]] = -kdf.max_speed
+        pdf.loc[pdf.max_speed < 5, ["shield"]] = -pdf.max_speed
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kser1, pser1)
+        self.assert_eq(kser2, pser2)
+
+        pdf = pdf_orig.copy()
+        kdf = kdf_orig.copy()
+        pser1 = pdf.max_speed
+        pser2 = pdf.shield
+        kser1 = kdf.max_speed
+        kser2 = kdf.shield
+        kdf.loc[another_kdf.max_speed < 5, ["shield"]] = -another_kdf.max_speed
+        pdf.loc[pdf.max_speed < 5, ["shield"]] = -pdf.max_speed
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kser1, pser1)
+        self.assert_eq(kser2, pser2)
+
+    def test_frame_iloc_setitem(self):
+        pdf = pd.DataFrame(
+            [[1, 2], [4, 5], [7, 8]],
+            index=["cobra", "viper", "sidewinder"],
+            columns=["max_speed", "shield"],
+        )
+        kdf = ps.DataFrame(pdf)
+        another_kdf = ps.DataFrame(pdf)
+
+        kdf.iloc[[0, 1, 2], 1] = -another_kdf.max_speed
+        pdf.iloc[[0, 1, 2], 1] = -pdf.max_speed
+        self.assert_eq(kdf, pdf)
+
+        with self.assertRaisesRegex(
+            ValueError, "shape mismatch",
+        ):
+            kdf.iloc[[1, 2], [1]] = -another_kdf.max_speed
+
+        kdf.iloc[[0, 1, 2], 1] = 10 * another_kdf.max_speed
+        pdf.iloc[[0, 1, 2], 1] = 10 * pdf.max_speed
+        self.assert_eq(kdf, pdf)
+
+        with self.assertRaisesRegex(ValueError, "shape mismatch"):
+            kdf.iloc[[0], 1] = 10 * another_kdf.max_speed
+
+    def test_series_loc_setitem(self):
+        pdf = pd.DataFrame({"x": [1, 2, 3], "y": [4, 5, 6]}, index=["cobra", "viper", "sidewinder"])
+        kdf = ps.from_pandas(pdf)
+        pser = pdf.x
+        psery = pdf.y
+        kser = kdf.x
+        ksery = kdf.y
+
+        pser_another = pd.Series([1, 2, 3], index=["cobra", "viper", "sidewinder"])
+        kser_another = ps.from_pandas(pser_another)
+
+        kser.loc[kser % 2 == 1] = -kser_another
+        pser.loc[pser % 2 == 1] = -pser_another
+        self.assert_eq(kser, pser)
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(ksery, psery)
+
+        pdf = pd.DataFrame({"x": [1, 2, 3], "y": [4, 5, 6]}, index=["cobra", "viper", "sidewinder"])
+        kdf = ps.from_pandas(pdf)
+        pser = pdf.x
+        psery = pdf.y
+        kser = kdf.x
+        ksery = kdf.y
+        kser.loc[kser_another % 2 == 1] = -kser
+        pser.loc[pser_another % 2 == 1] = -pser
+        self.assert_eq(kser, pser)
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(ksery, psery)
+
+        pdf = pd.DataFrame({"x": [1, 2, 3], "y": [4, 5, 6]}, index=["cobra", "viper", "sidewinder"])
+        kdf = ps.from_pandas(pdf)
+        pser = pdf.x
+        psery = pdf.y
+        kser = kdf.x
+        ksery = kdf.y
+        kser.loc[kser_another % 2 == 1] = -kser
+        pser.loc[pser_another % 2 == 1] = -pser
+        self.assert_eq(kser, pser)
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(ksery, psery)
+
+        pdf = pd.DataFrame({"x": [1, 2, 3], "y": [4, 5, 6]}, index=["cobra", "viper", "sidewinder"])
+        kdf = ps.from_pandas(pdf)
+        pser = pdf.x
+        psery = pdf.y
+        kser = kdf.x
+        ksery = kdf.y
+        kser.loc[kser_another % 2 == 1] = -kser_another
+        pser.loc[pser_another % 2 == 1] = -pser_another
+        self.assert_eq(kser, pser)
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(ksery, psery)
+
+        pdf = pd.DataFrame({"x": [1, 2, 3], "y": [4, 5, 6]}, index=["cobra", "viper", "sidewinder"])
+        kdf = ps.from_pandas(pdf)
+        pser = pdf.x
+        psery = pdf.y
+        kser = kdf.x
+        ksery = kdf.y
+        kser.loc[["viper", "sidewinder"]] = -kser_another
+        pser.loc[["viper", "sidewinder"]] = -pser_another
+        self.assert_eq(kser, pser)
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(ksery, psery)
+
+        pdf = pd.DataFrame({"x": [1, 2, 3], "y": [4, 5, 6]}, index=["cobra", "viper", "sidewinder"])
+        kdf = ps.from_pandas(pdf)
+        pser = pdf.x
+        psery = pdf.y
+        kser = kdf.x
+        ksery = kdf.y
+        kser.loc[kser_another % 2 == 1] = 10
+        pser.loc[pser_another % 2 == 1] = 10
+        self.assert_eq(kser, pser)
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(ksery, psery)
+
+    def test_series_iloc_setitem(self):
+        pdf = pd.DataFrame({"x": [1, 2, 3], "y": [4, 5, 6]}, index=["cobra", "viper", "sidewinder"])
+        kdf = ps.from_pandas(pdf)
+
+        pser = pdf.x
+        psery = pdf.y
+        kser = kdf.x
+        ksery = kdf.y
+
+        pser1 = pser + 1
+        kser1 = kser + 1
+
+        pser_another = pd.Series([1, 2, 3], index=["cobra", "viper", "sidewinder"])
+        kser_another = ps.from_pandas(pser_another)
+
+        kser.iloc[[0, 1, 2]] = -kser_another
+        pser.iloc[[0, 1, 2]] = -pser_another
+        self.assert_eq(kser, pser)
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(ksery, psery)
+
+        with self.assertRaisesRegex(
+            ValueError,
+            "cannot set using a list-like indexer with a different length than the value",
+        ):
+            kser.iloc[[1, 2]] = -kser_another
+
+        kser.iloc[[0, 1, 2]] = 10 * kser_another
+        pser.iloc[[0, 1, 2]] = 10 * pser_another
+        self.assert_eq(kser, pser)
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(ksery, psery)
+
+        with self.assertRaisesRegex(
+            ValueError,
+            "cannot set using a list-like indexer with a different length than the value",
+        ):
+            kser.iloc[[0]] = 10 * kser_another
+
+        kser1.iloc[[0, 1, 2]] = -kser_another
+        pser1.iloc[[0, 1, 2]] = -pser_another
+        self.assert_eq(kser1, pser1)
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(ksery, psery)
+
+        with self.assertRaisesRegex(
+            ValueError,
+            "cannot set using a list-like indexer with a different length than the value",
+        ):
+            kser1.iloc[[1, 2]] = -kser_another
+
+        pdf = pd.DataFrame({"x": [1, 2, 3], "y": [4, 5, 6]}, index=["cobra", "viper", "sidewinder"])
+        kdf = ps.from_pandas(pdf)
+
+        pser = pdf.x
+        psery = pdf.y
+        kser = kdf.x
+        ksery = kdf.y
+
+        piloc = pser.iloc
+        kiloc = kser.iloc
+
+        kiloc[[0, 1, 2]] = -kser_another
+        piloc[[0, 1, 2]] = -pser_another
+        self.assert_eq(kser, pser)
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(ksery, psery)
+
+        with self.assertRaisesRegex(
+            ValueError,
+            "cannot set using a list-like indexer with a different length than the value",
+        ):
+            kiloc[[1, 2]] = -kser_another
+
+        kiloc[[0, 1, 2]] = 10 * kser_another
+        piloc[[0, 1, 2]] = 10 * pser_another
+        self.assert_eq(kser, pser)
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(ksery, psery)
+
+        with self.assertRaisesRegex(
+            ValueError,
+            "cannot set using a list-like indexer with a different length than the value",
+        ):
+            kiloc[[0]] = 10 * kser_another
+
+    def test_update(self):
+        pdf = pd.DataFrame({"x": [1, 2, 3], "y": [10, 20, 30]})
+        kdf = ps.from_pandas(pdf)
+
+        pser = pdf.x
+        kser = kdf.x
+        pser.update(pd.Series([4, 5, 6]))
+        kser.update(ps.Series([4, 5, 6]))
+        self.assert_eq(kser.sort_index(), pser.sort_index())
+        self.assert_eq(kdf.sort_index(), pdf.sort_index())
+
+    def test_where(self):
+        pdf1 = pd.DataFrame({"A": [0, 1, 2, 3, 4], "B": [100, 200, 300, 400, 500]})
+        pdf2 = pd.DataFrame({"A": [0, -1, -2, -3, -4], "B": [-100, -200, -300, -400, -500]})
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        self.assert_eq(pdf1.where(pdf2 > 100), kdf1.where(kdf2 > 100).sort_index())
+
+        pdf1 = pd.DataFrame({"A": [-1, -2, -3, -4, -5], "B": [-100, -200, -300, -400, -500]})
+        pdf2 = pd.DataFrame({"A": [-10, -20, -30, -40, -50], "B": [-5, -4, -3, -2, -1]})
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        self.assert_eq(pdf1.where(pdf2 < -250), kdf1.where(kdf2 < -250).sort_index())
+
+        # multi-index columns
+        pdf1 = pd.DataFrame({("X", "A"): [0, 1, 2, 3, 4], ("X", "B"): [100, 200, 300, 400, 500]})
+        pdf2 = pd.DataFrame(
+            {("X", "A"): [0, -1, -2, -3, -4], ("X", "B"): [-100, -200, -300, -400, -500]}
+        )
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        self.assert_eq(pdf1.where(pdf2 > 100), kdf1.where(kdf2 > 100).sort_index())
+
+    def test_mask(self):
+        pdf1 = pd.DataFrame({"A": [0, 1, 2, 3, 4], "B": [100, 200, 300, 400, 500]})
+        pdf2 = pd.DataFrame({"A": [0, -1, -2, -3, -4], "B": [-100, -200, -300, -400, -500]})
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        self.assert_eq(pdf1.mask(pdf2 < 100), kdf1.mask(kdf2 < 100).sort_index())
+
+        pdf1 = pd.DataFrame({"A": [-1, -2, -3, -4, -5], "B": [-100, -200, -300, -400, -500]})
+        pdf2 = pd.DataFrame({"A": [-10, -20, -30, -40, -50], "B": [-5, -4, -3, -2, -1]})
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        self.assert_eq(pdf1.mask(pdf2 > -250), kdf1.mask(kdf2 > -250).sort_index())
+
+        # multi-index columns
+        pdf1 = pd.DataFrame({("X", "A"): [0, 1, 2, 3, 4], ("X", "B"): [100, 200, 300, 400, 500]})
+        pdf2 = pd.DataFrame(
+            {("X", "A"): [0, -1, -2, -3, -4], ("X", "B"): [-100, -200, -300, -400, -500]}
+        )
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        self.assert_eq(pdf1.mask(pdf2 < 100), kdf1.mask(kdf2 < 100).sort_index())
+
+    def test_multi_index_column_assignment_frame(self):
+        pdf = pd.DataFrame({"a": [1, 2, 3, 2], "b": [4.0, 2.0, 3.0, 1.0]})
+        pdf.columns = pd.MultiIndex.from_tuples([("a", "x"), ("a", "y")])
+        kdf = ps.DataFrame(pdf)
+
+        kdf["c"] = ps.Series([10, 20, 30, 20])
+        pdf["c"] = pd.Series([10, 20, 30, 20])
+
+        kdf[("d", "x")] = ps.Series([100, 200, 300, 200], name="1")
+        pdf[("d", "x")] = pd.Series([100, 200, 300, 200], name="1")
+
+        kdf[("d", "y")] = ps.Series([1000, 2000, 3000, 2000], name=("1", "2"))
+        pdf[("d", "y")] = pd.Series([1000, 2000, 3000, 2000], name=("1", "2"))
+
+        kdf["e"] = ps.Series([10000, 20000, 30000, 20000], name=("1", "2", "3"))
+        pdf["e"] = pd.Series([10000, 20000, 30000, 20000], name=("1", "2", "3"))
+
+        kdf[[("f", "x"), ("f", "y")]] = ps.DataFrame(
+            {"1": [100000, 200000, 300000, 200000], "2": [1000000, 2000000, 3000000, 2000000]}
+        )
+        pdf[[("f", "x"), ("f", "y")]] = pd.DataFrame(
+            {"1": [100000, 200000, 300000, 200000], "2": [1000000, 2000000, 3000000, 2000000]}
+        )
+
+        self.assert_eq(repr(kdf.sort_index()), repr(pdf))
+
+        with self.assertRaisesRegex(KeyError, "Key length \\(3\\) exceeds index depth \\(2\\)"):
+            kdf[("1", "2", "3")] = ps.Series([100, 200, 300, 200])
+
+    def test_series_dot(self):
+        pser = pd.Series([90, 91, 85], index=[2, 4, 1])
+        kser = ps.from_pandas(pser)
+        pser_other = pd.Series([90, 91, 85], index=[2, 4, 1])
+        kser_other = ps.from_pandas(pser_other)
+
+        self.assert_eq(kser.dot(kser_other), pser.dot(pser_other))
+
+        kser_other = ps.Series([90, 91, 85], index=[1, 2, 4])
+        pser_other = pd.Series([90, 91, 85], index=[1, 2, 4])
+
+        self.assert_eq(kser.dot(kser_other), pser.dot(pser_other))
+
+        # length of index is different
+        kser_other = ps.Series([90, 91, 85, 100], index=[2, 4, 1, 0])
+        with self.assertRaisesRegex(ValueError, "matrices are not aligned"):
+            kser.dot(kser_other)
+
+        # for MultiIndex
+        midx = pd.MultiIndex(
+            [["lama", "cow", "falcon"], ["speed", "weight", "length"]],
+            [[0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 1, 2, 0, 1, 2, 0, 1, 2]],
+        )
+        pser = pd.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], index=midx)
+        kser = ps.from_pandas(pser)
+        pser_other = pd.Series([-450, 20, 12, -30, -250, 15, -320, 100, 3], index=midx)
+        kser_other = ps.from_pandas(pser_other)
+        self.assert_eq(kser.dot(kser_other), pser.dot(pser_other))
+
+        pser = pd.Series([0, 1, 2, 3])
+        kser = ps.from_pandas(pser)
+
+        # DataFrame "other" without Index/MultiIndex as columns
+        pdf = pd.DataFrame([[0, 1], [-2, 3], [4, -5], [6, 7]])
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kser.dot(kdf), pser.dot(pdf))
+
+        # DataFrame "other" with Index as columns
+        pdf.columns = pd.Index(["x", "y"])
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kser.dot(kdf), pser.dot(pdf))
+        pdf.columns = pd.Index(["x", "y"], name="cols_name")
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kser.dot(kdf), pser.dot(pdf))
+
+        pdf = pdf.reindex([1, 0, 2, 3])
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kser.dot(kdf), pser.dot(pdf))
+
+        # DataFrame "other" with MultiIndex as columns
+        pdf.columns = pd.MultiIndex.from_tuples([("a", "x"), ("b", "y")])
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kser.dot(kdf), pser.dot(pdf))
+        pdf.columns = pd.MultiIndex.from_tuples(
+            [("a", "x"), ("b", "y")], names=["cols_name1", "cols_name2"]
+        )
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kser.dot(kdf), pser.dot(pdf))
+
+        kser = ps.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}).b
+        pser = kser.to_pandas()
+        kdf = ps.DataFrame({"c": [7, 8, 9]})
+        pdf = kdf.to_pandas()
+        self.assert_eq(kser.dot(kdf), pser.dot(pdf))
+
+    def test_frame_dot(self):
+        pdf = pd.DataFrame([[0, 1, -2, -1], [1, 1, 1, 1]])
+        kdf = ps.from_pandas(pdf)
+
+        pser = pd.Series([1, 1, 2, 1])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(kdf.dot(kser), pdf.dot(pser))
+
+        # Index reorder
+        pser = pser.reindex([1, 0, 2, 3])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(kdf.dot(kser), pdf.dot(pser))
+
+        # ser with name
+        pser.name = "ser"
+        kser = ps.from_pandas(pser)
+        self.assert_eq(kdf.dot(kser), pdf.dot(pser))
+
+        # df with MultiIndex as column (ser with MultiIndex)
+        arrays = [[1, 1, 2, 2], ["red", "blue", "red", "blue"]]
+        pidx = pd.MultiIndex.from_arrays(arrays, names=("number", "color"))
+        pser = pd.Series([1, 1, 2, 1], index=pidx)
+        pdf = pd.DataFrame([[0, 1, -2, -1], [1, 1, 1, 1]], columns=pidx)
+        kdf = ps.from_pandas(pdf)
+        kser = ps.from_pandas(pser)
+        self.assert_eq(kdf.dot(kser), pdf.dot(pser))
+
+        # df with Index as column (ser with Index)
+        pidx = pd.Index([1, 2, 3, 4], name="number")
+        pser = pd.Series([1, 1, 2, 1], index=pidx)
+        pdf = pd.DataFrame([[0, 1, -2, -1], [1, 1, 1, 1]], columns=pidx)
+        kdf = ps.from_pandas(pdf)
+        kser = ps.from_pandas(pser)
+        self.assert_eq(kdf.dot(kser), pdf.dot(pser))
+
+        # df with Index
+        pdf.index = pd.Index(["x", "y"], name="char")
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kdf.dot(kser), pdf.dot(pser))
+
+        # df with MultiIndex
+        pdf.index = pd.MultiIndex.from_arrays([[1, 1], ["red", "blue"]], names=("number", "color"))
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kdf.dot(kser), pdf.dot(pser))
+
+        pdf = pd.DataFrame([[1, 2], [3, 4]])
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kdf.dot(kdf[0]), pdf.dot(pdf[0]))
+        self.assert_eq(kdf.dot(kdf[0] * 10), pdf.dot(pdf[0] * 10))
+        self.assert_eq((kdf + 1).dot(kdf[0] * 10), (pdf + 1).dot(pdf[0] * 10))
+
+    def test_to_series_comparison(self):
+        kidx1 = ps.Index([1, 2, 3, 4, 5])
+        kidx2 = ps.Index([1, 2, 3, 4, 5])
+
+        self.assert_eq((kidx1.to_series() == kidx2.to_series()).all(), True)
+
+        kidx1.name = "koalas"
+        kidx2.name = "koalas"
+
+        self.assert_eq((kidx1.to_series() == kidx2.to_series()).all(), True)
+
+    def test_series_repeat(self):
+        pser1 = pd.Series(["a", "b", "c"], name="a")
+        pser2 = pd.Series([10, 20, 30], name="rep")
+        kser1 = ps.from_pandas(pser1)
+        kser2 = ps.from_pandas(pser2)
+
+        self.assert_eq(kser1.repeat(kser2).sort_index(), pser1.repeat(pser2).sort_index())
+
+    def test_series_ops(self):
+        pser1 = pd.Series([1, 2, 3, 4, 5, 6, 7], name="x", index=[11, 12, 13, 14, 15, 16, 17])
+        pser2 = pd.Series([1, 2, 3, 4, 5, 6, 7], name="x", index=[11, 12, 13, 14, 15, 16, 17])
+        pidx1 = pd.Index([10, 11, 12, 13, 14, 15, 16], name="x")
+        kser1 = ps.from_pandas(pser1)
+        kser2 = ps.from_pandas(pser2)
+        kidx1 = ps.from_pandas(pidx1)
+
+        self.assert_eq((kser1 + 1 + 10 * kser2).sort_index(), (pser1 + 1 + 10 * pser2).sort_index())
+        self.assert_eq(
+            (kser1 + 1 + 10 * kser2.rename()).sort_index(),
+            (pser1 + 1 + 10 * pser2.rename()).sort_index(),
+        )
+        self.assert_eq(
+            (kser1.rename() + 1 + 10 * kser2).sort_index(),
+            (pser1.rename() + 1 + 10 * pser2).sort_index(),
+        )
+        self.assert_eq(
+            (kser1.rename() + 1 + 10 * kser2.rename()).sort_index(),
+            (pser1.rename() + 1 + 10 * pser2.rename()).sort_index(),
+        )
+
+        self.assert_eq(kser1 + 1 + 10 * kidx1, pser1 + 1 + 10 * pidx1)
+        self.assert_eq(kser1.rename() + 1 + 10 * kidx1, pser1.rename() + 1 + 10 * pidx1)
+        self.assert_eq(kser1 + 1 + 10 * kidx1.rename(None), pser1 + 1 + 10 * pidx1.rename(None))
+        self.assert_eq(
+            kser1.rename() + 1 + 10 * kidx1.rename(None),
+            pser1.rename() + 1 + 10 * pidx1.rename(None),
+        )
+
+        self.assert_eq(kidx1 + 1 + 10 * kser1, pidx1 + 1 + 10 * pser1)
+        self.assert_eq(kidx1 + 1 + 10 * kser1.rename(), pidx1 + 1 + 10 * pser1.rename())
+        self.assert_eq(kidx1.rename(None) + 1 + 10 * kser1, pidx1.rename(None) + 1 + 10 * pser1)
+        self.assert_eq(
+            kidx1.rename(None) + 1 + 10 * kser1.rename(),
+            pidx1.rename(None) + 1 + 10 * pser1.rename(),
+        )
+
+        pidx2 = pd.Index([11, 12, 13])
+        kidx2 = ps.from_pandas(pidx2)
+
+        with self.assertRaisesRegex(
+            ValueError, "operands could not be broadcast together with shapes"
+        ):
+            kser1 + kidx2
+
+        with self.assertRaisesRegex(
+            ValueError, "operands could not be broadcast together with shapes"
+        ):
+            kidx2 + kser1
+
+    def test_index_ops(self):
+        pidx1 = pd.Index([1, 2, 3, 4, 5], name="x")
+        pidx2 = pd.Index([6, 7, 8, 9, 10], name="x")
+        kidx1 = ps.from_pandas(pidx1)
+        kidx2 = ps.from_pandas(pidx2)
+
+        self.assert_eq(kidx1 * 10 + kidx2, pidx1 * 10 + pidx2)
+        self.assert_eq(kidx1.rename(None) * 10 + kidx2, pidx1.rename(None) * 10 + pidx2)
+
+        if LooseVersion(pd.__version__) >= LooseVersion("1.0"):
+            self.assert_eq(kidx1 * 10 + kidx2.rename(None), pidx1 * 10 + pidx2.rename(None))
+        else:
+            self.assert_eq(
+                kidx1 * 10 + kidx2.rename(None), (pidx1 * 10 + pidx2.rename(None)).rename(None)
+            )
+
+        pidx3 = pd.Index([11, 12, 13])
+        kidx3 = ps.from_pandas(pidx3)
+
+        with self.assertRaisesRegex(
+            ValueError, "operands could not be broadcast together with shapes"
+        ):
+            kidx1 + kidx3
+
+        pidx1 = pd.Index([1, 2, 3, 4, 5], name="a")
+        pidx2 = pd.Index([6, 7, 8, 9, 10], name="a")
+        pidx3 = pd.Index([11, 12, 13, 14, 15], name="x")
+        kidx1 = ps.from_pandas(pidx1)
+        kidx2 = ps.from_pandas(pidx2)
+        kidx3 = ps.from_pandas(pidx3)
+
+        self.assert_eq(kidx1 * 10 + kidx2, pidx1 * 10 + pidx2)
+
+        if LooseVersion(pd.__version__) >= LooseVersion("1.0"):
+            self.assert_eq(kidx1 * 10 + kidx3, pidx1 * 10 + pidx3)
+        else:
+            self.assert_eq(kidx1 * 10 + kidx3, (pidx1 * 10 + pidx3).rename(None))
+
+    def test_align(self):
+        pdf1 = pd.DataFrame({"a": [1, 2, 3], "b": ["a", "b", "c"]}, index=[10, 20, 30])
+        pdf2 = pd.DataFrame({"a": [4, 5, 6], "c": ["d", "e", "f"]}, index=[10, 11, 12])
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        for join in ["outer", "inner", "left", "right"]:
+            for axis in [None, 0]:
+                kdf_l, kdf_r = kdf1.align(kdf2, join=join, axis=axis)
+                pdf_l, pdf_r = pdf1.align(pdf2, join=join, axis=axis)
+                self.assert_eq(kdf_l.sort_index(), pdf_l.sort_index())
+                self.assert_eq(kdf_r.sort_index(), pdf_r.sort_index())
+
+        pser1 = pd.Series([7, 8, 9], index=[10, 11, 12])
+        pser2 = pd.Series(["g", "h", "i"], index=[10, 20, 30])
+        kser1 = ps.from_pandas(pser1)
+        kser2 = ps.from_pandas(pser2)
+
+        for join in ["outer", "inner", "left", "right"]:
+            kser_l, kser_r = kser1.align(kser2, join=join)
+            pser_l, pser_r = pser1.align(pser2, join=join)
+            self.assert_eq(kser_l.sort_index(), pser_l.sort_index())
+            self.assert_eq(kser_r.sort_index(), pser_r.sort_index())
+
+            kdf_l, kser_r = kdf1.align(kser1, join=join, axis=0)
+            pdf_l, pser_r = pdf1.align(pser1, join=join, axis=0)
+            self.assert_eq(kdf_l.sort_index(), pdf_l.sort_index())
+            self.assert_eq(kser_r.sort_index(), pser_r.sort_index())
+
+            kser_l, kdf_r = kser1.align(kdf1, join=join)
+            pser_l, pdf_r = pser1.align(pdf1, join=join)
+            self.assert_eq(kser_l.sort_index(), pser_l.sort_index())
+            self.assert_eq(kdf_r.sort_index(), pdf_r.sort_index())
+
+        # multi-index columns
+        pdf3 = pd.DataFrame(
+            {("x", "a"): [4, 5, 6], ("y", "c"): ["d", "e", "f"]}, index=[10, 11, 12]
+        )
+        kdf3 = ps.from_pandas(pdf3)
+        pser3 = pdf3[("y", "c")]
+        kser3 = kdf3[("y", "c")]
+
+        for join in ["outer", "inner", "left", "right"]:
+            kdf_l, kdf_r = kdf1.align(kdf3, join=join, axis=0)
+            pdf_l, pdf_r = pdf1.align(pdf3, join=join, axis=0)
+            self.assert_eq(kdf_l.sort_index(), pdf_l.sort_index())
+            self.assert_eq(kdf_r.sort_index(), pdf_r.sort_index())
+
+            kser_l, kser_r = kser1.align(kser3, join=join)
+            pser_l, pser_r = pser1.align(pser3, join=join)
+            self.assert_eq(kser_l.sort_index(), pser_l.sort_index())
+            self.assert_eq(kser_r.sort_index(), pser_r.sort_index())
+
+            kdf_l, kser_r = kdf1.align(kser3, join=join, axis=0)
+            pdf_l, pser_r = pdf1.align(pser3, join=join, axis=0)
+            self.assert_eq(kdf_l.sort_index(), pdf_l.sort_index())
+            self.assert_eq(kser_r.sort_index(), pser_r.sort_index())
+
+            kser_l, kdf_r = kser3.align(kdf1, join=join)
+            pser_l, pdf_r = pser3.align(pdf1, join=join)
+            self.assert_eq(kser_l.sort_index(), pser_l.sort_index())
+            self.assert_eq(kdf_r.sort_index(), pdf_r.sort_index())
+
+        self.assertRaises(ValueError, lambda: kdf1.align(kdf3, axis=None))
+        self.assertRaises(ValueError, lambda: kdf1.align(kdf3, axis=1))
+
+    def test_pow_and_rpow(self):
+        pser = pd.Series([1, 2, np.nan])
+        kser = ps.from_pandas(pser)
+        pser_other = pd.Series([np.nan, 2, 3])
+        kser_other = ps.from_pandas(pser_other)
+
+        self.assert_eq(pser.pow(pser_other), kser.pow(kser_other).sort_index())
+        self.assert_eq(pser ** pser_other, (kser ** kser_other).sort_index())
+        self.assert_eq(pser.rpow(pser_other), kser.rpow(kser_other).sort_index())
+
+    def test_shift(self):
+        pdf = pd.DataFrame(
+            {
+                "Col1": [10, 20, 15, 30, 45],
+                "Col2": [13, 23, 18, 33, 48],
+                "Col3": [17, 27, 22, 37, 52],
+            },
+            index=np.random.rand(5),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            pdf.shift().loc[pdf["Col1"] == 20].astype(int), kdf.shift().loc[kdf["Col1"] == 20]
+        )
+        self.assert_eq(
+            pdf["Col2"].shift().loc[pdf["Col1"] == 20].astype(int),
+            kdf["Col2"].shift().loc[kdf["Col1"] == 20],
+        )
+
+    def test_diff(self):
+        pdf = pd.DataFrame(
+            {
+                "Col1": [10, 20, 15, 30, 45],
+                "Col2": [13, 23, 18, 33, 48],
+                "Col3": [17, 27, 22, 37, 52],
+            },
+            index=np.random.rand(5),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            pdf.diff().loc[pdf["Col1"] == 20].astype(int), kdf.diff().loc[kdf["Col1"] == 20]
+        )
+        self.assert_eq(
+            pdf["Col2"].diff().loc[pdf["Col1"] == 20].astype(int),
+            kdf["Col2"].diff().loc[kdf["Col1"] == 20],
+        )
+
+    def test_rank(self):
+        pdf = pd.DataFrame(
+            {
+                "Col1": [10, 20, 15, 30, 45],
+                "Col2": [13, 23, 18, 33, 48],
+                "Col3": [17, 27, 22, 37, 52],
+            },
+            index=np.random.rand(5),
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(pdf.rank().loc[pdf["Col1"] == 20], kdf.rank().loc[kdf["Col1"] == 20])
+        self.assert_eq(
+            pdf["Col2"].rank().loc[pdf["Col1"] == 20], kdf["Col2"].rank().loc[kdf["Col1"] == 20]
+        )
+
+
+class OpsOnDiffFramesDisabledTest(PandasOnSparkTestCase, SQLTestUtils):
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        set_option("compute.ops_on_diff_frames", False)
+
+    @classmethod
+    def tearDownClass(cls):
+        reset_option("compute.ops_on_diff_frames")
+        super().tearDownClass()
+
+    @property
+    def pdf1(self):
+        return pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5, 6, 7, 8, 9], "b": [4, 5, 6, 3, 2, 1, 0, 0, 0]},
+            index=[0, 1, 3, 5, 6, 8, 9, 9, 9],
+        )
+
+    @property
+    def pdf2(self):
+        return pd.DataFrame(
+            {"a": [9, 8, 7, 6, 5, 4, 3, 2, 1], "b": [0, 0, 0, 4, 5, 6, 1, 2, 3]},
+            index=list(range(9)),
+        )
+
+    @property
+    def kdf1(self):
+        return ps.from_pandas(self.pdf1)
+
+    @property
+    def kdf2(self):
+        return ps.from_pandas(self.pdf2)
+
+    def test_arithmetic(self):
+        with self.assertRaisesRegex(ValueError, "Cannot combine the series or dataframe"):
+            self.kdf1.a - self.kdf2.b
+
+        with self.assertRaisesRegex(ValueError, "Cannot combine the series or dataframe"):
+            self.kdf1.a - self.kdf2.a
+
+        with self.assertRaisesRegex(ValueError, "Cannot combine the series or dataframe"):
+            self.kdf1["a"] - self.kdf2["a"]
+
+        with self.assertRaisesRegex(ValueError, "Cannot combine the series or dataframe"):
+            self.kdf1 - self.kdf2
+
+    def test_assignment(self):
+        with self.assertRaisesRegex(ValueError, "Cannot combine the series or dataframe"):
+            kdf = ps.from_pandas(self.pdf1)
+            kdf["c"] = self.kdf1.a
+
+    def test_frame_loc_setitem(self):
+        pdf = pd.DataFrame(
+            [[1, 2], [4, 5], [7, 8]],
+            index=["cobra", "viper", "sidewinder"],
+            columns=["max_speed", "shield"],
+        )
+        kdf = ps.DataFrame(pdf)
+        another_kdf = ps.DataFrame(pdf)
+
+        with self.assertRaisesRegex(ValueError, "Cannot combine the series or dataframe"):
+            kdf.loc[["viper", "sidewinder"], ["shield"]] = another_kdf.max_speed
+
+        with self.assertRaisesRegex(ValueError, "Cannot combine the series or dataframe"):
+            kdf.loc[another_kdf.max_speed < 5, ["shield"]] = -kdf.max_speed
+
+        with self.assertRaisesRegex(ValueError, "Cannot combine the series or dataframe"):
+            kdf.loc[another_kdf.max_speed < 5, ["shield"]] = -another_kdf.max_speed
+
+    def test_frame_iloc_setitem(self):
+        pdf = pd.DataFrame(
+            [[1, 2], [4, 5], [7, 8]],
+            index=["cobra", "viper", "sidewinder"],
+            columns=["max_speed", "shield"],
+        )
+        kdf = ps.DataFrame(pdf)
+        another_kdf = ps.DataFrame(pdf)
+
+        with self.assertRaisesRegex(ValueError, "Cannot combine the series or dataframe"):
+            kdf.iloc[[1, 2], [1]] = another_kdf.max_speed.iloc[[1, 2]]
+
+    def test_series_loc_setitem(self):
+        pser = pd.Series([1, 2, 3], index=["cobra", "viper", "sidewinder"])
+        kser = ps.from_pandas(pser)
+
+        pser_another = pd.Series([1, 2, 3], index=["cobra", "viper", "sidewinder"])
+        kser_another = ps.from_pandas(pser_another)
+
+        with self.assertRaisesRegex(ValueError, "Cannot combine the series or dataframe"):
+            kser.loc[kser % 2 == 1] = -kser_another
+
+        with self.assertRaisesRegex(ValueError, "Cannot combine the series or dataframe"):
+            kser.loc[kser_another % 2 == 1] = -kser
+
+        with self.assertRaisesRegex(ValueError, "Cannot combine the series or dataframe"):
+            kser.loc[kser_another % 2 == 1] = -kser_another
+
+    def test_series_iloc_setitem(self):
+        pser = pd.Series([1, 2, 3], index=["cobra", "viper", "sidewinder"])
+        kser = ps.from_pandas(pser)
+
+        pser_another = pd.Series([1, 2, 3], index=["cobra", "viper", "sidewinder"])
+        kser_another = ps.from_pandas(pser_another)
+
+        with self.assertRaisesRegex(ValueError, "Cannot combine the series or dataframe"):
+            kser.iloc[[1]] = -kser_another.iloc[[1]]
+
+    def test_where(self):
+        pdf1 = pd.DataFrame({"A": [0, 1, 2, 3, 4], "B": [100, 200, 300, 400, 500]})
+        pdf2 = pd.DataFrame({"A": [0, -1, -2, -3, -4], "B": [-100, -200, -300, -400, -500]})
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        with self.assertRaisesRegex(ValueError, "Cannot combine the series or dataframe"):
+            kdf1.where(kdf2 > 100)
+
+        pdf1 = pd.DataFrame({"A": [-1, -2, -3, -4, -5], "B": [-100, -200, -300, -400, -500]})
+        pdf2 = pd.DataFrame({"A": [-10, -20, -30, -40, -50], "B": [-5, -4, -3, -2, -1]})
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        with self.assertRaisesRegex(ValueError, "Cannot combine the series or dataframe"):
+            kdf1.where(kdf2 < -250)
+
+    def test_mask(self):
+        pdf1 = pd.DataFrame({"A": [0, 1, 2, 3, 4], "B": [100, 200, 300, 400, 500]})
+        pdf2 = pd.DataFrame({"A": [0, -1, -2, -3, -4], "B": [-100, -200, -300, -400, -500]})
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        with self.assertRaisesRegex(ValueError, "Cannot combine the series or dataframe"):
+            kdf1.mask(kdf2 < 100)
+
+        pdf1 = pd.DataFrame({"A": [-1, -2, -3, -4, -5], "B": [-100, -200, -300, -400, -500]})
+        pdf2 = pd.DataFrame({"A": [-10, -20, -30, -40, -50], "B": [-5, -4, -3, -2, -1]})
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        with self.assertRaisesRegex(ValueError, "Cannot combine the series or dataframe"):
+            kdf1.mask(kdf2 > -250)
+
+    def test_align(self):
+        pdf1 = pd.DataFrame({"a": [1, 2, 3], "b": ["a", "b", "c"]}, index=[10, 20, 30])
+        pdf2 = pd.DataFrame({"a": [4, 5, 6], "c": ["d", "e", "f"]}, index=[10, 11, 12])
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        with self.assertRaisesRegex(ValueError, "Cannot combine the series or dataframe"):
+            kdf1.align(kdf2)
+
+        with self.assertRaisesRegex(ValueError, "Cannot combine the series or dataframe"):
+            kdf1.align(kdf2, axis=0)
+
+    def test_pow_and_rpow(self):
+        pser = pd.Series([1, 2, np.nan])
+        kser = ps.from_pandas(pser)
+        pser_other = pd.Series([np.nan, 2, 3])
+        kser_other = ps.from_pandas(pser_other)
+
+        with self.assertRaisesRegex(ValueError, "Cannot combine the series or dataframe"):
+            kser.pow(kser_other)
+        with self.assertRaisesRegex(ValueError, "Cannot combine the series or dataframe"):
+            kser ** kser_other
+        with self.assertRaisesRegex(ValueError, "Cannot combine the series or dataframe"):
+            kser.rpow(kser_other)
+
+
+if __name__ == "__main__":
+    from pyspark.pandas.tests.test_ops_on_diff_frames import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby.py b/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby.py
new file mode 100644
index 0000000000000..ce4653868dcd1
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby.py
@@ -0,0 +1,614 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import unittest
+
+import pandas as pd
+
+from pyspark import pandas as ps
+from pyspark.pandas.config import set_option, reset_option
+from pyspark.testing.pandasutils import PandasOnSparkTestCase
+from pyspark.testing.sqlutils import SQLTestUtils
+
+
+class OpsOnDiffFramesGroupByTest(PandasOnSparkTestCase, SQLTestUtils):
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        set_option("compute.ops_on_diff_frames", True)
+
+    @classmethod
+    def tearDownClass(cls):
+        reset_option("compute.ops_on_diff_frames")
+        super().tearDownClass()
+
+    def test_groupby_different_lengths(self):
+        pdfs1 = [
+            pd.DataFrame({"c": [4, 2, 7, 3, None, 1, 1, 1, 2], "d": list("abcdefght")}),
+            pd.DataFrame({"c": [4, 2, 7, None, 1, 1, 2], "d": list("abcdefg")}),
+            pd.DataFrame({"c": [4, 2, 7, 3, None, 1, 1, 1, 2, 2], "d": list("abcdefghti")}),
+        ]
+        pdfs2 = [
+            pd.DataFrame({"a": [1, 2, 6, 4, 4, 6, 4, 3, 7], "b": [4, 2, 7, 3, 3, 1, 1, 1, 2]}),
+            pd.DataFrame({"a": [1, 2, 6, 4, 4, 6, 4, 7], "b": [4, 2, 7, 3, 3, 1, 1, 2]}),
+            pd.DataFrame({"a": [1, 2, 6, 4, 4, 6, 4, 3, 7], "b": [4, 2, 7, 3, 3, 1, 1, 1, 2]}),
+        ]
+
+        for pdf1, pdf2 in zip(pdfs1, pdfs2):
+            kdf1 = ps.from_pandas(pdf1)
+            kdf2 = ps.from_pandas(pdf2)
+
+            for as_index in [True, False]:
+                if as_index:
+                    sort = lambda df: df.sort_index()
+                else:
+                    sort = lambda df: df.sort_values("c").reset_index(drop=True)
+                self.assert_eq(
+                    sort(kdf1.groupby(kdf2.a, as_index=as_index).sum()),
+                    sort(pdf1.groupby(pdf2.a, as_index=as_index).sum()),
+                    almost=as_index,
+                )
+
+                self.assert_eq(
+                    sort(kdf1.groupby(kdf2.a, as_index=as_index).c.sum()),
+                    sort(pdf1.groupby(pdf2.a, as_index=as_index).c.sum()),
+                    almost=as_index,
+                )
+                self.assert_eq(
+                    sort(kdf1.groupby(kdf2.a, as_index=as_index)["c"].sum()),
+                    sort(pdf1.groupby(pdf2.a, as_index=as_index)["c"].sum()),
+                    almost=as_index,
+                )
+
+    def test_groupby_multiindex_columns(self):
+        pdf1 = pd.DataFrame(
+            {("y", "c"): [4, 2, 7, 3, None, 1, 1, 1, 2], ("z", "d"): list("abcdefght")}
+        )
+        pdf2 = pd.DataFrame(
+            {("x", "a"): [1, 2, 6, 4, 4, 6, 4, 3, 7], ("x", "b"): [4, 2, 7, 3, 3, 1, 1, 1, 2]}
+        )
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        self.assert_eq(
+            kdf1.groupby(kdf2[("x", "a")]).sum().sort_index(),
+            pdf1.groupby(pdf2[("x", "a")]).sum().sort_index(),
+        )
+
+        self.assert_eq(
+            kdf1.groupby(kdf2[("x", "a")], as_index=False)
+            .sum()
+            .sort_values(("y", "c"))
+            .reset_index(drop=True),
+            pdf1.groupby(pdf2[("x", "a")], as_index=False)
+            .sum()
+            .sort_values(("y", "c"))
+            .reset_index(drop=True),
+        )
+        self.assert_eq(
+            kdf1.groupby(kdf2[("x", "a")])[[("y", "c")]].sum().sort_index(),
+            pdf1.groupby(pdf2[("x", "a")])[[("y", "c")]].sum().sort_index(),
+        )
+
+    def test_split_apply_combine_on_series(self):
+        pdf1 = pd.DataFrame({"C": [0.362, 0.227, 1.267, -0.562], "B": [1, 2, 3, 4]})
+        pdf2 = pd.DataFrame({"A": [1, 1, 2, 2]})
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        for as_index in [True, False]:
+            if as_index:
+                sort = lambda df: df.sort_index()
+            else:
+                sort = lambda df: df.sort_values(list(df.columns)).reset_index(drop=True)
+
+            with self.subTest(as_index=as_index):
+                self.assert_eq(
+                    sort(kdf1.groupby(kdf2.A, as_index=as_index).sum()),
+                    sort(pdf1.groupby(pdf2.A, as_index=as_index).sum()),
+                )
+                self.assert_eq(
+                    sort(kdf1.groupby(kdf2.A, as_index=as_index).B.sum()),
+                    sort(pdf1.groupby(pdf2.A, as_index=as_index).B.sum()),
+                )
+                self.assert_eq(
+                    sort(kdf1.groupby([kdf1.C, kdf2.A], as_index=as_index).sum()),
+                    sort(pdf1.groupby([pdf1.C, pdf2.A], as_index=as_index).sum()),
+                )
+                self.assert_eq(
+                    sort(kdf1.groupby([kdf1.C + 1, kdf2.A], as_index=as_index).sum()),
+                    sort(pdf1.groupby([pdf1.C + 1, pdf2.A], as_index=as_index).sum()),
+                )
+
+        self.assert_eq(
+            kdf1.B.groupby(kdf2.A).sum().sort_index(), pdf1.B.groupby(pdf2.A).sum().sort_index(),
+        )
+        self.assert_eq(
+            (kdf1.B + 1).groupby(kdf2.A).sum().sort_index(),
+            (pdf1.B + 1).groupby(pdf2.A).sum().sort_index(),
+        )
+
+        self.assert_eq(
+            kdf1.B.groupby(kdf2.A.rename()).sum().sort_index(),
+            pdf1.B.groupby(pdf2.A.rename()).sum().sort_index(),
+        )
+        self.assert_eq(
+            kdf1.B.rename().groupby(kdf2.A).sum().sort_index(),
+            pdf1.B.rename().groupby(pdf2.A).sum().sort_index(),
+        )
+        self.assert_eq(
+            kdf1.B.rename().groupby(kdf2.A.rename()).sum().sort_index(),
+            pdf1.B.rename().groupby(pdf2.A.rename()).sum().sort_index(),
+        )
+
+    def test_aggregate(self):
+        pdf1 = pd.DataFrame({"C": [0.362, 0.227, 1.267, -0.562], "B": [1, 2, 3, 4]})
+        pdf2 = pd.DataFrame({"A": [1, 1, 2, 2]})
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        for as_index in [True, False]:
+            if as_index:
+                sort = lambda df: df.sort_index()
+            else:
+                sort = lambda df: df.sort_values(list(df.columns)).reset_index(drop=True)
+
+            with self.subTest(as_index=as_index):
+                self.assert_eq(
+                    sort(kdf1.groupby(kdf2.A, as_index=as_index).agg("sum")),
+                    sort(pdf1.groupby(pdf2.A, as_index=as_index).agg("sum")),
+                )
+                self.assert_eq(
+                    sort(kdf1.groupby(kdf2.A, as_index=as_index).agg({"B": "min", "C": "sum"})),
+                    sort(pdf1.groupby(pdf2.A, as_index=as_index).agg({"B": "min", "C": "sum"})),
+                )
+                self.assert_eq(
+                    sort(
+                        kdf1.groupby(kdf2.A, as_index=as_index).agg(
+                            {"B": ["min", "max"], "C": "sum"}
+                        )
+                    ),
+                    sort(
+                        pdf1.groupby(pdf2.A, as_index=as_index).agg(
+                            {"B": ["min", "max"], "C": "sum"}
+                        )
+                    ),
+                )
+                self.assert_eq(
+                    sort(kdf1.groupby([kdf1.C, kdf2.A], as_index=as_index).agg("sum")),
+                    sort(pdf1.groupby([pdf1.C, pdf2.A], as_index=as_index).agg("sum")),
+                )
+                self.assert_eq(
+                    sort(kdf1.groupby([kdf1.C + 1, kdf2.A], as_index=as_index).agg("sum")),
+                    sort(pdf1.groupby([pdf1.C + 1, pdf2.A], as_index=as_index).agg("sum")),
+                )
+
+        # multi-index columns
+        columns = pd.MultiIndex.from_tuples([("Y", "C"), ("X", "B")])
+        pdf1.columns = columns
+        kdf1.columns = columns
+
+        columns = pd.MultiIndex.from_tuples([("X", "A")])
+        pdf2.columns = columns
+        kdf2.columns = columns
+
+        for as_index in [True, False]:
+            stats_kdf = kdf1.groupby(kdf2[("X", "A")], as_index=as_index).agg(
+                {("X", "B"): "min", ("Y", "C"): "sum"}
+            )
+            stats_pdf = pdf1.groupby(pdf2[("X", "A")], as_index=as_index).agg(
+                {("X", "B"): "min", ("Y", "C"): "sum"}
+            )
+            self.assert_eq(
+                stats_kdf.sort_values(by=[("X", "B"), ("Y", "C")]).reset_index(drop=True),
+                stats_pdf.sort_values(by=[("X", "B"), ("Y", "C")]).reset_index(drop=True),
+            )
+
+        stats_kdf = kdf1.groupby(kdf2[("X", "A")]).agg(
+            {("X", "B"): ["min", "max"], ("Y", "C"): "sum"}
+        )
+        stats_pdf = pdf1.groupby(pdf2[("X", "A")]).agg(
+            {("X", "B"): ["min", "max"], ("Y", "C"): "sum"}
+        )
+        self.assert_eq(
+            stats_kdf.sort_values(
+                by=[("X", "B", "min"), ("X", "B", "max"), ("Y", "C", "sum")]
+            ).reset_index(drop=True),
+            stats_pdf.sort_values(
+                by=[("X", "B", "min"), ("X", "B", "max"), ("Y", "C", "sum")]
+            ).reset_index(drop=True),
+        )
+
+    def test_duplicated_labels(self):
+        pdf1 = pd.DataFrame({"A": [3, 2, 1]})
+        pdf2 = pd.DataFrame({"A": [1, 2, 3]})
+        kdf1 = ps.from_pandas(pdf1)
+        kdf2 = ps.from_pandas(pdf2)
+
+        self.assert_eq(
+            kdf1.groupby(kdf2.A).sum().sort_index(), pdf1.groupby(pdf2.A).sum().sort_index()
+        )
+        self.assert_eq(
+            kdf1.groupby(kdf2.A, as_index=False).sum().sort_values("A").reset_index(drop=True),
+            pdf1.groupby(pdf2.A, as_index=False).sum().sort_values("A").reset_index(drop=True),
+        )
+
+    def test_apply(self):
+        pdf = pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5, 6], "b": [1, 1, 2, 3, 5, 8], "c": [1, 4, 9, 16, 25, 36]},
+            columns=["a", "b", "c"],
+        )
+        pkey = pd.Series([1, 1, 2, 3, 5, 8])
+        kdf = ps.from_pandas(pdf)
+        kkey = ps.from_pandas(pkey)
+
+        self.assert_eq(
+            kdf.groupby(kkey).apply(lambda x: x + x.min()).sort_index(),
+            pdf.groupby(pkey).apply(lambda x: x + x.min()).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kkey)["a"].apply(lambda x: x + x.min()).sort_index(),
+            pdf.groupby(pkey)["a"].apply(lambda x: x + x.min()).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kkey)[["a"]].apply(lambda x: x + x.min()).sort_index(),
+            pdf.groupby(pkey)[["a"]].apply(lambda x: x + x.min()).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(["a", kkey]).apply(lambda x: x + x.min()).sort_index(),
+            pdf.groupby(["a", pkey]).apply(lambda x: x + x.min()).sort_index(),
+        )
+
+    def test_transform(self):
+        pdf = pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5, 6], "b": [1, 1, 2, 3, 5, 8], "c": [1, 4, 9, 16, 25, 36]},
+            columns=["a", "b", "c"],
+        )
+        pkey = pd.Series([1, 1, 2, 3, 5, 8])
+        kdf = ps.from_pandas(pdf)
+        kkey = ps.from_pandas(pkey)
+
+        self.assert_eq(
+            kdf.groupby(kkey).transform(lambda x: x + x.min()).sort_index(),
+            pdf.groupby(pkey).transform(lambda x: x + x.min()).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kkey)["a"].transform(lambda x: x + x.min()).sort_index(),
+            pdf.groupby(pkey)["a"].transform(lambda x: x + x.min()).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kkey)[["a"]].transform(lambda x: x + x.min()).sort_index(),
+            pdf.groupby(pkey)[["a"]].transform(lambda x: x + x.min()).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(["a", kkey]).transform(lambda x: x + x.min()).sort_index(),
+            pdf.groupby(["a", pkey]).transform(lambda x: x + x.min()).sort_index(),
+        )
+
+    def test_filter(self):
+        pdf = pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5, 6], "b": [1, 1, 2, 3, 5, 8], "c": [1, 4, 9, 16, 25, 36]},
+            columns=["a", "b", "c"],
+        )
+        pkey = pd.Series([1, 1, 2, 3, 5, 8])
+        kdf = ps.from_pandas(pdf)
+        kkey = ps.from_pandas(pkey)
+
+        self.assert_eq(
+            kdf.groupby(kkey).filter(lambda x: any(x.a == 2)).sort_index(),
+            pdf.groupby(pkey).filter(lambda x: any(x.a == 2)).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kkey)["a"].filter(lambda x: any(x == 2)).sort_index(),
+            pdf.groupby(pkey)["a"].filter(lambda x: any(x == 2)).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kkey)[["a"]].filter(lambda x: any(x.a == 2)).sort_index(),
+            pdf.groupby(pkey)[["a"]].filter(lambda x: any(x.a == 2)).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(["a", kkey]).filter(lambda x: any(x.a == 2)).sort_index(),
+            pdf.groupby(["a", pkey]).filter(lambda x: any(x.a == 2)).sort_index(),
+        )
+
+    def test_head(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 1, 1, 1, 2, 2, 2, 3, 3, 3] * 3,
+                "b": [2, 3, 1, 4, 6, 9, 8, 10, 7, 5] * 3,
+                "c": [3, 5, 2, 5, 1, 2, 6, 4, 3, 6] * 3,
+            },
+        )
+        pkey = pd.Series([1, 1, 1, 1, 2, 2, 2, 3, 3, 3] * 3)
+        kdf = ps.from_pandas(pdf)
+        kkey = ps.from_pandas(pkey)
+
+        self.assert_eq(
+            pdf.groupby(pkey).head(2).sort_index(), kdf.groupby(kkey).head(2).sort_index()
+        )
+        self.assert_eq(
+            pdf.groupby("a")["b"].head(2).sort_index(), kdf.groupby("a")["b"].head(2).sort_index()
+        )
+        self.assert_eq(
+            pdf.groupby("a")[["b"]].head(2).sort_index(),
+            kdf.groupby("a")[["b"]].head(2).sort_index(),
+        )
+        self.assert_eq(
+            pdf.groupby([pkey, "b"]).head(2).sort_index(),
+            kdf.groupby([kkey, "b"]).head(2).sort_index(),
+        )
+
+    def test_cumcount(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 2, 3, 4, 5, 6] * 3,
+                "b": [1, 1, 2, 3, 5, 8] * 3,
+                "c": [1, 4, 9, 16, 25, 36] * 3,
+            },
+        )
+        pkey = pd.Series([1, 1, 2, 3, 5, 8] * 3)
+        kdf = ps.from_pandas(pdf)
+        kkey = ps.from_pandas(pkey)
+
+        for ascending in [True, False]:
+            self.assert_eq(
+                kdf.groupby(kkey).cumcount(ascending=ascending).sort_index(),
+                pdf.groupby(pkey).cumcount(ascending=ascending).sort_index(),
+            )
+            self.assert_eq(
+                kdf.groupby(kkey)["a"].cumcount(ascending=ascending).sort_index(),
+                pdf.groupby(pkey)["a"].cumcount(ascending=ascending).sort_index(),
+            )
+            self.assert_eq(
+                kdf.groupby(kkey)[["a"]].cumcount(ascending=ascending).sort_index(),
+                pdf.groupby(pkey)[["a"]].cumcount(ascending=ascending).sort_index(),
+            )
+
+    def test_cummin(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 2, 3, 4, 5, 6] * 3,
+                "b": [1, 1, 2, 3, 5, 8] * 3,
+                "c": [1, 4, 9, 16, 25, 36] * 3,
+            },
+        )
+        pkey = pd.Series([1, 1, 2, 3, 5, 8] * 3)
+        kdf = ps.from_pandas(pdf)
+        kkey = ps.from_pandas(pkey)
+
+        self.assert_eq(
+            kdf.groupby(kkey).cummin().sort_index(), pdf.groupby(pkey).cummin().sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby(kkey)["a"].cummin().sort_index(),
+            pdf.groupby(pkey)["a"].cummin().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kkey)[["a"]].cummin().sort_index(),
+            pdf.groupby(pkey)[["a"]].cummin().sort_index(),
+        )
+
+    def test_cummax(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 2, 3, 4, 5, 6] * 3,
+                "b": [1, 1, 2, 3, 5, 8] * 3,
+                "c": [1, 4, 9, 16, 25, 36] * 3,
+            },
+        )
+        pkey = pd.Series([1, 1, 2, 3, 5, 8] * 3)
+        kdf = ps.from_pandas(pdf)
+        kkey = ps.from_pandas(pkey)
+
+        self.assert_eq(
+            kdf.groupby(kkey).cummax().sort_index(), pdf.groupby(pkey).cummax().sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby(kkey)["a"].cummax().sort_index(),
+            pdf.groupby(pkey)["a"].cummax().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kkey)[["a"]].cummax().sort_index(),
+            pdf.groupby(pkey)[["a"]].cummax().sort_index(),
+        )
+
+    def test_cumsum(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 2, 3, 4, 5, 6] * 3,
+                "b": [1, 1, 2, 3, 5, 8] * 3,
+                "c": [1, 4, 9, 16, 25, 36] * 3,
+            },
+        )
+        pkey = pd.Series([1, 1, 2, 3, 5, 8] * 3)
+        kdf = ps.from_pandas(pdf)
+        kkey = ps.from_pandas(pkey)
+
+        self.assert_eq(
+            kdf.groupby(kkey).cumsum().sort_index(), pdf.groupby(pkey).cumsum().sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby(kkey)["a"].cumsum().sort_index(),
+            pdf.groupby(pkey)["a"].cumsum().sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kkey)[["a"]].cumsum().sort_index(),
+            pdf.groupby(pkey)[["a"]].cumsum().sort_index(),
+        )
+
+    def test_cumprod(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 2, 3, 4, 5, 6] * 3,
+                "b": [1, 1, 2, 3, 5, 8] * 3,
+                "c": [1, 4, 9, 16, 25, 36] * 3,
+            },
+        )
+        pkey = pd.Series([1, 1, 2, 3, 5, 8] * 3)
+        kdf = ps.from_pandas(pdf)
+        kkey = ps.from_pandas(pkey)
+
+        self.assert_eq(
+            kdf.groupby(kkey).cumprod().sort_index(),
+            pdf.groupby(pkey).cumprod().sort_index(),
+            almost=True,
+        )
+        self.assert_eq(
+            kdf.groupby(kkey)["a"].cumprod().sort_index(),
+            pdf.groupby(pkey)["a"].cumprod().sort_index(),
+            almost=True,
+        )
+        self.assert_eq(
+            kdf.groupby(kkey)[["a"]].cumprod().sort_index(),
+            pdf.groupby(pkey)[["a"]].cumprod().sort_index(),
+            almost=True,
+        )
+
+    def test_diff(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 2, 3, 4, 5, 6] * 3,
+                "b": [1, 1, 2, 3, 5, 8] * 3,
+                "c": [1, 4, 9, 16, 25, 36] * 3,
+            }
+        )
+        pkey = pd.Series([1, 1, 2, 3, 5, 8] * 3)
+        kdf = ps.from_pandas(pdf)
+        kkey = ps.from_pandas(pkey)
+
+        self.assert_eq(kdf.groupby(kkey).diff().sort_index(), pdf.groupby(pkey).diff().sort_index())
+        self.assert_eq(
+            kdf.groupby(kkey)["a"].diff().sort_index(), pdf.groupby(pkey)["a"].diff().sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby(kkey)[["a"]].diff().sort_index(),
+            pdf.groupby(pkey)[["a"]].diff().sort_index(),
+        )
+
+        self.assert_eq(kdf.groupby(kkey).diff().sum(), pdf.groupby(pkey).diff().sum().astype(int))
+        self.assert_eq(kdf.groupby(kkey)["a"].diff().sum(), pdf.groupby(pkey)["a"].diff().sum())
+
+    def test_rank(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 2, 3, 4, 5, 6] * 3,
+                "b": [1, 1, 2, 3, 5, 8] * 3,
+                "c": [1, 4, 9, 16, 25, 36] * 3,
+            },
+        )
+        pkey = pd.Series([1, 1, 2, 3, 5, 8] * 3)
+        kdf = ps.from_pandas(pdf)
+        kkey = ps.from_pandas(pkey)
+
+        self.assert_eq(kdf.groupby(kkey).rank().sort_index(), pdf.groupby(pkey).rank().sort_index())
+        self.assert_eq(
+            kdf.groupby(kkey)["a"].rank().sort_index(), pdf.groupby(pkey)["a"].rank().sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby(kkey)[["a"]].rank().sort_index(),
+            pdf.groupby(pkey)[["a"]].rank().sort_index(),
+        )
+
+        self.assert_eq(kdf.groupby(kkey).rank().sum(), pdf.groupby(pkey).rank().sum())
+        self.assert_eq(kdf.groupby(kkey)["a"].rank().sum(), pdf.groupby(pkey)["a"].rank().sum())
+
+    @unittest.skipIf(pd.__version__ < "0.24.0", "not supported before pandas 0.24.0")
+    def test_shift(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 1, 2, 2, 3, 3] * 3,
+                "b": [1, 1, 2, 2, 3, 4] * 3,
+                "c": [1, 4, 9, 16, 25, 36] * 3,
+            },
+        )
+        pkey = pd.Series([1, 1, 2, 2, 3, 4] * 3)
+        kdf = ps.from_pandas(pdf)
+        kkey = ps.from_pandas(pkey)
+
+        self.assert_eq(
+            kdf.groupby(kkey).shift().sort_index(), pdf.groupby(pkey).shift().sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby(kkey)["a"].shift().sort_index(), pdf.groupby(pkey)["a"].shift().sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby(kkey)[["a"]].shift().sort_index(),
+            pdf.groupby(pkey)[["a"]].shift().sort_index(),
+        )
+
+        self.assert_eq(kdf.groupby(kkey).shift().sum(), pdf.groupby(pkey).shift().sum().astype(int))
+        self.assert_eq(kdf.groupby(kkey)["a"].shift().sum(), pdf.groupby(pkey)["a"].shift().sum())
+
+    def test_fillna(self):
+        pdf = pd.DataFrame(
+            {
+                "A": [1, 1, 2, 2] * 3,
+                "B": [2, 4, None, 3] * 3,
+                "C": [None, None, None, 1] * 3,
+                "D": [0, 1, 5, 4] * 3,
+            }
+        )
+        pkey = pd.Series([1, 1, 2, 2] * 3)
+        kdf = ps.from_pandas(pdf)
+        kkey = ps.from_pandas(pkey)
+
+        self.assert_eq(
+            kdf.groupby(kkey).fillna(0).sort_index(), pdf.groupby(pkey).fillna(0).sort_index()
+        )
+        self.assert_eq(
+            kdf.groupby(kkey)["C"].fillna(0).sort_index(),
+            pdf.groupby(pkey)["C"].fillna(0).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kkey)[["C"]].fillna(0).sort_index(),
+            pdf.groupby(pkey)[["C"]].fillna(0).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kkey).fillna(method="bfill").sort_index(),
+            pdf.groupby(pkey).fillna(method="bfill").sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kkey)["C"].fillna(method="bfill").sort_index(),
+            pdf.groupby(pkey)["C"].fillna(method="bfill").sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kkey)[["C"]].fillna(method="bfill").sort_index(),
+            pdf.groupby(pkey)[["C"]].fillna(method="bfill").sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kkey).fillna(method="ffill").sort_index(),
+            pdf.groupby(pkey).fillna(method="ffill").sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kkey)["C"].fillna(method="ffill").sort_index(),
+            pdf.groupby(pkey)["C"].fillna(method="ffill").sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby(kkey)[["C"]].fillna(method="ffill").sort_index(),
+            pdf.groupby(pkey)[["C"]].fillna(method="ffill").sort_index(),
+        )
+
+
+if __name__ == "__main__":
+    from pyspark.pandas.tests.test_ops_on_diff_frames_groupby import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_expanding.py b/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_expanding.py
new file mode 100644
index 0000000000000..afd81854e8874
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_expanding.py
@@ -0,0 +1,134 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from distutils.version import LooseVersion
+
+import numpy as np
+import pandas as pd
+
+from pyspark import pandas as ps
+from pyspark.pandas.config import set_option, reset_option
+from pyspark.testing.pandasutils import PandasOnSparkTestCase, TestUtils
+
+
+class OpsOnDiffFramesGroupByExpandingTest(PandasOnSparkTestCase, TestUtils):
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        set_option("compute.ops_on_diff_frames", True)
+
+    @classmethod
+    def tearDownClass(cls):
+        reset_option("compute.ops_on_diff_frames")
+        super().tearDownClass()
+
+    def _test_groupby_expanding_func(self, f):
+        pser = pd.Series([1, 2, 3])
+        pkey = pd.Series([1, 2, 3], name="a")
+        kser = ps.from_pandas(pser)
+        kkey = ps.from_pandas(pkey)
+
+        self.assert_eq(
+            getattr(kser.groupby(kkey).expanding(2), f)().sort_index(),
+            getattr(pser.groupby(pkey).expanding(2), f)().sort_index(),
+        )
+
+        pdf = pd.DataFrame({"a": [1, 2, 3, 2], "b": [4.0, 2.0, 3.0, 1.0]})
+        pkey = pd.Series([1, 2, 3, 2], name="a")
+        kdf = ps.from_pandas(pdf)
+        kkey = ps.from_pandas(pkey)
+
+        self.assert_eq(
+            getattr(kdf.groupby(kkey).expanding(2), f)().sort_index(),
+            getattr(pdf.groupby(pkey).expanding(2), f)().sort_index(),
+        )
+        self.assert_eq(
+            getattr(kdf.groupby(kkey)["b"].expanding(2), f)().sort_index(),
+            getattr(pdf.groupby(pkey)["b"].expanding(2), f)().sort_index(),
+        )
+        self.assert_eq(
+            getattr(kdf.groupby(kkey)[["b"]].expanding(2), f)().sort_index(),
+            getattr(pdf.groupby(pkey)[["b"]].expanding(2), f)().sort_index(),
+        )
+
+    def test_groupby_expanding_count(self):
+        # The behaviour of ExpandingGroupby.count are different between pandas>=1.0.0 and lower,
+        # and we're following the behaviour of latest version of pandas.
+        if LooseVersion(pd.__version__) >= LooseVersion("1.0.0"):
+            self._test_groupby_expanding_func("count")
+        else:
+            # Series
+            kser = ps.Series([1, 2, 3])
+            kkey = ps.Series([1, 2, 3], name="a")
+            midx = pd.MultiIndex.from_tuples(
+                list(zip(kkey.to_pandas().values, kser.index.to_pandas().values)), names=["a", None]
+            )
+            expected_result = pd.Series([np.nan, np.nan, np.nan], index=midx)
+            self.assert_eq(
+                kser.groupby(kkey).expanding(2).count().sort_index(), expected_result.sort_index()
+            )
+
+            # DataFrame
+            kdf = ps.DataFrame({"a": [1, 2, 3, 2], "b": [4.0, 2.0, 3.0, 1.0]})
+            kkey = ps.Series([1, 2, 3, 2], name="a")
+            midx = pd.MultiIndex.from_tuples([(1, 0), (2, 1), (2, 3), (3, 2)], names=["a", None])
+            expected_result = pd.DataFrame(
+                {"a": [None, None, 2.0, None], "b": [None, None, 2.0, None]}, index=midx
+            )
+            self.assert_eq(
+                kdf.groupby(kkey).expanding(2).count().sort_index(), expected_result.sort_index()
+            )
+            expected_result = pd.Series([None, None, 2.0, None], index=midx, name="b")
+            self.assert_eq(
+                kdf.groupby(kkey)["b"].expanding(2).count().sort_index(),
+                expected_result.sort_index(),
+            )
+            expected_result = pd.DataFrame({"b": [None, None, 2.0, None]}, index=midx)
+            self.assert_eq(
+                kdf.groupby(kkey)[["b"]].expanding(2).count().sort_index(),
+                expected_result.sort_index(),
+            )
+
+    def test_groupby_expanding_min(self):
+        self._test_groupby_expanding_func("min")
+
+    def test_groupby_expanding_max(self):
+        self._test_groupby_expanding_func("max")
+
+    def test_groupby_expanding_mean(self):
+        self._test_groupby_expanding_func("mean")
+
+    def test_groupby_expanding_sum(self):
+        self._test_groupby_expanding_func("sum")
+
+    def test_groupby_expanding_std(self):
+        self._test_groupby_expanding_func("std")
+
+    def test_groupby_expanding_var(self):
+        self._test_groupby_expanding_func("var")
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.pandas.tests.test_ops_on_diff_frames_groupby_expanding import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_rolling.py b/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_rolling.py
new file mode 100644
index 0000000000000..158af35f61bbb
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_rolling.py
@@ -0,0 +1,97 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import pandas as pd
+
+from pyspark import pandas as ps
+from pyspark.pandas.config import set_option, reset_option
+from pyspark.testing.pandasutils import PandasOnSparkTestCase, TestUtils
+
+
+class OpsOnDiffFramesGroupByRollingTest(PandasOnSparkTestCase, TestUtils):
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        set_option("compute.ops_on_diff_frames", True)
+
+    @classmethod
+    def tearDownClass(cls):
+        reset_option("compute.ops_on_diff_frames")
+        super().tearDownClass()
+
+    def _test_groupby_rolling_func(self, f):
+        pser = pd.Series([1, 2, 3], name="a")
+        pkey = pd.Series([1, 2, 3], name="a")
+        kser = ps.from_pandas(pser)
+        kkey = ps.from_pandas(pkey)
+
+        self.assert_eq(
+            getattr(kser.groupby(kkey).rolling(2), f)().sort_index(),
+            getattr(pser.groupby(pkey).rolling(2), f)().sort_index(),
+        )
+
+        pdf = pd.DataFrame({"a": [1, 2, 3, 2], "b": [4.0, 2.0, 3.0, 1.0]})
+        pkey = pd.Series([1, 2, 3, 2], name="a")
+        kdf = ps.from_pandas(pdf)
+        kkey = ps.from_pandas(pkey)
+
+        self.assert_eq(
+            getattr(kdf.groupby(kkey).rolling(2), f)().sort_index(),
+            getattr(pdf.groupby(pkey).rolling(2), f)().sort_index(),
+        )
+        self.assert_eq(
+            getattr(kdf.groupby(kkey)["b"].rolling(2), f)().sort_index(),
+            getattr(pdf.groupby(pkey)["b"].rolling(2), f)().sort_index(),
+        )
+        self.assert_eq(
+            getattr(kdf.groupby(kkey)[["b"]].rolling(2), f)().sort_index(),
+            getattr(pdf.groupby(pkey)[["b"]].rolling(2), f)().sort_index(),
+        )
+
+    def test_groupby_rolling_count(self):
+        self._test_groupby_rolling_func("count")
+
+    def test_groupby_rolling_min(self):
+        self._test_groupby_rolling_func("min")
+
+    def test_groupby_rolling_max(self):
+        self._test_groupby_rolling_func("max")
+
+    def test_groupby_rolling_mean(self):
+        self._test_groupby_rolling_func("mean")
+
+    def test_groupby_rolling_sum(self):
+        self._test_groupby_rolling_func("sum")
+
+    def test_groupby_rolling_std(self):
+        # TODO: `std` now raise error in pandas 1.0.0
+        self._test_groupby_rolling_func("std")
+
+    def test_groupby_rolling_var(self):
+        self._test_groupby_rolling_func("var")
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.pandas.tests.test_ops_on_diff_frames_groupby_rolling import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_repr.py b/python/pyspark/pandas/tests/test_repr.py
new file mode 100644
index 0000000000000..e2b1c166f1c04
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_repr.py
@@ -0,0 +1,185 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import numpy as np
+
+from pyspark import pandas as ps
+from pyspark.pandas.config import set_option, reset_option, option_context
+from pyspark.testing.pandasutils import PandasOnSparkTestCase
+
+
+class ReprTest(PandasOnSparkTestCase):
+    max_display_count = 23
+
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        set_option("display.max_rows", ReprTest.max_display_count)
+
+    @classmethod
+    def tearDownClass(cls):
+        reset_option("display.max_rows")
+        super().tearDownClass()
+
+    def test_repr_dataframe(self):
+        kdf = ps.range(ReprTest.max_display_count)
+        self.assertTrue("Showing only the first" not in repr(kdf))
+        self.assert_eq(repr(kdf), repr(kdf.to_pandas()))
+
+        kdf = ps.range(ReprTest.max_display_count + 1)
+        self.assertTrue("Showing only the first" in repr(kdf))
+        self.assertTrue(
+            repr(kdf).startswith(repr(kdf.to_pandas().head(ReprTest.max_display_count)))
+        )
+
+        with option_context("display.max_rows", None):
+            kdf = ps.range(ReprTest.max_display_count + 1)
+            self.assert_eq(repr(kdf), repr(kdf.to_pandas()))
+
+    def test_repr_series(self):
+        kser = ps.range(ReprTest.max_display_count).id
+        self.assertTrue("Showing only the first" not in repr(kser))
+        self.assert_eq(repr(kser), repr(kser.to_pandas()))
+
+        kser = ps.range(ReprTest.max_display_count + 1).id
+        self.assertTrue("Showing only the first" in repr(kser))
+        self.assertTrue(
+            repr(kser).startswith(repr(kser.to_pandas().head(ReprTest.max_display_count)))
+        )
+
+        with option_context("display.max_rows", None):
+            kser = ps.range(ReprTest.max_display_count + 1).id
+            self.assert_eq(repr(kser), repr(kser.to_pandas()))
+
+        kser = ps.range(ReprTest.max_display_count).id.rename()
+        self.assertTrue("Showing only the first" not in repr(kser))
+        self.assert_eq(repr(kser), repr(kser.to_pandas()))
+
+        kser = ps.range(ReprTest.max_display_count + 1).id.rename()
+        self.assertTrue("Showing only the first" in repr(kser))
+        self.assertTrue(
+            repr(kser).startswith(repr(kser.to_pandas().head(ReprTest.max_display_count)))
+        )
+
+        with option_context("display.max_rows", None):
+            kser = ps.range(ReprTest.max_display_count + 1).id.rename()
+            self.assert_eq(repr(kser), repr(kser.to_pandas()))
+
+        kser = ps.MultiIndex.from_tuples(
+            [(100 * i, i) for i in range(ReprTest.max_display_count)]
+        ).to_series()
+        self.assertTrue("Showing only the first" not in repr(kser))
+        self.assert_eq(repr(kser), repr(kser.to_pandas()))
+
+        kser = ps.MultiIndex.from_tuples(
+            [(100 * i, i) for i in range(ReprTest.max_display_count + 1)]
+        ).to_series()
+        self.assertTrue("Showing only the first" in repr(kser))
+        self.assertTrue(
+            repr(kser).startswith(repr(kser.to_pandas().head(ReprTest.max_display_count)))
+        )
+
+        with option_context("display.max_rows", None):
+            kser = ps.MultiIndex.from_tuples(
+                [(100 * i, i) for i in range(ReprTest.max_display_count + 1)]
+            ).to_series()
+            self.assert_eq(repr(kser), repr(kser.to_pandas()))
+
+    def test_repr_indexes(self):
+        kidx = ps.range(ReprTest.max_display_count).index
+        self.assertTrue("Showing only the first" not in repr(kidx))
+        self.assert_eq(repr(kidx), repr(kidx.to_pandas()))
+
+        kidx = ps.range(ReprTest.max_display_count + 1).index
+        self.assertTrue("Showing only the first" in repr(kidx))
+        self.assertTrue(
+            repr(kidx).startswith(
+                repr(kidx.to_pandas().to_series().head(ReprTest.max_display_count).index)
+            )
+        )
+
+        with option_context("display.max_rows", None):
+            kidx = ps.range(ReprTest.max_display_count + 1).index
+            self.assert_eq(repr(kidx), repr(kidx.to_pandas()))
+
+        kidx = ps.MultiIndex.from_tuples([(100 * i, i) for i in range(ReprTest.max_display_count)])
+        self.assertTrue("Showing only the first" not in repr(kidx))
+        self.assert_eq(repr(kidx), repr(kidx.to_pandas()))
+
+        kidx = ps.MultiIndex.from_tuples(
+            [(100 * i, i) for i in range(ReprTest.max_display_count + 1)]
+        )
+        self.assertTrue("Showing only the first" in repr(kidx))
+        self.assertTrue(
+            repr(kidx).startswith(
+                repr(kidx.to_pandas().to_frame().head(ReprTest.max_display_count).index)
+            )
+        )
+
+        with option_context("display.max_rows", None):
+            kidx = ps.MultiIndex.from_tuples(
+                [(100 * i, i) for i in range(ReprTest.max_display_count + 1)]
+            )
+            self.assert_eq(repr(kidx), repr(kidx.to_pandas()))
+
+    def test_html_repr(self):
+        kdf = ps.range(ReprTest.max_display_count)
+        self.assertTrue("Showing only the first" not in kdf._repr_html_())
+        self.assertEqual(kdf._repr_html_(), kdf.to_pandas()._repr_html_())
+
+        kdf = ps.range(ReprTest.max_display_count + 1)
+        self.assertTrue("Showing only the first" in kdf._repr_html_())
+
+        with option_context("display.max_rows", None):
+            kdf = ps.range(ReprTest.max_display_count + 1)
+            self.assertEqual(kdf._repr_html_(), kdf.to_pandas()._repr_html_())
+
+    def test_repr_float_index(self):
+        kdf = ps.DataFrame(
+            {"a": np.random.rand(ReprTest.max_display_count)},
+            index=np.random.rand(ReprTest.max_display_count),
+        )
+        self.assertTrue("Showing only the first" not in repr(kdf))
+        self.assert_eq(repr(kdf), repr(kdf.to_pandas()))
+        self.assertTrue("Showing only the first" not in repr(kdf.a))
+        self.assert_eq(repr(kdf.a), repr(kdf.a.to_pandas()))
+        self.assertTrue("Showing only the first" not in repr(kdf.index))
+        self.assert_eq(repr(kdf.index), repr(kdf.index.to_pandas()))
+
+        self.assertTrue("Showing only the first" not in kdf._repr_html_())
+        self.assertEqual(kdf._repr_html_(), kdf.to_pandas()._repr_html_())
+
+        kdf = ps.DataFrame(
+            {"a": np.random.rand(ReprTest.max_display_count + 1)},
+            index=np.random.rand(ReprTest.max_display_count + 1),
+        )
+        self.assertTrue("Showing only the first" in repr(kdf))
+        self.assertTrue("Showing only the first" in repr(kdf.a))
+        self.assertTrue("Showing only the first" in repr(kdf.index))
+        self.assertTrue("Showing only the first" in kdf._repr_html_())
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.pandas.tests.test_repr import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_reshape.py b/python/pyspark/pandas/tests/test_reshape.py
new file mode 100644
index 0000000000000..96665dfa010cc
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_reshape.py
@@ -0,0 +1,294 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import datetime
+from decimal import Decimal
+from distutils.version import LooseVersion
+
+import numpy as np
+import pandas as pd
+
+from pyspark import pandas as ps
+from pyspark.pandas.utils import name_like_string
+from pyspark.testing.pandasutils import PandasOnSparkTestCase
+
+
+class ReshapeTest(PandasOnSparkTestCase):
+    def test_get_dummies(self):
+        for pdf_or_ps in [
+            pd.Series([1, 1, 1, 2, 2, 1, 3, 4]),
+            # pd.Series([1, 1, 1, 2, 2, 1, 3, 4], dtype='category'),
+            # pd.Series(pd.Categorical([1, 1, 1, 2, 2, 1, 3, 4],
+            #                          categories=[4, 3, 2, 1])),
+            pd.DataFrame(
+                {
+                    "a": [1, 2, 3, 4, 4, 3, 2, 1],
+                    # 'b': pd.Categorical(list('abcdabcd')),
+                    "b": list("abcdabcd"),
+                }
+            ),
+            pd.DataFrame({10: [1, 2, 3, 4, 4, 3, 2, 1], 20: list("abcdabcd")}),
+        ]:
+            kdf_or_kser = ps.from_pandas(pdf_or_ps)
+
+            self.assert_eq(ps.get_dummies(kdf_or_kser), pd.get_dummies(pdf_or_ps, dtype=np.int8))
+
+        kser = ps.Series([1, 1, 1, 2, 2, 1, 3, 4])
+        with self.assertRaisesRegex(
+            NotImplementedError, "get_dummies currently does not support sparse"
+        ):
+            ps.get_dummies(kser, sparse=True)
+
+    def test_get_dummies_object(self):
+        pdf = pd.DataFrame(
+            {
+                "a": [1, 2, 3, 4, 4, 3, 2, 1],
+                # 'a': pd.Categorical([1, 2, 3, 4, 4, 3, 2, 1]),
+                "b": list("abcdabcd"),
+                # 'c': pd.Categorical(list('abcdabcd')),
+                "c": list("abcdabcd"),
+            }
+        )
+        kdf = ps.from_pandas(pdf)
+
+        # Explicitly exclude object columns
+        self.assert_eq(
+            ps.get_dummies(kdf, columns=["a", "c"]),
+            pd.get_dummies(pdf, columns=["a", "c"], dtype=np.int8),
+        )
+
+        self.assert_eq(ps.get_dummies(kdf), pd.get_dummies(pdf, dtype=np.int8))
+        self.assert_eq(ps.get_dummies(kdf.b), pd.get_dummies(pdf.b, dtype=np.int8))
+        self.assert_eq(
+            ps.get_dummies(kdf, columns=["b"]), pd.get_dummies(pdf, columns=["b"], dtype=np.int8)
+        )
+
+        self.assertRaises(KeyError, lambda: ps.get_dummies(kdf, columns=("a", "c")))
+        self.assertRaises(TypeError, lambda: ps.get_dummies(kdf, columns="b"))
+
+        # non-string names
+        pdf = pd.DataFrame(
+            {10: [1, 2, 3, 4, 4, 3, 2, 1], 20: list("abcdabcd"), 30: list("abcdabcd")}
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            ps.get_dummies(kdf, columns=[10, 30]),
+            pd.get_dummies(pdf, columns=[10, 30], dtype=np.int8),
+        )
+
+        self.assertRaises(TypeError, lambda: ps.get_dummies(kdf, columns=10))
+
+    def test_get_dummies_date_datetime(self):
+        pdf = pd.DataFrame(
+            {
+                "d": [
+                    datetime.date(2019, 1, 1),
+                    datetime.date(2019, 1, 2),
+                    datetime.date(2019, 1, 1),
+                ],
+                "dt": [
+                    datetime.datetime(2019, 1, 1, 0, 0, 0),
+                    datetime.datetime(2019, 1, 1, 0, 0, 1),
+                    datetime.datetime(2019, 1, 1, 0, 0, 0),
+                ],
+            }
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(ps.get_dummies(kdf), pd.get_dummies(pdf, dtype=np.int8))
+        self.assert_eq(ps.get_dummies(kdf.d), pd.get_dummies(pdf.d, dtype=np.int8))
+        self.assert_eq(ps.get_dummies(kdf.dt), pd.get_dummies(pdf.dt, dtype=np.int8))
+
+    def test_get_dummies_boolean(self):
+        pdf = pd.DataFrame({"b": [True, False, True]})
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(ps.get_dummies(kdf), pd.get_dummies(pdf, dtype=np.int8))
+        self.assert_eq(ps.get_dummies(kdf.b), pd.get_dummies(pdf.b, dtype=np.int8))
+
+    def test_get_dummies_decimal(self):
+        pdf = pd.DataFrame({"d": [Decimal(1.0), Decimal(2.0), Decimal(1)]})
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(ps.get_dummies(kdf), pd.get_dummies(pdf, dtype=np.int8))
+        self.assert_eq(ps.get_dummies(kdf.d), pd.get_dummies(pdf.d, dtype=np.int8), almost=True)
+
+    def test_get_dummies_kwargs(self):
+        # pser = pd.Series([1, 1, 1, 2, 2, 1, 3, 4], dtype='category')
+        pser = pd.Series([1, 1, 1, 2, 2, 1, 3, 4])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(
+            ps.get_dummies(kser, prefix="X", prefix_sep="-"),
+            pd.get_dummies(pser, prefix="X", prefix_sep="-", dtype=np.int8),
+        )
+
+        self.assert_eq(
+            ps.get_dummies(kser, drop_first=True),
+            pd.get_dummies(pser, drop_first=True, dtype=np.int8),
+        )
+
+        # nan
+        # pser = pd.Series([1, 1, 1, 2, np.nan, 3, np.nan, 5], dtype='category')
+        pser = pd.Series([1, 1, 1, 2, np.nan, 3, np.nan, 5])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(ps.get_dummies(kser), pd.get_dummies(pser, dtype=np.int8), almost=True)
+
+        # dummy_na
+        self.assert_eq(
+            ps.get_dummies(kser, dummy_na=True), pd.get_dummies(pser, dummy_na=True, dtype=np.int8)
+        )
+
+    def test_get_dummies_prefix(self):
+        pdf = pd.DataFrame({"A": ["a", "b", "a"], "B": ["b", "a", "c"], "D": [0, 0, 1]})
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            ps.get_dummies(kdf, prefix=["foo", "bar"]),
+            pd.get_dummies(pdf, prefix=["foo", "bar"], dtype=np.int8),
+        )
+
+        self.assert_eq(
+            ps.get_dummies(kdf, prefix=["foo"], columns=["B"]),
+            pd.get_dummies(pdf, prefix=["foo"], columns=["B"], dtype=np.int8),
+        )
+
+        self.assert_eq(
+            ps.get_dummies(kdf, prefix={"A": "foo", "B": "bar"}),
+            pd.get_dummies(pdf, prefix={"A": "foo", "B": "bar"}, dtype=np.int8),
+        )
+
+        self.assert_eq(
+            ps.get_dummies(kdf, prefix={"B": "foo", "A": "bar"}),
+            pd.get_dummies(pdf, prefix={"B": "foo", "A": "bar"}, dtype=np.int8),
+        )
+
+        self.assert_eq(
+            ps.get_dummies(kdf, prefix={"A": "foo", "B": "bar"}, columns=["A", "B"]),
+            pd.get_dummies(pdf, prefix={"A": "foo", "B": "bar"}, columns=["A", "B"], dtype=np.int8),
+        )
+
+        with self.assertRaisesRegex(NotImplementedError, "string types"):
+            ps.get_dummies(kdf, prefix="foo")
+        with self.assertRaisesRegex(ValueError, "Length of 'prefix' \\(1\\) .* \\(2\\)"):
+            ps.get_dummies(kdf, prefix=["foo"])
+        with self.assertRaisesRegex(ValueError, "Length of 'prefix' \\(2\\) .* \\(1\\)"):
+            ps.get_dummies(kdf, prefix=["foo", "bar"], columns=["B"])
+
+        pser = pd.Series([1, 1, 1, 2, 2, 1, 3, 4], name="A")
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(
+            ps.get_dummies(kser, prefix="foo"), pd.get_dummies(pser, prefix="foo", dtype=np.int8)
+        )
+
+        # columns are ignored.
+        self.assert_eq(
+            ps.get_dummies(kser, prefix=["foo"], columns=["B"]),
+            pd.get_dummies(pser, prefix=["foo"], columns=["B"], dtype=np.int8),
+        )
+
+    def test_get_dummies_dtype(self):
+        pdf = pd.DataFrame(
+            {
+                # "A": pd.Categorical(['a', 'b', 'a'], categories=['a', 'b', 'c']),
+                "A": ["a", "b", "a"],
+                "B": [0, 0, 1],
+            }
+        )
+        kdf = ps.from_pandas(pdf)
+
+        if LooseVersion("0.23.0") <= LooseVersion(pd.__version__):
+            exp = pd.get_dummies(pdf, dtype="float64")
+        else:
+            exp = pd.get_dummies(pdf)
+            exp = exp.astype({"A_a": "float64", "A_b": "float64"})
+        res = ps.get_dummies(kdf, dtype="float64")
+        self.assert_eq(res, exp)
+
+    def test_get_dummies_multiindex_columns(self):
+        pdf = pd.DataFrame(
+            {
+                ("x", "a", "1"): [1, 2, 3, 4, 4, 3, 2, 1],
+                ("x", "b", "2"): list("abcdabcd"),
+                ("y", "c", "3"): list("abcdabcd"),
+            }
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            ps.get_dummies(kdf), pd.get_dummies(pdf, dtype=np.int8).rename(columns=name_like_string)
+        )
+        self.assert_eq(
+            ps.get_dummies(kdf, columns=[("y", "c", "3"), ("x", "a", "1")]),
+            pd.get_dummies(pdf, columns=[("y", "c", "3"), ("x", "a", "1")], dtype=np.int8).rename(
+                columns=name_like_string
+            ),
+        )
+        self.assert_eq(
+            ps.get_dummies(kdf, columns=["x"]),
+            pd.get_dummies(pdf, columns=["x"], dtype=np.int8).rename(columns=name_like_string),
+        )
+        self.assert_eq(
+            ps.get_dummies(kdf, columns=("x", "a")),
+            pd.get_dummies(pdf, columns=("x", "a"), dtype=np.int8).rename(columns=name_like_string),
+        )
+
+        self.assertRaises(KeyError, lambda: ps.get_dummies(kdf, columns=["z"]))
+        self.assertRaises(KeyError, lambda: ps.get_dummies(kdf, columns=("x", "c")))
+        self.assertRaises(ValueError, lambda: ps.get_dummies(kdf, columns=[("x",), "c"]))
+        self.assertRaises(TypeError, lambda: ps.get_dummies(kdf, columns="x"))
+
+        # non-string names
+        pdf = pd.DataFrame(
+            {
+                ("x", 1, "a"): [1, 2, 3, 4, 4, 3, 2, 1],
+                ("x", 2, "b"): list("abcdabcd"),
+                ("y", 3, "c"): list("abcdabcd"),
+            }
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            ps.get_dummies(kdf), pd.get_dummies(pdf, dtype=np.int8).rename(columns=name_like_string)
+        )
+        self.assert_eq(
+            ps.get_dummies(kdf, columns=[("y", 3, "c"), ("x", 1, "a")]),
+            pd.get_dummies(pdf, columns=[("y", 3, "c"), ("x", 1, "a")], dtype=np.int8).rename(
+                columns=name_like_string
+            ),
+        )
+        self.assert_eq(
+            ps.get_dummies(kdf, columns=["x"]),
+            pd.get_dummies(pdf, columns=["x"], dtype=np.int8).rename(columns=name_like_string),
+        )
+        self.assert_eq(
+            ps.get_dummies(kdf, columns=("x", 1)),
+            pd.get_dummies(pdf, columns=("x", 1), dtype=np.int8).rename(columns=name_like_string),
+        )
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.pandas.tests.test_reshape import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_rolling.py b/python/pyspark/pandas/tests/test_rolling.py
new file mode 100644
index 0000000000000..3827a6017e1f4
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_rolling.py
@@ -0,0 +1,184 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import numpy as np
+import pandas as pd
+
+import pyspark.pandas as ps
+from pyspark.testing.pandasutils import PandasOnSparkTestCase, TestUtils
+from pyspark.pandas.window import Rolling
+
+
+class RollingTest(PandasOnSparkTestCase, TestUtils):
+    def test_rolling_error(self):
+        with self.assertRaisesRegex(ValueError, "window must be >= 0"):
+            ps.range(10).rolling(window=-1)
+        with self.assertRaisesRegex(ValueError, "min_periods must be >= 0"):
+            ps.range(10).rolling(window=1, min_periods=-1)
+
+        with self.assertRaisesRegex(
+            TypeError, "kdf_or_kser must be a series or dataframe; however, got:.*int"
+        ):
+            Rolling(1, 2)
+
+    def _test_rolling_func(self, f):
+        pser = pd.Series([1, 2, 3], index=np.random.rand(3), name="a")
+        kser = ps.from_pandas(pser)
+        self.assert_eq(getattr(kser.rolling(2), f)(), getattr(pser.rolling(2), f)())
+        self.assert_eq(getattr(kser.rolling(2), f)().sum(), getattr(pser.rolling(2), f)().sum())
+
+        # Multiindex
+        pser = pd.Series(
+            [1, 2, 3],
+            index=pd.MultiIndex.from_tuples([("a", "x"), ("a", "y"), ("b", "z")]),
+            name="a",
+        )
+        kser = ps.from_pandas(pser)
+        self.assert_eq(getattr(kser.rolling(2), f)(), getattr(pser.rolling(2), f)())
+
+        pdf = pd.DataFrame(
+            {"a": [1.0, 2.0, 3.0, 2.0], "b": [4.0, 2.0, 3.0, 1.0]}, index=np.random.rand(4)
+        )
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(getattr(kdf.rolling(2), f)(), getattr(pdf.rolling(2), f)())
+        self.assert_eq(getattr(kdf.rolling(2), f)().sum(), getattr(pdf.rolling(2), f)().sum())
+
+        # Multiindex column
+        columns = pd.MultiIndex.from_tuples([("a", "x"), ("a", "y")])
+        pdf.columns = columns
+        kdf.columns = columns
+        self.assert_eq(getattr(kdf.rolling(2), f)(), getattr(pdf.rolling(2), f)())
+
+    def test_rolling_min(self):
+        self._test_rolling_func("min")
+
+    def test_rolling_max(self):
+        self._test_rolling_func("max")
+
+    def test_rolling_mean(self):
+        self._test_rolling_func("mean")
+
+    def test_rolling_sum(self):
+        self._test_rolling_func("sum")
+
+    def test_rolling_count(self):
+        self._test_rolling_func("count")
+
+    def test_rolling_std(self):
+        self._test_rolling_func("std")
+
+    def test_rolling_var(self):
+        self._test_rolling_func("var")
+
+    def _test_groupby_rolling_func(self, f):
+        pser = pd.Series([1, 2, 3, 2], index=np.random.rand(4), name="a")
+        kser = ps.from_pandas(pser)
+        self.assert_eq(
+            getattr(kser.groupby(kser).rolling(2), f)().sort_index(),
+            getattr(pser.groupby(pser).rolling(2), f)().sort_index(),
+        )
+        self.assert_eq(
+            getattr(kser.groupby(kser).rolling(2), f)().sum(),
+            getattr(pser.groupby(pser).rolling(2), f)().sum(),
+        )
+
+        # Multiindex
+        pser = pd.Series(
+            [1, 2, 3, 2],
+            index=pd.MultiIndex.from_tuples([("a", "x"), ("a", "y"), ("b", "z"), ("c", "z")]),
+            name="a",
+        )
+        kser = ps.from_pandas(pser)
+        self.assert_eq(
+            getattr(kser.groupby(kser).rolling(2), f)().sort_index(),
+            getattr(pser.groupby(pser).rolling(2), f)().sort_index(),
+        )
+
+        pdf = pd.DataFrame({"a": [1.0, 2.0, 3.0, 2.0], "b": [4.0, 2.0, 3.0, 1.0]})
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(
+            getattr(kdf.groupby(kdf.a).rolling(2), f)().sort_index(),
+            getattr(pdf.groupby(pdf.a).rolling(2), f)().sort_index(),
+        )
+        self.assert_eq(
+            getattr(kdf.groupby(kdf.a).rolling(2), f)().sum(),
+            getattr(pdf.groupby(pdf.a).rolling(2), f)().sum(),
+        )
+        self.assert_eq(
+            getattr(kdf.groupby(kdf.a + 1).rolling(2), f)().sort_index(),
+            getattr(pdf.groupby(pdf.a + 1).rolling(2), f)().sort_index(),
+        )
+        self.assert_eq(
+            getattr(kdf.b.groupby(kdf.a).rolling(2), f)().sort_index(),
+            getattr(pdf.b.groupby(pdf.a).rolling(2), f)().sort_index(),
+        )
+        self.assert_eq(
+            getattr(kdf.groupby(kdf.a)["b"].rolling(2), f)().sort_index(),
+            getattr(pdf.groupby(pdf.a)["b"].rolling(2), f)().sort_index(),
+        )
+        self.assert_eq(
+            getattr(kdf.groupby(kdf.a)[["b"]].rolling(2), f)().sort_index(),
+            getattr(pdf.groupby(pdf.a)[["b"]].rolling(2), f)().sort_index(),
+        )
+
+        # Multiindex column
+        columns = pd.MultiIndex.from_tuples([("a", "x"), ("a", "y")])
+        pdf.columns = columns
+        kdf.columns = columns
+        self.assert_eq(
+            getattr(kdf.groupby(("a", "x")).rolling(2), f)().sort_index(),
+            getattr(pdf.groupby(("a", "x")).rolling(2), f)().sort_index(),
+        )
+
+        self.assert_eq(
+            getattr(kdf.groupby([("a", "x"), ("a", "y")]).rolling(2), f)().sort_index(),
+            getattr(pdf.groupby([("a", "x"), ("a", "y")]).rolling(2), f)().sort_index(),
+        )
+
+    def test_groupby_rolling_count(self):
+        self._test_groupby_rolling_func("count")
+
+    def test_groupby_rolling_min(self):
+        self._test_groupby_rolling_func("min")
+
+    def test_groupby_rolling_max(self):
+        self._test_groupby_rolling_func("max")
+
+    def test_groupby_rolling_mean(self):
+        self._test_groupby_rolling_func("mean")
+
+    def test_groupby_rolling_sum(self):
+        self._test_groupby_rolling_func("sum")
+
+    def test_groupby_rolling_std(self):
+        # TODO: `std` now raise error in pandas 1.0.0
+        self._test_groupby_rolling_func("std")
+
+    def test_groupby_rolling_var(self):
+        self._test_groupby_rolling_func("var")
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.pandas.tests.test_rolling import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_series.py b/python/pyspark/pandas/tests/test_series.py
new file mode 100644
index 0000000000000..c9f3f7e3630a9
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_series.py
@@ -0,0 +1,2917 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import unittest
+from collections import defaultdict
+from distutils.version import LooseVersion
+import inspect
+from itertools import product
+from datetime import datetime, timedelta
+
+import numpy as np
+import pandas as pd
+from pyspark.ml.linalg import SparseVector
+from pyspark.sql import functions as F
+
+from pyspark import pandas as ps
+from pyspark.testing.pandasutils import (
+    have_tabulate,
+    PandasOnSparkTestCase,
+    SPARK_CONF_ARROW_ENABLED,
+    tabulate_requirement_message,
+)
+from pyspark.testing.sqlutils import SQLTestUtils
+from pyspark.pandas.exceptions import PandasNotImplementedError
+from pyspark.pandas.missing.series import MissingPandasLikeSeries
+from pyspark.pandas.typedef.typehints import (
+    extension_dtypes,
+    extension_dtypes_available,
+    extension_float_dtypes_available,
+    extension_object_dtypes_available,
+)
+
+
+class SeriesTest(PandasOnSparkTestCase, SQLTestUtils):
+    @property
+    def pser(self):
+        return pd.Series([1, 2, 3, 4, 5, 6, 7], name="x")
+
+    @property
+    def kser(self):
+        return ps.from_pandas(self.pser)
+
+    def test_series_ops(self):
+        pser = self.pser
+        kser = self.kser
+
+        self.assert_eq(kser + 1, pser + 1)
+        self.assert_eq(1 + kser, 1 + pser)
+        self.assert_eq(kser + 1 + 10 * kser, pser + 1 + 10 * pser)
+        self.assert_eq(kser + 1 + 10 * kser.index, pser + 1 + 10 * pser.index)
+        self.assert_eq(kser.index + 1 + 10 * kser, pser.index + 1 + 10 * pser)
+
+    def test_series_tuple_name(self):
+        pser = self.pser
+        pser.name = ("x", "a")
+
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(kser, pser)
+        self.assert_eq(kser.name, pser.name)
+
+        pser.name = ("y", "z")
+        kser.name = ("y", "z")
+
+        self.assert_eq(kser, pser)
+        self.assert_eq(kser.name, pser.name)
+
+    def test_repr_cache_invalidation(self):
+        # If there is any cache, inplace operations should invalidate it.
+        s = ps.range(10)["id"]
+        s.__repr__()
+        s.rename("a", inplace=True)
+        self.assertEqual(s.__repr__(), s.rename("a").__repr__())
+
+    def _check_extension(self, kser, pser):
+        if LooseVersion("1.1") <= LooseVersion(pd.__version__) < LooseVersion("1.2.2"):
+            self.assert_eq(kser, pser, check_exact=False)
+            self.assertTrue(isinstance(kser.dtype, extension_dtypes))
+        else:
+            self.assert_eq(kser, pser)
+
+    @unittest.skipIf(not extension_dtypes_available, "pandas extension dtypes are not available")
+    def test_extension_dtypes(self):
+        for pser in [
+            pd.Series([1, 2, None, 4], dtype="Int8"),
+            pd.Series([1, 2, None, 4], dtype="Int16"),
+            pd.Series([1, 2, None, 4], dtype="Int32"),
+            pd.Series([1, 2, None, 4], dtype="Int64"),
+        ]:
+            kser = ps.from_pandas(pser)
+
+            self._check_extension(kser, pser)
+            self._check_extension(kser + F.lit(1).cast("byte"), pser + 1)
+            self._check_extension(kser + kser, pser + pser)
+
+    @unittest.skipIf(
+        not extension_object_dtypes_available, "pandas extension object dtypes are not available"
+    )
+    def test_extension_object_dtypes(self):
+        # string
+        pser = pd.Series(["a", None, "c", "d"], dtype="string")
+        kser = ps.from_pandas(pser)
+
+        self._check_extension(kser, pser)
+
+        # boolean
+        pser = pd.Series([True, False, True, None], dtype="boolean")
+        kser = ps.from_pandas(pser)
+
+        self._check_extension(kser, pser)
+        self._check_extension(kser & kser, pser & pser)
+        self._check_extension(kser | kser, pser | pser)
+
+    @unittest.skipIf(
+        not extension_float_dtypes_available, "pandas extension float dtypes are not available"
+    )
+    def test_extension_float_dtypes(self):
+        for pser in [
+            pd.Series([1.0, 2.0, None, 4.0], dtype="Float32"),
+            pd.Series([1.0, 2.0, None, 4.0], dtype="Float64"),
+        ]:
+            kser = ps.from_pandas(pser)
+
+            self._check_extension(kser, pser)
+            self._check_extension(kser + 1, pser + 1)
+            self._check_extension(kser + kser, pser + pser)
+
+    def test_empty_series(self):
+        pser_a = pd.Series([], dtype="i1")
+        pser_b = pd.Series([], dtype="str")
+
+        self.assert_eq(ps.from_pandas(pser_a), pser_a)
+
+        kser_b = ps.from_pandas(pser_b)
+        self.assert_eq(kser_b, pser_b)
+
+        with self.sql_conf({SPARK_CONF_ARROW_ENABLED: False}):
+            self.assert_eq(ps.from_pandas(pser_a), pser_a)
+            self.assert_eq(ps.from_pandas(pser_b), pser_b)
+
+    def test_all_null_series(self):
+        pser_a = pd.Series([None, None, None], dtype="float64")
+        pser_b = pd.Series([None, None, None], dtype="str")
+
+        self.assert_eq(ps.from_pandas(pser_a), pser_a)
+
+        kser_b = ps.from_pandas(pser_b)
+        self.assert_eq(kser_b, pser_b)
+
+        with self.sql_conf({SPARK_CONF_ARROW_ENABLED: False}):
+            self.assert_eq(ps.from_pandas(pser_a), pser_a)
+            self.assert_eq(ps.from_pandas(pser_b), pser_b)
+
+    def test_head(self):
+        kser = self.kser
+        pser = self.pser
+
+        self.assert_eq(kser.head(3), pser.head(3))
+        self.assert_eq(kser.head(0), pser.head(0))
+        self.assert_eq(kser.head(-3), pser.head(-3))
+        self.assert_eq(kser.head(-10), pser.head(-10))
+
+    def test_last(self):
+        with self.assertRaises(TypeError):
+            self.kser.last("1D")
+
+        index = pd.date_range("2018-04-09", periods=4, freq="2D")
+        pser = pd.Series([1, 2, 3, 4], index=index)
+        kser = ps.from_pandas(pser)
+        self.assert_eq(kser.last("1D"), pser.last("1D"))
+
+    def test_first(self):
+        with self.assertRaises(TypeError):
+            self.kser.first("1D")
+
+        index = pd.date_range("2018-04-09", periods=4, freq="2D")
+        pser = pd.Series([1, 2, 3, 4], index=index)
+        kser = ps.from_pandas(pser)
+        self.assert_eq(kser.first("1D"), pser.first("1D"))
+
+    def test_rename(self):
+        pser = pd.Series([1, 2, 3, 4, 5, 6, 7], name="x")
+        kser = ps.from_pandas(pser)
+
+        pser.name = "renamed"
+        kser.name = "renamed"
+        self.assertEqual(kser.name, "renamed")
+        self.assert_eq(kser, pser)
+
+        pser.name = None
+        kser.name = None
+        self.assertEqual(kser.name, None)
+        self.assert_eq(kser, pser)
+
+        pidx = pser.index
+        kidx = kser.index
+        pidx.name = "renamed"
+        kidx.name = "renamed"
+        self.assertEqual(kidx.name, "renamed")
+        self.assert_eq(kidx, pidx)
+
+        expected_error_message = "Series.name must be a hashable type"
+        with self.assertRaisesRegex(TypeError, expected_error_message):
+            kser.name = ["renamed"]
+        with self.assertRaisesRegex(TypeError, expected_error_message):
+            kser.name = ["0", "1"]
+        with self.assertRaisesRegex(TypeError, expected_error_message):
+            ps.Series([1, 2, 3], name=["0", "1"])
+
+    def test_rename_method(self):
+        # Series name
+        pser = pd.Series([1, 2, 3, 4, 5, 6, 7], name="x")
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(kser.rename("y"), pser.rename("y"))
+        self.assertEqual(kser.name, "x")  # no mutation
+        self.assert_eq(kser.rename(), pser.rename())
+
+        self.assert_eq((kser.rename("y") + 1).head(), (pser.rename("y") + 1).head())
+
+        kser.rename("z", inplace=True)
+        pser.rename("z", inplace=True)
+        self.assertEqual(kser.name, "z")
+        self.assert_eq(kser, pser)
+
+        expected_error_message = "Series.name must be a hashable type"
+        with self.assertRaisesRegex(TypeError, expected_error_message):
+            kser.rename(["0", "1"])
+
+        # Series index
+        # pser = pd.Series(['a', 'b', 'c', 'd', 'e', 'f', 'g'], name='x')
+        # kser = ps.from_pandas(s)
+
+        # TODO: index
+        # res = kser.rename(lambda x: x ** 2)
+        # self.assert_eq(res, pser.rename(lambda x: x ** 2))
+
+        # res = kser.rename(pser)
+        # self.assert_eq(res, pser.rename(pser))
+
+        # res = kser.rename(kser)
+        # self.assert_eq(res, pser.rename(pser))
+
+        # res = kser.rename(lambda x: x**2, inplace=True)
+        # self.assertis(res, kser)
+        # s.rename(lambda x: x**2, inplace=True)
+        # self.assert_eq(kser, pser)
+
+    def test_rename_axis(self):
+        index = pd.Index(["A", "B", "C"], name="index")
+        pser = pd.Series([1.0, 2.0, 3.0], index=index, name="name")
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(
+            pser.rename_axis("index2").sort_index(), kser.rename_axis("index2").sort_index(),
+        )
+
+        self.assert_eq(
+            (pser + 1).rename_axis("index2").sort_index(),
+            (kser + 1).rename_axis("index2").sort_index(),
+        )
+
+        pser2 = pser.copy()
+        kser2 = kser.copy()
+        pser2.rename_axis("index2", inplace=True)
+        kser2.rename_axis("index2", inplace=True)
+        self.assert_eq(pser2.sort_index(), kser2.sort_index())
+
+        self.assertRaises(ValueError, lambda: kser.rename_axis(["index2", "index3"]))
+        self.assertRaises(TypeError, lambda: kser.rename_axis(mapper=["index2"], index=["index3"]))
+
+        # index/columns parameters and dict_like/functions mappers introduced in pandas 0.24.0
+        if LooseVersion(pd.__version__) >= LooseVersion("0.24.0"):
+            self.assert_eq(
+                pser.rename_axis(index={"index": "index2", "missing": "index4"}).sort_index(),
+                kser.rename_axis(index={"index": "index2", "missing": "index4"}).sort_index(),
+            )
+
+            self.assert_eq(
+                pser.rename_axis(index=str.upper).sort_index(),
+                kser.rename_axis(index=str.upper).sort_index(),
+            )
+        else:
+            expected = kser
+            expected.index.name = "index2"
+            result = kser.rename_axis(index={"index": "index2", "missing": "index4"}).sort_index()
+            self.assert_eq(expected, result)
+
+            expected = kser
+            expected.index.name = "INDEX"
+            result = kser.rename_axis(index=str.upper).sort_index()
+            self.assert_eq(expected, result)
+
+        index = pd.MultiIndex.from_tuples(
+            [("A", "B"), ("C", "D"), ("E", "F")], names=["index1", "index2"]
+        )
+        pser = pd.Series([1.0, 2.0, 3.0], index=index, name="name")
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(
+            pser.rename_axis(["index3", "index4"]).sort_index(),
+            kser.rename_axis(["index3", "index4"]).sort_index(),
+        )
+
+        self.assertRaises(ValueError, lambda: kser.rename_axis(["index3", "index4", "index5"]))
+
+        # index/columns parameters and dict_like/functions mappers introduced in pandas 0.24.0
+        if LooseVersion(pd.__version__) >= LooseVersion("0.24.0"):
+            self.assert_eq(
+                pser.rename_axis(
+                    index={"index1": "index3", "index2": "index4", "missing": "index5"}
+                ).sort_index(),
+                kser.rename_axis(
+                    index={"index1": "index3", "index2": "index4", "missing": "index5"}
+                ).sort_index(),
+            )
+
+            self.assert_eq(
+                pser.rename_axis(index=str.upper).sort_index(),
+                kser.rename_axis(index=str.upper).sort_index(),
+            )
+        else:
+            expected = kser
+            expected.index.names = ["index3", "index4"]
+            result = kser.rename_axis(
+                index={"index1": "index3", "index2": "index4", "missing": "index5"}
+            ).sort_index()
+            self.assert_eq(expected, result)
+
+            expected.index.names = ["INDEX1", "INDEX2"]
+            result = kser.rename_axis(index=str.upper).sort_index()
+            self.assert_eq(expected, result)
+
+    def test_or(self):
+        pdf = pd.DataFrame(
+            {
+                "left": [True, False, True, False, np.nan, np.nan, True, False, np.nan],
+                "right": [True, False, False, True, True, False, np.nan, np.nan, np.nan],
+            }
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf["left"] | kdf["right"], pdf["left"] | pdf["right"])
+        self.assert_eq(kdf["left"] | True, pdf["left"] | True)
+        self.assert_eq(kdf["left"] | False, pdf["left"] | False)
+        self.assert_eq(kdf["left"] | None, pdf["left"] | None)
+        self.assert_eq(True | kdf["right"], True | pdf["right"])
+        self.assert_eq(False | kdf["right"], False | pdf["right"])
+        self.assert_eq(None | kdf["right"], None | pdf["right"])
+
+    @unittest.skipIf(
+        not extension_object_dtypes_available, "pandas extension object dtypes are not available"
+    )
+    def test_or_extenstion_dtypes(self):
+        pdf = pd.DataFrame(
+            {
+                "left": [True, False, True, False, np.nan, np.nan, True, False, np.nan],
+                "right": [True, False, False, True, True, False, np.nan, np.nan, np.nan],
+            }
+        ).astype("boolean")
+        kdf = ps.from_pandas(pdf)
+
+        self._check_extension(kdf["left"] | kdf["right"], pdf["left"] | pdf["right"])
+        self._check_extension(kdf["left"] | True, pdf["left"] | True)
+        self._check_extension(kdf["left"] | False, pdf["left"] | False)
+        self._check_extension(kdf["left"] | pd.NA, pdf["left"] | pd.NA)
+        self._check_extension(True | kdf["right"], True | pdf["right"])
+        self._check_extension(False | kdf["right"], False | pdf["right"])
+        self._check_extension(pd.NA | kdf["right"], pd.NA | pdf["right"])
+
+    def test_and(self):
+        pdf = pd.DataFrame(
+            {
+                "left": [True, False, True, False, np.nan, np.nan, True, False, np.nan],
+                "right": [True, False, False, True, True, False, np.nan, np.nan, np.nan],
+            }
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf["left"] & kdf["right"], pdf["left"] & pdf["right"])
+        self.assert_eq(kdf["left"] & True, pdf["left"] & True)
+        self.assert_eq(kdf["left"] & False, pdf["left"] & False)
+        self.assert_eq(kdf["left"] & None, pdf["left"] & None)
+        self.assert_eq(True & kdf["right"], True & pdf["right"])
+        self.assert_eq(False & kdf["right"], False & pdf["right"])
+        self.assert_eq(None & kdf["right"], None & pdf["right"])
+
+    @unittest.skipIf(
+        not extension_object_dtypes_available, "pandas extension object dtypes are not available"
+    )
+    def test_and_extenstion_dtypes(self):
+        pdf = pd.DataFrame(
+            {
+                "left": [True, False, True, False, np.nan, np.nan, True, False, np.nan],
+                "right": [True, False, False, True, True, False, np.nan, np.nan, np.nan],
+            }
+        ).astype("boolean")
+        kdf = ps.from_pandas(pdf)
+
+        self._check_extension(kdf["left"] & kdf["right"], pdf["left"] & pdf["right"])
+        self._check_extension(kdf["left"] & True, pdf["left"] & True)
+        self._check_extension(kdf["left"] & False, pdf["left"] & False)
+        self._check_extension(kdf["left"] & pd.NA, pdf["left"] & pd.NA)
+        self._check_extension(True & kdf["right"], True & pdf["right"])
+        self._check_extension(False & kdf["right"], False & pdf["right"])
+        self._check_extension(pd.NA & kdf["right"], pd.NA & pdf["right"])
+
+    def test_to_numpy(self):
+        pser = pd.Series([1, 2, 3, 4, 5, 6, 7], name="x")
+
+        kser = ps.from_pandas(pser)
+        self.assert_eq(kser.to_numpy(), pser.values)
+
+    def test_isin(self):
+        pser = pd.Series(["lama", "cow", "lama", "beetle", "lama", "hippo"], name="animal")
+
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(kser.isin(["cow", "lama"]), pser.isin(["cow", "lama"]))
+        self.assert_eq(kser.isin(np.array(["cow", "lama"])), pser.isin(np.array(["cow", "lama"])))
+        self.assert_eq(kser.isin({"cow"}), pser.isin({"cow"}))
+
+        msg = "only list-like objects are allowed to be passed to isin()"
+        with self.assertRaisesRegex(TypeError, msg):
+            kser.isin(1)
+
+    def test_drop_duplicates(self):
+        pdf = pd.DataFrame({"animal": ["lama", "cow", "lama", "beetle", "lama", "hippo"]})
+        kdf = ps.from_pandas(pdf)
+
+        pser = pdf.animal
+        kser = kdf.animal
+
+        self.assert_eq(kser.drop_duplicates().sort_index(), pser.drop_duplicates().sort_index())
+        self.assert_eq(
+            kser.drop_duplicates(keep="last").sort_index(),
+            pser.drop_duplicates(keep="last").sort_index(),
+        )
+
+        # inplace
+        kser.drop_duplicates(keep=False, inplace=True)
+        pser.drop_duplicates(keep=False, inplace=True)
+        self.assert_eq(kser.sort_index(), pser.sort_index())
+        self.assert_eq(kdf, pdf)
+
+    def test_reindex(self):
+        index = ["A", "B", "C", "D", "E"]
+        pser = pd.Series([1.0, 2.0, 3.0, 4.0, None], index=index, name="x")
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(pser, kser)
+
+        self.assert_eq(
+            pser.reindex(["A", "B"]).sort_index(), kser.reindex(["A", "B"]).sort_index(),
+        )
+
+        self.assert_eq(
+            pser.reindex(["A", "B", "2", "3"]).sort_index(),
+            kser.reindex(["A", "B", "2", "3"]).sort_index(),
+        )
+
+        self.assert_eq(
+            pser.reindex(["A", "E", "2"], fill_value=0).sort_index(),
+            kser.reindex(["A", "E", "2"], fill_value=0).sort_index(),
+        )
+
+        self.assertRaises(TypeError, lambda: kser.reindex(index=123))
+
+    def test_reindex_like(self):
+        data = [1.0, 2.0, None]
+        index = pd.Index(["A", "B", "C"], name="index1")
+        pser = pd.Series(data=data, index=index, name="name1")
+        kser = ps.from_pandas(pser)
+
+        # Reindexing single Index on single Index
+        data2 = [3.0, None, 4.0]
+        index2 = pd.Index(["A", "C", "D"], name="index2")
+        pser2 = pd.Series(data=data2, index=index2, name="name2")
+        kser2 = ps.from_pandas(pser2)
+
+        self.assert_eq(
+            pser.reindex_like(pser2).sort_index(), kser.reindex_like(kser2).sort_index(),
+        )
+
+        self.assert_eq(
+            (pser + 1).reindex_like(pser2).sort_index(),
+            (kser + 1).reindex_like(kser2).sort_index(),
+        )
+
+        # Reindexing MultiIndex on single Index
+        index2 = pd.MultiIndex.from_tuples(
+            [("A", "G"), ("C", "D"), ("I", "J")], names=["index3", "index4"]
+        )
+        pser2 = pd.Series(data=data2, index=index2, name="name2")
+        kser2 = ps.from_pandas(pser2)
+
+        self.assert_eq(
+            pser.reindex_like(pser2).sort_index(), kser.reindex_like(kser2).sort_index(),
+        )
+
+        self.assertRaises(TypeError, lambda: kser.reindex_like(index2))
+        self.assertRaises(AssertionError, lambda: kser2.reindex_like(kser))
+
+        # Reindexing MultiIndex on MultiIndex
+        index = pd.MultiIndex.from_tuples(
+            [("A", "B"), ("C", "D"), ("E", "F")], names=["index1", "index2"]
+        )
+        pser = pd.Series(data=data, index=index, name="name1")
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(
+            pser.reindex_like(pser2).sort_index(), kser.reindex_like(kser2).sort_index(),
+        )
+
+        # Reindexing with DataFrame
+        index2 = pd.MultiIndex.from_tuples(
+            [("A", "B"), ("C", "D"), ("E", "F")], names=["name3", "name4"]
+        )
+        pdf = pd.DataFrame(data=data, index=index2)
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            pser.reindex_like(pdf).sort_index(), kser.reindex_like(kdf).sort_index(),
+        )
+
+    def test_fillna(self):
+        pdf = pd.DataFrame({"x": [np.nan, 2, 3, 4, np.nan, 6], "y": [np.nan, 2, 3, 4, np.nan, 6]})
+        kdf = ps.from_pandas(pdf)
+
+        pser = pdf.x
+        kser = kdf.x
+
+        self.assert_eq(kser.fillna(0), pser.fillna(0))
+        self.assert_eq(kser.fillna(np.nan).fillna(0), pser.fillna(np.nan).fillna(0))
+
+        kser.fillna(0, inplace=True)
+        pser.fillna(0, inplace=True)
+        self.assert_eq(kser, pser)
+        self.assert_eq(kdf, pdf)
+
+        # test considering series does not have NA/NaN values
+        kser.fillna(0, inplace=True)
+        pser.fillna(0, inplace=True)
+        self.assert_eq(kser, pser)
+
+        kser = kdf.x.rename("y")
+        pser = pdf.x.rename("y")
+        kser.fillna(0, inplace=True)
+        pser.fillna(0, inplace=True)
+        self.assert_eq(kser.head(), pser.head())
+
+        pser = pd.Series([1, 2, 3, 4, 5, 6], name="x")
+        kser = ps.from_pandas(pser)
+
+        pser.loc[3] = np.nan
+        kser.loc[3] = np.nan
+
+        self.assert_eq(kser.fillna(0), pser.fillna(0))
+        self.assert_eq(kser.fillna(method="ffill"), pser.fillna(method="ffill"))
+        self.assert_eq(kser.fillna(method="bfill"), pser.fillna(method="bfill"))
+
+        # inplace fillna on non-nullable column
+        pdf = pd.DataFrame({"a": [1, 2, None], "b": [1, 2, 3]})
+        kdf = ps.from_pandas(pdf)
+
+        pser = pdf.b
+        kser = kdf.b
+
+        self.assert_eq(kser.fillna(0), pser.fillna(0))
+        self.assert_eq(kser.fillna(np.nan).fillna(0), pser.fillna(np.nan).fillna(0))
+
+        kser.fillna(0, inplace=True)
+        pser.fillna(0, inplace=True)
+        self.assert_eq(kser, pser)
+        self.assert_eq(kdf, pdf)
+
+    def test_dropna(self):
+        pdf = pd.DataFrame({"x": [np.nan, 2, 3, 4, np.nan, 6]})
+        kdf = ps.from_pandas(pdf)
+
+        pser = pdf.x
+        kser = kdf.x
+
+        self.assert_eq(kser.dropna(), pser.dropna())
+
+        pser.dropna(inplace=True)
+        kser.dropna(inplace=True)
+        self.assert_eq(kser, pser)
+        self.assert_eq(kdf, pdf)
+
+    def test_nunique(self):
+        pser = pd.Series([1, 2, 1, np.nan])
+        kser = ps.from_pandas(pser)
+
+        # Assert NaNs are dropped by default
+        nunique_result = kser.nunique()
+        self.assertEqual(nunique_result, 2)
+        self.assert_eq(nunique_result, pser.nunique())
+
+        # Assert including NaN values
+        nunique_result = kser.nunique(dropna=False)
+        self.assertEqual(nunique_result, 3)
+        self.assert_eq(nunique_result, pser.nunique(dropna=False))
+
+        # Assert approximate counts
+        self.assertEqual(ps.Series(range(100)).nunique(approx=True), 103)
+        self.assertEqual(ps.Series(range(100)).nunique(approx=True, rsd=0.01), 100)
+
+    def test_value_counts(self):
+        # this is also containing test for Index & MultiIndex
+        pser = pd.Series(
+            [1, 2, 1, 3, 3, np.nan, 1, 4, 2, np.nan, 3, np.nan, 3, 1, 3],
+            index=[1, 2, 1, 3, 3, np.nan, 1, 4, 2, np.nan, 3, np.nan, 3, 1, 3],
+            name="x",
+        )
+        kser = ps.from_pandas(pser)
+
+        exp = pser.value_counts()
+        res = kser.value_counts()
+        self.assertEqual(res.name, exp.name)
+        self.assert_eq(res, exp)
+
+        self.assert_eq(kser.value_counts(normalize=True), pser.value_counts(normalize=True))
+        self.assert_eq(kser.value_counts(ascending=True), pser.value_counts(ascending=True))
+        self.assert_eq(
+            kser.value_counts(normalize=True, dropna=False),
+            pser.value_counts(normalize=True, dropna=False),
+        )
+        self.assert_eq(
+            kser.value_counts(ascending=True, dropna=False),
+            pser.value_counts(ascending=True, dropna=False),
+        )
+
+        self.assert_eq(
+            kser.index.value_counts(normalize=True), pser.index.value_counts(normalize=True)
+        )
+        self.assert_eq(
+            kser.index.value_counts(ascending=True), pser.index.value_counts(ascending=True)
+        )
+        self.assert_eq(
+            kser.index.value_counts(normalize=True, dropna=False),
+            pser.index.value_counts(normalize=True, dropna=False),
+        )
+        self.assert_eq(
+            kser.index.value_counts(ascending=True, dropna=False),
+            pser.index.value_counts(ascending=True, dropna=False),
+        )
+
+        with self.assertRaisesRegex(
+            NotImplementedError, "value_counts currently does not support bins"
+        ):
+            kser.value_counts(bins=3)
+
+        pser.name = "index"
+        kser.name = "index"
+        self.assert_eq(kser.value_counts(), pser.value_counts())
+
+        # Series from DataFrame
+        pdf = pd.DataFrame({"a": [2, 2, 3], "b": [None, 1, None]})
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.a.value_counts(normalize=True), pdf.a.value_counts(normalize=True))
+        self.assert_eq(kdf.a.value_counts(ascending=True), pdf.a.value_counts(ascending=True))
+        self.assert_eq(
+            kdf.a.value_counts(normalize=True, dropna=False),
+            pdf.a.value_counts(normalize=True, dropna=False),
+        )
+        self.assert_eq(
+            kdf.a.value_counts(ascending=True, dropna=False),
+            pdf.a.value_counts(ascending=True, dropna=False),
+        )
+
+        self.assert_eq(
+            kser.index.value_counts(normalize=True), pser.index.value_counts(normalize=True)
+        )
+        self.assert_eq(
+            kser.index.value_counts(ascending=True), pser.index.value_counts(ascending=True)
+        )
+        self.assert_eq(
+            kser.index.value_counts(normalize=True, dropna=False),
+            pser.index.value_counts(normalize=True, dropna=False),
+        )
+        self.assert_eq(
+            kser.index.value_counts(ascending=True, dropna=False),
+            pser.index.value_counts(ascending=True, dropna=False),
+        )
+
+        # Series with NaN index
+        pser = pd.Series([3, 2, 3, 1, 2, 3], index=[2.0, None, 5.0, 5.0, None, 5.0])
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(kser.value_counts(normalize=True), pser.value_counts(normalize=True))
+        self.assert_eq(kser.value_counts(ascending=True), pser.value_counts(ascending=True))
+        self.assert_eq(
+            kser.value_counts(normalize=True, dropna=False),
+            pser.value_counts(normalize=True, dropna=False),
+        )
+        self.assert_eq(
+            kser.value_counts(ascending=True, dropna=False),
+            pser.value_counts(ascending=True, dropna=False),
+        )
+
+        self.assert_eq(
+            kser.index.value_counts(normalize=True), pser.index.value_counts(normalize=True)
+        )
+        self.assert_eq(
+            kser.index.value_counts(ascending=True), pser.index.value_counts(ascending=True)
+        )
+        self.assert_eq(
+            kser.index.value_counts(normalize=True, dropna=False),
+            pser.index.value_counts(normalize=True, dropna=False),
+        )
+        self.assert_eq(
+            kser.index.value_counts(ascending=True, dropna=False),
+            pser.index.value_counts(ascending=True, dropna=False),
+        )
+
+        # Series with MultiIndex
+        pser.index = pd.MultiIndex.from_tuples(
+            [("x", "a"), ("x", "b"), ("y", "c"), ("x", "a"), ("y", "c"), ("x", "a")]
+        )
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(kser.value_counts(normalize=True), pser.value_counts(normalize=True))
+        self.assert_eq(kser.value_counts(ascending=True), pser.value_counts(ascending=True))
+        self.assert_eq(
+            kser.value_counts(normalize=True, dropna=False),
+            pser.value_counts(normalize=True, dropna=False),
+        )
+        self.assert_eq(
+            kser.value_counts(ascending=True, dropna=False),
+            pser.value_counts(ascending=True, dropna=False),
+        )
+
+        # FIXME: MultiIndex.value_counts returns wrong indices.
+        self.assert_eq(
+            kser.index.value_counts(normalize=True),
+            pser.index.value_counts(normalize=True),
+            almost=True,
+        )
+        self.assert_eq(
+            kser.index.value_counts(ascending=True),
+            pser.index.value_counts(ascending=True),
+            almost=True,
+        )
+        self.assert_eq(
+            kser.index.value_counts(normalize=True, dropna=False),
+            pser.index.value_counts(normalize=True, dropna=False),
+            almost=True,
+        )
+        self.assert_eq(
+            kser.index.value_counts(ascending=True, dropna=False),
+            pser.index.value_counts(ascending=True, dropna=False),
+            almost=True,
+        )
+
+        # Series with MultiIndex some of index has NaN
+        pser.index = pd.MultiIndex.from_tuples(
+            [("x", "a"), ("x", None), ("y", "c"), ("x", "a"), ("y", "c"), ("x", "a")]
+        )
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(kser.value_counts(normalize=True), pser.value_counts(normalize=True))
+        self.assert_eq(kser.value_counts(ascending=True), pser.value_counts(ascending=True))
+        self.assert_eq(
+            kser.value_counts(normalize=True, dropna=False),
+            pser.value_counts(normalize=True, dropna=False),
+        )
+        self.assert_eq(
+            kser.value_counts(ascending=True, dropna=False),
+            pser.value_counts(ascending=True, dropna=False),
+        )
+
+        # FIXME: MultiIndex.value_counts returns wrong indices.
+        self.assert_eq(
+            kser.index.value_counts(normalize=True),
+            pser.index.value_counts(normalize=True),
+            almost=True,
+        )
+        self.assert_eq(
+            kser.index.value_counts(ascending=True),
+            pser.index.value_counts(ascending=True),
+            almost=True,
+        )
+        self.assert_eq(
+            kser.index.value_counts(normalize=True, dropna=False),
+            pser.index.value_counts(normalize=True, dropna=False),
+            almost=True,
+        )
+        self.assert_eq(
+            kser.index.value_counts(ascending=True, dropna=False),
+            pser.index.value_counts(ascending=True, dropna=False),
+            almost=True,
+        )
+
+        # Series with MultiIndex some of index is NaN.
+        # This test only available for pandas >= 0.24.
+        if LooseVersion(pd.__version__) >= LooseVersion("0.24"):
+            pser.index = pd.MultiIndex.from_tuples(
+                [("x", "a"), None, ("y", "c"), ("x", "a"), ("y", "c"), ("x", "a")]
+            )
+            kser = ps.from_pandas(pser)
+
+            self.assert_eq(kser.value_counts(normalize=True), pser.value_counts(normalize=True))
+            self.assert_eq(kser.value_counts(ascending=True), pser.value_counts(ascending=True))
+            self.assert_eq(
+                kser.value_counts(normalize=True, dropna=False),
+                pser.value_counts(normalize=True, dropna=False),
+            )
+            self.assert_eq(
+                kser.value_counts(ascending=True, dropna=False),
+                pser.value_counts(ascending=True, dropna=False),
+            )
+
+            # FIXME: MultiIndex.value_counts returns wrong indices.
+            self.assert_eq(
+                kser.index.value_counts(normalize=True),
+                pser.index.value_counts(normalize=True),
+                almost=True,
+            )
+            self.assert_eq(
+                kser.index.value_counts(ascending=True),
+                pser.index.value_counts(ascending=True),
+                almost=True,
+            )
+            self.assert_eq(
+                kser.index.value_counts(normalize=True, dropna=False),
+                pser.index.value_counts(normalize=True, dropna=False),
+                almost=True,
+            )
+            self.assert_eq(
+                kser.index.value_counts(ascending=True, dropna=False),
+                pser.index.value_counts(ascending=True, dropna=False),
+                almost=True,
+            )
+
+    def test_nsmallest(self):
+        sample_lst = [1, 2, 3, 4, np.nan, 6]
+        pser = pd.Series(sample_lst, name="x")
+        kser = ps.Series(sample_lst, name="x")
+        self.assert_eq(kser.nsmallest(n=3), pser.nsmallest(n=3))
+        self.assert_eq(kser.nsmallest(), pser.nsmallest())
+        self.assert_eq((kser + 1).nsmallest(), (pser + 1).nsmallest())
+
+    def test_nlargest(self):
+        sample_lst = [1, 2, 3, 4, np.nan, 6]
+        pser = pd.Series(sample_lst, name="x")
+        kser = ps.Series(sample_lst, name="x")
+        self.assert_eq(kser.nlargest(n=3), pser.nlargest(n=3))
+        self.assert_eq(kser.nlargest(), pser.nlargest())
+        self.assert_eq((kser + 1).nlargest(), (pser + 1).nlargest())
+
+    def test_isnull(self):
+        pser = pd.Series([1, 2, 3, 4, np.nan, 6], name="x")
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(kser.notnull(), pser.notnull())
+        self.assert_eq(kser.isnull(), pser.isnull())
+
+        pser = self.pser
+        kser = self.kser
+
+        self.assert_eq(kser.notnull(), pser.notnull())
+        self.assert_eq(kser.isnull(), pser.isnull())
+
+    def test_all(self):
+        for pser in [
+            pd.Series([True, True], name="x"),
+            pd.Series([True, False], name="x"),
+            pd.Series([0, 1], name="x"),
+            pd.Series([1, 2, 3], name="x"),
+            pd.Series([True, True, None], name="x"),
+            pd.Series([True, False, None], name="x"),
+            pd.Series([], name="x"),
+            pd.Series([np.nan], name="x"),
+        ]:
+            kser = ps.from_pandas(pser)
+            self.assert_eq(kser.all(), pser.all())
+
+        pser = pd.Series([1, 2, 3, 4], name="x")
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq((kser % 2 == 0).all(), (pser % 2 == 0).all())
+
+        with self.assertRaisesRegex(
+            NotImplementedError, 'axis should be either 0 or "index" currently.'
+        ):
+            kser.all(axis=1)
+
+    def test_any(self):
+        for pser in [
+            pd.Series([False, False], name="x"),
+            pd.Series([True, False], name="x"),
+            pd.Series([0, 1], name="x"),
+            pd.Series([1, 2, 3], name="x"),
+            pd.Series([True, True, None], name="x"),
+            pd.Series([True, False, None], name="x"),
+            pd.Series([], name="x"),
+            pd.Series([np.nan], name="x"),
+        ]:
+            kser = ps.from_pandas(pser)
+            self.assert_eq(kser.any(), pser.any())
+
+        pser = pd.Series([1, 2, 3, 4], name="x")
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq((kser % 2 == 0).any(), (pser % 2 == 0).any())
+
+        with self.assertRaisesRegex(
+            NotImplementedError, 'axis should be either 0 or "index" currently.'
+        ):
+            kser.any(axis=1)
+
+    def test_reset_index(self):
+        pdf = pd.DataFrame({"foo": [1, 2, 3, 4]}, index=pd.Index(["a", "b", "c", "d"], name="idx"))
+        kdf = ps.from_pandas(pdf)
+
+        pser = pdf.foo
+        kser = kdf.foo
+
+        self.assert_eq(kser.reset_index(), pser.reset_index())
+        self.assert_eq(kser.reset_index(name="values"), pser.reset_index(name="values"))
+        self.assert_eq(kser.reset_index(drop=True), pser.reset_index(drop=True))
+
+        # inplace
+        kser.reset_index(drop=True, inplace=True)
+        pser.reset_index(drop=True, inplace=True)
+        self.assert_eq(kser, pser)
+        self.assert_eq(kdf, pdf)
+
+    def test_reset_index_with_default_index_types(self):
+        pser = pd.Series([1, 2, 3], name="0", index=np.random.rand(3))
+        kser = ps.from_pandas(pser)
+
+        with ps.option_context("compute.default_index_type", "sequence"):
+            self.assert_eq(kser.reset_index(), pser.reset_index())
+
+        with ps.option_context("compute.default_index_type", "distributed-sequence"):
+            # the order might be changed.
+            self.assert_eq(kser.reset_index().sort_index(), pser.reset_index())
+
+        with ps.option_context("compute.default_index_type", "distributed"):
+            # the index is different.
+            self.assert_eq(
+                kser.reset_index().to_pandas().reset_index(drop=True), pser.reset_index()
+            )
+
+    def test_index_to_series_reset_index(self):
+        def check(kser, pser):
+            self.assert_eq(kser.reset_index(), pser.reset_index())
+            self.assert_eq(kser.reset_index(drop=True), pser.reset_index(drop=True))
+
+            pser.reset_index(drop=True, inplace=True)
+            kser.reset_index(drop=True, inplace=True)
+            self.assert_eq(kser, pser)
+
+        pdf = pd.DataFrame(
+            {"a": [1, 2, 3, 4, 5, 6, 7, 8, 9], "b": [4, 5, 6, 3, 2, 1, 0, 0, 0]},
+            index=np.random.rand(9),
+        )
+        kdf = ps.from_pandas(pdf)
+        check(kdf.index.to_series(), pdf.index.to_series())
+        check(kdf.index.to_series(name="a"), pdf.index.to_series(name="a"))
+        check(kdf.index.to_series(name=("x", "a")), pdf.index.to_series(name=("x", "a")))
+
+    def test_sort_values(self):
+        pdf = pd.DataFrame({"x": [1, 2, 3, 4, 5, None, 7]})
+        kdf = ps.from_pandas(pdf)
+
+        pser = pdf.x
+        kser = kdf.x
+
+        self.assert_eq(kser.sort_values(), pser.sort_values())
+        self.assert_eq(kser.sort_values(ascending=False), pser.sort_values(ascending=False))
+        self.assert_eq(kser.sort_values(na_position="first"), pser.sort_values(na_position="first"))
+
+        self.assertRaises(ValueError, lambda: kser.sort_values(na_position="invalid"))
+
+        # inplace
+        # pandas raises an exception when the Series is derived from DataFrame
+        kser.sort_values(inplace=True)
+        self.assert_eq(kser, pser.sort_values())
+        self.assert_eq(kdf, pdf)
+
+        pser = pdf.x.copy()
+        kser = kdf.x.copy()
+
+        kser.sort_values(inplace=True)
+        pser.sort_values(inplace=True)
+        self.assert_eq(kser, pser)
+        self.assert_eq(kdf, pdf)
+
+    def test_sort_index(self):
+        pdf = pd.DataFrame({"x": [2, 1, np.nan]}, index=["b", "a", np.nan])
+        kdf = ps.from_pandas(pdf)
+
+        pser = pdf.x
+        kser = kdf.x
+
+        # Assert invalid parameters
+        self.assertRaises(NotImplementedError, lambda: kser.sort_index(axis=1))
+        self.assertRaises(NotImplementedError, lambda: kser.sort_index(kind="mergesort"))
+        self.assertRaises(ValueError, lambda: kser.sort_index(na_position="invalid"))
+
+        # Assert default behavior without parameters
+        self.assert_eq(kser.sort_index(), pser.sort_index())
+        # Assert sorting descending
+        self.assert_eq(kser.sort_index(ascending=False), pser.sort_index(ascending=False))
+        # Assert sorting NA indices first
+        self.assert_eq(kser.sort_index(na_position="first"), pser.sort_index(na_position="first"))
+
+        # Assert sorting inplace
+        # pandas sorts pdf.x by the index and update the column only
+        # when the Series is derived from DataFrame.
+        kser.sort_index(inplace=True)
+        self.assert_eq(kser, pser.sort_index())
+        self.assert_eq(kdf, pdf)
+
+        pser = pdf.x.copy()
+        kser = kdf.x.copy()
+
+        kser.sort_index(inplace=True)
+        pser.sort_index(inplace=True)
+        self.assert_eq(kser, pser)
+        self.assert_eq(kdf, pdf)
+
+        # Assert multi-indices
+        pser = pd.Series(range(4), index=[["b", "b", "a", "a"], [1, 0, 1, 0]], name="0")
+        kser = ps.from_pandas(pser)
+        self.assert_eq(kser.sort_index(), pser.sort_index())
+        self.assert_eq(kser.sort_index(level=[1, 0]), pser.sort_index(level=[1, 0]))
+
+        self.assert_eq(kser.reset_index().sort_index(), pser.reset_index().sort_index())
+
+    def test_to_datetime(self):
+        pser = pd.Series(["3/11/2000", "3/12/2000", "3/13/2000"] * 100)
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(
+            pd.to_datetime(pser, infer_datetime_format=True),
+            ps.to_datetime(kser, infer_datetime_format=True),
+        )
+
+    def test_missing(self):
+        kser = self.kser
+
+        missing_functions = inspect.getmembers(MissingPandasLikeSeries, inspect.isfunction)
+        unsupported_functions = [
+            name for (name, type_) in missing_functions if type_.__name__ == "unsupported_function"
+        ]
+        for name in unsupported_functions:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "method.*Series.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kser, name)()
+
+        deprecated_functions = [
+            name for (name, type_) in missing_functions if type_.__name__ == "deprecated_function"
+        ]
+        for name in deprecated_functions:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "method.*Series.*{}.*is deprecated".format(name)
+            ):
+                getattr(kser, name)()
+
+        missing_properties = inspect.getmembers(
+            MissingPandasLikeSeries, lambda o: isinstance(o, property)
+        )
+        unsupported_properties = [
+            name
+            for (name, type_) in missing_properties
+            if type_.fget.__name__ == "unsupported_property"
+        ]
+        for name in unsupported_properties:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "property.*Series.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kser, name)
+        deprecated_properties = [
+            name
+            for (name, type_) in missing_properties
+            if type_.fget.__name__ == "deprecated_property"
+        ]
+        for name in deprecated_properties:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "property.*Series.*{}.*is deprecated".format(name)
+            ):
+                getattr(kser, name)
+
+    def test_clip(self):
+        pser = pd.Series([0, 2, 4], index=np.random.rand(3))
+        kser = ps.from_pandas(pser)
+
+        # Assert list-like values are not accepted for 'lower' and 'upper'
+        msg = "List-like value are not supported for 'lower' and 'upper' at the moment"
+        with self.assertRaises(TypeError, msg=msg):
+            kser.clip(lower=[1])
+        with self.assertRaises(TypeError, msg=msg):
+            kser.clip(upper=[1])
+
+        # Assert no lower or upper
+        self.assert_eq(kser.clip(), pser.clip())
+        # Assert lower only
+        self.assert_eq(kser.clip(1), pser.clip(1))
+        # Assert upper only
+        self.assert_eq(kser.clip(upper=3), pser.clip(upper=3))
+        # Assert lower and upper
+        self.assert_eq(kser.clip(1, 3), pser.clip(1, 3))
+
+        # Assert behavior on string values
+        str_kser = ps.Series(["a", "b", "c"])
+        self.assert_eq(str_kser.clip(1, 3), str_kser)
+
+    def test_compare(self):
+        if LooseVersion(pd.__version__) >= LooseVersion("1.1"):
+            pser = pd.Series([1, 2])
+            kser = ps.from_pandas(pser)
+
+            res_kdf = kser.compare(kser)
+            self.assertTrue(res_kdf.empty)
+            self.assert_eq(res_kdf.columns, pd.Index(["self", "other"]))
+
+            self.assert_eq(pser.compare(pser + 1).sort_index(), kser.compare(kser + 1).sort_index())
+
+            pser = pd.Series([1, 2], index=["x", "y"])
+            kser = ps.from_pandas(pser)
+            self.assert_eq(pser.compare(pser + 1).sort_index(), kser.compare(kser + 1).sort_index())
+        else:
+            kser = ps.Series([1, 2])
+            res_kdf = kser.compare(kser)
+            self.assertTrue(res_kdf.empty)
+            self.assert_eq(res_kdf.columns, pd.Index(["self", "other"]))
+            expected = ps.DataFrame([[1, 2], [2, 3]], columns=["self", "other"])
+            self.assert_eq(expected, kser.compare(kser + 1).sort_index())
+
+            kser = ps.Series([1, 2], index=["x", "y"])
+            expected = ps.DataFrame([[1, 2], [2, 3]], index=["x", "y"], columns=["self", "other"])
+            self.assert_eq(expected, kser.compare(kser + 1).sort_index())
+
+    def test_is_unique(self):
+        # We can't use pandas' is_unique for comparison. pandas 0.23 ignores None
+        pser = pd.Series([1, 2, 2, None, None])
+        kser = ps.from_pandas(pser)
+        self.assertEqual(False, kser.is_unique)
+        self.assertEqual(False, (kser + 1).is_unique)
+
+        pser = pd.Series([1, None, None])
+        kser = ps.from_pandas(pser)
+        self.assertEqual(False, kser.is_unique)
+        self.assertEqual(False, (kser + 1).is_unique)
+
+        pser = pd.Series([1])
+        kser = ps.from_pandas(pser)
+        self.assertEqual(pser.is_unique, kser.is_unique)
+        self.assertEqual((pser + 1).is_unique, (kser + 1).is_unique)
+
+        pser = pd.Series([1, 1, 1])
+        kser = ps.from_pandas(pser)
+        self.assertEqual(pser.is_unique, kser.is_unique)
+        self.assertEqual((pser + 1).is_unique, (kser + 1).is_unique)
+
+    def test_to_list(self):
+        self.assert_eq(self.kser.tolist(), self.pser.tolist())
+
+    def test_append(self):
+        pser1 = pd.Series([1, 2, 3], name="0")
+        pser2 = pd.Series([4, 5, 6], name="0")
+        pser3 = pd.Series([4, 5, 6], index=[3, 4, 5], name="0")
+        kser1 = ps.from_pandas(pser1)
+        kser2 = ps.from_pandas(pser2)
+        kser3 = ps.from_pandas(pser3)
+
+        self.assert_eq(kser1.append(kser2), pser1.append(pser2))
+        self.assert_eq(kser1.append(kser3), pser1.append(pser3))
+        self.assert_eq(
+            kser1.append(kser2, ignore_index=True), pser1.append(pser2, ignore_index=True)
+        )
+
+        kser1.append(kser3, verify_integrity=True)
+        msg = "Indices have overlapping values"
+        with self.assertRaises(ValueError, msg=msg):
+            kser1.append(kser2, verify_integrity=True)
+
+    def test_map(self):
+        pser = pd.Series(["cat", "dog", None, "rabbit"])
+        kser = ps.from_pandas(pser)
+        # Currently Koalas doesn't return NaN as pandas does.
+        self.assert_eq(kser.map({}), pser.map({}).replace({pd.np.nan: None}))
+
+        d = defaultdict(lambda: "abc")
+        self.assertTrue("abc" in repr(kser.map(d)))
+        self.assert_eq(kser.map(d), pser.map(d))
+
+        def tomorrow(date) -> datetime:
+            return date + timedelta(days=1)
+
+        pser = pd.Series([datetime(2019, 10, 24)])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(kser.map(tomorrow), pser.map(tomorrow))
+
+    def test_add_prefix(self):
+        pser = pd.Series([1, 2, 3, 4], name="0")
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.add_prefix("item_"), kser.add_prefix("item_"))
+
+        pser = pd.Series(
+            [1, 2, 3],
+            name="0",
+            index=pd.MultiIndex.from_tuples([("A", "X"), ("A", "Y"), ("B", "X")]),
+        )
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.add_prefix("item_"), kser.add_prefix("item_"))
+
+    def test_add_suffix(self):
+        pser = pd.Series([1, 2, 3, 4], name="0")
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.add_suffix("_item"), kser.add_suffix("_item"))
+
+        pser = pd.Series(
+            [1, 2, 3],
+            name="0",
+            index=pd.MultiIndex.from_tuples([("A", "X"), ("A", "Y"), ("B", "X")]),
+        )
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.add_suffix("_item"), kser.add_suffix("_item"))
+
+    def test_cummin(self):
+        pser = pd.Series([1.0, None, 0.0, 4.0, 9.0])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.cummin(), kser.cummin())
+        self.assert_eq(pser.cummin(skipna=False), kser.cummin(skipna=False))
+        self.assert_eq(pser.cummin().sum(), kser.cummin().sum())
+
+        # with reversed index
+        pser.index = [4, 3, 2, 1, 0]
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.cummin(), kser.cummin())
+        self.assert_eq(pser.cummin(skipna=False), kser.cummin(skipna=False))
+
+    def test_cummax(self):
+        pser = pd.Series([1.0, None, 0.0, 4.0, 9.0])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.cummax(), kser.cummax())
+        self.assert_eq(pser.cummax(skipna=False), kser.cummax(skipna=False))
+        self.assert_eq(pser.cummax().sum(), kser.cummax().sum())
+
+        # with reversed index
+        pser.index = [4, 3, 2, 1, 0]
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.cummax(), kser.cummax())
+        self.assert_eq(pser.cummax(skipna=False), kser.cummax(skipna=False))
+
+    def test_cumsum(self):
+        pser = pd.Series([1.0, None, 0.0, 4.0, 9.0])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.cumsum(), kser.cumsum())
+        self.assert_eq(pser.cumsum(skipna=False), kser.cumsum(skipna=False))
+        self.assert_eq(pser.cumsum().sum(), kser.cumsum().sum())
+
+        # with reversed index
+        pser.index = [4, 3, 2, 1, 0]
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.cumsum(), kser.cumsum())
+        self.assert_eq(pser.cumsum(skipna=False), kser.cumsum(skipna=False))
+
+        # bool
+        pser = pd.Series([True, True, False, True])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.cumsum().astype(int), kser.cumsum())
+        self.assert_eq(pser.cumsum(skipna=False).astype(int), kser.cumsum(skipna=False))
+
+    def test_cumprod(self):
+        pser = pd.Series([1.0, None, 1.0, 4.0, 9.0])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.cumprod(), kser.cumprod())
+        self.assert_eq(pser.cumprod(skipna=False), kser.cumprod(skipna=False))
+        self.assert_eq(pser.cumprod().sum(), kser.cumprod().sum())
+
+        # with integer type
+        pser = pd.Series([1, 10, 1, 4, 9])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.cumprod(), kser.cumprod())
+        self.assert_eq(pser.cumprod(skipna=False), kser.cumprod(skipna=False))
+        self.assert_eq(pser.cumprod().sum(), kser.cumprod().sum())
+
+        # with reversed index
+        pser.index = [4, 3, 2, 1, 0]
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.cumprod(), kser.cumprod())
+        self.assert_eq(pser.cumprod(skipna=False), kser.cumprod(skipna=False))
+
+        # including zero
+        pser = pd.Series([1, 2, 0, 3])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.cumprod(), kser.cumprod())
+        self.assert_eq(pser.cumprod(skipna=False), kser.cumprod(skipna=False))
+
+        # including negative values
+        pser = pd.Series([1, -1, -2])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.cumprod(), kser.cumprod())
+        self.assert_eq(pser.cumprod(skipna=False), kser.cumprod(skipna=False))
+
+        # bool
+        pser = pd.Series([True, True, False, True])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.cumprod(), kser.cumprod())
+        self.assert_eq(pser.cumprod(skipna=False).astype(int), kser.cumprod(skipna=False))
+
+    def test_median(self):
+        with self.assertRaisesRegex(TypeError, "accuracy must be an integer; however"):
+            ps.Series([24.0, 21.0, 25.0, 33.0, 26.0]).median(accuracy="a")
+
+    def test_rank(self):
+        pser = pd.Series([1, 2, 3, 1], name="x")
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.rank(), kser.rank().sort_index())
+        self.assert_eq(pser.rank().sum(), kser.rank().sum())
+        self.assert_eq(pser.rank(ascending=False), kser.rank(ascending=False).sort_index())
+        self.assert_eq(pser.rank(method="min"), kser.rank(method="min").sort_index())
+        self.assert_eq(pser.rank(method="max"), kser.rank(method="max").sort_index())
+        self.assert_eq(pser.rank(method="first"), kser.rank(method="first").sort_index())
+        self.assert_eq(pser.rank(method="dense"), kser.rank(method="dense").sort_index())
+
+        msg = "method must be one of 'average', 'min', 'max', 'first', 'dense'"
+        with self.assertRaisesRegex(ValueError, msg):
+            kser.rank(method="nothing")
+
+    def test_round(self):
+        pser = pd.Series([0.028208, 0.038683, 0.877076], name="x")
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.round(2), kser.round(2))
+        msg = "decimals must be an integer"
+        with self.assertRaisesRegex(TypeError, msg):
+            kser.round(1.5)
+
+    def test_quantile(self):
+        pser = pd.Series([])
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(kser.quantile(0.5), pser.quantile(0.5))
+        self.assert_eq(kser.quantile([0.25, 0.5, 0.75]), pser.quantile([0.25, 0.5, 0.75]))
+
+        with self.assertRaisesRegex(TypeError, "accuracy must be an integer; however"):
+            ps.Series([24.0, 21.0, 25.0, 33.0, 26.0]).quantile(accuracy="a")
+        with self.assertRaisesRegex(TypeError, "q must be a float or an array of floats;"):
+            ps.Series([24.0, 21.0, 25.0, 33.0, 26.0]).quantile(q="a")
+        with self.assertRaisesRegex(TypeError, "q must be a float or an array of floats;"):
+            ps.Series([24.0, 21.0, 25.0, 33.0, 26.0]).quantile(q=["a"])
+
+        with self.assertRaisesRegex(TypeError, "Could not convert object \\(string\\) to numeric"):
+            ps.Series(["a", "b", "c"]).quantile()
+        with self.assertRaisesRegex(TypeError, "Could not convert object \\(string\\) to numeric"):
+            ps.Series(["a", "b", "c"]).quantile([0.25, 0.5, 0.75])
+
+    def test_idxmax(self):
+        pser = pd.Series(data=[1, 4, 5], index=["A", "B", "C"])
+        kser = ps.Series(pser)
+
+        self.assertEqual(kser.idxmax(), pser.idxmax())
+        self.assertEqual(kser.idxmax(skipna=False), pser.idxmax(skipna=False))
+
+        index = pd.MultiIndex.from_arrays(
+            [["a", "a", "b", "b"], ["c", "d", "e", "f"]], names=("first", "second")
+        )
+        pser = pd.Series(data=[1, 2, 4, 5], index=index)
+        kser = ps.Series(pser)
+
+        self.assertEqual(kser.idxmax(), pser.idxmax())
+        self.assertEqual(kser.idxmax(skipna=False), pser.idxmax(skipna=False))
+
+        kser = ps.Series([])
+        with self.assertRaisesRegex(ValueError, "an empty sequence"):
+            kser.idxmax()
+
+        pser = pd.Series([1, 100, None, 100, 1, 100], index=[10, 3, 5, 2, 1, 8])
+        kser = ps.Series(pser)
+
+        self.assertEqual(kser.idxmax(), pser.idxmax())
+        self.assertEqual(repr(kser.idxmax(skipna=False)), repr(pser.idxmax(skipna=False)))
+
+    def test_idxmin(self):
+        pser = pd.Series(data=[1, 4, 5], index=["A", "B", "C"])
+        kser = ps.Series(pser)
+
+        self.assertEqual(kser.idxmin(), pser.idxmin())
+        self.assertEqual(kser.idxmin(skipna=False), pser.idxmin(skipna=False))
+
+        index = pd.MultiIndex.from_arrays(
+            [["a", "a", "b", "b"], ["c", "d", "e", "f"]], names=("first", "second")
+        )
+        pser = pd.Series(data=[1, 2, 4, 5], index=index)
+        kser = ps.Series(pser)
+
+        self.assertEqual(kser.idxmin(), pser.idxmin())
+        self.assertEqual(kser.idxmin(skipna=False), pser.idxmin(skipna=False))
+
+        kser = ps.Series([])
+        with self.assertRaisesRegex(ValueError, "an empty sequence"):
+            kser.idxmin()
+
+        pser = pd.Series([1, 100, None, 100, 1, 100], index=[10, 3, 5, 2, 1, 8])
+        kser = ps.Series(pser)
+
+        self.assertEqual(kser.idxmin(), pser.idxmin())
+        self.assertEqual(repr(kser.idxmin(skipna=False)), repr(pser.idxmin(skipna=False)))
+
+    def test_shift(self):
+        pser = pd.Series([10, 20, 15, 30, 45], name="x")
+        kser = ps.Series(pser)
+
+        self.assert_eq(kser.shift(2), pser.shift(2))
+        self.assert_eq(kser.shift().shift(-1), pser.shift().shift(-1))
+        self.assert_eq(kser.shift().sum(), pser.shift().sum())
+
+        if LooseVersion(pd.__version__) < LooseVersion("0.24.2"):
+            self.assert_eq(kser.shift(periods=2), pser.shift(periods=2))
+        else:
+            self.assert_eq(kser.shift(periods=2, fill_value=0), pser.shift(periods=2, fill_value=0))
+        with self.assertRaisesRegex(TypeError, "periods should be an int; however"):
+            kser.shift(periods=1.5)
+
+    def test_diff(self):
+        pser = pd.Series([10, 20, 15, 30, 45], name="x")
+        kser = ps.Series(pser)
+
+        self.assert_eq(kser.diff(2), pser.diff(2))
+        self.assert_eq(kser.diff().diff(-1), pser.diff().diff(-1))
+        self.assert_eq(kser.diff().sum(), pser.diff().sum())
+
+    def _test_numeric_astype(self, pser):
+        kser = ps.Series(pser)
+
+        self.assert_eq(kser.astype(int), pser.astype(int))
+        self.assert_eq(kser.astype(np.int), pser.astype(np.int))
+        self.assert_eq(kser.astype(np.int8), pser.astype(np.int8))
+        self.assert_eq(kser.astype(np.int16), pser.astype(np.int16))
+        self.assert_eq(kser.astype(np.int32), pser.astype(np.int32))
+        self.assert_eq(kser.astype(np.int64), pser.astype(np.int64))
+        self.assert_eq(kser.astype(np.byte), pser.astype(np.byte))
+        self.assert_eq(kser.astype("int"), pser.astype("int"))
+        self.assert_eq(kser.astype("int8"), pser.astype("int8"))
+        self.assert_eq(kser.astype("int16"), pser.astype("int16"))
+        self.assert_eq(kser.astype("int32"), pser.astype("int32"))
+        self.assert_eq(kser.astype("int64"), pser.astype("int64"))
+        self.assert_eq(kser.astype("b"), pser.astype("b"))
+        self.assert_eq(kser.astype("byte"), pser.astype("byte"))
+        self.assert_eq(kser.astype("i"), pser.astype("i"))
+        self.assert_eq(kser.astype("long"), pser.astype("long"))
+        self.assert_eq(kser.astype("short"), pser.astype("short"))
+        self.assert_eq(kser.astype(np.float), pser.astype(np.float))
+        self.assert_eq(kser.astype(np.float32), pser.astype(np.float32))
+        self.assert_eq(kser.astype(np.float64), pser.astype(np.float64))
+        self.assert_eq(kser.astype("float"), pser.astype("float"))
+        self.assert_eq(kser.astype("float32"), pser.astype("float32"))
+        self.assert_eq(kser.astype("float64"), pser.astype("float64"))
+        self.assert_eq(kser.astype("double"), pser.astype("double"))
+        self.assert_eq(kser.astype("f"), pser.astype("f"))
+        self.assert_eq(kser.astype(bool), pser.astype(bool))
+        self.assert_eq(kser.astype("bool"), pser.astype("bool"))
+        self.assert_eq(kser.astype("?"), pser.astype("?"))
+        self.assert_eq(kser.astype(np.unicode_), pser.astype(np.unicode_))
+        self.assert_eq(kser.astype("str"), pser.astype("str"))
+        self.assert_eq(kser.astype("U"), pser.astype("U"))
+
+        if extension_dtypes_available:
+            from pandas import Int8Dtype, Int16Dtype, Int32Dtype, Int64Dtype
+
+            self._check_extension(kser.astype("Int8"), pser.astype("Int8"))
+            self._check_extension(kser.astype("Int16"), pser.astype("Int16"))
+            self._check_extension(kser.astype("Int32"), pser.astype("Int32"))
+            self._check_extension(kser.astype("Int64"), pser.astype("Int64"))
+            self._check_extension(kser.astype(Int8Dtype()), pser.astype(Int8Dtype()))
+            self._check_extension(kser.astype(Int16Dtype()), pser.astype(Int16Dtype()))
+            self._check_extension(kser.astype(Int32Dtype()), pser.astype(Int32Dtype()))
+            self._check_extension(kser.astype(Int64Dtype()), pser.astype(Int64Dtype()))
+
+        if extension_object_dtypes_available:
+            from pandas import StringDtype
+
+            if LooseVersion(pd.__version__) >= LooseVersion("1.1"):
+                self._check_extension(kser.astype("string"), pser.astype("string"))
+                self._check_extension(kser.astype(StringDtype()), pser.astype(StringDtype()))
+            else:
+                self._check_extension(
+                    kser.astype("string"),
+                    pd.Series(["10", "20", "15", "30", "45"], name="x", dtype="string"),
+                )
+                self._check_extension(
+                    kser.astype(StringDtype()),
+                    pd.Series(["10", "20", "15", "30", "45"], name="x", dtype=StringDtype()),
+                )
+
+        if extension_float_dtypes_available:
+            from pandas import Float32Dtype, Float64Dtype
+
+            self._check_extension(kser.astype("Float32"), pser.astype("Float32"))
+            self._check_extension(kser.astype("Float64"), pser.astype("Float64"))
+            self._check_extension(kser.astype(Float32Dtype()), pser.astype(Float32Dtype()))
+            self._check_extension(kser.astype(Float64Dtype()), pser.astype(Float64Dtype()))
+
+    def test_astype(self):
+        psers = [pd.Series([10, 20, 15, 30, 45], name="x")]
+
+        if extension_dtypes_available:
+            psers.append(pd.Series([10, 20, 15, 30, 45], name="x", dtype="Int64"))
+        if extension_float_dtypes_available:
+            psers.append(pd.Series([10, 20, 15, 30, 45], name="x", dtype="Float64"))
+
+        for pser in psers:
+            self._test_numeric_astype(pser)
+
+        pser = pd.Series([10, 20, 15, 30, 45, None, np.nan], name="x")
+        kser = ps.Series(pser)
+
+        self.assert_eq(kser.astype(bool), pser.astype(bool))
+        self.assert_eq(kser.astype(str), pser.astype(str))
+
+        pser = pd.Series(["hi", "hi ", " ", " \t", "", None], name="x")
+        kser = ps.Series(pser)
+
+        self.assert_eq(kser.astype(bool), pser.astype(bool))
+        if LooseVersion("1.1.1") <= LooseVersion(pd.__version__) < LooseVersion("1.1.4"):
+            # a pandas bug: https://github.com/databricks/koalas/pull/1818#issuecomment-703961980
+            self.assert_eq(kser.astype(str).tolist(), ["hi", "hi ", " ", " \t", "", "None"])
+        else:
+            self.assert_eq(kser.astype(str), pser.astype(str))
+        self.assert_eq(kser.str.strip().astype(bool), pser.str.strip().astype(bool))
+
+        if extension_object_dtypes_available:
+            from pandas import StringDtype
+
+            self._check_extension(kser.astype("string"), pser.astype("string"))
+            self._check_extension(kser.astype(StringDtype()), pser.astype(StringDtype()))
+
+        pser = pd.Series([True, False, None], name="x")
+        kser = ps.Series(pser)
+
+        self.assert_eq(kser.astype(bool), pser.astype(bool))
+        # Comment out the below test cause because pandas returns `None` or `nan` randomly
+        # self.assert_eq(kser.astype(str), pser.astype(str))
+
+        if extension_object_dtypes_available:
+            from pandas import BooleanDtype, StringDtype
+
+            self._check_extension(kser.astype("boolean"), pser.astype("boolean"))
+            self._check_extension(kser.astype(BooleanDtype()), pser.astype(BooleanDtype()))
+
+            if LooseVersion(pd.__version__) >= LooseVersion("1.1"):
+                self._check_extension(kser.astype("string"), pser.astype("string"))
+                self._check_extension(kser.astype(StringDtype()), pser.astype(StringDtype()))
+            else:
+                self._check_extension(
+                    kser.astype("string"),
+                    pd.Series(["True", "False", None], name="x", dtype="string"),
+                )
+                self._check_extension(
+                    kser.astype(StringDtype()),
+                    pd.Series(["True", "False", None], name="x", dtype=StringDtype()),
+                )
+
+        pser = pd.Series(["2020-10-27 00:00:01", None], name="x")
+        kser = ps.Series(pser)
+
+        self.assert_eq(kser.astype(np.datetime64), pser.astype(np.datetime64))
+        self.assert_eq(kser.astype("datetime64[ns]"), pser.astype("datetime64[ns]"))
+        self.assert_eq(kser.astype("M"), pser.astype("M"))
+        self.assert_eq(kser.astype("M").astype(str), pser.astype("M").astype(str))
+        # Comment out the below test cause because pandas returns `NaT` or `nan` randomly
+        # self.assert_eq(kser.astype("M").dt.date.astype(str), pser.astype("M").dt.date.astype(str))
+
+        if extension_object_dtypes_available:
+            from pandas import StringDtype
+
+            self._check_extension(
+                kser.astype("M").astype("string"), pser.astype("M").astype("string")
+            )
+            self._check_extension(
+                kser.astype("M").astype(StringDtype()), pser.astype("M").astype(StringDtype())
+            )
+
+        with self.assertRaisesRegex(TypeError, "not understood"):
+            kser.astype("int63")
+
+    def test_aggregate(self):
+        pser = pd.Series([10, 20, 15, 30, 45], name="x")
+        kser = ps.Series(pser)
+        msg = "func must be a string or list of strings"
+        with self.assertRaisesRegex(TypeError, msg):
+            kser.aggregate({"x": ["min", "max"]})
+        msg = (
+            "If the given function is a list, it " "should only contains function names as strings."
+        )
+        with self.assertRaisesRegex(ValueError, msg):
+            kser.aggregate(["min", max])
+
+    def test_drop(self):
+        pser = pd.Series([10, 20, 15, 30, 45], name="x")
+        kser = ps.Series(pser)
+
+        self.assert_eq(kser.drop(1), pser.drop(1))
+        self.assert_eq(kser.drop([1, 4]), pser.drop([1, 4]))
+
+        msg = "Need to specify at least one of 'labels' or 'index'"
+        with self.assertRaisesRegex(ValueError, msg):
+            kser.drop()
+        self.assertRaises(KeyError, lambda: kser.drop((0, 1)))
+
+        # For MultiIndex
+        midx = pd.MultiIndex(
+            [["lama", "cow", "falcon"], ["speed", "weight", "length"]],
+            [[0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 1, 2, 0, 1, 2, 0, 1, 2]],
+        )
+        pser = pd.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], index=midx)
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(kser.drop("lama"), pser.drop("lama"))
+        self.assert_eq(kser.drop(labels="weight", level=1), pser.drop(labels="weight", level=1))
+        self.assert_eq(kser.drop(("lama", "weight")), pser.drop(("lama", "weight")))
+        self.assert_eq(
+            kser.drop([("lama", "speed"), ("falcon", "weight")]),
+            pser.drop([("lama", "speed"), ("falcon", "weight")]),
+        )
+        self.assert_eq(kser.drop({"lama": "speed"}), pser.drop({"lama": "speed"}))
+
+        msg = "'level' should be less than the number of indexes"
+        with self.assertRaisesRegex(ValueError, msg):
+            kser.drop(labels="weight", level=2)
+
+        msg = (
+            "If the given index is a list, it "
+            "should only contains names as all tuples or all non tuples "
+            "that contain index names"
+        )
+        with self.assertRaisesRegex(ValueError, msg):
+            kser.drop(["lama", ["cow", "falcon"]])
+
+        msg = "Cannot specify both 'labels' and 'index'"
+        with self.assertRaisesRegex(ValueError, msg):
+            kser.drop("lama", index="cow")
+
+        msg = r"'Key length \(2\) exceeds index depth \(3\)'"
+        with self.assertRaisesRegex(KeyError, msg):
+            kser.drop(("lama", "speed", "x"))
+
+    def test_pop(self):
+        midx = pd.MultiIndex(
+            [["lama", "cow", "falcon"], ["speed", "weight", "length"]],
+            [[0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 1, 2, 0, 1, 2, 0, 1, 2]],
+        )
+        pdf = pd.DataFrame({"x": [45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3]}, index=midx)
+        kdf = ps.from_pandas(pdf)
+
+        pser = pdf.x
+        kser = kdf.x
+
+        self.assert_eq(kser.pop(("lama", "speed")), pser.pop(("lama", "speed")))
+        self.assert_eq(kser, pser)
+        self.assert_eq(kdf, pdf)
+
+        msg = r"'Key length \(3\) exceeds index depth \(2\)'"
+        with self.assertRaisesRegex(KeyError, msg):
+            kser.pop(("lama", "speed", "x"))
+
+    def test_replace(self):
+        pser = pd.Series([10, 20, 15, 30, np.nan], name="x")
+        kser = ps.Series(pser)
+
+        self.assert_eq(kser.replace(), pser.replace())
+        self.assert_eq(kser.replace({}), pser.replace({}))
+
+        self.assert_eq(kser.replace(np.nan, 45), pser.replace(np.nan, 45))
+        self.assert_eq(kser.replace([10, 15], 45), pser.replace([10, 15], 45))
+        self.assert_eq(kser.replace((10, 15), 45), pser.replace((10, 15), 45))
+        self.assert_eq(kser.replace([10, 15], [45, 50]), pser.replace([10, 15], [45, 50]))
+        self.assert_eq(kser.replace((10, 15), (45, 50)), pser.replace((10, 15), (45, 50)))
+
+        msg = "'to_replace' should be one of str, list, tuple, dict, int, float"
+        with self.assertRaisesRegex(TypeError, msg):
+            kser.replace(ps.range(5))
+        msg = "Replacement lists must match in length. Expecting 3 got 2"
+        with self.assertRaisesRegex(ValueError, msg):
+            kser.replace([10, 20, 30], [1, 2])
+        msg = "replace currently not support for regex"
+        with self.assertRaisesRegex(NotImplementedError, msg):
+            kser.replace(r"^1.$", regex=True)
+
+    def test_xs(self):
+        midx = pd.MultiIndex(
+            [["a", "b", "c"], ["lama", "cow", "falcon"], ["speed", "weight", "length"]],
+            [[0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 1, 2, 0, 1, 2, 0, 1, 2]],
+        )
+        pser = pd.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], index=midx)
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(kser.xs(("a", "lama", "speed")), pser.xs(("a", "lama", "speed")))
+
+    def test_duplicates(self):
+        psers = {
+            "test on texts": pd.Series(
+                ["lama", "cow", "lama", "beetle", "lama", "hippo"], name="animal"
+            ),
+            "test on numbers": pd.Series([1, 1, 2, 4, 3]),
+        }
+        keeps = ["first", "last", False]
+
+        for (msg, pser), keep in product(psers.items(), keeps):
+            with self.subTest(msg, keep=keep):
+                kser = ps.Series(pser)
+
+                self.assert_eq(
+                    pser.drop_duplicates(keep=keep).sort_values(),
+                    kser.drop_duplicates(keep=keep).sort_values(),
+                )
+
+    def test_update(self):
+        pser = pd.Series([10, 20, 15, 30, 45], name="x")
+        kser = ps.Series(pser)
+
+        msg = "'other' must be a Series"
+        with self.assertRaisesRegex(TypeError, msg):
+            kser.update(10)
+
+    def test_where(self):
+        pser1 = pd.Series([0, 1, 2, 3, 4])
+        kser1 = ps.from_pandas(pser1)
+
+        self.assert_eq(pser1.where(pser1 > 3), kser1.where(kser1 > 3).sort_index())
+
+    def test_mask(self):
+        pser1 = pd.Series([0, 1, 2, 3, 4])
+        kser1 = ps.from_pandas(pser1)
+
+        self.assert_eq(pser1.mask(pser1 > 3), kser1.mask(kser1 > 3).sort_index())
+
+    def test_truncate(self):
+        pser1 = pd.Series([10, 20, 30, 40, 50, 60, 70], index=[1, 2, 3, 4, 5, 6, 7])
+        kser1 = ps.Series(pser1)
+        pser2 = pd.Series([10, 20, 30, 40, 50, 60, 70], index=[7, 6, 5, 4, 3, 2, 1])
+        kser2 = ps.Series(pser2)
+
+        self.assert_eq(kser1.truncate(), pser1.truncate())
+        self.assert_eq(kser1.truncate(before=2), pser1.truncate(before=2))
+        self.assert_eq(kser1.truncate(after=5), pser1.truncate(after=5))
+        self.assert_eq(kser1.truncate(copy=False), pser1.truncate(copy=False))
+        self.assert_eq(kser1.truncate(2, 5, copy=False), pser1.truncate(2, 5, copy=False))
+        # The bug for these tests has been fixed in pandas 1.1.0.
+        if LooseVersion(pd.__version__) >= LooseVersion("1.1.0"):
+            self.assert_eq(kser2.truncate(4, 6), pser2.truncate(4, 6))
+            self.assert_eq(kser2.truncate(4, 6, copy=False), pser2.truncate(4, 6, copy=False))
+        else:
+            expected_kser = ps.Series([20, 30, 40], index=[6, 5, 4])
+            self.assert_eq(kser2.truncate(4, 6), expected_kser)
+            self.assert_eq(kser2.truncate(4, 6, copy=False), expected_kser)
+
+        kser = ps.Series([10, 20, 30, 40, 50, 60, 70], index=[1, 2, 3, 4, 3, 2, 1])
+        msg = "truncate requires a sorted index"
+        with self.assertRaisesRegex(ValueError, msg):
+            kser.truncate()
+
+        kser = ps.Series([10, 20, 30, 40, 50, 60, 70], index=[1, 2, 3, 4, 5, 6, 7])
+        msg = "Truncate: 2 must be after 5"
+        with self.assertRaisesRegex(ValueError, msg):
+            kser.truncate(5, 2)
+
+    def test_getitem(self):
+        pser = pd.Series([10, 20, 15, 30, 45], ["A", "A", "B", "C", "D"])
+        kser = ps.Series(pser)
+
+        self.assert_eq(kser["A"], pser["A"])
+        self.assert_eq(kser["B"], pser["B"])
+        self.assert_eq(kser[kser > 15], pser[pser > 15])
+
+        # for MultiIndex
+        midx = pd.MultiIndex(
+            [["a", "b", "c"], ["lama", "cow", "falcon"], ["speed", "weight", "length"]],
+            [[0, 0, 0, 0, 0, 0, 1, 1, 1], [0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 0, 0, 0, 1, 2, 0, 1, 2]],
+        )
+        pser = pd.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], name="0", index=midx)
+        kser = ps.Series(pser)
+
+        self.assert_eq(kser["a"], pser["a"])
+        self.assert_eq(kser["a", "lama"], pser["a", "lama"])
+        self.assert_eq(kser[kser > 1.5], pser[pser > 1.5])
+
+        msg = r"'Key length \(4\) exceeds index depth \(3\)'"
+        with self.assertRaisesRegex(KeyError, msg):
+            kser[("a", "lama", "speed", "x")]
+
+    def test_keys(self):
+        midx = pd.MultiIndex(
+            [["lama", "cow", "falcon"], ["speed", "weight", "length"]],
+            [[0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 1, 2, 0, 1, 2, 0, 1, 2]],
+        )
+        pser = pd.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], index=midx)
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(kser.keys(), pser.keys())
+
+    def test_index(self):
+        # to check setting name of Index properly.
+        idx = pd.Index([1, 2, 3, 4, 5, 6, 7, 8, 9])
+        pser = pd.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], index=idx)
+        kser = ps.from_pandas(pser)
+
+        kser.name = "koalas"
+        pser.name = "koalas"
+        self.assert_eq(kser.index.name, pser.index.name)
+
+        # for check setting names of MultiIndex properly.
+        kser.names = ["hello", "koalas"]
+        pser.names = ["hello", "koalas"]
+        self.assert_eq(kser.index.names, pser.index.names)
+
+    def test_pct_change(self):
+        pser = pd.Series([90, 91, 85], index=[2, 4, 1])
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(kser.pct_change(), pser.pct_change(), check_exact=False)
+        self.assert_eq(kser.pct_change().sum(), pser.pct_change().sum(), almost=True)
+        self.assert_eq(kser.pct_change(periods=2), pser.pct_change(periods=2), check_exact=False)
+        self.assert_eq(kser.pct_change(periods=-1), pser.pct_change(periods=-1), check_exact=False)
+        self.assert_eq(kser.pct_change(periods=-100000000), pser.pct_change(periods=-100000000))
+        self.assert_eq(kser.pct_change(periods=100000000), pser.pct_change(periods=100000000))
+
+        # for MultiIndex
+        midx = pd.MultiIndex(
+            [["lama", "cow", "falcon"], ["speed", "weight", "length"]],
+            [[0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 1, 2, 0, 1, 2, 0, 1, 2]],
+        )
+        pser = pd.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], index=midx)
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(kser.pct_change(), pser.pct_change(), check_exact=False)
+        self.assert_eq(kser.pct_change().sum(), pser.pct_change().sum(), almost=True)
+        self.assert_eq(kser.pct_change(periods=2), pser.pct_change(periods=2), check_exact=False)
+        self.assert_eq(kser.pct_change(periods=-1), pser.pct_change(periods=-1), check_exact=False)
+        self.assert_eq(kser.pct_change(periods=-100000000), pser.pct_change(periods=-100000000))
+        self.assert_eq(kser.pct_change(periods=100000000), pser.pct_change(periods=100000000))
+
+    def test_axes(self):
+        pser = pd.Series([90, 91, 85], index=[2, 4, 1])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(kser.axes, pser.axes)
+
+        # for MultiIndex
+        midx = pd.MultiIndex(
+            [["lama", "cow", "falcon"], ["speed", "weight", "length"]],
+            [[0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 1, 2, 0, 1, 2, 0, 1, 2]],
+        )
+        pser = pd.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], index=midx)
+        kser = ps.from_pandas(pser)
+        self.assert_eq(kser.axes, pser.axes)
+
+    def test_udt(self):
+        sparse_values = {0: 0.1, 1: 1.1}
+        sparse_vector = SparseVector(len(sparse_values), sparse_values)
+        pser = pd.Series([sparse_vector])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(kser, pser)
+
+    def test_repeat(self):
+        pser = pd.Series(["a", "b", "c"], name="0", index=np.random.rand(3))
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(kser.repeat(3).sort_index(), pser.repeat(3).sort_index())
+        self.assert_eq(kser.repeat(0).sort_index(), pser.repeat(0).sort_index())
+
+        self.assertRaises(ValueError, lambda: kser.repeat(-1))
+        self.assertRaises(TypeError, lambda: kser.repeat("abc"))
+
+        pdf = pd.DataFrame({"a": ["a", "b", "c"], "rep": [10, 20, 30]}, index=np.random.rand(3))
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.a.repeat(kdf.rep).sort_index(), pdf.a.repeat(pdf.rep).sort_index())
+
+    def test_take(self):
+        pser = pd.Series([100, 200, 300, 400, 500], name="Koalas")
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(kser.take([0, 2, 4]).sort_values(), pser.take([0, 2, 4]).sort_values())
+        self.assert_eq(
+            kser.take(range(0, 5, 2)).sort_values(), pser.take(range(0, 5, 2)).sort_values()
+        )
+        self.assert_eq(kser.take([-4, -2, 0]).sort_values(), pser.take([-4, -2, 0]).sort_values())
+        self.assert_eq(
+            kser.take(range(-2, 1, 2)).sort_values(), pser.take(range(-2, 1, 2)).sort_values()
+        )
+
+        # Checking the type of indices.
+        self.assertRaises(TypeError, lambda: kser.take(1))
+        self.assertRaises(TypeError, lambda: kser.take("1"))
+        self.assertRaises(TypeError, lambda: kser.take({1, 2}))
+        self.assertRaises(TypeError, lambda: kser.take({1: None, 2: None}))
+
+    def test_divmod(self):
+        pser = pd.Series([100, None, 300, None, 500], name="Koalas")
+        kser = ps.from_pandas(pser)
+
+        if LooseVersion(pd.__version__) >= LooseVersion("1.0.0"):
+            kdiv, kmod = kser.divmod(-100)
+            pdiv, pmod = pser.divmod(-100)
+            self.assert_eq(kdiv, pdiv)
+            self.assert_eq(kmod, pmod)
+
+            kdiv, kmod = kser.divmod(100)
+            pdiv, pmod = pser.divmod(100)
+            self.assert_eq(kdiv, pdiv)
+            self.assert_eq(kmod, pmod)
+        elif LooseVersion(pd.__version__) < LooseVersion("1.0.0"):
+            kdiv, kmod = kser.divmod(-100)
+            pdiv, pmod = pser.floordiv(-100), pser.mod(-100)
+            self.assert_eq(kdiv, pdiv)
+            self.assert_eq(kmod, pmod)
+
+            kdiv, kmod = kser.divmod(100)
+            pdiv, pmod = pser.floordiv(100), pser.mod(100)
+            self.assert_eq(kdiv, pdiv)
+            self.assert_eq(kmod, pmod)
+
+    def test_rdivmod(self):
+        pser = pd.Series([100, None, 300, None, 500])
+        kser = ps.from_pandas(pser)
+
+        if LooseVersion(pd.__version__) >= LooseVersion("1.0.0"):
+            krdiv, krmod = kser.rdivmod(-100)
+            prdiv, prmod = pser.rdivmod(-100)
+            self.assert_eq(krdiv, prdiv)
+            self.assert_eq(krmod, prmod)
+
+            krdiv, krmod = kser.rdivmod(100)
+            prdiv, prmod = pser.rdivmod(100)
+            self.assert_eq(krdiv, prdiv)
+            self.assert_eq(krmod, prmod)
+        elif LooseVersion(pd.__version__) < LooseVersion("1.0.0"):
+            krdiv, krmod = kser.rdivmod(-100)
+            prdiv, prmod = pser.rfloordiv(-100), pser.rmod(-100)
+            self.assert_eq(krdiv, prdiv)
+            self.assert_eq(krmod, prmod)
+
+            krdiv, krmod = kser.rdivmod(100)
+            prdiv, prmod = pser.rfloordiv(100), pser.rmod(100)
+            self.assert_eq(krdiv, prdiv)
+            self.assert_eq(krmod, prmod)
+
+    def test_mod(self):
+        pser = pd.Series([100, None, -300, None, 500, -700], name="Koalas")
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(kser.mod(-150), pser.mod(-150))
+        self.assert_eq(kser.mod(0), pser.mod(0))
+        self.assert_eq(kser.mod(150), pser.mod(150))
+
+        pdf = pd.DataFrame({"a": [100, None, -300, None, 500, -700], "b": [150] * 6})
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kdf.a.mod(kdf.b), pdf.a.mod(pdf.b))
+
+    def test_mode(self):
+        pser = pd.Series([0, 0, 1, 1, 1, np.nan, np.nan, np.nan])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(kser.mode(), pser.mode())
+        if LooseVersion(pd.__version__) >= LooseVersion("0.24"):
+            # The `dropna` argument is added in pandas 0.24.
+            self.assert_eq(
+                kser.mode(dropna=False).sort_values().reset_index(drop=True),
+                pser.mode(dropna=False).sort_values().reset_index(drop=True),
+            )
+
+        pser.name = "x"
+        kser = ps.from_pandas(pser)
+        self.assert_eq(kser.mode(), pser.mode())
+        if LooseVersion(pd.__version__) >= LooseVersion("0.24"):
+            # The `dropna` argument is added in pandas 0.24.
+            self.assert_eq(
+                kser.mode(dropna=False).sort_values().reset_index(drop=True),
+                pser.mode(dropna=False).sort_values().reset_index(drop=True),
+            )
+
+    def test_rmod(self):
+        pser = pd.Series([100, None, -300, None, 500, -700], name="Koalas")
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(kser.rmod(-150), pser.rmod(-150))
+        self.assert_eq(kser.rmod(0), pser.rmod(0))
+        self.assert_eq(kser.rmod(150), pser.rmod(150))
+
+        pdf = pd.DataFrame({"a": [100, None, -300, None, 500, -700], "b": [150] * 6})
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kdf.a.rmod(kdf.b), pdf.a.rmod(pdf.b))
+
+    def test_asof(self):
+        pser = pd.Series([1, 2, np.nan, 4], index=[10, 20, 30, 40], name="Koalas")
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(kser.asof(20), pser.asof(20))
+        self.assert_eq(kser.asof([5, 20]).sort_index(), pser.asof([5, 20]).sort_index())
+        self.assert_eq(kser.asof(100), pser.asof(100))
+        self.assert_eq(repr(kser.asof(-100)), repr(pser.asof(-100)))
+        self.assert_eq(kser.asof([-100, 100]).sort_index(), pser.asof([-100, 100]).sort_index())
+
+        # where cannot be an Index, Series or a DataFrame
+        self.assertRaises(ValueError, lambda: kser.asof(ps.Index([-100, 100])))
+        self.assertRaises(ValueError, lambda: kser.asof(ps.Series([-100, 100])))
+        self.assertRaises(ValueError, lambda: kser.asof(ps.DataFrame({"A": [1, 2, 3]})))
+        # asof is not supported for a MultiIndex
+        pser.index = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "c"), ("y", "d")])
+        kser = ps.from_pandas(pser)
+        self.assertRaises(ValueError, lambda: kser.asof(20))
+        # asof requires a sorted index (More precisely, should be a monotonic increasing)
+        kser = ps.Series([1, 2, np.nan, 4], index=[10, 30, 20, 40], name="Koalas")
+        self.assertRaises(ValueError, lambda: kser.asof(20))
+        kser = ps.Series([1, 2, np.nan, 4], index=[40, 30, 20, 10], name="Koalas")
+        self.assertRaises(ValueError, lambda: kser.asof(20))
+
+        pidx = pd.DatetimeIndex(["2013-12-31", "2014-01-02", "2014-01-03"])
+        pser = pd.Series([1, 2, np.nan], index=pidx)
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(kser.asof("2014-01-01"), pser.asof("2014-01-01"))
+        self.assert_eq(kser.asof("2014-01-02"), pser.asof("2014-01-02"))
+        self.assert_eq(repr(kser.asof("1999-01-02")), repr(pser.asof("1999-01-02")))
+
+    def test_squeeze(self):
+        # Single value
+        pser = pd.Series([90])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(kser.squeeze(), pser.squeeze())
+
+        # Single value with MultiIndex
+        midx = pd.MultiIndex.from_tuples([("a", "b", "c")])
+        pser = pd.Series([90], index=midx)
+        kser = ps.from_pandas(pser)
+        self.assert_eq(kser.squeeze(), pser.squeeze())
+
+        # Multiple values
+        pser = pd.Series([90, 91, 85])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(kser.squeeze(), pser.squeeze())
+
+        # Multiple values with MultiIndex
+        midx = pd.MultiIndex.from_tuples([("a", "x"), ("b", "y"), ("c", "z")])
+        pser = pd.Series([90, 91, 85], index=midx)
+        kser = ps.from_pandas(pser)
+        self.assert_eq(kser.squeeze(), pser.squeeze())
+
+    def test_swaplevel(self):
+        # MultiIndex with two levels
+        arrays = [[1, 1, 2, 2], ["red", "blue", "red", "blue"]]
+        pidx = pd.MultiIndex.from_arrays(arrays, names=("number", "color"))
+        pser = pd.Series(["a", "b", "c", "d"], index=pidx)
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.swaplevel(), kser.swaplevel())
+        self.assert_eq(pser.swaplevel(0, 1), kser.swaplevel(0, 1))
+        self.assert_eq(pser.swaplevel(1, 1), kser.swaplevel(1, 1))
+        self.assert_eq(pser.swaplevel("number", "color"), kser.swaplevel("number", "color"))
+
+        # MultiIndex with more than two levels
+        arrays = [[1, 1, 2, 2], ["red", "blue", "red", "blue"], ["l", "m", "s", "xs"]]
+        pidx = pd.MultiIndex.from_arrays(arrays, names=("number", "color", "size"))
+        pser = pd.Series(["a", "b", "c", "d"], index=pidx)
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.swaplevel(), kser.swaplevel())
+        self.assert_eq(pser.swaplevel(0, 1), kser.swaplevel(0, 1))
+        self.assert_eq(pser.swaplevel(0, 2), kser.swaplevel(0, 2))
+        self.assert_eq(pser.swaplevel(1, 2), kser.swaplevel(1, 2))
+        self.assert_eq(pser.swaplevel(1, 1), kser.swaplevel(1, 1))
+        self.assert_eq(pser.swaplevel(-1, -2), kser.swaplevel(-1, -2))
+        self.assert_eq(pser.swaplevel("number", "color"), kser.swaplevel("number", "color"))
+        self.assert_eq(pser.swaplevel("number", "size"), kser.swaplevel("number", "size"))
+        self.assert_eq(pser.swaplevel("color", "size"), kser.swaplevel("color", "size"))
+
+        # Error conditions
+        self.assertRaises(AssertionError, lambda: ps.Series([1, 2]).swaplevel())
+        self.assertRaises(IndexError, lambda: kser.swaplevel(0, 9))
+        self.assertRaises(KeyError, lambda: kser.swaplevel("not_number", "color"))
+        self.assertRaises(AssertionError, lambda: kser.swaplevel(copy=False))
+
+    def test_swapaxes(self):
+        pser = pd.Series([1, 2, 3], index=["x", "y", "z"], name="ser")
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(kser.swapaxes(0, 0), pser.swapaxes(0, 0))
+        self.assert_eq(kser.swapaxes("index", "index"), pser.swapaxes("index", "index"))
+        self.assert_eq((kser + 1).swapaxes(0, 0), (pser + 1).swapaxes(0, 0))
+
+        self.assertRaises(AssertionError, lambda: kser.swapaxes(0, 1, copy=False))
+        self.assertRaises(ValueError, lambda: kser.swapaxes(0, 1))
+        self.assertRaises(ValueError, lambda: kser.swapaxes("index", "columns"))
+
+    def test_div_zero_and_nan(self):
+        pser = pd.Series([100, None, -300, None, 500, -700, np.inf, -np.inf], name="Koalas")
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(pser.div(0), kser.div(0))
+        self.assert_eq(pser.truediv(0), kser.truediv(0))
+        self.assert_eq(pser / 0, kser / 0)
+        self.assert_eq(pser.div(np.nan), kser.div(np.nan))
+        self.assert_eq(pser.truediv(np.nan), kser.truediv(np.nan))
+        self.assert_eq(pser / np.nan, kser / np.nan)
+
+        # floordiv has different behavior in pandas > 1.0.0 when divide by 0
+        if LooseVersion(pd.__version__) >= LooseVersion("1.0.0"):
+            self.assert_eq(pser.floordiv(0), kser.floordiv(0))
+            self.assert_eq(pser // 0, kser // 0)
+        else:
+            result = pd.Series(
+                [np.inf, np.nan, -np.inf, np.nan, np.inf, -np.inf, np.inf, -np.inf], name="Koalas"
+            )
+            self.assert_eq(kser.floordiv(0), result)
+            self.assert_eq(kser // 0, result)
+        self.assert_eq(pser.floordiv(np.nan), kser.floordiv(np.nan))
+
+    def test_mad(self):
+        pser = pd.Series([1, 2, 3, 4], name="Koalas")
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(pser.mad(), kser.mad())
+
+        pser = pd.Series([None, -2, 5, 10, 50, np.nan, -20], name="Koalas")
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(pser.mad(), kser.mad())
+
+        pmidx = pd.MultiIndex.from_tuples(
+            [("a", "1"), ("a", "2"), ("b", "1"), ("b", "2"), ("c", "1")]
+        )
+        pser = pd.Series([1, 2, 3, 4, 5], name="Koalas")
+        pser.index = pmidx
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(pser.mad(), kser.mad())
+
+        pmidx = pd.MultiIndex.from_tuples(
+            [("a", "1"), ("a", "2"), ("b", "1"), ("b", "2"), ("c", "1")]
+        )
+        pser = pd.Series([None, -2, 5, 50, np.nan], name="Koalas")
+        pser.index = pmidx
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(pser.mad(), kser.mad())
+
+    def test_to_frame(self):
+        pser = pd.Series(["a", "b", "c"])
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(pser.to_frame(name="a"), kser.to_frame(name="a"))
+
+        # for MultiIndex
+        midx = pd.MultiIndex.from_tuples([("a", "x"), ("b", "y"), ("c", "z")])
+        pser = pd.Series(["a", "b", "c"], index=midx)
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(pser.to_frame(name="a"), kser.to_frame(name="a"))
+
+    def test_shape(self):
+        pser = pd.Series(["a", "b", "c"])
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(pser.shape, kser.shape)
+
+        # for MultiIndex
+        midx = pd.MultiIndex.from_tuples([("a", "x"), ("b", "y"), ("c", "z")])
+        pser = pd.Series(["a", "b", "c"], index=midx)
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(pser.shape, kser.shape)
+
+    @unittest.skipIf(not have_tabulate, tabulate_requirement_message)
+    def test_to_markdown(self):
+        pser = pd.Series(["elk", "pig", "dog", "quetzal"], name="animal")
+        kser = ps.from_pandas(pser)
+
+        # `to_markdown()` is supported in pandas >= 1.0.0 since it's newly added in pandas 1.0.0.
+        if LooseVersion(pd.__version__) < LooseVersion("1.0.0"):
+            self.assertRaises(NotImplementedError, lambda: kser.to_markdown())
+        else:
+            self.assert_eq(pser.to_markdown(), kser.to_markdown())
+
+    def test_unstack(self):
+        pser = pd.Series(
+            [10, -2, 4, 7],
+            index=pd.MultiIndex.from_tuples(
+                [("one", "a", "z"), ("one", "b", "x"), ("two", "a", "c"), ("two", "b", "v")],
+                names=["A", "B", "C"],
+            ),
+        )
+        kser = ps.from_pandas(pser)
+
+        levels = [-3, -2, -1, 0, 1, 2]
+        for level in levels:
+            pandas_result = pser.unstack(level=level)
+            koalas_result = kser.unstack(level=level).sort_index()
+            self.assert_eq(pandas_result, koalas_result)
+            self.assert_eq(pandas_result.index.names, koalas_result.index.names)
+            self.assert_eq(pandas_result.columns.names, koalas_result.columns.names)
+
+        # non-numeric datatypes
+        pser = pd.Series(
+            list("abcd"), index=pd.MultiIndex.from_product([["one", "two"], ["a", "b"]])
+        )
+        kser = ps.from_pandas(pser)
+
+        levels = [-2, -1, 0, 1]
+        for level in levels:
+            pandas_result = pser.unstack(level=level)
+            koalas_result = kser.unstack(level=level).sort_index()
+            self.assert_eq(pandas_result, koalas_result)
+            self.assert_eq(pandas_result.index.names, koalas_result.index.names)
+            self.assert_eq(pandas_result.columns.names, koalas_result.columns.names)
+
+        # Exceeding the range of level
+        self.assertRaises(IndexError, lambda: kser.unstack(level=3))
+        self.assertRaises(IndexError, lambda: kser.unstack(level=-4))
+        # Only support for MultiIndex
+        kser = ps.Series([10, -2, 4, 7])
+        self.assertRaises(ValueError, lambda: kser.unstack())
+
+    def test_item(self):
+        kser = ps.Series([10, 20])
+        self.assertRaises(ValueError, lambda: kser.item())
+
+    def test_filter(self):
+        pser = pd.Series([0, 1, 2], index=["one", "two", "three"])
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(pser.filter(items=["one", "three"]), kser.filter(items=["one", "three"]))
+        self.assert_eq(pser.filter(regex="e$"), kser.filter(regex="e$"))
+        self.assert_eq(pser.filter(like="hre"), kser.filter(like="hre"))
+
+        with self.assertRaisesRegex(ValueError, "Series does not support columns axis."):
+            kser.filter(like="hre", axis=1)
+
+        # for MultiIndex
+        midx = pd.MultiIndex.from_tuples([("one", "x"), ("two", "y"), ("three", "z")])
+        pser = pd.Series([0, 1, 2], index=midx)
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(
+            pser.filter(items=[("one", "x"), ("three", "z")]),
+            kser.filter(items=[("one", "x"), ("three", "z")]),
+        )
+
+        with self.assertRaisesRegex(TypeError, "Unsupported type list"):
+            kser.filter(items=[["one", "x"], ("three", "z")])
+
+        with self.assertRaisesRegex(ValueError, "The item should not be empty."):
+            kser.filter(items=[(), ("three", "z")])
+
+    def test_abs(self):
+        pser = pd.Series([-2, -1, 0, 1])
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(abs(kser), abs(pser))
+        self.assert_eq(np.abs(kser), np.abs(pser))
+
+    def test_bfill(self):
+        pdf = pd.DataFrame({"x": [np.nan, 2, 3, 4, np.nan, 6], "y": [np.nan, 2, 3, 4, np.nan, 6]})
+        kdf = ps.from_pandas(pdf)
+
+        pser = pdf.x
+        kser = kdf.x
+
+        self.assert_eq(kser.bfill(), pser.bfill())
+        self.assert_eq(kser.bfill()[0], pser.bfill()[0])
+
+        kser.bfill(inplace=True)
+        pser.bfill(inplace=True)
+        self.assert_eq(kser, pser)
+        self.assert_eq(kser[0], pser[0])
+        self.assert_eq(kdf, pdf)
+
+    def test_ffill(self):
+        pdf = pd.DataFrame({"x": [np.nan, 2, 3, 4, np.nan, 6], "y": [np.nan, 2, 3, 4, np.nan, 6]})
+        kdf = ps.from_pandas(pdf)
+
+        pser = pdf.x
+        kser = kdf.x
+
+        self.assert_eq(kser.ffill(), pser.ffill())
+        self.assert_eq(kser.ffill()[4], pser.ffill()[4])
+
+        kser.ffill(inplace=True)
+        pser.ffill(inplace=True)
+        self.assert_eq(kser, pser)
+        self.assert_eq(kser[4], pser[4])
+        self.assert_eq(kdf, pdf)
+
+    def test_iteritems(self):
+        pser = pd.Series(["A", "B", "C"])
+        kser = ps.from_pandas(pser)
+
+        for (p_name, p_items), (k_name, k_items) in zip(pser.iteritems(), kser.iteritems()):
+            self.assert_eq(p_name, k_name)
+            self.assert_eq(p_items, k_items)
+
+    def test_droplevel(self):
+        # droplevel is new in pandas 0.24.0
+        if LooseVersion(pd.__version__) >= LooseVersion("0.24.0"):
+            pser = pd.Series(
+                [1, 2, 3],
+                index=pd.MultiIndex.from_tuples(
+                    [("x", "a", "q"), ("x", "b", "w"), ("y", "c", "e")],
+                    names=["level_1", "level_2", "level_3"],
+                ),
+            )
+            kser = ps.from_pandas(pser)
+
+            self.assert_eq(pser.droplevel(0), kser.droplevel(0))
+            self.assert_eq(pser.droplevel("level_1"), kser.droplevel("level_1"))
+            self.assert_eq(pser.droplevel(-1), kser.droplevel(-1))
+            self.assert_eq(pser.droplevel([0]), kser.droplevel([0]))
+            self.assert_eq(pser.droplevel(["level_1"]), kser.droplevel(["level_1"]))
+            self.assert_eq(pser.droplevel((0,)), kser.droplevel((0,)))
+            self.assert_eq(pser.droplevel(("level_1",)), kser.droplevel(("level_1",)))
+            self.assert_eq(pser.droplevel([0, 2]), kser.droplevel([0, 2]))
+            self.assert_eq(
+                pser.droplevel(["level_1", "level_3"]), kser.droplevel(["level_1", "level_3"])
+            )
+            self.assert_eq(pser.droplevel((1, 2)), kser.droplevel((1, 2)))
+            self.assert_eq(
+                pser.droplevel(("level_2", "level_3")), kser.droplevel(("level_2", "level_3"))
+            )
+
+            with self.assertRaisesRegex(KeyError, "Level {0, 1, 2} not found"):
+                kser.droplevel({0, 1, 2})
+            with self.assertRaisesRegex(KeyError, "Level level_100 not found"):
+                kser.droplevel(["level_1", "level_100"])
+            with self.assertRaisesRegex(
+                IndexError, "Too many levels: Index has only 3 levels, not 11"
+            ):
+                kser.droplevel(10)
+            with self.assertRaisesRegex(
+                IndexError,
+                "Too many levels: Index has only 3 levels, -10 is not a valid level number",
+            ):
+                kser.droplevel(-10)
+            with self.assertRaisesRegex(
+                ValueError,
+                "Cannot remove 3 levels from an index with 3 levels: "
+                "at least one level must be left.",
+            ):
+                kser.droplevel([0, 1, 2])
+            with self.assertRaisesRegex(
+                ValueError,
+                "Cannot remove 5 levels from an index with 3 levels: "
+                "at least one level must be left.",
+            ):
+                kser.droplevel([1, 1, 1, 1, 1])
+
+            # Tupled names
+            pser.index.names = [("a", "1"), ("b", "2"), ("c", "3")]
+            kser = ps.from_pandas(pser)
+
+            self.assert_eq(
+                pser.droplevel([("a", "1"), ("c", "3")]), kser.droplevel([("a", "1"), ("c", "3")])
+            )
+
+    def test_dot(self):
+        pdf = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq((kdf["b"] * 10).dot(kdf["a"]), (pdf["b"] * 10).dot(pdf["a"]))
+        self.assert_eq((kdf["b"] * 10).dot(kdf), (pdf["b"] * 10).dot(pdf))
+        self.assert_eq((kdf["b"] * 10).dot(kdf + 1), (pdf["b"] * 10).dot(pdf + 1))
+
+    def test_tail(self):
+        pser = pd.Series(range(1000), name="Koalas")
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(pser.tail(), kser.tail())
+        self.assert_eq(pser.tail(10), kser.tail(10))
+        self.assert_eq(pser.tail(-990), kser.tail(-990))
+        self.assert_eq(pser.tail(0), kser.tail(0))
+        self.assert_eq(pser.tail(1001), kser.tail(1001))
+        self.assert_eq(pser.tail(-1001), kser.tail(-1001))
+        self.assert_eq((pser + 1).tail(), (kser + 1).tail())
+        self.assert_eq((pser + 1).tail(10), (kser + 1).tail(10))
+        self.assert_eq((pser + 1).tail(-990), (kser + 1).tail(-990))
+        self.assert_eq((pser + 1).tail(0), (kser + 1).tail(0))
+        self.assert_eq((pser + 1).tail(1001), (kser + 1).tail(1001))
+        self.assert_eq((pser + 1).tail(-1001), (kser + 1).tail(-1001))
+        with self.assertRaisesRegex(TypeError, "bad operand type for unary -: 'str'"):
+            kser.tail("10")
+
+    def test_product(self):
+        pser = pd.Series([10, 20, 30, 40, 50])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.prod(), kser.prod())
+
+        # Containing NA values
+        pser = pd.Series([10, np.nan, 30, np.nan, 50])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.prod(), kser.prod(), almost=True)
+
+        # All-NA values
+        pser = pd.Series([np.nan, np.nan, np.nan])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.prod(), kser.prod())
+
+        # Empty Series
+        pser = pd.Series([])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.prod(), kser.prod())
+
+        # Boolean Series
+        pser = pd.Series([True, True, True])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.prod(), kser.prod())
+
+        pser = pd.Series([False, False, False])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.prod(), kser.prod())
+
+        pser = pd.Series([True, False, True])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.prod(), kser.prod())
+
+        # With `min_count` parameter
+        pser = pd.Series([10, 20, 30, 40, 50])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.prod(min_count=5), kser.prod(min_count=5))
+        self.assert_eq(pser.prod(min_count=6), kser.prod(min_count=6))
+
+        pser = pd.Series([10, np.nan, 30, np.nan, 50])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.prod(min_count=3), kser.prod(min_count=3), almost=True)
+        self.assert_eq(pser.prod(min_count=4), kser.prod(min_count=4))
+
+        pser = pd.Series([np.nan, np.nan, np.nan])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.prod(min_count=1), kser.prod(min_count=1))
+
+        pser = pd.Series([])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.prod(min_count=1), kser.prod(min_count=1))
+
+        with self.assertRaisesRegex(TypeError, "Could not convert object \\(string\\) to numeric"):
+            ps.Series(["a", "b", "c"]).prod()
+        with self.assertRaisesRegex(
+            TypeError, "Could not convert datetime64\\[ns\\] \\(timestamp\\) to numeric"
+        ):
+            ps.Series([pd.Timestamp("2016-01-01") for _ in range(3)]).prod()
+
+    def test_hasnans(self):
+        # BooleanType
+        pser = pd.Series([True, False, True, True])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.hasnans, kser.hasnans)
+
+        pser = pd.Series([True, False, np.nan, True])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.hasnans, kser.hasnans)
+
+        # TimestampType
+        pser = pd.Series([pd.Timestamp("2020-07-30") for _ in range(3)])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.hasnans, kser.hasnans)
+
+        pser = pd.Series([pd.Timestamp("2020-07-30"), np.nan, pd.Timestamp("2020-07-30")])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.hasnans, kser.hasnans)
+
+    def test_last_valid_index(self):
+        pser = pd.Series([250, 1.5, 320, 1, 0.3, None, None, None, None])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.last_valid_index(), kser.last_valid_index())
+
+        # MultiIndex columns
+        midx = pd.MultiIndex(
+            [["lama", "cow", "falcon"], ["speed", "weight", "length"]],
+            [[0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 1, 2, 0, 1, 2, 0, 1, 2]],
+        )
+        pser.index = midx
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.last_valid_index(), kser.last_valid_index())
+
+        # Empty Series
+        pser = pd.Series([])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.last_valid_index(), kser.last_valid_index())
+
+    def test_first_valid_index(self):
+        # Empty Series
+        pser = pd.Series([])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.first_valid_index(), kser.first_valid_index())
+
+    def test_factorize(self):
+        pser = pd.Series(["a", "b", "a", "b"])
+        kser = ps.from_pandas(pser)
+        pcodes, puniques = pser.factorize(sort=True)
+        kcodes, kuniques = kser.factorize()
+        self.assert_eq(pcodes.tolist(), kcodes.to_list())
+        self.assert_eq(puniques, kuniques)
+
+        pser = pd.Series([5, 1, 5, 1])
+        kser = ps.from_pandas(pser)
+        pcodes, puniques = (pser + 1).factorize(sort=True)
+        kcodes, kuniques = (kser + 1).factorize()
+        self.assert_eq(pcodes.tolist(), kcodes.to_list())
+        self.assert_eq(puniques, kuniques)
+
+        pser = pd.Series(["a", "b", "a", "b"], name="ser", index=["w", "x", "y", "z"])
+        kser = ps.from_pandas(pser)
+        pcodes, puniques = pser.factorize(sort=True)
+        kcodes, kuniques = kser.factorize()
+        self.assert_eq(pcodes.tolist(), kcodes.to_list())
+        self.assert_eq(puniques, kuniques)
+
+        pser = pd.Series(
+            ["a", "b", "a", "b"], index=pd.MultiIndex.from_arrays([[4, 3, 2, 1], [1, 2, 3, 4]])
+        )
+        kser = ps.from_pandas(pser)
+        pcodes, puniques = pser.factorize(sort=True)
+        kcodes, kuniques = kser.factorize()
+        self.assert_eq(pcodes.tolist(), kcodes.to_list())
+        self.assert_eq(puniques, kuniques)
+
+        #
+        # Deals with None and np.nan
+        #
+        pser = pd.Series(["a", "b", "a", np.nan])
+        kser = ps.from_pandas(pser)
+        pcodes, puniques = pser.factorize(sort=True)
+        kcodes, kuniques = kser.factorize()
+        self.assert_eq(pcodes.tolist(), kcodes.to_list())
+        self.assert_eq(puniques, kuniques)
+
+        pser = pd.Series([1, None, 3, 2, 1])
+        kser = ps.from_pandas(pser)
+        pcodes, puniques = pser.factorize(sort=True)
+        kcodes, kuniques = kser.factorize()
+        self.assert_eq(pcodes.tolist(), kcodes.to_list())
+        self.assert_eq(puniques, kuniques)
+
+        pser = pd.Series(["a", None, "a"])
+        kser = ps.from_pandas(pser)
+        pcodes, puniques = pser.factorize(sort=True)
+        kcodes, kuniques = kser.factorize()
+        self.assert_eq(pcodes.tolist(), kcodes.to_list())
+        self.assert_eq(puniques, kuniques)
+
+        pser = pd.Series([None, np.nan])
+        kser = ps.from_pandas(pser)
+        pcodes, puniques = pser.factorize()
+        kcodes, kuniques = kser.factorize()
+        self.assert_eq(pcodes, kcodes.to_list())
+        # pandas: Float64Index([], dtype='float64')
+        self.assert_eq(pd.Index([]), kuniques)
+
+        pser = pd.Series([np.nan, np.nan])
+        kser = ps.from_pandas(pser)
+        pcodes, puniques = pser.factorize()
+        kcodes, kuniques = kser.factorize()
+        self.assert_eq(pcodes, kcodes.to_list())
+        # pandas: Float64Index([], dtype='float64')
+        self.assert_eq(pd.Index([]), kuniques)
+
+        #
+        # Deals with na_sentinel
+        #
+        # pandas >= 1.1.2 support na_sentinel=None
+        # pandas >= 0.24 support na_sentinel not to be -1
+        #
+        pd_below_1_1_2 = LooseVersion(pd.__version__) < LooseVersion("1.1.2")
+        pd_below_0_24 = LooseVersion(pd.__version__) < LooseVersion("0.24")
+
+        pser = pd.Series(["a", "b", "a", np.nan, None])
+        kser = ps.from_pandas(pser)
+
+        pcodes, puniques = pser.factorize(sort=True, na_sentinel=-2)
+        kcodes, kuniques = kser.factorize(na_sentinel=-2)
+        self.assert_eq([0, 1, 0, -2, -2] if pd_below_0_24 else pcodes.tolist(), kcodes.to_list())
+        self.assert_eq(puniques, kuniques)
+
+        pcodes, puniques = pser.factorize(sort=True, na_sentinel=2)
+        kcodes, kuniques = kser.factorize(na_sentinel=2)
+        self.assert_eq([0, 1, 0, 2, 2] if pd_below_0_24 else pcodes.tolist(), kcodes.to_list())
+        self.assert_eq(puniques, kuniques)
+
+        if not pd_below_1_1_2:
+            pcodes, puniques = pser.factorize(sort=True, na_sentinel=None)
+            kcodes, kuniques = kser.factorize(na_sentinel=None)
+            self.assert_eq(pcodes.tolist(), kcodes.to_list())
+            # puniques is Index(['a', 'b', nan], dtype='object')
+            self.assert_eq(ps.Index(["a", "b", None]), kuniques)
+
+            kser = ps.Series([1, 2, np.nan, 4, 5])  # Arrow takes np.nan as null
+            kser.loc[3] = np.nan  # Spark takes np.nan as NaN
+            kcodes, kuniques = kser.factorize(na_sentinel=None)
+            pcodes, puniques = kser.to_pandas().factorize(sort=True, na_sentinel=None)
+            self.assert_eq(pcodes.tolist(), kcodes.to_list())
+            self.assert_eq(puniques, kuniques)
+
+    def test_pad(self):
+        pser = pd.Series([np.nan, 2, 3, 4, np.nan, 6], name="x")
+        kser = ps.from_pandas(pser)
+
+        if LooseVersion(pd.__version__) >= LooseVersion("1.1"):
+            self.assert_eq(pser.pad(), kser.pad())
+
+            # Test `inplace=True`
+            pser.pad(inplace=True)
+            kser.pad(inplace=True)
+            self.assert_eq(pser, kser)
+        else:
+            expected = ps.Series([np.nan, 2, 3, 4, 4, 6], name="x")
+            self.assert_eq(expected, kser.pad())
+
+            # Test `inplace=True`
+            kser.pad(inplace=True)
+            self.assert_eq(expected, kser)
+
+    def test_explode(self):
+        if LooseVersion(pd.__version__) >= LooseVersion("0.25"):
+            pser = pd.Series([[1, 2, 3], [], None, [3, 4]])
+            kser = ps.from_pandas(pser)
+            self.assert_eq(pser.explode(), kser.explode(), almost=True)
+
+            # MultiIndex
+            pser.index = pd.MultiIndex.from_tuples([("a", "w"), ("b", "x"), ("c", "y"), ("d", "z")])
+            kser = ps.from_pandas(pser)
+            self.assert_eq(pser.explode(), kser.explode(), almost=True)
+
+            # non-array type Series
+            pser = pd.Series([1, 2, 3, 4])
+            kser = ps.from_pandas(pser)
+            self.assert_eq(pser.explode(), kser.explode())
+        else:
+            pser = pd.Series([[1, 2, 3], [], None, [3, 4]])
+            kser = ps.from_pandas(pser)
+            expected = pd.Series([1.0, 2.0, 3.0, None, None, 3.0, 4.0], index=[0, 0, 0, 1, 2, 3, 3])
+            self.assert_eq(kser.explode(), expected)
+
+            # MultiIndex
+            pser.index = pd.MultiIndex.from_tuples([("a", "w"), ("b", "x"), ("c", "y"), ("d", "z")])
+            kser = ps.from_pandas(pser)
+            expected = pd.Series(
+                [1.0, 2.0, 3.0, None, None, 3.0, 4.0],
+                index=pd.MultiIndex.from_tuples(
+                    [
+                        ("a", "w"),
+                        ("a", "w"),
+                        ("a", "w"),
+                        ("b", "x"),
+                        ("c", "y"),
+                        ("d", "z"),
+                        ("d", "z"),
+                    ]
+                ),
+            )
+            self.assert_eq(kser.explode(), expected)
+
+            # non-array type Series
+            pser = pd.Series([1, 2, 3, 4])
+            kser = ps.from_pandas(pser)
+            expected = pser
+            self.assert_eq(kser.explode(), expected)
+
+    def test_argsort(self):
+        # Without null values
+        pser = pd.Series([0, -100, 50, 100, 20], index=["A", "B", "C", "D", "E"])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.argsort().sort_index(), kser.argsort().sort_index())
+        self.assert_eq((-pser).argsort().sort_index(), (-kser).argsort().sort_index())
+
+        # MultiIndex
+        pser.index = pd.MultiIndex.from_tuples(
+            [("a", "v"), ("b", "w"), ("c", "x"), ("d", "y"), ("e", "z")]
+        )
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.argsort().sort_index(), kser.argsort().sort_index())
+        self.assert_eq((-pser).argsort().sort_index(), (-kser).argsort().sort_index())
+
+        # With name
+        pser.name = "Koalas"
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.argsort().sort_index(), kser.argsort().sort_index())
+        self.assert_eq((-pser).argsort().sort_index(), (-kser).argsort().sort_index())
+
+        # Series from Index
+        pidx = pd.Index([4.0, -6.0, 2.0, -100.0, 11.0, 20.0, 1.0, -99.0])
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(
+            pidx.to_series().argsort().sort_index(), kidx.to_series().argsort().sort_index()
+        )
+        self.assert_eq(
+            (-pidx.to_series()).argsort().sort_index(), (-kidx.to_series()).argsort().sort_index()
+        )
+
+        # Series from Index with name
+        pidx.name = "Koalas"
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(
+            pidx.to_series().argsort().sort_index(), kidx.to_series().argsort().sort_index()
+        )
+        self.assert_eq(
+            (-pidx.to_series()).argsort().sort_index(), (-kidx.to_series()).argsort().sort_index()
+        )
+
+        # Series from DataFrame
+        pdf = pd.DataFrame({"A": [4.0, -6.0, 2.0, np.nan, -100.0, 11.0, 20.0, np.nan, 1.0, -99.0]})
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.A.argsort().sort_index(), kdf.A.argsort().sort_index())
+        self.assert_eq((-pdf.A).argsort().sort_index(), (-kdf.A).argsort().sort_index())
+
+        # With null values
+        pser = pd.Series([0, -100, np.nan, 100, np.nan], index=["A", "B", "C", "D", "E"])
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.argsort().sort_index(), kser.argsort().sort_index())
+        self.assert_eq((-pser).argsort().sort_index(), (-kser).argsort().sort_index())
+
+        # MultiIndex with null values
+        pser.index = pd.MultiIndex.from_tuples(
+            [("a", "v"), ("b", "w"), ("c", "x"), ("d", "y"), ("e", "z")]
+        )
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.argsort().sort_index(), kser.argsort().sort_index())
+        self.assert_eq((-pser).argsort().sort_index(), (-kser).argsort().sort_index())
+
+        # With name with null values
+        pser.name = "Koalas"
+        kser = ps.from_pandas(pser)
+        self.assert_eq(pser.argsort().sort_index(), kser.argsort().sort_index())
+        self.assert_eq((-pser).argsort().sort_index(), (-kser).argsort().sort_index())
+
+        # Series from Index with null values
+        pidx = pd.Index([4.0, -6.0, 2.0, np.nan, -100.0, 11.0, 20.0, np.nan, 1.0, -99.0])
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(
+            pidx.to_series().argsort().sort_index(), kidx.to_series().argsort().sort_index()
+        )
+        self.assert_eq(
+            (-pidx.to_series()).argsort().sort_index(), (-kidx.to_series()).argsort().sort_index()
+        )
+
+        # Series from Index with name with null values
+        pidx.name = "Koalas"
+        kidx = ps.from_pandas(pidx)
+        self.assert_eq(
+            pidx.to_series().argsort().sort_index(), kidx.to_series().argsort().sort_index()
+        )
+        self.assert_eq(
+            (-pidx.to_series()).argsort().sort_index(), (-kidx.to_series()).argsort().sort_index()
+        )
+
+        # Series from DataFrame with null values
+        pdf = pd.DataFrame({"A": [4.0, -6.0, 2.0, np.nan, -100.0, 11.0, 20.0, np.nan, 1.0, -99.0]})
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(pdf.A.argsort().sort_index(), kdf.A.argsort().sort_index())
+        self.assert_eq((-pdf.A).argsort().sort_index(), (-kdf.A).argsort().sort_index())
+
+    def test_argmin_argmax(self):
+        pser = pd.Series(
+            {
+                "Corn Flakes": 100.0,
+                "Almond Delight": 110.0,
+                "Cinnamon Toast Crunch": 120.0,
+                "Cocoa Puff": 110.0,
+                "Expensive Flakes": 120.0,
+                "Cheap Flakes": 100.0,
+            },
+            name="Koalas",
+        )
+        kser = ps.from_pandas(pser)
+
+        if LooseVersion(pd.__version__) >= LooseVersion("1.0"):
+            self.assert_eq(pser.argmin(), kser.argmin())
+            self.assert_eq(pser.argmax(), kser.argmax())
+
+            # MultiIndex
+            pser.index = pd.MultiIndex.from_tuples(
+                [("a", "t"), ("b", "u"), ("c", "v"), ("d", "w"), ("e", "x"), ("f", "u")]
+            )
+            kser = ps.from_pandas(pser)
+            self.assert_eq(pser.argmin(), kser.argmin())
+            self.assert_eq(pser.argmax(), kser.argmax())
+
+            # Null Series
+            self.assert_eq(pd.Series([np.nan]).argmin(), ps.Series([np.nan]).argmin())
+            self.assert_eq(pd.Series([np.nan]).argmax(), ps.Series([np.nan]).argmax())
+        else:
+            self.assert_eq(pser.values.argmin(), kser.argmin())
+            self.assert_eq(pser.values.argmax(), kser.argmax())
+
+            # MultiIndex
+            pser.index = pd.MultiIndex.from_tuples(
+                [("a", "t"), ("b", "u"), ("c", "v"), ("d", "w"), ("e", "x"), ("f", "u")]
+            )
+            kser = ps.from_pandas(pser)
+            self.assert_eq(pser.values.argmin(), kser.argmin())
+            self.assert_eq(pser.values.argmax(), kser.argmax())
+
+            # Null Series
+            self.assert_eq(-1, ps.Series([np.nan]).argmin())
+            self.assert_eq(-1, ps.Series([np.nan]).argmax())
+
+        with self.assertRaisesRegex(ValueError, "attempt to get argmin of an empty sequence"):
+            ps.Series([]).argmin()
+        with self.assertRaisesRegex(ValueError, "attempt to get argmax of an empty sequence"):
+            ps.Series([]).argmax()
+
+    def test_backfill(self):
+        pser = pd.Series([np.nan, 2, 3, 4, np.nan, 6], name="x")
+        kser = ps.from_pandas(pser)
+
+        if LooseVersion(pd.__version__) >= LooseVersion("1.1"):
+            self.assert_eq(pser.backfill(), kser.backfill())
+
+            # Test `inplace=True`
+            pser.backfill(inplace=True)
+            kser.backfill(inplace=True)
+            self.assert_eq(pser, kser)
+        else:
+            expected = ps.Series([2.0, 2.0, 3.0, 4.0, 6.0, 6.0], name="x")
+            self.assert_eq(expected, kser.backfill())
+
+            # Test `inplace=True`
+            kser.backfill(inplace=True)
+            self.assert_eq(expected, kser)
+
+    def test_align(self):
+        pdf = pd.DataFrame({"a": [1, 2, 3], "b": ["a", "b", "c"]})
+        kdf = ps.from_pandas(pdf)
+
+        for join in ["outer", "inner", "left", "right"]:
+            for axis in [None, 0]:
+                kser_l, kser_r = kdf.a.align(kdf.b, join=join, axis=axis)
+                pser_l, pser_r = pdf.a.align(pdf.b, join=join, axis=axis)
+                self.assert_eq(kser_l, pser_l)
+                self.assert_eq(kser_r, pser_r)
+
+                kser_l, kdf_r = kdf.b.align(kdf[["b", "a"]], join=join, axis=axis)
+                pser_l, pdf_r = pdf.b.align(pdf[["b", "a"]], join=join, axis=axis)
+                self.assert_eq(kser_l, pser_l)
+                self.assert_eq(kdf_r, pdf_r)
+
+        self.assertRaises(ValueError, lambda: kdf.a.align(kdf.b, axis=1))
+
+    def test_pow_and_rpow(self):
+        pser = pd.Series([1, 2, np.nan])
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(pser.pow(np.nan), kser.pow(np.nan))
+        self.assert_eq(pser ** np.nan, kser ** np.nan)
+        self.assert_eq(pser.rpow(np.nan), kser.rpow(np.nan))
+        self.assert_eq(1 ** pser, 1 ** kser)
+
+    def test_between_time(self):
+        idx = pd.date_range("2018-04-09", periods=4, freq="1D20min")
+        pser = pd.Series([1, 2, 3, 4], index=idx)
+        kser = ps.from_pandas(pser)
+        self.assert_eq(
+            pser.between_time("0:15", "0:45").sort_index(),
+            kser.between_time("0:15", "0:45").sort_index(),
+        )
+
+        pser.index.name = "ts"
+        kser = ps.from_pandas(pser)
+        self.assert_eq(
+            pser.between_time("0:15", "0:45").sort_index(),
+            kser.between_time("0:15", "0:45").sort_index(),
+        )
+
+        pser.index.name = "index"
+        kser = ps.from_pandas(pser)
+        self.assert_eq(
+            pser.between_time("0:15", "0:45").sort_index(),
+            kser.between_time("0:15", "0:45").sort_index(),
+        )
+
+    def test_at_time(self):
+        idx = pd.date_range("2018-04-09", periods=4, freq="1D20min")
+        pser = pd.Series([1, 2, 3, 4], index=idx)
+        kser = ps.from_pandas(pser)
+        self.assert_eq(
+            pser.at_time("0:20").sort_index(), kser.at_time("0:20").sort_index(),
+        )
+
+        pser.index.name = "ts"
+        kser = ps.from_pandas(pser)
+        self.assert_eq(
+            pser.at_time("0:20").sort_index(), kser.at_time("0:20").sort_index(),
+        )
+
+        pser.index.name = "index"
+        kser = ps.from_pandas(pser)
+        self.assert_eq(
+            pser.at_time("0:20").sort_index(), kser.at_time("0:20").sort_index(),
+        )
+
+
+if __name__ == "__main__":
+    from pyspark.pandas.tests.test_series import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_series_conversion.py b/python/pyspark/pandas/tests/test_series_conversion.py
new file mode 100644
index 0000000000000..18ce24de74446
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_series_conversion.py
@@ -0,0 +1,76 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import unittest
+from distutils.version import LooseVersion
+
+import pandas as pd
+
+from pyspark import pandas as ps
+from pyspark.testing.pandasutils import PandasOnSparkTestCase
+from pyspark.testing.sqlutils import SQLTestUtils
+
+
+class SeriesConversionTest(PandasOnSparkTestCase, SQLTestUtils):
+    @property
+    def pser(self):
+        return pd.Series([1, 2, 3, 4, 5, 6, 7], name="x")
+
+    @property
+    def kser(self):
+        return ps.from_pandas(self.pser)
+
+    @unittest.skip("Pyperclip could not find a copy/paste mechanism for Linux.")
+    def test_to_clipboard(self):
+        pser = self.pser
+        kser = self.kser
+
+        self.assert_eq(kser.to_clipboard(), pser.to_clipboard())
+        self.assert_eq(kser.to_clipboard(excel=False), pser.to_clipboard(excel=False))
+        self.assert_eq(
+            kser.to_clipboard(sep=",", index=False), pser.to_clipboard(sep=",", index=False)
+        )
+
+    def test_to_latex(self):
+        pser = self.pser
+        kser = self.kser
+
+        self.assert_eq(kser.to_latex(), pser.to_latex())
+        self.assert_eq(kser.to_latex(col_space=2), pser.to_latex(col_space=2))
+        self.assert_eq(kser.to_latex(header=True), pser.to_latex(header=True))
+        self.assert_eq(kser.to_latex(index=False), pser.to_latex(index=False))
+        self.assert_eq(kser.to_latex(na_rep="-"), pser.to_latex(na_rep="-"))
+        self.assert_eq(kser.to_latex(float_format="%.1f"), pser.to_latex(float_format="%.1f"))
+        self.assert_eq(kser.to_latex(sparsify=False), pser.to_latex(sparsify=False))
+        self.assert_eq(kser.to_latex(index_names=False), pser.to_latex(index_names=False))
+        self.assert_eq(kser.to_latex(bold_rows=True), pser.to_latex(bold_rows=True))
+        # Can't specifying `encoding` without specifying `buf` as filename in pandas >= 1.0.0
+        # https://github.com/pandas-dev/pandas/blob/master/pandas/io/formats/format.py#L492-L495
+        if LooseVersion(pd.__version__) < LooseVersion("1.0.0"):
+            self.assert_eq(kser.to_latex(encoding="ascii"), pser.to_latex(encoding="ascii"))
+        self.assert_eq(kser.to_latex(decimal=","), pser.to_latex(decimal=","))
+
+
+if __name__ == "__main__":
+    from pyspark.pandas.tests.test_series_conversion import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_series_datetime.py b/python/pyspark/pandas/tests/test_series_datetime.py
new file mode 100644
index 0000000000000..deb44974839c4
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_series_datetime.py
@@ -0,0 +1,290 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import datetime
+import unittest
+
+import numpy as np
+import pandas as pd
+
+from pyspark import pandas as ps
+from pyspark.testing.pandasutils import PandasOnSparkTestCase
+from pyspark.testing.sqlutils import SQLTestUtils
+
+
+class SeriesDateTimeTest(PandasOnSparkTestCase, SQLTestUtils):
+    @property
+    def pdf1(self):
+        date1 = pd.Series(pd.date_range("2012-1-1 12:45:31", periods=3, freq="M"))
+        date2 = pd.Series(pd.date_range("2013-3-11 21:45:00", periods=3, freq="W"))
+        return pd.DataFrame(dict(start_date=date1, end_date=date2))
+
+    @property
+    def pd_start_date(self):
+        return self.pdf1["start_date"]
+
+    @property
+    def ks_start_date(self):
+        return ps.from_pandas(self.pd_start_date)
+
+    def check_func(self, func):
+        self.assert_eq(func(self.ks_start_date), func(self.pd_start_date))
+
+    def test_timestamp_subtraction(self):
+        pdf = self.pdf1
+        kdf = ps.from_pandas(pdf)
+
+        # Those fail in certain OSs presumably due to different
+        # timezone behaviours inherited from C library.
+
+        actual = (kdf["end_date"] - kdf["start_date"] - 1).to_pandas()
+        expected = (pdf["end_date"] - pdf["start_date"]) // np.timedelta64(1, "s") - 1
+        # self.assert_eq(actual, expected)
+
+        actual = (kdf["end_date"] - pd.Timestamp("2012-1-1 12:45:31") - 1).to_pandas()
+        expected = (pdf["end_date"] - pd.Timestamp("2012-1-1 12:45:31")) // np.timedelta64(
+            1, "s"
+        ) - 1
+        # self.assert_eq(actual, expected)
+
+        actual = (pd.Timestamp("2013-3-11 21:45:00") - kdf["start_date"] - 1).to_pandas()
+        expected = (pd.Timestamp("2013-3-11 21:45:00") - pdf["start_date"]) // np.timedelta64(
+            1, "s"
+        ) - 1
+        # self.assert_eq(actual, expected)
+
+        kdf = ps.DataFrame(
+            {"a": pd.date_range("2016-12-31", "2017-01-08", freq="D"), "b": pd.Series(range(9))}
+        )
+        expected_error_message = "datetime subtraction can only be applied to datetime series."
+        with self.assertRaisesRegex(TypeError, expected_error_message):
+            kdf["a"] - kdf["b"]
+        with self.assertRaisesRegex(TypeError, expected_error_message):
+            kdf["a"] - 1
+        with self.assertRaisesRegex(TypeError, expected_error_message):
+            1 - kdf["a"]
+
+    def test_arithmetic_op_exceptions(self):
+        kser = self.ks_start_date
+        py_datetime = self.pd_start_date.dt.to_pydatetime()
+        datetime_index = ps.Index(self.pd_start_date)
+
+        for other in [1, 0.1, kser, datetime_index, py_datetime]:
+            expected_err_msg = "addition can not be applied to date times."
+            self.assertRaisesRegex(TypeError, expected_err_msg, lambda: kser + other)
+            self.assertRaisesRegex(TypeError, expected_err_msg, lambda: other + kser)
+
+            expected_err_msg = "multiplication can not be applied to date times."
+            self.assertRaisesRegex(TypeError, expected_err_msg, lambda: kser * other)
+            self.assertRaisesRegex(TypeError, expected_err_msg, lambda: other * kser)
+
+            expected_err_msg = "division can not be applied to date times."
+            self.assertRaisesRegex(TypeError, expected_err_msg, lambda: kser / other)
+            self.assertRaisesRegex(TypeError, expected_err_msg, lambda: other / kser)
+            self.assertRaisesRegex(TypeError, expected_err_msg, lambda: kser // other)
+            self.assertRaisesRegex(TypeError, expected_err_msg, lambda: other // kser)
+
+            expected_err_msg = "modulo can not be applied to date times."
+            self.assertRaisesRegex(TypeError, expected_err_msg, lambda: kser % other)
+            self.assertRaisesRegex(TypeError, expected_err_msg, lambda: other % kser)
+
+        expected_err_msg = "datetime subtraction can only be applied to datetime series."
+
+        for other in [1, 0.1]:
+            self.assertRaisesRegex(TypeError, expected_err_msg, lambda: kser - other)
+            self.assertRaisesRegex(TypeError, expected_err_msg, lambda: other - kser)
+
+        self.assertRaisesRegex(TypeError, expected_err_msg, lambda: kser - other)
+        self.assertRaises(NotImplementedError, lambda: py_datetime - kser)
+
+    def test_date_subtraction(self):
+        pdf = self.pdf1
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            kdf["end_date"].dt.date - kdf["start_date"].dt.date,
+            (pdf["end_date"].dt.date - pdf["start_date"].dt.date).dt.days,
+        )
+
+        self.assert_eq(
+            kdf["end_date"].dt.date - datetime.date(2012, 1, 1),
+            (pdf["end_date"].dt.date - datetime.date(2012, 1, 1)).dt.days,
+        )
+
+        self.assert_eq(
+            datetime.date(2013, 3, 11) - kdf["start_date"].dt.date,
+            (datetime.date(2013, 3, 11) - pdf["start_date"].dt.date).dt.days,
+        )
+
+        kdf = ps.DataFrame(
+            {"a": pd.date_range("2016-12-31", "2017-01-08", freq="D"), "b": pd.Series(range(9))}
+        )
+        expected_error_message = "date subtraction can only be applied to date series."
+        with self.assertRaisesRegex(TypeError, expected_error_message):
+            kdf["a"].dt.date - kdf["b"]
+        with self.assertRaisesRegex(TypeError, expected_error_message):
+            kdf["a"].dt.date - 1
+        with self.assertRaisesRegex(TypeError, expected_error_message):
+            1 - kdf["a"].dt.date
+
+    @unittest.skip(
+        "It fails in certain OSs presumably due to different "
+        "timezone behaviours inherited from C library."
+    )
+    def test_div(self):
+        pdf = self.pdf1
+        kdf = ps.from_pandas(pdf)
+        for u in "D", "s", "ms":
+            duration = np.timedelta64(1, u)
+            self.assert_eq(
+                (kdf["end_date"] - kdf["start_date"]) / duration,
+                (pdf["end_date"] - pdf["start_date"]) / duration,
+            )
+
+    @unittest.skip("It is currently failed probably for the same reason in 'test_subtraction'")
+    def test_date(self):
+        self.check_func(lambda x: x.dt.date)
+
+    def test_time(self):
+        with self.assertRaises(NotImplementedError):
+            self.check_func(lambda x: x.dt.time)
+
+    def test_timetz(self):
+        with self.assertRaises(NotImplementedError):
+            self.check_func(lambda x: x.dt.timetz)
+
+    def test_year(self):
+        self.check_func(lambda x: x.dt.year)
+
+    def test_month(self):
+        self.check_func(lambda x: x.dt.month)
+
+    def test_day(self):
+        self.check_func(lambda x: x.dt.day)
+
+    def test_hour(self):
+        self.check_func(lambda x: x.dt.hour)
+
+    def test_minute(self):
+        self.check_func(lambda x: x.dt.minute)
+
+    def test_second(self):
+        self.check_func(lambda x: x.dt.second)
+
+    def test_microsecond(self):
+        self.check_func(lambda x: x.dt.microsecond)
+
+    def test_nanosecond(self):
+        with self.assertRaises(NotImplementedError):
+            self.check_func(lambda x: x.dt.nanosecond)
+
+    def test_week(self):
+        self.check_func(lambda x: x.dt.week)
+
+    def test_weekofyear(self):
+        self.check_func(lambda x: x.dt.weekofyear)
+
+    def test_dayofweek(self):
+        self.check_func(lambda x: x.dt.dayofweek)
+
+    def test_weekday(self):
+        self.check_func(lambda x: x.dt.weekday)
+
+    def test_dayofyear(self):
+        self.check_func(lambda x: x.dt.dayofyear)
+
+    def test_quarter(self):
+        self.check_func(lambda x: x.dt.dayofyear)
+
+    def test_is_month_start(self):
+        self.check_func(lambda x: x.dt.is_month_start)
+
+    def test_is_month_end(self):
+        self.check_func(lambda x: x.dt.is_month_end)
+
+    def test_is_quarter_start(self):
+        self.check_func(lambda x: x.dt.is_quarter_start)
+
+    def test_is_quarter_end(self):
+        self.check_func(lambda x: x.dt.is_quarter_end)
+
+    def test_is_year_start(self):
+        self.check_func(lambda x: x.dt.is_year_start)
+
+    def test_is_year_end(self):
+        self.check_func(lambda x: x.dt.is_year_end)
+
+    def test_is_leap_year(self):
+        self.check_func(lambda x: x.dt.is_leap_year)
+
+    def test_daysinmonth(self):
+        self.check_func(lambda x: x.dt.daysinmonth)
+
+    def test_days_in_month(self):
+        self.check_func(lambda x: x.dt.days_in_month)
+
+    @unittest.expectedFailure
+    def test_tz_localize(self):
+        self.check_func(lambda x: x.dt.tz_localize("America/New_York"))
+
+    @unittest.expectedFailure
+    def test_tz_convert(self):
+        self.check_func(lambda x: x.dt.tz_convert("America/New_York"))
+
+    def test_normalize(self):
+        self.check_func(lambda x: x.dt.normalize())
+
+    def test_strftime(self):
+        self.check_func(lambda x: x.dt.strftime("%Y-%m-%d"))
+
+    def test_round(self):
+        self.check_func(lambda x: x.dt.round(freq="min"))
+        self.check_func(lambda x: x.dt.round(freq="H"))
+
+    def test_floor(self):
+        self.check_func(lambda x: x.dt.floor(freq="min"))
+        self.check_func(lambda x: x.dt.floor(freq="H"))
+
+    def test_ceil(self):
+        self.check_func(lambda x: x.dt.floor(freq="min"))
+        self.check_func(lambda x: x.dt.floor(freq="H"))
+
+    @unittest.skip("Unsupported locale setting")
+    def test_month_name(self):
+        self.check_func(lambda x: x.dt.month_name())
+        self.check_func(lambda x: x.dt.month_name(locale="en_US.UTF-8"))
+
+    @unittest.skip("Unsupported locale setting")
+    def test_day_name(self):
+        self.check_func(lambda x: x.dt.day_name())
+        self.check_func(lambda x: x.dt.day_name(locale="en_US.UTF-8"))
+
+    def test_unsupported_type(self):
+        self.assertRaisesRegex(
+            ValueError, "Cannot call DatetimeMethods on type LongType", lambda: ps.Series([0]).dt
+        )
+
+
+if __name__ == "__main__":
+    from pyspark.pandas.tests.test_series_datetime import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_series_string.py b/python/pyspark/pandas/tests/test_series_string.py
new file mode 100644
index 0000000000000..057db77360987
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_series_string.py
@@ -0,0 +1,340 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import pandas as pd
+import numpy as np
+import re
+
+from pyspark import pandas as ps
+from pyspark.testing.pandasutils import PandasOnSparkTestCase
+from pyspark.testing.sqlutils import SQLTestUtils
+
+
+class SeriesStringTest(PandasOnSparkTestCase, SQLTestUtils):
+    @property
+    def pser(self):
+        return pd.Series(
+            [
+                "apples",
+                "Bananas",
+                "carrots",
+                "1",
+                "100",
+                "",
+                "\nleading-whitespace",
+                "trailing-Whitespace    \t",
+                None,
+                np.NaN,
+            ]
+        )
+
+    def check_func(self, func, almost=False):
+        self.check_func_on_series(func, self.pser, almost=almost)
+
+    def check_func_on_series(self, func, pser, almost=False):
+        self.assert_eq(func(ps.from_pandas(pser)), func(pser), almost=almost)
+
+    def test_string_add_str_num(self):
+        pdf = pd.DataFrame(dict(col1=["a"], col2=[1]))
+        kdf = ps.from_pandas(pdf)
+        with self.assertRaises(TypeError):
+            kdf["col1"] + kdf["col2"]
+
+    def test_string_add_assign(self):
+        pdf = pd.DataFrame(dict(col1=["a", "b", "c"], col2=["1", "2", "3"]))
+        kdf = ps.from_pandas(pdf)
+        kdf["col1"] += kdf["col2"]
+        pdf["col1"] += pdf["col2"]
+        self.assert_eq(kdf["col1"], pdf["col1"])
+
+    def test_string_add_str_str(self):
+        pdf = pd.DataFrame(dict(col1=["a", "b", "c"], col2=["1", "2", "3"]))
+        kdf = ps.from_pandas(pdf)
+
+        # TODO: Fix the Series names
+        self.assert_eq(kdf["col1"] + kdf["col2"], pdf["col1"] + pdf["col2"])
+        self.assert_eq(kdf["col2"] + kdf["col1"], pdf["col2"] + pdf["col1"])
+
+    def test_string_add_str_lit(self):
+        pdf = pd.DataFrame(dict(col1=["a", "b", "c"]))
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kdf["col1"] + "_lit", pdf["col1"] + "_lit")
+        self.assert_eq("_lit" + kdf["col1"], "_lit" + pdf["col1"])
+
+    def test_string_capitalize(self):
+        self.check_func(lambda x: x.str.capitalize())
+
+    def test_string_title(self):
+        self.check_func(lambda x: x.str.title())
+
+    def test_string_lower(self):
+        self.check_func(lambda x: x.str.lower())
+
+    def test_string_upper(self):
+        self.check_func(lambda x: x.str.upper())
+
+    def test_string_swapcase(self):
+        self.check_func(lambda x: x.str.swapcase())
+
+    def test_string_startswith(self):
+        pattern = "car"
+        self.check_func(lambda x: x.str.startswith(pattern))
+        self.check_func(lambda x: x.str.startswith(pattern, na=False))
+
+    def test_string_endswith(self):
+        pattern = "s"
+        self.check_func(lambda x: x.str.endswith(pattern))
+        self.check_func(lambda x: x.str.endswith(pattern, na=False))
+
+    def test_string_strip(self):
+        self.check_func(lambda x: x.str.strip())
+        self.check_func(lambda x: x.str.strip("es\t"))
+        self.check_func(lambda x: x.str.strip("1"))
+
+    def test_string_lstrip(self):
+        self.check_func(lambda x: x.str.lstrip())
+        self.check_func(lambda x: x.str.lstrip("\n1le"))
+        self.check_func(lambda x: x.str.lstrip("s"))
+
+    def test_string_rstrip(self):
+        self.check_func(lambda x: x.str.rstrip())
+        self.check_func(lambda x: x.str.rstrip("\t ec"))
+        self.check_func(lambda x: x.str.rstrip("0"))
+
+    def test_string_get(self):
+        self.check_func(lambda x: x.str.get(6))
+        self.check_func(lambda x: x.str.get(-1))
+
+    def test_string_isalnum(self):
+        self.check_func(lambda x: x.str.isalnum())
+
+    def test_string_isalpha(self):
+        self.check_func(lambda x: x.str.isalpha())
+
+    def test_string_isdigit(self):
+        self.check_func(lambda x: x.str.isdigit())
+
+    def test_string_isspace(self):
+        self.check_func(lambda x: x.str.isspace())
+
+    def test_string_islower(self):
+        self.check_func(lambda x: x.str.islower())
+
+    def test_string_isupper(self):
+        self.check_func(lambda x: x.str.isupper())
+
+    def test_string_istitle(self):
+        self.check_func(lambda x: x.str.istitle())
+
+    def test_string_isnumeric(self):
+        self.check_func(lambda x: x.str.isnumeric())
+
+    def test_string_isdecimal(self):
+        self.check_func(lambda x: x.str.isdecimal())
+
+    def test_string_cat(self):
+        kser = ps.from_pandas(self.pser)
+        with self.assertRaises(NotImplementedError):
+            kser.str.cat()
+
+    def test_string_center(self):
+        self.check_func(lambda x: x.str.center(0))
+        self.check_func(lambda x: x.str.center(10))
+        self.check_func(lambda x: x.str.center(10, "x"))
+
+    def test_string_contains(self):
+        self.check_func(lambda x: x.str.contains("le", regex=False))
+        self.check_func(lambda x: x.str.contains("White", case=True, regex=False))
+        self.check_func(lambda x: x.str.contains("apples|carrots", regex=True))
+        self.check_func(lambda x: x.str.contains("BANANAS", flags=re.IGNORECASE, na=False))
+
+    def test_string_count(self):
+        self.check_func(lambda x: x.str.count("wh|Wh"))
+        self.check_func(lambda x: x.str.count("WH", flags=re.IGNORECASE))
+
+    def test_string_decode(self):
+        kser = ps.from_pandas(self.pser)
+        with self.assertRaises(NotImplementedError):
+            kser.str.decode("utf-8")
+
+    def test_string_encode(self):
+        kser = ps.from_pandas(self.pser)
+        with self.assertRaises(NotImplementedError):
+            kser.str.encode("utf-8")
+
+    def test_string_extract(self):
+        kser = ps.from_pandas(self.pser)
+        with self.assertRaises(NotImplementedError):
+            kser.str.extract("pat")
+
+    def test_string_extractall(self):
+        kser = ps.from_pandas(self.pser)
+        with self.assertRaises(NotImplementedError):
+            kser.str.extractall("pat")
+
+    def test_string_find(self):
+        self.check_func(lambda x: x.str.find("a"))
+        self.check_func(lambda x: x.str.find("a", start=3))
+        self.check_func(lambda x: x.str.find("a", start=0, end=1))
+
+    def test_string_findall(self):
+        self.check_func_on_series(lambda x: x.str.findall("es|as").apply(str), self.pser[:-1])
+        self.check_func_on_series(
+            lambda x: x.str.findall("wh.*", flags=re.IGNORECASE).apply(str), self.pser[:-1]
+        )
+
+    def test_string_index(self):
+        pser = pd.Series(["tea", "eat"])
+        self.check_func_on_series(lambda x: x.str.index("ea"), pser)
+        with self.assertRaises(Exception):
+            self.check_func_on_series(lambda x: x.str.index("ea", start=0, end=2), pser)
+        with self.assertRaises(Exception):
+            self.check_func(lambda x: x.str.index("not-found"))
+
+    def test_string_join(self):
+        pser = pd.Series([["a", "b", "c"], ["xx", "yy", "zz"]])
+        self.check_func_on_series(lambda x: x.str.join("-"), pser)
+        self.check_func(lambda x: x.str.join("-"))
+
+    def test_string_len(self):
+        self.check_func(lambda x: x.str.len())
+        pser = pd.Series([["a", "b", "c"], ["xx"], []])
+        self.check_func_on_series(lambda x: x.str.len(), pser)
+
+    def test_string_ljust(self):
+        self.check_func(lambda x: x.str.ljust(0))
+        self.check_func(lambda x: x.str.ljust(10))
+        self.check_func(lambda x: x.str.ljust(30, "x"))
+
+    def test_string_match(self):
+        self.check_func(lambda x: x.str.match("in"))
+        self.check_func(lambda x: x.str.match("apples|carrots", na=False))
+        self.check_func(lambda x: x.str.match("White", case=True))
+        self.check_func(lambda x: x.str.match("BANANAS", flags=re.IGNORECASE))
+
+    def test_string_normalize(self):
+        self.check_func(lambda x: x.str.normalize("NFC"))
+        self.check_func(lambda x: x.str.normalize("NFKD"))
+
+    def test_string_pad(self):
+        self.check_func(lambda x: x.str.pad(10))
+        self.check_func(lambda x: x.str.pad(10, side="both"))
+        self.check_func(lambda x: x.str.pad(10, side="right", fillchar="-"))
+
+    def test_string_partition(self):
+        with self.assertRaises(NotImplementedError):
+            self.check_func(lambda x: x.str.partition())
+
+    def test_string_repeat(self):
+        self.check_func(lambda x: x.str.repeat(repeats=3))
+        with self.assertRaises(TypeError):
+            self.check_func(lambda x: x.str.repeat(repeats=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]))
+
+    def test_string_replace(self):
+        self.check_func(lambda x: x.str.replace("a.", "xx", regex=True))
+        self.check_func(lambda x: x.str.replace("a.", "xx", regex=False))
+        self.check_func(lambda x: x.str.replace("ing", "0", flags=re.IGNORECASE))
+        # reverse every lowercase word
+        repl = lambda m: m.group(0)[::-1]
+        self.check_func(lambda x: x.str.replace(r"[a-z]+", repl))
+        # compiled regex with flags
+        regex_pat = re.compile(r"WHITESPACE", flags=re.IGNORECASE)
+        self.check_func(lambda x: x.str.replace(regex_pat, "---"))
+
+    def test_string_rfind(self):
+        self.check_func(lambda x: x.str.rfind("a"))
+        self.check_func(lambda x: x.str.rfind("a", start=3))
+        self.check_func(lambda x: x.str.rfind("a", start=0, end=1))
+
+    def test_string_rindex(self):
+        pser = pd.Series(["teatea", "eateat"])
+        self.check_func_on_series(lambda x: x.str.rindex("ea"), pser)
+        with self.assertRaises(Exception):
+            self.check_func_on_series(lambda x: x.str.rindex("ea", start=0, end=2), pser)
+        with self.assertRaises(Exception):
+            self.check_func(lambda x: x.str.rindex("not-found"))
+
+    def test_string_rjust(self):
+        self.check_func(lambda x: x.str.rjust(0))
+        self.check_func(lambda x: x.str.rjust(10))
+        self.check_func(lambda x: x.str.rjust(30, "x"))
+
+    def test_string_rpartition(self):
+        with self.assertRaises(NotImplementedError):
+            self.check_func(lambda x: x.str.rpartition())
+
+    def test_string_slice(self):
+        self.check_func(lambda x: x.str.slice(start=1))
+        self.check_func(lambda x: x.str.slice(stop=3))
+        self.check_func(lambda x: x.str.slice(step=2))
+        self.check_func(lambda x: x.str.slice(start=0, stop=5, step=3))
+
+    def test_string_slice_replace(self):
+        self.check_func(lambda x: x.str.slice_replace(1, repl="X"))
+        self.check_func(lambda x: x.str.slice_replace(stop=2, repl="X"))
+        self.check_func(lambda x: x.str.slice_replace(start=1, stop=3, repl="X"))
+
+    def test_string_split(self):
+        self.check_func_on_series(lambda x: repr(x.str.split()), self.pser[:-1])
+        self.check_func_on_series(lambda x: repr(x.str.split(r"p*")), self.pser[:-1])
+        pser = pd.Series(["This is a sentence.", "This-is-a-long-word."])
+        self.check_func_on_series(lambda x: repr(x.str.split(n=2)), pser)
+        self.check_func_on_series(lambda x: repr(x.str.split(pat="-", n=2)), pser)
+        self.check_func_on_series(lambda x: x.str.split(n=2, expand=True), pser, almost=True)
+        with self.assertRaises(NotImplementedError):
+            self.check_func(lambda x: x.str.split(expand=True))
+
+    def test_string_rsplit(self):
+        self.check_func_on_series(lambda x: repr(x.str.rsplit()), self.pser[:-1])
+        self.check_func_on_series(lambda x: repr(x.str.rsplit(r"p*")), self.pser[:-1])
+        pser = pd.Series(["This is a sentence.", "This-is-a-long-word."])
+        self.check_func_on_series(lambda x: repr(x.str.rsplit(n=2)), pser)
+        self.check_func_on_series(lambda x: repr(x.str.rsplit(pat="-", n=2)), pser)
+        self.check_func_on_series(lambda x: x.str.rsplit(n=2, expand=True), pser, almost=True)
+        with self.assertRaises(NotImplementedError):
+            self.check_func(lambda x: x.str.rsplit(expand=True))
+
+    def test_string_translate(self):
+        m = str.maketrans({"a": "X", "e": "Y", "i": None})
+        self.check_func(lambda x: x.str.translate(m))
+
+    def test_string_wrap(self):
+        self.check_func(lambda x: x.str.wrap(5))
+        self.check_func(lambda x: x.str.wrap(5, expand_tabs=False))
+        self.check_func(lambda x: x.str.wrap(5, replace_whitespace=False))
+        self.check_func(lambda x: x.str.wrap(5, drop_whitespace=False))
+        self.check_func(lambda x: x.str.wrap(5, break_long_words=False))
+        self.check_func(lambda x: x.str.wrap(5, break_on_hyphens=False))
+
+    def test_string_zfill(self):
+        self.check_func(lambda x: x.str.zfill(10))
+
+    def test_string_get_dummies(self):
+        with self.assertRaises(NotImplementedError):
+            self.check_func(lambda x: x.str.get_dummies())
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.pandas.tests.test_series_string import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_sql.py b/python/pyspark/pandas/tests/test_sql.py
new file mode 100644
index 0000000000000..6c3405f0f03cd
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_sql.py
@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from pyspark import pandas as ps
+from pyspark.sql.utils import ParseException
+from pyspark.testing.pandasutils import PandasOnSparkTestCase
+from pyspark.testing.sqlutils import SQLTestUtils
+
+
+class SQLTest(PandasOnSparkTestCase, SQLTestUtils):
+    def test_error_variable_not_exist(self):
+        msg = "The key variable_foo in the SQL statement was not found.*"
+        with self.assertRaisesRegex(ValueError, msg):
+            ps.sql("select * from {variable_foo}")
+
+    def test_error_unsupported_type(self):
+        msg = "Unsupported variable type dict: {'a': 1}"
+        with self.assertRaisesRegex(ValueError, msg):
+            some_dict = {"a": 1}
+            ps.sql("select * from {some_dict}")
+
+    def test_error_bad_sql(self):
+        with self.assertRaises(ParseException):
+            ps.sql("this is not valid sql")
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.pandas.tests.test_sql import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_stats.py b/python/pyspark/pandas/tests/test_stats.py
new file mode 100644
index 0000000000000..905add4baaae1
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_stats.py
@@ -0,0 +1,409 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from distutils.version import LooseVersion
+
+import numpy as np
+import pandas as pd
+
+try:
+    from pandas._testing import makeMissingDataframe
+except ImportError:
+    from pandas.util.testing import makeMissingDataframe
+
+from pyspark import pandas as ps
+from pyspark.pandas.config import option_context
+from pyspark.testing.pandasutils import PandasOnSparkTestCase, SPARK_CONF_ARROW_ENABLED
+from pyspark.testing.sqlutils import SQLTestUtils
+
+
+class StatsTest(PandasOnSparkTestCase, SQLTestUtils):
+    def _test_stat_functions(self, pdf_or_pser, kdf_or_kser):
+        functions = ["max", "min", "mean", "sum", "count"]
+        for funcname in functions:
+            self.assert_eq(getattr(kdf_or_kser, funcname)(), getattr(pdf_or_pser, funcname)())
+
+        functions = ["std", "var", "product", "sem"]
+        for funcname in functions:
+            self.assert_eq(
+                getattr(kdf_or_kser, funcname)(),
+                getattr(pdf_or_pser, funcname)(),
+                check_exact=False,
+            )
+
+        functions = ["std", "var", "sem"]
+        for funcname in functions:
+            self.assert_eq(
+                getattr(kdf_or_kser, funcname)(ddof=0),
+                getattr(pdf_or_pser, funcname)(ddof=0),
+                check_exact=False,
+            )
+
+        # NOTE: To test skew, kurt, and median, just make sure they run.
+        #       The numbers are different in spark and pandas.
+        functions = ["skew", "kurt", "median"]
+        for funcname in functions:
+            getattr(kdf_or_kser, funcname)()
+
+    def test_stat_functions(self):
+        pdf = pd.DataFrame({"A": [1, 2, 3, 4], "B": [1, 2, 3, 4], "C": [1, np.nan, 3, np.nan]})
+        kdf = ps.from_pandas(pdf)
+        self._test_stat_functions(pdf.A, kdf.A)
+        self._test_stat_functions(pdf, kdf)
+
+        # empty
+        self._test_stat_functions(pdf.A.loc[[]], kdf.A.loc[[]])
+        self._test_stat_functions(pdf.loc[[]], kdf.loc[[]])
+
+    def test_stat_functions_multiindex_column(self):
+        arrays = [np.array(["A", "A", "B", "B"]), np.array(["one", "two", "one", "two"])]
+        pdf = pd.DataFrame(np.random.randn(3, 4), index=["A", "B", "C"], columns=arrays)
+        kdf = ps.from_pandas(pdf)
+        self._test_stat_functions(pdf.A, kdf.A)
+        self._test_stat_functions(pdf, kdf)
+
+    def test_stat_functions_with_no_numeric_columns(self):
+        pdf = pd.DataFrame(
+            {
+                "A": ["a", None, "c", "d", None, "f", "g"],
+                "B": ["A", "B", "C", None, "E", "F", None],
+            }
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self._test_stat_functions(pdf, kdf)
+
+    def test_sum(self):
+        pdf = pd.DataFrame({"a": [1, 2, 3, np.nan], "b": [0.1, np.nan, 0.3, np.nan]})
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.sum(), pdf.sum())
+        self.assert_eq(kdf.sum(axis=1), pdf.sum(axis=1))
+        self.assert_eq(kdf.sum(min_count=3), pdf.sum(min_count=3))
+        self.assert_eq(kdf.sum(axis=1, min_count=1), pdf.sum(axis=1, min_count=1))
+        self.assert_eq(kdf.loc[[]].sum(), pdf.loc[[]].sum())
+        self.assert_eq(kdf.loc[[]].sum(min_count=1), pdf.loc[[]].sum(min_count=1))
+
+        self.assert_eq(kdf["a"].sum(), pdf["a"].sum())
+        self.assert_eq(kdf["a"].sum(min_count=3), pdf["a"].sum(min_count=3))
+        self.assert_eq(kdf["b"].sum(min_count=3), pdf["b"].sum(min_count=3))
+        self.assert_eq(kdf["a"].loc[[]].sum(), pdf["a"].loc[[]].sum())
+        self.assert_eq(kdf["a"].loc[[]].sum(min_count=1), pdf["a"].loc[[]].sum(min_count=1))
+
+    def test_product(self):
+        pdf = pd.DataFrame(
+            {"a": [1, -2, -3, np.nan], "b": [0.1, np.nan, -0.3, np.nan], "c": [10, 20, 0, -10]}
+        )
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.product(), pdf.product(), check_exact=False)
+        self.assert_eq(kdf.product(axis=1), pdf.product(axis=1))
+        self.assert_eq(kdf.product(min_count=3), pdf.product(min_count=3), check_exact=False)
+        self.assert_eq(kdf.product(axis=1, min_count=1), pdf.product(axis=1, min_count=1))
+        self.assert_eq(kdf.loc[[]].product(), pdf.loc[[]].product())
+        self.assert_eq(kdf.loc[[]].product(min_count=1), pdf.loc[[]].product(min_count=1))
+
+        self.assert_eq(kdf["a"].product(), pdf["a"].product(), check_exact=False)
+        self.assert_eq(
+            kdf["a"].product(min_count=3), pdf["a"].product(min_count=3), check_exact=False
+        )
+        self.assert_eq(kdf["b"].product(min_count=3), pdf["b"].product(min_count=3))
+        self.assert_eq(kdf["c"].product(min_count=3), pdf["c"].product(min_count=3))
+        self.assert_eq(kdf["a"].loc[[]].product(), pdf["a"].loc[[]].product())
+        self.assert_eq(kdf["a"].loc[[]].product(min_count=1), pdf["a"].loc[[]].product(min_count=1))
+
+    def test_abs(self):
+        pdf = pd.DataFrame(
+            {
+                "A": [1, -2, np.nan, -4, 5],
+                "B": [1.0, -2, np.nan, -4, 5],
+                "C": [-6.0, -7, -8, np.nan, 10],
+                "D": ["a", "b", "c", "d", np.nan],
+                "E": [True, np.nan, False, True, True],
+            }
+        )
+        kdf = ps.from_pandas(pdf)
+        self.assert_eq(kdf.A.abs(), pdf.A.abs())
+        self.assert_eq(kdf.B.abs(), pdf.B.abs())
+        self.assert_eq(kdf.E.abs(), pdf.E.abs())
+        # pandas' bug?
+        # self.assert_eq(kdf[["B", "C", "E"]].abs(), pdf[["B", "C", "E"]].abs())
+        self.assert_eq(kdf[["B", "C"]].abs(), pdf[["B", "C"]].abs())
+        self.assert_eq(kdf[["E"]].abs(), pdf[["E"]].abs())
+
+        with self.assertRaisesRegex(
+            TypeError, "bad operand type for abs\\(\\): object \\(string\\)"
+        ):
+            kdf.abs()
+        with self.assertRaisesRegex(
+            TypeError, "bad operand type for abs\\(\\): object \\(string\\)"
+        ):
+            kdf.D.abs()
+
+    def test_axis_on_dataframe(self):
+        # The number of each count is intentionally big
+        # because when data is small, it executes a shortcut.
+        # Less than 'compute.shortcut_limit' will execute a shortcut
+        # by using collected pandas dataframe directly.
+        # now we set the 'compute.shortcut_limit' as 1000 explicitly
+        with option_context("compute.shortcut_limit", 1000):
+            pdf = pd.DataFrame(
+                {
+                    "A": [1, -2, 3, -4, 5] * 300,
+                    "B": [1.0, -2, 3, -4, 5] * 300,
+                    "C": [-6.0, -7, -8, -9, 10] * 300,
+                    "D": [True, False, True, False, False] * 300,
+                },
+                index=range(10, 15001, 10),
+            )
+            kdf = ps.from_pandas(pdf)
+            self.assert_eq(kdf.count(axis=1), pdf.count(axis=1))
+            self.assert_eq(kdf.var(axis=1), pdf.var(axis=1))
+            self.assert_eq(kdf.var(axis=1, ddof=0), pdf.var(axis=1, ddof=0))
+            self.assert_eq(kdf.std(axis=1), pdf.std(axis=1))
+            self.assert_eq(kdf.std(axis=1, ddof=0), pdf.std(axis=1, ddof=0))
+            self.assert_eq(kdf.max(axis=1), pdf.max(axis=1))
+            self.assert_eq(kdf.min(axis=1), pdf.min(axis=1))
+            self.assert_eq(kdf.sum(axis=1), pdf.sum(axis=1))
+            self.assert_eq(kdf.product(axis=1), pdf.product(axis=1))
+            self.assert_eq(kdf.kurtosis(axis=1), pdf.kurtosis(axis=1))
+            self.assert_eq(kdf.skew(axis=1), pdf.skew(axis=1))
+            self.assert_eq(kdf.mean(axis=1), pdf.mean(axis=1))
+            self.assert_eq(kdf.sem(axis=1), pdf.sem(axis=1))
+            self.assert_eq(kdf.sem(axis=1, ddof=0), pdf.sem(axis=1, ddof=0))
+
+            self.assert_eq(
+                kdf.count(axis=1, numeric_only=True), pdf.count(axis=1, numeric_only=True)
+            )
+            self.assert_eq(kdf.var(axis=1, numeric_only=True), pdf.var(axis=1, numeric_only=True))
+            self.assert_eq(
+                kdf.var(axis=1, ddof=0, numeric_only=True),
+                pdf.var(axis=1, ddof=0, numeric_only=True),
+            )
+            self.assert_eq(kdf.std(axis=1, numeric_only=True), pdf.std(axis=1, numeric_only=True))
+            self.assert_eq(
+                kdf.std(axis=1, ddof=0, numeric_only=True),
+                pdf.std(axis=1, ddof=0, numeric_only=True),
+            )
+            self.assert_eq(
+                kdf.max(axis=1, numeric_only=True), pdf.max(axis=1, numeric_only=True).astype(float)
+            )
+            self.assert_eq(
+                kdf.min(axis=1, numeric_only=True), pdf.min(axis=1, numeric_only=True).astype(float)
+            )
+            self.assert_eq(
+                kdf.sum(axis=1, numeric_only=True), pdf.sum(axis=1, numeric_only=True).astype(float)
+            )
+            self.assert_eq(
+                kdf.product(axis=1, numeric_only=True),
+                pdf.product(axis=1, numeric_only=True).astype(float),
+            )
+            self.assert_eq(
+                kdf.kurtosis(axis=1, numeric_only=True), pdf.kurtosis(axis=1, numeric_only=True)
+            )
+            self.assert_eq(kdf.skew(axis=1, numeric_only=True), pdf.skew(axis=1, numeric_only=True))
+            self.assert_eq(kdf.mean(axis=1, numeric_only=True), pdf.mean(axis=1, numeric_only=True))
+            self.assert_eq(kdf.sem(axis=1, numeric_only=True), pdf.sem(axis=1, numeric_only=True))
+            self.assert_eq(
+                kdf.sem(axis=1, ddof=0, numeric_only=True),
+                pdf.sem(axis=1, ddof=0, numeric_only=True),
+            )
+
+    def test_corr(self):
+        # Disable arrow execution since corr() is using UDT internally which is not supported.
+        with self.sql_conf({SPARK_CONF_ARROW_ENABLED: False}):
+            # DataFrame
+            # we do not handle NaNs for now
+            pdf = makeMissingDataframe(0.3, 42).fillna(0)
+            kdf = ps.from_pandas(pdf)
+
+            self.assert_eq(kdf.corr(), pdf.corr(), check_exact=False)
+
+            # Series
+            pser_a = pdf.A
+            pser_b = pdf.B
+            kser_a = kdf.A
+            kser_b = kdf.B
+
+            self.assertAlmostEqual(kser_a.corr(kser_b), pser_a.corr(pser_b))
+            self.assertRaises(TypeError, lambda: kser_a.corr(kdf))
+
+            # multi-index columns
+            columns = pd.MultiIndex.from_tuples([("X", "A"), ("X", "B"), ("Y", "C"), ("Z", "D")])
+            pdf.columns = columns
+            kdf.columns = columns
+
+            self.assert_eq(kdf.corr(), pdf.corr(), check_exact=False)
+
+            # Series
+            pser_xa = pdf[("X", "A")]
+            pser_xb = pdf[("X", "B")]
+            kser_xa = kdf[("X", "A")]
+            kser_xb = kdf[("X", "B")]
+
+            self.assert_eq(kser_xa.corr(kser_xb), pser_xa.corr(pser_xb), almost=True)
+
+    def test_cov_corr_meta(self):
+        # Disable arrow execution since corr() is using UDT internally which is not supported.
+        with self.sql_conf({SPARK_CONF_ARROW_ENABLED: False}):
+            pdf = pd.DataFrame(
+                {
+                    "a": np.array([1, 2, 3], dtype="i1"),
+                    "b": np.array([1, 2, 3], dtype="i2"),
+                    "c": np.array([1, 2, 3], dtype="i4"),
+                    "d": np.array([1, 2, 3]),
+                    "e": np.array([1.0, 2.0, 3.0], dtype="f4"),
+                    "f": np.array([1.0, 2.0, 3.0]),
+                    "g": np.array([True, False, True]),
+                    "h": np.array(list("abc")),
+                },
+                index=pd.Index([1, 2, 3], name="myindex"),
+            )
+            kdf = ps.from_pandas(pdf)
+            self.assert_eq(kdf.corr(), pdf.corr())
+
+    def test_stats_on_boolean_dataframe(self):
+        pdf = pd.DataFrame({"A": [True, False, True], "B": [False, False, True]})
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(kdf.min(), pdf.min())
+        self.assert_eq(kdf.max(), pdf.max())
+        self.assert_eq(kdf.count(), pdf.count())
+
+        self.assert_eq(kdf.sum(), pdf.sum())
+        self.assert_eq(kdf.product(), pdf.product())
+        self.assert_eq(kdf.mean(), pdf.mean())
+
+        self.assert_eq(kdf.var(), pdf.var(), check_exact=False)
+        self.assert_eq(kdf.var(ddof=0), pdf.var(ddof=0), check_exact=False)
+        self.assert_eq(kdf.std(), pdf.std(), check_exact=False)
+        self.assert_eq(kdf.std(ddof=0), pdf.std(ddof=0), check_exact=False)
+        self.assert_eq(kdf.sem(), pdf.sem(), check_exact=False)
+        self.assert_eq(kdf.sem(ddof=0), pdf.sem(ddof=0), check_exact=False)
+
+    def test_stats_on_boolean_series(self):
+        pser = pd.Series([True, False, True])
+        kser = ps.from_pandas(pser)
+
+        self.assert_eq(kser.min(), pser.min())
+        self.assert_eq(kser.max(), pser.max())
+        self.assert_eq(kser.count(), pser.count())
+
+        self.assert_eq(kser.sum(), pser.sum())
+        self.assert_eq(kser.product(), pser.product())
+        self.assert_eq(kser.mean(), pser.mean())
+
+        self.assert_eq(kser.var(), pser.var(), almost=True)
+        self.assert_eq(kser.var(ddof=0), pser.var(ddof=0), almost=True)
+        self.assert_eq(kser.std(), pser.std(), almost=True)
+        self.assert_eq(kser.std(ddof=0), pser.std(ddof=0), almost=True)
+        self.assert_eq(kser.sem(), pser.sem(), almost=True)
+        self.assert_eq(kser.sem(ddof=0), pser.sem(ddof=0), almost=True)
+
+    def test_stats_on_non_numeric_columns_should_be_discarded_if_numeric_only_is_true(self):
+        pdf = pd.DataFrame({"i": [0, 1, 2], "b": [False, False, True], "s": ["x", "y", "z"]})
+        kdf = ps.from_pandas(pdf)
+
+        self.assert_eq(
+            kdf[["i", "s"]].max(numeric_only=True), pdf[["i", "s"]].max(numeric_only=True)
+        )
+        self.assert_eq(
+            kdf[["b", "s"]].max(numeric_only=True), pdf[["b", "s"]].max(numeric_only=True)
+        )
+        self.assert_eq(
+            kdf[["i", "s"]].min(numeric_only=True), pdf[["i", "s"]].min(numeric_only=True)
+        )
+        self.assert_eq(
+            kdf[["b", "s"]].min(numeric_only=True), pdf[["b", "s"]].min(numeric_only=True)
+        )
+        self.assert_eq(kdf.count(numeric_only=True), pdf.count(numeric_only=True))
+
+        if LooseVersion(pd.__version__) >= LooseVersion("1.0.0"):
+            self.assert_eq(kdf.sum(numeric_only=True), pdf.sum(numeric_only=True))
+            self.assert_eq(kdf.product(numeric_only=True), pdf.product(numeric_only=True))
+        else:
+            self.assert_eq(kdf.sum(numeric_only=True), pdf.sum(numeric_only=True).astype(int))
+            self.assert_eq(
+                kdf.product(numeric_only=True), pdf.product(numeric_only=True).astype(int)
+            )
+
+        self.assert_eq(kdf.mean(numeric_only=True), pdf.mean(numeric_only=True))
+
+        self.assert_eq(kdf.var(numeric_only=True), pdf.var(numeric_only=True), check_exact=False)
+        self.assert_eq(
+            kdf.var(ddof=0, numeric_only=True),
+            pdf.var(ddof=0, numeric_only=True),
+            check_exact=False,
+        )
+        self.assert_eq(kdf.std(numeric_only=True), pdf.std(numeric_only=True), check_exact=False)
+        self.assert_eq(
+            kdf.std(ddof=0, numeric_only=True),
+            pdf.std(ddof=0, numeric_only=True),
+            check_exact=False,
+        )
+        self.assert_eq(kdf.sem(numeric_only=True), pdf.sem(numeric_only=True), check_exact=False)
+        self.assert_eq(
+            kdf.sem(ddof=0, numeric_only=True),
+            pdf.sem(ddof=0, numeric_only=True),
+            check_exact=False,
+        )
+
+        self.assert_eq(len(kdf.median(numeric_only=True)), len(pdf.median(numeric_only=True)))
+        self.assert_eq(len(kdf.kurtosis(numeric_only=True)), len(pdf.kurtosis(numeric_only=True)))
+        self.assert_eq(len(kdf.skew(numeric_only=True)), len(pdf.skew(numeric_only=True)))
+
+        self.assert_eq(
+            len(kdf.quantile(q=0.5, numeric_only=True)), len(pdf.quantile(q=0.5, numeric_only=True))
+        )
+        self.assert_eq(
+            len(kdf.quantile(q=[0.25, 0.5, 0.75], numeric_only=True)),
+            len(pdf.quantile(q=[0.25, 0.5, 0.75], numeric_only=True)),
+        )
+
+    def test_numeric_only_unsupported(self):
+        pdf = pd.DataFrame({"i": [0, 1, 2], "b": [False, False, True], "s": ["x", "y", "z"]})
+        kdf = ps.from_pandas(pdf)
+
+        if LooseVersion(pd.__version__) >= LooseVersion("1.0.0"):
+            self.assert_eq(kdf.sum(numeric_only=True), pdf.sum(numeric_only=True))
+            self.assert_eq(
+                kdf[["i", "b"]].sum(numeric_only=False), pdf[["i", "b"]].sum(numeric_only=False)
+            )
+        else:
+            self.assert_eq(kdf.sum(numeric_only=True), pdf.sum(numeric_only=True).astype(int))
+            self.assert_eq(
+                kdf[["i", "b"]].sum(numeric_only=False),
+                pdf[["i", "b"]].sum(numeric_only=False).astype(int),
+            )
+
+        with self.assertRaisesRegex(TypeError, "Could not convert object \\(string\\) to numeric"):
+            kdf.sum(numeric_only=False)
+
+        with self.assertRaisesRegex(TypeError, "Could not convert object \\(string\\) to numeric"):
+            kdf.s.sum()
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.pandas.tests.test_stats import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_typedef.py b/python/pyspark/pandas/tests/test_typedef.py
new file mode 100644
index 0000000000000..53fc3734f8f14
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_typedef.py
@@ -0,0 +1,437 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import sys
+import unittest
+import datetime
+import decimal
+from typing import List
+
+import pandas
+import pandas as pd
+from pandas.api.types import CategoricalDtype
+import numpy as np
+from pyspark.sql.types import (
+    ArrayType,
+    BinaryType,
+    BooleanType,
+    FloatType,
+    IntegerType,
+    LongType,
+    StringType,
+    StructField,
+    StructType,
+    ByteType,
+    ShortType,
+    DateType,
+    DecimalType,
+    DoubleType,
+    TimestampType,
+)
+
+from pyspark.pandas.typedef import (
+    as_spark_type,
+    extension_dtypes_available,
+    extension_float_dtypes_available,
+    extension_object_dtypes_available,
+    infer_return_type,
+    pandas_on_spark_type,
+)
+from pyspark import pandas as ps
+
+
+class TypeHintTests(unittest.TestCase):
+    @unittest.skipIf(
+        sys.version_info < (3, 7),
+        "Type inference from pandas instances is supported with Python 3.7+",
+    )
+    def test_infer_schema_from_pandas_instances(self):
+        def func() -> pd.Series[int]:
+            pass
+
+        inferred = infer_return_type(func)
+        self.assertEqual(inferred.dtype, np.int64)
+        self.assertEqual(inferred.spark_type, LongType())
+
+        def func() -> pd.Series[np.float]:
+            pass
+
+        inferred = infer_return_type(func)
+        self.assertEqual(inferred.dtype, np.float64)
+        self.assertEqual(inferred.spark_type, DoubleType())
+
+        def func() -> "pd.DataFrame[np.float, str]":
+            pass
+
+        expected = StructType([StructField("c0", DoubleType()), StructField("c1", StringType())])
+        inferred = infer_return_type(func)
+        self.assertEqual(inferred.dtypes, [np.float64, np.unicode_])
+        self.assertEqual(inferred.spark_type, expected)
+
+        def func() -> "pandas.DataFrame[np.float]":
+            pass
+
+        expected = StructType([StructField("c0", DoubleType())])
+        inferred = infer_return_type(func)
+        self.assertEqual(inferred.dtypes, [np.float64])
+        self.assertEqual(inferred.spark_type, expected)
+
+        def func() -> "pd.Series[int]":
+            pass
+
+        inferred = infer_return_type(func)
+        self.assertEqual(inferred.dtype, np.int64)
+        self.assertEqual(inferred.spark_type, LongType())
+
+        def func() -> pd.DataFrame[np.float, str]:
+            pass
+
+        expected = StructType([StructField("c0", DoubleType()), StructField("c1", StringType())])
+        inferred = infer_return_type(func)
+        self.assertEqual(inferred.dtypes, [np.float64, np.unicode_])
+        self.assertEqual(inferred.spark_type, expected)
+
+        def func() -> pd.DataFrame[np.float]:
+            pass
+
+        expected = StructType([StructField("c0", DoubleType())])
+        inferred = infer_return_type(func)
+        self.assertEqual(inferred.dtypes, [np.float64])
+        self.assertEqual(inferred.spark_type, expected)
+
+        pdf = pd.DataFrame({"a": [1, 2, 3], "b": [3, 4, 5]})
+
+        def func() -> pd.DataFrame[pdf.dtypes]:  # type: ignore
+            pass
+
+        expected = StructType([StructField("c0", LongType()), StructField("c1", LongType())])
+        inferred = infer_return_type(func)
+        self.assertEqual(inferred.dtypes, [np.int64, np.int64])
+        self.assertEqual(inferred.spark_type, expected)
+
+        pdf = pd.DataFrame({"a": [1, 2, 3], "b": pd.Categorical(["a", "b", "c"])})
+
+        def func() -> pd.Series[pdf.b.dtype]:  # type: ignore
+            pass
+
+        inferred = infer_return_type(func)
+        self.assertEqual(inferred.dtype, CategoricalDtype(categories=["a", "b", "c"]))
+        self.assertEqual(inferred.spark_type, LongType())
+
+        def func() -> pd.DataFrame[pdf.dtypes]:  # type: ignore
+            pass
+
+        expected = StructType([StructField("c0", LongType()), StructField("c1", LongType())])
+        inferred = infer_return_type(func)
+        self.assertEqual(inferred.dtypes, [np.int64, CategoricalDtype(categories=["a", "b", "c"])])
+        self.assertEqual(inferred.spark_type, expected)
+
+    def test_if_pandas_implements_class_getitem(self):
+        # the current type hint implementation of pandas DataFrame assumes pandas doesn't
+        # implement '__class_getitem__'. This test case is to make sure pandas
+        # doesn't implement them.
+        assert not ps._frame_has_class_getitem
+        assert not ps._series_has_class_getitem
+
+    @unittest.skipIf(
+        sys.version_info < (3, 7),
+        "Type inference from pandas instances is supported with Python 3.7+",
+    )
+    def test_infer_schema_with_names_pandas_instances(self):
+        def func() -> 'pd.DataFrame["a" : np.float, "b":str]':  # noqa: F405
+            pass
+
+        expected = StructType([StructField("a", DoubleType()), StructField("b", StringType())])
+        inferred = infer_return_type(func)
+        self.assertEqual(inferred.dtypes, [np.float64, np.unicode_])
+        self.assertEqual(inferred.spark_type, expected)
+
+        def func() -> "pd.DataFrame['a': np.float, 'b': int]":  # noqa: F405
+            pass
+
+        expected = StructType([StructField("a", DoubleType()), StructField("b", LongType())])
+        inferred = infer_return_type(func)
+        self.assertEqual(inferred.dtypes, [np.float64, np.int64])
+        self.assertEqual(inferred.spark_type, expected)
+
+        pdf = pd.DataFrame({"a": [1, 2, 3], "b": [3, 4, 5]})
+
+        def func() -> pd.DataFrame[zip(pdf.columns, pdf.dtypes)]:
+            pass
+
+        expected = StructType([StructField("a", LongType()), StructField("b", LongType())])
+        inferred = infer_return_type(func)
+        self.assertEqual(inferred.dtypes, [np.int64, np.int64])
+        self.assertEqual(inferred.spark_type, expected)
+
+        pdf = pd.DataFrame({("x", "a"): [1, 2, 3], ("y", "b"): [3, 4, 5]})
+
+        def func() -> pd.DataFrame[zip(pdf.columns, pdf.dtypes)]:
+            pass
+
+        expected = StructType(
+            [StructField("(x, a)", LongType()), StructField("(y, b)", LongType())]
+        )
+        inferred = infer_return_type(func)
+        self.assertEqual(inferred.dtypes, [np.int64, np.int64])
+        self.assertEqual(inferred.spark_type, expected)
+
+        pdf = pd.DataFrame({"a": [1, 2, 3], "b": pd.Categorical(["a", "b", "c"])})
+
+        def func() -> pd.DataFrame[zip(pdf.columns, pdf.dtypes)]:
+            pass
+
+        expected = StructType([StructField("a", LongType()), StructField("b", LongType())])
+        inferred = infer_return_type(func)
+        self.assertEqual(inferred.dtypes, [np.int64, CategoricalDtype(categories=["a", "b", "c"])])
+        self.assertEqual(inferred.spark_type, expected)
+
+    @unittest.skipIf(
+        sys.version_info < (3, 7),
+        "Type inference from pandas instances is supported with Python 3.7+",
+    )
+    def test_infer_schema_with_names_pandas_instances_negative(self):
+        def try_infer_return_type():
+            def f() -> 'pd.DataFrame["a" : np.float : 1, "b":str:2]':  # noqa: F405
+                pass
+
+            infer_return_type(f)
+
+        self.assertRaisesRegex(TypeError, "Type hints should be specified", try_infer_return_type)
+
+        class A:
+            pass
+
+        def try_infer_return_type():
+            def f() -> pd.DataFrame[A]:
+                pass
+
+            infer_return_type(f)
+
+        self.assertRaisesRegex(TypeError, "not understood", try_infer_return_type)
+
+        def try_infer_return_type():
+            def f() -> 'pd.DataFrame["a" : np.float : 1, "b":str:2]':  # noqa: F405
+                pass
+
+            infer_return_type(f)
+
+        self.assertRaisesRegex(TypeError, "Type hints should be specified", try_infer_return_type)
+
+        # object type
+        pdf = pd.DataFrame({"a": ["a", 2, None]})
+
+        def try_infer_return_type():
+            def f() -> pd.DataFrame[pdf.dtypes]:  # type: ignore
+                pass
+
+            infer_return_type(f)
+
+        self.assertRaisesRegex(TypeError, "object.*not understood", try_infer_return_type)
+
+        def try_infer_return_type():
+            def f() -> pd.Series[pdf.a.dtype]:  # type: ignore
+                pass
+
+            infer_return_type(f)
+
+        self.assertRaisesRegex(TypeError, "object.*not understood", try_infer_return_type)
+
+    def test_infer_schema_with_names_negative(self):
+        def try_infer_return_type():
+            def f() -> 'ps.DataFrame["a" : np.float : 1, "b":str:2]':  # noqa: F405
+                pass
+
+            infer_return_type(f)
+
+        self.assertRaisesRegex(TypeError, "Type hints should be specified", try_infer_return_type)
+
+        class A:
+            pass
+
+        def try_infer_return_type():
+            def f() -> ps.DataFrame[A]:
+                pass
+
+            infer_return_type(f)
+
+        self.assertRaisesRegex(TypeError, "not understood", try_infer_return_type)
+
+        def try_infer_return_type():
+            def f() -> 'ps.DataFrame["a" : np.float : 1, "b":str:2]':  # noqa: F405
+                pass
+
+            infer_return_type(f)
+
+        self.assertRaisesRegex(TypeError, "Type hints should be specified", try_infer_return_type)
+
+        # object type
+        pdf = pd.DataFrame({"a": ["a", 2, None]})
+
+        def try_infer_return_type():
+            def f() -> ps.DataFrame[pdf.dtypes]:  # type: ignore
+                pass
+
+            infer_return_type(f)
+
+        self.assertRaisesRegex(TypeError, "object.*not understood", try_infer_return_type)
+
+        def try_infer_return_type():
+            def f() -> ps.Series[pdf.a.dtype]:  # type: ignore
+                pass
+
+            infer_return_type(f)
+
+        self.assertRaisesRegex(TypeError, "object.*not understood", try_infer_return_type)
+
+    def test_as_spark_type_koalas_dtype(self):
+        type_mapper = {
+            # binary
+            np.character: (np.character, BinaryType()),
+            np.bytes_: (np.bytes_, BinaryType()),
+            np.string_: (np.bytes_, BinaryType()),
+            bytes: (np.bytes_, BinaryType()),
+            # integer
+            np.int8: (np.int8, ByteType()),
+            np.byte: (np.int8, ByteType()),
+            np.int16: (np.int16, ShortType()),
+            np.int32: (np.int32, IntegerType()),
+            np.int64: (np.int64, LongType()),
+            np.int: (np.int64, LongType()),
+            int: (np.int64, LongType()),
+            # floating
+            np.float32: (np.float32, FloatType()),
+            np.float: (np.float64, DoubleType()),
+            np.float64: (np.float64, DoubleType()),
+            float: (np.float64, DoubleType()),
+            # string
+            np.str: (np.unicode_, StringType()),
+            np.unicode_: (np.unicode_, StringType()),
+            str: (np.unicode_, StringType()),
+            # bool
+            np.bool: (np.bool, BooleanType()),
+            bool: (np.bool, BooleanType()),
+            # datetime
+            np.datetime64: (np.datetime64, TimestampType()),
+            datetime.datetime: (np.dtype("datetime64[ns]"), TimestampType()),
+            # DateType
+            datetime.date: (np.dtype("object"), DateType()),
+            # DecimalType
+            decimal.Decimal: (np.dtype("object"), DecimalType(38, 18)),
+            # ArrayType
+            np.ndarray: (np.dtype("object"), ArrayType(StringType())),
+            List[bytes]: (np.dtype("object"), ArrayType(BinaryType())),
+            List[np.character]: (np.dtype("object"), ArrayType(BinaryType())),
+            List[np.bytes_]: (np.dtype("object"), ArrayType(BinaryType())),
+            List[np.string_]: (np.dtype("object"), ArrayType(BinaryType())),
+            List[bool]: (np.dtype("object"), ArrayType(BooleanType())),
+            List[np.bool]: (np.dtype("object"), ArrayType(BooleanType())),
+            List[datetime.date]: (np.dtype("object"), ArrayType(DateType())),
+            List[np.int8]: (np.dtype("object"), ArrayType(ByteType())),
+            List[np.byte]: (np.dtype("object"), ArrayType(ByteType())),
+            List[decimal.Decimal]: (np.dtype("object"), ArrayType(DecimalType(38, 18))),
+            List[float]: (np.dtype("object"), ArrayType(DoubleType())),
+            List[np.float]: (np.dtype("object"), ArrayType(DoubleType())),
+            List[np.float64]: (np.dtype("object"), ArrayType(DoubleType())),
+            List[np.float32]: (np.dtype("object"), ArrayType(FloatType())),
+            List[np.int32]: (np.dtype("object"), ArrayType(IntegerType())),
+            List[int]: (np.dtype("object"), ArrayType(LongType())),
+            List[np.int]: (np.dtype("object"), ArrayType(LongType())),
+            List[np.int64]: (np.dtype("object"), ArrayType(LongType())),
+            List[np.int16]: (np.dtype("object"), ArrayType(ShortType())),
+            List[str]: (np.dtype("object"), ArrayType(StringType())),
+            List[np.unicode_]: (np.dtype("object"), ArrayType(StringType())),
+            List[datetime.datetime]: (np.dtype("object"), ArrayType(TimestampType())),
+            List[np.datetime64]: (np.dtype("object"), ArrayType(TimestampType())),
+            # CategoricalDtype
+            CategoricalDtype(categories=["a", "b", "c"]): (
+                CategoricalDtype(categories=["a", "b", "c"]),
+                LongType(),
+            ),
+        }
+
+        for numpy_or_python_type, (dtype, spark_type) in type_mapper.items():
+            self.assertEqual(as_spark_type(numpy_or_python_type), spark_type)
+            self.assertEqual(pandas_on_spark_type(numpy_or_python_type), (dtype, spark_type))
+
+        with self.assertRaisesRegex(TypeError, "Type uint64 was not understood."):
+            as_spark_type(np.dtype("uint64"))
+
+        with self.assertRaisesRegex(TypeError, "Type object was not understood."):
+            as_spark_type(np.dtype("object"))
+
+        with self.assertRaisesRegex(TypeError, "Type uint64 was not understood."):
+            pandas_on_spark_type(np.dtype("uint64"))
+
+        with self.assertRaisesRegex(TypeError, "Type object was not understood."):
+            pandas_on_spark_type(np.dtype("object"))
+
+    @unittest.skipIf(not extension_dtypes_available, "The pandas extension types are not available")
+    def test_as_spark_type_extension_dtypes(self):
+        from pandas import Int8Dtype, Int16Dtype, Int32Dtype, Int64Dtype
+
+        type_mapper = {
+            Int8Dtype(): ByteType(),
+            Int16Dtype(): ShortType(),
+            Int32Dtype(): IntegerType(),
+            Int64Dtype(): LongType(),
+        }
+
+        for extension_dtype, spark_type in type_mapper.items():
+            self.assertEqual(as_spark_type(extension_dtype), spark_type)
+            self.assertEqual(pandas_on_spark_type(extension_dtype), (extension_dtype, spark_type))
+
+    @unittest.skipIf(
+        not extension_object_dtypes_available, "The pandas extension object types are not available"
+    )
+    def test_as_spark_type_extension_object_dtypes(self):
+        from pandas import BooleanDtype, StringDtype
+
+        type_mapper = {
+            BooleanDtype(): BooleanType(),
+            StringDtype(): StringType(),
+        }
+
+        for extension_dtype, spark_type in type_mapper.items():
+            self.assertEqual(as_spark_type(extension_dtype), spark_type)
+            self.assertEqual(pandas_on_spark_type(extension_dtype), (extension_dtype, spark_type))
+
+    @unittest.skipIf(
+        not extension_float_dtypes_available, "The pandas extension float types are not available"
+    )
+    def test_as_spark_type_extension_float_dtypes(self):
+        from pandas import Float32Dtype, Float64Dtype
+
+        type_mapper = {
+            Float32Dtype(): FloatType(),
+            Float64Dtype(): DoubleType(),
+        }
+
+        for extension_dtype, spark_type in type_mapper.items():
+            self.assertEqual(as_spark_type(extension_dtype), spark_type)
+            self.assertEqual(pandas_on_spark_type(extension_dtype), (extension_dtype, spark_type))
+
+
+if __name__ == "__main__":
+    from pyspark.pandas.tests.test_typedef import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_utils.py b/python/pyspark/pandas/tests/test_utils.py
new file mode 100644
index 0000000000000..a3595486e0510
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_utils.py
@@ -0,0 +1,105 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import pandas as pd
+
+from pyspark.pandas.utils import (
+    lazy_property,
+    validate_arguments_and_invoke_function,
+    validate_bool_kwarg,
+)
+from pyspark.testing.pandasutils import PandasOnSparkTestCase
+from pyspark.testing.sqlutils import SQLTestUtils
+
+some_global_variable = 0
+
+
+class UtilsTest(PandasOnSparkTestCase, SQLTestUtils):
+
+    # a dummy to_html version with an extra parameter that pandas does not support
+    # used in test_validate_arguments_and_invoke_function
+    def to_html(self, max_rows=None, unsupported_param=None):
+        args = locals()
+
+        pdf = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}, index=[0, 1, 3])
+        validate_arguments_and_invoke_function(pdf, self.to_html, pd.DataFrame.to_html, args)
+
+    def to_clipboard(self, sep=",", **kwargs):
+        args = locals()
+
+        pdf = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}, index=[0, 1, 3])
+        validate_arguments_and_invoke_function(
+            pdf, self.to_clipboard, pd.DataFrame.to_clipboard, args
+        )
+
+        # Support for **kwargs
+        self.to_clipboard(sep=",", index=False)
+
+    def test_validate_arguments_and_invoke_function(self):
+        # This should pass and run fine
+        self.to_html()
+        self.to_html(unsupported_param=None)
+        self.to_html(max_rows=5)
+
+        # This should fail because we are explicitly setting an unsupported param
+        # to a non-default value
+        with self.assertRaises(TypeError):
+            self.to_html(unsupported_param=1)
+
+    def test_lazy_property(self):
+        obj = TestClassForLazyProp()
+        # If lazy prop is not working, the second test would fail (because it'd be 2)
+        self.assert_eq(obj.lazy_prop, 1)
+        self.assert_eq(obj.lazy_prop, 1)
+
+    def test_validate_bool_kwarg(self):
+        # This should pass and run fine
+        koalas = True
+        self.assert_eq(validate_bool_kwarg(koalas, "koalas"), True)
+        koalas = False
+        self.assert_eq(validate_bool_kwarg(koalas, "koalas"), False)
+        koalas = None
+        self.assert_eq(validate_bool_kwarg(koalas, "koalas"), None)
+
+        # This should fail because we are explicitly setting a non-boolean value
+        koalas = "true"
+        with self.assertRaisesRegex(
+            TypeError, 'For argument "koalas" expected type bool, received type str.'
+        ):
+            validate_bool_kwarg(koalas, "koalas")
+
+
+class TestClassForLazyProp:
+    def __init__(self):
+        self.some_variable = 0
+
+    @lazy_property
+    def lazy_prop(self):
+        self.some_variable += 1
+        return self.some_variable
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.pandas.tests.test_utils import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/tests/test_window.py b/python/pyspark/pandas/tests/test_window.py
new file mode 100644
index 0000000000000..8c347b8687626
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_window.py
@@ -0,0 +1,316 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import inspect
+
+from pyspark import pandas as ps
+from pyspark.pandas.exceptions import PandasNotImplementedError
+from pyspark.pandas.missing.window import (
+    MissingPandasLikeExpanding,
+    MissingPandasLikeRolling,
+    MissingPandasLikeExpandingGroupby,
+    MissingPandasLikeRollingGroupby,
+)
+from pyspark.testing.pandasutils import PandasOnSparkTestCase, TestUtils
+
+
+class ExpandingRollingTest(PandasOnSparkTestCase, TestUtils):
+    def test_missing(self):
+        kdf = ps.DataFrame({"a": [1, 2, 3, 4, 5, 6, 7, 8, 9]})
+
+        # Expanding functions
+        missing_functions = inspect.getmembers(MissingPandasLikeExpanding, inspect.isfunction)
+        unsupported_functions = [
+            name for (name, type_) in missing_functions if type_.__name__ == "unsupported_function"
+        ]
+        for name in unsupported_functions:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "method.*Expanding.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.expanding(1), name)()  # Frame
+
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "method.*Expanding.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.a.expanding(1), name)()  # Series
+
+        deprecated_functions = [
+            name for (name, type_) in missing_functions if type_.__name__ == "deprecated_function"
+        ]
+        for name in deprecated_functions:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "method.*Expanding.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.expanding(1), name)()  # Frame
+
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "method.*Expanding.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.a.expanding(1), name)()  # Series
+
+        # Rolling functions
+        missing_functions = inspect.getmembers(MissingPandasLikeRolling, inspect.isfunction)
+        unsupported_functions = [
+            name for (name, type_) in missing_functions if type_.__name__ == "unsupported_function"
+        ]
+        for name in unsupported_functions:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "method.*Rolling.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.rolling(1), name)()  # Frame
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "method.*Rolling.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.a.rolling(1), name)()  # Series
+
+        deprecated_functions = [
+            name for (name, type_) in missing_functions if type_.__name__ == "deprecated_function"
+        ]
+        for name in deprecated_functions:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "method.*Rolling.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.rolling(1), name)()  # Frame
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "method.*Rolling.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.a.rolling(1), name)()  # Series
+
+        # Expanding properties
+        missing_properties = inspect.getmembers(
+            MissingPandasLikeExpanding, lambda o: isinstance(o, property)
+        )
+        unsupported_properties = [
+            name
+            for (name, type_) in missing_properties
+            if type_.fget.__name__ == "unsupported_property"
+        ]
+        for name in unsupported_properties:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "property.*Expanding.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.expanding(1), name)  # Frame
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "property.*Expanding.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.a.expanding(1), name)  # Series
+
+        deprecated_properties = [
+            name
+            for (name, type_) in missing_properties
+            if type_.fget.__name__ == "deprecated_property"
+        ]
+        for name in deprecated_properties:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "property.*Expanding.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.expanding(1), name)  # Frame
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "property.*Expanding.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.a.expanding(1), name)  # Series
+
+        # Rolling properties
+        missing_properties = inspect.getmembers(
+            MissingPandasLikeRolling, lambda o: isinstance(o, property)
+        )
+        unsupported_properties = [
+            name
+            for (name, type_) in missing_properties
+            if type_.fget.__name__ == "unsupported_property"
+        ]
+        for name in unsupported_properties:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "property.*Rolling.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.rolling(1), name)()  # Frame
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "property.*Rolling.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.a.rolling(1), name)()  # Series
+        deprecated_properties = [
+            name
+            for (name, type_) in missing_properties
+            if type_.fget.__name__ == "deprecated_property"
+        ]
+        for name in deprecated_properties:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "property.*Rolling.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.rolling(1), name)()  # Frame
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "property.*Rolling.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.a.rolling(1), name)()  # Series
+
+    def test_missing_groupby(self):
+        kdf = ps.DataFrame({"a": [1, 2, 3, 4, 5, 6, 7, 8, 9]})
+
+        # Expanding functions
+        missing_functions = inspect.getmembers(
+            MissingPandasLikeExpandingGroupby, inspect.isfunction
+        )
+        unsupported_functions = [
+            name for (name, type_) in missing_functions if type_.__name__ == "unsupported_function"
+        ]
+        for name in unsupported_functions:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "method.*Expanding.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.groupby("a").expanding(1), name)()  # Frame
+
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "method.*Expanding.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.a.groupby(kdf.a).expanding(1), name)()  # Series
+
+        deprecated_functions = [
+            name for (name, type_) in missing_functions if type_.__name__ == "deprecated_function"
+        ]
+        for name in deprecated_functions:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "method.*Expanding.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.groupby("a").expanding(1), name)()  # Frame
+
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "method.*Expanding.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.a.groupby(kdf.a).expanding(1), name)()  # Series
+
+        # Rolling functions
+        missing_functions = inspect.getmembers(MissingPandasLikeRollingGroupby, inspect.isfunction)
+        unsupported_functions = [
+            name for (name, type_) in missing_functions if type_.__name__ == "unsupported_function"
+        ]
+        for name in unsupported_functions:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "method.*Rolling.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.groupby("a").rolling(1), name)()  # Frame
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "method.*Rolling.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.a.groupby(kdf.a).rolling(1), name)()  # Series
+
+        deprecated_functions = [
+            name for (name, type_) in missing_functions if type_.__name__ == "deprecated_function"
+        ]
+        for name in deprecated_functions:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "method.*Rolling.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.rolling(1), name)()  # Frame
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "method.*Rolling.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.a.rolling(1), name)()  # Series
+
+        # Expanding properties
+        missing_properties = inspect.getmembers(
+            MissingPandasLikeExpandingGroupby, lambda o: isinstance(o, property)
+        )
+        unsupported_properties = [
+            name
+            for (name, type_) in missing_properties
+            if type_.fget.__name__ == "unsupported_property"
+        ]
+        for name in unsupported_properties:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "property.*Expanding.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.groupby("a").expanding(1), name)()  # Frame
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "property.*Expanding.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.a.groupby(kdf.a).expanding(1), name)()  # Series
+
+        deprecated_properties = [
+            name
+            for (name, type_) in missing_properties
+            if type_.fget.__name__ == "deprecated_property"
+        ]
+        for name in deprecated_properties:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "property.*Expanding.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.expanding(1), name)  # Frame
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "property.*Expanding.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.a.expanding(1), name)  # Series
+
+        # Rolling properties
+        missing_properties = inspect.getmembers(
+            MissingPandasLikeRollingGroupby, lambda o: isinstance(o, property)
+        )
+        unsupported_properties = [
+            name
+            for (name, type_) in missing_properties
+            if type_.fget.__name__ == "unsupported_property"
+        ]
+        for name in unsupported_properties:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "property.*Rolling.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.groupby("a").rolling(1), name)()  # Frame
+            with self.assertRaisesRegex(
+                PandasNotImplementedError,
+                "property.*Rolling.*{}.*not implemented( yet\\.|\\. .+)".format(name),
+            ):
+                getattr(kdf.a.groupby(kdf.a).rolling(1), name)()  # Series
+        deprecated_properties = [
+            name
+            for (name, type_) in missing_properties
+            if type_.fget.__name__ == "deprecated_property"
+        ]
+        for name in deprecated_properties:
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "property.*Rolling.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.rolling(1), name)()  # Frame
+            with self.assertRaisesRegex(
+                PandasNotImplementedError, "property.*Rolling.*{}.*is deprecated".format(name)
+            ):
+                getattr(kdf.a.rolling(1), name)()  # Series
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.pandas.tests.test_window import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/typedef/__init__.py b/python/pyspark/pandas/typedef/__init__.py
new file mode 100644
index 0000000000000..5f7ea2834a52a
--- /dev/null
+++ b/python/pyspark/pandas/typedef/__init__.py
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from pyspark.pandas.typedef.typehints import *  # noqa: F401,F405
diff --git a/python/pyspark/pandas/typedef/string_typehints.py b/python/pyspark/pandas/typedef/string_typehints.py
new file mode 100644
index 0000000000000..4ad0826fcaebe
--- /dev/null
+++ b/python/pyspark/pandas/typedef/string_typehints.py
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import numpy as np  # noqa: F401
+import pandas  # noqa: F401
+import pandas as pd  # noqa: F401
+from numpy import *  # noqa: F401
+from pandas import *  # noqa: F401
+from inspect import getfullargspec  # noqa: F401
+
+
+def resolve_string_type_hint(tpe):
+    import pyspark.pandas as ps
+    from pyspark.pandas import DataFrame, Series
+
+    locs = {
+        "ps": ps,
+        "pyspark.pandas": ps,
+        "DataFrame": DataFrame,
+        "Series": Series,
+    }
+    # This is a hack to resolve the forward reference string.
+    exec("def func() -> %s: pass\narg_spec = getfullargspec(func)" % tpe, globals(), locs)
+    return locs["arg_spec"].annotations.get("return", None)
diff --git a/python/pyspark/pandas/typedef/typehints.py b/python/pyspark/pandas/typedef/typehints.py
new file mode 100644
index 0000000000000..a222ab10fc77e
--- /dev/null
+++ b/python/pyspark/pandas/typedef/typehints.py
@@ -0,0 +1,540 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Utilities to deal with types. This is mostly focused on python3.
+"""
+import datetime
+import decimal
+from inspect import getfullargspec, isclass
+from typing import Generic, List, Optional, Tuple, TypeVar, Union  # noqa: F401
+
+import numpy as np
+import pandas as pd
+from pandas.api.types import CategoricalDtype, pandas_dtype
+from pandas.api.extensions import ExtensionDtype
+
+try:
+    from pandas import Int8Dtype, Int16Dtype, Int32Dtype, Int64Dtype
+
+    extension_dtypes_available = True
+    extension_dtypes = (Int8Dtype, Int16Dtype, Int32Dtype, Int64Dtype)  # type: Tuple
+
+    try:
+        from pandas import BooleanDtype, StringDtype
+
+        extension_object_dtypes_available = True
+        extension_dtypes += (BooleanDtype, StringDtype)
+    except ImportError:
+        extension_object_dtypes_available = False
+
+    try:
+        from pandas import Float32Dtype, Float64Dtype
+
+        extension_float_dtypes_available = True
+        extension_dtypes += (Float32Dtype, Float64Dtype)
+    except ImportError:
+        extension_float_dtypes_available = False
+
+except ImportError:
+    extension_dtypes_available = False
+    extension_object_dtypes_available = False
+    extension_float_dtypes_available = False
+    extension_dtypes = ()
+
+import pyarrow as pa
+import pyspark.sql.types as types
+
+try:
+    from pyspark.sql.types import to_arrow_type, from_arrow_type
+except ImportError:
+    from pyspark.sql.pandas.types import to_arrow_type, from_arrow_type
+
+from pyspark import pandas as ps  # For running doctests and reference resolution in PyCharm.
+from pyspark.pandas.typedef.string_typehints import resolve_string_type_hint
+
+T = TypeVar("T")
+
+Scalar = Union[
+    int, float, bool, str, bytes, decimal.Decimal, datetime.date, datetime.datetime, None
+]
+
+Dtype = Union[np.dtype, ExtensionDtype]
+
+
+# A column of data, with the data type.
+class SeriesType(Generic[T]):
+    def __init__(self, dtype: Dtype, spark_type: types.DataType):
+        self.dtype = dtype
+        self.spark_type = spark_type
+
+    def __repr__(self):
+        return "SeriesType[{}]".format(self.spark_type)
+
+
+class DataFrameType(object):
+    def __init__(
+        self, dtypes: List[Dtype], spark_types: List[types.DataType], names: List[Optional[str]]
+    ):
+        from pyspark.pandas.utils import name_like_string
+
+        self.dtypes = dtypes
+        self.spark_type = types.StructType(
+            [
+                types.StructField(name_like_string(n) if n is not None else ("c%s" % i), t)
+                for i, (n, t) in enumerate(zip(names, spark_types))
+            ]
+        )  # type: types.StructType
+
+    def __repr__(self):
+        return "DataFrameType[{}]".format(self.spark_type)
+
+
+# The type is a scalar type that is furthermore understood by Spark.
+class ScalarType(object):
+    def __init__(self, dtype: Dtype, spark_type: types.DataType):
+        self.dtype = dtype
+        self.spark_type = spark_type
+
+    def __repr__(self):
+        return "ScalarType[{}]".format(self.spark_type)
+
+
+# The type is left unspecified or we do not know about this type.
+class UnknownType(object):
+    def __init__(self, tpe):
+        self.tpe = tpe
+
+    def __repr__(self):
+        return "UnknownType[{}]".format(self.tpe)
+
+
+class NameTypeHolder(object):
+    name = None
+    tpe = None
+
+
+def as_spark_type(tpe: Union[str, type, Dtype], *, raise_error: bool = True) -> types.DataType:
+    """
+    Given a Python type, returns the equivalent spark type.
+    Accepts:
+    - the built-in types in Python
+    - the built-in types in numpy
+    - list of pairs of (field_name, type)
+    - dictionaries of field_name -> type
+    - Python3's typing system
+    """
+    if isinstance(tpe, np.dtype) and tpe == np.dtype("object"):
+        pass
+    # ArrayType
+    elif tpe in (np.ndarray,):
+        return types.ArrayType(types.StringType())
+    elif hasattr(tpe, "__origin__") and issubclass(tpe.__origin__, list):  # type: ignore
+        element_type = as_spark_type(tpe.__args__[0], raise_error=raise_error)  # type: ignore
+        if element_type is None:
+            return None
+        return types.ArrayType(element_type)
+    # BinaryType
+    elif tpe in (bytes, np.character, np.bytes_, np.string_):
+        return types.BinaryType()
+    # BooleanType
+    elif tpe in (bool, np.bool, "bool", "?"):
+        return types.BooleanType()
+    # DateType
+    elif tpe in (datetime.date,):
+        return types.DateType()
+    # NumericType
+    elif tpe in (np.int8, np.byte, "int8", "byte", "b"):
+        return types.ByteType()
+    elif tpe in (decimal.Decimal,):
+        # TODO: considering about the precision & scale for decimal type.
+        return types.DecimalType(38, 18)
+    elif tpe in (float, np.float, np.float64, "float", "float64", "double"):
+        return types.DoubleType()
+    elif tpe in (np.float32, "float32", "f"):
+        return types.FloatType()
+    elif tpe in (np.int32, "int32", "i"):
+        return types.IntegerType()
+    elif tpe in (int, np.int, np.int64, "int", "int64", "long"):
+        return types.LongType()
+    elif tpe in (np.int16, "int16", "short"):
+        return types.ShortType()
+    # StringType
+    elif tpe in (str, np.unicode_, "str", "U"):
+        return types.StringType()
+    # TimestampType
+    elif tpe in (datetime.datetime, np.datetime64, "datetime64[ns]", "M"):
+        return types.TimestampType()
+
+    # categorical types
+    elif isinstance(tpe, CategoricalDtype) or (isinstance(tpe, str) and type == "category"):
+        return types.LongType()
+
+    # extension types
+    elif extension_dtypes_available:
+        # IntegralType
+        if isinstance(tpe, Int8Dtype) or (isinstance(tpe, str) and tpe == "Int8"):
+            return types.ByteType()
+        elif isinstance(tpe, Int16Dtype) or (isinstance(tpe, str) and tpe == "Int16"):
+            return types.ShortType()
+        elif isinstance(tpe, Int32Dtype) or (isinstance(tpe, str) and tpe == "Int32"):
+            return types.IntegerType()
+        elif isinstance(tpe, Int64Dtype) or (isinstance(tpe, str) and tpe == "Int64"):
+            return types.LongType()
+
+        if extension_object_dtypes_available:
+            # BooleanType
+            if isinstance(tpe, BooleanDtype) or (isinstance(tpe, str) and tpe == "boolean"):
+                return types.BooleanType()
+            # StringType
+            elif isinstance(tpe, StringDtype) or (isinstance(tpe, str) and tpe == "string"):
+                return types.StringType()
+
+        if extension_float_dtypes_available:
+            # FractionalType
+            if isinstance(tpe, Float32Dtype) or (isinstance(tpe, str) and tpe == "Float32"):
+                return types.FloatType()
+            elif isinstance(tpe, Float64Dtype) or (isinstance(tpe, str) and tpe == "Float64"):
+                return types.DoubleType()
+
+    if raise_error:
+        raise TypeError("Type %s was not understood." % tpe)
+    else:
+        return None
+
+
+def spark_type_to_pandas_dtype(
+    spark_type: types.DataType, *, use_extension_dtypes: bool = False
+) -> Dtype:
+    """ Return the given Spark DataType to pandas dtype. """
+
+    if use_extension_dtypes and extension_dtypes_available:
+        # IntegralType
+        if isinstance(spark_type, types.ByteType):
+            return Int8Dtype()
+        elif isinstance(spark_type, types.ShortType):
+            return Int16Dtype()
+        elif isinstance(spark_type, types.IntegerType):
+            return Int32Dtype()
+        elif isinstance(spark_type, types.LongType):
+            return Int64Dtype()
+
+        if extension_object_dtypes_available:
+            # BooleanType
+            if isinstance(spark_type, types.BooleanType):
+                return BooleanDtype()
+            # StringType
+            elif isinstance(spark_type, types.StringType):
+                return StringDtype()
+
+        # FractionalType
+        if extension_float_dtypes_available:
+            if isinstance(spark_type, types.FloatType):
+                return Float32Dtype()
+            elif isinstance(spark_type, types.DoubleType):
+                return Float64Dtype()
+
+    if isinstance(
+        spark_type,
+        (
+            types.DateType,
+            types.NullType,
+            types.ArrayType,
+            types.MapType,
+            types.StructType,
+            types.UserDefinedType,
+        ),
+    ):
+        return np.dtype("object")
+    elif isinstance(spark_type, types.TimestampType):
+        return np.dtype("datetime64[ns]")
+    else:
+        return np.dtype(to_arrow_type(spark_type).to_pandas_dtype())
+
+
+def pandas_on_spark_type(tpe) -> Tuple[Dtype, types.DataType]:
+    """
+    Convert input into a pandas only dtype object or a numpy dtype object,
+    and its corresponding Spark DataType.
+
+    Parameters
+    ----------
+    tpe : object to be converted
+
+    Returns
+    -------
+    tuple of np.dtype or a pandas dtype, and Spark DataType
+
+    Raises
+    ------
+    TypeError if not a dtype
+
+    Examples
+    --------
+    >>> pandas_on_spark_type(int)
+    (dtype('int64'), LongType)
+    >>> pandas_on_spark_type(str)
+    (dtype('<U'), StringType)
+    >>> pandas_on_spark_type(datetime.date)
+    (dtype('O'), DateType)
+    >>> pandas_on_spark_type(datetime.datetime)
+    (dtype('<M8[ns]'), TimestampType)
+    >>> pandas_on_spark_type(List[bool])
+    (dtype('O'), ArrayType(BooleanType,true))
+    """
+    try:
+        dtype = pandas_dtype(tpe)
+        spark_type = as_spark_type(dtype)
+    except TypeError:
+        spark_type = as_spark_type(tpe)
+        dtype = spark_type_to_pandas_dtype(spark_type)
+    return dtype, spark_type
+
+
+def infer_pd_series_spark_type(pser: pd.Series, dtype: Dtype) -> types.DataType:
+    """Infer Spark DataType from pandas Series dtype.
+
+    :param pser: :class:`pandas.Series` to be inferred
+    :param dtype: the Series' dtype
+    :return: the inferred Spark data type
+    """
+    if dtype == np.dtype("object"):
+        if len(pser) == 0 or pser.isnull().all():
+            return types.NullType()
+        elif hasattr(pser.iloc[0], "__UDT__"):
+            return pser.iloc[0].__UDT__
+        else:
+            return from_arrow_type(pa.Array.from_pandas(pser).type)
+    elif isinstance(dtype, CategoricalDtype):
+        # `pser` must already be converted to codes.
+        return as_spark_type(pser.dtype)
+    else:
+        return as_spark_type(dtype)
+
+
+def infer_return_type(f) -> Union[SeriesType, DataFrameType, ScalarType, UnknownType]:
+    """
+    Infer the return type from the return type annotation of the given function.
+
+    The returned type class indicates both dtypes (a pandas only dtype object
+    or a numpy dtype object) and its corresponding Spark DataType.
+
+    >>> def func() -> int:
+    ...    pass
+    >>> inferred = infer_return_type(func)
+    >>> inferred.dtype
+    dtype('int64')
+    >>> inferred.spark_type
+    LongType
+
+    >>> def func() -> ps.Series[int]:
+    ...    pass
+    >>> inferred = infer_return_type(func)
+    >>> inferred.dtype
+    dtype('int64')
+    >>> inferred.spark_type
+    LongType
+
+    >>> def func() -> ps.DataFrame[np.float, str]:
+    ...    pass
+    >>> inferred = infer_return_type(func)
+    >>> inferred.dtypes
+    [dtype('float64'), dtype('<U')]
+    >>> inferred.spark_type
+    StructType(List(StructField(c0,DoubleType,true),StructField(c1,StringType,true)))
+
+    >>> def func() -> ps.DataFrame[np.float]:
+    ...    pass
+    >>> inferred = infer_return_type(func)
+    >>> inferred.dtypes
+    [dtype('float64')]
+    >>> inferred.spark_type
+    StructType(List(StructField(c0,DoubleType,true)))
+
+    >>> def func() -> 'int':
+    ...    pass
+    >>> inferred = infer_return_type(func)
+    >>> inferred.dtype
+    dtype('int64')
+    >>> inferred.spark_type
+    LongType
+
+    >>> def func() -> 'ps.Series[int]':
+    ...    pass
+    >>> inferred = infer_return_type(func)
+    >>> inferred.dtype
+    dtype('int64')
+    >>> inferred.spark_type
+    LongType
+
+    >>> def func() -> 'ps.DataFrame[np.float, str]':
+    ...    pass
+    >>> inferred = infer_return_type(func)
+    >>> inferred.dtypes
+    [dtype('float64'), dtype('<U')]
+    >>> inferred.spark_type
+    StructType(List(StructField(c0,DoubleType,true),StructField(c1,StringType,true)))
+
+    >>> def func() -> 'ps.DataFrame[np.float]':
+    ...    pass
+    >>> inferred = infer_return_type(func)
+    >>> inferred.dtypes
+    [dtype('float64')]
+    >>> inferred.spark_type
+    StructType(List(StructField(c0,DoubleType,true)))
+
+    >>> def func() -> ps.DataFrame['a': np.float, 'b': int]:
+    ...     pass
+    >>> inferred = infer_return_type(func)
+    >>> inferred.dtypes
+    [dtype('float64'), dtype('int64')]
+    >>> inferred.spark_type
+    StructType(List(StructField(a,DoubleType,true),StructField(b,LongType,true)))
+
+    >>> def func() -> "ps.DataFrame['a': np.float, 'b': int]":
+    ...     pass
+    >>> inferred = infer_return_type(func)
+    >>> inferred.dtypes
+    [dtype('float64'), dtype('int64')]
+    >>> inferred.spark_type
+    StructType(List(StructField(a,DoubleType,true),StructField(b,LongType,true)))
+
+    >>> pdf = pd.DataFrame({"a": [1, 2, 3], "b": [3, 4, 5]})
+    >>> def func() -> ps.DataFrame[pdf.dtypes]:
+    ...     pass
+    >>> inferred = infer_return_type(func)
+    >>> inferred.dtypes
+    [dtype('int64'), dtype('int64')]
+    >>> inferred.spark_type
+    StructType(List(StructField(c0,LongType,true),StructField(c1,LongType,true)))
+
+    >>> pdf = pd.DataFrame({"a": [1, 2, 3], "b": [3, 4, 5]})
+    >>> def func() -> ps.DataFrame[zip(pdf.columns, pdf.dtypes)]:
+    ...     pass
+    >>> inferred = infer_return_type(func)
+    >>> inferred.dtypes
+    [dtype('int64'), dtype('int64')]
+    >>> inferred.spark_type
+    StructType(List(StructField(a,LongType,true),StructField(b,LongType,true)))
+
+    >>> pdf = pd.DataFrame({("x", "a"): [1, 2, 3], ("y", "b"): [3, 4, 5]})
+    >>> def func() -> ps.DataFrame[zip(pdf.columns, pdf.dtypes)]:
+    ...     pass
+    >>> inferred = infer_return_type(func)
+    >>> inferred.dtypes
+    [dtype('int64'), dtype('int64')]
+    >>> inferred.spark_type
+    StructType(List(StructField((x, a),LongType,true),StructField((y, b),LongType,true)))
+
+    >>> pdf = pd.DataFrame({"a": [1, 2, 3], "b": pd.Categorical([3, 4, 5])})
+    >>> def func() -> ps.DataFrame[pdf.dtypes]:
+    ...     pass
+    >>> inferred = infer_return_type(func)
+    >>> inferred.dtypes
+    [dtype('int64'), CategoricalDtype(categories=[3, 4, 5], ordered=False)]
+    >>> inferred.spark_type
+    StructType(List(StructField(c0,LongType,true),StructField(c1,LongType,true)))
+
+    >>> def func() -> ps.DataFrame[zip(pdf.columns, pdf.dtypes)]:
+    ...     pass
+    >>> inferred = infer_return_type(func)
+    >>> inferred.dtypes
+    [dtype('int64'), CategoricalDtype(categories=[3, 4, 5], ordered=False)]
+    >>> inferred.spark_type
+    StructType(List(StructField(a,LongType,true),StructField(b,LongType,true)))
+
+    >>> def func() -> ps.Series[pdf.b.dtype]:
+    ...     pass
+    >>> inferred = infer_return_type(func)
+    >>> inferred.dtype
+    CategoricalDtype(categories=[3, 4, 5], ordered=False)
+    >>> inferred.spark_type
+    LongType
+    """
+    # We should re-import to make sure the class 'SeriesType' is not treated as a class
+    # within this module locally. See Series.__class_getitem__ which imports this class
+    # canonically.
+    from pyspark.pandas.typedef import SeriesType, NameTypeHolder
+
+    spec = getfullargspec(f)
+    tpe = spec.annotations.get("return", None)
+    if isinstance(tpe, str):
+        # This type hint can happen when given hints are string to avoid forward reference.
+        tpe = resolve_string_type_hint(tpe)
+
+    if hasattr(tpe, "__origin__") and (
+        tpe.__origin__ == ps.DataFrame or tpe.__origin__ == ps.Series
+    ):
+        # When Python version is lower then 3.7. Unwrap it to a Tuple/SeriesType type hints.
+        tpe = tpe.__args__[0]
+
+    if hasattr(tpe, "__origin__") and issubclass(tpe.__origin__, SeriesType):
+        tpe = tpe.__args__[0]
+        if issubclass(tpe, NameTypeHolder):
+            tpe = tpe.tpe
+        dtype, spark_type = pandas_on_spark_type(tpe)
+        return SeriesType(dtype, spark_type)
+
+    # Note that, DataFrame type hints will create a Tuple.
+    # Python 3.6 has `__name__`. Python 3.7 and 3.8 have `_name`.
+    # Check if the name is Tuple.
+    name = getattr(tpe, "_name", getattr(tpe, "__name__", None))
+    if name == "Tuple":
+        tuple_type = tpe
+        if hasattr(tuple_type, "__tuple_params__"):
+            # Python 3.5.0 to 3.5.2 has '__tuple_params__' instead.
+            # See https://github.com/python/cpython/blob/v3.5.2/Lib/typing.py
+            parameters = getattr(tuple_type, "__tuple_params__")
+        else:
+            parameters = getattr(tuple_type, "__args__")
+        dtypes, spark_types = zip(
+            *(
+                pandas_on_spark_type(p.tpe)
+                if isclass(p) and issubclass(p, NameTypeHolder)
+                else pandas_on_spark_type(p)
+                for p in parameters
+            )
+        )
+        names = [
+            p.name if isclass(p) and issubclass(p, NameTypeHolder) else None for p in parameters
+        ]
+        return DataFrameType(list(dtypes), list(spark_types), names)
+
+    types = pandas_on_spark_type(tpe)
+    if types is None:
+        return UnknownType(tpe)
+    else:
+        return ScalarType(*types)
+
+
+def _test():
+    import doctest
+    import sys
+    import pyspark.pandas.typedef.typehints
+
+    globs = pyspark.pandas.typedef.typehints.__dict__.copy()
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.typedef.typehints,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/usage_logging/__init__.py b/python/pyspark/pandas/usage_logging/__init__.py
new file mode 100644
index 0000000000000..3e0fa7b7b667f
--- /dev/null
+++ b/python/pyspark/pandas/usage_logging/__init__.py
@@ -0,0 +1,269 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import functools
+import importlib
+import inspect
+import threading
+import time
+from types import ModuleType
+from typing import Union
+
+import pandas as pd
+
+from pyspark.pandas import config, namespace, sql_processor
+from pyspark.pandas.accessors import PandasOnSparkFrameMethods
+from pyspark.pandas.frame import DataFrame
+from pyspark.pandas.datetimes import DatetimeMethods
+from pyspark.pandas.groupby import DataFrameGroupBy, SeriesGroupBy
+from pyspark.pandas.indexes.base import Index
+from pyspark.pandas.indexes.category import CategoricalIndex
+from pyspark.pandas.indexes.datetimes import DatetimeIndex
+from pyspark.pandas.indexes.multi import MultiIndex
+from pyspark.pandas.indexes.numeric import Float64Index, Int64Index
+from pyspark.pandas.missing.frame import _MissingPandasLikeDataFrame
+from pyspark.pandas.missing.groupby import (
+    MissingPandasLikeDataFrameGroupBy,
+    MissingPandasLikeSeriesGroupBy,
+)
+from pyspark.pandas.missing.indexes import (
+    MissingPandasLikeCategoricalIndex,
+    MissingPandasLikeDatetimeIndex,
+    MissingPandasLikeIndex,
+    MissingPandasLikeMultiIndex,
+)
+from pyspark.pandas.missing.series import MissingPandasLikeSeries
+from pyspark.pandas.missing.window import (
+    MissingPandasLikeExpanding,
+    MissingPandasLikeRolling,
+    MissingPandasLikeExpandingGroupby,
+    MissingPandasLikeRollingGroupby,
+)
+from pyspark.pandas.series import Series
+from pyspark.pandas.spark.accessors import (
+    CachedSparkFrameMethods,
+    SparkFrameMethods,
+    SparkIndexOpsMethods,
+)
+from pyspark.pandas.strings import StringMethods
+from pyspark.pandas.window import Expanding, ExpandingGroupby, Rolling, RollingGroupby
+
+
+def attach(logger_module: Union[str, ModuleType]) -> None:
+    """
+    Attach the usage logger.
+
+    Parameters
+    ----------
+    logger_module : the module or module name contains the usage logger.
+        The module needs to provide `get_logger` function as an entry point of the plug-in
+        returning the usage logger.
+
+    See Also
+    --------
+    usage_logger : the reference implementation of the usage logger.
+    """
+    if isinstance(logger_module, str):
+        logger_module = importlib.import_module(logger_module)
+
+    logger = getattr(logger_module, "get_logger")()
+
+    modules = [config, namespace]
+    classes = [
+        DataFrame,
+        Series,
+        Index,
+        MultiIndex,
+        Int64Index,
+        Float64Index,
+        CategoricalIndex,
+        DatetimeIndex,
+        DataFrameGroupBy,
+        SeriesGroupBy,
+        DatetimeMethods,
+        StringMethods,
+        Expanding,
+        ExpandingGroupby,
+        Rolling,
+        RollingGroupby,
+        CachedSparkFrameMethods,
+        SparkFrameMethods,
+        SparkIndexOpsMethods,
+        PandasOnSparkFrameMethods,
+    ]
+
+    try:
+        from pyspark.pandas import mlflow
+
+        modules.append(mlflow)
+        classes.append(mlflow.PythonModelWrapper)
+    except ImportError:
+        pass
+
+    sql_processor._CAPTURE_SCOPES = 3  # type: ignore
+    modules.append(sql_processor)  # type: ignore
+
+    # Modules
+    for target_module in modules:
+        target_name = target_module.__name__.split(".")[-1]
+        for name in getattr(target_module, "__all__"):
+            func = getattr(target_module, name)
+            if not inspect.isfunction(func):
+                continue
+            setattr(target_module, name, _wrap_function(target_name, name, func, logger))
+
+    special_functions = set(
+        [
+            "__init__",
+            "__repr__",
+            "__str__",
+            "_repr_html_",
+            "__len__",
+            "__getitem__",
+            "__setitem__",
+            "__getattr__",
+        ]
+    )
+
+    # Classes
+    for target_class in classes:
+        for name, func in inspect.getmembers(target_class, inspect.isfunction):
+            if name.startswith("_") and name not in special_functions:
+                continue
+            setattr(target_class, name, _wrap_function(target_class.__name__, name, func, logger))
+
+        for name, prop in inspect.getmembers(target_class, lambda o: isinstance(o, property)):
+            if name.startswith("_"):
+                continue
+            setattr(target_class, name, _wrap_property(target_class.__name__, name, prop, logger))
+
+    # Missings
+    for original, missing in [
+        (pd.DataFrame, _MissingPandasLikeDataFrame),
+        (pd.Series, MissingPandasLikeSeries),
+        (pd.Index, MissingPandasLikeIndex),
+        (pd.MultiIndex, MissingPandasLikeMultiIndex),
+        (pd.CategoricalIndex, MissingPandasLikeCategoricalIndex),
+        (pd.DatetimeIndex, MissingPandasLikeDatetimeIndex),
+        (pd.core.groupby.DataFrameGroupBy, MissingPandasLikeDataFrameGroupBy),
+        (pd.core.groupby.SeriesGroupBy, MissingPandasLikeSeriesGroupBy),
+        (pd.core.window.Expanding, MissingPandasLikeExpanding),
+        (pd.core.window.Rolling, MissingPandasLikeRolling),
+        (pd.core.window.ExpandingGroupby, MissingPandasLikeExpandingGroupby),
+        (pd.core.window.RollingGroupby, MissingPandasLikeRollingGroupby),
+    ]:
+        for name, func in inspect.getmembers(missing, inspect.isfunction):
+            setattr(
+                missing,
+                name,
+                _wrap_missing_function(original.__name__, name, func, original, logger),
+            )
+
+        for name, prop in inspect.getmembers(missing, lambda o: isinstance(o, property)):
+            setattr(missing, name, _wrap_missing_property(original.__name__, name, prop, logger))
+
+
+_local = threading.local()
+
+
+def _wrap_function(class_name, function_name, func, logger):
+
+    signature = inspect.signature(func)
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        if hasattr(_local, "logging") and _local.logging:
+            # no need to log since this should be internal call.
+            return func(*args, **kwargs)
+        _local.logging = True
+        try:
+            start = time.perf_counter()
+            try:
+                res = func(*args, **kwargs)
+                logger.log_success(
+                    class_name, function_name, time.perf_counter() - start, signature
+                )
+                return res
+            except Exception as ex:
+                logger.log_failure(
+                    class_name, function_name, ex, time.perf_counter() - start, signature
+                )
+                raise
+        finally:
+            _local.logging = False
+
+    return wrapper
+
+
+def _wrap_property(class_name, property_name, prop, logger):
+    @property
+    def wrapper(self):
+        if hasattr(_local, "logging") and _local.logging:
+            # no need to log since this should be internal call.
+            return prop.fget(self)
+        _local.logging = True
+        try:
+            start = time.perf_counter()
+            try:
+                res = prop.fget(self)
+                logger.log_success(class_name, property_name, time.perf_counter() - start)
+                return res
+            except Exception as ex:
+                logger.log_failure(class_name, property_name, ex, time.perf_counter() - start)
+                raise
+        finally:
+            _local.logging = False
+
+    wrapper.__doc__ = prop.__doc__
+
+    if prop.fset is not None:
+        wrapper = wrapper.setter(_wrap_function(class_name, prop.fset.__name__, prop.fset, logger))
+
+    return wrapper
+
+
+def _wrap_missing_function(class_name, function_name, func, original, logger):
+
+    if not hasattr(original, function_name):
+        return func
+
+    signature = inspect.signature(getattr(original, function_name))
+
+    is_deprecated = func.__name__ == "deprecated_function"
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        finally:
+            logger.log_missing(class_name, function_name, is_deprecated, signature)
+
+    return wrapper
+
+
+def _wrap_missing_property(class_name, property_name, prop, logger):
+
+    is_deprecated = prop.fget.__name__ == "deprecated_property"
+
+    @property
+    def wrapper(self):
+        try:
+            return prop.fget(self)
+        finally:
+            logger.log_missing(class_name, property_name, is_deprecated)
+
+    return wrapper
diff --git a/python/pyspark/pandas/usage_logging/usage_logger.py b/python/pyspark/pandas/usage_logging/usage_logger.py
new file mode 100644
index 0000000000000..897a74eece079
--- /dev/null
+++ b/python/pyspark/pandas/usage_logging/usage_logger.py
@@ -0,0 +1,132 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+The reference implementation of usage logger using the Python standard logging library.
+"""
+
+from inspect import Signature
+import logging
+from typing import Any, Optional
+
+
+def get_logger() -> Any:
+    """ An entry point of the plug-in and return the usage logger. """
+    return PandasOnSparkUsageLogger()
+
+
+def _format_signature(signature):
+    return (
+        "({})".format(", ".join([p.name for p in signature.parameters.values()]))
+        if signature is not None
+        else ""
+    )
+
+
+class PandasOnSparkUsageLogger(object):
+    """
+    The reference implementation of usage logger.
+
+    The usage logger needs to provide the following methods:
+
+        - log_success(self, class_name, name, duration, signature=None)
+        - log_failure(self, class_name, name, ex, duration, signature=None)
+        - log_missing(self, class_name, name, is_deprecated=False, signature=None)
+    """
+
+    def __init__(self):
+        self.logger = logging.getLogger("pyspark.pandas.usage_logger")
+
+    def log_success(
+        self, class_name: str, name: str, duration: float, signature: Optional[Signature] = None
+    ) -> None:
+        """
+        Log the function or property call is successfully finished.
+
+        :param class_name: the target class name
+        :param name: the target function or property name
+        :param duration: the duration to finish the function or property call
+        :param signature: the signature if the target is a function, else None
+        """
+        if self.logger.isEnabledFor(logging.INFO):
+            msg = (
+                "A {function} `{class_name}.{name}{signature}` was successfully finished "
+                "after {duration:.3f} ms."
+            ).format(
+                class_name=class_name,
+                name=name,
+                signature=_format_signature(signature),
+                duration=duration * 1000,
+                function="function" if signature is not None else "property",
+            )
+            self.logger.info(msg)
+
+    def log_failure(
+        self,
+        class_name: str,
+        name: str,
+        ex: Exception,
+        duration: float,
+        signature: Optional[Signature] = None,
+    ) -> None:
+        """
+        Log the function or property call failed.
+
+        :param class_name: the target class name
+        :param name: the target function or property name
+        :param ex: the exception causing the failure
+        :param duration: the duration until the function or property call fails
+        :param signature: the signature if the target is a function, else None
+        """
+        if self.logger.isEnabledFor(logging.WARNING):
+            msg = (
+                "A {function} `{class_name}.{name}{signature}` was failed "
+                "after {duration:.3f} ms: {msg}"
+            ).format(
+                class_name=class_name,
+                name=name,
+                signature=_format_signature(signature),
+                msg=str(ex),
+                duration=duration * 1000,
+                function="function" if signature is not None else "property",
+            )
+            self.logger.warning(msg)
+
+    def log_missing(
+        self,
+        class_name: str,
+        name: str,
+        is_deprecated: bool = False,
+        signature: Optional[Signature] = None,
+    ) -> None:
+        """
+        Log the missing or deprecated function or property is called.
+
+        :param class_name: the target class name
+        :param name: the target function or property name
+        :param is_deprecated: True if the function or property is marked as deprecated
+        :param signature: the original function signature if the target is a function, else None
+        """
+        if self.logger.isEnabledFor(logging.INFO):
+            msg = "A {deprecated} {function} `{class_name}.{name}{signature}` was called.".format(
+                class_name=class_name,
+                name=name,
+                signature=_format_signature(signature),
+                function="function" if signature is not None else "property",
+                deprecated="deprecated" if is_deprecated else "missing",
+            )
+            self.logger.info(msg)
diff --git a/python/pyspark/pandas/utils.py b/python/pyspark/pandas/utils.py
new file mode 100644
index 0000000000000..948a786c63962
--- /dev/null
+++ b/python/pyspark/pandas/utils.py
@@ -0,0 +1,872 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+Commonly used utils in pandas-on-Spark.
+"""
+
+import functools
+from collections import OrderedDict
+from contextlib import contextmanager
+import os
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union, TYPE_CHECKING
+import warnings
+
+from pyspark import sql as spark
+from pyspark.sql import functions as F
+from pyspark.sql.types import DoubleType
+import pandas as pd
+from pandas.api.types import is_list_like
+
+# For running doctests and reference resolution in PyCharm.
+from pyspark import pandas as ps  # noqa: F401
+from pyspark.pandas.typedef.typehints import (
+    as_spark_type,
+    extension_dtypes,
+    spark_type_to_pandas_dtype,
+)
+
+if TYPE_CHECKING:
+    # This is required in old Python 3.5 to prevent circular reference.
+    from pyspark.pandas.base import IndexOpsMixin  # noqa: F401 (SPARK-34943)
+    from pyspark.pandas.frame import DataFrame  # noqa: F401 (SPARK-34943)
+    from pyspark.pandas.internal import InternalFrame  # noqa: F401 (SPARK-34943)
+
+
+ERROR_MESSAGE_CANNOT_COMBINE = (
+    "Cannot combine the series or dataframe because it comes from a different dataframe. "
+    "In order to allow this operation, enable 'compute.ops_on_diff_frames' option."
+)
+
+
+SPARK_CONF_ARROW_ENABLED = "spark.sql.execution.arrow.pyspark.enabled"
+
+
+def same_anchor(
+    this: Union["DataFrame", "IndexOpsMixin", "InternalFrame"],
+    that: Union["DataFrame", "IndexOpsMixin", "InternalFrame"],
+) -> bool:
+    """
+    Check if the anchors of the given DataFrame or Series are the same or not.
+    """
+    from pyspark.pandas.base import IndexOpsMixin
+    from pyspark.pandas.frame import DataFrame
+    from pyspark.pandas.internal import InternalFrame
+
+    if isinstance(this, InternalFrame):
+        this_internal = this
+    else:
+        assert isinstance(this, (DataFrame, IndexOpsMixin)), type(this)
+        this_internal = this._internal
+
+    if isinstance(that, InternalFrame):
+        that_internal = that
+    else:
+        assert isinstance(that, (DataFrame, IndexOpsMixin)), type(that)
+        that_internal = that._internal
+
+    return (
+        this_internal.spark_frame is that_internal.spark_frame
+        and this_internal.index_level == that_internal.index_level
+        and all(
+            this_scol._jc.equals(that_scol._jc)
+            for this_scol, that_scol in zip(
+                this_internal.index_spark_columns, that_internal.index_spark_columns
+            )
+        )
+    )
+
+
+def combine_frames(this, *args, how="full", preserve_order_column=False):
+    """
+    This method combines `this` DataFrame with a different `that` DataFrame or
+    Series from a different DataFrame.
+
+    It returns a DataFrame that has prefix `this_` and `that_` to distinct
+    the columns names from both DataFrames
+
+    It internally performs a join operation which can be expensive in general.
+    So, if `compute.ops_on_diff_frames` option is False,
+    this method throws an exception.
+    """
+    from pyspark.pandas.config import get_option
+    from pyspark.pandas.frame import DataFrame
+    from pyspark.pandas.internal import (
+        InternalFrame,
+        HIDDEN_COLUMNS,
+        NATURAL_ORDER_COLUMN_NAME,
+        SPARK_INDEX_NAME_FORMAT,
+    )
+    from pyspark.pandas.series import Series
+
+    if all(isinstance(arg, Series) for arg in args):
+        assert all(
+            same_anchor(arg, args[0]) for arg in args
+        ), "Currently only one different DataFrame (from given Series) is supported"
+        assert not same_anchor(this, args[0]), "We don't need to combine. All series is in this."
+        that = args[0]._kdf[list(args)]
+    elif len(args) == 1 and isinstance(args[0], DataFrame):
+        assert isinstance(args[0], DataFrame)
+        assert not same_anchor(
+            this, args[0]
+        ), "We don't need to combine. `this` and `that` are same."
+        that = args[0]
+    else:
+        raise AssertionError("args should be single DataFrame or " "single/multiple Series")
+
+    if get_option("compute.ops_on_diff_frames"):
+
+        def resolve(internal, side):
+            rename = lambda col: "__{}_{}".format(side, col)
+            internal = internal.resolved_copy
+            sdf = internal.spark_frame
+            sdf = internal.spark_frame.select(
+                [
+                    scol_for(sdf, col).alias(rename(col))
+                    for col in sdf.columns
+                    if col not in HIDDEN_COLUMNS
+                ]
+                + list(HIDDEN_COLUMNS)
+            )
+            return internal.copy(
+                spark_frame=sdf,
+                index_spark_columns=[
+                    scol_for(sdf, rename(col)) for col in internal.index_spark_column_names
+                ],
+                data_spark_columns=[
+                    scol_for(sdf, rename(col)) for col in internal.data_spark_column_names
+                ],
+            )
+
+        this_internal = resolve(this._internal, "this")
+        that_internal = resolve(that._internal, "that")
+
+        this_index_map = list(
+            zip(
+                this_internal.index_spark_column_names,
+                this_internal.index_names,
+                this_internal.index_dtypes,
+            )
+        )
+        that_index_map = list(
+            zip(
+                that_internal.index_spark_column_names,
+                that_internal.index_names,
+                that_internal.index_dtypes,
+            )
+        )
+        assert len(this_index_map) == len(that_index_map)
+
+        join_scols = []
+        merged_index_scols = []
+
+        # Note that the order of each element in index_map is guaranteed according to the index
+        # level.
+        this_and_that_index_map = list(zip(this_index_map, that_index_map))
+
+        this_sdf = this_internal.spark_frame.alias("this")
+        that_sdf = that_internal.spark_frame.alias("that")
+
+        # If the same named index is found, that's used.
+        index_column_names = []
+        index_use_extension_dtypes = []
+        for (
+            i,
+            ((this_column, this_name, this_dtype), (that_column, that_name, that_dtype)),
+        ) in enumerate(this_and_that_index_map):
+            if this_name == that_name:
+                # We should merge the Spark columns into one
+                # to mimic pandas' behavior.
+                this_scol = scol_for(this_sdf, this_column)
+                that_scol = scol_for(that_sdf, that_column)
+                join_scol = this_scol == that_scol
+                join_scols.append(join_scol)
+
+                column_name = SPARK_INDEX_NAME_FORMAT(i)
+                index_column_names.append(column_name)
+                index_use_extension_dtypes.append(
+                    any(isinstance(dtype, extension_dtypes) for dtype in [this_dtype, that_dtype])
+                )
+                merged_index_scols.append(
+                    F.when(this_scol.isNotNull(), this_scol).otherwise(that_scol).alias(column_name)
+                )
+            else:
+                raise ValueError("Index names must be exactly matched currently.")
+
+        assert len(join_scols) > 0, "cannot join with no overlapping index names"
+
+        joined_df = this_sdf.join(that_sdf, on=join_scols, how=how)
+
+        if preserve_order_column:
+            order_column = [scol_for(this_sdf, NATURAL_ORDER_COLUMN_NAME)]
+        else:
+            order_column = []
+
+        joined_df = joined_df.select(
+            merged_index_scols
+            + [
+                scol_for(this_sdf, this_internal.spark_column_name_for(label))
+                for label in this_internal.column_labels
+            ]
+            + [
+                scol_for(that_sdf, that_internal.spark_column_name_for(label))
+                for label in that_internal.column_labels
+            ]
+            + order_column
+        )
+
+        index_spark_columns = [scol_for(joined_df, col) for col in index_column_names]
+        index_dtypes = [
+            spark_type_to_pandas_dtype(field.dataType, use_extension_dtypes=use_extension_dtypes)
+            for field, use_extension_dtypes in zip(
+                joined_df.select(index_spark_columns).schema, index_use_extension_dtypes
+            )
+        ]
+
+        index_columns = set(index_column_names)
+        new_data_columns = [
+            col
+            for col in joined_df.columns
+            if col not in index_columns and col != NATURAL_ORDER_COLUMN_NAME
+        ]
+        data_dtypes = this_internal.data_dtypes + that_internal.data_dtypes
+
+        level = max(this_internal.column_labels_level, that_internal.column_labels_level)
+
+        def fill_label(label):
+            if label is None:
+                return ([""] * (level - 1)) + [None]
+            else:
+                return ([""] * (level - len(label))) + list(label)
+
+        column_labels = [
+            tuple(["this"] + fill_label(label)) for label in this_internal.column_labels
+        ] + [tuple(["that"] + fill_label(label)) for label in that_internal.column_labels]
+        column_label_names = (
+            [None] * (1 + level - this_internal.column_labels_level)
+        ) + this_internal.column_label_names
+        return DataFrame(
+            InternalFrame(
+                spark_frame=joined_df,
+                index_spark_columns=index_spark_columns,
+                index_names=this_internal.index_names,
+                index_dtypes=index_dtypes,
+                column_labels=column_labels,
+                data_spark_columns=[scol_for(joined_df, col) for col in new_data_columns],
+                data_dtypes=data_dtypes,
+                column_label_names=column_label_names,
+            )
+        )
+    else:
+        raise ValueError(ERROR_MESSAGE_CANNOT_COMBINE)
+
+
+def align_diff_frames(
+    resolve_func,
+    this: "DataFrame",
+    that: "DataFrame",
+    fillna: bool = True,
+    how: str = "full",
+    preserve_order_column: bool = False,
+) -> "DataFrame":
+    """
+    This method aligns two different DataFrames with a given `func`. Columns are resolved and
+    handled within the given `func`.
+    To use this, `compute.ops_on_diff_frames` should be True, for now.
+
+    :param resolve_func: Takes aligned (joined) DataFrame, the column of the current DataFrame, and
+        the column of another DataFrame. It returns an iterable that produces Series.
+
+        >>> from pyspark.pandas.config import set_option, reset_option
+        >>>
+        >>> set_option("compute.ops_on_diff_frames", True)
+        >>>
+        >>> kdf1 = ps.DataFrame({'a': [9, 8, 7, 6, 5, 4, 3, 2, 1]})
+        >>> kdf2 = ps.DataFrame({'a': [9, 8, 7, 6, 5, 4, 3, 2, 1]})
+        >>>
+        >>> def func(kdf, this_column_labels, that_column_labels):
+        ...    kdf  # conceptually this is A + B.
+        ...
+        ...    # Within this function, Series from A or B can be performed against `kdf`.
+        ...    this_label = this_column_labels[0]  # this is ('a',) from kdf1.
+        ...    that_label = that_column_labels[0]  # this is ('a',) from kdf2.
+        ...    new_series = (kdf[this_label] - kdf[that_label]).rename(str(this_label))
+        ...
+        ...    # This new series will be placed in new DataFrame.
+        ...    yield (new_series, this_label)
+        >>>
+        >>>
+        >>> align_diff_frames(func, kdf1, kdf2).sort_index()
+           a
+        0  0
+        1  0
+        2  0
+        3  0
+        4  0
+        5  0
+        6  0
+        7  0
+        8  0
+        >>> reset_option("compute.ops_on_diff_frames")
+
+    :param this: a DataFrame to align
+    :param that: another DataFrame to align
+    :param fillna: If True, it fills missing values in non-common columns in both `this` and `that`.
+        Otherwise, it returns as are.
+    :param how: join way. In addition, it affects how `resolve_func` resolves the column conflict.
+        - full: `resolve_func` should resolve only common columns from 'this' and 'that' DataFrames.
+            For instance, if 'this' has columns A, B, C and that has B, C, D, `this_columns` and
+            'that_columns' in this function are B, C and B, C.
+        - left: `resolve_func` should resolve columns including that columns.
+            For instance, if 'this' has columns A, B, C and that has B, C, D, `this_columns` is
+            B, C but `that_columns` are B, C, D.
+        - inner: Same as 'full' mode; however, internally performs inner join instead.
+    :return: Aligned DataFrame
+    """
+    from pyspark.pandas.frame import DataFrame
+
+    assert how == "full" or how == "left" or how == "inner"
+
+    this_column_labels = this._internal.column_labels
+    that_column_labels = that._internal.column_labels
+    common_column_labels = set(this_column_labels).intersection(that_column_labels)
+
+    # 1. Perform the join given two dataframes.
+    combined = combine_frames(this, that, how=how, preserve_order_column=preserve_order_column)
+
+    # 2. Apply the given function to transform the columns in a batch and keep the new columns.
+    combined_column_labels = combined._internal.column_labels
+
+    that_columns_to_apply = []
+    this_columns_to_apply = []
+    additional_that_columns = []
+    columns_to_keep = []
+    column_labels_to_keep = []
+
+    for combined_label in combined_column_labels:
+        for common_label in common_column_labels:
+            if combined_label == tuple(["this", *common_label]):
+                this_columns_to_apply.append(combined_label)
+                break
+            elif combined_label == tuple(["that", *common_label]):
+                that_columns_to_apply.append(combined_label)
+                break
+        else:
+            if how == "left" and combined_label in [
+                tuple(["that", *label]) for label in that_column_labels
+            ]:
+                # In this case, we will drop `that_columns` in `columns_to_keep` but passes
+                # it later to `func`. `func` should resolve it.
+                # Note that adding this into a separate list (`additional_that_columns`)
+                # is intentional so that `this_columns` and `that_columns` can be paired.
+                additional_that_columns.append(combined_label)
+            elif fillna:
+                columns_to_keep.append(F.lit(None).cast(DoubleType()).alias(str(combined_label)))
+                column_labels_to_keep.append(combined_label)
+            else:
+                columns_to_keep.append(combined._kser_for(combined_label))
+                column_labels_to_keep.append(combined_label)
+
+    that_columns_to_apply += additional_that_columns
+
+    # Should extract columns to apply and do it in a batch in case
+    # it adds new columns for example.
+    if len(this_columns_to_apply) > 0 or len(that_columns_to_apply) > 0:
+        kser_set, column_labels_applied = zip(
+            *resolve_func(combined, this_columns_to_apply, that_columns_to_apply)
+        )
+        columns_applied = list(kser_set)
+        column_labels_applied = list(column_labels_applied)
+    else:
+        columns_applied = []
+        column_labels_applied = []
+
+    applied = DataFrame(
+        combined._internal.with_new_columns(
+            columns_applied + columns_to_keep,
+            column_labels=column_labels_applied + column_labels_to_keep,
+        )
+    )  # type: DataFrame
+
+    # 3. Restore the names back and deduplicate columns.
+    this_labels = OrderedDict()
+    # Add columns in an order of its original frame.
+    for this_label in this_column_labels:
+        for new_label in applied._internal.column_labels:
+            if new_label[1:] not in this_labels and this_label == new_label[1:]:
+                this_labels[new_label[1:]] = new_label
+
+    # After that, we will add the rest columns.
+    other_labels = OrderedDict()
+    for new_label in applied._internal.column_labels:
+        if new_label[1:] not in this_labels:
+            other_labels[new_label[1:]] = new_label
+
+    kdf = applied[list(this_labels.values()) + list(other_labels.values())]
+    kdf.columns = kdf.columns.droplevel()
+    return kdf
+
+
+def is_testing():
+    """ Indicates whether Spark is currently running tests. """
+    return "SPARK_TESTING" in os.environ
+
+
+def default_session(conf=None):
+    if conf is None:
+        conf = dict()
+
+    builder = spark.SparkSession.builder.appName("pandas-on-Spark")
+    for key, value in conf.items():
+        builder = builder.config(key, value)
+    # Currently, pandas-on-Spark is dependent on such join due to 'compute.ops_on_diff_frames'
+    # configuration. This is needed with Spark 3.0+.
+    builder.config("spark.sql.analyzer.failAmbiguousSelfJoin", False)
+
+    if is_testing():
+        builder.config("spark.executor.allowSparkContext", False)
+
+    return builder.getOrCreate()
+
+
+@contextmanager
+def sql_conf(pairs, *, spark=None):
+    """
+    A convenient context manager to set `value` to the Spark SQL configuration `key` and
+    then restores it back when it exits.
+    """
+    assert isinstance(pairs, dict), "pairs should be a dictionary."
+
+    if spark is None:
+        spark = default_session()
+
+    keys = pairs.keys()
+    new_values = pairs.values()
+    old_values = [spark.conf.get(key, None) for key in keys]
+    for key, new_value in zip(keys, new_values):
+        spark.conf.set(key, new_value)
+    try:
+        yield
+    finally:
+        for key, old_value in zip(keys, old_values):
+            if old_value is None:
+                spark.conf.unset(key)
+            else:
+                spark.conf.set(key, old_value)
+
+
+def validate_arguments_and_invoke_function(
+    pobj: Union[pd.DataFrame, pd.Series],
+    pandas_on_spark_func: Callable,
+    pandas_func: Callable,
+    input_args: Dict,
+):
+    """
+    Invokes a pandas function.
+
+    This is created because different versions of pandas support different parameters, and as a
+    result when we code against the latest version, our users might get a confusing
+    "got an unexpected keyword argument" error if they are using an older version of pandas.
+
+    This function validates all the arguments, removes the ones that are not supported if they
+    are simply the default value (i.e. most likely the user didn't explicitly specify it). It
+    throws a TypeError if the user explicitly specify an argument that is not supported by the
+    pandas version available.
+
+    For example usage, look at DataFrame.to_html().
+
+    :param pobj: the pandas DataFrame or Series to operate on
+    :param pandas_on_spark_func: pandas-on-Spark function, used to get default parameter values
+    :param pandas_func: pandas function, used to check whether pandas supports all the arguments
+    :param input_args: arguments to pass to the pandas function, often created by using locals().
+                       Make sure locals() call is at the top of the function so it captures only
+                       input parameters, rather than local variables.
+    :return: whatever pandas_func returns
+    """
+    import inspect
+
+    # Makes a copy since whatever passed in is likely created by locals(), and we can't delete
+    # 'self' key from that.
+    args = input_args.copy()
+    del args["self"]
+
+    if "kwargs" in args:
+        # explode kwargs
+        kwargs = args["kwargs"]
+        del args["kwargs"]
+        args = {**args, **kwargs}
+
+    pandas_on_spark_params = inspect.signature(pandas_on_spark_func).parameters
+    pandas_params = inspect.signature(pandas_func).parameters
+
+    for param in pandas_on_spark_params.values():
+        if param.name not in pandas_params:
+            if args[param.name] == param.default:
+                del args[param.name]
+            else:
+                raise TypeError(
+                    (
+                        "The pandas version [%s] available does not support parameter '%s' "
+                        + "for function '%s'."
+                    )
+                    % (pd.__version__, param.name, pandas_func.__name__)
+                )
+
+    args["self"] = pobj
+    return pandas_func(**args)
+
+
+def lazy_property(fn):
+    """
+    Decorator that makes a property lazy-evaluated.
+
+    Copied from https://stevenloria.com/lazy-properties/
+    """
+    attr_name = "_lazy_" + fn.__name__
+
+    @property
+    @functools.wraps(fn)
+    def wrapped_lazy_property(self):
+        if not hasattr(self, attr_name):
+            setattr(self, attr_name, fn(self))
+        return getattr(self, attr_name)
+
+    def deleter(self):
+        if hasattr(self, attr_name):
+            delattr(self, attr_name)
+
+    return wrapped_lazy_property.deleter(deleter)
+
+
+def scol_for(sdf: spark.DataFrame, column_name: str) -> spark.Column:
+    """ Return Spark Column for the given column name. """
+    return sdf["`{}`".format(column_name)]
+
+
+def column_labels_level(column_labels: List[Tuple]) -> int:
+    """ Return the level of the column index. """
+    if len(column_labels) == 0:
+        return 1
+    else:
+        levels = set(1 if label is None else len(label) for label in column_labels)
+        assert len(levels) == 1, levels
+        return list(levels)[0]
+
+
+def name_like_string(name: Optional[Union[str, Tuple]]) -> str:
+    """
+    Return the name-like strings from str or tuple of str
+
+    Examples
+    --------
+    >>> name = 'abc'
+    >>> name_like_string(name)
+    'abc'
+
+    >>> name = ('abc',)
+    >>> name_like_string(name)
+    'abc'
+
+    >>> name = ('a', 'b', 'c')
+    >>> name_like_string(name)
+    '(a, b, c)'
+    """
+    if name is None:
+        name = ("__none__",)
+    elif is_list_like(name):
+        name = tuple([str(n) for n in name])
+    else:
+        name = (str(name),)
+    return ("(%s)" % ", ".join(name)) if len(name) > 1 else name[0]
+
+
+def is_name_like_tuple(value: Any, allow_none: bool = True, check_type: bool = False) -> bool:
+    """
+    Check the given tuple is be able to be used as a name.
+
+    Examples
+    --------
+    >>> is_name_like_tuple(('abc',))
+    True
+    >>> is_name_like_tuple((1,))
+    True
+    >>> is_name_like_tuple(('abc', 1, None))
+    True
+    >>> is_name_like_tuple(('abc', 1, None), check_type=True)
+    True
+    >>> is_name_like_tuple((1.0j,))
+    True
+    >>> is_name_like_tuple(tuple())
+    False
+    >>> is_name_like_tuple((list('abc'),))
+    False
+    >>> is_name_like_tuple(('abc', 1, None), allow_none=False)
+    False
+    >>> is_name_like_tuple((1.0j,), check_type=True)
+    False
+    """
+    if value is None:
+        return allow_none
+    elif not isinstance(value, tuple):
+        return False
+    elif len(value) == 0:
+        return False
+    elif not allow_none and any(v is None for v in value):
+        return False
+    elif any(is_list_like(v) or isinstance(v, slice) for v in value):
+        return False
+    elif check_type:
+        return all(
+            v is None or as_spark_type(type(v), raise_error=False) is not None for v in value
+        )
+    else:
+        return True
+
+
+def is_name_like_value(
+    value: Any, allow_none: bool = True, allow_tuple: bool = True, check_type: bool = False
+) -> bool:
+    """
+    Check the given value is like a name.
+
+    Examples
+    --------
+    >>> is_name_like_value('abc')
+    True
+    >>> is_name_like_value(1)
+    True
+    >>> is_name_like_value(None)
+    True
+    >>> is_name_like_value(('abc',))
+    True
+    >>> is_name_like_value(1.0j)
+    True
+    >>> is_name_like_value(list('abc'))
+    False
+    >>> is_name_like_value(None, allow_none=False)
+    False
+    >>> is_name_like_value(('abc',), allow_tuple=False)
+    False
+    >>> is_name_like_value(1.0j, check_type=True)
+    False
+    """
+    if value is None:
+        return allow_none
+    elif isinstance(value, tuple):
+        return allow_tuple and is_name_like_tuple(
+            value, allow_none=allow_none, check_type=check_type
+        )
+    elif is_list_like(value) or isinstance(value, slice):
+        return False
+    elif check_type:
+        return as_spark_type(type(value), raise_error=False) is not None
+    else:
+        return True
+
+
+def validate_axis(axis=0, none_axis=0):
+    """ Check the given axis is valid. """
+    # convert to numeric axis
+    axis = {None: none_axis, "index": 0, "columns": 1}.get(axis, axis)
+    if axis not in (none_axis, 0, 1):
+        raise ValueError("No axis named {0}".format(axis))
+    return axis
+
+
+def validate_bool_kwarg(value, arg_name):
+    """ Ensures that argument passed in arg_name is of type bool. """
+    if not (isinstance(value, bool) or value is None):
+        raise TypeError(
+            'For argument "{}" expected type bool, received '
+            "type {}.".format(arg_name, type(value).__name__)
+        )
+    return value
+
+
+def validate_how(how: str) -> str:
+    """ Check the given how for join is valid. """
+    if how == "full":
+        warnings.warn(
+            "Warning: While pandas-on-Spark will accept 'full', you should use 'outer' "
+            + "instead to be compatible with the pandas merge API",
+            UserWarning,
+        )
+    if how == "outer":
+        # 'outer' in pandas equals 'full' in Spark
+        how = "full"
+    if how not in ("inner", "left", "right", "full"):
+        raise ValueError(
+            "The 'how' parameter has to be amongst the following values: ",
+            "['inner', 'left', 'right', 'outer']",
+        )
+    return how
+
+
+def verify_temp_column_name(
+    df: Union["DataFrame", spark.DataFrame], column_name_or_label: Union[Any, Tuple]
+) -> Union[Any, Tuple]:
+    """
+    Verify that the given column name does not exist in the given pandas-on-Spark or
+    Spark DataFrame.
+
+    The temporary column names should start and end with `__`. In addition, `column_name_or_label`
+    expects a single string, or column labels when `df` is a pandas-on-Spark DataFrame.
+
+    >>> kdf = ps.DataFrame({("x", "a"): ['a', 'b', 'c']})
+    >>> kdf["__dummy__"] = 0
+    >>> kdf[("", "__dummy__")] = 1
+    >>> kdf  # doctest: +NORMALIZE_WHITESPACE
+       x __dummy__
+       a           __dummy__
+    0  a         0         1
+    1  b         0         1
+    2  c         0         1
+
+    >>> verify_temp_column_name(kdf, '__tmp__')
+    ('__tmp__', '')
+    >>> verify_temp_column_name(kdf, ('', '__tmp__'))
+    ('', '__tmp__')
+    >>> verify_temp_column_name(kdf, '__dummy__')
+    Traceback (most recent call last):
+    ...
+    AssertionError: ... `(__dummy__, )` ...
+    >>> verify_temp_column_name(kdf, ('', '__dummy__'))
+    Traceback (most recent call last):
+    ...
+    AssertionError: ... `(, __dummy__)` ...
+    >>> verify_temp_column_name(kdf, 'dummy')
+    Traceback (most recent call last):
+    ...
+    AssertionError: ... should be empty or start and end with `__`: ('dummy', '')
+    >>> verify_temp_column_name(kdf, ('', 'dummy'))
+    Traceback (most recent call last):
+    ...
+    AssertionError: ... should be empty or start and end with `__`: ('', 'dummy')
+
+    >>> internal = kdf._internal.resolved_copy
+    >>> sdf = internal.spark_frame
+    >>> sdf.select(internal.data_spark_columns).show()  # doctest: +NORMALIZE_WHITESPACE
+    +------+---------+-------------+
+    |(x, a)|__dummy__|(, __dummy__)|
+    +------+---------+-------------+
+    |     a|        0|            1|
+    |     b|        0|            1|
+    |     c|        0|            1|
+    +------+---------+-------------+
+
+    >>> verify_temp_column_name(sdf, '__tmp__')
+    '__tmp__'
+    >>> verify_temp_column_name(sdf, '__dummy__')
+    Traceback (most recent call last):
+    ...
+    AssertionError: ... `__dummy__` ... '(x, a)', '__dummy__', '(, __dummy__)', ...
+    >>> verify_temp_column_name(sdf, ('', '__dummy__'))
+    Traceback (most recent call last):
+    ...
+    AssertionError: <class 'tuple'>
+    >>> verify_temp_column_name(sdf, 'dummy')
+    Traceback (most recent call last):
+    ...
+    AssertionError: ... should start and end with `__`: dummy
+    """
+    from pyspark.pandas.frame import DataFrame
+
+    if isinstance(df, DataFrame):
+        if isinstance(column_name_or_label, str):
+            column_name = column_name_or_label
+
+            level = df._internal.column_labels_level
+            column_name_or_label = tuple([column_name_or_label] + ([""] * (level - 1)))
+        else:
+            column_name = name_like_string(column_name_or_label)
+
+        assert any(len(label) > 0 for label in column_name_or_label) and all(
+            label == "" or (label.startswith("__") and label.endswith("__"))
+            for label in column_name_or_label
+        ), "The temporary column name should be empty or start and end with `__`: {}".format(
+            column_name_or_label
+        )
+        assert all(
+            column_name_or_label != label for label in df._internal.column_labels
+        ), "The given column name `{}` already exists in the pandas-on-Spark DataFrame: {}".format(
+            name_like_string(column_name_or_label), df.columns
+        )
+        df = df._internal.resolved_copy.spark_frame
+    else:
+        assert isinstance(column_name_or_label, str), type(column_name_or_label)
+        assert column_name_or_label.startswith("__") and column_name_or_label.endswith(
+            "__"
+        ), "The temporary column name should start and end with `__`: {}".format(
+            column_name_or_label
+        )
+        column_name = column_name_or_label
+
+    assert isinstance(df, spark.DataFrame), type(df)
+    assert (
+        column_name not in df.columns
+    ), "The given column name `{}` already exists in the Spark DataFrame: {}".format(
+        column_name, df.columns
+    )
+
+    return column_name_or_label
+
+
+def compare_null_first(left, right, comp):
+    return (left.isNotNull() & right.isNotNull() & comp(left, right)) | (
+        left.isNull() & right.isNotNull()
+    )
+
+
+def compare_null_last(left, right, comp):
+    return (left.isNotNull() & right.isNotNull() & comp(left, right)) | (
+        left.isNotNull() & right.isNull()
+    )
+
+
+def compare_disallow_null(left, right, comp):
+    return left.isNotNull() & right.isNotNull() & comp(left, right)
+
+
+def compare_allow_null(left, right, comp):
+    return left.isNull() | right.isNull() | comp(left, right)
+
+
+def _test():
+    import os
+    import doctest
+    import sys
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.utils
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.utils.__dict__.copy()
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]").appName("pyspark.pandas.utils tests").getOrCreate()
+    )
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.utils,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/pandas/version.py b/python/pyspark/pandas/version.py
new file mode 100644
index 0000000000000..449df1d633135
--- /dev/null
+++ b/python/pyspark/pandas/version.py
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+__version__ = "1.7.0"
diff --git a/python/pyspark/pandas/window.py b/python/pyspark/pandas/window.py
new file mode 100644
index 0000000000000..06f74122050ea
--- /dev/null
+++ b/python/pyspark/pandas/window.py
@@ -0,0 +1,1767 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from functools import partial
+from typing import Any, Union, TYPE_CHECKING
+
+from pyspark.sql import Window
+from pyspark.sql import functions as F
+from pyspark.pandas.missing.window import (
+    MissingPandasLikeRolling,
+    MissingPandasLikeRollingGroupby,
+    MissingPandasLikeExpanding,
+    MissingPandasLikeExpandingGroupby,
+)
+
+# For running doctests and reference resolution in PyCharm.
+from pyspark import pandas as ps  # noqa: F401
+
+from pyspark.pandas.internal import NATURAL_ORDER_COLUMN_NAME, SPARK_INDEX_NAME_FORMAT
+from pyspark.pandas.utils import scol_for
+
+if TYPE_CHECKING:
+    from pyspark.pandas.frame import DataFrame  # noqa: F401 (SPARK-34943)
+    from pyspark.pandas.series import Series  # noqa: F401 (SPARK-34943)
+
+
+class RollingAndExpanding(object):
+    def __init__(self, kdf_or_kser, window, min_periods):
+        self._kdf_or_kser = kdf_or_kser
+        self._window = window
+        # This unbounded Window is later used to handle 'min_periods' for now.
+        self._unbounded_window = Window.orderBy(NATURAL_ORDER_COLUMN_NAME).rowsBetween(
+            Window.unboundedPreceding, Window.currentRow
+        )
+        self._min_periods = min_periods
+
+    def _apply_as_series_or_frame(self, func):
+        """
+        Wraps a function that handles Spark column in order
+        to support it in both pandas-on-Spark Series and DataFrame.
+        Note that the given `func` name should be same as the API's method name.
+        """
+        raise NotImplementedError(
+            "A class that inherits this class should implement this method "
+            "to handle the index and columns of output."
+        )
+
+    def count(self) -> Union["Series", "DataFrame"]:
+        def count(scol):
+            return F.count(scol).over(self._window)
+
+        return self._apply_as_series_or_frame(count).astype("float64")
+
+    def sum(self) -> Union["Series", "DataFrame"]:
+        def sum(scol):
+            return F.when(
+                F.row_number().over(self._unbounded_window) >= self._min_periods,
+                F.sum(scol).over(self._window),
+            ).otherwise(F.lit(None))
+
+        return self._apply_as_series_or_frame(sum)
+
+    def min(self) -> Union["Series", "DataFrame"]:
+        def min(scol):
+            return F.when(
+                F.row_number().over(self._unbounded_window) >= self._min_periods,
+                F.min(scol).over(self._window),
+            ).otherwise(F.lit(None))
+
+        return self._apply_as_series_or_frame(min)
+
+    def max(self) -> Union["Series", "DataFrame"]:
+        def max(scol):
+            return F.when(
+                F.row_number().over(self._unbounded_window) >= self._min_periods,
+                F.max(scol).over(self._window),
+            ).otherwise(F.lit(None))
+
+        return self._apply_as_series_or_frame(max)
+
+    def mean(self) -> Union["Series", "DataFrame"]:
+        def mean(scol):
+            return F.when(
+                F.row_number().over(self._unbounded_window) >= self._min_periods,
+                F.mean(scol).over(self._window),
+            ).otherwise(F.lit(None))
+
+        return self._apply_as_series_or_frame(mean)
+
+    def std(self) -> Union["Series", "DataFrame"]:
+        def std(scol):
+            return F.when(
+                F.row_number().over(self._unbounded_window) >= self._min_periods,
+                F.stddev(scol).over(self._window),
+            ).otherwise(F.lit(None))
+
+        return self._apply_as_series_or_frame(std)
+
+    def var(self) -> Union["Series", "DataFrame"]:
+        def var(scol):
+            return F.when(
+                F.row_number().over(self._unbounded_window) >= self._min_periods,
+                F.variance(scol).over(self._window),
+            ).otherwise(F.lit(None))
+
+        return self._apply_as_series_or_frame(var)
+
+
+class Rolling(RollingAndExpanding):
+    def __init__(self, kdf_or_kser, window, min_periods=None):
+        from pyspark.pandas import DataFrame, Series
+
+        if window < 0:
+            raise ValueError("window must be >= 0")
+        if (min_periods is not None) and (min_periods < 0):
+            raise ValueError("min_periods must be >= 0")
+        if min_periods is None:
+            # TODO: 'min_periods' is not equivalent in pandas because it does not count NA as
+            #  a value.
+            min_periods = window
+
+        if not isinstance(kdf_or_kser, (DataFrame, Series)):
+            raise TypeError(
+                "kdf_or_kser must be a series or dataframe; however, got: %s" % type(kdf_or_kser)
+            )
+
+        window = Window.orderBy(NATURAL_ORDER_COLUMN_NAME).rowsBetween(
+            Window.currentRow - (window - 1), Window.currentRow
+        )
+
+        super().__init__(kdf_or_kser, window, min_periods)
+
+    def __getattr__(self, item: str) -> Any:
+        if hasattr(MissingPandasLikeRolling, item):
+            property_or_func = getattr(MissingPandasLikeRolling, item)
+            if isinstance(property_or_func, property):
+                return property_or_func.fget(self)  # type: ignore
+            else:
+                return partial(property_or_func, self)
+        raise AttributeError(item)
+
+    def _apply_as_series_or_frame(self, func):
+        return self._kdf_or_kser._apply_series_op(
+            lambda kser: kser._with_new_scol(func(kser.spark.column)),  # TODO: dtype?
+            should_resolve=True,
+        )
+
+    def count(self) -> Union["Series", "DataFrame"]:
+        """
+        The rolling count of any non-NaN observations inside the window.
+
+        .. note:: the current implementation of this API uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Returns
+        -------
+        Series.expanding : Calling object with Series data.
+        DataFrame.expanding : Calling object with DataFrames.
+        Series.count : Count of the full Series.
+        DataFrame.count : Count of the full DataFrame.
+
+        Examples
+        --------
+        >>> s = ps.Series([2, 3, float("nan"), 10])
+        >>> s.rolling(1).count()
+        0    1.0
+        1    1.0
+        2    0.0
+        3    1.0
+        dtype: float64
+
+        >>> s.rolling(3).count()
+        0    1.0
+        1    2.0
+        2    2.0
+        3    2.0
+        dtype: float64
+
+        >>> s.to_frame().rolling(1).count()
+             0
+        0  1.0
+        1  1.0
+        2  0.0
+        3  1.0
+
+        >>> s.to_frame().rolling(3).count()
+             0
+        0  1.0
+        1  2.0
+        2  2.0
+        3  2.0
+        """
+        return super().count()
+
+    def sum(self) -> Union["Series", "DataFrame"]:
+        """
+        Calculate rolling summation of given DataFrame or Series.
+
+        .. note:: the current implementation of this API uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Returns
+        -------
+        Series or DataFrame
+            Same type as the input, with the same index, containing the
+            rolling summation.
+
+        See Also
+        --------
+        Series.expanding : Calling object with Series data.
+        DataFrame.expanding : Calling object with DataFrames.
+        Series.sum : Reducing sum for Series.
+        DataFrame.sum : Reducing sum for DataFrame.
+
+        Examples
+        --------
+        >>> s = ps.Series([4, 3, 5, 2, 6])
+        >>> s
+        0    4
+        1    3
+        2    5
+        3    2
+        4    6
+        dtype: int64
+
+        >>> s.rolling(2).sum()
+        0    NaN
+        1    7.0
+        2    8.0
+        3    7.0
+        4    8.0
+        dtype: float64
+
+        >>> s.rolling(3).sum()
+        0     NaN
+        1     NaN
+        2    12.0
+        3    10.0
+        4    13.0
+        dtype: float64
+
+        For DataFrame, each rolling summation is computed column-wise.
+
+        >>> df = ps.DataFrame({"A": s.to_numpy(), "B": s.to_numpy() ** 2})
+        >>> df
+           A   B
+        0  4  16
+        1  3   9
+        2  5  25
+        3  2   4
+        4  6  36
+
+        >>> df.rolling(2).sum()
+             A     B
+        0  NaN   NaN
+        1  7.0  25.0
+        2  8.0  34.0
+        3  7.0  29.0
+        4  8.0  40.0
+
+        >>> df.rolling(3).sum()
+              A     B
+        0   NaN   NaN
+        1   NaN   NaN
+        2  12.0  50.0
+        3  10.0  38.0
+        4  13.0  65.0
+        """
+        return super().sum()
+
+    def min(self) -> Union["Series", "DataFrame"]:
+        """
+        Calculate the rolling minimum.
+
+        .. note:: the current implementation of this API uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Returns
+        -------
+        Series or DataFrame
+            Returned object type is determined by the caller of the rolling
+            calculation.
+
+        See Also
+        --------
+        Series.rolling : Calling object with a Series.
+        DataFrame.rolling : Calling object with a DataFrame.
+        Series.min : Similar method for Series.
+        DataFrame.min : Similar method for DataFrame.
+
+        Examples
+        --------
+        >>> s = ps.Series([4, 3, 5, 2, 6])
+        >>> s
+        0    4
+        1    3
+        2    5
+        3    2
+        4    6
+        dtype: int64
+
+        >>> s.rolling(2).min()
+        0    NaN
+        1    3.0
+        2    3.0
+        3    2.0
+        4    2.0
+        dtype: float64
+
+        >>> s.rolling(3).min()
+        0    NaN
+        1    NaN
+        2    3.0
+        3    2.0
+        4    2.0
+        dtype: float64
+
+        For DataFrame, each rolling minimum is computed column-wise.
+
+        >>> df = ps.DataFrame({"A": s.to_numpy(), "B": s.to_numpy() ** 2})
+        >>> df
+           A   B
+        0  4  16
+        1  3   9
+        2  5  25
+        3  2   4
+        4  6  36
+
+        >>> df.rolling(2).min()
+             A    B
+        0  NaN  NaN
+        1  3.0  9.0
+        2  3.0  9.0
+        3  2.0  4.0
+        4  2.0  4.0
+
+        >>> df.rolling(3).min()
+             A    B
+        0  NaN  NaN
+        1  NaN  NaN
+        2  3.0  9.0
+        3  2.0  4.0
+        4  2.0  4.0
+        """
+        return super().min()
+
+    def max(self) -> Union["Series", "DataFrame"]:
+        """
+        Calculate the rolling maximum.
+
+        .. note:: the current implementation of this API uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Returns
+        -------
+        Series or DataFrame
+            Return type is determined by the caller.
+
+        See Also
+        --------
+        Series.rolling : Series rolling.
+        DataFrame.rolling : DataFrame rolling.
+        Series.max : Similar method for Series.
+        DataFrame.max : Similar method for DataFrame.
+
+        Examples
+        --------
+        >>> s = ps.Series([4, 3, 5, 2, 6])
+        >>> s
+        0    4
+        1    3
+        2    5
+        3    2
+        4    6
+        dtype: int64
+
+        >>> s.rolling(2).max()
+        0    NaN
+        1    4.0
+        2    5.0
+        3    5.0
+        4    6.0
+        dtype: float64
+
+        >>> s.rolling(3).max()
+        0    NaN
+        1    NaN
+        2    5.0
+        3    5.0
+        4    6.0
+        dtype: float64
+
+        For DataFrame, each rolling maximum is computed column-wise.
+
+        >>> df = ps.DataFrame({"A": s.to_numpy(), "B": s.to_numpy() ** 2})
+        >>> df
+           A   B
+        0  4  16
+        1  3   9
+        2  5  25
+        3  2   4
+        4  6  36
+
+        >>> df.rolling(2).max()
+             A     B
+        0  NaN   NaN
+        1  4.0  16.0
+        2  5.0  25.0
+        3  5.0  25.0
+        4  6.0  36.0
+
+        >>> df.rolling(3).max()
+             A     B
+        0  NaN   NaN
+        1  NaN   NaN
+        2  5.0  25.0
+        3  5.0  25.0
+        4  6.0  36.0
+        """
+        return super().max()
+
+    def mean(self) -> Union["Series", "DataFrame"]:
+        """
+        Calculate the rolling mean of the values.
+
+        .. note:: the current implementation of this API uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Returns
+        -------
+        Series or DataFrame
+            Returned object type is determined by the caller of the rolling
+            calculation.
+
+        See Also
+        --------
+        Series.rolling : Calling object with Series data.
+        DataFrame.rolling : Calling object with DataFrames.
+        Series.mean : Equivalent method for Series.
+        DataFrame.mean : Equivalent method for DataFrame.
+
+        Examples
+        --------
+        >>> s = ps.Series([4, 3, 5, 2, 6])
+        >>> s
+        0    4
+        1    3
+        2    5
+        3    2
+        4    6
+        dtype: int64
+
+        >>> s.rolling(2).mean()
+        0    NaN
+        1    3.5
+        2    4.0
+        3    3.5
+        4    4.0
+        dtype: float64
+
+        >>> s.rolling(3).mean()
+        0         NaN
+        1         NaN
+        2    4.000000
+        3    3.333333
+        4    4.333333
+        dtype: float64
+
+        For DataFrame, each rolling mean is computed column-wise.
+
+        >>> df = ps.DataFrame({"A": s.to_numpy(), "B": s.to_numpy() ** 2})
+        >>> df
+           A   B
+        0  4  16
+        1  3   9
+        2  5  25
+        3  2   4
+        4  6  36
+
+        >>> df.rolling(2).mean()
+             A     B
+        0  NaN   NaN
+        1  3.5  12.5
+        2  4.0  17.0
+        3  3.5  14.5
+        4  4.0  20.0
+
+        >>> df.rolling(3).mean()
+                  A          B
+        0       NaN        NaN
+        1       NaN        NaN
+        2  4.000000  16.666667
+        3  3.333333  12.666667
+        4  4.333333  21.666667
+        """
+        return super().mean()
+
+    def std(self) -> Union["Series", "DataFrame"]:
+        """
+        Calculate rolling standard deviation.
+
+        .. note:: the current implementation of this API uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Returns
+        -------
+        Series or DataFrame
+            Returns the same object type as the caller of the rolling calculation.
+
+        See Also
+        --------
+        Series.rolling : Calling object with Series data.
+        DataFrame.rolling : Calling object with DataFrames.
+        Series.std : Equivalent method for Series.
+        DataFrame.std : Equivalent method for DataFrame.
+        numpy.std : Equivalent method for Numpy array.
+
+        Examples
+        --------
+        >>> s = ps.Series([5, 5, 6, 7, 5, 5, 5])
+        >>> s.rolling(3).std()
+        0         NaN
+        1         NaN
+        2    0.577350
+        3    1.000000
+        4    1.000000
+        5    1.154701
+        6    0.000000
+        dtype: float64
+
+        For DataFrame, each rolling standard deviation is computed column-wise.
+
+        >>> df = ps.DataFrame({"A": s.to_numpy(), "B": s.to_numpy() ** 2})
+        >>> df.rolling(2).std()
+                  A          B
+        0       NaN        NaN
+        1  0.000000   0.000000
+        2  0.707107   7.778175
+        3  0.707107   9.192388
+        4  1.414214  16.970563
+        5  0.000000   0.000000
+        6  0.000000   0.000000
+        """
+        return super().std()
+
+    def var(self) -> Union["Series", "DataFrame"]:
+        """
+        Calculate unbiased rolling variance.
+
+        .. note:: the current implementation of this API uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Returns
+        -------
+        Series or DataFrame
+            Returns the same object type as the caller of the rolling calculation.
+
+        See Also
+        --------
+        Series.rolling : Calling object with Series data.
+        DataFrame.rolling : Calling object with DataFrames.
+        Series.var : Equivalent method for Series.
+        DataFrame.var : Equivalent method for DataFrame.
+        numpy.var : Equivalent method for Numpy array.
+
+        Examples
+        --------
+        >>> s = ps.Series([5, 5, 6, 7, 5, 5, 5])
+        >>> s.rolling(3).var()
+        0         NaN
+        1         NaN
+        2    0.333333
+        3    1.000000
+        4    1.000000
+        5    1.333333
+        6    0.000000
+        dtype: float64
+
+        For DataFrame, each unbiased rolling variance is computed column-wise.
+
+        >>> df = ps.DataFrame({"A": s.to_numpy(), "B": s.to_numpy() ** 2})
+        >>> df.rolling(2).var()
+             A      B
+        0  NaN    NaN
+        1  0.0    0.0
+        2  0.5   60.5
+        3  0.5   84.5
+        4  2.0  288.0
+        5  0.0    0.0
+        6  0.0    0.0
+        """
+        return super().var()
+
+
+class RollingGroupby(Rolling):
+    def __init__(self, groupby, window, min_periods=None):
+        from pyspark.pandas.groupby import SeriesGroupBy
+        from pyspark.pandas.groupby import DataFrameGroupBy
+
+        if isinstance(groupby, SeriesGroupBy):
+            kdf_or_kser = groupby._kser
+        elif isinstance(groupby, DataFrameGroupBy):
+            kdf_or_kser = groupby._kdf
+        else:
+            raise TypeError(
+                "groupby must be a SeriesGroupBy or DataFrameGroupBy; "
+                "however, got: %s" % type(groupby)
+            )
+
+        super().__init__(kdf_or_kser, window, min_periods)
+
+        self._groupby = groupby
+        self._window = self._window.partitionBy(*[ser.spark.column for ser in groupby._groupkeys])
+        self._unbounded_window = self._unbounded_window.partitionBy(
+            *[ser.spark.column for ser in groupby._groupkeys]
+        )
+
+    def __getattr__(self, item: str) -> Any:
+        if hasattr(MissingPandasLikeRollingGroupby, item):
+            property_or_func = getattr(MissingPandasLikeRollingGroupby, item)
+            if isinstance(property_or_func, property):
+                return property_or_func.fget(self)  # type: ignore
+            else:
+                return partial(property_or_func, self)
+        raise AttributeError(item)
+
+    def _apply_as_series_or_frame(self, func):
+        """
+        Wraps a function that handles Spark column in order
+        to support it in both pandas-on-Spark Series and DataFrame.
+        Note that the given `func` name should be same as the API's method name.
+        """
+        from pyspark.pandas import DataFrame
+        from pyspark.pandas.series import first_series
+        from pyspark.pandas.groupby import SeriesGroupBy
+
+        groupby = self._groupby
+        kdf = groupby._kdf
+
+        # Here we need to include grouped key as an index, and shift previous index.
+        #   [index_column0, index_column1] -> [grouped key, index_column0, index_column1]
+        new_index_scols = []
+        new_index_spark_column_names = []
+        new_index_names = []
+        new_index_dtypes = []
+        for groupkey in groupby._groupkeys:
+            index_column_name = SPARK_INDEX_NAME_FORMAT(len(new_index_scols))
+            new_index_scols.append(groupkey.spark.column.alias(index_column_name))
+            new_index_spark_column_names.append(index_column_name)
+            new_index_names.append(groupkey._column_label)
+            new_index_dtypes.append(groupkey.dtype)
+
+        for new_index_scol, index_name, index_dtype in zip(
+            kdf._internal.index_spark_columns, kdf._internal.index_names, kdf._internal.index_dtypes
+        ):
+            index_column_name = SPARK_INDEX_NAME_FORMAT(len(new_index_scols))
+            new_index_scols.append(new_index_scol.alias(index_column_name))
+            new_index_spark_column_names.append(index_column_name)
+            new_index_names.append(index_name)
+            new_index_dtypes.append(index_dtype)
+
+        if groupby._agg_columns_selected:
+            agg_columns = groupby._agg_columns
+        else:
+            agg_columns = [
+                kdf._kser_for(label)
+                for label in kdf._internal.column_labels
+                if label not in groupby._column_labels_to_exlcude
+            ]
+
+        applied = []
+        for agg_column in agg_columns:
+            applied.append(agg_column._with_new_scol(func(agg_column.spark.column)))  # TODO: dtype?
+
+        # Seems like pandas filters out when grouped key is NA.
+        cond = groupby._groupkeys[0].spark.column.isNotNull()
+        for c in groupby._groupkeys[1:]:
+            cond = cond | c.spark.column.isNotNull()
+
+        sdf = kdf._internal.spark_frame.filter(cond).select(
+            new_index_scols + [c.spark.column for c in applied]
+        )
+
+        internal = kdf._internal.copy(
+            spark_frame=sdf,
+            index_spark_columns=[scol_for(sdf, col) for col in new_index_spark_column_names],
+            index_names=new_index_names,
+            index_dtypes=new_index_dtypes,
+            column_labels=[c._column_label for c in applied],
+            data_spark_columns=[
+                scol_for(sdf, c._internal.data_spark_column_names[0]) for c in applied
+            ],
+            data_dtypes=[c.dtype for c in applied],
+        )
+
+        ret = DataFrame(internal)
+        if isinstance(groupby, SeriesGroupBy):
+            return first_series(ret)
+        else:
+            return ret
+
+    def count(self) -> Union["Series", "DataFrame"]:
+        """
+        The rolling count of any non-NaN observations inside the window.
+
+        Returns
+        -------
+        Series or DataFrame
+            Returned object type is determined by the caller of the expanding
+            calculation.
+
+        See Also
+        --------
+        Series.rolling : Calling object with Series data.
+        DataFrame.rolling : Calling object with DataFrames.
+        Series.count : Count of the full Series.
+        DataFrame.count : Count of the full DataFrame.
+
+        Examples
+        --------
+        >>> s = ps.Series([2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5])
+        >>> s.groupby(s).rolling(3).count().sort_index()
+        2  0     1.0
+           1     2.0
+        3  2     1.0
+           3     2.0
+           4     3.0
+        4  5     1.0
+           6     2.0
+           7     3.0
+           8     3.0
+        5  9     1.0
+           10    2.0
+        dtype: float64
+
+        For DataFrame, each rolling count is computed column-wise.
+
+        >>> df = ps.DataFrame({"A": s.to_numpy(), "B": s.to_numpy() ** 2})
+        >>> df.groupby(df.A).rolling(2).count().sort_index()  # doctest: +NORMALIZE_WHITESPACE
+                A    B
+        A
+        2 0   1.0  1.0
+          1   2.0  2.0
+        3 2   1.0  1.0
+          3   2.0  2.0
+          4   2.0  2.0
+        4 5   1.0  1.0
+          6   2.0  2.0
+          7   2.0  2.0
+          8   2.0  2.0
+        5 9   1.0  1.0
+          10  2.0  2.0
+        """
+        return super().count()
+
+    def sum(self) -> Union["Series", "DataFrame"]:
+        """
+        The rolling summation of any non-NaN observations inside the window.
+
+        Returns
+        -------
+        Series or DataFrame
+            Returned object type is determined by the caller of the rolling
+            calculation.
+
+        See Also
+        --------
+        Series.rolling : Calling object with Series data.
+        DataFrame.rolling : Calling object with DataFrames.
+        Series.sum : Sum of the full Series.
+        DataFrame.sum : Sum of the full DataFrame.
+
+        Examples
+        --------
+        >>> s = ps.Series([2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5])
+        >>> s.groupby(s).rolling(3).sum().sort_index()
+        2  0      NaN
+           1      NaN
+        3  2      NaN
+           3      NaN
+           4      9.0
+        4  5      NaN
+           6      NaN
+           7     12.0
+           8     12.0
+        5  9      NaN
+           10     NaN
+        dtype: float64
+
+        For DataFrame, each rolling summation is computed column-wise.
+
+        >>> df = ps.DataFrame({"A": s.to_numpy(), "B": s.to_numpy() ** 2})
+        >>> df.groupby(df.A).rolling(2).sum().sort_index()  # doctest: +NORMALIZE_WHITESPACE
+                 A     B
+        A
+        2 0    NaN   NaN
+          1    4.0   8.0
+        3 2    NaN   NaN
+          3    6.0  18.0
+          4    6.0  18.0
+        4 5    NaN   NaN
+          6    8.0  32.0
+          7    8.0  32.0
+          8    8.0  32.0
+        5 9    NaN   NaN
+          10  10.0  50.0
+        """
+        return super().sum()
+
+    def min(self) -> Union["Series", "DataFrame"]:
+        """
+        The rolling minimum of any non-NaN observations inside the window.
+
+        Returns
+        -------
+        Series or DataFrame
+            Returned object type is determined by the caller of the rolling
+            calculation.
+
+        See Also
+        --------
+        Series.rolling : Calling object with Series data.
+        DataFrame.rolling : Calling object with DataFrames.
+        Series.min : Min of the full Series.
+        DataFrame.min : Min of the full DataFrame.
+
+        Examples
+        --------
+        >>> s = ps.Series([2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5])
+        >>> s.groupby(s).rolling(3).min().sort_index()
+        2  0     NaN
+           1     NaN
+        3  2     NaN
+           3     NaN
+           4     3.0
+        4  5     NaN
+           6     NaN
+           7     4.0
+           8     4.0
+        5  9     NaN
+           10    NaN
+        dtype: float64
+
+        For DataFrame, each rolling minimum is computed column-wise.
+
+        >>> df = ps.DataFrame({"A": s.to_numpy(), "B": s.to_numpy() ** 2})
+        >>> df.groupby(df.A).rolling(2).min().sort_index()  # doctest: +NORMALIZE_WHITESPACE
+                A     B
+        A
+        2 0   NaN   NaN
+          1   2.0   4.0
+        3 2   NaN   NaN
+          3   3.0   9.0
+          4   3.0   9.0
+        4 5   NaN   NaN
+          6   4.0  16.0
+          7   4.0  16.0
+          8   4.0  16.0
+        5 9   NaN   NaN
+          10  5.0  25.0
+        """
+        return super().min()
+
+    def max(self) -> Union["Series", "DataFrame"]:
+        """
+        The rolling maximum of any non-NaN observations inside the window.
+
+        Returns
+        -------
+        Series or DataFrame
+            Returned object type is determined by the caller of the rolling
+            calculation.
+
+        See Also
+        --------
+        Series.rolling : Calling object with Series data.
+        DataFrame.rolling : Calling object with DataFrames.
+        Series.max : Max of the full Series.
+        DataFrame.max : Max of the full DataFrame.
+
+        Examples
+        --------
+        >>> s = ps.Series([2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5])
+        >>> s.groupby(s).rolling(3).max().sort_index()
+        2  0     NaN
+           1     NaN
+        3  2     NaN
+           3     NaN
+           4     3.0
+        4  5     NaN
+           6     NaN
+           7     4.0
+           8     4.0
+        5  9     NaN
+           10    NaN
+        dtype: float64
+
+        For DataFrame, each rolling maximum is computed column-wise.
+
+        >>> df = ps.DataFrame({"A": s.to_numpy(), "B": s.to_numpy() ** 2})
+        >>> df.groupby(df.A).rolling(2).max().sort_index()  # doctest: +NORMALIZE_WHITESPACE
+                A     B
+        A
+        2 0   NaN   NaN
+          1   2.0   4.0
+        3 2   NaN   NaN
+          3   3.0   9.0
+          4   3.0   9.0
+        4 5   NaN   NaN
+          6   4.0  16.0
+          7   4.0  16.0
+          8   4.0  16.0
+        5 9   NaN   NaN
+          10  5.0  25.0
+        """
+        return super().max()
+
+    def mean(self) -> Union["Series", "DataFrame"]:
+        """
+        The rolling mean of any non-NaN observations inside the window.
+
+        Returns
+        -------
+        Series or DataFrame
+            Returned object type is determined by the caller of the rolling
+            calculation.
+
+        See Also
+        --------
+        Series.rolling : Calling object with Series data.
+        DataFrame.rolling : Calling object with DataFrames.
+        Series.mean : Mean of the full Series.
+        DataFrame.mean : Mean of the full DataFrame.
+
+        Examples
+        --------
+        >>> s = ps.Series([2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5])
+        >>> s.groupby(s).rolling(3).mean().sort_index()
+        2  0     NaN
+           1     NaN
+        3  2     NaN
+           3     NaN
+           4     3.0
+        4  5     NaN
+           6     NaN
+           7     4.0
+           8     4.0
+        5  9     NaN
+           10    NaN
+        dtype: float64
+
+        For DataFrame, each rolling mean is computed column-wise.
+
+        >>> df = ps.DataFrame({"A": s.to_numpy(), "B": s.to_numpy() ** 2})
+        >>> df.groupby(df.A).rolling(2).mean().sort_index()  # doctest: +NORMALIZE_WHITESPACE
+                A     B
+        A
+        2 0   NaN   NaN
+          1   2.0   4.0
+        3 2   NaN   NaN
+          3   3.0   9.0
+          4   3.0   9.0
+        4 5   NaN   NaN
+          6   4.0  16.0
+          7   4.0  16.0
+          8   4.0  16.0
+        5 9   NaN   NaN
+          10  5.0  25.0
+        """
+        return super().mean()
+
+    def std(self) -> Union["Series", "DataFrame"]:
+        """
+        Calculate rolling standard deviation.
+
+        Returns
+        -------
+        Series or DataFrame
+            Returns the same object type as the caller of the rolling calculation.
+
+        See Also
+        --------
+        Series.rolling : Calling object with Series data.
+        DataFrame.rolling : Calling object with DataFrames.
+        Series.std : Equivalent method for Series.
+        DataFrame.std : Equivalent method for DataFrame.
+        numpy.std : Equivalent method for Numpy array.
+        """
+        return super().std()
+
+    def var(self) -> Union["Series", "DataFrame"]:
+        """
+        Calculate unbiased rolling variance.
+
+        Returns
+        -------
+        Series or DataFrame
+            Returns the same object type as the caller of the rolling calculation.
+
+        See Also
+        --------
+        Series.rolling : Calling object with Series data.
+        DataFrame.rolling : Calling object with DataFrames.
+        Series.var : Equivalent method for Series.
+        DataFrame.var : Equivalent method for DataFrame.
+        numpy.var : Equivalent method for Numpy array.
+        """
+        return super().var()
+
+
+class Expanding(RollingAndExpanding):
+    def __init__(self, kdf_or_kser, min_periods=1):
+        from pyspark.pandas import DataFrame, Series
+
+        if min_periods < 0:
+            raise ValueError("min_periods must be >= 0")
+
+        if not isinstance(kdf_or_kser, (DataFrame, Series)):
+            raise TypeError(
+                "kdf_or_kser must be a series or dataframe; however, got: %s" % type(kdf_or_kser)
+            )
+
+        window = Window.orderBy(NATURAL_ORDER_COLUMN_NAME).rowsBetween(
+            Window.unboundedPreceding, Window.currentRow
+        )
+
+        super().__init__(kdf_or_kser, window, min_periods)
+
+    def __getattr__(self, item: str) -> Any:
+        if hasattr(MissingPandasLikeExpanding, item):
+            property_or_func = getattr(MissingPandasLikeExpanding, item)
+            if isinstance(property_or_func, property):
+                return property_or_func.fget(self)  # type: ignore
+            else:
+                return partial(property_or_func, self)
+        raise AttributeError(item)
+
+    # TODO: when add 'center' and 'axis' parameter, should add to here too.
+    def __repr__(self):
+        return "Expanding [min_periods={}]".format(self._min_periods)
+
+    _apply_as_series_or_frame = Rolling._apply_as_series_or_frame
+
+    def count(self) -> Union["Series", "DataFrame"]:
+        """
+        The expanding count of any non-NaN observations inside the window.
+
+        .. note:: the current implementation of this API uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Returns
+        -------
+        Series or DataFrame
+            Returned object type is determined by the caller of the expanding
+            calculation.
+
+        See Also
+        --------
+        Series.expanding : Calling object with Series data.
+        DataFrame.expanding : Calling object with DataFrames.
+        Series.count : Count of the full Series.
+        DataFrame.count : Count of the full DataFrame.
+
+        Examples
+        --------
+        >>> s = ps.Series([2, 3, float("nan"), 10])
+        >>> s.expanding().count()
+        0    1.0
+        1    2.0
+        2    2.0
+        3    3.0
+        dtype: float64
+
+        >>> s.to_frame().expanding().count()
+             0
+        0  1.0
+        1  2.0
+        2  2.0
+        3  3.0
+        """
+
+        def count(scol):
+            return F.when(
+                F.row_number().over(self._unbounded_window) >= self._min_periods,
+                F.count(scol).over(self._window),
+            ).otherwise(F.lit(None))
+
+        return self._apply_as_series_or_frame(count).astype("float64")  # type: ignore
+
+    def sum(self) -> Union["Series", "DataFrame"]:
+        """
+        Calculate expanding summation of given DataFrame or Series.
+
+        .. note:: the current implementation of this API uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Returns
+        -------
+        Series or DataFrame
+            Same type as the input, with the same index, containing the
+            expanding summation.
+
+        See Also
+        --------
+        Series.expanding : Calling object with Series data.
+        DataFrame.expanding : Calling object with DataFrames.
+        Series.sum : Reducing sum for Series.
+        DataFrame.sum : Reducing sum for DataFrame.
+
+        Examples
+        --------
+        >>> s = ps.Series([1, 2, 3, 4, 5])
+        >>> s
+        0    1
+        1    2
+        2    3
+        3    4
+        4    5
+        dtype: int64
+
+        >>> s.expanding(3).sum()
+        0     NaN
+        1     NaN
+        2     6.0
+        3    10.0
+        4    15.0
+        dtype: float64
+
+        For DataFrame, each expanding summation is computed column-wise.
+
+        >>> df = ps.DataFrame({"A": s.to_numpy(), "B": s.to_numpy() ** 2})
+        >>> df
+           A   B
+        0  1   1
+        1  2   4
+        2  3   9
+        3  4  16
+        4  5  25
+
+        >>> df.expanding(3).sum()
+              A     B
+        0   NaN   NaN
+        1   NaN   NaN
+        2   6.0  14.0
+        3  10.0  30.0
+        4  15.0  55.0
+        """
+        return super().sum()
+
+    def min(self) -> Union["Series", "DataFrame"]:
+        """
+        Calculate the expanding minimum.
+
+        .. note:: the current implementation of this API uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Returns
+        -------
+        Series or DataFrame
+            Returned object type is determined by the caller of the expanding
+            calculation.
+
+        See Also
+        --------
+        Series.expanding : Calling object with a Series.
+        DataFrame.expanding : Calling object with a DataFrame.
+        Series.min : Similar method for Series.
+        DataFrame.min : Similar method for DataFrame.
+
+        Examples
+        --------
+        Performing a expanding minimum with a window size of 3.
+
+        >>> s = ps.Series([4, 3, 5, 2, 6])
+        >>> s.expanding(3).min()
+        0    NaN
+        1    NaN
+        2    3.0
+        3    2.0
+        4    2.0
+        dtype: float64
+        """
+        return super().min()
+
+    def max(self) -> Union["Series", "DataFrame"]:
+        """
+        Calculate the expanding maximum.
+
+        .. note:: the current implementation of this API uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Returns
+        -------
+        Series or DataFrame
+            Return type is determined by the caller.
+
+        See Also
+        --------
+        Series.expanding : Calling object with Series data.
+        DataFrame.expanding : Calling object with DataFrames.
+        Series.max : Similar method for Series.
+        DataFrame.max : Similar method for DataFrame.
+
+        Examples
+        --------
+        Performing a expanding minimum with a window size of 3.
+
+        >>> s = ps.Series([4, 3, 5, 2, 6])
+        >>> s.expanding(3).max()
+        0    NaN
+        1    NaN
+        2    5.0
+        3    5.0
+        4    6.0
+        dtype: float64
+        """
+        return super().max()
+
+    def mean(self) -> Union["Series", "DataFrame"]:
+        """
+        Calculate the expanding mean of the values.
+
+        .. note:: the current implementation of this API uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Returns
+        -------
+        Series or DataFrame
+            Returned object type is determined by the caller of the expanding
+            calculation.
+
+        See Also
+        --------
+        Series.expanding : Calling object with Series data.
+        DataFrame.expanding : Calling object with DataFrames.
+        Series.mean : Equivalent method for Series.
+        DataFrame.mean : Equivalent method for DataFrame.
+
+        Examples
+        --------
+        The below examples will show expanding mean calculations with window sizes of
+        two and three, respectively.
+
+        >>> s = ps.Series([1, 2, 3, 4])
+        >>> s.expanding(2).mean()
+        0    NaN
+        1    1.5
+        2    2.0
+        3    2.5
+        dtype: float64
+
+        >>> s.expanding(3).mean()
+        0    NaN
+        1    NaN
+        2    2.0
+        3    2.5
+        dtype: float64
+        """
+        return super().mean()
+
+    def std(self) -> Union["Series", "DataFrame"]:
+        """
+        Calculate expanding standard deviation.
+
+        .. note:: the current implementation of this API uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Returns
+        -------
+        Series or DataFrame
+            Returns the same object type as the caller of the expanding calculation.
+
+        See Also
+        --------
+        Series.expanding : Calling object with Series data.
+        DataFrame.expanding : Calling object with DataFrames.
+        Series.std : Equivalent method for Series.
+        DataFrame.std : Equivalent method for DataFrame.
+        numpy.std : Equivalent method for Numpy array.
+
+        Examples
+        --------
+        >>> s = ps.Series([5, 5, 6, 7, 5, 5, 5])
+        >>> s.expanding(3).std()
+        0         NaN
+        1         NaN
+        2    0.577350
+        3    0.957427
+        4    0.894427
+        5    0.836660
+        6    0.786796
+        dtype: float64
+
+        For DataFrame, each expanding standard deviation variance is computed column-wise.
+
+        >>> df = ps.DataFrame({"A": s.to_numpy(), "B": s.to_numpy() ** 2})
+        >>> df.expanding(2).std()
+                  A          B
+        0       NaN        NaN
+        1  0.000000   0.000000
+        2  0.577350   6.350853
+        3  0.957427  11.412712
+        4  0.894427  10.630146
+        5  0.836660   9.928075
+        6  0.786796   9.327379
+        """
+        return super().std()
+
+    def var(self) -> Union["Series", "DataFrame"]:
+        """
+        Calculate unbiased expanding variance.
+
+        .. note:: the current implementation of this API uses Spark's Window without
+            specifying partition specification. This leads to move all data into
+            single partition in single machine and could cause serious
+            performance degradation. Avoid this method against very large dataset.
+
+        Returns
+        -------
+        Series or DataFrame
+            Returns the same object type as the caller of the expanding calculation.
+
+        See Also
+        --------
+        Series.expanding : Calling object with Series data.
+        DataFrame.expanding : Calling object with DataFrames.
+        Series.var : Equivalent method for Series.
+        DataFrame.var : Equivalent method for DataFrame.
+        numpy.var : Equivalent method for Numpy array.
+
+        Examples
+        --------
+        >>> s = ps.Series([5, 5, 6, 7, 5, 5, 5])
+        >>> s.expanding(3).var()
+        0         NaN
+        1         NaN
+        2    0.333333
+        3    0.916667
+        4    0.800000
+        5    0.700000
+        6    0.619048
+        dtype: float64
+
+        For DataFrame, each unbiased expanding variance is computed column-wise.
+
+        >>> df = ps.DataFrame({"A": s.to_numpy(), "B": s.to_numpy() ** 2})
+        >>> df.expanding(2).var()
+                  A           B
+        0       NaN         NaN
+        1  0.000000    0.000000
+        2  0.333333   40.333333
+        3  0.916667  130.250000
+        4  0.800000  113.000000
+        5  0.700000   98.566667
+        6  0.619048   87.000000
+        """
+        return super().var()
+
+
+class ExpandingGroupby(Expanding):
+    def __init__(self, groupby, min_periods=1):
+        from pyspark.pandas.groupby import SeriesGroupBy
+        from pyspark.pandas.groupby import DataFrameGroupBy
+
+        if isinstance(groupby, SeriesGroupBy):
+            kdf_or_kser = groupby._kser
+        elif isinstance(groupby, DataFrameGroupBy):
+            kdf_or_kser = groupby._kdf
+        else:
+            raise TypeError(
+                "groupby must be a SeriesGroupBy or DataFrameGroupBy; "
+                "however, got: %s" % type(groupby)
+            )
+
+        super().__init__(kdf_or_kser, min_periods)
+
+        self._groupby = groupby
+        self._window = self._window.partitionBy(*[ser.spark.column for ser in groupby._groupkeys])
+        self._unbounded_window = self._window.partitionBy(
+            *[ser.spark.column for ser in groupby._groupkeys]
+        )
+
+    def __getattr__(self, item: str) -> Any:
+        if hasattr(MissingPandasLikeExpandingGroupby, item):
+            property_or_func = getattr(MissingPandasLikeExpandingGroupby, item)
+            if isinstance(property_or_func, property):
+                return property_or_func.fget(self)  # type: ignore
+            else:
+                return partial(property_or_func, self)
+        raise AttributeError(item)
+
+    _apply_as_series_or_frame = RollingGroupby._apply_as_series_or_frame  # type: ignore
+
+    def count(self) -> Union["Series", "DataFrame"]:
+        """
+        The expanding count of any non-NaN observations inside the window.
+
+        Returns
+        -------
+        Series or DataFrame
+            Returned object type is determined by the caller of the expanding
+            calculation.
+
+        See Also
+        --------
+        Series.expanding : Calling object with Series data.
+        DataFrame.expanding : Calling object with DataFrames.
+        Series.count : Count of the full Series.
+        DataFrame.count : Count of the full DataFrame.
+
+        Examples
+        --------
+        >>> s = ps.Series([2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5])
+        >>> s.groupby(s).expanding(3).count().sort_index()
+        2  0     NaN
+           1     NaN
+        3  2     NaN
+           3     NaN
+           4     3.0
+        4  5     NaN
+           6     NaN
+           7     3.0
+           8     4.0
+        5  9     NaN
+           10    NaN
+        dtype: float64
+
+        For DataFrame, each expanding count is computed column-wise.
+
+        >>> df = ps.DataFrame({"A": s.to_numpy(), "B": s.to_numpy() ** 2})
+        >>> df.groupby(df.A).expanding(2).count().sort_index()  # doctest: +NORMALIZE_WHITESPACE
+                A    B
+        A
+        2 0   NaN  NaN
+          1   2.0  2.0
+        3 2   NaN  NaN
+          3   2.0  2.0
+          4   3.0  3.0
+        4 5   NaN  NaN
+          6   2.0  2.0
+          7   3.0  3.0
+          8   4.0  4.0
+        5 9   NaN  NaN
+          10  2.0  2.0
+        """
+        return super().count()
+
+    def sum(self) -> Union["Series", "DataFrame"]:
+        """
+        Calculate expanding summation of given DataFrame or Series.
+
+        Returns
+        -------
+        Series or DataFrame
+            Same type as the input, with the same index, containing the
+            expanding summation.
+
+        See Also
+        --------
+        Series.expanding : Calling object with Series data.
+        DataFrame.expanding : Calling object with DataFrames.
+        Series.sum : Reducing sum for Series.
+        DataFrame.sum : Reducing sum for DataFrame.
+
+        Examples
+        --------
+        >>> s = ps.Series([2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5])
+        >>> s.groupby(s).expanding(3).sum().sort_index()
+        2  0      NaN
+           1      NaN
+        3  2      NaN
+           3      NaN
+           4      9.0
+        4  5      NaN
+           6      NaN
+           7     12.0
+           8     16.0
+        5  9      NaN
+           10     NaN
+        dtype: float64
+
+        For DataFrame, each expanding summation is computed column-wise.
+
+        >>> df = ps.DataFrame({"A": s.to_numpy(), "B": s.to_numpy() ** 2})
+        >>> df.groupby(df.A).expanding(2).sum().sort_index()  # doctest: +NORMALIZE_WHITESPACE
+                 A     B
+        A
+        2 0    NaN   NaN
+          1    4.0   8.0
+        3 2    NaN   NaN
+          3    6.0  18.0
+          4    9.0  27.0
+        4 5    NaN   NaN
+          6    8.0  32.0
+          7   12.0  48.0
+          8   16.0  64.0
+        5 9    NaN   NaN
+          10  10.0  50.0
+        """
+        return super().sum()
+
+    def min(self) -> Union["Series", "DataFrame"]:
+        """
+        Calculate the expanding minimum.
+
+        Returns
+        -------
+        Series or DataFrame
+            Returned object type is determined by the caller of the expanding
+            calculation.
+
+        See Also
+        --------
+        Series.expanding : Calling object with a Series.
+        DataFrame.expanding : Calling object with a DataFrame.
+        Series.min : Similar method for Series.
+        DataFrame.min : Similar method for DataFrame.
+
+        Examples
+        --------
+        >>> s = ps.Series([2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5])
+        >>> s.groupby(s).expanding(3).min().sort_index()
+        2  0     NaN
+           1     NaN
+        3  2     NaN
+           3     NaN
+           4     3.0
+        4  5     NaN
+           6     NaN
+           7     4.0
+           8     4.0
+        5  9     NaN
+           10    NaN
+        dtype: float64
+
+        For DataFrame, each expanding minimum is computed column-wise.
+
+        >>> df = ps.DataFrame({"A": s.to_numpy(), "B": s.to_numpy() ** 2})
+        >>> df.groupby(df.A).expanding(2).min().sort_index()  # doctest: +NORMALIZE_WHITESPACE
+                A     B
+        A
+        2 0   NaN   NaN
+          1   2.0   4.0
+        3 2   NaN   NaN
+          3   3.0   9.0
+          4   3.0   9.0
+        4 5   NaN   NaN
+          6   4.0  16.0
+          7   4.0  16.0
+          8   4.0  16.0
+        5 9   NaN   NaN
+          10  5.0  25.0
+        """
+        return super().min()
+
+    def max(self) -> Union["Series", "DataFrame"]:
+        """
+        Calculate the expanding maximum.
+
+        Returns
+        -------
+        Series or DataFrame
+            Return type is determined by the caller.
+
+        See Also
+        --------
+        Series.expanding : Calling object with Series data.
+        DataFrame.expanding : Calling object with DataFrames.
+        Series.max : Similar method for Series.
+        DataFrame.max : Similar method for DataFrame.
+
+        Examples
+        --------
+        >>> s = ps.Series([2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5])
+        >>> s.groupby(s).expanding(3).max().sort_index()
+        2  0     NaN
+           1     NaN
+        3  2     NaN
+           3     NaN
+           4     3.0
+        4  5     NaN
+           6     NaN
+           7     4.0
+           8     4.0
+        5  9     NaN
+           10    NaN
+        dtype: float64
+
+        For DataFrame, each expanding maximum is computed column-wise.
+
+        >>> df = ps.DataFrame({"A": s.to_numpy(), "B": s.to_numpy() ** 2})
+        >>> df.groupby(df.A).expanding(2).max().sort_index()  # doctest: +NORMALIZE_WHITESPACE
+                A     B
+        A
+        2 0   NaN   NaN
+          1   2.0   4.0
+        3 2   NaN   NaN
+          3   3.0   9.0
+          4   3.0   9.0
+        4 5   NaN   NaN
+          6   4.0  16.0
+          7   4.0  16.0
+          8   4.0  16.0
+        5 9   NaN   NaN
+          10  5.0  25.0
+        """
+        return super().max()
+
+    def mean(self) -> Union["Series", "DataFrame"]:
+        """
+        Calculate the expanding mean of the values.
+
+        Returns
+        -------
+        Series or DataFrame
+            Returned object type is determined by the caller of the expanding
+            calculation.
+
+        See Also
+        --------
+        Series.expanding : Calling object with Series data.
+        DataFrame.expanding : Calling object with DataFrames.
+        Series.mean : Equivalent method for Series.
+        DataFrame.mean : Equivalent method for DataFrame.
+
+        Examples
+        --------
+        >>> s = ps.Series([2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5])
+        >>> s.groupby(s).expanding(3).mean().sort_index()
+        2  0     NaN
+           1     NaN
+        3  2     NaN
+           3     NaN
+           4     3.0
+        4  5     NaN
+           6     NaN
+           7     4.0
+           8     4.0
+        5  9     NaN
+           10    NaN
+        dtype: float64
+
+        For DataFrame, each expanding mean is computed column-wise.
+
+        >>> df = ps.DataFrame({"A": s.to_numpy(), "B": s.to_numpy() ** 2})
+        >>> df.groupby(df.A).expanding(2).mean().sort_index()  # doctest: +NORMALIZE_WHITESPACE
+                A     B
+        A
+        2 0   NaN   NaN
+          1   2.0   4.0
+        3 2   NaN   NaN
+          3   3.0   9.0
+          4   3.0   9.0
+        4 5   NaN   NaN
+          6   4.0  16.0
+          7   4.0  16.0
+          8   4.0  16.0
+        5 9   NaN   NaN
+          10  5.0  25.0
+        """
+        return super().mean()
+
+    def std(self) -> Union["Series", "DataFrame"]:
+        """
+        Calculate expanding standard deviation.
+
+
+        Returns
+        -------
+        Series or DataFrame
+            Returns the same object type as the caller of the expanding calculation.
+
+        See Also
+        --------
+        Series.expanding: Calling object with Series data.
+        DataFrame.expanding : Calling object with DataFrames.
+        Series.std : Equivalent method for Series.
+        DataFrame.std : Equivalent method for DataFrame.
+        numpy.std : Equivalent method for Numpy array.
+        """
+        return super().std()
+
+    def var(self) -> Union["Series", "DataFrame"]:
+        """
+        Calculate unbiased expanding variance.
+
+        Returns
+        -------
+        Series or DataFrame
+            Returns the same object type as the caller of the expanding calculation.
+
+        See Also
+        --------
+        Series.expanding : Calling object with Series data.
+        DataFrame.expanding : Calling object with DataFrames.
+        Series.var : Equivalent method for Series.
+        DataFrame.var : Equivalent method for DataFrame.
+        numpy.var : Equivalent method for Numpy array.
+        """
+        return super().var()
+
+
+def _test():
+    import os
+    import doctest
+    import sys
+    from pyspark.sql import SparkSession
+    import pyspark.pandas.window
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    globs = pyspark.pandas.window.__dict__.copy()
+    globs["ps"] = pyspark.pandas
+    spark = (
+        SparkSession.builder.master("local[4]").appName("pyspark.pandas.window tests").getOrCreate()
+    )
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.pandas.window,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+    )
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/profiler.py b/python/pyspark/profiler.py
index b9423b7604873..99cfe71fda2b0 100644
--- a/python/pyspark/profiler.py
+++ b/python/pyspark/profiler.py
@@ -67,8 +67,6 @@ def show_profiles(self):
 
 class Profiler(object):
     """
-    .. note:: DeveloperApi
-
     PySpark supports custom profilers, this is to allow for different profilers to
     be used as well as outputting to different formats than what is provided in the
     BasicProfiler.
@@ -81,6 +79,8 @@ class Profiler(object):
 
     The profiler class is chosen when creating a SparkContext
 
+    Examples
+    --------
     >>> from pyspark import SparkConf, SparkContext
     >>> from pyspark import BasicProfiler
     >>> class MyCustomProfiler(BasicProfiler):
@@ -97,6 +97,10 @@ class Profiler(object):
     My custom profiles for RDD:1
     My custom profiles for RDD:3
     >>> sc.stop()
+
+    Notes
+    -----
+    This API is a developer API.
     """
 
     def __init__(self, ctx):
diff --git a/python/pyspark/profiler.pyi b/python/pyspark/profiler.pyi
new file mode 100644
index 0000000000000..7276da529fa17
--- /dev/null
+++ b/python/pyspark/profiler.pyi
@@ -0,0 +1,56 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Any, Callable, List, Optional, Tuple, Type
+
+import pstats
+
+from pyspark.accumulators import AccumulatorParam
+from pyspark.context import SparkContext
+
+class ProfilerCollector:
+    profiler_cls: Type[Profiler]
+    profile_dump_path: Optional[str]
+    profilers: List[Tuple[int, Profiler, bool]]
+    def __init__(
+        self, profiler_cls: Type[Profiler], dump_path: Optional[str] = ...
+    ) -> None: ...
+    def new_profiler(self, ctx: SparkContext) -> Profiler: ...
+    def add_profiler(self, id: int, profiler: Profiler) -> None: ...
+    def dump_profiles(self, path: str) -> None: ...
+    def show_profiles(self) -> None: ...
+
+class Profiler:
+    def __init__(self, ctx: SparkContext) -> None: ...
+    def profile(self, func: Callable[[], Any]) -> None: ...
+    def stats(self) -> pstats.Stats: ...
+    def show(self, id: int) -> None: ...
+    def dump(self, id: int, path: str) -> None: ...
+
+class PStatsParam(AccumulatorParam):
+    @staticmethod
+    def zero(value: pstats.Stats) -> None: ...
+    @staticmethod
+    def addInPlace(
+        value1: Optional[pstats.Stats], value2: Optional[pstats.Stats]
+    ) -> Optional[pstats.Stats]: ...
+
+class BasicProfiler(Profiler):
+    def __init__(self, ctx: SparkContext) -> None: ...
+    def profile(self, func: Callable[[], Any]) -> None: ...
+    def stats(self) -> pstats.Stats: ...
diff --git a/python/pyspark/py.typed b/python/pyspark/py.typed
new file mode 100644
index 0000000000000..b648ac9233330
--- /dev/null
+++ b/python/pyspark/py.typed
@@ -0,0 +1 @@
+partial
diff --git a/python/pyspark/rdd.py b/python/pyspark/rdd.py
index d0ac000ba3208..3fed2bc7d3247 100644
--- a/python/pyspark/rdd.py
+++ b/python/pyspark/rdd.py
@@ -18,7 +18,6 @@
 import copy
 import sys
 import os
-import re
 import operator
 import shlex
 import warnings
@@ -26,37 +25,29 @@
 import bisect
 import random
 from subprocess import Popen, PIPE
-from tempfile import NamedTemporaryFile
 from threading import Thread
 from collections import defaultdict
 from itertools import chain
 from functools import reduce
 from math import sqrt, log, isinf, isnan, pow, ceil
 
-if sys.version > '3':
-    basestring = unicode = str
-else:
-    from itertools import imap as map, ifilter as filter
-
 from pyspark.java_gateway import local_connect_and_auth
 from pyspark.serializers import AutoBatchedSerializer, BatchedSerializer, NoOpSerializer, \
     CartesianDeserializer, CloudPickleSerializer, PairDeserializer, PickleSerializer, \
-    UTF8Deserializer, pack_long, read_int, write_int
+    pack_long, read_int, write_int
 from pyspark.join import python_join, python_left_outer_join, \
     python_right_outer_join, python_full_outer_join, python_cogroup
 from pyspark.statcounter import StatCounter
 from pyspark.rddsampler import RDDSampler, RDDRangeSampler, RDDStratifiedSampler
 from pyspark.storagelevel import StorageLevel
-from pyspark.resource.executorrequests import ExecutorResourceRequests
-from pyspark.resource.resourceprofile import ResourceProfile
-from pyspark.resource.taskrequests import TaskResourceRequests
+from pyspark.resource.requests import ExecutorResourceRequests, TaskResourceRequests
+from pyspark.resource.profile import ResourceProfile
 from pyspark.resultiterable import ResultIterable
 from pyspark.shuffle import Aggregator, ExternalMerger, \
     get_used_memory, ExternalSorter, ExternalGroupBy
 from pyspark.traceback_utils import SCCallSiteSync
 from pyspark.util import fail_on_stopiteration, _parse_memory
 
-
 __all__ = ["RDD"]
 
 
@@ -88,13 +79,15 @@ def portable_hash(x):
 
     The algorithm is similar to that one used by CPython 2.7
 
+    Examples
+    --------
     >>> portable_hash(None)
     0
     >>> portable_hash((None, 1)) & 0xffffffff
     219750521
     """
 
-    if sys.version_info >= (3, 2, 3) and 'PYTHONHASHSEED' not in os.environ:
+    if 'PYTHONHASHSEED' not in os.environ:
         raise Exception("Randomness of hash of string should be disabled via PYTHONHASHSEED")
 
     if x is None:
@@ -117,6 +110,8 @@ class BoundedFloat(float):
     Bounded value is generated by approximate job, with confidence and low
     bound and high bound.
 
+    Examples
+    --------
     >>> BoundedFloat(100.0, 0.95, 95.0, 105.0)
     100.0
     """
@@ -132,8 +127,14 @@ def _create_local_socket(sock_info):
     """
     Create a local socket that can be used to load deserialized data from the JVM
 
-    :param sock_info: Tuple containing port number and authentication secret for a local socket.
-    :return: sockfile file descriptor of the local socket
+    Parameters
+    ----------
+    sock_info : tuple
+        Tuple containing port number and authentication secret for a local socket.
+
+    Returns
+    -------
+    sockfile file descriptor of the local socket
     """
     port = sock_info[0]
     auth_secret = sock_info[1]
@@ -148,9 +149,17 @@ def _load_from_socket(sock_info, serializer):
     """
     Connect to a local socket described by sock_info and use the given serializer to yield data
 
-    :param sock_info: Tuple containing port number and authentication secret for a local socket.
-    :param serializer: The PySpark serializer to use
-    :return: result of Serializer.load_stream, usually a generator that yields deserialized data
+    Parameters
+    ----------
+    sock_info : tuple
+        Tuple containing port number and authentication secret for a local socket.
+    serializer : :py:class:`Serializer`
+        The PySpark serializer to use
+
+    Returns
+    -------
+    result of :py:meth:`Serializer.load_stream`,
+    usually a generator that yields deserialized data
     """
     sockfile = _create_local_socket(sock_info)
     # The socket will be automatically closed when garbage-collected.
@@ -205,19 +214,6 @@ def __del__(self):
     return iter(PyLocalIterable(sock_info, serializer))
 
 
-def ignore_unicode_prefix(f):
-    """
-    Ignore the 'u' prefix of string in doc tests, to make it works
-    in both python 2 and 3
-    """
-    if sys.version >= '3':
-        # the representation of unicode string in Python 3 does not have prefix 'u',
-        # so remove the prefix 'u' for doc tests
-        literal_re = re.compile(r"(\W|^)[uU](['])", re.UNICODE)
-        f.__doc__ = literal_re.sub(r'\1\2', f.__doc__)
-    return f
-
-
 class Partitioner(object):
     def __init__(self, numPartitions, partitionFunc):
         self.numPartitions = numPartitions
@@ -294,6 +290,8 @@ def persist(self, storageLevel=StorageLevel.MEMORY_ONLY):
         a new storage level if the RDD does not have a storage level set yet.
         If no storage level is specified defaults to (`MEMORY_ONLY`).
 
+        Examples
+        --------
         >>> rdd = sc.parallelize(["b", "a", "c"])
         >>> rdd.persist().is_cached
         True
@@ -377,6 +375,8 @@ def map(self, f, preservesPartitioning=False):
         """
         Return a new RDD by applying a function to each element of this RDD.
 
+        Examples
+        --------
         >>> rdd = sc.parallelize(["b", "a", "c"])
         >>> sorted(rdd.map(lambda x: (x, 1)).collect())
         [('a', 1), ('b', 1), ('c', 1)]
@@ -390,6 +390,8 @@ def flatMap(self, f, preservesPartitioning=False):
         Return a new RDD by first applying a function to all elements of this
         RDD, and then flattening the results.
 
+        Examples
+        --------
         >>> rdd = sc.parallelize([2, 3, 4])
         >>> sorted(rdd.flatMap(lambda x: range(1, x)).collect())
         [1, 1, 1, 2, 2, 3]
@@ -404,6 +406,8 @@ def mapPartitions(self, f, preservesPartitioning=False):
         """
         Return a new RDD by applying a function to each partition of this RDD.
 
+        Examples
+        --------
         >>> rdd = sc.parallelize([1, 2, 3, 4], 2)
         >>> def f(iterator): yield sum(iterator)
         >>> rdd.mapPartitions(f).collect()
@@ -418,6 +422,8 @@ def mapPartitionsWithIndex(self, f, preservesPartitioning=False):
         Return a new RDD by applying a function to each partition of this RDD,
         while tracking the index of the original partition.
 
+        Examples
+        --------
         >>> rdd = sc.parallelize([1, 2, 3, 4], 4)
         >>> def f(splitIndex, iterator): yield splitIndex
         >>> rdd.mapPartitionsWithIndex(f).sum()
@@ -427,24 +433,32 @@ def mapPartitionsWithIndex(self, f, preservesPartitioning=False):
 
     def mapPartitionsWithSplit(self, f, preservesPartitioning=False):
         """
-        Deprecated: use mapPartitionsWithIndex instead.
 
         Return a new RDD by applying a function to each partition of this RDD,
         while tracking the index of the original partition.
 
+        .. deprecated:: 0.9.0
+            use :py:meth:`RDD.mapPartitionsWithIndex` instead.
+
+        Examples
+        --------
         >>> rdd = sc.parallelize([1, 2, 3, 4], 4)
         >>> def f(splitIndex, iterator): yield splitIndex
         >>> rdd.mapPartitionsWithSplit(f).sum()
         6
         """
-        warnings.warn("mapPartitionsWithSplit is deprecated; "
-                      "use mapPartitionsWithIndex instead", DeprecationWarning, stacklevel=2)
+        warnings.warn(
+            "mapPartitionsWithSplit is deprecated; use mapPartitionsWithIndex instead",
+            FutureWarning, stacklevel=2
+        )
         return self.mapPartitionsWithIndex(f, preservesPartitioning)
 
     def getNumPartitions(self):
         """
         Returns the number of partitions in RDD
 
+        Examples
+        --------
         >>> rdd = sc.parallelize([1, 2, 3, 4], 2)
         >>> rdd.getNumPartitions()
         2
@@ -455,6 +469,8 @@ def filter(self, f):
         """
         Return a new RDD containing only the elements that satisfy a predicate.
 
+        Examples
+        --------
         >>> rdd = sc.parallelize([1, 2, 3, 4, 5])
         >>> rdd.filter(lambda x: x % 2 == 0).collect()
         [2, 4]
@@ -467,6 +483,8 @@ def distinct(self, numPartitions=None):
         """
         Return a new RDD containing the distinct elements in this RDD.
 
+        Examples
+        --------
         >>> sorted(sc.parallelize([1, 1, 2, 3]).distinct().collect())
         [1, 2, 3]
         """
@@ -478,15 +496,24 @@ def sample(self, withReplacement, fraction, seed=None):
         """
         Return a sampled subset of this RDD.
 
-        :param withReplacement: can elements be sampled multiple times (replaced when sampled out)
-        :param fraction: expected size of the sample as a fraction of this RDD's size
+        Parameters
+        ----------
+        withReplacement : bool
+            can elements be sampled multiple times (replaced when sampled out)
+        fraction : float
+            expected size of the sample as a fraction of this RDD's size
             without replacement: probability that each element is chosen; fraction must be [0, 1]
             with replacement: expected number of times each element is chosen; fraction must be >= 0
-        :param seed: seed for the random number generator
+        seed : int, optional
+            seed for the random number generator
 
-        .. note:: This is not guaranteed to provide exactly the fraction specified of the total
-            count of the given :class:`DataFrame`.
+        Notes
+        -----
+        This is not guaranteed to provide exactly the fraction specified of the total
+        count of the given :class:`DataFrame`.
 
+        Examples
+        --------
         >>> rdd = sc.parallelize(range(100), 4)
         >>> 6 <= rdd.sample(False, 0.1, 81).count() <= 14
         True
@@ -498,10 +525,18 @@ def randomSplit(self, weights, seed=None):
         """
         Randomly splits this RDD with the provided weights.
 
-        :param weights: weights for splits, will be normalized if they don't sum to 1
-        :param seed: random seed
-        :return: split RDDs in a list
+        weights : list
+            weights for splits, will be normalized if they don't sum to 1
+        seed : int, optional
+            random seed
 
+        Returns
+        -------
+        list
+            split RDDs in a list
+
+        Examples
+        --------
         >>> rdd = sc.parallelize(range(500), 1)
         >>> rdd1, rdd2 = rdd.randomSplit([2, 3], 17)
         >>> len(rdd1.collect() + rdd2.collect())
@@ -525,9 +560,13 @@ def takeSample(self, withReplacement, num, seed=None):
         """
         Return a fixed-size sampled subset of this RDD.
 
-        .. note:: This method should only be used if the resulting array is expected
-            to be small, as all the data is loaded into the driver's memory.
+        Notes
+        -----
+        This method should only be used if the resulting array is expected
+        to be small, as all the data is loaded into the driver's memory.
 
+        Examples
+        --------
         >>> rdd = sc.parallelize(range(0, 10))
         >>> len(rdd.takeSample(True, 20, 1))
         20
@@ -613,6 +652,8 @@ def union(self, other):
         """
         Return the union of this RDD and another one.
 
+        Examples
+        --------
         >>> rdd = sc.parallelize([1, 1, 2, 3])
         >>> rdd.union(rdd).collect()
         [1, 1, 2, 3, 1, 1, 2, 3]
@@ -637,8 +678,12 @@ def intersection(self, other):
         Return the intersection of this RDD and another one. The output will
         not contain any duplicate elements, even if the input RDDs did.
 
-        .. note:: This method performs a shuffle internally.
+        Notes
+        -----
+        This method performs a shuffle internally.
 
+        Examples
+        --------
         >>> rdd1 = sc.parallelize([1, 10, 2, 3, 4, 5])
         >>> rdd2 = sc.parallelize([1, 6, 2, 3, 7, 8])
         >>> rdd1.intersection(rdd2).collect()
@@ -660,6 +705,8 @@ def __add__(self, other):
         """
         Return the union of this RDD and another one.
 
+        Examples
+        --------
         >>> rdd = sc.parallelize([1, 1, 2, 3])
         >>> (rdd + rdd).collect()
         [1, 1, 2, 3, 1, 1, 2, 3]
@@ -674,6 +721,8 @@ def repartitionAndSortWithinPartitions(self, numPartitions=None, partitionFunc=p
         Repartition the RDD according to the given partitioner and, within each resulting partition,
         sort records by their keys.
 
+        Examples
+        --------
         >>> rdd = sc.parallelize([(0, 5), (3, 8), (2, 6), (0, 8), (3, 8), (1, 3)])
         >>> rdd2 = rdd.repartitionAndSortWithinPartitions(2, lambda x: x % 2, True)
         >>> rdd2.glom().collect()
@@ -695,6 +744,8 @@ def sortByKey(self, ascending=True, numPartitions=None, keyfunc=lambda x: x):
         """
         Sorts this RDD, which is assumed to consist of (key, value) pairs.
 
+        Examples
+        --------
         >>> tmp = [('a', 1), ('b', 2), ('1', 3), ('d', 4), ('2', 5)]
         >>> sc.parallelize(tmp).sortByKey().first()
         ('1', 3)
@@ -751,6 +802,8 @@ def sortBy(self, keyfunc, ascending=True, numPartitions=None):
         """
         Sorts this RDD by the given keyfunc
 
+        Examples
+        --------
         >>> tmp = [('a', 1), ('b', 2), ('1', 3), ('d', 4), ('2', 5)]
         >>> sc.parallelize(tmp).sortBy(lambda x: x[0]).collect()
         [('1', 3), ('2', 5), ('a', 1), ('b', 2), ('d', 4)]
@@ -764,6 +817,8 @@ def glom(self):
         Return an RDD created by coalescing all elements within each partition
         into a list.
 
+        Examples
+        --------
         >>> rdd = sc.parallelize([1, 2, 3, 4], 2)
         >>> sorted(rdd.glom().collect())
         [[1, 2], [3, 4]]
@@ -778,6 +833,8 @@ def cartesian(self, other):
         RDD of all pairs of elements ``(a, b)`` where ``a`` is in `self` and
         ``b`` is in `other`.
 
+        Examples
+        --------
         >>> rdd = sc.parallelize([1, 2])
         >>> sorted(rdd.cartesian(rdd).collect())
         [(1, 1), (1, 2), (2, 1), (2, 2)]
@@ -791,6 +848,8 @@ def groupBy(self, f, numPartitions=None, partitionFunc=portable_hash):
         """
         Return an RDD of grouped items.
 
+        Examples
+        --------
         >>> rdd = sc.parallelize([1, 1, 2, 3, 5, 8])
         >>> result = rdd.groupBy(lambda x: x % 2).collect()
         >>> sorted([(x, sorted(y)) for (x, y) in result])
@@ -798,15 +857,23 @@ def groupBy(self, f, numPartitions=None, partitionFunc=portable_hash):
         """
         return self.map(lambda x: (f(x), x)).groupByKey(numPartitions, partitionFunc)
 
-    @ignore_unicode_prefix
     def pipe(self, command, env=None, checkCode=False):
         """
         Return an RDD created by piping elements to a forked external process.
 
+        Parameters
+        ----------
+        command : str
+            command to run.
+        env : dict, optional
+            environment variables to set.
+        checkCode : bool, optional
+            whether or not to check the return value of the shell command.
+
+        Examples
+        --------
         >>> sc.parallelize(['1', '2', '', '3']).pipe('cat').collect()
-        [u'1', u'2', u'', u'3']
-
-        :param checkCode: whether or not to check the return value of the shell command.
+        ['1', '2', '', '3']
         """
         if env is None:
             env = dict()
@@ -817,7 +884,7 @@ def func(iterator):
 
             def pipe_objs(out):
                 for obj in iterator:
-                    s = unicode(obj).rstrip('\n') + '\n'
+                    s = str(obj).rstrip('\n') + '\n'
                     out.write(s.encode('utf-8'))
                 out.close()
             Thread(target=pipe_objs, args=[pipe.stdin]).start()
@@ -838,6 +905,8 @@ def foreach(self, f):
         """
         Applies a function to all elements of this RDD.
 
+        Examples
+        --------
         >>> def f(x): print(x)
         >>> sc.parallelize([1, 2, 3, 4, 5]).foreach(f)
         """
@@ -853,6 +922,8 @@ def foreachPartition(self, f):
         """
         Applies a function to each partition of this RDD.
 
+        Examples
+        --------
         >>> def f(iterator):
         ...     for x in iterator:
         ...          print(x)
@@ -870,8 +941,10 @@ def collect(self):
         """
         Return a list that contains all of the elements in this RDD.
 
-        .. note:: This method should only be used if the resulting array is expected
-            to be small, as all the data is loaded into the driver's memory.
+        Notes
+        -----
+        This method should only be used if the resulting array is expected
+        to be small, as all the data is loaded into the driver's memory.
         """
         with SCCallSiteSync(self.context) as css:
             sock_info = self.ctx._jvm.PythonRDD.collectAndServe(self._jrdd.rdd())
@@ -879,12 +952,18 @@ def collect(self):
 
     def collectWithJobGroup(self, groupId, description, interruptOnCancel=False):
         """
-        .. note:: Experimental
-
         When collect rdd, use this method to specify job group.
 
         .. versionadded:: 3.0.0
+        .. deprecated:: 3.1.0
+            Use :class:`pyspark.InheritableThread` with the pinned thread mode enabled.
         """
+        warnings.warn(
+            "Deprecated in 3.1, Use pyspark.InheritableThread with "
+            "the pinned thread mode enabled.",
+            FutureWarning
+        )
+
         with SCCallSiteSync(self.context) as css:
             sock_info = self.ctx._jvm.PythonRDD.collectAndServeWithJobGroup(
                 self._jrdd.rdd(), groupId, description, interruptOnCancel)
@@ -895,6 +974,8 @@ def reduce(self, f):
         Reduces the elements of this RDD using the specified commutative and
         associative binary operator. Currently reduces partitions locally.
 
+        Examples
+        --------
         >>> from operator import add
         >>> sc.parallelize([1, 2, 3, 4, 5]).reduce(add)
         15
@@ -924,8 +1005,14 @@ def treeReduce(self, f, depth=2):
         """
         Reduces the elements of this RDD in a multi-level tree pattern.
 
-        :param depth: suggested depth of the tree (default: 2)
+        Parameters
+        ----------
+        f : function
+        depth : int, optional
+            suggested depth of the tree (default: 2)
 
+        Examples
+        --------
         >>> add = lambda x, y: x + y
         >>> rdd = sc.parallelize([-5, -4, -3, -2, -1, 1, 2, 3, 4], 10)
         >>> rdd.treeReduce(add)
@@ -974,6 +1061,8 @@ def fold(self, zeroValue, op):
         that are not commutative, the result may differ from that of a fold
         applied to a non-distributed collection.
 
+        Examples
+        --------
         >>> from operator import add
         >>> sc.parallelize([1, 2, 3, 4, 5]).fold(0, add)
         15
@@ -1005,6 +1094,8 @@ def aggregate(self, zeroValue, seqOp, combOp):
         the type of this RDD. Thus, we need one operation for merging a T into
         an U and one operation for merging two U
 
+        Examples
+        --------
         >>> seqOp = (lambda x, y: (x[0] + y, x[1] + 1))
         >>> combOp = (lambda x, y: (x[0] + y[0], x[1] + y[1]))
         >>> sc.parallelize([1, 2, 3, 4]).aggregate((0, 0), seqOp, combOp)
@@ -1031,8 +1122,11 @@ def treeAggregate(self, zeroValue, seqOp, combOp, depth=2):
         Aggregates the elements of this RDD in a multi-level tree
         pattern.
 
-        :param depth: suggested depth of the tree (default: 2)
+        depth : int, optional
+            suggested depth of the tree (default: 2)
 
+        Examples
+        --------
         >>> add = lambda x, y: x + y
         >>> rdd = sc.parallelize([-5, -4, -3, -2, -1, 1, 2, 3, 4], 10)
         >>> rdd.treeAggregate(0, add, add)
@@ -1082,8 +1176,13 @@ def max(self, key=None):
         """
         Find the maximum item in this RDD.
 
-        :param key: A function used to generate key for comparing
+        Parameters
+        ----------
+        key : function, optional
+            A function used to generate key for comparing
 
+        Examples
+        --------
         >>> rdd = sc.parallelize([1.0, 5.0, 43.0, 10.0])
         >>> rdd.max()
         43.0
@@ -1098,8 +1197,13 @@ def min(self, key=None):
         """
         Find the minimum item in this RDD.
 
-        :param key: A function used to generate key for comparing
+        Parameters
+        ----------
+        key : function, optional
+            A function used to generate key for comparing
 
+        Examples
+        --------
         >>> rdd = sc.parallelize([2.0, 5.0, 43.0, 10.0])
         >>> rdd.min()
         2.0
@@ -1114,6 +1218,8 @@ def sum(self):
         """
         Add up the elements in this RDD.
 
+        Examples
+        --------
         >>> sc.parallelize([1.0, 2.0, 3.0]).sum()
         6.0
         """
@@ -1123,6 +1229,8 @@ def count(self):
         """
         Return the number of elements in this RDD.
 
+        Examples
+        --------
         >>> sc.parallelize([2, 3, 4]).count()
         3
         """
@@ -1147,7 +1255,7 @@ def histogram(self, buckets):
         and 50 we would have a histogram of 1,0,1.
 
         If your histogram is evenly spaced (e.g. [0, 10, 20, 30]),
-        this can be switched from an O(log n) inseration to O(1) per
+        this can be switched from an O(log n) insertion to O(1) per
         element (where n is the number of buckets).
 
         Buckets must be sorted, not contain any duplicates, and have
@@ -1163,6 +1271,8 @@ def histogram(self, buckets):
 
         The return value is a tuple of buckets and histogram.
 
+        Examples
+        --------
         >>> rdd = sc.parallelize(range(51))
         >>> rdd.histogram(2)
         ([0, 25, 50], [25, 26])
@@ -1270,6 +1380,8 @@ def mean(self):
         """
         Compute the mean of this RDD's elements.
 
+        Examples
+        --------
         >>> sc.parallelize([1, 2, 3]).mean()
         2.0
         """
@@ -1279,6 +1391,8 @@ def variance(self):
         """
         Compute the variance of this RDD's elements.
 
+        Examples
+        --------
         >>> sc.parallelize([1, 2, 3]).variance()
         0.666...
         """
@@ -1288,6 +1402,8 @@ def stdev(self):
         """
         Compute the standard deviation of this RDD's elements.
 
+        Examples
+        --------
         >>> sc.parallelize([1, 2, 3]).stdev()
         0.816...
         """
@@ -1299,6 +1415,8 @@ def sampleStdev(self):
         corrects for bias in estimating the standard deviation by dividing by
         N-1 instead of N).
 
+        Examples
+        --------
         >>> sc.parallelize([1, 2, 3]).sampleStdev()
         1.0
         """
@@ -1309,6 +1427,8 @@ def sampleVariance(self):
         Compute the sample variance of this RDD's elements (which corrects
         for bias in estimating the variance by dividing by N-1 instead of N).
 
+        Examples
+        --------
         >>> sc.parallelize([1, 2, 3]).sampleVariance()
         1.0
         """
@@ -1319,6 +1439,8 @@ def countByValue(self):
         Return the count of each unique value in this RDD as a dictionary of
         (value, count) pairs.
 
+        Examples
+        --------
         >>> sorted(sc.parallelize([1, 2, 1, 2, 2], 2).countByValue().items())
         [(1, 2), (2, 3)]
         """
@@ -1338,11 +1460,15 @@ def top(self, num, key=None):
         """
         Get the top N elements from an RDD.
 
-        .. note:: This method should only be used if the resulting array is expected
-            to be small, as all the data is loaded into the driver's memory.
+        Notes
+        -----
+        This method should only be used if the resulting array is expected
+        to be small, as all the data is loaded into the driver's memory.
 
-        .. note:: It returns the list sorted in descending order.
+        It returns the list sorted in descending order.
 
+        Examples
+        --------
         >>> sc.parallelize([10, 4, 2, 12, 3]).top(1)
         [12]
         >>> sc.parallelize([2, 3, 4, 5, 6], 2).top(2)
@@ -1363,9 +1489,13 @@ def takeOrdered(self, num, key=None):
         Get the N elements from an RDD ordered in ascending order or as
         specified by the optional key function.
 
-        .. note:: this method should only be used if the resulting array is expected
-            to be small, as all the data is loaded into the driver's memory.
+        Notes
+        -----
+        This method should only be used if the resulting array is expected
+        to be small, as all the data is loaded into the driver's memory.
 
+        Examples
+        --------
         >>> sc.parallelize([10, 1, 2, 9, 3, 4, 5, 6, 7]).takeOrdered(6)
         [1, 2, 3, 4, 5, 6]
         >>> sc.parallelize([10, 1, 2, 9, 3, 4, 5, 6, 7], 2).takeOrdered(6, key=lambda x: -x)
@@ -1387,9 +1517,13 @@ def take(self, num):
 
         Translated from the Scala implementation in RDD#take().
 
-        .. note:: this method should only be used if the resulting array is expected
-            to be small, as all the data is loaded into the driver's memory.
+        Notes
+        -----
+        This method should only be used if the resulting array is expected
+        to be small, as all the data is loaded into the driver's memory.
 
+        Examples
+        --------
         >>> sc.parallelize([2, 3, 4, 5, 6]).cache().take(2)
         [2, 3]
         >>> sc.parallelize([2, 3, 4, 5, 6]).take(10)
@@ -1442,6 +1576,8 @@ def first(self):
         """
         Return the first element in this RDD.
 
+        Examples
+        --------
         >>> sc.parallelize([2, 3, 4]).first()
         2
         >>> sc.parallelize([]).first()
@@ -1458,8 +1594,12 @@ def isEmpty(self):
         """
         Returns true if and only if the RDD contains no elements at all.
 
-        .. note:: an RDD may be empty even when it has at least 1 partition.
+        Notes
+        -----
+        An RDD may be empty even when it has at least 1 partition.
 
+        Examples
+        --------
         >>> sc.parallelize([]).isEmpty()
         True
         >>> sc.parallelize([1]).isEmpty()
@@ -1474,9 +1614,14 @@ def saveAsNewAPIHadoopDataset(self, conf, keyConverter=None, valueConverter=None
         converted for output using either user specified converters or, by default,
         "org.apache.spark.api.python.JavaToWritableConverter".
 
-        :param conf: Hadoop job configuration, passed in as a dict
-        :param keyConverter: (None by default)
-        :param valueConverter: (None by default)
+        Parameters
+        ----------
+        conf : dict
+            Hadoop job configuration
+        keyConverter : str, optional
+            fully qualified classname of key converter (None by default)
+        valueConverter : str, optional
+            fully qualified classname of value converter (None by default)
         """
         jconf = self.ctx._dictToJavaMap(conf)
         pickledRDD = self._pickled()
@@ -1493,16 +1638,23 @@ def saveAsNewAPIHadoopFile(self, path, outputFormatClass, keyClass=None, valueCl
         `conf` is applied on top of the base Hadoop conf associated with the SparkContext
         of this RDD to create a merged Hadoop MapReduce job configuration for saving the data.
 
-        :param path: path to Hadoop file
-        :param outputFormatClass: fully qualified classname of Hadoop OutputFormat
-               (e.g. "org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat")
-        :param keyClass: fully qualified classname of key Writable class
-               (e.g. "org.apache.hadoop.io.IntWritable", None by default)
-        :param valueClass: fully qualified classname of value Writable class
-               (e.g. "org.apache.hadoop.io.Text", None by default)
-        :param keyConverter: (None by default)
-        :param valueConverter: (None by default)
-        :param conf: Hadoop job configuration, passed in as a dict (None by default)
+        path : str
+            path to Hadoop file
+        outputFormatClass : str
+            fully qualified classname of Hadoop OutputFormat
+            (e.g. "org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat")
+        keyClass : str, optional
+            fully qualified classname of key Writable class
+             (e.g. "org.apache.hadoop.io.IntWritable", None by default)
+        valueClass : str, optional
+            fully qualified classname of value Writable class
+            (e.g. "org.apache.hadoop.io.Text", None by default)
+        keyConverter : str, optional
+            fully qualified classname of key converter (None by default)
+        valueConverter : str, optional
+            fully qualified classname of value converter (None by default)
+        conf : dict, optional
+            Hadoop job configuration (None by default)
         """
         jconf = self.ctx._dictToJavaMap(conf)
         pickledRDD = self._pickled()
@@ -1518,9 +1670,14 @@ def saveAsHadoopDataset(self, conf, keyConverter=None, valueConverter=None):
         converted for output using either user specified converters or, by default,
         "org.apache.spark.api.python.JavaToWritableConverter".
 
-        :param conf: Hadoop job configuration, passed in as a dict
-        :param keyConverter: (None by default)
-        :param valueConverter: (None by default)
+        Parameters
+        ----------
+        conf : dict
+            Hadoop job configuration
+        keyConverter : str, optional
+            fully qualified classname of key converter (None by default)
+        valueConverter : str, optional
+            fully qualified classname of value converter (None by default)
         """
         jconf = self.ctx._dictToJavaMap(conf)
         pickledRDD = self._pickled()
@@ -1538,17 +1695,28 @@ def saveAsHadoopFile(self, path, outputFormatClass, keyClass=None, valueClass=No
         `conf` is applied on top of the base Hadoop conf associated with the SparkContext
         of this RDD to create a merged Hadoop MapReduce job configuration for saving the data.
 
-        :param path: path to Hadoop file
-        :param outputFormatClass: fully qualified classname of Hadoop OutputFormat
-               (e.g. "org.apache.hadoop.mapred.SequenceFileOutputFormat")
-        :param keyClass: fully qualified classname of key Writable class
-               (e.g. "org.apache.hadoop.io.IntWritable", None by default)
-        :param valueClass: fully qualified classname of value Writable class
-               (e.g. "org.apache.hadoop.io.Text", None by default)
-        :param keyConverter: (None by default)
-        :param valueConverter: (None by default)
-        :param conf: (None by default)
-        :param compressionCodecClass: (None by default)
+        Parameters
+        ----------
+        path : str
+            path to Hadoop file
+        outputFormatClass : str
+            fully qualified classname of Hadoop OutputFormat
+            (e.g. "org.apache.hadoop.mapred.SequenceFileOutputFormat")
+        keyClass : str, optional
+            fully qualified classname of key Writable class
+            (e.g. "org.apache.hadoop.io.IntWritable", None by default)
+        valueClass : str, optional
+            fully qualified classname of value Writable class
+            (e.g. "org.apache.hadoop.io.Text", None by default)
+        keyConverter : str, optional
+            fully qualified classname of key converter (None by default)
+        valueConverter : str, optional
+            fully qualified classname of value converter (None by default)
+        conf : dict, optional
+            (None by default)
+        compressionCodecClass : str
+            fully qualified classname of the compression codec class
+            i.e. "org.apache.hadoop.io.compress.GzipCodec" (None by default)
         """
         jconf = self.ctx._dictToJavaMap(conf)
         pickledRDD = self._pickled()
@@ -1567,8 +1735,13 @@ def saveAsSequenceFile(self, path, compressionCodecClass=None):
             1. Pyrolite is used to convert pickled Python RDD into RDD of Java objects.
             2. Keys and values of this Java RDD are converted to Writables and written out.
 
-        :param path: path to sequence file
-        :param compressionCodecClass: (None by default)
+        Parameters
+        ----------
+        path : str
+            path to sequence file
+        compressionCodecClass : str, optional
+            fully qualified classname of the compression codec class
+            i.e. "org.apache.hadoop.io.compress.GzipCodec" (None by default)
         """
         pickledRDD = self._pickled()
         self.ctx._jvm.PythonRDD.saveAsSequenceFile(pickledRDD._jrdd, True,
@@ -1580,6 +1753,9 @@ def saveAsPickleFile(self, path, batchSize=10):
         used is :class:`pyspark.serializers.PickleSerializer`, default batch size
         is 10.
 
+        Examples
+        --------
+        >>> from tempfile import NamedTemporaryFile
         >>> tmpFile = NamedTemporaryFile(delete=True)
         >>> tmpFile.close()
         >>> sc.parallelize([1, 2, 'spark', 'rdd']).saveAsPickleFile(tmpFile.name, 3)
@@ -1592,15 +1768,21 @@ def saveAsPickleFile(self, path, batchSize=10):
             ser = BatchedSerializer(PickleSerializer(), batchSize)
         self._reserialize(ser)._jrdd.saveAsObjectFile(path)
 
-    @ignore_unicode_prefix
     def saveAsTextFile(self, path, compressionCodecClass=None):
         """
         Save this RDD as a text file, using string representations of elements.
 
-        :param path: path to text file
-        :param compressionCodecClass: (None by default) string i.e.
-            "org.apache.hadoop.io.compress.GzipCodec"
-
+        Parameters
+        ----------
+        path : str
+            path to text file
+        compressionCodecClass : str, optional
+            fully qualified classname of the compression codec class
+            i.e. "org.apache.hadoop.io.compress.GzipCodec" (None by default)
+
+        Examples
+        --------
+        >>> from tempfile import NamedTemporaryFile
         >>> tempFile = NamedTemporaryFile(delete=True)
         >>> tempFile.close()
         >>> sc.parallelize(range(10)).saveAsTextFile(tempFile.name)
@@ -1611,6 +1793,7 @@ def saveAsTextFile(self, path, compressionCodecClass=None):
 
         Empty lines are tolerated when saving to text files.
 
+        >>> from tempfile import NamedTemporaryFile
         >>> tempFile2 = NamedTemporaryFile(delete=True)
         >>> tempFile2.close()
         >>> sc.parallelize(['', 'foo', '', 'bar', '']).saveAsTextFile(tempFile2.name)
@@ -1619,6 +1802,7 @@ def saveAsTextFile(self, path, compressionCodecClass=None):
 
         Using compressionCodecClass
 
+        >>> from tempfile import NamedTemporaryFile
         >>> tempFile3 = NamedTemporaryFile(delete=True)
         >>> tempFile3.close()
         >>> codec = "org.apache.hadoop.io.compress.GzipCodec"
@@ -1626,13 +1810,13 @@ def saveAsTextFile(self, path, compressionCodecClass=None):
         >>> from fileinput import input, hook_compressed
         >>> result = sorted(input(glob(tempFile3.name + "/part*.gz"), openhook=hook_compressed))
         >>> b''.join(result).decode('utf-8')
-        u'bar\\nfoo\\n'
+        'bar\\nfoo\\n'
         """
         def func(split, iterator):
             for x in iterator:
-                if not isinstance(x, (unicode, bytes)):
-                    x = unicode(x)
-                if isinstance(x, unicode):
+                if not isinstance(x, (str, bytes)):
+                    x = str(x)
+                if isinstance(x, str):
                     x = x.encode("utf-8")
                 yield x
         keyed = self.mapPartitionsWithIndex(func)
@@ -1649,9 +1833,13 @@ def collectAsMap(self):
         """
         Return the key-value pairs in this RDD to the master as a dictionary.
 
-        .. note:: this method should only be used if the resulting data is expected
-            to be small, as all the data is loaded into the driver's memory.
+        Notes
+        -----
+        This method should only be used if the resulting data is expected
+        to be small, as all the data is loaded into the driver's memory.
 
+        Examples
+        --------
         >>> m = sc.parallelize([(1, 2), (3, 4)]).collectAsMap()
         >>> m[1]
         2
@@ -1664,6 +1852,8 @@ def keys(self):
         """
         Return an RDD with the keys of each tuple.
 
+        Examples
+        --------
         >>> m = sc.parallelize([(1, 2), (3, 4)]).keys()
         >>> m.collect()
         [1, 3]
@@ -1674,6 +1864,8 @@ def values(self):
         """
         Return an RDD with the values of each tuple.
 
+        Examples
+        --------
         >>> m = sc.parallelize([(1, 2), (3, 4)]).values()
         >>> m.collect()
         [2, 4]
@@ -1691,6 +1883,8 @@ def reduceByKey(self, func, numPartitions=None, partitionFunc=portable_hash):
         the default parallelism level if `numPartitions` is not specified.
         Default partitioner is hash-partition.
 
+        Examples
+        --------
         >>> from operator import add
         >>> rdd = sc.parallelize([("a", 1), ("b", 1), ("a", 1)])
         >>> sorted(rdd.reduceByKey(add).collect())
@@ -1706,6 +1900,8 @@ def reduceByKeyLocally(self, func):
         This will also perform the merging locally on each mapper before
         sending results to a reducer, similarly to a "combiner" in MapReduce.
 
+        Examples
+        --------
         >>> from operator import add
         >>> rdd = sc.parallelize([("a", 1), ("b", 1), ("a", 1)])
         >>> sorted(rdd.reduceByKeyLocally(add).items())
@@ -1730,6 +1926,8 @@ def countByKey(self):
         Count the number of elements for each key, and return the result to the
         master as a dictionary.
 
+        Examples
+        --------
         >>> rdd = sc.parallelize([("a", 1), ("b", 1), ("a", 1)])
         >>> sorted(rdd.countByKey().items())
         [('a', 2), ('b', 1)]
@@ -1746,6 +1944,8 @@ def join(self, other, numPartitions=None):
 
         Performs a hash join across the cluster.
 
+        Examples
+        --------
         >>> x = sc.parallelize([("a", 1), ("b", 4)])
         >>> y = sc.parallelize([("a", 2), ("a", 3)])
         >>> sorted(x.join(y).collect())
@@ -1763,6 +1963,8 @@ def leftOuterJoin(self, other, numPartitions=None):
 
         Hash-partitions the resulting RDD into the given number of partitions.
 
+        Examples
+        --------
         >>> x = sc.parallelize([("a", 1), ("b", 4)])
         >>> y = sc.parallelize([("a", 2)])
         >>> sorted(x.leftOuterJoin(y).collect())
@@ -1780,6 +1982,8 @@ def rightOuterJoin(self, other, numPartitions=None):
 
         Hash-partitions the resulting RDD into the given number of partitions.
 
+        Examples
+        --------
         >>> x = sc.parallelize([("a", 1), ("b", 4)])
         >>> y = sc.parallelize([("a", 2)])
         >>> sorted(y.rightOuterJoin(x).collect())
@@ -1801,6 +2005,8 @@ def fullOuterJoin(self, other, numPartitions=None):
 
         Hash-partitions the resulting RDD into the given number of partitions.
 
+        Examples
+        --------
         >>> x = sc.parallelize([("a", 1), ("b", 4)])
         >>> y = sc.parallelize([("a", 2), ("c", 8)])
         >>> sorted(x.fullOuterJoin(y).collect())
@@ -1815,6 +2021,8 @@ def partitionBy(self, numPartitions, partitionFunc=portable_hash):
         """
         Return a copy of the RDD partitioned using the specified partitioner.
 
+        Examples
+        --------
         >>> pairs = sc.parallelize([1, 2, 3, 4, 2, 4, 1]).map(lambda x: (x, x))
         >>> sets = pairs.partitionBy(2).glom().collect()
         >>> len(set(sets[0]).intersection(set(sets[1])))
@@ -1904,9 +2112,13 @@ def combineByKey(self, createCombiner, mergeValue, mergeCombiners,
 
         In addition, users can control the partitioning of the output RDD.
 
-        .. note:: V and C can be different -- for example, one might group an RDD of type
+        Notes
+        -----
+        V and C can be different -- for example, one might group an RDD of type
             (Int, Int) into an RDD of type (Int, List[Int]).
 
+        Examples
+        --------
         >>> x = sc.parallelize([("a", 1), ("b", 1), ("a", 2)])
         >>> def to_list(a):
         ...     return [a]
@@ -1968,6 +2180,8 @@ def foldByKey(self, zeroValue, func, numPartitions=None, partitionFunc=portable_
         arbitrary number of times, and must not change the result
         (e.g., 0 for addition, or 1 for multiplication.).
 
+        Examples
+        --------
         >>> rdd = sc.parallelize([("a", 1), ("b", 1), ("a", 1)])
         >>> from operator import add
         >>> sorted(rdd.foldByKey(0, add).collect())
@@ -1988,10 +2202,14 @@ def groupByKey(self, numPartitions=None, partitionFunc=portable_hash):
         Group the values for each key in the RDD into a single sequence.
         Hash-partitions the resulting RDD with numPartitions partitions.
 
-        .. note:: If you are grouping in order to perform an aggregation (such as a
-            sum or average) over each key, using reduceByKey or aggregateByKey will
-            provide much better performance.
+        Notes
+        -----
+        If you are grouping in order to perform an aggregation (such as a
+        sum or average) over each key, using reduceByKey or aggregateByKey will
+        provide much better performance.
 
+        Examples
+        --------
         >>> rdd = sc.parallelize([("a", 1), ("b", 1), ("a", 1)])
         >>> sorted(rdd.groupByKey().mapValues(len).collect())
         [('a', 2), ('b', 1)]
@@ -2034,6 +2252,8 @@ def flatMapValues(self, f):
         without changing the keys; this also retains the original RDD's
         partitioning.
 
+        Examples
+        --------
         >>> x = sc.parallelize([("a", ["x", "y", "z"]), ("b", ["p", "r"])])
         >>> def f(x): return x
         >>> x.flatMapValues(f).collect()
@@ -2048,6 +2268,8 @@ def mapValues(self, f):
         without changing the keys; this also retains the original RDD's
         partitioning.
 
+        Examples
+        --------
         >>> x = sc.parallelize([("a", ["apple", "banana", "lemon"]), ("b", ["grapes"])])
         >>> def f(x): return len(x)
         >>> x.mapValues(f).collect()
@@ -2060,6 +2282,8 @@ def groupWith(self, other, *others):
         """
         Alias for cogroup but with support for multiple RDDs.
 
+        Examples
+        --------
         >>> w = sc.parallelize([("a", 5), ("b", 6)])
         >>> x = sc.parallelize([("a", 1), ("b", 4)])
         >>> y = sc.parallelize([("a", 2)])
@@ -2070,13 +2294,15 @@ def groupWith(self, other, *others):
         """
         return python_cogroup((self, other) + others, numPartitions=None)
 
-    # TODO: add variant with custom parittioner
+    # TODO: add variant with custom partitioner
     def cogroup(self, other, numPartitions=None):
         """
         For each key k in `self` or `other`, return a resulting RDD that
         contains a tuple with the list of values for that key in `self` as
         well as `other`.
 
+        Examples
+        --------
         >>> x = sc.parallelize([("a", 1), ("b", 4)])
         >>> y = sc.parallelize([("a", 2)])
         >>> [(x, tuple(map(list, y))) for x, y in sorted(list(x.cogroup(y).collect()))]
@@ -2090,6 +2316,8 @@ def sampleByKey(self, withReplacement, fractions, seed=None):
         Create a sample of this RDD using variable sampling rates for
         different keys as specified by fractions, a key to sampling rate map.
 
+        Examples
+        --------
         >>> fractions = {"a": 0.2, "b": 0.1}
         >>> rdd = sc.parallelize(fractions.keys()).cartesian(sc.parallelize(range(0, 1000)))
         >>> sample = dict(rdd.sampleByKey(False, fractions, 2).groupByKey().collect())
@@ -2110,6 +2338,8 @@ def subtractByKey(self, other, numPartitions=None):
         Return each (key, value) pair in `self` that has no pair with matching
         key in `other`.
 
+        Examples
+        --------
         >>> x = sc.parallelize([("a", 1), ("b", 4), ("b", 5), ("a", 2)])
         >>> y = sc.parallelize([("a", 3), ("c", None)])
         >>> sorted(x.subtractByKey(y).collect())
@@ -2124,6 +2354,8 @@ def subtract(self, other, numPartitions=None):
         """
         Return each value in `self` that is not contained in `other`.
 
+        Examples
+        --------
         >>> x = sc.parallelize([("a", 1), ("b", 4), ("b", 5), ("a", 3)])
         >>> y = sc.parallelize([("a", 3), ("c", None)])
         >>> sorted(x.subtract(y).collect())
@@ -2137,6 +2369,8 @@ def keyBy(self, f):
         """
         Creates tuples of the elements in this RDD by applying `f`.
 
+        Examples
+        --------
         >>> x = sc.parallelize(range(0,3)).keyBy(lambda x: x*x)
         >>> y = sc.parallelize(zip(range(0,5), range(0,5)))
         >>> [(x, list(map(list, y))) for x, y in sorted(x.cogroup(y).collect())]
@@ -2153,6 +2387,8 @@ def repartition(self, numPartitions):
          If you are decreasing the number of partitions in this RDD, consider
          using `coalesce`, which can avoid performing a shuffle.
 
+        Examples
+        --------
          >>> rdd = sc.parallelize([1,2,3,4,5,6,7], 4)
          >>> sorted(rdd.glom().collect())
          [[1], [2, 3], [4, 5], [6, 7]]
@@ -2167,6 +2403,8 @@ def coalesce(self, numPartitions, shuffle=False):
         """
         Return a new RDD that is reduced into `numPartitions` partitions.
 
+        Examples
+        --------
         >>> sc.parallelize([1, 2, 3, 4, 5], 3).glom().collect()
         [[1], [2, 3], [4, 5]]
         >>> sc.parallelize([1, 2, 3, 4, 5], 3).coalesce(1).glom().collect()
@@ -2193,6 +2431,8 @@ def zip(self, other):
         number of elements in each partition (e.g. one was made through
         a map on the other).
 
+        Examples
+        --------
         >>> x = sc.parallelize(range(0,5))
         >>> y = sc.parallelize(range(1000, 1005))
         >>> x.zip(y).collect()
@@ -2239,6 +2479,8 @@ def zipWithIndex(self):
         This method needs to trigger a spark job when this RDD contains
         more than one partitions.
 
+        Examples
+        --------
         >>> sc.parallelize(["a", "b", "c", "d"], 3).zipWithIndex().collect()
         [('a', 0), ('b', 1), ('c', 2), ('d', 3)]
         """
@@ -2263,6 +2505,8 @@ def zipWithUniqueId(self):
         method won't trigger a spark job, which is different from
         :meth:`zipWithIndex`.
 
+        Examples
+        --------
         >>> sc.parallelize(["a", "b", "c", "d", "e"], 3).zipWithUniqueId().collect()
         [('a', 0), ('b', 1), ('c', 4), ('d', 2), ('e', 5)]
         """
@@ -2282,14 +2526,15 @@ def name(self):
         if n:
             return n
 
-    @ignore_unicode_prefix
     def setName(self, name):
         """
         Assign a name to this RDD.
 
+        Examples
+        --------
         >>> rdd1 = sc.parallelize([1, 2])
         >>> rdd1.setName('RDD1').name()
-        u'RDD1'
+        'RDD1'
         """
         self._jrdd.setName(name)
         return self
@@ -2306,6 +2551,8 @@ def getStorageLevel(self):
         """
         Get the RDD's current storage level.
 
+        Examples
+        --------
         >>> rdd1 = sc.parallelize([1,2])
         >>> rdd1.getStorageLevel()
         StorageLevel(False, False, False, False, 1)
@@ -2341,6 +2588,8 @@ def lookup(self, key):
         is done efficiently if the RDD has a known partitioner by only
         searching the partition that the key maps to.
 
+        Examples
+        --------
         >>> l = range(1000)
         >>> rdd = sc.parallelize(zip(l, l), 10)
         >>> rdd.lookup(42)  # slow
@@ -2375,6 +2624,8 @@ def countApprox(self, timeout, confidence=0.95):
         Approximate version of count() that returns a potentially incomplete
         result within a timeout, even if not all tasks have finished.
 
+        Examples
+        --------
         >>> rdd = sc.parallelize(range(1000), 10)
         >>> rdd.countApprox(1000, 1.0)
         1000
@@ -2387,6 +2638,8 @@ def sumApprox(self, timeout, confidence=0.95):
         Approximate operation to return the sum within a timeout
         or meet the confidence.
 
+        Examples
+        --------
         >>> rdd = sc.parallelize(range(1000), 10)
         >>> r = sum(range(1000))
         >>> abs(rdd.sumApprox(1000) - r) / r < 0.05
@@ -2402,6 +2655,8 @@ def meanApprox(self, timeout, confidence=0.95):
         Approximate operation to return the mean within a timeout
         or meet the confidence.
 
+        Examples
+        --------
         >>> rdd = sc.parallelize(range(1000), 10)
         >>> r = sum(range(1000)) / 1000.0
         >>> abs(rdd.meanApprox(1000) - r) / r < 0.05
@@ -2416,15 +2671,22 @@ def countApproxDistinct(self, relativeSD=0.05):
         """
         Return approximate number of distinct elements in the RDD.
 
+        Parameters
+        ----------
+        relativeSD : float, optional
+            Relative accuracy. Smaller values create
+            counters that require more space.
+            It must be greater than 0.000017.
+
+        Notes
+        -----
         The algorithm used is based on streamlib's implementation of
         `"HyperLogLog in Practice: Algorithmic Engineering of a State
         of The Art Cardinality Estimation Algorithm", available here
         <https://doi.org/10.1145/2452376.2452456>`_.
 
-        :param relativeSD: Relative accuracy. Smaller values create
-                           counters that require more space.
-                           It must be greater than 0.000017.
-
+        Examples
+        --------
         >>> n = sc.parallelize(range(1000)).map(str).countApproxDistinct()
         >>> 900 < n < 1100
         True
@@ -2444,9 +2706,14 @@ def toLocalIterator(self, prefetchPartitions=False):
         The iterator will consume as much memory as the largest partition in this RDD.
         With prefetch it may consume up to the memory of the 2 largest partitions.
 
-        :param prefetchPartitions: If Spark should pre-fetch the next partition
-                                   before it is needed.
+        Parameters
+        ----------
+        prefetchPartitions : bool, optional
+            If Spark should pre-fetch the next partition
+            before it is needed.
 
+        Examples
+        --------
         >>> rdd = sc.parallelize(range(10))
         >>> [x for x in rdd.toLocalIterator()]
         [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
@@ -2459,22 +2726,31 @@ def toLocalIterator(self, prefetchPartitions=False):
 
     def barrier(self):
         """
-        .. note:: Experimental
-
         Marks the current stage as a barrier stage, where Spark must launch all tasks together.
         In case of a task failure, instead of only restarting the failed task, Spark will abort the
         entire stage and relaunch all tasks for this stage.
         The barrier execution mode feature is experimental and it only handles limited scenarios.
         Please read the linked SPIP and design docs to understand the limitations and future plans.
 
-        :return: an :class:`RDDBarrier` instance that provides actions within a barrier stage.
+        .. versionadded:: 2.4.0
 
-        .. seealso:: :class:`BarrierTaskContext`
-        .. seealso:: `SPIP: Barrier Execution Mode
-            <http://jira.apache.org/jira/browse/SPARK-24374>`_
-        .. seealso:: `Design Doc <https://jira.apache.org/jira/browse/SPARK-24582>`_
+        Returns
+        -------
+        :class:`RDDBarrier`
+            instance that provides actions within a barrier stage.
 
-        .. versionadded:: 2.4.0
+        See Also
+        --------
+        pyspark.BarrierTaskContext
+
+        Notes
+        -----
+        For additional information see
+
+        - `SPIP: Barrier Execution Mode <http://jira.apache.org/jira/browse/SPARK-24374>`_
+        - `Design Doc <https://jira.apache.org/jira/browse/SPARK-24582>`_
+
+        This API is experimental
         """
         return RDDBarrier(self)
 
@@ -2486,14 +2762,16 @@ def _is_barrier(self):
 
     def withResources(self, profile):
         """
-        .. note:: Experimental
-
         Specify a :class:`pyspark.resource.ResourceProfile` to use when calculating this RDD.
         This is only supported on certain cluster managers and currently requires dynamic
         allocation to be enabled. It will result in new executors with the resources specified
         being acquired to calculate the RDD.
 
         .. versionadded:: 3.1.0
+
+        Notes
+        -----
+        This API is experimental
         """
         self.has_resource_profile = True
         if profile._java_resource_profile is not None:
@@ -2511,13 +2789,19 @@ def withResources(self, profile):
 
     def getResourceProfile(self):
         """
-        .. note:: Experimental
-
         Get the :class:`pyspark.resource.ResourceProfile` specified with this RDD or None
         if it wasn't specified.
-        :return: the user specified ResourceProfile or None if none were specified
 
         .. versionadded:: 3.1.0
+
+        Returns
+        -------
+        :py:class:`pyspark.resource.ResourceProfile`
+            The the user specified profile or None if none were specified
+
+        Notes
+        -----
+        This API is experimental
         """
         rp = self._jrdd.getResourceProfile()
         if rp is not None:
@@ -2551,12 +2835,14 @@ def _wrap_function(sc, func, deserializer, serializer, profiler=None):
 class RDDBarrier(object):
 
     """
-    .. note:: Experimental
-
     Wraps an RDD in a barrier stage, which forces Spark to launch tasks of this stage together.
     :class:`RDDBarrier` instances are created by :func:`RDD.barrier`.
 
     .. versionadded:: 2.4.0
+
+    Notes
+    -----
+    This API is experimental
     """
 
     def __init__(self, rdd):
@@ -2564,14 +2850,16 @@ def __init__(self, rdd):
 
     def mapPartitions(self, f, preservesPartitioning=False):
         """
-        .. note:: Experimental
-
         Returns a new RDD by applying a function to each partition of the wrapped RDD,
         where tasks are launched together in a barrier stage.
         The interface is the same as :func:`RDD.mapPartitions`.
         Please see the API doc there.
 
         .. versionadded:: 2.4.0
+
+        Notes
+        -----
+        This API is experimental
         """
         def func(s, iterator):
             return f(iterator)
@@ -2579,8 +2867,6 @@ def func(s, iterator):
 
     def mapPartitionsWithIndex(self, f, preservesPartitioning=False):
         """
-        .. note:: Experimental
-
         Returns a new RDD by applying a function to each partition of the wrapped RDD, while
         tracking the index of the original partition. And all tasks are launched together
         in a barrier stage.
@@ -2588,6 +2874,10 @@ def mapPartitionsWithIndex(self, f, preservesPartitioning=False):
         Please see the API doc there.
 
         .. versionadded:: 3.0.0
+
+        Notes
+        -----
+        This API is experimental
         """
         return PipelinedRDD(self.rdd, f, preservesPartitioning, isFromBarrier=True)
 
@@ -2595,6 +2885,8 @@ def mapPartitionsWithIndex(self, f, preservesPartitioning=False):
 class PipelinedRDD(RDD):
 
     """
+    Examples
+    --------
     Pipelined maps:
 
     >>> rdd = sc.parallelize([1, 2, 3, 4])
@@ -2604,6 +2896,7 @@ class PipelinedRDD(RDD):
     [4, 8, 12, 16]
 
     Pipelined reduces:
+
     >>> from operator import add
     >>> rdd.map(lambda x: 2 * x).reduce(add)
     20
diff --git a/python/pyspark/rdd.pyi b/python/pyspark/rdd.pyi
new file mode 100644
index 0000000000000..a277cd9f7edae
--- /dev/null
+++ b/python/pyspark/rdd.pyi
@@ -0,0 +1,483 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import overload
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Generic,
+    Hashable,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    Tuple,
+    Union,
+    TypeVar,
+)
+from typing_extensions import Literal
+
+from numpy import int32, int64, float32, float64, ndarray  # type: ignore[import]
+
+from pyspark._typing import SupportsOrdering
+from pyspark.sql.pandas._typing import (
+    PandasScalarUDFType,
+    PandasScalarIterUDFType,
+    PandasGroupedMapUDFType,
+    PandasCogroupedMapUDFType,
+    PandasGroupedAggUDFType,
+    PandasMapIterUDFType,
+)
+import pyspark.context
+from pyspark.resultiterable import ResultIterable
+from pyspark.serializers import Serializer
+from pyspark.storagelevel import StorageLevel
+from pyspark.resource.requests import (  # noqa: F401
+    ExecutorResourceRequests,
+    TaskResourceRequests,
+)
+from pyspark.resource.profile import ResourceProfile
+from pyspark.statcounter import StatCounter
+from pyspark.sql.dataframe import DataFrame
+from pyspark.sql.types import StructType
+from pyspark.sql._typing import RowLike
+from py4j.java_gateway import JavaObject  # type: ignore[import]
+
+T = TypeVar("T")
+U = TypeVar("U")
+K = TypeVar("K", bound=Hashable)
+V = TypeVar("V")
+V1 = TypeVar("V1")
+V2 = TypeVar("V2")
+V3 = TypeVar("V3")
+O = TypeVar("O", bound=SupportsOrdering)
+NumberOrArray = TypeVar(
+    "NumberOrArray", float, int, complex, int32, int64, float32, float64, ndarray
+)
+
+def portable_hash(x: Hashable) -> int: ...
+
+class PythonEvalType:
+    NON_UDF: Literal[0]
+    SQL_BATCHED_UDF: Literal[100]
+    SQL_SCALAR_PANDAS_UDF: PandasScalarUDFType
+    SQL_GROUPED_MAP_PANDAS_UDF: PandasGroupedMapUDFType
+    SQL_GROUPED_AGG_PANDAS_UDF: PandasGroupedAggUDFType
+    SQL_WINDOW_AGG_PANDAS_UDF: Literal[203]
+    SQL_SCALAR_PANDAS_ITER_UDF: PandasScalarIterUDFType
+    SQL_MAP_PANDAS_ITER_UDF: PandasMapIterUDFType
+    SQL_COGROUPED_MAP_PANDAS_UDF: PandasCogroupedMapUDFType
+
+class BoundedFloat(float):
+    def __new__(
+        cls, mean: float, confidence: float, low: float, high: float
+    ) -> BoundedFloat: ...
+
+class Partitioner:
+    numPartitions: int
+    partitionFunc: Callable[[Any], int]
+    def __init__(
+        self, numPartitions: int, partitionFunc: Callable[[Any], int]
+    ) -> None: ...
+    def __eq__(self, other: Any) -> bool: ...
+    def __call__(self, k: Any) -> int: ...
+
+class RDD(Generic[T]):
+    is_cached: bool
+    is_checkpointed: bool
+    ctx: pyspark.context.SparkContext
+    partitioner: Optional[Partitioner]
+    def __init__(
+        self,
+        jrdd: JavaObject,
+        ctx: pyspark.context.SparkContext,
+        jrdd_deserializer: Serializer = ...,
+    ) -> None: ...
+    def id(self) -> int: ...
+    def __getnewargs__(self) -> Any: ...
+    @property
+    def context(self) -> pyspark.context.SparkContext: ...
+    def cache(self) -> RDD[T]: ...
+    def persist(self, storageLevel: StorageLevel = ...) -> RDD[T]: ...
+    def unpersist(self, blocking: bool = ...) -> RDD[T]: ...
+    def checkpoint(self) -> None: ...
+    def isCheckpointed(self) -> bool: ...
+    def localCheckpoint(self) -> None: ...
+    def isLocallyCheckpointed(self) -> bool: ...
+    def getCheckpointFile(self) -> Optional[str]: ...
+    def map(self, f: Callable[[T], U], preservesPartitioning: bool = ...) -> RDD[U]: ...
+    def flatMap(
+        self, f: Callable[[T], Iterable[U]], preservesPartitioning: bool = ...
+    ) -> RDD[U]: ...
+    def mapPartitions(
+        self, f: Callable[[Iterable[T]], Iterable[U]], preservesPartitioning: bool = ...
+    ) -> RDD[U]: ...
+    def mapPartitionsWithIndex(
+        self,
+        f: Callable[[int, Iterable[T]], Iterable[U]],
+        preservesPartitioning: bool = ...,
+    ) -> RDD[U]: ...
+    def mapPartitionsWithSplit(
+        self,
+        f: Callable[[int, Iterable[T]], Iterable[U]],
+        preservesPartitioning: bool = ...,
+    ) -> RDD[U]: ...
+    def getNumPartitions(self) -> int: ...
+    def filter(self, f: Callable[[T], bool]) -> RDD[T]: ...
+    def distinct(self, numPartitions: Optional[int] = ...) -> RDD[T]: ...
+    def sample(
+        self, withReplacement: bool, fraction: float, seed: Optional[int] = ...
+    ) -> RDD[T]: ...
+    def randomSplit(
+        self, weights: List[Union[int, float]], seed: Optional[int] = ...
+    ) -> List[RDD[T]]: ...
+    def takeSample(
+        self, withReplacement: bool, num: int, seed: Optional[int] = ...
+    ) -> List[T]: ...
+    def union(self, other: RDD[U]) -> RDD[Union[T, U]]: ...
+    def intersection(self, other: RDD[T]) -> RDD[T]: ...
+    def __add__(self, other: RDD[T]) -> RDD[T]: ...
+    @overload
+    def repartitionAndSortWithinPartitions(
+        self: RDD[Tuple[O, V]],
+        numPartitions: Optional[int] = ...,
+        partitionFunc: Callable[[O], int] = ...,
+        ascending: bool = ...,
+    ) -> RDD[Tuple[O, V]]: ...
+    @overload
+    def repartitionAndSortWithinPartitions(
+        self: RDD[Tuple[K, V]],
+        numPartitions: Optional[int],
+        partitionFunc: Callable[[K], int],
+        ascending: bool,
+        keyfunc: Callable[[K], O],
+    ) -> RDD[Tuple[K, V]]: ...
+    @overload
+    def repartitionAndSortWithinPartitions(
+        self: RDD[Tuple[K, V]],
+        numPartitions: Optional[int] = ...,
+        partitionFunc: Callable[[K], int] = ...,
+        ascending: bool = ...,
+        *,
+        keyfunc: Callable[[K], O]
+    ) -> RDD[Tuple[K, V]]: ...
+    @overload
+    def sortByKey(
+        self: RDD[Tuple[O, V]],
+        ascending: bool = ...,
+        numPartitions: Optional[int] = ...,
+    ) -> RDD[Tuple[K, V]]: ...
+    @overload
+    def sortByKey(
+        self: RDD[Tuple[K, V]],
+        ascending: bool,
+        numPartitions: int,
+        keyfunc: Callable[[K], O],
+    ) -> RDD[Tuple[K, V]]: ...
+    @overload
+    def sortByKey(
+        self: RDD[Tuple[K, V]],
+        ascending: bool = ...,
+        numPartitions: Optional[int] = ...,
+        *,
+        keyfunc: Callable[[K], O]
+    ) -> RDD[Tuple[K, V]]: ...
+    def sortBy(
+        self: RDD[T],
+        keyfunc: Callable[[T], O],
+        ascending: bool = ...,
+        numPartitions: Optional[int] = ...,
+    ) -> RDD[T]: ...
+    def glom(self) -> RDD[List[T]]: ...
+    def cartesian(self, other: RDD[U]) -> RDD[Tuple[T, U]]: ...
+    def groupBy(
+        self,
+        f: Callable[[T], K],
+        numPartitions: Optional[int] = ...,
+        partitionFunc: Callable[[K], int] = ...,
+    ) -> RDD[Tuple[K, Iterable[T]]]: ...
+    def pipe(
+        self, command: str, env: Optional[Dict[str, str]] = ..., checkCode: bool = ...
+    ) -> RDD[str]: ...
+    def foreach(self, f: Callable[[T], None]) -> None: ...
+    def foreachPartition(self, f: Callable[[Iterable[T]], None]) -> None: ...
+    def collect(self) -> List[T]: ...
+    def collectWithJobGroup(
+        self, groupId: str, description: str, interruptOnCancel: bool = ...
+    ) -> List[T]: ...
+    def reduce(self, f: Callable[[T, T], T]) -> T: ...
+    def treeReduce(self, f: Callable[[T, T], T], depth: int = ...) -> T: ...
+    def fold(self, zeroValue: T, op: Callable[[T, T], T]) -> T: ...
+    def aggregate(
+        self, zeroValue: U, seqOp: Callable[[U, T], U], combOp: Callable[[U, U], U]
+    ) -> U: ...
+    def treeAggregate(
+        self,
+        zeroValue: U,
+        seqOp: Callable[[U, T], U],
+        combOp: Callable[[U, U], U],
+        depth: int = ...,
+    ) -> U: ...
+    @overload
+    def max(self: RDD[O]) -> O: ...
+    @overload
+    def max(self, key: Callable[[T], O]) -> T: ...
+    @overload
+    def min(self: RDD[O]) -> O: ...
+    @overload
+    def min(self, key: Callable[[T], O]) -> T: ...
+    def sum(self: RDD[NumberOrArray]) -> NumberOrArray: ...
+    def count(self) -> int: ...
+    def stats(self: RDD[NumberOrArray]) -> StatCounter: ...
+    def histogram(self, buckets: List[T]) -> Tuple[List[T], List[int]]: ...
+    def mean(self: RDD[NumberOrArray]) -> NumberOrArray: ...
+    def variance(self: RDD[NumberOrArray]) -> NumberOrArray: ...
+    def stdev(self: RDD[NumberOrArray]) -> NumberOrArray: ...
+    def sampleStdev(self: RDD[NumberOrArray]) -> NumberOrArray: ...
+    def sampleVariance(self: RDD[NumberOrArray]) -> NumberOrArray: ...
+    def countByValue(self: RDD[K]) -> Dict[K, int]: ...
+    @overload
+    def top(self: RDD[O], num: int) -> List[O]: ...
+    @overload
+    def top(self: RDD[T], num: int, key: Callable[[T], O]) -> List[T]: ...
+    @overload
+    def takeOrdered(self: RDD[O], num: int) -> List[O]: ...
+    @overload
+    def takeOrdered(self: RDD[T], num: int, key: Callable[[T], O]) -> List[T]: ...
+    def take(self, num: int) -> List[T]: ...
+    def first(self) -> T: ...
+    def isEmpty(self) -> bool: ...
+    def saveAsNewAPIHadoopDataset(
+        self: RDD[Tuple[K, V]],
+        conf: Dict[str, str],
+        keyConverter: Optional[str] = ...,
+        valueConverter: Optional[str] = ...,
+    ) -> None: ...
+    def saveAsNewAPIHadoopFile(
+        self: RDD[Tuple[K, V]],
+        path: str,
+        outputFormatClass: str,
+        keyClass: Optional[str] = ...,
+        valueClass: Optional[str] = ...,
+        keyConverter: Optional[str] = ...,
+        valueConverter: Optional[str] = ...,
+        conf: Optional[Dict[str, str]] = ...,
+    ) -> None: ...
+    def saveAsHadoopDataset(
+        self: RDD[Tuple[K, V]],
+        conf: Dict[str, str],
+        keyConverter: Optional[str] = ...,
+        valueConverter: Optional[str] = ...,
+    ) -> None: ...
+    def saveAsHadoopFile(
+        self: RDD[Tuple[K, V]],
+        path: str,
+        outputFormatClass: str,
+        keyClass: Optional[str] = ...,
+        valueClass: Optional[str] = ...,
+        keyConverter: Optional[str] = ...,
+        valueConverter: Optional[str] = ...,
+        conf: Optional[str] = ...,
+        compressionCodecClass: Optional[str] = ...,
+    ) -> None: ...
+    def saveAsSequenceFile(
+        self: RDD[Tuple[K, V]], path: str, compressionCodecClass: Optional[str] = ...
+    ) -> None: ...
+    def saveAsPickleFile(self, path: str, batchSize: int = ...) -> None: ...
+    def saveAsTextFile(
+        self, path: str, compressionCodecClass: Optional[str] = ...
+    ) -> None: ...
+    def collectAsMap(self: RDD[Tuple[K, V]]) -> Dict[K, V]: ...
+    def keys(self: RDD[Tuple[K, V]]) -> RDD[K]: ...
+    def values(self: RDD[Tuple[K, V]]) -> RDD[V]: ...
+    def reduceByKey(
+        self: RDD[Tuple[K, V]],
+        func: Callable[[V, V], V],
+        numPartitions: Optional[int] = ...,
+        partitionFunc: Callable[[K], int] = ...,
+    ) -> RDD[Tuple[K, V]]: ...
+    def reduceByKeyLocally(
+        self: RDD[Tuple[K, V]], func: Callable[[V, V], V]
+    ) -> Dict[K, V]: ...
+    def countByKey(self: RDD[Tuple[K, V]]) -> Dict[K, int]: ...
+    def join(
+        self: RDD[Tuple[K, V]],
+        other: RDD[Tuple[K, U]],
+        numPartitions: Optional[int] = ...,
+    ) -> RDD[Tuple[K, Tuple[V, U]]]: ...
+    def leftOuterJoin(
+        self: RDD[Tuple[K, V]],
+        other: RDD[Tuple[K, U]],
+        numPartitions: Optional[int] = ...,
+    ) -> RDD[Tuple[K, Tuple[V, Optional[U]]]]: ...
+    def rightOuterJoin(
+        self: RDD[Tuple[K, V]],
+        other: RDD[Tuple[K, U]],
+        numPartitions: Optional[int] = ...,
+    ) -> RDD[Tuple[K, Tuple[Optional[V], U]]]: ...
+    def fullOuterJoin(
+        self: RDD[Tuple[K, V]],
+        other: RDD[Tuple[K, U]],
+        numPartitions: Optional[int] = ...,
+    ) -> RDD[Tuple[K, Tuple[Optional[V], Optional[U]]]]: ...
+    def partitionBy(
+        self: RDD[Tuple[K, V]],
+        numPartitions: int,
+        partitionFunc: Callable[[K], int] = ...,
+    ) -> RDD[Tuple[K, V]]: ...
+    def combineByKey(
+        self: RDD[Tuple[K, V]],
+        createCombiner: Callable[[V], U],
+        mergeValue: Callable[[U, V], U],
+        mergeCombiners: Callable[[U, U], U],
+        numPartitions: Optional[int] = ...,
+        partitionFunc: Callable[[K], int] = ...,
+    ) -> RDD[Tuple[K, U]]: ...
+    def aggregateByKey(
+        self: RDD[Tuple[K, V]],
+        zeroValue: U,
+        seqFunc: Callable[[U, V], U],
+        combFunc: Callable[[U, U], U],
+        numPartitions: Optional[int] = ...,
+        partitionFunc: Callable[[K], int] = ...,
+    ) -> RDD[Tuple[K, U]]: ...
+    def foldByKey(
+        self: RDD[Tuple[K, V]],
+        zeroValue: V,
+        func: Callable[[V, V], V],
+        numPartitions: Optional[int] = ...,
+        partitionFunc: Callable[[K], int] = ...,
+    ) -> RDD[Tuple[K, V]]: ...
+    def groupByKey(
+        self: RDD[Tuple[K, V]],
+        numPartitions: Optional[int] = ...,
+        partitionFunc: Callable[[K], int] = ...,
+    ) -> RDD[Tuple[K, Iterable[V]]]: ...
+    def flatMapValues(
+        self: RDD[Tuple[K, V]], f: Callable[[V], Iterable[U]]
+    ) -> RDD[Tuple[K, U]]: ...
+    def mapValues(self: RDD[Tuple[K, V]], f: Callable[[V], U]) -> RDD[Tuple[K, U]]: ...
+    @overload
+    def groupWith(
+        self: RDD[Tuple[K, V]], __o: RDD[Tuple[K, V1]]
+    ) -> RDD[Tuple[K, Tuple[ResultIterable[V], ResultIterable[V1]]]]: ...
+    @overload
+    def groupWith(
+        self: RDD[Tuple[K, V]], __o1: RDD[Tuple[K, V1]], __o2: RDD[Tuple[K, V2]]
+    ) -> RDD[
+        Tuple[K, Tuple[ResultIterable[V], ResultIterable[V1], ResultIterable[V2]]]
+    ]: ...
+    @overload
+    def groupWith(
+        self: RDD[Tuple[K, V]],
+        other1: RDD[Tuple[K, V1]],
+        other2: RDD[Tuple[K, V2]],
+        other3: RDD[Tuple[K, V3]],
+    ) -> RDD[
+        Tuple[
+            K,
+            Tuple[
+                ResultIterable[V],
+                ResultIterable[V1],
+                ResultIterable[V2],
+                ResultIterable[V3],
+            ],
+        ]
+    ]: ...
+    def cogroup(
+        self: RDD[Tuple[K, V]],
+        other: RDD[Tuple[K, U]],
+        numPartitions: Optional[int] = ...,
+    ) -> RDD[Tuple[K, Tuple[ResultIterable[V], ResultIterable[U]]]]: ...
+    def sampleByKey(
+        self: RDD[Tuple[K, V]],
+        withReplacement: bool,
+        fractions: Dict[K, Union[float, int]],
+        seed: Optional[int] = ...,
+    ) -> RDD[Tuple[K, V]]: ...
+    def subtractByKey(
+        self: RDD[Tuple[K, V]],
+        other: RDD[Tuple[K, U]],
+        numPartitions: Optional[int] = ...,
+    ) -> RDD[Tuple[K, V]]: ...
+    def subtract(
+        self: RDD[T], other: RDD[T], numPartitions: Optional[int] = ...
+    ) -> RDD[T]: ...
+    def keyBy(self: RDD[T], f: Callable[[T], K]) -> RDD[Tuple[K, T]]: ...
+    def repartition(self, numPartitions: int) -> RDD[T]: ...
+    def coalesce(self, numPartitions: int, shuffle: bool = ...) -> RDD[T]: ...
+    def zip(self, other: RDD[U]) -> RDD[Tuple[T, U]]: ...
+    def zipWithIndex(self) -> RDD[Tuple[T, int]]: ...
+    def zipWithUniqueId(self) -> RDD[Tuple[T, int]]: ...
+    def name(self) -> str: ...
+    def setName(self, name: str) -> RDD[T]: ...
+    def toDebugString(self) -> bytes: ...
+    def getStorageLevel(self) -> StorageLevel: ...
+    def lookup(self: RDD[Tuple[K, V]], key: K) -> List[V]: ...
+    def countApprox(self, timeout: int, confidence: float = ...) -> int: ...
+    def sumApprox(
+        self: RDD[Union[float, int]], timeout: int, confidence: float = ...
+    ) -> BoundedFloat: ...
+    def meanApprox(
+        self: RDD[Union[float, int]], timeout: int, confidence: float = ...
+    ) -> BoundedFloat: ...
+    def countApproxDistinct(self, relativeSD: float = ...) -> int: ...
+    def toLocalIterator(self, prefetchPartitions: bool = ...) -> Iterator[T]: ...
+    def barrier(self: RDD[T]) -> RDDBarrier[T]: ...
+    def withResources(self: RDD[T], profile: ResourceProfile) -> RDD[T]: ...
+    def getResourceProfile(self) -> Optional[ResourceProfile]: ...
+    @overload
+    def toDF(
+        self: RDD[RowLike],
+        schema: Optional[List[str]] = ...,
+        sampleRatio: Optional[float] = ...,
+    ) -> DataFrame: ...
+    @overload
+    def toDF(self: RDD[RowLike], schema: Optional[StructType] = ...) -> DataFrame: ...
+
+class RDDBarrier(Generic[T]):
+    rdd: RDD[T]
+    def __init__(self, rdd: RDD[T]) -> None: ...
+    def mapPartitions(
+        self, f: Callable[[Iterable[T]], Iterable[U]], preservesPartitioning: bool = ...
+    ) -> RDD[U]: ...
+    def mapPartitionsWithIndex(
+        self,
+        f: Callable[[int, Iterable[T]], Iterable[U]],
+        preservesPartitioning: bool = ...,
+    ) -> RDD[U]: ...
+
+class PipelinedRDD(RDD[U], Generic[T, U]):
+    func: Callable[[T], U]
+    preservesPartitioning: bool
+    is_cached: bool
+    is_checkpointed: bool
+    ctx: pyspark.context.SparkContext
+    prev: RDD[T]
+    partitioner: Optional[Partitioner]
+    is_barrier: bool
+    def __init__(
+        self,
+        prev: RDD[T],
+        func: Callable[[Iterable[T]], Iterable[U]],
+        preservesPartitioning: bool = ...,
+        isFromBarrier: bool = ...,
+    ) -> None: ...
+    def getNumPartitions(self) -> int: ...
+    def id(self) -> int: ...
diff --git a/python/pyspark/resource/__init__.py b/python/pyspark/resource/__init__.py
index 89070ec4adc7e..b5f4c4a6b1825 100644
--- a/python/pyspark/resource/__init__.py
+++ b/python/pyspark/resource/__init__.py
@@ -18,12 +18,13 @@
 """
 APIs to let users manipulate resource requirements.
 """
-from pyspark.resource.executorrequests import ExecutorResourceRequest, ExecutorResourceRequests
-from pyspark.resource.taskrequests import TaskResourceRequest, TaskResourceRequests
-from pyspark.resource.resourceprofilebuilder import ResourceProfileBuilder
-from pyspark.resource.resourceprofile import ResourceProfile
+from pyspark.resource.information import ResourceInformation
+from pyspark.resource.requests import TaskResourceRequest, TaskResourceRequests, \
+    ExecutorResourceRequest, ExecutorResourceRequests
+from pyspark.resource.profile import ResourceProfile, ResourceProfileBuilder
 
 __all__ = [
     "TaskResourceRequest", "TaskResourceRequests", "ExecutorResourceRequest",
-    "ExecutorResourceRequests", "ResourceProfile", "ResourceProfileBuilder",
+    "ExecutorResourceRequests", "ResourceProfile", "ResourceInformation",
+    "ResourceProfileBuilder",
 ]
diff --git a/python/pyspark/resource/executorrequests.py b/python/pyspark/resource/executorrequests.py
deleted file mode 100644
index 91a195c94b6e5..0000000000000
--- a/python/pyspark/resource/executorrequests.py
+++ /dev/null
@@ -1,169 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-from pyspark.resource.taskrequests import TaskResourceRequest
-from pyspark.util import _parse_memory
-
-
-class ExecutorResourceRequest(object):
-    """
-    .. note:: Evolving
-
-    An Executor resource request. This is used in conjunction with the ResourceProfile to
-    programmatically specify the resources needed for an RDD that will be applied at the
-    stage level.
-
-    This is used to specify what the resource requirements are for an Executor and how
-    Spark can find out specific details about those resources. Not all the parameters are
-    required for every resource type. Resources like GPUs are supported and have same limitations
-    as using the global spark configs spark.executor.resource.gpu.*. The amount, discoveryScript,
-    and vendor parameters for resources are all the same parameters a user would specify through the
-    configs: spark.executor.resource.{resourceName}.{amount, discoveryScript, vendor}.
-
-    For instance, a user wants to allocate an Executor with GPU resources on YARN. The user has
-    to specify the resource name (gpu), the amount or number of GPUs per Executor,
-    the discovery script would be specified so that when the Executor starts up it can
-    discovery what GPU addresses are available for it to use because YARN doesn't tell
-    Spark that, then vendor would not be used because its specific for Kubernetes.
-
-    See the configuration and cluster specific docs for more details.
-
-    Use `pyspark.ExecutorResourceRequests` class as a convenience API.
-
-    :param resourceName: Name of the resource
-    :param amount: Amount requesting
-    :param discoveryScript: Optional script used to discover the resources. This is required on some
-        cluster managers that don't tell Spark the addresses of the resources
-        allocated. The script runs on Executors startup to discover the addresses
-        of the resources available.
-    :param vendor: Vendor, required for some cluster managers
-
-    .. versionadded:: 3.1.0
-    """
-    def __init__(self, resourceName, amount, discoveryScript="", vendor=""):
-        self._name = resourceName
-        self._amount = amount
-        self._discovery_script = discoveryScript
-        self._vendor = vendor
-
-    @property
-    def resourceName(self):
-        return self._name
-
-    @property
-    def amount(self):
-        return self._amount
-
-    @property
-    def discoveryScript(self):
-        return self._discovery_script
-
-    @property
-    def vendor(self):
-        return self._vendor
-
-
-class ExecutorResourceRequests(object):
-
-    """
-    .. note:: Evolving
-
-    A set of Executor resource requests. This is used in conjunction with the
-    :class:`pyspark.resource.ResourceProfileBuilder` to programmatically specify the
-    resources needed for an RDD that will be applied at the stage level.
-
-    .. versionadded:: 3.1.0
-    """
-    _CORES = "cores"
-    _MEMORY = "memory"
-    _OVERHEAD_MEM = "memoryOverhead"
-    _PYSPARK_MEM = "pyspark.memory"
-
-    def __init__(self, _jvm=None, _requests=None):
-        from pyspark import SparkContext
-        _jvm = _jvm or SparkContext._jvm
-        if _jvm is not None:
-            self._java_executor_resource_requests = \
-                _jvm.org.apache.spark.resource.ExecutorResourceRequests()
-            if _requests is not None:
-                for k, v in _requests.items():
-                    if k == self._MEMORY:
-                        self._java_executor_resource_requests.memory(str(v.amount))
-                    elif k == self._OVERHEAD_MEM:
-                        self._java_executor_resource_requests.memoryOverhead(str(v.amount))
-                    elif k == self._PYSPARK_MEM:
-                        self._java_executor_resource_requests.pysparkMemory(str(v.amount))
-                    elif k == self._CORES:
-                        self._java_executor_resource_requests.cores(v.amount)
-                    else:
-                        self._java_executor_resource_requests.resource(v.resourceName, v.amount,
-                                                                       v.discoveryScript, v.vendor)
-        else:
-            self._java_executor_resource_requests = None
-            self._executor_resources = {}
-
-    def memory(self, amount):
-        if self._java_executor_resource_requests is not None:
-            self._java_executor_resource_requests.memory(amount)
-        else:
-            self._executor_resources[self._MEMORY] = ExecutorResourceRequest(self._MEMORY,
-                                                                             _parse_memory(amount))
-        return self
-
-    def memoryOverhead(self, amount):
-        if self._java_executor_resource_requests is not None:
-            self._java_executor_resource_requests.memoryOverhead(amount)
-        else:
-            self._executor_resources[self._OVERHEAD_MEM] = \
-                ExecutorResourceRequest(self._OVERHEAD_MEM, _parse_memory(amount))
-        return self
-
-    def pysparkMemory(self, amount):
-        if self._java_executor_resource_requests is not None:
-            self._java_executor_resource_requests.pysparkMemory(amount)
-        else:
-            self._executor_resources[self._PYSPARK_MEM] = \
-                ExecutorResourceRequest(self._PYSPARK_MEM, _parse_memory(amount))
-        return self
-
-    def cores(self, amount):
-        if self._java_executor_resource_requests is not None:
-            self._java_executor_resource_requests.cores(amount)
-        else:
-            self._executor_resources[self._CORES] = ExecutorResourceRequest(self._CORES, amount)
-        return self
-
-    def resource(self, resourceName, amount, discoveryScript="", vendor=""):
-        if self._java_executor_resource_requests is not None:
-            self._java_executor_resource_requests.resource(resourceName, amount, discoveryScript,
-                                                           vendor)
-        else:
-            self._executor_resources[resourceName] = \
-                ExecutorResourceRequest(resourceName, amount, discoveryScript, vendor)
-        return self
-
-    @property
-    def requests(self):
-        if self._java_executor_resource_requests is not None:
-            result = {}
-            execRes = self._java_executor_resource_requests.requestsJMap()
-            for k, v in execRes.items():
-                result[k] = ExecutorResourceRequest(v.resourceName(), v.amount(),
-                                                    v.discoveryScript(), v.vendor())
-            return result
-        else:
-            return self._executor_resources
diff --git a/python/pyspark/resource/information.py b/python/pyspark/resource/information.py
new file mode 100644
index 0000000000000..40209cae9b3d9
--- /dev/null
+++ b/python/pyspark/resource/information.py
@@ -0,0 +1,51 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+class ResourceInformation(object):
+
+    """
+    Class to hold information about a type of Resource. A resource could be a GPU, FPGA, etc.
+    The array of addresses are resource specific and its up to the user to interpret the address.
+
+    One example is GPUs, where the addresses would be the indices of the GPUs
+
+    .. versionadded:: 3.0.0
+
+    Parameters
+    ----------
+    name : str
+        the name of the resource
+    addresses : list
+        an array of strings describing the addresses of the resource
+
+    Notes
+    -----
+    This API is evolving.
+    """
+
+    def __init__(self, name, addresses):
+        self._name = name
+        self._addresses = addresses
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def addresses(self):
+        return self._addresses
diff --git a/python/pyspark/resource/information.pyi b/python/pyspark/resource/information.pyi
new file mode 100644
index 0000000000000..5474afa659746
--- /dev/null
+++ b/python/pyspark/resource/information.pyi
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import List
+
+class ResourceInformation:
+    def __init__(self, name: str, addresses: List[str]) -> None: ...
+    @property
+    def name(self) -> str: ...
+    @property
+    def addresses(self) -> List[str]: ...
diff --git a/python/pyspark/resource/profile.py b/python/pyspark/resource/profile.py
new file mode 100644
index 0000000000000..38a68bc74d97e
--- /dev/null
+++ b/python/pyspark/resource/profile.py
@@ -0,0 +1,172 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from pyspark.resource.requests import TaskResourceRequest, TaskResourceRequests, \
+    ExecutorResourceRequests, ExecutorResourceRequest
+
+
+class ResourceProfile(object):
+
+    """
+    Resource profile to associate with an RDD. A :class:`pyspark.resource.ResourceProfile`
+    allows the user to specify executor and task requirements for an RDD that will get
+    applied during a stage. This allows the user to change the resource requirements between
+    stages. This is meant to be immutable so user cannot change it after building.
+
+    .. versionadded:: 3.1.0
+
+    Notes
+    -----
+    This API is evolving.
+    """
+
+    def __init__(self, _java_resource_profile=None, _exec_req=None, _task_req=None):
+        if _java_resource_profile is not None:
+            self._java_resource_profile = _java_resource_profile
+        else:
+            self._java_resource_profile = None
+            self._executor_resource_requests = _exec_req or {}
+            self._task_resource_requests = _task_req or {}
+
+    @property
+    def id(self):
+        if self._java_resource_profile is not None:
+            return self._java_resource_profile.id()
+        else:
+            raise RuntimeError("SparkContext must be created to get the id, get the id "
+                               "after adding the ResourceProfile to an RDD")
+
+    @property
+    def taskResources(self):
+        if self._java_resource_profile is not None:
+            taskRes = self._java_resource_profile.taskResourcesJMap()
+            result = {}
+            for k, v in taskRes.items():
+                result[k] = TaskResourceRequest(v.resourceName(), v.amount())
+            return result
+        else:
+            return self._task_resource_requests
+
+    @property
+    def executorResources(self):
+        if self._java_resource_profile is not None:
+            execRes = self._java_resource_profile.executorResourcesJMap()
+            result = {}
+            for k, v in execRes.items():
+                result[k] = ExecutorResourceRequest(v.resourceName(), v.amount(),
+                                                    v.discoveryScript(), v.vendor())
+            return result
+        else:
+            return self._executor_resource_requests
+
+
+class ResourceProfileBuilder(object):
+
+    """
+    Resource profile Builder to build a resource profile to associate with an RDD.
+    A ResourceProfile allows the user to specify executor and task requirements for
+    an RDD that will get applied during a stage. This allows the user to change the
+    resource requirements between stages.
+
+    .. versionadded:: 3.1.0
+
+    Notes
+    -----
+    This API is evolving.
+    """
+
+    def __init__(self):
+        from pyspark.context import SparkContext
+        _jvm = SparkContext._jvm
+        if _jvm is not None:
+            self._jvm = _jvm
+            self._java_resource_profile_builder = \
+                _jvm.org.apache.spark.resource.ResourceProfileBuilder()
+        else:
+            self._jvm = None
+            self._java_resource_profile_builder = None
+            self._executor_resource_requests = {}
+            self._task_resource_requests = {}
+
+    def require(self, resourceRequest):
+        if isinstance(resourceRequest, TaskResourceRequests):
+            if self._java_resource_profile_builder is not None:
+                if resourceRequest._java_task_resource_requests is not None:
+                    self._java_resource_profile_builder.require(
+                        resourceRequest._java_task_resource_requests)
+                else:
+                    taskReqs = TaskResourceRequests(self._jvm, resourceRequest.requests)
+                    self._java_resource_profile_builder.require(
+                        taskReqs._java_task_resource_requests)
+            else:
+                self._task_resource_requests.update(resourceRequest.requests)
+        else:
+            if self._java_resource_profile_builder is not None:
+                if resourceRequest._java_executor_resource_requests is not None:
+                    self._java_resource_profile_builder.require(
+                        resourceRequest._java_executor_resource_requests)
+                else:
+                    execReqs = ExecutorResourceRequests(self._jvm, resourceRequest.requests)
+                    self._java_resource_profile_builder.require(
+                        execReqs._java_executor_resource_requests)
+            else:
+                self._executor_resource_requests.update(resourceRequest.requests)
+        return self
+
+    def clearExecutorResourceRequests(self):
+        if self._java_resource_profile_builder is not None:
+            self._java_resource_profile_builder.clearExecutorResourceRequests()
+        else:
+            self._executor_resource_requests = {}
+
+    def clearTaskResourceRequests(self):
+        if self._java_resource_profile_builder is not None:
+            self._java_resource_profile_builder.clearTaskResourceRequests()
+        else:
+            self._task_resource_requests = {}
+
+    @property
+    def taskResources(self):
+        if self._java_resource_profile_builder is not None:
+            taskRes = self._java_resource_profile_builder.taskResourcesJMap()
+            result = {}
+            for k, v in taskRes.items():
+                result[k] = TaskResourceRequest(v.resourceName(), v.amount())
+            return result
+        else:
+            return self._task_resource_requests
+
+    @property
+    def executorResources(self):
+        if self._java_resource_profile_builder is not None:
+            result = {}
+            execRes = self._java_resource_profile_builder.executorResourcesJMap()
+            for k, v in execRes.items():
+                result[k] = ExecutorResourceRequest(v.resourceName(), v.amount(),
+                                                    v.discoveryScript(), v.vendor())
+            return result
+        else:
+            return self._executor_resource_requests
+
+    @property
+    def build(self):
+        if self._java_resource_profile_builder is not None:
+            jresourceProfile = self._java_resource_profile_builder.build()
+            return ResourceProfile(_java_resource_profile=jresourceProfile)
+        else:
+            return ResourceProfile(_exec_req=self._executor_resource_requests,
+                                   _task_req=self._task_resource_requests)
diff --git a/python/pyspark/resource/profile.pyi b/python/pyspark/resource/profile.pyi
new file mode 100644
index 0000000000000..c8f23a5cac370
--- /dev/null
+++ b/python/pyspark/resource/profile.pyi
@@ -0,0 +1,60 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from pyspark.resource.requests import (  # noqa: F401
+    ExecutorResourceRequest as ExecutorResourceRequest,
+    ExecutorResourceRequests as ExecutorResourceRequests,
+    TaskResourceRequest as TaskResourceRequest,
+    TaskResourceRequests as TaskResourceRequests,
+)
+from typing import overload, Dict, Union, Optional
+from py4j.java_gateway import JavaObject  # type: ignore[import]
+
+class ResourceProfile:
+    @overload
+    def __init__(
+        self,
+        _java_resource_profile: JavaObject,
+    ) -> None: ...
+    @overload
+    def __init__(
+        self,
+        _java_resource_profile: None = ...,
+        _exec_req: Optional[Dict[str, ExecutorResourceRequest]] = ...,
+        _task_req: Optional[Dict[str, TaskResourceRequest]] = ...,
+    ) -> None: ...
+    @property
+    def id(self) -> int: ...
+    @property
+    def taskResources(self) -> Dict[str, TaskResourceRequest]: ...
+    @property
+    def executorResources(self) -> Dict[str, ExecutorResourceRequest]: ...
+
+class ResourceProfileBuilder:
+    def __init__(self) -> None: ...
+    def require(
+        self, resourceRequest: Union[ExecutorResourceRequest, TaskResourceRequests]
+    ) -> ResourceProfileBuilder: ...
+    def clearExecutorResourceRequests(self) -> None: ...
+    def clearTaskResourceRequests(self) -> None: ...
+    @property
+    def taskResources(self) -> Dict[str, TaskResourceRequest]: ...
+    @property
+    def executorResources(self) -> Dict[str, ExecutorResourceRequest]: ...
+    @property
+    def build(self) -> ResourceProfile: ...
diff --git a/python/pyspark/resource/requests.py b/python/pyspark/resource/requests.py
new file mode 100644
index 0000000000000..4deb22b5948f0
--- /dev/null
+++ b/python/pyspark/resource/requests.py
@@ -0,0 +1,281 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from pyspark.util import _parse_memory
+
+
+class ExecutorResourceRequest(object):
+    """
+    An Executor resource request. This is used in conjunction with the ResourceProfile to
+    programmatically specify the resources needed for an RDD that will be applied at the
+    stage level.
+
+    This is used to specify what the resource requirements are for an Executor and how
+    Spark can find out specific details about those resources. Not all the parameters are
+    required for every resource type. Resources like GPUs are supported and have same limitations
+    as using the global spark configs spark.executor.resource.gpu.*. The amount, discoveryScript,
+    and vendor parameters for resources are all the same parameters a user would specify through the
+    configs: spark.executor.resource.{resourceName}.{amount, discoveryScript, vendor}.
+
+    For instance, a user wants to allocate an Executor with GPU resources on YARN. The user has
+    to specify the resource name (gpu), the amount or number of GPUs per Executor,
+    the discovery script would be specified so that when the Executor starts up it can
+    discovery what GPU addresses are available for it to use because YARN doesn't tell
+    Spark that, then vendor would not be used because its specific for Kubernetes.
+
+    See the configuration and cluster specific docs for more details.
+
+    Use :py:class:`pyspark.ExecutorResourceRequests` class as a convenience API.
+
+    .. versionadded:: 3.1.0
+
+    Parameters
+    ----------
+    resourceName : str
+        Name of the resource
+    amount : str
+        Amount requesting
+    discoveryScript : str, optional
+        Optional script used to discover the resources. This is required on some
+        cluster managers that don't tell Spark the addresses of the resources
+        allocated. The script runs on Executors startup to discover the addresses
+        of the resources available.
+    vendor : str, optional
+        Vendor, required for some cluster managers
+
+    Notes
+    -----
+    This API is evolving.
+    """
+    def __init__(self, resourceName, amount, discoveryScript="", vendor=""):
+        self._name = resourceName
+        self._amount = amount
+        self._discovery_script = discoveryScript
+        self._vendor = vendor
+
+    @property
+    def resourceName(self):
+        return self._name
+
+    @property
+    def amount(self):
+        return self._amount
+
+    @property
+    def discoveryScript(self):
+        return self._discovery_script
+
+    @property
+    def vendor(self):
+        return self._vendor
+
+
+class ExecutorResourceRequests(object):
+
+    """
+    A set of Executor resource requests. This is used in conjunction with the
+    :class:`pyspark.resource.ResourceProfileBuilder` to programmatically specify the
+    resources needed for an RDD that will be applied at the stage level.
+
+    .. versionadded:: 3.1.0
+
+    Notes
+    -----
+    This API is evolving.
+    """
+    _CORES = "cores"
+    _MEMORY = "memory"
+    _OVERHEAD_MEM = "memoryOverhead"
+    _PYSPARK_MEM = "pyspark.memory"
+    _OFFHEAP_MEM = "offHeap"
+
+    def __init__(self, _jvm=None, _requests=None):
+        from pyspark import SparkContext
+        _jvm = _jvm or SparkContext._jvm
+        if _jvm is not None:
+            self._java_executor_resource_requests = \
+                _jvm.org.apache.spark.resource.ExecutorResourceRequests()
+            if _requests is not None:
+                for k, v in _requests.items():
+                    if k == self._MEMORY:
+                        self._java_executor_resource_requests.memory(str(v.amount))
+                    elif k == self._OVERHEAD_MEM:
+                        self._java_executor_resource_requests.memoryOverhead(str(v.amount))
+                    elif k == self._PYSPARK_MEM:
+                        self._java_executor_resource_requests.pysparkMemory(str(v.amount))
+                    elif k == self._CORES:
+                        self._java_executor_resource_requests.cores(v.amount)
+                    else:
+                        self._java_executor_resource_requests.resource(v.resourceName, v.amount,
+                                                                       v.discoveryScript, v.vendor)
+        else:
+            self._java_executor_resource_requests = None
+            self._executor_resources = {}
+
+    def memory(self, amount):
+        if self._java_executor_resource_requests is not None:
+            self._java_executor_resource_requests.memory(amount)
+        else:
+            self._executor_resources[self._MEMORY] = ExecutorResourceRequest(self._MEMORY,
+                                                                             _parse_memory(amount))
+        return self
+
+    def memoryOverhead(self, amount):
+        if self._java_executor_resource_requests is not None:
+            self._java_executor_resource_requests.memoryOverhead(amount)
+        else:
+            self._executor_resources[self._OVERHEAD_MEM] = \
+                ExecutorResourceRequest(self._OVERHEAD_MEM, _parse_memory(amount))
+        return self
+
+    def pysparkMemory(self, amount):
+        if self._java_executor_resource_requests is not None:
+            self._java_executor_resource_requests.pysparkMemory(amount)
+        else:
+            self._executor_resources[self._PYSPARK_MEM] = \
+                ExecutorResourceRequest(self._PYSPARK_MEM, _parse_memory(amount))
+        return self
+
+    def offheapMemory(self, amount):
+        if self._java_executor_resource_requests is not None:
+            self._java_executor_resource_requests.offHeapMemory(amount)
+        else:
+            self._executor_resources[self._OFFHEAP_MEM] = \
+                ExecutorResourceRequest(self._OFFHEAP_MEM, _parse_memory(amount))
+        return self
+
+    def cores(self, amount):
+        if self._java_executor_resource_requests is not None:
+            self._java_executor_resource_requests.cores(amount)
+        else:
+            self._executor_resources[self._CORES] = ExecutorResourceRequest(self._CORES, amount)
+        return self
+
+    def resource(self, resourceName, amount, discoveryScript="", vendor=""):
+        if self._java_executor_resource_requests is not None:
+            self._java_executor_resource_requests.resource(resourceName, amount, discoveryScript,
+                                                           vendor)
+        else:
+            self._executor_resources[resourceName] = \
+                ExecutorResourceRequest(resourceName, amount, discoveryScript, vendor)
+        return self
+
+    @property
+    def requests(self):
+        if self._java_executor_resource_requests is not None:
+            result = {}
+            execRes = self._java_executor_resource_requests.requestsJMap()
+            for k, v in execRes.items():
+                result[k] = ExecutorResourceRequest(v.resourceName(), v.amount(),
+                                                    v.discoveryScript(), v.vendor())
+            return result
+        else:
+            return self._executor_resources
+
+
+class TaskResourceRequest(object):
+    """
+    A task resource request. This is used in conjunction with the
+    :class:`pyspark.resource.ResourceProfile` to programmatically specify the resources
+    needed for an RDD that will be applied at the stage level. The amount is specified
+    as a Double to allow for saying you want more than 1 task per resource. Valid values
+    are less than or equal to 0.5 or whole numbers.
+    Use :class:`pyspark.resource.TaskResourceRequests` class as a convenience API.
+
+    Parameters
+    ----------
+    resourceName : str
+        Name of the resource
+    amount : float
+        Amount requesting as a float to support fractional resource requests.
+        Valid values are less than or equal to 0.5 or whole numbers.
+
+    .. versionadded:: 3.1.0
+
+    Notes
+    -----
+    This API is evolving.
+    """
+    def __init__(self, resourceName, amount):
+        self._name = resourceName
+        self._amount = float(amount)
+
+    @property
+    def resourceName(self):
+        return self._name
+
+    @property
+    def amount(self):
+        return self._amount
+
+
+class TaskResourceRequests(object):
+
+    """
+    A set of task resource requests. This is used in conjunction with the
+    :class:`pyspark.resource.ResourceProfileBuilder` to programmatically specify the resources
+    needed for an RDD that will be applied at the stage level.
+
+    .. versionadded:: 3.1.0
+
+    Notes
+    -----
+    This API is evolving.
+    """
+
+    _CPUS = "cpus"
+
+    def __init__(self, _jvm=None, _requests=None):
+        from pyspark import SparkContext
+        _jvm = _jvm or SparkContext._jvm
+        if _jvm is not None:
+            self._java_task_resource_requests = \
+                SparkContext._jvm.org.apache.spark.resource.TaskResourceRequests()
+            if _requests is not None:
+                for k, v in _requests.items():
+                    if k == self._CPUS:
+                        self._java_task_resource_requests.cpus(int(v.amount))
+                    else:
+                        self._java_task_resource_requests.resource(v.resourceName, v.amount)
+        else:
+            self._java_task_resource_requests = None
+            self._task_resources = {}
+
+    def cpus(self, amount):
+        if self._java_task_resource_requests is not None:
+            self._java_task_resource_requests.cpus(amount)
+        else:
+            self._task_resources[self._CPUS] = TaskResourceRequest(self._CPUS, amount)
+        return self
+
+    def resource(self, resourceName, amount):
+        if self._java_task_resource_requests is not None:
+            self._java_task_resource_requests.resource(resourceName, float(amount))
+        else:
+            self._task_resources[resourceName] = TaskResourceRequest(resourceName, amount)
+        return self
+
+    @property
+    def requests(self):
+        if self._java_task_resource_requests is not None:
+            result = {}
+            taskRes = self._java_task_resource_requests.requestsJMap()
+            for k, v in taskRes.items():
+                result[k] = TaskResourceRequest(v.resourceName(), v.amount())
+            return result
+        else:
+            return self._task_resources
diff --git a/python/pyspark/resource/requests.pyi b/python/pyspark/resource/requests.pyi
new file mode 100644
index 0000000000000..6ba14d65eb516
--- /dev/null
+++ b/python/pyspark/resource/requests.pyi
@@ -0,0 +1,83 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import overload, Dict, Optional
+
+from py4j.java_gateway import JVMView  # type: ignore[import]
+
+class ExecutorResourceRequest:
+    def __init__(
+        self,
+        resourceName: str,
+        amount: int,
+        discoveryScript: str = ...,
+        vendor: str = ...,
+    ) -> None: ...
+    @property
+    def resourceName(self) -> str: ...
+    @property
+    def amount(self) -> int: ...
+    @property
+    def discoveryScript(self) -> str: ...
+    @property
+    def vendor(self) -> str: ...
+
+class ExecutorResourceRequests:
+    @overload
+    def __init__(self, _jvm: JVMView) -> None: ...
+    @overload
+    def __init__(
+        self,
+        _jvm: None = ...,
+        _requests: Optional[Dict[str, ExecutorResourceRequest]] = ...,
+    ) -> None: ...
+    def memory(self, amount: str) -> ExecutorResourceRequests: ...
+    def memoryOverhead(self, amount: str) -> ExecutorResourceRequests: ...
+    def pysparkMemory(self, amount: str) -> ExecutorResourceRequests: ...
+    def offheapMemory(self, amount: str) -> ExecutorResourceRequests: ...
+    def cores(self, amount: int) -> ExecutorResourceRequests: ...
+    def resource(
+        self,
+        resourceName: str,
+        amount: int,
+        discoveryScript: str = ...,
+        vendor: str = ...,
+    ) -> ExecutorResourceRequests: ...
+    @property
+    def requests(self) -> Dict[str, ExecutorResourceRequest]: ...
+
+class TaskResourceRequest:
+    def __init__(self, resourceName: str, amount: float) -> None: ...
+    @property
+    def resourceName(self) -> str: ...
+    @property
+    def amount(self) -> float: ...
+
+class TaskResourceRequests:
+    @overload
+    def __init__(self, _jvm: JVMView) -> None: ...
+    @overload
+    def __init__(
+        self,
+        _jvm: None = ...,
+        _requests: Optional[Dict[str, TaskResourceRequest]] = ...,
+    ) -> None: ...
+    def cpus(self, amount: int) -> TaskResourceRequests: ...
+    def resource(self, resourceName: str, amount: float) -> TaskResourceRequests: ...
+    @property
+    def requests(self) -> Dict[str, TaskResourceRequest]: ...
diff --git a/python/pyspark/resource/resourceprofile.py b/python/pyspark/resource/resourceprofile.py
deleted file mode 100644
index 59e9ccb4b6ea0..0000000000000
--- a/python/pyspark/resource/resourceprofile.py
+++ /dev/null
@@ -1,72 +0,0 @@
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-from pyspark.resource.taskrequests import TaskResourceRequest
-from pyspark.resource.executorrequests import ExecutorResourceRequest
-
-
-class ResourceProfile(object):
-
-    """
-    .. note:: Evolving
-
-    Resource profile to associate with an RDD. A :class:`pyspark.resource.ResourceProfile`
-    allows the user to specify executor and task requirements for an RDD that will get
-    applied during a stage. This allows the user to change the resource requirements between
-    stages. This is meant to be immutable so user doesn't change it after building.
-
-    .. versionadded:: 3.1.0
-    """
-
-    def __init__(self, _java_resource_profile=None, _exec_req={}, _task_req={}):
-        if _java_resource_profile is not None:
-            self._java_resource_profile = _java_resource_profile
-        else:
-            self._java_resource_profile = None
-            self._executor_resource_requests = _exec_req
-            self._task_resource_requests = _task_req
-
-    @property
-    def id(self):
-        if self._java_resource_profile is not None:
-            return self._java_resource_profile.id()
-        else:
-            raise RuntimeError("SparkContext must be created to get the id, get the id "
-                               "after adding the ResourceProfile to an RDD")
-
-    @property
-    def taskResources(self):
-        if self._java_resource_profile is not None:
-            taskRes = self._java_resource_profile.taskResourcesJMap()
-            result = {}
-            for k, v in taskRes.items():
-                result[k] = TaskResourceRequest(v.resourceName(), v.amount())
-            return result
-        else:
-            return self._task_resource_requests
-
-    @property
-    def executorResources(self):
-        if self._java_resource_profile is not None:
-            execRes = self._java_resource_profile.executorResourcesJMap()
-            result = {}
-            for k, v in execRes.items():
-                result[k] = ExecutorResourceRequest(v.resourceName(), v.amount(),
-                                                    v.discoveryScript(), v.vendor())
-            return result
-        else:
-            return self._executor_resource_requests
diff --git a/python/pyspark/resource/resourceprofilebuilder.py b/python/pyspark/resource/resourceprofilebuilder.py
deleted file mode 100644
index 67654289d500f..0000000000000
--- a/python/pyspark/resource/resourceprofilebuilder.py
+++ /dev/null
@@ -1,117 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-from pyspark.resource.executorrequests import ExecutorResourceRequest,\
-    ExecutorResourceRequests
-from pyspark.resource.resourceprofile import ResourceProfile
-from pyspark.resource.taskrequests import TaskResourceRequest, TaskResourceRequests
-
-
-class ResourceProfileBuilder(object):
-
-    """
-    .. note:: Evolving
-
-    Resource profile Builder to build a resource profile to associate with an RDD.
-    A ResourceProfile allows the user to specify executor and task requirements for
-    an RDD that will get applied during a stage. This allows the user to change the
-    resource requirements between stages.
-
-    .. versionadded:: 3.1.0
-    """
-
-    def __init__(self):
-        from pyspark.context import SparkContext
-        _jvm = SparkContext._jvm
-        if _jvm is not None:
-            self._jvm = _jvm
-            self._java_resource_profile_builder = \
-                _jvm.org.apache.spark.resource.ResourceProfileBuilder()
-        else:
-            self._jvm = None
-            self._java_resource_profile_builder = None
-            self._executor_resource_requests = {}
-            self._task_resource_requests = {}
-
-    def require(self, resourceRequest):
-        if isinstance(resourceRequest, TaskResourceRequests):
-            if self._java_resource_profile_builder is not None:
-                if resourceRequest._java_task_resource_requests is not None:
-                    self._java_resource_profile_builder.require(
-                        resourceRequest._java_task_resource_requests)
-                else:
-                    taskReqs = TaskResourceRequests(self._jvm, resourceRequest.requests)
-                    self._java_resource_profile_builder.require(
-                        taskReqs._java_task_resource_requests)
-            else:
-                self._task_resource_requests.update(resourceRequest.requests)
-        else:
-            if self._java_resource_profile_builder is not None:
-                if resourceRequest._java_executor_resource_requests is not None:
-                    self._java_resource_profile_builder.require(
-                        resourceRequest._java_executor_resource_requests)
-                else:
-                    execReqs = ExecutorResourceRequests(self._jvm, resourceRequest.requests)
-                    self._java_resource_profile_builder.require(
-                        execReqs._java_executor_resource_requests)
-            else:
-                self._executor_resource_requests.update(resourceRequest.requests)
-        return self
-
-    def clearExecutorResourceRequests(self):
-        if self._java_resource_profile_builder is not None:
-            self._java_resource_profile_builder.clearExecutorResourceRequests()
-        else:
-            self._executor_resource_requests = {}
-
-    def clearTaskResourceRequests(self):
-        if self._java_resource_profile_builder is not None:
-            self._java_resource_profile_builder.clearTaskResourceRequests()
-        else:
-            self._task_resource_requests = {}
-
-    @property
-    def taskResources(self):
-        if self._java_resource_profile_builder is not None:
-            taskRes = self._java_resource_profile_builder.taskResourcesJMap()
-            result = {}
-            for k, v in taskRes.items():
-                result[k] = TaskResourceRequest(v.resourceName(), v.amount())
-            return result
-        else:
-            return self._task_resource_requests
-
-    @property
-    def executorResources(self):
-        if self._java_resource_profile_builder is not None:
-            result = {}
-            execRes = self._java_resource_profile_builder.executorResourcesJMap()
-            for k, v in execRes.items():
-                result[k] = ExecutorResourceRequest(v.resourceName(), v.amount(),
-                                                    v.discoveryScript(), v.vendor())
-            return result
-        else:
-            return self._executor_resource_requests
-
-    @property
-    def build(self):
-        if self._java_resource_profile_builder is not None:
-            jresourceProfile = self._java_resource_profile_builder.build()
-            return ResourceProfile(_java_resource_profile=jresourceProfile)
-        else:
-            return ResourceProfile(_exec_req=self._executor_resource_requests,
-                                   _task_req=self._task_resource_requests)
diff --git a/python/pyspark/resource/taskrequests.py b/python/pyspark/resource/taskrequests.py
deleted file mode 100644
index e8dca98d14b61..0000000000000
--- a/python/pyspark/resource/taskrequests.py
+++ /dev/null
@@ -1,102 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-
-class TaskResourceRequest(object):
-    """
-    .. note:: Evolving
-
-    A task resource request. This is used in conjuntion with the
-    :class:`pyspark.resource.ResourceProfile` to programmatically specify the resources
-    needed for an RDD that will be applied at the stage level. The amount is specified
-    as a Double to allow for saying you want more then 1 task per resource. Valid values
-    are less than or equal to 0.5 or whole numbers.
-    Use :class:`pyspark.resource.TaskResourceRequests` class as a convenience API.
-
-    :param resourceName: Name of the resource
-    :param amount: Amount requesting as a Double to support fractional resource requests.
-        Valid values are less than or equal to 0.5 or whole numbers.
-
-    .. versionadded:: 3.1.0
-    """
-    def __init__(self, resourceName, amount):
-        self._name = resourceName
-        self._amount = float(amount)
-
-    @property
-    def resourceName(self):
-        return self._name
-
-    @property
-    def amount(self):
-        return self._amount
-
-
-class TaskResourceRequests(object):
-
-    """
-    .. note:: Evolving
-
-    A set of task resource requests. This is used in conjuntion with the
-    :class:`pyspark.resource.ResourceProfileBuilder` to programmatically specify the resources
-    needed for an RDD that will be applied at the stage level.
-
-    .. versionadded:: 3.1.0
-    """
-
-    _CPUS = "cpus"
-
-    def __init__(self, _jvm=None, _requests=None):
-        from pyspark import SparkContext
-        _jvm = _jvm or SparkContext._jvm
-        if _jvm is not None:
-            self._java_task_resource_requests = \
-                SparkContext._jvm.org.apache.spark.resource.TaskResourceRequests()
-            if _requests is not None:
-                for k, v in _requests.items():
-                    if k == self._CPUS:
-                        self._java_task_resource_requests.cpus(int(v.amount))
-                    else:
-                        self._java_task_resource_requests.resource(v.resourceName, v.amount)
-        else:
-            self._java_task_resource_requests = None
-            self._task_resources = {}
-
-    def cpus(self, amount):
-        if self._java_task_resource_requests is not None:
-            self._java_task_resource_requests.cpus(amount)
-        else:
-            self._task_resources[self._CPUS] = TaskResourceRequest(self._CPUS, amount)
-        return self
-
-    def resource(self, resourceName, amount):
-        if self._java_task_resource_requests is not None:
-            self._java_task_resource_requests.resource(resourceName, float(amount))
-        else:
-            self._task_resources[resourceName] = TaskResourceRequest(resourceName, amount)
-        return self
-
-    @property
-    def requests(self):
-        if self._java_task_resource_requests is not None:
-            result = {}
-            taskRes = self._java_task_resource_requests.requestsJMap()
-            for k, v in taskRes.items():
-                result[k] = TaskResourceRequest(v.resourceName(), v.amount())
-            return result
-        else:
-            return self._task_resources
diff --git a/python/pyspark/resource/tests/test_resources.py b/python/pyspark/resource/tests/test_resources.py
index 9eb5a35e58df5..6149f1ff7205a 100644
--- a/python/pyspark/resource/tests/test_resources.py
+++ b/python/pyspark/resource/tests/test_resources.py
@@ -14,10 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-import random
 import unittest
 
-from pyspark.resource import ExecutorResourceRequests, ResourceProfile, ResourceProfileBuilder,\
+from pyspark.resource import ExecutorResourceRequests, ResourceProfileBuilder,\
     TaskResourceRequests
 
 
@@ -25,15 +24,16 @@ class ResourceProfileTests(unittest.TestCase):
     def test_profile_before_sc(self):
         rpb = ResourceProfileBuilder()
         ereqs = ExecutorResourceRequests().cores(2).memory("6g").memoryOverhead("1g")
-        ereqs.pysparkMemory("2g").resource("gpu", 2, "testGpus", "nvidia.com")
+        ereqs.pysparkMemory("2g").offheapMemory("3g").resource("gpu", 2, "testGpus", "nvidia.com")
         treqs = TaskResourceRequests().cpus(2).resource("gpu", 2)
 
         def assert_request_contents(exec_reqs, task_reqs):
-            self.assertEqual(len(exec_reqs), 5)
+            self.assertEqual(len(exec_reqs), 6)
             self.assertEqual(exec_reqs["cores"].amount, 2)
             self.assertEqual(exec_reqs["memory"].amount, 6144)
             self.assertEqual(exec_reqs["memoryOverhead"].amount, 1024)
             self.assertEqual(exec_reqs["pyspark.memory"].amount, 2048)
+            self.assertEqual(exec_reqs["offHeap"].amount, 3072)
             self.assertEqual(exec_reqs["gpu"].amount, 2)
             self.assertEqual(exec_reqs["gpu"].discoveryScript, "testGpus")
             self.assertEqual(exec_reqs["gpu"].resourceName, "gpu")
@@ -72,10 +72,10 @@ def assert_request_contents(exec_reqs, task_reqs):
 
 
 if __name__ == "__main__":
-    from pyspark.resource.tests.test_resources import *
+    from pyspark.resource.tests.test_resources import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/resourceinformation.py b/python/pyspark/resourceinformation.py
deleted file mode 100644
index aaed21374b6ee..0000000000000
--- a/python/pyspark/resourceinformation.py
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-
-class ResourceInformation(object):
-
-    """
-    .. note:: Evolving
-
-    Class to hold information about a type of Resource. A resource could be a GPU, FPGA, etc.
-    The array of addresses are resource specific and its up to the user to interpret the address.
-
-    One example is GPUs, where the addresses would be the indices of the GPUs
-
-    @param name the name of the resource
-    @param addresses an array of strings describing the addresses of the resource
-    """
-
-    def __init__(self, name, addresses):
-        self._name = name
-        self._addresses = addresses
-
-    @property
-    def name(self):
-        return self._name
-
-    @property
-    def addresses(self):
-        return self._addresses
diff --git a/python/pyspark/resultiterable.py b/python/pyspark/resultiterable.py
index c867b51877ffe..cd2a59513bb17 100644
--- a/python/pyspark/resultiterable.py
+++ b/python/pyspark/resultiterable.py
@@ -15,10 +15,7 @@
 # limitations under the License.
 #
 
-try:
-    from collections.abc import Iterable
-except ImportError:
-    from collections import Iterable
+from collections.abc import Iterable
 
 
 __all__ = ["ResultIterable"]
diff --git a/python/pyspark/resultiterable.pyi b/python/pyspark/resultiterable.pyi
new file mode 100644
index 0000000000000..69596ad82c8cc
--- /dev/null
+++ b/python/pyspark/resultiterable.pyi
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from pyspark._typing import SizedIterable
+from typing import Iterator, TypeVar
+
+T = TypeVar("T")
+
+class ResultIterable(SizedIterable[T]):
+    data: SizedIterable[T]
+    index: int
+    maxindex: int
+    def __init__(self, data: SizedIterable[T]) -> None: ...
+    def __iter__(self) -> Iterator[T]: ...
+    def __len__(self) -> int: ...
diff --git a/python/pyspark/serializers.py b/python/pyspark/serializers.py
index 49b7cb4546676..1b434d39312a6 100644
--- a/python/pyspark/serializers.py
+++ b/python/pyspark/serializers.py
@@ -24,6 +24,9 @@
 Other serializers, like :class:`MarshalSerializer`, support fewer datatypes but can be
 faster.
 
+
+Examples
+--------
 The serializer is chosen when creating :class:`SparkContext`:
 
 >>> from pyspark.context import SparkContext
@@ -58,18 +61,11 @@
 import collections
 import zlib
 import itertools
-
-if sys.version < '3':
-    import cPickle as pickle
-    from itertools import izip as zip, imap as map
-else:
-    import pickle
-    basestring = unicode = str
-    xrange = range
+import pickle
 pickle_protocol = pickle.HIGHEST_PROTOCOL
 
 from pyspark import cloudpickle
-from pyspark.util import _exception_message, print_exec
+from pyspark.util import print_exec
 
 
 __all__ = ["PickleSerializer", "MarshalSerializer", "UTF8Deserializer"]
@@ -132,11 +128,6 @@ class FramedSerializer(Serializer):
     where `length` is a 32-bit integer and data is `length` bytes.
     """
 
-    def __init__(self):
-        # On Python 2.6, we can't write bytearrays to streams, so we need to convert them
-        # to strings first. Check if the version number is that old.
-        self._only_write_strings = sys.version_info[0:2] <= (2, 6)
-
     def dump_stream(self, iterator, stream):
         for obj in iterator:
             self._write_with_length(obj, stream)
@@ -155,10 +146,7 @@ def _write_with_length(self, obj, stream):
         if len(serialized) > (1 << 31):
             raise ValueError("can not serialize object larger than 2G")
         write_int(len(serialized), stream)
-        if self._only_write_strings:
-            stream.write(str(serialized))
-        else:
-            stream.write(serialized)
+        stream.write(serialized)
 
     def _read_with_length(self, stream):
         length = read_int(stream)
@@ -204,7 +192,7 @@ def _batched(self, iterator):
             yield list(iterator)
         elif hasattr(iterator, "__len__") and hasattr(iterator, "__getslice__"):
             n = len(iterator)
-            for i in xrange(0, n, self.batchSize):
+            for i in range(0, n, self.batchSize):
                 yield iterator[i: i + self.batchSize]
         else:
             items = []
@@ -357,7 +345,7 @@ def dumps(self, obj):
 
 # Hack namedtuple, make it picklable
 
-__cls = {}
+__cls = {}  # type: ignore
 
 
 def _restore(name, fields, value):
@@ -395,23 +383,8 @@ def _copy_func(f):
         return types.FunctionType(f.__code__, f.__globals__, f.__name__,
                                   f.__defaults__, f.__closure__)
 
-    def _kwdefaults(f):
-        # __kwdefaults__ contains the default values of keyword-only arguments which are
-        # introduced from Python 3. The possible cases for __kwdefaults__ in namedtuple
-        # are as below:
-        #
-        # - Does not exist in Python 2.
-        # - Returns None in <= Python 3.5.x.
-        # - Returns a dictionary containing the default values to the keys from Python 3.6.x
-        #    (See https://bugs.python.org/issue25628).
-        kargs = getattr(f, "__kwdefaults__", None)
-        if kargs is None:
-            return {}
-        else:
-            return kargs
-
     _old_namedtuple = _copy_func(collections.namedtuple)
-    _old_namedtuple_kwdefaults = _kwdefaults(collections.namedtuple)
+    _old_namedtuple_kwdefaults = collections.namedtuple.__kwdefaults__
 
     def namedtuple(*args, **kwargs):
         for k, v in _old_namedtuple_kwdefaults.items():
@@ -453,12 +426,8 @@ class PickleSerializer(FramedSerializer):
     def dumps(self, obj):
         return pickle.dumps(obj, pickle_protocol)
 
-    if sys.version >= '3':
-        def loads(self, obj, encoding="bytes"):
-            return pickle.loads(obj, encoding=encoding)
-    else:
-        def loads(self, obj, encoding=None):
-            return pickle.loads(obj)
+    def loads(self, obj, encoding="bytes"):
+        return pickle.loads(obj, encoding=encoding)
 
 
 class CloudPickleSerializer(PickleSerializer):
@@ -469,7 +438,7 @@ def dumps(self, obj):
         except pickle.PickleError:
             raise
         except Exception as e:
-            emsg = _exception_message(e)
+            emsg = str(e)
             if "'i' format requires" in emsg:
                 msg = "Object too large to serialize: %s" % emsg
             else:
diff --git a/python/pyspark/shell.py b/python/pyspark/shell.py
index 65e3bdbc05ce8..25aadb16840c8 100644
--- a/python/pyspark/shell.py
+++ b/python/pyspark/shell.py
@@ -26,19 +26,16 @@
 import platform
 import warnings
 
-import py4j
-
-from pyspark import SparkConf
 from pyspark.context import SparkContext
-from pyspark.sql import SparkSession, SQLContext
+from pyspark.sql import SparkSession
 
 if os.environ.get("SPARK_EXECUTOR_URI"):
     SparkContext.setSystemProperty("spark.executor.uri", os.environ["SPARK_EXECUTOR_URI"])
 
-SparkContext._ensure_initialized()
+SparkContext._ensure_initialized()  # type: ignore
 
 try:
-    spark = SparkSession._create_shell_session()
+    spark = SparkSession._create_shell_session()  # type: ignore
 except Exception:
     import sys
     import traceback
@@ -65,6 +62,8 @@
     platform.python_version(),
     platform.python_build()[0],
     platform.python_build()[1]))
+print("Spark context Web UI available at %s" % (sc.uiWebUrl))
+print("Spark context available as 'sc' (master = %s, app id = %s)." % (sc.master, sc.applicationId))
 print("SparkSession available as 'spark'.")
 
 # The ./bin/pyspark script stores the old PYTHONSTARTUP value in OLD_PYTHONSTARTUP,
diff --git a/python/pyspark/shuffle.py b/python/pyspark/shuffle.py
index 5d2d63850e9b2..4ba846227188c 100644
--- a/python/pyspark/shuffle.py
+++ b/python/pyspark/shuffle.py
@@ -25,7 +25,7 @@
 import random
 import sys
 
-import pyspark.heapq3 as heapq
+import heapq
 from pyspark.serializers import BatchedSerializer, PickleSerializer, FlattenedValuesSerializer, \
     CompressedSerializer, AutoBatchedSerializer
 from pyspark.util import fail_on_stopiteration
@@ -177,6 +177,8 @@ class ExternalMerger(Merger):
     Finally, if any items were spilled into disks, each partition
     will be merged into `data` and be yielded, then cleared.
 
+    Examples
+    --------
     >>> agg = SimpleAggregator(lambda x, y: x + y)
     >>> merger = ExternalMerger(agg, 10)
     >>> N = 10000
@@ -416,13 +418,14 @@ def _cleanup(self):
 
 class ExternalSorter(object):
     """
-    ExtenalSorter will divide the elements into chunks, sort them in
+    ExternalSorter will divide the elements into chunks, sort them in
     memory and dump them into disks, finally merge them back.
 
     The spilling will only happen when the used memory goes above
     the limit.
 
-
+    Examples
+    --------
     >>> sorter = ExternalSorter(1)  # 1M
     >>> import random
     >>> l = list(range(1024))
@@ -498,7 +501,7 @@ def load(f):
         if current_chunk:
             chunks.append(iter(current_chunk))
 
-        return heapq.merge(chunks, key=key, reverse=reverse)
+        return heapq.merge(*chunks, key=key, reverse=reverse)
 
 
 class ExternalList(object):
@@ -506,6 +509,8 @@ class ExternalList(object):
     ExternalList can have many items which cannot be hold in memory in
     the same time.
 
+    Examples
+    --------
     >>> l = ExternalList(list(range(100)))
     >>> len(l)
     100
@@ -606,6 +611,8 @@ class ExternalListOfList(ExternalList):
     """
     An external list for list.
 
+    Examples
+    --------
     >>> l = ExternalListOfList([[i, i] for i in range(100)])
     >>> len(l)
     200
@@ -635,6 +642,8 @@ class GroupByKey(object):
     """
     Group a sorted iterator as [(k1, it1), (k2, it2), ...]
 
+    Examples
+    --------
     >>> k = [i // 3 for i in range(6)]
     >>> v = [[i] for i in range(6)]
     >>> g = GroupByKey(zip(k, v))
@@ -796,7 +805,7 @@ def load_partition(j):
 
         if self._sorted:
             # all the partitions are already sorted
-            sorted_items = heapq.merge(disk_items, key=operator.itemgetter(0))
+            sorted_items = heapq.merge(*disk_items, key=operator.itemgetter(0))
 
         else:
             # Flatten the combined values, so it will not consume huge
diff --git a/python/pyspark/sql/__init__.py b/python/pyspark/sql/__init__.py
index c28cb8c3b9cbe..af32469e82b43 100644
--- a/python/pyspark/sql/__init__.py
+++ b/python/pyspark/sql/__init__.py
@@ -39,9 +39,6 @@
     - :class:`pyspark.sql.Window`
       For working with window functions.
 """
-from __future__ import absolute_import
-
-
 from pyspark.sql.types import Row
 from pyspark.sql.context import SQLContext, HiveContext, UDFRegistration
 from pyspark.sql.session import SparkSession
diff --git a/python/pyspark/sql/__init__.pyi b/python/pyspark/sql/__init__.pyi
new file mode 100644
index 0000000000000..787be5647772e
--- /dev/null
+++ b/python/pyspark/sql/__init__.pyi
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from pyspark.sql.catalog import Catalog as Catalog  # noqa: F401
+from pyspark.sql.column import Column as Column  # noqa: F401
+from pyspark.sql.context import (  # noqa: F401
+    HiveContext as HiveContext,
+    SQLContext as SQLContext,
+    UDFRegistration as UDFRegistration,
+)
+from pyspark.sql.dataframe import (  # noqa: F401
+    DataFrame as DataFrame,
+    DataFrameNaFunctions as DataFrameNaFunctions,
+    DataFrameStatFunctions as DataFrameStatFunctions,
+)
+from pyspark.sql.group import GroupedData as GroupedData  # noqa: F401
+from pyspark.sql.pandas.group_ops import (  # noqa: F401
+    PandasCogroupedOps as PandasCogroupedOps,
+)
+from pyspark.sql.readwriter import (  # noqa: F401
+    DataFrameReader as DataFrameReader,
+    DataFrameWriter as DataFrameWriter,
+)
+from pyspark.sql.session import SparkSession as SparkSession  # noqa: F401
+from pyspark.sql.types import Row as Row  # noqa: F401
+from pyspark.sql.window import Window as Window, WindowSpec as WindowSpec  # noqa: F401
diff --git a/python/pyspark/sql/_typing.pyi b/python/pyspark/sql/_typing.pyi
new file mode 100644
index 0000000000000..799a73204a639
--- /dev/null
+++ b/python/pyspark/sql/_typing.pyi
@@ -0,0 +1,57 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import (
+    Any,
+    List,
+    Optional,
+    Tuple,
+    TypeVar,
+    Union,
+)
+from typing_extensions import Protocol
+
+import datetime
+import decimal
+
+from pyspark._typing import PrimitiveType
+import pyspark.sql.column
+import pyspark.sql.types
+from pyspark.sql.column import Column
+
+ColumnOrName = Union[pyspark.sql.column.Column, str]
+DecimalLiteral = decimal.Decimal
+DateTimeLiteral = Union[datetime.datetime, datetime.date]
+LiteralType = PrimitiveType
+AtomicDataTypeOrString = Union[pyspark.sql.types.AtomicType, str]
+DataTypeOrString = Union[pyspark.sql.types.DataType, str]
+OptionalPrimitiveType = Optional[PrimitiveType]
+
+RowLike = TypeVar("RowLike", List[Any], Tuple[Any, ...], pyspark.sql.types.Row)
+
+class SupportsOpen(Protocol):
+    def open(self, partition_id: int, epoch_id: int) -> bool: ...
+
+class SupportsProcess(Protocol):
+    def process(self, row: pyspark.sql.types.Row) -> None: ...
+
+class SupportsClose(Protocol):
+    def close(self, error: Exception) -> None: ...
+
+class UserDefinedFunctionLike(Protocol):
+    def __call__(self, *_: ColumnOrName) -> Column: ...
diff --git a/python/pyspark/sql/avro/functions.py b/python/pyspark/sql/avro/functions.py
index ed62a72d6c8fb..dbb8ab4ac24ff 100644
--- a/python/pyspark/sql/avro/functions.py
+++ b/python/pyspark/sql/avro/functions.py
@@ -20,15 +20,12 @@
 """
 
 
-from pyspark import since, SparkContext
-from pyspark.rdd import ignore_unicode_prefix
+from pyspark import SparkContext
 from pyspark.sql.column import Column, _to_java_column
 from pyspark.util import _print_missing_jar
 
 
-@ignore_unicode_prefix
-@since(3.0)
-def from_avro(data, jsonFormatSchema, options={}):
+def from_avro(data, jsonFormatSchema, options=None):
     """
     Converts a binary column of Avro format into its corresponding catalyst value.
     The specified schema must match the read data, otherwise the behavior is undefined:
@@ -36,32 +33,44 @@ def from_avro(data, jsonFormatSchema, options={}):
     To deserialize the data with a compatible and evolved schema, the expected Avro schema can be
     set via the option avroSchema.
 
-    Note: Avro is built-in but external data source module since Spark 2.4. Please deploy the
-    application as per the deployment section of "Apache Avro Data Source Guide".
+    .. versionadded:: 3.0.0
+
+    Parameters
+    ----------
+    data : :class:`~pyspark.sql.Column` or str
+        the binary column.
+    jsonFormatSchema : str
+        the avro schema in JSON string format.
+    options : dict, optional
+        options to control how the Avro record is parsed.
 
-    :param data: the binary column.
-    :param jsonFormatSchema: the avro schema in JSON string format.
-    :param options: options to control how the Avro record is parsed.
+    Notes
+    -----
+    Avro is built-in but external data source module since Spark 2.4. Please deploy the
+    application as per the deployment section of "Apache Avro Data Source Guide".
 
+    Examples
+    --------
     >>> from pyspark.sql import Row
     >>> from pyspark.sql.avro.functions import from_avro, to_avro
-    >>> data = [(1, Row(name='Alice', age=2))]
+    >>> data = [(1, Row(age=2, name='Alice'))]
     >>> df = spark.createDataFrame(data, ("key", "value"))
     >>> avroDf = df.select(to_avro(df.value).alias("avro"))
     >>> avroDf.collect()
     [Row(avro=bytearray(b'\\x00\\x00\\x04\\x00\\nAlice'))]
+
     >>> jsonFormatSchema = '''{"type":"record","name":"topLevelRecord","fields":
     ...     [{"name":"avro","type":[{"type":"record","name":"value","namespace":"topLevelRecord",
     ...     "fields":[{"name":"age","type":["long","null"]},
     ...     {"name":"name","type":["string","null"]}]},"null"]}]}'''
     >>> avroDf.select(from_avro(avroDf.avro, jsonFormatSchema).alias("value")).collect()
-    [Row(value=Row(avro=Row(age=2, name=u'Alice')))]
+    [Row(value=Row(avro=Row(age=2, name='Alice')))]
     """
 
     sc = SparkContext._active_spark_context
     try:
         jc = sc._jvm.org.apache.spark.sql.avro.functions.from_avro(
-            _to_java_column(data), jsonFormatSchema, options)
+            _to_java_column(data), jsonFormatSchema, options or {})
     except TypeError as e:
         if str(e) == "'JavaPackage' object is not callable":
             _print_missing_jar("Avro", "avro", "avro", sc.version)
@@ -69,24 +78,33 @@ def from_avro(data, jsonFormatSchema, options={}):
     return Column(jc)
 
 
-@ignore_unicode_prefix
-@since(3.0)
 def to_avro(data, jsonFormatSchema=""):
     """
     Converts a column into binary of avro format.
 
-    Note: Avro is built-in but external data source module since Spark 2.4. Please deploy the
-    application as per the deployment section of "Apache Avro Data Source Guide".
+    .. versionadded:: 3.0.0
+
+    Parameters
+    ----------
+    data : :class:`~pyspark.sql.Column` or str
+        the data column.
+    jsonFormatSchema : str, optional
+        user-specified output avro schema in JSON string format.
 
-    :param data: the data column.
-    :param jsonFormatSchema: user-specified output avro schema in JSON string format.
+    Notes
+    -----
+    Avro is built-in but external data source module since Spark 2.4. Please deploy the
+    application as per the deployment section of "Apache Avro Data Source Guide".
 
+    Examples
+    --------
     >>> from pyspark.sql import Row
     >>> from pyspark.sql.avro.functions import to_avro
     >>> data = ['SPADES']
     >>> df = spark.createDataFrame(data, "string")
     >>> df.select(to_avro(df.value).alias("suite")).collect()
     [Row(suite=bytearray(b'\\x00\\x0cSPADES'))]
+
     >>> jsonFormatSchema = '''["null", {"type": "enum", "name": "value",
     ...     "symbols": ["SPADES", "HEARTS", "DIAMONDS", "CLUBS"]}]'''
     >>> df.select(to_avro(df.value, jsonFormatSchema).alias("suite")).collect()
@@ -125,7 +143,7 @@ def _test():
         os.environ["PYSPARK_SUBMIT_ARGS"] = " ".join([jars_args, existing_args])
 
     import doctest
-    from pyspark.sql import Row, SparkSession
+    from pyspark.sql import SparkSession
     import pyspark.sql.avro.functions
     globs = pyspark.sql.avro.functions.__dict__.copy()
     spark = SparkSession.builder\
diff --git a/python/pyspark/sql/avro/functions.pyi b/python/pyspark/sql/avro/functions.pyi
new file mode 100644
index 0000000000000..49881335d8fcc
--- /dev/null
+++ b/python/pyspark/sql/avro/functions.pyi
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Dict, Optional
+
+from pyspark.sql._typing import ColumnOrName
+from pyspark.sql.column import Column
+
+def from_avro(
+    data: ColumnOrName, jsonFormatSchema: str, options: Optional[Dict[str, str]] = ...
+) -> Column: ...
+def to_avro(data: ColumnOrName, jsonFormatSchema: str = ...) -> Column: ...
diff --git a/python/pyspark/sql/catalog.py b/python/pyspark/sql/catalog.py
index 974251f63b37a..d4b7d4ef61cc4 100644
--- a/python/pyspark/sql/catalog.py
+++ b/python/pyspark/sql/catalog.py
@@ -20,10 +20,8 @@
 from collections import namedtuple
 
 from pyspark import since
-from pyspark.rdd import ignore_unicode_prefix, PythonEvalType
 from pyspark.sql.dataframe import DataFrame
-from pyspark.sql.udf import UserDefinedFunction
-from pyspark.sql.types import IntegerType, StringType, StructType
+from pyspark.sql.types import StructType
 
 
 Database = namedtuple("Database", "name description locationUri")
@@ -44,19 +42,16 @@ def __init__(self, sparkSession):
         self._jsparkSession = sparkSession._jsparkSession
         self._jcatalog = sparkSession._jsparkSession.catalog()
 
-    @ignore_unicode_prefix
     @since(2.0)
     def currentDatabase(self):
         """Returns the current default database in this session."""
         return self._jcatalog.currentDatabase()
 
-    @ignore_unicode_prefix
     @since(2.0)
     def setCurrentDatabase(self, dbName):
         """Sets the current default database in this session."""
         return self._jcatalog.setCurrentDatabase(dbName)
 
-    @ignore_unicode_prefix
     @since(2.0)
     def listDatabases(self):
         """Returns a list of databases available across all sessions."""
@@ -70,7 +65,6 @@ def listDatabases(self):
                 locationUri=jdb.locationUri()))
         return databases
 
-    @ignore_unicode_prefix
     @since(2.0)
     def listTables(self, dbName=None):
         """Returns a list of tables/views in the specified database.
@@ -92,7 +86,6 @@ def listTables(self, dbName=None):
                 isTemporary=jtable.isTemporary()))
         return tables
 
-    @ignore_unicode_prefix
     @since(2.0)
     def listFunctions(self, dbName=None):
         """Returns a list of functions registered in the specified database.
@@ -113,14 +106,16 @@ def listFunctions(self, dbName=None):
                 isTemporary=jfunction.isTemporary()))
         return functions
 
-    @ignore_unicode_prefix
-    @since(2.0)
     def listColumns(self, tableName, dbName=None):
         """Returns a list of columns for the given table/view in the specified database.
 
         If no database is specified, the current database is used.
 
-        Note: the order of arguments here is different from that of its JVM counterpart
+       .. versionadded:: 2.0.0
+
+        Notes
+        -----
+        the order of arguments here is different from that of its JVM counterpart
         because Python does not support method overloading.
         """
         if dbName is None:
@@ -138,7 +133,6 @@ def listColumns(self, tableName, dbName=None):
                 isBucket=jcolumn.isBucket()))
         return columns
 
-    @since(2.0)
     def createExternalTable(self, tableName, path=None, source=None, schema=None, **options):
         """Creates a table based on the dataset in a data source.
 
@@ -151,15 +145,20 @@ def createExternalTable(self, tableName, path=None, source=None, schema=None, **
         Optionally, a schema can be provided as the schema of the returned :class:`DataFrame` and
         created external table.
 
-        :return: :class:`DataFrame`
+        .. versionadded:: 2.0.0
+
+        Returns
+        -------
+        :class:`DataFrame`
         """
         warnings.warn(
             "createExternalTable is deprecated since Spark 2.2, please use createTable instead.",
-            DeprecationWarning)
+            FutureWarning
+        )
         return self.createTable(tableName, path, source, schema, **options)
 
-    @since(2.2)
-    def createTable(self, tableName, path=None, source=None, schema=None, **options):
+    def createTable(
+            self, tableName, path=None, source=None, schema=None, description=None, **options):
         """Creates a table based on the dataset in a data source.
 
         It returns the DataFrame associated with the table.
@@ -172,30 +171,45 @@ def createTable(self, tableName, path=None, source=None, schema=None, **options)
         Optionally, a schema can be provided as the schema of the returned :class:`DataFrame` and
         created table.
 
-        :return: :class:`DataFrame`
+        .. versionadded:: 2.2.0
+
+        Returns
+        -------
+        :class:`DataFrame`
+
+        .. versionchanged:: 3.1
+           Added the ``description`` parameter.
         """
         if path is not None:
             options["path"] = path
         if source is None:
             source = self._sparkSession._wrapped._conf.defaultDataSourceName()
+        if description is None:
+            description = ""
         if schema is None:
-            df = self._jcatalog.createTable(tableName, source, options)
+            df = self._jcatalog.createTable(tableName, source, description, options)
         else:
             if not isinstance(schema, StructType):
                 raise TypeError("schema should be StructType")
             scala_datatype = self._jsparkSession.parseDataType(schema.json())
-            df = self._jcatalog.createTable(tableName, source, scala_datatype, options)
+            df = self._jcatalog.createTable(
+                tableName, source, scala_datatype, description, options)
         return DataFrame(df, self._sparkSession._wrapped)
 
-    @since(2.0)
     def dropTempView(self, viewName):
         """Drops the local temporary view with the given view name in the catalog.
         If the view has been cached before, then it will also be uncached.
         Returns true if this view is dropped successfully, false otherwise.
 
-        Note that, the return type of this method was None in Spark 2.0, but changed to Boolean
+        .. versionadded:: 2.0.0
+
+        Notes
+        -----
+        The return type of this method was None in Spark 2.0, but changed to Boolean
         in Spark 2.1.
 
+        Examples
+        --------
         >>> spark.createDataFrame([(1, 1)]).createTempView("my_table")
         >>> spark.table("my_table").collect()
         [Row(_1=1, _2=1)]
@@ -207,12 +221,15 @@ def dropTempView(self, viewName):
         """
         self._jcatalog.dropTempView(viewName)
 
-    @since(2.1)
     def dropGlobalTempView(self, viewName):
         """Drops the global temporary view with the given view name in the catalog.
         If the view has been cached before, then it will also be uncached.
         Returns true if this view is dropped successfully, false otherwise.
 
+        .. versionadded:: 2.1.0
+
+        Examples
+        --------
         >>> spark.createDataFrame([(1, 1)]).createGlobalTempView("my_table")
         >>> spark.table("global_temp.my_table").collect()
         [Row(_1=1, _2=1)]
@@ -224,16 +241,19 @@ def dropGlobalTempView(self, viewName):
         """
         self._jcatalog.dropGlobalTempView(viewName)
 
-    @since(2.0)
     def registerFunction(self, name, f, returnType=None):
         """An alias for :func:`spark.udf.register`.
         See :meth:`pyspark.sql.UDFRegistration.register`.
 
-        .. note:: Deprecated in 2.3.0. Use :func:`spark.udf.register` instead.
+        .. versionadded:: 2.0.0
+
+        .. deprecated:: 2.3.0
+            Use :func:`spark.udf.register` instead.
         """
         warnings.warn(
             "Deprecated in 2.3.0. Use spark.udf.register instead.",
-            DeprecationWarning)
+            FutureWarning
+        )
         return self._sparkSession.udf.register(name, f, returnType)
 
     @since(2.0)
diff --git a/python/pyspark/sql/catalog.pyi b/python/pyspark/sql/catalog.pyi
new file mode 100644
index 0000000000000..cb5436ddf77ea
--- /dev/null
+++ b/python/pyspark/sql/catalog.pyi
@@ -0,0 +1,64 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Any, Callable, List, Optional
+from pyspark.sql._typing import UserDefinedFunctionLike
+from pyspark.sql.dataframe import DataFrame
+from pyspark.sql.session import SparkSession
+from pyspark.sql.types import DataType, StructType
+from collections import namedtuple
+
+Database = namedtuple("Database", "name description locationUri")
+
+Table = namedtuple("Table", "name database description tableType isTemporary")
+
+Column = namedtuple("Column", "name description dataType nullable isPartition isBucket")
+
+Function = namedtuple("Function", "name description className isTemporary")
+
+class Catalog:
+    def __init__(self, sparkSession: SparkSession) -> None: ...
+    def currentDatabase(self) -> str: ...
+    def setCurrentDatabase(self, dbName: str) -> None: ...
+    def listDatabases(self) -> List[Database]: ...
+    def listTables(self, dbName: Optional[str] = ...) -> List[Table]: ...
+    def listFunctions(self, dbName: Optional[str] = ...) -> List[Function]: ...
+    def listColumns(
+        self, tableName: str, dbName: Optional[str] = ...
+    ) -> List[Column]: ...
+    def createTable(
+        self,
+        tableName: str,
+        path: Optional[str] = ...,
+        source: Optional[str] = ...,
+        schema: Optional[StructType] = ...,
+        description: Optional[str] = ...,
+        **options: str
+    ) -> DataFrame: ...
+    def dropTempView(self, viewName: str) -> None: ...
+    def dropGlobalTempView(self, viewName: str) -> None: ...
+    def registerFunction(
+        self, name: str, f: Callable[..., Any], returnType: DataType = ...
+    ) -> UserDefinedFunctionLike: ...
+    def isCached(self, tableName: str) -> bool: ...
+    def cacheTable(self, tableName: str) -> None: ...
+    def uncacheTable(self, tableName: str) -> None: ...
+    def clearCache(self) -> None: ...
+    def refreshTable(self, tableName: str) -> None: ...
+    def recoverPartitions(self, tableName: str) -> None: ...
+    def refreshByPath(self, path: str) -> None: ...
diff --git a/python/pyspark/sql/column.py b/python/pyspark/sql/column.py
index ef4944c9121a4..391ee5edcd46b 100644
--- a/python/pyspark/sql/column.py
+++ b/python/pyspark/sql/column.py
@@ -19,16 +19,9 @@
 import json
 import warnings
 
-if sys.version >= '3':
-    basestring = str
-    long = int
-
-from py4j.java_gateway import is_instance_of
-
-from pyspark import copy_func, since
+from pyspark import copy_func
 from pyspark.context import SparkContext
-from pyspark.rdd import ignore_unicode_prefix
-from pyspark.sql.types import *
+from pyspark.sql.types import DataType, StructField, StructType, IntegerType, StringType
 
 __all__ = ["Column"]
 
@@ -46,7 +39,7 @@ def _create_column_from_name(name):
 def _to_java_column(col):
     if isinstance(col, Column):
         jcol = col._jc
-    elif isinstance(col, basestring):
+    elif isinstance(col, str):
         jcol = _create_column_from_name(col)
     else:
         raise TypeError(
@@ -148,7 +141,7 @@ class Column(object):
         df.colName + 1
         1 / df.colName
 
-    .. versionadded:: 1.3
+    .. versionadded:: 1.3.0
     """
 
     def __init__(self, jc):
@@ -182,8 +175,15 @@ def __init__(self, jc):
     _eqNullSafe_doc = """
     Equality test that is safe for null values.
 
-    :param other: a value or :class:`Column`
+    .. versionadded:: 2.3.0
+
+    Parameters
+    ----------
+    other
+        a value or :class:`Column`
 
+    Examples
+    --------
     >>> from pyspark.sql import Row
     >>> df1 = spark.createDataFrame([
     ...     Row(id=1, value='foo'),
@@ -226,11 +226,11 @@ def __init__(self, jc):
     |            true|          false|           false|
     +----------------+---------------+----------------+
 
-    .. note:: Unlike Pandas, PySpark doesn't consider NaN values to be NULL.
-       See the `NaN Semantics`_ for details.
-    .. _NaN Semantics:
-       https://spark.apache.org/docs/latest/sql-programming-guide.html#nan-semantics
-    .. versionadded:: 2.3.0
+    Notes
+    -----
+    Unlike Pandas, PySpark doesn't consider NaN values to be NULL. See the
+    `NaN Semantics <https://spark.apache.org/docs/latest/sql-ref-datatypes.html#nan-semantics>`_
+    for details.
     """
     eqNullSafe = _bin_op("eqNullSafe", _eqNullSafe_doc)
 
@@ -251,9 +251,14 @@ def __contains__(self, item):
     _bitwiseOR_doc = """
     Compute bitwise OR of this expression with another expression.
 
-    :param other: a value or :class:`Column` to calculate bitwise or(|) against
-                  this :class:`Column`.
+    Parameters
+    ----------
+    other
+        a value or :class:`Column` to calculate bitwise or(|) with
+        this :class:`Column`.
 
+    Examples
+    --------
     >>> from pyspark.sql import Row
     >>> df = spark.createDataFrame([Row(a=170, b=75)])
     >>> df.select(df.a.bitwiseOR(df.b)).collect()
@@ -262,9 +267,14 @@ def __contains__(self, item):
     _bitwiseAND_doc = """
     Compute bitwise AND of this expression with another expression.
 
-    :param other: a value or :class:`Column` to calculate bitwise and(&) against
-                  this :class:`Column`.
+    Parameters
+    ----------
+    other
+        a value or :class:`Column` to calculate bitwise and(&) with
+        this :class:`Column`.
 
+    Examples
+    --------
     >>> from pyspark.sql import Row
     >>> df = spark.createDataFrame([Row(a=170, b=75)])
     >>> df.select(df.a.bitwiseAND(df.b)).collect()
@@ -273,9 +283,14 @@ def __contains__(self, item):
     _bitwiseXOR_doc = """
     Compute bitwise XOR of this expression with another expression.
 
-    :param other: a value or :class:`Column` to calculate bitwise xor(^) against
-                  this :class:`Column`.
+    Parameters
+    ----------
+    other
+        a value or :class:`Column` to calculate bitwise xor(^) with
+        this :class:`Column`.
 
+    Examples
+    --------
     >>> from pyspark.sql import Row
     >>> df = spark.createDataFrame([Row(a=170, b=75)])
     >>> df.select(df.a.bitwiseXOR(df.b)).collect()
@@ -286,12 +301,15 @@ def __contains__(self, item):
     bitwiseAND = _bin_op("bitwiseAND", _bitwiseAND_doc)
     bitwiseXOR = _bin_op("bitwiseXOR", _bitwiseXOR_doc)
 
-    @since(1.3)
     def getItem(self, key):
         """
         An expression that gets an item at position ``ordinal`` out of a list,
         or gets an item by key out of a dict.
 
+        .. versionadded:: 1.3.0
+
+        Examples
+        --------
         >>> df = spark.createDataFrame([([1, 2], {"key": "value"})], ["l", "d"])
         >>> df.select(df.l.getItem(0), df.d.getItem("key")).show()
         +----+------+
@@ -305,14 +323,18 @@ def getItem(self, key):
                 "A column as 'key' in getItem is deprecated as of Spark 3.0, and will not "
                 "be supported in the future release. Use `column[key]` or `column.key` syntax "
                 "instead.",
-                DeprecationWarning)
+                FutureWarning
+            )
         return self[key]
 
-    @since(1.3)
     def getField(self, name):
         """
         An expression that gets a field by name in a StructField.
 
+        .. versionadded:: 1.3.0
+
+        Examples
+        --------
         >>> from pyspark.sql import Row
         >>> df = spark.createDataFrame([Row(r=Row(a=1, b="b"))])
         >>> df.select(df.r.getField("b")).show()
@@ -333,9 +355,96 @@ def getField(self, name):
                 "A column as 'name' in getField is deprecated as of Spark 3.0, and will not "
                 "be supported in the future release. Use `column[name]` or `column.name` syntax "
                 "instead.",
-                DeprecationWarning)
+                FutureWarning
+            )
         return self[name]
 
+    def withField(self, fieldName, col):
+        """
+        An expression that adds/replaces a field in :class:`StructType` by name.
+
+        .. versionadded:: 3.1.0
+
+        Examples
+        --------
+        >>> from pyspark.sql import Row
+        >>> from pyspark.sql.functions import lit
+        >>> df = spark.createDataFrame([Row(a=Row(b=1, c=2))])
+        >>> df.withColumn('a', df['a'].withField('b', lit(3))).select('a.b').show()
+        +---+
+        |  b|
+        +---+
+        |  3|
+        +---+
+        >>> df.withColumn('a', df['a'].withField('d', lit(4))).select('a.d').show()
+        +---+
+        |  d|
+        +---+
+        |  4|
+        +---+
+        """
+        if not isinstance(fieldName, str):
+            raise TypeError("fieldName should be a string")
+
+        if not isinstance(col, Column):
+            raise TypeError("col should be a Column")
+
+        return Column(self._jc.withField(fieldName, col._jc))
+
+    def dropFields(self, *fieldNames):
+        """
+        An expression that drops fields in :class:`StructType` by name.
+
+        .. versionadded:: 3.1.0
+
+        Examples
+        --------
+        >>> from pyspark.sql import Row
+        >>> from pyspark.sql.functions import col, lit
+        >>> df = spark.createDataFrame([
+        ...     Row(a=Row(b=1, c=2, d=3, e=Row(f=4, g=5, h=6)))])
+        >>> df.withColumn('a', df['a'].dropFields('b')).show()
+        +-----------------+
+        |                a|
+        +-----------------+
+        |{2, 3, {4, 5, 6}}|
+        +-----------------+
+
+        >>> df.withColumn('a', df['a'].dropFields('b', 'c')).show()
+        +--------------+
+        |             a|
+        +--------------+
+        |{3, {4, 5, 6}}|
+        +--------------+
+
+        This method supports dropping multiple nested fields directly e.g.
+
+        >>> df.withColumn("a", col("a").dropFields("e.g", "e.h")).show()
+        +--------------+
+        |             a|
+        +--------------+
+        |{1, 2, 3, {4}}|
+        +--------------+
+
+        However, if you are going to add/replace multiple nested fields,
+        it is preferred to extract out the nested struct before
+        adding/replacing multiple fields e.g.
+
+        >>> df.select(col("a").withField(
+        ...     "e", col("a.e").dropFields("g", "h")).alias("a")
+        ... ).show()
+        +--------------+
+        |             a|
+        +--------------+
+        |{1, 2, 3, {4}}|
+        +--------------+
+
+        """
+        sc = SparkContext._active_spark_context
+
+        jc = self._jc.dropFields(_to_seq(sc, fieldNames))
+        return Column(jc)
+
     def __getattr__(self, item):
         if item.startswith("__"):
             raise AttributeError(item)
@@ -356,68 +465,101 @@ def __iter__(self):
     _contains_doc = """
     Contains the other element. Returns a boolean :class:`Column` based on a string match.
 
-    :param other: string in line
+    Parameters
+    ----------
+    other
+        string in line. A value as a literal or a :class:`Column`.
 
+    Examples
+    --------
     >>> df.filter(df.name.contains('o')).collect()
-    [Row(age=5, name=u'Bob')]
+    [Row(age=5, name='Bob')]
     """
     _rlike_doc = """
     SQL RLIKE expression (LIKE with Regex). Returns a boolean :class:`Column` based on a regex
     match.
 
-    :param other: an extended regex expression
+    Parameters
+    ----------
+    other : str
+        an extended regex expression
 
+    Examples
+    --------
     >>> df.filter(df.name.rlike('ice$')).collect()
-    [Row(age=2, name=u'Alice')]
+    [Row(age=2, name='Alice')]
     """
     _like_doc = """
     SQL like expression. Returns a boolean :class:`Column` based on a SQL LIKE match.
 
-    :param other: a SQL LIKE pattern
+    Parameters
+    ----------
+    other : str
+        a SQL LIKE pattern
 
-    See :func:`rlike` for a regex version
+    See Also
+    --------
+    pyspark.sql.Column.rlike
 
+    Examples
+    --------
     >>> df.filter(df.name.like('Al%')).collect()
-    [Row(age=2, name=u'Alice')]
+    [Row(age=2, name='Alice')]
     """
     _startswith_doc = """
     String starts with. Returns a boolean :class:`Column` based on a string match.
 
-    :param other: string at start of line (do not use a regex `^`)
+    Parameters
+    ----------
+    other : :class:`Column` or str
+        string at start of line (do not use a regex `^`)
 
+    Examples
+    --------
     >>> df.filter(df.name.startswith('Al')).collect()
-    [Row(age=2, name=u'Alice')]
+    [Row(age=2, name='Alice')]
     >>> df.filter(df.name.startswith('^Al')).collect()
     []
     """
     _endswith_doc = """
     String ends with. Returns a boolean :class:`Column` based on a string match.
 
-    :param other: string at end of line (do not use a regex `$`)
+    Parameters
+    ----------
+    other : :class:`Column` or str
+        string at end of line (do not use a regex `$`)
 
+    Examples
+    --------
     >>> df.filter(df.name.endswith('ice')).collect()
-    [Row(age=2, name=u'Alice')]
+    [Row(age=2, name='Alice')]
     >>> df.filter(df.name.endswith('ice$')).collect()
     []
     """
 
-    contains = ignore_unicode_prefix(_bin_op("contains", _contains_doc))
-    rlike = ignore_unicode_prefix(_bin_op("rlike", _rlike_doc))
-    like = ignore_unicode_prefix(_bin_op("like", _like_doc))
-    startswith = ignore_unicode_prefix(_bin_op("startsWith", _startswith_doc))
-    endswith = ignore_unicode_prefix(_bin_op("endsWith", _endswith_doc))
+    contains = _bin_op("contains", _contains_doc)
+    rlike = _bin_op("rlike", _rlike_doc)
+    like = _bin_op("like", _like_doc)
+    startswith = _bin_op("startsWith", _startswith_doc)
+    endswith = _bin_op("endsWith", _endswith_doc)
 
-    @ignore_unicode_prefix
-    @since(1.3)
     def substr(self, startPos, length):
         """
         Return a :class:`Column` which is a substring of the column.
 
-        :param startPos: start position (int or Column)
-        :param length:  length of the substring (int or Column)
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        startPos : :class:`Column` or int
+            start position
+        length : :class:`Column` or int
+            length of the substring
 
+        Examples
+        --------
         >>> df.select(df.name.substr(1, 3).alias("col")).collect()
-        [Row(col=u'Ali'), Row(col=u'Bob')]
+        [Row(col='Ali'), Row(col='Bob')]
         """
         if type(startPos) != type(length):
             raise TypeError(
@@ -435,17 +577,19 @@ def substr(self, startPos, length):
             raise TypeError("Unexpected type: %s" % type(startPos))
         return Column(jc)
 
-    @ignore_unicode_prefix
-    @since(1.5)
     def isin(self, *cols):
         """
         A boolean expression that is evaluated to true if the value of this
         expression is contained by the evaluated values of the arguments.
 
+        .. versionadded:: 1.5.0
+
+        Examples
+        --------
         >>> df[df.name.isin("Bob", "Mike")].collect()
-        [Row(age=5, name=u'Bob')]
+        [Row(age=5, name='Bob')]
         >>> df[df.age.isin([1, 2, 3])].collect()
-        [Row(age=2, name=u'Alice')]
+        [Row(age=2, name='Alice')]
         """
         if len(cols) == 1 and isinstance(cols[0], (list, set)):
             cols = cols[0]
@@ -458,105 +602,136 @@ def isin(self, *cols):
     _asc_doc = """
     Returns a sort expression based on ascending order of the column.
 
+    Examples
+    --------
     >>> from pyspark.sql import Row
     >>> df = spark.createDataFrame([('Tom', 80), ('Alice', None)], ["name", "height"])
     >>> df.select(df.name).orderBy(df.name.asc()).collect()
-    [Row(name=u'Alice'), Row(name=u'Tom')]
+    [Row(name='Alice'), Row(name='Tom')]
     """
     _asc_nulls_first_doc = """
     Returns a sort expression based on ascending order of the column, and null values
     return before non-null values.
 
+    .. versionadded:: 2.4.0
+
+    Examples
+    --------
     >>> from pyspark.sql import Row
     >>> df = spark.createDataFrame([('Tom', 80), (None, 60), ('Alice', None)], ["name", "height"])
     >>> df.select(df.name).orderBy(df.name.asc_nulls_first()).collect()
-    [Row(name=None), Row(name=u'Alice'), Row(name=u'Tom')]
+    [Row(name=None), Row(name='Alice'), Row(name='Tom')]
 
-    .. versionadded:: 2.4
     """
     _asc_nulls_last_doc = """
     Returns a sort expression based on ascending order of the column, and null values
     appear after non-null values.
 
+    .. versionadded:: 2.4.0
+
+    Examples
+    --------
     >>> from pyspark.sql import Row
     >>> df = spark.createDataFrame([('Tom', 80), (None, 60), ('Alice', None)], ["name", "height"])
     >>> df.select(df.name).orderBy(df.name.asc_nulls_last()).collect()
-    [Row(name=u'Alice'), Row(name=u'Tom'), Row(name=None)]
+    [Row(name='Alice'), Row(name='Tom'), Row(name=None)]
 
-    .. versionadded:: 2.4
     """
     _desc_doc = """
     Returns a sort expression based on the descending order of the column.
 
+    .. versionadded:: 2.4.0
+
+    Examples
+    --------
     >>> from pyspark.sql import Row
     >>> df = spark.createDataFrame([('Tom', 80), ('Alice', None)], ["name", "height"])
     >>> df.select(df.name).orderBy(df.name.desc()).collect()
-    [Row(name=u'Tom'), Row(name=u'Alice')]
+    [Row(name='Tom'), Row(name='Alice')]
     """
     _desc_nulls_first_doc = """
     Returns a sort expression based on the descending order of the column, and null values
     appear before non-null values.
 
+    .. versionadded:: 2.4.0
+
+    Examples
+    --------
     >>> from pyspark.sql import Row
     >>> df = spark.createDataFrame([('Tom', 80), (None, 60), ('Alice', None)], ["name", "height"])
     >>> df.select(df.name).orderBy(df.name.desc_nulls_first()).collect()
-    [Row(name=None), Row(name=u'Tom'), Row(name=u'Alice')]
+    [Row(name=None), Row(name='Tom'), Row(name='Alice')]
 
-    .. versionadded:: 2.4
     """
     _desc_nulls_last_doc = """
     Returns a sort expression based on the descending order of the column, and null values
     appear after non-null values.
 
+    .. versionadded:: 2.4.0
+
+    Examples
+    --------
     >>> from pyspark.sql import Row
     >>> df = spark.createDataFrame([('Tom', 80), (None, 60), ('Alice', None)], ["name", "height"])
     >>> df.select(df.name).orderBy(df.name.desc_nulls_last()).collect()
-    [Row(name=u'Tom'), Row(name=u'Alice'), Row(name=None)]
-
-    .. versionadded:: 2.4
+    [Row(name='Tom'), Row(name='Alice'), Row(name=None)]
     """
 
-    asc = ignore_unicode_prefix(_unary_op("asc", _asc_doc))
-    asc_nulls_first = ignore_unicode_prefix(_unary_op("asc_nulls_first", _asc_nulls_first_doc))
-    asc_nulls_last = ignore_unicode_prefix(_unary_op("asc_nulls_last", _asc_nulls_last_doc))
-    desc = ignore_unicode_prefix(_unary_op("desc", _desc_doc))
-    desc_nulls_first = ignore_unicode_prefix(_unary_op("desc_nulls_first", _desc_nulls_first_doc))
-    desc_nulls_last = ignore_unicode_prefix(_unary_op("desc_nulls_last", _desc_nulls_last_doc))
+    asc = _unary_op("asc", _asc_doc)
+    asc_nulls_first = _unary_op("asc_nulls_first", _asc_nulls_first_doc)
+    asc_nulls_last = _unary_op("asc_nulls_last", _asc_nulls_last_doc)
+    desc = _unary_op("desc", _desc_doc)
+    desc_nulls_first = _unary_op("desc_nulls_first", _desc_nulls_first_doc)
+    desc_nulls_last = _unary_op("desc_nulls_last", _desc_nulls_last_doc)
 
     _isNull_doc = """
     True if the current expression is null.
 
+    Examples
+    --------
     >>> from pyspark.sql import Row
-    >>> df = spark.createDataFrame([Row(name=u'Tom', height=80), Row(name=u'Alice', height=None)])
+    >>> df = spark.createDataFrame([Row(name='Tom', height=80), Row(name='Alice', height=None)])
     >>> df.filter(df.height.isNull()).collect()
-    [Row(height=None, name=u'Alice')]
+    [Row(name='Alice', height=None)]
     """
     _isNotNull_doc = """
     True if the current expression is NOT null.
 
+    Examples
+    --------
     >>> from pyspark.sql import Row
-    >>> df = spark.createDataFrame([Row(name=u'Tom', height=80), Row(name=u'Alice', height=None)])
+    >>> df = spark.createDataFrame([Row(name='Tom', height=80), Row(name='Alice', height=None)])
     >>> df.filter(df.height.isNotNull()).collect()
-    [Row(height=80, name=u'Tom')]
+    [Row(name='Tom', height=80)]
     """
 
-    isNull = ignore_unicode_prefix(_unary_op("isNull", _isNull_doc))
-    isNotNull = ignore_unicode_prefix(_unary_op("isNotNull", _isNotNull_doc))
+    isNull = _unary_op("isNull", _isNull_doc)
+    isNotNull = _unary_op("isNotNull", _isNotNull_doc)
 
-    @since(1.3)
     def alias(self, *alias, **kwargs):
         """
         Returns this column aliased with a new name or names (in the case of expressions that
         return more than one column, such as explode).
 
-        :param alias: strings of desired column names (collects all positional arguments passed)
-        :param metadata: a dict of information to be stored in ``metadata`` attribute of the
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        alias : str
+            desired column names (collects all positional arguments passed)
+
+        Other Parameters
+        ----------------
+        metadata: dict
+            a dict of information to be stored in ``metadata`` attribute of the
             corresponding :class:`StructField <pyspark.sql.types.StructField>` (optional, keyword
             only argument)
 
-        .. versionchanged:: 2.2
-           Added optional ``metadata`` argument.
+            .. versionchanged:: 2.2.0
+               Added optional ``metadata`` argument.
 
+        Examples
+        --------
         >>> df.select(df.age.alias("age2")).collect()
         [Row(age2=2), Row(age2=5)]
         >>> df.select(df.age.alias("age3", metadata={'max': 99})).schema['age3'].metadata['max']
@@ -581,17 +756,19 @@ def alias(self, *alias, **kwargs):
 
     name = copy_func(alias, sinceversion=2.0, doc=":func:`name` is an alias for :func:`alias`.")
 
-    @ignore_unicode_prefix
-    @since(1.3)
     def cast(self, dataType):
         """ Convert the column into type ``dataType``.
 
+        .. versionadded:: 1.3.0
+
+        Examples
+        --------
         >>> df.select(df.age.cast("string").alias('ages')).collect()
-        [Row(ages=u'2'), Row(ages=u'5')]
+        [Row(ages='2'), Row(ages='5')]
         >>> df.select(df.age.cast(StringType()).alias('ages')).collect()
-        [Row(ages=u'2'), Row(ages=u'5')]
+        [Row(ages='2'), Row(ages='5')]
         """
-        if isinstance(dataType, basestring):
+        if isinstance(dataType, str):
             jc = self._jc.cast(dataType)
         elif isinstance(dataType, DataType):
             from pyspark.sql import SparkSession
@@ -604,12 +781,15 @@ def cast(self, dataType):
 
     astype = copy_func(cast, sinceversion=1.4, doc=":func:`astype` is an alias for :func:`cast`.")
 
-    @since(1.3)
     def between(self, lowerBound, upperBound):
         """
         A boolean expression that is evaluated to true if the value of this
         expression is between the given columns.
 
+        .. versionadded:: 1.3.0
+
+        Examples
+        --------
         >>> df.select(df.name, df.age.between(2, 4)).show()
         +-----+---------------------------+
         | name|((age >= 2) AND (age <= 4))|
@@ -620,17 +800,22 @@ def between(self, lowerBound, upperBound):
         """
         return (self >= lowerBound) & (self <= upperBound)
 
-    @since(1.4)
     def when(self, condition, value):
         """
         Evaluates a list of conditions and returns one of multiple possible result expressions.
         If :func:`Column.otherwise` is not invoked, None is returned for unmatched conditions.
 
-        See :func:`pyspark.sql.functions.when` for example usage.
+        .. versionadded:: 1.4.0
 
-        :param condition: a boolean :class:`Column` expression.
-        :param value: a literal value, or a :class:`Column` expression.
+        Parameters
+        ----------
+        condition : :class:`Column`
+            a boolean :class:`Column` expression.
+        value
+            a literal value, or a :class:`Column` expression.
 
+        Examples
+        --------
         >>> from pyspark.sql import functions as F
         >>> df.select(df.name, F.when(df.age > 4, 1).when(df.age < 3, -1).otherwise(0)).show()
         +-----+------------------------------------------------------------+
@@ -639,6 +824,10 @@ def when(self, condition, value):
         |Alice|                                                          -1|
         |  Bob|                                                           1|
         +-----+------------------------------------------------------------+
+
+        See Also
+        --------
+        pyspark.sql.functions.when
         """
         if not isinstance(condition, Column):
             raise TypeError("condition should be a Column")
@@ -646,16 +835,20 @@ def when(self, condition, value):
         jc = self._jc.when(condition._jc, v)
         return Column(jc)
 
-    @since(1.4)
     def otherwise(self, value):
         """
         Evaluates a list of conditions and returns one of multiple possible result expressions.
         If :func:`Column.otherwise` is not invoked, None is returned for unmatched conditions.
 
-        See :func:`pyspark.sql.functions.when` for example usage.
+        .. versionadded:: 1.4.0
 
-        :param value: a literal value, or a :class:`Column` expression.
+        Parameters
+        ----------
+        value
+            a literal value, or a :class:`Column` expression.
 
+        Examples
+        --------
         >>> from pyspark.sql import functions as F
         >>> df.select(df.name, F.when(df.age > 3, 1).otherwise(0)).show()
         +-----+-------------------------------------+
@@ -664,19 +857,31 @@ def otherwise(self, value):
         |Alice|                                    0|
         |  Bob|                                    1|
         +-----+-------------------------------------+
+
+        See Also
+        --------
+        pyspark.sql.functions.when
         """
         v = value._jc if isinstance(value, Column) else value
         jc = self._jc.otherwise(v)
         return Column(jc)
 
-    @since(1.4)
     def over(self, window):
         """
         Define a windowing column.
 
-        :param window: a :class:`WindowSpec`
-        :return: a Column
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        window : :class:`WindowSpec`
+
+        Returns
+        -------
+        :class:`Column`
 
+        Examples
+        --------
         >>> from pyspark.sql import Window
         >>> window = Window.partitionBy("name").orderBy("age") \
                 .rowsBetween(Window.unboundedPreceding, Window.currentRow)
@@ -703,7 +908,7 @@ def __nonzero__(self):
     __bool__ = __nonzero__
 
     def __repr__(self):
-        return 'Column<%s>' % self._jc.toString().encode('utf8')
+        return "Column<'%s'>" % self._jc.toString()
 
 
 def _test():
diff --git a/python/pyspark/sql/column.pyi b/python/pyspark/sql/column.pyi
new file mode 100644
index 0000000000000..36c1bcc1961a8
--- /dev/null
+++ b/python/pyspark/sql/column.pyi
@@ -0,0 +1,118 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import overload
+from typing import Any, Union
+
+from pyspark.sql._typing import LiteralType, DecimalLiteral, DateTimeLiteral
+from pyspark.sql.types import (  # noqa: F401
+    DataType,
+    StructField,
+    StructType,
+    IntegerType,
+    StringType,
+)
+from pyspark.sql.window import WindowSpec
+
+from py4j.java_gateway import JavaObject  # type: ignore[import]
+
+class Column:
+    def __init__(self, jc: JavaObject) -> None: ...
+    def __neg__(self) -> Column: ...
+    def __add__(self, other: Union[Column, LiteralType, DecimalLiteral]) -> Column: ...
+    def __sub__(self, other: Union[Column, LiteralType, DecimalLiteral]) -> Column: ...
+    def __mul__(self, other: Union[Column, LiteralType, DecimalLiteral]) -> Column: ...
+    def __div__(self, other: Union[Column, LiteralType, DecimalLiteral]) -> Column: ...
+    def __truediv__(
+        self, other: Union[Column, LiteralType, DecimalLiteral]
+    ) -> Column: ...
+    def __mod__(self, other: Union[Column, LiteralType, DecimalLiteral]) -> Column: ...
+    def __radd__(self, other: Union[LiteralType, DecimalLiteral]) -> Column: ...
+    def __rsub__(self, other: Union[LiteralType, DecimalLiteral]) -> Column: ...
+    def __rmul__(self, other: Union[LiteralType, DecimalLiteral]) -> Column: ...
+    def __rdiv__(self, other: Union[LiteralType, DecimalLiteral]) -> Column: ...
+    def __rtruediv__(self, other: Union[LiteralType, DecimalLiteral]) -> Column: ...
+    def __rmod__(self, other: Union[bool, int, float, DecimalLiteral]) -> Column: ...
+    def __pow__(self, other: Union[Column, LiteralType, DecimalLiteral]) -> Column: ...
+    def __rpow__(self, other: Union[LiteralType, DecimalLiteral]) -> Column: ...
+    def __eq__(self, other: Union[Column, LiteralType, DateTimeLiteral, DecimalLiteral]) -> Column: ...  # type: ignore[override]
+    def __ne__(self, other: Any) -> Column: ...  # type: ignore[override]
+    def __lt__(
+        self, other: Union[Column, LiteralType, DateTimeLiteral, DecimalLiteral]
+    ) -> Column: ...
+    def __le__(
+        self, other: Union[Column, LiteralType, DateTimeLiteral, DecimalLiteral]
+    ) -> Column: ...
+    def __ge__(
+        self, other: Union[Column, LiteralType, DateTimeLiteral, DecimalLiteral]
+    ) -> Column: ...
+    def __gt__(
+        self, other: Union[Column, LiteralType, DateTimeLiteral, DecimalLiteral]
+    ) -> Column: ...
+    def eqNullSafe(
+        self, other: Union[Column, LiteralType, DecimalLiteral]
+    ) -> Column: ...
+    def __and__(self, other: Column) -> Column: ...
+    def __or__(self, other: Column) -> Column: ...
+    def __invert__(self) -> Column: ...
+    def __rand__(self, other: Column) -> Column: ...
+    def __ror__(self, other: Column) -> Column: ...
+    def __contains__(self, other: Any) -> Column: ...
+    def __getitem__(self, other: Any) -> Column: ...
+    def bitwiseOR(self, other: Union[Column, int]) -> Column: ...
+    def bitwiseAND(self, other: Union[Column, int]) -> Column: ...
+    def bitwiseXOR(self, other: Union[Column, int]) -> Column: ...
+    def getItem(self, key: Any) -> Column: ...
+    def getField(self, name: Any) -> Column: ...
+    def withField(self, fieldName: str, col: Column) -> Column: ...
+    def dropFields(self, *fieldNames: str) -> Column: ...
+    def __getattr__(self, item: Any) -> Column: ...
+    def __iter__(self) -> None: ...
+    def rlike(self, item: str) -> Column: ...
+    def like(self, item: str) -> Column: ...
+    def startswith(self, item: Union[str, Column]) -> Column: ...
+    def endswith(self, item: Union[str, Column]) -> Column: ...
+    @overload
+    def substr(self, startPos: int, length: int) -> Column: ...
+    @overload
+    def substr(self, startPos: Column, length: Column) -> Column: ...
+    def __getslice__(self, startPos: int, length: int) -> Column: ...
+    def isin(self, *cols: Any) -> Column: ...
+    def asc(self) -> Column: ...
+    def asc_nulls_first(self) -> Column: ...
+    def asc_nulls_last(self) -> Column: ...
+    def desc(self) -> Column: ...
+    def desc_nulls_first(self) -> Column: ...
+    def desc_nulls_last(self) -> Column: ...
+    def isNull(self) -> Column: ...
+    def isNotNull(self) -> Column: ...
+    def alias(self, *alias: str, **kwargs: Any) -> Column: ...
+    def name(self, *alias: str) -> Column: ...
+    def cast(self, dataType: Union[DataType, str]) -> Column: ...
+    def astype(self, dataType: Union[DataType, str]) -> Column: ...
+    def between(
+        self,
+        lowerBound: Union[Column, LiteralType, DateTimeLiteral, DecimalLiteral],
+        upperBound: Union[Column, LiteralType, DateTimeLiteral, DecimalLiteral],
+    ) -> Column: ...
+    def when(self, condition: Column, value: Any) -> Column: ...
+    def otherwise(self, value: Any) -> Column: ...
+    def over(self, window: WindowSpec) -> Column: ...
+    def __nonzero__(self) -> None: ...
+    def __bool__(self) -> None: ...
+    def contains(self, item: Any) -> Column: ...
diff --git a/python/pyspark/sql/conf.py b/python/pyspark/sql/conf.py
index 71ea1631718f1..eab084a1faddf 100644
--- a/python/pyspark/sql/conf.py
+++ b/python/pyspark/sql/conf.py
@@ -18,10 +18,6 @@
 import sys
 
 from pyspark import since, _NoValue
-from pyspark.rdd import ignore_unicode_prefix
-
-if sys.version_info[0] >= 3:
-    basestring = str
 
 
 class RuntimeConfig(object):
@@ -34,13 +30,11 @@ def __init__(self, jconf):
         """Create a new RuntimeConfig that wraps the underlying JVM object."""
         self._jconf = jconf
 
-    @ignore_unicode_prefix
     @since(2.0)
     def set(self, key, value):
         """Sets the given Spark runtime configuration property."""
         self._jconf.set(key, value)
 
-    @ignore_unicode_prefix
     @since(2.0)
     def get(self, key, default=_NoValue):
         """Returns the value of Spark runtime configuration property for the given key,
@@ -54,7 +48,6 @@ def get(self, key, default=_NoValue):
                 self._checkType(default, "default")
             return self._jconf.get(key, default)
 
-    @ignore_unicode_prefix
     @since(2.0)
     def unset(self, key):
         """Resets the configuration property for the given key."""
@@ -62,11 +55,10 @@ def unset(self, key):
 
     def _checkType(self, obj, identifier):
         """Assert that an object is of type str."""
-        if not isinstance(obj, basestring):
+        if not isinstance(obj, str):
             raise TypeError("expected %s '%s' to be a string (was '%s')" %
                             (identifier, obj, type(obj).__name__))
 
-    @ignore_unicode_prefix
     @since(2.4)
     def isModifiable(self, key):
         """Indicates whether the configuration property with the given key
diff --git a/python/pyspark/sql/conf.pyi b/python/pyspark/sql/conf.pyi
new file mode 100644
index 0000000000000..3e88f845a1165
--- /dev/null
+++ b/python/pyspark/sql/conf.pyi
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Optional
+from py4j.java_gateway import JavaObject  # type: ignore[import]
+
+class RuntimeConfig:
+    def __init__(self, jconf: JavaObject) -> None: ...
+    def set(self, key: str, value: str) -> None: ...
+    def get(self, key: str, default: Optional[str] = ...) -> str: ...
+    def unset(self, key: str) -> None: ...
+    def isModifiable(self, key: str) -> bool: ...
diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py
index 956343a2310b8..9e5e650eb86e4 100644
--- a/python/pyspark/sql/context.py
+++ b/python/pyspark/sql/context.py
@@ -15,21 +15,15 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
 import sys
 import warnings
 
-if sys.version >= '3':
-    basestring = unicode = str
-
 from pyspark import since, _NoValue
-from pyspark.rdd import ignore_unicode_prefix
 from pyspark.sql.session import _monkey_patch_RDD, SparkSession
 from pyspark.sql.dataframe import DataFrame
 from pyspark.sql.readwriter import DataFrameReader
 from pyspark.sql.streaming import DataStreamReader
-from pyspark.sql.types import IntegerType, Row, StringType
-from pyspark.sql.udf import UDFRegistration
+from pyspark.sql.udf import UDFRegistration  # noqa: F401
 from pyspark.sql.utils import install_exception_handler
 
 __all__ = ["SQLContext", "HiveContext"]
@@ -44,37 +38,46 @@ class SQLContext(object):
     A SQLContext can be used create :class:`DataFrame`, register :class:`DataFrame` as
     tables, execute SQL over tables, cache tables, and read parquet files.
 
-    :param sparkContext: The :class:`SparkContext` backing this SQLContext.
-    :param sparkSession: The :class:`SparkSession` around which this SQLContext wraps.
-    :param jsqlContext: An optional JVM Scala SQLContext. If set, we do not instantiate a new
+    .. deprecated:: 3.0.0
+        Use :func:`SparkSession.builder.getOrCreate()` instead.
+
+    Parameters
+    ----------
+    sparkContext : :class:`SparkContext`
+        The :class:`SparkContext` backing this SQLContext.
+    sparkSession : :class:`SparkSession`
+        The :class:`SparkSession` around which this SQLContext wraps.
+    jsqlContext : optional
+        An optional JVM Scala SQLContext. If set, we do not instantiate a new
         SQLContext in the JVM, instead we make all calls to this object.
+        This is only for internal.
+
+    Examples
+    --------
+    >>> from datetime import datetime
+    >>> from pyspark.sql import Row
+    >>> sqlContext = SQLContext(sc)
+    >>> allTypes = sc.parallelize([Row(i=1, s="string", d=1.0, l=1,
+    ...     b=True, list=[1, 2, 3], dict={"s": 0}, row=Row(a=1),
+    ...     time=datetime(2014, 8, 1, 14, 1, 5))])
+    >>> df = allTypes.toDF()
+    >>> df.createOrReplaceTempView("allTypes")
+    >>> sqlContext.sql('select i+1, d+1, not b, list[1], dict["s"], time, row.a '
+    ...            'from allTypes where b and i > 0').collect()
+    [Row((i + 1)=2, (d + 1)=2.0, (NOT b)=False, list[1]=2, \
+        dict[s]=0, time=datetime.datetime(2014, 8, 1, 14, 1, 5), a=1)]
+    >>> df.rdd.map(lambda x: (x.i, x.s, x.d, x.l, x.b, x.time, x.row.a, x.list)).collect()
+    [(1, 'string', 1.0, 1, True, datetime.datetime(2014, 8, 1, 14, 1, 5), 1, [1, 2, 3])]
     """
 
     _instantiatedContext = None
 
-    @ignore_unicode_prefix
     def __init__(self, sparkContext, sparkSession=None, jsqlContext=None):
-        """Creates a new SQLContext.
-
-        .. note:: Deprecated in 3.0.0. Use :func:`SparkSession.builder.getOrCreate()` instead.
-
-        >>> from datetime import datetime
-        >>> sqlContext = SQLContext(sc)
-        >>> allTypes = sc.parallelize([Row(i=1, s="string", d=1.0, l=1,
-        ...     b=True, list=[1, 2, 3], dict={"s": 0}, row=Row(a=1),
-        ...     time=datetime(2014, 8, 1, 14, 1, 5))])
-        >>> df = allTypes.toDF()
-        >>> df.createOrReplaceTempView("allTypes")
-        >>> sqlContext.sql('select i+1, d+1, not b, list[1], dict["s"], time, row.a '
-        ...            'from allTypes where b and i > 0').collect()
-        [Row((i + CAST(1 AS BIGINT))=2, (d + CAST(1 AS DOUBLE))=2.0, (NOT b)=False, list[1]=2, \
-            dict[s]=0, time=datetime.datetime(2014, 8, 1, 14, 1, 5), a=1)]
-        >>> df.rdd.map(lambda x: (x.i, x.s, x.d, x.l, x.b, x.time, x.row.a, x.list)).collect()
-        [(1, u'string', 1.0, 1, True, datetime.datetime(2014, 8, 1, 14, 1, 5), 1, [1, 2, 3])]
-        """
-        warnings.warn(
-            "Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate() instead.",
-            DeprecationWarning)
+        if sparkSession is None:
+            warnings.warn(
+                "Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate() instead.",
+                FutureWarning
+            )
 
         self._sc = sparkContext
         self._jsc = self._sc._jsc
@@ -106,18 +109,23 @@ def _conf(self):
         return self.sparkSession._jsparkSession.sessionState().conf()
 
     @classmethod
-    @since(1.6)
     def getOrCreate(cls, sc):
         """
         Get the existing SQLContext or create a new one with given SparkContext.
 
-        :param sc: SparkContext
+        .. versionadded:: 1.6.0
+
+        .. deprecated:: 3.0.0
+            Use :func:`SparkSession.builder.getOrCreate()` instead.
 
-        .. note:: Deprecated in 3.0.0. Use :func:`SparkSession.builder.getOrCreate()` instead.
+        Parameters
+        ----------
+        sc : :class:`SparkContext`
         """
         warnings.warn(
             "Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate() instead.",
-            DeprecationWarning)
+            FutureWarning
+        )
 
         if (cls._instantiatedContext is None
                 or SQLContext._instantiatedContext._sc._jsc is None):
@@ -127,23 +135,23 @@ def getOrCreate(cls, sc):
             cls(sc, sparkSession, jsqlContext)
         return cls._instantiatedContext
 
-    @since(1.6)
     def newSession(self):
         """
         Returns a new SQLContext as new session, that has separate SQLConf,
         registered temporary views and UDFs, but shared SparkContext and
         table cache.
+
+        .. versionadded:: 1.6.0
         """
         return self.__class__(self._sc, self.sparkSession.newSession())
 
-    @since(1.3)
     def setConf(self, key, value):
         """Sets the given Spark SQL configuration property.
+
+        .. versionadded:: 1.3.0
         """
         self.sparkSession.conf.set(key, value)
 
-    @ignore_unicode_prefix
-    @since(1.3)
     def getConf(self, key, defaultValue=_NoValue):
         """Returns the value of Spark SQL configuration property for the given key.
 
@@ -151,38 +159,57 @@ def getConf(self, key, defaultValue=_NoValue):
         defaultValue. If the key is not set and defaultValue is not set, return
         the system default value.
 
+        .. versionadded:: 1.3.0
+
+        Examples
+        --------
         >>> sqlContext.getConf("spark.sql.shuffle.partitions")
-        u'200'
-        >>> sqlContext.getConf("spark.sql.shuffle.partitions", u"10")
-        u'10'
-        >>> sqlContext.setConf("spark.sql.shuffle.partitions", u"50")
-        >>> sqlContext.getConf("spark.sql.shuffle.partitions", u"10")
-        u'50'
+        '200'
+        >>> sqlContext.getConf("spark.sql.shuffle.partitions", "10")
+        '10'
+        >>> sqlContext.setConf("spark.sql.shuffle.partitions", "50")
+        >>> sqlContext.getConf("spark.sql.shuffle.partitions", "10")
+        '50'
         """
         return self.sparkSession.conf.get(key, defaultValue)
 
     @property
-    @since("1.3.1")
     def udf(self):
         """Returns a :class:`UDFRegistration` for UDF registration.
 
-        :return: :class:`UDFRegistration`
+        .. versionadded:: 1.3.1
+
+        Returns
+        -------
+        :class:`UDFRegistration`
         """
         return self.sparkSession.udf
 
-    @since(1.4)
     def range(self, start, end=None, step=1, numPartitions=None):
         """
         Create a :class:`DataFrame` with single :class:`pyspark.sql.types.LongType` column named
         ``id``, containing elements in a range from ``start`` to ``end`` (exclusive) with
         step value ``step``.
 
-        :param start: the start value
-        :param end: the end value (exclusive)
-        :param step: the incremental step (default: 1)
-        :param numPartitions: the number of partitions of the DataFrame
-        :return: :class:`DataFrame`
-
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        start : int
+            the start value
+        end : int, optional
+            the end value (exclusive)
+        step : int, optional
+            the incremental step (default: 1)
+        numPartitions : int, optional
+            the number of partitions of the DataFrame
+
+        Returns
+        -------
+        :class:`DataFrame`
+
+        Examples
+        --------
         >>> sqlContext.range(1, 7, 2).collect()
         [Row(id=1), Row(id=3), Row(id=5)]
 
@@ -193,28 +220,34 @@ def range(self, start, end=None, step=1, numPartitions=None):
         """
         return self.sparkSession.range(start, end, step, numPartitions)
 
-    @since(1.2)
     def registerFunction(self, name, f, returnType=None):
         """An alias for :func:`spark.udf.register`.
         See :meth:`pyspark.sql.UDFRegistration.register`.
 
-        .. note:: Deprecated in 2.3.0. Use :func:`spark.udf.register` instead.
+        .. versionadded:: 1.2.0
+
+        .. deprecated:: 2.3.0
+            Use :func:`spark.udf.register` instead.
         """
         warnings.warn(
             "Deprecated in 2.3.0. Use spark.udf.register instead.",
-            DeprecationWarning)
+            FutureWarning
+        )
         return self.sparkSession.udf.register(name, f, returnType)
 
-    @since(2.1)
     def registerJavaFunction(self, name, javaClassName, returnType=None):
         """An alias for :func:`spark.udf.registerJavaFunction`.
         See :meth:`pyspark.sql.UDFRegistration.registerJavaFunction`.
 
-        .. note:: Deprecated in 2.3.0. Use :func:`spark.udf.registerJavaFunction` instead.
+        .. versionadded:: 2.1.0
+
+        .. deprecated:: 2.3.0
+            Use :func:`spark.udf.registerJavaFunction` instead.
         """
         warnings.warn(
             "Deprecated in 2.3.0. Use spark.udf.registerJavaFunction instead.",
-            DeprecationWarning)
+            FutureWarning
+        )
         return self.sparkSession.udf.registerJavaFunction(name, javaClassName, returnType)
 
     # TODO(andrew): delete this once we refactor things to take in SparkSession
@@ -222,14 +255,19 @@ def _inferSchema(self, rdd, samplingRatio=None):
         """
         Infer schema from an RDD of Row or tuple.
 
-        :param rdd: an RDD of Row or tuple
-        :param samplingRatio: sampling ratio, or no sampling (default)
-        :return: :class:`pyspark.sql.types.StructType`
+        Parameters
+        ----------
+        rdd : :class:`RDD`
+            an RDD of Row or tuple
+        samplingRatio : float, optional
+            sampling ratio, or no sampling (default)
+
+        Returns
+        -------
+        :class:`pyspark.sql.types.StructType`
         """
         return self.sparkSession._inferSchema(rdd, samplingRatio)
 
-    @since(1.3)
-    @ignore_unicode_prefix
     def createDataFrame(self, data, schema=None, samplingRatio=None, verifySchema=True):
         """
         Creates a :class:`DataFrame` from an :class:`RDD`, a list or a :class:`pandas.DataFrame`.
@@ -250,51 +288,64 @@ def createDataFrame(self, data, schema=None, samplingRatio=None, verifySchema=Tr
         If schema inference is needed, ``samplingRatio`` is used to determined the ratio of
         rows used for schema inference. The first row will be used if ``samplingRatio`` is ``None``.
 
-        :param data: an RDD of any kind of SQL data representation(e.g. :class:`Row`,
+        .. versionadded:: 1.3.0
+
+        .. versionchanged:: 2.0.0
+           The ``schema`` parameter can be a :class:`pyspark.sql.types.DataType` or a
+           datatype string after 2.0.
+           If it's not a :class:`pyspark.sql.types.StructType`, it will be wrapped into a
+           :class:`pyspark.sql.types.StructType` and each record will also be wrapped into a tuple.
+
+        .. versionchanged:: 2.1.0
+           Added verifySchema.
+
+        Parameters
+        ----------
+        data : :class:`RDD` or iterable
+            an RDD of any kind of SQL data representation (:class:`Row`,
             :class:`tuple`, ``int``, ``boolean``, etc.), or :class:`list`, or
             :class:`pandas.DataFrame`.
-        :param schema: a :class:`pyspark.sql.types.DataType` or a datatype string or a list of
+        schema : :class:`pyspark.sql.types.DataType`, str or list, optional
+            a :class:`pyspark.sql.types.DataType` or a datatype string or a list of
             column names, default is None.  The data type string format equals to
             :class:`pyspark.sql.types.DataType.simpleString`, except that top level struct type can
             omit the ``struct<>`` and atomic types use ``typeName()`` as their format, e.g. use
             ``byte`` instead of ``tinyint`` for :class:`pyspark.sql.types.ByteType`.
             We can also use ``int`` as a short name for :class:`pyspark.sql.types.IntegerType`.
-        :param samplingRatio: the sample ratio of rows used for inferring
-        :param verifySchema: verify data types of every row against schema.
-        :return: :class:`DataFrame`
+        samplingRatio : float, optional
+            the sample ratio of rows used for inferring
+        verifySchema : bool, optional
+            verify data types of every row against schema. Enabled by default.
 
-        .. versionchanged:: 2.0
-           The ``schema`` parameter can be a :class:`pyspark.sql.types.DataType` or a
-           datatype string after 2.0.
-           If it's not a :class:`pyspark.sql.types.StructType`, it will be wrapped into a
-           :class:`pyspark.sql.types.StructType` and each record will also be wrapped into a tuple.
-
-        .. versionchanged:: 2.1
-           Added verifySchema.
+        Returns
+        -------
+        :class:`DataFrame`
 
+        Examples
+        --------
         >>> l = [('Alice', 1)]
         >>> sqlContext.createDataFrame(l).collect()
-        [Row(_1=u'Alice', _2=1)]
+        [Row(_1='Alice', _2=1)]
         >>> sqlContext.createDataFrame(l, ['name', 'age']).collect()
-        [Row(name=u'Alice', age=1)]
+        [Row(name='Alice', age=1)]
 
         >>> d = [{'name': 'Alice', 'age': 1}]
         >>> sqlContext.createDataFrame(d).collect()
-        [Row(age=1, name=u'Alice')]
+        [Row(age=1, name='Alice')]
 
         >>> rdd = sc.parallelize(l)
         >>> sqlContext.createDataFrame(rdd).collect()
-        [Row(_1=u'Alice', _2=1)]
+        [Row(_1='Alice', _2=1)]
         >>> df = sqlContext.createDataFrame(rdd, ['name', 'age'])
         >>> df.collect()
-        [Row(name=u'Alice', age=1)]
+        [Row(name='Alice', age=1)]
 
         >>> from pyspark.sql import Row
         >>> Person = Row('name', 'age')
         >>> person = rdd.map(lambda r: Person(*r))
         >>> df2 = sqlContext.createDataFrame(person)
         >>> df2.collect()
-        [Row(name=u'Alice', age=1)]
+        [Row(name='Alice', age=1)]
 
         >>> from pyspark.sql.types import *
         >>> schema = StructType([
@@ -302,15 +353,15 @@ def createDataFrame(self, data, schema=None, samplingRatio=None, verifySchema=Tr
         ...    StructField("age", IntegerType(), True)])
         >>> df3 = sqlContext.createDataFrame(rdd, schema)
         >>> df3.collect()
-        [Row(name=u'Alice', age=1)]
+        [Row(name='Alice', age=1)]
 
         >>> sqlContext.createDataFrame(df.toPandas()).collect()  # doctest: +SKIP
-        [Row(name=u'Alice', age=1)]
+        [Row(name='Alice', age=1)]
         >>> sqlContext.createDataFrame(pandas.DataFrame([[1, 2]])).collect()  # doctest: +SKIP
         [Row(0=1, 1=2)]
 
         >>> sqlContext.createDataFrame(rdd, "a: string, b: int").collect()
-        [Row(a=u'Alice', b=1)]
+        [Row(a='Alice', b=1)]
         >>> rdd = rdd.map(lambda row: row[1])
         >>> sqlContext.createDataFrame(rdd, "int").collect()
         [Row(value=1)]
@@ -321,26 +372,31 @@ def createDataFrame(self, data, schema=None, samplingRatio=None, verifySchema=Tr
         """
         return self.sparkSession.createDataFrame(data, schema, samplingRatio, verifySchema)
 
-    @since(1.3)
     def registerDataFrameAsTable(self, df, tableName):
         """Registers the given :class:`DataFrame` as a temporary table in the catalog.
 
         Temporary tables exist only during the lifetime of this instance of :class:`SQLContext`.
 
+        .. versionadded:: 1.3.0
+
+        Examples
+        --------
         >>> sqlContext.registerDataFrameAsTable(df, "table1")
         """
         df.createOrReplaceTempView(tableName)
 
-    @since(1.6)
     def dropTempTable(self, tableName):
         """ Remove the temporary table from catalog.
 
+        .. versionadded:: 1.6.0
+
+        Examples
+        --------
         >>> sqlContext.registerDataFrameAsTable(df, "table1")
         >>> sqlContext.dropTempTable("table1")
         """
         self.sparkSession.catalog.dropTempView(tableName)
 
-    @since(1.3)
     def createExternalTable(self, tableName, path=None, source=None, schema=None, **options):
         """Creates an external table based on the dataset in a data source.
 
@@ -353,31 +409,44 @@ def createExternalTable(self, tableName, path=None, source=None, schema=None, **
         Optionally, a schema can be provided as the schema of the returned :class:`DataFrame` and
         created external table.
 
-        :return: :class:`DataFrame`
+        .. versionadded:: 1.3.0
+
+        Returns
+        -------
+        :class:`DataFrame`
         """
         return self.sparkSession.catalog.createExternalTable(
             tableName, path, source, schema, **options)
 
-    @ignore_unicode_prefix
-    @since(1.0)
     def sql(self, sqlQuery):
         """Returns a :class:`DataFrame` representing the result of the given query.
 
-        :return: :class:`DataFrame`
+        .. versionadded:: 1.0.0
+
+        Returns
+        -------
+        :class:`DataFrame`
 
+        Examples
+        --------
         >>> sqlContext.registerDataFrameAsTable(df, "table1")
         >>> df2 = sqlContext.sql("SELECT field1 AS f1, field2 as f2 from table1")
         >>> df2.collect()
-        [Row(f1=1, f2=u'row1'), Row(f1=2, f2=u'row2'), Row(f1=3, f2=u'row3')]
+        [Row(f1=1, f2='row1'), Row(f1=2, f2='row2'), Row(f1=3, f2='row3')]
         """
         return self.sparkSession.sql(sqlQuery)
 
-    @since(1.0)
     def table(self, tableName):
         """Returns the specified table or view as a :class:`DataFrame`.
 
-        :return: :class:`DataFrame`
+        .. versionadded:: 1.0.0
+
+        Returns
+        -------
+        :class:`DataFrame`
 
+        Examples
+        --------
         >>> sqlContext.registerDataFrameAsTable(df, "table1")
         >>> df2 = sqlContext.table("table1")
         >>> sorted(df.collect()) == sorted(df2.collect())
@@ -385,8 +454,6 @@ def table(self, tableName):
         """
         return self.sparkSession.table(tableName)
 
-    @ignore_unicode_prefix
-    @since(1.3)
     def tables(self, dbName=None):
         """Returns a :class:`DataFrame` containing names of tables in the given database.
 
@@ -395,25 +462,43 @@ def tables(self, dbName=None):
         The returned DataFrame has two columns: ``tableName`` and ``isTemporary``
         (a column with :class:`BooleanType` indicating if a table is a temporary one or not).
 
-        :param dbName: string, name of the database to use.
-        :return: :class:`DataFrame`
+        .. versionadded:: 1.3.0
 
+        Parameters
+        ----------
+        dbName: str, optional
+            name of the database to use.
+
+        Returns
+        -------
+        :class:`DataFrame`
+
+        Examples
+        --------
         >>> sqlContext.registerDataFrameAsTable(df, "table1")
         >>> df2 = sqlContext.tables()
         >>> df2.filter("tableName = 'table1'").first()
-        Row(database=u'', tableName=u'table1', isTemporary=True)
+        Row(namespace='', tableName='table1', isTemporary=True)
         """
         if dbName is None:
             return DataFrame(self._ssql_ctx.tables(), self)
         else:
             return DataFrame(self._ssql_ctx.tables(dbName), self)
 
-    @since(1.3)
     def tableNames(self, dbName=None):
         """Returns a list of names of tables in the database ``dbName``.
 
-        :param dbName: string, name of the database to use. Default to the current database.
-        :return: list of table names, in string
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        dbName: str
+            name of the database to use. Default to the current database.
+
+        Returns
+        -------
+        list
+            list of table names, in string
 
         >>> sqlContext.registerDataFrameAsTable(df, "table1")
         >>> "table1" in sqlContext.tableNames()
@@ -442,26 +527,34 @@ def clearCache(self):
         self._ssql_ctx.clearCache()
 
     @property
-    @since(1.4)
     def read(self):
         """
         Returns a :class:`DataFrameReader` that can be used to read data
         in as a :class:`DataFrame`.
 
-        :return: :class:`DataFrameReader`
+        .. versionadded:: 1.4.0
+
+        Returns
+        -------
+        :class:`DataFrameReader`
         """
         return DataFrameReader(self)
 
     @property
-    @since(2.0)
     def readStream(self):
         """
         Returns a :class:`DataStreamReader` that can be used to read data streams
         as a streaming :class:`DataFrame`.
 
-        .. note:: Evolving.
+        .. versionadded:: 2.0.0
 
-        :return: :class:`DataStreamReader`
+        Notes
+        -----
+        This API is evolving.
+
+        Returns
+        -------
+        :class:`DataStreamReader`
 
         >>> text_sdf = sqlContext.readStream.text(tempfile.mkdtemp())
         >>> text_sdf.isStreaming
@@ -470,12 +563,15 @@ def readStream(self):
         return DataStreamReader(self)
 
     @property
-    @since(2.0)
     def streams(self):
         """Returns a :class:`StreamingQueryManager` that allows managing all the
         :class:`StreamingQuery` StreamingQueries active on `this` context.
 
-        .. note:: Evolving.
+        .. versionadded:: 2.0.0
+
+        Notes
+        -----
+        This API is evolving.
         """
         from pyspark.sql.streaming import StreamingQueryManager
         return StreamingQueryManager(self._ssql_ctx.streams())
@@ -487,18 +583,26 @@ class HiveContext(SQLContext):
     Configuration for Hive is read from ``hive-site.xml`` on the classpath.
     It supports running both SQL and HiveQL commands.
 
-    :param sparkContext: The SparkContext to wrap.
-    :param jhiveContext: An optional JVM Scala HiveContext. If set, we do not instantiate a new
+    .. deprecated:: 2.0.0
+        Use SparkSession.builder.enableHiveSupport().getOrCreate().
+
+    Parameters
+    ----------
+    sparkContext : :class:`SparkContext`
+        The SparkContext to wrap.
+    jhiveContext : optional
+        An optional JVM Scala HiveContext. If set, we do not instantiate a new
         :class:`HiveContext` in the JVM, instead we make all calls to this object.
+        This is only for internal use.
 
-    .. note:: Deprecated in 2.0.0. Use SparkSession.builder.enableHiveSupport().getOrCreate().
     """
 
     def __init__(self, sparkContext, jhiveContext=None):
         warnings.warn(
             "HiveContext is deprecated in Spark 2.0.0. Please use " +
             "SparkSession.builder.enableHiveSupport().getOrCreate() instead.",
-            DeprecationWarning)
+            FutureWarning
+        )
         if jhiveContext is None:
             sparkContext._conf.set("spark.sql.catalogImplementation", "hive")
             sparkSession = SparkSession.builder._sparkContext(sparkContext).getOrCreate()
diff --git a/python/pyspark/sql/context.pyi b/python/pyspark/sql/context.pyi
new file mode 100644
index 0000000000000..e8b61c8d59ba9
--- /dev/null
+++ b/python/pyspark/sql/context.pyi
@@ -0,0 +1,140 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from pyspark.sql._typing import UserDefinedFunctionLike
+
+from typing import overload
+from typing import Any, Callable, Iterable, List, Optional, Tuple, TypeVar, Union
+
+from py4j.java_gateway import JavaObject  # type: ignore[import]
+
+from pyspark.sql._typing import (
+    DateTimeLiteral,
+    LiteralType,
+    DecimalLiteral,
+    RowLike,
+)
+from pyspark.sql.pandas._typing import DataFrameLike
+from pyspark.context import SparkContext
+from pyspark.rdd import RDD
+from pyspark.sql.dataframe import DataFrame
+from pyspark.sql.session import SparkSession
+from pyspark.sql.types import AtomicType, DataType, StructType
+from pyspark.sql.udf import UDFRegistration as UDFRegistration
+from pyspark.sql.readwriter import DataFrameReader
+from pyspark.sql.streaming import DataStreamReader, StreamingQueryManager
+
+T = TypeVar("T")
+
+class SQLContext:
+    sparkSession: SparkSession
+    def __init__(
+        self,
+        sparkContext: SparkContext,
+        sparkSession: Optional[SparkSession] = ...,
+        jsqlContext: Optional[JavaObject] = ...,
+    ) -> None: ...
+    @classmethod
+    def getOrCreate(cls: type, sc: SparkContext) -> SQLContext: ...
+    def newSession(self) -> SQLContext: ...
+    def setConf(self, key: str, value: Union[bool, int, str]) -> None: ...
+    def getConf(self, key: str, defaultValue: Optional[str] = ...) -> str: ...
+    @property
+    def udf(self) -> UDFRegistration: ...
+    def range(
+        self,
+        start: int,
+        end: Optional[int] = ...,
+        step: int = ...,
+        numPartitions: Optional[int] = ...,
+    ) -> DataFrame: ...
+    def registerFunction(
+        self, name: str, f: Callable[..., Any], returnType: DataType = ...
+    ) -> UserDefinedFunctionLike: ...
+    def registerJavaFunction(
+        self, name: str, javaClassName: str, returnType: Optional[DataType] = ...
+    ) -> None: ...
+    @overload
+    def createDataFrame(
+        self,
+        data: Union[RDD[RowLike], Iterable[RowLike]],
+        samplingRatio: Optional[float] = ...,
+    ) -> DataFrame: ...
+    @overload
+    def createDataFrame(
+        self,
+        data: Union[RDD[RowLike], Iterable[RowLike]],
+        schema: Union[List[str], Tuple[str, ...]] = ...,
+        verifySchema: bool = ...,
+    ) -> DataFrame: ...
+    @overload
+    def createDataFrame(
+        self,
+        data: Union[
+            RDD[Union[DateTimeLiteral, LiteralType, DecimalLiteral]],
+            Iterable[Union[DateTimeLiteral, LiteralType, DecimalLiteral]],
+        ],
+        schema: Union[AtomicType, str],
+        verifySchema: bool = ...,
+    ) -> DataFrame: ...
+    @overload
+    def createDataFrame(
+        self,
+        data: Union[RDD[RowLike], Iterable[RowLike]],
+        schema: Union[StructType, str],
+        verifySchema: bool = ...,
+    ) -> DataFrame: ...
+    @overload
+    def createDataFrame(
+        self, data: DataFrameLike, samplingRatio: Optional[float] = ...
+    ) -> DataFrame: ...
+    @overload
+    def createDataFrame(
+        self,
+        data: DataFrameLike,
+        schema: Union[StructType, str],
+        verifySchema: bool = ...,
+    ) -> DataFrame: ...
+    def registerDataFrameAsTable(self, df: DataFrame, tableName: str) -> None: ...
+    def dropTempTable(self, tableName: str) -> None: ...
+    def createExternalTable(
+        self,
+        tableName: str,
+        path: Optional[str] = ...,
+        source: Optional[str] = ...,
+        schema: Optional[StructType] = ...,
+        **options: str
+    ) -> DataFrame: ...
+    def sql(self, sqlQuery: str) -> DataFrame: ...
+    def table(self, tableName: str) -> DataFrame: ...
+    def tables(self, dbName: Optional[str] = ...) -> DataFrame: ...
+    def tableNames(self, dbName: Optional[str] = ...) -> List[str]: ...
+    def cacheTable(self, tableName: str) -> None: ...
+    def uncacheTable(self, tableName: str) -> None: ...
+    def clearCache(self) -> None: ...
+    @property
+    def read(self) -> DataFrameReader: ...
+    @property
+    def readStream(self) -> DataStreamReader: ...
+    @property
+    def streams(self) -> StreamingQueryManager: ...
+
+class HiveContext(SQLContext):
+    def __init__(
+        self, sparkContext: SparkContext, jhiveContext: Optional[JavaObject] = ...
+    ) -> None: ...
+    def refreshTable(self, tableName: str) -> None: ...
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index a3ce87096e790..8fe263e1520dd 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -17,30 +17,21 @@
 
 import sys
 import random
-
-if sys.version >= '3':
-    basestring = unicode = str
-    long = int
-    from functools import reduce
-    from html import escape as html_escape
-else:
-    from itertools import imap as map
-    from cgi import escape as html_escape
-
 import warnings
+from functools import reduce
+from html import escape as html_escape
 
 from pyspark import copy_func, since, _NoValue
-from pyspark.rdd import RDD, _load_from_socket, _local_iterator_from_socket, \
-    ignore_unicode_prefix
+from pyspark.rdd import RDD, _load_from_socket, _local_iterator_from_socket
 from pyspark.serializers import BatchedSerializer, PickleSerializer, \
     UTF8Deserializer
 from pyspark.storagelevel import StorageLevel
 from pyspark.traceback_utils import SCCallSiteSync
 from pyspark.sql.types import _parse_datatype_json_string
 from pyspark.sql.column import Column, _to_seq, _to_list, _to_java_column
-from pyspark.sql.readwriter import DataFrameWriter
+from pyspark.sql.readwriter import DataFrameWriter, DataFrameWriterV2
 from pyspark.sql.streaming import DataStreamWriter
-from pyspark.sql.types import *
+from pyspark.sql.types import StructType, StructField, StringType, IntegerType
 from pyspark.sql.pandas.conversion import PandasConversionMixin
 from pyspark.sql.pandas.map_ops import PandasMapOpsMixin
 
@@ -71,7 +62,7 @@ class DataFrame(PandasMapOpsMixin, PandasConversionMixin):
         people.filter(people.age > 30).join(department, people.deptId == department.id) \\
           .groupBy(department.name, "gender").agg({"salary": "avg", "age": "max"})
 
-    .. versionadded:: 1.3
+    .. versionadded:: 1.3.0
     """
 
     def __init__(self, jdf, sql_ctx):
@@ -109,39 +100,46 @@ def stat(self):
         """
         return DataFrameStatFunctions(self)
 
-    @ignore_unicode_prefix
-    @since(1.3)
     def toJSON(self, use_unicode=True):
         """Converts a :class:`DataFrame` into a :class:`RDD` of string.
 
         Each row is turned into a JSON document as one element in the returned RDD.
 
+        .. versionadded:: 1.3.0
+
+        Examples
+        --------
         >>> df.toJSON().first()
-        u'{"age":2,"name":"Alice"}'
+        '{"age":2,"name":"Alice"}'
         """
         rdd = self._jdf.toJSON()
         return RDD(rdd.toJavaRDD(), self._sc, UTF8Deserializer(use_unicode))
 
-    @since(1.3)
     def registerTempTable(self, name):
         """Registers this DataFrame as a temporary table using the given name.
 
         The lifetime of this temporary table is tied to the :class:`SparkSession`
         that was used to create this :class:`DataFrame`.
 
+        .. versionadded:: 1.3.0
+
+        .. deprecated:: 2.0.0
+            Use :meth:`DataFrame.createOrReplaceTempView` instead.
+
+        Examples
+        --------
         >>> df.registerTempTable("people")
         >>> df2 = spark.sql("select * from people")
         >>> sorted(df.collect()) == sorted(df2.collect())
         True
         >>> spark.catalog.dropTempView("people")
-
-        .. note:: Deprecated in 2.0, use createOrReplaceTempView instead.
         """
         warnings.warn(
-            "Deprecated in 2.0, use createOrReplaceTempView instead.", DeprecationWarning)
+            "Deprecated in 2.0, use createOrReplaceTempView instead.",
+            FutureWarning
+        )
         self._jdf.createOrReplaceTempView(name)
 
-    @since(2.0)
     def createTempView(self, name):
         """Creates a local temporary view with this :class:`DataFrame`.
 
@@ -150,6 +148,10 @@ def createTempView(self, name):
         throws :class:`TempTableAlreadyExistsException`, if the view name already exists in the
         catalog.
 
+        .. versionadded:: 2.0.0
+
+        Examples
+        --------
         >>> df.createTempView("people")
         >>> df2 = spark.sql("select * from people")
         >>> sorted(df.collect()) == sorted(df2.collect())
@@ -163,13 +165,16 @@ def createTempView(self, name):
         """
         self._jdf.createTempView(name)
 
-    @since(2.0)
     def createOrReplaceTempView(self, name):
         """Creates or replaces a local temporary view with this :class:`DataFrame`.
 
         The lifetime of this temporary table is tied to the :class:`SparkSession`
         that was used to create this :class:`DataFrame`.
 
+        .. versionadded:: 2.0.0
+
+        Examples
+        --------
         >>> df.createOrReplaceTempView("people")
         >>> df2 = df.filter(df.age > 3)
         >>> df2.createOrReplaceTempView("people")
@@ -181,7 +186,6 @@ def createOrReplaceTempView(self, name):
         """
         self._jdf.createOrReplaceTempView(name)
 
-    @since(2.1)
     def createGlobalTempView(self, name):
         """Creates a global temporary view with this :class:`DataFrame`.
 
@@ -189,6 +193,10 @@ def createGlobalTempView(self, name):
         throws :class:`TempTableAlreadyExistsException`, if the view name already exists in the
         catalog.
 
+        .. versionadded:: 2.1.0
+
+        Examples
+        --------
         >>> df.createGlobalTempView("people")
         >>> df2 = spark.sql("select * from global_temp.people")
         >>> sorted(df.collect()) == sorted(df2.collect())
@@ -202,12 +210,15 @@ def createGlobalTempView(self, name):
         """
         self._jdf.createGlobalTempView(name)
 
-    @since(2.2)
     def createOrReplaceGlobalTempView(self, name):
         """Creates or replaces a global temporary view using the given name.
 
         The lifetime of this temporary view is tied to this Spark application.
 
+        .. versionadded:: 2.2.0
+
+        Examples
+        --------
         >>> df.createOrReplaceGlobalTempView("people")
         >>> df2 = df.filter(df.age > 3)
         >>> df2.createOrReplaceGlobalTempView("people")
@@ -220,34 +231,45 @@ def createOrReplaceGlobalTempView(self, name):
         self._jdf.createOrReplaceGlobalTempView(name)
 
     @property
-    @since(1.4)
     def write(self):
         """
         Interface for saving the content of the non-streaming :class:`DataFrame` out into external
         storage.
 
-        :return: :class:`DataFrameWriter`
+        .. versionadded:: 1.4.0
+
+        Returns
+        -------
+        :class:`DataFrameWriter`
         """
         return DataFrameWriter(self)
 
     @property
-    @since(2.0)
     def writeStream(self):
         """
         Interface for saving the content of the streaming :class:`DataFrame` out into external
         storage.
 
-        .. note:: Evolving.
+        .. versionadded:: 2.0.0
+
+        Notes
+        -----
+        This API is evolving.
 
-        :return: :class:`DataStreamWriter`
+        Returns
+        -------
+        :class:`DataStreamWriter`
         """
         return DataStreamWriter(self)
 
     @property
-    @since(1.3)
     def schema(self):
         """Returns the schema of this :class:`DataFrame` as a :class:`pyspark.sql.types.StructType`.
 
+        .. versionadded:: 1.3.0
+
+        Examples
+        --------
         >>> df.schema
         StructType(List(StructField(age,IntegerType,true),StructField(name,StringType,true)))
         """
@@ -259,10 +281,13 @@ def schema(self):
                     "Unable to parse datatype from schema. %s" % e)
         return self._schema
 
-    @since(1.3)
     def printSchema(self):
         """Prints out the schema in the tree format.
 
+        .. versionadded:: 1.3.0
+
+        Examples
+        --------
         >>> df.printSchema()
         root
          |-- age: integer (nullable = true)
@@ -271,12 +296,19 @@ def printSchema(self):
         """
         print(self._jdf.schema().treeString())
 
-    @since(1.3)
     def explain(self, extended=None, mode=None):
         """Prints the (logical and physical) plans to the console for debugging purpose.
 
-        :param extended: boolean, default ``False``. If ``False``, prints only the physical plan.
-        :param mode: specifies the expected output format of plans.
+        .. versionadded:: 1.3.0
+
+        parameters
+        ----------
+        extended : bool, optional
+            default ``False``. If ``False``, prints only the physical plan.
+            When this is a string without specifying the ``mode``, it works as the mode is
+            specified.
+        mode : str, optional
+            specifies the expected output format of plans.
 
             * ``simple``: Print only a physical plan.
             * ``extended``: Print both logical and physical plans.
@@ -285,6 +317,11 @@ def explain(self, extended=None, mode=None):
             * ``formatted``: Split explain output into two sections: a physical plan outline \
                 and node details.
 
+            .. versionchanged:: 3.0.0
+               Added optional argument `mode` to specify the expected output format of plans.
+
+        Examples
+        --------
         >>> df.explain()
         == Physical Plan ==
         *(1) Scan ExistingRDD[age#0,name#1]
@@ -306,12 +343,14 @@ def explain(self, extended=None, mode=None):
         Output [2]: [age#0, name#1]
         ...
 
-        .. versionchanged:: 3.0.0
-           Added optional argument `mode` to specify the expected output format of plans.
+        >>> df.explain("cost")
+        == Optimized Logical Plan ==
+        ...Statistics...
+        ...
         """
 
         if extended is not None and mode is not None:
-            raise Exception("extended and mode can not be specified simultaneously")
+            raise Exception("extended and mode should not be set together.")
 
         # For the no argument case: df.explain()
         is_no_argument = extended is None and mode is None
@@ -319,18 +358,22 @@ def explain(self, extended=None, mode=None):
         # For the cases below:
         #   explain(True)
         #   explain(extended=False)
-        is_extended_case = extended is not None and isinstance(extended, bool)
+        is_extended_case = isinstance(extended, bool) and mode is None
+
+        # For the case when extended is mode:
+        #   df.explain("formatted")
+        is_extended_as_mode = isinstance(extended, str) and mode is None
 
-        # For the mode specified: df.explain(mode="formatted")
-        is_mode_case = mode is not None and isinstance(mode, basestring)
+        # For the mode specified:
+        #   df.explain(mode="formatted")
+        is_mode_case = extended is None and isinstance(mode, str)
 
-        if not is_no_argument and not (is_extended_case or is_mode_case):
-            if extended is not None:
-                err_msg = "extended (optional) should be provided as bool" \
-                    ", got {0}".format(type(extended))
-            else:  # For mode case
-                err_msg = "mode (optional) should be provided as str, got {0}".format(type(mode))
-            raise TypeError(err_msg)
+        if not (is_no_argument or is_extended_case or is_extended_as_mode or is_mode_case):
+            argtypes = [
+                str(type(arg)) for arg in [extended, mode] if arg is not None]
+            raise TypeError(
+                "extended (optional) and mode (optional) should be a string "
+                "and bool; however, got [%s]." % ", ".join(argtypes))
 
         # Sets an explain mode depending on a given argument
         if is_no_argument:
@@ -339,16 +382,22 @@ def explain(self, extended=None, mode=None):
             explain_mode = "extended" if extended else "simple"
         elif is_mode_case:
             explain_mode = mode
+        elif is_extended_as_mode:
+            explain_mode = extended
 
         print(self._sc._jvm.PythonSQLUtils.explainString(self._jdf.queryExecution(), explain_mode))
 
-    @since(2.4)
     def exceptAll(self, other):
         """Return a new :class:`DataFrame` containing rows in this :class:`DataFrame` but
         not in another :class:`DataFrame` while preserving duplicates.
 
         This is equivalent to `EXCEPT ALL` in SQL.
+        As standard in SQL, this function resolves columns by position (not by name).
+
+        .. versionadded:: 2.4.0
 
+        Examples
+        --------
         >>> df1 = spark.createDataFrame(
         ...         [("a", 1), ("a", 1), ("a", 1), ("a", 2), ("b",  3), ("c", 4)], ["C1", "C2"])
         >>> df2 = spark.createDataFrame([("a", 1), ("b", 3)], ["C1", "C2"])
@@ -363,7 +412,6 @@ def exceptAll(self, other):
         |  c|  4|
         +---+---+
 
-        Also as standard in SQL, this function resolves columns by position (not by name).
         """
         return DataFrame(self._jdf.exceptAll(other._jdf), self.sql_ctx)
 
@@ -375,7 +423,6 @@ def isLocal(self):
         return self._jdf.isLocal()
 
     @property
-    @since(2.0)
     def isStreaming(self):
         """Returns ``True`` if this :class:`Dataset` contains one or more sources that continuously
         return data as it arrives. A :class:`Dataset` that reads data from a streaming source
@@ -384,21 +431,33 @@ def isStreaming(self):
         :func:`collect`) will throw an :class:`AnalysisException` when there is a streaming
         source present.
 
-        .. note:: Evolving
+        .. versionadded:: 2.0.0
+
+        Notes
+        -----
+        This API is evolving.
         """
         return self._jdf.isStreaming()
 
-    @since(1.3)
     def show(self, n=20, truncate=True, vertical=False):
         """Prints the first ``n`` rows to the console.
 
-        :param n: Number of rows to show.
-        :param truncate: If set to ``True``, truncate strings longer than 20 chars by default.
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        n : int, optional
+            Number of rows to show.
+        truncate : bool, optional
+            If set to ``True``, truncate strings longer than 20 chars by default.
             If set to a number greater than one, truncates long strings to length ``truncate``
             and align cells right.
-        :param vertical: If set to ``True``, print output rows vertically (one line
+        vertical : bool, optional
+            If set to ``True``, print output rows vertically (one line
             per column value).
 
+        Examples
+        --------
         >>> df
         DataFrame[age: int, name: string]
         >>> df.show()
@@ -469,35 +528,46 @@ def _repr_html_(self):
         else:
             return None
 
-    @since(2.1)
     def checkpoint(self, eager=True):
         """Returns a checkpointed version of this Dataset. Checkpointing can be used to truncate the
         logical plan of this :class:`DataFrame`, which is especially useful in iterative algorithms
         where the plan may grow exponentially. It will be saved to files inside the checkpoint
         directory set with :meth:`SparkContext.setCheckpointDir`.
 
-        :param eager: Whether to checkpoint this :class:`DataFrame` immediately
+        .. versionadded:: 2.1.0
+
+        Parameters
+        ----------
+        eager : bool, optional
+            Whether to checkpoint this :class:`DataFrame` immediately
 
-        .. note:: Experimental
+        Notes
+        -----
+        This API is experimental.
         """
         jdf = self._jdf.checkpoint(eager)
         return DataFrame(jdf, self.sql_ctx)
 
-    @since(2.3)
     def localCheckpoint(self, eager=True):
         """Returns a locally checkpointed version of this Dataset. Checkpointing can be used to
         truncate the logical plan of this :class:`DataFrame`, which is especially useful in
         iterative algorithms where the plan may grow exponentially. Local checkpoints are
         stored in the executors using the caching subsystem and therefore they are not reliable.
 
-        :param eager: Whether to checkpoint this :class:`DataFrame` immediately
+        .. versionadded:: 2.3.0
 
-        .. note:: Experimental
+        Parameters
+        ----------
+        eager : bool, optional
+            Whether to checkpoint this :class:`DataFrame` immediately
+
+        Notes
+        -----
+        This API is experimental.
         """
         jdf = self._jdf.localCheckpoint(eager)
         return DataFrame(jdf, self.sql_ctx)
 
-    @since(2.1)
     def withWatermark(self, eventTime, delayThreshold):
         """Defines an event time watermark for this :class:`DataFrame`. A watermark tracks a point
         in time before which we assume no more late data is going to arrive.
@@ -514,14 +584,25 @@ def withWatermark(self, eventTime, delayThreshold):
         to be at least `delayThreshold` behind the actual event time.  In some cases we may still
         process records that arrive more than `delayThreshold` late.
 
-        :param eventTime: the name of the column that contains the event time of the row.
-        :param delayThreshold: the minimum delay to wait to data to arrive late, relative to the
+        .. versionadded:: 2.1.0
+
+        Parameters
+        ----------
+        eventTime : str
+            the name of the column that contains the event time of the row.
+        delayThreshold : str
+            the minimum delay to wait to data to arrive late, relative to the
             latest record that has been processed in the form of an interval
             (e.g. "1 minute" or "5 hours").
 
-        .. note:: Evolving
+        Notes
+        -----
+        This API is evolving.
 
-        >>> sdf.select('name', sdf.time.cast('timestamp')).withWatermark('time', '10 minutes')
+        >>> from pyspark.sql.functions import timestamp_seconds
+        >>> sdf.select(
+        ...    'name',
+        ...    timestamp_seconds(sdf.time).alias('time')).withWatermark('time', '10 minutes')
         DataFrame[name: string, time: timestamp]
         """
         if not eventTime or type(eventTime) is not str:
@@ -531,14 +612,24 @@ def withWatermark(self, eventTime, delayThreshold):
         jdf = self._jdf.withWatermark(eventTime, delayThreshold)
         return DataFrame(jdf, self.sql_ctx)
 
-    @since(2.2)
     def hint(self, name, *parameters):
         """Specifies some hint on the current :class:`DataFrame`.
 
-        :param name: A name of the hint.
-        :param parameters: Optional parameters.
-        :return: :class:`DataFrame`
+        .. versionadded:: 2.2.0
+
+        Parameters
+        ----------
+        name : str
+            A name of the hint.
+        parameters : str, list, float or int
+            Optional parameters.
 
+        Returns
+        -------
+        :class:`DataFrame`
+
+        Examples
+        --------
         >>> df.join(df2.hint("broadcast"), "name").show()
         +----+---+------+
         |name|age|height|
@@ -552,7 +643,7 @@ def hint(self, name, *parameters):
         if not isinstance(name, str):
             raise TypeError("name should be provided as str, got {0}".format(type(name)))
 
-        allowed_types = (basestring, list, float, int)
+        allowed_types = (str, list, float, int)
         for p in parameters:
             if not isinstance(p, allowed_types):
                 raise TypeError(
@@ -562,29 +653,32 @@ def hint(self, name, *parameters):
         jdf = self._jdf.hint(name, self._jseq(parameters))
         return DataFrame(jdf, self.sql_ctx)
 
-    @since(1.3)
     def count(self):
         """Returns the number of rows in this :class:`DataFrame`.
 
+        .. versionadded:: 1.3.0
+
+        Examples
+        --------
         >>> df.count()
         2
         """
         return int(self._jdf.count())
 
-    @ignore_unicode_prefix
-    @since(1.3)
     def collect(self):
         """Returns all the records as a list of :class:`Row`.
 
+        .. versionadded:: 1.3.0
+
+        Examples
+        --------
         >>> df.collect()
-        [Row(age=2, name=u'Alice'), Row(age=5, name=u'Bob')]
+        [Row(age=2, name='Alice'), Row(age=5, name='Bob')]
         """
         with SCCallSiteSync(self._sc) as css:
             sock_info = self._jdf.collectToPython()
         return list(_load_from_socket(sock_info, BatchedSerializer(PickleSerializer())))
 
-    @ignore_unicode_prefix
-    @since(2.0)
     def toLocalIterator(self, prefetchPartitions=False):
         """
         Returns an iterator that contains all of the rows in this :class:`DataFrame`.
@@ -592,41 +686,49 @@ def toLocalIterator(self, prefetchPartitions=False):
         :class:`DataFrame`. With prefetch it may consume up to the memory of the 2 largest
         partitions.
 
-        :param prefetchPartitions: If Spark should pre-fetch the next partition
-                                   before it is needed.
+        .. versionadded:: 2.0.0
 
+        Parameters
+        ----------
+        prefetchPartitions : bool, optional
+            If Spark should pre-fetch the next partition  before it is needed.
+
+        Examples
+        --------
         >>> list(df.toLocalIterator())
-        [Row(age=2, name=u'Alice'), Row(age=5, name=u'Bob')]
+        [Row(age=2, name='Alice'), Row(age=5, name='Bob')]
         """
         with SCCallSiteSync(self._sc) as css:
             sock_info = self._jdf.toPythonIterator(prefetchPartitions)
         return _local_iterator_from_socket(sock_info, BatchedSerializer(PickleSerializer()))
 
-    @ignore_unicode_prefix
-    @since(1.3)
     def limit(self, num):
         """Limits the result count to the number specified.
 
+        .. versionadded:: 1.3.0
+
+        Examples
+        --------
         >>> df.limit(1).collect()
-        [Row(age=2, name=u'Alice')]
+        [Row(age=2, name='Alice')]
         >>> df.limit(0).collect()
         []
         """
         jdf = self._jdf.limit(num)
         return DataFrame(jdf, self.sql_ctx)
 
-    @ignore_unicode_prefix
-    @since(1.3)
     def take(self, num):
         """Returns the first ``num`` rows as a :class:`list` of :class:`Row`.
 
+        .. versionadded:: 1.3.0
+
+        Examples
+        --------
         >>> df.take(2)
-        [Row(age=2, name=u'Alice'), Row(age=5, name=u'Bob')]
+        [Row(age=2, name='Alice'), Row(age=5, name='Bob')]
         """
         return self.limit(num).collect()
 
-    @ignore_unicode_prefix
-    @since(3.0)
     def tail(self, num):
         """
         Returns the last ``num`` rows as a :class:`list` of :class:`Row`.
@@ -634,31 +736,41 @@ def tail(self, num):
         Running tail requires moving data into the application's driver process, and doing so with
         a very large ``num`` can crash the driver process with OutOfMemoryError.
 
+        .. versionadded:: 3.0.0
+
+        Examples
+        --------
         >>> df.tail(1)
-        [Row(age=5, name=u'Bob')]
+        [Row(age=5, name='Bob')]
         """
         with SCCallSiteSync(self._sc):
             sock_info = self._jdf.tailToPython(num)
         return list(_load_from_socket(sock_info, BatchedSerializer(PickleSerializer())))
 
-    @since(1.3)
     def foreach(self, f):
         """Applies the ``f`` function to all :class:`Row` of this :class:`DataFrame`.
 
         This is a shorthand for ``df.rdd.foreach()``.
 
+        .. versionadded:: 1.3.0
+
+        Examples
+        --------
         >>> def f(person):
         ...     print(person.name)
         >>> df.foreach(f)
         """
         self.rdd.foreach(f)
 
-    @since(1.3)
     def foreachPartition(self, f):
         """Applies the ``f`` function to each partition of this :class:`DataFrame`.
 
         This a shorthand for ``df.rdd.foreachPartition()``.
 
+        .. versionadded:: 1.3.0
+
+        Examples
+        --------
         >>> def f(people):
         ...     for person in people:
         ...         print(person.name)
@@ -666,24 +778,30 @@ def foreachPartition(self, f):
         """
         self.rdd.foreachPartition(f)
 
-    @since(1.3)
     def cache(self):
         """Persists the :class:`DataFrame` with the default storage level (`MEMORY_AND_DISK`).
 
-        .. note:: The default storage level has changed to `MEMORY_AND_DISK` to match Scala in 2.0.
+        .. versionadded:: 1.3.0
+
+        Notes
+        -----
+        The default storage level has changed to `MEMORY_AND_DISK` to match Scala in 2.0.
         """
         self.is_cached = True
         self._jdf.cache()
         return self
 
-    @since(1.3)
-    def persist(self, storageLevel=StorageLevel.MEMORY_AND_DISK):
+    def persist(self, storageLevel=StorageLevel.MEMORY_AND_DISK_DESER):
         """Sets the storage level to persist the contents of the :class:`DataFrame` across
         operations after the first time it is computed. This can only be used to assign
         a new storage level if the :class:`DataFrame` does not have a storage level set yet.
-        If no storage level is specified defaults to (`MEMORY_AND_DISK`).
+        If no storage level is specified defaults to (`MEMORY_AND_DISK_DESER`)
+
+        .. versionadded:: 1.3.0
 
-        .. note:: The default storage level has changed to `MEMORY_AND_DISK` to match Scala in 2.0.
+        Notes
+        -----
+        The default storage level has changed to `MEMORY_AND_DISK_DESER` to match Scala in 3.0.
         """
         self.is_cached = True
         javaStorageLevel = self._sc._getJavaStorageLevel(storageLevel)
@@ -691,10 +809,13 @@ def persist(self, storageLevel=StorageLevel.MEMORY_AND_DISK):
         return self
 
     @property
-    @since(2.1)
     def storageLevel(self):
         """Get the :class:`DataFrame`'s current storage level.
 
+        .. versionadded:: 2.1.0
+
+        Examples
+        --------
         >>> df.storageLevel
         StorageLevel(False, False, False, False, 1)
         >>> df.cache().storageLevel
@@ -710,24 +831,24 @@ def storageLevel(self):
                                      java_storage_level.replication())
         return storage_level
 
-    @since(1.3)
     def unpersist(self, blocking=False):
         """Marks the :class:`DataFrame` as non-persistent, and remove all blocks for it from
         memory and disk.
 
-        .. note:: `blocking` default has changed to ``False`` to match Scala in 2.0.
+        .. versionadded:: 1.3.0
+
+        Notes
+        -----
+        `blocking` default has changed to ``False`` to match Scala in 2.0.
         """
         self.is_cached = False
         self._jdf.unpersist(blocking)
         return self
 
-    @since(1.4)
     def coalesce(self, numPartitions):
         """
         Returns a new :class:`DataFrame` that has exactly `numPartitions` partitions.
 
-        :param numPartitions: int, to specify the target number of partitions
-
         Similar to coalesce defined on an :class:`RDD`, this operation results in a
         narrow dependency, e.g. if you go from 1000 partitions to 100 partitions,
         there will not be a shuffle, instead each of the 100 new partitions will
@@ -741,26 +862,42 @@ def coalesce(self, numPartitions):
         current upstream partitions will be executed in parallel (per whatever
         the current partitioning is).
 
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        numPartitions : int
+            specify the target number of partitions
+
+        Examples
+        --------
         >>> df.coalesce(1).rdd.getNumPartitions()
         1
         """
         return DataFrame(self._jdf.coalesce(numPartitions), self.sql_ctx)
 
-    @since(1.3)
     def repartition(self, numPartitions, *cols):
         """
         Returns a new :class:`DataFrame` partitioned by the given partitioning expressions. The
         resulting :class:`DataFrame` is hash partitioned.
 
-        :param numPartitions:
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        numPartitions : int
             can be an int to specify the target number of partitions or a Column.
             If it is a Column, it will be used as the first partitioning column. If not specified,
             the default number of partitions is used.
+        cols : str or :class:`Column`
+            partitioning columns.
 
-        .. versionchanged:: 1.6
-           Added optional arguments to specify the partitioning columns. Also made numPartitions
-           optional if partitioning columns are specified.
+            .. versionchanged:: 1.6
+               Added optional arguments to specify the partitioning columns. Also made numPartitions
+               optional if partitioning columns are specified.
 
+        Examples
+        --------
         >>> df.repartition(10).rdd.getNumPartitions()
         10
         >>> data = df.union(df).repartition("age")
@@ -802,31 +939,40 @@ def repartition(self, numPartitions, *cols):
             else:
                 return DataFrame(
                     self._jdf.repartition(numPartitions, self._jcols(*cols)), self.sql_ctx)
-        elif isinstance(numPartitions, (basestring, Column)):
+        elif isinstance(numPartitions, (str, Column)):
             cols = (numPartitions, ) + cols
             return DataFrame(self._jdf.repartition(self._jcols(*cols)), self.sql_ctx)
         else:
             raise TypeError("numPartitions should be an int or Column")
 
-    @since("2.4.0")
     def repartitionByRange(self, numPartitions, *cols):
         """
         Returns a new :class:`DataFrame` partitioned by the given partitioning expressions. The
         resulting :class:`DataFrame` is range partitioned.
 
-        :param numPartitions:
+        At least one partition-by expression must be specified.
+        When no explicit sort order is specified, "ascending nulls first" is assumed.
+
+        .. versionadded:: 2.4.0
+
+        Parameters
+        ----------
+        numPartitions : int
             can be an int to specify the target number of partitions or a Column.
             If it is a Column, it will be used as the first partitioning column. If not specified,
             the default number of partitions is used.
+        cols : str or :class:`Column`
+            partitioning columns.
 
-        At least one partition-by expression must be specified.
-        When no explicit sort order is specified, "ascending nulls first" is assumed.
-
-        Note that due to performance reasons this method uses sampling to estimate the ranges.
+        Notes
+        -----
+        Due to performance reasons this method uses sampling to estimate the ranges.
         Hence, the output may not be consistent, since sampling can return different values.
         The sample size can be controlled by the config
         `spark.sql.execution.rangeExchange.sampleSizePerPartition`.
 
+        Examples
+        --------
         >>> df.repartitionByRange(2, "age").rdd.getNumPartitions()
         2
         >>> df.show()
@@ -853,34 +999,47 @@ def repartitionByRange(self, numPartitions, *cols):
             else:
                 return DataFrame(
                     self._jdf.repartitionByRange(numPartitions, self._jcols(*cols)), self.sql_ctx)
-        elif isinstance(numPartitions, (basestring, Column)):
+        elif isinstance(numPartitions, (str, Column)):
             cols = (numPartitions,) + cols
             return DataFrame(self._jdf.repartitionByRange(self._jcols(*cols)), self.sql_ctx)
         else:
             raise TypeError("numPartitions should be an int, string or Column")
 
-    @since(1.3)
     def distinct(self):
         """Returns a new :class:`DataFrame` containing the distinct rows in this :class:`DataFrame`.
 
+        .. versionadded:: 1.3.0
+
+        Examples
+        --------
         >>> df.distinct().count()
         2
         """
         return DataFrame(self._jdf.distinct(), self.sql_ctx)
 
-    @since(1.3)
     def sample(self, withReplacement=None, fraction=None, seed=None):
         """Returns a sampled subset of this :class:`DataFrame`.
 
-        :param withReplacement: Sample with replacement or not (default ``False``).
-        :param fraction: Fraction of rows to generate, range [0.0, 1.0].
-        :param seed: Seed for sampling (default a random seed).
+        .. versionadded:: 1.3.0
 
-        .. note:: This is not guaranteed to provide exactly the fraction specified of the total
-            count of the given :class:`DataFrame`.
+        Parameters
+        ----------
+        withReplacement : bool, optional
+            Sample with replacement or not (default ``False``).
+        fraction : float, optional
+            Fraction of rows to generate, range [0.0, 1.0].
+        seed : int, optional
+            Seed for sampling (default a random seed).
 
-        .. note:: `fraction` is required and, `withReplacement` and `seed` are optional.
+        Notes
+        -----
+        This is not guaranteed to provide exactly the fraction specified of the total
+        count of the given :class:`DataFrame`.
 
+        `fraction` is required and, `withReplacement` and `seed` are optional.
+
+        Examples
+        --------
         >>> df = spark.range(10)
         >>> df.sample(0.5, 3).count()
         7
@@ -928,24 +1087,37 @@ def sample(self, withReplacement=None, fraction=None, seed=None):
             fraction = withReplacement
             withReplacement = None
 
-        seed = long(seed) if seed is not None else None
+        seed = int(seed) if seed is not None else None
         args = [arg for arg in [withReplacement, fraction, seed] if arg is not None]
         jdf = self._jdf.sample(*args)
         return DataFrame(jdf, self.sql_ctx)
 
-    @since(1.5)
     def sampleBy(self, col, fractions, seed=None):
         """
         Returns a stratified sample without replacement based on the
         fraction given on each stratum.
 
-        :param col: column that defines strata
-        :param fractions:
+        .. versionadded:: 1.5.0
+
+        Parameters
+        ----------
+        col : :class:`Column` or str
+            column that defines strata
+
+            .. versionchanged:: 3.0
+               Added sampling by a column of :class:`Column`
+        fractions : dict
             sampling fraction for each stratum. If a stratum is not
             specified, we treat its fraction as zero.
-        :param seed: random seed
-        :return: a new :class:`DataFrame` that represents the stratified sample
+        seed : int, optional
+            random seed
 
+        Returns
+        -------
+        a new :class:`DataFrame` that represents the stratified sample
+
+        Examples
+        --------
         >>> from pyspark.sql.functions import col
         >>> dataset = sqlContext.range(0, 100).select((col("id") % 3).alias("key"))
         >>> sampled = dataset.sampleBy("key", fractions={0: 0.1, 1: 0.2}, seed=0)
@@ -958,32 +1130,36 @@ def sampleBy(self, col, fractions, seed=None):
         +---+-----+
         >>> dataset.sampleBy(col("key"), fractions={2: 1.0}, seed=0).count()
         33
-
-        .. versionchanged:: 3.0
-           Added sampling by a column of :class:`Column`
         """
-        if isinstance(col, basestring):
+        if isinstance(col, str):
             col = Column(col)
         elif not isinstance(col, Column):
-            raise ValueError("col must be a string or a column, but got %r" % type(col))
+            raise TypeError("col must be a string or a column, but got %r" % type(col))
         if not isinstance(fractions, dict):
-            raise ValueError("fractions must be a dict but got %r" % type(fractions))
+            raise TypeError("fractions must be a dict but got %r" % type(fractions))
         for k, v in fractions.items():
-            if not isinstance(k, (float, int, long, basestring)):
-                raise ValueError("key must be float, int, long, or string, but got %r" % type(k))
+            if not isinstance(k, (float, int, str)):
+                raise TypeError("key must be float, int, or string, but got %r" % type(k))
             fractions[k] = float(v)
         col = col._jc
         seed = seed if seed is not None else random.randint(0, sys.maxsize)
         return DataFrame(self._jdf.stat().sampleBy(col, self._jmap(fractions), seed), self.sql_ctx)
 
-    @since(1.4)
     def randomSplit(self, weights, seed=None):
         """Randomly splits this :class:`DataFrame` with the provided weights.
 
-        :param weights: list of doubles as weights with which to split the :class:`DataFrame`.
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        weights : list
+            list of doubles as weights with which to split the :class:`DataFrame`.
             Weights will be normalized if they don't sum up to 1.0.
-        :param seed: The seed for sampling.
+        seed : int, optional
+            The seed for sampling.
 
+        Examples
+        --------
         >>> splits = df4.randomSplit([1.0, 2.0], 24)
         >>> splits[0].count()
         2
@@ -995,37 +1171,49 @@ def randomSplit(self, weights, seed=None):
             if w < 0.0:
                 raise ValueError("Weights must be positive. Found weight value: %s" % w)
         seed = seed if seed is not None else random.randint(0, sys.maxsize)
-        rdd_array = self._jdf.randomSplit(_to_list(self.sql_ctx._sc, weights), long(seed))
+        rdd_array = self._jdf.randomSplit(_to_list(self.sql_ctx._sc, weights), int(seed))
         return [DataFrame(rdd, self.sql_ctx) for rdd in rdd_array]
 
     @property
-    @since(1.3)
     def dtypes(self):
         """Returns all column names and their data types as a list.
 
+        .. versionadded:: 1.3.0
+
+        Examples
+        --------
         >>> df.dtypes
         [('age', 'int'), ('name', 'string')]
         """
         return [(str(f.name), f.dataType.simpleString()) for f in self.schema.fields]
 
     @property
-    @since(1.3)
     def columns(self):
         """Returns all column names as a list.
 
+        .. versionadded:: 1.3.0
+
+        Examples
+        --------
         >>> df.columns
         ['age', 'name']
         """
         return [f.name for f in self.schema.fields]
 
-    @since(2.3)
     def colRegex(self, colName):
         """
         Selects column based on the column name specified as a regex and returns it
         as :class:`Column`.
 
-        :param colName: string, column name specified as a regex.
+        .. versionadded:: 2.3.0
+
+        Parameters
+        ----------
+        colName : str
+            string, column name specified as a regex.
 
+        Examples
+        --------
         >>> df = spark.createDataFrame([("a", 1), ("b", 2), ("c",  3)], ["Col1", "Col2"])
         >>> df.select(df.colRegex("`(Col1)?+.+`")).show()
         +----+
@@ -1036,88 +1224,106 @@ def colRegex(self, colName):
         |   3|
         +----+
         """
-        if not isinstance(colName, basestring):
-            raise ValueError("colName should be provided as string")
+        if not isinstance(colName, str):
+            raise TypeError("colName should be provided as string")
         jc = self._jdf.colRegex(colName)
         return Column(jc)
 
-    @ignore_unicode_prefix
-    @since(1.3)
     def alias(self, alias):
         """Returns a new :class:`DataFrame` with an alias set.
 
-        :param alias: string, an alias name to be set for the :class:`DataFrame`.
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        alias : str
+            an alias name to be set for the :class:`DataFrame`.
 
+        Examples
+        --------
         >>> from pyspark.sql.functions import *
         >>> df_as1 = df.alias("df_as1")
         >>> df_as2 = df.alias("df_as2")
         >>> joined_df = df_as1.join(df_as2, col("df_as1.name") == col("df_as2.name"), 'inner')
         >>> joined_df.select("df_as1.name", "df_as2.name", "df_as2.age") \
                 .sort(desc("df_as1.name")).collect()
-        [Row(name=u'Bob', name=u'Bob', age=5), Row(name=u'Alice', name=u'Alice', age=2)]
+        [Row(name='Bob', name='Bob', age=5), Row(name='Alice', name='Alice', age=2)]
         """
-        assert isinstance(alias, basestring), "alias should be a string"
+        assert isinstance(alias, str), "alias should be a string"
         return DataFrame(getattr(self._jdf, "as")(alias), self.sql_ctx)
 
-    @ignore_unicode_prefix
-    @since(2.1)
     def crossJoin(self, other):
         """Returns the cartesian product with another :class:`DataFrame`.
 
-        :param other: Right side of the cartesian product.
+        .. versionadded:: 2.1.0
+
+        Parameters
+        ----------
+        other : :class:`DataFrame`
+            Right side of the cartesian product.
 
+        Examples
+        --------
         >>> df.select("age", "name").collect()
-        [Row(age=2, name=u'Alice'), Row(age=5, name=u'Bob')]
+        [Row(age=2, name='Alice'), Row(age=5, name='Bob')]
         >>> df2.select("name", "height").collect()
-        [Row(name=u'Tom', height=80), Row(name=u'Bob', height=85)]
+        [Row(name='Tom', height=80), Row(name='Bob', height=85)]
         >>> df.crossJoin(df2.select("height")).select("age", "name", "height").collect()
-        [Row(age=2, name=u'Alice', height=80), Row(age=2, name=u'Alice', height=85),
-         Row(age=5, name=u'Bob', height=80), Row(age=5, name=u'Bob', height=85)]
+        [Row(age=2, name='Alice', height=80), Row(age=2, name='Alice', height=85),
+         Row(age=5, name='Bob', height=80), Row(age=5, name='Bob', height=85)]
         """
 
         jdf = self._jdf.crossJoin(other._jdf)
         return DataFrame(jdf, self.sql_ctx)
 
-    @ignore_unicode_prefix
-    @since(1.3)
     def join(self, other, on=None, how=None):
         """Joins with another :class:`DataFrame`, using the given join expression.
 
-        :param other: Right side of the join
-        :param on: a string for the join column name, a list of column names,
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        other : :class:`DataFrame`
+            Right side of the join
+        on : str, list or :class:`Column`, optional
+            a string for the join column name, a list of column names,
             a join expression (Column), or a list of Columns.
             If `on` is a string or a list of strings indicating the name of the join column(s),
             the column(s) must exist on both sides, and this performs an equi-join.
-        :param how: str, default ``inner``. Must be one of: ``inner``, ``cross``, ``outer``,
+        how : str, optional
+            default ``inner``. Must be one of: ``inner``, ``cross``, ``outer``,
             ``full``, ``fullouter``, ``full_outer``, ``left``, ``leftouter``, ``left_outer``,
             ``right``, ``rightouter``, ``right_outer``, ``semi``, ``leftsemi``, ``left_semi``,
             ``anti``, ``leftanti`` and ``left_anti``.
 
+        Examples
+        --------
         The following performs a full outer join between ``df1`` and ``df2``.
+
         >>> from pyspark.sql.functions import desc
         >>> df.join(df2, df.name == df2.name, 'outer').select(df.name, df2.height) \
                 .sort(desc("name")).collect()
-        [Row(name=u'Bob', height=85), Row(name=u'Alice', height=None), Row(name=None, height=80)]
+        [Row(name='Bob', height=85), Row(name='Alice', height=None), Row(name=None, height=80)]
 
         >>> df.join(df2, 'name', 'outer').select('name', 'height').sort(desc("name")).collect()
-        [Row(name=u'Tom', height=80), Row(name=u'Bob', height=85), Row(name=u'Alice', height=None)]
+        [Row(name='Tom', height=80), Row(name='Bob', height=85), Row(name='Alice', height=None)]
 
         >>> cond = [df.name == df3.name, df.age == df3.age]
         >>> df.join(df3, cond, 'outer').select(df.name, df3.age).collect()
-        [Row(name=u'Alice', age=2), Row(name=u'Bob', age=5)]
+        [Row(name='Alice', age=2), Row(name='Bob', age=5)]
 
         >>> df.join(df2, 'name').select(df.name, df2.height).collect()
-        [Row(name=u'Bob', height=85)]
+        [Row(name='Bob', height=85)]
 
         >>> df.join(df4, ['name', 'age']).select(df.name, df.age).collect()
-        [Row(name=u'Bob', age=5)]
+        [Row(name='Bob', age=5)]
         """
 
         if on is not None and not isinstance(on, list):
             on = [on]
 
         if on is not None:
-            if isinstance(on[0], basestring):
+            if isinstance(on[0], str):
                 on = self._jseq(on)
             else:
                 assert isinstance(on[0], Column), "on should be Column or list of Column"
@@ -1131,19 +1337,29 @@ def join(self, other, on=None, how=None):
                 how = "inner"
             if on is None:
                 on = self._jseq([])
-            assert isinstance(how, basestring), "how should be basestring"
+            assert isinstance(how, str), "how should be a string"
             jdf = self._jdf.join(other._jdf, on, how)
         return DataFrame(jdf, self.sql_ctx)
 
-    @since(1.6)
     def sortWithinPartitions(self, *cols, **kwargs):
         """Returns a new :class:`DataFrame` with each partition sorted by the specified column(s).
 
-        :param cols: list of :class:`Column` or column names to sort by.
-        :param ascending: boolean or list of boolean (default ``True``).
+        .. versionadded:: 1.6.0
+
+        Parameters
+        ----------
+        cols : str, list or :class:`Column`, optional
+            list of :class:`Column` or column names to sort by.
+
+        Other Parameters
+        ----------------
+        ascending : bool or list, optional
+            boolean or list of boolean (default ``True``).
             Sort ascending vs. descending. Specify list for multiple sort orders.
             If a list is specified, length of the list must equal length of the `cols`.
 
+        Examples
+        --------
         >>> df.sortWithinPartitions("age", ascending=False).show()
         +---+-----+
         |age| name|
@@ -1155,29 +1371,38 @@ def sortWithinPartitions(self, *cols, **kwargs):
         jdf = self._jdf.sortWithinPartitions(self._sort_cols(cols, kwargs))
         return DataFrame(jdf, self.sql_ctx)
 
-    @ignore_unicode_prefix
-    @since(1.3)
     def sort(self, *cols, **kwargs):
         """Returns a new :class:`DataFrame` sorted by the specified column(s).
 
-        :param cols: list of :class:`Column` or column names to sort by.
-        :param ascending: boolean or list of boolean (default ``True``).
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        cols : str, list, or :class:`Column`, optional
+             list of :class:`Column` or column names to sort by.
+
+        Other Parameters
+        ----------------
+        ascending : bool or list, optional
+            boolean or list of boolean (default ``True``).
             Sort ascending vs. descending. Specify list for multiple sort orders.
             If a list is specified, length of the list must equal length of the `cols`.
 
+        Examples
+        --------
         >>> df.sort(df.age.desc()).collect()
-        [Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]
+        [Row(age=5, name='Bob'), Row(age=2, name='Alice')]
         >>> df.sort("age", ascending=False).collect()
-        [Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]
+        [Row(age=5, name='Bob'), Row(age=2, name='Alice')]
         >>> df.orderBy(df.age.desc()).collect()
-        [Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]
+        [Row(age=5, name='Bob'), Row(age=2, name='Alice')]
         >>> from pyspark.sql.functions import *
         >>> df.sort(asc("age")).collect()
-        [Row(age=2, name=u'Alice'), Row(age=5, name=u'Bob')]
+        [Row(age=2, name='Alice'), Row(age=5, name='Bob')]
         >>> df.orderBy(desc("age"), "name").collect()
-        [Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]
+        [Row(age=5, name='Bob'), Row(age=2, name='Alice')]
         >>> df.orderBy(["age", "name"], ascending=[0, 1]).collect()
-        [Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]
+        [Row(age=5, name='Bob'), Row(age=2, name='Alice')]
         """
         jdf = self._jdf.sort(self._sort_cols(cols, kwargs))
         return DataFrame(jdf, self.sql_ctx)
@@ -1220,17 +1445,24 @@ def _sort_cols(self, cols, kwargs):
             raise TypeError("ascending can only be boolean or list, but got %s" % type(ascending))
         return self._jseq(jcols)
 
-    @since("1.3.1")
     def describe(self, *cols):
         """Computes basic statistics for numeric and string columns.
 
+        .. versionadded:: 1.3.1
+
         This include count, mean, stddev, min, and max. If no columns are
         given, this function computes statistics for all numerical or string columns.
 
-        .. note:: This function is meant for exploratory data analysis, as we make no
-            guarantee about the backward compatibility of the schema of the resulting
-            :class:`DataFrame`.
+        Notes
+        -----
+        This function is meant for exploratory data analysis, as we make no
+        guarantee about the backward compatibility of the schema of the resulting
+        :class:`DataFrame`.
+
+        Use summary for expanded statistics and control over which statistics to compute.
 
+        Examples
+        --------
         >>> df.describe(['age']).show()
         +-------+------------------+
         |summary|               age|
@@ -1252,14 +1484,15 @@ def describe(self, *cols):
         |    max|                 5|  Bob|
         +-------+------------------+-----+
 
-        Use summary for expanded statistics and control over which statistics to compute.
+        See Also
+        --------
+        DataFrame.summary
         """
         if len(cols) == 1 and isinstance(cols[0], list):
             cols = cols[0]
         jdf = self._jdf.describe(self._jseq(cols))
         return DataFrame(jdf, self.sql_ctx)
 
-    @since("2.3.0")
     def summary(self, *statistics):
         """Computes specified statistics for numeric and string columns. Available statistics are:
         - count
@@ -1267,15 +1500,21 @@ def summary(self, *statistics):
         - stddev
         - min
         - max
-        - arbitrary approximate percentiles specified as a percentage (eg, 75%)
+        - arbitrary approximate percentiles specified as a percentage (e.g., 75%)
 
         If no statistics are given, this function computes count, mean, stddev, min,
         approximate quartiles (percentiles at 25%, 50%, and 75%), and max.
 
-        .. note:: This function is meant for exploratory data analysis, as we make no
-            guarantee about the backward compatibility of the schema of the resulting
-            :class:`DataFrame`.
+        .. versionadded:: 2.3.0
+
+        Notes
+        -----
+        This function is meant for exploratory data analysis, as we make no
+        guarantee about the backward compatibility of the schema of the resulting
+        :class:`DataFrame`.
 
+        Examples
+        --------
         >>> df.summary().show()
         +-------+------------------+-----+
         |summary|               age| name|
@@ -1310,60 +1549,76 @@ def summary(self, *statistics):
         |  count|  2|   2|
         +-------+---+----+
 
-        See also describe for basic statistics.
+        See Also
+        --------
+        DataFrame.display
         """
         if len(statistics) == 1 and isinstance(statistics[0], list):
             statistics = statistics[0]
         jdf = self._jdf.summary(self._jseq(statistics))
         return DataFrame(jdf, self.sql_ctx)
 
-    @ignore_unicode_prefix
-    @since(1.3)
     def head(self, n=None):
         """Returns the first ``n`` rows.
 
-        .. note:: This method should only be used if the resulting array is expected
-            to be small, as all the data is loaded into the driver's memory.
+        .. versionadded:: 1.3.0
+
+        Notes
+        -----
+        This method should only be used if the resulting array is expected
+        to be small, as all the data is loaded into the driver's memory.
 
-        :param n: int, default 1. Number of rows to return.
-        :return: If n is greater than 1, return a list of :class:`Row`.
-            If n is 1, return a single Row.
+        Parameters
+        ----------
+        n : int, optional
+            default 1. Number of rows to return.
 
+        Returns
+        -------
+        If n is greater than 1, return a list of :class:`Row`.
+        If n is 1, return a single Row.
+
+        Examples
+        --------
         >>> df.head()
-        Row(age=2, name=u'Alice')
+        Row(age=2, name='Alice')
         >>> df.head(1)
-        [Row(age=2, name=u'Alice')]
+        [Row(age=2, name='Alice')]
         """
         if n is None:
             rs = self.head(1)
             return rs[0] if rs else None
         return self.take(n)
 
-    @ignore_unicode_prefix
-    @since(1.3)
     def first(self):
         """Returns the first row as a :class:`Row`.
 
+        .. versionadded:: 1.3.0
+
+        Examples
+        --------
         >>> df.first()
-        Row(age=2, name=u'Alice')
+        Row(age=2, name='Alice')
         """
         return self.head()
 
-    @ignore_unicode_prefix
-    @since(1.3)
     def __getitem__(self, item):
         """Returns the column as a :class:`Column`.
 
+        .. versionadded:: 1.3.0
+
+        Examples
+        --------
         >>> df.select(df['age']).collect()
         [Row(age=2), Row(age=5)]
         >>> df[ ["name", "age"]].collect()
-        [Row(name=u'Alice', age=2), Row(name=u'Bob', age=5)]
+        [Row(name='Alice', age=2), Row(name='Bob', age=5)]
         >>> df[ df.age > 3 ].collect()
-        [Row(age=5, name=u'Bob')]
+        [Row(age=5, name='Bob')]
         >>> df[df[0] > 3].collect()
-        [Row(age=5, name=u'Bob')]
+        [Row(age=5, name='Bob')]
         """
-        if isinstance(item, basestring):
+        if isinstance(item, str):
             jc = self._jdf.apply(item)
             return Column(jc)
         elif isinstance(item, Column):
@@ -1376,10 +1631,13 @@ def __getitem__(self, item):
         else:
             raise TypeError("unexpected item type: %s" % type(item))
 
-    @since(1.3)
     def __getattr__(self, name):
         """Returns the :class:`Column` denoted by ``name``.
 
+        .. versionadded:: 1.3.0
+
+        Examples
+        --------
         >>> df.select(df.age).collect()
         [Row(age=2), Row(age=5)]
         """
@@ -1389,31 +1647,39 @@ def __getattr__(self, name):
         jc = self._jdf.apply(name)
         return Column(jc)
 
-    @ignore_unicode_prefix
-    @since(1.3)
     def select(self, *cols):
         """Projects a set of expressions and returns a new :class:`DataFrame`.
 
-        :param cols: list of column names (string) or expressions (:class:`Column`).
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        cols : str, :class:`Column`, or list
+            column names (string) or expressions (:class:`Column`).
             If one of the column names is '*', that column is expanded to include all columns
             in the current :class:`DataFrame`.
 
+        Examples
+        --------
         >>> df.select('*').collect()
-        [Row(age=2, name=u'Alice'), Row(age=5, name=u'Bob')]
+        [Row(age=2, name='Alice'), Row(age=5, name='Bob')]
         >>> df.select('name', 'age').collect()
-        [Row(name=u'Alice', age=2), Row(name=u'Bob', age=5)]
+        [Row(name='Alice', age=2), Row(name='Bob', age=5)]
         >>> df.select(df.name, (df.age + 10).alias('age')).collect()
-        [Row(name=u'Alice', age=12), Row(name=u'Bob', age=15)]
+        [Row(name='Alice', age=12), Row(name='Bob', age=15)]
         """
         jdf = self._jdf.select(self._jcols(*cols))
         return DataFrame(jdf, self.sql_ctx)
 
-    @since(1.3)
     def selectExpr(self, *expr):
         """Projects a set of SQL expressions and returns a new :class:`DataFrame`.
 
         This is a variant of :func:`select` that accepts SQL expressions.
 
+        .. versionadded:: 1.3.0
+
+        Examples
+        --------
         >>> df.selectExpr("age * 2", "abs(age)").collect()
         [Row((age * 2)=4, abs(age)=2), Row((age * 2)=10, abs(age)=5)]
         """
@@ -1422,27 +1688,32 @@ def selectExpr(self, *expr):
         jdf = self._jdf.selectExpr(self._jseq(expr))
         return DataFrame(jdf, self.sql_ctx)
 
-    @ignore_unicode_prefix
-    @since(1.3)
     def filter(self, condition):
         """Filters rows using the given condition.
 
         :func:`where` is an alias for :func:`filter`.
 
-        :param condition: a :class:`Column` of :class:`types.BooleanType`
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        condition : :class:`Column` or str
+            a :class:`Column` of :class:`types.BooleanType`
             or a string of SQL expression.
 
+        Examples
+        --------
         >>> df.filter(df.age > 3).collect()
-        [Row(age=5, name=u'Bob')]
+        [Row(age=5, name='Bob')]
         >>> df.where(df.age == 2).collect()
-        [Row(age=2, name=u'Alice')]
+        [Row(age=2, name='Alice')]
 
         >>> df.filter("age > 3").collect()
-        [Row(age=5, name=u'Bob')]
+        [Row(age=5, name='Bob')]
         >>> df.where("age = 2").collect()
-        [Row(age=2, name=u'Alice')]
+        [Row(age=2, name='Alice')]
         """
-        if isinstance(condition, basestring):
+        if isinstance(condition, str):
             jdf = self._jdf.filter(condition)
         elif isinstance(condition, Column):
             jdf = self._jdf.filter(condition._jc)
@@ -1450,8 +1721,6 @@ def filter(self, condition):
             raise TypeError("condition should be string or Column")
         return DataFrame(jdf, self.sql_ctx)
 
-    @ignore_unicode_prefix
-    @since(1.3)
     def groupBy(self, *cols):
         """Groups the :class:`DataFrame` using the specified columns,
         so we can run aggregation on them. See :class:`GroupedData`
@@ -1459,28 +1728,38 @@ def groupBy(self, *cols):
 
         :func:`groupby` is an alias for :func:`groupBy`.
 
-        :param cols: list of columns to group by.
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        cols : list, str or :class:`Column`
+            columns to group by.
             Each element should be a column name (string) or an expression (:class:`Column`).
 
+        Examples
+        --------
         >>> df.groupBy().avg().collect()
         [Row(avg(age)=3.5)]
         >>> sorted(df.groupBy('name').agg({'age': 'mean'}).collect())
-        [Row(name=u'Alice', avg(age)=2.0), Row(name=u'Bob', avg(age)=5.0)]
+        [Row(name='Alice', avg(age)=2.0), Row(name='Bob', avg(age)=5.0)]
         >>> sorted(df.groupBy(df.name).avg().collect())
-        [Row(name=u'Alice', avg(age)=2.0), Row(name=u'Bob', avg(age)=5.0)]
+        [Row(name='Alice', avg(age)=2.0), Row(name='Bob', avg(age)=5.0)]
         >>> sorted(df.groupBy(['name', df.age]).count().collect())
-        [Row(name=u'Alice', age=2, count=1), Row(name=u'Bob', age=5, count=1)]
+        [Row(name='Alice', age=2, count=1), Row(name='Bob', age=5, count=1)]
         """
         jgd = self._jdf.groupBy(self._jcols(*cols))
         from pyspark.sql.group import GroupedData
         return GroupedData(jgd, self)
 
-    @since(1.4)
     def rollup(self, *cols):
         """
         Create a multi-dimensional rollup for the current :class:`DataFrame` using
         the specified columns, so we can run aggregation on them.
 
+        .. versionadded:: 1.4.0
+
+        Examples
+        --------
         >>> df.rollup("name", df.age).count().orderBy("name", "age").show()
         +-----+----+-----+
         | name| age|count|
@@ -1496,12 +1775,15 @@ def rollup(self, *cols):
         from pyspark.sql.group import GroupedData
         return GroupedData(jgd, self)
 
-    @since(1.4)
     def cube(self, *cols):
         """
         Create a multi-dimensional cube for the current :class:`DataFrame` using
         the specified columns, so we can run aggregations on them.
 
+        .. versionadded:: 1.4.0
+
+        Examples
+        --------
         >>> df.cube("name", df.age).count().orderBy("name", "age").show()
         +-----+----+-----+
         | name| age|count|
@@ -1519,11 +1801,14 @@ def cube(self, *cols):
         from pyspark.sql.group import GroupedData
         return GroupedData(jgd, self)
 
-    @since(1.3)
     def agg(self, *exprs):
         """ Aggregate on the entire :class:`DataFrame` without groups
-        (shorthand for ``df.groupBy.agg()``).
+        (shorthand for ``df.groupBy().agg()``).
+
+        .. versionadded:: 1.3.0
 
+        Examples
+        --------
         >>> df.agg({"age": "max"}).collect()
         [Row(max(age)=5)]
         >>> from pyspark.sql import functions as F
@@ -1556,14 +1841,17 @@ def unionAll(self, other):
         """
         return self.union(other)
 
-    @since(2.3)
-    def unionByName(self, other):
+    def unionByName(self, other, allowMissingColumns=False):
         """ Returns a new :class:`DataFrame` containing union of rows in this and another
         :class:`DataFrame`.
 
         This is different from both `UNION ALL` and `UNION DISTINCT` in SQL. To do a SQL-style set
         union (that does deduplication of elements), use this function followed by :func:`distinct`.
 
+        .. versionadded:: 2.3.0
+
+        Examples
+        --------
         The difference between this function and :func:`union` is that this function
         resolves columns by name (not by position):
 
@@ -1576,8 +1864,27 @@ def unionByName(self, other):
         |   1|   2|   3|
         |   6|   4|   5|
         +----+----+----+
+
+        When the parameter `allowMissingColumns` is ``True``, the set of column names
+        in this and other :class:`DataFrame` can differ; missing columns will be filled with null.
+        Further, the missing columns of this :class:`DataFrame` will be added at the end
+        in the schema of the union result:
+
+        >>> df1 = spark.createDataFrame([[1, 2, 3]], ["col0", "col1", "col2"])
+        >>> df2 = spark.createDataFrame([[4, 5, 6]], ["col1", "col2", "col3"])
+        >>> df1.unionByName(df2, allowMissingColumns=True).show()
+        +----+----+----+----+
+        |col0|col1|col2|col3|
+        +----+----+----+----+
+        |   1|   2|   3|null|
+        |null|   4|   5|   6|
+        +----+----+----+----+
+
+        .. versionchanged:: 3.1.0
+           Added optional argument `allowMissingColumns` to specify whether to allow
+           missing columns.
         """
-        return DataFrame(self._jdf.unionByName(other._jdf), self.sql_ctx)
+        return DataFrame(self._jdf.unionByName(other._jdf, allowMissingColumns), self.sql_ctx)
 
     @since(1.3)
     def intersect(self, other):
@@ -1588,12 +1895,17 @@ def intersect(self, other):
         """
         return DataFrame(self._jdf.intersect(other._jdf), self.sql_ctx)
 
-    @since(2.4)
     def intersectAll(self, other):
         """ Return a new :class:`DataFrame` containing rows in both this :class:`DataFrame`
         and another :class:`DataFrame` while preserving duplicates.
 
-        This is equivalent to `INTERSECT ALL` in SQL.
+        This is equivalent to `INTERSECT ALL` in SQL. As standard in SQL, this function
+        resolves columns by position (not by name).
+
+        .. versionadded:: 2.4.0
+
+        Examples
+        --------
         >>> df1 = spark.createDataFrame([("a", 1), ("a", 1), ("b", 3), ("c", 4)], ["C1", "C2"])
         >>> df2 = spark.createDataFrame([("a", 1), ("a", 1), ("b", 3)], ["C1", "C2"])
 
@@ -1606,7 +1918,6 @@ def intersectAll(self, other):
         |  b|  3|
         +---+---+
 
-        Also as standard in SQL, this function resolves columns by position (not by name).
         """
         return DataFrame(self._jdf.intersectAll(other._jdf), self.sql_ctx)
 
@@ -1620,7 +1931,6 @@ def subtract(self, other):
         """
         return DataFrame(getattr(self._jdf, "except")(other._jdf), self.sql_ctx)
 
-    @since(1.4)
     def dropDuplicates(self, subset=None):
         """Return a new :class:`DataFrame` with duplicate rows removed,
         optionally only considering certain columns.
@@ -1633,25 +1943,29 @@ def dropDuplicates(self, subset=None):
 
         :func:`drop_duplicates` is an alias for :func:`dropDuplicates`.
 
+        .. versionadded:: 1.4.0
+
+        Examples
+        --------
         >>> from pyspark.sql import Row
         >>> df = sc.parallelize([ \\
         ...     Row(name='Alice', age=5, height=80), \\
         ...     Row(name='Alice', age=5, height=80), \\
         ...     Row(name='Alice', age=10, height=80)]).toDF()
         >>> df.dropDuplicates().show()
-        +---+------+-----+
-        |age|height| name|
-        +---+------+-----+
-        |  5|    80|Alice|
-        | 10|    80|Alice|
-        +---+------+-----+
+        +-----+---+------+
+        | name|age|height|
+        +-----+---+------+
+        |Alice|  5|    80|
+        |Alice| 10|    80|
+        +-----+---+------+
 
         >>> df.dropDuplicates(['name', 'height']).show()
-        +---+------+-----+
-        |age|height| name|
-        +---+------+-----+
-        |  5|    80|Alice|
-        +---+------+-----+
+        +-----+---+------+
+        | name|age|height|
+        +-----+---+------+
+        |Alice|  5|    80|
+        +-----+---+------+
         """
         if subset is None:
             jdf = self._jdf.dropDuplicates()
@@ -1659,19 +1973,27 @@ def dropDuplicates(self, subset=None):
             jdf = self._jdf.dropDuplicates(self._jseq(subset))
         return DataFrame(jdf, self.sql_ctx)
 
-    @since("1.3.1")
     def dropna(self, how='any', thresh=None, subset=None):
         """Returns a new :class:`DataFrame` omitting rows with null values.
         :func:`DataFrame.dropna` and :func:`DataFrameNaFunctions.drop` are aliases of each other.
 
-        :param how: 'any' or 'all'.
+        .. versionadded:: 1.3.1
+
+        Parameters
+        ----------
+        how : str, optional
+            'any' or 'all'.
             If 'any', drop a row if it contains any nulls.
             If 'all', drop a row only if all its values are null.
-        :param thresh: int, default None
+        thresh: int, optional
+            default None
             If specified, drop rows that have less than `thresh` non-null values.
             This overwrites the `how` parameter.
-        :param subset: optional list of column names to consider.
+        subset : str, tuple or list, optional
+            optional list of column names to consider.
 
+        Examples
+        --------
         >>> df4.na.drop().show()
         +---+------+-----+
         |age|height| name|
@@ -1684,31 +2006,37 @@ def dropna(self, how='any', thresh=None, subset=None):
 
         if subset is None:
             subset = self.columns
-        elif isinstance(subset, basestring):
+        elif isinstance(subset, str):
             subset = [subset]
         elif not isinstance(subset, (list, tuple)):
-            raise ValueError("subset should be a list or tuple of column names")
+            raise TypeError("subset should be a list or tuple of column names")
 
         if thresh is None:
             thresh = len(subset) if how == 'any' else 1
 
         return DataFrame(self._jdf.na().drop(thresh, self._jseq(subset)), self.sql_ctx)
 
-    @since("1.3.1")
     def fillna(self, value, subset=None):
         """Replace null values, alias for ``na.fill()``.
         :func:`DataFrame.fillna` and :func:`DataFrameNaFunctions.fill` are aliases of each other.
 
-        :param value: int, long, float, string, bool or dict.
+        .. versionadded:: 1.3.1
+
+        Parameters
+        ----------
+        value : int, float, string, bool or dict
             Value to replace null values with.
             If the value is a dict, then `subset` is ignored and `value` must be a mapping
             from column name (string) to replacement value. The replacement value must be
-            an int, long, float, boolean, or string.
-        :param subset: optional list of column names to consider.
+            an int, float, boolean, or string.
+        subset : str, tuple or list, optional
+            optional list of column names to consider.
             Columns specified in subset that do not have matching data type are ignored.
             For example, if `value` is a string, and subset contains a non-string column,
             then the non-string column is simply ignored.
 
+        Examples
+        --------
         >>> df4.na.fill(50).show()
         +---+------+-----+
         |age|height| name|
@@ -1738,13 +2066,13 @@ def fillna(self, value, subset=None):
         | 50|  null|unknown|
         +---+------+-------+
         """
-        if not isinstance(value, (float, int, long, basestring, bool, dict)):
-            raise ValueError("value should be a float, int, long, string, bool or dict")
+        if not isinstance(value, (float, int, str, bool, dict)):
+            raise TypeError("value should be a float, int, string, bool or dict")
 
         # Note that bool validates isinstance(int), but we don't want to
         # convert bools to floats
 
-        if not isinstance(value, bool) and isinstance(value, (int, long)):
+        if not isinstance(value, bool) and isinstance(value, int):
             value = float(value)
 
         if isinstance(value, dict):
@@ -1752,14 +2080,13 @@ def fillna(self, value, subset=None):
         elif subset is None:
             return DataFrame(self._jdf.na().fill(value), self.sql_ctx)
         else:
-            if isinstance(subset, basestring):
+            if isinstance(subset, str):
                 subset = [subset]
             elif not isinstance(subset, (list, tuple)):
-                raise ValueError("subset should be a list or tuple of column names")
+                raise TypeError("subset should be a list or tuple of column names")
 
             return DataFrame(self._jdf.na().fill(value, self._jseq(subset)), self.sql_ctx)
 
-    @since(1.4)
     def replace(self, to_replace, value=_NoValue, subset=None):
         """Returns a new :class:`DataFrame` replacing a value with another value.
         :func:`DataFrame.replace` and :func:`DataFrameNaFunctions.replace` are
@@ -1771,20 +2098,27 @@ def replace(self, to_replace, value=_NoValue, subset=None):
         floating point representation. In case of conflicts (for example with `{42: -1, 42.0: 1}`)
         and arbitrary replacement will be used.
 
-        :param to_replace: bool, int, long, float, string, list or dict.
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        to_replace : bool, int, float, string, list or dict
             Value to be replaced.
             If the value is a dict, then `value` is ignored or can be omitted, and `to_replace`
             must be a mapping between a value and a replacement.
-        :param value: bool, int, long, float, string, list or None.
-            The replacement value must be a bool, int, long, float, string or None. If `value` is a
+        value : bool, int, float, string or None, optional
+            The replacement value must be a bool, int, float, string or None. If `value` is a
             list, `value` should be of the same length and type as `to_replace`.
             If `value` is a scalar and `to_replace` is a sequence, then `value` is
             used as a replacement for each item in `to_replace`.
-        :param subset: optional list of column names to consider.
+        subset : list, optional
+            optional list of column names to consider.
             Columns specified in subset that do not have matching data type are ignored.
             For example, if `value` is a string, and subset contains a non-string column,
             then the non-string column is simply ignored.
 
+        Examples
+        --------
         >>> df4.na.replace(10, 20).show()
         +----+------+-----+
         | age|height| name|
@@ -1838,7 +2172,7 @@ def all_of(types):
 
             >>> all_of(bool)([True, False])
             True
-            >>> all_of(basestring)(["a", 1])
+            >>> all_of(str)(["a", 1])
             False
             """
             def all_of_(xs):
@@ -1846,33 +2180,33 @@ def all_of_(xs):
             return all_of_
 
         all_of_bool = all_of(bool)
-        all_of_str = all_of(basestring)
-        all_of_numeric = all_of((float, int, long))
+        all_of_str = all_of(str)
+        all_of_numeric = all_of((float, int))
 
         # Validate input types
-        valid_types = (bool, float, int, long, basestring, list, tuple)
+        valid_types = (bool, float, int, str, list, tuple)
         if not isinstance(to_replace, valid_types + (dict, )):
-            raise ValueError(
-                "to_replace should be a bool, float, int, long, string, list, tuple, or dict. "
+            raise TypeError(
+                "to_replace should be a bool, float, int, string, list, tuple, or dict. "
                 "Got {0}".format(type(to_replace)))
 
         if not isinstance(value, valid_types) and value is not None \
                 and not isinstance(to_replace, dict):
-            raise ValueError("If to_replace is not a dict, value should be "
-                             "a bool, float, int, long, string, list, tuple or None. "
-                             "Got {0}".format(type(value)))
+            raise TypeError("If to_replace is not a dict, value should be "
+                            "a bool, float, int, string, list, tuple or None. "
+                            "Got {0}".format(type(value)))
 
         if isinstance(to_replace, (list, tuple)) and isinstance(value, (list, tuple)):
             if len(to_replace) != len(value):
                 raise ValueError("to_replace and value lists should be of the same length. "
                                  "Got {0} and {1}".format(len(to_replace), len(value)))
 
-        if not (subset is None or isinstance(subset, (list, tuple, basestring))):
-            raise ValueError("subset should be a list or tuple of column names, "
-                             "column name or None. Got {0}".format(type(subset)))
+        if not (subset is None or isinstance(subset, (list, tuple, str))):
+            raise TypeError("subset should be a list or tuple of column names, "
+                            "column name or None. Got {0}".format(type(subset)))
 
         # Reshape input arguments if necessary
-        if isinstance(to_replace, (float, int, long, basestring)):
+        if isinstance(to_replace, (float, int, str)):
             to_replace = [to_replace]
 
         if isinstance(to_replace, dict):
@@ -1880,11 +2214,11 @@ def all_of_(xs):
             if value is not None:
                 warnings.warn("to_replace is a dict and value is not None. value will be ignored.")
         else:
-            if isinstance(value, (float, int, long, basestring)) or value is None:
+            if isinstance(value, (float, int, str)) or value is None:
                 value = [value for _ in range(len(to_replace))]
             rep_dict = dict(zip(to_replace, value))
 
-        if isinstance(subset, basestring):
+        if isinstance(subset, str):
             subset = [subset]
 
         # Verify we were not passed in mixed type generics.
@@ -1899,7 +2233,6 @@ def all_of_(xs):
             return DataFrame(
                 self._jdf.na().replace(self._jseq(subset), self._jmap(rep_dict)), self.sql_ctx)
 
-    @since(2.0)
     def approxQuantile(self, col, probabilities, relativeError):
         """
         Calculates the approximate quantiles of numerical columns of a
@@ -1922,29 +2255,39 @@ def approxQuantile(self, col, probabilities, relativeError):
         Note that null values will be ignored in numerical columns before calculation.
         For columns only containing null values, an empty list is returned.
 
-        :param col: str, list.
-          Can be a single column name, or a list of names for multiple columns.
-        :param probabilities: a list of quantile probabilities
-          Each number must belong to [0, 1].
-          For example 0 is the minimum, 0.5 is the median, 1 is the maximum.
-        :param relativeError:  The relative target precision to achieve
-          (>= 0). If set to zero, the exact quantiles are computed, which
-          could be very expensive. Note that values greater than 1 are
-          accepted but give the same result as 1.
-        :return:  the approximate quantiles at the given probabilities. If
-          the input `col` is a string, the output is a list of floats. If the
-          input `col` is a list or tuple of strings, the output is also a
-          list, but each element in it is a list of floats, i.e., the output
-          is a list of list of floats.
+        .. versionadded:: 2.0.0
 
-        .. versionchanged:: 2.2
-           Added support for multiple columns.
+        Parameters
+        ----------
+        col: str, tuple or list
+            Can be a single column name, or a list of names for multiple columns.
+
+            .. versionchanged:: 2.2
+               Added support for multiple columns.
+        probabilities : list or tuple
+            a list of quantile probabilities
+            Each number must belong to [0, 1].
+            For example 0 is the minimum, 0.5 is the median, 1 is the maximum.
+        relativeError : float
+            The relative target precision to achieve
+            (>= 0). If set to zero, the exact quantiles are computed, which
+            could be very expensive. Note that values greater than 1 are
+            accepted but give the same result as 1.
+
+        Returns
+        -------
+        list
+            the approximate quantiles at the given probabilities. If
+            the input `col` is a string, the output is a list of floats. If the
+            input `col` is a list or tuple of strings, the output is also a
+            list, but each element in it is a list of floats, i.e., the output
+            is a list of list of floats.
         """
 
-        if not isinstance(col, (basestring, list, tuple)):
-            raise ValueError("col should be a string, list or tuple, but got %r" % type(col))
+        if not isinstance(col, (str, list, tuple)):
+            raise TypeError("col should be a string, list or tuple, but got %r" % type(col))
 
-        isStr = isinstance(col, basestring)
+        isStr = isinstance(col, str)
 
         if isinstance(col, tuple):
             col = list(col)
@@ -1952,42 +2295,50 @@ def approxQuantile(self, col, probabilities, relativeError):
             col = [col]
 
         for c in col:
-            if not isinstance(c, basestring):
-                raise ValueError("columns should be strings, but got %r" % type(c))
+            if not isinstance(c, str):
+                raise TypeError("columns should be strings, but got %r" % type(c))
         col = _to_list(self._sc, col)
 
         if not isinstance(probabilities, (list, tuple)):
-            raise ValueError("probabilities should be a list or tuple")
+            raise TypeError("probabilities should be a list or tuple")
         if isinstance(probabilities, tuple):
             probabilities = list(probabilities)
         for p in probabilities:
-            if not isinstance(p, (float, int, long)) or p < 0 or p > 1:
-                raise ValueError("probabilities should be numerical (float, int, long) in [0,1].")
+            if not isinstance(p, (float, int)) or p < 0 or p > 1:
+                raise ValueError("probabilities should be numerical (float, int) in [0,1].")
         probabilities = _to_list(self._sc, probabilities)
 
-        if not isinstance(relativeError, (float, int, long)) or relativeError < 0:
-            raise ValueError("relativeError should be numerical (float, int, long) >= 0.")
+        if not isinstance(relativeError, (float, int)):
+            raise TypeError("relativeError should be numerical (float, int)")
+        if relativeError < 0:
+            raise ValueError("relativeError should be >= 0.")
         relativeError = float(relativeError)
 
         jaq = self._jdf.stat().approxQuantile(col, probabilities, relativeError)
         jaq_list = [list(j) for j in jaq]
         return jaq_list[0] if isStr else jaq_list
 
-    @since(1.4)
     def corr(self, col1, col2, method=None):
         """
         Calculates the correlation of two columns of a :class:`DataFrame` as a double value.
         Currently only supports the Pearson Correlation Coefficient.
         :func:`DataFrame.corr` and :func:`DataFrameStatFunctions.corr` are aliases of each other.
 
-        :param col1: The name of the first column
-        :param col2: The name of the second column
-        :param method: The correlation method. Currently only supports "pearson"
-        """
-        if not isinstance(col1, basestring):
-            raise ValueError("col1 should be a string.")
-        if not isinstance(col2, basestring):
-            raise ValueError("col2 should be a string.")
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        col1 : str
+            The name of the first column
+        col2 : str
+            The name of the second column
+        method : str, optional
+            The correlation method. Currently only supports "pearson"
+        """
+        if not isinstance(col1, str):
+            raise TypeError("col1 should be a string.")
+        if not isinstance(col2, str):
+            raise TypeError("col2 should be a string.")
         if not method:
             method = "pearson"
         if not method == "pearson":
@@ -1995,22 +2346,26 @@ def corr(self, col1, col2, method=None):
                              "coefficient is supported.")
         return self._jdf.stat().corr(col1, col2, method)
 
-    @since(1.4)
     def cov(self, col1, col2):
         """
         Calculate the sample covariance for the given columns, specified by their names, as a
         double value. :func:`DataFrame.cov` and :func:`DataFrameStatFunctions.cov` are aliases.
 
-        :param col1: The name of the first column
-        :param col2: The name of the second column
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        col1 : str
+            The name of the first column
+        col2 : str
+            The name of the second column
         """
-        if not isinstance(col1, basestring):
-            raise ValueError("col1 should be a string.")
-        if not isinstance(col2, basestring):
-            raise ValueError("col2 should be a string.")
+        if not isinstance(col1, str):
+            raise TypeError("col1 should be a string.")
+        if not isinstance(col2, str):
+            raise TypeError("col2 should be a string.")
         return self._jdf.stat().cov(col1, col2)
 
-    @since(1.4)
     def crosstab(self, col1, col2):
         """
         Computes a pair-wise frequency table of the given columns. Also known as a contingency
@@ -2021,18 +2376,23 @@ def crosstab(self, col1, col2):
         Pairs that have no occurrences will have zero as their counts.
         :func:`DataFrame.crosstab` and :func:`DataFrameStatFunctions.crosstab` are aliases.
 
-        :param col1: The name of the first column. Distinct items will make the first item of
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        col1 : str
+            The name of the first column. Distinct items will make the first item of
             each row.
-        :param col2: The name of the second column. Distinct items will make the column names
+        col2 : str
+            The name of the second column. Distinct items will make the column names
             of the :class:`DataFrame`.
         """
-        if not isinstance(col1, basestring):
-            raise ValueError("col1 should be a string.")
-        if not isinstance(col2, basestring):
-            raise ValueError("col2 should be a string.")
+        if not isinstance(col1, str):
+            raise TypeError("col1 should be a string.")
+        if not isinstance(col2, str):
+            raise TypeError("col2 should be a string.")
         return DataFrame(self._jdf.stat().crosstab(col1, col2), self.sql_ctx)
 
-    @since(1.4)
     def freqItems(self, cols, support=None):
         """
         Finding frequent items for columns, possibly with false positives. Using the
@@ -2040,25 +2400,31 @@ def freqItems(self, cols, support=None):
         "https://doi.org/10.1145/762471.762473, proposed by Karp, Schenker, and Papadimitriou".
         :func:`DataFrame.freqItems` and :func:`DataFrameStatFunctions.freqItems` are aliases.
 
-        .. note:: This function is meant for exploratory data analysis, as we make no
-            guarantee about the backward compatibility of the schema of the resulting
-            :class:`DataFrame`.
+        .. versionadded:: 1.4.0
 
-        :param cols: Names of the columns to calculate frequent items for as a list or tuple of
+        Parameters
+        ----------
+        cols : list or tuple
+            Names of the columns to calculate frequent items for as a list or tuple of
             strings.
-        :param support: The frequency with which to consider an item 'frequent'. Default is 1%.
+        support : float, optional
+            The frequency with which to consider an item 'frequent'. Default is 1%.
             The support must be greater than 1e-4.
+
+        Notes
+        -----
+        This function is meant for exploratory data analysis, as we make no
+        guarantee about the backward compatibility of the schema of the resulting
+        :class:`DataFrame`.
         """
         if isinstance(cols, tuple):
             cols = list(cols)
         if not isinstance(cols, list):
-            raise ValueError("cols must be a list or tuple of column names as strings.")
+            raise TypeError("cols must be a list or tuple of column names as strings.")
         if not support:
             support = 0.01
         return DataFrame(self._jdf.stat().freqItems(_to_seq(self._sc, cols), support), self.sql_ctx)
 
-    @ignore_unicode_prefix
-    @since(1.3)
     def withColumn(self, colName, col):
         """
         Returns a new :class:`DataFrame` by adding a column or replacing the
@@ -2067,62 +2433,83 @@ def withColumn(self, colName, col):
         The column expression must be an expression over this :class:`DataFrame`; attempting to add
         a column from some other :class:`DataFrame` will raise an error.
 
-        :param colName: string, name of the new column.
-        :param col: a :class:`Column` expression for the new column.
+        .. versionadded:: 1.3.0
 
-        .. note:: This method introduces a projection internally. Therefore, calling it multiple
-            times, for instance, via loops in order to add multiple columns can generate big
-            plans which can cause performance issues and even `StackOverflowException`.
-            To avoid this, use :func:`select` with the multiple columns at once.
+        Parameters
+        ----------
+        colName : str
+            string, name of the new column.
+        col : :class:`Column`
+            a :class:`Column` expression for the new column.
 
+        Notes
+        -----
+        This method introduces a projection internally. Therefore, calling it multiple
+        times, for instance, via loops in order to add multiple columns can generate big
+        plans which can cause performance issues and even `StackOverflowException`.
+        To avoid this, use :func:`select` with the multiple columns at once.
+
+        Examples
+        --------
         >>> df.withColumn('age2', df.age + 2).collect()
-        [Row(age=2, name=u'Alice', age2=4), Row(age=5, name=u'Bob', age2=7)]
+        [Row(age=2, name='Alice', age2=4), Row(age=5, name='Bob', age2=7)]
 
         """
-        assert isinstance(col, Column), "col should be Column"
+        if not isinstance(col, Column):
+            raise TypeError("col should be Column")
         return DataFrame(self._jdf.withColumn(colName, col._jc), self.sql_ctx)
 
-    @ignore_unicode_prefix
-    @since(1.3)
     def withColumnRenamed(self, existing, new):
         """Returns a new :class:`DataFrame` by renaming an existing column.
         This is a no-op if schema doesn't contain the given column name.
 
-        :param existing: string, name of the existing column to rename.
-        :param new: string, new name of the column.
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        existing : str
+            string, name of the existing column to rename.
+        new : str
+            string, new name of the column.
 
+        Examples
+        --------
         >>> df.withColumnRenamed('age', 'age2').collect()
-        [Row(age2=2, name=u'Alice'), Row(age2=5, name=u'Bob')]
+        [Row(age2=2, name='Alice'), Row(age2=5, name='Bob')]
         """
         return DataFrame(self._jdf.withColumnRenamed(existing, new), self.sql_ctx)
 
-    @since(1.4)
-    @ignore_unicode_prefix
     def drop(self, *cols):
         """Returns a new :class:`DataFrame` that drops the specified column.
         This is a no-op if schema doesn't contain the given column name(s).
 
-        :param cols: a string name of the column to drop, or a
-            :class:`Column` to drop, or a list of string name of the columns to drop.
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        cols: str or :class:`Column`
+            a name of the column, or the :class:`Column` to drop
 
+        Examples
+        --------
         >>> df.drop('age').collect()
-        [Row(name=u'Alice'), Row(name=u'Bob')]
+        [Row(name='Alice'), Row(name='Bob')]
 
         >>> df.drop(df.age).collect()
-        [Row(name=u'Alice'), Row(name=u'Bob')]
+        [Row(name='Alice'), Row(name='Bob')]
 
         >>> df.join(df2, df.name == df2.name, 'inner').drop(df.name).collect()
-        [Row(age=5, height=85, name=u'Bob')]
+        [Row(age=5, height=85, name='Bob')]
 
         >>> df.join(df2, df.name == df2.name, 'inner').drop(df2.name).collect()
-        [Row(age=5, name=u'Bob', height=85)]
+        [Row(age=5, name='Bob', height=85)]
 
         >>> df.join(df2, 'name', 'inner').drop('age', 'height').collect()
-        [Row(name=u'Bob')]
+        [Row(name='Bob')]
         """
         if len(cols) == 1:
             col = cols[0]
-            if isinstance(col, basestring):
+            if isinstance(col, str):
                 jdf = self._jdf.drop(col)
             elif isinstance(col, Column):
                 jdf = self._jdf.drop(col._jc)
@@ -2130,30 +2517,40 @@ def drop(self, *cols):
                 raise TypeError("col should be a string or a Column")
         else:
             for col in cols:
-                if not isinstance(col, basestring):
+                if not isinstance(col, str):
                     raise TypeError("each col in the param list should be a string")
             jdf = self._jdf.drop(self._jseq(cols))
 
         return DataFrame(jdf, self.sql_ctx)
 
-    @ignore_unicode_prefix
     def toDF(self, *cols):
         """Returns a new :class:`DataFrame` that with new specified column names
 
-        :param cols: list of new column names (string)
+        Parameters
+        ----------
+        cols : str
+            new column names
 
+        Examples
+        --------
         >>> df.toDF('f1', 'f2').collect()
-        [Row(f1=2, f2=u'Alice'), Row(f1=5, f2=u'Bob')]
+        [Row(f1=2, f2='Alice'), Row(f1=5, f2='Bob')]
         """
         jdf = self._jdf.toDF(self._jseq(cols))
         return DataFrame(jdf, self.sql_ctx)
 
-    @since(3.0)
     def transform(self, func):
         """Returns a new :class:`DataFrame`. Concise syntax for chaining custom transformations.
 
-        :param func: a function that takes and returns a :class:`DataFrame`.
+        .. versionadded:: 3.0.0
+
+        Parameters
+        ----------
+        func : function
+            a function that takes and returns a :class:`DataFrame`.
 
+        Examples
+        --------
         >>> from pyspark.sql.functions import col
         >>> df = spark.createDataFrame([(1, 1.0), (2, 2.0)], ["int", "float"])
         >>> def cast_all_to_int(input_df):
@@ -2173,21 +2570,26 @@ def transform(self, func):
                                               "should have been DataFrame." % type(result)
         return result
 
-    @since(3.1)
     def sameSemantics(self, other):
         """
         Returns `True` when the logical query plans inside both :class:`DataFrame`\\s are equal and
         therefore return same results.
 
-        .. note:: The equality comparison here is simplified by tolerating the cosmetic differences
-            such as attribute names.
+        .. versionadded:: 3.1.0
 
-        .. note:: This API can compare both :class:`DataFrame`\\s very fast but can still return
-            `False` on the :class:`DataFrame` that return the same results, for instance, from
-            different plans. Such false negative semantic can be useful when caching as an example.
+        Notes
+        -----
+        The equality comparison here is simplified by tolerating the cosmetic differences
+        such as attribute names.
 
-        .. note:: DeveloperApi
+        This API can compare both :class:`DataFrame`\\s very fast but can still return
+        `False` on the :class:`DataFrame` that return the same results, for instance, from
+        different plans. Such false negative semantic can be useful when caching as an example.
 
+        This API is a developer API.
+
+        Examples
+        --------
         >>> df1 = spark.range(10)
         >>> df2 = spark.range(10)
         >>> df1.withColumn("col1", df1.id * 2).sameSemantics(df2.withColumn("col1", df2.id * 2))
@@ -2198,20 +2600,25 @@ def sameSemantics(self, other):
         True
         """
         if not isinstance(other, DataFrame):
-            raise ValueError("other parameter should be of DataFrame; however, got %s"
-                             % type(other))
+            raise TypeError("other parameter should be of DataFrame; however, got %s"
+                            % type(other))
         return self._jdf.sameSemantics(other._jdf)
 
-    @since(3.1)
     def semanticHash(self):
         """
         Returns a hash code of the logical query plan against this :class:`DataFrame`.
 
-        .. note:: Unlike the standard hash code, the hash is calculated against the query plan
-            simplified by tolerating the cosmetic differences such as attribute names.
+        .. versionadded:: 3.1.0
+
+        Notes
+        -----
+        Unlike the standard hash code, the hash is calculated against the query plan
+        simplified by tolerating the cosmetic differences such as attribute names.
 
-        .. note:: DeveloperApi
+        This API is a developer API.
 
+        Examples
+        --------
         >>> spark.range(10).selectExpr("id as col0").semanticHash()  # doctest: +SKIP
         1855039936
         >>> spark.range(10).selectExpr("id as col1").semanticHash()  # doctest: +SKIP
@@ -2219,6 +2626,23 @@ def semanticHash(self):
         """
         return self._jdf.semanticHash()
 
+    def inputFiles(self):
+        """
+        Returns a best-effort snapshot of the files that compose this :class:`DataFrame`.
+        This method simply asks each constituent BaseRelation for its respective files and
+        takes the union of all results. Depending on the source relations, this may not find
+        all input files. Duplicates are removed.
+
+        .. versionadded:: 3.1.0
+
+        Examples
+        --------
+        >>> df = spark.read.load("examples/src/main/resources/people.json", format="json")
+        >>> len(df.inputFiles())
+        1
+        """
+        return list(self._jdf.inputFiles())
+
     where = copy_func(
         filter,
         sinceversion=1.3,
@@ -2239,6 +2663,25 @@ def semanticHash(self):
         sinceversion=1.4,
         doc=":func:`drop_duplicates` is an alias for :func:`dropDuplicates`.")
 
+    def writeTo(self, table):
+        """
+        Create a write configuration builder for v2 sources.
+
+        This builder is used to configure and execute write operations.
+
+        For example, to append or create or replace existing tables.
+
+        .. versionadded:: 3.1.0
+
+        Examples
+        --------
+        >>> df.writeTo("catalog.db.table").append()  # doctest: +SKIP
+        >>> df.writeTo(                              # doctest: +SKIP
+        ...     "catalog.db.table"
+        ... ).partitionedBy("col").createOrReplace()
+        """
+        return DataFrameWriterV2(self, table)
+
 
 def _to_scala_map(sc, jm):
     """
@@ -2317,7 +2760,6 @@ def _test():
     from pyspark.context import SparkContext
     from pyspark.sql import Row, SQLContext, SparkSession
     import pyspark.sql.dataframe
-    from pyspark.sql.functions import from_unixtime
     globs = pyspark.sql.dataframe.__dict__.copy()
     sc = SparkContext('local[4]', 'PythonTest')
     globs['sc'] = sc
@@ -2326,16 +2768,16 @@ def _test():
     globs['df'] = sc.parallelize([(2, 'Alice'), (5, 'Bob')])\
         .toDF(StructType([StructField('age', IntegerType()),
                           StructField('name', StringType())]))
-    globs['df2'] = sc.parallelize([Row(name='Tom', height=80), Row(name='Bob', height=85)]).toDF()
-    globs['df3'] = sc.parallelize([Row(name='Alice', age=2),
-                                   Row(name='Bob', age=5)]).toDF()
-    globs['df4'] = sc.parallelize([Row(name='Alice', age=10, height=80),
-                                   Row(name='Bob', age=5, height=None),
-                                   Row(name='Tom', age=None, height=None),
-                                   Row(name=None, age=None, height=None)]).toDF()
-    globs['df5'] = sc.parallelize([Row(name='Alice', spy=False, age=10),
-                                   Row(name='Bob', spy=None, age=5),
-                                   Row(name='Mallory', spy=True, age=None)]).toDF()
+    globs['df2'] = sc.parallelize([Row(height=80, name='Tom'), Row(height=85, name='Bob')]).toDF()
+    globs['df3'] = sc.parallelize([Row(age=2, name='Alice'),
+                                   Row(age=5, name='Bob')]).toDF()
+    globs['df4'] = sc.parallelize([Row(age=10, height=80, name='Alice'),
+                                   Row(age=5, height=None, name='Bob'),
+                                   Row(age=None, height=None, name='Tom'),
+                                   Row(age=None, height=None, name=None)]).toDF()
+    globs['df5'] = sc.parallelize([Row(age=10, name='Alice', spy=False),
+                                   Row(age=5, name='Bob', spy=None),
+                                   Row(age=None, name='Mallory', spy=True)]).toDF()
     globs['sdf'] = sc.parallelize([Row(name='Tom', time=1479441846),
                                    Row(name='Bob', time=1479442946)]).toDF()
 
diff --git a/python/pyspark/sql/dataframe.pyi b/python/pyspark/sql/dataframe.pyi
new file mode 100644
index 0000000000000..af1bac61c2291
--- /dev/null
+++ b/python/pyspark/sql/dataframe.pyi
@@ -0,0 +1,328 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import overload
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Iterator,
+    List,
+    Optional,
+    Tuple,
+    Union,
+)
+
+from py4j.java_gateway import JavaObject  # type: ignore[import]
+
+from pyspark.sql._typing import ColumnOrName, LiteralType, OptionalPrimitiveType
+from pyspark._typing import PrimitiveType
+from pyspark.sql.types import (  # noqa: F401
+    StructType,
+    StructField,
+    StringType,
+    IntegerType,
+    Row,
+)  # noqa: F401
+from pyspark.sql.context import SQLContext
+from pyspark.sql.group import GroupedData
+from pyspark.sql.readwriter import DataFrameWriter, DataFrameWriterV2
+from pyspark.sql.streaming import DataStreamWriter
+from pyspark.sql.column import Column
+from pyspark.rdd import RDD
+from pyspark.storagelevel import StorageLevel
+
+from pyspark.sql.pandas.conversion import PandasConversionMixin
+from pyspark.sql.pandas.map_ops import PandasMapOpsMixin
+
+class DataFrame(PandasMapOpsMixin, PandasConversionMixin):
+    sql_ctx: SQLContext
+    is_cached: bool
+    def __init__(self, jdf: JavaObject, sql_ctx: SQLContext) -> None: ...
+    @property
+    def rdd(self) -> RDD[Row]: ...
+    @property
+    def na(self) -> DataFrameNaFunctions: ...
+    @property
+    def stat(self) -> DataFrameStatFunctions: ...
+    def toJSON(self, use_unicode: bool = ...) -> RDD[str]: ...
+    def registerTempTable(self, name: str) -> None: ...
+    def createTempView(self, name: str) -> None: ...
+    def createOrReplaceTempView(self, name: str) -> None: ...
+    def createGlobalTempView(self, name: str) -> None: ...
+    @property
+    def write(self) -> DataFrameWriter: ...
+    @property
+    def writeStream(self) -> DataStreamWriter: ...
+    @property
+    def schema(self) -> StructType: ...
+    def printSchema(self) -> None: ...
+    def explain(
+        self, extended: Optional[Union[bool, str]] = ..., mode: Optional[str] = ...
+    ) -> None: ...
+    def exceptAll(self, other: DataFrame) -> DataFrame: ...
+    def isLocal(self) -> bool: ...
+    @property
+    def isStreaming(self) -> bool: ...
+    def show(
+        self, n: int = ..., truncate: Union[bool, int] = ..., vertical: bool = ...
+    ) -> None: ...
+    def checkpoint(self, eager: bool = ...) -> DataFrame: ...
+    def localCheckpoint(self, eager: bool = ...) -> DataFrame: ...
+    def withWatermark(
+        self, eventTime: str, delayThreshold: str
+    ) -> DataFrame: ...
+    def hint(self, name: str, *parameters: Union[PrimitiveType, List[PrimitiveType]]) -> DataFrame: ...
+    def count(self) -> int: ...
+    def collect(self) -> List[Row]: ...
+    def toLocalIterator(self, prefetchPartitions: bool = ...) -> Iterator[Row]: ...
+    def limit(self, num: int) -> DataFrame: ...
+    def take(self, num: int) -> List[Row]: ...
+    def tail(self, num: int) -> List[Row]: ...
+    def foreach(self, f: Callable[[Row], None]) -> None: ...
+    def foreachPartition(self, f: Callable[[Iterator[Row]], None]) -> None: ...
+    def cache(self) -> DataFrame: ...
+    def persist(self, storageLevel: StorageLevel = ...) -> DataFrame: ...
+    @property
+    def storageLevel(self) -> StorageLevel: ...
+    def unpersist(self, blocking: bool = ...) -> DataFrame: ...
+    def coalesce(self, numPartitions: int) -> DataFrame: ...
+    @overload
+    def repartition(self, numPartitions: int, *cols: ColumnOrName) -> DataFrame: ...
+    @overload
+    def repartition(self, *cols: ColumnOrName) -> DataFrame: ...
+    @overload
+    def repartitionByRange(
+        self, numPartitions: int, *cols: ColumnOrName
+    ) -> DataFrame: ...
+    @overload
+    def repartitionByRange(self, *cols: ColumnOrName) -> DataFrame: ...
+    def distinct(self) -> DataFrame: ...
+    @overload
+    def sample(self, fraction: float, seed: Optional[int] = ...) -> DataFrame: ...
+    @overload
+    def sample(
+        self,
+        withReplacement: Optional[bool],
+        fraction: float,
+        seed: Optional[int] = ...,
+    ) -> DataFrame: ...
+    def sampleBy(
+        self, col: ColumnOrName, fractions: Dict[Any, float], seed: Optional[int] = ...
+    ) -> DataFrame: ...
+    def randomSplit(
+        self, weights: List[float], seed: Optional[int] = ...
+    ) -> List[DataFrame]: ...
+    @property
+    def dtypes(self) -> List[Tuple[str, str]]: ...
+    @property
+    def columns(self) -> List[str]: ...
+    def colRegex(self, colName: str) -> Column: ...
+    def alias(self, alias: str) -> DataFrame: ...
+    def crossJoin(self, other: DataFrame) -> DataFrame: ...
+    def join(
+        self,
+        other: DataFrame,
+        on: Optional[Union[str, List[str], Column, List[Column]]] = ...,
+        how: Optional[str] = ...,
+    ) -> DataFrame: ...
+    def sortWithinPartitions(
+        self,
+        *cols: Union[str, Column, List[Union[str, Column]]],
+        ascending: Union[bool, List[bool]] = ...
+    ) -> DataFrame: ...
+    def sort(
+        self,
+        *cols: Union[str, Column, List[Union[str, Column]]],
+        ascending: Union[bool, List[bool]] = ...
+    ) -> DataFrame: ...
+    def orderBy(
+        self,
+        *cols: Union[str, Column, List[Union[str, Column]]],
+        ascending: Union[bool, List[bool]] = ...
+    ) -> DataFrame: ...
+    def describe(self, *cols: Union[str, List[str]]) -> DataFrame: ...
+    def summary(self, *statistics: str) -> DataFrame: ...
+    @overload
+    def head(self) -> Row: ...
+    @overload
+    def head(self, n: int) -> List[Row]: ...
+    def first(self) -> Row: ...
+    def __getitem__(self, item: Union[int, str, Column, List, Tuple]) -> Column: ...
+    def __getattr__(self, name: str) -> Column: ...
+    @overload
+    def select(self, *cols: ColumnOrName) -> DataFrame: ...
+    @overload
+    def select(self, __cols: Union[List[Column], List[str]]) -> DataFrame: ...
+    @overload
+    def selectExpr(self, *expr: str) -> DataFrame: ...
+    @overload
+    def selectExpr(self, *expr: List[str]) -> DataFrame: ...
+    def filter(self, condition: ColumnOrName) -> DataFrame: ...
+    @overload
+    def groupBy(self, *cols: ColumnOrName) -> GroupedData: ...
+    @overload
+    def groupBy(self, __cols: Union[List[Column], List[str]]) -> GroupedData: ...
+    @overload
+    def rollup(self, *cols: ColumnOrName) -> GroupedData: ...
+    @overload
+    def rollup(self, __cols: Union[List[Column], List[str]]) -> GroupedData: ...
+    @overload
+    def cube(self, *cols: ColumnOrName) -> GroupedData: ...
+    @overload
+    def cube(self, __cols: Union[List[Column], List[str]]) -> GroupedData: ...
+    def agg(self, *exprs: Union[Column, Dict[str, str]]) -> DataFrame: ...
+    def union(self, other: DataFrame) -> DataFrame: ...
+    def unionAll(self, other: DataFrame) -> DataFrame: ...
+    def unionByName(
+        self, other: DataFrame, allowMissingColumns: bool = ...
+    ) -> DataFrame: ...
+    def intersect(self, other: DataFrame) -> DataFrame: ...
+    def intersectAll(self, other: DataFrame) -> DataFrame: ...
+    def subtract(self, other: DataFrame) -> DataFrame: ...
+    def dropDuplicates(self, subset: Optional[List[str]] = ...) -> DataFrame: ...
+    def dropna(
+        self,
+        how: str = ...,
+        thresh: Optional[int] = ...,
+        subset: Optional[Union[str, Tuple[str, ...], List[str]]] = ...,
+    ) -> DataFrame: ...
+    @overload
+    def fillna(
+        self,
+        value: LiteralType,
+        subset: Optional[Union[str, Tuple[str, ...], List[str]]] = ...,
+    ) -> DataFrame: ...
+    @overload
+    def fillna(self, value: Dict[str, LiteralType]) -> DataFrame: ...
+    @overload
+    def replace(
+        self,
+        to_replace: LiteralType,
+        value: OptionalPrimitiveType,
+        subset: Optional[List[str]] = ...,
+    ) -> DataFrame: ...
+    @overload
+    def replace(
+        self,
+        to_replace: List[LiteralType],
+        value: List[OptionalPrimitiveType],
+        subset: Optional[List[str]] = ...,
+    ) -> DataFrame: ...
+    @overload
+    def replace(
+        self,
+        to_replace: Dict[LiteralType, OptionalPrimitiveType],
+        subset: Optional[List[str]] = ...,
+    ) -> DataFrame: ...
+    @overload
+    def replace(
+        self,
+        to_replace: List[LiteralType],
+        value: OptionalPrimitiveType,
+        subset: Optional[List[str]] = ...,
+    ) -> DataFrame: ...
+    def approxQuantile(
+        self,
+        col: Union[str, Tuple[str, ...], List[str]],
+        probabilities: Union[List[float], Tuple[float, ...]],
+        relativeError: float
+    ) -> List[float]: ...
+    def corr(self, col1: str, col2: str, method: Optional[str] = ...) -> float: ...
+    def cov(self, col1: str, col2: str) -> float: ...
+    def crosstab(self, col1: str, col2: str) -> DataFrame: ...
+    def freqItems(
+        self, cols: Union[List[str], Tuple[str]], support: Optional[float] = ...
+    ) -> DataFrame: ...
+    def withColumn(self, colName: str, col: Column) -> DataFrame: ...
+    def withColumnRenamed(self, existing: str, new: str) -> DataFrame: ...
+    @overload
+    def drop(self, cols: ColumnOrName) -> DataFrame: ...
+    @overload
+    def drop(self, *cols: str) -> DataFrame: ...
+    def toDF(self, *cols: ColumnOrName) -> DataFrame: ...
+    def transform(self, func: Callable[[DataFrame], DataFrame]) -> DataFrame: ...
+    @overload
+    def groupby(self, *cols: ColumnOrName) -> GroupedData: ...
+    @overload
+    def groupby(self, __cols: Union[List[Column], List[str]]) -> GroupedData: ...
+    def drop_duplicates(self, subset: Optional[List[str]] = ...) -> DataFrame: ...
+    def where(self, condition: ColumnOrName) -> DataFrame: ...
+    def sameSemantics(self, other: DataFrame) -> bool: ...
+    def semanticHash(self) -> int: ...
+    def inputFiles(self) -> List[str]: ...
+    def writeTo(self, table: str) -> DataFrameWriterV2: ...
+
+class DataFrameNaFunctions:
+    df: DataFrame
+    def __init__(self, df: DataFrame) -> None: ...
+    def drop(
+        self,
+        how: str = ...,
+        thresh: Optional[int] = ...,
+        subset: Optional[List[str]] = ...,
+    ) -> DataFrame: ...
+    @overload
+    def fill(
+        self, value: LiteralType, subset: Optional[List[str]] = ...
+    ) -> DataFrame: ...
+    @overload
+    def fill(self, value: Dict[str, LiteralType]) -> DataFrame: ...
+    @overload
+    def replace(
+        self,
+        to_replace: LiteralType,
+        value: OptionalPrimitiveType,
+        subset: Optional[List[str]] = ...,
+    ) -> DataFrame: ...
+    @overload
+    def replace(
+        self,
+        to_replace: List[LiteralType],
+        value: List[OptionalPrimitiveType],
+        subset: Optional[List[str]] = ...,
+    ) -> DataFrame: ...
+    @overload
+    def replace(
+        self,
+        to_replace: Dict[LiteralType, OptionalPrimitiveType],
+        subset: Optional[List[str]] = ...,
+    ) -> DataFrame: ...
+    @overload
+    def replace(
+        self,
+        to_replace: List[LiteralType],
+        value: OptionalPrimitiveType,
+        subset: Optional[List[str]] = ...,
+    ) -> DataFrame: ...
+
+class DataFrameStatFunctions:
+    df: DataFrame
+    def __init__(self, df: DataFrame) -> None: ...
+    def approxQuantile(
+        self, col: str, probabilities: List[float], relativeError: float
+    ) -> List[float]: ...
+    def corr(self, col1: str, col2: str, method: Optional[str] = ...) -> float: ...
+    def cov(self, col1: str, col2: str) -> float: ...
+    def crosstab(self, col1: str, col2: str) -> DataFrame: ...
+    def freqItems(
+        self, cols: List[str], support: Optional[float] = ...
+    ) -> DataFrame: ...
+    def sampleBy(
+        self, col: str, fractions: Dict[Any, float], seed: Optional[int] = ...
+    ) -> DataFrame: ...
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index de0d38e2aed12..6ea73430367fa 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -22,322 +22,850 @@
 import functools
 import warnings
 
-if sys.version < "3":
-    from itertools import imap as map
-
-if sys.version >= '3':
-    basestring = str
-
 from pyspark import since, SparkContext
-from pyspark.rdd import ignore_unicode_prefix, PythonEvalType
-from pyspark.sql.column import Column, _to_java_column, _to_seq, _create_column_from_literal, \
-    _create_column_from_name
+from pyspark.rdd import PythonEvalType
+from pyspark.sql.column import Column, _to_java_column, _to_seq, _create_column_from_literal
 from pyspark.sql.dataframe import DataFrame
 from pyspark.sql.types import StringType, DataType
 # Keep UserDefinedFunction import for backwards compatible import; moved in SPARK-22409
-from pyspark.sql.udf import UserDefinedFunction, _create_udf
+from pyspark.sql.udf import UserDefinedFunction, _create_udf  # noqa: F401
+from pyspark.sql.udf import _create_udf
 # Keep pandas_udf and PandasUDFType import for backwards compatible import; moved in SPARK-28264
-from pyspark.sql.pandas.functions import pandas_udf, PandasUDFType
+from pyspark.sql.pandas.functions import pandas_udf, PandasUDFType  # noqa: F401
 from pyspark.sql.utils import to_str
 
-# Note to developers: all of PySpark functions here take string as column names whenever possible.
-# Namely, if columns are referred as arguments, they can be always both Column or string,
-# even though there might be few exceptions for legacy or inevitable reasons.
-# If you are fixing other language APIs together, also please note that Scala side is not the case
-# since it requires to make every single overridden definition.
+# Note to developers: all of PySpark functions here take string as column names whenever possible.
+# Namely, if columns are referred as arguments, they can be always both Column or string,
+# even though there might be few exceptions for legacy or inevitable reasons.
+# If you are fixing other language APIs together, also please note that Scala side is not the case
+# since it requires to make every single overridden definition.
+
+
+def _get_get_jvm_function(name, sc):
+    """
+    Retrieves JVM function identified by name from
+    Java gateway associated with sc.
+    """
+    return getattr(sc._jvm.functions, name)
+
+
+def _invoke_function(name, *args):
+    """
+    Invokes JVM function identified by name with args
+    and wraps the result with :class:`~pyspark.sql.Column`.
+    """
+    jf = _get_get_jvm_function(name, SparkContext._active_spark_context)
+    return Column(jf(*args))
+
+
+def _invoke_function_over_column(name, col):
+    """
+    Invokes unary JVM function identified by name
+    and wraps the result with :class:`~pyspark.sql.Column`.
+    """
+    return _invoke_function(name, _to_java_column(col))
+
+
+def _invoke_binary_math_function(name, col1, col2):
+    """
+    Invokes binary JVM math function identified by name
+    and wraps the result with :class:`~pyspark.sql.Column`.
+    """
+    return _invoke_function(
+        name,
+        # For legacy reasons, the arguments here can be implicitly converted into floats,
+        # if they are not columns or strings.
+        _to_java_column(col1) if isinstance(col1, (str, Column)) else float(col1),
+        _to_java_column(col2) if isinstance(col2, (str, Column)) else float(col2)
+    )
+
+
+def _options_to_str(options=None):
+    if options:
+        return {key: to_str(value) for (key, value) in options.items()}
+    return {}
+
+
+def lit(col):
+    """
+    Creates a :class:`~pyspark.sql.Column` of literal value.
+
+    .. versionadded:: 1.3.0
+
+    Examples
+    --------
+    >>> df.select(lit(5).alias('height')).withColumn('spark_user', lit(True)).take(1)
+    [Row(height=5, spark_user=True)]
+    """
+    return col if isinstance(col, Column) else _invoke_function("lit", col)
+
+
+@since(1.3)
+def col(col):
+    """
+    Returns a :class:`~pyspark.sql.Column` based on the given column name.'
+    """
+    return _invoke_function("col", col)
+
+
+@since(1.3)
+def column(col):
+    """
+    Returns a :class:`~pyspark.sql.Column` based on the given column name.'
+    """
+    return col(col)
+
+
+@since(1.3)
+def asc(col):
+    """
+    Returns a sort expression based on the ascending order of the given column name.
+    """
+    return (
+        col.asc() if isinstance(col, Column)
+        else _invoke_function("asc", col)
+    )
+
+
+@since(1.3)
+def desc(col):
+    """
+    Returns a sort expression based on the descending order of the given column name.
+    """
+    return (
+        col.desc() if isinstance(col, Column)
+        else _invoke_function("desc", col)
+    )
+
+
+@since(1.3)
+def sqrt(col):
+    """
+    Computes the square root of the specified float value.
+    """
+    return _invoke_function_over_column("sqrt", col)
+
+
+@since(1.3)
+def abs(col):
+    """
+    Computes the absolute value.
+    """
+    return _invoke_function_over_column("abs", col)
+
+
+@since(1.3)
+def max(col):
+    """
+    Aggregate function: returns the maximum value of the expression in a group.
+    """
+    return _invoke_function_over_column("max", col)
+
+
+@since(1.3)
+def min(col):
+    """
+    Aggregate function: returns the minimum value of the expression in a group.
+    """
+    return _invoke_function_over_column("min", col)
+
+
+@since(1.3)
+def count(col):
+    """
+    Aggregate function: returns the number of items in a group.
+    """
+    return _invoke_function_over_column("count", col)
+
+
+@since(1.3)
+def sum(col):
+    """
+    Aggregate function: returns the sum of all values in the expression.
+    """
+    return _invoke_function_over_column("sum", col)
+
+
+@since(1.3)
+def avg(col):
+    """
+    Aggregate function: returns the average of the values in a group.
+    """
+    return _invoke_function_over_column("avg", col)
+
+
+@since(1.3)
+def mean(col):
+    """
+    Aggregate function: returns the average of the values in a group.
+    """
+    return _invoke_function_over_column("mean", col)
+
+
+@since(1.3)
+def sumDistinct(col):
+    """
+    Aggregate function: returns the sum of distinct values in the expression.
+
+    .. deprecated:: 3.2.0
+        Use :func:`sum_distinct` instead.
+    """
+    warnings.warn("Deprecated in 3.2, use sum_distinct instead.", FutureWarning)
+    return sum_distinct(col)
+
+
+@since(3.2)
+def sum_distinct(col):
+    """
+    Aggregate function: returns the sum of distinct values in the expression.
+    """
+    return _invoke_function_over_column("sum_distinct", col)
+
+
+def product(col):
+    """
+    Aggregate function: returns the product of the values in a group.
+
+    .. versionadded:: 3.2.0
+
+    Parameters
+    ----------
+    col : str, :class:`Column`
+        column containing values to be multiplied together
+
+    Examples
+    --------
+    >>> df = spark.range(1, 10).toDF('x').withColumn('mod3', col('x') % 3)
+    >>> prods = df.groupBy('mod3').agg(product('x').alias('product'))
+    >>> prods.orderBy('mod3').show()
+    +----+-------+
+    |mod3|product|
+    +----+-------+
+    |   0|  162.0|
+    |   1|   28.0|
+    |   2|   80.0|
+    +----+-------+
+
+    """
+    return _invoke_function_over_column("product", col)
+
+
+def acos(col):
+    """
+    .. versionadded:: 1.4.0
+
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
+        inverse cosine of `col`, as if computed by `java.lang.Math.acos()`
+    """
+    return _invoke_function_over_column("acos", col)
+
+
+def acosh(col):
+    """
+    Computes inverse hyperbolic cosine of the input column.
+
+    .. versionadded:: 3.1.0
+
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
+    """
+    return _invoke_function_over_column("acosh", col)
+
+
+def asin(col):
+    """
+    .. versionadded:: 1.3.0
+
+
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
+        inverse sine of `col`, as if computed by `java.lang.Math.asin()`
+    """
+    return _invoke_function_over_column("asin", col)
+
+
+def asinh(col):
+    """
+    Computes inverse hyperbolic sine of the input column.
+
+    .. versionadded:: 3.1.0
+
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
+    """
+    return _invoke_function_over_column("asinh", col)
+
+
+def atan(col):
+    """
+    .. versionadded:: 1.4.0
+
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
+        inverse tangent of `col`, as if computed by `java.lang.Math.atan()`
+    """
+    return _invoke_function_over_column("atan", col)
+
+
+def atanh(col):
+    """
+    Computes inverse hyperbolic tangent of the input column.
+
+    .. versionadded:: 3.1.0
+
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
+    """
+    return _invoke_function_over_column("atanh", col)
+
+
+@since(1.4)
+def cbrt(col):
+    """
+    Computes the cube-root of the given value.
+    """
+    return _invoke_function_over_column("cbrt", col)
+
+
+@since(1.4)
+def ceil(col):
+    """
+    Computes the ceiling of the given value.
+    """
+    return _invoke_function_over_column("ceil", col)
+
+
+def cos(col):
+    """
+    .. versionadded:: 1.4.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        angle in radians
+
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
+        cosine of the angle, as if computed by `java.lang.Math.cos()`.
+    """
+    return _invoke_function_over_column("cos", col)
+
+
+def cosh(col):
+    """
+    .. versionadded:: 1.4.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        hyperbolic angle
+
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
+        hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh()`
+    """
+    return _invoke_function_over_column("cosh", col)
+
+
+@since(1.4)
+def exp(col):
+    """
+    Computes the exponential of the given value.
+    """
+    return _invoke_function_over_column("exp", col)
+
+
+@since(1.4)
+def expm1(col):
+    """
+    Computes the exponential of the given value minus one.
+    """
+    return _invoke_function_over_column("expm1", col)
+
+
+@since(1.4)
+def floor(col):
+    """
+    Computes the floor of the given value.
+    """
+    return _invoke_function_over_column("floor", col)
+
+
+@since(1.4)
+def log(col):
+    """
+    Computes the natural logarithm of the given value.
+    """
+    return _invoke_function_over_column("log", col)
+
+
+@since(1.4)
+def log10(col):
+    """
+    Computes the logarithm of the given value in Base 10.
+    """
+    return _invoke_function_over_column("log10", col)
+
+
+@since(1.4)
+def log1p(col):
+    """
+    Computes the natural logarithm of the given value plus one.
+    """
+    return _invoke_function_over_column("log1p", col)
+
+
+@since(1.4)
+def rint(col):
+    """
+    Returns the double value that is closest in value to the argument and
+    is equal to a mathematical integer.
+    """
+    return _invoke_function_over_column("rint", col)
+
+
+@since(1.4)
+def signum(col):
+    """
+    Computes the signum of the given value.
+    """
+    return _invoke_function_over_column("signum", col)
+
+
+def sin(col):
+    """
+    .. versionadded:: 1.4.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
+        sine of the angle, as if computed by `java.lang.Math.sin()`
+    """
+    return _invoke_function_over_column("sin", col)
+
+
+def sinh(col):
+    """
+    .. versionadded:: 1.4.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        hyperbolic angle
+
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
+        hyperbolic sine of the given value,
+        as if computed by `java.lang.Math.sinh()`
+    """
+    return _invoke_function_over_column("sinh", col)
+
+
+def tan(col):
+    """
+    .. versionadded:: 1.4.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        angle in radians
+
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
+        tangent of the given value, as if computed by `java.lang.Math.tan()`
+    """
+    return _invoke_function_over_column("tan", col)
+
+
+def tanh(col):
+    """
+    .. versionadded:: 1.4.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        hyperbolic angle
+
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
+        hyperbolic tangent of the given value
+        as if computed by `java.lang.Math.tanh()`
+    """
+    return _invoke_function_over_column("tanh", col)
+
+
+@since(1.4)
+def toDegrees(col):
+    """
+    .. deprecated:: 2.1.0
+        Use :func:`degrees` instead.
+    """
+    warnings.warn("Deprecated in 2.1, use degrees instead.", FutureWarning)
+    return degrees(col)
+
+
+@since(1.4)
+def toRadians(col):
+    """
+    .. deprecated:: 2.1.0
+        Use :func:`radians` instead.
+    """
+    warnings.warn("Deprecated in 2.1, use radians instead.", FutureWarning)
+    return radians(col)
+
+
+@since(1.4)
+def bitwiseNOT(col):
+    """
+    Computes bitwise not.
+
+    .. deprecated:: 3.2.0
+        Use :func:`bitwise_not` instead.
+    """
+    warnings.warn("Deprecated in 3.2, use bitwise_not instead.", FutureWarning)
+    return bitwise_not(col)
+
+
+@since(3.2)
+def bitwise_not(col):
+    """
+    Computes bitwise not.
+    """
+    return _invoke_function_over_column("bitwise_not", col)
+
+
+@since(2.4)
+def asc_nulls_first(col):
+    """
+    Returns a sort expression based on the ascending order of the given
+    column name, and null values return before non-null values.
+    """
+    return (
+        col.asc_nulls_first() if isinstance(col, Column)
+        else _invoke_function("asc_nulls_first", col)
+    )
+
+
+@since(2.4)
+def asc_nulls_last(col):
+    """
+    Returns a sort expression based on the ascending order of the given
+    column name, and null values appear after non-null values.
+    """
+    return (
+        col.asc_nulls_last() if isinstance(col, Column)
+        else _invoke_function("asc_nulls_last", col)
+    )
+
+
+@since(2.4)
+def desc_nulls_first(col):
+    """
+    Returns a sort expression based on the descending order of the given
+    column name, and null values appear before non-null values.
+    """
+    return (
+        col.desc_nulls_first() if isinstance(col, Column)
+        else _invoke_function("desc_nulls_first", col)
+    )
+
+
+@since(2.4)
+def desc_nulls_last(col):
+    """
+    Returns a sort expression based on the descending order of the given
+    column name, and null values appear after non-null values.
+    """
+    return (
+        col.desc_nulls_last() if isinstance(col, Column)
+        else _invoke_function("desc_nulls_last", col)
+    )
+
+
+@since(1.6)
+def stddev(col):
+    """
+    Aggregate function: alias for stddev_samp.
+    """
+    return _invoke_function_over_column("stddev", col)
+
+
+@since(1.6)
+def stddev_samp(col):
+    """
+    Aggregate function: returns the unbiased sample standard deviation of
+    the expression in a group.
+    """
+    return _invoke_function_over_column("stddev_samp", col)
+
+
+@since(1.6)
+def stddev_pop(col):
+    """
+    Aggregate function: returns population standard deviation of
+    the expression in a group.
+    """
+    return _invoke_function_over_column("stddev_pop", col)
+
+
+@since(1.6)
+def variance(col):
+    """
+    Aggregate function: alias for var_samp
+    """
+    return _invoke_function_over_column("variance", col)
+
+
+@since(1.6)
+def var_samp(col):
+    """
+    Aggregate function: returns the unbiased sample variance of
+    the values in a group.
+    """
+    return _invoke_function_over_column("var_samp", col)
+
+
+@since(1.6)
+def var_pop(col):
+    """
+    Aggregate function: returns the population variance of the values in a group.
+    """
+    return _invoke_function_over_column("var_pop", col)
+
+
+@since(1.6)
+def skewness(col):
+    """
+    Aggregate function: returns the skewness of the values in a group.
+    """
+    return _invoke_function_over_column("skewness", col)
+
+
+@since(1.6)
+def kurtosis(col):
+    """
+    Aggregate function: returns the kurtosis of the values in a group.
+    """
+    return _invoke_function_over_column("kurtosis", col)
+
+
+def collect_list(col):
+    """
+    Aggregate function: returns a list of objects with duplicates.
+
+    .. versionadded:: 1.6.0
+
+    Notes
+    -----
+    The function is non-deterministic because the order of collected results depends
+    on the order of the rows which may be non-deterministic after a shuffle.
+
+    Examples
+    --------
+    >>> df2 = spark.createDataFrame([(2,), (5,), (5,)], ('age',))
+    >>> df2.agg(collect_list('age')).collect()
+    [Row(collect_list(age)=[2, 5, 5])]
+    """
+    return _invoke_function_over_column("collect_list", col)
+
+
+def collect_set(col):
+    """
+    Aggregate function: returns a set of objects with duplicate elements eliminated.
+
+    .. versionadded:: 1.6.0
+
+    Notes
+    -----
+    The function is non-deterministic because the order of collected results depends
+    on the order of the rows which may be non-deterministic after a shuffle.
+
+    Examples
+    --------
+    >>> df2 = spark.createDataFrame([(2,), (5,), (5,)], ('age',))
+    >>> df2.agg(collect_set('age')).collect()
+    [Row(collect_set(age)=[5, 2])]
+    """
+    return _invoke_function_over_column("collect_set", col)
+
+
+def degrees(col):
+    """
+    Converts an angle measured in radians to an approximately equivalent angle
+    measured in degrees.
+
+    .. versionadded:: 2.1.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        angle in radians
+
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
+        angle in degrees, as if computed by `java.lang.Math.toDegrees()`
+    """
+    return _invoke_function_over_column("degrees", col)
+
+
+def radians(col):
+    """
+    Converts an angle measured in degrees to an approximately equivalent angle
+    measured in radians.
+
+    .. versionadded:: 2.1.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        angle in degrees
+
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
+        angle in radians, as if computed by `java.lang.Math.toRadians()`
+    """
+    return _invoke_function_over_column("radians", col)
+
+
+def atan2(col1, col2):
+    """
+    .. versionadded:: 1.4.0
+
+    Parameters
+    ----------
+    col1 : str, :class:`~pyspark.sql.Column` or float
+        coordinate on y-axis
+    col2 : str, :class:`~pyspark.sql.Column` or float
+        coordinate on x-axis
+
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
+        the `theta` component of the point
+        (`r`, `theta`)
+        in polar coordinates that corresponds to the point
+        (`x`, `y`) in Cartesian coordinates,
+        as if computed by `java.lang.Math.atan2()`
+    """
+    return _invoke_binary_math_function("atan2", col1, col2)
+
+
+@since(1.4)
+def hypot(col1, col2):
+    """
+    Computes ``sqrt(a^2 + b^2)`` without intermediate overflow or underflow.
+    """
+    return _invoke_binary_math_function("hypot", col1, col2)
+
+
+@since(1.4)
+def pow(col1, col2):
+    """
+    Returns the value of the first argument raised to the power of the second argument.
+    """
+    return _invoke_binary_math_function("pow", col1, col2)
 
 
-def _create_function(name, doc=""):
-    """Create a PySpark function by its name"""
-    def _(col):
-        sc = SparkContext._active_spark_context
-        jc = getattr(sc._jvm.functions, name)(col._jc if isinstance(col, Column) else col)
-        return Column(jc)
-    _.__name__ = name
-    _.__doc__ = doc
-    return _
+@since(1.6)
+def row_number():
+    """
+    Window function: returns a sequential number starting at 1 within a window partition.
+    """
+    return _invoke_function("row_number")
 
 
-def _create_function_over_column(name, doc=""):
-    """Similar with `_create_function` but creates a PySpark function that takes a column
-    (as string as well). This is mainly for PySpark functions to take strings as
-    column names.
+@since(1.6)
+def dense_rank():
     """
-    def _(col):
-        sc = SparkContext._active_spark_context
-        jc = getattr(sc._jvm.functions, name)(_to_java_column(col))
-        return Column(jc)
-    _.__name__ = name
-    _.__doc__ = doc
-    return _
+    Window function: returns the rank of rows within a window partition, without any gaps.
+
+    The difference between rank and dense_rank is that dense_rank leaves no gaps in ranking
+    sequence when there are ties. That is, if you were ranking a competition using dense_rank
+    and had three people tie for second place, you would say that all three were in second
+    place and that the next person came in third. Rank would give me sequential numbers, making
+    the person that came in third place (after the ties) would register as coming in fifth.
 
+    This is equivalent to the DENSE_RANK function in SQL.
+    """
+    return _invoke_function("dense_rank")
 
-def _wrap_deprecated_function(func, message):
-    """ Wrap the deprecated function to print out deprecation warnings"""
-    def _(col):
-        warnings.warn(message, DeprecationWarning)
-        return func(col)
-    return functools.wraps(func)(_)
 
+@since(1.6)
+def rank():
+    """
+    Window function: returns the rank of rows within a window partition.
 
-def _create_binary_mathfunction(name, doc=""):
-    """ Create a binary mathfunction by name"""
-    def _(col1, col2):
-        sc = SparkContext._active_spark_context
-        # For legacy reasons, the arguments here can be implicitly converted into floats,
-        # if they are not columns or strings.
-        if isinstance(col1, Column):
-            arg1 = col1._jc
-        elif isinstance(col1, basestring):
-            arg1 = _create_column_from_name(col1)
-        else:
-            arg1 = float(col1)
-
-        if isinstance(col2, Column):
-            arg2 = col2._jc
-        elif isinstance(col2, basestring):
-            arg2 = _create_column_from_name(col2)
-        else:
-            arg2 = float(col2)
-
-        jc = getattr(sc._jvm.functions, name)(arg1, arg2)
-        return Column(jc)
-    _.__name__ = name
-    _.__doc__ = doc
-    return _
-
-
-def _create_window_function(name, doc=''):
-    """ Create a window function by name """
-    def _():
-        sc = SparkContext._active_spark_context
-        jc = getattr(sc._jvm.functions, name)()
-        return Column(jc)
-    _.__name__ = name
-    _.__doc__ = 'Window function: ' + doc
-    return _
-
-
-def _options_to_str(options):
-    return {key: to_str(value) for (key, value) in options.items()}
-
-_lit_doc = """
-    Creates a :class:`Column` of literal value.
+    The difference between rank and dense_rank is that dense_rank leaves no gaps in ranking
+    sequence when there are ties. That is, if you were ranking a competition using dense_rank
+    and had three people tie for second place, you would say that all three were in second
+    place and that the next person came in third. Rank would give me sequential numbers, making
+    the person that came in third place (after the ties) would register as coming in fifth.
 
-    >>> df.select(lit(5).alias('height')).withColumn('spark_user', lit(True)).take(1)
-    [Row(height=5, spark_user=True)]
+    This is equivalent to the RANK function in SQL.
     """
-_functions = {
-    'lit': _lit_doc,
-    'col': 'Returns a :class:`Column` based on the given column name.',
-    'column': 'Returns a :class:`Column` based on the given column name.',
-    'asc': 'Returns a sort expression based on the ascending order of the given column name.',
-    'desc': 'Returns a sort expression based on the descending order of the given column name.',
-}
-
-_functions_over_column = {
-    'sqrt': 'Computes the square root of the specified float value.',
-    'abs': 'Computes the absolute value.',
-
-    'max': 'Aggregate function: returns the maximum value of the expression in a group.',
-    'min': 'Aggregate function: returns the minimum value of the expression in a group.',
-    'count': 'Aggregate function: returns the number of items in a group.',
-    'sum': 'Aggregate function: returns the sum of all values in the expression.',
-    'avg': 'Aggregate function: returns the average of the values in a group.',
-    'mean': 'Aggregate function: returns the average of the values in a group.',
-    'sumDistinct': 'Aggregate function: returns the sum of distinct values in the expression.',
-}
-
-_functions_1_4_over_column = {
-    # unary math functions
-    'acos': ':return: inverse cosine of `col`, as if computed by `java.lang.Math.acos()`',
-    'asin': ':return: inverse sine of `col`, as if computed by `java.lang.Math.asin()`',
-    'atan': ':return: inverse tangent of `col`, as if computed by `java.lang.Math.atan()`',
-    'cbrt': 'Computes the cube-root of the given value.',
-    'ceil': 'Computes the ceiling of the given value.',
-    'cos': """:param col: angle in radians
-           :return: cosine of the angle, as if computed by `java.lang.Math.cos()`.""",
-    'cosh': """:param col: hyperbolic angle
-           :return: hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh()`""",
-    'exp': 'Computes the exponential of the given value.',
-    'expm1': 'Computes the exponential of the given value minus one.',
-    'floor': 'Computes the floor of the given value.',
-    'log': 'Computes the natural logarithm of the given value.',
-    'log10': 'Computes the logarithm of the given value in Base 10.',
-    'log1p': 'Computes the natural logarithm of the given value plus one.',
-    'rint': 'Returns the double value that is closest in value to the argument and' +
-            ' is equal to a mathematical integer.',
-    'signum': 'Computes the signum of the given value.',
-    'sin': """:param col: angle in radians
-           :return: sine of the angle, as if computed by `java.lang.Math.sin()`""",
-    'sinh': """:param col: hyperbolic angle
-           :return: hyperbolic sine of the given value,
-                    as if computed by `java.lang.Math.sinh()`""",
-    'tan': """:param col: angle in radians
-           :return: tangent of the given value, as if computed by `java.lang.Math.tan()`""",
-    'tanh': """:param col: hyperbolic angle
-            :return: hyperbolic tangent of the given value,
-                     as if computed by `java.lang.Math.tanh()`""",
-    'toDegrees': '.. note:: Deprecated in 2.1, use :func:`degrees` instead.',
-    'toRadians': '.. note:: Deprecated in 2.1, use :func:`radians` instead.',
-    'bitwiseNOT': 'Computes bitwise not.',
-}
-
-_functions_2_4 = {
-    'asc_nulls_first': 'Returns a sort expression based on the ascending order of the given' +
-                       ' column name, and null values return before non-null values.',
-    'asc_nulls_last': 'Returns a sort expression based on the ascending order of the given' +
-                      ' column name, and null values appear after non-null values.',
-    'desc_nulls_first': 'Returns a sort expression based on the descending order of the given' +
-                        ' column name, and null values appear before non-null values.',
-    'desc_nulls_last': 'Returns a sort expression based on the descending order of the given' +
-                       ' column name, and null values appear after non-null values',
-}
-
-_collect_list_doc = """
-    Aggregate function: returns a list of objects with duplicates.
+    return _invoke_function("rank")
 
-    .. note:: The function is non-deterministic because the order of collected results depends
-        on the order of the rows which may be non-deterministic after a shuffle.
 
-    >>> df2 = spark.createDataFrame([(2,), (5,), (5,)], ('age',))
-    >>> df2.agg(collect_list('age')).collect()
-    [Row(collect_list(age)=[2, 5, 5])]
+@since(1.6)
+def cume_dist():
     """
-_collect_set_doc = """
-    Aggregate function: returns a set of objects with duplicate elements eliminated.
+    Window function: returns the cumulative distribution of values within a window partition,
+    i.e. the fraction of rows that are below the current row.
+    """
+    return _invoke_function("cume_dist")
 
-    .. note:: The function is non-deterministic because the order of collected results depends
-        on the order of the rows which may be non-deterministic after a shuffle.
 
-    >>> df2 = spark.createDataFrame([(2,), (5,), (5,)], ('age',))
-    >>> df2.agg(collect_set('age')).collect()
-    [Row(collect_set(age)=[5, 2])]
+@since(1.6)
+def percent_rank():
     """
-_functions_1_6_over_column = {
-    # unary math functions
-    'stddev': 'Aggregate function: alias for stddev_samp.',
-    'stddev_samp': 'Aggregate function: returns the unbiased sample standard deviation of' +
-                   ' the expression in a group.',
-    'stddev_pop': 'Aggregate function: returns population standard deviation of' +
-                  ' the expression in a group.',
-    'variance': 'Aggregate function: alias for var_samp.',
-    'var_samp': 'Aggregate function: returns the unbiased sample variance of' +
-                ' the values in a group.',
-    'var_pop':  'Aggregate function: returns the population variance of the values in a group.',
-    'skewness': 'Aggregate function: returns the skewness of the values in a group.',
-    'kurtosis': 'Aggregate function: returns the kurtosis of the values in a group.',
-    'collect_list': _collect_list_doc,
-    'collect_set': _collect_set_doc
-}
-
-_functions_2_1_over_column = {
-    # unary math functions
-    'degrees': """
-               Converts an angle measured in radians to an approximately equivalent angle
-               measured in degrees.
-
-               :param col: angle in radians
-               :return: angle in degrees, as if computed by `java.lang.Math.toDegrees()`
-               """,
-    'radians': """
-               Converts an angle measured in degrees to an approximately equivalent angle
-               measured in radians.
-
-               :param col: angle in degrees
-               :return: angle in radians, as if computed by `java.lang.Math.toRadians()`
-               """,
-}
-
-# math functions that take two arguments as input
-_binary_mathfunctions = {
-    'atan2': """
-             :param col1: coordinate on y-axis
-             :param col2: coordinate on x-axis
-             :return: the `theta` component of the point
-                (`r`, `theta`)
-                in polar coordinates that corresponds to the point
-                (`x`, `y`) in Cartesian coordinates,
-                as if computed by `java.lang.Math.atan2()`
-             """,
-    'hypot': 'Computes ``sqrt(a^2 + b^2)`` without intermediate overflow or underflow.',
-    'pow': 'Returns the value of the first argument raised to the power of the second argument.',
-}
-
-_window_functions = {
-    'row_number':
-        """returns a sequential number starting at 1 within a window partition.""",
-    'dense_rank':
-        """returns the rank of rows within a window partition, without any gaps.
-
-        The difference between rank and dense_rank is that dense_rank leaves no gaps in ranking
-        sequence when there are ties. That is, if you were ranking a competition using dense_rank
-        and had three people tie for second place, you would say that all three were in second
-        place and that the next person came in third. Rank would give me sequential numbers, making
-        the person that came in third place (after the ties) would register as coming in fifth.
-
-        This is equivalent to the DENSE_RANK function in SQL.""",
-    'rank':
-        """returns the rank of rows within a window partition.
-
-        The difference between rank and dense_rank is that dense_rank leaves no gaps in ranking
-        sequence when there are ties. That is, if you were ranking a competition using dense_rank
-        and had three people tie for second place, you would say that all three were in second
-        place and that the next person came in third. Rank would give me sequential numbers, making
-        the person that came in third place (after the ties) would register as coming in fifth.
-
-        This is equivalent to the RANK function in SQL.""",
-    'cume_dist':
-        """returns the cumulative distribution of values within a window partition,
-        i.e. the fraction of rows that are below the current row.""",
-    'percent_rank':
-        """returns the relative rank (i.e. percentile) of rows within a window partition.""",
-}
-
-# Wraps deprecated functions (keys) with the messages (values).
-_functions_deprecated = {
-    'toDegrees': 'Deprecated in 2.1, use degrees instead.',
-    'toRadians': 'Deprecated in 2.1, use radians instead.',
-}
-
-for _name, _doc in _functions.items():
-    globals()[_name] = since(1.3)(_create_function(_name, _doc))
-for _name, _doc in _functions_over_column.items():
-    globals()[_name] = since(1.3)(_create_function_over_column(_name, _doc))
-for _name, _doc in _functions_1_4_over_column.items():
-    globals()[_name] = since(1.4)(_create_function_over_column(_name, _doc))
-for _name, _doc in _binary_mathfunctions.items():
-    globals()[_name] = since(1.4)(_create_binary_mathfunction(_name, _doc))
-for _name, _doc in _window_functions.items():
-    globals()[_name] = since(1.6)(_create_window_function(_name, _doc))
-for _name, _doc in _functions_1_6_over_column.items():
-    globals()[_name] = since(1.6)(_create_function_over_column(_name, _doc))
-for _name, _doc in _functions_2_1_over_column.items():
-    globals()[_name] = since(2.1)(_create_function_over_column(_name, _doc))
-for _name, _message in _functions_deprecated.items():
-    globals()[_name] = _wrap_deprecated_function(globals()[_name], _message)
-for _name, _doc in _functions_2_4.items():
-    globals()[_name] = since(2.4)(_create_function(_name, _doc))
-del _name, _doc
+    Window function: returns the relative rank (i.e. percentile) of rows within a window partition.
+    """
+    return _invoke_function("percent_rank")
 
 
 @since(1.3)
 def approxCountDistinct(col, rsd=None):
     """
-    .. note:: Deprecated in 2.1, use :func:`approx_count_distinct` instead.
+    .. deprecated:: 2.1.0
+        Use :func:`approx_count_distinct` instead.
     """
-    warnings.warn("Deprecated in 2.1, use approx_count_distinct instead.", DeprecationWarning)
+    warnings.warn("Deprecated in 2.1, use approx_count_distinct instead.", FutureWarning)
     return approx_count_distinct(col, rsd)
 
 
-@since(2.1)
 def approx_count_distinct(col, rsd=None):
-    """Aggregate function: returns a new :class:`Column` for approximate distinct count of
-    column `col`.
+    """Aggregate function: returns a new :class:`~pyspark.sql.Column` for approximate distinct count
+    of column `col`.
+
+    .. versionadded:: 2.1.0
 
-    :param rsd: maximum estimation error allowed (default = 0.05). For rsd < 0.01, it is more
-        efficient to use :func:`countDistinct`
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+    rsd : float, optional
+        maximum relative standard deviation allowed (default = 0.05).
+        For rsd < 0.01, it is more efficient to use :func:`count_distinct`
 
+    Examples
+    --------
     >>> df.agg(approx_count_distinct(df.age).alias('distinct_ages')).collect()
     [Row(distinct_ages=2)]
     """
@@ -357,10 +885,13 @@ def broadcast(df):
     return DataFrame(sc._jvm.functions.broadcast(df._jdf), df.sql_ctx)
 
 
-@since(1.4)
 def coalesce(*cols):
     """Returns the first column that is not null.
 
+    .. versionadded:: 1.4.0
+
+    Examples
+    --------
     >>> cDf = spark.createDataFrame([(None, None), (1, None), (None, 2)], ("a", "b"))
     >>> cDf.show()
     +----+----+
@@ -394,11 +925,14 @@ def coalesce(*cols):
     return Column(jc)
 
 
-@since(1.6)
 def corr(col1, col2):
-    """Returns a new :class:`Column` for the Pearson Correlation Coefficient for ``col1``
-    and ``col2``.
+    """Returns a new :class:`~pyspark.sql.Column` for the Pearson Correlation Coefficient for
+    ``col1`` and ``col2``.
+
+    .. versionadded:: 1.6.0
 
+    Examples
+    --------
     >>> a = range(20)
     >>> b = [2 * x for x in range(20)]
     >>> df = spark.createDataFrame(zip(a, b), ["a", "b"])
@@ -409,10 +943,14 @@ def corr(col1, col2):
     return Column(sc._jvm.functions.corr(_to_java_column(col1), _to_java_column(col2)))
 
 
-@since(2.0)
 def covar_pop(col1, col2):
-    """Returns a new :class:`Column` for the population covariance of ``col1`` and ``col2``.
+    """Returns a new :class:`~pyspark.sql.Column` for the population covariance of ``col1`` and
+    ``col2``.
 
+    .. versionadded:: 2.0.0
+
+    Examples
+    --------
     >>> a = [1] * 10
     >>> b = [1] * 10
     >>> df = spark.createDataFrame(zip(a, b), ["a", "b"])
@@ -423,10 +961,14 @@ def covar_pop(col1, col2):
     return Column(sc._jvm.functions.covar_pop(_to_java_column(col1), _to_java_column(col2)))
 
 
-@since(2.0)
 def covar_samp(col1, col2):
-    """Returns a new :class:`Column` for the sample covariance of ``col1`` and ``col2``.
+    """Returns a new :class:`~pyspark.sql.Column` for the sample covariance of ``col1`` and
+    ``col2``.
+
+    .. versionadded:: 2.0.0
 
+    Examples
+    --------
     >>> a = [1] * 10
     >>> b = [1] * 10
     >>> df = spark.createDataFrame(zip(a, b), ["a", "b"])
@@ -437,42 +979,62 @@ def covar_samp(col1, col2):
     return Column(sc._jvm.functions.covar_samp(_to_java_column(col1), _to_java_column(col2)))
 
 
-@since(1.3)
 def countDistinct(col, *cols):
+    """Returns a new :class:`~pyspark.sql.Column` for distinct count of ``col`` or ``cols``.
+
+    An alias of :func:`count_distinct`, and it is encouraged to use :func:`count_distinct`
+    directly.
+
+    .. versionadded:: 1.3.0
+    """
+    return count_distinct(col, *cols)
+
+
+def count_distinct(col, *cols):
     """Returns a new :class:`Column` for distinct count of ``col`` or ``cols``.
 
-    >>> df.agg(countDistinct(df.age, df.name).alias('c')).collect()
+    .. versionadded:: 3.2.0
+
+    Examples
+    --------
+    >>> df.agg(count_distinct(df.age, df.name).alias('c')).collect()
     [Row(c=2)]
 
-    >>> df.agg(countDistinct("age", "name").alias('c')).collect()
+    >>> df.agg(count_distinct("age", "name").alias('c')).collect()
     [Row(c=2)]
     """
     sc = SparkContext._active_spark_context
-    jc = sc._jvm.functions.countDistinct(_to_java_column(col), _to_seq(sc, cols, _to_java_column))
+    jc = sc._jvm.functions.count_distinct(_to_java_column(col), _to_seq(sc, cols, _to_java_column))
     return Column(jc)
 
 
-@since(1.3)
 def first(col, ignorenulls=False):
     """Aggregate function: returns the first value in a group.
 
     The function by default returns the first values it sees. It will return the first non-null
     value it sees when ignoreNulls is set to true. If all values are null, then null is returned.
 
-    .. note:: The function is non-deterministic because its results depends on the order of the
-        rows which may be non-deterministic after a shuffle.
+    .. versionadded:: 1.3.0
+
+    Notes
+    -----
+    The function is non-deterministic because its results depends on the order of the
+    rows which may be non-deterministic after a shuffle.
     """
     sc = SparkContext._active_spark_context
     jc = sc._jvm.functions.first(_to_java_column(col), ignorenulls)
     return Column(jc)
 
 
-@since(2.0)
 def grouping(col):
     """
     Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated
     or not, returns 1 for aggregated or 0 for not aggregated in the result set.
 
+    .. versionadded:: 2.0.0
+
+    Examples
+    --------
     >>> df.cube("name").agg(grouping("name"), sum("age")).orderBy("name").show()
     +-----+--------------+--------+
     | name|grouping(name)|sum(age)|
@@ -487,16 +1049,21 @@ def grouping(col):
     return Column(jc)
 
 
-@since(2.0)
 def grouping_id(*cols):
     """
     Aggregate function: returns the level of grouping, equals to
 
        (grouping(c1) << (n-1)) + (grouping(c2) << (n-2)) + ... + grouping(cn)
 
-    .. note:: The list of columns should match with grouping columns exactly, or empty (means all
-        the grouping columns).
+    .. versionadded:: 2.0.0
 
+    Notes
+    -----
+    The list of columns should match with grouping columns exactly, or empty (means all
+    the grouping columns).
+
+    Examples
+    --------
     >>> df.cube("name").agg(grouping_id(), sum("age")).orderBy("name").show()
     +-----+-------------+--------+
     | name|grouping_id()|sum(age)|
@@ -519,10 +1086,13 @@ def input_file_name():
     return Column(sc._jvm.functions.input_file_name())
 
 
-@since(1.6)
 def isnan(col):
     """An expression that returns true iff the column is NaN.
 
+    .. versionadded:: 1.6.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([(1.0, float('nan')), (float('nan'), 2.0)], ("a", "b"))
     >>> df.select(isnan("a").alias("r1"), isnan(df.a).alias("r2")).collect()
     [Row(r1=False, r2=False), Row(r1=True, r2=True)]
@@ -531,10 +1101,13 @@ def isnan(col):
     return Column(sc._jvm.functions.isnan(_to_java_column(col)))
 
 
-@since(1.6)
 def isnull(col):
     """An expression that returns true iff the column is null.
 
+    .. versionadded:: 1.6.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([(1, None), (None, 2)], ("a", "b"))
     >>> df.select(isnull("a").alias("r1"), isnull(df.a).alias("r2")).collect()
     [Row(r1=False, r2=False), Row(r1=True, r2=True)]
@@ -543,22 +1116,24 @@ def isnull(col):
     return Column(sc._jvm.functions.isnull(_to_java_column(col)))
 
 
-@since(1.3)
 def last(col, ignorenulls=False):
     """Aggregate function: returns the last value in a group.
 
     The function by default returns the last values it sees. It will return the last non-null
     value it sees when ignoreNulls is set to true. If all values are null, then null is returned.
 
-    .. note:: The function is non-deterministic because its results depends on the order of the
-        rows which may be non-deterministic after a shuffle.
+    .. versionadded:: 1.3.0
+
+    Notes
+    -----
+    The function is non-deterministic because its results depends on the order of the
+    rows which may be non-deterministic after a shuffle.
     """
     sc = SparkContext._active_spark_context
     jc = sc._jvm.functions.last(_to_java_column(col), ignorenulls)
     return Column(jc)
 
 
-@since(1.6)
 def monotonically_increasing_id():
     """A column that generates monotonically increasing 64-bit integers.
 
@@ -567,7 +1142,11 @@ def monotonically_increasing_id():
     within each partition in the lower 33 bits. The assumption is that the data frame has
     less than 1 billion partitions, and each partition has less than 8 billion records.
 
-    .. note:: The function is non-deterministic because its result depends on partition IDs.
+    .. versionadded:: 1.6.0
+
+    Notes
+    -----
+    The function is non-deterministic because its result depends on partition IDs.
 
     As an example, consider a :class:`DataFrame` with two partitions, each with 3 records.
     This expression would return the following IDs:
@@ -581,12 +1160,15 @@ def monotonically_increasing_id():
     return Column(sc._jvm.functions.monotonically_increasing_id())
 
 
-@since(1.6)
 def nanvl(col1, col2):
     """Returns col1 if it is not NaN, or col2 if col1 is NaN.
 
     Both inputs should be floating point columns (:class:`DoubleType` or :class:`FloatType`).
 
+    .. versionadded:: 1.6.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([(1.0, float('nan')), (float('nan'), 2.0)], ("a", "b"))
     >>> df.select(nanvl("a", "b").alias("r1"), nanvl(df.a, df.b).alias("r2")).collect()
     [Row(r1=1.0, r2=1.0), Row(r1=2.0, r2=2.0)]
@@ -595,9 +1177,10 @@ def nanvl(col1, col2):
     return Column(sc._jvm.functions.nanvl(_to_java_column(col1), _to_java_column(col2)))
 
 
-@since(3.1)
 def percentile_approx(col, percentage, accuracy=10000):
-    """Returns the approximate percentile value of numeric column col at the given percentage.
+    """Returns the approximate `percentile` of the numeric column `col` which is the smallest value
+    in the ordered `col` values (sorted from least to greatest) such that no more than `percentage`
+    of `col` values is less than the value or equal to that value.
     The value of percentage must be between 0.0 and 1.0.
 
     The accuracy parameter (default: 10000)
@@ -609,6 +1192,10 @@ def percentile_approx(col, percentage, accuracy=10000):
     In this case, returns the approximate percentile array of column col
     at the given percentage array.
 
+    .. versionadded:: 3.1.0
+
+    Examples
+    --------
     >>> key = (col("id") % 3).alias("key")
     >>> value = (randn(42) + key * 10).alias("value")
     >>> df = spark.range(0, 1000, 1, 1).select(key, value)
@@ -648,17 +1235,21 @@ def percentile_approx(col, percentage, accuracy=10000):
     return Column(sc._jvm.functions.percentile_approx(_to_java_column(col), percentage, accuracy))
 
 
-@ignore_unicode_prefix
-@since(1.4)
 def rand(seed=None):
     """Generates a random column with independent and identically distributed (i.i.d.) samples
     uniformly distributed in [0.0, 1.0).
 
-    .. note:: The function is non-deterministic in general case.
+    .. versionadded:: 1.4.0
 
+    Notes
+    -----
+    The function is non-deterministic in general case.
+
+    Examples
+    --------
     >>> df.withColumn('rand', rand(seed=42) * 3).collect()
-    [Row(age=2, name=u'Alice', rand=2.4052597283576684),
-     Row(age=5, name=u'Bob', rand=2.3913904055683974)]
+    [Row(age=2, name='Alice', rand=2.4052597283576684),
+     Row(age=5, name='Bob', rand=2.3913904055683974)]
     """
     sc = SparkContext._active_spark_context
     if seed is not None:
@@ -668,17 +1259,21 @@ def rand(seed=None):
     return Column(jc)
 
 
-@ignore_unicode_prefix
-@since(1.4)
 def randn(seed=None):
     """Generates a column with independent and identically distributed (i.i.d.) samples from
     the standard normal distribution.
 
-    .. note:: The function is non-deterministic in general case.
+    .. versionadded:: 1.4.0
 
+    Notes
+    -----
+    The function is non-deterministic in general case.
+
+    Examples
+    --------
     >>> df.withColumn('randn', randn(seed=42)).collect()
-    [Row(age=2, name=u'Alice', randn=1.1027054481455365),
-    Row(age=5, name=u'Bob', randn=0.7400395449950132)]
+    [Row(age=2, name='Alice', randn=1.1027054481455365),
+    Row(age=5, name='Bob', randn=0.7400395449950132)]
     """
     sc = SparkContext._active_spark_context
     if seed is not None:
@@ -688,12 +1283,15 @@ def randn(seed=None):
     return Column(jc)
 
 
-@since(1.5)
 def round(col, scale=0):
     """
     Round the given value to `scale` decimal places using HALF_UP rounding mode if `scale` >= 0
     or at integral part when `scale` < 0.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> spark.createDataFrame([(2.5,)], ['a']).select(round('a', 0).alias('r')).collect()
     [Row(r=3.0)]
     """
@@ -701,12 +1299,15 @@ def round(col, scale=0):
     return Column(sc._jvm.functions.round(_to_java_column(col), scale))
 
 
-@since(2.0)
 def bround(col, scale=0):
     """
     Round the given value to `scale` decimal places using HALF_EVEN rounding mode if `scale` >= 0
     or at integral part when `scale` < 0.
 
+    .. versionadded:: 2.0.0
+
+    Examples
+    --------
     >>> spark.createDataFrame([(2.5,)], ['a']).select(bround('a', 0).alias('r')).collect()
     [Row(r=2.0)]
     """
@@ -714,22 +1315,52 @@ def bround(col, scale=0):
     return Column(sc._jvm.functions.bround(_to_java_column(col), scale))
 
 
-@since(1.5)
 def shiftLeft(col, numBits):
     """Shift the given value numBits left.
 
-    >>> spark.createDataFrame([(21,)], ['a']).select(shiftLeft('a', 1).alias('r')).collect()
+    .. versionadded:: 1.5.0
+
+    .. deprecated:: 3.2.0
+        Use :func:`shiftleft` instead.
+    """
+    warnings.warn("Deprecated in 3.2, use shiftleft instead.", FutureWarning)
+    return shiftleft(col, numBits)
+
+
+def shiftleft(col, numBits):
+    """Shift the given value numBits left.
+
+    .. versionadded:: 3.2.0
+
+    Examples
+    --------
+    >>> spark.createDataFrame([(21,)], ['a']).select(shiftleft('a', 1).alias('r')).collect()
     [Row(r=42)]
     """
     sc = SparkContext._active_spark_context
-    return Column(sc._jvm.functions.shiftLeft(_to_java_column(col), numBits))
+    return Column(sc._jvm.functions.shiftleft(_to_java_column(col), numBits))
 
 
-@since(1.5)
 def shiftRight(col, numBits):
     """(Signed) shift the given value numBits right.
 
-    >>> spark.createDataFrame([(42,)], ['a']).select(shiftRight('a', 1).alias('r')).collect()
+    .. versionadded:: 1.5.0
+
+    .. deprecated:: 3.2.0
+        Use :func:`shiftright` instead.
+    """
+    warnings.warn("Deprecated in 3.2, use shiftright instead.", FutureWarning)
+    return shiftright(col, numBits)
+
+
+def shiftright(col, numBits):
+    """(Signed) shift the given value numBits right.
+
+    .. versionadded:: 3.2.0
+
+    Examples
+    --------
+    >>> spark.createDataFrame([(42,)], ['a']).select(shiftright('a', 1).alias('r')).collect()
     [Row(r=21)]
     """
     sc = SparkContext._active_spark_context
@@ -737,12 +1368,27 @@ def shiftRight(col, numBits):
     return Column(jc)
 
 
-@since(1.5)
 def shiftRightUnsigned(col, numBits):
     """Unsigned shift the given value numBits right.
 
+    .. versionadded:: 1.5.0
+
+    .. deprecated:: 3.2.0
+        Use :func:`shiftrightunsigned` instead.
+    """
+    warnings.warn("Deprecated in 3.2, use shiftrightunsigned instead.", FutureWarning)
+    return shiftrightunsigned(col, numBits)
+
+
+def shiftrightunsigned(col, numBits):
+    """Unsigned shift the given value numBits right.
+
+    .. versionadded:: 3.2.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([(-42,)], ['a'])
-    >>> df.select(shiftRightUnsigned('a', 1).alias('r')).collect()
+    >>> df.select(shiftrightunsigned('a', 1).alias('r')).collect()
     [Row(r=9223372036854775787)]
     """
     sc = SparkContext._active_spark_context
@@ -750,12 +1396,17 @@ def shiftRightUnsigned(col, numBits):
     return Column(jc)
 
 
-@since(1.6)
 def spark_partition_id():
     """A column for partition ID.
 
-    .. note:: This is indeterministic because it depends on data partitioning and task scheduling.
+    .. versionadded:: 1.6.0
+
+    Notes
+    -----
+    This is non deterministic because it depends on data partitioning and task scheduling.
 
+    Examples
+    --------
     >>> df.repartition(1).select(spark_partition_id().alias("pid")).collect()
     [Row(pid=0), Row(pid=0)]
     """
@@ -763,10 +1414,13 @@ def spark_partition_id():
     return Column(sc._jvm.functions.spark_partition_id())
 
 
-@since(1.5)
 def expr(str):
     """Parses the expression string into the column that it represents
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df.select(expr("length(name)")).collect()
     [Row(length(name)=5), Row(length(name)=3)]
     """
@@ -774,17 +1428,22 @@ def expr(str):
     return Column(sc._jvm.functions.expr(str))
 
 
-@ignore_unicode_prefix
-@since(1.4)
 def struct(*cols):
     """Creates a new struct column.
 
-    :param cols: list of column names (string) or list of :class:`Column` expressions
+    .. versionadded:: 1.4.0
+
+    Parameters
+    ----------
+    cols : list, set, str or :class:`~pyspark.sql.Column`
+        column names or :class:`~pyspark.sql.Column`\\s to contain in the output struct.
 
+    Examples
+    --------
     >>> df.select(struct('age', 'name').alias("struct")).collect()
-    [Row(struct=Row(age=2, name=u'Alice')), Row(struct=Row(age=5, name=u'Bob'))]
+    [Row(struct=Row(age=2, name='Alice')), Row(struct=Row(age=5, name='Bob'))]
     >>> df.select(struct([df.age, df.name]).alias("struct")).collect()
-    [Row(struct=Row(age=2, name=u'Alice')), Row(struct=Row(age=5, name=u'Bob'))]
+    [Row(struct=Row(age=2, name='Alice')), Row(struct=Row(age=5, name='Bob'))]
     """
     sc = SparkContext._active_spark_context
     if len(cols) == 1 and isinstance(cols[0], (list, set)):
@@ -793,12 +1452,15 @@ def struct(*cols):
     return Column(jc)
 
 
-@since(1.5)
 def greatest(*cols):
     """
     Returns the greatest value of the list of column names, skipping null values.
     This function takes at least 2 parameters. It will return null iff all parameters are null.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([(1, 4, 3)], ['a', 'b', 'c'])
     >>> df.select(greatest(df.a, df.b, df.c).alias("greatest")).collect()
     [Row(greatest=4)]
@@ -809,12 +1471,15 @@ def greatest(*cols):
     return Column(sc._jvm.functions.greatest(_to_seq(sc, cols, _to_java_column)))
 
 
-@since(1.5)
 def least(*cols):
     """
     Returns the least value of the list of column names, skipping null values.
     This function takes at least 2 parameters. It will return null iff all parameters are null.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([(1, 4, 3)], ['a', 'b', 'c'])
     >>> df.select(least(df.a, df.b, df.c).alias("least")).collect()
     [Row(least=1)]
@@ -825,13 +1490,19 @@ def least(*cols):
     return Column(sc._jvm.functions.least(_to_seq(sc, cols, _to_java_column)))
 
 
-@since(1.4)
 def when(condition, value):
     """Evaluates a list of conditions and returns one of multiple possible result expressions.
-    If :func:`Column.otherwise` is not invoked, None is returned for unmatched conditions.
+    If :func:`pyspark.sql.Column.otherwise` is not invoked, None is returned for unmatched
+    conditions.
+
+    .. versionadded:: 1.4.0
 
-    :param condition: a boolean :class:`Column` expression.
-    :param value: a literal value, or a :class:`Column` expression.
+    Parameters
+    ----------
+    condition : :class:`~pyspark.sql.Column`
+        a boolean :class:`~pyspark.sql.Column` expression.
+    value :
+        a literal value, or a :class:`~pyspark.sql.Column` expression.
 
     >>> df.select(when(df['age'] == 2, 3).otherwise(4).alias("age")).collect()
     [Row(age=3), Row(age=4)]
@@ -847,12 +1518,15 @@ def when(condition, value):
     return Column(jc)
 
 
-@since(1.5)
 def log(arg1, arg2=None):
     """Returns the first argument-based logarithm of the second argument.
 
     If there is only one argument, then this takes the natural logarithm of the argument.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df.select(log(10.0, df.age).alias('ten')).rdd.map(lambda l: str(l.ten)[:7]).collect()
     ['0.30102', '0.69897']
 
@@ -867,10 +1541,13 @@ def log(arg1, arg2=None):
     return Column(jc)
 
 
-@since(1.5)
 def log2(col):
     """Returns the base-2 logarithm of the argument.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> spark.createDataFrame([(4,)], ['a']).select(log2('a').alias('log2')).collect()
     [Row(log2=2.0)]
     """
@@ -878,25 +1555,30 @@ def log2(col):
     return Column(sc._jvm.functions.log2(_to_java_column(col)))
 
 
-@since(1.5)
-@ignore_unicode_prefix
 def conv(col, fromBase, toBase):
     """
     Convert a number in a string column from one base to another.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([("010101",)], ['n'])
     >>> df.select(conv(df.n, 2, 16).alias('hex')).collect()
-    [Row(hex=u'15')]
+    [Row(hex='15')]
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.conv(_to_java_column(col), fromBase, toBase))
 
 
-@since(1.5)
 def factorial(col):
     """
     Computes the factorial of the given value.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([(5,)], ['n'])
     >>> df.select(factorial(df.n).alias('f')).collect()
     [Row(f=120)]
@@ -907,41 +1589,78 @@ def factorial(col):
 
 # ---------------  Window functions ------------------------
 
-@since(1.4)
 def lag(col, offset=1, default=None):
     """
     Window function: returns the value that is `offset` rows before the current row, and
-    `defaultValue` if there is less than `offset` rows before the current row. For example,
+    `default` if there is less than `offset` rows before the current row. For example,
     an `offset` of one will return the previous row at any given point in the window partition.
 
     This is equivalent to the LAG function in SQL.
 
-    :param col: name of column or expression
-    :param offset: number of row to extend
-    :param default: default value
+    .. versionadded:: 1.4.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column or expression
+    offset : int, optional
+        number of row to extend
+    default : optional
+        default value
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.lag(_to_java_column(col), offset, default))
 
 
-@since(1.4)
 def lead(col, offset=1, default=None):
     """
     Window function: returns the value that is `offset` rows after the current row, and
-    `defaultValue` if there is less than `offset` rows after the current row. For example,
+    `default` if there is less than `offset` rows after the current row. For example,
     an `offset` of one will return the next row at any given point in the window partition.
 
     This is equivalent to the LEAD function in SQL.
 
-    :param col: name of column or expression
-    :param offset: number of row to extend
-    :param default: default value
+    .. versionadded:: 1.4.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column or expression
+    offset : int, optional
+        number of row to extend
+    default : optional
+        default value
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.lead(_to_java_column(col), offset, default))
 
 
-@since(1.4)
+def nth_value(col, offset, ignoreNulls=False):
+    """
+    Window function: returns the value that is the `offset`\\th row of the window frame
+    (counting from 1), and `null` if the size of window frame is less than `offset` rows.
+
+    It will return the `offset`\\th non-null value it sees when `ignoreNulls` is set to
+    true. If all values are null, then null is returned.
+
+    This is equivalent to the nth_value function in SQL.
+
+    .. versionadded:: 3.1.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column or expression
+    offset : int, optional
+        number of row to use as the value
+    ignoreNulls : bool, optional
+        indicates the Nth value should skip null in the
+        determination of which row to use
+    """
+    sc = SparkContext._active_spark_context
+    return Column(sc._jvm.functions.nth_value(_to_java_column(col), offset, ignoreNulls))
+
+
 def ntile(n):
     """
     Window function: returns the ntile group id (from 1 to `n` inclusive)
@@ -951,7 +1670,12 @@ def ntile(n):
 
     This is equivalent to the NTILE function in SQL.
 
-    :param n: an integer
+    .. versionadded:: 1.4.0
+
+    Parameters
+    ----------
+    n : int
+        an integer
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.ntile(int(n)))
@@ -962,7 +1686,8 @@ def ntile(n):
 @since(1.5)
 def current_date():
     """
-    Returns the current date as a :class:`DateType` column.
+    Returns the current date at the start of query evaluation as a :class:`DateType` column.
+    All calls of current_date within the same query return the same value.
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.current_date())
@@ -970,14 +1695,13 @@ def current_date():
 
 def current_timestamp():
     """
-    Returns the current timestamp as a :class:`TimestampType` column.
+    Returns the current timestamp at the start of query evaluation as a :class:`TimestampType`
+    column. All calls of current_timestamp within the same query return the same value.
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.current_timestamp())
 
 
-@ignore_unicode_prefix
-@since(1.5)
 def date_format(date, format):
     """
     Converts a date/timestamp/string to a value of string in the format specified by the date
@@ -987,22 +1711,31 @@ def date_format(date, format):
     pattern letters of `datetime pattern`_. can be used.
 
     .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
-    .. note:: Use when ever possible specialized functions like `year`. These benefit from a
-        specialized implementation.
 
+    .. versionadded:: 1.5.0
+
+    Notes
+    -----
+    Whenever possible, use specialized functions like `year`.
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])
     >>> df.select(date_format('dt', 'MM/dd/yyy').alias('date')).collect()
-    [Row(date=u'04/08/2015')]
+    [Row(date='04/08/2015')]
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.date_format(_to_java_column(date), format))
 
 
-@since(1.5)
 def year(col):
     """
     Extract the year of a given date as integer.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])
     >>> df.select(year('dt').alias('year')).collect()
     [Row(year=2015)]
@@ -1011,11 +1744,14 @@ def year(col):
     return Column(sc._jvm.functions.year(_to_java_column(col)))
 
 
-@since(1.5)
 def quarter(col):
     """
     Extract the quarter of a given date as integer.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])
     >>> df.select(quarter('dt').alias('quarter')).collect()
     [Row(quarter=2)]
@@ -1024,11 +1760,14 @@ def quarter(col):
     return Column(sc._jvm.functions.quarter(_to_java_column(col)))
 
 
-@since(1.5)
 def month(col):
     """
     Extract the month of a given date as integer.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])
     >>> df.select(month('dt').alias('month')).collect()
     [Row(month=4)]
@@ -1037,11 +1776,14 @@ def month(col):
     return Column(sc._jvm.functions.month(_to_java_column(col)))
 
 
-@since(2.3)
 def dayofweek(col):
     """
     Extract the day of the week of a given date as integer.
 
+    .. versionadded:: 2.3.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])
     >>> df.select(dayofweek('dt').alias('day')).collect()
     [Row(day=4)]
@@ -1050,11 +1792,14 @@ def dayofweek(col):
     return Column(sc._jvm.functions.dayofweek(_to_java_column(col)))
 
 
-@since(1.5)
 def dayofmonth(col):
     """
     Extract the day of the month of a given date as integer.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])
     >>> df.select(dayofmonth('dt').alias('day')).collect()
     [Row(day=8)]
@@ -1063,11 +1808,14 @@ def dayofmonth(col):
     return Column(sc._jvm.functions.dayofmonth(_to_java_column(col)))
 
 
-@since(1.5)
 def dayofyear(col):
     """
     Extract the day of the year of a given date as integer.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])
     >>> df.select(dayofyear('dt').alias('day')).collect()
     [Row(day=98)]
@@ -1076,11 +1824,14 @@ def dayofyear(col):
     return Column(sc._jvm.functions.dayofyear(_to_java_column(col)))
 
 
-@since(1.5)
 def hour(col):
     """
     Extract the hours of a given date as integer.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('2015-04-08 13:08:15',)], ['ts'])
     >>> df.select(hour('ts').alias('hour')).collect()
     [Row(hour=13)]
@@ -1089,11 +1840,14 @@ def hour(col):
     return Column(sc._jvm.functions.hour(_to_java_column(col)))
 
 
-@since(1.5)
 def minute(col):
     """
     Extract the minutes of a given date as integer.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('2015-04-08 13:08:15',)], ['ts'])
     >>> df.select(minute('ts').alias('minute')).collect()
     [Row(minute=8)]
@@ -1102,11 +1856,14 @@ def minute(col):
     return Column(sc._jvm.functions.minute(_to_java_column(col)))
 
 
-@since(1.5)
 def second(col):
     """
     Extract the seconds of a given date as integer.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('2015-04-08 13:08:15',)], ['ts'])
     >>> df.select(second('ts').alias('second')).collect()
     [Row(second=15)]
@@ -1115,11 +1872,14 @@ def second(col):
     return Column(sc._jvm.functions.second(_to_java_column(col)))
 
 
-@since(1.5)
 def weekofyear(col):
     """
     Extract the week number of a given date as integer.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])
     >>> df.select(weekofyear(df.dt).alias('week')).collect()
     [Row(week=15)]
@@ -1128,11 +1888,14 @@ def weekofyear(col):
     return Column(sc._jvm.functions.weekofyear(_to_java_column(col)))
 
 
-@since(1.5)
 def date_add(start, days):
     """
     Returns the date that is `days` days after `start`
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])
     >>> df.select(date_add(df.dt, 1).alias('next_date')).collect()
     [Row(next_date=datetime.date(2015, 4, 9))]
@@ -1141,11 +1904,14 @@ def date_add(start, days):
     return Column(sc._jvm.functions.date_add(_to_java_column(start), days))
 
 
-@since(1.5)
 def date_sub(start, days):
     """
     Returns the date that is `days` days before `start`
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])
     >>> df.select(date_sub(df.dt, 1).alias('prev_date')).collect()
     [Row(prev_date=datetime.date(2015, 4, 7))]
@@ -1154,11 +1920,14 @@ def date_sub(start, days):
     return Column(sc._jvm.functions.date_sub(_to_java_column(start), days))
 
 
-@since(1.5)
 def datediff(end, start):
     """
     Returns the number of days from `start` to `end`.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('2015-04-08','2015-05-10')], ['d1', 'd2'])
     >>> df.select(datediff(df.d2, df.d1).alias('diff')).collect()
     [Row(diff=32)]
@@ -1167,11 +1936,14 @@ def datediff(end, start):
     return Column(sc._jvm.functions.datediff(_to_java_column(end), _to_java_column(start)))
 
 
-@since(1.5)
 def add_months(start, months):
     """
     Returns the date that is `months` months after `start`
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])
     >>> df.select(add_months(df.dt, 1).alias('next_month')).collect()
     [Row(next_month=datetime.date(2015, 5, 8))]
@@ -1180,7 +1952,6 @@ def add_months(start, months):
     return Column(sc._jvm.functions.add_months(_to_java_column(start), months))
 
 
-@since(1.5)
 def months_between(date1, date2, roundOff=True):
     """
     Returns number of months between dates date1 and date2.
@@ -1189,6 +1960,10 @@ def months_between(date1, date2, roundOff=True):
     returns an integer (time of day will be ignored).
     The result is rounded off to 8 digits unless `roundOff` is set to `False`.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('1997-02-28 10:30:00', '1996-10-30')], ['date1', 'date2'])
     >>> df.select(months_between(df.date1, df.date2).alias('months')).collect()
     [Row(months=3.94959677)]
@@ -1200,13 +1975,18 @@ def months_between(date1, date2, roundOff=True):
         _to_java_column(date1), _to_java_column(date2), roundOff))
 
 
-@since(2.2)
 def to_date(col, format=None):
-    """Converts a :class:`Column` into :class:`pyspark.sql.types.DateType`
+    """Converts a :class:`~pyspark.sql.Column` into :class:`pyspark.sql.types.DateType`
     using the optionally specified format. Specify formats according to `datetime pattern`_.
     By default, it follows casting rules to :class:`pyspark.sql.types.DateType` if the format
     is omitted. Equivalent to ``col.cast("date")``.
 
+    .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+    .. versionadded:: 2.2.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('1997-02-28 10:30:00',)], ['t'])
     >>> df.select(to_date(df.t).alias('date')).collect()
     [Row(date=datetime.date(1997, 2, 28))]
@@ -1223,13 +2003,18 @@ def to_date(col, format=None):
     return Column(jc)
 
 
-@since(2.2)
 def to_timestamp(col, format=None):
-    """Converts a :class:`Column` into :class:`pyspark.sql.types.TimestampType`
+    """Converts a :class:`~pyspark.sql.Column` into :class:`pyspark.sql.types.TimestampType`
     using the optionally specified format. Specify formats according to `datetime pattern`_.
     By default, it follows casting rules to :class:`pyspark.sql.types.TimestampType` if the format
     is omitted. Equivalent to ``col.cast("timestamp")``.
 
+    .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+    .. versionadded:: 2.2.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('1997-02-28 10:30:00',)], ['t'])
     >>> df.select(to_timestamp(df.t).alias('dt')).collect()
     [Row(dt=datetime.datetime(1997, 2, 28, 10, 30))]
@@ -1246,13 +2031,20 @@ def to_timestamp(col, format=None):
     return Column(jc)
 
 
-@since(1.5)
 def trunc(date, format):
     """
     Returns date truncated to the unit specified by the format.
 
-    :param format: 'year', 'yyyy', 'yy' or 'month', 'mon', 'mm'
+    .. versionadded:: 1.5.0
 
+    Parameters
+    ----------
+    date : :class:`~pyspark.sql.Column` or str
+    format : str
+        'year', 'yyyy', 'yy' or 'month', 'mon', 'mm'
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('1997-02-28',)], ['d'])
     >>> df.select(trunc(df.d, 'year').alias('year')).collect()
     [Row(year=datetime.date(1997, 1, 1))]
@@ -1263,14 +2055,21 @@ def trunc(date, format):
     return Column(sc._jvm.functions.trunc(_to_java_column(date), format))
 
 
-@since(2.3)
 def date_trunc(format, timestamp):
     """
     Returns timestamp truncated to the unit specified by the format.
 
-    :param format: 'year', 'yyyy', 'yy', 'month', 'mon', 'mm',
+    .. versionadded:: 2.3.0
+
+    Parameters
+    ----------
+    format : str
+        'year', 'yyyy', 'yy', 'month', 'mon', 'mm',
         'day', 'dd', 'hour', 'minute', 'second', 'week', 'quarter'
+    timestamp : :class:`~pyspark.sql.Column` or str
 
+    Examples
+    --------
     >>> df = spark.createDataFrame([('1997-02-28 05:02:11',)], ['t'])
     >>> df.select(date_trunc('year', df.t).alias('year')).collect()
     [Row(year=datetime.datetime(1997, 1, 1, 0, 0))]
@@ -1281,7 +2080,6 @@ def date_trunc(format, timestamp):
     return Column(sc._jvm.functions.date_trunc(format, _to_java_column(timestamp)))
 
 
-@since(1.5)
 def next_day(date, dayOfWeek):
     """
     Returns the first date which is later than the value of the date column.
@@ -1289,6 +2087,10 @@ def next_day(date, dayOfWeek):
     Day of the week parameter is case insensitive, and accepts:
         "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun".
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('2015-07-27',)], ['d'])
     >>> df.select(next_day(df.d, 'Sun').alias('date')).collect()
     [Row(date=datetime.date(2015, 8, 2))]
@@ -1297,11 +2099,14 @@ def next_day(date, dayOfWeek):
     return Column(sc._jvm.functions.next_day(_to_java_column(date), dayOfWeek))
 
 
-@since(1.5)
 def last_day(date):
     """
     Returns the last day of the month which the given date belongs to.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('1997-02-10',)], ['d'])
     >>> df.select(last_day(df.d).alias('date')).collect()
     [Row(date=datetime.date(1997, 2, 28))]
@@ -1310,25 +2115,26 @@ def last_day(date):
     return Column(sc._jvm.functions.last_day(_to_java_column(date)))
 
 
-@ignore_unicode_prefix
-@since(1.5)
 def from_unixtime(timestamp, format="yyyy-MM-dd HH:mm:ss"):
     """
     Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC) to a string
     representing the timestamp of that moment in the current system time zone in the given
     format.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles")
     >>> time_df = spark.createDataFrame([(1428476400,)], ['unix_time'])
     >>> time_df.select(from_unixtime('unix_time').alias('ts')).collect()
-    [Row(ts=u'2015-04-08 00:00:00')]
+    [Row(ts='2015-04-08 00:00:00')]
     >>> spark.conf.unset("spark.sql.session.timeZone")
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.from_unixtime(_to_java_column(timestamp), format))
 
 
-@since(1.5)
 def unix_timestamp(timestamp=None, format='yyyy-MM-dd HH:mm:ss'):
     """
     Convert time string with given pattern ('yyyy-MM-dd HH:mm:ss', by default)
@@ -1337,6 +2143,10 @@ def unix_timestamp(timestamp=None, format='yyyy-MM-dd HH:mm:ss'):
 
     if `timestamp` is None, then it returns current timestamp.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles")
     >>> time_df = spark.createDataFrame([('2015-04-08',)], ['dt'])
     >>> time_df.select(unix_timestamp('dt', 'yyyy-MM-dd').alias('unix_time')).collect()
@@ -1349,7 +2159,6 @@ def unix_timestamp(timestamp=None, format='yyyy-MM-dd HH:mm:ss'):
     return Column(sc._jvm.functions.unix_timestamp(_to_java_column(timestamp), format))
 
 
-@since(1.5)
 def from_utc_timestamp(timestamp, tz):
     """
     This is a common function for databases supporting TIMESTAMP WITHOUT TIMEZONE. This function
@@ -1365,17 +2174,25 @@ def from_utc_timestamp(timestamp, tz):
     according to the timezone in the string, and finally display the result by converting the
     timestamp to string according to the session local timezone.
 
-    :param timestamp: the column that contains timestamps
-    :param tz: A string detailing the time zone ID that the input should be adjusted to. It should
-               be in the format of either region-based zone IDs or zone offsets. Region IDs must
-               have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in
-               the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are
-               supported as aliases of '+00:00'. Other short names are not recommended to use
-               because they can be ambiguous.
-
-    .. versionchanged:: 2.4
-       `tz` can take a :class:`Column` containing timezone ID strings.
-
+    .. versionadded:: 1.5.0
+
+    Parameters
+    ----------
+    timestamp : :class:`~pyspark.sql.Column` or str
+        the column that contains timestamps
+    tz : :class:`~pyspark.sql.Column` or str
+        A string detailing the time zone ID that the input should be adjusted to. It should
+        be in the format of either region-based zone IDs or zone offsets. Region IDs must
+        have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in
+        the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are
+        supported as aliases of '+00:00'. Other short names are not recommended to use
+        because they can be ambiguous.
+
+        .. versionchanged:: 2.4
+           `tz` can take a :class:`~pyspark.sql.Column` containing timezone ID strings.
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('1997-02-28 10:30:00', 'JST')], ['ts', 'tz'])
     >>> df.select(from_utc_timestamp(df.ts, "PST").alias('local_time')).collect()
     [Row(local_time=datetime.datetime(1997, 2, 28, 2, 30))]
@@ -1388,7 +2205,6 @@ def from_utc_timestamp(timestamp, tz):
     return Column(sc._jvm.functions.from_utc_timestamp(_to_java_column(timestamp), tz))
 
 
-@since(1.5)
 def to_utc_timestamp(timestamp, tz):
     """
     This is a common function for databases supporting TIMESTAMP WITHOUT TIMEZONE. This function
@@ -1404,17 +2220,25 @@ def to_utc_timestamp(timestamp, tz):
     according to the timezone in the string, and finally display the result by converting the
     timestamp to string according to the session local timezone.
 
-    :param timestamp: the column that contains timestamps
-    :param tz: A string detailing the time zone ID that the input should be adjusted to. It should
-               be in the format of either region-based zone IDs or zone offsets. Region IDs must
-               have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in
-               the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are
-               supported as aliases of '+00:00'. Other short names are not recommended to use
-               because they can be ambiguous.
-
-    .. versionchanged:: 2.4
-       `tz` can take a :class:`Column` containing timezone ID strings.
-
+    .. versionadded:: 1.5.0
+
+    Parameters
+    ----------
+    timestamp : :class:`~pyspark.sql.Column` or str
+        the column that contains timestamps
+    tz : :class:`~pyspark.sql.Column` or str
+        A string detailing the time zone ID that the input should be adjusted to. It should
+        be in the format of either region-based zone IDs or zone offsets. Region IDs must
+        have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in
+        the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are
+        upported as aliases of '+00:00'. Other short names are not recommended to use
+        because they can be ambiguous.
+
+        .. versionchanged:: 2.4.0
+           `tz` can take a :class:`~pyspark.sql.Column` containing timezone ID strings.
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('1997-02-28 10:30:00', 'JST')], ['ts', 'tz'])
     >>> df.select(to_utc_timestamp(df.ts, "PST").alias('utc_time')).collect()
     [Row(utc_time=datetime.datetime(1997, 2, 28, 18, 30))]
@@ -1427,8 +2251,28 @@ def to_utc_timestamp(timestamp, tz):
     return Column(sc._jvm.functions.to_utc_timestamp(_to_java_column(timestamp), tz))
 
 
-@since(2.0)
-@ignore_unicode_prefix
+def timestamp_seconds(col):
+    """
+    .. versionadded:: 3.1.0
+
+    Examples
+    --------
+    >>> from pyspark.sql.functions import timestamp_seconds
+    >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles")
+    >>> time_df = spark.createDataFrame([(1230219000,)], ['unix_time'])
+    >>> time_df.select(timestamp_seconds(time_df.unix_time).alias('ts')).show()
+    +-------------------+
+    |                 ts|
+    +-------------------+
+    |2008-12-25 07:30:00|
+    +-------------------+
+    >>> spark.conf.unset("spark.sql.session.timeZone")
+    """
+
+    sc = SparkContext._active_spark_context
+    return Column(sc._jvm.functions.timestamp_seconds(_to_java_column(col)))
+
+
 def window(timeColumn, windowDuration, slideDuration=None, startTime=None):
     """Bucketize rows into one or more time windows given a timestamp specifying column. Window
     starts are inclusive but the window ends are exclusive, e.g. 12:05 will be in the window
@@ -1448,11 +2292,15 @@ def window(timeColumn, windowDuration, slideDuration=None, startTime=None):
     The output column will be a struct called 'window' by default with the nested columns 'start'
     and 'end', where 'start' and 'end' will be of :class:`pyspark.sql.types.TimestampType`.
 
+    .. versionadded:: 2.0.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([("2016-03-11 09:00:07", 1)]).toDF("date", "val")
     >>> w = df.groupBy(window("date", "5 seconds")).agg(sum("val").alias("sum"))
     >>> w.select(w.window.start.cast("string").alias("start"),
     ...          w.window.end.cast("string").alias("end"), "sum").collect()
-    [Row(start=u'2016-03-11 09:00:05', end=u'2016-03-11 09:00:10', sum=1)]
+    [Row(start='2016-03-11 09:00:05', end='2016-03-11 09:00:10', sum=1)]
     """
     def check_string_field(field, fieldName):
         if not field or type(field) is not str:
@@ -1478,13 +2326,15 @@ def check_string_field(field, fieldName):
 
 # ---------------------------- misc functions ----------------------------------
 
-@since(1.5)
-@ignore_unicode_prefix
 def crc32(col):
     """
     Calculates the cyclic redundancy check value  (CRC32) of a binary column and
     returns the value as a bigint.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> spark.createDataFrame([('ABC',)], ['a']).select(crc32('a').alias('crc32')).collect()
     [Row(crc32=2743272264)]
     """
@@ -1492,104 +2342,219 @@ def crc32(col):
     return Column(sc._jvm.functions.crc32(_to_java_column(col)))
 
 
-@ignore_unicode_prefix
-@since(1.5)
 def md5(col):
     """Calculates the MD5 digest and returns the value as a 32 character hex string.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> spark.createDataFrame([('ABC',)], ['a']).select(md5('a').alias('hash')).collect()
-    [Row(hash=u'902fbdd2b1df0c4f70b4a5d23525e932')]
+    [Row(hash='902fbdd2b1df0c4f70b4a5d23525e932')]
     """
     sc = SparkContext._active_spark_context
     jc = sc._jvm.functions.md5(_to_java_column(col))
     return Column(jc)
 
 
-@ignore_unicode_prefix
-@since(1.5)
 def sha1(col):
     """Returns the hex string result of SHA-1.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> spark.createDataFrame([('ABC',)], ['a']).select(sha1('a').alias('hash')).collect()
-    [Row(hash=u'3c01bdbb26f358bab27f267924aa2c9a03fcfdb8')]
+    [Row(hash='3c01bdbb26f358bab27f267924aa2c9a03fcfdb8')]
     """
     sc = SparkContext._active_spark_context
     jc = sc._jvm.functions.sha1(_to_java_column(col))
     return Column(jc)
 
 
-@ignore_unicode_prefix
-@since(1.5)
 def sha2(col, numBits):
     """Returns the hex string result of SHA-2 family of hash functions (SHA-224, SHA-256, SHA-384,
     and SHA-512). The numBits indicates the desired bit length of the result, which must have a
     value of 224, 256, 384, 512, or 0 (which is equivalent to 256).
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> digests = df.select(sha2(df.name, 256).alias('s')).collect()
     >>> digests[0]
-    Row(s=u'3bc51062973c458d5a6f2d8d64a023246354ad7e064b1e4e009ec8a0699a3043')
+    Row(s='3bc51062973c458d5a6f2d8d64a023246354ad7e064b1e4e009ec8a0699a3043')
     >>> digests[1]
-    Row(s=u'cd9fb1e148ccd8442e5aa74904cc73bf6fb54d1d54d333bd596aa9bb4bb4e961')
+    Row(s='cd9fb1e148ccd8442e5aa74904cc73bf6fb54d1d54d333bd596aa9bb4bb4e961')
     """
     sc = SparkContext._active_spark_context
     jc = sc._jvm.functions.sha2(_to_java_column(col), numBits)
     return Column(jc)
 
 
-@since(2.0)
-def hash(*cols):
-    """Calculates the hash code of given columns, and returns the result as an int column.
+def hash(*cols):
+    """Calculates the hash code of given columns, and returns the result as an int column.
+
+    .. versionadded:: 2.0.0
+
+    Examples
+    --------
+    >>> spark.createDataFrame([('ABC',)], ['a']).select(hash('a').alias('hash')).collect()
+    [Row(hash=-757602832)]
+    """
+    sc = SparkContext._active_spark_context
+    jc = sc._jvm.functions.hash(_to_seq(sc, cols, _to_java_column))
+    return Column(jc)
+
+
+def xxhash64(*cols):
+    """Calculates the hash code of given columns using the 64-bit variant of the xxHash algorithm,
+    and returns the result as a long column.
+
+    .. versionadded:: 3.0.0
+
+    Examples
+    --------
+    >>> spark.createDataFrame([('ABC',)], ['a']).select(xxhash64('a').alias('hash')).collect()
+    [Row(hash=4105715581806190027)]
+    """
+    sc = SparkContext._active_spark_context
+    jc = sc._jvm.functions.xxhash64(_to_seq(sc, cols, _to_java_column))
+    return Column(jc)
+
+
+def assert_true(col, errMsg=None):
+    """
+    Returns null if the input column is true; throws an exception with the provided error message
+    otherwise.
+
+    .. versionadded:: 3.1.0
+
+    Examples
+    --------
+    >>> df = spark.createDataFrame([(0,1)], ['a', 'b'])
+    >>> df.select(assert_true(df.a < df.b).alias('r')).collect()
+    [Row(r=None)]
+    >>> df = spark.createDataFrame([(0,1)], ['a', 'b'])
+    >>> df.select(assert_true(df.a < df.b, df.a).alias('r')).collect()
+    [Row(r=None)]
+    >>> df = spark.createDataFrame([(0,1)], ['a', 'b'])
+    >>> df.select(assert_true(df.a < df.b, 'error').alias('r')).collect()
+    [Row(r=None)]
+    """
+    sc = SparkContext._active_spark_context
+    if errMsg is None:
+        return Column(sc._jvm.functions.assert_true(_to_java_column(col)))
+    if not isinstance(errMsg, (str, Column)):
+        raise TypeError(
+            "errMsg should be a Column or a str, got {}".format(type(errMsg))
+        )
+
+    errMsg = (
+        _create_column_from_literal(errMsg)
+        if isinstance(errMsg, str)
+        else _to_java_column(errMsg)
+    )
+    return Column(sc._jvm.functions.assert_true(_to_java_column(col), errMsg))
+
+
+@since(3.1)
+def raise_error(errMsg):
+    """
+    Throws an exception with the provided error message.
+    """
+    if not isinstance(errMsg, (str, Column)):
+        raise TypeError(
+            "errMsg should be a Column or a str, got {}".format(type(errMsg))
+        )
+
+    sc = SparkContext._active_spark_context
+    errMsg = (
+        _create_column_from_literal(errMsg)
+        if isinstance(errMsg, str)
+        else _to_java_column(errMsg)
+    )
+    return Column(sc._jvm.functions.raise_error(errMsg))
+
+
+# ---------------------- String/Binary functions ------------------------------
+
+@since(1.5)
+def upper(col):
+    """
+    Converts a string expression to upper case.
+    """
+    return _invoke_function_over_column("upper", col)
+
 
-    >>> spark.createDataFrame([('ABC',)], ['a']).select(hash('a').alias('hash')).collect()
-    [Row(hash=-757602832)]
+@since(1.5)
+def lower(col):
     """
-    sc = SparkContext._active_spark_context
-    jc = sc._jvm.functions.hash(_to_seq(sc, cols, _to_java_column))
-    return Column(jc)
+    Converts a string expression to lower case.
+    """
+    return _invoke_function_over_column("lower", col)
 
 
-@since(3.0)
-def xxhash64(*cols):
-    """Calculates the hash code of given columns using the 64-bit variant of the xxHash algorithm,
-    and returns the result as a long column.
+@since(1.5)
+def ascii(col):
+    """
+    Computes the numeric value of the first character of the string column.
+    """
+    return _invoke_function_over_column("ascii", col)
 
-    >>> spark.createDataFrame([('ABC',)], ['a']).select(xxhash64('a').alias('hash')).collect()
-    [Row(hash=4105715581806190027)]
+
+@since(1.5)
+def base64(col):
     """
-    sc = SparkContext._active_spark_context
-    jc = sc._jvm.functions.xxhash64(_to_seq(sc, cols, _to_java_column))
-    return Column(jc)
+    Computes the BASE64 encoding of a binary column and returns it as a string column.
+    """
+    return _invoke_function_over_column("base64", col)
 
 
-# ---------------------- String/Binary functions ------------------------------
+@since(1.5)
+def unbase64(col):
+    """
+    Decodes a BASE64 encoded string column and returns it as a binary column.
+    """
+    return _invoke_function_over_column("unbase64", col)
+
 
-_string_functions = {
-    'upper': 'Converts a string expression to upper case.',
-    'lower': 'Converts a string expression to lower case.',
-    'ascii': 'Computes the numeric value of the first character of the string column.',
-    'base64': 'Computes the BASE64 encoding of a binary column and returns it as a string column.',
-    'unbase64': 'Decodes a BASE64 encoded string column and returns it as a binary column.',
-    'ltrim': 'Trim the spaces from left end for the specified string value.',
-    'rtrim': 'Trim the spaces from right end for the specified string value.',
-    'trim': 'Trim the spaces from both ends for the specified string column.',
-}
+@since(1.5)
+def ltrim(col):
+    """
+    Trim the spaces from left end for the specified string value.
+    """
+    return _invoke_function_over_column("ltrim", col)
 
 
-for _name, _doc in _string_functions.items():
-    globals()[_name] = since(1.5)(_create_function_over_column(_name, _doc))
-del _name, _doc
+@since(1.5)
+def rtrim(col):
+    """
+    Trim the spaces from right end for the specified string value.
+    """
+    return _invoke_function_over_column("rtrim", col)
 
 
 @since(1.5)
-@ignore_unicode_prefix
+def trim(col):
+    """
+    Trim the spaces from both ends for the specified string column.
+    """
+    return _invoke_function_over_column("trim", col)
+
+
 def concat_ws(sep, *cols):
     """
     Concatenates multiple input string columns together into a single string column,
     using the given separator.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('abcd','123')], ['s', 'd'])
     >>> df.select(concat_ws('-', df.s, df.d).alias('s')).collect()
-    [Row(s=u'abcd-123')]
+    [Row(s='abcd-123')]
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.concat_ws(sep, _to_seq(sc, cols, _to_java_column)))
@@ -1615,49 +2580,61 @@ def encode(col, charset):
     return Column(sc._jvm.functions.encode(_to_java_column(col), charset))
 
 
-@ignore_unicode_prefix
-@since(1.5)
 def format_number(col, d):
     """
     Formats the number X to a format like '#,--#,--#.--', rounded to d decimal places
     with HALF_EVEN round mode, and returns the result as a string.
 
-    :param col: the column name of the numeric value to be formatted
-    :param d: the N decimal places
+    .. versionadded:: 1.5.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        the column name of the numeric value to be formatted
+    d : int
+        the N decimal places
 
     >>> spark.createDataFrame([(5,)], ['a']).select(format_number('a', 4).alias('v')).collect()
-    [Row(v=u'5.0000')]
+    [Row(v='5.0000')]
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.format_number(_to_java_column(col), d))
 
 
-@ignore_unicode_prefix
-@since(1.5)
 def format_string(format, *cols):
     """
     Formats the arguments in printf-style and returns the result as a string column.
 
-    :param format: string that can contain embedded format tags and used as result column's value
-    :param cols: list of column names (string) or list of :class:`Column` expressions to
-        be used in formatting
+    .. versionadded:: 1.5.0
 
+    Parameters
+    ----------
+    format : str
+        string that can contain embedded format tags and used as result column's value
+    cols : :class:`~pyspark.sql.Column` or str
+        column names or :class:`~pyspark.sql.Column`\\s to be used in formatting
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([(5, "hello")], ['a', 'b'])
     >>> df.select(format_string('%d %s', df.a, df.b).alias('v')).collect()
-    [Row(v=u'5 hello')]
+    [Row(v='5 hello')]
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.format_string(format, _to_seq(sc, cols, _to_java_column)))
 
 
-@since(1.5)
 def instr(str, substr):
     """
     Locate the position of the first occurrence of substr column in the given string.
     Returns null if either of the arguments are null.
 
-    .. note:: The position is not zero based, but 1 based index. Returns 0 if substr
-        could not be found in str.
+    .. versionadded:: 1.5.0
+
+    Notes
+    -----
+    The position is not zero based, but 1 based index. Returns 0 if substr
+    could not be found in str.
 
     >>> df = spark.createDataFrame([('abcd',)], ['s',])
     >>> df.select(instr(df.s, 'b').alias('s')).collect()
@@ -1667,12 +2644,15 @@ def instr(str, substr):
     return Column(sc._jvm.functions.instr(_to_java_column(str), substr))
 
 
-@since(3.0)
 def overlay(src, replace, pos, len=-1):
     """
     Overlay the specified portion of `src` with `replace`,
     starting from byte position `pos` of `src` and proceeding for `len` bytes.
 
+    .. versionadded:: 3.0.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([("SPARK_SQL", "CORE")], ("x", "y"))
     >>> df.select(overlay("x", "y", 7).alias("overlayed")).show()
     +----------+
@@ -1701,26 +2681,28 @@ def overlay(src, replace, pos, len=-1):
     ))
 
 
-@since(1.5)
-@ignore_unicode_prefix
 def substring(str, pos, len):
     """
     Substring starts at `pos` and is of length `len` when str is String type or
     returns the slice of byte array that starts at `pos` in byte and is of length `len`
     when str is Binary type.
 
-    .. note:: The position is not zero based, but 1 based index.
+    .. versionadded:: 1.5.0
+
+    Notes
+    -----
+    The position is not zero based, but 1 based index.
 
+    Examples
+    --------
     >>> df = spark.createDataFrame([('abcd',)], ['s',])
     >>> df.select(substring(df.s, 1, 2).alias('s')).collect()
-    [Row(s=u'ab')]
+    [Row(s='ab')]
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.substring(_to_java_column(str), pos, len))
 
 
-@since(1.5)
-@ignore_unicode_prefix
 def substring_index(str, delim, count):
     """
     Returns the substring from string str before count occurrences of the delimiter delim.
@@ -1728,21 +2710,27 @@ def substring_index(str, delim, count):
     returned. If count is negative, every to the right of the final delimiter (counting from the
     right) is returned. substring_index performs a case-sensitive match when searching for delim.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('a.b.c.d',)], ['s'])
     >>> df.select(substring_index(df.s, '.', 2).alias('s')).collect()
-    [Row(s=u'a.b')]
+    [Row(s='a.b')]
     >>> df.select(substring_index(df.s, '.', -3).alias('s')).collect()
-    [Row(s=u'b.c.d')]
+    [Row(s='b.c.d')]
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.substring_index(_to_java_column(str), delim, count))
 
 
-@ignore_unicode_prefix
-@since(1.5)
 def levenshtein(left, right):
     """Computes the Levenshtein distance of the two given strings.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df0 = spark.createDataFrame([('kitten', 'sitting',)], ['l', 'r'])
     >>> df0.select(levenshtein('l', 'r').alias('d')).collect()
     [Row(d=3)]
@@ -1752,18 +2740,28 @@ def levenshtein(left, right):
     return Column(jc)
 
 
-@since(1.5)
 def locate(substr, str, pos=1):
     """
     Locate the position of the first occurrence of substr in a string column, after position pos.
 
-    .. note:: The position is not zero based, but 1 based index. Returns 0 if substr
-        could not be found in str.
+    .. versionadded:: 1.5.0
+
+    Parameters
+    ----------
+    substr : str
+        a string
+    str : :class:`~pyspark.sql.Column` or str
+        a Column of :class:`pyspark.sql.types.StringType`
+    pos : int, optional
+        start position (zero based)
 
-    :param substr: a string
-    :param str: a Column of :class:`pyspark.sql.types.StringType`
-    :param pos: start position (zero based)
+    Notes
+    -----
+    The position is not zero based, but 1 based index. Returns 0 if substr
+    could not be found in str.
 
+    Examples
+    --------
     >>> df = spark.createDataFrame([('abcd',)], ['s',])
     >>> df.select(locate('b', df.s, 1).alias('s')).collect()
     [Row(s=2)]
@@ -1772,58 +2770,69 @@ def locate(substr, str, pos=1):
     return Column(sc._jvm.functions.locate(substr, _to_java_column(str), pos))
 
 
-@since(1.5)
-@ignore_unicode_prefix
 def lpad(col, len, pad):
     """
     Left-pad the string column to width `len` with `pad`.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('abcd',)], ['s',])
     >>> df.select(lpad(df.s, 6, '#').alias('s')).collect()
-    [Row(s=u'##abcd')]
+    [Row(s='##abcd')]
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.lpad(_to_java_column(col), len, pad))
 
 
-@since(1.5)
-@ignore_unicode_prefix
 def rpad(col, len, pad):
     """
     Right-pad the string column to width `len` with `pad`.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('abcd',)], ['s',])
     >>> df.select(rpad(df.s, 6, '#').alias('s')).collect()
-    [Row(s=u'abcd##')]
+    [Row(s='abcd##')]
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.rpad(_to_java_column(col), len, pad))
 
 
-@since(1.5)
-@ignore_unicode_prefix
 def repeat(col, n):
     """
     Repeats a string column n times, and returns it as a new string column.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('ab',)], ['s',])
     >>> df.select(repeat(df.s, 3).alias('s')).collect()
-    [Row(s=u'ababab')]
+    [Row(s='ababab')]
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.repeat(_to_java_column(col), n))
 
 
-@since(1.5)
-@ignore_unicode_prefix
 def split(str, pattern, limit=-1):
     """
     Splits str around matches of the given pattern.
 
-    :param str: a string expression to split
-    :param pattern: a string representing a regular expression. The regex string should be
+    .. versionadded:: 1.5.0
+
+    Parameters
+    ----------
+    str : :class:`~pyspark.sql.Column` or str
+        a string expression to split
+    pattern : str
+        a string representing a regular expression. The regex string should be
         a Java regular expression.
-    :param limit: an integer which controls the number of times `pattern` is applied.
+    limit : int, optional
+        an integer which controls the number of times `pattern` is applied.
 
         * ``limit > 0``: The resulting array's length will not be more than `limit`, and the
                          resulting array's last entry will contain all input beyond the last
@@ -1831,114 +2840,130 @@ def split(str, pattern, limit=-1):
         * ``limit <= 0``: `pattern` will be applied as many times as possible, and the resulting
                           array can be of any size.
 
-    .. versionchanged:: 3.0
-       `split` now takes an optional `limit` field. If not provided, default limit value is -1.
+        .. versionchanged:: 3.0
+           `split` now takes an optional `limit` field. If not provided, default limit value is -1.
 
+    Examples
+    --------
     >>> df = spark.createDataFrame([('oneAtwoBthreeC',)], ['s',])
     >>> df.select(split(df.s, '[ABC]', 2).alias('s')).collect()
-    [Row(s=[u'one', u'twoBthreeC'])]
+    [Row(s=['one', 'twoBthreeC'])]
     >>> df.select(split(df.s, '[ABC]', -1).alias('s')).collect()
-    [Row(s=[u'one', u'two', u'three', u''])]
+    [Row(s=['one', 'two', 'three', ''])]
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.split(_to_java_column(str), pattern, limit))
 
 
-@ignore_unicode_prefix
-@since(1.5)
 def regexp_extract(str, pattern, idx):
     r"""Extract a specific group matched by a Java regex, from the specified string column.
     If the regex did not match, or the specified group did not match, an empty string is returned.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('100-200',)], ['str'])
     >>> df.select(regexp_extract('str', r'(\d+)-(\d+)', 1).alias('d')).collect()
-    [Row(d=u'100')]
+    [Row(d='100')]
     >>> df = spark.createDataFrame([('foo',)], ['str'])
     >>> df.select(regexp_extract('str', r'(\d+)', 1).alias('d')).collect()
-    [Row(d=u'')]
+    [Row(d='')]
     >>> df = spark.createDataFrame([('aaaac',)], ['str'])
     >>> df.select(regexp_extract('str', '(a+)(b)?(c)', 2).alias('d')).collect()
-    [Row(d=u'')]
+    [Row(d='')]
     """
     sc = SparkContext._active_spark_context
     jc = sc._jvm.functions.regexp_extract(_to_java_column(str), pattern, idx)
     return Column(jc)
 
 
-@ignore_unicode_prefix
-@since(1.5)
 def regexp_replace(str, pattern, replacement):
     r"""Replace all substrings of the specified string value that match regexp with rep.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('100-200',)], ['str'])
     >>> df.select(regexp_replace('str', r'(\d+)', '--').alias('d')).collect()
-    [Row(d=u'-----')]
+    [Row(d='-----')]
     """
     sc = SparkContext._active_spark_context
     jc = sc._jvm.functions.regexp_replace(_to_java_column(str), pattern, replacement)
     return Column(jc)
 
 
-@ignore_unicode_prefix
-@since(1.5)
 def initcap(col):
     """Translate the first letter of each word to upper case in the sentence.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> spark.createDataFrame([('ab cd',)], ['a']).select(initcap("a").alias('v')).collect()
-    [Row(v=u'Ab Cd')]
+    [Row(v='Ab Cd')]
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.initcap(_to_java_column(col)))
 
 
-@since(1.5)
-@ignore_unicode_prefix
 def soundex(col):
     """
     Returns the SoundEx encoding for a string
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([("Peters",),("Uhrbach",)], ['name'])
     >>> df.select(soundex(df.name).alias("soundex")).collect()
-    [Row(soundex=u'P362'), Row(soundex=u'U612')]
+    [Row(soundex='P362'), Row(soundex='U612')]
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.soundex(_to_java_column(col)))
 
 
-@ignore_unicode_prefix
-@since(1.5)
 def bin(col):
     """Returns the string representation of the binary value of the given column.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df.select(bin(df.age).alias('c')).collect()
-    [Row(c=u'10'), Row(c=u'101')]
+    [Row(c='10'), Row(c='101')]
     """
     sc = SparkContext._active_spark_context
     jc = sc._jvm.functions.bin(_to_java_column(col))
     return Column(jc)
 
 
-@ignore_unicode_prefix
-@since(1.5)
 def hex(col):
     """Computes hex value of the given column, which could be :class:`pyspark.sql.types.StringType`,
     :class:`pyspark.sql.types.BinaryType`, :class:`pyspark.sql.types.IntegerType` or
     :class:`pyspark.sql.types.LongType`.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> spark.createDataFrame([('ABC', 3)], ['a', 'b']).select(hex('a'), hex('b')).collect()
-    [Row(hex(a)=u'414243', hex(b)=u'3')]
+    [Row(hex(a)='414243', hex(b)='3')]
     """
     sc = SparkContext._active_spark_context
     jc = sc._jvm.functions.hex(_to_java_column(col))
     return Column(jc)
 
 
-@ignore_unicode_prefix
-@since(1.5)
 def unhex(col):
     """Inverse of hex. Interprets each pair of characters as a hexadecimal number
     and converts to the byte representation of number.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> spark.createDataFrame([('414243',)], ['a']).select(unhex('a')).collect()
     [Row(unhex(a)=bytearray(b'ABC'))]
     """
@@ -1946,13 +2971,15 @@ def unhex(col):
     return Column(sc._jvm.functions.unhex(_to_java_column(col)))
 
 
-@ignore_unicode_prefix
-@since(1.5)
 def length(col):
     """Computes the character length of string data or number of bytes of binary data.
     The length of character data includes the trailing spaces. The length of binary data
     includes binary zeros.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> spark.createDataFrame([('ABC ',)], ['a']).select(length('a').alias('length')).collect()
     [Row(length=4)]
     """
@@ -1960,17 +2987,19 @@ def length(col):
     return Column(sc._jvm.functions.length(_to_java_column(col)))
 
 
-@ignore_unicode_prefix
-@since(1.5)
 def translate(srcCol, matching, replace):
     """A function translate any character in the `srcCol` by a character in `matching`.
     The characters in `replace` is corresponding to the characters in `matching`.
     The translate will happen when any character in the string matching with the character
     in the `matching`.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> spark.createDataFrame([('translate',)], ['a']).select(translate('a', "rnlt", "123") \\
     ...     .alias('r')).collect()
-    [Row(r=u'1a2s3ae')]
+    [Row(r='1a2s3ae')]
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.translate(_to_java_column(srcCol), matching, replace))
@@ -1978,18 +3007,23 @@ def translate(srcCol, matching, replace):
 
 # ---------------------- Collection functions ------------------------------
 
-@ignore_unicode_prefix
-@since(2.0)
 def create_map(*cols):
     """Creates a new map column.
 
-    :param cols: list of column names (string) or list of :class:`Column` expressions that are
+    .. versionadded:: 2.0.0
+
+    Parameters
+    ----------
+    cols : :class:`~pyspark.sql.Column` or str
+        column names or :class:`~pyspark.sql.Column`\\s that are
         grouped as key-value pairs, e.g. (key1, value1, key2, value2, ...).
 
+    Examples
+    --------
     >>> df.select(create_map('name', 'age').alias("map")).collect()
-    [Row(map={u'Alice': 2}), Row(map={u'Bob': 5})]
+    [Row(map={'Alice': 2}), Row(map={'Bob': 5})]
     >>> df.select(create_map([df.name, df.age]).alias("map")).collect()
-    [Row(map={u'Alice': 2}), Row(map={u'Bob': 5})]
+    [Row(map={'Alice': 2}), Row(map={'Bob': 5})]
     """
     sc = SparkContext._active_spark_context
     if len(cols) == 1 and isinstance(cols[0], (list, set)):
@@ -1998,32 +3032,45 @@ def create_map(*cols):
     return Column(jc)
 
 
-@since(2.4)
 def map_from_arrays(col1, col2):
     """Creates a new map from two arrays.
 
-    :param col1: name of column containing a set of keys. All elements should not be null
-    :param col2: name of column containing a set of values
+    .. versionadded:: 2.4.0
 
+    Parameters
+    ----------
+    col1 : :class:`~pyspark.sql.Column` or str
+        name of column containing a set of keys. All elements should not be null
+    col2 : :class:`~pyspark.sql.Column` or str
+        name of column containing a set of values
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([([2, 5], ['a', 'b'])], ['k', 'v'])
     >>> df.select(map_from_arrays(df.k, df.v).alias("map")).show()
     +----------------+
     |             map|
     +----------------+
-    |[2 -> a, 5 -> b]|
+    |{2 -> a, 5 -> b}|
     +----------------+
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.map_from_arrays(_to_java_column(col1), _to_java_column(col2)))
 
 
-@since(1.4)
 def array(*cols):
     """Creates a new array column.
 
-    :param cols: list of column names (string) or list of :class:`Column` expressions that have
+    .. versionadded:: 1.4.0
+
+    Parameters
+    ----------
+    cols : :class:`~pyspark.sql.Column` or str
+        column names or :class:`~pyspark.sql.Column`\\s that have
         the same data type.
 
+    Examples
+    --------
     >>> df.select(array('age', 'age').alias("arr")).collect()
     [Row(arr=[2, 2]), Row(arr=[5, 5])]
     >>> df.select(array([df.age, df.age]).alias("arr")).collect()
@@ -2036,15 +3083,22 @@ def array(*cols):
     return Column(jc)
 
 
-@since(1.5)
 def array_contains(col, value):
     """
     Collection function: returns null if the array is null, true if the array contains the
     given value, and false otherwise.
 
-    :param col: name of column containing array
-    :param value: value or column to check for in array
+    .. versionadded:: 1.5.0
 
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column containing array
+    value :
+        value or column to check for in array
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([(["a", "b", "c"],), ([],)], ['data'])
     >>> df.select(array_contains(df.data, "a")).collect()
     [Row(array_contains(data, a)=True), Row(array_contains(data, a)=False)]
@@ -2056,13 +3110,16 @@ def array_contains(col, value):
     return Column(sc._jvm.functions.array_contains(_to_java_column(col), value))
 
 
-@since(2.4)
 def arrays_overlap(a1, a2):
     """
     Collection function: returns true if the arrays contain any common non-null element; if not,
     returns null if both the arrays are non-empty and any of them contains a null element; returns
     false otherwise.
 
+    .. versionadded:: 2.4.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([(["a", "b"], ["b", "c"]), (["a"], ["b", "c"])], ['x', 'y'])
     >>> df.select(arrays_overlap(df.x, df.y).alias("overlap")).collect()
     [Row(overlap=True), Row(overlap=False)]
@@ -2071,36 +3128,50 @@ def arrays_overlap(a1, a2):
     return Column(sc._jvm.functions.arrays_overlap(_to_java_column(a1), _to_java_column(a2)))
 
 
-@since(2.4)
 def slice(x, start, length):
     """
     Collection function: returns an array containing  all the elements in `x` from index `start`
     (array indices start at 1, or from the end if `start` is negative) with the specified `length`.
 
-    :param x: the array to be sliced
-    :param start: the starting index
-    :param length: the length of the slice
+    .. versionadded:: 2.4.0
 
+    Parameters
+    ----------
+    x : :class:`~pyspark.sql.Column` or str
+        the array to be sliced
+    start : :class:`~pyspark.sql.Column` or int
+        the starting index
+    length : :class:`~pyspark.sql.Column` or int
+        the length of the slice
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([([1, 2, 3],), ([4, 5],)], ['x'])
     >>> df.select(slice(df.x, 2, 2).alias("sliced")).collect()
     [Row(sliced=[2, 3]), Row(sliced=[5])]
     """
     sc = SparkContext._active_spark_context
-    return Column(sc._jvm.functions.slice(_to_java_column(x), start, length))
+    return Column(sc._jvm.functions.slice(
+        _to_java_column(x),
+        start._jc if isinstance(start, Column) else start,
+        length._jc if isinstance(length, Column) else length
+    ))
 
 
-@ignore_unicode_prefix
-@since(2.4)
 def array_join(col, delimiter, null_replacement=None):
     """
     Concatenates the elements of `column` using the `delimiter`. Null values are replaced with
     `null_replacement` if set, otherwise they are ignored.
 
+    .. versionadded:: 2.4.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([(["a", "b", "c"],), (["a", None],)], ['data'])
     >>> df.select(array_join(df.data, ",").alias("joined")).collect()
-    [Row(joined=u'a,b,c'), Row(joined=u'a')]
+    [Row(joined='a,b,c'), Row(joined='a')]
     >>> df.select(array_join(df.data, ",", "NULL").alias("joined")).collect()
-    [Row(joined=u'a,b,c'), Row(joined=u'a,NULL')]
+    [Row(joined='a,b,c'), Row(joined='a,NULL')]
     """
     sc = SparkContext._active_spark_context
     if null_replacement is None:
@@ -2110,16 +3181,18 @@ def array_join(col, delimiter, null_replacement=None):
             _to_java_column(col), delimiter, null_replacement))
 
 
-@since(1.5)
-@ignore_unicode_prefix
 def concat(*cols):
     """
     Concatenates multiple input columns together into a single column.
     The function works with strings, binary and compatible array columns.
 
+    .. versionadded:: 1.5.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('abcd','123')], ['s', 'd'])
     >>> df.select(concat(df.s, df.d).alias('s')).collect()
-    [Row(s=u'abcd123')]
+    [Row(s='abcd123')]
 
     >>> df = spark.createDataFrame([([1, 2], [3, 4], [5]), ([1, 2], None, [3])], ['a', 'b', 'c'])
     >>> df.select(concat(df.a, df.b, df.c).alias("arr")).collect()
@@ -2129,15 +3202,20 @@ def concat(*cols):
     return Column(sc._jvm.functions.concat(_to_seq(sc, cols, _to_java_column)))
 
 
-@since(2.4)
 def array_position(col, value):
     """
     Collection function: Locates the position of the first occurrence of the given value
     in the given array. Returns null if either of the arguments are null.
 
-    .. note:: The position is not zero based, but 1 based index. Returns 0 if the given
-        value could not be found in the array.
+    .. versionadded:: 2.4.0
+
+    Notes
+    -----
+    The position is not zero based, but 1 based index. Returns 0 if the given
+    value could not be found in the array.
 
+    Examples
+    --------
     >>> df = spark.createDataFrame([(["c", "b", "a"],), ([],)], ['data'])
     >>> df.select(array_position(df.data, "a")).collect()
     [Row(array_position(data, a)=3), Row(array_position(data, a)=0)]
@@ -2146,21 +3224,29 @@ def array_position(col, value):
     return Column(sc._jvm.functions.array_position(_to_java_column(col), value))
 
 
-@ignore_unicode_prefix
-@since(2.4)
 def element_at(col, extraction):
     """
     Collection function: Returns element of array at given index in extraction if col is array.
     Returns value for the given key in extraction if col is map.
 
-    :param col: name of column containing array or map
-    :param extraction: index to check for in array or key to check for in map
+    .. versionadded:: 2.4.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column containing array or map
+    extraction :
+        index to check for in array or key to check for in map
 
-    .. note:: The position is not zero based, but 1 based index.
+    Notes
+    -----
+    The position is not zero based, but 1 based index.
 
+    Examples
+    --------
     >>> df = spark.createDataFrame([(["a", "b", "c"],), ([],)], ['data'])
     >>> df.select(element_at(df.data, 1)).collect()
-    [Row(element_at(data, 1)=u'a'), Row(element_at(data, 1)=None)]
+    [Row(element_at(data, 1)='a'), Row(element_at(data, 1)=None)]
 
     >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0},), ({},)], ['data'])
     >>> df.select(element_at(df.data, lit("a"))).collect()
@@ -2168,17 +3254,24 @@ def element_at(col, extraction):
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.element_at(
-        _to_java_column(col), lit(extraction)._jc))  # noqa: F821 'lit' is dynamically defined.
+        _to_java_column(col), lit(extraction)._jc))
 
 
-@since(2.4)
 def array_remove(col, element):
     """
     Collection function: Remove all elements that equal to element from the given array.
 
-    :param col: name of column containing array
-    :param element: element to be removed from the array
+    .. versionadded:: 2.4.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column containing array
+    element :
+        element to be removed from the array
 
+    Examples
+    --------
     >>> df = spark.createDataFrame([([1, 2, 3, 1, 1],), ([],)], ['data'])
     >>> df.select(array_remove(df.data, 1)).collect()
     [Row(array_remove(data, 1)=[2, 3]), Row(array_remove(data, 1)=[])]
@@ -2187,13 +3280,19 @@ def array_remove(col, element):
     return Column(sc._jvm.functions.array_remove(_to_java_column(col), element))
 
 
-@since(2.4)
 def array_distinct(col):
     """
     Collection function: removes duplicate values from the array.
 
-    :param col: name of column or expression
+    .. versionadded:: 2.4.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column or expression
 
+    Examples
+    --------
     >>> df = spark.createDataFrame([([1, 2, 3, 2],), ([4, 5, 5, 4],)], ['data'])
     >>> df.select(array_distinct(df.data)).collect()
     [Row(array_distinct(data)=[1, 2, 3]), Row(array_distinct(data)=[4, 5])]
@@ -2202,70 +3301,91 @@ def array_distinct(col):
     return Column(sc._jvm.functions.array_distinct(_to_java_column(col)))
 
 
-@ignore_unicode_prefix
-@since(2.4)
 def array_intersect(col1, col2):
     """
     Collection function: returns an array of the elements in the intersection of col1 and col2,
     without duplicates.
 
-    :param col1: name of column containing array
-    :param col2: name of column containing array
+    .. versionadded:: 2.4.0
+
+    Parameters
+    ----------
+    col1 : :class:`~pyspark.sql.Column` or str
+        name of column containing array
+    col2 : :class:`~pyspark.sql.Column` or str
+        name of column containing array
 
+    Examples
+    --------
     >>> from pyspark.sql import Row
     >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["c", "d", "a", "f"])])
     >>> df.select(array_intersect(df.c1, df.c2)).collect()
-    [Row(array_intersect(c1, c2)=[u'a', u'c'])]
+    [Row(array_intersect(c1, c2)=['a', 'c'])]
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.array_intersect(_to_java_column(col1), _to_java_column(col2)))
 
 
-@ignore_unicode_prefix
-@since(2.4)
 def array_union(col1, col2):
     """
     Collection function: returns an array of the elements in the union of col1 and col2,
     without duplicates.
 
-    :param col1: name of column containing array
-    :param col2: name of column containing array
+    .. versionadded:: 2.4.0
+
+    Parameters
+    ----------
+    col1 : :class:`~pyspark.sql.Column` or str
+        name of column containing array
+    col2 : :class:`~pyspark.sql.Column` or str
+        name of column containing array
 
+    Examples
+    --------
     >>> from pyspark.sql import Row
     >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["c", "d", "a", "f"])])
     >>> df.select(array_union(df.c1, df.c2)).collect()
-    [Row(array_union(c1, c2)=[u'b', u'a', u'c', u'd', u'f'])]
+    [Row(array_union(c1, c2)=['b', 'a', 'c', 'd', 'f'])]
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.array_union(_to_java_column(col1), _to_java_column(col2)))
 
 
-@ignore_unicode_prefix
-@since(2.4)
 def array_except(col1, col2):
     """
     Collection function: returns an array of the elements in col1 but not in col2,
     without duplicates.
 
-    :param col1: name of column containing array
-    :param col2: name of column containing array
+    .. versionadded:: 2.4.0
+
+    Parameters
+    ----------
+    col1 : :class:`~pyspark.sql.Column` or str
+        name of column containing array
+    col2 : :class:`~pyspark.sql.Column` or str
+        name of column containing array
 
+    Examples
+    --------
     >>> from pyspark.sql import Row
     >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["c", "d", "a", "f"])])
     >>> df.select(array_except(df.c1, df.c2)).collect()
-    [Row(array_except(c1, c2)=[u'b'])]
+    [Row(array_except(c1, c2)=['b'])]
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.array_except(_to_java_column(col1), _to_java_column(col2)))
 
 
-@since(1.4)
 def explode(col):
     """
     Returns a new row for each element in the given array or map.
     Uses the default column name `col` for elements in the array and
     `key` and `value` for elements in the map unless specified otherwise.
 
+    .. versionadded:: 1.4.0
+
+    Examples
+    --------
     >>> from pyspark.sql import Row
     >>> eDF = spark.createDataFrame([Row(a=1, intlist=[1,2,3], mapfield={"a": "b"})])
     >>> eDF.select(explode(eDF.intlist).alias("anInt")).collect()
@@ -2283,13 +3403,16 @@ def explode(col):
     return Column(jc)
 
 
-@since(2.1)
 def posexplode(col):
     """
     Returns a new row for each element with position in the given array or map.
     Uses the default column name `pos` for position, and `col` for elements in the
     array and `key` and `value` for elements in the map unless specified otherwise.
 
+    .. versionadded:: 2.1.0
+
+    Examples
+    --------
     >>> from pyspark.sql import Row
     >>> eDF = spark.createDataFrame([Row(a=1, intlist=[1,2,3], mapfield={"a": "b"})])
     >>> eDF.select(posexplode(eDF.intlist)).collect()
@@ -2307,7 +3430,6 @@ def posexplode(col):
     return Column(jc)
 
 
-@since(2.3)
 def explode_outer(col):
     """
     Returns a new row for each element in the given array or map.
@@ -2315,6 +3437,10 @@ def explode_outer(col):
     Uses the default column name `col` for elements in the array and
     `key` and `value` for elements in the map unless specified otherwise.
 
+    .. versionadded:: 2.3.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame(
     ...     [(1, ["foo", "bar"], {"x": 1.0}), (2, [], {}), (3, None, None)],
     ...     ("id", "an_array", "a_map")
@@ -2332,9 +3458,9 @@ def explode_outer(col):
     +---+----------+----+
     | id|     a_map| col|
     +---+----------+----+
-    |  1|[x -> 1.0]| foo|
-    |  1|[x -> 1.0]| bar|
-    |  2|        []|null|
+    |  1|{x -> 1.0}| foo|
+    |  1|{x -> 1.0}| bar|
+    |  2|        {}|null|
     |  3|      null|null|
     +---+----------+----+
     """
@@ -2343,7 +3469,6 @@ def explode_outer(col):
     return Column(jc)
 
 
-@since(2.3)
 def posexplode_outer(col):
     """
     Returns a new row for each element with position in the given array or map.
@@ -2351,6 +3476,10 @@ def posexplode_outer(col):
     Uses the default column name `pos` for position, and `col` for elements in the
     array and `key` and `value` for elements in the map unless specified otherwise.
 
+    .. versionadded:: 2.3.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame(
     ...     [(1, ["foo", "bar"], {"x": 1.0}), (2, [], {}), (3, None, None)],
     ...     ("id", "an_array", "a_map")
@@ -2367,9 +3496,9 @@ def posexplode_outer(col):
     +---+----------+----+----+
     | id|     a_map| pos| col|
     +---+----------+----+----+
-    |  1|[x -> 1.0]|   0| foo|
-    |  1|[x -> 1.0]|   1| bar|
-    |  2|        []|null|null|
+    |  1|{x -> 1.0}|   0| foo|
+    |  1|{x -> 1.0}|   1| bar|
+    |  2|        {}|null|null|
     |  3|      null|null|null|
     +---+----------+----+----+
     """
@@ -2378,60 +3507,79 @@ def posexplode_outer(col):
     return Column(jc)
 
 
-@ignore_unicode_prefix
-@since(1.6)
 def get_json_object(col, path):
     """
     Extracts json object from a json string based on json path specified, and returns json string
     of the extracted json object. It will return null if the input json string is invalid.
 
-    :param col: string column in json format
-    :param path: path to the json object to extract
+    .. versionadded:: 1.6.0
 
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        string column in json format
+    path : str
+        path to the json object to extract
+
+    Examples
+    --------
     >>> data = [("1", '''{"f1": "value1", "f2": "value2"}'''), ("2", '''{"f1": "value12"}''')]
     >>> df = spark.createDataFrame(data, ("key", "jstring"))
     >>> df.select(df.key, get_json_object(df.jstring, '$.f1').alias("c0"), \\
     ...                   get_json_object(df.jstring, '$.f2').alias("c1") ).collect()
-    [Row(key=u'1', c0=u'value1', c1=u'value2'), Row(key=u'2', c0=u'value12', c1=None)]
+    [Row(key='1', c0='value1', c1='value2'), Row(key='2', c0='value12', c1=None)]
     """
     sc = SparkContext._active_spark_context
     jc = sc._jvm.functions.get_json_object(_to_java_column(col), path)
     return Column(jc)
 
 
-@ignore_unicode_prefix
-@since(1.6)
 def json_tuple(col, *fields):
     """Creates a new row for a json column according to the given field names.
 
-    :param col: string column in json format
-    :param fields: list of fields to extract
+    .. versionadded:: 1.6.0
 
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        string column in json format
+    fields : str
+        fields to extract
+
+    Examples
+    --------
     >>> data = [("1", '''{"f1": "value1", "f2": "value2"}'''), ("2", '''{"f1": "value12"}''')]
     >>> df = spark.createDataFrame(data, ("key", "jstring"))
     >>> df.select(df.key, json_tuple(df.jstring, 'f1', 'f2')).collect()
-    [Row(key=u'1', c0=u'value1', c1=u'value2'), Row(key=u'2', c0=u'value12', c1=None)]
+    [Row(key='1', c0='value1', c1='value2'), Row(key='2', c0='value12', c1=None)]
     """
     sc = SparkContext._active_spark_context
     jc = sc._jvm.functions.json_tuple(_to_java_column(col), _to_seq(sc, fields))
     return Column(jc)
 
 
-@ignore_unicode_prefix
-@since(2.1)
-def from_json(col, schema, options={}):
+def from_json(col, schema, options=None):
     """
     Parses a column containing a JSON string into a :class:`MapType` with :class:`StringType`
     as keys type, :class:`StructType` or :class:`ArrayType` with
     the specified schema. Returns `null`, in the case of an unparseable string.
 
-    :param col: string column in json format
-    :param schema: a StructType or ArrayType of StructType to use when parsing the json column.
-    :param options: options to control parsing. accepts the same options as the json datasource
+    .. versionadded:: 2.1.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        string column in json format
+    schema : :class:`DataType` or str
+        a StructType or ArrayType of StructType to use when parsing the json column.
 
-    .. note:: Since Spark 2.3, the DDL-formatted string or a JSON format string is also
-              supported for ``schema``.
+        .. versionchanged:: 2.3
+            the DDL-formatted string is also supported for ``schema``.
+    options : dict, optional
+        options to control parsing. accepts the same options as the json datasource
 
+    Examples
+    --------
     >>> from pyspark.sql.types import *
     >>> data = [(1, '''{"a": 1}''')]
     >>> schema = StructType([StructField("a", IntegerType())])
@@ -2441,7 +3589,7 @@ def from_json(col, schema, options={}):
     >>> df.select(from_json(df.value, "a INT").alias("json")).collect()
     [Row(json=Row(a=1))]
     >>> df.select(from_json(df.value, "MAP<STRING,INT>").alias("json")).collect()
-    [Row(json={u'a': 1})]
+    [Row(json={'a': 1})]
     >>> data = [(1, '''[{"a": 1}]''')]
     >>> schema = ArrayType(StructType([StructField("a", IntegerType())]))
     >>> df = spark.createDataFrame(data, ("key", "value"))
@@ -2466,40 +3614,46 @@ def from_json(col, schema, options={}):
     return Column(jc)
 
 
-@ignore_unicode_prefix
-@since(2.1)
-def to_json(col, options={}):
+def to_json(col, options=None):
     """
     Converts a column containing a :class:`StructType`, :class:`ArrayType` or a :class:`MapType`
     into a JSON string. Throws an exception, in the case of an unsupported type.
 
-    :param col: name of column containing a struct, an array or a map.
-    :param options: options to control converting. accepts the same options as the JSON datasource.
-                    Additionally the function supports the `pretty` option which enables
-                    pretty JSON generation.
+    .. versionadded:: 2.1.0
 
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column containing a struct, an array or a map.
+    options : dict, optional
+        options to control converting. accepts the same options as the JSON datasource.
+        Additionally the function supports the `pretty` option which enables
+        pretty JSON generation.
+
+    Examples
+    --------
     >>> from pyspark.sql import Row
     >>> from pyspark.sql.types import *
-    >>> data = [(1, Row(name='Alice', age=2))]
+    >>> data = [(1, Row(age=2, name='Alice'))]
     >>> df = spark.createDataFrame(data, ("key", "value"))
     >>> df.select(to_json(df.value).alias("json")).collect()
-    [Row(json=u'{"age":2,"name":"Alice"}')]
-    >>> data = [(1, [Row(name='Alice', age=2), Row(name='Bob', age=3)])]
+    [Row(json='{"age":2,"name":"Alice"}')]
+    >>> data = [(1, [Row(age=2, name='Alice'), Row(age=3, name='Bob')])]
     >>> df = spark.createDataFrame(data, ("key", "value"))
     >>> df.select(to_json(df.value).alias("json")).collect()
-    [Row(json=u'[{"age":2,"name":"Alice"},{"age":3,"name":"Bob"}]')]
+    [Row(json='[{"age":2,"name":"Alice"},{"age":3,"name":"Bob"}]')]
     >>> data = [(1, {"name": "Alice"})]
     >>> df = spark.createDataFrame(data, ("key", "value"))
     >>> df.select(to_json(df.value).alias("json")).collect()
-    [Row(json=u'{"name":"Alice"}')]
+    [Row(json='{"name":"Alice"}')]
     >>> data = [(1, [{"name": "Alice"}, {"name": "Bob"}])]
     >>> df = spark.createDataFrame(data, ("key", "value"))
     >>> df.select(to_json(df.value).alias("json")).collect()
-    [Row(json=u'[{"name":"Alice"},{"name":"Bob"}]')]
+    [Row(json='[{"name":"Alice"},{"name":"Bob"}]')]
     >>> data = [(1, ["Alice", "Bob"])]
     >>> df = spark.createDataFrame(data, ("key", "value"))
     >>> df.select(to_json(df.value).alias("json")).collect()
-    [Row(json=u'["Alice","Bob"]')]
+    [Row(json='["Alice","Bob"]')]
     """
 
     sc = SparkContext._active_spark_context
@@ -2507,26 +3661,32 @@ def to_json(col, options={}):
     return Column(jc)
 
 
-@ignore_unicode_prefix
-@since(2.4)
-def schema_of_json(json, options={}):
+def schema_of_json(json, options=None):
     """
     Parses a JSON string and infers its schema in DDL format.
 
-    :param json: a JSON string or a string literal containing a JSON string.
-    :param options: options to control parsing. accepts the same options as the JSON datasource
+    .. versionadded:: 2.4.0
 
-    .. versionchanged:: 3.0
-       It accepts `options` parameter to control schema inferring.
+    Parameters
+    ----------
+    json : :class:`~pyspark.sql.Column` or str
+        a JSON string or a foldable string column containing a JSON string.
+    options : dict, optional
+        options to control parsing. accepts the same options as the JSON datasource
 
+        .. versionchanged:: 3.0
+           It accepts `options` parameter to control schema inferring.
+
+    Examples
+    --------
     >>> df = spark.range(1)
     >>> df.select(schema_of_json(lit('{"a": 0}')).alias("json")).collect()
-    [Row(json=u'struct<a:bigint>')]
+    [Row(json='STRUCT<`a`: BIGINT>')]
     >>> schema = schema_of_json('{a: 1}', {'allowUnquotedFieldNames':'true'})
     >>> df.select(schema.alias("json")).collect()
-    [Row(json=u'struct<a:bigint>')]
+    [Row(json='STRUCT<`a`: BIGINT>')]
     """
-    if isinstance(json, basestring):
+    if isinstance(json, str):
         col = _create_column_from_literal(json)
     elif isinstance(json, Column):
         col = _to_java_column(json)
@@ -2538,22 +3698,28 @@ def schema_of_json(json, options={}):
     return Column(jc)
 
 
-@ignore_unicode_prefix
-@since(3.0)
-def schema_of_csv(csv, options={}):
+def schema_of_csv(csv, options=None):
     """
     Parses a CSV string and infers its schema in DDL format.
 
-    :param col: a CSV string or a string literal containing a CSV string.
-    :param options: options to control parsing. accepts the same options as the CSV datasource
+    .. versionadded:: 3.0.0
+
+    Parameters
+    ----------
+    csv : :class:`~pyspark.sql.Column` or str
+        a CSV string or a foldable string column containing a CSV string.
+    options : dict, optional
+        options to control parsing. accepts the same options as the CSV datasource
 
+    Examples
+    --------
     >>> df = spark.range(1)
     >>> df.select(schema_of_csv(lit('1|a'), {'sep':'|'}).alias("csv")).collect()
-    [Row(csv=u'struct<_c0:int,_c1:string>')]
+    [Row(csv='STRUCT<`_c0`: INT, `_c1`: STRING>')]
     >>> df.select(schema_of_csv('1|a', {'sep':'|'}).alias("csv")).collect()
-    [Row(csv=u'struct<_c0:int,_c1:string>')]
+    [Row(csv='STRUCT<`_c0`: INT, `_c1`: STRING>')]
     """
-    if isinstance(csv, basestring):
+    if isinstance(csv, str):
         col = _create_column_from_literal(csv)
     elif isinstance(csv, Column):
         col = _to_java_column(csv)
@@ -2565,21 +3731,27 @@ def schema_of_csv(csv, options={}):
     return Column(jc)
 
 
-@ignore_unicode_prefix
-@since(3.0)
-def to_csv(col, options={}):
+def to_csv(col, options=None):
     """
     Converts a column containing a :class:`StructType` into a CSV string.
     Throws an exception, in the case of an unsupported type.
 
-    :param col: name of column containing a struct.
-    :param options: options to control converting. accepts the same options as the CSV datasource.
+    .. versionadded:: 3.0.0
 
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column containing a struct.
+    options: dict, optional
+        options to control converting. accepts the same options as the CSV datasource.
+
+    Examples
+    --------
     >>> from pyspark.sql import Row
-    >>> data = [(1, Row(name='Alice', age=2))]
+    >>> data = [(1, Row(age=2, name='Alice'))]
     >>> df = spark.createDataFrame(data, ("key", "value"))
     >>> df.select(to_csv(df.value).alias("csv")).collect()
-    [Row(csv=u'2,Alice')]
+    [Row(csv='2,Alice')]
     """
 
     sc = SparkContext._active_spark_context
@@ -2587,13 +3759,19 @@ def to_csv(col, options={}):
     return Column(jc)
 
 
-@since(1.5)
 def size(col):
     """
     Collection function: returns the length of the array or map stored in the column.
 
-    :param col: name of column or expression
+    .. versionadded:: 1.5.0
 
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column or expression
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([([1, 2, 3],),([1],),([],)], ['data'])
     >>> df.select(size(df.data)).collect()
     [Row(size(data)=3), Row(size(data)=1), Row(size(data)=0)]
@@ -2602,13 +3780,19 @@ def size(col):
     return Column(sc._jvm.functions.size(_to_java_column(col)))
 
 
-@since(2.4)
 def array_min(col):
     """
     Collection function: returns the minimum value of the array.
 
-    :param col: name of column or expression
+    .. versionadded:: 2.4.0
 
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column or expression
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([([2, 1, 3],), ([None, 10, -1],)], ['data'])
     >>> df.select(array_min(df.data).alias('min')).collect()
     [Row(min=1), Row(min=-1)]
@@ -2617,13 +3801,19 @@ def array_min(col):
     return Column(sc._jvm.functions.array_min(_to_java_column(col)))
 
 
-@since(2.4)
 def array_max(col):
     """
     Collection function: returns the maximum value of the array.
 
-    :param col: name of column or expression
+    .. versionadded:: 2.4.0
 
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column or expression
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([([2, 1, 3],), ([None, 10, -1],)], ['data'])
     >>> df.select(array_max(df.data).alias('max')).collect()
     [Row(max=3), Row(max=10)]
@@ -2632,7 +3822,6 @@ def array_max(col):
     return Column(sc._jvm.functions.array_max(_to_java_column(col)))
 
 
-@since(1.5)
 def sort_array(col, asc=True):
     """
     Collection function: sorts the input array in ascending or descending order according
@@ -2640,8 +3829,16 @@ def sort_array(col, asc=True):
     of the returned array in ascending order or at the end of the returned array in descending
     order.
 
-    :param col: name of column or expression
+    .. versionadded:: 1.5.0
 
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column or expression
+    asc : bool, optional
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([([2, 1, None, 3],),([1],),([],)], ['data'])
     >>> df.select(sort_array(df.data).alias('r')).collect()
     [Row(r=[None, 1, 2, 3]), Row(r=[1]), Row(r=[])]
@@ -2652,14 +3849,20 @@ def sort_array(col, asc=True):
     return Column(sc._jvm.functions.sort_array(_to_java_column(col), asc))
 
 
-@since(2.4)
 def array_sort(col):
     """
     Collection function: sorts the input array in ascending order. The elements of the input array
     must be orderable. Null elements will be placed at the end of the returned array.
 
-    :param col: name of column or expression
+    .. versionadded:: 2.4.0
 
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column or expression
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([([2, 1, None, 3],),([1],),([],)], ['data'])
     >>> df.select(array_sort(df.data).alias('r')).collect()
     [Row(r=[1, 2, 3, None]), Row(r=[1]), Row(r=[])]
@@ -2668,15 +3871,23 @@ def array_sort(col):
     return Column(sc._jvm.functions.array_sort(_to_java_column(col)))
 
 
-@since(2.4)
 def shuffle(col):
     """
     Collection function: Generates a random permutation of the given array.
 
-    .. note:: The function is non-deterministic.
+    .. versionadded:: 2.4.0
 
-    :param col: name of column or expression
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column or expression
 
+    Notes
+    -----
+    The function is non-deterministic.
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([([1, 20, 3, 5],), ([1, 20, None, 3],)], ['data'])
     >>> df.select(shuffle(df.data).alias('s')).collect()  # doctest: +SKIP
     [Row(s=[3, 1, 5, 20]), Row(s=[20, None, 3, 1])]
@@ -2685,17 +3896,22 @@ def shuffle(col):
     return Column(sc._jvm.functions.shuffle(_to_java_column(col)))
 
 
-@since(1.5)
-@ignore_unicode_prefix
 def reverse(col):
     """
     Collection function: returns a reversed string or an array with reverse order of elements.
 
-    :param col: name of column or expression
+    .. versionadded:: 1.5.0
 
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column or expression
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('Spark SQL',)], ['data'])
     >>> df.select(reverse(df.data).alias('s')).collect()
-    [Row(s=u'LQS krapS')]
+    [Row(s='LQS krapS')]
     >>> df = spark.createDataFrame([([2, 1, 3],) ,([1],) ,([],)], ['data'])
     >>> df.select(reverse(df.data).alias('r')).collect()
     [Row(r=[3, 1, 2]), Row(r=[1]), Row(r=[])]
@@ -2704,15 +3920,21 @@ def reverse(col):
     return Column(sc._jvm.functions.reverse(_to_java_column(col)))
 
 
-@since(2.4)
 def flatten(col):
     """
     Collection function: creates a single array from an array of arrays.
     If a structure of nested arrays is deeper than two levels,
     only one level of nesting is removed.
 
-    :param col: name of column or expression
+    .. versionadded:: 2.4.0
 
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column or expression
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([([[1, 2, 3], [4, 5], [6]],), ([None, [4, 5]],)], ['data'])
     >>> df.select(flatten(df.data).alias('r')).collect()
     [Row(r=[1, 2, 3, 4, 5, 6]), Row(r=None)]
@@ -2721,13 +3943,19 @@ def flatten(col):
     return Column(sc._jvm.functions.flatten(_to_java_column(col)))
 
 
-@since(2.3)
 def map_keys(col):
     """
     Collection function: Returns an unordered array containing the keys of the map.
 
-    :param col: name of column or expression
+    .. versionadded:: 2.3.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column or expression
 
+    Examples
+    --------
     >>> from pyspark.sql.functions import map_keys
     >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data")
     >>> df.select(map_keys("data").alias("keys")).show()
@@ -2741,13 +3969,19 @@ def map_keys(col):
     return Column(sc._jvm.functions.map_keys(_to_java_column(col)))
 
 
-@since(2.3)
 def map_values(col):
     """
     Collection function: Returns an unordered array containing the values of the map.
 
-    :param col: name of column or expression
+    .. versionadded:: 2.3.0
 
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column or expression
+
+    Examples
+    --------
     >>> from pyspark.sql.functions import map_values
     >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data")
     >>> df.select(map_values("data").alias("values")).show()
@@ -2761,55 +3995,69 @@ def map_values(col):
     return Column(sc._jvm.functions.map_values(_to_java_column(col)))
 
 
-@since(3.0)
 def map_entries(col):
     """
     Collection function: Returns an unordered array of all entries in the given map.
 
-    :param col: name of column or expression
+    .. versionadded:: 3.0.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column or expression
 
+    Examples
+    --------
     >>> from pyspark.sql.functions import map_entries
     >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data")
     >>> df.select(map_entries("data").alias("entries")).show()
     +----------------+
     |         entries|
     +----------------+
-    |[[1, a], [2, b]]|
+    |[{1, a}, {2, b}]|
     +----------------+
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.map_entries(_to_java_column(col)))
 
 
-@since(2.4)
 def map_from_entries(col):
     """
     Collection function: Returns a map created from the given array of entries.
 
-    :param col: name of column or expression
+    .. versionadded:: 2.4.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column or expression
 
+    Examples
+    --------
     >>> from pyspark.sql.functions import map_from_entries
     >>> df = spark.sql("SELECT array(struct(1, 'a'), struct(2, 'b')) as data")
     >>> df.select(map_from_entries("data").alias("map")).show()
     +----------------+
     |             map|
     +----------------+
-    |[1 -> a, 2 -> b]|
+    |{1 -> a, 2 -> b}|
     +----------------+
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.map_from_entries(_to_java_column(col)))
 
 
-@ignore_unicode_prefix
-@since(2.4)
 def array_repeat(col, count):
     """
     Collection function: creates an array containing a column repeated count times.
 
+    .. versionadded:: 2.4.0
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([('ab',)], ['data'])
     >>> df.select(array_repeat(df.data, 3).alias('r')).collect()
-    [Row(r=[u'ab', u'ab', u'ab'])]
+    [Row(r=['ab', 'ab', 'ab'])]
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.array_repeat(
@@ -2818,14 +4066,20 @@ def array_repeat(col, count):
     ))
 
 
-@since(2.4)
 def arrays_zip(*cols):
     """
     Collection function: Returns a merged array of structs in which the N-th struct contains all
     N-th values of input arrays.
 
-    :param cols: columns of arrays to be merged.
+    .. versionadded:: 2.4.0
+
+    Parameters
+    ----------
+    cols : :class:`~pyspark.sql.Column` or str
+        columns of arrays to be merged.
 
+    Examples
+    --------
     >>> from pyspark.sql.functions import arrays_zip
     >>> df = spark.createDataFrame([(([1, 2, 3], [2, 3, 4]))], ['vals1', 'vals2'])
     >>> df.select(arrays_zip(df.vals1, df.vals2).alias('zipped')).collect()
@@ -2835,19 +4089,25 @@ def arrays_zip(*cols):
     return Column(sc._jvm.functions.arrays_zip(_to_seq(sc, cols, _to_java_column)))
 
 
-@since(2.4)
 def map_concat(*cols):
     """Returns the union of all the given maps.
 
-    :param cols: list of column names (string) or list of :class:`Column` expressions
+    .. versionadded:: 2.4.0
+
+    Parameters
+    ----------
+    cols : :class:`~pyspark.sql.Column` or str
+        column names or :class:`~pyspark.sql.Column`\\s
 
+    Examples
+    --------
     >>> from pyspark.sql.functions import map_concat
     >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as map1, map(3, 'c') as map2")
     >>> df.select(map_concat("map1", "map2").alias("map3")).show(truncate=False)
     +------------------------+
     |map3                    |
     +------------------------+
-    |[1 -> a, 2 -> b, 3 -> c]|
+    |{1 -> a, 2 -> b, 3 -> c}|
     +------------------------+
     """
     sc = SparkContext._active_spark_context
@@ -2857,13 +4117,16 @@ def map_concat(*cols):
     return Column(jc)
 
 
-@since(2.4)
 def sequence(start, stop, step=None):
     """
     Generate a sequence of integers from `start` to `stop`, incrementing by `step`.
     If `step` is not set, incrementing by 1 if `start` is less than or equal to `stop`,
     otherwise -1.
 
+    .. versionadded:: 2.4.0
+
+    Examples
+    --------
     >>> df1 = spark.createDataFrame([(-2, 2)], ('C1', 'C2'))
     >>> df1.select(sequence('C1', 'C2').alias('r')).collect()
     [Row(r=[-2, -1, 0, 1, 2])]
@@ -2879,17 +4142,24 @@ def sequence(start, stop, step=None):
             _to_java_column(start), _to_java_column(stop), _to_java_column(step)))
 
 
-@ignore_unicode_prefix
-@since(3.0)
-def from_csv(col, schema, options={}):
+def from_csv(col, schema, options=None):
     """
     Parses a column containing a CSV string to a row with the specified schema.
     Returns `null`, in the case of an unparseable string.
 
-    :param col: string column in CSV format
-    :param schema: a string with schema in DDL format to use when parsing the CSV column.
-    :param options: options to control parsing. accepts the same options as the CSV datasource
+    .. versionadded:: 3.0.0
 
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        string column in CSV format
+    schema :class:`~pyspark.sql.Column` or str
+        a string with schema in DDL format to use when parsing the CSV column.
+    options : dict, optional
+        options to control parsing. accepts the same options as the CSV datasource
+
+    Examples
+    --------
     >>> data = [("1,2,3",)]
     >>> df = spark.createDataFrame(data, ("value",))
     >>> df.select(from_csv(df.value, "a INT, b INT, c INT").alias("csv")).collect()
@@ -2901,11 +4171,11 @@ def from_csv(col, schema, options={}):
     >>> df = spark.createDataFrame(data, ("value",))
     >>> options = {'ignoreLeadingWhiteSpace': True}
     >>> df.select(from_csv(df.value, "s string", options).alias("csv")).collect()
-    [Row(csv=Row(s=u'abc'))]
+    [Row(csv=Row(s='abc'))]
     """
 
     sc = SparkContext._active_spark_context
-    if isinstance(schema, basestring):
+    if isinstance(schema, str):
         schema = _create_column_from_literal(schema)
     elif isinstance(schema, Column):
         schema = _to_java_column(schema)
@@ -2921,7 +4191,9 @@ def _unresolved_named_lambda_variable(*name_parts):
     Create `o.a.s.sql.expressions.UnresolvedNamedLambdaVariable`,
     convert it to o.s.sql.Column and wrap in Python `Column`
 
-    :param name_parts: str
+    Parameters
+    ----------
+    name_parts : str
     """
     sc = SparkContext._active_spark_context
     name_parts_seq = _to_seq(sc, name_parts)
@@ -2942,7 +4214,7 @@ def _get_lambda_parameters(f):
     # We should exclude functions that use
     # variable args and keyword argnames
     # as well as keyword only args
-    supported_parmeter_types = {
+    supported_parameter_types = {
         inspect.Parameter.POSITIONAL_OR_KEYWORD,
         inspect.Parameter.POSITIONAL_ONLY,
     }
@@ -2957,7 +4229,7 @@ def _get_lambda_parameters(f):
         )
 
     # and all arguments can be used as positional
-    if not all(p.kind in supported_parmeter_types for p in parameters):
+    if not all(p.kind in supported_parameter_types for p in parameters):
         raise ValueError(
             "f should use only POSITIONAL or POSITIONAL OR KEYWORD arguments"
         )
@@ -2965,20 +4237,6 @@ def _get_lambda_parameters(f):
     return parameters
 
 
-def _get_lambda_parameters_legacy(f):
-    # TODO (SPARK-29909) Remove once 2.7 support is dropped
-    import inspect
-
-    spec = inspect.getargspec(f)
-    if not 1 <= len(spec.args) <= 3 or spec.varargs or spec.keywords:
-        raise ValueError(
-            "f should take between 1 and 3 arguments, but provided function takes {}".format(
-                spec
-            )
-        )
-    return spec.args
-
-
 def _create_lambda(f):
     """
     Create `o.a.s.sql.expressions.LambdaFunction` corresponding
@@ -2989,17 +4247,17 @@ def _create_lambda(f):
             - (Column, Column) -> Column: ...
             - (Column, Column, Column) -> Column: ...
     """
-    if sys.version_info >= (3, 3):
-        parameters = _get_lambda_parameters(f)
-    else:
-        parameters = _get_lambda_parameters_legacy(f)
+    parameters = _get_lambda_parameters(f)
 
     sc = SparkContext._active_spark_context
     expressions = sc._jvm.org.apache.spark.sql.catalyst.expressions
 
     argnames = ["x", "y", "z"]
     args = [
-        _unresolved_named_lambda_variable(arg) for arg in argnames[: len(parameters)]
+        _unresolved_named_lambda_variable(
+            expressions.UnresolvedNamedLambdaVariable.freshVarName(arg)
+        )
+        for arg in argnames[: len(parameters)]
     ]
 
     result = f(*args)
@@ -3035,26 +4293,35 @@ def _invoke_higher_order_function(name, cols, funs):
     return Column(sc._jvm.Column(expr(*jcols + jfuns)))
 
 
-@since(3.1)
 def transform(col, f):
     """
     Returns an array of elements after applying a transformation to each element in the input array.
 
-    :param col: name of column or expression
-    :param f: a function that is applied to each element of the input array.
+    .. versionadded:: 3.1.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column or expression
+    f : function
+        a function that is applied to each element of the input array.
         Can take one of the following forms:
 
         - Unary ``(x: Column) -> Column: ...``
         - Binary ``(x: Column, i: Column) -> Column...``, where the second argument is
             a 0-based index of the element.
 
-        and can use methods of :class:`pyspark.sql.Column`, functions defined in
+        and can use methods of :class:`~pyspark.sql.Column`, functions defined in
         :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
         Python ``UserDefinedFunctions`` are not supported
         (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
 
-    :return: a :class:`pyspark.sql.Column`
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
 
+    Examples
+    --------
     >>> df = spark.createDataFrame([(1, [1, 2, 3, 4])], ("key", "values"))
     >>> df.select(transform("values", lambda x: x * 2).alias("doubled")).show()
     +------------+
@@ -3075,19 +4342,26 @@ def transform(col, f):
     return _invoke_higher_order_function("ArrayTransform", [col], [f])
 
 
-@since(3.1)
 def exists(col, f):
     """
     Returns whether a predicate holds for one or more elements in the array.
 
-    :param col: name of column or expression
-    :param f: an function ``(x: Column) -> Column: ...``  returning the Boolean expression.
-        Can use methods of :class:`pyspark.sql.Column`, functions defined in
+    .. versionadded:: 3.1.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column or expression
+    f : function
+        ``(x: Column) -> Column: ...``  returning the Boolean expression.
+        Can use methods of :class:`~pyspark.sql.Column`, functions defined in
         :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
         Python ``UserDefinedFunctions`` are not supported
         (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
-    :return: a :class:`pyspark.sql.Column`
+    :return: a :class:`~pyspark.sql.Column`
 
+    Examples
+    --------
     >>> df = spark.createDataFrame([(1, [1, 2, 3, 4]), (2, [3, -1, 0])],("key", "values"))
     >>> df.select(exists("values", lambda x: x < 0).alias("any_negative")).show()
     +------------+
@@ -3100,19 +4374,29 @@ def exists(col, f):
     return _invoke_higher_order_function("ArrayExists", [col], [f])
 
 
-@since(3.1)
 def forall(col, f):
     """
     Returns whether a predicate holds for every element in the array.
 
-    :param col: name of column or expression
-    :param f: an function ``(x: Column) -> Column: ...``  returning the Boolean expression.
-        Can use methods of :class:`pyspark.sql.Column`, functions defined in
+    .. versionadded:: 3.1.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column or expression
+    f : function
+        ``(x: Column) -> Column: ...``  returning the Boolean expression.
+        Can use methods of :class:`~pyspark.sql.Column`, functions defined in
         :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
         Python ``UserDefinedFunctions`` are not supported
         (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
-    :return: a :class:`pyspark.sql.Column`
 
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
+
+    Examples
+    --------
     >>> df = spark.createDataFrame(
     ...     [(1, ["bar"]), (2, ["foo", "bar"]), (3, ["foobar", "foo"])],
     ...     ("key", "values")
@@ -3129,26 +4413,35 @@ def forall(col, f):
     return _invoke_higher_order_function("ArrayForAll", [col], [f])
 
 
-@since(3.1)
 def filter(col, f):
     """
     Returns an array of elements for which a predicate holds in a given array.
 
-    :param col: name of column or expression
-    :param f: A function that returns the Boolean expression.
+    .. versionadded:: 3.1.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column or expression
+    f : function
+        A function that returns the Boolean expression.
         Can take one of the following forms:
 
         - Unary ``(x: Column) -> Column: ...``
         - Binary ``(x: Column, i: Column) -> Column...``, where the second argument is
             a 0-based index of the element.
 
-        and can use methods of :class:`pyspark.sql.Column`, functions defined in
+        and can use methods of :class:`~pyspark.sql.Column`, functions defined in
         :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
         Python ``UserDefinedFunctions`` are not supported
         (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
 
-    :return: a :class:`pyspark.sql.Column`
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
 
+    Examples
+    --------
     >>> df = spark.createDataFrame(
     ...     [(1, ["2018-09-20",  "2019-02-03", "2019-07-01", "2020-06-01"])],
     ...     ("key", "values")
@@ -3167,26 +4460,38 @@ def filter(col, f):
     return _invoke_higher_order_function("ArrayFilter", [col], [f])
 
 
-@since(3.1)
-def aggregate(col, zero, merge, finish=None):
+def aggregate(col, initialValue, merge, finish=None):
     """
     Applies a binary operator to an initial state and all elements in the array,
     and reduces this to a single state. The final state is converted into the final result
     by applying a finish function.
 
-    Both functions can use methods of :class:`pyspark.sql.Column`, functions defined in
+    Both functions can use methods of :class:`~pyspark.sql.Column`, functions defined in
     :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
     Python ``UserDefinedFunctions`` are not supported
     (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
 
-    :param col: name of column or expression
-    :param zero: initial value. Name of column or expression
-    :param merge: a binary function ``(acc: Column, x: Column) -> Column...`` returning expression
+    .. versionadded:: 3.1.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column or expression
+    initialValue : :class:`~pyspark.sql.Column` or str
+        initial value. Name of column or expression
+    merge : function
+        a binary function ``(acc: Column, x: Column) -> Column...`` returning expression
         of the same type as ``zero``
-    :param finish: an optional unary function ``(x: Column) -> Column: ...``
+    finish : function
+        an optional unary function ``(x: Column) -> Column: ...``
         used to convert accumulated value.
-    :return: a :class:`pyspark.sql.Column`
 
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([(1, [20.0, 4.0, 2.0, 6.0, 10.0])], ("id", "values"))
     >>> df.select(aggregate("values", lit(0.0), lambda acc, x: acc + x).alias("sum")).show()
     +----+
@@ -3216,34 +4521,45 @@ def aggregate(col, zero, merge, finish=None):
     if finish is not None:
         return _invoke_higher_order_function(
             "ArrayAggregate",
-            [col, zero],
+            [col, initialValue],
             [merge, finish]
         )
 
     else:
         return _invoke_higher_order_function(
             "ArrayAggregate",
-            [col, zero],
+            [col, initialValue],
             [merge]
         )
 
 
-@since(3.1)
-def zip_with(col1, col2, f):
+def zip_with(left, right, f):
     """
     Merge two given arrays, element-wise, into a single array using a function.
     If one array is shorter, nulls are appended at the end to match the length of the longer
     array, before applying the function.
 
-    :param col1: name of the first column or expression
-    :param col2: name of the second column or expression
-    :param f: a binary function ``(x1: Column, x2: Column) -> Column...``
-        Can use methods of :class:`pyspark.sql.Column`, functions defined in
+    .. versionadded:: 3.1.0
+
+    Parameters
+    ----------
+    left : :class:`~pyspark.sql.Column` or str
+        name of the first column or expression
+    right : :class:`~pyspark.sql.Column` or str
+        name of the second column or expression
+    f : function
+        a binary function ``(x1: Column, x2: Column) -> Column...``
+        Can use methods of :class:`~pyspark.sql.Column`, functions defined in
         :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
         Python ``UserDefinedFunctions`` are not supported
         (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
-    :return: a :class:`pyspark.sql.Column`
 
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([(1, [1, 3, 5, 8], [0, 2, 4, 6])], ("id", "xs", "ys"))
     >>> df.select(zip_with("xs", "ys", lambda x, y: x ** y).alias("powers")).show(truncate=False)
     +---------------------------+
@@ -3260,23 +4576,33 @@ def zip_with(col1, col2, f):
     |[foo_1, bar_2, 3]|
     +-----------------+
     """
-    return _invoke_higher_order_function("ZipWith", [col1, col2], [f])
+    return _invoke_higher_order_function("ZipWith", [left, right], [f])
 
 
-@since(3.1)
 def transform_keys(col, f):
     """
     Applies a function to every key-value pair in a map and returns
     a map with the results of those applications as the new keys for the pairs.
 
-    :param col: name of column or expression
-    :param f: a binary function ``(k: Column, v: Column) -> Column...``
-        Can use methods of :class:`pyspark.sql.Column`, functions defined in
+    .. versionadded:: 3.1.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column or expression
+    f : function
+        a binary function ``(k: Column, v: Column) -> Column...``
+        Can use methods of :class:`~pyspark.sql.Column`, functions defined in
         :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
         Python ``UserDefinedFunctions`` are not supported
         (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
-    :return: a :class:`pyspark.sql.Column`
 
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([(1, {"foo": -2.0, "bar": 2.0})], ("id", "data"))
     >>> df.select(transform_keys(
     ...     "data", lambda k, _: upper(k)).alias("data_upper")
@@ -3284,26 +4610,36 @@ def transform_keys(col, f):
     +-------------------------+
     |data_upper               |
     +-------------------------+
-    |[BAR -> 2.0, FOO -> -2.0]|
+    |{BAR -> 2.0, FOO -> -2.0}|
     +-------------------------+
     """
     return _invoke_higher_order_function("TransformKeys", [col], [f])
 
 
-@since(3.1)
 def transform_values(col, f):
     """
     Applies a function to every key-value pair in a map and returns
     a map with the results of those applications as the new values for the pairs.
 
-    :param col: name of column or expression
-    :param f: a binary function ``(k: Column, v: Column) -> Column...``
-        Can use methods of :class:`pyspark.sql.Column`, functions defined in
+    .. versionadded:: 3.1.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column or expression
+    f : function
+        a binary function ``(k: Column, v: Column) -> Column...``
+        Can use methods of :class:`~pyspark.sql.Column`, functions defined in
         :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
         Python ``UserDefinedFunctions`` are not supported
         (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
-    :return: a :class:`pyspark.sql.Column`
 
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([(1, {"IT": 10.0, "SALES": 2.0, "OPS": 24.0})], ("id", "data"))
     >>> df.select(transform_values(
     ...     "data", lambda k, v: when(k.isin("IT", "OPS"), v + 10.0).otherwise(v)
@@ -3311,25 +4647,35 @@ def transform_values(col, f):
     +---------------------------------------+
     |new_data                               |
     +---------------------------------------+
-    |[OPS -> 34.0, IT -> 20.0, SALES -> 2.0]|
+    |{OPS -> 34.0, IT -> 20.0, SALES -> 2.0}|
     +---------------------------------------+
     """
     return _invoke_higher_order_function("TransformValues", [col], [f])
 
 
-@since(3.1)
 def map_filter(col, f):
     """
     Returns a map whose key-value pairs satisfy a predicate.
 
-    :param col: name of column or expression
-    :param f: a binary function ``(k: Column, v: Column) -> Column...``
-        Can use methods of :class:`pyspark.sql.Column`, functions defined in
+    .. versionadded:: 3.1.0
+
+    Parameters
+    ----------
+    col : :class:`~pyspark.sql.Column` or str
+        name of column or expression
+    f : function
+        a binary function ``(k: Column, v: Column) -> Column...``
+        Can use methods of :class:`~pyspark.sql.Column`, functions defined in
         :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
         Python ``UserDefinedFunctions`` are not supported
         (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
-    :return: a :class:`pyspark.sql.Column`
 
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([(1, {"foo": 42.0, "bar": 1.0, "baz": 32.0})], ("id", "data"))
     >>> df.select(map_filter(
     ...     "data", lambda _, v: v > 30.0).alias("data_filtered")
@@ -3337,26 +4683,37 @@ def map_filter(col, f):
     +--------------------------+
     |data_filtered             |
     +--------------------------+
-    |[baz -> 32.0, foo -> 42.0]|
+    |{baz -> 32.0, foo -> 42.0}|
     +--------------------------+
     """
     return _invoke_higher_order_function("MapFilter", [col], [f])
 
 
-@since(3.1)
 def map_zip_with(col1, col2, f):
     """
     Merge two given maps, key-wise into a single map using a function.
 
-    :param col1: name of the first column or expression
-    :param col2: name of the second column or expression
-    :param f: a ternary function ``(k: Column, v1: Column, v2: Column) -> Column...``
-        Can use methods of :class:`pyspark.sql.Column`, functions defined in
+    .. versionadded:: 3.1.0
+
+    Parameters
+    ----------
+    col1 : :class:`~pyspark.sql.Column` or str
+        name of the first column or expression
+    col2 : :class:`~pyspark.sql.Column` or str
+        name of the second column or expression
+    f : function
+        a ternary function ``(k: Column, v1: Column, v2: Column) -> Column...``
+        Can use methods of :class:`~pyspark.sql.Column`, functions defined in
         :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
         Python ``UserDefinedFunctions`` are not supported
         (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
-    :return: a :class:`pyspark.sql.Column`
 
+    Returns
+    -------
+    :class:`~pyspark.sql.Column`
+
+    Examples
+    --------
     >>> df = spark.createDataFrame([
     ...     (1, {"IT": 24.0, "SALES": 12.00}, {"IT": 2.0, "SALES": 1.4})],
     ...     ("id", "base", "ratio")
@@ -3367,37 +4724,161 @@ def map_zip_with(col1, col2, f):
     +---------------------------+
     |updated_data               |
     +---------------------------+
-    |[SALES -> 16.8, IT -> 48.0]|
+    |{SALES -> 16.8, IT -> 48.0}|
     +---------------------------+
     """
     return _invoke_higher_order_function("MapZipWith", [col1, col2], [f])
 
 
-# ---------------------------- User Defined Function ----------------------------------
+# ---------------------- Partition transform functions --------------------------------
 
-@since(1.3)
-def udf(f=None, returnType=StringType()):
-    """Creates a user defined function (UDF).
+def years(col):
+    """
+    Partition transform function: A transform for timestamps and dates
+    to partition data into years.
 
-    .. note:: The user-defined functions are considered deterministic by default. Due to
-        optimization, duplicate invocations may be eliminated or the function may even be invoked
-        more times than it is present in the query. If your function is not deterministic, call
-        `asNondeterministic` on the user defined function. E.g.:
+    .. versionadded:: 3.1.0
 
-    >>> from pyspark.sql.types import IntegerType
-    >>> import random
-    >>> random_udf = udf(lambda: int(random.random() * 100), IntegerType()).asNondeterministic()
+    Examples
+    --------
+    >>> df.writeTo("catalog.db.table").partitionedBy(  # doctest: +SKIP
+    ...     years("ts")
+    ... ).createOrReplace()
+
+    Notes
+    -----
+    This function can be used only in combination with
+    :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy`
+    method of the `DataFrameWriterV2`.
+
+    """
+    sc = SparkContext._active_spark_context
+    return Column(sc._jvm.functions.years(_to_java_column(col)))
+
+
+def months(col):
+    """
+    Partition transform function: A transform for timestamps and dates
+    to partition data into months.
+
+    .. versionadded:: 3.1.0
+
+    Examples
+    --------
+    >>> df.writeTo("catalog.db.table").partitionedBy(
+    ...     months("ts")
+    ... ).createOrReplace()  # doctest: +SKIP
+
+    Notes
+    -----
+    This function can be used only in combination with
+    :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy`
+    method of the `DataFrameWriterV2`.
+
+    """
+    sc = SparkContext._active_spark_context
+    return Column(sc._jvm.functions.months(_to_java_column(col)))
+
+
+def days(col):
+    """
+    Partition transform function: A transform for timestamps and dates
+    to partition data into days.
+
+    .. versionadded:: 3.1.0
+
+    Examples
+    --------
+    >>> df.writeTo("catalog.db.table").partitionedBy(  # doctest: +SKIP
+    ...     days("ts")
+    ... ).createOrReplace()
+
+    Notes
+    -----
+    This function can be used only in combination with
+    :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy`
+    method of the `DataFrameWriterV2`.
+
+    """
+    sc = SparkContext._active_spark_context
+    return Column(sc._jvm.functions.days(_to_java_column(col)))
+
+
+def hours(col):
+    """
+    Partition transform function: A transform for timestamps
+    to partition data into hours.
+
+    .. versionadded:: 3.1.0
 
-    .. note:: The user-defined functions do not support conditional expressions or short circuiting
-        in boolean expressions and it ends up with being executed all internally. If the functions
-        can fail on special rows, the workaround is to incorporate the condition into the functions.
+    Examples
+    --------
+    >>> df.writeTo("catalog.db.table").partitionedBy(   # doctest: +SKIP
+    ...     hours("ts")
+    ... ).createOrReplace()
 
-    .. note:: The user-defined functions do not take keyword arguments on the calling side.
+    Notes
+    -----
+    This function can be used only in combination with
+    :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy`
+    method of the `DataFrameWriterV2`.
 
-    :param f: python function if used as a standalone function
-    :param returnType: the return type of the user-defined function. The value can be either a
+    """
+    sc = SparkContext._active_spark_context
+    return Column(sc._jvm.functions.hours(_to_java_column(col)))
+
+
+def bucket(numBuckets, col):
+    """
+    Partition transform function: A transform for any type that partitions
+    by a hash of the input column.
+
+    .. versionadded:: 3.1.0
+
+    Examples
+    --------
+    >>> df.writeTo("catalog.db.table").partitionedBy(  # doctest: +SKIP
+    ...     bucket(42, "ts")
+    ... ).createOrReplace()
+
+    Notes
+    -----
+    This function can be used only in combination with
+    :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy`
+    method of the `DataFrameWriterV2`.
+
+    """
+    if not isinstance(numBuckets, (int, Column)):
+        raise TypeError(
+            "numBuckets should be a Column or an int, got {}".format(type(numBuckets))
+        )
+
+    sc = SparkContext._active_spark_context
+    numBuckets = (
+        _create_column_from_literal(numBuckets)
+        if isinstance(numBuckets, int)
+        else _to_java_column(numBuckets)
+    )
+    return Column(sc._jvm.functions.bucket(numBuckets, _to_java_column(col)))
+
+
+# ---------------------------- User Defined Function ----------------------------------
+
+def udf(f=None, returnType=StringType()):
+    """Creates a user defined function (UDF).
+
+    .. versionadded:: 1.3.0
+
+    Parameters
+    ----------
+    f : function
+        python function if used as a standalone function
+    returnType : :class:`pyspark.sql.types.DataType` or str
+        the return type of the user-defined function. The value can be either a
         :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string.
 
+    Examples
+    --------
     >>> from pyspark.sql.types import IntegerType
     >>> slen = udf(lambda s: len(s), IntegerType())
     >>> @udf
@@ -3417,6 +4898,23 @@ def udf(f=None, returnType=StringType()):
     +----------+--------------+------------+
     |         8|      JOHN DOE|          22|
     +----------+--------------+------------+
+
+    Notes
+    -----
+    The user-defined functions are considered deterministic by default. Due to
+    optimization, duplicate invocations may be eliminated or the function may even be invoked
+    more times than it is present in the query. If your function is not deterministic, call
+    `asNondeterministic` on the user defined function. E.g.:
+
+    >>> from pyspark.sql.types import IntegerType
+    >>> import random
+    >>> random_udf = udf(lambda: int(random.random() * 100), IntegerType()).asNondeterministic()
+
+    The user-defined functions do not support conditional expressions or short circuiting
+    in boolean expressions and it ends up with being executed all internally. If the functions
+    can fail on special rows, the workaround is to incorporate the condition into the functions.
+
+    The user-defined functions do not take keyword arguments on the calling side.
     """
 
     # The following table shows most of Python data and SQL type conversions in normal UDFs that
@@ -3462,13 +4960,6 @@ def udf(f=None, returnType=StringType()):
                            evalType=PythonEvalType.SQL_BATCHED_UDF)
 
 
-blacklist = ['map', 'since', 'ignore_unicode_prefix']
-__all__ = [k for k, v in globals().items()
-           if not k.startswith('_') and k[0].islower() and callable(v) and k not in blacklist]
-__all__ += ["PandasUDFType"]
-__all__.sort()
-
-
 def _test():
     import doctest
     from pyspark.sql import Row, SparkSession
@@ -3481,7 +4972,7 @@ def _test():
     sc = spark.sparkContext
     globs['sc'] = sc
     globs['spark'] = spark
-    globs['df'] = spark.createDataFrame([Row(name='Alice', age=2), Row(name='Bob', age=5)])
+    globs['df'] = spark.createDataFrame([Row(age=2, name='Alice'), Row(age=5, name='Bob')])
     (failure_count, test_count) = doctest.testmod(
         pyspark.sql.functions, globs=globs,
         optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE)
diff --git a/python/pyspark/sql/functions.pyi b/python/pyspark/sql/functions.pyi
new file mode 100644
index 0000000000000..674c5a7e653ad
--- /dev/null
+++ b/python/pyspark/sql/functions.pyi
@@ -0,0 +1,365 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import overload
+from typing import Any, Callable, Dict, List, Optional, Union
+
+from pyspark.sql._typing import (
+    ColumnOrName,
+    DataTypeOrString,
+)
+from pyspark.sql.pandas.functions import (  # noqa: F401
+    pandas_udf as pandas_udf,
+    PandasUDFType as PandasUDFType,
+)
+from pyspark.sql.column import Column
+from pyspark.sql.dataframe import DataFrame
+from pyspark.sql.types import (  # noqa: F401
+    ArrayType,
+    StringType,
+    StructType,
+    DataType,
+)
+from pyspark.sql.utils import to_str  # noqa: F401
+
+def approxCountDistinct(col: ColumnOrName, rsd: Optional[float] = ...) -> Column: ...
+def approx_count_distinct(col: ColumnOrName, rsd: Optional[float] = ...) -> Column: ...
+def broadcast(df: DataFrame) -> DataFrame: ...
+def coalesce(*cols: ColumnOrName) -> Column: ...
+def corr(col1: ColumnOrName, col2: ColumnOrName) -> Column: ...
+def covar_pop(col1: ColumnOrName, col2: ColumnOrName) -> Column: ...
+def covar_samp(col1: ColumnOrName, col2: ColumnOrName) -> Column: ...
+def countDistinct(col: ColumnOrName, *cols: ColumnOrName) -> Column: ...
+def count_distinct(col: ColumnOrName, *cols: ColumnOrName) -> Column: ...
+def first(col: ColumnOrName, ignorenulls: bool = ...) -> Column: ...
+def grouping(col: ColumnOrName) -> Column: ...
+def grouping_id(*cols: ColumnOrName) -> Column: ...
+def input_file_name() -> Column: ...
+def isnan(col: ColumnOrName) -> Column: ...
+def isnull(col: ColumnOrName) -> Column: ...
+def last(col: ColumnOrName, ignorenulls: bool = ...) -> Column: ...
+def monotonically_increasing_id() -> Column: ...
+def nanvl(col1: ColumnOrName, col2: ColumnOrName) -> Column: ...
+def percentile_approx(
+    col: ColumnOrName,
+    percentage: Union[Column, float, List[float], tuple[float]],
+    accuracy: Union[Column, float] = ...,
+) -> Column: ...
+def rand(seed: Optional[int] = ...) -> Column: ...
+def randn(seed: Optional[int] = ...) -> Column: ...
+def round(col: ColumnOrName, scale: int = ...) -> Column: ...
+def bround(col: ColumnOrName, scale: int = ...) -> Column: ...
+def shiftLeft(col: ColumnOrName, numBits: int) -> Column: ...
+def shiftleft(col: ColumnOrName, numBits: int) -> Column: ...
+def shiftRight(col: ColumnOrName, numBits: int) -> Column: ...
+def shiftright(col: ColumnOrName, numBits: int) -> Column: ...
+def shiftRightUnsigned(col: ColumnOrName, numBits: int) -> Column: ...
+def shiftrightunsigned(col: ColumnOrName, numBits: int) -> Column: ...
+def spark_partition_id() -> Column: ...
+def expr(str: str) -> Column: ...
+def struct(*cols: ColumnOrName) -> Column: ...
+def greatest(*cols: ColumnOrName) -> Column: ...
+def least(*cols: Column) -> Column: ...
+def when(condition: Column, value: Any) -> Column: ...
+@overload
+def log(arg1: ColumnOrName) -> Column: ...
+@overload
+def log(arg1: float, arg2: ColumnOrName) -> Column: ...
+def log2(col: ColumnOrName) -> Column: ...
+def conv(col: ColumnOrName, fromBase: int, toBase: int) -> Column: ...
+def factorial(col: ColumnOrName) -> Column: ...
+def lag(
+    col: ColumnOrName, offset: int = ..., default: Optional[Any] = ...
+) -> Column: ...
+def lead(
+    col: ColumnOrName, offset: int = ..., default: Optional[Any] = ...
+) -> Column: ...
+def nth_value(
+    col: ColumnOrName, offset: int, ignoreNulls: Optional[bool] = ...
+) -> Column: ...
+def ntile(n: int) -> Column: ...
+def current_date() -> Column: ...
+def current_timestamp() -> Column: ...
+def date_format(date: ColumnOrName, format: str) -> Column: ...
+def year(col: ColumnOrName) -> Column: ...
+def quarter(col: ColumnOrName) -> Column: ...
+def month(col: ColumnOrName) -> Column: ...
+def dayofweek(col: ColumnOrName) -> Column: ...
+def dayofmonth(col: ColumnOrName) -> Column: ...
+def dayofyear(col: ColumnOrName) -> Column: ...
+def hour(col: ColumnOrName) -> Column: ...
+def minute(col: ColumnOrName) -> Column: ...
+def second(col: ColumnOrName) -> Column: ...
+def weekofyear(col: ColumnOrName) -> Column: ...
+def date_add(start: ColumnOrName, days: int) -> Column: ...
+def date_sub(start: ColumnOrName, days: int) -> Column: ...
+def datediff(end: ColumnOrName, start: ColumnOrName) -> Column: ...
+def add_months(start: ColumnOrName, months: int) -> Column: ...
+def months_between(
+    date1: ColumnOrName, date2: ColumnOrName, roundOff: bool = ...
+) -> Column: ...
+def to_date(col: ColumnOrName, format: Optional[str] = ...) -> Column: ...
+@overload
+def to_timestamp(col: ColumnOrName) -> Column: ...
+@overload
+def to_timestamp(col: ColumnOrName, format: str) -> Column: ...
+def trunc(date: ColumnOrName, format: str) -> Column: ...
+def date_trunc(format: str, timestamp: ColumnOrName) -> Column: ...
+def next_day(date: ColumnOrName, dayOfWeek: str) -> Column: ...
+def last_day(date: ColumnOrName) -> Column: ...
+def from_unixtime(timestamp: ColumnOrName, format: str = ...) -> Column: ...
+def unix_timestamp(
+    timestamp: Optional[ColumnOrName] = ..., format: str = ...
+) -> Column: ...
+def from_utc_timestamp(timestamp: ColumnOrName, tz: ColumnOrName) -> Column: ...
+def to_utc_timestamp(timestamp: ColumnOrName, tz: ColumnOrName) -> Column: ...
+def timestamp_seconds(col: ColumnOrName) -> Column: ...
+def window(
+    timeColumn: ColumnOrName,
+    windowDuration: str,
+    slideDuration: Optional[str] = ...,
+    startTime: Optional[str] = ...,
+) -> Column: ...
+def crc32(col: ColumnOrName) -> Column: ...
+def md5(col: ColumnOrName) -> Column: ...
+def sha1(col: ColumnOrName) -> Column: ...
+def sha2(col: ColumnOrName, numBits: int) -> Column: ...
+def hash(*cols: ColumnOrName) -> Column: ...
+def xxhash64(*cols: ColumnOrName) -> Column: ...
+def assert_true(col: ColumnOrName, errMsg: Union[Column, str] = ...) -> Column: ...
+def raise_error(errMsg: Union[Column, str]) -> Column: ...
+def concat(*cols: ColumnOrName) -> Column: ...
+def concat_ws(sep: str, *cols: ColumnOrName) -> Column: ...
+def decode(col: ColumnOrName, charset: str) -> Column: ...
+def encode(col: ColumnOrName, charset: str) -> Column: ...
+def format_number(col: ColumnOrName, d: int) -> Column: ...
+def format_string(format: str, *cols: ColumnOrName) -> Column: ...
+def instr(str: ColumnOrName, substr: str) -> Column: ...
+def overlay(
+    src: ColumnOrName,
+    replace: ColumnOrName,
+    pos: Union[Column, int],
+    len: Union[Column, int] = ...,
+) -> Column: ...
+def substring(str: ColumnOrName, pos: int, len: int) -> Column: ...
+def substring_index(str: ColumnOrName, delim: str, count: int) -> Column: ...
+def levenshtein(left: ColumnOrName, right: ColumnOrName) -> Column: ...
+def locate(substr: str, str: ColumnOrName, pos: int = ...) -> Column: ...
+def lpad(col: ColumnOrName, len: int, pad: str) -> Column: ...
+def rpad(col: ColumnOrName, len: int, pad: str) -> Column: ...
+def repeat(col: ColumnOrName, n: int) -> Column: ...
+def split(str: ColumnOrName, pattern: str, limit: int = ...) -> Column: ...
+def regexp_extract(str: ColumnOrName, pattern: str, idx: int) -> Column: ...
+def regexp_replace(str: ColumnOrName, pattern: str, replacement: str) -> Column: ...
+def initcap(col: ColumnOrName) -> Column: ...
+def soundex(col: ColumnOrName) -> Column: ...
+def bin(col: ColumnOrName) -> Column: ...
+def hex(col: ColumnOrName) -> Column: ...
+def unhex(col: ColumnOrName) -> Column: ...
+def length(col: ColumnOrName) -> Column: ...
+def translate(srcCol: ColumnOrName, matching: str, replace: str) -> Column: ...
+def map_from_arrays(col1: ColumnOrName, col2: ColumnOrName) -> Column: ...
+def create_map(*cols: ColumnOrName) -> Column: ...
+def array(*cols: ColumnOrName) -> Column: ...
+def array_contains(col: ColumnOrName, value: Any) -> Column: ...
+def arrays_overlap(a1: ColumnOrName, a2: ColumnOrName) -> Column: ...
+def slice(
+    x: ColumnOrName, start: Union[Column, int], length: Union[Column, int]
+) -> Column: ...
+def array_join(
+    col: ColumnOrName, delimiter: str, null_replacement: Optional[str] = ...
+) -> Column: ...
+def array_position(col: ColumnOrName, value: Any) -> Column: ...
+def element_at(col: ColumnOrName, extraction: Any) -> Column: ...
+def array_remove(col: ColumnOrName, element: Any) -> Column: ...
+def array_distinct(col: ColumnOrName) -> Column: ...
+def array_intersect(col1: ColumnOrName, col2: ColumnOrName) -> Column: ...
+def array_union(col1: ColumnOrName, col2: ColumnOrName) -> Column: ...
+def array_except(col1: ColumnOrName, col2: ColumnOrName) -> Column: ...
+def explode(col: ColumnOrName) -> Column: ...
+def explode_outer(col: ColumnOrName) -> Column: ...
+def posexplode(col: ColumnOrName) -> Column: ...
+def posexplode_outer(col: ColumnOrName) -> Column: ...
+def get_json_object(col: ColumnOrName, path: str) -> Column: ...
+def json_tuple(col: ColumnOrName, *fields: str) -> Column: ...
+def from_json(
+    col: ColumnOrName,
+    schema: Union[ArrayType, StructType, Column, str],
+    options: Optional[Dict[str, str]] = ...,
+) -> Column: ...
+def to_json(col: ColumnOrName, options: Optional[Dict[str, str]] = ...) -> Column: ...
+def schema_of_json(json: ColumnOrName, options: Optional[Dict[str, str]] = ...) -> Column: ...
+def schema_of_csv(csv: ColumnOrName, options: Optional[Dict[str, str]] = ...) -> Column: ...
+def to_csv(col: ColumnOrName, options: Optional[Dict[str, str]] = ...) -> Column: ...
+def size(col: ColumnOrName) -> Column: ...
+def array_min(col: ColumnOrName) -> Column: ...
+def array_max(col: ColumnOrName) -> Column: ...
+def sort_array(col: ColumnOrName, asc: bool = ...) -> Column: ...
+def array_sort(col: ColumnOrName) -> Column: ...
+def shuffle(col: ColumnOrName) -> Column: ...
+def reverse(col: ColumnOrName) -> Column: ...
+def flatten(col: ColumnOrName) -> Column: ...
+def map_keys(col: ColumnOrName) -> Column: ...
+def map_values(col: ColumnOrName) -> Column: ...
+def map_entries(col: ColumnOrName) -> Column: ...
+def map_from_entries(col: ColumnOrName) -> Column: ...
+def array_repeat(col: ColumnOrName, count: Union[Column, int]) -> Column: ...
+def arrays_zip(*cols: ColumnOrName) -> Column: ...
+def map_concat(*cols: ColumnOrName) -> Column: ...
+def sequence(
+    start: ColumnOrName, stop: ColumnOrName, step: Optional[ColumnOrName] = ...
+) -> Column: ...
+def from_csv(
+    col: ColumnOrName,
+    schema: Union[StructType, Column, str],
+    options: Optional[Dict[str, str]] = ...,
+) -> Column: ...
+@overload
+def transform(col: ColumnOrName, f: Callable[[Column], Column]) -> Column: ...
+@overload
+def transform(col: ColumnOrName, f: Callable[[Column, Column], Column]) -> Column: ...
+def exists(col: ColumnOrName, f: Callable[[Column], Column]) -> Column: ...
+def forall(col: ColumnOrName, f: Callable[[Column], Column]) -> Column: ...
+@overload
+def filter(col: ColumnOrName, f: Callable[[Column], Column]) -> Column: ...
+@overload
+def filter(col: ColumnOrName, f: Callable[[Column, Column], Column]) -> Column: ...
+def aggregate(
+    col: ColumnOrName,
+    initialValue: ColumnOrName,
+    merge: Callable[[Column, Column], Column],
+    finish: Optional[Callable[[Column], Column]] = ...,
+) -> Column: ...
+def zip_with(
+    left: ColumnOrName,
+    right: ColumnOrName,
+    f: Callable[[Column, Column], Column],
+) -> Column: ...
+def transform_keys(
+    col: ColumnOrName, f: Callable[[Column, Column], Column]
+) -> Column: ...
+def transform_values(
+    col: ColumnOrName, f: Callable[[Column, Column], Column]
+) -> Column: ...
+def map_filter(col: ColumnOrName, f: Callable[[Column, Column], Column]) -> Column: ...
+def map_zip_with(
+    col1: ColumnOrName,
+    col2: ColumnOrName,
+    f: Callable[[Column, Column, Column], Column],
+) -> Column: ...
+def abs(col: ColumnOrName) -> Column: ...
+def acos(col: ColumnOrName) -> Column: ...
+def acosh(col: ColumnOrName) -> Column: ...
+def asc(col: ColumnOrName) -> Column: ...
+def asc_nulls_first(col: ColumnOrName) -> Column: ...
+def asc_nulls_last(col: ColumnOrName) -> Column: ...
+def ascii(col: ColumnOrName) -> Column: ...
+def asin(col: ColumnOrName) -> Column: ...
+def asinh(col: ColumnOrName) -> Column: ...
+def atan(col: ColumnOrName) -> Column: ...
+def atanh(col: ColumnOrName) -> Column: ...
+@overload
+def atan2(col1: ColumnOrName, col2: ColumnOrName) -> Column: ...
+@overload
+def atan2(col1: float, col2: ColumnOrName) -> Column: ...
+@overload
+def atan2(col1: ColumnOrName, col2: float) -> Column: ...
+def avg(col: ColumnOrName) -> Column: ...
+def base64(col: ColumnOrName) -> Column: ...
+def bitwiseNOT(col: ColumnOrName) -> Column: ...
+def bitwise_not(col: ColumnOrName) -> Column: ...
+def cbrt(col: ColumnOrName) -> Column: ...
+def ceil(col: ColumnOrName) -> Column: ...
+def col(col: str) -> Column: ...
+def collect_list(col: ColumnOrName) -> Column: ...
+def collect_set(col: ColumnOrName) -> Column: ...
+def column(col: str) -> Column: ...
+def cos(col: ColumnOrName) -> Column: ...
+def cosh(col: ColumnOrName) -> Column: ...
+def count(col: ColumnOrName) -> Column: ...
+def cume_dist() -> Column: ...
+def degrees(col: ColumnOrName) -> Column: ...
+def dense_rank() -> Column: ...
+def desc(col: ColumnOrName) -> Column: ...
+def desc_nulls_first(col: ColumnOrName) -> Column: ...
+def desc_nulls_last(col: ColumnOrName) -> Column: ...
+def exp(col: ColumnOrName) -> Column: ...
+def expm1(col: ColumnOrName) -> Column: ...
+def floor(col: ColumnOrName) -> Column: ...
+@overload
+def hypot(col1: ColumnOrName, col2: ColumnOrName) -> Column: ...
+@overload
+def hypot(col1: float, col2: ColumnOrName) -> Column: ...
+@overload
+def hypot(col1: ColumnOrName, col2: float) -> Column: ...
+def kurtosis(col: ColumnOrName) -> Column: ...
+def lit(col: Any) -> Column: ...
+def log10(col: ColumnOrName) -> Column: ...
+def log1p(col: ColumnOrName) -> Column: ...
+def lower(col: ColumnOrName) -> Column: ...
+def ltrim(col: ColumnOrName) -> Column: ...
+def max(col: ColumnOrName) -> Column: ...
+def mean(col: ColumnOrName) -> Column: ...
+def min(col: ColumnOrName) -> Column: ...
+def percent_rank() -> Column: ...
+@overload
+def pow(col1: ColumnOrName, col2: ColumnOrName) -> Column: ...
+@overload
+def pow(col1: float, col2: ColumnOrName) -> Column: ...
+@overload
+def pow(col1: ColumnOrName, col2: float) -> Column: ...
+def product(col: ColumnOrName) -> Column: ...
+def radians(col: ColumnOrName) -> Column: ...
+def rank() -> Column: ...
+def rint(col: ColumnOrName) -> Column: ...
+def row_number() -> Column: ...
+def rtrim(col: ColumnOrName) -> Column: ...
+def signum(col: ColumnOrName) -> Column: ...
+def sin(col: ColumnOrName) -> Column: ...
+def sinh(col: ColumnOrName) -> Column: ...
+def skewness(col: ColumnOrName) -> Column: ...
+def sqrt(col: ColumnOrName) -> Column: ...
+def stddev(col: ColumnOrName) -> Column: ...
+def stddev_pop(col: ColumnOrName) -> Column: ...
+def stddev_samp(col: ColumnOrName) -> Column: ...
+def sum(col: ColumnOrName) -> Column: ...
+def sumDistinct(col: ColumnOrName) -> Column: ...
+def sum_distinct(col: ColumnOrName) -> Column: ...
+def tan(col: ColumnOrName) -> Column: ...
+def tanh(col: ColumnOrName) -> Column: ...
+def toDegrees(col: ColumnOrName) -> Column: ...
+def toRadians(col: ColumnOrName) -> Column: ...
+def trim(col: ColumnOrName) -> Column: ...
+def unbase64(col: ColumnOrName) -> Column: ...
+def upper(col: ColumnOrName) -> Column: ...
+def var_pop(col: ColumnOrName) -> Column: ...
+def var_samp(col: ColumnOrName) -> Column: ...
+def variance(col: ColumnOrName) -> Column: ...
+@overload
+def udf(
+    f: Callable[..., Any], returnType: DataTypeOrString = ...
+) -> Callable[..., Column]: ...
+@overload
+def udf(
+    f: DataTypeOrString = ...,
+) -> Callable[[Callable[..., Any]], Callable[..., Column]]: ...
+@overload
+def udf(
+    *,
+    returnType: DataTypeOrString = ...,
+) -> Callable[[Callable[..., Any]], Callable[..., Column]]: ...
diff --git a/python/pyspark/sql/group.py b/python/pyspark/sql/group.py
index ac826bc64ad7e..d3cbf9268c9c7 100644
--- a/python/pyspark/sql/group.py
+++ b/python/pyspark/sql/group.py
@@ -17,12 +17,10 @@
 
 import sys
 
-from pyspark import since
-from pyspark.rdd import ignore_unicode_prefix
 from pyspark.sql.column import Column, _to_seq
 from pyspark.sql.dataframe import DataFrame
 from pyspark.sql.pandas.group_ops import PandasGroupedOpsMixin
-from pyspark.sql.types import *
+from pyspark.sql.types import StructType, StructField, IntegerType, StringType
 
 __all__ = ["GroupedData"]
 
@@ -60,8 +58,6 @@ def __init__(self, jgd, df):
         self._df = df
         self.sql_ctx = df.sql_ctx
 
-    @ignore_unicode_prefix
-    @since(1.3)
     def agg(self, *exprs):
         """Compute aggregates and returns the result as a :class:`DataFrame`.
 
@@ -83,26 +79,35 @@ def agg(self, *exprs):
 
         Alternatively, ``exprs`` can also be a list of aggregate :class:`Column` expressions.
 
-        .. note:: Built-in aggregation functions and group aggregate pandas UDFs cannot be mixed
-            in a single call to this function.
+        .. versionadded:: 1.3.0
 
-        :param exprs: a dict mapping from column name (string) to aggregate functions (string),
+        Parameters
+        ----------
+        exprs : dict
+            a dict mapping from column name (string) to aggregate functions (string),
             or a list of :class:`Column`.
 
+        Notes
+        -----
+        Built-in aggregation functions and group aggregate pandas UDFs cannot be mixed
+        in a single call to this function.
+
+        Examples
+        --------
         >>> gdf = df.groupBy(df.name)
         >>> sorted(gdf.agg({"*": "count"}).collect())
-        [Row(name=u'Alice', count(1)=1), Row(name=u'Bob', count(1)=1)]
+        [Row(name='Alice', count(1)=1), Row(name='Bob', count(1)=1)]
 
         >>> from pyspark.sql import functions as F
         >>> sorted(gdf.agg(F.min(df.age)).collect())
-        [Row(name=u'Alice', min(age)=2), Row(name=u'Bob', min(age)=5)]
+        [Row(name='Alice', min(age)=2), Row(name='Bob', min(age)=5)]
 
         >>> from pyspark.sql.functions import pandas_udf, PandasUDFType
         >>> @pandas_udf('int', PandasUDFType.GROUPED_AGG)  # doctest: +SKIP
         ... def min_udf(v):
         ...     return v.min()
         >>> sorted(gdf.agg(min_udf(df.age)).collect())  # doctest: +SKIP
-        [Row(name=u'Alice', min_udf(age)=2), Row(name=u'Bob', min_udf(age)=5)]
+        [Row(name='Alice', min_udf(age)=2), Row(name='Bob', min_udf(age)=5)]
         """
         assert exprs, "exprs should not be empty"
         if len(exprs) == 1 and isinstance(exprs[0], dict):
@@ -115,23 +120,32 @@ def agg(self, *exprs):
         return DataFrame(jdf, self.sql_ctx)
 
     @dfapi
-    @since(1.3)
     def count(self):
         """Counts the number of records for each group.
 
+        .. versionadded:: 1.3.0
+
+        Examples
+        --------
         >>> sorted(df.groupBy(df.age).count().collect())
         [Row(age=2, count=1), Row(age=5, count=1)]
         """
 
     @df_varargs_api
-    @since(1.3)
     def mean(self, *cols):
         """Computes average values for each numeric columns for each group.
 
         :func:`mean` is an alias for :func:`avg`.
 
-        :param cols: list of column names (string). Non-numeric columns are ignored.
+        .. versionadded:: 1.3.0
 
+        Parameters
+        ----------
+        cols : str
+            column names. Non-numeric columns are ignored.
+
+        Examples
+        --------
         >>> df.groupBy().mean('age').collect()
         [Row(avg(age)=3.5)]
         >>> df3.groupBy().mean('age', 'height').collect()
@@ -139,14 +153,20 @@ def mean(self, *cols):
         """
 
     @df_varargs_api
-    @since(1.3)
     def avg(self, *cols):
         """Computes average values for each numeric columns for each group.
 
         :func:`mean` is an alias for :func:`avg`.
 
-        :param cols: list of column names (string). Non-numeric columns are ignored.
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        cols : str
+            column names. Non-numeric columns are ignored.
 
+        Examples
+        --------
         >>> df.groupBy().avg('age').collect()
         [Row(avg(age)=3.5)]
         >>> df3.groupBy().avg('age', 'height').collect()
@@ -154,10 +174,13 @@ def avg(self, *cols):
         """
 
     @df_varargs_api
-    @since(1.3)
     def max(self, *cols):
         """Computes the max value for each numeric columns for each group.
 
+        .. versionadded:: 1.3.0
+
+        Examples
+        --------
         >>> df.groupBy().max('age').collect()
         [Row(max(age)=5)]
         >>> df3.groupBy().max('age', 'height').collect()
@@ -165,12 +188,18 @@ def max(self, *cols):
         """
 
     @df_varargs_api
-    @since(1.3)
     def min(self, *cols):
         """Computes the min value for each numeric column for each group.
 
-        :param cols: list of column names (string). Non-numeric columns are ignored.
+        .. versionadded:: 1.3.0
+
+        Parameters
+        ----------
+        cols : str
+            column names. Non-numeric columns are ignored.
 
+        Examples
+        --------
         >>> df.groupBy().min('age').collect()
         [Row(min(age)=2)]
         >>> df3.groupBy().min('age', 'height').collect()
@@ -178,19 +207,24 @@ def min(self, *cols):
         """
 
     @df_varargs_api
-    @since(1.3)
     def sum(self, *cols):
         """Compute the sum for each numeric columns for each group.
 
-        :param cols: list of column names (string). Non-numeric columns are ignored.
+        .. versionadded:: 1.3.0
 
+        Parameters
+        ----------
+        cols : str
+            column names. Non-numeric columns are ignored.
+
+        Examples
+        --------
         >>> df.groupBy().sum('age').collect()
         [Row(sum(age)=7)]
         >>> df3.groupBy().sum('age', 'height').collect()
         [Row(sum(age)=7, sum(height)=165)]
         """
 
-    @since(1.6)
     def pivot(self, pivot_col, values=None):
         """
         Pivots a column of the current :class:`DataFrame` and perform the specified aggregation.
@@ -198,9 +232,17 @@ def pivot(self, pivot_col, values=None):
         of distinct values to pivot on, and one that does not. The latter is more concise but less
         efficient, because Spark needs to first compute the list of distinct values internally.
 
-        :param pivot_col: Name of the column to pivot.
-        :param values: List of values that will be translated to columns in the output DataFrame.
+        .. versionadded:: 1.6.0
+
+        Parameters
+        ----------
+        pivot_col : str
+            Name of the column to pivot.
+        values :
+            List of values that will be translated to columns in the output DataFrame.
 
+        Examples
+        --------
         # Compute the sum of earnings for each year by course with each course as a separate column
 
         >>> df4.groupBy("year").pivot("course", ["dotNET", "Java"]).sum("earnings").collect()
diff --git a/python/pyspark/sql/group.pyi b/python/pyspark/sql/group.pyi
new file mode 100644
index 0000000000000..0b0df8c63cfdd
--- /dev/null
+++ b/python/pyspark/sql/group.pyi
@@ -0,0 +1,44 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import overload
+from typing import Dict, List, Optional
+
+from pyspark.sql._typing import LiteralType
+from pyspark.sql.context import SQLContext
+from pyspark.sql.column import Column
+from pyspark.sql.dataframe import DataFrame
+from pyspark.sql.pandas.group_ops import PandasGroupedOpsMixin
+from py4j.java_gateway import JavaObject  # type: ignore[import]
+
+class GroupedData(PandasGroupedOpsMixin):
+    sql_ctx: SQLContext
+    def __init__(self, jgd: JavaObject, df: DataFrame) -> None: ...
+    @overload
+    def agg(self, *exprs: Column) -> DataFrame: ...
+    @overload
+    def agg(self, __exprs: Dict[str, str]) -> DataFrame: ...
+    def count(self) -> DataFrame: ...
+    def mean(self, *cols: str) -> DataFrame: ...
+    def avg(self, *cols: str) -> DataFrame: ...
+    def max(self, *cols: str) -> DataFrame: ...
+    def min(self, *cols: str) -> DataFrame: ...
+    def sum(self, *cols: str) -> DataFrame: ...
+    def pivot(
+        self, pivot_col: str, values: Optional[List[LiteralType]] = ...
+    ) -> GroupedData: ...
diff --git a/python/pyspark/sql/pandas/_typing/__init__.pyi b/python/pyspark/sql/pandas/_typing/__init__.pyi
new file mode 100644
index 0000000000000..dda1b3341b31c
--- /dev/null
+++ b/python/pyspark/sql/pandas/_typing/__init__.pyi
@@ -0,0 +1,338 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import (
+    Any,
+    Callable,
+    Iterable,
+    NewType,
+    Tuple,
+    Type,
+    Union,
+)
+from typing_extensions import Protocol, Literal
+from types import FunctionType
+
+from pyspark.sql._typing import LiteralType
+from pyspark.sql.pandas._typing.protocols.frame import DataFrameLike as DataFrameLike
+from pyspark.sql.pandas._typing.protocols.series import SeriesLike as SeriesLike
+
+import pandas.core.frame  # type: ignore[import]
+import pandas.core.series  # type: ignore[import]
+
+# POC compatibility annotations
+PandasDataFrame: Type[DataFrameLike] = pandas.core.frame.DataFrame
+PandasSeries: Type[SeriesLike] = pandas.core.series.Series
+
+DataFrameOrSeriesLike = Union[DataFrameLike, SeriesLike]
+
+# UDF annotations
+PandasScalarUDFType = Literal[200]
+PandasScalarIterUDFType = Literal[204]
+PandasGroupedMapUDFType = Literal[201]
+PandasCogroupedMapUDFType = Literal[206]
+PandasGroupedAggUDFType = Literal[202]
+PandasMapIterUDFType = Literal[205]
+
+class PandasVariadicScalarToScalarFunction(Protocol):
+    def __call__(self, *_: DataFrameOrSeriesLike) -> SeriesLike: ...
+
+PandasScalarToScalarFunction = Union[
+    PandasVariadicScalarToScalarFunction,
+    Callable[[DataFrameOrSeriesLike], SeriesLike],
+    Callable[[DataFrameOrSeriesLike, DataFrameOrSeriesLike], SeriesLike],
+    Callable[
+        [DataFrameOrSeriesLike, DataFrameOrSeriesLike, DataFrameOrSeriesLike],
+        SeriesLike,
+    ],
+    Callable[
+        [
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+        ],
+        SeriesLike,
+    ],
+    Callable[
+        [
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+        ],
+        SeriesLike,
+    ],
+    Callable[
+        [
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+        ],
+        SeriesLike,
+    ],
+    Callable[
+        [
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+        ],
+        SeriesLike,
+    ],
+    Callable[
+        [
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+        ],
+        SeriesLike,
+    ],
+    Callable[
+        [
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+        ],
+        SeriesLike,
+    ],
+    Callable[
+        [
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+        ],
+        SeriesLike,
+    ],
+]
+
+class PandasVariadicScalarToStructFunction(Protocol):
+    def __call__(self, *_: DataFrameOrSeriesLike) -> DataFrameLike: ...
+
+PandasScalarToStructFunction = Union[
+    PandasVariadicScalarToStructFunction,
+    Callable[[DataFrameOrSeriesLike], DataFrameLike],
+    Callable[[DataFrameOrSeriesLike, DataFrameOrSeriesLike], DataFrameLike],
+    Callable[
+        [DataFrameOrSeriesLike, DataFrameOrSeriesLike, DataFrameOrSeriesLike],
+        DataFrameLike,
+    ],
+    Callable[
+        [
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+        ],
+        DataFrameLike,
+    ],
+    Callable[
+        [
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+        ],
+        DataFrameLike,
+    ],
+    Callable[
+        [
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+        ],
+        DataFrameLike,
+    ],
+    Callable[
+        [
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+        ],
+        DataFrameLike,
+    ],
+    Callable[
+        [
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+        ],
+        DataFrameLike,
+    ],
+    Callable[
+        [
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+        ],
+        DataFrameLike,
+    ],
+    Callable[
+        [
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+            DataFrameOrSeriesLike,
+        ],
+        DataFrameLike,
+    ],
+]
+
+PandasScalarIterFunction = Callable[
+    [Iterable[Union[DataFrameOrSeriesLike, Tuple[DataFrameOrSeriesLike, ...]]]],
+    Iterable[SeriesLike],
+]
+
+PandasGroupedMapFunction = Union[
+    Callable[[DataFrameLike], DataFrameLike],
+    Callable[[Any, DataFrameLike], DataFrameLike],
+]
+
+class PandasVariadicGroupedAggFunction(Protocol):
+    def __call__(self, *_: SeriesLike) -> LiteralType: ...
+
+PandasGroupedAggFunction = Union[
+    Callable[[SeriesLike], LiteralType],
+    Callable[[SeriesLike, SeriesLike], LiteralType],
+    Callable[[SeriesLike, SeriesLike, SeriesLike], LiteralType],
+    Callable[[SeriesLike, SeriesLike, SeriesLike, SeriesLike], LiteralType],
+    Callable[[SeriesLike, SeriesLike, SeriesLike, SeriesLike, SeriesLike], LiteralType],
+    Callable[
+        [SeriesLike, SeriesLike, SeriesLike, SeriesLike, SeriesLike, SeriesLike],
+        LiteralType,
+    ],
+    Callable[
+        [
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+        ],
+        LiteralType,
+    ],
+    Callable[
+        [
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+        ],
+        LiteralType,
+    ],
+    Callable[
+        [
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+        ],
+        LiteralType,
+    ],
+    Callable[
+        [
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+            SeriesLike,
+        ],
+        LiteralType,
+    ],
+    PandasVariadicGroupedAggFunction,
+]
+
+PandasMapIterFunction = Callable[[Iterable[DataFrameLike]], Iterable[DataFrameLike]]
+
+PandasCogroupedMapFunction = Callable[[DataFrameLike, DataFrameLike], DataFrameLike]
+
+MapIterPandasUserDefinedFunction = NewType(
+    "MapIterPandasUserDefinedFunction", FunctionType
+)
+GroupedMapPandasUserDefinedFunction = NewType(
+    "GroupedMapPandasUserDefinedFunction", FunctionType
+)
+CogroupedMapPandasUserDefinedFunction = NewType(
+    "CogroupedMapPandasUserDefinedFunction", FunctionType
+)
diff --git a/python/pyspark/sql/pandas/_typing/protocols/__init__.pyi b/python/pyspark/sql/pandas/_typing/protocols/__init__.pyi
new file mode 100644
index 0000000000000..217e5db960782
--- /dev/null
+++ b/python/pyspark/sql/pandas/_typing/protocols/__init__.pyi
@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/python/pyspark/sql/pandas/_typing/protocols/frame.pyi b/python/pyspark/sql/pandas/_typing/protocols/frame.pyi
new file mode 100644
index 0000000000000..9148e7a2dca8e
--- /dev/null
+++ b/python/pyspark/sql/pandas/_typing/protocols/frame.pyi
@@ -0,0 +1,428 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This Protocol reuses core Pandas annotation.
+# Overall pipeline looks as follows
+# - Stubgen pandas.core.frame
+# - Add Protocol as a base class
+# - Replace imports with Any
+
+import numpy.ma as np  # type: ignore[import]
+from typing import Any, Hashable, IO, Iterable, List, Optional, Sequence, Tuple, Union
+from typing_extensions import Protocol
+from .series import SeriesLike
+
+Axes = Any
+Dtype = Any
+Index = Any
+Renamer = Any
+Axis = Any
+Level = Any
+
+class DataFrameLike(Protocol):
+    def __init__(
+        self,
+        data: Any = ...,
+        index: Optional[Axes] = ...,
+        columns: Optional[Axes] = ...,
+        dtype: Optional[Dtype] = ...,
+        copy: bool = ...,
+    ) -> None: ...
+    @property
+    def axes(self) -> List[Index]: ...
+    @property
+    def shape(self) -> Tuple[int, int]: ...
+    @property
+    def style(self) -> Any: ...
+    def items(self) -> Iterable[Tuple[Optional[Hashable], SeriesLike]]: ...
+    def iteritems(self) -> Iterable[Tuple[Optional[Hashable], SeriesLike]]: ...
+    def iterrows(self) -> Iterable[Tuple[Optional[Hashable], SeriesLike]]: ...
+    def itertuples(self, index: bool = ..., name: str = ...): ...
+    def __len__(self) -> int: ...
+    def dot(self, other: Any): ...
+    def __matmul__(self, other: Any): ...
+    def __rmatmul__(self, other: Any): ...
+    @classmethod
+    def from_dict(
+        cls: Any, data: Any, orient: Any = ..., dtype: Any = ..., columns: Any = ...
+    ) -> DataFrameLike: ...
+    def to_numpy(self, dtype: Any = ..., copy: Any = ...) -> np.ndarray: ...
+    def to_dict(self, orient: str = ..., into: Any = ...): ...
+    def to_gbq(
+        self,
+        destination_table: Any,
+        project_id: Any = ...,
+        chunksize: Any = ...,
+        reauth: Any = ...,
+        if_exists: Any = ...,
+        auth_local_webserver: Any = ...,
+        table_schema: Any = ...,
+        location: Any = ...,
+        progress_bar: Any = ...,
+        credentials: Any = ...,
+    ) -> None: ...
+    @classmethod
+    def from_records(
+        cls: Any,
+        data: Any,
+        index: Any = ...,
+        exclude: Any = ...,
+        columns: Any = ...,
+        coerce_float: Any = ...,
+        nrows: Any = ...,
+    ) -> DataFrameLike: ...
+    def to_records(
+        self, index: Any = ..., column_dtypes: Any = ..., index_dtypes: Any = ...
+    ) -> np.recarray: ...
+    def to_stata(
+        self,
+        path: Any,
+        convert_dates: Optional[Any] = ...,
+        write_index: bool = ...,
+        byteorder: Optional[Any] = ...,
+        time_stamp: Optional[Any] = ...,
+        data_label: Optional[Any] = ...,
+        variable_labels: Optional[Any] = ...,
+        version: int = ...,
+        convert_strl: Optional[Any] = ...,
+    ) -> None: ...
+    def to_feather(self, path: Any) -> None: ...
+    def to_markdown(
+        self, buf: Optional[IO[str]] = ..., mode: Optional[str] = ..., **kwargs: Any
+    ) -> Optional[str]: ...
+    def to_parquet(
+        self,
+        path: Any,
+        engine: Any = ...,
+        compression: Any = ...,
+        index: Any = ...,
+        partition_cols: Any = ...,
+        **kwargs: Any
+    ) -> None: ...
+    def to_html(
+        self,
+        buf: Optional[Any] = ...,
+        columns: Optional[Any] = ...,
+        col_space: Optional[Any] = ...,
+        header: bool = ...,
+        index: bool = ...,
+        na_rep: str = ...,
+        formatters: Optional[Any] = ...,
+        float_format: Optional[Any] = ...,
+        sparsify: Optional[Any] = ...,
+        index_names: bool = ...,
+        justify: Optional[Any] = ...,
+        max_rows: Optional[Any] = ...,
+        max_cols: Optional[Any] = ...,
+        show_dimensions: bool = ...,
+        decimal: str = ...,
+        bold_rows: bool = ...,
+        classes: Optional[Any] = ...,
+        escape: bool = ...,
+        notebook: bool = ...,
+        border: Optional[Any] = ...,
+        table_id: Optional[Any] = ...,
+        render_links: bool = ...,
+        encoding: Optional[Any] = ...,
+    ): ...
+    def info(
+        self,
+        verbose: Any = ...,
+        buf: Any = ...,
+        max_cols: Any = ...,
+        memory_usage: Any = ...,
+        null_counts: Any = ...,
+    ) -> None: ...
+    def memory_usage(self, index: Any = ..., deep: Any = ...) -> SeriesLike: ...
+    def transpose(self, *args: Any, copy: bool = ...) -> DataFrameLike: ...
+    T: Any = ...
+    def __getitem__(self, key: Any): ...
+    def __setitem__(self, key: Any, value: Any): ...
+    def query(self, expr: Any, inplace: bool = ..., **kwargs: Any): ...
+    def eval(self, expr: Any, inplace: bool = ..., **kwargs: Any): ...
+    def select_dtypes(
+        self, include: Any = ..., exclude: Any = ...
+    ) -> DataFrameLike: ...
+    def insert(
+        self, loc: Any, column: Any, value: Any, allow_duplicates: Any = ...
+    ) -> None: ...
+    def assign(self, **kwargs: Any) -> DataFrameLike: ...
+    def lookup(self, row_labels: Any, col_labels: Any) -> np.ndarray: ...
+    def align(
+        self,
+        other: Any,
+        join: Any = ...,
+        axis: Any = ...,
+        level: Any = ...,
+        copy: Any = ...,
+        fill_value: Any = ...,
+        method: Any = ...,
+        limit: Any = ...,
+        fill_axis: Any = ...,
+        broadcast_axis: Any = ...,
+    ) -> DataFrameLike: ...
+    def reindex(self, *args: Any, **kwargs: Any) -> DataFrameLike: ...
+    def drop(
+        self,
+        labels: Optional[Any] = ...,
+        axis: int = ...,
+        index: Optional[Any] = ...,
+        columns: Optional[Any] = ...,
+        level: Optional[Any] = ...,
+        inplace: bool = ...,
+        errors: str = ...,
+    ): ...
+    def rename(
+        self,
+        mapper: Optional[Renamer] = ...,
+        *,
+        index: Optional[Renamer] = ...,
+        columns: Optional[Renamer] = ...,
+        axis: Optional[Axis] = ...,
+        copy: bool = ...,
+        inplace: bool = ...,
+        level: Optional[Level] = ...,
+        errors: str = ...
+    ) -> Optional[DataFrameLike]: ...
+    def fillna(
+        self,
+        value: Any = ...,
+        method: Any = ...,
+        axis: Any = ...,
+        inplace: Any = ...,
+        limit: Any = ...,
+        downcast: Any = ...,
+    ) -> Optional[DataFrameLike]: ...
+    def replace(
+        self,
+        to_replace: Optional[Any] = ...,
+        value: Optional[Any] = ...,
+        inplace: bool = ...,
+        limit: Optional[Any] = ...,
+        regex: bool = ...,
+        method: str = ...,
+    ): ...
+    def shift(
+        self,
+        periods: Any = ...,
+        freq: Any = ...,
+        axis: Any = ...,
+        fill_value: Any = ...,
+    ) -> DataFrameLike: ...
+    def set_index(
+        self,
+        keys: Any,
+        drop: bool = ...,
+        append: bool = ...,
+        inplace: bool = ...,
+        verify_integrity: bool = ...,
+    ): ...
+    def reset_index(
+        self,
+        level: Optional[Union[Hashable, Sequence[Hashable]]] = ...,
+        drop: bool = ...,
+        inplace: bool = ...,
+        col_level: Hashable = ...,
+        col_fill: Optional[Hashable] = ...,
+    ) -> Optional[DataFrameLike]: ...
+    def isna(self) -> DataFrameLike: ...
+    def isnull(self) -> DataFrameLike: ...
+    def notna(self) -> DataFrameLike: ...
+    def notnull(self) -> DataFrameLike: ...
+    def dropna(
+        self,
+        axis: int = ...,
+        how: str = ...,
+        thresh: Optional[Any] = ...,
+        subset: Optional[Any] = ...,
+        inplace: bool = ...,
+    ): ...
+    def drop_duplicates(
+        self,
+        subset: Optional[Union[Hashable, Sequence[Hashable]]] = ...,
+        keep: Union[str, bool] = ...,
+        inplace: bool = ...,
+        ignore_index: bool = ...,
+    ) -> Optional[DataFrameLike]: ...
+    def duplicated(
+        self,
+        subset: Optional[Union[Hashable, Sequence[Hashable]]] = ...,
+        keep: Union[str, bool] = ...,
+    ) -> SeriesLike: ...
+    def sort_values(
+        self,
+        by: Any,
+        axis: int = ...,
+        ascending: bool = ...,
+        inplace: bool = ...,
+        kind: str = ...,
+        na_position: str = ...,
+        ignore_index: bool = ...,
+    ): ...
+    def sort_index(
+        self,
+        axis: Any = ...,
+        level: Any = ...,
+        ascending: Any = ...,
+        inplace: Any = ...,
+        kind: Any = ...,
+        na_position: Any = ...,
+        sort_remaining: Any = ...,
+        ignore_index: bool = ...,
+    ) -> Any: ...
+    def nlargest(self, n: Any, columns: Any, keep: Any = ...) -> DataFrameLike: ...
+    def nsmallest(self, n: Any, columns: Any, keep: Any = ...) -> DataFrameLike: ...
+    def swaplevel(
+        self, i: Any = ..., j: Any = ..., axis: Any = ...
+    ) -> DataFrameLike: ...
+    def reorder_levels(self, order: Any, axis: Any = ...) -> DataFrameLike: ...
+    def combine(
+        self,
+        other: DataFrameLike,
+        func: Any,
+        fill_value: Any = ...,
+        overwrite: Any = ...,
+    ) -> DataFrameLike: ...
+    def combine_first(self, other: DataFrameLike) -> DataFrameLike: ...
+    def update(
+        self,
+        other: Any,
+        join: Any = ...,
+        overwrite: Any = ...,
+        filter_func: Any = ...,
+        errors: Any = ...,
+    ) -> None: ...
+    def groupby(
+        self,
+        by: Any = ...,
+        axis: Any = ...,
+        level: Any = ...,
+        as_index: bool = ...,
+        sort: bool = ...,
+        group_keys: bool = ...,
+        squeeze: bool = ...,
+        observed: bool = ...,
+    ) -> Any: ...
+    def pivot(
+        self, index: Any = ..., columns: Any = ..., values: Any = ...
+    ) -> DataFrameLike: ...
+    def pivot_table(
+        self,
+        values: Any = ...,
+        index: Any = ...,
+        columns: Any = ...,
+        aggfunc: Any = ...,
+        fill_value: Any = ...,
+        margins: Any = ...,
+        dropna: Any = ...,
+        margins_name: Any = ...,
+        observed: Any = ...,
+    ) -> DataFrameLike: ...
+    def stack(self, level: int = ..., dropna: bool = ...): ...
+    def explode(self, column: Union[str, Tuple]) -> DataFrameLike: ...
+    def unstack(self, level: int = ..., fill_value: Optional[Any] = ...): ...
+    def melt(
+        self,
+        id_vars: Any = ...,
+        value_vars: Any = ...,
+        var_name: Any = ...,
+        value_name: Any = ...,
+        col_level: Any = ...,
+    ) -> DataFrameLike: ...
+    def diff(self, periods: Any = ..., axis: Any = ...) -> DataFrameLike: ...
+    def aggregate(self, func: Any, axis: int = ..., *args: Any, **kwargs: Any): ...
+    agg: Any = ...
+    def transform(
+        self, func: Any, axis: Any = ..., *args: Any, **kwargs: Any
+    ) -> DataFrameLike: ...
+    def apply(
+        self,
+        func: Any,
+        axis: int = ...,
+        raw: bool = ...,
+        result_type: Optional[Any] = ...,
+        args: Any = ...,
+        **kwds: Any
+    ): ...
+    def applymap(self, func: Any) -> DataFrameLike: ...
+    def append(
+        self,
+        other: Any,
+        ignore_index: Any = ...,
+        verify_integrity: Any = ...,
+        sort: Any = ...,
+    ) -> DataFrameLike: ...
+    def join(
+        self,
+        other: Any,
+        on: Any = ...,
+        how: Any = ...,
+        lsuffix: Any = ...,
+        rsuffix: Any = ...,
+        sort: Any = ...,
+    ) -> DataFrameLike: ...
+    def merge(
+        self,
+        right: Any,
+        how: Any = ...,
+        on: Any = ...,
+        left_on: Any = ...,
+        right_on: Any = ...,
+        left_index: Any = ...,
+        right_index: Any = ...,
+        sort: Any = ...,
+        suffixes: Any = ...,
+        copy: Any = ...,
+        indicator: Any = ...,
+        validate: Any = ...,
+    ) -> DataFrameLike: ...
+    def round(
+        self, decimals: Any = ..., *args: Any, **kwargs: Any
+    ) -> DataFrameLike: ...
+    def corr(self, method: Any = ..., min_periods: Any = ...) -> DataFrameLike: ...
+    def cov(self, min_periods: Any = ...) -> DataFrameLike: ...
+    def corrwith(
+        self, other: Any, axis: Any = ..., drop: Any = ..., method: Any = ...
+    ) -> SeriesLike: ...
+    def count(
+        self, axis: int = ..., level: Optional[Any] = ..., numeric_only: bool = ...
+    ): ...
+    def nunique(self, axis: Any = ..., dropna: Any = ...) -> SeriesLike: ...
+    def idxmin(self, axis: Any = ..., skipna: Any = ...) -> SeriesLike: ...
+    def idxmax(self, axis: Any = ..., skipna: Any = ...) -> SeriesLike: ...
+    def mode(
+        self, axis: Any = ..., numeric_only: Any = ..., dropna: Any = ...
+    ) -> DataFrameLike: ...
+    def quantile(
+        self,
+        q: float = ...,
+        axis: int = ...,
+        numeric_only: bool = ...,
+        interpolation: str = ...,
+    ): ...
+    def to_timestamp(
+        self, freq: Any = ..., how: Any = ..., axis: Any = ..., copy: Any = ...
+    ) -> DataFrameLike: ...
+    def to_period(
+        self, freq: Any = ..., axis: Any = ..., copy: Any = ...
+    ) -> DataFrameLike: ...
+    def isin(self, values: Any) -> DataFrameLike: ...
+    plot: Any = ...
+    hist: Any = ...
+    boxplot: Any = ...
+    sparse: Any = ...
diff --git a/python/pyspark/sql/pandas/_typing/protocols/series.pyi b/python/pyspark/sql/pandas/_typing/protocols/series.pyi
new file mode 100644
index 0000000000000..f2de2e8b129fd
--- /dev/null
+++ b/python/pyspark/sql/pandas/_typing/protocols/series.pyi
@@ -0,0 +1,253 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This Protocol reuses core Pandas annotation.
+# Overall pipeline looks as follows
+# - Stubgen pandas.core.series
+# - Add Protocol as a base class
+# - Replace imports with Any
+
+import numpy as np  # type: ignore[import]
+from typing import Any, Callable, Hashable, IO, Optional
+from typing_extensions import Protocol
+
+groupby_generic = Any
+
+class SeriesLike(Protocol):
+    hasnans: Any = ...
+    div: Callable[[SeriesLike, Any], SeriesLike]
+    rdiv: Callable[[SeriesLike, Any], SeriesLike]
+    def __init__(
+        self,
+        data: Optional[Any] = ...,
+        index: Optional[Any] = ...,
+        dtype: Optional[Any] = ...,
+        name: Optional[Any] = ...,
+        copy: bool = ...,
+        fastpath: bool = ...,
+    ) -> None: ...
+    @property
+    def dtype(self): ...
+    @property
+    def dtypes(self): ...
+    @property
+    def name(self) -> Optional[Hashable]: ...
+    @name.setter
+    def name(self, value: Optional[Hashable]) -> None: ...
+    @property
+    def values(self): ...
+    def ravel(self, order: str = ...): ...
+    def __len__(self) -> int: ...
+    def view(self, dtype: Optional[Any] = ...): ...
+    def __array_ufunc__(
+        self, ufunc: Callable, method: str, *inputs: Any, **kwargs: Any
+    ) -> Any: ...
+    def __array__(self, dtype: Any = ...) -> np.ndarray: ...
+    __float__: Any = ...
+    __long__: Any = ...
+    __int__: Any = ...
+    @property
+    def axes(self): ...
+    def take(
+        self, indices: Any, axis: int = ..., is_copy: bool = ..., **kwargs: Any
+    ): ...
+    def __getitem__(self, key: Any): ...
+    def __setitem__(self, key: Any, value: Any) -> None: ...
+    def repeat(self, repeats: Any, axis: Optional[Any] = ...): ...
+    index: Any = ...
+    def reset_index(
+        self,
+        level: Optional[Any] = ...,
+        drop: bool = ...,
+        name: Optional[Any] = ...,
+        inplace: bool = ...,
+    ): ...
+    def to_string(
+        self,
+        buf: Optional[Any] = ...,
+        na_rep: str = ...,
+        float_format: Optional[Any] = ...,
+        header: bool = ...,
+        index: bool = ...,
+        length: bool = ...,
+        dtype: bool = ...,
+        name: bool = ...,
+        max_rows: Optional[Any] = ...,
+        min_rows: Optional[Any] = ...,
+    ): ...
+    def to_markdown(
+        self, buf: Optional[IO[str]] = ..., mode: Optional[str] = ..., **kwargs: Any
+    ) -> Optional[str]: ...
+    def items(self): ...
+    def iteritems(self): ...
+    def keys(self): ...
+    def to_dict(self, into: Any = ...): ...
+    def to_frame(self, name: Optional[Any] = ...): ...
+    def groupby(
+        self,
+        by: Any = ...,
+        axis: Any = ...,
+        level: Any = ...,
+        as_index: bool = ...,
+        sort: bool = ...,
+        group_keys: bool = ...,
+        squeeze: bool = ...,
+        observed: bool = ...,
+    ) -> Any: ...
+    def count(self, level: Optional[Any] = ...): ...
+    def mode(self, dropna: bool = ...): ...
+    def unique(self): ...
+    def drop_duplicates(self, keep: str = ..., inplace: bool = ...): ...
+    def duplicated(self, keep: str = ...): ...
+    def idxmin(
+        self, axis: int = ..., skipna: bool = ..., *args: Any, **kwargs: Any
+    ): ...
+    def idxmax(
+        self, axis: int = ..., skipna: bool = ..., *args: Any, **kwargs: Any
+    ): ...
+    def round(self, decimals: int = ..., *args: Any, **kwargs: Any): ...
+    def quantile(self, q: float = ..., interpolation: str = ...): ...
+    def corr(self, other: Any, method: str = ..., min_periods: Optional[Any] = ...): ...
+    def cov(self, other: Any, min_periods: Optional[Any] = ...): ...
+    def diff(self, periods: int = ...): ...
+    def autocorr(self, lag: int = ...): ...
+    def dot(self, other: Any): ...
+    def __matmul__(self, other: Any): ...
+    def __rmatmul__(self, other: Any): ...
+    def searchsorted(
+        self, value: Any, side: str = ..., sorter: Optional[Any] = ...
+    ): ...
+    def append(
+        self, to_append: Any, ignore_index: bool = ..., verify_integrity: bool = ...
+    ): ...
+    def combine(self, other: Any, func: Any, fill_value: Optional[Any] = ...): ...
+    def combine_first(self, other: Any): ...
+    def update(self, other: Any) -> None: ...
+    def sort_values(
+        self,
+        axis: int = ...,
+        ascending: bool = ...,
+        inplace: bool = ...,
+        kind: str = ...,
+        na_position: str = ...,
+        ignore_index: bool = ...,
+    ): ...
+    def sort_index(
+        self,
+        axis: Any = ...,
+        level: Any = ...,
+        ascending: Any = ...,
+        inplace: Any = ...,
+        kind: Any = ...,
+        na_position: Any = ...,
+        sort_remaining: Any = ...,
+        ignore_index: bool = ...,
+    ) -> Any: ...
+    def argsort(self, axis: int = ..., kind: str = ..., order: Optional[Any] = ...): ...
+    def nlargest(self, n: int = ..., keep: str = ...): ...
+    def nsmallest(self, n: int = ..., keep: str = ...): ...
+    def swaplevel(self, i: int = ..., j: int = ..., copy: bool = ...): ...
+    def reorder_levels(self, order: Any): ...
+    def explode(self) -> SeriesLike: ...
+    def unstack(self, level: int = ..., fill_value: Optional[Any] = ...): ...
+    def map(self, arg: Any, na_action: Optional[Any] = ...): ...
+    def aggregate(self, func: Any, axis: int = ..., *args: Any, **kwargs: Any): ...
+    agg: Any = ...
+    def transform(self, func: Any, axis: int = ..., *args: Any, **kwargs: Any): ...
+    def apply(
+        self, func: Any, convert_dtype: bool = ..., args: Any = ..., **kwds: Any
+    ): ...
+    def align(
+        self,
+        other: Any,
+        join: str = ...,
+        axis: Optional[Any] = ...,
+        level: Optional[Any] = ...,
+        copy: bool = ...,
+        fill_value: Optional[Any] = ...,
+        method: Optional[Any] = ...,
+        limit: Optional[Any] = ...,
+        fill_axis: int = ...,
+        broadcast_axis: Optional[Any] = ...,
+    ): ...
+    def rename(
+        self,
+        index: Optional[Any] = ...,
+        *,
+        axis: Optional[Any] = ...,
+        copy: bool = ...,
+        inplace: bool = ...,
+        level: Optional[Any] = ...,
+        errors: str = ...
+    ): ...
+    def reindex(self, index: Optional[Any] = ..., **kwargs: Any): ...
+    def drop(
+        self,
+        labels: Optional[Any] = ...,
+        axis: int = ...,
+        index: Optional[Any] = ...,
+        columns: Optional[Any] = ...,
+        level: Optional[Any] = ...,
+        inplace: bool = ...,
+        errors: str = ...,
+    ): ...
+    def fillna(
+        self,
+        value: Any = ...,
+        method: Any = ...,
+        axis: Any = ...,
+        inplace: Any = ...,
+        limit: Any = ...,
+        downcast: Any = ...,
+    ) -> Optional[SeriesLike]: ...
+    def replace(
+        self,
+        to_replace: Optional[Any] = ...,
+        value: Optional[Any] = ...,
+        inplace: bool = ...,
+        limit: Optional[Any] = ...,
+        regex: bool = ...,
+        method: str = ...,
+    ): ...
+    def shift(
+        self,
+        periods: int = ...,
+        freq: Optional[Any] = ...,
+        axis: int = ...,
+        fill_value: Optional[Any] = ...,
+    ): ...
+    def memory_usage(self, index: bool = ..., deep: bool = ...): ...
+    def isin(self, values: Any): ...
+    def between(self, left: Any, right: Any, inclusive: bool = ...): ...
+    def isna(self): ...
+    def isnull(self): ...
+    def notna(self): ...
+    def notnull(self): ...
+    def dropna(
+        self, axis: int = ..., inplace: bool = ..., how: Optional[Any] = ...
+    ): ...
+    def to_timestamp(
+        self, freq: Optional[Any] = ..., how: str = ..., copy: bool = ...
+    ): ...
+    def to_period(self, freq: Optional[Any] = ..., copy: bool = ...): ...
+    str: Any = ...
+    dt: Any = ...
+    cat: Any = ...
+    plot: Any = ...
+    sparse: Any = ...
+    hist: Any = ...
diff --git a/python/pyspark/sql/pandas/conversion.py b/python/pyspark/sql/pandas/conversion.py
index 251625ae412cf..92ef7ce313026 100644
--- a/python/pyspark/sql/pandas/conversion.py
+++ b/python/pyspark/sql/pandas/conversion.py
@@ -16,20 +16,14 @@
 #
 import sys
 import warnings
-if sys.version >= '3':
-    basestring = unicode = str
-    xrange = range
-else:
-    from itertools import izip as zip
 from collections import Counter
 
-from pyspark import since
 from pyspark.rdd import _load_from_socket
 from pyspark.sql.pandas.serializers import ArrowCollectSerializer
 from pyspark.sql.types import IntegralType
-from pyspark.sql.types import *
+from pyspark.sql.types import ByteType, ShortType, IntegerType, LongType, FloatType, \
+    DoubleType, BooleanType, MapType, TimestampType, StructType, DataType
 from pyspark.traceback_utils import SCCallSiteSync
-from pyspark.util import _exception_message
 
 
 class PandasConversionMixin(object):
@@ -38,18 +32,23 @@ class PandasConversionMixin(object):
     can use this class.
     """
 
-    @since(1.3)
     def toPandas(self):
         """
         Returns the contents of this :class:`DataFrame` as Pandas ``pandas.DataFrame``.
 
         This is only available if Pandas is installed and available.
 
-        .. note:: This method should only be used if the resulting Pandas's :class:`DataFrame` is
-            expected to be small, as all the data is loaded into the driver's memory.
+        .. versionadded:: 1.3.0
 
-        .. note:: Usage with spark.sql.execution.arrow.pyspark.enabled=True is experimental.
+        Notes
+        -----
+        This method should only be used if the resulting Pandas's :class:`DataFrame` is
+        expected to be small, as all the data is loaded into the driver's memory.
 
+        Usage with spark.sql.execution.arrow.pyspark.enabled=True is experimental.
+
+        Examples
+        --------
         >>> df.toPandas()  # doctest: +SKIP
            age   name
         0    2  Alice
@@ -84,7 +83,7 @@ def toPandas(self):
                         "failed by the reason below:\n  %s\n"
                         "Attempting non-optimization as "
                         "'spark.sql.execution.arrow.pyspark.fallback.enabled' is set to "
-                        "true." % _exception_message(e))
+                        "true." % str(e))
                     warnings.warn(msg)
                     use_arrow = False
                 else:
@@ -93,7 +92,7 @@ def toPandas(self):
                         "'spark.sql.execution.arrow.pyspark.enabled' is set to true, but has "
                         "reached the error below and will not continue because automatic fallback "
                         "with 'spark.sql.execution.arrow.pyspark.fallback.enabled' has been set to "
-                        "false.\n  %s" % _exception_message(e))
+                        "false.\n  %s" % str(e))
                     warnings.warn(msg)
                     raise
 
@@ -101,23 +100,43 @@ def toPandas(self):
             # of PyArrow is found, if 'spark.sql.execution.arrow.pyspark.enabled' is enabled.
             if use_arrow:
                 try:
-                    from pyspark.sql.pandas.types import _check_series_localize_timestamps
+                    from pyspark.sql.pandas.types import _check_series_localize_timestamps, \
+                        _convert_map_items_to_dict
                     import pyarrow
                     # Rename columns to avoid duplicated column names.
                     tmp_column_names = ['col_{}'.format(i) for i in range(len(self.columns))]
-                    batches = self.toDF(*tmp_column_names)._collect_as_arrow()
+                    self_destruct = self.sql_ctx._conf.arrowPySparkSelfDestructEnabled()
+                    batches = self.toDF(*tmp_column_names)._collect_as_arrow(
+                        split_batches=self_destruct)
                     if len(batches) > 0:
                         table = pyarrow.Table.from_batches(batches)
+                        # Ensure only the table has a reference to the batches, so that
+                        # self_destruct (if enabled) is effective
+                        del batches
                         # Pandas DataFrame created from PyArrow uses datetime64[ns] for date type
                         # values, but we should use datetime.date to match the behavior with when
                         # Arrow optimization is disabled.
-                        pdf = table.to_pandas(date_as_object=True)
+                        pandas_options = {'date_as_object': True}
+                        if self_destruct:
+                            # Configure PyArrow to use as little memory as possible:
+                            # self_destruct - free columns as they are converted
+                            # split_blocks - create a separate Pandas block for each column
+                            # use_threads - convert one column at a time
+                            pandas_options.update({
+                                'self_destruct': True,
+                                'split_blocks': True,
+                                'use_threads': False,
+                            })
+                        pdf = table.to_pandas(**pandas_options)
                         # Rename back to the original column names.
                         pdf.columns = self.columns
                         for field in self.schema:
                             if isinstance(field.dataType, TimestampType):
                                 pdf[field.name] = \
                                     _check_series_localize_timestamps(pdf[field.name], timezone)
+                            elif isinstance(field.dataType, MapType):
+                                pdf[field.name] = \
+                                    _convert_map_items_to_dict(pdf[field.name])
                         return pdf
                     else:
                         return pd.DataFrame.from_records([], columns=self.columns)
@@ -130,7 +149,7 @@ def toPandas(self):
                         "reached the error below and can not continue. Note that "
                         "'spark.sql.execution.arrow.pyspark.fallback.enabled' does not have an "
                         "effect on failures in the middle of "
-                        "computation.\n  %s" % _exception_message(e))
+                        "computation.\n  %s" % str(e))
                     warnings.warn(msg)
                     raise
 
@@ -222,10 +241,14 @@ def _to_corrected_pandas_type(dt):
         else:
             return None
 
-    def _collect_as_arrow(self):
+    def _collect_as_arrow(self, split_batches=False):
         """
         Returns all records as a list of ArrowRecordBatches, pyarrow must be installed
         and available on driver and worker Python environments.
+        This is an experimental feature.
+
+        :param split_batches: split batches such that each column is in its own allocation, so
+            that the selfDestruct optimization is effective; default False.
 
         .. note:: Experimental.
         """
@@ -238,7 +261,26 @@ def _collect_as_arrow(self):
 
         # Collect list of un-ordered batches where last element is a list of correct order indices
         try:
-            results = list(_load_from_socket((port, auth_secret), ArrowCollectSerializer()))
+            batch_stream = _load_from_socket((port, auth_secret), ArrowCollectSerializer())
+            if split_batches:
+                # When spark.sql.execution.arrow.pyspark.selfDestruct.enabled, ensure
+                # each column in each record batch is contained in its own allocation.
+                # Otherwise, selfDestruct does nothing; it frees each column as its
+                # converted, but each column will actually be a list of slices of record
+                # batches, and so no memory is actually freed until all columns are
+                # converted.
+                import pyarrow as pa
+                results = []
+                for batch_or_indices in batch_stream:
+                    if isinstance(batch_or_indices, pa.RecordBatch):
+                        batch_or_indices = pa.RecordBatch.from_arrays([
+                            # This call actually reallocates the array
+                            pa.concat_arrays([array])
+                            for array in batch_or_indices
+                        ], schema=batch_or_indices.schema)
+                    results.append(batch_or_indices)
+            else:
+                results = list(batch_stream)
         finally:
             # Join serving thread and raise any exceptions from collectAsArrowToPython
             jsocket_auth_server.getResult()
@@ -268,7 +310,7 @@ def createDataFrame(self, data, schema=None, samplingRatio=None, verifySchema=Tr
 
         # If no schema supplied by user then get the names of columns only
         if schema is None:
-            schema = [str(x) if not isinstance(x, basestring) else
+            schema = [str(x) if not isinstance(x, str) else
                       (x.encode('utf-8') if not isinstance(x, str) else x)
                       for x in data.columns]
 
@@ -276,8 +318,6 @@ def createDataFrame(self, data, schema=None, samplingRatio=None, verifySchema=Tr
             try:
                 return self._create_from_pandas_with_arrow(data, schema, timezone)
             except Exception as e:
-                from pyspark.util import _exception_message
-
                 if self._wrapped._conf.arrowPySparkFallbackEnabled():
                     msg = (
                         "createDataFrame attempted Arrow optimization because "
@@ -285,7 +325,7 @@ def createDataFrame(self, data, schema=None, samplingRatio=None, verifySchema=Tr
                         "failed by the reason below:\n  %s\n"
                         "Attempting non-optimization as "
                         "'spark.sql.execution.arrow.pyspark.fallback.enabled' is set to "
-                        "true." % _exception_message(e))
+                        "true." % str(e))
                     warnings.warn(msg)
                 else:
                     msg = (
@@ -293,7 +333,7 @@ def createDataFrame(self, data, schema=None, samplingRatio=None, verifySchema=Tr
                         "'spark.sql.execution.arrow.pyspark.enabled' is set to true, but has "
                         "reached the error below and will not continue because automatic "
                         "fallback with 'spark.sql.execution.arrow.pyspark.fallback.enabled' "
-                        "has been set to false.\n  %s" % _exception_message(e))
+                        "has been set to false.\n  %s" % str(e))
                     warnings.warn(msg)
                     raise
         data = self._convert_from_pandas(data, schema, timezone)
@@ -302,7 +342,11 @@ def createDataFrame(self, data, schema=None, samplingRatio=None, verifySchema=Tr
     def _convert_from_pandas(self, pdf, schema, timezone):
         """
          Convert a pandas.DataFrame to list of records that can be used to make a DataFrame
-         :return list of records
+
+         Returns
+         -------
+         list
+             list of records
         """
         from pyspark.sql import SparkSession
 
@@ -350,15 +394,23 @@ def _get_numpy_record_dtype(self, rec):
         """
         Used when converting a pandas.DataFrame to Spark using to_records(), this will correct
         the dtypes of fields in a record so they can be properly loaded into Spark.
-        :param rec: a numpy record to check field dtypes
-        :return corrected dtype for a numpy.record or None if no correction needed
+
+        Parameters
+        ----------
+        rec : numpy.record
+            a numpy record to check field dtypes
+
+        Returns
+        -------
+        numpy.dtype
+            corrected dtype for a numpy.record or None if no correction needed
         """
         import numpy as np
         cur_dtypes = rec.dtype
         col_names = cur_dtypes.names
         record_type_list = []
         has_rec_fix = False
-        for i in xrange(len(cur_dtypes)):
+        for i in range(len(cur_dtypes)):
             curr_type = cur_dtypes[i]
             # If type is a datetime64 timestamp, convert to microseconds
             # NOTE: if dtype is datetime[ns] then np.record.tolist() will output values as longs,
@@ -413,7 +465,7 @@ def _create_from_pandas_with_arrow(self, pdf, schema, timezone):
 
         # Slice the DataFrame to be batched
         step = -(-len(pdf) // self.sparkContext.defaultParallelism)  # round int up
-        pdf_slices = (pdf[start:start + step] for start in xrange(0, len(pdf), step))
+        pdf_slices = (pdf.iloc[start:start + step] for start in range(0, len(pdf), step))
 
         # Create list of Arrow (columns, type) for serializer dump_stream
         arrow_data = [[(c, t) for (_, c), t in zip(pdf_slice.iteritems(), arrow_types)]
diff --git a/python/pyspark/sql/pandas/conversion.pyi b/python/pyspark/sql/pandas/conversion.pyi
new file mode 100644
index 0000000000000..031852fcc053d
--- /dev/null
+++ b/python/pyspark/sql/pandas/conversion.pyi
@@ -0,0 +1,58 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import overload
+from typing import Optional, Union
+
+from pyspark.sql.pandas._typing import DataFrameLike
+from pyspark import since as since  # noqa: F401
+from pyspark.rdd import RDD  # noqa: F401
+import pyspark.sql.dataframe
+from pyspark.sql.pandas.serializers import (  # noqa: F401
+    ArrowCollectSerializer as ArrowCollectSerializer,
+)
+from pyspark.sql.types import (  # noqa: F401
+    BooleanType as BooleanType,
+    ByteType as ByteType,
+    DataType as DataType,
+    DoubleType as DoubleType,
+    FloatType as FloatType,
+    IntegerType as IntegerType,
+    IntegralType as IntegralType,
+    LongType as LongType,
+    ShortType as ShortType,
+    StructType as StructType,
+    TimestampType as TimestampType,
+)
+from pyspark.traceback_utils import SCCallSiteSync as SCCallSiteSync  # noqa: F401
+
+class PandasConversionMixin:
+    def toPandas(self) -> DataFrameLike: ...
+
+class SparkConversionMixin:
+    @overload
+    def createDataFrame(
+        self, data: DataFrameLike, samplingRatio: Optional[float] = ...
+    ) -> pyspark.sql.dataframe.DataFrame: ...
+    @overload
+    def createDataFrame(
+        self,
+        data: DataFrameLike,
+        schema: Union[StructType, str],
+        verifySchema: bool = ...,
+    ) -> pyspark.sql.dataframe.DataFrame: ...
diff --git a/python/pyspark/sql/pandas/functions.py b/python/pyspark/sql/pandas/functions.py
index 094dc357b6822..4cd0b196d3366 100644
--- a/python/pyspark/sql/pandas/functions.py
+++ b/python/pyspark/sql/pandas/functions.py
@@ -16,16 +16,14 @@
 #
 
 import functools
-import sys
 import warnings
+from inspect import getfullargspec
 
-from pyspark import since
 from pyspark.rdd import PythonEvalType
 from pyspark.sql.pandas.typehints import infer_eval_type
 from pyspark.sql.pandas.utils import require_minimum_pandas_version, require_minimum_pyarrow_version
 from pyspark.sql.types import DataType
 from pyspark.sql.udf import _create_udf
-from pyspark.util import _get_argspec
 
 
 class PandasUDFType(object):
@@ -40,7 +38,6 @@ class PandasUDFType(object):
     GROUPED_AGG = PythonEvalType.SQL_GROUPED_AGG_PANDAS_UDF
 
 
-@since(2.3)
 def pandas_udf(f=None, returnType=None, functionType=None):
     """
     Creates a pandas user defined function (a.k.a. vectorized user defined function).
@@ -51,14 +48,22 @@ def pandas_udf(f=None, returnType=None, functionType=None):
     additional configuration is required. A Pandas UDF behaves as a regular PySpark function
     API in general.
 
-    :param f: user-defined function. A python function if used as a standalone function
-    :param returnType: the return type of the user-defined function. The value can be either a
-        :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string.
-    :param functionType: an enum value in :class:`pyspark.sql.functions.PandasUDFType`.
-        Default: SCALAR.
+    .. versionadded:: 2.3.0
 
-        .. note:: This parameter exists for compatibility. Using Python type hints is encouraged.
+    Parameters
+    ----------
+    f : function, optional
+        user-defined function. A python function if used as a standalone function
+    returnType : :class:`pyspark.sql.types.DataType` or str, optional
+        the return type of the user-defined function. The value can be either a
+        :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string.
+    functionType : int, optional
+        an enum value in :class:`pyspark.sql.functions.PandasUDFType`.
+        Default: SCALAR. This parameter exists for compatibility.
+        Using Python type hints is encouraged.
 
+    Examples
+    --------
     In order to use this API, customarily the below are imported:
 
     >>> import pandas as pd
@@ -94,7 +99,7 @@ def pandas_udf(f=None, returnType=None, functionType=None):
     ...     s3['col2'] = s1 + s2.str.len()
     ...     return s3
     ...
-    >>> # Create a Spark DataFrame that has three columns including a sturct column.
+    >>> # Create a Spark DataFrame that has three columns including a struct column.
     ... df = spark.createDataFrame(
     ...     [[1, "a string", ("a nested string",)]],
     ...     "long_col long, string_col string, struct_col struct<col1:string>")
@@ -109,7 +114,7 @@ def pandas_udf(f=None, returnType=None, functionType=None):
     |    |-- col1: string (nullable = true)
     |    |-- col2: long (nullable = true)
 
-    In the following sections, it describes the cominations of the supported type hints. For
+    In the following sections, it describes the combinations of the supported type hints. For
     simplicity, `pandas.DataFrame` variant is omitted.
 
     * Series to Series
@@ -264,30 +269,32 @@ def calculate(iterator: Iterator[pd.Series]) -> Iterator[pd.Series]:
             Therefore, mutating the input series is not allowed and will cause incorrect results.
             For the same reason, users should also not rely on the index of the input series.
 
-        .. seealso:: :meth:`pyspark.sql.GroupedData.agg` and :class:`pyspark.sql.Window`
-
-    .. note:: The user-defined functions do not support conditional expressions or short circuiting
-        in boolean expressions and it ends up with being executed all internally. If the functions
-        can fail on special rows, the workaround is to incorporate the condition into the functions.
-
-    .. note:: The user-defined functions do not take keyword arguments on the calling side.
-
-    .. note:: The data type of returned `pandas.Series` from the user-defined functions should be
-        matched with defined `returnType` (see :meth:`types.to_arrow_type` and
-        :meth:`types.from_arrow_type`). When there is mismatch between them, Spark might do
-        conversion on returned data. The conversion is not guaranteed to be correct and results
-        should be checked for accuracy by users.
-
-    .. note:: Currently,
-        :class:`pyspark.sql.types.MapType`,
-        :class:`pyspark.sql.types.ArrayType` of :class:`pyspark.sql.types.TimestampType` and
-        nested :class:`pyspark.sql.types.StructType`
-        are currently not supported as output types.
-
-    .. seealso:: :meth:`pyspark.sql.DataFrame.mapInPandas`
-    .. seealso:: :meth:`pyspark.sql.GroupedData.applyInPandas`
-    .. seealso:: :meth:`pyspark.sql.PandasCogroupedOps.applyInPandas`
-    .. seealso:: :meth:`pyspark.sql.UDFRegistration.register`
+    Notes
+    -----
+    The user-defined functions do not support conditional expressions or short circuiting
+    in boolean expressions and it ends up with being executed all internally. If the functions
+    can fail on special rows, the workaround is to incorporate the condition into the functions.
+
+    The user-defined functions do not take keyword arguments on the calling side.
+
+    The data type of returned `pandas.Series` from the user-defined functions should be
+    matched with defined `returnType` (see :meth:`types.to_arrow_type` and
+    :meth:`types.from_arrow_type`). When there is mismatch between them, Spark might do
+    conversion on returned data. The conversion is not guaranteed to be correct and results
+    should be checked for accuracy by users.
+
+    Currently,
+    :class:`pyspark.sql.types.ArrayType` of :class:`pyspark.sql.types.TimestampType` and
+    nested :class:`pyspark.sql.types.StructType`
+    are currently not supported as output types.
+
+    See Also
+    --------
+    pyspark.sql.GroupedData.agg
+    pyspark.sql.DataFrame.mapInPandas
+    pyspark.sql.GroupedData.applyInPandas
+    pyspark.sql.PandasCogroupedOps.applyInPandas
+    pyspark.sql.UDFRegistration.register
     """
 
     # The following table shows most of Pandas data and SQL type conversions in Pandas UDFs that
@@ -295,30 +302,30 @@ def calculate(iterator: Iterator[pd.Series]) -> Iterator[pd.Series]:
     # future. The table might have to be eventually documented externally.
     # Please see SPARK-28132's PR to see the codes in order to generate the table below.
     #
-    # +-----------------------------+----------------------+------------------+------------------+------------------+--------------------+--------------------+------------------+------------------+------------------+------------------+--------------+--------------+--------------+-----------------------------------+-----------------------------------------------------+-----------------+--------------------+-----------------------------+--------------+-----------------+------------------+-----------+--------------------------------+  # noqa
-    # |SQL Type \ Pandas Value(Type)|None(object(NoneType))|        True(bool)|           1(int8)|          1(int16)|            1(int32)|            1(int64)|          1(uint8)|         1(uint16)|         1(uint32)|         1(uint64)|  1.0(float16)|  1.0(float32)|  1.0(float64)|1970-01-01 00:00:00(datetime64[ns])|1970-01-01 00:00:00-05:00(datetime64[ns, US/Eastern])|a(object(string))|  1(object(Decimal))|[1 2 3](object(array[int32]))| 1.0(float128)|(1+0j)(complex64)|(1+0j)(complex128)|A(category)|1 days 00:00:00(timedelta64[ns])|  # noqa
-    # +-----------------------------+----------------------+------------------+------------------+------------------+--------------------+--------------------+------------------+------------------+------------------+------------------+--------------+--------------+--------------+-----------------------------------+-----------------------------------------------------+-----------------+--------------------+-----------------------------+--------------+-----------------+------------------+-----------+--------------------------------+  # noqa
-    # |                      boolean|                  None|              True|              True|              True|                True|                True|              True|              True|              True|              True|          True|          True|          True|                                  X|                                                    X|                X|                   X|                            X|             X|                X|                 X|          X|                               X|  # noqa
-    # |                      tinyint|                  None|                 1|                 1|                 1|                   1|                   1|                 1|                 1|                 1|                 1|             1|             1|             1|                                  X|                                                    X|                X|                   1|                            X|             X|                X|                 X|          0|                               X|  # noqa
-    # |                     smallint|                  None|                 1|                 1|                 1|                   1|                   1|                 1|                 1|                 1|                 1|             1|             1|             1|                                  X|                                                    X|                X|                   1|                            X|             X|                X|                 X|          X|                               X|  # noqa
-    # |                          int|                  None|                 1|                 1|                 1|                   1|                   1|                 1|                 1|                 1|                 1|             1|             1|             1|                                  X|                                                    X|                X|                   1|                            X|             X|                X|                 X|          X|                               X|  # noqa
-    # |                       bigint|                  None|                 1|                 1|                 1|                   1|                   1|                 1|                 1|                 1|                 1|             1|             1|             1|                                  0|                                       18000000000000|                X|                   1|                            X|             X|                X|                 X|          X|                               X|  # noqa
-    # |                        float|                  None|               1.0|               1.0|               1.0|                 1.0|                 1.0|               1.0|               1.0|               1.0|               1.0|           1.0|           1.0|           1.0|                                  X|                                                    X|                X|                   X|                            X|             X|                X|                 X|          X|                               X|  # noqa
-    # |                       double|                  None|               1.0|               1.0|               1.0|                 1.0|                 1.0|               1.0|               1.0|               1.0|               1.0|           1.0|           1.0|           1.0|                                  X|                                                    X|                X|                   X|                            X|             X|                X|                 X|          X|                               X|  # noqa
-    # |                         date|                  None|                 X|                 X|                 X|datetime.date(197...|                   X|                 X|                 X|                 X|                 X|             X|             X|             X|               datetime.date(197...|                                 datetime.date(197...|                X|datetime.date(197...|                            X|             X|                X|                 X|          X|                               X|  # noqa
-    # |                    timestamp|                  None|                 X|                 X|                 X|                   X|datetime.datetime...|                 X|                 X|                 X|                 X|             X|             X|             X|               datetime.datetime...|                                 datetime.datetime...|                X|datetime.datetime...|                            X|             X|                X|                 X|          X|                               X|  # noqa
-    # |                       string|                  None|                ''|                ''|                ''|              '\x01'|              '\x01'|                ''|                ''|            '\x01'|            '\x01'|            ''|            ''|            ''|                                  X|                                                    X|              'a'|                   X|                            X|            ''|                X|                ''|          X|                               X|  # noqa
-    # |                decimal(10,0)|                  None|                 X|                 X|                 X|                   X|                   X|                 X|                 X|                 X|                 X|             X|             X|             X|                                  X|                                                    X|                X|        Decimal('1')|                            X|             X|                X|                 X|          X|                               X|  # noqa
-    # |                   array<int>|                  None|                 X|                 X|                 X|                   X|                   X|                 X|                 X|                 X|                 X|             X|             X|             X|                                  X|                                                    X|                X|                   X|                    [1, 2, 3]|             X|                X|                 X|          X|                               X|  # noqa
-    # |              map<string,int>|                     X|                 X|                 X|                 X|                   X|                   X|                 X|                 X|                 X|                 X|             X|             X|             X|                                  X|                                                    X|                X|                   X|                            X|             X|                X|                 X|          X|                               X|  # noqa
-    # |               struct<_1:int>|                     X|                 X|                 X|                 X|                   X|                   X|                 X|                 X|                 X|                 X|             X|             X|             X|                                  X|                                                    X|                X|                   X|                            X|             X|                X|                 X|          X|                               X|  # noqa
-    # |                       binary|                  None|bytearray(b'\x01')|bytearray(b'\x01')|bytearray(b'\x01')|  bytearray(b'\x01')|  bytearray(b'\x01')|bytearray(b'\x01')|bytearray(b'\x01')|bytearray(b'\x01')|bytearray(b'\x01')|bytearray(b'')|bytearray(b'')|bytearray(b'')|                     bytearray(b'')|                                       bytearray(b'')|  bytearray(b'a')|                   X|                            X|bytearray(b'')|   bytearray(b'')|    bytearray(b'')|          X|                  bytearray(b'')|  # noqa
-    # +-----------------------------+----------------------+------------------+------------------+------------------+--------------------+--------------------+------------------+------------------+------------------+------------------+--------------+--------------+--------------+-----------------------------------+-----------------------------------------------------+-----------------+--------------------+-----------------------------+--------------+-----------------+------------------+-----------+--------------------------------+  # noqa
+    # +-----------------------------+----------------------+------------------+------------------+------------------+--------------------+--------------------+------------------+------------------+------------------+------------------+--------------+--------------+--------------+-----------------------------------+-----------------------------------------------------+-----------------+--------------------+-----------------------------+--------------+-----------------+------------------+---------------+--------------------------------+  # noqa
+    # |SQL Type \ Pandas Value(Type)|None(object(NoneType))|        True(bool)|           1(int8)|          1(int16)|            1(int32)|            1(int64)|          1(uint8)|         1(uint16)|         1(uint32)|         1(uint64)|  1.0(float16)|  1.0(float32)|  1.0(float64)|1970-01-01 00:00:00(datetime64[ns])|1970-01-01 00:00:00-05:00(datetime64[ns, US/Eastern])|a(object(string))|  1(object(Decimal))|[1 2 3](object(array[int32]))| 1.0(float128)|(1+0j)(complex64)|(1+0j)(complex128)|    A(category)|1 days 00:00:00(timedelta64[ns])|  # noqa
+    # +-----------------------------+----------------------+------------------+------------------+------------------+--------------------+--------------------+------------------+------------------+------------------+------------------+--------------+--------------+--------------+-----------------------------------+-----------------------------------------------------+-----------------+--------------------+-----------------------------+--------------+-----------------+------------------+---------------+--------------------------------+  # noqa
+    # |                      boolean|                  None|              True|              True|              True|                True|                True|              True|              True|              True|              True|          True|          True|          True|                                  X|                                                    X|                X|                   X|                            X|             X|                X|                 X|              X|                               X|  # noqa
+    # |                      tinyint|                  None|                 1|                 1|                 1|                   1|                   1|                 1|                 1|                 1|                 1|             1|             1|             1|                                  X|                                                    X|                X|                   1|                            X|             X|                X|                 X|              X|                               X|  # noqa
+    # |                     smallint|                  None|                 1|                 1|                 1|                   1|                   1|                 1|                 1|                 1|                 1|             1|             1|             1|                                  X|                                                    X|                X|                   1|                            X|             X|                X|                 X|              X|                               X|  # noqa
+    # |                          int|                  None|                 1|                 1|                 1|                   1|                   1|                 1|                 1|                 1|                 1|             1|             1|             1|                                  X|                                                    X|                X|                   1|                            X|             X|                X|                 X|              X|                               X|  # noqa
+    # |                       bigint|                  None|                 1|                 1|                 1|                   1|                   1|                 1|                 1|                 1|                 1|             1|             1|             1|                                  0|                                       18000000000000|                X|                   1|                            X|             X|                X|                 X|              X|                  86400000000000|  # noqa
+    # |                        float|                  None|               1.0|               1.0|               1.0|                 1.0|                 1.0|               1.0|               1.0|               1.0|               1.0|           1.0|           1.0|           1.0|                                  X|                                                    X|                X|                   X|                            X|             X|                X|                 X|              X|                               X|  # noqa
+    # |                       double|                  None|               1.0|               1.0|               1.0|                 1.0|                 1.0|               1.0|               1.0|               1.0|               1.0|           1.0|           1.0|           1.0|                                  X|                                                    X|                X|                   X|                            X|             X|                X|                 X|              X|                               X|  # noqa
+    # |                         date|                  None|                 X|                 X|                 X|datetime.date(197...|                   X|                 X|                 X|                 X|                 X|             X|             X|             X|               datetime.date(197...|                                 datetime.date(197...|                X|datetime.date(197...|                            X|             X|                X|                 X|              X|                               X|  # noqa
+    # |                    timestamp|                  None|                 X|                 X|                 X|                   X|datetime.datetime...|                 X|                 X|                 X|                 X|             X|             X|             X|               datetime.datetime...|                                 datetime.datetime...|                X|datetime.datetime...|                            X|             X|                X|                 X|              X|                               X|  # noqa
+    # |                       string|                  None|                 X|                 X|                 X|                   X|                   X|                 X|                 X|                 X|                 X|             X|             X|             X|                                  X|                                                    X|              'a'|                   X|                            X|             X|                X|                 X|            'A'|                               X|  # noqa
+    # |                decimal(10,0)|                  None|                 X|                 X|                 X|                   X|                   X|                 X|                 X|                 X|                 X|             X|             X|             X|                                  X|                                                    X|                X|        Decimal('1')|                            X|             X|                X|                 X|              X|                               X|  # noqa
+    # |                   array<int>|                  None|                 X|                 X|                 X|                   X|                   X|                 X|                 X|                 X|                 X|             X|             X|             X|                                  X|                                                    X|                X|                   X|                    [1, 2, 3]|             X|                X|                 X|              X|                               X|  # noqa
+    # |              map<string,int>|                     X|                 X|                 X|                 X|                   X|                   X|                 X|                 X|                 X|                 X|             X|             X|             X|                                  X|                                                    X|                X|                   X|                            X|             X|                X|                 X|              X|                               X|  # noqa
+    # |               struct<_1:int>|                     X|                 X|                 X|                 X|                   X|                   X|                 X|                 X|                 X|                 X|             X|             X|             X|                                  X|                                                    X|                X|                   X|                            X|             X|                X|                 X|              X|                               X|  # noqa
+    # |                       binary|                  None|bytearray(b'\x01')|bytearray(b'\x01')|bytearray(b'\x01')|  bytearray(b'\x01')|  bytearray(b'\x01')|bytearray(b'\x01')|bytearray(b'\x01')|bytearray(b'\x01')|bytearray(b'\x01')|bytearray(b'')|bytearray(b'')|bytearray(b'')|                     bytearray(b'')|                                       bytearray(b'')|  bytearray(b'a')|                   X|                            X|bytearray(b'')|   bytearray(b'')|    bytearray(b'')|bytearray(b'A')|                  bytearray(b'')|  # noqa
+    # +-----------------------------+----------------------+------------------+------------------+------------------+--------------------+--------------------+------------------+------------------+------------------+------------------+--------------+--------------+--------------+-----------------------------------+-----------------------------------------------------+-----------------+--------------------+-----------------------------+--------------+-----------------+------------------+---------------+--------------------------------+  # noqa    #
     #
     # Note: DDL formatted string is used for 'SQL Type' for simplicity. This string can be
     #       used in `returnType`.
     # Note: The values inside of the table are generated by `repr`.
-    # Note: Python 3.7.3, Pandas 0.24.2 and PyArrow 0.13.0 are used.
+    # Note: Python 3.7.3, Pandas 1.1.1 and PyArrow 1.0.1 are used.
     # Note: Timezone is KST.
     # Note: 'X' means it throws an exception during the conversion.
     require_minimum_pandas_version()
@@ -371,30 +378,29 @@ def calculate(iterator: Iterator[pd.Series]) -> Iterator[pd.Series]:
 
 
 def _create_pandas_udf(f, returnType, evalType):
-    argspec = _get_argspec(f)
+    argspec = getfullargspec(f)
 
     # pandas UDF by type hints.
-    if sys.version_info >= (3, 6):
-        from inspect import signature
-
-        if evalType in [PythonEvalType.SQL_SCALAR_PANDAS_UDF,
-                        PythonEvalType.SQL_SCALAR_PANDAS_ITER_UDF,
-                        PythonEvalType.SQL_GROUPED_AGG_PANDAS_UDF]:
-            warnings.warn(
-                "In Python 3.6+ and Spark 3.0+, it is preferred to specify type hints for "
-                "pandas UDF instead of specifying pandas UDF type which will be deprecated "
-                "in the future releases. See SPARK-28264 for more details.", UserWarning)
-        elif evalType in [PythonEvalType.SQL_GROUPED_MAP_PANDAS_UDF,
-                          PythonEvalType.SQL_MAP_PANDAS_ITER_UDF,
-                          PythonEvalType.SQL_COGROUPED_MAP_PANDAS_UDF]:
-            # In case of 'SQL_GROUPED_MAP_PANDAS_UDF',  deprecation warning is being triggered
-            # at `apply` instead.
-            # In case of 'SQL_MAP_PANDAS_ITER_UDF' and 'SQL_COGROUPED_MAP_PANDAS_UDF', the
-            # evaluation type will always be set.
-            pass
-        elif len(argspec.annotations) > 0:
-            evalType = infer_eval_type(signature(f))
-            assert evalType is not None
+    from inspect import signature
+
+    if evalType in [PythonEvalType.SQL_SCALAR_PANDAS_UDF,
+                    PythonEvalType.SQL_SCALAR_PANDAS_ITER_UDF,
+                    PythonEvalType.SQL_GROUPED_AGG_PANDAS_UDF]:
+        warnings.warn(
+            "In Python 3.6+ and Spark 3.0+, it is preferred to specify type hints for "
+            "pandas UDF instead of specifying pandas UDF type which will be deprecated "
+            "in the future releases. See SPARK-28264 for more details.", UserWarning)
+    elif evalType in [PythonEvalType.SQL_GROUPED_MAP_PANDAS_UDF,
+                      PythonEvalType.SQL_MAP_PANDAS_ITER_UDF,
+                      PythonEvalType.SQL_COGROUPED_MAP_PANDAS_UDF]:
+        # In case of 'SQL_GROUPED_MAP_PANDAS_UDF',  deprecation warning is being triggered
+        # at `apply` instead.
+        # In case of 'SQL_MAP_PANDAS_ITER_UDF' and 'SQL_COGROUPED_MAP_PANDAS_UDF', the
+        # evaluation type will always be set.
+        pass
+    elif len(argspec.annotations) > 0:
+        evalType = infer_eval_type(signature(f))
+        assert evalType is not None
 
     if evalType is None:
         # Set default is scalar UDF.
diff --git a/python/pyspark/sql/pandas/functions.pyi b/python/pyspark/sql/pandas/functions.pyi
new file mode 100644
index 0000000000000..09318e43f8aa1
--- /dev/null
+++ b/python/pyspark/sql/pandas/functions.pyi
@@ -0,0 +1,176 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import overload
+from typing import Union, Callable
+
+from pyspark.sql._typing import (
+    AtomicDataTypeOrString,
+    UserDefinedFunctionLike,
+)
+from pyspark.sql.pandas._typing import (
+    GroupedMapPandasUserDefinedFunction,
+    MapIterPandasUserDefinedFunction,
+    CogroupedMapPandasUserDefinedFunction,
+    PandasCogroupedMapFunction,
+    PandasCogroupedMapUDFType,
+    PandasGroupedAggFunction,
+    PandasGroupedAggUDFType,
+    PandasGroupedMapFunction,
+    PandasGroupedMapUDFType,
+    PandasMapIterFunction,
+    PandasMapIterUDFType,
+    PandasScalarIterFunction,
+    PandasScalarIterUDFType,
+    PandasScalarToScalarFunction,
+    PandasScalarToStructFunction,
+    PandasScalarUDFType,
+)
+
+from pyspark import since as since  # noqa: F401
+from pyspark.rdd import PythonEvalType as PythonEvalType  # noqa: F401
+from pyspark.sql.types import ArrayType, StructType
+
+class PandasUDFType:
+    SCALAR: PandasScalarUDFType
+    SCALAR_ITER: PandasScalarIterUDFType
+    GROUPED_MAP: PandasGroupedMapUDFType
+    GROUPED_AGG: PandasGroupedAggUDFType
+
+@overload
+def pandas_udf(
+    f: PandasScalarToScalarFunction,
+    returnType: Union[AtomicDataTypeOrString, ArrayType],
+    functionType: PandasScalarUDFType,
+) -> UserDefinedFunctionLike: ...
+@overload
+def pandas_udf(f: Union[AtomicDataTypeOrString, ArrayType], returnType: PandasScalarUDFType) -> Callable[[PandasScalarToScalarFunction], UserDefinedFunctionLike]: ...  # type: ignore[misc]
+@overload
+def pandas_udf(f: Union[AtomicDataTypeOrString, ArrayType], *, functionType: PandasScalarUDFType) -> Callable[[PandasScalarToScalarFunction], UserDefinedFunctionLike]: ...  # type: ignore[misc]
+@overload
+def pandas_udf(*, returnType: Union[AtomicDataTypeOrString, ArrayType], functionType: PandasScalarUDFType) -> Callable[[PandasScalarToScalarFunction], UserDefinedFunctionLike]: ...  # type: ignore[misc]
+@overload
+def pandas_udf(
+    f: PandasScalarToStructFunction,
+    returnType: Union[StructType, str],
+    functionType: PandasScalarUDFType,
+) -> UserDefinedFunctionLike: ...
+@overload
+def pandas_udf(f: Union[StructType, str], returnType: PandasScalarUDFType) -> Callable[[PandasScalarToStructFunction], UserDefinedFunctionLike]: ...  # type: ignore[misc]
+@overload
+def pandas_udf(f: Union[StructType, str], *, functionType: PandasScalarUDFType) -> Callable[[PandasScalarToStructFunction], UserDefinedFunctionLike]: ...  # type: ignore[misc]
+@overload
+def pandas_udf(*, returnType: Union[StructType, str], functionType: PandasScalarUDFType) -> Callable[[PandasScalarToStructFunction], UserDefinedFunctionLike]: ...  # type: ignore[misc]
+@overload
+def pandas_udf(
+    f: PandasScalarIterFunction,
+    returnType: Union[AtomicDataTypeOrString, ArrayType],
+    functionType: PandasScalarIterUDFType,
+) -> UserDefinedFunctionLike: ...
+@overload
+def pandas_udf(
+    f: Union[AtomicDataTypeOrString, ArrayType], returnType: PandasScalarIterUDFType
+) -> Callable[[PandasScalarIterFunction], UserDefinedFunctionLike]: ...
+@overload
+def pandas_udf(
+    *,
+    returnType: Union[AtomicDataTypeOrString, ArrayType],
+    functionType: PandasScalarIterUDFType
+) -> Callable[[PandasScalarIterFunction], UserDefinedFunctionLike]: ...
+@overload
+def pandas_udf(
+    f: Union[AtomicDataTypeOrString, ArrayType],
+    *,
+    functionType: PandasScalarIterUDFType
+) -> Callable[[PandasScalarIterFunction], UserDefinedFunctionLike]: ...
+@overload
+def pandas_udf(
+    f: PandasGroupedMapFunction,
+    returnType: Union[StructType, str],
+    functionType: PandasGroupedMapUDFType,
+) -> GroupedMapPandasUserDefinedFunction: ...
+@overload
+def pandas_udf(
+    f: Union[StructType, str], returnType: PandasGroupedMapUDFType
+) -> Callable[[PandasGroupedMapFunction], GroupedMapPandasUserDefinedFunction]: ...
+@overload
+def pandas_udf(
+    *, returnType: Union[StructType, str], functionType: PandasGroupedMapUDFType
+) -> Callable[[PandasGroupedMapFunction], GroupedMapPandasUserDefinedFunction]: ...
+@overload
+def pandas_udf(
+    f: Union[StructType, str], *, functionType: PandasGroupedMapUDFType
+) -> Callable[[PandasGroupedMapFunction], GroupedMapPandasUserDefinedFunction]: ...
+@overload
+def pandas_udf(
+    f: PandasGroupedAggFunction,
+    returnType: Union[AtomicDataTypeOrString, ArrayType],
+    functionType: PandasGroupedAggUDFType,
+) -> UserDefinedFunctionLike: ...
+@overload
+def pandas_udf(
+    f: Union[AtomicDataTypeOrString, ArrayType], returnType: PandasGroupedAggUDFType
+) -> Callable[[PandasGroupedAggFunction], UserDefinedFunctionLike]: ...
+@overload
+def pandas_udf(
+    *,
+    returnType: Union[AtomicDataTypeOrString, ArrayType],
+    functionType: PandasGroupedAggUDFType
+) -> Callable[[PandasGroupedAggFunction], UserDefinedFunctionLike]: ...
+@overload
+def pandas_udf(
+    f: Union[AtomicDataTypeOrString, ArrayType],
+    *,
+    functionType: PandasGroupedAggUDFType
+) -> Callable[[PandasGroupedAggFunction], UserDefinedFunctionLike]: ...
+@overload
+def pandas_udf(
+    f: PandasMapIterFunction,
+    returnType: Union[StructType, str],
+    functionType: PandasMapIterUDFType,
+) -> MapIterPandasUserDefinedFunction: ...
+@overload
+def pandas_udf(
+    f: Union[StructType, str], returnType: PandasMapIterUDFType
+) -> Callable[[PandasMapIterFunction], MapIterPandasUserDefinedFunction]: ...
+@overload
+def pandas_udf(
+    *, returnType: Union[StructType, str], functionType: PandasMapIterUDFType
+) -> Callable[[PandasMapIterFunction], MapIterPandasUserDefinedFunction]: ...
+@overload
+def pandas_udf(
+    f: Union[StructType, str], *, functionType: PandasMapIterUDFType
+) -> Callable[[PandasMapIterFunction], MapIterPandasUserDefinedFunction]: ...
+@overload
+def pandas_udf(
+    f: PandasCogroupedMapFunction,
+    returnType: Union[StructType, str],
+    functionType: PandasCogroupedMapUDFType,
+) -> CogroupedMapPandasUserDefinedFunction: ...
+@overload
+def pandas_udf(
+    f: Union[StructType, str], returnType: PandasCogroupedMapUDFType
+) -> Callable[[PandasCogroupedMapFunction], CogroupedMapPandasUserDefinedFunction]: ...
+@overload
+def pandas_udf(
+    *, returnType: Union[StructType, str], functionType: PandasCogroupedMapUDFType
+) -> Callable[[PandasCogroupedMapFunction], CogroupedMapPandasUserDefinedFunction]: ...
+@overload
+def pandas_udf(
+    f: Union[StructType, str], *, functionType: PandasCogroupedMapUDFType
+) -> Callable[[PandasCogroupedMapFunction], CogroupedMapPandasUserDefinedFunction]: ...
diff --git a/python/pyspark/sql/pandas/group_ops.py b/python/pyspark/sql/pandas/group_ops.py
index b93f0516cadb1..8d4f67e2c7502 100644
--- a/python/pyspark/sql/pandas/group_ops.py
+++ b/python/pyspark/sql/pandas/group_ops.py
@@ -17,7 +17,6 @@
 import sys
 import warnings
 
-from pyspark import since
 from pyspark.rdd import PythonEvalType
 from pyspark.sql.column import Column
 from pyspark.sql.dataframe import DataFrame
@@ -29,19 +28,27 @@ class PandasGroupedOpsMixin(object):
     can use this class.
     """
 
-    @since(2.3)
     def apply(self, udf):
         """
         It is an alias of :meth:`pyspark.sql.GroupedData.applyInPandas`; however, it takes a
         :meth:`pyspark.sql.functions.pandas_udf` whereas
         :meth:`pyspark.sql.GroupedData.applyInPandas` takes a Python native function.
 
-        .. note:: It is preferred to use :meth:`pyspark.sql.GroupedData.applyInPandas` over this
-            API. This API will be deprecated in the future releases.
+        .. versionadded:: 2.3.0
 
-        :param udf: a grouped map user-defined function returned by
+        Parameters
+        ----------
+        udf : :func:`pyspark.sql.functions.pandas_udf`
+            a grouped map user-defined function returned by
             :func:`pyspark.sql.functions.pandas_udf`.
 
+        Notes
+        -----
+        It is preferred to use :meth:`pyspark.sql.GroupedData.applyInPandas` over this
+        API. This API will be deprecated in the future releases.
+
+        Examples
+        --------
         >>> from pyspark.sql.functions import pandas_udf, PandasUDFType
         >>> df = spark.createDataFrame(
         ...     [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)],
@@ -61,8 +68,9 @@ def apply(self, udf):
         |  2| 1.1094003924504583|
         +---+-------------------+
 
-        .. seealso:: :meth:`pyspark.sql.functions.pandas_udf`
-
+        See Also
+        --------
+        pyspark.sql.functions.pandas_udf
         """
         # Columns are special because hasattr always return True
         if isinstance(udf, Column) or not hasattr(udf, 'func') \
@@ -77,7 +85,6 @@ def apply(self, udf):
 
         return self.applyInPandas(udf.func, schema=udf.returnType)
 
-    @since(3.0)
     def applyInPandas(self, func, schema):
         """
         Maps each group of the current :class:`DataFrame` using a pandas udf and returns the result
@@ -94,11 +101,19 @@ def applyInPandas(self, func, schema):
         field data types by position if not strings, e.g. integer indices.
         The length of the returned `pandas.DataFrame` can be arbitrary.
 
-        :param func: a Python native function that takes a `pandas.DataFrame`, and outputs a
+        .. versionadded:: 3.0.0
+
+        Parameters
+        ----------
+        func : function
+            a Python native function that takes a `pandas.DataFrame`, and outputs a
             `pandas.DataFrame`.
-        :param schema: the return type of the `func` in PySpark. The value can be either a
+        schema : :class:`pyspark.sql.types.DataType` or str
+            the return type of the `func` in PySpark. The value can be either a
             :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string.
 
+        Examples
+        --------
         >>> import pandas as pd  # doctest: +SKIP
         >>> from pyspark.sql.functions import pandas_udf, ceil
         >>> df = spark.createDataFrame(
@@ -141,6 +156,7 @@ def applyInPandas(self, func, schema):
         |  1|1.5|
         |  2|6.0|
         +---+---+
+
         >>> def sum_func(key, pdf):
         ...     # key is a tuple of two numpy.int64s, which is the values
         ...     # of 'id' and 'ceil(df.v / 2)' for the current group
@@ -156,19 +172,23 @@ def applyInPandas(self, func, schema):
         |  2|          2| 3.0|
         +---+-----------+----+
 
-        .. note:: This function requires a full shuffle. All the data of a group will be loaded
-            into memory, so the user should be aware of the potential OOM risk if data is skewed
-            and certain groups are too large to fit in memory.
+        Notes
+        -----
+        This function requires a full shuffle. All the data of a group will be loaded
+        into memory, so the user should be aware of the potential OOM risk if data is skewed
+        and certain groups are too large to fit in memory.
 
-        .. note:: If returning a new `pandas.DataFrame` constructed with a dictionary, it is
-            recommended to explicitly index the columns by name to ensure the positions are correct,
-            or alternatively use an `OrderedDict`.
-            For example, `pd.DataFrame({'id': ids, 'a': data}, columns=['id', 'a'])` or
-            `pd.DataFrame(OrderedDict([('id', ids), ('a', data)]))`.
+        If returning a new `pandas.DataFrame` constructed with a dictionary, it is
+        recommended to explicitly index the columns by name to ensure the positions are correct,
+        or alternatively use an `OrderedDict`.
+        For example, `pd.DataFrame({'id': ids, 'a': data}, columns=['id', 'a'])` or
+        `pd.DataFrame(OrderedDict([('id', ids), ('a', data)]))`.
 
-        .. note:: Experimental
+        This API is experimental.
 
-        .. seealso:: :meth:`pyspark.sql.functions.pandas_udf`
+        See Also
+        --------
+        pyspark.sql.functions.pandas_udf
         """
         from pyspark.sql import GroupedData
         from pyspark.sql.functions import pandas_udf, PandasUDFType
@@ -182,12 +202,13 @@ def applyInPandas(self, func, schema):
         jdf = self._jgd.flatMapGroupsInPandas(udf_column._jc.expr())
         return DataFrame(jdf, self.sql_ctx)
 
-    @since(3.0)
     def cogroup(self, other):
         """
         Cogroups this group with another group so that we can run cogrouped operations.
 
-        See :class:`CoGroupedData` for the operations that can be run.
+        .. versionadded:: 3.0.0
+
+        See :class:`PandasCogroupedOps` for the operations that can be run.
         """
         from pyspark.sql import GroupedData
 
@@ -201,9 +222,11 @@ class PandasCogroupedOps(object):
     A logical grouping of two :class:`GroupedData`,
     created by :func:`GroupedData.cogroup`.
 
-    .. note:: Experimental
+    .. versionadded:: 3.0.0
 
-    .. versionadded:: 3.0
+    Notes
+    -----
+    This API is experimental.
     """
 
     def __init__(self, gd1, gd2):
@@ -211,7 +234,6 @@ def __init__(self, gd1, gd2):
         self._gd2 = gd2
         self.sql_ctx = gd1.sql_ctx
 
-    @since(3.0)
     def applyInPandas(self, func, schema):
         """
         Applies a function to each cogroup using pandas and returns the result
@@ -228,12 +250,20 @@ def applyInPandas(self, func, schema):
         field data types by position if not strings, e.g. integer indices.
         The length of the returned `pandas.DataFrame` can be arbitrary.
 
-        :param func: a Python native function that takes two `pandas.DataFrame`\\s, and
+        .. versionadded:: 3.0.0
+
+        Parameters
+        ----------
+        func : function
+            a Python native function that takes two `pandas.DataFrame`\\s, and
             outputs a `pandas.DataFrame`, or that takes one tuple (grouping keys) and two
-            pandas ``DataFrame``s, and outputs a pandas ``DataFrame``.
-        :param schema: the return type of the `func` in PySpark. The value can be either a
+            pandas ``DataFrame``\\s, and outputs a pandas ``DataFrame``.
+        schema : :class:`pyspark.sql.types.DataType` or str
+            the return type of the `func` in PySpark. The value can be either a
             :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string.
 
+        Examples
+        --------
         >>> from pyspark.sql.functions import pandas_udf
         >>> df1 = spark.createDataFrame(
         ...     [(20000101, 1, 1.0), (20000101, 2, 2.0), (20000102, 1, 3.0), (20000102, 2, 4.0)],
@@ -275,20 +305,23 @@ def applyInPandas(self, func, schema):
         |20000102|  1|3.0|  x|
         +--------+---+---+---+
 
-        .. note:: This function requires a full shuffle. All the data of a cogroup will be loaded
-            into memory, so the user should be aware of the potential OOM risk if data is skewed
-            and certain groups are too large to fit in memory.
-
-        .. note:: If returning a new `pandas.DataFrame` constructed with a dictionary, it is
-            recommended to explicitly index the columns by name to ensure the positions are correct,
-            or alternatively use an `OrderedDict`.
-            For example, `pd.DataFrame({'id': ids, 'a': data}, columns=['id', 'a'])` or
-            `pd.DataFrame(OrderedDict([('id', ids), ('a', data)]))`.
+        Notes
+        -----
+        This function requires a full shuffle. All the data of a cogroup will be loaded
+        into memory, so the user should be aware of the potential OOM risk if data is skewed
+        and certain groups are too large to fit in memory.
 
-        .. note:: Experimental
+        If returning a new `pandas.DataFrame` constructed with a dictionary, it is
+        recommended to explicitly index the columns by name to ensure the positions are correct,
+        or alternatively use an `OrderedDict`.
+        For example, `pd.DataFrame({'id': ids, 'a': data}, columns=['id', 'a'])` or
+        `pd.DataFrame(OrderedDict([('id', ids), ('a', data)]))`.
 
-        .. seealso:: :meth:`pyspark.sql.functions.pandas_udf`
+        This API is experimental.
 
+        See Also
+        --------
+        pyspark.sql.functions.pandas_udf
         """
         from pyspark.sql.pandas.functions import pandas_udf
 
diff --git a/python/pyspark/sql/pandas/group_ops.pyi b/python/pyspark/sql/pandas/group_ops.pyi
new file mode 100644
index 0000000000000..2c543e0dc77b9
--- /dev/null
+++ b/python/pyspark/sql/pandas/group_ops.pyi
@@ -0,0 +1,49 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Union
+
+from pyspark.sql.pandas._typing import (
+    GroupedMapPandasUserDefinedFunction,
+    PandasGroupedMapFunction,
+    PandasCogroupedMapFunction,
+)
+
+from pyspark import since as since  # noqa: F401
+from pyspark.rdd import PythonEvalType as PythonEvalType  # noqa: F401
+from pyspark.sql.column import Column as Column  # noqa: F401
+from pyspark.sql.context import SQLContext
+import pyspark.sql.group
+from pyspark.sql.dataframe import DataFrame as DataFrame
+from pyspark.sql.types import StructType
+
+class PandasGroupedOpsMixin:
+    def cogroup(self, other: pyspark.sql.group.GroupedData) -> PandasCogroupedOps: ...
+    def apply(self, udf: GroupedMapPandasUserDefinedFunction) -> DataFrame: ...
+    def applyInPandas(
+        self, func: PandasGroupedMapFunction, schema: Union[StructType, str]
+    ) -> DataFrame: ...
+
+class PandasCogroupedOps:
+    sql_ctx: SQLContext
+    def __init__(
+        self, gd1: pyspark.sql.group.GroupedData, gd2: pyspark.sql.group.GroupedData
+    ) -> None: ...
+    def applyInPandas(
+        self, func: PandasCogroupedMapFunction, schema: Union[StructType, str]
+    ) -> DataFrame: ...
diff --git a/python/pyspark/sql/pandas/map_ops.py b/python/pyspark/sql/pandas/map_ops.py
index 9835e88c6ac21..63fe37113e7cc 100644
--- a/python/pyspark/sql/pandas/map_ops.py
+++ b/python/pyspark/sql/pandas/map_ops.py
@@ -16,7 +16,6 @@
 #
 import sys
 
-from pyspark import since
 from pyspark.rdd import PythonEvalType
 
 
@@ -26,7 +25,6 @@ class PandasMapOpsMixin(object):
     can use this class.
     """
 
-    @since(3.0)
     def mapInPandas(self, func, schema):
         """
         Maps an iterator of batches in the current :class:`DataFrame` using a Python native
@@ -40,11 +38,19 @@ def mapInPandas(self, func, schema):
         Each `pandas.DataFrame` size can be controlled by
         `spark.sql.execution.arrow.maxRecordsPerBatch`.
 
-        :param func: a Python native function that takes an iterator of `pandas.DataFrame`\\s, and
+        .. versionadded:: 3.0.0
+
+        Parameters
+        ----------
+        func : function
+            a Python native function that takes an iterator of `pandas.DataFrame`\\s, and
             outputs an iterator of `pandas.DataFrame`\\s.
-        :param schema: the return type of the `func` in PySpark. The value can be either a
+        schema : :class:`pyspark.sql.types.DataType` or str
+            the return type of the `func` in PySpark. The value can be either a
             :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string.
 
+        Examples
+        --------
         >>> from pyspark.sql.functions import pandas_udf
         >>> df = spark.createDataFrame([(1, 21), (2, 30)], ("id", "age"))
         >>> def filter_func(iterator):
@@ -57,9 +63,13 @@ def mapInPandas(self, func, schema):
         |  1| 21|
         +---+---+
 
-        .. seealso:: :meth:`pyspark.sql.functions.pandas_udf`
+        Notes
+        -----
+        This API is experimental
 
-        .. note:: Experimental
+        See Also
+        --------
+        pyspark.sql.functions.pandas_udf
         """
         from pyspark.sql import DataFrame
         from pyspark.sql.pandas.functions import pandas_udf
diff --git a/python/pyspark/sql/pandas/map_ops.pyi b/python/pyspark/sql/pandas/map_ops.pyi
new file mode 100644
index 0000000000000..cab885278c388
--- /dev/null
+++ b/python/pyspark/sql/pandas/map_ops.pyi
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Union
+
+from pyspark.sql.pandas._typing import PandasMapIterFunction
+from pyspark import since as since  # noqa: F401
+from pyspark.rdd import PythonEvalType as PythonEvalType  # noqa: F401
+from pyspark.sql.types import StructType
+import pyspark.sql.dataframe
+
+class PandasMapOpsMixin:
+    def mapInPandas(
+        self, udf: PandasMapIterFunction, schema: Union[StructType, str]
+    ) -> pyspark.sql.dataframe.DataFrame: ...
diff --git a/python/pyspark/sql/pandas/serializers.py b/python/pyspark/sql/pandas/serializers.py
index 4dd15d14b9c53..2dcfdc1046049 100644
--- a/python/pyspark/sql/pandas/serializers.py
+++ b/python/pyspark/sql/pandas/serializers.py
@@ -19,13 +19,6 @@
 Serializers for PyArrow and pandas conversions. See `pyspark.serializers` for more details.
 """
 
-import sys
-if sys.version < '3':
-    from itertools import izip as zip
-else:
-    basestring = unicode = str
-    xrange = range
-
 from pyspark.serializers import Serializer, read_int, write_int, UTF8Deserializer
 
 
@@ -67,7 +60,7 @@ def load_stream(self, stream):
             raise RuntimeError("An error occurred while calling "
                                "ArrowCollectSerializer.load_stream: {}".format(error_msg))
         batch_order = []
-        for i in xrange(num):
+        for i in range(num):
             index = read_int(stream)
             batch_order.append(index)
         yield batch_order
@@ -107,9 +100,14 @@ class ArrowStreamPandasSerializer(ArrowStreamSerializer):
     """
     Serializes Pandas.Series as Arrow data with Arrow streaming format.
 
-    :param timezone: A timezone to respect when handling timestamp values
-    :param safecheck: If True, conversion from Arrow to Pandas checks for overflow/truncation
-    :param assign_cols_by_name: If True, then Pandas DataFrames will get columns by name
+    Parameters
+    ----------
+    timezone : str
+        A timezone to respect when handling timestamp values
+    safecheck : bool
+        If True, conversion from Arrow to Pandas checks for overflow/truncation
+    assign_cols_by_name : bool
+        If True, then Pandas DataFrames will get columns by name
     """
 
     def __init__(self, timezone, safecheck, assign_cols_by_name):
@@ -119,7 +117,8 @@ def __init__(self, timezone, safecheck, assign_cols_by_name):
         self._assign_cols_by_name = assign_cols_by_name
 
     def arrow_to_pandas(self, arrow_column):
-        from pyspark.sql.pandas.types import _check_series_localize_timestamps
+        from pyspark.sql.pandas.types import _check_series_localize_timestamps, \
+            _convert_map_items_to_dict
         import pyarrow
 
         # If the given column is a date type column, creates a series of datetime.date directly
@@ -129,6 +128,8 @@ def arrow_to_pandas(self, arrow_column):
 
         if pyarrow.types.is_timestamp(arrow_column.type):
             return _check_series_localize_timestamps(s, self._timezone)
+        elif pyarrow.types.is_map(arrow_column.type):
+            return _convert_map_items_to_dict(s)
         else:
             return s
 
@@ -137,12 +138,21 @@ def _create_batch(self, series):
         Create an Arrow record batch from the given pandas.Series or list of Series,
         with optional type.
 
-        :param series: A single pandas.Series, list of Series, or list of (series, arrow_type)
-        :return: Arrow RecordBatch
+        Parameters
+        ----------
+        series : pandas.Series or list
+            A single series, list of series, or list of (series, arrow_type)
+
+        Returns
+        -------
+        pyarrow.RecordBatch
+            Arrow RecordBatch
         """
         import pandas as pd
         import pyarrow as pa
-        from pyspark.sql.pandas.types import _check_series_convert_timestamps_internal
+        from pyspark.sql.pandas.types import _check_series_convert_timestamps_internal, \
+            _convert_dict_to_map_items
+        from pandas.api.types import is_categorical_dtype
         # Make input conform to [(series1, type1), (series2, type2), ...]
         if not isinstance(series, (list, tuple)) or \
                 (len(series) == 2 and isinstance(series[1], pa.DataType)):
@@ -154,15 +164,23 @@ def create_array(s, t):
             # Ensure timestamp series are in expected form for Spark internal representation
             if t is not None and pa.types.is_timestamp(t):
                 s = _check_series_convert_timestamps_internal(s, self._timezone)
+            elif t is not None and pa.types.is_map(t):
+                s = _convert_dict_to_map_items(s)
+            elif is_categorical_dtype(s.dtype):
+                # Note: This can be removed once minimum pyarrow version is >= 0.16.1
+                s = s.astype(s.dtypes.categories.dtype)
             try:
                 array = pa.Array.from_pandas(s, mask=mask, type=t, safe=self._safecheck)
-            except pa.ArrowException as e:
-                error_msg = "Exception thrown when converting pandas.Series (%s) to Arrow " + \
-                            "Array (%s). It can be caused by overflows or other unsafe " + \
-                            "conversions warned by Arrow. Arrow safe type check can be " + \
-                            "disabled by using SQL config " + \
-                            "`spark.sql.execution.pandas.convertToArrowArraySafely`."
-                raise RuntimeError(error_msg % (s.dtype, t), e)
+            except ValueError as e:
+                if self._safecheck:
+                    error_msg = "Exception thrown when converting pandas.Series (%s) to " + \
+                                "Arrow Array (%s). It can be caused by overflows or other " + \
+                                "unsafe conversions warned by Arrow. Arrow safe type check " + \
+                                "can be disabled by using SQL config " + \
+                                "`spark.sql.execution.pandas.convertToArrowArraySafely`."
+                    raise ValueError(error_msg % (s.dtype, t)) from e
+                else:
+                    raise e
             return array
 
         arrs = []
@@ -176,7 +194,7 @@ def create_array(s, t):
                 if len(s) == 0 and len(s.columns) == 0:
                     arrs_names = [(pa.array([], type=field.type), field.name) for field in t]
                 # Assign result columns by schema name if user labeled with strings
-                elif self._assign_cols_by_name and any(isinstance(name, basestring)
+                elif self._assign_cols_by_name and any(isinstance(name, str)
                                                        for name in s.columns):
                     arrs_names = [(create_array(s[field.name], field.type), field.name)
                                   for field in t]
@@ -190,7 +208,7 @@ def create_array(s, t):
             else:
                 arrs.append(create_array(s, t))
 
-        return pa.RecordBatch.from_arrays(arrs, ["_%d" % i for i in xrange(len(arrs))])
+        return pa.RecordBatch.from_arrays(arrs, ["_%d" % i for i in range(len(arrs))])
 
     def dump_stream(self, iterator, stream):
         """
diff --git a/python/pyspark/sql/pandas/typehints.py b/python/pyspark/sql/pandas/typehints.py
index b0323ba1697df..e696f677cd154 100644
--- a/python/pyspark/sql/pandas/typehints.py
+++ b/python/pyspark/sql/pandas/typehints.py
@@ -98,8 +98,8 @@ def infer_eval_type(sig):
                 a,
                 parameter_check_func=lambda ua: ua == pd.Series or ua == pd.DataFrame)
             for a in parameters_sig) and (
-            # It's tricky to whitelist which types pd.Series constructor can take.
-            # Simply blacklist common types used here for now (which becomes object
+            # It's tricky to include only types which pd.Series constructor can take.
+            # Simply exclude common types used here for now (which becomes object
             # types Spark can't recognize).
             return_annotation != pd.Series and
             return_annotation != pd.DataFrame and
diff --git a/python/pyspark/sql/pandas/types.py b/python/pyspark/sql/pandas/types.py
index d1edf3f9c47c1..489b46691a135 100644
--- a/python/pyspark/sql/pandas/types.py
+++ b/python/pyspark/sql/pandas/types.py
@@ -20,12 +20,15 @@
 pandas instances during the type conversion.
 """
 
-from pyspark.sql.types import *
+from pyspark.sql.types import BooleanType, ByteType, ShortType, IntegerType, LongType, \
+    FloatType, DoubleType, DecimalType, StringType, BinaryType, DateType, TimestampType, \
+    ArrayType, MapType, StructType, StructField, NullType
 
 
 def to_arrow_type(dt):
     """ Convert Spark data type to pyarrow type
     """
+    from distutils.version import LooseVersion
     import pyarrow as pa
     if type(dt) == BooleanType:
         arrow_type = pa.bool_()
@@ -56,12 +59,21 @@ def to_arrow_type(dt):
         if type(dt.elementType) in [StructType, TimestampType]:
             raise TypeError("Unsupported type in conversion to Arrow: " + str(dt))
         arrow_type = pa.list_(to_arrow_type(dt.elementType))
+    elif type(dt) == MapType:
+        if LooseVersion(pa.__version__) < LooseVersion("2.0.0"):
+            raise TypeError("MapType is only supported with pyarrow 2.0.0 and above")
+        if type(dt.keyType) in [StructType, TimestampType] or \
+                type(dt.valueType) in [StructType, TimestampType]:
+            raise TypeError("Unsupported type in conversion to Arrow: " + str(dt))
+        arrow_type = pa.map_(to_arrow_type(dt.keyType), to_arrow_type(dt.valueType))
     elif type(dt) == StructType:
         if any(type(field.dataType) == StructType for field in dt):
             raise TypeError("Nested StructType not supported in conversion to Arrow")
         fields = [pa.field(field.name, to_arrow_type(field.dataType), nullable=field.nullable)
                   for field in dt]
         arrow_type = pa.struct(fields)
+    elif type(dt) == NullType:
+        arrow_type = pa.null()
     else:
         raise TypeError("Unsupported type in conversion to Arrow: " + str(dt))
     return arrow_type
@@ -79,6 +91,8 @@ def to_arrow_schema(schema):
 def from_arrow_type(at):
     """ Convert pyarrow type to Spark data type.
     """
+    from distutils.version import LooseVersion
+    import pyarrow as pa
     import pyarrow.types as types
     if types.is_boolean(at):
         spark_type = BooleanType()
@@ -108,12 +122,22 @@ def from_arrow_type(at):
         if types.is_timestamp(at.value_type):
             raise TypeError("Unsupported type in conversion from Arrow: " + str(at))
         spark_type = ArrayType(from_arrow_type(at.value_type))
+    elif types.is_map(at):
+        if LooseVersion(pa.__version__) < LooseVersion("2.0.0"):
+            raise TypeError("MapType is only supported with pyarrow 2.0.0 and above")
+        if types.is_timestamp(at.key_type) or types.is_timestamp(at.item_type):
+            raise TypeError("Unsupported type in conversion from Arrow: " + str(at))
+        spark_type = MapType(from_arrow_type(at.key_type), from_arrow_type(at.item_type))
     elif types.is_struct(at):
         if any(types.is_struct(field.type) for field in at):
             raise TypeError("Nested StructType not supported in conversion from Arrow: " + str(at))
         return StructType(
             [StructField(field.name, from_arrow_type(field.type), nullable=field.nullable)
              for field in at])
+    elif types.is_dictionary(at):
+        spark_type = from_arrow_type(at.value_type)
+    elif types.is_null(at):
+        spark_type = NullType()
     else:
         raise TypeError("Unsupported type in conversion from Arrow: " + str(at))
     return spark_type
@@ -149,9 +173,16 @@ def _check_series_localize_timestamps(s, timezone):
     If the input series is not a timestamp series, then the same series is returned. If the input
     series is a timestamp series, then a converted series is returned.
 
-    :param s: pandas.Series
-    :param timezone: the timezone to convert. if None then use local timezone
-    :return pandas.Series that have been converted to tz-naive
+    Parameters
+    ----------
+    s : pandas.Series
+    timezone : str
+        the timezone to convert. if None then use local timezone
+
+    Returns
+    -------
+    pandas.Series
+        `pandas.Series` that have been converted to tz-naive
     """
     from pyspark.sql.pandas.utils import require_minimum_pandas_version
     require_minimum_pandas_version()
@@ -170,9 +201,16 @@ def _check_series_convert_timestamps_internal(s, timezone):
     Convert a tz-naive timestamp in the specified timezone or local timezone to UTC normalized for
     Spark internal storage
 
-    :param s: a pandas.Series
-    :param timezone: the timezone to convert. if None then use local timezone
-    :return pandas.Series where if it is a timestamp, has been UTC normalized without a time zone
+    Parameters
+    ----------
+    s : pandas.Series
+    timezone : str
+        the timezone to convert. if None then use local timezone
+
+    Returns
+    -------
+    pandas.Series
+        `pandas.Series` where if it is a timestamp, has been UTC normalized without a time zone
     """
     from pyspark.sql.pandas.utils import require_minimum_pandas_version
     require_minimum_pandas_version()
@@ -222,10 +260,18 @@ def _check_series_convert_timestamps_localize(s, from_timezone, to_timezone):
     """
     Convert timestamp to timezone-naive in the specified timezone or local timezone
 
-    :param s: a pandas.Series
-    :param from_timezone: the timezone to convert from. if None then use local timezone
-    :param to_timezone: the timezone to convert to. if None then use local timezone
-    :return pandas.Series where if it is a timestamp, has been converted to tz-naive
+    Parameters
+    ----------
+    s : pandas.Series
+    from_timezone : str
+        the timezone to convert from. if None then use local timezone
+    to_timezone : str
+        the timezone to convert to. if None then use local timezone
+
+    Returns
+    -------
+    pandas.Series
+        `pandas.Series` where if it is a timestamp, has been converted to tz-naive
     """
     from pyspark.sql.pandas.utils import require_minimum_pandas_version
     require_minimum_pandas_version()
@@ -250,9 +296,16 @@ def _check_series_convert_timestamps_local_tz(s, timezone):
     """
     Convert timestamp to timezone-naive in the specified timezone or local timezone
 
-    :param s: a pandas.Series
-    :param timezone: the timezone to convert to. if None then use local timezone
-    :return pandas.Series where if it is a timestamp, has been converted to tz-naive
+    Parameters
+    ----------
+    s : pandas.Series
+    timezone : str
+        the timezone to convert to. if None then use local timezone
+
+    Returns
+    -------
+    pandas.Series
+        `pandas.Series` where if it is a timestamp, has been converted to tz-naive
     """
     return _check_series_convert_timestamps_localize(s, None, timezone)
 
@@ -261,8 +314,35 @@ def _check_series_convert_timestamps_tz_local(s, timezone):
     """
     Convert timestamp to timezone-naive in the specified timezone or local timezone
 
-    :param s: a pandas.Series
-    :param timezone: the timezone to convert from. if None then use local timezone
-    :return pandas.Series where if it is a timestamp, has been converted to tz-naive
+    Parameters
+    ----------
+    s : pandas.Series
+    timezone : str
+        the timezone to convert from. if None then use local timezone
+
+    Returns
+    -------
+    pandas.Series
+        `pandas.Series` where if it is a timestamp, has been converted to tz-naive
     """
     return _check_series_convert_timestamps_localize(s, timezone, None)
+
+
+def _convert_map_items_to_dict(s):
+    """
+    Convert a series with items as list of (key, value), as made from an Arrow column of map type,
+    to dict for compatibility with non-arrow MapType columns.
+    :param s: pandas.Series of lists of (key, value) pairs
+    :return: pandas.Series of dictionaries
+    """
+    return s.apply(lambda m: None if m is None else {k: v for k, v in m})
+
+
+def _convert_dict_to_map_items(s):
+    """
+    Convert a series of dictionaries to list of (key, value) pairs to match expected data
+    for Arrow column of map type.
+    :param s: pandas.Series of dictionaries
+    :return: pandas.Series of lists of (key, value) pairs
+    """
+    return s.apply(lambda d: list(d.items()) if d is not None else None)
diff --git a/python/pyspark/sql/pandas/utils.py b/python/pyspark/sql/pandas/utils.py
index 481aa3e6432e1..b22603a779701 100644
--- a/python/pyspark/sql/pandas/utils.py
+++ b/python/pyspark/sql/pandas/utils.py
@@ -26,11 +26,12 @@ def require_minimum_pandas_version():
     try:
         import pandas
         have_pandas = True
-    except ImportError:
+    except ImportError as error:
         have_pandas = False
+        raised_error = error
     if not have_pandas:
         raise ImportError("Pandas >= %s must be installed; however, "
-                          "it was not found." % minimum_pandas_version)
+                          "it was not found." % minimum_pandas_version) from raised_error
     if LooseVersion(pandas.__version__) < LooseVersion(minimum_pandas_version):
         raise ImportError("Pandas >= %s must be installed; however, "
                           "your version was %s." % (minimum_pandas_version, pandas.__version__))
@@ -40,18 +41,19 @@ def require_minimum_pyarrow_version():
     """ Raise ImportError if minimum version of pyarrow is not installed
     """
     # TODO(HyukjinKwon): Relocate and deduplicate the version specification.
-    minimum_pyarrow_version = "0.15.1"
+    minimum_pyarrow_version = "1.0.0"
 
     from distutils.version import LooseVersion
     import os
     try:
         import pyarrow
         have_arrow = True
-    except ImportError:
+    except ImportError as error:
         have_arrow = False
+        raised_error = error
     if not have_arrow:
         raise ImportError("PyArrow >= %s must be installed; however, "
-                          "it was not found." % minimum_pyarrow_version)
+                          "it was not found." % minimum_pyarrow_version) from raised_error
     if LooseVersion(pyarrow.__version__) < LooseVersion(minimum_pyarrow_version):
         raise ImportError("PyArrow >= %s must be installed; however, "
                           "your version was %s." % (minimum_pyarrow_version, pyarrow.__version__))
diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py
index 336345e383729..31c1f2f7ca3c5 100644
--- a/python/pyspark/sql/readwriter.py
+++ b/python/pyspark/sql/readwriter.py
@@ -14,18 +14,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
 import sys
 
-if sys.version >= '3':
-    basestring = unicode = str
-
 from py4j.java_gateway import JavaClass
 
 from pyspark import RDD, since
-from pyspark.rdd import ignore_unicode_prefix
-from pyspark.sql.column import _to_seq
-from pyspark.sql.types import *
+from pyspark.sql.column import _to_seq, _to_java_column
+from pyspark.sql.types import StructType
 from pyspark.sql import utils
 from pyspark.sql.utils import to_str
 
@@ -62,12 +57,18 @@ def _df(self, jdf):
         from pyspark.sql.dataframe import DataFrame
         return DataFrame(jdf, self._spark)
 
-    @since(1.4)
     def format(self, source):
         """Specifies the input data source format.
 
-        :param source: string, name of the data source, e.g. 'json', 'parquet'.
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        source : str
+            string, name of the data source, e.g. 'json', 'parquet'.
 
+        Examples
+        --------
         >>> df = spark.read.format('json').load('python/test_support/sql/people.json')
         >>> df.dtypes
         [('age', 'bigint'), ('name', 'string')]
@@ -76,7 +77,6 @@ def format(self, source):
         self._jreader = self._jreader.format(source)
         return self
 
-    @since(1.4)
     def schema(self, schema):
         """Specifies the input schema.
 
@@ -84,8 +84,13 @@ def schema(self, schema):
         By specifying the schema here, the underlying data source can skip the schema
         inference step, and thus speed up data loading.
 
-        :param schema: a :class:`pyspark.sql.types.StructType` object or a DDL-formatted string
-                       (For example ``col0 INT, col1 DOUBLE``).
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        schema : :class:`pyspark.sql.types.StructType` or str
+            a :class:`pyspark.sql.types.StructType` object or a DDL-formatted string
+            (For example ``col0 INT, col1 DOUBLE``).
 
         >>> s = spark.read.schema("col0 INT, col1 DOUBLE")
         """
@@ -94,7 +99,7 @@ def schema(self, schema):
         if isinstance(schema, StructType):
             jschema = spark._jsparkSession.parseDataType(schema.json())
             self._jreader = self._jreader.schema(jschema)
-        elif isinstance(schema, basestring):
+        elif isinstance(schema, str):
             self._jreader = self._jreader.schema(schema)
         else:
             raise TypeError("schema should be StructType or string")
@@ -120,6 +125,12 @@ def option(self, key, value):
             * ``pathGlobFilter``: an optional glob pattern to only include files with paths matching
                 the pattern. The syntax follows org.apache.hadoop.fs.GlobFilter.
                 It does not change the behavior of partition discovery.
+            * ``modifiedBefore``: an optional timestamp to only include files with
+                modification times occurring before the specified time. The provided timestamp
+                must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)
+            * ``modifiedAfter``: an optional timestamp to only include files with
+                modification times occurring after the specified time. The provided timestamp
+                must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)
         """
         self._jreader = self._jreader.option(key, to_str(value))
         return self
@@ -144,21 +155,36 @@ def options(self, **options):
             * ``pathGlobFilter``: an optional glob pattern to only include files with paths matching
                 the pattern. The syntax follows org.apache.hadoop.fs.GlobFilter.
                 It does not change the behavior of partition discovery.
+            * ``modifiedBefore``: an optional timestamp to only include files with
+                modification times occurring before the specified time. The provided timestamp
+                must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)
+            * ``modifiedAfter``: an optional timestamp to only include files with
+                modification times occurring after the specified time. The provided timestamp
+                must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)
         """
         for k in options:
             self._jreader = self._jreader.option(k, to_str(options[k]))
         return self
 
-    @since(1.4)
     def load(self, path=None, format=None, schema=None, **options):
         """Loads data from a data source and returns it as a :class:`DataFrame`.
 
-        :param path: optional string or a list of string for file-system backed data sources.
-        :param format: optional string for format of the data source. Default to 'parquet'.
-        :param schema: optional :class:`pyspark.sql.types.StructType` for the input schema
-                       or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``).
-        :param options: all other string options
-
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        path : str or list, optional
+            optional string or a list of string for file-system backed data sources.
+        format : str, optional
+            optional string for format of the data source. Default to 'parquet'.
+        schema : :class:`pyspark.sql.types.StructType` or str, optional
+            optional :class:`pyspark.sql.types.StructType` for the input schema
+            or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``).
+        **options : dict
+            all other string options
+
+        Examples
+        --------
         >>> df = spark.read.format("parquet").load('python/test_support/sql/parquet_partitioned',
         ...     opt1=True, opt2=1, opt3='str')
         >>> df.dtypes
@@ -174,7 +200,7 @@ def load(self, path=None, format=None, schema=None, **options):
         if schema is not None:
             self.schema(schema)
         self.options(**options)
-        if isinstance(path, basestring):
+        if isinstance(path, str):
             return self._df(self._jreader.load(path))
         elif path is not None:
             if type(path) != list:
@@ -183,14 +209,14 @@ def load(self, path=None, format=None, schema=None, **options):
         else:
             return self._df(self._jreader.load())
 
-    @since(1.4)
     def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
              allowComments=None, allowUnquotedFieldNames=None, allowSingleQuotes=None,
              allowNumericLeadingZero=None, allowBackslashEscapingAnyCharacter=None,
              mode=None, columnNameOfCorruptRecord=None, dateFormat=None, timestampFormat=None,
              multiLine=None, allowUnquotedControlChars=None, lineSep=None, samplingRatio=None,
              dropFieldIfAllNull=None, encoding=None, locale=None, pathGlobFilter=None,
-             recursiveFileLookup=None):
+             recursiveFileLookup=None, allowNonNumericNumbers=None,
+             modifiedBefore=None, modifiedAfter=None):
         """
         Loads JSON files and returns the results as a :class:`DataFrame`.
 
@@ -200,81 +226,125 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
         If the ``schema`` parameter is not specified, this function goes
         through the input once to determine the input schema.
 
-        :param path: string represents path to the JSON dataset, or a list of paths,
-                     or RDD of Strings storing JSON objects.
-        :param schema: an optional :class:`pyspark.sql.types.StructType` for the input schema or
-                       a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``).
-        :param primitivesAsString: infers all primitive values as a string type. If None is set,
-                                   it uses the default value, ``false``.
-        :param prefersDecimal: infers all floating-point values as a decimal type. If the values
-                               do not fit in decimal, then it infers them as doubles. If None is
-                               set, it uses the default value, ``false``.
-        :param allowComments: ignores Java/C++ style comment in JSON records. If None is set,
-                              it uses the default value, ``false``.
-        :param allowUnquotedFieldNames: allows unquoted JSON field names. If None is set,
-                                        it uses the default value, ``false``.
-        :param allowSingleQuotes: allows single quotes in addition to double quotes. If None is
-                                        set, it uses the default value, ``true``.
-        :param allowNumericLeadingZero: allows leading zeros in numbers (e.g. 00012). If None is
-                                        set, it uses the default value, ``false``.
-        :param allowBackslashEscapingAnyCharacter: allows accepting quoting of all character
-                                                   using backslash quoting mechanism. If None is
-                                                   set, it uses the default value, ``false``.
-        :param mode: allows a mode for dealing with corrupt records during parsing. If None is
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        path : str, list or :class:`RDD`
+            string represents path to the JSON dataset, or a list of paths,
+            or RDD of Strings storing JSON objects.
+        schema : :class:`pyspark.sql.types.StructType` or str, optional
+            an optional :class:`pyspark.sql.types.StructType` for the input schema or
+            a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``).
+        primitivesAsString : str or bool, optional
+            infers all primitive values as a string type. If None is set,
+            it uses the default value, ``false``.
+        prefersDecimal : str or bool, optional
+            infers all floating-point values as a decimal type. If the values
+            do not fit in decimal, then it infers them as doubles. If None is
+            set, it uses the default value, ``false``.
+        allowComments : str or bool, optional
+            ignores Java/C++ style comment in JSON records. If None is set,
+            it uses the default value, ``false``.
+        allowUnquotedFieldNames : str or bool, optional
+            allows unquoted JSON field names. If None is set,
+            it uses the default value, ``false``.
+        allowSingleQuotes : str or bool, optional
+            allows single quotes in addition to double quotes. If None is
+            set, it uses the default value, ``true``.
+        allowNumericLeadingZero : str or bool, optional
+            allows leading zeros in numbers (e.g. 00012). If None is
+            set, it uses the default value, ``false``.
+        allowBackslashEscapingAnyCharacter : str or bool, optional
+            allows accepting quoting of all character
+            using backslash quoting mechanism. If None is
+            set, it uses the default value, ``false``.
+        mode : str, optional
+            allows a mode for dealing with corrupt records during parsing. If None is
                      set, it uses the default value, ``PERMISSIVE``.
 
-                * ``PERMISSIVE``: when it meets a corrupted record, puts the malformed string \
-                  into a field configured by ``columnNameOfCorruptRecord``, and sets malformed \
-                  fields to ``null``. To keep corrupt records, an user can set a string type \
-                  field named ``columnNameOfCorruptRecord`` in an user-defined schema. If a \
-                  schema does not have the field, it drops corrupt records during parsing. \
-                  When inferring a schema, it implicitly adds a ``columnNameOfCorruptRecord`` \
-                  field in an output schema.
-                *  ``DROPMALFORMED``: ignores the whole corrupted records.
-                *  ``FAILFAST``: throws an exception when it meets corrupted records.
-
-        :param columnNameOfCorruptRecord: allows renaming the new field having malformed string
-                                          created by ``PERMISSIVE`` mode. This overrides
-                                          ``spark.sql.columnNameOfCorruptRecord``. If None is set,
-                                          it uses the value specified in
-                                          ``spark.sql.columnNameOfCorruptRecord``.
-        :param dateFormat: sets the string that indicates a date format. Custom date formats
-                           follow the formats at `datetime pattern`_.
-                           This applies to date type. If None is set, it uses the
-                           default value, ``yyyy-MM-dd``.
-        :param timestampFormat: sets the string that indicates a timestamp format.
-                                Custom date formats follow the formats at `datetime pattern`_.
-                                This applies to timestamp type. If None is set, it uses the
-                                default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``.
-        :param multiLine: parse one record, which may span multiple lines, per file. If None is
-                          set, it uses the default value, ``false``.
-        :param allowUnquotedControlChars: allows JSON Strings to contain unquoted control
-                                          characters (ASCII characters with value less than 32,
-                                          including tab and line feed characters) or not.
-        :param encoding: allows to forcibly set one of standard basic or extended encoding for
-                         the JSON files. For example UTF-16BE, UTF-32LE. If None is set,
-                         the encoding of input JSON will be detected automatically
-                         when the multiLine option is set to ``true``.
-        :param lineSep: defines the line separator that should be used for parsing. If None is
-                        set, it covers all ``\\r``, ``\\r\\n`` and ``\\n``.
-        :param samplingRatio: defines fraction of input JSON objects used for schema inferring.
-                              If None is set, it uses the default value, ``1.0``.
-        :param dropFieldIfAllNull: whether to ignore column of all null values or empty
-                                   array/struct during schema inference. If None is set, it
-                                   uses the default value, ``false``.
-        :param locale: sets a locale as language tag in IETF BCP 47 format. If None is set,
-                       it uses the default value, ``en-US``. For instance, ``locale`` is used while
-                       parsing dates and timestamps.
-        :param pathGlobFilter: an optional glob pattern to only include files with paths matching
-                               the pattern. The syntax follows `org.apache.hadoop.fs.GlobFilter`.
-                               It does not change the behavior of `partition discovery`_.
-        :param recursiveFileLookup: recursively scan a directory for files. Using this option
-                                    disables `partition discovery`_.
-
-        .. _partition discovery:
-          https://spark.apache.org/docs/latest/sql-data-sources-parquet.html#partition-discovery
-        .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
-
+            * ``PERMISSIVE``: when it meets a corrupted record, puts the malformed string \
+              into a field configured by ``columnNameOfCorruptRecord``, and sets malformed \
+              fields to ``null``. To keep corrupt records, an user can set a string type \
+              field named ``columnNameOfCorruptRecord`` in an user-defined schema. If a \
+              schema does not have the field, it drops corrupt records during parsing. \
+              When inferring a schema, it implicitly adds a ``columnNameOfCorruptRecord`` \
+              field in an output schema.
+            *  ``DROPMALFORMED``: ignores the whole corrupted records.
+            *  ``FAILFAST``: throws an exception when it meets corrupted records.
+
+        columnNameOfCorruptRecord: str, optional
+            allows renaming the new field having malformed string
+            created by ``PERMISSIVE`` mode. This overrides
+            ``spark.sql.columnNameOfCorruptRecord``. If None is set,
+            it uses the value specified in
+            ``spark.sql.columnNameOfCorruptRecord``.
+        dateFormat : str, optional
+            sets the string that indicates a date format. Custom date formats
+            follow the formats at
+            `datetime pattern <https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html>`_.  # noqa
+            This applies to date type. If None is set, it uses the
+            default value, ``yyyy-MM-dd``.
+        timestampFormat : str, optional
+            sets the string that indicates a timestamp format.
+            Custom date formats follow the formats at
+            `datetime pattern <https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html>`_.  # noqa
+            This applies to timestamp type. If None is set, it uses the
+            default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``.
+        multiLine : str or bool, optional
+            parse one record, which may span multiple lines, per file. If None is
+            set, it uses the default value, ``false``.
+        allowUnquotedControlChars : str or bool, optional
+            allows JSON Strings to contain unquoted control
+            characters (ASCII characters with value less than 32,
+            including tab and line feed characters) or not.
+        encoding : str or bool, optional
+            allows to forcibly set one of standard basic or extended encoding for
+            the JSON files. For example UTF-16BE, UTF-32LE. If None is set,
+            the encoding of input JSON will be detected automatically
+            when the multiLine option is set to ``true``.
+        lineSep : str, optional
+            defines the line separator that should be used for parsing. If None is
+            set, it covers all ``\\r``, ``\\r\\n`` and ``\\n``.
+        samplingRatio : str or float, optional
+            defines fraction of input JSON objects used for schema inferring.
+            If None is set, it uses the default value, ``1.0``.
+        dropFieldIfAllNull : str or bool, optional
+            whether to ignore column of all null values or empty
+            array/struct during schema inference. If None is set, it
+            uses the default value, ``false``.
+        locale : str, optional
+            sets a locale as language tag in IETF BCP 47 format. If None is set,
+            it uses the default value, ``en-US``. For instance, ``locale`` is used while
+            parsing dates and timestamps.
+        pathGlobFilter : str or bool, optional
+            an optional glob pattern to only include files with paths matching
+            the pattern. The syntax follows `org.apache.hadoop.fs.GlobFilter`.
+            It does not change the behavior of
+            `partition discovery <https://spark.apache.org/docs/latest/sql-data-sources-parquet.html#partition-discovery>`_.  # noqa
+        recursiveFileLookup : str or bool, optional
+            recursively scan a directory for files. Using this option
+            disables
+            `partition discovery <https://spark.apache.org/docs/latest/sql-data-sources-parquet.html#partition-discovery>`_.  # noqa
+        allowNonNumericNumbers : str or bool
+            allows JSON parser to recognize set of "Not-a-Number" (NaN)
+            tokens as legal floating number values. If None is set,
+            it uses the default value, ``true``.
+
+                * ``+INF``: for positive infinity, as well as alias of
+                            ``+Infinity`` and ``Infinity``.
+                *  ``-INF``: for negative infinity, alias ``-Infinity``.
+                *  ``NaN``: for other not-a-numbers, like result of division by zero.
+        modifiedBefore : an optional timestamp to only include files with
+            modification times occurring before the specified time. The provided timestamp
+            must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)
+        modifiedAfter : an optional timestamp to only include files with
+            modification times occurring after the specified time. The provided timestamp
+            must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)
+
+
+        Examples
+        --------
         >>> df1 = spark.read.json('python/test_support/sql/people.json')
         >>> df1.dtypes
         [('age', 'bigint'), ('name', 'string')]
@@ -293,17 +363,19 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
             timestampFormat=timestampFormat, multiLine=multiLine,
             allowUnquotedControlChars=allowUnquotedControlChars, lineSep=lineSep,
             samplingRatio=samplingRatio, dropFieldIfAllNull=dropFieldIfAllNull, encoding=encoding,
-            locale=locale, pathGlobFilter=pathGlobFilter, recursiveFileLookup=recursiveFileLookup)
-        if isinstance(path, basestring):
+            locale=locale, pathGlobFilter=pathGlobFilter, recursiveFileLookup=recursiveFileLookup,
+            modifiedBefore=modifiedBefore, modifiedAfter=modifiedAfter,
+            allowNonNumericNumbers=allowNonNumericNumbers)
+        if isinstance(path, str):
             path = [path]
         if type(path) == list:
             return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))
         elif isinstance(path, RDD):
             def func(iterator):
                 for x in iterator:
-                    if not isinstance(x, basestring):
-                        x = unicode(x)
-                    if isinstance(x, unicode):
+                    if not isinstance(x, str):
+                        x = str(x)
+                    if isinstance(x, str):
                         x = x.encode("utf-8")
                     yield x
             keyed = path.mapPartitions(func)
@@ -313,12 +385,18 @@ def func(iterator):
         else:
             raise TypeError("path can be only string, list or RDD")
 
-    @since(1.4)
     def table(self, tableName):
         """Returns the specified table as a :class:`DataFrame`.
 
-        :param tableName: string, name of the table.
+        .. versionadded:: 1.4.0
 
+        Parameters
+        ----------
+        tableName : str
+            string, name of the table.
+
+        Examples
+        --------
         >>> df = spark.read.parquet('python/test_support/sql/parquet_partitioned')
         >>> df.createOrReplaceTempView('tmpTable')
         >>> spark.read.table('tmpTable').dtypes
@@ -326,36 +404,89 @@ def table(self, tableName):
         """
         return self._df(self._jreader.table(tableName))
 
-    @since(1.4)
     def parquet(self, *paths, **options):
         """
         Loads Parquet files, returning the result as a :class:`DataFrame`.
 
-        :param mergeSchema: sets whether we should merge schemas collected from all
-                            Parquet part-files. This will override
-                            ``spark.sql.parquet.mergeSchema``. The default value is specified in
-                            ``spark.sql.parquet.mergeSchema``.
-        :param pathGlobFilter: an optional glob pattern to only include files with paths matching
-                               the pattern. The syntax follows `org.apache.hadoop.fs.GlobFilter`.
-                               It does not change the behavior of `partition discovery`_.
-        :param recursiveFileLookup: recursively scan a directory for files. Using this option
-                                    disables `partition discovery`_.
-
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        paths : str
+
+        Other Parameters
+        ----------------
+        mergeSchema : str or bool, optional
+            sets whether we should merge schemas collected from all
+            Parquet part-files. This will override
+            ``spark.sql.parquet.mergeSchema``. The default value is specified in
+            ``spark.sql.parquet.mergeSchema``.
+        pathGlobFilter : str or bool, optional
+            an optional glob pattern to only include files with paths matching
+            the pattern. The syntax follows `org.apache.hadoop.fs.GlobFilter`.
+            It does not change the behavior of
+            `partition discovery <https://spark.apache.org/docs/latest/sql-data-sources-parquet.html#partition-discovery>`_.  # noqa
+        recursiveFileLookup : str or bool, optional
+            recursively scan a directory for files. Using this option
+            disables
+            `partition discovery <https://spark.apache.org/docs/latest/sql-data-sources-parquet.html#partition-discovery>`_.  # noqa
+
+            modification times occurring before the specified time. The provided timestamp
+            must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)
+        modifiedBefore (batch only) : an optional timestamp to only include files with
+            modification times occurring before the specified time. The provided timestamp
+            must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)
+        modifiedAfter (batch only) : an optional timestamp to only include files with
+            modification times occurring after the specified time. The provided timestamp
+            must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)
+        datetimeRebaseMode : str, optional
+            the rebasing mode for the values of the ``DATE``, ``TIMESTAMP_MICROS``,
+            ``TIMESTAMP_MILLIS`` logical types from the Julian to Proleptic Gregorian calendar.
+
+                * ``EXCEPTION``: Spark fails in reads of ancient dates/timestamps
+                                 that are ambiguous between the two calendars.
+                *  ``CORRECTED``: loading of dates/timestamps without rebasing.
+                *  ``LEGACY``: perform rebasing of ancient dates/timestamps from the Julian
+                               to Proleptic Gregorian calendar.
+
+            If None is set, the value of the SQL config
+            ``spark.sql.parquet.datetimeRebaseModeInRead`` is used by default.
+        int96RebaseMode : str, optional
+            the rebasing mode for ``INT96`` timestamps from the Julian to
+            Proleptic Gregorian calendar.
+
+                * ``EXCEPTION``: Spark fails in reads of ancient ``INT96`` timestamps
+                                 that are ambiguous between the two calendars.
+                *  ``CORRECTED``: loading of ``INT96`` timestamps without rebasing.
+                *  ``LEGACY``: perform rebasing of ancient ``INT96`` timestamps from the Julian
+                               to Proleptic Gregorian calendar.
+
+            If None is set, the value of the SQL config
+            ``spark.sql.parquet.int96RebaseModeInRead`` is used by default.
+
+        Examples
+        --------
         >>> df = spark.read.parquet('python/test_support/sql/parquet_partitioned')
         >>> df.dtypes
         [('name', 'string'), ('year', 'int'), ('month', 'int'), ('day', 'int')]
         """
         mergeSchema = options.get('mergeSchema', None)
         pathGlobFilter = options.get('pathGlobFilter', None)
+        modifiedBefore = options.get('modifiedBefore', None)
+        modifiedAfter = options.get('modifiedAfter', None)
         recursiveFileLookup = options.get('recursiveFileLookup', None)
+        datetimeRebaseMode = options.get('datetimeRebaseMode', None)
+        int96RebaseMode = options.get('int96RebaseMode', None)
         self._set_opts(mergeSchema=mergeSchema, pathGlobFilter=pathGlobFilter,
-                       recursiveFileLookup=recursiveFileLookup)
+                       recursiveFileLookup=recursiveFileLookup, modifiedBefore=modifiedBefore,
+                       modifiedAfter=modifiedAfter, datetimeRebaseMode=datetimeRebaseMode,
+                       int96RebaseMode=int96RebaseMode)
+
         return self._df(self._jreader.parquet(_to_seq(self._spark._sc, paths)))
 
-    @ignore_unicode_prefix
-    @since(1.6)
     def text(self, paths, wholetext=False, lineSep=None, pathGlobFilter=None,
-             recursiveFileLookup=None):
+             recursiveFileLookup=None, modifiedBefore=None,
+             modifiedAfter=None):
         """
         Loads text files and returns a :class:`DataFrame` whose schema starts with a
         string column named "value", and followed by partitioned columns if there
@@ -364,31 +495,53 @@ def text(self, paths, wholetext=False, lineSep=None, pathGlobFilter=None,
 
         By default, each line in the text file is a new row in the resulting DataFrame.
 
-        :param paths: string, or list of strings, for input path(s).
-        :param wholetext: if true, read each file from input path(s) as a single row.
-        :param lineSep: defines the line separator that should be used for parsing. If None is
-                        set, it covers all ``\\r``, ``\\r\\n`` and ``\\n``.
-        :param pathGlobFilter: an optional glob pattern to only include files with paths matching
-                               the pattern. The syntax follows `org.apache.hadoop.fs.GlobFilter`.
-                               It does not change the behavior of `partition discovery`_.
-        :param recursiveFileLookup: recursively scan a directory for files. Using this option
-                                    disables `partition discovery`_.
-
+        .. versionadded:: 1.6.0
+
+        Parameters
+        ----------
+        paths : str or list
+            string, or list of strings, for input path(s).
+        wholetext : str or bool, optional
+            if true, read each file from input path(s) as a single row.
+        lineSep : str, optional
+            defines the line separator that should be used for parsing. If None is
+            set, it covers all ``\\r``, ``\\r\\n`` and ``\\n``.
+        pathGlobFilter : str or bool, optional
+            an optional glob pattern to only include files with paths matching
+            the pattern. The syntax follows `org.apache.hadoop.fs.GlobFilter`.
+            It does not change the behavior of
+            `partition discovery <https://spark.apache.org/docs/latest/sql-data-sources-parquet.html#partition-discovery>`_.  # noqa
+        recursiveFileLookup : str or bool, optional
+            recursively scan a directory for files. Using this option disables
+            `partition discovery <https://spark.apache.org/docs/latest/sql-data-sources-parquet.html#partition-discovery>`_.  # noqa
+
+            modification times occurring before the specified time. The provided timestamp
+            must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)
+        modifiedBefore (batch only) : an optional timestamp to only include files with
+            modification times occurring before the specified time. The provided timestamp
+            must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)
+        modifiedAfter (batch only) : an optional timestamp to only include files with
+            modification times occurring after the specified time. The provided timestamp
+            must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)
+
+        Examples
+        --------
         >>> df = spark.read.text('python/test_support/sql/text-test.txt')
         >>> df.collect()
-        [Row(value=u'hello'), Row(value=u'this')]
+        [Row(value='hello'), Row(value='this')]
         >>> df = spark.read.text('python/test_support/sql/text-test.txt', wholetext=True)
         >>> df.collect()
-        [Row(value=u'hello\\nthis')]
+        [Row(value='hello\\nthis')]
         """
         self._set_opts(
             wholetext=wholetext, lineSep=lineSep, pathGlobFilter=pathGlobFilter,
-            recursiveFileLookup=recursiveFileLookup)
-        if isinstance(paths, basestring):
+            recursiveFileLookup=recursiveFileLookup, modifiedBefore=modifiedBefore,
+            modifiedAfter=modifiedAfter)
+
+        if isinstance(paths, str):
             paths = [paths]
         return self._df(self._jreader.text(self._spark._sc._jvm.PythonUtils.toSeq(paths)))
 
-    @since(2.0)
     def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=None,
             comment=None, header=None, inferSchema=None, ignoreLeadingWhiteSpace=None,
             ignoreTrailingWhiteSpace=None, nullValue=None, nanValue=None, positiveInf=None,
@@ -396,119 +549,193 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
             maxCharsPerColumn=None, maxMalformedLogPerPartition=None, mode=None,
             columnNameOfCorruptRecord=None, multiLine=None, charToEscapeQuoteEscaping=None,
             samplingRatio=None, enforceSchema=None, emptyValue=None, locale=None, lineSep=None,
-            pathGlobFilter=None, recursiveFileLookup=None):
+            pathGlobFilter=None, recursiveFileLookup=None, modifiedBefore=None, modifiedAfter=None,
+            unescapedQuoteHandling=None):
         r"""Loads a CSV file and returns the result as a  :class:`DataFrame`.
 
         This function will go through the input once to determine the input schema if
         ``inferSchema`` is enabled. To avoid going through the entire data once, disable
         ``inferSchema`` option or specify the schema explicitly using ``schema``.
 
-        :param path: string, or list of strings, for input path(s),
-                     or RDD of Strings storing CSV rows.
-        :param schema: an optional :class:`pyspark.sql.types.StructType` for the input schema
-                       or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``).
-        :param sep: sets a separator (one or more characters) for each field and value. If None is
-                    set, it uses the default value, ``,``.
-        :param encoding: decodes the CSV files by the given encoding type. If None is set,
-                         it uses the default value, ``UTF-8``.
-        :param quote: sets a single character used for escaping quoted values where the
-                      separator can be part of the value. If None is set, it uses the default
-                      value, ``"``. If you would like to turn off quotations, you need to set an
-                      empty string.
-        :param escape: sets a single character used for escaping quotes inside an already
-                       quoted value. If None is set, it uses the default value, ``\``.
-        :param comment: sets a single character used for skipping lines beginning with this
-                        character. By default (None), it is disabled.
-        :param header: uses the first line as names of columns. If None is set, it uses the
-                       default value, ``false``.
-        :param inferSchema: infers the input schema automatically from data. It requires one extra
-                       pass over the data. If None is set, it uses the default value, ``false``.
-        :param enforceSchema: If it is set to ``true``, the specified or inferred schema will be
-                              forcibly applied to datasource files, and headers in CSV files will be
-                              ignored. If the option is set to ``false``, the schema will be
-                              validated against all headers in CSV files or the first header in RDD
-                              if the ``header`` option is set to ``true``. Field names in the schema
-                              and column names in CSV headers are checked by their positions
-                              taking into account ``spark.sql.caseSensitive``. If None is set,
-                              ``true`` is used by default. Though the default value is ``true``,
-                              it is recommended to disable the ``enforceSchema`` option
-                              to avoid incorrect results.
-        :param ignoreLeadingWhiteSpace: A flag indicating whether or not leading whitespaces from
-                                        values being read should be skipped. If None is set, it
-                                        uses the default value, ``false``.
-        :param ignoreTrailingWhiteSpace: A flag indicating whether or not trailing whitespaces from
-                                         values being read should be skipped. If None is set, it
-                                         uses the default value, ``false``.
-        :param nullValue: sets the string representation of a null value. If None is set, it uses
-                          the default value, empty string. Since 2.0.1, this ``nullValue`` param
-                          applies to all supported types including the string type.
-        :param nanValue: sets the string representation of a non-number value. If None is set, it
-                         uses the default value, ``NaN``.
-        :param positiveInf: sets the string representation of a positive infinity value. If None
-                            is set, it uses the default value, ``Inf``.
-        :param negativeInf: sets the string representation of a negative infinity value. If None
-                            is set, it uses the default value, ``Inf``.
-        :param dateFormat: sets the string that indicates a date format. Custom date formats
-                           follow the formats at `datetime pattern`_.
-                           This applies to date type. If None is set, it uses the
-                           default value, ``yyyy-MM-dd``.
-        :param timestampFormat: sets the string that indicates a timestamp format.
-                                Custom date formats follow the formats at `datetime pattern`_.
-                                This applies to timestamp type. If None is set, it uses the
-                                default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``.
-        :param maxColumns: defines a hard limit of how many columns a record can have. If None is
-                           set, it uses the default value, ``20480``.
-        :param maxCharsPerColumn: defines the maximum number of characters allowed for any given
-                                  value being read. If None is set, it uses the default value,
-                                  ``-1`` meaning unlimited length.
-        :param maxMalformedLogPerPartition: this parameter is no longer used since Spark 2.2.0.
-                                            If specified, it is ignored.
-        :param mode: allows a mode for dealing with corrupt records during parsing. If None is
-                     set, it uses the default value, ``PERMISSIVE``. Note that Spark tries to
-                     parse only required columns in CSV under column pruning. Therefore, corrupt
-                     records can be different based on required set of fields. This behavior can
-                     be controlled by ``spark.sql.csv.parser.columnPruning.enabled``
-                     (enabled by default).
-
-                * ``PERMISSIVE``: when it meets a corrupted record, puts the malformed string \
-                  into a field configured by ``columnNameOfCorruptRecord``, and sets malformed \
-                  fields to ``null``. To keep corrupt records, an user can set a string type \
-                  field named ``columnNameOfCorruptRecord`` in an user-defined schema. If a \
-                  schema does not have the field, it drops corrupt records during parsing. \
-                  A record with less/more tokens than schema is not a corrupted record to CSV. \
-                  When it meets a record having fewer tokens than the length of the schema, \
-                  sets ``null`` to extra fields. When the record has more tokens than the \
-                  length of the schema, it drops extra tokens.
-                * ``DROPMALFORMED``: ignores the whole corrupted records.
-                * ``FAILFAST``: throws an exception when it meets corrupted records.
-
-        :param columnNameOfCorruptRecord: allows renaming the new field having malformed string
-                                          created by ``PERMISSIVE`` mode. This overrides
-                                          ``spark.sql.columnNameOfCorruptRecord``. If None is set,
-                                          it uses the value specified in
-                                          ``spark.sql.columnNameOfCorruptRecord``.
-        :param multiLine: parse records, which may span multiple lines. If None is
-                          set, it uses the default value, ``false``.
-        :param charToEscapeQuoteEscaping: sets a single character used for escaping the escape for
-                                          the quote character. If None is set, the default value is
-                                          escape character when escape and quote characters are
-                                          different, ``\0`` otherwise.
-        :param samplingRatio: defines fraction of rows used for schema inferring.
-                              If None is set, it uses the default value, ``1.0``.
-        :param emptyValue: sets the string representation of an empty value. If None is set, it uses
-                           the default value, empty string.
-        :param locale: sets a locale as language tag in IETF BCP 47 format. If None is set,
-                       it uses the default value, ``en-US``. For instance, ``locale`` is used while
-                       parsing dates and timestamps.
-        :param lineSep: defines the line separator that should be used for parsing. If None is
-                        set, it covers all ``\\r``, ``\\r\\n`` and ``\\n``.
-                        Maximum length is 1 character.
-        :param pathGlobFilter: an optional glob pattern to only include files with paths matching
-                               the pattern. The syntax follows `org.apache.hadoop.fs.GlobFilter`.
-                               It does not change the behavior of `partition discovery`_.
-        :param recursiveFileLookup: recursively scan a directory for files. Using this option
-                                    disables `partition discovery`_.
-
+        .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        path : str or list
+            string, or list of strings, for input path(s),
+            or RDD of Strings storing CSV rows.
+        schema : :class:`pyspark.sql.types.StructType` or str, optional
+            an optional :class:`pyspark.sql.types.StructType` for the input schema
+            or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``).
+        sep : str, optional
+            sets a separator (one or more characters) for each field and value. If None is
+            set, it uses the default value, ``,``.
+        encoding : str, optional
+            decodes the CSV files by the given encoding type. If None is set,
+            it uses the default value, ``UTF-8``.
+        quote : str, optional
+            sets a single character used for escaping quoted values where the
+            separator can be part of the value. If None is set, it uses the default
+            value, ``"``. If you would like to turn off quotations, you need to set an
+            empty string.
+        escape : str, optional
+            sets a single character used for escaping quotes inside an already
+            quoted value. If None is set, it uses the default value, ``\``.
+        comment : str, optional
+            sets a single character used for skipping lines beginning with this
+            character. By default (None), it is disabled.
+        header : str or bool, optional
+            uses the first line as names of columns. If None is set, it uses the
+            default value, ``false``.
+
+            .. note:: if the given path is a RDD of Strings, this header
+                option will remove all lines same with the header if exists.
+
+        inferSchema : str or bool, optional
+            infers the input schema automatically from data. It requires one extra
+            pass over the data. If None is set, it uses the default value, ``false``.
+        enforceSchema : str or bool, optional
+            If it is set to ``true``, the specified or inferred schema will be
+            forcibly applied to datasource files, and headers in CSV files will be
+            ignored. If the option is set to ``false``, the schema will be
+            validated against all headers in CSV files or the first header in RDD
+            if the ``header`` option is set to ``true``. Field names in the schema
+            and column names in CSV headers are checked by their positions
+            taking into account ``spark.sql.caseSensitive``. If None is set,
+            ``true`` is used by default. Though the default value is ``true``,
+            it is recommended to disable the ``enforceSchema`` option
+            to avoid incorrect results.
+        ignoreLeadingWhiteSpace : str or bool, optional
+            A flag indicating whether or not leading whitespaces from
+            values being read should be skipped. If None is set, it
+            uses the default value, ``false``.
+        ignoreTrailingWhiteSpace : str or bool, optional
+            A flag indicating whether or not trailing whitespaces from
+            values being read should be skipped. If None is set, it
+            uses the default value, ``false``.
+        nullValue : str, optional
+            sets the string representation of a null value. If None is set, it uses
+            the default value, empty string. Since 2.0.1, this ``nullValue`` param
+            applies to all supported types including the string type.
+        nanValue : str, optional
+            sets the string representation of a non-number value. If None is set, it
+            uses the default value, ``NaN``.
+        positiveInf : str, optional
+            sets the string representation of a positive infinity value. If None
+            is set, it uses the default value, ``Inf``.
+        negativeInf : str, optional
+            sets the string representation of a negative infinity value. If None
+            is set, it uses the default value, ``Inf``.
+        dateFormat : str, optional
+            sets the string that indicates a date format. Custom date formats
+            follow the formats at
+            `datetime pattern <https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html>`_.  # noqa
+            This applies to date type. If None is set, it uses the
+            default value, ``yyyy-MM-dd``.
+        timestampFormat : str, optional
+            sets the string that indicates a timestamp format.
+            Custom date formats follow the formats at
+            `datetime pattern <https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html>`_.  # noqa
+            This applies to timestamp type. If None is set, it uses the
+            default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``.
+        maxColumns : str or int, optional
+            defines a hard limit of how many columns a record can have. If None is
+            set, it uses the default value, ``20480``.
+        maxCharsPerColumn : str or int, optional
+            defines the maximum number of characters allowed for any given
+            value being read. If None is set, it uses the default value,
+            ``-1`` meaning unlimited length.
+        maxMalformedLogPerPartition : str or int, optional
+            this parameter is no longer used since Spark 2.2.0.
+            If specified, it is ignored.
+        mode : str, optional
+            allows a mode for dealing with corrupt records during parsing. If None is
+            set, it uses the default value, ``PERMISSIVE``. Note that Spark tries to
+            parse only required columns in CSV under column pruning. Therefore, corrupt
+            records can be different based on required set of fields. This behavior can
+            be controlled by ``spark.sql.csv.parser.columnPruning.enabled``
+            (enabled by default).
+
+            * ``PERMISSIVE``: when it meets a corrupted record, puts the malformed string \
+              into a field configured by ``columnNameOfCorruptRecord``, and sets malformed \
+              fields to ``null``. To keep corrupt records, an user can set a string type \
+              field named ``columnNameOfCorruptRecord`` in an user-defined schema. If a \
+              schema does not have the field, it drops corrupt records during parsing. \
+              A record with less/more tokens than schema is not a corrupted record to CSV. \
+              When it meets a record having fewer tokens than the length of the schema, \
+              sets ``null`` to extra fields. When the record has more tokens than the \
+              length of the schema, it drops extra tokens.
+            * ``DROPMALFORMED``: ignores the whole corrupted records.
+            * ``FAILFAST``: throws an exception when it meets corrupted records.
+
+        columnNameOfCorruptRecord : str, optional
+            allows renaming the new field having malformed string
+            created by ``PERMISSIVE`` mode. This overrides
+            ``spark.sql.columnNameOfCorruptRecord``. If None is set,
+            it uses the value specified in
+            ``spark.sql.columnNameOfCorruptRecord``.
+        multiLine : str or bool, optional
+            parse records, which may span multiple lines. If None is
+            set, it uses the default value, ``false``.
+        charToEscapeQuoteEscaping : str, optional
+            sets a single character used for escaping the escape for
+            the quote character. If None is set, the default value is
+            escape character when escape and quote characters are
+            different, ``\0`` otherwise.
+        samplingRatio : str or float, optional
+            defines fraction of rows used for schema inferring.
+            If None is set, it uses the default value, ``1.0``.
+        emptyValue : str, optional
+            sets the string representation of an empty value. If None is set, it uses
+            the default value, empty string.
+        locale : str, optional
+            sets a locale as language tag in IETF BCP 47 format. If None is set,
+            it uses the default value, ``en-US``. For instance, ``locale`` is used while
+            parsing dates and timestamps.
+        lineSep : str, optional
+            defines the line separator that should be used for parsing. If None is
+            set, it covers all ``\\r``, ``\\r\\n`` and ``\\n``.
+            Maximum length is 1 character.
+        pathGlobFilter : str or bool, optional
+            an optional glob pattern to only include files with paths matching
+            the pattern. The syntax follows `org.apache.hadoop.fs.GlobFilter`.
+            It does not change the behavior of
+            `partition discovery <https://spark.apache.org/docs/latest/sql-data-sources-parquet.html#partition-discovery>`_.  # noqa
+        recursiveFileLookup : str or bool, optional
+            recursively scan a directory for files. Using this option disables
+            `partition discovery <https://spark.apache.org/docs/latest/sql-data-sources-parquet.html#partition-discovery>`_.  # noqa
+
+            modification times occurring before the specified time. The provided timestamp
+            must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)
+        modifiedBefore (batch only) : an optional timestamp to only include files with
+            modification times occurring before the specified time. The provided timestamp
+            must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)
+        modifiedAfter (batch only) : an optional timestamp to only include files with
+            modification times occurring after the specified time. The provided timestamp
+            must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)
+        unescapedQuoteHandling : str, optional
+            defines how the CsvParser will handle values with unescaped quotes. If None is
+            set, it uses the default value, ``STOP_AT_DELIMITER``.
+
+            * ``STOP_AT_CLOSING_QUOTE``: If unescaped quotes are found in the input, accumulate
+              the quote character and proceed parsing the value as a quoted value, until a closing
+              quote is found.
+            * ``BACK_TO_DELIMITER``: If unescaped quotes are found in the input, consider the value
+              as an unquoted value. This will make the parser accumulate all characters of the current
+              parsed value until the delimiter is found. If no delimiter is found in the value, the
+              parser will continue accumulating characters from the input until a delimiter or line
+              ending is found.
+            * ``STOP_AT_DELIMITER``: If unescaped quotes are found in the input, consider the value
+              as an unquoted value. This will make the parser accumulate all characters until the
+              delimiter or a line ending is found in the input.
+            * ``SKIP_VALUE``: If unescaped quotes are found in the input, the content parsed
+              for the given value will be skipped and the value set in nullValue will be produced
+              instead.
+            * ``RAISE_ERROR``: If unescaped quotes are found in the input, a TextParsingException
+              will be thrown.
+
+        Examples
+        --------
         >>> df = spark.read.csv('python/test_support/sql/ages.csv')
         >>> df.dtypes
         [('_c0', 'string'), ('_c1', 'string')]
@@ -528,17 +755,19 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
             columnNameOfCorruptRecord=columnNameOfCorruptRecord, multiLine=multiLine,
             charToEscapeQuoteEscaping=charToEscapeQuoteEscaping, samplingRatio=samplingRatio,
             enforceSchema=enforceSchema, emptyValue=emptyValue, locale=locale, lineSep=lineSep,
-            pathGlobFilter=pathGlobFilter, recursiveFileLookup=recursiveFileLookup)
-        if isinstance(path, basestring):
+            pathGlobFilter=pathGlobFilter, recursiveFileLookup=recursiveFileLookup,
+            modifiedBefore=modifiedBefore, modifiedAfter=modifiedAfter,
+            unescapedQuoteHandling=unescapedQuoteHandling)
+        if isinstance(path, str):
             path = [path]
         if type(path) == list:
             return self._df(self._jreader.csv(self._spark._sc._jvm.PythonUtils.toSeq(path)))
         elif isinstance(path, RDD):
             def func(iterator):
                 for x in iterator:
-                    if not isinstance(x, basestring):
-                        x = unicode(x)
-                    if isinstance(x, unicode):
+                    if not isinstance(x, str):
+                        x = str(x)
+                    if isinstance(x, str):
                         x = x.encode("utf-8")
                     yield x
             keyed = path.mapPartitions(func)
@@ -555,30 +784,51 @@ def func(iterator):
         else:
             raise TypeError("path can be only string, list or RDD")
 
-    @since(1.5)
-    def orc(self, path, mergeSchema=None, pathGlobFilter=None, recursiveFileLookup=None):
+    def orc(self, path, mergeSchema=None, pathGlobFilter=None, recursiveFileLookup=None,
+            modifiedBefore=None, modifiedAfter=None):
         """Loads ORC files, returning the result as a :class:`DataFrame`.
 
-        :param mergeSchema: sets whether we should merge schemas collected from all
-                            ORC part-files. This will override ``spark.sql.orc.mergeSchema``.
-                            The default value is specified in ``spark.sql.orc.mergeSchema``.
-        :param pathGlobFilter: an optional glob pattern to only include files with paths matching
-                               the pattern. The syntax follows `org.apache.hadoop.fs.GlobFilter`.
-                               It does not change the behavior of `partition discovery`_.
-        :param recursiveFileLookup: recursively scan a directory for files. Using this option
-                                    disables `partition discovery`_.
-
+        .. versionadded:: 1.5.0
+
+        Parameters
+        ----------
+        path : str or list
+        mergeSchema : str or bool, optional
+            sets whether we should merge schemas collected from all
+            ORC part-files. This will override ``spark.sql.orc.mergeSchema``.
+            The default value is specified in ``spark.sql.orc.mergeSchema``.
+        pathGlobFilter : str or bool
+            an optional glob pattern to only include files with paths matching
+            the pattern. The syntax follows `org.apache.hadoop.fs.GlobFilter`.
+            It does not change the behavior of
+            `partition discovery <https://spark.apache.org/docs/latest/sql-data-sources-parquet.html#partition-discovery>`_.  # noqa
+        recursiveFileLookup : str or bool
+            recursively scan a directory for files. Using this option
+            disables
+            `partition discovery <https://spark.apache.org/docs/latest/sql-data-sources-parquet.html#partition-discovery>`_.  # noqa
+
+            modification times occurring before the specified time. The provided timestamp
+            must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)
+        modifiedBefore : an optional timestamp to only include files with
+            modification times occurring before the specified time. The provided timestamp
+            must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)
+        modifiedAfter : an optional timestamp to only include files with
+            modification times occurring after the specified time. The provided timestamp
+            must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)
+
+        Examples
+        --------
         >>> df = spark.read.orc('python/test_support/sql/orc_partitioned')
         >>> df.dtypes
         [('a', 'bigint'), ('b', 'int'), ('c', 'int')]
         """
         self._set_opts(mergeSchema=mergeSchema, pathGlobFilter=pathGlobFilter,
+                       modifiedBefore=modifiedBefore, modifiedAfter=modifiedAfter,
                        recursiveFileLookup=recursiveFileLookup)
-        if isinstance(path, basestring):
+        if isinstance(path, str):
             path = [path]
         return self._df(self._jreader.orc(_to_seq(self._spark._sc, path)))
 
-    @since(1.4)
     def jdbc(self, url, table, column=None, lowerBound=None, upperBound=None, numPartitions=None,
              predicates=None, properties=None):
         """
@@ -586,31 +836,48 @@ def jdbc(self, url, table, column=None, lowerBound=None, upperBound=None, numPar
         accessible via JDBC URL ``url`` and connection ``properties``.
 
         Partitions of the table will be retrieved in parallel if either ``column`` or
-        ``predicates`` is specified. ``lowerBound`, ``upperBound`` and ``numPartitions``
+        ``predicates`` is specified. ``lowerBound``, ``upperBound`` and ``numPartitions``
         is needed when ``column`` is specified.
 
         If both ``column`` and ``predicates`` are specified, ``column`` will be used.
 
-        .. note:: Don't create too many partitions in parallel on a large cluster;
-            otherwise Spark might crash your external database systems.
-
-        :param url: a JDBC URL of the form ``jdbc:subprotocol:subname``
-        :param table: the name of the table
-        :param column: the name of a column of numeric, date, or timestamp type
-                       that will be used for partitioning;
-                       if this parameter is specified, then ``numPartitions``, ``lowerBound``
-                       (inclusive), and ``upperBound`` (exclusive) will form partition strides
-                       for generated WHERE clause expressions used to split the column
-                       ``column`` evenly
-        :param lowerBound: the minimum value of ``column`` used to decide partition stride
-        :param upperBound: the maximum value of ``column`` used to decide partition stride
-        :param numPartitions: the number of partitions
-        :param predicates: a list of expressions suitable for inclusion in WHERE clauses;
-                           each one defines one partition of the :class:`DataFrame`
-        :param properties: a dictionary of JDBC database connection arguments. Normally at
-                           least properties "user" and "password" with their corresponding values.
-                           For example { 'user' : 'SYSTEM', 'password' : 'mypassword' }
-        :return: a DataFrame
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        url : str
+            a JDBC URL of the form ``jdbc:subprotocol:subname``
+        table : str
+            the name of the table
+        column : str, optional
+            the name of a column of numeric, date, or timestamp type
+            that will be used for partitioning;
+            if this parameter is specified, then ``numPartitions``, ``lowerBound``
+            (inclusive), and ``upperBound`` (exclusive) will form partition strides
+            for generated WHERE clause expressions used to split the column
+            ``column`` evenly
+        lowerBound : str or int, optional
+            the minimum value of ``column`` used to decide partition stride
+        upperBound : str or int, optional
+            the maximum value of ``column`` used to decide partition stride
+        numPartitions : int, optional
+            the number of partitions
+        predicates : list, optional
+            a list of expressions suitable for inclusion in WHERE clauses;
+            each one defines one partition of the :class:`DataFrame`
+        properties : dict, optional
+            a dictionary of JDBC database connection arguments. Normally at
+            least properties "user" and "password" with their corresponding values.
+            For example { 'user' : 'SYSTEM', 'password' : 'mypassword' }
+
+        Notes
+        -----
+        Don't create too many partitions in parallel on a large cluster;
+        otherwise Spark might crash your external database systems.
+
+        Returns
+        -------
+        :class:`DataFrame`
         """
         if properties is None:
             properties = dict()
@@ -648,7 +915,6 @@ def _sq(self, jsq):
         from pyspark.sql.streaming import StreamingQuery
         return StreamingQuery(jsq)
 
-    @since(1.4)
     def mode(self, saveMode):
         """Specifies the behavior when data or table already exists.
 
@@ -659,6 +925,10 @@ def mode(self, saveMode):
         * `error` or `errorifexists`: Throw an exception if data already exists.
         * `ignore`: Silently ignore this operation if data already exists.
 
+        .. versionadded:: 1.4.0
+
+        Examples
+        --------
         >>> df.write.mode('append').parquet(os.path.join(tempfile.mkdtemp(), 'data'))
         """
         # At the JVM side, the default value of mode is already set to "error".
@@ -667,12 +937,18 @@ def mode(self, saveMode):
             self._jwrite = self._jwrite.mode(saveMode)
         return self
 
-    @since(1.4)
     def format(self, source):
         """Specifies the underlying output data source.
 
-        :param source: string, name of the data source, e.g. 'json', 'parquet'.
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        source : str
+            string, name of the data source, e.g. 'json', 'parquet'.
 
+        Examples
+        --------
         >>> df.write.format('json').save(os.path.join(tempfile.mkdtemp(), 'data'))
         """
         self._jwrite = self._jwrite.format(source)
@@ -721,15 +997,21 @@ def options(self, **options):
             self._jwrite = self._jwrite.option(k, to_str(options[k]))
         return self
 
-    @since(1.4)
     def partitionBy(self, *cols):
         """Partitions the output by the given columns on the file system.
 
         If specified, the output is laid out on the file system similar
         to Hive's partitioning scheme.
 
-        :param cols: name of columns
+        .. versionadded:: 1.4.0
 
+        Parameters
+        ----------
+        cols : str or list
+            name of columns
+
+        Examples
+        --------
         >>> df.write.partitionBy('year', 'month').parquet(os.path.join(tempfile.mkdtemp(), 'data'))
         """
         if len(cols) == 1 and isinstance(cols[0], (list, tuple)):
@@ -737,18 +1019,28 @@ def partitionBy(self, *cols):
         self._jwrite = self._jwrite.partitionBy(_to_seq(self._spark._sc, cols))
         return self
 
-    @since(2.3)
     def bucketBy(self, numBuckets, col, *cols):
         """Buckets the output by the given columns.If specified,
         the output is laid out on the file system similar to Hive's bucketing scheme.
 
-        :param numBuckets: the number of buckets to save
-        :param col: a name of a column, or a list of names.
-        :param cols: additional names (optional). If `col` is a list it should be empty.
+        .. versionadded:: 2.3.0
+
+        Parameters
+        ----------
+        numBuckets : int
+            the number of buckets to save
+        col : str, list or tuple
+            a name of a column, or a list of names.
+        cols : str
+            additional names (optional). If `col` is a list it should be empty.
 
-        .. note:: Applicable for file-based data sources in combination with
-                  :py:meth:`DataFrameWriter.saveAsTable`.
+        Notes
+        -----
+        Applicable for file-based data sources in combination with
+        :py:meth:`DataFrameWriter.saveAsTable`.
 
+        Examples
+        --------
         >>> (df.write.format('parquet')  # doctest: +SKIP
         ...     .bucketBy(100, 'year', 'month')
         ...     .mode("overwrite")
@@ -763,19 +1055,26 @@ def bucketBy(self, numBuckets, col, *cols):
 
             col, cols = col[0], col[1:]
 
-        if not all(isinstance(c, basestring) for c in cols) or not(isinstance(col, basestring)):
+        if not all(isinstance(c, str) for c in cols) or not(isinstance(col, str)):
             raise TypeError("all names should be `str`")
 
         self._jwrite = self._jwrite.bucketBy(numBuckets, col, _to_seq(self._spark._sc, cols))
         return self
 
-    @since(2.3)
     def sortBy(self, col, *cols):
         """Sorts the output in each bucket by the given columns on the file system.
 
-        :param col: a name of a column, or a list of names.
-        :param cols: additional names (optional). If `col` is a list it should be empty.
+        .. versionadded:: 2.3.0
 
+        Parameters
+        ----------
+        col : str, tuple or list
+            a name of a column, or a list of names.
+        cols : str
+            additional names (optional). If `col` is a list it should be empty.
+
+        Examples
+        --------
         >>> (df.write.format('parquet')  # doctest: +SKIP
         ...     .bucketBy(100, 'year', 'month')
         ...     .sortBy('day')
@@ -788,13 +1087,12 @@ def sortBy(self, col, *cols):
 
             col, cols = col[0], col[1:]
 
-        if not all(isinstance(c, basestring) for c in cols) or not(isinstance(col, basestring)):
+        if not all(isinstance(c, str) for c in cols) or not(isinstance(col, str)):
             raise TypeError("all names should be `str`")
 
         self._jwrite = self._jwrite.sortBy(col, _to_seq(self._spark._sc, cols))
         return self
 
-    @since(1.4)
     def save(self, path=None, format=None, mode=None, partitionBy=None, **options):
         """Saves the contents of the :class:`DataFrame` to a data source.
 
@@ -802,18 +1100,29 @@ def save(self, path=None, format=None, mode=None, partitionBy=None, **options):
         If ``format`` is not specified, the default data source configured by
         ``spark.sql.sources.default`` will be used.
 
-        :param path: the path in a Hadoop supported file system
-        :param format: the format used to save
-        :param mode: specifies the behavior of the save operation when data already exists.
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        path : str, optional
+            the path in a Hadoop supported file system
+        format : str, optional
+            the format used to save
+        mode : str, optional
+            specifies the behavior of the save operation when data already exists.
 
             * ``append``: Append contents of this :class:`DataFrame` to existing data.
             * ``overwrite``: Overwrite existing data.
             * ``ignore``: Silently ignore this operation if data already exists.
             * ``error`` or ``errorifexists`` (default case): Throw an exception if data already \
                 exists.
-        :param partitionBy: names of partitioning columns
-        :param options: all other string options
+        partitionBy : list, optional
+            names of partitioning columns
+        **options : dict
+            all other string options
 
+        Examples
+        --------
         >>> df.write.mode("append").save(os.path.join(tempfile.mkdtemp(), 'data'))
         """
         self.mode(mode).options(**options)
@@ -839,7 +1148,6 @@ def insertInto(self, tableName, overwrite=None):
             self.mode("overwrite" if overwrite else "append")
         self._jwrite.insertInto(tableName)
 
-    @since(1.4)
     def saveAsTable(self, name, format=None, mode=None, partitionBy=None, **options):
         """Saves the content of the :class:`DataFrame` as the specified table.
 
@@ -853,12 +1161,21 @@ def saveAsTable(self, name, format=None, mode=None, partitionBy=None, **options)
         * `error` or `errorifexists`: Throw an exception if data already exists.
         * `ignore`: Silently ignore this operation if data already exists.
 
-        :param name: the table name
-        :param format: the format used to save
-        :param mode: one of `append`, `overwrite`, `error`, `errorifexists`, `ignore` \
-                     (default: error)
-        :param partitionBy: names of partitioning columns
-        :param options: all other string options
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        name : str
+            the table name
+        format : str, optional
+            the format used to save
+        mode : str, optional
+            one of `append`, `overwrite`, `error`, `errorifexists`, `ignore` \
+            (default: error)
+        partitionBy : str or list
+            names of partitioning columns
+        **options : dict
+            all other string options
         """
         self.mode(mode).options(**options)
         if partitionBy is not None:
@@ -867,39 +1184,54 @@ def saveAsTable(self, name, format=None, mode=None, partitionBy=None, **options)
             self.format(format)
         self._jwrite.saveAsTable(name)
 
-    @since(1.4)
     def json(self, path, mode=None, compression=None, dateFormat=None, timestampFormat=None,
              lineSep=None, encoding=None, ignoreNullFields=None):
         """Saves the content of the :class:`DataFrame` in JSON format
         (`JSON Lines text format or newline-delimited JSON <http://jsonlines.org/>`_) at the
         specified path.
 
-        :param path: the path in any Hadoop supported file system
-        :param mode: specifies the behavior of the save operation when data already exists.
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        path : str
+            the path in any Hadoop supported file system
+        mode : str, optional
+            specifies the behavior of the save operation when data already exists.
 
             * ``append``: Append contents of this :class:`DataFrame` to existing data.
             * ``overwrite``: Overwrite existing data.
             * ``ignore``: Silently ignore this operation if data already exists.
             * ``error`` or ``errorifexists`` (default case): Throw an exception if data already \
                 exists.
-        :param compression: compression codec to use when saving to file. This can be one of the
-                            known case-insensitive shorten names (none, bzip2, gzip, lz4,
-                            snappy and deflate).
-        :param dateFormat: sets the string that indicates a date format. Custom date formats
-                           follow the formats at `datetime pattern`_.
-                           This applies to date type. If None is set, it uses the
-                           default value, ``yyyy-MM-dd``.
-        :param timestampFormat: sets the string that indicates a timestamp format.
-                                Custom date formats follow the formats at `datetime pattern`_.
-                                This applies to timestamp type. If None is set, it uses the
-                                default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``.
-        :param encoding: specifies encoding (charset) of saved json files. If None is set,
-                        the default UTF-8 charset will be used.
-        :param lineSep: defines the line separator that should be used for writing. If None is
-                        set, it uses the default value, ``\\n``.
-        :param ignoreNullFields: Whether to ignore null fields when generating JSON objects.
-                        If None is set, it uses the default value, ``true``.
-
+        compression : str, optional
+            compression codec to use when saving to file. This can be one of the
+            known case-insensitive shorten names (none, bzip2, gzip, lz4,
+            snappy and deflate).
+        dateFormat : str, optional
+            sets the string that indicates a date format. Custom date formats
+            follow the formats at
+            `datetime pattern <https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html>`_.  # noqa
+            This applies to date type. If None is set, it uses the
+            default value, ``yyyy-MM-dd``.
+        timestampFormat : str, optional
+            sets the string that indicates a timestamp format.
+            Custom date formats follow the formats at
+            `datetime pattern <https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html>`_.  # noqa
+            This applies to timestamp type. If None is set, it uses the
+            default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``.
+        encoding : str, optional
+            specifies encoding (charset) of saved json files. If None is set,
+            the default UTF-8 charset will be used.
+        lineSep : str, optional
+            defines the line separator that should be used for writing. If None is
+            set, it uses the default value, ``\\n``.
+        ignoreNullFields : str or bool, optional
+            Whether to ignore null fields when generating JSON objects.
+            If None is set, it uses the default value, ``true``.
+
+        Examples
+        --------
         >>> df.write.json(os.path.join(tempfile.mkdtemp(), 'data'))
         """
         self.mode(mode)
@@ -908,25 +1240,34 @@ def json(self, path, mode=None, compression=None, dateFormat=None, timestampForm
             lineSep=lineSep, encoding=encoding, ignoreNullFields=ignoreNullFields)
         self._jwrite.json(path)
 
-    @since(1.4)
     def parquet(self, path, mode=None, partitionBy=None, compression=None):
         """Saves the content of the :class:`DataFrame` in Parquet format at the specified path.
 
-        :param path: the path in any Hadoop supported file system
-        :param mode: specifies the behavior of the save operation when data already exists.
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        path : str
+            the path in any Hadoop supported file system
+        mode : str, optional
+            specifies the behavior of the save operation when data already exists.
 
             * ``append``: Append contents of this :class:`DataFrame` to existing data.
             * ``overwrite``: Overwrite existing data.
             * ``ignore``: Silently ignore this operation if data already exists.
             * ``error`` or ``errorifexists`` (default case): Throw an exception if data already \
                 exists.
-        :param partitionBy: names of partitioning columns
-        :param compression: compression codec to use when saving to file. This can be one of the
-                            known case-insensitive shorten names (none, uncompressed, snappy, gzip,
-                            lzo, brotli, lz4, and zstd). This will override
-                            ``spark.sql.parquet.compression.codec``. If None is set, it uses the
-                            value specified in ``spark.sql.parquet.compression.codec``.
-
+        partitionBy : str or list, optional
+            names of partitioning columns
+        compression : str, optional
+            compression codec to use when saving to file. This can be one of the
+            known case-insensitive shorten names (none, uncompressed, snappy, gzip,
+            lzo, brotli, lz4, and zstd). This will override
+            ``spark.sql.parquet.compression.codec``. If None is set, it uses the
+            value specified in ``spark.sql.parquet.compression.codec``.
+
+        Examples
+        --------
         >>> df.write.parquet(os.path.join(tempfile.mkdtemp(), 'data'))
         """
         self.mode(mode)
@@ -935,17 +1276,23 @@ def parquet(self, path, mode=None, partitionBy=None, compression=None):
         self._set_opts(compression=compression)
         self._jwrite.parquet(path)
 
-    @since(1.6)
     def text(self, path, compression=None, lineSep=None):
         """Saves the content of the DataFrame in a text file at the specified path.
         The text files will be encoded as UTF-8.
 
-        :param path: the path in any Hadoop supported file system
-        :param compression: compression codec to use when saving to file. This can be one of the
-                            known case-insensitive shorten names (none, bzip2, gzip, lz4,
-                            snappy and deflate).
-        :param lineSep: defines the line separator that should be used for writing. If None is
-                        set, it uses the default value, ``\\n``.
+        .. versionadded:: 1.6.0
+
+        Parameters
+        ----------
+        path : str
+            the path in any Hadoop supported file system
+        compression : str, optional
+            compression codec to use when saving to file. This can be one of the
+            known case-insensitive shorten names (none, bzip2, gzip, lz4,
+            snappy and deflate).
+        lineSep : str, optional
+            defines the line separator that should be used for writing. If None is
+            set, it uses the default value, ``\\n``.
 
         The DataFrame must have only one column that is of string type.
         Each row becomes a new line in the output file.
@@ -953,15 +1300,20 @@ def text(self, path, compression=None, lineSep=None):
         self._set_opts(compression=compression, lineSep=lineSep)
         self._jwrite.text(path)
 
-    @since(2.0)
     def csv(self, path, mode=None, compression=None, sep=None, quote=None, escape=None,
             header=None, nullValue=None, escapeQuotes=None, quoteAll=None, dateFormat=None,
             timestampFormat=None, ignoreLeadingWhiteSpace=None, ignoreTrailingWhiteSpace=None,
             charToEscapeQuoteEscaping=None, encoding=None, emptyValue=None, lineSep=None):
         r"""Saves the content of the :class:`DataFrame` in CSV format at the specified path.
 
-        :param path: the path in any Hadoop supported file system
-        :param mode: specifies the behavior of the save operation when data already exists.
+        .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        path : str
+            the path in any Hadoop supported file system
+        mode : str, optional
+            specifies the behavior of the save operation when data already exists.
 
             * ``append``: Append contents of this :class:`DataFrame` to existing data.
             * ``overwrite``: Overwrite existing data.
@@ -969,51 +1321,71 @@ def csv(self, path, mode=None, compression=None, sep=None, quote=None, escape=No
             * ``error`` or ``errorifexists`` (default case): Throw an exception if data already \
                 exists.
 
-        :param compression: compression codec to use when saving to file. This can be one of the
-                            known case-insensitive shorten names (none, bzip2, gzip, lz4,
-                            snappy and deflate).
-        :param sep: sets a separator (one or more characters) for each field and value. If None is
-                    set, it uses the default value, ``,``.
-        :param quote: sets a single character used for escaping quoted values where the
-                      separator can be part of the value. If None is set, it uses the default
-                      value, ``"``. If an empty string is set, it uses ``u0000`` (null character).
-        :param escape: sets a single character used for escaping quotes inside an already
-                       quoted value. If None is set, it uses the default value, ``\``
-        :param escapeQuotes: a flag indicating whether values containing quotes should always
-                             be enclosed in quotes. If None is set, it uses the default value
-                             ``true``, escaping all values containing a quote character.
-        :param quoteAll: a flag indicating whether all values should always be enclosed in
-                          quotes. If None is set, it uses the default value ``false``,
-                          only escaping values containing a quote character.
-        :param header: writes the names of columns as the first line. If None is set, it uses
-                       the default value, ``false``.
-        :param nullValue: sets the string representation of a null value. If None is set, it uses
-                          the default value, empty string.
-        :param dateFormat: sets the string that indicates a date format. Custom date formats follow
-                           the formats at `datetime pattern`_.
-                           This applies to date type. If None is set, it uses the
-                           default value, ``yyyy-MM-dd``.
-        :param timestampFormat: sets the string that indicates a timestamp format.
-                                Custom date formats follow the formats at `datetime pattern`_.
-                                This applies to timestamp type. If None is set, it uses the
-                                default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``.
-        :param ignoreLeadingWhiteSpace: a flag indicating whether or not leading whitespaces from
-                                        values being written should be skipped. If None is set, it
-                                        uses the default value, ``true``.
-        :param ignoreTrailingWhiteSpace: a flag indicating whether or not trailing whitespaces from
-                                         values being written should be skipped. If None is set, it
-                                         uses the default value, ``true``.
-        :param charToEscapeQuoteEscaping: sets a single character used for escaping the escape for
-                                          the quote character. If None is set, the default value is
-                                          escape character when escape and quote characters are
-                                          different, ``\0`` otherwise..
-        :param encoding: sets the encoding (charset) of saved csv files. If None is set,
-                         the default UTF-8 charset will be used.
-        :param emptyValue: sets the string representation of an empty value. If None is set, it uses
-                           the default value, ``""``.
-        :param lineSep: defines the line separator that should be used for writing. If None is
-                        set, it uses the default value, ``\\n``. Maximum length is 1 character.
-
+        compression : str, optional
+            compression codec to use when saving to file. This can be one of the
+            known case-insensitive shorten names (none, bzip2, gzip, lz4,
+            snappy and deflate).
+        sep : str, optional
+            sets a separator (one or more characters) for each field and value. If None is
+            set, it uses the default value, ``,``.
+        quote : str, optional
+            sets a single character used for escaping quoted values where the
+            separator can be part of the value. If None is set, it uses the default
+            value, ``"``. If an empty string is set, it uses ``u0000`` (null character).
+        escape : str, optional
+            sets a single character used for escaping quotes inside an already
+            quoted value. If None is set, it uses the default value, ``\``
+        escapeQuotes : str or bool, optional
+            a flag indicating whether values containing quotes should always
+            be enclosed in quotes. If None is set, it uses the default value
+            ``true``, escaping all values containing a quote character.
+        quoteAll : str or bool, optional
+            a flag indicating whether all values should always be enclosed in
+            quotes. If None is set, it uses the default value ``false``,
+            only escaping values containing a quote character.
+        header : str or bool, optional
+            writes the names of columns as the first line. If None is set, it uses
+            the default value, ``false``.
+        nullValue : str, optional
+            sets the string representation of a null value. If None is set, it uses
+            the default value, empty string.
+        dateFormat : str, optional
+            sets the string that indicates a date format. Custom date formats follow
+            the formats at
+            `datetime pattern <https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html>`_.  # noqa
+            This applies to date type. If None is set, it uses the
+            default value, ``yyyy-MM-dd``.
+        timestampFormat : str, optional
+            sets the string that indicates a timestamp format.
+            Custom date formats follow the formats at
+            `datetime pattern <https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html>`_.  # noqa
+            This applies to timestamp type. If None is set, it uses the
+            default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``.
+        ignoreLeadingWhiteSpace : str or bool, optional
+            a flag indicating whether or not leading whitespaces from
+            values being written should be skipped. If None is set, it
+            uses the default value, ``true``.
+        ignoreTrailingWhiteSpace : str or bool, optional
+            a flag indicating whether or not trailing whitespaces from
+            values being written should be skipped. If None is set, it
+            uses the default value, ``true``.
+        charToEscapeQuoteEscaping : str, optional
+            sets a single character used for escaping the escape for
+            the quote character. If None is set, the default value is
+            escape character when escape and quote characters are
+            different, ``\0`` otherwise..
+        encoding : str, optional
+            sets the encoding (charset) of saved csv files. If None is set,
+            the default UTF-8 charset will be used.
+        emptyValue : str, optional
+            sets the string representation of an empty value. If None is set, it uses
+            the default value, ``""``.
+        lineSep : str, optional
+            defines the line separator that should be used for writing. If None is
+            set, it uses the default value, ``\\n``. Maximum length is 1 character.
+
+        Examples
+        --------
         >>> df.write.csv(os.path.join(tempfile.mkdtemp(), 'data'))
         """
         self.mode(mode)
@@ -1026,25 +1398,34 @@ def csv(self, path, mode=None, compression=None, sep=None, quote=None, escape=No
                        encoding=encoding, emptyValue=emptyValue, lineSep=lineSep)
         self._jwrite.csv(path)
 
-    @since(1.5)
     def orc(self, path, mode=None, partitionBy=None, compression=None):
         """Saves the content of the :class:`DataFrame` in ORC format at the specified path.
 
-        :param path: the path in any Hadoop supported file system
-        :param mode: specifies the behavior of the save operation when data already exists.
+        .. versionadded:: 1.5.0
+
+        Parameters
+        ----------
+        path : str
+            the path in any Hadoop supported file system
+        mode : str, optional
+            specifies the behavior of the save operation when data already exists.
 
             * ``append``: Append contents of this :class:`DataFrame` to existing data.
             * ``overwrite``: Overwrite existing data.
             * ``ignore``: Silently ignore this operation if data already exists.
             * ``error`` or ``errorifexists`` (default case): Throw an exception if data already \
                 exists.
-        :param partitionBy: names of partitioning columns
-        :param compression: compression codec to use when saving to file. This can be one of the
-                            known case-insensitive shorten names (none, snappy, zlib, and lzo).
-                            This will override ``orc.compress`` and
-                            ``spark.sql.orc.compression.codec``. If None is set, it uses the value
-                            specified in ``spark.sql.orc.compression.codec``.
-
+        partitionBy : str or list, optional
+            names of partitioning columns
+        compression : str, optional
+            compression codec to use when saving to file. This can be one of the
+            known case-insensitive shorten names (none, snappy, zlib, lzo, and zstd).
+            This will override ``orc.compress`` and
+            ``spark.sql.orc.compression.codec``. If None is set, it uses the value
+            specified in ``spark.sql.orc.compression.codec``.
+
+        Examples
+        --------
         >>> orc_df = spark.read.orc('python/test_support/sql/orc_partitioned')
         >>> orc_df.write.orc(os.path.join(tempfile.mkdtemp(), 'data'))
         """
@@ -1054,25 +1435,34 @@ def orc(self, path, mode=None, partitionBy=None, compression=None):
         self._set_opts(compression=compression)
         self._jwrite.orc(path)
 
-    @since(1.4)
     def jdbc(self, url, table, mode=None, properties=None):
         """Saves the content of the :class:`DataFrame` to an external database table via JDBC.
 
-        .. note:: Don't create too many partitions in parallel on a large cluster;
-            otherwise Spark might crash your external database systems.
+        .. versionadded:: 1.4.0
 
-        :param url: a JDBC URL of the form ``jdbc:subprotocol:subname``
-        :param table: Name of the table in the external database.
-        :param mode: specifies the behavior of the save operation when data already exists.
+        Parameters
+        ----------
+        url : str
+            a JDBC URL of the form ``jdbc:subprotocol:subname``
+        table : str
+            Name of the table in the external database.
+        mode : str, optional
+            specifies the behavior of the save operation when data already exists.
 
             * ``append``: Append contents of this :class:`DataFrame` to existing data.
             * ``overwrite``: Overwrite existing data.
             * ``ignore``: Silently ignore this operation if data already exists.
             * ``error`` or ``errorifexists`` (default case): Throw an exception if data already \
                 exists.
-        :param properties: a dictionary of JDBC database connection arguments. Normally at
-                           least properties "user" and "password" with their corresponding values.
-                           For example { 'user' : 'SYSTEM', 'password' : 'mypassword' }
+        properties : dict
+            a dictionary of JDBC database connection arguments. Normally at
+            least properties "user" and "password" with their corresponding values.
+            For example { 'user' : 'SYSTEM', 'password' : 'mypassword' }
+
+        Notes
+        -----
+        Don't create too many partitions in parallel on a large cluster;
+        otherwise Spark might crash your external database systems.
         """
         if properties is None:
             properties = dict()
@@ -1082,13 +1472,151 @@ def jdbc(self, url, table, mode=None, properties=None):
         self.mode(mode)._jwrite.jdbc(url, table, jprop)
 
 
+class DataFrameWriterV2(object):
+    """
+    Interface used to write a class:`pyspark.sql.dataframe.DataFrame`
+    to external storage using the v2 API.
+
+    .. versionadded:: 3.1.0
+    """
+
+    def __init__(self, df, table):
+        self._df = df
+        self._spark = df.sql_ctx
+        self._jwriter = df._jdf.writeTo(table)
+
+    @since(3.1)
+    def using(self, provider):
+        """
+        Specifies a provider for the underlying output data source.
+        Spark's default catalog supports "parquet", "json", etc.
+        """
+        self._jwriter.using(provider)
+        return self
+
+    @since(3.1)
+    def option(self, key, value):
+        """
+        Add a write option.
+        """
+        self._jwriter.option(key, to_str(value))
+        return self
+
+    @since(3.1)
+    def options(self, **options):
+        """
+        Add write options.
+        """
+        options = {k: to_str(v) for k, v in options.items()}
+        self._jwriter.options(options)
+        return self
+
+    @since(3.1)
+    def tableProperty(self, property, value):
+        """
+        Add table property.
+        """
+        self._jwriter.tableProperty(property, value)
+        return self
+
+    @since(3.1)
+    def partitionedBy(self, col, *cols):
+        """
+        Partition the output table created by `create`, `createOrReplace`, or `replace` using
+        the given columns or transforms.
+
+        When specified, the table data will be stored by these values for efficient reads.
+
+        For example, when a table is partitioned by day, it may be stored
+        in a directory layout like:
+
+        * `table/day=2019-06-01/`
+        * `table/day=2019-06-02/`
+
+        Partitioning is one of the most widely used techniques to optimize physical data layout.
+        It provides a coarse-grained index for skipping unnecessary data reads when queries have
+        predicates on the partitioned columns. In order for partitioning to work well, the number
+        of distinct values in each column should typically be less than tens of thousands.
+
+        `col` and `cols` support only the following functions:
+
+        * :py:func:`pyspark.sql.functions.years`
+        * :py:func:`pyspark.sql.functions.months`
+        * :py:func:`pyspark.sql.functions.days`
+        * :py:func:`pyspark.sql.functions.hours`
+        * :py:func:`pyspark.sql.functions.bucket`
+
+        """
+        col = _to_java_column(col)
+        cols = _to_seq(self._spark._sc, [_to_java_column(c) for c in cols])
+        return self
+
+    @since(3.1)
+    def create(self):
+        """
+        Create a new table from the contents of the data frame.
+
+        The new table's schema, partition layout, properties, and other configuration will be
+        based on the configuration set on this writer.
+        """
+        self._jwriter.create()
+
+    @since(3.1)
+    def replace(self):
+        """
+        Replace an existing table with the contents of the data frame.
+
+        The existing table's schema, partition layout, properties, and other configuration will be
+        replaced with the contents of the data frame and the configuration set on this writer.
+        """
+        self._jwriter.replace()
+
+    @since(3.1)
+    def createOrReplace(self):
+        """
+        Create a new table or replace an existing table with the contents of the data frame.
+
+        The output table's schema, partition layout, properties,
+        and other configuration will be based on the contents of the data frame
+        and the configuration set on this writer.
+        If the table exists, its configuration and data will be replaced.
+        """
+        self._jwriter.createOrReplace()
+
+    @since(3.1)
+    def append(self):
+        """
+        Append the contents of the data frame to the output table.
+        """
+        self._jwriter.append()
+
+    @since(3.1)
+    def overwrite(self, condition):
+        """
+        Overwrite rows matching the given filter condition with the contents of the data frame in
+        the output table.
+        """
+        self._jwriter.overwrite(condition)
+
+    @since(3.1)
+    def overwritePartitions(self):
+        """
+        Overwrite all partition for which the data frame contains at least one row with the contents
+        of the data frame in the output table.
+
+        This operation is equivalent to Hive's `INSERT OVERWRITE ... PARTITION`, which replaces
+        partitions dynamically depending on the contents of the data frame.
+        """
+        self._jwriter.overwritePartitions()
+
+
 def _test():
     import doctest
     import os
     import tempfile
     import py4j
     from pyspark.context import SparkContext
-    from pyspark.sql import SparkSession, Row
+    from pyspark.sql import SparkSession
     import pyspark.sql.readwriter
 
     os.chdir(os.environ["SPARK_HOME"])
diff --git a/python/pyspark/sql/readwriter.pyi b/python/pyspark/sql/readwriter.pyi
new file mode 100644
index 0000000000000..c3b9a428f22b3
--- /dev/null
+++ b/python/pyspark/sql/readwriter.pyi
@@ -0,0 +1,257 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import overload
+from typing import Dict, List, Optional, Tuple, Union
+
+from pyspark.sql._typing import OptionalPrimitiveType
+from pyspark.sql.dataframe import DataFrame
+from pyspark.rdd import RDD
+from pyspark.sql.column import Column
+from pyspark.sql.context import SQLContext
+from pyspark.sql.types import StructType
+
+PathOrPaths = Union[str, List[str]]
+TupleOrListOfString = Union[List[str], Tuple[str, ...]]
+
+class OptionUtils: ...
+
+class DataFrameReader(OptionUtils):
+    def __init__(self, spark: SQLContext) -> None: ...
+    def format(self, source: str) -> DataFrameReader: ...
+    def schema(self, schema: Union[StructType, str]) -> DataFrameReader: ...
+    def option(self, key: str, value: OptionalPrimitiveType) -> DataFrameReader: ...
+    def options(self, **options: OptionalPrimitiveType) -> DataFrameReader: ...
+    def load(
+        self,
+        path: Optional[PathOrPaths] = ...,
+        format: Optional[str] = ...,
+        schema: Optional[Union[StructType, str]] = ...,
+        **options: OptionalPrimitiveType
+    ) -> DataFrame: ...
+    def json(
+        self,
+        path: Union[str, List[str], RDD[str]],
+        schema: Optional[Union[StructType, str]] = ...,
+        primitivesAsString: Optional[Union[bool, str]] = ...,
+        prefersDecimal: Optional[Union[bool, str]] = ...,
+        allowComments: Optional[Union[bool, str]] = ...,
+        allowUnquotedFieldNames: Optional[Union[bool, str]] = ...,
+        allowSingleQuotes: Optional[Union[bool, str]] = ...,
+        allowNumericLeadingZero: Optional[Union[bool, str]] = ...,
+        allowBackslashEscapingAnyCharacter: Optional[Union[bool, str]] = ...,
+        mode: Optional[str] = ...,
+        columnNameOfCorruptRecord: Optional[str] = ...,
+        dateFormat: Optional[str] = ...,
+        timestampFormat: Optional[str] = ...,
+        multiLine: Optional[Union[bool, str]] = ...,
+        allowUnquotedControlChars: Optional[Union[bool, str]] = ...,
+        lineSep: Optional[str] = ...,
+        samplingRatio: Optional[Union[float, str]] = ...,
+        dropFieldIfAllNull: Optional[Union[bool, str]] = ...,
+        encoding: Optional[str] = ...,
+        locale: Optional[str] = ...,
+        pathGlobFilter: Optional[Union[bool, str]] = ...,
+        recursiveFileLookup: Optional[Union[bool, str]] = ...,
+        allowNonNumericNumbers: Optional[Union[bool, str]] = ...,
+    ) -> DataFrame: ...
+    def table(self, tableName: str) -> DataFrame: ...
+    def parquet(self, *paths: str, **options: OptionalPrimitiveType) -> DataFrame: ...
+    def text(
+        self,
+        paths: PathOrPaths,
+        wholetext: bool = ...,
+        lineSep: Optional[str] = ...,
+        pathGlobFilter: Optional[Union[bool, str]] = ...,
+        recursiveFileLookup: Optional[Union[bool, str]] = ...,
+    ) -> DataFrame: ...
+    def csv(
+        self,
+        path: PathOrPaths,
+        schema: Optional[Union[StructType, str]] = ...,
+        sep: Optional[str] = ...,
+        encoding: Optional[str] = ...,
+        quote: Optional[str] = ...,
+        escape: Optional[str] = ...,
+        comment: Optional[str] = ...,
+        header: Optional[Union[bool, str]] = ...,
+        inferSchema: Optional[Union[bool, str]] = ...,
+        ignoreLeadingWhiteSpace: Optional[Union[bool, str]] = ...,
+        ignoreTrailingWhiteSpace: Optional[Union[bool, str]] = ...,
+        nullValue: Optional[str] = ...,
+        nanValue: Optional[str] = ...,
+        positiveInf: Optional[str] = ...,
+        negativeInf: Optional[str] = ...,
+        dateFormat: Optional[str] = ...,
+        timestampFormat: Optional[str] = ...,
+        maxColumns: Optional[Union[int, str]] = ...,
+        maxCharsPerColumn: Optional[Union[int, str]] = ...,
+        maxMalformedLogPerPartition: Optional[Union[int, str]] = ...,
+        mode: Optional[str] = ...,
+        columnNameOfCorruptRecord: Optional[str] = ...,
+        multiLine: Optional[Union[bool, str]] = ...,
+        charToEscapeQuoteEscaping: Optional[str] = ...,
+        samplingRatio: Optional[Union[float, str]] = ...,
+        enforceSchema: Optional[Union[bool, str]] = ...,
+        emptyValue: Optional[str] = ...,
+        locale: Optional[str] = ...,
+        lineSep: Optional[str] = ...,
+        pathGlobFilter: Optional[Union[bool, str]] = ...,
+        recursiveFileLookup: Optional[Union[bool, str]] = ...,
+        unescapedQuoteHandling: Optional[str] = ...,
+    ) -> DataFrame: ...
+    def orc(
+        self,
+        path: PathOrPaths,
+        mergeSchema: Optional[bool] = ...,
+        pathGlobFilter: Optional[Union[bool, str]] = ...,
+        recursiveFileLookup: Optional[Union[bool, str]] = ...,
+    ) -> DataFrame: ...
+    @overload
+    def jdbc(
+        self, url: str, table: str, *, properties: Optional[Dict[str, str]] = ...
+    ) -> DataFrame: ...
+    @overload
+    def jdbc(
+        self,
+        url: str,
+        table: str,
+        column: str,
+        lowerBound: Union[int, str],
+        upperBound: Union[int, str],
+        numPartitions: int,
+        *,
+        properties: Optional[Dict[str, str]] = ...
+    ) -> DataFrame: ...
+    @overload
+    def jdbc(
+        self,
+        url: str,
+        table: str,
+        *,
+        predicates: List[str],
+        properties: Optional[Dict[str, str]] = ...
+    ) -> DataFrame: ...
+
+class DataFrameWriter(OptionUtils):
+    def __init__(self, df: DataFrame) -> None: ...
+    def mode(self, saveMode: str) -> DataFrameWriter: ...
+    def format(self, source: str) -> DataFrameWriter: ...
+    def option(self, key: str, value: OptionalPrimitiveType) -> DataFrameWriter: ...
+    def options(self, **options: OptionalPrimitiveType) -> DataFrameWriter: ...
+    @overload
+    def partitionBy(self, *cols: str) -> DataFrameWriter: ...
+    @overload
+    def partitionBy(self, __cols: List[str]) -> DataFrameWriter: ...
+    @overload
+    def bucketBy(self, numBuckets: int, col: str, *cols: str) -> DataFrameWriter: ...
+    @overload
+    def bucketBy(
+        self, numBuckets: int, col: TupleOrListOfString
+    ) -> DataFrameWriter: ...
+    @overload
+    def sortBy(self, col: str, *cols: str) -> DataFrameWriter: ...
+    @overload
+    def sortBy(self, col: TupleOrListOfString) -> DataFrameWriter: ...
+    def save(
+        self,
+        path: Optional[str] = ...,
+        format: Optional[str] = ...,
+        mode: Optional[str] = ...,
+        partitionBy: Optional[Union[str, List[str]]] = ...,
+        **options: OptionalPrimitiveType
+    ) -> None: ...
+    def insertInto(self, tableName: str, overwrite: Optional[bool] = ...) -> None: ...
+    def saveAsTable(
+        self,
+        name: str,
+        format: Optional[str] = ...,
+        mode: Optional[str] = ...,
+        partitionBy: Optional[Union[str, List[str]]] = ...,
+        **options: OptionalPrimitiveType
+    ) -> None: ...
+    def json(
+        self,
+        path: str,
+        mode: Optional[str] = ...,
+        compression: Optional[str] = ...,
+        dateFormat: Optional[str] = ...,
+        timestampFormat: Optional[str] = ...,
+        lineSep: Optional[str] = ...,
+        encoding: Optional[str] = ...,
+        ignoreNullFields: Optional[Union[bool, str]] = ...,
+    ) -> None: ...
+    def parquet(
+        self,
+        path: str,
+        mode: Optional[str] = ...,
+        partitionBy: Optional[Union[str, List[str]]] = ...,
+        compression: Optional[str] = ...,
+    ) -> None: ...
+    def text(
+        self, path: str, compression: Optional[str] = ..., lineSep: Optional[str] = ...
+    ) -> None: ...
+    def csv(
+        self,
+        path: str,
+        mode: Optional[str] = ...,
+        compression: Optional[str] = ...,
+        sep: Optional[str] = ...,
+        quote: Optional[str] = ...,
+        escape: Optional[str] = ...,
+        header: Optional[Union[bool, str]] = ...,
+        nullValue: Optional[str] = ...,
+        escapeQuotes: Optional[Union[bool, str]] = ...,
+        quoteAll: Optional[Union[bool, str]] = ...,
+        dateFormat: Optional[str] = ...,
+        timestampFormat: Optional[str] = ...,
+        ignoreLeadingWhiteSpace: Optional[Union[bool, str]] = ...,
+        ignoreTrailingWhiteSpace: Optional[Union[bool, str]] = ...,
+        charToEscapeQuoteEscaping: Optional[str] = ...,
+        encoding: Optional[str] = ...,
+        emptyValue: Optional[str] = ...,
+        lineSep: Optional[str] = ...,
+    ) -> None: ...
+    def orc(
+        self,
+        path: str,
+        mode: Optional[str] = ...,
+        partitionBy: Optional[Union[str, List[str]]] = ...,
+        compression: Optional[str] = ...,
+    ) -> None: ...
+    def jdbc(
+        self,
+        url: str,
+        table: str,
+        mode: Optional[str] = ...,
+        properties: Optional[Dict[str, str]] = ...,
+    ) -> None: ...
+
+class DataFrameWriterV2:
+    def __init__(self, df: DataFrame, table: str) -> None: ...
+    def using(self, provider: str) -> DataFrameWriterV2: ...
+    def option(self, key: str, value: OptionalPrimitiveType) -> DataFrameWriterV2: ...
+    def options(self, **options: OptionalPrimitiveType) -> DataFrameWriterV2: ...
+    def tableProperty(self, property: str, value: str) -> DataFrameWriterV2: ...
+    def partitionedBy(self, col: Column, *cols: Column) -> DataFrameWriterV2: ...
+    def create(self) -> None: ...
+    def replace(self) -> None: ...
+    def createOrReplace(self) -> None: ...
+    def append(self) -> None: ...
+    def overwrite(self, condition: Column) -> None: ...
+    def overwritePartitions(self) -> None: ...
diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py
index be4fa20a04327..740ceb31f7d16 100644
--- a/python/pyspark/sql/session.py
+++ b/python/pyspark/sql/session.py
@@ -15,28 +15,19 @@
 # limitations under the License.
 #
 
-# To disallow implicit relative import. Remove this once we drop Python 2.
-from __future__ import absolute_import
-from __future__ import print_function
 import sys
 import warnings
 from functools import reduce
 from threading import RLock
 
-if sys.version >= '3':
-    basestring = unicode = str
-    xrange = range
-else:
-    from itertools import imap as map
-
 from pyspark import since
-from pyspark.rdd import RDD, ignore_unicode_prefix
+from pyspark.rdd import RDD
 from pyspark.sql.conf import RuntimeConfig
 from pyspark.sql.dataframe import DataFrame
 from pyspark.sql.pandas.conversion import SparkConversionMixin
 from pyspark.sql.readwriter import DataFrameReader
 from pyspark.sql.streaming import DataStreamReader
-from pyspark.sql.types import Row, DataType, StringType, StructType, \
+from pyspark.sql.types import DataType, StructType, \
     _make_type_verifier, _infer_schema, _has_nulltype, _merge_type, _create_converter, \
     _parse_datatype_string
 from pyspark.sql.utils import install_exception_handler
@@ -51,12 +42,26 @@ def toDF(self, schema=None, sampleRatio=None):
 
         This is a shorthand for ``spark.createDataFrame(rdd, schema, sampleRatio)``
 
-        :param schema: a :class:`pyspark.sql.types.StructType` or list of names of columns
-        :param samplingRatio: the sample ratio of rows used for inferring
-        :return: a DataFrame
+        Parameters
+        ----------
+        schema : :class:`pyspark.sql.types.DataType`, str or list, optional
+            a :class:`pyspark.sql.types.DataType` or a datatype string or a list of
+            column names, default is None.  The data type string format equals to
+            :class:`pyspark.sql.types.DataType.simpleString`, except that top level struct type can
+            omit the ``struct<>`` and atomic types use ``typeName()`` as their format, e.g. use
+            ``byte`` instead of ``tinyint`` for :class:`pyspark.sql.types.ByteType`.
+            We can also use ``int`` as a short name for :class:`pyspark.sql.types.IntegerType`.
+        sampleRatio : float, optional
+            the sample ratio of rows used for inferring
+
+        Returns
+        -------
+        :class:`DataFrame`
 
+        Examples
+        --------
         >>> rdd.toDF().collect()
-        [Row(name=u'Alice', age=1)]
+        [Row(name='Alice', age=1)]
         """
         return sparkSession.createDataFrame(self, schema, sampleRatio)
 
@@ -70,14 +75,31 @@ class SparkSession(SparkConversionMixin):
     tables, execute SQL over tables, cache tables, and read parquet files.
     To create a SparkSession, use the following builder pattern:
 
+    .. autoattribute:: builder
+       :annotation:
+
+    Examples
+    --------
     >>> spark = SparkSession.builder \\
     ...     .master("local") \\
     ...     .appName("Word Count") \\
     ...     .config("spark.some.config.option", "some-value") \\
     ...     .getOrCreate()
 
-    .. autoattribute:: builder
-       :annotation:
+    >>> from datetime import datetime
+    >>> from pyspark.sql import Row
+    >>> spark = SparkSession(sc)
+    >>> allTypes = sc.parallelize([Row(i=1, s="string", d=1.0, l=1,
+    ...     b=True, list=[1, 2, 3], dict={"s": 0}, row=Row(a=1),
+    ...     time=datetime(2014, 8, 1, 14, 1, 5))])
+    >>> df = allTypes.toDF()
+    >>> df.createOrReplaceTempView("allTypes")
+    >>> spark.sql('select i+1, d+1, not b, list[1], dict["s"], time, row.a '
+    ...            'from allTypes where b and i > 0').collect()
+    [Row((i + 1)=2, (d + 1)=2.0, (NOT b)=False, list[1]=2, \
+        dict[s]=0, time=datetime.datetime(2014, 8, 1, 14, 1, 5), a=1)]
+    >>> df.rdd.map(lambda x: (x.i, x.s, x.d, x.l, x.b, x.time, x.row.a, x.list)).collect()
+    [(1, 'string', 1.0, 1, True, datetime.datetime(2014, 8, 1, 14, 1, 5), 1, [1, 2, 3])]
     """
 
     class Builder(object):
@@ -88,11 +110,23 @@ class Builder(object):
         _options = {}
         _sc = None
 
-        @since(2.0)
         def config(self, key=None, value=None, conf=None):
             """Sets a config option. Options set using this method are automatically propagated to
             both :class:`SparkConf` and :class:`SparkSession`'s own configuration.
 
+            .. versionadded:: 2.0.0
+
+            Parameters
+            ----------
+            key : str, optional
+                a key name string for configuration property
+            value : str, optional
+                a value for configuration property
+            conf : :class:`SparkConf`, optional
+                an instance of :class:`SparkConf`
+
+            Examples
+            --------
             For an existing SparkConf, use `conf` parameter.
 
             >>> from pyspark.conf import SparkConf
@@ -104,9 +138,6 @@ def config(self, key=None, value=None, conf=None):
             >>> SparkSession.builder.config("spark.some.config.option", "some-value")
             <pyspark.sql.session...
 
-            :param key: a key name string for configuration property
-            :param value: a value for configuration property
-            :param conf: an instance of :class:`SparkConf`
             """
             with self._lock:
                 if conf is None:
@@ -116,23 +147,31 @@ def config(self, key=None, value=None, conf=None):
                         self._options[k] = v
                 return self
 
-        @since(2.0)
         def master(self, master):
             """Sets the Spark master URL to connect to, such as "local" to run locally, "local[4]"
             to run locally with 4 cores, or "spark://master:7077" to run on a Spark standalone
             cluster.
 
-            :param master: a url for spark master
+            .. versionadded:: 2.0.0
+
+            Parameters
+            ----------
+            master : str
+                a url for spark master
             """
             return self.config("spark.master", master)
 
-        @since(2.0)
         def appName(self, name):
             """Sets a name for the application, which will be shown in the Spark web UI.
 
             If no application name is set, a randomly generated name will be used.
 
-            :param name: an application name
+            .. versionadded:: 2.0.0
+
+            Parameters
+            ----------
+            name : str
+                an application name
             """
             return self.config("spark.app.name", name)
 
@@ -148,11 +187,14 @@ def _sparkContext(self, sc):
                 self._sc = sc
                 return self
 
-        @since(2.0)
         def getOrCreate(self):
             """Gets an existing :class:`SparkSession` or, if there is no existing one, creates a
             new one based on the options set in this builder.
 
+            .. versionadded:: 2.0.0
+
+            Examples
+            --------
             This method first checks whether there is a valid global default SparkSession, and if
             yes, return that one. If no valid global default SparkSession exists, the method
             creates a new SparkSession and assigns the newly created SparkSession as the global
@@ -197,24 +239,7 @@ def getOrCreate(self):
     _instantiatedSession = None
     _activeSession = None
 
-    @ignore_unicode_prefix
     def __init__(self, sparkContext, jsparkSession=None):
-        """Creates a new SparkSession.
-
-        >>> from datetime import datetime
-        >>> spark = SparkSession(sc)
-        >>> allTypes = sc.parallelize([Row(i=1, s="string", d=1.0, l=1,
-        ...     b=True, list=[1, 2, 3], dict={"s": 0}, row=Row(a=1),
-        ...     time=datetime(2014, 8, 1, 14, 1, 5))])
-        >>> df = allTypes.toDF()
-        >>> df.createOrReplaceTempView("allTypes")
-        >>> spark.sql('select i+1, d+1, not b, list[1], dict["s"], time, row.a '
-        ...            'from allTypes where b and i > 0').collect()
-        [Row((i + CAST(1 AS BIGINT))=2, (d + CAST(1 AS DOUBLE))=2.0, (NOT b)=False, list[1]=2, \
-            dict[s]=0, time=datetime.datetime(2014, 8, 1, 14, 1, 5), a=1)]
-        >>> df.rdd.map(lambda x: (x.i, x.s, x.d, x.l, x.b, x.time, x.row.a, x.list)).collect()
-        [(1, u'string', 1.0, 1, True, datetime.datetime(2014, 8, 1, 14, 1, 5), 1, [1, 2, 3])]
-        """
         from pyspark.sql.context import SQLContext
         self._sc = sparkContext
         self._jsc = self._sc._jsc
@@ -262,10 +287,19 @@ def newSession(self):
         return self.__class__(self._sc, self._jsparkSession.newSession())
 
     @classmethod
-    @since(3.0)
     def getActiveSession(cls):
         """
-        Returns the active SparkSession for the current thread, returned by the builder.
+        Returns the active SparkSession for the current thread, returned by the builder
+
+        .. versionadded:: 3.0.0
+
+        Returns
+        -------
+        :class:`SparkSession`
+            Spark session if an active session exists for the current thread
+
+        Examples
+        --------
         >>> s = SparkSession.getActiveSession()
         >>> l = [('Alice', 1)]
         >>> rdd = s.sparkContext.parallelize(l)
@@ -304,18 +338,25 @@ def conf(self):
         This is the interface through which the user can get and set all Spark and Hadoop
         configurations that are relevant to Spark SQL. When getting the value of a config,
         this defaults to the value set in the underlying :class:`SparkContext`, if any.
+
+        Returns
+        -------
+        :class:`pyspark.sql.conf.RuntimeConfig`
         """
         if not hasattr(self, "_conf"):
             self._conf = RuntimeConfig(self._jsparkSession.conf())
         return self._conf
 
     @property
-    @since(2.0)
     def catalog(self):
         """Interface through which the user may create, drop, alter or query underlying
         databases, tables, functions, etc.
 
-        :return: :class:`Catalog`
+        .. versionadded:: 2.0.0
+
+        Returns
+        -------
+        :class:`Catalog`
         """
         from pyspark.sql.catalog import Catalog
         if not hasattr(self, "_catalog"):
@@ -323,28 +364,43 @@ def catalog(self):
         return self._catalog
 
     @property
-    @since(2.0)
     def udf(self):
         """Returns a :class:`UDFRegistration` for UDF registration.
 
-        :return: :class:`UDFRegistration`
+        .. versionadded:: 2.0.0
+
+        Returns
+        -------
+        :class:`UDFRegistration`
         """
         from pyspark.sql.udf import UDFRegistration
         return UDFRegistration(self)
 
-    @since(2.0)
     def range(self, start, end=None, step=1, numPartitions=None):
         """
         Create a :class:`DataFrame` with single :class:`pyspark.sql.types.LongType` column named
         ``id``, containing elements in a range from ``start`` to ``end`` (exclusive) with
         step value ``step``.
 
-        :param start: the start value
-        :param end: the end value (exclusive)
-        :param step: the incremental step (default: 1)
-        :param numPartitions: the number of partitions of the DataFrame
-        :return: :class:`DataFrame`
-
+        .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        start : int
+            the start value
+        end : int, optional
+            the end value (exclusive)
+        step : int, optional
+            the incremental step (default: 1)
+        numPartitions : int, optional
+            the number of partitions of the DataFrame
+
+        Returns
+        -------
+        :class:`DataFrame`
+
+        Examples
+        --------
         >>> spark.range(1, 7, 2).collect()
         [Row(id=1), Row(id=3), Row(id=5)]
 
@@ -365,18 +421,21 @@ def range(self, start, end=None, step=1, numPartitions=None):
 
     def _inferSchemaFromList(self, data, names=None):
         """
-        Infer schema from list of Row or tuple.
+        Infer schema from list of Row, dict, or tuple.
+
+        Parameters
+        ----------
+        data : iterable
+            list of Row, dict, or tuple
+        names : list, optional
+            list of column names
 
-        :param data: list of Row or tuple
-        :param names: list of column names
-        :return: :class:`pyspark.sql.types.StructType`
+        Returns
+        -------
+        :class:`pyspark.sql.types.StructType`
         """
         if not data:
             raise ValueError("can not infer schema from empty dataset")
-        first = data[0]
-        if type(first) is dict:
-            warnings.warn("inferring schema from dict is deprecated,"
-                          "please use pyspark.sql.Row instead")
         schema = reduce(_merge_type, (_infer_schema(row, names) for row in data))
         if _has_nulltype(schema):
             raise ValueError("Some of types cannot be determined after inferring")
@@ -384,19 +443,24 @@ def _inferSchemaFromList(self, data, names=None):
 
     def _inferSchema(self, rdd, samplingRatio=None, names=None):
         """
-        Infer schema from an RDD of Row or tuple.
+        Infer schema from an RDD of Row, dict, or tuple.
 
-        :param rdd: an RDD of Row or tuple
-        :param samplingRatio: sampling ratio, or no sampling (default)
-        :return: :class:`pyspark.sql.types.StructType`
+        Parameters
+        ----------
+        rdd : :class:`RDD`
+            an RDD of Row, dict, or tuple
+        samplingRatio : float, optional
+            sampling ratio, or no sampling (default)
+        names : list, optional
+
+        Returns
+        -------
+        :class:`pyspark.sql.types.StructType`
         """
         first = rdd.first()
         if not first:
             raise ValueError("The first row in RDD is empty, "
                              "can not infer schema")
-        if type(first) is dict:
-            warnings.warn("Using RDD of dict to inferSchema is deprecated. "
-                          "Use pyspark.sql.Row instead")
 
         if samplingRatio is None:
             schema = _infer_schema(first, names=names)
@@ -488,8 +552,6 @@ def _create_shell_session():
 
         return SparkSession.builder.getOrCreate()
 
-    @since(2.0)
-    @ignore_unicode_prefix
     def createDataFrame(self, data, schema=None, samplingRatio=None, verifySchema=True):
         """
         Creates a :class:`DataFrame` from an :class:`RDD`, a list or a :class:`pandas.DataFrame`.
@@ -510,51 +572,62 @@ def createDataFrame(self, data, schema=None, samplingRatio=None, verifySchema=Tr
         If schema inference is needed, ``samplingRatio`` is used to determined the ratio of
         rows used for schema inference. The first row will be used if ``samplingRatio`` is ``None``.
 
-        :param data: an RDD of any kind of SQL data representation (e.g. row, tuple, int, boolean,
-            etc.), :class:`list`, or :class:`pandas.DataFrame`.
-        :param schema: a :class:`pyspark.sql.types.DataType` or a datatype string or a list of
-            column names, default is ``None``.  The data type string format equals to
-            :class:`pyspark.sql.types.DataType.simpleString`, except that top level struct type can
-            omit the ``struct<>`` and atomic types use ``typeName()`` as their format, e.g. use
-            ``byte`` instead of ``tinyint`` for :class:`pyspark.sql.types.ByteType`. We can also use
-            ``int`` as a short name for ``IntegerType``.
-        :param samplingRatio: the sample ratio of rows used for inferring
-        :param verifySchema: verify data types of every row against schema.
-        :return: :class:`DataFrame`
+        .. versionadded:: 2.0.0
 
-        .. versionchanged:: 2.1
+        .. versionchanged:: 2.1.0
            Added verifySchema.
 
-        .. note:: Usage with spark.sql.execution.arrow.pyspark.enabled=True is experimental.
-
-        .. note:: When Arrow optimization is enabled, strings inside Pandas DataFrame in Python
-            2 are converted into bytes as they are bytes in Python 2 whereas regular strings are
-            left as strings. When using strings in Python 2, use unicode `u""` as Python standard
-            practice.
-
+        Parameters
+        ----------
+        data : :class:`RDD` or iterable
+            an RDD of any kind of SQL data representation (:class:`Row`,
+            :class:`tuple`, ``int``, ``boolean``, etc.), or :class:`list`, or
+            :class:`pandas.DataFrame`.
+        schema : :class:`pyspark.sql.types.DataType`, str or list, optional
+            a :class:`pyspark.sql.types.DataType` or a datatype string or a list of
+            column names, default is None.  The data type string format equals to
+            :class:`pyspark.sql.types.DataType.simpleString`, except that top level struct type can
+            omit the ``struct<>`` and atomic types use ``typeName()`` as their format, e.g. use
+            ``byte`` instead of ``tinyint`` for :class:`pyspark.sql.types.ByteType`.
+            We can also use ``int`` as a short name for :class:`pyspark.sql.types.IntegerType`.
+        samplingRatio : float, optional
+            the sample ratio of rows used for inferring
+        verifySchema : bool, optional
+            verify data types of every row against schema. Enabled by default.
+
+        Returns
+        -------
+        :class:`DataFrame`
+
+        Notes
+        -----
+        Usage with spark.sql.execution.arrow.pyspark.enabled=True is experimental.
+
+        Examples
+        --------
         >>> l = [('Alice', 1)]
         >>> spark.createDataFrame(l).collect()
-        [Row(_1=u'Alice', _2=1)]
+        [Row(_1='Alice', _2=1)]
         >>> spark.createDataFrame(l, ['name', 'age']).collect()
-        [Row(name=u'Alice', age=1)]
+        [Row(name='Alice', age=1)]
 
         >>> d = [{'name': 'Alice', 'age': 1}]
         >>> spark.createDataFrame(d).collect()
-        [Row(age=1, name=u'Alice')]
+        [Row(age=1, name='Alice')]
 
         >>> rdd = sc.parallelize(l)
         >>> spark.createDataFrame(rdd).collect()
-        [Row(_1=u'Alice', _2=1)]
+        [Row(_1='Alice', _2=1)]
         >>> df = spark.createDataFrame(rdd, ['name', 'age'])
         >>> df.collect()
-        [Row(name=u'Alice', age=1)]
+        [Row(name='Alice', age=1)]
 
         >>> from pyspark.sql import Row
         >>> Person = Row('name', 'age')
         >>> person = rdd.map(lambda r: Person(*r))
         >>> df2 = spark.createDataFrame(person)
         >>> df2.collect()
-        [Row(name=u'Alice', age=1)]
+        [Row(name='Alice', age=1)]
 
         >>> from pyspark.sql.types import *
         >>> schema = StructType([
@@ -562,15 +635,15 @@ def createDataFrame(self, data, schema=None, samplingRatio=None, verifySchema=Tr
         ...    StructField("age", IntegerType(), True)])
         >>> df3 = spark.createDataFrame(rdd, schema)
         >>> df3.collect()
-        [Row(name=u'Alice', age=1)]
+        [Row(name='Alice', age=1)]
 
         >>> spark.createDataFrame(df.toPandas()).collect()  # doctest: +SKIP
-        [Row(name=u'Alice', age=1)]
+        [Row(name='Alice', age=1)]
         >>> spark.createDataFrame(pandas.DataFrame([[1, 2]])).collect()  # doctest: +SKIP
         [Row(0=1, 1=2)]
 
         >>> spark.createDataFrame(rdd, "a: string, b: int").collect()
-        [Row(a=u'Alice', b=1)]
+        [Row(a='Alice', b=1)]
         >>> rdd = rdd.map(lambda row: row[1])
         >>> spark.createDataFrame(rdd, "int").collect()
         [Row(value=1)]
@@ -584,7 +657,7 @@ def createDataFrame(self, data, schema=None, samplingRatio=None, verifySchema=Tr
         if isinstance(data, DataFrame):
             raise TypeError("data is already a DataFrame")
 
-        if isinstance(schema, basestring):
+        if isinstance(schema, str):
             schema = _parse_datatype_string(schema)
         elif isinstance(schema, (list, tuple)):
             # Must re-encode any unicode strings to be consistent with StructField names
@@ -631,26 +704,35 @@ def prepare(obj):
         df._schema = schema
         return df
 
-    @ignore_unicode_prefix
-    @since(2.0)
     def sql(self, sqlQuery):
         """Returns a :class:`DataFrame` representing the result of the given query.
 
-        :return: :class:`DataFrame`
+        .. versionadded:: 2.0.0
 
+        Returns
+        -------
+        :class:`DataFrame`
+
+        Examples
+        --------
         >>> df.createOrReplaceTempView("table1")
         >>> df2 = spark.sql("SELECT field1 AS f1, field2 as f2 from table1")
         >>> df2.collect()
-        [Row(f1=1, f2=u'row1'), Row(f1=2, f2=u'row2'), Row(f1=3, f2=u'row3')]
+        [Row(f1=1, f2='row1'), Row(f1=2, f2='row2'), Row(f1=3, f2='row3')]
         """
         return DataFrame(self._jsparkSession.sql(sqlQuery), self._wrapped)
 
-    @since(2.0)
     def table(self, tableName):
         """Returns the specified table as a :class:`DataFrame`.
 
-        :return: :class:`DataFrame`
+        .. versionadded:: 2.0.0
+
+        Returns
+        -------
+        :class:`DataFrame`
 
+        Examples
+        --------
         >>> df.createOrReplaceTempView("table1")
         >>> df2 = spark.table("table1")
         >>> sorted(df.collect()) == sorted(df2.collect())
@@ -659,38 +741,51 @@ def table(self, tableName):
         return DataFrame(self._jsparkSession.table(tableName), self._wrapped)
 
     @property
-    @since(2.0)
     def read(self):
         """
         Returns a :class:`DataFrameReader` that can be used to read data
         in as a :class:`DataFrame`.
 
-        :return: :class:`DataFrameReader`
+        .. versionadded:: 2.0.0
+
+        Returns
+        -------
+        :class:`DataFrameReader`
         """
         return DataFrameReader(self._wrapped)
 
     @property
-    @since(2.0)
     def readStream(self):
         """
         Returns a :class:`DataStreamReader` that can be used to read data streams
         as a streaming :class:`DataFrame`.
 
-        .. note:: Evolving.
+        .. versionadded:: 2.0.0
 
-        :return: :class:`DataStreamReader`
+        Notes
+        -----
+        This API is evolving.
+
+        Returns
+        -------
+        :class:`DataStreamReader`
         """
         return DataStreamReader(self._wrapped)
 
     @property
-    @since(2.0)
     def streams(self):
         """Returns a :class:`StreamingQueryManager` that allows managing all the
         :class:`StreamingQuery` instances active on `this` context.
 
-        .. note:: Evolving.
+        .. versionadded:: 2.0.0
+
+        Notes
+        -----
+        This API is evolving.
 
-        :return: :class:`StreamingQueryManager`
+        Returns
+        -------
+        :class:`StreamingQueryManager`
         """
         from pyspark.sql.streaming import StreamingQueryManager
         return StreamingQueryManager(self._jsparkSession.streams())
diff --git a/python/pyspark/sql/session.pyi b/python/pyspark/sql/session.pyi
new file mode 100644
index 0000000000000..6cd2d3bed2b2f
--- /dev/null
+++ b/python/pyspark/sql/session.pyi
@@ -0,0 +1,131 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import overload
+from typing import Any, Iterable, List, Optional, Tuple, Type, TypeVar, Union
+from types import TracebackType
+
+from py4j.java_gateway import JavaObject  # type: ignore[import]
+
+from pyspark.sql._typing import DateTimeLiteral, LiteralType, DecimalLiteral, RowLike
+from pyspark.sql.pandas._typing import DataFrameLike
+from pyspark.conf import SparkConf
+from pyspark.context import SparkContext
+from pyspark.rdd import RDD
+from pyspark.sql.catalog import Catalog
+from pyspark.sql.conf import RuntimeConfig
+from pyspark.sql.dataframe import DataFrame
+from pyspark.sql.pandas.conversion import SparkConversionMixin
+from pyspark.sql.types import AtomicType, StructType
+from pyspark.sql.readwriter import DataFrameReader
+from pyspark.sql.streaming import DataStreamReader, StreamingQueryManager
+from pyspark.sql.udf import UDFRegistration
+
+T = TypeVar("T")
+
+class SparkSession(SparkConversionMixin):
+    class Builder:
+        @overload
+        def config(self, *, conf: SparkConf) -> SparkSession.Builder: ...
+        @overload
+        def config(self, key: str, value: Any) -> SparkSession.Builder: ...
+        def master(self, master: str) -> SparkSession.Builder: ...
+        def appName(self, name: str) -> SparkSession.Builder: ...
+        def enableHiveSupport(self) -> SparkSession.Builder: ...
+        def getOrCreate(self) -> SparkSession: ...
+    builder: SparkSession.Builder
+    def __init__(
+        self, sparkContext: SparkContext, jsparkSession: Optional[JavaObject] = ...
+    ) -> None: ...
+    def newSession(self) -> SparkSession: ...
+    @classmethod
+    def getActiveSession(cls) -> SparkSession: ...
+    @property
+    def sparkContext(self) -> SparkContext: ...
+    @property
+    def version(self) -> str: ...
+    @property
+    def conf(self) -> RuntimeConfig: ...
+    @property
+    def catalog(self) -> Catalog: ...
+    @property
+    def udf(self) -> UDFRegistration: ...
+    def range(
+        self,
+        start: int,
+        end: Optional[int] = ...,
+        step: int = ...,
+        numPartitions: Optional[int] = ...,
+    ) -> DataFrame: ...
+    @overload
+    def createDataFrame(
+        self,
+        data: Union[RDD[RowLike], Iterable[RowLike]],
+        samplingRatio: Optional[float] = ...,
+    ) -> DataFrame: ...
+    @overload
+    def createDataFrame(
+        self,
+        data: Union[RDD[RowLike], Iterable[RowLike]],
+        schema: Union[List[str], Tuple[str, ...]] = ...,
+        verifySchema: bool = ...,
+    ) -> DataFrame: ...
+    @overload
+    def createDataFrame(
+        self,
+        data: Union[
+            RDD[Union[DateTimeLiteral, LiteralType, DecimalLiteral]],
+            Iterable[Union[DateTimeLiteral, LiteralType, DecimalLiteral]],
+        ],
+        schema: Union[AtomicType, str],
+        verifySchema: bool = ...,
+    ) -> DataFrame: ...
+    @overload
+    def createDataFrame(
+        self,
+        data: Union[RDD[RowLike], Iterable[RowLike]],
+        schema: Union[StructType, str],
+        verifySchema: bool = ...,
+    ) -> DataFrame: ...
+    @overload
+    def createDataFrame(
+        self, data: DataFrameLike, samplingRatio: Optional[float] = ...
+    ) -> DataFrame: ...
+    @overload
+    def createDataFrame(
+        self,
+        data: DataFrameLike,
+        schema: Union[StructType, str],
+        verifySchema: bool = ...,
+    ) -> DataFrame: ...
+    def sql(self, sqlQuery: str) -> DataFrame: ...
+    def table(self, tableName: str) -> DataFrame: ...
+    @property
+    def read(self) -> DataFrameReader: ...
+    @property
+    def readStream(self) -> DataStreamReader: ...
+    @property
+    def streams(self) -> StreamingQueryManager: ...
+    def stop(self) -> None: ...
+    def __enter__(self) -> SparkSession: ...
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> None: ...
diff --git a/python/pyspark/sql/streaming.py b/python/pyspark/sql/streaming.py
index 2450a4c93c460..2c90d7f2dee70 100644
--- a/python/pyspark/sql/streaming.py
+++ b/python/pyspark/sql/streaming.py
@@ -18,16 +18,12 @@
 import sys
 import json
 
-if sys.version >= '3':
-    basestring = str
-
 from py4j.java_gateway import java_import
 
 from pyspark import since, keyword_only
-from pyspark.rdd import ignore_unicode_prefix
 from pyspark.sql.column import _to_seq
 from pyspark.sql.readwriter import OptionUtils, to_str
-from pyspark.sql.types import *
+from pyspark.sql.types import StructType, StructField, StringType
 from pyspark.sql.utils import ForeachBatchFunction, StreamingQueryException
 
 __all__ = ["StreamingQuery", "StreamingQueryManager", "DataStreamReader", "DataStreamWriter"]
@@ -38,9 +34,11 @@ class StreamingQuery(object):
     A handle to a query that is executing continuously in the background as new data arrives.
     All these methods are thread-safe.
 
-    .. note:: Evolving
+    .. versionadded:: 2.0.0
 
-    .. versionadded:: 2.0
+    Notes
+    -----
+    This API is evolving.
     """
 
     def __init__(self, jsq):
@@ -120,13 +118,16 @@ def recentProgress(self):
         return [json.loads(p.json()) for p in self._jsq.recentProgress()]
 
     @property
-    @since(2.1)
     def lastProgress(self):
         """
         Returns the most recent :class:`StreamingQueryProgress` update of this streaming query or
         None if there were no progress updates
 
-        :return: a map
+        .. versionadded:: 2.1.0
+
+        Returns
+        -------
+        dict
         """
         lastProgress = self._jsq.lastProgress()
         if lastProgress:
@@ -134,15 +135,18 @@ def lastProgress(self):
         else:
             return None
 
-    @since(2.0)
     def processAllAvailable(self):
         """Blocks until all available data in the source has been processed and committed to the
         sink. This method is intended for testing.
 
-        .. note:: In the case of continually arriving data, this method may block forever.
-            Additionally, this method is only guaranteed to block until data that has been
-            synchronously appended data to a stream source prior to invocation.
-            (i.e. `getOffset` must immediately reflect the addition).
+        .. versionadded:: 2.0.0
+
+        Notes
+        -----
+        In the case of continually arriving data, this method may block forever.
+        Additionally, this method is only guaranteed to block until data that has been
+        synchronously appended data to a stream source prior to invocation.
+        (i.e. `getOffset` must immediately reflect the addition).
         """
         return self._jsq.processAllAvailable()
 
@@ -152,12 +156,18 @@ def stop(self):
         """
         self._jsq.stop()
 
-    @since(2.1)
     def explain(self, extended=False):
         """Prints the (logical and physical) plans to the console for debugging purpose.
 
-        :param extended: boolean, default ``False``. If ``False``, prints only the physical plan.
+        .. versionadded:: 2.1.0
 
+        Parameters
+        ----------
+        extended : bool, optional
+            default ``False``. If ``False``, prints only the physical plan.
+
+        Examples
+        --------
         >>> sq = sdf.writeStream.format('memory').queryName('query_explain').start()
         >>> sq.processAllAvailable() # Wait a bit to generate the runtime plans.
         >>> sq.explain()
@@ -178,10 +188,14 @@ def explain(self, extended=False):
         # We should print it in the Python process.
         print(self._jsq.explainInternal(extended))
 
-    @since(2.1)
     def exception(self):
         """
-        :return: the StreamingQueryException if the query was terminated by an exception, or None.
+        .. versionadded:: 2.1.0
+
+        Returns
+        -------
+        :class:`StreamingQueryException`
+            the StreamingQueryException if the query was terminated by an exception, or None.
         """
         if self._jsq.exception().isDefined():
             je = self._jsq.exception().get()
@@ -195,38 +209,44 @@ def exception(self):
 class StreamingQueryManager(object):
     """A class to manage all the :class:`StreamingQuery` StreamingQueries active.
 
-    .. note:: Evolving
+    .. versionadded:: 2.0.0
 
-    .. versionadded:: 2.0
+    Notes
+    -----
+    This API is evolving.
     """
 
     def __init__(self, jsqm):
         self._jsqm = jsqm
 
     @property
-    @ignore_unicode_prefix
-    @since(2.0)
     def active(self):
         """Returns a list of active queries associated with this SQLContext
 
+        .. versionadded:: 2.0.0
+
+        Examples
+        --------
         >>> sq = sdf.writeStream.format('memory').queryName('this_query').start()
         >>> sqm = spark.streams
         >>> # get the list of active streaming queries
         >>> [q.name for q in sqm.active]
-        [u'this_query']
+        ['this_query']
         >>> sq.stop()
         """
         return [StreamingQuery(jsq) for jsq in self._jsqm.active()]
 
-    @ignore_unicode_prefix
-    @since(2.0)
     def get(self, id):
         """Returns an active query from this SQLContext or throws exception if an active query
         with this name doesn't exist.
 
+        .. versionadded:: 2.0.0
+
+        Examples
+        --------
         >>> sq = sdf.writeStream.format('memory').queryName('this_query').start()
         >>> sq.name
-        u'this_query'
+        'this_query'
         >>> sq = spark.streams.get(sq.id)
         >>> sq.isActive
         True
@@ -265,11 +285,14 @@ def awaitAnyTermination(self, timeout=None):
         else:
             return self._jsqm.awaitAnyTermination()
 
-    @since(2.0)
     def resetTerminated(self):
         """Forget about past terminated queries so that :func:`awaitAnyTermination()` can be used
         again to wait for new terminations.
 
+        .. versionadded:: 2.0.0
+
+        Examples
+        --------
         >>> spark.streams.resetTerminated()
         """
         self._jsqm.resetTerminated()
@@ -281,9 +304,11 @@ class DataStreamReader(OptionUtils):
     storage systems (e.g. file systems, key-value stores, etc).
     Use :attr:`SparkSession.readStream <pyspark.sql.SparkSession.readStream>` to access this.
 
-    .. note:: Evolving.
+    .. versionadded:: 2.0.0
 
-    .. versionadded:: 2.0
+    Notes
+    -----
+    This API is evolving.
     """
 
     def __init__(self, spark):
@@ -294,20 +319,27 @@ def _df(self, jdf):
         from pyspark.sql.dataframe import DataFrame
         return DataFrame(jdf, self._spark)
 
-    @since(2.0)
     def format(self, source):
         """Specifies the input data source format.
 
-        .. note:: Evolving.
+        .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        source : str
+            name of the data source, e.g. 'json', 'parquet'.
 
-        :param source: string, name of the data source, e.g. 'json', 'parquet'.
+        Notes
+        -----
+        This API is evolving.
 
+        Examples
+        --------
         >>> s = spark.readStream.format("text")
         """
         self._jreader = self._jreader.format(source)
         return self
 
-    @since(2.0)
     def schema(self, schema):
         """Specifies the input schema.
 
@@ -315,11 +347,20 @@ def schema(self, schema):
         By specifying the schema here, the underlying data source can skip the schema
         inference step, and thus speed up data loading.
 
-        .. note:: Evolving.
+        .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        schema : :class:`pyspark.sql.types.StructType` or str
+            a :class:`pyspark.sql.types.StructType` object or a DDL-formatted string
+            (For example ``col0 INT, col1 DOUBLE``).
 
-        :param schema: a :class:`pyspark.sql.types.StructType` object or a DDL-formatted string
-                       (For example ``col0 INT, col1 DOUBLE``).
+        Notes
+        -----
+        This API is evolving.
 
+        Examples
+        --------
         >>> s = spark.readStream.schema(sdf_schema)
         >>> s = spark.readStream.schema("col0 INT, col1 DOUBLE")
         """
@@ -328,13 +369,12 @@ def schema(self, schema):
         if isinstance(schema, StructType):
             jschema = spark._jsparkSession.parseDataType(schema.json())
             self._jreader = self._jreader.schema(jschema)
-        elif isinstance(schema, basestring):
+        elif isinstance(schema, str):
             self._jreader = self._jreader.schema(schema)
         else:
             raise TypeError("schema should be StructType or string")
         return self
 
-    @since(2.0)
     def option(self, key, value):
         """Adds an input option for the underlying data source.
 
@@ -352,14 +392,19 @@ def option(self, key, value):
                 ambiguous. If it isn't set, the current value of the SQL config
                 ``spark.sql.session.timeZone`` is used by default.
 
-        .. note:: Evolving.
+        .. versionadded:: 2.0.0
 
+        Notes
+        -----
+        This API is evolving.
+
+        Examples
+        --------
         >>> s = spark.readStream.option("x", 1)
         """
         self._jreader = self._jreader.option(key, to_str(value))
         return self
 
-    @since(2.0)
     def options(self, **options):
         """Adds input options for the underlying data source.
 
@@ -377,27 +422,44 @@ def options(self, **options):
                 ambiguous. If it isn't set, the current value of the SQL config
                 ``spark.sql.session.timeZone`` is used by default.
 
-        .. note:: Evolving.
+        .. versionadded:: 2.0.0
 
+        Notes
+        -----
+        This API is evolving.
+
+        Examples
+        --------
         >>> s = spark.readStream.options(x="1", y=2)
         """
         for k in options:
             self._jreader = self._jreader.option(k, to_str(options[k]))
         return self
 
-    @since(2.0)
     def load(self, path=None, format=None, schema=None, **options):
         """Loads a data stream from a data source and returns it as a
         :class:`DataFrame <pyspark.sql.DataFrame>`.
 
-        .. note:: Evolving.
-
-        :param path: optional string for file-system backed data sources.
-        :param format: optional string for format of the data source. Default to 'parquet'.
-        :param schema: optional :class:`pyspark.sql.types.StructType` for the input schema
-                       or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``).
-        :param options: all other string options
-
+        .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        path : str, optional
+            optional string for file-system backed data sources.
+        format : str, optional
+            optional string for format of the data source. Default to 'parquet'.
+        schema : :class:`pyspark.sql.types.StructType` or str, optional
+            optional :class:`pyspark.sql.types.StructType` for the input schema
+            or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``).
+        **options : dict
+            all other string options
+
+        Notes
+        -----
+        This API is evolving.
+
+        Examples
+        --------
         >>> json_sdf = spark.readStream.format("json") \\
         ...     .schema(sdf_schema) \\
         ...     .load(tempfile.mkdtemp())
@@ -419,14 +481,13 @@ def load(self, path=None, format=None, schema=None, **options):
         else:
             return self._df(self._jreader.load())
 
-    @since(2.0)
     def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
              allowComments=None, allowUnquotedFieldNames=None, allowSingleQuotes=None,
              allowNumericLeadingZero=None, allowBackslashEscapingAnyCharacter=None,
              mode=None, columnNameOfCorruptRecord=None, dateFormat=None, timestampFormat=None,
              multiLine=None,  allowUnquotedControlChars=None, lineSep=None, locale=None,
              dropFieldIfAllNull=None, encoding=None, pathGlobFilter=None,
-             recursiveFileLookup=None):
+             recursiveFileLookup=None, allowNonNumericNumbers=None):
         """
         Loads a JSON file stream and returns the results as a :class:`DataFrame`.
 
@@ -436,81 +497,119 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
         If the ``schema`` parameter is not specified, this function goes
         through the input once to determine the input schema.
 
-        .. note:: Evolving.
-
-        :param path: string represents path to the JSON dataset,
-                     or RDD of Strings storing JSON objects.
-        :param schema: an optional :class:`pyspark.sql.types.StructType` for the input schema
-                       or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``).
-        :param primitivesAsString: infers all primitive values as a string type. If None is set,
-                                   it uses the default value, ``false``.
-        :param prefersDecimal: infers all floating-point values as a decimal type. If the values
-                               do not fit in decimal, then it infers them as doubles. If None is
-                               set, it uses the default value, ``false``.
-        :param allowComments: ignores Java/C++ style comment in JSON records. If None is set,
-                              it uses the default value, ``false``.
-        :param allowUnquotedFieldNames: allows unquoted JSON field names. If None is set,
-                                        it uses the default value, ``false``.
-        :param allowSingleQuotes: allows single quotes in addition to double quotes. If None is
-                                        set, it uses the default value, ``true``.
-        :param allowNumericLeadingZero: allows leading zeros in numbers (e.g. 00012). If None is
-                                        set, it uses the default value, ``false``.
-        :param allowBackslashEscapingAnyCharacter: allows accepting quoting of all character
-                                                   using backslash quoting mechanism. If None is
-                                                   set, it uses the default value, ``false``.
-        :param mode: allows a mode for dealing with corrupt records during parsing. If None is
-                     set, it uses the default value, ``PERMISSIVE``.
-
-                * ``PERMISSIVE``: when it meets a corrupted record, puts the malformed string \
-                  into a field configured by ``columnNameOfCorruptRecord``, and sets malformed \
-                  fields to ``null``. To keep corrupt records, an user can set a string type \
-                  field named ``columnNameOfCorruptRecord`` in an user-defined schema. If a \
-                  schema does not have the field, it drops corrupt records during parsing. \
-                  When inferring a schema, it implicitly adds a ``columnNameOfCorruptRecord`` \
-                  field in an output schema.
-                *  ``DROPMALFORMED``: ignores the whole corrupted records.
-                *  ``FAILFAST``: throws an exception when it meets corrupted records.
-
-        :param columnNameOfCorruptRecord: allows renaming the new field having malformed string
-                                          created by ``PERMISSIVE`` mode. This overrides
-                                          ``spark.sql.columnNameOfCorruptRecord``. If None is set,
-                                          it uses the value specified in
-                                          ``spark.sql.columnNameOfCorruptRecord``.
-        :param dateFormat: sets the string that indicates a date format. Custom date formats
-                           follow the formats at `datetime pattern`_.
-                           This applies to date type. If None is set, it uses the
-                           default value, ``yyyy-MM-dd``.
-        :param timestampFormat: sets the string that indicates a timestamp format.
-                                Custom date formats follow the formats at `datetime pattern`_.
-                                This applies to timestamp type. If None is set, it uses the
-                                default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``.
-        :param multiLine: parse one record, which may span multiple lines, per file. If None is
-                          set, it uses the default value, ``false``.
-        :param allowUnquotedControlChars: allows JSON Strings to contain unquoted control
-                                          characters (ASCII characters with value less than 32,
-                                          including tab and line feed characters) or not.
-        :param lineSep: defines the line separator that should be used for parsing. If None is
-                        set, it covers all ``\\r``, ``\\r\\n`` and ``\\n``.
-        :param locale: sets a locale as language tag in IETF BCP 47 format. If None is set,
-                       it uses the default value, ``en-US``. For instance, ``locale`` is used while
-                       parsing dates and timestamps.
-        :param dropFieldIfAllNull: whether to ignore column of all null values or empty
-                                   array/struct during schema inference. If None is set, it
-                                   uses the default value, ``false``.
-        :param encoding: allows to forcibly set one of standard basic or extended encoding for
-                         the JSON files. For example UTF-16BE, UTF-32LE. If None is set,
-                         the encoding of input JSON will be detected automatically
-                         when the multiLine option is set to ``true``.
-        :param pathGlobFilter: an optional glob pattern to only include files with paths matching
-                               the pattern. The syntax follows `org.apache.hadoop.fs.GlobFilter`.
-                               It does not change the behavior of `partition discovery`_.
-        :param recursiveFileLookup: recursively scan a directory for files. Using this option
-                                    disables `partition discovery`_.
-
-        .. _partition discovery:
-          https://spark.apache.org/docs/latest/sql-data-sources-parquet.html#partition-discovery
-        .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
-
+        .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        path : str
+            string represents path to the JSON dataset,
+            or RDD of Strings storing JSON objects.
+        schema : :class:`pyspark.sql.types.StructType` or str, optional
+            an optional :class:`pyspark.sql.types.StructType` for the input schema
+            or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``).
+        primitivesAsString : str or bool, optional
+            infers all primitive values as a string type. If None is set,
+            it uses the default value, ``false``.
+        prefersDecimal : str or bool, optional
+            infers all floating-point values as a decimal type. If the values
+            do not fit in decimal, then it infers them as doubles. If None is
+            set, it uses the default value, ``false``.
+        allowComments : str or bool, optional
+            ignores Java/C++ style comment in JSON records. If None is set,
+            it uses the default value, ``false``.
+        allowUnquotedFieldNames : str or bool, optional
+            allows unquoted JSON field names. If None is set,
+            it uses the default value, ``false``.
+        allowSingleQuotes : str or bool, optional
+            allows single quotes in addition to double quotes. If None is
+            set, it uses the default value, ``true``.
+        allowNumericLeadingZero : str or bool, optional
+            allows leading zeros in numbers (e.g. 00012). If None is
+            set, it uses the default value, ``false``.
+        allowBackslashEscapingAnyCharacter : str or bool, optional
+            allows accepting quoting of all character
+            using backslash quoting mechanism. If None is
+            set, it uses the default value, ``false``.
+        mode : str, optional
+            allows a mode for dealing with corrupt records during parsing. If None is
+            set, it uses the default value, ``PERMISSIVE``.
+
+            * ``PERMISSIVE``: when it meets a corrupted record, puts the malformed string \
+              into a field configured by ``columnNameOfCorruptRecord``, and sets malformed \
+              fields to ``null``. To keep corrupt records, an user can set a string type \
+              field named ``columnNameOfCorruptRecord`` in an user-defined schema. If a \
+              schema does not have the field, it drops corrupt records during parsing. \
+              When inferring a schema, it implicitly adds a ``columnNameOfCorruptRecord`` \
+              field in an output schema.
+            *  ``DROPMALFORMED``: ignores the whole corrupted records.
+            *  ``FAILFAST``: throws an exception when it meets corrupted records.
+
+        columnNameOfCorruptRecord : str, optional
+            allows renaming the new field having malformed string
+            created by ``PERMISSIVE`` mode. This overrides
+            ``spark.sql.columnNameOfCorruptRecord``. If None is set,
+            it uses the value specified in
+            ``spark.sql.columnNameOfCorruptRecord``.
+        dateFormat : str, optional
+            sets the string that indicates a date format. Custom date formats
+            follow the formats at
+            `datetime pattern <https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html>`_.  # noqa
+            This applies to date type. If None is set, it uses the
+            default value, ``yyyy-MM-dd``.
+        timestampFormat : str, optional
+            sets the string that indicates a timestamp format.
+            Custom date formats follow the formats at
+            `datetime pattern <https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html>`_.  # noqa
+            This applies to timestamp type. If None is set, it uses the
+            default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``.
+        multiLine : str or bool, optional
+            parse one record, which may span multiple lines, per file. If None is
+            set, it uses the default value, ``false``.
+        allowUnquotedControlChars : str or bool, optional
+            allows JSON Strings to contain unquoted control
+            characters (ASCII characters with value less than 32,
+            including tab and line feed characters) or not.
+        lineSep : str, optional
+            defines the line separator that should be used for parsing. If None is
+            set, it covers all ``\\r``, ``\\r\\n`` and ``\\n``.
+        locale : str, optional
+            sets a locale as language tag in IETF BCP 47 format. If None is set,
+            it uses the default value, ``en-US``. For instance, ``locale`` is used while
+            parsing dates and timestamps.
+        dropFieldIfAllNull : str or bool, optional
+            whether to ignore column of all null values or empty
+            array/struct during schema inference. If None is set, it
+            uses the default value, ``false``.
+        encoding : str or bool, optional
+            allows to forcibly set one of standard basic or extended encoding for
+            the JSON files. For example UTF-16BE, UTF-32LE. If None is set,
+            the encoding of input JSON will be detected automatically
+            when the multiLine option is set to ``true``.
+        pathGlobFilter : str or bool, optional
+            an optional glob pattern to only include files with paths matching
+            the pattern. The syntax follows `org.apache.hadoop.fs.GlobFilter`.
+            It does not change the behavior of
+            `partition discovery <https://spark.apache.org/docs/latest/sql-data-sources-parquet.html#partition-discovery>`_.  # noqa
+        recursiveFileLookup : str or bool, optional
+            recursively scan a directory for files. Using this option
+            disables
+            `partition discovery <https://spark.apache.org/docs/latest/sql-data-sources-parquet.html#partition-discovery>`_.  # noqa
+        allowNonNumericNumbers : str or bool, optional
+            allows JSON parser to recognize set of "Not-a-Number" (NaN)
+            tokens as legal floating number values. If None is set,
+            it uses the default value, ``true``.
+
+                * ``+INF``: for positive infinity, as well as alias of
+                            ``+Infinity`` and ``Infinity``.
+                *  ``-INF``: for negative infinity, alias ``-Infinity``.
+                *  ``NaN``: for other not-a-numbers, like result of division by zero.
+
+        Notes
+        -----
+        This API is evolving.
+
+        Examples
+        --------
         >>> json_sdf = spark.readStream.json(tempfile.mkdtemp(), schema = sdf_schema)
         >>> json_sdf.isStreaming
         True
@@ -526,27 +625,35 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
             timestampFormat=timestampFormat, multiLine=multiLine,
             allowUnquotedControlChars=allowUnquotedControlChars, lineSep=lineSep, locale=locale,
             dropFieldIfAllNull=dropFieldIfAllNull, encoding=encoding,
-            pathGlobFilter=pathGlobFilter, recursiveFileLookup=recursiveFileLookup)
-        if isinstance(path, basestring):
+            pathGlobFilter=pathGlobFilter, recursiveFileLookup=recursiveFileLookup,
+            allowNonNumericNumbers=allowNonNumericNumbers)
+        if isinstance(path, str):
             return self._df(self._jreader.json(path))
         else:
             raise TypeError("path can be only a single string")
 
-    @since(2.3)
     def orc(self, path, mergeSchema=None, pathGlobFilter=None, recursiveFileLookup=None):
         """Loads a ORC file stream, returning the result as a :class:`DataFrame`.
 
-        .. note:: Evolving.
-
-        :param mergeSchema: sets whether we should merge schemas collected from all
-                            ORC part-files. This will override ``spark.sql.orc.mergeSchema``.
-                            The default value is specified in ``spark.sql.orc.mergeSchema``.
-        :param pathGlobFilter: an optional glob pattern to only include files with paths matching
-                               the pattern. The syntax follows `org.apache.hadoop.fs.GlobFilter`.
-                               It does not change the behavior of `partition discovery`_.
-        :param recursiveFileLookup: recursively scan a directory for files. Using this option
-            disables `partition discovery`_.
-
+        .. versionadded:: 2.3.0
+
+        Parameters
+        ----------
+        mergeSchema : str or bool, optional
+            sets whether we should merge schemas collected from all
+            ORC part-files. This will override ``spark.sql.orc.mergeSchema``.
+            The default value is specified in ``spark.sql.orc.mergeSchema``.
+        pathGlobFilter : str or bool, optional
+            an optional glob pattern to only include files with paths matching
+            the pattern. The syntax follows `org.apache.hadoop.fs.GlobFilter`.
+            It does not change the behavior of `partition discovery`_.
+        recursiveFileLookup : str or bool, optional
+            recursively scan a directory for files. Using this option
+            disables
+            `partition discovery <https://spark.apache.org/docs/latest/sql-data-sources-parquet.html#partition-discovery>`_.  # noqa
+
+        Examples
+        --------
         >>> orc_sdf = spark.readStream.schema(sdf_schema).orc(tempfile.mkdtemp())
         >>> orc_sdf.isStreaming
         True
@@ -555,28 +662,60 @@ def orc(self, path, mergeSchema=None, pathGlobFilter=None, recursiveFileLookup=N
         """
         self._set_opts(mergeSchema=mergeSchema, pathGlobFilter=pathGlobFilter,
                        recursiveFileLookup=recursiveFileLookup)
-        if isinstance(path, basestring):
+        if isinstance(path, str):
             return self._df(self._jreader.orc(path))
         else:
             raise TypeError("path can be only a single string")
 
-    @since(2.0)
-    def parquet(self, path, mergeSchema=None, pathGlobFilter=None, recursiveFileLookup=None):
+    def parquet(self, path, mergeSchema=None, pathGlobFilter=None, recursiveFileLookup=None,
+                datetimeRebaseMode=None, int96RebaseMode=None):
         """
         Loads a Parquet file stream, returning the result as a :class:`DataFrame`.
 
-        .. note:: Evolving.
-
-        :param mergeSchema: sets whether we should merge schemas collected from all
-                            Parquet part-files. This will override
-                            ``spark.sql.parquet.mergeSchema``. The default value is specified in
-                            ``spark.sql.parquet.mergeSchema``.
-        :param pathGlobFilter: an optional glob pattern to only include files with paths matching
-                               the pattern. The syntax follows `org.apache.hadoop.fs.GlobFilter`.
-                               It does not change the behavior of `partition discovery`_.
-        :param recursiveFileLookup: recursively scan a directory for files. Using this option
-                                    disables `partition discovery`_.
-
+        .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        mergeSchema : str or bool, optional
+            sets whether we should merge schemas collected from all
+            Parquet part-files. This will override
+            ``spark.sql.parquet.mergeSchema``. The default value is specified in
+            ``spark.sql.parquet.mergeSchema``.
+        pathGlobFilter : str or bool, optional
+            an optional glob pattern to only include files with paths matching
+            the pattern. The syntax follows `org.apache.hadoop.fs.GlobFilter`.
+            It does not change the behavior of `partition discovery`_.
+        recursiveFileLookup : str or bool, optional
+            recursively scan a directory for files. Using this option
+            disables
+            `partition discovery <https://spark.apache.org/docs/latest/sql-data-sources-parquet.html#partition-discovery>`_.  # noqa
+        datetimeRebaseMode : str, optional
+            the rebasing mode for the values of the ``DATE``, ``TIMESTAMP_MICROS``,
+            ``TIMESTAMP_MILLIS`` logical types from the Julian to Proleptic Gregorian calendar.
+
+                * ``EXCEPTION``: Spark fails in reads of ancient dates/timestamps
+                                 that are ambiguous between the two calendars.
+                *  ``CORRECTED``: loading of dates/timestamps without rebasing.
+                *  ``LEGACY``: perform rebasing of ancient dates/timestamps from the Julian
+                               to Proleptic Gregorian calendar.
+
+            If None is set, the value of the SQL config
+            ``spark.sql.parquet.datetimeRebaseModeInRead`` is used by default.
+        int96RebaseMode : str, optional
+            the rebasing mode for ``INT96`` timestamps from the Julian to
+            Proleptic Gregorian calendar.
+
+                * ``EXCEPTION``: Spark fails in reads of ancient ``INT96`` timestamps
+                                 that are ambiguous between the two calendars.
+                *  ``CORRECTED``: loading of ``INT96`` timestamps without rebasing.
+                *  ``LEGACY``: perform rebasing of ancient ``INT96`` timestamps from the Julian
+                               to Proleptic Gregorian calendar.
+
+            If None is set, the value of the SQL config
+            ``spark.sql.parquet.int96RebaseModeInRead`` is used by default.
+
+        Examples
+        --------
         >>> parquet_sdf = spark.readStream.schema(sdf_schema).parquet(tempfile.mkdtemp())
         >>> parquet_sdf.isStreaming
         True
@@ -584,14 +723,13 @@ def parquet(self, path, mergeSchema=None, pathGlobFilter=None, recursiveFileLook
         True
         """
         self._set_opts(mergeSchema=mergeSchema, pathGlobFilter=pathGlobFilter,
-                       recursiveFileLookup=recursiveFileLookup)
-        if isinstance(path, basestring):
+                       recursiveFileLookup=recursiveFileLookup,
+                       datetimeRebaseMode=datetimeRebaseMode, int96RebaseMode=int96RebaseMode)
+        if isinstance(path, str):
             return self._df(self._jreader.parquet(path))
         else:
             raise TypeError("path can be only a single string")
 
-    @ignore_unicode_prefix
-    @since(2.0)
     def text(self, path, wholetext=False, lineSep=None, pathGlobFilter=None,
              recursiveFileLookup=None):
         """
@@ -602,18 +740,32 @@ def text(self, path, wholetext=False, lineSep=None, pathGlobFilter=None,
 
         By default, each line in the text file is a new row in the resulting DataFrame.
 
-        .. note:: Evolving.
-
-        :param paths: string, or list of strings, for input path(s).
-        :param wholetext: if true, read each file from input path(s) as a single row.
-        :param lineSep: defines the line separator that should be used for parsing. If None is
-                        set, it covers all ``\\r``, ``\\r\\n`` and ``\\n``.
-        :param pathGlobFilter: an optional glob pattern to only include files with paths matching
-                               the pattern. The syntax follows `org.apache.hadoop.fs.GlobFilter`.
-                               It does not change the behavior of `partition discovery`_.
-        :param recursiveFileLookup: recursively scan a directory for files. Using this option
-                                    disables `partition discovery`_.
-
+        .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        paths : str or list
+            string, or list of strings, for input path(s).
+        wholetext : str or bool, optional
+            if true, read each file from input path(s) as a single row.
+        lineSep : str, optional
+            defines the line separator that should be used for parsing. If None is
+            set, it covers all ``\\r``, ``\\r\\n`` and ``\\n``.
+        pathGlobFilter : str or bool, optional
+            an optional glob pattern to only include files with paths matching
+            the pattern. The syntax follows `org.apache.hadoop.fs.GlobFilter`.
+            It does not change the behavior of `partition discovery`_.
+        recursiveFileLookup : str or bool, optional
+            recursively scan a directory for files. Using this option
+            disables
+            `partition discovery <https://spark.apache.org/docs/latest/sql-data-sources-parquet.html#partition-discovery>`_.  # noqa
+
+        Notes
+        -----
+        This API is evolving.
+
+        Examples
+        --------
         >>> text_sdf = spark.readStream.text(tempfile.mkdtemp())
         >>> text_sdf.isStreaming
         True
@@ -623,12 +775,11 @@ def text(self, path, wholetext=False, lineSep=None, pathGlobFilter=None,
         self._set_opts(
             wholetext=wholetext, lineSep=lineSep, pathGlobFilter=pathGlobFilter,
             recursiveFileLookup=recursiveFileLookup)
-        if isinstance(path, basestring):
+        if isinstance(path, str):
             return self._df(self._jreader.text(path))
         else:
             raise TypeError("path can be only a single string")
 
-    @since(2.0)
     def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=None,
             comment=None, header=None, inferSchema=None, ignoreLeadingWhiteSpace=None,
             ignoreTrailingWhiteSpace=None, nullValue=None, nanValue=None, positiveInf=None,
@@ -636,114 +787,174 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
             maxCharsPerColumn=None, maxMalformedLogPerPartition=None, mode=None,
             columnNameOfCorruptRecord=None, multiLine=None, charToEscapeQuoteEscaping=None,
             enforceSchema=None, emptyValue=None, locale=None, lineSep=None,
-            pathGlobFilter=None, recursiveFileLookup=None):
+            pathGlobFilter=None, recursiveFileLookup=None, unescapedQuoteHandling=None):
         r"""Loads a CSV file stream and returns the result as a :class:`DataFrame`.
 
         This function will go through the input once to determine the input schema if
         ``inferSchema`` is enabled. To avoid going through the entire data once, disable
         ``inferSchema`` option or specify the schema explicitly using ``schema``.
 
-        .. note:: Evolving.
-
-        :param path: string, or list of strings, for input path(s).
-        :param schema: an optional :class:`pyspark.sql.types.StructType` for the input schema
-                       or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``).
-        :param sep: sets a separator (one or more characters) for each field and value. If None is
-                    set, it uses the default value, ``,``.
-        :param encoding: decodes the CSV files by the given encoding type. If None is set,
-                         it uses the default value, ``UTF-8``.
-        :param quote: sets a single character used for escaping quoted values where the
-                      separator can be part of the value. If None is set, it uses the default
-                      value, ``"``. If you would like to turn off quotations, you need to set an
-                      empty string.
-        :param escape: sets a single character used for escaping quotes inside an already
-                       quoted value. If None is set, it uses the default value, ``\``.
-        :param comment: sets a single character used for skipping lines beginning with this
-                        character. By default (None), it is disabled.
-        :param header: uses the first line as names of columns. If None is set, it uses the
-                       default value, ``false``.
-        :param inferSchema: infers the input schema automatically from data. It requires one extra
-                       pass over the data. If None is set, it uses the default value, ``false``.
-        :param enforceSchema: If it is set to ``true``, the specified or inferred schema will be
-                              forcibly applied to datasource files, and headers in CSV files will be
-                              ignored. If the option is set to ``false``, the schema will be
-                              validated against all headers in CSV files or the first header in RDD
-                              if the ``header`` option is set to ``true``. Field names in the schema
-                              and column names in CSV headers are checked by their positions
-                              taking into account ``spark.sql.caseSensitive``. If None is set,
-                              ``true`` is used by default. Though the default value is ``true``,
-                              it is recommended to disable the ``enforceSchema`` option
-                              to avoid incorrect results.
-        :param ignoreLeadingWhiteSpace: a flag indicating whether or not leading whitespaces from
-                                        values being read should be skipped. If None is set, it
-                                        uses the default value, ``false``.
-        :param ignoreTrailingWhiteSpace: a flag indicating whether or not trailing whitespaces from
-                                         values being read should be skipped. If None is set, it
-                                         uses the default value, ``false``.
-        :param nullValue: sets the string representation of a null value. If None is set, it uses
-                          the default value, empty string. Since 2.0.1, this ``nullValue`` param
-                          applies to all supported types including the string type.
-        :param nanValue: sets the string representation of a non-number value. If None is set, it
-                         uses the default value, ``NaN``.
-        :param positiveInf: sets the string representation of a positive infinity value. If None
-                            is set, it uses the default value, ``Inf``.
-        :param negativeInf: sets the string representation of a negative infinity value. If None
-                            is set, it uses the default value, ``Inf``.
-        :param dateFormat: sets the string that indicates a date format. Custom date formats
-                           follow the formats at `datetime pattern`_.
-                           This applies to date type. If None is set, it uses the
-                           default value, ``yyyy-MM-dd``.
-        :param timestampFormat: sets the string that indicates a timestamp format.
-                                Custom date formats follow the formats at `datetime pattern`_.
-                                This applies to timestamp type. If None is set, it uses the
-                                default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``.
-        :param maxColumns: defines a hard limit of how many columns a record can have. If None is
-                           set, it uses the default value, ``20480``.
-        :param maxCharsPerColumn: defines the maximum number of characters allowed for any given
-                                  value being read. If None is set, it uses the default value,
-                                  ``-1`` meaning unlimited length.
-        :param maxMalformedLogPerPartition: this parameter is no longer used since Spark 2.2.0.
-                                            If specified, it is ignored.
-        :param mode: allows a mode for dealing with corrupt records during parsing. If None is
-                     set, it uses the default value, ``PERMISSIVE``.
-
-                * ``PERMISSIVE``: when it meets a corrupted record, puts the malformed string \
-                  into a field configured by ``columnNameOfCorruptRecord``, and sets malformed \
-                  fields to ``null``. To keep corrupt records, an user can set a string type \
-                  field named ``columnNameOfCorruptRecord`` in an user-defined schema. If a \
-                  schema does not have the field, it drops corrupt records during parsing. \
-                  A record with less/more tokens than schema is not a corrupted record to CSV. \
-                  When it meets a record having fewer tokens than the length of the schema, \
-                  sets ``null`` to extra fields. When the record has more tokens than the \
-                  length of the schema, it drops extra tokens.
-                * ``DROPMALFORMED``: ignores the whole corrupted records.
-                * ``FAILFAST``: throws an exception when it meets corrupted records.
-
-        :param columnNameOfCorruptRecord: allows renaming the new field having malformed string
-                                          created by ``PERMISSIVE`` mode. This overrides
-                                          ``spark.sql.columnNameOfCorruptRecord``. If None is set,
-                                          it uses the value specified in
-                                          ``spark.sql.columnNameOfCorruptRecord``.
-        :param multiLine: parse one record, which may span multiple lines. If None is
-                          set, it uses the default value, ``false``.
-        :param charToEscapeQuoteEscaping: sets a single character used for escaping the escape for
-                                          the quote character. If None is set, the default value is
-                                          escape character when escape and quote characters are
-                                          different, ``\0`` otherwise..
-        :param emptyValue: sets the string representation of an empty value. If None is set, it uses
-                           the default value, empty string.
-        :param locale: sets a locale as language tag in IETF BCP 47 format. If None is set,
-                       it uses the default value, ``en-US``. For instance, ``locale`` is used while
-                       parsing dates and timestamps.
-        :param lineSep: defines the line separator that should be used for parsing. If None is
-                        set, it covers all ``\\r``, ``\\r\\n`` and ``\\n``.
-                        Maximum length is 1 character.
-        :param pathGlobFilter: an optional glob pattern to only include files with paths matching
-                               the pattern. The syntax follows `org.apache.hadoop.fs.GlobFilter`.
-                               It does not change the behavior of `partition discovery`_.
-        :param recursiveFileLookup: recursively scan a directory for files. Using this option
-                                    disables `partition discovery`_.
-
+        Parameters
+        ----------
+        path : str or list
+            string, or list of strings, for input path(s).
+        schema : :class:`pyspark.sql.types.StructType` or str, optional
+            an optional :class:`pyspark.sql.types.StructType` for the input schema
+            or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``).
+        sep : str, optional
+            sets a separator (one or more characters) for each field and value. If None is
+            set, it uses the default value, ``,``.
+        encoding : str, optional
+            decodes the CSV files by the given encoding type. If None is set,
+            it uses the default value, ``UTF-8``.
+        quote : str, optional sets a single character used for escaping quoted values where the
+            separator can be part of the value. If None is set, it uses the default
+            value, ``"``. If you would like to turn off quotations, you need to set an
+            empty string.
+        escape : str, optional
+            sets a single character used for escaping quotes inside an already
+            quoted value. If None is set, it uses the default value, ``\``.
+        comment : str, optional
+            sets a single character used for skipping lines beginning with this
+            character. By default (None), it is disabled.
+        header : str or bool, optional
+            uses the first line as names of columns. If None is set, it uses the
+            default value, ``false``.
+        inferSchema : str or bool, optional
+            infers the input schema automatically from data. It requires one extra
+            pass over the data. If None is set, it uses the default value, ``false``.
+        enforceSchema : str or bool, optional
+            If it is set to ``true``, the specified or inferred schema will be
+            forcibly applied to datasource files, and headers in CSV files will be
+            ignored. If the option is set to ``false``, the schema will be
+            validated against all headers in CSV files or the first header in RDD
+            if the ``header`` option is set to ``true``. Field names in the schema
+            and column names in CSV headers are checked by their positions
+            taking into account ``spark.sql.caseSensitive``. If None is set,
+            ``true`` is used by default. Though the default value is ``true``,
+            it is recommended to disable the ``enforceSchema`` option
+            to avoid incorrect results.
+        ignoreLeadingWhiteSpace : str or bool, optional
+            a flag indicating whether or not leading whitespaces from
+            values being read should be skipped. If None is set, it
+            uses the default value, ``false``.
+        ignoreTrailingWhiteSpace : str or bool, optional
+            a flag indicating whether or not trailing whitespaces from
+            values being read should be skipped. If None is set, it
+            uses the default value, ``false``.
+        nullValue : str, optional
+            sets the string representation of a null value. If None is set, it uses
+            the default value, empty string. Since 2.0.1, this ``nullValue`` param
+            applies to all supported types including the string type.
+        nanValue : str, optional
+            sets the string representation of a non-number value. If None is set, it
+            uses the default value, ``NaN``.
+        positiveInf : str, optional
+            sets the string representation of a positive infinity value. If None
+            is set, it uses the default value, ``Inf``.
+        negativeInf : str, optional
+            sets the string representation of a negative infinity value. If None
+            is set, it uses the default value, ``Inf``.
+        dateFormat : str, optional
+            sets the string that indicates a date format. Custom date formats
+            follow the formats at
+            `datetime pattern <https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html>`_.  # noqa
+            This applies to date type. If None is set, it uses the
+            default value, ``yyyy-MM-dd``.
+        timestampFormat : str, optional
+            sets the string that indicates a timestamp format.
+            Custom date formats follow the formats at
+            `datetime pattern <https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html>`_.  # noqa
+            This applies to timestamp type. If None is set, it uses the
+            default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``.
+        maxColumns : str or int, optional
+            defines a hard limit of how many columns a record can have. If None is
+            set, it uses the default value, ``20480``.
+        maxCharsPerColumn : str or int, optional
+            defines the maximum number of characters allowed for any given
+            value being read. If None is set, it uses the default value,
+            ``-1`` meaning unlimited length.
+        maxMalformedLogPerPartition : str or int, optional
+            this parameter is no longer used since Spark 2.2.0.
+            If specified, it is ignored.
+        mode : str, optional
+            allows a mode for dealing with corrupt records during parsing. If None is
+            set, it uses the default value, ``PERMISSIVE``.
+
+            * ``PERMISSIVE``: when it meets a corrupted record, puts the malformed string \
+              into a field configured by ``columnNameOfCorruptRecord``, and sets malformed \
+              fields to ``null``. To keep corrupt records, an user can set a string type \
+              field named ``columnNameOfCorruptRecord`` in an user-defined schema. If a \
+              schema does not have the field, it drops corrupt records during parsing. \
+              A record with less/more tokens than schema is not a corrupted record to CSV. \
+              When it meets a record having fewer tokens than the length of the schema, \
+              sets ``null`` to extra fields. When the record has more tokens than the \
+              length of the schema, it drops extra tokens.
+            * ``DROPMALFORMED``: ignores the whole corrupted records.
+            * ``FAILFAST``: throws an exception when it meets corrupted records.
+
+        columnNameOfCorruptRecord : str, optional
+            allows renaming the new field having malformed string
+            created by ``PERMISSIVE`` mode. This overrides
+            ``spark.sql.columnNameOfCorruptRecord``. If None is set,
+            it uses the value specified in
+            ``spark.sql.columnNameOfCorruptRecord``.
+        multiLine : str or bool, optional
+            parse one record, which may span multiple lines. If None is
+            set, it uses the default value, ``false``.
+        charToEscapeQuoteEscaping : str, optional
+            sets a single character used for escaping the escape for
+            the quote character. If None is set, the default value is
+            escape character when escape and quote characters are
+            different, ``\0`` otherwise.
+        emptyValue : str, optional
+            sets the string representation of an empty value. If None is set, it uses
+            the default value, empty string.
+        locale : str, optional
+            sets a locale as language tag in IETF BCP 47 format. If None is set,
+            it uses the default value, ``en-US``. For instance, ``locale`` is used while
+            parsing dates and timestamps.
+        lineSep : str, optional
+            defines the line separator that should be used for parsing. If None is
+            set, it covers all ``\\r``, ``\\r\\n`` and ``\\n``.
+            Maximum length is 1 character.
+        pathGlobFilter : str or bool, optional
+            an optional glob pattern to only include files with paths matching
+            the pattern. The syntax follows `org.apache.hadoop.fs.GlobFilter`.
+            It does not change the behavior of
+            `partition discovery <https://spark.apache.org/docs/latest/sql-data-sources-parquet.html#partition-discovery>`_.  # noqa
+        recursiveFileLookup : str or bool, optional
+            recursively scan a directory for files. Using this option disables
+            `partition discovery <https://spark.apache.org/docs/latest/sql-data-sources-parquet.html#partition-discovery>`_.  # noqa
+        unescapedQuoteHandling : str, optional
+            defines how the CsvParser will handle values with unescaped quotes. If None is
+            set, it uses the default value, ``STOP_AT_DELIMITER``.
+
+            * ``STOP_AT_CLOSING_QUOTE``: If unescaped quotes are found in the input, accumulate
+              the quote character and proceed parsing the value as a quoted value, until a closing
+              quote is found.
+            * ``BACK_TO_DELIMITER``: If unescaped quotes are found in the input, consider the value
+              as an unquoted value. This will make the parser accumulate all characters of the current
+              parsed value until the delimiter is found. If no delimiter is found in the value, the
+              parser will continue accumulating characters from the input until a delimiter or line
+              ending is found.
+            * ``STOP_AT_DELIMITER``: If unescaped quotes are found in the input, consider the value
+              as an unquoted value. This will make the parser accumulate all characters until the
+              delimiter or a line ending is found in the input.
+            * ``SKIP_VALUE``: If unescaped quotes are found in the input, the content parsed
+              for the given value will be skipped and the value set in nullValue will be produced
+              instead.
+            * ``RAISE_ERROR``: If unescaped quotes are found in the input, a TextParsingException
+              will be thrown.
+
+        .. versionadded:: 2.0.0
+
+        Notes
+        -----
+        This API is evolving.
+
+        Examples
+        --------
         >>> csv_sdf = spark.readStream.csv(tempfile.mkdtemp(), schema = sdf_schema)
         >>> csv_sdf.isStreaming
         True
@@ -761,12 +972,41 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
             columnNameOfCorruptRecord=columnNameOfCorruptRecord, multiLine=multiLine,
             charToEscapeQuoteEscaping=charToEscapeQuoteEscaping, enforceSchema=enforceSchema,
             emptyValue=emptyValue, locale=locale, lineSep=lineSep,
-            pathGlobFilter=pathGlobFilter, recursiveFileLookup=recursiveFileLookup)
-        if isinstance(path, basestring):
+            pathGlobFilter=pathGlobFilter, recursiveFileLookup=recursiveFileLookup,
+            unescapedQuoteHandling=unescapedQuoteHandling)
+        if isinstance(path, str):
             return self._df(self._jreader.csv(path))
         else:
             raise TypeError("path can be only a single string")
 
+    def table(self, tableName):
+        """Define a Streaming DataFrame on a Table. The DataSource corresponding to the table should
+        support streaming mode.
+
+        .. versionadded:: 3.1.0
+
+        Parameters
+        ----------
+        tableName : str
+            string, for the name of the table.
+
+        Returns
+        --------
+        :class:`DataFrame`
+
+        Notes
+        -----
+        This API is evolving.
+
+        Examples
+        --------
+        >>> spark.readStream.table('input_table') # doctest: +SKIP
+        """
+        if isinstance(tableName, str):
+            return self._df(self._jreader.table(tableName))
+        else:
+            raise TypeError("tableName can be only a single string")
+
 
 class DataStreamWriter(object):
     """
@@ -775,9 +1015,11 @@ class DataStreamWriter(object):
     Use :attr:`DataFrame.writeStream <pyspark.sql.DataFrame.writeStream>`
     to access this.
 
-    .. note:: Evolving.
+    .. versionadded:: 2.0.0
 
-    .. versionadded:: 2.0
+    Notes
+    -----
+    This API is evolving.
     """
 
     def __init__(self, df):
@@ -789,22 +1031,27 @@ def _sq(self, jsq):
         from pyspark.sql.streaming import StreamingQuery
         return StreamingQuery(jsq)
 
-    @since(2.0)
     def outputMode(self, outputMode):
         """Specifies how data of a streaming DataFrame/Dataset is written to a streaming sink.
 
+        .. versionadded:: 2.0.0
+
         Options include:
 
         * `append`: Only the new rows in the streaming DataFrame/Dataset will be written to
            the sink
         * `complete`: All the rows in the streaming DataFrame/Dataset will be written to the sink
-           every time these is some updates
+           every time these are some updates
         * `update`: only the rows that were updated in the streaming DataFrame/Dataset will be
            written to the sink every time there are some updates. If the query doesn't contain
            aggregations, it will be equivalent to `append` mode.
 
-       .. note:: Evolving.
+        Notes
+        -----
+        This API is evolving.
 
+        Examples
+        --------
         >>> writer = sdf.writeStream.outputMode('append')
         """
         if not outputMode or type(outputMode) != str or len(outputMode.strip()) == 0:
@@ -812,20 +1059,27 @@ def outputMode(self, outputMode):
         self._jwrite = self._jwrite.outputMode(outputMode)
         return self
 
-    @since(2.0)
     def format(self, source):
         """Specifies the underlying output data source.
 
-        .. note:: Evolving.
+        .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        source : str
+            string, name of the data source, which for now can be 'parquet'.
 
-        :param source: string, name of the data source, which for now can be 'parquet'.
+        Notes
+        -----
+        This API is evolving.
 
+        Examples
+        --------
         >>> writer = sdf.writeStream.format('json')
         """
         self._jwrite = self._jwrite.format(source)
         return self
 
-    @since(2.0)
     def option(self, key, value):
         """Adds an output option for the underlying data source.
 
@@ -843,12 +1097,15 @@ def option(self, key, value):
                 ambiguous. If it isn't set, the current value of the SQL config
                 ``spark.sql.session.timeZone`` is used by default.
 
-        .. note:: Evolving.
+        .. versionadded:: 2.0.0
+
+        Notes
+        -----
+        This API is evolving.
         """
         self._jwrite = self._jwrite.option(key, to_str(value))
         return self
 
-    @since(2.0)
     def options(self, **options):
         """Adds output options for the underlying data source.
 
@@ -866,39 +1123,56 @@ def options(self, **options):
                 ambiguous. If it isn't set, the current value of the SQL config
                 ``spark.sql.session.timeZone`` is used by default.
 
-       .. note:: Evolving.
+        .. versionadded:: 2.0.0
+
+        Notes
+        -----
+        This API is evolving.
         """
         for k in options:
             self._jwrite = self._jwrite.option(k, to_str(options[k]))
         return self
 
-    @since(2.0)
     def partitionBy(self, *cols):
         """Partitions the output by the given columns on the file system.
 
         If specified, the output is laid out on the file system similar
         to Hive's partitioning scheme.
 
-        .. note:: Evolving.
+        .. versionadded:: 2.0.0
 
-        :param cols: name of columns
+        Parameters
+        ----------
+        cols : str or list
+            name of columns
 
+        Notes
+        -----
+        This API is evolving.
         """
         if len(cols) == 1 and isinstance(cols[0], (list, tuple)):
             cols = cols[0]
         self._jwrite = self._jwrite.partitionBy(_to_seq(self._spark._sc, cols))
         return self
 
-    @since(2.0)
     def queryName(self, queryName):
         """Specifies the name of the :class:`StreamingQuery` that can be started with
         :func:`start`. This name must be unique among all the currently active queries
         in the associated SparkSession.
 
-        .. note:: Evolving.
+        .. versionadded:: 2.0.0
 
-        :param queryName: unique name for the query
+        Parameters
+        ----------
+        queryName : str
+            unique name for the query
 
+        Notes
+        -----
+        This API is evolving.
+
+        Examples
+        --------
         >>> writer = sdf.writeStream.queryName('streaming_query')
         """
         if not queryName or type(queryName) != str or len(queryName.strip()) == 0:
@@ -907,22 +1181,32 @@ def queryName(self, queryName):
         return self
 
     @keyword_only
-    @since(2.0)
-    def trigger(self, processingTime=None, once=None, continuous=None):
+    def trigger(self, *, processingTime=None, once=None, continuous=None):
         """Set the trigger for the stream query. If this is not set it will run the query as fast
         as possible, which is equivalent to setting the trigger to ``processingTime='0 seconds'``.
 
-        .. note:: Evolving.
-
-        :param processingTime: a processing time interval as a string, e.g. '5 seconds', '1 minute'.
-                               Set a trigger that runs a microbatch query periodically based on the
-                               processing time. Only one trigger can be set.
-        :param once: if set to True, set a trigger that processes only one batch of data in a
-                     streaming query then terminates the query. Only one trigger can be set.
-        :param continuous: a time interval as a string, e.g. '5 seconds', '1 minute'.
-                           Set a trigger that runs a continuous query with a given checkpoint
-                           interval. Only one trigger can be set.
-
+        .. versionadded:: 2.0.0
+
+        Parameters
+        ----------
+        processingTime : str, optional
+            a processing time interval as a string, e.g. '5 seconds', '1 minute'.
+            Set a trigger that runs a microbatch query periodically based on the
+            processing time. Only one trigger can be set.
+        once : bool, optional
+            if set to True, set a trigger that processes only one batch of data in a
+            streaming query then terminates the query. Only one trigger can be set.
+        continuous : str, optional
+            a time interval as a string, e.g. '5 seconds', '1 minute'.
+            Set a trigger that runs a continuous query with a given checkpoint
+            interval. Only one trigger can be set.
+
+        Notes
+        -----
+        This API is evolving.
+
+        Examples
+        --------
         >>> # trigger the query for execution every 5 seconds
         >>> writer = sdf.writeStream.trigger(processingTime='5 seconds')
         >>> # trigger the query for just once batch of data
@@ -962,7 +1246,6 @@ def trigger(self, processingTime=None, once=None, continuous=None):
         self._jwrite = self._jwrite.trigger(jTrigger)
         return self
 
-    @since(2.4)
     def foreach(self, f):
         """
         Sets the output of the streaming query to be processed using the provided writer ``f``.
@@ -1030,8 +1313,14 @@ def foreach(self, f):
                 returns successfully (irrespective of the return value), except if the Python
                 crashes in the middle.
 
-        .. note:: Evolving.
+        .. versionadded:: 2.4.0
+
+        Notes
+        -----
+        This API is evolving.
 
+        Examples
+        --------
         >>> # Print every row using a function
         >>> def print_row(row):
         ...     print(row)
@@ -1124,7 +1413,6 @@ def func_with_open_process_close(partition_id, iterator):
         self._jwrite.foreach(jForeachWriter)
         return self
 
-    @since(2.4)
     def foreachBatch(self, func):
         """
         Sets the output of the streaming query to be processed using the provided
@@ -1136,8 +1424,14 @@ def foreachBatch(self, func):
         to exactly same for the same batchId (assuming all operations are deterministic in the
         query).
 
-        .. note:: Evolving.
+        .. versionadded:: 2.4.0
+
+        Notes
+        -----
+        This API is evolving.
 
+        Examples
+        --------
         >>> def func(batch_df, batch_id):
         ...     batch_df.collect()
         ...
@@ -1153,8 +1447,6 @@ def foreachBatch(self, func):
         ensure_callback_server_started(gw)
         return self
 
-    @ignore_unicode_prefix
-    @since(2.0)
     def start(self, path=None, format=None, outputMode=None, partitionBy=None, queryName=None,
               **options):
         """Streams the contents of the :class:`DataFrame` to a data source.
@@ -1163,37 +1455,51 @@ def start(self, path=None, format=None, outputMode=None, partitionBy=None, query
         If ``format`` is not specified, the default data source configured by
         ``spark.sql.sources.default`` will be used.
 
-        .. note:: Evolving.
+        .. versionadded:: 2.0.0
 
-        :param path: the path in a Hadoop supported file system
-        :param format: the format used to save
-        :param outputMode: specifies how data of a streaming DataFrame/Dataset is written to a
-                           streaming sink.
+        Parameters
+        ----------
+        path : str, optional
+            the path in a Hadoop supported file system
+        format : str, optional
+            the format used to save
+        outputMode : str, optional
+            specifies how data of a streaming DataFrame/Dataset is written to a
+            streaming sink.
 
             * `append`: Only the new rows in the streaming DataFrame/Dataset will be written to the
               sink
             * `complete`: All the rows in the streaming DataFrame/Dataset will be written to the
-              sink every time these is some updates
+              sink every time these are some updates
             * `update`: only the rows that were updated in the streaming DataFrame/Dataset will be
               written to the sink every time there are some updates. If the query doesn't contain
               aggregations, it will be equivalent to `append` mode.
-        :param partitionBy: names of partitioning columns
-        :param queryName: unique name for the query
-        :param options: All other string options. You may want to provide a `checkpointLocation`
-                        for most streams, however it is not required for a `memory` stream.
-
+        partitionBy : str or list, optional
+            names of partitioning columns
+        queryName : str, optional
+            unique name for the query
+        **options : dict
+            All other string options. You may want to provide a `checkpointLocation`
+            for most streams, however it is not required for a `memory` stream.
+
+        Notes
+        -----
+        This API is evolving.
+
+        Examples
+        --------
         >>> sq = sdf.writeStream.format('memory').queryName('this_query').start()
         >>> sq.isActive
         True
         >>> sq.name
-        u'this_query'
+        'this_query'
         >>> sq.stop()
         >>> sq.isActive
         False
         >>> sq = sdf.writeStream.trigger(processingTime='5 seconds').start(
         ...     queryName='that_query', outputMode="append", format='memory')
         >>> sq.name
-        u'that_query'
+        'that_query'
         >>> sq.isActive
         True
         >>> sq.stop()
@@ -1212,12 +1518,82 @@ def start(self, path=None, format=None, outputMode=None, partitionBy=None, query
         else:
             return self._sq(self._jwrite.start(path))
 
+    def toTable(self, tableName, format=None, outputMode=None, partitionBy=None, queryName=None,
+                **options):
+        """
+        Starts the execution of the streaming query, which will continually output results to the
+        given table as new data arrives.
+
+        The returned :class:`StreamingQuery` object can be used to interact with the stream.
+
+        .. versionadded:: 3.1.0
+
+        Parameters
+        ----------
+        tableName : str
+            string, for the name of the table.
+        format : str, optional
+            the format used to save.
+        outputMode : str, optional
+            specifies how data of a streaming DataFrame/Dataset is written to a
+            streaming sink.
+
+            * `append`: Only the new rows in the streaming DataFrame/Dataset will be written to the
+              sink
+            * `complete`: All the rows in the streaming DataFrame/Dataset will be written to the
+              sink every time these are some updates
+            * `update`: only the rows that were updated in the streaming DataFrame/Dataset will be
+              written to the sink every time there are some updates. If the query doesn't contain
+              aggregations, it will be equivalent to `append` mode.
+        partitionBy : str or list, optional
+            names of partitioning columns
+        queryName : str, optional
+            unique name for the query
+        **options : dict
+            All other string options. You may want to provide a `checkpointLocation`.
+
+        Notes
+        -----
+        This API is evolving.
+
+        For v1 table, partitioning columns provided by `partitionBy` will be respected no matter
+        the table exists or not. A new table will be created if the table not exists.
+
+        For v2 table, `partitionBy` will be ignored if the table already exists. `partitionBy` will
+        be respected only if the v2 table does not exist. Besides, the v2 table created by this API
+        lacks some functionalities (e.g., customized properties, options, and serde info). If you
+        need them, please create the v2 table manually before the execution to avoid creating a
+        table with incomplete information.
+
+        Examples
+        --------
+        >>> sdf.writeStream.format('parquet').queryName('query').toTable('output_table')
+        ... # doctest: +SKIP
+
+        >>> sdf.writeStream.trigger(processingTime='5 seconds').toTable(
+        ...     'output_table',
+        ...     queryName='that_query',
+        ...     outputMode="append",
+        ...     format='parquet',
+        ...     checkpointLocation='/tmp/checkpoint') # doctest: +SKIP
+        """
+        self.options(**options)
+        if outputMode is not None:
+            self.outputMode(outputMode)
+        if partitionBy is not None:
+            self.partitionBy(partitionBy)
+        if format is not None:
+            self.format(format)
+        if queryName is not None:
+            self.queryName(queryName)
+        return self._sq(self._jwrite.toTable(tableName))
+
 
 def _test():
     import doctest
     import os
     import tempfile
-    from pyspark.sql import Row, SparkSession, SQLContext
+    from pyspark.sql import SparkSession, SQLContext
     import pyspark.sql.streaming
 
     os.chdir(os.environ["SPARK_HOME"])
@@ -1225,8 +1601,8 @@ def _test():
     globs = pyspark.sql.streaming.__dict__.copy()
     try:
         spark = SparkSession.builder.getOrCreate()
-    except py4j.protocol.Py4JError:
-        spark = SparkSession(sc)
+    except py4j.protocol.Py4JError:  # noqa: F821
+        spark = SparkSession(sc)  # noqa: F821
 
     globs['tempfile'] = tempfile
     globs['os'] = os
diff --git a/python/pyspark/sql/streaming.pyi b/python/pyspark/sql/streaming.pyi
new file mode 100644
index 0000000000000..1d05483c012f1
--- /dev/null
+++ b/python/pyspark/sql/streaming.pyi
@@ -0,0 +1,197 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import overload
+from typing import Any, Callable, Dict, List, Optional, Union
+
+from pyspark.sql._typing import SupportsProcess, OptionalPrimitiveType
+from pyspark.sql.context import SQLContext
+from pyspark.sql.dataframe import DataFrame
+from pyspark.sql.readwriter import OptionUtils
+from pyspark.sql.types import Row, StructType
+from pyspark.sql.utils import StreamingQueryException
+
+from py4j.java_gateway import JavaObject  # type: ignore[import]
+
+class StreamingQuery:
+    def __init__(self, jsq: JavaObject) -> None: ...
+    @property
+    def id(self) -> str: ...
+    @property
+    def runId(self) -> str: ...
+    @property
+    def name(self) -> str: ...
+    @property
+    def isActive(self) -> bool: ...
+    def awaitTermination(self, timeout: Optional[int] = ...) -> Optional[bool]: ...
+    @property
+    def status(self) -> Dict[str, Any]: ...
+    @property
+    def recentProgress(self) -> List[Dict[str, Any]]: ...
+    @property
+    def lastProgress(self) -> Optional[Dict[str, Any]]: ...
+    def processAllAvailable(self) -> None: ...
+    def stop(self) -> None: ...
+    def explain(self, extended: bool = ...) -> None: ...
+    def exception(self) -> Optional[StreamingQueryException]: ...
+
+class StreamingQueryManager:
+    def __init__(self, jsqm: JavaObject) -> None: ...
+    @property
+    def active(self) -> List[StreamingQuery]: ...
+    def get(self, id: str) -> StreamingQuery: ...
+    def awaitAnyTermination(self, timeout: Optional[int] = ...) -> bool: ...
+    def resetTerminated(self) -> None: ...
+
+class DataStreamReader(OptionUtils):
+    def __init__(self, spark: SQLContext) -> None: ...
+    def format(self, source: str) -> DataStreamReader: ...
+    def schema(self, schema: Union[StructType, str]) -> DataStreamReader: ...
+    def option(self, key: str, value: OptionalPrimitiveType) -> DataStreamReader: ...
+    def options(self, **options: OptionalPrimitiveType) -> DataStreamReader: ...
+    def load(
+        self,
+        path: Optional[str] = ...,
+        format: Optional[str] = ...,
+        schema: Optional[Union[StructType, str]] = ...,
+        **options: OptionalPrimitiveType
+    ) -> DataFrame: ...
+    def json(
+        self,
+        path: str,
+        schema: Optional[Union[StructType, str]] = ...,
+        primitivesAsString: Optional[Union[bool, str]] = ...,
+        prefersDecimal: Optional[Union[bool, str]] = ...,
+        allowComments: Optional[Union[bool, str]] = ...,
+        allowUnquotedFieldNames: Optional[Union[bool, str]] = ...,
+        allowSingleQuotes: Optional[Union[bool, str]] = ...,
+        allowNumericLeadingZero: Optional[Union[bool, str]] = ...,
+        allowBackslashEscapingAnyCharacter: Optional[Union[bool, str]] = ...,
+        mode: Optional[str] = ...,
+        columnNameOfCorruptRecord: Optional[str] = ...,
+        dateFormat: Optional[str] = ...,
+        timestampFormat: Optional[str] = ...,
+        multiLine: Optional[Union[bool, str]] = ...,
+        allowUnquotedControlChars: Optional[Union[bool, str]] = ...,
+        lineSep: Optional[str] = ...,
+        locale: Optional[str] = ...,
+        dropFieldIfAllNull: Optional[Union[bool, str]] = ...,
+        encoding: Optional[str] = ...,
+        pathGlobFilter: Optional[Union[bool, str]] = ...,
+        recursiveFileLookup: Optional[Union[bool, str]] = ...,
+        allowNonNumericNumbers: Optional[Union[bool, str]] = ...,
+    ) -> DataFrame: ...
+    def orc(
+        self,
+        path: str,
+        mergeSchema: Optional[bool] = ...,
+        pathGlobFilter: Optional[Union[bool, str]] = ...,
+        recursiveFileLookup: Optional[Union[bool, str]] = ...,
+    ) -> DataFrame: ...
+    def parquet(
+        self,
+        path: str,
+        mergeSchema: Optional[bool] = ...,
+        pathGlobFilter: Optional[Union[bool, str]] = ...,
+        recursiveFileLookup: Optional[Union[bool, str]] = ...,
+    ) -> DataFrame: ...
+    def text(
+        self,
+        path: str,
+        wholetext: bool = ...,
+        lineSep: Optional[str] = ...,
+        pathGlobFilter: Optional[Union[bool, str]] = ...,
+        recursiveFileLookup: Optional[Union[bool, str]] = ...,
+    ) -> DataFrame: ...
+    def csv(
+        self,
+        path: str,
+        schema: Optional[Union[StructType, str]] = ...,
+        sep: Optional[str] = ...,
+        encoding: Optional[str] = ...,
+        quote: Optional[str] = ...,
+        escape: Optional[str] = ...,
+        comment: Optional[str] = ...,
+        header: Optional[Union[bool, str]] = ...,
+        inferSchema: Optional[Union[bool, str]] = ...,
+        ignoreLeadingWhiteSpace: Optional[Union[bool, str]] = ...,
+        ignoreTrailingWhiteSpace: Optional[Union[bool, str]] = ...,
+        nullValue: Optional[str] = ...,
+        nanValue: Optional[str] = ...,
+        positiveInf: Optional[str] = ...,
+        negativeInf: Optional[str] = ...,
+        dateFormat: Optional[str] = ...,
+        timestampFormat: Optional[str] = ...,
+        maxColumns: Optional[Union[int, str]] = ...,
+        maxCharsPerColumn: Optional[Union[int, str]] = ...,
+        mode: Optional[str] = ...,
+        columnNameOfCorruptRecord: Optional[str] = ...,
+        multiLine: Optional[Union[bool, str]] = ...,
+        charToEscapeQuoteEscaping: Optional[Union[bool, str]] = ...,
+        enforceSchema: Optional[Union[bool, str]] = ...,
+        emptyValue: Optional[str] = ...,
+        locale: Optional[str] = ...,
+        lineSep: Optional[str] = ...,
+        pathGlobFilter: Optional[Union[bool, str]] = ...,
+        recursiveFileLookup: Optional[Union[bool, str]] = ...,
+        unescapedQuoteHandling: Optional[str] = ...,
+    ) -> DataFrame: ...
+    def table(self, tableName: str) -> DataFrame: ...
+
+class DataStreamWriter:
+    def __init__(self, df: DataFrame) -> None: ...
+    def outputMode(self, outputMode: str) -> DataStreamWriter: ...
+    def format(self, source: str) -> DataStreamWriter: ...
+    def option(self, key: str, value: OptionalPrimitiveType) -> DataStreamWriter: ...
+    def options(self, **options: OptionalPrimitiveType) -> DataStreamWriter: ...
+    @overload
+    def partitionBy(self, *cols: str) -> DataStreamWriter: ...
+    @overload
+    def partitionBy(self, __cols: List[str]) -> DataStreamWriter: ...
+    def queryName(self, queryName: str) -> DataStreamWriter: ...
+    @overload
+    def trigger(self, processingTime: str) -> DataStreamWriter: ...
+    @overload
+    def trigger(self, once: bool) -> DataStreamWriter: ...
+    @overload
+    def trigger(self, continuous: bool) -> DataStreamWriter: ...
+    def start(
+        self,
+        path: Optional[str] = ...,
+        format: Optional[str] = ...,
+        outputMode: Optional[str] = ...,
+        partitionBy: Optional[Union[str, List[str]]] = ...,
+        queryName: Optional[str] = ...,
+        **options: OptionalPrimitiveType
+    ) -> StreamingQuery: ...
+    @overload
+    def foreach(self, f: Callable[[Row], None]) -> DataStreamWriter: ...
+    @overload
+    def foreach(self, f: SupportsProcess) -> DataStreamWriter: ...
+    def foreachBatch(
+        self, func: Callable[[DataFrame, int], None]
+    ) -> DataStreamWriter: ...
+    def toTable(
+        self,
+        tableName: str,
+        format: Optional[str] = ...,
+        outputMode: Optional[str] = ...,
+        partitionBy: Optional[Union[str, List[str]]] = ...,
+        queryName: Optional[str] = ...,
+        **options: OptionalPrimitiveType
+    ) -> StreamingQuery: ...
diff --git a/python/pyspark/sql/tests/test_arrow.py b/python/pyspark/sql/tests/test_arrow.py
index 004c79f290213..184360749fbe5 100644
--- a/python/pyspark/sql/tests/test_arrow.py
+++ b/python/pyspark/sql/tests/test_arrow.py
@@ -21,27 +21,29 @@
 import time
 import unittest
 import warnings
+from distutils.version import LooseVersion
 
 from pyspark import SparkContext, SparkConf
 from pyspark.sql import Row, SparkSession
-from pyspark.sql.functions import udf
-from pyspark.sql.types import *
+from pyspark.sql.functions import rand, udf
+from pyspark.sql.types import StructType, StringType, IntegerType, LongType, \
+    FloatType, DoubleType, DecimalType, DateType, TimestampType, BinaryType, StructField, \
+    ArrayType, NullType
 from pyspark.testing.sqlutils import ReusedSQLTestCase, have_pandas, have_pyarrow, \
     pandas_requirement_message, pyarrow_requirement_message
 from pyspark.testing.utils import QuietTest
-from pyspark.util import _exception_message
 
 if have_pandas:
     import pandas as pd
-    from pandas.util.testing import assert_frame_equal
+    from pandas.testing import assert_frame_equal
 
 if have_pyarrow:
-    import pyarrow as pa
+    import pyarrow as pa  # noqa: F401
 
 
 @unittest.skipIf(
     not have_pandas or not have_pyarrow,
-    pandas_requirement_message or pyarrow_requirement_message)
+    pandas_requirement_message or pyarrow_requirement_message)  # type: ignore
 class ArrowTests(ReusedSQLTestCase):
 
     @classmethod
@@ -75,7 +77,7 @@ def setUpClass(cls):
         # Disable fallback by default to easily detect the failures.
         cls.spark.conf.set("spark.sql.execution.arrow.pyspark.fallback.enabled", "false")
 
-        cls.schema = StructType([
+        cls.schema_wo_null = StructType([
             StructField("1_str_t", StringType(), True),
             StructField("2_int_t", IntegerType(), True),
             StructField("3_long_t", LongType(), True),
@@ -85,14 +87,18 @@ def setUpClass(cls):
             StructField("7_date_t", DateType(), True),
             StructField("8_timestamp_t", TimestampType(), True),
             StructField("9_binary_t", BinaryType(), True)])
-        cls.data = [(u"a", 1, 10, 0.2, 2.0, Decimal("2.0"),
-                     date(1969, 1, 1), datetime(1969, 1, 1, 1, 1, 1), bytearray(b"a")),
-                    (u"b", 2, 20, 0.4, 4.0, Decimal("4.0"),
-                     date(2012, 2, 2), datetime(2012, 2, 2, 2, 2, 2), bytearray(b"bb")),
-                    (u"c", 3, 30, 0.8, 6.0, Decimal("6.0"),
-                     date(2100, 3, 3), datetime(2100, 3, 3, 3, 3, 3), bytearray(b"ccc")),
-                    (u"d", 4, 40, 1.0, 8.0, Decimal("8.0"),
-                     date(2262, 4, 12), datetime(2262, 3, 3, 3, 3, 3), bytearray(b"dddd"))]
+        cls.schema = cls.schema_wo_null.add("10_null_t", NullType(), True)
+        cls.data_wo_null = [
+            (u"a", 1, 10, 0.2, 2.0, Decimal("2.0"),
+             date(1969, 1, 1), datetime(1969, 1, 1, 1, 1, 1), bytearray(b"a")),
+            (u"b", 2, 20, 0.4, 4.0, Decimal("4.0"),
+             date(2012, 2, 2), datetime(2012, 2, 2, 2, 2, 2), bytearray(b"bb")),
+            (u"c", 3, 30, 0.8, 6.0, Decimal("6.0"),
+             date(2100, 3, 3), datetime(2100, 3, 3, 3, 3, 3), bytearray(b"ccc")),
+            (u"d", 4, 40, 1.0, 8.0, Decimal("8.0"),
+             date(2262, 4, 12), datetime(2262, 3, 3, 3, 3, 3), bytearray(b"dddd")),
+        ]
+        cls.data = [tuple(list(d) + [None]) for d in cls.data_wo_null]
 
     @classmethod
     def tearDownClass(cls):
@@ -113,9 +119,10 @@ def create_pandas_data_frame(self):
         return pd.DataFrame(data=data_dict)
 
     def test_toPandas_fallback_enabled(self):
+        ts = datetime.datetime(2015, 11, 1, 0, 30)
         with self.sql_conf({"spark.sql.execution.arrow.pyspark.fallback.enabled": True}):
-            schema = StructType([StructField("map", MapType(StringType(), IntegerType()), True)])
-            df = self.spark.createDataFrame([({u'a': 1},)], schema=schema)
+            schema = StructType([StructField("a", ArrayType(TimestampType()), True)])
+            df = self.spark.createDataFrame([([ts],)], schema=schema)
             with QuietTest(self.sc):
                 with self.warnings_lock:
                     with warnings.catch_warnings(record=True) as warns:
@@ -127,20 +134,20 @@ def test_toPandas_fallback_enabled(self):
                             warn.message for warn in warns if isinstance(warn.message, UserWarning)]
                         self.assertTrue(len(user_warns) > 0)
                         self.assertTrue(
-                            "Attempting non-optimization" in _exception_message(user_warns[-1]))
-                        assert_frame_equal(pdf, pd.DataFrame({u'map': [{u'a': 1}]}))
+                            "Attempting non-optimization" in str(user_warns[-1]))
+                        assert_frame_equal(pdf, pd.DataFrame({"a": [[ts]]}))
 
     def test_toPandas_fallback_disabled(self):
-        schema = StructType([StructField("map", MapType(StringType(), IntegerType()), True)])
+        schema = StructType([StructField("a", ArrayType(TimestampType()), True)])
         df = self.spark.createDataFrame([(None,)], schema=schema)
         with QuietTest(self.sc):
             with self.warnings_lock:
-                with self.assertRaisesRegexp(Exception, 'Unsupported type'):
+                with self.assertRaisesRegex(Exception, 'Unsupported type'):
                     df.toPandas()
 
     def test_null_conversion(self):
-        df_null = self.spark.createDataFrame([tuple([None for _ in range(len(self.data[0]))])] +
-                                             self.data)
+        df_null = self.spark.createDataFrame(
+            [tuple([None for _ in range(len(self.data_wo_null[0]))])] + self.data_wo_null)
         pdf = df_null.toPandas()
         null_counts = pdf.isnull().sum().tolist()
         self.assertTrue(all([c == 1 for c in null_counts]))
@@ -189,6 +196,37 @@ def test_pandas_round_trip(self):
         pdf_arrow = df.toPandas()
         assert_frame_equal(pdf_arrow, pdf)
 
+    def test_pandas_self_destruct(self):
+        import pyarrow as pa
+        rows = 2 ** 10
+        cols = 4
+        expected_bytes = rows * cols * 8
+        df = self.spark.range(0, rows).select(*[rand() for _ in range(cols)])
+        # Test the self_destruct behavior by testing _collect_as_arrow directly
+        allocation_before = pa.total_allocated_bytes()
+        batches = df._collect_as_arrow(split_batches=True)
+        table = pa.Table.from_batches(batches)
+        del batches
+        pdf_split = table.to_pandas(self_destruct=True, split_blocks=True, use_threads=False)
+        allocation_after = pa.total_allocated_bytes()
+        difference = allocation_after - allocation_before
+        # Should be around 1x the data size (table should not hold on to any memory)
+        self.assertGreaterEqual(difference, 0.9 * expected_bytes)
+        self.assertLessEqual(difference, 1.1 * expected_bytes)
+
+        with self.sql_conf({"spark.sql.execution.arrow.pyspark.selfDestruct.enabled": False}):
+            no_self_destruct_pdf = df.toPandas()
+            # Note while memory usage is 2x data size here (both table and pdf hold on to
+            # memory), in this case Arrow still only tracks 1x worth of memory (since the
+            # batches are not allocated by Arrow in this case), so we can't make any
+            # assertions here
+
+        with self.sql_conf({"spark.sql.execution.arrow.pyspark.selfDestruct.enabled": True}):
+            self_destruct_pdf = df.toPandas()
+
+        assert_frame_equal(pdf_split, no_self_destruct_pdf)
+        assert_frame_equal(pdf_split, self_destruct_pdf)
+
     def test_filtered_frame(self):
         df = self.spark.range(3).toDF("i")
         pdf = df.filter("i < 0").toPandas()
@@ -212,7 +250,7 @@ def raise_exception():
         exception_udf = udf(raise_exception, IntegerType())
         df = df.withColumn("error", exception_udf())
         with QuietTest(self.sc):
-            with self.assertRaisesRegexp(Exception, 'My error'):
+            with self.assertRaisesRegex(Exception, 'My error'):
                 df.toPandas()
 
     def _createDataFrame_toggle(self, pdf, schema=None):
@@ -226,7 +264,7 @@ def _createDataFrame_toggle(self, pdf, schema=None):
     def test_createDataFrame_toggle(self):
         pdf = self.create_pandas_data_frame()
         df_no_arrow, df_arrow = self._createDataFrame_toggle(pdf, schema=self.schema)
-        self.assertEquals(df_no_arrow.collect(), df_arrow.collect())
+        self.assertEqual(df_no_arrow.collect(), df_arrow.collect())
 
     def test_createDataFrame_respect_session_timezone(self):
         from datetime import timedelta
@@ -256,41 +294,42 @@ def test_createDataFrame_respect_session_timezone(self):
     def test_createDataFrame_with_schema(self):
         pdf = self.create_pandas_data_frame()
         df = self.spark.createDataFrame(pdf, schema=self.schema)
-        self.assertEquals(self.schema, df.schema)
+        self.assertEqual(self.schema, df.schema)
         pdf_arrow = df.toPandas()
         assert_frame_equal(pdf_arrow, pdf)
 
     def test_createDataFrame_with_incorrect_schema(self):
         pdf = self.create_pandas_data_frame()
         fields = list(self.schema)
-        fields[0], fields[1] = fields[1], fields[0]  # swap str with int
+        fields[5], fields[6] = fields[6], fields[5]  # swap decimal with date
         wrong_schema = StructType(fields)
-        with QuietTest(self.sc):
-            with self.assertRaisesRegexp(Exception, "integer.*required"):
-                self.spark.createDataFrame(pdf, schema=wrong_schema)
+        with self.sql_conf({"spark.sql.execution.pandas.convertToArrowArraySafely": False}):
+            with QuietTest(self.sc):
+                with self.assertRaisesRegex(Exception, "[D|d]ecimal.*got.*date"):
+                    self.spark.createDataFrame(pdf, schema=wrong_schema)
 
     def test_createDataFrame_with_names(self):
         pdf = self.create_pandas_data_frame()
         new_names = list(map(str, range(len(self.schema.fieldNames()))))
         # Test that schema as a list of column names gets applied
         df = self.spark.createDataFrame(pdf, schema=list(new_names))
-        self.assertEquals(df.schema.fieldNames(), new_names)
+        self.assertEqual(df.schema.fieldNames(), new_names)
         # Test that schema as tuple of column names gets applied
         df = self.spark.createDataFrame(pdf, schema=tuple(new_names))
-        self.assertEquals(df.schema.fieldNames(), new_names)
+        self.assertEqual(df.schema.fieldNames(), new_names)
 
     def test_createDataFrame_column_name_encoding(self):
         pdf = pd.DataFrame({u'a': [1]})
         columns = self.spark.createDataFrame(pdf).columns
         self.assertTrue(isinstance(columns[0], str))
-        self.assertEquals(columns[0], 'a')
+        self.assertEqual(columns[0], 'a')
         columns = self.spark.createDataFrame(pdf, [u'b']).columns
         self.assertTrue(isinstance(columns[0], str))
-        self.assertEquals(columns[0], 'b')
+        self.assertEqual(columns[0], 'b')
 
     def test_createDataFrame_with_single_data_type(self):
         with QuietTest(self.sc):
-            with self.assertRaisesRegexp(ValueError, ".*IntegerType.*not supported.*"):
+            with self.assertRaisesRegex(ValueError, ".*IntegerType.*not supported.*"):
                 self.spark.createDataFrame(pd.DataFrame({"a": [1]}), schema="int")
 
     def test_createDataFrame_does_not_modify_input(self):
@@ -308,7 +347,7 @@ def test_schema_conversion_roundtrip(self):
         from pyspark.sql.pandas.types import from_arrow_schema, to_arrow_schema
         arrow_schema = to_arrow_schema(self.schema)
         schema_rt = from_arrow_schema(arrow_schema)
-        self.assertEquals(self.schema, schema_rt)
+        self.assertEqual(self.schema, schema_rt)
 
     def test_createDataFrame_with_array_type(self):
         pdf = pd.DataFrame({"a": [[1, 2], [3, 4]], "b": [[u"x", u"y"], [u"y", u"z"]]})
@@ -334,6 +373,62 @@ def test_toPandas_with_array_type(self):
                 self.assertTrue(expected[r][e] == result_arrow[r][e] and
                                 result[r][e] == result_arrow[r][e])
 
+    def test_createDataFrame_with_map_type(self):
+        map_data = [{"a": 1}, {"b": 2, "c": 3}, {}, None, {"d": None}]
+
+        pdf = pd.DataFrame({"id": [0, 1, 2, 3, 4], "m": map_data})
+        schema = "id long, m map<string, long>"
+
+        with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": False}):
+            df = self.spark.createDataFrame(pdf, schema=schema)
+
+        if LooseVersion(pa.__version__) < LooseVersion("2.0.0"):
+            with QuietTest(self.sc):
+                with self.assertRaisesRegex(Exception, "MapType.*only.*pyarrow 2.0.0"):
+                    self.spark.createDataFrame(pdf, schema=schema)
+        else:
+            df_arrow = self.spark.createDataFrame(pdf, schema=schema)
+
+            result = df.collect()
+            result_arrow = df_arrow.collect()
+
+            self.assertEqual(len(result), len(result_arrow))
+            for row, row_arrow in zip(result, result_arrow):
+                i, m = row
+                _, m_arrow = row_arrow
+                self.assertEqual(m, map_data[i])
+                self.assertEqual(m_arrow, map_data[i])
+
+    def test_toPandas_with_map_type(self):
+        pdf = pd.DataFrame({"id": [0, 1, 2, 3],
+                            "m": [{}, {"a": 1}, {"a": 1, "b": 2}, {"a": 1, "b": 2, "c": 3}]})
+
+        with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": False}):
+            df = self.spark.createDataFrame(pdf, schema="id long, m map<string, long>")
+
+        if LooseVersion(pa.__version__) < LooseVersion("2.0.0"):
+            with QuietTest(self.sc):
+                with self.assertRaisesRegex(Exception, "MapType.*only.*pyarrow 2.0.0"):
+                    df.toPandas()
+        else:
+            pdf_non, pdf_arrow = self._toPandas_arrow_toggle(df)
+            assert_frame_equal(pdf_arrow, pdf_non)
+
+    def test_toPandas_with_map_type_nulls(self):
+        pdf = pd.DataFrame({"id": [0, 1, 2, 3, 4],
+                            "m": [{"a": 1}, {"b": 2, "c": 3}, {}, None, {"d": None}]})
+
+        with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": False}):
+            df = self.spark.createDataFrame(pdf, schema="id long, m map<string, long>")
+
+        if LooseVersion(pa.__version__) < LooseVersion("2.0.0"):
+            with QuietTest(self.sc):
+                with self.assertRaisesRegex(Exception, "MapType.*only.*pyarrow 2.0.0"):
+                    df.toPandas()
+        else:
+            pdf_non, pdf_arrow = self._toPandas_arrow_toggle(df)
+            assert_frame_equal(pdf_arrow, pdf_non)
+
     def test_createDataFrame_with_int_col_names(self):
         import numpy as np
         pdf = pd.DataFrame(np.random.rand(4, 2))
@@ -343,26 +438,28 @@ def test_createDataFrame_with_int_col_names(self):
         self.assertEqual(pdf_col_names, df_arrow.columns)
 
     def test_createDataFrame_fallback_enabled(self):
+        ts = datetime.datetime(2015, 11, 1, 0, 30)
         with QuietTest(self.sc):
             with self.sql_conf({"spark.sql.execution.arrow.pyspark.fallback.enabled": True}):
                 with warnings.catch_warnings(record=True) as warns:
                     # we want the warnings to appear even if this test is run from a subclass
                     warnings.simplefilter("always")
                     df = self.spark.createDataFrame(
-                        pd.DataFrame([[{u'a': 1}]]), "a: map<string, int>")
+                        pd.DataFrame({"a": [[ts]]}), "a: array<timestamp>")
                     # Catch and check the last UserWarning.
                     user_warns = [
                         warn.message for warn in warns if isinstance(warn.message, UserWarning)]
                     self.assertTrue(len(user_warns) > 0)
                     self.assertTrue(
-                        "Attempting non-optimization" in _exception_message(user_warns[-1]))
-                    self.assertEqual(df.collect(), [Row(a={u'a': 1})])
+                        "Attempting non-optimization" in str(user_warns[-1]))
+                    self.assertEqual(df.collect(), [Row(a=[ts])])
 
     def test_createDataFrame_fallback_disabled(self):
         with QuietTest(self.sc):
-            with self.assertRaisesRegexp(TypeError, 'Unsupported type'):
+            with self.assertRaisesRegex(TypeError, 'Unsupported type'):
                 self.spark.createDataFrame(
-                    pd.DataFrame([[{u'a': 1}]]), "a: map<string, int>")
+                    pd.DataFrame({"a": [[datetime.datetime(2015, 11, 1, 0, 30)]]}),
+                    "a: array<timestamp>")
 
     # Regression test for SPARK-23314
     def test_timestamp_dst(self):
@@ -415,10 +512,56 @@ def run_test(num_records, num_parts, max_records, use_delay=False):
         for case in cases:
             run_test(*case)
 
+    def test_createDateFrame_with_category_type(self):
+        pdf = pd.DataFrame({"A": [u"a", u"b", u"c", u"a"]})
+        pdf["B"] = pdf["A"].astype('category')
+        category_first_element = dict(enumerate(pdf['B'].cat.categories))[0]
+
+        with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": True}):
+            arrow_df = self.spark.createDataFrame(pdf)
+            arrow_type = arrow_df.dtypes[1][1]
+            result_arrow = arrow_df.toPandas()
+            arrow_first_category_element = result_arrow["B"][0]
+
+        with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": False}):
+            df = self.spark.createDataFrame(pdf)
+            spark_type = df.dtypes[1][1]
+            result_spark = df.toPandas()
+            spark_first_category_element = result_spark["B"][0]
+
+        assert_frame_equal(result_spark, result_arrow)
+
+        # ensure original category elements are string
+        self.assertIsInstance(category_first_element, str)
+        # spark data frame and arrow execution mode enabled data frame type must match pandas
+        self.assertEqual(spark_type, 'string')
+        self.assertEqual(arrow_type, 'string')
+        self.assertIsInstance(arrow_first_category_element, str)
+        self.assertIsInstance(spark_first_category_element, str)
+
+    def test_createDataFrame_with_float_index(self):
+        # SPARK-32098: float index should not produce duplicated or truncated Spark DataFrame
+        self.assertEqual(
+            self.spark.createDataFrame(
+                pd.DataFrame({'a': [1, 2, 3]}, index=[2., 3., 4.])).distinct().count(), 3)
+
+    def test_no_partition_toPandas(self):
+        # SPARK-32301: toPandas should work from a Spark DataFrame with no partitions
+        # Forward-ported from SPARK-32300.
+        pdf = self.spark.sparkContext.emptyRDD().toDF("col1 int").toPandas()
+        self.assertEqual(len(pdf), 0)
+        self.assertEqual(list(pdf.columns), ["col1"])
+
+    def test_createDataFrame_empty_partition(self):
+        pdf = pd.DataFrame({"c1": [1], "c2": ["string"]})
+        df = self.spark.createDataFrame(pdf)
+        self.assertEqual([Row(c1=1, c2='string')], df.collect())
+        self.assertGreater(self.spark.sparkContext.defaultParallelism, len(pdf))
+
 
 @unittest.skipIf(
     not have_pandas or not have_pyarrow,
-    pandas_requirement_message or pyarrow_requirement_message)
+    pandas_requirement_message or pyarrow_requirement_message)  # type: ignore
 class MaxResultArrowTests(unittest.TestCase):
     # These tests are separate as 'spark.driver.maxResultSize' configuration
     # is a static configuration to Spark context.
@@ -438,7 +581,7 @@ def tearDownClass(cls):
             cls.spark.stop()
 
     def test_exception_by_max_results(self):
-        with self.assertRaisesRegexp(Exception, "is bigger than"):
+        with self.assertRaisesRegex(Exception, "is bigger than"):
             self.spark.range(0, 10000, 1, 100).toPandas()
 
 
@@ -450,10 +593,10 @@ def conf(cls):
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_arrow import *
+    from pyspark.sql.tests.test_arrow import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/sql/tests/test_catalog.py b/python/pyspark/sql/tests/test_catalog.py
index ebe81d0325031..56e7c97020662 100644
--- a/python/pyspark/sql/tests/test_catalog.py
+++ b/python/pyspark/sql/tests/test_catalog.py
@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+from pyspark.sql.types import StructType, StructField, IntegerType
 from pyspark.sql.utils import AnalysisException
 from pyspark.testing.sqlutils import ReusedSQLTestCase
 
@@ -24,11 +25,11 @@ class CatalogTests(ReusedSQLTestCase):
     def test_current_database(self):
         spark = self.spark
         with self.database("some_db"):
-            self.assertEquals(spark.catalog.currentDatabase(), "default")
+            self.assertEqual(spark.catalog.currentDatabase(), "default")
             spark.sql("CREATE DATABASE some_db")
             spark.catalog.setCurrentDatabase("some_db")
-            self.assertEquals(spark.catalog.currentDatabase(), "some_db")
-            self.assertRaisesRegexp(
+            self.assertEqual(spark.catalog.currentDatabase(), "some_db")
+            self.assertRaisesRegex(
                 AnalysisException,
                 "does_not_exist",
                 lambda: spark.catalog.setCurrentDatabase("does_not_exist"))
@@ -37,56 +38,68 @@ def test_list_databases(self):
         spark = self.spark
         with self.database("some_db"):
             databases = [db.name for db in spark.catalog.listDatabases()]
-            self.assertEquals(databases, ["default"])
+            self.assertEqual(databases, ["default"])
             spark.sql("CREATE DATABASE some_db")
             databases = [db.name for db in spark.catalog.listDatabases()]
-            self.assertEquals(sorted(databases), ["default", "some_db"])
+            self.assertEqual(sorted(databases), ["default", "some_db"])
 
     def test_list_tables(self):
         from pyspark.sql.catalog import Table
         spark = self.spark
         with self.database("some_db"):
             spark.sql("CREATE DATABASE some_db")
-            with self.table("tab1", "some_db.tab2"):
+            with self.table("tab1", "some_db.tab2", "tab3_via_catalog"):
                 with self.tempView("temp_tab"):
-                    self.assertEquals(spark.catalog.listTables(), [])
-                    self.assertEquals(spark.catalog.listTables("some_db"), [])
+                    self.assertEqual(spark.catalog.listTables(), [])
+                    self.assertEqual(spark.catalog.listTables("some_db"), [])
                     spark.createDataFrame([(1, 1)]).createOrReplaceTempView("temp_tab")
                     spark.sql("CREATE TABLE tab1 (name STRING, age INT) USING parquet")
                     spark.sql("CREATE TABLE some_db.tab2 (name STRING, age INT) USING parquet")
+
+                    schema = StructType([StructField("a", IntegerType(), True)])
+                    description = "this a table created via Catalog.createTable()"
+                    spark.catalog.createTable(
+                        "tab3_via_catalog", schema=schema, description=description)
+
                     tables = sorted(spark.catalog.listTables(), key=lambda t: t.name)
                     tablesDefault = \
                         sorted(spark.catalog.listTables("default"), key=lambda t: t.name)
                     tablesSomeDb = \
                         sorted(spark.catalog.listTables("some_db"), key=lambda t: t.name)
-                    self.assertEquals(tables, tablesDefault)
-                    self.assertEquals(len(tables), 2)
-                    self.assertEquals(len(tablesSomeDb), 2)
-                    self.assertEquals(tables[0], Table(
+                    self.assertEqual(tables, tablesDefault)
+                    self.assertEqual(len(tables), 3)
+                    self.assertEqual(len(tablesSomeDb), 2)
+                    self.assertEqual(tables[0], Table(
                         name="tab1",
                         database="default",
                         description=None,
                         tableType="MANAGED",
                         isTemporary=False))
-                    self.assertEquals(tables[1], Table(
+                    self.assertEqual(tables[1], Table(
+                        name="tab3_via_catalog",
+                        database="default",
+                        description=description,
+                        tableType="MANAGED",
+                        isTemporary=False))
+                    self.assertEqual(tables[2], Table(
                         name="temp_tab",
                         database=None,
                         description=None,
                         tableType="TEMPORARY",
                         isTemporary=True))
-                    self.assertEquals(tablesSomeDb[0], Table(
+                    self.assertEqual(tablesSomeDb[0], Table(
                         name="tab2",
                         database="some_db",
                         description=None,
                         tableType="MANAGED",
                         isTemporary=False))
-                    self.assertEquals(tablesSomeDb[1], Table(
+                    self.assertEqual(tablesSomeDb[1], Table(
                         name="temp_tab",
                         database=None,
                         description=None,
                         tableType="TEMPORARY",
                         isTemporary=True))
-                    self.assertRaisesRegexp(
+                    self.assertRaisesRegex(
                         AnalysisException,
                         "does_not_exist",
                         lambda: spark.catalog.listTables("does_not_exist"))
@@ -106,12 +119,12 @@ def test_list_functions(self):
             self.assertTrue("to_timestamp" in functions)
             self.assertTrue("to_unix_timestamp" in functions)
             self.assertTrue("current_database" in functions)
-            self.assertEquals(functions["+"], Function(
+            self.assertEqual(functions["+"], Function(
                 name="+",
                 description=None,
                 className="org.apache.spark.sql.catalyst.expressions.Add",
                 isTemporary=True))
-            self.assertEquals(functions, functionsDefault)
+            self.assertEqual(functions, functionsDefault)
 
             with self.function("func1", "some_db.func2"):
                 spark.catalog.registerFunction("temp_func", lambda x: str(x))
@@ -128,7 +141,7 @@ def test_list_functions(self):
                 self.assertTrue("temp_func" in newFunctionsSomeDb)
                 self.assertTrue("func1" not in newFunctionsSomeDb)
                 self.assertTrue("func2" in newFunctionsSomeDb)
-                self.assertRaisesRegexp(
+                self.assertRaisesRegex(
                     AnalysisException,
                     "does_not_exist",
                     lambda: spark.catalog.listFunctions("does_not_exist"))
@@ -145,16 +158,16 @@ def test_list_columns(self):
                 columns = sorted(spark.catalog.listColumns("tab1"), key=lambda c: c.name)
                 columnsDefault = \
                     sorted(spark.catalog.listColumns("tab1", "default"), key=lambda c: c.name)
-                self.assertEquals(columns, columnsDefault)
-                self.assertEquals(len(columns), 2)
-                self.assertEquals(columns[0], Column(
+                self.assertEqual(columns, columnsDefault)
+                self.assertEqual(len(columns), 2)
+                self.assertEqual(columns[0], Column(
                     name="age",
                     description=None,
                     dataType="int",
                     nullable=True,
                     isPartition=False,
                     isBucket=False))
-                self.assertEquals(columns[1], Column(
+                self.assertEqual(columns[1], Column(
                     name="name",
                     description=None,
                     dataType="string",
@@ -163,26 +176,26 @@ def test_list_columns(self):
                     isBucket=False))
                 columns2 = \
                     sorted(spark.catalog.listColumns("tab2", "some_db"), key=lambda c: c.name)
-                self.assertEquals(len(columns2), 2)
-                self.assertEquals(columns2[0], Column(
+                self.assertEqual(len(columns2), 2)
+                self.assertEqual(columns2[0], Column(
                     name="nickname",
                     description=None,
                     dataType="string",
                     nullable=True,
                     isPartition=False,
                     isBucket=False))
-                self.assertEquals(columns2[1], Column(
+                self.assertEqual(columns2[1], Column(
                     name="tolerance",
                     description=None,
                     dataType="float",
                     nullable=True,
                     isPartition=False,
                     isBucket=False))
-                self.assertRaisesRegexp(
+                self.assertRaisesRegex(
                     AnalysisException,
                     "tab2",
                     lambda: spark.catalog.listColumns("tab2"))
-                self.assertRaisesRegexp(
+                self.assertRaisesRegex(
                     AnalysisException,
                     "does_not_exist",
                     lambda: spark.catalog.listColumns("does_not_exist"))
@@ -190,10 +203,10 @@ def test_list_columns(self):
 
 if __name__ == "__main__":
     import unittest
-    from pyspark.sql.tests.test_catalog import *
+    from pyspark.sql.tests.test_catalog import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/sql/tests/test_column.py b/python/pyspark/sql/tests/test_column.py
index 58bf896a10c2a..c2530b294eedb 100644
--- a/python/pyspark/sql/tests/test_column.py
+++ b/python/pyspark/sql/tests/test_column.py
@@ -16,10 +16,8 @@
 # limitations under the License.
 #
 
-import sys
-
 from pyspark.sql import Column, Row
-from pyspark.sql.types import *
+from pyspark.sql.types import StructType, StructField, LongType
 from pyspark.sql.utils import AnalysisException
 from pyspark.testing.sqlutils import ReusedSQLTestCase
 
@@ -49,7 +47,7 @@ def test_validate_column_types(self):
         self.assertTrue("Column" in _to_java_column(u"a").getClass().toString())
         self.assertTrue("Column" in _to_java_column(self.spark.range(1).id).getClass().toString())
 
-        self.assertRaisesRegexp(
+        self.assertRaisesRegex(
             TypeError,
             "Invalid argument, not a string or column",
             lambda: _to_java_column(1))
@@ -60,7 +58,7 @@ class A():
         self.assertRaises(TypeError, lambda: _to_java_column(A()))
         self.assertRaises(TypeError, lambda: _to_java_column([]))
 
-        self.assertRaisesRegexp(
+        self.assertRaisesRegex(
             TypeError,
             "Invalid argument, not a string or column",
             lambda: udf(lambda x: x)(None))
@@ -81,9 +79,9 @@ def test_column_operators(self):
             cs.startswith('a'), cs.endswith('a'), ci.eqNullSafe(cs)
         self.assertTrue(all(isinstance(c, Column) for c in css))
         self.assertTrue(isinstance(ci.cast(LongType()), Column))
-        self.assertRaisesRegexp(ValueError,
-                                "Cannot apply 'in' operator against a column",
-                                lambda: 1 in cs)
+        self.assertRaisesRegex(ValueError,
+                               "Cannot apply 'in' operator against a column",
+                               lambda: 1 in cs)
 
     def test_column_accessor(self):
         from pyspark.sql.functions import col
@@ -109,12 +107,8 @@ def test_access_column(self):
         self.assertRaises(TypeError, lambda: df[{}])
 
     def test_column_name_with_non_ascii(self):
-        if sys.version >= '3':
-            columnName = "数量"
-            self.assertTrue(isinstance(columnName, str))
-        else:
-            columnName = unicode("数量", "utf-8")
-            self.assertTrue(isinstance(columnName, unicode))
+        columnName = "数量"
+        self.assertTrue(isinstance(columnName, str))
         schema = StructType([StructField(columnName, LongType(), True)])
         df = self.spark.createDataFrame([(1,)], schema)
         self.assertEqual(schema, df.schema)
@@ -122,6 +116,7 @@ def test_column_name_with_non_ascii(self):
         self.assertEqual([("数量", 'bigint')], df.dtypes)
         self.assertEqual(1, df.select("数量").first()[0])
         self.assertEqual(1, df.select(df["数量"]).first()[0])
+        self.assertTrue(columnName in repr(df[columnName]))
 
     def test_field_accessor(self):
         df = self.sc.parallelize([Row(l=[1], r=Row(a=1, b="b"), d={"k": "v"})]).toDF()
@@ -144,14 +139,54 @@ def test_bitwise_operations(self):
         self.assertEqual(170 ^ 75, result['(a ^ b)'])
         result = df.select(functions.bitwiseNOT(df.b)).collect()[0].asDict()
         self.assertEqual(~75, result['~b'])
+        result = df.select(functions.bitwise_not(df.b)).collect()[0].asDict()
+        self.assertEqual(~75, result['~b'])
 
+    def test_with_field(self):
+        from pyspark.sql.functions import lit, col
+        df = self.spark.createDataFrame([Row(a=Row(b=1, c=2))])
+        self.assertIsInstance(df['a'].withField('b', lit(3)), Column)
+        self.assertIsInstance(df['a'].withField('d', lit(3)), Column)
+        result = df.withColumn('a', df['a'].withField('d', lit(3))).collect()[0].asDict()
+        self.assertEqual(3, result['a']['d'])
+        result = df.withColumn('a', df['a'].withField('b', lit(3))).collect()[0].asDict()
+        self.assertEqual(3, result['a']['b'])
+
+        self.assertRaisesRegex(TypeError,
+                               'col should be a Column',
+                               lambda: df['a'].withField('b', 3))
+        self.assertRaisesRegex(TypeError,
+                               'fieldName should be a string',
+                               lambda: df['a'].withField(col('b'), lit(3)))
+
+    def test_drop_fields(self):
+        df = self.spark.createDataFrame([Row(a=Row(b=1, c=2, d=Row(e=3, f=4)))])
+        self.assertIsInstance(df["a"].dropFields("b"), Column)
+        self.assertIsInstance(df["a"].dropFields("b", "c"), Column)
+        self.assertIsInstance(df["a"].dropFields("d.e"), Column)
+
+        result = df.select(
+            df["a"].dropFields("b").alias("a1"),
+            df["a"].dropFields("d.e").alias("a2"),
+        ).first().asDict(True)
+
+        self.assertTrue(
+            "b" not in result["a1"] and
+            "c" in result["a1"] and
+            "d" in result["a1"]
+        )
+
+        self.assertTrue(
+            "e" not in result["a2"]["d"] and
+            "f" in result["a2"]["d"]
+        )
 
 if __name__ == "__main__":
     import unittest
-    from pyspark.sql.tests.test_column import *
+    from pyspark.sql.tests.test_column import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/sql/tests/test_conf.py b/python/pyspark/sql/tests/test_conf.py
index 9ec10c4cb6d23..9222e2b8272d6 100644
--- a/python/pyspark/sql/tests/test_conf.py
+++ b/python/pyspark/sql/tests/test_conf.py
@@ -28,7 +28,7 @@ def test_conf(self):
         self.assertEqual(spark.conf.get("bogo"), "ta")
         self.assertEqual(spark.conf.get("bogo", "not.read"), "ta")
         self.assertEqual(spark.conf.get("not.set", "ta"), "ta")
-        self.assertRaisesRegexp(Exception, "not.set", lambda: spark.conf.get("not.set"))
+        self.assertRaisesRegex(Exception, "not.set", lambda: spark.conf.get("not.set"))
         spark.conf.unset("bogo")
         self.assertEqual(spark.conf.get("bogo", "colombia"), "colombia")
 
@@ -46,10 +46,10 @@ def test_conf(self):
 
 if __name__ == "__main__":
     import unittest
-    from pyspark.sql.tests.test_conf import *
+    from pyspark.sql.tests.test_conf import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/sql/tests/test_context.py b/python/pyspark/sql/tests/test_context.py
index 3b1b638ed4aa6..ff1db31072df9 100644
--- a/python/pyspark/sql/tests/test_context.py
+++ b/python/pyspark/sql/tests/test_context.py
@@ -19,18 +19,13 @@
 import sys
 import tempfile
 import unittest
-try:
-    from importlib import reload  # Python 3.4+ only.
-except ImportError:
-    # Otherwise, we will stick to Python 2's built-in reload.
-    pass
+from importlib import reload
 
 import py4j
 
 from pyspark import SparkContext, SQLContext
 from pyspark.sql import Row, SparkSession
-from pyspark.sql.types import *
-from pyspark.sql.window import Window
+from pyspark.sql.types import StructType, StringType, StructField
 from pyspark.testing.utils import ReusedPySparkTestCase
 
 
@@ -112,99 +107,6 @@ def test_save_and_load_table(self):
 
         shutil.rmtree(tmpPath)
 
-    def test_window_functions(self):
-        df = self.spark.createDataFrame([(1, "1"), (2, "2"), (1, "2"), (1, "2")], ["key", "value"])
-        w = Window.partitionBy("value").orderBy("key")
-        from pyspark.sql import functions as F
-        sel = df.select(df.value, df.key,
-                        F.max("key").over(w.rowsBetween(0, 1)),
-                        F.min("key").over(w.rowsBetween(0, 1)),
-                        F.count("key").over(w.rowsBetween(float('-inf'), float('inf'))),
-                        F.row_number().over(w),
-                        F.rank().over(w),
-                        F.dense_rank().over(w),
-                        F.ntile(2).over(w))
-        rs = sorted(sel.collect())
-        expected = [
-            ("1", 1, 1, 1, 1, 1, 1, 1, 1),
-            ("2", 1, 1, 1, 3, 1, 1, 1, 1),
-            ("2", 1, 2, 1, 3, 2, 1, 1, 1),
-            ("2", 2, 2, 2, 3, 3, 3, 2, 2)
-        ]
-        for r, ex in zip(rs, expected):
-            self.assertEqual(tuple(r), ex[:len(r)])
-
-    def test_window_functions_without_partitionBy(self):
-        df = self.spark.createDataFrame([(1, "1"), (2, "2"), (1, "2"), (1, "2")], ["key", "value"])
-        w = Window.orderBy("key", df.value)
-        from pyspark.sql import functions as F
-        sel = df.select(df.value, df.key,
-                        F.max("key").over(w.rowsBetween(0, 1)),
-                        F.min("key").over(w.rowsBetween(0, 1)),
-                        F.count("key").over(w.rowsBetween(float('-inf'), float('inf'))),
-                        F.row_number().over(w),
-                        F.rank().over(w),
-                        F.dense_rank().over(w),
-                        F.ntile(2).over(w))
-        rs = sorted(sel.collect())
-        expected = [
-            ("1", 1, 1, 1, 4, 1, 1, 1, 1),
-            ("2", 1, 1, 1, 4, 2, 2, 2, 1),
-            ("2", 1, 2, 1, 4, 3, 2, 2, 2),
-            ("2", 2, 2, 2, 4, 4, 4, 3, 2)
-        ]
-        for r, ex in zip(rs, expected):
-            self.assertEqual(tuple(r), ex[:len(r)])
-
-    def test_window_functions_cumulative_sum(self):
-        df = self.spark.createDataFrame([("one", 1), ("two", 2)], ["key", "value"])
-        from pyspark.sql import functions as F
-
-        # Test cumulative sum
-        sel = df.select(
-            df.key,
-            F.sum(df.value).over(Window.rowsBetween(Window.unboundedPreceding, 0)))
-        rs = sorted(sel.collect())
-        expected = [("one", 1), ("two", 3)]
-        for r, ex in zip(rs, expected):
-            self.assertEqual(tuple(r), ex[:len(r)])
-
-        # Test boundary values less than JVM's Long.MinValue and make sure we don't overflow
-        sel = df.select(
-            df.key,
-            F.sum(df.value).over(Window.rowsBetween(Window.unboundedPreceding - 1, 0)))
-        rs = sorted(sel.collect())
-        expected = [("one", 1), ("two", 3)]
-        for r, ex in zip(rs, expected):
-            self.assertEqual(tuple(r), ex[:len(r)])
-
-        # Test boundary values greater than JVM's Long.MaxValue and make sure we don't overflow
-        frame_end = Window.unboundedFollowing + 1
-        sel = df.select(
-            df.key,
-            F.sum(df.value).over(Window.rowsBetween(Window.currentRow, frame_end)))
-        rs = sorted(sel.collect())
-        expected = [("one", 3), ("two", 2)]
-        for r, ex in zip(rs, expected):
-            self.assertEqual(tuple(r), ex[:len(r)])
-
-    def test_collect_functions(self):
-        df = self.spark.createDataFrame([(1, "1"), (2, "2"), (1, "2"), (1, "2")], ["key", "value"])
-        from pyspark.sql import functions
-
-        self.assertEqual(
-            sorted(df.select(functions.collect_set(df.key).alias('r')).collect()[0].r),
-            [1, 2])
-        self.assertEqual(
-            sorted(df.select(functions.collect_list(df.key).alias('r')).collect()[0].r),
-            [1, 1, 1, 2])
-        self.assertEqual(
-            sorted(df.select(functions.collect_set(df.value).alias('r')).collect()[0].r),
-            ["1", "2"])
-        self.assertEqual(
-            sorted(df.select(functions.collect_list(df.value).alias('r')).collect()[0].r),
-            ["1", "2", "2", "2"])
-
     def test_limit_and_take(self):
         df = self.spark.range(1, 1000, numPartitions=10)
 
@@ -223,13 +125,6 @@ def assert_runs_only_one_job_stage_and_task(job_group_name, f):
         # Regression test for SPARK-17514: limit(n).collect() should the perform same as take(n)
         assert_runs_only_one_job_stage_and_task("collect_limit", lambda: df.limit(1).collect())
 
-    def test_datetime_functions(self):
-        from pyspark.sql import functions
-        from datetime import date
-        df = self.spark.range(1).selectExpr("'2017-01-22' as dateCol")
-        parse_result = df.select(functions.to_date(functions.col("dateCol"))).first()
-        self.assertEquals(date(2017, 1, 22), parse_result['to_date(dateCol)'])
-
     def test_unbounded_frames(self):
         from pyspark.sql import functions as F
         from pyspark.sql import window
@@ -277,10 +172,10 @@ def test_get_or_create(self):
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_context import *
+    from pyspark.sql.tests.test_context import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/sql/tests/test_dataframe.py b/python/pyspark/sql/tests/test_dataframe.py
index 9861178158f85..3e961cba7e395 100644
--- a/python/pyspark/sql/tests/test_dataframe.py
+++ b/python/pyspark/sql/tests/test_dataframe.py
@@ -17,11 +17,14 @@
 
 import os
 import pydoc
+import shutil
+import tempfile
 import time
 import unittest
 
 from pyspark.sql import SparkSession, Row
-from pyspark.sql.types import *
+from pyspark.sql.types import StringType, IntegerType, DoubleType, StructType, StructField, \
+    BooleanType, DateType, TimestampType, FloatType
 from pyspark.sql.utils import AnalysisException, IllegalArgumentException
 from pyspark.testing.sqlutils import ReusedSQLTestCase, SQLTestUtils, have_pyarrow, have_pandas, \
     pandas_requirement_message, pyarrow_requirement_message
@@ -316,7 +319,7 @@ def test_replace(self):
         self.assertTupleEqual(row, (u'Alice', 20, None))
 
         # should fail if subset is not list, tuple or None
-        with self.assertRaises(ValueError):
+        with self.assertRaises(TypeError):
             self.spark.createDataFrame(
                 [(u'Alice', 10, 80.1)], schema).replace({10: 11}, subset=1).first()
 
@@ -326,7 +329,7 @@ def test_replace(self):
                 [(u'Alice', 10, 80.1)], schema).replace(["Alice", "Bob"], ["Eve"]).first()
 
         # should fail if when received unexpected type
-        with self.assertRaises(ValueError):
+        with self.assertRaises(TypeError):
             from datetime import datetime
             self.spark.createDataFrame(
                 [(u'Alice', 10, 80.1)], schema).replace(datetime.now(), datetime.now()).first()
@@ -340,7 +343,7 @@ def test_replace(self):
             self.spark.createDataFrame(
                 [(u'Alice', 10, 80.1)], schema).replace({u"Alice": u"Bob", 10: 20}).first()
 
-        with self.assertRaisesRegexp(
+        with self.assertRaisesRegex(
                 TypeError,
                 'value argument is required when to_replace is not a dictionary.'):
             self.spark.createDataFrame(
@@ -377,8 +380,6 @@ def test_generic_hints(self):
 
     # add tests for SPARK-23647 (test more types for hint)
     def test_extended_hint_types(self):
-        from pyspark.sql import DataFrame
-
         df = self.spark.range(10e10).toDF("id")
         such_a_nice_list = ["itworks1", "itworks2", "itworks3"]
         hinted_df = df.hint("my awesome hint", 1.2345, "what", such_a_nice_list)
@@ -389,7 +390,7 @@ def test_extended_hint_types(self):
         self.assertEqual(3, logical_plan.toString().count("itworks"))
 
     def test_sample(self):
-        self.assertRaisesRegexp(
+        self.assertRaisesRegex(
             TypeError,
             "should be a bool, float and number",
             lambda: self.spark.range(1).sample())
@@ -425,12 +426,12 @@ def test_toDF_with_schema_string(self):
         self.assertEqual(df.collect(), data)
 
         # number of fields must match.
-        self.assertRaisesRegexp(Exception, "Length of object",
-                                lambda: rdd.toDF("key: int").collect())
+        self.assertRaisesRegex(Exception, "Length of object",
+                               lambda: rdd.toDF("key: int").collect())
 
         # field types mismatch will cause exception at runtime.
-        self.assertRaisesRegexp(Exception, "FloatType can not accept",
-                                lambda: rdd.toDF("key: float, value: string").collect())
+        self.assertRaisesRegex(Exception, "FloatType can not accept",
+                               lambda: rdd.toDF("key: float, value: string").collect())
 
         # flat schema values will be wrapped into row.
         df = rdd.map(lambda row: row.key).toDF("int")
@@ -490,15 +491,15 @@ def test_cache(self):
             spark.catalog.clearCache()
             self.assertFalse(spark.catalog.isCached("tab1"))
             self.assertFalse(spark.catalog.isCached("tab2"))
-            self.assertRaisesRegexp(
+            self.assertRaisesRegex(
                 AnalysisException,
                 "does_not_exist",
                 lambda: spark.catalog.isCached("does_not_exist"))
-            self.assertRaisesRegexp(
+            self.assertRaisesRegex(
                 AnalysisException,
                 "does_not_exist",
                 lambda: spark.catalog.cacheTable("does_not_exist"))
-            self.assertRaisesRegexp(
+            self.assertRaisesRegex(
                 AnalysisException,
                 "does_not_exist",
                 lambda: spark.catalog.uncacheTable("does_not_exist"))
@@ -517,19 +518,19 @@ def _to_pandas(self):
         df = self.spark.createDataFrame(data, schema)
         return df.toPandas()
 
-    @unittest.skipIf(not have_pandas, pandas_requirement_message)
+    @unittest.skipIf(not have_pandas, pandas_requirement_message)  # type: ignore
     def test_to_pandas(self):
         import numpy as np
         pdf = self._to_pandas()
         types = pdf.dtypes
-        self.assertEquals(types[0], np.int32)
-        self.assertEquals(types[1], np.object)
-        self.assertEquals(types[2], np.bool)
-        self.assertEquals(types[3], np.float32)
-        self.assertEquals(types[4], np.object)  # datetime.date
-        self.assertEquals(types[5], 'datetime64[ns]')
-
-    @unittest.skipIf(not have_pandas, pandas_requirement_message)
+        self.assertEqual(types[0], np.int32)
+        self.assertEqual(types[1], np.object)
+        self.assertEqual(types[2], np.bool)
+        self.assertEqual(types[3], np.float32)
+        self.assertEqual(types[4], np.object)  # datetime.date
+        self.assertEqual(types[5], 'datetime64[ns]')
+
+    @unittest.skipIf(not have_pandas, pandas_requirement_message)  # type: ignore
     def test_to_pandas_with_duplicated_column_names(self):
         import numpy as np
 
@@ -539,10 +540,10 @@ def test_to_pandas_with_duplicated_column_names(self):
                 df = self.spark.sql(sql)
                 pdf = df.toPandas()
                 types = pdf.dtypes
-                self.assertEquals(types.iloc[0], np.int32)
-                self.assertEquals(types.iloc[1], np.int32)
+                self.assertEqual(types.iloc[0], np.int32)
+                self.assertEqual(types.iloc[1], np.int32)
 
-    @unittest.skipIf(not have_pandas, pandas_requirement_message)
+    @unittest.skipIf(not have_pandas, pandas_requirement_message)  # type: ignore
     def test_to_pandas_on_cross_join(self):
         import numpy as np
 
@@ -559,16 +560,16 @@ def test_to_pandas_on_cross_join(self):
                 df = self.spark.sql(sql)
                 pdf = df.toPandas()
                 types = pdf.dtypes
-                self.assertEquals(types.iloc[0], np.int32)
-                self.assertEquals(types.iloc[1], np.int32)
+                self.assertEqual(types.iloc[0], np.int32)
+                self.assertEqual(types.iloc[1], np.int32)
 
     @unittest.skipIf(have_pandas, "Required Pandas was found.")
     def test_to_pandas_required_pandas_not_found(self):
         with QuietTest(self.sc):
-            with self.assertRaisesRegexp(ImportError, 'Pandas >= .* must be installed'):
+            with self.assertRaisesRegex(ImportError, 'Pandas >= .* must be installed'):
                 self._to_pandas()
 
-    @unittest.skipIf(not have_pandas, pandas_requirement_message)
+    @unittest.skipIf(not have_pandas, pandas_requirement_message)  # type: ignore
     def test_to_pandas_avoid_astype(self):
         import numpy as np
         schema = StructType().add("a", IntegerType()).add("b", StringType())\
@@ -576,11 +577,11 @@ def test_to_pandas_avoid_astype(self):
         data = [(1, "foo", 16777220), (None, "bar", None)]
         df = self.spark.createDataFrame(data, schema)
         types = df.toPandas().dtypes
-        self.assertEquals(types[0], np.float64)  # doesn't convert to np.int32 due to NaN value.
-        self.assertEquals(types[1], np.object)
-        self.assertEquals(types[2], np.float64)
+        self.assertEqual(types[0], np.float64)  # doesn't convert to np.int32 due to NaN value.
+        self.assertEqual(types[1], np.object)
+        self.assertEqual(types[2], np.float64)
 
-    @unittest.skipIf(not have_pandas, pandas_requirement_message)
+    @unittest.skipIf(not have_pandas, pandas_requirement_message)  # type: ignore
     def test_to_pandas_from_empty_dataframe(self):
         with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": False}):
             # SPARK-29188 test that toPandas() on an empty dataframe has the correct dtypes
@@ -600,7 +601,7 @@ def test_to_pandas_from_empty_dataframe(self):
             dtypes_when_empty_df = self.spark.sql(sql).filter("False").toPandas().dtypes
             self.assertTrue(np.all(dtypes_when_empty_df == dtypes_when_nonempty_df))
 
-    @unittest.skipIf(not have_pandas, pandas_requirement_message)
+    @unittest.skipIf(not have_pandas, pandas_requirement_message)  # type: ignore
     def test_to_pandas_from_null_dataframe(self):
         with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": False}):
             # SPARK-29188 test that toPandas() on a dataframe with only nulls has correct dtypes
@@ -628,7 +629,7 @@ def test_to_pandas_from_null_dataframe(self):
             self.assertEqual(types[7], np.object)
             self.assertTrue(np.can_cast(np.datetime64, types[8]))
 
-    @unittest.skipIf(not have_pandas, pandas_requirement_message)
+    @unittest.skipIf(not have_pandas, pandas_requirement_message)  # type: ignore
     def test_to_pandas_from_mixed_dataframe(self):
         with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": False}):
             # SPARK-29188 test that toPandas() on a dataframe with some nulls has correct dtypes
@@ -642,7 +643,7 @@ def test_to_pandas_from_mixed_dataframe(self):
             CAST(col6 AS DOUBLE) AS double,
             CAST(col7 AS BOOLEAN) AS boolean,
             CAST(col8 AS STRING) AS string,
-            CAST(col9 AS TIMESTAMP) AS timestamp
+            timestamp_seconds(col9) AS timestamp
             FROM VALUES (1, 1, 1, 1, 1, 1, 1, 1, 1),
                         (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)
             """
@@ -656,7 +657,7 @@ def test_create_dataframe_from_array_of_long(self):
         df = self.spark.createDataFrame(data)
         self.assertEqual(df.first(), Row(longarray=[-9223372036854775808, 0, 9223372036854775807]))
 
-    @unittest.skipIf(not have_pandas, pandas_requirement_message)
+    @unittest.skipIf(not have_pandas, pandas_requirement_message)  # type: ignore
     def test_create_dataframe_from_pandas_with_timestamp(self):
         import pandas as pd
         from datetime import datetime
@@ -674,7 +675,7 @@ def test_create_dataframe_from_pandas_with_timestamp(self):
     @unittest.skipIf(have_pandas, "Required Pandas was found.")
     def test_create_dataframe_required_pandas_not_found(self):
         with QuietTest(self.sc):
-            with self.assertRaisesRegexp(
+            with self.assertRaisesRegex(
                     ImportError,
                     "(Pandas >= .* must be installed|No module named '?pandas'?)"):
                 import pandas as pd
@@ -684,10 +685,10 @@ def test_create_dataframe_required_pandas_not_found(self):
                 self.spark.createDataFrame(pdf)
 
     # Regression test for SPARK-23360
-    @unittest.skipIf(not have_pandas, pandas_requirement_message)
+    @unittest.skipIf(not have_pandas, pandas_requirement_message)  # type: ignore
     def test_create_dataframe_from_pandas_with_dst(self):
         import pandas as pd
-        from pandas.util.testing import assert_frame_equal
+        from pandas.testing import assert_frame_equal
         from datetime import datetime
 
         pdf = pd.DataFrame({'time': [datetime(2015, 10, 31, 22, 30)]})
@@ -723,7 +724,7 @@ def test_repr_behaviors(self):
                 ||22222|22222|
                 |+-----+-----+
                 |"""
-            self.assertEquals(re.sub(pattern, '', expected1), df.__repr__())
+            self.assertEqual(re.sub(pattern, '', expected1), df.__repr__())
             with self.sql_conf({"spark.sql.repl.eagerEval.truncate": 3}):
                 expected2 = """+---+-----+
                 ||key|value|
@@ -732,7 +733,7 @@ def test_repr_behaviors(self):
                 ||222|  222|
                 |+---+-----+
                 |"""
-                self.assertEquals(re.sub(pattern, '', expected2), df.__repr__())
+                self.assertEqual(re.sub(pattern, '', expected2), df.__repr__())
                 with self.sql_conf({"spark.sql.repl.eagerEval.maxNumRows": 1}):
                     expected3 = """+---+-----+
                     ||key|value|
@@ -741,7 +742,7 @@ def test_repr_behaviors(self):
                     |+---+-----+
                     |only showing top 1 row
                     |"""
-                    self.assertEquals(re.sub(pattern, '', expected3), df.__repr__())
+                    self.assertEqual(re.sub(pattern, '', expected3), df.__repr__())
 
         # test when eager evaluation is enabled and _repr_html_ will be called
         with self.sql_conf({"spark.sql.repl.eagerEval.enabled": True}):
@@ -751,7 +752,7 @@ def test_repr_behaviors(self):
                 |<tr><td>22222</td><td>22222</td></tr>
                 |</table>
                 |"""
-            self.assertEquals(re.sub(pattern, '', expected1), df._repr_html_())
+            self.assertEqual(re.sub(pattern, '', expected1), df._repr_html_())
             with self.sql_conf({"spark.sql.repl.eagerEval.truncate": 3}):
                 expected2 = """<table border='1'>
                     |<tr><th>key</th><th>value</th></tr>
@@ -759,7 +760,7 @@ def test_repr_behaviors(self):
                     |<tr><td>222</td><td>222</td></tr>
                     |</table>
                     |"""
-                self.assertEquals(re.sub(pattern, '', expected2), df._repr_html_())
+                self.assertEqual(re.sub(pattern, '', expected2), df._repr_html_())
                 with self.sql_conf({"spark.sql.repl.eagerEval.maxNumRows": 1}):
                     expected3 = """<table border='1'>
                         |<tr><th>key</th><th>value</th></tr>
@@ -767,19 +768,19 @@ def test_repr_behaviors(self):
                         |</table>
                         |only showing top 1 row
                         |"""
-                    self.assertEquals(re.sub(pattern, '', expected3), df._repr_html_())
+                    self.assertEqual(re.sub(pattern, '', expected3), df._repr_html_())
 
         # test when eager evaluation is disabled and _repr_html_ will be called
         with self.sql_conf({"spark.sql.repl.eagerEval.enabled": False}):
             expected = "DataFrame[key: bigint, value: string]"
-            self.assertEquals(None, df._repr_html_())
-            self.assertEquals(expected, df.__repr__())
+            self.assertEqual(None, df._repr_html_())
+            self.assertEqual(expected, df.__repr__())
             with self.sql_conf({"spark.sql.repl.eagerEval.truncate": 3}):
-                self.assertEquals(None, df._repr_html_())
-                self.assertEquals(expected, df.__repr__())
+                self.assertEqual(None, df._repr_html_())
+                self.assertEqual(expected, df.__repr__())
                 with self.sql_conf({"spark.sql.repl.eagerEval.maxNumRows": 1}):
-                    self.assertEquals(None, df._repr_html_())
-                    self.assertEquals(expected, df.__repr__())
+                    self.assertEqual(None, df._repr_html_())
+                    self.assertEqual(expected, df.__repr__())
 
     def test_to_local_iterator(self):
         df = self.spark.range(8, numPartitions=4)
@@ -817,9 +818,25 @@ def test_to_local_iterator_not_fully_consumed(self):
 
     def test_same_semantics_error(self):
         with QuietTest(self.sc):
-            with self.assertRaisesRegexp(ValueError, "should be of DataFrame.*int"):
+            with self.assertRaisesRegex(TypeError, "should be of DataFrame.*int"):
                 self.spark.range(10).sameSemantics(1)
 
+    def test_input_files(self):
+        tpath = tempfile.mkdtemp()
+        shutil.rmtree(tpath)
+        try:
+            self.spark.range(1, 100, 1, 10).write.parquet(tpath)
+            # read parquet file and get the input files list
+            input_files_list = self.spark.read.parquet(tpath).inputFiles()
+
+            # input files list should contain 10 entries
+            self.assertEqual(len(input_files_list), 10)
+            # all file paths in list must contain tpath
+            for file_path in input_files_list:
+                self.assertTrue(tpath in file_path)
+        finally:
+            shutil.rmtree(tpath)
+
 
 class QueryExecutionListenerTests(unittest.TestCase, SQLTestUtils):
     # These tests are separate because it uses 'spark.sql.queryExecutionListeners' which is
@@ -872,7 +889,7 @@ def test_query_execution_listener_on_collect(self):
 
     @unittest.skipIf(
         not have_pandas or not have_pyarrow,
-        pandas_requirement_message or pyarrow_requirement_message)
+        pandas_requirement_message or pyarrow_requirement_message)  # type: ignore
     def test_query_execution_listener_on_collect_with_arrow(self):
         with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": True}):
             self.assertFalse(
@@ -887,10 +904,10 @@ def test_query_execution_listener_on_collect_with_arrow(self):
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_dataframe import *
+    from pyspark.sql.tests.test_dataframe import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/sql/tests/test_datasources.py b/python/pyspark/sql/tests/test_datasources.py
index a2e73ca610ec6..26a6c58dbad6b 100644
--- a/python/pyspark/sql/tests/test_datasources.py
+++ b/python/pyspark/sql/tests/test_datasources.py
@@ -19,7 +19,7 @@
 import tempfile
 
 from pyspark.sql import Row
-from pyspark.sql.types import *
+from pyspark.sql.types import IntegerType, StructField, StructType, LongType, StringType
 from pyspark.testing.sqlutils import ReusedSQLTestCase
 
 
@@ -107,7 +107,7 @@ def test_read_text_file_list(self):
         df = self.spark.read.text(['python/test_support/sql/text-test.txt',
                                    'python/test_support/sql/text-test.txt'])
         count = df.count()
-        self.assertEquals(count, 4)
+        self.assertEqual(count, 4)
 
     def test_json_sampling_ratio(self):
         rdd = self.spark.sparkContext.range(0, 100, 1, 1) \
@@ -115,14 +115,14 @@ def test_json_sampling_ratio(self):
         schema = self.spark.read.option('inferSchema', True) \
             .option('samplingRatio', 0.5) \
             .json(rdd).schema
-        self.assertEquals(schema, StructType([StructField("a", LongType(), True)]))
+        self.assertEqual(schema, StructType([StructField("a", LongType(), True)]))
 
     def test_csv_sampling_ratio(self):
         rdd = self.spark.sparkContext.range(0, 100, 1, 1) \
             .map(lambda x: '0.1' if x == 1 else str(x))
         schema = self.spark.read.option('inferSchema', True)\
             .csv(rdd, samplingRatio=0.5).schema
-        self.assertEquals(schema, StructType([StructField("_c0", IntegerType(), True)]))
+        self.assertEqual(schema, StructType([StructField("_c0", IntegerType(), True)]))
 
     def test_checking_csv_header(self):
         path = tempfile.mkdtemp()
@@ -135,7 +135,7 @@ def test_checking_csv_header(self):
                 StructField('f1', IntegerType(), nullable=True)])
             df = self.spark.read.option('header', 'true').schema(schema)\
                 .csv(path, enforceSchema=False)
-            self.assertRaisesRegexp(
+            self.assertRaisesRegex(
                 Exception,
                 "CSV header does not conform to the schema",
                 lambda: df.collect())
@@ -154,17 +154,17 @@ def test_ignore_column_of_all_nulls(self):
                 StructField('b', LongType(), nullable=True),
                 StructField('c', StringType(), nullable=True)])
             readback = self.spark.read.json(path, dropFieldIfAllNull=True)
-            self.assertEquals(readback.schema, schema)
+            self.assertEqual(readback.schema, schema)
         finally:
             shutil.rmtree(path)
 
 
 if __name__ == "__main__":
     import unittest
-    from pyspark.sql.tests.test_datasources import *
+    from pyspark.sql.tests.test_datasources import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/sql/tests/test_functions.py b/python/pyspark/sql/tests/test_functions.py
index 52ae74df5d4f2..082d61b732429 100644
--- a/python/pyspark/sql/tests/test_functions.py
+++ b/python/pyspark/sql/tests/test_functions.py
@@ -16,12 +16,14 @@
 #
 
 import datetime
-import sys
 from itertools import chain
 import re
 
-from pyspark.sql import Row
-from pyspark.sql.functions import udf, input_file_name, col, percentile_approx, lit
+from py4j.protocol import Py4JJavaError
+from pyspark.sql import Row, Window
+from pyspark.sql.functions import udf, input_file_name, col, percentile_approx, \
+    lit, assert_true, sum_distinct, sumDistinct, shiftleft, shiftLeft, shiftRight, \
+    shiftright, shiftrightunsigned, shiftRightUnsigned
 from pyspark.testing.sqlutils import ReusedSQLTestCase
 
 
@@ -116,6 +118,7 @@ def assert_close(a, b):
             c = get_values(b)
             diff = [abs(v - c[k]) < 1e-6 for k, v in enumerate(a)]
             return sum(diff) == len(a)
+
         assert_close([math.cos(i) for i in range(10)],
                      df.select(functions.cos(df.a)).collect())
         assert_close([math.cos(i) for i in range(10)],
@@ -139,6 +142,21 @@ def assert_close(a, b):
         assert_close([math.hypot(i, 2) for i in range(10)],
                      df.select(functions.hypot(df.a, 2)).collect())
 
+    def test_inverse_trig_functions(self):
+        from pyspark.sql import functions
+
+        funs = [
+            (functions.acosh, "ACOSH"),
+            (functions.asinh, "ASINH"),
+            (functions.atanh, "ATANH"),
+        ]
+
+        cols = ["a", functions.col("a")]
+
+        for f, alias in funs:
+            for c in cols:
+                self.assertIn(f"{alias}(a)", repr(f(c)))
+
     def test_rand_functions(self):
         df = self.df
         from pyspark.sql import functions
@@ -161,18 +179,20 @@ def test_rand_functions(self):
 
     def test_string_functions(self):
         from pyspark.sql import functions
-        from pyspark.sql.functions import col, lit, _string_functions
+        from pyspark.sql.functions import col, lit
+        string_functions = [
+            "upper", "lower", "ascii",
+            "base64", "unbase64",
+            "ltrim", "rtrim", "trim"
+        ]
+
         df = self.spark.createDataFrame([['nick']], schema=['name'])
-        self.assertRaisesRegexp(
+        self.assertRaisesRegex(
             TypeError,
             "must be the same type",
             lambda: df.select(col('name').substr(0, lit(1))))
-        if sys.version_info.major == 2:
-            self.assertRaises(
-                TypeError,
-                lambda: df.select(col('name').substr(long(0), long(1))))
 
-        for name in _string_functions.keys():
+        for name in string_functions:
             self.assertEqual(
                 df.select(getattr(functions, name)("name")).first()[0],
                 df.select(getattr(functions, name)(col("name"))).first()[0])
@@ -259,25 +279,60 @@ def test_approxQuantile(self):
         self.assertTrue(isinstance(aqt[1], list))
         self.assertEqual(len(aqt[1]), 3)
         self.assertTrue(all(isinstance(q, float) for q in aqt[1]))
-        self.assertRaises(ValueError, lambda: df.stat.approxQuantile(123, [0.1, 0.9], 0.1))
-        self.assertRaises(ValueError, lambda: df.stat.approxQuantile(("a", 123), [0.1, 0.9], 0.1))
-        self.assertRaises(ValueError, lambda: df.stat.approxQuantile(["a", 123], [0.1, 0.9], 0.1))
+        self.assertRaises(TypeError, lambda: df.stat.approxQuantile(123, [0.1, 0.9], 0.1))
+        self.assertRaises(TypeError, lambda: df.stat.approxQuantile(("a", 123), [0.1, 0.9], 0.1))
+        self.assertRaises(TypeError, lambda: df.stat.approxQuantile(["a", 123], [0.1, 0.9], 0.1))
+
+    def test_sorting_functions_with_column(self):
+        from pyspark.sql import functions
+        from pyspark.sql.column import Column
+
+        funs = [
+            functions.asc_nulls_first, functions.asc_nulls_last,
+            functions.desc_nulls_first, functions.desc_nulls_last
+        ]
+        exprs = [col("x"), "x"]
+
+        for fun in funs:
+            for expr in exprs:
+                res = fun(expr)
+                self.assertIsInstance(res, Column)
+                self.assertIn(
+                    f"""'x {fun.__name__.replace("_", " ").upper()}'""",
+                    str(res)
+                )
+
+        for expr in exprs:
+            res = functions.asc(expr)
+            self.assertIsInstance(res, Column)
+            self.assertIn(
+                """'x ASC NULLS FIRST'""",
+                str(res)
+            )
+
+        for expr in exprs:
+            res = functions.desc(expr)
+            self.assertIsInstance(res, Column)
+            self.assertIn(
+                """'x DESC NULLS LAST'""",
+                str(res)
+            )
 
     def test_sort_with_nulls_order(self):
         from pyspark.sql import functions
 
         df = self.spark.createDataFrame(
             [('Tom', 80), (None, 60), ('Alice', 50)], ["name", "height"])
-        self.assertEquals(
+        self.assertEqual(
             df.select(df.name).orderBy(functions.asc_nulls_first('name')).collect(),
             [Row(name=None), Row(name=u'Alice'), Row(name=u'Tom')])
-        self.assertEquals(
+        self.assertEqual(
             df.select(df.name).orderBy(functions.asc_nulls_last('name')).collect(),
             [Row(name=u'Alice'), Row(name=u'Tom'), Row(name=None)])
-        self.assertEquals(
+        self.assertEqual(
             df.select(df.name).orderBy(functions.desc_nulls_first('name')).collect(),
             [Row(name=None), Row(name=u'Tom'), Row(name=u'Alice')])
-        self.assertEquals(
+        self.assertEqual(
             df.select(df.name).orderBy(functions.desc_nulls_last('name')).collect(),
             [Row(name=u'Tom'), Row(name=u'Alice'), Row(name=None)])
 
@@ -296,12 +351,31 @@ def test_input_file_name_reset_for_rdd(self):
         for result in results:
             self.assertEqual(result[0], '')
 
+    def test_slice(self):
+        from pyspark.sql.functions import lit, size, slice
+
+        df = self.spark.createDataFrame([([1, 2, 3],), ([4, 5],)], ['x'])
+
+        self.assertEqual(
+            df.select(slice(df.x, 2, 2).alias("sliced")).collect(),
+            df.select(slice(df.x, lit(2), lit(2)).alias("sliced")).collect(),
+        )
+
+        self.assertEqual(
+            df.select(slice(df.x, size(df.x) - 1, lit(1)).alias("sliced")).collect(),
+            [Row(sliced=[2]), Row(sliced=[4])]
+        )
+        self.assertEqual(
+            df.select(slice(df.x, lit(1), size(df.x) - 1).alias("sliced")).collect(),
+            [Row(sliced=[1, 2]), Row(sliced=[4])]
+        )
+
     def test_array_repeat(self):
         from pyspark.sql.functions import array_repeat, lit
 
         df = self.spark.range(1)
 
-        self.assertEquals(
+        self.assertEqual(
             df.select(array_repeat("id", 3)).toDF("val").collect(),
             df.select(array_repeat("id", lit(3))).toDF("val").collect(),
         )
@@ -362,8 +436,42 @@ def test_percentile_approx(self):
 
         self.assertListEqual(actual, expected)
 
+    def test_nth_value(self):
+        from pyspark.sql import Window
+        from pyspark.sql.functions import nth_value
+
+        df = self.spark.createDataFrame([
+            ("a", 0, None),
+            ("a", 1, "x"),
+            ("a", 2, "y"),
+            ("a", 3, "z"),
+            ("a", 4, None),
+            ("b", 1, None),
+            ("b", 2, None)], schema=("key", "order", "value"))
+        w = Window.partitionBy("key").orderBy("order")
+
+        rs = df.select(
+            df.key,
+            df.order,
+            nth_value("value", 2).over(w),
+            nth_value("value", 2, False).over(w),
+            nth_value("value", 2, True).over(w)).collect()
+
+        expected = [
+            ("a", 0, None, None, None),
+            ("a", 1, "x", "x", None),
+            ("a", 2, "x", "x", "y"),
+            ("a", 3, "x", "x", "y"),
+            ("a", 4, "x", "x", "y"),
+            ("b", 1, None, None, None),
+            ("b", 2, None, None, None)
+        ]
+
+        for r, ex in zip(sorted(rs), sorted(expected)):
+            self.assertEqual(tuple(r), ex[:len(r)])
+
     def test_higher_order_function_failures(self):
-        from pyspark.sql.functions import col, exists, transform
+        from pyspark.sql.functions import col, transform
 
         # Should fail with varargs
         with self.assertRaises(ValueError):
@@ -385,13 +493,201 @@ def test_higher_order_function_failures(self):
         with self.assertRaises(ValueError):
             transform(col("foo"), lambda x: 1)
 
+    def test_nested_higher_order_function(self):
+        # SPARK-35382: lambda vars must be resolved properly in nested higher order functions
+        from pyspark.sql.functions import flatten, struct, transform
+
+        df = self.spark.sql("SELECT array(1, 2, 3) as numbers, array('a', 'b', 'c') as letters")
+
+        actual = df.select(flatten(
+            transform(
+                "numbers",
+                lambda number: transform(
+                    "letters",
+                    lambda letter: struct(number.alias("n"), letter.alias("l"))
+                )
+            )
+        )).first()[0]
+
+        expected = [(1, "a"), (1, "b"), (1, "c"),
+                    (2, "a"), (2, "b"), (2, "c"),
+                    (3, "a"), (3, "b"), (3, "c")]
+
+        self.assertEquals(actual, expected)
+
+    def test_window_functions(self):
+        df = self.spark.createDataFrame([(1, "1"), (2, "2"), (1, "2"), (1, "2")], ["key", "value"])
+        w = Window.partitionBy("value").orderBy("key")
+        from pyspark.sql import functions as F
+        sel = df.select(df.value, df.key,
+                        F.max("key").over(w.rowsBetween(0, 1)),
+                        F.min("key").over(w.rowsBetween(0, 1)),
+                        F.count("key").over(w.rowsBetween(float('-inf'), float('inf'))),
+                        F.row_number().over(w),
+                        F.rank().over(w),
+                        F.dense_rank().over(w),
+                        F.ntile(2).over(w))
+        rs = sorted(sel.collect())
+        expected = [
+            ("1", 1, 1, 1, 1, 1, 1, 1, 1),
+            ("2", 1, 1, 1, 3, 1, 1, 1, 1),
+            ("2", 1, 2, 1, 3, 2, 1, 1, 1),
+            ("2", 2, 2, 2, 3, 3, 3, 2, 2)
+        ]
+        for r, ex in zip(rs, expected):
+            self.assertEqual(tuple(r), ex[:len(r)])
+
+    def test_window_functions_without_partitionBy(self):
+        df = self.spark.createDataFrame([(1, "1"), (2, "2"), (1, "2"), (1, "2")], ["key", "value"])
+        w = Window.orderBy("key", df.value)
+        from pyspark.sql import functions as F
+        sel = df.select(df.value, df.key,
+                        F.max("key").over(w.rowsBetween(0, 1)),
+                        F.min("key").over(w.rowsBetween(0, 1)),
+                        F.count("key").over(w.rowsBetween(float('-inf'), float('inf'))),
+                        F.row_number().over(w),
+                        F.rank().over(w),
+                        F.dense_rank().over(w),
+                        F.ntile(2).over(w))
+        rs = sorted(sel.collect())
+        expected = [
+            ("1", 1, 1, 1, 4, 1, 1, 1, 1),
+            ("2", 1, 1, 1, 4, 2, 2, 2, 1),
+            ("2", 1, 2, 1, 4, 3, 2, 2, 2),
+            ("2", 2, 2, 2, 4, 4, 4, 3, 2)
+        ]
+        for r, ex in zip(rs, expected):
+            self.assertEqual(tuple(r), ex[:len(r)])
+
+    def test_window_functions_cumulative_sum(self):
+        df = self.spark.createDataFrame([("one", 1), ("two", 2)], ["key", "value"])
+        from pyspark.sql import functions as F
+
+        # Test cumulative sum
+        sel = df.select(
+            df.key,
+            F.sum(df.value).over(Window.rowsBetween(Window.unboundedPreceding, 0)))
+        rs = sorted(sel.collect())
+        expected = [("one", 1), ("two", 3)]
+        for r, ex in zip(rs, expected):
+            self.assertEqual(tuple(r), ex[:len(r)])
+
+        # Test boundary values less than JVM's Long.MinValue and make sure we don't overflow
+        sel = df.select(
+            df.key,
+            F.sum(df.value).over(Window.rowsBetween(Window.unboundedPreceding - 1, 0)))
+        rs = sorted(sel.collect())
+        expected = [("one", 1), ("two", 3)]
+        for r, ex in zip(rs, expected):
+            self.assertEqual(tuple(r), ex[:len(r)])
+
+        # Test boundary values greater than JVM's Long.MaxValue and make sure we don't overflow
+        frame_end = Window.unboundedFollowing + 1
+        sel = df.select(
+            df.key,
+            F.sum(df.value).over(Window.rowsBetween(Window.currentRow, frame_end)))
+        rs = sorted(sel.collect())
+        expected = [("one", 3), ("two", 2)]
+        for r, ex in zip(rs, expected):
+            self.assertEqual(tuple(r), ex[:len(r)])
+
+    def test_collect_functions(self):
+        df = self.spark.createDataFrame([(1, "1"), (2, "2"), (1, "2"), (1, "2")], ["key", "value"])
+        from pyspark.sql import functions
+
+        self.assertEqual(
+            sorted(df.select(functions.collect_set(df.key).alias('r')).collect()[0].r),
+            [1, 2])
+        self.assertEqual(
+            sorted(df.select(functions.collect_list(df.key).alias('r')).collect()[0].r),
+            [1, 1, 1, 2])
+        self.assertEqual(
+            sorted(df.select(functions.collect_set(df.value).alias('r')).collect()[0].r),
+            ["1", "2"])
+        self.assertEqual(
+            sorted(df.select(functions.collect_list(df.value).alias('r')).collect()[0].r),
+            ["1", "2", "2", "2"])
+
+    def test_datetime_functions(self):
+        from pyspark.sql import functions
+        from datetime import date
+        df = self.spark.range(1).selectExpr("'2017-01-22' as dateCol")
+        parse_result = df.select(functions.to_date(functions.col("dateCol"))).first()
+        self.assertEqual(date(2017, 1, 22), parse_result['to_date(dateCol)'])
+
+    def test_assert_true(self):
+        from pyspark.sql.functions import assert_true
+
+        df = self.spark.range(3)
+
+        self.assertEqual(
+            df.select(assert_true(df.id < 3)).toDF("val").collect(),
+            [Row(val=None), Row(val=None), Row(val=None)],
+        )
+
+        with self.assertRaises(Py4JJavaError) as cm:
+            df.select(assert_true(df.id < 2, 'too big')).toDF("val").collect()
+        self.assertIn("java.lang.RuntimeException", str(cm.exception))
+        self.assertIn("too big", str(cm.exception))
+
+        with self.assertRaises(Py4JJavaError) as cm:
+            df.select(assert_true(df.id < 2, df.id * 1e6)).toDF("val").collect()
+        self.assertIn("java.lang.RuntimeException", str(cm.exception))
+        self.assertIn("2000000", str(cm.exception))
+
+        with self.assertRaises(TypeError) as cm:
+            df.select(assert_true(df.id < 2, 5))
+        self.assertEqual(
+            "errMsg should be a Column or a str, got <class 'int'>",
+            str(cm.exception)
+        )
+
+    def test_raise_error(self):
+        from pyspark.sql.functions import raise_error
+
+        df = self.spark.createDataFrame([Row(id="foobar")])
+
+        with self.assertRaises(Py4JJavaError) as cm:
+            df.select(raise_error(df.id)).collect()
+        self.assertIn("java.lang.RuntimeException", str(cm.exception))
+        self.assertIn("foobar", str(cm.exception))
+
+        with self.assertRaises(Py4JJavaError) as cm:
+            df.select(raise_error("barfoo")).collect()
+        self.assertIn("java.lang.RuntimeException", str(cm.exception))
+        self.assertIn("barfoo", str(cm.exception))
+
+        with self.assertRaises(TypeError) as cm:
+            df.select(raise_error(None))
+        self.assertEqual(
+            "errMsg should be a Column or a str, got <class 'NoneType'>",
+            str(cm.exception)
+        )
+
+    def test_sum_distinct(self):
+        self.spark.range(10).select(
+            assert_true(sum_distinct(col("id")) == sumDistinct(col("id")))).collect()
+
+    def test_shiftleft(self):
+        self.spark.range(10).select(
+            assert_true(shiftLeft(col("id"), 2) == shiftleft(col("id"), 2))).collect()
+
+    def test_shiftright(self):
+        self.spark.range(10).select(
+            assert_true(shiftRight(col("id"), 2) == shiftright(col("id"), 2))).collect()
+
+    def test_shiftrightunsigned(self):
+        self.spark.range(10).select(
+            assert_true(
+                shiftRightUnsigned(col("id"), 2) == shiftrightunsigned(col("id"), 2))).collect()
+
 
 if __name__ == "__main__":
     import unittest
-    from pyspark.sql.tests.test_functions import *
+    from pyspark.sql.tests.test_functions import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/sql/tests/test_group.py b/python/pyspark/sql/tests/test_group.py
index 3261fa1836203..e603334fc8040 100644
--- a/python/pyspark/sql/tests/test_group.py
+++ b/python/pyspark/sql/tests/test_group.py
@@ -31,15 +31,16 @@ def test_aggregator(self):
         self.assertEqual((0, u'99'),
                          tuple(g.agg(functions.first(df.key), functions.last(df.value)).first()))
         self.assertTrue(95 < g.agg(functions.approx_count_distinct(df.key)).first()[0])
+        # test deprecated countDistinct
         self.assertEqual(100, g.agg(functions.countDistinct(df.value)).first()[0])
 
 
 if __name__ == "__main__":
     import unittest
-    from pyspark.sql.tests.test_group import *
+    from pyspark.sql.tests.test_group import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/sql/tests/test_pandas_cogrouped_map.py b/python/pyspark/sql/tests/test_pandas_cogrouped_map.py
index 3ed9d2ac62fd3..94a12bfb3f656 100644
--- a/python/pyspark/sql/tests/test_pandas_cogrouped_map.py
+++ b/python/pyspark/sql/tests/test_pandas_cogrouped_map.py
@@ -16,30 +16,24 @@
 #
 
 import unittest
-import sys
 
-from pyspark.sql.functions import array, explode, col, lit, udf, sum, pandas_udf, PandasUDFType
-from pyspark.sql.types import DoubleType, StructType, StructField
+from pyspark.sql.functions import array, explode, col, lit, udf, pandas_udf
+from pyspark.sql.types import DoubleType, StructType, StructField, Row
 from pyspark.testing.sqlutils import ReusedSQLTestCase, have_pandas, have_pyarrow, \
     pandas_requirement_message, pyarrow_requirement_message
 from pyspark.testing.utils import QuietTest
 
 if have_pandas:
     import pandas as pd
-    from pandas.util.testing import assert_frame_equal, assert_series_equal
+    from pandas.testing import assert_frame_equal
 
 if have_pyarrow:
-    import pyarrow as pa
-
-
-# Tests below use pd.DataFrame.assign that will infer mixed types (unicode/str) for column names
-# From kwargs w/ Python 2, so need to set check_column_type=False and avoid this check
-_check_column_type = sys.version >= '3'
+    import pyarrow as pa  # noqa: F401
 
 
 @unittest.skipIf(
     not have_pandas or not have_pyarrow,
-    pandas_requirement_message or pyarrow_requirement_message)
+    pandas_requirement_message or pyarrow_requirement_message)  # type: ignore[arg-type]
 class CogroupedMapInPandasTests(ReusedSQLTestCase):
 
     @property
@@ -109,7 +103,7 @@ def merge_pandas(l, r):
             'v2': [90, 100, 110]
         })
 
-        assert_frame_equal(expected, result, check_column_type=_check_column_type)
+        assert_frame_equal(expected, result)
 
     def test_empty_group_by(self):
         left = self.data1
@@ -130,7 +124,7 @@ def merge_pandas(l, r):
             .merge(left, right, on=['id', 'k']) \
             .sort_values(by=['id', 'k'])
 
-        assert_frame_equal(expected, result, check_column_type=_check_column_type)
+        assert_frame_equal(expected, result)
 
     def test_mixed_scalar_udfs_followed_by_cogrouby_apply(self):
         df = self.spark.range(0, 10).toDF('v1')
@@ -141,8 +135,8 @@ def test_mixed_scalar_udfs_followed_by_cogrouby_apply(self):
             .applyInPandas(lambda x, y: pd.DataFrame([(x.sum().sum(), y.sum().sum())]),
                            'sum1 int, sum2 int').collect()
 
-        self.assertEquals(result[0]['sum1'], 165)
-        self.assertEquals(result[0]['sum2'], 165)
+        self.assertEqual(result[0]['sum1'], 165)
+        self.assertEqual(result[0]['sum2'], 165)
 
     def test_with_key_left(self):
         self._test_with_key(self.data1, self.data1, isLeft=True)
@@ -173,26 +167,54 @@ def left_assign_key(key, l, _):
         expected = self.data1.toPandas()
         expected = expected.assign(key=expected.id % 2 == 0)
 
-        assert_frame_equal(expected, result, check_column_type=_check_column_type)
+        assert_frame_equal(expected, result)
 
     def test_wrong_return_type(self):
         # Test that we get a sensible exception invalid values passed to apply
         left = self.data1
         right = self.data2
         with QuietTest(self.sc):
-            with self.assertRaisesRegexp(
+            with self.assertRaisesRegex(
                     NotImplementedError,
-                    'Invalid return type.*MapType'):
+                    'Invalid return type.*ArrayType.*TimestampType'):
                 left.groupby('id').cogroup(right.groupby('id')).applyInPandas(
-                    lambda l, r: l, 'id long, v map<int, int>')
+                    lambda l, r: l, 'id long, v array<timestamp>')
 
     def test_wrong_args(self):
         left = self.data1
         right = self.data2
-        with self.assertRaisesRegexp(ValueError, 'Invalid function'):
+        with self.assertRaisesRegex(ValueError, 'Invalid function'):
             left.groupby('id').cogroup(right.groupby('id')) \
                 .applyInPandas(lambda: 1, StructType([StructField("d", DoubleType())]))
 
+    def test_case_insensitive_grouping_column(self):
+        # SPARK-31915: case-insensitive grouping column should work.
+        df1 = self.spark.createDataFrame([(1, 1)], ("column", "value"))
+
+        row = df1.groupby("ColUmn").cogroup(
+            df1.groupby("COLUMN")
+        ).applyInPandas(lambda r, l: r + l, "column long, value long").first()
+        self.assertEqual(row.asDict(), Row(column=2, value=2).asDict())
+
+        df2 = self.spark.createDataFrame([(1, 1)], ("column", "value"))
+
+        row = df1.groupby("ColUmn").cogroup(
+            df2.groupby("COLUMN")
+        ).applyInPandas(lambda r, l: r + l, "column long, value long").first()
+        self.assertEqual(row.asDict(), Row(column=2, value=2).asDict())
+
+    def test_self_join(self):
+        # SPARK-34319: self-join with FlatMapCoGroupsInPandas
+        df = self.spark.createDataFrame([(1, 1)], ("column", "value"))
+
+        row = df.groupby("ColUmn").cogroup(
+            df.groupby("COLUMN")
+        ).applyInPandas(lambda r, l: r + l, "column long, value long")
+
+        row = row.join(row).first()
+
+        self.assertEqual(row.asDict(), Row(column=2, value=2).asDict())
+
     @staticmethod
     def _test_with_key(left, right, isLeft):
 
@@ -208,7 +230,7 @@ def right_assign_key(key, l, r):
         expected = left.toPandas() if isLeft else right.toPandas()
         expected = expected.assign(key=expected.id)
 
-        assert_frame_equal(expected, result, check_column_type=_check_column_type)
+        assert_frame_equal(expected, result)
 
     @staticmethod
     def _test_merge(left, right, output_schema='id long, k int, v int, v2 int'):
@@ -230,14 +252,14 @@ def merge_pandas(l, r):
             .merge(left, right, on=['id', 'k']) \
             .sort_values(by=['id', 'k'])
 
-        assert_frame_equal(expected, result, check_column_type=_check_column_type)
+        assert_frame_equal(expected, result)
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_pandas_cogrouped_map import *
+    from pyspark.sql.tests.test_pandas_cogrouped_map import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/sql/tests/test_pandas_grouped_map.py b/python/pyspark/sql/tests/test_pandas_grouped_map.py
index ff53a0c6f2cf2..9216600d91e8d 100644
--- a/python/pyspark/sql/tests/test_pandas_grouped_map.py
+++ b/python/pyspark/sql/tests/test_pandas_grouped_map.py
@@ -17,7 +17,6 @@
 
 import datetime
 import unittest
-import sys
 
 from collections import OrderedDict
 from decimal import Decimal
@@ -25,27 +24,24 @@
 from pyspark.sql import Row
 from pyspark.sql.functions import array, explode, col, lit, udf, sum, pandas_udf, PandasUDFType, \
     window
-from pyspark.sql.types import *
+from pyspark.sql.types import IntegerType, DoubleType, ArrayType, BinaryType, ByteType, \
+    LongType, DecimalType, ShortType, FloatType, StringType, BooleanType, StructType, \
+    StructField, NullType, TimestampType
 from pyspark.testing.sqlutils import ReusedSQLTestCase, have_pandas, have_pyarrow, \
     pandas_requirement_message, pyarrow_requirement_message
 from pyspark.testing.utils import QuietTest
 
 if have_pandas:
     import pandas as pd
-    from pandas.util.testing import assert_frame_equal
+    from pandas.testing import assert_frame_equal
 
 if have_pyarrow:
-    import pyarrow as pa
-
-
-# Tests below use pd.DataFrame.assign that will infer mixed types (unicode/str) for column names
-# from kwargs w/ Python 2, so need to set check_column_type=False and avoid this check
-_check_column_type = sys.version >= '3'
+    import pyarrow as pa  # noqa: F401
 
 
 @unittest.skipIf(
     not have_pandas or not have_pyarrow,
-    pandas_requirement_message or pyarrow_requirement_message)
+    pandas_requirement_message or pyarrow_requirement_message)  # type: ignore[arg-type]
 class GroupedMapInPandasTests(ReusedSQLTestCase):
 
     @property
@@ -61,14 +57,15 @@ def test_supported_types(self):
             4, 5, 1.1,
             2.2, Decimal(1.123),
             [1, 2, 2], True, 'hello',
-            bytearray([0x01, 0x02])
+            bytearray([0x01, 0x02]),
+            None
         ]
         output_fields = [
             ('id', IntegerType()), ('byte', ByteType()), ('short', ShortType()),
             ('int', IntegerType()), ('long', LongType()), ('float', FloatType()),
             ('double', DoubleType()), ('decim', DecimalType(10, 3)),
             ('array', ArrayType(IntegerType())), ('bool', BooleanType()), ('str', StringType()),
-            ('bin', BinaryType())
+            ('bin', BinaryType()), ('null', NullType())
         ]
 
         output_schema = StructType([StructField(*x) for x in output_fields])
@@ -87,7 +84,8 @@ def test_supported_types(self):
                 bool=False if pdf.bool else True,
                 str=pdf.str + 'there',
                 array=pdf.array,
-                bin=pdf.bin
+                bin=pdf.bin,
+                null=pdf.null
             ),
             output_schema,
             PandasUDFType.GROUPED_MAP
@@ -105,7 +103,8 @@ def test_supported_types(self):
                 bool=False if pdf.bool else True,
                 str=pdf.str + 'there',
                 array=pdf.array,
-                bin=pdf.bin
+                bin=pdf.bin,
+                null=pdf.null
             ),
             output_schema,
             PandasUDFType.GROUPED_MAP
@@ -124,7 +123,8 @@ def test_supported_types(self):
                 bool=False if pdf.bool else True,
                 str=pdf.str + 'there',
                 array=pdf.array,
-                bin=pdf.bin
+                bin=pdf.bin,
+                null=pdf.null
             ),
             output_schema,
             PandasUDFType.GROUPED_MAP
@@ -139,9 +139,9 @@ def test_supported_types(self):
         result3 = df.groupby('id').apply(udf3).sort('id').toPandas()
         expected3 = expected1
 
-        assert_frame_equal(expected1, result1, check_column_type=_check_column_type)
-        assert_frame_equal(expected2, result2, check_column_type=_check_column_type)
-        assert_frame_equal(expected3, result3, check_column_type=_check_column_type)
+        assert_frame_equal(expected1, result1)
+        assert_frame_equal(expected2, result2)
+        assert_frame_equal(expected3, result3)
 
     def test_array_type_correct(self):
         df = self.data.withColumn("arr", array(col("id"))).repartition(1, "id")
@@ -159,12 +159,12 @@ def test_array_type_correct(self):
 
         result = df.groupby('id').apply(udf).sort('id').toPandas()
         expected = df.toPandas().groupby('id').apply(udf.func).reset_index(drop=True)
-        assert_frame_equal(expected, result, check_column_type=_check_column_type)
+        assert_frame_equal(expected, result)
 
     def test_register_grouped_map_udf(self):
         foo_udf = pandas_udf(lambda x: x, "id long", PandasUDFType.GROUPED_MAP)
         with QuietTest(self.sc):
-            with self.assertRaisesRegexp(
+            with self.assertRaisesRegex(
                     ValueError,
                     'f.*SQL_BATCHED_UDF.*SQL_SCALAR_PANDAS_UDF.*SQL_GROUPED_AGG_PANDAS_UDF.*'):
                 self.spark.catalog.registerFunction("foo_udf", foo_udf)
@@ -181,7 +181,7 @@ def foo(pdf):
 
         result = df.groupby('id').apply(foo).sort('id').toPandas()
         expected = df.toPandas().groupby('id').apply(foo.func).reset_index(drop=True)
-        assert_frame_equal(expected, result, check_column_type=_check_column_type)
+        assert_frame_equal(expected, result)
 
     def test_coerce(self):
         df = self.data
@@ -195,7 +195,7 @@ def test_coerce(self):
         result = df.groupby('id').apply(foo).sort('id').toPandas()
         expected = df.toPandas().groupby('id').apply(foo.func).reset_index(drop=True)
         expected = expected.assign(v=expected.v.astype('float64'))
-        assert_frame_equal(expected, result, check_column_type=_check_column_type)
+        assert_frame_equal(expected, result)
 
     def test_complex_groupby(self):
         df = self.data
@@ -213,7 +213,7 @@ def normalize(pdf):
         expected = pdf.groupby(pdf['id'] % 2 == 0, as_index=False).apply(normalize.func)
         expected = expected.sort_values(['id', 'v']).reset_index(drop=True)
         expected = expected.assign(norm=expected.norm.astype('float64'))
-        assert_frame_equal(expected, result, check_column_type=_check_column_type)
+        assert_frame_equal(expected, result)
 
     def test_empty_groupby(self):
         df = self.data
@@ -231,7 +231,7 @@ def normalize(pdf):
         expected = normalize.func(pdf)
         expected = expected.sort_values(['id', 'v']).reset_index(drop=True)
         expected = expected.assign(norm=expected.norm.astype('float64'))
-        assert_frame_equal(expected, result, check_column_type=_check_column_type)
+        assert_frame_equal(expected, result)
 
     def test_datatype_string(self):
         df = self.data
@@ -244,52 +244,50 @@ def test_datatype_string(self):
 
         result = df.groupby('id').apply(foo_udf).sort('id').toPandas()
         expected = df.toPandas().groupby('id').apply(foo_udf.func).reset_index(drop=True)
-        assert_frame_equal(expected, result, check_column_type=_check_column_type)
+        assert_frame_equal(expected, result)
 
     def test_wrong_return_type(self):
         with QuietTest(self.sc):
-            with self.assertRaisesRegexp(
+            with self.assertRaisesRegex(
                     NotImplementedError,
-                    'Invalid return type.*grouped map Pandas UDF.*MapType'):
+                    'Invalid return type.*grouped map Pandas UDF.*ArrayType.*TimestampType'):
                 pandas_udf(
                     lambda pdf: pdf,
-                    'id long, v map<int, int>',
+                    'id long, v array<timestamp>',
                     PandasUDFType.GROUPED_MAP)
 
     def test_wrong_args(self):
         df = self.data
 
         with QuietTest(self.sc):
-            with self.assertRaisesRegexp(ValueError, 'Invalid udf'):
+            with self.assertRaisesRegex(ValueError, 'Invalid udf'):
                 df.groupby('id').apply(lambda x: x)
-            with self.assertRaisesRegexp(ValueError, 'Invalid udf'):
+            with self.assertRaisesRegex(ValueError, 'Invalid udf'):
                 df.groupby('id').apply(udf(lambda x: x, DoubleType()))
-            with self.assertRaisesRegexp(ValueError, 'Invalid udf'):
+            with self.assertRaisesRegex(ValueError, 'Invalid udf'):
                 df.groupby('id').apply(sum(df.v))
-            with self.assertRaisesRegexp(ValueError, 'Invalid udf'):
+            with self.assertRaisesRegex(ValueError, 'Invalid udf'):
                 df.groupby('id').apply(df.v + 1)
-            with self.assertRaisesRegexp(ValueError, 'Invalid function'):
+            with self.assertRaisesRegex(ValueError, 'Invalid function'):
                 df.groupby('id').apply(
                     pandas_udf(lambda: 1, StructType([StructField("d", DoubleType())])))
-            with self.assertRaisesRegexp(ValueError, 'Invalid udf'):
+            with self.assertRaisesRegex(ValueError, 'Invalid udf'):
                 df.groupby('id').apply(pandas_udf(lambda x, y: x, DoubleType()))
-            with self.assertRaisesRegexp(ValueError, 'Invalid udf.*GROUPED_MAP'):
+            with self.assertRaisesRegex(ValueError, 'Invalid udf.*GROUPED_MAP'):
                 df.groupby('id').apply(
                     pandas_udf(lambda x, y: x, DoubleType(), PandasUDFType.SCALAR))
 
     def test_unsupported_types(self):
         common_err_msg = 'Invalid return type.*grouped map Pandas UDF.*'
         unsupported_types = [
-            StructField('map', MapType(StringType(), IntegerType())),
             StructField('arr_ts', ArrayType(TimestampType())),
-            StructField('null', NullType()),
             StructField('struct', StructType([StructField('l', LongType())])),
         ]
 
         for unsupported_type in unsupported_types:
             schema = StructType([StructField('id', LongType(), True), unsupported_type])
             with QuietTest(self.sc):
-                with self.assertRaisesRegexp(NotImplementedError, common_err_msg):
+                with self.assertRaisesRegex(NotImplementedError, common_err_msg):
                     pandas_udf(lambda x: x, schema, PandasUDFType.GROUPED_MAP)
 
     # Regression test for SPARK-23314
@@ -301,7 +299,7 @@ def test_timestamp_dst(self):
         df = self.spark.createDataFrame(dt, 'timestamp').toDF('time')
         foo_udf = pandas_udf(lambda pdf: pdf, 'time timestamp', PandasUDFType.GROUPED_MAP)
         result = df.groupby('time').apply(foo_udf).sort('time')
-        assert_frame_equal(df.toPandas(), result.toPandas(), check_column_type=_check_column_type)
+        assert_frame_equal(df.toPandas(), result.toPandas())
 
     def test_udf_with_key(self):
         import numpy as np
@@ -355,26 +353,26 @@ def foo3(key, pdf):
         expected1 = pdf.groupby('id', as_index=False)\
             .apply(lambda x: udf1.func((x.id.iloc[0],), x))\
             .sort_values(['id', 'v']).reset_index(drop=True)
-        assert_frame_equal(expected1, result1, check_column_type=_check_column_type)
+        assert_frame_equal(expected1, result1)
 
         # Test groupby expression
         result2 = df.groupby(df.id % 2).apply(udf1).sort('id', 'v').toPandas()
         expected2 = pdf.groupby(pdf.id % 2, as_index=False)\
             .apply(lambda x: udf1.func((x.id.iloc[0] % 2,), x))\
             .sort_values(['id', 'v']).reset_index(drop=True)
-        assert_frame_equal(expected2, result2, check_column_type=_check_column_type)
+        assert_frame_equal(expected2, result2)
 
         # Test complex groupby
         result3 = df.groupby(df.id, df.v % 2).apply(udf2).sort('id', 'v').toPandas()
         expected3 = pdf.groupby([pdf.id, pdf.v % 2], as_index=False)\
             .apply(lambda x: udf2.func((x.id.iloc[0], (x.v % 2).iloc[0],), x))\
             .sort_values(['id', 'v']).reset_index(drop=True)
-        assert_frame_equal(expected3, result3, check_column_type=_check_column_type)
+        assert_frame_equal(expected3, result3)
 
         # Test empty groupby
         result4 = df.groupby().apply(udf3).sort('id', 'v').toPandas()
         expected4 = udf3.func((), pdf)
-        assert_frame_equal(expected4, result4, check_column_type=_check_column_type)
+        assert_frame_equal(expected4, result4)
 
     def test_column_order(self):
 
@@ -407,7 +405,7 @@ def change_col_order(pdf):
             .select('id', 'u', 'v').toPandas()
         pd_result = grouped_pdf.apply(change_col_order)
         expected = pd_result.sort_values(['id', 'v']).reset_index(drop=True)
-        assert_frame_equal(expected, result, check_column_type=_check_column_type)
+        assert_frame_equal(expected, result)
 
         # Function returns a pdf with positional columns, indexed by range
         def range_col_order(pdf):
@@ -426,7 +424,7 @@ def range_col_order(pdf):
         pd_result = grouped_pdf.apply(range_col_order)
         rename_pdf(pd_result, ['id', 'u', 'v'])
         expected = pd_result.sort_values(['id', 'v']).reset_index(drop=True)
-        assert_frame_equal(expected, result, check_column_type=_check_column_type)
+        assert_frame_equal(expected, result)
 
         # Function returns a pdf with columns indexed with integers
         def int_index(pdf):
@@ -444,21 +442,22 @@ def int_index(pdf):
         pd_result = grouped_pdf.apply(int_index)
         rename_pdf(pd_result, ['id', 'u', 'v'])
         expected = pd_result.sort_values(['id', 'v']).reset_index(drop=True)
-        assert_frame_equal(expected, result, check_column_type=_check_column_type)
+        assert_frame_equal(expected, result)
 
         @pandas_udf('id long, v int', PandasUDFType.GROUPED_MAP)
         def column_name_typo(pdf):
             return pd.DataFrame({'iid': pdf.id, 'v': pdf.v})
 
-        @pandas_udf('id long, v int', PandasUDFType.GROUPED_MAP)
+        @pandas_udf('id long, v decimal', PandasUDFType.GROUPED_MAP)
         def invalid_positional_types(pdf):
-            return pd.DataFrame([(u'a', 1.2)])
+            return pd.DataFrame([(1, datetime.date(2020, 10, 5))])
 
-        with QuietTest(self.sc):
-            with self.assertRaisesRegexp(Exception, "KeyError: 'id'"):
-                grouped_df.apply(column_name_typo).collect()
-            with self.assertRaisesRegexp(Exception, "an integer is required"):
-                grouped_df.apply(invalid_positional_types).collect()
+        with self.sql_conf({"spark.sql.execution.pandas.convertToArrowArraySafely": False}):
+            with QuietTest(self.sc):
+                with self.assertRaisesRegex(Exception, "KeyError: 'id'"):
+                    grouped_df.apply(column_name_typo).collect()
+                with self.assertRaisesRegex(Exception, "[D|d]ecimal.*got.*date"):
+                    grouped_df.apply(invalid_positional_types).collect()
 
     def test_positional_assignment_conf(self):
         with self.sql_conf({
@@ -486,9 +485,9 @@ def dummy_pandas_udf(df):
         # this was throwing an AnalysisException before SPARK-24208
         res = df_with_pandas.alias('temp0').join(df_with_pandas.alias('temp1'),
                                                  col('temp0.key') == col('temp1.key'))
-        self.assertEquals(res.count(), 5)
+        self.assertEqual(res.count(), 5)
 
-    def test_mixed_scalar_udfs_followed_by_grouby_apply(self):
+    def test_mixed_scalar_udfs_followed_by_groupby_apply(self):
         df = self.spark.range(0, 10).toDF('v1')
         df = df.withColumn('v2', udf(lambda x: x + 1, 'int')(df['v1'])) \
             .withColumn('v3', pandas_udf(lambda x: x + 2, 'int')(df['v1']))
@@ -498,7 +497,7 @@ def test_mixed_scalar_udfs_followed_by_grouby_apply(self):
                               'sum int',
                               PandasUDFType.GROUPED_MAP))
 
-        self.assertEquals(result.collect()[0]['sum'], 165)
+        self.assertEqual(result.collect()[0]['sum'], 165)
 
     def test_grouped_with_empty_partition(self):
         data = [Row(id=1, x=2), Row(id=1, x=3), Row(id=2, x=4)]
@@ -545,13 +544,13 @@ def f(pdf):
 
     def test_grouped_over_window_with_key(self):
 
-        data = [(0, 1, "2018-03-10T00:00:00+00:00", False),
-                (1, 2, "2018-03-11T00:00:00+00:00", False),
-                (2, 2, "2018-03-12T00:00:00+00:00", False),
-                (3, 3, "2018-03-15T00:00:00+00:00", False),
-                (4, 3, "2018-03-16T00:00:00+00:00", False),
-                (5, 3, "2018-03-17T00:00:00+00:00", False),
-                (6, 3, "2018-03-21T00:00:00+00:00", False)]
+        data = [(0, 1, "2018-03-10T00:00:00+00:00", [0]),
+                (1, 2, "2018-03-11T00:00:00+00:00", [0]),
+                (2, 2, "2018-03-12T00:00:00+00:00", [0]),
+                (3, 3, "2018-03-15T00:00:00+00:00", [0]),
+                (4, 3, "2018-03-16T00:00:00+00:00", [0]),
+                (5, 3, "2018-03-17T00:00:00+00:00", [0]),
+                (6, 3, "2018-03-21T00:00:00+00:00", [0])]
 
         expected_window = [
             {'start': datetime.datetime(2018, 3, 10, 0, 0),
@@ -562,37 +561,60 @@ def test_grouped_over_window_with_key(self):
              'end': datetime.datetime(2018, 3, 25, 0, 0)},
         ]
 
-        expected = {0: (1, expected_window[0]),
-                    1: (2, expected_window[0]),
-                    2: (2, expected_window[0]),
-                    3: (3, expected_window[1]),
-                    4: (3, expected_window[1]),
-                    5: (3, expected_window[1]),
-                    6: (3, expected_window[2])}
+        expected_key = {0: (1, expected_window[0]),
+                        1: (2, expected_window[0]),
+                        2: (2, expected_window[0]),
+                        3: (3, expected_window[1]),
+                        4: (3, expected_window[1]),
+                        5: (3, expected_window[1]),
+                        6: (3, expected_window[2])}
+
+        # id -> array of group with len of num records in window
+        expected = {0: [1],
+                    1: [2, 2],
+                    2: [2, 2],
+                    3: [3, 3, 3],
+                    4: [3, 3, 3],
+                    5: [3, 3, 3],
+                    6: [3]}
 
         df = self.spark.createDataFrame(data, ['id', 'group', 'ts', 'result'])
         df = df.select(col('id'), col('group'), col('ts').cast('timestamp'), col('result'))
 
-        @pandas_udf(df.schema, PandasUDFType.GROUPED_MAP)
         def f(key, pdf):
             group = key[0]
             window_range = key[1]
-            # Result will be True if group and window range equal to expected
-            is_expected = pdf.id.apply(lambda id: (expected[id][0] == group and
-                                                   expected[id][1] == window_range))
-            return pdf.assign(result=is_expected)
 
-        result = df.groupby('group', window('ts', '5 days')).apply(f).select('result').collect()
+            # Make sure the key with group and window values are correct
+            for _, i in pdf.id.iteritems():
+                assert expected_key[i][0] == group, "{} != {}".format(expected_key[i][0], group)
+                assert expected_key[i][1] == window_range, \
+                    "{} != {}".format(expected_key[i][1], window_range)
+
+            return pdf.assign(result=[[group] * len(pdf)] * len(pdf))
+
+        result = df.groupby('group', window('ts', '5 days')).applyInPandas(f, df.schema)\
+            .select('id', 'result').collect()
+
+        for r in result:
+            self.assertListEqual(expected[r[0]], r[1])
+
+    def test_case_insensitive_grouping_column(self):
+        # SPARK-31915: case-insensitive grouping column should work.
+        def my_pandas_udf(pdf):
+            return pdf.assign(score=0.5)
 
-        # Check that all group and window_range values from udf matched expected
-        self.assertTrue(all([r[0] for r in result]))
+        df = self.spark.createDataFrame([[1, 1]], ["column", "score"])
+        row = df.groupby('COLUMN').applyInPandas(
+            my_pandas_udf, schema="column integer, score float").first()
+        self.assertEqual(row.asDict(), Row(column=1, score=0.5).asDict())
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_pandas_grouped_map import *
+    from pyspark.sql.tests.test_pandas_grouped_map import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/sql/tests/test_pandas_map.py b/python/pyspark/sql/tests/test_pandas_map.py
index f1956a2523e48..e8f92de417dda 100644
--- a/python/pyspark/sql/tests/test_pandas_map.py
+++ b/python/pyspark/sql/tests/test_pandas_map.py
@@ -15,14 +15,9 @@
 # limitations under the License.
 #
 import os
-import sys
 import time
 import unittest
 
-if sys.version >= '3':
-    unicode = str
-
-from pyspark.sql.functions import pandas_udf, PandasUDFType
 from pyspark.testing.sqlutils import ReusedSQLTestCase, have_pandas, have_pyarrow, \
     pandas_requirement_message, pyarrow_requirement_message
 
@@ -32,7 +27,7 @@
 
 @unittest.skipIf(
     not have_pandas or not have_pyarrow,
-    pandas_requirement_message or pyarrow_requirement_message)
+    pandas_requirement_message or pyarrow_requirement_message)  # type: ignore[arg-type]
 class MapInPandasTests(ReusedSQLTestCase):
 
     @classmethod
@@ -66,7 +61,7 @@ def func(iterator):
         df = self.spark.range(10)
         actual = df.mapInPandas(func, 'id long').collect()
         expected = df.collect()
-        self.assertEquals(actual, expected)
+        self.assertEqual(actual, expected)
 
     def test_multiple_columns(self):
         data = [(1, "foo"), (2, None), (3, "bar"), (4, "bar")]
@@ -80,7 +75,7 @@ def func(iterator):
 
         actual = df.mapInPandas(func, df.schema).collect()
         expected = df.collect()
-        self.assertEquals(actual, expected)
+        self.assertEqual(actual, expected)
 
     def test_different_output_length(self):
         def func(iterator):
@@ -89,7 +84,7 @@ def func(iterator):
 
         df = self.spark.range(10)
         actual = df.repartition(1).mapInPandas(func, 'a long').collect()
-        self.assertEquals(set((r.a for r in actual)), set(range(100)))
+        self.assertEqual(set((r.a for r in actual)), set(range(100)))
 
     def test_empty_iterator(self):
         def empty_iter(_):
@@ -115,14 +110,22 @@ def func(iterator):
         df = self.spark.range(10)
         actual = df.mapInPandas(func, 'id long').mapInPandas(func, 'id long').collect()
         expected = df.collect()
-        self.assertEquals(actual, expected)
+        self.assertEqual(actual, expected)
+
+    def test_self_join(self):
+        # SPARK-34319: self-join with MapInPandas
+        df1 = self.spark.range(10)
+        df2 = df1.mapInPandas(lambda iter: iter, 'id long')
+        actual = df2.join(df2).collect()
+        expected = df1.join(df1).collect()
+        self.assertEqual(sorted(actual), sorted(expected))
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_pandas_map import *
+    from pyspark.sql.tests.test_pandas_map import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/sql/tests/test_pandas_udf.py b/python/pyspark/sql/tests/test_pandas_udf.py
index 4218f5cfc401f..975eb4680dd04 100644
--- a/python/pyspark/sql/tests/test_pandas_udf.py
+++ b/python/pyspark/sql/tests/test_pandas_udf.py
@@ -18,19 +18,17 @@
 import unittest
 
 from pyspark.sql.functions import udf, pandas_udf, PandasUDFType
-from pyspark.sql.types import *
-from pyspark.sql.utils import ParseException
+from pyspark.sql.types import DoubleType, StructType, StructField, LongType
+from pyspark.sql.utils import ParseException, PythonException
 from pyspark.rdd import PythonEvalType
 from pyspark.testing.sqlutils import ReusedSQLTestCase, have_pandas, have_pyarrow, \
     pandas_requirement_message, pyarrow_requirement_message
 from pyspark.testing.utils import QuietTest
 
-from py4j.protocol import Py4JJavaError
-
 
 @unittest.skipIf(
     not have_pandas or not have_pyarrow,
-    pandas_requirement_message or pyarrow_requirement_message)
+    pandas_requirement_message or pyarrow_requirement_message)  # type: ignore[arg-type]
 class PandasUDFTests(ReusedSQLTestCase):
 
     def test_pandas_udf_basic(self):
@@ -116,31 +114,31 @@ def test_udf_wrong_arg(self):
                 @pandas_udf('blah')
                 def foo(x):
                     return x
-            with self.assertRaisesRegexp(ValueError, 'Invalid return type.*None'):
+            with self.assertRaisesRegex(ValueError, 'Invalid return type.*None'):
                 @pandas_udf(functionType=PandasUDFType.SCALAR)
                 def foo(x):
                     return x
-            with self.assertRaisesRegexp(ValueError, 'Invalid function'):
+            with self.assertRaisesRegex(ValueError, 'Invalid function'):
                 @pandas_udf('double', 100)
                 def foo(x):
                     return x
 
-            with self.assertRaisesRegexp(ValueError, '0-arg pandas_udfs.*not.*supported'):
+            with self.assertRaisesRegex(ValueError, '0-arg pandas_udfs.*not.*supported'):
                 pandas_udf(lambda: 1, LongType(), PandasUDFType.SCALAR)
-            with self.assertRaisesRegexp(ValueError, '0-arg pandas_udfs.*not.*supported'):
+            with self.assertRaisesRegex(ValueError, '0-arg pandas_udfs.*not.*supported'):
                 @pandas_udf(LongType(), PandasUDFType.SCALAR)
                 def zero_with_type():
                     return 1
 
-            with self.assertRaisesRegexp(TypeError, 'Invalid return type'):
+            with self.assertRaisesRegex(TypeError, 'Invalid return type'):
                 @pandas_udf(returnType=PandasUDFType.GROUPED_MAP)
                 def foo(df):
                     return df
-            with self.assertRaisesRegexp(TypeError, 'Invalid return type'):
+            with self.assertRaisesRegex(TypeError, 'Invalid return type'):
                 @pandas_udf(returnType='double', functionType=PandasUDFType.GROUPED_MAP)
                 def foo(df):
                     return df
-            with self.assertRaisesRegexp(ValueError, 'Invalid function'):
+            with self.assertRaisesRegex(ValueError, 'Invalid function'):
                 @pandas_udf(returnType='k int, v double', functionType=PandasUDFType.GROUPED_MAP)
                 def foo(k, v, w):
                     return k
@@ -156,15 +154,15 @@ def foofoo(x, y):
         df = self.spark.range(0, 100)
 
         # plain udf (test for SPARK-23754)
-        self.assertRaisesRegexp(
-            Py4JJavaError,
+        self.assertRaisesRegex(
+            PythonException,
             exc_message,
             df.withColumn('v', udf(foo)('id')).collect
         )
 
         # pandas scalar udf
-        self.assertRaisesRegexp(
-            Py4JJavaError,
+        self.assertRaisesRegex(
+            PythonException,
             exc_message,
             df.withColumn(
                 'v', pandas_udf(foo, 'double', PandasUDFType.SCALAR)('id')
@@ -172,16 +170,16 @@ def foofoo(x, y):
         )
 
         # pandas grouped map
-        self.assertRaisesRegexp(
-            Py4JJavaError,
+        self.assertRaisesRegex(
+            PythonException,
             exc_message,
             df.groupBy('id').apply(
                 pandas_udf(foo, df.schema, PandasUDFType.GROUPED_MAP)
             ).collect
         )
 
-        self.assertRaisesRegexp(
-            Py4JJavaError,
+        self.assertRaisesRegex(
+            PythonException,
             exc_message,
             df.groupBy('id').apply(
                 pandas_udf(foofoo, df.schema, PandasUDFType.GROUPED_MAP)
@@ -189,8 +187,8 @@ def foofoo(x, y):
         )
 
         # pandas grouped agg
-        self.assertRaisesRegexp(
-            Py4JJavaError,
+        self.assertRaisesRegex(
+            PythonException,
             exc_message,
             df.groupBy('id').agg(
                 pandas_udf(foo, 'double', PandasUDFType.GROUPED_AGG)('id')
@@ -212,8 +210,8 @@ def udf(column):
         # Since 0.11.0, PyArrow supports the feature to raise an error for unsafe cast.
         with self.sql_conf({
                 "spark.sql.execution.pandas.convertToArrowArraySafely": True}):
-            with self.assertRaisesRegexp(Exception,
-                                         "Exception thrown when converting pandas.Series"):
+            with self.assertRaisesRegex(Exception,
+                                        "Exception thrown when converting pandas.Series"):
                 df.select(['A']).withColumn('udf', udf('A')).collect()
 
         # Disabling Arrow safe type check.
@@ -233,8 +231,8 @@ def udf(column):
         # When enabling safe type check, Arrow 0.11.0+ disallows overflow cast.
         with self.sql_conf({
                 "spark.sql.execution.pandas.convertToArrowArraySafely": True}):
-            with self.assertRaisesRegexp(Exception,
-                                         "Exception thrown when converting pandas.Series"):
+            with self.assertRaisesRegex(Exception,
+                                        "Exception thrown when converting pandas.Series"):
                 df.withColumn('udf', udf('id')).collect()
 
         # Disabling safe type check, let Arrow do the cast anyway.
@@ -243,10 +241,10 @@ def udf(column):
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_pandas_udf import *
+    from pyspark.sql.tests.test_pandas_udf import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/sql/tests/test_pandas_udf_grouped_agg.py b/python/pyspark/sql/tests/test_pandas_udf_grouped_agg.py
index 224c8ce97f559..0c78855cd9123 100644
--- a/python/pyspark/sql/tests/test_pandas_udf_grouped_agg.py
+++ b/python/pyspark/sql/tests/test_pandas_udf_grouped_agg.py
@@ -21,20 +21,21 @@
 from pyspark.sql import Row
 from pyspark.sql.functions import array, explode, col, lit, mean, sum, \
     udf, pandas_udf, PandasUDFType
-from pyspark.sql.types import *
+from pyspark.sql.types import ArrayType, TimestampType
 from pyspark.sql.utils import AnalysisException
 from pyspark.testing.sqlutils import ReusedSQLTestCase, have_pandas, have_pyarrow, \
     pandas_requirement_message, pyarrow_requirement_message
 from pyspark.testing.utils import QuietTest
 
+
 if have_pandas:
     import pandas as pd
-    from pandas.util.testing import assert_frame_equal
+    from pandas.testing import assert_frame_equal
 
 
 @unittest.skipIf(
     not have_pandas or not have_pyarrow,
-    pandas_requirement_message or pyarrow_requirement_message)
+    pandas_requirement_message or pyarrow_requirement_message)  # type: ignore[arg-type]
 class GroupedAggPandasUDFTests(ReusedSQLTestCase):
 
     @property
@@ -50,7 +51,7 @@ def python_plus_one(self):
         @udf('double')
         def plus_one(v):
             assert isinstance(v, (int, float))
-            return v + 1
+            return float(v + 1)
         return plus_one
 
     @property
@@ -144,21 +145,21 @@ def test_basic(self):
 
     def test_unsupported_types(self):
         with QuietTest(self.sc):
-            with self.assertRaisesRegexp(NotImplementedError, 'not supported'):
+            with self.assertRaisesRegex(NotImplementedError, 'not supported'):
                 pandas_udf(
                     lambda x: x,
                     ArrayType(ArrayType(TimestampType())),
                     PandasUDFType.GROUPED_AGG)
 
         with QuietTest(self.sc):
-            with self.assertRaisesRegexp(NotImplementedError, 'not supported'):
+            with self.assertRaisesRegex(NotImplementedError, 'not supported'):
                 @pandas_udf('mean double, std double', PandasUDFType.GROUPED_AGG)
                 def mean_and_std_udf(v):
                     return v.mean(), v.std()
 
         with QuietTest(self.sc):
-            with self.assertRaisesRegexp(NotImplementedError, 'not supported'):
-                @pandas_udf(MapType(DoubleType(), DoubleType()), PandasUDFType.GROUPED_AGG)
+            with self.assertRaisesRegex(NotImplementedError, 'not supported'):
+                @pandas_udf(ArrayType(TimestampType()), PandasUDFType.GROUPED_AGG)
                 def mean_and_std_udf(v):
                     return {v.mean(): v.std()}
 
@@ -315,16 +316,18 @@ def test_complex_groupby(self):
         expected3 = df.groupby(df.id, df.v % 2).agg(sum(df.v)).orderBy(df.id, df.v % 2)
 
         # groupby one python UDF
-        result4 = df.groupby(plus_one(df.id)).agg(sum_udf(df.v))
-        expected4 = df.groupby(plus_one(df.id)).agg(sum(df.v))
+        result4 = df.groupby(plus_one(df.id)).agg(sum_udf(df.v)).sort('plus_one(id)')
+        expected4 = df.groupby(plus_one(df.id)).agg(sum(df.v)).sort('plus_one(id)')
 
         # groupby one scalar pandas UDF
         result5 = df.groupby(plus_two(df.id)).agg(sum_udf(df.v)).sort('sum(v)')
         expected5 = df.groupby(plus_two(df.id)).agg(sum(df.v)).sort('sum(v)')
 
         # groupby one expression and one python UDF
-        result6 = df.groupby(df.v % 2, plus_one(df.id)).agg(sum_udf(df.v))
-        expected6 = df.groupby(df.v % 2, plus_one(df.id)).agg(sum(df.v))
+        result6 = (df.groupby(df.v % 2, plus_one(df.id))
+                   .agg(sum_udf(df.v)).sort(['(v % 2)', 'plus_one(id)']))
+        expected6 = (df.groupby(df.v % 2, plus_one(df.id))
+                     .agg(sum(df.v)).sort(['(v % 2)', 'plus_one(id)']))
 
         # groupby one expression and one scalar pandas UDF
         result7 = (df.groupby(df.v % 2, plus_two(df.id))
@@ -427,7 +430,7 @@ def test_array_type(self):
 
         array_udf = pandas_udf(lambda x: [1.0, 2.0], 'array<double>', PandasUDFType.GROUPED_AGG)
         result1 = df.groupby('id').agg(array_udf(df['v']).alias('v2'))
-        self.assertEquals(result1.first()['v2'], [1.0, 2.0])
+        self.assertEqual(result1.first()['v2'], [1.0, 2.0])
 
     def test_invalid_args(self):
         df = self.data
@@ -435,19 +438,19 @@ def test_invalid_args(self):
         mean_udf = self.pandas_agg_mean_udf
 
         with QuietTest(self.sc):
-            with self.assertRaisesRegexp(
+            with self.assertRaisesRegex(
                     AnalysisException,
                     'nor.*aggregate function'):
                 df.groupby(df.id).agg(plus_one(df.v)).collect()
 
         with QuietTest(self.sc):
-            with self.assertRaisesRegexp(
+            with self.assertRaisesRegex(
                     AnalysisException,
                     'aggregate function.*argument.*aggregate function'):
                 df.groupby(df.id).agg(mean_udf(mean_udf(df.v))).collect()
 
         with QuietTest(self.sc):
-            with self.assertRaisesRegexp(
+            with self.assertRaisesRegex(
                     AnalysisException,
                     'mixture.*aggregate function.*group aggregate pandas UDF'):
                 df.groupby(df.id).agg(mean_udf(df.v), mean(df.v)).collect()
@@ -510,10 +513,10 @@ def mean(x):
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_pandas_udf_grouped_agg import *
+    from pyspark.sql.tests.test_pandas_udf_grouped_agg import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/sql/tests/test_pandas_udf_scalar.py b/python/pyspark/sql/tests/test_pandas_udf_scalar.py
index 7260e80e2cfca..2eb2dec00106e 100644
--- a/python/pyspark/sql/tests/test_pandas_udf_scalar.py
+++ b/python/pyspark/sql/tests/test_pandas_udf_scalar.py
@@ -14,28 +14,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-import datetime
 import os
 import random
 import shutil
-import sys
 import tempfile
 import time
 import unittest
-
-if sys.version >= '3':
-    unicode = str
-
 from datetime import date, datetime
 from decimal import Decimal
+from distutils.version import LooseVersion
 
 from pyspark import TaskContext
 from pyspark.rdd import PythonEvalType
 from pyspark.sql import Column
 from pyspark.sql.functions import array, col, expr, lit, sum, struct, udf, pandas_udf, \
     PandasUDFType
-from pyspark.sql.types import Row
-from pyspark.sql.types import *
+from pyspark.sql.types import IntegerType, ByteType, StructType, ShortType, BooleanType, \
+    LongType, FloatType, DoubleType, DecimalType, StringType, ArrayType, StructField, \
+    Row, TimestampType, MapType, DateType, BinaryType
 from pyspark.sql.utils import AnalysisException
 from pyspark.testing.sqlutils import ReusedSQLTestCase, test_compiled,\
     test_not_compiled_message, have_pandas, have_pyarrow, pandas_requirement_message, \
@@ -46,12 +42,12 @@
     import pandas as pd
 
 if have_pyarrow:
-    import pyarrow as pa
+    import pyarrow as pa  # noqa: F401
 
 
 @unittest.skipIf(
     not have_pandas or not have_pyarrow,
-    pandas_requirement_message or pyarrow_requirement_message)
+    pandas_requirement_message or pyarrow_requirement_message)  # type: ignore
 class ScalarPandasUDFTests(ReusedSQLTestCase):
 
     @classmethod
@@ -137,7 +133,7 @@ def test_vectorized_udf_basic(self):
                             long_f(col('long')), float_f(col('float')),
                             double_f(col('double')), decimal_f('decimal'),
                             bool_f(col('bool')), array_long_f('array_long'))
-            self.assertEquals(df.collect(), res.collect())
+            self.assertEqual(df.collect(), res.collect())
 
     def test_register_nondeterministic_vectorized_udf_basic(self):
         random_pandas_udf = pandas_udf(
@@ -173,7 +169,7 @@ def test_vectorized_udf_null_boolean(self):
         for udf_type in [PandasUDFType.SCALAR, PandasUDFType.SCALAR_ITER]:
             bool_f = pandas_udf(lambda x: x, BooleanType(), udf_type)
             res = df.select(bool_f(col('bool')))
-            self.assertEquals(df.collect(), res.collect())
+            self.assertEqual(df.collect(), res.collect())
 
     def test_vectorized_udf_null_byte(self):
         data = [(None,), (2,), (3,), (4,)]
@@ -182,7 +178,7 @@ def test_vectorized_udf_null_byte(self):
         for udf_type in [PandasUDFType.SCALAR, PandasUDFType.SCALAR_ITER]:
             byte_f = pandas_udf(lambda x: x, ByteType(), udf_type)
             res = df.select(byte_f(col('byte')))
-            self.assertEquals(df.collect(), res.collect())
+            self.assertEqual(df.collect(), res.collect())
 
     def test_vectorized_udf_null_short(self):
         data = [(None,), (2,), (3,), (4,)]
@@ -191,7 +187,7 @@ def test_vectorized_udf_null_short(self):
         for udf_type in [PandasUDFType.SCALAR, PandasUDFType.SCALAR_ITER]:
             short_f = pandas_udf(lambda x: x, ShortType(), udf_type)
             res = df.select(short_f(col('short')))
-            self.assertEquals(df.collect(), res.collect())
+            self.assertEqual(df.collect(), res.collect())
 
     def test_vectorized_udf_null_int(self):
         data = [(None,), (2,), (3,), (4,)]
@@ -200,7 +196,7 @@ def test_vectorized_udf_null_int(self):
         for udf_type in [PandasUDFType.SCALAR, PandasUDFType.SCALAR_ITER]:
             int_f = pandas_udf(lambda x: x, IntegerType(), udf_type)
             res = df.select(int_f(col('int')))
-            self.assertEquals(df.collect(), res.collect())
+            self.assertEqual(df.collect(), res.collect())
 
     def test_vectorized_udf_null_long(self):
         data = [(None,), (2,), (3,), (4,)]
@@ -209,7 +205,7 @@ def test_vectorized_udf_null_long(self):
         for udf_type in [PandasUDFType.SCALAR, PandasUDFType.SCALAR_ITER]:
             long_f = pandas_udf(lambda x: x, LongType(), udf_type)
             res = df.select(long_f(col('long')))
-            self.assertEquals(df.collect(), res.collect())
+            self.assertEqual(df.collect(), res.collect())
 
     def test_vectorized_udf_null_float(self):
         data = [(3.0,), (5.0,), (-1.0,), (None,)]
@@ -218,7 +214,7 @@ def test_vectorized_udf_null_float(self):
         for udf_type in [PandasUDFType.SCALAR, PandasUDFType.SCALAR_ITER]:
             float_f = pandas_udf(lambda x: x, FloatType(), udf_type)
             res = df.select(float_f(col('float')))
-            self.assertEquals(df.collect(), res.collect())
+            self.assertEqual(df.collect(), res.collect())
 
     def test_vectorized_udf_null_double(self):
         data = [(3.0,), (5.0,), (-1.0,), (None,)]
@@ -227,7 +223,7 @@ def test_vectorized_udf_null_double(self):
         for udf_type in [PandasUDFType.SCALAR, PandasUDFType.SCALAR_ITER]:
             double_f = pandas_udf(lambda x: x, DoubleType(), udf_type)
             res = df.select(double_f(col('double')))
-            self.assertEquals(df.collect(), res.collect())
+            self.assertEqual(df.collect(), res.collect())
 
     def test_vectorized_udf_null_decimal(self):
         data = [(Decimal(3.0),), (Decimal(5.0),), (Decimal(-1.0),), (None,)]
@@ -236,7 +232,7 @@ def test_vectorized_udf_null_decimal(self):
         for udf_type in [PandasUDFType.SCALAR, PandasUDFType.SCALAR_ITER]:
             decimal_f = pandas_udf(lambda x: x, DecimalType(38, 18), udf_type)
             res = df.select(decimal_f(col('decimal')))
-            self.assertEquals(df.collect(), res.collect())
+            self.assertEqual(df.collect(), res.collect())
 
     def test_vectorized_udf_null_string(self):
         data = [("foo",), (None,), ("bar",), ("bar",)]
@@ -245,7 +241,7 @@ def test_vectorized_udf_null_string(self):
         for udf_type in [PandasUDFType.SCALAR, PandasUDFType.SCALAR_ITER]:
             str_f = pandas_udf(lambda x: x, StringType(), udf_type)
             res = df.select(str_f(col('str')))
-            self.assertEquals(df.collect(), res.collect())
+            self.assertEqual(df.collect(), res.collect())
 
     def test_vectorized_udf_string_in_udf(self):
         df = self.spark.range(10)
@@ -259,7 +255,7 @@ def iter_f(it):
             str_f = pandas_udf(f, StringType(), udf_type)
             actual = df.select(str_f(col('id')))
             expected = df.select(col('id').cast('string'))
-            self.assertEquals(expected.collect(), actual.collect())
+            self.assertEqual(expected.collect(), actual.collect())
 
     def test_vectorized_udf_datatype_string(self):
         df = self.spark.range(10).select(
@@ -283,7 +279,7 @@ def test_vectorized_udf_datatype_string(self):
                             long_f(col('long')), float_f(col('float')),
                             double_f(col('double')), decimal_f('decimal'),
                             bool_f(col('bool')))
-            self.assertEquals(df.collect(), res.collect())
+            self.assertEqual(df.collect(), res.collect())
 
     def test_vectorized_udf_null_binary(self):
         data = [(bytearray(b"a"),), (None,), (bytearray(b"bb"),), (bytearray(b"ccc"),)]
@@ -292,7 +288,7 @@ def test_vectorized_udf_null_binary(self):
         for udf_type in [PandasUDFType.SCALAR, PandasUDFType.SCALAR_ITER]:
             str_f = pandas_udf(lambda x: x, BinaryType(), udf_type)
             res = df.select(str_f(col('binary')))
-            self.assertEquals(df.collect(), res.collect())
+            self.assertEqual(df.collect(), res.collect())
 
     def test_vectorized_udf_array_type(self):
         data = [([1, 2],), ([3, 4],)]
@@ -301,7 +297,7 @@ def test_vectorized_udf_array_type(self):
         for udf_type in [PandasUDFType.SCALAR, PandasUDFType.SCALAR_ITER]:
             array_f = pandas_udf(lambda x: x, ArrayType(IntegerType()), udf_type)
             result = df.select(array_f(col('array')))
-            self.assertEquals(df.collect(), result.collect())
+            self.assertEqual(df.collect(), result.collect())
 
     def test_vectorized_udf_null_array(self):
         data = [([1, 2],), (None,), (None,), ([3, 4],), (None,)]
@@ -310,7 +306,7 @@ def test_vectorized_udf_null_array(self):
         for udf_type in [PandasUDFType.SCALAR, PandasUDFType.SCALAR_ITER]:
             array_f = pandas_udf(lambda x: x, ArrayType(IntegerType()), udf_type)
             result = df.select(array_f(col('array')))
-            self.assertEquals(df.collect(), result.collect())
+            self.assertEqual(df.collect(), result.collect())
 
     def test_vectorized_udf_struct_type(self):
         df = self.spark.range(10)
@@ -319,7 +315,7 @@ def test_vectorized_udf_struct_type(self):
             StructField('str', StringType())])
 
         def scalar_func(id):
-            return pd.DataFrame({'id': id, 'str': id.apply(unicode)})
+            return pd.DataFrame({'id': id, 'str': id.apply(str)})
 
         def iter_func(it):
             for id in it:
@@ -379,11 +375,25 @@ def test_vectorized_udf_nested_struct(self):
 
         for udf_type in [PandasUDFType.SCALAR, PandasUDFType.SCALAR_ITER]:
             with QuietTest(self.sc):
-                with self.assertRaisesRegexp(
+                with self.assertRaisesRegex(
                         Exception,
                         'Invalid return type with scalar Pandas UDFs'):
                     pandas_udf(lambda x: x, returnType=nested_type, functionType=udf_type)
 
+    def test_vectorized_udf_map_type(self):
+        data = [({},), ({"a": 1},), ({"a": 1, "b": 2},), ({"a": 1, "b": 2, "c": 3},)]
+        schema = StructType([StructField("map", MapType(StringType(), LongType()))])
+        df = self.spark.createDataFrame(data, schema=schema)
+        for udf_type in [PandasUDFType.SCALAR, PandasUDFType.SCALAR_ITER]:
+            if LooseVersion(pa.__version__) < LooseVersion("2.0.0"):
+                with QuietTest(self.sc):
+                    with self.assertRaisesRegex(Exception, "MapType.*not supported"):
+                        pandas_udf(lambda x: x, MapType(StringType(), LongType()), udf_type)
+            else:
+                map_f = pandas_udf(lambda x: x, MapType(StringType(), LongType()), udf_type)
+                result = df.select(map_f(col('map')))
+                self.assertEqual(df.collect(), result.collect())
+
     def test_vectorized_udf_complex(self):
         df = self.spark.range(10).select(
             col('id').cast('int').alias('a'),
@@ -412,7 +422,7 @@ def iter_mul(it):
                                  (iter_add, iter_power2, iter_mul)]:
             res = df.select(add(col('a'), col('b')), power2(col('a')), mul(col('b'), col('c')))
             expected = df.select(expr('a + b'), expr('power(2, a)'), expr('b * c'))
-            self.assertEquals(expected.collect(), res.collect())
+            self.assertEqual(expected.collect(), res.collect())
 
     def test_vectorized_udf_exception(self):
         df = self.spark.range(10)
@@ -425,14 +435,14 @@ def iter_raise_exception(it):
 
         for raise_exception in [scalar_raise_exception, iter_raise_exception]:
             with QuietTest(self.sc):
-                with self.assertRaisesRegexp(Exception, 'division( or modulo)? by zero'):
+                with self.assertRaisesRegex(Exception, 'division( or modulo)? by zero'):
                     df.select(raise_exception(col('id'))).collect()
 
     def test_vectorized_udf_invalid_length(self):
         df = self.spark.range(10)
         raise_exception = pandas_udf(lambda _: pd.Series(1), LongType())
         with QuietTest(self.sc):
-            with self.assertRaisesRegexp(
+            with self.assertRaisesRegex(
                     Exception,
                     'Result vector from pandas_udf was not the required length'):
                 df.select(raise_exception(col('id'))).collect()
@@ -443,7 +453,7 @@ def iter_udf_wong_output_size(it):
                 yield pd.Series(1)
 
         with QuietTest(self.sc):
-            with self.assertRaisesRegexp(
+            with self.assertRaisesRegex(
                     Exception,
                     "The length of output in Scalar iterator.*"
                     "the length of output was 1"):
@@ -459,7 +469,7 @@ def iter_udf_not_reading_all_input(it):
         with self.sql_conf({"spark.sql.execution.arrow.maxRecordsPerBatch": 3}):
             df1 = self.spark.range(10).repartition(1)
             with QuietTest(self.sc):
-                with self.assertRaisesRegexp(
+                with self.assertRaisesRegex(
                         Exception,
                         "pandas iterator UDF should exhaust"):
                     df1.select(iter_udf_not_reading_all_input(col('id'))).collect()
@@ -476,7 +486,7 @@ def test_vectorized_udf_chained(self):
 
         for f, g in [(scalar_f, scalar_g), (iter_f, iter_g)]:
             res = df.select(g(f(col('id'))))
-            self.assertEquals(df.collect(), res.collect())
+            self.assertEqual(df.collect(), res.collect())
 
     def test_vectorized_udf_chained_struct_type(self):
         df = self.spark.range(10)
@@ -486,14 +496,14 @@ def test_vectorized_udf_chained_struct_type(self):
 
         @pandas_udf(return_type)
         def scalar_f(id):
-            return pd.DataFrame({'id': id, 'str': id.apply(unicode)})
+            return pd.DataFrame({'id': id, 'str': id.apply(str)})
 
         scalar_g = pandas_udf(lambda x: x, return_type)
 
         @pandas_udf(return_type, PandasUDFType.SCALAR_ITER)
         def iter_f(it):
             for id in it:
-                yield pd.DataFrame({'id': id, 'str': id.apply(unicode)})
+                yield pd.DataFrame({'id': id, 'str': id.apply(str)})
 
         iter_g = pandas_udf(lambda x: x, return_type, PandasUDFType.SCALAR_ITER)
 
@@ -507,10 +517,10 @@ def iter_f(it):
     def test_vectorized_udf_wrong_return_type(self):
         with QuietTest(self.sc):
             for udf_type in [PandasUDFType.SCALAR, PandasUDFType.SCALAR_ITER]:
-                with self.assertRaisesRegexp(
+                with self.assertRaisesRegex(
                         NotImplementedError,
-                        'Invalid return type.*scalar Pandas UDF.*MapType'):
-                    pandas_udf(lambda x: x, MapType(LongType(), LongType()), udf_type)
+                        'Invalid return type.*scalar Pandas UDF.*ArrayType.*TimestampType'):
+                    pandas_udf(lambda x: x, ArrayType(TimestampType()), udf_type)
 
     def test_vectorized_udf_return_scalar(self):
         df = self.spark.range(10)
@@ -519,7 +529,7 @@ def test_vectorized_udf_return_scalar(self):
                             PandasUDFType.SCALAR_ITER)
         for f in [scalar_f, iter_f]:
             with QuietTest(self.sc):
-                with self.assertRaisesRegexp(Exception, 'Return.*type.*Series'):
+                with self.assertRaisesRegex(Exception, 'Return.*type.*Series'):
                     df.select(f(col('id'))).collect()
 
     def test_vectorized_udf_decorator(self):
@@ -535,14 +545,14 @@ def iter_identity(x):
 
         for identity in [scalar_identity, iter_identity]:
             res = df.select(identity(col('id')))
-            self.assertEquals(df.collect(), res.collect())
+            self.assertEqual(df.collect(), res.collect())
 
     def test_vectorized_udf_empty_partition(self):
         df = self.spark.createDataFrame(self.sc.parallelize([Row(id=1)], 2))
         for udf_type in [PandasUDFType.SCALAR, PandasUDFType.SCALAR_ITER]:
             f = pandas_udf(lambda x: x, LongType(), udf_type)
             res = df.select(f(col('id')))
-            self.assertEquals(df.collect(), res.collect())
+            self.assertEqual(df.collect(), res.collect())
 
     def test_vectorized_udf_struct_with_empty_partition(self):
         df = self.spark.createDataFrame(self.sc.parallelize([Row(id=1)], 2))\
@@ -575,16 +585,16 @@ def iter_f(it):
 
         for f in [scalar_f, iter_f]:
             res = df.select(f(col('id'), col('id')))
-            self.assertEquals(df.collect(), res.collect())
+            self.assertEqual(df.collect(), res.collect())
 
     def test_vectorized_udf_unsupported_types(self):
         with QuietTest(self.sc):
             for udf_type in [PandasUDFType.SCALAR, PandasUDFType.SCALAR_ITER]:
-                with self.assertRaisesRegexp(
+                with self.assertRaisesRegex(
                         NotImplementedError,
-                        'Invalid return type.*scalar Pandas UDF.*MapType'):
-                    pandas_udf(lambda x: x, MapType(StringType(), IntegerType()), udf_type)
-                with self.assertRaisesRegexp(
+                        'Invalid return type.*scalar Pandas UDF.*ArrayType.*TimestampType'):
+                    pandas_udf(lambda x: x, ArrayType(TimestampType()), udf_type)
+                with self.assertRaisesRegex(
                         NotImplementedError,
                         'Invalid return type.*scalar Pandas UDF.*ArrayType.StructType'):
                     pandas_udf(lambda x: x,
@@ -627,10 +637,10 @@ def iter_check_data(it):
             result = df.withColumn("check_data",
                                    check_data(col("idx"), col("date"), col("date_copy"))).collect()
 
-            self.assertEquals(len(data), len(result))
+            self.assertEqual(len(data), len(result))
             for i in range(len(result)):
-                self.assertEquals(data[i][1], result[i][1])  # "date" col
-                self.assertEquals(data[i][1], result[i][2])  # "date_copy" col
+                self.assertEqual(data[i][1], result[i][1])  # "date" col
+                self.assertEqual(data[i][1], result[i][2])  # "date_copy" col
                 self.assertIsNone(result[i][3])  # "check_data" col
 
     def test_vectorized_udf_timestamps(self):
@@ -676,10 +686,10 @@ def iter_check_data(it):
             result = df.withColumn("check_data", check_data(col("idx"), col("timestamp"),
                                                             col("timestamp_copy"))).collect()
             # Check that collection values are correct
-            self.assertEquals(len(data), len(result))
+            self.assertEqual(len(data), len(result))
             for i in range(len(result)):
-                self.assertEquals(data[i][1], result[i][1])  # "timestamp" col
-                self.assertEquals(data[i][1], result[i][2])  # "timestamp_copy" col
+                self.assertEqual(data[i][1], result[i][1])  # "timestamp" col
+                self.assertEqual(data[i][1], result[i][2])  # "timestamp_copy" col
                 self.assertIsNone(result[i][3])  # "check_data" col
 
     def test_vectorized_udf_return_timestamp_tz(self):
@@ -703,7 +713,7 @@ def iter_gen_timestamps(it):
                 i, ts = r
                 ts_tz = pd.Timestamp(i, unit='D', tz='America/Los_Angeles').to_pydatetime()
                 expected = spark_ts_t.fromInternal(spark_ts_t.toInternal(ts_tz))
-                self.assertEquals(expected, ts)
+                self.assertEqual(expected, ts)
 
     def test_vectorized_udf_check_config(self):
         with self.sql_conf({"spark.sql.execution.arrow.maxRecordsPerBatch": 3}):
@@ -789,9 +799,9 @@ def test_nondeterministic_vectorized_udf_in_aggregate(self):
         for random_udf in [self.nondeterministic_vectorized_udf,
                            self.nondeterministic_vectorized_iter_udf]:
             with QuietTest(self.sc):
-                with self.assertRaisesRegexp(AnalysisException, 'nondeterministic'):
+                with self.assertRaisesRegex(AnalysisException, 'nondeterministic'):
                     df.groupby(df.id).agg(sum(random_udf(df.id))).collect()
-                with self.assertRaisesRegexp(AnalysisException, 'nondeterministic'):
+                with self.assertRaisesRegex(AnalysisException, 'nondeterministic'):
                     df.agg(sum(random_udf(df.id))).collect()
 
     def test_register_vectorized_udf_basic(self):
@@ -815,8 +825,8 @@ def iter_original_add(it):
             res2 = self.spark.sql(
                 "SELECT add1(t.a, t.b) FROM (SELECT id as a, id as b FROM range(10)) t")
             expected = df.select(expr('a + b'))
-            self.assertEquals(expected.collect(), res1.collect())
-            self.assertEquals(expected.collect(), res2.collect())
+            self.assertEqual(expected.collect(), res1.collect())
+            self.assertEqual(expected.collect(), res2.collect())
 
     def test_scalar_iter_udf_init(self):
         import numpy as np
@@ -844,7 +854,7 @@ def test_close(batch_iter):
             finally:
                 raise RuntimeError("reached finally block")
         with QuietTest(self.sc):
-            with self.assertRaisesRegexp(Exception, "reached finally block"):
+            with self.assertRaisesRegex(Exception, "reached finally block"):
                 self.spark.range(1).select(test_close(col("id"))).collect()
 
     def test_scalar_iter_udf_close_early(self):
@@ -895,23 +905,32 @@ def test_timestamp_dst(self):
         for udf_type in [PandasUDFType.SCALAR, PandasUDFType.SCALAR_ITER]:
             foo_udf = pandas_udf(lambda x: x, 'timestamp', udf_type)
             result = df.withColumn('time', foo_udf(df.time))
-            self.assertEquals(df.collect(), result.collect())
+            self.assertEqual(df.collect(), result.collect())
+
+    def test_udf_category_type(self):
+
+        @pandas_udf('string')
+        def to_category_func(x):
+            return x.astype('category')
+
+        pdf = pd.DataFrame({"A": [u"a", u"b", u"c", u"a"]})
+        df = self.spark.createDataFrame(pdf)
+        df = df.withColumn("B", to_category_func(df['A']))
+        result_spark = df.toPandas()
+
+        spark_type = df.dtypes[1][1]
+        # spark data frame and arrow execution mode enabled data frame type must match pandas
+        self.assertEqual(spark_type, 'string')
+
+        # Check result of column 'B' must be equal to column 'A' in type and values
+        pd.testing.assert_series_equal(result_spark["A"], result_spark["B"], check_names=False)
 
-    @unittest.skipIf(sys.version_info[:2] < (3, 5), "Type hints are supported from Python 3.5.")
     def test_type_annotation(self):
         # Regression test to check if type hints can be used. See SPARK-23569.
-        # Note that it throws an error during compilation in lower Python versions if 'exec'
-        # is not used. Also, note that we explicitly use another dictionary to avoid modifications
-        # in the current 'locals()'.
-        #
-        # Hyukjin: I think it's an ugly way to test issues about syntax specific in
-        # higher versions of Python, which we shouldn't encourage. This was the last resort
-        # I could come up with at that time.
-        _locals = {}
-        exec(
-            "import pandas as pd\ndef noop(col: pd.Series) -> pd.Series: return col",
-            _locals)
-        df = self.spark.range(1).select(pandas_udf(f=_locals['noop'], returnType='bigint')('id'))
+        def noop(col: pd.Series) -> pd.Series:
+            return col
+
+        df = self.spark.range(1).select(pandas_udf(f=noop, returnType='bigint')('id'))
         self.assertEqual(df.first()[0], 0)
 
     def test_mixed_udf(self):
@@ -984,11 +1003,11 @@ def f4_iter(it):
             df_chained_4 = df.withColumn('f4_f2_f1', f4(f2(f1(df['v']))))
             df_chained_5 = df.withColumn('f4_f3_f1', f4(f3(f1(df['v']))))
 
-            self.assertEquals(expected_chained_1, df_chained_1.collect())
-            self.assertEquals(expected_chained_2, df_chained_2.collect())
-            self.assertEquals(expected_chained_3, df_chained_3.collect())
-            self.assertEquals(expected_chained_4, df_chained_4.collect())
-            self.assertEquals(expected_chained_5, df_chained_5.collect())
+            self.assertEqual(expected_chained_1, df_chained_1.collect())
+            self.assertEqual(expected_chained_2, df_chained_2.collect())
+            self.assertEqual(expected_chained_3, df_chained_3.collect())
+            self.assertEqual(expected_chained_4, df_chained_4.collect())
+            self.assertEqual(expected_chained_5, df_chained_5.collect())
 
             # Test multiple mixed UDF expressions in a single projection
             df_multi_1 = df \
@@ -1026,8 +1045,8 @@ def f4_iter(it):
                 .withColumn('f4_f3_f2', f4(f3(f2(col('v'))))) \
                 .withColumn('f4_f3_f2_f1', f4(f3(f2(f1(col('v'))))))
 
-            self.assertEquals(expected_multi, df_multi_1.collect())
-            self.assertEquals(expected_multi, df_multi_2.collect())
+            self.assertEqual(expected_multi, df_multi_1.collect())
+            self.assertEqual(expected_multi, df_multi_2.collect())
 
     def test_mixed_udf_and_sql(self):
         df = self.spark.range(0, 1).toDF('v')
@@ -1088,10 +1107,10 @@ def f3i(it):
                 .withColumn('f3_f1_f2', f3(f1(f2(df['v'])))) \
                 .withColumn('f3_f2_f1', f3(f2(f1(df['v']))))
 
-            self.assertEquals(expected, df1.collect())
+            self.assertEqual(expected, df1.collect())
 
     # SPARK-24721
-    @unittest.skipIf(not test_compiled, test_not_compiled_message)
+    @unittest.skipIf(not test_compiled, test_not_compiled_message)  # type: ignore
     def test_datasource_with_udf(self):
         # Same as SQLTests.test_datasource_with_udf, but with Pandas UDF
         # This needs to a separate test because Arrow dependency is optional
@@ -1119,26 +1138,26 @@ def test_datasource_with_udf(self):
             for df in [filesource_df, datasource_df, datasource_v2_df]:
                 result = df.withColumn('c', c1)
                 expected = df.withColumn('c', lit(2))
-                self.assertEquals(expected.collect(), result.collect())
+                self.assertEqual(expected.collect(), result.collect())
 
             for df in [filesource_df, datasource_df, datasource_v2_df]:
                 result = df.withColumn('c', c2)
                 expected = df.withColumn('c', col('i') + 1)
-                self.assertEquals(expected.collect(), result.collect())
+                self.assertEqual(expected.collect(), result.collect())
 
             for df in [filesource_df, datasource_df, datasource_v2_df]:
                 for f in [f1, f2]:
                     result = df.filter(f)
-                    self.assertEquals(0, result.count())
+                    self.assertEqual(0, result.count())
         finally:
             shutil.rmtree(path)
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_pandas_udf_scalar import *
+    from pyspark.sql.tests.test_pandas_udf_scalar import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/sql/tests/test_pandas_udf_typehints.py b/python/pyspark/sql/tests/test_pandas_udf_typehints.py
index 2582080056864..e30f43181ae96 100644
--- a/python/pyspark/sql/tests/test_pandas_udf_typehints.py
+++ b/python/pyspark/sql/tests/test_pandas_udf_typehints.py
@@ -14,9 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-import sys
 import unittest
 import inspect
+from typing import Union, Iterator, Tuple
 
 from pyspark.sql.functions import mean, lit
 from pyspark.testing.sqlutils import ReusedSQLTestCase, \
@@ -24,209 +24,162 @@
     pyarrow_requirement_message
 from pyspark.sql.pandas.typehints import infer_eval_type
 from pyspark.sql.pandas.functions import pandas_udf, PandasUDFType
+from pyspark.sql import Row
 
 if have_pandas:
     import pandas as pd
-    from pandas.util.testing import assert_frame_equal
-
-python_requirement_message = "pandas UDF with type hints are supported with Python 3.6+."
+    import numpy as np
+    from pandas.testing import assert_frame_equal
 
 
 @unittest.skipIf(
-    not have_pandas or not have_pyarrow or sys.version_info[:2] < (3, 6),
-    pandas_requirement_message or pyarrow_requirement_message or python_requirement_message)
+    not have_pandas or not have_pyarrow,
+    pandas_requirement_message or pyarrow_requirement_message)  # type: ignore[arg-type]
 class PandasUDFTypeHintsTests(ReusedSQLTestCase):
-    # Note that, we should remove `exec` once we drop Python 2 in this class.
-
-    def setUp(self):
-        self.local = {'pd': pd}
-
     def test_type_annotation_scalar(self):
-        exec(
-            "def func(col: pd.Series) -> pd.Series: pass",
-            self.local)
+        def func(col: pd.Series) -> pd.Series:
+            pass
         self.assertEqual(
-            infer_eval_type(inspect.signature(self.local['func'])), PandasUDFType.SCALAR)
+            infer_eval_type(inspect.signature(func)), PandasUDFType.SCALAR)
 
-        exec(
-            "def func(col: pd.DataFrame, col1: pd.Series) -> pd.DataFrame: pass",
-            self.local)
+        def func(col: pd.DataFrame, col1: pd.Series) -> pd.DataFrame:
+            pass
         self.assertEqual(
-            infer_eval_type(inspect.signature(self.local['func'])), PandasUDFType.SCALAR)
+            infer_eval_type(inspect.signature(func)), PandasUDFType.SCALAR)
 
-        exec(
-            "def func(col: pd.DataFrame, *args: pd.Series) -> pd.Series: pass",
-            self.local)
+        def func(col: pd.DataFrame, *args: pd.Series) -> pd.Series:
+            pass
         self.assertEqual(
-            infer_eval_type(inspect.signature(self.local['func'])), PandasUDFType.SCALAR)
+            infer_eval_type(inspect.signature(func)), PandasUDFType.SCALAR)
 
-        exec(
-            "def func(col: pd.Series, *args: pd.Series, **kwargs: pd.DataFrame) -> pd.Series:\n"
-            "    pass",
-            self.local)
+        def func(col: pd.Series, *args: pd.Series, **kwargs: pd.DataFrame) -> pd.Series:
+            pass
         self.assertEqual(
-            infer_eval_type(inspect.signature(self.local['func'])), PandasUDFType.SCALAR)
+            infer_eval_type(inspect.signature(func)), PandasUDFType.SCALAR)
 
-        exec(
-            "def func(col: pd.Series, *, col2: pd.DataFrame) -> pd.DataFrame:\n"
-            "    pass",
-            self.local)
+        def func(col: pd.Series, *, col2: pd.DataFrame) -> pd.DataFrame:
+            pass
         self.assertEqual(
-            infer_eval_type(inspect.signature(self.local['func'])), PandasUDFType.SCALAR)
+            infer_eval_type(inspect.signature(func)), PandasUDFType.SCALAR)
 
-        exec(
-            "from typing import Union\n"
-            "def func(col: Union[pd.Series, pd.DataFrame], *, col2: pd.DataFrame) -> pd.Series:\n"
-            "    pass",
-            self.local)
+        def func(col: Union[pd.Series, pd.DataFrame], *, col2: pd.DataFrame) -> pd.Series:
+            pass
         self.assertEqual(
-            infer_eval_type(inspect.signature(self.local['func'])), PandasUDFType.SCALAR)
+            infer_eval_type(inspect.signature(func)), PandasUDFType.SCALAR)
 
     def test_type_annotation_scalar_iter(self):
-        exec(
-            "from typing import Iterator\n"
-            "def func(iter: Iterator[pd.Series]) -> Iterator[pd.Series]: pass",
-            self.local)
+        def func(iter: Iterator[pd.Series]) -> Iterator[pd.Series]:
+            pass
         self.assertEqual(
-            infer_eval_type(inspect.signature(self.local['func'])), PandasUDFType.SCALAR_ITER)
+            infer_eval_type(inspect.signature(func)), PandasUDFType.SCALAR_ITER)
 
-        exec(
-            "from typing import Iterator, Tuple\n"
-            "def func(iter: Iterator[Tuple[pd.DataFrame, pd.Series]]) -> Iterator[pd.DataFrame]:\n"
-            "    pass",
-            self.local)
+        def func(iter: Iterator[Tuple[pd.DataFrame, pd.Series]]) -> Iterator[pd.DataFrame]:
+            pass
         self.assertEqual(
-            infer_eval_type(inspect.signature(self.local['func'])), PandasUDFType.SCALAR_ITER)
+            infer_eval_type(inspect.signature(func)), PandasUDFType.SCALAR_ITER)
 
-        exec(
-            "from typing import Iterator, Tuple\n"
-            "def func(iter: Iterator[Tuple[pd.DataFrame, ...]]) -> Iterator[pd.Series]: pass",
-            self.local)
+        def func(iter: Iterator[Tuple[pd.DataFrame, ...]]) -> Iterator[pd.Series]:
+            pass
         self.assertEqual(
-            infer_eval_type(inspect.signature(self.local['func'])), PandasUDFType.SCALAR_ITER)
+            infer_eval_type(inspect.signature(func)), PandasUDFType.SCALAR_ITER)
 
-        exec(
-            "from typing import Iterator, Tuple, Union\n"
-            "def func(iter: Iterator[Tuple[Union[pd.DataFrame, pd.Series], ...]])"
-            " -> Iterator[pd.Series]: pass",
-            self.local)
+        def func(
+            iter: Iterator[Tuple[Union[pd.DataFrame, pd.Series], ...]]
+        ) -> Iterator[pd.Series]:
+            pass
         self.assertEqual(
-            infer_eval_type(inspect.signature(self.local['func'])), PandasUDFType.SCALAR_ITER)
+            infer_eval_type(inspect.signature(func)), PandasUDFType.SCALAR_ITER)
 
     def test_type_annotation_group_agg(self):
-        exec(
-            "def func(col: pd.Series) -> str: pass",
-            self.local)
+
+        def func(col: pd.Series) -> str:
+            pass
         self.assertEqual(
-            infer_eval_type(inspect.signature(self.local['func'])), PandasUDFType.GROUPED_AGG)
+            infer_eval_type(inspect.signature(func)), PandasUDFType.GROUPED_AGG)
 
-        exec(
-            "def func(col: pd.DataFrame, col1: pd.Series) -> int: pass",
-            self.local)
+        def func(col: pd.DataFrame, col1: pd.Series) -> int:
+            pass
         self.assertEqual(
-            infer_eval_type(inspect.signature(self.local['func'])), PandasUDFType.GROUPED_AGG)
+            infer_eval_type(inspect.signature(func)), PandasUDFType.GROUPED_AGG)
 
-        exec(
-            "from pyspark.sql import Row\n"
-            "def func(col: pd.DataFrame, *args: pd.Series) -> Row: pass",
-            self.local)
+        def func(col: pd.DataFrame, *args: pd.Series) -> Row:
+            pass
         self.assertEqual(
-            infer_eval_type(inspect.signature(self.local['func'])), PandasUDFType.GROUPED_AGG)
+            infer_eval_type(inspect.signature(func)), PandasUDFType.GROUPED_AGG)
 
-        exec(
-            "def func(col: pd.Series, *args: pd.Series, **kwargs: pd.DataFrame) -> str:\n"
-            "    pass",
-            self.local)
+        def func(col: pd.Series, *args: pd.Series, **kwargs: pd.DataFrame) -> str:
+            pass
         self.assertEqual(
-            infer_eval_type(inspect.signature(self.local['func'])), PandasUDFType.GROUPED_AGG)
+            infer_eval_type(inspect.signature(func)), PandasUDFType.GROUPED_AGG)
 
-        exec(
-            "def func(col: pd.Series, *, col2: pd.DataFrame) -> float:\n"
-            "    pass",
-            self.local)
+        def func(col: pd.Series, *, col2: pd.DataFrame) -> float:
+            pass
         self.assertEqual(
-            infer_eval_type(inspect.signature(self.local['func'])), PandasUDFType.GROUPED_AGG)
+            infer_eval_type(inspect.signature(func)), PandasUDFType.GROUPED_AGG)
 
-        exec(
-            "from typing import Union\n"
-            "def func(col: Union[pd.Series, pd.DataFrame], *, col2: pd.DataFrame) -> float:\n"
-            "    pass",
-            self.local)
+        def func(col: Union[pd.Series, pd.DataFrame], *, col2: pd.DataFrame) -> float:
+            pass
         self.assertEqual(
-            infer_eval_type(inspect.signature(self.local['func'])), PandasUDFType.GROUPED_AGG)
+            infer_eval_type(inspect.signature(func)), PandasUDFType.GROUPED_AGG)
 
     def test_type_annotation_negative(self):
-        exec(
-            "def func(col: str) -> pd.Series: pass",
-            self.local)
+
+        def func(col: str) -> pd.Series:
+            pass
         self.assertRaisesRegex(
             NotImplementedError,
             "Unsupported signature.*str",
-            infer_eval_type, inspect.signature(self.local['func']))
+            infer_eval_type, inspect.signature(func))
 
-        exec(
-            "def func(col: pd.DataFrame, col1: int) -> pd.DataFrame: pass",
-            self.local)
+        def func(col: pd.DataFrame, col1: int) -> pd.DataFrame:
+            pass
         self.assertRaisesRegex(
             NotImplementedError,
             "Unsupported signature.*int",
-            infer_eval_type, inspect.signature(self.local['func']))
+            infer_eval_type, inspect.signature(func))
 
-        exec(
-            "from typing import Union\n"
-            "def func(col: Union[pd.DataFrame, str], col1: int) -> pd.DataFrame: pass",
-            self.local)
+        def func(col: Union[pd.DataFrame, str], col1: int) -> pd.DataFrame:
+            pass
         self.assertRaisesRegex(
             NotImplementedError,
             "Unsupported signature.*str",
-            infer_eval_type, inspect.signature(self.local['func']))
+            infer_eval_type, inspect.signature(func))
 
-        exec(
-            "from typing import Tuple\n"
-            "def func(col: pd.Series) -> Tuple[pd.DataFrame]: pass",
-            self.local)
+        def func(col: pd.Series) -> Tuple[pd.DataFrame]:
+            pass
         self.assertRaisesRegex(
             NotImplementedError,
             "Unsupported signature.*Tuple",
-            infer_eval_type, inspect.signature(self.local['func']))
+            infer_eval_type, inspect.signature(func))
 
-        exec(
-            "def func(col, *args: pd.Series) -> pd.Series: pass",
-            self.local)
+        def func(col, *args: pd.Series) -> pd.Series:
+            pass
         self.assertRaisesRegex(
             ValueError,
             "should be specified.*Series",
-            infer_eval_type, inspect.signature(self.local['func']))
+            infer_eval_type, inspect.signature(func))
 
-        exec(
-            "def func(col: pd.Series, *args: pd.Series, **kwargs: pd.DataFrame):\n"
-            "    pass",
-            self.local)
+        def func(col: pd.Series, *args: pd.Series, **kwargs: pd.DataFrame):
+            pass
         self.assertRaisesRegex(
             ValueError,
             "should be specified.*Series",
-            infer_eval_type, inspect.signature(self.local['func']))
+            infer_eval_type, inspect.signature(func))
 
-        exec(
-            "def func(col: pd.Series, *, col2) -> pd.DataFrame:\n"
-            "    pass",
-            self.local)
+        def func(col: pd.Series, *, col2) -> pd.DataFrame:
+            pass
         self.assertRaisesRegex(
             ValueError,
             "should be specified.*Series",
-            infer_eval_type, inspect.signature(self.local['func']))
+            infer_eval_type, inspect.signature(func))
 
     def test_scalar_udf_type_hint(self):
         df = self.spark.range(10).selectExpr("id", "id as v")
 
-        exec(
-            "import typing\n"
-            "def plus_one(v: typing.Union[pd.Series, pd.DataFrame]) -> pd.Series:\n"
-            "    return v + 1",
-            self.local)
-
-        plus_one = pandas_udf("long")(self.local["plus_one"])
+        def plus_one(v: Union[pd.Series, pd.DataFrame]) -> pd.Series:
+            return v + 1
 
+        plus_one = pandas_udf("long")(plus_one)
         actual = df.select(plus_one(df.v).alias("plus_one"))
         expected = df.selectExpr("(v + 1) as plus_one")
         assert_frame_equal(expected.toPandas(), actual.toPandas())
@@ -234,14 +187,11 @@ def test_scalar_udf_type_hint(self):
     def test_scalar_iter_udf_type_hint(self):
         df = self.spark.range(10).selectExpr("id", "id as v")
 
-        exec(
-            "import typing\n"
-            "def plus_one(itr: typing.Iterator[pd.Series]) -> typing.Iterator[pd.Series]:\n"
-            "    for s in itr:\n"
-            "        yield s + 1",
-            self.local)
+        def plus_one(itr: Iterator[pd.Series]) -> Iterator[pd.Series]:
+            for s in itr:
+                yield s + 1
 
-        plus_one = pandas_udf("long")(self.local["plus_one"])
+        plus_one = pandas_udf("long")(plus_one)
 
         actual = df.select(plus_one(df.v).alias("plus_one"))
         expected = df.selectExpr("(v + 1) as plus_one")
@@ -249,13 +199,11 @@ def test_scalar_iter_udf_type_hint(self):
 
     def test_group_agg_udf_type_hint(self):
         df = self.spark.range(10).selectExpr("id", "id as v")
-        exec(
-            "import numpy as np\n"
-            "def weighted_mean(v: pd.Series, w: pd.Series) -> float:\n"
-            "    return np.average(v, weights=w)",
-            self.local)
 
-        weighted_mean = pandas_udf("double")(self.local["weighted_mean"])
+        def weighted_mean(v: pd.Series, w: pd.Series) -> float:
+            return np.average(v, weights=w)
+
+        weighted_mean = pandas_udf("double")(weighted_mean)
 
         actual = df.groupby('id').agg(weighted_mean(df.v, lit(1.0))).sort('id')
         expected = df.groupby('id').agg(mean(df.v).alias('weighted_mean(v, 1.0)')).sort('id')
@@ -263,12 +211,9 @@ def test_group_agg_udf_type_hint(self):
 
     def test_ignore_type_hint_in_group_apply_in_pandas(self):
         df = self.spark.range(10)
-        exec(
-            "def pandas_plus_one(v: pd.DataFrame) -> pd.DataFrame:\n"
-            "    return v + 1",
-            self.local)
 
-        pandas_plus_one = self.local["pandas_plus_one"]
+        def pandas_plus_one(v: pd.DataFrame) -> pd.DataFrame:
+            return v + 1
 
         actual = df.groupby('id').applyInPandas(pandas_plus_one, schema=df.schema).sort('id')
         expected = df.selectExpr("id + 1 as id")
@@ -276,12 +221,9 @@ def test_ignore_type_hint_in_group_apply_in_pandas(self):
 
     def test_ignore_type_hint_in_cogroup_apply_in_pandas(self):
         df = self.spark.range(10)
-        exec(
-            "def pandas_plus_one(left: pd.DataFrame, right: pd.DataFrame) -> pd.DataFrame:\n"
-            "    return left + 1",
-            self.local)
 
-        pandas_plus_one = self.local["pandas_plus_one"]
+        def pandas_plus_one(left: pd.DataFrame, right: pd.DataFrame) -> pd.DataFrame:
+            return left + 1
 
         actual = df.groupby('id').cogroup(
             self.spark.range(10).groupby("id")
@@ -291,13 +233,9 @@ def test_ignore_type_hint_in_cogroup_apply_in_pandas(self):
 
     def test_ignore_type_hint_in_map_in_pandas(self):
         df = self.spark.range(10)
-        exec(
-            "from typing import Iterator\n"
-            "def pandas_plus_one(iter: Iterator[pd.DataFrame]) -> Iterator[pd.DataFrame]:\n"
-            "    return map(lambda v: v + 1, iter)",
-            self.local)
 
-        pandas_plus_one = self.local["pandas_plus_one"]
+        def pandas_plus_one(iter: Iterator[pd.DataFrame]) -> Iterator[pd.DataFrame]:
+            return map(lambda v: v + 1, iter)
 
         actual = df.mapInPandas(pandas_plus_one, schema=df.schema)
         expected = df.selectExpr("id + 1 as id")
@@ -305,10 +243,10 @@ def test_ignore_type_hint_in_map_in_pandas(self):
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_pandas_udf_typehints import *
+    from pyspark.sql.tests.test_pandas_udf_typehints import *  # noqa: #401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/sql/tests/test_pandas_udf_window.py b/python/pyspark/sql/tests/test_pandas_udf_window.py
index 4c969abef4920..18379b8a8c8c0 100644
--- a/python/pyspark/sql/tests/test_pandas_udf_window.py
+++ b/python/pyspark/sql/tests/test_pandas_udf_window.py
@@ -26,12 +26,12 @@
 from pyspark.testing.utils import QuietTest
 
 if have_pandas:
-    from pandas.util.testing import assert_frame_equal
+    from pandas.testing import assert_frame_equal
 
 
 @unittest.skipIf(
     not have_pandas or not have_pyarrow,
-    pandas_requirement_message or pyarrow_requirement_message)
+    pandas_requirement_message or pyarrow_requirement_message)  # type: ignore[arg-type]
 class WindowPandasUDFTests(ReusedSQLTestCase):
     @property
     def data(self):
@@ -43,7 +43,11 @@ def data(self):
 
     @property
     def python_plus_one(self):
-        return udf(lambda v: v + 1, 'double')
+        @udf('double')
+        def plus_one(v):
+            assert isinstance(v, float)
+            return v + 1
+        return plus_one
 
     @property
     def pandas_scalar_time_two(self):
@@ -241,14 +245,14 @@ def test_array_type(self):
 
         array_udf = pandas_udf(lambda x: [1.0, 2.0], 'array<double>', PandasUDFType.GROUPED_AGG)
         result1 = df.withColumn('v2', array_udf(df['v']).over(w))
-        self.assertEquals(result1.first()['v2'], [1.0, 2.0])
+        self.assertEqual(result1.first()['v2'], [1.0, 2.0])
 
     def test_invalid_args(self):
         df = self.data
         w = self.unbounded_window
 
         with QuietTest(self.sc):
-            with self.assertRaisesRegexp(
+            with self.assertRaisesRegex(
                     AnalysisException,
                     '.*not supported within a window function'):
                 foo_udf = pandas_udf(lambda x: x, 'v double', PandasUDFType.GROUPED_MAP)
@@ -352,10 +356,10 @@ def test_bounded_mixed(self):
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_pandas_udf_window import *
+    from pyspark.sql.tests.test_pandas_udf_window import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/sql/tests/test_readwriter.py b/python/pyspark/sql/tests/test_readwriter.py
index 2530cc2ebf224..80b4118ae796a 100644
--- a/python/pyspark/sql/tests/test_readwriter.py
+++ b/python/pyspark/sql/tests/test_readwriter.py
@@ -19,7 +19,9 @@
 import shutil
 import tempfile
 
-from pyspark.sql.types import *
+from pyspark.sql.functions import col
+from pyspark.sql.readwriter import DataFrameWriterV2
+from pyspark.sql.types import StructType, StructField, StringType
 from pyspark.testing.sqlutils import ReusedSQLTestCase
 
 
@@ -163,12 +165,46 @@ def test_insert_into(self):
             self.assertEqual(6, self.spark.sql("select * from test_table").count())
 
 
+class ReadwriterV2Tests(ReusedSQLTestCase):
+    def test_api(self):
+        df = self.df
+        writer = df.writeTo("testcat.t")
+        self.assertIsInstance(writer, DataFrameWriterV2)
+        self.assertIsInstance(writer.option("property", "value"), DataFrameWriterV2)
+        self.assertIsInstance(writer.options(property="value"), DataFrameWriterV2)
+        self.assertIsInstance(writer.using("source"), DataFrameWriterV2)
+        self.assertIsInstance(writer.partitionedBy("id"), DataFrameWriterV2)
+        self.assertIsInstance(writer.partitionedBy(col("id")), DataFrameWriterV2)
+        self.assertIsInstance(writer.tableProperty("foo", "bar"), DataFrameWriterV2)
+
+    def test_partitioning_functions(self):
+        import datetime
+        from pyspark.sql.functions import years, months, days, hours, bucket
+
+        df = self.spark.createDataFrame(
+            [(1, datetime.datetime(2000, 1, 1), "foo")],
+            ("id", "ts", "value")
+        )
+
+        writer = df.writeTo("testcat.t")
+
+        self.assertIsInstance(writer.partitionedBy(years("ts")), DataFrameWriterV2)
+        self.assertIsInstance(writer.partitionedBy(months("ts")), DataFrameWriterV2)
+        self.assertIsInstance(writer.partitionedBy(days("ts")), DataFrameWriterV2)
+        self.assertIsInstance(writer.partitionedBy(hours("ts")), DataFrameWriterV2)
+        self.assertIsInstance(writer.partitionedBy(bucket(11, "id")), DataFrameWriterV2)
+        self.assertIsInstance(writer.partitionedBy(bucket(11, col("id"))), DataFrameWriterV2)
+        self.assertIsInstance(
+            writer.partitionedBy(bucket(3, "id"), hours(col("ts"))), DataFrameWriterV2
+        )
+
+
 if __name__ == "__main__":
     import unittest
-    from pyspark.sql.tests.test_readwriter import *
+    from pyspark.sql.tests.test_readwriter import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/sql/tests/test_serde.py b/python/pyspark/sql/tests/test_serde.py
index 052a5b2835245..ce087ff4ce550 100644
--- a/python/pyspark/sql/tests/test_serde.py
+++ b/python/pyspark/sql/tests/test_serde.py
@@ -22,7 +22,7 @@
 
 from pyspark.sql import Row
 from pyspark.sql.functions import lit
-from pyspark.sql.types import *
+from pyspark.sql.types import StructType, StructField, DecimalType, BinaryType
 from pyspark.testing.sqlutils import ReusedSQLTestCase, UTCOffsetTimezone
 
 
@@ -139,10 +139,10 @@ def test_bytes_as_binary_type(self):
 
 if __name__ == "__main__":
     import unittest
-    from pyspark.sql.tests.test_serde import *
+    from pyspark.sql.tests.test_serde import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/sql/tests/test_session.py b/python/pyspark/sql/tests/test_session.py
index 5e4166e6f8e71..7faeb1857b983 100644
--- a/python/pyspark/sql/tests/test_session.py
+++ b/python/pyspark/sql/tests/test_session.py
@@ -358,10 +358,10 @@ def test_use_custom_class_for_extensions(self):
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_session import *
+    from pyspark.sql.tests.test_session import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/sql/tests/test_streaming.py b/python/pyspark/sql/tests/test_streaming.py
index d19ca075f56c3..44cdde0f2e8a9 100644
--- a/python/pyspark/sql/tests/test_streaming.py
+++ b/python/pyspark/sql/tests/test_streaming.py
@@ -20,8 +20,9 @@
 import tempfile
 import time
 
+from pyspark.sql import Row
 from pyspark.sql.functions import lit
-from pyspark.sql.types import *
+from pyspark.sql.types import StructType, StructField, IntegerType, StringType
 from pyspark.testing.sqlutils import ReusedSQLTestCase
 
 
@@ -68,9 +69,12 @@ def test_stream_read_options(self):
     def test_stream_read_options_overwrite(self):
         bad_schema = StructType([StructField("test", IntegerType(), False)])
         schema = StructType([StructField("data", StringType(), False)])
-        df = self.spark.readStream.format('csv').option('path', 'python/test_support/sql/fake') \
-            .schema(bad_schema)\
-            .load(path='python/test_support/sql/streaming', schema=schema, format='text')
+        # SPARK-32516 disables the overwrite behavior by default.
+        with self.sql_conf({"spark.sql.legacy.pathOptionBehavior.enabled": True}):
+            df = self.spark.readStream.format('csv')\
+                .option('path', 'python/test_support/sql/fake')\
+                .schema(bad_schema)\
+                .load(path='python/test_support/sql/streaming', schema=schema, format='text')
         self.assertTrue(df.isStreaming)
         self.assertEqual(df.schema.simpleString(), "struct<data:string>")
 
@@ -110,10 +114,12 @@ def test_stream_save_options_overwrite(self):
         chk = os.path.join(tmpPath, 'chk')
         fake1 = os.path.join(tmpPath, 'fake1')
         fake2 = os.path.join(tmpPath, 'fake2')
-        q = df.writeStream.option('checkpointLocation', fake1)\
-            .format('memory').option('path', fake2) \
-            .queryName('fake_query').outputMode('append') \
-            .start(path=out, format='parquet', queryName='this_query', checkpointLocation=chk)
+        # SPARK-32516 disables the overwrite behavior by default.
+        with self.sql_conf({"spark.sql.legacy.pathOptionBehavior.enabled": True}):
+            q = df.writeStream.option('checkpointLocation', fake1)\
+                .format('memory').option('path', fake2) \
+                .queryName('fake_query').outputMode('append') \
+                .start(path=out, format='parquet', queryName='this_query', checkpointLocation=chk)
 
         try:
             self.assertEqual(q.name, 'this_query')
@@ -564,13 +570,36 @@ def collectBatch(df, id):
             if q:
                 q.stop()
 
+    def test_streaming_read_from_table(self):
+        with self.table("input_table", "this_query"):
+            self.spark.sql("CREATE TABLE input_table (value string) USING parquet")
+            self.spark.sql("INSERT INTO input_table VALUES ('aaa'), ('bbb'), ('ccc')")
+            df = self.spark.readStream.table("input_table")
+            self.assertTrue(df.isStreaming)
+            q = df.writeStream.format('memory').queryName('this_query').start()
+            q.processAllAvailable()
+            q.stop()
+            result = self.spark.sql("SELECT * FROM this_query ORDER BY value").collect()
+            self.assertEqual(
+                set([Row(value='aaa'), Row(value='bbb'), Row(value='ccc')]), set(result))
+
+    def test_streaming_write_to_table(self):
+        with self.table("output_table"), tempfile.TemporaryDirectory() as tmpdir:
+            df = self.spark.readStream.format("rate").option("rowsPerSecond", 10).load()
+            q = df.writeStream.toTable("output_table", format='parquet', checkpointLocation=tmpdir)
+            self.assertTrue(q.isActive)
+            time.sleep(3)
+            q.stop()
+            result = self.spark.sql("SELECT value FROM output_table").collect()
+            self.assertTrue(len(result) > 0)
+
 
 if __name__ == "__main__":
     import unittest
-    from pyspark.sql.tests.test_streaming import *
+    from pyspark.sql.tests.test_streaming import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/sql/tests/test_types.py b/python/pyspark/sql/tests/test_types.py
index 81402f52af3b3..eb4caf05d1af0 100644
--- a/python/pyspark/sql/tests/test_types.py
+++ b/python/pyspark/sql/tests/test_types.py
@@ -25,10 +25,16 @@
 import unittest
 
 from pyspark.sql import Row
-from pyspark.sql.functions import col, UserDefinedFunction
-from pyspark.sql.types import *
-from pyspark.sql.types import _array_signed_int_typecode_ctype_mappings, _array_type_mappings, \
+from pyspark.sql.functions import col
+from pyspark.sql.udf import UserDefinedFunction
+from pyspark.sql.utils import AnalysisException
+from pyspark.sql.types import ByteType, ShortType, IntegerType, FloatType, DateType, \
+    TimestampType, MapType, StringType, StructType, StructField, ArrayType, DoubleType, LongType, \
+    DecimalType, BinaryType, BooleanType, NullType
+from pyspark.sql.types import (  # type: ignore
+    _array_signed_int_typecode_ctype_mappings, _array_type_mappings,
     _array_unsigned_int_typecode_ctype_mappings, _infer_type, _make_type_verifier, _merge_type
+)
 from pyspark.testing.sqlutils import ReusedSQLTestCase, ExamplePointUDT, PythonOnlyUDT, \
     ExamplePoint, PythonOnlyPoint, MyObject
 
@@ -56,7 +62,7 @@ def test_infer_schema_to_local(self):
         self.assertEqual(10, df3.count())
 
     def test_apply_schema_to_dict_and_rows(self):
-        schema = StructType().add("b", StringType()).add("a", IntegerType())
+        schema = StructType().add("a", IntegerType()).add("b", StringType())
         input = [{"a": 1}, {"b": "coffee"}]
         rdd = self.sc.parallelize(input)
         for verify in [False, True]:
@@ -72,7 +78,6 @@ def test_apply_schema_to_dict_and_rows(self):
             self.assertEqual(10, df4.count())
 
     def test_create_dataframe_schema_mismatch(self):
-        input = [Row(a=1)]
         rdd = self.sc.parallelize(range(3)).map(lambda i: Row(a=i))
         schema = StructType([StructField("a", IntegerType()), StructField("b", StringType())])
         df = self.spark.createDataFrame(rdd, schema)
@@ -175,7 +180,7 @@ def test_infer_schema_not_enough_names(self):
         self.assertEqual(df.columns, ['col1', '_2'])
 
     def test_infer_schema_fails(self):
-        with self.assertRaisesRegexp(TypeError, 'field a'):
+        with self.assertRaisesRegex(TypeError, 'field a'):
             self.spark.createDataFrame(self.spark.sparkContext.parallelize([[1, 1], ["x", 1]]),
                                        schema=["a", "b"], samplingRatio=0.99)
 
@@ -437,6 +442,14 @@ def test_cast_to_string_with_udt(self):
         result = df.select(col('point').cast('string'), col('pypoint').cast('string')).head()
         self.assertEqual(result, Row(point=u'(1.0, 2.0)', pypoint=u'[3.0, 4.0]'))
 
+    def test_cast_to_udt_with_udt(self):
+        from pyspark.sql.functions import col
+        row = Row(point=ExamplePoint(1.0, 2.0), python_only_point=PythonOnlyPoint(1.0, 2.0))
+        df = self.spark.createDataFrame([row])
+        self.assertRaises(AnalysisException, lambda: df.select(col("point").cast(PythonOnlyUDT())))
+        self.assertRaises(AnalysisException,
+                          lambda: df.select(col("python_only_point").cast(ExamplePointUDT())))
+
     def test_struct_type(self):
         struct1 = StructType().add("f1", StringType(), True).add("f2", StringType(), True, None)
         struct2 = StructType([StructField("f1", StringType(), True),
@@ -540,7 +553,6 @@ def test_infer_long_type(self):
         self.assertEqual(_infer_type(2**61), LongType())
         self.assertEqual(_infer_type(2**71), LongType())
 
-    @unittest.skipIf(sys.version < "3", "only Python 3 infers bytes as binary type")
     def test_infer_binary_type(self):
         binaryrow = [Row(f1='a', f2=b"abcd")]
         df = self.sc.parallelize(binaryrow).toDF()
@@ -566,18 +578,18 @@ def test_merge_type(self):
             ArrayType(LongType()),
             ArrayType(LongType())
         ), ArrayType(LongType()))
-        with self.assertRaisesRegexp(TypeError, 'element in array'):
+        with self.assertRaisesRegex(TypeError, 'element in array'):
             _merge_type(ArrayType(LongType()), ArrayType(DoubleType()))
 
         self.assertEqual(_merge_type(
             MapType(StringType(), LongType()),
             MapType(StringType(), LongType())
         ), MapType(StringType(), LongType()))
-        with self.assertRaisesRegexp(TypeError, 'key of map'):
+        with self.assertRaisesRegex(TypeError, 'key of map'):
             _merge_type(
                 MapType(StringType(), LongType()),
                 MapType(DoubleType(), LongType()))
-        with self.assertRaisesRegexp(TypeError, 'value of map'):
+        with self.assertRaisesRegex(TypeError, 'value of map'):
             _merge_type(
                 MapType(StringType(), LongType()),
                 MapType(StringType(), DoubleType()))
@@ -586,7 +598,7 @@ def test_merge_type(self):
             StructType([StructField("f1", LongType()), StructField("f2", StringType())]),
             StructType([StructField("f1", LongType()), StructField("f2", StringType())])
         ), StructType([StructField("f1", LongType()), StructField("f2", StringType())]))
-        with self.assertRaisesRegexp(TypeError, 'field f1'):
+        with self.assertRaisesRegex(TypeError, 'field f1'):
             _merge_type(
                 StructType([StructField("f1", LongType()), StructField("f2", StringType())]),
                 StructType([StructField("f1", DoubleType()), StructField("f2", StringType())]))
@@ -595,7 +607,7 @@ def test_merge_type(self):
             StructType([StructField("f1", StructType([StructField("f2", LongType())]))]),
             StructType([StructField("f1", StructType([StructField("f2", LongType())]))])
         ), StructType([StructField("f1", StructType([StructField("f2", LongType())]))]))
-        with self.assertRaisesRegexp(TypeError, 'field f2 in field f1'):
+        with self.assertRaisesRegex(TypeError, 'field f2 in field f1'):
             _merge_type(
                 StructType([StructField("f1", StructType([StructField("f2", LongType())]))]),
                 StructType([StructField("f1", StructType([StructField("f2", StringType())]))]))
@@ -604,7 +616,7 @@ def test_merge_type(self):
             StructType([StructField("f1", ArrayType(LongType())), StructField("f2", StringType())]),
             StructType([StructField("f1", ArrayType(LongType())), StructField("f2", StringType())])
         ), StructType([StructField("f1", ArrayType(LongType())), StructField("f2", StringType())]))
-        with self.assertRaisesRegexp(TypeError, 'element in array field f1'):
+        with self.assertRaisesRegex(TypeError, 'element in array field f1'):
             _merge_type(
                 StructType([
                     StructField("f1", ArrayType(LongType())),
@@ -623,7 +635,7 @@ def test_merge_type(self):
         ), StructType([
             StructField("f1", MapType(StringType(), LongType())),
             StructField("f2", StringType())]))
-        with self.assertRaisesRegexp(TypeError, 'value of map field f1'):
+        with self.assertRaisesRegex(TypeError, 'value of map field f1'):
             _merge_type(
                 StructType([
                     StructField("f1", MapType(StringType(), LongType())),
@@ -636,7 +648,7 @@ def test_merge_type(self):
             StructType([StructField("f1", ArrayType(MapType(StringType(), LongType())))]),
             StructType([StructField("f1", ArrayType(MapType(StringType(), LongType())))])
         ), StructType([StructField("f1", ArrayType(MapType(StringType(), LongType())))]))
-        with self.assertRaisesRegexp(TypeError, 'key of map element in array field f1'):
+        with self.assertRaisesRegex(TypeError, 'key of map element in array field f1'):
             _merge_type(
                 StructType([StructField("f1", ArrayType(MapType(StringType(), LongType())))]),
                 StructType([StructField("f1", ArrayType(MapType(DoubleType(), LongType())))])
@@ -665,10 +677,6 @@ def assertCollectSuccess(typecode, value):
             supported_string_types += ['u']
             # test unicode
             assertCollectSuccess('u', u'a')
-        if sys.version_info[0] < 3:
-            supported_string_types += ['c']
-            # test string
-            assertCollectSuccess('c', 'a')
 
         # supported float and double
         #
@@ -721,15 +729,12 @@ def assertCollectSuccess(typecode, value):
         #
         # Keys in _array_type_mappings is a complete list of all supported types,
         # and types not in _array_type_mappings are considered unsupported.
-        # `array.typecodes` are not supported in python 2.
-        if sys.version_info[0] < 3:
-            all_types = set(['c', 'b', 'B', 'u', 'h', 'H', 'i', 'I', 'l', 'L', 'f', 'd'])
-        else:
-            all_types = set(array.typecodes)
+        # PyPy seems not having array.typecodes.
+        all_types = set(['b', 'B', 'u', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'f', 'd'])
         unsupported_types = all_types - set(supported_types)
         # test unsupported types
         for t in unsupported_types:
-            with self.assertRaises(TypeError):
+            with self.assertRaisesRegex(TypeError, "infer the type of the field myarray"):
                 a = array.array(t)
                 self.spark.createDataFrame([Row(myarray=a)]).collect()
 
@@ -766,10 +771,7 @@ def test_row_without_column_name(self):
         self.assertEqual(repr(row), "<Row('Alice', 11)>")
 
         # test __repr__ with unicode values
-        if sys.version_info.major >= 3:
-            self.assertEqual(repr(Row("数", "量")), "<Row('数', '量')>")
-        else:
-            self.assertEqual(repr(Row(u"数", u"量")), r"<Row(u'\u6570', u'\u91cf')>")
+        self.assertEqual(repr(Row("数", "量")), "<Row('数', '量')>")
 
     def test_empty_row(self):
         row = Row()
@@ -787,13 +789,13 @@ def test_invalid_create_row(self):
 class DataTypeVerificationTests(unittest.TestCase):
 
     def test_verify_type_exception_msg(self):
-        self.assertRaisesRegexp(
+        self.assertRaisesRegex(
             ValueError,
             "test_name",
             lambda: _make_type_verifier(StringType(), nullable=False, name="test_name")(None))
 
         schema = StructType([StructField('a', StructType([StructField('b', IntegerType())]))])
-        self.assertRaisesRegexp(
+        self.assertRaisesRegex(
             TypeError,
             "field b in field a",
             lambda: _make_type_verifier(schema)([["data"]]))
@@ -887,7 +889,6 @@ def __init__(self, **kwargs):
             ({"s": "a", "f": 1.0}, schema),
             (Row(s="a", i=1), schema),
             (Row(s="a", i=None), schema),
-            (Row(s="a", i=1, f=1.0), schema),
             (["a", 1], schema),
             (["a", None], schema),
             (("a", 1), schema),
@@ -972,25 +973,20 @@ def __init__(self, **kwargs):
             with self.assertRaises(exp, msg=msg):
                 _make_type_verifier(data_type, nullable=False)(obj)
 
-    @unittest.skipIf(sys.version_info[:2] < (3, 6), "Create Row without sorting fields")
     def test_row_without_field_sorting(self):
-        sorting_enabled_tmp = Row._row_field_sorting_enabled
-        Row._row_field_sorting_enabled = False
-
         r = Row(b=1, a=2)
         TestRow = Row("b", "a")
         expected = TestRow(1, 2)
 
         self.assertEqual(r, expected)
         self.assertEqual(repr(r), "Row(b=1, a=2)")
-        Row._row_field_sorting_enabled = sorting_enabled_tmp
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_types import *
+    from pyspark.sql.tests.test_types import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/sql/tests/test_udf.py b/python/pyspark/sql/tests/test_udf.py
index 061d3f5e1f7ac..0d13361dcabe6 100644
--- a/python/pyspark/sql/tests/test_udf.py
+++ b/python/pyspark/sql/tests/test_udf.py
@@ -23,8 +23,10 @@
 
 from pyspark import SparkContext
 from pyspark.sql import SparkSession, Column, Row
-from pyspark.sql.functions import UserDefinedFunction, udf
-from pyspark.sql.types import *
+from pyspark.sql.functions import udf
+from pyspark.sql.udf import UserDefinedFunction
+from pyspark.sql.types import StringType, IntegerType, BooleanType, DoubleType, LongType, \
+    ArrayType, StructType, StructField
 from pyspark.sql.utils import AnalysisException
 from pyspark.testing.sqlutils import ReusedSQLTestCase, test_compiled, test_not_compiled_message
 from pyspark.testing.utils import QuietTest
@@ -96,7 +98,7 @@ def test_udf_registration_return_type_none(self):
 
     def test_udf_registration_return_type_not_none(self):
         with QuietTest(self.sc):
-            with self.assertRaisesRegexp(TypeError, "Invalid return type"):
+            with self.assertRaisesRegex(TypeError, "Invalid return type"):
                 self.spark.catalog.registerFunction(
                     "f", UserDefinedFunction(lambda x, y: len(x) + y, StringType()), StringType())
 
@@ -147,9 +149,9 @@ def test_nondeterministic_udf_in_aggregate(self):
         df = self.spark.range(10)
 
         with QuietTest(self.sc):
-            with self.assertRaisesRegexp(AnalysisException, "nondeterministic"):
+            with self.assertRaisesRegex(AnalysisException, "nondeterministic"):
                 df.groupby('id').agg(sum(udf_random_col())).collect()
-            with self.assertRaisesRegexp(AnalysisException, "nondeterministic"):
+            with self.assertRaisesRegex(AnalysisException, "nondeterministic"):
                 df.agg(sum(udf_random_col())).collect()
 
     def test_chained_udf(self):
@@ -201,7 +203,7 @@ def test_udf_in_join_condition(self):
         # Cross join.
         df = left.join(right, f("a", "b"))
         with self.sql_conf({"spark.sql.crossJoin.enabled": False}):
-            with self.assertRaisesRegexp(AnalysisException, 'Detected implicit cartesian product'):
+            with self.assertRaisesRegex(AnalysisException, 'Detected implicit cartesian product'):
                 df.collect()
         with self.sql_conf({"spark.sql.crossJoin.enabled": True}):
             self.assertEqual(df.collect(), [Row(a=1, b=1)])
@@ -236,7 +238,7 @@ def test_udf_not_supported_in_join_condition(self):
         f = udf(lambda a, b: a == b, BooleanType())
 
         def runWithJoinType(join_type, type_string):
-            with self.assertRaisesRegexp(
+            with self.assertRaisesRegex(
                     AnalysisException,
                     'Using PythonUDF.*%s is not supported.' % type_string):
                 left.join(right, [f("a", "b"), left.a1 == right.b1], join_type).collect()
@@ -283,7 +285,6 @@ def test_broadcast_in_udf(self):
     def test_udf_with_filter_function(self):
         df = self.spark.createDataFrame([(1, "1"), (2, "2"), (1, "2"), (1, "2")], ["key", "value"])
         from pyspark.sql.functions import col
-        from pyspark.sql.types import BooleanType
 
         my_filter = udf(lambda a: a < 2, BooleanType())
         sel = df.select(col("key"), col("value")).filter((my_filter(col("key"))) & (df.value < "2"))
@@ -292,7 +293,6 @@ def test_udf_with_filter_function(self):
     def test_udf_with_aggregate_function(self):
         df = self.spark.createDataFrame([(1, "1"), (2, "2"), (1, "2"), (1, "2")], ["key", "value"])
         from pyspark.sql.functions import col, sum
-        from pyspark.sql.types import BooleanType
 
         my_filter = udf(lambda a: a == 1, BooleanType())
         sel = df.select(col("key")).distinct().filter(my_filter(col("key")))
@@ -357,20 +357,46 @@ def test_udf_registration_returns_udf(self):
             df.select(add_four("id").alias("plus_four")).collect()
         )
 
+    @unittest.skipIf(not test_compiled, test_not_compiled_message)  # type: ignore
+    def test_register_java_function(self):
+        self.spark.udf.registerJavaFunction(
+            "javaStringLength", "test.org.apache.spark.sql.JavaStringLength", IntegerType())
+        [value] = self.spark.sql("SELECT javaStringLength('test')").first()
+        self.assertEqual(value, 4)
+
+        self.spark.udf.registerJavaFunction(
+            "javaStringLength2", "test.org.apache.spark.sql.JavaStringLength")
+        [value] = self.spark.sql("SELECT javaStringLength2('test')").first()
+        self.assertEqual(value, 4)
+
+        self.spark.udf.registerJavaFunction(
+            "javaStringLength3", "test.org.apache.spark.sql.JavaStringLength", "integer")
+        [value] = self.spark.sql("SELECT javaStringLength3('test')").first()
+        self.assertEqual(value, 4)
+
+    @unittest.skipIf(not test_compiled, test_not_compiled_message)  # type: ignore
+    def test_register_java_udaf(self):
+        self.spark.udf.registerJavaUDAF("javaUDAF", "test.org.apache.spark.sql.MyDoubleAvg")
+        df = self.spark.createDataFrame([(1, "a"), (2, "b"), (3, "a")], ["id", "name"])
+        df.createOrReplaceTempView("df")
+        row = self.spark.sql(
+            "SELECT name, javaUDAF(id) as avg from df group by name order by name desc").first()
+        self.assertEqual(row.asDict(), Row(name='b', avg=102.0).asDict())
+
     def test_non_existed_udf(self):
         spark = self.spark
-        self.assertRaisesRegexp(AnalysisException, "Can not load class non_existed_udf",
-                                lambda: spark.udf.registerJavaFunction("udf1", "non_existed_udf"))
+        self.assertRaisesRegex(AnalysisException, "Can not load class non_existed_udf",
+                               lambda: spark.udf.registerJavaFunction("udf1", "non_existed_udf"))
 
         # This is to check if a deprecated 'SQLContext.registerJavaFunction' can call its alias.
         sqlContext = spark._wrapped
-        self.assertRaisesRegexp(AnalysisException, "Can not load class non_existed_udf",
-                                lambda: sqlContext.registerJavaFunction("udf1", "non_existed_udf"))
+        self.assertRaisesRegex(AnalysisException, "Can not load class non_existed_udf",
+                               lambda: sqlContext.registerJavaFunction("udf1", "non_existed_udf"))
 
     def test_non_existed_udaf(self):
         spark = self.spark
-        self.assertRaisesRegexp(AnalysisException, "Can not load class non_existed_udaf",
-                                lambda: spark.udf.registerJavaUDAF("udaf1", "non_existed_udaf"))
+        self.assertRaisesRegex(AnalysisException, "Can not load class non_existed_udaf",
+                               lambda: spark.udf.registerJavaUDAF("udaf1", "non_existed_udaf"))
 
     def test_udf_with_input_file_name(self):
         from pyspark.sql.functions import input_file_name
@@ -433,13 +459,12 @@ def test_udf_with_string_return_type(self):
 
         self.assertTupleEqual(expected, actual)
 
-    def test_udf_shouldnt_accept_noncallable_object(self):
+    def test_udf_should_not_accept_noncallable_object(self):
         non_callable = None
         self.assertRaises(TypeError, UserDefinedFunction, non_callable, StringType())
 
     def test_udf_with_decorator(self):
         from pyspark.sql.functions import lit
-        from pyspark.sql.types import IntegerType, DoubleType
 
         @udf(IntegerType())
         def add_one(x):
@@ -495,8 +520,6 @@ def as_double(x):
         )
 
     def test_udf_wrapper(self):
-        from pyspark.sql.types import IntegerType
-
         def f(x):
             """Identity"""
             return x
@@ -538,7 +561,7 @@ def test_nonparam_udf_with_aggregate(self):
         self.assertEqual(rows, [Row(_1=1, _2=2, a=u'const_str')])
 
     # SPARK-24721
-    @unittest.skipIf(not test_compiled, test_not_compiled_message)
+    @unittest.skipIf(not test_compiled, test_not_compiled_message)  # type: ignore
     def test_datasource_with_udf(self):
         from pyspark.sql.functions import lit, col
 
@@ -564,17 +587,17 @@ def test_datasource_with_udf(self):
             for df in [filesource_df, datasource_df, datasource_v2_df]:
                 result = df.withColumn('c', c1)
                 expected = df.withColumn('c', lit(2))
-                self.assertEquals(expected.collect(), result.collect())
+                self.assertEqual(expected.collect(), result.collect())
 
             for df in [filesource_df, datasource_df, datasource_v2_df]:
                 result = df.withColumn('c', c2)
                 expected = df.withColumn('c', col('i') + 1)
-                self.assertEquals(expected.collect(), result.collect())
+                self.assertEqual(expected.collect(), result.collect())
 
             for df in [filesource_df, datasource_df, datasource_v2_df]:
                 for f in [f1, f2]:
                     result = df.filter(f)
-                    self.assertEquals(0, result.count())
+                    self.assertEqual(0, result.count())
         finally:
             shutil.rmtree(path)
 
@@ -642,6 +665,26 @@ def f(*a):
         r = df.select(fUdf(*df.columns))
         self.assertEqual(r.first()[0], "success")
 
+    def test_udf_cache(self):
+        func = lambda x: x
+
+        df = self.spark.range(1)
+        df.select(udf(func)("id")).cache()
+
+        self.assertEqual(df.select(udf(func)("id"))._jdf.queryExecution()
+                         .withCachedData().getClass().getSimpleName(), 'InMemoryRelation')
+
+    # SPARK-34545
+    def test_udf_input_serialization_valuecompare_disabled(self):
+        def f(e):
+            return e[0]
+
+        df = self.spark.createDataFrame([((1.0, 1.0), (1, 1))], ['c1', 'c2'])
+        result = df.select("*", udf(f, DoubleType())("c1").alias('c3'),
+                           udf(f, IntegerType())("c2").alias('c4'))
+        self.assertEqual(result.collect(),
+                         [Row(c1=Row(_1=1.0, _2=1.0), c2=Row(_1=1, _2=1), c3=1.0, c4=1)])
+
 
 class UDFInitializationTests(unittest.TestCase):
     def tearDown(self):
@@ -651,7 +694,7 @@ def tearDown(self):
         if SparkContext._active_spark_context is not None:
             SparkContext._active_spark_context.stop()
 
-    def test_udf_init_shouldnt_initialize_context(self):
+    def test_udf_init_should_not_initialize_context(self):
         UserDefinedFunction(lambda x: x, StringType())
 
         self.assertIsNone(
@@ -665,10 +708,10 @@ def test_udf_init_shouldnt_initialize_context(self):
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_udf import *
+    from pyspark.sql.tests.test_udf import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/sql/tests/test_utils.py b/python/pyspark/sql/tests/test_utils.py
index 072ea08085fba..005f0e892b60f 100644
--- a/python/pyspark/sql/tests/test_utils.py
+++ b/python/pyspark/sql/tests/test_utils.py
@@ -31,31 +31,30 @@ def test_capture_user_friendly_exception(self):
         try:
             self.spark.sql("select `中文字段`")
         except AnalysisException as e:
-            self.assertRegexpMatches(str(e), "cannot resolve '`中文字段`'")
+            self.assertRegex(str(e), "cannot resolve '`中文字段`'")
 
     def test_capture_parse_exception(self):
         self.assertRaises(ParseException, lambda: self.spark.sql("abc"))
 
     def test_capture_illegalargument_exception(self):
-        self.assertRaisesRegexp(IllegalArgumentException, "Setting negative mapred.reduce.tasks",
-                                lambda: self.spark.sql("SET mapred.reduce.tasks=-1"))
+        self.assertRaisesRegex(IllegalArgumentException, "Setting negative mapred.reduce.tasks",
+                               lambda: self.spark.sql("SET mapred.reduce.tasks=-1"))
         df = self.spark.createDataFrame([(1, 2)], ["a", "b"])
-        self.assertRaisesRegexp(IllegalArgumentException, "1024 is not in the permitted values",
-                                lambda: df.select(sha2(df.a, 1024)).collect())
+        self.assertRaisesRegex(IllegalArgumentException, "1024 is not in the permitted values",
+                               lambda: df.select(sha2(df.a, 1024)).collect())
         try:
             df.select(sha2(df.a, 1024)).collect()
         except IllegalArgumentException as e:
-            self.assertRegexpMatches(e.desc, "1024 is not in the permitted values")
-            self.assertRegexpMatches(e.stackTrace,
-                                     "org.apache.spark.sql.functions")
+            self.assertRegex(e.desc, "1024 is not in the permitted values")
+            self.assertRegex(e.stackTrace, "org.apache.spark.sql.functions")
 
 
 if __name__ == "__main__":
     import unittest
-    from pyspark.sql.tests.test_utils import *
+    from pyspark.sql.tests.test_utils import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index 320a68dffe7a3..c0948b6e6e379 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -15,7 +15,6 @@
 # limitations under the License.
 #
 
-import os
 import sys
 import decimal
 import time
@@ -26,11 +25,6 @@
 import base64
 from array import array
 import ctypes
-import warnings
-
-if sys.version >= "3":
-    long = int
-    basestring = unicode = str
 
 from py4j.protocol import register_input_converter
 from py4j.java_gateway import JavaClass
@@ -108,13 +102,12 @@ def __call__(cls):
         return cls._instances[cls]
 
 
-class NullType(DataType):
+class NullType(DataType, metaclass=DataTypeSingleton):
     """Null type.
 
     The data type representing None, used for the types that cannot be inferred.
     """
-
-    __metaclass__ = DataTypeSingleton
+    pass
 
 
 class AtomicType(DataType):
@@ -127,11 +120,10 @@ class NumericType(AtomicType):
     """
 
 
-class IntegralType(NumericType):
+class IntegralType(NumericType, metaclass=DataTypeSingleton):
     """Integral data types.
     """
-
-    __metaclass__ = DataTypeSingleton
+    pass
 
 
 class FractionalType(NumericType):
@@ -139,33 +131,28 @@ class FractionalType(NumericType):
     """
 
 
-class StringType(AtomicType):
+class StringType(AtomicType, metaclass=DataTypeSingleton):
     """String data type.
     """
-
-    __metaclass__ = DataTypeSingleton
+    pass
 
 
-class BinaryType(AtomicType):
+class BinaryType(AtomicType, metaclass=DataTypeSingleton):
     """Binary (byte array) data type.
     """
+    pass
 
-    __metaclass__ = DataTypeSingleton
 
-
-class BooleanType(AtomicType):
+class BooleanType(AtomicType, metaclass=DataTypeSingleton):
     """Boolean data type.
     """
-
-    __metaclass__ = DataTypeSingleton
+    pass
 
 
-class DateType(AtomicType):
+class DateType(AtomicType, metaclass=DataTypeSingleton):
     """Date (datetime.date) data type.
     """
 
-    __metaclass__ = DataTypeSingleton
-
     EPOCH_ORDINAL = datetime.datetime(1970, 1, 1).toordinal()
 
     def needConversion(self):
@@ -180,12 +167,10 @@ def fromInternal(self, v):
             return datetime.date.fromordinal(v + self.EPOCH_ORDINAL)
 
 
-class TimestampType(AtomicType):
+class TimestampType(AtomicType, metaclass=DataTypeSingleton):
     """Timestamp (datetime.datetime) data type.
     """
 
-    __metaclass__ = DataTypeSingleton
-
     def needConversion(self):
         return True
 
@@ -213,8 +198,12 @@ class DecimalType(FractionalType):
     When creating a DecimalType, the default precision and scale is (10, 0). When inferring
     schema from decimal.Decimal objects, it will be DecimalType(38, 18).
 
-    :param precision: the maximum (i.e. total) number of digits (default: 10)
-    :param scale: the number of digits on right side of dot. (default: 0)
+    Parameters
+    ----------
+    precision : int, optional
+        the maximum (i.e. total) number of digits (default: 10)
+    scale : int, optional
+        the number of digits on right side of dot. (default: 0)
     """
 
     def __init__(self, precision=10, scale=0):
@@ -232,18 +221,16 @@ def __repr__(self):
         return "DecimalType(%d,%d)" % (self.precision, self.scale)
 
 
-class DoubleType(FractionalType):
+class DoubleType(FractionalType, metaclass=DataTypeSingleton):
     """Double data type, representing double precision floats.
     """
+    pass
 
-    __metaclass__ = DataTypeSingleton
 
-
-class FloatType(FractionalType):
+class FloatType(FractionalType, metaclass=DataTypeSingleton):
     """Float data type, representing single precision floats.
     """
-
-    __metaclass__ = DataTypeSingleton
+    pass
 
 
 class ByteType(IntegralType):
@@ -280,17 +267,22 @@ def simpleString(self):
 class ArrayType(DataType):
     """Array data type.
 
-    :param elementType: :class:`DataType` of each element in the array.
-    :param containsNull: boolean, whether the array can contain null (None) values.
+    Parameters
+    ----------
+    elementType : :class:`DataType`
+        :class:`DataType` of each element in the array.
+    containsNull : bool, optional
+        whether the array can contain null (None) values.
+
+    Examples
+    --------
+    >>> ArrayType(StringType()) == ArrayType(StringType(), True)
+    True
+    >>> ArrayType(StringType(), False) == ArrayType(StringType())
+    False
     """
 
     def __init__(self, elementType, containsNull=True):
-        """
-        >>> ArrayType(StringType()) == ArrayType(StringType(), True)
-        True
-        >>> ArrayType(StringType(), False) == ArrayType(StringType())
-        False
-        """
         assert isinstance(elementType, DataType),\
             "elementType %s should be an instance of %s" % (elementType, DataType)
         self.elementType = elementType
@@ -330,22 +322,30 @@ def fromInternal(self, obj):
 class MapType(DataType):
     """Map data type.
 
-    :param keyType: :class:`DataType` of the keys in the map.
-    :param valueType: :class:`DataType` of the values in the map.
-    :param valueContainsNull: indicates whether values can contain null (None) values.
-
+    Parameters
+    ----------
+    keyType : :class:`DataType`
+        :class:`DataType` of the keys in the map.
+    valueType : :class:`DataType`
+        :class:`DataType` of the values in the map.
+    valueContainsNull : bool, optional
+        indicates whether values can contain null (None) values.
+
+    Notes
+    -----
     Keys in a map data type are not allowed to be null (None).
+
+    Examples
+    --------
+    >>> (MapType(StringType(), IntegerType())
+    ...        == MapType(StringType(), IntegerType(), True))
+    True
+    >>> (MapType(StringType(), IntegerType(), False)
+    ...        == MapType(StringType(), FloatType()))
+    False
     """
 
     def __init__(self, keyType, valueType, valueContainsNull=True):
-        """
-        >>> (MapType(StringType(), IntegerType())
-        ...        == MapType(StringType(), IntegerType(), True))
-        True
-        >>> (MapType(StringType(), IntegerType(), False)
-        ...        == MapType(StringType(), FloatType()))
-        False
-        """
         assert isinstance(keyType, DataType),\
             "keyType %s should be an instance of %s" % (keyType, DataType)
         assert isinstance(valueType, DataType),\
@@ -392,26 +392,31 @@ def fromInternal(self, obj):
 class StructField(DataType):
     """A field in :class:`StructType`.
 
-    :param name: string, name of the field.
-    :param dataType: :class:`DataType` of the field.
-    :param nullable: boolean, whether the field can be null (None) or not.
-    :param metadata: a dict from string to simple type that can be toInternald to JSON automatically
+    Parameters
+    ----------
+    name : str
+        name of the field.
+    dataType : :class:`DataType`
+        :class:`DataType` of the field.
+    nullable : bool
+        whether the field can be null (None) or not.
+    metadata : dict
+        a dict from string to simple type that can be toInternald to JSON automatically
+
+    Examples
+    --------
+    >>> (StructField("f1", StringType(), True)
+    ...      == StructField("f1", StringType(), True))
+    True
+    >>> (StructField("f1", StringType(), True)
+    ...      == StructField("f2", StringType(), True))
+    False
     """
 
     def __init__(self, name, dataType, nullable=True, metadata=None):
-        """
-        >>> (StructField("f1", StringType(), True)
-        ...      == StructField("f1", StringType(), True))
-        True
-        >>> (StructField("f1", StringType(), True)
-        ...      == StructField("f2", StringType(), True))
-        False
-        """
         assert isinstance(dataType, DataType),\
             "dataType %s should be an instance of %s" % (dataType, DataType)
-        assert isinstance(name, basestring), "field name %s should be string" % (name)
-        if not isinstance(name, str):
-            name = name.encode('utf-8')
+        assert isinstance(name, str), "field name %s should be a string" % (name)
         self.name = name
         self.dataType = dataType
         self.nullable = nullable
@@ -460,24 +465,25 @@ class StructType(DataType):
     Iterating a :class:`StructType` will iterate over its :class:`StructField`\\s.
     A contained :class:`StructField` can be accessed by its name or position.
 
+    Examples
+    --------
     >>> struct1 = StructType([StructField("f1", StringType(), True)])
     >>> struct1["f1"]
     StructField(f1,StringType,true)
     >>> struct1[0]
     StructField(f1,StringType,true)
+
+    >>> struct1 = StructType([StructField("f1", StringType(), True)])
+    >>> struct2 = StructType([StructField("f1", StringType(), True)])
+    >>> struct1 == struct2
+    True
+    >>> struct1 = StructType([StructField("f1", StringType(), True)])
+    >>> struct2 = StructType([StructField("f1", StringType(), True),
+    ...     StructField("f2", IntegerType(), False)])
+    >>> struct1 == struct2
+    False
     """
     def __init__(self, fields=None):
-        """
-        >>> struct1 = StructType([StructField("f1", StringType(), True)])
-        >>> struct2 = StructType([StructField("f1", StringType(), True)])
-        >>> struct1 == struct2
-        True
-        >>> struct1 = StructType([StructField("f1", StringType(), True)])
-        >>> struct2 = StructType([StructField("f1", StringType(), True),
-        ...     StructField("f2", IntegerType(), False)])
-        >>> struct1 == struct2
-        False
-        """
         if not fields:
             self.fields = []
             self.names = []
@@ -500,6 +506,23 @@ def add(self, field, data_type=None, nullable=True, metadata=None):
                metadata(optional). The data_type parameter may be either a String or a
                DataType object.
 
+        Parameters
+        ----------
+        field : str or :class:`StructField`
+            Either the name of the field or a StructField object
+        data_type : :class:`DataType`, optional
+            If present, the DataType of the StructField to create
+        nullable : bool, optional
+            Whether the field to add should be nullable (default True)
+        metadata : dict, optional
+            Any additional metadata (default None)
+
+        Returns
+        -------
+        :class:`StructType`
+
+        Examples
+        --------
         >>> struct1 = StructType().add("f1", StringType(), True).add("f2", StringType(), True, None)
         >>> struct2 = StructType([StructField("f1", StringType(), True), \\
         ...     StructField("f2", StringType(), True, None)])
@@ -513,12 +536,6 @@ def add(self, field, data_type=None, nullable=True, metadata=None):
         >>> struct2 = StructType([StructField("f1", StringType(), True)])
         >>> struct1 == struct2
         True
-
-        :param field: Either the name of the field or a StructField object
-        :param data_type: If present, the DataType of the StructField to create
-        :param nullable: Whether the field to add should be nullable (default True)
-        :param metadata: Any additional metadata (default None)
-        :return: a new updated StructType
         """
         if isinstance(field, StructField):
             self.fields.append(field)
@@ -582,6 +599,8 @@ def fieldNames(self):
         """
         Returns all field names in a list.
 
+        Examples
+        --------
         >>> struct = StructType([StructField("f1", StringType(), True)])
         >>> struct.fieldNames()
         ['f1']
@@ -613,8 +632,6 @@ def toInternal(self, obj):
         else:
             if isinstance(obj, dict):
                 return tuple(obj.get(n) for n in self.names)
-            elif isinstance(obj, Row) and getattr(obj, "__from_dict__", False):
-                return tuple(obj[n] for n in self.names)
             elif isinstance(obj, (list, tuple)):
                 return tuple(obj)
             elif hasattr(obj, "__dict__"):
@@ -766,6 +783,8 @@ def _parse_datatype_string(s):
     for :class:`IntegerType`. Since Spark 2.3, this also supports a schema in a DDL-formatted
     string and case-insensitive strings.
 
+    Examples
+    --------
     >>> _parse_datatype_string("int ")
     IntegerType
     >>> _parse_datatype_string("INT ")
@@ -824,6 +843,9 @@ def from_ddl_datatype(type_str):
 
 def _parse_datatype_json_string(json_string):
     """Parses the given data type JSON string.
+
+    Examples
+    --------
     >>> import pickle
     >>> def check_datatype(datatype):
     ...     pickled = pickle.loads(pickle.dumps(datatype))
@@ -904,19 +926,9 @@ def _parse_datatype_json_value(json_value):
     datetime.date: DateType,
     datetime.datetime: TimestampType,
     datetime.time: TimestampType,
+    bytes: BinaryType,
 }
 
-if sys.version < "3":
-    _type_mappings.update({
-        unicode: StringType,
-        long: LongType,
-    })
-
-if sys.version >= "3":
-    _type_mappings.update({
-        bytes: BinaryType,
-    })
-
 # Mapping Python array types to Spark SQL DataType
 # We should be careful here. The size of these types in python depends on C
 # implementation. We need to make sure that this conversion does not lose any
@@ -990,20 +1002,6 @@ def _int_size_to_type(size):
 if sys.version_info[0] < 4:
     _array_type_mappings['u'] = StringType
 
-# Type code 'c' are only available at python 2
-if sys.version_info[0] < 3:
-    _array_type_mappings['c'] = StringType
-
-# SPARK-21465:
-# In python2, array of 'L' happened to be mistakenly, just partially supported. To
-# avoid breaking user's code, we should keep this partial support. Below is a
-# dirty hacking to keep this partial support and pass the unit test.
-import platform
-if sys.version_info[0] < 3 and platform.python_implementation() != 'PyPy':
-    if 'L' not in _array_type_mappings.keys():
-        _array_type_mappings['L'] = LongType
-        _array_unsigned_int_typecode_ctype_mappings['L'] = ctypes.c_uint
-
 
 def _infer_type(obj):
     """Infer the DataType from obj
@@ -1066,7 +1064,12 @@ def _infer_schema(row, names=None):
     else:
         raise TypeError("Can not infer schema for type: %s" % type(row))
 
-    fields = [StructField(k, _infer_type(v), True) for k, v in items]
+    fields = []
+    for k, v in items:
+        try:
+            fields.append(StructField(k, _infer_type(v), True))
+        except TypeError as e:
+            raise TypeError("Unable to infer the type of the field {}.".format(k)) from e
     return StructType(fields)
 
 
@@ -1187,14 +1190,14 @@ def convert_struct(obj):
 
 _acceptable_types = {
     BooleanType: (bool,),
-    ByteType: (int, long),
-    ShortType: (int, long),
-    IntegerType: (int, long),
-    LongType: (int, long),
+    ByteType: (int,),
+    ShortType: (int,),
+    IntegerType: (int,),
+    LongType: (int,),
     FloatType: (float,),
     DoubleType: (float,),
     DecimalType: (decimal.Decimal,),
-    StringType: (str, unicode),
+    StringType: (str,),
     BinaryType: (bytearray, bytes),
     DateType: (datetime.date, datetime.datetime),
     TimestampType: (datetime.datetime,),
@@ -1213,6 +1216,8 @@ def _make_type_verifier(dataType, nullable=True, name=None):
     within the allowed range, e.g. using 128 as ByteType will overflow. Note that, Python float is
     not checked, so it will become infinity when cast to Java float, if it overflows.
 
+    Examples
+    --------
     >>> _make_type_verifier(StructType([]))(None)
     >>> _make_type_verifier(StringType())("")
     >>> _make_type_verifier(LongType())(0)
@@ -1376,10 +1381,6 @@ def verify_struct(obj):
             if isinstance(obj, dict):
                 for f, verifier in verifiers:
                     verifier(obj.get(f))
-            elif isinstance(obj, Row) and getattr(obj, "__from_dict__", False):
-                # the order in obj could be different than dataType.fields
-                for f, verifier in verifiers:
-                    verifier(obj[f])
             elif isinstance(obj, (tuple, list)):
                 if len(obj) != len(verifiers):
                     raise ValueError(
@@ -1436,23 +1437,16 @@ class Row(tuple):
     It is not allowed to omit a named argument to represent that the value is
     None or missing. This should be explicitly set to None in this case.
 
-    NOTE: As of Spark 3.0.0, Rows created from named arguments no longer have
-    field names sorted alphabetically and will be ordered in the position as
-    entered. To enable sorting for Rows compatible with Spark 2.x, set the
-    environment variable "PYSPARK_ROW_FIELD_SORTING_ENABLED" to "true". This
-    option is deprecated and will be removed in future versions of Spark. For
-    Python versions < 3.6, the order of named arguments is not guaranteed to
-    be the same as entered, see https://www.python.org/dev/peps/pep-0468. In
-    this case, a warning will be issued and the Row will fallback to sort the
-    field names automatically.
-
-    NOTE: Examples with Row in pydocs are run with the environment variable
-    "PYSPARK_ROW_FIELD_SORTING_ENABLED" set to "true" which results in output
-    where fields are sorted.
+    .. versionchanged:: 3.0.0
+        Rows created from named arguments no longer have
+        field names sorted alphabetically and will be ordered in the position as
+        entered.
 
+    Examples
+    --------
     >>> row = Row(name="Alice", age=11)
     >>> row
-    Row(age=11, name='Alice')
+    Row(name='Alice', age=11)
     >>> row['name'], row['age']
     ('Alice', 11)
     >>> row.name, row.age
@@ -1476,47 +1470,22 @@ class Row(tuple):
     Row(name='Alice', age=11)
 
     This form can also be used to create rows as tuple values, i.e. with unnamed
-    fields. Beware that such Row objects have different equality semantics:
+    fields.
 
     >>> row1 = Row("Alice", 11)
     >>> row2 = Row(name="Alice", age=11)
     >>> row1 == row2
-    False
-    >>> row3 = Row(a="Alice", b=11)
-    >>> row1 == row3
     True
     """
 
-    # Remove after Python < 3.6 dropped, see SPARK-29748
-    _row_field_sorting_enabled = \
-        os.environ.get('PYSPARK_ROW_FIELD_SORTING_ENABLED', 'false').lower() == 'true'
-
-    if _row_field_sorting_enabled:
-        warnings.warn("The environment variable 'PYSPARK_ROW_FIELD_SORTING_ENABLED' "
-                      "is deprecated and will be removed in future versions of Spark")
-
     def __new__(cls, *args, **kwargs):
         if args and kwargs:
             raise ValueError("Can not use both args "
                              "and kwargs to create Row")
         if kwargs:
-            if not Row._row_field_sorting_enabled and sys.version_info[:2] < (3, 6):
-                warnings.warn("To use named arguments for Python version < 3.6, Row fields will be "
-                              "automatically sorted. This warning can be skipped by setting the "
-                              "environment variable 'PYSPARK_ROW_FIELD_SORTING_ENABLED' to 'true'.")
-                Row._row_field_sorting_enabled = True
-
             # create row objects
-            if Row._row_field_sorting_enabled:
-                # Remove after Python < 3.6 dropped, see SPARK-29748
-                names = sorted(kwargs.keys())
-                row = tuple.__new__(cls, [kwargs[n] for n in names])
-                row.__fields__ = names
-                row.__from_dict__ = True
-            else:
-                row = tuple.__new__(cls, list(kwargs.values()))
-                row.__fields__ = list(kwargs.keys())
-
+            row = tuple.__new__(cls, list(kwargs.values()))
+            row.__fields__ = list(kwargs.keys())
             return row
         else:
             # create row class or objects
@@ -1526,18 +1495,25 @@ def asDict(self, recursive=False):
         """
         Return as a dict
 
-        :param recursive: turns the nested Rows to dict (default: False).
-
-        .. note:: If a row contains duplicate field names, e.g., the rows of a join
-            between two :class:`DataFrame` that both have the fields of same names,
-            one of the duplicate fields will be selected by ``asDict``. ``__getitem__``
-            will also return one of the duplicate fields, however returned value might
-            be different to ``asDict``.
-
+        Parameters
+        ----------
+        recursive : bool, optional
+            turns the nested Rows to dict (default: False).
+
+        Notes
+        -----
+        If a row contains duplicate field names, e.g., the rows of a join
+        between two :class:`DataFrame` that both have the fields of same names,
+        one of the duplicate fields will be selected by ``asDict``. ``__getitem__``
+        will also return one of the duplicate fields, however returned value might
+        be different to ``asDict``.
+
+        Examples
+        --------
         >>> Row(name="Alice", age=11).asDict() == {'name': 'Alice', 'age': 11}
         True
         >>> row = Row(key=1, value=Row(name='a', age=2))
-        >>> row.asDict() == {'key': 1, 'value': Row(age=2, name='a')}
+        >>> row.asDict() == {'key': 1, 'value': Row(name='a', age=2)}
         True
         >>> row.asDict(True) == {'key': 1, 'value': {'name': 'a', 'age': 2}}
         True
@@ -1600,7 +1576,7 @@ def __getattr__(self, item):
             raise AttributeError(item)
 
     def __setattr__(self, key, value):
-        if key != '__fields__' and key != "__from_dict__":
+        if key != '__fields__':
             raise Exception("Row is read-only")
         self.__dict__[key] = value
 
diff --git a/python/pyspark/sql/types.pyi b/python/pyspark/sql/types.pyi
new file mode 100644
index 0000000000000..3adf823d99a82
--- /dev/null
+++ b/python/pyspark/sql/types.pyi
@@ -0,0 +1,205 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import overload
+from typing import Any, Callable, Dict, Iterator, List, Optional, Union, Tuple, Type, TypeVar
+from py4j.java_gateway import JavaGateway, JavaObject
+import datetime
+
+T = TypeVar("T")
+U = TypeVar("U")
+
+class DataType:
+    def __hash__(self) -> int: ...
+    def __eq__(self, other: Any) -> bool: ...
+    def __ne__(self, other: Any) -> bool: ...
+    @classmethod
+    def typeName(cls) -> str: ...
+    def simpleString(self) -> str: ...
+    def jsonValue(self) -> Union[str, Dict[str, Any]]: ...
+    def json(self) -> str: ...
+    def needConversion(self) -> bool: ...
+    def toInternal(self, obj: Any) -> Any: ...
+    def fromInternal(self, obj: Any) -> Any: ...
+
+class DataTypeSingleton(type):
+    def __call__(cls: Type[T]) -> T: ...  # type: ignore
+
+class NullType(DataType, metaclass=DataTypeSingleton): ...
+class AtomicType(DataType): ...
+class NumericType(AtomicType): ...
+class IntegralType(NumericType, metaclass=DataTypeSingleton): ...
+class FractionalType(NumericType): ...
+class StringType(AtomicType, metaclass=DataTypeSingleton): ...
+class BinaryType(AtomicType, metaclass=DataTypeSingleton): ...
+class BooleanType(AtomicType, metaclass=DataTypeSingleton): ...
+
+class DateType(AtomicType, metaclass=DataTypeSingleton):
+    EPOCH_ORDINAL: int
+    def needConversion(self) -> bool: ...
+    def toInternal(self, d: datetime.date) -> int: ...
+    def fromInternal(self, v: int) -> datetime.date: ...
+
+class TimestampType(AtomicType, metaclass=DataTypeSingleton):
+    def needConversion(self) -> bool: ...
+    def toInternal(self, dt: datetime.datetime) -> int: ...
+    def fromInternal(self, ts: int) -> datetime.datetime: ...
+
+class DecimalType(FractionalType):
+    precision: int
+    scale: int
+    hasPrecisionInfo: bool
+    def __init__(self, precision: int = ..., scale: int = ...) -> None: ...
+    def simpleString(self) -> str: ...
+    def jsonValue(self) -> str: ...
+
+class DoubleType(FractionalType, metaclass=DataTypeSingleton): ...
+class FloatType(FractionalType, metaclass=DataTypeSingleton): ...
+
+class ByteType(IntegralType):
+    def simpleString(self) -> str: ...
+
+class IntegerType(IntegralType):
+    def simpleString(self) -> str: ...
+
+class LongType(IntegralType):
+    def simpleString(self) -> str: ...
+
+class ShortType(IntegralType):
+    def simpleString(self) -> str: ...
+
+class ArrayType(DataType):
+    elementType: DataType
+    containsNull: bool
+    def __init__(self, elementType: DataType, containsNull: bool = ...) -> None: ...
+    def simpleString(self) -> str: ...
+    def jsonValue(self) -> Dict[str, Any]: ...
+    @classmethod
+    def fromJson(cls, json: Dict[str, Any]) -> ArrayType: ...
+    def needConversion(self) -> bool: ...
+    def toInternal(self, obj: List[Optional[T]]) -> List[Optional[T]]: ...
+    def fromInternal(self, obj: List[Optional[T]]) -> List[Optional[T]]: ...
+
+class MapType(DataType):
+    keyType: DataType
+    valueType: DataType
+    valueContainsNull: bool
+    def __init__(
+        self, keyType: DataType, valueType: DataType, valueContainsNull: bool = ...
+    ) -> None: ...
+    def simpleString(self) -> str: ...
+    def jsonValue(self) -> Dict[str, Any]: ...
+    @classmethod
+    def fromJson(cls, json: Dict[str, Any]) -> MapType: ...
+    def needConversion(self) -> bool: ...
+    def toInternal(self, obj: Dict[T, Optional[U]]) -> Dict[T, Optional[U]]: ...
+    def fromInternal(self, obj: Dict[T, Optional[U]]) -> Dict[T, Optional[U]]: ...
+
+class StructField(DataType):
+    name: str
+    dataType: DataType
+    nullable: bool
+    metadata: Dict[str, Any]
+    def __init__(
+        self,
+        name: str,
+        dataType: DataType,
+        nullable: bool = ...,
+        metadata: Optional[Dict[str, Any]] = ...,
+    ) -> None: ...
+    def simpleString(self) -> str: ...
+    def jsonValue(self) -> Dict[str, Any]: ...
+    @classmethod
+    def fromJson(cls, json: Dict[str, Any]) -> StructField: ...
+    def needConversion(self) -> bool: ...
+    def toInternal(self, obj: T) -> T: ...
+    def fromInternal(self, obj: T) -> T: ...
+
+class StructType(DataType):
+    fields: List[StructField]
+    names: List[str]
+    def __init__(self, fields: Optional[List[StructField]] = ...) -> None: ...
+    @overload
+    def add(
+        self,
+        field: str,
+        data_type: Union[str, DataType],
+        nullable: bool = ...,
+        metadata: Optional[Dict[str, Any]] = ...,
+    ) -> StructType: ...
+    @overload
+    def add(self, field: StructField) -> StructType: ...
+    def __iter__(self) -> Iterator[StructField]: ...
+    def __len__(self) -> int: ...
+    def __getitem__(self, key: Union[str, int]) -> StructField: ...
+    def simpleString(self) -> str: ...
+    def jsonValue(self) -> Dict[str, Any]: ...
+    @classmethod
+    def fromJson(cls, json: Dict[str, Any]) -> StructType: ...
+    def fieldNames(self) -> List[str]: ...
+    def needConversion(self) -> bool: ...
+    def toInternal(self, obj: Tuple) -> Tuple: ...
+    def fromInternal(self, obj: Tuple) -> Row: ...
+
+class UserDefinedType(DataType):
+    @classmethod
+    def typeName(cls) -> str: ...
+    @classmethod
+    def sqlType(cls) -> DataType: ...
+    @classmethod
+    def module(cls) -> str: ...
+    @classmethod
+    def scalaUDT(cls) -> str: ...
+    def needConversion(self) -> bool: ...
+    def toInternal(self, obj: Any) -> Any: ...
+    def fromInternal(self, obj: Any) -> Any: ...
+    def serialize(self, obj: Any) -> Any: ...
+    def deserialize(self, datum: Any) -> Any: ...
+    def simpleString(self) -> str: ...
+    def json(self) -> str: ...
+    def jsonValue(self) -> Dict[str, Any]: ...
+    @classmethod
+    def fromJson(cls, json: Dict[str, Any]) -> UserDefinedType: ...
+    def __eq__(self, other: Any) -> bool: ...
+
+class Row(tuple):
+    @overload
+    def __new__(self, *args: str) -> Row: ...
+    @overload
+    def __new__(self, **kwargs: Any) -> Row: ...
+    @overload
+    def __init__(self, *args: str) -> None: ...
+    @overload
+    def __init__(self, **kwargs: Any) -> None: ...
+    def asDict(self, recursive: bool = ...) -> Dict[str, Any]: ...
+    def __contains__(self, item: Any) -> bool: ...
+    def __call__(self, *args: Any) -> Row: ...
+    def __getitem__(self, item: Any) -> Any: ...
+    def __getattr__(self, item: str) -> Any: ...
+    def __setattr__(self, key: Any, value: Any) -> None: ...
+    def __reduce__(
+        self,
+    ) -> Tuple[Callable[[List[str], List[Any]], Row], Tuple[List[str], Tuple]]: ...
+
+class DateConverter:
+    def can_convert(self, obj: Any) -> bool: ...
+    def convert(self, obj: datetime.date, gateway_client: JavaGateway) -> JavaObject: ...
+
+class DatetimeConverter:
+    def can_convert(self, obj: Any) -> bool: ...
+    def convert(self, obj: datetime.datetime, gateway_client: JavaGateway) -> JavaObject: ...
diff --git a/python/pyspark/sql/udf.py b/python/pyspark/sql/udf.py
index 10546ecacc57f..e20b9a1096d77 100644
--- a/python/pyspark/sql/udf.py
+++ b/python/pyspark/sql/udf.py
@@ -20,8 +20,8 @@
 import functools
 import sys
 
-from pyspark import SparkContext, since
-from pyspark.rdd import _prepare_for_python_RDD, PythonEvalType, ignore_unicode_prefix
+from pyspark import SparkContext
+from pyspark.rdd import _prepare_for_python_RDD, PythonEvalType
 from pyspark.sql.column import Column, _to_java_column, _to_seq
 from pyspark.sql.types import StringType, DataType, StructType, _parse_datatype_string
 from pyspark.sql.pandas.types import to_arrow_type
@@ -36,10 +36,10 @@ def _wrap_function(sc, func, returnType):
                                   sc.pythonVer, broadcast_vars, sc._javaAccumulator)
 
 
-def _create_udf(f, returnType, evalType):
+def _create_udf(f, returnType, evalType, name=None, deterministic=True):
     # Set the name of the UserDefinedFunction object to be the name of function f
     udf_obj = UserDefinedFunction(
-        f, returnType=returnType, name=None, evalType=evalType, deterministic=True)
+        f, returnType=returnType, name=name, evalType=evalType, deterministic=deterministic)
     return udf_obj._wrapped()
 
 
@@ -49,9 +49,11 @@ class UserDefinedFunction(object):
 
     .. versionadded:: 1.3
 
-    .. note:: The constructor of this class is not supposed to be directly called.
-        Use :meth:`pyspark.sql.functions.udf` or :meth:`pyspark.sql.functions.pandas_udf`
-        to create this instance.
+    Notes
+    -----
+    The constructor of this class is not supposed to be directly called.
+    Use :meth:`pyspark.sql.functions.udf` or :meth:`pyspark.sql.functions.pandas_udf`
+    to create this instance.
     """
     def __init__(self, func,
                  returnType=StringType(),
@@ -206,6 +208,7 @@ def wrapper(*args):
         wrapper.deterministic = self.deterministic
         wrapper.asNondeterministic = functools.wraps(
             self.asNondeterministic)(lambda: self.asNondeterministic()._wrapped())
+        wrapper._unwrapped = self
         return wrapper
 
     def asNondeterministic(self):
@@ -232,27 +235,39 @@ class UDFRegistration(object):
     def __init__(self, sparkSession):
         self.sparkSession = sparkSession
 
-    @ignore_unicode_prefix
-    @since("1.3.1")
     def register(self, name, f, returnType=None):
         """Register a Python function (including lambda function) or a user-defined function
         as a SQL function.
 
-        :param name: name of the user-defined function in SQL statements.
-        :param f: a Python function, or a user-defined function. The user-defined function can
+        .. versionadded:: 1.3.1
+
+        Parameters
+        ----------
+        name : str,
+            name of the user-defined function in SQL statements.
+        f : function, :meth:`pyspark.sql.functions.udf` or :meth:`pyspark.sql.functions.pandas_udf`
+            a Python function, or a user-defined function. The user-defined function can
             be either row-at-a-time or vectorized. See :meth:`pyspark.sql.functions.udf` and
             :meth:`pyspark.sql.functions.pandas_udf`.
-        :param returnType: the return type of the registered user-defined function. The value can
+        returnType : :class:`pyspark.sql.types.DataType` or str, optional
+            the return type of the registered user-defined function. The value can
             be either a :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string.
-        :return: a user-defined function.
+            `returnType` can be optionally specified when `f` is a Python function but not
+            when `f` is a user-defined function. Please see the examples below.
+
+        Returns
+        -------
+        function
+            a user-defined function
 
+        Notes
+        -----
         To register a nondeterministic Python function, users need to first build
         a nondeterministic user-defined function for the Python function and then register it
         as a SQL function.
 
-        `returnType` can be optionally specified when `f` is a Python function but not
-        when `f` is a user-defined function. Please see below.
-
+        Examples
+        --------
         1. When `f` is a Python function:
 
             `returnType` defaults to string type and can be optionally specified. The produced
@@ -261,10 +276,10 @@ def register(self, name, f, returnType=None):
 
             >>> strlen = spark.udf.register("stringLengthString", lambda x: len(x))
             >>> spark.sql("SELECT stringLengthString('test')").collect()
-            [Row(stringLengthString(test)=u'4')]
+            [Row(stringLengthString(test)='4')]
 
             >>> spark.sql("SELECT 'foo' AS text").select(strlen("text")).collect()
-            [Row(stringLengthString(text)=u'3')]
+            [Row(stringLengthString(text)='3')]
 
             >>> from pyspark.sql.types import IntegerType
             >>> _ = spark.udf.register("stringLengthInt", lambda x: len(x), IntegerType())
@@ -276,7 +291,7 @@ def register(self, name, f, returnType=None):
             >>> spark.sql("SELECT stringLengthInt('test')").collect()
             [Row(stringLengthInt(test)=4)]
 
-        2. When `f` is a user-defined function:
+        2. When `f` is a user-defined function (from Spark 2.3.0):
 
             Spark uses the return type of the given user-defined function as the return type of
             the registered user-defined function. `returnType` should not be specified.
@@ -316,8 +331,6 @@ def register(self, name, f, returnType=None):
             >>> spark.sql(q).collect()  # doctest: +SKIP
             [Row(sum_udf(v1)=1), Row(sum_udf(v1)=5)]
 
-            .. note:: Registration for a user-defined function (case 2.) was added from
-                Spark 2.3.0.
         """
 
         # This is to check whether the input function is from a user-defined function or
@@ -336,46 +349,56 @@ def register(self, name, f, returnType=None):
                     "Invalid f: f must be SQL_BATCHED_UDF, SQL_SCALAR_PANDAS_UDF, "
                     "SQL_SCALAR_PANDAS_ITER_UDF, SQL_GROUPED_AGG_PANDAS_UDF or "
                     "SQL_MAP_PANDAS_ITER_UDF.")
-            register_udf = UserDefinedFunction(f.func, returnType=f.returnType, name=name,
-                                               evalType=f.evalType,
-                                               deterministic=f.deterministic)
+            register_udf = _create_udf(
+                f.func, returnType=f.returnType, name=name,
+                evalType=f.evalType, deterministic=f.deterministic)._unwrapped
             return_udf = f
         else:
             if returnType is None:
                 returnType = StringType()
-            register_udf = UserDefinedFunction(f, returnType=returnType, name=name,
-                                               evalType=PythonEvalType.SQL_BATCHED_UDF)
-            return_udf = register_udf._wrapped()
+            return_udf = _create_udf(
+                f, returnType=returnType, evalType=PythonEvalType.SQL_BATCHED_UDF, name=name)
+            register_udf = return_udf._unwrapped
         self.sparkSession._jsparkSession.udf().registerPython(name, register_udf._judf)
         return return_udf
 
-    @ignore_unicode_prefix
-    @since(2.3)
     def registerJavaFunction(self, name, javaClassName, returnType=None):
         """Register a Java user-defined function as a SQL function.
 
         In addition to a name and the function itself, the return type can be optionally specified.
         When the return type is not specified we would infer it via reflection.
 
-        :param name: name of the user-defined function
-        :param javaClassName: fully qualified name of java class
-        :param returnType: the return type of the registered Java function. The value can be either
+        .. versionadded:: 2.3.0
+
+        Parameters
+        ----------
+        name : str
+            name of the user-defined function
+        javaClassName : str
+            fully qualified name of java class
+        returnType : :class:`pyspark.sql.types.DataType` or str, optional
+            the return type of the registered Java function. The value can be either
             a :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string.
 
+        Examples
+        --------
         >>> from pyspark.sql.types import IntegerType
         >>> spark.udf.registerJavaFunction(
         ...     "javaStringLength", "test.org.apache.spark.sql.JavaStringLength", IntegerType())
-        >>> spark.sql("SELECT javaStringLength('test')").collect()
+        ... # doctest: +SKIP
+        >>> spark.sql("SELECT javaStringLength('test')").collect()  # doctest: +SKIP
         [Row(javaStringLength(test)=4)]
 
         >>> spark.udf.registerJavaFunction(
         ...     "javaStringLength2", "test.org.apache.spark.sql.JavaStringLength")
-        >>> spark.sql("SELECT javaStringLength2('test')").collect()
+        ... # doctest: +SKIP
+        >>> spark.sql("SELECT javaStringLength2('test')").collect()  # doctest: +SKIP
         [Row(javaStringLength2(test)=4)]
 
         >>> spark.udf.registerJavaFunction(
         ...     "javaStringLength3", "test.org.apache.spark.sql.JavaStringLength", "integer")
-        >>> spark.sql("SELECT javaStringLength3('test')").collect()
+        ... # doctest: +SKIP
+        >>> spark.sql("SELECT javaStringLength3('test')").collect()  # doctest: +SKIP
         [Row(javaStringLength3(test)=4)]
         """
 
@@ -386,20 +409,25 @@ def registerJavaFunction(self, name, javaClassName, returnType=None):
             jdt = self.sparkSession._jsparkSession.parseDataType(returnType.json())
         self.sparkSession._jsparkSession.udf().registerJava(name, javaClassName, jdt)
 
-    @ignore_unicode_prefix
-    @since(2.3)
     def registerJavaUDAF(self, name, javaClassName):
         """Register a Java user-defined aggregate function as a SQL function.
 
-        :param name: name of the user-defined aggregate function
-        :param javaClassName: fully qualified name of java class
+        .. versionadded:: 2.3.0
+
+        name : str
+            name of the user-defined aggregate function
+        javaClassName : str
+            fully qualified name of java class
 
+        Examples
+        --------
         >>> spark.udf.registerJavaUDAF("javaUDAF", "test.org.apache.spark.sql.MyDoubleAvg")
+        ... # doctest: +SKIP
         >>> df = spark.createDataFrame([(1, "a"),(2, "b"), (3, "a")],["id", "name"])
         >>> df.createOrReplaceTempView("df")
-        >>> spark.sql("SELECT name, javaUDAF(id) as avg from df group by name order by name desc") \
-                .collect()
-        [Row(name=u'b', avg=102.0), Row(name=u'a', avg=102.0)]
+        >>> q = "SELECT name, javaUDAF(id) as avg from df group by name order by name desc"
+        >>> spark.sql(q).collect()  # doctest: +SKIP
+        [Row(name='b', avg=102.0), Row(name='a', avg=102.0)]
         """
 
         self.sparkSession._jsparkSession.udf().registerJavaUDAF(name, javaClassName)
@@ -415,9 +443,6 @@ def _test():
         .appName("sql.udf tests")\
         .getOrCreate()
     globs['spark'] = spark
-    # Hack to skip the unit tests in register. These are currently being tested in proper tests.
-    # We should reenable this test once we completely drop Python 2.
-    del pyspark.sql.udf.UDFRegistration.register
     (failure_count, test_count) = doctest.testmod(
         pyspark.sql.udf, globs=globs,
         optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE)
diff --git a/python/pyspark/sql/udf.pyi b/python/pyspark/sql/udf.pyi
new file mode 100644
index 0000000000000..ea61397a67ba1
--- /dev/null
+++ b/python/pyspark/sql/udf.pyi
@@ -0,0 +1,58 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Any, Callable, Optional
+
+from pyspark.sql._typing import ColumnOrName, DataTypeOrString, UserDefinedFunctionLike
+from pyspark.sql.column import Column
+from pyspark.sql.types import DataType
+import pyspark.sql.session
+
+class UserDefinedFunction:
+    func: Callable[..., Any]
+    evalType: int
+    deterministic: bool
+    def __init__(
+        self,
+        func: Callable[..., Any],
+        returnType: DataTypeOrString = ...,
+        name: Optional[str] = ...,
+        evalType: int = ...,
+        deterministic: bool = ...,
+    ) -> None: ...
+    @property
+    def returnType(self) -> DataType: ...
+    def __call__(self, *cols: ColumnOrName) -> Column: ...
+    def asNondeterministic(self) -> UserDefinedFunction: ...
+
+class UDFRegistration:
+    sparkSession: pyspark.sql.session.SparkSession
+    def __init__(self, sparkSession: pyspark.sql.session.SparkSession) -> None: ...
+    def register(
+        self,
+        name: str,
+        f: Callable[..., Any],
+        returnType: Optional[DataTypeOrString] = ...,
+    ) -> UserDefinedFunctionLike: ...
+    def registerJavaFunction(
+        self,
+        name: str,
+        javaClassName: str,
+        returnType: Optional[DataTypeOrString] = ...,
+    ) -> None: ...
+    def registerJavaUDAF(self, name: str, javaClassName: str) -> None: ...
diff --git a/python/pyspark/sql/utils.py b/python/pyspark/sql/utils.py
index 147ac3325efd9..f5db783d2b5bc 100644
--- a/python/pyspark/sql/utils.py
+++ b/python/pyspark/sql/utils.py
@@ -16,10 +16,8 @@
 #
 
 import py4j
-import sys
 
-if sys.version_info.major >= 3:
-    unicode = str
+from pyspark import SparkContext
 
 
 class CapturedException(Exception):
@@ -29,12 +27,12 @@ def __init__(self, desc, stackTrace, cause=None):
         self.cause = convert_exception(cause) if cause is not None else None
 
     def __str__(self):
+        sql_conf = SparkContext._jvm.org.apache.spark.sql.internal.SQLConf.get()
+        debug_enabled = sql_conf.pysparkJVMStacktraceEnabled()
         desc = self.desc
-        # encode unicode instance for python2 for human readable description
-        if sys.version_info.major < 3 and isinstance(desc, unicode):
-            return str(desc.encode('utf-8'))
-        else:
-            return str(desc)
+        if debug_enabled:
+            desc = desc + "\n\nJVM stacktrace:\n%s" % self.stackTrace
+        return str(desc)
 
 
 class AnalysisException(CapturedException):
@@ -67,6 +65,12 @@ class QueryExecutionException(CapturedException):
     """
 
 
+class PythonException(CapturedException):
+    """
+    Exceptions thrown from Python workers.
+    """
+
+
 class UnknownException(CapturedException):
     """
     None of the above exceptions.
@@ -75,21 +79,30 @@ class UnknownException(CapturedException):
 
 def convert_exception(e):
     s = e.toString()
-    stackTrace = '\n\t at '.join(map(lambda x: x.toString(), e.getStackTrace()))
     c = e.getCause()
+    stacktrace = SparkContext._jvm.org.apache.spark.util.Utils.exceptionString(e)
+
     if s.startswith('org.apache.spark.sql.AnalysisException: '):
-        return AnalysisException(s.split(': ', 1)[1], stackTrace, c)
+        return AnalysisException(s.split(': ', 1)[1], stacktrace, c)
     if s.startswith('org.apache.spark.sql.catalyst.analysis'):
-        return AnalysisException(s.split(': ', 1)[1], stackTrace, c)
+        return AnalysisException(s.split(': ', 1)[1], stacktrace, c)
     if s.startswith('org.apache.spark.sql.catalyst.parser.ParseException: '):
-        return ParseException(s.split(': ', 1)[1], stackTrace, c)
+        return ParseException(s.split(': ', 1)[1], stacktrace, c)
     if s.startswith('org.apache.spark.sql.streaming.StreamingQueryException: '):
-        return StreamingQueryException(s.split(': ', 1)[1], stackTrace, c)
+        return StreamingQueryException(s.split(': ', 1)[1], stacktrace, c)
     if s.startswith('org.apache.spark.sql.execution.QueryExecutionException: '):
-        return QueryExecutionException(s.split(': ', 1)[1], stackTrace, c)
+        return QueryExecutionException(s.split(': ', 1)[1], stacktrace, c)
     if s.startswith('java.lang.IllegalArgumentException: '):
-        return IllegalArgumentException(s.split(': ', 1)[1], stackTrace, c)
-    return UnknownException(s, stackTrace, c)
+        return IllegalArgumentException(s.split(': ', 1)[1], stacktrace, c)
+    if c is not None and (
+            c.toString().startswith('org.apache.spark.api.python.PythonException: ')
+            # To make sure this only catches Python UDFs.
+            and any(map(lambda v: "org.apache.spark.sql.execution.python" in v.toString(),
+                        c.getStackTrace()))):
+        msg = ("\n  An exception was thrown from the Python worker. "
+               "Please see the stack trace below.\n%s" % c.getMessage())
+        return PythonException(msg, stacktrace)
+    return UnknownException(s, stacktrace, c)
 
 
 def capture_sql_exception(f):
@@ -99,7 +112,9 @@ def deco(*a, **kw):
         except py4j.protocol.Py4JJavaError as e:
             converted = convert_exception(e.java_exception)
             if not isinstance(converted, UnknownException):
-                raise converted
+                # Hide where the exception came from that shows a non-Pythonic
+                # JVM exception message.
+                raise converted from None
             else:
                 raise
     return deco
@@ -126,14 +141,20 @@ def install_exception_handler():
 def toJArray(gateway, jtype, arr):
     """
     Convert python list to java type array
-    :param gateway: Py4j Gateway
-    :param jtype: java type of element in array
-    :param arr: python type list
+
+    Parameters
+    ----------
+    gateway :
+        Py4j Gateway
+    jtype :
+        java type of element in array
+    arr :
+        python type list
     """
-    jarr = gateway.new_array(jtype, len(arr))
+    jarray = gateway.new_array(jtype, len(arr))
     for i in range(0, len(arr)):
-        jarr[i] = arr[i]
-    return jarr
+        jarray[i] = arr[i]
+    return jarray
 
 
 def require_test_compiled():
diff --git a/python/pyspark/sql/window.py b/python/pyspark/sql/window.py
index 82f74346ba928..753ac6e10b3b3 100644
--- a/python/pyspark/sql/window.py
+++ b/python/pyspark/sql/window.py
@@ -34,19 +34,21 @@ class Window(object):
     """
     Utility functions for defining window in DataFrames.
 
-    For example:
+    .. versionadded:: 1.4
+
+    Notes
+    -----
+    When ordering is not defined, an unbounded window frame (rowFrame,
+    unboundedPreceding, unboundedFollowing) is used by default. When ordering is defined,
+    a growing window frame (rangeFrame, unboundedPreceding, currentRow) is used by default.
 
+    Examples
+    --------
     >>> # ORDER BY date ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
     >>> window = Window.orderBy("date").rowsBetween(Window.unboundedPreceding, Window.currentRow)
 
     >>> # PARTITION BY country ORDER BY date RANGE BETWEEN 3 PRECEDING AND 3 FOLLOWING
     >>> window = Window.orderBy("date").partitionBy("country").rangeBetween(-3, 3)
-
-    .. note:: When ordering is not defined, an unbounded window frame (rowFrame,
-         unboundedPreceding, unboundedFollowing) is used by default. When ordering is defined,
-         a growing window frame (rangeFrame, unboundedPreceding, currentRow) is used by default.
-
-    .. versionadded:: 1.4
     """
 
     _JAVA_MIN_LONG = -(1 << 63)  # -9223372036854775808
@@ -81,7 +83,6 @@ def orderBy(*cols):
         return WindowSpec(jspec)
 
     @staticmethod
-    @since(2.1)
     def rowsBetween(start, end):
         """
         Creates a :class:`WindowSpec` with the frame boundaries defined,
@@ -101,6 +102,21 @@ def rowsBetween(start, end):
         offset of -1 and a upper bound offset of +2. The frame for row with index 5 would range from
         index 4 to index 7.
 
+        .. versionadded:: 2.1.0
+
+        Parameters
+        ----------
+        start : int
+            boundary start, inclusive.
+            The frame is unbounded if this is ``Window.unboundedPreceding``, or
+            any value less than or equal to -9223372036854775808.
+        end : int
+            boundary end, inclusive.
+            The frame is unbounded if this is ``Window.unboundedFollowing``, or
+            any value greater than or equal to 9223372036854775807.
+
+        Examples
+        --------
         >>> from pyspark.sql import Window
         >>> from pyspark.sql import functions as func
         >>> from pyspark.sql import SQLContext
@@ -121,12 +137,6 @@ def rowsBetween(start, end):
         |  3|       b|  3|
         +---+--------+---+
 
-        :param start: boundary start, inclusive.
-                      The frame is unbounded if this is ``Window.unboundedPreceding``, or
-                      any value less than or equal to -9223372036854775808.
-        :param end: boundary end, inclusive.
-                    The frame is unbounded if this is ``Window.unboundedFollowing``, or
-                    any value greater than or equal to 9223372036854775807.
         """
         if start <= Window._PRECEDING_THRESHOLD:
             start = Window.unboundedPreceding
@@ -137,7 +147,6 @@ def rowsBetween(start, end):
         return WindowSpec(jspec)
 
     @staticmethod
-    @since(2.1)
     def rangeBetween(start, end):
         """
         Creates a :class:`WindowSpec` with the frame boundaries defined,
@@ -160,6 +169,21 @@ def rangeBetween(start, end):
         unbounded, because no value modification is needed, in this case multiple and non-numeric
         ORDER BY expression are allowed.
 
+        .. versionadded:: 2.1.0
+
+        Parameters
+        ----------
+        start : int
+            boundary start, inclusive.
+            The frame is unbounded if this is ``Window.unboundedPreceding``, or
+            any value less than or equal to max(-sys.maxsize, -9223372036854775808).
+        end : int
+            boundary end, inclusive.
+            The frame is unbounded if this is ``Window.unboundedFollowing``, or
+            any value greater than or equal to min(sys.maxsize, 9223372036854775807).
+
+        Examples
+        --------
         >>> from pyspark.sql import Window
         >>> from pyspark.sql import functions as func
         >>> from pyspark.sql import SQLContext
@@ -180,12 +204,6 @@ def rangeBetween(start, end):
         |  3|       b|  3|
         +---+--------+---+
 
-        :param start: boundary start, inclusive.
-                      The frame is unbounded if this is ``Window.unboundedPreceding``, or
-                      any value less than or equal to max(-sys.maxsize, -9223372036854775808).
-        :param end: boundary end, inclusive.
-                    The frame is unbounded if this is ``Window.unboundedFollowing``, or
-                    any value greater than or equal to min(sys.maxsize, 9223372036854775807).
         """
         if start <= Window._PRECEDING_THRESHOLD:
             start = Window.unboundedPreceding
@@ -203,31 +221,38 @@ class WindowSpec(object):
 
     Use the static methods in :class:`Window` to create a :class:`WindowSpec`.
 
-    .. versionadded:: 1.4
+    .. versionadded:: 1.4.0
     """
 
     def __init__(self, jspec):
         self._jspec = jspec
 
-    @since(1.4)
     def partitionBy(self, *cols):
         """
         Defines the partitioning columns in a :class:`WindowSpec`.
 
-        :param cols: names of columns or expressions
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        cols : str, :class:`Column` or list
+            names of columns or expressions
         """
         return WindowSpec(self._jspec.partitionBy(_to_java_cols(cols)))
 
-    @since(1.4)
     def orderBy(self, *cols):
         """
         Defines the ordering columns in a :class:`WindowSpec`.
 
-        :param cols: names of columns or expressions
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        cols : str, :class:`Column` or list
+            names of columns or expressions
         """
         return WindowSpec(self._jspec.orderBy(_to_java_cols(cols)))
 
-    @since(1.4)
     def rowsBetween(self, start, end):
         """
         Defines the frame boundaries, from `start` (inclusive) to `end` (inclusive).
@@ -240,12 +265,18 @@ def rowsBetween(self, start, end):
         and ``Window.currentRow`` to specify special boundary values, rather than using integral
         values directly.
 
-        :param start: boundary start, inclusive.
-                      The frame is unbounded if this is ``Window.unboundedPreceding``, or
-                      any value less than or equal to max(-sys.maxsize, -9223372036854775808).
-        :param end: boundary end, inclusive.
-                    The frame is unbounded if this is ``Window.unboundedFollowing``, or
-                    any value greater than or equal to min(sys.maxsize, 9223372036854775807).
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        start : int
+            boundary start, inclusive.
+            The frame is unbounded if this is ``Window.unboundedPreceding``, or
+            any value less than or equal to max(-sys.maxsize, -9223372036854775808).
+        end : int
+            boundary end, inclusive.
+            The frame is unbounded if this is ``Window.unboundedFollowing``, or
+            any value greater than or equal to min(sys.maxsize, 9223372036854775807).
         """
         if start <= Window._PRECEDING_THRESHOLD:
             start = Window.unboundedPreceding
@@ -253,7 +284,6 @@ def rowsBetween(self, start, end):
             end = Window.unboundedFollowing
         return WindowSpec(self._jspec.rowsBetween(start, end))
 
-    @since(1.4)
     def rangeBetween(self, start, end):
         """
         Defines the frame boundaries, from `start` (inclusive) to `end` (inclusive).
@@ -266,12 +296,18 @@ def rangeBetween(self, start, end):
         and ``Window.currentRow`` to specify special boundary values, rather than using integral
         values directly.
 
-        :param start: boundary start, inclusive.
-                      The frame is unbounded if this is ``Window.unboundedPreceding``, or
-                      any value less than or equal to max(-sys.maxsize, -9223372036854775808).
-        :param end: boundary end, inclusive.
-                    The frame is unbounded if this is ``Window.unboundedFollowing``, or
-                    any value greater than or equal to min(sys.maxsize, 9223372036854775807).
+        .. versionadded:: 1.4.0
+
+        Parameters
+        ----------
+        start : int
+            boundary start, inclusive.
+            The frame is unbounded if this is ``Window.unboundedPreceding``, or
+            any value less than or equal to max(-sys.maxsize, -9223372036854775808).
+        end : int
+            boundary end, inclusive.
+            The frame is unbounded if this is ``Window.unboundedFollowing``, or
+            any value greater than or equal to min(sys.maxsize, 9223372036854775807).
         """
         if start <= Window._PRECEDING_THRESHOLD:
             start = Window.unboundedPreceding
diff --git a/python/pyspark/sql/window.pyi b/python/pyspark/sql/window.pyi
new file mode 100644
index 0000000000000..4fdc468df112a
--- /dev/null
+++ b/python/pyspark/sql/window.pyi
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from typing import List, Union
+
+from pyspark.sql._typing import ColumnOrName
+from py4j.java_gateway import JavaObject  # type: ignore[import]
+
+class Window:
+    unboundedPreceding: int
+    unboundedFollowing: int
+    currentRow: int
+    @staticmethod
+    def partitionBy(*cols: Union[ColumnOrName, List[ColumnOrName]]) -> WindowSpec: ...
+    @staticmethod
+    def orderBy(*cols: Union[ColumnOrName, List[ColumnOrName]]) -> WindowSpec: ...
+    @staticmethod
+    def rowsBetween(start: int, end: int) -> WindowSpec: ...
+    @staticmethod
+    def rangeBetween(start: int, end: int) -> WindowSpec: ...
+
+class WindowSpec:
+    def __init__(self, jspec: JavaObject) -> None: ...
+    def partitionBy(self, *cols: Union[ColumnOrName, List[ColumnOrName]]) -> WindowSpec: ...
+    def orderBy(self, *cols: Union[ColumnOrName, List[ColumnOrName]]) -> WindowSpec: ...
+    def rowsBetween(self, start: int, end: int) -> WindowSpec: ...
+    def rangeBetween(self, start: int, end: int) -> WindowSpec: ...
diff --git a/python/pyspark/statcounter.py b/python/pyspark/statcounter.py
index 03ea0b6d33c9d..9e56c43f873d9 100644
--- a/python/pyspark/statcounter.py
+++ b/python/pyspark/statcounter.py
@@ -134,6 +134,8 @@ def sampleStdev(self):
     def asDict(self, sample=False):
         """Returns the :class:`StatCounter` members as a ``dict``.
 
+        Examples
+        --------
         >>> sc.parallelize([1., 2., 3., 4.]).stats().asDict()
         {'count': 4L,
          'max': 4.0,
diff --git a/python/pyspark/statcounter.pyi b/python/pyspark/statcounter.pyi
new file mode 100644
index 0000000000000..38e5970501527
--- /dev/null
+++ b/python/pyspark/statcounter.pyi
@@ -0,0 +1,44 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Any, Dict, Iterable, Optional, Union
+
+maximum: Any
+minimum: Any
+sqrt: Any
+
+class StatCounter:
+    n: int
+    mu: float
+    m2: float
+    maxValue: float
+    minValue: float
+    def __init__(self, values: Optional[Iterable[float]] = ...) -> None: ...
+    def merge(self, value: float) -> StatCounter: ...
+    def mergeStats(self, other: StatCounter) -> StatCounter: ...
+    def copy(self) -> StatCounter: ...
+    def count(self) -> int: ...
+    def mean(self) -> float: ...
+    def sum(self) -> float: ...
+    def min(self) -> float: ...
+    def max(self) -> float: ...
+    def variance(self) -> float: ...
+    def sampleVariance(self) -> float: ...
+    def stdev(self) -> float: ...
+    def sampleStdev(self) -> float: ...
+    def asDict(self, sample: bool = ...) -> Dict[str, Union[float, int]]: ...
diff --git a/python/pyspark/status.pyi b/python/pyspark/status.pyi
new file mode 100644
index 0000000000000..0558e245f49cc
--- /dev/null
+++ b/python/pyspark/status.pyi
@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import List, NamedTuple, Optional
+from py4j.java_gateway import JavaArray, JavaObject  # type: ignore[import]
+
+class SparkJobInfo(NamedTuple):
+    jobId: int
+    stageIds: JavaArray
+    status: str
+
+class SparkStageInfo(NamedTuple):
+    stageId: int
+    currentAttemptId: int
+    name: str
+    numTasks: int
+    numActiveTasks: int
+    numCompletedTasks: int
+    numFailedTasks: int
+
+class StatusTracker:
+    def __init__(self, jtracker: JavaObject) -> None: ...
+    def getJobIdsForGroup(self, jobGroup: Optional[str] = ...) -> List[int]: ...
+    def getActiveStageIds(self) -> List[int]: ...
+    def getActiveJobsIds(self) -> List[int]: ...
+    def getJobInfo(self, jobId: int) -> SparkJobInfo: ...
+    def getStageInfo(self, stageId: int) -> SparkStageInfo: ...
diff --git a/python/pyspark/storagelevel.py b/python/pyspark/storagelevel.py
index 951af45bb3227..ecf8e5c82ea56 100644
--- a/python/pyspark/storagelevel.py
+++ b/python/pyspark/storagelevel.py
@@ -51,8 +51,10 @@ def __str__(self):
 
 StorageLevel.DISK_ONLY = StorageLevel(True, False, False, False)
 StorageLevel.DISK_ONLY_2 = StorageLevel(True, False, False, False, 2)
+StorageLevel.DISK_ONLY_3 = StorageLevel(True, False, False, False, 3)
 StorageLevel.MEMORY_ONLY = StorageLevel(False, True, False, False)
 StorageLevel.MEMORY_ONLY_2 = StorageLevel(False, True, False, False, 2)
 StorageLevel.MEMORY_AND_DISK = StorageLevel(True, True, False, False)
 StorageLevel.MEMORY_AND_DISK_2 = StorageLevel(True, True, False, False, 2)
 StorageLevel.OFF_HEAP = StorageLevel(True, True, True, False, 1)
+StorageLevel.MEMORY_AND_DISK_DESER = StorageLevel(True, True, False, True)
diff --git a/python/pyspark/storagelevel.pyi b/python/pyspark/storagelevel.pyi
new file mode 100644
index 0000000000000..2eb05850bae78
--- /dev/null
+++ b/python/pyspark/storagelevel.pyi
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import ClassVar
+
+class StorageLevel:
+    DISK_ONLY: ClassVar[StorageLevel]
+    DISK_ONLY_2: ClassVar[StorageLevel]
+    MEMORY_ONLY: ClassVar[StorageLevel]
+    MEMORY_ONLY_2: ClassVar[StorageLevel]
+    DISK_ONLY_3: ClassVar[StorageLevel]
+    MEMORY_AND_DISK: ClassVar[StorageLevel]
+    MEMORY_AND_DISK_2: ClassVar[StorageLevel]
+    OFF_HEAP: ClassVar[StorageLevel]
+
+    useDisk: bool
+    useMemory: bool
+    useOffHeap: bool
+    deserialized: bool
+    replication: int
+    def __init__(
+        self,
+        useDisk: bool,
+        useMemory: bool,
+        useOffHeap: bool,
+        deserialized: bool,
+        replication: int = ...,
+    ) -> None: ...
diff --git a/python/pyspark/streaming/context.py b/python/pyspark/streaming/context.py
index 769121c19ff4d..2e6d7ede88551 100644
--- a/python/pyspark/streaming/context.py
+++ b/python/pyspark/streaming/context.py
@@ -15,9 +15,7 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
-from py4j.java_gateway import java_import
+from py4j.java_gateway import java_import, is_instance_of
 
 from pyspark import RDD, SparkConf
 from pyspark.serializers import NoOpSerializer, UTF8Deserializer, CloudPickleSerializer
@@ -38,6 +36,14 @@ class StreamingContext(object):
     be started and stopped using `context.start()` and `context.stop()`,
     respectively. `context.awaitTermination()` allows the current thread
     to wait for the termination of the context by `stop()` or by an exception.
+
+    Parameters
+    ----------
+    sparkContext : :class:`SparkContext`
+        SparkContext object.
+    batchDuration : int, optional
+        the time interval (in seconds) at which streaming
+        data will be divided into batches
     """
     _transformerSerializer = None
 
@@ -45,13 +51,6 @@ class StreamingContext(object):
     _activeContext = None
 
     def __init__(self, sparkContext, batchDuration=None, jssc=None):
-        """
-        Create a new StreamingContext.
-
-        :param sparkContext: :class:`SparkContext` object.
-        :param batchDuration: the time interval (in seconds) at which streaming
-                              data will be divided into batches
-        """
 
         self._sc = sparkContext
         self._jvm = self._sc._jvm
@@ -92,8 +91,12 @@ def getOrCreate(cls, checkpointPath, setupFunc):
         recreated from the checkpoint data. If the data does not exist, then the provided setupFunc
         will be used to create a new context.
 
-        :param checkpointPath: Checkpoint directory used in an earlier streaming program
-        :param setupFunc:      Function to create a new context and setup DStreams
+        Parameters
+        ----------
+        checkpointPath : str
+            Checkpoint directory used in an earlier streaming program
+        setupFunc : function
+            Function to create a new context and setup DStreams
         """
         cls._ensure_initialized()
         gw = SparkContext._gateway
@@ -149,10 +152,14 @@ def getActiveOrCreate(cls, checkpointPath, setupFunc):
         valid checkpoint data, then setupFunc will be called to create a new context and setup
         DStreams.
 
-        :param checkpointPath: Checkpoint directory used in an earlier streaming program. Can be
-                               None if the intention is to always create a new context when there
-                               is no active context.
-        :param setupFunc:      Function to create a new JavaStreamingContext and setup DStreams
+        Parameters
+        ----------
+        checkpointPath : str
+            Checkpoint directory used in an earlier streaming program. Can be
+            None if the intention is to always create a new context when there
+            is no active context.
+        setupFunc : function
+            Function to create a new JavaStreamingContext and setup DStreams
         """
 
         if setupFunc is None:
@@ -183,7 +190,10 @@ def awaitTermination(self, timeout=None):
         """
         Wait for the execution to stop.
 
-        :param timeout: time to wait in seconds
+        Parameters
+        ----------
+        timeout : int, optional
+            time to wait in seconds
         """
         if timeout is None:
             self._jssc.awaitTermination()
@@ -196,7 +206,10 @@ def awaitTerminationOrTimeout(self, timeout):
         throw the reported error during the execution; or `false` if the
         waiting time elapsed before returning from the method.
 
-        :param timeout: time to wait in seconds
+        Parameters
+        ----------
+        timeout : int
+            time to wait in seconds
         """
         return self._jssc.awaitTerminationOrTimeout(int(timeout * 1000))
 
@@ -205,9 +218,13 @@ def stop(self, stopSparkContext=True, stopGraceFully=False):
         Stop the execution of the streams, with option of ensuring all
         received data has been processed.
 
-        :param stopSparkContext: Stop the associated SparkContext or not
-        :param stopGracefully: Stop gracefully by waiting for the processing
-                              of all received data to be completed
+        Parameters
+        ----------
+        stopSparkContext : bool, optional
+            Stop the associated SparkContext or not
+        stopGracefully : bool, optional
+            Stop gracefully by waiting for the processing of all received
+            data to be completed
         """
         self._jssc.stop(stopSparkContext, stopGraceFully)
         StreamingContext._activeContext = None
@@ -223,8 +240,10 @@ def remember(self, duration):
         the RDDs (if the developer wishes to query old data outside the
         DStream computation).
 
-        :param duration: Minimum duration (in seconds) that each DStream
-                        should remember its RDDs
+        Parameters
+        ----------
+        duration : int
+            Minimum duration (in seconds) that each DStream should remember its RDDs
         """
         self._jssc.remember(self._jduration(duration))
 
@@ -233,8 +252,10 @@ def checkpoint(self, directory):
         Sets the context to periodically checkpoint the DStream operations for master
         fault-tolerance. The graph will be checkpointed every batch interval.
 
-        :param directory: HDFS-compatible directory where the checkpoint data
-                         will be reliably stored
+        Parameters
+        ----------
+        directory : str
+            HDFS-compatible directory where the checkpoint data will be reliably stored
         """
         self._jssc.checkpoint(directory)
 
@@ -244,9 +265,14 @@ def socketTextStream(self, hostname, port, storageLevel=StorageLevel.MEMORY_AND_
         a TCP socket and receive byte is interpreted as UTF8 encoded ``\\n`` delimited
         lines.
 
-        :param hostname:      Hostname to connect to for receiving data
-        :param port:          Port to connect to for receiving data
-        :param storageLevel:  Storage level to use for storing the received objects
+        Parameters
+        ----------
+        hostname : str
+            Hostname to connect to for receiving data
+        port : int
+            Port to connect to for receiving data
+        storageLevel : :class:`pyspark.StorageLevel`, optional
+            Storage level to use for storing the received objects
         """
         jlevel = self._sc._getJavaStorageLevel(storageLevel)
         return DStream(self._jssc.socketTextStream(hostname, port, jlevel), self,
@@ -255,7 +281,7 @@ def socketTextStream(self, hostname, port, storageLevel=StorageLevel.MEMORY_AND_
     def textFileStream(self, directory):
         """
         Create an input stream that monitors a Hadoop-compatible file system
-        for new files and reads them as text files. Files must be wrriten to the
+        for new files and reads them as text files. Files must be written to the
         monitored directory by "moving" them from another location within the same
         file system. File names starting with . are ignored.
         The text files must be encoded as UTF-8.
@@ -270,8 +296,12 @@ def binaryRecordsStream(self, directory, recordLength):
         them from another location within the same file system.
         File names starting with . are ignored.
 
-        :param directory:       Directory to load data from
-        :param recordLength:    Length of each record in bytes
+        Parameters
+        ----------
+        directory : str
+            Directory to load data from
+        recordLength : int
+            Length of each record in bytes
         """
         return DStream(self._jssc.binaryRecordsStream(directory, recordLength), self,
                        NoOpSerializer())
@@ -288,11 +318,18 @@ def queueStream(self, rdds, oneAtATime=True, default=None):
         Create an input stream from a queue of RDDs or list. In each batch,
         it will process either one or all of the RDDs returned by the queue.
 
-        .. note:: Changes to the queue after the stream is created will not be recognized.
+        Parameters
+        ----------
+        rdds : list
+            Queue of RDDs
+        oneAtATime : bool, optional
+            pick one rdd each time or pick all of them once.
+        default : :class:`pyspark.RDD`, optional
+            The default rdd if no more in rdds
 
-        :param rdds:       Queue of RDDs
-        :param oneAtATime: pick one rdd each time or pick all of them once.
-        :param default:    The default rdd if no more in rdds
+        Notes
+        -----
+        Changes to the queue after the stream is created will not be recognized.
         """
         if default and not isinstance(default, RDD):
             default = self._sc.parallelize(default)
@@ -341,8 +378,17 @@ def union(self, *dstreams):
             raise ValueError("All DStreams should have same serializer")
         if len(set(s._slideDuration for s in dstreams)) > 1:
             raise ValueError("All DStreams should have same slide duration")
-        cls = SparkContext._jvm.org.apache.spark.streaming.api.java.JavaDStream
-        jdstreams = SparkContext._gateway.new_array(cls, len(dstreams))
+        jdstream_cls = SparkContext._jvm.org.apache.spark.streaming.api.java.JavaDStream
+        jpair_dstream_cls = SparkContext._jvm.org.apache.spark.streaming.api.java.JavaPairDStream
+        gw = SparkContext._gateway
+        if is_instance_of(gw, dstreams[0]._jdstream, jdstream_cls):
+            cls = jdstream_cls
+        elif is_instance_of(gw, dstreams[0]._jdstream, jpair_dstream_cls):
+            cls = jpair_dstream_cls
+        else:
+            cls_name = dstreams[0]._jdstream.getClass().getCanonicalName()
+            raise TypeError("Unsupported Java DStream class %s" % cls_name)
+        jdstreams = gw.new_array(cls, len(dstreams))
         for i in range(0, len(dstreams)):
             jdstreams[i] = dstreams[i]._jdstream
         return DStream(self._jssc.union(jdstreams), self, dstreams[0]._jrdd_deserializer)
diff --git a/python/pyspark/streaming/context.pyi b/python/pyspark/streaming/context.pyi
new file mode 100644
index 0000000000000..117a6742e6b6b
--- /dev/null
+++ b/python/pyspark/streaming/context.pyi
@@ -0,0 +1,75 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Any, Callable, List, Optional, TypeVar
+
+from py4j.java_gateway import JavaObject  # type: ignore[import]
+
+from pyspark.context import SparkContext
+from pyspark.rdd import RDD
+from pyspark.storagelevel import StorageLevel
+from pyspark.streaming.dstream import DStream
+from pyspark.streaming.listener import StreamingListener
+
+T = TypeVar("T")
+
+class StreamingContext:
+    def __init__(
+        self,
+        sparkContext: SparkContext,
+        batchDuration: int = ...,
+        jssc: Optional[JavaObject] = ...,
+    ) -> None: ...
+    @classmethod
+    def getOrCreate(
+        cls, checkpointPath: str, setupFunc: Callable[[], StreamingContext]
+    ) -> StreamingContext: ...
+    @classmethod
+    def getActive(cls) -> StreamingContext: ...
+    @classmethod
+    def getActiveOrCreate(
+        cls, checkpointPath: str, setupFunc: Callable[[], StreamingContext]
+    ) -> StreamingContext: ...
+    @property
+    def sparkContext(self) -> SparkContext: ...
+    def start(self) -> None: ...
+    def awaitTermination(self, timeout: Optional[int] = ...) -> None: ...
+    def awaitTerminationOrTimeout(self, timeout: int) -> None: ...
+    def stop(
+        self, stopSparkContext: bool = ..., stopGraceFully: bool = ...
+    ) -> None: ...
+    def remember(self, duration: int) -> None: ...
+    def checkpoint(self, directory: str) -> None: ...
+    def socketTextStream(
+        self, hostname: str, port: int, storageLevel: StorageLevel = ...
+    ) -> DStream[str]: ...
+    def textFileStream(self, directory: str) -> DStream[str]: ...
+    def binaryRecordsStream(
+        self, directory: str, recordLength: int
+    ) -> DStream[bytes]: ...
+    def queueStream(
+        self,
+        rdds: List[RDD[T]],
+        oneAtATime: bool = ...,
+        default: Optional[RDD[T]] = ...,
+    ) -> DStream[T]: ...
+    def transform(
+        self, dstreams: List[DStream[Any]], transformFunc: Callable[..., RDD[T]]
+    ) -> DStream[T]: ...
+    def union(self, *dstreams: DStream[T]) -> DStream[T]: ...
+    def addStreamingListener(self, streamingListener: StreamingListener) -> None: ...
diff --git a/python/pyspark/streaming/dstream.py b/python/pyspark/streaming/dstream.py
index 60562a6c92aff..6ef164ae5a11a 100644
--- a/python/pyspark/streaming/dstream.py
+++ b/python/pyspark/streaming/dstream.py
@@ -15,17 +15,11 @@
 # limitations under the License.
 #
 
-import sys
 import operator
 import time
 from itertools import chain
 from datetime import datetime
 
-if sys.version < "3":
-    from itertools import imap as map, ifilter as filter
-else:
-    long = int
-
 from py4j.protocol import Py4JJavaError
 
 from pyspark import RDD
@@ -167,7 +161,10 @@ def pprint(self, num=10):
         """
         Print the first num elements of each RDD generated in this DStream.
 
-        :param num: the number of elements from the first will be printed.
+        Parameters
+        ----------
+        num : int, optional
+            the number of elements from the first will be printed.
         """
         def takeAndPrint(time, rdd):
             taken = rdd.take(num + 1)
@@ -229,8 +226,11 @@ def checkpoint(self, interval):
         """
         Enable periodic checkpointing of RDDs of this DStream
 
-        :param interval: time in seconds, after each period of that, generated
-                         RDD will be checkpointed
+        Parameters
+        ----------
+        interval : int
+            time in seconds, after each period of that, generated
+            RDD will be checkpointed
         """
         self.is_checkpointed = True
         self._jdstream.checkpoint(self._ssc._jduration(interval))
@@ -333,8 +333,11 @@ def union(self, other):
         """
         Return a new DStream by unifying data of another DStream with this DStream.
 
-        :param other: Another DStream having the same interval (i.e., slideDuration)
-                     as this DStream.
+        Parameters
+        ----------
+        other : :class:`DStream`
+            Another DStream having the same interval (i.e., slideDuration)
+            as this DStream.
         """
         if self._slideDuration != other._slideDuration:
             raise ValueError("the two DStream should have same slide duration")
@@ -404,7 +407,7 @@ def _jtime(self, timestamp):
         """
         if isinstance(timestamp, datetime):
             timestamp = time.mktime(timestamp.timetuple())
-        return self._sc._jvm.Time(long(timestamp * 1000))
+        return self._sc._jvm.Time(int(timestamp * 1000))
 
     def slice(self, begin, end):
         """
@@ -418,10 +421,12 @@ def slice(self, begin, end):
     def _validate_window_param(self, window, slide):
         duration = self._jdstream.dstream().slideDuration().milliseconds()
         if int(window * 1000) % duration != 0:
-            raise ValueError("windowDuration must be multiple of the slide duration (%d ms)"
+            raise ValueError("windowDuration must be multiple of the parent "
+                             "dstream's slide (batch) duration (%d ms)"
                              % duration)
         if slide and int(slide * 1000) % duration != 0:
-            raise ValueError("slideDuration must be multiple of the slide duration (%d ms)"
+            raise ValueError("slideDuration must be multiple of the parent "
+                             "dstream's slide (batch) duration (%d ms)"
                              % duration)
 
     def window(self, windowDuration, slideDuration=None):
@@ -429,11 +434,15 @@ def window(self, windowDuration, slideDuration=None):
         Return a new DStream in which each RDD contains all the elements in seen in a
         sliding window of time over this DStream.
 
-        :param windowDuration: width of the window; must be a multiple of this DStream's
-                              batching interval
-        :param slideDuration:  sliding interval of the window (i.e., the interval after which
-                              the new DStream will generate RDDs); must be a multiple of this
-                              DStream's batching interval
+        Parameters
+        ----------
+        windowDuration : int
+            width of the window; must be a multiple of this DStream's
+            batching interval
+        slideDuration : int, optional
+            sliding interval of the window (i.e., the interval after which
+            the new DStream will generate RDDs); must be a multiple of this
+            DStream's batching interval
         """
         self._validate_window_param(windowDuration, slideDuration)
         d = self._ssc._jduration(windowDuration)
@@ -455,15 +464,21 @@ def reduceByWindow(self, reduceFunc, invReduceFunc, windowDuration, slideDuratio
         2. "inverse reduce" the old values that left the window (e.g., subtracting old counts)
         This is more efficient than `invReduceFunc` is None.
 
-        :param reduceFunc:     associative and commutative reduce function
-        :param invReduceFunc:  inverse reduce function of `reduceFunc`; such that for all y,
-                               and invertible x:
-                               `invReduceFunc(reduceFunc(x, y), x) = y`
-        :param windowDuration: width of the window; must be a multiple of this DStream's
-                               batching interval
-        :param slideDuration:  sliding interval of the window (i.e., the interval after which
-                               the new DStream will generate RDDs); must be a multiple of this
-                               DStream's batching interval
+        Parameters
+        ----------
+        reduceFunc : function
+            associative and commutative reduce function
+        invReduceFunc : function
+            inverse reduce function of `reduceFunc`; such that for all y,
+            and invertible x:
+            `invReduceFunc(reduceFunc(x, y), x) = y`
+        windowDuration : int
+            width of the window; must be a multiple of this DStream's
+            batching interval
+        slideDuration : int
+            sliding interval of the window (i.e., the interval after which
+            the new DStream will generate RDDs); must be a multiple of this
+            DStream's batching interval
         """
         keyed = self.map(lambda x: (1, x))
         reduced = keyed.reduceByKeyAndWindow(reduceFunc, invReduceFunc,
@@ -487,12 +502,17 @@ def countByValueAndWindow(self, windowDuration, slideDuration, numPartitions=Non
         Return a new DStream in which each RDD contains the count of distinct elements in
         RDDs in a sliding window over this DStream.
 
-        :param windowDuration: width of the window; must be a multiple of this DStream's
-                              batching interval
-        :param slideDuration:  sliding interval of the window (i.e., the interval after which
-                              the new DStream will generate RDDs); must be a multiple of this
-                              DStream's batching interval
-        :param numPartitions:  number of partitions of each RDD in the new DStream.
+        Parameters
+        ----------
+        windowDuration : int
+            width of the window; must be a multiple of this DStream's
+            batching interval
+        slideDuration : int
+            sliding interval of the window (i.e., the interval after which
+            the new DStream will generate RDDs); must be a multiple of this
+            DStream's batching interval
+        numPartitions : int, optional
+            number of partitions of each RDD in the new DStream.
         """
         keyed = self.map(lambda x: (x, 1))
         counted = keyed.reduceByKeyAndWindow(operator.add, operator.sub,
@@ -504,12 +524,17 @@ def groupByKeyAndWindow(self, windowDuration, slideDuration, numPartitions=None)
         Return a new DStream by applying `groupByKey` over a sliding window.
         Similar to `DStream.groupByKey()`, but applies it over a sliding window.
 
-        :param windowDuration: width of the window; must be a multiple of this DStream's
-                              batching interval
-        :param slideDuration:  sliding interval of the window (i.e., the interval after which
-                              the new DStream will generate RDDs); must be a multiple of this
-                              DStream's batching interval
-        :param numPartitions:  Number of partitions of each RDD in the new DStream.
+        Parameters
+        ----------
+        windowDuration : int
+            width of the window; must be a multiple of this DStream's
+            batching interval
+        slideDuration : int
+            sliding interval of the window (i.e., the interval after which
+            the new DStream will generate RDDs); must be a multiple of this
+            DStream's batching interval
+        numPartitions : int, optional
+            Number of partitions of each RDD in the new DStream.
         """
         ls = self.mapValues(lambda x: [x])
         grouped = ls.reduceByKeyAndWindow(lambda a, b: a.extend(b) or a, lambda a, b: a[len(b):],
@@ -528,17 +553,25 @@ def reduceByKeyAndWindow(self, func, invFunc, windowDuration, slideDuration=None
         `invFunc` can be None, then it will reduce all the RDDs in window, could be slower
         than having `invFunc`.
 
-        :param func:           associative and commutative reduce function
-        :param invFunc:        inverse function of `reduceFunc`
-        :param windowDuration: width of the window; must be a multiple of this DStream's
-                              batching interval
-        :param slideDuration:  sliding interval of the window (i.e., the interval after which
-                              the new DStream will generate RDDs); must be a multiple of this
-                              DStream's batching interval
-        :param numPartitions:  number of partitions of each RDD in the new DStream.
-        :param filterFunc:     function to filter expired key-value pairs;
-                              only pairs that satisfy the function are retained
-                              set this to null if you do not want to filter
+        Parameters
+        ----------
+        func : function
+            associative and commutative reduce function
+        invFunc : function
+            inverse function of `reduceFunc`
+        windowDuration : int
+            width of the window; must be a multiple of this DStream's
+            batching interval
+        slideDuration : int, optional
+            sliding interval of the window (i.e., the interval after which
+            the new DStream will generate RDDs); must be a multiple of this
+            DStream's batching interval
+        numPartitions : int, optional
+            number of partitions of each RDD in the new DStream.
+        filterFunc : function, optional
+            function to filter expired key-value pairs;
+            only pairs that satisfy the function are retained
+            set this to null if you do not want to filter
         """
         self._validate_window_param(windowDuration, slideDuration)
         if numPartitions is None:
@@ -578,8 +611,11 @@ def updateStateByKey(self, updateFunc, numPartitions=None, initialRDD=None):
         Return a new "state" DStream where the state for each key is updated by applying
         the given function on the previous state of the key and the new values of the key.
 
-        :param updateFunc: State update function. If this function returns None, then
-                           corresponding state key-value pair will be eliminated.
+        Parameters
+        ----------
+        updateFunc : function
+            State update function. If this function returns None, then
+            corresponding state key-value pair will be eliminated.
         """
         if numPartitions is None:
             numPartitions = self._sc.defaultParallelism
diff --git a/python/pyspark/streaming/dstream.pyi b/python/pyspark/streaming/dstream.pyi
new file mode 100644
index 0000000000000..1521d838fc2b5
--- /dev/null
+++ b/python/pyspark/streaming/dstream.pyi
@@ -0,0 +1,216 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import overload
+from typing import (
+    Callable,
+    Generic,
+    Hashable,
+    Iterable,
+    List,
+    Optional,
+    Tuple,
+    TypeVar,
+    Union,
+)
+import datetime
+from pyspark.rdd import RDD
+import pyspark.serializers
+from pyspark.storagelevel import StorageLevel
+import pyspark.streaming.context
+
+from py4j.java_gateway import JavaObject
+
+S = TypeVar("S")
+T = TypeVar("T")
+U = TypeVar("U")
+K = TypeVar("K", bound=Hashable)
+V = TypeVar("V")
+
+class DStream(Generic[T]):
+    is_cached: bool
+    is_checkpointed: bool
+    def __init__(
+        self,
+        jdstream: JavaObject,
+        ssc: pyspark.streaming.context.StreamingContext,
+        jrdd_deserializer: pyspark.serializers.Serializer,
+    ) -> None: ...
+    def context(self) -> pyspark.streaming.context.StreamingContext: ...
+    def count(self) -> DStream[int]: ...
+    def filter(self, f: Callable[[T], bool]) -> DStream[T]: ...
+    def flatMap(
+        self: DStream[T],
+        f: Callable[[T], Iterable[U]],
+        preservesPartitioning: bool = ...,
+    ) -> DStream[U]: ...
+    def map(
+        self: DStream[T], f: Callable[[T], U], preservesPartitioning: bool = ...
+    ) -> DStream[U]: ...
+    def mapPartitions(
+        self, f: Callable[[Iterable[T]], Iterable[U]], preservesPartitioning: bool = ...
+    ) -> DStream[U]: ...
+    def mapPartitionsWithIndex(
+        self,
+        f: Callable[[int, Iterable[T]], Iterable[U]],
+        preservesPartitioning: bool = ...,
+    ) -> DStream[U]: ...
+    def reduce(self, func: Callable[[T, T], T]) -> DStream[T]: ...
+    def reduceByKey(
+        self: DStream[Tuple[K, V]],
+        func: Callable[[V, V], V],
+        numPartitions: Optional[int] = ...,
+    ) -> DStream[Tuple[K, V]]: ...
+    def combineByKey(
+        self: DStream[Tuple[K, V]],
+        createCombiner: Callable[[V], U],
+        mergeValue: Callable[[U, V], U],
+        mergeCombiners: Callable[[U, U], U],
+        numPartitions: Optional[int] = ...,
+    ) -> DStream[Tuple[K, U]]: ...
+    def partitionBy(
+        self: DStream[Tuple[K, V]],
+        numPartitions: int,
+        partitionFunc: Callable[[K], int] = ...,
+    ) -> DStream[Tuple[K, V]]: ...
+    @overload
+    def foreachRDD(self, func: Callable[[RDD[T]], None]) -> None: ...
+    @overload
+    def foreachRDD(self, func: Callable[[datetime.datetime, RDD[T]], None]) -> None: ...
+    def pprint(self, num: int = ...) -> None: ...
+    def mapValues(
+        self: DStream[Tuple[K, V]], f: Callable[[V], U]
+    ) -> DStream[Tuple[K, U]]: ...
+    def flatMapValues(
+        self: DStream[Tuple[K, V]], f: Callable[[V], Iterable[U]]
+    ) -> DStream[Tuple[K, U]]: ...
+    def glom(self) -> DStream[List[T]]: ...
+    def cache(self) -> DStream[T]: ...
+    def persist(self, storageLevel: StorageLevel) -> DStream[T]: ...
+    def checkpoint(self, interval: int) -> DStream[T]: ...
+    def groupByKey(
+        self: DStream[Tuple[K, V]], numPartitions: Optional[int] = ...
+    ) -> DStream[Tuple[K, Iterable[V]]]: ...
+    def countByValue(self) -> DStream[Tuple[T, int]]: ...
+    def saveAsTextFiles(self, prefix: str, suffix: Optional[str] = ...) -> None: ...
+    @overload
+    def transform(self, func: Callable[[RDD[T]], RDD[U]]) -> TransformedDStream[U]: ...
+    @overload
+    def transform(
+        self, func: Callable[[datetime.datetime, RDD[T]], RDD[U]]
+    ) -> TransformedDStream[U]: ...
+    @overload
+    def transformWith(
+        self,
+        func: Callable[[RDD[T], RDD[U]], RDD[V]],
+        other: RDD[U],
+        keepSerializer: bool = ...,
+    ) -> DStream[V]: ...
+    @overload
+    def transformWith(
+        self,
+        func: Callable[[datetime.datetime, RDD[T], RDD[U]], RDD[V]],
+        other: RDD[U],
+        keepSerializer: bool = ...,
+    ) -> DStream[V]: ...
+    def repartition(self, numPartitions: int) -> DStream[T]: ...
+    def union(self, other: DStream[U]) -> DStream[Union[T, U]]: ...
+    def cogroup(
+        self: DStream[Tuple[K, V]],
+        other: DStream[Tuple[K, U]],
+        numPartitions: Optional[int] = ...,
+    ) -> DStream[Tuple[K, Tuple[List[V], List[U]]]]: ...
+    def join(
+        self: DStream[Tuple[K, V]],
+        other: DStream[Tuple[K, U]],
+        numPartitions: Optional[int] = ...,
+    ) -> DStream[Tuple[K, Tuple[V, U]]]: ...
+    def leftOuterJoin(
+        self: DStream[Tuple[K, V]],
+        other: DStream[Tuple[K, U]],
+        numPartitions: Optional[int] = ...,
+    ) -> DStream[Tuple[K, Tuple[V, Optional[U]]]]: ...
+    def rightOuterJoin(
+        self: DStream[Tuple[K, V]],
+        other: DStream[Tuple[K, U]],
+        numPartitions: Optional[int] = ...,
+    ) -> DStream[Tuple[K, Tuple[Optional[V], U]]]: ...
+    def fullOuterJoin(
+        self: DStream[Tuple[K, V]],
+        other: DStream[Tuple[K, U]],
+        numPartitions: Optional[int] = ...,
+    ) -> DStream[Tuple[K, Tuple[Optional[V], Optional[U]]]]: ...
+    def slice(
+        self, begin: Union[datetime.datetime, int], end: Union[datetime.datetime, int]
+    ) -> List[RDD[T]]: ...
+    def window(
+        self, windowDuration: int, slideDuration: Optional[int] = ...
+    ) -> DStream[T]: ...
+    def reduceByWindow(
+        self,
+        reduceFunc: Callable[[T, T], T],
+        invReduceFunc: Optional[Callable[[T, T], T]],
+        windowDuration: int,
+        slideDuration: int,
+    ) -> DStream[T]: ...
+    def countByWindow(
+        self, windowDuration: int, slideDuration: int
+    ) -> DStream[Tuple[T, int]]: ...
+    def countByValueAndWindow(
+        self,
+        windowDuration: int,
+        slideDuration: int,
+        numPartitions: Optional[int] = ...,
+    ) -> DStream[Tuple[T, int]]: ...
+    def groupByKeyAndWindow(
+        self: DStream[Tuple[K, V]],
+        windowDuration: int,
+        slideDuration: int,
+        numPartitions: Optional[int] = ...,
+    ) -> DStream[Tuple[K, Iterable[V]]]: ...
+    def reduceByKeyAndWindow(
+        self: DStream[Tuple[K, V]],
+        func: Callable[[V, V], V],
+        invFunc: Optional[Callable[[V, V], V]],
+        windowDuration: int,
+        slideDuration: Optional[int] = ...,
+        numPartitions: Optional[int] = ...,
+        filterFunc: Optional[Callable[[Tuple[K, V]], bool]] = ...,
+    ) -> DStream[Tuple[K, V]]: ...
+    def updateStateByKey(
+        self: DStream[Tuple[K, V]],
+        updateFunc: Callable[[Iterable[V], Optional[S]], S],
+        numPartitions: Optional[int] = ...,
+        initialRDD: Optional[RDD[Tuple[K, S]]] = ...,
+    ) -> DStream[Tuple[K, S]]: ...
+
+class TransformedDStream(DStream[U]):
+    is_cached: bool
+    is_checkpointed: bool
+    func: Callable
+    prev: DStream
+    @overload
+    def __init__(
+        self: DStream[U], prev: DStream[T], func: Callable[[RDD[T]], RDD[U]]
+    ) -> None: ...
+    @overload
+    def __init__(
+        self: DStream[U],
+        prev: DStream[T],
+        func: Callable[[datetime.datetime, RDD[T]], RDD[U]],
+    ) -> None: ...
diff --git a/python/pyspark/streaming/kinesis.py b/python/pyspark/streaming/kinesis.py
index 729ec97505aad..25a1a72666ce5 100644
--- a/python/pyspark/streaming/kinesis.py
+++ b/python/pyspark/streaming/kinesis.py
@@ -43,38 +43,59 @@ def createStream(ssc, kinesisAppName, streamName, endpointUrl, regionName,
         Create an input stream that pulls messages from a Kinesis stream. This uses the
         Kinesis Client Library (KCL) to pull messages from Kinesis.
 
-        .. note:: The given AWS credentials will get saved in DStream checkpoints if checkpointing
-            is enabled. Make sure that your checkpoint directory is secure.
+        Parameters
+        ----------
+        ssc : :class:`StreamingContext`
+            StreamingContext object
+        kinesisAppName : str
+            Kinesis application name used by the Kinesis Client Library (KCL) to
+            update DynamoDB
+        streamName : str
+            Kinesis stream name
+        endpointUrl : str
+            Url of Kinesis service (e.g., https://kinesis.us-east-1.amazonaws.com)
+        regionName : str
+            Name of region used by the Kinesis Client Library (KCL) to update
+            DynamoDB (lease coordination and checkpointing) and CloudWatch (metrics)
+        initialPositionInStream : int
+            In the absence of Kinesis checkpoint info, this is the
+            worker's initial starting position in the stream. The
+            values are either the beginning of the stream per Kinesis'
+            limit of 24 hours (InitialPositionInStream.TRIM_HORIZON) or
+            the tip of the stream (InitialPositionInStream.LATEST).
+        checkpointInterval : int
+            Checkpoint interval(in seconds) for Kinesis checkpointing. See the Kinesis
+            Spark Streaming documentation for more details on the different
+            types of checkpoints.
+        storageLevel : :class:`pyspark.StorageLevel`, optional
+            Storage level to use for storing the received objects (default is
+            StorageLevel.MEMORY_AND_DISK_2)
+        awsAccessKeyId : str, optional
+            AWS AccessKeyId (default is None. If None, will use
+            DefaultAWSCredentialsProviderChain)
+        awsSecretKey : str, optional
+            AWS SecretKey (default is None. If None, will use
+            DefaultAWSCredentialsProviderChain)
+        decoder : function, optional
+            A function used to decode value (default is utf8_decoder)
+        stsAssumeRoleArn : str, optional
+            ARN of IAM role to assume when using STS sessions to read from
+            the Kinesis stream (default is None).
+        stsSessionName : str, optional
+            Name to uniquely identify STS sessions used to read from Kinesis
+            stream, if STS is being used (default is None).
+        stsExternalId : str, optional
+            External ID that can be used to validate against the assumed IAM
+            role's trust policy, if STS is being used (default is None).
 
-        :param ssc:  StreamingContext object
-        :param kinesisAppName:  Kinesis application name used by the Kinesis Client Library (KCL) to
-                                update DynamoDB
-        :param streamName:  Kinesis stream name
-        :param endpointUrl:  Url of Kinesis service (e.g., https://kinesis.us-east-1.amazonaws.com)
-        :param regionName:  Name of region used by the Kinesis Client Library (KCL) to update
-                            DynamoDB (lease coordination and checkpointing) and CloudWatch (metrics)
-        :param initialPositionInStream:  In the absence of Kinesis checkpoint info, this is the
-                                         worker's initial starting position in the stream. The
-                                         values are either the beginning of the stream per Kinesis'
-                                         limit of 24 hours (InitialPositionInStream.TRIM_HORIZON) or
-                                         the tip of the stream (InitialPositionInStream.LATEST).
-        :param checkpointInterval:  Checkpoint interval for Kinesis checkpointing. See the Kinesis
-                                    Spark Streaming documentation for more details on the different
-                                    types of checkpoints.
-        :param storageLevel:  Storage level to use for storing the received objects (default is
-                              StorageLevel.MEMORY_AND_DISK_2)
-        :param awsAccessKeyId:  AWS AccessKeyId (default is None. If None, will use
-                                DefaultAWSCredentialsProviderChain)
-        :param awsSecretKey:  AWS SecretKey (default is None. If None, will use
-                              DefaultAWSCredentialsProviderChain)
-        :param decoder:  A function used to decode value (default is utf8_decoder)
-        :param stsAssumeRoleArn: ARN of IAM role to assume when using STS sessions to read from
-                                 the Kinesis stream (default is None).
-        :param stsSessionName: Name to uniquely identify STS sessions used to read from Kinesis
-                               stream, if STS is being used (default is None).
-        :param stsExternalId: External ID that can be used to validate against the assumed IAM
-                              role's trust policy, if STS is being used (default is None).
-        :return: A DStream object
+        Returns
+        -------
+        A DStream object
+
+        Notes
+        -----
+        The given AWS credentials will get saved in DStream checkpoints if checkpointing
+        is enabled. Make sure that your checkpoint directory is secure.
         """
         jlevel = ssc._sc._getJavaStorageLevel(storageLevel)
         jduration = ssc._jduration(checkpointInterval)
diff --git a/python/pyspark/streaming/kinesis.pyi b/python/pyspark/streaming/kinesis.pyi
new file mode 100644
index 0000000000000..399c37f869620
--- /dev/null
+++ b/python/pyspark/streaming/kinesis.pyi
@@ -0,0 +1,49 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Callable, Optional, TypeVar
+from pyspark.storagelevel import StorageLevel
+from pyspark.streaming.context import StreamingContext
+from pyspark.streaming.dstream import DStream
+
+T = TypeVar("T")
+
+def utf8_decoder(s: Optional[bytes]) -> str: ...
+
+class KinesisUtils:
+    @staticmethod
+    def createStream(
+        ssc: StreamingContext,
+        kinesisAppName: str,
+        streamName: str,
+        endpointUrl: str,
+        regionName: str,
+        initialPositionInStream: str,
+        checkpointInterval: int,
+        storageLevel: StorageLevel = ...,
+        awsAccessKeyId: Optional[str] = ...,
+        awsSecretKey: Optional[str] = ...,
+        decoder: Callable[[Optional[bytes]], T] = ...,
+        stsAssumeRoleArn: Optional[str] = ...,
+        stsSessionName: Optional[str] = ...,
+        stsExternalId: Optional[str] = ...,
+    ) -> DStream[T]: ...
+
+class InitialPositionInStream:
+    LATEST: int
+    TRIM_HORIZON: int
diff --git a/python/pyspark/streaming/listener.pyi b/python/pyspark/streaming/listener.pyi
new file mode 100644
index 0000000000000..4033529607cea
--- /dev/null
+++ b/python/pyspark/streaming/listener.pyi
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# NOTE: This dynamically typed stub was automatically generated by stubgen.
+
+from typing import Any
+
+class StreamingListener:
+    def __init__(self) -> None: ...
+    def onStreamingStarted(self, streamingStarted: Any) -> None: ...
+    def onReceiverStarted(self, receiverStarted: Any) -> None: ...
+    def onReceiverError(self, receiverError: Any) -> None: ...
+    def onReceiverStopped(self, receiverStopped: Any) -> None: ...
+    def onBatchSubmitted(self, batchSubmitted: Any) -> None: ...
+    def onBatchStarted(self, batchStarted: Any) -> None: ...
+    def onBatchCompleted(self, batchCompleted: Any) -> None: ...
+    def onOutputOperationStarted(self, outputOperationStarted: Any) -> None: ...
+    def onOutputOperationCompleted(self, outputOperationCompleted: Any) -> None: ...
+    class Java:
+        implements: Any = ...
diff --git a/python/pyspark/streaming/tests/test_context.py b/python/pyspark/streaming/tests/test_context.py
index 69a209ad878d6..b255796cdcdd7 100644
--- a/python/pyspark/streaming/tests/test_context.py
+++ b/python/pyspark/streaming/tests/test_context.py
@@ -175,10 +175,10 @@ def test_await_termination_or_timeout(self):
 
 if __name__ == "__main__":
     import unittest
-    from pyspark.streaming.tests.test_context import *
+    from pyspark.streaming.tests.test_context import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/streaming/tests/test_dstream.py b/python/pyspark/streaming/tests/test_dstream.py
index 7ecdf6b0b12db..ea5353c77b6b2 100644
--- a/python/pyspark/streaming/tests/test_dstream.py
+++ b/python/pyspark/streaming/tests/test_dstream.py
@@ -24,14 +24,15 @@
 from itertools import chain
 import platform
 
-from pyspark import SparkConf, SparkContext, RDD
+from pyspark import SparkConf, SparkContext
 from pyspark.streaming import StreamingContext
 from pyspark.testing.streamingutils import PySparkStreamingTestCase
 
 
 @unittest.skipIf(
-    "pypy" in platform.python_implementation().lower() and "COVERAGE_PROCESS_START" in os.environ,
-    "PyPy implementation causes to hang DStream tests forever when Coverage report is used.")
+    "pypy" in platform.python_implementation().lower(),
+    "The tests fail in PyPy3 implementation for an unknown reason. "
+    "With PyPy, it causes to hang DStream tests forever when Coverage report is used.")
 class BasicOperationTests(PySparkStreamingTestCase):
 
     def test_map(self):
@@ -394,8 +395,9 @@ def failed_func(i):
 
 
 @unittest.skipIf(
-    "pypy" in platform.python_implementation().lower() and "COVERAGE_PROCESS_START" in os.environ,
-    "PyPy implementation causes to hang DStream tests forever when Coverage report is used.")
+    "pypy" in platform.python_implementation().lower(),
+    "The tests fail in PyPy3 implementation for an unknown reason. "
+    "With PyPy, it causes to hang DStream tests forever when Coverage report is used.")
 class WindowFunctionTests(PySparkStreamingTestCase):
 
     timeout = 15
@@ -474,8 +476,9 @@ def func(dstream):
 
 
 @unittest.skipIf(
-    "pypy" in platform.python_implementation().lower() and "COVERAGE_PROCESS_START" in os.environ,
-    "PyPy implementation causes to hang DStream tests forever when Coverage report is used.")
+    "pypy" in platform.python_implementation().lower(),
+    "The tests fail in PyPy3 implementation for an unknown reason. "
+    "With PyPy, it causes to hang DStream tests forever when Coverage report is used.")
 class CheckpointTests(unittest.TestCase):
 
     setupCalled = False
@@ -641,10 +644,10 @@ def check_output(n):
 
 
 if __name__ == "__main__":
-    from pyspark.streaming.tests.test_dstream import *
+    from pyspark.streaming.tests.test_dstream import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/streaming/tests/test_kinesis.py b/python/pyspark/streaming/tests/test_kinesis.py
index a2da230821dea..70c9a012e7a03 100644
--- a/python/pyspark/streaming/tests/test_kinesis.py
+++ b/python/pyspark/streaming/tests/test_kinesis.py
@@ -80,10 +80,10 @@ def get_output(_, rdd):
 
 
 if __name__ == "__main__":
-    from pyspark.streaming.tests.test_kinesis import *
+    from pyspark.streaming.tests.test_kinesis import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/streaming/tests/test_listener.py b/python/pyspark/streaming/tests/test_listener.py
index 48c5783bf8ff5..e4dab1bba3a6c 100644
--- a/python/pyspark/streaming/tests/test_listener.py
+++ b/python/pyspark/streaming/tests/test_listener.py
@@ -149,10 +149,10 @@ def func(dstream):
 
 if __name__ == "__main__":
     import unittest
-    from pyspark.streaming.tests.test_listener import *
+    from pyspark.streaming.tests.test_listener import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/streaming/util.py b/python/pyspark/streaming/util.py
index b4b9f97feb7ca..84de9445761d4 100644
--- a/python/pyspark/streaming/util.py
+++ b/python/pyspark/streaming/util.py
@@ -139,6 +139,8 @@ def rddToFileName(prefix, suffix, timestamp):
     """
     Return string prefix-time(.suffix)
 
+    Examples
+    --------
     >>> rddToFileName("spark", None, 12345678910)
     'spark-12345678910'
     >>> rddToFileName("spark", "tmp", 12345678910)
diff --git a/python/pyspark/taskcontext.py b/python/pyspark/taskcontext.py
index 8f419a5e8446a..1afbe630846af 100644
--- a/python/pyspark/taskcontext.py
+++ b/python/pyspark/taskcontext.py
@@ -14,10 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
-from __future__ import print_function
-import json
-
 from pyspark.java_gateway import local_connect_and_auth
 from pyspark.serializers import read_int, write_int, write_with_length, UTF8Deserializer
 
@@ -64,7 +60,9 @@ def get(cls):
         Return the currently active TaskContext. This can be called inside of
         user functions to access contextual information about running tasks.
 
-        .. note:: Must be called on the worker, not the driver. Returns None if not initialized.
+        Notes
+        -----
+        Must be called on the worker, not the driver. Returns None if not initialized.
         """
         return cls._taskContext
 
@@ -148,12 +146,14 @@ def _load_from_socket(port, auth_secret, function, all_gather_message=None):
 class BarrierTaskContext(TaskContext):
 
     """
-    .. note:: Experimental
-
     A :class:`TaskContext` with extra contextual info and tooling for tasks in a barrier stage.
     Use :func:`BarrierTaskContext.get` to obtain the barrier context for a running barrier task.
 
     .. versionadded:: 2.4.0
+
+    Notes
+    -----
+    This API is experimental
     """
 
     _port = None
@@ -173,14 +173,16 @@ def _getOrCreate(cls):
     @classmethod
     def get(cls):
         """
-        .. note:: Experimental
-
         Return the currently active :class:`BarrierTaskContext`.
         This can be called inside of user functions to access contextual information about
         running tasks.
 
-        .. note:: Must be called on the worker, not the driver. Returns None if not initialized.
-            An Exception will raise if it is not in a barrier stage.
+        Notes
+        -----
+        Must be called on the worker, not the driver. Returns None if not initialized.
+        An Exception will raise if it is not in a barrier stage.
+
+        This API is experimental
         """
         if not isinstance(cls._taskContext, BarrierTaskContext):
             raise Exception('It is not in a barrier stage')
@@ -197,17 +199,19 @@ def _initialize(cls, port, secret):
 
     def barrier(self):
         """
-        .. note:: Experimental
-
         Sets a global barrier and waits until all tasks in this stage hit this barrier.
         Similar to `MPI_Barrier` function in MPI, this function blocks until all tasks
         in the same stage have reached this routine.
 
+        .. versionadded:: 2.4.0
+
         .. warning:: In a barrier stage, each task much have the same number of `barrier()`
             calls, in all possible code branches.
             Otherwise, you may get the job hanging or a SparkException after timeout.
 
-        .. versionadded:: 2.4.0
+        Notes
+        -----
+        This API is experimental
         """
         if self._port is None or self._secret is None:
             raise Exception("Not supported to call barrier() before initialize " +
@@ -217,20 +221,22 @@ def barrier(self):
 
     def allGather(self, message=""):
         """
-        .. note:: Experimental
-
         This function blocks until all tasks in the same stage have reached this routine.
         Each task passes in a message and returns with a list of all the messages passed in
         by each of those tasks.
 
+        .. versionadded:: 3.0.0
+
         .. warning:: In a barrier stage, each task much have the same number of `allGather()`
             calls, in all possible code branches.
             Otherwise, you may get the job hanging or a SparkException after timeout.
 
-        .. versionadded:: 3.0.0
+        Notes
+        -----
+        This API is experimental
         """
         if not isinstance(message, str):
-            raise ValueError("Argument `message` must be of type `str`")
+            raise TypeError("Argument `message` must be of type `str`")
         elif self._port is None or self._secret is None:
             raise Exception("Not supported to call barrier() before initialize " +
                             "BarrierTaskContext.")
@@ -239,12 +245,14 @@ def allGather(self, message=""):
 
     def getTaskInfos(self):
         """
-        .. note:: Experimental
-
         Returns :class:`BarrierTaskInfo` for all tasks in this barrier stage,
         ordered by partition ID.
 
         .. versionadded:: 2.4.0
+
+        Notes
+        -----
+        This API is experimental
         """
         if self._port is None or self._secret is None:
             raise Exception("Not supported to call getTaskInfos() before initialize " +
@@ -256,13 +264,18 @@ def getTaskInfos(self):
 
 class BarrierTaskInfo(object):
     """
-    .. note:: Experimental
-
     Carries all task infos of a barrier task.
 
-    :var address: The IPv4 address (host:port) of the executor that the barrier task is running on
-
     .. versionadded:: 2.4.0
+
+    Attributes
+    ----------
+    address : str
+        The IPv4 address (host:port) of the executor that the barrier task is running on
+
+    Notes
+    -----
+    This API is experimental
     """
 
     def __init__(self, address):
diff --git a/python/pyspark/taskcontext.pyi b/python/pyspark/taskcontext.pyi
new file mode 100644
index 0000000000000..3415c69f02177
--- /dev/null
+++ b/python/pyspark/taskcontext.pyi
@@ -0,0 +1,45 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Dict, List
+from typing_extensions import Literal
+from pyspark.resource.information import ResourceInformation
+
+class TaskContext:
+    def __new__(cls) -> TaskContext: ...
+    @classmethod
+    def get(cls) -> TaskContext: ...
+    def stageId(self) -> int: ...
+    def partitionId(self) -> int: ...
+    def attemptNumber(self) -> int: ...
+    def taskAttemptId(self) -> int: ...
+    def getLocalProperty(self, key: str) -> str: ...
+    def resources(self) -> Dict[str, ResourceInformation]: ...
+
+BARRIER_FUNCTION = Literal[1]
+
+class BarrierTaskContext(TaskContext):
+    @classmethod
+    def get(cls) -> BarrierTaskContext: ...
+    def barrier(self) -> None: ...
+    def allGather(self, message: str = ...) -> List[str]: ...
+    def getTaskInfos(self) -> List[BarrierTaskInfo]: ...
+
+class BarrierTaskInfo:
+    address: str
+    def __init__(self, address: str) -> None: ...
diff --git a/python/pyspark/testing/mlutils.py b/python/pyspark/testing/mlutils.py
index a36d0709d8013..d6edf9d64af49 100644
--- a/python/pyspark/testing/mlutils.py
+++ b/python/pyspark/testing/mlutils.py
@@ -17,10 +17,14 @@
 
 import numpy as np
 
+from pyspark import keyword_only
 from pyspark.ml import Estimator, Model, Transformer, UnaryTransformer
+from pyspark.ml.evaluation import Evaluator
 from pyspark.ml.param import Param, Params, TypeConverters
+from pyspark.ml.param.shared import HasMaxIter, HasRegParam
+from pyspark.ml.classification import Classifier, ClassificationModel
 from pyspark.ml.util import DefaultParamsReadable, DefaultParamsWritable
-from pyspark.ml.wrapper import _java2py
+from pyspark.ml.wrapper import _java2py  # type: ignore
 from pyspark.sql import DataFrame, SparkSession
 from pyspark.sql.types import DoubleType
 from pyspark.testing.utils import ReusedPySparkTestCase as PySparkTestCase
@@ -28,7 +32,8 @@
 
 def check_params(test_self, py_stage, check_params_exist=True):
     """
-    Checks common requirements for Params.params:
+    Checks common requirements for :py:class:`PySpark.ml.Params.params`:
+
       - set of params exist in Java and Python and are ordered by names
       - param parent has the same UID as the object's UID
       - default param value from Java matches value in Python
@@ -116,7 +121,8 @@ def _transform(self, dataset):
 
 class MockUnaryTransformer(UnaryTransformer, DefaultParamsReadable, DefaultParamsWritable):
 
-    shift = Param(Params._dummy(), "shift", "The amount by which to shift " +
+    shift = Param(Params._dummy(),  # type: ignore
+                  "shift", "The amount by which to shift " +
                   "data in a DataFrame",
                   typeConverter=TypeConverters.toFloat)
 
@@ -159,3 +165,86 @@ def _fit(self, dataset):
 
 class MockModel(MockTransformer, Model, HasFake):
     pass
+
+
+class _DummyLogisticRegressionParams(HasMaxIter, HasRegParam):
+    def setMaxIter(self, value):
+        return self._set(maxIter=value)
+
+    def setRegParam(self, value):
+        return self._set(regParam=value)
+
+
+# This is a dummy LogisticRegression used in test for python backend estimator/model
+class DummyLogisticRegression(Classifier, _DummyLogisticRegressionParams,
+                              DefaultParamsReadable, DefaultParamsWritable):
+    @keyword_only
+    def __init__(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
+                 maxIter=100, regParam=0.0, rawPredictionCol="rawPrediction"):
+        super(DummyLogisticRegression, self).__init__()
+        kwargs = self._input_kwargs
+        self.setParams(**kwargs)
+
+    @keyword_only
+    def setParams(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
+                  maxIter=100, regParam=0.0, rawPredictionCol="rawPrediction"):
+        kwargs = self._input_kwargs
+        self._set(**kwargs)
+        return self
+
+    def _fit(self, dataset):
+        # Do nothing but create a dummy model
+        return self._copyValues(DummyLogisticRegressionModel())
+
+
+class DummyLogisticRegressionModel(ClassificationModel, _DummyLogisticRegressionParams,
+                                   DefaultParamsReadable, DefaultParamsWritable):
+
+    def __init__(self):
+        super(DummyLogisticRegressionModel, self).__init__()
+
+    def _transform(self, dataset):
+        # A dummy transform impl which always predict label 1
+        from pyspark.sql.functions import array, lit
+        from pyspark.ml.functions import array_to_vector
+        rawPredCol = self.getRawPredictionCol()
+        if rawPredCol:
+            dataset = dataset.withColumn(
+                rawPredCol, array_to_vector(array(lit(-100.0), lit(100.0))))
+        predCol = self.getPredictionCol()
+        if predCol:
+            dataset = dataset.withColumn(predCol, lit(1.0))
+
+        return dataset
+
+    @property
+    def numClasses(self):
+        # a dummy implementation for test.
+        return 2
+
+    @property
+    def intercept(self):
+        # a dummy implementation for test.
+        return 0.0
+
+    # This class only used in test. The following methods/properties are not used in tests.
+
+    @property
+    def coefficients(self):
+        raise NotImplementedError()
+
+    def predictRaw(self, value):
+        raise NotImplementedError()
+
+    def numFeatures(self):
+        raise NotImplementedError()
+
+    def predict(self, value):
+        raise NotImplementedError()
+
+
+class DummyEvaluator(Evaluator, DefaultParamsReadable, DefaultParamsWritable):
+
+    def _evaluate(self, dataset):
+        # a dummy implementation for test.
+        return 1.0
diff --git a/python/pyspark/testing/pandasutils.py b/python/pyspark/testing/pandasutils.py
new file mode 100644
index 0000000000000..4a5bfe8d56424
--- /dev/null
+++ b/python/pyspark/testing/pandasutils.py
@@ -0,0 +1,373 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import functools
+import shutil
+import tempfile
+import unittest
+import warnings
+from contextlib import contextmanager
+from distutils.version import LooseVersion
+
+import pandas as pd
+from pandas.api.types import is_list_like
+from pandas.testing import assert_frame_equal, assert_index_equal, assert_series_equal
+
+from pyspark import pandas as ps
+from pyspark.pandas.frame import DataFrame
+from pyspark.pandas.indexes import Index
+from pyspark.pandas.series import Series
+from pyspark.pandas.utils import default_session, SPARK_CONF_ARROW_ENABLED
+from pyspark.testing.sqlutils import SQLTestUtils
+
+
+tabulate_requirement_message = None
+try:
+    from tabulate import tabulate  # noqa: F401
+except ImportError as e:
+    # If tabulate requirement is not satisfied, skip related tests.
+    tabulate_requirement_message = str(e)
+have_tabulate = tabulate_requirement_message is None
+
+matplotlib_requirement_message = None
+try:
+    import matplotlib  # type: ignore # noqa: F401
+except ImportError as e:
+    # If matplotlib requirement is not satisfied, skip related tests.
+    matplotlib_requirement_message = str(e)
+have_matplotlib = matplotlib_requirement_message is None
+
+plotly_requirement_message = None
+try:
+    import plotly  # type: ignore # noqa: F401
+except ImportError as e:
+    # If plotly requirement is not satisfied, skip related tests.
+    plotly_requirement_message = str(e)
+have_plotly = plotly_requirement_message is None
+
+
+class PandasOnSparkTestCase(unittest.TestCase, SQLTestUtils):
+    @classmethod
+    def setUpClass(cls):
+        cls.spark = default_session()
+        cls.spark.conf.set(SPARK_CONF_ARROW_ENABLED, True)
+
+    @classmethod
+    def tearDownClass(cls):
+        # We don't stop Spark session to reuse across all tests.
+        # The Spark session will be started and stopped at PyTest session level.
+        # Please see databricks/koalas/conftest.py.
+        pass
+
+    def assertPandasEqual(self, left, right, check_exact=True):
+        if isinstance(left, pd.DataFrame) and isinstance(right, pd.DataFrame):
+            try:
+                if LooseVersion(pd.__version__) >= LooseVersion("1.1"):
+                    kwargs = dict(check_freq=False)
+                else:
+                    kwargs = dict()
+
+                assert_frame_equal(
+                    left,
+                    right,
+                    check_index_type=("equiv" if len(left.index) > 0 else False),
+                    check_column_type=("equiv" if len(left.columns) > 0 else False),
+                    check_exact=check_exact,
+                    **kwargs
+                )
+            except AssertionError as e:
+                msg = (
+                    str(e)
+                    + "\n\nLeft:\n%s\n%s" % (left, left.dtypes)
+                    + "\n\nRight:\n%s\n%s" % (right, right.dtypes)
+                )
+                raise AssertionError(msg) from e
+        elif isinstance(left, pd.Series) and isinstance(right, pd.Series):
+            try:
+                if LooseVersion(pd.__version__) >= LooseVersion("1.1"):
+                    kwargs = dict(check_freq=False)
+                else:
+                    kwargs = dict()
+
+                assert_series_equal(
+                    left,
+                    right,
+                    check_index_type=("equiv" if len(left.index) > 0 else False),
+                    check_exact=check_exact,
+                    **kwargs
+                )
+            except AssertionError as e:
+                msg = (
+                    str(e)
+                    + "\n\nLeft:\n%s\n%s" % (left, left.dtype)
+                    + "\n\nRight:\n%s\n%s" % (right, right.dtype)
+                )
+                raise AssertionError(msg) from e
+        elif isinstance(left, pd.Index) and isinstance(right, pd.Index):
+            try:
+                assert_index_equal(left, right, check_exact=check_exact)
+            except AssertionError as e:
+                msg = (
+                    str(e)
+                    + "\n\nLeft:\n%s\n%s" % (left, left.dtype)
+                    + "\n\nRight:\n%s\n%s" % (right, right.dtype)
+                )
+                raise AssertionError(msg) from e
+        else:
+            raise ValueError("Unexpected values: (%s, %s)" % (left, right))
+
+    def assertPandasAlmostEqual(self, left, right):
+        """
+        This function checks if given pandas objects approximately same,
+        which means the conditions below:
+          - Both objects are nullable
+          - Compare floats rounding to the number of decimal places, 7 after
+            dropping missing values (NaN, NaT, None)
+        """
+        if isinstance(left, pd.DataFrame) and isinstance(right, pd.DataFrame):
+            msg = (
+                "DataFrames are not almost equal: "
+                + "\n\nLeft:\n%s\n%s" % (left, left.dtypes)
+                + "\n\nRight:\n%s\n%s" % (right, right.dtypes)
+            )
+            self.assertEqual(left.shape, right.shape, msg=msg)
+            for lcol, rcol in zip(left.columns, right.columns):
+                self.assertEqual(lcol, rcol, msg=msg)
+                for lnull, rnull in zip(left[lcol].isnull(), right[rcol].isnull()):
+                    self.assertEqual(lnull, rnull, msg=msg)
+                for lval, rval in zip(left[lcol].dropna(), right[rcol].dropna()):
+                    self.assertAlmostEqual(lval, rval, msg=msg)
+            self.assertEqual(left.columns.names, right.columns.names, msg=msg)
+        elif isinstance(left, pd.Series) and isinstance(right, pd.Series):
+            msg = (
+                "Series are not almost equal: "
+                + "\n\nLeft:\n%s\n%s" % (left, left.dtype)
+                + "\n\nRight:\n%s\n%s" % (right, right.dtype)
+            )
+            self.assertEqual(left.name, right.name, msg=msg)
+            self.assertEqual(len(left), len(right), msg=msg)
+            for lnull, rnull in zip(left.isnull(), right.isnull()):
+                self.assertEqual(lnull, rnull, msg=msg)
+            for lval, rval in zip(left.dropna(), right.dropna()):
+                self.assertAlmostEqual(lval, rval, msg=msg)
+        elif isinstance(left, pd.MultiIndex) and isinstance(right, pd.MultiIndex):
+            msg = (
+                "MultiIndices are not almost equal: "
+                + "\n\nLeft:\n%s\n%s" % (left, left.dtype)
+                + "\n\nRight:\n%s\n%s" % (right, right.dtype)
+            )
+            self.assertEqual(len(left), len(right), msg=msg)
+            for lval, rval in zip(left, right):
+                self.assertAlmostEqual(lval, rval, msg=msg)
+        elif isinstance(left, pd.Index) and isinstance(right, pd.Index):
+            msg = (
+                "Indices are not almost equal: "
+                + "\n\nLeft:\n%s\n%s" % (left, left.dtype)
+                + "\n\nRight:\n%s\n%s" % (right, right.dtype)
+            )
+            self.assertEqual(len(left), len(right), msg=msg)
+            for lnull, rnull in zip(left.isnull(), right.isnull()):
+                self.assertEqual(lnull, rnull, msg=msg)
+            for lval, rval in zip(left.dropna(), right.dropna()):
+                self.assertAlmostEqual(lval, rval, msg=msg)
+        else:
+            raise ValueError("Unexpected values: (%s, %s)" % (left, right))
+
+    def assert_eq(self, left, right, check_exact=True, almost=False):
+        """
+        Asserts if two arbitrary objects are equal or not. If given objects are Koalas DataFrame
+        or Series, they are converted into pandas' and compared.
+
+        :param left: object to compare
+        :param right: object to compare
+        :param check_exact: if this is False, the comparison is done less precisely.
+        :param almost: if this is enabled, the comparison is delegated to `unittest`'s
+                       `assertAlmostEqual`. See its documentation for more details.
+        """
+        lobj = self._to_pandas(left)
+        robj = self._to_pandas(right)
+        if isinstance(lobj, (pd.DataFrame, pd.Series, pd.Index)):
+            if almost:
+                self.assertPandasAlmostEqual(lobj, robj)
+            else:
+                self.assertPandasEqual(lobj, robj, check_exact=check_exact)
+        elif is_list_like(lobj) and is_list_like(robj):
+            self.assertTrue(len(left) == len(right))
+            for litem, ritem in zip(left, right):
+                self.assert_eq(litem, ritem, check_exact=check_exact, almost=almost)
+        elif (lobj is not None and pd.isna(lobj)) and (robj is not None and pd.isna(robj)):
+            pass
+        else:
+            if almost:
+                self.assertAlmostEqual(lobj, robj)
+            else:
+                self.assertEqual(lobj, robj)
+
+    @staticmethod
+    def _to_pandas(obj):
+        if isinstance(obj, (DataFrame, Series, Index)):
+            return obj.to_pandas()
+        else:
+            return obj
+
+
+class TestUtils(object):
+    @contextmanager
+    def temp_dir(self):
+        tmp = tempfile.mkdtemp()
+        try:
+            yield tmp
+        finally:
+            shutil.rmtree(tmp)
+
+    @contextmanager
+    def temp_file(self):
+        with self.temp_dir() as tmp:
+            yield tempfile.mktemp(dir=tmp)
+
+
+class ComparisonTestBase(PandasOnSparkTestCase):
+    @property
+    def kdf(self):
+        return ps.from_pandas(self.pdf)
+
+    @property
+    def pdf(self):
+        return self.kdf.to_pandas()
+
+
+def compare_both(f=None, almost=True):
+
+    if f is None:
+        return functools.partial(compare_both, almost=almost)
+    elif isinstance(f, bool):
+        return functools.partial(compare_both, almost=f)
+
+    @functools.wraps(f)
+    def wrapped(self):
+        if almost:
+            compare = self.assertPandasAlmostEqual
+        else:
+            compare = self.assertPandasEqual
+
+        for result_pandas, result_spark in zip(f(self, self.pdf), f(self, self.kdf)):
+            compare(result_pandas, result_spark.to_pandas())
+
+    return wrapped
+
+
+@contextmanager
+def assert_produces_warning(
+    expected_warning=Warning,
+    filter_level="always",
+    check_stacklevel=True,
+    raise_on_extra_warnings=True,
+):
+    """
+    Context manager for running code expected to either raise a specific
+    warning, or not raise any warnings. Verifies that the code raises the
+    expected warning, and that it does not raise any other unexpected
+    warnings. It is basically a wrapper around ``warnings.catch_warnings``.
+
+    Notes
+    -----
+    Replicated from pandas/_testing/_warnings.py.
+
+    Parameters
+    ----------
+    expected_warning : {Warning, False, None}, default Warning
+        The type of Exception raised. ``exception.Warning`` is the base
+        class for all warnings. To check that no warning is returned,
+        specify ``False`` or ``None``.
+    filter_level : str or None, default "always"
+        Specifies whether warnings are ignored, displayed, or turned
+        into errors.
+        Valid values are:
+        * "error" - turns matching warnings into exceptions
+        * "ignore" - discard the warning
+        * "always" - always emit a warning
+        * "default" - print the warning the first time it is generated
+          from each location
+        * "module" - print the warning the first time it is generated
+          from each module
+        * "once" - print the warning the first time it is generated
+    check_stacklevel : bool, default True
+        If True, displays the line that called the function containing
+        the warning to show were the function is called. Otherwise, the
+        line that implements the function is displayed.
+    raise_on_extra_warnings : bool, default True
+        Whether extra warnings not of the type `expected_warning` should
+        cause the test to fail.
+
+    Examples
+    --------
+    >>> import warnings
+    >>> with assert_produces_warning():
+    ...     warnings.warn(UserWarning())
+    ...
+    >>> with assert_produces_warning(False): # doctest: +SKIP
+    ...     warnings.warn(RuntimeWarning())
+    ...
+    Traceback (most recent call last):
+        ...
+    AssertionError: Caused unexpected warning(s): ['RuntimeWarning'].
+    >>> with assert_produces_warning(UserWarning): # doctest: +SKIP
+    ...     warnings.warn(RuntimeWarning())
+    Traceback (most recent call last):
+        ...
+    AssertionError: Did not see expected warning of class 'UserWarning'
+    ..warn:: This is *not* thread-safe.
+    """
+    __tracebackhide__ = True
+
+    with warnings.catch_warnings(record=True) as w:
+
+        saw_warning = False
+        warnings.simplefilter(filter_level)
+        yield w
+        extra_warnings = []
+
+        for actual_warning in w:
+            if expected_warning and issubclass(actual_warning.category, expected_warning):
+                saw_warning = True
+
+                if check_stacklevel and issubclass(
+                    actual_warning.category, (FutureWarning, DeprecationWarning)
+                ):
+                    from inspect import getframeinfo, stack
+
+                    caller = getframeinfo(stack()[2][0])
+                    msg = (
+                        "Warning not set with correct stacklevel. ",
+                        "File where warning is raised: {} != ".format(actual_warning.filename),
+                        "{}. Warning message: {}".format(caller.filename, actual_warning.message),
+                    )
+                    assert actual_warning.filename == caller.filename, msg
+            else:
+                extra_warnings.append(
+                    (
+                        actual_warning.category.__name__,
+                        actual_warning.message,
+                        actual_warning.filename,
+                        actual_warning.lineno,
+                    )
+                )
+        if expected_warning:
+            msg = "Did not see expected warning of class {}".format(repr(expected_warning.__name__))
+            assert saw_warning, msg
+        if raise_on_extra_warnings and extra_warnings:
+            raise AssertionError("Caused unexpected warning(s): {}".format(repr(extra_warnings)))
diff --git a/python/pyspark/testing/sqlutils.py b/python/pyspark/testing/sqlutils.py
index 085fce6daa4ec..a394e8eecc69e 100644
--- a/python/pyspark/testing/sqlutils.py
+++ b/python/pyspark/testing/sqlutils.py
@@ -24,7 +24,6 @@
 from pyspark.sql import SparkSession
 from pyspark.sql.types import ArrayType, DoubleType, UserDefinedType, Row
 from pyspark.testing.utils import ReusedPySparkTestCase
-from pyspark.util import _exception_message
 
 
 pandas_requirement_message = None
@@ -33,7 +32,7 @@
     require_minimum_pandas_version()
 except ImportError as e:
     # If Pandas version requirement is not satisfied, skip related tests.
-    pandas_requirement_message = _exception_message(e)
+    pandas_requirement_message = str(e)
 
 pyarrow_requirement_message = None
 try:
@@ -41,14 +40,14 @@
     require_minimum_pyarrow_version()
 except ImportError as e:
     # If Arrow version requirement is not satisfied, skip related tests.
-    pyarrow_requirement_message = _exception_message(e)
+    pyarrow_requirement_message = str(e)
 
 test_not_compiled_message = None
 try:
     from pyspark.sql.utils import require_test_compiled
     require_test_compiled()
 except Exception as e:
-    test_not_compiled_message = _exception_message(e)
+    test_not_compiled_message = str(e)
 
 have_pandas = pandas_requirement_message is None
 have_pyarrow = pyarrow_requirement_message is None
@@ -148,7 +147,7 @@ class PythonOnlyPoint(ExamplePoint):
     """
     An example class to demonstrate UDT in only Python
     """
-    __UDT__ = PythonOnlyUDT()
+    __UDT__ = PythonOnlyUDT()  # type: ignore
 
 
 class MyObject(object):
diff --git a/python/pyspark/testing/streamingutils.py b/python/pyspark/testing/streamingutils.py
index a6abc2ef673b7..84c186d0d3262 100644
--- a/python/pyspark/testing/streamingutils.py
+++ b/python/pyspark/testing/streamingutils.py
@@ -37,7 +37,7 @@
                                           "spark-streaming-kinesis-asl-assembly-",
                                           "spark-streaming-kinesis-asl-assembly_")
     if kinesis_asl_assembly_jar is None:
-        kinesis_requirement_message = (
+        kinesis_requirement_message = (  # type: ignore
             "Skipping all Kinesis Python tests as the optional Kinesis project was "
             "not compiled into a JAR. To run these tests, "
             "you need to build Spark with 'build/sbt -Pkinesis-asl assembly/package "
@@ -47,7 +47,7 @@
         existing_args = os.environ.get("PYSPARK_SUBMIT_ARGS", "pyspark-shell")
         jars_args = "--jars %s" % kinesis_asl_assembly_jar
         os.environ["PYSPARK_SUBMIT_ARGS"] = " ".join([jars_args, existing_args])
-        kinesis_requirement_message = None
+        kinesis_requirement_message = None  # type: ignore
 
 should_test_kinesis = kinesis_requirement_message is None
 
@@ -116,7 +116,10 @@ def _collect(self, dstream, n, block=True):
         """
         Collect each RDDs into the returned list.
 
-        :return: list, which will have the collected items.
+        Returns
+        -------
+        list
+            which will have the collected items.
         """
         result = []
 
@@ -137,9 +140,14 @@ def get_output(_, rdd):
 
     def _test_func(self, input, func, expected, sort=False, input2=None):
         """
-        :param input: dataset for the test. This should be list of lists.
-        :param func: wrapped function. This function should return PythonDStream object.
-        :param expected: expected output for this testcase.
+        Parameters
+        ----------
+        input : list
+            dataset for the test. This should be list of lists.
+        func : function
+            wrapped function. This function should return PythonDStream object.
+        expected
+            expected output for this testcase.
         """
         if not isinstance(input[0], RDD):
             input = [self.sc.parallelize(d, 1) for d in input]
diff --git a/python/pyspark/testing/utils.py b/python/pyspark/testing/utils.py
index cda902b6f44d4..bbd93d1c38807 100644
--- a/python/pyspark/testing/utils.py
+++ b/python/pyspark/testing/utils.py
@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+
 import glob
 import os
 import struct
@@ -27,15 +28,15 @@
 have_scipy = False
 have_numpy = False
 try:
-    import scipy.sparse
+    import scipy.sparse  # noqa: F401
     have_scipy = True
-except:
+except ImportError:
     # No SciPy, but that's okay, we'll skip those tests
     pass
 try:
-    import numpy as np
+    import numpy as np  # noqa: F401
     have_numpy = True
-except:
+except ImportError:
     # No NumPy, but that's okay, we'll skip those tests
     pass
 
@@ -56,18 +57,22 @@ def eventually(condition, timeout=30.0, catch_assertions=False):
     Wait a given amount of time for a condition to pass, else fail with an error.
     This is a helper utility for PySpark tests.
 
-    :param condition: Function that checks for termination conditions.
-                      condition() can return:
-                       - True: Conditions met. Return without error.
-                       - other value: Conditions not met yet. Continue. Upon timeout,
-                                      include last such value in error message.
-                      Note that this method may be called at any time during
-                      streaming execution (e.g., even before any results
-                      have been created).
-    :param timeout: Number of seconds to wait.  Default 30 seconds.
-    :param catch_assertions: If False (default), do not catch AssertionErrors.
-                             If True, catch AssertionErrors; continue, but save
-                             error to throw upon timeout.
+    Parameters
+    ----------
+    condition : function
+        Function that checks for termination conditions. condition() can return:
+            - True: Conditions met. Return without error.
+            - other value: Conditions not met yet. Continue. Upon timeout,
+              include last such value in error message.
+              Note that this method may be called at any time during
+              streaming execution (e.g., even before any results
+              have been created).
+    timeout : int
+        Number of seconds to wait.  Default 30 seconds.
+    catch_assertions : bool
+        If False (default), do not catch AssertionErrors.
+        If True, catch AssertionErrors; continue, but save
+        error to throw upon timeout.
     """
     start_time = time()
     lastValue = None
diff --git a/python/pyspark/tests/test_appsubmit.py b/python/pyspark/tests/test_appsubmit.py
index 0eff5148294e2..3f45bf039d3a9 100644
--- a/python/pyspark/tests/test_appsubmit.py
+++ b/python/pyspark/tests/test_appsubmit.py
@@ -238,10 +238,10 @@ def test_user_configuration(self):
 
 
 if __name__ == "__main__":
-    from pyspark.tests.test_appsubmit import *
+    from pyspark.tests.test_appsubmit import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/tests/test_broadcast.py b/python/pyspark/tests/test_broadcast.py
index 02b0d799bda66..c35c5a68e4986 100644
--- a/python/pyspark/tests/test_broadcast.py
+++ b/python/pyspark/tests/test_broadcast.py
@@ -145,10 +145,10 @@ def random_bytes(n):
 
 
 if __name__ == '__main__':
-    from pyspark.tests.test_broadcast import *
+    from pyspark.tests.test_broadcast import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/tests/test_conf.py b/python/pyspark/tests/test_conf.py
index d51fd3d1f43b9..a8d65b8919777 100644
--- a/python/pyspark/tests/test_conf.py
+++ b/python/pyspark/tests/test_conf.py
@@ -33,10 +33,10 @@ def test_memory_conf(self):
 
 
 if __name__ == "__main__":
-    from pyspark.tests.test_conf import *
+    from pyspark.tests.test_conf import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/tests/test_context.py b/python/pyspark/tests/test_context.py
index 5833bf9f96fb3..8397ef1c4b62d 100644
--- a/python/pyspark/tests/test_context.py
+++ b/python/pyspark/tests/test_context.py
@@ -43,6 +43,7 @@ def test_basic_checkpointing(self):
 
         self.assertFalse(flatMappedRDD.isCheckpointed())
         self.assertTrue(flatMappedRDD.getCheckpointFile() is None)
+        self.assertFalse(self.sc.getCheckpointDir() is None)
 
         flatMappedRDD.checkpoint()
         result = flatMappedRDD.collect()
@@ -51,6 +52,8 @@ def test_basic_checkpointing(self):
         self.assertEqual(flatMappedRDD.collect(), result)
         self.assertEqual("file:" + self.checkpointDir.name,
                          os.path.dirname(os.path.dirname(flatMappedRDD.getCheckpointFile())))
+        self.assertEqual(self.sc.getCheckpointDir(),
+                         os.path.dirname(flatMappedRDD.getCheckpointFile()))
 
     def test_checkpoint_and_restore(self):
         parCollection = self.sc.parallelize([1, 2, 3, 4])
@@ -93,7 +96,7 @@ def test_add_py_file(self):
         # this job fails due to `userlibrary` not being on the Python path:
         # disable logging in log4j temporarily
         def func(x):
-            from userlibrary import UserClass
+            from userlibrary import UserClass  # type: ignore
             return UserClass().hello()
         with QuietTest(self.sc):
             self.assertRaises(Exception, self.sc.parallelize(range(2)).map(func).first)
@@ -126,7 +129,7 @@ def test_add_py_file_locally(self):
         # To ensure that we're actually testing addPyFile's effects, check that
         # this fails due to `userlibrary` not being on the Python path:
         def func():
-            from userlibrary import UserClass
+            from userlibrary import UserClass  # noqa: F401
         self.assertRaises(ImportError, func)
         path = os.path.join(SPARK_HOME, "python/test_support/userlibrary.py")
         self.sc.addPyFile(path)
@@ -137,7 +140,8 @@ def test_add_egg_file_locally(self):
         # To ensure that we're actually testing addPyFile's effects, check that
         # this fails due to `userlibrary` not being on the Python path:
         def func():
-            from userlib import UserClass
+            from userlib import UserClass  # type: ignore[import]
+            UserClass()
         self.assertRaises(ImportError, func)
         path = os.path.join(SPARK_HOME, "python/test_support/userlib-0.1.zip")
         self.sc.addPyFile(path)
@@ -147,11 +151,11 @@ def func():
     def test_overwrite_system_module(self):
         self.sc.addPyFile(os.path.join(SPARK_HOME, "python/test_support/SimpleHTTPServer.py"))
 
-        import SimpleHTTPServer
+        import SimpleHTTPServer  # type: ignore[import]
         self.assertEqual("My Server", SimpleHTTPServer.__name__)
 
         def func(x):
-            import SimpleHTTPServer
+            import SimpleHTTPServer  # type: ignore[import]
             return SimpleHTTPServer.__name__
 
         self.assertEqual(["My Server"], self.sc.parallelize(range(1)).map(func).collect())
@@ -171,8 +175,8 @@ def test_parallelize_eager_cleanup(self):
         with SparkContext() as sc:
             temp_files = os.listdir(sc._temp_dir)
             rdd = sc.parallelize([0, 1, 2])
-            post_parallalize_temp_files = os.listdir(sc._temp_dir)
-            self.assertEqual(temp_files, post_parallalize_temp_files)
+            post_parallelize_temp_files = os.listdir(sc._temp_dir)
+            self.assertEqual(temp_files, post_parallelize_temp_files)
 
     def test_set_conf(self):
         # This is for an internal use case. When there is an existing SparkContext,
@@ -267,6 +271,25 @@ def test_resources(self):
             resources = sc.resources
             self.assertEqual(len(resources), 0)
 
+    def test_disallow_to_create_spark_context_in_executors(self):
+        # SPARK-32160: SparkContext should not be created in executors.
+        with SparkContext("local-cluster[3, 1, 1024]") as sc:
+            with self.assertRaises(Exception) as context:
+                sc.range(2).foreach(lambda _: SparkContext())
+            self.assertIn("SparkContext should only be created and accessed on the driver.",
+                          str(context.exception))
+
+    def test_allow_to_create_spark_context_in_executors(self):
+        # SPARK-32160: SparkContext can be created in executors if the config is set.
+
+        def create_spark_context():
+            conf = SparkConf().set("spark.executor.allowSparkContext", "true")
+            with SparkContext(conf=conf):
+                pass
+
+        with SparkContext("local-cluster[3, 1, 1024]") as sc:
+            sc.range(2).foreach(lambda _: create_spark_context())
+
 
 class ContextTestsWithResources(unittest.TestCase):
 
@@ -299,10 +322,10 @@ def tearDown(self):
 
 
 if __name__ == "__main__":
-    from pyspark.tests.test_context import *
+    from pyspark.tests.test_context import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/tests/test_daemon.py b/python/pyspark/tests/test_daemon.py
index 898fb39d9ece5..c3fd89fef72c2 100644
--- a/python/pyspark/tests/test_daemon.py
+++ b/python/pyspark/tests/test_daemon.py
@@ -73,10 +73,10 @@ def test_termination_sigterm(self):
 
 
 if __name__ == "__main__":
-    from pyspark.tests.test_daemon import *
+    from pyspark.tests.test_daemon import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/tests/test_install_spark.py b/python/pyspark/tests/test_install_spark.py
new file mode 100644
index 0000000000000..f761e0088cd77
--- /dev/null
+++ b/python/pyspark/tests/test_install_spark.py
@@ -0,0 +1,109 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import os
+import tempfile
+import unittest
+
+from pyspark.install import install_spark, DEFAULT_HADOOP, DEFAULT_HIVE, \
+    UNSUPPORTED_COMBINATIONS, checked_versions, checked_package_name
+
+
+class SparkInstallationTestCase(unittest.TestCase):
+
+    def test_install_spark(self):
+        # Test only one case. Testing this is expensive because it needs to download
+        # the Spark distribution.
+        spark_version, hadoop_version, hive_version = checked_versions("3.0.1", "3.2", "2.3")
+
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            install_spark(
+                dest=tmp_dir,
+                spark_version=spark_version,
+                hadoop_version=hadoop_version,
+                hive_version=hive_version)
+
+            self.assertTrue(os.path.isdir("%s/jars" % tmp_dir))
+            self.assertTrue(os.path.exists("%s/bin/spark-submit" % tmp_dir))
+            self.assertTrue(os.path.exists("%s/RELEASE" % tmp_dir))
+
+    def test_package_name(self):
+        self.assertEqual(
+            "spark-3.0.0-bin-hadoop3.2",
+            checked_package_name("spark-3.0.0", "hadoop3.2", "hive2.3"))
+
+    def test_checked_versions(self):
+        test_version = "3.0.1"  # Just pick one version to test.
+
+        # Positive test cases
+        self.assertEqual(
+            ("spark-3.0.0", "hadoop2.7", "hive2.3"),
+            checked_versions("spark-3.0.0", "hadoop2.7", "hive2.3"))
+
+        self.assertEqual(
+            ("spark-3.0.0", "hadoop2.7", "hive2.3"),
+            checked_versions("3.0.0", "2.7", "2.3"))
+
+        self.assertEqual(
+            ("spark-2.4.1", "without-hadoop", "hive2.3"),
+            checked_versions("2.4.1", "without", "2.3"))
+
+        self.assertEqual(
+            ("spark-3.0.1", "without-hadoop", "hive2.3"),
+            checked_versions("spark-3.0.1", "without-hadoop", "hive2.3"))
+
+        # Negative test cases
+        for (hadoop_version, hive_version) in UNSUPPORTED_COMBINATIONS:
+            with self.assertRaisesRegex(RuntimeError, 'Hive.*should.*Hadoop'):
+                checked_versions(
+                    spark_version=test_version,
+                    hadoop_version=hadoop_version,
+                    hive_version=hive_version)
+
+        with self.assertRaisesRegex(RuntimeError, "Spark version should start with 'spark-'"):
+            checked_versions(
+                spark_version="malformed",
+                hadoop_version=DEFAULT_HADOOP,
+                hive_version=DEFAULT_HIVE)
+
+        with self.assertRaisesRegex(RuntimeError, "Spark distribution.*malformed.*"):
+            checked_versions(
+                spark_version=test_version,
+                hadoop_version="malformed",
+                hive_version=DEFAULT_HIVE)
+
+        with self.assertRaisesRegex(RuntimeError, "Spark distribution.*malformed.*"):
+            checked_versions(
+                spark_version=test_version,
+                hadoop_version=DEFAULT_HADOOP,
+                hive_version="malformed")
+
+        with self.assertRaisesRegex(RuntimeError, "Spark distribution of hive1.2 is not supported"):
+            checked_versions(
+                spark_version=test_version,
+                hadoop_version="hadoop3.2",
+                hive_version="hive1.2")
+
+
+if __name__ == "__main__":
+    from pyspark.tests.test_install_spark import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/tests/test_join.py b/python/pyspark/tests/test_join.py
index 138d062e72260..63dd1cfef9a6a 100644
--- a/python/pyspark/tests/test_join.py
+++ b/python/pyspark/tests/test_join.py
@@ -59,10 +59,10 @@ def test_narrow_dependency_in_join(self):
 
 if __name__ == "__main__":
     import unittest
-    from pyspark.tests.test_join import *
+    from pyspark.tests.test_join import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/tests/test_pin_thread.py b/python/pyspark/tests/test_pin_thread.py
index 657d129fe63bb..b612796c963a0 100644
--- a/python/pyspark/tests/test_pin_thread.py
+++ b/python/pyspark/tests/test_pin_thread.py
@@ -16,11 +16,10 @@
 #
 import os
 import time
-import random
 import threading
 import unittest
 
-from pyspark import SparkContext, SparkConf
+from pyspark import SparkContext, SparkConf, InheritableThread
 
 
 class PinThreadTests(unittest.TestCase):
@@ -143,13 +142,34 @@ def run_job(job_group, index):
                 is_job_cancelled[i],
                 "Thread {i}: Job in group B did not succeeded.".format(i=i))
 
+    def test_inheritable_local_property(self):
+        self.sc.setLocalProperty("a", "hi")
+        expected = []
+
+        def get_inner_local_prop():
+            expected.append(self.sc.getLocalProperty("b"))
+
+        def get_outer_local_prop():
+            expected.append(self.sc.getLocalProperty("a"))
+            self.sc.setLocalProperty("b", "hello")
+            t2 = InheritableThread(target=get_inner_local_prop)
+            t2.start()
+            t2.join()
+
+        t1 = InheritableThread(target=get_outer_local_prop)
+        t1.start()
+        t1.join()
+
+        self.assertEqual(self.sc.getLocalProperty("b"), None)
+        self.assertEqual(expected, ["hi", "hello"])
+
 
 if __name__ == "__main__":
     import unittest
-    from pyspark.tests.test_pin_thread import *
+    from pyspark.tests.test_pin_thread import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/tests/test_profiler.py b/python/pyspark/tests/test_profiler.py
index 04ca5a3896bf4..e621321283dab 100644
--- a/python/pyspark/tests/test_profiler.py
+++ b/python/pyspark/tests/test_profiler.py
@@ -19,15 +19,11 @@
 import sys
 import tempfile
 import unittest
+from io import StringIO
 
 from pyspark import SparkConf, SparkContext, BasicProfiler
 from pyspark.testing.utils import PySparkTestCase
 
-if sys.version >= "3":
-    from io import StringIO
-else:
-    from StringIO import StringIO
-
 
 class ProfilerTests(PySparkTestCase):
 
@@ -89,11 +85,11 @@ class ProfilerTests2(unittest.TestCase):
     def test_profiler_disabled(self):
         sc = SparkContext(conf=SparkConf().set("spark.python.profile", "false"))
         try:
-            self.assertRaisesRegexp(
+            self.assertRaisesRegex(
                 RuntimeError,
                 "'spark.python.profile' configuration must be set",
                 lambda: sc.show_profiles())
-            self.assertRaisesRegexp(
+            self.assertRaisesRegex(
                 RuntimeError,
                 "'spark.python.profile' configuration must be set",
                 lambda: sc.dump_profiles("/tmp/abc"))
@@ -102,10 +98,10 @@ def test_profiler_disabled(self):
 
 
 if __name__ == "__main__":
-    from pyspark.tests.test_profiler import *
+    from pyspark.tests.test_profiler import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/tests/test_rdd.py b/python/pyspark/tests/test_rdd.py
index 62ad4221d7078..b17c039889a71 100644
--- a/python/pyspark/tests/test_rdd.py
+++ b/python/pyspark/tests/test_rdd.py
@@ -18,7 +18,6 @@
 import hashlib
 import os
 import random
-import sys
 import tempfile
 import time
 from glob import glob
@@ -26,15 +25,12 @@
 from py4j.protocol import Py4JJavaError
 
 from pyspark import shuffle, RDD
-from pyspark.resource import ExecutorResourceRequests, ResourceProfile, ResourceProfileBuilder,\
+from pyspark.resource import ExecutorResourceRequests, ResourceProfileBuilder,\
     TaskResourceRequests
 from pyspark.serializers import CloudPickleSerializer, BatchedSerializer, PickleSerializer,\
     MarshalSerializer, UTF8Deserializer, NoOpSerializer
 from pyspark.testing.utils import ReusedPySparkTestCase, SPARK_HOME, QuietTest
 
-if sys.version_info[0] >= 3:
-    xrange = range
-
 
 global_func = lambda: "Hi"
 
@@ -168,6 +164,17 @@ def test_zip_chaining(self):
             set([(x, (x, x)) for x in 'abc'])
         )
 
+    def test_union_pair_rdd(self):
+        # SPARK-31788: test if pair RDDs can be combined by union.
+        rdd = self.sc.parallelize([1, 2])
+        pair_rdd = rdd.zip(rdd)
+        unionRDD = self.sc.union([pair_rdd, pair_rdd])
+        self.assertEqual(
+            set(unionRDD.collect()),
+            set([(1, 1), (2, 2), (1, 1), (2, 2)])
+        )
+        self.assertEqual(unionRDD.count(), 4)
+
     def test_deleting_input_files(self):
         # Regression test for SPARK-1025
         tempFile = tempfile.NamedTemporaryFile(delete=False)
@@ -182,15 +189,13 @@ def test_deleting_input_files(self):
 
     def test_sampling_default_seed(self):
         # Test for SPARK-3995 (default seed setting)
-        data = self.sc.parallelize(xrange(1000), 1)
+        data = self.sc.parallelize(range(1000), 1)
         subset = data.takeSample(False, 10)
         self.assertEqual(len(subset), 10)
 
     def test_aggregate_mutable_zero_value(self):
         # Test for SPARK-9021; uses aggregate and treeAggregate to build dict
         # representing a counter of ints
-        # NOTE: dict is used instead of collections.Counter for Python 2.6
-        # compatibility
         from collections import defaultdict
 
         # Show that single or multiple partitions work
@@ -251,8 +256,6 @@ def comboOp(x, y):
     def test_fold_mutable_zero_value(self):
         # Test for SPARK-9021; uses fold to merge an RDD of dict counters into
         # a single dict
-        # NOTE: dict is used instead of collections.Counter for Python 2.6
-        # compatibility
         from collections import defaultdict
 
         counts1 = defaultdict(int, dict((i, 1) for i in range(10)))
@@ -428,7 +431,7 @@ def run(f, sc):
 
     def test_large_closure(self):
         N = 200000
-        data = [float(i) for i in xrange(N)]
+        data = [float(i) for i in range(N)]
         rdd = self.sc.parallelize(range(1), 1).map(lambda x: len(data))
         self.assertEqual(N, rdd.first())
         # regression test for SPARK-6886
@@ -453,8 +456,8 @@ def test_zip_with_different_serializers(self):
 
     def test_zip_with_different_object_sizes(self):
         # regress test for SPARK-5973
-        a = self.sc.parallelize(xrange(10000)).map(lambda i: '*' * i)
-        b = self.sc.parallelize(xrange(10000, 20000)).map(lambda i: '*' * i)
+        a = self.sc.parallelize(range(10000)).map(lambda i: '*' * i)
+        b = self.sc.parallelize(range(10000, 20000)).map(lambda i: '*' * i)
         self.assertEqual(10000, a.zip(b).count())
 
     def test_zip_with_different_number_of_items(self):
@@ -476,7 +479,7 @@ def test_zip_with_different_number_of_items(self):
             self.assertRaises(Exception, lambda: a.zip(b).count())
 
     def test_count_approx_distinct(self):
-        rdd = self.sc.parallelize(xrange(1000))
+        rdd = self.sc.parallelize(range(1000))
         self.assertTrue(950 < rdd.countApproxDistinct(0.03) < 1050)
         self.assertTrue(950 < rdd.map(float).countApproxDistinct(0.03) < 1050)
         self.assertTrue(950 < rdd.map(str).countApproxDistinct(0.03) < 1050)
@@ -630,7 +633,7 @@ def test_distinct(self):
     def test_external_group_by_key(self):
         self.sc._conf.set("spark.python.worker.memory", "1m")
         N = 2000001
-        kv = self.sc.parallelize(xrange(N)).map(lambda x: (x % 3, x))
+        kv = self.sc.parallelize(range(N)).map(lambda x: (x % 3, x))
         gkv = kv.groupByKey().cache()
         self.assertEqual(3, gkv.count())
         filtered = gkv.filter(lambda kv: kv[0] == 1)
@@ -687,7 +690,7 @@ def test_multiple_python_java_RDD_conversions(self):
 
     # Regression test for SPARK-6294
     def test_take_on_jrdd(self):
-        rdd = self.sc.parallelize(xrange(1 << 20)).map(lambda x: str(x))
+        rdd = self.sc.parallelize(range(1 << 20)).map(lambda x: str(x))
         rdd._jrdd.first()
 
     def test_sortByKey_uses_all_partitions_not_only_first_and_last(self):
@@ -730,25 +733,25 @@ def stopit(*x):
         keyed_rdd = self.sc.parallelize((x % 2, x) for x in range(10))
         msg = "Caught StopIteration thrown from user's code; failing the task"
 
-        self.assertRaisesRegexp(Py4JJavaError, msg, seq_rdd.map(stopit).collect)
-        self.assertRaisesRegexp(Py4JJavaError, msg, seq_rdd.filter(stopit).collect)
-        self.assertRaisesRegexp(Py4JJavaError, msg, seq_rdd.foreach, stopit)
-        self.assertRaisesRegexp(Py4JJavaError, msg, seq_rdd.reduce, stopit)
-        self.assertRaisesRegexp(Py4JJavaError, msg, seq_rdd.fold, 0, stopit)
-        self.assertRaisesRegexp(Py4JJavaError, msg, seq_rdd.foreach, stopit)
-        self.assertRaisesRegexp(Py4JJavaError, msg,
-                                seq_rdd.cartesian(seq_rdd).flatMap(stopit).collect)
+        self.assertRaisesRegex(Py4JJavaError, msg, seq_rdd.map(stopit).collect)
+        self.assertRaisesRegex(Py4JJavaError, msg, seq_rdd.filter(stopit).collect)
+        self.assertRaisesRegex(Py4JJavaError, msg, seq_rdd.foreach, stopit)
+        self.assertRaisesRegex(Py4JJavaError, msg, seq_rdd.reduce, stopit)
+        self.assertRaisesRegex(Py4JJavaError, msg, seq_rdd.fold, 0, stopit)
+        self.assertRaisesRegex(Py4JJavaError, msg, seq_rdd.foreach, stopit)
+        self.assertRaisesRegex(Py4JJavaError, msg,
+                               seq_rdd.cartesian(seq_rdd).flatMap(stopit).collect)
 
         # these methods call the user function both in the driver and in the executor
         # the exception raised is different according to where the StopIteration happens
         # RuntimeError is raised if in the driver
         # Py4JJavaError is raised if in the executor (wraps the RuntimeError raised in the worker)
-        self.assertRaisesRegexp((Py4JJavaError, RuntimeError), msg,
-                                keyed_rdd.reduceByKeyLocally, stopit)
-        self.assertRaisesRegexp((Py4JJavaError, RuntimeError), msg,
-                                seq_rdd.aggregate, 0, stopit, lambda *x: 1)
-        self.assertRaisesRegexp((Py4JJavaError, RuntimeError), msg,
-                                seq_rdd.aggregate, 0, lambda *x: 1, stopit)
+        self.assertRaisesRegex((Py4JJavaError, RuntimeError), msg,
+                               keyed_rdd.reduceByKeyLocally, stopit)
+        self.assertRaisesRegex((Py4JJavaError, RuntimeError), msg,
+                               seq_rdd.aggregate, 0, stopit, lambda *x: 1)
+        self.assertRaisesRegex((Py4JJavaError, RuntimeError), msg,
+                               seq_rdd.aggregate, 0, lambda *x: 1, stopit)
 
     def test_overwritten_global_func(self):
         # Regression test for SPARK-27000
@@ -765,7 +768,7 @@ def fail(_):
 
         rdd = self.sc.range(10).map(fail)
 
-        with self.assertRaisesRegexp(Exception, "local iterator error"):
+        with self.assertRaisesRegex(Exception, "local iterator error"):
             for _ in rdd.toLocalIterator():
                 pass
 
@@ -878,10 +881,10 @@ def run_job(job_group, index):
 
 if __name__ == "__main__":
     import unittest
-    from pyspark.tests.test_rdd import *
+    from pyspark.tests.test_rdd import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/tests/test_rddbarrier.py b/python/pyspark/tests/test_rddbarrier.py
index 8534fb4abb876..ba2c4b9ba84d4 100644
--- a/python/pyspark/tests/test_rddbarrier.py
+++ b/python/pyspark/tests/test_rddbarrier.py
@@ -40,10 +40,10 @@ def f(index, iterator):
 
 if __name__ == "__main__":
     import unittest
-    from pyspark.tests.test_rddbarrier import *
+    from pyspark.tests.test_rddbarrier import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/tests/test_readwrite.py b/python/pyspark/tests/test_readwrite.py
index 734b7e4789f61..145b53a5eaaa1 100644
--- a/python/pyspark/tests/test_readwrite.py
+++ b/python/pyspark/tests/test_readwrite.py
@@ -16,10 +16,8 @@
 #
 import os
 import shutil
-import sys
 import tempfile
 import unittest
-from array import array
 
 from pyspark.testing.utils import ReusedPySparkTestCase, SPARK_HOME
 
@@ -38,104 +36,6 @@ def tearDownClass(cls):
         ReusedPySparkTestCase.tearDownClass()
         shutil.rmtree(cls.tempdir.name)
 
-    @unittest.skipIf(sys.version >= "3", "serialize array of byte")
-    def test_sequencefiles(self):
-        basepath = self.tempdir.name
-        ints = sorted(self.sc.sequenceFile(basepath + "/sftestdata/sfint/",
-                                           "org.apache.hadoop.io.IntWritable",
-                                           "org.apache.hadoop.io.Text").collect())
-        ei = [(1, u'aa'), (1, u'aa'), (2, u'aa'), (2, u'bb'), (2, u'bb'), (3, u'cc')]
-        self.assertEqual(ints, ei)
-
-        doubles = sorted(self.sc.sequenceFile(basepath + "/sftestdata/sfdouble/",
-                                              "org.apache.hadoop.io.DoubleWritable",
-                                              "org.apache.hadoop.io.Text").collect())
-        ed = [(1.0, u'aa'), (1.0, u'aa'), (2.0, u'aa'), (2.0, u'bb'), (2.0, u'bb'), (3.0, u'cc')]
-        self.assertEqual(doubles, ed)
-
-        bytes = sorted(self.sc.sequenceFile(basepath + "/sftestdata/sfbytes/",
-                                            "org.apache.hadoop.io.IntWritable",
-                                            "org.apache.hadoop.io.BytesWritable").collect())
-        ebs = [(1, bytearray('aa', 'utf-8')),
-               (1, bytearray('aa', 'utf-8')),
-               (2, bytearray('aa', 'utf-8')),
-               (2, bytearray('bb', 'utf-8')),
-               (2, bytearray('bb', 'utf-8')),
-               (3, bytearray('cc', 'utf-8'))]
-        self.assertEqual(bytes, ebs)
-
-        text = sorted(self.sc.sequenceFile(basepath + "/sftestdata/sftext/",
-                                           "org.apache.hadoop.io.Text",
-                                           "org.apache.hadoop.io.Text").collect())
-        et = [(u'1', u'aa'),
-              (u'1', u'aa'),
-              (u'2', u'aa'),
-              (u'2', u'bb'),
-              (u'2', u'bb'),
-              (u'3', u'cc')]
-        self.assertEqual(text, et)
-
-        bools = sorted(self.sc.sequenceFile(basepath + "/sftestdata/sfbool/",
-                                            "org.apache.hadoop.io.IntWritable",
-                                            "org.apache.hadoop.io.BooleanWritable").collect())
-        eb = [(1, False), (1, True), (2, False), (2, False), (2, True), (3, True)]
-        self.assertEqual(bools, eb)
-
-        nulls = sorted(self.sc.sequenceFile(basepath + "/sftestdata/sfnull/",
-                                            "org.apache.hadoop.io.IntWritable",
-                                            "org.apache.hadoop.io.BooleanWritable").collect())
-        en = [(1, None), (1, None), (2, None), (2, None), (2, None), (3, None)]
-        self.assertEqual(nulls, en)
-
-        maps = self.sc.sequenceFile(basepath + "/sftestdata/sfmap/",
-                                    "org.apache.hadoop.io.IntWritable",
-                                    "org.apache.hadoop.io.MapWritable").collect()
-        em = [(1, {}),
-              (1, {3.0: u'bb'}),
-              (2, {1.0: u'aa'}),
-              (2, {1.0: u'cc'}),
-              (3, {2.0: u'dd'})]
-        for v in maps:
-            self.assertTrue(v in em)
-
-        # arrays get pickled to tuples by default
-        tuples = sorted(self.sc.sequenceFile(
-            basepath + "/sftestdata/sfarray/",
-            "org.apache.hadoop.io.IntWritable",
-            "org.apache.spark.api.python.DoubleArrayWritable").collect())
-        et = [(1, ()),
-              (2, (3.0, 4.0, 5.0)),
-              (3, (4.0, 5.0, 6.0))]
-        self.assertEqual(tuples, et)
-
-        # with custom converters, primitive arrays can stay as arrays
-        arrays = sorted(self.sc.sequenceFile(
-            basepath + "/sftestdata/sfarray/",
-            "org.apache.hadoop.io.IntWritable",
-            "org.apache.spark.api.python.DoubleArrayWritable",
-            valueConverter="org.apache.spark.api.python.WritableToDoubleArrayConverter").collect())
-        ea = [(1, array('d')),
-              (2, array('d', [3.0, 4.0, 5.0])),
-              (3, array('d', [4.0, 5.0, 6.0]))]
-        self.assertEqual(arrays, ea)
-
-        clazz = sorted(self.sc.sequenceFile(basepath + "/sftestdata/sfclass/",
-                                            "org.apache.hadoop.io.Text",
-                                            "org.apache.spark.api.python.TestWritable").collect())
-        cname = u'org.apache.spark.api.python.TestWritable'
-        ec = [(u'1', {u'__class__': cname, u'double': 1.0, u'int': 1, u'str': u'test1'}),
-              (u'2', {u'__class__': cname, u'double': 2.3, u'int': 2, u'str': u'test2'}),
-              (u'3', {u'__class__': cname, u'double': 3.1, u'int': 3, u'str': u'test3'}),
-              (u'4', {u'__class__': cname, u'double': 4.2, u'int': 4, u'str': u'test4'}),
-              (u'5', {u'__class__': cname, u'double': 5.5, u'int': 5, u'str': u'test56'})]
-        self.assertEqual(clazz, ec)
-
-        unbatched_clazz = sorted(self.sc.sequenceFile(basepath + "/sftestdata/sfclass/",
-                                                      "org.apache.hadoop.io.Text",
-                                                      "org.apache.spark.api.python.TestWritable",
-                                                      ).collect())
-        self.assertEqual(unbatched_clazz, ec)
-
     def test_oldhadoop(self):
         basepath = self.tempdir.name
         ints = sorted(self.sc.hadoopFile(basepath + "/sftestdata/sfint/",
@@ -249,51 +149,6 @@ def setUp(self):
     def tearDown(self):
         shutil.rmtree(self.tempdir.name, ignore_errors=True)
 
-    @unittest.skipIf(sys.version >= "3", "serialize array of byte")
-    def test_sequencefiles(self):
-        basepath = self.tempdir.name
-        ei = [(1, u'aa'), (1, u'aa'), (2, u'aa'), (2, u'bb'), (2, u'bb'), (3, u'cc')]
-        self.sc.parallelize(ei).saveAsSequenceFile(basepath + "/sfint/")
-        ints = sorted(self.sc.sequenceFile(basepath + "/sfint/").collect())
-        self.assertEqual(ints, ei)
-
-        ed = [(1.0, u'aa'), (1.0, u'aa'), (2.0, u'aa'), (2.0, u'bb'), (2.0, u'bb'), (3.0, u'cc')]
-        self.sc.parallelize(ed).saveAsSequenceFile(basepath + "/sfdouble/")
-        doubles = sorted(self.sc.sequenceFile(basepath + "/sfdouble/").collect())
-        self.assertEqual(doubles, ed)
-
-        ebs = [(1, bytearray(b'\x00\x07spam\x08')), (2, bytearray(b'\x00\x07spam\x08'))]
-        self.sc.parallelize(ebs).saveAsSequenceFile(basepath + "/sfbytes/")
-        bytes = sorted(self.sc.sequenceFile(basepath + "/sfbytes/").collect())
-        self.assertEqual(bytes, ebs)
-
-        et = [(u'1', u'aa'),
-              (u'2', u'bb'),
-              (u'3', u'cc')]
-        self.sc.parallelize(et).saveAsSequenceFile(basepath + "/sftext/")
-        text = sorted(self.sc.sequenceFile(basepath + "/sftext/").collect())
-        self.assertEqual(text, et)
-
-        eb = [(1, False), (1, True), (2, False), (2, False), (2, True), (3, True)]
-        self.sc.parallelize(eb).saveAsSequenceFile(basepath + "/sfbool/")
-        bools = sorted(self.sc.sequenceFile(basepath + "/sfbool/").collect())
-        self.assertEqual(bools, eb)
-
-        en = [(1, None), (1, None), (2, None), (2, None), (2, None), (3, None)]
-        self.sc.parallelize(en).saveAsSequenceFile(basepath + "/sfnull/")
-        nulls = sorted(self.sc.sequenceFile(basepath + "/sfnull/").collect())
-        self.assertEqual(nulls, en)
-
-        em = [(1, {}),
-              (1, {3.0: u'bb'}),
-              (2, {1.0: u'aa'}),
-              (2, {1.0: u'cc'}),
-              (3, {2.0: u'dd'})]
-        self.sc.parallelize(em).saveAsSequenceFile(basepath + "/sfmap/")
-        maps = self.sc.sequenceFile(basepath + "/sfmap/").collect()
-        for v in maps:
-            self.assertTrue(v, em)
-
     def test_oldhadoop(self):
         basepath = self.tempdir.name
         dict_data = [(1, {}),
@@ -361,46 +216,6 @@ def test_newhadoop(self):
             conf=input_conf).collect())
         self.assertEqual(new_dataset, data)
 
-    @unittest.skipIf(sys.version >= "3", "serialize of array")
-    def test_newhadoop_with_array(self):
-        basepath = self.tempdir.name
-        # use custom ArrayWritable types and converters to handle arrays
-        array_data = [(1, array('d')),
-                      (1, array('d', [1.0, 2.0, 3.0])),
-                      (2, array('d', [3.0, 4.0, 5.0]))]
-        self.sc.parallelize(array_data).saveAsNewAPIHadoopFile(
-            basepath + "/newhadoop/",
-            "org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat",
-            "org.apache.hadoop.io.IntWritable",
-            "org.apache.spark.api.python.DoubleArrayWritable",
-            valueConverter="org.apache.spark.api.python.DoubleArrayToWritableConverter")
-        result = sorted(self.sc.newAPIHadoopFile(
-            basepath + "/newhadoop/",
-            "org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat",
-            "org.apache.hadoop.io.IntWritable",
-            "org.apache.spark.api.python.DoubleArrayWritable",
-            valueConverter="org.apache.spark.api.python.WritableToDoubleArrayConverter").collect())
-        self.assertEqual(result, array_data)
-
-        conf = {
-            "mapreduce.job.outputformat.class":
-                "org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat",
-            "mapreduce.job.output.key.class": "org.apache.hadoop.io.IntWritable",
-            "mapreduce.job.output.value.class": "org.apache.spark.api.python.DoubleArrayWritable",
-            "mapreduce.output.fileoutputformat.outputdir": basepath + "/newdataset/"
-        }
-        self.sc.parallelize(array_data).saveAsNewAPIHadoopDataset(
-            conf,
-            valueConverter="org.apache.spark.api.python.DoubleArrayToWritableConverter")
-        input_conf = {"mapreduce.input.fileinputformat.inputdir": basepath + "/newdataset/"}
-        new_dataset = sorted(self.sc.newAPIHadoopRDD(
-            "org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat",
-            "org.apache.hadoop.io.IntWritable",
-            "org.apache.spark.api.python.DoubleArrayWritable",
-            valueConverter="org.apache.spark.api.python.WritableToDoubleArrayConverter",
-            conf=input_conf).collect())
-        self.assertEqual(new_dataset, array_data)
-
     def test_newolderror(self):
         basepath = self.tempdir.name
         rdd = self.sc.parallelize(range(1, 4)).map(lambda x: (x, "a" * x))
@@ -489,10 +304,10 @@ def test_malformed_RDD(self):
 
 
 if __name__ == "__main__":
-    from pyspark.tests.test_readwrite import *
+    from pyspark.tests.test_readwrite import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/tests/test_serializers.py b/python/pyspark/tests/test_serializers.py
index 8caf9da85a5b4..cc7838e595b8a 100644
--- a/python/pyspark/tests/test_serializers.py
+++ b/python/pyspark/tests/test_serializers.py
@@ -19,10 +19,10 @@
 import unittest
 
 from pyspark import serializers
-from pyspark.serializers import *
 from pyspark.serializers import CloudPickleSerializer, CompressedSerializer, \
     AutoBatchedSerializer, BatchedSerializer, AutoSerializer, NoOpSerializer, PairDeserializer, \
-    FlattenedValuesSerializer, CartesianDeserializer
+    FlattenedValuesSerializer, CartesianDeserializer, PickleSerializer, UTF8Deserializer, \
+    MarshalSerializer
 from pyspark.testing.utils import PySparkTestCase, read_int, write_int, ByteArrayOutput, \
     have_numpy, have_scipy
 
@@ -87,7 +87,7 @@ def __getattr__(self, item):
     def test_pickling_file_handles(self):
         # to be corrected with SPARK-11160
         try:
-            import xmlrunner
+            import xmlrunner  # type: ignore[import]  # noqa: F401
         except ImportError:
             ser = CloudPickleSerializer()
             out1 = sys.stderr
@@ -114,10 +114,7 @@ def foo():
 
     def test_compressed_serializer(self):
         ser = CompressedSerializer(PickleSerializer())
-        try:
-            from StringIO import StringIO
-        except ImportError:
-            from io import BytesIO as StringIO
+        from io import BytesIO as StringIO
         io = StringIO()
         ser.dump_stream(["abc", u"123", range(5)], io)
         io.seek(0)
@@ -227,10 +224,10 @@ def test_chunked_stream(self):
 
 
 if __name__ == "__main__":
-    from pyspark.tests.test_serializers import *
+    from pyspark.tests.test_serializers import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/tests/test_shuffle.py b/python/pyspark/tests/test_shuffle.py
index d50ba632d6cd4..6a245a26b4551 100644
--- a/python/pyspark/tests/test_shuffle.py
+++ b/python/pyspark/tests/test_shuffle.py
@@ -15,7 +15,6 @@
 # limitations under the License.
 #
 import random
-import sys
 import unittest
 
 from py4j.protocol import Py4JJavaError
@@ -23,15 +22,12 @@
 from pyspark import shuffle, PickleSerializer, SparkConf, SparkContext
 from pyspark.shuffle import Aggregator, ExternalMerger, ExternalSorter
 
-if sys.version_info[0] >= 3:
-    xrange = range
-
 
 class MergerTests(unittest.TestCase):
 
     def setUp(self):
         self.N = 1 << 12
-        self.l = [i for i in xrange(self.N)]
+        self.l = [i for i in range(self.N)]
         self.data = list(zip(self.l, self.l))
         self.agg = Aggregator(lambda x: [x],
                               lambda x, y: x.append(y) or x,
@@ -42,26 +38,26 @@ def test_small_dataset(self):
         m.mergeValues(self.data)
         self.assertEqual(m.spills, 0)
         self.assertEqual(sum(sum(v) for k, v in m.items()),
-                         sum(xrange(self.N)))
+                         sum(range(self.N)))
 
         m = ExternalMerger(self.agg, 1000)
         m.mergeCombiners(map(lambda x_y1: (x_y1[0], [x_y1[1]]), self.data))
         self.assertEqual(m.spills, 0)
         self.assertEqual(sum(sum(v) for k, v in m.items()),
-                         sum(xrange(self.N)))
+                         sum(range(self.N)))
 
     def test_medium_dataset(self):
         m = ExternalMerger(self.agg, 20)
         m.mergeValues(self.data)
         self.assertTrue(m.spills >= 1)
         self.assertEqual(sum(sum(v) for k, v in m.items()),
-                         sum(xrange(self.N)))
+                         sum(range(self.N)))
 
         m = ExternalMerger(self.agg, 10)
         m.mergeCombiners(map(lambda x_y2: (x_y2[0], [x_y2[1]]), self.data * 3))
         self.assertTrue(m.spills >= 1)
         self.assertEqual(sum(sum(v) for k, v in m.items()),
-                         sum(xrange(self.N)) * 3)
+                         sum(range(self.N)) * 3)
 
     def test_huge_dataset(self):
         m = ExternalMerger(self.agg, 5, partitions=3)
@@ -171,10 +167,10 @@ def test_external_sort_in_rdd(self):
 
 
 if __name__ == "__main__":
-    from pyspark.tests.test_shuffle import *
+    from pyspark.tests.test_shuffle import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/tests/test_taskcontext.py b/python/pyspark/tests/test_taskcontext.py
index 90e4bcdfadc03..e7161be48b55a 100644
--- a/python/pyspark/tests/test_taskcontext.py
+++ b/python/pyspark/tests/test_taskcontext.py
@@ -16,7 +16,6 @@
 #
 import os
 import random
-import shutil
 import stat
 import sys
 import tempfile
@@ -24,10 +23,7 @@
 import unittest
 
 from pyspark import SparkConf, SparkContext, TaskContext, BarrierTaskContext
-from pyspark.testing.utils import PySparkTestCase, SPARK_HOME
-
-if sys.version_info[0] >= 3:
-    xrange = range
+from pyspark.testing.utils import PySparkTestCase, SPARK_HOME, eventually
 
 
 class TaskContextTests(PySparkTestCase):
@@ -128,12 +124,12 @@ def f(iterator):
 
         def context_barrier(x):
             tc = BarrierTaskContext.get()
-            time.sleep(random.randint(1, 10))
+            time.sleep(random.randint(1, 5) * 2)
             tc.barrier()
             return time.time()
 
         times = rdd.barrier().mapPartitions(f).map(context_barrier).collect()
-        self.assertTrue(max(times) - min(times) < 1)
+        self.assertTrue(max(times) - min(times) < 2)
 
     def test_all_gather(self):
         """
@@ -236,7 +232,7 @@ def f(iterator):
 
         def context_barrier(x):
             tc = BarrierTaskContext.get()
-            time.sleep(random.randint(1, 10))
+            time.sleep(random.randint(1, 5) * 2)
             tc.barrier()
             return (time.time(), os.getpid())
 
@@ -244,16 +240,16 @@ def context_barrier(x):
         times = list(map(lambda x: x[0], result))
         pids = list(map(lambda x: x[1], result))
         # check both barrier and worker reuse effect
-        self.assertTrue(max(times) - min(times) < 1)
+        self.assertTrue(max(times) - min(times) < 2)
         for pid in pids:
             self.assertTrue(pid in worker_pids)
 
-    def test_task_context_correct_with_python_worker_reuse(self):
+    def check_task_context_correct_with_python_worker_reuse(self):
         """Verify the task context correct when reused python worker"""
         # start a normal job first to start all workers and get all worker pids
-        worker_pids = self.sc.parallelize(xrange(2), 2).map(lambda x: os.getpid()).collect()
+        worker_pids = self.sc.parallelize(range(2), 2).map(lambda x: os.getpid()).collect()
         # the worker will reuse in this barrier job
-        rdd = self.sc.parallelize(xrange(10), 2)
+        rdd = self.sc.parallelize(range(10), 2)
 
         def context(iterator):
             tp = TaskContext.get().partitionId()
@@ -267,7 +263,6 @@ def context(iterator):
         # normal stage after normal stage
         normal_result = rdd.mapPartitions(context).collect()
         tps, bps, pids = zip(*normal_result)
-        print(tps)
         self.assertTrue(tps == (0, 1))
         self.assertTrue(bps == (-1, -1))
         for pid in pids:
@@ -286,6 +281,14 @@ def context(iterator):
         self.assertTrue(bps == (-1, -1))
         for pid in pids:
             self.assertTrue(pid in worker_pids)
+        return True
+
+    def test_task_context_correct_with_python_worker_reuse(self):
+        # Retrying the check as the PIDs from Python workers might be different even
+        # when reusing Python workers is enabled if a Python worker is dead for some reasons
+        # (e.g., socket connection failure) and new Python worker is created.
+        eventually(
+            self.check_task_context_correct_with_python_worker_reuse, catch_assertions=True)
 
     def tearDown(self):
         self.sc.stop()
@@ -325,10 +328,10 @@ def tearDown(self):
 
 if __name__ == "__main__":
     import unittest
-    from pyspark.tests.test_taskcontext import *
+    from pyspark.tests.test_taskcontext import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/tests/test_util.py b/python/pyspark/tests/test_util.py
index 81bfb66e7019d..a25c41b296944 100644
--- a/python/pyspark/tests/test_util.py
+++ b/python/pyspark/tests/test_util.py
@@ -61,14 +61,12 @@ def set(self, x=None, other=None, other_x=None):
 
 
 class UtilTests(PySparkTestCase):
-    def test_py4j_exception_message(self):
-        from pyspark.util import _exception_message
-
+    def test_py4j_str(self):
         with self.assertRaises(Py4JJavaError) as context:
             # This attempts java.lang.String(null) which throws an NPE.
             self.sc._jvm.java.lang.String(None)
 
-        self.assertTrue('NullPointerException' in _exception_message(context.exception))
+        self.assertTrue('NullPointerException' in str(context.exception))
 
     def test_parsing_version_string(self):
         from pyspark.util import VersionUtils
@@ -76,10 +74,10 @@ def test_parsing_version_string(self):
 
 
 if __name__ == "__main__":
-    from pyspark.tests.test_util import *
+    from pyspark.tests.test_util import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/tests/test_worker.py b/python/pyspark/tests/test_worker.py
index dba9298ee161a..51ebee4de7cec 100644
--- a/python/pyspark/tests/test_worker.py
+++ b/python/pyspark/tests/test_worker.py
@@ -16,14 +16,13 @@
 # limitations under the License.
 #
 import os
-import sys
 import tempfile
 import threading
 import time
 import unittest
 has_resource_module = True
 try:
-    import resource
+    import resource  # noqa: F401
 except ImportError:
     has_resource_module = False
 
@@ -32,9 +31,6 @@
 from pyspark import SparkConf, SparkContext
 from pyspark.testing.utils import ReusedPySparkTestCase, PySparkTestCase, QuietTest
 
-if sys.version_info[0] >= 3:
-    xrange = range
-
 
 class WorkerTests(ReusedPySparkTestCase):
     def test_cancel_task(self):
@@ -88,17 +84,26 @@ def run():
             self.fail("daemon had been killed")
 
         # run a normal job
-        rdd = self.sc.parallelize(xrange(100), 1)
+        rdd = self.sc.parallelize(range(100), 1)
         self.assertEqual(100, rdd.map(str).count())
 
     def test_after_exception(self):
         def raise_exception(_):
             raise Exception()
-        rdd = self.sc.parallelize(xrange(100), 1)
+        rdd = self.sc.parallelize(range(100), 1)
         with QuietTest(self.sc):
             self.assertRaises(Exception, lambda: rdd.foreach(raise_exception))
         self.assertEqual(100, rdd.map(str).count())
 
+    def test_after_non_exception_error(self):
+        # SPARK-33339: Pyspark application will hang due to non Exception
+        def raise_system_exit(_):
+            raise SystemExit()
+        rdd = self.sc.parallelize(range(100), 1)
+        with QuietTest(self.sc):
+            self.assertRaises(Exception, lambda: rdd.foreach(raise_system_exit))
+        self.assertEqual(100, rdd.map(str).count())
+
     def test_after_jvm_exception(self):
         tempFile = tempfile.NamedTemporaryFile(delete=False)
         tempFile.write(b"Hello World!")
@@ -110,22 +115,22 @@ def test_after_jvm_exception(self):
         with QuietTest(self.sc):
             self.assertRaises(Exception, lambda: filtered_data.count())
 
-        rdd = self.sc.parallelize(xrange(100), 1)
+        rdd = self.sc.parallelize(range(100), 1)
         self.assertEqual(100, rdd.map(str).count())
 
     def test_accumulator_when_reuse_worker(self):
         from pyspark.accumulators import INT_ACCUMULATOR_PARAM
         acc1 = self.sc.accumulator(0, INT_ACCUMULATOR_PARAM)
-        self.sc.parallelize(xrange(100), 20).foreach(lambda x: acc1.add(x))
+        self.sc.parallelize(range(100), 20).foreach(lambda x: acc1.add(x))
         self.assertEqual(sum(range(100)), acc1.value)
 
         acc2 = self.sc.accumulator(0, INT_ACCUMULATOR_PARAM)
-        self.sc.parallelize(xrange(100), 20).foreach(lambda x: acc2.add(x))
+        self.sc.parallelize(range(100), 20).foreach(lambda x: acc2.add(x))
         self.assertEqual(sum(range(100)), acc2.value)
         self.assertEqual(sum(range(100)), acc1.value)
 
     def test_reuse_worker_after_take(self):
-        rdd = self.sc.parallelize(xrange(100000), 1)
+        rdd = self.sc.parallelize(range(100000), 1)
         self.assertEqual(0, rdd.first())
 
         def count():
@@ -138,7 +143,7 @@ def count():
         t.daemon = True
         t.start()
         t.join(5)
-        self.assertTrue(not t.isAlive())
+        self.assertTrue(not t.is_alive())
         self.assertEqual(100000, rdd.count())
 
     def test_with_different_versions_of_python(self):
@@ -160,17 +165,13 @@ def f():
 
             self.sc.parallelize([1]).map(lambda x: f()).count()
         except Py4JJavaError as e:
-            if sys.version_info.major < 3:
-                # we have to use unicode here to avoid UnicodeDecodeError
-                self.assertRegexpMatches(unicode(e).encode("utf-8"), "exception with 中")
-            else:
-                self.assertRegexpMatches(str(e), "exception with 中")
+            self.assertRegex(str(e), "exception with 中")
 
 
 class WorkerReuseTest(PySparkTestCase):
 
-    def test_reuse_worker_of_parallelize_xrange(self):
-        rdd = self.sc.parallelize(xrange(20), 8)
+    def test_reuse_worker_of_parallelize_range(self):
+        rdd = self.sc.parallelize(range(20), 8)
         previous_pids = rdd.map(lambda x: os.getpid()).collect()
         current_pids = rdd.map(lambda x: os.getpid()).collect()
         for pid in current_pids:
@@ -189,7 +190,7 @@ def setUp(self):
         self.sc = SparkContext('local[4]', class_name, conf=conf)
 
     def test_memory_limit(self):
-        rdd = self.sc.parallelize(xrange(1), 1)
+        rdd = self.sc.parallelize(range(1), 1)
 
         def getrlimit():
             import resource
@@ -207,10 +208,10 @@ def tearDown(self):
 
 if __name__ == "__main__":
     import unittest
-    from pyspark.tests.test_worker import *
+    from pyspark.tests.test_worker import *  # noqa: F401
 
     try:
-        import xmlrunner
+        import xmlrunner  # type: ignore[import]
         testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
     except ImportError:
         testRunner = None
diff --git a/python/pyspark/util.py b/python/pyspark/util.py
index cc614a2c0eaed..09c5963927456 100644
--- a/python/pyspark/util.py
+++ b/python/pyspark/util.py
@@ -16,53 +16,18 @@
 # limitations under the License.
 #
 
+import itertools
+import os
+import platform
 import re
 import sys
+import threading
 import traceback
-import os
-import warnings
-import inspect
-from py4j.protocol import Py4JJavaError
-
-__all__ = []
+import types
 
+from py4j.clientserver import ClientServer
 
-def _exception_message(excp):
-    """Return the message from an exception as either a str or unicode object.  Supports both
-    Python 2 and Python 3.
-
-    >>> msg = "Exception message"
-    >>> excp = Exception(msg)
-    >>> msg == _exception_message(excp)
-    True
-
-    >>> msg = u"unicöde"
-    >>> excp = Exception(msg)
-    >>> msg == _exception_message(excp)
-    True
-    """
-    if isinstance(excp, Py4JJavaError):
-        # 'Py4JJavaError' doesn't contain the stack trace available on the Java side in 'message'
-        # attribute in Python 2. We should call 'str' function on this exception in general but
-        # 'Py4JJavaError' has an issue about addressing non-ascii strings. So, here we work
-        # around by the direct call, '__str__()'. Please see SPARK-23517.
-        return excp.__str__()
-    if hasattr(excp, "message"):
-        return excp.message
-    return str(excp)
-
-
-def _get_argspec(f):
-    """
-    Get argspec of a function. Supports both Python 2 and Python 3.
-    """
-    if sys.version_info[0] < 3:
-        argspec = inspect.getargspec(f)
-    else:
-        # `getargspec` is deprecated since python3.0 (incompatible with function annotations).
-        # See SPARK-23569.
-        argspec = inspect.getfullargspec(f)
-    return argspec
+__all__ = []  # type: ignore
 
 
 def print_exec(stream):
@@ -80,13 +45,14 @@ def majorMinorVersion(sparkVersion):
         Given a Spark version string, return the (major version number, minor version number).
         E.g., for 2.0.1-SNAPSHOT, return (2, 0).
 
+        Examples
+        --------
         >>> sparkVersion = "2.4.0"
         >>> VersionUtils.majorMinorVersion(sparkVersion)
         (2, 4)
         >>> sparkVersion = "2.3.0-SNAPSHOT"
         >>> VersionUtils.majorMinorVersion(sparkVersion)
         (2, 3)
-
         """
         m = re.search(r'^(\d+)\.(\d+)(\..*)?$', sparkVersion)
         if m is not None:
@@ -114,31 +80,142 @@ def wrapper(*args, **kwargs):
     return wrapper
 
 
-def _warn_pin_thread(name):
-    if os.environ.get("PYSPARK_PIN_THREAD", "false").lower() == "true":
-        msg = (
-            "PYSPARK_PIN_THREAD feature is enabled. "
-            "However, note that it cannot inherit the local properties from the parent thread "
-            "although it isolates each thread on PVM and JVM with its own local properties. "
-            "\n"
-            "To work around this, you should manually copy and set the local properties from "
-            "the parent thread to the child thread when you create another thread.")
-    else:
-        msg = (
-            "Currently, '%s' (set to local properties) with multiple threads does "
-            "not properly work. "
-            "\n"
-            "Internally threads on PVM and JVM are not synced, and JVM thread can be reused "
-            "for multiple threads on PVM, which fails to isolate local properties for each "
-            "thread on PVM. "
-            "\n"
-            "To work around this, you can set PYSPARK_PIN_THREAD to true (see SPARK-22340). "
-            "However, note that it cannot inherit the local properties from the parent thread "
-            "although it isolates each thread on PVM and JVM with its own local properties. "
-            "\n"
-            "To work around this, you should manually copy and set the local properties from "
-            "the parent thread to the child thread when you create another thread." % name)
-    warnings.warn(msg, UserWarning)
+def walk_tb(tb):
+    while tb is not None:
+        yield tb
+        tb = tb.tb_next
+
+
+def try_simplify_traceback(tb):
+    """
+    Simplify the traceback. It removes the tracebacks in the current package, and only
+    shows the traceback that is related to the thirdparty and user-specified codes.
+
+    Returns
+    -------
+    TracebackType or None
+      Simplified traceback instance. It returns None if it fails to simplify.
+
+    Notes
+    -----
+    This keeps the tracebacks once it sees they are from a different file even
+    though the following tracebacks are from the current package.
+
+    Examples
+    --------
+    >>> import importlib
+    >>> import sys
+    >>> import traceback
+    >>> import tempfile
+    >>> with tempfile.TemporaryDirectory() as tmp_dir:
+    ...     with open("%s/dummy_module.py" % tmp_dir, "w") as f:
+    ...         _ = f.write(
+    ...             'def raise_stop_iteration():\\n'
+    ...             '    raise StopIteration()\\n\\n'
+    ...             'def simple_wrapper(f):\\n'
+    ...             '    def wrapper(*a, **k):\\n'
+    ...             '        return f(*a, **k)\\n'
+    ...             '    return wrapper\\n')
+    ...         f.flush()
+    ...         spec = importlib.util.spec_from_file_location(
+    ...             "dummy_module", "%s/dummy_module.py" % tmp_dir)
+    ...         dummy_module = importlib.util.module_from_spec(spec)
+    ...         spec.loader.exec_module(dummy_module)
+    >>> def skip_doctest_traceback(tb):
+    ...     import pyspark
+    ...     root = os.path.dirname(pyspark.__file__)
+    ...     pairs = zip(walk_tb(tb), traceback.extract_tb(tb))
+    ...     for cur_tb, cur_frame in pairs:
+    ...         if cur_frame.filename.startswith(root):
+    ...             return cur_tb
+
+    Regular exceptions should show the file name of the current package as below.
+
+    >>> exc_info = None
+    >>> try:
+    ...     fail_on_stopiteration(dummy_module.raise_stop_iteration)()
+    ... except Exception as e:
+    ...     tb = sys.exc_info()[-1]
+    ...     e.__cause__ = None
+    ...     exc_info = "".join(
+    ...         traceback.format_exception(type(e), e, tb))
+    >>> print(exc_info)  # doctest: +NORMALIZE_WHITESPACE, +ELLIPSIS
+    Traceback (most recent call last):
+      File ...
+        ...
+      File "/.../pyspark/util.py", line ...
+        ...
+    RuntimeError: ...
+    >>> "pyspark/util.py" in exc_info
+    True
+
+    If the traceback is simplified with this method, it hides the current package file name:
+
+    >>> exc_info = None
+    >>> try:
+    ...     fail_on_stopiteration(dummy_module.raise_stop_iteration)()
+    ... except Exception as e:
+    ...     tb = try_simplify_traceback(sys.exc_info()[-1])
+    ...     e.__cause__ = None
+    ...     exc_info = "".join(
+    ...         traceback.format_exception(
+    ...             type(e), e, try_simplify_traceback(skip_doctest_traceback(tb))))
+    >>> print(exc_info)  # doctest: +NORMALIZE_WHITESPACE, +ELLIPSIS
+    RuntimeError: ...
+    >>> "pyspark/util.py" in exc_info
+    False
+
+    In the case below, the traceback contains the current package in the middle.
+    In this case, it just hides the top occurrence only.
+
+    >>> exc_info = None
+    >>> try:
+    ...     fail_on_stopiteration(dummy_module.simple_wrapper(
+    ...         fail_on_stopiteration(dummy_module.raise_stop_iteration)))()
+    ... except Exception as e:
+    ...     tb = sys.exc_info()[-1]
+    ...     e.__cause__ = None
+    ...     exc_info_a = "".join(
+    ...         traceback.format_exception(type(e), e, tb))
+    ...     exc_info_b = "".join(
+    ...         traceback.format_exception(
+    ...             type(e), e, try_simplify_traceback(skip_doctest_traceback(tb))))
+    >>> exc_info_a.count("pyspark/util.py")
+    2
+    >>> exc_info_b.count("pyspark/util.py")
+    1
+    """
+    if "pypy" in platform.python_implementation().lower():
+        # Traceback modification is not supported with PyPy in PySpark.
+        return None
+    if sys.version_info[:2] < (3, 7):
+        # Traceback creation is not supported Python < 3.7.
+        # See https://bugs.python.org/issue30579.
+        return None
+
+    import pyspark
+
+    root = os.path.dirname(pyspark.__file__)
+    tb_next = None
+    new_tb = None
+    pairs = zip(walk_tb(tb), traceback.extract_tb(tb))
+    last_seen = []
+
+    for cur_tb, cur_frame in pairs:
+        if not cur_frame.filename.startswith(root):
+            # Filter the stacktrace from the PySpark source itself.
+            last_seen = [(cur_tb, cur_frame)]
+            break
+
+    for cur_tb, cur_frame in reversed(list(itertools.chain(last_seen, pairs))):
+        # Once we have seen the file names outside, don't skip.
+        new_tb = types.TracebackType(
+            tb_next=tb_next,
+            tb_frame=cur_tb.tb_frame,
+            tb_lasti=cur_tb.tb_frame.f_lasti,
+            tb_lineno=cur_tb.tb_frame.f_lineno)
+        tb_next = new_tb
+    return new_tb
 
 
 def _print_missing_jar(lib_name, pkg_name, jar_name, spark_version):
@@ -173,6 +250,8 @@ def _parse_memory(s):
     Parse a memory string in the format supported by Java (e.g. 1g, 200m) and
     return the value in MiB
 
+    Examples
+    --------
     >>> _parse_memory("256m")
     256
     >>> _parse_memory("2g")
@@ -183,8 +262,71 @@ def _parse_memory(s):
         raise ValueError("invalid format: " + s)
     return int(float(s[:-1]) * units[s[-1].lower()])
 
+
+class InheritableThread(threading.Thread):
+    """
+    Thread that is recommended to be used in PySpark instead of :class:`threading.Thread`
+    when the pinned thread mode is enabled. The usage of this class is exactly same as
+    :class:`threading.Thread` but correctly inherits the inheritable properties specific
+    to JVM thread such as ``InheritableThreadLocal``.
+
+    Also, note that pinned thread mode does not close the connection from Python
+    to JVM when the thread is finished in the Python side. With this class, Python
+    garbage-collects the Python thread instance and also closes the connection
+    which finishes JVM thread correctly.
+
+    When the pinned thread mode is off, this works as :class:`threading.Thread`.
+
+    .. versionadded:: 3.1.0
+
+
+    Notes
+    -----
+    This API is experimental.
+    """
+    def __init__(self, target, *args, **kwargs):
+        from pyspark import SparkContext
+
+        sc = SparkContext._active_spark_context
+
+        if isinstance(sc._gateway, ClientServer):
+            # Here's when the pinned-thread mode (PYSPARK_PIN_THREAD) is on.
+            properties = sc._jsc.sc().getLocalProperties().clone()
+            self._sc = sc
+
+            def copy_local_properties(*a, **k):
+                sc._jsc.sc().setLocalProperties(properties)
+                return target(*a, **k)
+
+            super(InheritableThread, self).__init__(
+                target=copy_local_properties, *args, **kwargs)
+        else:
+            super(InheritableThread, self).__init__(target=target, *args, **kwargs)
+
+    def __del__(self):
+        from pyspark import SparkContext
+
+        if isinstance(SparkContext._gateway, ClientServer):
+            thread_connection = self._sc._jvm._gateway_client.thread_connection.connection()
+            if thread_connection is not None:
+                connections = self._sc._jvm._gateway_client.deque
+
+                # Reuse the lock for Py4J in PySpark
+                with SparkContext._lock:
+                    for i in range(len(connections)):
+                        if connections[i] is thread_connection:
+                            connections[i].close()
+                            del connections[i]
+                            break
+                    else:
+                        # Just in case the connection was not closed but removed from the queue.
+                        thread_connection.close()
+
+
 if __name__ == "__main__":
     import doctest
-    (failure_count, test_count) = doctest.testmod()
-    if failure_count:
-        sys.exit(-1)
+
+    if "pypy" not in platform.python_implementation().lower() and sys.version_info[:2] >= (3, 7):
+        (failure_count, test_count) = doctest.testmod()
+        if failure_count:
+            sys.exit(-1)
diff --git a/python/pyspark/version.py b/python/pyspark/version.py
index e8da19fc44185..935795190797f 100644
--- a/python/pyspark/version.py
+++ b/python/pyspark/version.py
@@ -16,4 +16,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.1.0.dev0"
+__version__ = "3.2.0.dev0"
diff --git a/python/pyspark/version.pyi b/python/pyspark/version.pyi
new file mode 100644
index 0000000000000..444dae62f0c09
--- /dev/null
+++ b/python/pyspark/version.pyi
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+__version__: str
diff --git a/python/pyspark/worker.py b/python/pyspark/worker.py
index 988941e7550b9..84ba31f162e12 100644
--- a/python/pyspark/worker.py
+++ b/python/pyspark/worker.py
@@ -18,11 +18,11 @@
 """
 Worker that receives input from Piped RDD.
 """
-from __future__ import print_function
-from __future__ import absolute_import
 import os
 import sys
 import time
+from inspect import currentframe, getframeinfo, getfullargspec
+import importlib
 # 'resource' is a Unix specific module.
 has_resource_module = True
 try:
@@ -30,13 +30,14 @@
 except ImportError:
     has_resource_module = False
 import traceback
+import warnings
 
 from pyspark.accumulators import _accumulatorRegistry
 from pyspark.broadcast import Broadcast, _broadcastRegistry
 from pyspark.java_gateway import local_connect_and_auth
 from pyspark.taskcontext import BarrierTaskContext, TaskContext
 from pyspark.files import SparkFiles
-from pyspark.resourceinformation import ResourceInformation
+from pyspark.resource import ResourceInformation
 from pyspark.rdd import PythonEvalType
 from pyspark.serializers import write_with_length, write_int, read_long, read_bool, \
     write_long, read_int, SpecialLengths, UTF8Deserializer, PickleSerializer, \
@@ -44,14 +45,9 @@
 from pyspark.sql.pandas.serializers import ArrowStreamPandasUDFSerializer, CogroupUDFSerializer
 from pyspark.sql.pandas.types import to_arrow_type
 from pyspark.sql.types import StructType
-from pyspark.util import _get_argspec, fail_on_stopiteration
+from pyspark.util import fail_on_stopiteration, try_simplify_traceback
 from pyspark import shuffle
 
-if sys.version >= '3':
-    basestring = str
-else:
-    from itertools import imap as map  # use iterator map by default
-
 pickleSer = PickleSerializer()
 utf8_deserializer = UTF8Deserializer()
 
@@ -64,7 +60,7 @@ def report_times(outfile, boot, init, finish):
 
 
 def add_path(path):
-    # worker can be used, so donot add path multiple times
+    # worker can be used, so do not add path multiple times
     if path not in sys.path:
         # overwrite system packages
         sys.path.insert(1, path)
@@ -272,10 +268,10 @@ def read_single_udf(pickleSer, infile, eval_type, runner_conf, udf_index):
     elif eval_type == PythonEvalType.SQL_MAP_PANDAS_ITER_UDF:
         return arg_offsets, wrap_pandas_iter_udf(func, return_type)
     elif eval_type == PythonEvalType.SQL_GROUPED_MAP_PANDAS_UDF:
-        argspec = _get_argspec(chained_func)  # signature was lost when wrapping it
+        argspec = getfullargspec(chained_func)  # signature was lost when wrapping it
         return arg_offsets, wrap_grouped_map_pandas_udf(func, return_type, argspec)
     elif eval_type == PythonEvalType.SQL_COGROUPED_MAP_PANDAS_UDF:
-        argspec = _get_argspec(chained_func)  # signature was lost when wrapping it
+        argspec = getfullargspec(chained_func)  # signature was lost when wrapping it
         return arg_offsets, wrap_cogrouped_map_pandas_udf(func, return_type, argspec)
     elif eval_type == PythonEvalType.SQL_GROUPED_AGG_PANDAS_UDF:
         return arg_offsets, wrap_grouped_agg_pandas_udf(func, return_type)
@@ -342,11 +338,13 @@ def read_udfs(pickleSer, infile, eval_type):
             pickleSer, infile, eval_type, runner_conf, udf_index=0)
 
         def func(_, iterator):
-            num_input_rows = [0]  # TODO(SPARK-29909): Use nonlocal after we drop Python 2.
+            num_input_rows = 0
 
             def map_batch(batch):
+                nonlocal num_input_rows
+
                 udf_args = [batch[offset] for offset in arg_offsets]
-                num_input_rows[0] += len(udf_args[0])
+                num_input_rows += len(udf_args[0])
                 if len(udf_args) == 1:
                     return udf_args[0]
                 else:
@@ -363,7 +361,7 @@ def map_batch(batch):
                 # by consuming the input iterator in user side. Therefore,
                 # it's very unlikely the output length is higher than
                 # input length.
-                assert is_map_iter or num_output_rows <= num_input_rows[0], \
+                assert is_map_iter or num_output_rows <= num_input_rows, \
                     "Pandas SCALAR_ITER UDF outputted more rows than input rows."
                 yield (result_batch, result_type)
 
@@ -376,11 +374,11 @@ def map_batch(batch):
                     raise RuntimeError("pandas iterator UDF should exhaust the input "
                                        "iterator.")
 
-                if num_output_rows != num_input_rows[0]:
+                if num_output_rows != num_input_rows:
                     raise RuntimeError(
                         "The length of output in Scalar iterator pandas UDF should be "
                         "the same with the input's; however, the length of output was %d and the "
-                        "length of input was %d." % (num_output_rows, num_input_rows[0]))
+                        "length of input was %d." % (num_output_rows, num_input_rows))
 
         # profiling is not supported for UDF
         return func, None, ser, ser
@@ -390,7 +388,10 @@ def extract_key_value_indexes(grouped_arg_offsets):
         Helper function to extract the key and value indexes from arg_offsets for the grouped and
         cogrouped pandas udfs. See BasePandasGroupExec.resolveArgOffsets for equivalent scala code.
 
-        :param grouped_arg_offsets:  List containing the key and value indexes of columns of the
+        Parameters
+        ----------
+        grouped_arg_offsets:  list
+            List containing the key and value indexes of columns of the
             DataFrames to be passed to the udf. It consists of n repeating groups where n is the
             number of DataFrames.  Each group has the following format:
                 group[0]: length of group
@@ -471,7 +472,7 @@ def main(infile, outfile):
         version = utf8_deserializer.loads(infile)
         if version != "%d.%d" % sys.version_info[:2]:
             raise Exception(("Python in worker has different version %s than that in " +
-                             "driver %s, PySpark cannot run with different minor versions." +
+                             "driver %s, PySpark cannot run with different minor versions. " +
                              "Please check environment variables PYSPARK_PYTHON and " +
                              "PYSPARK_DRIVER_PYTHON are correctly set.") %
                             ("%d.%d" % sys.version_info[:2], version))
@@ -500,7 +501,14 @@ def main(infile, outfile):
 
             except (resource.error, OSError, ValueError) as e:
                 # not all systems support resource limits, so warn instead of failing
-                print("WARN: Failed to set memory limit: {0}\n".format(e), file=sys.stderr)
+                lineno = getframeinfo(
+                    currentframe()).lineno + 1 if currentframe() is not None else 0
+                print(warnings.formatwarning(
+                    "Failed to set memory limit: {0}".format(e),
+                    ResourceWarning,
+                    __file__,
+                    lineno
+                ), file=sys.stderr)
 
         # initialize global state
         taskContext = None
@@ -548,9 +556,8 @@ def main(infile, outfile):
         for _ in range(num_python_includes):
             filename = utf8_deserializer.loads(infile)
             add_path(os.path.join(spark_files_dir, filename))
-        if sys.version > '3':
-            import importlib
-            importlib.invalidate_caches()
+
+        importlib.invalidate_caches()
 
         # fetch names and values of broadcast variables
         needs_broadcast_decryption_server = read_bool(infile)
@@ -608,21 +615,23 @@ def process():
         # reuse.
         TaskContext._setTaskContext(None)
         BarrierTaskContext._setTaskContext(None)
-    except Exception:
+    except BaseException as e:
         try:
-            exc_info = traceback.format_exc()
-            if isinstance(exc_info, bytes):
-                # exc_info may contains other encoding bytes, replace the invalid bytes and convert
-                # it back to utf-8 again
-                exc_info = exc_info.decode("utf-8", "replace").encode("utf-8")
-            else:
-                exc_info = exc_info.encode("utf-8")
+            exc_info = None
+            if os.environ.get("SPARK_SIMPLIFIED_TRACEBACK", False):
+                tb = try_simplify_traceback(sys.exc_info()[-1])
+                if tb is not None:
+                    e.__cause__ = None
+                    exc_info = "".join(traceback.format_exception(type(e), e, tb))
+            if exc_info is None:
+                exc_info = traceback.format_exc()
+
             write_int(SpecialLengths.PYTHON_EXCEPTION_THROWN, outfile)
-            write_with_length(exc_info, outfile)
+            write_with_length(exc_info.encode("utf-8"), outfile)
         except IOError:
             # JVM close the socket
             pass
-        except Exception:
+        except BaseException:
             # Write the error to stderr if it happened while serializing
             print("PySpark worker failed with exception:", file=sys.stderr)
             print(traceback.format_exc(), file=sys.stderr)
diff --git a/python/run-tests-with-coverage b/python/run-tests-with-coverage
index 457821037d43c..3bc61e5326e31 100755
--- a/python/run-tests-with-coverage
+++ b/python/run-tests-with-coverage
@@ -56,7 +56,7 @@ export COVERAGE_PROCESS_START="$FWDIR/.coveragerc"
 unset COVERAGE_PROCESS_START
 
 # Coverage could generate empty coverage data files. Remove it to get rid of warnings when combining.
-find $COVERAGE_DIR/coverage_data -size 0 -print0 | xargs -0 rm
+find $COVERAGE_DIR/coverage_data -size 0 -print0 | xargs -0 rm -fr
 echo "Combining collected coverage data under $COVERAGE_DIR/coverage_data"
 $COV_EXEC combine
 echo "Reporting the coverage data at $COVERAGE_DIR/coverage_data/coverage"
diff --git a/python/run-tests.py b/python/run-tests.py
index b677a5134ec93..fd9f287710b29 100755
--- a/python/run-tests.py
+++ b/python/run-tests.py
@@ -28,10 +28,7 @@
 from threading import Thread, Lock
 import time
 import uuid
-if sys.version < '3':
-    import Queue
-else:
-    import queue as Queue
+import queue as Queue
 from multiprocessing import Manager
 
 
@@ -51,7 +48,7 @@ def print_red(text):
     print('\033[31m' + text + '\033[0m')
 
 
-SKIPPED_TESTS = Manager().dict()
+SKIPPED_TESTS = None
 LOG_FILE = os.path.join(SPARK_HOME, "python/unit-tests.log")
 FAILURE_REPORTING_LOCK = Lock()
 LOGGER = logging.getLogger()
@@ -75,7 +72,8 @@ def run_individual_python_test(target_dir, test_name, pyspark_python):
         'SPARK_PREPEND_CLASSES': '1',
         'PYSPARK_PYTHON': which(pyspark_python),
         'PYSPARK_DRIVER_PYTHON': which(pyspark_python),
-        'PYSPARK_ROW_FIELD_SORTING_ENABLED': 'true'
+        # Preserve legacy nested timezone behavior for pyarrow>=2, remove after SPARK-32285
+        'PYARROW_IGNORE_TIMEZONE': '1',
     })
 
     # Create a unique temp directory under 'target/' for each run. The TMPDIR variable is
@@ -85,12 +83,17 @@ def run_individual_python_test(target_dir, test_name, pyspark_python):
         tmp_dir = os.path.join(target_dir, str(uuid.uuid4()))
     os.mkdir(tmp_dir)
     env["TMPDIR"] = tmp_dir
+    metastore_dir = os.path.join(tmp_dir, str(uuid.uuid4()))
+    while os.path.isdir(metastore_dir):
+        metastore_dir = os.path.join(metastore_dir, str(uuid.uuid4()))
+    os.mkdir(metastore_dir)
 
     # Also override the JVM's temp directory by setting driver and executor options.
     java_options = "-Djava.io.tmpdir={0} -Dio.netty.tryReflectionSetAccessible=true".format(tmp_dir)
     spark_args = [
         "--conf", "spark.driver.extraJavaOptions='{0}'".format(java_options),
         "--conf", "spark.executor.extraJavaOptions='{0}'".format(java_options),
+        "--conf", "spark.sql.warehouse.dir='{0}'".format(metastore_dir),
         "pyspark-shell"
     ]
     env["PYSPARK_SUBMIT_ARGS"] = " ".join(spark_args)
@@ -142,6 +145,7 @@ def run_individual_python_test(target_dir, test_name, pyspark_python):
             skipped_counts = len(skipped_tests)
             if skipped_counts > 0:
                 key = (pyspark_python, test_name)
+                assert SKIPPED_TESTS is not None
                 SKIPPED_TESTS[key] = skipped_tests
             per_test_output.close()
         except:
@@ -161,7 +165,7 @@ def run_individual_python_test(target_dir, test_name, pyspark_python):
 
 
 def get_default_python_executables():
-    python_execs = [x for x in ["python3.6", "python2.7", "pypy"] if which(x)]
+    python_execs = [x for x in ["python3.6", "pypy3"] if which(x)]
 
     if "python3.6" not in python_execs:
         p = which("python3")
@@ -272,10 +276,11 @@ def main():
             [python_exec, "--version"], stderr=subprocess.STDOUT, universal_newlines=True).strip())
         if should_test_modules:
             for module in modules_to_test:
-                if python_implementation not in module.blacklisted_python_implementations:
+                if python_implementation not in module.excluded_python_implementations:
                     for test_goal in module.python_test_goals:
                         heavy_tests = ['pyspark.streaming.tests', 'pyspark.mllib.tests',
-                                       'pyspark.tests', 'pyspark.sql.tests', 'pyspark.ml.tests']
+                                       'pyspark.tests', 'pyspark.sql.tests', 'pyspark.ml.tests',
+                                       'pyspark.pandas.tests']
                         if any(map(lambda prefix: test_goal.startswith(prefix), heavy_tests)):
                             priority = 0
                         else:
@@ -322,4 +327,5 @@ def process_queue(task_queue):
 
 
 if __name__ == "__main__":
+    SKIPPED_TESTS = Manager().dict()
     main()
diff --git a/python/setup.py b/python/setup.py
index afbd601b04a94..5c4a1aeea083e 100755
--- a/python/setup.py
+++ b/python/setup.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
@@ -16,24 +16,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
+import importlib.util
 import glob
 import os
 import sys
 from setuptools import setup
+from setuptools.command.install import install
 from shutil import copyfile, copytree, rmtree
 
-if sys.version_info < (2, 7):
-    print("Python versions prior to 2.7 are not supported for pip installed PySpark.",
-          file=sys.stderr)
-    sys.exit(-1)
-
 try:
     exec(open('pyspark/version.py').read())
 except IOError:
     print("Failed to load PySpark version file for packaging. You must be in Spark's python dir.",
           file=sys.stderr)
     sys.exit(-1)
+try:
+    spec = importlib.util.spec_from_file_location("install", "pyspark/install.py")
+    install_module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(install_module)
+except IOError:
+    print("Failed to load the installing module (pyspark/install.py) which had to be "
+          "packaged together.",
+          file=sys.stderr)
+    sys.exit(-1)
 VERSION = __version__  # noqa
 # A temporary path so we can access above the Python project root and fetch scripts and jars we need
 TEMP_PATH = "deps"
@@ -102,11 +107,45 @@ def _supports_symlinks():
               file=sys.stderr)
         sys.exit(-1)
 
-# If you are changing the versions here, please also change ./python/pyspark/sql/utils.py
+# If you are changing the versions here, please also change ./python/pyspark/sql/pandas/utils.py
 # For Arrow, you should also check ./pom.xml and ensure there are no breaking changes in the
 # binary format protocol with the Java version, see ARROW_HOME/format/* for specifications.
+# Also don't forget to update python/docs/source/getting_started/install.rst.
 _minimum_pandas_version = "0.23.2"
-_minimum_pyarrow_version = "0.15.1"
+_minimum_pyarrow_version = "1.0.0"
+
+
+class InstallCommand(install):
+    # TODO(SPARK-32837) leverage pip's custom options
+
+    def run(self):
+        install.run(self)
+
+        # Make sure the destination is always clean.
+        spark_dist = os.path.join(self.install_lib, "pyspark", "spark-distribution")
+        rmtree(spark_dist, ignore_errors=True)
+
+        if ("PYSPARK_HADOOP_VERSION" in os.environ) or ("PYSPARK_HIVE_VERSION" in os.environ):
+            # Note that PYSPARK_VERSION environment is just a testing purpose.
+            # PYSPARK_HIVE_VERSION environment variable is also internal for now in case
+            # we support another version of Hive in the future.
+            spark_version, hadoop_version, hive_version = install_module.checked_versions(
+                os.environ.get("PYSPARK_VERSION", VERSION).lower(),
+                os.environ.get("PYSPARK_HADOOP_VERSION", install_module.DEFAULT_HADOOP).lower(),
+                os.environ.get("PYSPARK_HIVE_VERSION", install_module.DEFAULT_HIVE).lower())
+
+            if ("PYSPARK_VERSION" not in os.environ and
+                ((install_module.DEFAULT_HADOOP, install_module.DEFAULT_HIVE) ==
+                    (hadoop_version, hive_version))):
+                # Do not download and install if they are same as default.
+                return
+
+            install_module.install_spark(
+                dest=spark_dist,
+                spark_version=spark_version,
+                hadoop_version=hadoop_version,
+                hive_version=hive_version)
+
 
 try:
     # We copy the shell script to be under pyspark/python/pyspark so that the launcher scripts
@@ -167,6 +206,7 @@ def _supports_symlinks():
         author_email='dev@spark.apache.org',
         url='https://github.com/apache/spark/tree/master/python',
         packages=['pyspark',
+                  'pyspark.cloudpickle',
                   'pyspark.mllib',
                   'pyspark.mllib.linalg',
                   'pyspark.mllib.stat',
@@ -180,6 +220,13 @@ def _supports_symlinks():
                   'pyspark.bin',
                   'pyspark.sbin',
                   'pyspark.jars',
+                  'pyspark.pandas',
+                  'pyspark.pandas.indexes',
+                  'pyspark.pandas.missing',
+                  'pyspark.pandas.plot',
+                  'pyspark.pandas.spark',
+                  'pyspark.pandas.typedef',
+                  'pyspark.pandas.usage_logging',
                   'pyspark.python.pyspark',
                   'pyspark.python.lib',
                   'pyspark.data',
@@ -208,27 +255,36 @@ def _supports_symlinks():
             'pyspark.examples.src.main.python': ['*.py', '*/*.py']},
         scripts=scripts,
         license='http://www.apache.org/licenses/LICENSE-2.0',
-        install_requires=['py4j==0.10.9'],
+        # Don't forget to update python/docs/source/getting_started/install.rst
+        # if you're updating the versions or dependencies.
+        install_requires=['py4j==0.10.9.2'],
         extras_require={
             'ml': ['numpy>=1.7'],
             'mllib': ['numpy>=1.7'],
             'sql': [
                 'pandas>=%s' % _minimum_pandas_version,
                 'pyarrow>=%s' % _minimum_pyarrow_version,
-            ]
+            ],
+            'pandas_on_spark': [
+                'pandas>=%s' % _minimum_pandas_version,
+                'pyarrow>=%s' % _minimum_pyarrow_version,
+                'numpy>=1.14,<1.20.0',
+            ],
         },
+        python_requires='>=3.6',
         classifiers=[
             'Development Status :: 5 - Production/Stable',
             'License :: OSI Approved :: Apache Software License',
-            'Programming Language :: Python :: 2.7',
-            'Programming Language :: Python :: 3',
-            'Programming Language :: Python :: 3.4',
-            'Programming Language :: Python :: 3.5',
             'Programming Language :: Python :: 3.6',
             'Programming Language :: Python :: 3.7',
             'Programming Language :: Python :: 3.8',
+            'Programming Language :: Python :: 3.9',
             'Programming Language :: Python :: Implementation :: CPython',
-            'Programming Language :: Python :: Implementation :: PyPy']
+            'Programming Language :: Python :: Implementation :: PyPy',
+            'Typing :: Typed'],
+        cmdclass={
+            'install': InstallCommand,
+        },
     )
 finally:
     # We only cleanup the symlink farm if we were in Spark, otherwise we are installing rather than
diff --git a/python/test_support/userlibrary.py b/python/test_support/userlibrary.py
index 73fd26e71f10d..90cd30723ddfe 100755
--- a/python/test_support/userlibrary.py
+++ b/python/test_support/userlibrary.py
@@ -16,7 +16,7 @@
 #
 
 """
-Used to test shipping of code depenencies with SparkContext.addPyFile().
+Used to test shipping of code dependencies with SparkContext.addPyFile().
 """
 
 
diff --git a/repl/pom.xml b/repl/pom.xml
index a1079e7a6fe6a..a982af21d86f9 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/repl/src/main/scala/org/apache/spark/repl/Main.scala b/repl/src/main/scala-2.12/org/apache/spark/repl/Main.scala
similarity index 100%
rename from repl/src/main/scala/org/apache/spark/repl/Main.scala
rename to repl/src/main/scala-2.12/org/apache/spark/repl/Main.scala
diff --git a/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala b/repl/src/main/scala-2.12/org/apache/spark/repl/SparkILoop.scala
similarity index 100%
rename from repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala
rename to repl/src/main/scala-2.12/org/apache/spark/repl/SparkILoop.scala
diff --git a/repl/src/main/scala-2.13/org/apache/spark/repl/Main.scala b/repl/src/main/scala-2.13/org/apache/spark/repl/Main.scala
new file mode 100644
index 0000000000000..95115934ed1d6
--- /dev/null
+++ b/repl/src/main/scala-2.13/org/apache/spark/repl/Main.scala
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.repl
+
+import java.io.File
+import java.net.URI
+import java.util.Locale
+
+import scala.tools.nsc.GenericRunnerSettings
+
+import org.apache.spark._
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.internal.StaticSQLConf.CATALOG_IMPLEMENTATION
+import org.apache.spark.util.Utils
+
+object Main extends Logging {
+
+  initializeLogIfNecessary(true)
+  Signaling.cancelOnInterrupt()
+
+  val conf = new SparkConf()
+  val rootDir =
+    conf.getOption("spark.repl.classdir").getOrElse(Utils.getLocalDir(conf))
+  val outputDir = Utils.createTempDir(root = rootDir, namePrefix = "repl")
+
+  var sparkContext: SparkContext = _
+  var sparkSession: SparkSession = _
+  // this is a public var because tests reset it.
+  var interp: SparkILoop = _
+
+  private var hasErrors = false
+  private var isShellSession = false
+
+  private def scalaOptionError(msg: String): Unit = {
+    hasErrors = true
+    // scalastyle:off println
+    Console.err.println(msg)
+    // scalastyle:on println
+  }
+
+  def main(args: Array[String]): Unit = {
+    isShellSession = true
+    doMain(args, new SparkILoop)
+  }
+
+  // Visible for testing
+  private[repl] def doMain(args: Array[String], _interp: SparkILoop): Unit = {
+    interp = _interp
+    val jars = Utils
+      .getLocalUserJarsForShell(conf)
+      // Remove file:///, file:// or file:/ scheme if exists for each jar
+      .map { x =>
+        if (x.startsWith("file:")) new File(new URI(x)).getPath else x
+      }
+      .mkString(File.pathSeparator)
+    val interpArguments = List(
+      "-Yrepl-class-based",
+      "-Yrepl-outdir",
+      s"${outputDir.getAbsolutePath}",
+      "-classpath",
+      jars
+    ) ++ args.toList
+
+    val settings = new GenericRunnerSettings(scalaOptionError)
+    settings.processArguments(interpArguments, true)
+
+    if (!hasErrors) {
+      interp.run(settings) // Repl starts and goes in loop of R.E.P.L
+      Option(sparkContext).foreach(_.stop)
+    }
+  }
+
+  def createSparkSession(): SparkSession = {
+    try {
+      val execUri = System.getenv("SPARK_EXECUTOR_URI")
+      conf.setIfMissing("spark.app.name", "Spark shell")
+      // SparkContext will detect this configuration and register it with the RpcEnv's
+      // file server, setting spark.repl.class.uri to the actual URI for executors to
+      // use. This is sort of ugly but since executors are started as part of SparkContext
+      // initialization in certain cases, there's an initialization order issue that prevents
+      // this from being set after SparkContext is instantiated.
+      conf.set("spark.repl.class.outputDir", outputDir.getAbsolutePath())
+      if (execUri != null) {
+        conf.set("spark.executor.uri", execUri)
+      }
+      if (System.getenv("SPARK_HOME") != null) {
+        conf.setSparkHome(System.getenv("SPARK_HOME"))
+      }
+
+      val builder = SparkSession.builder.config(conf)
+      if (conf
+            .get(CATALOG_IMPLEMENTATION.key, "hive")
+            .toLowerCase(Locale.ROOT) == "hive") {
+        if (SparkSession.hiveClassesArePresent) {
+          // In the case that the property is not set at all, builder's config
+          // does not have this value set to 'hive' yet. The original default
+          // behavior is that when there are hive classes, we use hive catalog.
+          sparkSession = builder.enableHiveSupport().getOrCreate()
+          logInfo("Created Spark session with Hive support")
+        } else {
+          // Need to change it back to 'in-memory' if no hive classes are found
+          // in the case that the property is set to hive in spark-defaults.conf
+          builder.config(CATALOG_IMPLEMENTATION.key, "in-memory")
+          sparkSession = builder.getOrCreate()
+          logInfo("Created Spark session")
+        }
+      } else {
+        // In the case that the property is set but not to 'hive', the internal
+        // default is 'in-memory'. So the sparkSession will use in-memory catalog.
+        sparkSession = builder.getOrCreate()
+        logInfo("Created Spark session")
+      }
+      sparkContext = sparkSession.sparkContext
+      sparkSession
+    } catch {
+      case e: Exception if isShellSession =>
+        logError("Failed to initialize Spark session.", e)
+        sys.exit(1)
+    }
+  }
+
+}
diff --git a/repl/src/main/scala-2.13/org/apache/spark/repl/SparkILoop.scala b/repl/src/main/scala-2.13/org/apache/spark/repl/SparkILoop.scala
new file mode 100644
index 0000000000000..861cf5a740ce1
--- /dev/null
+++ b/repl/src/main/scala-2.13/org/apache/spark/repl/SparkILoop.scala
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.repl
+
+import java.io.{BufferedReader, PrintWriter}
+
+// scalastyle:off println
+import scala.Predef.{println => _, _}
+import scala.tools.nsc.GenericRunnerSettings
+import scala.tools.nsc.Settings
+import scala.tools.nsc.interpreter.shell.{ILoop, ShellConfig}
+import scala.tools.nsc.util.stringFromStream
+import scala.util.Properties.{javaVersion, javaVmName, versionString}
+// scalastyle:on println
+
+/**
+ *  A Spark-specific interactive shell.
+ */
+class SparkILoop(in0: BufferedReader, out: PrintWriter)
+  extends ILoop(ShellConfig(new GenericRunnerSettings(_ => ())), in0, out) {
+  def this() = this(null, new PrintWriter(Console.out, true))
+
+  val initializationCommands: Seq[String] = Seq(
+    """
+    @transient val spark = if (org.apache.spark.repl.Main.sparkSession != null) {
+        org.apache.spark.repl.Main.sparkSession
+      } else {
+        org.apache.spark.repl.Main.createSparkSession()
+      }
+    @transient val sc = {
+      val _sc = spark.sparkContext
+      if (_sc.getConf.getBoolean("spark.ui.reverseProxy", false)) {
+        val proxyUrl = _sc.getConf.get("spark.ui.reverseProxyUrl", null)
+        if (proxyUrl != null) {
+          println(
+            s"Spark Context Web UI is available at ${proxyUrl}/proxy/${_sc.applicationId}")
+        } else {
+          println(s"Spark Context Web UI is available at Spark Master Public URL")
+        }
+      } else {
+        _sc.uiWebUrl.foreach {
+          webUrl => println(s"Spark context Web UI available at ${webUrl}")
+        }
+      }
+      println("Spark context available as 'sc' " +
+        s"(master = ${_sc.master}, app id = ${_sc.applicationId}).")
+      println("Spark session available as 'spark'.")
+      _sc
+    }
+    """,
+    "import org.apache.spark.SparkContext._",
+    "import spark.implicits._",
+    "import spark.sql",
+    "import org.apache.spark.sql.functions._"
+  )
+
+  override protected def internalReplAutorunCode(): Seq[String] =
+    initializationCommands
+
+  def initializeSpark(): Unit = {
+    if (!intp.reporter.hasErrors) {
+      // `savingReplayStack` removes the commands from session history.
+      savingReplayStack {
+        initializationCommands.foreach(intp quietRun _)
+      }
+    } else {
+      throw new RuntimeException(
+        s"Scala $versionString interpreter encountered " +
+          "errors during initialization"
+      )
+    }
+  }
+
+  /** Print a welcome message */
+  override def printWelcome(): Unit = {
+    import org.apache.spark.SPARK_VERSION
+    echo("""Welcome to
+      ____              __
+     / __/__  ___ _____/ /__
+    _\ \/ _ \/ _ `/ __/  '_/
+   /___/ .__/\_,_/_/ /_/\_\   version %s
+      /_/
+         """.format(SPARK_VERSION))
+    val welcomeMsg = "Using Scala %s (%s, Java %s)".format(
+      versionString,
+      javaVmName,
+      javaVersion
+    )
+    echo(welcomeMsg)
+    echo("Type in expressions to have them evaluated.")
+    echo("Type :help for more information.")
+  }
+
+  /** Available commands */
+  override def commands: List[LoopCommand] = standardCommands
+
+  override def resetCommand(line: String): Unit = {
+    super.resetCommand(line)
+    initializeSpark()
+    echo(
+      "Note that after :reset, state of SparkSession and SparkContext is unchanged."
+    )
+  }
+
+  override def replay(): Unit = {
+    initializeSpark()
+    super.replay()
+  }
+}
+
+object SparkILoop {
+
+  /**
+   * Creates an interpreter loop with default settings and feeds
+   * the given code to it as input.
+   */
+  def run(code: String, sets: Settings = new Settings): String = {
+    import java.io.{BufferedReader, StringReader, OutputStreamWriter}
+
+    stringFromStream { ostream =>
+      Console.withOut(ostream) {
+        val input = new BufferedReader(new StringReader(code))
+        val output = new PrintWriter(new OutputStreamWriter(ostream), true)
+        val repl = new SparkILoop(input, output)
+
+        if (sets.classpath.isDefault) {
+          sets.classpath.value = sys.props("java.class.path")
+        }
+        repl.run(sets)
+      }
+    }
+  }
+  def run(lines: List[String]): String = run(lines.map(_ + "\n").mkString)
+}
diff --git a/repl/src/test/scala-2.12/org/apache/spark/repl/Repl2Suite.scala b/repl/src/test/scala-2.12/org/apache/spark/repl/Repl2Suite.scala
new file mode 100644
index 0000000000000..90af9ec299efc
--- /dev/null
+++ b/repl/src/test/scala-2.12/org/apache/spark/repl/Repl2Suite.scala
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.repl
+
+import java.io._
+
+import scala.tools.nsc.interpreter.SimpleReader
+
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.spark.{SparkContext, SparkFunSuite}
+
+class Repl2Suite extends SparkFunSuite with BeforeAndAfterAll {
+  test("propagation of local properties") {
+    // A mock ILoop that doesn't install the SIGINT handler.
+    class ILoop(out: PrintWriter) extends SparkILoop(None, out) {
+      settings = new scala.tools.nsc.Settings
+      settings.usejavacp.value = true
+      org.apache.spark.repl.Main.interp = this
+      in = SimpleReader()
+    }
+
+    val out = new StringWriter()
+    Main.interp = new ILoop(new PrintWriter(out))
+    Main.sparkContext = new SparkContext("local", "repl-test")
+    Main.interp.createInterpreter()
+
+    Main.sparkContext.setLocalProperty("someKey", "someValue")
+
+    // Make sure the value we set in the caller to interpret is propagated in the thread that
+    // interprets the command.
+    Main.interp.interpret("org.apache.spark.repl.Main.sparkContext.getLocalProperty(\"someKey\")")
+    assert(out.toString.contains("someValue"))
+
+    Main.sparkContext.stop()
+    System.clearProperty("spark.driver.port")
+  }
+}
diff --git a/repl/src/test/scala-2.12/org/apache/spark/repl/SingletonRepl2Suite.scala b/repl/src/test/scala-2.12/org/apache/spark/repl/SingletonRepl2Suite.scala
new file mode 100644
index 0000000000000..a4eff392a2c99
--- /dev/null
+++ b/repl/src/test/scala-2.12/org/apache/spark/repl/SingletonRepl2Suite.scala
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.repl
+
+import java.io._
+
+import org.apache.spark.SparkFunSuite
+
+/**
+ * A special test suite for REPL that all test cases share one REPL instance.
+ */
+class SingletonRepl2Suite extends SparkFunSuite {
+  private val out = new StringWriter()
+  private val in = new PipedOutputStream()
+  private var thread: Thread = _
+
+  private val CONF_EXECUTOR_CLASSPATH = "spark.executor.extraClassPath"
+  private val oldExecutorClasspath = System.getProperty(CONF_EXECUTOR_CLASSPATH)
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+
+    val classpath = System.getProperty("java.class.path")
+    System.setProperty(CONF_EXECUTOR_CLASSPATH, classpath)
+
+    Main.conf.set("spark.master", "local-cluster[2,1,1024]")
+    val interp = new SparkILoop(
+      new BufferedReader(new InputStreamReader(new PipedInputStream(in))),
+      new PrintWriter(out))
+
+    // Forces to create new SparkContext
+    Main.sparkContext = null
+    Main.sparkSession = null
+
+    // Starts a new thread to run the REPL interpreter, so that we won't block.
+    thread = new Thread(() => Main.doMain(Array("-classpath", classpath), interp))
+    thread.setDaemon(true)
+    thread.start()
+
+    waitUntil(() => out.toString.contains("Type :help for more information"))
+  }
+
+  override def afterAll(): Unit = {
+    in.close()
+    thread.join()
+    if (oldExecutorClasspath != null) {
+      System.setProperty(CONF_EXECUTOR_CLASSPATH, oldExecutorClasspath)
+    } else {
+      System.clearProperty(CONF_EXECUTOR_CLASSPATH)
+    }
+    super.afterAll()
+  }
+
+  private def waitUntil(cond: () => Boolean): Unit = {
+    import scala.concurrent.duration._
+    import org.scalatest.concurrent.Eventually._
+
+    eventually(timeout(50.seconds), interval(500.millis)) {
+      assert(cond(), "current output: " + out.toString)
+    }
+  }
+
+  /**
+   * Run the given commands string in a globally shared interpreter instance. Note that the given
+   * commands should not crash the interpreter, to not affect other test cases.
+   */
+  def runInterpreter(input: String): String = {
+    val currentOffset = out.getBuffer.length()
+    // append a special statement to the end of the given code, so that we can know what's
+    // the final output of this code snippet and rely on it to wait until the output is ready.
+    val timestamp = System.currentTimeMillis()
+    in.write((input + s"\nval _result_$timestamp = 1\n").getBytes)
+    in.flush()
+    val stopMessage = s"_result_$timestamp: Int = 1"
+    waitUntil(() => out.getBuffer.substring(currentOffset).contains(stopMessage))
+    out.getBuffer.substring(currentOffset)
+  }
+
+  def assertContains(message: String, output: String): Unit = {
+    val isContain = output.contains(message)
+    assert(isContain,
+      "Interpreter output did not contain '" + message + "':\n" + output)
+  }
+
+  def assertDoesNotContain(message: String, output: String): Unit = {
+    val isContain = output.contains(message)
+    assert(!isContain,
+      "Interpreter output contained '" + message + "':\n" + output)
+  }
+
+  test("SPARK-31399: should clone+clean line object w/ non-serializable state in ClosureCleaner") {
+    // Test ClosureCleaner when a closure captures the enclosing `this` REPL line object, and that
+    // object contains an unused non-serializable field.
+    // Specifically, the closure in this test case contains a directly nested closure, and the
+    // capture is triggered by the inner closure.
+    // `ns` should be nulled out, but `topLevelValue` should stay intact.
+
+    // Can't use :paste mode because PipedOutputStream/PipedInputStream doesn't work well with the
+    // EOT control character (i.e. Ctrl+D).
+    // Just write things on a single line to emulate :paste mode.
+
+    // NOTE: in order for this test case to trigger the intended scenario, the following three
+    //       variables need to be in the same "input", which will make the REPL pack them into the
+    //       same REPL line object:
+    //         - ns: a non-serializable state, not accessed by the closure;
+    //         - topLevelValue: a serializable state, accessed by the closure;
+    //         - closure: the starting closure, captures the enclosing REPL line object.
+    val output = runInterpreter(
+      """
+        |class NotSerializableClass(val x: Int)
+        |val ns = new NotSerializableClass(42); val topLevelValue = "someValue"; val closure =
+        |(j: Int) => {
+        |  (1 to j).flatMap { x =>
+        |    (1 to x).map { y => y + topLevelValue }
+        |  }
+        |}
+        |val r = sc.parallelize(0 to 2).map(closure).collect
+      """.stripMargin)
+    assertContains("r: Array[scala.collection.immutable.IndexedSeq[String]] = " +
+      "Array(Vector(), Vector(1someValue), Vector(1someValue, 1someValue, 2someValue))", output)
+//    assertContains("r: Array[IndexedSeq[String]] = " +
+//      "Array(Vector(), Vector(1someValue), Vector(1someValue, 1someValue, 2someValue))", output)
+    assertDoesNotContain("Exception", output)
+  }
+
+  test("SPARK-31399: ClosureCleaner should discover indirectly nested closure in inner class") {
+    // Similar to the previous test case, but with indirect closure nesting instead.
+    // There's still nested closures involved, but the inner closure is indirectly nested in the
+    // outer closure, with a level of inner class in between them.
+    // This changes how the inner closure references/captures the outer closure/enclosing `this`
+    // REPL line object, and covers a different code path in inner closure discovery.
+
+    // `ns` should be nulled out, but `topLevelValue` should stay intact.
+
+    val output = runInterpreter(
+      """
+        |class NotSerializableClass(val x: Int)
+        |val ns = new NotSerializableClass(42); val topLevelValue = "someValue"; val closure =
+        |(j: Int) => {
+        |  class InnerFoo {
+        |    val innerClosure = (x: Int) => (1 to x).map { y => y + topLevelValue }
+        |  }
+        |  val innerFoo = new InnerFoo
+        |  (1 to j).flatMap(innerFoo.innerClosure)
+        |}
+        |val r = sc.parallelize(0 to 2).map(closure).collect
+      """.stripMargin)
+    assertContains("r: Array[scala.collection.immutable.IndexedSeq[String]] = " +
+       "Array(Vector(), Vector(1someValue), Vector(1someValue, 1someValue, 2someValue))", output)
+//    assertContains("r: Array[IndexedSeq[String]] = " +
+//       "Array(Vector(), Vector(1someValue), Vector(1someValue, 1someValue, 2someValue))", output)
+    assertDoesNotContain("Array(Vector(), Vector(1null), Vector(1null, 1null, 2null)", output)
+    assertDoesNotContain("Exception", output)
+  }
+
+ }
diff --git a/repl/src/test/scala-2.13/org/apache/spark/repl/Repl2Suite.scala b/repl/src/test/scala-2.13/org/apache/spark/repl/Repl2Suite.scala
new file mode 100644
index 0000000000000..a93284a129e28
--- /dev/null
+++ b/repl/src/test/scala-2.13/org/apache/spark/repl/Repl2Suite.scala
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.repl
+
+import java.io._
+import java.nio.file.Files
+
+import org.apache.log4j.{Level, LogManager, PropertyConfigurator}
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.spark.{SparkContext, SparkFunSuite}
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.internal.StaticSQLConf.CATALOG_IMPLEMENTATION
+
+class Repl2Suite extends SparkFunSuite with BeforeAndAfterAll {
+  test("propagation of local properties") {
+    // A mock ILoop that doesn't install the SIGINT handler.
+    class ILoop(out: PrintWriter) extends SparkILoop(null, out)
+
+    val out = new StringWriter()
+    Main.interp = new ILoop(new PrintWriter(out))
+    Main.sparkContext = new SparkContext("local", "repl-test")
+    val settings = new scala.tools.nsc.Settings
+    settings.usejavacp.value = true
+    Main.interp.createInterpreter(settings)
+
+    Main.sparkContext.setLocalProperty("someKey", "someValue")
+
+    // Make sure the value we set in the caller to interpret is propagated in the thread that
+    // interprets the command.
+    Main.interp.interpret("org.apache.spark.repl.Main.sparkContext.getLocalProperty(\"someKey\")")
+    assert(out.toString.contains("someValue"))
+
+    Main.sparkContext.stop()
+    System.clearProperty("spark.driver.port")
+  }
+}
diff --git a/repl/src/test/scala-2.13/org/apache/spark/repl/SingletonRepl2Suite.scala b/repl/src/test/scala-2.13/org/apache/spark/repl/SingletonRepl2Suite.scala
new file mode 100644
index 0000000000000..b153a0261aaf5
--- /dev/null
+++ b/repl/src/test/scala-2.13/org/apache/spark/repl/SingletonRepl2Suite.scala
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.repl
+
+import java.io._
+
+import org.apache.spark.SparkFunSuite
+
+/**
+ * A special test suite for REPL that all test cases share one REPL instance.
+ */
+class SingletonRepl2Suite extends SparkFunSuite {
+
+  private val out = new StringWriter()
+  private val in = new PipedOutputStream()
+  private var thread: Thread = _
+
+  private val CONF_EXECUTOR_CLASSPATH = "spark.executor.extraClassPath"
+  private val oldExecutorClasspath = System.getProperty(CONF_EXECUTOR_CLASSPATH)
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+
+    val classpath = System.getProperty("java.class.path")
+    System.setProperty(CONF_EXECUTOR_CLASSPATH, classpath)
+
+    Main.conf.set("spark.master", "local-cluster[2,1,1024]")
+    val interp = new SparkILoop(
+      new BufferedReader(new InputStreamReader(new PipedInputStream(in))),
+      new PrintWriter(out))
+
+    // Forces to create new SparkContext
+    Main.sparkContext = null
+    Main.sparkSession = null
+
+    // Starts a new thread to run the REPL interpreter, so that we won't block.
+    thread = new Thread(() => Main.doMain(Array("-classpath", classpath), interp))
+    thread.setDaemon(true)
+    thread.start()
+
+    waitUntil(() => out.toString.contains("Type :help for more information"))
+  }
+
+  override def afterAll(): Unit = {
+    in.close()
+    thread.join()
+    if (oldExecutorClasspath != null) {
+      System.setProperty(CONF_EXECUTOR_CLASSPATH, oldExecutorClasspath)
+    } else {
+      System.clearProperty(CONF_EXECUTOR_CLASSPATH)
+    }
+    super.afterAll()
+  }
+
+  private def waitUntil(cond: () => Boolean): Unit = {
+    import scala.concurrent.duration._
+    import org.scalatest.concurrent.Eventually._
+
+    eventually(timeout(50.seconds), interval(500.millis)) {
+      assert(cond(), "current output: " + out.toString)
+    }
+  }
+
+  /**
+   * Run the given commands string in a globally shared interpreter instance. Note that the given
+   * commands should not crash the interpreter, to not affect other test cases.
+   */
+  def runInterpreter(input: String): String = {
+    val currentOffset = out.getBuffer.length()
+    // append a special statement to the end of the given code, so that we can know what's
+    // the final output of this code snippet and rely on it to wait until the output is ready.
+    val timestamp = System.currentTimeMillis()
+    in.write((input + s"\nval _result_$timestamp = 1\n").getBytes)
+    in.flush()
+    val stopMessage = s"_result_$timestamp: Int = 1"
+    waitUntil(() => out.getBuffer.substring(currentOffset).contains(stopMessage))
+    out.getBuffer.substring(currentOffset)
+  }
+
+  def assertContains(message: String, output: String): Unit = {
+    val isContain = output.contains(message)
+    assert(isContain,
+      "Interpreter output did not contain '" + message + "':\n" + output)
+  }
+
+  def assertDoesNotContain(message: String, output: String): Unit = {
+    val isContain = output.contains(message)
+    assert(!isContain,
+      "Interpreter output contained '" + message + "':\n" + output)
+  }
+
+  test("SPARK-31399: should clone+clean line object w/ non-serializable state in ClosureCleaner") {
+    // Test ClosureCleaner when a closure captures the enclosing `this` REPL line object, and that
+    // object contains an unused non-serializable field.
+    // Specifically, the closure in this test case contains a directly nested closure, and the
+    // capture is triggered by the inner closure.
+    // `ns` should be nulled out, but `topLevelValue` should stay intact.
+
+    // Can't use :paste mode because PipedOutputStream/PipedInputStream doesn't work well with the
+    // EOT control character (i.e. Ctrl+D).
+    // Just write things on a single line to emulate :paste mode.
+
+    // NOTE: in order for this test case to trigger the intended scenario, the following three
+    //       variables need to be in the same "input", which will make the REPL pack them into the
+    //       same REPL line object:
+    //         - ns: a non-serializable state, not accessed by the closure;
+    //         - topLevelValue: a serializable state, accessed by the closure;
+    //         - closure: the starting closure, captures the enclosing REPL line object.
+    val output = runInterpreter(
+      """
+        |class NotSerializableClass(val x: Int)
+        |val ns = new NotSerializableClass(42); val topLevelValue = "someValue"; val closure =
+        |(j: Int) => {
+        |  (1 to j).flatMap { x =>
+        |    (1 to x).map { y => y + topLevelValue }
+        |  }
+        |}
+        |val r = sc.parallelize(0 to 2).map(closure).collect
+      """.stripMargin)
+//    assertContains("r: Array[scala.collection.immutable.IndexedSeq[String]] = " +
+//      "Array(Vector(), Vector(1someValue), Vector(1someValue, 1someValue, 2someValue))", output)
+    assertContains("r: Array[IndexedSeq[String]] = " +
+      "Array(Vector(), Vector(1someValue), Vector(1someValue, 1someValue, 2someValue))", output)
+    assertDoesNotContain("Exception", output)
+  }
+
+  test("SPARK-31399: ClosureCleaner should discover indirectly nested closure in inner class") {
+    // Similar to the previous test case, but with indirect closure nesting instead.
+    // There's still nested closures involved, but the inner closure is indirectly nested in the
+    // outer closure, with a level of inner class in between them.
+    // This changes how the inner closure references/captures the outer closure/enclosing `this`
+    // REPL line object, and covers a different code path in inner closure discovery.
+
+    // `ns` should be nulled out, but `topLevelValue` should stay intact.
+
+    val output = runInterpreter(
+      """
+        |class NotSerializableClass(val x: Int)
+        |val ns = new NotSerializableClass(42); val topLevelValue = "someValue"; val closure =
+        |(j: Int) => {
+        |  class InnerFoo {
+        |    val innerClosure = (x: Int) => (1 to x).map { y => y + topLevelValue }
+        |  }
+        |  val innerFoo = new InnerFoo
+        |  (1 to j).flatMap(innerFoo.innerClosure)
+        |}
+        |val r = sc.parallelize(0 to 2).map(closure).collect
+      """.stripMargin)
+//    assertContains("r: Array[scala.collection.immutable.IndexedSeq[String]] = " +
+//       "Array(Vector(), Vector(1someValue), Vector(1someValue, 1someValue, 2someValue))", output)
+    assertContains("r: Array[IndexedSeq[String]] = " +
+       "Array(Vector(), Vector(1someValue), Vector(1someValue, 1someValue, 2someValue))", output)
+    assertDoesNotContain("Array(Vector(), Vector(1null), Vector(1null, 1null, 2null)", output)
+    assertDoesNotContain("Exception", output)
+  }
+}
diff --git a/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala b/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala
index 5428fa4ee9df7..23ea3fee2505b 100644
--- a/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala
+++ b/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala
@@ -28,7 +28,6 @@ import java.util.Collections
 import javax.tools.{JavaFileObject, SimpleJavaFileObject, ToolProvider}
 
 import scala.io.Source
-import scala.language.implicitConversions
 
 import com.google.common.io.Files
 import org.mockito.ArgumentMatchers.{any, anyString}
@@ -114,10 +113,9 @@ class ExecutorClassLoaderSuite
     val classLoader = new ExecutorClassLoader(
       new SparkConf(), null, url1, parentLoader, true)
 
-    // load 'scala.Option', using ClassforName to do the exact same behavior as
-    // what JavaDeserializationStream does
-
     // scalastyle:off classforname
+    // load 'scala.Option', using Class.forName to do the exact same behavior as
+    // what JavaDeserializationStream does
     val optionClass = Class.forName("scala.Option", false, classLoader)
     // scalastyle:on classforname
 
@@ -233,6 +231,7 @@ class ExecutorClassLoaderSuite
       .setMaster("local")
       .setAppName("executor-class-loader-test")
       .set("spark.network.timeout", "11s")
+      .set("spark.network.timeoutInterval", "11s")
       .set("spark.repl.class.outputDir", tempDir1.getAbsolutePath)
     val sc = new SparkContext(conf)
     try {
diff --git a/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala b/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala
index 1e92b36c336d8..6566d29d16e91 100644
--- a/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala
+++ b/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala
@@ -20,12 +20,10 @@ package org.apache.spark.repl
 import java.io._
 import java.nio.file.Files
 
-import scala.tools.nsc.interpreter.SimpleReader
-
 import org.apache.log4j.{Level, LogManager, PropertyConfigurator}
 import org.scalatest.BeforeAndAfterAll
 
-import org.apache.spark.{SparkContext, SparkFunSuite}
+import org.apache.spark.SparkFunSuite
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.internal.StaticSQLConf.CATALOG_IMPLEMENTATION
@@ -86,31 +84,6 @@ class ReplSuite extends SparkFunSuite with BeforeAndAfterAll {
       "Interpreter output contained '" + message + "':\n" + output)
   }
 
-  test("propagation of local properties") {
-    // A mock ILoop that doesn't install the SIGINT handler.
-    class ILoop(out: PrintWriter) extends SparkILoop(None, out) {
-      settings = new scala.tools.nsc.Settings
-      settings.usejavacp.value = true
-      org.apache.spark.repl.Main.interp = this
-      in = SimpleReader()
-    }
-
-    val out = new StringWriter()
-    Main.interp = new ILoop(new PrintWriter(out))
-    Main.sparkContext = new SparkContext("local", "repl-test")
-    Main.interp.createInterpreter()
-
-    Main.sparkContext.setLocalProperty("someKey", "someValue")
-
-    // Make sure the value we set in the caller to interpret is propagated in the thread that
-    // interprets the command.
-    Main.interp.interpret("org.apache.spark.repl.Main.sparkContext.getLocalProperty(\"someKey\")")
-    assert(out.toString.contains("someValue"))
-
-    Main.sparkContext.stop()
-    System.clearProperty("spark.driver.port")
-  }
-
   test("SPARK-15236: use Hive catalog") {
     // turn on the INFO log so that it is possible the code will dump INFO
     // entry for using "HiveMetastore"
diff --git a/repl/src/test/scala/org/apache/spark/repl/SingletonReplSuite.scala b/repl/src/test/scala/org/apache/spark/repl/SingletonReplSuite.scala
index e11a54bc88070..4795306692f7a 100644
--- a/repl/src/test/scala/org/apache/spark/repl/SingletonReplSuite.scala
+++ b/repl/src/test/scala/org/apache/spark/repl/SingletonReplSuite.scala
@@ -380,67 +380,6 @@ class SingletonReplSuite extends SparkFunSuite {
     assertDoesNotContain("Exception", output)
   }
 
-  test("SPARK-31399: should clone+clean line object w/ non-serializable state in ClosureCleaner") {
-    // Test ClosureCleaner when a closure captures the enclosing `this` REPL line object, and that
-    // object contains an unused non-serializable field.
-    // Specifically, the closure in this test case contains a directly nested closure, and the
-    // capture is triggered by the inner closure.
-    // `ns` should be nulled out, but `topLevelValue` should stay intact.
-
-    // Can't use :paste mode because PipedOutputStream/PipedInputStream doesn't work well with the
-    // EOT control character (i.e. Ctrl+D).
-    // Just write things on a single line to emulate :paste mode.
-
-    // NOTE: in order for this test case to trigger the intended scenario, the following three
-    //       variables need to be in the same "input", which will make the REPL pack them into the
-    //       same REPL line object:
-    //         - ns: a non-serializable state, not accessed by the closure;
-    //         - topLevelValue: a serializable state, accessed by the closure;
-    //         - closure: the starting closure, captures the enclosing REPL line object.
-    val output = runInterpreter(
-      """
-        |class NotSerializableClass(val x: Int)
-        |val ns = new NotSerializableClass(42); val topLevelValue = "someValue"; val closure =
-        |(j: Int) => {
-        |  (1 to j).flatMap { x =>
-        |    (1 to x).map { y => y + topLevelValue }
-        |  }
-        |}
-        |val r = sc.parallelize(0 to 2).map(closure).collect
-      """.stripMargin)
-    assertContains("r: Array[scala.collection.immutable.IndexedSeq[String]] = " +
-      "Array(Vector(), Vector(1someValue), Vector(1someValue, 1someValue, 2someValue))", output)
-    assertDoesNotContain("Exception", output)
-  }
-
-  test("SPARK-31399: ClosureCleaner should discover indirectly nested closure in inner class") {
-    // Similar to the previous test case, but with indirect closure nesting instead.
-    // There's still nested closures involved, but the inner closure is indirectly nested in the
-    // outer closure, with a level of inner class in between them.
-    // This changes how the inner closure references/captures the outer closure/enclosing `this`
-    // REPL line object, and covers a different code path in inner closure discovery.
-
-    // `ns` should be nulled out, but `topLevelValue` should stay intact.
-
-    val output = runInterpreter(
-      """
-        |class NotSerializableClass(val x: Int)
-        |val ns = new NotSerializableClass(42); val topLevelValue = "someValue"; val closure =
-        |(j: Int) => {
-        |  class InnerFoo {
-        |    val innerClosure = (x: Int) => (1 to x).map { y => y + topLevelValue }
-        |  }
-        |  val innerFoo = new InnerFoo
-        |  (1 to j).flatMap(innerFoo.innerClosure)
-        |}
-        |val r = sc.parallelize(0 to 2).map(closure).collect
-      """.stripMargin)
-    assertContains("r: Array[scala.collection.immutable.IndexedSeq[String]] = " +
-       "Array(Vector(), Vector(1someValue), Vector(1someValue, 1someValue, 2someValue))", output)
-    assertDoesNotContain("Array(Vector(), Vector(1null), Vector(1null, 1null, 2null)", output)
-    assertDoesNotContain("Exception", output)
-  }
-
   test("newProductSeqEncoder with REPL defined class") {
     val output = runInterpreter(
       """
diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml
index b527816015c63..be4b970ee805c 100644
--- a/resource-managers/kubernetes/core/pom.xml
+++ b/resource-managers/kubernetes/core/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
@@ -29,8 +29,6 @@
   <name>Spark Project Kubernetes</name>
   <properties>
     <sbt.project.name>kubernetes</sbt.project.name>
-    <!-- Note: Please update the kubernetes client version in kubernetes/integration-tests/pom.xml -->
-    <kubernetes.client.version>4.7.1</kubernetes.client.version>
   </properties>
 
   <dependencies>
@@ -57,7 +55,7 @@
     <dependency>
       <groupId>io.fabric8</groupId>
       <artifactId>kubernetes-client</artifactId>
-      <version>${kubernetes.client.version}</version>
+      <version>${kubernetes-client.version}</version>
       <exclusions>
         <exclusion>
           <groupId>com.fasterxml.jackson.core</groupId>
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
index 274b859fef96d..aa41b7ec2d115 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
@@ -20,10 +20,19 @@ import java.util.concurrent.TimeUnit
 
 import org.apache.spark.deploy.k8s.Constants._
 import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config.{PYSPARK_DRIVER_PYTHON, PYSPARK_PYTHON}
 import org.apache.spark.internal.config.ConfigBuilder
 
 private[spark] object Config extends Logging {
 
+  val DECOMMISSION_SCRIPT =
+    ConfigBuilder("spark.kubernetes.decommission.script")
+      .doc("The location of the script to use for graceful decommissioning")
+      .version("3.2.0")
+      .stringConf
+      .createWithDefault("/opt/decom.sh")
+
+
   val KUBERNETES_CONTEXT =
     ConfigBuilder("spark.kubernetes.context")
       .doc("The desired context from your K8S config file used to configure the K8S " +
@@ -45,6 +54,22 @@ private[spark] object Config extends Logging {
       .stringConf
       .createWithDefault(KUBERNETES_MASTER_INTERNAL_URL)
 
+  val KUBERNETES_DRIVER_SERVICE_DELETE_ON_TERMINATION =
+    ConfigBuilder("spark.kubernetes.driver.service.deleteOnTermination")
+      .doc("If true, driver service will be deleted on Spark application termination. " +
+        "If false, it will be cleaned up when the driver pod is deletion.")
+      .version("3.2.0")
+      .booleanConf
+      .createWithDefault(true)
+
+  val KUBERNETES_DRIVER_OWN_PVC =
+    ConfigBuilder("spark.kubernetes.driver.ownPersistentVolumeClaim")
+      .doc("If true, driver pod becomes the owner of on-demand persistent volume claims " +
+        "instead of the executor pods")
+      .version("3.2.0")
+      .booleanConf
+      .createWithDefault(false)
+
   val KUBERNETES_NAMESPACE =
     ConfigBuilder("spark.kubernetes.namespace")
       .doc("The namespace that will be used for running the driver and executor pods.")
@@ -90,6 +115,14 @@ private[spark] object Config extends Logging {
       .toSequence
       .createWithDefault(Nil)
 
+  val CONFIG_MAP_MAXSIZE =
+    ConfigBuilder("spark.kubernetes.configMap.maxSize")
+      .doc("Max size limit for a config map. This is configurable as per" +
+        " https://etcd.io/docs/v3.4.0/dev-guide/limit/ on k8s server end.")
+      .version("3.1.0")
+      .longConf
+      .createWithDefault(1572864) // 1.5 MiB
+
   val KUBERNETES_AUTH_DRIVER_CONF_PREFIX = "spark.kubernetes.authenticate.driver"
   val KUBERNETES_AUTH_EXECUTOR_CONF_PREFIX = "spark.kubernetes.authenticate.executor"
   val KUBERNETES_AUTH_DRIVER_MOUNTED_CONF_PREFIX = "spark.kubernetes.authenticate.driver.mounted"
@@ -207,10 +240,36 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.executor.podNamePrefix")
       .doc("Prefix to use in front of the executor pod names.")
       .version("2.3.0")
-      .internal()
       .stringConf
       .createOptional
 
+  val KUBERNETES_EXECUTOR_DISABLE_CONFIGMAP =
+    ConfigBuilder("spark.kubernetes.executor.disableConfigMap")
+      .doc("If true, disable ConfigMap creation for executors.")
+      .version("3.2.0")
+      .booleanConf
+      .createWithDefault(false)
+
+  val KUBERNETES_DRIVER_POD_FEATURE_STEPS =
+    ConfigBuilder("spark.kubernetes.driver.pod.featureSteps")
+      .doc("Class names of an extra driver pod feature step implementing " +
+        "KubernetesFeatureConfigStep. This is a developer API. Comma separated. " +
+        "Runs after all of Spark internal feature steps.")
+      .version("3.2.0")
+      .stringConf
+      .toSequence
+      .createWithDefault(Nil)
+
+  val KUBERNETES_EXECUTOR_POD_FEATURE_STEPS =
+    ConfigBuilder("spark.kubernetes.executor.pod.featureSteps")
+      .doc("Class name of an extra executor pod feature step implementing " +
+        "KubernetesFeatureConfigStep. This is a developer API. Comma separated. " +
+        "Runs after all of Spark internal feature steps.")
+      .version("3.2.0")
+      .stringConf
+      .toSequence
+      .createWithDefault(Nil)
+
   val KUBERNETES_ALLOCATION_BATCH_SIZE =
     ConfigBuilder("spark.kubernetes.allocation.batch.size")
       .doc("Number of pods to launch at once in each round of executor allocation.")
@@ -227,6 +286,15 @@ private[spark] object Config extends Logging {
       .checkValue(value => value > 0, "Allocation batch delay must be a positive time value.")
       .createWithDefaultString("1s")
 
+  val KUBERNETES_ALLOCATION_EXECUTOR_TIMEOUT =
+    ConfigBuilder("spark.kubernetes.allocation.executor.timeout")
+      .doc("Time to wait before a newly created executor POD request, which does not reached " +
+        "the POD pending state yet, considered timedout and will be deleted.")
+      .version("3.1.0")
+      .timeConf(TimeUnit.MILLISECONDS)
+      .checkValue(value => value > 0, "Allocation executor timeout must be a positive time value.")
+      .createWithDefaultString("600s")
+
   val KUBERNETES_EXECUTOR_LOST_REASON_CHECK_MAX_ATTEMPTS =
     ConfigBuilder("spark.kubernetes.executor.lostCheck.maxAttempts")
       .doc("Maximum number of attempts allowed for checking the reason of an executor loss " +
@@ -285,12 +353,19 @@ private[spark] object Config extends Logging {
 
   val PYSPARK_MAJOR_PYTHON_VERSION =
     ConfigBuilder("spark.kubernetes.pyspark.pythonVersion")
-      .doc("This sets the major Python version. Either 2 or 3. (Python2 or Python3)")
+      .doc(
+        s"(Deprecated since Spark 3.1, please set '${PYSPARK_PYTHON.key}' and " +
+        s"'${PYSPARK_DRIVER_PYTHON.key}' configurations or $ENV_PYSPARK_PYTHON and " +
+        s"$ENV_PYSPARK_DRIVER_PYTHON environment variables instead.)")
       .version("2.4.0")
       .stringConf
-      .checkValue(pv => List("2", "3").contains(pv),
-        "Ensure that major Python version is either Python2 or Python3")
-      .createWithDefault("3")
+      .checkValue("3" == _,
+        "Python 2 was dropped from Spark 3.1, and only 3 is allowed in " +
+          "this configuration. Note that this configuration was deprecated in Spark 3.1. " +
+          s"Please set '${PYSPARK_PYTHON.key}' and '${PYSPARK_DRIVER_PYTHON.key}' " +
+          s"configurations or $ENV_PYSPARK_PYTHON and $ENV_PYSPARK_DRIVER_PYTHON environment " +
+          "variables instead.")
+      .createOptional
 
   val KUBERNETES_KERBEROS_KRB5_FILE =
     ConfigBuilder("spark.kubernetes.kerberos.krb5.path")
@@ -384,6 +459,13 @@ private[spark] object Config extends Logging {
   val KUBERNETES_AUTH_SUBMISSION_CONF_PREFIX =
     "spark.kubernetes.authenticate.submission"
 
+  val KUBERNETES_TRUST_CERTIFICATES =
+    ConfigBuilder("spark.kubernetes.trust.certificates")
+      .doc("If set to true then client can submit to kubernetes cluster only with token")
+      .version("3.2.0")
+      .booleanConf
+      .createWithDefault(false)
+
   val KUBERNETES_NODE_SELECTOR_PREFIX = "spark.kubernetes.node.selector."
 
   val KUBERNETES_DELETE_EXECUTORS =
@@ -412,11 +494,30 @@ private[spark] object Config extends Logging {
   val KUBERNETES_FILE_UPLOAD_PATH =
     ConfigBuilder("spark.kubernetes.file.upload.path")
       .doc("Hadoop compatible file system path where files from the local file system " +
-        "will be uploded to in cluster mode.")
+        "will be uploaded to in cluster mode.")
       .version("3.0.0")
       .stringConf
       .createOptional
 
+  val KUBERNETES_EXECUTOR_CHECK_ALL_CONTAINERS =
+    ConfigBuilder("spark.kubernetes.executor.checkAllContainers")
+      .doc("If set to true, all containers in the executor pod will be checked when reporting" +
+        "executor status.")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(false)
+
+  val KUBERNETES_EXECUTOR_MISSING_POD_DETECT_DELTA =
+    ConfigBuilder("spark.kubernetes.executor.missingPodDetectDelta")
+      .doc("When a registered executor's POD is missing from the Kubernetes API server's polled " +
+        "list of PODs then this delta time is taken as the accepted time difference between the " +
+        "registration time and the time of the polling. After this time the POD is considered " +
+        "missing from the cluster and the executor will be removed.")
+      .version("3.1.1")
+      .timeConf(TimeUnit.MILLISECONDS)
+      .checkValue(delay => delay > 0, "delay must be a positive time value")
+      .createWithDefaultString("30s")
+
   val KUBERNETES_DRIVER_LABEL_PREFIX = "spark.kubernetes.driver.label."
   val KUBERNETES_DRIVER_ANNOTATION_PREFIX = "spark.kubernetes.driver.annotation."
   val KUBERNETES_DRIVER_SERVICE_ANNOTATION_PREFIX = "spark.kubernetes.driver.service.annotation."
@@ -439,6 +540,7 @@ private[spark] object Config extends Logging {
   val KUBERNETES_VOLUMES_MOUNT_READONLY_KEY = "mount.readOnly"
   val KUBERNETES_VOLUMES_OPTIONS_PATH_KEY = "options.path"
   val KUBERNETES_VOLUMES_OPTIONS_CLAIM_NAME_KEY = "options.claimName"
+  val KUBERNETES_VOLUMES_OPTIONS_CLAIM_STORAGE_CLASS_KEY = "options.storageClass"
   val KUBERNETES_VOLUMES_OPTIONS_MEDIUM_KEY = "options.medium"
   val KUBERNETES_VOLUMES_OPTIONS_SIZE_LIMIT_KEY = "options.sizeLimit"
   val KUBERNETES_VOLUMES_OPTIONS_SERVER_KEY = "options.server"
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
index 759c205b52d9b..543ca12594763 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
@@ -21,6 +21,7 @@ private[spark] object Constants {
   // Labels
   val SPARK_APP_ID_LABEL = "spark-app-selector"
   val SPARK_EXECUTOR_ID_LABEL = "spark-exec-id"
+  val SPARK_RESOURCE_PROFILE_ID_LABEL = "spark-exec-resourceprofile-id"
   val SPARK_ROLE_LABEL = "spark-role"
   val SPARK_POD_DRIVER_ROLE = "driver"
   val SPARK_POD_EXECUTOR_ROLE = "executor"
@@ -54,6 +55,7 @@ private[spark] object Constants {
   val ENV_DRIVER_URL = "SPARK_DRIVER_URL"
   val ENV_EXECUTOR_CORES = "SPARK_EXECUTOR_CORES"
   val ENV_EXECUTOR_MEMORY = "SPARK_EXECUTOR_MEMORY"
+  val ENV_EXECUTOR_DIRS = "SPARK_EXECUTOR_DIRS"
   val ENV_APPLICATION_ID = "SPARK_APPLICATION_ID"
   val ENV_EXECUTOR_ID = "SPARK_EXECUTOR_ID"
   val ENV_EXECUTOR_POD_IP = "SPARK_EXECUTOR_POD_IP"
@@ -62,28 +64,30 @@ private[spark] object Constants {
   val ENV_DRIVER_BIND_ADDRESS = "SPARK_DRIVER_BIND_ADDRESS"
   val ENV_SPARK_CONF_DIR = "SPARK_CONF_DIR"
   val ENV_SPARK_USER = "SPARK_USER"
+  val ENV_RESOURCE_PROFILE_ID = "SPARK_RESOURCE_PROFILE_ID"
   // Spark app configs for containers
-  val SPARK_CONF_VOLUME = "spark-conf-volume"
+  val SPARK_CONF_VOLUME_DRIVER = "spark-conf-volume-driver"
+  val SPARK_CONF_VOLUME_EXEC = "spark-conf-volume-exec"
   val SPARK_CONF_DIR_INTERNAL = "/opt/spark/conf"
   val SPARK_CONF_FILE_NAME = "spark.properties"
   val SPARK_CONF_PATH = s"$SPARK_CONF_DIR_INTERNAL/$SPARK_CONF_FILE_NAME"
   val ENV_HADOOP_TOKEN_FILE_LOCATION = "HADOOP_TOKEN_FILE_LOCATION"
 
   // BINDINGS
-  val ENV_PYSPARK_MAJOR_PYTHON_VERSION = "PYSPARK_MAJOR_PYTHON_VERSION"
+  val ENV_PYSPARK_PYTHON = "PYSPARK_PYTHON"
+  val ENV_PYSPARK_DRIVER_PYTHON = "PYSPARK_DRIVER_PYTHON"
 
   // Pod spec templates
   val EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME = "pod-spec-template.yml"
   val EXECUTOR_POD_SPEC_TEMPLATE_MOUNTPATH = "/opt/spark/pod-template"
   val POD_TEMPLATE_VOLUME = "pod-template-volume"
-  val POD_TEMPLATE_CONFIGMAP = "podspec-configmap"
+  val POD_TEMPLATE_CONFIGMAP = "driver-podspec-conf-map"
   val POD_TEMPLATE_KEY = "podspec-configmap-key"
 
   // Miscellaneous
   val KUBERNETES_MASTER_INTERNAL_URL = "https://kubernetes.default.svc"
   val DEFAULT_DRIVER_CONTAINER_NAME = "spark-kubernetes-driver"
   val DEFAULT_EXECUTOR_CONTAINER_NAME = "spark-kubernetes-executor"
-  val MEMORY_OVERHEAD_MIN_MIB = 384L
   val NON_JVM_MEMORY_OVERHEAD_FACTOR = 0.4d
 
   // Hadoop Configuration
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala
index 6bd7fa81c0e37..087eeee277d1e 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala
@@ -26,6 +26,7 @@ import org.apache.spark.deploy.k8s.Constants._
 import org.apache.spark.deploy.k8s.submit._
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.ConfigEntry
+import org.apache.spark.resource.ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID
 import org.apache.spark.util.Utils
 
 /**
@@ -56,7 +57,7 @@ private[spark] abstract class KubernetesConf(val sparkConf: SparkConf) {
   }
 
   def workerDecommissioning: Boolean =
-    sparkConf.get(org.apache.spark.internal.config.Worker.WORKER_DECOMMISSION_ENABLED)
+    sparkConf.get(org.apache.spark.internal.config.DECOMMISSION_ENABLED)
 
   def nodeSelector: Map[String, String] =
     KubernetesUtils.parsePrefixedKeyValuePairs(sparkConf, KUBERNETES_NODE_SELECTOR_PREFIX)
@@ -132,7 +133,8 @@ private[spark] class KubernetesExecutorConf(
     sparkConf: SparkConf,
     val appId: String,
     val executorId: String,
-    val driverPod: Option[Pod])
+    val driverPod: Option[Pod],
+    val resourceProfileId: Int = DEFAULT_RESOURCE_PROFILE_ID)
   extends KubernetesConf(sparkConf) with Logging {
 
   override val resourceNamePrefix: String = {
@@ -144,7 +146,8 @@ private[spark] class KubernetesExecutorConf(
     val presetLabels = Map(
       SPARK_EXECUTOR_ID_LABEL -> executorId,
       SPARK_APP_ID_LABEL -> appId,
-      SPARK_ROLE_LABEL -> SPARK_POD_EXECUTOR_ROLE)
+      SPARK_ROLE_LABEL -> SPARK_POD_EXECUTOR_ROLE,
+      SPARK_RESOURCE_PROFILE_ID_LABEL -> resourceProfileId.toString)
 
     val executorCustomLabels = KubernetesUtils.parsePrefixedKeyValuePairs(
       sparkConf, KUBERNETES_EXECUTOR_LABEL_PREFIX)
@@ -217,8 +220,9 @@ private[spark] object KubernetesConf {
       sparkConf: SparkConf,
       executorId: String,
       appId: String,
-      driverPod: Option[Pod]): KubernetesExecutorConf = {
-    new KubernetesExecutorConf(sparkConf.clone(), appId, executorId, driverPod)
+      driverPod: Option[Pod],
+      resourceProfileId: Int = DEFAULT_RESOURCE_PROFILE_ID): KubernetesExecutorConf = {
+    new KubernetesExecutorConf(sparkConf.clone(), appId, executorId, driverPod, resourceProfileId)
   }
 
   def getResourceNamePrefix(appName: String): String = {
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesExecutorSpec.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesExecutorSpec.scala
new file mode 100644
index 0000000000000..4db9211c1e42a
--- /dev/null
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesExecutorSpec.scala
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.deploy.k8s
+
+import io.fabric8.kubernetes.api.model.HasMetadata
+
+private[spark] case class KubernetesExecutorSpec(
+    pod: SparkPod,
+    executorKubernetesResources: Seq[HasMetadata])
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
index c49f4a15de974..0c8d9646a2b4e 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
@@ -19,25 +19,34 @@ package org.apache.spark.deploy.k8s
 import java.io.{File, IOException}
 import java.net.URI
 import java.security.SecureRandom
-import java.util.UUID
+import java.util.{Collections, UUID}
 
 import scala.collection.JavaConverters._
 
-import io.fabric8.kubernetes.api.model.{Container, ContainerBuilder, ContainerStateRunning, ContainerStateTerminated, ContainerStateWaiting, ContainerStatus, Pod, PodBuilder, Quantity}
+import io.fabric8.kubernetes.api.model.{Container, ContainerBuilder, ContainerStateRunning, ContainerStateTerminated, ContainerStateWaiting, ContainerStatus, HasMetadata, OwnerReferenceBuilder, Pod, PodBuilder, Quantity}
 import io.fabric8.kubernetes.client.KubernetesClient
 import org.apache.commons.codec.binary.Hex
 import org.apache.hadoop.fs.{FileSystem, Path}
 
 import org.apache.spark.{SparkConf, SparkException}
+import org.apache.spark.annotation.{DeveloperApi, Since, Unstable}
 import org.apache.spark.deploy.SparkHadoopUtil
 import org.apache.spark.deploy.k8s.Config.KUBERNETES_FILE_UPLOAD_PATH
 import org.apache.spark.internal.Logging
 import org.apache.spark.launcher.SparkLauncher
 import org.apache.spark.resource.ResourceUtils
 import org.apache.spark.util.{Clock, SystemClock, Utils}
+import org.apache.spark.util.DependencyUtils.downloadFile
 import org.apache.spark.util.Utils.getHadoopFileSystem
 
-private[spark] object KubernetesUtils extends Logging {
+/**
+ * :: DeveloperApi ::
+ *
+ * A utility class used for K8s operations internally and for implementing ExternalClusterManagers.
+ */
+@Unstable
+@DeveloperApi
+object KubernetesUtils extends Logging {
 
   private val systemClock = new SystemClock()
   private lazy val RNG = new SecureRandom()
@@ -50,12 +59,14 @@ private[spark] object KubernetesUtils extends Logging {
    * @param prefix the given property name prefix
    * @return a Map storing the configuration property keys and values
    */
+  @Since("2.3.0")
   def parsePrefixedKeyValuePairs(
       sparkConf: SparkConf,
       prefix: String): Map[String, String] = {
     sparkConf.getAllWithPrefix(prefix).toMap
   }
 
+  @Since("3.0.0")
   def requireBothOrNeitherDefined(
       opt1: Option[_],
       opt2: Option[_],
@@ -65,6 +76,7 @@ private[spark] object KubernetesUtils extends Logging {
     requireSecondIfFirstIsDefined(opt2, opt1, errMessageWhenFirstIsMissing)
   }
 
+  @Since("3.0.0")
   def requireSecondIfFirstIsDefined(
       opt1: Option[_],
       opt2: Option[_],
@@ -74,16 +86,22 @@ private[spark] object KubernetesUtils extends Logging {
     }
   }
 
+  @Since("2.3.0")
   def requireNandDefined(opt1: Option[_], opt2: Option[_], errMessage: String): Unit = {
     opt1.foreach { _ => require(opt2.isEmpty, errMessage) }
     opt2.foreach { _ => require(opt1.isEmpty, errMessage) }
   }
 
+  @Since("3.2.0")
   def loadPodFromTemplate(
       kubernetesClient: KubernetesClient,
-      templateFile: File,
-      containerName: Option[String]): SparkPod = {
+      templateFileName: String,
+      containerName: Option[String],
+      conf: SparkConf): SparkPod = {
     try {
+      val hadoopConf = SparkHadoopUtil.get.newConfiguration(conf)
+      val localFile = downloadFile(templateFileName, Utils.createTempDir(), conf, hadoopConf)
+      val templateFile = new File(new java.net.URI(localFile).getPath)
       val pod = kubernetesClient.pods().load(templateFile).get()
       selectSparkContainer(pod, containerName)
     } catch {
@@ -94,6 +112,7 @@ private[spark] object KubernetesUtils extends Logging {
     }
   }
 
+  @Since("3.0.0")
   def selectSparkContainer(pod: Pod, containerName: Option[String]): SparkPod = {
     def selectNamedContainer(
       containers: List[Container], name: String): Option[(Container, List[Container])] =
@@ -120,8 +139,10 @@ private[spark] object KubernetesUtils extends Logging {
       }.getOrElse(SparkPod(pod, new ContainerBuilder().build()))
   }
 
+  @Since("2.4.0")
   def parseMasterUrl(url: String): String = url.substring("k8s://".length)
 
+  @Since("3.0.0")
   def formatPairsBundle(pairs: Seq[(String, String)], indent: Int = 1) : String = {
     // Use more loggable format if value is null or empty
     val indentStr = "\t" * indent
@@ -136,6 +157,7 @@ private[spark] object KubernetesUtils extends Logging {
    * @param pod Pod
    * @return Human readable pod state
    */
+  @Since("3.0.0")
   def formatPodState(pod: Pod): String = {
     val details = Seq[(String, String)](
       // pod metadata
@@ -159,6 +181,7 @@ private[spark] object KubernetesUtils extends Logging {
     formatPairsBundle(details)
   }
 
+  @Since("3.0.0")
   def containersDescription(p: Pod, indent: Int = 1): String = {
     p.getStatus.getContainerStatuses.asScala.map { status =>
       Seq(
@@ -168,6 +191,7 @@ private[spark] object KubernetesUtils extends Logging {
     }.map(p => formatPairsBundle(p, indent)).mkString("\n\n")
   }
 
+  @Since("3.0.0")
   def containerStatusDescription(containerStatus: ContainerStatus)
     : Seq[(String, String)] = {
     val state = containerStatus.getState
@@ -195,6 +219,7 @@ private[spark] object KubernetesUtils extends Logging {
       }.getOrElse(Seq(("container state", "N/A")))
   }
 
+  @Since("3.0.0")
   def formatTime(time: String): String = {
     if (time != null) time else "N/A"
   }
@@ -207,6 +232,7 @@ private[spark] object KubernetesUtils extends Logging {
    * This avoids using a UUID for uniqueness (too long), and relying solely on the current time
    * (not unique enough).
    */
+  @Since("3.0.0")
   def uniqueID(clock: Clock = systemClock): String = {
     val random = new Array[Byte](3)
     synchronized {
@@ -223,6 +249,7 @@ private[spark] object KubernetesUtils extends Logging {
    * It assumes we can use the Kubernetes device plugin format: vendor-domain/resource.
    * It returns a set with a tuple of vendor-domain/resource and Quantity for each resource.
    */
+  @Since("3.0.0")
   def buildResourcesQuantities(
       componentName: String,
       sparkConf: SparkConf): Map[String, Quantity] = {
@@ -242,6 +269,7 @@ private[spark] object KubernetesUtils extends Logging {
   /**
    * Upload files and modify their uris
    */
+  @Since("3.0.0")
   def uploadAndTransformFileUris(fileUris: Iterable[String], conf: Option[SparkConf] = None)
     : Iterable[String] = {
     fileUris.map { uri =>
@@ -256,19 +284,29 @@ private[spark] object KubernetesUtils extends Logging {
     }
   }
 
+  @Since("3.0.0")
   def isLocalAndResolvable(resource: String): Boolean = {
     resource != SparkLauncher.NO_RESOURCE &&
       isLocalDependency(Utils.resolveURI(resource))
   }
 
-  def renameMainAppResource(resource: String, conf: SparkConf): String = {
+  @Since("3.1.1")
+  def renameMainAppResource(
+      resource: String,
+      conf: Option[SparkConf] = None,
+      shouldUploadLocal: Boolean): String = {
     if (isLocalAndResolvable(resource)) {
-      SparkLauncher.NO_RESOURCE
+      if (shouldUploadLocal) {
+        uploadFileUri(resource, conf)
+      } else {
+        SparkLauncher.NO_RESOURCE
+      }
     } else {
       resource
-   }
+    }
   }
 
+  @Since("3.0.0")
   def uploadFileUri(uri: String, conf: Option[SparkConf] = None): String = {
     conf match {
       case Some(sConf) =>
@@ -313,6 +351,7 @@ private[spark] object KubernetesUtils extends Logging {
     }
   }
 
+  @Since("3.0.0")
   def buildPodWithServiceAccount(serviceAccount: Option[String], pod: SparkPod): Option[Pod] = {
     serviceAccount.map { account =>
       new PodBuilder(pod.pod)
@@ -323,4 +362,23 @@ private[spark] object KubernetesUtils extends Logging {
         .build()
     }
   }
+
+  // Add a OwnerReference to the given resources making the pod an owner of them so when
+  // the pod is deleted, the resources are garbage collected.
+  @Since("3.1.1")
+  def addOwnerReference(pod: Pod, resources: Seq[HasMetadata]): Unit = {
+    if (pod != null) {
+      val reference = new OwnerReferenceBuilder()
+        .withName(pod.getMetadata.getName)
+        .withApiVersion(pod.getApiVersion)
+        .withUid(pod.getMetadata.getUid)
+        .withKind(pod.getKind)
+        .withController(true)
+        .build()
+      resources.foreach { resource =>
+        val originalMetadata = resource.getMetadata
+        originalMetadata.setOwnerReferences(Collections.singletonList(reference))
+      }
+    }
+  }
 }
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesVolumeSpec.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesVolumeSpec.scala
index f9faa435c81c5..3f7355de18911 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesVolumeSpec.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesVolumeSpec.scala
@@ -21,7 +21,10 @@ private[spark] sealed trait KubernetesVolumeSpecificConf
 private[spark] case class KubernetesHostPathVolumeConf(hostPath: String)
   extends KubernetesVolumeSpecificConf
 
-private[spark] case class KubernetesPVCVolumeConf(claimName: String)
+private[spark] case class KubernetesPVCVolumeConf(
+    claimName: String,
+    storageClass: Option[String] = None,
+    size: Option[String] = None)
   extends KubernetesVolumeSpecificConf
 
 private[spark] case class KubernetesEmptyDirVolumeConf(
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesVolumeUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesVolumeUtils.scala
index 7f821d37ac816..b2eacca042794 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesVolumeUtils.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesVolumeUtils.scala
@@ -67,11 +67,19 @@ private[spark] object KubernetesVolumeUtils {
     volumeType match {
       case KUBERNETES_VOLUMES_HOSTPATH_TYPE =>
         val pathKey = s"$volumeType.$volumeName.$KUBERNETES_VOLUMES_OPTIONS_PATH_KEY"
+        verifyOptionKey(options, pathKey, KUBERNETES_VOLUMES_HOSTPATH_TYPE)
         KubernetesHostPathVolumeConf(options(pathKey))
 
       case KUBERNETES_VOLUMES_PVC_TYPE =>
         val claimNameKey = s"$volumeType.$volumeName.$KUBERNETES_VOLUMES_OPTIONS_CLAIM_NAME_KEY"
-        KubernetesPVCVolumeConf(options(claimNameKey))
+        val storageClassKey =
+          s"$volumeType.$volumeName.$KUBERNETES_VOLUMES_OPTIONS_CLAIM_STORAGE_CLASS_KEY"
+        val sizeLimitKey = s"$volumeType.$volumeName.$KUBERNETES_VOLUMES_OPTIONS_SIZE_LIMIT_KEY"
+        verifyOptionKey(options, claimNameKey, KUBERNETES_VOLUMES_PVC_TYPE)
+        KubernetesPVCVolumeConf(
+          options(claimNameKey),
+          options.get(storageClassKey),
+          options.get(sizeLimitKey))
 
       case KUBERNETES_VOLUMES_EMPTYDIR_TYPE =>
         val mediumKey = s"$volumeType.$volumeName.$KUBERNETES_VOLUMES_OPTIONS_MEDIUM_KEY"
@@ -81,6 +89,8 @@ private[spark] object KubernetesVolumeUtils {
       case KUBERNETES_VOLUMES_NFS_TYPE =>
         val pathKey = s"$volumeType.$volumeName.$KUBERNETES_VOLUMES_OPTIONS_PATH_KEY"
         val serverKey = s"$volumeType.$volumeName.$KUBERNETES_VOLUMES_OPTIONS_SERVER_KEY"
+        verifyOptionKey(options, pathKey, KUBERNETES_VOLUMES_NFS_TYPE)
+        verifyOptionKey(options, serverKey, KUBERNETES_VOLUMES_NFS_TYPE)
         KubernetesNFSVolumeConf(
           options(pathKey),
           options(serverKey))
@@ -89,4 +99,10 @@ private[spark] object KubernetesVolumeUtils {
         throw new IllegalArgumentException(s"Kubernetes Volume type `$volumeType` is not supported")
     }
   }
+
+  private def verifyOptionKey(options: Map[String, String], key: String, msg: String): Unit = {
+    if (!options.isDefinedAt(key)) {
+      throw new NoSuchElementException(key + s" is required for $msg")
+    }
+  }
 }
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala
index 459259f77796c..15b44474445a4 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala
@@ -83,6 +83,7 @@ private[spark] object SparkKubernetesClientFactory extends Logging {
       .withWebsocketPingInterval(0)
       .withRequestTimeout(clientType.requestTimeout(sparkConf))
       .withConnectionTimeout(clientType.connectionTimeout(sparkConf))
+      .withTrustCerts(sparkConf.get(KUBERNETES_TRUST_CERTIFICATES))
       .withOption(oauthTokenValue) {
         (token, configBuilder) => configBuilder.withOauthToken(token)
       }.withOption(oauthTokenFile) {
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkPod.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkPod.scala
index fd1196368a7ff..c2298e7ca77c6 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkPod.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkPod.scala
@@ -18,7 +18,16 @@ package org.apache.spark.deploy.k8s
 
 import io.fabric8.kubernetes.api.model.{Container, ContainerBuilder, Pod, PodBuilder}
 
-private[spark] case class SparkPod(pod: Pod, container: Container) {
+import org.apache.spark.annotation.{DeveloperApi, Unstable}
+
+/**
+ * :: DeveloperApi ::
+ *
+ * Represents a SparkPod consisting of pod and the container within the pod.
+ */
+@Unstable
+@DeveloperApi
+case class SparkPod(pod: Pod, container: Container) {
 
   /**
    * Convenience method to apply a series of chained transformations to a pod.
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
index 63f18129328b0..cec8272beed57 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
@@ -16,6 +16,8 @@
  */
 package org.apache.spark.deploy.k8s.features
 
+import javax.ws.rs.core.UriBuilder
+
 import scala.collection.JavaConverters._
 import scala.collection.mutable
 
@@ -27,6 +29,7 @@ import org.apache.spark.deploy.k8s.Config._
 import org.apache.spark.deploy.k8s.Constants._
 import org.apache.spark.deploy.k8s.submit._
 import org.apache.spark.internal.config._
+import org.apache.spark.resource.ResourceProfile
 import org.apache.spark.ui.SparkUI
 import org.apache.spark.util.Utils
 
@@ -66,7 +69,8 @@ private[spark] class BasicDriverFeatureStep(conf: KubernetesDriverConf)
 
   private val memoryOverheadMiB = conf
     .get(DRIVER_MEMORY_OVERHEAD)
-    .getOrElse(math.max((overheadFactor * driverMemoryMiB).toInt, MEMORY_OVERHEAD_MIN_MIB))
+    .getOrElse(math.max((overheadFactor * driverMemoryMiB).toInt,
+      ResourceProfile.MEMORY_OVERHEAD_MIN_MIB))
   private val driverMemoryWithOverheadMiB = driverMemoryMiB + memoryOverheadMiB
 
   override def configurePod(pod: SparkPod): SparkPod = {
@@ -157,11 +161,25 @@ private[spark] class BasicDriverFeatureStep(conf: KubernetesDriverConf)
       KUBERNETES_DRIVER_SUBMIT_CHECK.key -> "true",
       MEMORY_OVERHEAD_FACTOR.key -> overheadFactor.toString)
     // try upload local, resolvable files to a hadoop compatible file system
-    Seq(JARS, FILES).foreach { key =>
-      val value = conf.get(key).filter(uri => KubernetesUtils.isLocalAndResolvable(uri))
+    Seq(JARS, FILES, ARCHIVES, SUBMIT_PYTHON_FILES).foreach { key =>
+      val uris = conf.get(key).filter(uri => KubernetesUtils.isLocalAndResolvable(uri))
+      val value = {
+        if (key == ARCHIVES) {
+          uris.map(UriBuilder.fromUri(_).fragment(null).build()).map(_.toString)
+        } else {
+          uris
+        }
+      }
       val resolved = KubernetesUtils.uploadAndTransformFileUris(value, Some(conf.sparkConf))
       if (resolved.nonEmpty) {
-        additionalProps.put(key.key, resolved.mkString(","))
+        val resolvedValue = if (key == ARCHIVES) {
+          uris.zip(resolved).map { case (uri, r) =>
+            UriBuilder.fromUri(r).fragment(new java.net.URI(uri).getFragment).build().toString
+          }
+        } else {
+          resolved
+        }
+        additionalProps.put(key.key, resolvedValue.mkString(","))
       }
     }
     additionalProps.toMap
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
index f575241de9540..74158f2665583 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
@@ -24,16 +24,18 @@ import org.apache.spark.{SecurityManager, SparkConf, SparkException}
 import org.apache.spark.deploy.k8s._
 import org.apache.spark.deploy.k8s.Config._
 import org.apache.spark.deploy.k8s.Constants._
+import org.apache.spark.deploy.k8s.submit.KubernetesClientUtils
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config._
-import org.apache.spark.internal.config.Python._
+import org.apache.spark.resource.{ExecutorResourceRequest, ResourceProfile}
 import org.apache.spark.rpc.RpcEndpointAddress
 import org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend
 import org.apache.spark.util.Utils
 
 private[spark] class BasicExecutorFeatureStep(
     kubernetesConf: KubernetesExecutorConf,
-    secMgr: SecurityManager)
+    secMgr: SecurityManager,
+    resourceProfile: ResourceProfile)
   extends KubernetesFeatureConfigStep with Logging {
 
   // Consider moving some of these fields to KubernetesConf or KubernetesExecutorSpecificConf
@@ -50,36 +52,50 @@ private[spark] class BasicExecutorFeatureStep(
     kubernetesConf.get(DRIVER_HOST_ADDRESS),
     kubernetesConf.sparkConf.getInt(DRIVER_PORT.key, DEFAULT_DRIVER_PORT),
     CoarseGrainedSchedulerBackend.ENDPOINT_NAME).toString
-  private val executorMemoryMiB = kubernetesConf.get(EXECUTOR_MEMORY)
-  private val executorMemoryString = kubernetesConf.get(
-    EXECUTOR_MEMORY.key, EXECUTOR_MEMORY.defaultValueString)
-
-  private val memoryOverheadMiB = kubernetesConf
-    .get(EXECUTOR_MEMORY_OVERHEAD)
-    .getOrElse(math.max(
-      (kubernetesConf.get(MEMORY_OVERHEAD_FACTOR) * executorMemoryMiB).toInt,
-      MEMORY_OVERHEAD_MIN_MIB))
-  private val executorMemoryWithOverhead = executorMemoryMiB + memoryOverheadMiB
-  private val executorMemoryTotal =
-    if (kubernetesConf.get(APP_RESOURCE_TYPE) == Some(APP_RESOURCE_TYPE_PYTHON)) {
-      executorMemoryWithOverhead +
-        kubernetesConf.get(PYSPARK_EXECUTOR_MEMORY).map(_.toInt).getOrElse(0)
-    } else {
-      executorMemoryWithOverhead
-    }
 
-  private val executorCores = kubernetesConf.sparkConf.get(EXECUTOR_CORES)
+  private val isDefaultProfile = resourceProfile.id == ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID
+  private val isPythonApp = kubernetesConf.get(APP_RESOURCE_TYPE) == Some(APP_RESOURCE_TYPE_PYTHON)
+  private val disableConfigMap = kubernetesConf.get(KUBERNETES_EXECUTOR_DISABLE_CONFIGMAP)
+
+  val execResources = ResourceProfile.getResourcesForClusterManager(
+    resourceProfile.id,
+    resourceProfile.executorResources,
+    kubernetesConf.get(MEMORY_OVERHEAD_FACTOR),
+    kubernetesConf.sparkConf,
+    isPythonApp,
+    Map.empty)
+
+  private val executorMemoryString = s"${execResources.executorMemoryMiB}m"
+  // we don't include any kubernetes conf specific requests or limits when using custom
+  // ResourceProfiles because we don't have a way of overriding them if needed
   private val executorCoresRequest =
-    if (kubernetesConf.sparkConf.contains(KUBERNETES_EXECUTOR_REQUEST_CORES)) {
+    if (isDefaultProfile && kubernetesConf.sparkConf.contains(KUBERNETES_EXECUTOR_REQUEST_CORES)) {
       kubernetesConf.get(KUBERNETES_EXECUTOR_REQUEST_CORES).get
     } else {
-      executorCores.toString
+      execResources.cores.toString
     }
   private val executorLimitCores = kubernetesConf.get(KUBERNETES_EXECUTOR_LIMIT_CORES)
 
+  private def buildExecutorResourcesQuantities(
+      customResources: Set[ExecutorResourceRequest]): Map[String, Quantity] = {
+    customResources.map { request =>
+      val vendorDomain = if (request.vendor.nonEmpty) {
+        request.vendor
+      } else {
+        throw new SparkException(s"Resource: ${request.resourceName} was requested, " +
+          "but vendor was not specified.")
+      }
+      val quantity = new Quantity(request.amount.toString)
+      (KubernetesConf.buildKubernetesResourceName(vendorDomain, request.resourceName), quantity)
+    }.toMap
+  }
+
   override def configurePod(pod: SparkPod): SparkPod = {
     val name = s"$executorPodNamePrefix-exec-${kubernetesConf.executorId}"
-
+    val configMapName = KubernetesClientUtils.configMapNameExecutor
+    val confFilesMap = KubernetesClientUtils
+      .buildSparkConfDirFilesMap(configMapName, kubernetesConf.sparkConf, Map.empty)
+    val keyToPaths = KubernetesClientUtils.buildKeyToPathObjects(confFilesMap)
     // hostname must be no longer than 63 characters, so take the last 63 characters of the pod
     // name as the hostname.  This preserves uniqueness since the end of name contains
     // executorId
@@ -89,22 +105,21 @@ private[spark] class BasicExecutorFeatureStep(
       // Replace dangerous characters in the remaining string with a safe alternative.
       .replaceAll("[^\\w-]+", "_")
 
-    val executorMemoryQuantity = new Quantity(s"${executorMemoryTotal}Mi")
+    val executorMemoryQuantity = new Quantity(s"${execResources.totalMemMiB}Mi")
     val executorCpuQuantity = new Quantity(executorCoresRequest)
-
     val executorResourceQuantities =
-      KubernetesUtils.buildResourcesQuantities(SPARK_EXECUTOR_PREFIX,
-        kubernetesConf.sparkConf)
+      buildExecutorResourcesQuantities(execResources.customResources.values.toSet)
 
     val executorEnv: Seq[EnvVar] = {
         (Seq(
           (ENV_DRIVER_URL, driverUrl),
-          (ENV_EXECUTOR_CORES, executorCores.toString),
+          (ENV_EXECUTOR_CORES, execResources.cores.toString),
           (ENV_EXECUTOR_MEMORY, executorMemoryString),
           (ENV_APPLICATION_ID, kubernetesConf.appId),
           // This is to set the SPARK_CONF_DIR to be /opt/spark/conf
           (ENV_SPARK_CONF_DIR, SPARK_CONF_DIR_INTERNAL),
-          (ENV_EXECUTOR_ID, kubernetesConf.executorId)
+          (ENV_EXECUTOR_ID, kubernetesConf.executorId),
+          (ENV_RESOURCE_PROFILE_ID, resourceProfile.id.toString)
         ) ++ kubernetesConf.environment).map { case (k, v) =>
           new EnvVarBuilder()
             .withName(k)
@@ -151,6 +166,11 @@ private[spark] class BasicExecutorFeatureStep(
             .build()
         }
       }
+    executorEnv.find(_.getName == ENV_EXECUTOR_DIRS).foreach { e =>
+      e.setValue(e.getValue
+        .replaceAll(ENV_APPLICATION_ID, kubernetesConf.appId)
+        .replaceAll(ENV_EXECUTOR_ID, kubernetesConf.executorId))
+    }
 
     val requiredPorts = Seq(
       (BLOCK_MANAGER_PORT_NAME, blockManagerPort))
@@ -161,6 +181,13 @@ private[spark] class BasicExecutorFeatureStep(
           .build()
       }
 
+    if (!isDefaultProfile) {
+      if (pod.container != null && pod.container.getResources() != null) {
+        logDebug("NOT using the default profile and removing template resources")
+        pod.container.setResources(new ResourceRequirements())
+      }
+    }
+
     val executorContainer = new ContainerBuilder(pod.container)
       .withName(Option(pod.container.getName).getOrElse(DEFAULT_EXECUTOR_CONTAINER_NAME))
       .withImage(executorContainerImage)
@@ -179,14 +206,28 @@ private[spark] class BasicExecutorFeatureStep(
       .withPorts(requiredPorts.asJava)
       .addToArgs("executor")
       .build()
-    val containerWithLimitCores = executorLimitCores.map { limitCores =>
-      val executorCpuLimitQuantity = new Quantity(limitCores)
+    val executorContainerWithConfVolume = if (disableConfigMap) {
+      executorContainer
+    } else {
       new ContainerBuilder(executorContainer)
-        .editResources()
+        .addNewVolumeMount()
+          .withName(SPARK_CONF_VOLUME_EXEC)
+          .withMountPath(SPARK_CONF_DIR_INTERNAL)
+          .endVolumeMount()
+        .build()
+    }
+    val containerWithLimitCores = if (isDefaultProfile) {
+      executorLimitCores.map { limitCores =>
+        val executorCpuLimitQuantity = new Quantity(limitCores)
+        new ContainerBuilder(executorContainerWithConfVolume)
+          .editResources()
           .addToLimits("cpu", executorCpuLimitQuantity)
           .endResources()
-        .build()
-    }.getOrElse(executorContainer)
+          .build()
+      }.getOrElse(executorContainerWithConfVolume)
+    } else {
+      executorContainer
+    }
     val containerWithLifecycle =
       if (!kubernetesConf.workerDecommissioning) {
         logInfo("Decommissioning not enabled, skipping shutdown script")
@@ -196,7 +237,7 @@ private[spark] class BasicExecutorFeatureStep(
         new ContainerBuilder(containerWithLimitCores).withNewLifecycle()
           .withNewPreStop()
             .withNewExec()
-              .addToCommand("/opt/decom.sh")
+              .addToCommand(kubernetesConf.get(DECOMMISSION_SCRIPT))
             .endExec()
           .endPreStop()
           .endLifecycle()
@@ -211,7 +252,7 @@ private[spark] class BasicExecutorFeatureStep(
         .withUid(pod.getMetadata.getUid)
         .build()
     }
-    val executorPod = new PodBuilder(pod.pod)
+    val executorPodBuilder = new PodBuilder(pod.pod)
       .editOrNewMetadata()
         .withName(name)
         .addToLabels(kubernetesConf.labels.asJava)
@@ -223,9 +264,20 @@ private[spark] class BasicExecutorFeatureStep(
         .withRestartPolicy("Never")
         .addToNodeSelector(kubernetesConf.nodeSelector.asJava)
         .addToImagePullSecrets(kubernetesConf.imagePullSecrets: _*)
+    val executorPod = if (disableConfigMap) {
+      executorPodBuilder.endSpec().build()
+    } else {
+      executorPodBuilder
+        .addNewVolume()
+          .withName(SPARK_CONF_VOLUME_EXEC)
+          .withNewConfigMap()
+            .withItems(keyToPaths.asJava)
+            .withName(configMapName)
+            .endConfigMap()
+          .endVolume()
         .endSpec()
       .build()
-
+    }
     kubernetesConf.get(KUBERNETES_EXECUTOR_SCHEDULER_NAME)
       .foreach(executorPod.getSpec.setSchedulerName)
 
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStep.scala
index ebe44855f1d0d..8015a1af3e17d 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStep.scala
@@ -24,6 +24,8 @@ import org.apache.spark.deploy.k8s._
 import org.apache.spark.deploy.k8s.Config._
 import org.apache.spark.deploy.k8s.Constants._
 import org.apache.spark.deploy.k8s.submit._
+import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config.{PYSPARK_DRIVER_PYTHON, PYSPARK_PYTHON}
 import org.apache.spark.launcher.SparkLauncher
 
 /**
@@ -31,7 +33,7 @@ import org.apache.spark.launcher.SparkLauncher
  * executors can also find the app code.
  */
 private[spark] class DriverCommandFeatureStep(conf: KubernetesDriverConf)
-  extends KubernetesFeatureConfigStep {
+  extends KubernetesFeatureConfigStep with Logging {
 
   override def configurePod(pod: SparkPod): SparkPod = {
     conf.mainAppResource match {
@@ -62,18 +64,50 @@ private[spark] class DriverCommandFeatureStep(conf: KubernetesDriverConf)
   }
 
   private def configureForJava(pod: SparkPod, res: String): SparkPod = {
-    val driverContainer = baseDriverContainer(pod, res).build()
+    // re-write primary resource, app jar is also added to spark.jars by default in SparkSubmit
+    // no uploading takes place here
+    val newResName = KubernetesUtils
+      .renameMainAppResource(resource = res, shouldUploadLocal = false)
+    val driverContainer = baseDriverContainer(pod, newResName).build()
     SparkPod(pod.pod, driverContainer)
   }
 
+  // Exposed for testing purpose.
+  private[spark] def environmentVariables: Map[String, String] = sys.env
+
   private def configureForPython(pod: SparkPod, res: String): SparkPod = {
-    val pythonEnvs =
-      Seq(new EnvVarBuilder()
-          .withName(ENV_PYSPARK_MAJOR_PYTHON_VERSION)
-          .withValue(conf.get(PYSPARK_MAJOR_PYTHON_VERSION))
-        .build())
+    if (conf.get(PYSPARK_MAJOR_PYTHON_VERSION).isDefined) {
+      logWarning(
+          s"${PYSPARK_MAJOR_PYTHON_VERSION.key} was deprecated in Spark 3.1. " +
+          s"Please set '${PYSPARK_PYTHON.key}' and '${PYSPARK_DRIVER_PYTHON.key}' " +
+          s"configurations or $ENV_PYSPARK_PYTHON and $ENV_PYSPARK_DRIVER_PYTHON environment " +
+          "variables instead.")
+    }
 
-    val pythonContainer = baseDriverContainer(pod, res)
+    val pythonEnvs =
+      Seq(
+        conf.get(PYSPARK_PYTHON)
+          .orElse(environmentVariables.get(ENV_PYSPARK_PYTHON)).map { value =>
+          new EnvVarBuilder()
+            .withName(ENV_PYSPARK_PYTHON)
+            .withValue(value)
+            .build()
+        },
+        conf.get(PYSPARK_DRIVER_PYTHON)
+          .orElse(conf.get(PYSPARK_PYTHON))
+          .orElse(environmentVariables.get(ENV_PYSPARK_DRIVER_PYTHON))
+          .orElse(environmentVariables.get(ENV_PYSPARK_PYTHON)).map { value =>
+          new EnvVarBuilder()
+            .withName(ENV_PYSPARK_DRIVER_PYTHON)
+            .withValue(value)
+            .build()
+        }
+      ).flatten
+
+    // re-write primary resource to be the remote one and upload the related file
+    val newResName = KubernetesUtils
+      .renameMainAppResource(res, Option(conf.sparkConf), true)
+    val pythonContainer = baseDriverContainer(pod, newResName)
       .addAllToEnv(pythonEnvs.asJava)
       .build()
 
@@ -88,7 +122,7 @@ private[spark] class DriverCommandFeatureStep(conf: KubernetesDriverConf)
   private def baseDriverContainer(pod: SparkPod, resource: String): ContainerBuilder = {
     // re-write primary resource, app jar is also added to spark.jars by default in SparkSubmit
     val resolvedResource = if (conf.mainAppResource.isInstanceOf[JavaMainAppResource]) {
-      KubernetesUtils.renameMainAppResource(resource, conf.sparkConf)
+      KubernetesUtils.renameMainAppResource(resource, Option(conf.sparkConf), false)
     } else {
       resource
     }
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStep.scala
index 1e9c60c871479..bd118f2439d21 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStep.scala
@@ -70,6 +70,7 @@ private[spark] class DriverServiceFeatureStep(
       .withNewMetadata()
         .withName(resolvedServiceName)
         .addToAnnotations(kubernetesConf.serviceAnnotations.asJava)
+        .addToLabels(SPARK_APP_ID_LABEL, kubernetesConf.appId)
         .endMetadata()
       .withNewSpec()
         .withClusterIP("None")
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/EnvSecretsFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/EnvSecretsFeatureStep.scala
index d78f04dcc40e6..222e19c5e20f1 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/EnvSecretsFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/EnvSecretsFeatureStep.scala
@@ -18,7 +18,7 @@ package org.apache.spark.deploy.k8s.features
 
 import scala.collection.JavaConverters._
 
-import io.fabric8.kubernetes.api.model.{ContainerBuilder, EnvVarBuilder, HasMetadata}
+import io.fabric8.kubernetes.api.model.{ContainerBuilder, EnvVarBuilder}
 
 import org.apache.spark.deploy.k8s.{KubernetesConf, SparkPod}
 
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesFeatureConfigStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesFeatureConfigStep.scala
index 58cdaa3cadd6b..3fec92644b956 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesFeatureConfigStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesFeatureConfigStep.scala
@@ -18,13 +18,18 @@ package org.apache.spark.deploy.k8s.features
 
 import io.fabric8.kubernetes.api.model.HasMetadata
 
+import org.apache.spark.annotation.{DeveloperApi, Unstable}
 import org.apache.spark.deploy.k8s.SparkPod
 
 /**
+ * :: DeveloperApi ::
+ *
  * A collection of functions that together represent a "feature" in pods that are launched for
  * Spark drivers and executors.
  */
-private[spark] trait KubernetesFeatureConfigStep {
+@Unstable
+@DeveloperApi
+trait KubernetesFeatureConfigStep {
 
   /**
    * Apply modifications on the given pod in accordance to this feature. This can include attaching
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountSecretsFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountSecretsFeatureStep.scala
index f4e1a3a326729..9de7686c8a9c0 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountSecretsFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountSecretsFeatureStep.scala
@@ -16,7 +16,7 @@
  */
 package org.apache.spark.deploy.k8s.features
 
-import io.fabric8.kubernetes.api.model.{ContainerBuilder, HasMetadata, PodBuilder, VolumeBuilder, VolumeMountBuilder}
+import io.fabric8.kubernetes.api.model.{ContainerBuilder, PodBuilder, VolumeBuilder, VolumeMountBuilder}
 
 import org.apache.spark.deploy.k8s.{KubernetesConf, SparkPod}
 
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala
index e598a38e7f36f..4e1647372ecdc 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala
@@ -16,12 +16,19 @@
  */
 package org.apache.spark.deploy.k8s.features
 
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+
 import io.fabric8.kubernetes.api.model._
 
 import org.apache.spark.deploy.k8s._
+import org.apache.spark.deploy.k8s.Constants.{ENV_EXECUTOR_ID, SPARK_APP_ID_LABEL}
 
 private[spark] class MountVolumesFeatureStep(conf: KubernetesConf)
   extends KubernetesFeatureConfigStep {
+  import MountVolumesFeatureStep._
+
+  val additionalResources = ArrayBuffer.empty[HasMetadata]
 
   override def configurePod(pod: SparkPod): SparkPod = {
     val (volumeMounts, volumes) = constructVolumes(conf.volumes).unzip
@@ -42,7 +49,12 @@ private[spark] class MountVolumesFeatureStep(conf: KubernetesConf)
   private def constructVolumes(
     volumeSpecs: Iterable[KubernetesVolumeSpec]
   ): Iterable[(VolumeMount, Volume)] = {
-    volumeSpecs.map { spec =>
+    val duplicateMountPaths = volumeSpecs.map(_.mountPath).toSeq.groupBy(identity).collect {
+      case (x, ys) if ys.length > 1 => s"'$x'"
+    }
+    require(duplicateMountPaths.isEmpty,
+      s"Found duplicated mountPath: ${duplicateMountPaths.mkString(", ")}")
+    volumeSpecs.zipWithIndex.map { case (spec, i) =>
       val volumeMount = new VolumeMountBuilder()
         .withMountPath(spec.mountPath)
         .withReadOnly(spec.mountReadOnly)
@@ -56,7 +68,34 @@ private[spark] class MountVolumesFeatureStep(conf: KubernetesConf)
           new VolumeBuilder()
             .withHostPath(new HostPathVolumeSource(hostPath, ""))
 
-        case KubernetesPVCVolumeConf(claimName) =>
+        case KubernetesPVCVolumeConf(claimNameTemplate, storageClass, size) =>
+          val claimName = conf match {
+            case c: KubernetesExecutorConf =>
+              claimNameTemplate
+                .replaceAll(PVC_ON_DEMAND,
+                  s"${conf.resourceNamePrefix}-exec-${c.executorId}$PVC_POSTFIX-$i")
+                .replaceAll(ENV_EXECUTOR_ID, c.executorId)
+            case _ =>
+              claimNameTemplate
+                .replaceAll(PVC_ON_DEMAND, s"${conf.resourceNamePrefix}-driver$PVC_POSTFIX-$i")
+          }
+          if (storageClass.isDefined && size.isDefined) {
+            additionalResources.append(new PersistentVolumeClaimBuilder()
+              .withKind(PVC)
+              .withApiVersion("v1")
+              .withNewMetadata()
+                .withName(claimName)
+                .addToLabels(SPARK_APP_ID_LABEL, conf.sparkConf.getAppId)
+                .endMetadata()
+              .withNewSpec()
+                .withStorageClassName(storageClass.get)
+                .withAccessModes(PVC_ACCESS_MODE)
+                .withResources(new ResourceRequirementsBuilder()
+                  .withRequests(Map("storage" -> new Quantity(size.get)).asJava).build())
+                .endSpec()
+              .build())
+          }
+
           new VolumeBuilder()
             .withPersistentVolumeClaim(
               new PersistentVolumeClaimVolumeSource(claimName, spec.mountReadOnly))
@@ -77,4 +116,15 @@ private[spark] class MountVolumesFeatureStep(conf: KubernetesConf)
       (volumeMount, volume)
     }
   }
+
+  override def getAdditionalKubernetesResources(): Seq[HasMetadata] = {
+    additionalResources.toSeq
+  }
+}
+
+private[spark] object MountVolumesFeatureStep {
+  val PVC_ON_DEMAND = "OnDemand"
+  val PVC = "PersistentVolumeClaim"
+  val PVC_POSTFIX = "-pvc"
+  val PVC_ACCESS_MODE = "ReadWriteOnce"
 }
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStep.scala
index 7f41ca43589b6..9788b2722ddfc 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStep.scala
@@ -22,15 +22,20 @@ import java.nio.charset.StandardCharsets
 import com.google.common.io.Files
 import io.fabric8.kubernetes.api.model.{ConfigMapBuilder, ContainerBuilder, HasMetadata, PodBuilder}
 
+import org.apache.spark.deploy.SparkHadoopUtil
 import org.apache.spark.deploy.k8s.{KubernetesConf, SparkPod}
 import org.apache.spark.deploy.k8s.Config._
 import org.apache.spark.deploy.k8s.Constants._
+import org.apache.spark.util.DependencyUtils.downloadFile
+import org.apache.spark.util.Utils
 
 private[spark] class PodTemplateConfigMapStep(conf: KubernetesConf)
   extends KubernetesFeatureConfigStep {
 
   private val hasTemplate = conf.contains(KUBERNETES_EXECUTOR_PODTEMPLATE_FILE)
 
+  private val configmapName = s"${conf.resourceNamePrefix}-$POD_TEMPLATE_CONFIGMAP"
+
   def configurePod(pod: SparkPod): SparkPod = {
     if (hasTemplate) {
       val podWithVolume = new PodBuilder(pod.pod)
@@ -38,7 +43,7 @@ private[spark] class PodTemplateConfigMapStep(conf: KubernetesConf)
             .addNewVolume()
               .withName(POD_TEMPLATE_VOLUME)
               .withNewConfigMap()
-                .withName(POD_TEMPLATE_CONFIGMAP)
+                .withName(configmapName)
                 .addNewItem()
                   .withKey(POD_TEMPLATE_KEY)
                   .withPath(EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME)
@@ -73,10 +78,13 @@ private[spark] class PodTemplateConfigMapStep(conf: KubernetesConf)
   override def getAdditionalKubernetesResources(): Seq[HasMetadata] = {
     if (hasTemplate) {
       val podTemplateFile = conf.get(KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).get
-      val podTemplateString = Files.toString(new File(podTemplateFile), StandardCharsets.UTF_8)
+      val hadoopConf = SparkHadoopUtil.get.newConfiguration(conf.sparkConf)
+      val uri = downloadFile(podTemplateFile, Utils.createTempDir(), conf.sparkConf, hadoopConf)
+      val file = new java.net.URI(uri).getPath
+      val podTemplateString = Files.toString(new File(file), StandardCharsets.UTF_8)
       Seq(new ConfigMapBuilder()
           .withNewMetadata()
-            .withName(POD_TEMPLATE_CONFIGMAP)
+            .withName(configmapName)
           .endMetadata()
           .addToData(POD_TEMPLATE_KEY, podTemplateString)
         .build())
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/K8sSubmitOps.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/K8sSubmitOps.scala
index d45c06772c76a..0238d5eafdea1 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/K8sSubmitOps.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/K8sSubmitOps.scala
@@ -19,7 +19,7 @@ package org.apache.spark.deploy.k8s.submit
 import scala.collection.JavaConverters._
 
 import K8SSparkSubmitOperation.getGracePeriod
-import io.fabric8.kubernetes.api.model.{DoneablePod, Pod, PodList}
+import io.fabric8.kubernetes.api.model.{Pod, PodList}
 import io.fabric8.kubernetes.client.KubernetesClient
 import io.fabric8.kubernetes.client.dsl.{NonNamespaceOperation, PodResource}
 
@@ -33,7 +33,7 @@ import org.apache.spark.util.{CommandLineLoggingUtils, Utils}
 
 private sealed trait K8sSubmitOp extends CommandLineLoggingUtils {
   type NON_NAMESPACED_PODS =
-    NonNamespaceOperation[Pod, PodList, DoneablePod, PodResource[Pod, DoneablePod]]
+    NonNamespaceOperation[Pod, PodList, PodResource[Pod]]
   def executeOnPod(pName: String, namespace: Option[String], sparkConf: SparkConf)
       (implicit client: KubernetesClient): Unit
   def executeOnGlob(pods: List[Pod], ns: Option[String], sparkConf: SparkConf)
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala
index b4155fed8aa24..d479d6688a8c0 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala
@@ -16,20 +16,23 @@
  */
 package org.apache.spark.deploy.k8s.submit
 
-import java.io.StringWriter
-import java.util.{Collections, UUID}
-import java.util.Properties
+import java.util.UUID
 
-import io.fabric8.kubernetes.api.model._
-import io.fabric8.kubernetes.client.KubernetesClient
+import scala.collection.JavaConverters._
 import scala.collection.mutable
+import scala.util.control.Breaks._
 import scala.util.control.NonFatal
 
+import io.fabric8.kubernetes.api.model._
+import io.fabric8.kubernetes.client.{KubernetesClient, Watch}
+import io.fabric8.kubernetes.client.Watcher.Action
+
 import org.apache.spark.SparkConf
 import org.apache.spark.deploy.SparkApplication
 import org.apache.spark.deploy.k8s._
 import org.apache.spark.deploy.k8s.Config._
 import org.apache.spark.deploy.k8s.Constants._
+import org.apache.spark.deploy.k8s.KubernetesUtils.addOwnerReference
 import org.apache.spark.internal.Logging
 import org.apache.spark.util.Utils
 
@@ -101,8 +104,11 @@ private[spark] class Client(
 
   def run(): Unit = {
     val resolvedDriverSpec = builder.buildFromFeatures(conf, kubernetesClient)
-    val configMapName = s"${conf.resourceNamePrefix}-driver-conf-map"
-    val configMap = buildConfigMap(configMapName, resolvedDriverSpec.systemProperties)
+    val configMapName = KubernetesClientUtils.configMapNameDriver
+    val confFilesMap = KubernetesClientUtils.buildSparkConfDirFilesMap(configMapName,
+      conf.sparkConf, resolvedDriverSpec.systemProperties)
+    val configMap = KubernetesClientUtils.buildConfigMap(configMapName, confFilesMap)
+
     // The include of the ENV_VAR for "SPARK_CONF_DIR" is to allow for the
     // Spark command builder to pickup on the Java Options present in the ConfigMap
     val resolvedDriverContainer = new ContainerBuilder(resolvedDriverSpec.pod.container)
@@ -111,7 +117,7 @@ private[spark] class Client(
         .withValue(SPARK_CONF_DIR_INTERNAL)
         .endEnv()
       .addNewVolumeMount()
-        .withName(SPARK_CONF_VOLUME)
+        .withName(SPARK_CONF_VOLUME_DRIVER)
         .withMountPath(SPARK_CONF_DIR_INTERNAL)
         .endVolumeMount()
       .build()
@@ -119,67 +125,47 @@ private[spark] class Client(
       .editSpec()
         .addToContainers(resolvedDriverContainer)
         .addNewVolume()
-          .withName(SPARK_CONF_VOLUME)
+          .withName(SPARK_CONF_VOLUME_DRIVER)
           .withNewConfigMap()
+            .withItems(KubernetesClientUtils.buildKeyToPathObjects(confFilesMap).asJava)
             .withName(configMapName)
             .endConfigMap()
           .endVolume()
         .endSpec()
       .build()
     val driverPodName = resolvedDriverPod.getMetadata.getName
-    Utils.tryWithResource(
-      kubernetesClient
-        .pods()
-        .withName(driverPodName)
-        .watch(watcher)) { _ =>
-      val createdDriverPod = kubernetesClient.pods().create(resolvedDriverPod)
-      try {
-        val otherKubernetesResources =
-          resolvedDriverSpec.driverKubernetesResources ++ Seq(configMap)
-        addDriverOwnerReference(createdDriverPod, otherKubernetesResources)
-        kubernetesClient.resourceList(otherKubernetesResources: _*).createOrReplace()
-      } catch {
-        case NonFatal(e) =>
-          kubernetesClient.pods().delete(createdDriverPod)
-          throw e
-      }
 
-      val sId = Seq(conf.namespace, driverPodName).mkString(":")
-      watcher.watchOrStop(sId)
+    var watch: Watch = null
+    val createdDriverPod = kubernetesClient.pods().create(resolvedDriverPod)
+    try {
+      val otherKubernetesResources = resolvedDriverSpec.driverKubernetesResources ++ Seq(configMap)
+      addOwnerReference(createdDriverPod, otherKubernetesResources)
+      kubernetesClient.resourceList(otherKubernetesResources: _*).createOrReplace()
+    } catch {
+      case NonFatal(e) =>
+        kubernetesClient.pods().delete(createdDriverPod)
+        throw e
     }
-  }
-
-  // Add a OwnerReference to the given resources making the driver pod an owner of them so when
-  // the driver pod is deleted, the resources are garbage collected.
-  private def addDriverOwnerReference(driverPod: Pod, resources: Seq[HasMetadata]): Unit = {
-    val driverPodOwnerReference = new OwnerReferenceBuilder()
-      .withName(driverPod.getMetadata.getName)
-      .withApiVersion(driverPod.getApiVersion)
-      .withUid(driverPod.getMetadata.getUid)
-      .withKind(driverPod.getKind)
-      .withController(true)
-      .build()
-    resources.foreach { resource =>
-      val originalMetadata = resource.getMetadata
-      originalMetadata.setOwnerReferences(Collections.singletonList(driverPodOwnerReference))
-    }
-  }
-
-  // Build a Config Map that will house spark conf properties in a single file for spark-submit
-  private def buildConfigMap(configMapName: String, conf: Map[String, String]): ConfigMap = {
-    val properties = new Properties()
-    conf.foreach { case (k, v) =>
-      properties.setProperty(k, v)
+    val sId = Seq(conf.namespace, driverPodName).mkString(":")
+    breakable {
+      while (true) {
+        val podWithName = kubernetesClient
+          .pods()
+          .withName(driverPodName)
+        // Reset resource to old before we start the watch, this is important for race conditions
+        watcher.reset()
+        watch = podWithName.watch(watcher)
+
+        // Send the latest pod state we know to the watcher to make sure we didn't miss anything
+        watcher.eventReceived(Action.MODIFIED, podWithName.get())
+
+        // Break the while loop if the pod is completed or we don't want to wait
+        if(watcher.watchOrStop(sId)) {
+          watch.close()
+          break
+        }
+      }
     }
-    val propertiesWriter = new StringWriter()
-    properties.store(propertiesWriter,
-      s"Java properties built from Kubernetes config map with name: $configMapName")
-    new ConfigMapBuilder()
-      .withNewMetadata()
-        .withName(configMapName)
-        .endMetadata()
-      .addToData(SPARK_CONF_FILE_NAME, propertiesWriter.toString)
-      .build()
   }
 }
 
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientUtils.scala
new file mode 100644
index 0000000000000..4207077677c25
--- /dev/null
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientUtils.scala
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.deploy.k8s.submit
+
+import java.io.{File, StringWriter}
+import java.nio.charset.MalformedInputException
+import java.util.Properties
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+import scala.io.{Codec, Source}
+
+import io.fabric8.kubernetes.api.model.{ConfigMap, ConfigMapBuilder, KeyToPath}
+
+import org.apache.spark.SparkConf
+import org.apache.spark.deploy.k8s.{Config, Constants, KubernetesUtils}
+import org.apache.spark.deploy.k8s.Constants.ENV_SPARK_CONF_DIR
+import org.apache.spark.internal.Logging
+
+private[spark] object KubernetesClientUtils extends Logging {
+
+  // Config map name can be 63 chars at max.
+  def configMapName(prefix: String): String = s"${prefix.take(54)}-conf-map"
+
+  val configMapNameExecutor: String = configMapName(s"spark-exec-${KubernetesUtils.uniqueID()}")
+
+  val configMapNameDriver: String = configMapName(s"spark-drv-${KubernetesUtils.uniqueID()}")
+
+  private def buildStringFromPropertiesMap(configMapName: String,
+      propertiesMap: Map[String, String]): String = {
+    val properties = new Properties()
+    propertiesMap.foreach { case (k, v) =>
+      properties.setProperty(k, v)
+    }
+    val propertiesWriter = new StringWriter()
+    properties.store(propertiesWriter,
+      s"Java properties built from Kubernetes config map with name: $configMapName")
+    propertiesWriter.toString
+  }
+
+  /**
+   * Build, file -> 'file's content' map of all the selected files in SPARK_CONF_DIR.
+   */
+  def buildSparkConfDirFilesMap(
+      configMapName: String,
+      sparkConf: SparkConf,
+      resolvedPropertiesMap: Map[String, String]): Map[String, String] = synchronized {
+    val loadedConfFilesMap = KubernetesClientUtils.loadSparkConfDirFiles(sparkConf)
+    // Add resolved spark conf to the loaded configuration files map.
+    if (resolvedPropertiesMap.nonEmpty) {
+      val resolvedProperties: String = KubernetesClientUtils
+        .buildStringFromPropertiesMap(configMapName, resolvedPropertiesMap)
+      loadedConfFilesMap ++ Map(Constants.SPARK_CONF_FILE_NAME -> resolvedProperties)
+    } else {
+      loadedConfFilesMap
+    }
+  }
+
+  def buildKeyToPathObjects(confFilesMap: Map[String, String]): Seq[KeyToPath] = {
+    confFilesMap.map {
+      case (fileName: String, _: String) =>
+        val filePermissionMode = 420  // 420 is decimal for octal literal 0644.
+        new KeyToPath(fileName, filePermissionMode, fileName)
+    }.toList.sortBy(x => x.getKey) // List is sorted to make mocking based tests work
+  }
+
+  /**
+   * Build a Config Map that will hold the content for environment variable SPARK_CONF_DIR
+   * on remote pods.
+   */
+  def buildConfigMap(configMapName: String, confFileMap: Map[String, String],
+      withLabels: Map[String, String] = Map()): ConfigMap = {
+    new ConfigMapBuilder()
+      .withNewMetadata()
+        .withName(configMapName)
+        .withLabels(withLabels.asJava)
+        .endMetadata()
+      .addToData(confFileMap.asJava)
+      .build()
+  }
+
+  private def orderFilesBySize(confFiles: Seq[File]): Seq[File] = {
+    val fileToFileSizePairs = confFiles.map(f => (f, f.getName.length + f.length()))
+    // sort first by name and then by length, so that during tests we have consistent results.
+    fileToFileSizePairs.sortBy(f => f._1).sortBy(f => f._2).map(_._1)
+  }
+
+  // exposed for testing
+  private[submit] def loadSparkConfDirFiles(conf: SparkConf): Map[String, String] = {
+    val confDir = Option(conf.getenv(ENV_SPARK_CONF_DIR)).orElse(
+      conf.getOption("spark.home").map(dir => s"$dir/conf"))
+    val maxSize = conf.get(Config.CONFIG_MAP_MAXSIZE)
+    if (confDir.isDefined) {
+      val confFiles: Seq[File] = listConfFiles(confDir.get, maxSize)
+      val orderedConfFiles = orderFilesBySize(confFiles)
+      var truncatedMapSize: Long = 0
+      val truncatedMap = mutable.HashMap[String, String]()
+      val skippedFiles = mutable.HashSet[String]()
+      var source: Source = Source.fromString("") // init with empty source.
+      for (file <- orderedConfFiles) {
+        try {
+          source = Source.fromFile(file)(Codec.UTF8)
+          val (fileName, fileContent) = file.getName -> source.mkString
+          if ((truncatedMapSize + fileName.length + fileContent.length) < maxSize) {
+            truncatedMap.put(fileName, fileContent)
+            truncatedMapSize = truncatedMapSize + (fileName.length + fileContent.length)
+          } else {
+            skippedFiles.add(fileName)
+          }
+        } catch {
+          case e: MalformedInputException =>
+            logWarning(
+              s"Unable to read a non UTF-8 encoded file ${file.getAbsolutePath}. Skipping...", e)
+            None
+        } finally {
+          source.close()
+        }
+      }
+      if (truncatedMap.nonEmpty) {
+        logInfo(s"Spark configuration files loaded from $confDir :" +
+          s" ${truncatedMap.keys.mkString(",")}")
+      }
+      if (skippedFiles.nonEmpty) {
+        logWarning(s"Skipped conf file(s) ${skippedFiles.mkString(",")}, due to size constraint." +
+          s" Please see, config: `${Config.CONFIG_MAP_MAXSIZE.key}` for more details.")
+      }
+      truncatedMap.toMap
+    } else {
+      Map.empty[String, String]
+    }
+  }
+
+  private def listConfFiles(confDir: String, maxSize: Long): Seq[File] = {
+    // At the moment configmaps do not support storing binary content (i.e. skip jar,tar,gzip,zip),
+    // and configMaps do not allow for size greater than 1.5 MiB(configurable).
+    // https://etcd.io/docs/v3.4.0/dev-guide/limit/
+    def testIfTooLargeOrBinary(f: File): Boolean = (f.length() + f.getName.length > maxSize) ||
+      f.getName.matches(".*\\.(gz|zip|jar|tar)")
+
+    // We exclude all the template files and user provided spark conf or properties,
+    // Spark properties are resolved in a different step.
+    def testIfSparkConfOrTemplates(f: File) = f.getName.matches(".*\\.template") ||
+      f.getName.matches("spark.*(conf|properties)")
+
+    val fileFilter = (f: File) => {
+      f.isFile && !testIfTooLargeOrBinary(f) && !testIfSparkConfOrTemplates(f)
+    }
+    val confFiles: Seq[File] = {
+      val dir = new File(confDir)
+      if (dir.isDirectory) {
+        dir.listFiles.filter(x => fileFilter(x)).toSeq
+      } else {
+        Nil
+      }
+    }
+    confFiles
+  }
+}
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala
index 43639a3b7dc1b..b8b93bb22b137 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala
@@ -16,12 +16,11 @@
  */
 package org.apache.spark.deploy.k8s.submit
 
-import java.io.File
-
 import io.fabric8.kubernetes.client.KubernetesClient
 
 import org.apache.spark.deploy.k8s._
 import org.apache.spark.deploy.k8s.features._
+import org.apache.spark.util.Utils
 
 private[spark] class KubernetesDriverBuilder {
 
@@ -32,11 +31,17 @@ private[spark] class KubernetesDriverBuilder {
       .map { file =>
         KubernetesUtils.loadPodFromTemplate(
           client,
-          new File(file),
-          conf.get(Config.KUBERNETES_DRIVER_PODTEMPLATE_CONTAINER_NAME))
+          file,
+          conf.get(Config.KUBERNETES_DRIVER_PODTEMPLATE_CONTAINER_NAME),
+          conf.sparkConf)
       }
       .getOrElse(SparkPod.initialPod())
 
+    val userFeatures = conf.get(Config.KUBERNETES_DRIVER_POD_FEATURE_STEPS)
+      .map { className =>
+        Utils.classForName(className).newInstance().asInstanceOf[KubernetesFeatureConfigStep]
+      }
+
     val features = Seq(
       new BasicDriverFeatureStep(conf),
       new DriverKubernetesCredentialsFeatureStep(conf),
@@ -48,7 +53,7 @@ private[spark] class KubernetesDriverBuilder {
       new HadoopConfDriverFeatureStep(conf),
       new KerberosConfDriverFeatureStep(conf),
       new PodTemplateConfigMapStep(conf),
-      new LocalDirsFeatureStep(conf))
+      new LocalDirsFeatureStep(conf)) ++ userFeatures
 
     val spec = KubernetesDriverSpec(
       initialPod,
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala
index ce3c80c0f85b1..81d9145725393 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala
@@ -17,7 +17,7 @@
 package org.apache.spark.deploy.k8s.submit
 
 import io.fabric8.kubernetes.api.model.Pod
-import io.fabric8.kubernetes.client.{KubernetesClientException, Watcher}
+import io.fabric8.kubernetes.client.{Watcher, WatcherException}
 import io.fabric8.kubernetes.client.Watcher.Action
 
 import org.apache.spark.deploy.k8s.Config._
@@ -26,7 +26,8 @@ import org.apache.spark.deploy.k8s.KubernetesUtils._
 import org.apache.spark.internal.Logging
 
 private[k8s] trait LoggingPodStatusWatcher extends Watcher[Pod] {
-  def watchOrStop(submissionId: String): Unit
+  def watchOrStop(submissionId: String): Boolean
+  def reset(): Unit
 }
 
 /**
@@ -42,10 +43,16 @@ private[k8s] class LoggingPodStatusWatcherImpl(conf: KubernetesDriverConf)
 
   private var podCompleted = false
 
+  private var resourceTooOldReceived = false
+
   private var pod = Option.empty[Pod]
 
   private def phase: String = pod.map(_.getStatus.getPhase).getOrElse("unknown")
 
+  override def reset(): Unit = {
+    resourceTooOldReceived = false
+  }
+
   override def eventReceived(action: Action, pod: Pod): Unit = {
     this.pod = Option(pod)
     action match {
@@ -60,7 +67,17 @@ private[k8s] class LoggingPodStatusWatcherImpl(conf: KubernetesDriverConf)
     }
   }
 
-  override def onClose(e: KubernetesClientException): Unit = {
+  override def onClose(e: WatcherException): Unit = {
+    logDebug(s"Stopping watching application $appId with last-observed phase $phase")
+    if(e != null && e.isHttpGone) {
+      resourceTooOldReceived = true
+      logDebug(s"Got HTTP Gone code, resource version changed in k8s api: $e")
+    } else {
+      closeWatch()
+    }
+  }
+
+  override def onClose(): Unit = {
     logDebug(s"Stopping watching application $appId with last-observed phase $phase")
     closeWatch()
   }
@@ -78,20 +95,26 @@ private[k8s] class LoggingPodStatusWatcherImpl(conf: KubernetesDriverConf)
     this.notifyAll()
   }
 
-  override def watchOrStop(sId: String): Unit = if (conf.get(WAIT_FOR_APP_COMPLETION)) {
+  override def watchOrStop(sId: String): Boolean = if (conf.get(WAIT_FOR_APP_COMPLETION)) {
     logInfo(s"Waiting for application ${conf.appName} with submission ID $sId to finish...")
     val interval = conf.get(REPORT_INTERVAL)
     synchronized {
-      while (!podCompleted) {
+      while (!podCompleted && !resourceTooOldReceived) {
         wait(interval)
         logInfo(s"Application status for $appId (phase: $phase)")
       }
     }
-    logInfo(
-      pod.map { p => s"Container final statuses:\n\n${containersDescription(p)}" }
-        .getOrElse("No containers were found in the driver pod."))
-    logInfo(s"Application ${conf.appName} with submission ID $sId finished")
+
+    if(podCompleted) {
+      logInfo(
+        pod.map { p => s"Container final statuses:\n\n${containersDescription(p)}" }
+          .getOrElse("No containers were found in the driver pod."))
+      logInfo(s"Application ${conf.appName} with submission ID $sId finished")
+    }
+    podCompleted
   } else {
     logInfo(s"Deployed Spark application ${conf.appName} with submission ID $sId into Kubernetes")
+    // Always act like the application has completed since we don't want to wait for app completion
+    true
   }
 }
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala
index b6ea1faeda972..d54f665a380f3 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala
@@ -16,17 +16,25 @@
  */
 package org.apache.spark.scheduler.cluster.k8s
 
-import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger, AtomicLong}
+import java.time.Instant
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.atomic.{AtomicInteger, AtomicLong}
 
-import io.fabric8.kubernetes.api.model.PodBuilder
-import io.fabric8.kubernetes.client.KubernetesClient
+import scala.collection.JavaConverters._
 import scala.collection.mutable
+import scala.util.control.NonFatal
+
+import io.fabric8.kubernetes.api.model.{PersistentVolumeClaim, PodBuilder}
+import io.fabric8.kubernetes.client.KubernetesClient
 
 import org.apache.spark.{SecurityManager, SparkConf, SparkException}
 import org.apache.spark.deploy.k8s.Config._
 import org.apache.spark.deploy.k8s.Constants._
 import org.apache.spark.deploy.k8s.KubernetesConf
+import org.apache.spark.deploy.k8s.KubernetesUtils.addOwnerReference
 import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config.DYN_ALLOCATION_EXECUTOR_IDLE_TIMEOUT
+import org.apache.spark.resource.ResourceProfile
 import org.apache.spark.util.{Clock, Utils}
 
 private[spark] class ExecutorPodsAllocator(
@@ -39,13 +47,21 @@ private[spark] class ExecutorPodsAllocator(
 
   private val EXECUTOR_ID_COUNTER = new AtomicLong(0L)
 
-  private val totalExpectedExecutors = new AtomicInteger(0)
+  // ResourceProfile id -> total expected executors per profile, currently we don't remove
+  // any resource profiles - https://issues.apache.org/jira/browse/SPARK-30749
+  private val totalExpectedExecutorsPerResourceProfileId = new ConcurrentHashMap[Int, Int]()
+
+  private val rpIdToResourceProfile = new mutable.HashMap[Int, ResourceProfile]
 
   private val podAllocationSize = conf.get(KUBERNETES_ALLOCATION_BATCH_SIZE)
 
   private val podAllocationDelay = conf.get(KUBERNETES_ALLOCATION_BATCH_DELAY)
 
-  private val podCreationTimeout = math.max(podAllocationDelay * 5, 60000)
+  private val podCreationTimeout = math.max(
+    podAllocationDelay * 5,
+    conf.get(KUBERNETES_ALLOCATION_EXECUTOR_TIMEOUT))
+
+  private val executorIdleTimeout = conf.get(DYN_ALLOCATION_EXECUTOR_IDLE_TIMEOUT) * 1000
 
   private val namespace = conf.get(KUBERNETES_NAMESPACE)
 
@@ -54,7 +70,7 @@ private[spark] class ExecutorPodsAllocator(
 
   private val shouldDeleteExecutors = conf.get(KUBERNETES_DELETE_EXECUTORS)
 
-  private val driverPod = kubernetesDriverPodName
+  val driverPod = kubernetesDriverPodName
     .map(name => Option(kubernetesClient.pods()
       .withName(name)
       .get())
@@ -63,29 +79,38 @@ private[spark] class ExecutorPodsAllocator(
           s"namespace $namespace (this was supposed to be the driver pod.).")))
 
   // Executor IDs that have been requested from Kubernetes but have not been detected in any
-  // snapshot yet. Mapped to the timestamp when they were created.
-  private val newlyCreatedExecutors = mutable.Map.empty[Long, Long]
+  // snapshot yet. Mapped to the (ResourceProfile id, timestamp) when they were created.
+  private val newlyCreatedExecutors = mutable.LinkedHashMap.empty[Long, (Int, Long)]
+
+  // Executor IDs that have been requested from Kubernetes but have not been detected in any POD
+  // snapshot yet but already known by the scheduler backend. Mapped to the ResourceProfile id.
+  private val schedulerKnownNewlyCreatedExecs = mutable.LinkedHashMap.empty[Long, Int]
 
   private val dynamicAllocationEnabled = Utils.isDynamicAllocationEnabled(conf)
 
-  private val hasPendingPods = new AtomicBoolean()
+  // visible for tests
+  private[k8s] val numOutstandingPods = new AtomicInteger()
 
-  private var lastSnapshot = ExecutorPodsSnapshot(Nil)
+  private var lastSnapshot = ExecutorPodsSnapshot()
 
   // Executors that have been deleted by this allocator but not yet detected as deleted in
   // a snapshot from the API server. This is used to deny registration from these executors
   // if they happen to come up before the deletion takes effect.
   @volatile private var deletedExecutorIds = Set.empty[Long]
 
-  def start(applicationId: String): Unit = {
+  def start(applicationId: String, schedulerBackend: KubernetesClusterSchedulerBackend): Unit = {
     snapshotsStore.addSubscriber(podAllocationDelay) {
-      onNewSnapshots(applicationId, _)
+      onNewSnapshots(applicationId, schedulerBackend, _)
     }
   }
 
-  def setTotalExpectedExecutors(total: Int): Unit = {
-    totalExpectedExecutors.set(total)
-    if (!hasPendingPods.get()) {
+  def setTotalExpectedExecutors(resourceProfileToTotalExecs: Map[ResourceProfile, Int]): Unit = {
+    resourceProfileToTotalExecs.foreach { case (rp, numExecs) =>
+      rpIdToResourceProfile.getOrElseUpdate(rp.id, rp)
+      totalExpectedExecutorsPerResourceProfileId.put(rp.id, numExecs)
+    }
+    logDebug(s"Set total expected execs to $totalExpectedExecutorsPerResourceProfileId")
+    if (numOutstandingPods.get() == 0) {
       snapshotsStore.notifySubscribers()
     }
   }
@@ -94,8 +119,19 @@ private[spark] class ExecutorPodsAllocator(
 
   private def onNewSnapshots(
       applicationId: String,
+      schedulerBackend: KubernetesClusterSchedulerBackend,
       snapshots: Seq[ExecutorPodsSnapshot]): Unit = {
-    newlyCreatedExecutors --= snapshots.flatMap(_.executorPods.keys)
+    val k8sKnownExecIds = snapshots.flatMap(_.executorPods.keys)
+    newlyCreatedExecutors --= k8sKnownExecIds
+    schedulerKnownNewlyCreatedExecs --= k8sKnownExecIds
+
+    // transfer the scheduler backend known executor requests from the newlyCreatedExecutors
+    // to the schedulerKnownNewlyCreatedExecs
+    val schedulerKnownExecs = schedulerBackend.getExecutorIds().map(_.toLong).toSet
+    schedulerKnownNewlyCreatedExecs ++=
+      newlyCreatedExecutors.filterKeys(schedulerKnownExecs.contains(_)).mapValues(_._1)
+    newlyCreatedExecutors --= schedulerKnownNewlyCreatedExecs.keySet
+
     // For all executors we've created against the API but have not seen in a snapshot
     // yet - check the current time. If the current time has exceeded some threshold,
     // assume that the pod was either never created (the API server never properly
@@ -103,7 +139,7 @@ private[spark] class ExecutorPodsAllocator(
     // both the creation and deletion events. In either case, delete the missing pod
     // if possible, and mark such a pod to be rescheduled below.
     val currentTime = clock.getTimeMillis()
-    val timedOut = newlyCreatedExecutors.flatMap { case (execId, timeCreated) =>
+    val timedOut = newlyCreatedExecutors.flatMap { case (execId, (_, timeCreated)) =>
       if (currentTime - timeCreated > podCreationTimeout) {
         Some(execId)
       } else {
@@ -136,116 +172,199 @@ private[spark] class ExecutorPodsAllocator(
       lastSnapshot = snapshots.last
     }
 
-    val currentRunningCount = lastSnapshot.executorPods.values.count {
-      case PodRunning(_) => true
-      case _ => false
-    }
-
-    val currentPendingExecutors = lastSnapshot.executorPods
-      .filter {
-        case (_, PodPending(_)) => true
-        case _ => false
-      }
-      .map { case (id, _) => id }
-
     // Make a local, non-volatile copy of the reference since it's used multiple times. This
     // is the only method that modifies the list, so this is safe.
     var _deletedExecutorIds = deletedExecutorIds
-
     if (snapshots.nonEmpty) {
-      logDebug(s"Pod allocation status: $currentRunningCount running, " +
-        s"${currentPendingExecutors.size} pending, " +
-        s"${newlyCreatedExecutors.size} unacknowledged.")
-
       val existingExecs = lastSnapshot.executorPods.keySet
       _deletedExecutorIds = _deletedExecutorIds.filter(existingExecs.contains)
     }
 
-    val currentTotalExpectedExecutors = totalExpectedExecutors.get
-
-    // This variable is used later to print some debug logs. It's updated when cleaning up
-    // excess pod requests, since currentPendingExecutors is immutable.
-    var knownPendingCount = currentPendingExecutors.size
-
-    // It's possible that we have outstanding pods that are outdated when dynamic allocation
-    // decides to downscale the application. So check if we can release any pending pods early
-    // instead of waiting for them to time out. Drop them first from the unacknowledged list,
-    // then from the pending.
-    //
-    // TODO: with dynamic allocation off, handle edge cases if we end up with more running
-    // executors than expected.
-    val knownPodCount = currentRunningCount + currentPendingExecutors.size +
-      newlyCreatedExecutors.size
-    if (knownPodCount > currentTotalExpectedExecutors) {
-      val excess = knownPodCount - currentTotalExpectedExecutors
-      val knownPendingToDelete = currentPendingExecutors.take(excess - newlyCreatedExecutors.size)
-      val toDelete = newlyCreatedExecutors.keys.take(excess).toList ++ knownPendingToDelete
-
-      if (toDelete.nonEmpty) {
-        logInfo(s"Deleting ${toDelete.size} excess pod requests (${toDelete.mkString(",")}).")
-        _deletedExecutorIds = _deletedExecutorIds ++ toDelete
+    val notDeletedPods = lastSnapshot.executorPods.filterKeys(!_deletedExecutorIds.contains(_))
+    // Map the pods into per ResourceProfile id so we can check per ResourceProfile,
+    // add a fast path if not using other ResourceProfiles.
+    val rpIdToExecsAndPodState =
+      mutable.HashMap[Int, mutable.HashMap[Long, ExecutorPodState]]()
+    if (totalExpectedExecutorsPerResourceProfileId.size <= 1) {
+      rpIdToExecsAndPodState(ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID) =
+        mutable.HashMap.empty ++= notDeletedPods
+    } else {
+      notDeletedPods.foreach { case (execId, execPodState) =>
+        val rpId = execPodState.pod.getMetadata.getLabels.get(SPARK_RESOURCE_PROFILE_ID_LABEL).toInt
+        val execPods = rpIdToExecsAndPodState.getOrElseUpdate(rpId,
+          mutable.HashMap[Long, ExecutorPodState]())
+        execPods(execId) = execPodState
+      }
+    }
 
-        Utils.tryLogNonFatalError {
-          kubernetesClient
-            .pods()
-            .withField("status.phase", "Pending")
-            .withLabel(SPARK_APP_ID_LABEL, applicationId)
-            .withLabel(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE)
-            .withLabelIn(SPARK_EXECUTOR_ID_LABEL, toDelete.sorted.map(_.toString): _*)
-            .delete()
-          newlyCreatedExecutors --= toDelete
-          knownPendingCount -= knownPendingToDelete.size
+    var totalPendingCount = 0
+    // The order we request executors for each ResourceProfile is not guaranteed.
+    totalExpectedExecutorsPerResourceProfileId.asScala.foreach { case (rpId, targetNum) =>
+      val podsForRpId = rpIdToExecsAndPodState.getOrElse(rpId, mutable.HashMap.empty)
+
+      val currentRunningCount = podsForRpId.values.count {
+        case PodRunning(_) => true
+        case _ => false
+      }
+
+      val (schedulerKnownPendingExecsForRpId, currentPendingExecutorsForRpId) = podsForRpId.filter {
+        case (_, PodPending(_)) => true
+        case _ => false
+      }.partition { case (k, _) =>
+        schedulerKnownExecs.contains(k)
+      }
+      // This variable is used later to print some debug logs. It's updated when cleaning up
+      // excess pod requests, since currentPendingExecutorsForRpId is immutable.
+      var knownPendingCount = currentPendingExecutorsForRpId.size
+
+      val newlyCreatedExecutorsForRpId =
+        newlyCreatedExecutors.filter { case (_, (waitingRpId, _)) =>
+          rpId == waitingRpId
         }
+
+      val schedulerKnownNewlyCreatedExecsForRpId =
+        schedulerKnownNewlyCreatedExecs.filter { case (_, waitingRpId) =>
+          rpId == waitingRpId
+        }
+
+      if (podsForRpId.nonEmpty) {
+        logDebug(s"ResourceProfile Id: $rpId " +
+          s"pod allocation status: $currentRunningCount running, " +
+          s"${currentPendingExecutorsForRpId.size} unknown pending, " +
+          s"${schedulerKnownPendingExecsForRpId.size} scheduler backend known pending, " +
+          s"${newlyCreatedExecutorsForRpId.size} unknown newly created, " +
+          s"${schedulerKnownNewlyCreatedExecsForRpId.size} scheduler backend known newly created.")
       }
-    }
 
-    if (newlyCreatedExecutors.isEmpty
-        && currentPendingExecutors.isEmpty
-        && currentRunningCount < currentTotalExpectedExecutors) {
-      val numExecutorsToAllocate = math.min(
-        currentTotalExpectedExecutors - currentRunningCount, podAllocationSize)
-      logInfo(s"Going to request $numExecutorsToAllocate executors from Kubernetes.")
-      for ( _ <- 0 until numExecutorsToAllocate) {
-        val newExecutorId = EXECUTOR_ID_COUNTER.incrementAndGet()
-        val executorConf = KubernetesConf.createExecutorConf(
-          conf,
-          newExecutorId.toString,
-          applicationId,
-          driverPod)
-        val executorPod = executorBuilder.buildFromFeatures(executorConf, secMgr,
-          kubernetesClient)
-        val podWithAttachedContainer = new PodBuilder(executorPod.pod)
-          .editOrNewSpec()
-          .addToContainers(executorPod.container)
-          .endSpec()
-          .build()
-        kubernetesClient.pods().create(podWithAttachedContainer)
-        newlyCreatedExecutors(newExecutorId) = clock.getTimeMillis()
-        logDebug(s"Requested executor with id $newExecutorId from Kubernetes.")
+      // It's possible that we have outstanding pods that are outdated when dynamic allocation
+      // decides to downscale the application. So check if we can release any pending pods early
+      // instead of waiting for them to time out. Drop them first from the unacknowledged list,
+      // then from the pending. However, in order to prevent too frequent fluctuation, newly
+      // requested pods are protected during executorIdleTimeout period.
+      //
+      // TODO: with dynamic allocation off, handle edge cases if we end up with more running
+      // executors than expected.
+      val knownPodCount = currentRunningCount +
+        currentPendingExecutorsForRpId.size + schedulerKnownPendingExecsForRpId.size +
+        newlyCreatedExecutorsForRpId.size + schedulerKnownNewlyCreatedExecsForRpId.size
+
+      if (knownPodCount > targetNum) {
+        val excess = knownPodCount - targetNum
+        val newlyCreatedToDelete = newlyCreatedExecutorsForRpId
+          .filter { case (_, (_, createTime)) =>
+            currentTime - createTime > executorIdleTimeout
+          }.keys.take(excess).toList
+        val knownPendingToDelete = currentPendingExecutorsForRpId
+          .filter(x => isExecutorIdleTimedOut(x._2, currentTime))
+          .take(excess - newlyCreatedToDelete.size)
+          .map { case (id, _) => id }
+        val toDelete = newlyCreatedToDelete ++ knownPendingToDelete
+
+        if (toDelete.nonEmpty) {
+          logInfo(s"Deleting ${toDelete.size} excess pod requests (${toDelete.mkString(",")}).")
+          _deletedExecutorIds = _deletedExecutorIds ++ toDelete
+
+          Utils.tryLogNonFatalError {
+            kubernetesClient
+              .pods()
+              .withField("status.phase", "Pending")
+              .withLabel(SPARK_APP_ID_LABEL, applicationId)
+              .withLabel(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE)
+              .withLabelIn(SPARK_EXECUTOR_ID_LABEL, toDelete.sorted.map(_.toString): _*)
+              .delete()
+            newlyCreatedExecutors --= newlyCreatedToDelete
+            knownPendingCount -= knownPendingToDelete.size
+          }
+        }
       }
-    }
 
+      if (newlyCreatedExecutorsForRpId.isEmpty
+        && knownPodCount < targetNum) {
+        requestNewExecutors(targetNum, knownPodCount, applicationId, rpId)
+      }
+      totalPendingCount += knownPendingCount
+
+      // The code below just prints debug messages, which are only useful when there's a change
+      // in the snapshot state. Since the messages are a little spammy, avoid them when we know
+      // there are no useful updates.
+      if (log.isDebugEnabled && snapshots.nonEmpty) {
+        val outstanding = knownPendingCount + newlyCreatedExecutorsForRpId.size
+        if (currentRunningCount >= targetNum && !dynamicAllocationEnabled) {
+          logDebug(s"Current number of running executors for ResourceProfile Id $rpId is " +
+            "equal to the number of requested executors. Not scaling up further.")
+        } else {
+          if (outstanding > 0) {
+            logDebug(s"Still waiting for $outstanding executors for ResourceProfile " +
+              s"Id $rpId before requesting more.")
+          }
+        }
+      }
+    }
     deletedExecutorIds = _deletedExecutorIds
 
     // Update the flag that helps the setTotalExpectedExecutors() callback avoid triggering this
-    // update method when not needed.
-    hasPendingPods.set(knownPendingCount + newlyCreatedExecutors.size > 0)
-
-    // The code below just prints debug messages, which are only useful when there's a change
-    // in the snapshot state. Since the messages are a little spammy, avoid them when we know
-    // there are no useful updates.
-    if (!log.isDebugEnabled || snapshots.isEmpty) {
-      return
-    }
+    // update method when not needed. PODs known by the scheduler backend are not counted here as
+    // they considered running PODs and they should not block upscaling.
+    numOutstandingPods.set(totalPendingCount + newlyCreatedExecutors.size)
+  }
 
-    if (currentRunningCount >= currentTotalExpectedExecutors && !dynamicAllocationEnabled) {
-      logDebug("Current number of running executors is equal to the number of requested" +
-        " executors. Not scaling up further.")
-    } else {
-      val outstanding = knownPendingCount + newlyCreatedExecutors.size
-      if (outstanding > 0) {
-        logDebug(s"Still waiting for $outstanding executors before requesting more.")
+  private def requestNewExecutors(
+      expected: Int,
+      running: Int,
+      applicationId: String,
+      resourceProfileId: Int): Unit = {
+    val numExecutorsToAllocate = math.min(expected - running, podAllocationSize)
+    logInfo(s"Going to request $numExecutorsToAllocate executors from Kubernetes for " +
+      s"ResourceProfile Id: $resourceProfileId, target: $expected running: $running.")
+    for ( _ <- 0 until numExecutorsToAllocate) {
+      val newExecutorId = EXECUTOR_ID_COUNTER.incrementAndGet()
+      val executorConf = KubernetesConf.createExecutorConf(
+        conf,
+        newExecutorId.toString,
+        applicationId,
+        driverPod,
+        resourceProfileId)
+      val resolvedExecutorSpec = executorBuilder.buildFromFeatures(executorConf, secMgr,
+        kubernetesClient, rpIdToResourceProfile(resourceProfileId))
+      val executorPod = resolvedExecutorSpec.pod
+      val podWithAttachedContainer = new PodBuilder(executorPod.pod)
+        .editOrNewSpec()
+        .addToContainers(executorPod.container)
+        .endSpec()
+        .build()
+      val createdExecutorPod = kubernetesClient.pods().create(podWithAttachedContainer)
+      try {
+        val resources = resolvedExecutorSpec.executorKubernetesResources
+        addOwnerReference(createdExecutorPod, resources)
+        resources
+          .filter(_.getKind == "PersistentVolumeClaim")
+          .foreach { resource =>
+            if (conf.get(KUBERNETES_DRIVER_OWN_PVC) && driverPod.nonEmpty) {
+              addOwnerReference(driverPod.get, Seq(resource))
+            }
+            val pvc = resource.asInstanceOf[PersistentVolumeClaim]
+            logInfo(s"Trying to create PersistentVolumeClaim ${pvc.getMetadata.getName} with " +
+              s"StorageClass ${pvc.getSpec.getStorageClassName}")
+            kubernetesClient.persistentVolumeClaims().create(pvc)
+          }
+        newlyCreatedExecutors(newExecutorId) = (resourceProfileId, clock.getTimeMillis())
+        logDebug(s"Requested executor with id $newExecutorId from Kubernetes.")
+      } catch {
+        case NonFatal(e) =>
+          kubernetesClient.pods().delete(createdExecutorPod)
+          throw e
       }
     }
   }
+
+  private def isExecutorIdleTimedOut(state: ExecutorPodState, currentTime: Long): Boolean = {
+    try {
+      val startTime = Instant.parse(state.pod.getStatus.getStartTime).toEpochMilli()
+      currentTime - startTime > executorIdleTimeout
+    } catch {
+      case _: Exception =>
+        logDebug(s"Cannot get startTime of pod ${state.pod}")
+        true
+    }
+  }
 }
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala
index 5d91e52cb23e8..e255de4d2dd9e 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala
@@ -16,7 +16,9 @@
  */
 package org.apache.spark.scheduler.cluster.k8s
 
-import com.google.common.cache.Cache
+import java.util.concurrent.TimeUnit
+
+import com.google.common.cache.CacheBuilder
 import io.fabric8.kubernetes.api.model.{Pod, PodBuilder}
 import io.fabric8.kubernetes.client.KubernetesClient
 import scala.collection.JavaConverters._
@@ -33,16 +35,23 @@ import org.apache.spark.util.Utils
 private[spark] class ExecutorPodsLifecycleManager(
     val conf: SparkConf,
     kubernetesClient: KubernetesClient,
-    snapshotsStore: ExecutorPodsSnapshotsStore,
-    // Use a best-effort to track which executors have been removed already. It's not generally
-    // job-breaking if we remove executors more than once but it's ideal if we make an attempt
-    // to avoid doing so. Expire cache entries so that this data structure doesn't grow beyond
-    // bounds.
-    removedExecutorsCache: Cache[java.lang.Long, java.lang.Long]) extends Logging {
+    snapshotsStore: ExecutorPodsSnapshotsStore) extends Logging {
 
   import ExecutorPodsLifecycleManager._
 
   private lazy val shouldDeleteExecutors = conf.get(KUBERNETES_DELETE_EXECUTORS)
+  private lazy val missingPodDetectDelta = conf.get(KUBERNETES_EXECUTOR_MISSING_POD_DETECT_DELTA)
+
+  // Use a best-effort to track which executors have been removed already. It's not generally
+  // job-breaking if we remove executors more than once but it's ideal if we make an attempt
+  // to avoid doing so. Expire cache entries so that this data structure doesn't grow beyond
+  // bounds.
+  private lazy val removedExecutorsCache =
+    CacheBuilder.newBuilder()
+      .expireAfterWrite(3, TimeUnit.MINUTES)
+      .build[java.lang.Long, java.lang.Long]()
+
+  private var lastFullSnapshotTs: Long = 0
 
   // Keep track of which pods are inactive to avoid contacting the API server multiple times.
   // This set is cleaned up when a snapshot containing the updated pod is processed.
@@ -109,33 +118,33 @@ private[spark] class ExecutorPodsLifecycleManager(
     // Reconcile the case where Spark claims to know about an executor but the corresponding pod
     // is missing from the cluster. This would occur if we miss a deletion event and the pod
     // transitions immediately from running to absent. We only need to check against the latest
-    // snapshot for this, and we don't do this for executors in the deleted executors cache or
-    // that we just removed in this round.
-    val lostExecutors = if (snapshots.nonEmpty) {
-      schedulerBackend.getExecutorIds().map(_.toLong).toSet --
+    // fresh full snapshot (coming from ExecutorPodsPollingSnapshotSource) for this, and we don't
+    // do this for executors in the deleted executors cache or that we just removed in this round.
+    if (snapshots.nonEmpty && lastFullSnapshotTs != snapshots.last.fullSnapshotTs) {
+      lastFullSnapshotTs = snapshots.last.fullSnapshotTs
+      val lostExecutorsWithRegistrationTs =
+        schedulerBackend.getExecutorsWithRegistrationTs().map(t => (t._1.toLong, t._2)) --
         snapshots.last.executorPods.keySet -- execIdsRemovedInThisRound
-    } else {
-      Nil
-    }
 
-    lostExecutors.foreach { lostId =>
-      if (removedExecutorsCache.getIfPresent(lostId) == null) {
-        val exitReasonMessage = s"The executor with ID $lostId was not found in the" +
-          s" cluster but we didn't get a reason why. Marking the executor as failed. The" +
-          s" executor may have been deleted but the driver missed the deletion event."
-        logDebug(exitReasonMessage)
-        val exitReason = ExecutorExited(
-          UNKNOWN_EXIT_CODE,
-          exitCausedByApp = false,
-          exitReasonMessage)
-        schedulerBackend.doRemoveExecutor(lostId.toString, exitReason)
+      lostExecutorsWithRegistrationTs.foreach { case (lostExecId, lostExecRegistrationTs) =>
+        if (removedExecutorsCache.getIfPresent(lostExecId) == null &&
+            lastFullSnapshotTs - lostExecRegistrationTs > missingPodDetectDelta) {
+          val exitReasonMessage = s"The executor with ID $lostExecId (registered at " +
+            s"$lostExecRegistrationTs ms) was not found in the cluster at the polling time " +
+            s"($lastFullSnapshotTs ms) which is after the accepted detect delta time " +
+            s"($missingPodDetectDelta ms) configured by " +
+            s"`${KUBERNETES_EXECUTOR_MISSING_POD_DETECT_DELTA.key}`. " +
+            "The executor may have been deleted but the driver missed the deletion event. " +
+            "Marking this executor as failed."
+          logDebug(exitReasonMessage)
+          val exitReason = ExecutorExited(
+            UNKNOWN_EXIT_CODE,
+            exitCausedByApp = false,
+            exitReasonMessage)
+          schedulerBackend.doRemoveExecutor(lostExecId.toString, exitReason)
+        }
       }
     }
-
-    if (lostExecutors.nonEmpty) {
-      logDebug(s"Removed executors with ids ${lostExecutors.mkString(",")}" +
-        s" from Spark that were either found to be deleted or non-existent in the cluster.")
-    }
   }
 
   private def onFinalNonDeletedState(
@@ -212,10 +221,17 @@ private[spark] class ExecutorPodsLifecycleManager(
     val pod = podState.pod
     val reason = Option(pod.getStatus.getReason)
     val message = Option(pod.getStatus.getMessage)
+    val explained = describeExitCode(exitCode)
+    val exitMsg = s"The executor with id $execId exited with exit code $explained."
+    val reasonStr = reason.map(r => s"The API gave the following brief reason: ${r}")
+    val msgStr = message.map(m => s"The API gave the following message: ${m}")
+
+
     s"""
-       |The executor with id $execId exited with exit code $exitCode.
-       |The API gave the following brief reason: ${reason.getOrElse("N/A")}
-       |The API gave the following message: ${message.getOrElse("N/A")}
+       |${exitMsg}
+       |${reasonStr.getOrElse("")}
+       |${msgStr.getOrElse("")}
+       |
        |The API gave the following container statuses:
        |
        |${containersDescription(pod)}
@@ -237,5 +253,25 @@ private[spark] class ExecutorPodsLifecycleManager(
 
 private object ExecutorPodsLifecycleManager {
   val UNKNOWN_EXIT_CODE = -1
-}
 
+  // A utility function to try and help people figure out whats gone wrong faster.
+  def describeExitCode(code: Int): String = {
+    val humanStr = code match {
+      case 0 => "(success)"
+      case 1 => "(generic, look at logs to clarify)"
+      case 42 => "(Douglas Adams fan)"
+      // Spark specific
+      case 10 | 50 => "(Uncaught exception)"
+      case 52 => "(JVM OOM)"
+      case 53 => "(DiskStore failed to create temp dir)"
+      // K8s & JVM specific exit codes
+      case 126 => "(not executable - possibly perm or arch)"
+      case 137 => "(SIGKILL, possible container OOM)"
+      case 139 => "(SIGSEGV: that's unexpected)"
+      case 255 => "(exit-1, your guess is as good as mine)"
+      case _ => "(unexpected)"
+    }
+    s"${code}${humanStr}"
+  }
+
+}
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsPollingSnapshotSource.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsPollingSnapshotSource.scala
index fd8f6979c9e65..da7fe7cdda328 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsPollingSnapshotSource.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsPollingSnapshotSource.scala
@@ -62,7 +62,7 @@ private[spark] class ExecutorPodsPollingSnapshotSource(
         .withoutLabel(SPARK_EXECUTOR_INACTIVE_LABEL, "true")
         .list()
         .getItems
-        .asScala)
+        .asScala.toSeq)
     }
   }
 
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshot.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshot.scala
index 30030ab539048..76c17cf933fe6 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshot.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshot.scala
@@ -18,6 +18,9 @@ package org.apache.spark.scheduler.cluster.k8s
 
 import java.util.Locale
 
+import scala.collection.JavaConverters._
+
+import io.fabric8.kubernetes.api.model.ContainerStateTerminated
 import io.fabric8.kubernetes.api.model.Pod
 
 import org.apache.spark.deploy.k8s.Constants._
@@ -26,23 +29,35 @@ import org.apache.spark.internal.Logging
 /**
  * An immutable view of the current executor pods that are running in the cluster.
  */
-private[spark] case class ExecutorPodsSnapshot(executorPods: Map[Long, ExecutorPodState]) {
+private[spark] case class ExecutorPodsSnapshot(
+    executorPods: Map[Long, ExecutorPodState],
+    fullSnapshotTs: Long) {
 
   import ExecutorPodsSnapshot._
 
   def withUpdate(updatedPod: Pod): ExecutorPodsSnapshot = {
     val newExecutorPods = executorPods ++ toStatesByExecutorId(Seq(updatedPod))
-    new ExecutorPodsSnapshot(newExecutorPods)
+    new ExecutorPodsSnapshot(newExecutorPods, fullSnapshotTs)
   }
 }
 
 object ExecutorPodsSnapshot extends Logging {
+  private var shouldCheckAllContainers: Boolean = _
+  private var sparkContainerName: String = DEFAULT_EXECUTOR_CONTAINER_NAME
 
-  def apply(executorPods: Seq[Pod]): ExecutorPodsSnapshot = {
-    ExecutorPodsSnapshot(toStatesByExecutorId(executorPods))
+  def apply(executorPods: Seq[Pod], fullSnapshotTs: Long): ExecutorPodsSnapshot = {
+    ExecutorPodsSnapshot(toStatesByExecutorId(executorPods), fullSnapshotTs)
   }
 
-  def apply(): ExecutorPodsSnapshot = ExecutorPodsSnapshot(Map.empty[Long, ExecutorPodState])
+  def apply(): ExecutorPodsSnapshot = ExecutorPodsSnapshot(Map.empty[Long, ExecutorPodState], 0)
+
+  def setShouldCheckAllContainers(watchAllContainers: Boolean): Unit = {
+    shouldCheckAllContainers = watchAllContainers
+  }
+
+  def setSparkContainerName(containerName: String): Unit = {
+    sparkContainerName = containerName
+  }
 
   private def toStatesByExecutorId(executorPods: Seq[Pod]): Map[Long, ExecutorPodState] = {
     executorPods.map { pod =>
@@ -59,7 +74,32 @@ object ExecutorPodsSnapshot extends Logging {
         case "pending" =>
           PodPending(pod)
         case "running" =>
-          PodRunning(pod)
+          // If we're checking all containers look for any non-zero exits
+          if (shouldCheckAllContainers &&
+            "Never" == pod.getSpec.getRestartPolicy &&
+            pod.getStatus.getContainerStatuses.stream
+              .map[ContainerStateTerminated](cs => cs.getState.getTerminated)
+              .anyMatch(t => t != null && t.getExitCode != 0)) {
+            PodFailed(pod)
+          } else {
+            // Otherwise look for the Spark container and get the exit code if present.
+            val sparkContainerExitCode = pod.getStatus.getContainerStatuses.asScala
+              .find(_.getName() == sparkContainerName).flatMap(x => Option(x.getState))
+              .flatMap(x => Option(x.getTerminated)).flatMap(x => Option(x.getExitCode))
+              .map(_.toInt)
+            sparkContainerExitCode match {
+              case Some(t) =>
+                t match {
+                  case 0 =>
+                    PodSucceeded(pod)
+                  case _ =>
+                    PodFailed(pod)
+                }
+              // No exit code means we are running.
+              case _ =>
+                PodRunning(pod)
+            }
+          }
         case "failed" =>
           PodFailed(pod)
         case "succeeded" =>
@@ -79,7 +119,8 @@ object ExecutorPodsSnapshot extends Logging {
       (
         pod.getStatus == null ||
         pod.getStatus.getPhase == null ||
-        pod.getStatus.getPhase.toLowerCase(Locale.ROOT) != "terminating"
+          (pod.getStatus.getPhase.toLowerCase(Locale.ROOT) != "terminating" &&
+           pod.getStatus.getPhase.toLowerCase(Locale.ROOT) != "running")
       ))
   }
 }
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreImpl.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreImpl.scala
index d68dc3ebef5d8..df8769b1ed255 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreImpl.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreImpl.scala
@@ -28,6 +28,8 @@ import scala.util.control.NonFatal
 import io.fabric8.kubernetes.api.model.Pod
 
 import org.apache.spark.internal.Logging
+import org.apache.spark.util.Clock
+import org.apache.spark.util.SystemClock
 import org.apache.spark.util.ThreadUtils
 
 /**
@@ -52,9 +54,11 @@ import org.apache.spark.util.ThreadUtils
  * time-windowed chunks. Each subscriber can choose to receive their snapshot chunks at different
  * time intervals.
  * <br>
- * The subcriber notification callback is guaranteed to be called from a single thread at a time.
+ * The subscriber notification callback is guaranteed to be called from a single thread at a time.
  */
-private[spark] class ExecutorPodsSnapshotsStoreImpl(subscribersExecutor: ScheduledExecutorService)
+private[spark] class ExecutorPodsSnapshotsStoreImpl(
+    subscribersExecutor: ScheduledExecutorService,
+    clock: Clock = new SystemClock)
   extends ExecutorPodsSnapshotsStore with Logging {
 
   private val SNAPSHOT_LOCK = new Object()
@@ -99,7 +103,7 @@ private[spark] class ExecutorPodsSnapshotsStoreImpl(subscribersExecutor: Schedul
   }
 
   override def replaceSnapshot(newSnapshot: Seq[Pod]): Unit = SNAPSHOT_LOCK.synchronized {
-    currentSnapshot = ExecutorPodsSnapshot(newSnapshot)
+    currentSnapshot = ExecutorPodsSnapshot(newSnapshot, clock.getTimeMillis())
     addCurrentSnapshotToSubscribers()
   }
 
@@ -131,15 +135,18 @@ private[spark] class ExecutorPodsSnapshotsStoreImpl(subscribersExecutor: Schedul
           try {
             val snapshots = new ArrayList[ExecutorPodsSnapshot]()
             snapshotsBuffer.drainTo(snapshots)
-            onNewSnapshots(snapshots.asScala)
+            onNewSnapshots(snapshots.asScala.toSeq)
           } catch {
+            case e: IllegalArgumentException =>
+              logError("Going to stop due to IllegalArgumentException", e)
+              System.exit(1)
             case NonFatal(e) => logWarning("Exception when notifying snapshot subscriber.", e)
           } finally {
             lock.unlock()
           }
 
           if (notificationCount.decrementAndGet() > 0) {
-            // There was another concurrent request for this subcriber. Schedule a task to
+            // There was another concurrent request for this subscriber. Schedule a task to
             // immediately process snapshots again, so that the subscriber can pick up any
             // changes that may have happened between the time it started looking at snapshots
             // above, and the time the concurrent request arrived.
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsWatchSnapshotSource.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsWatchSnapshotSource.scala
index a6749a644e00c..762878cbacac6 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsWatchSnapshotSource.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsWatchSnapshotSource.scala
@@ -19,7 +19,7 @@ package org.apache.spark.scheduler.cluster.k8s
 import java.io.Closeable
 
 import io.fabric8.kubernetes.api.model.Pod
-import io.fabric8.kubernetes.client.{KubernetesClient, KubernetesClientException, Watcher}
+import io.fabric8.kubernetes.client.{KubernetesClient, Watcher, WatcherException}
 import io.fabric8.kubernetes.client.Watcher.Action
 
 import org.apache.spark.deploy.k8s.Constants._
@@ -58,10 +58,14 @@ private[spark] class ExecutorPodsWatchSnapshotSource(
       snapshotsStore.updatePod(pod)
     }
 
-    override def onClose(e: KubernetesClientException): Unit = {
+    override def onClose(e: WatcherException): Unit = {
       logWarning("Kubernetes client has been closed (this is expected if the application is" +
         " shutting down.)", e)
     }
+
+    override def onClose(): Unit = {
+      logWarning("Kubernetes client has been closed.")
+    }
   }
 
 }
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala
index b9d7a7083f41a..4d28d3816dbde 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala
@@ -17,15 +17,13 @@
 package org.apache.spark.scheduler.cluster.k8s
 
 import java.io.File
-import java.util.concurrent.TimeUnit
 
-import com.google.common.cache.CacheBuilder
 import io.fabric8.kubernetes.client.Config
 
 import org.apache.spark.SparkContext
 import org.apache.spark.deploy.k8s.{KubernetesConf, KubernetesUtils, SparkKubernetesClientFactory}
 import org.apache.spark.deploy.k8s.Config._
-import org.apache.spark.deploy.k8s.Constants._
+import org.apache.spark.deploy.k8s.Constants.DEFAULT_EXECUTOR_CONTAINER_NAME
 import org.apache.spark.internal.Logging
 import org.apache.spark.scheduler.{ExternalClusterManager, SchedulerBackend, TaskScheduler, TaskSchedulerImpl}
 import org.apache.spark.util.{SystemClock, ThreadUtils}
@@ -50,10 +48,14 @@ private[spark] class KubernetesClusterManager extends ExternalClusterManager wit
       require(sc.conf.get(KUBERNETES_DRIVER_POD_NAME).isDefined,
         "If the application is deployed using spark-submit in cluster mode, the driver pod name " +
           "must be provided.")
+      val serviceAccountToken =
+        Some(new File(Config.KUBERNETES_SERVICE_ACCOUNT_TOKEN_PATH)).filter(_.exists)
+      val serviceAccountCaCrt =
+        Some(new File(Config.KUBERNETES_SERVICE_ACCOUNT_CA_CRT_PATH)).filter(_.exists)
       (KUBERNETES_AUTH_DRIVER_MOUNTED_CONF_PREFIX,
         sc.conf.get(KUBERNETES_DRIVER_MASTER_URL),
-        Some(new File(Config.KUBERNETES_SERVICE_ACCOUNT_TOKEN_PATH)),
-        Some(new File(Config.KUBERNETES_SERVICE_ACCOUNT_CA_CRT_PATH)))
+        serviceAccountToken,
+        serviceAccountCaCrt)
     } else {
       (KUBERNETES_AUTH_CLIENT_MODE_PREFIX,
         KubernetesUtils.parseMasterUrl(masterURL),
@@ -84,25 +86,28 @@ private[spark] class KubernetesClusterManager extends ExternalClusterManager wit
     if (sc.conf.get(KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).isDefined) {
       KubernetesUtils.loadPodFromTemplate(
         kubernetesClient,
-        new File(sc.conf.get(KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).get),
-        sc.conf.get(KUBERNETES_EXECUTOR_PODTEMPLATE_CONTAINER_NAME))
+        sc.conf.get(KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).get,
+        sc.conf.get(KUBERNETES_EXECUTOR_PODTEMPLATE_CONTAINER_NAME),
+        sc.conf)
     }
 
     val schedulerExecutorService = ThreadUtils.newDaemonSingleThreadScheduledExecutor(
       "kubernetes-executor-maintenance")
 
+    ExecutorPodsSnapshot.setShouldCheckAllContainers(
+      sc.conf.get(KUBERNETES_EXECUTOR_CHECK_ALL_CONTAINERS))
+    val sparkContainerName = sc.conf.get(KUBERNETES_EXECUTOR_PODTEMPLATE_CONTAINER_NAME)
+      .getOrElse(DEFAULT_EXECUTOR_CONTAINER_NAME)
+    ExecutorPodsSnapshot.setSparkContainerName(sparkContainerName)
     val subscribersExecutor = ThreadUtils
       .newDaemonThreadPoolScheduledExecutor(
         "kubernetes-executor-snapshots-subscribers", 2)
     val snapshotsStore = new ExecutorPodsSnapshotsStoreImpl(subscribersExecutor)
-    val removedExecutorsCache = CacheBuilder.newBuilder()
-      .expireAfterWrite(3, TimeUnit.MINUTES)
-      .build[java.lang.Long, java.lang.Long]()
+
     val executorPodsLifecycleEventHandler = new ExecutorPodsLifecycleManager(
       sc.conf,
       kubernetesClient,
-      snapshotsStore,
-      removedExecutorsCache)
+      snapshotsStore)
 
     val executorPodsAllocator = new ExecutorPodsAllocator(
       sc.conf,
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackend.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackend.scala
index 5655ef50d214f..5dad6a3b7622b 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackend.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackend.scala
@@ -20,17 +20,21 @@ import java.util.concurrent.{ScheduledExecutorService, TimeUnit}
 
 import scala.concurrent.Future
 
+import io.fabric8.kubernetes.api.model.Pod
 import io.fabric8.kubernetes.client.KubernetesClient
 
 import org.apache.spark.SparkContext
 import org.apache.spark.deploy.k8s.Config._
 import org.apache.spark.deploy.k8s.Constants._
+import org.apache.spark.deploy.k8s.KubernetesUtils
+import org.apache.spark.deploy.k8s.submit.KubernetesClientUtils
 import org.apache.spark.deploy.security.HadoopDelegationTokenManager
 import org.apache.spark.internal.config.SCHEDULER_MIN_REGISTERED_RESOURCES_RATIO
 import org.apache.spark.resource.ResourceProfile
-import org.apache.spark.rpc.RpcAddress
+import org.apache.spark.rpc.{RpcAddress, RpcCallContext}
 import org.apache.spark.scheduler.{ExecutorKilled, ExecutorLossReason, TaskSchedulerImpl}
 import org.apache.spark.scheduler.cluster.{CoarseGrainedSchedulerBackend, SchedulerBackendUtils}
+import org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages.RegisterExecutor
 import org.apache.spark.util.{ThreadUtils, Utils}
 
 private[spark] class KubernetesClusterSchedulerBackend(
@@ -54,15 +58,26 @@ private[spark] class KubernetesClusterSchedulerBackend(
 
   private val initialExecutors = SchedulerBackendUtils.getInitialTargetExecutorNumber(conf)
 
+  private val shouldDeleteDriverService = conf.get(KUBERNETES_DRIVER_SERVICE_DELETE_ON_TERMINATION)
+
   private val shouldDeleteExecutors = conf.get(KUBERNETES_DELETE_EXECUTORS)
 
   private val defaultProfile = scheduler.sc.resourceProfileManager.defaultResourceProfile
 
   // Allow removeExecutor to be accessible by ExecutorPodsLifecycleEventHandler
   private[k8s] def doRemoveExecutor(executorId: String, reason: ExecutorLossReason): Unit = {
-    if (isExecutorActive(executorId)) {
-      removeExecutor(executorId, reason)
-    }
+    removeExecutor(executorId, reason)
+  }
+
+  private def setUpExecutorConfigMap(driverPod: Option[Pod]): Unit = {
+    val configMapName = KubernetesClientUtils.configMapNameExecutor
+    val confFilesMap = KubernetesClientUtils
+      .buildSparkConfDirFilesMap(configMapName, conf, Map.empty)
+    val labels =
+      Map(SPARK_APP_ID_LABEL -> applicationId(), SPARK_ROLE_LABEL -> SPARK_POD_EXECUTOR_ROLE)
+    val configMap = KubernetesClientUtils.buildConfigMap(configMapName, confFilesMap, labels)
+    KubernetesUtils.addOwnerReference(driverPod.orNull, Seq(configMap))
+    kubernetesClient.configMaps().create(configMap)
   }
 
   /**
@@ -78,15 +93,23 @@ private[spark] class KubernetesClusterSchedulerBackend(
 
   override def start(): Unit = {
     super.start()
-    podAllocator.setTotalExpectedExecutors(initialExecutors)
+    val initExecs = Map(defaultProfile -> initialExecutors)
+    podAllocator.setTotalExpectedExecutors(initExecs)
     lifecycleEventHandler.start(this)
-    podAllocator.start(applicationId())
+    podAllocator.start(applicationId(), this)
     watchEvents.start(applicationId())
     pollEvents.start(applicationId())
+    if (!conf.get(KUBERNETES_EXECUTOR_DISABLE_CONFIGMAP)) {
+      setUpExecutorConfigMap(podAllocator.driverPod)
+    }
   }
 
   override def stop(): Unit = {
-    super.stop()
+    // When `CoarseGrainedSchedulerBackend.stop` throws `SparkException`,
+    // K8s cluster scheduler should log and proceed in order to delete the K8s cluster resources.
+    Utils.tryLogNonFatalError {
+      super.stop()
+    }
 
     Utils.tryLogNonFatalError {
       snapshotsStore.stop()
@@ -100,6 +123,22 @@ private[spark] class KubernetesClusterSchedulerBackend(
       pollEvents.stop()
     }
 
+    if (shouldDeleteDriverService) {
+      Utils.tryLogNonFatalError {
+        kubernetesClient
+          .services()
+          .withLabel(SPARK_APP_ID_LABEL, applicationId())
+          .delete()
+      }
+    }
+
+    Utils.tryLogNonFatalError {
+      kubernetesClient
+        .persistentVolumeClaims()
+        .withLabel(SPARK_APP_ID_LABEL, applicationId())
+        .delete()
+    }
+
     if (shouldDeleteExecutors) {
       Utils.tryLogNonFatalError {
         kubernetesClient
@@ -108,6 +147,15 @@ private[spark] class KubernetesClusterSchedulerBackend(
           .withLabel(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE)
           .delete()
       }
+      if (!conf.get(KUBERNETES_EXECUTOR_DISABLE_CONFIGMAP)) {
+        Utils.tryLogNonFatalError {
+          kubernetesClient
+            .configMaps()
+            .withLabel(SPARK_APP_ID_LABEL, applicationId())
+            .withLabel(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE)
+            .delete()
+        }
+      }
     }
 
     Utils.tryLogNonFatalError {
@@ -121,7 +169,7 @@ private[spark] class KubernetesClusterSchedulerBackend(
 
   override def doRequestTotalExecutors(
       resourceProfileToTotalExecs: Map[ResourceProfile, Int]): Future[Boolean] = {
-    podAllocator.setTotalExpectedExecutors(resourceProfileToTotalExecs(defaultProfile))
+    podAllocator.setTotalExpectedExecutors(resourceProfileToTotalExecs)
     Future.successful(true)
   }
 
@@ -185,11 +233,17 @@ private[spark] class KubernetesClusterSchedulerBackend(
     Some(new HadoopDelegationTokenManager(conf, sc.hadoopConfiguration, driverEndpoint))
   }
 
-  override protected def isBlacklisted(executorId: String, hostname: String): Boolean = {
+  override protected def isExecutorExcluded(executorId: String, hostname: String): Boolean = {
     podAllocator.isDeleted(executorId)
   }
 
   private class KubernetesDriverEndpoint extends DriverEndpoint {
+    private def ignoreRegisterExecutorAtStoppedContext: PartialFunction[Any, Unit] = {
+      case _: RegisterExecutor if sc.isStopped => // No-op
+    }
+
+    override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] =
+      ignoreRegisterExecutorAtStoppedContext.orElse(super.receiveAndReply(context))
 
     override def onDisconnected(rpcAddress: RpcAddress): Unit = {
       // Don't do anything besides disabling the executor - allow the Kubernetes API events to
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala
index 22bff2c807330..1a62d08a7b413 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala
@@ -16,38 +16,57 @@
  */
 package org.apache.spark.scheduler.cluster.k8s
 
-import java.io.File
-
 import io.fabric8.kubernetes.client.KubernetesClient
 
 import org.apache.spark.SecurityManager
 import org.apache.spark.deploy.k8s._
 import org.apache.spark.deploy.k8s.features._
+import org.apache.spark.resource.ResourceProfile
+import org.apache.spark.util.Utils
 
 private[spark] class KubernetesExecutorBuilder {
 
   def buildFromFeatures(
       conf: KubernetesExecutorConf,
       secMgr: SecurityManager,
-      client: KubernetesClient): SparkPod = {
+      client: KubernetesClient,
+      resourceProfile: ResourceProfile): KubernetesExecutorSpec = {
     val initialPod = conf.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE)
       .map { file =>
         KubernetesUtils.loadPodFromTemplate(
           client,
-          new File(file),
-          conf.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_CONTAINER_NAME))
+          file,
+          conf.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_CONTAINER_NAME),
+          conf.sparkConf)
       }
       .getOrElse(SparkPod.initialPod())
 
+    val userFeatures = conf.get(Config.KUBERNETES_EXECUTOR_POD_FEATURE_STEPS)
+      .map { className =>
+        Utils.classForName(className).newInstance().asInstanceOf[KubernetesFeatureConfigStep]
+      }
+
     val features = Seq(
-      new BasicExecutorFeatureStep(conf, secMgr),
+      new BasicExecutorFeatureStep(conf, secMgr, resourceProfile),
       new ExecutorKubernetesCredentialsFeatureStep(conf),
       new MountSecretsFeatureStep(conf),
       new EnvSecretsFeatureStep(conf),
       new MountVolumesFeatureStep(conf),
-      new LocalDirsFeatureStep(conf))
+      new LocalDirsFeatureStep(conf)) ++ userFeatures
+
+    val spec = KubernetesExecutorSpec(
+      initialPod,
+      executorKubernetesResources = Seq.empty)
 
-    features.foldLeft(initialPod) { case (pod, feature) => feature.configurePod(pod) }
+    // If using a template this will always get the resources from that and combine
+    // them with any Spark conf or ResourceProfile resources.
+    features.foldLeft(spec) { case (spec, feature) =>
+      val configuredPod = feature.configurePod(spec.pod)
+      val addedResources = feature.getAdditionalKubernetesResources()
+      KubernetesExecutorSpec(
+        configuredPod,
+        spec.executorKubernetesResources ++ addedResources)
+    }
   }
 
 }
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/Fabric8Aliases.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/Fabric8Aliases.scala
index 527fc6b0d8f87..792f3096eee3f 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/Fabric8Aliases.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/Fabric8Aliases.scala
@@ -16,15 +16,16 @@
  */
 package org.apache.spark.deploy.k8s
 
-import io.fabric8.kubernetes.api.model.{DoneablePod, HasMetadata, Pod, PodList}
-import io.fabric8.kubernetes.client.{Watch, Watcher}
-import io.fabric8.kubernetes.client.dsl.{FilterWatchListDeletable, MixedOperation, NamespaceListVisitFromServerGetDeleteRecreateWaitApplicable, PodResource}
+import io.fabric8.kubernetes.api.model.{ConfigMap, ConfigMapList, HasMetadata, Pod, PodList}
+import io.fabric8.kubernetes.client.dsl.{FilterWatchListDeletable, MixedOperation, NamespaceListVisitFromServerGetDeleteRecreateWaitApplicable, PodResource, Resource}
 
 object Fabric8Aliases {
-  type PODS = MixedOperation[Pod, PodList, DoneablePod, PodResource[Pod, DoneablePod]]
-  type LABELED_PODS = FilterWatchListDeletable[
-    Pod, PodList, java.lang.Boolean, Watch, Watcher[Pod]]
-  type SINGLE_POD = PodResource[Pod, DoneablePod]
+  type PODS = MixedOperation[Pod, PodList, PodResource[Pod]]
+  type CONFIG_MAPS = MixedOperation[
+    ConfigMap, ConfigMapList, Resource[ConfigMap]]
+  type LABELED_PODS = FilterWatchListDeletable[Pod, PodList]
+  type LABELED_CONFIG_MAPS = FilterWatchListDeletable[ConfigMap, ConfigMapList]
+  type SINGLE_POD = PodResource[Pod]
   type RESOURCE_LIST = NamespaceListVisitFromServerGetDeleteRecreateWaitApplicable[
-    HasMetadata, Boolean]
+    HasMetadata]
 }
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesConfSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesConfSuite.scala
index 1ca4dbc8674fb..0b973227fe0fd 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesConfSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesConfSuite.scala
@@ -23,6 +23,7 @@ import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.deploy.k8s.Config._
 import org.apache.spark.deploy.k8s.Constants._
 import org.apache.spark.deploy.k8s.submit._
+import org.apache.spark.resource.ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID
 
 class KubernetesConfSuite extends SparkFunSuite {
 
@@ -96,6 +97,17 @@ class KubernetesConfSuite extends SparkFunSuite {
       Some(DRIVER_POD))
     assert(conf.executorId === EXECUTOR_ID)
     assert(conf.driverPod.get === DRIVER_POD)
+    assert(conf.resourceProfileId === DEFAULT_RESOURCE_PROFILE_ID)
+  }
+
+  test("resource profile not default.") {
+    val conf = KubernetesConf.createExecutorConf(
+      new SparkConf(false),
+      EXECUTOR_ID,
+      KubernetesTestConf.APP_ID,
+      Some(DRIVER_POD),
+      10)
+    assert(conf.resourceProfileId === 10)
   }
 
   test("Image pull secrets.") {
@@ -134,7 +146,8 @@ class KubernetesConfSuite extends SparkFunSuite {
     assert(conf.labels === Map(
       SPARK_EXECUTOR_ID_LABEL -> EXECUTOR_ID,
       SPARK_APP_ID_LABEL -> KubernetesTestConf.APP_ID,
-      SPARK_ROLE_LABEL -> SPARK_POD_EXECUTOR_ROLE) ++ CUSTOM_LABELS)
+      SPARK_ROLE_LABEL -> SPARK_POD_EXECUTOR_ROLE,
+      SPARK_RESOURCE_PROFILE_ID_LABEL -> DEFAULT_RESOURCE_PROFILE_ID.toString) ++ CUSTOM_LABELS)
     assert(conf.annotations === CUSTOM_ANNOTATIONS)
     assert(conf.secretNamesToMountPaths === SECRET_NAMES_TO_MOUNT_PATHS)
     assert(conf.secretEnvNamesToKeyRefs === SECRET_ENV_VARS)
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesTestConf.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesTestConf.scala
index d6871a6c2866a..0567f32c23134 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesTestConf.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesTestConf.scala
@@ -21,7 +21,6 @@ import io.fabric8.kubernetes.api.model.Pod
 
 import org.apache.spark.SparkConf
 import org.apache.spark.deploy.k8s.Config._
-import org.apache.spark.deploy.k8s.Constants._
 import org.apache.spark.deploy.k8s.submit.{JavaMainAppResource, MainAppResource}
 
 /**
@@ -114,9 +113,12 @@ object KubernetesTestConf {
           (KUBERNETES_VOLUMES_HOSTPATH_TYPE,
             Map(KUBERNETES_VOLUMES_OPTIONS_PATH_KEY -> path))
 
-        case KubernetesPVCVolumeConf(claimName) =>
+        case KubernetesPVCVolumeConf(claimName, storageClass, sizeLimit) =>
+          val sconf = storageClass
+            .map { s => (KUBERNETES_VOLUMES_OPTIONS_CLAIM_STORAGE_CLASS_KEY, s) }.toMap
+          val lconf = sizeLimit.map { l => (KUBERNETES_VOLUMES_OPTIONS_SIZE_LIMIT_KEY, l) }.toMap
           (KUBERNETES_VOLUMES_PVC_TYPE,
-            Map(KUBERNETES_VOLUMES_OPTIONS_CLAIM_NAME_KEY -> claimName))
+            Map(KUBERNETES_VOLUMES_OPTIONS_CLAIM_NAME_KEY -> claimName) ++ sconf ++ lconf)
 
         case KubernetesEmptyDirVolumeConf(medium, sizeLimit) =>
           val mconf = medium.map { m => (KUBERNETES_VOLUMES_OPTIONS_MEDIUM_KEY, m) }.toMap
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesUtilsSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesUtilsSuite.scala
index 7c231586af935..ef57a4b861508 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesUtilsSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesUtilsSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.deploy.k8s
 
 import scala.collection.JavaConverters._
 
-import io.fabric8.kubernetes.api.model.{Container, ContainerBuilder, PodBuilder}
+import io.fabric8.kubernetes.api.model.{ContainerBuilder, PodBuilder}
 
 import org.apache.spark.SparkFunSuite
 
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesVolumeUtilsSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesVolumeUtilsSuite.scala
index 6596c5e2ad2e7..156740d7c8aee 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesVolumeUtilsSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesVolumeUtilsSuite.scala
@@ -49,14 +49,14 @@ class KubernetesVolumeUtilsSuite extends SparkFunSuite {
     val sparkConf = new SparkConf(false)
     sparkConf.set("test.persistentVolumeClaim.volumeName.mount.path", "/path")
     sparkConf.set("test.persistentVolumeClaim.volumeName.mount.readOnly", "true")
-    sparkConf.set("test.persistentVolumeClaim.volumeName.options.claimName", "claimeName")
+    sparkConf.set("test.persistentVolumeClaim.volumeName.options.claimName", "claimName")
 
     val volumeSpec = KubernetesVolumeUtils.parseVolumesWithPrefix(sparkConf, "test.").head
     assert(volumeSpec.volumeName === "volumeName")
     assert(volumeSpec.mountPath === "/path")
     assert(volumeSpec.mountReadOnly)
     assert(volumeSpec.volumeConf.asInstanceOf[KubernetesPVCVolumeConf] ===
-      KubernetesPVCVolumeConf("claimeName"))
+      KubernetesPVCVolumeConf("claimName"))
   }
 
   test("Parses emptyDir volumes correctly") {
@@ -118,6 +118,17 @@ class KubernetesVolumeUtilsSuite extends SparkFunSuite {
     assert(e.getMessage.contains("hostPath.volumeName.options.path"))
   }
 
+  test("SPARK-33063: Fails on missing option key in persistentVolumeClaim") {
+    val sparkConf = new SparkConf(false)
+    sparkConf.set("test.persistentVolumeClaim.volumeName.mount.path", "/path")
+    sparkConf.set("test.persistentVolumeClaim.volumeName.mount.readOnly", "true")
+
+    val e = intercept[NoSuchElementException] {
+      KubernetesVolumeUtils.parseVolumesWithPrefix(sparkConf, "test.")
+    }
+    assert(e.getMessage.contains("persistentVolumeClaim.volumeName.options.claimName"))
+  }
+
   test("Parses read-only nfs volumes correctly") {
     val sparkConf = new SparkConf(false)
     sparkConf.set("test.nfs.volumeName.mount.path", "/path")
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/PodBuilderSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/PodBuilderSuite.scala
index 26bd317de8ec6..2051c932e5393 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/PodBuilderSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/PodBuilderSuite.scala
@@ -26,13 +26,15 @@ import org.mockito.Mockito.{mock, never, verify, when}
 import scala.collection.JavaConverters._
 
 import org.apache.spark.{SparkConf, SparkException, SparkFunSuite}
-import org.apache.spark.deploy.k8s._
+import org.apache.spark.deploy.k8s.features.KubernetesFeatureConfigStep
 import org.apache.spark.internal.config.ConfigEntry
 
 abstract class PodBuilderSuite extends SparkFunSuite {
 
   protected def templateFileConf: ConfigEntry[_]
 
+  protected def userFeatureStepsConf: ConfigEntry[_]
+
   protected def buildPod(sparkConf: SparkConf, client: KubernetesClient): SparkPod
 
   private val baseConf = new SparkConf(false)
@@ -51,6 +53,19 @@ abstract class PodBuilderSuite extends SparkFunSuite {
     verifyPod(pod)
   }
 
+  test("configure a custom test step") {
+    val client = mockKubernetesClient()
+    val sparkConf = baseConf.clone()
+      .set(userFeatureStepsConf.key,
+        "org.apache.spark.deploy.k8s.TestStepTwo," +
+        "org.apache.spark.deploy.k8s.TestStep")
+      .set(templateFileConf.key, "template-file.yaml")
+    val pod = buildPod(sparkConf, client)
+    verifyPod(pod)
+    assert(pod.container.getVolumeMounts.asScala.exists(_.getName == "so_long"))
+    assert(pod.container.getVolumeMounts.asScala.exists(_.getName == "so_long_two"))
+  }
+
   test("complain about misconfigured pod template") {
     val client = mockKubernetesClient(
       new PodBuilder()
@@ -68,8 +83,8 @@ abstract class PodBuilderSuite extends SparkFunSuite {
   private def mockKubernetesClient(pod: Pod = podWithSupportedFeatures()): KubernetesClient = {
     val kubernetesClient = mock(classOf[KubernetesClient])
     val pods =
-      mock(classOf[MixedOperation[Pod, PodList, DoneablePod, PodResource[Pod, DoneablePod]]])
-    val podResource = mock(classOf[PodResource[Pod, DoneablePod]])
+      mock(classOf[MixedOperation[Pod, PodList, PodResource[Pod]]])
+    val podResource = mock(classOf[PodResource[Pod]])
     when(kubernetesClient.pods()).thenReturn(pods)
     when(pods.load(any(classOf[File]))).thenReturn(podResource)
     when(podResource.get()).thenReturn(pod)
@@ -174,3 +189,63 @@ abstract class PodBuilderSuite extends SparkFunSuite {
   }
 
 }
+
+/**
+ * A test user feature step.
+ */
+class TestStep extends KubernetesFeatureConfigStep {
+  import io.fabric8.kubernetes.api.model._
+
+  override def configurePod(pod: SparkPod): SparkPod = {
+    val localDirVolumes = Seq(new VolumeBuilder().withName("so_long").build())
+    val localDirVolumeMounts = Seq(
+      new VolumeMountBuilder().withName("so_long")
+        .withMountPath("and_thanks_for_all_the_fish")
+        .build()
+    )
+
+    val podWithLocalDirVolumes = new PodBuilder(pod.pod)
+      .editSpec()
+        .addToVolumes(localDirVolumes: _*)
+        .endSpec()
+      .build()
+    val containerWithLocalDirVolumeMounts = new ContainerBuilder(pod.container)
+      .addNewEnv()
+        .withName("CUSTOM_SPARK_LOCAL_DIRS")
+        .withValue("fishyfishyfishy")
+        .endEnv()
+      .addToVolumeMounts(localDirVolumeMounts: _*)
+      .build()
+    SparkPod(podWithLocalDirVolumes, containerWithLocalDirVolumeMounts)
+  }
+}
+
+/**
+ * A test user feature step.
+ */
+class TestStepTwo extends KubernetesFeatureConfigStep {
+  import io.fabric8.kubernetes.api.model._
+
+  override def configurePod(pod: SparkPod): SparkPod = {
+    val localDirVolumes = Seq(new VolumeBuilder().withName("so_long_two").build())
+    val localDirVolumeMounts = Seq(
+      new VolumeMountBuilder().withName("so_long_two")
+        .withMountPath("and_thanks_for_all_the_fish_eh")
+        .build()
+    )
+
+    val podWithLocalDirVolumes = new PodBuilder(pod.pod)
+      .editSpec()
+        .addToVolumes(localDirVolumes: _*)
+        .endSpec()
+      .build()
+    val containerWithLocalDirVolumeMounts = new ContainerBuilder(pod.container)
+      .addNewEnv()
+        .withName("CUSTOM_SPARK_LOCAL_DIRS_TWO")
+        .withValue("fishyfishyfishyTWO")
+        .endEnv()
+      .addToVolumeMounts(localDirVolumeMounts: _*)
+      .build()
+    SparkPod(podWithLocalDirVolumes, containerWithLocalDirVolumeMounts)
+  }
+}
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
index c8c934b436ffa..858b4f1494b8e 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
@@ -28,7 +28,7 @@ import org.apache.spark.deploy.k8s.features.KubernetesFeaturesTestUtils.TestReso
 import org.apache.spark.deploy.k8s.submit._
 import org.apache.spark.internal.config._
 import org.apache.spark.internal.config.UI._
-import org.apache.spark.resource.ResourceID
+import org.apache.spark.resource.{ResourceID, ResourceProfile}
 import org.apache.spark.resource.ResourceUtils._
 import org.apache.spark.util.Utils
 
@@ -191,7 +191,8 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
   ).foreach { case (name, resource, factor, expectedFactor) =>
     test(s"memory overhead factor: $name") {
       // Choose a driver memory where the default memory overhead is > MEMORY_OVERHEAD_MIN_MIB
-      val driverMem = MEMORY_OVERHEAD_MIN_MIB / MEMORY_OVERHEAD_FACTOR.defaultValue.get * 2
+      val driverMem =
+        ResourceProfile.MEMORY_OVERHEAD_MIN_MIB / MEMORY_OVERHEAD_FACTOR.defaultValue.get * 2
 
       // main app resource, overhead factor
       val sparkConf = new SparkConf(false)
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
index da50372d04c73..82cbed7509b95 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
@@ -27,14 +27,14 @@ import io.fabric8.kubernetes.api.model._
 import org.scalatest.BeforeAndAfter
 
 import org.apache.spark.{SecurityManager, SparkConf, SparkException, SparkFunSuite}
-import org.apache.spark.deploy.k8s.{KubernetesExecutorConf, KubernetesTestConf, SparkPod}
+import org.apache.spark.deploy.k8s.{KubernetesExecutorConf, KubernetesTestConf, SecretVolumeUtils, SparkPod}
 import org.apache.spark.deploy.k8s.Config._
 import org.apache.spark.deploy.k8s.Constants._
 import org.apache.spark.deploy.k8s.features.KubernetesFeaturesTestUtils.TestResourceInformation
 import org.apache.spark.internal.config
 import org.apache.spark.internal.config._
 import org.apache.spark.internal.config.Python._
-import org.apache.spark.resource.ResourceID
+import org.apache.spark.resource._
 import org.apache.spark.resource.ResourceUtils._
 import org.apache.spark.resource.TestResourceIDs._
 import org.apache.spark.rpc.RpcEndpointAddress
@@ -55,6 +55,7 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
   private val RESOURCE_NAME_PREFIX = "base"
   private val EXECUTOR_IMAGE = "executor-image"
   private val LABELS = Map("label1key" -> "label1value")
+  private var defaultProfile: ResourceProfile = _
   private val TEST_IMAGE_PULL_SECRETS = Seq("my-1secret-1", "my-secret-2")
   private val TEST_IMAGE_PULL_SECRET_OBJECTS =
     TEST_IMAGE_PULL_SECRETS.map { secret =>
@@ -84,6 +85,7 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
       .set(config.DRIVER_PORT, DRIVER_PORT)
       .set(IMAGE_PULL_SECRETS, TEST_IMAGE_PULL_SECRETS)
       .set("spark.kubernetes.resource.type", "java")
+    initDefaultProfile(baseConf)
   }
 
   private def newExecutorConf(
@@ -95,10 +97,17 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
       environment = environment)
   }
 
+  private def initDefaultProfile(baseConf: SparkConf): Unit = {
+    ResourceProfile.clearDefaultProfile()
+    defaultProfile = ResourceProfile.getOrCreateDefaultProfile(baseConf)
+  }
+
   test("test spark resource missing vendor") {
     baseConf.set(EXECUTOR_GPU_ID.amountConf, "2")
-    val step = new BasicExecutorFeatureStep(newExecutorConf(), new SecurityManager(baseConf))
     val error = intercept[SparkException] {
+      initDefaultProfile(baseConf)
+      val step = new BasicExecutorFeatureStep(newExecutorConf(), new SecurityManager(baseConf),
+        defaultProfile)
       val executor = step.configurePod(SparkPod.initialPod())
     }.getMessage()
     assert(error.contains("Resource: gpu was requested, but vendor was not specified"))
@@ -106,9 +115,10 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
 
   test("test spark resource missing amount") {
     baseConf.set(EXECUTOR_GPU_ID.vendorConf, "nvidia.com")
-
-    val step = new BasicExecutorFeatureStep(newExecutorConf(), new SecurityManager(baseConf))
     val error = intercept[SparkException] {
+      initDefaultProfile(baseConf)
+      val step = new BasicExecutorFeatureStep(newExecutorConf(), new SecurityManager(baseConf),
+      defaultProfile)
       val executor = step.configurePod(SparkPod.initialPod())
     }.getMessage()
     assert(error.contains("You must specify an amount for gpu"))
@@ -124,7 +134,9 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
       baseConf.set(testRInfo.rId.amountConf, testRInfo.count)
       baseConf.set(testRInfo.rId.vendorConf, testRInfo.vendor)
     }
-    val step = new BasicExecutorFeatureStep(newExecutorConf(), new SecurityManager(baseConf))
+    initDefaultProfile(baseConf)
+    val step = new BasicExecutorFeatureStep(newExecutorConf(), new SecurityManager(baseConf),
+      defaultProfile)
     val executor = step.configurePod(SparkPod.initialPod())
 
     assert(executor.container.getResources.getLimits.size() === 3)
@@ -137,7 +149,8 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
   }
 
   test("basic executor pod has reasonable defaults") {
-    val step = new BasicExecutorFeatureStep(newExecutorConf(), new SecurityManager(baseConf))
+    val step = new BasicExecutorFeatureStep(newExecutorConf(), new SecurityManager(baseConf),
+      defaultProfile)
     val executor = step.configurePod(SparkPod.initialPod())
 
     // The executor pod name and default labels.
@@ -147,17 +160,17 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
     }
     assert(executor.pod.getSpec.getImagePullSecrets.asScala === TEST_IMAGE_PULL_SECRET_OBJECTS)
 
-    // There is exactly 1 container with no volume mounts and default memory limits.
+    // There is exactly 1 container with 1 volume mount and default memory limits.
     // Default memory limit is 1024M + 384M (minimum overhead constant).
     assert(executor.container.getImage === EXECUTOR_IMAGE)
-    assert(executor.container.getVolumeMounts.isEmpty)
+    assert(executor.container.getVolumeMounts.size() == 1)
     assert(executor.container.getResources.getLimits.size() === 1)
     assert(amountAndFormat(executor.container.getResources
       .getLimits.get("memory")) === "1408Mi")
 
-    // The pod has no node selector, volumes.
+    // The pod has no node selector, and 1 volume.
     assert(executor.pod.getSpec.getNodeSelector.isEmpty)
-    assert(executor.pod.getSpec.getVolumes.isEmpty)
+    assert(executor.pod.getSpec.getVolumes.size() == 1)
 
     checkEnv(executor, baseConf, Map())
     checkOwnerReferences(executor.pod, DRIVER_POD_UID)
@@ -167,7 +180,9 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
     val longPodNamePrefix = "loremipsumdolorsitametvimatelitrefficiendisuscipianturvixlegeresple"
 
     baseConf.set(KUBERNETES_EXECUTOR_POD_NAME_PREFIX, longPodNamePrefix)
-    val step = new BasicExecutorFeatureStep(newExecutorConf(), new SecurityManager(baseConf))
+    initDefaultProfile(baseConf)
+    val step = new BasicExecutorFeatureStep(newExecutorConf(), new SecurityManager(baseConf),
+      defaultProfile)
     assert(step.configurePod(SparkPod.initialPod()).pod.getSpec.getHostname.length === 63)
   }
 
@@ -175,7 +190,9 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
     val invalidPrefix = "abcdef-*_/[]{}+==.,;'\"-----------------------------------------------"
 
     baseConf.set(KUBERNETES_EXECUTOR_POD_NAME_PREFIX, invalidPrefix)
-    val step = new BasicExecutorFeatureStep(newExecutorConf(), new SecurityManager(baseConf))
+    initDefaultProfile(baseConf)
+    val step = new BasicExecutorFeatureStep(newExecutorConf(), new SecurityManager(baseConf),
+      defaultProfile)
     val hostname = step.configurePod(SparkPod.initialPod()).pod.getSpec().getHostname()
     assert(hostname.length <= 63)
     assert(InternetDomainName.isValid(hostname))
@@ -184,8 +201,10 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
   test("classpath and extra java options get translated into environment variables") {
     baseConf.set(config.EXECUTOR_JAVA_OPTIONS, "foo=bar")
     baseConf.set(config.EXECUTOR_CLASS_PATH, "bar=baz")
+    initDefaultProfile(baseConf)
     val kconf = newExecutorConf(environment = Map("qux" -> "quux"))
-    val step = new BasicExecutorFeatureStep(kconf, new SecurityManager(baseConf))
+    val step = new BasicExecutorFeatureStep(kconf, new SecurityManager(baseConf),
+      defaultProfile)
     val executor = step.configurePod(SparkPod.initialPod())
 
     checkEnv(executor, baseConf,
@@ -195,11 +214,23 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
     checkOwnerReferences(executor.pod, DRIVER_POD_UID)
   }
 
+  test("SPARK-32655 Support appId/execId placeholder in SPARK_EXECUTOR_DIRS") {
+    val kconf = newExecutorConf(environment = Map(ENV_EXECUTOR_DIRS ->
+      "/p1/SPARK_APPLICATION_ID/SPARK_EXECUTOR_ID,/p2/SPARK_APPLICATION_ID/SPARK_EXECUTOR_ID"))
+    val step = new BasicExecutorFeatureStep(kconf, new SecurityManager(baseConf),
+      defaultProfile)
+    val executor = step.configurePod(SparkPod.initialPod())
+
+    checkEnv(executor, baseConf, Map(ENV_EXECUTOR_DIRS ->
+      s"/p1/${KubernetesTestConf.APP_ID}/1,/p2/${KubernetesTestConf.APP_ID}/1"))
+  }
+
   test("test executor pyspark memory") {
     baseConf.set("spark.kubernetes.resource.type", "python")
     baseConf.set(PYSPARK_EXECUTOR_MEMORY, 42L)
-
-    val step = new BasicExecutorFeatureStep(newExecutorConf(), new SecurityManager(baseConf))
+    initDefaultProfile(baseConf)
+    val step = new BasicExecutorFeatureStep(newExecutorConf(), new SecurityManager(baseConf),
+      defaultProfile)
     val executor = step.configurePod(SparkPod.initialPod())
     // This is checking that basic executor + executorMemory = 1408 + 42 = 1450
     assert(amountAndFormat(executor.container.getResources.getRequests.get("memory")) === "1450Mi")
@@ -214,7 +245,7 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
     secMgr.initializeAuth()
 
     val step = new BasicExecutorFeatureStep(KubernetesTestConf.createExecutorConf(sparkConf = conf),
-      secMgr)
+      secMgr, defaultProfile)
 
     val executor = step.configurePod(SparkPod.initialPod())
     checkEnv(executor, conf, Map(SecurityManager.ENV_AUTH_SECRET -> secMgr.getSecretKey()))
@@ -230,15 +261,86 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
       .set("spark.master", "k8s://127.0.0.1")
     val secMgr = new SecurityManager(conf)
     secMgr.initializeAuth()
-
     val step = new BasicExecutorFeatureStep(KubernetesTestConf.createExecutorConf(sparkConf = conf),
-      secMgr)
+      secMgr, defaultProfile)
 
     val executor = step.configurePod(SparkPod.initialPod())
     assert(!KubernetesFeaturesTestUtils.containerHasEnvVar(
       executor.container, SecurityManager.ENV_AUTH_SECRET))
   }
 
+  test("SPARK-32661 test executor offheap memory") {
+    baseConf.set(MEMORY_OFFHEAP_ENABLED, true)
+    baseConf.set("spark.memory.offHeap.size", "42m")
+    initDefaultProfile(baseConf)
+
+    val step = new BasicExecutorFeatureStep(newExecutorConf(), new SecurityManager(baseConf),
+      defaultProfile)
+    val executor = step.configurePod(SparkPod.initialPod())
+    // This is checking that basic executor + executorMemory = 1408 + 42 = 1450
+    assert(amountAndFormat(executor.container.getResources.getRequests.get("memory")) === "1450Mi")
+  }
+
+  test("basic resourceprofile") {
+    baseConf.set("spark.kubernetes.resource.type", "python")
+    initDefaultProfile(baseConf)
+    val rpb = new ResourceProfileBuilder()
+    val ereq = new ExecutorResourceRequests()
+    val treq = new TaskResourceRequests()
+    ereq.cores(4).memory("2g").memoryOverhead("1g").pysparkMemory("3g")
+    treq.cpus(2)
+    rpb.require(ereq).require(treq)
+    val rp = rpb.build()
+    val step = new BasicExecutorFeatureStep(newExecutorConf(), new SecurityManager(baseConf), rp)
+    val executor = step.configurePod(SparkPod.initialPod())
+
+    assert(amountAndFormat(executor.container.getResources
+      .getRequests.get("cpu")) === "4")
+    assert(amountAndFormat(executor.container.getResources
+      .getLimits.get("memory")) === "6144Mi")
+  }
+
+  test("resourceprofile with gpus") {
+    val rpb = new ResourceProfileBuilder()
+    val ereq = new ExecutorResourceRequests()
+    val treq = new TaskResourceRequests()
+    ereq.cores(2).resource("gpu", 2, "/path/getGpusResources.sh", "nvidia.com")
+    treq.cpus(1)
+    rpb.require(ereq).require(treq)
+    val rp = rpb.build()
+    val step = new BasicExecutorFeatureStep(newExecutorConf(), new SecurityManager(baseConf), rp)
+    val executor = step.configurePod(SparkPod.initialPod())
+
+    assert(amountAndFormat(executor.container.getResources
+      .getLimits.get("memory")) === "1408Mi")
+    assert(amountAndFormat(executor.container.getResources
+      .getRequests.get("cpu")) === "2")
+
+    assert(executor.container.getResources.getLimits.size() === 2)
+    assert(amountAndFormat(executor.container.getResources.getLimits.get("nvidia.com/gpu")) === "2")
+  }
+
+  test("Verify spark conf dir is mounted as configmap volume on executor pod's container.") {
+    val baseDriverPod = SparkPod.initialPod()
+    val step = new BasicExecutorFeatureStep(newExecutorConf(), new SecurityManager(baseConf),
+      defaultProfile)
+    val podConfigured = step.configurePod(baseDriverPod)
+    assert(SecretVolumeUtils.containerHasVolume(podConfigured.container,
+      SPARK_CONF_VOLUME_EXEC, SPARK_CONF_DIR_INTERNAL))
+    assert(SecretVolumeUtils.podHasVolume(podConfigured.pod, SPARK_CONF_VOLUME_EXEC))
+  }
+
+  test("SPARK-34316 Disable configmap volume on executor pod's container") {
+    baseConf.set(KUBERNETES_EXECUTOR_DISABLE_CONFIGMAP, true)
+    val baseDriverPod = SparkPod.initialPod()
+    val step = new BasicExecutorFeatureStep(newExecutorConf(), new SecurityManager(baseConf),
+      defaultProfile)
+    val podConfigured = step.configurePod(baseDriverPod)
+    assert(!SecretVolumeUtils.containerHasVolume(podConfigured.container,
+      SPARK_CONF_VOLUME_EXEC, SPARK_CONF_DIR_INTERNAL))
+    assert(!SecretVolumeUtils.podHasVolume(podConfigured.pod, SPARK_CONF_VOLUME_EXEC))
+  }
+
   // There is always exactly one controller reference, and it points to the driver pod.
   private def checkOwnerReferences(executor: Pod, driverPodUid: String): Unit = {
     assert(executor.getMetadata.getOwnerReferences.size() === 1)
@@ -255,11 +357,12 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
       ENV_EXECUTOR_ID -> "1",
       ENV_DRIVER_URL -> DRIVER_ADDRESS.toString,
       ENV_EXECUTOR_CORES -> "1",
-      ENV_EXECUTOR_MEMORY -> "1g",
+      ENV_EXECUTOR_MEMORY -> "1024m",
       ENV_APPLICATION_ID -> KubernetesTestConf.APP_ID,
       ENV_SPARK_CONF_DIR -> SPARK_CONF_DIR_INTERNAL,
       ENV_EXECUTOR_POD_IP -> null,
-      ENV_SPARK_USER -> Utils.getCurrentUserName())
+      ENV_SPARK_USER -> Utils.getCurrentUserName(),
+      ENV_RESOURCE_PROFILE_ID -> "0")
 
     val extraJavaOptsStart = additionalEnvVars.keys.count(_.startsWith(ENV_JAVA_OPT_PREFIX))
     val extraJavaOpts = Utils.sparkJavaOpts(conf, SparkConf.isExecutorStartupConf)
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStepSuite.scala
index 829943f16beac..ebbb42f225c51 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStepSuite.scala
@@ -20,11 +20,9 @@ import scala.collection.JavaConverters._
 
 import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.deploy.k8s._
-import org.apache.spark.deploy.k8s.Config._
 import org.apache.spark.deploy.k8s.Constants._
 import org.apache.spark.deploy.k8s.submit._
-import org.apache.spark.internal.config._
-import org.apache.spark.util.Utils
+import org.apache.spark.internal.config.{PYSPARK_DRIVER_PYTHON, PYSPARK_PYTHON}
 
 class DriverCommandFeatureStepSuite extends SparkFunSuite {
 
@@ -43,7 +41,6 @@ class DriverCommandFeatureStepSuite extends SparkFunSuite {
   test("python resource") {
     val mainResource = "local:/main.py"
     val sparkConf = new SparkConf(false)
-      .set(PYSPARK_MAJOR_PYTHON_VERSION, "2")
     val spec = applyFeatureStep(
       PythonMainAppResource(mainResource),
       conf = sparkConf,
@@ -54,12 +51,51 @@ class DriverCommandFeatureStepSuite extends SparkFunSuite {
       "--properties-file", SPARK_CONF_PATH,
       "--class", KubernetesTestConf.MAIN_CLASS,
       mainResource, "5", "7", "9"))
+  }
+
+  test("python executable precedence") {
+    val mainResource = "local:/main.py"
 
-    val envs = spec.pod.container.getEnv.asScala
-      .map { env => (env.getName, env.getValue) }
-      .toMap
-    val expected = Map(ENV_PYSPARK_MAJOR_PYTHON_VERSION -> "2")
-    assert(envs === expected)
+    val pythonExecutables = Seq(
+      (Some("conf_py"), Some("conf_driver_py"), Some("env_py"), Some("env_driver_py")),
+      (Some("conf_py"), None, Some("env_py"), Some("env_driver_py")),
+      (None, None, Some("env_py"), Some("env_driver_py")),
+      (None, None, Some("env_py"), None)
+    )
+
+    val expectedResults = Seq(
+      ("conf_py", "conf_driver_py"),
+      ("conf_py", "conf_py"),
+      ("env_py", "env_driver_py"),
+      ("env_py", "env_py")
+    )
+
+    pythonExecutables.zip(expectedResults).foreach { case (pythonExecutable, expected) =>
+      val sparkConf = new SparkConf(false)
+      val (confPy, confDriverPy, envPy, envDriverPy) = pythonExecutable
+      confPy.foreach(sparkConf.set(PYSPARK_PYTHON, _))
+      confDriverPy.foreach(sparkConf.set(PYSPARK_DRIVER_PYTHON, _))
+      val pythonEnvs = Map(
+        (
+          envPy.map(v => ENV_PYSPARK_PYTHON -> v :: Nil) ++
+          envDriverPy.map(v => ENV_PYSPARK_DRIVER_PYTHON -> v :: Nil)
+        ).flatten.toArray: _*)
+
+      val spec = applyFeatureStep(
+        PythonMainAppResource(mainResource),
+        conf = sparkConf,
+        appArgs = Array("foo"),
+        env = pythonEnvs)
+
+      val envs = spec.pod.container.getEnv.asScala
+        .map { env => (env.getName, env.getValue) }
+        .toMap
+
+      val (expectedEnvPy, expectedDriverPy) = expected
+      assert(envs === Map(
+        ENV_PYSPARK_PYTHON -> expectedEnvPy,
+        ENV_PYSPARK_DRIVER_PYTHON -> expectedDriverPy))
+    }
   }
 
   test("R resource") {
@@ -93,7 +129,6 @@ class DriverCommandFeatureStepSuite extends SparkFunSuite {
   test("SPARK-25355: python resource args with proxy-user") {
     val mainResource = "local:/main.py"
     val sparkConf = new SparkConf(false)
-      .set(PYSPARK_MAJOR_PYTHON_VERSION, "2")
     val spec = applyFeatureStep(
       PythonMainAppResource(mainResource),
       conf = sparkConf,
@@ -128,13 +163,16 @@ class DriverCommandFeatureStepSuite extends SparkFunSuite {
       resource: MainAppResource,
       conf: SparkConf = new SparkConf(false),
       appArgs: Array[String] = Array(),
-      proxyUser: Option[String] = None): KubernetesDriverSpec = {
+      proxyUser: Option[String] = None,
+      env: Map[String, String] = Map.empty[String, String]): KubernetesDriverSpec = {
     val kubernetesConf = KubernetesTestConf.createDriverConf(
       sparkConf = conf,
       mainAppResource = resource,
       appArgs = appArgs,
       proxyUser = proxyUser)
-    val step = new DriverCommandFeatureStep(kubernetesConf)
+    val step = new DriverCommandFeatureStep(kubernetesConf) {
+      private[spark] override val environmentVariables: Map[String, String] = env
+    }
     val pod = step.configurePod(SparkPod.initialPod())
     val props = step.getAdditionalPodSystemProperties()
     KubernetesDriverSpec(pod, Nil, props)
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStepSuite.scala
index 18afd10395566..2e5e60eb39f35 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStepSuite.scala
@@ -25,7 +25,6 @@ import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.deploy.k8s.{KubernetesTestConf, SparkPod}
 import org.apache.spark.deploy.k8s.Config._
 import org.apache.spark.deploy.k8s.Constants._
-import org.apache.spark.deploy.k8s.submit.JavaMainAppResource
 import org.apache.spark.internal.config._
 import org.apache.spark.internal.config.UI._
 import org.apache.spark.util.ManualClock
@@ -64,6 +63,7 @@ class DriverServiceFeatureStepSuite extends SparkFunSuite {
       8080,
       4080,
       s"${kconf.resourceNamePrefix}${DriverServiceFeatureStep.DRIVER_SVC_POSTFIX}",
+      kconf.appId,
       driverService)
   }
 
@@ -96,6 +96,7 @@ class DriverServiceFeatureStepSuite extends SparkFunSuite {
       DEFAULT_BLOCKMANAGER_PORT,
       UI_PORT.defaultValue.get,
       s"${kconf.resourceNamePrefix}${DriverServiceFeatureStep.DRIVER_SVC_POSTFIX}",
+      kconf.appId,
       resolvedService)
     val additionalProps = configurationStep.getAdditionalPodSystemProperties()
     assert(additionalProps(DRIVER_PORT.key) === DEFAULT_DRIVER_PORT.toString)
@@ -164,8 +165,11 @@ class DriverServiceFeatureStepSuite extends SparkFunSuite {
       blockManagerPort: Int,
       drierUIPort: Int,
       expectedServiceName: String,
+      appId: String,
       service: Service): Unit = {
     assert(service.getMetadata.getName === expectedServiceName)
+    assert(service.getMetadata.getLabels.containsKey(SPARK_APP_ID_LABEL) &&
+      service.getMetadata.getLabels.get(SPARK_APP_ID_LABEL).equals(appId))
     assert(service.getSpec.getClusterIP === "None")
     DRIVER_LABELS.foreach { case (k, v) =>
       assert(service.getSpec.getSelector.get(k) === v)
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/HadoopConfDriverFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/HadoopConfDriverFeatureStepSuite.scala
index e1c01dbdc7358..c078e69b8a14b 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/HadoopConfDriverFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/HadoopConfDriverFeatureStepSuite.scala
@@ -27,7 +27,6 @@ import io.fabric8.kubernetes.api.model.ConfigMap
 import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.deploy.k8s._
 import org.apache.spark.deploy.k8s.Constants._
-import org.apache.spark.deploy.k8s.submit.JavaMainAppResource
 import org.apache.spark.util.{SparkConfWithEnv, Utils}
 
 class HadoopConfDriverFeatureStepSuite extends SparkFunSuite {
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/KerberosConfDriverFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/KerberosConfDriverFeatureStepSuite.scala
index 41ca3a94ce7a7..094fcb39782f4 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/KerberosConfDriverFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/KerberosConfDriverFeatureStepSuite.scala
@@ -26,14 +26,13 @@ import com.google.common.io.Files
 import io.fabric8.kubernetes.api.model.{ConfigMap, Secret}
 import org.apache.commons.codec.binary.Base64
 import org.apache.hadoop.io.Text
-import org.apache.hadoop.security.{Credentials, UserGroupInformation}
+import org.apache.hadoop.security.UserGroupInformation
 
 import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.deploy.SparkHadoopUtil
 import org.apache.spark.deploy.k8s._
 import org.apache.spark.deploy.k8s.Config._
 import org.apache.spark.deploy.k8s.Constants._
-import org.apache.spark.deploy.k8s.submit.JavaMainAppResource
 import org.apache.spark.internal.config._
 import org.apache.spark.util.Utils
 
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStepSuite.scala
index 0d0ed50c0927a..38f8fac1858f1 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStepSuite.scala
@@ -18,7 +18,7 @@ package org.apache.spark.deploy.k8s.features
 
 import scala.collection.JavaConverters._
 
-import org.apache.spark.{SparkConf, SparkFunSuite}
+import org.apache.spark.SparkFunSuite
 import org.apache.spark.deploy.k8s._
 
 class MountVolumesFeatureStepSuite extends SparkFunSuite {
@@ -42,7 +42,7 @@ class MountVolumesFeatureStepSuite extends SparkFunSuite {
     assert(configuredPod.container.getVolumeMounts.get(0).getReadOnly === false)
   }
 
-  test("Mounts pesistentVolumeClaims") {
+  test("Mounts persistentVolumeClaims") {
     val volumeConf = KubernetesVolumeSpec(
       "testVolume",
       "/tmp",
@@ -64,6 +64,65 @@ class MountVolumesFeatureStepSuite extends SparkFunSuite {
 
   }
 
+  test("SPARK-32713 Mounts parameterized persistentVolumeClaims in executors") {
+    val volumeConf = KubernetesVolumeSpec(
+      "testVolume",
+      "/tmp",
+      "",
+      true,
+      KubernetesPVCVolumeConf("pvc-spark-SPARK_EXECUTOR_ID")
+    )
+    val driverConf = KubernetesTestConf.createDriverConf(volumes = Seq(volumeConf))
+    val driverStep = new MountVolumesFeatureStep(driverConf)
+    val driverPod = driverStep.configurePod(SparkPod.initialPod())
+
+    assert(driverPod.pod.getSpec.getVolumes.size() === 1)
+    val driverPVC = driverPod.pod.getSpec.getVolumes.get(0).getPersistentVolumeClaim
+    assert(driverPVC.getClaimName === "pvc-spark-SPARK_EXECUTOR_ID")
+
+    val executorConf = KubernetesTestConf.createExecutorConf(volumes = Seq(volumeConf))
+    val executorStep = new MountVolumesFeatureStep(executorConf)
+    val executorPod = executorStep.configurePod(SparkPod.initialPod())
+
+    assert(executorPod.pod.getSpec.getVolumes.size() === 1)
+    val executorPVC = executorPod.pod.getSpec.getVolumes.get(0).getPersistentVolumeClaim
+    assert(executorPVC.getClaimName === s"pvc-spark-${KubernetesTestConf.EXECUTOR_ID}")
+  }
+
+  test("Create and mounts persistentVolumeClaims in driver") {
+    val volumeConf = KubernetesVolumeSpec(
+      "testVolume",
+      "/tmp",
+      "",
+      true,
+      KubernetesPVCVolumeConf("OnDemand")
+    )
+    val kubernetesConf = KubernetesTestConf.createDriverConf(volumes = Seq(volumeConf))
+    val step = new MountVolumesFeatureStep(kubernetesConf)
+    val configuredPod = step.configurePod(SparkPod.initialPod())
+
+    assert(configuredPod.pod.getSpec.getVolumes.size() === 1)
+    val pvcClaim = configuredPod.pod.getSpec.getVolumes.get(0).getPersistentVolumeClaim
+    assert(pvcClaim.getClaimName.endsWith("-driver-pvc-0"))
+  }
+
+  test("Create and mount persistentVolumeClaims in executors") {
+    val volumeConf = KubernetesVolumeSpec(
+      "testVolume",
+      "/tmp",
+      "",
+      true,
+      KubernetesPVCVolumeConf(MountVolumesFeatureStep.PVC_ON_DEMAND)
+    )
+    val executorConf = KubernetesTestConf.createExecutorConf(volumes = Seq(volumeConf))
+    val executorStep = new MountVolumesFeatureStep(executorConf)
+    val executorPod = executorStep.configurePod(SparkPod.initialPod())
+
+    assert(executorPod.pod.getSpec.getVolumes.size() === 1)
+    val executorPVC = executorPod.pod.getSpec.getVolumes.get(0).getPersistentVolumeClaim
+    assert(executorPVC.getClaimName.endsWith("-exec-1-pvc-0"))
+  }
+
   test("Mounts emptyDir") {
     val volumeConf = KubernetesVolumeSpec(
       "testVolume",
@@ -177,6 +236,31 @@ class MountVolumesFeatureStepSuite extends SparkFunSuite {
     assert(configuredPod.container.getVolumeMounts.size() === 2)
   }
 
+  test("mountPath should be unique") {
+    val hpVolumeConf = KubernetesVolumeSpec(
+      "hpVolume",
+      "/data",
+      "",
+      false,
+      KubernetesHostPathVolumeConf("/hostPath/tmp")
+    )
+    val pvcVolumeConf = KubernetesVolumeSpec(
+      "checkpointVolume",
+      "/data",
+      "",
+      true,
+      KubernetesPVCVolumeConf("pvcClaim")
+    )
+    val kubernetesConf = KubernetesTestConf.createDriverConf(
+      volumes = Seq(hpVolumeConf, pvcVolumeConf))
+
+    val step = new MountVolumesFeatureStep(kubernetesConf)
+    val m = intercept[IllegalArgumentException] {
+      step.configurePod(SparkPod.initialPod())
+    }.getMessage
+    assert(m.contains("Found duplicated mountPath: '/data'"))
+  }
+
   test("Mounts subpath on emptyDir") {
     val volumeConf = KubernetesVolumeSpec(
       "testVolume",
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStepSuite.scala
index 051320fa44c5e..1b38fd6a0d2ab 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStepSuite.scala
@@ -16,13 +16,14 @@
  */
 package org.apache.spark.deploy.k8s.features
 
-import java.io.{File, PrintWriter}
+import java.io.PrintWriter
 import java.nio.file.Files
 
 import io.fabric8.kubernetes.api.model.ConfigMap
 
 import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.deploy.k8s._
+import org.apache.spark.deploy.k8s.Constants._
 import org.apache.spark.util.Utils
 
 class PodTemplateConfigMapStepSuite extends SparkFunSuite {
@@ -56,8 +57,9 @@ class PodTemplateConfigMapStepSuite extends SparkFunSuite {
 
     assert(configuredPod.pod.getSpec.getVolumes.size() === 1)
     val volume = configuredPod.pod.getSpec.getVolumes.get(0)
+    val generatedResourceName = s"${kubernetesConf.resourceNamePrefix}-$POD_TEMPLATE_CONFIGMAP"
     assert(volume.getName === Constants.POD_TEMPLATE_VOLUME)
-    assert(volume.getConfigMap.getName === Constants.POD_TEMPLATE_CONFIGMAP)
+    assert(volume.getConfigMap.getName === generatedResourceName)
     assert(volume.getConfigMap.getItems.size() === 1)
     assert(volume.getConfigMap.getItems.get(0).getKey === Constants.POD_TEMPLATE_KEY)
     assert(volume.getConfigMap.getItems.get(0).getPath ===
@@ -70,7 +72,7 @@ class PodTemplateConfigMapStepSuite extends SparkFunSuite {
 
     val resources = step.getAdditionalKubernetesResources()
     assert(resources.size === 1)
-    assert(resources.head.getMetadata.getName === Constants.POD_TEMPLATE_CONFIGMAP)
+    assert(resources.head.getMetadata.getName === generatedResourceName)
     assert(resources.head.isInstanceOf[ConfigMap])
     val configMap = resources.head.asInstanceOf[ConfigMap]
     assert(configMap.getData.size() === 1)
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/ClientSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/ClientSuite.scala
index 5d49ac0bbaafa..07bf090c1fc47 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/ClientSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/ClientSuite.scala
@@ -16,6 +16,12 @@
  */
 package org.apache.spark.deploy.k8s.submit
 
+import java.io.File
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+
+import scala.collection.JavaConverters._
+
 import io.fabric8.kubernetes.api.model._
 import io.fabric8.kubernetes.client.{KubernetesClient, Watch}
 import io.fabric8.kubernetes.client.dsl.PodResource
@@ -24,10 +30,11 @@ import org.mockito.Mockito.{verify, when}
 import org.scalatest.BeforeAndAfter
 import org.scalatestplus.mockito.MockitoSugar._
 
-import org.apache.spark.SparkFunSuite
+import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.deploy.k8s._
 import org.apache.spark.deploy.k8s.Constants._
 import org.apache.spark.deploy.k8s.Fabric8Aliases._
+import org.apache.spark.util.Utils
 
 class ClientSuite extends SparkFunSuite with BeforeAndAfter {
 
@@ -66,27 +73,36 @@ class ClientSuite extends SparkFunSuite with BeforeAndAfter {
       .withValue(SPARK_CONF_DIR_INTERNAL)
       .endEnv()
     .addNewVolumeMount()
-      .withName(SPARK_CONF_VOLUME)
+      .withName(SPARK_CONF_VOLUME_DRIVER)
       .withMountPath(SPARK_CONF_DIR_INTERNAL)
       .endVolumeMount()
     .build()
-  private val FULL_EXPECTED_POD = new PodBuilder(BUILT_DRIVER_POD)
-    .editSpec()
-      .addToContainers(FULL_EXPECTED_CONTAINER)
-      .addNewVolume()
-        .withName(SPARK_CONF_VOLUME)
-        .withNewConfigMap().withName(s"$KUBERNETES_RESOURCE_PREFIX-driver-conf-map").endConfigMap()
-        .endVolume()
-      .endSpec()
-    .build()
 
-  private val POD_WITH_OWNER_REFERENCE = new PodBuilder(FULL_EXPECTED_POD)
-    .editMetadata()
-      .withUid(DRIVER_POD_UID)
-      .endMetadata()
-    .withApiVersion(DRIVER_POD_API_VERSION)
-    .withKind(DRIVER_POD_KIND)
-    .build()
+  private val KEY_TO_PATH =
+    new KeyToPath(SPARK_CONF_FILE_NAME, 420, SPARK_CONF_FILE_NAME)
+
+  private def fullExpectedPod(keyToPaths: List[KeyToPath] = List(KEY_TO_PATH)) =
+    new PodBuilder(BUILT_DRIVER_POD)
+      .editSpec()
+        .addToContainers(FULL_EXPECTED_CONTAINER)
+        .addNewVolume()
+          .withName(SPARK_CONF_VOLUME_DRIVER)
+          .withNewConfigMap()
+            .withItems(keyToPaths.asJava)
+            .withName(KubernetesClientUtils.configMapNameDriver)
+            .endConfigMap()
+          .endVolume()
+        .endSpec()
+      .build()
+
+  private def podWithOwnerReference(keyToPaths: List[KeyToPath] = List(KEY_TO_PATH)) =
+    new PodBuilder(fullExpectedPod(keyToPaths))
+      .editMetadata()
+        .withUid(DRIVER_POD_UID)
+        .endMetadata()
+      .withApiVersion(DRIVER_POD_API_VERSION)
+      .withKind(DRIVER_POD_KIND)
+      .build()
 
   private val ADDITIONAL_RESOURCES_WITH_OWNER_REFERENCES = ADDITIONAL_RESOURCES.map { secret =>
     new SecretBuilder(secret)
@@ -109,7 +125,7 @@ class ClientSuite extends SparkFunSuite with BeforeAndAfter {
   private var podOperations: PODS = _
 
   @Mock
-  private var namedPods: PodResource[Pod, DoneablePod] = _
+  private var namedPods: PodResource[Pod] = _
 
   @Mock
   private var loggingPodStatusWatcher: LoggingPodStatusWatcher = _
@@ -125,7 +141,7 @@ class ClientSuite extends SparkFunSuite with BeforeAndAfter {
   private var createdResourcesArgumentCaptor: ArgumentCaptor[HasMetadata] = _
 
   before {
-    MockitoAnnotations.initMocks(this)
+    MockitoAnnotations.openMocks(this).close()
     kconf = KubernetesTestConf.createDriverConf(
       resourceNamePrefix = Some(KUBERNETES_RESOURCE_PREFIX))
     when(driverBuilder.buildFromFeatures(kconf, kubernetesClient)).thenReturn(BUILT_KUBERNETES_SPEC)
@@ -134,8 +150,9 @@ class ClientSuite extends SparkFunSuite with BeforeAndAfter {
 
     createdPodArgumentCaptor = ArgumentCaptor.forClass(classOf[Pod])
     createdResourcesArgumentCaptor = ArgumentCaptor.forClass(classOf[HasMetadata])
-    when(podOperations.create(FULL_EXPECTED_POD)).thenReturn(POD_WITH_OWNER_REFERENCE)
+    when(podOperations.create(fullExpectedPod())).thenReturn(podWithOwnerReference())
     when(namedPods.watch(loggingPodStatusWatcher)).thenReturn(mock[Watch])
+    when(loggingPodStatusWatcher.watchOrStop(kconf.namespace + ":" + POD_NAME)).thenReturn(true)
     doReturn(resourceList)
       .when(kubernetesClient)
       .resourceList(createdResourcesArgumentCaptor.capture())
@@ -148,7 +165,7 @@ class ClientSuite extends SparkFunSuite with BeforeAndAfter {
       kubernetesClient,
       loggingPodStatusWatcher)
     submissionClient.run()
-    verify(podOperations).create(FULL_EXPECTED_POD)
+    verify(podOperations).create(fullExpectedPod())
   }
 
   test("The client should create Kubernetes resources") {
@@ -168,12 +185,78 @@ class ClientSuite extends SparkFunSuite with BeforeAndAfter {
     assert(configMaps.nonEmpty)
     val configMap = configMaps.head
     assert(configMap.getMetadata.getName ===
-      s"$KUBERNETES_RESOURCE_PREFIX-driver-conf-map")
+      KubernetesClientUtils.configMapNameDriver)
     assert(configMap.getData.containsKey(SPARK_CONF_FILE_NAME))
     assert(configMap.getData.get(SPARK_CONF_FILE_NAME).contains("conf1key=conf1value"))
     assert(configMap.getData.get(SPARK_CONF_FILE_NAME).contains("conf2key=conf2value"))
   }
 
+  test("All files from SPARK_CONF_DIR, " +
+    "except templates, spark config, binary files and are within size limit, " +
+    "should be populated to pod's configMap.") {
+    def testSetup: (SparkConf, Seq[String]) = {
+      val tempDir = Utils.createTempDir()
+      val sparkConf = new SparkConf(loadDefaults = false)
+        .setSparkHome(tempDir.getAbsolutePath)
+
+      val tempConfDir = new File(s"${tempDir.getAbsolutePath}/conf")
+      tempConfDir.mkdir()
+      // File names - which should not get mounted on the resultant config map.
+      val filteredConfFileNames =
+        Set("spark-env.sh.template", "spark.properties", "spark-defaults.conf",
+          "test.gz", "test2.jar", "non_utf8.txt")
+      val confFileNames = (for (i <- 1 to 5) yield s"testConf.$i") ++
+        List("spark-env.sh") ++ filteredConfFileNames
+
+      val testConfFiles = (for (i <- confFileNames) yield {
+        val file = new File(s"${tempConfDir.getAbsolutePath}/$i")
+        if (i.startsWith("non_utf8")) { // filling some non-utf-8 binary
+          Files.write(file.toPath, Array[Byte](0x00.toByte, 0xA1.toByte))
+        } else {
+          Files.write(file.toPath, "conf1key=conf1value".getBytes(StandardCharsets.UTF_8))
+        }
+        file.getName
+      })
+      assert(tempConfDir.listFiles().length == confFileNames.length)
+      val expectedConfFiles: Seq[String] = testConfFiles.filterNot(filteredConfFileNames.contains)
+      (sparkConf, expectedConfFiles)
+    }
+
+    val (sparkConf: SparkConf, expectedConfFiles: Seq[String]) = testSetup
+
+    val expectedKeyToPaths = (expectedConfFiles.map(x => new KeyToPath(x, 420, x)).toList ++
+      List(KEY_TO_PATH)).sortBy(x => x.getKey)
+
+    when(podOperations.create(fullExpectedPod(expectedKeyToPaths)))
+      .thenReturn(podWithOwnerReference(expectedKeyToPaths))
+
+    kconf = KubernetesTestConf.createDriverConf(sparkConf = sparkConf,
+      resourceNamePrefix = Some(KUBERNETES_RESOURCE_PREFIX))
+
+    assert(kconf.sparkConf.getOption("spark.home").isDefined)
+    when(driverBuilder.buildFromFeatures(kconf, kubernetesClient)).thenReturn(BUILT_KUBERNETES_SPEC)
+
+    val submissionClient = new Client(
+      kconf,
+      driverBuilder,
+      kubernetesClient,
+      loggingPodStatusWatcher)
+    submissionClient.run()
+    val otherCreatedResources = createdResourcesArgumentCaptor.getAllValues
+
+    val configMaps = otherCreatedResources.toArray
+      .filter(_.isInstanceOf[ConfigMap]).map(_.asInstanceOf[ConfigMap])
+    assert(configMaps.nonEmpty)
+    val configMapName = KubernetesClientUtils.configMapNameDriver
+    val configMap: ConfigMap = configMaps.head
+    assert(configMap.getMetadata.getName == configMapName)
+    val configMapLoadedFiles = configMap.getData.keySet().asScala.toSet
+    assert(configMapLoadedFiles === expectedConfFiles.toSet ++ Set(SPARK_CONF_FILE_NAME))
+    for (f <- configMapLoadedFiles) {
+      assert(configMap.getData.get(f).contains("conf1key=conf1value"))
+    }
+  }
+
   test("Waiting for app completion should stall on the watcher") {
     val submissionClient = new Client(
       kconf,
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/K8sSubmitOpSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/K8sSubmitOpSuite.scala
index d8be13280c2e6..142d3fe112d69 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/K8sSubmitOpSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/K8sSubmitOpSuite.scala
@@ -45,10 +45,10 @@ class K8sSubmitOpSuite extends SparkFunSuite with BeforeAndAfter {
   private var podOperations: PODS = _
 
   @Mock
-  private var driverPodOperations1: PodResource[Pod, DoneablePod] = _
+  private var driverPodOperations1: PodResource[Pod] = _
 
   @Mock
-  private var driverPodOperations2: PodResource[Pod, DoneablePod] = _
+  private var driverPodOperations2: PodResource[Pod] = _
 
   @Mock
   private var kubernetesClient: KubernetesClient = _
@@ -57,7 +57,7 @@ class K8sSubmitOpSuite extends SparkFunSuite with BeforeAndAfter {
   private var err: PrintStream = _
 
   before {
-    MockitoAnnotations.initMocks(this)
+    MockitoAnnotations.openMocks(this).close()
     when(kubernetesClient.pods()).thenReturn(podOperations)
     when(podOperations.inNamespace(namespace)).thenReturn(podOperations)
     when(podOperations.delete(podList.asJava)).thenReturn(true)
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientUtilsSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientUtilsSuite.scala
new file mode 100644
index 0000000000000..ee672cc041330
--- /dev/null
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientUtilsSuite.scala
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.deploy.k8s.submit
+
+import java.io.File
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+
+import org.scalatest.BeforeAndAfter
+
+import org.apache.spark.{SparkConf, SparkFunSuite}
+import org.apache.spark.deploy.k8s.Config
+import org.apache.spark.util.Utils
+
+class KubernetesClientUtilsSuite extends SparkFunSuite with BeforeAndAfter {
+
+  def testSetup(inputFiles: Map[String, Array[Byte]]): SparkConf = {
+    val tempDir = Utils.createTempDir()
+    val sparkConf = new SparkConf(loadDefaults = false)
+      .setSparkHome(tempDir.getAbsolutePath)
+
+    val tempConfDir = new File(s"${tempDir.getAbsolutePath}/conf")
+    tempConfDir.mkdir()
+    for (i <- inputFiles) yield {
+      val file = new File(s"${tempConfDir.getAbsolutePath}/${i._1}")
+      Files.write(file.toPath, i._2)
+      file.getName
+    }
+    sparkConf
+  }
+
+  test("verify load files, loads only allowed files and not the disallowed files.") {
+    val input: Map[String, Array[Byte]] = Map("test.txt" -> "test123", "z12.zip" -> "zZ",
+      "rere.jar" -> "@31", "spark.jar" -> "@31", "_test" -> "", "sample.conf" -> "conf")
+      .map(f => f._1 -> f._2.getBytes(StandardCharsets.UTF_8)) ++
+      Map("binary-file.conf" -> Array[Byte](0x00.toByte, 0xA1.toByte))
+    val sparkConf = testSetup(input)
+    val output = KubernetesClientUtils.loadSparkConfDirFiles(sparkConf)
+    val expectedOutput = Map("test.txt" -> "test123", "sample.conf" -> "conf", "_test" -> "")
+    assert(output === expectedOutput)
+  }
+
+  test("verify load files, truncates the content to maxSize, when keys are very large in number.") {
+    val input = (for (i <- 10000 to 1 by -1) yield (s"testConf.${i}" -> "test123456")).toMap
+    val sparkConf = testSetup(input.map(f => f._1 -> f._2.getBytes(StandardCharsets.UTF_8)))
+      .set(Config.CONFIG_MAP_MAXSIZE.key, "60")
+    val output = KubernetesClientUtils.loadSparkConfDirFiles(sparkConf)
+    val expectedOutput = Map("testConf.1" -> "test123456", "testConf.2" -> "test123456")
+    assert(output === expectedOutput)
+    val output1 = KubernetesClientUtils.loadSparkConfDirFiles(
+      sparkConf.set(Config.CONFIG_MAP_MAXSIZE.key, "250000"))
+    assert(output1 === input)
+  }
+
+  test("verify load files, truncates the content to maxSize, when keys are equal in length.") {
+    val input = (for (i <- 9 to 1 by -1) yield (s"testConf.${i}" -> "test123456")).toMap
+    val sparkConf = testSetup(input.map(f => f._1 -> f._2.getBytes(StandardCharsets.UTF_8)))
+      .set(Config.CONFIG_MAP_MAXSIZE.key, "80")
+    val output = KubernetesClientUtils.loadSparkConfDirFiles(sparkConf)
+    val expectedOutput = Map("testConf.1" -> "test123456", "testConf.2" -> "test123456",
+      "testConf.3" -> "test123456")
+    assert(output === expectedOutput)
+  }
+}
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala
index 6518c91a1a1fd..f9802ff967f82 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala
@@ -28,9 +28,12 @@ class KubernetesDriverBuilderSuite extends PodBuilderSuite {
     Config.KUBERNETES_DRIVER_PODTEMPLATE_FILE
   }
 
+  override protected def userFeatureStepsConf: ConfigEntry[_] = {
+    Config.KUBERNETES_DRIVER_POD_FEATURE_STEPS
+  }
+
   override protected def buildPod(sparkConf: SparkConf, client: KubernetesClient): SparkPod = {
     val conf = KubernetesTestConf.createDriverConf(sparkConf = sparkConf)
     new KubernetesDriverBuilder().buildFromFeatures(conf, client).pod
   }
-
 }
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/DeterministicExecutorPodsSnapshotsStore.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/DeterministicExecutorPodsSnapshotsStore.scala
index 9ac7e0222054a..dbe2f29872ad1 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/DeterministicExecutorPodsSnapshotsStore.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/DeterministicExecutorPodsSnapshotsStore.scala
@@ -19,8 +19,16 @@ package org.apache.spark.scheduler.cluster.k8s
 import io.fabric8.kubernetes.api.model.Pod
 import scala.collection.mutable
 
+import org.apache.spark.deploy.k8s.Constants.DEFAULT_EXECUTOR_CONTAINER_NAME
+import org.apache.spark.util.ManualClock
+
 class DeterministicExecutorPodsSnapshotsStore extends ExecutorPodsSnapshotsStore {
 
+  ExecutorPodsSnapshot.setShouldCheckAllContainers(false)
+  ExecutorPodsSnapshot.setSparkContainerName(DEFAULT_EXECUTOR_CONTAINER_NAME)
+
+  val clock = new ManualClock()
+
   private val snapshotsBuffer = mutable.Buffer.empty[ExecutorPodsSnapshot]
   private val subscribers = mutable.Buffer.empty[Seq[ExecutorPodsSnapshot] => Unit]
 
@@ -35,7 +43,7 @@ class DeterministicExecutorPodsSnapshotsStore extends ExecutorPodsSnapshotsStore
   override def stop(): Unit = {}
 
   override def notifySubscribers(): Unit = {
-    subscribers.foreach(_(snapshotsBuffer))
+    subscribers.foreach(_(snapshotsBuffer.toSeq))
     snapshotsBuffer.clear()
   }
 
@@ -45,7 +53,7 @@ class DeterministicExecutorPodsSnapshotsStore extends ExecutorPodsSnapshotsStore
   }
 
   override def replaceSnapshot(newSnapshot: Seq[Pod]): Unit = {
-    currentSnapshot = ExecutorPodsSnapshot(newSnapshot)
+    currentSnapshot = ExecutorPodsSnapshot(newSnapshot, clock.getTimeMillis())
     snapshotsBuffer += currentSnapshot
   }
 
@@ -54,7 +62,7 @@ class DeterministicExecutorPodsSnapshotsStore extends ExecutorPodsSnapshotsStore
       case (_, PodDeleted(_)) => false
       case _ => true
     }
-    currentSnapshot = ExecutorPodsSnapshot(nonDeleted)
+    currentSnapshot = ExecutorPodsSnapshot(nonDeleted, clock.getTimeMillis())
     snapshotsBuffer += currentSnapshot
   }
 }
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorLifecycleTestUtils.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorLifecycleTestUtils.scala
index 2e883623a4b1c..41cba573d89c2 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorLifecycleTestUtils.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorLifecycleTestUtils.scala
@@ -16,19 +16,24 @@
  */
 package org.apache.spark.scheduler.cluster.k8s
 
+import java.time.Instant
+
 import io.fabric8.kubernetes.api.model.{ContainerBuilder, Pod, PodBuilder}
 
 import org.apache.spark.deploy.k8s.Constants._
 import org.apache.spark.deploy.k8s.SparkPod
+import org.apache.spark.resource.ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID
 
 object ExecutorLifecycleTestUtils {
 
   val TEST_SPARK_APP_ID = "spark-app-id"
 
-  def failedExecutorWithoutDeletion(executorId: Long): Pod = {
-    new PodBuilder(podWithAttachedContainerForId(executorId))
+  def failedExecutorWithoutDeletion(
+      executorId: Long, rpId: Int = DEFAULT_RESOURCE_PROFILE_ID): Pod = {
+    new PodBuilder(podWithAttachedContainerForId(executorId, rpId))
       .editOrNewStatus()
         .withPhase("failed")
+        .withStartTime(Instant.now.toString)
         .addNewContainerStatus()
           .withName("spark-executor")
           .withImage("k8s-spark")
@@ -55,48 +60,108 @@ object ExecutorLifecycleTestUtils {
       .build()
   }
 
-  def pendingExecutor(executorId: Long): Pod = {
-    new PodBuilder(podWithAttachedContainerForId(executorId))
+  def pendingExecutor(executorId: Long, rpId: Int = DEFAULT_RESOURCE_PROFILE_ID): Pod = {
+    new PodBuilder(podWithAttachedContainerForId(executorId, rpId))
       .editOrNewStatus()
         .withPhase("pending")
+        .withStartTime(Instant.now.toString)
         .endStatus()
       .build()
   }
 
-  def runningExecutor(executorId: Long): Pod = {
-    new PodBuilder(podWithAttachedContainerForId(executorId))
+  def runningExecutor(executorId: Long, rpId: Int = DEFAULT_RESOURCE_PROFILE_ID): Pod = {
+    new PodBuilder(podWithAttachedContainerForId(executorId, rpId))
       .editOrNewStatus()
         .withPhase("running")
+        .withStartTime(Instant.now.toString)
         .endStatus()
       .build()
   }
 
-  def succeededExecutor(executorId: Long): Pod = {
-    new PodBuilder(podWithAttachedContainerForId(executorId))
+  /**
+   * [SPARK-30821]
+   * This creates a pod with one container in running state and one container in failed
+   * state (terminated with non-zero exit code). This pod is used for unit-testing the
+   * spark.kubernetes.executor.checkAllContainers Spark Conf.
+   */
+  def runningExecutorWithFailedContainer(
+      executorId: Long, rpId: Int = DEFAULT_RESOURCE_PROFILE_ID): Pod = {
+    new PodBuilder(podWithAttachedContainerForId(executorId, rpId))
+      .editOrNewStatus()
+        .withPhase("running")
+        .addNewContainerStatus()
+          .withNewState()
+            .withNewTerminated()
+              .withExitCode(1)
+            .endTerminated()
+          .endState()
+        .endContainerStatus()
+        .addNewContainerStatus()
+          .withNewState()
+            .withNewRunning()
+            .endRunning()
+          .endState()
+        .endContainerStatus()
+      .endStatus()
+      .build()
+  }
+
+  /**
+   * This creates a pod with a finished executor and running sidecar
+   */
+  def finishedExecutorWithRunningSidecar(
+      executorId: Long, exitCode: Int): Pod = {
+    new PodBuilder(podWithAttachedContainerForId(executorId, DEFAULT_RESOURCE_PROFILE_ID))
+      .editOrNewStatus()
+        .withPhase("running")
+        .addNewContainerStatus()
+          .withName(DEFAULT_EXECUTOR_CONTAINER_NAME)
+          .withNewState()
+            .withNewTerminated()
+              .withMessage("message")
+              .withExitCode(exitCode)
+            .endTerminated()
+          .endState()
+        .endContainerStatus()
+        .addNewContainerStatus()
+          .withName("SIDECARFRIEND")
+          .withNewState()
+            .withNewRunning()
+            .endRunning()
+          .endState()
+        .endContainerStatus()
+      .endStatus()
+      .build()
+  }
+
+  def succeededExecutor(executorId: Long, rpId: Int = DEFAULT_RESOURCE_PROFILE_ID): Pod = {
+    new PodBuilder(podWithAttachedContainerForId(executorId, rpId))
       .editOrNewStatus()
         .withPhase("succeeded")
         .endStatus()
       .build()
   }
 
-  def deletedExecutor(executorId: Long): Pod = {
-    new PodBuilder(podWithAttachedContainerForId(executorId))
+  def deletedExecutor(executorId: Long, rpId: Int = DEFAULT_RESOURCE_PROFILE_ID): Pod = {
+    new PodBuilder(podWithAttachedContainerForId(executorId, rpId))
       .editOrNewMetadata()
         .withDeletionTimestamp("523012521")
         .endMetadata()
       .build()
   }
 
-  def unknownExecutor(executorId: Long): Pod = {
-    new PodBuilder(podWithAttachedContainerForId(executorId))
+  def unknownExecutor(executorId: Long, rpId: Int = DEFAULT_RESOURCE_PROFILE_ID): Pod = {
+    new PodBuilder(podWithAttachedContainerForId(executorId, rpId))
       .editOrNewStatus()
         .withPhase("unknown")
         .endStatus()
       .build()
   }
 
-  def podWithAttachedContainerForId(executorId: Long): Pod = {
-    val sparkPod = executorPodWithId(executorId)
+  def podWithAttachedContainerForId(
+      executorId: Long,
+      rpId: Int = DEFAULT_RESOURCE_PROFILE_ID): Pod = {
+    val sparkPod = executorPodWithId(executorId, rpId)
     val podWithAttachedContainer = new PodBuilder(sparkPod.pod)
       .editOrNewSpec()
         .addToContainers(sparkPod.container)
@@ -105,14 +170,18 @@ object ExecutorLifecycleTestUtils {
     podWithAttachedContainer
   }
 
-  def executorPodWithId(executorId: Long): SparkPod = {
+  def executorPodWithId(executorId: Long, rpId: Int = DEFAULT_RESOURCE_PROFILE_ID): SparkPod = {
     val pod = new PodBuilder()
       .withNewMetadata()
         .withName(s"spark-executor-$executorId")
         .addToLabels(SPARK_APP_ID_LABEL, TEST_SPARK_APP_ID)
         .addToLabels(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE)
         .addToLabels(SPARK_EXECUTOR_ID_LABEL, executorId.toString)
-        .endMetadata()
+        .addToLabels(SPARK_RESOURCE_PROFILE_ID_LABEL, rpId.toString)
+      .endMetadata()
+      .editOrNewSpec()
+        .withRestartPolicy("Never")
+      .endSpec()
       .build()
     val container = new ContainerBuilder()
       .withName("spark-executor")
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocatorSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocatorSuite.scala
index a0abded3823bb..c5291ab973aa3 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocatorSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocatorSuite.scala
@@ -16,7 +16,9 @@
  */
 package org.apache.spark.scheduler.cluster.k8s
 
-import io.fabric8.kubernetes.api.model.{DoneablePod, Pod, PodBuilder}
+import java.time.Instant
+
+import io.fabric8.kubernetes.api.model.{Pod, PodBuilder}
 import io.fabric8.kubernetes.client.KubernetesClient
 import io.fabric8.kubernetes.client.dsl.PodResource
 import org.mockito.{Mock, MockitoAnnotations}
@@ -27,10 +29,12 @@ import org.mockito.stubbing.Answer
 import org.scalatest.BeforeAndAfter
 
 import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
-import org.apache.spark.deploy.k8s.{KubernetesExecutorConf, SparkPod}
+import org.apache.spark.deploy.k8s.{KubernetesExecutorConf, KubernetesExecutorSpec}
 import org.apache.spark.deploy.k8s.Config._
 import org.apache.spark.deploy.k8s.Constants._
 import org.apache.spark.deploy.k8s.Fabric8Aliases._
+import org.apache.spark.internal.config.DYN_ALLOCATION_EXECUTOR_IDLE_TIMEOUT
+import org.apache.spark.resource._
 import org.apache.spark.scheduler.cluster.k8s.ExecutorLifecycleTestUtils._
 import org.apache.spark.util.ManualClock
 
@@ -47,11 +51,17 @@ class ExecutorPodsAllocatorSuite extends SparkFunSuite with BeforeAndAfter {
       .endMetadata()
     .build()
 
-  private val conf = new SparkConf().set(KUBERNETES_DRIVER_POD_NAME, driverPodName)
+  private val conf = new SparkConf()
+    .set(KUBERNETES_DRIVER_POD_NAME, driverPodName)
+    .set(DYN_ALLOCATION_EXECUTOR_IDLE_TIMEOUT.key, "10s")
 
+  private val defaultProfile: ResourceProfile = ResourceProfile.getOrCreateDefaultProfile(conf)
   private val podAllocationSize = conf.get(KUBERNETES_ALLOCATION_BATCH_SIZE)
   private val podAllocationDelay = conf.get(KUBERNETES_ALLOCATION_BATCH_DELAY)
-  private val podCreationTimeout = math.max(podAllocationDelay * 5, 60000L)
+  private val executorIdleTimeout = conf.get(DYN_ALLOCATION_EXECUTOR_IDLE_TIMEOUT) * 1000
+  private val podCreationTimeout = math.max(podAllocationDelay * 5,
+    conf.get(KUBERNETES_ALLOCATION_EXECUTOR_TIMEOUT))
+
   private val secMgr = new SecurityManager(conf)
 
   private var waitForExecutorPodsClock: ManualClock = _
@@ -66,32 +76,37 @@ class ExecutorPodsAllocatorSuite extends SparkFunSuite with BeforeAndAfter {
   private var labeledPods: LABELED_PODS = _
 
   @Mock
-  private var driverPodOperations: PodResource[Pod, DoneablePod] = _
+  private var driverPodOperations: PodResource[Pod] = _
 
   @Mock
   private var executorBuilder: KubernetesExecutorBuilder = _
 
+  @Mock
+  private var schedulerBackend: KubernetesClusterSchedulerBackend = _
+
   private var snapshotsStore: DeterministicExecutorPodsSnapshotsStore = _
 
   private var podsAllocatorUnderTest: ExecutorPodsAllocator = _
 
   before {
-    MockitoAnnotations.initMocks(this)
+    MockitoAnnotations.openMocks(this).close()
     when(kubernetesClient.pods()).thenReturn(podOperations)
     when(podOperations.withName(driverPodName)).thenReturn(driverPodOperations)
     when(driverPodOperations.get).thenReturn(driverPod)
     when(executorBuilder.buildFromFeatures(any(classOf[KubernetesExecutorConf]), meq(secMgr),
-      meq(kubernetesClient))).thenAnswer(executorPodAnswer())
+      meq(kubernetesClient), any(classOf[ResourceProfile]))).thenAnswer(executorPodAnswer())
     snapshotsStore = new DeterministicExecutorPodsSnapshotsStore()
     waitForExecutorPodsClock = new ManualClock(0L)
     podsAllocatorUnderTest = new ExecutorPodsAllocator(
       conf, secMgr, executorBuilder, kubernetesClient, snapshotsStore, waitForExecutorPodsClock)
-    podsAllocatorUnderTest.start(TEST_SPARK_APP_ID)
+    when(schedulerBackend.getExecutorIds).thenReturn(Seq.empty)
+    podsAllocatorUnderTest.start(TEST_SPARK_APP_ID, schedulerBackend)
   }
 
   test("Initially request executors in batches. Do not request another batch if the" +
     " first has not finished.") {
-    podsAllocatorUnderTest.setTotalExpectedExecutors(podAllocationSize + 1)
+    podsAllocatorUnderTest.setTotalExpectedExecutors(Map(defaultProfile -> (podAllocationSize + 1)))
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 5)
     for (nextId <- 1 to podAllocationSize) {
       verify(podOperations).create(podWithAttachedContainerForId(nextId))
     }
@@ -100,29 +115,35 @@ class ExecutorPodsAllocatorSuite extends SparkFunSuite with BeforeAndAfter {
 
   test("Request executors in batches. Allow another batch to be requested if" +
     " all pending executors start running.") {
-    podsAllocatorUnderTest.setTotalExpectedExecutors(podAllocationSize + 1)
+    podsAllocatorUnderTest.setTotalExpectedExecutors(Map(defaultProfile -> (podAllocationSize + 1)))
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 5)
     for (execId <- 1 until podAllocationSize) {
       snapshotsStore.updatePod(runningExecutor(execId))
     }
     snapshotsStore.notifySubscribers()
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 1)
     verify(podOperations, never()).create(podWithAttachedContainerForId(podAllocationSize + 1))
     snapshotsStore.updatePod(runningExecutor(podAllocationSize))
     snapshotsStore.notifySubscribers()
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 1)
     verify(podOperations).create(podWithAttachedContainerForId(podAllocationSize + 1))
     snapshotsStore.updatePod(runningExecutor(podAllocationSize))
     snapshotsStore.notifySubscribers()
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 1)
     verify(podOperations, times(podAllocationSize + 1)).create(any(classOf[Pod]))
   }
 
   test("When a current batch reaches error states immediately, re-request" +
     " them on the next batch.") {
-    podsAllocatorUnderTest.setTotalExpectedExecutors(podAllocationSize)
+    podsAllocatorUnderTest.setTotalExpectedExecutors(Map(defaultProfile -> podAllocationSize))
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 5)
     for (execId <- 1 until podAllocationSize) {
       snapshotsStore.updatePod(runningExecutor(execId))
     }
     val failedPod = failedExecutorWithoutDeletion(podAllocationSize)
     snapshotsStore.updatePod(failedPod)
     snapshotsStore.notifySubscribers()
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 1)
     verify(podOperations).create(podWithAttachedContainerForId(podAllocationSize + 1))
   }
 
@@ -137,10 +158,12 @@ class ExecutorPodsAllocatorSuite extends SparkFunSuite with BeforeAndAfter {
     when(podOperations
       .withLabelIn(SPARK_EXECUTOR_ID_LABEL, "1"))
       .thenReturn(labeledPods)
-    podsAllocatorUnderTest.setTotalExpectedExecutors(1)
+    podsAllocatorUnderTest.setTotalExpectedExecutors(Map(defaultProfile -> 1))
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 1)
     verify(podOperations).create(podWithAttachedContainerForId(1))
     waitForExecutorPodsClock.setTime(podCreationTimeout + 1)
     snapshotsStore.notifySubscribers()
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 1)
     verify(labeledPods).delete()
     verify(podOperations).create(podWithAttachedContainerForId(2))
   }
@@ -159,19 +182,25 @@ class ExecutorPodsAllocatorSuite extends SparkFunSuite with BeforeAndAfter {
       .withLabelIn(meq(SPARK_EXECUTOR_ID_LABEL), any()))
       .thenReturn(podOperations)
 
+    val startTime = Instant.now.toEpochMilli
+    waitForExecutorPodsClock.setTime(startTime)
+
     // Target 1 executor, make sure it's requested, even with an empty initial snapshot.
-    podsAllocatorUnderTest.setTotalExpectedExecutors(1)
+    podsAllocatorUnderTest.setTotalExpectedExecutors(Map(defaultProfile -> 1))
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 1)
     verify(podOperations).create(podWithAttachedContainerForId(1))
 
     // Mark executor as running, verify that subsequent allocation cycle is a no-op.
     snapshotsStore.updatePod(runningExecutor(1))
     snapshotsStore.notifySubscribers()
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 0)
     verify(podOperations, times(1)).create(any())
     verify(podOperations, never()).delete()
 
     // Request 3 more executors, make sure all are requested.
-    podsAllocatorUnderTest.setTotalExpectedExecutors(4)
+    podsAllocatorUnderTest.setTotalExpectedExecutors(Map(defaultProfile -> 4))
     snapshotsStore.notifySubscribers()
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 3)
     verify(podOperations).create(podWithAttachedContainerForId(2))
     verify(podOperations).create(podWithAttachedContainerForId(3))
     verify(podOperations).create(podWithAttachedContainerForId(4))
@@ -180,12 +209,15 @@ class ExecutorPodsAllocatorSuite extends SparkFunSuite with BeforeAndAfter {
     snapshotsStore.updatePod(runningExecutor(2))
     snapshotsStore.updatePod(pendingExecutor(3))
     snapshotsStore.notifySubscribers()
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 2)
     verify(podOperations, times(4)).create(any())
     verify(podOperations, never()).delete()
 
     // Scale down to 1. Pending executors (both acknowledged and not) should be deleted.
-    podsAllocatorUnderTest.setTotalExpectedExecutors(1)
+    waitForExecutorPodsClock.advance(executorIdleTimeout * 2)
+    podsAllocatorUnderTest.setTotalExpectedExecutors(Map(defaultProfile -> 1))
     snapshotsStore.notifySubscribers()
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 0)
     verify(podOperations, times(4)).create(any())
     verify(podOperations).withLabelIn(SPARK_EXECUTOR_ID_LABEL, "3", "4")
     verify(podOperations).delete()
@@ -198,13 +230,349 @@ class ExecutorPodsAllocatorSuite extends SparkFunSuite with BeforeAndAfter {
     snapshotsStore.updatePod(deletedExecutor(4))
     snapshotsStore.removeDeletedExecutors()
     snapshotsStore.notifySubscribers()
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 0)
     assert(!podsAllocatorUnderTest.isDeleted("3"))
     assert(!podsAllocatorUnderTest.isDeleted("4"))
   }
 
-  private def executorPodAnswer(): Answer[SparkPod] =
+  test("SPARK-34334: correctly identify timed out pending pod requests as excess") {
+    when(podOperations
+      .withField("status.phase", "Pending"))
+      .thenReturn(podOperations)
+    when(podOperations
+      .withLabel(SPARK_APP_ID_LABEL, TEST_SPARK_APP_ID))
+      .thenReturn(podOperations)
+    when(podOperations
+      .withLabel(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE))
+      .thenReturn(podOperations)
+    when(podOperations
+      .withLabelIn(meq(SPARK_EXECUTOR_ID_LABEL), any()))
+      .thenReturn(podOperations)
+
+    val startTime = Instant.now.toEpochMilli
+    waitForExecutorPodsClock.setTime(startTime)
+
+    podsAllocatorUnderTest.setTotalExpectedExecutors(Map(defaultProfile -> 1))
+    verify(podOperations).create(podWithAttachedContainerForId(1))
+    verify(podOperations).create(any())
+
+    snapshotsStore.updatePod(pendingExecutor(1))
+    snapshotsStore.notifySubscribers()
+
+    waitForExecutorPodsClock.advance(executorIdleTimeout)
+
+    podsAllocatorUnderTest.setTotalExpectedExecutors(Map(defaultProfile -> 2))
+    snapshotsStore.notifySubscribers()
+    verify(podOperations).create(podWithAttachedContainerForId(2))
+
+    podsAllocatorUnderTest.setTotalExpectedExecutors(Map(defaultProfile -> 1))
+    snapshotsStore.notifySubscribers()
+
+    verify(podOperations, never()).withLabelIn(SPARK_EXECUTOR_ID_LABEL, "1")
+    verify(podOperations, never()).delete()
+
+    waitForExecutorPodsClock.advance(executorIdleTimeout)
+    snapshotsStore.notifySubscribers()
+
+    // before SPARK-34334 this verify() call failed as the non-timed out newly created request
+    // decreased the number of requests taken from timed out pending pod requests
+    verify(podOperations).withLabelIn(SPARK_EXECUTOR_ID_LABEL, "1")
+    verify(podOperations).delete()
+  }
+
+  test("SPARK-33099: Respect executor idle timeout configuration") {
+    when(podOperations
+      .withField("status.phase", "Pending"))
+      .thenReturn(podOperations)
+    when(podOperations
+      .withLabel(SPARK_APP_ID_LABEL, TEST_SPARK_APP_ID))
+      .thenReturn(podOperations)
+    when(podOperations
+      .withLabel(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE))
+      .thenReturn(podOperations)
+    when(podOperations
+      .withLabelIn(meq(SPARK_EXECUTOR_ID_LABEL), any()))
+      .thenReturn(podOperations)
+
+    val startTime = Instant.now.toEpochMilli
+    waitForExecutorPodsClock.setTime(startTime)
+
+    podsAllocatorUnderTest.setTotalExpectedExecutors(Map(defaultProfile -> 5))
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 5)
+    verify(podOperations).create(podWithAttachedContainerForId(1))
+    verify(podOperations).create(podWithAttachedContainerForId(2))
+    verify(podOperations).create(podWithAttachedContainerForId(3))
+    verify(podOperations).create(podWithAttachedContainerForId(4))
+    verify(podOperations).create(podWithAttachedContainerForId(5))
+    verify(podOperations, times(5)).create(any())
+
+    snapshotsStore.updatePod(pendingExecutor(1))
+    snapshotsStore.updatePod(pendingExecutor(2))
+
+    // Newly created executors (both acknowledged and not) are protected by executorIdleTimeout
+    podsAllocatorUnderTest.setTotalExpectedExecutors(Map(defaultProfile -> 0))
+    snapshotsStore.notifySubscribers()
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 5)
+    verify(podOperations, never()).withLabelIn(SPARK_EXECUTOR_ID_LABEL, "1", "2", "3", "4", "5")
+    verify(podOperations, never()).delete()
+
+    // Newly created executors (both acknowledged and not) are cleaned up.
+    waitForExecutorPodsClock.advance(executorIdleTimeout * 2)
+    when(schedulerBackend.getExecutorIds).thenReturn(Seq("1", "3", "4"))
+    snapshotsStore.notifySubscribers()
+    // SPARK-34361: even as 1, 3 and 4 are not timed out as they are considered as known PODs so
+    // this is why they are not counted into the outstanding PODs and /they are not removed even
+    // though executor 1 is still in pending state and executor 3 and 4 are new request without
+    // any state reported by kubernetes and all the three are already timed out
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 0)
+    verify(podOperations).withLabelIn(SPARK_EXECUTOR_ID_LABEL, "2", "5")
+    verify(podOperations).delete()
+  }
+
+  /**
+   * This test covers some downscaling and upscaling of dynamic allocation on kubernetes
+   * along with multiple resource profiles (default and rp) when some executors
+   * already know by the scheduler backend.
+   *
+   * Legend:
+   *
+   * N-: newly created not known by the scheduler backend
+   * N+: newly created known by the scheduler backend
+   * P- / P+ : pending (not know / known) by the scheduler backend
+   * D: deleted
+   *                                       |   default    ||         rp        | expected
+   *                                       |              ||                   | outstanding
+   *                                       | 1  | 2  | 3  || 4  | 5  | 6  | 7  | PODs
+   * ==========================================================================================
+   *  0) setTotalExpectedExecs with        | N- | N- | N- || N- | N- | N- | N- |
+   *       default->3, ro->4               |    |    |    ||    |    |    |    |      7
+   * ------------------------------------------------------------------------------------------
+   *  1) make 1 from each rp               | N+ | N- | N- || N+ | N- | N- | N- |
+   *     known by backend                  |    |    |    ||    |    |    |    |      5
+   * -------------------------------------------------------------------------------------------
+   *  2) some more backend known + pending | N+ | P+ | P- || N+ | P+ | P- | N- |      3
+   * -------------------------------------------------------------------------------------------
+   *  3) advance time with idle timeout    |    |    |    ||    |    |    |    |
+   *     setTotalExpectedExecs with        | N+ | P+ | D  || N+ | P+ | D  | D  |      0
+   *       default->1, rp->1               |    |    |    ||    |    |    |    |
+   * -------------------------------------------------------------------------------------------
+   *  4) setTotalExpectedExecs with        | N+ | P+ | D  || N+ | P+ | D  | D  |      0 and
+   *       default->2, rp->2               |    |    |    ||    |    |    |    | no new POD req.
+   * ===========================================================================================
+   *
+   *  5) setTotalExpectedExecs with default -> 3, rp -> 3 which will lead to creation of the new
+   *     PODs: 8 and 9
+   */
+  test("SPARK-34361: scheduler backend known pods with multiple resource profiles at downscaling") {
+    when(podOperations
+      .withField("status.phase", "Pending"))
+      .thenReturn(podOperations)
+    when(podOperations
+      .withLabel(SPARK_APP_ID_LABEL, TEST_SPARK_APP_ID))
+      .thenReturn(podOperations)
+    when(podOperations
+      .withLabel(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE))
+      .thenReturn(podOperations)
+    when(podOperations
+      .withLabelIn(meq(SPARK_EXECUTOR_ID_LABEL), any()))
+      .thenReturn(podOperations)
+
+    val startTime = Instant.now.toEpochMilli
+    waitForExecutorPodsClock.setTime(startTime)
+
+    val rpb = new ResourceProfileBuilder()
+    val ereq = new ExecutorResourceRequests()
+    val treq = new TaskResourceRequests()
+    ereq.cores(4).memory("2g")
+    treq.cpus(2)
+    rpb.require(ereq).require(treq)
+    val rp = rpb.build()
+
+    // 0) request 3 PODs for the default and 4 PODs for the other resource profile
+    podsAllocatorUnderTest.setTotalExpectedExecutors(Map(defaultProfile -> 3, rp -> 4))
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 7)
+    verify(podOperations).create(podWithAttachedContainerForId(1, defaultProfile.id))
+    verify(podOperations).create(podWithAttachedContainerForId(2, defaultProfile.id))
+    verify(podOperations).create(podWithAttachedContainerForId(3, defaultProfile.id))
+    verify(podOperations).create(podWithAttachedContainerForId(4, rp.id))
+    verify(podOperations).create(podWithAttachedContainerForId(5, rp.id))
+    verify(podOperations).create(podWithAttachedContainerForId(6, rp.id))
+    verify(podOperations).create(podWithAttachedContainerForId(7, rp.id))
+
+    // 1) make 1 POD known by the scheduler backend for each resource profile
+    when(schedulerBackend.getExecutorIds).thenReturn(Seq("1", "4"))
+    snapshotsStore.notifySubscribers()
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 5,
+      "scheduler backend known PODs are not outstanding")
+    verify(podOperations, times(7)).create(any())
+
+    // 2) make 1 extra POD known by the scheduler backend for each resource profile
+    // and make some to pending
+    when(schedulerBackend.getExecutorIds).thenReturn(Seq("1", "2", "4", "5"))
+    snapshotsStore.updatePod(pendingExecutor(2, defaultProfile.id))
+    snapshotsStore.updatePod(pendingExecutor(3, defaultProfile.id))
+    snapshotsStore.updatePod(pendingExecutor(5, rp.id))
+    snapshotsStore.updatePod(pendingExecutor(6, rp.id))
+    snapshotsStore.notifySubscribers()
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 3)
+    verify(podOperations, times(7)).create(any())
+
+    // 3) downscale to 1 POD for default and 1 POD for the other resource profile
+    waitForExecutorPodsClock.advance(executorIdleTimeout * 2)
+    podsAllocatorUnderTest.setTotalExpectedExecutors(Map(defaultProfile -> 1, rp -> 1))
+    snapshotsStore.notifySubscribers()
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 0)
+    verify(podOperations, times(7)).create(any())
+    verify(podOperations, times(2)).delete()
+    assert(podsAllocatorUnderTest.isDeleted("3"))
+    assert(podsAllocatorUnderTest.isDeleted("6"))
+    assert(podsAllocatorUnderTest.isDeleted("7"))
+
+    // 4) upscale to 2 PODs for default and 2 for the other resource profile but as there is still
+    // 2 PODs known by the scheduler backend there must be no new POD requested to be created
+    podsAllocatorUnderTest.setTotalExpectedExecutors(Map(defaultProfile -> 2, rp -> 2))
+    snapshotsStore.notifySubscribers()
+    verify(podOperations, times(7)).create(any())
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 0)
+    verify(podOperations, times(7)).create(any())
+
+    // 5) requesting 1 more executor for each resource
+    podsAllocatorUnderTest.setTotalExpectedExecutors(Map(defaultProfile -> 3, rp -> 3))
+    snapshotsStore.notifySubscribers()
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 2)
+    verify(podOperations, times(9)).create(any())
+    verify(podOperations).create(podWithAttachedContainerForId(8, defaultProfile.id))
+    verify(podOperations).create(podWithAttachedContainerForId(9, rp.id))
+  }
+
+  test("SPARK-33288: multiple resource profiles") {
+    when(podOperations
+      .withField("status.phase", "Pending"))
+      .thenReturn(podOperations)
+    when(podOperations
+      .withLabel(SPARK_APP_ID_LABEL, TEST_SPARK_APP_ID))
+      .thenReturn(podOperations)
+    when(podOperations
+      .withLabel(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE))
+      .thenReturn(podOperations)
+    when(podOperations
+      .withLabelIn(meq(SPARK_EXECUTOR_ID_LABEL), any()))
+      .thenReturn(podOperations)
+
+    val startTime = Instant.now.toEpochMilli
+    waitForExecutorPodsClock.setTime(startTime)
+
+    val rpb = new ResourceProfileBuilder()
+    val ereq = new ExecutorResourceRequests()
+    val treq = new TaskResourceRequests()
+    ereq.cores(4).memory("2g")
+    treq.cpus(2)
+    rpb.require(ereq).require(treq)
+    val rp = rpb.build()
+
+    // Target 1 executor for default profile, 2 for other profile,
+    // make sure it's requested, even with an empty initial snapshot.
+    podsAllocatorUnderTest.setTotalExpectedExecutors(Map(defaultProfile -> 1, rp -> 2))
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 3)
+    verify(podOperations).create(podWithAttachedContainerForId(1, defaultProfile.id))
+    verify(podOperations).create(podWithAttachedContainerForId(2, rp.id))
+    verify(podOperations).create(podWithAttachedContainerForId(3, rp.id))
+
+    // Mark executor as running, verify that subsequent allocation cycle is a no-op.
+    snapshotsStore.updatePod(runningExecutor(1, defaultProfile.id))
+    snapshotsStore.updatePod(runningExecutor(2, rp.id))
+    snapshotsStore.updatePod(runningExecutor(3, rp.id))
+    snapshotsStore.notifySubscribers()
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 0)
+    verify(podOperations, times(3)).create(any())
+    verify(podOperations, never()).delete()
+
+    // Request 3 more executors for default profile and 1 more for other profile,
+    // make sure all are requested.
+    podsAllocatorUnderTest.setTotalExpectedExecutors(Map(defaultProfile -> 4, rp -> 3))
+    snapshotsStore.notifySubscribers()
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 4)
+    verify(podOperations).create(podWithAttachedContainerForId(4, defaultProfile.id))
+    verify(podOperations).create(podWithAttachedContainerForId(5, defaultProfile.id))
+    verify(podOperations).create(podWithAttachedContainerForId(6, defaultProfile.id))
+    verify(podOperations).create(podWithAttachedContainerForId(7, rp.id))
+
+    // Mark 4 as running, 5 and 7 as pending. Allocation cycle should do nothing.
+    snapshotsStore.updatePod(runningExecutor(4, defaultProfile.id))
+    snapshotsStore.updatePod(pendingExecutor(5, defaultProfile.id))
+    snapshotsStore.updatePod(pendingExecutor(7, rp.id))
+    snapshotsStore.notifySubscribers()
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 3)
+    verify(podOperations, times(7)).create(any())
+    verify(podOperations, never()).delete()
+
+    // Scale down to 1 for both resource profiles. Pending executors
+    // (both acknowledged and not) should be deleted.
+    waitForExecutorPodsClock.advance(executorIdleTimeout * 2)
+    podsAllocatorUnderTest.setTotalExpectedExecutors(Map(defaultProfile -> 1, rp -> 1))
+    snapshotsStore.notifySubscribers()
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 0)
+    verify(podOperations, times(7)).create(any())
+    verify(podOperations).withLabelIn(SPARK_EXECUTOR_ID_LABEL, "5", "6")
+    verify(podOperations).withLabelIn(SPARK_EXECUTOR_ID_LABEL, "7")
+    verify(podOperations, times(2)).delete()
+    assert(podsAllocatorUnderTest.isDeleted("5"))
+    assert(podsAllocatorUnderTest.isDeleted("6"))
+    assert(podsAllocatorUnderTest.isDeleted("7"))
+
+    // Update the snapshot to not contain the deleted executors, make sure the
+    // allocator cleans up internal state.
+    snapshotsStore.updatePod(deletedExecutor(5))
+    snapshotsStore.updatePod(deletedExecutor(6))
+    snapshotsStore.updatePod(deletedExecutor(7))
+    snapshotsStore.removeDeletedExecutors()
+    snapshotsStore.notifySubscribers()
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 0)
+    assert(!podsAllocatorUnderTest.isDeleted("5"))
+    assert(!podsAllocatorUnderTest.isDeleted("6"))
+    assert(!podsAllocatorUnderTest.isDeleted("7"))
+  }
+
+  test("SPARK-33262: pod allocator does not stall with pending pods") {
+    when(podOperations
+      .withLabel(SPARK_APP_ID_LABEL, TEST_SPARK_APP_ID))
+      .thenReturn(podOperations)
+    when(podOperations
+      .withLabel(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE))
+      .thenReturn(podOperations)
+    when(podOperations
+      .withLabelIn(SPARK_EXECUTOR_ID_LABEL, "1"))
+      .thenReturn(labeledPods)
+    when(podOperations
+      .withLabelIn(SPARK_EXECUTOR_ID_LABEL, "2", "3", "4", "5", "6"))
+      .thenReturn(podOperations)
+
+    podsAllocatorUnderTest.setTotalExpectedExecutors(Map(defaultProfile -> 6))
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 5)
+    // Initial request of pods
+    verify(podOperations).create(podWithAttachedContainerForId(1))
+    verify(podOperations).create(podWithAttachedContainerForId(2))
+    verify(podOperations).create(podWithAttachedContainerForId(3))
+    verify(podOperations).create(podWithAttachedContainerForId(4))
+    verify(podOperations).create(podWithAttachedContainerForId(5))
+    // 4 come up, 1 pending
+    snapshotsStore.updatePod(pendingExecutor(1))
+    snapshotsStore.updatePod(runningExecutor(2))
+    snapshotsStore.updatePod(runningExecutor(3))
+    snapshotsStore.updatePod(runningExecutor(4))
+    snapshotsStore.updatePod(runningExecutor(5))
+    // We move forward one allocation cycle
+    waitForExecutorPodsClock.setTime(podAllocationDelay + 1)
+    snapshotsStore.notifySubscribers()
+    assert(podsAllocatorUnderTest.numOutstandingPods.get() == 2)
+    // We request pod 6
+    verify(podOperations).create(podWithAttachedContainerForId(6))
+  }
+
+  private def executorPodAnswer(): Answer[KubernetesExecutorSpec] =
     (invocation: InvocationOnMock) => {
       val k8sConf: KubernetesExecutorConf = invocation.getArgument(0)
-      executorPodWithId(k8sConf.executorId.toInt)
+      KubernetesExecutorSpec(executorPodWithId(k8sConf.executorId.toInt,
+        k8sConf.resourceProfileId.toInt), Seq.empty)
   }
 }
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManagerSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManagerSuite.scala
index fb6f3ace7a914..e3ec53adef6ab 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManagerSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManagerSuite.scala
@@ -16,8 +16,7 @@
  */
 package org.apache.spark.scheduler.cluster.k8s
 
-import com.google.common.cache.CacheBuilder
-import io.fabric8.kubernetes.api.model.{DoneablePod, Pod}
+import io.fabric8.kubernetes.api.model.Pod
 import io.fabric8.kubernetes.client.KubernetesClient
 import io.fabric8.kubernetes.client.dsl.PodResource
 import org.mockito.{ArgumentCaptor, Mock, MockitoAnnotations}
@@ -38,7 +37,7 @@ import org.apache.spark.scheduler.cluster.k8s.ExecutorLifecycleTestUtils._
 
 class ExecutorPodsLifecycleManagerSuite extends SparkFunSuite with BeforeAndAfter {
 
-  private var namedExecutorPods: mutable.Map[String, PodResource[Pod, DoneablePod]] = _
+  private var namedExecutorPods: mutable.Map[String, PodResource[Pod]] = _
 
   @Mock
   private var kubernetesClient: KubernetesClient = _
@@ -53,18 +52,16 @@ class ExecutorPodsLifecycleManagerSuite extends SparkFunSuite with BeforeAndAfte
   private var eventHandlerUnderTest: ExecutorPodsLifecycleManager = _
 
   before {
-    MockitoAnnotations.initMocks(this)
-    val removedExecutorsCache = CacheBuilder.newBuilder().build[java.lang.Long, java.lang.Long]
+    MockitoAnnotations.openMocks(this).close()
     snapshotsStore = new DeterministicExecutorPodsSnapshotsStore()
-    namedExecutorPods = mutable.Map.empty[String, PodResource[Pod, DoneablePod]]
-    when(schedulerBackend.getExecutorIds()).thenReturn(Seq.empty[String])
+    namedExecutorPods = mutable.Map.empty[String, PodResource[Pod]]
+    when(schedulerBackend.getExecutorsWithRegistrationTs()).thenReturn(Map.empty[String, Long])
     when(kubernetesClient.pods()).thenReturn(podOperations)
     when(podOperations.withName(any(classOf[String]))).thenAnswer(namedPodsAnswer())
     eventHandlerUnderTest = new ExecutorPodsLifecycleManager(
       new SparkConf(),
       kubernetesClient,
-      snapshotsStore,
-      removedExecutorsCache)
+      snapshotsStore)
     eventHandlerUnderTest.start(schedulerBackend)
   }
 
@@ -72,7 +69,7 @@ class ExecutorPodsLifecycleManagerSuite extends SparkFunSuite with BeforeAndAfte
     val failedPod = failedExecutorWithoutDeletion(1)
     snapshotsStore.updatePod(failedPod)
     snapshotsStore.notifySubscribers()
-    val msg = exitReasonMessage(1, failedPod)
+    val msg = exitReasonMessage(1, failedPod, 1)
     val expectedLossReason = ExecutorExited(1, exitCausedByApp = true, msg)
     verify(schedulerBackend).doRemoveExecutor("1", expectedLossReason)
     verify(namedExecutorPods(failedPod.getMetadata.getName)).delete()
@@ -84,7 +81,7 @@ class ExecutorPodsLifecycleManagerSuite extends SparkFunSuite with BeforeAndAfte
     snapshotsStore.notifySubscribers()
     snapshotsStore.updatePod(failedPod)
     snapshotsStore.notifySubscribers()
-    val msg = exitReasonMessage(1, failedPod)
+    val msg = exitReasonMessage(1, failedPod, 1)
     val expectedLossReason = ExecutorExited(1, exitCausedByApp = true, msg)
     verify(schedulerBackend, times(1)).doRemoveExecutor("1", expectedLossReason)
     verify(namedExecutorPods(failedPod.getMetadata.getName), times(2)).delete()
@@ -92,13 +89,23 @@ class ExecutorPodsLifecycleManagerSuite extends SparkFunSuite with BeforeAndAfte
 
   test("When the scheduler backend lists executor ids that aren't present in the cluster," +
     " remove those executors from Spark.") {
-    when(schedulerBackend.getExecutorIds()).thenReturn(Seq("1"))
-    val msg = s"The executor with ID 1 was not found in the cluster but we didn't" +
-      s" get a reason why. Marking the executor as failed. The executor may have been" +
-      s" deleted but the driver missed the deletion event."
-    val expectedLossReason = ExecutorExited(-1, exitCausedByApp = false, msg)
+      when(schedulerBackend.getExecutorsWithRegistrationTs()).thenReturn(Map("1" -> 7L))
+    val missingPodDelta =
+      eventHandlerUnderTest.conf.get(Config.KUBERNETES_EXECUTOR_MISSING_POD_DETECT_DELTA)
+    snapshotsStore.clock.advance(missingPodDelta + 7)
+    snapshotsStore.replaceSnapshot(Seq.empty[Pod])
+    snapshotsStore.notifySubscribers()
+    verify(schedulerBackend, never()).doRemoveExecutor(any(), any())
+
+    // 1 more millisecond and the accepted delta is over so the missing POD will be detected
+    snapshotsStore.clock.advance(1)
     snapshotsStore.replaceSnapshot(Seq.empty[Pod])
     snapshotsStore.notifySubscribers()
+    val msg = "The executor with ID 1 (registered at 7 ms) was not found in the cluster at " +
+      "the polling time (30008 ms) which is after the accepted detect delta time (30000 ms) " +
+      "configured by `spark.kubernetes.executor.missingPodDetectDelta`. The executor may have " +
+      "been deleted but the driver missed the deletion event. Marking this executor as failed."
+    val expectedLossReason = ExecutorExited(-1, exitCausedByApp = false, msg)
     verify(schedulerBackend).doRemoveExecutor("1", expectedLossReason)
   }
 
@@ -107,7 +114,7 @@ class ExecutorPodsLifecycleManagerSuite extends SparkFunSuite with BeforeAndAfte
     eventHandlerUnderTest.conf.set(Config.KUBERNETES_DELETE_EXECUTORS, false)
     snapshotsStore.updatePod(failedPod)
     snapshotsStore.notifySubscribers()
-    val msg = exitReasonMessage(1, failedPod)
+    val msg = exitReasonMessage(1, failedPod, 1)
     val expectedLossReason = ExecutorExited(1, exitCausedByApp = true, msg)
     verify(schedulerBackend).doRemoveExecutor("1", expectedLossReason)
     verify(namedExecutorPods(failedPod.getMetadata.getName), never()).delete()
@@ -119,23 +126,30 @@ class ExecutorPodsLifecycleManagerSuite extends SparkFunSuite with BeforeAndAfte
     assert(pod.getMetadata().getLabels().get(SPARK_EXECUTOR_INACTIVE_LABEL) === "true")
   }
 
-  private def exitReasonMessage(failedExecutorId: Int, failedPod: Pod): String = {
+  private def exitReasonMessage(execId: Int, failedPod: Pod, exitCode: Int): String = {
     val reason = Option(failedPod.getStatus.getReason)
     val message = Option(failedPod.getStatus.getMessage)
+    val explained = ExecutorPodsLifecycleManager.describeExitCode(exitCode)
+    val exitMsg = s"The executor with id $execId exited with exit code $explained."
+    val reasonStr = reason.map(r => s"The API gave the following brief reason: ${r}")
+    val msgStr = message.map(m => s"The API gave the following message: ${m}")
+
+
     s"""
-       |The executor with id $failedExecutorId exited with exit code 1.
-       |The API gave the following brief reason: ${reason.getOrElse("N/A")}
-       |The API gave the following message: ${message.getOrElse("N/A")}
+       |${exitMsg}
+       |${reasonStr.getOrElse("")}
+       |${msgStr.getOrElse("")}
+       |
        |The API gave the following container statuses:
        |
        |${containersDescription(failedPod)}
       """.stripMargin
   }
 
-  private def namedPodsAnswer(): Answer[PodResource[Pod, DoneablePod]] =
+  private def namedPodsAnswer(): Answer[PodResource[Pod]] =
     (invocation: InvocationOnMock) => {
       val podName: String = invocation.getArgument(0)
       namedExecutorPods.getOrElseUpdate(
-        podName, mock(classOf[PodResource[Pod, DoneablePod]]))
+        podName, mock(classOf[PodResource[Pod]]))
     }
 }
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsPollingSnapshotSourceSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsPollingSnapshotSourceSuite.scala
index 63e43bd40c728..b9938f7c02c8d 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsPollingSnapshotSourceSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsPollingSnapshotSourceSuite.scala
@@ -59,7 +59,7 @@ class ExecutorPodsPollingSnapshotSourceSuite extends SparkFunSuite with BeforeAn
   private var pollingSourceUnderTest: ExecutorPodsPollingSnapshotSource = _
 
   before {
-    MockitoAnnotations.initMocks(this)
+    MockitoAnnotations.openMocks(this).close()
     pollingExecutor = new DeterministicScheduler()
     pollingSourceUnderTest = new ExecutorPodsPollingSnapshotSource(
       sparkConf,
@@ -77,13 +77,15 @@ class ExecutorPodsPollingSnapshotSourceSuite extends SparkFunSuite with BeforeAn
   }
 
   test("Items returned by the API should be pushed to the event queue") {
+    val exec1 = runningExecutor(1)
+    val exec2 = runningExecutor(2)
     when(activeExecutorPods.list())
       .thenReturn(new PodListBuilder()
         .addToItems(
-          runningExecutor(1),
-          runningExecutor(2))
+          exec1,
+          exec2)
         .build())
     pollingExecutor.tick(pollingInterval, TimeUnit.MILLISECONDS)
-    verify(eventQueue).replaceSnapshot(Seq(runningExecutor(1), runningExecutor(2)))
+    verify(eventQueue).replaceSnapshot(Seq(exec1, exec2))
   }
 }
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotSuite.scala
index 70e19c904eddb..5e66726927e36 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotSuite.scala
@@ -16,45 +16,72 @@
  */
 package org.apache.spark.scheduler.cluster.k8s
 
+import io.fabric8.kubernetes.api.model.Pod
+
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.scheduler.cluster.k8s.ExecutorLifecycleTestUtils._
 
 class ExecutorPodsSnapshotSuite extends SparkFunSuite {
 
+  def testCase(pod: Pod, state: Pod => ExecutorPodState): (Pod, ExecutorPodState) =
+    (pod, state(pod))
+
+  def doTest(testCases: Seq[(Pod, ExecutorPodState)]): Unit = {
+    val snapshot = ExecutorPodsSnapshot(testCases.map(_._1), 0)
+    for (((_, state), i) <- testCases.zipWithIndex) {
+      assertResult(state.getClass.getName, s"executor ID $i") {
+        snapshot.executorPods(i).getClass.getName
+      }
+    }
+  }
+
   test("States are interpreted correctly from pod metadata.") {
-    val pods = Seq(
-      pendingExecutor(0),
-      runningExecutor(1),
-      succeededExecutor(2),
-      failedExecutorWithoutDeletion(3),
-      deletedExecutor(4),
-      unknownExecutor(5))
-    val snapshot = ExecutorPodsSnapshot(pods)
-    assert(snapshot.executorPods ===
-      Map(
-        0L -> PodPending(pods(0)),
-        1L -> PodRunning(pods(1)),
-        2L -> PodSucceeded(pods(2)),
-        3L -> PodFailed(pods(3)),
-        4L -> PodDeleted(pods(4)),
-        5L -> PodUnknown(pods(5))))
+    ExecutorPodsSnapshot.setShouldCheckAllContainers(false)
+    val testCases = Seq(
+      testCase(pendingExecutor(0), PodPending),
+      testCase(runningExecutor(1), PodRunning),
+      testCase(succeededExecutor(2), PodSucceeded),
+      testCase(failedExecutorWithoutDeletion(3), PodFailed),
+      testCase(deletedExecutor(4), PodDeleted),
+      testCase(unknownExecutor(5), PodUnknown),
+      testCase(finishedExecutorWithRunningSidecar(6, 0), PodSucceeded),
+      testCase(finishedExecutorWithRunningSidecar(7, 1), PodFailed)
+    )
+    doTest(testCases)
+  }
+
+  test("SPARK-30821: States are interpreted correctly from pod metadata"
+    + " when configured to check all containers.") {
+    ExecutorPodsSnapshot.setShouldCheckAllContainers(true)
+    val testCases = Seq(
+      testCase(pendingExecutor(0), PodPending),
+      testCase(runningExecutor(1), PodRunning),
+      testCase(runningExecutorWithFailedContainer(2), PodFailed),
+      testCase(succeededExecutor(3), PodSucceeded),
+      testCase(failedExecutorWithoutDeletion(4), PodFailed),
+      testCase(deletedExecutor(5), PodDeleted),
+      testCase(unknownExecutor(6), PodUnknown)
+    )
+    doTest(testCases)
   }
 
   test("Updates add new pods for non-matching ids and edit existing pods for matching ids") {
+    ExecutorPodsSnapshot.setShouldCheckAllContainers(false)
     val originalPods = Seq(
       pendingExecutor(0),
       runningExecutor(1))
-    val originalSnapshot = ExecutorPodsSnapshot(originalPods)
+    val originalSnapshot = ExecutorPodsSnapshot(originalPods, 0)
     val snapshotWithUpdatedPod = originalSnapshot.withUpdate(succeededExecutor(1))
     assert(snapshotWithUpdatedPod.executorPods ===
       Map(
         0L -> PodPending(originalPods(0)),
         1L -> PodSucceeded(succeededExecutor(1))))
-    val snapshotWithNewPod = snapshotWithUpdatedPod.withUpdate(pendingExecutor(2))
+    val pendingExec = pendingExecutor(2)
+    val snapshotWithNewPod = snapshotWithUpdatedPod.withUpdate(pendingExec)
     assert(snapshotWithNewPod.executorPods ===
       Map(
         0L -> PodPending(originalPods(0)),
         1L -> PodSucceeded(succeededExecutor(1)),
-        2L -> PodPending(pendingExecutor(2))))
+        2L -> PodPending(pendingExec)))
   }
 }
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreSuite.scala
index cf54b3c4eb329..b4240bbacf106 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreSuite.scala
@@ -26,15 +26,19 @@ import scala.collection.mutable
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.deploy.k8s.Constants._
+import org.apache.spark.util.ManualClock
 
 class ExecutorPodsSnapshotsStoreSuite extends SparkFunSuite with BeforeAndAfter {
 
   private var eventBufferScheduler: DeterministicScheduler = _
   private var eventQueueUnderTest: ExecutorPodsSnapshotsStoreImpl = _
+  private var clock: ManualClock = _
 
   before {
     eventBufferScheduler = new DeterministicScheduler()
-    eventQueueUnderTest = new ExecutorPodsSnapshotsStoreImpl(eventBufferScheduler)
+    clock = new ManualClock()
+    eventQueueUnderTest = new ExecutorPodsSnapshotsStoreImpl(eventBufferScheduler, clock)
+    ExecutorPodsSnapshot.setShouldCheckAllContainers(false)
   }
 
   test("Subscribers get notified of events periodically.") {
@@ -51,6 +55,7 @@ class ExecutorPodsSnapshotsStoreSuite extends SparkFunSuite with BeforeAndAfter
     assert(receivedSnapshots1 === Seq(ExecutorPodsSnapshot()))
     assert(receivedSnapshots2 === Seq(ExecutorPodsSnapshot()))
 
+    clock.advance(100)
     pushPodWithIndex(1)
     // Force time to move forward so that the buffer is emitted, scheduling the
     // processing task on the subscription executor...
@@ -59,7 +64,7 @@ class ExecutorPodsSnapshotsStoreSuite extends SparkFunSuite with BeforeAndAfter
 
     assert(receivedSnapshots1 === Seq(
       ExecutorPodsSnapshot(),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1)))))
+      ExecutorPodsSnapshot(Seq(podWithIndex(1)), 0)))
     assert(receivedSnapshots2 === Seq(ExecutorPodsSnapshot()))
 
     eventBufferScheduler.tick(1000, TimeUnit.MILLISECONDS)
@@ -67,29 +72,29 @@ class ExecutorPodsSnapshotsStoreSuite extends SparkFunSuite with BeforeAndAfter
     // Don't repeat snapshots
     assert(receivedSnapshots1 === Seq(
       ExecutorPodsSnapshot(),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1)))))
+      ExecutorPodsSnapshot(Seq(podWithIndex(1)), 0)))
     assert(receivedSnapshots2 === Seq(
       ExecutorPodsSnapshot(),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1)))))
+      ExecutorPodsSnapshot(Seq(podWithIndex(1)), 0)))
     pushPodWithIndex(2)
     pushPodWithIndex(3)
     eventBufferScheduler.tick(1000, TimeUnit.MILLISECONDS)
 
     assert(receivedSnapshots1 === Seq(
       ExecutorPodsSnapshot(),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1))),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1), podWithIndex(2))),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1), podWithIndex(2), podWithIndex(3)))))
+      ExecutorPodsSnapshot(Seq(podWithIndex(1)), 0),
+      ExecutorPodsSnapshot(Seq(podWithIndex(1), podWithIndex(2)), 0),
+      ExecutorPodsSnapshot(Seq(podWithIndex(1), podWithIndex(2), podWithIndex(3)), 0)))
     assert(receivedSnapshots2 === Seq(
       ExecutorPodsSnapshot(),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1)))))
+      ExecutorPodsSnapshot(Seq(podWithIndex(1)), 0)))
 
     eventBufferScheduler.tick(1000, TimeUnit.MILLISECONDS)
     assert(receivedSnapshots1 === Seq(
       ExecutorPodsSnapshot(),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1))),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1), podWithIndex(2))),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1), podWithIndex(2), podWithIndex(3)))))
+      ExecutorPodsSnapshot(Seq(podWithIndex(1)), 0),
+      ExecutorPodsSnapshot(Seq(podWithIndex(1), podWithIndex(2)), 0),
+      ExecutorPodsSnapshot(Seq(podWithIndex(1), podWithIndex(2), podWithIndex(3)), 0)))
     assert(receivedSnapshots1 === receivedSnapshots2)
   }
 
@@ -112,13 +117,14 @@ class ExecutorPodsSnapshotsStoreSuite extends SparkFunSuite with BeforeAndAfter
     eventBufferScheduler.tick(1000, TimeUnit.MILLISECONDS)
     assert(receivedSnapshots === Seq(
       ExecutorPodsSnapshot(),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1)))))
+      ExecutorPodsSnapshot(Seq(podWithIndex(1)), 0)))
+    clock.advance(100)
     eventQueueUnderTest.replaceSnapshot(Seq(podWithIndex(2)))
     eventBufferScheduler.tick(1000, TimeUnit.MILLISECONDS)
     assert(receivedSnapshots === Seq(
       ExecutorPodsSnapshot(),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1))),
-      ExecutorPodsSnapshot(Seq(podWithIndex(2)))))
+      ExecutorPodsSnapshot(Seq(podWithIndex(1)), 0),
+      ExecutorPodsSnapshot(Seq(podWithIndex(2)), 100)))
   }
 
   private def pushPodWithIndex(index: Int): Unit =
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsWatchSnapshotSourceSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsWatchSnapshotSourceSuite.scala
index ac1968b4ff810..cddb5f6da44f4 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsWatchSnapshotSourceSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsWatchSnapshotSourceSuite.scala
@@ -53,7 +53,7 @@ class ExecutorPodsWatchSnapshotSourceSuite extends SparkFunSuite with BeforeAndA
   private var watchSourceUnderTest: ExecutorPodsWatchSnapshotSource = _
 
   before {
-    MockitoAnnotations.initMocks(this)
+    MockitoAnnotations.openMocks(this).close()
     watch = ArgumentCaptor.forClass(classOf[Watcher[Pod]])
     when(kubernetesClient.pods()).thenReturn(podOperations)
     when(podOperations.withLabel(SPARK_APP_ID_LABEL, TEST_SPARK_APP_ID))
@@ -67,9 +67,11 @@ class ExecutorPodsWatchSnapshotSourceSuite extends SparkFunSuite with BeforeAndA
   }
 
   test("Watch events should be pushed to the snapshots store as snapshot updates.") {
-    watch.getValue.eventReceived(Action.ADDED, runningExecutor(1))
-    watch.getValue.eventReceived(Action.MODIFIED, runningExecutor(2))
-    verify(eventQueue).updatePod(runningExecutor(1))
-    verify(eventQueue).updatePod(runningExecutor(2))
+    val exec1 = runningExecutor(1)
+    val exec2 = runningExecutor(2)
+    watch.getValue.eventReceived(Action.ADDED, exec1)
+    watch.getValue.eventReceived(Action.MODIFIED, exec2)
+    verify(eventQueue).updatePod(exec1)
+    verify(eventQueue).updatePod(exec2)
   }
 }
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackendSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackendSuite.scala
index 8c683e85dd5e2..5dd84e853620e 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackendSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackendSuite.scala
@@ -31,10 +31,10 @@ import org.apache.spark.{SparkConf, SparkContext, SparkEnv, SparkFunSuite}
 import org.apache.spark.deploy.k8s.Config._
 import org.apache.spark.deploy.k8s.Constants._
 import org.apache.spark.deploy.k8s.Fabric8Aliases._
-import org.apache.spark.resource.ResourceProfileManager
+import org.apache.spark.resource.{ResourceProfile, ResourceProfileManager}
 import org.apache.spark.rpc.{RpcEndpoint, RpcEndpointRef, RpcEnv}
-import org.apache.spark.scheduler.{ExecutorKilled, TaskSchedulerImpl}
-import org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages.RemoveExecutor
+import org.apache.spark.scheduler.{ExecutorKilled, LiveListenerBus, TaskSchedulerImpl}
+import org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages.{RegisterExecutor, RemoveExecutor, StopDriver}
 import org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend
 import org.apache.spark.scheduler.cluster.k8s.ExecutorLifecycleTestUtils.TEST_SPARK_APP_ID
 
@@ -66,6 +66,12 @@ class KubernetesClusterSchedulerBackendSuite extends SparkFunSuite with BeforeAn
   @Mock
   private var labeledPods: LABELED_PODS = _
 
+  @Mock
+  private var configMapsOperations: CONFIG_MAPS = _
+
+  @Mock
+  private var labledConfigMaps: LABELED_CONFIG_MAPS = _
+
   @Mock
   private var taskScheduler: TaskSchedulerImpl = _
 
@@ -87,10 +93,12 @@ class KubernetesClusterSchedulerBackendSuite extends SparkFunSuite with BeforeAn
   private var driverEndpoint: ArgumentCaptor[RpcEndpoint] = _
   private var schedulerBackendUnderTest: KubernetesClusterSchedulerBackend = _
 
-  private val resourceProfileManager = new ResourceProfileManager(sparkConf)
+  private val listenerBus = new LiveListenerBus(new SparkConf())
+  private val resourceProfileManager = new ResourceProfileManager(sparkConf, listenerBus)
+  private val defaultProfile = ResourceProfile.getOrCreateDefaultProfile(sparkConf)
 
   before {
-    MockitoAnnotations.initMocks(this)
+    MockitoAnnotations.openMocks(this).close()
     when(taskScheduler.sc).thenReturn(sc)
     when(sc.conf).thenReturn(sparkConf)
     when(sc.resourceProfileManager).thenReturn(resourceProfileManager)
@@ -103,6 +111,8 @@ class KubernetesClusterSchedulerBackendSuite extends SparkFunSuite with BeforeAn
         driverEndpoint.capture()))
       .thenReturn(driverEndpointRef)
     when(kubernetesClient.pods()).thenReturn(podOperations)
+    when(kubernetesClient.configMaps()).thenReturn(configMapsOperations)
+    when(podAllocator.driverPod).thenReturn(None)
     schedulerBackendUnderTest = new KubernetesClusterSchedulerBackend(
       taskScheduler,
       sc,
@@ -117,21 +127,27 @@ class KubernetesClusterSchedulerBackendSuite extends SparkFunSuite with BeforeAn
 
   test("Start all components") {
     schedulerBackendUnderTest.start()
-    verify(podAllocator).setTotalExpectedExecutors(3)
-    verify(podAllocator).start(TEST_SPARK_APP_ID)
+    verify(podAllocator).setTotalExpectedExecutors(Map(defaultProfile -> 3))
+    verify(podAllocator).start(TEST_SPARK_APP_ID, schedulerBackendUnderTest)
     verify(lifecycleEventHandler).start(schedulerBackendUnderTest)
     verify(watchEvents).start(TEST_SPARK_APP_ID)
     verify(pollEvents).start(TEST_SPARK_APP_ID)
+    verify(configMapsOperations).create(any())
   }
 
   test("Stop all components") {
     when(podOperations.withLabel(SPARK_APP_ID_LABEL, TEST_SPARK_APP_ID)).thenReturn(labeledPods)
     when(labeledPods.withLabel(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE)).thenReturn(labeledPods)
+    when(configMapsOperations.withLabel(SPARK_APP_ID_LABEL, TEST_SPARK_APP_ID))
+      .thenReturn(labledConfigMaps)
+    when(labledConfigMaps.withLabel(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE))
+      .thenReturn(labledConfigMaps)
     schedulerBackendUnderTest.stop()
     verify(eventQueue).stop()
     verify(watchEvents).stop()
     verify(pollEvents).stop()
     verify(labeledPods).delete()
+    verify(labledConfigMaps).delete()
     verify(kubernetesClient).close()
   }
 
@@ -142,10 +158,10 @@ class KubernetesClusterSchedulerBackendSuite extends SparkFunSuite with BeforeAn
 
     backend.start()
     backend.doRemoveExecutor("1", ExecutorKilled)
-    verify(driverEndpointRef, never()).send(RemoveExecutor("1", ExecutorKilled))
+    verify(driverEndpointRef).send(RemoveExecutor("1", ExecutorKilled))
 
     backend.doRemoveExecutor("2", ExecutorKilled)
-    verify(driverEndpointRef, never()).send(RemoveExecutor("1", ExecutorKilled))
+    verify(driverEndpointRef).send(RemoveExecutor("2", ExecutorKilled))
   }
 
   test("Kill executors") {
@@ -174,4 +190,21 @@ class KubernetesClusterSchedulerBackendSuite extends SparkFunSuite with BeforeAn
       TimeUnit.MILLISECONDS)
     verify(labeledPods).delete()
   }
+
+  test("SPARK-34407: CoarseGrainedSchedulerBackend.stop may throw SparkException") {
+    schedulerBackendUnderTest.start()
+
+    when(driverEndpointRef.askSync[Boolean](StopDriver)).thenThrow(new RuntimeException)
+    schedulerBackendUnderTest.stop()
+
+    // Verify the last operation of `schedulerBackendUnderTest.stop`.
+    verify(kubernetesClient).close()
+  }
+
+  test("SPARK-34469: Ignore RegisterExecutor when SparkContext is stopped") {
+    when(sc.isStopped).thenReturn(true)
+    val endpoint = schedulerBackendUnderTest.createDriverEndpoint()
+    endpoint.receiveAndReply(null).apply(
+      RegisterExecutor("1", null, "host1", 1, Map.empty, Map.empty, Map.empty, 0))
+  }
 }
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala
index bd716174a8271..ec60c6fc0bf82 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala
@@ -21,6 +21,7 @@ import io.fabric8.kubernetes.client.KubernetesClient
 import org.apache.spark.{SecurityManager, SparkConf}
 import org.apache.spark.deploy.k8s._
 import org.apache.spark.internal.config.ConfigEntry
+import org.apache.spark.resource.ResourceProfile
 
 class KubernetesExecutorBuilderSuite extends PodBuilderSuite {
 
@@ -28,11 +29,16 @@ class KubernetesExecutorBuilderSuite extends PodBuilderSuite {
     Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE
   }
 
+  override protected def userFeatureStepsConf: ConfigEntry[_] = {
+    Config.KUBERNETES_EXECUTOR_POD_FEATURE_STEPS
+  }
+
   override protected def buildPod(sparkConf: SparkConf, client: KubernetesClient): SparkPod = {
     sparkConf.set("spark.driver.host", "https://driver.host.com")
     val conf = KubernetesTestConf.createExecutorConf(sparkConf = sparkConf)
     val secMgr = new SecurityManager(sparkConf)
-    new KubernetesExecutorBuilder().buildFromFeatures(conf, secMgr, client)
+    val defaultProfile = ResourceProfile.getOrCreateDefaultProfile(sparkConf)
+    new KubernetesExecutorBuilder().buildFromFeatures(conf, secMgr, client, defaultProfile).pod
   }
 
 }
diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
index b722123195dc2..8c3db7e243d8b 100644
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-ARG java_image_tag=8-jre-slim
+ARG java_image_tag=11-jre-slim
 
 FROM openjdk:${java_image_tag}
 
@@ -28,7 +28,7 @@ ARG spark_uid=185
 # docker build -t spark:latest -f kubernetes/dockerfiles/spark/Dockerfile .
 
 RUN set -ex && \
-    sed -i 's/http:/https:/g' /etc/apt/sources.list && \
+    sed -i 's/http:\/\/deb.\(.*\)/https:\/\/deb.\1/g' /etc/apt/sources.list && \
     apt-get update && \
     ln -s /lib /lib64 && \
     apt install -y bash tini libc6 libpam-modules krb5-user libnss3 procps && \
diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/R/Dockerfile b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/R/Dockerfile
index 59f375b707ca7..2dd4d8c4d1e95 100644
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/R/Dockerfile
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/R/Dockerfile
@@ -25,7 +25,15 @@ USER 0
 
 RUN mkdir ${SPARK_HOME}/R
 
-RUN apt-get update && apt install -y r-base r-base-dev && rm -rf /var/cache/apt/*
+# Install R 3.6.3 (http://cloud.r-project.org/bin/linux/debian/)
+RUN \
+  apt-get update && \
+  apt install -y gnupg && \
+  echo "deb http://cloud.r-project.org/bin/linux/debian buster-cran35/" >> /etc/apt/sources.list && \
+  (apt-key adv --keyserver keys.gnupg.net --recv-key 'E19F5F87128899B192B1A2C2AD5F960A256A04AF' || apt-key adv --keyserver keys.openpgp.org --recv-key 'E19F5F87128899B192B1A2C2AD5F960A256A04AF') && \
+  apt-get update && \
+  apt install -y -t buster-cran35 r-base r-base-dev && \
+  rm -rf /var/cache/apt/*
 
 COPY R ${SPARK_HOME}/R
 ENV R_HOME /usr/lib/R
diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/python/Dockerfile b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/python/Dockerfile
index 8dfc5f7ff60c5..2f082f559ca5c 100644
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/python/Dockerfile
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/python/Dockerfile
@@ -24,15 +24,9 @@ WORKDIR /
 USER 0
 
 RUN mkdir ${SPARK_HOME}/python
-# TODO: Investigate running both pip and pip3 via virtualenvs
 RUN apt-get update && \
-    apt install -y python python-pip && \
     apt install -y python3 python3-pip && \
-    # We remove ensurepip since it adds no functionality since pip is
-    # installed on the image and it just takes up 1.6MB on the image
-    rm -r /usr/lib/python*/ensurepip && \
-    pip install --upgrade pip setuptools && \
-    # You may install with python3 packages by using pip3.6
+    pip3 install --upgrade pip setuptools && \
     # Removed the .cache to save space
     rm -r /root/.cache && rm -rf /var/cache/apt/*
 
diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/decom.sh b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/decom.sh
index 8a5208d49a70f..cd973df257f07 100755
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/decom.sh
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/decom.sh
@@ -32,4 +32,4 @@ timeout 60 tail --pid=${WORKER_PID} -f /dev/null
 date
 echo "Done"
 date
-sleep 30
+sleep 1
diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
index 813a70c6e7ec3..f722471906bfb 100755
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
@@ -44,16 +44,11 @@ if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
   SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
 fi
 
-if [ "$PYSPARK_MAJOR_PYTHON_VERSION" == "2" ]; then
-    pyv="$(python -V 2>&1)"
-    export PYTHON_VERSION="${pyv:7}"
-    export PYSPARK_PYTHON="python"
-    export PYSPARK_DRIVER_PYTHON="python"
-elif [ "$PYSPARK_MAJOR_PYTHON_VERSION" == "3" ]; then
-    pyv3="$(python3 -V 2>&1)"
-    export PYTHON_VERSION="${pyv3:7}"
-    export PYSPARK_PYTHON="python3"
-    export PYSPARK_DRIVER_PYTHON="python3"
+if ! [ -z ${PYSPARK_PYTHON+x} ]; then
+    export PYSPARK_PYTHON
+fi
+if ! [ -z ${PYSPARK_DRIVER_PYTHON+x} ]; then
+    export PYSPARK_DRIVER_PYTHON
 fi
 
 # If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor.
@@ -66,6 +61,12 @@ if ! [ -z ${HADOOP_CONF_DIR+x} ]; then
   SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH";
 fi
 
+if ! [ -z ${SPARK_CONF_DIR+x} ]; then
+  SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH";
+elif ! [ -z ${SPARK_HOME+x} ]; then
+  SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
+fi
+
 case "$1" in
   driver)
     shift 1
@@ -90,6 +91,7 @@ case "$1" in
       --cores $SPARK_EXECUTOR_CORES
       --app-id $SPARK_APPLICATION_ID
       --hostname $SPARK_EXECUTOR_POD_IP
+      --resourceProfileId $SPARK_RESOURCE_PROFILE_ID
     )
     ;;
 
diff --git a/resource-managers/kubernetes/integration-tests/README.md b/resource-managers/kubernetes/integration-tests/README.md
index 18b91916208d6..9a0689f7c451d 100644
--- a/resource-managers/kubernetes/integration-tests/README.md
+++ b/resource-managers/kubernetes/integration-tests/README.md
@@ -17,7 +17,11 @@ To run tests with Java 11 instead of Java 8, use `--java-image-tag` to specify t
 
     ./dev/dev-run-integration-tests.sh --java-image-tag 11-jre-slim
 
-The minimum tested version of Minikube is 0.23.0. The kube-dns addon must be enabled. Minikube should
+To run tests with Hadoop 2.7 instead of Hadoop 3.2, use `--hadoop-profile`.
+
+    ./dev/dev-run-integration-tests.sh --hadoop-profile hadoop-2.7
+
+The minimum tested version of Minikube is 1.7.3. The kube-dns addon must be enabled. Minikube should
 run with a minimum of 4 CPUs and 6G of memory:
 
     minikube start --cpus 4 --memory 6144
@@ -36,7 +40,7 @@ default this is set to `minikube`, the available backends are their prerequisite
 
 ### `minikube`
 
-Uses the local `minikube` cluster, this requires that `minikube` 0.23.0 or greater be installed and that it be allocated 
+Uses the local `minikube` cluster, this requires that `minikube` 1.7.3 or greater be installed and that it be allocated
 at least 4 CPUs and 6GB memory (some users have reported success with as few as 3 CPUs and 4GB memory).  The tests will 
 check if `minikube` is started and abort early if it isn't currently running.
 
@@ -122,7 +126,7 @@ If you prefer to run just the integration tests directly, then you can customise
 properties to Maven.  For example:
 
     mvn integration-test -am -pl :spark-kubernetes-integration-tests_2.12 \
-                            -Pkubernetes -Pkubernetes-integration-tests \ 
+                            -Pkubernetes -Pkubernetes-integration-tests \
                             -Phadoop-2.7 -Dhadoop.version=2.7.4 \
                             -Dspark.kubernetes.test.sparkTgz=spark-3.0.0-SNAPSHOT-bin-example.tgz \
                             -Dspark.kubernetes.test.imageTag=sometag \
diff --git a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
index 292abe91d35b6..8d10985b4f26a 100755
--- a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
+++ b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
@@ -35,6 +35,8 @@ CONTEXT=
 INCLUDE_TAGS="k8s"
 EXCLUDE_TAGS=
 JAVA_VERSION="8"
+BUILD_DEPENDENCIES_MVN_FLAG="-am"
+HADOOP_PROFILE="hadoop-3.2"
 MVN="$TEST_ROOT_DIR/build/mvn"
 
 SCALA_VERSION=$("$MVN" help:evaluate -Dexpression=scala.binary.version 2>/dev/null\
@@ -112,6 +114,13 @@ while (( "$#" )); do
       JAVA_VERSION="$2"
       shift
       ;;
+    --hadoop-profile)
+      HADOOP_PROFILE="$2"
+      shift
+      ;;
+    --skip-building-dependencies)
+      BUILD_DEPENDENCIES_MVN_FLAG=""
+      ;;
     *)
       echo "Unexpected command line flag $2 $1."
       exit 1
@@ -171,4 +180,4 @@ properties+=(
   -Dlog4j.logger.org.apache.spark=DEBUG
 )
 
-$TEST_ROOT_DIR/build/mvn integration-test -f $TEST_ROOT_DIR/pom.xml -pl resource-managers/kubernetes/integration-tests -am -Pscala-$SCALA_VERSION -Pkubernetes -Pkubernetes-integration-tests ${properties[@]}
+$TEST_ROOT_DIR/build/mvn install -f $TEST_ROOT_DIR/pom.xml -pl resource-managers/kubernetes/integration-tests $BUILD_DEPENDENCIES_MVN_FLAG -Pscala-$SCALA_VERSION -P$HADOOP_PROFILE -Pkubernetes -Pkubernetes-integration-tests ${properties[@]}
diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
index 7a889c427b41e..987dfd25f7c54 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -20,18 +20,14 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
   <artifactId>spark-kubernetes-integration-tests_2.12</artifactId>
   <properties>
     <download-maven-plugin.version>1.3.0</download-maven-plugin.version>
-    <exec-maven-plugin.version>1.4.0</exec-maven-plugin.version>
     <extraScalaTestArgs></extraScalaTestArgs>
-    <kubernetes-client.version>4.7.1</kubernetes-client.version>
-    <scala-maven-plugin.version>3.2.2</scala-maven-plugin.version>
-    <scalatest-maven-plugin.version>1.0</scalatest-maven-plugin.version>
     <sbt.project.name>kubernetes-integration-tests</sbt.project.name>
 
     <!-- Integration Test Configuration Properties -->
@@ -77,12 +73,6 @@
       <artifactId>spark-tags_${scala.binary.version}</artifactId>
       <type>test-jar</type>
     </dependency>
-    <dependency>
-      <groupId>com.amazonaws</groupId>
-      <artifactId>aws-java-sdk</artifactId>
-      <version>1.7.4</version>
-      <scope>test</scope>
-    </dependency>
   </dependencies>
 
   <build>
@@ -121,6 +111,9 @@
 
                 <argument>--spark-tgz</argument>
                 <argument>${spark.kubernetes.test.sparkTgz}</argument>
+
+                <argument>--test-exclude-tags</argument>
+                <argument>"${test.exclude.tags}"</argument>
               </arguments>
             </configuration>
           </execution>
@@ -186,4 +179,31 @@
 
   </build>
 
+  <profiles>
+    <profile>
+      <id>hadoop-2.7</id>
+      <dependencies>
+        <dependency>
+          <groupId>com.amazonaws</groupId>
+          <artifactId>aws-java-sdk</artifactId>
+          <version>1.7.4</version>
+          <scope>test</scope>
+        </dependency>
+      </dependencies>
+    </profile>
+    <profile>
+      <id>hadoop-3.2</id>
+      <activation>
+        <activeByDefault>true</activeByDefault>
+      </activation>
+      <dependencies>
+        <dependency>
+          <groupId>com.amazonaws</groupId>
+          <artifactId>aws-java-sdk-bundle</artifactId>
+          <version>1.11.375</version>
+          <scope>test</scope>
+        </dependency>
+      </dependencies>
+    </profile>
+  </profiles>
 </project>
diff --git a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh
index ab906604fce06..beda56cf37c94 100755
--- a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh
+++ b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh
@@ -25,6 +25,8 @@ IMAGE_REPO="docker.io/kubespark"
 IMAGE_TAG="N/A"
 JAVA_IMAGE_TAG="8-jre-slim"
 SPARK_TGZ="N/A"
+MVN="$TEST_ROOT_DIR/build/mvn"
+EXCLUDE_TAGS=""
 
 # Parse arguments
 while (( "$#" )); do
@@ -57,6 +59,10 @@ while (( "$#" )); do
       SPARK_TGZ="$2"
       shift
       ;;
+    --test-exclude-tags)
+      EXCLUDE_TAGS="$2"
+      shift
+      ;;
     *)
       break
       ;;
@@ -84,7 +90,11 @@ fi
 # If there is a specific Spark image skip building and extraction/copy
 if [[ $IMAGE_TAG == "N/A" ]];
 then
-  IMAGE_TAG=$(uuidgen);
+  VERSION=$("$MVN" help:evaluate -Dexpression=project.version \
+    | grep -v "INFO"\
+    | grep -v "WARNING"\
+    | tail -n 1)
+  IMAGE_TAG=${VERSION}_$(uuidgen)
   cd $SPARK_INPUT_DIR
 
   # OpenJDK base-image tag (e.g. 8-jre-slim, 11-jre-slim)
@@ -94,7 +104,10 @@ then
   LANGUAGE_BINDING_BUILD_ARGS="-p $DOCKER_FILE_BASE_PATH/bindings/python/Dockerfile"
 
   # Build SparkR image
-  LANGUAGE_BINDING_BUILD_ARGS="$LANGUAGE_BINDING_BUILD_ARGS -R $DOCKER_FILE_BASE_PATH/bindings/R/Dockerfile"
+  tags=(${EXCLUDE_TAGS//,/ })
+  if [[ ! ${tags[@]} =~ "r" ]]; then
+    LANGUAGE_BINDING_BUILD_ARGS="$LANGUAGE_BINDING_BUILD_ARGS -R $DOCKER_FILE_BASE_PATH/bindings/R/Dockerfile"
+  fi
 
   # Unset SPARK_HOME to let the docker-image-tool script detect SPARK_HOME. Otherwise, it cannot
   # indicate the unpacked directory as its home. See SPARK-28550.
diff --git a/resource-managers/kubernetes/integration-tests/src/test/resources/log-config-test-log4j.properties b/resource-managers/kubernetes/integration-tests/src/test/resources/log-config-test-log4j.properties
new file mode 100644
index 0000000000000..d3e13d8542ba1
--- /dev/null
+++ b/resource-managers/kubernetes/integration-tests/src/test/resources/log-config-test-log4j.properties
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This log4j config file is for integration test SparkConfPropagateSuite.
+log4j.rootCategory=DEBUG, console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c: %m%n
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
index 76221e46d1cf4..1c121236303f4 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
@@ -18,6 +18,7 @@ package org.apache.spark.deploy.k8s.integrationtest
 
 import io.fabric8.kubernetes.api.model.Pod
 
+import org.apache.spark.TestUtils
 import org.apache.spark.launcher.SparkLauncher
 
 private[spark] trait BasicTestsSuite { k8sSuite: KubernetesSuite =>
@@ -99,9 +100,12 @@ private[spark] trait BasicTestsSuite { k8sSuite: KubernetesSuite =>
   }
 
   test("Run SparkRemoteFileTest using a remote data file", k8sTestTag) {
-    sparkAppConf
-      .set("spark.files", REMOTE_PAGE_RANK_DATA_FILE)
-    runSparkRemoteCheckAndVerifyCompletion(appArgs = Array(REMOTE_PAGE_RANK_FILE_NAME))
+    assert(sys.props.contains("spark.test.home"), "spark.test.home is not set!")
+    TestUtils.withHttpServer(sys.props("spark.test.home")) { baseURL =>
+      sparkAppConf
+        .set("spark.files", baseURL.toString + REMOTE_PAGE_RANK_DATA_FILE)
+      runSparkRemoteCheckAndVerifyCompletion(appArgs = Array(REMOTE_PAGE_RANK_FILE_NAME))
+    }
   }
 }
 
@@ -110,7 +114,6 @@ private[spark] object BasicTestsSuite {
   val CONTAINER_LOCAL_FILE_DOWNLOAD_PATH = "/var/spark-data/spark-files"
   val CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE =
      s"$CONTAINER_LOCAL_FILE_DOWNLOAD_PATH/pagerank_data.txt"
-  val REMOTE_PAGE_RANK_DATA_FILE =
-    "https://raw.githubusercontent.com/apache/spark/master/data/mllib/pagerank_data.txt"
+  val REMOTE_PAGE_RANK_DATA_FILE = "data/mllib/pagerank_data.txt"
   val REMOTE_PAGE_RANK_FILE_NAME = "pagerank_data.txt"
 }
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ClientModeTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ClientModeTestsSuite.scala
index 4e205f69ce039..1a9724afe30c9 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ClientModeTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ClientModeTestsSuite.scala
@@ -16,6 +16,7 @@
  */
 package org.apache.spark.deploy.k8s.integrationtest
 
+import io.fabric8.kubernetes.api.model.{PodBuilder, ServiceBuilder}
 import org.scalatest.concurrent.Eventually
 import scala.collection.JavaConverters._
 
@@ -32,7 +33,7 @@ private[spark] trait ClientModeTestsSuite { k8sSuite: KubernetesSuite =>
       .getKubernetesClient
       .services()
       .inNamespace(kubernetesTestComponents.namespace)
-      .createNew()
+      .create(new ServiceBuilder()
         .withNewMetadata()
           .withName(s"$driverPodName-svc")
           .endMetadata()
@@ -50,13 +51,13 @@ private[spark] trait ClientModeTestsSuite { k8sSuite: KubernetesSuite =>
             .withNewTargetPort(blockManagerPort)
             .endPort()
           .endSpec()
-        .done()
+        .build())
     try {
       val driverPod = testBackend
         .getKubernetesClient
         .pods()
         .inNamespace(kubernetesTestComponents.namespace)
-        .createNew()
+        .create(new PodBuilder()
           .withNewMetadata()
           .withName(driverPodName)
           .withLabels(labels.asJava)
@@ -93,7 +94,7 @@ private[spark] trait ClientModeTestsSuite { k8sSuite: KubernetesSuite =>
             .addToArgs("10")
             .endContainer()
           .endSpec()
-        .done()
+        .build())
       Eventually.eventually(TIMEOUT, INTERVAL) {
         assert(kubernetesTestComponents.kubernetesClient
           .pods()
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DecommissionSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DecommissionSuite.scala
index becf9415c7506..75c27f64e3307 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DecommissionSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DecommissionSuite.scala
@@ -16,7 +16,11 @@
  */
 package org.apache.spark.deploy.k8s.integrationtest
 
-import org.apache.spark.internal.config.Worker
+import org.scalatest.concurrent.PatienceConfiguration
+import org.scalatest.time.Minutes
+import org.scalatest.time.Span
+
+import org.apache.spark.internal.config
 
 private[spark] trait DecommissionSuite { k8sSuite: KubernetesSuite =>
 
@@ -25,21 +29,125 @@ private[spark] trait DecommissionSuite { k8sSuite: KubernetesSuite =>
 
   test("Test basic decommissioning", k8sTestTag) {
     sparkAppConf
-      .set(Worker.WORKER_DECOMMISSION_ENABLED.key, "true")
-      .set("spark.kubernetes.pyspark.pythonVersion", "3")
+      .set(config.DECOMMISSION_ENABLED.key, "true")
+      .set("spark.kubernetes.container.image", pyImage)
+      .set(config.STORAGE_DECOMMISSION_ENABLED.key, "true")
+      .set(config.STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED.key, "true")
+      .set(config.STORAGE_DECOMMISSION_RDD_BLOCKS_ENABLED.key, "true")
+      // Ensure we have somewhere to migrate our data too
+      .set("spark.executor.instances", "3")
+      // The default of 30 seconds is fine, but for testing we just want to get this done fast.
+      .set("spark.storage.decommission.replicationReattemptInterval", "1")
+
+    runSparkApplicationAndVerifyCompletion(
+      appResource = PYSPARK_DECOMISSIONING,
+      mainClass = "",
+      expectedDriverLogOnCompletion = Seq(
+        "Finished waiting, stopping Spark",
+        "Decommission executors",
+        "Final accumulator value is: 100"),
+      appArgs = Array.empty[String],
+      driverPodChecker = doBasicDriverPyPodCheck,
+      executorPodChecker = doBasicExecutorPyPodCheck,
+      isJVM = false,
+      pyFiles = None,
+      executorPatience = None,
+      decommissioningTest = true)
+  }
+
+  test("Test basic decommissioning with shuffle cleanup", k8sTestTag) {
+    sparkAppConf
+      .set(config.DECOMMISSION_ENABLED.key, "true")
+      .set("spark.kubernetes.container.image", pyImage)
+      .set(config.STORAGE_DECOMMISSION_ENABLED.key, "true")
+      .set(config.STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED.key, "true")
+      .set(config.STORAGE_DECOMMISSION_RDD_BLOCKS_ENABLED.key, "true")
+      .set(config.DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED.key, "true")
+      .set(config.DYN_ALLOCATION_SHUFFLE_TRACKING_TIMEOUT.key, "400")
+      // Ensure we have somewhere to migrate our data too
+      .set("spark.executor.instances", "3")
+      // The default of 30 seconds is fine, but for testing we just want to get this done fast.
+      .set("spark.storage.decommission.replicationReattemptInterval", "1")
+
+    runSparkApplicationAndVerifyCompletion(
+      appResource = PYSPARK_DECOMISSIONING_CLEANUP,
+      mainClass = "",
+      expectedDriverLogOnCompletion = Seq(
+        "Finished waiting, stopping Spark",
+        "Decommission executors"),
+      appArgs = Array.empty[String],
+      driverPodChecker = doBasicDriverPyPodCheck,
+      executorPodChecker = doBasicExecutorPyPodCheck,
+      isJVM = false,
+      pyFiles = None,
+      executorPatience = None,
+      decommissioningTest = true)
+  }
+
+  test("Test decommissioning with dynamic allocation & shuffle cleanups", k8sTestTag) {
+    sparkAppConf
+      .set(config.DECOMMISSION_ENABLED.key, "true")
+      .set("spark.kubernetes.container.image", pyImage)
+      .set(config.STORAGE_DECOMMISSION_ENABLED.key, "true")
+      .set(config.STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED.key, "true")
+      .set(config.STORAGE_DECOMMISSION_RDD_BLOCKS_ENABLED.key, "true")
+      .set(config.DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED.key, "true")
+      .set(config.DYN_ALLOCATION_SHUFFLE_TRACKING_TIMEOUT.key, "30")
+      .set(config.DYN_ALLOCATION_CACHED_EXECUTOR_IDLE_TIMEOUT.key, "30")
+      .set(config.DYN_ALLOCATION_EXECUTOR_IDLE_TIMEOUT.key, "5")
+      .set(config.DYN_ALLOCATION_MIN_EXECUTORS.key, "1")
+      .set(config.DYN_ALLOCATION_INITIAL_EXECUTORS.key, "2")
+      .set(config.DYN_ALLOCATION_ENABLED.key, "true")
+      // The default of 30 seconds is fine, but for testing we just want to get this done fast.
+      .set("spark.storage.decommission.replicationReattemptInterval", "1")
+
+    var execLogs: String = ""
+
+    runSparkApplicationAndVerifyCompletion(
+      appResource = PYSPARK_SCALE,
+      mainClass = "",
+      expectedDriverLogOnCompletion = Seq(
+        "Finished waiting, stopping Spark",
+        "Decommission executors",
+        "Remove reason statistics: (gracefully decommissioned: 1, decommision unfinished: 0, " +
+          "driver killed: 0, unexpectedly exited: 0)."),
+      appArgs = Array.empty[String],
+      driverPodChecker = doBasicDriverPyPodCheck,
+      executorPodChecker = doBasicExecutorPyPodCheck,
+      isJVM = false,
+      pyFiles = None,
+      executorPatience = Some(None, Some(DECOMMISSIONING_FINISHED_TIMEOUT)),
+      decommissioningTest = false)
+  }
+
+  test("Test decommissioning timeouts", k8sTestTag) {
+    sparkAppConf
+      .set(config.DECOMMISSION_ENABLED.key, "true")
       .set("spark.kubernetes.container.image", pyImage)
+      .set(config.STORAGE_DECOMMISSION_ENABLED.key, "true")
+      .set(config.STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED.key, "true")
+      .set(config.STORAGE_DECOMMISSION_RDD_BLOCKS_ENABLED.key, "true")
+      // Ensure we have somewhere to migrate our data too
+      .set("spark.executor.instances", "3")
+      // Set super high so the timeout is triggered
+      .set("spark.storage.decommission.replicationReattemptInterval", "8640000")
+      // Set super low so the timeout is triggered
+      .set(config.EXECUTOR_DECOMMISSION_FORCE_KILL_TIMEOUT.key, "10")
 
     runSparkApplicationAndVerifyCompletion(
       appResource = PYSPARK_DECOMISSIONING,
       mainClass = "",
-      expectedLogOnCompletion = Seq(
+      expectedDriverLogOnCompletion = Seq(
         "Finished waiting, stopping Spark",
-        "decommissioning executor"),
+        "Decommission executors",
+        "failed to decommission in 10, killing",
+        "killed by driver."),
       appArgs = Array.empty[String],
       driverPodChecker = doBasicDriverPyPodCheck,
       executorPodChecker = doBasicExecutorPyPodCheck,
-      appLocator = appLocator,
       isJVM = false,
+      pyFiles = None,
+      executorPatience = None,
       decommissioningTest = true)
   }
 }
@@ -47,4 +155,7 @@ private[spark] trait DecommissionSuite { k8sSuite: KubernetesSuite =>
 private[spark] object DecommissionSuite {
   val TEST_LOCAL_PYSPARK: String = "local:///opt/spark/tests/"
   val PYSPARK_DECOMISSIONING: String = TEST_LOCAL_PYSPARK + "decommissioning.py"
+  val PYSPARK_DECOMISSIONING_CLEANUP: String = TEST_LOCAL_PYSPARK + "decommissioning_cleanup.py"
+  val PYSPARK_SCALE: String = TEST_LOCAL_PYSPARK + "autoscale.py"
+  val DECOMMISSIONING_FINISHED_TIMEOUT = PatienceConfiguration.Timeout(Span(4, Minutes))
 }
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala
index 2d90c06e36390..812fd21f4bc55 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala
@@ -24,13 +24,16 @@ import com.amazonaws.auth.BasicAWSCredentials
 import com.amazonaws.services.s3.AmazonS3Client
 import io.fabric8.kubernetes.api.model._
 import io.fabric8.kubernetes.api.model.apps.StatefulSetBuilder
+import org.apache.hadoop.util.VersionInfo
 import org.scalatest.concurrent.{Eventually, PatienceConfiguration}
 import org.scalatest.time.{Minutes, Span}
 
 import org.apache.spark.SparkException
 import org.apache.spark.deploy.k8s.integrationtest.DepsTestsSuite.{DEPS_TIMEOUT, FILE_CONTENTS, HOST_PATH}
 import org.apache.spark.deploy.k8s.integrationtest.KubernetesSuite.{INTERVAL, MinikubeTag, TIMEOUT}
+import org.apache.spark.deploy.k8s.integrationtest.Utils.getExamplesJarName
 import org.apache.spark.deploy.k8s.integrationtest.backend.minikube.Minikube
+import org.apache.spark.internal.config.{ARCHIVES, PYSPARK_DRIVER_PYTHON, PYSPARK_PYTHON}
 
 private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
   import KubernetesSuite.k8sTestTag
@@ -120,71 +123,134 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
       .endSpec()
       .build()
 
-    kubernetesTestComponents
+    // try until the service from a previous test is deleted
+    Eventually.eventually(TIMEOUT, INTERVAL) (kubernetesTestComponents
       .kubernetesClient
       .services()
-      .create(minioService)
+      .create(minioService))
 
-    kubernetesTestComponents
+    // try until the stateful set of a previous test is deleted
+    Eventually.eventually(TIMEOUT, INTERVAL) (kubernetesTestComponents
       .kubernetesClient
       .apps()
       .statefulSets()
-      .create(minioStatefulSet)
+      .create(minioStatefulSet))
   }
 
- private def deleteMinioStorage(): Unit = {
+  private def deleteMinioStorage(): Unit = {
     kubernetesTestComponents
       .kubernetesClient
       .apps()
       .statefulSets()
       .withName(cName)
+      .withGracePeriod(0)
       .delete()
 
     kubernetesTestComponents
       .kubernetesClient
       .services()
       .withName(svcName)
+      .withGracePeriod(0)
       .delete()
   }
 
   test("Launcher client dependencies", k8sTestTag, MinikubeTag) {
-    val fileName = Utils.createTempFile(FILE_CONTENTS, HOST_PATH)
-    try {
-      setupMinioStorage()
-      val minioUrlStr = getServiceUrl(svcName)
-      val minioUrl = new URL(minioUrlStr)
-      val minioHost = minioUrl.getHost
-      val minioPort = minioUrl.getPort
-      val examplesJar = Utils.getExamplesJarAbsolutePath(sparkHomeDir)
-      sparkAppConf
-        .set("spark.hadoop.fs.s3a.access.key", ACCESS_KEY)
-        .set("spark.hadoop.fs.s3a.secret.key", SECRET_KEY)
-        .set("spark.hadoop.fs.s3a.connection.ssl.enabled", "false")
-        .set("spark.hadoop.fs.s3a.endpoint", s"$minioHost:$minioPort")
-        .set("spark.kubernetes.file.upload.path", s"s3a://$BUCKET")
-        .set("spark.files", s"$HOST_PATH/$fileName")
-        .set("spark.hadoop.fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem")
-        .set("spark.jars.packages", "com.amazonaws:aws-java-sdk:" +
-          "1.7.4,org.apache.hadoop:hadoop-aws:2.7.6")
-        .set("spark.driver.extraJavaOptions", "-Divy.cache.dir=/tmp -Divy.home=/tmp")
-      createS3Bucket(ACCESS_KEY, SECRET_KEY, minioUrlStr)
+    tryDepsTest({
+      val fileName = Utils.createTempFile(FILE_CONTENTS, HOST_PATH)
+      sparkAppConf.set("spark.files", s"$HOST_PATH/$fileName")
+      val examplesJar = Utils.getTestFileAbsolutePath(getExamplesJarName(), sparkHomeDir)
       runSparkRemoteCheckAndVerifyCompletion(appResource = examplesJar,
         appArgs = Array(fileName),
         timeout = Option(DEPS_TIMEOUT))
-    } finally {
-      // make sure this always runs
-      deleteMinioStorage()
+    })
+  }
+
+  test("SPARK-33615: Launcher client archives", k8sTestTag, MinikubeTag) {
+    tryDepsTest {
+      val fileName = Utils.createTempFile(FILE_CONTENTS, HOST_PATH)
+      Utils.createTarGzFile(s"$HOST_PATH/$fileName", s"$HOST_PATH/$fileName.tar.gz")
+      sparkAppConf.set(ARCHIVES.key, s"$HOST_PATH/$fileName.tar.gz#test_tar_gz")
+      val examplesJar = Utils.getTestFileAbsolutePath(getExamplesJarName(), sparkHomeDir)
+      runSparkRemoteCheckAndVerifyCompletion(appResource = examplesJar,
+        appArgs = Array(s"test_tar_gz/$fileName"),
+        timeout = Option(DEPS_TIMEOUT))
     }
   }
 
-  private def extractS3Key(data: String, key: String): String = {
-    data.split("\n")
-      .filter(_.contains(key))
-      .head
-      .split(":")
-      .last
-      .trim
-      .replaceAll("[,|\"]", "")
+  test(
+    "SPARK-33748: Launcher python client respecting PYSPARK_PYTHON", k8sTestTag, MinikubeTag) {
+    val fileName = Utils.createTempFile(
+      """
+        |#!/usr/bin/env bash
+        |export IS_CUSTOM_PYTHON=1
+        |python3 "$@"
+      """.stripMargin, HOST_PATH)
+    Utils.createTarGzFile(s"$HOST_PATH/$fileName", s"$HOST_PATH/$fileName.tgz")
+    sparkAppConf.set(ARCHIVES.key, s"$HOST_PATH/$fileName.tgz#test_env")
+    val pySparkFiles = Utils.getTestFileAbsolutePath("python_executable_check.py", sparkHomeDir)
+    testPython(pySparkFiles,
+      Seq(
+        s"PYSPARK_PYTHON: ./test_env/$fileName",
+        s"PYSPARK_DRIVER_PYTHON: ./test_env/$fileName",
+        "Custom Python used on executor: True",
+        "Custom Python used on driver: True"),
+      env = Map("PYSPARK_PYTHON" -> s"./test_env/$fileName"))
+  }
+
+  test(
+    "SPARK-33748: Launcher python client respecting " +
+      s"${PYSPARK_PYTHON.key} and ${PYSPARK_DRIVER_PYTHON.key}", k8sTestTag, MinikubeTag) {
+    val fileName = Utils.createTempFile(
+      """
+        |#!/usr/bin/env bash
+        |export IS_CUSTOM_PYTHON=1
+        |python3 "$@"
+      """.stripMargin, HOST_PATH)
+    Utils.createTarGzFile(s"$HOST_PATH/$fileName", s"$HOST_PATH/$fileName.tgz")
+    sparkAppConf.set(ARCHIVES.key, s"$HOST_PATH/$fileName.tgz#test_env")
+    sparkAppConf.set(PYSPARK_PYTHON.key, s"./test_env/$fileName")
+    sparkAppConf.set(PYSPARK_DRIVER_PYTHON.key, "python3")
+    val pySparkFiles = Utils.getTestFileAbsolutePath("python_executable_check.py", sparkHomeDir)
+    testPython(pySparkFiles,
+      Seq(
+        s"PYSPARK_PYTHON: ./test_env/$fileName",
+        "PYSPARK_DRIVER_PYTHON: python3",
+        "Custom Python used on executor: True",
+        "Custom Python used on driver: False"))
+  }
+
+  test("Launcher python client dependencies using a zip file", k8sTestTag, MinikubeTag) {
+    val pySparkFiles = Utils.getTestFileAbsolutePath("pyfiles.py", sparkHomeDir)
+    val inDepsFile = Utils.getTestFileAbsolutePath("py_container_checks.py", sparkHomeDir)
+    val outDepsFile = s"${inDepsFile.substring(0, inDepsFile.lastIndexOf("."))}.zip"
+    Utils.createZipFile(inDepsFile, outDepsFile)
+    testPython(
+      pySparkFiles,
+      Seq(
+        "Python runtime version check is: True",
+        "Python environment version check is: True",
+        "Python runtime version check for executor is: True"),
+      Some(outDepsFile))
+  }
+
+  private def testPython(
+      pySparkFiles: String,
+      expectedDriverLogs: Seq[String],
+      depsFile: Option[String] = None,
+      env: Map[String, String] = Map.empty[String, String]): Unit = {
+    tryDepsTest {
+      setPythonSparkConfProperties(sparkAppConf)
+      runSparkApplicationAndVerifyCompletion(
+        appResource = pySparkFiles,
+        mainClass = "",
+        expectedDriverLogOnCompletion = expectedDriverLogs,
+        appArgs = Array("python3"),
+        driverPodChecker = doBasicDriverPyPodCheck,
+        executorPodChecker = doBasicExecutorPyPodCheck,
+        isJVM = false,
+        pyFiles = depsFile,
+        env = env)
+    }
   }
 
   private def createS3Bucket(accessKey: String, secretKey: String, endPoint: String): Unit = {
@@ -218,6 +284,47 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
       url
     }
   }
+
+  private def getServiceHostAndPort(minioUrlStr : String) : (String, Int) = {
+    val minioUrl = new URL(minioUrlStr)
+    (minioUrl.getHost, minioUrl.getPort)
+  }
+
+  private def setCommonSparkConfPropertiesForS3Access(
+      conf: SparkAppConf,
+      minioUrlStr: String): Unit = {
+    val (minioHost, minioPort) = getServiceHostAndPort(minioUrlStr)
+    val packages = if (Utils.isHadoop3) {
+      s"org.apache.hadoop:hadoop-aws:${VersionInfo.getVersion}"
+    } else {
+      "com.amazonaws:aws-java-sdk:1.7.4,org.apache.hadoop:hadoop-aws:2.7.6"
+    }
+    conf.set("spark.hadoop.fs.s3a.access.key", ACCESS_KEY)
+      .set("spark.hadoop.fs.s3a.secret.key", SECRET_KEY)
+      .set("spark.hadoop.fs.s3a.connection.ssl.enabled", "false")
+      .set("spark.hadoop.fs.s3a.endpoint", s"$minioHost:$minioPort")
+      .set("spark.kubernetes.file.upload.path", s"s3a://$BUCKET")
+      .set("spark.hadoop.fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem")
+      .set("spark.jars.packages", packages)
+      .set("spark.driver.extraJavaOptions", "-Divy.cache.dir=/tmp -Divy.home=/tmp")
+  }
+
+  private def setPythonSparkConfProperties(conf: SparkAppConf): Unit = {
+    sparkAppConf.set("spark.kubernetes.container.image", pyImage)
+  }
+
+  private def tryDepsTest(runTest: => Unit): Unit = {
+    try {
+      setupMinioStorage()
+      val minioUrlStr = getServiceUrl(svcName)
+      createS3Bucket(ACCESS_KEY, SECRET_KEY, minioUrlStr)
+      setCommonSparkConfPropertiesForS3Access(sparkAppConf, minioUrlStr)
+      runTest
+    } finally {
+      // make sure this always runs
+      deleteMinioStorage()
+    }
+  }
 }
 
 private[spark] object DepsTestsSuite {
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
index 4de7e70c1f409..5007171ec153f 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
@@ -25,25 +25,27 @@ import scala.collection.JavaConverters._
 import com.google.common.base.Charsets
 import com.google.common.io.Files
 import io.fabric8.kubernetes.api.model.Pod
-import io.fabric8.kubernetes.client.{KubernetesClientException, Watcher}
+import io.fabric8.kubernetes.client.{Watcher, WatcherException}
 import io.fabric8.kubernetes.client.Watcher.Action
 import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll, Tag}
-import org.scalatest.Matchers
 import org.scalatest.concurrent.{Eventually, PatienceConfiguration}
 import org.scalatest.concurrent.PatienceConfiguration.{Interval, Timeout}
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 import org.scalatest.time.{Minutes, Seconds, Span}
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.deploy.k8s.integrationtest.TestConstants._
 import org.apache.spark.deploy.k8s.integrationtest.backend.{IntegrationTestBackend, IntegrationTestBackendFactory}
+import org.apache.spark.deploy.k8s.integrationtest.backend.minikube.Minikube
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config._
 
 class KubernetesSuite extends SparkFunSuite
-  with BeforeAndAfterAll with BeforeAndAfter with BasicTestsSuite with SecretsTestsSuite
-  with PythonTestsSuite with ClientModeTestsSuite with PodTemplateSuite with PVTestsSuite
-  with DepsTestsSuite with DecommissionSuite with RTestsSuite with Logging with Eventually
-  with Matchers {
+  with BeforeAndAfterAll with BeforeAndAfter with BasicTestsSuite with SparkConfPropagateSuite
+  with SecretsTestsSuite with PythonTestsSuite with ClientModeTestsSuite with PodTemplateSuite
+  with PVTestsSuite with DepsTestsSuite with DecommissionSuite with RTestsSuite with Logging
+  with Eventually with Matchers {
 
 
   import KubernetesSuite._
@@ -71,6 +73,42 @@ class KubernetesSuite extends SparkFunSuite
   private val extraExecTotalMemory =
     s"${(1024 + memOverheadConstant*1024 + additionalMemory).toInt}"
 
+  protected override def logForFailedTest(): Unit = {
+    logInfo("\n\n===== EXTRA LOGS FOR THE FAILED TEST\n")
+    logInfo("BEGIN DESCRIBE PODS for application\n" +
+      Minikube.describePods(s"spark-app-locator=$appLocator").mkString("\n"))
+    logInfo("END DESCRIBE PODS for the application")
+    val driverPodOption = kubernetesTestComponents.kubernetesClient
+      .pods()
+      .withLabel("spark-app-locator", appLocator)
+      .withLabel("spark-role", "driver")
+      .list()
+      .getItems
+      .asScala
+      .headOption
+    driverPodOption.foreach { driverPod =>
+      logInfo("BEGIN driver POD log\n" +
+        kubernetesTestComponents.kubernetesClient
+          .pods()
+          .withName(driverPod.getMetadata.getName)
+          .getLog)
+      logInfo("END driver POD log")
+    }
+    kubernetesTestComponents.kubernetesClient
+      .pods()
+      .withLabel("spark-app-locator", appLocator)
+      .withLabel("spark-role", "executor")
+      .list()
+      .getItems.asScala.foreach { execPod =>
+        logInfo(s"\nBEGIN executor (${execPod.getMetadata.getName}) POD log:\n" +
+          kubernetesTestComponents.kubernetesClient
+            .pods()
+            .withName(execPod.getMetadata.getName)
+            .getLog)
+        logInfo(s"END executor (${execPod.getMetadata.getName}) POD log")
+      }
+  }
+
   /**
    * Build the image ref for the given image name, taking the repo and tag from the
    * test configuration.
@@ -157,6 +195,7 @@ class KubernetesSuite extends SparkFunSuite
       kubernetesTestComponents.deleteNamespace()
     }
     deleteDriverPod()
+    deleteExecutorPod()
   }
 
   protected def runSparkPiAndVerifyCompletion(
@@ -164,16 +203,15 @@ class KubernetesSuite extends SparkFunSuite
       driverPodChecker: Pod => Unit = doBasicDriverPodCheck,
       executorPodChecker: Pod => Unit = doBasicExecutorPodCheck,
       appArgs: Array[String] = Array.empty[String],
-      appLocator: String = appLocator,
       isJVM: Boolean = true ): Unit = {
     runSparkApplicationAndVerifyCompletion(
       appResource,
       SPARK_PI_MAIN_CLASS,
       Seq("Pi is roughly 3"),
+      Seq(),
       appArgs,
       driverPodChecker,
       executorPodChecker,
-      appLocator,
       isJVM)
   }
 
@@ -183,7 +221,6 @@ class KubernetesSuite extends SparkFunSuite
       driverPodChecker: Pod => Unit = doBasicDriverPodCheck,
       executorPodChecker: Pod => Unit = doBasicExecutorPodCheck,
       appArgs: Array[String] = Array.empty[String],
-      appLocator: String = appLocator,
       isJVM: Boolean = true,
       interval: Option[PatienceConfiguration.Interval] = None): Unit = {
     runSparkApplicationAndVerifyCompletion(
@@ -191,10 +228,10 @@ class KubernetesSuite extends SparkFunSuite
       SPARK_DFS_READ_WRITE_TEST,
       Seq(s"Success! Local Word Count $wordCount and " +
     s"DFS Word Count $wordCount agree."),
+      Seq(),
       appArgs,
       driverPodChecker,
       executorPodChecker,
-      appLocator,
       isJVM,
       None,
       Option((interval, None)))
@@ -205,16 +242,15 @@ class KubernetesSuite extends SparkFunSuite
       driverPodChecker: Pod => Unit = doBasicDriverPodCheck,
       executorPodChecker: Pod => Unit = doBasicExecutorPodCheck,
       appArgs: Array[String],
-      appLocator: String = appLocator,
       timeout: Option[PatienceConfiguration.Timeout] = None): Unit = {
     runSparkApplicationAndVerifyCompletion(
       appResource,
       SPARK_REMOTE_MAIN_CLASS,
       Seq(s"Mounting of ${appArgs.head} was true"),
+      Seq(),
       appArgs,
       driverPodChecker,
       executorPodChecker,
-      appLocator,
       true,
       executorPatience = Option((None, timeout)))
   }
@@ -260,15 +296,16 @@ class KubernetesSuite extends SparkFunSuite
   protected def runSparkApplicationAndVerifyCompletion(
       appResource: String,
       mainClass: String,
-      expectedLogOnCompletion: Seq[String],
+      expectedDriverLogOnCompletion: Seq[String],
+      expectedExecutorLogOnCompletion: Seq[String] = Seq(),
       appArgs: Array[String],
       driverPodChecker: Pod => Unit,
       executorPodChecker: Pod => Unit,
-      appLocator: String,
       isJVM: Boolean,
       pyFiles: Option[String] = None,
       executorPatience: Option[(Option[Interval], Option[Timeout])] = None,
-      decommissioningTest: Boolean = false): Unit = {
+      decommissioningTest: Boolean = false,
+      env: Map[String, String] = Map.empty[String, String]): Unit = {
 
   // scalastyle:on argcount
     val appArguments = SparkAppArguments(
@@ -277,6 +314,7 @@ class KubernetesSuite extends SparkFunSuite
       appArgs = appArgs)
 
     val execPods = scala.collection.mutable.Map[String, Pod]()
+    val podsDeleted = scala.collection.mutable.HashSet[String]()
     val (patienceInterval, patienceTimeout) = {
       executorPatience match {
         case Some(patience) => (patience._1.getOrElse(INTERVAL), patience._2.getOrElse(TIMEOUT))
@@ -305,7 +343,8 @@ class KubernetesSuite extends SparkFunSuite
       .withLabel("spark-role", "executor")
       .watch(new Watcher[Pod] {
         logDebug("Beginning watch of executors")
-        override def onClose(cause: KubernetesClientException): Unit =
+        override def onClose(): Unit = logInfo("Ending watch of executors")
+        override def onClose(cause: WatcherException): Unit =
           logInfo("Ending watch of executors")
         override def eventReceived(action: Watcher.Action, resource: Pod): Unit = {
           val name = resource.getMetadata.getName
@@ -325,24 +364,36 @@ class KubernetesSuite extends SparkFunSuite
                   val result = checkPodReady(namespace, name)
                   result shouldBe (true)
                 }
-                // Look for the string that indicates we're good to clean up
-                // on the driver
+                // Look for the string that indicates we're good to trigger decom on the driver
                 logDebug("Waiting for first collect...")
                 Eventually.eventually(TIMEOUT, INTERVAL) {
                   assert(kubernetesTestComponents.kubernetesClient
                     .pods()
                     .withName(driverPodName)
                     .getLog
-                    .contains("Waiting to give nodes time to finish."),
+                    .contains("Waiting to give nodes time to finish migration, decom exec 1."),
                     "Decommission test did not complete first collect.")
                 }
                 // Delete the pod to simulate cluster scale down/migration.
-                val pod = kubernetesTestComponents.kubernetesClient.pods().withName(name)
-                pod.delete()
+                // This will allow the pod to remain up for the grace period
+                // We set an intentionally long grace period to test that Spark
+                // exits once the blocks are done migrating and doesn't wait for the
+                // entire grace period if it does not need to.
+                kubernetesTestComponents.kubernetesClient.pods()
+                  .withName(name).withGracePeriod(Int.MaxValue).delete()
                 logDebug(s"Triggered pod decom/delete: $name deleted")
+                // Make sure this pod is deleted
+                Eventually.eventually(TIMEOUT, INTERVAL) {
+                  assert(podsDeleted.contains(name))
+                }
+                // Then make sure this pod is replaced
+                Eventually.eventually(TIMEOUT, INTERVAL) {
+                  assert(execPods.size == 3)
+                }
               }
             case Action.DELETED | Action.ERROR =>
               execPods.remove(name)
+              podsDeleted += name
           }
         }
       })
@@ -354,7 +405,8 @@ class KubernetesSuite extends SparkFunSuite
       TIMEOUT.value.toSeconds.toInt,
       sparkHomeDir,
       isJVM,
-      pyFiles)
+      pyFiles,
+      env)
 
     val driverPod = kubernetesTestComponents.kubernetesClient
       .pods()
@@ -363,25 +415,46 @@ class KubernetesSuite extends SparkFunSuite
       .list()
       .getItems
       .get(0)
-
     driverPodChecker(driverPod)
-    // If we're testing decommissioning we delete all the executors, but we should have
-    // an executor at some point.
-    Eventually.eventually(patienceTimeout, patienceInterval) {
+
+    // If we're testing decommissioning we an executors, but we should have an executor
+    // at some point.
+    Eventually.eventually(TIMEOUT, patienceInterval) {
       execPods.values.nonEmpty should be (true)
     }
-    execWatcher.close()
     execPods.values.foreach(executorPodChecker(_))
+
+    val execPod: Option[Pod] = if (expectedExecutorLogOnCompletion.nonEmpty) {
+      Some(kubernetesTestComponents.kubernetesClient
+        .pods()
+        .withLabel("spark-app-locator", appLocator)
+        .withLabel("spark-role", "executor")
+        .list()
+        .getItems
+        .get(0))
+    } else {
+      None
+    }
+
     Eventually.eventually(patienceTimeout, patienceInterval) {
-      expectedLogOnCompletion.foreach { e =>
+      expectedDriverLogOnCompletion.foreach { e =>
         assert(kubernetesTestComponents.kubernetesClient
           .pods()
           .withName(driverPod.getMetadata.getName)
           .getLog
           .contains(e),
-          s"The application did not complete, did not find str ${e}")
+          s"The application did not complete, driver log did not contain str ${e}")
+      }
+      expectedExecutorLogOnCompletion.foreach { e =>
+        assert(kubernetesTestComponents.kubernetesClient
+          .pods()
+          .withName(execPod.get.getMetadata.getName)
+          .getLog
+          .contains(e),
+          s"The application did not complete, executor log did not contain str ${e}")
       }
     }
+    execWatcher.close()
   }
 
   protected def doBasicDriverPodCheck(driverPod: Pod): Unit = {
@@ -472,15 +545,34 @@ class KubernetesSuite extends SparkFunSuite
         .get() == null)
     }
   }
+
+  private def deleteExecutorPod(): Unit = {
+    kubernetesTestComponents
+      .kubernetesClient
+      .pods()
+      .withLabel("spark-app-locator", appLocator)
+      .withLabel("spark-role", "executor")
+      .delete()
+    Eventually.eventually(TIMEOUT, INTERVAL) {
+      assert(kubernetesTestComponents.kubernetesClient
+        .pods()
+        .withLabel("spark-app-locator", appLocator)
+        .withLabel("spark-role", "executor")
+        .list()
+        .getItems.isEmpty)
+    }
+  }
 }
 
 private[spark] object KubernetesSuite {
   val k8sTestTag = Tag("k8s")
+  val pvTestTag = Tag("persistentVolume")
+  val rTestTag = Tag("r")
   val MinikubeTag = Tag("minikube")
   val SPARK_PI_MAIN_CLASS: String = "org.apache.spark.examples.SparkPi"
   val SPARK_DFS_READ_WRITE_TEST = "org.apache.spark.examples.DFSReadWriteTest"
   val SPARK_REMOTE_MAIN_CLASS: String = "org.apache.spark.examples.SparkRemoteFileTest"
   val SPARK_DRIVER_MAIN_CLASS: String = "org.apache.spark.examples.DriverSubmissionTest"
-  val TIMEOUT = PatienceConfiguration.Timeout(Span(2, Minutes))
+  val TIMEOUT = PatienceConfiguration.Timeout(Span(3, Minutes))
   val INTERVAL = PatienceConfiguration.Interval(Span(1, Seconds))
 }
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala
index 4cfda8ab9ba77..411857f0229db 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala
@@ -21,8 +21,8 @@ import java.util.UUID
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable
-import scala.collection.mutable.ArrayBuffer
 
+import io.fabric8.kubernetes.api.model.NamespaceBuilder
 import io.fabric8.kubernetes.client.DefaultKubernetesClient
 import org.scalatest.concurrent.Eventually
 
@@ -45,11 +45,11 @@ private[spark] class KubernetesTestComponents(defaultClient: DefaultKubernetesCl
   val clientConfig = kubernetesClient.getConfiguration
 
   def createNamespace(): Unit = {
-    defaultClient.namespaces.createNew()
+    defaultClient.namespaces.create(new NamespaceBuilder()
       .withNewMetadata()
       .withName(namespace)
       .endMetadata()
-      .done()
+      .build())
   }
 
   def deleteNamespace(): Unit = {
@@ -69,7 +69,7 @@ private[spark] class KubernetesTestComponents(defaultClient: DefaultKubernetesCl
       .set("spark.master", s"k8s://${kubernetesClient.getMasterUrl}")
       .set("spark.kubernetes.namespace", namespace)
       .set("spark.executor.cores", "1")
-      .set("spark.executors.instances", "1")
+      .set("spark.executor.instances", "1")
       .set("spark.app.name", "spark-test-app")
       .set(IS_TESTING.key, "false")
       .set(UI_ENABLED.key, "true")
@@ -110,7 +110,8 @@ private[spark] object SparkAppLauncher extends Logging {
       timeoutSecs: Int,
       sparkHomeDir: Path,
       isJVM: Boolean,
-      pyFiles: Option[String] = None): Unit = {
+      pyFiles: Option[String] = None,
+      env: Map[String, String] = Map.empty[String, String]): Unit = {
     val sparkSubmitExecutable = sparkHomeDir.resolve(Paths.get("bin", "spark-submit"))
     logInfo(s"Launching a spark app with arguments $appArguments and conf $appConf")
     val preCommandLine = if (isJVM) {
@@ -131,6 +132,6 @@ private[spark] object SparkAppLauncher extends Logging {
       commandLine ++= appArguments.appArgs
     }
     logInfo(s"Launching a spark app with command line: ${commandLine.mkString(" ")}")
-    ProcessUtils.executeProcess(commandLine.toArray, timeoutSecs)
+    ProcessUtils.executeProcess(commandLine.toArray, timeoutSecs, env = env)
   }
 }
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
index 86f8cddcef7da..2f1a7aa071336 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
@@ -122,7 +122,7 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite =>
     }
   }
 
-  test("PVs with local storage", k8sTestTag, MinikubeTag) {
+  test("PVs with local storage", pvTestTag, MinikubeTag) {
     sparkAppConf
       .set(s"spark.kubernetes.driver.volumes.persistentVolumeClaim.data.mount.path",
         CONTAINER_MOUNT_PATH)
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala
index 7d76a22e6d363..e259979ad0329 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala
@@ -19,6 +19,7 @@ package org.apache.spark.deploy.k8s.integrationtest
 import java.nio.charset.StandardCharsets
 import java.util.concurrent.TimeUnit
 
+import scala.collection.JavaConverters._
 import scala.collection.mutable.ArrayBuffer
 import scala.io.Source
 
@@ -32,21 +33,26 @@ object ProcessUtils extends Logging {
   def executeProcess(
       fullCommand: Array[String],
       timeout: Long,
-      dumpErrors: Boolean = false): Seq[String] = {
+      dumpOutput: Boolean = true,
+      dumpErrors: Boolean = true,
+      env: Map[String, String] = Map.empty[String, String]): Seq[String] = {
     val pb = new ProcessBuilder().command(fullCommand: _*)
+    pb.environment().putAll(env.asJava)
     pb.redirectErrorStream(true)
     val proc = pb.start()
     val outputLines = new ArrayBuffer[String]
     Utils.tryWithResource(proc.getInputStream)(
       Source.fromInputStream(_, StandardCharsets.UTF_8.name()).getLines().foreach { line =>
-        logInfo(line)
+        if (dumpOutput) {
+          logInfo(line)
+        }
         outputLines += line
       })
     assert(proc.waitFor(timeout, TimeUnit.SECONDS),
       s"Timed out while executing ${fullCommand.mkString(" ")}")
     assert(proc.exitValue == 0,
-      s"Failed to execute ${fullCommand.mkString(" ")}" +
+      s"Failed to execute -- ${fullCommand.mkString(" ")} --" +
         s"${if (dumpErrors) "\n" + outputLines.mkString("\n")}")
-    outputLines
+    outputLines.toSeq
   }
 }
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PythonTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PythonTestsSuite.scala
index 3a6e5e7a6255d..d17aec0a80ddb 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PythonTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PythonTestsSuite.scala
@@ -27,48 +27,26 @@ private[spark] trait PythonTestsSuite { k8sSuite: KubernetesSuite =>
     runSparkApplicationAndVerifyCompletion(
       appResource = PYSPARK_PI,
       mainClass = "",
-      expectedLogOnCompletion = Seq("Pi is roughly 3"),
+      expectedDriverLogOnCompletion = Seq("Pi is roughly 3"),
       appArgs = Array("5"),
       driverPodChecker = doBasicDriverPyPodCheck,
       executorPodChecker = doBasicExecutorPyPodCheck,
-      appLocator = appLocator,
       isJVM = false)
   }
 
-  test("Run PySpark with Python2 to test a pyfiles example", k8sTestTag) {
+  test("Run PySpark to test a pyfiles example", k8sTestTag) {
     sparkAppConf
       .set("spark.kubernetes.container.image", pyImage)
-      .set("spark.kubernetes.pyspark.pythonVersion", "2")
     runSparkApplicationAndVerifyCompletion(
       appResource = PYSPARK_FILES,
       mainClass = "",
-      expectedLogOnCompletion = Seq(
-        "Python runtime version check is: True",
-        "Python environment version check is: True",
-        "Python runtime version check for executor is: True"),
-      appArgs = Array("python"),
-      driverPodChecker = doBasicDriverPyPodCheck,
-      executorPodChecker = doBasicExecutorPyPodCheck,
-      appLocator = appLocator,
-      isJVM = false,
-      pyFiles = Some(PYSPARK_CONTAINER_TESTS))
-  }
-
-  test("Run PySpark with Python3 to test a pyfiles example", k8sTestTag) {
-    sparkAppConf
-      .set("spark.kubernetes.container.image", pyImage)
-      .set("spark.kubernetes.pyspark.pythonVersion", "3")
-    runSparkApplicationAndVerifyCompletion(
-      appResource = PYSPARK_FILES,
-      mainClass = "",
-      expectedLogOnCompletion = Seq(
+      expectedDriverLogOnCompletion = Seq(
         "Python runtime version check is: True",
         "Python environment version check is: True",
         "Python runtime version check for executor is: True"),
       appArgs = Array("python3"),
       driverPodChecker = doBasicDriverPyPodCheck,
       executorPodChecker = doBasicExecutorPyPodCheck,
-      appLocator = appLocator,
       isJVM = false,
       pyFiles = Some(PYSPARK_CONTAINER_TESTS))
   }
@@ -76,18 +54,16 @@ private[spark] trait PythonTestsSuite { k8sSuite: KubernetesSuite =>
   test("Run PySpark with memory customization", k8sTestTag) {
     sparkAppConf
       .set("spark.kubernetes.container.image", pyImage)
-      .set("spark.kubernetes.pyspark.pythonVersion", "3")
       .set("spark.kubernetes.memoryOverheadFactor", s"$memOverheadConstant")
       .set("spark.executor.pyspark.memory", s"${additionalMemory}m")
     runSparkApplicationAndVerifyCompletion(
       appResource = PYSPARK_MEMORY_CHECK,
       mainClass = "",
-      expectedLogOnCompletion = Seq(
+      expectedDriverLogOnCompletion = Seq(
         "PySpark Worker Memory Check is: True"),
       appArgs = Array(s"$additionalMemoryInBytes"),
       driverPodChecker = doDriverMemoryCheck,
       executorPodChecker = doExecutorMemoryCheck,
-      appLocator = appLocator,
       isJVM = false,
       pyFiles = Some(PYSPARK_CONTAINER_TESTS))
   }
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/RTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/RTestsSuite.scala
index e81562a923228..0046f75f49aff 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/RTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/RTestsSuite.scala
@@ -19,18 +19,17 @@ package org.apache.spark.deploy.k8s.integrationtest
 private[spark] trait RTestsSuite { k8sSuite: KubernetesSuite =>
 
   import RTestsSuite._
-  import KubernetesSuite.k8sTestTag
+  import KubernetesSuite.{k8sTestTag, rTestTag}
 
-  test("Run SparkR on simple dataframe.R example", k8sTestTag) {
+  test("Run SparkR on simple dataframe.R example", k8sTestTag, rTestTag) {
     sparkAppConf.set("spark.kubernetes.container.image", rImage)
     runSparkApplicationAndVerifyCompletion(
       appResource = SPARK_R_DATAFRAME_TEST,
       mainClass = "",
-      expectedLogOnCompletion = Seq("name: string (nullable = true)", "1 Justin"),
+      expectedDriverLogOnCompletion = Seq("name: string (nullable = true)", "1 Justin"),
       appArgs = Array.empty[String],
       driverPodChecker = doBasicDriverRPodCheck,
       executorPodChecker = doBasicExecutorRPodCheck,
-      appLocator = appLocator,
       isJVM = false)
   }
 }
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SecretsTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SecretsTestsSuite.scala
index 54a9dbf07816c..39ae94b17c63b 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SecretsTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SecretsTestsSuite.scala
@@ -22,8 +22,8 @@ import scala.collection.JavaConverters._
 
 import io.fabric8.kubernetes.api.model.{Pod, SecretBuilder}
 import org.apache.commons.codec.binary.Base64
-import org.apache.commons.io.output.ByteArrayOutputStream
 import org.scalatest.concurrent.Eventually
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.deploy.k8s.integrationtest.KubernetesSuite._
 
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SparkConfPropagateSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SparkConfPropagateSuite.scala
new file mode 100644
index 0000000000000..3651398b1d627
--- /dev/null
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SparkConfPropagateSuite.scala
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.deploy.k8s.integrationtest
+
+import java.io.File
+import java.net.URL
+import java.nio.file.Files
+
+import scala.io.Source
+
+private[spark] trait SparkConfPropagateSuite { k8sSuite: KubernetesSuite =>
+  import KubernetesSuite.{k8sTestTag, SPARK_PI_MAIN_CLASS}
+
+  test("Verify logging configuration is picked from the provided SPARK_CONF_DIR/log4j.properties",
+    k8sTestTag) {
+    val loggingConfigFileName = "log-config-test-log4j.properties"
+    val loggingConfURL: URL = this.getClass.getClassLoader.getResource(loggingConfigFileName)
+    assert(loggingConfURL != null, "Logging configuration file not available.")
+
+    val content = Source.createBufferedSource(loggingConfURL.openStream()).getLines().mkString("\n")
+    val logConfFilePath = s"${sparkHomeDir.toFile}/conf/log4j.properties"
+
+    try {
+      Files.write(new File(logConfFilePath).toPath, content.getBytes)
+
+      sparkAppConf.set("spark.driver.extraJavaOptions", "-Dlog4j.debug")
+      sparkAppConf.set("spark.executor.extraJavaOptions", "-Dlog4j.debug")
+      sparkAppConf.set("spark.kubernetes.executor.deleteOnTermination", "false")
+
+      val log4jExpectedLog =
+        s"log4j: Reading configuration from URL file:/opt/spark/conf/log4j.properties"
+
+      runSparkApplicationAndVerifyCompletion(
+        appResource = containerLocalSparkDistroExamplesJar,
+        mainClass = SPARK_PI_MAIN_CLASS,
+        expectedDriverLogOnCompletion = (Seq("DEBUG",
+          log4jExpectedLog,
+          "Pi is roughly 3")),
+        expectedExecutorLogOnCompletion = Seq(log4jExpectedLog),
+        appArgs = Array.empty[String],
+        driverPodChecker = doBasicDriverPodCheck,
+        executorPodChecker = doBasicExecutorPodCheck,
+        isJVM = true)
+    } finally {
+      new File(logConfFilePath).delete()
+    }
+  }
+}
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SparkReadinessWatcher.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SparkReadinessWatcher.scala
index f1fd6dc19ce54..6a3dfd5bf791d 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SparkReadinessWatcher.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SparkReadinessWatcher.scala
@@ -20,22 +20,25 @@ import java.util.concurrent.TimeUnit
 
 import com.google.common.util.concurrent.SettableFuture
 import io.fabric8.kubernetes.api.model.HasMetadata
-import io.fabric8.kubernetes.client.{KubernetesClientException, Watcher}
+import io.fabric8.kubernetes.client.{Watcher, WatcherException}
 import io.fabric8.kubernetes.client.Watcher.Action
 import io.fabric8.kubernetes.client.internal.readiness.Readiness
 
 private[spark] class SparkReadinessWatcher[T <: HasMetadata] extends Watcher[T] {
 
   private val signal = SettableFuture.create[Boolean]
+  private val readiness = Readiness.getInstance()
 
   override def eventReceived(action: Action, resource: T): Unit = {
     if ((action == Action.MODIFIED || action == Action.ADDED) &&
-        Readiness.isReady(resource)) {
+        readiness.isReady(resource)) {
       signal.set(true)
     }
   }
 
-  override def onClose(cause: KubernetesClientException): Unit = {}
+  override def onClose(): Unit = {}
+
+  override def onClose(cause: WatcherException): Unit = {}
 
   def waitUntilReady(): Boolean = signal.get(60, TimeUnit.SECONDS)
 }
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala
index 9f85805b9d315..cc258533c2c8d 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala
@@ -16,15 +16,20 @@
  */
 package org.apache.spark.deploy.k8s.integrationtest
 
-import java.io.{Closeable, File, PrintWriter}
+import java.io.{Closeable, File, FileInputStream, FileOutputStream, PrintWriter}
 import java.nio.file.{Files, Path}
 import java.util.concurrent.CountDownLatch
+import java.util.zip.{ZipEntry, ZipOutputStream}
 
 import scala.collection.JavaConverters._
 
 import io.fabric8.kubernetes.client.dsl.ExecListener
 import okhttp3.Response
+import org.apache.commons.compress.archivers.tar.{TarArchiveEntry, TarArchiveOutputStream}
+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream
+import org.apache.commons.compress.utils.IOUtils
 import org.apache.commons.io.output.ByteArrayOutputStream
+import org.apache.hadoop.util.VersionInfo
 
 import org.apache.spark.{SPARK_VERSION, SparkException}
 import org.apache.spark.internal.Logging
@@ -57,15 +62,15 @@ object Utils extends Logging {
       val openLatch: CountDownLatch = new CountDownLatch(1)
       val closeLatch: CountDownLatch = new CountDownLatch(1)
 
-      override def onOpen(response: Response) {
+      override def onOpen(response: Response): Unit = {
         openLatch.countDown()
       }
 
-      override def onClose(a: Int, b: String) {
+      override def onClose(a: Int, b: String): Unit = {
         closeLatch.countDown()
       }
 
-      override def onFailure(e: Throwable, r: Response) {
+      override def onFailure(e: Throwable, r: Response): Unit = {
       }
 
       def waitForInputStreamToConnect(): Unit = {
@@ -111,24 +116,64 @@ object Utils extends Logging {
     filename
   }
 
-  def getExamplesJarAbsolutePath(sparkHomeDir: Path): String = {
-    val jarName = getExamplesJarName()
-    val jarPathsFound = Files
+  def getTestFileAbsolutePath(fileName: String, sparkHomeDir: Path): String = {
+    val filePathsFound = Files
       .walk(sparkHomeDir)
       .filter(Files.isRegularFile(_))
-      .filter((f: Path) => {f.toFile.getName == jarName})
+      .filter((f: Path) => {f.toFile.getName == fileName})
     // we should not have more than one here under current test build dir
     // we only need one though
-    val jarPath = jarPathsFound
+    val filePath = filePathsFound
       .iterator()
       .asScala
       .map(_.toAbsolutePath.toString)
       .toArray
       .headOption
-    jarPath match {
-      case Some(jar) => jar
-      case _ => throw new SparkException(s"No valid $jarName file was found " +
+    filePath match {
+      case Some(file) => file
+      case _ => throw new SparkException(s"No valid $fileName file was found " +
         s"under spark home test dir ${sparkHomeDir.toAbsolutePath}!")
     }
   }
+
+  def isHadoop3(): Boolean = {
+    VersionInfo.getVersion.startsWith("3")
+  }
+
+  def createZipFile(inFile: String, outFile: String): Unit = {
+    val fileToZip = new File(inFile)
+    val fis = new FileInputStream(fileToZip)
+    val fos = new FileOutputStream(outFile)
+    val zipOut = new ZipOutputStream(fos)
+    val zipEntry = new ZipEntry(fileToZip.getName)
+    zipOut.putNextEntry(zipEntry)
+    IOUtils.copy(fis, zipOut)
+    IOUtils.closeQuietly(fis)
+    IOUtils.closeQuietly(zipOut)
+  }
+
+  def createTarGzFile(inFile: String, outFile: String): Unit = {
+    val oFile = new File(outFile)
+    val fileToTarGz = new File(inFile)
+    Utils.tryWithResource(
+      new FileInputStream(fileToTarGz)
+    ) { fis =>
+      Utils.tryWithResource(
+        new TarArchiveOutputStream(
+          new GzipCompressorOutputStream(
+            new FileOutputStream(oFile)))
+      ) { tOut =>
+        val tarEntry = new TarArchiveEntry(fileToTarGz, fileToTarGz.getName)
+        // Each entry does not keep the file permission from the input file.
+        // Setting permissions in the input file do not work. Just simply set
+        // to 777.
+        tarEntry.setMode(0x81ff)
+        tOut.putArchiveEntry(tarEntry)
+        IOUtils.copy(fis, tOut)
+        tOut.closeArchiveEntry()
+        tOut.finish()
+      }
+    }
+    oFile.deleteOnExit()
+  }
 }
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/KubeConfigBackend.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/KubeConfigBackend.scala
index be1834c0b5dea..0fbed4a220e68 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/KubeConfigBackend.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/KubeConfigBackend.scala
@@ -16,8 +16,6 @@
  */
 package org.apache.spark.deploy.k8s.integrationtest.backend.cloud
 
-import java.nio.file.Paths
-
 import io.fabric8.kubernetes.client.{Config, DefaultKubernetesClient}
 import io.fabric8.kubernetes.client.utils.Utils
 import org.apache.commons.lang3.StringUtils
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala
index a7d35b67d1b92..1ebc64445b717 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala
@@ -16,9 +16,8 @@
  */
 package org.apache.spark.deploy.k8s.integrationtest.backend.minikube
 
-import java.nio.file.Paths
-
-import io.fabric8.kubernetes.client.{ConfigBuilder, DefaultKubernetesClient}
+import io.fabric8.kubernetes.client.Config
+import io.fabric8.kubernetes.client.DefaultKubernetesClient
 
 import org.apache.spark.deploy.k8s.integrationtest.ProcessUtils
 import org.apache.spark.internal.Logging
@@ -26,82 +25,64 @@ import org.apache.spark.internal.Logging
 // TODO support windows
 private[spark] object Minikube extends Logging {
   private val MINIKUBE_STARTUP_TIMEOUT_SECONDS = 60
-  private val HOST_PREFIX = "host:"
-  private val KUBELET_PREFIX = "kubelet:"
-  private val APISERVER_PREFIX = "apiserver:"
-  private val KUBECTL_PREFIX = "kubectl:"
-  private val KUBECONFIG_PREFIX = "kubeconfig:"
+  private val VERSION_PREFIX = "minikube version: "
+  private val HOST_PREFIX = "host: "
+  private val KUBELET_PREFIX = "kubelet: "
+  private val APISERVER_PREFIX = "apiserver: "
+  private val KUBECTL_PREFIX = "kubectl: "
+  private val KUBECONFIG_PREFIX = "kubeconfig: "
   private val MINIKUBE_VM_PREFIX = "minikubeVM: "
   private val MINIKUBE_PREFIX = "minikube: "
-  private val MINIKUBE_PATH = ".minikube"
 
-  def logVersion(): Unit = {
-    logInfo(executeMinikube("version").mkString("\n"))
-  }
+  lazy val minikubeVersionString =
+    executeMinikube(true, "version").find(_.contains(VERSION_PREFIX)).get
 
-  def getMinikubeIp: String = {
-    val outputs = executeMinikube("ip")
-      .filter(_.matches("^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}$"))
-    assert(outputs.size == 1, "Unexpected amount of output from minikube ip")
-    outputs.head
-  }
+  def logVersion(): Unit =
+    logInfo(minikubeVersionString)
 
-  def getMinikubeStatus: MinikubeStatus.Value = {
-    val statusString = executeMinikube("status")
-    logInfo(s"Minikube status command output:\n$statusString")
-    // up to minikube version v0.30.0 use this to check for minikube status
-    val oldMinikube = statusString
-      .filter(line => line.contains(MINIKUBE_VM_PREFIX) || line.contains(MINIKUBE_PREFIX))
-
-    if (oldMinikube.isEmpty) {
-      getIfNewMinikubeStatus(statusString)
-    } else {
-      val finalStatusString = oldMinikube
-        .head
-        .replaceFirst(MINIKUBE_VM_PREFIX, "")
-        .replaceFirst(MINIKUBE_PREFIX, "")
-      MinikubeStatus.unapply(finalStatusString)
-        .getOrElse(throw new IllegalStateException(s"Unknown status $statusString"))
+  def getKubernetesClient: DefaultKubernetesClient = {
+    // only the three-part version number is matched (the optional suffix like "-beta.0" is dropped)
+    val versionArrayOpt = "\\d+\\.\\d+\\.\\d+".r
+      .findFirstIn(minikubeVersionString.split(VERSION_PREFIX)(1))
+      .map(_.split('.').map(_.toInt))
+
+    versionArrayOpt match {
+      case Some(Array(x, y, z)) =>
+        if (Ordering.Tuple3[Int, Int, Int].lt((x, y, z), (1, 7, 3))) {
+          assert(false, s"Unsupported Minikube version is detected: $minikubeVersionString." +
+            "For integration testing Minikube version 1.7.3 or greater is expected.")
+        }
+      case _ =>
+        assert(false, s"Unexpected version format detected in `$minikubeVersionString`." +
+          "For minikube version a three-part version number is expected (the optional " +
+          "non-numeric suffix is intentionally dropped)")
     }
-  }
 
-  def getKubernetesClient: DefaultKubernetesClient = {
-    val kubernetesMaster = s"https://${getMinikubeIp}:8443"
-    val userHome = System.getProperty("user.home")
-    val kubernetesConf = new ConfigBuilder()
-      .withApiVersion("v1")
-      .withMasterUrl(kubernetesMaster)
-      .withCaCertFile(
-        Paths.get(userHome, MINIKUBE_PATH, "ca.crt").toFile.getAbsolutePath)
-      .withClientCertFile(
-        Paths.get(userHome, MINIKUBE_PATH, "apiserver.crt").toFile.getAbsolutePath)
-      .withClientKeyFile(
-        Paths.get(userHome, MINIKUBE_PATH, "apiserver.key").toFile.getAbsolutePath)
-      .build()
-    new DefaultKubernetesClient(kubernetesConf)
+    new DefaultKubernetesClient(Config.autoConfigure("minikube"))
   }
 
-  // Covers minikube status output after Minikube V0.30.
-  private def getIfNewMinikubeStatus(statusString: Seq[String]): MinikubeStatus.Value = {
-    val hostString = statusString.find(_.contains(s"$HOST_PREFIX "))
-    val kubeletString = statusString.find(_.contains(s"$KUBELET_PREFIX "))
-    val apiserverString = statusString.find(_.contains(s"$APISERVER_PREFIX "))
-    val kubectlString = statusString.find(_.contains(s"$KUBECTL_PREFIX "))
-    val kubeconfigString = statusString.find(_.contains(s"$KUBECONFIG_PREFIX "))
+  def getMinikubeStatus(): MinikubeStatus.Value = {
+    val statusString = executeMinikube(true, "status")
+    logInfo(s"Minikube status command output:\n$statusString")
+    val hostString = statusString.find(_.contains(HOST_PREFIX))
+    val kubeletString = statusString.find(_.contains(KUBELET_PREFIX))
+    val apiserverString = statusString.find(_.contains(APISERVER_PREFIX))
+    val kubectlString = statusString.find(_.contains(KUBECTL_PREFIX))
+    val kubeconfigString = statusString.find(_.contains(KUBECONFIG_PREFIX))
     val hasConfigStatus = kubectlString.isDefined || kubeconfigString.isDefined
 
     if (hostString.isEmpty || kubeletString.isEmpty || apiserverString.isEmpty ||
         !hasConfigStatus) {
       MinikubeStatus.NONE
     } else {
-      val status1 = hostString.get.replaceFirst(s"$HOST_PREFIX ", "")
-      val status2 = kubeletString.get.replaceFirst(s"$KUBELET_PREFIX ", "")
-      val status3 = apiserverString.get.replaceFirst(s"$APISERVER_PREFIX ", "")
+      val status1 = hostString.get.split(HOST_PREFIX)(1)
+      val status2 = kubeletString.get.split(KUBELET_PREFIX)(1)
+      val status3 = apiserverString.get.split(APISERVER_PREFIX)(1)
       val isConfigured = if (kubectlString.isDefined) {
-        val cfgStatus = kubectlString.get.replaceFirst(s"$KUBECTL_PREFIX ", "")
+        val cfgStatus = kubectlString.get.split(KUBECTL_PREFIX)(1)
         cfgStatus.contains("Correctly Configured:")
       } else {
-        kubeconfigString.get.replaceFirst(s"$KUBECONFIG_PREFIX ", "") == "Configured"
+        kubeconfigString.get.split(KUBECONFIG_PREFIX)(1) == "Configured"
       }
       if (isConfigured) {
         val stats = List(status1, status2, status3)
@@ -118,15 +99,22 @@ private[spark] object Minikube extends Logging {
     }
   }
 
-  private def executeMinikube(action: String, args: String*): Seq[String] = {
+  def executeMinikube(logOutput: Boolean, action: String, args: String*): Seq[String] = {
     ProcessUtils.executeProcess(
-      Array("bash", "-c", s"minikube $action ${args.mkString(" ")}"),
-      MINIKUBE_STARTUP_TIMEOUT_SECONDS)
+      Array("bash", "-c", s"MINIKUBE_IN_STYLE=true minikube $action ${args.mkString(" ")}"),
+      MINIKUBE_STARTUP_TIMEOUT_SECONDS, dumpOutput = logOutput).filter { x =>
+      !x.contains("There is a newer version of minikube") &&
+      !x.contains("https://github.com/kubernetes")
+    }
   }
 
   def minikubeServiceAction(args: String*): String = {
-    executeMinikube("service", args: _*).head
+    executeMinikube(true, "service", args: _*).head
   }
+
+  def describePods(labels: String): Seq[String] =
+    Minikube.executeMinikube(false, "kubectl", "--", "describe", "pods", "--all-namespaces",
+      "-l", labels)
 }
 
 private[spark] object MinikubeStatus extends Enumeration {
diff --git a/resource-managers/kubernetes/integration-tests/tests/autoscale.py b/resource-managers/kubernetes/integration-tests/tests/autoscale.py
new file mode 100644
index 0000000000000..809b698fcdd8c
--- /dev/null
+++ b/resource-managers/kubernetes/integration-tests/tests/autoscale.py
@@ -0,0 +1,49 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import sys
+import time
+
+from pyspark.sql import SparkSession
+
+
+if __name__ == "__main__":
+    """
+        Usage: autoscale
+    """
+    print("Starting autoscale test")
+    spark = SparkSession \
+        .builder \
+        .appName("AutoScale") \
+        .getOrCreate()
+    sc = spark._sc
+
+    initialRdd = sc.parallelize(range(100), 5)
+    # Trigger a shuffle so there are shuffle blocks to migrate
+    rdd = initialRdd.map(lambda x: (x, x)).groupByKey()
+    rdd.collect()
+    numCores = sc._jsc.sc().getExecutorMemoryStatus().size()
+    print("Have " + str(numCores))
+    print("Waiting for dynamic alloc")
+    time.sleep(150)
+    print("Finished waiting!")
+    rdd.count()
+    rdd.collect()
+    print("Finished waiting, stopping Spark.")
+    spark.stop()
+    print("Done, exiting Python")
+    sys.exit(0)
diff --git a/resource-managers/kubernetes/integration-tests/tests/decommissioning.py b/resource-managers/kubernetes/integration-tests/tests/decommissioning.py
index f68f24d49763d..0880e8ab275b3 100644
--- a/resource-managers/kubernetes/integration-tests/tests/decommissioning.py
+++ b/resource-managers/kubernetes/integration-tests/tests/decommissioning.py
@@ -28,17 +28,27 @@
     print("Starting decom test")
     spark = SparkSession \
         .builder \
-        .appName("PyMemoryTest") \
+        .appName("DecomTest") \
         .getOrCreate()
     sc = spark._sc
-    rdd = sc.parallelize(range(10))
-    rdd.collect()
-    print("Waiting to give nodes time to finish.")
-    time.sleep(5)
+    acc = sc.accumulator(0)
+
+    def addToAcc(x):
+        acc.add(1)
+        return x
+
+    initialRdd = sc.parallelize(range(100), 5)
+    accRdd = initialRdd.map(addToAcc)
+    # Trigger a shuffle so there are shuffle blocks to migrate
+    rdd = accRdd.map(lambda x: (x, x)).groupByKey()
     rdd.collect()
-    print("Waiting some more....")
-    time.sleep(10)
+    print("1st accumulator value is: " + str(acc.value))
+    print("Waiting to give nodes time to finish migration, decom exec 1.")
+    print("...")
+    time.sleep(30)
+    rdd.count()
     rdd.collect()
+    print("Final accumulator value is: " + str(acc.value))
     print("Finished waiting, stopping Spark.")
     spark.stop()
     print("Done, exiting Python")
diff --git a/resource-managers/kubernetes/integration-tests/tests/decommissioning_cleanup.py b/resource-managers/kubernetes/integration-tests/tests/decommissioning_cleanup.py
new file mode 100644
index 0000000000000..8af558ee5214e
--- /dev/null
+++ b/resource-managers/kubernetes/integration-tests/tests/decommissioning_cleanup.py
@@ -0,0 +1,59 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import sys
+import time
+
+from pyspark.sql import SparkSession
+
+
+if __name__ == "__main__":
+    """
+        Usage: decommissioning
+    """
+    print("Starting decom test")
+    spark = SparkSession \
+        .builder \
+        .appName("DecomTest") \
+        .getOrCreate()
+    sc = spark._sc
+    acc = sc.accumulator(0)
+
+    def addToAcc(x):
+        acc.add(1)
+        return x
+
+    initialRdd = sc.parallelize(range(100), 5)
+    accRdd = initialRdd.map(addToAcc)
+    # Trigger a shuffle so there are shuffle blocks to migrate
+    rdd = accRdd.map(lambda x: (x, x)).groupByKey()
+    # Make enough shuffle files to increase the chance of the race condition.
+    for i in range(1, 2):
+        shuffleRdd = sc.parallelize(range(1, 10), 5).map(lambda x: (x, x)).groupByKey()
+        shuffleRdd.collect()
+    rdd.collect()
+    print("1st accumulator value is: " + str(acc.value))
+    print("Waiting to give nodes time to finish migration, decom exec 1.")
+    print("...")
+    time.sleep(30)
+    rdd.count()
+    rdd.collect()
+    print("Final accumulator value is: " + str(acc.value))
+    print("Finished waiting, stopping Spark.")
+    spark.stop()
+    print("Done, exiting Python")
+    sys.exit(0)
diff --git a/resource-managers/kubernetes/integration-tests/tests/py_container_checks.py b/resource-managers/kubernetes/integration-tests/tests/py_container_checks.py
index f6b3be2806c82..e6c0137c0405f 100644
--- a/resource-managers/kubernetes/integration-tests/tests/py_container_checks.py
+++ b/resource-managers/kubernetes/integration-tests/tests/py_container_checks.py
@@ -24,7 +24,7 @@ def version_check(python_env, major_python_version):
         These are various tests to test the Python container image.
         This file will be distributed via --py-files in the e2e tests.
     """
-    env_version = os.environ.get('PYSPARK_PYTHON')
+    env_version = os.environ.get('PYSPARK_PYTHON', 'python3')
     print("Python runtime version check is: " +
           str(sys.version_info[0] == major_python_version))
 
diff --git a/resource-managers/kubernetes/integration-tests/tests/pyfiles.py b/resource-managers/kubernetes/integration-tests/tests/pyfiles.py
index ba55b75803276..73c53be482c03 100644
--- a/resource-managers/kubernetes/integration-tests/tests/pyfiles.py
+++ b/resource-managers/kubernetes/integration-tests/tests/pyfiles.py
@@ -14,9 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
-from __future__ import print_function
-
 import sys
 
 from pyspark.sql import SparkSession
@@ -34,7 +31,7 @@
 
     from py_container_checks import version_check
     # Begin of Python container checks
-    version_check(sys.argv[1], 2 if sys.argv[1] == "python" else 3)
+    version_check(sys.argv[1], 3)
 
     # Check python executable at executors
     spark.udf.register("get_sys_ver",
diff --git a/resource-managers/kubernetes/integration-tests/tests/python_executable_check.py b/resource-managers/kubernetes/integration-tests/tests/python_executable_check.py
new file mode 100644
index 0000000000000..89fd2aacab1a3
--- /dev/null
+++ b/resource-managers/kubernetes/integration-tests/tests/python_executable_check.py
@@ -0,0 +1,40 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import os
+
+from pyspark.sql import SparkSession
+
+
+if __name__ == "__main__":
+    spark = SparkSession \
+        .builder \
+        .appName("PythonExecutableTest") \
+        .getOrCreate()
+
+    # Check python executable at executors
+    is_custom_python_executor = spark.range(1).rdd.map(
+        lambda _: "IS_CUSTOM_PYTHON" in os.environ).first()
+
+    print("PYSPARK_PYTHON: %s" % os.environ.get("PYSPARK_PYTHON"))
+    print("PYSPARK_DRIVER_PYTHON: %s" % os.environ.get("PYSPARK_DRIVER_PYTHON"))
+
+    print("Custom Python used on executor: %s" % is_custom_python_executor)
+
+    is_custom_python_driver = "IS_CUSTOM_PYTHON" in os.environ
+    print("Custom Python used on driver: %s" % is_custom_python_driver)
+
+    spark.stop()
diff --git a/resource-managers/kubernetes/integration-tests/tests/worker_memory_check.py b/resource-managers/kubernetes/integration-tests/tests/worker_memory_check.py
index d312a29f388e4..74559a0b54402 100644
--- a/resource-managers/kubernetes/integration-tests/tests/worker_memory_check.py
+++ b/resource-managers/kubernetes/integration-tests/tests/worker_memory_check.py
@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 import resource
 import sys
 
diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml
index 54a8d66ea1ad6..b9b3642498992 100644
--- a/resource-managers/mesos/pom.xml
+++ b/resource-managers/mesos/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/config.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/config.scala
index e1c0d18b73a2b..38df43d71b897 100644
--- a/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/config.scala
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/config.scala
@@ -113,6 +113,16 @@ package object config {
       .stringConf
       .createOptional
 
+  private[spark] val DISPATCHER_QUEUE =
+    ConfigBuilder("spark.mesos.dispatcher.queue")
+      .doc("Set the name of the dispatcher queue to which the application is submitted. " +
+        "The specified queue must be added to the dispatcher " +
+        "with \"spark.mesos.dispatcher.queue.[QueueName]\". If no queue is specified, then " +
+        "the application is submitted to the \"default\" queue with 0.0 priority.")
+      .version("3.1.0")
+      .stringConf
+      .createWithDefaultString("default")
+
   private[spark] val DRIVER_LABELS =
     ConfigBuilder("spark.mesos.driver.labels")
       .doc("Mesos labels to add to the driver.  Labels are free-form key-value pairs. Key-value " +
@@ -173,6 +183,14 @@ package object config {
       .stringConf
       .createOptional
 
+  private[spark] val DRIVER_MEMORY_OVERHEAD =
+    ConfigBuilder("spark.mesos.driver.memoryOverhead")
+      .doc("The amount of additional memory, specified in MB, to be allocated to the driver. " +
+        "By default, the overhead will be larger of either 384 or 10% of spark.driver.memory. " +
+        "Only applies to cluster mode.")
+      .intConf
+      .createOptional
+
   private[spark] val EXECUTOR_URI =
     ConfigBuilder("spark.executor.uri").version("0.8.0").stringConf.createOptional
 
@@ -220,7 +238,7 @@ package object config {
     ConfigBuilder("spark.mesos.appJar.local.resolution.mode")
       .doc("Provides support for the `local:///` scheme to reference the app jar resource in " +
         "cluster mode. If user uses a local resource (`local:///path/to/jar`) and the config " +
-        "option is not used it defaults to `host` eg. the mesos fetcher tries to get the " +
+        "option is not used it defaults to `host` e.g. the mesos fetcher tries to get the " +
         "resource from the host's file system. If the value is unknown it prints a warning msg " +
         "in the dispatcher logs and defaults to `host`. If the value is `container` then spark " +
         "submit in the container will use the jar in the container's path: `/path/to/jar`.")
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/ui/DriverPage.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/ui/DriverPage.scala
index 8dc123e93fe16..97ef153177674 100644
--- a/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/ui/DriverPage.scala
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/ui/DriverPage.scala
@@ -22,7 +22,7 @@ import javax.servlet.http.HttpServletRequest
 import scala.xml.Node
 
 import org.apache.spark.deploy.Command
-import org.apache.spark.deploy.mesos.MesosDriverDescription
+import org.apache.spark.deploy.mesos.{config, MesosDriverDescription}
 import org.apache.spark.scheduler.cluster.mesos.{MesosClusterRetryState, MesosClusterSubmissionState}
 import org.apache.spark.ui.{UIUtils, WebUIPage}
 
@@ -92,8 +92,8 @@ private[ui] class DriverPage(parent: MesosClusterUI) extends WebUIPage("driver")
   private def launchedRow(submissionState: Option[MesosClusterSubmissionState]): Seq[Node] = {
     submissionState.map { state =>
       <tr>
-        <td>Mesos Slave ID</td>
-        <td>{state.slaveId.getValue}</td>
+        <td>Mesos Agent ID</td>
+        <td>{state.agentId.getValue}</td>
       </tr>
       <tr>
         <td>Mesos Task ID</td>
@@ -153,6 +153,13 @@ private[ui] class DriverPage(parent: MesosClusterUI) extends WebUIPage("driver")
     <tr>
       <td>Memory</td><td>{driver.mem}</td>
     </tr>
+    <tr>
+      <td>Queue</td>
+      <td>
+        {driver.conf.get(
+        "spark.mesos.dispatcher.queue", config.DISPATCHER_QUEUE.defaultValueString)}
+      </td>
+    </tr>
     <tr>
       <td>Submitted</td><td>{UIUtils.formatDate(driver.submissionDate)}</td>
     </tr>
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/ui/MesosClusterPage.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/ui/MesosClusterPage.scala
index 173a9b86e7de6..5c62ddb37684d 100644
--- a/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/ui/MesosClusterPage.scala
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/ui/MesosClusterPage.scala
@@ -23,7 +23,7 @@ import scala.xml.Node
 
 import org.apache.mesos.Protos.TaskStatus
 
-import org.apache.spark.deploy.mesos.MesosDriverDescription
+import org.apache.spark.deploy.mesos.{config, MesosDriverDescription}
 import org.apache.spark.deploy.mesos.config._
 import org.apache.spark.scheduler.cluster.mesos.MesosClusterSubmissionState
 import org.apache.spark.ui.{UIUtils, WebUIPage}
@@ -36,12 +36,12 @@ private[mesos] class MesosClusterPage(parent: MesosClusterUI) extends WebUIPage(
 
     val driverHeader = Seq("Driver ID")
     val historyHeader = historyServerURL.map(url => Seq("History")).getOrElse(Nil)
-    val submissionHeader = Seq("Submit Date", "Main Class", "Driver Resources")
+    val submissionHeader = Seq("Queue", "Submit Date", "Main Class", "Driver Resources")
     val sandboxHeader = Seq("Sandbox")
 
     val queuedHeaders = driverHeader ++ submissionHeader
     val driverHeaders = driverHeader ++ historyHeader ++ submissionHeader ++
-      Seq("Start Date", "Mesos Slave ID", "State") ++ sandboxHeader
+      Seq("Start Date", "Mesos Agent ID", "State") ++ sandboxHeader
     val retryHeaders = Seq("Driver ID", "Submit Date", "Description") ++
       Seq("Last Failed Status", "Next Retry Time", "Attempt Count")
     val queuedTable = UIUtils.listingTable(queuedHeaders, queuedRow, state.queuedDrivers)
@@ -69,6 +69,10 @@ private[mesos] class MesosClusterPage(parent: MesosClusterUI) extends WebUIPage(
     val id = submission.submissionId
     <tr>
       <td><a href={s"driver?id=$id"}>{id}</a></td>
+      <td>
+        {submission.conf.get(
+        "spark.mesos.dispatcher.queue", config.DISPATCHER_QUEUE.defaultValueString)}
+      </td>
       <td>{UIUtils.formatDate(submission.submissionDate)}</td>
       <td>{submission.command.mainClass}</td>
       <td>cpus: {submission.cores}, mem: {submission.mem}</td>
@@ -81,7 +85,7 @@ private[mesos] class MesosClusterPage(parent: MesosClusterUI) extends WebUIPage(
 
     val sandboxCol = if (proxy.isDefined) {
       val clusterSchedulerId = parent.scheduler.getSchedulerState().frameworkId
-      val sandBoxUri = s"${proxy.get}/#/agents/${state.slaveId.getValue}/frameworks/" +
+      val sandBoxUri = s"${proxy.get}/#/agents/${state.agentId.getValue}/frameworks/" +
         s"${clusterSchedulerId}/executors/${id}/browse"
       <a href={sandBoxUri}>Sandbox</a>
     } else {
@@ -99,11 +103,15 @@ private[mesos] class MesosClusterPage(parent: MesosClusterUI) extends WebUIPage(
     <tr>
       <td><a href={s"driver?id=$id"}>{id}</a></td>
       {historyCol}
+      <td>
+        {state.driverDescription.conf.get(
+        "spark.mesos.dispatcher.queue", config.DISPATCHER_QUEUE.defaultValueString)}
+      </td>
       <td>{UIUtils.formatDate(state.driverDescription.submissionDate)}</td>
       <td>{state.driverDescription.command.mainClass}</td>
       <td>cpus: {state.driverDescription.cores}, mem: {state.driverDescription.mem}</td>
       <td>{UIUtils.formatDate(state.startDate)}</td>
-      <td>{state.slaveId.getValue}</td>
+      <td>{state.agentId.getValue}</td>
       <td>{stateString(state.mesosTaskStatus)}</td>
       <td>{sandboxCol}</td>
     </tr>
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/ui/MesosClusterUI.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/ui/MesosClusterUI.scala
index c0cdcda14291f..e260fb8e25f4c 100644
--- a/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/ui/MesosClusterUI.scala
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/ui/MesosClusterUI.scala
@@ -20,7 +20,6 @@ package org.apache.spark.deploy.mesos.ui
 import org.apache.spark.{SecurityManager, SparkConf}
 import org.apache.spark.scheduler.cluster.mesos.MesosClusterScheduler
 import org.apache.spark.ui.{SparkUI, WebUI}
-import org.apache.spark.ui.JettyUtils._
 
 /**
  * UI that displays driver results from the [[org.apache.spark.deploy.mesos.MesosClusterDispatcher]]
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/executor/MesosExecutorBackend.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/executor/MesosExecutorBackend.scala
index 47243e83d1335..b023cf1fa4bb2 100644
--- a/resource-managers/mesos/src/main/scala/org/apache/spark/executor/MesosExecutorBackend.scala
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/executor/MesosExecutorBackend.scala
@@ -26,18 +26,16 @@ import org.apache.mesos.Protos.{TaskStatus => MesosTaskStatus, _}
 import org.apache.mesos.protobuf.ByteString
 
 import org.apache.spark.{SparkConf, SparkEnv, TaskState}
-import org.apache.spark.TaskState
 import org.apache.spark.deploy.SparkHadoopUtil
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.EXECUTOR_ID
 import org.apache.spark.resource.ResourceInformation
 import org.apache.spark.scheduler.TaskDescription
-import org.apache.spark.scheduler.cluster.mesos.MesosSchedulerUtils
+import org.apache.spark.scheduler.cluster.mesos.MesosSchedulerBackendUtil
 import org.apache.spark.util.Utils
 
 private[spark] class MesosExecutorBackend
   extends MesosExecutor
-  with MesosSchedulerUtils // TODO: fix
   with ExecutorBackend
   with Logging {
 
@@ -48,7 +46,7 @@ private[spark] class MesosExecutorBackend
     val mesosTaskId = TaskID.newBuilder().setValue(taskId.toString).build()
     driver.sendStatusUpdate(MesosTaskStatus.newBuilder()
       .setTaskId(mesosTaskId)
-      .setState(taskStateToMesos(state))
+      .setState(MesosSchedulerBackendUtil.taskStateToMesos(state))
       .setData(ByteString.copyFrom(data))
       .build())
   }
@@ -57,7 +55,7 @@ private[spark] class MesosExecutorBackend
       driver: ExecutorDriver,
       executorInfo: ExecutorInfo,
       frameworkInfo: FrameworkInfo,
-      slaveInfo: SlaveInfo): Unit = {
+      agentInfo: SlaveInfo): Unit = {
 
     // Get num cores for this task from ExecutorInfo, created in MesosSchedulerBackend.
     val cpusPerTask = executorInfo.getResourcesList.asScala
@@ -78,11 +76,11 @@ private[spark] class MesosExecutorBackend
     val conf = new SparkConf(loadDefaults = true).setAll(properties)
     conf.set(EXECUTOR_ID, executorId)
     val env = SparkEnv.createExecutorEnv(
-      conf, executorId, slaveInfo.getHostname, cpusPerTask, None, isLocal = false)
+      conf, executorId, agentInfo.getHostname, cpusPerTask, None, isLocal = false)
 
     executor = new Executor(
       executorId,
-      slaveInfo.getHostname,
+      agentInfo.getHostname,
       env,
       resources = Map.empty[String, ResourceInformation])
   }
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala
index 289b109a42747..16cffd03135df 100644
--- a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala
@@ -21,15 +21,16 @@ import java.io.File
 import java.util.{Collections, Date, List => JList}
 
 import scala.collection.JavaConverters._
+import scala.collection.immutable
 import scala.collection.mutable
 import scala.collection.mutable.ArrayBuffer
 
 import org.apache.mesos.{Scheduler, SchedulerDriver}
-import org.apache.mesos.Protos.{TaskState => MesosTaskState, _}
+import org.apache.mesos.Protos.{SlaveID => AgentID, TaskState => MesosTaskState, _}
 import org.apache.mesos.Protos.Environment.Variable
 import org.apache.mesos.Protos.TaskStatus.Reason
 
-import org.apache.spark.{SecurityManager, SparkConf, SparkException, TaskState}
+import org.apache.spark.{SparkConf, SparkException, TaskState}
 import org.apache.spark.deploy.mesos.{config, MesosDriverDescription}
 import org.apache.spark.deploy.rest.{CreateSubmissionResponse, KillSubmissionResponse, SubmissionStatusResponse}
 import org.apache.spark.internal.config._
@@ -41,7 +42,7 @@ import org.apache.spark.util.Utils
  * @param driverDescription Submitted driver description from
  * [[org.apache.spark.deploy.rest.mesos.MesosRestServer]]
  * @param taskId Mesos TaskID generated for the task
- * @param slaveId Slave ID that the task is assigned to
+ * @param agentId Agent ID that the task is assigned to
  * @param mesosTaskStatus The last known task status update.
  * @param startDate The date the task was launched
  * @param finishDate The date the task finished
@@ -50,7 +51,7 @@ import org.apache.spark.util.Utils
 private[spark] class MesosClusterSubmissionState(
     val driverDescription: MesosDriverDescription,
     val taskId: TaskID,
-    val slaveId: SlaveID,
+    val agentId: AgentID,
     var mesosTaskStatus: Option[TaskStatus],
     var startDate: Date,
     var finishDate: Option[Date],
@@ -59,7 +60,7 @@ private[spark] class MesosClusterSubmissionState(
 
   def copy(): MesosClusterSubmissionState = {
     new MesosClusterSubmissionState(
-      driverDescription, taskId, slaveId, mesosTaskStatus, startDate, finishDate, frameworkId)
+      driverDescription, taskId, agentId, mesosTaskStatus, startDate, finishDate, frameworkId)
   }
 }
 
@@ -113,7 +114,7 @@ private[spark] class MesosDriverState(
  * A Mesos scheduler that is responsible for launching submitted Spark drivers in cluster mode
  * as Mesos tasks in a Mesos cluster.
  * All drivers are launched asynchronously by the framework, which will eventually be launched
- * by one of the slaves in the cluster. The results of the driver will be stored in slave's task
+ * by one of the agents in the cluster. The results of the driver will be stored in agent's task
  * sandbox which is accessible by visiting the Mesos UI.
  * This scheduler supports recovery by persisting all its state and performs task reconciliation
  * on recover, which gets all the latest state for all the drivers from Mesos master.
@@ -121,16 +122,17 @@ private[spark] class MesosDriverState(
 private[spark] class MesosClusterScheduler(
     engineFactory: MesosClusterPersistenceEngineFactory,
     conf: SparkConf)
-  extends Scheduler with MesosSchedulerUtils {
+  extends Scheduler with MesosSchedulerUtils with MesosScheduler {
   var frameworkUrl: String = _
   private val metricsSystem =
-    MetricsSystem.createMetricsSystem(MetricsSystemInstances.MESOS_CLUSTER, conf,
-      new SecurityManager(conf))
+    MetricsSystem.createMetricsSystem(MetricsSystemInstances.MESOS_CLUSTER, conf)
   private val master = conf.get("spark.master")
   private val appName = conf.get("spark.app.name")
   private val queuedCapacity = conf.get(config.MAX_DRIVERS)
   private val retainedDrivers = conf.get(config.RETAINED_DRIVERS)
   private val maxRetryWaitTime = conf.get(config.CLUSTER_RETRY_WAIT_MAX_SECONDS)
+  private val queues: immutable.Map[String, Float] =
+    conf.getAllWithPrefix("spark.mesos.dispatcher.queue.").map(t => (t._1, t._2.toFloat)).toMap
   private val schedulerState = engineFactory.createEngine("scheduler")
   private val stateLock = new Object()
   // Keyed by submission id
@@ -139,12 +141,24 @@ private[spark] class MesosClusterScheduler(
   private var frameworkId: String = null
   // Holds all the launched drivers and current launch state, keyed by submission id.
   private val launchedDrivers = new mutable.HashMap[String, MesosClusterSubmissionState]()
-  // Holds a map of driver id to expected slave id that is passed to Mesos for reconciliation.
+  // Holds a map of driver id to expected agent id that is passed to Mesos for reconciliation.
   // All drivers that are loaded after failover are added here, as we need get the latest
   // state of the tasks from Mesos. Keyed by task Id.
-  private val pendingRecover = new mutable.HashMap[String, SlaveID]()
+  private val pendingRecover = new mutable.HashMap[String, AgentID]()
   // Stores all the submitted drivers that hasn't been launched, keyed by submission id
-  private val queuedDrivers = new ArrayBuffer[MesosDriverDescription]()
+  // and sorted by priority, then by submission date
+  private val driverOrdering = new Ordering[MesosDriverDescription] {
+    override def compare(x: MesosDriverDescription, y: MesosDriverDescription): Int = {
+      val xp = getDriverPriority(x)
+      val yp = getDriverPriority(y)
+      if (xp != yp) {
+        xp compare yp
+      } else {
+        y.submissionDate.compareTo(x.submissionDate)
+      }
+    }
+  }
+  private val queuedDrivers = new mutable.TreeSet[MesosDriverDescription]()(driverOrdering.reverse)
   // All supervised drivers that are waiting to retry after termination, keyed by submission id
   private val pendingRetryDrivers = new ArrayBuffer[MesosDriverDescription]()
   private val queuedDriversState = engineFactory.createEngine("driverQueue")
@@ -277,7 +291,7 @@ private[spark] class MesosClusterScheduler(
     stateLock.synchronized {
       launchedDriversState.fetchAll[MesosClusterSubmissionState]().foreach { state =>
         launchedDrivers(state.driverDescription.submissionId) = state
-        pendingRecover(state.taskId.getValue) = state.slaveId
+        pendingRecover(state.taskId.getValue) = state.agentId
       }
       queuedDriversState.fetchAll[MesosDriverDescription]().foreach(d => queuedDrivers += d)
       // There is potential timing issue where a queued driver might have been launched
@@ -348,10 +362,10 @@ private[spark] class MesosClusterScheduler(
       if (!pendingRecover.isEmpty) {
         // Start task reconciliation if we need to recover.
         val statuses = pendingRecover.collect {
-          case (taskId, slaveId) =>
+          case (taskId, agentId) =>
             val newStatus = TaskStatus.newBuilder()
               .setTaskId(TaskID.newBuilder().setValue(taskId).build())
-              .setSlaveId(slaveId)
+              .setSlaveId(agentId)
               .setState(MesosTaskState.TASK_STAGING)
               .build()
             launchedDrivers.get(getSubmissionIdFromTaskId(taskId))
@@ -374,6 +388,16 @@ private[spark] class MesosClusterScheduler(
     s"${frameworkId}-${desc.submissionId}${retries}"
   }
 
+  private[mesos] def getDriverPriority(desc: MesosDriverDescription): Float = {
+    val defaultQueueName = config.DISPATCHER_QUEUE.defaultValueString
+    val queueName = desc.conf.get("spark.mesos.dispatcher.queue", defaultQueueName)
+    if (queueName != defaultQueueName) {
+      queues.getOrElse(queueName, throw new NoSuchElementException(queueName))
+    } else {
+      0.0f
+    }
+  }
+
   private def getDriverTaskId(desc: MesosDriverDescription): String = {
     val sId = desc.submissionId
     desc.retryState.map(state => sId + s"${RETRY_SEP}${state.retries.toString}").getOrElse(sId)
@@ -383,13 +407,13 @@ private[spark] class MesosClusterScheduler(
     taskId.split(s"${RETRY_SEP}").head
   }
 
-  private def adjust[A, B](m: collection.Map[A, B], k: A, default: B)(f: B => B) = {
+  private def adjust[A, B](m: Map[A, B], k: A, default: B)(f: B => B) = {
     m.updated(k, f(m.getOrElse(k, default)))
   }
 
   private def getDriverEnvironment(desc: MesosDriverDescription): Environment = {
     // TODO(mgummelt): Don't do this here.  This should be passed as a --conf
-    val commandEnv = adjust(desc.command.environment, "SPARK_SUBMIT_OPTS", "")(
+    val commandEnv = adjust(desc.command.environment.toMap, "SPARK_SUBMIT_OPTS", "")(
       v => s"$v -D${config.DRIVER_FRAMEWORK_ID.key}=${getDriverFrameworkID(desc)}"
     )
 
@@ -536,19 +560,21 @@ private[spark] class MesosClusterScheduler(
     val formattedFiles = pyFiles.map { path =>
       new File(sandboxPath, path.split("/").last).toString()
     }.mkString(",")
-    options ++= Seq("--py-files", formattedFiles)
+    if (!formattedFiles.equals("")) {
+      options ++= Seq("--py-files", formattedFiles)
+    }
 
     // --conf
-    val replicatedOptionsBlacklist = Set(
+    val replicatedOptionsExcludeList = Set(
       JARS.key, // Avoids duplicate classes in classpath
       SUBMIT_DEPLOY_MODE.key, // this would be set to `cluster`, but we need client
       "spark.master" // this contains the address of the dispatcher, not master
     )
     val defaultConf = conf.getAllWithPrefix(config.DISPATCHER_DRIVER_DEFAULT_PREFIX).toMap
     val driverConf = desc.conf.getAll
-      .filter { case (key, _) => !replicatedOptionsBlacklist.contains(key) }
+      .filter { case (key, _) => !replicatedOptionsExcludeList.contains(key) }
       .toMap
-    (defaultConf ++ driverConf).foreach { case (key, value) =>
+    (defaultConf ++ driverConf).toSeq.sortBy(_._1).foreach { case (key, value) =>
       options ++= Seq("--conf", s"${key}=${value}") }
 
     options.map(shellEscape)
@@ -587,7 +613,7 @@ private[spark] class MesosClusterScheduler(
     val (remainingResources, cpuResourcesToUse) =
       partitionResources(offer.remainingResources, "cpus", desc.cores)
     val (finalResources, memResourcesToUse) =
-      partitionResources(remainingResources.asJava, "mem", desc.mem)
+      partitionResources(remainingResources.asJava, "mem", driverContainerMemory(desc))
     offer.remainingResources = finalResources.asJava
 
     val appName = desc.conf.get("spark.app.name")
@@ -619,7 +645,7 @@ private[spark] class MesosClusterScheduler(
       tasks: mutable.HashMap[OfferID, ArrayBuffer[TaskInfo]]): Unit = {
     for (submission <- candidates) {
       val driverCpu = submission.cores
-      val driverMem = submission.mem
+      val driverMem = driverContainerMemory(submission)
       val driverConstraints =
         parseConstraintString(submission.conf.get(config.DRIVER_CONSTRAINTS))
       logTrace(s"Finding offer to launch driver with cpu: $driverCpu, mem: $driverMem, " +
@@ -657,7 +683,7 @@ private[spark] class MesosClusterScheduler(
             finishedDrivers += new MesosClusterSubmissionState(
               submission,
               TaskID.newBuilder().setValue(submission.submissionId).build(),
-              SlaveID.newBuilder().setValue("").build(),
+              AgentID.newBuilder().setValue("").build(),
               None,
               null,
               None,
@@ -686,14 +712,14 @@ private[spark] class MesosClusterScheduler(
       }
 
       scheduleTasks(
-        copyBuffer(driversToRetry),
+        copyBuffer(driversToRetry).toSeq,
         removeFromPendingRetryDrivers,
         currentOffers,
         tasks)
 
       // Then we walk through the queued drivers and try to schedule them.
       scheduleTasks(
-        copyBuffer(queuedDrivers),
+        copyBuffer(queuedDrivers).toSeq,
         removeFromQueuedDrivers,
         currentOffers,
         tasks)
@@ -708,7 +734,7 @@ private[spark] class MesosClusterScheduler(
   }
 
   private def copyBuffer(
-      buffer: ArrayBuffer[MesosDriverDescription]): ArrayBuffer[MesosDriverDescription] = {
+      buffer: TraversableOnce[MesosDriverDescription]): ArrayBuffer[MesosDriverDescription] = {
     val newBuffer = new ArrayBuffer[MesosDriverDescription](buffer.size)
     buffer.copyToBuffer(newBuffer)
     newBuffer
@@ -731,7 +757,7 @@ private[spark] class MesosClusterScheduler(
   override def reregistered(driver: SchedulerDriver, masterInfo: MasterInfo): Unit = {
     logInfo(s"Framework re-registered with master ${masterInfo.getId}")
   }
-  override def slaveLost(driver: SchedulerDriver, slaveId: SlaveID): Unit = {}
+  override def agentLost(driver: SchedulerDriver, agentId: AgentID): Unit = {}
   override def error(driver: SchedulerDriver, error: String): Unit = {
     logError("Error received: " + error)
     markErr()
@@ -815,23 +841,23 @@ private[spark] class MesosClusterScheduler(
   override def frameworkMessage(
       driver: SchedulerDriver,
       executorId: ExecutorID,
-      slaveId: SlaveID,
+      agentId: AgentID,
       message: Array[Byte]): Unit = {}
 
   override def executorLost(
       driver: SchedulerDriver,
       executorId: ExecutorID,
-      slaveId: SlaveID,
+      agentId: AgentID,
       status: Int): Unit = {}
 
   private def removeFromQueuedDrivers(subId: String): Boolean = {
-    val index = queuedDrivers.indexWhere(_.submissionId == subId)
-    if (index != -1) {
-      queuedDrivers.remove(index)
+    val matchOption = queuedDrivers.find(_.submissionId == subId)
+    if (matchOption.isEmpty) {
+      false
+    } else {
+      queuedDrivers.remove(matchOption.get)
       queuedDriversState.expunge(subId)
       true
-    } else {
-      false
     }
   }
 
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
index 0b447025c8a7a..6fedce61d8208 100644
--- a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
@@ -27,20 +27,21 @@ import scala.collection.JavaConverters._
 import scala.collection.mutable
 import scala.concurrent.Future
 
-import org.apache.mesos.Protos.{TaskInfo => MesosTaskInfo, _}
+import org.apache.mesos.Protos.{SlaveID => AgentID, TaskInfo => MesosTaskInfo, _}
 import org.apache.mesos.SchedulerDriver
 
 import org.apache.spark.{SecurityManager, SparkConf, SparkContext, SparkException, TaskState}
 import org.apache.spark.deploy.mesos.config._
 import org.apache.spark.deploy.security.HadoopDelegationTokenManager
 import org.apache.spark.internal.config
+import org.apache.spark.internal.config.Network
 import org.apache.spark.internal.config.Tests.IS_TESTING
 import org.apache.spark.launcher.{LauncherBackend, SparkAppHandle}
 import org.apache.spark.network.netty.SparkTransportConf
 import org.apache.spark.network.shuffle.mesos.MesosExternalBlockStoreClient
 import org.apache.spark.resource.ResourceProfile
-import org.apache.spark.rpc.{RpcEndpointAddress, RpcEndpointRef}
-import org.apache.spark.scheduler.{SlaveLost, TaskSchedulerImpl}
+import org.apache.spark.rpc.RpcEndpointAddress
+import org.apache.spark.scheduler.{ExecutorProcessLost, TaskSchedulerImpl}
 import org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend
 import org.apache.spark.util.Utils
 
@@ -60,10 +61,11 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
     master: String,
     securityManager: SecurityManager)
   extends CoarseGrainedSchedulerBackend(scheduler, sc.env.rpcEnv)
-    with org.apache.mesos.Scheduler with MesosSchedulerUtils {
+  with MesosScheduler
+  with MesosSchedulerUtils {
 
-  // Blacklist a slave after this many failures
-  private val MAX_SLAVE_FAILURES = 2
+  // Exclude an agent after this many failures
+  private val MAX_AGENT_FAILURES = 2
 
   private val maxCoresOption = conf.get(config.CORES_MAX)
 
@@ -116,10 +118,10 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
   // executor limit
   private var launchingExecutors = false
 
-  // SlaveID -> Slave
-  // This map accumulates entries for the duration of the job.  Slaves are never deleted, because
+  // AgentID -> Agent
+  // This map accumulates entries for the duration of the job.  Agents are never deleted, because
   // we need to maintain e.g. failure state and connection state.
-  private val slaves = new mutable.HashMap[String, Slave]
+  private val agents = new mutable.HashMap[String, Agent]
 
   /**
    * The total number of executors we aim to have. Undefined when not using dynamic allocation.
@@ -147,7 +149,7 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
   private val extraCoresPerExecutor = conf.get(EXTRA_CORES_PER_EXECUTOR)
 
   // Offer constraints
-  private val slaveOfferConstraints =
+  private val agentOfferConstraints =
     parseConstraintString(sc.conf.get(CONSTRAINTS))
 
   // Reject offers with mismatched constraints in seconds
@@ -354,7 +356,7 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
   }
 
   /**
-   * Method called by Mesos to offer resources on slaves. We respond by launching an executor,
+   * Method called by Mesos to offer resources on agents. We respond by launching an executor,
    * unless we've already launched more than we wanted to.
    */
   override def resourceOffers(d: org.apache.mesos.SchedulerDriver, offers: JList[Offer]): Unit = {
@@ -384,7 +386,7 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
 
       val (matchedOffers, unmatchedOffers) = offers.asScala.partition { offer =>
         val offerAttributes = toAttributeMap(offer.getAttributesList)
-        matchesAttributeRequirements(slaveOfferConstraints, offerAttributes)
+        matchesAttributeRequirements(agentOfferConstraints, offerAttributes)
       }
 
       declineUnmatchedOffers(d, unmatchedOffers)
@@ -441,7 +443,7 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
           val ports = getRangeResource(task.getResourcesList, "ports").mkString(",")
 
           logDebug(s"Launching Mesos task: ${taskId.getValue} with mem: $mem cpu: $cpus" +
-            s" ports: $ports" + s" on slave with slave id: ${task.getSlaveId.getValue} ")
+            s" ports: $ports" + s" on agent with agent id: ${task.getSlaveId.getValue} ")
         }
 
         driver.launchTasks(
@@ -490,23 +492,23 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
     val tasks = new mutable.HashMap[OfferID, List[MesosTaskInfo]].withDefaultValue(Nil)
 
     // offerID -> resources
-    val remainingResources = mutable.Map(offers.map(offer =>
-      (offer.getId.getValue, offer.getResourcesList)): _*)
+    val remainingResources = mutable.Map[String, JList[Resource]]()
+    remainingResources ++= offers.map(offer => (offer.getId.getValue, offer.getResourcesList))
 
     var launchTasks = true
 
-    // TODO(mgummelt): combine offers for a single slave
+    // TODO(mgummelt): combine offers for a single agent
     //
     // round-robin create executors on the available offers
     while (launchTasks) {
       launchTasks = false
 
       for (offer <- offers) {
-        val slaveId = offer.getSlaveId.getValue
+        val agentId = offer.getSlaveId.getValue
         val offerId = offer.getId.getValue
         val resources = remainingResources(offerId)
 
-        if (canLaunchTask(slaveId, offer.getHostname, resources)) {
+        if (canLaunchTask(agentId, offer.getHostname, resources)) {
           // Create a task
           launchTasks = true
           val taskId = newMesosTaskId()
@@ -517,7 +519,7 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
           val taskCPUs = executorCores(offerCPUs)
           val taskMemory = executorMemory(sc)
 
-          slaves.getOrElseUpdate(slaveId, new Slave(offer.getHostname)).taskIDs.add(taskId)
+          agents.getOrElseUpdate(agentId, new Agent(offer.getHostname)).taskIDs.add(taskId)
 
           val (resourcesLeft, resourcesToUse) =
             partitionTaskResources(resources, taskCPUs, taskMemory, taskGPUs)
@@ -540,8 +542,8 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
             gpusByTaskId(taskId) = taskGPUs
           }
         } else {
-          logDebug(s"Cannot launch a task for offer with id: $offerId on slave " +
-            s"with id: $slaveId. Requirements were not met for this offer.")
+          logDebug(s"Cannot launch a task for offer with id: $offerId on agent " +
+            s"with id: $agentId. Requirements were not met for this offer.")
         }
       }
     }
@@ -573,7 +575,7 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
       cpuResourcesToUse ++ memResourcesToUse ++ portResourcesToUse ++ gpuResourcesToUse)
   }
 
-  private def canLaunchTask(slaveId: String, offerHostname: String,
+  private def canLaunchTask(agentId: String, offerHostname: String,
                             resources: JList[Resource]): Boolean = {
     val offerMem = getResource(resources, "mem")
     val offerCPUs = getResource(resources, "cpus").toInt
@@ -587,7 +589,7 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
       cpus + totalCoresAcquired <= maxCores &&
       mem <= offerMem &&
       numExecutors < executorLimit &&
-      slaves.get(slaveId).map(_.taskFailures).getOrElse(0) < MAX_SLAVE_FAILURES &&
+      agents.get(agentId).map(_.taskFailures).getOrElse(0) < MAX_AGENT_FAILURES &&
       meetsPortRequirements &&
       satisfiesLocality(offerHostname)
   }
@@ -606,7 +608,7 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
     }
 
     // Check the locality information
-    val currentHosts = slaves.values.filter(_.taskIDs.nonEmpty).map(_.hostname).toSet
+    val currentHosts = agents.values.filter(_.taskIDs.nonEmpty).map(_.hostname).toSet
     val allDesiredHosts = hostToLocalTaskCount.map { case (k, v) => k }.toSet
 
     // Try to match locality for hosts which do not have executors yet, to potentially
@@ -622,13 +624,13 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
 
   override def statusUpdate(d: org.apache.mesos.SchedulerDriver, status: TaskStatus): Unit = {
     val taskId = status.getTaskId.getValue
-    val slaveId = status.getSlaveId.getValue
+    val agentId = status.getSlaveId.getValue
     val state = mesosToTaskState(status.getState)
 
     logInfo(s"Mesos task $taskId is now ${status.getState}")
 
     stateLock.synchronized {
-      val slave = slaves(slaveId)
+      val agent = agents(agentId)
 
       // If the shuffle service is enabled, have the driver register with each one of the
       // shuffle services. This allows the shuffle services to clean up state associated with
@@ -636,23 +638,25 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
       // this through Mesos, since the shuffle services are set up independently.
       if (state.equals(TaskState.RUNNING) &&
           shuffleServiceEnabled &&
-          !slave.shuffleRegistered) {
+          !agent.shuffleRegistered) {
         assume(mesosExternalShuffleClient.isDefined,
           "External shuffle client was not instantiated even though shuffle service is enabled.")
         // TODO: Remove this and allow the MesosExternalShuffleService to detect
         // framework termination when new Mesos Framework HTTP API is available.
         val externalShufflePort = conf.get(config.SHUFFLE_SERVICE_PORT)
 
-        logDebug(s"Connecting to shuffle service on slave $slaveId, " +
-            s"host ${slave.hostname}, port $externalShufflePort for app ${conf.getAppId}")
+        logDebug(s"Connecting to shuffle service on agent $agentId, " +
+            s"host ${agent.hostname}, port $externalShufflePort for app ${conf.getAppId}")
 
         mesosExternalShuffleClient.get
           .registerDriverWithShuffleService(
-            slave.hostname,
+            agent.hostname,
             externalShufflePort,
-            sc.conf.get(config.STORAGE_BLOCKMANAGER_SLAVE_TIMEOUT),
+            sc.conf.get(
+              config.STORAGE_BLOCKMANAGER_HEARTBEAT_TIMEOUT
+            ).getOrElse(Utils.timeStringAsMs(s"${sc.conf.get(Network.NETWORK_TIMEOUT)}s")),
             sc.conf.get(config.EXECUTOR_HEARTBEAT_INTERVAL))
-        slave.shuffleRegistered = true
+        agent.shuffleRegistered = true
       }
 
       if (TaskState.isFinished(state)) {
@@ -666,16 +670,16 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
           totalGpusAcquired -= gpus
           gpusByTaskId -= taskId
         }
-        // If it was a failure, mark the slave as failed for blacklisting purposes
+        // If it was a failure, mark the agent as failed for excluding purposes
         if (TaskState.isFailed(state)) {
-          slave.taskFailures += 1
+          agent.taskFailures += 1
 
-          if (slave.taskFailures >= MAX_SLAVE_FAILURES) {
-            logInfo(s"Blacklisting Mesos slave $slaveId due to too many failures; " +
+          if (agent.taskFailures >= MAX_AGENT_FAILURES) {
+            logInfo(s"Excluding Mesos agent $agentId due to too many failures; " +
                 "is Spark installed on it?")
           }
         }
-        executorTerminated(d, slaveId, taskId, s"Executor finished with state $state")
+        executorTerminated(d, agentId, taskId, s"Executor finished with state $state")
         // In case we'd rejected everything before but have now lost a node
         d.reviveOffers()
       }
@@ -708,7 +712,7 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
     // See SPARK-12330
     val startTime = System.nanoTime()
 
-    // slaveIdsWithExecutors has no memory barrier, so this is eventually consistent
+    // agentIdsWithExecutors has no memory barrier, so this is eventually consistent
     while (numExecutors() > 0 &&
       System.nanoTime() - startTime < shutdownTimeoutMS * 1000L * 1000L) {
       Thread.sleep(100)
@@ -729,15 +733,15 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
   }
 
   override def frameworkMessage(
-      d: org.apache.mesos.SchedulerDriver, e: ExecutorID, s: SlaveID, b: Array[Byte]): Unit = {}
+      d: org.apache.mesos.SchedulerDriver, e: ExecutorID, s: AgentID, b: Array[Byte]): Unit = {}
 
   /**
-   * Called when a slave is lost or a Mesos task finished. Updates local view on
+   * Called when a agent is lost or a Mesos task finished. Updates local view on
    * what tasks are running. It also notifies the driver that an executor was removed.
    */
   private def executorTerminated(
       d: org.apache.mesos.SchedulerDriver,
-      slaveId: String,
+      agentId: String,
       taskId: String,
       reason: String): Unit = {
     stateLock.synchronized {
@@ -745,18 +749,18 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
       // removeExecutor() internally will send a message to the driver endpoint but
       // the driver endpoint is not available now, otherwise an exception will be thrown.
       if (!stopCalled) {
-        removeExecutor(taskId, SlaveLost(reason))
+        removeExecutor(taskId, ExecutorProcessLost(reason))
       }
-      slaves(slaveId).taskIDs.remove(taskId)
+      agents(agentId).taskIDs.remove(taskId)
     }
   }
 
-  override def slaveLost(d: org.apache.mesos.SchedulerDriver, slaveId: SlaveID): Unit = {
-    logInfo(s"Mesos slave lost: ${slaveId.getValue}")
+  override def agentLost(d: org.apache.mesos.SchedulerDriver, agentId: AgentID): Unit = {
+    logInfo(s"Mesos agent lost: ${agentId.getValue}")
   }
 
   override def executorLost(
-      d: org.apache.mesos.SchedulerDriver, e: ExecutorID, s: SlaveID, status: Int): Unit = {
+      d: org.apache.mesos.SchedulerDriver, e: ExecutorID, s: AgentID, status: Int): Unit = {
     logInfo("Mesos executor lost: %s".format(e.getValue))
   }
 
@@ -770,7 +774,7 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
       resourceProfileToTotalExecs: Map[ResourceProfile, Int]
   ): Future[Boolean] = Future.successful {
     // We don't truly know if we can fulfill the full amount of executors
-    // since at coarse grain it depends on the amount of slaves available.
+    // since at coarse grain it depends on the amount of agents available.
     val numExecs = resourceProfileToTotalExecs.getOrElse(defaultProfile, 0)
     logInfo("Capping the total amount of executors to " + numExecs)
     executorLimitOption = Some(numExecs)
@@ -800,11 +804,11 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
   }
 
   private def numExecutors(): Int = {
-    slaves.values.map(_.taskIDs.size).sum
+    agents.values.map(_.taskIDs.size).sum
   }
 }
 
-private class Slave(val hostname: String) {
+private class Agent(val hostname: String) {
   val taskIDs = new mutable.HashSet[String]()
   var taskFailures = 0
   var shuffleRegistered = false
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosFineGrainedSchedulerBackend.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosFineGrainedSchedulerBackend.scala
index f1e3fcab7e6af..586c2bdd67cfa 100644
--- a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosFineGrainedSchedulerBackend.scala
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosFineGrainedSchedulerBackend.scala
@@ -23,7 +23,8 @@ import java.util.{ArrayList => JArrayList, Collections, List => JList}
 import scala.collection.JavaConverters._
 import scala.collection.mutable.{HashMap, HashSet}
 
-import org.apache.mesos.Protos.{ExecutorInfo => MesosExecutorInfo, TaskInfo => MesosTaskInfo, _}
+import org.apache.mesos.Protos.{ExecutorInfo => MesosExecutorInfo, SlaveID => AgentID,
+  TaskInfo => MesosTaskInfo, _}
 import org.apache.mesos.SchedulerDriver
 import org.apache.mesos.protobuf.ByteString
 
@@ -46,12 +47,12 @@ private[spark] class MesosFineGrainedSchedulerBackend(
     sc: SparkContext,
     master: String)
   extends SchedulerBackend
-  with org.apache.mesos.Scheduler
+  with MesosScheduler
   with MesosSchedulerUtils {
 
-  // Stores the slave ids that has launched a Mesos executor.
-  val slaveIdToExecutorInfo = new HashMap[String, MesosExecutorInfo]
-  val taskIdToSlaveId = new HashMap[Long, String]
+  // Stores the agent ids that has launched a Mesos executor.
+  val agentIdToExecutorInfo = new HashMap[String, MesosExecutorInfo]
+  val taskIdToAgentId = new HashMap[Long, String]
 
   // An ExecutorInfo for our tasks
   var execArgs: Array[Byte] = null
@@ -64,7 +65,7 @@ private[spark] class MesosFineGrainedSchedulerBackend(
   private[mesos] val mesosExecutorCores = sc.conf.get(mesosConfig.EXECUTOR_CORES)
 
   // Offer constraints
-  private[this] val slaveOfferConstraints =
+  private[this] val agentOfferConstraints =
     parseConstraintString(sc.conf.get(mesosConfig.CONSTRAINTS))
 
   // reject offers with mismatched constraints in seconds
@@ -217,7 +218,7 @@ private[spark] class MesosFineGrainedSchedulerBackend(
     val builder = new StringBuilder
     tasks.asScala.foreach { t =>
       builder.append("Task id: ").append(t.getTaskId.getValue).append("\n")
-        .append("Slave id: ").append(t.getSlaveId.getValue).append("\n")
+        .append("Agent id: ").append(t.getSlaveId.getValue).append("\n")
         .append("Task resources: ").append(t.getResourcesList).append("\n")
         .append("Executor resources: ").append(t.getExecutor.getResourcesList)
         .append("---------------------------------------------\n")
@@ -226,7 +227,7 @@ private[spark] class MesosFineGrainedSchedulerBackend(
   }
 
   /**
-   * Method called by Mesos to offer resources on slaves. We respond by asking our active task sets
+   * Method called by Mesos to offer resources on agents. We respond by asking our active task sets
    * for tasks in order of priority. We fill each node with tasks in a round-robin manner so that
    * tasks are balanced across the cluster.
    */
@@ -237,7 +238,7 @@ private[spark] class MesosFineGrainedSchedulerBackend(
         offers.asScala.partition { o =>
           val offerAttributes = toAttributeMap(o.getAttributesList)
           val meetsConstraints =
-            matchesAttributeRequirements(slaveOfferConstraints, offerAttributes)
+            matchesAttributeRequirements(agentOfferConstraints, offerAttributes)
 
           // add some debug messaging
           if (!meetsConstraints) {
@@ -259,7 +260,7 @@ private[spark] class MesosFineGrainedSchedulerBackend(
       val (usableOffers, unUsableOffers) = offersMatchingConstraints.partition { o =>
         val mem = getResource(o.getResourcesList, "mem")
         val cpus = getResource(o.getResourcesList, "cpus")
-        val slaveId = o.getSlaveId.getValue
+        val agentId = o.getSlaveId.getValue
         val offerAttributes = toAttributeMap(o.getAttributesList)
 
         // check offers for
@@ -269,7 +270,7 @@ private[spark] class MesosFineGrainedSchedulerBackend(
         val meetsCPURequirements = cpus >= (mesosExecutorCores + scheduler.CPUS_PER_TASK)
         val meetsRequirements =
           (meetsMemoryRequirements && meetsCPURequirements) ||
-          (slaveIdToExecutorInfo.contains(slaveId) && cpus >= scheduler.CPUS_PER_TASK)
+          (agentIdToExecutorInfo.contains(agentId) && cpus >= scheduler.CPUS_PER_TASK)
         val debugstr = if (meetsRequirements) "Accepting" else "Declining"
         logDebug(s"$debugstr offer: ${o.getId.getValue} with attributes: "
           + s"$offerAttributes mem: $mem cpu: $cpus")
@@ -281,10 +282,10 @@ private[spark] class MesosFineGrainedSchedulerBackend(
       unUsableOffers.foreach(o => d.declineOffer(o.getId))
 
       val workerOffers = usableOffers.map { o =>
-        val cpus = if (slaveIdToExecutorInfo.contains(o.getSlaveId.getValue)) {
+        val cpus = if (agentIdToExecutorInfo.contains(o.getSlaveId.getValue)) {
           getResource(o.getResourcesList, "cpus").toInt
         } else {
-          // If the Mesos executor has not been started on this slave yet, set aside a few
+          // If the Mesos executor has not been started on this agent yet, set aside a few
           // cores for the Mesos executor by offering fewer cores to the Spark executor
           (getResource(o.getResourcesList, "cpus") - mesosExecutorCores).toInt
         }
@@ -294,51 +295,51 @@ private[spark] class MesosFineGrainedSchedulerBackend(
           cpus)
       }.toIndexedSeq
 
-      val slaveIdToOffer = usableOffers.map(o => o.getSlaveId.getValue -> o).toMap
-      val slaveIdToWorkerOffer = workerOffers.map(o => o.executorId -> o).toMap
-      val slaveIdToResources = new HashMap[String, JList[Resource]]()
+      val agentIdToOffer = usableOffers.map(o => o.getSlaveId.getValue -> o).toMap
+      val agentIdToWorkerOffer = workerOffers.map(o => o.executorId -> o).toMap
+      val agentIdToResources = new HashMap[String, JList[Resource]]()
       usableOffers.foreach { o =>
-        slaveIdToResources(o.getSlaveId.getValue) = o.getResourcesList
+        agentIdToResources(o.getSlaveId.getValue) = o.getResourcesList
       }
 
       val mesosTasks = new HashMap[String, JArrayList[MesosTaskInfo]]
 
-      val slavesIdsOfAcceptedOffers = HashSet[String]()
+      val agentsIdsOfAcceptedOffers = HashSet[String]()
 
       // Call into the TaskSchedulerImpl
       val acceptedOffers = scheduler.resourceOffers(workerOffers).filter(!_.isEmpty)
       acceptedOffers
         .foreach { offer =>
           offer.foreach { taskDesc =>
-            val slaveId = taskDesc.executorId
-            slavesIdsOfAcceptedOffers += slaveId
-            taskIdToSlaveId(taskDesc.taskId) = slaveId
+            val agentId = taskDesc.executorId
+            agentsIdsOfAcceptedOffers += agentId
+            taskIdToAgentId(taskDesc.taskId) = agentId
             val (mesosTask, remainingResources) = createMesosTask(
               taskDesc,
-              slaveIdToResources(slaveId),
-              slaveId)
-            mesosTasks.getOrElseUpdate(slaveId, new JArrayList[MesosTaskInfo])
+              agentIdToResources(agentId),
+              agentId)
+            mesosTasks.getOrElseUpdate(agentId, new JArrayList[MesosTaskInfo])
               .add(mesosTask)
-            slaveIdToResources(slaveId) = remainingResources
+            agentIdToResources(agentId) = remainingResources
           }
         }
 
       // Reply to the offers
       val filters = Filters.newBuilder().setRefuseSeconds(1).build() // TODO: lower timeout?
 
-      mesosTasks.foreach { case (slaveId, tasks) =>
-        slaveIdToWorkerOffer.get(slaveId).foreach(o =>
-          listenerBus.post(SparkListenerExecutorAdded(System.currentTimeMillis(), slaveId,
+      mesosTasks.foreach { case (agentId, tasks) =>
+        agentIdToWorkerOffer.get(agentId).foreach(o =>
+          listenerBus.post(SparkListenerExecutorAdded(System.currentTimeMillis(), agentId,
             // TODO: Add support for log urls for Mesos
             new ExecutorInfo(o.host, o.cores, Map.empty, Map.empty)))
         )
-        logTrace(s"Launching Mesos tasks on slave '$slaveId', tasks:\n${getTasksSummary(tasks)}")
-        d.launchTasks(Collections.singleton(slaveIdToOffer(slaveId).getId), tasks, filters)
+        logTrace(s"Launching Mesos tasks on agent '$agentId', tasks:\n${getTasksSummary(tasks)}")
+        d.launchTasks(Collections.singleton(agentIdToOffer(agentId).getId), tasks, filters)
       }
 
       // Decline offers that weren't used
       // NOTE: This logic assumes that we only get a single offer for each host in a given batch
-      for (o <- usableOffers if !slavesIdsOfAcceptedOffers.contains(o.getSlaveId.getValue)) {
+      for (o <- usableOffers if !agentsIdsOfAcceptedOffers.contains(o.getSlaveId.getValue)) {
         d.declineOffer(o.getId)
       }
     }
@@ -348,19 +349,19 @@ private[spark] class MesosFineGrainedSchedulerBackend(
   def createMesosTask(
       task: TaskDescription,
       resources: JList[Resource],
-      slaveId: String): (MesosTaskInfo, JList[Resource]) = {
+      agentId: String): (MesosTaskInfo, JList[Resource]) = {
     val taskId = TaskID.newBuilder().setValue(task.taskId.toString).build()
-    val (executorInfo, remainingResources) = if (slaveIdToExecutorInfo.contains(slaveId)) {
-      (slaveIdToExecutorInfo(slaveId), resources)
+    val (executorInfo, remainingResources) = if (agentIdToExecutorInfo.contains(agentId)) {
+      (agentIdToExecutorInfo(agentId), resources)
     } else {
-      createExecutorInfo(resources, slaveId)
+      createExecutorInfo(resources, agentId)
     }
-    slaveIdToExecutorInfo(slaveId) = executorInfo
+    agentIdToExecutorInfo(agentId) = executorInfo
     val (finalResources, cpuResources) =
       partitionResources(remainingResources, "cpus", scheduler.CPUS_PER_TASK)
     val taskInfo = MesosTaskInfo.newBuilder()
       .setTaskId(taskId)
-      .setSlaveId(SlaveID.newBuilder().setValue(slaveId).build())
+      .setSlaveId(AgentID.newBuilder().setValue(agentId).build())
       .setExecutor(executorInfo)
       .setName(task.name)
       .addAllResources(cpuResources.asJava)
@@ -375,12 +376,12 @@ private[spark] class MesosFineGrainedSchedulerBackend(
       val state = mesosToTaskState(status.getState)
       synchronized {
         if (TaskState.isFailed(mesosToTaskState(status.getState))
-          && taskIdToSlaveId.contains(tid)) {
-          // We lost the executor on this slave, so remember that it's gone
-          removeExecutor(taskIdToSlaveId(tid), "Lost executor")
+          && taskIdToAgentId.contains(tid)) {
+          // We lost the executor on this agent, so remember that it's gone
+          removeExecutor(taskIdToAgentId(tid), "Lost executor")
         }
         if (TaskState.isFinished(state)) {
-          taskIdToSlaveId.remove(tid)
+          taskIdToAgentId.remove(tid)
         }
       }
       scheduler.statusUpdate(tid, state, status.getData.asReadOnlyByteBuffer)
@@ -406,39 +407,39 @@ private[spark] class MesosFineGrainedSchedulerBackend(
   }
 
   override def frameworkMessage(
-      d: org.apache.mesos.SchedulerDriver, e: ExecutorID, s: SlaveID, b: Array[Byte]): Unit = {}
+      d: org.apache.mesos.SchedulerDriver, e: ExecutorID, s: AgentID, b: Array[Byte]): Unit = {}
 
   /**
-   * Remove executor associated with slaveId in a thread safe manner.
+   * Remove executor associated with agentId in a thread safe manner.
    */
-  private def removeExecutor(slaveId: String, reason: String) = {
+  private def removeExecutor(agentId: String, reason: String) = {
     synchronized {
-      listenerBus.post(SparkListenerExecutorRemoved(System.currentTimeMillis(), slaveId, reason))
-      slaveIdToExecutorInfo -= slaveId
+      listenerBus.post(SparkListenerExecutorRemoved(System.currentTimeMillis(), agentId, reason))
+      agentIdToExecutorInfo -= agentId
     }
   }
 
-  private def recordSlaveLost(
-      d: org.apache.mesos.SchedulerDriver, slaveId: SlaveID, reason: ExecutorLossReason): Unit = {
+  private def recordAgentLost(
+      d: org.apache.mesos.SchedulerDriver, agentId: AgentID, reason: ExecutorLossReason): Unit = {
     inClassLoader() {
-      logInfo("Mesos slave lost: " + slaveId.getValue)
-      removeExecutor(slaveId.getValue, reason.toString)
-      scheduler.executorLost(slaveId.getValue, reason)
+      logInfo("Mesos agent lost: " + agentId.getValue)
+      removeExecutor(agentId.getValue, reason.toString)
+      scheduler.executorLost(agentId.getValue, reason)
     }
   }
 
-  override def slaveLost(d: org.apache.mesos.SchedulerDriver, slaveId: SlaveID): Unit = {
-    recordSlaveLost(d, slaveId, SlaveLost())
+  override def agentLost(d: org.apache.mesos.SchedulerDriver, agentId: AgentID): Unit = {
+    recordAgentLost(d, agentId, ExecutorProcessLost())
   }
 
   override def executorLost(
       d: org.apache.mesos.SchedulerDriver,
       executorId: ExecutorID,
-      slaveId: SlaveID,
+      agentId: AgentID,
       status: Int): Unit = {
-    logInfo("Executor lost: %s, marking slave %s as lost".format(executorId.getValue,
-                                                                 slaveId.getValue))
-    recordSlaveLost(d, slaveId, ExecutorExited(status, exitCausedByApp = true))
+    logInfo("Executor lost: %s, marking agent %s as lost".format(executorId.getValue,
+                                                                 agentId.getValue))
+    recordAgentLost(d, agentId, ExecutorExited(status, exitCausedByApp = true))
   }
 
   override def killTask(
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosScheduler.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosScheduler.scala
new file mode 100644
index 0000000000000..f55b9efb3e64b
--- /dev/null
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosScheduler.scala
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.scheduler.cluster.mesos
+
+import org.apache.mesos.Protos.{SlaveID => AgentID}
+
+trait MesosScheduler extends org.apache.mesos.Scheduler {
+  override def slaveLost(d: org.apache.mesos.SchedulerDriver, agentId: AgentID): Unit = {
+    agentLost(d, agentId)
+  }
+
+  def agentLost(d: org.apache.mesos.SchedulerDriver, agentId: AgentID): Unit
+}
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala
index 7b2f6a2535eda..e9fbff730f07e 100644
--- a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala
@@ -17,22 +17,23 @@
 
 package org.apache.spark.scheduler.cluster.mesos
 
-import org.apache.mesos.Protos.{ContainerInfo, Environment, Image, NetworkInfo, Parameter, Secret, Volume}
+import org.apache.mesos.Protos.{ContainerInfo, Image, NetworkInfo, Parameter, Secret,
+  TaskState => MesosTaskState, Volume}
 import org.apache.mesos.Protos.ContainerInfo.{DockerInfo, MesosInfo}
 import org.apache.mesos.Protos.Environment.Variable
 import org.apache.mesos.protobuf.ByteString
 
-import org.apache.spark.SparkConf
+import org.apache.spark.{SparkConf, TaskState}
 import org.apache.spark.SparkException
 import org.apache.spark.deploy.mesos.config._
 import org.apache.spark.deploy.mesos.config.MesosSecretConfig
 import org.apache.spark.internal.Logging
 
 /**
- * A collection of utility functions which can be used by both the
- * MesosSchedulerBackend and the [[MesosFineGrainedSchedulerBackend]].
+ * A collection of utility functions which can be used by the
+ * MesosSchedulerBackend, [[MesosFineGrainedSchedulerBackend]] and the MesosExecutorBackend.
  */
-private[mesos] object MesosSchedulerBackendUtil extends Logging {
+private[spark] object MesosSchedulerBackendUtil extends Logging {
   /**
    * Parse a list of volume specs, each of which
    * takes the form [host-dir:]container-dir[:rw|:ro].
@@ -137,7 +138,7 @@ private[mesos] object MesosSchedulerBackendUtil extends Logging {
     val containerInfo = ContainerInfo.newBuilder()
       .setType(containerType)
 
-    conf.get(EXECUTOR_DOCKER_IMAGE).map { image =>
+    conf.get(EXECUTOR_DOCKER_IMAGE).foreach { image =>
       val forcePullImage = conf
         .get(EXECUTOR_DOCKER_FORCE_PULL_IMAGE).contains(true)
 
@@ -294,4 +295,13 @@ private[mesos] object MesosSchedulerBackendUtil extends Logging {
       .setImage(imageProto)
       .build
   }
+
+  def taskStateToMesos(state: TaskState.TaskState): MesosTaskState = state match {
+    case TaskState.LAUNCHING => MesosTaskState.TASK_STARTING
+    case TaskState.RUNNING => MesosTaskState.TASK_RUNNING
+    case TaskState.FINISHED => MesosTaskState.TASK_FINISHED
+    case TaskState.FAILED => MesosTaskState.TASK_FAILED
+    case TaskState.KILLED => MesosTaskState.TASK_KILLED
+    case TaskState.LOST => MesosTaskState.TASK_LOST
+  }
 }
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala
index ed3bd358d4082..38f83df00e428 100644
--- a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala
@@ -32,11 +32,11 @@ import org.apache.mesos.{MesosSchedulerDriver, Protos, Scheduler, SchedulerDrive
 import org.apache.mesos.Protos.{TaskState => MesosTaskState, _}
 import org.apache.mesos.Protos.FrameworkInfo.Capability
 import org.apache.mesos.Protos.Resource.ReservationInfo
-import org.apache.mesos.protobuf.{ByteString, GeneratedMessageV3}
+import org.apache.mesos.protobuf.GeneratedMessageV3
 
 import org.apache.spark.{SparkConf, SparkContext, SparkException}
 import org.apache.spark.TaskState
-import org.apache.spark.deploy.mesos.{config => mesosConfig}
+import org.apache.spark.deploy.mesos.{config => mesosConfig, MesosDriverDescription}
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.{Status => _, _}
 import org.apache.spark.util.Utils
@@ -304,16 +304,15 @@ trait MesosSchedulerUtils extends Logging {
    * Match the requirements (if any) to the offer attributes.
    * if attribute requirements are not specified - return true
    * else if attribute is defined and no values are given, simple attribute presence is performed
-   * else if attribute name and value is specified, subset match is performed on slave attributes
+   * else if attribute name and value is specified, subset match is performed on agent attributes
    */
   def matchesAttributeRequirements(
-      slaveOfferConstraints: Map[String, Set[String]],
+      agentOfferConstraints: Map[String, Set[String]],
       offerAttributes: Map[String, GeneratedMessageV3]): Boolean = {
-    slaveOfferConstraints.forall {
+    agentOfferConstraints.forall {
       // offer has the required attribute and subsumes the required values for that attribute
       case (name, requiredValues) =>
         offerAttributes.get(name) match {
-          case None => false
           case Some(_) if requiredValues.isEmpty => true // empty value matches presence
           case Some(scalarValue: Value.Scalar) =>
             // check if provided values is less than equal to the offered values
@@ -332,6 +331,7 @@ trait MesosSchedulerUtils extends Logging {
             // check if the specified value is equal, if multiple values are specified
             // we succeed if any of them match.
             requiredValues.contains(textValue.getValue)
+          case _ => false
         }
     }
   }
@@ -356,7 +356,7 @@ trait MesosSchedulerUtils extends Logging {
    *                       https://github.com/apache/mesos/blob/master/src/common/values.cpp
    *                       https://github.com/apache/mesos/blob/master/src/common/attributes.cpp
    *
-   * @param constraintsVal constains string consisting of ';' separated key-value pairs (separated
+   * @param constraintsVal contains string consisting of ';' separated key-value pairs (separated
    *                       by ':')
    * @return  Map of constraints to match resources offers.
    */
@@ -379,7 +379,7 @@ trait MesosSchedulerUtils extends Logging {
           } else {
             v.split(',').toSet
           }
-        )
+        ).toMap
       } catch {
         case NonFatal(e) =>
           throw new IllegalArgumentException(s"Bad constraint string: $constraintsVal", e)
@@ -405,6 +405,21 @@ trait MesosSchedulerUtils extends Logging {
       sc.executorMemory
   }
 
+  /**
+   * Return the amount of memory to allocate to each driver, taking into account
+   * container overheads.
+   *
+   * @param driverDesc used to get driver memory
+   * @return memory requirement defined as `DRIVER_MEMORY_OVERHEAD` if set in the config,
+   *         otherwise the larger of `MEMORY_OVERHEAD_MINIMUM (=384MB)` or
+   *         `MEMORY_OVERHEAD_FRACTION (=0.1) * driverMemory`
+   */
+  def driverContainerMemory(driverDesc: MesosDriverDescription): Int = {
+    val defaultMem = math.max(MEMORY_OVERHEAD_FRACTION * driverDesc.mem, MEMORY_OVERHEAD_MINIMUM)
+    driverDesc.conf.get(mesosConfig.DRIVER_MEMORY_OVERHEAD).getOrElse(defaultMem.toInt) +
+      driverDesc.mem
+  }
+
   def setupUris(uris: Seq[String],
                 builder: CommandInfo.Builder,
                 useFetcherCache: Boolean = false): Unit = {
@@ -574,15 +589,6 @@ trait MesosSchedulerUtils extends Logging {
          MesosTaskState.TASK_UNREACHABLE => TaskState.LOST
   }
 
-  def taskStateToMesos(state: TaskState.TaskState): MesosTaskState = state match {
-    case TaskState.LAUNCHING => MesosTaskState.TASK_STARTING
-    case TaskState.RUNNING => MesosTaskState.TASK_RUNNING
-    case TaskState.FINISHED => MesosTaskState.TASK_FINISHED
-    case TaskState.FAILED => MesosTaskState.TASK_FAILED
-    case TaskState.KILLED => MesosTaskState.TASK_KILLED
-    case TaskState.LOST => MesosTaskState.TASK_LOST
-  }
-
   protected def declineOffer(
     driver: org.apache.mesos.SchedulerDriver,
     offer: Offer,
@@ -612,4 +618,3 @@ trait MesosSchedulerUtils extends Logging {
     }
   }
 }
-
diff --git a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterSchedulerSuite.scala b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterSchedulerSuite.scala
index 9a50142b51d97..9a1862d32dc13 100644
--- a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterSchedulerSuite.scala
+++ b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterSchedulerSuite.scala
@@ -105,7 +105,8 @@ class MesosClusterSchedulerSuite extends SparkFunSuite with LocalSparkContext wi
     val response = scheduler.submitDriver(
       new MesosDriverDescription("d1", "jar", 1200, 1.5, true,
         command,
-        Map((config.EXECUTOR_HOME.key, "test"), ("spark.app.name", "test")),
+        Map((config.EXECUTOR_HOME.key, "test"), ("spark.app.name", "test"),
+          (config.DRIVER_MEMORY_OVERHEAD.key, "0")),
         "s1",
         new Date()))
     assert(response.success)
@@ -146,7 +147,7 @@ class MesosClusterSchedulerSuite extends SparkFunSuite with LocalSparkContext wi
     val resources = taskInfo.getResourcesList
     assert(scheduler.getResource(resources, "cpus") == 1.5)
     assert(scheduler.getResource(resources, "mem") == 1200)
-    val resourcesSeq: Seq[Resource] = resources.asScala
+    val resourcesSeq: Seq[Resource] = resources.asScala.toSeq
     val cpus = resourcesSeq.filter(_.getName == "cpus").toList
     assert(cpus.size == 2)
     assert(cpus.exists(_.getRole() == "role2"))
@@ -200,6 +201,60 @@ class MesosClusterSchedulerSuite extends SparkFunSuite with LocalSparkContext wi
     })
   }
 
+  test("SPARK-22256: supports spark.mesos.driver.memoryOverhead with 384mb default") {
+    setScheduler()
+
+    val mem = 1000
+    val cpu = 1
+
+    val response = scheduler.submitDriver(
+      new MesosDriverDescription("d1", "jar", mem, cpu, true,
+        command,
+        Map("spark.mesos.executor.home" -> "test",
+          "spark.app.name" -> "test"),
+        "s1",
+        new Date()))
+    assert(response.success)
+
+    val offer = Utils.createOffer("o1", "s1", mem*2, cpu)
+    scheduler.resourceOffers(driver, List(offer).asJava)
+    val tasks = Utils.verifyTaskLaunched(driver, "o1")
+    // 1384.0
+    val taskMem = tasks.head.getResourcesList
+      .asScala
+      .filter(_.getName.equals("mem"))
+      .map(_.getScalar.getValue)
+      .head
+    assert(1384.0 === taskMem)
+  }
+
+  test("SPARK-22256: supports spark.mesos.driver.memoryOverhead with 10% default") {
+    setScheduler()
+
+    val mem = 10000
+    val cpu = 1
+
+    val response = scheduler.submitDriver(
+      new MesosDriverDescription("d1", "jar", mem, cpu, true,
+        command,
+        Map("spark.mesos.executor.home" -> "test",
+          "spark.app.name" -> "test"),
+        "s1",
+        new Date()))
+    assert(response.success)
+
+    val offer = Utils.createOffer("o1", "s1", mem*2, cpu)
+    scheduler.resourceOffers(driver, List(offer).asJava)
+    val tasks = Utils.verifyTaskLaunched(driver, "o1")
+    // 11000.0
+    val taskMem = tasks.head.getResourcesList
+      .asScala
+      .filter(_.getName.equals("mem"))
+      .map(_.getScalar.getValue)
+      .head
+    assert(11000.0 === taskMem)
+  }
+
   test("supports spark.mesos.driverEnv.*") {
     setScheduler()
 
@@ -211,7 +266,9 @@ class MesosClusterSchedulerSuite extends SparkFunSuite with LocalSparkContext wi
         command,
         Map(config.EXECUTOR_HOME.key -> "test",
           "spark.app.name" -> "test",
-          config.DRIVER_ENV_PREFIX + "TEST_ENV" -> "TEST_VAL"),
+          config.DRIVER_ENV_PREFIX + "TEST_ENV" -> "TEST_VAL",
+          config.DRIVER_MEMORY_OVERHEAD.key -> "0"
+        ),
         "s1",
         new Date()))
     assert(response.success)
@@ -236,7 +293,8 @@ class MesosClusterSchedulerSuite extends SparkFunSuite with LocalSparkContext wi
         Map(config.EXECUTOR_HOME.key -> "test",
           "spark.app.name" -> "test",
           config.NETWORK_NAME.key -> "test-network-name",
-          config.NETWORK_LABELS.key -> "key1:val1,key2:val2"),
+          config.NETWORK_LABELS.key -> "key1:val1,key2:val2",
+          config.DRIVER_MEMORY_OVERHEAD.key -> "0"),
         "s1",
         new Date()))
 
@@ -266,7 +324,8 @@ class MesosClusterSchedulerSuite extends SparkFunSuite with LocalSparkContext wi
         command,
         Map(config.EXECUTOR_HOME.key -> "test",
           config.ENABLE_FETCHER_CACHE.key -> "true",
-          "spark.app.name" -> "test"),
+          "spark.app.name" -> "test",
+          config.DRIVER_MEMORY_OVERHEAD.key -> "0"),
         "s1",
         new Date()))
 
@@ -290,7 +349,8 @@ class MesosClusterSchedulerSuite extends SparkFunSuite with LocalSparkContext wi
       new MesosDriverDescription("d1", "jar", mem, cpu, true,
         command,
         Map(config.EXECUTOR_HOME.key -> "test",
-          "spark.app.name" -> "test"),
+          "spark.app.name" -> "test",
+          config.DRIVER_MEMORY_OVERHEAD.key -> "0"),
         "s1",
         new Date()))
 
@@ -315,7 +375,8 @@ class MesosClusterSchedulerSuite extends SparkFunSuite with LocalSparkContext wi
         command,
         Map(config.EXECUTOR_HOME.key -> "test",
           config.ENABLE_FETCHER_CACHE.key -> "false",
-          "spark.app.name" -> "test"),
+          "spark.app.name" -> "test",
+          config.DRIVER_MEMORY_OVERHEAD.key -> "0"),
         "s1",
         new Date()))
 
@@ -349,7 +410,8 @@ class MesosClusterSchedulerSuite extends SparkFunSuite with LocalSparkContext wi
           command,
           Map(config.EXECUTOR_HOME.key -> "test",
             "spark.app.name" -> "test",
-            config.DRIVER_CONSTRAINTS.key -> driverConstraints),
+            config.DRIVER_CONSTRAINTS.key -> driverConstraints,
+            config.DRIVER_MEMORY_OVERHEAD.key -> "0"),
           "s1",
           new Date()))
       assert(response.success)
@@ -387,7 +449,8 @@ class MesosClusterSchedulerSuite extends SparkFunSuite with LocalSparkContext wi
         command,
         Map(config.EXECUTOR_HOME.key -> "test",
           "spark.app.name" -> "test",
-          config.DRIVER_LABELS.key -> "key:value"),
+          config.DRIVER_LABELS.key -> "key:value",
+          config.DRIVER_MEMORY_OVERHEAD.key -> "0"),
         "s1",
         new Date()))
 
@@ -413,7 +476,7 @@ class MesosClusterSchedulerSuite extends SparkFunSuite with LocalSparkContext wi
       new MesosDriverDescription("d1", "jar", 100, 1, true, command,
         Map((config.EXECUTOR_HOME.key, "test"), ("spark.app.name", "test")), "s1", new Date()))
     assert(response.success)
-    val slaveId = SlaveID.newBuilder().setValue("s1").build()
+    val agentId = SlaveID.newBuilder().setValue("s1").build()
     val offer = Offer.newBuilder()
       .addResources(
         Resource.newBuilder().setRole("*")
@@ -425,7 +488,7 @@ class MesosClusterSchedulerSuite extends SparkFunSuite with LocalSparkContext wi
           .setType(Type.SCALAR))
       .setId(OfferID.newBuilder().setValue("o1").build())
       .setFrameworkId(FrameworkID.newBuilder().setValue("f1").build())
-      .setSlaveId(slaveId)
+      .setSlaveId(agentId)
       .setHostname("host1")
       .build()
     // Offer the resource to launch the submitted driver
@@ -438,7 +501,7 @@ class MesosClusterSchedulerSuite extends SparkFunSuite with LocalSparkContext wi
 
     val taskStatus = TaskStatus.newBuilder()
       .setTaskId(TaskID.newBuilder().setValue(response.submissionId).build())
-      .setSlaveId(slaveId)
+      .setSlaveId(agentId)
       .setState(MesosTaskState.TASK_KILLED)
       .build()
     // Update the status of the killed task
@@ -592,10 +655,10 @@ class MesosClusterSchedulerSuite extends SparkFunSuite with LocalSparkContext wi
 
     val expectedCmd = "cd spark-version*;  " +
         "bin/spark-submit --name \"app name\" --master mesos://mesos://localhost:5050 " +
-        "--driver-cores 1.0 --driver-memory 1000M --class Main --py-files  " +
-        "--conf spark.executor.uri=s3a://bucket/spark-version.tgz " +
+        "--driver-cores 1.0 --driver-memory 1000M --class Main " +
         "--conf \"another.conf=\\\\value\" " +
         "--conf \"spark.app.name=app name\" " +
+        "--conf spark.executor.uri=s3a://bucket/spark-version.tgz " +
         "../jar " +
         "\"--a=\\$2\" " +
         "--b \"x y z\""
@@ -603,6 +666,136 @@ class MesosClusterSchedulerSuite extends SparkFunSuite with LocalSparkContext wi
     assert(scheduler.getDriverCommandValue(driverDesc) == expectedCmd)
   }
 
+  test("SPARK-23499: Test dispatcher priority queue with non float value") {
+    val conf = new SparkConf()
+    conf.set("spark.mesos.dispatcher.queue.ROUTINE", "1.0")
+    conf.set("spark.mesos.dispatcher.queue.URGENT", "abc")
+    conf.set("spark.mesos.dispatcher.queue.EXCEPTIONAL", "3.0")
+    assertThrows[NumberFormatException] {
+      setScheduler(conf.getAll.toMap)
+    }
+  }
+
+  test("SPARK-23499: Get driver priority") {
+    val conf = new SparkConf()
+    conf.set("spark.mesos.dispatcher.queue.ROUTINE", "1.0")
+    conf.set("spark.mesos.dispatcher.queue.URGENT", "2.0")
+    conf.set("spark.mesos.dispatcher.queue.EXCEPTIONAL", "3.0")
+    setScheduler(conf.getAll.toMap)
+
+    val mem = 1000
+    val cpu = 1
+
+    // Test queue not declared in scheduler
+    var desc = new MesosDriverDescription("d1", "jar", mem, cpu, true,
+      command,
+      Map("spark.mesos.dispatcher.queue" -> "dummy"),
+      "s1",
+      new Date())
+
+    assertThrows[NoSuchElementException] {
+      scheduler.getDriverPriority(desc)
+    }
+
+    // Test with no specified queue
+    desc = new MesosDriverDescription("d1", "jar", mem, cpu, true,
+      command,
+      Map[String, String](),
+      "s2",
+      new Date())
+
+    assert(scheduler.getDriverPriority(desc) == 0.0f)
+
+    // Test with "default" queue specified
+    desc = new MesosDriverDescription("d1", "jar", mem, cpu, true,
+      command,
+      Map("spark.mesos.dispatcher.queue" -> "default"),
+      "s3",
+      new Date())
+
+    assert(scheduler.getDriverPriority(desc) == 0.0f)
+
+    // Test queue declared in scheduler
+    desc = new MesosDriverDescription("d1", "jar", mem, cpu, true,
+      command,
+      Map("spark.mesos.dispatcher.queue" -> "ROUTINE"),
+      "s4",
+      new Date())
+
+    assert(scheduler.getDriverPriority(desc) == 1.0f)
+
+    // Test other queue declared in scheduler
+    desc = new MesosDriverDescription("d1", "jar", mem, cpu, true,
+      command,
+      Map("spark.mesos.dispatcher.queue" -> "URGENT"),
+      "s5",
+      new Date())
+
+    assert(scheduler.getDriverPriority(desc) == 2.0f)
+  }
+
+  test("SPARK-23499: Can queue drivers with priority") {
+    val conf = new SparkConf()
+    conf.set("spark.mesos.dispatcher.queue.ROUTINE", "1.0")
+    conf.set("spark.mesos.dispatcher.queue.URGENT", "2.0")
+    conf.set("spark.mesos.dispatcher.queue.EXCEPTIONAL", "3.0")
+    setScheduler(conf.getAll.toMap)
+
+    val mem = 1000
+    val cpu = 1
+
+    val response0 = scheduler.submitDriver(
+      new MesosDriverDescription("d1", "jar", 100, 1, true, command,
+        Map("spark.mesos.dispatcher.queue" -> "ROUTINE"), "s0", new Date()))
+    assert(response0.success)
+
+    val response1 = scheduler.submitDriver(
+      new MesosDriverDescription("d1", "jar", 100, 1, true, command,
+        Map[String, String](), "s1", new Date()))
+    assert(response1.success)
+
+    val response2 = scheduler.submitDriver(
+      new MesosDriverDescription("d1", "jar", 100, 1, true, command,
+        Map("spark.mesos.dispatcher.queue" -> "EXCEPTIONAL"), "s2", new Date()))
+    assert(response2.success)
+
+    val response3 = scheduler.submitDriver(
+      new MesosDriverDescription("d1", "jar", 100, 1, true, command,
+        Map("spark.mesos.dispatcher.queue" -> "URGENT"), "s3", new Date()))
+    assert(response3.success)
+
+    val state = scheduler.getSchedulerState()
+    val queuedDrivers = state.queuedDrivers.toList
+    assert(queuedDrivers(0).submissionId == response2.submissionId)
+    assert(queuedDrivers(1).submissionId == response3.submissionId)
+    assert(queuedDrivers(2).submissionId == response0.submissionId)
+    assert(queuedDrivers(3).submissionId == response1.submissionId)
+  }
+
+  test("SPARK-23499: Can queue drivers with negative priority") {
+    val conf = new SparkConf()
+    conf.set("spark.mesos.dispatcher.queue.LOWER", "-1.0")
+    setScheduler(conf.getAll.toMap)
+
+    val mem = 1000
+    val cpu = 1
+
+    val response0 = scheduler.submitDriver(
+      new MesosDriverDescription("d1", "jar", 100, 1, true, command,
+        Map("spark.mesos.dispatcher.queue" -> "LOWER"), "s0", new Date()))
+    assert(response0.success)
+
+    val response1 = scheduler.submitDriver(
+      new MesosDriverDescription("d1", "jar", 100, 1, true, command,
+        Map[String, String](), "s1", new Date()))
+    assert(response1.success)
+
+    val state = scheduler.getSchedulerState()
+    val queuedDrivers = state.queuedDrivers.toList
+    assert(queuedDrivers(0).submissionId == response1.submissionId)
+    assert(queuedDrivers(1).submissionId == response0.submissionId)
+  }
+
   private def launchDriverTask(addlSparkConfVars: Map[String, String]): List[TaskInfo] = {
     setScheduler()
     val mem = 1000
@@ -615,7 +808,8 @@ class MesosClusterSchedulerSuite extends SparkFunSuite with LocalSparkContext wi
       true,
       command,
       Map(config.EXECUTOR_HOME.key -> "test",
-        "spark.app.name" -> "test") ++
+        "spark.app.name" -> "test",
+        config.DRIVER_MEMORY_OVERHEAD.key -> "0") ++
         addlSparkConfVars,
       "s1",
       new Date())
diff --git a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala
index 5ab277ed87a72..2b7272a490376 100644
--- a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala
+++ b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala
@@ -105,7 +105,7 @@ class MesosCoarseGrainedSchedulerBackendSuite extends SparkFunSuite
     backend.statusUpdate(driver, status)
     verify(driver, times(1)).reviveOffers()
 
-    // Launches a new task on a valid offer from the same slave
+    // Launches a new task on a valid offer from the same agent
     offerResources(List(offer2))
     verifyTaskLaunched(driver, "o2")
   }
@@ -250,7 +250,7 @@ class MesosCoarseGrainedSchedulerBackendSuite extends SparkFunSuite
     verifyTaskLaunched(driver, "o2")
   }
 
-  test("mesos creates multiple executors on a single slave") {
+  test("mesos creates multiple executors on a single agent") {
     val executorCores = 4
     setBackend(Map(EXECUTOR_CORES.key -> executorCores.toString))
 
@@ -727,10 +727,10 @@ class MesosCoarseGrainedSchedulerBackendSuite extends SparkFunSuite
 
   private case class Resources(mem: Int, cpus: Int, gpus: Int = 0)
 
-  private def registerMockExecutor(executorId: String, slaveId: String, cores: Integer) = {
+  private def registerMockExecutor(executorId: String, agentId: String, cores: Integer) = {
     val mockEndpointRef = mock[RpcEndpointRef]
     val mockAddress = mock[RpcAddress]
-    val message = RegisterExecutor(executorId, mockEndpointRef, slaveId, cores, Map.empty,
+    val message = RegisterExecutor(executorId, mockEndpointRef, agentId, cores, Map.empty,
       Map.empty, Map.empty, ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID)
 
     backend.driverEndpoint.askSync[Boolean](message)
@@ -766,10 +766,10 @@ class MesosCoarseGrainedSchedulerBackendSuite extends SparkFunSuite
     }
   }
 
-  private def createTaskStatus(taskId: String, slaveId: String, state: TaskState): TaskStatus = {
+  private def createTaskStatus(taskId: String, agentId: String, state: TaskState): TaskStatus = {
     TaskStatus.newBuilder()
       .setTaskId(TaskID.newBuilder().setValue(taskId).build())
-      .setSlaveId(SlaveID.newBuilder().setValue(slaveId).build())
+      .setSlaveId(SlaveID.newBuilder().setValue(agentId).build())
       .setState(state)
       .build
   }
@@ -833,7 +833,7 @@ class MesosCoarseGrainedSchedulerBackendSuite extends SparkFunSuite
     when(driver.start()).thenReturn(Protos.Status.DRIVER_RUNNING)
 
     taskScheduler = mock[TaskSchedulerImpl]
-    when(taskScheduler.nodeBlacklist).thenReturn(Set[String]())
+    when(taskScheduler.excludedNodes).thenReturn(Set[String]())
     when(taskScheduler.sc).thenReturn(sc)
 
     externalShuffleClient = mock[MesosExternalBlockStoreClient]
diff --git a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosFineGrainedSchedulerBackendSuite.scala b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosFineGrainedSchedulerBackendSuite.scala
index 36ed84858dbfb..10030a20f0884 100644
--- a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosFineGrainedSchedulerBackendSuite.scala
+++ b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosFineGrainedSchedulerBackendSuite.scala
@@ -178,7 +178,7 @@ class MesosFineGrainedSchedulerBackendSuite
     val (execInfo, _) = backend.createExecutorInfo(
       Arrays.asList(backend.createResource("cpus", 4)), "mockExecutor")
     assert(execInfo.getContainer.getDocker.getImage.equals("spark/mock"))
-    assert(execInfo.getContainer.getDocker.getForcePullImage.equals(true))
+    assert(execInfo.getContainer.getDocker.getForcePullImage)
     val portmaps = execInfo.getContainer.getDocker.getPortMappingsList
     assert(portmaps.get(0).getHostPort.equals(80))
     assert(portmaps.get(0).getContainerPort.equals(8080))
@@ -264,10 +264,12 @@ class MesosFineGrainedSchedulerBackendSuite
       partitionId = 0,
       addedFiles = mutable.Map.empty[String, Long],
       addedJars = mutable.Map.empty[String, Long],
+      addedArchives = mutable.Map.empty[String, Long],
       properties = new Properties(),
       resources = immutable.Map.empty[String, ResourceInformation],
       ByteBuffer.wrap(new Array[Byte](0)))
-    when(taskScheduler.resourceOffers(expectedWorkerOffers)).thenReturn(Seq(Seq(taskDesc)))
+    when(taskScheduler.resourceOffers(
+      expectedWorkerOffers.toIndexedSeq)).thenReturn(Seq(Seq(taskDesc)))
     when(taskScheduler.CPUS_PER_TASK).thenReturn(2)
 
     val capture = ArgumentCaptor.forClass(classOf[Collection[TaskInfo]])
@@ -376,10 +378,12 @@ class MesosFineGrainedSchedulerBackendSuite
       partitionId = 0,
       addedFiles = mutable.Map.empty[String, Long],
       addedJars = mutable.Map.empty[String, Long],
+      addedArchives = mutable.Map.empty[String, Long],
       properties = new Properties(),
       resources = immutable.Map.empty[String, ResourceInformation],
       ByteBuffer.wrap(new Array[Byte](0)))
-    when(taskScheduler.resourceOffers(expectedWorkerOffers)).thenReturn(Seq(Seq(taskDesc)))
+    when(taskScheduler.resourceOffers(
+      expectedWorkerOffers.toIndexedSeq)).thenReturn(Seq(Seq(taskDesc)))
     when(taskScheduler.CPUS_PER_TASK).thenReturn(1)
 
     val capture = ArgumentCaptor.forClass(classOf[Collection[TaskInfo]])
diff --git a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtilsSuite.scala b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtilsSuite.scala
index 0ed6fe66c56eb..f2efe86c3f9ec 100644
--- a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtilsSuite.scala
+++ b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtilsSuite.scala
@@ -25,7 +25,8 @@ import scala.collection.JavaConverters._
 import com.google.common.io.Files
 import org.apache.mesos.Protos.{FrameworkInfo, Resource, Value}
 import org.mockito.Mockito._
-import org.scalatest._
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 import org.scalatestplus.mockito.MockitoSugar
 
 import org.apache.spark.{SparkConf, SparkContext, SparkException, SparkFunSuite}
diff --git a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/Utils.scala b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/Utils.scala
index 5a4bf1dd2d409..92676cc4e7395 100644
--- a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/Utils.scala
+++ b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/Utils.scala
@@ -46,7 +46,7 @@ object Utils {
 
   def createOffer(
                    offerId: String,
-                   slaveId: String,
+                   agentId: String,
                    mem: Int,
                    cpus: Int,
                    ports: Option[(Long, Long)] = None,
@@ -77,8 +77,8 @@ object Utils {
     builder.setId(createOfferId(offerId))
       .setFrameworkId(FrameworkID.newBuilder()
       .setValue("f1"))
-      .setSlaveId(SlaveID.newBuilder().setValue(slaveId))
-      .setHostname(s"host${slaveId}")
+      .setSlaveId(SlaveID.newBuilder().setValue(agentId))
+      .setHostname(s"host${agentId}")
       .addAllAttributes(attributes.asJava)
       .build()
   }
@@ -101,8 +101,8 @@ object Utils {
     OfferID.newBuilder().setValue(offerId).build()
   }
 
-  def createSlaveId(slaveId: String): SlaveID = {
-    SlaveID.newBuilder().setValue(slaveId).build()
+  def createAgentId(agentId: String): SlaveID = {
+    SlaveID.newBuilder().setValue(agentId).build()
   }
 
   def createExecutorId(executorId: String): ExecutorID = {
@@ -227,4 +227,3 @@ object Utils {
       .build()
   }
 }
-
diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
index b84180abfa200..a6629539f59b5 100644
--- a/resource-managers/yarn/pom.xml
+++ b/resource-managers/yarn/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
@@ -30,8 +30,54 @@
   <properties>
     <sbt.project.name>yarn</sbt.project.name>
     <jersey-1.version>1.19</jersey-1.version>
+    <spark.yarn.isHadoopProvided>false</spark.yarn.isHadoopProvided>
   </properties>
 
+  <profiles>
+    <profile>
+      <id>hadoop-provided</id>
+      <properties>
+        <spark.yarn.isHadoopProvided>true</spark.yarn.isHadoopProvided>
+      </properties>
+    </profile>
+    <profile>
+      <id>hadoop-2.7</id>
+      <dependencies>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-yarn-api</artifactId>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-yarn-common</artifactId>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-yarn-client</artifactId>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-yarn-server-tests</artifactId>
+          <classifier>tests</classifier>
+          <scope>test</scope>
+        </dependency>
+        <!--
+          Hack to exclude org.apache.hadoop:hadoop-yarn-server-resourcemanager:jar:tests.
+          See the parent pom.xml for more details.
+        -->
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+          <scope>test</scope>
+        </dependency>
+      </dependencies>
+    </profile>
+  </profiles>
+
   <dependencies>
     <dependency>
       <groupId>org.apache.spark</groupId>
@@ -59,23 +105,20 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-yarn-api</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-yarn-common</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+      <artifactId>${hadoop-client-api.artifact}</artifactId>
+      <version>${hadoop.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-yarn-client</artifactId>
+      <artifactId>${hadoop-client-runtime.artifact}</artifactId>
+      <version>${hadoop.version}</version>
+      <scope>${hadoop.deps.scope}</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-client</artifactId>
+      <artifactId>${hadoop-client-minicluster.artifact}</artifactId>
+      <version>${hadoop.version}</version>
+      <scope>test</scope>
     </dependency>
 
     <!-- Explicit listing of transitive deps that are shaded. Otherwise, odd compiler crashes. -->
@@ -125,13 +168,6 @@
       <scope>test</scope>
     </dependency>
 
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-yarn-server-tests</artifactId>
-      <classifier>tests</classifier>
-      <scope>test</scope>
-    </dependency>
-
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>
@@ -201,6 +237,12 @@
   <build>
     <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
     <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
+    <resources>
+      <resource>
+        <directory>src/main/resources</directory>
+        <filtering>true</filtering>
+      </resource>
+    </resources>
   </build>
 
 </project>
diff --git a/resource-managers/yarn/src/main/resources/org/apache/spark/deploy/yarn/config.properties b/resource-managers/yarn/src/main/resources/org/apache/spark/deploy/yarn/config.properties
new file mode 100644
index 0000000000000..b923dbab954f5
--- /dev/null
+++ b/resource-managers/yarn/src/main/resources/org/apache/spark/deploy/yarn/config.properties
@@ -0,0 +1 @@
+spark.yarn.isHadoopProvided = ${spark.yarn.isHadoopProvided}
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 9a6a43914bca3..ebe1286235542 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -19,7 +19,7 @@ package org.apache.spark.deploy.yarn
 
 import java.io.{File, IOException}
 import java.lang.reflect.{InvocationTargetException, Modifier}
-import java.net.{URI, URL}
+import java.net.{URI, URL, URLEncoder}
 import java.security.PrivilegedExceptionAction
 import java.util.concurrent.{TimeoutException, TimeUnit}
 
@@ -36,7 +36,6 @@ import org.apache.hadoop.yarn.api._
 import org.apache.hadoop.yarn.api.records._
 import org.apache.hadoop.yarn.conf.YarnConfiguration
 import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException
-import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils
 import org.apache.hadoop.yarn.util.{ConverterUtils, Records}
 
 import org.apache.spark._
@@ -51,6 +50,7 @@ import org.apache.spark.internal.config.UI._
 import org.apache.spark.metrics.{MetricsSystem, MetricsSystemInstances}
 import org.apache.spark.resource.ResourceProfile
 import org.apache.spark.rpc._
+import org.apache.spark.scheduler.MiscellaneousProcessDetails
 import org.apache.spark.scheduler.cluster.{CoarseGrainedSchedulerBackend, YarnSchedulerBackend}
 import org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages._
 import org.apache.spark.util._
@@ -66,6 +66,11 @@ private[spark] class ApplicationMaster(
   // TODO: Currently, task to container is computed once (TaskSetManager) - which need not be
   // optimal as more containers are available. Might need to handle this better.
 
+  private def extractLogUrls: Map[String, String] = {
+    YarnContainerInfoHelper.getLogUrls(SparkHadoopUtil.
+      newConfiguration(sparkConf), None).getOrElse(Map())
+  }
+
   private val appAttemptId =
     if (System.getenv(ApplicationConstants.Environment.CONTAINER_ID.name()) != null) {
       YarnSparkHadoopUtil.getContainerId.getApplicationAttemptId()
@@ -211,9 +216,13 @@ private[spark] class ApplicationMaster(
   final def run(): Int = {
     try {
       val attemptID = if (isClusterMode) {
-        // Set the web ui port to be ephemeral for yarn so we don't conflict with
-        // other spark processes running on the same box
-        System.setProperty(UI_PORT.key, "0")
+        // Set the web ui port to be ephemeral for yarn if not set explicitly
+        // so we don't conflict with other spark processes running on the same box
+        // If set explicitly, Web UI will attempt to run on UI_PORT and try
+        // incrementally until UI_PORT + `spark.port.maxRetries`
+        if (System.getProperty(UI_PORT.key) == null) {
+          System.setProperty(UI_PORT.key, "0")
+        }
 
         // Set the master and deploy mode property to match the requested mode.
         System.setProperty("spark.master", "yarn")
@@ -304,7 +313,8 @@ private[spark] class ApplicationMaster(
       // The client-mode AM doesn't listen for incoming connections, so report an invalid port.
       registerAM(Utils.localHostName, -1, sparkConf,
         sparkConf.getOption("spark.driver.appUIAddress"), appAttemptId)
-      addAmIpFilter(Some(driverRef), ProxyUriUtils.getPath(appAttemptId.getApplicationId))
+      val encodedAppId = URLEncoder.encode(appAttemptId.getApplicationId.toString, "UTF-8")
+      addAmIpFilter(Some(driverRef), s"/proxy/$encodedAppId")
       createAllocator(driverRef, sparkConf, clientRpcEnv, appAttemptId, cachedResourcesConf)
       reporterThread.join()
     } catch {
@@ -477,8 +487,7 @@ private[spark] class ApplicationMaster(
     rpcEnv.setupEndpoint("YarnAM", new AMEndpoint(rpcEnv, driverRef))
 
     allocator.allocateResources()
-    val ms = MetricsSystem.createMetricsSystem(MetricsSystemInstances.APPLICATION_MASTER,
-      sparkConf, securityMgr)
+    val ms = MetricsSystem.createMetricsSystem(MetricsSystemInstances.APPLICATION_MASTER, sparkConf)
     val prefix = _sparkConf.get(YARN_METRICS_NAMESPACE).getOrElse(appId)
     ms.registerSource(new ApplicationMasterSource(prefix, allocator))
     // do not register static sources in this case as per SPARK-25277
@@ -563,10 +572,10 @@ private[spark] class ApplicationMaster(
           finish(FinalApplicationStatus.FAILED,
             ApplicationMaster.EXIT_MAX_EXECUTOR_FAILURES,
             s"Max number of executor failures ($maxNumExecutorFailures) reached")
-        } else if (allocator.isAllNodeBlacklisted) {
+        } else if (allocator.isAllNodeExcluded) {
           finish(FinalApplicationStatus.FAILED,
             ApplicationMaster.EXIT_MAX_EXECUTOR_FAILURES,
-            "Due to executor failures all available nodes are blacklisted")
+            "Due to executor failures all available nodes are excluded")
         } else {
           logDebug("Sending progress")
           allocator.allocateResources()
@@ -686,7 +695,7 @@ private[spark] class ApplicationMaster(
     val params = client.getAmIpFilterParams(yarnConf, proxyBase)
     driver match {
       case Some(d) =>
-        d.send(AddWebUIFilter(amFilter, params.toMap, proxyBase))
+        d.send(AddWebUIFilter(amFilter, params, proxyBase))
 
       case None =>
         System.setProperty(UI_FILTERS.key, amFilter)
@@ -773,6 +782,20 @@ private[spark] class ApplicationMaster(
 
     override def onStart(): Unit = {
       driver.send(RegisterClusterManager(self))
+      // if deployment mode for yarn Application is client
+      // then send the AM Log Info to spark driver
+      if (!isClusterMode) {
+        val hostPort = YarnContainerInfoHelper.getNodeManagerHttpAddress(None)
+        val yarnAMID = "yarn-am"
+        val info = new MiscellaneousProcessDetails(hostPort,
+          sparkConf.get(AM_CORES), extractLogUrls)
+        driver.send(MiscellaneousProcessAdded(System.currentTimeMillis(), yarnAMID, info))
+      }
+    }
+
+    override def receive: PartialFunction[Any, Unit] = {
+      case UpdateDelegationTokens(tokens) =>
+        SparkHadoopUtil.get.addDelegationTokens(tokens, sparkConf)
     }
 
     override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
@@ -783,7 +806,7 @@ private[spark] class ApplicationMaster(
               r.resourceProfileToTotalExecs,
               r.numLocalityAwareTasksPerResourceProfileId,
               r.hostToLocalTaskCount,
-              r.nodeBlacklist)) {
+              r.excludedNodes)) {
               resetAllocatorInterval()
             }
             context.reply(true)
@@ -809,9 +832,6 @@ private[spark] class ApplicationMaster(
           case None =>
             logWarning("Container allocator is not ready to find executor loss reasons yet.")
         }
-
-      case UpdateDelegationTokens(tokens) =>
-        SparkHadoopUtil.get.addDelegationTokens(tokens, sparkConf)
     }
 
     override def onDisconnected(remoteAddress: RpcAddress): Unit = {
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index fc429d6fb1972..427202fbe962e 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -21,15 +21,16 @@ import java.io.{FileSystem => _, _}
 import java.net.{InetAddress, UnknownHostException, URI}
 import java.nio.ByteBuffer
 import java.nio.charset.StandardCharsets
+import java.nio.file.Files
 import java.util.{Locale, Properties, UUID}
 import java.util.zip.{ZipEntry, ZipOutputStream}
 
 import scala.collection.JavaConverters._
+import scala.collection.immutable.{Map => IMap}
 import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet, ListBuffer, Map}
 import scala.util.control.NonFatal
 
 import com.google.common.base.Objects
-import com.google.common.io.Files
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs._
 import org.apache.hadoop.fs.permission.FsPermission
@@ -57,8 +58,9 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config._
 import org.apache.spark.internal.config.Python._
 import org.apache.spark.launcher.{LauncherBackend, SparkAppHandle, YarnCommandBuilderUtils}
+import org.apache.spark.resource.ResourceProfile
 import org.apache.spark.rpc.RpcEnv
-import org.apache.spark.util.{CallerContext, Utils}
+import org.apache.spark.util.{CallerContext, Utils, YarnContainerInfoHelper}
 
 private[spark] class Client(
     val args: ClientArguments,
@@ -87,7 +89,8 @@ private[spark] class Client(
   private val amMemoryOverhead = {
     val amMemoryOverheadEntry = if (isClusterMode) DRIVER_MEMORY_OVERHEAD else AM_MEMORY_OVERHEAD
     sparkConf.get(amMemoryOverheadEntry).getOrElse(
-      math.max((MEMORY_OVERHEAD_FACTOR * amMemory).toLong, MEMORY_OVERHEAD_MIN)).toInt
+      math.max((MEMORY_OVERHEAD_FACTOR * amMemory).toLong,
+        ResourceProfile.MEMORY_OVERHEAD_MIN_MIB)).toInt
   }
   private val amCores = if (isClusterMode) {
     sparkConf.get(DRIVER_CORES)
@@ -98,9 +101,10 @@ private[spark] class Client(
   // Executor related configurations
   private val executorMemory = sparkConf.get(EXECUTOR_MEMORY)
   // Executor offHeap memory in MiB.
-  protected val executorOffHeapMemory = YarnSparkHadoopUtil.executorOffHeapMemorySizeAsMb(sparkConf)
+  protected val executorOffHeapMemory = Utils.executorOffHeapMemorySizeAsMb(sparkConf)
   private val executorMemoryOverhead = sparkConf.get(EXECUTOR_MEMORY_OVERHEAD).getOrElse(
-    math.max((MEMORY_OVERHEAD_FACTOR * executorMemory).toLong, MEMORY_OVERHEAD_MIN)).toInt
+    math.max((MEMORY_OVERHEAD_FACTOR * executorMemory).toLong,
+      ResourceProfile.MEMORY_OVERHEAD_MIN_MIB)).toInt
 
   private val isPython = sparkConf.get(IS_PYTHON_APP)
   private val pysparkWorkerMemory: Int = if (isPython) {
@@ -181,10 +185,12 @@ private[spark] class Client(
 
       // The app staging dir based on the STAGING_DIR configuration if configured
       // otherwise based on the users home directory.
+      // scalastyle:off FileSystemGet
       val appStagingBaseDir = sparkConf.get(STAGING_DIR)
         .map { new Path(_, UserGroupInformation.getCurrentUser.getShortUserName) }
         .getOrElse(FileSystem.get(hadoopConf).getHomeDirectory())
       stagingDirPath = new Path(appStagingBaseDir, getAppStagingDir(appId))
+      // scalastyle:on FileSystemGet
 
       new CallerContext("CLIENT", sparkConf.get(APP_CALLER_CONTEXT),
         Option(appId.toString)).setCurrentContext()
@@ -512,6 +518,32 @@ private[spark] class Client(
       require(localizedPath != null, "Keytab file already distributed.")
     }
 
+    // If we passed in a ivySettings file, make sure we copy the file to the distributed cache
+    // in cluster mode so that the driver can access it
+    val ivySettings = sparkConf.getOption("spark.jars.ivySettings")
+    val ivySettingsLocalizedPath: Option[String] = ivySettings match {
+      case Some(ivySettingsPath) if isClusterMode =>
+        val uri = new URI(ivySettingsPath)
+        Option(uri.getScheme).getOrElse("file") match {
+          case "file" =>
+            val ivySettingsFile = new File(uri.getPath)
+            require(ivySettingsFile.exists(), s"Ivy settings file $ivySettingsFile not found")
+            require(ivySettingsFile.isFile(), s"Ivy settings file $ivySettingsFile is not a" +
+              "normal file")
+            // Generate a file name that can be used for the ivySettings file, that does not
+            // conflict with any user file.
+            val localizedFileName = Some(ivySettingsFile.getName() + "-" +
+              UUID.randomUUID().toString)
+            val (_, localizedPath) = distribute(ivySettingsPath, destName = localizedFileName)
+            require(localizedPath != null, "IvySettings file already distributed.")
+            Some(localizedPath)
+          case scheme =>
+            throw new IllegalArgumentException(s"Scheme $scheme not supported in " +
+              "spark.jars.ivySettings")
+        }
+      case _ => None
+    }
+
     /**
      * Add Spark to the cache. There are two settings that control what files to add to the cache:
      * - if a Spark archive is defined, use the archive. The archive is expected to contain
@@ -553,7 +585,7 @@ private[spark] class Client(
           }
 
           // Propagate the local URIs to the containers using the configuration.
-          sparkConf.set(SPARK_JARS, localJars)
+          sparkConf.set(SPARK_JARS, localJars.toSeq)
 
         case None =>
           // No configuration, so fall back to uploading local jar files.
@@ -570,7 +602,7 @@ private[spark] class Client(
             jarsDir.listFiles().foreach { f =>
               if (f.isFile && f.getName.toLowerCase(Locale.ROOT).endsWith(".jar") && f.canRead) {
                 jarsStream.putNextEntry(new ZipEntry(f.getName))
-                Files.copy(f, jarsStream)
+                Files.copy(f.toPath, jarsStream)
                 jarsStream.closeEntry()
               }
             }
@@ -628,14 +660,19 @@ private[spark] class Client(
       }
     }
     if (cachedSecondaryJarLinks.nonEmpty) {
-      sparkConf.set(SECONDARY_JARS, cachedSecondaryJarLinks)
+      sparkConf.set(SECONDARY_JARS, cachedSecondaryJarLinks.toSeq)
     }
 
     if (isClusterMode && args.primaryPyFile != null) {
       distribute(args.primaryPyFile, appMasterOnly = true)
     }
 
-    pySparkArchives.foreach { f => distribute(f) }
+    pySparkArchives.foreach { f =>
+      val uri = Utils.resolveURI(f)
+      if (uri.getScheme != Utils.LOCAL_SCHEME) {
+        distribute(f)
+      }
+    }
 
     // The python files list needs to be treated especially. All files that are not an
     // archive need to be placed in a subdirectory that will be added to PYTHONPATH.
@@ -661,7 +698,18 @@ private[spark] class Client(
     val remoteFs = FileSystem.get(remoteConfArchivePath.toUri(), hadoopConf)
     cachedResourcesConf.set(CACHED_CONF_ARCHIVE, remoteConfArchivePath.toString())
 
-    val localConfArchive = new Path(createConfArchive().toURI())
+    val confsToOverride = Map.empty[String, String]
+    // If propagating the keytab to the AM, override the keytab name with the name of the
+    // distributed file.
+    amKeytabFileName.foreach { kt => confsToOverride.put(KEYTAB.key, kt) }
+
+    // If propagating the ivySettings file to the distributed cache, override the ivySettings
+    // file name with the name of the distributed file.
+    ivySettingsLocalizedPath.foreach { path =>
+      confsToOverride.put("spark.jars.ivySettings", path)
+    }
+
+    val localConfArchive = new Path(createConfArchive(confsToOverride).toURI())
     copyFileToRemote(destDir, localConfArchive, replication, symlinkCache, force = true,
       destName = Some(LOCALIZED_CONF_ARCHIVE))
 
@@ -690,8 +738,10 @@ private[spark] class Client(
    *
    * The archive also contains some Spark configuration. Namely, it saves the contents of
    * SparkConf in a file to be loaded by the AM process.
+   *
+   * @param confsToOverride configs that should overriden when creating the final spark conf file
    */
-  private def createConfArchive(): File = {
+  private def createConfArchive(confsToOverride: Map[String, String]): File = {
     val hadoopConfFiles = new HashMap[String, File]()
 
     // SPARK_CONF_DIR shows up in the classpath before HADOOP_CONF_DIR/YARN_CONF_DIR
@@ -753,7 +803,7 @@ private[spark] class Client(
             if url.getProtocol == "file" } {
         val file = new File(url.getPath())
         confStream.putNextEntry(new ZipEntry(file.getName()))
-        Files.copy(file, confStream)
+        Files.copy(file.toPath, confStream)
         confStream.closeEntry()
       }
 
@@ -764,7 +814,7 @@ private[spark] class Client(
       hadoopConfFiles.foreach { case (name, file) =>
         if (file.canRead()) {
           confStream.putNextEntry(new ZipEntry(s"$LOCALIZED_HADOOP_CONF_DIR/$name"))
-          Files.copy(file, confStream)
+          Files.copy(file.toPath, confStream)
           confStream.closeEntry()
         }
       }
@@ -777,11 +827,7 @@ private[spark] class Client(
 
       // Save Spark configuration to a file in the archive.
       val props = confToProperties(sparkConf)
-
-      // If propagating the keytab to the AM, override the keytab name with the name of the
-      // distributed file.
-      amKeytabFileName.foreach { kt => props.setProperty(KEYTAB.key, kt) }
-
+      confsToOverride.foreach { case (k, v) => props.setProperty(k, v)}
       writePropertiesToArchive(props, SPARK_CONF_FILE, confStream)
 
       // Write the distributed cache config to the archive.
@@ -1058,7 +1104,7 @@ private[spark] class Client(
             logError(s"Application $appId not found.")
             cleanupStagingDir()
             return YarnAppReport(YarnApplicationState.KILLED, FinalApplicationStatus.KILLED, None)
-          case NonFatal(e) =>
+          case NonFatal(e) if !e.isInstanceOf[InterruptedIOException] =>
             val msg = s"Failed to contact YARN for application $appId."
             logError(msg, e)
             // Don't necessarily clean up staging dir because status is unknown
@@ -1073,9 +1119,9 @@ private[spark] class Client(
         // If DEBUG is enabled, log report details every iteration
         // Otherwise, log them every time the application changes state
         if (log.isDebugEnabled) {
-          logDebug(formatReportDetails(report))
+          logDebug(formatReportDetails(report, getDriverLogsLink(report)))
         } else if (lastState != state) {
-          logInfo(formatReportDetails(report))
+          logInfo(formatReportDetails(report, getDriverLogsLink(report)))
         }
       }
 
@@ -1145,7 +1191,17 @@ private[spark] class Client(
     appMaster
   }
 
-  private def formatReportDetails(report: ApplicationReport): String = {
+  /**
+   * Format an application report and optionally, links to driver logs, in a human-friendly manner.
+   *
+   * @param report The application report from YARN.
+   * @param driverLogsLinks A map of driver log files and their links. Keys are the file names
+   *                        (e.g. `stdout`), and values are the links. If empty, nothing will be
+   *                        printed.
+   * @return Human-readable version of the input data.
+   */
+  private def formatReportDetails(report: ApplicationReport,
+    driverLogsLinks: IMap[String, String]): String = {
     val details = Seq[(String, String)](
       ("client token", getClientToken(report)),
       ("diagnostics", report.getDiagnostics),
@@ -1156,7 +1212,7 @@ private[spark] class Client(
       ("final status", report.getFinalApplicationStatus.toString),
       ("tracking URL", report.getTrackingUrl),
       ("user", report.getUser)
-    )
+    ) ++ driverLogsLinks.map { case (fname, link) => (s"Driver Logs ($fname)", link) }
 
     // Use more loggable format if value is null or empty
     details.map { case (k, v) =>
@@ -1165,6 +1221,35 @@ private[spark] class Client(
     }.mkString("")
   }
 
+  /**
+   * Fetch links to the logs of the driver for the given application report. This requires
+   * query the ResourceManager via RPC. Returns an empty map if the links could not be fetched.
+   * If this feature is disabled via [[CLIENT_INCLUDE_DRIVER_LOGS_LINK]], or if the application
+   * report indicates that the driver container isn't currently running, an empty map is
+   * returned immediately.
+   */
+  private def getDriverLogsLink(appReport: ApplicationReport): IMap[String, String] = {
+    if (!sparkConf.get(CLIENT_INCLUDE_DRIVER_LOGS_LINK)
+      || appReport.getYarnApplicationState != YarnApplicationState.RUNNING) {
+      return IMap.empty
+    }
+    try {
+      Option(appReport.getCurrentApplicationAttemptId)
+        .flatMap(attemptId => Option(yarnClient.getApplicationAttemptReport(attemptId)))
+        .flatMap(attemptReport => Option(attemptReport.getAMContainerId))
+        .flatMap(amContainerId => Option(yarnClient.getContainerReport(amContainerId)))
+        .flatMap(containerReport => Option(containerReport.getLogUrl))
+        .map(YarnContainerInfoHelper.getLogUrlsFromBaseUrl)
+        .getOrElse(IMap.empty)
+    } catch {
+      case e: Exception =>
+        logWarning(s"Unable to get driver log links for $appId: $e")
+        // Include the full stack trace only at DEBUG level to reduce verbosity
+        logDebug(s"Unable to get driver log links for $appId", e)
+        IMap.empty
+    }
+  }
+
   /**
    * Submit an application to the ResourceManager.
    * If set spark.yarn.submit.waitAppCompletion to true, it will stay alive
@@ -1179,7 +1264,7 @@ private[spark] class Client(
       val report = getApplicationReport(appId)
       val state = report.getYarnApplicationState
       logInfo(s"Application report for $appId (state: $state)")
-      logInfo(formatReportDetails(report))
+      logInfo(formatReportDetails(report, getDriverLogsLink(report)))
       if (state == YarnApplicationState.FAILED || state == YarnApplicationState.KILLED) {
         throw new SparkException(s"Application $appId finished with status: $state")
       }
@@ -1579,6 +1664,7 @@ private[spark] class YarnClusterApplication extends SparkApplication {
     // so remove them from sparkConf here for yarn mode.
     conf.remove(JARS)
     conf.remove(FILES)
+    conf.remove(ARCHIVES)
 
     new Client(new ClientArguments(args), conf, null).run()
   }
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ClientDistributedCacheManager.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ClientDistributedCacheManager.scala
index e6e0ea38ade94..e02fbd0c91495 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ClientDistributedCacheManager.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ClientDistributedCacheManager.scala
@@ -91,11 +91,11 @@ private[spark] class ClientDistributedCacheManager() extends Logging {
    * Writes down information about cached files needed in executors to the given configuration.
    */
   def updateConfiguration(conf: SparkConf): Unit = {
-    conf.set(CACHED_FILES, distCacheEntries.map(_.uri.toString))
-    conf.set(CACHED_FILES_SIZES, distCacheEntries.map(_.size))
-    conf.set(CACHED_FILES_TIMESTAMPS, distCacheEntries.map(_.modTime))
-    conf.set(CACHED_FILES_VISIBILITIES, distCacheEntries.map(_.visibility.name()))
-    conf.set(CACHED_FILES_TYPES, distCacheEntries.map(_.resType.name()))
+    conf.set(CACHED_FILES, distCacheEntries.map(_.uri.toString).toSeq)
+    conf.set(CACHED_FILES_SIZES, distCacheEntries.map(_.size).toSeq)
+    conf.set(CACHED_FILES_TIMESTAMPS, distCacheEntries.map(_.modTime).toSeq)
+    conf.set(CACHED_FILES_VISIBILITIES, distCacheEntries.map(_.visibility.name()).toSeq)
+    conf.set(CACHED_FILES_TYPES, distCacheEntries.map(_.resType.name()).toSeq)
   }
 
   /**
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
index d9262bbac6586..717ce57b902c1 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
@@ -19,12 +19,11 @@ package org.apache.spark.deploy.yarn
 
 import java.io.File
 import java.nio.ByteBuffer
-import java.util.{Collections, Locale}
+import java.util.Collections
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable.{HashMap, ListBuffer}
 
-import org.apache.hadoop.HadoopIllegalArgumentException
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.io.DataOutputBuffer
 import org.apache.hadoop.security.UserGroupInformation
@@ -40,7 +39,6 @@ import org.apache.spark.{SecurityManager, SparkConf, SparkException}
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config._
 import org.apache.spark.network.util.JavaUtils
-import org.apache.spark.resource.ResourceProfile
 import org.apache.spark.util.Utils
 
 private[yarn] class ExecutorRunnable(
@@ -117,7 +115,9 @@ private[yarn] class ExecutorRunnable(
           // Authentication is not enabled, so just provide dummy metadata
           ByteBuffer.allocate(0)
         }
-      ctx.setServiceData(Collections.singletonMap("spark_shuffle", secretBytes))
+      val serviceName = sparkConf.get(SHUFFLE_SERVICE_NAME)
+      logInfo(s"Initializing service data for shuffle service using name '$serviceName'")
+      ctx.setServiceData(Collections.singletonMap(serviceName, secretBytes))
     }
 
     // Send the start request to the ContainerManager
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/LocalityPreferredContainerPlacementStrategy.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/LocalityPreferredContainerPlacementStrategy.scala
index 5640f7ede33df..7ac5beac76e20 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/LocalityPreferredContainerPlacementStrategy.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/LocalityPreferredContainerPlacementStrategy.scala
@@ -21,7 +21,7 @@ import scala.collection.JavaConverters._
 import scala.collection.mutable.{ArrayBuffer, HashMap, Set}
 
 import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.yarn.api.records.{ContainerId, Resource}
+import org.apache.hadoop.yarn.api.records.ContainerId
 import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest
 
 import org.apache.spark.SparkConf
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ResourceRequestHelper.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ResourceRequestHelper.scala
index 3d800be9e210b..3aabc467a9de9 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ResourceRequestHelper.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ResourceRequestHelper.scala
@@ -43,6 +43,8 @@ private object ResourceRequestHelper extends Logging {
   private val RESOURCE_NOT_FOUND = "org.apache.hadoop.yarn.exceptions.ResourceNotFoundException"
   val YARN_GPU_RESOURCE_CONFIG = "yarn.io/gpu"
   val YARN_FPGA_RESOURCE_CONFIG = "yarn.io/fpga"
+  private[yarn] val resourceNameMapping =
+    Map(GPU -> YARN_GPU_RESOURCE_CONFIG, FPGA -> YARN_FPGA_RESOURCE_CONFIG)
   @volatile private var numResourceErrors: Int = 0
 
   private[yarn] def getYarnResourcesAndAmounts(
@@ -76,7 +78,7 @@ private object ResourceRequestHelper extends Logging {
       confPrefix: String,
       sparkConf: SparkConf
   ): Map[String, String] = {
-    Map(GPU -> YARN_GPU_RESOURCE_CONFIG, FPGA -> YARN_FPGA_RESOURCE_CONFIG).map {
+    resourceNameMapping.map {
       case (rName, yarnName) =>
         (yarnName -> sparkConf.get(new ResourceID(confPrefix, rName).amountConf, "0"))
     }.filter { case (_, count) => count.toLong > 0 }
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
index cd0e7d5c87bc8..f236fc39f61fa 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
@@ -37,7 +37,6 @@ import org.apache.spark.deploy.yarn.YarnSparkHadoopUtil._
 import org.apache.spark.deploy.yarn.config._
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config._
-import org.apache.spark.internal.config.Python._
 import org.apache.spark.resource.ResourceProfile
 import org.apache.spark.resource.ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID
 import org.apache.spark.rpc.{RpcCallContext, RpcEndpointRef}
@@ -159,37 +158,10 @@ private[yarn] class YarnAllocator(
 
   private[spark] val failureTracker = new FailureTracker(sparkConf, clock)
 
-  private val allocatorBlacklistTracker =
-    new YarnAllocatorBlacklistTracker(sparkConf, amClient, failureTracker)
-
-  // Executor memory in MiB.
-  protected val executorMemory = sparkConf.get(EXECUTOR_MEMORY).toInt
-  // Executor offHeap memory in MiB.
-  protected val executorOffHeapMemory = YarnSparkHadoopUtil.executorOffHeapMemorySizeAsMb(sparkConf)
-  // Additional memory overhead.
-  protected val memoryOverhead: Int = sparkConf.get(EXECUTOR_MEMORY_OVERHEAD).getOrElse(
-    math.max((MEMORY_OVERHEAD_FACTOR * executorMemory).toInt, MEMORY_OVERHEAD_MIN)).toInt
-  protected val pysparkWorkerMemory: Int = if (sparkConf.get(IS_PYTHON_APP)) {
-    sparkConf.get(PYSPARK_EXECUTOR_MEMORY).map(_.toInt).getOrElse(0)
-  } else {
-    0
-  }
-  // Number of cores per executor for the default profile
-  protected val defaultExecutorCores = sparkConf.get(EXECUTOR_CORES)
-
-  private val executorResourceRequests =
-    getYarnResourcesAndAmounts(sparkConf, config.YARN_EXECUTOR_RESOURCE_TYPES_PREFIX) ++
-    getYarnResourcesFromSparkResources(SPARK_EXECUTOR_PREFIX, sparkConf)
-
-  // Resource capability requested for each executor for the default profile
-  private[yarn] val defaultResource: Resource = {
-    val resource: Resource = Resource.newInstance(
-      executorMemory + executorOffHeapMemory + memoryOverhead + pysparkWorkerMemory,
-      defaultExecutorCores)
-    ResourceRequestHelper.setResourceRequests(executorResourceRequests, resource)
-    logDebug(s"Created resource capability: $resource")
-    resource
-  }
+  private val allocatorNodeHealthTracker =
+    new YarnAllocatorNodeHealthTracker(sparkConf, amClient, failureTracker)
+
+  private val isPythonApp = sparkConf.get(IS_PYTHON_APP)
 
   private val launcherPool = ThreadUtils.newDaemonCachedThreadPool(
     "ContainerLauncher", sparkConf.get(CONTAINER_LAUNCH_MAX_THREADS))
@@ -211,11 +183,10 @@ private[yarn] class YarnAllocator(
       new HashMap[String, mutable.Set[ContainerId]]()
     runningExecutorsPerResourceProfileId.put(DEFAULT_RESOURCE_PROFILE_ID, mutable.HashSet[String]())
     numExecutorsStartingPerResourceProfileId(DEFAULT_RESOURCE_PROFILE_ID) = new AtomicInteger(0)
-    targetNumExecutorsPerResourceProfileId(DEFAULT_RESOURCE_PROFILE_ID) =
-      SchedulerBackendUtils.getInitialTargetExecutorNumber(sparkConf)
-    rpIdToYarnResource.put(DEFAULT_RESOURCE_PROFILE_ID, defaultResource)
-    rpIdToResourceProfile(DEFAULT_RESOURCE_PROFILE_ID) =
-      ResourceProfile.getOrCreateDefaultProfile(sparkConf)
+    val initTargetExecNum = SchedulerBackendUtils.getInitialTargetExecutorNumber(sparkConf)
+    targetNumExecutorsPerResourceProfileId(DEFAULT_RESOURCE_PROFILE_ID) = initTargetExecNum
+    val defaultProfile = ResourceProfile.getOrCreateDefaultProfile(sparkConf)
+    createYarnResourceForResourceProfile(defaultProfile)
   }
 
   initDefaultProfile()
@@ -238,7 +209,7 @@ private[yarn] class YarnAllocator(
 
   def getNumExecutorsFailed: Int = failureTracker.numFailedExecutors
 
-  def isAllNodeBlacklisted: Boolean = allocatorBlacklistTracker.isAllNodeBlacklisted
+  def isAllNodeExcluded: Boolean = allocatorNodeHealthTracker.isAllNodeExcluded
 
   /**
    * A sequence of pending container requests that have not yet been fulfilled.
@@ -292,57 +263,65 @@ private[yarn] class YarnAllocator(
   private def getPendingAtLocation(
       location: String): Map[Int, Seq[ContainerRequest]] = synchronized {
     val allContainerRequests = new mutable.HashMap[Int, Seq[ContainerRequest]]
-    rpIdToResourceProfile.keys.map { id =>
+    rpIdToResourceProfile.keys.foreach { id =>
       val profResource = rpIdToYarnResource.get(id)
       val result = amClient.getMatchingRequests(getContainerPriority(id), location, profResource)
         .asScala.flatMap(_.asScala)
-      allContainerRequests(id) = result
+      allContainerRequests(id) = result.toSeq
     }
     allContainerRequests.toMap
   }
 
   // if a ResourceProfile hasn't been seen yet, create the corresponding YARN Resource for it
-  private def createYarnResourceForResourceProfile(
-      resourceProfileToTotalExecs: Map[ResourceProfile, Int]): Unit = synchronized {
-    resourceProfileToTotalExecs.foreach { case (rp, num) =>
-      if (!rpIdToYarnResource.contains(rp.id)) {
-        // Start with the application or default settings
-        var heapMem = executorMemory.toLong
-        // Note we currently don't support off heap memory in ResourceProfile - SPARK-30794
-        var offHeapMem = executorOffHeapMemory.toLong
-        var overheadMem = memoryOverhead.toLong
-        var pysparkMem = pysparkWorkerMemory.toLong
-        var cores = defaultExecutorCores
-        val customResources = new mutable.HashMap[String, String]
-        // track the resource profile if not already there
-        getOrUpdateRunningExecutorForRPId(rp.id)
-        logInfo(s"Resource profile ${rp.id} doesn't exist, adding it")
-        val execResources = rp.executorResources
-        execResources.foreach { case (r, execReq) =>
-          r match {
-            case ResourceProfile.MEMORY =>
-              heapMem = execReq.amount
-            case ResourceProfile.OVERHEAD_MEM =>
-              overheadMem = execReq.amount
-            case ResourceProfile.PYSPARK_MEM =>
-              pysparkMem = execReq.amount
-            case ResourceProfile.CORES =>
-              cores = execReq.amount.toInt
-            case "gpu" =>
-              customResources(YARN_GPU_RESOURCE_CONFIG) = execReq.amount.toString
-            case "fpga" =>
-              customResources(YARN_FPGA_RESOURCE_CONFIG) = execReq.amount.toString
-            case rName =>
-              customResources(rName) = execReq.amount.toString
-          }
+  private def createYarnResourceForResourceProfile(rp: ResourceProfile): Unit = synchronized {
+    if (!rpIdToYarnResource.contains(rp.id)) {
+      // track the resource profile if not already there
+      getOrUpdateRunningExecutorForRPId(rp.id)
+      logInfo(s"Resource profile ${rp.id} doesn't exist, adding it")
+      val resourcesWithDefaults =
+        ResourceProfile.getResourcesForClusterManager(rp.id, rp.executorResources,
+          MEMORY_OVERHEAD_FACTOR, sparkConf, isPythonApp,
+          ResourceRequestHelper.resourceNameMapping)
+      val customSparkResources =
+        resourcesWithDefaults.customResources.map { case (name, execReq) =>
+          (name, execReq.amount.toString)
         }
-        val totalMem = (heapMem + offHeapMem + overheadMem + pysparkMem).toInt
-        val resource = Resource.newInstance(totalMem, cores)
-        ResourceRequestHelper.setResourceRequests(customResources.toMap, resource)
-        logDebug(s"Created resource capability: $resource")
-        rpIdToYarnResource.putIfAbsent(rp.id, resource)
-        rpIdToResourceProfile(rp.id) = rp
+      // There is a difference in the way custom resources are handled between
+      // the base default profile and custom ResourceProfiles. To allow for the user
+      // to request YARN containers with extra resources without Spark scheduling on
+      // them, the user can specify resources via the <code>spark.yarn.executor.resource.</code>
+      // config. Those configs are only used in the base default profile though and do
+      // not get propogated into any other custom ResourceProfiles. This is because
+      // there would be no way to remove them if you wanted a stage to not have them.
+      // This results in your default profile getting custom resources defined in
+      // <code>spark.yarn.executor.resource.</code> plus spark defined resources of
+      // GPU or FPGA. Spark converts GPU and FPGA resources into the YARN built in
+      // types <code>yarn.io/gpu</code>) and <code>yarn.io/fpga</code>, but does not
+      // know the mapping of any other resources. Any other Spark custom resources
+      // are not propogated to YARN for the default profile. So if you want Spark
+      // to schedule based off a custom resource and have it requested from YARN, you
+      // must specify it in both YARN (<code>spark.yarn.{driver/executor}.resource.</code>)
+      // and Spark (<code>spark.{driver/executor}.resource.</code>) configs. Leave the Spark
+      // config off if you only want YARN containers with the extra resources but Spark not to
+      // schedule using them. Now for custom ResourceProfiles, it doesn't currently have a way
+      // to only specify YARN resources without Spark scheduling off of them. This means for
+      // custom ResourceProfiles we propogate all the resources defined in the ResourceProfile
+      // to YARN. We still convert GPU and FPGA to the YARN build in types as well. This requires
+      // that the name of any custom resources you specify match what they are defined as in YARN.
+      val customResources = if (rp.id == DEFAULT_RESOURCE_PROFILE_ID) {
+        getYarnResourcesAndAmounts(sparkConf, config.YARN_EXECUTOR_RESOURCE_TYPES_PREFIX) ++
+          customSparkResources.filterKeys { r =>
+            (r == YARN_GPU_RESOURCE_CONFIG || r == YARN_FPGA_RESOURCE_CONFIG)
+          }
+      } else {
+        customSparkResources
       }
+      val resource =
+        Resource.newInstance(resourcesWithDefaults.totalMemMiB.toInt, resourcesWithDefaults.cores)
+      ResourceRequestHelper.setResourceRequests(customResources, resource)
+      logDebug(s"Created resource capability: $resource")
+      rpIdToYarnResource.putIfAbsent(rp.id, resource)
+      rpIdToResourceProfile(rp.id) = rp
     }
   }
 
@@ -357,26 +336,25 @@ private[yarn] class YarnAllocator(
    *                                                  placement hint.
    * @param hostToLocalTaskCount a map of preferred hostname to possible task counts for each
    *                             ResourceProfile id to be used as container placement hint.
-   * @param nodeBlacklist blacklisted nodes, which is passed in to avoid allocating new containers
-   *                      on them. It will be used to update the application master's blacklist.
+   * @param excludedNodes excluded nodes, which is passed in to avoid allocating new containers
+   *                      on them. It will be used to update the applications excluded node list.
    * @return Whether the new requested total is different than the old value.
    */
   def requestTotalExecutorsWithPreferredLocalities(
       resourceProfileToTotalExecs: Map[ResourceProfile, Int],
       numLocalityAwareTasksPerResourceProfileId: Map[Int, Int],
       hostToLocalTaskCountPerResourceProfileId: Map[Int, Map[String, Int]],
-      nodeBlacklist: Set[String]): Boolean = synchronized {
+      excludedNodes: Set[String]): Boolean = synchronized {
     this.numLocalityAwareTasksPerResourceProfileId = numLocalityAwareTasksPerResourceProfileId
     this.hostToLocalTaskCountPerResourceProfileId = hostToLocalTaskCountPerResourceProfileId
 
-    createYarnResourceForResourceProfile(resourceProfileToTotalExecs)
-
     val res = resourceProfileToTotalExecs.map { case (rp, numExecs) =>
+      createYarnResourceForResourceProfile(rp)
       if (numExecs != getOrUpdateTargetNumExecutorsForRPId(rp.id)) {
         logInfo(s"Driver requested a total number of $numExecs executor(s) " +
           s"for resource profile id: ${rp.id}.")
         targetNumExecutorsPerResourceProfileId(rp.id) = numExecs
-        allocatorBlacklistTracker.setSchedulerBlacklistedNodes(nodeBlacklist)
+        allocatorNodeHealthTracker.setSchedulerExcludedNodes(excludedNodes)
         true
       } else {
         false
@@ -415,7 +393,7 @@ private[yarn] class YarnAllocator(
     val allocateResponse = amClient.allocate(progressIndicator)
 
     val allocatedContainers = allocateResponse.getAllocatedContainers()
-    allocatorBlacklistTracker.setNumClusterNodes(allocateResponse.getNumClusterNodes)
+    allocatorNodeHealthTracker.setNumClusterNodes(allocateResponse.getNumClusterNodes)
 
     if (allocatedContainers.size > 0) {
       logDebug(("Allocated containers: %d. Current executor count: %d. " +
@@ -426,13 +404,13 @@ private[yarn] class YarnAllocator(
           getNumExecutorsStarting,
           allocateResponse.getAvailableResources))
 
-      handleAllocatedContainers(allocatedContainers.asScala)
+      handleAllocatedContainers(allocatedContainers.asScala.toSeq)
     }
 
     val completedContainers = allocateResponse.getCompletedContainersStatuses()
     if (completedContainers.size > 0) {
       logDebug("Completed %d containers".format(completedContainers.size))
-      processCompletedContainers(completedContainers.asScala)
+      processCompletedContainers(completedContainers.asScala.toSeq)
       logDebug("Finished processing %d completed containers. Current running executor count: %d."
         .format(completedContainers.size, getNumExecutorsRunning))
     }
@@ -476,7 +454,7 @@ private[yarn] class YarnAllocator(
           var requestContainerMessage = s"Will request $missing executor container(s) for " +
             s" ResourceProfile Id: $rpId, each with " +
             s"${resource.getVirtualCores} core(s) and " +
-            s"${resource.getMemory} MB memory (including $memoryOverhead MB of overhead)"
+            s"${resource.getMemory} MB memory."
           if (ResourceRequestHelper.isYarnResourceTypesAvailable() &&
             ResourceRequestHelper.isYarnCustomResourcesNonEmpty(resource)) {
             requestContainerMessage ++= s" with custom resources: " + resource.toString
@@ -722,9 +700,10 @@ private[yarn] class YarnAllocator(
       }
 
       val rp = rpIdToResourceProfile(rpId)
+      val defaultResources = ResourceProfile.getDefaultProfileExecutorResources(sparkConf)
       val containerMem = rp.executorResources.get(ResourceProfile.MEMORY).
-        map(_.amount.toInt).getOrElse(executorMemory)
-      val containerCores = rp.getExecutorCores.getOrElse(defaultExecutorCores)
+        map(_.amount).getOrElse(defaultResources.executorMemoryMiB).toInt
+      val containerCores = rp.getExecutorCores.getOrElse(defaultResources.cores)
       val rpRunningExecs = getOrUpdateRunningExecutorForRPId(rpId).size
       if (rpRunningExecs < getOrUpdateTargetNumExecutorsForRPId(rpId)) {
         getOrUpdateNumExecutorsStartingForRPId(rpId).incrementAndGet()
@@ -802,7 +781,7 @@ private[yarn] class YarnAllocator(
         val (exitCausedByApp, containerExitReason) = exitStatus match {
           case ContainerExitStatus.SUCCESS =>
             (false, s"Executor for container $containerId exited because of a YARN event (e.g., " +
-              "pre-emption) and not because of an error in the running job.")
+              "preemption) and not because of an error in the running job.")
           case ContainerExitStatus.PREEMPTED =>
             // Preemption is not the fault of the running tasks, since YARN preempts containers
             // merely to do resource sharing, and tasks that fail due to preempted executors could
@@ -826,7 +805,7 @@ private[yarn] class YarnAllocator(
               s"$diag Consider boosting ${EXECUTOR_MEMORY_OVERHEAD.key}."
             (true, message)
           case other_exit_status =>
-            // SPARK-26269: follow YARN's blacklisting behaviour(see https://github
+            // SPARK-26269: follow YARN's behaviour(see https://github
             // .com/apache/hadoop/blob/228156cfd1b474988bc4fedfbf7edddc87db41e3/had
             // oop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/ap
             // ache/hadoop/yarn/util/Apps.java#L273 for details)
@@ -836,7 +815,7 @@ private[yarn] class YarnAllocator(
                 s". Diagnostics: ${completedContainer.getDiagnostics}.")
             } else {
               // completed container from a bad node
-              allocatorBlacklistTracker.handleResourceAllocationFailure(hostOpt)
+              allocatorNodeHealthTracker.handleResourceAllocationFailure(hostOpt)
               (true, s"Container from a bad node: $containerId$onHostStr" +
                 s". Exit status: ${completedContainer.getExitStatus}" +
                 s". Diagnostics: ${completedContainer.getDiagnostics}.")
@@ -960,7 +939,7 @@ private[yarn] class YarnAllocator(
       }
     }
 
-    (localityMatched, localityUnMatched, localityFree)
+    (localityMatched.toSeq, localityUnMatched.toSeq, localityFree.toSeq)
   }
 
 }
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocatorBlacklistTracker.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocatorBlacklistTracker.scala
deleted file mode 100644
index fa8c9610220c8..0000000000000
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocatorBlacklistTracker.scala
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.deploy.yarn
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-import scala.collection.mutable.HashMap
-
-import org.apache.hadoop.yarn.client.api.AMRMClient
-import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest
-
-import org.apache.spark.SparkConf
-import org.apache.spark.deploy.yarn.config._
-import org.apache.spark.internal.Logging
-import org.apache.spark.internal.config._
-import org.apache.spark.scheduler.BlacklistTracker
-import org.apache.spark.util.{Clock, SystemClock}
-
-/**
- * YarnAllocatorBlacklistTracker is responsible for tracking the blacklisted nodes
- * and synchronizing the node list to YARN.
- *
- * Blacklisted nodes are coming from two different sources:
- *
- * <ul>
- *   <li> from the scheduler as task level blacklisted nodes
- *   <li> from this class (tracked here) as YARN resource allocation problems
- * </ul>
- *
- * The reason to realize this logic here (and not in the driver) is to avoid possible delays
- * between synchronizing the blacklisted nodes with YARN and resource allocations.
- */
-private[spark] class YarnAllocatorBlacklistTracker(
-    sparkConf: SparkConf,
-    amClient: AMRMClient[ContainerRequest],
-    failureTracker: FailureTracker)
-  extends Logging {
-
-  private val blacklistTimeoutMillis = BlacklistTracker.getBlacklistTimeout(sparkConf)
-
-  private val launchBlacklistEnabled = sparkConf.get(YARN_EXECUTOR_LAUNCH_BLACKLIST_ENABLED)
-
-  private val maxFailuresPerHost = sparkConf.get(MAX_FAILED_EXEC_PER_NODE)
-
-  private val excludeNodes = sparkConf.get(YARN_EXCLUDE_NODES).toSet
-
-  private val allocatorBlacklist = new HashMap[String, Long]()
-
-  private var currentBlacklistedYarnNodes = Set.empty[String]
-
-  private var schedulerBlacklist = Set.empty[String]
-
-  private var numClusterNodes = Int.MaxValue
-
-  def setNumClusterNodes(numClusterNodes: Int): Unit = {
-    this.numClusterNodes = numClusterNodes
-  }
-
-  def handleResourceAllocationFailure(hostOpt: Option[String]): Unit = {
-    hostOpt match {
-      case Some(hostname) if launchBlacklistEnabled =>
-        // failures on an already blacklisted nodes are not even tracked.
-        // otherwise, such failures could shutdown the application
-        // as resource requests are asynchronous
-        // and a late failure response could exceed MAX_EXECUTOR_FAILURES
-        if (!schedulerBlacklist.contains(hostname) &&
-            !allocatorBlacklist.contains(hostname)) {
-          failureTracker.registerFailureOnHost(hostname)
-          updateAllocationBlacklistedNodes(hostname)
-        }
-      case _ =>
-        failureTracker.registerExecutorFailure()
-    }
-  }
-
-  private def updateAllocationBlacklistedNodes(hostname: String): Unit = {
-    val failuresOnHost = failureTracker.numFailuresOnHost(hostname)
-    if (failuresOnHost > maxFailuresPerHost) {
-      logInfo(s"blacklisting $hostname as YARN allocation failed $failuresOnHost times")
-      allocatorBlacklist.put(
-        hostname,
-        failureTracker.clock.getTimeMillis() + blacklistTimeoutMillis)
-      refreshBlacklistedNodes()
-    }
-  }
-
-  def setSchedulerBlacklistedNodes(schedulerBlacklistedNodesWithExpiry: Set[String]): Unit = {
-    this.schedulerBlacklist = schedulerBlacklistedNodesWithExpiry
-    refreshBlacklistedNodes()
-  }
-
-  def isAllNodeBlacklisted: Boolean = currentBlacklistedYarnNodes.size >= numClusterNodes
-
-  private def refreshBlacklistedNodes(): Unit = {
-    removeExpiredYarnBlacklistedNodes()
-    val allBlacklistedNodes = excludeNodes ++ schedulerBlacklist ++ allocatorBlacklist.keySet
-    synchronizeBlacklistedNodeWithYarn(allBlacklistedNodes)
-  }
-
-  private def synchronizeBlacklistedNodeWithYarn(nodesToBlacklist: Set[String]): Unit = {
-    // Update blacklist information to YARN ResourceManager for this application,
-    // in order to avoid allocating new Containers on the problematic nodes.
-    val additions = (nodesToBlacklist -- currentBlacklistedYarnNodes).toList.sorted
-    val removals = (currentBlacklistedYarnNodes -- nodesToBlacklist).toList.sorted
-    if (additions.nonEmpty) {
-      logInfo(s"adding nodes to YARN application master's blacklist: $additions")
-    }
-    if (removals.nonEmpty) {
-      logInfo(s"removing nodes from YARN application master's blacklist: $removals")
-    }
-    if (additions.nonEmpty || removals.nonEmpty) {
-      amClient.updateBlacklist(additions.asJava, removals.asJava)
-    }
-    currentBlacklistedYarnNodes = nodesToBlacklist
-  }
-
-  private def removeExpiredYarnBlacklistedNodes(): Unit = {
-    val now = failureTracker.clock.getTimeMillis()
-    allocatorBlacklist.retain { (_, expiryTime) => expiryTime > now }
-  }
-}
-
-/**
- * FailureTracker is responsible for tracking executor failures both for each host separately
- * and for all hosts altogether.
- */
-private[spark] class FailureTracker(
-    sparkConf: SparkConf,
-    val clock: Clock = new SystemClock) extends Logging {
-
-  private val executorFailuresValidityInterval =
-    sparkConf.get(config.EXECUTOR_ATTEMPT_FAILURE_VALIDITY_INTERVAL_MS).getOrElse(-1L)
-
-  // Queue to store the timestamp of failed executors for each host
-  private val failedExecutorsTimeStampsPerHost = mutable.Map[String, mutable.Queue[Long]]()
-
-  private val failedExecutorsTimeStamps = new mutable.Queue[Long]()
-
-  private def updateAndCountFailures(failedExecutorsWithTimeStamps: mutable.Queue[Long]): Int = {
-    val endTime = clock.getTimeMillis()
-    while (executorFailuresValidityInterval > 0 &&
-        failedExecutorsWithTimeStamps.nonEmpty &&
-        failedExecutorsWithTimeStamps.head < endTime - executorFailuresValidityInterval) {
-      failedExecutorsWithTimeStamps.dequeue()
-    }
-    failedExecutorsWithTimeStamps.size
-  }
-
-  def numFailedExecutors: Int = synchronized {
-    updateAndCountFailures(failedExecutorsTimeStamps)
-  }
-
-  def registerFailureOnHost(hostname: String): Unit = synchronized {
-    val timeMillis = clock.getTimeMillis()
-    failedExecutorsTimeStamps.enqueue(timeMillis)
-    val failedExecutorsOnHost =
-      failedExecutorsTimeStampsPerHost.getOrElse(hostname, {
-        val failureOnHost = mutable.Queue[Long]()
-        failedExecutorsTimeStampsPerHost.put(hostname, failureOnHost)
-        failureOnHost
-      })
-    failedExecutorsOnHost.enqueue(timeMillis)
-  }
-
-  def registerExecutorFailure(): Unit = synchronized {
-    val timeMillis = clock.getTimeMillis()
-    failedExecutorsTimeStamps.enqueue(timeMillis)
-  }
-
-  def numFailuresOnHost(hostname: String): Int = {
-    failedExecutorsTimeStampsPerHost.get(hostname).map { failedExecutorsOnHost =>
-      updateAndCountFailures(failedExecutorsOnHost)
-    }.getOrElse(0)
-  }
-
-}
-
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocatorNodeHealthTracker.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocatorNodeHealthTracker.scala
new file mode 100644
index 0000000000000..de9e190361428
--- /dev/null
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocatorNodeHealthTracker.scala
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.deploy.yarn
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+import scala.collection.mutable.HashMap
+
+import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest
+
+import org.apache.spark.SparkConf
+import org.apache.spark.deploy.yarn.config._
+import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config._
+import org.apache.spark.scheduler.HealthTracker
+import org.apache.spark.util.{Clock, SystemClock}
+
+/**
+ * YarnAllocatorNodeHealthTracker is responsible for tracking the health of nodes
+ * and synchronizing the node list to YARN as to which nodes are excluded.
+ *
+ * Excluding nodes are coming from two different sources:
+ *
+ * <ul>
+ *   <li> from the scheduler as task level excluded nodes
+ *   <li> from this class (tracked here) as YARN resource allocation problems
+ * </ul>
+ *
+ * The reason to realize this logic here (and not in the driver) is to avoid possible delays
+ * between synchronizing the excluded nodes with YARN and resource allocations.
+ */
+private[spark] class YarnAllocatorNodeHealthTracker(
+    sparkConf: SparkConf,
+    amClient: AMRMClient[ContainerRequest],
+    failureTracker: FailureTracker)
+  extends Logging {
+
+  private val excludeOnFailureTimeoutMillis = HealthTracker.getExludeOnFailureTimeout(sparkConf)
+
+  private val launchExcludeOnFailureEnabled =
+    sparkConf.get(YARN_EXECUTOR_LAUNCH_EXCLUDE_ON_FAILURE_ENABLED)
+
+  private val maxFailuresPerHost = sparkConf.get(MAX_FAILED_EXEC_PER_NODE)
+
+  private val excludeNodes = sparkConf.get(YARN_EXCLUDE_NODES).toSet
+
+  private val allocatorExcludedNodeList = new HashMap[String, Long]()
+
+  private var currentExcludededYarnNodes = Set.empty[String]
+
+  private var schedulerExcludedNodeList = Set.empty[String]
+
+  private var numClusterNodes = Int.MaxValue
+
+  def setNumClusterNodes(numClusterNodes: Int): Unit = {
+    this.numClusterNodes = numClusterNodes
+  }
+
+  def handleResourceAllocationFailure(hostOpt: Option[String]): Unit = {
+    hostOpt match {
+      case Some(hostname) if launchExcludeOnFailureEnabled =>
+        // failures on an already excluded node are not even tracked.
+        // otherwise, such failures could shutdown the application
+        // as resource requests are asynchronous
+        // and a late failure response could exceed MAX_EXECUTOR_FAILURES
+        if (!schedulerExcludedNodeList.contains(hostname) &&
+            !allocatorExcludedNodeList.contains(hostname)) {
+          failureTracker.registerFailureOnHost(hostname)
+          updateAllocationExcludedNodes(hostname)
+        }
+      case _ =>
+        failureTracker.registerExecutorFailure()
+    }
+  }
+
+  private def updateAllocationExcludedNodes(hostname: String): Unit = {
+    val failuresOnHost = failureTracker.numFailuresOnHost(hostname)
+    if (failuresOnHost > maxFailuresPerHost) {
+      logInfo(s"excluding $hostname as YARN allocation failed $failuresOnHost times")
+      allocatorExcludedNodeList.put(
+        hostname,
+        failureTracker.clock.getTimeMillis() + excludeOnFailureTimeoutMillis)
+      refreshExcludedNodes()
+    }
+  }
+
+  def setSchedulerExcludedNodes(schedulerExcludedNodesWithExpiry: Set[String]): Unit = {
+    this.schedulerExcludedNodeList = schedulerExcludedNodesWithExpiry
+    refreshExcludedNodes()
+  }
+
+  def isAllNodeExcluded: Boolean = {
+    if (numClusterNodes <= 0) {
+      logWarning("No available nodes reported, please check Resource Manager.")
+      false
+    } else {
+      currentExcludededYarnNodes.size >= numClusterNodes
+    }
+  }
+
+  private def refreshExcludedNodes(): Unit = {
+    removeExpiredYarnExcludedNodes()
+    val allExcludedNodes =
+      excludeNodes ++ schedulerExcludedNodeList ++ allocatorExcludedNodeList.keySet
+    synchronizeExcludedNodesWithYarn(allExcludedNodes)
+  }
+
+  private def synchronizeExcludedNodesWithYarn(nodesToExclude: Set[String]): Unit = {
+    // Update YARN with the nodes that are excluded for this application,
+    // in order to avoid allocating new Containers on the problematic nodes.
+    val additions = (nodesToExclude -- currentExcludededYarnNodes).toList.sorted
+    val removals = (currentExcludededYarnNodes -- nodesToExclude).toList.sorted
+    if (additions.nonEmpty) {
+      logInfo(s"adding nodes to YARN application master's excluded node list: $additions")
+    }
+    if (removals.nonEmpty) {
+      logInfo(s"removing nodes from YARN application master's excluded node list: $removals")
+    }
+    if (additions.nonEmpty || removals.nonEmpty) {
+      // Note YARNs api for excluding nodes is updateBlacklist.
+      // TODO - We need to update once Hadoop changes -
+      // https://issues.apache.org/jira/browse/HADOOP-17169
+      amClient.updateBlacklist(additions.asJava, removals.asJava)
+    }
+    currentExcludededYarnNodes = nodesToExclude
+  }
+
+  private def removeExpiredYarnExcludedNodes(): Unit = {
+    val now = failureTracker.clock.getTimeMillis()
+    allocatorExcludedNodeList.retain { (_, expiryTime) => expiryTime > now }
+  }
+}
+
+/**
+ * FailureTracker is responsible for tracking executor failures both for each host separately
+ * and for all hosts altogether.
+ */
+private[spark] class FailureTracker(
+    sparkConf: SparkConf,
+    val clock: Clock = new SystemClock) extends Logging {
+
+  private val executorFailuresValidityInterval =
+    sparkConf.get(config.EXECUTOR_ATTEMPT_FAILURE_VALIDITY_INTERVAL_MS).getOrElse(-1L)
+
+  // Queue to store the timestamp of failed executors for each host
+  private val failedExecutorsTimeStampsPerHost = mutable.Map[String, mutable.Queue[Long]]()
+
+  private val failedExecutorsTimeStamps = new mutable.Queue[Long]()
+
+  private def updateAndCountFailures(failedExecutorsWithTimeStamps: mutable.Queue[Long]): Int = {
+    val endTime = clock.getTimeMillis()
+    while (executorFailuresValidityInterval > 0 &&
+        failedExecutorsWithTimeStamps.nonEmpty &&
+        failedExecutorsWithTimeStamps.head < endTime - executorFailuresValidityInterval) {
+      failedExecutorsWithTimeStamps.dequeue()
+    }
+    failedExecutorsWithTimeStamps.size
+  }
+
+  def numFailedExecutors: Int = synchronized {
+    updateAndCountFailures(failedExecutorsTimeStamps)
+  }
+
+  def registerFailureOnHost(hostname: String): Unit = synchronized {
+    val timeMillis = clock.getTimeMillis()
+    failedExecutorsTimeStamps.enqueue(timeMillis)
+    val failedExecutorsOnHost =
+      failedExecutorsTimeStampsPerHost.getOrElse(hostname, {
+        val failureOnHost = mutable.Queue[Long]()
+        failedExecutorsTimeStampsPerHost.put(hostname, failureOnHost)
+        failureOnHost
+      })
+    failedExecutorsOnHost.enqueue(timeMillis)
+  }
+
+  def registerExecutorFailure(): Unit = synchronized {
+    val timeMillis = clock.getTimeMillis()
+    failedExecutorsTimeStamps.enqueue(timeMillis)
+  }
+
+  def numFailuresOnHost(hostname: String): Int = {
+    failedExecutorsTimeStampsPerHost.get(hostname).map { failedExecutorsOnHost =>
+      updateAndCountFailures(failedExecutorsOnHost)
+    }.getOrElse(0)
+  }
+
+}
+
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClient.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClient.scala
index 7c67493c33160..2f272be60ba25 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClient.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClient.scala
@@ -30,6 +30,7 @@ import org.apache.spark.{SecurityManager, SparkConf}
 import org.apache.spark.deploy.yarn.config._
 import org.apache.spark.internal.Logging
 import org.apache.spark.rpc.RpcEndpointRef
+import org.apache.spark.util.Utils
 
 /**
  * Handles registering and unregistering the application with the YARN ResourceManager.
@@ -107,7 +108,7 @@ private[spark] class YarnRMClient extends Logging {
     // so not all stable releases have it.
     val prefix = WebAppUtils.getHttpSchemePrefix(conf)
     val proxies = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf)
-    val hosts = proxies.asScala.map(_.split(":").head)
+    val hosts = proxies.asScala.map(proxy => Utils.parseHostPort(proxy)._1)
     val uriBases = proxies.asScala.map { proxy => prefix + proxy + proxyBase }
     val params =
       Map("PROXY_HOSTS" -> hosts.mkString(","), "PROXY_URI_BASES" -> uriBases.mkString(","))
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnSparkHadoopUtil.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnSparkHadoopUtil.scala
index 9cefc4011c930..09766bf97d8f3 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnSparkHadoopUtil.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnSparkHadoopUtil.scala
@@ -26,8 +26,8 @@ import org.apache.hadoop.yarn.api.records.{ApplicationAccessType, ContainerId, P
 import org.apache.hadoop.yarn.util.ConverterUtils
 
 import org.apache.spark.{SecurityManager, SparkConf}
-import org.apache.spark.internal.config._
 import org.apache.spark.launcher.YarnCommandBuilderUtils
+import org.apache.spark.resource.ExecutorResourceRequest
 import org.apache.spark.util.Utils
 
 object YarnSparkHadoopUtil {
@@ -37,7 +37,6 @@ object YarnSparkHadoopUtil {
   // the common cases. Memory overhead tends to grow with container size.
 
   val MEMORY_OVERHEAD_FACTOR = 0.10
-  val MEMORY_OVERHEAD_MIN = 384L
 
   val ANY_HOST = "*"
 
@@ -107,7 +106,7 @@ object YarnSparkHadoopUtil {
    * Not killing the task leaves various aspects of the executor and (to some extent) the jvm in
    * an inconsistent state.
    * TODO: If the OOM is not recoverable by rescheduling it on different node, then do
-   * 'something' to fail job ... akin to blacklisting trackers in mapred ?
+   * 'something' to fail job ... akin to unhealthy trackers in mapred ?
    *
    * The handler if an OOM Exception is thrown by the JVM must be configured on Windows
    * differently: the 'taskkill' command should be used, whereas Unix-based systems use 'kill'.
@@ -184,16 +183,11 @@ object YarnSparkHadoopUtil {
   }
 
   /**
-   * Convert MEMORY_OFFHEAP_SIZE to MB Unit, return 0 if MEMORY_OFFHEAP_ENABLED is false.
+   * Get offHeap memory size from [[ExecutorResourceRequest]]
+   * return 0 if MEMORY_OFFHEAP_ENABLED is false.
    */
-  def executorOffHeapMemorySizeAsMb(sparkConf: SparkConf): Int = {
-    if (sparkConf.get(MEMORY_OFFHEAP_ENABLED)) {
-      val sizeInMB = Utils.memoryStringToMb(sparkConf.get(MEMORY_OFFHEAP_SIZE).toString)
-      require(sizeInMB > 0,
-        s"${MEMORY_OFFHEAP_SIZE.key} must be > 0 when ${MEMORY_OFFHEAP_ENABLED.key} == true")
-      sizeInMB
-    } else {
-      0
-    }
+  def executorOffHeapMemorySizeAsMb(sparkConf: SparkConf,
+    execRequest: ExecutorResourceRequest): Long = {
+    Utils.checkOffHeapEnabled(sparkConf, execRequest.amount)
   }
 }
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala
index b4257a48664c6..89a4af2d2a741 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala
@@ -17,12 +17,14 @@
 
 package org.apache.spark.deploy.yarn
 
+import java.util.Properties
 import java.util.concurrent.TimeUnit
 
+import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.ConfigBuilder
 import org.apache.spark.network.util.ByteUnit
 
-package object config {
+package object config extends Logging {
 
   /* Common app configuration. */
 
@@ -74,10 +76,11 @@ package object config {
     .doc("Whether to populate Hadoop classpath from `yarn.application.classpath` and " +
       "`mapreduce.application.classpath` Note that if this is set to `false`, it requires " +
       "a `with-Hadoop` Spark distribution that bundles Hadoop runtime or user has to provide " +
-      "a Hadoop installation separately.")
+      "a Hadoop installation separately. By default, for `with-hadoop` Spark distribution, " +
+      "this is set to `false`; for `no-hadoop` distribution, this is set to `true`.")
     .version("2.4.6")
     .booleanConf
-    .createWithDefault(true)
+    .createWithDefault(isHadoopProvided())
 
   private[spark] val GATEWAY_ROOT_PATH = ConfigBuilder("spark.yarn.config.gatewayPath")
     .doc("Root of configuration paths that is present on gateway nodes, and will be replaced " +
@@ -186,6 +189,15 @@ package object config {
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("1s")
 
+  private[spark] val CLIENT_INCLUDE_DRIVER_LOGS_LINK =
+    ConfigBuilder("spark.yarn.includeDriverLogsLink")
+      .doc("In cluster mode, whether the client application report includes links to the driver "
+          + "container's logs. This requires polling the ResourceManager's REST API, so it "
+          + "places some additional load on the RM.")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(false)
+
   /* Shared Client-mode AM / Driver configuration. */
 
   private[spark] val AM_MAX_WAIT_TIME = ConfigBuilder("spark.yarn.am.waitTime")
@@ -376,14 +388,15 @@ package object config {
     .stringConf
     .createOptional
 
-  /* YARN allocator-level blacklisting related config entries. */
-  private[spark] val YARN_EXECUTOR_LAUNCH_BLACKLIST_ENABLED =
-    ConfigBuilder("spark.yarn.blacklist.executor.launch.blacklisting.enabled")
-      .version("2.4.0")
+  /* YARN allocator-level excludeOnFailure related config entries. */
+  private[spark] val YARN_EXECUTOR_LAUNCH_EXCLUDE_ON_FAILURE_ENABLED =
+    ConfigBuilder("spark.yarn.executor.launch.excludeOnFailure.enabled")
+      .version("3.1.0")
+      .withAlternative("spark.yarn.blacklist.executor.launch.blacklisting.enabled")
       .booleanConf
       .createWithDefault(false)
 
-  /* Initially blacklisted YARN nodes. */
+  /* Initially excluded YARN nodes. */
   private[spark] val YARN_EXCLUDE_NODES = ConfigBuilder("spark.yarn.exclude.nodes")
     .version("3.0.0")
     .stringConf
@@ -394,4 +407,20 @@ package object config {
   private[yarn] val YARN_DRIVER_RESOURCE_TYPES_PREFIX = "spark.yarn.driver.resource."
   private[yarn] val YARN_AM_RESOURCE_TYPES_PREFIX = "spark.yarn.am.resource."
 
+  def isHadoopProvided(): Boolean = IS_HADOOP_PROVIDED
+
+  private lazy val IS_HADOOP_PROVIDED: Boolean = {
+    val configPath = "org/apache/spark/deploy/yarn/config.properties"
+    val propertyKey = "spark.yarn.isHadoopProvided"
+    try {
+      val prop = new Properties()
+      prop.load(ClassLoader.getSystemClassLoader.getResourceAsStream(configPath))
+      prop.getProperty(propertyKey).toBoolean
+    } catch {
+      case e: Exception =>
+        log.warn(s"Can not load the default value of `$propertyKey` from " +
+          s"`$configPath` with error, ${e.toString}. Using `false` as a default value.")
+        false
+    }
+  }
 }
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/executor/YarnCoarseGrainedExecutorBackend.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/executor/YarnCoarseGrainedExecutorBackend.scala
index 669e39fb7c1c7..ce46ffa06f0fe 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/executor/YarnCoarseGrainedExecutorBackend.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/executor/YarnCoarseGrainedExecutorBackend.scala
@@ -73,8 +73,8 @@ private[spark] object YarnCoarseGrainedExecutorBackend extends Logging {
     val createFn: (RpcEnv, CoarseGrainedExecutorBackend.Arguments, SparkEnv, ResourceProfile) =>
       CoarseGrainedExecutorBackend = { case (rpcEnv, arguments, env, resourceProfile) =>
       new YarnCoarseGrainedExecutorBackend(rpcEnv, arguments.driverUrl, arguments.executorId,
-        arguments.bindAddress, arguments.hostname, arguments.cores, arguments.userClassPath, env,
-        arguments.resourcesFileOpt, resourceProfile)
+        arguments.bindAddress, arguments.hostname, arguments.cores, arguments.userClassPath.toSeq,
+        env, arguments.resourcesFileOpt, resourceProfile)
     }
     val backendArgs = CoarseGrainedExecutorBackend.parseArguments(args,
       this.getClass.getCanonicalName.stripSuffix("$"))
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/launcher/YarnCommandBuilderUtils.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/launcher/YarnCommandBuilderUtils.scala
index 0c3d080cca254..d000287cb7a96 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/launcher/YarnCommandBuilderUtils.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/launcher/YarnCommandBuilderUtils.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.launcher
 
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ListBuffer
 import scala.util.Properties
 
 /**
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala
index cb0de5a0d50b4..8a55e612ce719 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.scheduler.cluster
 
+import java.io.InterruptedIOException
+
 import scala.collection.mutable.ArrayBuffer
 
 import org.apache.hadoop.yarn.api.records.YarnApplicationState
@@ -121,7 +123,8 @@ private[spark] class YarnClientSchedulerBackend(
         allowInterrupt = false
         sc.stop()
       } catch {
-        case e: InterruptedException => logInfo("Interrupting monitor thread")
+        case _: InterruptedException | _: InterruptedIOException =>
+          logInfo("Interrupting monitor thread")
       }
     }
 
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackend.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackend.scala
index e428bab4f96f3..c5c4594170c91 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackend.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackend.scala
@@ -18,11 +18,10 @@
 package org.apache.spark.scheduler.cluster
 
 import java.util.EnumSet
-import java.util.concurrent.atomic.{AtomicBoolean}
+import java.util.concurrent.atomic.AtomicBoolean
 import javax.servlet.DispatcherType
 
-import scala.concurrent.ExecutionContext.Implicits.global
-import scala.concurrent.Future
+import scala.concurrent.{ExecutionContext, Future}
 import scala.util.{Failure, Success}
 import scala.util.control.NonFatal
 
@@ -30,14 +29,14 @@ import org.apache.hadoop.yarn.api.records.{ApplicationAttemptId, ApplicationId}
 
 import org.apache.spark.SparkContext
 import org.apache.spark.deploy.security.HadoopDelegationTokenManager
-import org.apache.spark.internal.Logging
-import org.apache.spark.internal.config
+import org.apache.spark.internal.{config, Logging}
 import org.apache.spark.internal.config.UI._
 import org.apache.spark.resource.ResourceProfile
 import org.apache.spark.rpc._
 import org.apache.spark.scheduler._
 import org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages._
-import org.apache.spark.util.{RpcUtils, ThreadUtils}
+import org.apache.spark.storage.{BlockManagerId, BlockManagerMaster}
+import org.apache.spark.util.{RpcUtils, ThreadUtils, Utils}
 
 /**
  * Abstract Yarn scheduler backend that contains common logic
@@ -67,12 +66,32 @@ private[spark] abstract class YarnSchedulerBackend(
 
   private implicit val askTimeout = RpcUtils.askRpcTimeout(sc.conf)
 
+  /**
+   * Declare implicit single thread execution context for futures doRequestTotalExecutors and
+   * doKillExecutors below, avoiding using the global execution context that may cause conflict
+   * with user code's execution of futures.
+   */
+  private implicit val schedulerEndpointEC = ExecutionContext.fromExecutorService(
+      ThreadUtils.newDaemonSingleThreadExecutor("yarn-scheduler-endpoint"))
+
   /** Application ID. */
   protected var appId: Option[ApplicationId] = None
 
   /** Attempt ID. This is unset for client-mode schedulers */
   private var attemptId: Option[ApplicationAttemptId] = None
 
+  private val blockManagerMaster: BlockManagerMaster = sc.env.blockManager.master
+
+  private val minMergersThresholdRatio =
+    conf.get(config.SHUFFLE_MERGER_LOCATIONS_MIN_THRESHOLD_RATIO)
+
+  private val minMergersStaticThreshold =
+    conf.get(config.SHUFFLE_MERGER_LOCATIONS_MIN_STATIC_THRESHOLD)
+
+  private val maxNumExecutors = conf.get(config.DYN_ALLOCATION_MAX_EXECUTORS)
+
+  private val numExecutors = conf.get(config.EXECUTOR_INSTANCES).getOrElse(0)
+
   /**
    * Bind to YARN. This *must* be done before calling [[start()]].
    *
@@ -125,13 +144,13 @@ private[spark] abstract class YarnSchedulerBackend(
 
   private[cluster] def prepareRequestExecutors(
       resourceProfileToTotalExecs: Map[ResourceProfile, Int]): RequestExecutors = {
-    val nodeBlacklist: Set[String] = scheduler.nodeBlacklist()
-    // For locality preferences, ignore preferences for nodes that are blacklisted
+    val excludedNodes: Set[String] = scheduler.excludedNodes()
+    // For locality preferences, ignore preferences for nodes that are excluded
     val filteredRPHostToLocalTaskCount = rpHostToLocalTaskCount.map { case (rpid, v) =>
-      (rpid, v.filter { case (host, count) => !nodeBlacklist.contains(host) })
+      (rpid, v.filter { case (host, count) => !excludedNodes.contains(host) })
     }
     RequestExecutors(resourceProfileToTotalExecs, numLocalityAwareTasksPerResourceProfileId,
-      filteredRPHostToLocalTaskCount, nodeBlacklist)
+      filteredRPHostToLocalTaskCount, excludedNodes)
   }
 
   /**
@@ -154,6 +173,36 @@ private[spark] abstract class YarnSchedulerBackend(
     totalRegisteredExecutors.get() >= totalExpectedExecutors * minRegisteredRatio
   }
 
+  override def getShufflePushMergerLocations(
+      numPartitions: Int,
+      resourceProfileId: Int): Seq[BlockManagerId] = {
+    // TODO (SPARK-33481) This is a naive way of calculating numMergersDesired for a stage,
+    // TODO we can use better heuristics to calculate numMergersDesired for a stage.
+    val maxExecutors = if (Utils.isDynamicAllocationEnabled(sc.getConf)) {
+      maxNumExecutors
+    } else {
+      numExecutors
+    }
+    val tasksPerExecutor = sc.resourceProfileManager
+      .resourceProfileFromId(resourceProfileId).maxTasksPerExecutor(sc.conf)
+    val numMergersDesired = math.min(
+      math.max(1, math.ceil(numPartitions / tasksPerExecutor).toInt), maxExecutors)
+    val minMergersNeeded = math.max(minMergersStaticThreshold,
+      math.floor(numMergersDesired * minMergersThresholdRatio).toInt)
+
+    // Request for numMergersDesired shuffle mergers to BlockManagerMasterEndpoint
+    // and if it's less than minMergersNeeded, we disable push based shuffle.
+    val mergerLocations = blockManagerMaster
+      .getShufflePushMergerLocations(numMergersDesired, scheduler.excludedNodes())
+    if (mergerLocations.size < numMergersDesired && mergerLocations.size < minMergersNeeded) {
+      Seq.empty[BlockManagerId]
+    } else {
+      logDebug(s"The number of shuffle mergers desired ${numMergersDesired}" +
+        s" and available locations are ${mergerLocations.length}")
+      mergerLocations
+    }
+  }
+
   /**
    * Add filters to the SparkUI.
    */
@@ -257,13 +306,14 @@ private[spark] abstract class YarnSchedulerBackend(
               case NonFatal(e) =>
                 logWarning(s"Attempted to get executor loss reason" +
                   s" for executor id ${executorId} at RPC address ${executorRpcAddress}," +
-                  s" but got no response. Marking as slave lost.", e)
-                RemoveExecutor(executorId, SlaveLost())
+                  s" but got no response. Marking as agent lost.", e)
+                RemoveExecutor(executorId, ExecutorProcessLost())
             }(ThreadUtils.sameThread)
         case None =>
           logWarning("Attempted to check for an executor loss reason" +
             " before the AM has registered!")
-          Future.successful(RemoveExecutor(executorId, SlaveLost("AM is not yet registered.")))
+          Future.successful(RemoveExecutor(executorId,
+            ExecutorProcessLost("AM is not yet registered.")))
       }
 
       removeExecutorMessage.foreach { message => driverEndpoint.send(message) }
@@ -283,6 +333,12 @@ private[spark] abstract class YarnSchedulerBackend(
           logWarning(s"Requesting driver to remove executor $executorId for reason $reason")
           driverEndpoint.send(r)
         }
+
+      // In case of yarn Miscellaneous Process is Spark AM Container
+      // Launched for the deploy mode client
+      case processInfo @ MiscellaneousProcessAdded(_, _, _) =>
+        logDebug(s"Sending the Spark AM info for yarn client mode")
+        driverEndpoint.send(processInfo)
     }
 
     override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/util/YarnContainerInfoHelper.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/util/YarnContainerInfoHelper.scala
index 5e39422e868b7..854fe18c22430 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/util/YarnContainerInfoHelper.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/util/YarnContainerInfoHelper.scala
@@ -28,6 +28,16 @@ import org.apache.spark.deploy.yarn.YarnSparkHadoopUtil
 import org.apache.spark.internal.Logging
 
 private[spark] object YarnContainerInfoHelper extends Logging {
+
+  private[this] val DRIVER_LOG_FILE_NAMES = Seq("stdout", "stderr")
+  private[this] val DRIVER_LOG_START_OFFSET = -4096
+
+  def getLogUrlsFromBaseUrl(baseUrl: String): Map[String, String] = {
+    DRIVER_LOG_FILE_NAMES.map { fname =>
+      fname -> s"$baseUrl/$fname?start=$DRIVER_LOG_START_OFFSET"
+    }.toMap
+  }
+
   def getLogUrls(
       conf: Configuration,
       container: Option[Container]): Option[Map[String, String]] = {
@@ -42,9 +52,7 @@ private[spark] object YarnContainerInfoHelper extends Logging {
       val baseUrl = s"$httpScheme$httpAddress/node/containerlogs/$containerId/$user"
       logDebug(s"Base URL for logs: $baseUrl")
 
-      Some(Map(
-        "stdout" -> s"$baseUrl/stdout?start=-4096",
-        "stderr" -> s"$baseUrl/stderr?start=-4096"))
+      Some(getLogUrlsFromBaseUrl(baseUrl))
     } catch {
       case e: Exception =>
         logInfo("Error while building executor logs - executor logs will not be available", e)
diff --git a/resource-managers/yarn/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java b/resource-managers/yarn/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java
index df0ebcc9871ac..89e012ecd42e1 100644
--- a/resource-managers/yarn/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java
+++ b/resource-managers/yarn/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java
@@ -112,7 +112,7 @@ public static int waitForPort(int port, int retries)
    * The ports are all closed afterwards,
    * so other network services started may grab those same ports.
    *
-   * @param numPorts number of required port nubmers
+   * @param numPorts number of required port numbers
    * @return array of available port numbers
    * @throws IOException
    */
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/BaseYarnClusterSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/BaseYarnClusterSuite.scala
index f8ef0d08d829c..4189423d1ccc7 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/BaseYarnClusterSuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/BaseYarnClusterSuite.scala
@@ -28,8 +28,10 @@ import scala.concurrent.duration._
 import com.google.common.io.Files
 import org.apache.hadoop.yarn.conf.YarnConfiguration
 import org.apache.hadoop.yarn.server.MiniYARNCluster
-import org.scalatest.{BeforeAndAfterAll, Matchers}
+import org.scalactic.source.Position
+import org.scalatest.{BeforeAndAfterAll, Tag}
 import org.scalatest.concurrent.Eventually._
+import org.scalatest.matchers.must.Matchers
 
 import org.apache.spark._
 import org.apache.spark.deploy.yarn.config._
@@ -40,6 +42,7 @@ import org.apache.spark.util.Utils
 
 abstract class BaseYarnClusterSuite
   extends SparkFunSuite with BeforeAndAfterAll with Matchers with Logging {
+  private var isBindSuccessful = true
 
   // log4j configuration for the YARN containers, so that their output is collected
   // by YARN instead of trying to overwrite unit-tests.log.
@@ -63,6 +66,14 @@ abstract class BaseYarnClusterSuite
 
   def newYarnConfig(): YarnConfiguration
 
+  override protected def test(testName: String, testTags: Tag*)(testFun: => Any)
+                             (implicit pos: Position): Unit = {
+    super.test(testName, testTags: _*) {
+      assume(isBindSuccessful, "Mini Yarn cluster should be able to bind.")
+      testFun
+    }
+  }
+
   override def beforeAll(): Unit = {
     super.beforeAll()
 
@@ -79,9 +90,26 @@ abstract class BaseYarnClusterSuite
     yarnConf.set("yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage",
       "100.0")
 
-    yarnCluster = new MiniYARNCluster(getClass().getName(), 1, 1, 1)
-    yarnCluster.init(yarnConf)
-    yarnCluster.start()
+    // capacity-scheduler.xml is missing in hadoop-client-minicluster so this is a workaround
+    yarnConf.set("yarn.scheduler.capacity.root.queues", "default")
+    yarnConf.setInt("yarn.scheduler.capacity.root.default.capacity", 100)
+    yarnConf.setFloat("yarn.scheduler.capacity.root.default.user-limit-factor", 1)
+    yarnConf.setInt("yarn.scheduler.capacity.root.default.maximum-capacity", 100)
+    yarnConf.set("yarn.scheduler.capacity.root.default.state", "RUNNING")
+    yarnConf.set("yarn.scheduler.capacity.root.default.acl_submit_applications", "*")
+    yarnConf.set("yarn.scheduler.capacity.root.default.acl_administer_queue", "*")
+    yarnConf.setInt("yarn.scheduler.capacity.node-locality-delay", -1)
+
+    try {
+      yarnCluster = new MiniYARNCluster(getClass().getName(), 1, 1, 1)
+      yarnCluster.init(yarnConf)
+      yarnCluster.start()
+    } catch {
+      case e: Throwable if org.apache.commons.lang3.exception.ExceptionUtils.indexOfThrowable(
+          e, classOf[java.net.BindException]) != -1 =>
+        isBindSuccessful = false
+        return
+    }
 
     // There's a race in MiniYARNCluster in which start() may return before the RM has updated
     // its address in the configuration. You can see this in the logs by noticing that when
@@ -117,7 +145,7 @@ abstract class BaseYarnClusterSuite
 
   override def afterAll(): Unit = {
     try {
-      yarnCluster.stop()
+      if (yarnCluster != null) yarnCluster.stop()
     } finally {
       super.afterAll()
     }
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
index 9c5c376ce5357..ea3acec3bb78b 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
@@ -43,7 +43,8 @@ import org.apache.hadoop.yarn.util.Records
 import org.mockito.ArgumentMatchers.{any, anyBoolean, anyShort, eq => meq}
 import org.mockito.Mockito._
 import org.mockito.invocation.InvocationOnMock
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.{SparkConf, SparkException, SparkFunSuite, TestUtils}
 import org.apache.spark.deploy.yarn.ResourceRequestHelper._
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ContainerPlacementStrategySuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ContainerPlacementStrategySuite.scala
index d83a0d2efec1f..d64aad9ad470b 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ContainerPlacementStrategySuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ContainerPlacementStrategySuite.scala
@@ -19,9 +19,9 @@ package org.apache.spark.deploy.yarn
 
 import org.apache.hadoop.yarn.api.records._
 import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest
-import org.scalatest.{BeforeAndAfterEach, Matchers}
+import org.scalatest.BeforeAndAfterEach
+import org.scalatest.matchers.must.Matchers
 
-import org.apache.spark.SparkConf
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.resource.ResourceProfile
 
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/FailureTrackerSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/FailureTrackerSuite.scala
index 4f77b9c99dd25..bfefdbc903b0c 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/FailureTrackerSuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/FailureTrackerSuite.scala
@@ -16,7 +16,8 @@
  */
 package org.apache.spark.deploy.yarn
 
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.util.ManualClock
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala
index 3c9209c292418..465de4860a627 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala
@@ -33,7 +33,7 @@ class LocalityPlacementStrategySuite extends SparkFunSuite {
   test("handle large number of containers and tasks (SPARK-18750)") {
     // Run the test in a thread with a small stack size, since the original issue
     // surfaced as a StackOverflowError.
-    var error: Throwable = null
+    @volatile var error: Throwable = null
 
     val runnable = new Runnable() {
       override def run(): Unit = try {
@@ -43,14 +43,22 @@ class LocalityPlacementStrategySuite extends SparkFunSuite {
       }
     }
 
-    val thread = new Thread(new ThreadGroup("test"), runnable, "test-thread", 32 * 1024)
+    val thread = new Thread(new ThreadGroup("test"), runnable, "test-thread", 256 * 1024)
+    thread.setDaemon(true)
     thread.start()
-    thread.join()
+    val secondsToWait = 30
+    thread.join(secondsToWait * 1000)
+    if (thread.isAlive()) {
+      error = new RuntimeException(
+        "Timeout at waiting for thread to stop (its stack trace is added to the exception)")
+      error.setStackTrace(thread.getStackTrace)
+      thread.interrupt()
+    }
 
     if (error != null) {
       val errors = new StringWriter()
       error.printStackTrace(new PrintWriter(errors))
-      fail(s"StackOverflowError should not be thrown; however, got:\n\n$errors")
+      fail(s"Failed with an exception or a timeout at thread join:\n\n$errors")
     }
   }
 
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ResourceRequestHelperSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ResourceRequestHelperSuite.scala
index f5ec531e26e0c..3e54dd0034307 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ResourceRequestHelperSuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ResourceRequestHelperSuite.scala
@@ -19,7 +19,8 @@ package org.apache.spark.deploy.yarn
 
 import org.apache.hadoop.yarn.api.records.Resource
 import org.apache.hadoop.yarn.util.Records
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.{SparkConf, SparkException, SparkFunSuite}
 import org.apache.spark.deploy.yarn.ResourceRequestHelper._
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorBlacklistTrackerSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorBlacklistTrackerSuite.scala
deleted file mode 100644
index c07a4ac76b989..0000000000000
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorBlacklistTrackerSuite.scala
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.deploy.yarn
-
-import java.util.Arrays
-import java.util.Collections
-
-import org.apache.hadoop.yarn.client.api.AMRMClient
-import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest
-import org.mockito.Mockito._
-import org.scalatest.{BeforeAndAfterEach, Matchers}
-
-import org.apache.spark.{SparkConf, SparkFunSuite}
-import org.apache.spark.deploy.yarn.config.{YARN_EXCLUDE_NODES, YARN_EXECUTOR_LAUNCH_BLACKLIST_ENABLED}
-import org.apache.spark.internal.config.{BLACKLIST_TIMEOUT_CONF, MAX_FAILED_EXEC_PER_NODE}
-import org.apache.spark.util.ManualClock
-
-class YarnAllocatorBlacklistTrackerSuite extends SparkFunSuite with Matchers
-  with BeforeAndAfterEach {
-
-  val BLACKLIST_TIMEOUT = 100L
-  val MAX_FAILED_EXEC_PER_NODE_VALUE = 2
-
-  var sparkConf: SparkConf = _
-  var amClientMock: AMRMClient[ContainerRequest] = _
-  var clock: ManualClock = _
-
-  override def beforeEach(): Unit = {
-    sparkConf = new SparkConf()
-    sparkConf.set(BLACKLIST_TIMEOUT_CONF, BLACKLIST_TIMEOUT)
-    sparkConf.set(YARN_EXECUTOR_LAUNCH_BLACKLIST_ENABLED, true)
-    sparkConf.set(MAX_FAILED_EXEC_PER_NODE, MAX_FAILED_EXEC_PER_NODE_VALUE)
-    clock = new ManualClock()
-    amClientMock = mock(classOf[AMRMClient[ContainerRequest]])
-    super.beforeEach()
-  }
-
-  private def createYarnAllocatorBlacklistTracker(
-      sparkConf: SparkConf = sparkConf): YarnAllocatorBlacklistTracker = {
-    val failureTracker = new FailureTracker(sparkConf, clock)
-    val yarnBlacklistTracker =
-      new YarnAllocatorBlacklistTracker(sparkConf, amClientMock, failureTracker)
-    yarnBlacklistTracker.setNumClusterNodes(4)
-    yarnBlacklistTracker
-  }
-
-  test("expiring its own blacklisted nodes") {
-    val yarnBlacklistTracker = createYarnAllocatorBlacklistTracker()
-    (1 to MAX_FAILED_EXEC_PER_NODE_VALUE).foreach {
-      _ => {
-        yarnBlacklistTracker.handleResourceAllocationFailure(Some("host"))
-        // host should not be blacklisted at these failures as MAX_FAILED_EXEC_PER_NODE is 2
-        verify(amClientMock, never())
-          .updateBlacklist(Arrays.asList("host"), Collections.emptyList())
-      }
-    }
-
-    yarnBlacklistTracker.handleResourceAllocationFailure(Some("host"))
-    // the third failure on the host triggers the blacklisting
-    verify(amClientMock).updateBlacklist(Arrays.asList("host"), Collections.emptyList())
-
-    clock.advance(BLACKLIST_TIMEOUT)
-
-    // trigger synchronisation of blacklisted nodes with YARN
-    yarnBlacklistTracker.setSchedulerBlacklistedNodes(Set())
-    verify(amClientMock).updateBlacklist(Collections.emptyList(), Arrays.asList("host"))
-  }
-
-  test("not handling the expiry of scheduler blacklisted nodes") {
-    val yarnBlacklistTracker = createYarnAllocatorBlacklistTracker()
-
-    yarnBlacklistTracker.setSchedulerBlacklistedNodes(Set("host1", "host2"))
-    verify(amClientMock)
-      .updateBlacklist(Arrays.asList("host1", "host2"), Collections.emptyList())
-
-    // advance timer more then host1, host2 expiry time
-    clock.advance(200L)
-
-    // expired blacklisted nodes (simulating a resource request)
-    yarnBlacklistTracker.setSchedulerBlacklistedNodes(Set("host1", "host2"))
-    // no change is communicated to YARN regarding the blacklisting
-    verify(amClientMock, times(0)).updateBlacklist(Collections.emptyList(), Collections.emptyList())
-  }
-
-  test("combining scheduler and allocation blacklist") {
-    sparkConf.set(YARN_EXCLUDE_NODES, Seq("initial1", "initial2"))
-    val yarnBlacklistTracker = createYarnAllocatorBlacklistTracker(sparkConf)
-    yarnBlacklistTracker.setSchedulerBlacklistedNodes(Set())
-
-    // initial1 and initial2 is added as blacklisted nodes at the very first updateBlacklist call
-    // and they are never removed
-    verify(amClientMock)
-      .updateBlacklist(Arrays.asList("initial1", "initial2"), Collections.emptyList())
-
-    (1 to MAX_FAILED_EXEC_PER_NODE_VALUE).foreach {
-      _ => {
-        yarnBlacklistTracker.handleResourceAllocationFailure(Some("host1"))
-        // host1 should not be blacklisted at these failures as MAX_FAILED_EXEC_PER_NODE is 2
-        verify(amClientMock, never())
-          .updateBlacklist(Arrays.asList("host1"), Collections.emptyList())
-      }
-    }
-
-    // as this is the third failure on host1 the node will be blacklisted
-    yarnBlacklistTracker.handleResourceAllocationFailure(Some("host1"))
-    verify(amClientMock)
-      .updateBlacklist(Arrays.asList("host1"), Collections.emptyList())
-
-    yarnBlacklistTracker.setSchedulerBlacklistedNodes(Set("host2", "host3"))
-    verify(amClientMock)
-      .updateBlacklist(Arrays.asList("host2", "host3"), Collections.emptyList())
-
-    clock.advance(10L)
-
-    yarnBlacklistTracker.setSchedulerBlacklistedNodes(Set("host3", "host4"))
-    verify(amClientMock)
-      .updateBlacklist(Arrays.asList("host4"), Arrays.asList("host2"))
-  }
-
-  test("blacklist all available nodes") {
-    val yarnBlacklistTracker = createYarnAllocatorBlacklistTracker()
-    yarnBlacklistTracker.setSchedulerBlacklistedNodes(Set("host1", "host2", "host3"))
-    verify(amClientMock)
-      .updateBlacklist(Arrays.asList("host1", "host2", "host3"), Collections.emptyList())
-
-    clock.advance(60L)
-    (1 to MAX_FAILED_EXEC_PER_NODE_VALUE).foreach {
-      _ => {
-        yarnBlacklistTracker.handleResourceAllocationFailure(Some("host4"))
-        // host4 should not be blacklisted at these failures as MAX_FAILED_EXEC_PER_NODE is 2
-        verify(amClientMock, never())
-          .updateBlacklist(Arrays.asList("host4"), Collections.emptyList())
-      }
-    }
-
-    // the third failure on the host triggers the blacklisting
-    yarnBlacklistTracker.handleResourceAllocationFailure(Some("host4"))
-
-    verify(amClientMock).updateBlacklist(Arrays.asList("host4"), Collections.emptyList())
-    assert(yarnBlacklistTracker.isAllNodeBlacklisted)
-  }
-
-}
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorHealthTrackerSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorHealthTrackerSuite.scala
new file mode 100644
index 0000000000000..c2fd5ff316592
--- /dev/null
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorHealthTrackerSuite.scala
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.deploy.yarn
+
+import java.util.Arrays
+import java.util.Collections
+
+import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest
+import org.mockito.Mockito._
+import org.scalatest.BeforeAndAfterEach
+import org.scalatest.matchers.must.Matchers
+
+import org.apache.spark.{SparkConf, SparkFunSuite}
+import org.apache.spark.deploy.yarn.config.{YARN_EXCLUDE_NODES, YARN_EXECUTOR_LAUNCH_EXCLUDE_ON_FAILURE_ENABLED}
+import org.apache.spark.internal.config.{EXCLUDE_ON_FAILURE_TIMEOUT_CONF, MAX_FAILED_EXEC_PER_NODE}
+import org.apache.spark.util.ManualClock
+
+class YarnAllocatorHealthTrackerSuite extends SparkFunSuite with Matchers
+  with BeforeAndAfterEach {
+
+  val EXCLUDE_TIMEOUT = 100L
+  val MAX_FAILED_EXEC_PER_NODE_VALUE = 2
+
+  var sparkConf: SparkConf = _
+  var amClientMock: AMRMClient[ContainerRequest] = _
+  var clock: ManualClock = _
+
+  override def beforeEach(): Unit = {
+    sparkConf = new SparkConf()
+    sparkConf.set(EXCLUDE_ON_FAILURE_TIMEOUT_CONF, EXCLUDE_TIMEOUT)
+    sparkConf.set(YARN_EXECUTOR_LAUNCH_EXCLUDE_ON_FAILURE_ENABLED, true)
+    sparkConf.set(MAX_FAILED_EXEC_PER_NODE, MAX_FAILED_EXEC_PER_NODE_VALUE)
+    clock = new ManualClock()
+    amClientMock = mock(classOf[AMRMClient[ContainerRequest]])
+    super.beforeEach()
+  }
+
+  private def createYarnAllocatorHealthTracker(
+      sparkConf: SparkConf = sparkConf): YarnAllocatorNodeHealthTracker = {
+    val failureTracker = new FailureTracker(sparkConf, clock)
+    val yarnHealthTracker =
+      new YarnAllocatorNodeHealthTracker(sparkConf, amClientMock, failureTracker)
+    yarnHealthTracker.setNumClusterNodes(4)
+    yarnHealthTracker
+  }
+
+  test("expiring its own excluded nodes") {
+    val yarnHealthTracker = createYarnAllocatorHealthTracker()
+    (1 to MAX_FAILED_EXEC_PER_NODE_VALUE).foreach {
+      _ => {
+        yarnHealthTracker.handleResourceAllocationFailure(Some("host"))
+        // host should not be excluded at these failures as MAX_FAILED_EXEC_PER_NODE is 2
+        verify(amClientMock, never())
+          .updateBlacklist(Arrays.asList("host"), Collections.emptyList())
+      }
+    }
+
+    yarnHealthTracker.handleResourceAllocationFailure(Some("host"))
+    // the third failure on the host triggers the exclusion
+    verify(amClientMock).updateBlacklist(Arrays.asList("host"), Collections.emptyList())
+
+    clock.advance(EXCLUDE_TIMEOUT)
+
+    // trigger synchronisation of excluded nodes with YARN
+    yarnHealthTracker.setSchedulerExcludedNodes(Set())
+    verify(amClientMock).updateBlacklist(Collections.emptyList(), Arrays.asList("host"))
+  }
+
+  test("not handling the expiry of scheduler excluded nodes") {
+    val yarnHealthTracker = createYarnAllocatorHealthTracker()
+
+    yarnHealthTracker.setSchedulerExcludedNodes(Set("host1", "host2"))
+    verify(amClientMock)
+      .updateBlacklist(Arrays.asList("host1", "host2"), Collections.emptyList())
+
+    // advance timer more then host1, host2 expiry time
+    clock.advance(200L)
+
+    // expired excluded nodes (simulating a resource request)
+    yarnHealthTracker.setSchedulerExcludedNodes(Set("host1", "host2"))
+    // no change is communicated to YARN regarding the exclusion
+    verify(amClientMock, times(0)).updateBlacklist(Collections.emptyList(), Collections.emptyList())
+  }
+
+  test("combining scheduler and allocation excluded node list") {
+    sparkConf.set(YARN_EXCLUDE_NODES, Seq("initial1", "initial2"))
+    val yarnHealthTracker = createYarnAllocatorHealthTracker(sparkConf)
+    yarnHealthTracker.setSchedulerExcludedNodes(Set())
+
+    // initial1 and initial2 is added as excluded nodes at the very first updateBlacklist call
+    // and they are never removed
+    verify(amClientMock)
+      .updateBlacklist(Arrays.asList("initial1", "initial2"), Collections.emptyList())
+
+    (1 to MAX_FAILED_EXEC_PER_NODE_VALUE).foreach {
+      _ => {
+        yarnHealthTracker.handleResourceAllocationFailure(Some("host1"))
+        // host1 should not be excluded at these failures as MAX_FAILED_EXEC_PER_NODE is 2
+        verify(amClientMock, never())
+          .updateBlacklist(Arrays.asList("host1"), Collections.emptyList())
+      }
+    }
+
+    // as this is the third failure on host1 the node will be excluded
+    yarnHealthTracker.handleResourceAllocationFailure(Some("host1"))
+    verify(amClientMock)
+      .updateBlacklist(Arrays.asList("host1"), Collections.emptyList())
+
+    yarnHealthTracker.setSchedulerExcludedNodes(Set("host2", "host3"))
+    verify(amClientMock)
+      .updateBlacklist(Arrays.asList("host2", "host3"), Collections.emptyList())
+
+    clock.advance(10L)
+
+    yarnHealthTracker.setSchedulerExcludedNodes(Set("host3", "host4"))
+    verify(amClientMock)
+      .updateBlacklist(Arrays.asList("host4"), Arrays.asList("host2"))
+  }
+
+  test("exclude all available nodes") {
+    val yarnHealthTracker = createYarnAllocatorHealthTracker()
+    yarnHealthTracker.setSchedulerExcludedNodes(Set("host1", "host2", "host3"))
+    verify(amClientMock)
+      .updateBlacklist(Arrays.asList("host1", "host2", "host3"), Collections.emptyList())
+
+    clock.advance(60L)
+    (1 to MAX_FAILED_EXEC_PER_NODE_VALUE).foreach {
+      _ => {
+        yarnHealthTracker.handleResourceAllocationFailure(Some("host4"))
+        // host4 should not be excluded at these failures as MAX_FAILED_EXEC_PER_NODE is 2
+        verify(amClientMock, never())
+          .updateBlacklist(Arrays.asList("host4"), Collections.emptyList())
+      }
+    }
+
+    // the third failure on the host triggers the exclusion
+    yarnHealthTracker.handleResourceAllocationFailure(Some("host4"))
+
+    verify(amClientMock).updateBlacklist(Arrays.asList("host4"), Collections.emptyList())
+    assert(yarnHealthTracker.isAllNodeExcluded)
+  }
+
+}
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorSuite.scala
index 2003d0bb87b2d..9a7eed60bc096 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorSuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorSuite.scala
@@ -22,13 +22,16 @@ import java.util.Collections
 import scala.collection.JavaConverters._
 import scala.collection.mutable
 
+import org.apache.hadoop.net.{Node, NodeBase}
 import org.apache.hadoop.yarn.api.records._
 import org.apache.hadoop.yarn.client.api.AMRMClient
 import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest
 import org.apache.hadoop.yarn.conf.YarnConfiguration
 import org.mockito.ArgumentCaptor
 import org.mockito.Mockito._
-import org.scalatest.{BeforeAndAfterEach, Matchers}
+import org.scalatest.BeforeAndAfterEach
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
 import org.apache.spark.deploy.SparkHadoopUtil
@@ -48,6 +51,9 @@ class MockResolver extends SparkRackResolver(SparkHadoopUtil.get.conf) {
     if (hostName == "host3") "/rack2" else "/rack1"
   }
 
+  override def resolve(hostNames: Seq[String]): Seq[Node] =
+    hostNames.map(n => new NodeBase(n, resolve(n)))
+
 }
 
 class YarnAllocatorSuite extends SparkFunSuite with Matchers with BeforeAndAfterEach {
@@ -73,7 +79,7 @@ class YarnAllocatorSuite extends SparkFunSuite with Matchers with BeforeAndAfter
   // priority has to be 0 to match default profile id
   val RM_REQUEST_PRIORITY = Priority.newInstance(0)
   val defaultRPId = ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID
-  val defaultRP = ResourceProfile.getOrCreateDefaultProfile(sparkConf)
+  var defaultRP = ResourceProfile.getOrCreateDefaultProfile(sparkConf)
 
   override def beforeEach(): Unit = {
     super.beforeEach()
@@ -112,6 +118,9 @@ class YarnAllocatorSuite extends SparkFunSuite with Matchers with BeforeAndAfter
     for ((name, value) <- additionalConfigs) {
       sparkConfClone.set(name, value)
     }
+    // different spark confs means we need to reinit the default profile
+    ResourceProfile.clearDefaultProfile()
+    defaultRP = ResourceProfile.getOrCreateDefaultProfile(sparkConfClone)
 
     val allocator = new YarnAllocator(
       "not used",
@@ -266,12 +275,13 @@ class YarnAllocatorSuite extends SparkFunSuite with Matchers with BeforeAndAfter
       Map(s"${YARN_EXECUTOR_RESOURCE_TYPES_PREFIX}${GPU}.${AMOUNT}" -> "2G"))
 
     handler.updateResourceRequests()
-    val container = createContainer("host1", resource = handler.defaultResource)
+    val defaultResource = handler.rpIdToYarnResource.get(defaultRPId)
+    val container = createContainer("host1", resource = defaultResource)
     handler.handleAllocatedContainers(Array(container))
 
     // get amount of memory and vcores from resource, so effectively skipping their validation
-    val expectedResources = Resource.newInstance(handler.defaultResource.getMemory(),
-      handler.defaultResource.getVirtualCores)
+    val expectedResources = Resource.newInstance(defaultResource.getMemory(),
+      defaultResource.getVirtualCores)
     setResourceRequests(Map("gpu" -> "2G"), expectedResources)
     val captor = ArgumentCaptor.forClass(classOf[ContainerRequest])
 
@@ -294,7 +304,8 @@ class YarnAllocatorSuite extends SparkFunSuite with Matchers with BeforeAndAfter
     val (handler, _) = createAllocator(1, mockAmClient, sparkResources)
 
     handler.updateResourceRequests()
-    val yarnRInfo = ResourceRequestTestHelper.getResources(handler.defaultResource)
+    val defaultResource = handler.rpIdToYarnResource.get(defaultRPId)
+    val yarnRInfo = ResourceRequestTestHelper.getResources(defaultResource)
     val allResourceInfo = yarnRInfo.map( rInfo => (rInfo.name -> rInfo.value) ).toMap
     assert(allResourceInfo.get(YARN_GPU_RESOURCE_CONFIG).nonEmpty)
     assert(allResourceInfo.get(YARN_GPU_RESOURCE_CONFIG).get === 3)
@@ -521,9 +532,10 @@ class YarnAllocatorSuite extends SparkFunSuite with Matchers with BeforeAndAfter
     handler.getNumUnexpectedContainerRelease should be (2)
   }
 
-  test("blacklisted nodes reflected in amClient requests") {
-    // Internally we track the set of blacklisted nodes, but yarn wants us to send *changes*
-    // to the blacklist.  This makes sure we are sending the right updates.
+  test("excluded nodes reflected in amClient requests") {
+    // Internally we track the set of excluded nodes, but yarn wants us to send *changes*
+    // to it. Note the YARN api uses the term blacklist for excluded nodes.
+    // This makes sure we are sending the right updates.
     val mockAmClient = mock(classOf[AMRMClient[ContainerRequest]])
     val (handler, _) = createAllocator(4, mockAmClient)
     val resourceProfileToTotalExecs = mutable.HashMap(defaultRP -> 1)
@@ -532,14 +544,14 @@ class YarnAllocatorSuite extends SparkFunSuite with Matchers with BeforeAndAfter
       numLocalityAwareTasksPerResourceProfileId.toMap, Map(), Set("hostA"))
     verify(mockAmClient).updateBlacklist(Seq("hostA").asJava, Seq[String]().asJava)
 
-    val blacklistedNodes = Set(
+    val excludedNodes = Set(
       "hostA",
       "hostB"
     )
 
     resourceProfileToTotalExecs(defaultRP) = 2
     handler.requestTotalExecutorsWithPreferredLocalities(resourceProfileToTotalExecs.toMap,
-      numLocalityAwareTasksPerResourceProfileId.toMap, Map(), blacklistedNodes)
+      numLocalityAwareTasksPerResourceProfileId.toMap, Map(), excludedNodes)
     verify(mockAmClient).updateBlacklist(Seq("hostB").asJava, Seq[String]().asJava)
     resourceProfileToTotalExecs(defaultRP) = 3
     handler.requestTotalExecutorsWithPreferredLocalities(resourceProfileToTotalExecs.toMap,
@@ -590,7 +602,7 @@ class YarnAllocatorSuite extends SparkFunSuite with Matchers with BeforeAndAfter
     handler.getNumExecutorsFailed should be (0)
   }
 
-  test("SPARK-26269: YarnAllocator should have same blacklist behaviour with YARN") {
+  test("SPARK-26269: YarnAllocator should have same excludeOnFailure behaviour with YARN") {
     val rmClientSpy = spy(rmClient)
     val maxExecutors = 11
 
@@ -598,7 +610,7 @@ class YarnAllocatorSuite extends SparkFunSuite with Matchers with BeforeAndAfter
       maxExecutors,
       rmClientSpy,
       Map(
-        YARN_EXECUTOR_LAUNCH_BLACKLIST_ENABLED.key -> "true",
+        YARN_EXECUTOR_LAUNCH_EXCLUDE_ON_FAILURE_ENABLED.key -> "true",
         MAX_FAILED_EXEC_PER_NODE.key -> "0"))
     handler.updateResourceRequests()
 
@@ -606,7 +618,7 @@ class YarnAllocatorSuite extends SparkFunSuite with Matchers with BeforeAndAfter
     val ids = 0 to maxExecutors
     val containers = createContainers(hosts, ids)
 
-    val nonBlacklistedStatuses = Seq(
+    val nonExcludedStatuses = Seq(
       ContainerExitStatus.SUCCESS,
       ContainerExitStatus.PREEMPTED,
       ContainerExitStatus.KILLED_EXCEEDED_VMEM,
@@ -617,24 +629,24 @@ class YarnAllocatorSuite extends SparkFunSuite with Matchers with BeforeAndAfter
       ContainerExitStatus.ABORTED,
       ContainerExitStatus.DISKS_FAILED)
 
-    val nonBlacklistedContainerStatuses = nonBlacklistedStatuses.zipWithIndex.map {
+    val nonExcludedContainerStatuses = nonExcludedStatuses.zipWithIndex.map {
       case (exitStatus, idx) => createContainerStatus(containers(idx).getId, exitStatus)
     }
 
-    val BLACKLISTED_EXIT_CODE = 1
-    val blacklistedStatuses = Seq(ContainerExitStatus.INVALID, BLACKLISTED_EXIT_CODE)
+    val EXCLUDED_EXIT_CODE = 1
+    val excludedStatuses = Seq(ContainerExitStatus.INVALID, EXCLUDED_EXIT_CODE)
 
-    val blacklistedContainerStatuses = blacklistedStatuses.zip(9 until maxExecutors).map {
+    val excludedContainerStatuses = excludedStatuses.zip(9 until maxExecutors).map {
       case (exitStatus, idx) => createContainerStatus(containers(idx).getId, exitStatus)
     }
 
     handler.handleAllocatedContainers(containers.slice(0, 9))
-    handler.processCompletedContainers(nonBlacklistedContainerStatuses)
+    handler.processCompletedContainers(nonExcludedContainerStatuses)
     verify(rmClientSpy, never())
       .updateBlacklist(hosts.slice(0, 9).asJava, Collections.emptyList())
 
     handler.handleAllocatedContainers(containers.slice(9, 11))
-    handler.processCompletedContainers(blacklistedContainerStatuses)
+    handler.processCompletedContainers(excludedContainerStatuses)
     verify(rmClientSpy)
       .updateBlacklist(hosts.slice(9, 10).asJava, Collections.emptyList())
     verify(rmClientSpy)
@@ -653,9 +665,10 @@ class YarnAllocatorSuite extends SparkFunSuite with Matchers with BeforeAndAfter
       sparkConf.set(MEMORY_OFFHEAP_SIZE, offHeapMemoryInByte)
       val (handler, _) = createAllocator(maxExecutors = 1,
         additionalConfigs = Map(EXECUTOR_MEMORY.key -> executorMemory.toString))
-      val memory = handler.defaultResource.getMemory
+      val defaultResource = handler.rpIdToYarnResource.get(defaultRPId)
+      val memory = defaultResource.getMemory
       assert(memory ==
-        executorMemory + offHeapMemoryInMB + YarnSparkHadoopUtil.MEMORY_OVERHEAD_MIN)
+        executorMemory + offHeapMemoryInMB + ResourceProfile.MEMORY_OVERHEAD_MIN_MIB)
     } finally {
       sparkConf.set(MEMORY_OFFHEAP_ENABLED, originalOffHeapEnabled)
       sparkConf.set(MEMORY_OFFHEAP_SIZE, originalOffHeapSize)
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala
index b7c9e83446012..26ff3bf2971f6 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala
@@ -28,8 +28,10 @@ import scala.io.Source
 import com.google.common.io.{ByteStreams, Files}
 import org.apache.hadoop.yarn.conf.YarnConfiguration
 import org.apache.hadoop.yarn.util.ConverterUtils
-import org.scalatest.Matchers
 import org.scalatest.concurrent.Eventually._
+import org.scalatest.exceptions.TestFailedException
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark._
 import org.apache.spark.api.python.PythonUtils
@@ -51,6 +53,13 @@ import org.apache.spark.util.{Utils, YarnContainerInfoHelper}
 @ExtendedYarnTest
 class YarnClusterSuite extends BaseYarnClusterSuite {
 
+  private val pythonExecutablePath = {
+    // To make sure to use the same Python executable.
+    val maybePath = TestUtils.getAbsolutePathFromExecutable("python3")
+    assert(maybePath.isDefined)
+    maybePath.get
+  }
+
   override def newYarnConfig(): YarnConfiguration = new YarnConfiguration()
 
   private val TEST_PYFILE = """
@@ -174,9 +183,9 @@ class YarnClusterSuite extends BaseYarnClusterSuite {
       clientMode = false,
       extraConf = Map(
         "spark.yarn.appMasterEnv.PYSPARK_DRIVER_PYTHON"
-          -> sys.env.getOrElse("PYSPARK_DRIVER_PYTHON", "python"),
+          -> sys.env.getOrElse("PYSPARK_DRIVER_PYTHON", pythonExecutablePath),
         "spark.yarn.appMasterEnv.PYSPARK_PYTHON"
-          -> sys.env.getOrElse("PYSPARK_PYTHON", "python")),
+          -> sys.env.getOrElse("PYSPARK_PYTHON", pythonExecutablePath)),
       extraEnv = Map(
         "PYSPARK_DRIVER_PYTHON" -> "not python",
         "PYSPARK_PYTHON" -> "not python"))
@@ -222,6 +231,37 @@ class YarnClusterSuite extends BaseYarnClusterSuite {
     }
   }
 
+  test("running Spark in yarn-cluster mode displays driver log links") {
+    val log4jConf = new File(tempDir, "log4j.properties")
+    val logOutFile = new File(tempDir, "logs")
+    Files.write(
+      s"""log4j.rootCategory=DEBUG,file
+         |log4j.appender.file=org.apache.log4j.FileAppender
+         |log4j.appender.file.file=$logOutFile
+         |log4j.appender.file.layout=org.apache.log4j.PatternLayout
+         |""".stripMargin,
+      log4jConf, StandardCharsets.UTF_8)
+    // Since this test is trying to extract log output from the SparkSubmit process itself,
+    // standard options to the Spark process don't take effect. Leverage the java-opts file which
+    // will get picked up for the SparkSubmit process.
+    val confDir = new File(tempDir, "conf")
+    confDir.mkdir()
+    val javaOptsFile = new File(confDir, "java-opts")
+    Files.write(s"-Dlog4j.configuration=file://$log4jConf\n", javaOptsFile, StandardCharsets.UTF_8)
+
+    val result = File.createTempFile("result", null, tempDir)
+    val finalState = runSpark(clientMode = false,
+      mainClassName(YarnClusterDriver.getClass),
+      appArgs = Seq(result.getAbsolutePath),
+      extraEnv = Map("SPARK_CONF_DIR" -> confDir.getAbsolutePath),
+      extraConf = Map(CLIENT_INCLUDE_DRIVER_LOGS_LINK.key -> true.toString))
+    checkResult(finalState, result)
+    val logOutput = Files.toString(logOutFile, StandardCharsets.UTF_8)
+    val logFilePattern = raw"""(?s).+\sDriver Logs \(<NAME>\): https?://.+/<NAME>(\?\S+)?\s.+"""
+    logOutput should fullyMatch regex logFilePattern.replace("<NAME>", "stdout")
+    logOutput should fullyMatch regex logFilePattern.replace("<NAME>", "stderr")
+  }
+
   test("timeout to get SparkContext in cluster mode triggers failure") {
     val timeout = 2000
     val finalState = runSpark(false, mainClassName(SparkContextTimeoutApp.getClass),
@@ -274,7 +314,10 @@ class YarnClusterSuite extends BaseYarnClusterSuite {
         s"$sparkHome/python")
     val extraEnvVars = Map(
       "PYSPARK_ARCHIVES_PATH" -> pythonPath.map("local:" + _).mkString(File.pathSeparator),
-      "PYTHONPATH" -> pythonPath.mkString(File.pathSeparator)) ++ extraEnv
+      "PYTHONPATH" -> pythonPath.mkString(File.pathSeparator),
+      "PYSPARK_DRIVER_PYTHON" -> pythonExecutablePath,
+      "PYSPARK_PYTHON" -> pythonExecutablePath
+    ) ++ extraEnv
 
     val moduleDir = {
       val subdir = new File(tempDir, "pyModules")
@@ -326,6 +369,64 @@ class YarnClusterSuite extends BaseYarnClusterSuite {
     )
     checkResult(finalState, result, "true")
   }
+
+  def createEmptyIvySettingsFile: File = {
+    val emptyIvySettings = File.createTempFile("ivy", ".xml")
+    Files.write("<ivysettings />", emptyIvySettings, StandardCharsets.UTF_8)
+    emptyIvySettings
+  }
+
+  test("SPARK-34472: ivySettings file with no scheme or file:// scheme should be " +
+    "localized on driver in cluster mode") {
+    val emptyIvySettings = createEmptyIvySettingsFile
+    // For file:// URIs or URIs without scheme, make sure that ivySettings conf was changed
+    // to the localized file. So the expected ivySettings path on the driver will start with
+    // the file name and then some random UUID suffix
+    testIvySettingsDistribution(clientMode = false, emptyIvySettings.getAbsolutePath,
+      emptyIvySettings.getName, prefixMatch = true)
+    testIvySettingsDistribution(clientMode = false, s"file://${emptyIvySettings.getAbsolutePath}",
+      emptyIvySettings.getName, prefixMatch = true)
+  }
+
+  test("SPARK-34472: ivySettings file with no scheme or file:// scheme should retain " +
+    "user provided path in client mode") {
+    val emptyIvySettings = createEmptyIvySettingsFile
+    // In client mode, the file is present locally on the driver and so does not need to be
+    // distributed. So the user provided path should be kept as is.
+    testIvySettingsDistribution(clientMode = true, emptyIvySettings.getAbsolutePath,
+      emptyIvySettings.getAbsolutePath)
+    testIvySettingsDistribution(clientMode = true, s"file://${emptyIvySettings.getAbsolutePath}",
+      s"file://${emptyIvySettings.getAbsolutePath}")
+  }
+
+  test("SPARK-34472: ivySettings file with non-file:// schemes should throw an error") {
+    val emptyIvySettings = createEmptyIvySettingsFile
+    val e1 = intercept[TestFailedException] {
+      testIvySettingsDistribution(clientMode = false,
+        s"local://${emptyIvySettings.getAbsolutePath}", "")
+    }
+    assert(e1.getMessage.contains("IllegalArgumentException: " +
+      "Scheme local not supported in spark.jars.ivySettings"))
+    val e2 = intercept[TestFailedException] {
+      testIvySettingsDistribution(clientMode = false,
+        s"hdfs://${emptyIvySettings.getAbsolutePath}", "")
+    }
+    assert(e2.getMessage.contains("IllegalArgumentException: " +
+      "Scheme hdfs not supported in spark.jars.ivySettings"))
+  }
+
+  def testIvySettingsDistribution(clientMode: Boolean, ivySettingsPath: String,
+    expectedIvySettingsPrefixOnDriver: String, prefixMatch: Boolean = false): Unit = {
+    val result = File.createTempFile("result", null, tempDir)
+    val outFile = File.createTempFile("out", null, tempDir)
+    val finalState = runSpark(clientMode = clientMode,
+      mainClassName(YarnAddJarTest.getClass),
+      appArgs = Seq(result.getAbsolutePath, expectedIvySettingsPrefixOnDriver,
+        prefixMatch.toString),
+      extraConf = Map("spark.jars.ivySettings" -> ivySettingsPath),
+      outFile = Option(outFile))
+    checkResult(finalState, result, outFile = Option(outFile))
+  }
 }
 
 private[spark] class SaveExecutorInfo extends SparkListener {
@@ -438,7 +539,7 @@ private object YarnClusterDriver extends Logging with Matchers {
       executorInfos.foreach { info =>
         assert(info.logUrlMap.nonEmpty)
         info.logUrlMap.values.foreach { url =>
-          val log = Source.fromURL(url).mkString
+          val log = Utils.tryWithResource(Source.fromURL(url))(_.mkString)
           assert(
             !log.contains(SECRET_PASSWORD),
             s"Executor logs contain sensitive info (${SECRET_PASSWORD}): \n${log} "
@@ -457,7 +558,7 @@ private object YarnClusterDriver extends Logging with Matchers {
         assert(driverLogs.contains("stdout"))
         val urlStr = driverLogs("stderr")
         driverLogs.foreach { kv =>
-          val log = Source.fromURL(kv._2).mkString
+          val log = Utils.tryWithResource(Source.fromURL(kv._2))(_.mkString)
           assert(
             !log.contains(SECRET_PASSWORD),
             s"Driver logs contain sensitive info (${SECRET_PASSWORD}): \n${log} "
@@ -541,6 +642,50 @@ private object YarnClasspathTest extends Logging {
 
 }
 
+private object YarnAddJarTest extends Logging {
+  def main(args: Array[String]): Unit = {
+    if (args.length != 3) {
+      // scalastyle:off println
+      System.err.println(
+        s"""
+           |Invalid command line: ${args.mkString(" ")}
+           |
+           |Usage: YarnAddJarTest [result file] [expected ivy settings path] [prefix match]
+        """.stripMargin)
+      // scalastyle:on println
+      System.exit(1)
+    }
+
+    val resultPath = args(0)
+    val expectedIvySettingsPath = args(1)
+    val prefixMatch = args(2).toBoolean
+    val sc = new SparkContext(new SparkConf())
+
+    var result = "failure"
+    try {
+      val settingsFile = sc.getConf.get("spark.jars.ivySettings")
+      if (prefixMatch) {
+        assert(settingsFile !== expectedIvySettingsPath)
+        assert(settingsFile.startsWith(expectedIvySettingsPath))
+      } else {
+        assert(settingsFile === expectedIvySettingsPath)
+      }
+
+      val caught = intercept[RuntimeException] {
+        sc.addJar("ivy://org.fake-project.test:test:1.0.0")
+      }
+      if (caught.getMessage.contains("unresolved dependency: org.fake-project.test#test")) {
+        // "unresolved dependency" is expected as the dependency does not exist
+        // but exception like "Ivy settings file <file> does not exist" should result in failure
+        result = "success"
+      }
+    } finally {
+      Files.write(result, new File(resultPath), StandardCharsets.UTF_8)
+      sc.stop()
+    }
+  }
+}
+
 private object YarnLauncherTestApp {
 
   def main(args: Array[String]): Unit = {
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnShuffleAlternateNameConfigSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnShuffleAlternateNameConfigSuite.scala
new file mode 100644
index 0000000000000..db001a946fddf
--- /dev/null
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnShuffleAlternateNameConfigSuite.scala
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.deploy.yarn
+
+import java.net.URLClassLoader
+
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+
+import org.apache.spark._
+import org.apache.spark.internal.config._
+import org.apache.spark.network.yarn.{YarnShuffleService, YarnTestAccessor}
+import org.apache.spark.tags.ExtendedYarnTest
+
+/**
+ * SPARK-34828: Integration test for the external shuffle service with an alternate name and
+ * configs (by using a configuration overlay)
+ */
+@ExtendedYarnTest
+class YarnShuffleAlternateNameConfigSuite extends YarnShuffleIntegrationSuite {
+
+  private[this] val shuffleServiceName = "custom_shuffle_service_name"
+
+  override def newYarnConfig(): YarnConfiguration = {
+    val yarnConfig = super.newYarnConfig()
+    yarnConfig.set(YarnConfiguration.NM_AUX_SERVICES, shuffleServiceName)
+    yarnConfig.set(YarnConfiguration.NM_AUX_SERVICE_FMT.format(shuffleServiceName),
+      classOf[YarnShuffleService].getCanonicalName)
+    val overlayConf = new YarnConfiguration()
+    // Enable authentication in the base NodeManager conf but not in the client. This would break
+    // shuffle, unless the shuffle service conf overlay overrides to turn off authentication.
+    overlayConf.setBoolean(NETWORK_AUTH_ENABLED.key, true)
+    // Add the authentication conf to a separate config object used as an overlay rather than
+    // setting it directly. This is necessary because a config overlay will override previous
+    // config overlays, but not configs which were set directly on the config object.
+    yarnConfig.addResource(overlayConf)
+    yarnConfig
+  }
+
+  override protected def extraSparkConf(): Map[String, String] =
+    super.extraSparkConf() ++ Map(SHUFFLE_SERVICE_NAME.key -> shuffleServiceName)
+
+  override def beforeAll(): Unit = {
+    val configFileContent =
+      s"""<?xml version="1.0" encoding="UTF-8"?>
+         |<configuration>
+         |  <property>
+         |    <name>${NETWORK_AUTH_ENABLED.key}</name>
+         |    <value>false</value>
+         |  </property>
+         |</configuration>
+         |""".stripMargin
+    val jarFile = TestUtils.createJarWithFiles(Map(
+      YarnTestAccessor.getShuffleServiceConfOverlayResourceName -> configFileContent
+    ))
+    // Configure a custom classloader which includes the conf overlay as a resource
+    val oldClassLoader = Thread.currentThread().getContextClassLoader
+    Thread.currentThread().setContextClassLoader(new URLClassLoader(Array(jarFile)))
+    try {
+      super.beforeAll()
+    } finally {
+      Thread.currentThread().setContextClassLoader(oldClassLoader)
+    }
+  }
+}
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnShuffleIntegrationSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnShuffleIntegrationSuite.scala
index 8c62069a8dd67..547bfca2891f1 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnShuffleIntegrationSuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnShuffleIntegrationSuite.scala
@@ -23,7 +23,8 @@ import java.nio.charset.StandardCharsets
 import com.google.common.io.Files
 import org.apache.commons.io.FileUtils
 import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark._
 import org.apache.spark.deploy.yarn.config._
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnSparkHadoopUtilSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnSparkHadoopUtilSuite.scala
index c88bb292aa77f..5b762f606112c 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnSparkHadoopUtilSuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnSparkHadoopUtilSuite.scala
@@ -23,12 +23,12 @@ import java.nio.charset.StandardCharsets
 import com.google.common.io.{ByteStreams, Files}
 import org.apache.hadoop.yarn.api.records.ApplicationAccessType
 import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
 import org.apache.spark.deploy.SparkHadoopUtil
 import org.apache.spark.internal.Logging
-import org.apache.spark.internal.config._
 import org.apache.spark.internal.config.UI._
 import org.apache.spark.util.{ResetSystemProperties, Utils}
 
@@ -141,31 +141,4 @@ class YarnSparkHadoopUtilSuite extends SparkFunSuite with Matchers with Logging
     }
 
   }
-
-  test("executorOffHeapMemorySizeAsMb when MEMORY_OFFHEAP_ENABLED is false") {
-    val executorOffHeapMemory = YarnSparkHadoopUtil.executorOffHeapMemorySizeAsMb(new SparkConf())
-    assert(executorOffHeapMemory == 0)
-  }
-
-  test("executorOffHeapMemorySizeAsMb when MEMORY_OFFHEAP_ENABLED is true") {
-    val offHeapMemoryInMB = 50
-    val offHeapMemory: Long = offHeapMemoryInMB * 1024 * 1024
-    val sparkConf = new SparkConf()
-      .set(MEMORY_OFFHEAP_ENABLED, true)
-      .set(MEMORY_OFFHEAP_SIZE, offHeapMemory)
-    val executorOffHeapMemory = YarnSparkHadoopUtil.executorOffHeapMemorySizeAsMb(sparkConf)
-    assert(executorOffHeapMemory == offHeapMemoryInMB)
-  }
-
-  test("executorMemoryOverhead when MEMORY_OFFHEAP_ENABLED is true, " +
-    "but MEMORY_OFFHEAP_SIZE not config scene") {
-    val sparkConf = new SparkConf()
-      .set(MEMORY_OFFHEAP_ENABLED, true)
-    val expected =
-      s"${MEMORY_OFFHEAP_SIZE.key} must be > 0 when ${MEMORY_OFFHEAP_ENABLED.key} == true"
-    val message = intercept[IllegalArgumentException] {
-      YarnSparkHadoopUtil.executorOffHeapMemorySizeAsMb(sparkConf)
-    }.getMessage
-    assert(message.contains(expected))
-  }
 }
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceMetricsSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceMetricsSuite.scala
index 632c66d77b707..9239d891aae3b 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceMetricsSuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceMetricsSuite.scala
@@ -21,7 +21,8 @@ import scala.collection.JavaConverters._
 import org.apache.hadoop.metrics2.MetricsRecordBuilder
 import org.mockito.ArgumentMatchers.{any, anyDouble, anyInt, anyLong}
 import org.mockito.Mockito.{mock, times, verify, when}
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.network.server.OneForOneStreamManager
@@ -39,7 +40,7 @@ class YarnShuffleServiceMetricsSuite extends SparkFunSuite with Matchers {
     val allMetrics = Set(
       "openBlockRequestLatencyMillis", "registerExecutorRequestLatencyMillis",
       "blockTransferRateBytes", "registeredExecutorsSize", "numActiveConnections",
-      "numCaughtExceptions")
+      "numCaughtExceptions", "finalizeShuffleMergeLatencyMillis")
 
     metrics.getMetrics.keySet().asScala should be (allMetrics)
   }
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceSuite.scala
index 1a5a099217f55..d6d1715223e35 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceSuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceSuite.scala
@@ -34,14 +34,18 @@ import org.apache.hadoop.service.ServiceStateException
 import org.apache.hadoop.yarn.api.records.ApplicationId
 import org.apache.hadoop.yarn.conf.YarnConfiguration
 import org.apache.hadoop.yarn.server.api.{ApplicationInitializationContext, ApplicationTerminationContext}
-import org.scalatest.{BeforeAndAfterEach, Matchers}
+import org.mockito.Mockito.{mock, when}
+import org.scalatest.BeforeAndAfterEach
 import org.scalatest.concurrent.Eventually._
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.SecurityManager
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.internal.config._
-import org.apache.spark.network.shuffle.ShuffleTestAccessor
+import org.apache.spark.network.shuffle.{ExternalBlockHandler, RemoteBlockPushResolver, ShuffleTestAccessor}
 import org.apache.spark.network.shuffle.protocol.ExecutorShuffleInfo
+import org.apache.spark.network.util.TransportConf
 import org.apache.spark.util.Utils
 
 class YarnShuffleServiceSuite extends SparkFunSuite with Matchers with BeforeAndAfterEach {
@@ -52,6 +56,9 @@ class YarnShuffleServiceSuite extends SparkFunSuite with Matchers with BeforeAnd
 
   override def beforeEach(): Unit = {
     super.beforeEach()
+    // Ensure that each test uses a fresh metrics system
+    DefaultMetricsSystem.shutdown()
+    DefaultMetricsSystem.setInstance(new MetricsSystemImpl())
     yarnConfig = new YarnConfiguration()
     yarnConfig.set(YarnConfiguration.NM_AUX_SERVICES, "spark_shuffle")
     yarnConfig.set(YarnConfiguration.NM_AUX_SERVICE_FMT.format("spark_shuffle"),
@@ -246,7 +253,7 @@ class YarnShuffleServiceSuite extends SparkFunSuite with Matchers with BeforeAnd
     ShuffleTestAccessor.getExecutorInfo(app2Id, "exec-2", resolver2) should be (Some(shuffleInfo2))
     s2.stop()
 
-    // another stop & restart should be fine though (eg., we recover from previous corruption)
+    // another stop & restart should be fine though (e.g., we recover from previous corruption)
     s3 = new YarnShuffleService
     s3.setRecoveryPath(new Path(recoveryLocalDir.toURI))
     s3.init(yarnConfig)
@@ -403,9 +410,43 @@ class YarnShuffleServiceSuite extends SparkFunSuite with Matchers with BeforeAnd
       "openBlockRequestLatencyMillis",
       "registeredExecutorsSize",
       "registerExecutorRequestLatencyMillis",
+      "finalizeShuffleMergeLatencyMillis",
       "shuffle-server.usedDirectMemory",
       "shuffle-server.usedHeapMemory"
     ))
   }
 
+  test("SPARK-34828: metrics should be registered with configured name") {
+    s1 = new YarnShuffleService
+    yarnConfig.set(YarnShuffleService.SPARK_SHUFFLE_SERVICE_METRICS_NAMESPACE_KEY, "fooMetrics")
+    s1.init(yarnConfig)
+
+    assert(DefaultMetricsSystem.instance.getSource("sparkShuffleService") === null)
+    assert(DefaultMetricsSystem.instance.getSource("fooMetrics")
+        .isInstanceOf[YarnShuffleServiceMetrics])
+  }
+
+  test("create default merged shuffle file manager instance") {
+    val mockConf = mock(classOf[TransportConf])
+    when(mockConf.mergedShuffleFileManagerImpl).thenReturn(
+      "org.apache.spark.network.shuffle.ExternalBlockHandler$NoOpMergedShuffleFileManager")
+    val mergeMgr = YarnShuffleService.newMergedShuffleFileManagerInstance(mockConf)
+    assert(mergeMgr.isInstanceOf[ExternalBlockHandler.NoOpMergedShuffleFileManager])
+  }
+
+  test("create remote block push resolver instance") {
+    val mockConf = mock(classOf[TransportConf])
+    when(mockConf.mergedShuffleFileManagerImpl).thenReturn(
+      "org.apache.spark.network.shuffle.RemoteBlockPushResolver")
+    val mergeMgr = YarnShuffleService.newMergedShuffleFileManagerInstance(mockConf)
+    assert(mergeMgr.isInstanceOf[RemoteBlockPushResolver])
+  }
+
+  test("invalid class name of merge manager will use noop instance") {
+    val mockConf = mock(classOf[TransportConf])
+    when(mockConf.mergedShuffleFileManagerImpl).thenReturn(
+      "org.apache.spark.network.shuffle.NotExistent")
+    val mergeMgr = YarnShuffleService.newMergedShuffleFileManagerInstance(mockConf)
+    assert(mergeMgr.isInstanceOf[ExternalBlockHandler.NoOpMergedShuffleFileManager])
+  }
 }
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnTestAccessor.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnTestAccessor.scala
index db322cd18e150..d87cc26384729 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnTestAccessor.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnTestAccessor.scala
@@ -34,4 +34,7 @@ object YarnTestAccessor {
     service.registeredExecutorFile
   }
 
+  def getShuffleServiceConfOverlayResourceName: String = {
+    YarnShuffleService.SHUFFLE_SERVICE_CONF_OVERLAY_RESOURCE_NAME
+  }
 }
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackendSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackendSuite.scala
index 9003c2f630975..7959bb55d7ffc 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackendSuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackendSuite.scala
@@ -44,9 +44,10 @@ class YarnSchedulerBackendSuite extends SparkFunSuite with MockitoSugar with Loc
   }
 
   private class TestTaskSchedulerImpl(sc: SparkContext) extends TaskSchedulerImpl(sc) {
-    val blacklistedNodes = new AtomicReference[Set[String]]()
-    def setNodeBlacklist(nodeBlacklist: Set[String]): Unit = blacklistedNodes.set(nodeBlacklist)
-    override def nodeBlacklist(): Set[String] = blacklistedNodes.get()
+    val excludedNodesList = new AtomicReference[Set[String]]()
+    def setNodeExcludeList(nodeExcludeList: Set[String]): Unit =
+      excludedNodesList.set(nodeExcludeList)
+    override def excludedNodes(): Set[String] = excludedNodesList.get()
   }
 
   private class TestYarnSchedulerBackend(scheduler: TaskSchedulerImpl, sc: SparkContext)
@@ -56,7 +57,7 @@ class YarnSchedulerBackendSuite extends SparkFunSuite with MockitoSugar with Loc
     }
   }
 
-  test("RequestExecutors reflects node blacklist and is serializable") {
+  test("RequestExecutors reflects node excludelist and is serializable") {
     sc = new SparkContext("local", "YarnSchedulerBackendSuite")
     // Subclassing the TaskSchedulerImpl here instead of using Mockito. For details see SPARK-26891.
     val sched = new TestTaskSchedulerImpl(sc)
@@ -65,7 +66,7 @@ class YarnSchedulerBackendSuite extends SparkFunSuite with MockitoSugar with Loc
     val ser = new JavaSerializer(sc.conf).newInstance()
     val defaultResourceProf = ResourceProfile.getOrCreateDefaultProfile(sc.getConf)
     for {
-      blacklist <- IndexedSeq(Set[String](), Set("a", "b", "c"))
+      excludelist <- IndexedSeq(Set[String](), Set("a", "b", "c"))
       numRequested <- 0 until 10
       hostToLocalCount <- IndexedSeq(
         Map(defaultResourceProf.id -> Map.empty[String, Int]),
@@ -73,14 +74,14 @@ class YarnSchedulerBackendSuite extends SparkFunSuite with MockitoSugar with Loc
       )
     } {
       yarnSchedulerBackendExtended.setHostToLocalTaskCount(hostToLocalCount)
-      sched.setNodeBlacklist(blacklist)
+      sched.setNodeExcludeList(excludelist)
       val request = Map(defaultResourceProf -> numRequested)
       val req = yarnSchedulerBackendExtended.prepareRequestExecutors(request)
       assert(req.resourceProfileToTotalExecs(defaultResourceProf) === numRequested)
-      assert(req.nodeBlacklist === blacklist)
+      assert(req.excludedNodes === excludelist)
       val hosts =
         req.hostToLocalTaskCount(ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID).keySet
-      assert(hosts.intersect(blacklist).isEmpty)
+      assert(hosts.intersect(excludelist).isEmpty)
       // Serialize to make sure serialization doesn't throw an error
       ser.serialize(req)
     }
diff --git a/sbin/decommission-slave.sh b/sbin/decommission-slave.sh
old mode 100644
new mode 100755
index 4bbf257ff1d3a..858bede1d2878
--- a/sbin/decommission-slave.sh
+++ b/sbin/decommission-slave.sh
@@ -17,41 +17,7 @@
 # limitations under the License.
 #
 
-# A shell script to decommission all workers on a single slave
-#
-# Environment variables
-#
-#   SPARK_WORKER_INSTANCES The number of worker instances that should be
-#                          running on this slave.  Default is 1.
-
-# Usage: decommission-slave.sh [--block-until-exit]
-#   Decommissions all slaves on this worker machine
-
-set -ex
-
-if [ -z "${SPARK_HOME}" ]; then
-  export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
-fi
-
-. "${SPARK_HOME}/sbin/spark-config.sh"
-
-. "${SPARK_HOME}/bin/load-spark-env.sh"
-
-if [ "$SPARK_WORKER_INSTANCES" = "" ]; then
-  "${SPARK_HOME}/sbin"/spark-daemon.sh decommission org.apache.spark.deploy.worker.Worker 1
-else
-  for ((i=0; i<$SPARK_WORKER_INSTANCES; i++)); do
-    "${SPARK_HOME}/sbin"/spark-daemon.sh decommission org.apache.spark.deploy.worker.Worker $(( $i + 1 ))
-  done
-fi
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
-# Check if --block-until-exit is set.
-# This is done for systems which block on the decomissioning script and on exit
-# shut down the entire system (e.g. K8s).
-if [ "$1" == "--block-until-exit" ]; then
-  shift
-  # For now we only block on the 0th instance if there multiple instances.
-  instance=$1
-  pid="$SPARK_PID_DIR/spark-$SPARK_IDENT_STRING-$command-$instance.pid"
-  wait $pid
-fi
+>&2 echo "This script is deprecated, use decommission-worker.sh"
+"${DIR}/decommission-worker.sh" "$@"
diff --git a/sbin/decommission-worker.sh b/sbin/decommission-worker.sh
new file mode 100755
index 0000000000000..07e1e1771f7c1
--- /dev/null
+++ b/sbin/decommission-worker.sh
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# A shell script to decommission all workers on a single worker
+#
+# Environment variables
+#
+#   SPARK_WORKER_INSTANCES The number of worker instances that should be
+#                          running on this worker machine.  Default is 1.
+
+# Usage: decommission-worker.sh [--block-until-exit]
+#   Decommissions all workers on this worker machine.
+
+set -ex
+
+if [ -z "${SPARK_HOME}" ]; then
+  export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
+
+. "${SPARK_HOME}/sbin/spark-config.sh"
+
+. "${SPARK_HOME}/bin/load-spark-env.sh"
+
+if [ "$SPARK_WORKER_INSTANCES" = "" ]; then
+  "${SPARK_HOME}/sbin"/spark-daemon.sh decommission org.apache.spark.deploy.worker.Worker 1
+else
+  for ((i=0; i<$SPARK_WORKER_INSTANCES; i++)); do
+    "${SPARK_HOME}/sbin"/spark-daemon.sh decommission org.apache.spark.deploy.worker.Worker $(( $i + 1 ))
+  done
+fi
+
+# Check if --block-until-exit is set.
+# This is done for systems which block on the decommissioning script and on exit
+# shut down the entire system (e.g. K8s).
+if [ "$1" == "--block-until-exit" ]; then
+  shift
+  # For now we only block on the 0th instance if there multiple instances.
+  instance=$1
+  pid="$SPARK_PID_DIR/spark-$SPARK_IDENT_STRING-$command-$instance.pid"
+  wait $pid
+fi
diff --git a/sbin/slaves.sh b/sbin/slaves.sh
index c971aa3296b09..b92007ecdfad5 100755
--- a/sbin/slaves.sh
+++ b/sbin/slaves.sh
@@ -17,87 +17,7 @@
 # limitations under the License.
 #
 
-# Run a shell command on all slave hosts.
-#
-# Environment Variables
-#
-#   SPARK_SLAVES    File naming remote hosts.
-#     Default is ${SPARK_CONF_DIR}/slaves.
-#   SPARK_CONF_DIR  Alternate conf dir. Default is ${SPARK_HOME}/conf.
-#   SPARK_SLAVE_SLEEP Seconds to sleep between spawning remote commands.
-#   SPARK_SSH_OPTS Options passed to ssh when running remote commands.
-##
-
-usage="Usage: slaves.sh [--config <conf-dir>] command..."
-
-# if no args specified, show usage
-if [ $# -le 0 ]; then
-  echo $usage
-  exit 1
-fi
-
-if [ -z "${SPARK_HOME}" ]; then
-  export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
-fi
-
-. "${SPARK_HOME}/sbin/spark-config.sh"
-
-# If the slaves file is specified in the command line,
-# then it takes precedence over the definition in
-# spark-env.sh. Save it here.
-if [ -f "$SPARK_SLAVES" ]; then
-  HOSTLIST=`cat "$SPARK_SLAVES"`
-fi
-
-# Check if --config is passed as an argument. It is an optional parameter.
-# Exit if the argument is not a directory.
-if [ "$1" == "--config" ]
-then
-  shift
-  conf_dir="$1"
-  if [ ! -d "$conf_dir" ]
-  then
-    echo "ERROR : $conf_dir is not a directory"
-    echo $usage
-    exit 1
-  else
-    export SPARK_CONF_DIR="$conf_dir"
-  fi
-  shift
-fi
-
-. "${SPARK_HOME}/bin/load-spark-env.sh"
-
-if [ "$HOSTLIST" = "" ]; then
-  if [ "$SPARK_SLAVES" = "" ]; then
-    if [ -f "${SPARK_CONF_DIR}/slaves" ]; then
-      HOSTLIST=`cat "${SPARK_CONF_DIR}/slaves"`
-    else
-      HOSTLIST=localhost
-    fi
-  else
-    HOSTLIST=`cat "${SPARK_SLAVES}"`
-  fi
-fi
-
-
-
-# By default disable strict host key checking
-if [ "$SPARK_SSH_OPTS" = "" ]; then
-  SPARK_SSH_OPTS="-o StrictHostKeyChecking=no"
-fi
-
-for slave in `echo "$HOSTLIST"|sed  "s/#.*$//;/^$/d"`; do
-  if [ -n "${SPARK_SSH_FOREGROUND}" ]; then
-    ssh $SPARK_SSH_OPTS "$slave" $"${@// /\\ }" \
-      2>&1 | sed "s/^/$slave: /"
-  else
-    ssh $SPARK_SSH_OPTS "$slave" $"${@// /\\ }" \
-      2>&1 | sed "s/^/$slave: /" &
-  fi
-  if [ "$SPARK_SLAVE_SLEEP" != "" ]; then
-    sleep $SPARK_SLAVE_SLEEP
-  fi
-done
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
-wait
+>&2 echo "This script is deprecated, use workers.sh"
+"${DIR}/workers.sh" "$@"
diff --git a/sbin/spark-config.sh b/sbin/spark-config.sh
index b53442ec096a1..ae8b2231f504b 100755
--- a/sbin/spark-config.sh
+++ b/sbin/spark-config.sh
@@ -28,6 +28,6 @@ export SPARK_CONF_DIR="${SPARK_CONF_DIR:-"${SPARK_HOME}/conf"}"
 # Add the PySpark classes to the PYTHONPATH:
 if [ -z "${PYSPARK_PYTHONPATH_SET}" ]; then
   export PYTHONPATH="${SPARK_HOME}/python:${PYTHONPATH}"
-  export PYTHONPATH="${SPARK_HOME}/python/lib/py4j-0.10.9-src.zip:${PYTHONPATH}"
+  export PYTHONPATH="${SPARK_HOME}/python/lib/py4j-0.10.9.2-src.zip:${PYTHONPATH}"
   export PYSPARK_PYTHONPATH_SET=1
 fi
diff --git a/sbin/spark-daemon.sh b/sbin/spark-daemon.sh
index 81f2fd40a706f..e563f7bff1667 100755
--- a/sbin/spark-daemon.sh
+++ b/sbin/spark-daemon.sh
@@ -23,6 +23,7 @@
 #
 #   SPARK_CONF_DIR  Alternate conf dir. Default is ${SPARK_HOME}/conf.
 #   SPARK_LOG_DIR   Where log files are stored. ${SPARK_HOME}/logs by default.
+#   SPARK_LOG_MAX_FILES Max log files of Spark daemons can rotate to. Default is 5.
 #   SPARK_MASTER    host:path where spark code should be rsync'd from
 #   SPARK_PID_DIR   The pid files are stored. /tmp by default.
 #   SPARK_IDENT_STRING   A string representing this instance of spark. $USER by default
@@ -74,10 +75,16 @@ shift
 spark_rotate_log ()
 {
     log=$1;
-    num=5;
-    if [ -n "$2" ]; then
-	num=$2
+
+    if [[ -z ${SPARK_LOG_MAX_FILES} ]]; then
+      num=5
+    elif [[ ${SPARK_LOG_MAX_FILES} -gt 0 ]]; then
+      num=${SPARK_LOG_MAX_FILES}
+    else
+      echo "Error: SPARK_LOG_MAX_FILES must be a positive number, but got ${SPARK_LOG_MAX_FILES}"
+      exit -1
     fi
+
     if [ -f "$log" ]; then # rotate logs
 	while [ $num -gt 1 ]; do
 	    prev=`expr $num - 1`
diff --git a/sbin/spark-daemons.sh b/sbin/spark-daemons.sh
index dec2f4432df39..9a5e5f3a09c1d 100755
--- a/sbin/spark-daemons.sh
+++ b/sbin/spark-daemons.sh
@@ -17,7 +17,7 @@
 # limitations under the License.
 #
 
-# Run a Spark command on all slave hosts.
+# Run a Spark command on all worker hosts.
 
 usage="Usage: spark-daemons.sh [--config <conf-dir>] [start|stop] command instance-number args..."
 
@@ -33,4 +33,4 @@ fi
 
 . "${SPARK_HOME}/sbin/spark-config.sh"
 
-exec "${SPARK_HOME}/sbin/slaves.sh" cd "${SPARK_HOME}" \; "${SPARK_HOME}/sbin/spark-daemon.sh" "$@"
+exec "${SPARK_HOME}/sbin/workers.sh" cd "${SPARK_HOME}" \; "${SPARK_HOME}/sbin/spark-daemon.sh" "$@"
diff --git a/sbin/start-all.sh b/sbin/start-all.sh
index a5d30d274ea6e..064074e07922b 100755
--- a/sbin/start-all.sh
+++ b/sbin/start-all.sh
@@ -19,7 +19,7 @@
 
 # Start all spark daemons.
 # Starts the master on this node.
-# Starts a worker on each node specified in conf/slaves
+# Starts a worker on each node specified in conf/workers
 
 if [ -z "${SPARK_HOME}" ]; then
   export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
@@ -32,4 +32,4 @@ fi
 "${SPARK_HOME}/sbin"/start-master.sh
 
 # Start Workers
-"${SPARK_HOME}/sbin"/start-slaves.sh
+"${SPARK_HOME}/sbin"/start-workers.sh
diff --git a/sbin/start-slave.sh b/sbin/start-slave.sh
index 9b3b26b07842b..68682532f02ee 100755
--- a/sbin/start-slave.sh
+++ b/sbin/start-slave.sh
@@ -17,76 +17,7 @@
 # limitations under the License.
 #
 
-# Starts a slave on the machine this script is executed on.
-#
-# Environment Variables
-#
-#   SPARK_WORKER_INSTANCES  The number of worker instances to run on this
-#                           slave.  Default is 1. Note it has been deprecate since Spark 3.0.
-#   SPARK_WORKER_PORT       The base port number for the first worker. If set,
-#                           subsequent workers will increment this number.  If
-#                           unset, Spark will find a valid port number, but
-#                           with no guarantee of a predictable pattern.
-#   SPARK_WORKER_WEBUI_PORT The base port for the web interface of the first
-#                           worker.  Subsequent workers will increment this
-#                           number.  Default is 8081.
-
-if [ -z "${SPARK_HOME}" ]; then
-  export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
-fi
-
-# NOTE: This exact class name is matched downstream by SparkSubmit.
-# Any changes need to be reflected there.
-CLASS="org.apache.spark.deploy.worker.Worker"
-
-if [[ $# -lt 1 ]] || [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then
-  echo "Usage: ./sbin/start-slave.sh <master> [options]"
-  pattern="Usage:"
-  pattern+="\|Using Spark's default log4j profile:"
-  pattern+="\|Started daemon with process name"
-  pattern+="\|Registered signal handler for"
-
-  "${SPARK_HOME}"/bin/spark-class $CLASS --help 2>&1 | grep -v "$pattern" 1>&2
-  exit 1
-fi
-
-. "${SPARK_HOME}/sbin/spark-config.sh"
-
-. "${SPARK_HOME}/bin/load-spark-env.sh"
-
-# First argument should be the master; we need to store it aside because we may
-# need to insert arguments between it and the other arguments
-MASTER=$1
-shift
-
-# Determine desired worker port
-if [ "$SPARK_WORKER_WEBUI_PORT" = "" ]; then
-  SPARK_WORKER_WEBUI_PORT=8081
-fi
-
-# Start up the appropriate number of workers on this machine.
-# quick local function to start a worker
-function start_instance {
-  WORKER_NUM=$1
-  shift
-
-  if [ "$SPARK_WORKER_PORT" = "" ]; then
-    PORT_FLAG=
-    PORT_NUM=
-  else
-    PORT_FLAG="--port"
-    PORT_NUM=$(( $SPARK_WORKER_PORT + $WORKER_NUM - 1 ))
-  fi
-  WEBUI_PORT=$(( $SPARK_WORKER_WEBUI_PORT + $WORKER_NUM - 1 ))
-
-  "${SPARK_HOME}/sbin"/spark-daemon.sh start $CLASS $WORKER_NUM \
-     --webui-port "$WEBUI_PORT" $PORT_FLAG $PORT_NUM $MASTER "$@"
-}
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
-if [ "$SPARK_WORKER_INSTANCES" = "" ]; then
-  start_instance 1 "$@"
-else
-  for ((i=0; i<$SPARK_WORKER_INSTANCES; i++)); do
-    start_instance $(( 1 + $i )) "$@"
-  done
-fi
+>&2 echo "This script is deprecated, use start-worker.sh"
+"${DIR}/start-worker.sh" "$@"
diff --git a/sbin/start-slaves.sh b/sbin/start-slaves.sh
index f5269df523dac..9b113d9f2e0f4 100755
--- a/sbin/start-slaves.sh
+++ b/sbin/start-slaves.sh
@@ -17,30 +17,7 @@
 # limitations under the License.
 #
 
-# Starts a slave instance on each machine specified in the conf/slaves file.
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
-if [ -z "${SPARK_HOME}" ]; then
-  export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
-fi
-
-. "${SPARK_HOME}/sbin/spark-config.sh"
-. "${SPARK_HOME}/bin/load-spark-env.sh"
-
-# Find the port number for the master
-if [ "$SPARK_MASTER_PORT" = "" ]; then
-  SPARK_MASTER_PORT=7077
-fi
-
-if [ "$SPARK_MASTER_HOST" = "" ]; then
-  case `uname` in
-      (SunOS)
-	  SPARK_MASTER_HOST="`/usr/sbin/check-hostname | awk '{print $NF}'`"
-	  ;;
-      (*)
-	  SPARK_MASTER_HOST="`hostname -f`"
-	  ;;
-  esac
-fi
-
-# Launch the slaves
-"${SPARK_HOME}/sbin/slaves.sh" cd "${SPARK_HOME}" \; "${SPARK_HOME}/sbin/start-slave.sh" "spark://$SPARK_MASTER_HOST:$SPARK_MASTER_PORT"
+>&2 echo "This script is deprecated, use start-workers.sh"
+"${DIR}/start-workers.sh" "$@"
diff --git a/sbin/start-worker.sh b/sbin/start-worker.sh
new file mode 100755
index 0000000000000..fd58f01bac2eb
--- /dev/null
+++ b/sbin/start-worker.sh
@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Starts a worker on the machine this script is executed on.
+#
+# Environment Variables
+#
+#   SPARK_WORKER_INSTANCES  The number of worker instances to run on this
+#                           worker.  Default is 1. Note it has been deprecate since Spark 3.0.
+#   SPARK_WORKER_PORT       The base port number for the first worker. If set,
+#                           subsequent workers will increment this number.  If
+#                           unset, Spark will find a valid port number, but
+#                           with no guarantee of a predictable pattern.
+#   SPARK_WORKER_WEBUI_PORT The base port for the web interface of the first
+#                           worker.  Subsequent workers will increment this
+#                           number.  Default is 8081.
+
+if [ -z "${SPARK_HOME}" ]; then
+  export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
+
+# NOTE: This exact class name is matched downstream by SparkSubmit.
+# Any changes need to be reflected there.
+CLASS="org.apache.spark.deploy.worker.Worker"
+
+if [[ $# -lt 1 ]] || [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then
+  echo "Usage: ./sbin/start-worker.sh <master> [options]"
+  pattern="Usage:"
+  pattern+="\|Using Spark's default log4j profile:"
+  pattern+="\|Started daemon with process name"
+  pattern+="\|Registered signal handler for"
+
+  "${SPARK_HOME}"/bin/spark-class $CLASS --help 2>&1 | grep -v "$pattern" 1>&2
+  exit 1
+fi
+
+. "${SPARK_HOME}/sbin/spark-config.sh"
+
+. "${SPARK_HOME}/bin/load-spark-env.sh"
+
+# First argument should be the master; we need to store it aside because we may
+# need to insert arguments between it and the other arguments
+MASTER=$1
+shift
+
+# Determine desired worker port
+if [ "$SPARK_WORKER_WEBUI_PORT" = "" ]; then
+  SPARK_WORKER_WEBUI_PORT=8081
+fi
+
+# Start up the appropriate number of workers on this machine.
+# quick local function to start a worker
+function start_instance {
+  WORKER_NUM=$1
+  shift
+
+  if [ "$SPARK_WORKER_PORT" = "" ]; then
+    PORT_FLAG=
+    PORT_NUM=
+  else
+    PORT_FLAG="--port"
+    PORT_NUM=$(( $SPARK_WORKER_PORT + $WORKER_NUM - 1 ))
+  fi
+  WEBUI_PORT=$(( $SPARK_WORKER_WEBUI_PORT + $WORKER_NUM - 1 ))
+
+  "${SPARK_HOME}/sbin"/spark-daemon.sh start $CLASS $WORKER_NUM \
+     --webui-port "$WEBUI_PORT" $PORT_FLAG $PORT_NUM $MASTER "$@"
+}
+
+if [ "$SPARK_WORKER_INSTANCES" = "" ]; then
+  start_instance 1 "$@"
+else
+  for ((i=0; i<$SPARK_WORKER_INSTANCES; i++)); do
+    start_instance $(( 1 + $i )) "$@"
+  done
+fi
diff --git a/sbin/start-workers.sh b/sbin/start-workers.sh
new file mode 100755
index 0000000000000..3867ef3ccf255
--- /dev/null
+++ b/sbin/start-workers.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Starts a worker instance on each machine specified in the conf/workers file.
+
+if [ -z "${SPARK_HOME}" ]; then
+  export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
+
+. "${SPARK_HOME}/sbin/spark-config.sh"
+. "${SPARK_HOME}/bin/load-spark-env.sh"
+
+# Find the port number for the master
+if [ "$SPARK_MASTER_PORT" = "" ]; then
+  SPARK_MASTER_PORT=7077
+fi
+
+if [ "$SPARK_MASTER_HOST" = "" ]; then
+  case `uname` in
+      (SunOS)
+          SPARK_MASTER_HOST="`/usr/sbin/check-hostname | awk '{print $NF}'`"
+          ;;
+      (*)
+          SPARK_MASTER_HOST="`hostname -f`"
+          ;;
+  esac
+fi
+
+# Launch the workers
+"${SPARK_HOME}/sbin/workers.sh" cd "${SPARK_HOME}" \; "${SPARK_HOME}/sbin/start-worker.sh" "spark://$SPARK_MASTER_HOST:$SPARK_MASTER_PORT"
diff --git a/sbin/stop-all.sh b/sbin/stop-all.sh
index 4e476ca05cb05..2c40905cd499b 100755
--- a/sbin/stop-all.sh
+++ b/sbin/stop-all.sh
@@ -27,8 +27,8 @@ fi
 # Load the Spark configuration
 . "${SPARK_HOME}/sbin/spark-config.sh"
 
-# Stop the slaves, then the master
-"${SPARK_HOME}/sbin"/stop-slaves.sh
+# Stop the workers, then the master
+"${SPARK_HOME}/sbin"/stop-workers.sh
 "${SPARK_HOME}/sbin"/stop-master.sh
 
 if [ "$1" == "--wait" ]
@@ -36,7 +36,7 @@ then
   printf "Waiting for workers to shut down..."
   while true
   do
-    running=`${SPARK_HOME}/sbin/slaves.sh ps -ef | grep -v grep | grep deploy.worker.Worker`
+    running=`${SPARK_HOME}/sbin/workers.sh ps -ef | grep -v grep | grep deploy.worker.Worker`
     if [ -z "$running" ]
     then
       printf "\nAll workers successfully shut down.\n"
diff --git a/sbin/stop-slave.sh b/sbin/stop-slave.sh
index 685bcf59b33aa..71ed29987d4a1 100755
--- a/sbin/stop-slave.sh
+++ b/sbin/stop-slave.sh
@@ -17,28 +17,7 @@
 # limitations under the License.
 #
 
-# A shell script to stop all workers on a single slave
-#
-# Environment variables
-#
-#   SPARK_WORKER_INSTANCES The number of worker instances that should be
-#                          running on this slave.  Default is 1.
-
-# Usage: stop-slave.sh
-#   Stops all slaves on this worker machine
-
-if [ -z "${SPARK_HOME}" ]; then
-  export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
-fi
-
-. "${SPARK_HOME}/sbin/spark-config.sh"
-
-. "${SPARK_HOME}/bin/load-spark-env.sh"
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
-if [ "$SPARK_WORKER_INSTANCES" = "" ]; then
-  "${SPARK_HOME}/sbin"/spark-daemon.sh stop org.apache.spark.deploy.worker.Worker 1
-else
-  for ((i=0; i<$SPARK_WORKER_INSTANCES; i++)); do
-    "${SPARK_HOME}/sbin"/spark-daemon.sh stop org.apache.spark.deploy.worker.Worker $(( $i + 1 ))
-  done
-fi
+>&2 echo "This script is deprecated, use stop-worker.sh"
+"${DIR}/stop-worker.sh" "$@"
diff --git a/sbin/stop-slaves.sh b/sbin/stop-slaves.sh
index a57441b52a04a..c0aca6868efe3 100755
--- a/sbin/stop-slaves.sh
+++ b/sbin/stop-slaves.sh
@@ -17,12 +17,7 @@
 # limitations under the License.
 #
 
-if [ -z "${SPARK_HOME}" ]; then
-  export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
-fi
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
-. "${SPARK_HOME}/sbin/spark-config.sh"
-
-. "${SPARK_HOME}/bin/load-spark-env.sh"
-
-"${SPARK_HOME}/sbin/slaves.sh" cd "${SPARK_HOME}" \; "${SPARK_HOME}/sbin"/stop-slave.sh
+>&2 echo "This script is deprecated, use stop-workers.sh"
+"${DIR}/stop-workers.sh" "$@"
diff --git a/sbin/stop-worker.sh b/sbin/stop-worker.sh
new file mode 100755
index 0000000000000..112b62ecffa27
--- /dev/null
+++ b/sbin/stop-worker.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# A shell script to stop all workers on a single worker
+#
+# Environment variables
+#
+#   SPARK_WORKER_INSTANCES The number of worker instances that should be
+#                          running on this worker machine.  Default is 1.
+
+# Usage: stop-worker.sh
+#   Stops all workers on this worker machine
+
+if [ -z "${SPARK_HOME}" ]; then
+  export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
+
+. "${SPARK_HOME}/sbin/spark-config.sh"
+
+. "${SPARK_HOME}/bin/load-spark-env.sh"
+
+if [ "$SPARK_WORKER_INSTANCES" = "" ]; then
+  "${SPARK_HOME}/sbin"/spark-daemon.sh stop org.apache.spark.deploy.worker.Worker 1
+else
+  for ((i=0; i<$SPARK_WORKER_INSTANCES; i++)); do
+    "${SPARK_HOME}/sbin"/spark-daemon.sh stop org.apache.spark.deploy.worker.Worker $(( $i + 1 ))
+  done
+fi
diff --git a/sbin/stop-workers.sh b/sbin/stop-workers.sh
new file mode 100755
index 0000000000000..552800f522222
--- /dev/null
+++ b/sbin/stop-workers.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [ -z "${SPARK_HOME}" ]; then
+  export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
+
+. "${SPARK_HOME}/sbin/spark-config.sh"
+
+. "${SPARK_HOME}/bin/load-spark-env.sh"
+
+"${SPARK_HOME}/sbin/workers.sh" cd "${SPARK_HOME}" \; "${SPARK_HOME}/sbin"/stop-worker.sh
diff --git a/sbin/workers.sh b/sbin/workers.sh
new file mode 100755
index 0000000000000..cab0330723a6c
--- /dev/null
+++ b/sbin/workers.sh
@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Run a shell command on all worker hosts.
+#
+# Environment Variables
+#
+#   SPARK_WORKERS    File naming remote hosts.
+#     Default is ${SPARK_CONF_DIR}/workers.
+#   SPARK_CONF_DIR  Alternate conf dir. Default is ${SPARK_HOME}/conf.
+#   SPARK_WORKER_SLEEP Seconds to sleep between spawning remote commands.
+#   SPARK_SSH_OPTS Options passed to ssh when running remote commands.
+##
+
+usage="Usage: workers.sh [--config <conf-dir>] command..."
+
+# if no args specified, show usage
+if [ $# -le 0 ]; then
+  echo $usage
+  exit 1
+fi
+
+if [ -z "${SPARK_HOME}" ]; then
+  export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
+
+. "${SPARK_HOME}/sbin/spark-config.sh"
+
+# If the workers file is specified in the command line,
+# then it takes precedence over the definition in
+# spark-env.sh. Save it here.
+if [ -f "$SPARK_WORKERS" ]; then
+  HOSTLIST=`cat "$SPARK_WORKERS"`
+fi
+if [ -f "$SPARK_SLAVES" ]; then
+  >&2 echo "SPARK_SLAVES is deprecated, use SPARK_WORKERS"
+  HOSTLIST=`cat "$SPARK_SLAVES"`
+fi
+
+
+# Check if --config is passed as an argument. It is an optional parameter.
+# Exit if the argument is not a directory.
+if [ "$1" == "--config" ]
+then
+  shift
+  conf_dir="$1"
+  if [ ! -d "$conf_dir" ]
+  then
+    echo "ERROR : $conf_dir is not a directory"
+    echo $usage
+    exit 1
+  else
+    export SPARK_CONF_DIR="$conf_dir"
+  fi
+  shift
+fi
+
+. "${SPARK_HOME}/bin/load-spark-env.sh"
+
+if [ "$HOSTLIST" = "" ]; then
+  if [ "$SPARK_SLAVES" = "" ] && [ "$SPARK_WORKERS" = "" ]; then
+    if [ -f "${SPARK_CONF_DIR}/workers" ]; then
+      HOSTLIST=`cat "${SPARK_CONF_DIR}/workers"`
+    elif [ -f "${SPARK_CONF_DIR}/slaves" ]; then
+      HOSTLIST=`cat "${SPARK_CONF_DIR}/slaves"`
+    else
+      HOSTLIST=localhost
+    fi
+  else
+    if [ -f "$SPARK_WORKERS" ]; then
+      HOSTLIST=`cat "$SPARK_WORKERS"`
+    fi
+    if [ -f "$SPARK_SLAVES" ]; then
+      >&2 echo "SPARK_SLAVES is deprecated, use SPARK_WORKERS"
+      HOSTLIST=`cat "$SPARK_SLAVES"`
+    fi
+  fi
+fi
+
+
+
+# By default disable strict host key checking
+if [ "$SPARK_SSH_OPTS" = "" ]; then
+  SPARK_SSH_OPTS="-o StrictHostKeyChecking=no"
+fi
+
+for host in `echo "$HOSTLIST"|sed  "s/#.*$//;/^$/d"`; do
+  if [ -n "${SPARK_SSH_FOREGROUND}" ]; then
+    ssh $SPARK_SSH_OPTS "$host" $"${@// /\\ }" \
+      2>&1 | sed "s/^/$host: /"
+  else
+    ssh $SPARK_SSH_OPTS "$host" $"${@// /\\ }" \
+      2>&1 | sed "s/^/$host: /" &
+  fi
+  if [ "$SPARK_WORKER_SLEEP" != "" ]; then
+    sleep $SPARK_WORKER_SLEEP
+  fi
+  if [ "$SPARK_SLAVE_SLEEP" != "" ]; then
+    >&2 echo "SPARK_SLAVE_SLEEP is deprecated, use SPARK_WORKER_SLEEP"
+    sleep $SPARK_SLAVE_SLEEP
+  fi
+done
+
+wait
diff --git a/scalastyle-config.xml b/scalastyle-config.xml
index 73ac14fdba1cf..c1dc57be56dff 100644
--- a/scalastyle-config.xml
+++ b/scalastyle-config.xml
@@ -264,6 +264,20 @@ This file is divided into 3 sections:
     of Commons Lang 2 (package org.apache.commons.lang.*)</customMessage>
   </check>
 
+  <check customId="FileSystemGet" level="error" class="org.scalastyle.file.RegexChecker" enabled="true">
+    <parameters><parameter name="regex">FileSystem.get\([a-zA-Z_$][a-zA-Z_$0-9]*\)</parameter></parameters>
+    <customMessage><![CDATA[
+      Are you sure that you want to use "FileSystem.get(Configuration conf)"? If the input
+      configuration is not set properly, a default FileSystem instance will be returned. It can
+      lead to errors when you deal with multiple file systems. Please consider using
+      "FileSystem.get(URI uri, Configuration conf)" or "Path.getFileSystem(Configuration conf)" instead.
+      If you must use the method "FileSystem.get(Configuration conf)", wrap the code block with
+      // scalastyle:off FileSystemGet
+      FileSystem.get(...)
+      // scalastyle:on FileSystemGet
+    ]]></customMessage>
+  </check>
+
   <check customId="extractopt" level="error" class="org.scalastyle.scalariform.TokenChecker" enabled="true">
     <parameters><parameter name="regex">extractOpt</parameter></parameters>
     <customMessage>Use jsonOption(x).map(.extract[T]) instead of .extractOpt[T], as the latter
diff --git a/sql/catalyst/benchmarks/GenericArrayDataBenchmark-jdk11-results.txt b/sql/catalyst/benchmarks/GenericArrayDataBenchmark-jdk11-results.txt
index 6a5a9b781e894..a3ef8452bb03c 100644
--- a/sql/catalyst/benchmarks/GenericArrayDataBenchmark-jdk11-results.txt
+++ b/sql/catalyst/benchmarks/GenericArrayDataBenchmark-jdk11-results.txt
@@ -1,10 +1,10 @@
-OpenJDK 64-Bit Server VM 11.0.5+10 on Mac OS X 10.14.6
-Intel(R) Core(TM) i5-8210Y CPU @ 1.60GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 constructor:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-arrayOfAny                                            6              7           1       1770.9           0.6       1.0X
-arrayOfAnyAsObject                                    6              7           2       1709.3           0.6       1.0X
-arrayOfAnyAsSeq                                       5              6           2       2195.5           0.5       1.2X
-arrayOfInt                                          452            469          13         22.1          45.2       0.0X
-arrayOfIntAsObject                                  678            690          11         14.7          67.8       0.0X
+arrayOfAny                                            5              6           1       1957.6           0.5       1.0X
+arrayOfAnyAsObject                                    5              6           1       2001.6           0.5       1.0X
+arrayOfAnyAsSeq                                       5              6           1       2040.4           0.5       1.0X
+arrayOfInt                                          578            595          20         17.3          57.8       0.0X
+arrayOfIntAsObject                                  830            847          18         12.0          83.0       0.0X
 
diff --git a/sql/catalyst/benchmarks/GenericArrayDataBenchmark-results.txt b/sql/catalyst/benchmarks/GenericArrayDataBenchmark-results.txt
index 02971749662f5..88de9c13130d0 100644
--- a/sql/catalyst/benchmarks/GenericArrayDataBenchmark-results.txt
+++ b/sql/catalyst/benchmarks/GenericArrayDataBenchmark-results.txt
@@ -1,10 +1,10 @@
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_191-b12 on Mac OS X 10.14.6
-Intel(R) Core(TM) i5-8210Y CPU @ 1.60GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 constructor:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-arrayOfAny                                            7              8           2       1471.6           0.7       1.0X
-arrayOfAnyAsObject                                  197            207           9         50.7          19.7       0.0X
-arrayOfAnyAsSeq                                      25             27           2        398.0           2.5       0.3X
-arrayOfInt                                          613            630          15         16.3          61.3       0.0X
-arrayOfIntAsObject                                  866            872           8         11.5          86.6       0.0X
+arrayOfAny                                            3              4           0       2907.2           0.3       1.0X
+arrayOfAnyAsObject                                  274            276           2         36.5          27.4       0.0X
+arrayOfAnyAsSeq                                      21             23           1        470.1           2.1       0.2X
+arrayOfInt                                          727            735           8         13.8          72.7       0.0X
+arrayOfIntAsObject                                  998           1002           6         10.0          99.8       0.0X
 
diff --git a/sql/catalyst/benchmarks/HashBenchmark-jdk11-results.txt b/sql/catalyst/benchmarks/HashBenchmark-jdk11-results.txt
index d6452891a3ddd..6454809cd6218 100644
--- a/sql/catalyst/benchmarks/HashBenchmark-jdk11-results.txt
+++ b/sql/catalyst/benchmarks/HashBenchmark-jdk11-results.txt
@@ -2,69 +2,69 @@
 single ints
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Hash For single ints:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-interpreted version                                3285           3286           1        163.4           6.1       1.0X
-codegen version                                    6838           6838           0         78.5          12.7       0.5X
-codegen version 64-bit                             6247           6247           0         85.9          11.6       0.5X
-codegen HiveHash version                           4927           4927           0        109.0           9.2       0.7X
+interpreted version                                4452           4453           1        120.6           8.3       1.0X
+codegen version                                    5888           5942          77         91.2          11.0       0.8X
+codegen version 64-bit                             5760           5782          30         93.2          10.7       0.8X
+codegen HiveHash version                           4697           4744          67        114.3           8.7       0.9X
 
 
 ================================================================================================
 single longs
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Hash For single longs:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-interpreted version                                3615           3615           0        148.5           6.7       1.0X
-codegen version                                    9630           9633           5         55.8          17.9       0.4X
-codegen version 64-bit                             6763           6764           2         79.4          12.6       0.5X
-codegen HiveHash version                           5709           5711           2         94.0          10.6       0.6X
+interpreted version                                5628           5654          36         95.4          10.5       1.0X
+codegen version                                    8911           8913           4         60.3          16.6       0.6X
+codegen version 64-bit                             6828           6830           3         78.6          12.7       0.8X
+codegen HiveHash version                           5731           5734           4         93.7          10.7       1.0X
 
 
 ================================================================================================
 normal
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Hash For normal:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-interpreted version                                2946           2965          27          0.7        1404.7       1.0X
-codegen version                                    2386           2386           1          0.9        1137.7       1.2X
-codegen version 64-bit                              895            895           0          2.3         426.8       3.3X
-codegen HiveHash version                           4454           4463          14          0.5        2123.6       0.7X
+interpreted version                                3207           3212           7          0.7        1529.4       1.0X
+codegen version                                    2518           2522           6          0.8        1200.9       1.3X
+codegen version 64-bit                             1004           1009           7          2.1         478.7       3.2X
+codegen HiveHash version                           5074           5076           3          0.4        2419.5       0.6X
 
 
 ================================================================================================
 array
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Hash For array:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-interpreted version                                2325           2374          68          0.1       17740.6       1.0X
-codegen version                                    4928           4977          69          0.0       37597.1       0.5X
-codegen version 64-bit                             3404           3408           5          0.0       25973.0       0.7X
-codegen HiveHash version                           1491           1512          30          0.1       11376.3       1.6X
+interpreted version                                2309           2314           7          0.1       17619.7       1.0X
+codegen version                                    4207           4214           9          0.0       32098.7       0.5X
+codegen version 64-bit                             3518           3522           6          0.0       26842.6       0.7X
+codegen HiveHash version                           1681           1698          24          0.1       12823.2       1.4X
 
 
 ================================================================================================
 map
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Hash For map:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-interpreted version                                   0              0           0         45.1          22.2       1.0X
-codegen version                                     271            278           8          0.0       66062.7       0.0X
-codegen version 64-bit                              208            210           2          0.0       50775.0       0.0X
-codegen HiveHash version                             52             53           0          0.1       12794.9       0.0X
+interpreted version                                   0              0           0         42.7          23.4       1.0X
+codegen version                                     207            209           1          0.0       50608.9       0.0X
+codegen version 64-bit                              203            204           1          0.0       49444.7       0.0X
+codegen HiveHash version                             56             57           0          0.1       13725.9       0.0X
 
 
diff --git a/sql/catalyst/benchmarks/HashBenchmark-results.txt b/sql/catalyst/benchmarks/HashBenchmark-results.txt
index 8075df73399b9..a8596bc917cf1 100644
--- a/sql/catalyst/benchmarks/HashBenchmark-results.txt
+++ b/sql/catalyst/benchmarks/HashBenchmark-results.txt
@@ -2,69 +2,69 @@
 single ints
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 Hash For single ints:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-interpreted version                                3364           3364           0        159.6           6.3       1.0X
-codegen version                                    6921           6924           4         77.6          12.9       0.5X
-codegen version 64-bit                             6139           6140           2         87.5          11.4       0.5X
-codegen HiveHash version                           5097           5099           2        105.3           9.5       0.7X
+interpreted version                                3181           3184           4        168.8           5.9       1.0X
+codegen version                                    4371           4372           1        122.8           8.1       0.7X
+codegen version 64-bit                             3869           3870           1        138.7           7.2       0.8X
+codegen HiveHash version                           3633           3636           4        147.8           6.8       0.9X
 
 
 ================================================================================================
 single longs
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 Hash For single longs:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-interpreted version                                3838           3838           0        139.9           7.1       1.0X
-codegen version                                    9690           9703          19         55.4          18.0       0.4X
-codegen version 64-bit                             6876           6877           1         78.1          12.8       0.6X
-codegen HiveHash version                           5717           5718           1         93.9          10.6       0.7X
+interpreted version                                3747           3747           0        143.3           7.0       1.0X
+codegen version                                    7617           7626          12         70.5          14.2       0.5X
+codegen version 64-bit                             4622           4623           0        116.1           8.6       0.8X
+codegen HiveHash version                           4423           4424           1        121.4           8.2       0.8X
 
 
 ================================================================================================
 normal
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 Hash For normal:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-interpreted version                                2914           2918           6          0.7        1389.6       1.0X
-codegen version                                    2337           2341           6          0.9        1114.5       1.2X
-codegen version 64-bit                              910            911           3          2.3         433.9       3.2X
-codegen HiveHash version                           4479           4480           1          0.5        2135.9       0.7X
+interpreted version                                2760           2808          67          0.8        1316.1       1.0X
+codegen version                                    1901           1901           0          1.1         906.7       1.5X
+codegen version 64-bit                              769            771           3          2.7         366.5       3.6X
+codegen HiveHash version                           3713           3723          13          0.6        1770.6       0.7X
 
 
 ================================================================================================
 array
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 Hash For array:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-interpreted version                                2081           2133          74          0.1       15878.7       1.0X
-codegen version                                    4385           4506         172          0.0       33452.2       0.5X
-codegen version 64-bit                             3518           3552          48          0.0       26842.8       0.6X
-codegen HiveHash version                           1715           1752          52          0.1       13085.4       1.2X
+interpreted version                                1677           1692          21          0.1       12797.1       1.0X
+codegen version                                    2609           2697         124          0.1       19907.4       0.6X
+codegen version 64-bit                             2673           2692          27          0.0       20392.7       0.6X
+codegen HiveHash version                           1363           1382          28          0.1       10396.1       1.2X
 
 
 ================================================================================================
 map
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 Hash For map:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-interpreted version                                   0              0           0         50.4          19.8       1.0X
-codegen version                                     195            204           8          0.0       47681.7       0.0X
-codegen version 64-bit                              186            190           3          0.0       45296.8       0.0X
-codegen HiveHash version                             48             48           0          0.1       11610.3       0.0X
+interpreted version                                   0              0           0         46.7          21.4       1.0X
+codegen version                                     159            160           1          0.0       38910.2       0.0X
+codegen version 64-bit                              155            156           1          0.0       37908.4       0.0X
+codegen HiveHash version                             42             43           0          0.1       10321.6       0.0X
 
 
diff --git a/sql/catalyst/benchmarks/HashByteArrayBenchmark-jdk11-results.txt b/sql/catalyst/benchmarks/HashByteArrayBenchmark-jdk11-results.txt
index d8d43068929ed..5f3d0a657eb4e 100644
--- a/sql/catalyst/benchmarks/HashByteArrayBenchmark-jdk11-results.txt
+++ b/sql/catalyst/benchmarks/HashByteArrayBenchmark-jdk11-results.txt
@@ -2,76 +2,76 @@
 Benchmark for MurMurHash 3 and xxHash64
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Hash byte arrays with length 8:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Murmur3_x86_32                                       14             14           0        147.0           6.8       1.0X
-xxHash 64-bit                                        18             18           0        119.7           8.4       0.8X
-HiveHasher                                           16             16           0        129.9           7.7       0.9X
+Murmur3_x86_32                                       15             16           1        140.1           7.1       1.0X
+xxHash 64-bit                                        18             19           1        118.8           8.4       0.8X
+HiveHasher                                           17             18           1        122.4           8.2       0.9X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Hash byte arrays with length 16:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Murmur3_x86_32                                       23             23           0         89.8          11.1       1.0X
-xxHash 64-bit                                        22             23           0         93.3          10.7       1.0X
-HiveHasher                                           26             26           0         79.4          12.6       0.9X
+Murmur3_x86_32                                       23             24           1         90.1          11.1       1.0X
+xxHash 64-bit                                        21             22           2         98.5          10.2       1.1X
+HiveHasher                                           29             31           2         71.3          14.0       0.8X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Hash byte arrays with length 24:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Murmur3_x86_32                                       32             32           0         65.5          15.3       1.0X
-xxHash 64-bit                                        26             26           0         80.8          12.4       1.2X
-HiveHasher                                           41             41           0         50.8          19.7       0.8X
+Murmur3_x86_32                                       31             32           2         68.5          14.6       1.0X
+xxHash 64-bit                                        24             25           1         87.4          11.4       1.3X
+HiveHasher                                           45             48           3         46.8          21.4       0.7X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Hash byte arrays with length 31:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Murmur3_x86_32                                       49             49           0         43.2          23.2       1.0X
-xxHash 64-bit                                        44             44           0         48.0          20.8       1.1X
-HiveHasher                                           55             56           0         37.8          26.4       0.9X
+Murmur3_x86_32                                       48             50           2         44.0          22.7       1.0X
+xxHash 64-bit                                        42             45           2         50.2          19.9       1.1X
+HiveHasher                                           62             65           2         34.1          29.3       0.8X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Hash byte arrays with length 95:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Murmur3_x86_32                                      127            127           0         16.5          60.5       1.0X
-xxHash 64-bit                                        83             83           0         25.3          39.5       1.5X
-HiveHasher                                          196            196           0         10.7          93.3       0.6X
+Murmur3_x86_32                                      113            118           3         18.6          53.8       1.0X
+xxHash 64-bit                                        86             89           2         24.3          41.1       1.3X
+HiveHasher                                          218            226           7          9.6         103.9       0.5X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Hash byte arrays with length 287:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Murmur3_x86_32                                      327            327           0          6.4         155.9       1.0X
-xxHash 64-bit                                       138            138           0         15.2          65.9       2.4X
-HiveHasher                                          628            628           0          3.3         299.6       0.5X
+Murmur3_x86_32                                      346            352           6          6.1         165.2       1.0X
+xxHash 64-bit                                       155            158           2         13.5          73.8       2.2X
+HiveHasher                                          722            728           7          2.9         344.5       0.5X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Hash byte arrays with length 1055:        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Murmur3_x86_32                                     1145           1145           0          1.8         545.9       1.0X
-xxHash 64-bit                                       370            371           0          5.7         176.6       3.1X
-HiveHasher                                         2325           2325           0          0.9        1108.6       0.5X
+Murmur3_x86_32                                     1217           1224           9          1.7         580.3       1.0X
+xxHash 64-bit                                       412            417           3          5.1         196.5       3.0X
+HiveHasher                                         2593           2630          52          0.8        1236.6       0.5X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Hash byte arrays with length 2079:        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Murmur3_x86_32                                     2209           2211           3          0.9        1053.4       1.0X
-xxHash 64-bit                                       615            615           0          3.4         293.0       3.6X
-HiveHasher                                         4590           4590           0          0.5        2188.7       0.5X
+Murmur3_x86_32                                     2451           2454           4          0.9        1168.8       1.0X
+xxHash 64-bit                                       621            631           7          3.4         296.0       3.9X
+HiveHasher                                         5243           5295          73          0.4        2500.1       0.5X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Hash byte arrays with length 8223:        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Murmur3_x86_32                                     8687           8702          22          0.2        4142.2       1.0X
-xxHash 64-bit                                      2033           2034           1          1.0         969.5       4.3X
-HiveHasher                                        18216          18218           2          0.1        8686.1       0.5X
+Murmur3_x86_32                                     9280           9405         177          0.2        4425.1       1.0X
+xxHash 64-bit                                      1943           1945           3          1.1         926.5       4.8X
+HiveHasher                                        20457          20458           1          0.1        9754.6       0.5X
 
 
diff --git a/sql/catalyst/benchmarks/HashByteArrayBenchmark-results.txt b/sql/catalyst/benchmarks/HashByteArrayBenchmark-results.txt
index 83bd970e14392..35582ff03b5be 100644
--- a/sql/catalyst/benchmarks/HashByteArrayBenchmark-results.txt
+++ b/sql/catalyst/benchmarks/HashByteArrayBenchmark-results.txt
@@ -2,76 +2,76 @@
 Benchmark for MurMurHash 3 and xxHash64
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Hash byte arrays with length 8:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Murmur3_x86_32                                       15             15           0        138.0           7.2       1.0X
-xxHash 64-bit                                        17             17           0        125.7           8.0       0.9X
-HiveHasher                                           16             16           0        134.4           7.4       1.0X
+Murmur3_x86_32                                       11             12           1        195.8           5.1       1.0X
+xxHash 64-bit                                        11             13           1        190.4           5.3       1.0X
+HiveHasher                                           12             13           1        181.4           5.5       0.9X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Hash byte arrays with length 16:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Murmur3_x86_32                                       24             24           0         86.8          11.5       1.0X
-xxHash 64-bit                                        22             22           0         96.5          10.4       1.1X
-HiveHasher                                           31             31           0         66.8          15.0       0.8X
+Murmur3_x86_32                                       16             19           1        128.5           7.8       1.0X
+xxHash 64-bit                                        14             16           1        145.7           6.9       1.1X
+HiveHasher                                           22             25           1         93.6          10.7       0.7X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Hash byte arrays with length 24:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Murmur3_x86_32                                       34             34           0         61.9          16.2       1.0X
-xxHash 64-bit                                        26             26           0         79.6          12.6       1.3X
-HiveHasher                                           48             48           0         44.0          22.7       0.7X
+Murmur3_x86_32                                       23             26           2         90.3          11.1       1.0X
+xxHash 64-bit                                        18             20           1        119.6           8.4       1.3X
+HiveHasher                                           35             38           2         60.7          16.5       0.7X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Hash byte arrays with length 31:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Murmur3_x86_32                                       50             50           0         41.9          23.9       1.0X
-xxHash 64-bit                                        40             40           0         52.4          19.1       1.3X
-HiveHasher                                           61             61           0         34.4          29.1       0.8X
+Murmur3_x86_32                                       38             41           2         55.6          18.0       1.0X
+xxHash 64-bit                                        30             33           2         69.9          14.3       1.3X
+HiveHasher                                           48             53           2         43.7          22.9       0.8X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Hash byte arrays with length 95:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Murmur3_x86_32                                      132            133           0         15.9          63.1       1.0X
-xxHash 64-bit                                        77             78           0         27.4          36.5       1.7X
-HiveHasher                                          209            209           0         10.0          99.6       0.6X
+Murmur3_x86_32                                       92            101           6         22.8          43.8       1.0X
+xxHash 64-bit                                        56             63           3         37.4          26.7       1.6X
+HiveHasher                                          165            175           5         12.7          78.7       0.6X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Hash byte arrays with length 287:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Murmur3_x86_32                                      333            334           0          6.3         158.9       1.0X
-xxHash 64-bit                                       123            123           0         17.1          58.6       2.7X
-HiveHasher                                          630            630           0          3.3         300.3       0.5X
+Murmur3_x86_32                                      276            294          13          7.6         131.7       1.0X
+xxHash 64-bit                                        94            103           5         22.2          45.0       2.9X
+HiveHasher                                          549            574          19          3.8         261.6       0.5X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Hash byte arrays with length 1055:        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Murmur3_x86_32                                     1151           1151           0          1.8         548.9       1.0X
-xxHash 64-bit                                       321            321           0          6.5         153.2       3.6X
-HiveHasher                                         2332           2332           1          0.9        1111.8       0.5X
+Murmur3_x86_32                                     1009           1016          10          2.1         481.3       1.0X
+xxHash 64-bit                                       246            260           7          8.5         117.4       4.1X
+HiveHasher                                         2243           2257          19          0.9        1069.6       0.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Hash byte arrays with length 2079:        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Murmur3_x86_32                                     2213           2213           0          0.9        1055.2       1.0X
-xxHash 64-bit                                       550            550           0          3.8         262.3       4.0X
-HiveHasher                                         4599           4599           1          0.5        2192.8       0.5X
+Murmur3_x86_32                                     2021           2036          22          1.0         963.7       1.0X
+xxHash 64-bit                                       458            466           7          4.6         218.2       4.4X
+HiveHasher                                         4473           4525          73          0.5        2132.9       0.5X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Hash byte arrays with length 8223:        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Murmur3_x86_32                                     8660           8664           6          0.2        4129.3       1.0X
-xxHash 64-bit                                      1889           1893           6          1.1         900.6       4.6X
-HiveHasher                                        18269          18272           5          0.1        8711.3       0.5X
+Murmur3_x86_32                                     7706           7771          92          0.3        3674.5       1.0X
+xxHash 64-bit                                      1512           1533          29          1.4         721.1       5.1X
+HiveHasher                                        17151          17155           5          0.1        8178.4       0.4X
 
 
diff --git a/sql/catalyst/benchmarks/UnsafeProjectionBenchmark-jdk11-results.txt b/sql/catalyst/benchmarks/UnsafeProjectionBenchmark-jdk11-results.txt
index 8b56fcbebc79c..8f18280c59c88 100644
--- a/sql/catalyst/benchmarks/UnsafeProjectionBenchmark-jdk11-results.txt
+++ b/sql/catalyst/benchmarks/UnsafeProjectionBenchmark-jdk11-results.txt
@@ -2,13 +2,13 @@
 unsafe projection
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 unsafe projection:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-single long                                        2218           2219           1        121.0           8.3       1.0X
-single nullable long                               3200           3201           1         83.9          11.9       0.7X
-7 primitive types                                  7809           7813           5         34.4          29.1       0.3X
-7 nullable primitive types                        11906          11908           3         22.5          44.4       0.2X
+single long                                        2009           2009           0        133.6           7.5       1.0X
+single nullable long                               3300           3301           0         81.3          12.3       0.6X
+7 primitive types                                  6499           6508          13         41.3          24.2       0.3X
+7 nullable primitive types                        11725          11741          22         22.9          43.7       0.2X
 
 
diff --git a/sql/catalyst/benchmarks/UnsafeProjectionBenchmark-results.txt b/sql/catalyst/benchmarks/UnsafeProjectionBenchmark-results.txt
index 7b30e2075f010..4d484090e29b3 100644
--- a/sql/catalyst/benchmarks/UnsafeProjectionBenchmark-results.txt
+++ b/sql/catalyst/benchmarks/UnsafeProjectionBenchmark-results.txt
@@ -2,13 +2,13 @@
 unsafe projection
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 unsafe projection:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-single long                                        2173           2173           0        123.5           8.1       1.0X
-single nullable long                               3156           3157           2         85.0          11.8       0.7X
-7 primitive types                                  6725           6726           2         39.9          25.1       0.3X
-7 nullable primitive types                        11399          11400           2         23.5          42.5       0.2X
+single long                                        1610           1629          27        166.7           6.0       1.0X
+single nullable long                               2349           2397          68        114.3           8.8       0.7X
+7 primitive types                                  5446           5508          88         49.3          20.3       0.3X
+7 nullable primitive types                         8743           8843         142         30.7          32.6       0.2X
 
 
diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
index 9edbb7fec97d0..583738bc561a9 100644
--- a/sql/catalyst/pom.xml
+++ b/sql/catalyst/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
@@ -104,6 +104,10 @@
       <groupId>org.antlr</groupId>
       <artifactId>antlr4-runtime</artifactId>
     </dependency>
+    <dependency>
+      <groupId>javax.xml.bind</groupId>
+      <artifactId>jaxb-api</artifactId>
+    </dependency>
     <dependency>
       <groupId>commons-codec</groupId>
       <artifactId>commons-codec</artifactId>
@@ -117,6 +121,10 @@
       <groupId>org.apache.arrow</groupId>
       <artifactId>arrow-vector</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.arrow</groupId>
+      <artifactId>arrow-memory-netty</artifactId>
+    </dependency>
   </dependencies>
   <build>
     <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
index b03e6372a8eae..b019769dac8de 100644
--- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
+++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
@@ -16,12 +16,12 @@
 
 grammar SqlBase;
 
-@members {
+@parser::members {
   /**
    * When false, INTERSECT is given the greater precedence over the other set
    * operations (UNION, EXCEPT and MINUS) as per the SQL standard.
    */
-  public boolean legacy_setops_precedence_enbled = false;
+  public boolean legacy_setops_precedence_enabled = false;
 
   /**
    * When false, a literal with an exponent would be converted into
@@ -29,6 +29,13 @@ grammar SqlBase;
    */
   public boolean legacy_exponent_literal_as_decimal_enabled = false;
 
+  /**
+   * When true, the behavior of keywords follows ANSI SQL standard.
+   */
+  public boolean SQL_standard_keyword_behavior = false;
+}
+
+@lexer::members {
   /**
    * Verify whether current token is a valid decimal token (which contains dot).
    * Returns true if the character that follows the token is not a digit or letter or underscore.
@@ -51,11 +58,6 @@ grammar SqlBase;
     }
   }
 
-  /**
-   * When true, the behavior of keywords follows ANSI SQL standard.
-   */
-  public boolean SQL_standard_keyword_behavior = false;
-
   /**
    * This method will be called when we see '/*' and try to match it as a bracketed comment.
    * If the next character is '+', it should be parsed as hint later, and we cannot match
@@ -117,20 +119,9 @@ statement
         (RESTRICT | CASCADE)?                                          #dropNamespace
     | SHOW (DATABASES | NAMESPACES) ((FROM | IN) multipartIdentifier)?
         (LIKE? pattern=STRING)?                                        #showNamespaces
-    | createTableHeader ('(' colTypeList ')')? tableProvider
+    | createTableHeader ('(' colTypeList ')')? tableProvider?
         createTableClauses
         (AS? query)?                                                   #createTable
-    | createTableHeader ('(' columns=colTypeList ')')?
-        (commentSpec |
-        (PARTITIONED BY '(' partitionColumns=colTypeList ')' |
-        PARTITIONED BY partitionColumnNames=identifierList) |
-        bucketSpec |
-        skewSpec |
-        rowFormat |
-        createFileFormat |
-        locationSpec |
-        (TBLPROPERTIES tableProps=tablePropertyList))*
-        (AS? query)?                                                   #createHiveTable
     | CREATE TABLE (IF NOT EXISTS)? target=tableIdentifier
         LIKE source=tableIdentifier
         (tableProvider |
@@ -138,11 +129,13 @@ statement
         createFileFormat |
         locationSpec |
         (TBLPROPERTIES tableProps=tablePropertyList))*                 #createTableLike
-    | replaceTableHeader ('(' colTypeList ')')? tableProvider
+    | replaceTableHeader ('(' colTypeList ')')? tableProvider?
         createTableClauses
         (AS? query)?                                                   #replaceTable
     | ANALYZE TABLE multipartIdentifier partitionSpec? COMPUTE STATISTICS
         (identifier | FOR COLUMNS identifierSeq | FOR ALL COLUMNS)?    #analyze
+    | ANALYZE TABLES ((FROM | IN) multipartIdentifier)? COMPUTE STATISTICS
+        (identifier)?                                                  #analyzeTables
     | ALTER TABLE multipartIdentifier
         ADD (COLUMN | COLUMNS)
         columns=qualifiedColTypeWithPositionList                       #addTableColumns
@@ -207,7 +200,7 @@ statement
     | SHOW TABLES ((FROM | IN) multipartIdentifier)?
         (LIKE? pattern=STRING)?                                        #showTables
     | SHOW TABLE EXTENDED ((FROM | IN) ns=multipartIdentifier)?
-        LIKE pattern=STRING partitionSpec?                             #showTable
+        LIKE pattern=STRING partitionSpec?                             #showTableExtended
     | SHOW TBLPROPERTIES table=multipartIdentifier
         ('(' key=tablePropertyKey ')')?                                #showTblProperties
     | SHOW COLUMNS (FROM | IN) table=multipartIdentifier
@@ -229,6 +222,7 @@ statement
         comment=(STRING | NULL)                                        #commentNamespace
     | COMMENT ON TABLE multipartIdentifier IS comment=(STRING | NULL)  #commentTable
     | REFRESH TABLE multipartIdentifier                                #refreshTable
+    | REFRESH FUNCTION multipartIdentifier                             #refreshFunction
     | REFRESH (STRING | .*?)                                           #refreshResource
     | CACHE LAZY? TABLE multipartIdentifier
         (OPTIONS options=tablePropertyList)? (AS? query)?              #cacheTable
@@ -237,14 +231,30 @@ statement
     | LOAD DATA LOCAL? INPATH path=STRING OVERWRITE? INTO TABLE
         multipartIdentifier partitionSpec?                             #loadData
     | TRUNCATE TABLE multipartIdentifier partitionSpec?                #truncateTable
-    | MSCK REPAIR TABLE multipartIdentifier                            #repairTable
-    | op=(ADD | LIST) identifier (STRING | .*?)                        #manageResource
+    | MSCK REPAIR TABLE multipartIdentifier
+        (option=(ADD|DROP|SYNC) PARTITIONS)?                           #repairTable
+    | op=(ADD | LIST) identifier .*?                                   #manageResource
     | SET ROLE .*?                                                     #failNativeCommand
+    | SET TIME ZONE interval                                           #setTimeZone
+    | SET TIME ZONE timezone=(STRING | LOCAL)                          #setTimeZone
+    | SET TIME ZONE .*?                                                #setTimeZone
+    | SET configKey EQ configValue                                     #setQuotedConfiguration
+    | SET configKey (EQ .*?)?                                          #setQuotedConfiguration
+    | SET .*? EQ configValue                                           #setQuotedConfiguration
     | SET .*?                                                          #setConfiguration
-    | RESET                                                            #resetConfiguration
+    | RESET configKey                                                  #resetQuotedConfiguration
+    | RESET .*?                                                        #resetConfiguration
     | unsupportedHiveNativeCommands .*?                                #failNativeCommand
     ;
 
+configKey
+    : quotedIdentifier
+    ;
+
+configValue
+    : quotedIdentifier
+    ;
+
 unsupportedHiveNativeCommands
     : kw1=CREATE kw2=ROLE
     | kw1=DROP kw2=ROLE
@@ -325,8 +335,8 @@ query
     ;
 
 insertInto
-    : INSERT OVERWRITE TABLE? multipartIdentifier (partitionSpec (IF NOT EXISTS)?)?                         #insertOverwriteTable
-    | INSERT INTO TABLE? multipartIdentifier partitionSpec? (IF NOT EXISTS)?                                #insertIntoTable
+    : INSERT OVERWRITE TABLE? multipartIdentifier (partitionSpec (IF NOT EXISTS)?)?  identifierList?        #insertOverwriteTable
+    | INSERT INTO TABLE? multipartIdentifier partitionSpec? (IF NOT EXISTS)? identifierList?                #insertIntoTable
     | INSERT OVERWRITE LOCAL? DIRECTORY path=STRING rowFormat? createFileFormat?                            #insertOverwriteHiveDir
     | INSERT OVERWRITE LOCAL? DIRECTORY (path=STRING)? tableProvider (OPTIONS options=tablePropertyList)?   #insertOverwriteDir
     ;
@@ -375,8 +385,11 @@ tableProvider
 
 createTableClauses
     :((OPTIONS options=tablePropertyList) |
-     (PARTITIONED BY partitioning=transformList) |
+     (PARTITIONED BY partitioning=partitionFieldList) |
+     skewSpec |
      bucketSpec |
+     rowFormat |
+     createFileFormat |
      locationSpec |
      commentSpec |
      (TBLPROPERTIES tableProps=tablePropertyList))*
@@ -456,11 +469,11 @@ multiInsertQueryBody
 
 queryTerm
     : queryPrimary                                                                       #queryTermDefault
-    | left=queryTerm {legacy_setops_precedence_enbled}?
+    | left=queryTerm {legacy_setops_precedence_enabled}?
         operator=(INTERSECT | UNION | EXCEPT | SETMINUS) setQuantifier? right=queryTerm  #setOperation
-    | left=queryTerm {!legacy_setops_precedence_enbled}?
+    | left=queryTerm {!legacy_setops_precedence_enabled}?
         operator=INTERSECT setQuantifier? right=queryTerm                                #setOperation
-    | left=queryTerm {!legacy_setops_precedence_enbled}?
+    | left=queryTerm {!legacy_setops_precedence_enabled}?
         operator=(UNION | EXCEPT | SETMINUS) setQuantifier? right=queryTerm              #setOperation
     ;
 
@@ -496,7 +509,11 @@ fromStatementBody
 querySpecification
     : transformClause
       fromClause?
-      whereClause?                                                          #transformQuerySpecification
+      lateralView*
+      whereClause?
+      aggregationClause?
+      havingClause?
+      windowClause?                                                         #transformQuerySpecification
     | selectClause
       fromClause?
       lateralView*
@@ -507,9 +524,9 @@ querySpecification
     ;
 
 transformClause
-    : (SELECT kind=TRANSFORM '(' namedExpressionSeq ')'
-            | kind=MAP namedExpressionSeq
-            | kind=REDUCE namedExpressionSeq)
+    : (SELECT kind=TRANSFORM '(' setQuantifier? expressionSeq ')'
+            | kind=MAP setQuantifier? expressionSeq
+            | kind=REDUCE setQuantifier? expressionSeq)
       inRowFormat=rowFormat?
       (RECORDWRITER recordWriter=STRING)?
       USING script=STRING
@@ -575,11 +592,27 @@ fromClause
     ;
 
 aggregationClause
-    : GROUP BY groupingExpressions+=expression (',' groupingExpressions+=expression)* (
+    : GROUP BY groupingExpressionsWithGroupingAnalytics+=groupByClause
+        (',' groupingExpressionsWithGroupingAnalytics+=groupByClause)*
+    | GROUP BY groupingExpressions+=expression (',' groupingExpressions+=expression)* (
       WITH kind=ROLLUP
     | WITH kind=CUBE
     | kind=GROUPING SETS '(' groupingSet (',' groupingSet)* ')')?
-    | GROUP BY kind=GROUPING SETS '(' groupingSet (',' groupingSet)* ')'
+    ;
+
+groupByClause
+    : groupingAnalytics
+    | expression
+    ;
+
+groupingAnalytics
+    : (ROLLUP | CUBE) '(' groupingSet (',' groupingSet)* ')'
+    | GROUPING SETS '(' groupingElement (',' groupingElement)* ')'
+    ;
+
+groupingElement
+    : groupingAnalytics
+    | groupingSet
     ;
 
 groupingSet
@@ -682,7 +715,7 @@ inlineTable
     ;
 
 functionTable
-    : funcName=errorCapturingIdentifier '(' (expression (',' expression)*)? ')' tableAlias
+    : funcName=functionName '(' (expression (',' expression)*)? ')' tableAlias
     ;
 
 tableAlias
@@ -723,8 +756,13 @@ namedExpressionSeq
     : namedExpression (',' namedExpression)*
     ;
 
-transformList
-    : '(' transforms+=transform (',' transforms+=transform)* ')'
+partitionFieldList
+    : '(' fields+=partitionField (',' fields+=partitionField)* ')'
+    ;
+
+partitionField
+    : transform  #partitionTransform
+    | colType    #partitionColumn
     ;
 
 transform
@@ -742,6 +780,10 @@ expression
     : booleanExpression
     ;
 
+expressionSeq
+    : expression (',' expression)*
+    ;
+
 booleanExpression
     : NOT booleanExpression                                        #logicalNot
     | EXISTS '(' query ')'                                         #exists
@@ -777,7 +819,7 @@ primaryExpression
     : name=(CURRENT_DATE | CURRENT_TIMESTAMP)                                                  #currentDatetime
     | CASE whenClause+ (ELSE elseExpression=expression)? END                                   #searchedCase
     | CASE value=expression whenClause+ (ELSE elseExpression=expression)? END                  #simpleCase
-    | CAST '(' expression AS dataType ')'                                                      #cast
+    | name=(CAST | TRY_CAST) '(' expression AS dataType ')'                                    #cast
     | STRUCT '(' (argument+=namedExpression (',' argument+=namedExpression)*)? ')'             #struct
     | FIRST '(' expression (IGNORE NULLS)? ')'                                                 #first
     | LAST '(' expression (IGNORE NULLS)? ')'                                                  #last
@@ -788,7 +830,8 @@ primaryExpression
     | '(' namedExpression (',' namedExpression)+ ')'                                           #rowConstructor
     | '(' query ')'                                                                            #subqueryExpression
     | functionName '(' (setQuantifier? argument+=expression (',' argument+=expression)*)? ')'
-       (FILTER '(' WHERE where=booleanExpression ')')? (OVER windowSpec)?                      #functionCall
+       (FILTER '(' WHERE where=booleanExpression ')')?
+       (nullsOption=(IGNORE | RESPECT) NULLS)? ( OVER windowSpec)?                             #functionCall
     | identifier '->' expression                                                               #lambda
     | '(' identifier (',' identifier)+ ')' '->' expression                                     #lambda
     | value=primaryExpression '[' index=valueExpression ']'                                    #subscript
@@ -838,7 +881,7 @@ errorCapturingMultiUnitsInterval
     ;
 
 multiUnitsInterval
-    : (intervalValue intervalUnit)+
+    : (intervalValue unit+=identifier)+
     ;
 
 errorCapturingUnitToUnitInterval
@@ -846,22 +889,11 @@ errorCapturingUnitToUnitInterval
     ;
 
 unitToUnitInterval
-    : value=intervalValue from=intervalUnit TO to=intervalUnit
+    : value=intervalValue from=identifier TO to=identifier
     ;
 
 intervalValue
-    : (PLUS | MINUS)? (INTEGER_VALUE | DECIMAL_VALUE)
-    | STRING
-    ;
-
-intervalUnit
-    : DAY
-    | HOUR
-    | MINUTE
-    | MONTH
-    | SECOND
-    | YEAR
-    | identifier
+    : (PLUS | MINUS)? (INTEGER_VALUE | DECIMAL_VALUE | STRING)
     ;
 
 colPosition
@@ -872,6 +904,8 @@ dataType
     : complex=ARRAY '<' dataType '>'                            #complexDataType
     | complex=MAP '<' dataType ',' dataType '>'                 #complexDataType
     | complex=STRUCT ('<' complexColTypeList? '>' | NEQ)        #complexDataType
+    | INTERVAL YEAR TO MONTH                                    #yearMonthIntervalDataType
+    | INTERVAL DAY TO SECOND                                    #dayTimeIntervalDataType
     | identifier ('(' INTEGER_VALUE (',' INTEGER_VALUE)* ')')?  #primitiveDataType
     ;
 
@@ -988,6 +1022,7 @@ number
     | MINUS? SMALLINT_LITERAL         #smallIntLiteral
     | MINUS? TINYINT_LITERAL          #tinyIntLiteral
     | MINUS? DOUBLE_LITERAL           #doubleLiteral
+    | MINUS? FLOAT_LITERAL            #floatLiteral
     | MINUS? BIGDECIMAL_LITERAL       #bigDecimalLiteral
     ;
 
@@ -1009,10 +1044,12 @@ alterColumnAction
 // You can find the full keywords list by searching "Start of the keywords list" in this file.
 // The non-reserved keywords are listed below. Keywords not in this list are reserved keywords.
 ansiNonReserved
+//--ANSI-NON-RESERVED-START
     : ADD
     | AFTER
     | ALTER
     | ANALYZE
+    | ANTI
     | ARCHIVE
     | ARRAY
     | ASC
@@ -1042,6 +1079,7 @@ ansiNonReserved
     | DATA
     | DATABASE
     | DATABASES
+    | DAY
     | DBPROPERTIES
     | DEFINED
     | DELETE
@@ -1100,6 +1138,7 @@ ansiNonReserved
     | MAP
     | MATCHED
     | MERGE
+    | MONTH
     | MSCK
     | NAMESPACE
     | NAMESPACES
@@ -1135,6 +1174,7 @@ ansiNonReserved
     | REPAIR
     | REPLACE
     | RESET
+    | RESPECT
     | RESTRICT
     | REVOKE
     | RLIKE
@@ -1145,10 +1185,13 @@ ansiNonReserved
     | ROW
     | ROWS
     | SCHEMA
+    | SECOND
+    | SEMI
     | SEPARATED
     | SERDE
     | SERDEPROPERTIES
     | SET
+    | SETMINUS
     | SETS
     | SHOW
     | SKEWED
@@ -1161,6 +1204,7 @@ ansiNonReserved
     | STRUCT
     | SUBSTR
     | SUBSTRING
+    | SYNC
     | TABLES
     | TABLESAMPLE
     | TBLPROPERTIES
@@ -1173,6 +1217,8 @@ ansiNonReserved
     | TRIM
     | TRUE
     | TRUNCATE
+    | TRY_CAST
+    | TYPE
     | UNARCHIVE
     | UNBOUNDED
     | UNCACHE
@@ -1184,6 +1230,9 @@ ansiNonReserved
     | VIEW
     | VIEWS
     | WINDOW
+    | YEAR
+    | ZONE
+//--ANSI-NON-RESERVED-END
     ;
 
 // When `SQL_standard_keyword_behavior=false`, there are 2 kinds of keywords in Spark SQL.
@@ -1214,6 +1263,7 @@ strictNonReserved
     ;
 
 nonReserved
+//--DEFAULT-NON-RESERVED-START
     : ADD
     | AFTER
     | ALL
@@ -1308,7 +1358,6 @@ nonReserved
     | GROUP
     | GROUPING
     | HAVING
-    | HOUR
     | IF
     | IGNORE
     | IMPORT
@@ -1341,7 +1390,6 @@ nonReserved
     | MAP
     | MATCHED
     | MERGE
-    | MINUTE
     | MONTH
     | MSCK
     | NAMESPACE
@@ -1387,6 +1435,7 @@ nonReserved
     | REPAIR
     | REPLACE
     | RESET
+    | RESPECT
     | RESTRICT
     | REVOKE
     | RLIKE
@@ -1417,6 +1466,7 @@ nonReserved
     | STRUCT
     | SUBSTR
     | SUBSTRING
+    | SYNC
     | TABLE
     | TABLES
     | TABLESAMPLE
@@ -1424,6 +1474,7 @@ nonReserved
     | TEMPORARY
     | TERMINATED
     | THEN
+    | TIME
     | TO
     | TOUCH
     | TRAILING
@@ -1433,6 +1484,7 @@ nonReserved
     | TRIM
     | TRUE
     | TRUNCATE
+    | TRY_CAST
     | TYPE
     | UNARCHIVE
     | UNBOUNDED
@@ -1452,15 +1504,17 @@ nonReserved
     | WINDOW
     | WITH
     | YEAR
+    | ZONE
+//--DEFAULT-NON-RESERVED-END
     ;
 
 // NOTE: If you add a new token in the list below, you should update the list of keywords
-// in `docs/sql-keywords.md`. If the token is a non-reserved keyword,
-// please update `ansiNonReserved` and `nonReserved` as well.
+// and reserved tag in `docs/sql-ref-ansi-compliance.md#sql-keywords`.
 
 //============================
 // Start of the keywords list
 //============================
+//--SPARK-KEYWORD-LIST-START
 ADD: 'ADD';
 AFTER: 'AFTER';
 ALL: 'ALL';
@@ -1510,10 +1564,10 @@ CURRENT_DATE: 'CURRENT_DATE';
 CURRENT_TIME: 'CURRENT_TIME';
 CURRENT_TIMESTAMP: 'CURRENT_TIMESTAMP';
 CURRENT_USER: 'CURRENT_USER';
+DAY: 'DAY';
 DATA: 'DATA';
 DATABASE: 'DATABASE';
 DATABASES: 'DATABASES' | 'SCHEMAS';
-DAY: 'DAY';
 DBPROPERTIES: 'DBPROPERTIES';
 DEFINED: 'DEFINED';
 DELETE: 'DELETE';
@@ -1525,6 +1579,7 @@ DIRECTORIES: 'DIRECTORIES';
 DIRECTORY: 'DIRECTORY';
 DISTINCT: 'DISTINCT';
 DISTRIBUTE: 'DISTRIBUTE';
+DIV: 'DIV';
 DROP: 'DROP';
 ELSE: 'ELSE';
 END: 'END';
@@ -1558,7 +1613,6 @@ GRANT: 'GRANT';
 GROUP: 'GROUP';
 GROUPING: 'GROUPING';
 HAVING: 'HAVING';
-HOUR: 'HOUR';
 IF: 'IF';
 IGNORE: 'IGNORE';
 IMPORT: 'IMPORT';
@@ -1595,7 +1649,6 @@ MACRO: 'MACRO';
 MAP: 'MAP';
 MATCHED: 'MATCHED';
 MERGE: 'MERGE';
-MINUTE: 'MINUTE';
 MONTH: 'MONTH';
 MSCK: 'MSCK';
 NAMESPACE: 'NAMESPACE';
@@ -1643,6 +1696,7 @@ RENAME: 'RENAME';
 REPAIR: 'REPAIR';
 REPLACE: 'REPLACE';
 RESET: 'RESET';
+RESPECT: 'RESPECT';
 RESTRICT: 'RESTRICT';
 REVOKE: 'REVOKE';
 RIGHT: 'RIGHT';
@@ -1653,8 +1707,8 @@ ROLLBACK: 'ROLLBACK';
 ROLLUP: 'ROLLUP';
 ROW: 'ROW';
 ROWS: 'ROWS';
-SCHEMA: 'SCHEMA';
 SECOND: 'SECOND';
+SCHEMA: 'SCHEMA';
 SELECT: 'SELECT';
 SEMI: 'SEMI';
 SEPARATED: 'SEPARATED';
@@ -1676,6 +1730,7 @@ STRATIFY: 'STRATIFY';
 STRUCT: 'STRUCT';
 SUBSTR: 'SUBSTR';
 SUBSTRING: 'SUBSTRING';
+SYNC: 'SYNC';
 TABLE: 'TABLE';
 TABLES: 'TABLES';
 TABLESAMPLE: 'TABLESAMPLE';
@@ -1683,6 +1738,7 @@ TBLPROPERTIES: 'TBLPROPERTIES';
 TEMPORARY: 'TEMPORARY' | 'TEMP';
 TERMINATED: 'TERMINATED';
 THEN: 'THEN';
+TIME: 'TIME';
 TO: 'TO';
 TOUCH: 'TOUCH';
 TRAILING: 'TRAILING';
@@ -1692,6 +1748,7 @@ TRANSFORM: 'TRANSFORM';
 TRIM: 'TRIM';
 TRUE: 'TRUE';
 TRUNCATE: 'TRUNCATE';
+TRY_CAST: 'TRY_CAST';
 TYPE: 'TYPE';
 UNARCHIVE: 'UNARCHIVE';
 UNBOUNDED: 'UNBOUNDED';
@@ -1713,6 +1770,8 @@ WHERE: 'WHERE';
 WINDOW: 'WINDOW';
 WITH: 'WITH';
 YEAR: 'YEAR';
+ZONE: 'ZONE';
+//--SPARK-KEYWORD-LIST-END
 //============================
 // End of the keywords list
 //============================
@@ -1731,7 +1790,6 @@ MINUS: '-';
 ASTERISK: '*';
 SLASH: '/';
 PERCENT: '%';
-DIV: 'DIV';
 TILDE: '~';
 AMPERSAND: '&';
 PIPE: '|';
@@ -1768,6 +1826,11 @@ DECIMAL_VALUE
     : DECIMAL_DIGITS {isValidDecimal()}?
     ;
 
+FLOAT_LITERAL
+    : DIGIT+ EXPONENT? 'F'
+    | DECIMAL_DIGITS EXPONENT? 'F' {isValidDecimal()}?
+    ;
+
 DOUBLE_LITERAL
     : DIGIT+ EXPONENT? 'D'
     | DECIMAL_DIGITS EXPONENT? 'D' {isValidDecimal()}?
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionInfo.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionInfo.java
index a500822b21f02..70064b6ee9585 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionInfo.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionInfo.java
@@ -40,8 +40,11 @@ public class ExpressionInfo {
     private String deprecated;
 
     private static final Set<String> validGroups =
-        new HashSet<>(Arrays.asList("agg_funcs", "array_funcs", "datetime_funcs",
-            "json_funcs", "map_funcs", "window_funcs"));
+        new HashSet<>(Arrays.asList("agg_funcs", "array_funcs", "binary_funcs", "bitwise_funcs",
+            "collection_funcs", "predicate_funcs", "conditional_funcs", "conversion_funcs",
+            "csv_funcs", "datetime_funcs", "generator_funcs", "hash_funcs", "json_funcs",
+            "lambda_funcs", "map_funcs", "math_funcs", "misc_funcs", "string_funcs", "struct_funcs",
+            "window_funcs", "xml_funcs", "table_funcs"));
 
     public String getClassName() {
         return className;
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/FixedLengthRowBasedKeyValueBatch.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/FixedLengthRowBasedKeyValueBatch.java
index df52f9c2d5496..25400beeb0d75 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/FixedLengthRowBasedKeyValueBatch.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/FixedLengthRowBasedKeyValueBatch.java
@@ -46,6 +46,8 @@ private long getKeyOffsetForFixedLengthRecords(int rowId) {
   public UnsafeRow appendRow(Object kbase, long koff, int klen,
                              Object vbase, long voff, int vlen) {
     // if run out of max supported rows or page size, return null
+    assert(vlen == this.vlen);
+    assert(klen == this.klen);
     if (numRows >= capacity || page == null || page.size() - pageCursor < recordLength) {
       return null;
     }
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/RowBasedKeyValueBatch.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/RowBasedKeyValueBatch.java
index 6344cf18c11b8..40d360d84fb0f 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/RowBasedKeyValueBatch.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/RowBasedKeyValueBatch.java
@@ -79,13 +79,11 @@ public static RowBasedKeyValueBatch allocate(StructType keySchema, StructType va
     boolean allFixedLength = true;
     // checking if there is any variable length fields
     // there is probably a more succinct impl of this
-    for (String name : keySchema.fieldNames()) {
-      allFixedLength = allFixedLength
-              && UnsafeRow.isFixedLength(keySchema.apply(name).dataType());
+    for (StructField field : keySchema.fields()) {
+      allFixedLength = allFixedLength && UnsafeRow.isFixedLength(field.dataType());
     }
-    for (String name : valueSchema.fieldNames()) {
-      allFixedLength = allFixedLength
-              && UnsafeRow.isFixedLength(valueSchema.apply(name).dataType());
+    for (StructField field : valueSchema.fields()) {
+      allFixedLength = allFixedLength && UnsafeRow.isFixedLength(field.dataType());
     }
 
     if (allFixedLength) {
@@ -97,7 +95,7 @@ public static RowBasedKeyValueBatch allocate(StructType keySchema, StructType va
 
   protected RowBasedKeyValueBatch(StructType keySchema, StructType valueSchema, int maxRows,
                                 TaskMemoryManager manager) {
-    super(manager, manager.pageSizeBytes(), manager.getTungstenMemoryMode());
+    super(manager, manager.getTungstenMemoryMode());
 
     this.keySchema = keySchema;
     this.valueSchema = valueSchema;
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/SpecializedGettersReader.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/SpecializedGettersReader.java
index ea0648a6cb909..90f340b51c3e0 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/SpecializedGettersReader.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/SpecializedGettersReader.java
@@ -83,6 +83,12 @@ public static Object read(
     if (handleUserDefinedType && dataType instanceof UserDefinedType) {
       return obj.get(ordinal, ((UserDefinedType)dataType).sqlType());
     }
+    if (dataType instanceof DayTimeIntervalType) {
+      return obj.getLong(ordinal);
+    }
+    if (dataType instanceof YearMonthIntervalType) {
+      return obj.getInt(ordinal);
+    }
 
     throw new UnsupportedOperationException("Unsupported data type " + dataType.simpleString());
   }
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
index 034894bd86085..0c6685d76fd04 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
@@ -90,7 +90,9 @@ public static int calculateBitSetWidthInBytes(int numFields) {
           FloatType,
           DoubleType,
           DateType,
-          TimestampType
+          TimestampType,
+          YearMonthIntervalType,
+          DayTimeIntervalType
         })));
   }
 
@@ -288,7 +290,7 @@ public void setDecimal(int ordinal, Decimal value, int precision) {
       Platform.putLong(baseObject, baseOffset + cursor, 0L);
       Platform.putLong(baseObject, baseOffset + cursor + 8, 0L);
 
-      if (value == null) {
+      if (value == null || !value.changePrecision(precision, value.scale())) {
         setNullAt(ordinal);
         // keep the offset for future update
         Platform.putLong(baseObject, getFieldOffset(ordinal), cursor << 32);
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/XXH64.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/XXH64.java
index eb5051b284073..4c356256836c2 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/XXH64.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/XXH64.java
@@ -16,6 +16,8 @@
  */
 package org.apache.spark.sql.catalyst.expressions;
 
+import java.nio.ByteOrder;
+
 import org.apache.spark.unsafe.Platform;
 import org.apache.spark.unsafe.types.UTF8String;
 
@@ -31,6 +33,7 @@
  */
 // scalastyle: on
 public final class XXH64 {
+  private static final boolean isBigEndian = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN);
 
   private static final long PRIME64_1 = 0x9E3779B185EBCA87L;
   private static final long PRIME64_2 = 0xC2B2AE3D27D4EB4FL;
@@ -93,7 +96,11 @@ public static long hashUnsafeBytes(Object base, long offset, int length, long se
     offset += length & -8;
 
     if (offset + 4L <= end) {
-      hash ^= (Platform.getInt(base, offset) & 0xFFFFFFFFL) * PRIME64_1;
+      int k1 = Platform.getInt(base, offset);
+      if (isBigEndian) {
+        k1 = Integer.reverseBytes(k1);
+      }
+      hash ^= (k1 & 0xFFFFFFFFL) * PRIME64_1;
       hash = Long.rotateLeft(hash, 23) * PRIME64_2 + PRIME64_3;
       offset += 4L;
     }
@@ -130,21 +137,22 @@ private static long hashBytesByWords(Object base, long offset, int length, long
       long v4 = seed - PRIME64_1;
 
       do {
-        v1 += Platform.getLong(base, offset) * PRIME64_2;
-        v1 = Long.rotateLeft(v1, 31);
-        v1 *= PRIME64_1;
-
-        v2 += Platform.getLong(base, offset + 8) * PRIME64_2;
-        v2 = Long.rotateLeft(v2, 31);
-        v2 *= PRIME64_1;
-
-        v3 += Platform.getLong(base, offset + 16) * PRIME64_2;
-        v3 = Long.rotateLeft(v3, 31);
-        v3 *= PRIME64_1;
-
-        v4 += Platform.getLong(base, offset + 24) * PRIME64_2;
-        v4 = Long.rotateLeft(v4, 31);
-        v4 *= PRIME64_1;
+        long k1 = Platform.getLong(base, offset);
+        long k2 = Platform.getLong(base, offset + 8);
+        long k3 = Platform.getLong(base, offset + 16);
+        long k4 = Platform.getLong(base, offset + 24);
+
+        if (isBigEndian) {
+          k1 = Long.reverseBytes(k1);
+          k2 = Long.reverseBytes(k2);
+          k3 = Long.reverseBytes(k3);
+          k4 = Long.reverseBytes(k4);
+        }
+
+        v1 = Long.rotateLeft(v1 + (k1 * PRIME64_2), 31) * PRIME64_1;
+        v2 = Long.rotateLeft(v2 + (k2 * PRIME64_2), 31) * PRIME64_1;
+        v3 = Long.rotateLeft(v3 + (k3 * PRIME64_2), 31) * PRIME64_1;
+        v4 = Long.rotateLeft(v4 + (k4 * PRIME64_2), 31) * PRIME64_1;
 
         offset += 32L;
       } while (offset <= limit);
@@ -186,6 +194,9 @@ private static long hashBytesByWords(Object base, long offset, int length, long
     long limit = end - 8;
     while (offset <= limit) {
       long k1 = Platform.getLong(base, offset);
+      if (isBigEndian) {
+        k1 = Long.reverseBytes(k1);
+      }
       hash ^= Long.rotateLeft(k1 * PRIME64_2, 31) * PRIME64_1;
       hash = Long.rotateLeft(hash, 27) * PRIME64_1 + PRIME64_4;
       offset += 8L;
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/util/CharVarcharCodegenUtils.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/util/CharVarcharCodegenUtils.java
new file mode 100644
index 0000000000000..581f4bb6d259f
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/util/CharVarcharCodegenUtils.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util;
+
+import org.apache.spark.unsafe.types.UTF8String;
+
+public class CharVarcharCodegenUtils {
+  private static final UTF8String SPACE = UTF8String.fromString(" ");
+
+  private static UTF8String trimTrailingSpaces(
+      UTF8String inputStr, int numChars, int limit) {
+    int numTailSpacesToTrim = numChars - limit;
+    UTF8String trimmed = inputStr.trimTrailingSpaces(numTailSpacesToTrim);
+    if (trimmed.numChars() > limit) {
+      throw new RuntimeException("Exceeds char/varchar type length limitation: " + limit);
+    } else {
+      return trimmed;
+    }
+  }
+
+  public static UTF8String charTypeWriteSideCheck(UTF8String inputStr, int limit) {
+    int numChars = inputStr.numChars();
+    if (numChars == limit) {
+      return inputStr;
+    } else if (numChars < limit) {
+      return inputStr.rpad(limit, SPACE);
+    } else {
+      return trimTrailingSpaces(inputStr, numChars, limit);
+    }
+  }
+
+  public static UTF8String varcharTypeWriteSideCheck(UTF8String inputStr, int limit) {
+    int numChars = inputStr.numChars();
+    if (numChars <= limit) {
+      return inputStr;
+    } else {
+      return trimTrailingSpaces(inputStr, numChars, limit);
+    }
+  }
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DelegatingCatalogExtension.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DelegatingCatalogExtension.java
index d07d299d65a58..34f07b12b3666 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DelegatingCatalogExtension.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DelegatingCatalogExtension.java
@@ -99,6 +99,11 @@ public boolean dropTable(Identifier ident) {
     return asTableCatalog().dropTable(ident);
   }
 
+  @Override
+  public boolean purgeTable(Identifier ident) {
+    return asTableCatalog().purgeTable(ident);
+  }
+
   @Override
   public void renameTable(
       Identifier oldIdent,
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/FunctionCatalog.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/FunctionCatalog.java
new file mode 100644
index 0000000000000..651c9148c4702
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/FunctionCatalog.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.catalog;
+
+import org.apache.spark.sql.catalyst.analysis.NoSuchFunctionException;
+import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException;
+import org.apache.spark.sql.connector.catalog.functions.UnboundFunction;
+
+/**
+ * Catalog methods for working with Functions.
+ */
+public interface FunctionCatalog extends CatalogPlugin {
+
+  /**
+   * List the functions in a namespace from the catalog.
+   * <p>
+   * If there are no functions in the namespace, implementations should return an empty array.
+   *
+   * @param namespace a multi-part namespace
+   * @return an array of Identifiers for functions
+   * @throws NoSuchNamespaceException If the namespace does not exist (optional).
+   */
+  Identifier[] listFunctions(String[] namespace) throws NoSuchNamespaceException;
+
+  /**
+   * Load a function by {@link Identifier identifier} from the catalog.
+   *
+   * @param ident a function identifier
+   * @return an unbound function instance
+   * @throws NoSuchFunctionException If the function doesn't exist
+   */
+  UnboundFunction loadFunction(Identifier ident) throws NoSuchFunctionException;
+
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/IdentifierImpl.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/IdentifierImpl.java
index b6c84cdeb0d3e..ce8734b1a162e 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/IdentifierImpl.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/IdentifierImpl.java
@@ -25,6 +25,7 @@
 import com.google.common.base.Preconditions;
 
 import org.apache.spark.annotation.Evolving;
+import org.apache.spark.sql.catalyst.util.package$;
 
 /**
  *  An {@link Identifier} implementation.
@@ -55,7 +56,7 @@ public String name() {
   @Override
   public String toString() {
     return Stream.concat(Stream.of(namespace), Stream.of(name))
-      .map(CatalogV2Implicits::quoteIfNeeded)
+      .map(package$.MODULE$::quoteIfNeeded)
       .collect(Collectors.joining("."));
   }
 
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/MetadataColumn.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/MetadataColumn.java
new file mode 100644
index 0000000000000..65f31229764fe
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/MetadataColumn.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.connector.catalog;
+
+import org.apache.spark.annotation.Evolving;
+import org.apache.spark.sql.connector.expressions.Transform;
+import org.apache.spark.sql.types.DataType;
+
+/**
+ * Interface for a metadata column.
+ * <p>
+ * A metadata column can expose additional metadata about a row. For example, rows from Kafka can
+ * use metadata columns to expose a message's topic, partition number, and offset.
+ * <p>
+ * A metadata column could also be the result of a transform applied to a value in the row. For
+ * example, a partition value produced by bucket(id, 16) could be exposed by a metadata column. In
+ * this case, {@link #transform()} should return a non-null {@link Transform} that produced the
+ * metadata column's values.
+ *
+ * @since 3.1.0
+ */
+@Evolving
+public interface MetadataColumn {
+  /**
+   * The name of this metadata column.
+   *
+   * @return a String name
+   */
+  String name();
+
+  /**
+   * The data type of values in this metadata column.
+   *
+   * @return a {@link DataType}
+   */
+  DataType dataType();
+
+  /**
+   * @return whether values produced by this metadata column may be null
+   */
+  default boolean isNullable() {
+    return true;
+  }
+
+  /**
+   * Documentation for this metadata column, or null.
+   *
+   * @return a documentation String
+   */
+  default String comment() {
+    return null;
+  }
+
+  /**
+   * The {@link Transform} used to produce this metadata column from data rows, or null.
+   *
+   * @return a {@link Transform} used to produce the column's values, or null if there isn't one
+   */
+  default Transform transform() {
+    return null;
+  }
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsAtomicPartitionManagement.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsAtomicPartitionManagement.java
new file mode 100644
index 0000000000000..e2c693f2d0a92
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsAtomicPartitionManagement.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.catalog;
+
+import java.util.Map;
+
+import org.apache.spark.annotation.Experimental;
+import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.catalyst.analysis.NoSuchPartitionException;
+import org.apache.spark.sql.catalyst.analysis.PartitionAlreadyExistsException;
+import org.apache.spark.sql.catalyst.analysis.PartitionsAlreadyExistException;
+
+/**
+ * An atomic partition interface of {@link Table} to operate multiple partitions atomically.
+ * <p>
+ * These APIs are used to modify table partition or partition metadata,
+ * they will change the table data as well.
+ * <ul>
+ *   <li>{@link #createPartitions}: add an array of partitions and any data they contain to the
+ *   table</li>
+ *   <li>{@link #dropPartitions}: remove an array of partitions and any data they contain from
+ *   the table</li>
+ *   <li>{@link #purgePartitions}: remove an array of partitions and any data they contain from
+ *   the table by skipping a trash even if it is supported</li>
+ *   <li>{@link #truncatePartitions}: truncate an array of partitions by removing partitions
+ *   data</li>
+ * </ul>
+ *
+ * @since 3.1.0
+ */
+@Experimental
+public interface SupportsAtomicPartitionManagement extends SupportsPartitionManagement {
+
+  @Override
+  default void createPartition(
+      InternalRow ident,
+      Map<String, String> properties)
+      throws PartitionAlreadyExistsException, UnsupportedOperationException {
+    try {
+      createPartitions(new InternalRow[]{ident}, new Map[]{properties});
+    } catch (PartitionsAlreadyExistException e) {
+      throw new PartitionAlreadyExistsException(e.getMessage());
+    }
+  }
+
+  @Override
+  default boolean dropPartition(InternalRow ident) {
+    return dropPartitions(new InternalRow[]{ident});
+  }
+
+  /**
+   * Create an array of partitions atomically in table.
+   * <p>
+   * If any partition already exists,
+   * the operation of createPartitions need to be safely rolled back.
+   *
+   * @param idents an array of new partition identifiers
+   * @param properties the metadata of the partitions
+   * @throws PartitionsAlreadyExistException If any partition already exists for the identifier
+   * @throws UnsupportedOperationException If partition property is not supported
+   */
+  void createPartitions(
+      InternalRow[] idents,
+      Map<String, String>[] properties)
+      throws PartitionsAlreadyExistException, UnsupportedOperationException;
+
+  /**
+   * Drop an array of partitions atomically from table.
+   * <p>
+   * If any partition doesn't exists,
+   * the operation of dropPartitions need to be safely rolled back.
+   *
+   * @param idents an array of partition identifiers
+   * @return true if partitions were deleted, false if any partition not exists
+   */
+  boolean dropPartitions(InternalRow[] idents);
+
+  /**
+   * Drop an array of partitions atomically from table, and completely remove partitions data
+   * by skipping a trash even if it is supported.
+   * <p>
+   * If any partition doesn't exists,
+   * the operation of purgePartitions need to be safely rolled back.
+   *
+   * @param idents an array of partition identifiers
+   * @return true if partitions were deleted, false if any partition not exists
+   * @throws NoSuchPartitionException If any partition identifier to alter doesn't exist
+   * @throws UnsupportedOperationException If partition purging is not supported
+   *
+   * @since 3.2.0
+   */
+  default boolean purgePartitions(InternalRow[] idents)
+    throws NoSuchPartitionException, UnsupportedOperationException {
+    throw new UnsupportedOperationException("Partition purge is not supported");
+  }
+
+  /**
+   * Truncate an array of partitions atomically from table, and completely remove partitions data.
+   * <p>
+   * If any partition doesn't exists,
+   * the operation of truncatePartitions need to be safely rolled back.
+   *
+   * @param idents an array of partition identifiers
+   * @return true if partitions were truncated successfully otherwise false
+   * @throws NoSuchPartitionException If any partition identifier to truncate doesn't exist
+   * @throws UnsupportedOperationException If partition truncate is not supported
+   *
+   * @since 3.2.0
+   */
+  default boolean truncatePartitions(InternalRow[] idents)
+      throws NoSuchPartitionException, UnsupportedOperationException {
+    throw new UnsupportedOperationException("Partitions truncate is not supported");
+  }
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsDelete.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsDelete.java
index 106f3283a62c8..6a28bcac3e4f0 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsDelete.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsDelete.java
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.connector.catalog;
 
 import org.apache.spark.annotation.Evolving;
+import org.apache.spark.sql.sources.AlwaysTrue;
 import org.apache.spark.sql.sources.Filter;
 
 /**
@@ -27,9 +28,33 @@
  * @since 3.0.0
  */
 @Evolving
-public interface SupportsDelete {
+public interface SupportsDelete extends TruncatableTable {
+
+  /**
+   * Checks whether it is possible to delete data from a data source table that matches filter
+   * expressions.
+   * <p>
+   * Rows should be deleted from the data source iff all of the filter expressions match.
+   * That is, the expressions must be interpreted as a set of filters that are ANDed together.
+   * <p>
+   * Spark will call this method at planning time to check whether {@link #deleteWhere(Filter[])}
+   * would reject the delete operation because it requires significant effort. If this method
+   * returns false, Spark will not call {@link #deleteWhere(Filter[])} and will try to rewrite
+   * the delete operation and produce row-level changes if the data source table supports deleting
+   * individual records.
+   *
+   * @param filters filter expressions, used to select rows to delete when all expressions match
+   * @return true if the delete operation can be performed
+   *
+   * @since 3.1.0
+   */
+  default boolean canDeleteWhere(Filter[] filters) {
+    return true;
+  }
+
   /**
-   * Delete data from a data source table that matches filter expressions.
+   * Delete data from a data source table that matches filter expressions. Note that this method
+   * will be invoked only if {@link #canDeleteWhere(Filter[])} returns true.
    * <p>
    * Rows are deleted from the data source iff all of the filter expressions match. That is, the
    * expressions must be interpreted as a set of filters that are ANDed together.
@@ -44,4 +69,14 @@ public interface SupportsDelete {
    * @throws IllegalArgumentException If the delete is rejected due to required effort
    */
   void deleteWhere(Filter[] filters);
+
+  @Override
+  default boolean truncateTable() {
+    Filter[] filters = new Filter[] { new AlwaysTrue() };
+    boolean canDelete = canDeleteWhere(filters);
+    if (canDelete) {
+      deleteWhere(filters);
+    }
+    return canDelete;
+  }
 }
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsMetadataColumns.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsMetadataColumns.java
new file mode 100644
index 0000000000000..b7b715bd456ab
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsMetadataColumns.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.connector.catalog;
+
+import org.apache.spark.annotation.Evolving;
+import org.apache.spark.sql.connector.read.SupportsPushDownRequiredColumns;
+import org.apache.spark.sql.types.StructField;
+import org.apache.spark.sql.types.StructType;
+
+/**
+ * An interface for exposing data columns for a table that are not in the table schema. For example,
+ * a file source could expose a "file" column that contains the path of the file that contained each
+ * row.
+ * <p>
+ * The columns returned by {@link #metadataColumns()} may be passed as {@link StructField} in
+ * requested projections. Sources that implement this interface and column projection using
+ * {@link SupportsPushDownRequiredColumns} must accept metadata fields passed to
+ * {@link SupportsPushDownRequiredColumns#pruneColumns(StructType)}.
+ * <p>
+ * If a table column and a metadata column have the same name, the metadata column will never be
+ * requested. It is recommended that Table implementations reject data column name that conflict
+ * with metadata column names.
+ *
+ * @since 3.1.0
+ */
+@Evolving
+public interface SupportsMetadataColumns extends Table {
+  /**
+   * Metadata columns that are supported by this {@link Table}.
+   * <p>
+   * The columns returned by this method may be passed as {@link StructField} in requested
+   * projections using {@link SupportsPushDownRequiredColumns#pruneColumns(StructType)}.
+   * <p>
+   * If a table column and a metadata column have the same name, the metadata column will never be
+   * requested and is ignored. It is recommended that Table implementations reject data column names
+   * that conflict with metadata column names.
+   *
+   * @return an array of {@link MetadataColumn}
+   */
+  MetadataColumn[] metadataColumns();
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsPartitionManagement.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsPartitionManagement.java
new file mode 100644
index 0000000000000..ec2b61a766499
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsPartitionManagement.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.catalog;
+
+import java.util.Map;
+
+import org.apache.spark.annotation.Experimental;
+import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.catalyst.analysis.NoSuchPartitionException;
+import org.apache.spark.sql.catalyst.analysis.PartitionAlreadyExistsException;
+import org.apache.spark.sql.types.StructType;
+
+/**
+ * A partition interface of {@link Table}.
+ * A partition is composed of identifier and properties,
+ * and properties contains metadata information of the partition.
+ * <p>
+ * These APIs are used to modify table partition identifier or partition metadata.
+ * In some cases, they will change the table data as well.
+ * <ul>
+ *   <li>{@link #createPartition}: add a partition and any data it contains to the table</li>
+ *   <li>{@link #dropPartition}: remove a partition and any data it contains from the table</li>
+ *   <li>{@link #purgePartition}: remove a partition and any data it contains from the table by
+ *   skipping a trash even if it is supported.</li>
+ *   <li>{@link #replacePartitionMetadata}: point a partition to a new location, which will swap
+ *   one location's data for the other</li>
+ *   <li>{@link #truncatePartition}: remove partition data from the table</li>
+ * </ul>
+ *
+ * @since 3.1.0
+ */
+@Experimental
+public interface SupportsPartitionManagement extends Table {
+
+    /**
+     * Get the partition schema of table,
+     * this must be consistent with ${@link Table#partitioning()}.
+     * @return the partition schema of table
+     */
+    StructType partitionSchema();
+
+    /**
+     * Create a partition in table.
+     *
+     * @param ident a new partition identifier
+     * @param properties the metadata of a partition
+     * @throws PartitionAlreadyExistsException If a partition already exists for the identifier
+     * @throws UnsupportedOperationException If partition property is not supported
+     */
+    void createPartition(
+        InternalRow ident,
+        Map<String, String> properties)
+        throws PartitionAlreadyExistsException, UnsupportedOperationException;
+
+    /**
+     * Drop a partition from table.
+     *
+     * @param ident a partition identifier
+     * @return true if a partition was deleted, false if no partition exists for the identifier
+     */
+    boolean dropPartition(InternalRow ident);
+
+    /**
+     * Drop a partition from the table and completely remove partition data by skipping a trash
+     * even if it is supported.
+     *
+     * @param ident a partition identifier
+     * @return true if a partition was deleted, false if no partition exists for the identifier
+     * @throws NoSuchPartitionException If the partition identifier to alter doesn't exist
+     * @throws UnsupportedOperationException If partition purging is not supported
+     *
+     * @since 3.2.0
+     */
+    default boolean purgePartition(InternalRow ident)
+      throws NoSuchPartitionException, UnsupportedOperationException {
+      throw new UnsupportedOperationException("Partition purge is not supported");
+    }
+
+    /**
+     * Test whether a partition exists using an {@link InternalRow ident} from the table.
+     *
+     * @param ident a partition identifier which must contain all partition fields in order
+     * @return true if the partition exists, false otherwise
+     */
+    default boolean partitionExists(InternalRow ident) {
+      String[] partitionNames = partitionSchema().names();
+      if (ident.numFields() == partitionNames.length) {
+        return listPartitionIdentifiers(partitionNames, ident).length > 0;
+      } else {
+        throw new IllegalArgumentException("The number of fields (" + ident.numFields() +
+          ") in the partition identifier is not equal to the partition schema length (" +
+          partitionNames.length + "). The identifier might not refer to one partition.");
+      }
+    }
+
+    /**
+     * Replace the partition metadata of the existing partition.
+     *
+     * @param ident the partition identifier of the existing partition
+     * @param properties the new metadata of the partition
+     * @throws NoSuchPartitionException If the partition identifier to alter doesn't exist
+     * @throws UnsupportedOperationException If partition property is not supported
+     */
+    void replacePartitionMetadata(
+        InternalRow ident,
+        Map<String, String> properties)
+        throws NoSuchPartitionException, UnsupportedOperationException;
+
+    /**
+     * Retrieve the partition metadata of the existing partition.
+     *
+     * @param ident a partition identifier
+     * @return the metadata of the partition
+     * @throws UnsupportedOperationException If partition property is not supported
+     */
+    Map<String, String> loadPartitionMetadata(InternalRow ident)
+        throws UnsupportedOperationException;
+
+    /**
+     * List the identifiers of all partitions that match to the ident by names.
+     *
+     * @param names the names of partition values in the identifier.
+     * @param ident a partition identifier values.
+     * @return an array of Identifiers for the partitions
+     */
+    InternalRow[] listPartitionIdentifiers(String[] names, InternalRow ident);
+
+    /**
+     * Rename an existing partition of the table.
+     *
+     * @param from an existing partition identifier to rename
+     * @param to new partition identifier
+     * @return true if renaming completes successfully otherwise false
+     * @throws UnsupportedOperationException If partition renaming is not supported
+     * @throws PartitionAlreadyExistsException If the `to` partition exists already
+     * @throws NoSuchPartitionException If the `from` partition does not exist
+     *
+     * @since 3.2.0
+     */
+    default boolean renamePartition(InternalRow from, InternalRow to)
+        throws UnsupportedOperationException,
+               PartitionAlreadyExistsException,
+               NoSuchPartitionException {
+      throw new UnsupportedOperationException("Partition renaming is not supported");
+    }
+
+    /**
+     * Truncate a partition in the table by completely removing partition data.
+     *
+     * @param ident a partition identifier
+     * @return true if the partition was truncated successfully otherwise false
+     * @throws NoSuchPartitionException If the partition identifier to alter doesn't exist
+     * @throws UnsupportedOperationException If partition truncation is not supported
+     *
+     * @since 3.2.0
+     */
+    default boolean truncatePartition(InternalRow ident)
+        throws NoSuchPartitionException, UnsupportedOperationException {
+      throw new UnsupportedOperationException("Partition truncate is not supported");
+    }
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCapability.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCapability.java
index 68161d7225fcf..5bb42fb4b313d 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCapability.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCapability.java
@@ -96,7 +96,7 @@ public enum TableCapability {
   /**
    * Signals that the table supports append writes using the V1 InsertableRelation interface.
    * <p>
-   * Tables that return this capability must create a V1WriteBuilder and may also support additional
+   * Tables that return this capability must create a V1Write and may also support additional
    * write modes, like {@link #TRUNCATE}, and {@link #OVERWRITE_BY_FILTER}, but cannot support
    * {@link #OVERWRITE_DYNAMIC}.
    */
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java
index 1809b9cdb52e5..4163d86bcc54b 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java
@@ -46,6 +46,11 @@ public interface TableCatalog extends CatalogPlugin {
    */
   String PROP_LOCATION = "location";
 
+  /**
+   * A reserved property to specify a table was created with EXTERNAL.
+   */
+  String PROP_EXTERNAL = "external";
+
   /**
    * A reserved property to specify the description of the table.
    */
@@ -61,6 +66,11 @@ public interface TableCatalog extends CatalogPlugin {
    */
   String PROP_OWNER = "owner";
 
+  /**
+   * A prefix used to pass OPTIONS in table properties
+   */
+  String OPTION_PREFIX = "option.";
+
   /**
    * List the tables in a namespace from the catalog.
    * <p>
@@ -162,6 +172,26 @@ Table alterTable(
    */
   boolean dropTable(Identifier ident);
 
+  /**
+   * Drop a table in the catalog and completely remove its data by skipping a trash even if it is
+   * supported.
+   * <p>
+   * If the catalog supports views and contains a view for the identifier and not a table, this
+   * must not drop the view and must return false.
+   * <p>
+   * If the catalog supports to purge a table, this method should be overridden.
+   * The default implementation throws {@link UnsupportedOperationException}.
+   *
+   * @param ident a table identifier
+   * @return true if a table was deleted, false if no table exists for the identifier
+   * @throws UnsupportedOperationException If table purging is not supported
+   *
+   * @since 3.1.0
+   */
+  default boolean purgeTable(Identifier ident) throws UnsupportedOperationException {
+    throw new UnsupportedOperationException("Purge table is not supported.");
+  }
+
   /**
    * Renames a table in the catalog.
    * <p>
@@ -176,7 +206,7 @@ Table alterTable(
    * @param newIdent the new table identifier of the table
    * @throws NoSuchTableException If the table to rename doesn't exist or is a view
    * @throws TableAlreadyExistsException If the new table name already exists or is a view
-   * @throws UnsupportedOperationException If the namespaces of old and new identiers do not
+   * @throws UnsupportedOperationException If the namespaces of old and new identifiers do not
    *                                       match (optional)
    */
   void renameTable(Identifier oldIdent, Identifier newIdent)
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableProvider.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableProvider.java
index 82731e2c8e1e8..6b68fbe123019 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableProvider.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableProvider.java
@@ -77,12 +77,18 @@ default Transform[] inferPartitioning(CaseInsensitiveStringMap options) {
 
   /**
    * Returns true if the source has the ability of accepting external table metadata when getting
-   * tables. The external table metadata includes user-specified schema from
-   * `DataFrameReader`/`DataStreamReader` and schema/partitioning stored in Spark catalog.
+   * tables. The external table metadata includes:
+   * <ol>
+   *   <li>For table reader: user-specified schema from {@code DataFrameReader}/{@code
+   *   DataStreamReader} and schema/partitioning stored in Spark catalog.</li>
+   *   <li>For table writer: the schema of the input {@code Dataframe} of
+   *   {@code DataframeWriter}/{@code DataStreamWriter}.</li>
+   * </ol>
    * <p>
    * By default this method returns false, which means the schema and partitioning passed to
-   * `getTable` are from the infer methods. Please override it if this source has expensive
-   * schema/partitioning inference and wants external table metadata to avoid inference.
+   * {@link #getTable(StructType, Transform[], Map)} are from the infer methods. Please override it
+   * if this source has expensive schema/partitioning inference and wants external table metadata
+   * to avoid inference.
    */
   default boolean supportsExternalMetadata() {
     return false;
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TruncatableTable.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TruncatableTable.java
new file mode 100644
index 0000000000000..a69f3849130e8
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TruncatableTable.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.catalog;
+
+import org.apache.spark.annotation.Evolving;
+
+/**
+ * Represents a table which can be atomically truncated.
+ */
+@Evolving
+public interface TruncatableTable extends Table {
+  /**
+   * Truncate a table by removing all rows from the table atomically.
+   *
+   * @return true if a table was truncated successfully otherwise false
+   *
+   * @since 3.2.0
+   */
+  boolean truncateTable();
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/functions/AggregateFunction.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/functions/AggregateFunction.java
new file mode 100644
index 0000000000000..4181feafed101
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/functions/AggregateFunction.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.catalog.functions;
+
+import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.types.DataType;
+
+import java.io.Serializable;
+
+/**
+ * Interface for a function that produces a result value by aggregating over multiple input rows.
+ * <p>
+ * For each input row, Spark will call the {@link #update} method which should evaluate the row
+ * and update the aggregation state. The JVM type of result values produced by
+ * {@link #produceResult} must be the type used by Spark's
+ * InternalRow API for the {@link DataType SQL data type} returned by {@link #resultType()}.
+ * Please refer to class documentation of {@link ScalarFunction} for the mapping between
+ * {@link DataType} and the JVM type.
+ * <p>
+ * All implementations must support partial aggregation by implementing merge so that Spark can
+ * partially aggregate and shuffle intermediate results, instead of shuffling all rows for an
+ * aggregate. This reduces the impact of data skew and the amount of data shuffled to produce the
+ * result.
+ * <p>
+ * Intermediate aggregation state must be {@link Serializable} so that state produced by parallel
+ * tasks can be serialized, shuffled, and then merged to produce a final result.
+ *
+ * @param <S> the JVM type for the aggregation's intermediate state; must be {@link Serializable}
+ * @param <R> the JVM type of result values
+ */
+public interface AggregateFunction<S extends Serializable, R> extends BoundFunction {
+
+  /**
+   * Initialize state for an aggregation.
+   * <p>
+   * This method is called one or more times for every group of values to initialize intermediate
+   * aggregation state. More than one intermediate aggregation state variable may be used when the
+   * aggregation is run in parallel tasks.
+   * <p>
+   * Implementations that return null must support null state passed into all other methods.
+   *
+   * @return a state instance or null
+   */
+  S newAggregationState();
+
+  /**
+   * Update the aggregation state with a new row.
+   * <p>
+   * This is called for each row in a group to update an intermediate aggregation state.
+   *
+   * @param state intermediate aggregation state
+   * @param input an input row
+   * @return updated aggregation state
+   */
+  S update(S state, InternalRow input);
+
+  /**
+   * Merge two partial aggregation states.
+   * <p>
+   * This is called to merge intermediate aggregation states that were produced by parallel tasks.
+   *
+   * @param leftState intermediate aggregation state
+   * @param rightState intermediate aggregation state
+   * @return combined aggregation state
+   */
+  S merge(S leftState, S rightState);
+
+  /**
+   * Produce the aggregation result based on intermediate state.
+   *
+   * @param state intermediate aggregation state
+   * @return a result value
+   */
+  R produceResult(S state);
+
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/functions/BoundFunction.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/functions/BoundFunction.java
new file mode 100644
index 0000000000000..c53f94a16893e
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/functions/BoundFunction.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.catalog.functions;
+
+import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.types.IntegerType;
+import org.apache.spark.sql.types.StructType;
+
+import java.util.UUID;
+
+/**
+ * Represents a function that is bound to an input type.
+ */
+public interface BoundFunction extends Function {
+
+  /**
+   * Returns the required {@link DataType data types} of the input values to this function.
+   * <p>
+   * If the types returned differ from the types passed to {@link UnboundFunction#bind(StructType)},
+   * Spark will cast input values to the required data types. This allows implementations to
+   * delegate input value casting to Spark.
+   *
+   * @return an array of input value data types
+   */
+  DataType[] inputTypes();
+
+  /**
+   * Returns the {@link DataType data type} of values produced by this function.
+   * <p>
+   * For example, a "plus" function may return {@link IntegerType} when it is bound to arguments
+   * that are also {@link IntegerType}.
+   *
+   * @return a data type for values produced by this function
+   */
+  DataType resultType();
+
+  /**
+   * Returns whether the values produced by this function may be null.
+   * <p>
+   * For example, a "plus" function may return false when it is bound to arguments that are always
+   * non-null, but true when either argument may be null.
+   *
+   * @return true if values produced by this function may be null, false otherwise
+   */
+  default boolean isResultNullable() {
+    return true;
+  }
+
+  /**
+   * Returns whether this function result is deterministic.
+   * <p>
+   * By default, functions are assumed to be deterministic. Functions that are not deterministic
+   * should override this method so that Spark can ensure the function runs only once for a given
+   * input.
+   *
+   * @return true if this function is deterministic, false otherwise
+   */
+  default boolean isDeterministic() {
+    return true;
+  }
+
+  /**
+   * Returns the canonical name of this function, used to determine if functions are equivalent.
+   * <p>
+   * The canonical name is used to determine whether two functions are the same when loaded by
+   * different catalogs. For example, the same catalog implementation may be used for by two
+   * environments, "prod" and "test". Functions produced by the catalogs may be equivalent, but
+   * loaded using different names, like "test.func_name" and "prod.func_name".
+   * <p>
+   * Names returned by this function should be unique and unlikely to conflict with similar
+   * functions in other catalogs. For example, many catalogs may define a "bucket" function with a
+   * different implementation. Adding context, like "com.mycompany.bucket(string)", is recommended
+   * to avoid unintentional collisions.
+   *
+   * @return a canonical name for this function
+   */
+  default String canonicalName() {
+    // by default, use a random UUID so a function is never equivalent to another, even itself.
+    // this method is not required so that generated implementations (or careless ones) are not
+    // added and forgotten. for example, returning "" as a place-holder could cause unnecessary
+    // bugs if not replaced before release.
+    return UUID.randomUUID().toString();
+  }
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/functions/Function.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/functions/Function.java
new file mode 100644
index 0000000000000..b7f14eb271ef6
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/functions/Function.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.catalog.functions;
+
+import java.io.Serializable;
+
+/**
+ * Base class for user-defined functions.
+ */
+public interface Function extends Serializable {
+
+  /**
+   * A name to identify this function. Implementations should provide a meaningful name, like the
+   * database and function name from the catalog.
+   */
+  String name();
+
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/functions/ScalarFunction.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/functions/ScalarFunction.java
new file mode 100644
index 0000000000000..858ab923490fc
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/functions/ScalarFunction.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.catalog.functions;
+
+import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.types.DataType;
+
+/**
+ * Interface for a function that produces a result value for each input row.
+ * <p>
+ * To evaluate each input row, Spark will first try to lookup and use a "magic method" (described
+ * below) through Java reflection. If the method is not found, Spark will call
+ * {@link #produceResult(InternalRow)} as a fallback approach.
+ * <p>
+ * The JVM type of result values produced by this function must be the type used by Spark's
+ * InternalRow API for the {@link DataType SQL data type} returned by {@link #resultType()}.
+ * The mapping between {@link DataType} and the corresponding JVM type is defined below.
+ * <p>
+ * <b>IMPORTANT</b>: the default implementation of {@link #produceResult} throws
+ * {@link UnsupportedOperationException}. Users must choose to either override this method, or
+ * implement a magic method with name {@link #MAGIC_METHOD_NAME}, which takes individual parameters
+ * instead of a {@link InternalRow}. The magic method approach is generally recommended because it
+ * provides better performance over the default {@link #produceResult}, due to optimizations such
+ * as whole-stage codegen, elimination of Java boxing, etc.
+ * <p>
+ * In addition, for stateless Java functions, users can optionally define the
+ * {@link #MAGIC_METHOD_NAME} as a static method, which further avoids certain runtime costs such
+ * as Java dynamic dispatch.
+ * <p>
+ * For example, a scalar UDF for adding two integers can be defined as follow with the magic
+ * method approach:
+ *
+ * <pre>
+ *   public class IntegerAdd implements{@code ScalarFunction<Integer>} {
+ *     public DataType[] inputTypes() {
+ *       return new DataType[] { DataTypes.IntegerType, DataTypes.IntegerType };
+ *     }
+ *     public int invoke(int left, int right) {
+ *       return left + right;
+ *     }
+ *   }
+ * </pre>
+ * In the above, since {@link #MAGIC_METHOD_NAME} is defined, and also that it has
+ * matching parameter types and return type, Spark will use it to evaluate inputs.
+ * <p>
+ * As another example, in the following:
+ * <pre>
+ *   public class IntegerAdd implements{@code ScalarFunction<Integer>} {
+ *     public DataType[] inputTypes() {
+ *       return new DataType[] { DataTypes.IntegerType, DataTypes.IntegerType };
+ *     }
+ *     public static int invoke(int left, int right) {
+ *       return left + right;
+ *     }
+ *     public Integer produceResult(InternalRow input) {
+ *       return input.getInt(0) + input.getInt(1);
+ *     }
+ *   }
+ * </pre>
+ *
+ * the class defines both the magic method and the {@link #produceResult}, and Spark will use
+ * {@link #MAGIC_METHOD_NAME} over the {@link #produceResult(InternalRow)} as it takes higher
+ * precedence. Also note that the magic method is annotated as a static method in this case.
+ * <p>
+ * Resolution on magic method is done during query analysis, where Spark looks up the magic
+ * method by first converting the actual input SQL data types to their corresponding Java types
+ * following the mapping defined below, and then checking if there is a matching method from all the
+ * declared methods in the UDF class, using method name and the Java types.
+ * <p>
+ * The following are the mapping from {@link DataType SQL data type} to Java type which is used
+ * by Spark to infer parameter types for the magic methods as well as return value type for
+ * {@link #produceResult}:
+ * <ul>
+ *   <li>{@link org.apache.spark.sql.types.BooleanType}: {@code boolean}</li>
+ *   <li>{@link org.apache.spark.sql.types.ByteType}: {@code byte}</li>
+ *   <li>{@link org.apache.spark.sql.types.ShortType}: {@code short}</li>
+ *   <li>{@link org.apache.spark.sql.types.IntegerType}: {@code int}</li>
+ *   <li>{@link org.apache.spark.sql.types.LongType}: {@code long}</li>
+ *   <li>{@link org.apache.spark.sql.types.FloatType}: {@code float}</li>
+ *   <li>{@link org.apache.spark.sql.types.DoubleType}: {@code double}</li>
+ *   <li>{@link org.apache.spark.sql.types.StringType}:
+ *       {@link org.apache.spark.unsafe.types.UTF8String}</li>
+ *   <li>{@link org.apache.spark.sql.types.DateType}: {@code int}</li>
+ *   <li>{@link org.apache.spark.sql.types.TimestampType}: {@code long}</li>
+ *   <li>{@link org.apache.spark.sql.types.BinaryType}: {@code byte[]}</li>
+ *   <li>{@link org.apache.spark.sql.types.DayTimeIntervalType}: {@code long}</li>
+ *   <li>{@link org.apache.spark.sql.types.YearMonthIntervalType}: {@code int}</li>
+ *   <li>{@link org.apache.spark.sql.types.DecimalType}:
+ *       {@link org.apache.spark.sql.types.Decimal}</li>
+ *   <li>{@link org.apache.spark.sql.types.StructType}: {@link InternalRow}</li>
+ *   <li>{@link org.apache.spark.sql.types.ArrayType}:
+ *       {@link org.apache.spark.sql.catalyst.util.ArrayData}</li>
+ *   <li>{@link org.apache.spark.sql.types.MapType}:
+ *       {@link org.apache.spark.sql.catalyst.util.MapData}</li>
+ * </ul>
+ *
+ * @param <R> the JVM type of result values, MUST be consistent with the {@link DataType}
+ *          returned via {@link #resultType()}, according to the mapping above.
+ */
+public interface ScalarFunction<R> extends BoundFunction {
+  String MAGIC_METHOD_NAME = "invoke";
+
+  /**
+   * Applies the function to an input row to produce a value.
+   *
+   * @param input an input row
+   * @return a result value
+   */
+  default R produceResult(InternalRow input) {
+    throw new UnsupportedOperationException(
+        "Cannot find a compatible ScalarFunction#produceResult");
+  }
+
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/functions/UnboundFunction.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/functions/UnboundFunction.java
new file mode 100644
index 0000000000000..c7dd4c2b881c3
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/functions/UnboundFunction.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.catalog.functions;
+
+import org.apache.spark.sql.types.StructType;
+
+/**
+ * Represents a user-defined function that is not bound to input types.
+ */
+public interface UnboundFunction extends Function {
+
+  /**
+   * Bind this function to an input type.
+   * <p>
+   * If the input type is not supported, implementations must throw
+   * {@link UnsupportedOperationException}.
+   * <p>
+   * For example, a "length" function that only supports a single string argument should throw
+   * UnsupportedOperationException if the struct has more than one field or if that field is not a
+   * string, and it may optionally throw if the field is nullable.
+   *
+   * @param inputType a struct type for inputs that will be passed to the bound function
+   * @return a function that can process rows with the given input type
+   * @throws UnsupportedOperationException If the function cannot be applied to the input type
+   */
+  BoundFunction bind(StructType inputType);
+
+  /**
+   * Returns Function documentation.
+   *
+   * @return this function's documentation
+   */
+  String description();
+
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/distributions/ClusteredDistribution.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/distributions/ClusteredDistribution.java
new file mode 100644
index 0000000000000..dcc3d191461ce
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/distributions/ClusteredDistribution.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.distributions;
+
+import org.apache.spark.annotation.Experimental;
+import org.apache.spark.sql.connector.expressions.Expression;
+
+/**
+ * A distribution where tuples that share the same values for clustering expressions are co-located
+ * in the same partition.
+ *
+ * @since 3.2.0
+ */
+@Experimental
+public interface ClusteredDistribution extends Distribution {
+  /**
+   * Returns clustering expressions.
+   */
+  Expression[] clustering();
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/distributions/Distribution.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/distributions/Distribution.java
new file mode 100644
index 0000000000000..95d68ea2d1abe
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/distributions/Distribution.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.distributions;
+
+import org.apache.spark.annotation.Experimental;
+
+/**
+ * An interface that defines how data is distributed across partitions.
+ *
+ * @since 3.2.0
+ */
+@Experimental
+public interface Distribution {}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/distributions/Distributions.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/distributions/Distributions.java
new file mode 100644
index 0000000000000..da5d6f8c81a3f
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/distributions/Distributions.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.distributions;
+
+import org.apache.spark.annotation.Experimental;
+import org.apache.spark.sql.connector.expressions.Expression;
+import org.apache.spark.sql.connector.expressions.SortOrder;
+
+/**
+ * Helper methods to create distributions to pass into Spark.
+ *
+ * @since 3.2.0
+ */
+@Experimental
+public class Distributions {
+  private Distributions() {
+  }
+
+  /**
+   * Creates a distribution where no promises are made about co-location of data.
+   */
+  public static UnspecifiedDistribution unspecified() {
+    return LogicalDistributions.unspecified();
+  }
+
+  /**
+   * Creates a distribution where tuples that share the same values for clustering expressions are
+   * co-located in the same partition.
+   */
+  public static ClusteredDistribution clustered(Expression[] clustering) {
+    return LogicalDistributions.clustered(clustering);
+  }
+
+  /**
+   * Creates a distribution where tuples have been ordered across partitions according
+   * to ordering expressions, but not necessarily within a given partition.
+   */
+  public static OrderedDistribution ordered(SortOrder[] ordering) {
+    return LogicalDistributions.ordered(ordering);
+  }
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/distributions/OrderedDistribution.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/distributions/OrderedDistribution.java
new file mode 100644
index 0000000000000..3456178d8e64f
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/distributions/OrderedDistribution.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.distributions;
+
+import org.apache.spark.annotation.Experimental;
+import org.apache.spark.sql.connector.expressions.SortOrder;
+
+/**
+ * A distribution where tuples have been ordered across partitions according
+ * to ordering expressions, but not necessarily within a given partition.
+ *
+ * @since 3.2.0
+ */
+@Experimental
+public interface OrderedDistribution extends Distribution {
+  /**
+   * Returns ordering expressions.
+   */
+  SortOrder[] ordering();
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/distributions/UnspecifiedDistribution.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/distributions/UnspecifiedDistribution.java
new file mode 100644
index 0000000000000..ea18d8906cfd0
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/distributions/UnspecifiedDistribution.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.distributions;
+
+import org.apache.spark.annotation.Experimental;
+
+/**
+ * A distribution where no promises are made about co-location of data.
+ *
+ * @since 3.2.0
+ */
+@Experimental
+public interface UnspecifiedDistribution extends Distribution {}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expressions.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expressions.java
index 791dc969ab008..6aed8896e9f58 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expressions.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expressions.java
@@ -164,4 +164,30 @@ public static Transform hours(String column) {
     return LogicalExpressions.hours(Expressions.column(column));
   }
 
+  /**
+   * Create a sort expression.
+   *
+   * @param expr an expression to produce values to sort
+   * @param direction direction of the sort
+   * @param nullOrder null order of the sort
+   * @return a SortOrder
+   *
+   * @since 3.2.0
+   */
+  public static SortOrder sort(Expression expr, SortDirection direction, NullOrdering nullOrder) {
+    return LogicalExpressions.sort(expr, direction, nullOrder);
+  }
+
+  /**
+   * Create a sort expression.
+   *
+   * @param expr an expression to produce values to sort
+   * @param direction direction of the sort
+   * @return a SortOrder
+   *
+   * @since 3.2.0
+   */
+  public static SortOrder sort(Expression expr, SortDirection direction) {
+    return LogicalExpressions.sort(expr, direction, direction.defaultNullOrdering());
+  }
 }
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/NullOrdering.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/NullOrdering.java
new file mode 100644
index 0000000000000..669d1c8443b15
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/NullOrdering.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.expressions;
+
+import org.apache.spark.annotation.Experimental;
+
+/**
+ * A null order used in sorting expressions.
+ *
+ * @since 3.2.0
+ */
+@Experimental
+public enum NullOrdering {
+  NULLS_FIRST, NULLS_LAST;
+
+  @Override
+  public String toString() {
+    switch (this) {
+      case NULLS_FIRST:
+        return "NULLS FIRST";
+      case NULLS_LAST:
+        return "NULLS LAST";
+      default:
+        throw new IllegalArgumentException("Unexpected null order: " + this);
+    }
+  }
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/SortDirection.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/SortDirection.java
new file mode 100644
index 0000000000000..e42dd5bcc2e13
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/SortDirection.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.expressions;
+
+import org.apache.spark.annotation.Experimental;
+
+import static org.apache.spark.sql.connector.expressions.NullOrdering.NULLS_FIRST;
+import static org.apache.spark.sql.connector.expressions.NullOrdering.NULLS_LAST;
+
+/**
+ * A sort direction used in sorting expressions.
+ * <p>
+ * Each direction has a default null ordering that is implied if no null ordering is specified
+ * explicitly.
+ *
+ * @since 3.2.0
+ */
+@Experimental
+public enum SortDirection {
+  ASCENDING(NULLS_FIRST), DESCENDING(NULLS_LAST);
+
+  private final NullOrdering defaultNullOrdering;
+
+  SortDirection(NullOrdering defaultNullOrdering) {
+    this.defaultNullOrdering = defaultNullOrdering;
+  }
+
+  /**
+   * Returns the default null ordering to use if no null ordering is specified explicitly.
+   */
+  public NullOrdering defaultNullOrdering() {
+    return defaultNullOrdering;
+  }
+
+  @Override
+  public String toString() {
+    switch (this) {
+      case ASCENDING:
+        return "ASC";
+      case DESCENDING:
+        return "DESC";
+      default:
+        throw new IllegalArgumentException("Unexpected sort direction: " + this);
+    }
+  }
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/SortOrder.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/SortOrder.java
new file mode 100644
index 0000000000000..72252457df26e
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/SortOrder.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.expressions;
+
+import org.apache.spark.annotation.Experimental;
+
+/**
+ * Represents a sort order in the public expression API.
+ *
+ * @since 3.2.0
+ */
+@Experimental
+public interface SortOrder extends Expression {
+  /**
+   * Returns the sort expression.
+   */
+  Expression expression();
+
+  /**
+   * Returns the sort direction.
+   */
+  SortDirection direction();
+
+  /**
+   * Returns the null ordering.
+   */
+  NullOrdering nullOrdering();
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/metric/CustomAvgMetric.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/metric/CustomAvgMetric.java
new file mode 100644
index 0000000000000..71e83002dda07
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/metric/CustomAvgMetric.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.metric;
+
+import org.apache.spark.annotation.Evolving;
+
+import java.util.Arrays;
+import java.text.DecimalFormat;
+
+/**
+ * Built-in `CustomMetric` that computes average of metric values. Note that please extend this
+ * class and override `name` and `description` to create your custom metric for real usage.
+ *
+ * @since 3.2.0
+ */
+@Evolving
+public abstract class CustomAvgMetric implements CustomMetric {
+  @Override
+  public String aggregateTaskMetrics(long[] taskMetrics) {
+    if (taskMetrics.length > 0) {
+      double average = ((double)Arrays.stream(taskMetrics).sum()) / taskMetrics.length;
+      return new DecimalFormat("#0.000").format(average);
+    } else {
+      return "0";
+    }
+  }
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/metric/CustomMetric.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/metric/CustomMetric.java
new file mode 100644
index 0000000000000..4c4151ad96975
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/metric/CustomMetric.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.metric;
+
+import org.apache.spark.annotation.Evolving;
+
+/**
+ * A custom metric. Data source can define supported custom metrics using this interface.
+ * During query execution, Spark will collect the task metrics using {@link CustomTaskMetric}
+ * and combine the metrics at the driver side. How to combine task metrics is defined by the
+ * metric class with the same metric name.
+ *
+ * @since 3.2.0
+ */
+@Evolving
+public interface CustomMetric {
+  /**
+   * Returns the name of custom metric.
+   */
+  String name();
+
+  /**
+   * Returns the description of custom metric.
+   */
+  String description();
+
+  /**
+   * The initial value of this metric.
+   */
+  long initialValue = 0L;
+
+  /**
+   * Given an array of task metric values, returns aggregated final metric value.
+   */
+  String aggregateTaskMetrics(long[] taskMetrics);
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/metric/CustomSumMetric.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/metric/CustomSumMetric.java
new file mode 100644
index 0000000000000..ba28e9b9187ee
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/metric/CustomSumMetric.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.metric;
+
+import org.apache.spark.annotation.Evolving;
+
+import java.util.Arrays;
+
+/**
+ * Built-in `CustomMetric` that sums up metric values. Note that please extend this class
+ * and override `name` and `description` to create your custom metric for real usage.
+ *
+ * @since 3.2.0
+ */
+@Evolving
+public abstract class CustomSumMetric implements CustomMetric {
+  @Override
+  public String aggregateTaskMetrics(long[] taskMetrics) {
+    return String.valueOf(Arrays.stream(taskMetrics).sum());
+  }
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/metric/CustomTaskMetric.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/metric/CustomTaskMetric.java
new file mode 100644
index 0000000000000..1b6f04d927913
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/metric/CustomTaskMetric.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.metric;
+
+import org.apache.spark.annotation.Evolving;
+import org.apache.spark.sql.connector.read.PartitionReader;
+
+/**
+ * A custom task metric. This is a logical representation of a metric reported by data sources
+ * at the executor side. During query execution, Spark will collect the task metrics per partition
+ * by {@link PartitionReader} and update internal metrics based on collected metric values.
+ * For streaming query, Spark will collect and combine metrics for a final result per micro batch.
+ * <p>
+ * The metrics will be gathered during query execution back to the driver and then combined. How
+ * the task metrics are combined is defined by corresponding {@link CustomMetric} with same metric
+ * name. The final result will be shown up in the data source scan operator in Spark UI.
+ *
+ * @since 3.2.0
+ */
+@Evolving
+public interface CustomTaskMetric {
+  /**
+   * Returns the name of custom task metric.
+   */
+  String name();
+
+  /**
+   * Returns the long value of custom task metric.
+   */
+  long value();
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Batch.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Batch.java
index 73aefa55ae5e1..6861f168b9d33 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Batch.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Batch.java
@@ -36,10 +36,8 @@ public interface Batch {
    * <p>
    * If the {@link Scan} supports filter pushdown, this Batch is likely configured with a filter
    * and is responsible for creating splits for that filter, which is not a full scan.
-   * </p>
    * <p>
    * This method will be called only once during a data source scan, to launch one Spark job.
-   * </p>
    */
   InputPartition[] planInputPartitions();
 
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/InputPartition.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/InputPartition.java
index 03eec00168012..b9e7c375b488b 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/InputPartition.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/InputPartition.java
@@ -24,7 +24,7 @@
 /**
  * A serializable representation of an input partition returned by
  * {@link Batch#planInputPartitions()} and the corresponding ones in streaming .
- *
+ * <p>
  * Note that {@link InputPartition} will be serialized and sent to executors, then
  * {@link PartitionReader} will be created by
  * {@link PartitionReaderFactory#createReader(InputPartition)} or
@@ -42,11 +42,11 @@ public interface InputPartition extends Serializable {
    * faster, but Spark does not guarantee to run the input partition reader on these locations.
    * The implementations should make sure that it can be run on any location.
    * The location is a string representing the host name.
-   *
+   * <p>
    * Note that if a host name cannot be recognized by Spark, it will be ignored as it was not in
    * the returned locations. The default return value is empty string array, which means this
    * input partition's reader has no location preference.
-   *
+   * <p>
    * If this method fails (by throwing an exception), the action will fail and no Spark job will be
    * submitted.
    */
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/PartitionReader.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/PartitionReader.java
index 23fbd95800e26..5286bbf9f85a1 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/PartitionReader.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/PartitionReader.java
@@ -21,12 +21,13 @@
 import java.io.IOException;
 
 import org.apache.spark.annotation.Evolving;
+import org.apache.spark.sql.connector.metric.CustomTaskMetric;
 
 /**
  * A partition reader returned by {@link PartitionReaderFactory#createReader(InputPartition)} or
  * {@link PartitionReaderFactory#createColumnarReader(InputPartition)}. It's responsible for
  * outputting data for a RDD partition.
- *
+ * <p>
  * Note that, Currently the type `T` can only be {@link org.apache.spark.sql.catalyst.InternalRow}
  * for normal data sources, or {@link org.apache.spark.sql.vectorized.ColumnarBatch} for columnar
  * data sources(whose {@link PartitionReaderFactory#supportColumnarReads(InputPartition)}
@@ -48,4 +49,13 @@ public interface PartitionReader<T> extends Closeable {
    * Return the current record. This method should return same value until `next` is called.
    */
   T get();
+
+  /**
+   * Returns an array of custom task metrics. By default it returns empty array. Note that it is
+   * not recommended to put heavy logic in this method as it may affect reading performance.
+   */
+  default CustomTaskMetric[] currentMetricsValues() {
+    CustomTaskMetric[] NO_METRICS = {};
+    return NO_METRICS;
+  }
 }
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/PartitionReaderFactory.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/PartitionReaderFactory.java
index 9dded247e89f3..52204fd3bf4a7 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/PartitionReaderFactory.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/PartitionReaderFactory.java
@@ -25,7 +25,7 @@
 
 /**
  * A factory used to create {@link PartitionReader} instances.
- *
+ * <p>
  * If Spark fails to execute any methods in the implementations of this interface or in the returned
  * {@link PartitionReader} (by throwing an exception), corresponding Spark task would fail and
  * get retried until hitting the maximum retry times.
@@ -37,7 +37,7 @@ public interface PartitionReaderFactory extends Serializable {
 
   /**
    * Returns a row-based partition reader to read data from the given {@link InputPartition}.
-   *
+   * <p>
    * Implementations probably need to cast the input partition to the concrete
    * {@link InputPartition} class defined for the data source.
    */
@@ -45,7 +45,7 @@ public interface PartitionReaderFactory extends Serializable {
 
   /**
    * Returns a columnar partition reader to read data from the given {@link InputPartition}.
-   *
+   * <p>
    * Implementations probably need to cast the input partition to the concrete
    * {@link InputPartition} class defined for the data source.
    */
@@ -57,7 +57,7 @@ default PartitionReader<ColumnarBatch> createColumnarReader(InputPartition parti
    * Returns true if the given {@link InputPartition} should be read by Spark in a columnar way.
    * This means, implementations must also implement {@link #createColumnarReader(InputPartition)}
    * for the input partitions that this method returns true.
-   *
+   * <p>
    * As of Spark 2.4, Spark can only read all input partition in a columnar way, or none of them.
    * Data source can't mix columnar and row-based partitions. This may be relaxed in future
    * versions.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Scan.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Scan.java
index 4146f217985b4..0c009f5c56d06 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Scan.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Scan.java
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.connector.read;
 
 import org.apache.spark.annotation.Evolving;
+import org.apache.spark.sql.connector.metric.CustomMetric;
 import org.apache.spark.sql.connector.read.streaming.ContinuousStream;
 import org.apache.spark.sql.connector.read.streaming.MicroBatchStream;
 import org.apache.spark.sql.types.StructType;
@@ -102,4 +103,13 @@ default MicroBatchStream toMicroBatchStream(String checkpointLocation) {
   default ContinuousStream toContinuousStream(String checkpointLocation) {
     throw new UnsupportedOperationException(description() + ": Continuous scan are not supported");
   }
+
+  /**
+   * Returns an array of supported custom metrics with name and description.
+   * By default it returns empty array.
+   */
+  default CustomMetric[] supportedCustomMetrics() {
+    CustomMetric[] NO_METRICS = {};
+    return NO_METRICS;
+  }
 }
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsPushDownFilters.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsPushDownFilters.java
index 6594af27734c7..5ab9092c9a1bd 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsPushDownFilters.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsPushDownFilters.java
@@ -39,14 +39,17 @@ public interface SupportsPushDownFilters extends ScanBuilder {
 
   /**
    * Returns the filters that are pushed to the data source via {@link #pushFilters(Filter[])}.
-   *
+   * <p>
    * There are 3 kinds of filters:
-   *  1. pushable filters which don't need to be evaluated again after scanning.
-   *  2. pushable filters which still need to be evaluated after scanning, e.g. parquet
-   *     row group filter.
-   *  3. non-pushable filters.
+   * <ol>
+   *  <li>pushable filters which don't need to be evaluated again after scanning.</li>
+   *  <li>pushable filters which still need to be evaluated after scanning, e.g. parquet row
+   *  group filter.</li>
+   *  <li>non-pushable filters.</li>
+   * </ol>
+   * <p>
    * Both case 1 and 2 should be considered as pushed filters and should be returned by this method.
-   *
+   * <p>
    * It's possible that there is no filters in the query and {@link #pushFilters(Filter[])}
    * is never called, empty array should be returned for this case.
    */
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsPushDownRequiredColumns.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsPushDownRequiredColumns.java
index 4f7da3c2a38f4..34de67bcf466a 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsPushDownRequiredColumns.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsPushDownRequiredColumns.java
@@ -32,11 +32,11 @@ public interface SupportsPushDownRequiredColumns extends ScanBuilder {
 
   /**
    * Applies column pruning w.r.t. the given requiredSchema.
-   *
+   * <p>
    * Implementation should try its best to prune the unnecessary columns or nested fields, but it's
    * also OK to do the pruning partially, e.g., a data source may not be able to prune nested
    * fields, and only prune top-level columns.
-   *
+   * <p>
    * Note that, {@link Scan#readSchema()} implementation should take care of the column
    * pruning applied here.
    */
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportPartitioning.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportPartitioning.java
index e7a27e07498fb..51d56bdf0a3ef 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportPartitioning.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportPartitioning.java
@@ -23,7 +23,7 @@
 /**
  * A mix in interface for {@link Scan}. Data sources can implement this interface to
  * report data partitioning and try to avoid shuffle at Spark side.
- *
+ * <p>
  * Note that, when a {@link Scan} implementation creates exactly one {@link InputPartition},
  * Spark may avoid adding a shuffle even if the reader does not implement this interface.
  *
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportStatistics.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportStatistics.java
index b839fd5a4a726..031749dee0350 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportStatistics.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportStatistics.java
@@ -22,10 +22,10 @@
 /**
  * A mix in interface for {@link Scan}. Data sources can implement this interface to
  * report statistics to Spark.
- *
- * As of Spark 2.4, statistics are reported to the optimizer before any operator is pushed to the
- * data source. Implementations that return more accurate statistics based on pushed operators will
- * not improve query performance until the planner can push operators before getting stats.
+ * <p>
+ * As of Spark 3.0, statistics are reported to the optimizer after operators are pushed to the
+ * data source. Implementations may return more accurate statistics based on pushed operators
+ * which may improve query performance by providing better information to the optimizer.
  *
  * @since 3.0.0
  */
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/Distribution.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/Distribution.java
index 264b268e247e1..a5911a820ac10 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/Distribution.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/Distribution.java
@@ -24,14 +24,15 @@
  * An interface to represent data distribution requirement, which specifies how the records should
  * be distributed among the data partitions (one {@link PartitionReader} outputs data for one
  * partition).
+ * <p>
  * Note that this interface has nothing to do with the data ordering inside one
  * partition(the output records of a single {@link PartitionReader}).
- *
+ * <p>
  * The instance of this interface is created and provided by Spark, then consumed by
  * {@link Partitioning#satisfy(Distribution)}. This means data source developers don't need to
  * implement this interface, but need to catch as more concrete implementations of this interface
  * as possible in {@link Partitioning#satisfy(Distribution)}.
- *
+ * <p>
  * Concrete implementations until now:
  * <ul>
  *   <li>{@link ClusteredDistribution}</li>
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/Partitioning.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/Partitioning.java
index c2c474696f667..7befab4ec5365 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/Partitioning.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/Partitioning.java
@@ -40,7 +40,7 @@ public interface Partitioning {
   /**
    * Returns true if this partitioning can satisfy the given distribution, which means Spark does
    * not need to shuffle the output data of this data source for some certain operations.
-   *
+   * <p>
    * Note that, Spark may add new concrete implementations of {@link Distribution} in new releases.
    * This method should be aware of it and always return false for unrecognized distributions. It's
    * recommended to check every Spark new release and support new distributions if possible, to
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousPartitionReader.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousPartitionReader.java
index c2ad9ec244a0d..a33c64192d82a 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousPartitionReader.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousPartitionReader.java
@@ -30,7 +30,7 @@ public interface ContinuousPartitionReader<T> extends PartitionReader<T> {
 
   /**
    * Get the offset of the current record, or the start offset if no records have been read.
-   *
+   * <p>
    * The execution engine will call this method along with get() to keep track of the current
    * offset. When an epoch ends, the offset of the previous record in each partition will be saved
    * as a restart checkpoint.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousStream.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousStream.java
index a84578fe461a3..98943f642934a 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousStream.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousStream.java
@@ -62,7 +62,7 @@ public interface ContinuousStream extends SparkDataStream {
    * The execution engine will call this method in every epoch to determine if new input
    * partitions need to be generated, which may be required if for example the underlying
    * source system has had partitions added or removed.
-   *
+   * <p>
    * If true, the Spark job to scan this continuous data stream will be interrupted and Spark will
    * launch it again with a new list of {@link InputPartition input partitions}.
    */
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/Offset.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/Offset.java
index efb8ebb684f06..818f378624ef6 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/Offset.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/Offset.java
@@ -22,6 +22,7 @@
 /**
  * An abstract representation of progress through a {@link MicroBatchStream} or
  * {@link ContinuousStream}.
+ * <p>
  * During execution, offsets provided by the data source implementation will be logged and used as
  * restart checkpoints. Each source should provide an offset implementation which the source can use
  * to reconstruct a position in the stream up to which data has been seen/processed.
@@ -33,6 +34,7 @@ public abstract class Offset {
     /**
      * A JSON-serialized representation of an Offset that is
      * used for saving offsets to the offset log.
+     * <p>
      * Note: We assume that equivalent/equal offsets serialize to
      * identical JSON strings.
      *
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/PartitionOffset.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/PartitionOffset.java
index faee230467bea..2d194f52ee939 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/PartitionOffset.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/PartitionOffset.java
@@ -24,7 +24,7 @@
 /**
  * Used for per-partition offsets in continuous processing. ContinuousReader implementations will
  * provide a method to merge these into a global Offset.
- *
+ * <p>
  * These offsets must be serializable.
  *
  * @since 3.0.0
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ReadAllAvailable.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ReadAllAvailable.java
index 5a946ad14b3a2..4aab6b578fc15 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ReadAllAvailable.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ReadAllAvailable.java
@@ -22,7 +22,7 @@
 /**
  * Represents a {@link ReadLimit} where the {@link MicroBatchStream} must scan all the data
  * available at the streaming source. This is meant to be a hard specification as being able
- * to return all available data is necessary for Trigger.Once() to work correctly.
+ * to return all available data is necessary for {@code Trigger.Once()} to work correctly.
  * If a source is unable to scan all available data, then it must throw an error.
  *
  * @see SupportsAdmissionControl#latestOffset(Offset, ReadLimit)
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ReportsSourceMetrics.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ReportsSourceMetrics.java
new file mode 100644
index 0000000000000..705cd1ad04be9
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ReportsSourceMetrics.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.read.streaming;
+
+import java.util.Map;
+import java.util.Optional;
+
+import org.apache.spark.annotation.Evolving;
+
+/**
+ * A mix-in interface for {@link SparkDataStream} streaming sources to signal that they can report
+ * metrics.
+ */
+@Evolving
+public interface ReportsSourceMetrics extends SparkDataStream {
+    /**
+     * Returns the metrics reported by the streaming source with respect to
+     * the latest consumed offset.
+     *
+     * @param latestConsumedOffset the end offset (exclusive) of the latest triggered batch.
+     */
+    Map<String, String> metrics(Optional<Offset> latestConsumedOffset);
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/SparkDataStream.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/SparkDataStream.java
index 95703e255ea4e..c8cf2419ceeb3 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/SparkDataStream.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/SparkDataStream.java
@@ -22,7 +22,7 @@
 /**
  * The base interface representing a readable data stream in a Spark streaming query. It's
  * responsible to manage the offsets of the streaming source in the streaming query.
- *
+ * <p>
  * Data sources should implement concrete data stream interfaces:
  * {@link MicroBatchStream} and {@link ContinuousStream}.
  *
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/SupportsAdmissionControl.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/SupportsAdmissionControl.java
index 027763ce6fcdf..b60a37a22308a 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/SupportsAdmissionControl.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/SupportsAdmissionControl.java
@@ -25,7 +25,7 @@
  * contract of triggers, e.g. Trigger.Once() requires that a micro-batch process all data
  * available to the system at the start of the micro-batch. Alternatively, sources can decide to
  * limit ingest through data source options.
- *
+ * <p>
  * Through this interface, a MicroBatchStream should be able to return the next offset that it will
  * process until given a {@link ReadLimit}.
  *
@@ -44,13 +44,21 @@ public interface SupportsAdmissionControl extends SparkDataStream {
    * Returns the most recent offset available given a read limit. The start offset can be used
    * to figure out how much new data should be read given the limit. Users should implement this
    * method instead of latestOffset for a MicroBatchStream or getOffset for Source.
-   *
+   * <p>
    * When this method is called on a `Source`, the source can return `null` if there is no
    * data to process. In addition, for the very first micro-batch, the `startOffset` will be
    * null as well.
-   *
+   * <p>
    * When this method is called on a MicroBatchStream, the `startOffset` will be `initialOffset`
    * for the very first micro-batch. The source can return `null` if there is no data to process.
    */
   Offset latestOffset(Offset startOffset, ReadLimit limit);
+
+  /**
+   * Returns the most recent offset available.
+   * <p>
+   * The source can return `null`, if there is no data to process or the source does not support
+   * to this method.
+   */
+  default Offset reportLatestOffset() { return null; }
 }
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/BatchWrite.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/BatchWrite.java
index c4b073458eac8..8c068928415f4 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/BatchWrite.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/BatchWrite.java
@@ -21,20 +21,22 @@
 
 /**
  * An interface that defines how to write the data to data source for batch processing.
- *
+ * <p>
  * The writing procedure is:
- *   1. Create a writer factory by {@link #createBatchWriterFactory(PhysicalWriteInfo)}, serialize
- *      and send it to all the partitions of the input data(RDD).
- *   2. For each partition, create the data writer, and write the data of the partition with this
- *      writer. If all the data are written successfully, call {@link DataWriter#commit()}. If
- *      exception happens during the writing, call {@link DataWriter#abort()}.
- *   3. If all writers are successfully committed, call {@link #commit(WriterCommitMessage[])}. If
- *      some writers are aborted, or the job failed with an unknown reason, call
- *      {@link #abort(WriterCommitMessage[])}.
- *
+ * <ol>
+ *   <li>Create a writer factory by {@link #createBatchWriterFactory(PhysicalWriteInfo)}, serialize
+ *     and send it to all the partitions of the input data(RDD).</li>
+ *   <li>For each partition, create the data writer, and write the data of the partition with this
+ *     writer. If all the data are written successfully, call {@link DataWriter#commit()}. If
+ *     exception happens during the writing, call {@link DataWriter#abort()}.</li>
+ *   <li>If all writers are successfully committed, call {@link #commit(WriterCommitMessage[])}. If
+ *     some writers are aborted, or the job failed with an unknown reason, call
+ *     {@link #abort(WriterCommitMessage[])}.</li>
+ * </ol>
+ * <p>
  * While Spark will retry failed writing tasks, Spark won't retry failed writing jobs. Users should
  * do it manually in their Spark applications if they want to retry.
- *
+ * <p>
  * Please refer to the documentation of commit/abort methods for detailed specifications.
  *
  * @since 3.0.0
@@ -44,7 +46,7 @@ public interface BatchWrite {
 
   /**
    * Creates a writer factory which will be serialized and sent to executors.
-   *
+   * <p>
    * If this method fails (by throwing an exception), the action will fail and no Spark job will be
    * submitted.
    *
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/DataWriter.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/DataWriter.java
index 0022a71491738..1c074801481ce 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/DataWriter.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/DataWriter.java
@@ -25,17 +25,17 @@
 /**
  * A data writer returned by {@link DataWriterFactory#createWriter(int, long)} and is
  * responsible for writing data for an input RDD partition.
- *
+ * <p>
  * One Spark task has one exclusive data writer, so there is no thread-safe concern.
- *
+ * <p>
  * {@link #write(Object)} is called for each record in the input RDD partition. If one record fails
  * the {@link #write(Object)}, {@link #abort()} is called afterwards and the remaining records will
  * not be processed. If all records are successfully written, {@link #commit()} is called.
- *
+ * <p>
  * Once a data writer returns successfully from {@link #commit()} or {@link #abort()}, Spark will
  * call {@link #close()} to let DataWriter doing resource cleanup. After calling {@link #close()},
  * its lifecycle is over and Spark will not use it again.
- *
+ * <p>
  * If this data writer succeeds(all records are successfully written and {@link #commit()}
  * succeeds), a {@link WriterCommitMessage} will be sent to the driver side and pass to
  * {@link BatchWrite#commit(WriterCommitMessage[])} with commit messages from other data
@@ -44,7 +44,7 @@
  * In each retry, {@link DataWriterFactory#createWriter(int, long)} will receive a
  * different `taskId`. Spark will call {@link BatchWrite#abort(WriterCommitMessage[])}
  * when the configured number of retries is exhausted.
- *
+ * <p>
  * Besides the retry mechanism, Spark may launch speculative tasks if the existing writing task
  * takes too long to finish. Different from retried tasks, which are launched one by one after the
  * previous one fails, speculative tasks are running simultaneously. It's possible that one input
@@ -54,8 +54,9 @@
  * these data writers can commit successfully. Or implementations can allow all of them to commit
  * successfully, and have a way to revert committed data writers without the commit message, because
  * Spark only accepts the commit message that arrives first and ignore others.
- *
- * Note that, Currently the type `T` can only be {@link org.apache.spark.sql.catalyst.InternalRow}.
+ * <p>
+ * Note that, Currently the type {@code T} can only be
+ * {@link org.apache.spark.sql.catalyst.InternalRow}.
  *
  * @since 3.0.0
  */
@@ -64,7 +65,7 @@ public interface DataWriter<T> extends Closeable {
 
   /**
    * Writes one record.
-   *
+   * <p>
    * If this method fails (by throwing an exception), {@link #abort()} will be called and this
    * data writer is considered to have been failed.
    *
@@ -76,12 +77,12 @@ public interface DataWriter<T> extends Closeable {
    * Commits this writer after all records are written successfully, returns a commit message which
    * will be sent back to driver side and passed to
    * {@link BatchWrite#commit(WriterCommitMessage[])}.
-   *
+   * <p>
    * The written data should only be visible to data source readers after
    * {@link BatchWrite#commit(WriterCommitMessage[])} succeeds, which means this method
    * should still "hide" the written data and ask the {@link BatchWrite} at driver side to
    * do the final commit via {@link WriterCommitMessage}.
-   *
+   * <p>
    * If this method fails (by throwing an exception), {@link #abort()} will be called and this
    * data writer is considered to have been failed.
    *
@@ -92,10 +93,10 @@ public interface DataWriter<T> extends Closeable {
   /**
    * Aborts this writer if it is failed. Implementations should clean up the data for already
    * written records.
-   *
+   * <p>
    * This method will only be called if there is one record failed to write, or {@link #commit()}
    * failed.
-   *
+   * <p>
    * If this method fails(by throwing an exception), the underlying data source may have garbage
    * that need to be cleaned by {@link BatchWrite#abort(WriterCommitMessage[])} or manually,
    * but these garbage should not be visible to data source readers.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/DataWriterFactory.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/DataWriterFactory.java
index 33b7ff3dd27b1..cb14a7060716a 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/DataWriterFactory.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/DataWriterFactory.java
@@ -27,7 +27,7 @@
  * A factory of {@link DataWriter} returned by
  * {@link BatchWrite#createBatchWriterFactory(PhysicalWriteInfo)}, which is responsible for
  * creating and initializing the actual data writer at executor side.
- *
+ * <p>
  * Note that, the writer factory will be serialized and sent to executors, then the data writer
  * will be created on executors and do the actual writing. So this interface must be
  * serializable and {@link DataWriter} doesn't need to be.
@@ -42,7 +42,7 @@ public interface DataWriterFactory extends Serializable {
    * object instance when sending data to the data writer, for better performance. Data writers
    * are responsible for defensive copies if necessary, e.g. copy the data before buffer it in a
    * list.
-   *
+   * <p>
    * If this method fails (by throwing an exception), the corresponding Spark write task would fail
    * and get retried until hitting the maximum retry times.
    *
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/LogicalWriteInfo.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/LogicalWriteInfo.java
index 0638970df925e..e472a130187b9 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/LogicalWriteInfo.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/LogicalWriteInfo.java
@@ -35,7 +35,7 @@ public interface LogicalWriteInfo {
   CaseInsensitiveStringMap options();
 
   /**
-   * `queryId` is a unique string of the query. It's possible that there are many queries
+   * {@code queryId} is a unique string of the query. It's possible that there are many queries
    * running at the same time, or a query is restarted and resumed. {@link BatchWrite} can use
    * this id to identify the query.
    */
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/RequiresDistributionAndOrdering.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/RequiresDistributionAndOrdering.java
new file mode 100644
index 0000000000000..2284086f99f6e
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/RequiresDistributionAndOrdering.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.write;
+
+import org.apache.spark.annotation.Experimental;
+import org.apache.spark.sql.connector.distributions.Distribution;
+import org.apache.spark.sql.connector.distributions.UnspecifiedDistribution;
+import org.apache.spark.sql.connector.expressions.SortOrder;
+
+/**
+ * A write that requires a specific distribution and ordering of data.
+ *
+ * @since 3.2.0
+ */
+@Experimental
+public interface RequiresDistributionAndOrdering extends Write {
+  /**
+   * Returns the distribution required by this write.
+   * <p>
+   * Spark will distribute incoming records across partitions to satisfy the required distribution
+   * before passing the records to the data source table on write.
+   * <p>
+   * Implementations may return {@link UnspecifiedDistribution} if they don't require any specific
+   * distribution of data on write.
+   *
+   * @return the required distribution
+   */
+  Distribution requiredDistribution();
+
+  /**
+   * Returns the number of partitions required by this write.
+   * <p>
+   * Implementations may override this to require a specific number of input partitions.
+   * <p>
+   * Note that Spark doesn't support the number of partitions on {@link UnspecifiedDistribution},
+   * the query will fail if the number of partitions are provided but the distribution is
+   * unspecified.
+   *
+   * @return the required number of partitions, any value less than 1 mean no requirement.
+   */
+  default int requiredNumPartitions() { return 0; }
+
+  /**
+   * Returns the ordering required by this write.
+   * <p>
+   * Spark will order incoming records within partitions to satisfy the required ordering
+   * before passing those records to the data source table on write.
+   * <p>
+   * Implementations may return an empty array if they don't require any specific ordering of data
+   * on write.
+   *
+   * @return the required ordering
+   */
+  SortOrder[] requiredOrdering();
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/Write.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/Write.java
new file mode 100644
index 0000000000000..873680415d447
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/Write.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.write;
+
+import org.apache.spark.annotation.Evolving;
+import org.apache.spark.sql.connector.catalog.Table;
+import org.apache.spark.sql.connector.catalog.TableCapability;
+import org.apache.spark.sql.connector.write.streaming.StreamingWrite;
+
+/**
+ * A logical representation of a data source write.
+ * <p>
+ * This logical representation is shared between batch and streaming write. Data sources must
+ * implement the corresponding methods in this interface to match what the table promises
+ * to support. For example, {@link #toBatch()} must be implemented if the {@link Table} that
+ * creates this {@link Write} returns {@link TableCapability#BATCH_WRITE} support in its
+ * {@link Table#capabilities()}.
+ *
+ * @since 3.2.0
+ */
+@Evolving
+public interface Write {
+
+  /**
+   * Returns the description associated with this write.
+   */
+  default String description() {
+    return this.getClass().toString();
+  }
+
+  /**
+   * Returns a {@link BatchWrite} to write data to batch source. By default this method throws
+   * exception, data sources must overwrite this method to provide an implementation, if the
+   * {@link Table} that creates this write returns {@link TableCapability#BATCH_WRITE} support in
+   * its {@link Table#capabilities()}.
+   */
+  default BatchWrite toBatch() {
+    throw new UnsupportedOperationException(description() + ": Batch write is not supported");
+  }
+
+  /**
+   * Returns a {@link StreamingWrite} to write data to streaming source. By default this method
+   * throws exception, data sources must overwrite this method to provide an implementation, if the
+   * {@link Table} that creates this write returns {@link TableCapability#STREAMING_WRITE} support
+   * in its {@link Table#capabilities()}.
+   */
+  default StreamingWrite toStreaming() {
+    throw new UnsupportedOperationException(description() + ": Streaming write is not supported");
+  }
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/WriteBuilder.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/WriteBuilder.java
index 5398ca46e9777..51ca167a4c0df 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/WriteBuilder.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/WriteBuilder.java
@@ -18,15 +18,13 @@
 package org.apache.spark.sql.connector.write;
 
 import org.apache.spark.annotation.Evolving;
-import org.apache.spark.sql.connector.catalog.Table;
-import org.apache.spark.sql.connector.catalog.TableCapability;
 import org.apache.spark.sql.connector.write.streaming.StreamingWrite;
 
 /**
- * An interface for building the {@link BatchWrite}. Implementations can mix in some interfaces to
+ * An interface for building the {@link Write}. Implementations can mix in some interfaces to
  * support different ways to write data to data sources.
- *
- * Unless modified by a mixin interface, the {@link BatchWrite} configured by this builder is to
+ * <p>
+ * Unless modified by a mixin interface, the {@link Write} configured by this builder is to
  * append data without affecting existing data.
  *
  * @since 3.0.0
@@ -35,22 +33,41 @@
 public interface WriteBuilder {
 
   /**
-   * Returns a {@link BatchWrite} to write data to batch source. By default this method throws
-   * exception, data sources must overwrite this method to provide an implementation, if the
-   * {@link Table} that creates this write returns {@link TableCapability#BATCH_WRITE} support in
-   * its {@link Table#capabilities()}.
+   * Returns a logical {@link Write} shared between batch and streaming.
+   *
+   * @since 3.2.0
+   */
+  default Write build() {
+    return new Write() {
+      @Override
+      public BatchWrite toBatch() {
+        return buildForBatch();
+      }
+
+      @Override
+      public StreamingWrite toStreaming() {
+        return buildForStreaming();
+      }
+    };
+  }
+
+  /**
+   * Returns a {@link BatchWrite} to write data to batch source.
+   *
+   * @deprecated use {@link #build()} instead.
    */
+  @Deprecated
   default BatchWrite buildForBatch() {
     throw new UnsupportedOperationException(getClass().getName() +
       " does not support batch write");
   }
 
   /**
-   * Returns a {@link StreamingWrite} to write data to streaming source. By default this method
-   * throws exception, data sources must overwrite this method to provide an implementation, if the
-   * {@link Table} that creates this write returns {@link TableCapability#STREAMING_WRITE} support
-   * in its {@link Table#capabilities()}.
+   * Returns a {@link StreamingWrite} to write data to streaming source.
+   *
+   * @deprecated use {@link #build()} instead.
    */
+  @Deprecated
   default StreamingWrite buildForStreaming() {
     throw new UnsupportedOperationException(getClass().getName() +
       " does not support streaming write");
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/WriterCommitMessage.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/WriterCommitMessage.java
index f340d20c3549f..b76d531f78975 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/WriterCommitMessage.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/WriterCommitMessage.java
@@ -26,7 +26,7 @@
  * A commit message returned by {@link DataWriter#commit()} and will be sent back to the driver side
  * as the input parameter of {@link BatchWrite#commit(WriterCommitMessage[])} or
  * {@link StreamingWrite#commit(long, WriterCommitMessage[])}.
- *
+ * <p>
  * This is an empty interface, data sources should define their own message class and use it when
  * generating messages at executor side and handling the messages at driver side.
  *
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/streaming/StreamingDataWriterFactory.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/streaming/StreamingDataWriterFactory.java
index 0923d07e7e5a3..b430e1e59792a 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/streaming/StreamingDataWriterFactory.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/streaming/StreamingDataWriterFactory.java
@@ -29,7 +29,7 @@
  * A factory of {@link DataWriter} returned by
  * {@link StreamingWrite#createStreamingWriterFactory(PhysicalWriteInfo)}, which is responsible for
  * creating and initializing the actual data writer at executor side.
- *
+ * <p>
  * Note that, the writer factory will be serialized and sent to executors, then the data writer
  * will be created on executors and do the actual writing. So this interface must be
  * serializable and {@link DataWriter} doesn't need to be.
@@ -44,7 +44,7 @@ public interface StreamingDataWriterFactory extends Serializable {
    * object instance when sending data to the data writer, for better performance. Data writers
    * are responsible for defensive copies if necessary, e.g. copy the data before buffer it in a
    * list.
-   *
+   * <p>
    * If this method fails (by throwing an exception), the corresponding Spark write task would fail
    * and get retried until hitting the maximum retry times.
    *
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/streaming/StreamingWrite.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/streaming/StreamingWrite.java
index e3dec3b2ff55e..20694f0b051d3 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/streaming/StreamingWrite.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/streaming/StreamingWrite.java
@@ -26,19 +26,21 @@
  * An interface that defines how to write the data to data source in streaming queries.
  *
  * The writing procedure is:
- *   1. Create a writer factory by {@link #createStreamingWriterFactory(PhysicalWriteInfo)},
- *      serialize and send it to all the partitions of the input data(RDD).
- *   2. For each epoch in each partition, create the data writer, and write the data of the epoch in
- *      the partition with this writer. If all the data are written successfully, call
- *      {@link DataWriter#commit()}. If exception happens during the writing, call
- *      {@link DataWriter#abort()}.
- *   3. If writers in all partitions of one epoch are successfully committed, call
- *      {@link #commit(long, WriterCommitMessage[])}. If some writers are aborted, or the job failed
- *      with an unknown reason, call {@link #abort(long, WriterCommitMessage[])}.
- *
+ * <ol>
+ *   <li>Create a writer factory by {@link #createStreamingWriterFactory(PhysicalWriteInfo)},
+ *   serialize and send it to all the partitions of the input data(RDD).</li>
+ *   <li>For each epoch in each partition, create the data writer, and write the data of the
+ *   epoch in the partition with this writer. If all the data are written successfully, call
+ *   {@link DataWriter#commit()}. If exception happens during the writing, call
+ *   {@link DataWriter#abort()}.</li>
+ *   <li>If writers in all partitions of one epoch are successfully committed, call
+ *   {@link #commit(long, WriterCommitMessage[])}. If some writers are aborted, or the job failed
+ *   with an unknown reason, call {@link #abort(long, WriterCommitMessage[])}.</li>
+ * </ol>
+ * <p>
  * While Spark will retry failed writing tasks, Spark won't retry failed writing jobs. Users should
  * do it manually in their Spark applications if they want to retry.
- *
+ * <p>
  * Please refer to the documentation of commit/abort methods for detailed specifications.
  *
  * @since 3.0.0
@@ -48,7 +50,7 @@ public interface StreamingWrite {
 
   /**
    * Creates a writer factory which will be serialized and sent to executors.
-   *
+   * <p>
    * If this method fails (by throwing an exception), the action will fail and no Spark job will be
    * submitted.
    *
@@ -60,14 +62,14 @@ public interface StreamingWrite {
    * Commits this writing job for the specified epoch with a list of commit messages. The commit
    * messages are collected from successful data writers and are produced by
    * {@link DataWriter#commit()}.
-   *
+   * <p>
    * If this method fails (by throwing an exception), this writing job is considered to have been
    * failed, and the execution engine will attempt to call
    * {@link #abort(long, WriterCommitMessage[])}.
-   *
-   * The execution engine may call `commit` multiple times for the same epoch in some circumstances.
-   * To support exactly-once data semantics, implementations must ensure that multiple commits for
-   * the same epoch are idempotent.
+   * <p>
+   * The execution engine may call {@code commit} multiple times for the same epoch in some
+   * circumstances. To support exactly-once data semantics, implementations must ensure that
+   * multiple commits for the same epoch are idempotent.
    */
   void commit(long epochId, WriterCommitMessage[] messages);
 
@@ -75,10 +77,10 @@ public interface StreamingWrite {
    * Aborts this writing job because some data writers are failed and keep failing when retried, or
    * the Spark job fails with some unknown reasons, or {@link #commit(long, WriterCommitMessage[])}
    * fails.
-   *
+   * <p>
    * If this method fails (by throwing an exception), the underlying data source may require manual
    * cleanup.
-   *
+   * <p>
    * Unless the abort is triggered by the failure of commit, the given messages will have some
    * null slots, as there may be only a few data writers that were committed before the abort
    * happens, or some data writers were committed but their commit messages haven't reached the
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/streaming/GroupStateTimeout.java b/sql/catalyst/src/main/java/org/apache/spark/sql/streaming/GroupStateTimeout.java
index 5f6a46f2b8e89..a814525f870c9 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/streaming/GroupStateTimeout.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/streaming/GroupStateTimeout.java
@@ -23,8 +23,9 @@
 
 /**
  * Represents the type of timeouts possible for the Dataset operations
- * `mapGroupsWithState` and `flatMapGroupsWithState`. See documentation on
- * `GroupState` for more details.
+ * {@code mapGroupsWithState} and {@code flatMapGroupsWithState}.
+ * <p>
+ * See documentation on {@code GroupState} for more details.
  *
  * @since 2.2.0
  */
@@ -33,21 +34,29 @@
 public class GroupStateTimeout {
 
   /**
-   * Timeout based on processing time. The duration of timeout can be set for each group in
-   * `map/flatMapGroupsWithState` by calling `GroupState.setTimeoutDuration()`. See documentation
-   * on `GroupState` for more details.
+   * Timeout based on processing time.
+   * <p>
+   * The duration of timeout can be set for each group in
+   * {@code map/flatMapGroupsWithState} by calling {@code GroupState.setTimeoutDuration()}.
+   * <p>
+   * See documentation on {@code GroupState} for more details.
    */
   public static GroupStateTimeout ProcessingTimeTimeout() {
     return ProcessingTimeTimeout$.MODULE$;
   }
 
   /**
-   * Timeout based on event-time. The event-time timestamp for timeout can be set for each
-   * group in `map/flatMapGroupsWithState` by calling `GroupState.setTimeoutTimestamp()`.
-   * In addition, you have to define the watermark in the query using `Dataset.withWatermark`.
+   * Timeout based on event-time.
+   * <p>
+   * The event-time timestamp for timeout can be set for each
+   * group in {@code map/flatMapGroupsWithState} by calling
+   * {@code GroupState.setTimeoutTimestamp()}.
+   * In addition, you have to define the watermark in the query using
+   * {@code Dataset.withWatermark}.
    * When the watermark advances beyond the set timestamp of a group and the group has not
-   * received any data, then the group times out. See documentation on
-   * `GroupState` for more details.
+   * received any data, then the group times out.
+   * <p>
+   * See documentation on {@code GroupState} for more details.
    */
   public static GroupStateTimeout EventTimeTimeout() { return EventTimeTimeout$.MODULE$; }
 
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/types/DataTypes.java b/sql/catalyst/src/main/java/org/apache/spark/sql/types/DataTypes.java
index d786374f69e20..eac0d91acdba7 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/types/DataTypes.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/types/DataTypes.java
@@ -94,6 +94,16 @@ public class DataTypes {
    */
   public static final DataType NullType = NullType$.MODULE$;
 
+  /**
+   * Gets the DayTimeIntervalType object.
+   */
+  public static final DataType DayTimeIntervalType = DayTimeIntervalType$.MODULE$;
+
+  /**
+   * Gets the YearMonthIntervalType object.
+   */
+  public static final DataType YearMonthIntervalType = YearMonthIntervalType$.MODULE$;
+
   /**
    * Creates an ArrayType by specifying the data type of elements ({@code elementType}).
    * The field of {@code containsNull} is set to {@code true}.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ArrowColumnVector.java b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ArrowColumnVector.java
index d2220dc2668f9..fe48670cb35fa 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ArrowColumnVector.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ArrowColumnVector.java
@@ -17,15 +17,18 @@
 
 package org.apache.spark.sql.vectorized;
 
-import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.vector.*;
 import org.apache.arrow.vector.complex.*;
+import org.apache.arrow.vector.holders.NullableIntervalDayHolder;
 import org.apache.arrow.vector.holders.NullableVarCharHolder;
 
 import org.apache.spark.sql.util.ArrowUtils;
 import org.apache.spark.sql.types.*;
 import org.apache.spark.unsafe.types.UTF8String;
 
+import static org.apache.spark.sql.catalyst.util.DateTimeConstants.MICROS_PER_DAY;
+import static org.apache.spark.sql.catalyst.util.DateTimeConstants.MICROS_PER_MILLIS;
+
 /**
  * A column vector backed by Apache Arrow. Currently calendar interval type and map type are not
  * supported.
@@ -171,6 +174,12 @@ public ArrowColumnVector(ValueVector vector) {
       for (int i = 0; i < childColumns.length; ++i) {
         childColumns[i] = new ArrowColumnVector(structVector.getVectorById(i));
       }
+    } else if (vector instanceof NullVector) {
+      accessor = new NullAccessor((NullVector) vector);
+    } else if (vector instanceof IntervalYearVector) {
+      accessor = new IntervalYearAccessor((IntervalYearVector) vector);
+    } else if (vector instanceof IntervalDayVector) {
+      accessor = new IntervalDayAccessor((IntervalDayVector) vector);
     } else {
       throw new UnsupportedOperationException();
     }
@@ -458,10 +467,8 @@ final boolean isNullAt(int rowId) {
 
     @Override
     final ColumnarArray getArray(int rowId) {
-      ArrowBuf offsets = accessor.getOffsetBuffer();
-      int index = rowId * ListVector.OFFSET_WIDTH;
-      int start = offsets.getInt(index);
-      int end = offsets.getInt(index + ListVector.OFFSET_WIDTH);
+      int start = accessor.getElementStartIndex(rowId);
+      int end = accessor.getElementEndIndex(rowId);
       return new ColumnarArray(arrayData, start, end - start);
     }
   }
@@ -502,4 +509,44 @@ final ColumnarMap getMap(int rowId) {
       return new ColumnarMap(keys, values, offset, length);
     }
   }
+
+  private static class NullAccessor extends ArrowVectorAccessor {
+
+    NullAccessor(NullVector vector) {
+      super(vector);
+    }
+  }
+
+  private static class IntervalYearAccessor extends ArrowVectorAccessor {
+
+    private final IntervalYearVector accessor;
+
+    IntervalYearAccessor(IntervalYearVector vector) {
+      super(vector);
+      this.accessor = vector;
+    }
+
+    @Override
+    int getInt(int rowId) {
+      return accessor.get(rowId);
+    }
+  }
+
+  private static class IntervalDayAccessor extends ArrowVectorAccessor {
+
+    private final IntervalDayVector accessor;
+    private final NullableIntervalDayHolder intervalDayHolder = new NullableIntervalDayHolder();
+
+    IntervalDayAccessor(IntervalDayVector vector) {
+      super(vector);
+      this.accessor = vector;
+    }
+
+    @Override
+    long getLong(int rowId) {
+      accessor.get(rowId, intervalDayHolder);
+      return Math.addExact(Math.multiplyExact(intervalDayHolder.days, MICROS_PER_DAY),
+                           intervalDayHolder.milliseconds * MICROS_PER_MILLIS);
+    }
+  }
 }
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnVector.java b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnVector.java
index 2158ef8324845..5ef4fba193e0c 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnVector.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnVector.java
@@ -26,22 +26,23 @@
  * An interface representing in-memory columnar data in Spark. This interface defines the main APIs
  * to access the data, as well as their batched versions. The batched versions are considered to be
  * faster and preferable whenever possible.
- *
+ * <p>
  * Most of the APIs take the rowId as a parameter. This is the batch local 0-based row id for values
  * in this ColumnVector.
- *
- * Spark only calls specific `get` method according to the data type of this {@link ColumnVector},
+ * <p>
+ * Spark only calls specific {@code get} method according to the data type of this
+ * {@link ColumnVector},
  * e.g. if it's int type, Spark is guaranteed to only call {@link #getInt(int)} or
  * {@link #getInts(int, int)}.
- *
+ * <p>
  * ColumnVector supports all the data types including nested types. To handle nested types,
  * ColumnVector can have children and is a tree structure. Please refer to {@link #getStruct(int)},
  * {@link #getArray(int)} and {@link #getMap(int)} for the details about how to implement nested
  * types.
- *
+ * <p>
  * ColumnVector is expected to be reused during the entire data loading process, to avoid allocating
  * memory again and again.
- *
+ * <p>
  * ColumnVector is meant to maximize CPU efficiency but not to minimize storage footprint.
  * Implementations should prefer computing efficiency over storage efficiency when design the
  * format. Since it is expected to reuse the ColumnVector instance while loading data, the storage
@@ -57,9 +58,10 @@ public abstract class ColumnVector implements AutoCloseable {
 
   /**
    * Cleans up memory for this column vector. The column vector is not usable after this.
-   *
-   * This overwrites `AutoCloseable.close` to remove the `throws` clause, as column vector is
-   * in-memory and we don't expect any exception to happen during closing.
+   * <p>
+   * This overwrites {@link AutoCloseable#close} to remove the
+   * {@code throws} clause, as column vector is in-memory and we don't expect any exception to
+   * happen during closing.
    */
   @Override
   public abstract void close();
@@ -75,19 +77,19 @@ public abstract class ColumnVector implements AutoCloseable {
   public abstract int numNulls();
 
   /**
-   * Returns whether the value at rowId is NULL.
+   * Returns whether the value at {@code rowId} is NULL.
    */
   public abstract boolean isNullAt(int rowId);
 
   /**
-   * Returns the boolean type value for rowId. The return value is undefined and can be anything,
-   * if the slot for rowId is null.
+   * Returns the boolean type value for {@code rowId}. The return value is undefined and can be
+   * anything, if the slot for {@code rowId} is null.
    */
   public abstract boolean getBoolean(int rowId);
 
   /**
-   * Gets boolean type values from [rowId, rowId + count). The return values for the null slots
-   * are undefined and can be anything.
+   * Gets boolean type values from {@code [rowId, rowId + count)}. The return values for the null
+   * slots are undefined and can be anything.
    */
   public boolean[] getBooleans(int rowId, int count) {
     boolean[] res = new boolean[count];
@@ -98,13 +100,13 @@ public boolean[] getBooleans(int rowId, int count) {
   }
 
   /**
-   * Returns the byte type value for rowId. The return value is undefined and can be anything,
-   * if the slot for rowId is null.
+   * Returns the byte type value for {@code rowId}. The return value is undefined and can be
+   * anything, if the slot for {@code rowId} is null.
    */
   public abstract byte getByte(int rowId);
 
   /**
-   * Gets byte type values from [rowId, rowId + count). The return values for the null slots
+   * Gets byte type values from {@code [rowId, rowId + count)}. The return values for the null slots
    * are undefined and can be anything.
    */
   public byte[] getBytes(int rowId, int count) {
@@ -116,14 +118,14 @@ public byte[] getBytes(int rowId, int count) {
   }
 
   /**
-   * Returns the short type value for rowId. The return value is undefined and can be anything,
-   * if the slot for rowId is null.
+   * Returns the short type value for {@code rowId}. The return value is undefined and can be
+   * anything, if the slot for {@code rowId} is null.
    */
   public abstract short getShort(int rowId);
 
   /**
-   * Gets short type values from [rowId, rowId + count). The return values for the null slots
-   * are undefined and can be anything.
+   * Gets short type values from {@code [rowId, rowId + count)}. The return values for the null
+   * slots are undefined and can be anything.
    */
   public short[] getShorts(int rowId, int count) {
     short[] res = new short[count];
@@ -134,13 +136,13 @@ public short[] getShorts(int rowId, int count) {
   }
 
   /**
-   * Returns the int type value for rowId. The return value is undefined and can be anything,
-   * if the slot for rowId is null.
+   * Returns the int type value for {@code rowId}. The return value is undefined and can be
+   * anything, if the slot for {@code rowId} is null.
    */
   public abstract int getInt(int rowId);
 
   /**
-   * Gets int type values from [rowId, rowId + count). The return values for the null slots
+   * Gets int type values from {@code [rowId, rowId + count)}. The return values for the null slots
    * are undefined and can be anything.
    */
   public int[] getInts(int rowId, int count) {
@@ -152,13 +154,13 @@ public int[] getInts(int rowId, int count) {
   }
 
   /**
-   * Returns the long type value for rowId. The return value is undefined and can be anything,
-   * if the slot for rowId is null.
+   * Returns the long type value for {@code rowId}. The return value is undefined and can be
+   * anything, if the slot for {@code rowId} is null.
    */
   public abstract long getLong(int rowId);
 
   /**
-   * Gets long type values from [rowId, rowId + count). The return values for the null slots
+   * Gets long type values from {@code [rowId, rowId + count)}. The return values for the null slots
    * are undefined and can be anything.
    */
   public long[] getLongs(int rowId, int count) {
@@ -170,14 +172,14 @@ public long[] getLongs(int rowId, int count) {
   }
 
   /**
-   * Returns the float type value for rowId. The return value is undefined and can be anything,
-   * if the slot for rowId is null.
+   * Returns the float type value for {@code rowId}. The return value is undefined and can be
+   * anything, if the slot for {@code rowId} is null.
    */
   public abstract float getFloat(int rowId);
 
   /**
-   * Gets float type values from [rowId, rowId + count). The return values for the null slots
-   * are undefined and can be anything.
+   * Gets float type values from {@code [rowId, rowId + count)}. The return values for the null
+   * slots are undefined and can be anything.
    */
   public float[] getFloats(int rowId, int count) {
     float[] res = new float[count];
@@ -188,14 +190,14 @@ public float[] getFloats(int rowId, int count) {
   }
 
   /**
-   * Returns the double type value for rowId. The return value is undefined and can be anything,
-   * if the slot for rowId is null.
+   * Returns the double type value for {@code rowId}. The return value is undefined and can be
+   * anything, if the slot for {@code rowId} is null.
    */
   public abstract double getDouble(int rowId);
 
   /**
-   * Gets double type values from [rowId, rowId + count). The return values for the null slots
-   * are undefined and can be anything.
+   * Gets double type values from {@code [rowId, rowId + count)}. The return values for the null
+   * slots are undefined and can be anything.
    */
   public double[] getDoubles(int rowId, int count) {
     double[] res = new double[count];
@@ -206,8 +208,9 @@ public double[] getDoubles(int rowId, int count) {
   }
 
   /**
-   * Returns the struct type value for rowId. If the slot for rowId is null, it should return null.
-   *
+   * Returns the struct type value for {@code rowId}. If the slot for {@code rowId} is null, it
+   * should return null.
+   * <p>
    * To support struct type, implementations must implement {@link #getChild(int)} and make this
    * vector a tree structure. The number of child vectors must be same as the number of fields of
    * the struct type, and each child vector is responsible to store the data for its corresponding
@@ -219,8 +222,9 @@ public final ColumnarRow getStruct(int rowId) {
   }
 
   /**
-   * Returns the array type value for rowId. If the slot for rowId is null, it should return null.
-   *
+   * Returns the array type value for {@code rowId}. If the slot for {@code rowId} is null, it
+   * should return null.
+   * <p>
    * To support array type, implementations must construct an {@link ColumnarArray} and return it in
    * this method. {@link ColumnarArray} requires a {@link ColumnVector} that stores the data of all
    * the elements of all the arrays in this vector, and an offset and length which points to a range
@@ -232,12 +236,13 @@ public final ColumnarRow getStruct(int rowId) {
   public abstract ColumnarArray getArray(int rowId);
 
   /**
-   * Returns the map type value for rowId. If the slot for rowId is null, it should return null.
-   *
+   * Returns the map type value for {@code rowId}. If the slot for {@code rowId} is null, it
+   * should return null.
+   * <p>
    * In Spark, map type value is basically a key data array and a value data array. A key from the
    * key array with a index and a value from the value array with the same index contribute to
    * an entry of this map type value.
-   *
+   * <p>
    * To support map type, implementations must construct a {@link ColumnarMap} and return it in
    * this method. {@link ColumnarMap} requires a {@link ColumnVector} that stores the data of all
    * the keys of all the maps in this vector, and another {@link ColumnVector} that stores the data
@@ -247,31 +252,35 @@ public final ColumnarRow getStruct(int rowId) {
   public abstract ColumnarMap getMap(int ordinal);
 
   /**
-   * Returns the decimal type value for rowId. If the slot for rowId is null, it should return null.
+   * Returns the decimal type value for {@code rowId}. If the slot for {@code rowId} is null, it
+   * should return null.
    */
   public abstract Decimal getDecimal(int rowId, int precision, int scale);
 
   /**
-   * Returns the string type value for rowId. If the slot for rowId is null, it should return null.
-   * Note that the returned UTF8String may point to the data of this column vector, please copy it
-   * if you want to keep it after this column vector is freed.
+   * Returns the string type value for {@code rowId}. If the slot for {@code rowId} is null, it
+   * should return null.
+   * <p>
+   * Note that the returned {@link UTF8String} may point to the data of this column vector,
+   * please copy it if you want to keep it after this column vector is freed.
    */
   public abstract UTF8String getUTF8String(int rowId);
 
   /**
-   * Returns the binary type value for rowId. If the slot for rowId is null, it should return null.
+   * Returns the binary type value for {@code rowId}. If the slot for {@code rowId} is null, it
+   * should return null.
    */
   public abstract byte[] getBinary(int rowId);
 
   /**
-   * Returns the calendar interval type value for rowId. If the slot for rowId is null, it should
-   * return null.
-   *
+   * Returns the calendar interval type value for {@code rowId}. If the slot for
+   * {@code rowId} is null, it should return null.
+   * <p>
    * In Spark, calendar interval type value is basically two integer values representing the number
    * of months and days in this interval, and a long value representing the number of microseconds
    * in this interval. An interval type vector is the same as a struct type vector with 3 fields:
-   * `months`, `days` and `microseconds`.
-   *
+   * {@code months}, {@code days} and {@code microseconds}.
+   * <p>
    * To support interval type, implementations must implement {@link #getChild(int)} and define 3
    * child vectors: the first child vector is an int type vector, containing all the month values of
    * all the interval values in this vector. The second child vector is an int type vector,
@@ -288,7 +297,7 @@ public final CalendarInterval getInterval(int rowId) {
   }
 
   /**
-   * @return child [[ColumnVector]] at the given ordinal.
+   * @return child {@link ColumnVector} at the given ordinal.
    */
   public abstract ColumnVector getChild(int ordinal);
 
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarMap.java b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarMap.java
index 35648e386c4f1..6b3d518746dc3 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarMap.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarMap.java
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.vectorized;
 
+import org.apache.spark.sql.catalyst.util.ArrayBasedMapData;
 import org.apache.spark.sql.catalyst.util.MapData;
 
 /**
@@ -47,7 +48,7 @@ public ColumnarArray valueArray() {
   }
 
   @Override
-  public ColumnarMap copy() {
-    throw new UnsupportedOperationException();
+  public MapData copy() {
+    return new ArrayBasedMapData(keys.copy(), values.copy());
   }
 }
diff --git a/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/expressions/AttributeMap.scala b/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/expressions/AttributeMap.scala
index 75a8bec018a1f..42b92d4593c77 100644
--- a/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/expressions/AttributeMap.scala
+++ b/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/expressions/AttributeMap.scala
@@ -26,6 +26,8 @@ object AttributeMap {
   def apply[A](kvs: Seq[(Attribute, A)]): AttributeMap[A] = {
     new AttributeMap(kvs.map(kv => (kv._1.exprId, kv)).toMap)
   }
+
+  def empty[A]: AttributeMap[A] = new AttributeMap(Map.empty)
 }
 
 class AttributeMap[A](val baseMap: Map[ExprId, (Attribute, A)])
diff --git a/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/expressions/ExpressionSet.scala b/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/expressions/ExpressionSet.scala
deleted file mode 100644
index 5198c6dd2fdf0..0000000000000
--- a/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/expressions/ExpressionSet.scala
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.catalyst.expressions
-
-import scala.collection.{mutable, GenTraversableOnce}
-import scala.collection.mutable.ArrayBuffer
-
-object ExpressionSet {
-  /** Constructs a new [[ExpressionSet]] by applying [[Canonicalize]] to `expressions`. */
-  def apply(expressions: TraversableOnce[Expression]): ExpressionSet = {
-    val set = new ExpressionSet()
-    expressions.foreach(set.add)
-    set
-  }
-}
-
-/**
- * A [[Set]] where membership is determined based on determinacy and a canonical representation of
- * an [[Expression]] (i.e. one that attempts to ignore cosmetic differences).
- * See [[Canonicalize]] for more details.
- *
- * Internally this set uses the canonical representation, but keeps also track of the original
- * expressions to ease debugging.  Since different expressions can share the same canonical
- * representation, this means that operations that extract expressions from this set are only
- * guaranteed to see at least one such expression.  For example:
- *
- * {{{
- *   val set = ExpressionSet(a + 1, 1 + a)
- *
- *   set.iterator => Iterator(a + 1)
- *   set.contains(a + 1) => true
- *   set.contains(1 + a) => true
- *   set.contains(a + 2) => false
- * }}}
- *
- * For non-deterministic expressions, they are always considered as not contained in the [[Set]].
- * On adding a non-deterministic expression, simply append it to the original expressions.
- * This is consistent with how we define `semanticEquals` between two expressions.
- */
-class ExpressionSet protected(
-    protected val baseSet: mutable.Set[Expression] = new mutable.HashSet,
-    protected val originals: mutable.Buffer[Expression] = new ArrayBuffer)
-  extends Set[Expression] {
-
-  //  Note: this class supports Scala 2.12. A parallel source tree has a 2.13 implementation.
-
-  protected def add(e: Expression): Unit = {
-    if (!e.deterministic) {
-      originals += e
-    } else if (!baseSet.contains(e.canonicalized) ) {
-      baseSet.add(e.canonicalized)
-      originals += e
-    }
-  }
-
-  override def contains(elem: Expression): Boolean = baseSet.contains(elem.canonicalized)
-
-  override def +(elem: Expression): ExpressionSet = {
-    val newSet = new ExpressionSet(baseSet.clone(), originals.clone())
-    newSet.add(elem)
-    newSet
-  }
-
-  override def ++(elems: GenTraversableOnce[Expression]): ExpressionSet = {
-    val newSet = new ExpressionSet(baseSet.clone(), originals.clone())
-    elems.foreach(newSet.add)
-    newSet
-  }
-
-  override def -(elem: Expression): ExpressionSet = {
-    if (elem.deterministic) {
-      val newBaseSet = baseSet.clone().filterNot(_ == elem.canonicalized)
-      val newOriginals = originals.clone().filterNot(_.canonicalized == elem.canonicalized)
-      new ExpressionSet(newBaseSet, newOriginals)
-    } else {
-      new ExpressionSet(baseSet.clone(), originals.clone())
-    }
-  }
-
-  override def iterator: Iterator[Expression] = originals.iterator
-
-  /**
-   * Returns a string containing both the post [[Canonicalize]] expressions and the original
-   * expressions in this set.
-   */
-  def toDebugString: String =
-    s"""
-       |baseSet: ${baseSet.mkString(", ")}
-       |originals: ${originals.mkString(", ")}
-     """.stripMargin
-}
diff --git a/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala b/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
index 699b42bd6d60d..14b8f620017f6 100644
--- a/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
+++ b/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
@@ -39,8 +39,12 @@ class CaseInsensitiveMap[T] private (val originalMap: Map[String, T]) extends Ma
   override def contains(k: String): Boolean =
     keyLowerCasedMap.contains(k.toLowerCase(Locale.ROOT))
 
-  override def +[B1 >: T](kv: (String, B1)): Map[String, B1] = {
-    new CaseInsensitiveMap(originalMap + kv)
+  override def +[B1 >: T](kv: (String, B1)): CaseInsensitiveMap[B1] = {
+    new CaseInsensitiveMap(originalMap.filter(!_._1.equalsIgnoreCase(kv._1)) + kv)
+  }
+
+  def ++(xs: TraversableOnce[(String, T)]): CaseInsensitiveMap[T] = {
+    xs.foldLeft(this)(_ + _)
   }
 
   override def iterator: Iterator[(String, T)] = keyLowerCasedMap.iterator
@@ -48,6 +52,8 @@ class CaseInsensitiveMap[T] private (val originalMap: Map[String, T]) extends Ma
   override def -(key: String): Map[String, T] = {
     new CaseInsensitiveMap(originalMap.filter(!_._1.equalsIgnoreCase(key)))
   }
+
+  def toMap: Map[String, T] = originalMap
 }
 
 object CaseInsensitiveMap {
diff --git a/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/expressions/AttributeMap.scala b/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/expressions/AttributeMap.scala
index 4caa3d0461875..e6b53e3e6548f 100644
--- a/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/expressions/AttributeMap.scala
+++ b/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/expressions/AttributeMap.scala
@@ -26,6 +26,8 @@ object AttributeMap {
   def apply[A](kvs: Seq[(Attribute, A)]): AttributeMap[A] = {
     new AttributeMap(kvs.map(kv => (kv._1.exprId, kv)).toMap)
   }
+
+  def empty[A]: AttributeMap[A] = new AttributeMap(Map.empty)
 }
 
 class AttributeMap[A](val baseMap: Map[ExprId, (Attribute, A)])
diff --git a/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/expressions/ExpressionSet.scala b/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/expressions/ExpressionSet.scala
deleted file mode 100644
index cf74b98b31b0c..0000000000000
--- a/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/expressions/ExpressionSet.scala
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.catalyst.expressions
-
-import scala.collection.mutable
-import scala.collection.mutable.ArrayBuffer
-
-object ExpressionSet {
-  /** Constructs a new [[ExpressionSet]] by applying [[Canonicalize]] to `expressions`. */
-  def apply(expressions: TraversableOnce[Expression]): ExpressionSet = {
-    val set = new ExpressionSet()
-    expressions.foreach(set.add)
-    set
-  }
-}
-
-/**
- * A [[Set]] where membership is determined based on determinacy and a canonical representation of
- * an [[Expression]] (i.e. one that attempts to ignore cosmetic differences).
- * See [[Canonicalize]] for more details.
- *
- * Internally this set uses the canonical representation, but keeps also track of the original
- * expressions to ease debugging.  Since different expressions can share the same canonical
- * representation, this means that operations that extract expressions from this set are only
- * guaranteed to see at least one such expression.  For example:
- *
- * {{{
- *   val set = ExpressionSet(a + 1, 1 + a)
- *
- *   set.iterator => Iterator(a + 1)
- *   set.contains(a + 1) => true
- *   set.contains(1 + a) => true
- *   set.contains(a + 2) => false
- * }}}
- *
- * For non-deterministic expressions, they are always considered as not contained in the [[Set]].
- * On adding a non-deterministic expression, simply append it to the original expressions.
- * This is consistent with how we define `semanticEquals` between two expressions.
- */
-class ExpressionSet protected(
-    protected val baseSet: mutable.Set[Expression] = new mutable.HashSet,
-    protected val originals: mutable.Buffer[Expression] = new ArrayBuffer)
-  extends Set[Expression] {
-
-  //  Note: this class supports Scala 2.13. A parallel source tree has a 2.12 implementation.
-
-  protected def add(e: Expression): Unit = {
-    if (!e.deterministic) {
-      originals += e
-    } else if (!baseSet.contains(e.canonicalized) ) {
-      baseSet.add(e.canonicalized)
-      originals += e
-    }
-  }
-
-  override def contains(elem: Expression): Boolean = baseSet.contains(elem.canonicalized)
-
-  override def incl(elem: Expression): ExpressionSet = {
-    val newSet = new ExpressionSet(baseSet.clone(), originals.clone())
-    newSet.add(elem)
-    newSet
-  }
-
-  override def excl(elem: Expression): ExpressionSet = {
-    if (elem.deterministic) {
-      val newBaseSet = baseSet.clone().filterNot(_ == elem.canonicalized)
-      val newOriginals = originals.clone().filterNot(_.canonicalized == elem.canonicalized)
-      new ExpressionSet(newBaseSet, newOriginals)
-    } else {
-      new ExpressionSet(baseSet.clone(), originals.clone())
-    }
-  }
-
-  override def iterator: Iterator[Expression] = originals.iterator
-
-  /**
-   * Returns a string containing both the post [[Canonicalize]] expressions and the original
-   * expressions in this set.
-   */
-  def toDebugString: String =
-    s"""
-       |baseSet: ${baseSet.mkString(", ")}
-       |originals: ${originals.mkString(", ")}
-     """.stripMargin
-}
diff --git a/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala b/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
index 1f8cc6c0f83a2..e18a01810d2eb 100644
--- a/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
+++ b/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
@@ -39,8 +39,14 @@ class CaseInsensitiveMap[T] private (val originalMap: Map[String, T]) extends Ma
   override def contains(k: String): Boolean =
     keyLowerCasedMap.contains(k.toLowerCase(Locale.ROOT))
 
-  override def updated[B1 >: T](key: String, value: B1): Map[String, B1] = {
-    new CaseInsensitiveMap[B1](originalMap + (key -> value))
+  override def updated[B1 >: T](key: String, value: B1): CaseInsensitiveMap[B1] = {
+    new CaseInsensitiveMap[B1](originalMap.filter(!_._1.equalsIgnoreCase(key)) + (key -> value))
+  }
+
+  override def +[B1 >: T](kv: (String, B1)): CaseInsensitiveMap[B1] = this.updated(kv._1, kv._2)
+
+  def ++(xs: IterableOnce[(String, T)]): CaseInsensitiveMap[T] = {
+    xs.iterator.foldLeft(this) { (m, kv) => m.updated(kv._1, kv._2) }
   }
 
   override def iterator: Iterator[(String, T)] = keyLowerCasedMap.iterator
@@ -48,6 +54,8 @@ class CaseInsensitiveMap[T] private (val originalMap: Map[String, T]) extends Ma
   override def removed(key: String): Map[String, T] = {
     new CaseInsensitiveMap(originalMap.filter(!_._1.equalsIgnoreCase(key)))
   }
+
+  def toMap: Map[String, T] = originalMap
 }
 
 object CaseInsensitiveMap {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala
index f5c87677ab9eb..1dfbff5c6df5b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala
@@ -48,9 +48,11 @@ class AnalysisException protected[sql] (
 
   // Outputs an exception without the logical plan.
   // For testing only
-  def getSimpleMessage: String = {
+  def getSimpleMessage: String = if (line.isDefined || startPosition.isDefined) {
     val lineAnnotation = line.map(l => s" line $l").getOrElse("")
     val positionAnnotation = startPosition.map(p => s" pos $p").getOrElse("")
     s"$message;$lineAnnotation$positionAnnotation"
+  } else {
+    message
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoders.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoders.scala
index 5d31b5bbf12af..d50829578e6fe 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoders.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoders.scala
@@ -135,6 +135,22 @@ object Encoders {
    */
   def BINARY: Encoder[Array[Byte]] = ExpressionEncoder()
 
+  /**
+   * Creates an encoder that serializes instances of the `java.time.Duration` class
+   * to the internal representation of nullable Catalyst's DayTimeIntervalType.
+   *
+   * @since 3.2.0
+   */
+  def DURATION: Encoder[java.time.Duration] = ExpressionEncoder()
+
+  /**
+   * Creates an encoder that serializes instances of the `java.time.Period` class
+   * to the internal representation of nullable Catalyst's YearMonthIntervalType.
+   *
+   * @since 3.2.0
+   */
+  def PERIOD: Encoder[java.time.Period] = ExpressionEncoder()
+
   /**
    * Creates an encoder for Java Bean of type T.
    *
@@ -146,7 +162,7 @@ object Encoders {
    *  - String
    *  - java.math.BigDecimal, java.math.BigInteger
    *  - time related: java.sql.Date, java.sql.Timestamp, java.time.LocalDate, java.time.Instant
-   *  - collection types: only array and java.util.List currently, map support is in progress
+   *  - collection types: array, java.util.List, and map
    *  - nested java bean.
    *
    * @since 1.6.0
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala
index 4487a2d7f4358..d43c57ed0f5c8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala
@@ -160,6 +160,7 @@ trait Row extends Serializable {
    *   ByteType -> java.lang.Byte
    *   ShortType -> java.lang.Short
    *   IntegerType -> java.lang.Integer
+   *   LongType -> java.lang.Long
    *   FloatType -> java.lang.Float
    *   DoubleType -> java.lang.Double
    *   StringType -> String
@@ -188,6 +189,7 @@ trait Row extends Serializable {
    *   ByteType -> java.lang.Byte
    *   ShortType -> java.lang.Short
    *   IntegerType -> java.lang.Integer
+   *   LongType -> java.lang.Long
    *   FloatType -> java.lang.Float
    *   DoubleType -> java.lang.Double
    *   StringType -> String
@@ -314,7 +316,7 @@ trait Row extends Serializable {
    *
    * @throws ClassCastException when data type does not match.
    */
-  def getSeq[T](i: Int): Seq[T] = getAs[Seq[T]](i)
+  def getSeq[T](i: Int): Seq[T] = getAs[scala.collection.Seq[T]](i).toSeq
 
   /**
    * Returns the value at position i of array type as `java.util.List`.
@@ -349,7 +351,7 @@ trait Row extends Serializable {
   /**
    * Returns the value at position i.
    * For primitive types if value is null it returns 'zero value' specific for primitive
-   * ie. 0 for Int - use isNullAt to ensure that value is not null
+   * i.e. 0 for Int - use isNullAt to ensure that value is not null
    *
    * @throws ClassCastException when data type does not match.
    */
@@ -358,7 +360,7 @@ trait Row extends Serializable {
   /**
    * Returns the value of a given fieldName.
    * For primitive types if value is null it returns 'zero value' specific for primitive
-   * ie. 0 for Int - use isNullAt to ensure that value is not null
+   * i.e. 0 for Int - use isNullAt to ensure that value is not null
    *
    * @throws UnsupportedOperationException when schema is not defined.
    * @throws IllegalArgumentException when fieldName do not exist.
@@ -379,7 +381,7 @@ trait Row extends Serializable {
   /**
    * Returns a Map consisting of names and values for the requested fieldNames
    * For primitive types if value is null it returns 'zero value' specific for primitive
-   * ie. 0 for Int - use isNullAt to ensure that value is not null
+   * i.e. 0 for Int - use isNullAt to ensure that value is not null
    *
    * @throws UnsupportedOperationException when schema is not defined.
    * @throws IllegalArgumentException when fieldName do not exist.
@@ -571,14 +573,10 @@ trait Row extends Serializable {
       case (s: String, _) => JString(s)
       case (b: Array[Byte], BinaryType) =>
         JString(Base64.getEncoder.encodeToString(b))
-      case (d: LocalDate, _) =>
-        JString(dateFormatter.format(DateTimeUtils.localDateToDays(d)))
-      case (d: Date, _) =>
-        JString(dateFormatter.format(DateTimeUtils.fromJavaDate(d)))
-      case (i: Instant, _) =>
-        JString(timestampFormatter.format(DateTimeUtils.instantToMicros(i)))
-      case (t: Timestamp, _) =>
-        JString(timestampFormatter.format(DateTimeUtils.fromJavaTimestamp(t)))
+      case (d: LocalDate, _) => JString(dateFormatter.format(d))
+      case (d: Date, _) => JString(dateFormatter.format(d))
+      case (i: Instant, _) => JString(timestampFormatter.format(i))
+      case (t: Timestamp, _) => JString(timestampFormatter.format(t))
       case (i: CalendarInterval, _) => JString(i.toString)
       case (a: Array[_], ArrayType(elementType, _)) =>
         iteratorToJsonArray(a.iterator, elementType)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala
index 34d2f45e715e9..ccf0a50b734f7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala
@@ -21,7 +21,7 @@ import java.lang.{Iterable => JavaIterable}
 import java.math.{BigDecimal => JavaBigDecimal}
 import java.math.{BigInteger => JavaBigInteger}
 import java.sql.{Date, Timestamp}
-import java.time.{Instant, LocalDate}
+import java.time.{Duration, Instant, LocalDate, Period}
 import java.util.{Map => JavaMap}
 import javax.annotation.Nullable
 
@@ -74,6 +74,8 @@ object CatalystTypeConverters {
       case LongType => LongConverter
       case FloatType => FloatConverter
       case DoubleType => DoubleConverter
+      case DayTimeIntervalType => DurationConverter
+      case YearMonthIntervalType => PeriodConverter
       case dataType: DataType => IdentityConverter(dataType)
     }
     converter.asInstanceOf[CatalystTypeConverter[Any, Any, Any]]
@@ -164,7 +166,7 @@ object CatalystTypeConverters {
       scalaValue match {
         case a: Array[_] =>
           new GenericArrayData(a.map(elementConverter.toCatalyst))
-        case s: Seq[_] =>
+        case s: scala.collection.Seq[_] =>
           new GenericArrayData(s.map(elementConverter.toCatalyst).toArray)
         case i: JavaIterable[_] =>
           val iter = i.iterator
@@ -291,6 +293,7 @@ object CatalystTypeConverters {
       case str: String => UTF8String.fromString(str)
       case utf8: UTF8String => utf8
       case chr: Char => UTF8String.fromString(chr.toString)
+      case ac: Array[Char] => UTF8String.fromString(String.valueOf(ac))
       case other => throw new IllegalArgumentException(
         s"The value (${other.toString}) of the type (${other.getClass.getCanonicalName}) "
           + s"cannot be converted to the string type")
@@ -301,18 +304,23 @@ object CatalystTypeConverters {
       row.getUTF8String(column).toString
   }
 
-  private object DateConverter extends CatalystTypeConverter[Date, Date, Any] {
-    override def toCatalystImpl(scalaValue: Date): Int = DateTimeUtils.fromJavaDate(scalaValue)
+  private object DateConverter extends CatalystTypeConverter[Any, Date, Any] {
+    override def toCatalystImpl(scalaValue: Any): Int = scalaValue match {
+      case d: Date => DateTimeUtils.fromJavaDate(d)
+      case l: LocalDate => DateTimeUtils.localDateToDays(l)
+      case other => throw new IllegalArgumentException(
+        s"The value (${other.toString}) of the type (${other.getClass.getCanonicalName}) "
+          + s"cannot be converted to the ${DateType.sql} type")
+    }
     override def toScala(catalystValue: Any): Date =
       if (catalystValue == null) null else DateTimeUtils.toJavaDate(catalystValue.asInstanceOf[Int])
     override def toScalaImpl(row: InternalRow, column: Int): Date =
       DateTimeUtils.toJavaDate(row.getInt(column))
   }
 
-  private object LocalDateConverter extends CatalystTypeConverter[LocalDate, LocalDate, Any] {
-    override def toCatalystImpl(scalaValue: LocalDate): Int = {
-      DateTimeUtils.localDateToDays(scalaValue)
-    }
+  private object LocalDateConverter extends CatalystTypeConverter[Any, LocalDate, Any] {
+    override def toCatalystImpl(scalaValue: Any): Int =
+      DateConverter.toCatalystImpl(scalaValue)
     override def toScala(catalystValue: Any): LocalDate = {
       if (catalystValue == null) null
       else DateTimeUtils.daysToLocalDate(catalystValue.asInstanceOf[Int])
@@ -321,9 +329,14 @@ object CatalystTypeConverters {
       DateTimeUtils.daysToLocalDate(row.getInt(column))
   }
 
-  private object TimestampConverter extends CatalystTypeConverter[Timestamp, Timestamp, Any] {
-    override def toCatalystImpl(scalaValue: Timestamp): Long =
-      DateTimeUtils.fromJavaTimestamp(scalaValue)
+  private object TimestampConverter extends CatalystTypeConverter[Any, Timestamp, Any] {
+    override def toCatalystImpl(scalaValue: Any): Long = scalaValue match {
+      case t: Timestamp => DateTimeUtils.fromJavaTimestamp(t)
+      case i: Instant => DateTimeUtils.instantToMicros(i)
+      case other => throw new IllegalArgumentException(
+        s"The value (${other.toString}) of the type (${other.getClass.getCanonicalName}) "
+          + s"cannot be converted to the ${TimestampType.sql} type")
+    }
     override def toScala(catalystValue: Any): Timestamp =
       if (catalystValue == null) null
       else DateTimeUtils.toJavaTimestamp(catalystValue.asInstanceOf[Long])
@@ -331,9 +344,9 @@ object CatalystTypeConverters {
       DateTimeUtils.toJavaTimestamp(row.getLong(column))
   }
 
-  private object InstantConverter extends CatalystTypeConverter[Instant, Instant, Any] {
-    override def toCatalystImpl(scalaValue: Instant): Long =
-      DateTimeUtils.instantToMicros(scalaValue)
+  private object InstantConverter extends CatalystTypeConverter[Any, Instant, Any] {
+    override def toCatalystImpl(scalaValue: Any): Long =
+      TimestampConverter.toCatalystImpl(scalaValue)
     override def toScala(catalystValue: Any): Instant =
       if (catalystValue == null) null
       else DateTimeUtils.microsToInstant(catalystValue.asInstanceOf[Long])
@@ -399,6 +412,30 @@ object CatalystTypeConverters {
     override def toScalaImpl(row: InternalRow, column: Int): Double = row.getDouble(column)
   }
 
+  private object DurationConverter extends CatalystTypeConverter[Duration, Duration, Any] {
+    override def toCatalystImpl(scalaValue: Duration): Long = {
+      IntervalUtils.durationToMicros(scalaValue)
+    }
+    override def toScala(catalystValue: Any): Duration = {
+      if (catalystValue == null) null
+      else IntervalUtils.microsToDuration(catalystValue.asInstanceOf[Long])
+    }
+    override def toScalaImpl(row: InternalRow, column: Int): Duration =
+      IntervalUtils.microsToDuration(row.getLong(column))
+  }
+
+  private object PeriodConverter extends CatalystTypeConverter[Period, Period, Any] {
+    override def toCatalystImpl(scalaValue: Period): Int = {
+      IntervalUtils.periodToMonths(scalaValue)
+    }
+    override def toScala(catalystValue: Any): Period = {
+      if (catalystValue == null) null
+      else IntervalUtils.monthsToPeriod(catalystValue.asInstanceOf[Int])
+    }
+    override def toScalaImpl(row: InternalRow, column: Int): Period =
+      IntervalUtils.monthsToPeriod(row.getInt(column))
+  }
+
   /**
    * Creates a converter function that will convert Scala objects to the specified Catalyst type.
    * Typical use case would be converting a collection of rows that have the same schema. You will
@@ -456,12 +493,16 @@ object CatalystTypeConverters {
     case d: JavaBigDecimal => new DecimalConverter(DecimalType(d.precision, d.scale)).toCatalyst(d)
     case seq: Seq[Any] => new GenericArrayData(seq.map(convertToCatalyst).toArray)
     case r: Row => InternalRow(r.toSeq.map(convertToCatalyst): _*)
-    case arr: Array[Any] => new GenericArrayData(arr.map(convertToCatalyst))
+    case arr: Array[Byte] => arr
+    case arr: Array[Char] => StringConverter.toCatalyst(arr)
+    case arr: Array[_] => new GenericArrayData(arr.map(convertToCatalyst))
     case map: Map[_, _] =>
       ArrayBasedMapData(
         map,
         (key: Any) => convertToCatalyst(key),
         (value: Any) => convertToCatalyst(value))
+    case d: Duration => DurationConverter.toCatalyst(d)
+    case p: Period => PeriodConverter.toCatalyst(p)
     case other => other
   }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala
index e55c25c4b0c54..eaa7c17bfd312 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst
 import org.apache.spark.sql.catalyst.analysis.UnresolvedExtractValue
 import org.apache.spark.sql.catalyst.expressions.{Expression, GetStructField, UpCast}
 import org.apache.spark.sql.catalyst.expressions.objects.{AssertNotNull, Invoke, StaticInvoke}
-import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.catalyst.util.{DateTimeUtils, IntervalUtils}
 import org.apache.spark.sql.types._
 
 object DeserializerBuildHelper {
@@ -143,6 +143,24 @@ object DeserializerBuildHelper {
       returnNullable = false)
   }
 
+  def createDeserializerForDuration(path: Expression): Expression = {
+    StaticInvoke(
+      IntervalUtils.getClass,
+      ObjectType(classOf[java.time.Duration]),
+      "microsToDuration",
+      path :: Nil,
+      returnNullable = false)
+  }
+
+  def createDeserializerForPeriod(path: Expression): Expression = {
+    StaticInvoke(
+      IntervalUtils.getClass,
+      ObjectType(classOf[java.time.Period]),
+      "monthsToPeriod",
+      path :: Nil,
+      returnNullable = false)
+  }
+
   /**
    * When we build the `deserializer` for an encoder, we set up a lot of "unresolved" stuff
    * and lost the required data type, which may lead to runtime error if the real type doesn't
@@ -161,6 +179,10 @@ object DeserializerBuildHelper {
     case _: StructType => expr
     case _: ArrayType => expr
     case _: MapType => expr
+    case _: DecimalType =>
+      // For Scala/Java `BigDecimal`, we accept decimal types of any valid precision/scale.
+      // Here we use the `DecimalType` object to indicate it.
+      UpCast(expr, DecimalType, walkedTypePath.getPaths)
     case _ => UpCast(expr, expected, walkedTypePath.getPaths)
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/InternalRow.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/InternalRow.scala
index f98b59edd4226..fd74f60c0c47e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/InternalRow.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/InternalRow.scala
@@ -132,8 +132,10 @@ object InternalRow {
       case BooleanType => (input, ordinal) => input.getBoolean(ordinal)
       case ByteType => (input, ordinal) => input.getByte(ordinal)
       case ShortType => (input, ordinal) => input.getShort(ordinal)
-      case IntegerType | DateType => (input, ordinal) => input.getInt(ordinal)
-      case LongType | TimestampType => (input, ordinal) => input.getLong(ordinal)
+      case IntegerType | DateType | YearMonthIntervalType =>
+        (input, ordinal) => input.getInt(ordinal)
+      case LongType | TimestampType | DayTimeIntervalType =>
+        (input, ordinal) => input.getLong(ordinal)
       case FloatType => (input, ordinal) => input.getFloat(ordinal)
       case DoubleType => (input, ordinal) => input.getDouble(ordinal)
       case StringType => (input, ordinal) => input.getUTF8String(ordinal)
@@ -167,8 +169,10 @@ object InternalRow {
     case BooleanType => (input, v) => input.setBoolean(ordinal, v.asInstanceOf[Boolean])
     case ByteType => (input, v) => input.setByte(ordinal, v.asInstanceOf[Byte])
     case ShortType => (input, v) => input.setShort(ordinal, v.asInstanceOf[Short])
-    case IntegerType | DateType => (input, v) => input.setInt(ordinal, v.asInstanceOf[Int])
-    case LongType | TimestampType => (input, v) => input.setLong(ordinal, v.asInstanceOf[Long])
+    case IntegerType | DateType | YearMonthIntervalType =>
+      (input, v) => input.setInt(ordinal, v.asInstanceOf[Int])
+    case LongType | TimestampType | DayTimeIntervalType =>
+      (input, v) => input.setLong(ordinal, v.asInstanceOf[Long])
     case FloatType => (input, v) => input.setFloat(ordinal, v.asInstanceOf[Float])
     case DoubleType => (input, v) => input.setDouble(ordinal, v.asInstanceOf[Double])
     case CalendarIntervalType =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala
index 2248e2eb02597..541b78336ba04 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala
@@ -118,6 +118,8 @@ object JavaTypeInference {
       case c: Class[_] if c == classOf[java.sql.Date] => (DateType, true)
       case c: Class[_] if c == classOf[java.time.Instant] => (TimestampType, true)
       case c: Class[_] if c == classOf[java.sql.Timestamp] => (TimestampType, true)
+      case c: Class[_] if c == classOf[java.time.Duration] => (DayTimeIntervalType, true)
+      case c: Class[_] if c == classOf[java.time.Period] => (YearMonthIntervalType, true)
 
       case _ if typeToken.isArray =>
         val (dataType, nullable) = inferDataType(typeToken.getComponentType, seenTypeSet)
@@ -249,6 +251,12 @@ object JavaTypeInference {
       case c if c == classOf[java.sql.Timestamp] =>
         createDeserializerForSqlTimestamp(path)
 
+      case c if c == classOf[java.time.Duration] =>
+        createDeserializerForDuration(path)
+
+      case c if c == classOf[java.time.Period] =>
+        createDeserializerForPeriod(path)
+
       case c if c == classOf[java.lang.String] =>
         createDeserializerForString(path, returnNullable = true)
 
@@ -406,6 +414,10 @@ object JavaTypeInference {
 
         case c if c == classOf[java.sql.Date] => createSerializerForSqlDate(inputObject)
 
+        case c if c == classOf[java.time.Duration] => createSerializerForJavaDuration(inputObject)
+
+        case c if c == classOf[java.time.Period] => createSerializerForJavaPeriod(inputObject)
+
         case c if c == classOf[java.math.BigDecimal] =>
           createSerializerForJavaBigDecimal(inputObject)
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/OrderedFilters.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/OrderedFilters.scala
new file mode 100644
index 0000000000000..b7c8a0140ea66
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/OrderedFilters.scala
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst
+
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.sources
+import org.apache.spark.sql.types.StructType
+
+/**
+ * An instance of the class compiles filters to predicates and sorts them in
+ * the order which allows to apply the predicates to an internal row with partially
+ * initialized values, for instance converted from parsed CSV fields.
+ *
+ * @param filters The filters pushed down to a datasource.
+ * @param requiredSchema The schema with only fields requested by the upper layer.
+ */
+class OrderedFilters(filters: Seq[sources.Filter], requiredSchema: StructType)
+  extends StructFilters(filters, requiredSchema) {
+  /**
+   * Converted filters to predicates and grouped by maximum field index
+   * in the read schema. For example, if an filter refers to 2 attributes
+   * attrA with field index 5 and attrB with field index 10 in the read schema:
+   *   0 === $"attrA" or $"attrB" < 100
+   * the filter is compiled to a predicate, and placed to the `predicates`
+   * array at the position 10. In this way, if there is a row with initialized
+   * fields from the 0 to 10 index, the predicate can be applied to the row
+   * to check that the row should be skipped or not.
+   * Multiple predicates with the same maximum reference index are combined
+   * by the `And` expression.
+   */
+  private val predicates: Array[BasePredicate] = {
+    val len = requiredSchema.fields.length
+    val groupedPredicates = Array.fill[BasePredicate](len)(null)
+    val groupedFilters = Array.fill(len)(Seq.empty[sources.Filter])
+    for (filter <- filters) {
+      val refs = filter.references
+      val index = if (refs.isEmpty) {
+        // For example, `AlwaysTrue` and `AlwaysFalse` doesn't have any references
+        // Filters w/o refs always return the same result. Taking into account
+        // that predicates are combined via `And`, we can apply such filters only
+        // once at the position 0.
+        0
+      } else {
+        // readSchema must contain attributes of all filters.
+        // Accordingly, `fieldIndex()` returns a valid index always.
+        refs.map(requiredSchema.fieldIndex).max
+      }
+      groupedFilters(index) :+= filter
+    }
+    if (len > 0 && groupedFilters(0).nonEmpty) {
+      // We assume that filters w/o refs like `AlwaysTrue` and `AlwaysFalse`
+      // can be evaluated faster that others. We put them in front of others.
+      val (literals, others) = groupedFilters(0).partition(_.references.isEmpty)
+      groupedFilters(0) = literals ++ others
+    }
+    for (i <- 0 until len) {
+      if (groupedFilters(i).nonEmpty) {
+        groupedPredicates(i) = toPredicate(groupedFilters(i))
+      }
+    }
+    groupedPredicates
+  }
+
+  /**
+   * Applies all filters that refer to row fields at the positions from 0 to `index`.
+   * @param row The internal row to check.
+   * @param index Maximum field index. The function assumes that all fields
+   *              from 0 to `index` position are set.
+   * @return false` iff row fields at the position from 0 to `index` pass filters
+   *         or there are no applicable filters
+   *         otherwise `false` if at least one of the filters returns `false`.
+   */
+  def skipRow(row: InternalRow, index: Int): Boolean = {
+    assert(0 <= index && index < requiredSchema.fields.length,
+      "Index is out of the valid range: it must point out to a field of the required schema.")
+    val predicate = predicates(index)
+    predicate != null && !predicate.eval(row)
+  }
+
+  // The filters are applied sequentially, and no need to track which filter references
+  // point out to already set row values. The `reset()` method is trivial because
+  // the filters don't have any states.
+  def reset(): Unit = {}
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/QueryPlanningTracker.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/QueryPlanningTracker.scala
index cd75407c7ee7a..35551d8ba77dc 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/QueryPlanningTracker.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/QueryPlanningTracker.scala
@@ -28,7 +28,7 @@ import org.apache.spark.util.BoundedPriorityQueue
  * There are two separate concepts we track:
  *
  * 1. Phases: These are broad scope phases in query planning, as listed below, i.e. analysis,
- * optimizationm and physical planning (just planning).
+ * optimization and physical planning (just planning).
  *
  * 2. Rules: These are the individual Catalyst rules that we track. In addition to time, we also
  * track the number of invocations and effective invocations.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SQLConfHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SQLConfHelper.scala
new file mode 100644
index 0000000000000..cee35cdb8d840
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SQLConfHelper.scala
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst
+
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * Trait for getting the active SQLConf.
+ */
+trait SQLConfHelper {
+
+  /**
+   * The active config object within the current scope.
+   * See [[SQLConf.get]] for more information.
+   */
+  def conf: SQLConf = SQLConf.get
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
index 05de21b9eafec..73a6367be9adf 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
@@ -28,6 +28,7 @@ import org.apache.spark.sql.catalyst.analysis.GetColumnByOrdinal
 import org.apache.spark.sql.catalyst.expressions.{Expression, _}
 import org.apache.spark.sql.catalyst.expressions.objects._
 import org.apache.spark.sql.catalyst.util.{ArrayData, MapData}
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 
@@ -232,9 +233,20 @@ object ScalaReflection extends ScalaReflection {
       case t if isSubtype(t, localTypeOf[java.time.Instant]) =>
         createDeserializerForInstant(path)
 
+      case t if isSubtype(t, localTypeOf[java.lang.Enum[_]]) =>
+        createDeserializerForTypesSupportValueOf(
+          Invoke(path, "toString", ObjectType(classOf[String]), returnNullable = false),
+          getClassFromType(t))
+
       case t if isSubtype(t, localTypeOf[java.sql.Timestamp]) =>
         createDeserializerForSqlTimestamp(path)
 
+      case t if isSubtype(t, localTypeOf[java.time.Duration]) =>
+        createDeserializerForDuration(path)
+
+      case t if isSubtype(t, localTypeOf[java.time.Period]) =>
+        createDeserializerForPeriod(path)
+
       case t if isSubtype(t, localTypeOf[java.lang.String]) =>
         createDeserializerForString(path, returnNullable = false)
 
@@ -284,7 +296,7 @@ object ScalaReflection extends ScalaReflection {
 
       // We serialize a `Set` to Catalyst array. When we deserialize a Catalyst array
       // to a `Set`, if there are duplicated elements, the elements will be de-duplicated.
-      case t if isSubtype(t, localTypeOf[Seq[_]]) ||
+      case t if isSubtype(t, localTypeOf[scala.collection.Seq[_]]) ||
           isSubtype(t, localTypeOf[scala.collection.Set[_]]) =>
         val TypeRef(_, _, Seq(elementType)) = t
         val Schema(dataType, elementNullable) = schemaFor(elementType)
@@ -377,6 +389,23 @@ object ScalaReflection extends ScalaReflection {
           expressions.Literal.create(null, ObjectType(cls)),
           newInstance
         )
+
+      case t if isSubtype(t, localTypeOf[Enumeration#Value]) =>
+        // package example
+        // object Foo extends Enumeration {
+        //  type Foo = Value
+        //  val E1, E2 = Value
+        // }
+        // the fullName of tpe is example.Foo.Foo, but we need example.Foo so that
+        // we can call example.Foo.withName to deserialize string to enumeration.
+        val parent = t.asInstanceOf[TypeRef].pre.typeSymbol.asClass
+        val cls = mirror.runtimeClass(parent)
+        StaticInvoke(
+          cls,
+          ObjectType(getClassFromType(t)),
+          "withName",
+          createDeserializerForString(path, false) :: Nil,
+          returnNullable = false)
     }
   }
 
@@ -448,10 +477,9 @@ object ScalaReflection extends ScalaReflection {
       // Since List[_] also belongs to localTypeOf[Product], we put this case before
       // "case t if definedByConstructorParams(t)" to make sure it will match to the
       // case "localTypeOf[Seq[_]]"
-      case t if isSubtype(t, localTypeOf[Seq[_]]) =>
+      case t if isSubtype(t, localTypeOf[scala.collection.Seq[_]]) =>
         val TypeRef(_, _, Seq(elementType)) = t
         toCatalystArray(inputObject, elementType)
-
       case t if isSubtype(t, localTypeOf[Array[_]]) =>
         val TypeRef(_, _, Seq(elementType)) = t
         toCatalystArray(inputObject, elementType)
@@ -501,6 +529,12 @@ object ScalaReflection extends ScalaReflection {
 
       case t if isSubtype(t, localTypeOf[java.sql.Date]) => createSerializerForSqlDate(inputObject)
 
+      case t if isSubtype(t, localTypeOf[java.time.Duration]) =>
+        createSerializerForJavaDuration(inputObject)
+
+      case t if isSubtype(t, localTypeOf[java.time.Period]) =>
+        createSerializerForJavaPeriod(inputObject)
+
       case t if isSubtype(t, localTypeOf[BigDecimal]) =>
         createSerializerForScalaBigDecimal(inputObject)
 
@@ -510,6 +544,9 @@ object ScalaReflection extends ScalaReflection {
       case t if isSubtype(t, localTypeOf[java.math.BigInteger]) =>
         createSerializerForJavaBigInteger(inputObject)
 
+      case t if isSubtype(t, localTypeOf[java.lang.Enum[_]]) =>
+        createSerializerForJavaEnum(inputObject)
+
       case t if isSubtype(t, localTypeOf[scala.math.BigInt]) =>
         createSerializerForScalaBigInt(inputObject)
 
@@ -562,6 +599,14 @@ object ScalaReflection extends ScalaReflection {
         }
         createSerializerForObject(inputObject, fields)
 
+      case t if isSubtype(t, localTypeOf[Enumeration#Value]) =>
+        createSerializerForString(
+          Invoke(
+            inputObject,
+            "toString",
+            ObjectType(classOf[java.lang.String]),
+            returnNullable = false))
+
       case _ =>
         throw new UnsupportedOperationException(
           s"No Encoder found for $tpe\n" + walkedTypePath)
@@ -686,7 +731,7 @@ object ScalaReflection extends ScalaReflection {
         val TypeRef(_, _, Seq(elementType)) = t
         val Schema(dataType, nullable) = schemaFor(elementType)
         Schema(ArrayType(dataType, containsNull = nullable), nullable = true)
-      case t if isSubtype(t, localTypeOf[Seq[_]]) =>
+      case t if isSubtype(t, localTypeOf[scala.collection.Seq[_]]) =>
         val TypeRef(_, _, Seq(elementType)) = t
         val Schema(dataType, nullable) = schemaFor(elementType)
         Schema(ArrayType(dataType, containsNull = nullable), nullable = true)
@@ -708,6 +753,10 @@ object ScalaReflection extends ScalaReflection {
       case t if isSubtype(t, localTypeOf[java.sql.Date]) => Schema(DateType, nullable = true)
       case t if isSubtype(t, localTypeOf[CalendarInterval]) =>
         Schema(CalendarIntervalType, nullable = true)
+      case t if isSubtype(t, localTypeOf[java.time.Duration]) =>
+        Schema(DayTimeIntervalType, nullable = true)
+      case t if isSubtype(t, localTypeOf[java.time.Period]) =>
+        Schema(YearMonthIntervalType, nullable = true)
       case t if isSubtype(t, localTypeOf[BigDecimal]) =>
         Schema(DecimalType.SYSTEM_DEFAULT, nullable = true)
       case t if isSubtype(t, localTypeOf[java.math.BigDecimal]) =>
@@ -725,6 +774,7 @@ object ScalaReflection extends ScalaReflection {
       case t if isSubtype(t, localTypeOf[java.lang.Short]) => Schema(ShortType, nullable = true)
       case t if isSubtype(t, localTypeOf[java.lang.Byte]) => Schema(ByteType, nullable = true)
       case t if isSubtype(t, localTypeOf[java.lang.Boolean]) => Schema(BooleanType, nullable = true)
+      case t if isSubtype(t, localTypeOf[java.lang.Enum[_]]) => Schema(StringType, nullable = true)
       case t if isSubtype(t, definitions.IntTpe) => Schema(IntegerType, nullable = false)
       case t if isSubtype(t, definitions.LongTpe) => Schema(LongType, nullable = false)
       case t if isSubtype(t, definitions.DoubleTpe) => Schema(DoubleType, nullable = false)
@@ -739,6 +789,8 @@ object ScalaReflection extends ScalaReflection {
             val Schema(dataType, nullable) = schemaFor(fieldType)
             StructField(fieldName, dataType, nullable)
           }), nullable = true)
+      case t if isSubtype(t, localTypeOf[Enumeration#Value]) =>
+        Schema(StringType, nullable = true)
       case other =>
         throw new UnsupportedOperationException(s"Schema for type $other is not supported")
     }
@@ -802,7 +854,9 @@ object ScalaReflection extends ScalaReflection {
     DateType -> classOf[DateType.InternalType],
     TimestampType -> classOf[TimestampType.InternalType],
     BinaryType -> classOf[BinaryType.InternalType],
-    CalendarIntervalType -> classOf[CalendarInterval]
+    CalendarIntervalType -> classOf[CalendarInterval],
+    DayTimeIntervalType -> classOf[DayTimeIntervalType.InternalType],
+    YearMonthIntervalType -> classOf[YearMonthIntervalType.InternalType]
   )
 
   val typeBoxedJavaMapping = Map[DataType, Class[_]](
@@ -814,7 +868,9 @@ object ScalaReflection extends ScalaReflection {
     FloatType -> classOf[java.lang.Float],
     DoubleType -> classOf[java.lang.Double],
     DateType -> classOf[java.lang.Integer],
-    TimestampType -> classOf[java.lang.Long]
+    TimestampType -> classOf[java.lang.Long],
+    DayTimeIntervalType -> classOf[java.lang.Long],
+    YearMonthIntervalType -> classOf[java.lang.Integer]
   )
 
   def dataTypeJavaClass(dt: DataType): Class[_] = {
@@ -867,10 +923,6 @@ trait ScalaReflection extends Logging {
 
   import universe._
 
-  // The Predef.Map is scala.collection.immutable.Map.
-  // Since the map values can be mutable, we explicitly import scala.collection.Map at here.
-  import scala.collection.Map
-
   /**
    * Any codes calling `scala.reflect.api.Types.TypeApi.<:<` should be wrapped by this method to
    * clean up the Scala reflection garbage automatically. Otherwise, it will leak some objects to
@@ -952,7 +1004,7 @@ trait ScalaReflection extends Logging {
       val primaryConstructorSymbol: Option[Symbol] = constructorSymbol.asTerm.alternatives.find(
         s => s.isMethod && s.asMethod.isPrimaryConstructor)
       if (primaryConstructorSymbol.isEmpty) {
-        sys.error("Internal SQL error: Product object did not have a primary constructor.")
+        throw QueryExecutionErrors.primaryConstructorNotFoundError(tpe.getClass)
       } else {
         primaryConstructorSymbol.get.asMethod.paramLists
       }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SerializerBuildHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SerializerBuildHelper.scala
index 85acaa11230b4..f80fab573c9a8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SerializerBuildHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SerializerBuildHelper.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst
 
 import org.apache.spark.sql.catalyst.expressions.{CheckOverflow, CreateNamedStruct, Expression, IsNull, UnsafeArrayData}
 import org.apache.spark.sql.catalyst.expressions.objects._
-import org.apache.spark.sql.catalyst.util.{DateTimeUtils, GenericArrayData}
+import org.apache.spark.sql.catalyst.util.{DateTimeUtils, GenericArrayData, IntervalUtils}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
@@ -74,6 +74,9 @@ object SerializerBuildHelper {
       returnNullable = false)
   }
 
+  def createSerializerForJavaEnum(inputObject: Expression): Expression =
+    createSerializerForString(Invoke(inputObject, "name", ObjectType(classOf[String])))
+
   def createSerializerForSqlTimestamp(inputObject: Expression): Expression = {
     StaticInvoke(
       DateTimeUtils.getClass,
@@ -101,6 +104,24 @@ object SerializerBuildHelper {
       returnNullable = false)
   }
 
+  def createSerializerForJavaDuration(inputObject: Expression): Expression = {
+    StaticInvoke(
+      IntervalUtils.getClass,
+      DayTimeIntervalType,
+      "durationToMicros",
+      inputObject :: Nil,
+      returnNullable = false)
+  }
+
+  def createSerializerForJavaPeriod(inputObject: Expression): Expression = {
+    StaticInvoke(
+      IntervalUtils.getClass,
+      YearMonthIntervalType,
+      "periodToMonths",
+      inputObject :: Nil,
+      returnNullable = false)
+  }
+
   def createSerializerForJavaBigDecimal(inputObject: Expression): Expression = {
     CheckOverflow(StaticInvoke(
       Decimal.getClass,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/StructFilters.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/StructFilters.scala
new file mode 100644
index 0000000000000..ff67b6fccfae9
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/StructFilters.scala
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst
+
+import scala.util.Try
+
+import org.apache.spark.sql.catalyst.StructFilters._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.sources
+import org.apache.spark.sql.types.{BooleanType, StructType}
+
+/**
+ * The class provides API for applying pushed down filters to partially or
+ * fully set internal rows that have the struct schema.
+ *
+ * `StructFilters` assumes that:
+ *   - `reset()` is called before any `skipRow()` calls for new row.
+ *
+ * @param pushedFilters The pushed down source filters. The filters should refer to
+ *                      the fields of the provided schema.
+ * @param schema The required schema of records from datasource files.
+ */
+abstract class StructFilters(pushedFilters: Seq[sources.Filter], schema: StructType) {
+
+  protected val filters = StructFilters.pushedFilters(pushedFilters.toArray, schema)
+
+  /**
+   * Applies pushed down source filters to the given row assuming that
+   * value at `index` has been already set.
+   *
+   * @param row The row with fully or partially set values.
+   * @param index The index of already set value.
+   * @return `true` if currently processed row can be skipped otherwise false.
+   */
+  def skipRow(row: InternalRow, index: Int): Boolean
+
+  /**
+   * Resets states of pushed down filters. The method must be called before
+   * processing any new row otherwise `skipRow()` may return wrong result.
+   */
+  def reset(): Unit
+
+  /**
+   * Compiles source filters to a predicate.
+   */
+  def toPredicate(filters: Seq[sources.Filter]): BasePredicate = {
+    val reducedExpr = filters
+      .sortBy(_.references.length)
+      .flatMap(filterToExpression(_, toRef))
+      .reduce(And)
+    Predicate.create(reducedExpr)
+  }
+
+  // Finds a filter attribute in the schema and converts it to a `BoundReference`
+  private def toRef(attr: String): Option[BoundReference] = {
+    // The names have been normalized and case sensitivity is not a concern here.
+    schema.getFieldIndex(attr).map { index =>
+      val field = schema(index)
+      BoundReference(index, field.dataType, field.nullable)
+    }
+  }
+}
+
+object StructFilters {
+  private def checkFilterRefs(filter: sources.Filter, fieldNames: Set[String]): Boolean = {
+    // The names have been normalized and case sensitivity is not a concern here.
+    filter.references.forall(fieldNames.contains)
+  }
+
+  /**
+   * Returns the filters currently supported by the datasource.
+   * @param filters The filters pushed down to the datasource.
+   * @param schema data schema of datasource files.
+   * @return a sub-set of `filters` that can be handled by the datasource.
+   */
+  def pushedFilters(filters: Array[sources.Filter], schema: StructType): Array[sources.Filter] = {
+    val fieldNames = schema.fieldNames.toSet
+    filters.filter(checkFilterRefs(_, fieldNames))
+  }
+
+  private def zip[A, B](a: Option[A], b: Option[B]): Option[(A, B)] = {
+    a.zip(b).headOption
+  }
+
+  private def toLiteral(value: Any): Option[Literal] = {
+    Try(Literal(value)).toOption
+  }
+
+  /**
+   * Converts a filter to an expression and binds it to row positions.
+   *
+   * @param filter The filter to convert.
+   * @param toRef The function converts a filter attribute to a bound reference.
+   * @return some expression with resolved attributes or `None` if the conversion
+   *         of the given filter to an expression is impossible.
+   */
+  def filterToExpression(
+      filter: sources.Filter,
+      toRef: String => Option[BoundReference]): Option[Expression] = {
+    def zipAttributeAndValue(name: String, value: Any): Option[(BoundReference, Literal)] = {
+      zip(toRef(name), toLiteral(value))
+    }
+    def translate(filter: sources.Filter): Option[Expression] = filter match {
+      case sources.And(left, right) =>
+        zip(translate(left), translate(right)).map(And.tupled)
+      case sources.Or(left, right) =>
+        zip(translate(left), translate(right)).map(Or.tupled)
+      case sources.Not(child) =>
+        translate(child).map(Not)
+      case sources.EqualTo(attribute, value) =>
+        zipAttributeAndValue(attribute, value).map(EqualTo.tupled)
+      case sources.EqualNullSafe(attribute, value) =>
+        zipAttributeAndValue(attribute, value).map(EqualNullSafe.tupled)
+      case sources.IsNull(attribute) =>
+        toRef(attribute).map(IsNull)
+      case sources.IsNotNull(attribute) =>
+        toRef(attribute).map(IsNotNull)
+      case sources.In(attribute, values) =>
+        val literals = values.toSeq.flatMap(toLiteral)
+        if (literals.length == values.length) {
+          toRef(attribute).map(In(_, literals))
+        } else {
+          None
+        }
+      case sources.GreaterThan(attribute, value) =>
+        zipAttributeAndValue(attribute, value).map(GreaterThan.tupled)
+      case sources.GreaterThanOrEqual(attribute, value) =>
+        zipAttributeAndValue(attribute, value).map(GreaterThanOrEqual.tupled)
+      case sources.LessThan(attribute, value) =>
+        zipAttributeAndValue(attribute, value).map(LessThan.tupled)
+      case sources.LessThanOrEqual(attribute, value) =>
+        zipAttributeAndValue(attribute, value).map(LessThanOrEqual.tupled)
+      case sources.StringContains(attribute, value) =>
+        zipAttributeAndValue(attribute, value).map(Contains.tupled)
+      case sources.StringStartsWith(attribute, value) =>
+        zipAttributeAndValue(attribute, value).map(StartsWith.tupled)
+      case sources.StringEndsWith(attribute, value) =>
+        zipAttributeAndValue(attribute, value).map(EndsWith.tupled)
+      case sources.AlwaysTrue() =>
+        Some(Literal(true, BooleanType))
+      case sources.AlwaysFalse() =>
+        Some(Literal(false, BooleanType))
+    }
+    translate(filter)
+  }
+}
+
+class NoopFilters extends StructFilters(Seq.empty, new StructType()) {
+  override def skipRow(row: InternalRow, index: Int): Boolean = false
+  override def reset(): Unit = {}
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlreadyExistException.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlreadyExistException.scala
index 7e5d56a7d1196..70f821d5f8af0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlreadyExistException.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlreadyExistException.scala
@@ -18,9 +18,11 @@
 package org.apache.spark.sql.catalyst.analysis
 
 import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
 import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
 import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.types.StructType
 
 /**
  * Thrown by a catalog when an item already exists. The analyzer will rethrow the exception
@@ -35,7 +37,8 @@ class NamespaceAlreadyExistsException(message: String) extends AnalysisException
   }
 }
 
-class TableAlreadyExistsException(message: String) extends AnalysisException(message) {
+class TableAlreadyExistsException(message: String, cause: Option[Throwable] = None)
+  extends AnalysisException(message, cause = cause) {
   def this(db: String, table: String) = {
     this(s"Table or view '$table' already exists in database '$db'")
   }
@@ -48,14 +51,30 @@ class TableAlreadyExistsException(message: String) extends AnalysisException(mes
 class TempTableAlreadyExistsException(table: String)
   extends TableAlreadyExistsException(s"Temporary view '$table' already exists")
 
-class PartitionAlreadyExistsException(db: String, table: String, spec: TablePartitionSpec)
-  extends AnalysisException(
-    s"Partition already exists in table '$table' database '$db':\n" + spec.mkString("\n"))
+class PartitionAlreadyExistsException(message: String) extends AnalysisException(message) {
+  def this(db: String, table: String, spec: TablePartitionSpec) = {
+    this(s"Partition already exists in table '$table' database '$db':\n" + spec.mkString("\n"))
+  }
+
+  def this(tableName: String, partitionIdent: InternalRow, partitionSchema: StructType) = {
+    this(s"Partition already exists in table $tableName:" +
+      partitionIdent.toSeq(partitionSchema).zip(partitionSchema.map(_.name))
+        .map( kv => s"${kv._1} -> ${kv._2}").mkString(","))
+  }
+}
 
-class PartitionsAlreadyExistException(db: String, table: String, specs: Seq[TablePartitionSpec])
-  extends AnalysisException(
-    s"The following partitions already exists in table '$table' database '$db':\n"
+class PartitionsAlreadyExistException(message: String) extends AnalysisException(message) {
+  def this(db: String, table: String, specs: Seq[TablePartitionSpec]) = {
+    this(s"The following partitions already exists in table '$table' database '$db':\n"
       + specs.mkString("\n===\n"))
+  }
+
+  def this(tableName: String, partitionIdents: Seq[InternalRow], partitionSchema: StructType) = {
+    this(s"The following partitions already exists in table $tableName:" +
+      partitionIdents.map(_.toSeq(partitionSchema).zip(partitionSchema.map(_.name))
+        .map( kv => s"${kv._1} -> ${kv._2}").mkString(",")).mkString("\n===\n"))
+  }
+}
 
 class FunctionAlreadyExistsException(db: String, func: String)
   extends AnalysisException(s"Function '$func' already exists in database '$db'")
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 91f852da82533..9954ca08c2921 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.catalyst.analysis
 
+import java.lang.reflect.{Method, Modifier}
 import java.util
 import java.util.Locale
 import java.util.concurrent.atomic.AtomicBoolean
@@ -29,41 +30,50 @@ import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst._
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.encoders.OuterScopes
-import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.expressions.{Expression, FrameLessOffsetWindowFunction, _}
 import org.apache.spark.sql.catalyst.expressions.SubExprUtils._
 import org.apache.spark.sql.catalyst.expressions.aggregate._
 import org.apache.spark.sql.catalyst.expressions.objects._
+import org.apache.spark.sql.catalyst.optimizer.OptimizeUpdateFields
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules._
+import org.apache.spark.sql.catalyst.streaming.StreamingRelationV2
+import org.apache.spark.sql.catalyst.trees.AlwaysProcess
 import org.apache.spark.sql.catalyst.trees.TreeNodeRef
-import org.apache.spark.sql.catalyst.util.toPrettySQL
+import org.apache.spark.sql.catalyst.trees.TreePattern._
+import org.apache.spark.sql.catalyst.util.{toPrettySQL, CharVarcharUtils}
 import org.apache.spark.sql.connector.catalog._
 import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
 import org.apache.spark.sql.connector.catalog.TableChange.{AddColumn, After, ColumnChange, ColumnPosition, DeleteColumn, RenameColumn, UpdateColumnComment, UpdateColumnNullability, UpdateColumnPosition, UpdateColumnType}
+import org.apache.spark.sql.connector.catalog.functions.{AggregateFunction => V2AggregateFunction, BoundFunction, ScalarFunction}
+import org.apache.spark.sql.connector.catalog.functions.ScalarFunction.MAGIC_METHOD_NAME
 import org.apache.spark.sql.connector.expressions.{FieldReference, IdentityTransform, Transform}
+import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
 import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.{PartitionOverwriteMode, StoreAssignmentPolicy}
 import org.apache.spark.sql.types._
-import org.apache.spark.sql.util.CaseInsensitiveStringMap
+import org.apache.spark.sql.util.{CaseInsensitiveStringMap, SchemaUtils}
+import org.apache.spark.unsafe.types.UTF8String
+import org.apache.spark.util.Utils
 
 /**
- * A trivial [[Analyzer]] with a dummy [[SessionCatalog]] and [[EmptyFunctionRegistry]].
- * Used for testing when all relations are already filled in and the analyzer needs only
- * to resolve attribute references.
+ * A trivial [[Analyzer]] with a dummy [[SessionCatalog]], [[EmptyFunctionRegistry]] and
+ * [[EmptyTableFunctionRegistry]]. Used for testing when all relations are already filled
+ * in and the analyzer needs only to resolve attribute references.
  */
 object SimpleAnalyzer extends Analyzer(
   new CatalogManager(
-    new SQLConf().copy(SQLConf.CASE_SENSITIVE -> true),
     FakeV2SessionCatalog,
     new SessionCatalog(
       new InMemoryCatalog,
       EmptyFunctionRegistry,
-      new SQLConf().copy(SQLConf.CASE_SENSITIVE -> true)) {
+      EmptyTableFunctionRegistry) {
       override def createDatabase(dbDefinition: CatalogDatabase, ignoreIfExists: Boolean): Unit = {}
-    }),
-  new SQLConf().copy(SQLConf.CASE_SENSITIVE -> true))
+    })) {
+  override def resolver: Resolver = caseSensitiveResolution
+}
 
 object FakeV2SessionCatalog extends TableCatalog {
   private def fail() = throw new UnsupportedOperationException
@@ -84,8 +94,8 @@ object FakeV2SessionCatalog extends TableCatalog {
 }
 
 /**
- * Provides a way to keep state during the analysis, this enables us to decouple the concerns
- * of analysis environment from the catalog.
+ * Provides a way to keep state during the analysis, mostly for resolving views. This enables us to
+ * decouple the concerns of analysis environment from the catalog.
  * The state that is kept here is per-query.
  *
  * Note this is thread local.
@@ -95,14 +105,24 @@ object FakeV2SessionCatalog extends TableCatalog {
  *                            views.
  * @param nestedViewDepth The nested depth in the view resolution, this enables us to limit the
  *                        depth of nested views.
+ * @param maxNestedViewDepth The maximum allowed depth of nested view resolution.
  * @param relationCache A mapping from qualified table names to resolved relations. This can ensure
  *                      that the table is resolved only once if a table is used multiple times
  *                      in a query.
+ * @param referredTempViewNames All the temp view names referred by the current view we are
+ *                              resolving. It's used to make sure the relation resolution is
+ *                              consistent between view creation and view resolution. For example,
+ *                              if `t` was a permanent table when the current view was created, it
+ *                              should still be a permanent table when resolving the current view,
+ *                              even if a temp view `t` has been created.
  */
 case class AnalysisContext(
     catalogAndNamespace: Seq[String] = Nil,
     nestedViewDepth: Int = 0,
-    relationCache: mutable.Map[Seq[String], LogicalPlan] = mutable.Map.empty)
+    maxNestedViewDepth: Int = -1,
+    relationCache: mutable.Map[Seq[String], LogicalPlan] = mutable.Map.empty,
+    referredTempViewNames: Seq[Seq[String]] = Seq.empty,
+    referredTempFunctionNames: Seq[String] = Seq.empty)
 
 object AnalysisContext {
   private val value = new ThreadLocal[AnalysisContext]() {
@@ -114,10 +134,21 @@ object AnalysisContext {
 
   private def set(context: AnalysisContext): Unit = value.set(context)
 
-  def withAnalysisContext[A](catalogAndNamespace: Seq[String])(f: => A): A = {
+  def withAnalysisContext[A](viewDesc: CatalogTable)(f: => A): A = {
     val originContext = value.get()
+    val maxNestedViewDepth = if (originContext.maxNestedViewDepth == -1) {
+      // Here we start to resolve views, get `maxNestedViewDepth` from configs.
+      SQLConf.get.maxNestedViewDepth
+    } else {
+      originContext.maxNestedViewDepth
+    }
     val context = AnalysisContext(
-      catalogAndNamespace, originContext.nestedViewDepth + 1, originContext.relationCache)
+      viewDesc.viewCatalogAndNamespace,
+      originContext.nestedViewDepth + 1,
+      maxNestedViewDepth,
+      originContext.relationCache,
+      viewDesc.viewReferredTempViewNames,
+      viewDesc.viewReferredTempFunctionNames)
     set(context)
     try f finally { set(originContext) }
   }
@@ -127,29 +158,24 @@ object AnalysisContext {
  * Provides a logical query plan analyzer, which translates [[UnresolvedAttribute]]s and
  * [[UnresolvedRelation]]s into fully typed objects using information in a [[SessionCatalog]].
  */
-class Analyzer(
-    override val catalogManager: CatalogManager,
-    conf: SQLConf,
-    maxIterations: Int)
-  extends RuleExecutor[LogicalPlan] with CheckAnalysis with LookupCatalog {
+class Analyzer(override val catalogManager: CatalogManager)
+  extends RuleExecutor[LogicalPlan] with CheckAnalysis with SQLConfHelper {
 
   private val v1SessionCatalog: SessionCatalog = catalogManager.v1SessionCatalog
 
+  override protected def isPlanIntegral(plan: LogicalPlan): Boolean = {
+    !Utils.isTesting || LogicalPlanIntegrity.checkIfExprIdsAreGloballyUnique(plan)
+  }
+
   override def isView(nameParts: Seq[String]): Boolean = v1SessionCatalog.isView(nameParts)
 
   // Only for tests.
-  def this(catalog: SessionCatalog, conf: SQLConf) = {
-    this(
-      new CatalogManager(conf, FakeV2SessionCatalog, catalog),
-      conf,
-      conf.analyzerMaxIterations)
-  }
-
-  def this(catalogManager: CatalogManager, conf: SQLConf) = {
-    this(catalogManager, conf, conf.analyzerMaxIterations)
+  def this(catalog: SessionCatalog) = {
+    this(new CatalogManager(FakeV2SessionCatalog, catalog))
   }
 
   def executeAndCheck(plan: LogicalPlan, tracker: QueryPlanningTracker): LogicalPlan = {
+    if (plan.analyzed) return plan
     AnalysisHelper.markInAnalyzer {
       val analyzed = executeAndTrack(plan, tracker)
       try {
@@ -181,9 +207,9 @@ class Analyzer(
    * If the plan cannot be resolved within maxIterations, analyzer will throw exception to inform
    * user to increase the value of SQLConf.ANALYZER_MAX_ITERATIONS.
    */
-  protected val fixedPoint =
+  protected def fixedPoint =
     FixedPoint(
-      maxIterations,
+      conf.analyzerMaxIterations,
       errorOnExceed = true,
       maxIterationsSetting = SQLConf.ANALYZER_MAX_ITERATIONS.key)
 
@@ -199,24 +225,41 @@ class Analyzer(
    */
   val postHocResolutionRules: Seq[Rule[LogicalPlan]] = Nil
 
-  lazy val batches: Seq[Batch] = Seq(
-    Batch("Hints", fixedPoint,
-      new ResolveHints.ResolveJoinStrategyHints(conf),
-      new ResolveHints.ResolveCoalesceHints(conf)),
-    Batch("Simple Sanity Check", Once,
-      LookupFunctions),
+  private def typeCoercionRules(): List[Rule[LogicalPlan]] = if (conf.ansiEnabled) {
+    AnsiTypeCoercion.typeCoercionRules
+  } else {
+    TypeCoercion.typeCoercionRules
+  }
+
+  override def batches: Seq[Batch] = Seq(
     Batch("Substitution", fixedPoint,
+      // This rule optimizes `UpdateFields` expression chains so looks more like optimization rule.
+      // However, when manipulating deeply nested schema, `UpdateFields` expression tree could be
+      // very complex and make analysis impossible. Thus we need to optimize `UpdateFields` early
+      // at the beginning of analysis.
+      OptimizeUpdateFields,
       CTESubstitution,
       WindowsSubstitution,
       EliminateUnions,
-      new SubstituteUnresolvedOrdinals(conf)),
+      SubstituteUnresolvedOrdinals),
+    Batch("Disable Hints", Once,
+      new ResolveHints.DisableHints),
+    Batch("Hints", fixedPoint,
+      ResolveHints.ResolveJoinStrategyHints,
+      ResolveHints.ResolveCoalesceHints),
+    Batch("Simple Sanity Check", Once,
+      LookupFunctions),
     Batch("Resolution", fixedPoint,
-      ResolveTableValuedFunctions ::
+      ResolveTableValuedFunctions(v1SessionCatalog) ::
       ResolveNamespace(catalogManager) ::
       new ResolveCatalogs(catalogManager) ::
+      ResolveUserSpecifiedColumns ::
       ResolveInsertInto ::
       ResolveRelations ::
       ResolveTables ::
+      ResolvePartitionSpec ::
+      AddMetadataColumns ::
+      DeduplicateRelations ::
       ResolveReferences ::
       ResolveCreateNamedStruct ::
       ResolveDeserializer ::
@@ -241,28 +284,36 @@ class Analyzer(
       GlobalAggregates ::
       ResolveAggregateFunctions ::
       TimeWindowing ::
-      ResolveInlineTables(conf) ::
-      ResolveHigherOrderFunctions(v1SessionCatalog) ::
-      ResolveLambdaVariables(conf) ::
-      ResolveTimeZone(conf) ::
+      ResolveInlineTables ::
+      ResolveHigherOrderFunctions(catalogManager) ::
+      ResolveLambdaVariables ::
+      ResolveTimeZone ::
       ResolveRandomSeed ::
       ResolveBinaryArithmetic ::
-      TypeCoercion.typeCoercionRules(conf) ++
+      ResolveUnion ::
+      typeCoercionRules ++
       extendedResolutionRules : _*),
-    Batch("Post-Hoc Resolution", Once, postHocResolutionRules: _*),
+    Batch("Apply Char Padding", Once,
+      ApplyCharTypePadding),
+    Batch("Post-Hoc Resolution", Once,
+      Seq(ResolveCommandsWithIfExists) ++
+      postHocResolutionRules: _*),
     Batch("Normalize Alter Table", Once, ResolveAlterTableChanges),
     Batch("Remove Unresolved Hints", Once,
-      new ResolveHints.RemoveAllHints(conf)),
+      new ResolveHints.RemoveAllHints),
     Batch("Nondeterministic", Once,
       PullOutNondeterministic),
     Batch("UDF", Once,
-      HandleNullInputsForUDF),
+      HandleNullInputsForUDF,
+      ResolveEncodersInUDF),
     Batch("UpdateNullability", Once,
       UpdateAttributeNullability),
     Batch("Subquery", Once,
       UpdateOuterReferences),
     Batch("Cleanup", fixedPoint,
-      CleanupAliases)
+      CleanupAliases),
+    Batch("HandleAnalysisOnlyCommand", Once,
+      HandleAnalysisOnlyCommand)
   )
 
   /**
@@ -293,37 +344,58 @@ class Analyzer(
    * 2. otherwise, stays the same.
    */
   object ResolveBinaryArithmetic extends Rule[LogicalPlan] {
-    override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
-      case p: LogicalPlan => p.transformExpressionsUp {
-        case a @ Add(l, r) if a.childrenResolved => (l.dataType, r.dataType) match {
-          case (CalendarIntervalType, CalendarIntervalType) => a
-          case (DateType, CalendarIntervalType) => DateAddInterval(l, r)
-          case (_, CalendarIntervalType) => Cast(TimeAdd(l, r), l.dataType)
-          case (CalendarIntervalType, DateType) => DateAddInterval(r, l)
-          case (CalendarIntervalType, _) => Cast(TimeAdd(r, l), r.dataType)
+    override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
+      AlwaysProcess.fn, ruleId) {
+      case p: LogicalPlan => p.transformExpressionsUpWithPruning(AlwaysProcess.fn, ruleId) {
+        case a @ Add(l, r, f) if a.childrenResolved => (l.dataType, r.dataType) match {
+          case (DateType, DayTimeIntervalType) => TimeAdd(Cast(l, TimestampType), r)
+          case (DayTimeIntervalType, DateType) => TimeAdd(Cast(r, TimestampType), l)
+          case (DateType, YearMonthIntervalType) => DateAddYMInterval(l, r)
+          case (YearMonthIntervalType, DateType) => DateAddYMInterval(r, l)
+          case (TimestampType, YearMonthIntervalType) => TimestampAddYMInterval(l, r)
+          case (YearMonthIntervalType, TimestampType) => TimestampAddYMInterval(r, l)
+          case (CalendarIntervalType, CalendarIntervalType) |
+               (DayTimeIntervalType, DayTimeIntervalType) => a
+          case (DateType, CalendarIntervalType) => DateAddInterval(l, r, ansiEnabled = f)
+          case (_, CalendarIntervalType | DayTimeIntervalType) => Cast(TimeAdd(l, r), l.dataType)
+          case (CalendarIntervalType, DateType) => DateAddInterval(r, l, ansiEnabled = f)
+          case (CalendarIntervalType | DayTimeIntervalType, _) => Cast(TimeAdd(r, l), r.dataType)
           case (DateType, dt) if dt != StringType => DateAdd(l, r)
           case (dt, DateType) if dt != StringType => DateAdd(r, l)
           case _ => a
         }
-        case s @ Subtract(l, r) if s.childrenResolved => (l.dataType, r.dataType) match {
-          case (CalendarIntervalType, CalendarIntervalType) => s
+        case s @ Subtract(l, r, f) if s.childrenResolved => (l.dataType, r.dataType) match {
+          case (DateType, DayTimeIntervalType) =>
+            DatetimeSub(l, r, TimeAdd(Cast(l, TimestampType), UnaryMinus(r, f)))
+          case (DateType, YearMonthIntervalType) =>
+            DatetimeSub(l, r, DateAddYMInterval(l, UnaryMinus(r, f)))
+          case (TimestampType, YearMonthIntervalType) =>
+            DatetimeSub(l, r, TimestampAddYMInterval(l, UnaryMinus(r, f)))
+          case (CalendarIntervalType, CalendarIntervalType) |
+               (DayTimeIntervalType, DayTimeIntervalType) => s
           case (DateType, CalendarIntervalType) =>
-            DatetimeSub(l, r, DateAddInterval(l, UnaryMinus(r)))
-          case (_, CalendarIntervalType) =>
-            Cast(DatetimeSub(l, r, TimeAdd(l, UnaryMinus(r))), l.dataType)
+            DatetimeSub(l, r, DateAddInterval(l, UnaryMinus(r, f), ansiEnabled = f))
+          case (_, CalendarIntervalType | DayTimeIntervalType) =>
+            Cast(DatetimeSub(l, r, TimeAdd(l, UnaryMinus(r, f))), l.dataType)
           case (TimestampType, _) => SubtractTimestamps(l, r)
           case (_, TimestampType) => SubtractTimestamps(l, r)
           case (_, DateType) => SubtractDates(l, r)
           case (DateType, dt) if dt != StringType => DateSub(l, r)
           case _ => s
         }
-        case m @ Multiply(l, r) if m.childrenResolved => (l.dataType, r.dataType) match {
-          case (CalendarIntervalType, _) => MultiplyInterval(l, r)
-          case (_, CalendarIntervalType) => MultiplyInterval(r, l)
+        case m @ Multiply(l, r, f) if m.childrenResolved => (l.dataType, r.dataType) match {
+          case (CalendarIntervalType, _) => MultiplyInterval(l, r, f)
+          case (_, CalendarIntervalType) => MultiplyInterval(r, l, f)
+          case (YearMonthIntervalType, _) => MultiplyYMInterval(l, r)
+          case (_, YearMonthIntervalType) => MultiplyYMInterval(r, l)
+          case (DayTimeIntervalType, _) => MultiplyDTInterval(l, r)
+          case (_, DayTimeIntervalType) => MultiplyDTInterval(r, l)
           case _ => m
         }
-        case d @ Divide(l, r) if d.childrenResolved => (l.dataType, r.dataType) match {
-          case (CalendarIntervalType, _) => DivideInterval(l, r)
+        case d @ Divide(l, r, f) if d.childrenResolved => (l.dataType, r.dataType) match {
+          case (CalendarIntervalType, _) => DivideInterval(l, r, f)
+          case (YearMonthIntervalType, _) => DivideYMInterval(l, r)
+          case (DayTimeIntervalType, _) => DivideDTInterval(l, r)
           case _ => d
         }
       }
@@ -334,14 +406,13 @@ class Analyzer(
    * Substitute child plan with WindowSpecDefinitions.
    */
   object WindowsSubstitution extends Rule[LogicalPlan] {
-    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
+    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
+      AlwaysProcess.fn, ruleId) {
       // Lookup WindowSpecDefinitions. This rule works with unresolved children.
       case WithWindowDefinition(windowDefinitions, child) => child.resolveExpressions {
         case UnresolvedWindowExpression(c, WindowSpecReference(windowName)) =>
-          val errorMessage =
-            s"Window specification $windowName is not defined in the WINDOW clause."
-          val windowSpecDefinition =
-            windowDefinitions.getOrElse(windowName, failAnalysis(errorMessage))
+          val windowSpecDefinition = windowDefinitions.getOrElse(windowName,
+            throw QueryCompilationErrors.windowSpecificationNotDefinedError(windowName))
           WindowExpression(c, windowSpecDefinition)
       }
     }
@@ -371,13 +442,11 @@ class Analyzer(
     private def hasUnresolvedAlias(exprs: Seq[NamedExpression]) =
       exprs.exists(_.find(_.isInstanceOf[UnresolvedAlias]).isDefined)
 
-    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
+    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
+      AlwaysProcess.fn, ruleId) {
       case Aggregate(groups, aggs, child) if child.resolved && hasUnresolvedAlias(aggs) =>
         Aggregate(groups, assignAliases(aggs), child)
 
-      case g: GroupingSets if g.child.resolved && hasUnresolvedAlias(g.aggregations) =>
-        g.copy(aggregations = assignAliases(g.aggregations))
-
       case Pivot(groupByOpt, pivotColumn, pivotValues, aggregates, child)
         if child.resolved && groupByOpt.isDefined && hasUnresolvedAlias(groupByOpt.get) =>
         Pivot(Some(assignAliases(groupByOpt.get)), pivotColumn, pivotValues, aggregates, child)
@@ -388,40 +457,6 @@ class Analyzer(
   }
 
   object ResolveGroupingAnalytics extends Rule[LogicalPlan] {
-    /*
-     *  GROUP BY a, b, c WITH ROLLUP
-     *  is equivalent to
-     *  GROUP BY a, b, c GROUPING SETS ( (a, b, c), (a, b), (a), ( ) ).
-     *  Group Count: N + 1 (N is the number of group expressions)
-     *
-     *  We need to get all of its subsets for the rule described above, the subset is
-     *  represented as sequence of expressions.
-     */
-    def rollupExprs(exprs: Seq[Expression]): Seq[Seq[Expression]] = exprs.inits.toIndexedSeq
-
-    /*
-     *  GROUP BY a, b, c WITH CUBE
-     *  is equivalent to
-     *  GROUP BY a, b, c GROUPING SETS ( (a, b, c), (a, b), (b, c), (a, c), (a), (b), (c), ( ) ).
-     *  Group Count: 2 ^ N (N is the number of group expressions)
-     *
-     *  We need to get all of its subsets for a given GROUPBY expression, the subsets are
-     *  represented as sequence of expressions.
-     */
-    def cubeExprs(exprs: Seq[Expression]): Seq[Seq[Expression]] = {
-      // `cubeExprs0` is recursive and returns a lazy Stream. Here we call `toIndexedSeq` to
-      // materialize it and avoid serialization problems later on.
-      cubeExprs0(exprs).toIndexedSeq
-    }
-
-    def cubeExprs0(exprs: Seq[Expression]): Seq[Seq[Expression]] = exprs.toList match {
-      case x :: xs =>
-        val initial = cubeExprs0(xs)
-        initial.map(x +: _) ++ initial
-      case Nil =>
-        Seq(Seq.empty)
-    }
-
     private[analysis] def hasGroupingFunction(e: Expression): Boolean = {
       e.collectFirst {
         case g: Grouping => g
@@ -439,9 +474,7 @@ class Analyzer(
               e.groupByExprs.map(_.canonicalized) == groupByExprs.map(_.canonicalized)) {
             Alias(gid, toPrettySQL(e))()
           } else {
-            throw new AnalysisException(
-              s"Columns of grouping_id (${e.groupByExprs.mkString(",")}) does not match " +
-                s"grouping columns (${groupByExprs.mkString(",")})")
+            throw QueryCompilationErrors.groupingIDMismatchError(e, groupByExprs)
           }
         case e @ Grouping(col: Expression) =>
           val idx = groupByExprs.indexWhere(_.semanticEquals(col))
@@ -449,8 +482,7 @@ class Analyzer(
             Alias(Cast(BitwiseAnd(ShiftRight(gid, Literal(groupByExprs.length - 1 - idx)),
               Literal(1L)), ByteType), toPrettySQL(e))()
           } else {
-            throw new AnalysisException(s"Column of grouping ($col) can't be found " +
-              s"in grouping columns ${groupByExprs.mkString(",")}")
+            throw QueryCompilationErrors.groupingColInvalidError(col, groupByExprs)
           }
       }
     }
@@ -460,7 +492,7 @@ class Analyzer(
      */
     private def constructGroupByAlias(groupByExprs: Seq[Expression]): Seq[Alias] = {
       groupByExprs.map {
-        case e: NamedExpression => Alias(e, e.name)()
+        case e: NamedExpression => Alias(e, e.name)(qualifier = e.qualifier)
         case other => Alias(other, other.toString)()
       }
     }
@@ -488,7 +520,7 @@ class Analyzer(
       val groupingSetsAttributes = selectedGroupByExprs.map { groupingSetExprs =>
         groupingSetExprs.map { expr =>
           val alias = groupByAliases.find(_.child.semanticEquals(expr)).getOrElse(
-            failAnalysis(s"$expr doesn't show up in the GROUP BY list $groupByAliases"))
+            throw QueryCompilationErrors.selectExprNotInGroupByError(expr, groupByAliases))
           // Map alias to expanded attribute.
           expandedAttributes.find(_.semanticEquals(alias.toAttribute)).getOrElse(
             alias.toAttribute)
@@ -533,28 +565,6 @@ class Analyzer(
       }.asInstanceOf[NamedExpression]
     }
 
-    private def getFinalGroupByExpressions(
-        selectedGroupByExprs: Seq[Seq[Expression]],
-        groupByExprs: Seq[Expression]): Seq[Expression] = {
-      // In case of ANSI-SQL compliant syntax for GROUPING SETS, groupByExprs is optional and
-      // can be null. In such case, we derive the groupByExprs from the user supplied values for
-      // grouping sets.
-      if (groupByExprs == Nil) {
-        selectedGroupByExprs.flatten.foldLeft(Seq.empty[Expression]) { (result, currentExpr) =>
-          // Only unique expressions are included in the group by expressions and is determined
-          // based on their semantic equality. Example. grouping sets ((a * b), (b * a)) results
-          // in grouping expression (a * b)
-          if (result.find(_.semanticEquals(currentExpr)).isDefined) {
-            result
-          } else {
-            result :+ currentExpr
-          }
-        }
-      } else {
-        groupByExprs
-      }
-    }
-
     /*
      * Construct [[Aggregate]] operator from Cube/Rollup/GroupingSets.
      */
@@ -563,25 +573,23 @@ class Analyzer(
         groupByExprs: Seq[Expression],
         aggregationExprs: Seq[NamedExpression],
         child: LogicalPlan): LogicalPlan = {
-      val finalGroupByExpressions = getFinalGroupByExpressions(selectedGroupByExprs, groupByExprs)
 
-      if (finalGroupByExpressions.size > GroupingID.dataType.defaultSize * 8) {
-        throw new AnalysisException(
-          s"Grouping sets size cannot be greater than ${GroupingID.dataType.defaultSize * 8}")
+      if (groupByExprs.size > GroupingID.dataType.defaultSize * 8) {
+        throw QueryCompilationErrors.groupingSizeTooLargeError(GroupingID.dataType.defaultSize * 8)
       }
 
       // Expand works by setting grouping expressions to null as determined by the
       // `selectedGroupByExprs`. To prevent these null values from being used in an aggregate
       // instead of the original value we need to create new aliases for all group by expressions
       // that will only be used for the intended purpose.
-      val groupByAliases = constructGroupByAlias(finalGroupByExpressions)
+      val groupByAliases = constructGroupByAlias(groupByExprs)
 
       val gid = AttributeReference(VirtualColumn.groupingIdName, GroupingID.dataType, false)()
       val expand = constructExpand(selectedGroupByExprs, child, groupByAliases, gid)
       val groupingAttrs = expand.output.drop(child.output.length)
 
       val aggregations = constructAggregateExprs(
-        finalGroupByExpressions, aggregationExprs, groupByAliases, groupingAttrs, gid)
+        groupByExprs, aggregationExprs, groupByAliases, groupingAttrs, gid)
 
       Aggregate(groupingAttrs, aggregations, expand)
     }
@@ -593,27 +601,22 @@ class Analyzer(
           val gid = a.groupingExpressions.last
           if (!gid.isInstanceOf[AttributeReference]
             || gid.asInstanceOf[AttributeReference].name != VirtualColumn.groupingIdName) {
-            failAnalysis(s"grouping()/grouping_id() can only be used with GroupingSets/Cube/Rollup")
+            throw QueryCompilationErrors.groupingMustWithGroupingSetsOrCubeOrRollupError()
           }
           a.groupingExpressions.take(a.groupingExpressions.length - 1)
       }.getOrElse {
-        failAnalysis(s"grouping()/grouping_id() can only be used with GroupingSets/Cube/Rollup")
+        throw QueryCompilationErrors.groupingMustWithGroupingSetsOrCubeOrRollupError()
       }
     }
 
-    private def tryResolveHavingCondition(h: UnresolvedHaving): LogicalPlan = {
-      val aggForResolving = h.child match {
-        // For CUBE/ROLLUP expressions, to avoid resolving repeatedly, here we delete them from
-        // groupingExpressions for condition resolving.
-        case a @ Aggregate(Seq(c @ Cube(groupByExprs)), _, _) =>
-          a.copy(groupingExpressions = groupByExprs)
-        case a @ Aggregate(Seq(r @ Rollup(groupByExprs)), _, _) =>
-          a.copy(groupingExpressions = groupByExprs)
-        case g: GroupingSets =>
-          Aggregate(
-            getFinalGroupByExpressions(g.selectedGroupByExprs, g.groupByExprs),
-            g.aggregations, g.child)
-      }
+    private def tryResolveHavingCondition(
+        h: UnresolvedHaving,
+        aggregate: Aggregate,
+        selectedGroupByExprs: Seq[Seq[Expression]],
+        groupByExprs: Seq[Expression]): LogicalPlan = {
+      // For CUBE/ROLLUP expressions, to avoid resolving repeatedly, here we delete them from
+      // groupingExpressions for condition resolving.
+      val aggForResolving = aggregate.copy(groupingExpressions = groupByExprs)
       // Try resolving the condition of the filter as though it is in the aggregate clause
       val resolvedInfo =
         ResolveAggregateFunctions.resolveFilterCondInAggregate(h.havingCondition, aggForResolving)
@@ -621,17 +624,8 @@ class Analyzer(
       // Push the aggregate expressions into the aggregate (if any).
       if (resolvedInfo.nonEmpty) {
         val (extraAggExprs, resolvedHavingCond) = resolvedInfo.get
-        val newChild = h.child match {
-          case Aggregate(Seq(c @ Cube(groupByExprs)), aggregateExpressions, child) =>
-            constructAggregate(
-              cubeExprs(groupByExprs), groupByExprs, aggregateExpressions ++ extraAggExprs, child)
-          case Aggregate(Seq(r @ Rollup(groupByExprs)), aggregateExpressions, child) =>
-            constructAggregate(
-              rollupExprs(groupByExprs), groupByExprs, aggregateExpressions ++ extraAggExprs, child)
-          case x: GroupingSets =>
-            constructAggregate(
-              x.selectedGroupByExprs, x.groupByExprs, x.aggregations ++ extraAggExprs, x.child)
-        }
+        val newChild = constructAggregate(selectedGroupByExprs, groupByExprs,
+          aggregate.aggregateExpressions ++ extraAggExprs, aggregate.child)
 
         // Since the exprId of extraAggExprs will be changed in the constructed aggregate, and the
         // aggregateExpressions keeps the input order. So here we build an exprMap to resolve the
@@ -652,31 +646,19 @@ class Analyzer(
     // This require transformDown to resolve having condition when generating aggregate node for
     // CUBE/ROLLUP/GROUPING SETS. This also replace grouping()/grouping_id() in resolved
     // Filter/Sort.
-    def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperatorsDown {
-      case h @ UnresolvedHaving(
-          _, agg @ Aggregate(Seq(c @ Cube(groupByExprs)), aggregateExpressions, _))
-          if agg.childrenResolved && (groupByExprs ++ aggregateExpressions).forall(_.resolved) =>
-        tryResolveHavingCondition(h)
-      case h @ UnresolvedHaving(
-          _, agg @ Aggregate(Seq(r @ Rollup(groupByExprs)), aggregateExpressions, _))
-          if agg.childrenResolved && (groupByExprs ++ aggregateExpressions).forall(_.resolved) =>
-        tryResolveHavingCondition(h)
-      case h @ UnresolvedHaving(_, g: GroupingSets)
-          if g.childrenResolved && g.expressions.forall(_.resolved) =>
-        tryResolveHavingCondition(h)
+    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsDownWithPruning(
+      AlwaysProcess.fn, ruleId) {
+      case h @ UnresolvedHaving(_, agg @ Aggregate(
+        GroupingAnalytics(selectedGroupByExprs, groupByExprs), aggExprs, _))
+        if agg.childrenResolved && aggExprs.forall(_.resolved) =>
+        tryResolveHavingCondition(h, agg, selectedGroupByExprs, groupByExprs)
 
       case a if !a.childrenResolved => a // be sure all of the children are resolved.
 
       // Ensure group by expressions and aggregate expressions have been resolved.
-      case Aggregate(Seq(c @ Cube(groupByExprs)), aggregateExpressions, child)
-        if (groupByExprs ++ aggregateExpressions).forall(_.resolved) =>
-        constructAggregate(cubeExprs(groupByExprs), groupByExprs, aggregateExpressions, child)
-      case Aggregate(Seq(r @ Rollup(groupByExprs)), aggregateExpressions, child)
-        if (groupByExprs ++ aggregateExpressions).forall(_.resolved) =>
-        constructAggregate(rollupExprs(groupByExprs), groupByExprs, aggregateExpressions, child)
-      // Ensure all the expressions have been resolved.
-      case x: GroupingSets if x.expressions.forall(_.resolved) =>
-        constructAggregate(x.selectedGroupByExprs, x.groupByExprs, x.aggregations, x.child)
+      case Aggregate(GroupingAnalytics(selectedGroupByExprs, groupByExprs), aggExprs, child)
+        if aggExprs.forall(_.resolved) =>
+        constructAggregate(selectedGroupByExprs, groupByExprs, aggExprs, child)
 
       // We should make sure all expressions in condition have been resolved.
       case f @ Filter(cond, child) if hasGroupingFunction(cond) && cond.resolved =>
@@ -697,14 +679,14 @@ class Analyzer(
   }
 
   object ResolvePivot extends Rule[LogicalPlan] {
-    def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
+    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsWithPruning(
+      AlwaysProcess.fn, ruleId) {
       case p: Pivot if !p.childrenResolved || !p.aggregates.forall(_.resolved)
         || (p.groupByExprsOpt.isDefined && !p.groupByExprsOpt.get.forall(_.resolved))
         || !p.pivotColumn.resolved || !p.pivotValues.forall(_.resolved) => p
       case Pivot(groupByExprsOpt, pivotColumn, pivotValues, aggregates, child) =>
         if (!RowOrdering.isOrderable(pivotColumn.dataType)) {
-          throw new AnalysisException(
-            s"Invalid pivot column '${pivotColumn}'. Pivot columns must be comparable.")
+          throw QueryCompilationErrors.unorderablePivotColError(pivotColumn)
         }
         // Check all aggregate expressions.
         aggregates.foreach(checkValidAggregateExpression)
@@ -715,13 +697,10 @@ class Analyzer(
             case _ => value.foldable
           }
           if (!foldable) {
-            throw new AnalysisException(
-              s"Literal expressions required for pivot values, found '$value'")
+            throw QueryCompilationErrors.nonLiteralPivotValError(value)
           }
           if (!Cast.canCast(value.dataType, pivotColumn.dataType)) {
-            throw new AnalysisException(s"Invalid pivot value '$value': " +
-              s"value data type ${value.dataType.simpleString} does not match " +
-              s"pivot column data type ${pivotColumn.dataType.catalogString}")
+            throw QueryCompilationErrors.pivotValDataTypeMismatchError(value, pivotColumn)
           }
           Cast(value, pivotColumn.dataType, Some(conf.sessionLocalTimeZone)).eval(EmptyRow)
         }
@@ -788,9 +767,9 @@ class Analyzer(
                 // AggregateFunction's with the exception of First and Last in their default mode
                 // (which we handle) and possibly some Hive UDAF's.
                 case First(expr, _) =>
-                  First(ifExpr(expr), Literal(true))
+                  First(ifExpr(expr), true)
                 case Last(expr, _) =>
-                  Last(ifExpr(expr), Literal(true))
+                  Last(ifExpr(expr), true)
                 case a: AggregateFunction =>
                   a.withNewChildren(a.children.map(ifExpr))
               }.transform {
@@ -811,11 +790,9 @@ class Analyzer(
     private def checkValidAggregateExpression(expr: Expression): Unit = expr match {
       case _: AggregateExpression => // OK and leave the argument check to CheckAnalysis.
       case expr: PythonUDF if PythonUDF.isGroupedAggPandasUDF(expr) =>
-        failAnalysis("Pandas UDF aggregate expressions are currently not supported in pivot.")
+        throw QueryCompilationErrors.pandasUDFAggregateNotSupportedInPivotError()
       case e: Attribute =>
-        failAnalysis(
-          s"Aggregate expression required for pivot, but '${e.sql}' " +
-          s"did not appear in any aggregate function.")
+        throw QueryCompilationErrors.aggregateExpressionRequiredForPivotError(e.sql)
       case e => e.children.foreach(checkValidAggregateExpression)
     }
   }
@@ -823,10 +800,14 @@ class Analyzer(
   case class ResolveNamespace(catalogManager: CatalogManager)
     extends Rule[LogicalPlan] with LookupCatalog {
     def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
-      case s @ ShowTables(UnresolvedNamespace(Seq()), _) =>
+      case s @ ShowTables(UnresolvedNamespace(Seq()), _, _) =>
         s.copy(namespace = ResolvedNamespace(currentCatalog, catalogManager.currentNamespace))
-      case s @ ShowViews(UnresolvedNamespace(Seq()), _) =>
+      case s @ ShowTableExtended(UnresolvedNamespace(Seq()), _, _, _) =>
         s.copy(namespace = ResolvedNamespace(currentCatalog, catalogManager.currentNamespace))
+      case s @ ShowViews(UnresolvedNamespace(Seq()), _, _) =>
+        s.copy(namespace = ResolvedNamespace(currentCatalog, catalogManager.currentNamespace))
+      case a @ AnalyzeTables(UnresolvedNamespace(Seq()), _) =>
+        a.copy(namespace = ResolvedNamespace(currentCatalog, catalogManager.currentNamespace))
       case UnresolvedNamespace(Seq()) =>
         ResolvedNamespace(currentCatalog, Seq.empty[String])
       case UnresolvedNamespace(CatalogAndNamespace(catalog, ns)) =>
@@ -835,44 +816,100 @@ class Analyzer(
   }
 
   private def isResolvingView: Boolean = AnalysisContext.get.catalogAndNamespace.nonEmpty
+  private def isReferredTempViewName(nameParts: Seq[String]): Boolean = {
+    AnalysisContext.get.referredTempViewNames.exists { n =>
+      (n.length == nameParts.length) && n.zip(nameParts).forall {
+        case (a, b) => resolver(a, b)
+      }
+    }
+  }
+
+  private def unwrapRelationPlan(plan: LogicalPlan): LogicalPlan = {
+    EliminateSubqueryAliases(plan) match {
+      case v: View if v.isTempViewStoringAnalyzedPlan => v.child
+      case other => other
+    }
+  }
 
   /**
    * Resolve relations to temp views. This is not an actual rule, and is called by
    * [[ResolveTables]] and [[ResolveRelations]].
    */
   object ResolveTempViews extends Rule[LogicalPlan] {
-    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
-      case u @ UnresolvedRelation(ident) =>
-        lookupTempView(ident).getOrElse(u)
-      case i @ InsertIntoStatement(UnresolvedRelation(ident), _, _, _, _) =>
-        lookupTempView(ident)
+    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
+      AlwaysProcess.fn, ruleId) {
+      case u @ UnresolvedRelation(ident, _, isStreaming) =>
+        lookupAndResolveTempView(ident, isStreaming).getOrElse(u)
+      case i @ InsertIntoStatement(UnresolvedRelation(ident, _, false), _, _, _, _, _) =>
+        lookupAndResolveTempView(ident)
           .map(view => i.copy(table = view))
           .getOrElse(i)
-      case u @ UnresolvedTable(ident) =>
+      // TODO (SPARK-27484): handle streaming write commands when we have them.
+      case write: V2WriteCommand =>
+        write.table match {
+          case UnresolvedRelation(ident, _, false) =>
+            lookupAndResolveTempView(ident).map(unwrapRelationPlan).map {
+              case r: DataSourceV2Relation => write.withNewTable(r)
+              case _ => throw QueryCompilationErrors.writeIntoTempViewNotAllowedError(ident.quoted)
+            }.getOrElse(write)
+          case _ => write
+        }
+      case u @ UnresolvedTable(ident, cmd, _) =>
         lookupTempView(ident).foreach { _ =>
-          u.failAnalysis(s"${ident.quoted} is a temp view not table.")
+          throw QueryCompilationErrors.expectTableNotViewError(
+            ResolvedView(ident.asIdentifier, isTemp = true), cmd, u.relationTypeMismatchHint, u)
         }
         u
-      case u @ UnresolvedTableOrView(ident) =>
-        lookupTempView(ident).map(_ => ResolvedView(ident.asIdentifier)).getOrElse(u)
+      case u @ UnresolvedView(ident, cmd, allowTemp, _) =>
+        lookupTempView(ident).map { _ =>
+          if (!allowTemp) {
+            u.failAnalysis(s"${ident.quoted} is a temp view. '$cmd' expects a permanent view.")
+          }
+          ResolvedView(ident.asIdentifier, isTemp = true)
+        }
+        .getOrElse(u)
+      case u @ UnresolvedTableOrView(ident, cmd, allowTempView) =>
+        lookupTempView(ident)
+          .map { _ =>
+            if (!allowTempView) {
+              throw QueryCompilationErrors.expectTableOrPermanentViewNotTempViewError(
+                ident.quoted, cmd, u)
+            }
+            ResolvedView(ident.asIdentifier, isTemp = true)
+          }
+          .getOrElse(u)
     }
 
-    def lookupTempView(identifier: Seq[String]): Option[LogicalPlan] = {
+    private def lookupTempView(
+        identifier: Seq[String],
+        isStreaming: Boolean = false): Option[LogicalPlan] = {
       // Permanent View can't refer to temp views, no need to lookup at all.
-      if (isResolvingView) return None
+      if (isResolvingView && !isReferredTempViewName(identifier)) return None
 
-      identifier match {
+      val tmpView = identifier match {
         case Seq(part1) => v1SessionCatalog.lookupTempView(part1)
         case Seq(part1, part2) => v1SessionCatalog.lookupGlobalTempView(part1, part2)
         case _ => None
       }
+
+      if (isStreaming && tmpView.nonEmpty && !tmpView.get.isStreaming) {
+        throw QueryCompilationErrors.readNonStreamingTempViewError(identifier.quoted)
+      }
+      tmpView
+    }
+
+    private def lookupAndResolveTempView(
+        identifier: Seq[String],
+        isStreaming: Boolean = false): Option[LogicalPlan] = {
+      lookupTempView(identifier, isStreaming).map(ResolveRelations.resolveViews)
     }
   }
 
-  // If we are resolving relations insides views, we need to expand single-part relation names with
-  // the current catalog and namespace of when the view was created.
-  private def expandRelationName(nameParts: Seq[String]): Seq[String] = {
-    if (!isResolvingView) return nameParts
+  // If we are resolving database objects (relations, functions, etc.) insides views, we may need to
+  // expand single or multi-part identifiers with the current catalog and namespace of when the
+  // view was created.
+  private def expandIdentifier(nameParts: Seq[String]): Seq[String] = {
+    if (!isResolvingView || isReferredTempViewName(nameParts)) return nameParts
 
     if (nameParts.length == 1) {
       AnalysisContext.get.catalogAndNamespace :+ nameParts.head
@@ -883,35 +920,123 @@ class Analyzer(
     }
   }
 
+  /**
+   * Adds metadata columns to output for child relations when nodes are missing resolved attributes.
+   *
+   * References to metadata columns are resolved using columns from [[LogicalPlan.metadataOutput]],
+   * but the relation's output does not include the metadata columns until the relation is replaced.
+   * Unless this rule adds metadata to the relation's output, the analyzer will detect that nothing
+   * produces the columns.
+   *
+   * This rule only adds metadata columns when a node is resolved but is missing input from its
+   * children. This ensures that metadata columns are not added to the plan unless they are used. By
+   * checking only resolved nodes, this ensures that * expansion is already done so that metadata
+   * columns are not accidentally selected by *. This rule resolves operators downwards to avoid
+   * projecting away metadata columns prematurely.
+   */
+  object AddMetadataColumns extends Rule[LogicalPlan] {
+
+    import org.apache.spark.sql.catalyst.util._
+
+    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsDownWithPruning(
+      AlwaysProcess.fn, ruleId) {
+      // Add metadata output to all node types
+      case node if node.children.nonEmpty && node.resolved && hasMetadataCol(node) =>
+        val inputAttrs = AttributeSet(node.children.flatMap(_.output))
+        val metaCols = getMetadataAttributes(node).filterNot(inputAttrs.contains)
+        if (metaCols.isEmpty) {
+          node
+        } else {
+          val newNode = addMetadataCol(node)
+          // We should not change the output schema of the plan. We should project away the extra
+          // metadata columns if necessary.
+          if (newNode.sameOutput(node)) {
+            newNode
+          } else {
+            Project(node.output, newNode)
+          }
+        }
+    }
+
+    private def getMetadataAttributes(plan: LogicalPlan): Seq[Attribute] = {
+      plan.expressions.flatMap(_.collect {
+        case a: Attribute if a.isMetadataCol => a
+        case a: Attribute
+          if plan.children.exists(c => c.metadataOutput.exists(_.exprId == a.exprId)) =>
+          plan.children.collectFirst {
+            case c if c.metadataOutput.exists(_.exprId == a.exprId) =>
+              c.metadataOutput.find(_.exprId == a.exprId).get
+          }.get
+      })
+    }
+
+    private def hasMetadataCol(plan: LogicalPlan): Boolean = {
+      plan.expressions.exists(_.find {
+        case a: Attribute =>
+          // If an attribute is resolved before being labeled as metadata
+          // (i.e. from the originating Dataset), we check with expression ID
+          a.isMetadataCol ||
+            plan.children.exists(c => c.metadataOutput.exists(_.exprId == a.exprId))
+        case _ => false
+      }.isDefined)
+    }
+
+    private def addMetadataCol(plan: LogicalPlan): LogicalPlan = plan match {
+      case r: DataSourceV2Relation => r.withMetadataColumns()
+      case p: Project =>
+        p.copy(
+          projectList = p.metadataOutput ++ p.projectList,
+          child = addMetadataCol(p.child))
+      case _ => plan.withNewChildren(plan.children.map(addMetadataCol))
+    }
+  }
+
   /**
    * Resolve table relations with concrete relations from v2 catalog.
    *
    * [[ResolveRelations]] still resolves v1 tables.
    */
   object ResolveTables extends Rule[LogicalPlan] {
-    def apply(plan: LogicalPlan): LogicalPlan = ResolveTempViews(plan).resolveOperatorsUp {
+    def apply(plan: LogicalPlan): LogicalPlan = ResolveTempViews(plan).
+      resolveOperatorsUpWithPruning(AlwaysProcess.fn, ruleId) {
       case u: UnresolvedRelation =>
-        lookupV2Relation(u.multipartIdentifier)
-          .map { rel =>
-            val ident = rel.identifier.get
-            SubqueryAlias(rel.catalog.get.name +: ident.namespace :+ ident.name, rel)
+        lookupV2Relation(u.multipartIdentifier, u.options, u.isStreaming)
+          .map { relation =>
+            val (catalog, ident) = relation match {
+              case ds: DataSourceV2Relation => (ds.catalog, ds.identifier.get)
+              case s: StreamingRelationV2 => (s.catalog, s.identifier.get)
+            }
+            SubqueryAlias(catalog.get.name +: ident.namespace :+ ident.name, relation)
           }.getOrElse(u)
 
-      case u @ UnresolvedTable(NonSessionCatalogAndIdentifier(catalog, ident)) =>
+      case u @ UnresolvedTable(NonSessionCatalogAndIdentifier(catalog, ident), _, _) =>
         CatalogV2Util.loadTable(catalog, ident)
-          .map(ResolvedTable(catalog.asTableCatalog, ident, _))
+          .map(table => ResolvedTable.create(catalog.asTableCatalog, ident, table))
           .getOrElse(u)
 
-      case u @ UnresolvedTableOrView(NonSessionCatalogAndIdentifier(catalog, ident)) =>
+      case u @ UnresolvedTableOrView(NonSessionCatalogAndIdentifier(catalog, ident), _, _) =>
         CatalogV2Util.loadTable(catalog, ident)
-          .map(ResolvedTable(catalog.asTableCatalog, ident, _))
+          .map(table => ResolvedTable.create(catalog.asTableCatalog, ident, table))
           .getOrElse(u)
 
-      case i @ InsertIntoStatement(u: UnresolvedRelation, _, _, _, _) if i.query.resolved =>
-        lookupV2Relation(u.multipartIdentifier)
+      case i @ InsertIntoStatement(u @ UnresolvedRelation(_, _, false), _, _, _, _, _)
+          if i.query.resolved =>
+        lookupV2Relation(u.multipartIdentifier, u.options, false)
           .map(v2Relation => i.copy(table = v2Relation))
           .getOrElse(i)
 
+      // TODO (SPARK-27484): handle streaming write commands when we have them.
+      case write: V2WriteCommand =>
+        write.table match {
+          case u: UnresolvedRelation if !u.isStreaming =>
+            lookupV2Relation(u.multipartIdentifier, u.options, false).map {
+              case r: DataSourceV2Relation => write.withNewTable(r)
+              case other => throw new IllegalStateException(
+                "[BUG] unexpected plan returned by `lookupV2Relation`: " + other)
+            }.getOrElse(write)
+          case _ => write
+        }
+
       case alter @ AlterTable(_, _, u: UnresolvedV2Relation, _) =>
         CatalogV2Util.loadRelation(u.catalog, u.tableName)
           .map(rel => alter.copy(table = rel))
@@ -924,12 +1049,20 @@ class Analyzer(
     /**
      * Performs the lookup of DataSourceV2 Tables from v2 catalog.
      */
-    private def lookupV2Relation(identifier: Seq[String]): Option[DataSourceV2Relation] =
-      expandRelationName(identifier) match {
+    private def lookupV2Relation(
+        identifier: Seq[String],
+        options: CaseInsensitiveStringMap,
+        isStreaming: Boolean): Option[LogicalPlan] =
+      expandIdentifier(identifier) match {
         case NonSessionCatalogAndIdentifier(catalog, ident) =>
           CatalogV2Util.loadTable(catalog, ident) match {
             case Some(table) =>
-              Some(DataSourceV2Relation.create(table, Some(catalog), Some(ident)))
+              if (isStreaming) {
+                Some(StreamingRelationV2(None, table.name, table, options,
+                  table.schema.toAttributes, Some(catalog), Some(ident), None))
+              } else {
+                Some(DataSourceV2Relation.create(table, Some(catalog), Some(ident), options))
+              }
             case None => None
           }
         case _ => None
@@ -946,63 +1079,101 @@ class Analyzer(
     // look at `AnalysisContext.catalogAndNamespace` when resolving relations with single-part name.
     // If `AnalysisContext.catalogAndNamespace` is non-empty, analyzer will expand single-part names
     // with it, instead of current catalog and namespace.
-    private def resolveViews(plan: LogicalPlan): LogicalPlan = plan match {
+    def resolveViews(plan: LogicalPlan): LogicalPlan = plan match {
       // The view's child should be a logical plan parsed from the `desc.viewText`, the variable
       // `viewText` should be defined, or else we throw an error on the generation of the View
       // operator.
-      case view @ View(desc, _, child) if !child.resolved =>
+      case view @ View(desc, isTempView, child) if !child.resolved =>
         // Resolve all the UnresolvedRelations and Views in the child.
-        val newChild = AnalysisContext.withAnalysisContext(desc.viewCatalogAndNamespace) {
-          if (AnalysisContext.get.nestedViewDepth > conf.maxNestedViewDepth) {
-            view.failAnalysis(s"The depth of view ${desc.identifier} exceeds the maximum " +
-              s"view resolution depth (${conf.maxNestedViewDepth}). Analysis is aborted to " +
-              s"avoid errors. Increase the value of ${SQLConf.MAX_NESTED_VIEW_DEPTH.key} to work " +
-              "around this.")
+        val newChild = AnalysisContext.withAnalysisContext(desc) {
+          val nestedViewDepth = AnalysisContext.get.nestedViewDepth
+          val maxNestedViewDepth = AnalysisContext.get.maxNestedViewDepth
+          if (nestedViewDepth > maxNestedViewDepth) {
+            throw QueryCompilationErrors.viewDepthExceedsMaxResolutionDepthError(
+              desc.identifier, maxNestedViewDepth, view)
+          }
+          SQLConf.withExistingConf(View.effectiveSQLConf(desc.viewSQLConfigs, isTempView)) {
+            executeSameContext(child)
           }
-          executeSameContext(child)
         }
+        // Fail the analysis eagerly because outside AnalysisContext, the unresolved operators
+        // inside a view maybe resolved incorrectly.
+        checkAnalysis(newChild)
         view.copy(child = newChild)
       case p @ SubqueryAlias(_, view: View) =>
         p.copy(child = resolveViews(view))
       case _ => plan
     }
 
-    def apply(plan: LogicalPlan): LogicalPlan = ResolveTempViews(plan).resolveOperatorsUp {
-      case i @ InsertIntoStatement(table, _, _, _, _) if i.query.resolved =>
+    def apply(plan: LogicalPlan): LogicalPlan = ResolveTempViews(plan).
+      resolveOperatorsUpWithPruning(AlwaysProcess.fn, ruleId) {
+      case i @ InsertIntoStatement(table, _, _, _, _, _) if i.query.resolved =>
         val relation = table match {
-          case u: UnresolvedRelation =>
-            lookupRelation(u.multipartIdentifier).getOrElse(u)
+          case u @ UnresolvedRelation(_, _, false) =>
+            lookupRelation(u.multipartIdentifier, u.options, false).getOrElse(u)
           case other => other
         }
 
-        EliminateSubqueryAliases(relation) match {
+        // Inserting into a file-based temporary view is allowed.
+        // (e.g., spark.read.parquet("path").createOrReplaceTempView("t").
+        // Thus, we need to look at the raw plan if `relation` is a temporary view.
+        unwrapRelationPlan(relation) match {
           case v: View =>
-            table.failAnalysis(s"Inserting into a view is not allowed. View: ${v.desc.identifier}.")
+            throw QueryCompilationErrors.insertIntoViewNotAllowedError(v.desc.identifier, table)
           case other => i.copy(table = other)
         }
 
+      // TODO (SPARK-27484): handle streaming write commands when we have them.
+      case write: V2WriteCommand =>
+        write.table match {
+          case u: UnresolvedRelation if !u.isStreaming =>
+            lookupRelation(u.multipartIdentifier, u.options, false)
+              .map(EliminateSubqueryAliases(_))
+              .map {
+                case v: View => throw QueryCompilationErrors.writeIntoViewNotAllowedError(
+                  v.desc.identifier, write)
+                case u: UnresolvedCatalogRelation =>
+                  throw QueryCompilationErrors.writeIntoV1TableNotAllowedError(
+                    u.tableMeta.identifier, write)
+                case r: DataSourceV2Relation => write.withNewTable(r)
+                case other => throw new IllegalStateException(
+                  "[BUG] unexpected plan returned by `lookupRelation`: " + other)
+              }.getOrElse(write)
+          case _ => write
+        }
+
       case u: UnresolvedRelation =>
-        lookupRelation(u.multipartIdentifier).map(resolveViews).getOrElse(u)
+        lookupRelation(u.multipartIdentifier, u.options, u.isStreaming)
+          .map(resolveViews).getOrElse(u)
 
-      case u @ UnresolvedTable(identifier) =>
+      case u @ UnresolvedTable(identifier, cmd, relationTypeMismatchHint) =>
         lookupTableOrView(identifier).map {
           case v: ResolvedView =>
-            u.failAnalysis(s"${v.identifier.quoted} is a view not table.")
+            throw QueryCompilationErrors.expectTableNotViewError(
+              v, cmd, relationTypeMismatchHint, u)
           case table => table
         }.getOrElse(u)
 
-      case u @ UnresolvedTableOrView(identifier) =>
+      case u @ UnresolvedView(identifier, cmd, _, relationTypeMismatchHint) =>
+        lookupTableOrView(identifier).map {
+          case t: ResolvedTable =>
+            throw QueryCompilationErrors.expectViewNotTableError(
+              t, cmd, relationTypeMismatchHint, u)
+          case view => view
+        }.getOrElse(u)
+
+      case u @ UnresolvedTableOrView(identifier, _, _) =>
         lookupTableOrView(identifier).getOrElse(u)
     }
 
     private def lookupTableOrView(identifier: Seq[String]): Option[LogicalPlan] = {
-      expandRelationName(identifier) match {
+      expandIdentifier(identifier) match {
         case SessionCatalogAndIdentifier(catalog, ident) =>
           CatalogV2Util.loadTable(catalog, ident).map {
             case v1Table: V1Table if v1Table.v1Table.tableType == CatalogTableType.VIEW =>
-              ResolvedView(ident)
+              ResolvedView(ident, isTemp = false)
             case table =>
-              ResolvedTable(catalog.asTableCatalog, ident, table)
+              ResolvedTable.create(catalog.asTableCatalog, ident, table)
           }
         case _ => None
       }
@@ -1012,44 +1183,76 @@ class Analyzer(
     // 1) If the resolved catalog is not session catalog, return None.
     // 2) If a relation is not found in the catalog, return None.
     // 3) If a v1 table is found, create a v1 relation. Otherwise, create a v2 relation.
-    private def lookupRelation(identifier: Seq[String]): Option[LogicalPlan] = {
-      expandRelationName(identifier) match {
+    private def lookupRelation(
+        identifier: Seq[String],
+        options: CaseInsensitiveStringMap,
+        isStreaming: Boolean): Option[LogicalPlan] = {
+      expandIdentifier(identifier) match {
         case SessionCatalogAndIdentifier(catalog, ident) =>
-          def loaded = CatalogV2Util.loadTable(catalog, ident).map {
+          lazy val loaded = CatalogV2Util.loadTable(catalog, ident).map {
             case v1Table: V1Table =>
-              v1SessionCatalog.getRelation(v1Table.v1Table)
+              if (isStreaming) {
+                if (v1Table.v1Table.tableType == CatalogTableType.VIEW) {
+                  throw QueryCompilationErrors.permanentViewNotSupportedByStreamingReadingAPIError(
+                    identifier.quoted)
+                }
+                SubqueryAlias(
+                  catalog.name +: ident.asMultipartIdentifier,
+                  UnresolvedCatalogRelation(v1Table.v1Table, options, isStreaming = true))
+              } else {
+                v1SessionCatalog.getRelation(v1Table.v1Table, options)
+              }
             case table =>
-              SubqueryAlias(
-                catalog.name +: ident.asMultipartIdentifier,
-                DataSourceV2Relation.create(table, Some(catalog), Some(ident)))
+              if (isStreaming) {
+                val v1Fallback = table match {
+                  case withFallback: V2TableWithV1Fallback =>
+                    Some(UnresolvedCatalogRelation(withFallback.v1Table, isStreaming = true))
+                  case _ => None
+                }
+                SubqueryAlias(
+                  catalog.name +: ident.asMultipartIdentifier,
+                  StreamingRelationV2(None, table.name, table, options, table.schema.toAttributes,
+                    Some(catalog), Some(ident), v1Fallback))
+              } else {
+                SubqueryAlias(
+                  catalog.name +: ident.asMultipartIdentifier,
+                  DataSourceV2Relation.create(table, Some(catalog), Some(ident), options))
+              }
           }
           val key = catalog.name +: ident.namespace :+ ident.name
-          Option(AnalysisContext.get.relationCache.getOrElseUpdate(key, loaded.orNull))
+          AnalysisContext.get.relationCache.get(key).map(_.transform {
+            case multi: MultiInstanceRelation =>
+              val newRelation = multi.newInstance()
+              newRelation.copyTagsFrom(multi)
+              newRelation
+          }).orElse {
+            loaded.foreach(AnalysisContext.get.relationCache.update(key, _))
+            loaded
+          }
         case _ => None
       }
     }
   }
 
   object ResolveInsertInto extends Rule[LogicalPlan] {
-    override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
-      case i @ InsertIntoStatement(r: DataSourceV2Relation, _, _, _, _) if i.query.resolved =>
+    override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsWithPruning(
+      AlwaysProcess.fn, ruleId) {
+      case i @ InsertIntoStatement(r: DataSourceV2Relation, _, _, _, _, _)
+          if i.query.resolved && i.userSpecifiedCols.isEmpty =>
         // ifPartitionNotExists is append with validation, but validation is not supported
         if (i.ifPartitionNotExists) {
-          throw new AnalysisException(
-            s"Cannot write, IF NOT EXISTS is not supported for table: ${r.table.name}")
+          throw QueryCompilationErrors.unsupportedIfNotExistsError(r.table.name)
         }
 
         val partCols = partitionColumnNames(r.table)
         validatePartitionSpec(partCols, i.partitionSpec)
 
-        val staticPartitions = i.partitionSpec.filter(_._2.isDefined).mapValues(_.get)
+        val staticPartitions = i.partitionSpec.filter(_._2.isDefined).mapValues(_.get).toMap
         val query = addStaticPartitionColumns(r, i.query, staticPartitions)
-        val dynamicPartitionOverwrite = partCols.size > staticPartitions.size &&
-          conf.partitionOverwriteMode == PartitionOverwriteMode.DYNAMIC
 
         if (!i.overwrite) {
           AppendData.byPosition(r, query)
-        } else if (dynamicPartitionOverwrite) {
+        } else if (conf.partitionOverwriteMode == PartitionOverwriteMode.DYNAMIC) {
           OverwritePartitionsDynamic.byPosition(r, query)
         } else {
           OverwriteByExpression.byPosition(r, query, staticDeleteExpression(r, staticPartitions))
@@ -1075,8 +1278,7 @@ class Analyzer(
         partitionColumnNames.find(name => conf.resolver(name, partitionName)) match {
           case Some(_) =>
           case None =>
-            throw new AnalysisException(
-              s"PARTITION clause cannot contain a non-partition column name: $partitionName")
+            throw QueryCompilationErrors.nonPartitionColError(partitionName)
         }
       }
     }
@@ -1098,8 +1300,7 @@ class Analyzer(
               case Some(attr) =>
                 attr.name -> staticName
               case _ =>
-                throw new AnalysisException(
-                  s"Cannot add static value for unknown column: $staticName")
+                throw QueryCompilationErrors.addStaticValToUnknownColError(staticName)
             }).toMap
 
           val queryColumns = query.output.iterator
@@ -1111,7 +1312,11 @@ class Analyzer(
           relation.output.flatMap { col =>
             outputNameToStaticName.get(col.name).flatMap(staticPartitions.get) match {
               case Some(staticValue) =>
-                Some(Alias(Cast(Literal(staticValue), col.dataType), col.name)())
+                // SPARK-30844: try our best to follow StoreAssignmentPolicy for static partition
+                // values but not completely follow because we can't do static type checking due to
+                // the reason that the parser has erased the type info of static partition values
+                // and converted them to string.
+                Some(Alias(AnsiCast(Literal(staticValue), col.dataType), col.name)())
               case _ if queryColumns.hasNext =>
                 Some(queryColumns.next)
               case _ =>
@@ -1139,9 +1344,11 @@ class Analyzer(
               // ResolveOutputRelation runs, using the query's column names that will match the
               // table names at that point. because resolution happens after a future rule, create
               // an UnresolvedAttribute.
-              EqualTo(UnresolvedAttribute(attr.name), Cast(Literal(value), attr.dataType))
+              EqualNullSafe(
+                UnresolvedAttribute.quoted(attr.name),
+                Cast(Literal(value), attr.dataType))
             case None =>
-              throw new AnalysisException(s"Unknown static partition column: $name")
+              throw QueryCompilationErrors.unknownStaticPartitionColError(name)
           }
         }.reduce(And)
       }
@@ -1153,255 +1360,49 @@ class Analyzer(
    * a logical plan node's children.
    */
   object ResolveReferences extends Rule[LogicalPlan] {
-    /**
-     * Generate a new logical plan for the right child with different expression IDs
-     * for all conflicting attributes.
-     */
-    private def dedupRight (left: LogicalPlan, right: LogicalPlan): LogicalPlan = {
-      val conflictingAttributes = left.outputSet.intersect(right.outputSet)
-      logDebug(s"Conflicting attributes ${conflictingAttributes.mkString(",")} " +
-        s"between $left and $right")
 
-      /**
-       * For LogicalPlan likes MultiInstanceRelation, Project, Aggregate, etc, whose output doesn't
-       * inherit directly from its children, we could just stop collect on it. Because we could
-       * always replace all the lower conflict attributes with the new attributes from the new
-       * plan. Theoretically, we should do recursively collect for Generate and Window but we leave
-       * it to the next batch to reduce possible overhead because this should be a corner case.
-       */
-      def collectConflictPlans(plan: LogicalPlan): Seq[(LogicalPlan, LogicalPlan)] = plan match {
-        // Handle base relations that might appear more than once.
-        case oldVersion: MultiInstanceRelation
-            if oldVersion.outputSet.intersect(conflictingAttributes).nonEmpty =>
-          val newVersion = oldVersion.newInstance()
-          Seq((oldVersion, newVersion))
-
-        case oldVersion: SerializeFromObject
-            if oldVersion.outputSet.intersect(conflictingAttributes).nonEmpty =>
-          Seq((oldVersion, oldVersion.copy(
-            serializer = oldVersion.serializer.map(_.newInstance()))))
-
-        // Handle projects that create conflicting aliases.
-        case oldVersion @ Project(projectList, _)
-            if findAliases(projectList).intersect(conflictingAttributes).nonEmpty =>
-          Seq((oldVersion, oldVersion.copy(projectList = newAliases(projectList))))
-
-        case oldVersion @ Aggregate(_, aggregateExpressions, _)
-            if findAliases(aggregateExpressions).intersect(conflictingAttributes).nonEmpty =>
-          Seq((oldVersion, oldVersion.copy(
-            aggregateExpressions = newAliases(aggregateExpressions))))
-
-        case oldVersion @ FlatMapGroupsInPandas(_, _, output, _)
-            if oldVersion.outputSet.intersect(conflictingAttributes).nonEmpty =>
-          Seq((oldVersion, oldVersion.copy(output = output.map(_.newInstance()))))
-
-        case oldVersion: Generate
-            if oldVersion.producedAttributes.intersect(conflictingAttributes).nonEmpty =>
-          val newOutput = oldVersion.generatorOutput.map(_.newInstance())
-          Seq((oldVersion, oldVersion.copy(generatorOutput = newOutput)))
-
-        case oldVersion: Expand
-            if oldVersion.producedAttributes.intersect(conflictingAttributes).nonEmpty =>
-          val producedAttributes = oldVersion.producedAttributes
-          val newOutput = oldVersion.output.map { attr =>
-            if (producedAttributes.contains(attr)) {
-              attr.newInstance()
-            } else {
-              attr
-            }
-          }
-          Seq((oldVersion, oldVersion.copy(output = newOutput)))
-
-        case oldVersion @ Window(windowExpressions, _, _, child)
-            if AttributeSet(windowExpressions.map(_.toAttribute)).intersect(conflictingAttributes)
-            .nonEmpty =>
-          Seq((oldVersion, oldVersion.copy(windowExpressions = newAliases(windowExpressions))))
-
-        case _ => plan.children.flatMap(collectConflictPlans)
-      }
-
-      val conflictPlans = collectConflictPlans(right)
-
-      /*
-       * Note that it's possible `conflictPlans` can be empty which implies that there
-       * is a logical plan node that produces new references that this rule cannot handle.
-       * When that is the case, there must be another rule that resolves these conflicts.
-       * Otherwise, the analysis will fail.
-       */
-      if (conflictPlans.isEmpty) {
-        right
-      } else {
-        val attributeRewrites = AttributeMap(conflictPlans.flatMap {
-          case (oldRelation, newRelation) => oldRelation.output.zip(newRelation.output)})
-        val conflictPlanMap = conflictPlans.toMap
-        // transformDown so that we can replace all the old Relations in one turn due to
-        // the reason that `conflictPlans` are also collected in pre-order.
-        right transformDown {
-          case r => conflictPlanMap.getOrElse(r, r)
-        } transformUp {
-          case other => other transformExpressions {
-            case a: Attribute =>
-              dedupAttr(a, attributeRewrites)
-            case s: SubqueryExpression =>
-              s.withNewPlan(dedupOuterReferencesInSubquery(s.plan, attributeRewrites))
-          }
-        }
-      }
-    }
-
-    private def dedupAttr(attr: Attribute, attrMap: AttributeMap[Attribute]): Attribute = {
-      val exprId = attrMap.getOrElse(attr, attr).exprId
-      attr.withExprId(exprId)
-    }
-
-    /**
-     * The outer plan may have been de-duplicated and the function below updates the
-     * outer references to refer to the de-duplicated attributes.
-     *
-     * For example (SQL):
-     * {{{
-     *   SELECT * FROM t1
-     *   INTERSECT
-     *   SELECT * FROM t1
-     *   WHERE EXISTS (SELECT 1
-     *                 FROM t2
-     *                 WHERE t1.c1 = t2.c1)
-     * }}}
-     * Plan before resolveReference rule.
-     *    'Intersect
-     *    :- Project [c1#245, c2#246]
-     *    :  +- SubqueryAlias t1
-     *    :     +- Relation[c1#245,c2#246] parquet
-     *    +- 'Project [*]
-     *       +- Filter exists#257 [c1#245]
-     *       :  +- Project [1 AS 1#258]
-     *       :     +- Filter (outer(c1#245) = c1#251)
-     *       :        +- SubqueryAlias t2
-     *       :           +- Relation[c1#251,c2#252] parquet
-     *       +- SubqueryAlias t1
-     *          +- Relation[c1#245,c2#246] parquet
-     * Plan after the resolveReference rule.
-     *    Intersect
-     *    :- Project [c1#245, c2#246]
-     *    :  +- SubqueryAlias t1
-     *    :     +- Relation[c1#245,c2#246] parquet
-     *    +- Project [c1#259, c2#260]
-     *       +- Filter exists#257 [c1#259]
-     *       :  +- Project [1 AS 1#258]
-     *       :     +- Filter (outer(c1#259) = c1#251) => Updated
-     *       :        +- SubqueryAlias t2
-     *       :           +- Relation[c1#251,c2#252] parquet
-     *       +- SubqueryAlias t1
-     *          +- Relation[c1#259,c2#260] parquet  => Outer plan's attributes are de-duplicated.
-     */
-    private def dedupOuterReferencesInSubquery(
-        plan: LogicalPlan,
-        attrMap: AttributeMap[Attribute]): LogicalPlan = {
-      plan transformDown { case currentFragment =>
-        currentFragment transformExpressions {
-          case OuterReference(a: Attribute) =>
-            OuterReference(dedupAttr(a, attrMap))
-          case s: SubqueryExpression =>
-            s.withNewPlan(dedupOuterReferencesInSubquery(s.plan, attrMap))
+    /** Return true if there're conflicting attributes among children's outputs of a plan */
+    def hasConflictingAttrs(p: LogicalPlan): Boolean = {
+      p.children.length > 1 && {
+        // Note that duplicated attributes are allowed within a single node,
+        // e.g., df.select($"a", $"a"), so we should only check conflicting
+        // attributes between nodes.
+        val uniqueAttrs = mutable.HashSet[ExprId]()
+        p.children.head.outputSet.foreach(a => uniqueAttrs.add(a.exprId))
+        p.children.tail.exists { child =>
+          val uniqueSize = uniqueAttrs.size
+          val childSize = child.outputSet.size
+          child.outputSet.foreach(a => uniqueAttrs.add(a.exprId))
+          uniqueSize + childSize > uniqueAttrs.size
         }
       }
     }
 
-    /**
-     * Resolves the attribute and extract value expressions(s) by traversing the
-     * input expression in top down manner. The traversal is done in top-down manner as
-     * we need to skip over unbound lambda function expression. The lambda expressions are
-     * resolved in a different rule [[ResolveLambdaVariables]]
-     *
-     * Example :
-     * SELECT transform(array(1, 2, 3), (x, i) -> x + i)"
-     *
-     * In the case above, x and i are resolved as lambda variables in [[ResolveLambdaVariables]]
-     *
-     * Note : In this routine, the unresolved attributes are resolved from the input plan's
-     * children attributes.
-     */
-    private def resolveExpressionTopDown(e: Expression, q: LogicalPlan): Expression = {
-      if (e.resolved) return e
-      e match {
-        case f: LambdaFunction if !f.bound => f
-        case u @ UnresolvedAttribute(nameParts) =>
-          // Leave unchanged if resolution fails. Hopefully will be resolved next round.
-          val result =
-            withPosition(u) {
-              q.resolveChildren(nameParts, resolver)
-                .orElse(resolveLiteralFunction(nameParts, u, q))
-                .getOrElse(u)
-            }
-          logDebug(s"Resolving $u to $result")
-          result
-        case UnresolvedExtractValue(child, fieldExpr) if child.resolved =>
-          ExtractValue(child, fieldExpr, resolver)
-        case _ => e.mapChildren(resolveExpressionTopDown(_, q))
-      }
-    }
-
-    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
+    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
+      AlwaysProcess.fn, ruleId) {
       case p: LogicalPlan if !p.childrenResolved => p
 
+      // Wait for the rule `DeduplicateRelations` to resolve conflicting attrs first.
+      case p: LogicalPlan if hasConflictingAttrs(p) => p
+
       // If the projection list contains Stars, expand it.
       case p: Project if containsStar(p.projectList) =>
         p.copy(projectList = buildExpandedProjectList(p.projectList, p.child))
       // If the aggregate function argument contains Stars, expand it.
       case a: Aggregate if containsStar(a.aggregateExpressions) =>
         if (a.groupingExpressions.exists(_.isInstanceOf[UnresolvedOrdinal])) {
-          failAnalysis(
-            "Star (*) is not allowed in select list when GROUP BY ordinal position is used")
+          throw QueryCompilationErrors.starNotAllowedWhenGroupByOrdinalPositionUsedError()
         } else {
           a.copy(aggregateExpressions = buildExpandedProjectList(a.aggregateExpressions, a.child))
         }
-      // If the script transformation input contains Stars, expand it.
-      case t: ScriptTransformation if containsStar(t.input) =>
-        t.copy(
-          input = t.input.flatMap {
-            case s: Star => s.expand(t.child, resolver)
-            case o => o :: Nil
-          }
-        )
       case g: Generate if containsStar(g.generator.children) =>
-        failAnalysis("Invalid usage of '*' in explode/json_tuple/UDTF")
-
-      // To resolve duplicate expression IDs for Join and Intersect
-      case j @ Join(left, right, _, _, _) if !j.duplicateResolved =>
-        j.copy(right = dedupRight(left, right))
-      case f @ FlatMapCoGroupsInPandas(leftAttributes, rightAttributes, _, _, left, right) =>
-        val leftRes = leftAttributes
-          .map(x => resolveExpressionBottomUp(x, left).asInstanceOf[Attribute])
-        val rightRes = rightAttributes
-          .map(x => resolveExpressionBottomUp(x, right).asInstanceOf[Attribute])
-        f.copy(leftAttributes = leftRes, rightAttributes = rightRes)
-      // intersect/except will be rewritten to join at the begininng of optimizer. Here we need to
-      // deduplicate the right side plan, so that we won't produce an invalid self-join later.
-      case i @ Intersect(left, right, _) if !i.duplicateResolved =>
-        i.copy(right = dedupRight(left, right))
-      case e @ Except(left, right, _) if !e.duplicateResolved =>
-        e.copy(right = dedupRight(left, right))
-      case u @ Union(children) if !u.duplicateResolved =>
-        // Use projection-based de-duplication for Union to avoid breaking the checkpoint sharing
-        // feature in streaming.
-        val newChildren = children.foldRight(Seq.empty[LogicalPlan]) { (head, tail) =>
-          head +: tail.map {
-            case child if head.outputSet.intersect(child.outputSet).isEmpty =>
-              child
-            case child =>
-              val projectList = child.output.map { attr =>
-                Alias(attr, attr.name)()
-              }
-              Project(projectList, child)
-          }
-        }
-        u.copy(children = newChildren)
+        throw QueryCompilationErrors.invalidStarUsageError("explode/json_tuple/UDTF")
 
       // When resolve `SortOrder`s in Sort based on child, don't report errors as
       // we still have chance to resolve it based on its descendants
       case s @ Sort(ordering, global, child) if child.resolved && !s.resolved =>
         val newOrdering =
-          ordering.map(order => resolveExpressionBottomUp(order, child).asInstanceOf[SortOrder])
+          ordering.map(order => resolveExpressionByPlanOutput(order, child).asInstanceOf[SortOrder])
         Sort(newOrdering, global, child)
 
       // A special case for Generate, because the output of Generate should not be resolved by
@@ -1409,7 +1410,7 @@ class Analyzer(
       case g @ Generate(generator, _, _, _, _, _) if generator.resolved => g
 
       case g @ Generate(generator, join, outer, qualifier, output, child) =>
-        val newG = resolveExpressionBottomUp(generator, child, throws = true)
+        val newG = resolveExpressionByPlanOutput(generator, child, throws = true)
         if (newG.fastEquals(generator)) {
           g
         } else {
@@ -1420,16 +1421,34 @@ class Analyzer(
       // rule: ResolveDeserializer.
       case plan if containsDeserializer(plan.expressions) => plan
 
-      // SPARK-25942: Resolves aggregate expressions with `AppendColumns`'s children, instead of
-      // `AppendColumns`, because `AppendColumns`'s serializer might produce conflict attribute
-      // names leading to ambiguous references exception.
-      case a @ Aggregate(groupingExprs, aggExprs, appendColumns: AppendColumns) =>
-        a.mapExpressions(resolveExpressionTopDown(_, appendColumns))
+      // SPARK-31670: Resolve Struct field in groupByExpressions and aggregateExpressions
+      // with CUBE/ROLLUP will be wrapped with alias like Alias(GetStructField, name) with
+      // different ExprId. This cause aggregateExpressions can't be replaced by expanded
+      // groupByExpressions in `ResolveGroupingAnalytics.constructAggregateExprs()`, we trim
+      // unnecessary alias of GetStructField here.
+      case a: Aggregate =>
+        val planForResolve = a.child match {
+          // SPARK-25942: Resolves aggregate expressions with `AppendColumns`'s children, instead of
+          // `AppendColumns`, because `AppendColumns`'s serializer might produce conflict attribute
+          // names leading to ambiguous references exception.
+          case appendColumns: AppendColumns => appendColumns
+          case _ => a
+        }
 
-      case o: OverwriteByExpression if !o.outputResolved =>
-        // do not resolve expression attributes until the query attributes are resolved against the
-        // table by ResolveOutputRelation. that rule will alias the attributes to the table's names.
-        o
+        val resolvedGroupingExprs = a.groupingExpressions
+          .map(resolveExpressionByPlanChildren(_, planForResolve))
+          .map(trimTopLevelGetStructFieldAlias)
+
+        val resolvedAggExprs = a.aggregateExpressions
+          .map(resolveExpressionByPlanChildren(_, planForResolve))
+            .map(_.asInstanceOf[NamedExpression])
+
+        a.copy(resolvedGroupingExprs, resolvedAggExprs, a.child)
+
+      case o: OverwriteByExpression if o.table.resolved =>
+        // The delete condition of `OverwriteByExpression` will be passed to the table
+        // implementation and should be resolved based on the table schema.
+        o.copy(deleteExpr = resolveExpressionByPlanOutput(o.deleteExpr, o.table))
 
       case m @ MergeIntoTable(targetTable, sourceTable, _, _, _)
         if !m.resolved && targetTable.resolved && sourceTable.resolved =>
@@ -1444,28 +1463,49 @@ class Analyzer(
           case _ =>
             val newMatchedActions = m.matchedActions.map {
               case DeleteAction(deleteCondition) =>
-                val resolvedDeleteCondition = deleteCondition.map(resolveExpressionTopDown(_, m))
+                val resolvedDeleteCondition = deleteCondition.map(
+                  resolveExpressionByPlanChildren(_, m))
                 DeleteAction(resolvedDeleteCondition)
               case UpdateAction(updateCondition, assignments) =>
-                val resolvedUpdateCondition = updateCondition.map(resolveExpressionTopDown(_, m))
-                // The update value can access columns from both target and source tables.
+                val resolvedUpdateCondition = updateCondition.map(
+                  resolveExpressionByPlanChildren(_, m))
                 UpdateAction(
                   resolvedUpdateCondition,
+                  // The update value can access columns from both target and source tables.
                   resolveAssignments(assignments, m, resolveValuesWithSourceOnly = false))
+              case UpdateStarAction(updateCondition) =>
+                val assignments = targetTable.output.map { attr =>
+                  Assignment(attr, UnresolvedAttribute(Seq(attr.name)))
+                }
+                UpdateAction(
+                  updateCondition.map(resolveExpressionByPlanChildren(_, m)),
+                  // For UPDATE *, the value must from source table.
+                  resolveAssignments(assignments, m, resolveValuesWithSourceOnly = true))
               case o => o
             }
             val newNotMatchedActions = m.notMatchedActions.map {
               case InsertAction(insertCondition, assignments) =>
                 // The insert action is used when not matched, so its condition and value can only
                 // access columns from the source table.
-                val resolvedInsertCondition =
-                  insertCondition.map(resolveExpressionTopDown(_, Project(Nil, m.sourceTable)))
+                val resolvedInsertCondition = insertCondition.map(
+                  resolveExpressionByPlanChildren(_, Project(Nil, m.sourceTable)))
+                InsertAction(
+                  resolvedInsertCondition,
+                  resolveAssignments(assignments, m, resolveValuesWithSourceOnly = true))
+              case InsertStarAction(insertCondition) =>
+                // The insert action is used when not matched, so its condition and value can only
+                // access columns from the source table.
+                val resolvedInsertCondition = insertCondition.map(
+                  resolveExpressionByPlanChildren(_, Project(Nil, m.sourceTable)))
+                val assignments = targetTable.output.map { attr =>
+                  Assignment(attr, UnresolvedAttribute(Seq(attr.name)))
+                }
                 InsertAction(
                   resolvedInsertCondition,
                   resolveAssignments(assignments, m, resolveValuesWithSourceOnly = true))
               case o => o
             }
-            val resolvedMergeCondition = resolveExpressionTopDown(m.mergeCondition, m)
+            val resolvedMergeCondition = resolveExpressionByPlanChildren(m.mergeCondition, m)
             m.copy(mergeCondition = resolvedMergeCondition,
               matchedActions = newMatchedActions,
               notMatchedActions = newNotMatchedActions)
@@ -1475,49 +1515,53 @@ class Analyzer(
       case h: UnresolvedHaving => h
 
       case q: LogicalPlan =>
-        logTrace(s"Attempting to resolve ${q.simpleString(SQLConf.get.maxToStringFields)}")
-        q.mapExpressions(resolveExpressionTopDown(_, q))
+        logTrace(s"Attempting to resolve ${q.simpleString(conf.maxToStringFields)}")
+        q.mapExpressions(resolveExpressionByPlanChildren(_, q))
     }
 
     def resolveAssignments(
         assignments: Seq[Assignment],
         mergeInto: MergeIntoTable,
         resolveValuesWithSourceOnly: Boolean): Seq[Assignment] = {
-      if (assignments.isEmpty) {
-        val expandedColumns = mergeInto.targetTable.output
-        val expandedValues = mergeInto.sourceTable.output
-        expandedColumns.zip(expandedValues).map(kv => Assignment(kv._1, kv._2))
-      } else {
-        assignments.map { assign =>
-          val resolvedKey = assign.key match {
-            case c if !c.resolved =>
-              resolveExpressionTopDown(c, Project(Nil, mergeInto.targetTable))
-            case o => o
-          }
-          val resolvedValue = assign.value match {
-            // The update values may contain target and/or source references.
-            case c if !c.resolved =>
-              if (resolveValuesWithSourceOnly) {
-                resolveExpressionTopDown(c, Project(Nil, mergeInto.sourceTable))
-              } else {
-                resolveExpressionTopDown(c, mergeInto)
-              }
-            case o => o
-          }
-          Assignment(resolvedKey, resolvedValue)
+      assignments.map { assign =>
+        val resolvedKey = assign.key match {
+          case c if !c.resolved =>
+            resolveMergeExprOrFail(c, Project(Nil, mergeInto.targetTable))
+          case o => o
+        }
+        val resolvedValue = assign.value match {
+          // The update values may contain target and/or source references.
+          case c if !c.resolved =>
+            if (resolveValuesWithSourceOnly) {
+              resolveMergeExprOrFail(c, Project(Nil, mergeInto.sourceTable))
+            } else {
+              resolveMergeExprOrFail(c, mergeInto)
+            }
+          case o => o
         }
+        Assignment(resolvedKey, resolvedValue)
       }
     }
 
-    def newAliases(expressions: Seq[NamedExpression]): Seq[NamedExpression] = {
-      expressions.map {
-        case a: Alias => Alias(a.child, a.name)()
-        case other => other
+    private def resolveMergeExprOrFail(e: Expression, p: LogicalPlan): Expression = {
+      val resolved = resolveExpressionByPlanChildren(e, p)
+      resolved.references.filter(!_.resolved).foreach { a =>
+        // Note: This will throw error only on unresolved attribute issues,
+        // not other resolution errors like mismatched data types.
+        val cols = p.inputSet.toSeq.map(_.sql).mkString(", ")
+        a.failAnalysis(s"cannot resolve ${a.sql} in MERGE command given columns [$cols]")
       }
+      resolved
     }
 
-    def findAliases(projectList: Seq[NamedExpression]): AttributeSet = {
-      AttributeSet(projectList.collect { case a: Alias => a.toAttribute })
+    // This method is used to trim groupByExpressions/selectedGroupByExpressions's top-level
+    // GetStructField Alias. Since these expression are not NamedExpression originally,
+    // we are safe to trim top-level GetStructField Alias.
+    def trimTopLevelGetStructFieldAlias(e: Expression): Expression = {
+      e match {
+        case Alias(s: GetStructField, _) => s
+        case other => other
+      }
     }
 
     /**
@@ -1548,6 +1592,20 @@ class Analyzer(
     def expandStarExpression(expr: Expression, child: LogicalPlan): Expression = {
       expr.transformUp {
         case f1: UnresolvedFunction if containsStar(f1.arguments) =>
+          // SPECIAL CASE: We want to block count(tblName.*) because in spark, count(tblName.*) will
+          // be expanded while count(*) will be converted to count(1). They will produce different
+          // results and confuse users if there is any null values. For count(t1.*, t2.*), it is
+          // still allowed, since it's well-defined in spark.
+          if (!conf.allowStarWithSingleTableIdentifierInCount &&
+              f1.nameParts == Seq("count") &&
+              f1.arguments.length == 1) {
+            f1.arguments.foreach {
+              case u: UnresolvedStar if u.isQualifiedByTable(child, resolver) =>
+                throw QueryCompilationErrors
+                  .singleTableStarInCountNotAllowedError(u.target.get.mkString("."))
+              case _ => // do nothing
+            }
+          }
           f1.copy(arguments = f1.arguments.flatMap {
             case s: Star => s.expand(child, resolver)
             case o => o :: Nil
@@ -1575,7 +1633,7 @@ class Analyzer(
           })
         // count(*) has been replaced by count(1)
         case o if containsStar(o.children) =>
-          failAnalysis(s"Invalid usage of '*' in expression '${o.prettyName}'")
+          throw QueryCompilationErrors.invalidStarUsageError(s"expression '${o.prettyName}'")
       }
     }
   }
@@ -1584,74 +1642,143 @@ class Analyzer(
     exprs.exists(_.find(_.isInstanceOf[UnresolvedDeserializer]).isDefined)
   }
 
+  // support CURRENT_DATE, CURRENT_TIMESTAMP, and grouping__id
+  private val literalFunctions: Seq[(String, () => Expression, Expression => String)] = Seq(
+    (CurrentDate().prettyName, () => CurrentDate(), toPrettySQL(_)),
+    (CurrentTimestamp().prettyName, () => CurrentTimestamp(), toPrettySQL(_)),
+    (VirtualColumn.hiveGroupingIdName, () => GroupingID(Nil), _ => VirtualColumn.hiveGroupingIdName)
+  )
+
   /**
    * Literal functions do not require the user to specify braces when calling them
    * When an attributes is not resolvable, we try to resolve it as a literal function.
    */
-  private def resolveLiteralFunction(
-      nameParts: Seq[String],
-      attribute: UnresolvedAttribute,
-      plan: LogicalPlan): Option[Expression] = {
+  private def resolveLiteralFunction(nameParts: Seq[String]): Option[NamedExpression] = {
     if (nameParts.length != 1) return None
-    val isNamedExpression = plan match {
-      case Aggregate(_, aggregateExpressions, _) => aggregateExpressions.contains(attribute)
-      case Project(projectList, _) => projectList.contains(attribute)
-      case Window(windowExpressions, _, _, _) => windowExpressions.contains(attribute)
-      case _ => false
-    }
-    val wrapper: Expression => Expression =
-      if (isNamedExpression) f => Alias(f, toPrettySQL(f))() else identity
-    // support CURRENT_DATE and CURRENT_TIMESTAMP
-    val literalFunctions = Seq(CurrentDate(), CurrentTimestamp())
     val name = nameParts.head
-    val func = literalFunctions.find(e => caseInsensitiveResolution(e.prettyName, name))
-    func.map(wrapper)
+    literalFunctions.find(func => caseInsensitiveResolution(func._1, name)).map {
+      case (_, getFuncExpr, getAliasName) =>
+        val funcExpr = getFuncExpr()
+        Alias(funcExpr, getAliasName(funcExpr))()
+    }
   }
 
   /**
-   * Resolves the attribute, column value and extract value expressions(s) by traversing the
-   * input expression in bottom-up manner. In order to resolve the nested complex type fields
-   * correctly, this function makes use of `throws` parameter to control when to raise an
-   * AnalysisException.
+   * Resolves `UnresolvedAttribute`, `GetColumnByOrdinal` and extract value expressions(s) by
+   * traversing the input expression in top-down manner. It must be top-down because we need to
+   * skip over unbound lambda function expression. The lambda expressions are resolved in a
+   * different place [[ResolveLambdaVariables]].
    *
    * Example :
-   * SELECT a.b FROM t ORDER BY b[0].d
+   * SELECT transform(array(1, 2, 3), (x, i) -> x + i)"
    *
-   * In the above example, in b needs to be resolved before d can be resolved. Given we are
-   * doing a bottom up traversal, it will first attempt to resolve d and fail as b has not
-   * been resolved yet. If `throws` is false, this function will handle the exception by
-   * returning the original attribute. In this case `d` will be resolved in subsequent passes
-   * after `b` is resolved.
+   * In the case above, x and i are resolved as lambda variables in [[ResolveLambdaVariables]].
    */
-  protected[sql] def resolveExpressionBottomUp(
+  private def resolveExpression(
       expr: Expression,
-      plan: LogicalPlan,
-      throws: Boolean = false): Expression = {
-    if (expr.resolved) return expr
-    // Resolve expression in one round.
-    // If throws == false or the desired attribute doesn't exist
-    // (like try to resolve `a.b` but `a` doesn't exist), fail and return the origin one.
-    // Else, throw exception.
-    try {
-      expr transformUp {
-        case GetColumnByOrdinal(ordinal, _) => plan.output(ordinal)
+      resolveColumnByName: Seq[String] => Option[Expression],
+      getAttrCandidates: () => Seq[Attribute],
+      throws: Boolean): Expression = {
+    def innerResolve(e: Expression, isTopLevel: Boolean): Expression = {
+      if (e.resolved) return e
+      e match {
+        case f: LambdaFunction if !f.bound => f
+
+        case GetColumnByOrdinal(ordinal, _) =>
+          val attrCandidates = getAttrCandidates()
+          assert(ordinal >= 0 && ordinal < attrCandidates.length)
+          attrCandidates(ordinal)
+
+        case GetViewColumnByNameAndOrdinal(viewName, colName, ordinal, expectedNumCandidates) =>
+          val attrCandidates = getAttrCandidates()
+          val matched = attrCandidates.filter(a => resolver(a.name, colName))
+          if (matched.length != expectedNumCandidates) {
+            throw QueryCompilationErrors.incompatibleViewSchemaChange(
+              viewName, colName, expectedNumCandidates, matched)
+          }
+          matched(ordinal)
+
         case u @ UnresolvedAttribute(nameParts) =>
-          val result =
-            withPosition(u) {
-              plan.resolve(nameParts, resolver)
-                .orElse(resolveLiteralFunction(nameParts, u, plan))
-                .getOrElse(u)
-            }
+          val result = withPosition(u) {
+            resolveColumnByName(nameParts).orElse(resolveLiteralFunction(nameParts)).map {
+              // We trim unnecessary alias here. Note that, we cannot trim the alias at top-level,
+              // as we should resolve `UnresolvedAttribute` to a named expression. The caller side
+              // can trim the top-level alias if it's safe to do so. Since we will call
+              // CleanupAliases later in Analyzer, trim non top-level unnecessary alias is safe.
+              case Alias(child, _) if !isTopLevel => child
+              case other => other
+            }.getOrElse(u)
+          }
           logDebug(s"Resolving $u to $result")
           result
-        case UnresolvedExtractValue(child, fieldName) if child.resolved =>
-          ExtractValue(child, fieldName, resolver)
+
+        case u @ UnresolvedExtractValue(child, fieldName) =>
+          val newChild = innerResolve(child, isTopLevel = false)
+          if (newChild.resolved) {
+            ExtractValue(newChild, fieldName, resolver)
+          } else {
+            u.copy(child = newChild)
+          }
+
+        case _ => e.mapChildren(innerResolve(_, isTopLevel = false))
       }
+    }
+
+    try {
+      innerResolve(expr, isTopLevel = true)
     } catch {
-      case a: AnalysisException if !throws => expr
+      case _: AnalysisException if !throws => expr
     }
   }
 
+  /**
+   * Resolves `UnresolvedAttribute`, `GetColumnByOrdinal` and extract value expressions(s) by the
+   * input plan's output attributes. In order to resolve the nested fields correctly, this function
+   * makes use of `throws` parameter to control when to raise an AnalysisException.
+   *
+   * Example :
+   * SELECT * FROM t ORDER BY a.b
+   *
+   * In the above example, after `a` is resolved to a struct-type column, we may fail to resolve `b`
+   * if there is no such nested field named "b". We should not fail and wait for other rules to
+   * resolve it if possible.
+   */
+  def resolveExpressionByPlanOutput(
+      expr: Expression,
+      plan: LogicalPlan,
+      throws: Boolean = false): Expression = {
+    resolveExpression(
+      expr,
+      resolveColumnByName = nameParts => {
+        plan.resolve(nameParts, resolver)
+      },
+      getAttrCandidates = () => plan.output,
+      throws = throws)
+  }
+
+  /**
+   * Resolves `UnresolvedAttribute`, `GetColumnByOrdinal` and extract value expressions(s) by the
+   * input plan's children output attributes.
+   *
+   * @param e The expression need to be resolved.
+   * @param q The LogicalPlan whose children are used to resolve expression's attribute.
+   * @return resolved Expression.
+   */
+  def resolveExpressionByPlanChildren(
+      e: Expression,
+      q: LogicalPlan): Expression = {
+    resolveExpression(
+      e,
+      resolveColumnByName = nameParts => {
+        q.resolveChildren(nameParts, resolver)
+      },
+      getAttrCandidates = () => {
+        assert(q.children.length == 1)
+        q.children.head.output
+      },
+      throws = true)
+  }
+
   /**
    * In many dialects of SQL it is valid to use ordinal positions in order/sort by and group by
    * clauses. This rule is to convert ordinal positions to the corresponding expressions in the
@@ -1666,7 +1793,8 @@ class Analyzer(
    * have no effect on the results.
    */
   object ResolveOrdinalInOrderByAndGroupBy extends Rule[LogicalPlan] {
-    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
+    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
+      AlwaysProcess.fn, ruleId) {
       case p if !p.childrenResolved => p
       // Replace the index with the related attribute for ORDER BY,
       // which is a 1-base position of the projection list.
@@ -1675,11 +1803,9 @@ class Analyzer(
         val newOrders = orders map {
           case s @ SortOrder(UnresolvedOrdinal(index), direction, nullOrdering, _) =>
             if (index > 0 && index <= child.output.size) {
-              SortOrder(child.output(index - 1), direction, nullOrdering, Set.empty)
+              SortOrder(child.output(index - 1), direction, nullOrdering, Seq.empty)
             } else {
-              s.failAnalysis(
-                s"ORDER BY position $index is not in select list " +
-                  s"(valid range is [1, ${child.output.size}])")
+              throw QueryCompilationErrors.orderByPositionRangeError(index, child.output.size, s)
             }
           case o => o
         }
@@ -1688,18 +1814,38 @@ class Analyzer(
       // Replace the index with the corresponding expression in aggregateExpressions. The index is
       // a 1-base position of aggregateExpressions, which is output columns (select expression)
       case Aggregate(groups, aggs, child) if aggs.forall(_.resolved) &&
-        groups.exists(_.isInstanceOf[UnresolvedOrdinal]) =>
-        val newGroups = groups.map {
-          case u @ UnresolvedOrdinal(index) if index > 0 && index <= aggs.size =>
-            aggs(index - 1)
-          case ordinal @ UnresolvedOrdinal(index) =>
-            ordinal.failAnalysis(
-              s"GROUP BY position $index is not in select list " +
-                s"(valid range is [1, ${aggs.size}])")
-          case o => o
-        }
+        groups.exists(containUnresolvedOrdinal) =>
+        val newGroups = groups.map(resolveGroupByExpressionOrdinal(_, aggs))
         Aggregate(newGroups, aggs, child)
     }
+
+    private def containUnresolvedOrdinal(e: Expression): Boolean = e match {
+      case _: UnresolvedOrdinal => true
+      case gs: BaseGroupingSets => gs.children.exists(containUnresolvedOrdinal)
+      case _ => false
+    }
+
+    private def resolveGroupByExpressionOrdinal(
+        expr: Expression,
+        aggs: Seq[Expression]): Expression = expr match {
+      case ordinal @ UnresolvedOrdinal(index) =>
+        withPosition(ordinal) {
+          if (index > 0 && index <= aggs.size) {
+            val ordinalExpr = aggs(index - 1)
+            if (ordinalExpr.find(_.isInstanceOf[AggregateExpression]).nonEmpty) {
+              throw QueryCompilationErrors.groupByPositionRefersToAggregateFunctionError(
+                index, ordinalExpr)
+            } else {
+              ordinalExpr
+            }
+          } else {
+            throw QueryCompilationErrors.groupByPositionRangeError(index, aggs.size)
+          }
+        }
+      case gs: BaseGroupingSets =>
+        gs.withNewChildren(gs.children.map(resolveGroupByExpressionOrdinal(_, aggs)))
+      case others => others
+    }
   }
 
   /**
@@ -1722,18 +1868,15 @@ class Analyzer(
       }}
     }
 
-    override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
+    // Group by alias is not allowed in ANSI mode.
+    private def allowGroupByAlias: Boolean = conf.groupByAliases && !conf.ansiEnabled
+
+    override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
+      AlwaysProcess.fn, ruleId) {
       case agg @ Aggregate(groups, aggs, child)
-          if conf.groupByAliases && child.resolved && aggs.forall(_.resolved) &&
+          if allowGroupByAlias && child.resolved && aggs.forall(_.resolved) &&
             groups.exists(!_.resolved) =>
         agg.copy(groupingExpressions = mayResolveAttrByAggregateExprs(groups, aggs, child))
-
-      case gs @ GroupingSets(selectedGroups, groups, child, aggs)
-          if conf.groupByAliases && child.resolved && aggs.forall(_.resolved) &&
-            groups.exists(_.isInstanceOf[UnresolvedAttribute]) =>
-        gs.copy(
-          selectedGroupByExprs = selectedGroups.map(mayResolveAttrByAggregateExprs(_, aggs, child)),
-          groupByExprs = mayResolveAttrByAggregateExprs(groups, aggs, child))
     }
   }
 
@@ -1746,7 +1889,8 @@ class Analyzer(
    * The HAVING clause could also used a grouping columns that is not presented in the SELECT.
    */
   object ResolveMissingReferences extends Rule[LogicalPlan] {
-    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
+    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
+      AlwaysProcess.fn, ruleId) {
       // Skip sort with aggregate. This will be handled in ResolveAggregateFunctions
       case sa @ Sort(_, _, child: Aggregate) => sa
 
@@ -1771,13 +1915,22 @@ class Analyzer(
           val newFilter = Filter(newCond.head, newChild)
           Project(child.output, newFilter)
         }
+
+      case r @ RepartitionByExpression(partitionExprs, child, _)
+          if (!r.resolved || r.missingInput.nonEmpty) && child.resolved =>
+        val (newPartitionExprs, newChild) = resolveExprsAndAddMissingAttrs(partitionExprs, child)
+        if (child.output == newChild.output) {
+          r.copy(newPartitionExprs, newChild)
+        } else {
+          Project(child.output, r.copy(newPartitionExprs, newChild))
+        }
     }
 
     /**
-     * This method tries to resolve expressions and find missing attributes recursively. Specially,
-     * when the expressions used in `Sort` or `Filter` contain unresolved attributes or resolved
-     * attributes which are missed from child output. This method tries to find the missing
-     * attributes out and add into the projection.
+     * This method tries to resolve expressions and find missing attributes recursively.
+     * Specifically, when the expressions used in `Sort` or `Filter` contain unresolved attributes
+     * or resolved attributes which are missing from child output. This method tries to find the
+     * missing attributes and add them into the projection.
      */
     private def resolveExprsAndAddMissingAttrs(
         exprs: Seq[Expression], plan: LogicalPlan): (Seq[Expression], LogicalPlan) = {
@@ -1789,7 +1942,7 @@ class Analyzer(
         plan match {
           case p: Project =>
             // Resolving expressions against current plan.
-            val maybeResolvedExprs = exprs.map(resolveExpressionBottomUp(_, p))
+            val maybeResolvedExprs = exprs.map(resolveExpressionByPlanOutput(_, p))
             // Recursively resolving expressions on the child of current plan.
             val (newExprs, newChild) = resolveExprsAndAddMissingAttrs(maybeResolvedExprs, p.child)
             // If some attributes used by expressions are resolvable only on the rewritten child
@@ -1798,7 +1951,7 @@ class Analyzer(
             (newExprs, Project(p.projectList ++ missingAttrs, newChild))
 
           case a @ Aggregate(groupExprs, aggExprs, child) =>
-            val maybeResolvedExprs = exprs.map(resolveExpressionBottomUp(_, a))
+            val maybeResolvedExprs = exprs.map(resolveExpressionByPlanOutput(_, a))
             val (newExprs, newChild) = resolveExprsAndAddMissingAttrs(maybeResolvedExprs, child)
             val missingAttrs = (AttributeSet(newExprs) -- a.outputSet).intersect(newChild.outputSet)
             if (missingAttrs.forall(attr => groupExprs.exists(_.semanticEquals(attr)))) {
@@ -1810,20 +1963,20 @@ class Analyzer(
             }
 
           case g: Generate =>
-            val maybeResolvedExprs = exprs.map(resolveExpressionBottomUp(_, g))
+            val maybeResolvedExprs = exprs.map(resolveExpressionByPlanOutput(_, g))
             val (newExprs, newChild) = resolveExprsAndAddMissingAttrs(maybeResolvedExprs, g.child)
             (newExprs, g.copy(unrequiredChildIndex = Nil, child = newChild))
 
           // For `Distinct` and `SubqueryAlias`, we can't recursively resolve and add attributes
           // via its children.
           case u: UnaryNode if !u.isInstanceOf[Distinct] && !u.isInstanceOf[SubqueryAlias] =>
-            val maybeResolvedExprs = exprs.map(resolveExpressionBottomUp(_, u))
+            val maybeResolvedExprs = exprs.map(resolveExpressionByPlanOutput(_, u))
             val (newExprs, newChild) = resolveExprsAndAddMissingAttrs(maybeResolvedExprs, u.child)
             (newExprs, u.withNewChildren(Seq(newChild)))
 
           // For other operators, we can't recursively resolve and add attributes via its children.
           case other =>
-            (exprs.map(resolveExpressionBottomUp(_, other)), other)
+            (exprs.map(resolveExpressionByPlanOutput(_, other)), other)
         }
       }
     }
@@ -1844,17 +1997,19 @@ class Analyzer(
     override def apply(plan: LogicalPlan): LogicalPlan = {
       val externalFunctionNameSet = new mutable.HashSet[FunctionIdentifier]()
       plan.resolveExpressions {
-        case f: UnresolvedFunction
-          if externalFunctionNameSet.contains(normalizeFuncName(f.name)) => f
-        case f: UnresolvedFunction if v1SessionCatalog.isRegisteredFunction(f.name) => f
-        case f: UnresolvedFunction if v1SessionCatalog.isPersistentFunction(f.name) =>
-          externalFunctionNameSet.add(normalizeFuncName(f.name))
-          f
-        case f: UnresolvedFunction =>
-          withPosition(f) {
-            throw new NoSuchFunctionException(
-              f.name.database.getOrElse(v1SessionCatalog.getCurrentDatabase),
-              f.name.funcName)
+        case f @ UnresolvedFunction(AsFunctionIdentifier(ident), _, _, _, _) =>
+          if (externalFunctionNameSet.contains(normalizeFuncName(ident)) ||
+            v1SessionCatalog.isRegisteredFunction(ident)) {
+            f
+          } else if (v1SessionCatalog.isPersistentFunction(ident)) {
+            externalFunctionNameSet.add(normalizeFuncName(ident))
+            f
+          } else {
+            withPosition(f) {
+              throw new NoSuchFunctionException(
+                ident.database.getOrElse(v1SessionCatalog.getCurrentDatabase),
+                ident.funcName)
+            }
           }
       }
     }
@@ -1880,57 +2035,98 @@ class Analyzer(
   }
 
   /**
+   * Replaces [[UnresolvedFunc]]s with concrete [[LogicalPlan]]s.
    * Replaces [[UnresolvedFunction]]s with concrete [[Expression]]s.
    */
   object ResolveFunctions extends Rule[LogicalPlan] {
     val trimWarningEnabled = new AtomicBoolean(true)
-    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
+    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
+      AlwaysProcess.fn, ruleId) {
+      // Resolve functions with concrete relations from v2 catalog.
+      case UnresolvedFunc(multipartIdent) =>
+        val funcIdent = parseSessionCatalogFunctionIdentifier(multipartIdent)
+        ResolvedFunc(Identifier.of(funcIdent.database.toArray, funcIdent.funcName))
+
       case q: LogicalPlan =>
-        q transformExpressions {
+        q.transformExpressionsWithPruning(AlwaysProcess.fn, ruleId) {
           case u if !u.childrenResolved => u // Skip until children are resolved.
-          case u: UnresolvedAttribute if resolver(u.name, VirtualColumn.hiveGroupingIdName) =>
-            withPosition(u) {
-              Alias(GroupingID(Nil), VirtualColumn.hiveGroupingIdName)()
-            }
           case u @ UnresolvedGenerator(name, children) =>
             withPosition(u) {
               v1SessionCatalog.lookupFunction(name, children) match {
                 case generator: Generator => generator
-                case other =>
-                  failAnalysis(s"$name is expected to be a generator. However, " +
-                    s"its class is ${other.getClass.getCanonicalName}, which is not a generator.")
+                case other => throw QueryCompilationErrors.generatorNotExpectedError(
+                  name, other.getClass.getCanonicalName)
               }
             }
-          case u @ UnresolvedFunction(funcId, arguments, isDistinct, filter) =>
-            withPosition(u) {
-              v1SessionCatalog.lookupFunction(funcId, arguments) match {
+
+          case u @ UnresolvedFunction(AsFunctionIdentifier(ident), arguments, isDistinct, filter,
+            ignoreNulls) => withPosition(u) {
+              v1SessionCatalog.lookupFunction(ident, arguments) match {
                 // AggregateWindowFunctions are AggregateFunctions that can only be evaluated within
                 // the context of a Window clause. They do not need to be wrapped in an
                 // AggregateExpression.
                 case wf: AggregateWindowFunction =>
-                  if (isDistinct || filter.isDefined) {
-                    failAnalysis("DISTINCT or FILTER specified, " +
-                      s"but ${wf.prettyName} is not an aggregate function")
+                  if (isDistinct) {
+                    throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
+                      wf.prettyName, "DISTINCT")
+                  } else if (filter.isDefined) {
+                    throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
+                      wf.prettyName, "FILTER clause")
+                  } else if (ignoreNulls) {
+                    wf match {
+                      case nthValue: NthValue =>
+                        nthValue.copy(ignoreNulls = ignoreNulls)
+                      case _ =>
+                        throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
+                          wf.prettyName, "IGNORE NULLS")
+                    }
                   } else {
                     wf
                   }
+                case owf: FrameLessOffsetWindowFunction =>
+                  if (isDistinct) {
+                    throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
+                      owf.prettyName, "DISTINCT")
+                  } else if (filter.isDefined) {
+                    throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
+                      owf.prettyName, "FILTER clause")
+                  } else if (ignoreNulls) {
+                    owf match {
+                      case lead: Lead =>
+                        lead.copy(ignoreNulls = ignoreNulls)
+                      case lag: Lag =>
+                        lag.copy(ignoreNulls = ignoreNulls)
+                    }
+                  } else {
+                    owf
+                  }
                 // We get an aggregate function, we need to wrap it in an AggregateExpression.
                 case agg: AggregateFunction =>
-                  // TODO: SPARK-30276 Support Filter expression allows simultaneous use of DISTINCT
-                  if (filter.isDefined) {
-                    if (isDistinct) {
-                      failAnalysis("DISTINCT and FILTER cannot be used in aggregate functions " +
-                        "at the same time")
-                    } else if (!filter.get.deterministic) {
-                      failAnalysis("FILTER expression is non-deterministic, " +
-                        "it cannot be used in aggregate functions")
+                  if (filter.isDefined && !filter.get.deterministic) {
+                    throw QueryCompilationErrors.nonDeterministicFilterInAggregateError
+                  }
+                  if (ignoreNulls) {
+                    val aggFunc = agg match {
+                      case first: First => first.copy(ignoreNulls = ignoreNulls)
+                      case last: Last => last.copy(ignoreNulls = ignoreNulls)
+                      case _ =>
+                        throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
+                          agg.prettyName, "IGNORE NULLS")
                     }
+                    AggregateExpression(aggFunc, Complete, isDistinct, filter)
+                  } else {
+                    AggregateExpression(agg, Complete, isDistinct, filter)
                   }
-                  AggregateExpression(agg, Complete, isDistinct, filter)
                 // This function is not an aggregate function, just return the resolved one.
-                case other if (isDistinct || filter.isDefined) =>
-                  failAnalysis("DISTINCT or FILTER specified, " +
-                    s"but ${other.prettyName} is not an aggregate function")
+                case other if isDistinct =>
+                  throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
+                    other.prettyName, "DISTINCT")
+                case other if filter.isDefined =>
+                  throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
+                    other.prettyName, "FILTER clause")
+                case other if ignoreNulls =>
+                  throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
+                    other.prettyName, "IGNORE NULLS")
                 case e: String2TrimExpression if arguments.size == 2 =>
                   if (trimWarningEnabled.get) {
                     log.warn("Two-parameter TRIM/LTRIM/RTRIM function signatures are deprecated." +
@@ -1942,9 +2138,126 @@ class Analyzer(
                 case other =>
                   other
               }
+          }
+
+          case u @ UnresolvedFunction(nameParts, arguments, isDistinct, filter, ignoreNulls) =>
+            withPosition(u) {
+              expandIdentifier(nameParts) match {
+                case NonSessionCatalogAndIdentifier(catalog, ident) =>
+                  if (!catalog.isFunctionCatalog) {
+                    throw new AnalysisException(s"Trying to lookup function '$ident' in " +
+                      s"catalog '${catalog.name()}', but it is not a FunctionCatalog.")
+                  }
+
+                  val unbound = catalog.asFunctionCatalog.loadFunction(ident)
+                  val inputType = StructType(arguments.zipWithIndex.map {
+                    case (exp, pos) => StructField(s"_$pos", exp.dataType, exp.nullable)
+                  })
+                  val bound = try {
+                    unbound.bind(inputType)
+                  } catch {
+                    case unsupported: UnsupportedOperationException =>
+                      throw new AnalysisException(s"Function '${unbound.name}' cannot process " +
+                        s"input: (${arguments.map(_.dataType.simpleString).mkString(", ")}): " +
+                        unsupported.getMessage, cause = Some(unsupported))
+                  }
+
+                  bound match {
+                    case scalarFunc: ScalarFunction[_] =>
+                      processV2ScalarFunction(scalarFunc, inputType, arguments, isDistinct,
+                        filter, ignoreNulls)
+                    case aggFunc: V2AggregateFunction[_, _] =>
+                      processV2AggregateFunction(aggFunc, arguments, isDistinct, filter,
+                        ignoreNulls)
+                    case _ =>
+                      failAnalysis(s"Function '${bound.name()}' does not implement ScalarFunction" +
+                        s" or AggregateFunction")
+                  }
+
+                case _ => u
+              }
             }
         }
     }
+
+    private def processV2ScalarFunction(
+        scalarFunc: ScalarFunction[_],
+        inputType: StructType,
+        arguments: Seq[Expression],
+        isDistinct: Boolean,
+        filter: Option[Expression],
+        ignoreNulls: Boolean): Expression = {
+      if (isDistinct) {
+        throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
+          scalarFunc.name(), "DISTINCT")
+      } else if (filter.isDefined) {
+        throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
+          scalarFunc.name(), "FILTER clause")
+      } else if (ignoreNulls) {
+        throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
+          scalarFunc.name(), "IGNORE NULLS")
+      } else {
+        // TODO: implement type coercion by looking at input type from the UDF. We
+        //  may also want to check if the parameter types from the magic method
+        //  match the input type through `BoundFunction.inputTypes`.
+        val argClasses = inputType.fields.map(_.dataType)
+        findMethod(scalarFunc, MAGIC_METHOD_NAME, argClasses) match {
+          case Some(m) if Modifier.isStatic(m.getModifiers) =>
+            StaticInvoke(scalarFunc.getClass, scalarFunc.resultType(),
+              MAGIC_METHOD_NAME, arguments, returnNullable = scalarFunc.isResultNullable)
+          case Some(_) =>
+            val caller = Literal.create(scalarFunc, ObjectType(scalarFunc.getClass))
+            Invoke(caller, MAGIC_METHOD_NAME, scalarFunc.resultType(),
+              arguments, returnNullable = scalarFunc.isResultNullable)
+          case _ =>
+            // TODO: handle functions defined in Scala too - in Scala, even if a
+            //  subclass do not override the default method in parent interface
+            //  defined in Java, the method can still be found from
+            //  `getDeclaredMethod`.
+            // since `inputType` is a `StructType`, it is mapped to a `InternalRow`
+            // which we can use to lookup the `produceResult` method.
+            findMethod(scalarFunc, "produceResult", Seq(inputType)) match {
+              case Some(_) =>
+                ApplyFunctionExpression(scalarFunc, arguments)
+              case None =>
+                failAnalysis(s"ScalarFunction '${scalarFunc.name()}' neither implement" +
+                  s" magic method nor override 'produceResult'")
+            }
+        }
+      }
+    }
+
+    private def processV2AggregateFunction(
+        aggFunc: V2AggregateFunction[_, _],
+        arguments: Seq[Expression],
+        isDistinct: Boolean,
+        filter: Option[Expression],
+        ignoreNulls: Boolean): Expression = {
+      if (ignoreNulls) {
+        throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
+          aggFunc.name(), "IGNORE NULLS")
+      }
+      val aggregator = V2Aggregator(aggFunc, arguments)
+      AggregateExpression(aggregator, Complete, isDistinct, filter)
+    }
+
+    /**
+     * Check if the input `fn` implements the given `methodName` with parameter types specified
+     * via `inputType`.
+     */
+    private def findMethod(
+        fn: BoundFunction,
+        methodName: String,
+        inputType: Seq[DataType]): Option[Method] = {
+      val cls = fn.getClass
+      try {
+        val argClasses = inputType.map(ScalaReflection.dataTypeJavaClass)
+        Some(cls.getDeclaredMethod(methodName, argClasses: _*))
+      } catch {
+        case _: NoSuchMethodException =>
+          None
+      }
+    }
   }
 
   /**
@@ -2024,7 +2337,8 @@ class Analyzer(
      *     outer plan to get evaluated.
      */
     private def resolveSubQueries(plan: LogicalPlan, plans: Seq[LogicalPlan]): LogicalPlan = {
-      plan transformExpressions {
+      plan.transformAllExpressionsWithPruning(_.containsAnyPattern(SCALAR_SUBQUERY,
+        EXISTS_SUBQUERY, IN_SUBQUERY), ruleId) {
         case s @ ScalarSubquery(sub, _, exprId) if !sub.resolved =>
           resolveSubQuery(s, plans)(ScalarSubquery(_, _, exprId))
         case e @ Exists(sub, _, exprId) if !sub.resolved =>
@@ -2041,7 +2355,8 @@ class Analyzer(
     /**
      * Resolve and rewrite all subqueries in an operator tree..
      */
-    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
+    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
+      _.containsAnyPattern(SCALAR_SUBQUERY, EXISTS_SUBQUERY, IN_SUBQUERY), ruleId) {
       // In case of HAVING (a filter after an aggregate) we use both the aggregate and
       // its child for resolution.
       case f @ Filter(_, a: Aggregate) if f.childrenResolved =>
@@ -2049,8 +2364,8 @@ class Analyzer(
       // Only a few unary nodes (Project/Filter/Aggregate) can contain subqueries.
       case q: UnaryNode if q.childrenResolved =>
         resolveSubQueries(q, q.children)
-      case j: Join if j.childrenResolved =>
-        resolveSubQueries(j, Seq(j, j.left, j.right))
+      case j: Join if j.childrenResolved && j.duplicateResolved =>
+        resolveSubQueries(j, j.children)
       case s: SupportsSubquery if s.childrenResolved =>
         resolveSubQueries(s, s.children)
     }
@@ -2061,7 +2376,8 @@ class Analyzer(
    */
   object ResolveSubqueryColumnAliases extends Rule[LogicalPlan] {
 
-     def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
+     def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
+       AlwaysProcess.fn, ruleId) {
       case u @ UnresolvedSubqueryColumnAliases(columnNames, child) if child.resolved =>
         // Resolves output attributes if a query has alias names in its subquery:
         // e.g., SELECT * FROM (SELECT 1 AS a, 1 AS b) t(col1, col2)
@@ -2069,9 +2385,8 @@ class Analyzer(
         // Checks if the number of the aliases equals to the number of output columns
         // in the subquery.
         if (columnNames.size != outputAttrs.size) {
-          u.failAnalysis("Number of column aliases does not match number of columns. " +
-            s"Number of column aliases: ${columnNames.size}; " +
-            s"number of columns: ${outputAttrs.size}.")
+          throw QueryCompilationErrors.aliasNumberNotMatchColumnNumberError(
+            columnNames.size, outputAttrs.size, u)
         }
         val aliases = outputAttrs.zip(columnNames).map { case (attr, aliasName) =>
           Alias(attr, aliasName)()
@@ -2084,7 +2399,8 @@ class Analyzer(
    * Turns projections that contain aggregate expressions into aggregations.
    */
   object GlobalAggregates extends Rule[LogicalPlan] {
-    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperators {
+    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
+      AlwaysProcess.fn, ruleId) {
       case Project(projectList, child) if containsAggregates(projectList) =>
         Aggregate(Nil, projectList, child)
     }
@@ -2112,8 +2428,9 @@ class Analyzer(
    * those in a HAVING clause or ORDER BY clause.  These expressions are pushed down to the
    * underlying aggregate operator and then projected away after the original operator.
    */
-  object ResolveAggregateFunctions extends Rule[LogicalPlan] {
-    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
+  object ResolveAggregateFunctions extends Rule[LogicalPlan] with AliasHelper {
+    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
+      AlwaysProcess.fn, ruleId) {
       // Resolve aggregate with having clause to Filter(..., Aggregate()). Note, to avoid wrongly
       // resolve the having condition expression, here we skip resolving it in ResolveReferences
       // and transform it to Filter after aggregate is resolved. See more details in SPARK-31519.
@@ -2132,39 +2449,48 @@ class Analyzer(
           val unresolvedSortOrders = sortOrder.filter { s =>
             !s.resolved || !s.references.subsetOf(aggregate.outputSet) || containsAggregate(s)
           }
-          val aliasedOrdering =
-            unresolvedSortOrders.map(o => Alias(o.child, "aggOrder")())
-          val aggregatedOrdering = aggregate.copy(aggregateExpressions = aliasedOrdering)
+          val aliasedOrdering = unresolvedSortOrders.map(o => Alias(o.child, "aggOrder")())
+
+          val aggregateWithExtraOrdering = aggregate.copy(
+            aggregateExpressions = aggregate.aggregateExpressions ++ aliasedOrdering)
+
           val resolvedAggregate: Aggregate =
-            executeSameContext(aggregatedOrdering).asInstanceOf[Aggregate]
-          val resolvedAliasedOrdering: Seq[Alias] =
-            resolvedAggregate.aggregateExpressions.asInstanceOf[Seq[Alias]]
+            executeSameContext(aggregateWithExtraOrdering).asInstanceOf[Aggregate]
+
+          val (reResolvedAggExprs, resolvedAliasedOrdering) =
+            resolvedAggregate.aggregateExpressions.splitAt(aggregate.aggregateExpressions.length)
 
           // If we pass the analysis check, then the ordering expressions should only reference to
           // aggregate expressions or grouping expressions, and it's safe to push them down to
           // Aggregate.
           checkAnalysis(resolvedAggregate)
 
-          val originalAggExprs = aggregate.aggregateExpressions.map(
-            CleanupAliases.trimNonTopLevelAliases(_).asInstanceOf[NamedExpression])
+          val originalAggExprs = aggregate.aggregateExpressions.map(trimNonTopLevelAliases)
 
           // If the ordering expression is same with original aggregate expression, we don't need
           // to push down this ordering expression and can reference the original aggregate
           // expression instead.
           val needsPushDown = ArrayBuffer.empty[NamedExpression]
-          val evaluatedOrderings = resolvedAliasedOrdering.zip(unresolvedSortOrders).map {
-            case (evaluated, order) =>
-              val index = originalAggExprs.indexWhere {
-                case Alias(child, _) => child semanticEquals evaluated.child
-                case other => other semanticEquals evaluated.child
-              }
+          val orderToAlias = unresolvedSortOrders.zip(aliasedOrdering)
+          val evaluatedOrderings =
+            resolvedAliasedOrdering.asInstanceOf[Seq[Alias]].zip(orderToAlias).map {
+              case (evaluated, (order, aliasOrder)) =>
+                val index = reResolvedAggExprs.indexWhere {
+                  case Alias(child, _) => child semanticEquals evaluated.child
+                  case other => other semanticEquals evaluated.child
+                }
 
-              if (index == -1) {
-                needsPushDown += evaluated
-                order.copy(child = evaluated.toAttribute)
-              } else {
-                order.copy(child = originalAggExprs(index).toAttribute)
-              }
+                if (index == -1) {
+                  if (hasCharVarchar(evaluated)) {
+                    needsPushDown += aliasOrder
+                    order.copy(child = aliasOrder)
+                  } else {
+                    needsPushDown += evaluated
+                    order.copy(child = evaluated.toAttribute)
+                  }
+                } else {
+                  order.copy(child = originalAggExprs(index).toAttribute)
+                }
           }
 
           val sortOrdersMap = unresolvedSortOrders
@@ -2189,6 +2515,13 @@ class Analyzer(
         }
     }
 
+    def hasCharVarchar(expr: Alias): Boolean = {
+      expr.find {
+        case ne: NamedExpression => CharVarcharUtils.getRawType(ne.metadata).nonEmpty
+        case _ => false
+      }.nonEmpty
+    }
+
     def containsAggregate(condition: Expression): Boolean = {
       condition.find(_.isInstanceOf[AggregateExpression]).isDefined
     }
@@ -2232,7 +2565,7 @@ class Analyzer(
               }
           }
           if (aggregateExpressions.nonEmpty) {
-            Some(aggregateExpressions, transformedAggregateFilter)
+            Some(aggregateExpressions.toSeq, transformedAggregateFilter)
           } else {
             None
           }
@@ -2289,7 +2622,7 @@ class Analyzer(
             case _ => false
           }.isDefined }
       }
-      CleanupAliases.trimNonTopLevelAliases(expr) match {
+      trimNonTopLevelAliases(expr) match {
         case UnresolvedAlias(g: Generator, _) => hasInnerGenerator(g)
         case Alias(g: Generator, _) => hasInnerGenerator(g)
         case MultiAlias(g: Generator, _) => hasInnerGenerator(g)
@@ -2329,26 +2662,23 @@ class Analyzer(
       }
     }
 
-    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
+    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
+      AlwaysProcess.fn, ruleId) {
       case Project(projectList, _) if projectList.exists(hasNestedGenerator) =>
         val nestedGenerator = projectList.find(hasNestedGenerator).get
-        throw new AnalysisException("Generators are not supported when it's nested in " +
-          "expressions, but got: " + toPrettySQL(trimAlias(nestedGenerator)))
+        throw QueryCompilationErrors.nestedGeneratorError(trimAlias(nestedGenerator))
 
       case Project(projectList, _) if projectList.count(hasGenerator) > 1 =>
         val generators = projectList.filter(hasGenerator).map(trimAlias)
-        throw new AnalysisException("Only one generator allowed per select clause but found " +
-          generators.size + ": " + generators.map(toPrettySQL).mkString(", "))
+        throw QueryCompilationErrors.moreThanOneGeneratorError(generators, "select")
 
       case Aggregate(_, aggList, _) if aggList.exists(hasNestedGenerator) =>
         val nestedGenerator = aggList.find(hasNestedGenerator).get
-        throw new AnalysisException("Generators are not supported when it's nested in " +
-          "expressions, but got: " + toPrettySQL(trimAlias(nestedGenerator)))
+        throw QueryCompilationErrors.nestedGeneratorError(trimAlias(nestedGenerator))
 
       case Aggregate(_, aggList, _) if aggList.count(hasGenerator) > 1 =>
         val generators = aggList.filter(hasGenerator).map(trimAlias)
-        throw new AnalysisException("Only one generator allowed per aggregate clause but found " +
-          generators.size + ": " + generators.map(toPrettySQL).mkString(", "))
+        throw QueryCompilationErrors.moreThanOneGeneratorError(generators, "aggregate")
 
       case agg @ Aggregate(groupList, aggList, child) if aggList.forall {
           case AliasedGenerator(_, _, _) => true
@@ -2359,7 +2689,7 @@ class Analyzer(
 
         val projectExprs = Array.ofDim[NamedExpression](aggList.length)
         val newAggList = aggList
-          .map(CleanupAliases.trimNonTopLevelAliases(_).asInstanceOf[NamedExpression])
+          .map(trimNonTopLevelAliases)
           .zipWithIndex
           .flatMap {
             case (AliasedGenerator(generator, names, outer), idx) =>
@@ -2398,19 +2728,16 @@ class Analyzer(
         p
 
       case p @ Project(projectList, child) =>
-        // Holds the resolved generator, if one exists in the project list.
-        var resolvedGenerator: Generate = null
-
-        val newProjectList = projectList
-          .map(CleanupAliases.trimNonTopLevelAliases(_).asInstanceOf[NamedExpression])
-          .flatMap {
-            case AliasedGenerator(generator, names, outer) if generator.childrenResolved =>
-              // It's a sanity check, this should not happen as the previous case will throw
-              // exception earlier.
-              assert(resolvedGenerator == null, "More than one generator found in SELECT.")
-
-              resolvedGenerator =
-                Generate(
+        val (resolvedGenerator, newProjectList) = projectList
+          .map(trimNonTopLevelAliases)
+          .foldLeft((None: Option[Generate], Nil: Seq[NamedExpression])) { (res, e) =>
+            e match {
+              case AliasedGenerator(generator, names, outer) if generator.childrenResolved =>
+                // It's a sanity check, this should not happen as the previous case will throw
+                // exception earlier.
+                assert(res._1.isEmpty, "More than one generator found in SELECT.")
+
+                val g = Generate(
                   generator,
                   unrequiredChildIndex = Nil,
                   outer = outer,
@@ -2418,12 +2745,14 @@ class Analyzer(
                   generatorOutput = ResolveGenerate.makeGeneratorOutput(generator, names),
                   child)
 
-              resolvedGenerator.generatorOutput
-            case other => other :: Nil
+                (Some(g), res._2 ++ g.generatorOutput)
+              case other =>
+                (res._1, res._2 :+ other)
+            }
           }
 
-        if (resolvedGenerator != null) {
-          Project(newProjectList, resolvedGenerator)
+        if (resolvedGenerator.isDefined) {
+          Project(newProjectList, resolvedGenerator.get)
         } else {
           p
         }
@@ -2431,8 +2760,7 @@ class Analyzer(
       case g: Generate => g
 
       case p if p.expressions.exists(hasGenerator) =>
-        throw new AnalysisException("Generators are not supported outside the SELECT clause, but " +
-          "got: " + p.simpleString(SQLConf.get.maxToStringFields))
+        throw QueryCompilationErrors.generatorOutsideSelectError(p)
     }
   }
 
@@ -2446,7 +2774,8 @@ class Analyzer(
    * that wrap the [[Generator]].
    */
   object ResolveGenerate extends Rule[LogicalPlan] {
-    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
+    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
+      AlwaysProcess.fn, ruleId) {
       case g: Generate if !g.child.resolved || !g.generator.resolved => g
       case g: Generate if !g.resolved =>
         g.copy(generatorOutput = makeGeneratorOutput(g.generator, g.generatorOutput.map(_.name)))
@@ -2468,10 +2797,8 @@ class Analyzer(
       } else if (names.isEmpty) {
         elementAttrs
       } else {
-        failAnalysis(
-          "The number of aliases supplied in the AS clause does not match the number of columns " +
-          s"output by the UDTF expected ${elementAttrs.size} aliases but got " +
-          s"${names.mkString(",")} ")
+        throw QueryCompilationErrors.aliasesNumberNotMatchUDTFOutputError(
+          elementAttrs.size, names.mkString(","))
       }
     }
   }
@@ -2501,6 +2828,8 @@ class Analyzer(
    *    [[Window]] operator and inserts it into the plan tree.
    */
   object ExtractWindowExpressions extends Rule[LogicalPlan] {
+    type Spec = (Seq[Expression], Seq[SortOrder], WindowFunctionType)
+
     private def hasWindowFunction(exprs: Seq[Expression]): Boolean =
       exprs.exists(hasWindowFunction)
 
@@ -2578,8 +2907,7 @@ class Analyzer(
             wsc.copy(partitionSpec = newPartitionSpec, orderSpec = newOrderSpec)
 
           case WindowExpression(ae: AggregateExpression, _) if ae.filter.isDefined =>
-            failAnalysis(
-              "window aggregate function with filter predicate is not supported yet.")
+            throw QueryCompilationErrors.windowAggregateFunctionWithFilterNotSupportedError
 
           // Extract Windowed AggregateExpression
           case we @ WindowExpression(
@@ -2592,8 +2920,7 @@ class Analyzer(
             WindowExpression(newAgg, spec)
 
           case AggregateExpression(aggFunc, _, _, _, _) if hasWindowFunction(aggFunc.children) =>
-            failAnalysis("It is not allowed to use a window function inside an aggregate " +
-              "function. Please use the inner window function in a sub-query.")
+            throw QueryCompilationErrors.windowFunctionInsideAggregateFunctionNotAllowedError
 
           // Extracts AggregateExpression. For example, for SUM(x) - Sum(y) OVER (...),
           // we need to extract SUM(x).
@@ -2645,8 +2972,11 @@ class Analyzer(
         }.asInstanceOf[NamedExpression]
       }
 
+      // SPARK-32616: Use a linked hash map to maintains the insertion order of the Window
+      // operators, so the query with multiple Window operators can have the determined plan.
+      val groupedWindowExpressions = mutable.LinkedHashMap.empty[Spec, ArrayBuffer[NamedExpression]]
       // Second, we group extractedWindowExprBuffer based on their Partition and Order Specs.
-      val groupedWindowExpressions = extractedWindowExprBuffer.groupBy { expr =>
+      extractedWindowExprBuffer.foreach { expr =>
         val distinctWindowSpec = expr.collect {
           case window: WindowExpression => window.windowSpec
         }.distinct
@@ -2654,24 +2984,27 @@ class Analyzer(
         // We do a final check and see if we only have a single Window Spec defined in an
         // expressions.
         if (distinctWindowSpec.isEmpty) {
-          failAnalysis(s"$expr does not have any WindowExpression.")
+          throw QueryCompilationErrors.expressionWithoutWindowExpressionError(expr)
         } else if (distinctWindowSpec.length > 1) {
           // newExpressionsWithWindowFunctions only have expressions with a single
           // WindowExpression. If we reach here, we have a bug.
-          failAnalysis(s"$expr has multiple Window Specifications ($distinctWindowSpec)." +
-            s"Please file a bug report with this error message, stack trace, and the query.")
+          throw QueryCompilationErrors.expressionWithMultiWindowExpressionsError(
+            expr, distinctWindowSpec)
         } else {
           val spec = distinctWindowSpec.head
-          (spec.partitionSpec, spec.orderSpec, WindowFunctionType.functionType(expr))
+          val specKey = (spec.partitionSpec, spec.orderSpec, WindowFunctionType.functionType(expr))
+          val windowExprs = groupedWindowExpressions
+            .getOrElseUpdate(specKey, new ArrayBuffer[NamedExpression])
+          windowExprs += expr
         }
-      }.toSeq
+      }
 
       // Third, we aggregate them by adding each Window operator for each Window Spec and then
       // setting this to the child of the next Window operator.
       val windowOps =
         groupedWindowExpressions.foldLeft(child) {
           case (last, ((partitionSpec, orderSpec, _), windowExpressions)) =>
-            Window(windowExpressions, partitionSpec, orderSpec, last)
+            Window(windowExpressions.toSeq, partitionSpec, orderSpec, last)
         }
 
       // Finally, we create a Project to output windowOps's output
@@ -2681,20 +3014,21 @@ class Analyzer(
 
     // We have to use transformDown at here to make sure the rule of
     // "Aggregate with Having clause" will be triggered.
-    def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperatorsDown {
+    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsDownWithPruning(
+      AlwaysProcess.fn, ruleId) {
 
       case Filter(condition, _) if hasWindowFunction(condition) =>
-        failAnalysis("It is not allowed to use window functions inside WHERE clause")
+        throw QueryCompilationErrors.windowFunctionNotAllowedError("WHERE")
 
       case UnresolvedHaving(condition, _) if hasWindowFunction(condition) =>
-        failAnalysis("It is not allowed to use window functions inside HAVING clause")
+        throw QueryCompilationErrors.windowFunctionNotAllowedError("HAVING")
 
       // Aggregate with Having clause. This rule works with an unresolved Aggregate because
       // a resolved Aggregate will not have Window Functions.
       case f @ UnresolvedHaving(condition, a @ Aggregate(groupingExprs, aggregateExprs, child))
         if child.resolved &&
-           hasWindowFunction(aggregateExprs) &&
-           a.expressions.forall(_.resolved) =>
+          hasWindowFunction(aggregateExprs) &&
+          a.expressions.forall(_.resolved) =>
         val (windowExpressions, aggregateExpressions) = extract(aggregateExprs)
         // Create an Aggregate operator to evaluate aggregation functions.
         val withAggregate = Aggregate(groupingExprs, aggregateExpressions, child)
@@ -2711,7 +3045,7 @@ class Analyzer(
       // Aggregate without Having clause.
       case a @ Aggregate(groupingExprs, aggregateExprs, child)
         if hasWindowFunction(aggregateExprs) &&
-           a.expressions.forall(_.resolved) =>
+          a.expressions.forall(_.resolved) =>
         val (windowExpressions, aggregateExpressions) = extract(aggregateExprs)
         // Create an Aggregate operator to evaluate aggregation functions.
         val withAggregate = Aggregate(groupingExprs, aggregateExpressions, child)
@@ -2739,64 +3073,19 @@ class Analyzer(
     }
   }
 
-  /**
-   * Pulls out nondeterministic expressions from LogicalPlan which is not Project or Filter,
-   * put them into an inner Project and finally project them away at the outer Project.
-   */
-  object PullOutNondeterministic extends Rule[LogicalPlan] {
-    override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
-      case p if !p.resolved => p // Skip unresolved nodes.
-      case p: Project => p
-      case f: Filter => f
-
-      case a: Aggregate if a.groupingExpressions.exists(!_.deterministic) =>
-        val nondeterToAttr = getNondeterToAttr(a.groupingExpressions)
-        val newChild = Project(a.child.output ++ nondeterToAttr.values, a.child)
-        a.transformExpressions { case e =>
-          nondeterToAttr.get(e).map(_.toAttribute).getOrElse(e)
-        }.copy(child = newChild)
-
-      // Don't touch collect metrics. Top-level metrics are not supported (check analysis will fail)
-      // and we want to retain them inside the aggregate functions.
-      case m: CollectMetrics => m
-
-      // todo: It's hard to write a general rule to pull out nondeterministic expressions
-      // from LogicalPlan, currently we only do it for UnaryNode which has same output
-      // schema with its child.
-      case p: UnaryNode if p.output == p.child.output && p.expressions.exists(!_.deterministic) =>
-        val nondeterToAttr = getNondeterToAttr(p.expressions)
-        val newPlan = p.transformExpressions { case e =>
-          nondeterToAttr.get(e).map(_.toAttribute).getOrElse(e)
-        }
-        val newChild = Project(p.child.output ++ nondeterToAttr.values, p.child)
-        Project(p.output, newPlan.withNewChildren(newChild :: Nil))
-    }
-
-    private def getNondeterToAttr(exprs: Seq[Expression]): Map[Expression, NamedExpression] = {
-      exprs.filterNot(_.deterministic).flatMap { expr =>
-        val leafNondeterministic = expr.collect { case n: Nondeterministic => n }
-        leafNondeterministic.distinct.map { e =>
-          val ne = e match {
-            case n: NamedExpression => n
-            case _ => Alias(e, "_nondeterministic")()
-          }
-          e -> ne
-        }
-      }.toMap
-    }
-  }
-
   /**
    * Set the seed for random number generation.
    */
   object ResolveRandomSeed extends Rule[LogicalPlan] {
     private lazy val random = new Random()
 
-    override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
+    override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
+      _.containsPattern(EXPRESSION_WITH_RANDOM_SEED), ruleId) {
       case p if p.resolved => p
-      case p => p transformExpressionsUp {
-        case Uuid(None) => Uuid(Some(random.nextLong()))
-        case Shuffle(child, None) => Shuffle(child, Some(random.nextLong()))
+      case p => p.transformExpressionsUpWithPruning(
+        _.containsPattern(EXPRESSION_WITH_RANDOM_SEED), ruleId) {
+        case e: ExpressionWithRandomSeed if e.seedExpression == UnresolvedSeed =>
+          e.withNewSeed(random.nextLong())
       }
     }
   }
@@ -2813,13 +3102,12 @@ class Analyzer(
 
       case p => p transformExpressionsUp {
 
-        case udf @ ScalaUDF(_, _, inputs, _, _, _, _)
-            if udf.inputPrimitives.contains(true) =>
+        case udf: ScalaUDF if udf.inputPrimitives.contains(true) =>
           // Otherwise, add special handling of null for fields that can't accept null.
           // The result of operations like this, when passed null, is generally to return null.
-          assert(udf.inputPrimitives.length == inputs.length)
+          assert(udf.inputPrimitives.length == udf.children.length)
 
-          val inputPrimitivesPair = udf.inputPrimitives.zip(inputs)
+          val inputPrimitivesPair = udf.inputPrimitives.zip(udf.children)
           val inputNullCheck = inputPrimitivesPair.collect {
             case (isPrimitive, input) if isPrimitive && input.nullable =>
               IsNull(input)
@@ -2847,14 +3135,53 @@ class Analyzer(
     }
   }
 
+  /**
+   * Resolve the encoders for the UDF by explicitly given the attributes. We give the
+   * attributes explicitly in order to handle the case where the data type of the input
+   * value is not the same with the internal schema of the encoder, which could cause
+   * data loss. For example, the encoder should not cast the input value to Decimal(38, 18)
+   * if the actual data type is Decimal(30, 0).
+   *
+   * The resolved encoders then will be used to deserialize the internal row to Scala value.
+   */
+  object ResolveEncodersInUDF extends Rule[LogicalPlan] {
+    override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
+      AlwaysProcess.fn, ruleId) {
+      case p if !p.resolved => p // Skip unresolved nodes.
+
+      case p => p.transformExpressionsUpWithPruning(AlwaysProcess.fn, ruleId) {
+
+        case udf: ScalaUDF if udf.inputEncoders.nonEmpty =>
+          val boundEncoders = udf.inputEncoders.zipWithIndex.map { case (encOpt, i) =>
+            val dataType = udf.children(i).dataType
+            encOpt.map { enc =>
+              val attrs = if (enc.isSerializedAsStructForTopLevel) {
+                dataType.asInstanceOf[StructType].toAttributes
+              } else {
+                // the field name doesn't matter here, so we use
+                // a simple literal to avoid any overhead
+                new StructType().add("input", dataType).toAttributes
+              }
+              enc.resolveAndBind(attrs)
+            }
+          }
+          udf.copy(inputEncoders = boundEncoders)
+      }
+    }
+  }
+
   /**
    * Check and add proper window frames for all window functions.
    */
   object ResolveWindowFrame extends Rule[LogicalPlan] {
-    def apply(plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
+    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveExpressionsWithPruning(
+      _.containsPattern(WINDOW_EXPRESSION), ruleId) {
+      case WindowExpression(wf: FrameLessOffsetWindowFunction,
+        WindowSpecDefinition(_, _, f: SpecifiedWindowFrame)) if wf.frame != f =>
+        throw QueryCompilationErrors.cannotSpecifyWindowFrameError(wf.prettyName)
       case WindowExpression(wf: WindowFunction, WindowSpecDefinition(_, _, f: SpecifiedWindowFrame))
           if wf.frame != UnspecifiedFrame && wf.frame != f =>
-        failAnalysis(s"Window Frame $f must match the required frame ${wf.frame}")
+        throw QueryCompilationErrors.windowFrameNotMatchRequiredFrameError(f, wf.frame)
       case WindowExpression(wf: WindowFunction, s @ WindowSpecDefinition(_, _, UnspecifiedFrame))
           if wf.frame != UnspecifiedFrame =>
         WindowExpression(wf, s.copy(frameSpecification = wf.frame))
@@ -2873,11 +3200,10 @@ class Analyzer(
    * Check and add order to [[AggregateWindowFunction]]s.
    */
   object ResolveWindowOrder extends Rule[LogicalPlan] {
-    def apply(plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
+    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveExpressionsWithPruning(
+      _.containsPattern(WINDOW_EXPRESSION), ruleId) {
       case WindowExpression(wf: WindowFunction, spec) if spec.orderSpec.isEmpty =>
-        failAnalysis(s"Window function $wf requires window to be ordered, please add ORDER BY " +
-          s"clause. For example SELECT $wf(value_expr) OVER (PARTITION BY window_partition " +
-          s"ORDER BY window_ordering) from table")
+        throw QueryCompilationErrors.windowFunctionWithWindowFrameNotOrderedError(wf)
       case WindowExpression(rank: RankLike, spec) if spec.resolved =>
         val order = spec.orderSpec.map(_.child)
         WindowExpression(rank.withOrder(order), spec)
@@ -2889,7 +3215,8 @@ class Analyzer(
    * Then apply a Project on a normal Join to eliminate natural or using join.
    */
   object ResolveNaturalAndUsingJoin extends Rule[LogicalPlan] {
-    override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
+    override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
+      _.containsPattern(NATURAL_LIKE_JOIN), ruleId) {
       case j @ Join(left, right, UsingJoin(joinType, usingCols), _, hint)
           if left.resolved && right.resolved && j.duplicateResolved =>
         commonNaturalJoinProcessing(left, right, joinType, usingCols, None, hint)
@@ -2910,52 +3237,71 @@ class Analyzer(
    * - Detect plans that are not compatible with the output table and throw AnalysisException
    */
   object ResolveOutputRelation extends Rule[LogicalPlan] {
-    override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperators {
-      case append @ AppendData(table, query, _, isByName)
-          if table.resolved && query.resolved && !append.outputResolved =>
+    override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsWithPruning(
+      AlwaysProcess.fn, ruleId) {
+      case v2Write: V2WriteCommand
+          if v2Write.table.resolved && v2Write.query.resolved && !v2Write.outputResolved =>
         validateStoreAssignmentPolicy()
-        val projection =
-          TableOutputResolver.resolveOutputColumns(table.name, table.output, query, isByName, conf)
-
-        if (projection != query) {
-          append.copy(query = projection)
+        val projection = TableOutputResolver.resolveOutputColumns(
+          v2Write.table.name, v2Write.table.output, v2Write.query, v2Write.isByName, conf)
+        if (projection != v2Write.query) {
+          val cleanedTable = v2Write.table match {
+            case r: DataSourceV2Relation =>
+              r.copy(output = r.output.map(CharVarcharUtils.cleanAttrMetadata))
+            case other => other
+          }
+          v2Write.withNewQuery(projection).withNewTable(cleanedTable)
         } else {
-          append
+          v2Write
         }
+    }
+  }
 
-      case overwrite @ OverwriteByExpression(table, _, query, _, isByName)
-          if table.resolved && query.resolved && !overwrite.outputResolved =>
-        validateStoreAssignmentPolicy()
-        val projection =
-          TableOutputResolver.resolveOutputColumns(table.name, table.output, query, isByName, conf)
+  object ResolveUserSpecifiedColumns extends Rule[LogicalPlan] {
+    override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsWithPruning(
+      AlwaysProcess.fn, ruleId) {
+      case i: InsertIntoStatement if i.table.resolved && i.query.resolved &&
+          i.userSpecifiedCols.nonEmpty =>
+        val resolved = resolveUserSpecifiedColumns(i)
+        val projection = addColumnListOnQuery(i.table.output, resolved, i.query)
+        i.copy(userSpecifiedCols = Nil, query = projection)
+    }
 
-        if (projection != query) {
-          overwrite.copy(query = projection)
-        } else {
-          overwrite
-        }
+    private def resolveUserSpecifiedColumns(i: InsertIntoStatement): Seq[NamedExpression] = {
+      SchemaUtils.checkColumnNameDuplication(
+        i.userSpecifiedCols, "in the column list", resolver)
 
-      case overwrite @ OverwritePartitionsDynamic(table, query, _, isByName)
-          if table.resolved && query.resolved && !overwrite.outputResolved =>
-        validateStoreAssignmentPolicy()
-        val projection =
-          TableOutputResolver.resolveOutputColumns(table.name, table.output, query, isByName, conf)
+      i.userSpecifiedCols.map { col =>
+          i.table.resolve(Seq(col), resolver)
+            .getOrElse(throw QueryCompilationErrors.cannotResolveUserSpecifiedColumnsError(
+              col, i.table))
+      }
+    }
 
-        if (projection != query) {
-          overwrite.copy(query = projection)
-        } else {
-          overwrite
-        }
+    private def addColumnListOnQuery(
+        tableOutput: Seq[Attribute],
+        cols: Seq[NamedExpression],
+        query: LogicalPlan): LogicalPlan = {
+      if (cols.size != query.output.size) {
+        throw QueryCompilationErrors.writeTableWithMismatchedColumnsError(
+          cols.size, query.output.size, query)
+      }
+      val nameToQueryExpr = cols.zip(query.output).toMap
+      // Static partition columns in the table output should not appear in the column list
+      // they will be handled in another rule ResolveInsertInto
+      val reordered = tableOutput.flatMap { nameToQueryExpr.get(_).orElse(None) }
+      if (reordered == query.output) {
+        query
+      } else {
+        Project(reordered, query)
+      }
     }
   }
 
   private def validateStoreAssignmentPolicy(): Unit = {
     // SPARK-28730: LEGACY store assignment policy is disallowed in data source v2.
     if (conf.storeAssignmentPolicy == StoreAssignmentPolicy.LEGACY) {
-      val configKey = SQLConf.STORE_ASSIGNMENT_POLICY.key
-      throw new AnalysisException(s"""
-        |"LEGACY" store assignment policy is disallowed in Spark data source V2.
-        |Please set the configuration $configKey to other values.""".stripMargin)
+      throw QueryCompilationErrors.legacyStoreAssignmentPolicyError()
     }
   }
 
@@ -2965,17 +3311,17 @@ class Analyzer(
       joinType: JoinType,
       joinNames: Seq[String],
       condition: Option[Expression],
-      hint: JoinHint) = {
+      hint: JoinHint): LogicalPlan = {
+    import org.apache.spark.sql.catalyst.util._
+
     val leftKeys = joinNames.map { keyName =>
       left.output.find(attr => resolver(attr.name, keyName)).getOrElse {
-        throw new AnalysisException(s"USING column `$keyName` cannot be resolved on the left " +
-          s"side of the join. The left-side columns: [${left.output.map(_.name).mkString(", ")}]")
+        throw QueryCompilationErrors.unresolvedUsingColForJoinError(keyName, left, "left")
       }
     }
     val rightKeys = joinNames.map { keyName =>
       right.output.find(attr => resolver(attr.name, keyName)).getOrElse {
-        throw new AnalysisException(s"USING column `$keyName` cannot be resolved on the right " +
-          s"side of the join. The right-side columns: [${right.output.map(_.name).mkString(", ")}]")
+        throw QueryCompilationErrors.unresolvedUsingColForJoinError(keyName, right, "right")
       }
     }
     val joinPairs = leftKeys.zip(rightKeys)
@@ -2987,26 +3333,33 @@ class Analyzer(
     val rUniqueOutput = right.output.filterNot(att => rightKeys.contains(att))
 
     // the output list looks like: join keys, columns from left, columns from right
-    val projectList = joinType match {
+    val (projectList, hiddenList) = joinType match {
       case LeftOuter =>
-        leftKeys ++ lUniqueOutput ++ rUniqueOutput.map(_.withNullability(true))
+        (leftKeys ++ lUniqueOutput ++ rUniqueOutput.map(_.withNullability(true)), rightKeys)
       case LeftExistence(_) =>
-        leftKeys ++ lUniqueOutput
+        (leftKeys ++ lUniqueOutput, Seq.empty)
       case RightOuter =>
-        rightKeys ++ lUniqueOutput.map(_.withNullability(true)) ++ rUniqueOutput
+        (rightKeys ++ lUniqueOutput.map(_.withNullability(true)) ++ rUniqueOutput, leftKeys)
       case FullOuter =>
         // in full outer join, joinCols should be non-null if there is.
         val joinedCols = joinPairs.map { case (l, r) => Alias(Coalesce(Seq(l, r)), l.name)() }
-        joinedCols ++
+        (joinedCols ++
           lUniqueOutput.map(_.withNullability(true)) ++
-          rUniqueOutput.map(_.withNullability(true))
+          rUniqueOutput.map(_.withNullability(true)),
+          leftKeys ++ rightKeys)
       case _ : InnerLike =>
-        leftKeys ++ lUniqueOutput ++ rUniqueOutput
+        (leftKeys ++ lUniqueOutput ++ rUniqueOutput, rightKeys)
       case _ =>
-        sys.error("Unsupported natural join type " + joinType)
+        throw QueryExecutionErrors.unsupportedNaturalJoinTypeError(joinType)
     }
-    // use Project to trim unnecessary fields
-    Project(projectList, Join(left, right, joinType, newCondition, hint))
+    // use Project to hide duplicated common keys
+    // propagate hidden columns from nested USING/NATURAL JOINs
+    val project = Project(projectList, Join(left, right, joinType, newCondition, hint))
+    project.setTagValue(
+      Project.hiddenOutputTag,
+      hiddenList.map(_.markAsSupportsQualifiedStar()) ++
+        project.child.metadataOutput.filter(_.supportsQualifiedStar))
+    project
   }
 
   /**
@@ -3014,11 +3367,12 @@ class Analyzer(
    * to the given input attributes.
    */
   object ResolveDeserializer extends Rule[LogicalPlan] {
-    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
+    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
+      AlwaysProcess.fn, ruleId) {
       case p if !p.childrenResolved => p
       case p if p.resolved => p
 
-      case p => p transformExpressions {
+      case p => p.transformExpressionsWithPruning(AlwaysProcess.fn, ruleId) {
         case UnresolvedDeserializer(deserializer, inputAttributes) =>
           val inputs = if (inputAttributes.isEmpty) {
             p.children.flatMap(_.output)
@@ -3027,7 +3381,7 @@ class Analyzer(
           }
 
           validateTopLevelTupleFields(deserializer, inputs)
-          val resolved = resolveExpressionBottomUp(
+          val resolved = resolveExpressionByPlanOutput(
             deserializer, LocalRelation(inputs), throws = true)
           val result = resolved transformDown {
             case UnresolvedMapObjects(func, inputData, cls) if inputData.resolved =>
@@ -3038,7 +3392,8 @@ class Analyzer(
                       ExtractValue(child, fieldName, resolver)
                   }
                 case other =>
-                  throw new AnalysisException("need an array field but got " + other.catalogString)
+                  throw QueryCompilationErrors.dataTypeMismatchForDeserializerError(other,
+                    "array")
               }
             case u: UnresolvedCatalystToExternalMap if u.child.resolved =>
               u.child.dataType match {
@@ -3048,7 +3403,7 @@ class Analyzer(
                       ExtractValue(child, fieldName, resolver)
                   }
                 case other =>
-                  throw new AnalysisException("need a map field but got " + other.catalogString)
+                  throw QueryCompilationErrors.dataTypeMismatchForDeserializerError(other, "map")
               }
           }
           validateNestedTupleFields(result)
@@ -3057,8 +3412,7 @@ class Analyzer(
     }
 
     private def fail(schema: StructType, maxOrdinal: Int): Unit = {
-      throw new AnalysisException(s"Try to map ${schema.catalogString} to Tuple${maxOrdinal + 1}" +
-        ", but failed as the number of fields does not line up.")
+      throw QueryCompilationErrors.fieldNumberMismatchForDeserializerError(schema, maxOrdinal)
     }
 
     /**
@@ -3109,18 +3463,16 @@ class Analyzer(
    * constructed is an inner class.
    */
   object ResolveNewInstance extends Rule[LogicalPlan] {
-    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
+    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
+      AlwaysProcess.fn, ruleId) {
       case p if !p.childrenResolved => p
       case p if p.resolved => p
 
-      case p => p transformExpressions {
+      case p => p.transformExpressionsUpWithPruning(AlwaysProcess.fn, ruleId) {
         case n: NewInstance if n.childrenResolved && !n.resolved =>
           val outer = OuterScopes.getOuterScope(n.cls)
           if (outer == null) {
-            throw new AnalysisException(
-              s"Unable to generate an encoder for inner class `${n.cls.getName}` without " +
-                "access to the scope that this class was defined in.\n" +
-                "Try moving this class out of its parent class.")
+            throw QueryCompilationErrors.outerScopeFailureForNewInstanceError(n.cls.getName)
           }
           n.copy(outerPointer = Some(outer))
       }
@@ -3136,29 +3488,39 @@ class Analyzer(
         case l: LambdaVariable => "array element"
         case e => e.sql
       }
-      throw new AnalysisException(s"Cannot up cast $fromStr from " +
-        s"${from.dataType.catalogString} to ${to.catalogString}.\n" +
-        "The type path of the target object is:\n" + walkedTypePath.mkString("", "\n", "\n") +
-        "You can either add an explicit cast to the input data or choose a higher precision " +
-        "type of the field in the target object")
+      throw QueryCompilationErrors.upCastFailureError(fromStr, from, to, walkedTypePath)
     }
 
-    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
+    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
+      AlwaysProcess.fn, ruleId) {
       case p if !p.childrenResolved => p
       case p if p.resolved => p
 
-      case p => p transformExpressions {
+      case p => p.transformExpressionsWithPruning(AlwaysProcess.fn, ruleId) {
         case u @ UpCast(child, _, _) if !child.resolved => u
 
-        case UpCast(child, dt: AtomicType, _)
-            if SQLConf.get.getConf(SQLConf.LEGACY_LOOSE_UPCAST) &&
+        case UpCast(_, target, _) if target != DecimalType && !target.isInstanceOf[DataType] =>
+          throw QueryCompilationErrors.unsupportedAbstractDataTypeForUpCastError(target)
+
+        case UpCast(child, target, walkedTypePath) if target == DecimalType
+          && child.dataType.isInstanceOf[DecimalType] =>
+          assert(walkedTypePath.nonEmpty,
+            "object DecimalType should only be used inside ExpressionEncoder")
+
+          // SPARK-31750: if we want to upcast to the general decimal type, and the `child` is
+          // already decimal type, we can remove the `Upcast` and accept any precision/scale.
+          // This can happen for cases like `spark.read.parquet("/tmp/file").as[BigDecimal]`.
+          child
+
+        case UpCast(child, target: AtomicType, _)
+            if conf.getConf(SQLConf.LEGACY_LOOSE_UPCAST) &&
               child.dataType == StringType =>
-          Cast(child, dt.asNullable)
+          Cast(child, target.asNullable)
 
-        case UpCast(child, dataType, walkedTypePath) if !Cast.canUpCast(child.dataType, dataType) =>
-          fail(child, dataType, walkedTypePath)
+        case u @ UpCast(child, _, walkedTypePath) if !Cast.canUpCast(child.dataType, u.dataType) =>
+          fail(child, u.dataType, walkedTypePath)
 
-        case UpCast(child, dataType, _) => Cast(child, dataType.asNullable)
+        case u @ UpCast(child, _, _) => Cast(child, u.dataType.asNullable)
       }
     }
   }
@@ -3221,7 +3583,8 @@ class Analyzer(
               Some(typeChange)
             } else {
               val (fieldNames, field) = fieldOpt.get
-              if (field.dataType == typeChange.newDataType()) {
+              val dt = CharVarcharUtils.getRawType(field.metadata).getOrElse(field.dataType)
+              if (dt == typeChange.newDataType()) {
                 // The user didn't want the field to change, so remove this change
                 None
               } else {
@@ -3284,8 +3647,7 @@ class Analyzer(
 
           case column: ColumnChange =>
             // This is informational for future developers
-            throw new UnsupportedOperationException(
-              "Please add an implementation for a column change here")
+            throw QueryExecutionErrors.columnChangeUnsupportedError
           case other => Some(other)
         }
 
@@ -3317,13 +3679,25 @@ class Analyzer(
             case Some(colName) =>
               ColumnPosition.after(colName)
             case None =>
-              throw new AnalysisException("Couldn't find the reference column for " +
-                s"$after at $parentName")
+              throw QueryCompilationErrors.referenceColNotFoundForAlterTableChangesError(after,
+                parentName)
           }
         case other => other
       }
     }
   }
+
+  /**
+   * A rule that marks a command as analyzed so that its children are removed to avoid
+   * being optimized. This rule should run after all other analysis rules are run.
+   */
+  object HandleAnalysisOnlyCommand extends Rule[LogicalPlan] {
+    override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
+      case c: AnalysisOnlyCommand if c.resolved =>
+        checkAnalysis(c)
+        c.markAsAnalyzed()
+    }
+  }
 }
 
 /**
@@ -3334,7 +3708,7 @@ object EliminateSubqueryAliases extends Rule[LogicalPlan] {
   // This is also called in the beginning of the optimization phase, and as a result
   // is using transformUp rather than resolveOperators.
   def apply(plan: LogicalPlan): LogicalPlan = AnalysisHelper.allowInvokingTransformsInAnalyzer {
-    plan transformUp {
+    plan.transformUpWithPruning(AlwaysProcess.fn, ruleId) {
       case SubqueryAlias(_, child) => child
     }
   }
@@ -3344,8 +3718,9 @@ object EliminateSubqueryAliases extends Rule[LogicalPlan] {
  * Removes [[Union]] operators from the plan if it just has one child.
  */
 object EliminateUnions extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
-    case Union(children) if children.size == 1 => children.head
+  def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsWithPruning(
+    AlwaysProcess.fn, ruleId) {
+    case u: Union if u.children.size == 1 => u.children.head
   }
 }
 
@@ -3356,45 +3731,23 @@ object EliminateUnions extends Rule[LogicalPlan] {
  * are not in its `children`, e.g. `RuntimeReplaceable`, the transformation for Aliases in this
  * rule can't work for those parameters.
  */
-object CleanupAliases extends Rule[LogicalPlan] {
-  def trimAliases(e: Expression): Expression = {
-    e.transformDown {
-      case Alias(child, _) => child
-      case MultiAlias(child, _) => child
-    }
-  }
-
-  def trimNonTopLevelAliases(e: Expression): Expression = e match {
-    case a: Alias =>
-      a.copy(child = trimAliases(a.child))(
-        exprId = a.exprId,
-        qualifier = a.qualifier,
-        explicitMetadata = Some(a.metadata))
-    case a: MultiAlias =>
-      a.copy(child = trimAliases(a.child))
-    case other => trimAliases(other)
-  }
-
+object CleanupAliases extends Rule[LogicalPlan] with AliasHelper {
   override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
     case Project(projectList, child) =>
-      val cleanedProjectList =
-        projectList.map(trimNonTopLevelAliases(_).asInstanceOf[NamedExpression])
+      val cleanedProjectList = projectList.map(trimNonTopLevelAliases)
       Project(cleanedProjectList, child)
 
     case Aggregate(grouping, aggs, child) =>
-      val cleanedAggs = aggs.map(trimNonTopLevelAliases(_).asInstanceOf[NamedExpression])
+      val cleanedAggs = aggs.map(trimNonTopLevelAliases)
       Aggregate(grouping.map(trimAliases), cleanedAggs, child)
 
     case Window(windowExprs, partitionSpec, orderSpec, child) =>
-      val cleanedWindowExprs =
-        windowExprs.map(e => trimNonTopLevelAliases(e).asInstanceOf[NamedExpression])
+      val cleanedWindowExprs = windowExprs.map(trimNonTopLevelAliases)
       Window(cleanedWindowExprs, partitionSpec.map(trimAliases),
         orderSpec.map(trimAliases(_).asInstanceOf[SortOrder]), child)
 
     case CollectMetrics(name, metrics, child) =>
-      val cleanedMetrics = metrics.map {
-        e => trimNonTopLevelAliases(e).asInstanceOf[NamedExpression]
-      }
+      val cleanedMetrics = metrics.map(trimNonTopLevelAliases)
       CollectMetrics(name, cleanedMetrics, child)
 
     // Operators that operate on objects should only have expressions from encoders, which should
@@ -3460,7 +3813,8 @@ object TimeWindowing extends Rule[LogicalPlan] {
    * @return the logical plan that will generate the time windows using the Expand operator, with
    *         the Filter operator for correctness and Project for usability.
    */
-  def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
+    AlwaysProcess.fn, ruleId) {
     case p: LogicalPlan if p.children.size == 1 =>
       val child = p.children.head
       val windowExpressions =
@@ -3536,8 +3890,7 @@ object TimeWindowing extends Rule[LogicalPlan] {
           renamedPlan.withNewChildren(substitutedPlan :: Nil)
         }
       } else if (numWindowExpr > 1) {
-        p.failAnalysis("Multiple time window expressions would result in a cartesian product " +
-          "of rows, therefore they are currently not supported.")
+        throw QueryCompilationErrors.multiTimeWindowExpressionsNotSupportedError(p)
       } else {
         p // Return unchanged. Analyzer will throw exception later
       }
@@ -3548,7 +3901,8 @@ object TimeWindowing extends Rule[LogicalPlan] {
  * Resolve a [[CreateNamedStruct]] if it contains [[NamePlaceholder]]s.
  */
 object ResolveCreateNamedStruct extends Rule[LogicalPlan] {
-  override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveExpressions {
+  override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveExpressionsWithPruning(
+    AlwaysProcess.fn, ruleId) {
     case e: CreateNamedStruct if !e.resolved =>
       val children = e.children.grouped(2).flatMap {
         case Seq(NamePlaceholder, e: NamedExpression) if e.resolved =>
@@ -3611,9 +3965,9 @@ object UpdateOuterReferences extends Rule[LogicalPlan] {
   }
 
   def apply(plan: LogicalPlan): LogicalPlan = {
-    plan resolveOperators {
+    plan.resolveOperatorsWithPruning(_.containsAllPatterns(PLAN_EXPRESSION, FILTER), ruleId) {
       case f @ Filter(_, a: Aggregate) if f.resolved =>
-        f transformExpressions {
+        f.transformExpressionsWithPruning(_.containsPattern(PLAN_EXPRESSION), ruleId) {
           case s: SubqueryExpression if s.children.nonEmpty =>
             // Collect the aliases from output of aggregate.
             val outerAliases = a.aggregateExpressions collect { case a: Alias => a }
@@ -3623,3 +3977,127 @@ object UpdateOuterReferences extends Rule[LogicalPlan] {
     }
   }
 }
+
+/**
+ * This rule performs string padding for char type comparison.
+ *
+ * When comparing char type column/field with string literal or char type column/field,
+ * right-pad the shorter one to the longer length.
+ */
+object ApplyCharTypePadding extends Rule[LogicalPlan] {
+
+  object AttrOrOuterRef {
+    def unapply(e: Expression): Option[Attribute] = e match {
+      case a: Attribute => Some(a)
+      case OuterReference(a: Attribute) => Some(a)
+      case _ => None
+    }
+  }
+
+  override def apply(plan: LogicalPlan): LogicalPlan = {
+    plan.resolveOperatorsUp {
+      case operator => operator.transformExpressionsUp {
+        case e if !e.childrenResolved => e
+
+        // String literal is treated as char type when it's compared to a char type column.
+        // We should pad the shorter one to the longer length.
+        case b @ BinaryComparison(e @ AttrOrOuterRef(attr), lit) if lit.foldable =>
+          padAttrLitCmp(e, attr.metadata, lit).map { newChildren =>
+            b.withNewChildren(newChildren)
+          }.getOrElse(b)
+
+        case b @ BinaryComparison(lit, e @ AttrOrOuterRef(attr)) if lit.foldable =>
+          padAttrLitCmp(e, attr.metadata, lit).map { newChildren =>
+            b.withNewChildren(newChildren.reverse)
+          }.getOrElse(b)
+
+        case i @ In(e @ AttrOrOuterRef(attr), list)
+          if attr.dataType == StringType && list.forall(_.foldable) =>
+          CharVarcharUtils.getRawType(attr.metadata).flatMap {
+            case CharType(length) =>
+              val (nulls, literalChars) =
+                list.map(_.eval().asInstanceOf[UTF8String]).partition(_ == null)
+              val literalCharLengths = literalChars.map(_.numChars())
+              val targetLen = (length +: literalCharLengths).max
+              Some(i.copy(
+                value = addPadding(e, length, targetLen),
+                list = list.zip(literalCharLengths).map {
+                  case (lit, charLength) => addPadding(lit, charLength, targetLen)
+                } ++ nulls.map(Literal.create(_, StringType))))
+            case _ => None
+          }.getOrElse(i)
+
+        // For char type column or inner field comparison, pad the shorter one to the longer length.
+        case b @ BinaryComparison(e1 @ AttrOrOuterRef(left), e2 @ AttrOrOuterRef(right))
+            // For the same attribute, they must be the same length and no padding is needed.
+            if !left.semanticEquals(right) =>
+          val outerRefs = (e1, e2) match {
+            case (_: OuterReference, _: OuterReference) => Seq(left, right)
+            case (_: OuterReference, _) => Seq(left)
+            case (_, _: OuterReference) => Seq(right)
+            case _ => Nil
+          }
+          val newChildren = CharVarcharUtils.addPaddingInStringComparison(Seq(left, right))
+          if (outerRefs.nonEmpty) {
+            b.withNewChildren(newChildren.map(_.transform {
+              case a: Attribute if outerRefs.exists(_.semanticEquals(a)) => OuterReference(a)
+            }))
+          } else {
+            b.withNewChildren(newChildren)
+          }
+
+        case i @ In(e @ AttrOrOuterRef(attr), list) if list.forall(_.isInstanceOf[Attribute]) =>
+          val newChildren = CharVarcharUtils.addPaddingInStringComparison(
+            attr +: list.map(_.asInstanceOf[Attribute]))
+          if (e.isInstanceOf[OuterReference]) {
+            i.copy(
+              value = newChildren.head.transform {
+                case a: Attribute if a.semanticEquals(attr) => OuterReference(a)
+              },
+              list = newChildren.tail)
+          } else {
+            i.copy(value = newChildren.head, list = newChildren.tail)
+          }
+      }
+    }
+  }
+
+  private def padAttrLitCmp(
+      expr: Expression,
+      metadata: Metadata,
+      lit: Expression): Option[Seq[Expression]] = {
+    if (expr.dataType == StringType) {
+      CharVarcharUtils.getRawType(metadata).flatMap {
+        case CharType(length) =>
+          val str = lit.eval().asInstanceOf[UTF8String]
+          if (str == null) {
+            None
+          } else {
+            val stringLitLen = str.numChars()
+            if (length < stringLitLen) {
+              Some(Seq(StringRPad(expr, Literal(stringLitLen)), lit))
+            } else if (length > stringLitLen) {
+              Some(Seq(expr, StringRPad(lit, Literal(length))))
+            } else {
+              None
+            }
+          }
+        case _ => None
+      }
+    } else {
+      None
+    }
+  }
+
+  private def padOuterRefAttrCmp(outerAttr: Attribute, attr: Attribute): Seq[Expression] = {
+    val Seq(r, newAttr) = CharVarcharUtils.addPaddingInStringComparison(Seq(outerAttr, attr))
+    val newOuterRef = r.transform {
+      case ar: Attribute if ar.semanticEquals(outerAttr) => OuterReference(ar)
+    }
+    Seq(newOuterRef, newAttr)
+  }
+
+  private def addPadding(expr: Expression, charLength: Int, targetLength: Int): Expression = {
+    if (targetLength > charLength) StringRPad(expr, Literal(targetLength)) else expr
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercion.scala
new file mode 100644
index 0000000000000..3732b2eb0e302
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercion.scala
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import org.apache.spark.sql.catalyst.analysis.TypeCoercion.numericPrecedence
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.types._
+
+/**
+ * In Spark ANSI mode, the type coercion rules are based on the type precedence lists of the input
+ * data types.
+ * As per the section "Type precedence list determination" of "ISO/IEC 9075-2:2011
+ * Information technology - Database languages - SQL - Part 2: Foundation (SQL/Foundation)",
+ * the type precedence lists of primitive data types are as following:
+ *   * Byte: Byte, Short, Int, Long, Decimal, Float, Double
+ *   * Short: Short, Int, Long, Decimal, Float, Double
+ *   * Int: Int, Long, Decimal, Float, Double
+ *   * Long: Long, Decimal, Float, Double
+ *   * Decimal: Float, Double, or any wider Numeric type
+ *   * Float: Float, Double
+ *   * Double: Double
+ *   * String: String
+ *   * Date: Date, Timestamp
+ *   * Timestamp: Timestamp
+ *   * Binary: Binary
+ *   * Boolean: Boolean
+ *   * Interval: Interval
+ * As for complex data types, Spark will determine the precedent list recursively based on their
+ * sub-types and nullability.
+ *
+ * With the definition of type precedent list, the general type coercion rules are as following:
+ *   * Data type S is allowed to be implicitly cast as type T iff T is in the precedence list of S
+ *   * Comparison is allowed iff the data type precedence list of both sides has at least one common
+ *     element. When evaluating the comparison, Spark casts both sides as the tightest common data
+ *     type of their precedent lists.
+ *   * There should be at least one common data type among all the children's precedence lists for
+ *     the following operators. The data type of the operator is the tightest common precedent
+ *     data type.
+ *       * In
+ *       * Except
+ *       * Intersect
+ *       * Greatest
+ *       * Least
+ *       * Union
+ *       * If
+ *       * CaseWhen
+ *       * CreateArray
+ *       * Array Concat
+ *       * Sequence
+ *       * MapConcat
+ *       * CreateMap
+ *   * For complex types (struct, array, map), Spark recursively looks into the element type and
+ *     applies the rules above.
+ *  Note: this new type coercion system will allow implicit converting String type literals as other
+ *  primitive types, in case of breaking too many existing Spark SQL queries. This is a special
+ *  rule and it is not from the ANSI SQL standard.
+ */
+object AnsiTypeCoercion extends TypeCoercionBase {
+  override def typeCoercionRules: List[Rule[LogicalPlan]] =
+    WidenSetOperationTypes ::
+    CombinedTypeCoercionRule(
+      InConversion ::
+      PromoteStringLiterals ::
+      DecimalPrecision ::
+      FunctionArgumentConversion ::
+      ConcatCoercion ::
+      MapZipWithCoercion ::
+      EltCoercion ::
+      CaseWhenCoercion ::
+      IfCoercion ::
+      StackCoercion ::
+      Division ::
+      IntegralDivision ::
+      ImplicitTypeCasts ::
+      DateTimeOperations ::
+      WindowFrameCoercion ::
+      StringLiteralCoercion :: Nil) :: Nil
+
+  val findTightestCommonType: (DataType, DataType) => Option[DataType] = {
+    case (t1, t2) if t1 == t2 => Some(t1)
+    case (NullType, t1) => Some(t1)
+    case (t1, NullType) => Some(t1)
+
+    case (t1: IntegralType, t2: DecimalType) if t2.isWiderThan(t1) =>
+      Some(t2)
+    case (t1: DecimalType, t2: IntegralType) if t1.isWiderThan(t2) =>
+      Some(t1)
+
+    case (t1: NumericType, t2: NumericType)
+        if !t1.isInstanceOf[DecimalType] && !t2.isInstanceOf[DecimalType] =>
+      val index = numericPrecedence.lastIndexWhere(t => t == t1 || t == t2)
+      val widerType = numericPrecedence(index)
+      if (widerType == FloatType) {
+        // If the input type is an Integral type and a Float type, simply return Double type as
+        // the tightest common type to avoid potential precision loss on converting the Integral
+        // type as Float type.
+        Some(DoubleType)
+      } else {
+        Some(widerType)
+      }
+
+    case (_: TimestampType, _: DateType) | (_: DateType, _: TimestampType) =>
+      Some(TimestampType)
+
+    case (t1, t2) => findTypeForComplex(t1, t2, findTightestCommonType)
+  }
+
+  override def findWiderTypeForTwo(t1: DataType, t2: DataType): Option[DataType] = {
+    findTightestCommonType(t1, t2)
+      .orElse(findWiderTypeForDecimal(t1, t2))
+      .orElse(findTypeForComplex(t1, t2, findWiderTypeForTwo))
+  }
+
+  override def findWiderCommonType(types: Seq[DataType]): Option[DataType] = {
+    types.foldLeft[Option[DataType]](Some(NullType))((r, c) =>
+      r match {
+        case Some(d) => findWiderTypeForTwo(d, c)
+        case _ => None
+      })
+  }
+
+  override def implicitCast(e: Expression, expectedType: AbstractDataType): Option[Expression] = {
+    implicitCast(e.dataType, expectedType, e.foldable).map { dt =>
+      if (dt == e.dataType) e else Cast(e, dt)
+    }
+  }
+
+  /**
+   * In Ansi mode, the implicit cast is only allow when `expectedType` is in the type precedent
+   * list of `inType`.
+   */
+  private def implicitCast(
+      inType: DataType,
+      expectedType: AbstractDataType,
+      isInputFoldable: Boolean): Option[DataType] = {
+    (inType, expectedType) match {
+      // If the expected type equals the input type, no need to cast.
+      case _ if expectedType.acceptsType(inType) => Some(inType)
+
+      // Cast null type (usually from null literals) into target types
+      // By default, the result type is `target.defaultConcreteType`. When the target type is
+      // `TypeCollection`, there is another branch to find the "closet convertible data type" below.
+      case (NullType, target) if !target.isInstanceOf[TypeCollection] =>
+        Some(target.defaultConcreteType)
+
+      // This type coercion system will allow implicit converting String type literals as other
+      // primitive types, in case of breaking too many existing Spark SQL queries.
+      case (StringType, a: AtomicType) if isInputFoldable =>
+        Some(a)
+
+      // If the target type is any Numeric type, convert the String type literal as Double type.
+      case (StringType, NumericType) if isInputFoldable =>
+        Some(DoubleType)
+
+      // If the target type is any Decimal type, convert the String type literal as Double type.
+      case (StringType, DecimalType) if isInputFoldable =>
+        Some(DecimalType.SYSTEM_DEFAULT)
+
+      // If input is a numeric type but not decimal, and we expect a decimal type,
+      // cast the input to decimal.
+      case (d: NumericType, DecimalType) => Some(DecimalType.forType(d))
+
+      case (n1: NumericType, n2: NumericType) =>
+        val widerType = findWiderTypeForTwo(n1, n2)
+        widerType match {
+          // if the expected type is Float type, we should still return Float type.
+          case Some(DoubleType) if n1 != DoubleType && n2 == FloatType => Some(FloatType)
+
+          case Some(dt) if dt == n2 => Some(dt)
+
+          case _ => None
+        }
+
+      case (DateType, TimestampType) => Some(TimestampType)
+
+      // When we reach here, input type is not acceptable for any types in this type collection,
+      // first try to find the all the expected types we can implicitly cast:
+      //   1. if there is no convertible data types, return None;
+      //   2. if there is only one convertible data type, cast input as it;
+      //   3. otherwise if there are multiple convertible data types, find the closet convertible
+      //      data type among them. If there is no such a data type, return None.
+      case (_, TypeCollection(types)) =>
+        // Since Spark contains special objects like `NumericType` and `DecimalType`, which accepts
+        // multiple types and they are `AbstractDataType` instead of `DataType`, here we use the
+        // conversion result their representation.
+        val convertibleTypes = types.flatMap(implicitCast(inType, _, isInputFoldable))
+        if (convertibleTypes.isEmpty) {
+          None
+        } else {
+          // find the closet convertible data type, which can be implicit cast to all other
+          // convertible types.
+          val closestConvertibleType = convertibleTypes.find { dt =>
+            convertibleTypes.forall { target =>
+              implicitCast(dt, target, isInputFoldable = false).isDefined
+            }
+          }
+          // If the closet convertible type is Float type and the convertible types contains Double
+          // type, simply return Double type as the closet convertible type to avoid potential
+          // precision loss on converting the Integral type as Float type.
+          if (closestConvertibleType.contains(FloatType) && convertibleTypes.contains(DoubleType)) {
+            Some(DoubleType)
+          } else {
+            closestConvertibleType
+          }
+        }
+
+      // Implicit cast between array types.
+      //
+      // Compare the nullabilities of the from type and the to type, check whether the cast of
+      // the nullability is resolvable by the following rules:
+      // 1. If the nullability of the to type is true, the cast is always allowed;
+      // 2. If the nullabilities of both the from type and the to type are false, the cast is
+      //    allowed.
+      // 3. Otherwise, the cast is not allowed
+      case (ArrayType(fromType, containsNullFrom), ArrayType(toType: DataType, containsNullTo))
+          if Cast.resolvableNullability(containsNullFrom, containsNullTo) =>
+        implicitCast(fromType, toType, isInputFoldable).map(ArrayType(_, containsNullTo))
+
+      // Implicit cast between Map types.
+      // Follows the same semantics of implicit casting between two array types.
+      // Refer to documentation above.
+      case (MapType(fromKeyType, fromValueType, fn), MapType(toKeyType, toValueType, tn))
+          if Cast.resolvableNullability(fn, tn) =>
+        val newKeyType = implicitCast(fromKeyType, toKeyType, isInputFoldable)
+        val newValueType = implicitCast(fromValueType, toValueType, isInputFoldable)
+        if (newKeyType.isDefined && newValueType.isDefined) {
+          Some(MapType(newKeyType.get, newValueType.get, tn))
+        } else {
+          None
+        }
+
+      case _ => None
+    }
+  }
+
+  override def canCast(from: DataType, to: DataType): Boolean = AnsiCast.canCast(from, to)
+
+  /**
+   * Promotes string literals that appear in arithmetic and comparison expressions.
+   */
+  object PromoteStringLiterals extends TypeCoercionRule {
+    private def castExpr(expr: Expression, targetType: DataType): Expression = {
+      expr.dataType match {
+        case NullType => Literal.create(null, targetType)
+        case l if l != targetType => Cast(expr, targetType)
+        case _ => expr
+      }
+    }
+
+    override def transform: PartialFunction[Expression, Expression] = {
+      // Skip nodes who's children have not been resolved yet.
+      case e if !e.childrenResolved => e
+
+      case b @ BinaryOperator(left @ StringType(), right @ AtomicType()) if left.foldable =>
+        b.makeCopy(Array(castExpr(left, right.dataType), right))
+
+      case b @ BinaryOperator(left @ AtomicType(), right @ StringType()) if right.foldable =>
+        b.makeCopy(Array(left, castExpr(right, left.dataType)))
+
+      case Abs(e @ StringType(), failOnError) if e.foldable => Abs(Cast(e, DoubleType), failOnError)
+      case m @ UnaryMinus(e @ StringType(), _) if e.foldable =>
+        m.withNewChildren(Seq(Cast(e, DoubleType)))
+      case UnaryPositive(e @ StringType()) if e.foldable => UnaryPositive(Cast(e, DoubleType))
+
+      // Promotes string literals in `In predicate`.
+      case p @ In(a, b)
+        if a.dataType != StringType && b.exists( e => e.foldable && e.dataType == StringType) =>
+        val newList = b.map {
+          case e @ StringType() if e.foldable => Cast(e, a.dataType)
+          case other => other
+        }
+        p.makeCopy(Array(a, newList))
+    }
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala
index 9f0eff5017f38..de0a6bbf7e1aa 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala
@@ -23,7 +23,6 @@ import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
 import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, SubqueryAlias, With}
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.{LEGACY_CTE_PRECEDENCE_POLICY, LegacyBehaviorPolicy}
 
 /**
@@ -31,7 +30,7 @@ import org.apache.spark.sql.internal.SQLConf.{LEGACY_CTE_PRECEDENCE_POLICY, Lega
  */
 object CTESubstitution extends Rule[LogicalPlan] {
   def apply(plan: LogicalPlan): LogicalPlan = {
-    LegacyBehaviorPolicy.withName(SQLConf.get.getConf(LEGACY_CTE_PRECEDENCE_POLICY)) match {
+    LegacyBehaviorPolicy.withName(conf.getConf(LEGACY_CTE_PRECEDENCE_POLICY)) match {
       case LegacyBehaviorPolicy.EXCEPTION =>
         assertNoNameConflictsInCTE(plan)
         traverseAndSubstituteCTE(plan)
@@ -57,7 +56,7 @@ object CTESubstitution extends Rule[LogicalPlan] {
       plan: LogicalPlan,
       outerCTERelationNames: Seq[String] = Nil,
       startOfQuery: Boolean = true): Unit = {
-    val resolver = SQLConf.get.resolver
+    val resolver = conf.resolver
     plan match {
       case With(child, relations) =>
         val newNames = mutable.ArrayBuffer.empty[String]
@@ -72,10 +71,10 @@ object CTESubstitution extends Rule[LogicalPlan] {
             }
             // CTE relation is defined as `SubqueryAlias`. Here we skip it and check the child
             // directly, so that `startOfQuery` is set correctly.
-            assertNoNameConflictsInCTE(relation.child, newNames)
+            assertNoNameConflictsInCTE(relation.child, newNames.toSeq)
             newNames += name
         }
-        assertNoNameConflictsInCTE(child, newNames, startOfQuery = false)
+        assertNoNameConflictsInCTE(child, newNames.toSeq, startOfQuery = false)
 
       case other =>
         other.subqueries.foreach(assertNoNameConflictsInCTE(_, outerCTERelationNames))
@@ -162,16 +161,16 @@ object CTESubstitution extends Rule[LogicalPlan] {
         traverseAndSubstituteCTE(relation)
       }
       // CTE definition can reference a previous one
-      resolvedCTERelations += (name -> substituteCTE(innerCTEResolved, resolvedCTERelations))
+      resolvedCTERelations += (name -> substituteCTE(innerCTEResolved, resolvedCTERelations.toSeq))
     }
-    resolvedCTERelations
+    resolvedCTERelations.toSeq
   }
 
   private def substituteCTE(
       plan: LogicalPlan,
       cteRelations: Seq[(String, LogicalPlan)]): LogicalPlan =
     plan resolveOperatorsUp {
-      case u @ UnresolvedRelation(Seq(table)) =>
+      case u @ UnresolvedRelation(Seq(table), _, _) =>
         cteRelations.find(r => plan.conf.resolver(r._1, table)).map(_._2).getOrElse(u)
 
       case other =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CannotReplaceMissingTableException.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CannotReplaceMissingTableException.scala
index 123d8aff25fb9..311f722034545 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CannotReplaceMissingTableException.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CannotReplaceMissingTableException.scala
@@ -26,4 +26,4 @@ class CannotReplaceMissingTableException(
     cause: Option[Throwable] = None)
   extends AnalysisException(
     s"Table $tableIdentifier cannot be replaced as it did not exist." +
-    s" Use CREATE OR REPLACE TABLE to create the table.", cause = cause)
+      s" Use CREATE OR REPLACE TABLE to create the table.", cause = cause)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
index 9e325d0c2e4e1..526ecef81a3c7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
@@ -25,15 +25,17 @@ import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
 import org.apache.spark.sql.catalyst.optimizer.BooleanSimplification
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.catalyst.util.TypeUtils
+import org.apache.spark.sql.catalyst.util.{CharVarcharUtils, TypeUtils}
+import org.apache.spark.sql.connector.catalog.{LookupCatalog, SupportsAtomicPartitionManagement, SupportsPartitionManagement, Table}
 import org.apache.spark.sql.connector.catalog.TableChange.{AddColumn, After, ColumnPosition, DeleteColumn, RenameColumn, UpdateColumnComment, UpdateColumnNullability, UpdateColumnPosition, UpdateColumnType}
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 /**
  * Throws user facing errors when passed invalid queries that fail to analyze.
  */
-trait CheckAnalysis extends PredicateHelper {
+trait CheckAnalysis extends PredicateHelper with LookupCatalog {
 
   protected def isView(nameParts: Seq[String]): Boolean
 
@@ -93,20 +95,40 @@ trait CheckAnalysis extends PredicateHelper {
 
       case p if p.analyzed => // Skip already analyzed sub-plans
 
+      case leaf: LeafNode if leaf.output.map(_.dataType).exists(CharVarcharUtils.hasCharVarchar) =>
+        throw new IllegalStateException(
+          "[BUG] logical plan should not have output of char/varchar type: " + leaf)
+
       case u: UnresolvedNamespace =>
         u.failAnalysis(s"Namespace not found: ${u.multipartIdentifier.quoted}")
 
       case u: UnresolvedTable =>
         u.failAnalysis(s"Table not found: ${u.multipartIdentifier.quoted}")
 
+      case u @ UnresolvedView(NonSessionCatalogAndIdentifier(catalog, ident), cmd, _, _) =>
+        u.failAnalysis(
+          s"Cannot specify catalog `${catalog.name}` for view ${ident.quoted} " +
+            "because view support in v2 catalog has not been implemented yet. " +
+            s"$cmd expects a view.")
+
+      case u: UnresolvedView =>
+        u.failAnalysis(s"View not found: ${u.multipartIdentifier.quoted}")
+
       case u: UnresolvedTableOrView =>
-        u.failAnalysis(s"Table or view not found: ${u.multipartIdentifier.quoted}")
+        val viewStr = if (u.allowTempView) "view" else "permanent view"
+        u.failAnalysis(
+          s"Table or $viewStr not found: ${u.multipartIdentifier.quoted}")
 
       case u: UnresolvedRelation =>
         u.failAnalysis(s"Table or view not found: ${u.multipartIdentifier.quoted}")
 
-      case InsertIntoStatement(u: UnresolvedRelation, _, _, _, _) =>
-        failAnalysis(s"Table not found: ${u.multipartIdentifier.quoted}")
+      case InsertIntoStatement(u: UnresolvedRelation, _, _, _, _, _) =>
+        u.failAnalysis(s"Table not found: ${u.multipartIdentifier.quoted}")
+
+      // TODO (SPARK-27484): handle streaming write commands when we have them.
+      case write: V2WriteCommand if write.table.isInstanceOf[UnresolvedRelation] =>
+        val tblName = write.table.asInstanceOf[UnresolvedRelation].multipartIdentifier
+        write.table.failAnalysis(s"Table or view not found: ${tblName.quoted}")
 
       case u: UnresolvedV2Relation if isView(u.originalNameParts) =>
         u.failAnalysis(
@@ -122,6 +144,23 @@ trait CheckAnalysis extends PredicateHelper {
       case AlterTable(_, _, u: UnresolvedV2Relation, _) =>
         failAnalysis(s"Table not found: ${u.originalNameParts.quoted}")
 
+      case command: V2PartitionCommand =>
+        command.table match {
+          case r @ ResolvedTable(_, _, table, _) => table match {
+            case t: SupportsPartitionManagement =>
+              if (t.partitionSchema.isEmpty) {
+                failAnalysis(s"Table ${r.name} is not partitioned.")
+              }
+            case _ =>
+              failAnalysis(s"Table ${r.name} does not support partition management.")
+          }
+          case _ =>
+        }
+
+      // `ShowTableExtended` should have been converted to the v1 command if the table is v1.
+      case _: ShowTableExtended =>
+        throw new AnalysisException("SHOW TABLE EXTENDED is not supported for v2 tables.")
+
       case operator: LogicalPlan =>
         // Check argument data types of higher-order functions downwards first.
         // If the arguments of the higher-order functions are resolved but the type check fails,
@@ -158,20 +197,26 @@ trait CheckAnalysis extends PredicateHelper {
           case g: GroupingID =>
             failAnalysis("grouping_id() can only be used with GroupingSets/Cube/Rollup")
 
+          case e: Expression if e.children.exists(_.isInstanceOf[WindowFunction]) &&
+              !e.isInstanceOf[WindowExpression] =>
+            val w = e.children.find(_.isInstanceOf[WindowFunction]).get
+            failAnalysis(s"Window function $w requires an OVER clause.")
+
           case w @ WindowExpression(AggregateExpression(_, _, true, _, _), _) =>
             failAnalysis(s"Distinct window functions are not supported: $w")
 
-          case w @ WindowExpression(_: OffsetWindowFunction,
+          case w @ WindowExpression(wf: FrameLessOffsetWindowFunction,
             WindowSpecDefinition(_, order, frame: SpecifiedWindowFrame))
              if order.isEmpty || !frame.isOffset =>
-            failAnalysis("An offset window function can only be evaluated in an ordered " +
+            failAnalysis(s"${wf.prettyName} function can only be evaluated in an ordered " +
               s"row-based window frame with a single offset: $w")
 
           case w @ WindowExpression(e, s) =>
             // Only allow window functions with an aggregate expression or an offset window
             // function or a Pandas window UDF.
             e match {
-              case _: AggregateExpression | _: OffsetWindowFunction | _: AggregateWindowFunction =>
+              case _: AggregateExpression | _: FrameLessOffsetWindowFunction |
+                  _: AggregateWindowFunction =>
                 w
               case f: PythonUDF if PythonUDF.isWindowPandasUDF(f) =>
                 w
@@ -182,6 +227,10 @@ trait CheckAnalysis extends PredicateHelper {
           case s: SubqueryExpression =>
             checkSubqueryExpression(operator, s)
             s
+
+          case e: ExpressionWithRandomSeed if !e.seedExpression.foldable =>
+            failAnalysis(
+              s"Input argument to ${e.prettyName} must be a constant.")
         }
 
         operator match {
@@ -205,7 +254,7 @@ trait CheckAnalysis extends PredicateHelper {
               s"join condition '${condition.sql}' " +
                 s"of type ${condition.dataType.catalogString} is not a boolean.")
 
-          case Aggregate(groupingExprs, aggregateExprs, child) =>
+          case a @ Aggregate(groupingExprs, aggregateExprs, child) =>
             def isAggregateExpression(expr: Expression): Boolean = {
               expr.isInstanceOf[AggregateExpression] || PythonUDF.isGroupedAggPandasUDF(expr)
             }
@@ -250,6 +299,12 @@ trait CheckAnalysis extends PredicateHelper {
                     s"nor is it an aggregate function. " +
                     "Add to group by or wrap in first() (or first_value) if you don't care " +
                     "which value you get.")
+              case s: ScalarSubquery
+                  if s.children.nonEmpty && !groupingExprs.exists(_.semanticEquals(s)) =>
+                failAnalysis(s"Correlated scalar subquery '${s.sql}' is neither " +
+                  "present in the group by, nor in an aggregate function. Add it to group by " +
+                  "using ordinal position or wrap it in first() (or first_value) if you don't " +
+                  "care which value you get.")
               case e if groupingExprs.exists(_.semanticEquals(e)) => // OK
               case e => e.children.foreach(checkValidAggregateExpression)
             }
@@ -337,7 +392,8 @@ trait CheckAnalysis extends PredicateHelper {
             def ordinalNumber(i: Int): String = i match {
               case 0 => "first"
               case 1 => "second"
-              case i => s"${i}th"
+              case 2 => "third"
+              case i => s"${i + 1}th"
             }
             val ref = dataTypes(operator.children.head)
             operator.children.tail.zipWithIndex.foreach { case (child, ti) =>
@@ -384,46 +440,6 @@ trait CheckAnalysis extends PredicateHelper {
           case write: V2WriteCommand if write.resolved =>
             write.query.schema.foreach(f => TypeUtils.failWithIntervalType(f.dataType))
 
-          // If the view output doesn't have the same number of columns neither with the child
-          // output, nor with the query column names, throw an AnalysisException.
-          // If the view's child output can't up cast to the view output,
-          // throw an AnalysisException, too.
-          case v @ View(desc, output, child) if child.resolved && !v.sameOutput(child) =>
-            val queryColumnNames = desc.viewQueryColumnNames
-            val queryOutput = if (queryColumnNames.nonEmpty) {
-              if (output.length != queryColumnNames.length) {
-                // If the view output doesn't have the same number of columns with the query column
-                // names, throw an AnalysisException.
-                throw new AnalysisException(
-                  s"The view output ${output.mkString("[", ",", "]")} doesn't have the same" +
-                    "number of columns with the query column names " +
-                    s"${queryColumnNames.mkString("[", ",", "]")}")
-              }
-              val resolver = SQLConf.get.resolver
-              queryColumnNames.map { colName =>
-                child.output.find { attr =>
-                  resolver(attr.name, colName)
-                }.getOrElse(throw new AnalysisException(
-                  s"Attribute with name '$colName' is not found in " +
-                    s"'${child.output.map(_.name).mkString("(", ",", ")")}'"))
-              }
-            } else {
-              child.output
-            }
-
-            output.zip(queryOutput).foreach {
-              case (attr, originAttr) if !attr.dataType.sameType(originAttr.dataType) =>
-                // The dataType of the output attributes may be not the same with that of the view
-                // output, so we should cast the attribute to the dataType of the view output
-                // attribute. Will throw an AnalysisException if the cast is not a up-cast.
-                if (!Cast.canUpCast(originAttr.dataType, attr.dataType)) {
-                  throw new AnalysisException(s"Cannot up cast ${originAttr.sql} from " +
-                    s"${originAttr.dataType.catalogString} to ${attr.dataType.catalogString} " +
-                    "as it may truncate\n")
-                }
-              case _ =>
-            }
-
           case alter: AlterTable if alter.table.resolved =>
             val table = alter.table
             def findField(operation: String, fieldName: Array[String]): StructField = {
@@ -494,7 +510,12 @@ trait CheckAnalysis extends PredicateHelper {
                 TypeUtils.failWithIntervalType(add.dataType())
                 colsToAdd(parentName) = fieldsAdded :+ add.fieldNames().last
               case update: UpdateColumnType =>
-                val field = findField("update", update.fieldNames)
+                val field = {
+                  val f = findField("update", update.fieldNames)
+                  CharVarcharUtils.getRawType(f.metadata)
+                    .map(dt => f.copy(dataType = dt))
+                    .getOrElse(f)
+                }
                 val fieldName = update.fieldNames.quoted
                 update.newDataType match {
                   case _: StructType =>
@@ -515,7 +536,16 @@ trait CheckAnalysis extends PredicateHelper {
                   case _ =>
                     // update is okay
                 }
-                if (!Cast.canUpCast(field.dataType, update.newDataType)) {
+
+                // We don't need to handle nested types here which shall fail before
+                def canAlterColumnType(from: DataType, to: DataType): Boolean = (from, to) match {
+                  case (CharType(l1), CharType(l2)) => l1 == l2
+                  case (CharType(l1), VarcharType(l2)) => l1 <= l2
+                  case (VarcharType(l1), VarcharType(l2)) => l1 <= l2
+                  case _ => Cast.canUpCast(from, to)
+                }
+
+                if (!canAlterColumnType(field.dataType, update.newDataType)) {
                   alter.failAnalysis(
                     s"Cannot update ${table.name} field $fieldName: " +
                         s"${field.dataType.simpleString} cannot be cast to " +
@@ -552,7 +582,7 @@ trait CheckAnalysis extends PredicateHelper {
               // no validation needed for set and remove property
             }
 
-          case _ => // Fallbacks to the following checks
+          case _ => // Falls back to the following checks
         }
 
         operator match {
@@ -629,8 +659,7 @@ trait CheckAnalysis extends PredicateHelper {
                """.stripMargin)
 
           case _: UnresolvedHint =>
-            throw new IllegalStateException(
-              "Internal error: logical hint operator should have been removed during analysis")
+            throw QueryExecutionErrors.logicalHintOperatorNotRemovedDuringAnalysisError
 
           case f @ Filter(condition, _)
             if PlanHelper.specialExpressionsInUnsupportedOperator(f).nonEmpty =>
@@ -706,6 +735,11 @@ trait CheckAnalysis extends PredicateHelper {
       case child => child
     }
 
+    // Check whether the given expressions contains the subquery expression.
+    def containsExpr(expressions: Seq[Expression]): Boolean = {
+      expressions.exists(_.find(_.semanticEquals(expr)).isDefined)
+    }
+
     // Validate the subquery plan.
     checkAnalysis(expr.plan)
 
@@ -721,13 +755,22 @@ trait CheckAnalysis extends PredicateHelper {
           cleanQueryInScalarSubquery(query) match {
             case a: Aggregate => checkAggregateInScalarSubquery(conditions, query, a)
             case Filter(_, a: Aggregate) => checkAggregateInScalarSubquery(conditions, query, a)
+            case p: LogicalPlan if p.maxRows.exists(_ <= 1) => // Ok
             case fail => failAnalysis(s"Correlated scalar subqueries must be aggregated: $fail")
           }
 
           // Only certain operators are allowed to host subquery expression containing
           // outer references.
           plan match {
-            case _: Filter | _: Aggregate | _: Project | _: SupportsSubquery => // Ok
+            case _: Filter | _: Project | _: SupportsSubquery => // Ok
+            case a: Aggregate =>
+              // If the correlated scalar subquery is in the grouping expressions of an Aggregate,
+              // it must also be in the aggregate expressions to be rewritten in the optimization
+              // phase.
+              if (containsExpr(a.groupingExpressions) && !containsExpr(a.aggregateExpressions)) {
+                failAnalysis("Correlated scalar subqueries in the group by clause " +
+                  s"must also be in the aggregate expressions:\n$a")
+              }
             case other => failAnalysis(
               "Correlated scalar sub-queries can only be used in a " +
                 s"Filter/Aggregate/Project and a few commands: $plan")
@@ -850,18 +893,76 @@ trait CheckAnalysis extends PredicateHelper {
     // +- SubqueryAlias t1, `t1`
     // +- Project [_1#73 AS c1#76, _2#74 AS c2#77]
     // +- LocalRelation [_1#73, _2#74]
-    def failOnNonEqualCorrelatedPredicate(found: Boolean, p: LogicalPlan): Unit = {
-      if (found) {
+    // SPARK-35080: The same issue can happen to correlated equality predicates when
+    // they do not guarantee one-to-one mapping between inner and outer attributes.
+    // For example:
+    // Table:
+    //   t1(a, b): [(0, 6), (1, 5), (2, 4)]
+    //   t2(c): [(6)]
+    //
+    // Query:
+    //   SELECT c, (SELECT COUNT(*) FROM t1 WHERE a + b = c) FROM t2
+    //
+    // Original subquery plan:
+    //   Aggregate [count(1)]
+    //   +- Filter ((a + b) = outer(c))
+    //      +- LocalRelation [a, b]
+    //
+    // Plan after pulling up correlated predicates:
+    //   Aggregate [a, b] [count(1), a, b]
+    //   +- LocalRelation [a, b]
+    //
+    // Plan after rewrite:
+    //   Project [c1, count(1)]
+    //   +- Join LeftOuter ((a + b) = c)
+    //      :- LocalRelation [c]
+    //      +- Aggregate [a, b] [count(1), a, b]
+    //         +- LocalRelation [a, b]
+    //
+    // The right hand side of the join transformed from the subquery will output
+    //   count(1) | a | b
+    //      1     | 0 | 6
+    //      1     | 1 | 5
+    //      1     | 2 | 4
+    // and the plan after rewrite will give the original query incorrect results.
+    def failOnUnsupportedCorrelatedPredicate(predicates: Seq[Expression], p: LogicalPlan): Unit = {
+      if (predicates.nonEmpty) {
         // Report a non-supported case as an exception
-        failAnalysis(s"Correlated column is not allowed in a non-equality predicate:\n$p")
+        failAnalysis("Correlated column is not allowed in predicate " +
+          s"${predicates.map(_.sql).mkString}:\n$p")
       }
     }
 
-    var foundNonEqualCorrelatedPred: Boolean = false
+    def containsAttribute(e: Expression): Boolean = {
+      e.find(_.isInstanceOf[Attribute]).isDefined
+    }
+
+    // Given a correlated predicate, check if it is either a non-equality predicate or
+    // equality predicate that does not guarantee one-on-one mapping between inner and
+    // outer attributes. When the correlated predicate does not contain any attribute
+    // (i.e. only has outer references), it is supported and should return false. E.G.:
+    //   (a = outer(c)) -> false
+    //   (outer(c) = outer(d)) -> false
+    //   (a > outer(c)) -> true
+    //   (a + b = outer(c)) -> true
+    // The last one is true because there can be multiple combinations of (a, b) that
+    // satisfy the equality condition. For example, if outer(c) = 0, then both (0, 0)
+    // and (-1, 1) can make the predicate evaluate to true.
+    def isUnsupportedPredicate(condition: Expression): Boolean = condition match {
+      // Only allow equality condition with one side being an attribute and another
+      // side being an expression without attributes from the inner query. Note
+      // OuterReference is a leaf node and will not be found here.
+      case Equality(_: Attribute, b) => containsAttribute(b)
+      case Equality(a, _: Attribute) => containsAttribute(a)
+      case e @ Equality(_, _) => containsAttribute(e)
+      case _ => true
+    }
+
+    val unsupportedPredicates = mutable.ArrayBuffer.empty[Expression]
 
     // Simplify the predicates before validating any unsupported correlation patterns in the plan.
     AnalysisHelper.allowInvokingTransformsInAnalyzer { BooleanSimplification(sub).foreachUp {
-      // Whitelist operators allowed in a correlated subquery
+      // Approve operators allowed in a correlated subquery
       // There are 4 categories:
       // 1. Operators that are allowed anywhere in a correlated subquery, and,
       //    by definition of the operators, they either do not contain
@@ -900,22 +1001,17 @@ trait CheckAnalysis extends PredicateHelper {
       // The other operator is Join. Filter can be anywhere in a correlated subquery.
       case f: Filter =>
         val (correlated, _) = splitConjunctivePredicates(f.condition).partition(containsOuter)
-
-        // Find any non-equality correlated predicates
-        foundNonEqualCorrelatedPred = foundNonEqualCorrelatedPred || correlated.exists {
-          case _: EqualTo | _: EqualNullSafe => false
-          case _ => true
-        }
+        unsupportedPredicates ++= correlated.filter(isUnsupportedPredicate)
         failOnInvalidOuterReference(f)
 
       // Aggregate cannot host any correlated expressions
       // It can be on a correlation path if the correlation contains
-      // only equality correlated predicates.
+      // only supported correlated equality predicates.
       // It cannot be on a correlation path if the correlation has
       // non-equality correlated predicates.
       case a: Aggregate =>
         failOnInvalidOuterReference(a)
-        failOnNonEqualCorrelatedPredicate(foundNonEqualCorrelatedPred, a)
+        failOnUnsupportedCorrelatedPredicate(unsupportedPredicates.toSeq, a)
 
       // Join can host correlated expressions.
       case j @ Join(left, right, joinType, _, _) =>
@@ -964,4 +1060,24 @@ trait CheckAnalysis extends PredicateHelper {
         failOnOuterReferenceInSubTree(p)
     }}
   }
+
+  // Make sure that table is able to alter partition.
+  private def checkAlterTablePartition(
+      table: Table, parts: Seq[PartitionSpec]): Unit = {
+    (table, parts) match {
+      case (table, _) if !table.isInstanceOf[SupportsPartitionManagement] =>
+        failAnalysis(s"Table ${table.name()} can not alter partitions.")
+
+      case (_, parts) if parts.exists(_.isInstanceOf[UnresolvedPartitionSpec]) =>
+        failAnalysis("PartitionSpecs are not resolved")
+
+      // Skip atomic partition tables
+      case (_: SupportsAtomicPartitionManagement, _) =>
+      case (_: SupportsPartitionManagement, parts) if parts.size > 1 =>
+        failAnalysis(
+          s"Nonatomic partition table ${table.name()} can not alter multiple partitions.")
+
+      case _ =>
+    }
+  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecision.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecision.scala
index f2d607e5b737c..bf128cd3753e3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecision.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecision.scala
@@ -19,9 +19,6 @@ package org.apache.spark.sql.catalyst.analysis
 
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.Literal._
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 
@@ -82,56 +79,64 @@ object DecimalPrecision extends TypeCoercionRule {
     PromotePrecision(Cast(e, dataType))
   }
 
-  private def nullOnOverflow: Boolean = !SQLConf.get.ansiEnabled
+  override def transform: PartialFunction[Expression, Expression] = {
+    decimalAndDecimal()
+      .orElse(integralAndDecimalLiteral)
+      .orElse(nondecimalAndDecimal(conf.literalPickMinimumPrecision))
+  }
 
-  override protected def coerceTypes(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
-    // fix decimal precision for expressions
-    case q => q.transformExpressionsUp(
-      decimalAndDecimal.orElse(integralAndDecimalLiteral).orElse(nondecimalAndDecimal))
+  private[catalyst] def decimalAndDecimal(): PartialFunction[Expression, Expression] = {
+    decimalAndDecimal(conf.decimalOperationsAllowPrecisionLoss, !conf.ansiEnabled)
   }
 
   /** Decimal precision promotion for +, -, *, /, %, pmod, and binary comparison. */
-  private[catalyst] val decimalAndDecimal: PartialFunction[Expression, Expression] = {
+  private[catalyst] def decimalAndDecimal(allowPrecisionLoss: Boolean, nullOnOverflow: Boolean)
+    : PartialFunction[Expression, Expression] = {
     // Skip nodes whose children have not been resolved yet
     case e if !e.childrenResolved => e
 
     // Skip nodes who is already promoted
     case e: BinaryArithmetic if e.left.isInstanceOf[PromotePrecision] => e
 
-    case Add(e1 @ DecimalType.Expression(p1, s1), e2 @ DecimalType.Expression(p2, s2)) =>
+    case a @ Add(e1 @ DecimalType.Expression(p1, s1), e2 @ DecimalType.Expression(p2, s2), _) =>
       val resultScale = max(s1, s2)
-      val resultType = if (SQLConf.get.decimalOperationsAllowPrecisionLoss) {
+      val resultType = if (allowPrecisionLoss) {
         DecimalType.adjustPrecisionScale(max(p1 - s1, p2 - s2) + resultScale + 1,
           resultScale)
       } else {
         DecimalType.bounded(max(p1 - s1, p2 - s2) + resultScale + 1, resultScale)
       }
-      CheckOverflow(Add(promotePrecision(e1, resultType), promotePrecision(e2, resultType)),
+      CheckOverflow(
+        a.copy(left = promotePrecision(e1, resultType), right = promotePrecision(e2, resultType)),
         resultType, nullOnOverflow)
 
-    case Subtract(e1 @ DecimalType.Expression(p1, s1), e2 @ DecimalType.Expression(p2, s2)) =>
+    case s @ Subtract(e1 @ DecimalType.Expression(p1, s1),
+        e2 @ DecimalType.Expression(p2, s2), _) =>
       val resultScale = max(s1, s2)
-      val resultType = if (SQLConf.get.decimalOperationsAllowPrecisionLoss) {
+      val resultType = if (allowPrecisionLoss) {
         DecimalType.adjustPrecisionScale(max(p1 - s1, p2 - s2) + resultScale + 1,
           resultScale)
       } else {
         DecimalType.bounded(max(p1 - s1, p2 - s2) + resultScale + 1, resultScale)
       }
-      CheckOverflow(Subtract(promotePrecision(e1, resultType), promotePrecision(e2, resultType)),
+      CheckOverflow(
+        s.copy(left = promotePrecision(e1, resultType), right = promotePrecision(e2, resultType)),
         resultType, nullOnOverflow)
 
-    case Multiply(e1 @ DecimalType.Expression(p1, s1), e2 @ DecimalType.Expression(p2, s2)) =>
-      val resultType = if (SQLConf.get.decimalOperationsAllowPrecisionLoss) {
+    case m @ Multiply(
+        e1 @ DecimalType.Expression(p1, s1), e2 @ DecimalType.Expression(p2, s2), _) =>
+      val resultType = if (allowPrecisionLoss) {
         DecimalType.adjustPrecisionScale(p1 + p2 + 1, s1 + s2)
       } else {
         DecimalType.bounded(p1 + p2 + 1, s1 + s2)
       }
       val widerType = widerDecimalType(p1, s1, p2, s2)
-      CheckOverflow(Multiply(promotePrecision(e1, widerType), promotePrecision(e2, widerType)),
+      CheckOverflow(
+        m.copy(left = promotePrecision(e1, widerType), right = promotePrecision(e2, widerType)),
         resultType, nullOnOverflow)
 
-    case Divide(e1 @ DecimalType.Expression(p1, s1), e2 @ DecimalType.Expression(p2, s2)) =>
-      val resultType = if (SQLConf.get.decimalOperationsAllowPrecisionLoss) {
+    case d @ Divide(e1 @ DecimalType.Expression(p1, s1), e2 @ DecimalType.Expression(p2, s2), _) =>
+      val resultType = if (allowPrecisionLoss) {
         // Precision: p1 - s1 + s2 + max(6, s1 + p2 + 1)
         // Scale: max(6, s1 + p2 + 1)
         val intDig = p1 - s1 + s2
@@ -149,37 +154,41 @@ object DecimalPrecision extends TypeCoercionRule {
         DecimalType.bounded(intDig + decDig, decDig)
       }
       val widerType = widerDecimalType(p1, s1, p2, s2)
-      CheckOverflow(Divide(promotePrecision(e1, widerType), promotePrecision(e2, widerType)),
+      CheckOverflow(
+        d.copy(left = promotePrecision(e1, widerType), right = promotePrecision(e2, widerType)),
         resultType, nullOnOverflow)
 
-    case Remainder(e1 @ DecimalType.Expression(p1, s1), e2 @ DecimalType.Expression(p2, s2)) =>
-      val resultType = if (SQLConf.get.decimalOperationsAllowPrecisionLoss) {
+    case r @ Remainder(
+        e1 @ DecimalType.Expression(p1, s1), e2 @ DecimalType.Expression(p2, s2), _) =>
+      val resultType = if (allowPrecisionLoss) {
         DecimalType.adjustPrecisionScale(min(p1 - s1, p2 - s2) + max(s1, s2), max(s1, s2))
       } else {
         DecimalType.bounded(min(p1 - s1, p2 - s2) + max(s1, s2), max(s1, s2))
       }
       // resultType may have lower precision, so we cast them into wider type first.
       val widerType = widerDecimalType(p1, s1, p2, s2)
-      CheckOverflow(Remainder(promotePrecision(e1, widerType), promotePrecision(e2, widerType)),
+      CheckOverflow(
+        r.copy(left = promotePrecision(e1, widerType), right = promotePrecision(e2, widerType)),
         resultType, nullOnOverflow)
 
-    case Pmod(e1 @ DecimalType.Expression(p1, s1), e2 @ DecimalType.Expression(p2, s2)) =>
-      val resultType = if (SQLConf.get.decimalOperationsAllowPrecisionLoss) {
+    case p @ Pmod(e1 @ DecimalType.Expression(p1, s1), e2 @ DecimalType.Expression(p2, s2), _) =>
+      val resultType = if (allowPrecisionLoss) {
         DecimalType.adjustPrecisionScale(min(p1 - s1, p2 - s2) + max(s1, s2), max(s1, s2))
       } else {
         DecimalType.bounded(min(p1 - s1, p2 - s2) + max(s1, s2), max(s1, s2))
       }
       // resultType may have lower precision, so we cast them into wider type first.
       val widerType = widerDecimalType(p1, s1, p2, s2)
-      CheckOverflow(Pmod(promotePrecision(e1, widerType), promotePrecision(e2, widerType)),
+      CheckOverflow(
+        p.copy(left = promotePrecision(e1, widerType), right = promotePrecision(e2, widerType)),
         resultType, nullOnOverflow)
 
     case expr @ IntegralDivide(
-        e1 @ DecimalType.Expression(p1, s1), e2 @ DecimalType.Expression(p2, s2)) =>
+        e1 @ DecimalType.Expression(p1, s1), e2 @ DecimalType.Expression(p2, s2), _) =>
       val widerType = widerDecimalType(p1, s1, p2, s2)
-      val promotedExpr = IntegralDivide(
-        promotePrecision(e1, widerType),
-        promotePrecision(e2, widerType))
+      val promotedExpr = expr.copy(
+        left = promotePrecision(e1, widerType),
+        right = promotePrecision(e2, widerType))
       if (expr.dataType.isInstanceOf[DecimalType]) {
         // This follows division rule
         val intDig = p1 - s1 + s2
@@ -295,7 +304,8 @@ object DecimalPrecision extends TypeCoercionRule {
    * Type coercion for BinaryOperator in which one side is a non-decimal numeric, and the other
    * side is a decimal.
    */
-  private val nondecimalAndDecimal: PartialFunction[Expression, Expression] = {
+  private def nondecimalAndDecimal(literalPickMinimumPrecision: Boolean)
+    : PartialFunction[Expression, Expression] = {
     // Promote integers inside a binary expression with fixed-precision decimals to decimals,
     // and fixed-precision decimals in an expression with floats / doubles to doubles
     case b @ BinaryOperator(left, right) if left.dataType != right.dataType =>
@@ -312,11 +322,11 @@ object DecimalPrecision extends TypeCoercionRule {
         // become DECIMAL(38, 16), safely having a much lower precision loss.
         case (l: Literal, r) if r.dataType.isInstanceOf[DecimalType] &&
             l.dataType.isInstanceOf[IntegralType] &&
-            SQLConf.get.literalPickMinimumPrecision =>
+            literalPickMinimumPrecision =>
           b.makeCopy(Array(Cast(l, DecimalType.fromLiteral(l)), r))
         case (l, r: Literal) if l.dataType.isInstanceOf[DecimalType] &&
             r.dataType.isInstanceOf[IntegralType] &&
-            SQLConf.get.literalPickMinimumPrecision =>
+            literalPickMinimumPrecision =>
           b.makeCopy(Array(l, Cast(r, DecimalType.fromLiteral(r))))
         // Promote integers inside a binary expression with fixed-precision decimals to decimals,
         // and fixed-precision decimals in an expression with floats / doubles to doubles
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DeduplicateRelations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DeduplicateRelations.scala
new file mode 100644
index 0000000000000..85412e48ce861
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DeduplicateRelations.scala
@@ -0,0 +1,252 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import scala.collection.mutable.ArrayBuffer
+
+import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeMap, AttributeSet, NamedExpression, SubqueryExpression}
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.AlwaysProcess
+
+object DeduplicateRelations extends Rule[LogicalPlan] {
+  override def apply(plan: LogicalPlan): LogicalPlan = {
+    renewDuplicatedRelations(Nil, plan)._1.resolveOperatorsUpWithPruning(
+      AlwaysProcess.fn, ruleId) {
+      case p: LogicalPlan if !p.childrenResolved => p
+      // To resolve duplicate expression IDs for Join.
+      case j @ Join(left, right, _, _, _) if !j.duplicateResolved =>
+        j.copy(right = dedupRight(left, right))
+      // intersect/except will be rewritten to join at the beginning of optimizer. Here we need to
+      // deduplicate the right side plan, so that we won't produce an invalid self-join later.
+      case i @ Intersect(left, right, _) if !i.duplicateResolved =>
+        i.copy(right = dedupRight(left, right))
+      case e @ Except(left, right, _) if !e.duplicateResolved =>
+        e.copy(right = dedupRight(left, right))
+      // Only after we finish by-name resolution for Union
+      case u: Union if !u.byName && !u.duplicateResolved =>
+        // Use projection-based de-duplication for Union to avoid breaking the checkpoint sharing
+        // feature in streaming.
+        val newChildren = u.children.foldRight(Seq.empty[LogicalPlan]) { (head, tail) =>
+          head +: tail.map {
+            case child if head.outputSet.intersect(child.outputSet).isEmpty =>
+              child
+            case child =>
+              val projectList = child.output.map { attr =>
+                Alias(attr, attr.name)()
+              }
+              Project(projectList, child)
+          }
+        }
+        u.copy(children = newChildren)
+      case m @ MergeIntoTable(targetTable, sourceTable, _, _, _) if !m.duplicateResolved =>
+        m.copy(sourceTable = dedupRight(targetTable, sourceTable))
+    }
+  }
+
+  /**
+   * Deduplicate any duplicated relations of a LogicalPlan
+   * @param existingRelations the known unique relations for a LogicalPlan
+   * @param plan the LogicalPlan that requires the deduplication
+   * @return (the new LogicalPlan which already deduplicate all duplicated relations (if any),
+   *         all relations of the new LogicalPlan )
+   */
+  private def renewDuplicatedRelations(
+      existingRelations: Seq[MultiInstanceRelation],
+      plan: LogicalPlan): (LogicalPlan, Seq[MultiInstanceRelation]) = plan match {
+    case p: LogicalPlan if p.isStreaming => (plan, Nil)
+
+    case m: MultiInstanceRelation =>
+      if (isDuplicated(existingRelations, m)) {
+        val newNode = m.newInstance()
+        newNode.copyTagsFrom(m)
+        (newNode, Nil)
+      } else {
+        (m, Seq(m))
+      }
+
+    case plan: LogicalPlan =>
+      val relations = ArrayBuffer.empty[MultiInstanceRelation]
+      val newPlan = if (plan.children.nonEmpty) {
+        val newChildren = ArrayBuffer.empty[LogicalPlan]
+        for (c <- plan.children) {
+          val (renewed, collected) = renewDuplicatedRelations(existingRelations ++ relations, c)
+          newChildren += renewed
+          relations ++= collected
+        }
+
+        if (plan.childrenResolved) {
+          val attrMap = AttributeMap(
+            plan
+              .children
+              .flatMap(_.output).zip(newChildren.flatMap(_.output))
+              .filter { case (a1, a2) => a1.exprId != a2.exprId }
+          )
+          plan.withNewChildren(newChildren.toSeq).rewriteAttrs(attrMap)
+        } else {
+          plan.withNewChildren(newChildren.toSeq)
+        }
+      } else {
+        plan
+      }
+
+      val planWithNewSubquery = newPlan.transformExpressions {
+        case subquery: SubqueryExpression =>
+          val (renewed, collected) = renewDuplicatedRelations(
+            existingRelations ++ relations, subquery.plan)
+          relations ++= collected
+          subquery.withNewPlan(renewed)
+      }
+      (planWithNewSubquery, relations.toSeq)
+  }
+
+  private def isDuplicated(
+      existingRelations: Seq[MultiInstanceRelation],
+      relation: MultiInstanceRelation): Boolean = {
+    existingRelations.exists { er =>
+      er.asInstanceOf[LogicalPlan].outputSet
+        .intersect(relation.asInstanceOf[LogicalPlan].outputSet).nonEmpty
+    }
+  }
+
+  /**
+   * Generate a new logical plan for the right child with different expression IDs
+   * for all conflicting attributes.
+   */
+  private def dedupRight(left: LogicalPlan, right: LogicalPlan): LogicalPlan = {
+    val conflictingAttributes = left.outputSet.intersect(right.outputSet)
+    logDebug(s"Conflicting attributes ${conflictingAttributes.mkString(",")} " +
+      s"between $left and $right")
+
+    /**
+     * For LogicalPlan likes MultiInstanceRelation, Project, Aggregate, etc, whose output doesn't
+     * inherit directly from its children, we could just stop collect on it. Because we could
+     * always replace all the lower conflict attributes with the new attributes from the new
+     * plan. Theoretically, we should do recursively collect for Generate and Window but we leave
+     * it to the next batch to reduce possible overhead because this should be a corner case.
+     */
+    def collectConflictPlans(plan: LogicalPlan): Seq[(LogicalPlan, LogicalPlan)] = plan match {
+      // Handle base relations that might appear more than once.
+      case oldVersion: MultiInstanceRelation
+          if oldVersion.outputSet.intersect(conflictingAttributes).nonEmpty =>
+        val newVersion = oldVersion.newInstance()
+        newVersion.copyTagsFrom(oldVersion)
+        Seq((oldVersion, newVersion))
+
+      case oldVersion: SerializeFromObject
+          if oldVersion.outputSet.intersect(conflictingAttributes).nonEmpty =>
+        Seq((oldVersion, oldVersion.copy(
+          serializer = oldVersion.serializer.map(_.newInstance()))))
+
+      // Handle projects that create conflicting aliases.
+      case oldVersion @ Project(projectList, _)
+          if findAliases(projectList).intersect(conflictingAttributes).nonEmpty =>
+        Seq((oldVersion, oldVersion.copy(projectList = newAliases(projectList))))
+
+      // We don't need to search child plan recursively if the projectList of a Project
+      // is only composed of Alias and doesn't contain any conflicting attributes.
+      // Because, even if the child plan has some conflicting attributes, the attributes
+      // will be aliased to non-conflicting attributes by the Project at the end.
+      case _ @ Project(projectList, _)
+        if findAliases(projectList).size == projectList.size =>
+        Nil
+
+      case oldVersion @ Aggregate(_, aggregateExpressions, _)
+          if findAliases(aggregateExpressions).intersect(conflictingAttributes).nonEmpty =>
+        Seq((oldVersion, oldVersion.copy(
+          aggregateExpressions = newAliases(aggregateExpressions))))
+
+      // We don't search the child plan recursively for the same reason as the above Project.
+      case _ @ Aggregate(_, aggregateExpressions, _)
+        if findAliases(aggregateExpressions).size == aggregateExpressions.size =>
+        Nil
+
+      case oldVersion @ FlatMapGroupsInPandas(_, _, output, _)
+          if oldVersion.outputSet.intersect(conflictingAttributes).nonEmpty =>
+        Seq((oldVersion, oldVersion.copy(output = output.map(_.newInstance()))))
+
+      case oldVersion @ FlatMapCoGroupsInPandas(_, _, _, output, _, _)
+        if oldVersion.outputSet.intersect(conflictingAttributes).nonEmpty =>
+        Seq((oldVersion, oldVersion.copy(output = output.map(_.newInstance()))))
+
+      case oldVersion @ MapInPandas(_, output, _)
+        if oldVersion.outputSet.intersect(conflictingAttributes).nonEmpty =>
+        Seq((oldVersion, oldVersion.copy(output = output.map(_.newInstance()))))
+
+      case oldVersion: Generate
+          if oldVersion.producedAttributes.intersect(conflictingAttributes).nonEmpty =>
+        val newOutput = oldVersion.generatorOutput.map(_.newInstance())
+        Seq((oldVersion, oldVersion.copy(generatorOutput = newOutput)))
+
+      case oldVersion: Expand
+          if oldVersion.producedAttributes.intersect(conflictingAttributes).nonEmpty =>
+        val producedAttributes = oldVersion.producedAttributes
+        val newOutput = oldVersion.output.map { attr =>
+          if (producedAttributes.contains(attr)) {
+            attr.newInstance()
+          } else {
+            attr
+          }
+        }
+        Seq((oldVersion, oldVersion.copy(output = newOutput)))
+
+      case oldVersion @ Window(windowExpressions, _, _, child)
+          if AttributeSet(windowExpressions.map(_.toAttribute)).intersect(conflictingAttributes)
+          .nonEmpty =>
+        Seq((oldVersion, oldVersion.copy(windowExpressions = newAliases(windowExpressions))))
+
+      case oldVersion @ ScriptTransformation(_, output, _, _)
+          if AttributeSet(output).intersect(conflictingAttributes).nonEmpty =>
+        Seq((oldVersion, oldVersion.copy(output = output.map(_.newInstance()))))
+
+      case _ => plan.children.flatMap(collectConflictPlans)
+    }
+
+    val conflictPlans = collectConflictPlans(right)
+
+    /*
+     * Note that it's possible `conflictPlans` can be empty which implies that there
+     * is a logical plan node that produces new references that this rule cannot handle.
+     * When that is the case, there must be another rule that resolves these conflicts.
+     * Otherwise, the analysis will fail.
+     */
+    if (conflictPlans.isEmpty) {
+      right
+    } else {
+      val planMapping = conflictPlans.toMap
+      right.transformUpWithNewOutput {
+        case oldPlan =>
+          val newPlanOpt = planMapping.get(oldPlan)
+          newPlanOpt.map { newPlan =>
+            newPlan -> oldPlan.output.zip(newPlan.output)
+          }.getOrElse(oldPlan -> Nil)
+      }
+    }
+  }
+
+  private def newAliases(expressions: Seq[NamedExpression]): Seq[NamedExpression] = {
+    expressions.map {
+      case a: Alias => Alias(a.child, a.name)()
+      case other => other
+    }
+  }
+
+  private def findAliases(projectList: Seq[NamedExpression]): AttributeSet = {
+    AttributeSet(projectList.collect { case a: Alias => a.toAttribute })
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index 26d7b3ef3dcd5..4cc87493b5a25 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -26,11 +26,12 @@ import scala.reflect.ClassTag
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.FunctionIdentifier
-import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate._
 import org.apache.spark.sql.catalyst.expressions.xml._
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Range}
 import org.apache.spark.sql.catalyst.trees.TreeNodeTag
+import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.types._
 
 
@@ -42,7 +43,9 @@ import org.apache.spark.sql.types._
  *   2) the database name is always case-sensitive here, callers are responsible to
  *      format the database name w.r.t. case-sensitive config.
  */
-trait FunctionRegistry {
+trait FunctionRegistryBase[T] {
+
+  type FunctionBuilder = Seq[Expression] => T
 
   final def registerFunction(name: FunctionIdentifier, builder: FunctionBuilder): Unit = {
     val info = new ExpressionInfo(
@@ -63,7 +66,7 @@ trait FunctionRegistry {
   }
 
   @throws[AnalysisException]("If function does not exist")
-  def lookupFunction(name: FunctionIdentifier, children: Seq[Expression]): Expression
+  def lookupFunction(name: FunctionIdentifier, children: Seq[Expression]): T
 
   /* List all of the registered function names. */
   def listFunction(): Seq[FunctionIdentifier]
@@ -82,15 +85,94 @@ trait FunctionRegistry {
 
   /** Clear all registered functions. */
   def clear(): Unit
+}
 
-  /** Create a copy of this registry with identical functions as this registry. */
-  override def clone(): FunctionRegistry = throw new CloneNotSupportedException()
+object FunctionRegistryBase {
+
+  /**
+   * Return an expression info and a function builder for the function as defined by
+   * T using the given name.
+   */
+  def build[T : ClassTag](name: String): (ExpressionInfo, Seq[Expression] => T) = {
+    val runtimeClass = scala.reflect.classTag[T].runtimeClass
+    // For `RuntimeReplaceable`, skip the constructor with most arguments, which is the main
+    // constructor and contains non-parameter `child` and should not be used as function builder.
+    val constructors = if (classOf[RuntimeReplaceable].isAssignableFrom(runtimeClass)) {
+      val all = runtimeClass.getConstructors
+      val maxNumArgs = all.map(_.getParameterCount).max
+      all.filterNot(_.getParameterCount == maxNumArgs)
+    } else {
+      runtimeClass.getConstructors
+    }
+    // See if we can find a constructor that accepts Seq[Expression]
+    val varargCtor = constructors.find(_.getParameterTypes.toSeq == Seq(classOf[Seq[_]]))
+    val builder = (expressions: Seq[Expression]) => {
+      if (varargCtor.isDefined) {
+        // If there is an apply method that accepts Seq[Expression], use that one.
+        try {
+          varargCtor.get.newInstance(expressions).asInstanceOf[T]
+        } catch {
+          // the exception is an invocation exception. To get a meaningful message, we need the
+          // cause.
+          case e: Exception => throw new AnalysisException(e.getCause.getMessage)
+        }
+      } else {
+        // Otherwise, find a constructor method that matches the number of arguments, and use that.
+        val params = Seq.fill(expressions.size)(classOf[Expression])
+        val f = constructors.find(_.getParameterTypes.toSeq == params).getOrElse {
+          val validParametersCount = constructors
+            .filter(_.getParameterTypes.forall(_ == classOf[Expression]))
+            .map(_.getParameterCount).distinct.sorted
+          throw QueryCompilationErrors.invalidFunctionArgumentNumberError(
+            validParametersCount, name, params)
+        }
+        try {
+          f.newInstance(expressions : _*).asInstanceOf[T]
+        } catch {
+          // the exception is an invocation exception. To get a meaningful message, we need the
+          // cause.
+          case e: Exception => throw new AnalysisException(e.getCause.getMessage)
+        }
+      }
+    }
+
+    (expressionInfo(name), builder)
+  }
+
+  /**
+   * Creates an [[ExpressionInfo]] for the function as defined by T using the given name.
+   */
+  def expressionInfo[T : ClassTag](name: String): ExpressionInfo = {
+    val clazz = scala.reflect.classTag[T].runtimeClass
+    val df = clazz.getAnnotation(classOf[ExpressionDescription])
+    if (df != null) {
+      if (df.extended().isEmpty) {
+        new ExpressionInfo(
+          clazz.getCanonicalName,
+          null,
+          name,
+          df.usage(),
+          df.arguments(),
+          df.examples(),
+          df.note(),
+          df.group(),
+          df.since(),
+          df.deprecated())
+      } else {
+        // This exists for the backward compatibility with old `ExpressionDescription`s defining
+        // the extended description in `extended()`.
+        new ExpressionInfo(clazz.getCanonicalName, null, name, df.usage(), df.extended())
+      }
+    } else {
+      new ExpressionInfo(clazz.getCanonicalName, name)
+    }
+  }
 }
 
-class SimpleFunctionRegistry extends FunctionRegistry with Logging {
+trait SimpleFunctionRegistryBase[T] extends FunctionRegistryBase[T] with Logging {
 
   @GuardedBy("this")
-  private val functionBuilders =
+  protected val functionBuilders =
     new mutable.HashMap[FunctionIdentifier, (ExpressionInfo, FunctionBuilder)]
 
   // Resolution of the function name is always case insensitive, but the database name
@@ -112,10 +194,10 @@ class SimpleFunctionRegistry extends FunctionRegistry with Logging {
     }
   }
 
-  override def lookupFunction(name: FunctionIdentifier, children: Seq[Expression]): Expression = {
+  override def lookupFunction(name: FunctionIdentifier, children: Seq[Expression]): T = {
     val func = synchronized {
       functionBuilders.get(normalizeFuncName(name)).map(_._2).getOrElse {
-        throw new AnalysisException(s"undefined function $name")
+        throw QueryCompilationErrors.functionUndefinedError(name)
       }
     }
     func(children)
@@ -141,27 +223,19 @@ class SimpleFunctionRegistry extends FunctionRegistry with Logging {
   override def clear(): Unit = synchronized {
     functionBuilders.clear()
   }
-
-  override def clone(): SimpleFunctionRegistry = synchronized {
-    val registry = new SimpleFunctionRegistry
-    functionBuilders.iterator.foreach { case (name, (info, builder)) =>
-      registry.registerFunction(name, info, builder)
-    }
-    registry
-  }
 }
 
 /**
  * A trivial catalog that returns an error when a function is requested. Used for testing when all
  * functions are already filled in and the analyzer needs only to resolve attribute references.
  */
-object EmptyFunctionRegistry extends FunctionRegistry {
+trait EmptyFunctionRegistryBase[T] extends FunctionRegistryBase[T] {
   override def registerFunction(
       name: FunctionIdentifier, info: ExpressionInfo, builder: FunctionBuilder): Unit = {
     throw new UnsupportedOperationException
   }
 
-  override def lookupFunction(name: FunctionIdentifier, children: Seq[Expression]): Expression = {
+  override def lookupFunction(name: FunctionIdentifier, children: Seq[Expression]): T = {
     throw new UnsupportedOperationException
   }
 
@@ -184,10 +258,33 @@ object EmptyFunctionRegistry extends FunctionRegistry {
   override def clear(): Unit = {
     throw new UnsupportedOperationException
   }
+}
 
-  override def clone(): FunctionRegistry = this
+trait FunctionRegistry extends FunctionRegistryBase[Expression] {
+
+  /** Create a copy of this registry with identical functions as this registry. */
+  override def clone(): FunctionRegistry = throw new CloneNotSupportedException()
 }
 
+class SimpleFunctionRegistry
+    extends SimpleFunctionRegistryBase[Expression]
+    with FunctionRegistry {
+
+  override def clone(): SimpleFunctionRegistry = synchronized {
+    val registry = new SimpleFunctionRegistry
+    functionBuilders.iterator.foreach { case (name, (info, builder)) =>
+      registry.registerFunction(name, info, builder)
+    }
+    registry
+  }
+}
+
+object EmptyFunctionRegistry
+    extends EmptyFunctionRegistryBase[Expression]
+    with FunctionRegistry {
+
+  override def clone(): FunctionRegistry = this
+}
 
 object FunctionRegistry {
 
@@ -274,6 +371,7 @@ object FunctionRegistry {
     expression[Tan]("tan"),
     expression[Cot]("cot"),
     expression[Tanh]("tanh"),
+    expression[WidthBucket]("width_bucket"),
 
     expression[Add]("+"),
     expression[Subtract]("-"),
@@ -281,6 +379,8 @@ object FunctionRegistry {
     expression[Divide]("/"),
     expression[IntegralDivide]("div"),
     expression[Remainder]("%"),
+    expression[TryAdd]("try_add"),
+    expression[TryDivide]("try_divide"),
 
     // aggregate functions
     expression[HyperLogLogPlusPlus]("approx_count_distinct"),
@@ -339,7 +439,7 @@ object FunctionRegistry {
     expression[GetJsonObject]("get_json_object"),
     expression[InitCap]("initcap"),
     expression[StringInstr]("instr"),
-    expression[Lower]("lcase"),
+    expression[Lower]("lcase", true),
     expression[Length]("length"),
     expression[Levenshtein]("levenshtein"),
     expression[Like]("like"),
@@ -350,10 +450,13 @@ object FunctionRegistry {
     expression[StringTrimLeft]("ltrim"),
     expression[JsonTuple]("json_tuple"),
     expression[ParseUrl]("parse_url"),
-    expression[StringLocate]("position"),
+    expression[StringLocate]("position", true),
     expression[FormatString]("printf", true),
     expression[RegExpExtract]("regexp_extract"),
+    expression[RegExpExtractAll]("regexp_extract_all"),
     expression[RegExpReplace]("regexp_replace"),
+    expression[RLike]("regexp_like", true),
+    expression[RLike]("regexp", true),
     expression[StringRepeat]("repeat"),
     expression[StringReplace]("replace"),
     expression[Overlay]("overlay"),
@@ -371,6 +474,7 @@ object FunctionRegistry {
     expression[SubstringIndex]("substring_index"),
     expression[StringTranslate]("translate"),
     expression[StringTrim]("trim"),
+    expression[StringTrimBoth]("btrim"),
     expression[Upper]("ucase", true),
     expression[UnBase64]("unbase64"),
     expression[Unhex]("unhex"),
@@ -389,6 +493,7 @@ object FunctionRegistry {
     expression[AddMonths]("add_months"),
     expression[CurrentDate]("current_date"),
     expression[CurrentTimestamp]("current_timestamp"),
+    expression[CurrentTimeZone]("current_timezone"),
     expression[DateDiff]("datediff"),
     expression[DateAdd]("date_add"),
     expression[DateFormatClass]("date_format"),
@@ -424,6 +529,14 @@ object FunctionRegistry {
     expression[MakeInterval]("make_interval"),
     expression[DatePart]("date_part"),
     expression[Extract]("extract"),
+    expression[DateFromUnixDate]("date_from_unix_date"),
+    expression[UnixDate]("unix_date"),
+    expression[SecondsToTimestamp]("timestamp_seconds"),
+    expression[MillisToTimestamp]("timestamp_millis"),
+    expression[MicrosToTimestamp]("timestamp_micros"),
+    expression[UnixSeconds]("unix_seconds"),
+    expression[UnixMillis]("unix_millis"),
+    expression[UnixMicros]("unix_micros"),
 
     // collection functions
     expression[CreateArray]("array"),
@@ -474,6 +587,7 @@ object FunctionRegistry {
 
     // misc functions
     expression[AssertTrue]("assert_true"),
+    expression[RaiseError]("raise_error"),
     expression[Crc32]("crc32"),
     expression[Md5]("md5"),
     expression[Uuid]("uuid"),
@@ -488,14 +602,13 @@ object FunctionRegistry {
     expression[InputFileBlockLength]("input_file_block_length"),
     expression[MonotonicallyIncreasingID]("monotonically_increasing_id"),
     expression[CurrentDatabase]("current_database"),
+    expression[CurrentCatalog]("current_catalog"),
     expression[CallMethodViaReflection]("reflect"),
     expression[CallMethodViaReflection]("java_method", true),
     expression[SparkVersion]("version"),
     expression[TypeOf]("typeof"),
 
     // grouping sets
-    expression[Cube]("cube"),
-    expression[Rollup]("rollup"),
     expression[Grouping]("grouping"),
     expression[GroupingID]("grouping_id"),
 
@@ -504,6 +617,7 @@ object FunctionRegistry {
     expression[Lag]("lag"),
     expression[RowNumber]("row_number"),
     expression[CumeDist]("cume_dist"),
+    expression[NthValue]("nth_value"),
     expression[NTile]("ntile"),
     expression[Rank]("rank"),
     expression[DenseRank]("dense_rank"),
@@ -534,6 +648,8 @@ object FunctionRegistry {
     expression[BitAndAgg]("bit_and"),
     expression[BitOrAgg]("bit_or"),
     expression[BitXorAgg]("bit_xor"),
+    expression[BitwiseGet]("bit_get"),
+    expression[BitwiseGet]("getbit", true),
 
     // json
     expression[StructsToJson]("to_json"),
@@ -575,66 +691,15 @@ object FunctionRegistry {
   val functionSet: Set[FunctionIdentifier] = builtin.listFunction().toSet
 
   /** See usage above. */
-  private def expression[T <: Expression](name: String, setAlias: Boolean = false)
-      (implicit tag: ClassTag[T]): (String, (ExpressionInfo, FunctionBuilder)) = {
-
-    // For `RuntimeReplaceable`, skip the constructor with most arguments, which is the main
-    // constructor and contains non-parameter `child` and should not be used as function builder.
-    val constructors = if (classOf[RuntimeReplaceable].isAssignableFrom(tag.runtimeClass)) {
-      val all = tag.runtimeClass.getConstructors
-      val maxNumArgs = all.map(_.getParameterCount).max
-      all.filterNot(_.getParameterCount == maxNumArgs)
-    } else {
-      tag.runtimeClass.getConstructors
+  private def expression[T <: Expression : ClassTag](name: String, setAlias: Boolean = false)
+      : (String, (ExpressionInfo, FunctionBuilder)) = {
+    val (expressionInfo, builder) = FunctionRegistryBase.build[T](name)
+    val newBuilder = (expressions: Seq[Expression]) => {
+      val expr = builder(expressions)
+      if (setAlias) expr.setTagValue(FUNC_ALIAS, name)
+      expr
     }
-    // See if we can find a constructor that accepts Seq[Expression]
-    val varargCtor = constructors.find(_.getParameterTypes.toSeq == Seq(classOf[Seq[_]]))
-    val builder = (expressions: Seq[Expression]) => {
-      if (varargCtor.isDefined) {
-        // If there is an apply method that accepts Seq[Expression], use that one.
-        try {
-          val exp = varargCtor.get.newInstance(expressions).asInstanceOf[Expression]
-          if (setAlias) exp.setTagValue(FUNC_ALIAS, name)
-          exp
-        } catch {
-          // the exception is an invocation exception. To get a meaningful message, we need the
-          // cause.
-          case e: Exception => throw new AnalysisException(e.getCause.getMessage)
-        }
-      } else {
-        // Otherwise, find a constructor method that matches the number of arguments, and use that.
-        val params = Seq.fill(expressions.size)(classOf[Expression])
-        val f = constructors.find(_.getParameterTypes.toSeq == params).getOrElse {
-          val validParametersCount = constructors
-            .filter(_.getParameterTypes.forall(_ == classOf[Expression]))
-            .map(_.getParameterCount).distinct.sorted
-          val invalidArgumentsMsg = if (validParametersCount.length == 0) {
-            s"Invalid arguments for function $name"
-          } else {
-            val expectedNumberOfParameters = if (validParametersCount.length == 1) {
-              validParametersCount.head.toString
-            } else {
-              validParametersCount.init.mkString("one of ", ", ", " and ") +
-                validParametersCount.last
-            }
-            s"Invalid number of arguments for function $name. " +
-              s"Expected: $expectedNumberOfParameters; Found: ${params.length}"
-          }
-          throw new AnalysisException(invalidArgumentsMsg)
-        }
-        try {
-          val exp = f.newInstance(expressions : _*).asInstanceOf[Expression]
-          if (setAlias) exp.setTagValue(FUNC_ALIAS, name)
-          exp
-        } catch {
-          // the exception is an invocation exception. To get a meaningful message, we need the
-          // cause.
-          case e: Exception => throw new AnalysisException(e.getCause.getMessage)
-        }
-      }
-    }
-
-    (name, (expressionInfo[T](name), builder))
+    (name, (expressionInfo, newBuilder))
   }
 
   /**
@@ -648,46 +713,18 @@ object FunctionRegistry {
       dataType: DataType): (String, (ExpressionInfo, FunctionBuilder)) = {
     val builder = (args: Seq[Expression]) => {
       if (args.size != 1) {
-        throw new AnalysisException(s"Function $name accepts only one argument")
+        throw QueryCompilationErrors.functionAcceptsOnlyOneArgumentError(name)
       }
       Cast(args.head, dataType)
     }
     val clazz = scala.reflect.classTag[Cast].runtimeClass
     val usage = "_FUNC_(expr) - Casts the value `expr` to the target data type `_FUNC_`."
     val expressionInfo =
-      new ExpressionInfo(clazz.getCanonicalName, null, name, usage, "", "", "", "", "", "")
+      new ExpressionInfo(clazz.getCanonicalName, null, name, usage, "", "", "",
+        "conversion_funcs", "2.0.1", "")
     (name, (expressionInfo, builder))
   }
 
-  /**
-   * Creates an [[ExpressionInfo]] for the function as defined by expression T using the given name.
-   */
-  private def expressionInfo[T <: Expression : ClassTag](name: String): ExpressionInfo = {
-    val clazz = scala.reflect.classTag[T].runtimeClass
-    val df = clazz.getAnnotation(classOf[ExpressionDescription])
-    if (df != null) {
-      if (df.extended().isEmpty) {
-        new ExpressionInfo(
-          clazz.getCanonicalName,
-          null,
-          name,
-          df.usage(),
-          df.arguments(),
-          df.examples(),
-          df.note(),
-          df.group(),
-          df.since(),
-          df.deprecated())
-      } else {
-        // This exists for the backward compatibility with old `ExpressionDescription`s defining
-        // the extended description in `extended()`.
-        new ExpressionInfo(clazz.getCanonicalName, null, name, df.usage(), df.extended())
-      }
-    } else {
-      new ExpressionInfo(clazz.getCanonicalName, name)
-    }
-  }
-
   private def expressionGeneratorOuter[T <: Generator : ClassTag](name: String)
     : (String, (ExpressionInfo, FunctionBuilder)) = {
     val (_, (info, generatorBuilder)) = expression[T](name)
@@ -697,3 +734,66 @@ object FunctionRegistry {
     (name, (info, outerBuilder))
   }
 }
+
+/**
+ * A catalog for looking up table functions.
+ */
+trait TableFunctionRegistry extends FunctionRegistryBase[LogicalPlan] {
+
+  /** Create a copy of this registry with identical functions as this registry. */
+  override def clone(): TableFunctionRegistry = throw new CloneNotSupportedException()
+}
+
+class SimpleTableFunctionRegistry extends SimpleFunctionRegistryBase[LogicalPlan]
+    with TableFunctionRegistry {
+
+  override def clone(): SimpleTableFunctionRegistry = synchronized {
+    val registry = new SimpleTableFunctionRegistry
+    functionBuilders.iterator.foreach { case (name, (info, builder)) =>
+      registry.registerFunction(name, info, builder)
+    }
+    registry
+  }
+}
+
+object EmptyTableFunctionRegistry extends EmptyFunctionRegistryBase[LogicalPlan]
+    with TableFunctionRegistry {
+
+  override def clone(): TableFunctionRegistry = this
+}
+
+object TableFunctionRegistry {
+
+  type TableFunctionBuilder = Seq[Expression] => LogicalPlan
+
+  private def logicalPlan[T <: LogicalPlan : ClassTag](name: String)
+      : (String, (ExpressionInfo, TableFunctionBuilder)) = {
+    val (info, builder) = FunctionRegistryBase.build[T](name)
+    val newBuilder = (expressions: Seq[Expression]) => {
+      try {
+        builder(expressions)
+      } catch {
+        case e: AnalysisException =>
+          val argTypes = expressions.map(_.dataType.typeName).mkString(", ")
+          throw QueryCompilationErrors.cannotApplyTableValuedFunctionError(
+            name, argTypes, info.getUsage, e.getMessage)
+      }
+    }
+    (name, (info, newBuilder))
+  }
+
+  val logicalPlans: Map[String, (ExpressionInfo, TableFunctionBuilder)] = Map(
+    logicalPlan[Range]("range")
+  )
+
+  val builtin: SimpleTableFunctionRegistry = {
+    val fr = new SimpleTableFunctionRegistry
+    logicalPlans.foreach {
+      case (name, (info, builder)) =>
+        fr.registerFunction(FunctionIdentifier(name), info, builder)
+    }
+    fr
+  }
+
+  val functionSet: Set[FunctionIdentifier] = builtin.listFunction().toSet
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/NoSuchItemException.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/NoSuchItemException.scala
index 9b5b059908c00..ba5a9c618c650 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/NoSuchItemException.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/NoSuchItemException.scala
@@ -18,9 +18,11 @@
 package org.apache.spark.sql.catalyst.analysis
 
 import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
 import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
 import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.types.StructType
 
 
 /**
@@ -30,13 +32,15 @@ import org.apache.spark.sql.connector.catalog.Identifier
 class NoSuchDatabaseException(
     val db: String) extends NoSuchNamespaceException(s"Database '$db' not found")
 
-class NoSuchNamespaceException(message: String) extends AnalysisException(message) {
+class NoSuchNamespaceException(message: String, cause: Option[Throwable] = None)
+  extends AnalysisException(message, cause = cause) {
   def this(namespace: Array[String]) = {
     this(s"Namespace '${namespace.quoted}' not found")
   }
 }
 
-class NoSuchTableException(message: String) extends AnalysisException(message) {
+class NoSuchTableException(message: String, cause: Option[Throwable] = None)
+  extends AnalysisException(message, cause = cause) {
   def this(db: String, table: String) = {
     this(s"Table or view '$table' not found in database '$db'")
   }
@@ -46,25 +50,48 @@ class NoSuchTableException(message: String) extends AnalysisException(message) {
   }
 }
 
-class NoSuchPartitionException(
-    db: String,
-    table: String,
-    spec: TablePartitionSpec)
-  extends AnalysisException(
-    s"Partition not found in table '$table' database '$db':\n" + spec.mkString("\n"))
+class NoSuchPartitionException(message: String) extends AnalysisException(message) {
+  def this(db: String, table: String, spec: TablePartitionSpec) = {
+    this(s"Partition not found in table '$table' database '$db':\n" + spec.mkString("\n"))
+  }
+
+  def this(tableName: String, partitionIdent: InternalRow, partitionSchema: StructType) = {
+    this(s"Partition not found in table $tableName: "
+      + partitionIdent.toSeq(partitionSchema).zip(partitionSchema.map(_.name))
+        .map( kv => s"${kv._1} -> ${kv._2}").mkString(","))
+  }
+}
 
 class NoSuchPermanentFunctionException(db: String, func: String)
   extends AnalysisException(s"Function '$func' not found in database '$db'")
 
-class NoSuchFunctionException(db: String, func: String, cause: Option[Throwable] = None)
-  extends AnalysisException(
-    s"Undefined function: '$func'. This function is neither a registered temporary function nor " +
-    s"a permanent function registered in the database '$db'.", cause = cause)
+class NoSuchFunctionException(
+    msg: String,
+    cause: Option[Throwable]) extends AnalysisException(msg, cause = cause) {
 
-class NoSuchPartitionsException(db: String, table: String, specs: Seq[TablePartitionSpec])
-  extends AnalysisException(
-    s"The following partitions not found in table '$table' database '$db':\n"
+  def this(db: String, func: String, cause: Option[Throwable] = None) = {
+    this(s"Undefined function: '$func'. " +
+        s"This function is neither a registered temporary function nor " +
+        s"a permanent function registered in the database '$db'.", cause = cause)
+  }
+
+  def this(identifier: Identifier) = {
+    this(s"Undefined function: ${identifier.quoted}", cause = None)
+  }
+}
+
+class NoSuchPartitionsException(message: String) extends AnalysisException(message) {
+  def this(db: String, table: String, specs: Seq[TablePartitionSpec]) = {
+    this(s"The following partitions not found in table '$table' database '$db':\n"
       + specs.mkString("\n===\n"))
+  }
+
+  def this(tableName: String, partitionIdents: Seq[InternalRow], partitionSchema: StructType) = {
+    this(s"The following partitions not found in table $tableName: "
+      + partitionIdents.map(_.toSeq(partitionSchema).zip(partitionSchema.map(_.name))
+        .map( kv => s"${kv._1} -> ${kv._2}").mkString(",")).mkString("\n===\n"))
+  }
+}
 
 class NoSuchTempFunctionException(func: String)
   extends AnalysisException(s"Temporary function '$func' not found")
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/PullOutNondeterministic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/PullOutNondeterministic.scala
new file mode 100644
index 0000000000000..3431c9327f1d5
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/PullOutNondeterministic.scala
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.Rule
+
+/**
+ * Pulls out nondeterministic expressions from LogicalPlan which is not Project or Filter,
+ * put them into an inner Project and finally project them away at the outer Project.
+ */
+object PullOutNondeterministic extends Rule[LogicalPlan] {
+  override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperatorsUp applyLocally
+
+  val applyLocally: PartialFunction[LogicalPlan, LogicalPlan] = {
+    case p if !p.resolved => p // Skip unresolved nodes.
+    case p: Project => p
+    case f: Filter => f
+
+    case a: Aggregate if a.groupingExpressions.exists(!_.deterministic) =>
+      val nondeterToAttr = getNondeterToAttr(a.groupingExpressions)
+      val newChild = Project(a.child.output ++ nondeterToAttr.values, a.child)
+      a.transformExpressions { case e =>
+        nondeterToAttr.get(e).map(_.toAttribute).getOrElse(e)
+      }.copy(child = newChild)
+
+    // Don't touch collect metrics. Top-level metrics are not supported (check analysis will fail)
+    // and we want to retain them inside the aggregate functions.
+    case m: CollectMetrics => m
+
+    // todo: It's hard to write a general rule to pull out nondeterministic expressions
+    // from LogicalPlan, currently we only do it for UnaryNode which has same output
+    // schema with its child.
+    case p: UnaryNode if p.output == p.child.output && p.expressions.exists(!_.deterministic) =>
+      val nondeterToAttr = getNondeterToAttr(p.expressions)
+      val newPlan = p.transformExpressions { case e =>
+        nondeterToAttr.get(e).map(_.toAttribute).getOrElse(e)
+      }
+      val newChild = Project(p.child.output ++ nondeterToAttr.values, p.child)
+      Project(p.output, newPlan.withNewChildren(newChild :: Nil))
+  }
+
+  private def getNondeterToAttr(exprs: Seq[Expression]): Map[Expression, NamedExpression] = {
+    exprs.filterNot(_.deterministic).flatMap { expr =>
+      val leafNondeterministic = expr.collect {
+        case n: Nondeterministic => n
+        case udf: UserDefinedExpression if !udf.deterministic => udf
+      }
+      leafNondeterministic.distinct.map { e =>
+        val ne = e match {
+          case n: NamedExpression => n
+          case _ => Alias(e, "_nondeterministic")()
+        }
+        e -> ne
+      }
+    }.toMap
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala
index 2a0a944e4849c..3b876882162f5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala
@@ -17,10 +17,9 @@
 
 package org.apache.spark.sql.catalyst.analysis
 
-import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.connector.catalog.{CatalogManager, CatalogPlugin, LookupCatalog, SupportsNamespaces, TableCatalog, TableChange}
+import org.apache.spark.sql.connector.catalog.{CatalogManager, CatalogPlugin, LookupCatalog, TableChange}
 
 /**
  * Resolves catalogs from the multi-part identifiers in SQL statements, and convert the statements
@@ -34,7 +33,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
   override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
     case AlterTableAddColumnsStatement(
          nameParts @ NonSessionCatalogAndTable(catalog, tbl), cols) =>
-      cols.foreach(c => failCharType(c.dataType))
+      cols.foreach(c => failNullType(c.dataType))
       val changes = cols.map { col =>
         TableChange.addColumn(
           col.name.toArray,
@@ -47,7 +46,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
 
     case AlterTableReplaceColumnsStatement(
         nameParts @ NonSessionCatalogAndTable(catalog, tbl), cols) =>
-      cols.foreach(c => failCharType(c.dataType))
+      cols.foreach(c => failNullType(c.dataType))
       val changes: Seq[TableChange] = loadTable(catalog, tbl.asIdentifier) match {
         case Some(table) =>
           // REPLACE COLUMNS deletes all the existing columns and adds new columns specified.
@@ -69,7 +68,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
 
     case a @ AlterTableAlterColumnStatement(
          nameParts @ NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _) =>
-      a.dataType.foreach(failCharType)
+      a.dataType.foreach(failNullType)
       val colName = a.column.toArray
       val typeChange = a.dataType.map { newDataType =>
         TableChange.updateColumnType(colName, newDataType)
@@ -99,109 +98,60 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
       val changes = cols.map(col => TableChange.deleteColumn(col.toArray))
       createAlterTable(nameParts, catalog, tbl, changes)
 
-    case AlterTableSetPropertiesStatement(
-         nameParts @ NonSessionCatalogAndTable(catalog, tbl), props) =>
-      val changes = props.map { case (key, value) =>
-        TableChange.setProperty(key, value)
-      }.toSeq
-      createAlterTable(nameParts, catalog, tbl, changes)
-
-    // TODO: v2 `UNSET TBLPROPERTIES` should respect the ifExists flag.
-    case AlterTableUnsetPropertiesStatement(
-         nameParts @ NonSessionCatalogAndTable(catalog, tbl), keys, _) =>
-      val changes = keys.map(key => TableChange.removeProperty(key))
-      createAlterTable(nameParts, catalog, tbl, changes)
-
-    case AlterTableSetLocationStatement(
-         nameParts @ NonSessionCatalogAndTable(catalog, tbl), partitionSpec, newLoc) =>
-      if (partitionSpec.nonEmpty) {
-        throw new AnalysisException(
-          "ALTER TABLE SET LOCATION does not support partition for v2 tables.")
-      }
-      val changes = Seq(TableChange.setProperty(TableCatalog.PROP_LOCATION, newLoc))
-      createAlterTable(nameParts, catalog, tbl, changes)
-
-    case AlterViewSetPropertiesStatement(
-         NonSessionCatalogAndTable(catalog, tbl), props) =>
-      throw new AnalysisException(
-        s"Can not specify catalog `${catalog.name}` for view ${tbl.quoted} " +
-          s"because view support in catalog has not been implemented yet")
-
-    case AlterViewUnsetPropertiesStatement(
-         NonSessionCatalogAndTable(catalog, tbl), keys, ifExists) =>
-      throw new AnalysisException(
-        s"Can not specify catalog `${catalog.name}` for view ${tbl.quoted} " +
-          s"because view support in catalog has not been implemented yet")
-
-    case RenameTableStatement(NonSessionCatalogAndTable(catalog, oldName), newNameParts, isView) =>
-      if (isView) {
-        throw new AnalysisException("Renaming view is not supported in v2 catalogs.")
-      }
-      RenameTable(catalog.asTableCatalog, oldName.asIdentifier, newNameParts.asIdentifier)
-
-    case DescribeColumnStatement(
-         NonSessionCatalogAndTable(catalog, tbl), colNameParts, isExtended) =>
-      throw new AnalysisException("Describing columns is not supported for v2 tables.")
-
     case c @ CreateTableStatement(
-         NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
-      assertNoCharTypeInSchema(c.tableSchema)
+         NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _, _) =>
+      assertNoNullTypeInSchema(c.tableSchema)
       CreateV2Table(
         catalog.asTableCatalog,
         tbl.asIdentifier,
         c.tableSchema,
         // convert the bucket spec and add it as a transform
         c.partitioning ++ c.bucketSpec.map(_.asTransform),
-        convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
+        convertTableProperties(c),
         ignoreIfExists = c.ifNotExists)
 
     case c @ CreateTableAsSelectStatement(
-         NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _) =>
+         NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _, _, _) =>
+      if (c.asSelect.resolved) {
+        assertNoNullTypeInSchema(c.asSelect.schema)
+      }
       CreateTableAsSelect(
         catalog.asTableCatalog,
         tbl.asIdentifier,
         // convert the bucket spec and add it as a transform
         c.partitioning ++ c.bucketSpec.map(_.asTransform),
         c.asSelect,
-        convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
+        convertTableProperties(c),
         writeOptions = c.writeOptions,
         ignoreIfExists = c.ifNotExists)
 
-    case RefreshTableStatement(NonSessionCatalogAndTable(catalog, tbl)) =>
-      RefreshTable(catalog.asTableCatalog, tbl.asIdentifier)
-
     case c @ ReplaceTableStatement(
-         NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
-      assertNoCharTypeInSchema(c.tableSchema)
+         NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _) =>
+      assertNoNullTypeInSchema(c.tableSchema)
       ReplaceTable(
         catalog.asTableCatalog,
         tbl.asIdentifier,
         c.tableSchema,
         // convert the bucket spec and add it as a transform
         c.partitioning ++ c.bucketSpec.map(_.asTransform),
-        convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
+        convertTableProperties(c),
         orCreate = c.orCreate)
 
     case c @ ReplaceTableAsSelectStatement(
-         NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _) =>
+         NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _, _) =>
+      if (c.asSelect.resolved) {
+        assertNoNullTypeInSchema(c.asSelect.schema)
+      }
       ReplaceTableAsSelect(
         catalog.asTableCatalog,
         tbl.asIdentifier,
         // convert the bucket spec and add it as a transform
         c.partitioning ++ c.bucketSpec.map(_.asTransform),
         c.asSelect,
-        convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
+        convertTableProperties(c),
         writeOptions = c.writeOptions,
         orCreate = c.orCreate)
 
-    case DropTableStatement(NonSessionCatalogAndTable(catalog, tbl), ifExists, _) =>
-      DropTable(catalog.asTableCatalog, tbl.asIdentifier, ifExists)
-
-    case DropViewStatement(NonSessionCatalogAndTable(catalog, viewName), _) =>
-      throw new AnalysisException(
-        s"Can not specify catalog `${catalog.name}` for view ${viewName.quoted} " +
-          s"because view support in catalog has not been implemented yet")
-
     case c @ CreateNamespaceStatement(CatalogAndNamespace(catalog, ns), _, _)
         if !isSessionCatalog(catalog) =>
       CreateNamespace(catalog.asNamespaceCatalog, ns, c.ifNotExists, c.properties)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCommandsWithIfExists.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCommandsWithIfExists.scala
new file mode 100644
index 0000000000000..60f86b31a4bdf
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCommandsWithIfExists.scala
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import org.apache.spark.sql.catalyst.plans.logical.{DropTable, DropView, LogicalPlan, NoopCommand, UncacheTable}
+import org.apache.spark.sql.catalyst.rules.Rule
+
+/**
+ * A rule for handling commands when the table or temp view is not resolved.
+ * These commands support a flag, "ifExists", so that they do not fail when a relation is not
+ * resolved. If the "ifExists" flag is set to true. the plan is resolved to [[NoopCommand]],
+ */
+object ResolveCommandsWithIfExists extends Rule[LogicalPlan] {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
+    case DropTable(u: UnresolvedTableOrView, ifExists, _) if ifExists =>
+      NoopCommand("DROP TABLE", u.multipartIdentifier)
+    case DropView(u: UnresolvedView, ifExists) if ifExists =>
+      NoopCommand("DROP VIEW", u.multipartIdentifier)
+    case UncacheTable(u: UnresolvedRelation, ifExists, _) if ifExists =>
+      NoopCommand("UNCACHE TABLE", u.multipartIdentifier)
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveHints.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveHints.scala
index 81de086e78f91..e33450d22bfd2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveHints.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveHints.scala
@@ -21,11 +21,11 @@ import java.util.Locale
 
 import scala.collection.mutable
 
-import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.expressions.{Ascending, Expression, IntegerLiteral, SortOrder}
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.catalyst.trees.CurrentOrigin
+import org.apache.spark.sql.catalyst.trees.{AlwaysProcess, CurrentOrigin}
+import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.internal.SQLConf
 
 
@@ -50,10 +50,10 @@ object ResolveHints {
    *
    * This rule must happen before common table expressions.
    */
-  class ResolveJoinStrategyHints(conf: SQLConf) extends Rule[LogicalPlan] {
+  object ResolveJoinStrategyHints extends Rule[LogicalPlan] {
     private val STRATEGY_HINT_NAMES = JoinStrategyHint.strategies.flatMap(_.hintAliases)
 
-    private val hintErrorHandler = conf.hintErrorHandler
+    private def hintErrorHandler = conf.hintErrorHandler
 
     def resolver: Resolver = conf.resolver
 
@@ -105,7 +105,7 @@ object ResolveHints {
 
       val newNode = CurrentOrigin.withOrigin(plan.origin) {
         plan match {
-          case ResolvedHint(u @ UnresolvedRelation(ident), hint)
+          case ResolvedHint(u @ UnresolvedRelation(ident, _, _), hint)
               if matchedIdentifierInHint(ident) =>
             ResolvedHint(u, createHintInfo(hintName).merge(hint, hintErrorHandler))
 
@@ -113,7 +113,7 @@ object ResolveHints {
               if matchedIdentifierInHint(extractIdentifier(r)) =>
             ResolvedHint(r, createHintInfo(hintName).merge(hint, hintErrorHandler))
 
-          case UnresolvedRelation(ident) if matchedIdentifierInHint(ident) =>
+          case UnresolvedRelation(ident, _, _) if matchedIdentifierInHint(ident) =>
             ResolvedHint(plan, createHintInfo(hintName))
 
           case r: SubqueryAlias if matchedIdentifierInHint(extractIdentifier(r)) =>
@@ -143,7 +143,8 @@ object ResolveHints {
       }
     }
 
-    def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperatorsUp {
+    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
+      AlwaysProcess.fn, ruleId) {
       case h: UnresolvedHint if STRATEGY_HINT_NAMES.contains(h.name.toUpperCase(Locale.ROOT)) =>
         if (h.parameters.isEmpty) {
           // If there is no table alias specified, apply the hint on the entire subtree.
@@ -153,8 +154,8 @@ object ResolveHints {
           val relationNamesInHint = h.parameters.map {
             case tableName: String => UnresolvedAttribute.parseAttributeName(tableName)
             case tableId: UnresolvedAttribute => tableId.nameParts
-            case unsupported => throw new AnalysisException("Join strategy hint parameter " +
-              s"should be an identifier or string but was $unsupported (${unsupported.getClass}")
+            case unsupported =>
+              throw QueryCompilationErrors.joinStrategyHintParameterNotSupportedError(unsupported)
           }.toSet
           val relationsInHintWithMatch = new mutable.HashSet[Seq[String]]
           val applied = applyJoinStrategyHint(
@@ -171,7 +172,9 @@ object ResolveHints {
   /**
    * COALESCE Hint accepts names "COALESCE", "REPARTITION", and "REPARTITION_BY_RANGE".
    */
-  class ResolveCoalesceHints(conf: SQLConf) extends Rule[LogicalPlan] {
+  object ResolveCoalesceHints extends Rule[LogicalPlan] {
+
+    val COALESCE_HINT_NAMES: Set[String] = Set("COALESCE", "REPARTITION", "REPARTITION_BY_RANGE")
 
     /**
      * This function handles hints for "COALESCE" and "REPARTITION".
@@ -183,16 +186,14 @@ object ResolveHints {
       val hintName = hint.name.toUpperCase(Locale.ROOT)
 
       def createRepartitionByExpression(
-          numPartitions: Int, partitionExprs: Seq[Any]): RepartitionByExpression = {
+          numPartitions: Option[Int], partitionExprs: Seq[Any]): RepartitionByExpression = {
         val sortOrders = partitionExprs.filter(_.isInstanceOf[SortOrder])
-        if (sortOrders.nonEmpty) throw new IllegalArgumentException(
-          s"""Invalid partitionExprs specified: $sortOrders
-             |For range partitioning use REPARTITION_BY_RANGE instead.
-           """.stripMargin)
+        if (sortOrders.nonEmpty) {
+          throw QueryCompilationErrors.invalidRepartitionExpressionsError(sortOrders)
+        }
         val invalidParams = partitionExprs.filter(!_.isInstanceOf[UnresolvedAttribute])
         if (invalidParams.nonEmpty) {
-          throw new AnalysisException(s"$hintName Hint parameter should include columns, but " +
-            s"${invalidParams.mkString(", ")} found")
+          throw QueryCompilationErrors.invalidHintParameterError(hintName, invalidParams)
         }
         RepartitionByExpression(
           partitionExprs.map(_.asInstanceOf[Expression]), hint.child, numPartitions)
@@ -205,14 +206,14 @@ object ResolveHints {
           Repartition(numPartitions, shuffle, hint.child)
         // The "COALESCE" hint (shuffle = false) must have a partition number only
         case _ if !shuffle =>
-          throw new AnalysisException(s"$hintName Hint expects a partition number as a parameter")
+          throw QueryCompilationErrors.invalidCoalesceHintParameterError(hintName)
 
         case param @ Seq(IntegerLiteral(numPartitions), _*) if shuffle =>
-          createRepartitionByExpression(numPartitions, param.tail)
+          createRepartitionByExpression(Some(numPartitions), param.tail)
         case param @ Seq(numPartitions: Int, _*) if shuffle =>
-          createRepartitionByExpression(numPartitions, param.tail)
+          createRepartitionByExpression(Some(numPartitions), param.tail)
         case param @ Seq(_*) if shuffle =>
-          createRepartitionByExpression(conf.numShufflePartitions, param)
+          createRepartitionByExpression(None, param)
       }
     }
 
@@ -224,11 +225,10 @@ object ResolveHints {
       val hintName = hint.name.toUpperCase(Locale.ROOT)
 
       def createRepartitionByExpression(
-          numPartitions: Int, partitionExprs: Seq[Any]): RepartitionByExpression = {
+          numPartitions: Option[Int], partitionExprs: Seq[Any]): RepartitionByExpression = {
         val invalidParams = partitionExprs.filter(!_.isInstanceOf[UnresolvedAttribute])
         if (invalidParams.nonEmpty) {
-          throw new AnalysisException(s"$hintName Hint parameter should include columns, but " +
-            s"${invalidParams.mkString(", ")} found")
+          throw QueryCompilationErrors.invalidHintParameterError(hintName, invalidParams)
         }
         val sortOrder = partitionExprs.map {
           case expr: SortOrder => expr
@@ -239,15 +239,16 @@ object ResolveHints {
 
       hint.parameters match {
         case param @ Seq(IntegerLiteral(numPartitions), _*) =>
-          createRepartitionByExpression(numPartitions, param.tail)
+          createRepartitionByExpression(Some(numPartitions), param.tail)
         case param @ Seq(numPartitions: Int, _*) =>
-          createRepartitionByExpression(numPartitions, param.tail)
+          createRepartitionByExpression(Some(numPartitions), param.tail)
         case param @ Seq(_*) =>
-          createRepartitionByExpression(conf.numShufflePartitions, param)
+          createRepartitionByExpression(None, param)
       }
     }
 
-    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperators {
+    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsWithPruning(
+      AlwaysProcess.fn, ruleId) {
       case hint @ UnresolvedHint(hintName, _, _) => hintName.toUpperCase(Locale.ROOT) match {
           case "REPARTITION" =>
             createRepartition(shuffle = true, hint)
@@ -260,17 +261,13 @@ object ResolveHints {
     }
   }
 
-  object ResolveCoalesceHints {
-    val COALESCE_HINT_NAMES: Set[String] = Set("COALESCE", "REPARTITION", "REPARTITION_BY_RANGE")
-  }
-
   /**
    * Removes all the hints, used to remove invalid hints provided by the user.
    * This must be executed after all the other hint rules are executed.
    */
-  class RemoveAllHints(conf: SQLConf) extends Rule[LogicalPlan] {
+  class RemoveAllHints extends Rule[LogicalPlan] {
 
-    private val hintErrorHandler = conf.hintErrorHandler
+    private def hintErrorHandler = conf.hintErrorHandler
 
     def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperatorsUp {
       case h: UnresolvedHint =>
@@ -278,4 +275,15 @@ object ResolveHints {
         h.child
     }
   }
+
+  /**
+   * Removes all the hints when `spark.sql.optimizer.disableHints` is set.
+   * This is executed at the very beginning of the Analyzer to disable
+   * the hint functionality.
+   */
+  class DisableHints extends RemoveAllHints {
+    override def apply(plan: LogicalPlan): LogicalPlan = {
+      if (conf.getConf(SQLConf.DISABLE_HINTS)) super.apply(plan) else plan
+    }
+  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveInlineTables.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveInlineTables.scala
index 4edfe507a7580..91d724dc0135a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveInlineTables.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveInlineTables.scala
@@ -22,14 +22,15 @@ import scala.util.control.NonFatal
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan}
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.catalyst.trees.AlwaysProcess
 import org.apache.spark.sql.types.{StructField, StructType}
 
 /**
  * An analyzer rule that replaces [[UnresolvedInlineTable]] with [[LocalRelation]].
  */
-case class ResolveInlineTables(conf: SQLConf) extends Rule[LogicalPlan] with CastSupport {
-  override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
+object ResolveInlineTables extends Rule[LogicalPlan] with CastSupport {
+  override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsWithPruning(
+    AlwaysProcess.fn, ruleId) {
     case table: UnresolvedInlineTable if table.expressionsResolved =>
       validateInputDimension(table)
       validateInputEvaluable(table)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolvePartitionSpec.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolvePartitionSpec.scala
new file mode 100644
index 0000000000000..79b7b0c5ba35e
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolvePartitionSpec.scala
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
+import org.apache.spark.sql.catalyst.expressions.{Cast, Literal}
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, V2PartitionCommand}
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.util.CharVarcharUtils
+import org.apache.spark.sql.connector.catalog.SupportsPartitionManagement
+import org.apache.spark.sql.types._
+import org.apache.spark.sql.util.PartitioningUtils.{normalizePartitionSpec, requireExactMatchedPartitionSpec}
+
+/**
+ * Resolve [[UnresolvedPartitionSpec]] to [[ResolvedPartitionSpec]] in partition related commands.
+ */
+object ResolvePartitionSpec extends Rule[LogicalPlan] {
+
+  def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
+    case command: V2PartitionCommand if command.childrenResolved && !command.resolved =>
+      command.table match {
+        case r @ ResolvedTable(_, _, table: SupportsPartitionManagement, _) =>
+          command.transformExpressions {
+            case partSpecs: UnresolvedPartitionSpec =>
+              val partitionSchema = table.partitionSchema()
+              resolvePartitionSpec(
+                r.name,
+                partSpecs,
+                partitionSchema,
+                command.allowPartialPartitionSpec)
+          }
+        case _ => command
+      }
+  }
+
+  private def resolvePartitionSpec(
+      tableName: String,
+      partSpec: UnresolvedPartitionSpec,
+      partSchema: StructType,
+      allowPartitionSpec: Boolean): ResolvedPartitionSpec = {
+    val normalizedSpec = normalizePartitionSpec(
+      partSpec.spec,
+      partSchema,
+      tableName,
+      conf.resolver)
+    if (!allowPartitionSpec) {
+      requireExactMatchedPartitionSpec(tableName, normalizedSpec, partSchema.fieldNames)
+    }
+    val partitionNames = normalizedSpec.keySet
+    val requestedFields = partSchema.filter(field => partitionNames.contains(field.name))
+    ResolvedPartitionSpec(
+      requestedFields.map(_.name),
+      convertToPartIdent(normalizedSpec, requestedFields),
+      partSpec.location)
+  }
+
+  private[sql] def convertToPartIdent(
+      partitionSpec: TablePartitionSpec,
+      schema: Seq[StructField]): InternalRow = {
+    val partValues = schema.map { part =>
+      val raw = partitionSpec.get(part.name).orNull
+      val dt = CharVarcharUtils.replaceCharVarcharWithString(part.dataType)
+      Cast(Literal.create(raw, StringType), dt, Some(conf.sessionLocalTimeZone)).eval()
+    }
+    InternalRow.fromSeq(partValues)
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableValuedFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableValuedFunctions.scala
index 983e4b0e901cf..a3f7ec5d3b8d8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableValuedFunctions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableValuedFunctions.scala
@@ -17,146 +17,42 @@
 
 package org.apache.spark.sql.catalyst.analysis
 
-import java.util.Locale
-
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.expressions.{Alias, Expression}
-import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project, Range}
+import org.apache.spark.sql.catalyst.catalog.SessionCatalog
+import org.apache.spark.sql.catalyst.expressions.Alias
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project}
 import org.apache.spark.sql.catalyst.rules._
-import org.apache.spark.sql.types.{DataType, IntegerType, LongType}
 
 /**
  * Rule that resolves table-valued function references.
  */
-object ResolveTableValuedFunctions extends Rule[LogicalPlan] {
-  /**
-   * List of argument names and their types, used to declare a function.
-   */
-  private case class ArgumentList(args: (String, DataType)*) {
-    /**
-     * Try to cast the expressions to satisfy the expected types of this argument list. If there
-     * are any types that cannot be casted, then None is returned.
-     */
-    def implicitCast(values: Seq[Expression]): Option[Seq[Expression]] = {
-      if (args.length == values.length) {
-        val casted = values.zip(args).map { case (value, (_, expectedType)) =>
-          TypeCoercion.ImplicitTypeCasts.implicitCast(value, expectedType)
-        }
-        if (casted.forall(_.isDefined)) {
-          return Some(casted.map(_.get))
-        }
-      }
-      None
-    }
-
-    override def toString: String = {
-      args.map { a =>
-        s"${a._1}: ${a._2.typeName}"
-      }.mkString(", ")
-    }
-  }
-
-  /**
-   * A TVF maps argument lists to resolver functions that accept those arguments. Using a map
-   * here allows for function overloading.
-   */
-  private type TVF = Map[ArgumentList, Seq[Any] => LogicalPlan]
-
-  /**
-   * TVF builder.
-   */
-  private def tvf(args: (String, DataType)*)(pf: PartialFunction[Seq[Any], LogicalPlan])
-      : (ArgumentList, Seq[Any] => LogicalPlan) = {
-    (ArgumentList(args: _*),
-     pf orElse {
-       case arguments =>
-         // This is caught again by the apply function and rethrow with richer information about
-         // position, etc, for a better error message.
-         throw new AnalysisException(
-           "Invalid arguments for resolved function: " + arguments.mkString(", "))
-     })
-  }
-
-  /**
-   * Internal registry of table-valued functions.
-   */
-  private val builtinFunctions: Map[String, TVF] = Map(
-    "range" -> Map(
-      /* range(end) */
-      tvf("end" -> LongType) { case Seq(end: Long) =>
-        Range(0, end, 1, None)
-      },
-
-      /* range(start, end) */
-      tvf("start" -> LongType, "end" -> LongType) { case Seq(start: Long, end: Long) =>
-        Range(start, end, 1, None)
-      },
-
-      /* range(start, end, step) */
-      tvf("start" -> LongType, "end" -> LongType, "step" -> LongType) {
-        case Seq(start: Long, end: Long, step: Long) =>
-          Range(start, end, step, None)
-      },
-
-      /* range(start, end, step, numPartitions) */
-      tvf("start" -> LongType, "end" -> LongType, "step" -> LongType,
-          "numPartitions" -> IntegerType) {
-        case Seq(start: Long, end: Long, step: Long, numPartitions: Int) =>
-          Range(start, end, step, Some(numPartitions))
-      })
-  )
+case class ResolveTableValuedFunctions(catalog: SessionCatalog) extends Rule[LogicalPlan] {
 
   override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
     case u: UnresolvedTableValuedFunction if u.functionArgs.forall(_.resolved) =>
-      // The whole resolution is somewhat difficult to understand here due to too much abstractions.
-      // We should probably rewrite the following at some point. Reynold was just here to improve
-      // error messages and didn't have time to do a proper rewrite.
-      val resolvedFunc = builtinFunctions.get(u.functionName.toLowerCase(Locale.ROOT)) match {
-        case Some(tvf) =>
-
-          def failAnalysis(): Nothing = {
-            val argTypes = u.functionArgs.map(_.dataType.typeName).mkString(", ")
+      withPosition(u) {
+        val resolvedFunc = try {
+          catalog.lookupTableFunction(u.name, u.functionArgs)
+        } catch {
+          case _: NoSuchFunctionException =>
+            u.failAnalysis(s"could not resolve `${u.name}` to a table-valued function")
+        }
+        // If alias names assigned, add `Project` with the aliases
+        if (u.outputNames.nonEmpty) {
+          val outputAttrs = resolvedFunc.output
+          // Checks if the number of the aliases is equal to expected one
+          if (u.outputNames.size != outputAttrs.size) {
             u.failAnalysis(
-              s"""error: table-valued function ${u.functionName} with alternatives:
-                 |${tvf.keys.map(_.toString).toSeq.sorted.map(x => s" ($x)").mkString("\n")}
-                 |cannot be applied to: ($argTypes)""".stripMargin)
+              s"Number of given aliases does not match number of output columns. " +
+                s"Function name: ${u.name}; number of aliases: " +
+                s"${u.outputNames.size}; number of output columns: ${outputAttrs.size}.")
           }
-
-          val resolved = tvf.flatMap { case (argList, resolver) =>
-            argList.implicitCast(u.functionArgs) match {
-              case Some(casted) =>
-                try {
-                  Some(resolver(casted.map(_.eval())))
-                } catch {
-                  case e: AnalysisException =>
-                    failAnalysis()
-                }
-              case _ =>
-                None
-            }
-          }
-          resolved.headOption.getOrElse {
-            failAnalysis()
+          val aliases = outputAttrs.zip(u.outputNames).map {
+            case (attr, name) => Alias(attr, name)()
           }
-        case _ =>
-          u.failAnalysis(s"could not resolve `${u.functionName}` to a table-valued function")
-      }
-
-      // If alias names assigned, add `Project` with the aliases
-      if (u.outputNames.nonEmpty) {
-        val outputAttrs = resolvedFunc.output
-        // Checks if the number of the aliases is equal to expected one
-        if (u.outputNames.size != outputAttrs.size) {
-          u.failAnalysis(s"Number of given aliases does not match number of output columns. " +
-            s"Function name: ${u.functionName}; number of aliases: " +
-            s"${u.outputNames.size}; number of output columns: ${outputAttrs.size}.")
-        }
-        val aliases = outputAttrs.zip(u.outputNames).map {
-          case (attr, name) => Alias(attr, name)()
+          Project(aliases, resolvedFunc)
+        } else {
+          resolvedFunc
         }
-        Project(aliases, resolvedFunc)
-      } else {
-        resolvedFunc
       }
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveUnion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveUnion.scala
new file mode 100644
index 0000000000000..5715a4118de9b
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveUnion.scala
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import scala.collection.mutable
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.optimizer.{CombineUnions, OptimizeUpdateFields}
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project, Union}
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.AlwaysProcess
+import org.apache.spark.sql.types._
+import org.apache.spark.sql.util.SchemaUtils
+import org.apache.spark.unsafe.types.UTF8String
+
+/**
+ * Resolves different children of Union to a common set of columns.
+ */
+object ResolveUnion extends Rule[LogicalPlan] {
+  /**
+   * This method sorts columns recursively in a struct expression based on column names.
+   */
+  private def sortStructFields(expr: Expression): Expression = {
+    val existingExprs = expr.dataType.asInstanceOf[StructType].fieldNames.zipWithIndex.map {
+      case (name, i) =>
+        val fieldExpr = GetStructField(KnownNotNull(expr), i)
+        if (fieldExpr.dataType.isInstanceOf[StructType]) {
+          (name, sortStructFields(fieldExpr))
+        } else {
+          (name, fieldExpr)
+        }
+    }.sortBy(_._1).flatMap(pair => Seq(Literal(pair._1), pair._2))
+
+    val newExpr = CreateNamedStruct(existingExprs)
+    if (expr.nullable) {
+      If(IsNull(expr), Literal(null, newExpr.dataType), newExpr)
+    } else {
+      newExpr
+    }
+  }
+
+  /**
+   * Assumes input expressions are field expression of `CreateNamedStruct`. This method
+   * sorts the expressions based on field names.
+   */
+  private def sortFieldExprs(fieldExprs: Seq[Expression]): Seq[Expression] = {
+    fieldExprs.grouped(2).map { e =>
+      Seq(e.head, e.last)
+    }.toSeq.sortBy { pair =>
+      assert(pair.head.isInstanceOf[Literal])
+      pair.head.eval().asInstanceOf[UTF8String].toString
+    }.flatten
+  }
+
+  /**
+   * This helper method sorts fields in a `UpdateFields` expression by field name.
+   */
+  private def sortStructFieldsInWithFields(expr: Expression): Expression = expr transformUp {
+    case u: UpdateFields if u.resolved =>
+      u.evalExpr match {
+        case i @ If(IsNull(_), _, CreateNamedStruct(fieldExprs)) =>
+          val sorted = sortFieldExprs(fieldExprs)
+          val newStruct = CreateNamedStruct(sorted)
+          i.copy(trueValue = Literal(null, newStruct.dataType), falseValue = newStruct)
+        case CreateNamedStruct(fieldExprs) =>
+          val sorted = sortFieldExprs(fieldExprs)
+          val newStruct = CreateNamedStruct(sorted)
+          newStruct
+        case other =>
+          throw new IllegalStateException(s"`UpdateFields` has incorrect expression: $other. " +
+            "Please file a bug report with this error message, stack trace, and the query.")
+      }
+  }
+
+  /**
+   * Adds missing fields recursively into given `col` expression, based on the target `StructType`.
+   * This is called by `compareAndAddFields` when we find two struct columns with same name but
+   * different nested fields. This method will find out the missing nested fields from `col` to
+   * `target` struct and add these missing nested fields. Currently we don't support finding out
+   * missing nested fields of struct nested in array or struct nested in map.
+   */
+  private def addFields(col: NamedExpression, target: StructType): Expression = {
+    assert(col.dataType.isInstanceOf[StructType], "Only support StructType.")
+
+    val resolver = conf.resolver
+    val missingFieldsOpt =
+      StructType.findMissingFields(col.dataType.asInstanceOf[StructType], target, resolver)
+
+    // We need to sort columns in result, because we might add another column in other side.
+    // E.g., we want to union two structs "a int, b long" and "a int, c string".
+    // If we don't sort, we will have "a int, b long, c string" and
+    // "a int, c string, b long", which are not compatible.
+    if (missingFieldsOpt.isEmpty) {
+      sortStructFields(col)
+    } else {
+      missingFieldsOpt.map { s =>
+        val struct = addFieldsInto(col, s.fields)
+        // Combines `WithFields`s to reduce expression tree.
+        val reducedStruct = struct.transformUp(OptimizeUpdateFields.optimizeUpdateFields)
+        val sorted = sortStructFieldsInWithFields(reducedStruct)
+        sorted
+      }.get
+    }
+  }
+
+  /**
+   * Adds missing fields recursively into given `col` expression. The missing fields are given
+   * in `fields`. For example, given `col` as "z struct<z:int, y:int>, x int", and `fields` is
+   * "z struct<w:long>, w string". This method will add a nested `z.w` field and a top-level
+   * `w` field to `col` and fill null values for them. Note that because we might also add missing
+   * fields at other side of Union, we must make sure corresponding attributes at two sides have
+   * same field order in structs, so when we adding missing fields, we will sort the fields based on
+   * field names. So the data type of returned expression will be
+   * "w string, x int, z struct<w:long, y:int, z:int>".
+   */
+  private def addFieldsInto(
+      col: Expression,
+      fields: Seq[StructField]): Expression = {
+    fields.foldLeft(col) { case (currCol, field) =>
+      field.dataType match {
+        case st: StructType =>
+          val resolver = conf.resolver
+          val colField = currCol.dataType.asInstanceOf[StructType]
+            .find(f => resolver(f.name, field.name))
+          if (colField.isEmpty) {
+            // The whole struct is missing. Add a null.
+            UpdateFields(currCol, field.name, Literal(null, st))
+          } else {
+            UpdateFields(currCol, field.name,
+              addFieldsInto(ExtractValue(currCol, Literal(field.name), resolver), st.fields))
+          }
+        case dt =>
+          UpdateFields(currCol, field.name, Literal(null, dt))
+      }
+    }
+  }
+
+  /**
+   * This method will compare right to left plan's outputs. If there is one struct attribute
+   * at right side has same name with left side struct attribute, but two structs are not the
+   * same data type, i.e., some missing (nested) fields at right struct attribute, then this
+   * method will try to add missing (nested) fields into the right attribute with null values.
+   */
+  private def compareAndAddFields(
+      left: LogicalPlan,
+      right: LogicalPlan,
+      allowMissingCol: Boolean): (Seq[NamedExpression], Seq[NamedExpression]) = {
+    val resolver = conf.resolver
+    val leftOutputAttrs = left.output
+    val rightOutputAttrs = right.output
+
+    val aliased = mutable.ArrayBuffer.empty[Attribute]
+
+    val rightProjectList = leftOutputAttrs.map { lattr =>
+      val found = rightOutputAttrs.find { rattr => resolver(lattr.name, rattr.name) }
+      if (found.isDefined) {
+        val foundAttr = found.get
+        val foundDt = foundAttr.dataType
+        (foundDt, lattr.dataType) match {
+          case (source: StructType, target: StructType)
+              if allowMissingCol && !source.sameType(target) =>
+            // Having an output with same name, but different struct type.
+            // We need to add missing fields. Note that if there are deeply nested structs such as
+            // nested struct of array in struct, we don't support to add missing deeply nested field
+            // like that. We will sort columns in the struct expression to make sure two sides of
+            // union have consistent schema.
+            aliased += foundAttr
+            Alias(addFields(foundAttr, target), foundAttr.name)()
+          case _ =>
+            // We don't need/try to add missing fields if:
+            // 1. The attributes of left and right side are the same struct type
+            // 2. The attributes are not struct types. They might be primitive types, or array, map
+            //    types. We don't support adding missing fields of nested structs in array or map
+            //    types now.
+            // 3. `allowMissingCol` is disabled.
+            foundAttr
+        }
+      } else {
+        if (allowMissingCol) {
+          Alias(Literal(null, lattr.dataType), lattr.name)()
+        } else {
+          throw new AnalysisException(
+            s"""Cannot resolve column name "${lattr.name}" among """ +
+              s"""(${rightOutputAttrs.map(_.name).mkString(", ")})""")
+        }
+      }
+    }
+
+    (rightProjectList, aliased.toSeq)
+  }
+
+  private def unionTwoSides(
+      left: LogicalPlan,
+      right: LogicalPlan,
+      allowMissingCol: Boolean): LogicalPlan = {
+    val rightOutputAttrs = right.output
+
+    // Builds a project list for `right` based on `left` output names
+    val (rightProjectList, aliased) = compareAndAddFields(left, right, allowMissingCol)
+
+    // Delegates failure checks to `CheckAnalysis`
+    val notFoundAttrs = rightOutputAttrs.diff(rightProjectList ++ aliased)
+    val rightChild = Project(rightProjectList ++ notFoundAttrs, right)
+
+    // Builds a project for `logicalPlan` based on `right` output names, if allowing
+    // missing columns.
+    val leftChild = if (allowMissingCol) {
+      // Add missing (nested) fields to left plan.
+      val (leftProjectList, _) = compareAndAddFields(rightChild, left, allowMissingCol)
+      if (leftProjectList.map(_.toAttribute) != left.output) {
+        Project(leftProjectList, left)
+      } else {
+        left
+      }
+    } else {
+      left
+    }
+    Union(leftChild, rightChild)
+  }
+
+  // Check column name duplication
+  private def checkColumnNames(left: LogicalPlan, right: LogicalPlan): Unit = {
+    val caseSensitiveAnalysis = conf.caseSensitiveAnalysis
+    val leftOutputAttrs = left.output
+    val rightOutputAttrs = right.output
+
+    SchemaUtils.checkColumnNameDuplication(
+      leftOutputAttrs.map(_.name),
+      "in the left attributes",
+      caseSensitiveAnalysis)
+    SchemaUtils.checkColumnNameDuplication(
+      rightOutputAttrs.map(_.name),
+      "in the right attributes",
+      caseSensitiveAnalysis)
+  }
+
+  def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
+    AlwaysProcess.fn, ruleId) {
+    case e if !e.childrenResolved => e
+
+    case Union(children, byName, allowMissingCol) if byName =>
+      val union = children.reduceLeft { (left, right) =>
+        checkColumnNames(left, right)
+        unionTwoSides(left, right, allowMissingCol)
+      }
+      CombineUnions(union)
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/StreamingJoinHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/StreamingJoinHelper.scala
index 6a2ff4b91e68d..d8e200d6b01e4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/StreamingJoinHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/StreamingJoinHelper.scala
@@ -55,7 +55,7 @@ object StreamingJoinHelper extends PredicateHelper with Logging {
    * given the join condition and the event time watermark. This is how it works.
    * - The condition is split into conjunctive predicates, and we find the predicates of the
    *   form `leftTime + c1 < rightTime + c2`   (or <=, >, >=).
-   * - We canoncalize the predicate and solve it with the event time watermark value to find the
+   * - We canonicalize the predicate and solve it with the event time watermark value to find the
    *  value of the state watermark.
    * This function is supposed to make best-effort attempt to get the state watermark. If there is
    * any error, it will return None.
@@ -94,7 +94,7 @@ object StreamingJoinHelper extends PredicateHelper with Logging {
 
       // The generated the state watermark cleanup expression is inclusive of the state watermark.
       // If state watermark is W, all state where timestamp <= W will be cleaned up.
-      // Now when the canonicalized join condition solves to leftTime >= W, we dont want to clean
+      // Now when the canonicalized join condition solves to leftTime >= W, we don't want to clean
       // up leftTime <= W. Rather we should clean up leftTime <= W - 1. Hence the -1 below.
       val stateWatermark = predicate match {
         case LessThan(l, r) => getStateWatermarkSafely(l, r)
@@ -189,7 +189,7 @@ object StreamingJoinHelper extends PredicateHelper with Logging {
           if attributesWithEventWatermark.contains(a) && metadata.contains(delayKey) =>
           Multiply(Literal(eventWatermark.get.toDouble), Literal(1000.0))
       }
-    }.reduceLeft(Add)
+    }.reduceLeft(Add(_, _))
 
     // Calculate the constraint value
     logInfo(s"Final expression to evaluate constraint:\t$exprWithWatermarkSubstituted")
@@ -226,14 +226,14 @@ object StreamingJoinHelper extends PredicateHelper with Logging {
      */
     def collect(expr: Expression, negate: Boolean): Seq[Expression] = {
       expr match {
-        case Add(left, right) =>
+        case Add(left, right, _) =>
           collect(left, negate) ++ collect(right, negate)
-        case Subtract(left, right) =>
+        case Subtract(left, right, _) =>
           collect(left, negate) ++ collect(right, !negate)
         case TimeAdd(left, right, _) =>
           collect(left, negate) ++ collect(right, negate)
         case DatetimeSub(_, _, child) => collect(child, negate)
-        case UnaryMinus(child) =>
+        case UnaryMinus(child, _) =>
           collect(child, !negate)
         case CheckOverflow(child, _, _) =>
           collect(child, negate)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/SubstituteUnresolvedOrdinals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/SubstituteUnresolvedOrdinals.scala
index 860d20f897690..6f92d7fce72e4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/SubstituteUnresolvedOrdinals.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/SubstituteUnresolvedOrdinals.scala
@@ -17,24 +17,32 @@
 
 package org.apache.spark.sql.catalyst.analysis
 
-import org.apache.spark.sql.catalyst.expressions.{Expression, Literal, SortOrder}
+import org.apache.spark.sql.catalyst.expressions.{BaseGroupingSets, Expression, Literal, SortOrder}
 import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LogicalPlan, Sort}
 import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.AlwaysProcess
 import org.apache.spark.sql.catalyst.trees.CurrentOrigin.withOrigin
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.IntegerType
 
 /**
  * Replaces ordinal in 'order by' or 'group by' with UnresolvedOrdinal expression.
  */
-class SubstituteUnresolvedOrdinals(conf: SQLConf) extends Rule[LogicalPlan] {
-  private def isIntLiteral(e: Expression) = e match {
+object SubstituteUnresolvedOrdinals extends Rule[LogicalPlan] {
+  private def containIntLiteral(e: Expression): Boolean = e match {
     case Literal(_, IntegerType) => true
+    case gs: BaseGroupingSets => gs.children.exists(containIntLiteral)
     case _ => false
   }
 
-  def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
-    case s: Sort if conf.orderByOrdinal && s.order.exists(o => isIntLiteral(o.child)) =>
+  private def substituteUnresolvedOrdinal(expression: Expression): Expression = expression match {
+    case ordinal @ Literal(index: Int, IntegerType) =>
+      withOrigin(ordinal.origin)(UnresolvedOrdinal(index))
+    case e => e
+  }
+
+  def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsWithPruning(
+    AlwaysProcess.fn, ruleId) {
+    case s: Sort if conf.orderByOrdinal && s.order.exists(o => containIntLiteral(o.child)) =>
       val newOrders = s.order.map {
         case order @ SortOrder(ordinal @ Literal(index: Int, IntegerType), _, _, _) =>
           val newOrdinal = withOrigin(ordinal.origin)(UnresolvedOrdinal(index))
@@ -43,10 +51,12 @@ class SubstituteUnresolvedOrdinals(conf: SQLConf) extends Rule[LogicalPlan] {
       }
       withOrigin(s.origin)(s.copy(order = newOrders))
 
-    case a: Aggregate if conf.groupByOrdinal && a.groupingExpressions.exists(isIntLiteral) =>
+    case a: Aggregate if conf.groupByOrdinal && a.groupingExpressions.exists(containIntLiteral) =>
       val newGroups = a.groupingExpressions.map {
         case ordinal @ Literal(index: Int, IntegerType) =>
           withOrigin(ordinal.origin)(UnresolvedOrdinal(index))
+        case gs: BaseGroupingSets =>
+          withOrigin(gs.origin)(gs.withNewChildren(gs.children.map(substituteUnresolvedOrdinal)))
         case other => other
       }
       withOrigin(a.origin)(a.copy(groupingExpressions = newGroups))
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TableOutputResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TableOutputResolver.scala
index 4f33ca99c02db..d5c407b47c5be 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TableOutputResolver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TableOutputResolver.scala
@@ -22,6 +22,7 @@ import scala.collection.mutable
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.expressions.{Alias, AnsiCast, Attribute, Cast, NamedExpression}
 import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project}
+import org.apache.spark.sql.catalyst.util.CharVarcharUtils
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy
 import org.apache.spark.sql.types.DataType
@@ -93,19 +94,17 @@ object TableOutputResolver {
       tableAttr.metadata == queryExpr.metadata) {
       Some(queryExpr)
     } else {
-      // Renaming is needed for handling the following cases like
-      // 1) Column names/types do not match, e.g., INSERT INTO TABLE tab1 SELECT 1, 2
-      // 2) Target tables have column metadata
-      storeAssignmentPolicy match {
+      val casted = storeAssignmentPolicy match {
         case StoreAssignmentPolicy.ANSI =>
-          Some(Alias(
-            AnsiCast(queryExpr, tableAttr.dataType, Option(conf.sessionLocalTimeZone)),
-            tableAttr.name)(explicitMetadata = Option(tableAttr.metadata)))
+          AnsiCast(queryExpr, tableAttr.dataType, Option(conf.sessionLocalTimeZone))
         case _ =>
-          Some(Alias(
-            Cast(queryExpr, tableAttr.dataType, Option(conf.sessionLocalTimeZone)),
-            tableAttr.name)(explicitMetadata = Option(tableAttr.metadata)))
+          Cast(queryExpr, tableAttr.dataType, Option(conf.sessionLocalTimeZone))
       }
+      val exprWithStrLenCheck = CharVarcharUtils.stringLengthCheck(casted, tableAttr)
+      // Renaming is needed for handling the following cases like
+      // 1) Column names/types do not match, e.g., INSERT INTO TABLE tab1 SELECT 1, 2
+      // 2) Target tables have column metadata
+      Some(Alias(exprWithStrLenCheck, tableAttr.name)(explicitMetadata = Some(tableAttr.metadata)))
     }
 
     storeAssignmentPolicy match {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
index d970bf466fb81..2aa6543a55c98 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
@@ -31,125 +31,52 @@ import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
-
-/**
- * A collection of [[Rule]] that can be used to coerce differing types that participate in
- * operations into compatible ones.
- *
- * Notes about type widening / tightest common types: Broadly, there are two cases when we need
- * to widen data types (e.g. union, binary comparison). In case 1, we are looking for a common
- * data type for two or more data types, and in this case no loss of precision is allowed. Examples
- * include type inference in JSON (e.g. what's the column's data type if one row is an integer
- * while the other row is a long?). In case 2, we are looking for a widened data type with
- * some acceptable loss of precision (e.g. there is no common type for double and decimal because
- * double's range is larger than decimal, and yet decimal is more precise than double, but in
- * union we would cast the decimal into double).
- */
-object TypeCoercion {
-
-  def typeCoercionRules(conf: SQLConf): List[Rule[LogicalPlan]] =
-    InConversion(conf) ::
-      WidenSetOperationTypes ::
-      PromoteStrings(conf) ::
-      DecimalPrecision ::
-      BooleanEquality ::
-      FunctionArgumentConversion ::
-      ConcatCoercion(conf) ::
-      MapZipWithCoercion ::
-      EltCoercion(conf) ::
-      CaseWhenCoercion ::
-      IfCoercion ::
-      StackCoercion ::
-      Division ::
-      ImplicitTypeCasts ::
-      DateTimeOperations ::
-      WindowFrameCoercion ::
-      StringLiteralCoercion ::
-      Nil
-
-  // See https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types.
-  // The conversion for integral and floating point types have a linear widening hierarchy:
-  val numericPrecedence =
-    IndexedSeq(
-      ByteType,
-      ShortType,
-      IntegerType,
-      LongType,
-      FloatType,
-      DoubleType)
+abstract class TypeCoercionBase {
+  /**
+   * A collection of [[Rule]] that can be used to coerce differing types that participate in
+   * operations into compatible ones.
+   */
+  def typeCoercionRules: List[Rule[LogicalPlan]]
 
   /**
-   * Case 1 type widening (see the classdoc comment above for TypeCoercion).
-   *
    * Find the tightest common type of two types that might be used in a binary expression.
    * This handles all numeric types except fixed-precision decimals interacting with each other or
    * with primitive types, because in that case the precision and scale of the result depends on
    * the operation. Those rules are implemented in [[DecimalPrecision]].
    */
-  val findTightestCommonType: (DataType, DataType) => Option[DataType] = {
-    case (t1, t2) if t1 == t2 => Some(t1)
-    case (NullType, t1) => Some(t1)
-    case (t1, NullType) => Some(t1)
-
-    case (t1: IntegralType, t2: DecimalType) if t2.isWiderThan(t1) =>
-      Some(t2)
-    case (t1: DecimalType, t2: IntegralType) if t1.isWiderThan(t2) =>
-      Some(t1)
-
-    // Promote numeric types to the highest of the two
-    case (t1: NumericType, t2: NumericType)
-        if !t1.isInstanceOf[DecimalType] && !t2.isInstanceOf[DecimalType] =>
-      val index = numericPrecedence.lastIndexWhere(t => t == t1 || t == t2)
-      Some(numericPrecedence(index))
-
-    case (_: TimestampType, _: DateType) | (_: DateType, _: TimestampType) =>
-      Some(TimestampType)
-
-    case (t1, t2) => findTypeForComplex(t1, t2, findTightestCommonType)
-  }
-
-  /** Promotes all the way to StringType. */
-  private def stringPromotion(dt1: DataType, dt2: DataType): Option[DataType] = (dt1, dt2) match {
-    case (StringType, t2: AtomicType) if t2 != BinaryType && t2 != BooleanType => Some(StringType)
-    case (t1: AtomicType, StringType) if t1 != BinaryType && t1 != BooleanType => Some(StringType)
-    case _ => None
-  }
+  val findTightestCommonType: (DataType, DataType) => Option[DataType]
 
   /**
-   * This function determines the target type of a comparison operator when one operand
-   * is a String and the other is not. It also handles when one op is a Date and the
-   * other is a Timestamp by making the target type to be String.
+   * Looking for a widened data type of two given data types with some acceptable loss of precision.
+   * E.g. there is no common type for double and decimal because double's range
+   * is larger than decimal, and yet decimal is more precise than double, but in
+   * union we would cast the decimal into double.
    */
-  private def findCommonTypeForBinaryComparison(
-      dt1: DataType, dt2: DataType, conf: SQLConf): Option[DataType] = (dt1, dt2) match {
-    case (StringType, DateType)
-      => if (conf.castDatetimeToString) Some(StringType) else Some(DateType)
-    case (DateType, StringType)
-      => if (conf.castDatetimeToString) Some(StringType) else Some(DateType)
-    case (StringType, TimestampType)
-      => if (conf.castDatetimeToString) Some(StringType) else Some(TimestampType)
-    case (TimestampType, StringType)
-      => if (conf.castDatetimeToString) Some(StringType) else Some(TimestampType)
-    case (StringType, NullType) => Some(StringType)
-    case (NullType, StringType) => Some(StringType)
+  def findWiderTypeForTwo(t1: DataType, t2: DataType): Option[DataType]
 
-    // Cast to TimestampType when we compare DateType with TimestampType
-    // i.e. TimeStamp('2017-03-01 00:00:00') eq Date('2017-03-01') = true
-    case (TimestampType, DateType) => Some(TimestampType)
-    case (DateType, TimestampType) => Some(TimestampType)
+  /**
+   * Looking for a widened data type of a given sequence of data types with some acceptable loss
+   * of precision.
+   * E.g. there is no common type for double and decimal because double's range
+   * is larger than decimal, and yet decimal is more precise than double, but in
+   * union we would cast the decimal into double.
+   */
+  def findWiderCommonType(types: Seq[DataType]): Option[DataType]
 
-    // There is no proper decimal type we can pick,
-    // using double type is the best we can do.
-    // See SPARK-22469 for details.
-    case (n: DecimalType, s: StringType) => Some(DoubleType)
-    case (s: StringType, n: DecimalType) => Some(DoubleType)
+  /**
+   * Given an expected data type, try to cast the expression and return the cast expression.
+   *
+   * If the expression already fits the input type, we simply return the expression itself.
+   * If the expression has an incompatible type that cannot be implicitly cast, return None.
+   */
+  def implicitCast(e: Expression, expectedType: AbstractDataType): Option[Expression]
 
-    case (l: StringType, r: AtomicType) if r != StringType => Some(r)
-    case (l: AtomicType, r: StringType) if l != StringType => Some(l)
-    case (l, r) => None
-  }
+  /**
+   * Whether casting `from` as `to` is valid.
+   */
+  def canCast(from: DataType, to: DataType): Boolean
 
-  private def findTypeForComplex(
+  protected def findTypeForComplex(
       t1: DataType,
       t2: DataType,
       findTypeFunc: (DataType, DataType) => Option[DataType]): Option[DataType] = (t1, t2) match {
@@ -181,65 +108,24 @@ object TypeCoercion {
   }
 
   /**
-   * The method finds a common type for data types that differ only in nullable flags, including
-   * `nullable`, `containsNull` of [[ArrayType]] and `valueContainsNull` of [[MapType]].
-   * If the input types are different besides nullable flags, None is returned.
+   * Finds a wider type when one or both types are decimals. If the wider decimal type exceeds
+   * system limitation, this rule will truncate the decimal type. If a decimal and other fractional
+   * types are compared, returns a double type.
    */
-  def findCommonTypeDifferentOnlyInNullFlags(t1: DataType, t2: DataType): Option[DataType] = {
-    if (t1 == t2) {
-      Some(t1)
-    } else {
-      findTypeForComplex(t1, t2, findCommonTypeDifferentOnlyInNullFlags)
-    }
-  }
-
-  def findCommonTypeDifferentOnlyInNullFlags(types: Seq[DataType]): Option[DataType] = {
-    if (types.isEmpty) {
-      None
-    } else {
-      types.tail.foldLeft[Option[DataType]](Some(types.head)) {
-        case (Some(t1), t2) => findCommonTypeDifferentOnlyInNullFlags(t1, t2)
-        case _ => None
-      }
+  protected def findWiderTypeForDecimal(dt1: DataType, dt2: DataType): Option[DataType] = {
+    (dt1, dt2) match {
+      case (t1: DecimalType, t2: DecimalType) =>
+        Some(DecimalPrecision.widerDecimalType(t1, t2))
+      case (t: IntegralType, d: DecimalType) =>
+        Some(DecimalPrecision.widerDecimalType(DecimalType.forType(t), d))
+      case (d: DecimalType, t: IntegralType) =>
+        Some(DecimalPrecision.widerDecimalType(DecimalType.forType(t), d))
+      case (_: FractionalType, _: DecimalType) | (_: DecimalType, _: FractionalType) =>
+        Some(DoubleType)
+      case _ => None
     }
   }
 
-  /**
-   * Case 2 type widening (see the classdoc comment above for TypeCoercion).
-   *
-   * i.e. the main difference with [[findTightestCommonType]] is that here we allow some
-   * loss of precision when widening decimal and double, and promotion to string.
-   */
-  def findWiderTypeForTwo(t1: DataType, t2: DataType): Option[DataType] = {
-    findTightestCommonType(t1, t2)
-      .orElse(findWiderTypeForDecimal(t1, t2))
-      .orElse(stringPromotion(t1, t2))
-      .orElse(findTypeForComplex(t1, t2, findWiderTypeForTwo))
-  }
-
-  /**
-   * Whether the data type contains StringType.
-   */
-  def hasStringType(dt: DataType): Boolean = dt match {
-    case StringType => true
-    case ArrayType(et, _) => hasStringType(et)
-    // Add StructType if we support string promotion for struct fields in the future.
-    case _ => false
-  }
-
-  private def findWiderCommonType(types: Seq[DataType]): Option[DataType] = {
-    // findWiderTypeForTwo doesn't satisfy the associative law, i.e. (a op b) op c may not equal
-    // to a op (b op c). This is only a problem for StringType or nested StringType in ArrayType.
-    // Excluding these types, findWiderTypeForTwo satisfies the associative law. For instance,
-    // (TimestampType, IntegerType, StringType) should have StringType as the wider common type.
-    val (stringTypes, nonStringTypes) = types.partition(hasStringType(_))
-    (stringTypes.distinct ++ nonStringTypes).foldLeft[Option[DataType]](Some(NullType))((r, c) =>
-      r match {
-        case Some(d) => findWiderTypeForTwo(d, c)
-        case _ => None
-      })
-  }
-
   /**
    * Similar to [[findWiderTypeForTwo]] that can handle decimal types, but can't promote to
    * string. If the wider decimal type exceeds system limitation, this rule will truncate
@@ -260,25 +146,6 @@ object TypeCoercion {
     })
   }
 
-  /**
-   * Finds a wider type when one or both types are decimals. If the wider decimal type exceeds
-   * system limitation, this rule will truncate the decimal type. If a decimal and other fractional
-   * types are compared, returns a double type.
-   */
-  private def findWiderTypeForDecimal(dt1: DataType, dt2: DataType): Option[DataType] = {
-    (dt1, dt2) match {
-      case (t1: DecimalType, t2: DecimalType) =>
-        Some(DecimalPrecision.widerDecimalType(t1, t2))
-      case (t: IntegralType, d: DecimalType) =>
-        Some(DecimalPrecision.widerDecimalType(DecimalType.forType(t), d))
-      case (d: DecimalType, t: IntegralType) =>
-        Some(DecimalPrecision.widerDecimalType(DecimalType.forType(t), d))
-      case (_: FractionalType, _: DecimalType) | (_: DecimalType, _: FractionalType) =>
-        Some(DoubleType)
-      case _ => None
-    }
-  }
-
   /**
    * Check whether the given types are equal ignoring nullable, containsNull and valueContainsNull.
    */
@@ -300,50 +167,91 @@ object TypeCoercion {
   }
 
   /**
-   * Widens numeric types and converts strings to numbers when appropriate.
-   *
-   * Loosely based on rules from "Hadoop: The Definitive Guide" 2nd edition, by Tom White
+   * Type coercion rule that combines multiple type coercion rules and applies them in a single tree
+   * traversal.
+   */
+  case class CombinedTypeCoercionRule(rules: Seq[TypeCoercionRule]) extends TypeCoercionRule {
+    override def transform: PartialFunction[Expression, Expression] = {
+      val transforms = rules.map(_.transform)
+      Function.unlift { e: Expression =>
+        val result = transforms.foldLeft(e) {
+          case (current, transform) => transform.applyOrElse(current, identity[Expression])
+        }
+        if (result ne e) {
+          Some(result)
+        } else {
+          None
+        }
+      }
+    }
+
+    override val ruleName: String = rules.map(_.ruleName).mkString("Combined[", ", ", "]")
+  }
+
+  /**
+   * Widens the data types of the children of Union/Except/Intersect.
+   * 1. When ANSI mode is off:
+   *   Loosely based on rules from "Hadoop: The Definitive Guide" 2nd edition, by Tom White
    *
-   * The implicit conversion rules can be summarized as follows:
-   *   - Any integral numeric type can be implicitly converted to a wider type.
-   *   - All the integral numeric types, FLOAT, and (perhaps surprisingly) STRING can be implicitly
-   *     converted to DOUBLE.
-   *   - TINYINT, SMALLINT, and INT can all be converted to FLOAT.
-   *   - BOOLEAN types cannot be converted to any other type.
-   *   - Any integral numeric type can be implicitly converted to decimal type.
-   *   - two different decimal types will be converted into a wider decimal type for both of them.
-   *   - decimal type will be converted into double if there float or double together with it.
+   *   The implicit conversion rules can be summarized as follows:
+   *     - Any integral numeric type can be implicitly converted to a wider type.
+   *     - All the integral numeric types, FLOAT, and (perhaps surprisingly) STRING can be
+   *       implicitly converted to DOUBLE.
+   *     - TINYINT, SMALLINT, and INT can all be converted to FLOAT.
+   *     - BOOLEAN types cannot be converted to any other type.
+   *     - Any integral numeric type can be implicitly converted to decimal type.
+   *     - two different decimal types will be converted into a wider decimal type for both of them.
+   *     - decimal type will be converted into double if there float or double together with it.
    *
-   * Additionally, all types when UNION-ed with strings will be promoted to strings.
-   * Other string conversions are handled by PromoteStrings.
+   *   All types when UNION-ed with strings will be promoted to
+   *   strings. Other string conversions are handled by PromoteStrings.
    *
-   * Widening types might result in loss of precision in the following cases:
-   * - IntegerType to FloatType
-   * - LongType to FloatType
-   * - LongType to DoubleType
-   * - DecimalType to Double
+   *   Widening types might result in loss of precision in the following cases:
+   *   - IntegerType to FloatType
+   *   - LongType to FloatType
+   *   - LongType to DoubleType
+   *   - DecimalType to Double
    *
-   * This rule is only applied to Union/Except/Intersect
+   * 2. When ANSI mode is on:
+   *   The implicit conversion is determined by the closest common data type from the precedent
+   *   lists from left and right child. See the comments of Object `AnsiTypeCoercion` for details.
    */
   object WidenSetOperationTypes extends Rule[LogicalPlan] {
 
-    def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperatorsUp {
-      case s @ Except(left, right, isAll) if s.childrenResolved &&
-        left.output.length == right.output.length && !s.resolved =>
-        val newChildren: Seq[LogicalPlan] = buildNewChildrenWithWiderTypes(left :: right :: Nil)
-        assert(newChildren.length == 2)
-        Except(newChildren.head, newChildren.last, isAll)
+    override def apply(plan: LogicalPlan): LogicalPlan = {
+      plan resolveOperatorsUpWithNewOutput {
+        case s @ Except(left, right, isAll) if s.childrenResolved &&
+          left.output.length == right.output.length && !s.resolved =>
+          val newChildren: Seq[LogicalPlan] = buildNewChildrenWithWiderTypes(left :: right :: Nil)
+          if (newChildren.isEmpty) {
+            s -> Nil
+          } else {
+            assert(newChildren.length == 2)
+            val attrMapping = left.output.zip(newChildren.head.output)
+            Except(newChildren.head, newChildren.last, isAll) -> attrMapping
+          }
 
-      case s @ Intersect(left, right, isAll) if s.childrenResolved &&
-        left.output.length == right.output.length && !s.resolved =>
-        val newChildren: Seq[LogicalPlan] = buildNewChildrenWithWiderTypes(left :: right :: Nil)
-        assert(newChildren.length == 2)
-        Intersect(newChildren.head, newChildren.last, isAll)
+        case s @ Intersect(left, right, isAll) if s.childrenResolved &&
+          left.output.length == right.output.length && !s.resolved =>
+          val newChildren: Seq[LogicalPlan] = buildNewChildrenWithWiderTypes(left :: right :: Nil)
+          if (newChildren.isEmpty) {
+            s -> Nil
+          } else {
+            assert(newChildren.length == 2)
+            val attrMapping = left.output.zip(newChildren.head.output)
+            Intersect(newChildren.head, newChildren.last, isAll) -> attrMapping
+          }
 
-      case s: Union if s.childrenResolved &&
+        case s: Union if s.childrenResolved && !s.byName &&
           s.children.forall(_.output.length == s.children.head.output.length) && !s.resolved =>
-        val newChildren: Seq[LogicalPlan] = buildNewChildrenWithWiderTypes(s.children)
-        s.makeCopy(Array(newChildren))
+          val newChildren: Seq[LogicalPlan] = buildNewChildrenWithWiderTypes(s.children)
+          if (newChildren.isEmpty) {
+            s -> Nil
+          } else {
+            val attrMapping = s.children.head.output.zip(newChildren.head.output)
+            s.copy(children = newChildren) -> attrMapping
+          }
+      }
     }
 
     /** Build new children with the widest types for each attribute among all the children */
@@ -359,8 +267,7 @@ object TypeCoercion {
         // Add an extra Project if the targetTypes are different from the original types.
         children.map(widenTypes(_, targetTypes))
       } else {
-        // Unable to find a target type to widen, then just return the original set.
-        children
+        Nil
       }
     }
 
@@ -386,63 +293,14 @@ object TypeCoercion {
     /** Given a plan, add an extra project on top to widen some columns' data types. */
     private def widenTypes(plan: LogicalPlan, targetTypes: Seq[DataType]): LogicalPlan = {
       val casted = plan.output.zip(targetTypes).map {
-        case (e, dt) if e.dataType != dt => Alias(Cast(e, dt), e.name)()
+        case (e, dt) if e.dataType != dt =>
+          Alias(Cast(e, dt, Some(conf.sessionLocalTimeZone)), e.name)()
         case (e, _) => e
       }
       Project(casted, plan)
     }
   }
 
-  /**
-   * Promotes strings that appear in arithmetic expressions.
-   */
-  case class PromoteStrings(conf: SQLConf) extends TypeCoercionRule {
-    private def castExpr(expr: Expression, targetType: DataType): Expression = {
-      (expr.dataType, targetType) match {
-        case (NullType, dt) => Literal.create(null, targetType)
-        case (l, dt) if (l != dt) => Cast(expr, targetType)
-        case _ => expr
-      }
-    }
-
-    override protected def coerceTypes(
-        plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
-      // Skip nodes who's children have not been resolved yet.
-      case e if !e.childrenResolved => e
-
-      case a @ BinaryArithmetic(left @ StringType(), right)
-        if right.dataType != CalendarIntervalType =>
-        a.makeCopy(Array(Cast(left, DoubleType), right))
-      case a @ BinaryArithmetic(left, right @ StringType())
-        if left.dataType != CalendarIntervalType =>
-        a.makeCopy(Array(left, Cast(right, DoubleType)))
-
-      // For equality between string and timestamp we cast the string to a timestamp
-      // so that things like rounding of subsecond precision does not affect the comparison.
-      case p @ Equality(left @ StringType(), right @ TimestampType()) =>
-        p.makeCopy(Array(Cast(left, TimestampType), right))
-      case p @ Equality(left @ TimestampType(), right @ StringType()) =>
-        p.makeCopy(Array(left, Cast(right, TimestampType)))
-
-      case p @ BinaryComparison(left, right)
-          if findCommonTypeForBinaryComparison(left.dataType, right.dataType, conf).isDefined =>
-        val commonType = findCommonTypeForBinaryComparison(left.dataType, right.dataType, conf).get
-        p.makeCopy(Array(castExpr(left, commonType), castExpr(right, commonType)))
-
-      case Abs(e @ StringType()) => Abs(Cast(e, DoubleType))
-      case Sum(e @ StringType()) => Sum(Cast(e, DoubleType))
-      case Average(e @ StringType()) => Average(Cast(e, DoubleType))
-      case StddevPop(e @ StringType()) => StddevPop(Cast(e, DoubleType))
-      case StddevSamp(e @ StringType()) => StddevSamp(Cast(e, DoubleType))
-      case UnaryMinus(e @ StringType()) => UnaryMinus(Cast(e, DoubleType))
-      case UnaryPositive(e @ StringType()) => UnaryPositive(Cast(e, DoubleType))
-      case VariancePop(e @ StringType()) => VariancePop(Cast(e, DoubleType))
-      case VarianceSamp(e @ StringType()) => VarianceSamp(Cast(e, DoubleType))
-      case Skewness(e @ StringType()) => Skewness(Cast(e, DoubleType))
-      case Kurtosis(e @ StringType()) => Kurtosis(Cast(e, DoubleType))
-    }
-  }
-
   /**
    * Handles type coercion for both IN expression with subquery and IN
    * expressions without subquery.
@@ -457,9 +315,8 @@ object TypeCoercion {
    *    operator type is found the original expression will be returned and an
    *    Analysis Exception will be raised at the type checking phase.
    */
-  case class InConversion(conf: SQLConf) extends TypeCoercionRule {
-    override protected def coerceTypes(
-        plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
+  object InConversion extends TypeCoercionRule {
+    override val transform: PartialFunction[Expression, Expression] = {
       // Skip nodes who's children have not been resolved yet.
       case e if !e.childrenResolved => e
 
@@ -502,65 +359,20 @@ object TypeCoercion {
   }
 
   /**
-   * Changes numeric values to booleans so that expressions like true = 1 can be evaluated.
+   * This ensure that the types for various functions are as expected.
    */
-  object BooleanEquality extends Rule[LogicalPlan] {
-    private val trueValues = Seq(1.toByte, 1.toShort, 1, 1L, Decimal.ONE)
-    private val falseValues = Seq(0.toByte, 0.toShort, 0, 0L, Decimal.ZERO)
+  object FunctionArgumentConversion extends TypeCoercionRule {
 
-    def apply(plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
+    override val transform: PartialFunction[Expression, Expression] = {
       // Skip nodes who's children have not been resolved yet.
       case e if !e.childrenResolved => e
 
-      // Hive treats (true = 1) as true and (false = 0) as true,
-      // all other cases are considered as false.
-
-      // We may simplify the expression if one side is literal numeric values
-      // TODO: Maybe these rules should go into the optimizer.
-      case EqualTo(bool @ BooleanType(), Literal(value, _: NumericType))
-        if trueValues.contains(value) => bool
-      case EqualTo(bool @ BooleanType(), Literal(value, _: NumericType))
-        if falseValues.contains(value) => Not(bool)
-      case EqualTo(Literal(value, _: NumericType), bool @ BooleanType())
-        if trueValues.contains(value) => bool
-      case EqualTo(Literal(value, _: NumericType), bool @ BooleanType())
-        if falseValues.contains(value) => Not(bool)
-      case EqualNullSafe(bool @ BooleanType(), Literal(value, _: NumericType))
-        if trueValues.contains(value) => And(IsNotNull(bool), bool)
-      case EqualNullSafe(bool @ BooleanType(), Literal(value, _: NumericType))
-        if falseValues.contains(value) => And(IsNotNull(bool), Not(bool))
-      case EqualNullSafe(Literal(value, _: NumericType), bool @ BooleanType())
-        if trueValues.contains(value) => And(IsNotNull(bool), bool)
-      case EqualNullSafe(Literal(value, _: NumericType), bool @ BooleanType())
-        if falseValues.contains(value) => And(IsNotNull(bool), Not(bool))
-
-      case EqualTo(left @ BooleanType(), right @ NumericType()) =>
-        EqualTo(Cast(left, right.dataType), right)
-      case EqualTo(left @ NumericType(), right @ BooleanType()) =>
-        EqualTo(left, Cast(right, left.dataType))
-      case EqualNullSafe(left @ BooleanType(), right @ NumericType()) =>
-        EqualNullSafe(Cast(left, right.dataType), right)
-      case EqualNullSafe(left @ NumericType(), right @ BooleanType()) =>
-        EqualNullSafe(left, Cast(right, left.dataType))
-    }
-  }
-
-  /**
-   * This ensure that the types for various functions are as expected.
-   */
-  object FunctionArgumentConversion extends TypeCoercionRule {
-
-    override protected def coerceTypes(
-        plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
-      // Skip nodes who's children have not been resolved yet.
-      case e if !e.childrenResolved => e
-
-      case a @ CreateArray(children, _) if !haveSameType(children.map(_.dataType)) =>
-        val types = children.map(_.dataType)
-        findWiderCommonType(types) match {
-          case Some(finalDataType) => a.copy(children.map(castIfNotSameType(_, finalDataType)))
-          case None => a
-        }
+      case a @ CreateArray(children, _) if !haveSameType(children.map(_.dataType)) =>
+        val types = children.map(_.dataType)
+        findWiderCommonType(types) match {
+          case Some(finalDataType) => a.copy(children.map(castIfNotSameType(_, finalDataType)))
+          case None => a
+        }
 
       case c @ Concat(children) if children.forall(c => ArrayType.acceptsType(c.dataType)) &&
         !haveSameType(c.inputTypesForMerging) =>
@@ -573,7 +385,7 @@ object TypeCoercion {
       case aj @ ArrayJoin(arr, d, nr) if !ArrayType(StringType).acceptsType(arr.dataType) &&
         ArrayType.acceptsType(arr.dataType) =>
         val containsNull = arr.dataType.asInstanceOf[ArrayType].containsNull
-        ImplicitTypeCasts.implicitCast(arr, ArrayType(StringType, containsNull)) match {
+        implicitCast(arr, ArrayType(StringType, containsNull)) match {
           case Some(castedArr) => ArrayJoin(castedArr, d, nr)
           case None => aj
         }
@@ -610,17 +422,6 @@ object TypeCoercion {
 
         m.copy(newKeys.zip(newValues).flatMap { case (k, v) => Seq(k, v) })
 
-      // Promote SUM, SUM DISTINCT and AVERAGE to largest types to prevent overflows.
-      case s @ Sum(e @ DecimalType()) => s // Decimal is already the biggest.
-      case Sum(e @ IntegralType()) if e.dataType != LongType => Sum(Cast(e, LongType))
-      case Sum(e @ FractionalType()) if e.dataType != DoubleType => Sum(Cast(e, DoubleType))
-
-      case s @ Average(e @ DecimalType()) => s // Decimal is already the biggest.
-      case Average(e @ IntegralType()) if e.dataType != LongType =>
-        Average(Cast(e, LongType))
-      case Average(e @ FractionalType()) if e.dataType != DoubleType =>
-        Average(Cast(e, DoubleType))
-
       // Hive lets you do aggregation of timestamps... for some reason
       case Sum(e @ TimestampType()) => Sum(Cast(e, DoubleType))
       case Average(e @ TimestampType()) => Average(Cast(e, DoubleType))
@@ -631,8 +432,10 @@ object TypeCoercion {
       case c @ Coalesce(es) if !haveSameType(c.inputTypesForMerging) =>
         val types = es.map(_.dataType)
         findWiderCommonType(types) match {
-          case Some(finalDataType) => Coalesce(es.map(castIfNotSameType(_, finalDataType)))
-          case None => c
+          case Some(finalDataType) =>
+            Coalesce(es.map(castIfNotSameType(_, finalDataType)))
+          case None =>
+            c
         }
 
       // When finding wider type for `Greatest` and `Least`, we should handle decimal types even if
@@ -665,8 +468,7 @@ object TypeCoercion {
    * converted to fractional types.
    */
   object Division extends TypeCoercionRule {
-    override protected def coerceTypes(
-        plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
+    override val transform: PartialFunction[Expression, Expression] = {
       // Skip nodes who has not been resolved yet,
       // as this is an extra rule which should be applied at last.
       case e if !e.childrenResolved => e
@@ -674,8 +476,8 @@ object TypeCoercion {
       // Decimal and Double remain the same
       case d: Divide if d.dataType == DoubleType => d
       case d: Divide if d.dataType.isInstanceOf[DecimalType] => d
-      case Divide(left, right) if isNumericOrNull(left) && isNumericOrNull(right) =>
-        Divide(Cast(left, DoubleType), Cast(right, DoubleType))
+      case d @ Divide(left, right, _) if isNumericOrNull(left) && isNumericOrNull(right) =>
+        d.copy(left = Cast(left, DoubleType), right = Cast(right, DoubleType))
     }
 
     private def isNumericOrNull(ex: Expression): Boolean = {
@@ -684,12 +486,28 @@ object TypeCoercion {
     }
   }
 
+  /**
+   * The DIV operator always returns long-type value.
+   * This rule cast the integral inputs to long type, to avoid overflow during calculation.
+   */
+  object IntegralDivision extends TypeCoercionRule {
+    override val transform: PartialFunction[Expression, Expression] = {
+      case e if !e.childrenResolved => e
+      case d @ IntegralDivide(left, right, _) =>
+        d.copy(left = mayCastToLong(left), right = mayCastToLong(right))
+    }
+
+    private def mayCastToLong(expr: Expression): Expression = expr.dataType match {
+      case _: ByteType | _: ShortType | _: IntegerType => Cast(expr, LongType)
+      case _ => expr
+    }
+  }
+
   /**
    * Coerces the type of different branches of a CASE WHEN statement to a common type.
    */
   object CaseWhenCoercion extends TypeCoercionRule {
-    override protected def coerceTypes(
-        plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
+    override val transform: PartialFunction[Expression, Expression] = {
       case c: CaseWhen if c.childrenResolved && !haveSameType(c.inputTypesForMerging) =>
         val maybeCommonType = findWiderCommonType(c.inputTypesForMerging)
         maybeCommonType.map { commonType =>
@@ -706,8 +524,7 @@ object TypeCoercion {
    * Coerces the type of different branches of If statement to a common type.
    */
   object IfCoercion extends TypeCoercionRule {
-    override protected def coerceTypes(
-        plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
+    override val transform: PartialFunction[Expression, Expression] = {
       case e if !e.childrenResolved => e
       // Find tightest common type for If, if the true value and false value have different types.
       case i @ If(pred, left, right) if !haveSameType(i.inputTypesForMerging) =>
@@ -727,7 +544,7 @@ object TypeCoercion {
    * Coerces NullTypes in the Stack expression to the column types of the corresponding positions.
    */
   object StackCoercion extends TypeCoercionRule {
-    override def coerceTypes(plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
+    override val transform: PartialFunction[Expression, Expression] = {
       case s @ Stack(children) if s.childrenResolved && s.hasFoldableNumRows =>
         Stack(children.zipWithIndex.map {
           // The first child is the number of rows for stack.
@@ -745,21 +562,17 @@ object TypeCoercion {
    * If `spark.sql.function.concatBinaryAsString` is false and all children types are binary,
    * the expected types are binary. Otherwise, the expected ones are strings.
    */
-  case class ConcatCoercion(conf: SQLConf) extends TypeCoercionRule {
-
-    override protected def coerceTypes(plan: LogicalPlan): LogicalPlan = {
-      plan resolveOperators { case p =>
-        p transformExpressionsUp {
-          // Skip nodes if unresolved or empty children
-          case c @ Concat(children) if !c.childrenResolved || children.isEmpty => c
-          case c @ Concat(children) if conf.concatBinaryAsString ||
-            !children.map(_.dataType).forall(_ == BinaryType) =>
-            val newChildren = c.children.map { e =>
-              ImplicitTypeCasts.implicitCast(e, StringType).getOrElse(e)
-            }
-            c.copy(children = newChildren)
+  object ConcatCoercion extends TypeCoercionRule {
+
+    override val transform: PartialFunction[Expression, Expression] = {
+      // Skip nodes if unresolved or empty children
+      case c @ Concat(children) if !c.childrenResolved || children.isEmpty => c
+      case c @ Concat(children) if conf.concatBinaryAsString ||
+        !children.map(_.dataType).forall(_ == BinaryType) =>
+        val newChildren = c.children.map { e =>
+          implicitCast(e, StringType).getOrElse(e)
         }
-      }
+        c.copy(children = newChildren)
     }
   }
 
@@ -768,7 +581,7 @@ object TypeCoercion {
    * to a common type.
    */
   object MapZipWithCoercion extends TypeCoercionRule {
-    override protected def coerceTypes(plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
+    override val transform: PartialFunction[Expression, Expression] = {
       // Lambda function isn't resolved when the rule is executed.
       case m @ MapZipWith(left, right, function) if m.arguments.forall(a => a.resolved &&
           MapType.acceptsType(a.dataType)) && !m.leftKeyType.sameType(m.rightKeyType) =>
@@ -793,32 +606,28 @@ object TypeCoercion {
    * If `spark.sql.function.eltOutputAsString` is false and all children types are binary,
    * the expected types are binary. Otherwise, the expected ones are strings.
    */
-  case class EltCoercion(conf: SQLConf) extends TypeCoercionRule {
-
-    override protected def coerceTypes(plan: LogicalPlan): LogicalPlan = {
-      plan resolveOperators { case p =>
-        p transformExpressionsUp {
-          // Skip nodes if unresolved or not enough children
-          case c @ Elt(children) if !c.childrenResolved || children.size < 2 => c
-          case c @ Elt(children) =>
-            val index = children.head
-            val newIndex = ImplicitTypeCasts.implicitCast(index, IntegerType).getOrElse(index)
-            val newInputs = if (conf.eltOutputAsString ||
-              !children.tail.map(_.dataType).forall(_ == BinaryType)) {
-              children.tail.map { e =>
-                ImplicitTypeCasts.implicitCast(e, StringType).getOrElse(e)
-              }
-            } else {
-              children.tail
-            }
-            c.copy(children = newIndex +: newInputs)
+  object EltCoercion extends TypeCoercionRule {
+
+    override val transform: PartialFunction[Expression, Expression] = {
+      // Skip nodes if unresolved or not enough children
+      case c @ Elt(children, _) if !c.childrenResolved || children.size < 2 => c
+      case c @ Elt(children, _) =>
+        val index = children.head
+        val newIndex = implicitCast(index, IntegerType).getOrElse(index)
+        val newInputs = if (conf.eltOutputAsString ||
+          !children.tail.map(_.dataType).forall(_ == BinaryType)) {
+          children.tail.map { e =>
+            implicitCast(e, StringType).getOrElse(e)
+          }
+        } else {
+          children.tail
         }
-      }
+        c.copy(children = newIndex +: newInputs)
     }
   }
 
-  object DateTimeOperations extends Rule[LogicalPlan] {
-    override def apply(plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
+  object DateTimeOperations extends TypeCoercionRule {
+    override val transform: PartialFunction[Expression, Expression] = {
       // Skip nodes who's children have not been resolved yet.
       case e if !e.childrenResolved => e
       case d @ DateAdd(TimestampType(), _) => d.copy(startDate = Cast(d.startDate, DateType))
@@ -826,10 +635,10 @@ object TypeCoercion {
       case d @ DateSub(TimestampType(), _) => d.copy(startDate = Cast(d.startDate, DateType))
       case d @ DateSub(StringType(), _) => d.copy(startDate = Cast(d.startDate, DateType))
 
-      case s @ SubtractTimestamps(DateType(), _) =>
-        s.copy(endTimestamp = Cast(s.endTimestamp, TimestampType))
-      case s @ SubtractTimestamps(_, DateType()) =>
-        s.copy(startTimestamp = Cast(s.startTimestamp, TimestampType))
+      case s @ SubtractTimestamps(DateType(), _, _, _) =>
+        s.copy(left = Cast(s.left, TimestampType))
+      case s @ SubtractTimestamps(_, DateType(), _, _) =>
+        s.copy(right = Cast(s.right, TimestampType))
 
       case t @ TimeAdd(StringType(), _, _) => t.copy(start = Cast(t.start, TimestampType))
     }
@@ -852,8 +661,7 @@ object TypeCoercion {
       }
     }
 
-    override protected def coerceTypes(
-        plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
+    override val transform: PartialFunction[Expression, Expression] = {
       // Skip nodes who's children have not been resolved yet.
       case e if !e.childrenResolved => e
 
@@ -905,7 +713,7 @@ object TypeCoercion {
           }
 
         }
-        udf.withNewChildren(children)
+        udf.copy(children = children)
     }
 
     private def udfInputToCastType(input: DataType, expectedType: DataType): DataType = {
@@ -932,108 +740,13 @@ object TypeCoercion {
         case (_, other) => other
       }
     }
-
-    /**
-     * Given an expected data type, try to cast the expression and return the cast expression.
-     *
-     * If the expression already fits the input type, we simply return the expression itself.
-     * If the expression has an incompatible type that cannot be implicitly cast, return None.
-     */
-    def implicitCast(e: Expression, expectedType: AbstractDataType): Option[Expression] = {
-      implicitCast(e.dataType, expectedType).map { dt =>
-        if (dt == e.dataType) e else Cast(e, dt)
-      }
-    }
-
-    private def implicitCast(inType: DataType, expectedType: AbstractDataType): Option[DataType] = {
-      // Note that ret is nullable to avoid typing a lot of Some(...) in this local scope.
-      // We wrap immediately an Option after this.
-      @Nullable val ret: DataType = (inType, expectedType) match {
-        // If the expected type is already a parent of the input type, no need to cast.
-        case _ if expectedType.acceptsType(inType) => inType
-
-        // Cast null type (usually from null literals) into target types
-        case (NullType, target) => target.defaultConcreteType
-
-        // If the function accepts any numeric type and the input is a string, we follow the hive
-        // convention and cast that input into a double
-        case (StringType, NumericType) => NumericType.defaultConcreteType
-
-        // Implicit cast among numeric types. When we reach here, input type is not acceptable.
-
-        // If input is a numeric type but not decimal, and we expect a decimal type,
-        // cast the input to decimal.
-        case (d: NumericType, DecimalType) => DecimalType.forType(d)
-        // For any other numeric types, implicitly cast to each other, e.g. long -> int, int -> long
-        case (_: NumericType, target: NumericType) => target
-
-        // Implicit cast between date time types
-        case (DateType, TimestampType) => TimestampType
-        case (TimestampType, DateType) => DateType
-
-        // Implicit cast from/to string
-        case (StringType, DecimalType) => DecimalType.SYSTEM_DEFAULT
-        case (StringType, target: NumericType) => target
-        case (StringType, DateType) => DateType
-        case (StringType, TimestampType) => TimestampType
-        case (StringType, BinaryType) => BinaryType
-        // Cast any atomic type to string.
-        case (any: AtomicType, StringType) if any != StringType => StringType
-
-        // When we reach here, input type is not acceptable for any types in this type collection,
-        // try to find the first one we can implicitly cast.
-        case (_, TypeCollection(types)) =>
-          types.flatMap(implicitCast(inType, _)).headOption.orNull
-
-        // Implicit cast between array types.
-        //
-        // Compare the nullabilities of the from type and the to type, check whether the cast of
-        // the nullability is resolvable by the following rules:
-        // 1. If the nullability of the to type is true, the cast is always allowed;
-        // 2. If the nullability of the to type is false, and the nullability of the from type is
-        // true, the cast is never allowed;
-        // 3. If the nullabilities of both the from type and the to type are false, the cast is
-        // allowed only when Cast.forceNullable(fromType, toType) is false.
-        case (ArrayType(fromType, fn), ArrayType(toType: DataType, true)) =>
-          implicitCast(fromType, toType).map(ArrayType(_, true)).orNull
-
-        case (ArrayType(fromType, true), ArrayType(toType: DataType, false)) => null
-
-        case (ArrayType(fromType, false), ArrayType(toType: DataType, false))
-            if !Cast.forceNullable(fromType, toType) =>
-          implicitCast(fromType, toType).map(ArrayType(_, false)).orNull
-
-        // Implicit cast between Map types.
-        // Follows the same semantics of implicit casting between two array types.
-        // Refer to documentation above. Make sure that both key and values
-        // can not be null after the implicit cast operation by calling forceNullable
-        // method.
-        case (MapType(fromKeyType, fromValueType, fn), MapType(toKeyType, toValueType, tn))
-            if !Cast.forceNullable(fromKeyType, toKeyType) && Cast.resolvableNullability(fn, tn) =>
-          if (Cast.forceNullable(fromValueType, toValueType) && !tn) {
-            null
-          } else {
-            val newKeyType = implicitCast(fromKeyType, toKeyType).orNull
-            val newValueType = implicitCast(fromValueType, toValueType).orNull
-            if (newKeyType != null && newValueType != null) {
-              MapType(newKeyType, newValueType, tn)
-            } else {
-              null
-            }
-          }
-
-        case _ => null
-      }
-      Option(ret)
-    }
   }
 
   /**
    * Cast WindowFrame boundaries to the type they operate upon.
    */
   object WindowFrameCoercion extends TypeCoercionRule {
-    override protected def coerceTypes(
-        plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
+    override val transform: PartialFunction[Expression, Expression] = {
       case s @ WindowSpecDefinition(_, Seq(order), SpecifiedWindowFrame(RangeFrame, lower, upper))
           if order.resolved =>
         s.copy(frameSpecification = SpecifiedWindowFrame(
@@ -1047,7 +760,7 @@ object TypeCoercion {
         case (e: SpecialFrameBoundary, _) => e
         case (e, _: DateType) => e
         case (e, _: TimestampType) => e
-        case (e: Expression, t) if e.dataType != t && Cast.canCast(e.dataType, t) =>
+        case (e: Expression, t) if e.dataType != t && canCast(e.dataType, t) =>
           Cast(e, t)
         case _ => boundary
       }
@@ -1060,7 +773,7 @@ object TypeCoercion {
    * TODO(SPARK-28589): implement ANSI type type coercion and handle string literals.
    */
   object StringLiteralCoercion extends TypeCoercionRule {
-    override protected def coerceTypes(plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
+    override val transform: PartialFunction[Expression, Expression] = {
       // Skip nodes who's children have not been resolved yet.
       case e if !e.childrenResolved => e
       case DateAdd(l, r) if r.dataType == StringType && r.foldable =>
@@ -1083,45 +796,406 @@ object TypeCoercion {
   }
 }
 
+/**
+ * A collection of [[Rule]] that can be used to coerce differing types that participate in
+ * operations into compatible ones.
+ *
+ * Notes about type widening / tightest common types: Broadly, there are two cases when we need
+ * to widen data types (e.g. union, binary comparison). In case 1, we are looking for a common
+ * data type for two or more data types, and in this case no loss of precision is allowed. Examples
+ * include type inference in JSON (e.g. what's the column's data type if one row is an integer
+ * while the other row is a long?). In case 2, we are looking for a widened data type with
+ * some acceptable loss of precision (e.g. there is no common type for double and decimal because
+ * double's range is larger than decimal, and yet decimal is more precise than double, but in
+ * union we would cast the decimal into double).
+ */
+object TypeCoercion extends TypeCoercionBase {
+
+  override def typeCoercionRules: List[Rule[LogicalPlan]] =
+    WidenSetOperationTypes ::
+    CombinedTypeCoercionRule(
+      InConversion ::
+      PromoteStrings ::
+      DecimalPrecision ::
+      BooleanEquality ::
+      FunctionArgumentConversion ::
+      ConcatCoercion ::
+      MapZipWithCoercion ::
+      EltCoercion ::
+      CaseWhenCoercion ::
+      IfCoercion ::
+      StackCoercion ::
+      Division ::
+      IntegralDivision ::
+      ImplicitTypeCasts ::
+      DateTimeOperations ::
+      WindowFrameCoercion ::
+      StringLiteralCoercion :: Nil) :: Nil
+
+  override def canCast(from: DataType, to: DataType): Boolean = Cast.canCast(from, to)
+
+  // See https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types.
+  // The conversion for integral and floating point types have a linear widening hierarchy:
+  val numericPrecedence =
+    IndexedSeq(
+      ByteType,
+      ShortType,
+      IntegerType,
+      LongType,
+      FloatType,
+      DoubleType)
+
+  override val findTightestCommonType: (DataType, DataType) => Option[DataType] = {
+      case (t1, t2) if t1 == t2 => Some(t1)
+      case (NullType, t1) => Some(t1)
+      case (t1, NullType) => Some(t1)
+
+      case (t1: IntegralType, t2: DecimalType) if t2.isWiderThan(t1) =>
+        Some(t2)
+      case (t1: DecimalType, t2: IntegralType) if t1.isWiderThan(t2) =>
+        Some(t1)
+
+      // Promote numeric types to the highest of the two
+      case (t1: NumericType, t2: NumericType)
+          if !t1.isInstanceOf[DecimalType] && !t2.isInstanceOf[DecimalType] =>
+        val index = numericPrecedence.lastIndexWhere(t => t == t1 || t == t2)
+        Some(numericPrecedence(index))
+
+      case (_: TimestampType, _: DateType) | (_: DateType, _: TimestampType) =>
+        Some(TimestampType)
+
+      case (t1, t2) => findTypeForComplex(t1, t2, findTightestCommonType)
+  }
+
+  /** Promotes all the way to StringType. */
+  private def stringPromotion(dt1: DataType, dt2: DataType): Option[DataType] = (dt1, dt2) match {
+    case (StringType, t2: AtomicType) if t2 != BinaryType && t2 != BooleanType => Some(StringType)
+    case (t1: AtomicType, StringType) if t1 != BinaryType && t1 != BooleanType => Some(StringType)
+    case _ => None
+  }
+
+  /**
+   * This function determines the target type of a comparison operator when one operand
+   * is a String and the other is not. It also handles when one op is a Date and the
+   * other is a Timestamp by making the target type to be String.
+   */
+  def findCommonTypeForBinaryComparison(
+      dt1: DataType, dt2: DataType, conf: SQLConf): Option[DataType] = (dt1, dt2) match {
+    case (StringType, DateType)
+      => if (conf.castDatetimeToString) Some(StringType) else Some(DateType)
+    case (DateType, StringType)
+      => if (conf.castDatetimeToString) Some(StringType) else Some(DateType)
+    case (StringType, TimestampType)
+      => if (conf.castDatetimeToString) Some(StringType) else Some(TimestampType)
+    case (TimestampType, StringType)
+      => if (conf.castDatetimeToString) Some(StringType) else Some(TimestampType)
+    case (StringType, NullType) => Some(StringType)
+    case (NullType, StringType) => Some(StringType)
+
+    // Cast to TimestampType when we compare DateType with TimestampType
+    // i.e. TimeStamp('2017-03-01 00:00:00') eq Date('2017-03-01') = true
+    case (TimestampType, DateType) => Some(TimestampType)
+    case (DateType, TimestampType) => Some(TimestampType)
+
+    // There is no proper decimal type we can pick,
+    // using double type is the best we can do.
+    // See SPARK-22469 for details.
+    case (n: DecimalType, s: StringType) => Some(DoubleType)
+    case (s: StringType, n: DecimalType) => Some(DoubleType)
+
+    case (l: StringType, r: AtomicType) if r != StringType => Some(r)
+    case (l: AtomicType, r: StringType) if l != StringType => Some(l)
+    case (l, r) => None
+  }
+
+  override def findWiderTypeForTwo(t1: DataType, t2: DataType): Option[DataType] = {
+    findTightestCommonType(t1, t2)
+      .orElse(findWiderTypeForDecimal(t1, t2))
+      .orElse(stringPromotion(t1, t2))
+      .orElse(findTypeForComplex(t1, t2, findWiderTypeForTwo))
+  }
+
+  override def findWiderCommonType(types: Seq[DataType]): Option[DataType] = {
+    // findWiderTypeForTwo doesn't satisfy the associative law, i.e. (a op b) op c may not equal
+    // to a op (b op c). This is only a problem for StringType or nested StringType in ArrayType.
+    // Excluding these types, findWiderTypeForTwo satisfies the associative law. For instance,
+    // (TimestampType, IntegerType, StringType) should have StringType as the wider common type.
+    val (stringTypes, nonStringTypes) = types.partition(hasStringType(_))
+    (stringTypes.distinct ++ nonStringTypes).foldLeft[Option[DataType]](Some(NullType))((r, c) =>
+      r match {
+        case Some(d) => findWiderTypeForTwo(d, c)
+        case _ => None
+      })
+  }
+
+  override def implicitCast(e: Expression, expectedType: AbstractDataType): Option[Expression] = {
+    implicitCast(e.dataType, expectedType).map { dt =>
+      if (dt == e.dataType) e else Cast(e, dt)
+    }
+  }
+
+  private def implicitCast(inType: DataType, expectedType: AbstractDataType): Option[DataType] = {
+    // Note that ret is nullable to avoid typing a lot of Some(...) in this local scope.
+    // We wrap immediately an Option after this.
+    @Nullable val ret: DataType = (inType, expectedType) match {
+      // If the expected type is already a parent of the input type, no need to cast.
+      case _ if expectedType.acceptsType(inType) => inType
+
+      // Cast null type (usually from null literals) into target types
+      case (NullType, target) => target.defaultConcreteType
+
+      // If the function accepts any numeric type and the input is a string, we follow the hive
+      // convention and cast that input into a double
+      case (StringType, NumericType) => NumericType.defaultConcreteType
+
+      // Implicit cast among numeric types. When we reach here, input type is not acceptable.
+
+      // If input is a numeric type but not decimal, and we expect a decimal type,
+      // cast the input to decimal.
+      case (d: NumericType, DecimalType) => DecimalType.forType(d)
+      // For any other numeric types, implicitly cast to each other, e.g. long -> int, int -> long
+      case (_: NumericType, target: NumericType) => target
+
+      // Implicit cast between date time types
+      case (DateType, TimestampType) => TimestampType
+      case (TimestampType, DateType) => DateType
+
+      // Implicit cast from/to string
+      case (StringType, DecimalType) => DecimalType.SYSTEM_DEFAULT
+      case (StringType, target: NumericType) => target
+      case (StringType, DateType) => DateType
+      case (StringType, TimestampType) => TimestampType
+      case (StringType, BinaryType) => BinaryType
+      // Cast any atomic type to string.
+      case (any: AtomicType, StringType) if any != StringType => StringType
+
+      // When we reach here, input type is not acceptable for any types in this type collection,
+      // try to find the first one we can implicitly cast.
+      case (_, TypeCollection(types)) =>
+        types.flatMap(implicitCast(inType, _)).headOption.orNull
+
+      // Implicit cast between array types.
+      //
+      // Compare the nullabilities of the from type and the to type, check whether the cast of
+      // the nullability is resolvable by the following rules:
+      // 1. If the nullability of the to type is true, the cast is always allowed;
+      // 2. If the nullability of the to type is false, and the nullability of the from type is
+      // true, the cast is never allowed;
+      // 3. If the nullabilities of both the from type and the to type are false, the cast is
+      // allowed only when Cast.forceNullable(fromType, toType) is false.
+      case (ArrayType(fromType, fn), ArrayType(toType: DataType, true)) =>
+        implicitCast(fromType, toType).map(ArrayType(_, true)).orNull
+
+      case (ArrayType(fromType, true), ArrayType(toType: DataType, false)) => null
+
+      case (ArrayType(fromType, false), ArrayType(toType: DataType, false))
+          if !Cast.forceNullable(fromType, toType) =>
+        implicitCast(fromType, toType).map(ArrayType(_, false)).orNull
+
+      // Implicit cast between Map types.
+      // Follows the same semantics of implicit casting between two array types.
+      // Refer to documentation above. Make sure that both key and values
+      // can not be null after the implicit cast operation by calling forceNullable
+      // method.
+      case (MapType(fromKeyType, fromValueType, fn), MapType(toKeyType, toValueType, tn))
+          if !Cast.forceNullable(fromKeyType, toKeyType) && Cast.resolvableNullability(fn, tn) =>
+        if (Cast.forceNullable(fromValueType, toValueType) && !tn) {
+          null
+        } else {
+          val newKeyType = implicitCast(fromKeyType, toKeyType).orNull
+          val newValueType = implicitCast(fromValueType, toValueType).orNull
+          if (newKeyType != null && newValueType != null) {
+            MapType(newKeyType, newValueType, tn)
+          } else {
+            null
+          }
+        }
+
+      case _ => null
+    }
+    Option(ret)
+  }
+
+  /**
+   * The method finds a common type for data types that differ only in nullable flags, including
+   * `nullable`, `containsNull` of [[ArrayType]] and `valueContainsNull` of [[MapType]].
+   * If the input types are different besides nullable flags, None is returned.
+   */
+  def findCommonTypeDifferentOnlyInNullFlags(t1: DataType, t2: DataType): Option[DataType] = {
+    if (t1 == t2) {
+      Some(t1)
+    } else {
+      findTypeForComplex(t1, t2, findCommonTypeDifferentOnlyInNullFlags)
+    }
+  }
+
+  def findCommonTypeDifferentOnlyInNullFlags(types: Seq[DataType]): Option[DataType] = {
+    if (types.isEmpty) {
+      None
+    } else {
+      types.tail.foldLeft[Option[DataType]](Some(types.head)) {
+        case (Some(t1), t2) => findCommonTypeDifferentOnlyInNullFlags(t1, t2)
+        case _ => None
+      }
+    }
+  }
+
+  /**
+   * Whether the data type contains StringType.
+   */
+  def hasStringType(dt: DataType): Boolean = dt match {
+    case StringType => true
+    case ArrayType(et, _) => hasStringType(et)
+    // Add StructType if we support string promotion for struct fields in the future.
+    case _ => false
+  }
+
+  /**
+   * Promotes strings that appear in arithmetic expressions.
+   */
+  object PromoteStrings extends TypeCoercionRule {
+    private def castExpr(expr: Expression, targetType: DataType): Expression = {
+      (expr.dataType, targetType) match {
+        case (NullType, dt) => Literal.create(null, targetType)
+        case (l, dt) if (l != dt) => Cast(expr, targetType)
+        case _ => expr
+      }
+    }
+
+    override def transform: PartialFunction[Expression, Expression] = {
+      // Skip nodes who's children have not been resolved yet.
+      case e if !e.childrenResolved => e
+
+      case a @ BinaryArithmetic(left @ StringType(), right)
+        if right.dataType != CalendarIntervalType =>
+        a.makeCopy(Array(Cast(left, DoubleType), right))
+      case a @ BinaryArithmetic(left, right @ StringType())
+        if left.dataType != CalendarIntervalType =>
+        a.makeCopy(Array(left, Cast(right, DoubleType)))
+
+      // For equality between string and timestamp we cast the string to a timestamp
+      // so that things like rounding of subsecond precision does not affect the comparison.
+      case p @ Equality(left @ StringType(), right @ TimestampType()) =>
+        p.makeCopy(Array(Cast(left, TimestampType), right))
+      case p @ Equality(left @ TimestampType(), right @ StringType()) =>
+        p.makeCopy(Array(left, Cast(right, TimestampType)))
+
+      case p @ BinaryComparison(left, right)
+          if findCommonTypeForBinaryComparison(left.dataType, right.dataType, conf).isDefined =>
+        val commonType = findCommonTypeForBinaryComparison(left.dataType, right.dataType, conf).get
+        p.makeCopy(Array(castExpr(left, commonType), castExpr(right, commonType)))
+
+      case Abs(e @ StringType(), failOnError) => Abs(Cast(e, DoubleType), failOnError)
+      case Sum(e @ StringType()) => Sum(Cast(e, DoubleType))
+      case Average(e @ StringType()) => Average(Cast(e, DoubleType))
+      case s @ StddevPop(e @ StringType(), _) =>
+        s.withNewChildren(Seq(Cast(e, DoubleType)))
+      case s @ StddevSamp(e @ StringType(), _) =>
+        s.withNewChildren(Seq(Cast(e, DoubleType)))
+      case m @ UnaryMinus(e @ StringType(), _) => m.withNewChildren(Seq(Cast(e, DoubleType)))
+      case UnaryPositive(e @ StringType()) => UnaryPositive(Cast(e, DoubleType))
+      case v @ VariancePop(e @ StringType(), _) =>
+        v.withNewChildren(Seq(Cast(e, DoubleType)))
+      case v @ VarianceSamp(e @ StringType(), _) =>
+        v.withNewChildren(Seq(Cast(e, DoubleType)))
+      case s @ Skewness(e @ StringType(), _) =>
+        s.withNewChildren(Seq(Cast(e, DoubleType)))
+      case k @ Kurtosis(e @ StringType(), _) =>
+        k.withNewChildren(Seq(Cast(e, DoubleType)))
+    }
+  }
+
+  /**
+   * Changes numeric values to booleans so that expressions like true = 1 can be evaluated.
+   */
+  object BooleanEquality extends TypeCoercionRule {
+    private val trueValues = Seq(1.toByte, 1.toShort, 1, 1L, Decimal.ONE)
+    private val falseValues = Seq(0.toByte, 0.toShort, 0, 0L, Decimal.ZERO)
+
+    override def transform: PartialFunction[Expression, Expression] = {
+      // Skip nodes who's children have not been resolved yet.
+      case e if !e.childrenResolved => e
+
+      // Hive treats (true = 1) as true and (false = 0) as true,
+      // all other cases are considered as false.
+
+      // We may simplify the expression if one side is literal numeric values
+      // TODO: Maybe these rules should go into the optimizer.
+      case EqualTo(bool @ BooleanType(), Literal(value, _: NumericType))
+        if trueValues.contains(value) => bool
+      case EqualTo(bool @ BooleanType(), Literal(value, _: NumericType))
+        if falseValues.contains(value) => Not(bool)
+      case EqualTo(Literal(value, _: NumericType), bool @ BooleanType())
+        if trueValues.contains(value) => bool
+      case EqualTo(Literal(value, _: NumericType), bool @ BooleanType())
+        if falseValues.contains(value) => Not(bool)
+      case EqualNullSafe(bool @ BooleanType(), Literal(value, _: NumericType))
+        if trueValues.contains(value) => And(IsNotNull(bool), bool)
+      case EqualNullSafe(bool @ BooleanType(), Literal(value, _: NumericType))
+        if falseValues.contains(value) => And(IsNotNull(bool), Not(bool))
+      case EqualNullSafe(Literal(value, _: NumericType), bool @ BooleanType())
+        if trueValues.contains(value) => And(IsNotNull(bool), bool)
+      case EqualNullSafe(Literal(value, _: NumericType), bool @ BooleanType())
+        if falseValues.contains(value) => And(IsNotNull(bool), Not(bool))
+
+      case EqualTo(left @ BooleanType(), right @ NumericType()) =>
+        EqualTo(Cast(left, right.dataType), right)
+      case EqualTo(left @ NumericType(), right @ BooleanType()) =>
+        EqualTo(left, Cast(right, left.dataType))
+      case EqualNullSafe(left @ BooleanType(), right @ NumericType()) =>
+        EqualNullSafe(Cast(left, right.dataType), right)
+      case EqualNullSafe(left @ NumericType(), right @ BooleanType()) =>
+        EqualNullSafe(left, Cast(right, left.dataType))
+    }
+  }
+}
+
 trait TypeCoercionRule extends Rule[LogicalPlan] with Logging {
   /**
    * Applies any changes to [[AttributeReference]] data types that are made by the transform method
    * to instances higher in the query tree.
    */
   def apply(plan: LogicalPlan): LogicalPlan = {
-    val newPlan = coerceTypes(plan)
-    if (plan.fastEquals(newPlan)) {
-      plan
-    } else {
-      propagateTypes(newPlan)
+    val typeCoercionFn = transform
+    def rewrite(plan: LogicalPlan): LogicalPlan = {
+      val withNewChildren = plan.mapChildren(rewrite)
+      if (!withNewChildren.childrenResolved) {
+        withNewChildren
+      } else {
+        // Only propagate types if the children have changed.
+        val withPropagatedTypes = if (withNewChildren ne plan) {
+          propagateTypes(withNewChildren)
+        } else {
+          plan
+        }
+        withPropagatedTypes.transformExpressionsUp(typeCoercionFn)
+      }
     }
+    rewrite(plan)
   }
 
-  protected def coerceTypes(plan: LogicalPlan): LogicalPlan
-
-  private def propagateTypes(plan: LogicalPlan): LogicalPlan = plan resolveOperatorsUp {
-    // No propagation required for leaf nodes.
-    case q: LogicalPlan if q.children.isEmpty => q
+  def transform: PartialFunction[Expression, Expression]
 
-    // Don't propagate types from unresolved children.
-    case q: LogicalPlan if !q.childrenResolved => q
-
-    case q: LogicalPlan =>
-      val inputMap = q.inputSet.toSeq.map(a => (a.exprId, a)).toMap
-      q transformExpressions {
+  private def propagateTypes(plan: LogicalPlan): LogicalPlan = {
+    // Check if the inputs have changed.
+    val references = AttributeMap(plan.references.collect {
+      case a if a.resolved => a -> a
+    }.toSeq)
+    def sameButDifferent(a: Attribute): Boolean = {
+      references.get(a).exists(b => b.dataType != a.dataType || b.nullable != a.nullable)
+    }
+    val inputMap = AttributeMap(plan.inputSet.collect {
+      case a if a.resolved && sameButDifferent(a) => a -> a
+    }.toSeq)
+    if (inputMap.isEmpty) {
+      // Nothing changed.
+      plan
+    } else {
+      // Update the references if the dataType/nullability has changed.
+      plan transformExpressions {
         case a: AttributeReference =>
-          inputMap.get(a.exprId) match {
-            // This can happen when an Attribute reference is born in a non-leaf node, for
-            // example due to a call to an external script like in the Transform operator.
-            // TODO: Perhaps those should actually be aliases?
-            case None => a
-            // Leave the same if the dataTypes match.
-            case Some(newType) if a.dataType == newType.dataType => a
-            case Some(newType) =>
-              logDebug(s"Promoting $a from ${a.dataType} to ${newType.dataType} in " +
-                s" ${q.simpleString(SQLConf.get.maxToStringFields)}")
-              newType
-          }
+          inputMap.getOrElse(a, a)
       }
+    }
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala
index 423f89fefa093..a3a85cb12077f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala
@@ -19,11 +19,12 @@ package org.apache.spark.sql.catalyst.analysis
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.expressions.{Attribute, CurrentDate, CurrentTimestamp, MonotonicallyIncreasingID, Now}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, CurrentDate, CurrentTimestamp, GroupingSets, MonotonicallyIncreasingID, Now}
 import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.streaming.InternalOutputModes
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.streaming.OutputMode
 
 /**
@@ -40,10 +41,15 @@ object UnsupportedOperationChecker extends Logging {
     }
   }
 
+  /**
+   * Checks for possible correctness issue in chained stateful operators. The behavior is
+   * controlled by SQL config `spark.sql.streaming.statefulOperator.checkCorrectness.enabled`.
+   * Once it is enabled, an analysis exception will be thrown. Otherwise, Spark will just
+   * print a warning message.
+   */
   def checkStreamingQueryGlobalWatermarkLimit(
       plan: LogicalPlan,
-      outputMode: OutputMode,
-      failWhenDetected: Boolean): Unit = {
+      outputMode: OutputMode): Unit = {
     def isStatefulOperationPossiblyEmitLateRows(p: LogicalPlan): Boolean = p match {
       case s: Aggregate
         if s.isStreaming && outputMode == InternalOutputModes.Append => true
@@ -62,6 +68,8 @@ object UnsupportedOperationChecker extends Logging {
       case _ => false
     }
 
+    val failWhenDetected = SQLConf.get.statefulOperatorCorrectnessCheckEnabled
+
     try {
       plan.foreach { subPlan =>
         if (isStatefulOperation(subPlan)) {
@@ -73,7 +81,10 @@ object UnsupportedOperationChecker extends Logging {
               "The query contains stateful operation which can emit rows older than " +
               "the current watermark plus allowed late record delay, which are \"late rows\"" +
               " in downstream stateful operations and these rows can be discarded. " +
-              "Please refer the programming guide doc for more details."
+              "Please refer the programming guide doc for more details. If you understand " +
+              "the possible risk of correctness issue and still need to run the query, " +
+              "you can disable this check by setting the config " +
+              "`spark.sql.streaming.statefulOperator.checkCorrectness.enabled` to false."
             throwError(errorMsg)(plan)
           }
         }
@@ -91,7 +102,13 @@ object UnsupportedOperationChecker extends Logging {
 
     /** Collect all the streaming aggregates in a sub plan */
     def collectStreamingAggregates(subplan: LogicalPlan): Seq[Aggregate] = {
-      subplan.collect { case a: Aggregate if a.isStreaming => a }
+      subplan.collect {
+        case a: Aggregate if a.isStreaming => a
+        // Since the Distinct node will be replaced to Aggregate in the optimizer rule
+        // [[ReplaceDistinctWithAggregate]], here we also need to check all Distinct node by
+        // assuming it as Aggregate.
+        case d @ Distinct(c: LogicalPlan) if d.isStreaming => Aggregate(c.output, c.output, c)
+      }
     }
 
     val mapGroupsWithStates = plan.collect {
@@ -186,15 +203,22 @@ object UnsupportedOperationChecker extends Logging {
       // Operations that cannot exists anywhere in a streaming plan
       subPlan match {
 
-        case Aggregate(_, aggregateExpressions, child) =>
+        case Aggregate(groupingExpressions, aggregateExpressions, child) =>
           val distinctAggExprs = aggregateExpressions.flatMap { expr =>
             expr.collect { case ae: AggregateExpression if ae.isDistinct => ae }
           }
+          val haveGroupingSets = groupingExpressions.exists(_.isInstanceOf[GroupingSets])
+
           throwErrorIf(
             child.isStreaming && distinctAggExprs.nonEmpty,
             "Distinct aggregations are not supported on streaming DataFrames/Datasets. Consider " +
               "using approx_count_distinct() instead.")
 
+          throwErrorIf(
+            child.isStreaming && haveGroupingSets,
+            "Grouping Sets is not supported on streaming DataFrames/Datasets"
+          )
+
         case _: Command =>
           throwError("Commands like CreateTable*, AlterTable*, Show* are not supported with " +
             "streaming DataFrames/Datasets")
@@ -206,11 +230,11 @@ object UnsupportedOperationChecker extends Logging {
         case m: FlatMapGroupsWithState if m.isStreaming =>
 
           // Check compatibility with output modes and aggregations in query
-          val aggsAfterFlatMapGroups = collectStreamingAggregates(plan)
+          val aggsInQuery = collectStreamingAggregates(plan)
 
           if (m.isMapGroupsWithState) {                       // check mapGroupsWithState
             // allowed only in update query output mode and without aggregation
-            if (aggsAfterFlatMapGroups.nonEmpty) {
+            if (aggsInQuery.nonEmpty) {
               throwError(
                 "mapGroupsWithState is not supported with aggregation " +
                   "on a streaming DataFrame/Dataset")
@@ -219,8 +243,8 @@ object UnsupportedOperationChecker extends Logging {
                 "mapGroupsWithState is not supported with " +
                   s"$outputMode output mode on a streaming DataFrame/Dataset")
             }
-          } else {                                           // check latMapGroupsWithState
-            if (aggsAfterFlatMapGroups.isEmpty) {
+          } else {                                           // check flatMapGroupsWithState
+            if (aggsInQuery.isEmpty) {
               // flatMapGroupsWithState without aggregation: operation's output mode must
               // match query output mode
               m.outputMode match {
@@ -246,7 +270,7 @@ object UnsupportedOperationChecker extends Logging {
               } else if (collectStreamingAggregates(m).nonEmpty) {
                 throwError(
                   "flatMapGroupsWithState in append mode is not supported after " +
-                    s"aggregation on a streaming DataFrame/Dataset")
+                    "aggregation on a streaming DataFrame/Dataset")
               }
             }
           }
@@ -270,7 +294,7 @@ object UnsupportedOperationChecker extends Logging {
           throwError("dropDuplicates is not supported after aggregation on a " +
             "streaming DataFrame/Dataset")
 
-        case Join(left, right, joinType, condition, _) =>
+        case j @ Join(left, right, joinType, condition, _) =>
           if (left.isStreaming && right.isStreaming && outputMode != InternalOutputModes.Append) {
             throwError("Join between two streaming DataFrames/Datasets is not supported" +
               s" in ${outputMode} output mode, only in Append output mode")
@@ -281,56 +305,40 @@ object UnsupportedOperationChecker extends Logging {
               // no further validations needed
 
             case FullOuter =>
-              if (left.isStreaming || right.isStreaming) {
-                throwError("Full outer joins with streaming DataFrames/Datasets are not supported")
+              if (left.isStreaming && !right.isStreaming) {
+                throwError("FullOuter joins with streaming DataFrames/Datasets on the left " +
+                  "and a static DataFrame/Dataset on the right is not supported")
+              } else if (!left.isStreaming && right.isStreaming) {
+                throwError("FullOuter joins with streaming DataFrames/Datasets on the right " +
+                  "and a static DataFrame/Dataset on the left is not supported")
+              } else if (left.isStreaming && right.isStreaming) {
+                checkForStreamStreamJoinWatermark(j)
               }
 
-            case LeftSemi | LeftAnti =>
+            case LeftAnti =>
               if (right.isStreaming) {
-                throwError("Left semi/anti joins with a streaming DataFrame/Dataset " +
+                throwError(s"$LeftAnti joins with a streaming DataFrame/Dataset " +
                     "on the right are not supported")
               }
 
-            // We support streaming left outer joins with static on the right always, and with
-            // stream on both sides under the appropriate conditions.
-            case LeftOuter =>
+            // We support streaming left outer and left semi joins with static on the right always,
+            // and with stream on both sides under the appropriate conditions.
+            case LeftOuter | LeftSemi =>
               if (!left.isStreaming && right.isStreaming) {
-                throwError("Left outer join with a streaming DataFrame/Dataset " +
+                throwError(s"$joinType join with a streaming DataFrame/Dataset " +
                   "on the right and a static DataFrame/Dataset on the left is not supported")
               } else if (left.isStreaming && right.isStreaming) {
-                val watermarkInJoinKeys = StreamingJoinHelper.isWatermarkInJoinKeys(subPlan)
-
-                val hasValidWatermarkRange =
-                  StreamingJoinHelper.getStateValueWatermark(
-                    left.outputSet, right.outputSet, condition, Some(1000000)).isDefined
-
-                if (!watermarkInJoinKeys && !hasValidWatermarkRange) {
-                  throwError("Stream-stream outer join between two streaming DataFrame/Datasets " +
-                    "is not supported without a watermark in the join keys, or a watermark on " +
-                    "the nullable side and an appropriate range condition")
-                }
+                checkForStreamStreamJoinWatermark(j)
               }
 
             // We support streaming right outer joins with static on the left always, and with
             // stream on both sides under the appropriate conditions.
             case RightOuter =>
               if (left.isStreaming && !right.isStreaming) {
-                throwError("Right outer join with a streaming DataFrame/Dataset on the left and " +
+                throwError("RightOuter join with a streaming DataFrame/Dataset on the left and " +
                     "a static DataFrame/DataSet on the right not supported")
               } else if (left.isStreaming && right.isStreaming) {
-                val isWatermarkInJoinKeys = StreamingJoinHelper.isWatermarkInJoinKeys(subPlan)
-
-                // Check if the nullable side has a watermark, and there's a range condition which
-                // implies a state value watermark on the first side.
-                val hasValidWatermarkRange =
-                    StreamingJoinHelper.getStateValueWatermark(
-                      right.outputSet, left.outputSet, condition, Some(1000000)).isDefined
-
-                if (!isWatermarkInJoinKeys && !hasValidWatermarkRange) {
-                  throwError("Stream-stream outer join between two streaming DataFrame/Datasets " +
-                    "is not supported without a watermark in the join keys, or a watermark on " +
-                    "the nullable side and an appropriate range condition")
-                }
+                checkForStreamStreamJoinWatermark(j)
               }
 
             case NaturalJoin(_) | UsingJoin(_, _) =>
@@ -349,11 +357,8 @@ object UnsupportedOperationChecker extends Logging {
         case Except(left, right, _) if right.isStreaming =>
           throwError("Except on a streaming DataFrame/Dataset on the right is not supported")
 
-        case Intersect(left, right, _) if left.isStreaming && right.isStreaming =>
-          throwError("Intersect between two streaming DataFrames/Datasets is not supported")
-
-        case GroupingSets(_, _, child, _) if child.isStreaming =>
-          throwError("GroupingSets is not supported on streaming DataFrames/Datasets")
+        case Intersect(left, right, _) if left.isStreaming || right.isStreaming =>
+          throwError("Intersect of streaming DataFrames/Datasets is not supported")
 
         case GlobalLimit(_, _) | LocalLimit(_, _)
             if subPlan.children.forall(_.isStreaming) && outputMode == InternalOutputModes.Update =>
@@ -381,7 +386,7 @@ object UnsupportedOperationChecker extends Logging {
       checkUnsupportedExpressions(subPlan)
     }
 
-    checkStreamingQueryGlobalWatermarkLimit(plan, outputMode, failWhenDetected = false)
+    checkStreamingQueryGlobalWatermarkLimit(plan, outputMode)
   }
 
   def checkForContinuous(plan: LogicalPlan, outputMode: OutputMode): Unit = {
@@ -392,18 +397,8 @@ object UnsupportedOperationChecker extends Logging {
         case (_: Project | _: Filter | _: MapElements | _: MapPartitions |
               _: DeserializeToObject | _: SerializeFromObject | _: SubqueryAlias |
               _: TypedFilter) =>
+        case v: View if v.isTempViewStoringAnalyzedPlan =>
         case node if node.nodeName == "StreamingRelationV2" =>
-        case Repartition(1, false, _) =>
-        case node: Aggregate =>
-          val aboveSinglePartitionCoalesce = node.find {
-            case Repartition(1, false, _) => true
-            case _ => false
-          }.isDefined
-
-          if (!aboveSinglePartitionCoalesce) {
-            throwError(s"In continuous processing mode, coalesce(1) must be called before " +
-              s"aggregate operation ${node.nodeName}.")
-          }
         case node =>
           throwError(s"Continuous processing does not support ${node.nodeName} operations.")
       }
@@ -431,4 +426,34 @@ object UnsupportedOperationChecker extends Logging {
     throw new AnalysisException(
       msg, operator.origin.line, operator.origin.startPosition, Some(operator))
   }
+
+  private def checkForStreamStreamJoinWatermark(join: Join): Unit = {
+    val watermarkInJoinKeys = StreamingJoinHelper.isWatermarkInJoinKeys(join)
+
+    // Check if the nullable side has a watermark, and there's a range condition which
+    // implies a state value watermark on the first side.
+    val hasValidWatermarkRange = join.joinType match {
+      case LeftOuter | LeftSemi => StreamingJoinHelper.getStateValueWatermark(
+        join.left.outputSet, join.right.outputSet, join.condition, Some(1000000)).isDefined
+      case RightOuter => StreamingJoinHelper.getStateValueWatermark(
+        join.right.outputSet, join.left.outputSet, join.condition, Some(1000000)).isDefined
+      case FullOuter =>
+        Seq((join.left.outputSet, join.right.outputSet),
+          (join.right.outputSet, join.left.outputSet)).exists {
+          case (attributesToFindStateWatermarkFor, attributesWithEventWatermark) =>
+            StreamingJoinHelper.getStateValueWatermark(attributesToFindStateWatermarkFor,
+              attributesWithEventWatermark, join.condition, Some(1000000)).isDefined
+        }
+      case _ =>
+        throwError(
+          s"Join type ${join.joinType} is not supported with streaming DataFrame/Dataset")(join)
+    }
+
+    if (!watermarkInJoinKeys && !hasValidWatermarkRange) {
+      throwError(
+        s"Stream-stream ${join.joinType} join between two streaming DataFrame/Datasets " +
+          "is not supported without a watermark in the join keys, or a watermark on " +
+          "the nullable side and an appropriate range condition")(join)
+    }
+  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UpdateAttributeNullability.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UpdateAttributeNullability.scala
index 3eae34da7e502..5004108d348b6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UpdateAttributeNullability.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UpdateAttributeNullability.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.catalyst.rules.Rule
  * Updates nullability of Attributes in a resolved LogicalPlan by using the nullability of
  * corresponding Attributes of its children output Attributes. This step is needed because
  * users can use a resolved AttributeReference in the Dataset API and outer joins
- * can change the nullability of an AttribtueReference. Without this rule, a nullable column's
+ * can change the nullability of an AttributeReference. Without this rule, a nullable column's
  * nullable field can be actually set as non-nullable, which cause illegal optimization
  * (e.g., NULL propagation) and wrong answers.
  * See SPARK-13484 and SPARK-13801 for the concrete queries of this case.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/higherOrderFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/higherOrderFunctions.scala
index 11f94762d43e0..eec538cbc84bb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/higherOrderFunctions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/higherOrderFunctions.scala
@@ -17,11 +17,12 @@
 
 package org.apache.spark.sql.catalyst.analysis
 
-import org.apache.spark.sql.catalyst.catalog.SessionCatalog
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.catalyst.trees.AlwaysProcess
+import org.apache.spark.sql.connector.catalog.{CatalogManager, LookupCatalog}
+import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.types.DataType
 
 /**
@@ -30,16 +31,21 @@ import org.apache.spark.sql.types.DataType
  * so we need to resolve higher order function when all children are either resolved or a lambda
  * function.
  */
-case class ResolveHigherOrderFunctions(catalog: SessionCatalog) extends Rule[LogicalPlan] {
+case class ResolveHigherOrderFunctions(catalogManager: CatalogManager)
+  extends Rule[LogicalPlan] with LookupCatalog {
 
   override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveExpressions {
-    case u @ UnresolvedFunction(fn, children, false, filter)
+    case u @ UnresolvedFunction(AsFunctionIdentifier(ident), children, false, filter, ignoreNulls)
         if hasLambdaAndResolvedArguments(children) =>
       withPosition(u) {
-        catalog.lookupFunction(fn, children) match {
+        catalogManager.v1SessionCatalog.lookupFunction(ident, children) match {
           case func: HigherOrderFunction =>
             filter.foreach(_.failAnalysis("FILTER predicate specified, " +
               s"but ${func.prettyName} is not an aggregate function"))
+            if (ignoreNulls) {
+              throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
+                func.prettyName, "IGNORE NULLS")
+            }
             func
           case other => other.failAnalysis(
             "A lambda function should only be used in a higher order function. However, " +
@@ -70,11 +76,11 @@ case class ResolveHigherOrderFunctions(catalog: SessionCatalog) extends Rule[Log
  *      be a lambda function defined in an outer scope, or a attribute in produced by the plan's
  *      child. If names are duplicate, the name defined in the most inner scope is used.
  */
-case class ResolveLambdaVariables(conf: SQLConf) extends Rule[LogicalPlan] {
+object ResolveLambdaVariables extends Rule[LogicalPlan] {
 
   type LambdaVariableMap = Map[String, NamedExpression]
 
-  private val canonicalizer = {
+  private def canonicalizer = {
     if (!conf.caseSensitiveAnalysis) {
       // scalastyle:off caselocale
       s: String => s.toLowerCase
@@ -85,7 +91,7 @@ case class ResolveLambdaVariables(conf: SQLConf) extends Rule[LogicalPlan] {
   }
 
   override def apply(plan: LogicalPlan): LogicalPlan = {
-    plan.resolveOperators {
+    plan.resolveOperatorsWithPruning(AlwaysProcess.fn, ruleId) {
       case q: LogicalPlan =>
         q.mapExpressions(resolve(_, Map.empty))
     }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/timeZoneAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/timeZoneAnalysis.scala
index a27aa845bf0ae..d6c0a0fed66c3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/timeZoneAnalysis.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/timeZoneAnalysis.scala
@@ -16,17 +16,18 @@
  */
 package org.apache.spark.sql.catalyst.analysis
 
+import org.apache.spark.sql.catalyst.SQLConfHelper
 import org.apache.spark.sql.catalyst.expressions.{Cast, Expression, ListQuery, TimeZoneAwareExpression}
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.catalyst.trees.AlwaysProcess
 import org.apache.spark.sql.types.DataType
 
 /**
  * Replace [[TimeZoneAwareExpression]] without timezone id by its copy with session local
  * time zone.
  */
-case class ResolveTimeZone(conf: SQLConf) extends Rule[LogicalPlan] {
+object ResolveTimeZone extends Rule[LogicalPlan] {
   private val transformTimeZoneExprs: PartialFunction[Expression, Expression] = {
     case e: TimeZoneAwareExpression if e.timeZoneId.isEmpty =>
       e.withTimeZone(conf.sessionLocalTimeZone)
@@ -38,7 +39,7 @@ case class ResolveTimeZone(conf: SQLConf) extends Rule[LogicalPlan] {
   }
 
   override def apply(plan: LogicalPlan): LogicalPlan =
-    plan.resolveExpressions(transformTimeZoneExprs)
+    plan.resolveExpressionsWithPruning(AlwaysProcess.fn, ruleId)(transformTimeZoneExprs)
 
   def resolveTimeZones(e: Expression): Expression = e.transform(transformTimeZoneExprs)
 }
@@ -47,10 +48,7 @@ case class ResolveTimeZone(conf: SQLConf) extends Rule[LogicalPlan] {
  * Mix-in trait for constructing valid [[Cast]] expressions.
  */
 trait CastSupport {
-  /**
-   * Configuration used to create a valid cast expression.
-   */
-  def conf: SQLConf
+  self: SQLConfHelper =>
 
   /**
    * Create a Cast expression with the session local time zone.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
index b28be042c43f5..6fcde63ca225c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
@@ -19,30 +19,34 @@ package org.apache.spark.sql.catalyst.analysis
 
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.{FunctionIdentifier, InternalRow, TableIdentifier}
-import org.apache.spark.sql.catalyst.errors.TreeNodeException
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
-import org.apache.spark.sql.catalyst.parser.ParserUtils
-import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LeafNode, LogicalPlan, UnaryNode}
-import org.apache.spark.sql.catalyst.trees.TreeNode
-import org.apache.spark.sql.catalyst.util.quoteIdentifier
+import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
+import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, UnaryNode}
+import org.apache.spark.sql.catalyst.util._
 import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog}
+import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
 import org.apache.spark.sql.types.{DataType, Metadata, StructType}
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
 /**
  * Thrown when an invalid attempt is made to access a property of a tree that has yet to be fully
  * resolved.
  */
-class UnresolvedException[TreeType <: TreeNode[_]](tree: TreeType, function: String)
-  extends TreeNodeException(tree, s"Invalid call to $function on unresolved object", null)
+class UnresolvedException(function: String)
+  extends AnalysisException(s"Invalid call to $function on unresolved object")
 
 /**
  * Holds the name of a relation that has yet to be looked up in a catalog.
  *
  * @param multipartIdentifier table name
+ * @param options options to scan this relation. Only applicable to v2 table scan.
  */
 case class UnresolvedRelation(
-    multipartIdentifier: Seq[String]) extends LeafNode with NamedRelation {
+    multipartIdentifier: Seq[String],
+    options: CaseInsensitiveStringMap = CaseInsensitiveStringMap.empty(),
+    override val isStreaming: Boolean = false)
+  extends LeafNode with NamedRelation {
   import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
 
   /** Returns a `.` separated name for this relation. */
@@ -56,6 +60,14 @@ case class UnresolvedRelation(
 }
 
 object UnresolvedRelation {
+  def apply(
+      tableIdentifier: TableIdentifier,
+      extraOptions: CaseInsensitiveStringMap,
+      isStreaming: Boolean): UnresolvedRelation = {
+    UnresolvedRelation(
+      tableIdentifier.database.toSeq :+ tableIdentifier.table, extraOptions, isStreaming)
+  }
+
   def apply(tableIdentifier: TableIdentifier): UnresolvedRelation =
     UnresolvedRelation(tableIdentifier.database.toSeq :+ tableIdentifier.table)
 }
@@ -108,13 +120,13 @@ case class UnresolvedInlineTable(
  *   select t.a from range(10) t(a);
  * }}}
  *
- * @param functionName name of this table-value function
+ * @param name qualified name of this table-value function
  * @param functionArgs list of function arguments
  * @param outputNames alias names of function output columns. If these names given, an analyzer
  *                    adds [[Project]] to rename the output columns.
  */
 case class UnresolvedTableValuedFunction(
-    functionName: String,
+    name: FunctionIdentifier,
     functionArgs: Seq[Expression],
     outputNames: Seq[String])
   extends LeafNode {
@@ -124,6 +136,15 @@ case class UnresolvedTableValuedFunction(
   override lazy val resolved = false
 }
 
+object UnresolvedTableValuedFunction {
+  def apply(
+      name: String,
+      functionArgs: Seq[Expression],
+      outputNames: Seq[String]): UnresolvedTableValuedFunction = {
+    UnresolvedTableValuedFunction(FunctionIdentifier(name), functionArgs, outputNames)
+  }
+}
+
 /**
  * Holds the name of an attribute that has yet to be resolved.
  */
@@ -132,10 +153,10 @@ case class UnresolvedAttribute(nameParts: Seq[String]) extends Attribute with Un
   def name: String =
     nameParts.map(n => if (n.contains(".")) s"`$n`" else n).mkString(".")
 
-  override def exprId: ExprId = throw new UnresolvedException(this, "exprId")
-  override def dataType: DataType = throw new UnresolvedException(this, "dataType")
-  override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
-  override def qualifier: Seq[String] = throw new UnresolvedException(this, "qualifier")
+  override def exprId: ExprId = throw new UnresolvedException("exprId")
+  override def dataType: DataType = throw new UnresolvedException("dataType")
+  override def nullable: Boolean = throw new UnresolvedException("nullable")
+  override def qualifier: Seq[String] = throw new UnresolvedException("qualifier")
   override lazy val resolved = false
 
   override def newInstance(): UnresolvedAttribute = this
@@ -147,17 +168,15 @@ case class UnresolvedAttribute(nameParts: Seq[String]) extends Attribute with Un
 
   override def toString: String = s"'$name"
 
-  override def sql: String = name match {
-    case ParserUtils.escapedIdentifier(_) | ParserUtils.qualifiedEscapedIdentifier(_, _) => name
-    case _ => quoteIdentifier(name)
-  }
+  override def sql: String = nameParts.map(quoteIfNeeded(_)).mkString(".")
 }
 
 object UnresolvedAttribute {
   /**
    * Creates an [[UnresolvedAttribute]], parsing segments separated by dots ('.').
    */
-  def apply(name: String): UnresolvedAttribute = new UnresolvedAttribute(name.split("\\."))
+  def apply(name: String): UnresolvedAttribute =
+    new UnresolvedAttribute(CatalystSqlParser.parseMultipartIdentifier(name))
 
   /**
    * Creates an [[UnresolvedAttribute]], from a single quoted string (for example using backticks in
@@ -177,10 +196,10 @@ object UnresolvedAttribute {
    * Used to split attribute name by dot with backticks rule.
    * Backticks must appear in pairs, and the quoted string must be a complete name part,
    * which means `ab..c`e.f is not allowed.
-   * Escape character is not supported now, so we can't use backtick inside name part.
+   * We can use backtick only inside quoted name parts.
    */
   def parseAttributeName(name: String): Seq[String] = {
-    def e = new AnalysisException(s"syntax error in attribute name: $name")
+    def e = QueryCompilationErrors.attributeNameSyntaxError(name)
     val nameParts = scala.collection.mutable.ArrayBuffer.empty[String]
     val tmp = scala.collection.mutable.ArrayBuffer.empty[Char]
     var inBacktick = false
@@ -189,8 +208,13 @@ object UnresolvedAttribute {
       val char = name(i)
       if (inBacktick) {
         if (char == '`') {
-          inBacktick = false
-          if (i + 1 < name.length && name(i + 1) != '.') throw e
+          if (i + 1 < name.length && name(i + 1) == '`') {
+            tmp += '`'
+            i += 1
+          } else {
+            inBacktick = false
+            if (i + 1 < name.length && name(i + 1) != '.') throw e
+          }
         } else {
           tmp += char
         }
@@ -222,46 +246,71 @@ object UnresolvedAttribute {
 case class UnresolvedGenerator(name: FunctionIdentifier, children: Seq[Expression])
   extends Generator {
 
-  override def elementSchema: StructType = throw new UnresolvedException(this, "elementTypes")
-  override def dataType: DataType = throw new UnresolvedException(this, "dataType")
-  override def foldable: Boolean = throw new UnresolvedException(this, "foldable")
-  override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
+  override def elementSchema: StructType = throw new UnresolvedException("elementTypes")
+  override def dataType: DataType = throw new UnresolvedException("dataType")
+  override def foldable: Boolean = throw new UnresolvedException("foldable")
+  override def nullable: Boolean = throw new UnresolvedException("nullable")
   override lazy val resolved = false
 
   override def prettyName: String = name.unquotedString
   override def toString: String = s"'$name(${children.mkString(", ")})"
 
   override def eval(input: InternalRow = null): TraversableOnce[InternalRow] =
-    throw new UnsupportedOperationException(s"Cannot evaluate expression: $this")
+    throw QueryExecutionErrors.cannotEvaluateExpressionError(this)
 
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode =
-    throw new UnsupportedOperationException(s"Cannot generate code for expression: $this")
+    throw QueryExecutionErrors.cannotGenerateCodeForExpressionError(this)
 
   override def terminate(): TraversableOnce[InternalRow] =
-    throw new UnsupportedOperationException(s"Cannot terminate expression: $this")
+    throw QueryExecutionErrors.cannotTerminateGeneratorError(this)
+
+  override protected def withNewChildrenInternal(
+    newChildren: IndexedSeq[Expression]): UnresolvedGenerator = copy(children = newChildren)
 }
 
 case class UnresolvedFunction(
-    name: FunctionIdentifier,
+    nameParts: Seq[String],
     arguments: Seq[Expression],
     isDistinct: Boolean,
-    filter: Option[Expression] = None)
+    filter: Option[Expression] = None,
+    ignoreNulls: Boolean = false)
   extends Expression with Unevaluable {
+  import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
 
   override def children: Seq[Expression] = arguments ++ filter.toSeq
 
-  override def dataType: DataType = throw new UnresolvedException(this, "dataType")
-  override def foldable: Boolean = throw new UnresolvedException(this, "foldable")
-  override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
+  override def dataType: DataType = throw new UnresolvedException("dataType")
+  override def nullable: Boolean = throw new UnresolvedException("nullable")
   override lazy val resolved = false
 
-  override def prettyName: String = name.unquotedString
-  override def toString: String = s"'$name(${children.mkString(", ")})"
+  override def prettyName: String = nameParts.quoted
+  override def toString: String = {
+    val distinct = if (isDistinct) "distinct " else ""
+    s"'${nameParts.quoted}($distinct${children.mkString(", ")})"
+  }
+
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[Expression]): UnresolvedFunction = {
+    if (filter.isDefined) {
+      copy(arguments = newChildren.dropRight(1), filter = Some(newChildren.last))
+    } else {
+      copy(arguments = newChildren)
+    }
+  }
 }
 
 object UnresolvedFunction {
+  import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+  def apply(
+      name: FunctionIdentifier,
+      arguments: Seq[Expression],
+      isDistinct: Boolean): UnresolvedFunction = {
+    UnresolvedFunction(name.asMultipart, arguments, isDistinct)
+  }
+
   def apply(name: String, arguments: Seq[Expression], isDistinct: Boolean): UnresolvedFunction = {
-    UnresolvedFunction(FunctionIdentifier(name, None), arguments, isDistinct)
+    UnresolvedFunction(Seq(name), arguments, isDistinct)
   }
 }
 
@@ -271,13 +320,13 @@ object UnresolvedFunction {
  */
 abstract class Star extends LeafExpression with NamedExpression {
 
-  override def name: String = throw new UnresolvedException(this, "name")
-  override def exprId: ExprId = throw new UnresolvedException(this, "exprId")
-  override def dataType: DataType = throw new UnresolvedException(this, "dataType")
-  override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
-  override def qualifier: Seq[String] = throw new UnresolvedException(this, "qualifier")
-  override def toAttribute: Attribute = throw new UnresolvedException(this, "toAttribute")
-  override def newInstance(): NamedExpression = throw new UnresolvedException(this, "newInstance")
+  override def name: String = throw new UnresolvedException("name")
+  override def exprId: ExprId = throw new UnresolvedException("exprId")
+  override def dataType: DataType = throw new UnresolvedException("dataType")
+  override def nullable: Boolean = throw new UnresolvedException("nullable")
+  override def qualifier: Seq[String] = throw new UnresolvedException("qualifier")
+  override def toAttribute: Attribute = throw new UnresolvedException("toAttribute")
+  override def newInstance(): NamedExpression = throw new UnresolvedException("newInstance")
   override lazy val resolved = false
 
   def expand(input: LogicalPlan, resolver: Resolver): Seq[NamedExpression]
@@ -301,11 +350,11 @@ case class UnresolvedStar(target: Option[Seq[String]]) extends Star with Unevalu
    * Returns true if the nameParts is a subset of the last elements of qualifier of the attribute.
    *
    * For example, the following should all return true:
-   *   - `SELECT ns1.ns2.t.* FROM ns1.n2.t` where nameParts is Seq("ns1", "ns2", "t") and
+   *   - `SELECT ns1.ns2.t.* FROM ns1.ns2.t` where nameParts is Seq("ns1", "ns2", "t") and
    *     qualifier is Seq("ns1", "ns2", "t").
-   *   - `SELECT ns2.t.* FROM ns1.n2.t` where nameParts is Seq("ns2", "t") and
+   *   - `SELECT ns2.t.* FROM ns1.ns2.t` where nameParts is Seq("ns2", "t") and
    *     qualifier is Seq("ns1", "ns2", "t").
-   *   - `SELECT t.* FROM ns1.n2.t` where nameParts is Seq("t") and
+   *   - `SELECT t.* FROM ns1.ns2.t` where nameParts is Seq("t") and
    *     qualifier is Seq("ns1", "ns2", "t").
    */
   private def matchedQualifier(
@@ -320,13 +369,20 @@ case class UnresolvedStar(target: Option[Seq[String]]) extends Star with Unevalu
     nameParts.corresponds(qualifierList)(resolver)
   }
 
+  def isQualifiedByTable(input: LogicalPlan, resolver: Resolver): Boolean = {
+    target.exists(nameParts => input.output.exists(matchedQualifier(_, nameParts, resolver)))
+  }
+
   override def expand(
       input: LogicalPlan,
       resolver: Resolver): Seq[NamedExpression] = {
-    // If there is no table specified, use all input attributes.
+    // If there is no table specified, use all non-hidden input attributes.
     if (target.isEmpty) return input.output
 
-    val expandedAttributes = input.output.filter(matchedQualifier(_, target.get, resolver))
+    // If there is a table specified, use hidden input attributes as well
+    val hiddenOutput = input.metadataOutput.filter(_.supportsQualifiedStar)
+    val expandedAttributes = (hiddenOutput ++ input.output).filter(
+      matchedQualifier(_, target.get, resolver))
 
     if (expandedAttributes.nonEmpty) return expandedAttributes
 
@@ -343,13 +399,13 @@ case class UnresolvedStar(target: Option[Seq[String]]) extends Star with Unevalu
         }
 
         case _ =>
-          throw new AnalysisException("Can only star expand struct data types. Attribute: `" +
-            target.get + "`")
+          throw QueryCompilationErrors.starExpandDataTypeNotSupportedError(target.get)
       }
     } else {
       val from = input.inputSet.map(_.name).mkString(", ")
       val targetString = target.get.mkString(".")
-      throw new AnalysisException(s"cannot resolve '$targetString.*' given input columns '$from'")
+      throw QueryCompilationErrors.cannotResolveStarExpandGivenInputColumnsError(
+        targetString, from)
     }
   }
 
@@ -392,24 +448,26 @@ case class UnresolvedRegex(regexPattern: String, table: Option[String], caseSens
 case class MultiAlias(child: Expression, names: Seq[String])
   extends UnaryExpression with NamedExpression with Unevaluable {
 
-  override def name: String = throw new UnresolvedException(this, "name")
+  override def name: String = throw new UnresolvedException("name")
 
-  override def exprId: ExprId = throw new UnresolvedException(this, "exprId")
+  override def exprId: ExprId = throw new UnresolvedException("exprId")
 
-  override def dataType: DataType = throw new UnresolvedException(this, "dataType")
+  override def dataType: DataType = throw new UnresolvedException("dataType")
 
-  override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
+  override def nullable: Boolean = throw new UnresolvedException("nullable")
 
-  override def qualifier: Seq[String] = throw new UnresolvedException(this, "qualifier")
+  override def qualifier: Seq[String] = throw new UnresolvedException("qualifier")
 
-  override def toAttribute: Attribute = throw new UnresolvedException(this, "toAttribute")
+  override def toAttribute: Attribute = throw new UnresolvedException("toAttribute")
 
-  override def newInstance(): NamedExpression = throw new UnresolvedException(this, "newInstance")
+  override def newInstance(): NamedExpression = throw new UnresolvedException("newInstance")
 
   override lazy val resolved = false
 
   override def toString: String = s"$child AS $names"
 
+  override protected def withNewChildInternal(newChild: Expression): MultiAlias =
+    copy(child = newChild)
 }
 
 /**
@@ -419,7 +477,7 @@ case class MultiAlias(child: Expression, names: Seq[String])
  * @param expressions Expressions to expand.
  */
 case class ResolvedStar(expressions: Seq[NamedExpression]) extends Star with Unevaluable {
-  override def newInstance(): NamedExpression = throw new UnresolvedException(this, "newInstance")
+  override def newInstance(): NamedExpression = throw new UnresolvedException("newInstance")
   override def expand(input: LogicalPlan, resolver: Resolver): Seq[NamedExpression] = expressions
   override def toString: String = expressions.mkString("ResolvedStar(", ", ", ")")
 }
@@ -438,13 +496,17 @@ case class UnresolvedExtractValue(child: Expression, extraction: Expression)
   override def left: Expression = child
   override def right: Expression = extraction
 
-  override def dataType: DataType = throw new UnresolvedException(this, "dataType")
-  override def foldable: Boolean = throw new UnresolvedException(this, "foldable")
-  override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
+  override def dataType: DataType = throw new UnresolvedException("dataType")
+  override def nullable: Boolean = throw new UnresolvedException("nullable")
   override lazy val resolved = false
 
   override def toString: String = s"$child[$extraction]"
   override def sql: String = s"${child.sql}[${extraction.sql}]"
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): UnresolvedExtractValue = {
+      copy(child = newLeft, extraction = newRight)
+  }
 }
 
 /**
@@ -460,15 +522,18 @@ case class UnresolvedAlias(
     aliasFunc: Option[Expression => String] = None)
   extends UnaryExpression with NamedExpression with Unevaluable {
 
-  override def toAttribute: Attribute = throw new UnresolvedException(this, "toAttribute")
-  override def qualifier: Seq[String] = throw new UnresolvedException(this, "qualifier")
-  override def exprId: ExprId = throw new UnresolvedException(this, "exprId")
-  override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
-  override def dataType: DataType = throw new UnresolvedException(this, "dataType")
-  override def name: String = throw new UnresolvedException(this, "name")
-  override def newInstance(): NamedExpression = throw new UnresolvedException(this, "newInstance")
+  override def toAttribute: Attribute = throw new UnresolvedException("toAttribute")
+  override def qualifier: Seq[String] = throw new UnresolvedException("qualifier")
+  override def exprId: ExprId = throw new UnresolvedException("exprId")
+  override def nullable: Boolean = throw new UnresolvedException("nullable")
+  override def dataType: DataType = throw new UnresolvedException("dataType")
+  override def name: String = throw new UnresolvedException("name")
+  override def newInstance(): NamedExpression = throw new UnresolvedException("newInstance")
 
   override lazy val resolved = false
+
+  override protected def withNewChildInternal(newChild: Expression): UnresolvedAlias =
+    copy(child = newChild)
 }
 
 /**
@@ -490,6 +555,9 @@ case class UnresolvedSubqueryColumnAliases(
   override def output: Seq[Attribute] = Nil
 
   override lazy val resolved = false
+
+  override protected def withNewChildInternal(
+    newChild: LogicalPlan): UnresolvedSubqueryColumnAliases = copy(child = newChild)
 }
 
 /**
@@ -508,16 +576,28 @@ case class UnresolvedDeserializer(deserializer: Expression, inputAttributes: Seq
   require(inputAttributes.forall(_.resolved), "Input attributes must all be resolved.")
 
   override def child: Expression = deserializer
-  override def dataType: DataType = throw new UnresolvedException(this, "dataType")
-  override def foldable: Boolean = throw new UnresolvedException(this, "foldable")
-  override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
+  override def dataType: DataType = throw new UnresolvedException("dataType")
+  override def nullable: Boolean = throw new UnresolvedException("nullable")
   override lazy val resolved = false
+
+  override protected def withNewChildInternal(newChild: Expression): UnresolvedDeserializer =
+    copy(deserializer = newChild)
 }
 
 case class GetColumnByOrdinal(ordinal: Int, dataType: DataType) extends LeafExpression
   with Unevaluable with NonSQLExpression {
-  override def foldable: Boolean = throw new UnresolvedException(this, "foldable")
-  override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
+  override def nullable: Boolean = throw new UnresolvedException("nullable")
+  override lazy val resolved = false
+}
+
+case class GetViewColumnByNameAndOrdinal(
+    viewName: String,
+    colName: String,
+    ordinal: Int,
+    expectedNumCandidates: Int)
+  extends LeafExpression with Unevaluable with NonSQLExpression {
+  override def dataType: DataType = throw new UnresolvedException("dataType")
+  override def nullable: Boolean = throw new UnresolvedException("nullable")
   override lazy val resolved = false
 }
 
@@ -533,9 +613,8 @@ case class GetColumnByOrdinal(ordinal: Int, dataType: DataType) extends LeafExpr
  */
 case class UnresolvedOrdinal(ordinal: Int)
     extends LeafExpression with Unevaluable with NonSQLExpression {
-  override def dataType: DataType = throw new UnresolvedException(this, "dataType")
-  override def foldable: Boolean = throw new UnresolvedException(this, "foldable")
-  override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
+  override def dataType: DataType = throw new UnresolvedException("dataType")
+  override def nullable: Boolean = throw new UnresolvedException("nullable")
   override lazy val resolved = false
 }
 
@@ -549,4 +628,15 @@ case class UnresolvedHaving(
   extends UnaryNode {
   override lazy val resolved: Boolean = false
   override def output: Seq[Attribute] = child.output
+  override protected def withNewChildInternal(newChild: LogicalPlan): UnresolvedHaving =
+    copy(child = newChild)
+}
+
+/**
+ * A place holder expression used in random functions, will be replaced after analyze.
+ */
+case object UnresolvedSeed extends LeafExpression with Unevaluable {
+  override def nullable: Boolean = throw new UnresolvedException("nullable")
+  override def dataType: DataType = throw new UnresolvedException("dataType")
+  override lazy val resolved = false
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/v2ResolutionPlans.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/v2ResolutionPlans.scala
index f3d40c6d36cc3..b50c306805436 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/v2ResolutionPlans.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/v2ResolutionPlans.scala
@@ -17,9 +17,14 @@
 
 package org.apache.spark.sql.catalyst.analysis
 
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan}
-import org.apache.spark.sql.connector.catalog.{CatalogPlugin, Identifier, SupportsNamespaces, Table, TableCatalog}
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
+import org.apache.spark.sql.catalyst.expressions.{Attribute, LeafExpression, Unevaluable}
+import org.apache.spark.sql.catalyst.plans.logical.LeafNode
+import org.apache.spark.sql.catalyst.util.CharVarcharUtils
+import org.apache.spark.sql.connector.catalog.{CatalogPlugin, Identifier, Table, TableCatalog}
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+import org.apache.spark.sql.types.DataType
 
 /**
  * Holds the name of a namespace that has yet to be looked up in a catalog. It will be resolved to
@@ -35,7 +40,24 @@ case class UnresolvedNamespace(multipartIdentifier: Seq[String]) extends LeafNod
  * Holds the name of a table that has yet to be looked up in a catalog. It will be resolved to
  * [[ResolvedTable]] during analysis.
  */
-case class UnresolvedTable(multipartIdentifier: Seq[String]) extends LeafNode {
+case class UnresolvedTable(
+    multipartIdentifier: Seq[String],
+    commandName: String,
+    relationTypeMismatchHint: Option[String]) extends LeafNode {
+  override lazy val resolved: Boolean = false
+
+  override def output: Seq[Attribute] = Nil
+}
+
+/**
+ * Holds the name of a view that has yet to be looked up in a catalog. It will be resolved to
+ * [[ResolvedView]] during analysis.
+ */
+case class UnresolvedView(
+    multipartIdentifier: Seq[String],
+    commandName: String,
+    allowTemp: Boolean,
+    relationTypeMismatchHint: Option[String]) extends LeafNode {
   override lazy val resolved: Boolean = false
 
   override def output: Seq[Attribute] = Nil
@@ -45,7 +67,32 @@ case class UnresolvedTable(multipartIdentifier: Seq[String]) extends LeafNode {
  * Holds the name of a table or view that has yet to be looked up in a catalog. It will
  * be resolved to [[ResolvedTable]] or [[ResolvedView]] during analysis.
  */
-case class UnresolvedTableOrView(multipartIdentifier: Seq[String]) extends LeafNode {
+case class UnresolvedTableOrView(
+    multipartIdentifier: Seq[String],
+    commandName: String,
+    allowTempView: Boolean) extends LeafNode {
+  override lazy val resolved: Boolean = false
+  override def output: Seq[Attribute] = Nil
+}
+
+sealed trait PartitionSpec extends LeafExpression with Unevaluable {
+  override def dataType: DataType = throw new IllegalStateException(
+    "PartitionSpec.dataType should not be called.")
+  override def nullable: Boolean = throw new IllegalStateException(
+    "PartitionSpec.nullable should not be called.")
+}
+
+case class UnresolvedPartitionSpec(
+    spec: TablePartitionSpec,
+    location: Option[String] = None) extends PartitionSpec {
+  override lazy val resolved = false
+}
+
+/**
+ * Holds the name of a function that has yet to be looked up in a catalog. It will be resolved to
+ * [[ResolvedFunc]] during analysis.
+ */
+case class UnresolvedFunc(multipartIdentifier: Seq[String]) extends LeafNode {
   override lazy val resolved: Boolean = false
   override def output: Seq[Attribute] = Nil
 }
@@ -61,16 +108,49 @@ case class ResolvedNamespace(catalog: CatalogPlugin, namespace: Seq[String])
 /**
  * A plan containing resolved table.
  */
-case class ResolvedTable(catalog: TableCatalog, identifier: Identifier, table: Table)
+case class ResolvedTable(
+    catalog: TableCatalog,
+    identifier: Identifier,
+    table: Table,
+    outputAttributes: Seq[Attribute])
   extends LeafNode {
-  override def output: Seq[Attribute] = Nil
+  override def output: Seq[Attribute] = {
+    val qualifier = catalog.name +: identifier.namespace :+ identifier.name
+    outputAttributes.map(_.withQualifier(qualifier))
+  }
+  def name: String = (catalog.name +: identifier.namespace() :+ identifier.name()).quoted
 }
 
+object ResolvedTable {
+  def create(
+      catalog: TableCatalog,
+      identifier: Identifier,
+      table: Table): ResolvedTable = {
+    val schema = CharVarcharUtils.replaceCharVarcharWithStringInSchema(table.schema)
+    ResolvedTable(catalog, identifier, table, schema.toAttributes)
+  }
+}
+
+case class ResolvedPartitionSpec(
+    names: Seq[String],
+    ident: InternalRow,
+    location: Option[String] = None) extends PartitionSpec
+
 /**
  * A plan containing resolved (temp) views.
  */
 // TODO: create a generic representation for temp view, v1 view and v2 view, after we add view
 //       support to v2 catalog. For now we only need the identifier to fallback to v1 command.
-case class ResolvedView(identifier: Identifier) extends LeafNode {
+case class ResolvedView(identifier: Identifier, isTemp: Boolean) extends LeafNode {
+  override def output: Seq[Attribute] = Nil
+}
+
+/**
+ * A plan containing resolved function.
+ */
+// TODO: create a generic representation for v1, v2 function, after we add function
+//       support to v2 catalog. For now we only need the identifier to fallback to v1 command.
+case class ResolvedFunc(identifier: Identifier)
+  extends LeafNode {
   override def output: Seq[Attribute] = Nil
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/view.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/view.scala
index 65601640fa044..66a5052d86f0a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/view.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/view.scala
@@ -17,77 +17,21 @@
 
 package org.apache.spark.sql.catalyst.analysis
 
-import org.apache.spark.sql.catalyst.expressions.Alias
-import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project, View}
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, View}
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.internal.SQLConf
 
 /**
  * This file defines view types and analysis rules related to views.
  */
 
 /**
- * This rule has two goals:
- *
- * 1. Removes [[View]] operators from the plan. The operator is respected till the end of analysis
- * stage because we want to see which part of an analyzed logical plan is generated from a view.
- *
- * 2. Make sure that a view's child plan produces the view's output attributes. We try to wrap the
- * child by:
- * 1. Generate the `queryOutput` by:
- *    1.1. If the query column names are defined, map the column names to attributes in the child
- *         output by name(This is mostly for handling view queries like SELECT * FROM ..., the
- *         schema of the referenced table/view may change after the view has been created, so we
- *         have to save the output of the query to `viewQueryColumnNames`, and restore them during
- *         view resolution, in this way, we are able to get the correct view column ordering and
- *         omit the extra columns that we don't require);
- *    1.2. Else set the child output attributes to `queryOutput`.
- * 2. Map the `queryOutput` to view output by index, if the corresponding attributes don't match,
- *    try to up cast and alias the attribute in `queryOutput` to the attribute in the view output.
- * 3. Add a Project over the child, with the new output generated by the previous steps.
- *
- * Once reaches this rule, it means `CheckAnalysis` did necessary checks on number of columns
- * between the view output and the child output or the query column names. `CheckAnalysis` also
- * checked the cast from the view's child to the Project is up-cast.
- *
- * This should be only done after the batch of Resolution, because the view attributes are not
- * completely resolved during the batch of Resolution.
+ * This rule removes [[View]] operators from the plan. The operator is respected till the end of
+ * analysis stage because we want to see which part of an analyzed logical plan is generated from a
+ * view.
  */
 object EliminateView extends Rule[LogicalPlan] with CastSupport {
-  override def conf: SQLConf = SQLConf.get
-
   override def apply(plan: LogicalPlan): LogicalPlan = plan transformUp {
-    // The child has the different output attributes with the View operator. Adds a Project over
-    // the child of the view.
-    case v @ View(desc, output, child) if child.resolved && !v.sameOutput(child) =>
-      val resolver = conf.resolver
-      val queryColumnNames = desc.viewQueryColumnNames
-      val queryOutput = if (queryColumnNames.nonEmpty) {
-        // Find the attribute that has the expected attribute name from an attribute list, the names
-        // are compared using conf.resolver.
-        // `CheckAnalysis` already guarantees the expected attribute can be found for sure.
-        desc.viewQueryColumnNames.map { colName =>
-          child.output.find(attr => resolver(attr.name, colName)).get
-        }
-      } else {
-        // For view created before Spark 2.2.0, the view text is already fully qualified, the plan
-        // output is the same with the view output.
-        child.output
-      }
-      // Map the attributes in the query output to the attributes in the view output by index.
-      val newOutput = output.zip(queryOutput).map {
-        case (attr, originAttr) if !attr.semanticEquals(originAttr) =>
-          // `CheckAnalysis` already guarantees that the cast is a up-cast for sure.
-          Alias(cast(originAttr, attr.dataType), attr.name)(exprId = attr.exprId,
-            qualifier = attr.qualifier, explicitMetadata = Some(attr.metadata))
-        case (_, originAttr) => originAttr
-      }
-      Project(newOutput, child)
-
-    // The child should have the same output attributes with the View operator, so we simply
-    // remove the View operator.
-    case View(_, _, child) =>
-      child
+    case View(_, _, child) => child
   }
 }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalog.scala
index db930cf7890e6..5643bf8b3a9b7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalog.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.catalyst.catalog
 
-import org.apache.spark.sql.catalyst.analysis.{FunctionAlreadyExistsException, NoSuchDatabaseException, NoSuchFunctionException, NoSuchPartitionException, NoSuchTableException}
+import org.apache.spark.sql.catalyst.analysis.{FunctionAlreadyExistsException, NoSuchDatabaseException, NoSuchFunctionException, NoSuchTableException}
 import org.apache.spark.sql.catalyst.expressions.Expression
 import org.apache.spark.sql.types.StructType
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala
index ae3b75dc3334b..c2b74604715b2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.analysis.Resolver
 import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
 import org.apache.spark.sql.catalyst.expressions.{And, AttributeReference, BoundReference, Expression, Predicate}
+import org.apache.spark.sql.catalyst.util.CharVarcharUtils
 
 object ExternalCatalogUtils {
   // This duplicates default value of Hive `ConfVars.DEFAULTPARTITIONNAME`, since catalyst doesn't
@@ -118,12 +119,16 @@ object ExternalCatalogUtils {
     }
   }
 
-  def getPartitionPathString(col: String, value: String): String = {
-    val partitionString = if (value == null || value.isEmpty) {
+  def getPartitionValueString(value: String): String = {
+    if (value == null || value.isEmpty) {
       DEFAULT_PARTITION_NAME
     } else {
       escapePathName(value)
     }
+  }
+
+  def getPartitionPathString(col: String, value: String): String = {
+    val partitionString = getPartitionValueString(value)
     escapePathName(col) + "=" + partitionString
   }
 
@@ -135,7 +140,8 @@ object ExternalCatalogUtils {
     if (predicates.isEmpty) {
       inputPartitions
     } else {
-      val partitionSchema = catalogTable.partitionSchema
+      val partitionSchema = CharVarcharUtils.replaceCharVarcharWithStringInSchema(
+        catalogTable.partitionSchema)
       val partitionColumnNames = catalogTable.partitionColumnNames.toSet
 
       val nonPartitionPruningPredicates = predicates.filterNot {
@@ -159,6 +165,10 @@ object ExternalCatalogUtils {
     }
   }
 
+  private def isNullPartitionValue(value: String): Boolean = {
+    value == null || value == DEFAULT_PARTITION_NAME
+  }
+
   /**
    * Returns true if `spec1` is a partial partition spec w.r.t. `spec2`, e.g. PARTITION (a=1) is a
    * partial partition spec w.r.t. PARTITION (a=1,b=2).
@@ -167,9 +177,15 @@ object ExternalCatalogUtils {
       spec1: TablePartitionSpec,
       spec2: TablePartitionSpec): Boolean = {
     spec1.forall {
+      case (partitionColumn, value) if isNullPartitionValue(value) =>
+        isNullPartitionValue(spec2(partitionColumn))
       case (partitionColumn, value) => spec2(partitionColumn) == value
     }
   }
+
+  def convertNullPartitionValues(spec: TablePartitionSpec): TablePartitionSpec = {
+    spec.mapValues(v => if (v == null) DEFAULT_PARTITION_NAME else v).map(identity).toMap
+  }
 }
 
 object CatalogUtils {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/GlobalTempViewManager.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/GlobalTempViewManager.scala
index 6095ac0bc9c50..5141c66f86cd8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/GlobalTempViewManager.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/GlobalTempViewManager.scala
@@ -21,10 +21,9 @@ import javax.annotation.concurrent.GuardedBy
 
 import scala.collection.mutable
 
-import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.analysis.TempTableAlreadyExistsException
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.util.StringUtils
+import org.apache.spark.sql.errors.QueryCompilationErrors
 
 
 /**
@@ -40,12 +39,12 @@ class GlobalTempViewManager(val database: String) {
 
   /** List of view definitions, mapping from view name to logical plan. */
   @GuardedBy("this")
-  private val viewDefinitions = new mutable.HashMap[String, LogicalPlan]
+  private val viewDefinitions = new mutable.HashMap[String, TemporaryViewRelation]
 
   /**
    * Returns the global view definition which matches the given name, or None if not found.
    */
-  def get(name: String): Option[LogicalPlan] = synchronized {
+  def get(name: String): Option[TemporaryViewRelation] = synchronized {
     viewDefinitions.get(name)
   }
 
@@ -55,7 +54,7 @@ class GlobalTempViewManager(val database: String) {
    */
   def create(
       name: String,
-      viewDefinition: LogicalPlan,
+      viewDefinition: TemporaryViewRelation,
       overrideIfExists: Boolean): Unit = synchronized {
     if (!overrideIfExists && viewDefinitions.contains(name)) {
       throw new TempTableAlreadyExistsException(name)
@@ -68,7 +67,7 @@ class GlobalTempViewManager(val database: String) {
    */
   def update(
       name: String,
-      viewDefinition: LogicalPlan): Boolean = synchronized {
+      viewDefinition: TemporaryViewRelation): Boolean = synchronized {
     if (viewDefinitions.contains(name)) {
       viewDefinitions.put(name, viewDefinition)
       true
@@ -92,8 +91,7 @@ class GlobalTempViewManager(val database: String) {
   def rename(oldName: String, newName: String): Boolean = synchronized {
     if (viewDefinitions.contains(oldName)) {
       if (viewDefinitions.contains(newName)) {
-        throw new AnalysisException(
-          s"rename temporary view from '$oldName' to '$newName': destination view already exists")
+        throw QueryCompilationErrors.renameTempViewToExistingViewError(oldName, newName)
       }
 
       val viewDefinition = viewDefinitions(oldName)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala
index 31644a5ae4e35..ab7e49581a9f4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala
@@ -24,13 +24,13 @@ import scala.collection.mutable
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.Path
 
-import org.apache.spark.{SparkConf, SparkException}
-import org.apache.spark.sql.AnalysisException
+import org.apache.spark.SparkConf
 import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
 import org.apache.spark.sql.catalyst.analysis._
 import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils._
 import org.apache.spark.sql.catalyst.expressions.Expression
 import org.apache.spark.sql.catalyst.util.StringUtils
+import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
 import org.apache.spark.sql.types.StructType
 
 /**
@@ -94,6 +94,15 @@ class InMemoryCatalog(
     }
   }
 
+  private def toCatalogPartitionSpec = ExternalCatalogUtils.convertNullPartitionValues(_)
+  private def toCatalogPartitionSpecs(specs: Seq[TablePartitionSpec]): Seq[TablePartitionSpec] = {
+    specs.map(toCatalogPartitionSpec)
+  }
+  private def toCatalogPartitionSpec(
+      parts: Seq[CatalogTablePartition]): Seq[CatalogTablePartition] = {
+    parts.map(part => part.copy(spec = toCatalogPartitionSpec(part.spec)))
+  }
+
   // --------------------------------------------------------------------------
   // Databases
   // --------------------------------------------------------------------------
@@ -112,8 +121,8 @@ class InMemoryCatalog(
         fs.mkdirs(location)
       } catch {
         case e: IOException =>
-          throw new SparkException(s"Unable to create database ${dbDefinition.name} as failed " +
-            s"to create its directory ${dbDefinition.locationUri}", e)
+          throw QueryExecutionErrors.unableToCreateDatabaseAsFailedToCreateDirectoryError(
+            dbDefinition, e)
       }
       catalog.put(dbDefinition.name, new DatabaseDesc(dbDefinition))
     }
@@ -127,10 +136,10 @@ class InMemoryCatalog(
       if (!cascade) {
         // If cascade is false, make sure the database is empty.
         if (catalog(db).tables.nonEmpty) {
-          throw new AnalysisException(s"Database $db is not empty. One or more tables exist.")
+          throw QueryCompilationErrors.databaseNotEmptyError(db, "tables")
         }
         if (catalog(db).functions.nonEmpty) {
-          throw new AnalysisException(s"Database '$db' is not empty. One or more functions exist.")
+          throw QueryCompilationErrors.databaseNotEmptyError(db, "functions")
         }
       }
       // Remove the database.
@@ -141,8 +150,8 @@ class InMemoryCatalog(
         fs.delete(location, true)
       } catch {
         case e: IOException =>
-          throw new SparkException(s"Unable to drop database ${dbDefinition.name} as failed " +
-            s"to delete its directory ${dbDefinition.locationUri}", e)
+          throw QueryExecutionErrors.unableToDropDatabaseAsFailedToDeleteDirectoryError(
+            dbDefinition, e)
       }
       catalog.remove(db)
     } else {
@@ -209,8 +218,8 @@ class InMemoryCatalog(
           fs.mkdirs(defaultTableLocation)
         } catch {
           case e: IOException =>
-            throw new SparkException(s"Unable to create table $table as failed " +
-              s"to create its directory $defaultTableLocation", e)
+            throw QueryExecutionErrors.unableToCreateTableAsFailedToCreateDirectoryError(
+              table, defaultTableLocation, e)
         }
         tableDefinition.withNewStorage(locationUri = Some(defaultTableLocation.toUri))
       } else {
@@ -239,7 +248,7 @@ class InMemoryCatalog(
             fs.delete(partitionPath, true)
           } catch {
             case e: IOException =>
-              throw new SparkException(s"Unable to delete partition path $partitionPath", e)
+              throw QueryExecutionErrors.unableToDeletePartitionPathError(partitionPath, e)
           }
         }
         assert(tableMeta.storage.locationUri.isDefined,
@@ -252,8 +261,8 @@ class InMemoryCatalog(
           fs.delete(dir, true)
         } catch {
           case e: IOException =>
-            throw new SparkException(s"Unable to drop table $table as failed " +
-              s"to delete its directory $dir", e)
+            throw QueryExecutionErrors.unableToDropTableAsFailedToDeleteDirectoryError(
+              table, dir, e)
         }
       }
       catalog(db).tables.remove(table)
@@ -284,8 +293,8 @@ class InMemoryCatalog(
         fs.rename(oldDir, newDir)
       } catch {
         case e: IOException =>
-          throw new SparkException(s"Unable to rename table $oldName to $newName as failed " +
-            s"to rename its directory $oldDir", e)
+          throw QueryExecutionErrors.unableToRenameTableAsFailedToRenameDirectoryError(
+            oldName, newName, oldDir, e)
       }
       oldDesc.table = oldDesc.table.withNewStorage(locationUri = Some(newDir.toUri))
     }
@@ -333,8 +342,7 @@ class InMemoryCatalog(
   }
 
   override def tableExists(db: String, table: String): Boolean = synchronized {
-    requireDbExists(db)
-    catalog(db).tables.contains(table)
+    catalog.contains(db) && catalog(db).tables.contains(table)
   }
 
   override def listTables(db: String): Seq[String] = synchronized {
@@ -358,7 +366,7 @@ class InMemoryCatalog(
       loadPath: String,
       isOverwrite: Boolean,
       isSrcLocal: Boolean): Unit = {
-    throw new UnsupportedOperationException("loadTable is not implemented")
+    throw QueryExecutionErrors.methodNotImplementedError("loadTable")
   }
 
   override def loadPartition(
@@ -369,7 +377,7 @@ class InMemoryCatalog(
       isOverwrite: Boolean,
       inheritTableSpecs: Boolean,
       isSrcLocal: Boolean): Unit = {
-    throw new UnsupportedOperationException("loadPartition is not implemented.")
+    throw QueryExecutionErrors.methodNotImplementedError("loadPartition")
   }
 
   override def loadDynamicPartitions(
@@ -379,7 +387,7 @@ class InMemoryCatalog(
       partition: TablePartitionSpec,
       replace: Boolean,
       numDP: Int): Unit = {
-    throw new UnsupportedOperationException("loadDynamicPartitions is not implemented.")
+    throw QueryExecutionErrors.methodNotImplementedError("loadDynamicPartitions")
   }
 
   // --------------------------------------------------------------------------
@@ -389,10 +397,11 @@ class InMemoryCatalog(
   override def createPartitions(
       db: String,
       table: String,
-      parts: Seq[CatalogTablePartition],
+      newParts: Seq[CatalogTablePartition],
       ignoreIfExists: Boolean): Unit = synchronized {
     requireTableExists(db, table)
     val existingParts = catalog(db).tables(table).partitions
+    val parts = toCatalogPartitionSpec(newParts)
     if (!ignoreIfExists) {
       val dupSpecs = parts.collect { case p if existingParts.contains(p.spec) => p.spec }
       if (dupSpecs.nonEmpty) {
@@ -416,7 +425,7 @@ class InMemoryCatalog(
         }
       } catch {
         case e: IOException =>
-          throw new SparkException(s"Unable to create partition path $partitionPath", e)
+          throw QueryExecutionErrors.unableToCreatePartitionPathError(partitionPath, e)
       }
 
       existingParts.put(
@@ -428,12 +437,13 @@ class InMemoryCatalog(
   override def dropPartitions(
       db: String,
       table: String,
-      partSpecs: Seq[TablePartitionSpec],
+      parts: Seq[TablePartitionSpec],
       ignoreIfNotExists: Boolean,
       purge: Boolean,
       retainData: Boolean): Unit = synchronized {
     requireTableExists(db, table)
     val existingParts = catalog(db).tables(table).partitions
+    val partSpecs = toCatalogPartitionSpecs(parts)
     if (!ignoreIfNotExists) {
       val missingSpecs = partSpecs.collect { case s if !existingParts.contains(s) => s }
       if (missingSpecs.nonEmpty) {
@@ -457,7 +467,7 @@ class InMemoryCatalog(
           fs.delete(partitionPath, true)
         } catch {
           case e: IOException =>
-            throw new SparkException(s"Unable to delete partition path $partitionPath", e)
+            throw QueryExecutionErrors.unableToDeletePartitionPathError(partitionPath, e)
         }
       }
       existingParts.remove(p)
@@ -467,8 +477,10 @@ class InMemoryCatalog(
   override def renamePartitions(
       db: String,
       table: String,
-      specs: Seq[TablePartitionSpec],
-      newSpecs: Seq[TablePartitionSpec]): Unit = synchronized {
+      fromSpecs: Seq[TablePartitionSpec],
+      toSpecs: Seq[TablePartitionSpec]): Unit = synchronized {
+    val specs = toCatalogPartitionSpecs(fromSpecs)
+    val newSpecs = toCatalogPartitionSpecs(toSpecs)
     require(specs.size == newSpecs.size, "number of old and new partition specs differ")
     requirePartitionsExist(db, table, specs)
     requirePartitionsNotExist(db, table, newSpecs)
@@ -490,7 +502,7 @@ class InMemoryCatalog(
           fs.rename(oldPartPath, newPartPath)
         } catch {
           case e: IOException =>
-            throw new SparkException(s"Unable to rename partition path $oldPartPath", e)
+            throw QueryExecutionErrors.unableToRenamePartitionPathError(oldPartPath, e)
         }
         oldPartition.copy(
           spec = newSpec,
@@ -507,7 +519,8 @@ class InMemoryCatalog(
   override def alterPartitions(
       db: String,
       table: String,
-      parts: Seq[CatalogTablePartition]): Unit = synchronized {
+      alterParts: Seq[CatalogTablePartition]): Unit = synchronized {
+    val parts = toCatalogPartitionSpec(alterParts)
     requirePartitionsExist(db, table, parts.map(p => p.spec))
     parts.foreach { p =>
       catalog(db).tables(table).partitions.put(p.spec, p)
@@ -517,7 +530,8 @@ class InMemoryCatalog(
   override def getPartition(
       db: String,
       table: String,
-      spec: TablePartitionSpec): CatalogTablePartition = synchronized {
+      partSpec: TablePartitionSpec): CatalogTablePartition = synchronized {
+    val spec = toCatalogPartitionSpec(partSpec)
     requirePartitionsExist(db, table, Seq(spec))
     catalog(db).tables(table).partitions(spec)
   }
@@ -525,7 +539,8 @@ class InMemoryCatalog(
   override def getPartitionOption(
       db: String,
       table: String,
-      spec: TablePartitionSpec): Option[CatalogTablePartition] = synchronized {
+      partSpec: TablePartitionSpec): Option[CatalogTablePartition] = synchronized {
+    val spec = toCatalogPartitionSpec(partSpec)
     if (!partitionExists(db, table, spec)) {
       None
     } else {
@@ -536,12 +551,17 @@ class InMemoryCatalog(
   override def listPartitionNames(
       db: String,
       table: String,
-      partialSpec: Option[TablePartitionSpec] = None): Seq[String] = synchronized {
+      partSpec: Option[TablePartitionSpec] = None): Seq[String] = synchronized {
     val partitionColumnNames = getTable(db, table).partitionColumnNames
-
+    val partialSpec = partSpec.map(toCatalogPartitionSpec)
     listPartitions(db, table, partialSpec).map { partition =>
       partitionColumnNames.map { name =>
-        escapePathName(name) + "=" + escapePathName(partition.spec(name))
+        val partValue = if (partition.spec(name) == null) {
+          DEFAULT_PARTITION_NAME
+        } else {
+          escapePathName(partition.spec(name))
+        }
+        escapePathName(name) + "=" + partValue
       }.mkString("/")
     }.sorted
   }
@@ -552,7 +572,7 @@ class InMemoryCatalog(
       partialSpec: Option[TablePartitionSpec] = None): Seq[CatalogTablePartition] = synchronized {
     requireTableExists(db, table)
 
-    partialSpec match {
+    partialSpec.map(toCatalogPartitionSpec) match {
       case None => catalog(db).tables(table).partitions.values.toSeq
       case Some(partial) =>
         catalog(db).tables(table).partitions.toSeq.collect {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InvalidUDFClassException.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InvalidUDFClassException.scala
new file mode 100644
index 0000000000000..bc02efd5113c2
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InvalidUDFClassException.scala
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.catalog
+
+import org.apache.spark.sql.AnalysisException
+
+/**
+ * Thrown when a query failed for invalid function class, usually because a SQL
+ * function's class does not follow the rules of the UDF/UDAF/UDTF class definition.
+ */
+class InvalidUDFClassException private[sql](message: String)
+  extends AnalysisException(message, None, None, None, None) {
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
index b79857cdccd22..0813d41af1617 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.catalog
 import java.net.URI
 import java.util.Locale
 import java.util.concurrent.Callable
+import java.util.concurrent.TimeUnit
 import javax.annotation.concurrent.GuardedBy
 
 import scala.collection.mutable
@@ -34,14 +35,16 @@ import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst._
 import org.apache.spark.sql.catalyst.analysis._
 import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder
-import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionInfo, ImplicitCastInputTypes}
+import org.apache.spark.sql.catalyst.expressions.{Alias, Expression, ExpressionInfo, ImplicitCastInputTypes, UpCast}
 import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, ParserInterface}
-import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, SubqueryAlias, View}
-import org.apache.spark.sql.catalyst.util.StringUtils
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project, SubqueryAlias, View}
+import org.apache.spark.sql.catalyst.util.{CharVarcharUtils, StringUtils}
 import org.apache.spark.sql.connector.catalog.CatalogManager
+import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.StaticSQLConf.GLOBAL_TEMP_DATABASE
 import org.apache.spark.sql.types.StructType
+import org.apache.spark.sql.util.{CaseInsensitiveStringMap, PartitioningUtils}
 import org.apache.spark.util.Utils
 
 object SessionCatalog {
@@ -59,10 +62,12 @@ class SessionCatalog(
     externalCatalogBuilder: () => ExternalCatalog,
     globalTempViewManagerBuilder: () => GlobalTempViewManager,
     functionRegistry: FunctionRegistry,
-    conf: SQLConf,
+    tableFunctionRegistry: TableFunctionRegistry,
     hadoopConf: Configuration,
     parser: ParserInterface,
-    functionResourceLoader: FunctionResourceLoader) extends Logging {
+    functionResourceLoader: FunctionResourceLoader,
+    cacheSize: Int = SQLConf.get.tableRelationCacheSize,
+    cacheTTL: Long = SQLConf.get.metadataCacheTTL) extends SQLConfHelper with Logging {
   import SessionCatalog._
   import CatalogTypes.TablePartitionSpec
 
@@ -70,23 +75,44 @@ class SessionCatalog(
   def this(
       externalCatalog: ExternalCatalog,
       functionRegistry: FunctionRegistry,
-      conf: SQLConf) {
+      tableFunctionRegistry: TableFunctionRegistry,
+      conf: SQLConf) = {
     this(
       () => externalCatalog,
       () => new GlobalTempViewManager(conf.getConf(GLOBAL_TEMP_DATABASE)),
       functionRegistry,
-      conf,
+      tableFunctionRegistry,
       new Configuration(),
-      new CatalystSqlParser(conf),
-      DummyFunctionResourceLoader)
+      new CatalystSqlParser(),
+      DummyFunctionResourceLoader,
+      conf.tableRelationCacheSize,
+      conf.metadataCacheTTL)
   }
 
   // For testing only.
-  def this(externalCatalog: ExternalCatalog) {
-    this(
-      externalCatalog,
-      new SimpleFunctionRegistry,
-      new SQLConf().copy(SQLConf.CASE_SENSITIVE -> true))
+  def this(
+      externalCatalog: ExternalCatalog,
+      functionRegistry: FunctionRegistry,
+      conf: SQLConf) = {
+    this(externalCatalog, functionRegistry, new SimpleTableFunctionRegistry, conf)
+  }
+
+  // For testing only.
+  def this(
+      externalCatalog: ExternalCatalog,
+      functionRegistry: FunctionRegistry,
+      tableFunctionRegistry: TableFunctionRegistry) = {
+    this(externalCatalog, functionRegistry, tableFunctionRegistry, SQLConf.get)
+  }
+
+  // For testing only.
+  def this(externalCatalog: ExternalCatalog, functionRegistry: FunctionRegistry) = {
+    this(externalCatalog, functionRegistry, SQLConf.get)
+  }
+
+  // For testing only.
+  def this(externalCatalog: ExternalCatalog) = {
+    this(externalCatalog, new SimpleFunctionRegistry)
   }
 
   lazy val externalCatalog = externalCatalogBuilder()
@@ -94,7 +120,7 @@ class SessionCatalog(
 
   /** List of temporary views, mapping from table name to their logical plan. */
   @GuardedBy("this")
-  protected val tempViews = new mutable.HashMap[String, LogicalPlan]
+  protected val tempViews = new mutable.HashMap[String, TemporaryViewRelation]
 
   // Note: we track current database here because certain operations do not explicitly
   // specify the database (e.g. DROP TABLE my_table). In these cases we must first
@@ -114,8 +140,7 @@ class SessionCatalog(
    */
   private def validateName(name: String): Unit = {
     if (!validNameFormat.pattern.matcher(name).matches()) {
-      throw new AnalysisException(s"`$name` is not a valid name for tables/databases. " +
-        "Valid names only contain alphabet characters, numbers and _.")
+      throw QueryCompilationErrors.invalidNameForTableOrDatabaseError(name)
     }
   }
 
@@ -134,8 +159,14 @@ class SessionCatalog(
   }
 
   private val tableRelationCache: Cache[QualifiedTableName, LogicalPlan] = {
-    val cacheSize = conf.tableRelationCacheSize
-    CacheBuilder.newBuilder().maximumSize(cacheSize).build[QualifiedTableName, LogicalPlan]()
+    var builder = CacheBuilder.newBuilder()
+      .maximumSize(cacheSize)
+
+    if (cacheTTL > 0) {
+      builder = builder.expireAfterWrite(cacheTTL, TimeUnit.SECONDS)
+    }
+
+    builder.build[QualifiedTableName, LogicalPlan]()
   }
 
   /** This method provides a way to get a cached plan. */
@@ -158,6 +189,13 @@ class SessionCatalog(
     tableRelationCache.invalidate(key)
   }
 
+  /** This method discards any cached table relation plans for the given table identifier. */
+  def invalidateCachedTable(name: TableIdentifier): Unit = {
+    val dbName = formatDatabaseName(name.database.getOrElse(currentDb))
+    val tableName = formatTableName(name.table)
+    invalidateCachedTable(QualifiedTableName(dbName, tableName))
+  }
+
   /** This method provides a way to invalidate all the cached plans. */
   def invalidateAllCachedTables(): Unit = {
     tableRelationCache.invalidateAll()
@@ -204,21 +242,28 @@ class SessionCatalog(
   def createDatabase(dbDefinition: CatalogDatabase, ignoreIfExists: Boolean): Unit = {
     val dbName = formatDatabaseName(dbDefinition.name)
     if (dbName == globalTempViewManager.database) {
-      throw new AnalysisException(
-        s"${globalTempViewManager.database} is a system preserved database, " +
-          "you cannot create a database with this name.")
+      throw QueryCompilationErrors.cannotCreateDatabaseWithSameNameAsPreservedDatabaseError(
+        globalTempViewManager.database)
     }
     validateName(dbName)
-    val qualifiedPath = makeQualifiedPath(dbDefinition.locationUri)
     externalCatalog.createDatabase(
-      dbDefinition.copy(name = dbName, locationUri = qualifiedPath),
+      dbDefinition.copy(name = dbName, locationUri = makeQualifiedDBPath(dbDefinition.locationUri)),
       ignoreIfExists)
   }
 
+  private def makeQualifiedDBPath(locationUri: URI): URI = {
+    if (locationUri.isAbsolute) {
+      locationUri
+    } else {
+      val fullPath = new Path(conf.warehousePath, CatalogUtils.URIToString(locationUri))
+      makeQualifiedPath(fullPath.toUri)
+    }
+  }
+
   def dropDatabase(db: String, ignoreIfNotExists: Boolean, cascade: Boolean): Unit = {
     val dbName = formatDatabaseName(db)
     if (dbName == DEFAULT_DATABASE) {
-      throw new AnalysisException(s"Can not drop default database")
+      throw QueryCompilationErrors.cannotDropDefaultDatabaseError
     }
     if (cascade && databaseExists(dbName)) {
       listTables(dbName).foreach { t =>
@@ -231,7 +276,8 @@ class SessionCatalog(
   def alterDatabase(dbDefinition: CatalogDatabase): Unit = {
     val dbName = formatDatabaseName(dbDefinition.name)
     requireDbExists(dbName)
-    externalCatalog.alterDatabase(dbDefinition.copy(name = dbName))
+    externalCatalog.alterDatabase(dbDefinition.copy(
+      name = dbName, locationUri = makeQualifiedDBPath(dbDefinition.locationUri)))
   }
 
   def getDatabaseMetadata(db: String): CatalogDatabase = {
@@ -258,11 +304,8 @@ class SessionCatalog(
   def setCurrentDatabase(db: String): Unit = {
     val dbName = formatDatabaseName(db)
     if (dbName == globalTempViewManager.database) {
-      throw new AnalysisException(
-        s"${globalTempViewManager.database} is a system preserved database, " +
-          "you cannot use it as current database. To access global temporary views, you should " +
-          "use qualified name with the GLOBAL_TEMP_DATABASE, e.g. SELECT * FROM " +
-          s"${globalTempViewManager.database}.viewName.")
+      throw QueryCompilationErrors.cannotUsePreservedDatabaseAsCurrentDatabaseError(
+        globalTempViewManager.database)
     }
     requireDbExists(dbName)
     synchronized { currentDb = dbName }
@@ -273,8 +316,7 @@ class SessionCatalog(
    * by users.
    */
   def getDefaultDBPath(db: String): URI = {
-    val database = formatDatabaseName(db)
-    new Path(new Path(conf.warehousePath), database + ".db").toUri
+    CatalogUtils.stringToURI(formatDatabaseName(db) + ".db")
   }
 
   // ----------------------------------------------------------------------------
@@ -298,6 +340,11 @@ class SessionCatalog(
       tableDefinition: CatalogTable,
       ignoreIfExists: Boolean,
       validateLocation: Boolean = true): Unit = {
+    val isExternal = tableDefinition.tableType == CatalogTableType.EXTERNAL
+    if (isExternal && tableDefinition.storage.locationUri.isEmpty) {
+      throw QueryCompilationErrors.createExternalTableWithoutLocationError
+    }
+
     val db = formatDatabaseName(tableDefinition.identifier.database.getOrElse(getCurrentDatabase))
     val table = formatTableName(tableDefinition.identifier.table)
     val tableIdentifier = TableIdentifier(table, Some(db))
@@ -307,7 +354,7 @@ class SessionCatalog(
       && !tableDefinition.storage.locationUri.get.isAbsolute) {
       // make the location of the table qualified.
       val qualifiedTableLocation =
-        makeQualifiedPath(tableDefinition.storage.locationUri.get)
+        makeQualifiedTablePath(tableDefinition.storage.locationUri.get, db)
       tableDefinition.copy(
         storage = tableDefinition.storage.copy(locationUri = Some(qualifiedTableLocation)),
         identifier = tableIdentifier)
@@ -334,12 +381,22 @@ class SessionCatalog(
       val fs = tableLocation.getFileSystem(hadoopConf)
 
       if (fs.exists(tableLocation) && fs.listStatus(tableLocation).nonEmpty) {
-        throw new AnalysisException(s"Can not create the managed table('${table.identifier}')" +
-          s". The associated location('${tableLocation.toString}') already exists.")
+        throw QueryCompilationErrors.cannotOperateManagedTableWithExistingLocationError(
+          "create", table.identifier, tableLocation)
       }
     }
   }
 
+  private def makeQualifiedTablePath(locationUri: URI, database: String): URI = {
+    if (locationUri.isAbsolute) {
+      locationUri
+    } else {
+      val dbName = formatDatabaseName(database)
+      val dbLocation = makeQualifiedDBPath(getDatabaseMetadata(dbName).locationUri)
+      new Path(new Path(dbLocation), CatalogUtils.URIToString(locationUri)).toUri
+    }
+  }
+
   /**
    * Alter the metadata of an existing metastore table identified by `tableDefinition`.
    *
@@ -359,7 +416,7 @@ class SessionCatalog(
       && !tableDefinition.storage.locationUri.get.isAbsolute) {
       // make the location of the table qualified.
       val qualifiedTableLocation =
-        makeQualifiedPath(tableDefinition.storage.locationUri.get)
+        makeQualifiedTablePath(tableDefinition.storage.locationUri.get, db)
       tableDefinition.copy(
         storage = tableDefinition.storage.copy(locationUri = Some(qualifiedTableLocation)),
         identifier = tableIdentifier)
@@ -393,11 +450,7 @@ class SessionCatalog(
     val nonExistentColumnNames =
       oldDataSchema.map(_.name).filterNot(columnNameResolved(newDataSchema, _))
     if (nonExistentColumnNames.nonEmpty) {
-      throw new AnalysisException(
-        s"""
-           |Some existing schema fields (${nonExistentColumnNames.mkString("[", ",", "]")}) are
-           |not present in the new schema. We don't support dropping columns yet.
-         """.stripMargin)
+      throw QueryCompilationErrors.dropNonExistentColumnsNotSupportedError(nonExistentColumnNames)
     }
 
     externalCatalog.alterTableDataSchema(db, table, newDataSchema)
@@ -426,8 +479,8 @@ class SessionCatalog(
    * Return whether a table/view with the specified name exists. If no database is specified, check
    * with current database.
    */
-  def tableExists(name: TableIdentifier): Boolean = synchronized {
-    val db = formatDatabaseName(name.database.getOrElse(currentDb))
+  def tableExists(name: TableIdentifier): Boolean = {
+    val db = formatDatabaseName(name.database.getOrElse(getCurrentDatabase))
     val table = formatTableName(name.table)
     externalCatalog.tableExists(db, table)
   }
@@ -435,10 +488,23 @@ class SessionCatalog(
   /**
    * Retrieve the metadata of an existing permanent table/view. If no database is specified,
    * assume the table/view is in the current database.
+   * We replace char/varchar with "annotated" string type in the table schema, as the query
+   * engine doesn't support char/varchar yet.
    */
   @throws[NoSuchDatabaseException]
   @throws[NoSuchTableException]
   def getTableMetadata(name: TableIdentifier): CatalogTable = {
+    val t = getTableRawMetadata(name)
+    t.copy(schema = CharVarcharUtils.replaceCharVarcharWithStringInSchema(t.schema))
+  }
+
+  /**
+   * Retrieve the metadata of an existing permanent table/view. If no database is specified,
+   * assume the table/view is in the current database.
+   */
+  @throws[NoSuchDatabaseException]
+  @throws[NoSuchTableException]
+  def getTableRawMetadata(name: TableIdentifier): CatalogTable = {
     val db = formatDatabaseName(name.database.getOrElse(getCurrentDatabase))
     val table = formatTableName(name.table)
     requireDbExists(db)
@@ -460,10 +526,8 @@ class SessionCatalog(
       if (dbs.distinct.size != 1) {
         val tables = names.map(name => formatTableName(name.table))
         val qualifiedTableNames = dbs.zip(tables).map { case (d, t) => QualifiedTableName(d, t)}
-        throw new AnalysisException(
-          s"Only the tables/views belong to the same database can be retrieved. Querying " +
-          s"tables/views are $qualifiedTableNames"
-        )
+        throw QueryCompilationErrors.cannotRetrieveTableOrViewNotInSameDatabaseError(
+          qualifiedTableNames)
       }
       val db = formatDatabaseName(dbs.head)
       requireDbExists(db)
@@ -528,13 +592,13 @@ class SessionCatalog(
    */
   def createTempView(
       name: String,
-      tableDefinition: LogicalPlan,
+      viewDefinition: TemporaryViewRelation,
       overrideIfExists: Boolean): Unit = synchronized {
     val table = formatTableName(name)
     if (tempViews.contains(table) && !overrideIfExists) {
       throw new TempTableAlreadyExistsException(name)
     }
-    tempViews.put(table, tableDefinition)
+    tempViews.put(table, viewDefinition)
   }
 
   /**
@@ -542,7 +606,7 @@ class SessionCatalog(
    */
   def createGlobalTempView(
       name: String,
-      viewDefinition: LogicalPlan,
+      viewDefinition: TemporaryViewRelation,
       overrideIfExists: Boolean): Unit = {
     globalTempViewManager.create(formatTableName(name), viewDefinition, overrideIfExists)
   }
@@ -553,7 +617,7 @@ class SessionCatalog(
    */
   def alterTempViewDefinition(
       name: TableIdentifier,
-      viewDefinition: LogicalPlan): Boolean = synchronized {
+      viewDefinition: TemporaryViewRelation): Boolean = synchronized {
     val viewName = formatTableName(name.table)
     if (name.database.isEmpty) {
       if (tempViews.contains(viewName)) {
@@ -572,10 +636,17 @@ class SessionCatalog(
   /**
    * Return a local temporary view exactly as it was stored.
    */
-  def getTempView(name: String): Option[LogicalPlan] = synchronized {
+  def getRawTempView(name: String): Option[TemporaryViewRelation] = synchronized {
     tempViews.get(formatTableName(name))
   }
 
+  /**
+   * Generate a [[View]] operator from the temporary view stored.
+   */
+  def getTempView(name: String): Option[View] = synchronized {
+    getRawTempView(name).map(getTempViewPlan)
+  }
+
   def getTempViewNames(): Seq[String] = synchronized {
     tempViews.keySet.toSeq
   }
@@ -583,10 +654,17 @@ class SessionCatalog(
   /**
    * Return a global temporary view exactly as it was stored.
    */
-  def getGlobalTempView(name: String): Option[LogicalPlan] = {
+  def getRawGlobalTempView(name: String): Option[TemporaryViewRelation] = {
     globalTempViewManager.get(formatTableName(name))
   }
 
+  /**
+   * Generate a [[View]] operator from the global temporary view stored.
+   */
+  def getGlobalTempView(name: String): Option[View] = {
+    getRawGlobalTempView(name).map(getTempViewPlan)
+  }
+
   /**
    * Drop a local temporary view.
    *
@@ -621,21 +699,10 @@ class SessionCatalog(
   def getTempViewOrPermanentTableMetadata(name: TableIdentifier): CatalogTable = synchronized {
     val table = formatTableName(name.table)
     if (name.database.isEmpty) {
-      getTempView(table).map { plan =>
-        CatalogTable(
-          identifier = TableIdentifier(table),
-          tableType = CatalogTableType.VIEW,
-          storage = CatalogStorageFormat.empty,
-          schema = plan.output.toStructType)
-      }.getOrElse(getTableMetadata(name))
+      tempViews.get(table).map(_.tableMeta).getOrElse(getTableMetadata(name))
     } else if (formatDatabaseName(name.database.get) == globalTempViewManager.database) {
-      globalTempViewManager.get(table).map { plan =>
-        CatalogTable(
-          identifier = TableIdentifier(table, Some(globalTempViewManager.database)),
-          tableType = CatalogTableType.VIEW,
-          storage = CatalogStorageFormat.empty,
-          schema = plan.output.toStructType)
-      }.getOrElse(throw new NoSuchTableException(globalTempViewManager.database, table))
+      globalTempViewManager.get(table).map(_.tableMeta)
+        .getOrElse(throw new NoSuchTableException(globalTempViewManager.database, table))
     } else {
       getTableMetadata(name)
     }
@@ -654,8 +721,7 @@ class SessionCatalog(
     val db = formatDatabaseName(oldName.database.getOrElse(currentDb))
     newName.database.map(formatDatabaseName).foreach { newDb =>
       if (db != newDb) {
-        throw new AnalysisException(
-          s"RENAME TABLE source and destination databases do not match: '$db' != '$newDb'")
+        throw QueryCompilationErrors.renameTableSourceAndDestinationMismatchError(db, newDb)
       }
     }
 
@@ -673,13 +739,12 @@ class SessionCatalog(
         externalCatalog.renameTable(db, oldTableName, newTableName)
       } else {
         if (newName.database.isDefined) {
-          throw new AnalysisException(
-            s"RENAME TEMPORARY VIEW from '$oldName' to '$newName': cannot specify database " +
-              s"name '${newName.database.get}' in the destination table")
+          throw QueryCompilationErrors.cannotRenameTempViewWithDatabaseSpecifiedError(
+            oldName, newName)
         }
         if (tempViews.contains(newTableName)) {
-          throw new AnalysisException(s"RENAME TEMPORARY VIEW from '$oldName' to '$newName': " +
-            "destination table already exists")
+          throw QueryCompilationErrors.cannotRenameTempViewToExistingTableError(
+            oldName, newName)
         }
         val table = tempViews(oldTableName)
         tempViews.remove(oldTableName)
@@ -744,39 +809,90 @@ class SessionCatalog(
       val table = formatTableName(name.table)
       if (db == globalTempViewManager.database) {
         globalTempViewManager.get(table).map { viewDef =>
-          SubqueryAlias(table, db, viewDef)
+          SubqueryAlias(table, db, getTempViewPlan(viewDef))
         }.getOrElse(throw new NoSuchTableException(db, table))
       } else if (name.database.isDefined || !tempViews.contains(table)) {
         val metadata = externalCatalog.getTable(db, table)
         getRelation(metadata)
       } else {
-        SubqueryAlias(table, tempViews(table))
+        SubqueryAlias(table, getTempViewPlan(tempViews(table)))
       }
     }
   }
 
-  def getRelation(metadata: CatalogTable): LogicalPlan = {
+  def getRelation(
+      metadata: CatalogTable,
+      options: CaseInsensitiveStringMap = CaseInsensitiveStringMap.empty()): LogicalPlan = {
     val name = metadata.identifier
     val db = formatDatabaseName(name.database.getOrElse(currentDb))
     val table = formatTableName(name.table)
     val multiParts = Seq(CatalogManager.SESSION_CATALOG_NAME, db, table)
 
     if (metadata.tableType == CatalogTableType.VIEW) {
-      val viewText = metadata.viewText.getOrElse(sys.error("Invalid view without text."))
-      logDebug(s"'$viewText' will be used for the view($table).")
       // The relation is a view, so we wrap the relation by:
       // 1. Add a [[View]] operator over the relation to keep track of the view desc;
       // 2. Wrap the logical plan in a [[SubqueryAlias]] which tracks the name of the view.
-      val child = View(
-        desc = metadata,
-        output = metadata.schema.toAttributes,
-        child = parser.parsePlan(viewText))
-      SubqueryAlias(multiParts, child)
+      SubqueryAlias(multiParts, fromCatalogTable(metadata, isTempView = false))
     } else {
-      SubqueryAlias(multiParts, UnresolvedCatalogRelation(metadata))
+      SubqueryAlias(multiParts, UnresolvedCatalogRelation(metadata, options))
     }
   }
 
+  private def getTempViewPlan(viewInfo: TemporaryViewRelation): View = viewInfo.plan match {
+    case Some(p) => View(desc = viewInfo.tableMeta, isTempView = true, child = p)
+    case None => fromCatalogTable(viewInfo.tableMeta, isTempView = true)
+  }
+
+  private def fromCatalogTable(metadata: CatalogTable, isTempView: Boolean): View = {
+    val viewText = metadata.viewText.getOrElse {
+      throw new IllegalStateException("Invalid view without text.")
+    }
+    val viewConfigs = metadata.viewSQLConfigs
+    val parsedPlan = SQLConf.withExistingConf(View.effectiveSQLConf(viewConfigs, isTempView)) {
+      parser.parsePlan(viewText)
+    }
+    val viewColumnNames = if (metadata.viewQueryColumnNames.isEmpty) {
+      // For view created before Spark 2.2.0, the view text is already fully qualified, the plan
+      // output is the same with the view output.
+      metadata.schema.fieldNames.toSeq
+    } else {
+      assert(metadata.viewQueryColumnNames.length == metadata.schema.length)
+      metadata.viewQueryColumnNames
+    }
+
+    // For view queries like `SELECT * FROM t`, the schema of the referenced table/view may
+    // change after the view has been created. We need to add an extra SELECT to pick the columns
+    // according to the recorded column names (to get the correct view column ordering and omit
+    // the extra columns that we don't require), with UpCast (to make sure the type change is
+    // safe) and Alias (to respect user-specified view column names) according to the view schema
+    // in the catalog.
+    // Note that, the column names may have duplication, e.g. `CREATE VIEW v(x, y) AS
+    // SELECT 1 col, 2 col`. We need to make sure that the matching attributes have the same
+    // number of duplications, and pick the corresponding attribute by ordinal.
+    val viewConf = View.effectiveSQLConf(metadata.viewSQLConfigs, isTempView)
+    val normalizeColName: String => String = if (viewConf.caseSensitiveAnalysis) {
+      identity
+    } else {
+      _.toLowerCase(Locale.ROOT)
+    }
+    val nameToCounts = viewColumnNames.groupBy(normalizeColName).mapValues(_.length)
+    val nameToCurrentOrdinal = scala.collection.mutable.HashMap.empty[String, Int]
+
+    val projectList = viewColumnNames.zip(metadata.schema).map { case (name, field) =>
+      val normalizedName = normalizeColName(name)
+      val count = nameToCounts(normalizedName)
+      val col = if (count > 1) {
+        val ordinal = nameToCurrentOrdinal.getOrElse(normalizedName, 0)
+        nameToCurrentOrdinal(normalizedName) = ordinal + 1
+        GetViewColumnByNameAndOrdinal(metadata.identifier.toString, name, ordinal, count)
+      } else {
+        UnresolvedAttribute.quoted(name)
+      }
+      Alias(UpCast(col, field.dataType), field.name)(explicitMetadata = Some(field.metadata))
+    }
+    View(desc = metadata, isTempView = isTempView, child = Project(projectList, parsedPlan))
+  }
+
   def lookupTempView(table: String): Option[SubqueryAlias] = {
     val formattedTable = formatTableName(table)
     getTempView(formattedTable).map { view =>
@@ -796,11 +912,24 @@ class SessionCatalog(
     }
   }
 
-  // TODO: merge it with `isTemporaryTable`.
+  /**
+   * Return whether the given name parts belong to a temporary or global temporary view.
+   */
   def isTempView(nameParts: Seq[String]): Boolean = {
     if (nameParts.length > 2) return false
     import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-    isTemporaryTable(nameParts.asTableIdentifier)
+    isTempView(nameParts.asTableIdentifier)
+  }
+
+  def lookupTempView(name: TableIdentifier): Option[View] = {
+    val tableName = formatTableName(name.table)
+    if (name.database.isEmpty) {
+      tempViews.get(tableName).map(getTempViewPlan)
+    } else if (formatDatabaseName(name.database.get) == globalTempViewManager.database) {
+      globalTempViewManager.get(tableName).map(getTempViewPlan)
+    } else {
+      None
+    }
   }
 
   /**
@@ -809,15 +938,8 @@ class SessionCatalog(
    * Note: The temporary view cache is checked only when database is not
    * explicitly specified.
    */
-  def isTemporaryTable(name: TableIdentifier): Boolean = synchronized {
-    val table = formatTableName(name.table)
-    if (name.database.isEmpty) {
-      tempViews.contains(table)
-    } else if (formatDatabaseName(name.database.get) == globalTempViewManager.database) {
-      globalTempViewManager.get(table).isDefined
-    } else {
-      false
-    }
+  def isTempView(name: TableIdentifier): Boolean = synchronized {
+    lookupTempView(name).isDefined
   }
 
   def isView(nameParts: Seq[String]): Boolean = {
@@ -911,24 +1033,34 @@ class SessionCatalog(
   }
 
   /**
-   * Refresh the cache entry for a metastore table, if any.
+   * Refresh table entries in structures maintained by the session catalog such as:
+   *   - The map of temporary or global temporary view names to their logical plans
+   *   - The relation cache which maps table identifiers to their logical plans
+   *
+   * For temp views, it refreshes their logical plans, and as a consequence of that it can refresh
+   * the file indexes of the base relations (`HadoopFsRelation` for instance) used in the views.
+   * The method still keeps the views in the internal lists of session catalog.
+   *
+   * For tables/views, it removes their entries from the relation cache.
+   *
+   * The method is supposed to use in the following situations:
+   *   1. The logical plan of a table/view was changed, and cached table/view data is cleared
+   *      explicitly. For example, like in `AlterTableRenameCommand` which re-caches the table
+   *      itself. Otherwise if you need to refresh cached data, consider using of
+   *      `CatalogImpl.refreshTable()`.
+   *   2. A table/view doesn't exist, and need to only remove its entry in the relation cache since
+   *      the cached data is invalidated explicitly like in `DropTableCommand` which uncaches
+   *      table/view data itself.
+   *   3. Meta-data (such as file indexes) of any relation used in a temporary view should be
+   *      updated.
    */
   def refreshTable(name: TableIdentifier): Unit = synchronized {
-    val dbName = formatDatabaseName(name.database.getOrElse(currentDb))
-    val tableName = formatTableName(name.table)
-
-    // Go through temporary views and invalidate them.
-    // If the database is defined, this may be a global temporary view.
-    // If the database is not defined, there is a good chance this is a temp view.
-    if (name.database.isEmpty) {
-      tempViews.get(tableName).foreach(_.refresh())
-    } else if (dbName == globalTempViewManager.database) {
-      globalTempViewManager.get(tableName).foreach(_.refresh())
+    lookupTempView(name).map(_.refresh).getOrElse {
+      val dbName = formatDatabaseName(name.database.getOrElse(currentDb))
+      val tableName = formatTableName(name.table)
+      val qualifiedTableName = QualifiedTableName(dbName, tableName)
+      tableRelationCache.invalidate(qualifiedTableName)
     }
-
-    // Also invalidate the table relation cache.
-    val qualifiedTableName = QualifiedTableName(dbName, tableName)
-    tableRelationCache.invalidate(qualifiedTableName)
   }
 
   /**
@@ -1104,10 +1236,10 @@ class SessionCatalog(
    */
   private def requireNonEmptyValueInPartitionSpec(specs: Seq[TablePartitionSpec]): Unit = {
     specs.foreach { s =>
-      if (s.values.exists(_.isEmpty)) {
+      if (s.values.exists(v => v != null && v.isEmpty)) {
         val spec = s.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]")
-        throw new AnalysisException(
-          s"Partition spec is invalid. The spec ($spec) contains an empty partition column value")
+        throw QueryCompilationErrors.invalidPartitionSpecError(
+          s"The spec ($spec) contains an empty partition column value")
       }
     }
   }
@@ -1119,14 +1251,11 @@ class SessionCatalog(
   private def requireExactMatchedPartitionSpec(
       specs: Seq[TablePartitionSpec],
       table: CatalogTable): Unit = {
-    val defined = table.partitionColumnNames.sorted
-    specs.foreach { s =>
-      if (s.keys.toSeq.sorted != defined) {
-        throw new AnalysisException(
-          s"Partition spec is invalid. The spec (${s.keys.mkString(", ")}) must match " +
-            s"the partition spec (${table.partitionColumnNames.mkString(", ")}) defined in " +
-            s"table '${table.identifier}'")
-      }
+    specs.foreach { spec =>
+      PartitioningUtils.requireExactMatchedPartitionSpec(
+        table.identifier.toString,
+        spec,
+        table.partitionColumnNames)
     }
   }
 
@@ -1140,10 +1269,10 @@ class SessionCatalog(
     val defined = table.partitionColumnNames
     specs.foreach { s =>
       if (!s.keys.forall(defined.contains)) {
-        throw new AnalysisException(
-          s"Partition spec is invalid. The spec (${s.keys.mkString(", ")}) must be contained " +
-            s"within the partition spec (${table.partitionColumnNames.mkString(", ")}) defined " +
-            s"in table '${table.identifier}'")
+        throw QueryCompilationErrors.invalidPartitionSpecError(
+          s"The spec (${s.keys.mkString(", ")}) must be contained " +
+          s"within the partition spec (${table.partitionColumnNames.mkString(", ")}) defined " +
+          s"in table '${table.identifier}'")
       }
     }
   }
@@ -1258,7 +1387,7 @@ class SessionCatalog(
    * Check if the function with the specified name exists
    */
   def functionExists(name: FunctionIdentifier): Boolean = {
-    functionRegistry.functionExists(name) || {
+    functionRegistry.functionExists(name) || tableFunctionRegistry.functionExists(name) || {
       val db = formatDatabaseName(name.database.getOrElse(getCurrentDatabase))
       requireDbExists(db)
       externalCatalog.functionExists(db, name.funcName)
@@ -1292,20 +1421,24 @@ class SessionCatalog(
       Utils.classForName("org.apache.spark.sql.expressions.UserDefinedAggregateFunction")
     if (clsForUDAF.isAssignableFrom(clazz)) {
       val cls = Utils.classForName("org.apache.spark.sql.execution.aggregate.ScalaUDAF")
-      val e = cls.getConstructor(classOf[Seq[Expression]], clsForUDAF, classOf[Int], classOf[Int])
-        .newInstance(input,
-          clazz.getConstructor().newInstance().asInstanceOf[Object], Int.box(1), Int.box(1))
+      val e = cls.getConstructor(
+          classOf[Seq[Expression]], clsForUDAF, classOf[Int], classOf[Int], classOf[Option[String]])
+        .newInstance(
+          input,
+          clazz.getConstructor().newInstance().asInstanceOf[Object],
+          Int.box(1),
+          Int.box(1),
+          Some(name))
         .asInstanceOf[ImplicitCastInputTypes]
 
       // Check input argument size
       if (e.inputTypes.size != input.size) {
-        throw new AnalysisException(s"Invalid number of arguments for function $name. " +
-          s"Expected: ${e.inputTypes.size}; Found: ${input.size}")
+        throw QueryCompilationErrors.invalidFunctionArgumentsError(
+          name, e.inputTypes.size.toString, input.size)
       }
       e
     } else {
-      throw new AnalysisException(s"No handler for UDAF '${clazz.getCanonicalName}'. " +
-        s"Use sparkSession.udf.register(...) instead.")
+      throw QueryCompilationErrors.noHandlerForUDAFError(clazz.getCanonicalName)
     }
   }
 
@@ -1326,26 +1459,35 @@ class SessionCatalog(
       functionBuilder: Option[FunctionBuilder] = None): Unit = {
     val func = funcDefinition.identifier
     if (functionRegistry.functionExists(func) && !overrideIfExists) {
-      throw new AnalysisException(s"Function $func already exists")
+      throw QueryCompilationErrors.functionAlreadyExistsError(func)
     }
     val info = new ExpressionInfo(funcDefinition.className, func.database.orNull, func.funcName)
     val builder =
       functionBuilder.getOrElse {
         val className = funcDefinition.className
         if (!Utils.classIsLoadable(className)) {
-          throw new AnalysisException(s"Can not load class '$className' when registering " +
-            s"the function '$func', please make sure it is on the classpath")
+          throw QueryCompilationErrors.cannotLoadClassWhenRegisteringFunctionError(className, func)
         }
         makeFunctionBuilder(func.unquotedString, className)
       }
     functionRegistry.registerFunction(func, info, builder)
   }
 
+  /**
+   * Unregister a temporary or permanent function from a session-specific [[FunctionRegistry]]
+   * Return true if function exists.
+   */
+  def unregisterFunction(name: FunctionIdentifier): Boolean = {
+    functionRegistry.dropFunction(name)
+  }
+
   /**
    * Drop a temporary function.
    */
   def dropTempFunction(name: String, ignoreIfNotExists: Boolean): Unit = {
-    if (!functionRegistry.dropFunction(FunctionIdentifier(name)) && !ignoreIfNotExists) {
+    if (!functionRegistry.dropFunction(FunctionIdentifier(name)) &&
+        !tableFunctionRegistry.dropFunction(FunctionIdentifier(name)) &&
+        !ignoreIfNotExists) {
       throw new NoSuchTempFunctionException(name)
     }
   }
@@ -1360,8 +1502,9 @@ class SessionCatalog(
     // A temporary function is a function that has been registered in functionRegistry
     // without a database name, and is neither a built-in function nor a Hive function
     name.database.isEmpty &&
-      functionRegistry.functionExists(name) &&
+      (functionRegistry.functionExists(name) || tableFunctionRegistry.functionExists(name)) &&
       !FunctionRegistry.builtin.functionExists(name) &&
+      !TableFunctionRegistry.builtin.functionExists(name) &&
       !hiveFunctions.contains(name.funcName.toLowerCase(Locale.ROOT))
   }
 
@@ -1374,7 +1517,7 @@ class SessionCatalog(
    * session. If not existed, return false.
    */
   def isRegisteredFunction(name: FunctionIdentifier): Boolean = {
-    functionRegistry.functionExists(name)
+    functionRegistry.functionExists(name) || tableFunctionRegistry.functionExists(name)
   }
 
   /**
@@ -1400,6 +1543,7 @@ class SessionCatalog(
     val qualifiedName = name.copy(database = database)
     functionRegistry.lookupFunction(name)
       .orElse(functionRegistry.lookupFunction(qualifiedName))
+      .orElse(tableFunctionRegistry.lookupFunction(name))
       .getOrElse {
         val db = qualifiedName.database.get
         requireDbExists(db)
@@ -1416,7 +1560,7 @@ class SessionCatalog(
   }
 
   /**
-   * Return an [[Expression]] that represents the specified function, assuming it exists.
+   * Look up a specific function, assuming it exists.
    *
    * For a temporary function or a permanent function that has been loaded,
    * this method will simply lookup the function through the
@@ -1428,25 +1572,46 @@ class SessionCatalog(
    * based on the function class and put the builder into the FunctionRegistry.
    * The name of this function in the FunctionRegistry will be `databaseName.functionName`.
    */
-  def lookupFunction(
+  private def lookupFunction[T](
       name: FunctionIdentifier,
-      children: Seq[Expression]): Expression = synchronized {
+      children: Seq[Expression],
+      registry: FunctionRegistryBase[T]): T = synchronized {
+    import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
+
     // Note: the implementation of this function is a little bit convoluted.
     // We probably shouldn't use a single FunctionRegistry to register all three kinds of functions
     // (built-in, temp, and external).
-    if (name.database.isEmpty && functionRegistry.functionExists(name)) {
-      // This function has been already loaded into the function registry.
-      return functionRegistry.lookupFunction(name, children)
+    if (name.database.isEmpty && registry.functionExists(name)) {
+      val referredTempFunctionNames = AnalysisContext.get.referredTempFunctionNames
+      val isResolvingView = AnalysisContext.get.catalogAndNamespace.nonEmpty
+      // Lookup the function as a temporary or a built-in function (i.e. without database) and
+      // 1. if we are not resolving view, we don't care about the function type and just return it.
+      // 2. if we are resolving view, only return a temp function if it's referred by this view.
+      if (!isResolvingView ||
+          !isTemporaryFunction(name) ||
+          referredTempFunctionNames.contains(name.funcName)) {
+        // This function has been already loaded into the function registry.
+        return registry.lookupFunction(name, children)
+      }
+    }
+
+    // Get the database from AnalysisContext if it's defined, otherwise, use current database
+    val currentDatabase = AnalysisContext.get.catalogAndNamespace match {
+      case Seq() => getCurrentDatabase
+      case Seq(_, db) => db
+      case Seq(catalog, namespace @ _*) =>
+        throw new IllegalStateException(s"[BUG] unexpected v2 catalog: $catalog, and " +
+          s"namespace: ${namespace.quoted} in v1 function lookup")
     }
 
     // If the name itself is not qualified, add the current database to it.
-    val database = formatDatabaseName(name.database.getOrElse(getCurrentDatabase))
+    val database = formatDatabaseName(name.database.getOrElse(currentDatabase))
     val qualifiedName = name.copy(database = Some(database))
 
-    if (functionRegistry.functionExists(qualifiedName)) {
+    if (registry.functionExists(qualifiedName)) {
       // This function has been already loaded into the function registry.
       // Unlike the above block, we find this function by using the qualified name.
-      return functionRegistry.lookupFunction(qualifiedName, children)
+      return registry.lookupFunction(qualifiedName, children)
     }
 
     // The function has not been loaded to the function registry, which means
@@ -1457,7 +1622,6 @@ class SessionCatalog(
       externalCatalog.getFunction(database, name.funcName)
     } catch {
       case _: AnalysisException => failFunctionLookup(name)
-      case _: NoSuchPermanentFunctionException => failFunctionLookup(name)
     }
     loadFunctionResources(catalogFunction.resources)
     // Please note that qualifiedName is provided by the user. However,
@@ -1467,7 +1631,21 @@ class SessionCatalog(
     // At here, we preserve the input from the user.
     registerFunction(catalogFunction.copy(identifier = qualifiedName), overrideIfExists = false)
     // Now, we need to create the Expression.
-    functionRegistry.lookupFunction(qualifiedName, children)
+    registry.lookupFunction(qualifiedName, children)
+  }
+
+  /**
+   * Return an [[Expression]] that represents the specified function, assuming it exists.
+   */
+  def lookupFunction(name: FunctionIdentifier, children: Seq[Expression]): Expression = {
+    lookupFunction[Expression](name, children, functionRegistry)
+  }
+
+  /**
+   * Return a [[LogicalPlan]] that represents the specified function, assuming it exists.
+   */
+  def lookupTableFunction(name: FunctionIdentifier, children: Seq[Expression]): LogicalPlan = {
+    lookupFunction[LogicalPlan](name, children, tableFunctionRegistry)
   }
 
   /**
@@ -1488,7 +1666,9 @@ class SessionCatalog(
     val dbFunctions = externalCatalog.listFunctions(dbName, pattern).map { f =>
       FunctionIdentifier(f, Some(dbName)) }
     val loadedFunctions = StringUtils
-      .filterPattern(functionRegistry.listFunction().map(_.unquotedString), pattern).map { f =>
+      .filterPattern(
+        (functionRegistry.listFunction() ++ tableFunctionRegistry.listFunction())
+          .map(_.unquotedString), pattern).map { f =>
         // In functionRegistry, function names are stored as an unquoted format.
         Try(parser.parseFunctionIdentifier(f)) match {
           case Success(e) => e
@@ -1502,6 +1682,7 @@ class SessionCatalog(
     // so there can be duplicates.
     functions.map {
       case f if FunctionRegistry.functionSet.contains(f) => (f, "SYSTEM")
+      case f if TableFunctionRegistry.functionSet.contains(f) => (f, "SYSTEM")
       case f => (f, "USER")
     }.distinct
   }
@@ -1536,6 +1717,7 @@ class SessionCatalog(
     clearTempTables()
     globalTempViewManager.clear()
     functionRegistry.clear()
+    tableFunctionRegistry.clear()
     tableRelationCache.invalidateAll()
     // restore built-in functions
     FunctionRegistry.builtin.listFunction().foreach { f =>
@@ -1545,6 +1727,14 @@ class SessionCatalog(
       require(functionBuilder.isDefined, s"built-in function '$f' is missing function builder")
       functionRegistry.registerFunction(f, expressionInfo.get, functionBuilder.get)
     }
+    // restore built-in table functions
+    TableFunctionRegistry.builtin.listFunction().foreach { f =>
+      val expressionInfo = TableFunctionRegistry.builtin.lookupFunction(f)
+      val functionBuilder = TableFunctionRegistry.builtin.lookupFunctionBuilder(f)
+      require(expressionInfo.isDefined, s"built-in function '$f' is missing expression info")
+      require(functionBuilder.isDefined, s"built-in function '$f' is missing function builder")
+      tableFunctionRegistry.registerFunction(f, expressionInfo.get, functionBuilder.get)
+    }
   }
 
   /**
@@ -1562,7 +1752,7 @@ class SessionCatalog(
   }
 
   /**
-   * Validate the new locatoin before renaming a managed table, which should be non-existent.
+   * Validate the new location before renaming a managed table, which should be non-existent.
    */
   private def validateNewLocationOfRename(
       oldName: TableIdentifier,
@@ -1574,8 +1764,8 @@ class SessionCatalog(
       val newTableLocation = new Path(new Path(databaseLocation), formatTableName(newName.table))
       val fs = newTableLocation.getFileSystem(hadoopConf)
       if (fs.exists(newTableLocation)) {
-        throw new AnalysisException(s"Can not rename the managed table('$oldName')" +
-          s". The associated location('$newTableLocation') already exists.")
+        throw QueryCompilationErrors.cannotOperateManagedTableWithExistingLocationError(
+          "rename", oldName, newTableLocation)
       }
     }
   }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/functionResources.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/functionResources.scala
index 67bf2d06c95dd..7ebe3d8c5f880 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/functionResources.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/functionResources.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.catalog
 
 import java.util.Locale
 
-import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.errors.QueryCompilationErrors
 
 /** A trait that represents the type of a resourced needed by a function. */
 abstract class FunctionResourceType(val resourceType: String)
@@ -40,7 +40,7 @@ object FunctionResourceType {
       case "file" => FileResource
       case "archive" => ArchiveResource
       case other =>
-        throw new AnalysisException(s"Resource Type '$resourceType' is not supported.")
+        throw QueryCompilationErrors.resourceTypeNotSupportedError(resourceType)
     }
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
index d02776b5d86f8..307c67b5ddda7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
@@ -24,18 +24,23 @@ import java.util.Date
 import scala.collection.mutable
 import scala.util.control.NonFatal
 
+import org.apache.commons.lang3.StringUtils
+import org.json4s.JsonAST.{JArray, JString}
+import org.json4s.jackson.JsonMethods._
+
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.{FunctionIdentifier, InternalRow, TableIdentifier}
+import org.apache.spark.sql.catalyst.{FunctionIdentifier, InternalRow, SQLConfHelper, TableIdentifier}
 import org.apache.spark.sql.catalyst.analysis.MultiInstanceRelation
+import org.apache.spark.sql.catalyst.catalog.CatalogTable.VIEW_STORING_ANALYZED_PLAN
 import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeMap, AttributeReference, Cast, ExprId, Literal}
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.plans.logical.statsEstimation.EstimationUtils
-import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateFormatter, DateTimeUtils, TimestampFormatter}
-import org.apache.spark.sql.catalyst.util.quoteIdentifier
+import org.apache.spark.sql.catalyst.util._
 import org.apache.spark.sql.connector.catalog.CatalogManager
+import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
 
 /**
@@ -141,7 +146,7 @@ case class CatalogTablePartition(
   /** Return the partition location, assuming it is specified. */
   def location: URI = storage.locationUri.getOrElse {
     val specString = spec.map { case (k, v) => s"$k=$v" }.mkString(", ")
-    throw new AnalysisException(s"Partition [$specString] did not specify locationUri")
+    throw QueryCompilationErrors.partitionNotSpecifyLocationUriError(specString)
   }
 
   /**
@@ -175,13 +180,11 @@ case class CatalogTablePartition(
 case class BucketSpec(
     numBuckets: Int,
     bucketColumnNames: Seq[String],
-    sortColumnNames: Seq[String]) {
-  def conf: SQLConf = SQLConf.get
+    sortColumnNames: Seq[String]) extends SQLConfHelper {
 
   if (numBuckets <= 0 || numBuckets > conf.bucketingMaxBuckets) {
-    throw new AnalysisException(
-      s"Number of buckets should be greater than 0 but less than or equal to " +
-        s"bucketing.maxBuckets (`${conf.bucketingMaxBuckets}`). Got `$numBuckets`")
+    throw QueryCompilationErrors.invalidBucketNumberError(
+      conf.bucketingMaxBuckets, numBuckets)
   }
 
   override def toString: String = {
@@ -271,12 +274,12 @@ case class CatalogTable(
 
   /** Return the database this table was specified to belong to, assuming it exists. */
   def database: String = identifier.database.getOrElse {
-    throw new AnalysisException(s"table $identifier did not specify database")
+    throw QueryCompilationErrors.tableNotSpecifyDatabaseError(identifier)
   }
 
   /** Return the table location, assuming it is specified. */
   def location: URI = storage.locationUri.getOrElse {
-    throw new AnalysisException(s"table $identifier did not specify locationUri")
+    throw QueryCompilationErrors.tableNotSpecifyLocationUriError(identifier)
   }
 
   /** Return the fully qualified name of this table, assuming the database was specified. */
@@ -292,8 +295,7 @@ case class CatalogTable(
       (0 until numParts).map { index =>
         properties.getOrElse(
           s"$VIEW_CATALOG_AND_NAMESPACE_PART_PREFIX$index",
-          throw new AnalysisException("Corrupted table name context in catalog: " +
-            s"$numParts parts expected, but part $index is missing.")
+          throw QueryCompilationErrors.corruptedTableNameContextInCatalogError(numParts, index)
         )
       }
     } else if (properties.contains(VIEW_DEFAULT_DATABASE)) {
@@ -304,6 +306,21 @@ case class CatalogTable(
     }
   }
 
+  /**
+   * Return the SQL configs of when the view was created, the configs are applied when parsing and
+   * analyzing the view, should be empty if the CatalogTable is not a View or created by older
+   * versions of Spark(before 3.1.0).
+   */
+  def viewSQLConfigs: Map[String, String] = {
+    try {
+      for ((key, value) <- properties if key.startsWith(CatalogTable.VIEW_SQL_CONFIG_PREFIX))
+        yield (key.substring(CatalogTable.VIEW_SQL_CONFIG_PREFIX.length), value)
+    } catch {
+      case e: Exception =>
+        throw QueryCompilationErrors.corruptedViewSQLConfigsInCatalogError(e)
+    }
+  }
+
   /**
    * Return the output column names of the query that creates a view, the column names are used to
    * resolve a view, should be empty if the CatalogTable is not a View or created by older versions
@@ -315,11 +332,42 @@ case class CatalogTable(
       index <- 0 until numCols.toInt
     } yield properties.getOrElse(
       s"$VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX$index",
-      throw new AnalysisException("Corrupted view query output column names in catalog: " +
-        s"$numCols parts expected, but part $index is missing.")
+      throw QueryCompilationErrors.corruptedViewQueryOutputColumnsInCatalogError(numCols, index)
     )
   }
 
+  /**
+   * Return temporary view names the current view was referred. should be empty if the
+   * CatalogTable is not a Temporary View or created by older versions of Spark(before 3.1.0).
+   */
+  def viewReferredTempViewNames: Seq[Seq[String]] = {
+    try {
+      properties.get(VIEW_REFERRED_TEMP_VIEW_NAMES).map { json =>
+        parse(json).asInstanceOf[JArray].arr.map { namePartsJson =>
+          namePartsJson.asInstanceOf[JArray].arr.map(_.asInstanceOf[JString].s)
+        }
+      }.getOrElse(Seq.empty)
+    } catch {
+      case e: Exception =>
+        throw QueryCompilationErrors.corruptedViewReferredTempViewInCatalogError(e)
+    }
+  }
+
+  /**
+   * Return temporary function names the current view was referred. should be empty if the
+   * CatalogTable is not a Temporary View or created by older versions of Spark(before 3.1.0).
+   */
+  def viewReferredTempFunctionNames: Seq[String] = {
+    try {
+      properties.get(VIEW_REFERRED_TEMP_FUNCTION_NAMES).map { json =>
+        parse(json).asInstanceOf[JArray].arr.map(_.asInstanceOf[JString].s)
+      }.getOrElse(Seq.empty)
+    } catch {
+      case e: Exception =>
+        throw QueryCompilationErrors.corruptedViewReferredTempFunctionsInCatalogError(e)
+    }
+  }
+
   /** Syntactic sugar to update a field in `storage`. */
   def withNewStorage(
       locationUri: Option[URI] = storage.locationUri,
@@ -335,7 +383,10 @@ case class CatalogTable(
 
   def toLinkedHashMap: mutable.LinkedHashMap[String, String] = {
     val map = new mutable.LinkedHashMap[String, String]()
-    val tableProperties = properties.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]")
+    val tableProperties = properties
+      .filterKeys(!_.startsWith(VIEW_PREFIX))
+      .toSeq.sortBy(_._1)
+      .map(p => p._1 + "=" + p._2).mkString("[", ", ", "]")
     val partitionColumns = partitionColumnNames.map(quoteIdentifier).mkString("[", ", ", "]")
     val lastAccess = {
       if (lastAccessTime <= 0) "UNKNOWN" else new Date(lastAccessTime).toString
@@ -410,9 +461,87 @@ object CatalogTable {
     props.toMap
   }
 
+  val VIEW_SQL_CONFIG_PREFIX = VIEW_PREFIX + "sqlConfig."
+
   val VIEW_QUERY_OUTPUT_PREFIX = VIEW_PREFIX + "query.out."
   val VIEW_QUERY_OUTPUT_NUM_COLUMNS = VIEW_QUERY_OUTPUT_PREFIX + "numCols"
   val VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX = VIEW_QUERY_OUTPUT_PREFIX + "col."
+
+  val VIEW_REFERRED_TEMP_VIEW_NAMES = VIEW_PREFIX + "referredTempViewNames"
+  val VIEW_REFERRED_TEMP_FUNCTION_NAMES = VIEW_PREFIX + "referredTempFunctionsNames"
+
+  val VIEW_STORING_ANALYZED_PLAN = VIEW_PREFIX + "storingAnalyzedPlan"
+
+  def splitLargeTableProp(
+      key: String,
+      value: String,
+      addProp: (String, String) => Unit,
+      defaultThreshold: Int): Unit = {
+    val threshold = SQLConf.get.getConf(SQLConf.HIVE_TABLE_PROPERTY_LENGTH_THRESHOLD)
+      .getOrElse(defaultThreshold)
+    if (value.length <= threshold) {
+      addProp(key, value)
+    } else {
+      val parts = value.grouped(threshold).toSeq
+      addProp(s"$key.numParts", parts.length.toString)
+      parts.zipWithIndex.foreach { case (part, index) =>
+        addProp(s"$key.part.$index", part)
+      }
+    }
+  }
+
+  def readLargeTableProp(props: Map[String, String], key: String): Option[String] = {
+    props.get(key).orElse {
+      if (props.filterKeys(_.startsWith(key)).isEmpty) {
+        None
+      } else {
+        val numParts = props.get(s"$key.numParts")
+        if (numParts.isEmpty) {
+          throw QueryCompilationErrors.cannotReadCorruptedTablePropertyError(key)
+        } else {
+          val parts = (0 until numParts.get.toInt).map { index =>
+            props.getOrElse(s"$key.part.$index", {
+              throw QueryCompilationErrors.cannotReadCorruptedTablePropertyError(
+                key, s"Missing part $index, $numParts parts are expected.")
+            })
+          }
+          Some(parts.mkString)
+        }
+      }
+    }
+  }
+
+  def isLargeTableProp(originalKey: String, propKey: String): Boolean = {
+    propKey == originalKey || propKey == s"$originalKey.numParts" ||
+      propKey.startsWith(s"$originalKey.part.")
+  }
+
+  def normalize(table: CatalogTable): CatalogTable = {
+    val nondeterministicProps = Set(
+      "CreateTime",
+      "transient_lastDdlTime",
+      "grantTime",
+      "lastUpdateTime",
+      "last_modified_by",
+      "last_modified_time",
+      "Owner:",
+      // The following are hive specific schema parameters which we do not need to match exactly.
+      "totalNumberFiles",
+      "maxFileSize",
+      "minFileSize"
+    )
+
+    table.copy(
+      createTime = 0L,
+      lastAccessTime = 0L,
+      properties = table.properties
+        .filterKeys(!nondeterministicProps.contains(_))
+        .map(identity)
+        .toMap,
+      stats = None,
+      ignoredProperties = Map.empty
+    )
+  }
 }
 
 /**
@@ -487,7 +616,11 @@ case class CatalogColumnStat(
     min.foreach { v => map.put(s"${colName}.${CatalogColumnStat.KEY_MIN_VALUE}", v) }
     max.foreach { v => map.put(s"${colName}.${CatalogColumnStat.KEY_MAX_VALUE}", v) }
     histogram.foreach { h =>
-      map.put(s"${colName}.${CatalogColumnStat.KEY_HISTOGRAM}", HistogramSerializer.serialize(h))
+      CatalogTable.splitLargeTableProp(
+        s"$colName.${CatalogColumnStat.KEY_HISTOGRAM}",
+        HistogramSerializer.serialize(h),
+        map.put,
+        4000)
     }
     map.toMap
   }
@@ -525,7 +658,7 @@ object CatalogColumnStat extends Logging {
     TimestampFormatter(
       format = "yyyy-MM-dd HH:mm:ss.SSSSSS",
       zoneId = ZoneOffset.UTC,
-      needVarLengthSecondFraction = isParsing)
+      isParsing = isParsing)
   }
 
   /**
@@ -549,8 +682,8 @@ object CatalogColumnStat extends Logging {
       // This version of Spark does not use min/max for binary/string types so we ignore it.
       case BinaryType | StringType => null
       case _ =>
-        throw new AnalysisException("Column statistics deserialization is not supported for " +
-          s"column $name of data type: $dataType.")
+        throw QueryCompilationErrors.columnStatisticsDeserializationNotSupportedError(
+          name, dataType)
     }
   }
 
@@ -566,8 +699,8 @@ object CatalogColumnStat extends Logging {
       case _: DecimalType => v.asInstanceOf[Decimal].toJavaBigDecimal
       // This version of Spark does not use min/max for binary/string types so we ignore it.
       case _ =>
-        throw new AnalysisException("Column statistics serialization is not supported for " +
-          s"column $colName of data type: $dataType.")
+        throw QueryCompilationErrors.columnStatisticsSerializationNotSupportedError(
+          colName, dataType)
     }
     externalValue.toString
   }
@@ -591,7 +724,8 @@ object CatalogColumnStat extends Logging {
         nullCount = map.get(s"${colName}.${KEY_NULL_COUNT}").map(v => BigInt(v.toLong)),
         avgLen = map.get(s"${colName}.${KEY_AVG_LEN}").map(_.toLong),
         maxLen = map.get(s"${colName}.${KEY_MAX_LEN}").map(_.toLong),
-        histogram = map.get(s"${colName}.${KEY_HISTOGRAM}").map(HistogramSerializer.deserialize),
+        histogram = CatalogTable.readLargeTableProp(map, s"$colName.$KEY_HISTOGRAM")
+          .map(HistogramSerializer.deserialize),
         version = map(s"${colName}.${KEY_VERSION}").toInt
       ))
     } catch {
@@ -639,12 +773,30 @@ object CatalogTypes {
  * A placeholder for a table relation, which will be replaced by concrete relation like
  * `LogicalRelation` or `HiveTableRelation`, during analysis.
  */
-case class UnresolvedCatalogRelation(tableMeta: CatalogTable) extends LeafNode {
+case class UnresolvedCatalogRelation(
+    tableMeta: CatalogTable,
+    options: CaseInsensitiveStringMap = CaseInsensitiveStringMap.empty(),
+    override val isStreaming: Boolean = false) extends LeafNode {
   assert(tableMeta.identifier.database.isDefined)
   override lazy val resolved: Boolean = false
   override def output: Seq[Attribute] = Nil
 }
 
+/**
+ * A wrapper to store the temporary view info, will be kept in `SessionCatalog`
+ * and will be transformed to `View` during analysis. If the temporary view is
+ * storing an analyzed plan, `plan` is set to the analyzed plan for the view.
+ */
+case class TemporaryViewRelation(
+    tableMeta: CatalogTable,
+    plan: Option[LogicalPlan] = None) extends LeafNode {
+  require(plan.isEmpty ||
+    (plan.get.resolved && tableMeta.properties.contains(VIEW_STORING_ANALYZED_PLAN)))
+
+  override lazy val resolved: Boolean = false
+  override def output: Seq[Attribute] = Nil
+}
+
 /**
  * A `LogicalPlan` that represents a hive table.
  *
@@ -667,27 +819,61 @@ case class HiveTableRelation(
   def isPartitioned: Boolean = partitionCols.nonEmpty
 
   override def doCanonicalize(): HiveTableRelation = copy(
-    tableMeta = tableMeta.copy(
-      storage = CatalogStorageFormat.empty,
-      createTime = -1
-    ),
+    tableMeta = CatalogTable.normalize(tableMeta),
     dataCols = dataCols.zipWithIndex.map {
       case (attr, index) => attr.withExprId(ExprId(index))
     },
     partitionCols = partitionCols.zipWithIndex.map {
       case (attr, index) => attr.withExprId(ExprId(index + dataCols.length))
-    }
+    },
+    tableStats = None
   )
 
   override def computeStats(): Statistics = {
     tableMeta.stats.map(_.toPlanStats(output, conf.cboEnabled || conf.planStatsEnabled))
       .orElse(tableStats)
       .getOrElse {
-      throw new IllegalStateException("table stats must be specified.")
+      throw QueryExecutionErrors.tableStatsNotSpecifiedError
     }
   }
 
   override def newInstance(): HiveTableRelation = copy(
     dataCols = dataCols.map(_.newInstance()),
     partitionCols = partitionCols.map(_.newInstance()))
+
+  override def simpleString(maxFields: Int): String = {
+    val catalogTable = tableMeta.storage.serde match {
+      case Some(serde) => tableMeta.identifier :: serde :: Nil
+      case _ => tableMeta.identifier :: Nil
+    }
+
+    var metadata = Map(
+      "CatalogTable" -> catalogTable.mkString(", "),
+      "Data Cols" -> truncatedString(dataCols, "[", ", ", "]", maxFields),
+      "Partition Cols" -> truncatedString(partitionCols, "[", ", ", "]", maxFields)
+    )
+
+    if (prunedPartitions.nonEmpty) {
+      metadata += ("Pruned Partitions" -> {
+        val parts = prunedPartitions.get.map { part =>
+          val spec = part.spec.map { case (k, v) => s"$k=$v" }.mkString(", ")
+          if (part.storage.serde.nonEmpty && part.storage.serde != tableMeta.storage.serde) {
+            s"($spec, ${part.storage.serde.get})"
+          } else {
+            s"($spec)"
+          }
+        }
+        truncatedString(parts, "[", ", ", "]", maxFields)
+      })
+    }
+
+    val metadataEntries = metadata.toSeq.map {
+      case (key, value) if key == "CatalogTable" => value
+      case (key, value) =>
+        key + ": " + StringUtils.abbreviate(value, SQLConf.get.maxMetadataStringLength)
+    }
+
+    val metadataStr = truncatedString(metadataEntries, "[", ", ", "]", maxFields)
+    s"$nodeName $metadataStr"
+  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtils.scala
index 3e83c1dcb4758..b800caccbf852 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtils.scala
@@ -25,8 +25,13 @@ object CSVExprUtils {
    * This is currently being used in CSV reading path and CSV schema inference.
    */
   def filterCommentAndEmpty(iter: Iterator[String], options: CSVOptions): Iterator[String] = {
-    iter.filter { line =>
-      line.trim.nonEmpty && !line.startsWith(options.comment.toString)
+    if (options.isCommentSet) {
+      val commentPrefix = options.comment.toString
+      iter.filter { line =>
+        line.trim.nonEmpty && !line.startsWith(commentPrefix)
+      }
+    } else {
+      iter.filter(_.trim.nonEmpty)
     }
   }
 
@@ -34,7 +39,7 @@ object CSVExprUtils {
     if (options.isCommentSet) {
       val commentPrefix = options.comment.toString
       iter.dropWhile { line =>
-        line.trim.isEmpty || line.trim.startsWith(commentPrefix)
+        line.trim.isEmpty || line.startsWith(commentPrefix)
       }
     } else {
       iter.dropWhile(_.trim.isEmpty)
@@ -74,7 +79,7 @@ object CSVExprUtils {
       case Seq('\\', '\"') => '\"'
       case Seq('\\', '\'') => '\''
       case Seq('\\', '\\') => '\\'
-      case _ if str == """\u0000""" => '\u0000'
+      case _ if str == "\u0000" => '\u0000'
       case Seq('\\', _) =>
         throw new IllegalArgumentException(s"Unsupported special character for delimiter: $str")
       case _ =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVFilters.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVFilters.scala
deleted file mode 100644
index b50a76a496556..0000000000000
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVFilters.scala
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.catalyst.csv
-
-import scala.util.Try
-
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.sources
-import org.apache.spark.sql.types.{BooleanType, StructType}
-
-/**
- * An instance of the class compiles filters to predicates and allows to
- * apply the predicates to an internal row with partially initialized values
- * converted from parsed CSV fields.
- *
- * @param filters The filters pushed down to CSV datasource.
- * @param requiredSchema The schema with only fields requested by the upper layer.
- */
-class CSVFilters(filters: Seq[sources.Filter], requiredSchema: StructType) {
-  /**
-   * Converted filters to predicates and grouped by maximum field index
-   * in the read schema. For example, if an filter refers to 2 attributes
-   * attrA with field index 5 and attrB with field index 10 in the read schema:
-   *   0 === $"attrA" or $"attrB" < 100
-   * the filter is compiled to a predicate, and placed to the `predicates`
-   * array at the position 10. In this way, if there is a row with initialized
-   * fields from the 0 to 10 index, the predicate can be applied to the row
-   * to check that the row should be skipped or not.
-   * Multiple predicates with the same maximum reference index are combined
-   * by the `And` expression.
-   */
-  private val predicates: Array[BasePredicate] = {
-    val len = requiredSchema.fields.length
-    val groupedPredicates = Array.fill[BasePredicate](len)(null)
-    if (SQLConf.get.csvFilterPushDown) {
-      val groupedFilters = Array.fill(len)(Seq.empty[sources.Filter])
-      for (filter <- filters) {
-        val refs = filter.references
-        val index = if (refs.isEmpty) {
-          // For example, AlwaysTrue and AlwaysFalse doesn't have any references
-          // Filters w/o refs always return the same result. Taking into account
-          // that predicates are combined via And, we can apply such filters only
-          // once at the position 0.
-          0
-        } else {
-          // readSchema must contain attributes of all filters.
-          // Accordingly, fieldIndex() returns a valid index always.
-          refs.map(requiredSchema.fieldIndex).max
-        }
-        groupedFilters(index) :+= filter
-      }
-      if (len > 0 && !groupedFilters(0).isEmpty) {
-        // We assume that filters w/o refs like AlwaysTrue and AlwaysFalse
-        // can be evaluated faster that others. We put them in front of others.
-        val (literals, others) = groupedFilters(0).partition(_.references.isEmpty)
-        groupedFilters(0) = literals ++ others
-      }
-      for (i <- 0 until len) {
-        if (!groupedFilters(i).isEmpty) {
-          val reducedExpr = groupedFilters(i)
-            .flatMap(CSVFilters.filterToExpression(_, toRef))
-            .reduce(And)
-          groupedPredicates(i) = Predicate.create(reducedExpr)
-        }
-      }
-    }
-    groupedPredicates
-  }
-
-  /**
-   * Applies all filters that refer to row fields at the positions from 0 to index.
-   * @param row The internal row to check.
-   * @param index Maximum field index. The function assumes that all fields
-   *              from 0 to index position are set.
-   * @return false iff row fields at the position from 0 to index pass filters
-   *         or there are no applicable filters
-   *         otherwise false if at least one of the filters returns false.
-   */
-  def skipRow(row: InternalRow, index: Int): Boolean = {
-    val predicate = predicates(index)
-    predicate != null && !predicate.eval(row)
-  }
-
-  // Finds a filter attribute in the read schema and converts it to a `BoundReference`
-  private def toRef(attr: String): Option[BoundReference] = {
-    requiredSchema.getFieldIndex(attr).map { index =>
-      val field = requiredSchema(index)
-      BoundReference(requiredSchema.fieldIndex(attr), field.dataType, field.nullable)
-    }
-  }
-}
-
-object CSVFilters {
-  private def checkFilterRefs(filter: sources.Filter, schema: StructType): Boolean = {
-    val fieldNames = schema.fields.map(_.name).toSet
-    filter.references.forall(fieldNames.contains(_))
-  }
-
-  /**
-   * Returns the filters currently supported by CSV datasource.
-   * @param filters The filters pushed down to CSV datasource.
-   * @param schema data schema of CSV files.
-   * @return a sub-set of `filters` that can be handled by CSV datasource.
-   */
-  def pushedFilters(filters: Array[sources.Filter], schema: StructType): Array[sources.Filter] = {
-    filters.filter(checkFilterRefs(_, schema))
-  }
-
-  private def zip[A, B](a: Option[A], b: Option[B]): Option[(A, B)] = {
-    a.zip(b).headOption
-  }
-
-  private def toLiteral(value: Any): Option[Literal] = {
-    Try(Literal(value)).toOption
-  }
-
-  /**
-   * Converts a filter to an expression and binds it to row positions.
-   *
-   * @param filter The filter to convert.
-   * @param toRef The function converts a filter attribute to a bound reference.
-   * @return some expression with resolved attributes or None if the conversion
-   *         of the given filter to an expression is impossible.
-   */
-  def filterToExpression(
-      filter: sources.Filter,
-      toRef: String => Option[BoundReference]): Option[Expression] = {
-    def zipAttributeAndValue(name: String, value: Any): Option[(BoundReference, Literal)] = {
-      zip(toRef(name), toLiteral(value))
-    }
-    def translate(filter: sources.Filter): Option[Expression] = filter match {
-      case sources.And(left, right) =>
-        zip(translate(left), translate(right)).map(And.tupled)
-      case sources.Or(left, right) =>
-        zip(translate(left), translate(right)).map(Or.tupled)
-      case sources.Not(child) =>
-        translate(child).map(Not)
-      case sources.EqualTo(attribute, value) =>
-        zipAttributeAndValue(attribute, value).map(EqualTo.tupled)
-      case sources.EqualNullSafe(attribute, value) =>
-        zipAttributeAndValue(attribute, value).map(EqualNullSafe.tupled)
-      case sources.IsNull(attribute) =>
-        toRef(attribute).map(IsNull)
-      case sources.IsNotNull(attribute) =>
-        toRef(attribute).map(IsNotNull)
-      case sources.In(attribute, values) =>
-        val literals = values.toSeq.flatMap(toLiteral)
-        if (literals.length == values.length) {
-          toRef(attribute).map(In(_, literals))
-        } else {
-          None
-        }
-      case sources.GreaterThan(attribute, value) =>
-        zipAttributeAndValue(attribute, value).map(GreaterThan.tupled)
-      case sources.GreaterThanOrEqual(attribute, value) =>
-        zipAttributeAndValue(attribute, value).map(GreaterThanOrEqual.tupled)
-      case sources.LessThan(attribute, value) =>
-        zipAttributeAndValue(attribute, value).map(LessThan.tupled)
-      case sources.LessThanOrEqual(attribute, value) =>
-        zipAttributeAndValue(attribute, value).map(LessThanOrEqual.tupled)
-      case sources.StringContains(attribute, value) =>
-        zipAttributeAndValue(attribute, value).map(Contains.tupled)
-      case sources.StringStartsWith(attribute, value) =>
-        zipAttributeAndValue(attribute, value).map(StartsWith.tupled)
-      case sources.StringEndsWith(attribute, value) =>
-        zipAttributeAndValue(attribute, value).map(EndsWith.tupled)
-      case sources.AlwaysTrue() =>
-        Some(Literal(true, BooleanType))
-      case sources.AlwaysFalse() =>
-        Some(Literal(false, BooleanType))
-    }
-    translate(filter)
-  }
-}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
index a7c243537acb7..fd9e30d155148 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
@@ -35,7 +35,7 @@ class CSVInferSchema(val options: CSVOptions) extends Serializable {
     options.zoneId,
     options.locale,
     legacyFormat = FAST_DATE_FORMAT,
-    needVarLengthSecondFraction = true)
+    isParsing = true)
 
   private val decimalParser = if (options.locale == Locale.US) {
     // Special handling the default locale for backward compatibility
@@ -102,13 +102,11 @@ class CSVInferSchema(val options: CSVOptions) extends Serializable {
     if (field == null || field.isEmpty || field == options.nullValue) {
       typeSoFar
     } else {
-      typeSoFar match {
+      val typeElemInfer = typeSoFar match {
         case NullType => tryParseInteger(field)
         case IntegerType => tryParseInteger(field)
         case LongType => tryParseLong(field)
-        case _: DecimalType =>
-          // DecimalTypes have different precisions and scales, so we try to find the common type.
-          compatibleType(typeSoFar, tryParseDecimal(field)).getOrElse(StringType)
+        case _: DecimalType => tryParseDecimal(field)
         case DoubleType => tryParseDouble(field)
         case TimestampType => tryParseTimestamp(field)
         case BooleanType => tryParseBoolean(field)
@@ -116,6 +114,7 @@ class CSVInferSchema(val options: CSVOptions) extends Serializable {
         case other: DataType =>
           throw new UnsupportedOperationException(s"Unexpected data type $other")
       }
+      compatibleType(typeSoFar, typeElemInfer).getOrElse(StringType)
     }
   }
 
@@ -144,7 +143,7 @@ class CSVInferSchema(val options: CSVOptions) extends Serializable {
       // The conversion can fail when the `field` is not a form of number.
       val bigDecimal = decimalParser(field)
       // Because many other formats do not support decimal, it reduces the cases for
-      // decimals by disallowing values having scale (eg. `1.1`).
+      // decimals by disallowing values having scale (e.g. `1.1`).
       if (bigDecimal.scale <= 0) {
         // `DecimalType` conversion can fail when
         //   1. The precision is bigger than 38.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala
index 9d09cab4fd482..b4b25a895016c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala
@@ -166,8 +166,6 @@ class CSVOptions(
 
   val quoteAll = getBool("quoteAll", false)
 
-  val inputBufferSize = 128
-
   /**
    * The max error content length in CSV parser/writer exception message.
    */
@@ -213,6 +211,15 @@ class CSVOptions(
   }
   val lineSeparatorInWrite: Option[String] = lineSeparator
 
+  val inputBufferSize: Option[Int] = parameters.get("inputBufferSize").map(_.toInt)
+    .orElse(SQLConf.get.getConf(SQLConf.CSV_INPUT_BUFFER_SIZE))
+
+  /**
+   * The handling method to be used when unescaped quotes are found in the input.
+   */
+  val unescapedQuoteHandling: UnescapedQuoteHandling = UnescapedQuoteHandling.valueOf(parameters
+    .getOrElse("unescapedQuoteHandling", "STOP_AT_DELIMITER").toUpperCase(Locale.ROOT))
+
   def asWriterSettings: CsvWriterSettings = {
     val writerSettings = new CsvWriterSettings()
     val format = writerSettings.getFormat
@@ -220,7 +227,9 @@ class CSVOptions(
     format.setQuote(quote)
     format.setQuoteEscape(escape)
     charToEscapeQuoteEscaping.foreach(format.setCharToEscapeQuoteEscaping)
-    format.setComment(comment)
+    if (isCommentSet) {
+      format.setComment(comment)
+    }
     lineSeparatorInWrite.foreach(format.setLineSeparator)
 
     writerSettings.setIgnoreLeadingWhitespaces(ignoreLeadingWhiteSpaceFlagInWrite)
@@ -242,17 +251,21 @@ class CSVOptions(
     format.setQuoteEscape(escape)
     lineSeparator.foreach(format.setLineSeparator)
     charToEscapeQuoteEscaping.foreach(format.setCharToEscapeQuoteEscaping)
-    format.setComment(comment)
+    if (isCommentSet) {
+      format.setComment(comment)
+    } else {
+      settings.setCommentProcessingEnabled(false)
+    }
 
     settings.setIgnoreLeadingWhitespaces(ignoreLeadingWhiteSpaceInRead)
     settings.setIgnoreTrailingWhitespaces(ignoreTrailingWhiteSpaceInRead)
     settings.setReadInputOnSeparateThread(false)
-    settings.setInputBufferSize(inputBufferSize)
+    inputBufferSize.foreach(settings.setInputBufferSize)
     settings.setMaxColumns(maxColumns)
     settings.setNullValue(nullValue)
     settings.setEmptyValue(emptyValueInRead)
     settings.setMaxCharsPerColumn(maxCharsPerColumn)
-    settings.setUnescapedQuoteHandling(UnescapedQuoteHandling.STOP_AT_DELIMITER)
+    settings.setUnescapedQuoteHandling(unescapedQuoteHandling)
     settings.setLineSeparatorDetectionEnabled(lineSeparatorInRead.isEmpty && multiLine)
     lineSeparatorInRead.foreach { _ =>
       settings.setNormalizeLineEndingsWithinQuotes(!multiLine)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala
index 4990da2bf3797..a3ee129cd6d64 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala
@@ -47,12 +47,13 @@ class UnivocityGenerator(
     options.zoneId,
     options.locale,
     legacyFormat = FAST_DATE_FORMAT,
-    needVarLengthSecondFraction = false)
+    isParsing = false)
   private val dateFormatter = DateFormatter(
     options.dateFormat,
     options.zoneId,
     options.locale,
-    legacyFormat = FAST_DATE_FORMAT)
+    legacyFormat = FAST_DATE_FORMAT,
+    isParsing = false)
 
   private def makeConverter(dataType: DataType): ValueConverter = dataType match {
     case DateType =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
index 8e87a82769471..b5c14a193ddee 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
@@ -25,10 +25,11 @@ import com.univocity.parsers.csv.CsvParser
 
 import org.apache.spark.SparkUpgradeException
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.{InternalRow, NoopFilters, OrderedFilters}
 import org.apache.spark.sql.catalyst.expressions.{ExprUtils, GenericInternalRow}
 import org.apache.spark.sql.catalyst.util._
 import org.apache.spark.sql.catalyst.util.LegacyDateFormats.FAST_DATE_FORMAT
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
@@ -85,19 +86,24 @@ class UnivocityParser(
   // We preallocate it avoid unnecessary allocations.
   private val noRows = None
 
-  private val timestampFormatter = TimestampFormatter(
+  private lazy val timestampFormatter = TimestampFormatter(
     options.timestampFormat,
     options.zoneId,
     options.locale,
     legacyFormat = FAST_DATE_FORMAT,
-    needVarLengthSecondFraction = true)
-  private val dateFormatter = DateFormatter(
+    isParsing = true)
+  private lazy val dateFormatter = DateFormatter(
     options.dateFormat,
     options.zoneId,
     options.locale,
-    legacyFormat = FAST_DATE_FORMAT)
+    legacyFormat = FAST_DATE_FORMAT,
+    isParsing = true)
 
-  private val csvFilters = new CSVFilters(filters, requiredSchema)
+  private val csvFilters = if (SQLConf.get.csvFilterPushDown) {
+    new OrderedFilters(filters, requiredSchema)
+  } else {
+    new NoopFilters
+  }
 
   // Retrieve the raw record string.
   private def getCurrentInput: UTF8String = {
@@ -185,7 +191,7 @@ class UnivocityParser(
           case NonFatal(e) =>
             // If fails to parse, then tries the way used in 2.0 and 1.x for backwards
             // compatibility.
-            val str = UTF8String.fromString(DateTimeUtils.cleanLegacyTimestampStr(datum))
+            val str = DateTimeUtils.cleanLegacyTimestampStr(UTF8String.fromString(datum))
             DateTimeUtils.stringToTimestamp(str, options.zoneId).getOrElse(throw e)
         }
       }
@@ -198,7 +204,7 @@ class UnivocityParser(
           case NonFatal(e) =>
             // If fails to parse, then tries the way used in 2.0 and 1.x for backwards
             // compatibility.
-            val str = UTF8String.fromString(DateTimeUtils.cleanLegacyTimestampStr(datum))
+            val str = DateTimeUtils.cleanLegacyTimestampStr(UTF8String.fromString(datum))
             DateTimeUtils.stringToDate(str, options.zoneId).getOrElse(throw e)
         }
       }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
index fe3fea5e35b1b..626ece33f157f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.catalyst
 
 import java.sql.{Date, Timestamp}
-import java.time.LocalDate
+import java.time.{Instant, LocalDate}
 
 import scala.language.implicitConversions
 
@@ -31,6 +31,7 @@ import org.apache.spark.sql.catalyst.expressions.objects.Invoke
 import org.apache.spark.sql.catalyst.plans.{Inner, JoinType}
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.UTF8String
 
 /**
  * A collection of implicit conversions that create a DSL for constructing catalyst data structures.
@@ -102,6 +103,14 @@ package object dsl {
     def like(other: Expression, escapeChar: Char = '\\'): Expression =
       Like(expr, other, escapeChar)
     def rlike(other: Expression): Expression = RLike(expr, other)
+    def likeAll(others: Expression*): Expression =
+      LikeAll(expr, others.map(_.eval(EmptyRow).asInstanceOf[UTF8String]))
+    def notLikeAll(others: Expression*): Expression =
+      NotLikeAll(expr, others.map(_.eval(EmptyRow).asInstanceOf[UTF8String]))
+    def likeAny(others: Expression*): Expression =
+      LikeAny(expr, others.map(_.eval(EmptyRow).asInstanceOf[UTF8String]))
+    def notLikeAny(others: Expression*): Expression =
+      NotLikeAny(expr, others.map(_.eval(EmptyRow).asInstanceOf[UTF8String]))
     def contains(other: Expression): Expression = Contains(expr, other)
     def startsWith(other: Expression): Expression = StartsWith(expr, other)
     def endsWith(other: Expression): Expression = EndsWith(expr, other)
@@ -121,14 +130,16 @@ package object dsl {
       if (expr.resolved && expr.dataType.sameType(to)) {
         expr
       } else {
-        Cast(expr, to)
+        val cast = Cast(expr, to)
+        cast.setTagValue(Cast.USER_SPECIFIED_CAST, true)
+        cast
       }
     }
 
     def asc: SortOrder = SortOrder(expr, Ascending)
-    def asc_nullsLast: SortOrder = SortOrder(expr, Ascending, NullsLast, Set.empty)
+    def asc_nullsLast: SortOrder = SortOrder(expr, Ascending, NullsLast, Seq.empty)
     def desc: SortOrder = SortOrder(expr, Descending)
-    def desc_nullsFirst: SortOrder = SortOrder(expr, Descending, NullsFirst, Set.empty)
+    def desc_nullsFirst: SortOrder = SortOrder(expr, Descending, NullsFirst, Seq.empty)
     def as(alias: String): NamedExpression = Alias(expr, alias)()
     def as(alias: Symbol): NamedExpression = Alias(expr, alias.name)()
   }
@@ -152,6 +163,7 @@ package object dsl {
     implicit def bigDecimalToLiteral(d: java.math.BigDecimal): Literal = Literal(d)
     implicit def decimalToLiteral(d: Decimal): Literal = Literal(d)
     implicit def timestampToLiteral(t: Timestamp): Literal = Literal(t)
+    implicit def instantToLiteral(i: Instant): Literal = Literal(i)
     implicit def binaryToLiteral(a: Array[Byte]): Literal = Literal(a)
 
     implicit def symbolToUnresolvedAttribute(s: Symbol): analysis.UnresolvedAttribute =
@@ -167,20 +179,41 @@ package object dsl {
     }
 
     def rand(e: Long): Expression = Rand(e)
-    def sum(e: Expression): Expression = Sum(e).toAggregateExpression()
-    def sumDistinct(e: Expression): Expression = Sum(e).toAggregateExpression(isDistinct = true)
-    def count(e: Expression): Expression = Count(e).toAggregateExpression()
+    def sum(e: Expression, filter: Option[Expression] = None): Expression =
+      Sum(e).toAggregateExpression(isDistinct = false, filter = filter)
+    def sumDistinct(e: Expression, filter: Option[Expression] = None): Expression =
+      Sum(e).toAggregateExpression(isDistinct = true, filter = filter)
+    def count(e: Expression, filter: Option[Expression] = None): Expression =
+      Count(e).toAggregateExpression(isDistinct = false, filter = filter)
     def countDistinct(e: Expression*): Expression =
       Count(e).toAggregateExpression(isDistinct = true)
-    def approxCountDistinct(e: Expression, rsd: Double = 0.05): Expression =
-      HyperLogLogPlusPlus(e, rsd).toAggregateExpression()
-    def avg(e: Expression): Expression = Average(e).toAggregateExpression()
-    def first(e: Expression): Expression = new First(e).toAggregateExpression()
-    def last(e: Expression): Expression = new Last(e).toAggregateExpression()
-    def min(e: Expression): Expression = Min(e).toAggregateExpression()
-    def minDistinct(e: Expression): Expression = Min(e).toAggregateExpression(isDistinct = true)
-    def max(e: Expression): Expression = Max(e).toAggregateExpression()
-    def maxDistinct(e: Expression): Expression = Max(e).toAggregateExpression(isDistinct = true)
+    def countDistinctWithFilter(filter: Expression, e: Expression*): Expression =
+      Count(e).toAggregateExpression(isDistinct = true, filter = Some(filter))
+    def approxCountDistinct(
+        e: Expression,
+        rsd: Double = 0.05,
+        filter: Option[Expression] = None): Expression =
+      HyperLogLogPlusPlus(e, rsd).toAggregateExpression(isDistinct = false, filter = filter)
+    def avg(e: Expression, filter: Option[Expression] = None): Expression =
+      Average(e).toAggregateExpression(isDistinct = false, filter = filter)
+    def first(e: Expression, filter: Option[Expression] = None): Expression =
+      new First(e).toAggregateExpression(isDistinct = false, filter = filter)
+    def last(e: Expression, filter: Option[Expression] = None): Expression =
+      new Last(e).toAggregateExpression(isDistinct = false, filter = filter)
+    def min(e: Expression, filter: Option[Expression] = None): Expression =
+      Min(e).toAggregateExpression(isDistinct = false, filter = filter)
+    def minDistinct(e: Expression, filter: Option[Expression] = None): Expression =
+      Min(e).toAggregateExpression(isDistinct = true, filter = filter)
+    def max(e: Expression, filter: Option[Expression] = None): Expression =
+      Max(e).toAggregateExpression(isDistinct = false, filter = filter)
+    def maxDistinct(e: Expression, filter: Option[Expression] = None): Expression =
+      Max(e).toAggregateExpression(isDistinct = true, filter = filter)
+    def bitAnd(e: Expression, filter: Option[Expression] = None): Expression =
+      BitAndAgg(e).toAggregateExpression(isDistinct = false, filter = filter)
+    def bitOr(e: Expression, filter: Option[Expression] = None): Expression =
+      BitOrAgg(e).toAggregateExpression(isDistinct = false, filter = filter)
+    def bitXor(e: Expression, filter: Option[Expression] = None): Expression =
+      BitXorAgg(e).toAggregateExpression(isDistinct = false, filter = filter)
     def upper(e: Expression): Expression = Upper(e)
     def lower(e: Expression): Expression = Lower(e)
     def coalesce(args: Expression*): Expression = Coalesce(args)
@@ -217,6 +250,9 @@ package object dsl {
       override def expr: Expression = Literal(s)
       def attr: UnresolvedAttribute = analysis.UnresolvedAttribute(s)
     }
+    implicit class DslAttr(attr: UnresolvedAttribute) extends ImplicitAttribute {
+      def s: String = attr.name
+    }
 
     abstract class ImplicitAttribute extends ImplicitOperators {
       def s: String
@@ -261,6 +297,16 @@ package object dsl {
       /** Creates a new AttributeReference of type timestamp */
       def timestamp: AttributeReference = AttributeReference(s, TimestampType, nullable = true)()
 
+      /** Creates a new AttributeReference of the day-time interval type */
+      def dayTimeInterval: AttributeReference = {
+        AttributeReference(s, DayTimeIntervalType, nullable = true)()
+      }
+
+      /** Creates a new AttributeReference of the year-month interval type */
+      def yearMonthInterval: AttributeReference = {
+        AttributeReference(s, YearMonthIntervalType, nullable = true)()
+      }
+
       /** Creates a new AttributeReference of type binary */
       def binary: AttributeReference = AttributeReference(s, BinaryType, nullable = true)()
 
@@ -404,7 +450,7 @@ package object dsl {
           partition: Map[String, Option[String]] = Map.empty,
           overwrite: Boolean = false,
           ifPartitionNotExists: Boolean = false): LogicalPlan =
-        InsertIntoStatement(table, partition, logicalPlan, overwrite, ifPartitionNotExists)
+        InsertIntoStatement(table, partition, Nil, logicalPlan, overwrite, ifPartitionNotExists)
 
       def as(alias: String): LogicalPlan = SubqueryAlias(alias, logicalPlan)
 
@@ -417,11 +463,24 @@ package object dsl {
       def distribute(exprs: Expression*)(n: Int): LogicalPlan =
         RepartitionByExpression(exprs, logicalPlan, numPartitions = n)
 
-      def analyze: LogicalPlan =
-        EliminateSubqueryAliases(analysis.SimpleAnalyzer.execute(logicalPlan))
+      def analyze: LogicalPlan = {
+        val analyzed = analysis.SimpleAnalyzer.execute(logicalPlan)
+        analysis.SimpleAnalyzer.checkAnalysis(analyzed)
+        EliminateSubqueryAliases(analyzed)
+      }
 
       def hint(name: String, parameters: Any*): LogicalPlan =
         UnresolvedHint(name, parameters, logicalPlan)
+
+      def sample(
+          lowerBound: Double,
+          upperBound: Double,
+          withReplacement: Boolean,
+          seed: Long): LogicalPlan = {
+        Sample(lowerBound, upperBound, withReplacement, seed, logicalPlan)
+      }
+
+      def deduplicate(colNames: Attribute*): LogicalPlan = Deduplicate(colNames, logicalPlan)
     }
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala
index f08416fcaba8a..3995c0c5ca633 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.sql.catalyst.encoders
 
-import java.io.ObjectInputStream
-
 import scala.reflect.ClassTag
 import scala.reflect.runtime.universe.{typeTag, TypeTag}
 
@@ -31,6 +29,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjectio
 import org.apache.spark.sql.catalyst.expressions.objects.{AssertNotNull, InitializeJavaBean, Invoke, NewInstance}
 import org.apache.spark.sql.catalyst.optimizer.{ReassignLambdaVariableID, SimplifyCasts}
 import org.apache.spark.sql.catalyst.plans.logical.{CatalystSerde, DeserializeToObject, LeafNode, LocalRelation}
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{ObjectType, StringType, StructField, StructType}
 import org.apache.spark.unsafe.types.UTF8String
@@ -190,7 +189,7 @@ object ExpressionEncoder {
   }
 
   /**
-   * Function that serializesa an object of type `T` to an [[InternalRow]]. This class is not
+   * Function that serializes an object of type `T` to an [[InternalRow]]. This class is not
    * thread-safe. Note that multiple calls to `apply(..)` return the same actual [[InternalRow]]
    * object.  Thus, the caller should copy the result before making another call if required.
    */
@@ -406,7 +405,7 @@ case class ExpressionEncoder[T](
   def assertUnresolved(): Unit = {
     (deserializer +:  serializer).foreach(_.foreach {
       case a: AttributeReference if a.name != "loopVar" =>
-        sys.error(s"Unresolved encoder expected, but $a was found.")
+        throw QueryExecutionErrors.notExpectedUnresolvedEncoderError(a)
       case _ =>
     })
   }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/OuterScopes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/OuterScopes.scala
index 665b2cd1274fd..2d8f02860dd7d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/OuterScopes.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/OuterScopes.scala
@@ -48,7 +48,7 @@ object OuterScopes {
    * useful for inner class defined in REPL.
    */
   def getOuterScope(innerCls: Class[_]): () => AnyRef = {
-    assert(innerCls.isMemberClass)
+    assert(Utils.isMemberClass(innerCls))
     val outerClassName = innerCls.getDeclaringClass.getName
     val outer = outerScopes.get(outerClassName)
     if (outer == null) {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala
index 765018f07d87a..b67f70754e89a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala
@@ -53,6 +53,9 @@ import org.apache.spark.sql.types._
  *   TimestampType -> java.sql.Timestamp if spark.sql.datetime.java8API.enabled is false
  *   TimestampType -> java.time.Instant if spark.sql.datetime.java8API.enabled is true
  *
+ *   DayTimeIntervalType -> java.time.Duration
+ *   YearMonthIntervalType -> java.time.Period
+ *
  *   BinaryType -> byte array
  *   ArrayType -> scala.collection.Seq or Array
  *   MapType -> scala.collection.Map
@@ -108,6 +111,10 @@ object RowEncoder {
         createSerializerForSqlDate(inputObject)
       }
 
+    case DayTimeIntervalType => createSerializerForJavaDuration(inputObject)
+
+    case YearMonthIntervalType => createSerializerForJavaPeriod(inputObject)
+
     case d: DecimalType =>
       CheckOverflow(StaticInvoke(
         Decimal.getClass,
@@ -226,6 +233,8 @@ object RowEncoder {
       } else {
         ObjectType(classOf[java.sql.Date])
       }
+    case DayTimeIntervalType => ObjectType(classOf[java.time.Duration])
+    case YearMonthIntervalType => ObjectType(classOf[java.time.Period])
     case _: DecimalType => ObjectType(classOf[java.math.BigDecimal])
     case StringType => ObjectType(classOf[java.lang.String])
     case _: ArrayType => ObjectType(classOf[scala.collection.Seq[_]])
@@ -281,6 +290,10 @@ object RowEncoder {
         createDeserializerForSqlDate(input)
       }
 
+    case DayTimeIntervalType => createDeserializerForDuration(input)
+
+    case YearMonthIntervalType => createDeserializerForPeriod(input)
+
     case _: DecimalType => createDeserializerForJavaBigDecimal(input, returnNullable = false)
 
     case StringType => createDeserializerForString(input, returnNullable = false)
@@ -291,9 +304,11 @@ object RowEncoder {
           MapObjects(deserializerFor(_), input, et),
           "array",
           ObjectType(classOf[Array[_]]), returnNullable = false)
+      // TODO should use `scala.collection.immutable.ArrayDeq.unsafeMake` method to create
+      //  `immutable.Seq` in Scala 2.13 when Scala version compatibility is no longer required.
       StaticInvoke(
         scala.collection.mutable.WrappedArray.getClass,
-        ObjectType(classOf[Seq[_]]),
+        ObjectType(classOf[scala.collection.Seq[_]]),
         "make",
         arrayData :: Nil,
         returnNullable = false)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/package.scala
index 59f7969e56144..dfa74b6ef7e6f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/package.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.catalyst
 
 import org.apache.spark.sql.Encoder
+import org.apache.spark.sql.errors.QueryExecutionErrors
 
 package object encoders {
   /**
@@ -30,6 +31,6 @@ package object encoders {
     case e: ExpressionEncoder[A] =>
       e.assertUnresolved()
       e
-    case _ => sys.error(s"Only expression encoders are supported today")
+    case _ => throw QueryExecutionErrors.unsupportedEncoderError()
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/errors/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/errors/package.scala
deleted file mode 100644
index 09a6b769da3f4..0000000000000
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/errors/package.scala
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.catalyst
-
-import scala.util.control.NonFatal
-
-import org.apache.spark.SparkException
-import org.apache.spark.sql.catalyst.trees.TreeNode
-
-/**
- * Functions for attaching and retrieving trees that are associated with errors.
- */
-package object errors {
-
-  class TreeNodeException[TreeType <: TreeNode[_]](
-      @transient val tree: TreeType,
-      msg: String,
-      cause: Throwable)
-    extends Exception(msg, cause) {
-
-    val treeString = tree.toString
-
-    // Yes, this is the same as a default parameter, but... those don't seem to work with SBT
-    // external project dependencies for some reason.
-    def this(tree: TreeType, msg: String) = this(tree, msg, null)
-
-    override def getMessage: String = {
-      s"${super.getMessage}, tree:${if (treeString contains "\n") "\n" else " "}$tree"
-    }
-  }
-
-  /**
-   *  Wraps any exceptions that are thrown while executing `f` in a
-   *  [[catalyst.errors.TreeNodeException TreeNodeException]], attaching the provided `tree`.
-   */
-  def attachTree[TreeType <: TreeNode[_], A](tree: TreeType, msg: String = "")(f: => A): A = {
-    try f catch {
-      // SPARK-16748: We do not want SparkExceptions from job failures in the planning phase
-      // to create TreeNodeException. Hence, wrap exception only if it is not SparkException.
-      case NonFatal(e) if !e.isInstanceOf[SparkException] =>
-        throw new TreeNodeException(tree, msg, e)
-    }
-  }
-}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/AliasHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/AliasHelper.scala
new file mode 100644
index 0000000000000..1f3f762662252
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/AliasHelper.scala
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions
+
+import org.apache.spark.sql.catalyst.analysis.MultiAlias
+import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
+import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Project}
+
+/**
+ * Helper methods for collecting and replacing aliases.
+ */
+trait AliasHelper {
+
+  protected def getAliasMap(plan: Project): AttributeMap[Alias] = {
+    // Create a map of Aliases to their values from the child projection.
+    // e.g., 'SELECT a + b AS c, d ...' produces Map(c -> Alias(a + b, c)).
+    getAliasMap(plan.projectList)
+  }
+
+  protected def getAliasMap(plan: Aggregate): AttributeMap[Alias] = {
+    // Find all the aliased expressions in the aggregate list that don't include any actual
+    // AggregateExpression or PythonUDF, and create a map from the alias to the expression
+    val aliasMap = plan.aggregateExpressions.collect {
+      case a: Alias if a.child.find(e => e.isInstanceOf[AggregateExpression] ||
+        PythonUDF.isGroupedAggPandasUDF(e)).isEmpty =>
+        (a.toAttribute, a)
+    }
+    AttributeMap(aliasMap)
+  }
+
+  protected def getAliasMap(exprs: Seq[NamedExpression]): AttributeMap[Alias] = {
+    // Create a map of Aliases to their values from the child projection.
+    // e.g., 'SELECT a + b AS c, d ...' produces Map(c -> Alias(a + b, c)).
+    AttributeMap(exprs.collect { case a: Alias => (a.toAttribute, a) })
+  }
+
+  /**
+   * Replace all attributes, that reference an alias, with the aliased expression
+   */
+  protected def replaceAlias(
+      expr: Expression,
+      aliasMap: AttributeMap[Alias]): Expression = {
+    // Use transformUp to prevent infinite recursion when the replacement expression
+    // redefines the same ExprId,
+    trimAliases(expr.transformUp {
+      case a: Attribute => aliasMap.getOrElse(a, a)
+    })
+  }
+
+  /**
+   * Replace all attributes, that reference an alias, with the aliased expression,
+   * but keep the name of the outermost attribute.
+   */
+  protected def replaceAliasButKeepName(
+     expr: NamedExpression,
+     aliasMap: AttributeMap[Alias]): NamedExpression = {
+    // Use transformUp to prevent infinite recursion when the replacement expression
+    // redefines the same ExprId,
+    trimNonTopLevelAliases(expr.transformUp {
+      case a: Attribute => aliasMap.getOrElse(a, a)
+    }).asInstanceOf[NamedExpression]
+  }
+
+  protected def trimAliases(e: Expression): Expression = {
+    e.transformDown {
+      case Alias(child, _) => child
+      case MultiAlias(child, _) => child
+    }
+  }
+
+  protected def trimNonTopLevelAliases[T <: Expression](e: T): T = {
+    val res = e match {
+      case a: Alias =>
+        a.copy(child = trimAliases(a.child))(
+          exprId = a.exprId,
+          qualifier = a.qualifier,
+          explicitMetadata = Some(a.metadata),
+          nonInheritableMetadataKeys = a.nonInheritableMetadataKeys)
+      case a: MultiAlias =>
+        a.copy(child = trimAliases(a.child))
+      case other => trimAliases(other)
+    }
+
+    res.asInstanceOf[T]
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ApplyFunctionExpression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ApplyFunctionExpression.scala
new file mode 100644
index 0000000000000..aa6faceffcddb
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ApplyFunctionExpression.scala
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
+import org.apache.spark.sql.connector.catalog.functions.ScalarFunction
+import org.apache.spark.sql.types.DataType
+
+case class ApplyFunctionExpression(
+    function: ScalarFunction[_],
+    children: Seq[Expression]) extends Expression with UserDefinedExpression with CodegenFallback {
+  override def nullable: Boolean = function.isResultNullable
+  override def name: String = function.name()
+  override def dataType: DataType = function.resultType()
+
+  private lazy val reusedRow = new GenericInternalRow(children.size)
+
+  /** Returns the result of evaluating this expression on a given input Row */
+  override def eval(input: InternalRow): Any = {
+    var i = 0
+    while (i < children.length) {
+      val expr = children(i)
+      reusedRow.update(i, expr.eval(input))
+      i += 1
+    }
+
+    function.produceResult(reusedRow)
+  }
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
+    copy(children = newChildren)
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/AttributeSet.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/AttributeSet.scala
index 038ebb2037a27..b059e4f849664 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/AttributeSet.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/AttributeSet.scala
@@ -37,7 +37,11 @@ object AttributeSet {
   val empty = apply(Iterable.empty)
 
   /** Constructs a new [[AttributeSet]] that contains a single [[Attribute]]. */
-  def apply(a: Attribute): AttributeSet = new AttributeSet(Set(new AttributeEquals(a)))
+  def apply(a: Attribute): AttributeSet = {
+    val baseSet = new mutable.LinkedHashSet[AttributeEquals]
+    baseSet += new AttributeEquals(a)
+    new AttributeSet(baseSet)
+  }
 
   /** Constructs a new [[AttributeSet]] given a sequence of [[Expression Expressions]]. */
   def apply(baseSet: Iterable[Expression]): AttributeSet = {
@@ -47,7 +51,7 @@ object AttributeSet {
   /** Constructs a new [[AttributeSet]] given a sequence of [[AttributeSet]]s. */
   def fromAttributeSets(sets: Iterable[AttributeSet]): AttributeSet = {
     val baseSet = sets.foldLeft(new mutable.LinkedHashSet[AttributeEquals]())( _ ++= _.baseSet)
-    new AttributeSet(baseSet.toSet)
+    new AttributeSet(baseSet)
   }
 }
 
@@ -62,7 +66,7 @@ object AttributeSet {
  * and also makes doing transformations hard (we always try keep older trees instead of new ones
  * when the transformation was a no-op).
  */
-class AttributeSet private (val baseSet: Set[AttributeEquals])
+class AttributeSet private (private val baseSet: mutable.LinkedHashSet[AttributeEquals])
   extends Iterable[Attribute] with Serializable {
 
   override def hashCode: Int = baseSet.hashCode()
@@ -101,10 +105,12 @@ class AttributeSet private (val baseSet: Set[AttributeEquals])
    */
   def --(other: Iterable[NamedExpression]): AttributeSet = {
     other match {
+      // SPARK-32755: `--` method behave differently under scala 2.12 and 2.13,
+      // use a Scala 2.12 based code to maintains the insertion order in Scala 2.13
       case otherSet: AttributeSet =>
-        new AttributeSet(baseSet -- otherSet.baseSet)
+        new AttributeSet(baseSet.clone() --= otherSet.baseSet)
       case _ =>
-        new AttributeSet(baseSet -- other.map(a => new AttributeEquals(a.toAttribute)))
+        new AttributeSet(baseSet.clone() --= other.map(a => new AttributeEquals(a.toAttribute)))
     }
   }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BoundAttribute.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BoundAttribute.scala
index 7ae5924b20faf..4d303aa95b78a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BoundAttribute.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BoundAttribute.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.catalyst.expressions
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.errors.attachTree
 import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodeGenerator, ExprCode, FalseLiteral, JavaCode}
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
 import org.apache.spark.sql.types._
@@ -72,17 +71,16 @@ object BindReferences extends Logging {
       input: AttributeSeq,
       allowFailures: Boolean = false): A = {
     expression.transform { case a: AttributeReference =>
-      attachTree(a, "Binding attribute") {
-        val ordinal = input.indexOf(a.exprId)
-        if (ordinal == -1) {
-          if (allowFailures) {
-            a
-          } else {
-            sys.error(s"Couldn't find $a in ${input.attrs.mkString("[", ",", "]")}")
-          }
+      val ordinal = input.indexOf(a.exprId)
+      if (ordinal == -1) {
+        if (allowFailures) {
+          a
         } else {
-          BoundReference(ordinal, a.dataType, input(ordinal).nullable)
+          throw new IllegalStateException(
+            s"Couldn't find $a in ${input.attrs.mkString("[", ",", "]")}")
         }
+      } else {
+        BoundReference(ordinal, a.dataType, input(ordinal).nullable)
       }
     }.asInstanceOf[A] // Kind of a hack, but safe.  TODO: Tighten return type when possible.
   }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala
index e6a4c8f1d3749..7cb830d115689 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala
@@ -51,9 +51,11 @@ import org.apache.spark.util.Utils
        c33fb387-8500-4bfa-81d2-6e0e3e930df2
       > SELECT _FUNC_('java.util.UUID', 'fromString', 'a5cf6c42-0c85-418f-af6c-3e4e5b1328f2');
        a5cf6c42-0c85-418f-af6c-3e4e5b1328f2
-  """)
+  """,
+  since = "2.0.0",
+  group = "misc_funcs")
 case class CallMethodViaReflection(children: Seq[Expression])
-  extends Expression with CodegenFallback {
+  extends Nondeterministic with CodegenFallback {
 
   override def prettyName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("reflect")
 
@@ -76,11 +78,11 @@ case class CallMethodViaReflection(children: Seq[Expression])
     }
   }
 
-  override lazy val deterministic: Boolean = false
   override def nullable: Boolean = true
   override val dataType: DataType = StringType
+  override protected def initializeInternal(partitionIndex: Int): Unit = {}
 
-  override def eval(input: InternalRow): Any = {
+  override protected def evalInternal(input: InternalRow): Any = {
     var i = 0
     while (i < argExprs.length) {
       buffer(i) = argExprs(i).eval(input).asInstanceOf[Object]
@@ -112,6 +114,9 @@ case class CallMethodViaReflection(children: Seq[Expression])
 
   /** A temporary buffer used to hold intermediate results returned by children. */
   @transient private lazy val buffer = new Array[Object](argExprs.length)
+
+  override protected def withNewChildrenInternal(
+    newChildren: IndexedSeq[Expression]): CallMethodViaReflection = copy(children = newChildren)
 }
 
 object CallMethodViaReflection {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Canonicalize.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Canonicalize.scala
index a8031086d82f7..2765ec7d8a0eb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Canonicalize.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Canonicalize.scala
@@ -70,8 +70,11 @@ object Canonicalize {
 
   /** Rearrange expressions that are commutative or associative. */
   private def expressionReorder(e: Expression): Expression = e match {
-    case a: Add => orderCommutative(a, { case Add(l, r) => Seq(l, r) }).reduce(Add)
-    case m: Multiply => orderCommutative(m, { case Multiply(l, r) => Seq(l, r) }).reduce(Multiply)
+    // TODO: do not reorder consecutive `Add`s or `Multiply`s with different `failOnError` flags
+    case a @ Add(_, _, f) =>
+      orderCommutative(a, { case Add(l, r, _) => Seq(l, r) }).reduce(Add(_, _, f))
+    case m @ Multiply(_, _, f) =>
+      orderCommutative(m, { case Multiply(l, r, _) => Seq(l, r) }).reduce(Multiply(_, _, f))
 
     case o: Or =>
       orderCommutative(o, { case Or(l, r) if l.deterministic && r.deterministic => Seq(l, r) })
@@ -80,6 +83,13 @@ object Canonicalize {
       orderCommutative(a, { case And(l, r) if l.deterministic && r.deterministic => Seq(l, r)})
         .reduce(And)
 
+    case o: BitwiseOr =>
+      orderCommutative(o, { case BitwiseOr(l, r) => Seq(l, r) }).reduce(BitwiseOr)
+    case a: BitwiseAnd =>
+      orderCommutative(a, { case BitwiseAnd(l, r) => Seq(l, r) }).reduce(BitwiseAnd)
+    case x: BitwiseXor =>
+      orderCommutative(x, { case BitwiseXor(l, r) => Seq(l, r) }).reduce(BitwiseXor)
+
     case EqualTo(l, r) if l.hashCode() > r.hashCode() => EqualTo(r, l)
     case EqualNullSafe(l, r) if l.hashCode() > r.hashCode() => EqualNullSafe(r, l)
 
@@ -99,6 +109,13 @@ object Canonicalize {
     // order the list in the In operator
     case In(value, list) if list.length > 1 => In(value, list.sortBy(_.hashCode()))
 
+    case g: Greatest =>
+      val newChildren = orderCommutative(g, { case Greatest(children) => children })
+      Greatest(newChildren)
+    case l: Least =>
+      val newChildren = orderCommutative(l, { case Least(children) => children })
+      Least(newChildren)
+
     case _ => e
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index 9bc86d4fb4e42..c52578d913682 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -17,19 +17,22 @@
 
 package org.apache.spark.sql.catalyst.expressions
 
-import java.math.{BigDecimal => JavaBigDecimal}
 import java.time.ZoneId
 import java.util.Locale
 import java.util.concurrent.TimeUnit._
 
-import org.apache.spark.SparkException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, TypeCoercion}
+import org.apache.spark.sql.catalyst.expressions.Cast.{forceNullable, resolvableNullability}
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
+import org.apache.spark.sql.catalyst.trees.TreeNodeTag
+import org.apache.spark.sql.catalyst.trees.TreePattern.{CAST, TreePattern}
 import org.apache.spark.sql.catalyst.util._
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils._
+import org.apache.spark.sql.catalyst.util.IntervalStringStyles.ANSI_STYLE
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.UTF8StringBuilder
@@ -38,6 +41,11 @@ import org.apache.spark.unsafe.types.UTF8String.{IntWrapper, LongWrapper}
 
 object Cast {
 
+  /**
+   * A tag to decide if a CAST is specified by user.
+   */
+  val USER_SPECIFIED_CAST = new TreeNodeTag[Boolean]("user_specified_cast")
+
   /**
    * Returns true iff we can cast `from` type to `to` type.
    */
@@ -65,6 +73,8 @@ object Cast {
     case (TimestampType, DateType) => true
 
     case (StringType, CalendarIntervalType) => true
+    case (StringType, DayTimeIntervalType) => true
+    case (StringType, YearMonthIntervalType) => true
 
     case (StringType, _: NumericType) => true
     case (BooleanType, _: NumericType) => true
@@ -92,8 +102,7 @@ object Cast {
                 toField.nullable)
         }
 
-    case (udt1: UserDefinedType[_], udt2: UserDefinedType[_]) if udt1.userClass == udt2.userClass =>
-      true
+    case (udt1: UserDefinedType[_], udt2: UserDefinedType[_]) if udt2.acceptsType(udt1) => true
 
     case _ => false
   }
@@ -108,8 +117,8 @@ object Cast {
    */
   def needsTimeZone(from: DataType, to: DataType): Boolean = (from, to) match {
     case (StringType, TimestampType | DateType) => true
+    case (TimestampType | DateType, StringType) => true
     case (DateType, TimestampType) => true
-    case (TimestampType, StringType) => true
     case (TimestampType, DateType) => true
     case (ArrayType(fromType, _), ArrayType(toType, _)) => needsTimeZone(fromType, toType)
     case (MapType(fromKey, fromValue, _), MapType(toKey, toValue, _)) =>
@@ -156,6 +165,8 @@ object Cast {
             resolvableNullability(f1.nullable, f2.nullable) && canUpCast(f1.dataType, f2.dataType)
         }
 
+    case (from: UserDefinedType[_], to: UserDefinedType[_]) if to.acceptsType(from) => true
+
     case _ => false
   }
 
@@ -256,17 +267,26 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
 
   def dataType: DataType
 
+  /**
+   * Returns true iff we can cast `from` type to `to` type.
+   */
+  def canCast(from: DataType, to: DataType): Boolean
+
+  /**
+   * Returns the error message if casting from one type to another one is invalid.
+   */
+  def typeCheckFailureMessage: String
+
   override def toString: String = {
     val ansi = if (ansiEnabled) "ansi_" else ""
     s"${ansi}cast($child as ${dataType.simpleString})"
   }
 
   override def checkInputDataTypes(): TypeCheckResult = {
-    if (Cast.canCast(child.dataType, dataType)) {
+    if (canCast(child.dataType, dataType)) {
       TypeCheckResult.TypeCheckSuccess
     } else {
-      TypeCheckResult.TypeCheckFailure(
-        s"cannot cast ${child.dataType.catalogString} to ${dataType.catalogString}")
+      TypeCheckResult.TypeCheckFailure(typeCheckFailureMessage)
     }
   }
 
@@ -282,11 +302,15 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
   def needsTimeZone: Boolean = Cast.needsTimeZone(child.dataType, dataType)
 
   // [[func]] assumes the input is no longer null because eval already does the null check.
-  @inline private[this] def buildCast[T](a: Any, func: T => Any): Any = func(a.asInstanceOf[T])
+  @inline protected[this] def buildCast[T](a: Any, func: T => Any): Any = func(a.asInstanceOf[T])
 
   private lazy val dateFormatter = DateFormatter(zoneId)
   private lazy val timestampFormatter = TimestampFormatter.getFractionFormatter(zoneId)
 
+  private val legacyCastToStr = SQLConf.get.getConf(SQLConf.LEGACY_COMPLEX_TYPES_TO_STRING)
+  // The brackets that are used in casting structs and maps to strings
+  private val (leftBracket, rightBracket) = if (legacyCastToStr) ("[", "]") else ("{", "}")
+
   // UDFToString
   private[this] def castToString(from: DataType): Any => Any = from match {
     case CalendarIntervalType =>
@@ -294,20 +318,24 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
     case BinaryType => buildCast[Array[Byte]](_, UTF8String.fromBytes)
     case DateType => buildCast[Int](_, d => UTF8String.fromString(dateFormatter.format(d)))
     case TimestampType => buildCast[Long](_,
-      t => UTF8String.fromString(DateTimeUtils.timestampToString(timestampFormatter, t)))
+      t => UTF8String.fromString(timestampFormatter.format(t)))
     case ArrayType(et, _) =>
       buildCast[ArrayData](_, array => {
         val builder = new UTF8StringBuilder
         builder.append("[")
         if (array.numElements > 0) {
           val toUTF8String = castToString(et)
-          if (!array.isNullAt(0)) {
+          if (array.isNullAt(0)) {
+            if (!legacyCastToStr) builder.append("null")
+          } else {
             builder.append(toUTF8String(array.get(0, et)).asInstanceOf[UTF8String])
           }
           var i = 1
           while (i < array.numElements) {
             builder.append(",")
-            if (!array.isNullAt(i)) {
+            if (array.isNullAt(i)) {
+              if (!legacyCastToStr) builder.append(" null")
+            } else {
               builder.append(" ")
               builder.append(toUTF8String(array.get(i, et)).asInstanceOf[UTF8String])
             }
@@ -320,7 +348,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
     case MapType(kt, vt, _) =>
       buildCast[MapData](_, map => {
         val builder = new UTF8StringBuilder
-        builder.append("[")
+        builder.append(leftBracket)
         if (map.numElements > 0) {
           val keyArray = map.keyArray()
           val valueArray = map.valueArray()
@@ -328,7 +356,9 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
           val valueToUTF8String = castToString(vt)
           builder.append(keyToUTF8String(keyArray.get(0, kt)).asInstanceOf[UTF8String])
           builder.append(" ->")
-          if (!valueArray.isNullAt(0)) {
+          if (valueArray.isNullAt(0)) {
+            if (!legacyCastToStr) builder.append(" null")
+          } else {
             builder.append(" ")
             builder.append(valueToUTF8String(valueArray.get(0, vt)).asInstanceOf[UTF8String])
           }
@@ -337,7 +367,9 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
             builder.append(", ")
             builder.append(keyToUTF8String(keyArray.get(i, kt)).asInstanceOf[UTF8String])
             builder.append(" ->")
-            if (!valueArray.isNullAt(i)) {
+            if (valueArray.isNullAt(i)) {
+              if (!legacyCastToStr) builder.append(" null")
+            } else {
               builder.append(" ")
               builder.append(valueToUTF8String(valueArray.get(i, vt))
                 .asInstanceOf[UTF8String])
@@ -345,35 +377,45 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
             i += 1
           }
         }
-        builder.append("]")
+        builder.append(rightBracket)
         builder.build()
       })
     case StructType(fields) =>
       buildCast[InternalRow](_, row => {
         val builder = new UTF8StringBuilder
-        builder.append("[")
+        builder.append(leftBracket)
         if (row.numFields > 0) {
           val st = fields.map(_.dataType)
           val toUTF8StringFuncs = st.map(castToString)
-          if (!row.isNullAt(0)) {
+          if (row.isNullAt(0)) {
+            if (!legacyCastToStr) builder.append("null")
+          } else {
             builder.append(toUTF8StringFuncs(0)(row.get(0, st(0))).asInstanceOf[UTF8String])
           }
           var i = 1
           while (i < row.numFields) {
             builder.append(",")
-            if (!row.isNullAt(i)) {
+            if (row.isNullAt(i)) {
+              if (!legacyCastToStr) builder.append(" null")
+            } else {
               builder.append(" ")
               builder.append(toUTF8StringFuncs(i)(row.get(i, st(i))).asInstanceOf[UTF8String])
             }
             i += 1
           }
         }
-        builder.append("]")
+        builder.append(rightBracket)
         builder.build()
       })
     case pudt: PythonUserDefinedType => castToString(pudt.sqlType)
     case udt: UserDefinedType[_] =>
       buildCast[Any](_, o => UTF8String.fromString(udt.deserialize(o).toString))
+    case YearMonthIntervalType =>
+      buildCast[Int](_, i => UTF8String.fromString(
+        IntervalUtils.toYearMonthIntervalString(i, ANSI_STYLE)))
+    case DayTimeIntervalType =>
+      buildCast[Long](_, i => UTF8String.fromString(
+        IntervalUtils.toDayTimeIntervalString(i, ANSI_STYLE)))
     case _ => buildCast[Any](_, o => UTF8String.fromString(o.toString))
   }
 
@@ -395,7 +437,11 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
         } else if (StringUtils.isFalseString(s)) {
           false
         } else {
-          null
+          if (ansiEnabled) {
+            throw new UnsupportedOperationException(s"invalid input syntax for type boolean: $s")
+          } else {
+            null
+          }
         }
       })
     case TimestampType =>
@@ -422,7 +468,13 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
   // TimestampConverter
   private[this] def castToTimestamp(from: DataType): Any => Any = from match {
     case StringType =>
-      buildCast[UTF8String](_, utfs => DateTimeUtils.stringToTimestamp(utfs, zoneId).orNull)
+      buildCast[UTF8String](_, utfs => {
+        if (ansiEnabled) {
+          DateTimeUtils.stringToTimestampAnsi(utfs, zoneId)
+        } else {
+          DateTimeUtils.stringToTimestamp(utfs, zoneId).orNull
+        }
+      })
     case BooleanType =>
       buildCast[Boolean](_, b => if (b) 1L else 0)
     case LongType =>
@@ -434,7 +486,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
     case ByteType =>
       buildCast[Byte](_, b => longToTimestamp(b.toLong))
     case DateType =>
-      buildCast[Int](_, d => epochDaysToMicros(d, zoneId))
+      buildCast[Int](_, d => daysToMicros(d, zoneId))
     // TimestampWritable.decimalToTimestamp
     case DecimalType() =>
       buildCast[Decimal](_, d => decimalToTimestamp(d))
@@ -467,11 +519,15 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
   // DateConverter
   private[this] def castToDate(from: DataType): Any => Any = from match {
     case StringType =>
-      buildCast[UTF8String](_, s => DateTimeUtils.stringToDate(s, zoneId).orNull)
+      if (ansiEnabled) {
+        buildCast[UTF8String](_, s => DateTimeUtils.stringToDateAnsi(s, zoneId))
+      } else {
+        buildCast[UTF8String](_, s => DateTimeUtils.stringToDate(s, zoneId).orNull)
+      }
     case TimestampType =>
       // throw valid precision more than seconds, according to Hive.
       // Timestamp.nanos is in 0 to 999,999,999, no more than a second.
-      buildCast[Long](_, t => microsToEpochDays(t, zoneId))
+      buildCast[Long](_, t => microsToDays(t, zoneId))
   }
 
   // IntervalConverter
@@ -480,6 +536,14 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
       buildCast[UTF8String](_, s => IntervalUtils.safeStringToInterval(s))
   }
 
+  private[this] def castToDayTimeInterval(from: DataType): Any => Any = from match {
+    case StringType => buildCast[UTF8String](_, s => IntervalUtils.castStringToDTInterval(s))
+  }
+
+  private[this] def castToYearMonthInterval(from: DataType): Any => Any = from match {
+    case StringType => buildCast[UTF8String](_, s => IntervalUtils.castStringToYMInterval(s))
+  }
+
   // LongConverter
   private[this] def castToLong(from: DataType): Any => Any = from match {
     case StringType if ansiEnabled =>
@@ -516,7 +580,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
         if (longValue == longValue.toInt) {
           longValue.toInt
         } else {
-          throw new ArithmeticException(s"Casting $t to int causes overflow")
+          throw QueryExecutionErrors.castingCauseOverflowError(t, IntegerType.catalogString)
         }
       })
     case TimestampType =>
@@ -548,7 +612,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
         if (longValue == longValue.toShort) {
           longValue.toShort
         } else {
-          throw new ArithmeticException(s"Casting $t to short causes overflow")
+          throw QueryExecutionErrors.castingCauseOverflowError(t, ShortType.catalogString)
         }
       })
     case TimestampType =>
@@ -559,12 +623,12 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
           x.exactNumeric.asInstanceOf[Numeric[Any]].toInt(b)
         } catch {
           case _: ArithmeticException =>
-            throw new ArithmeticException(s"Casting $b to short causes overflow")
+            throw QueryExecutionErrors.castingCauseOverflowError(b, ShortType.catalogString)
         }
         if (intValue == intValue.toShort) {
           intValue.toShort
         } else {
-          throw new ArithmeticException(s"Casting $b to short causes overflow")
+          throw QueryExecutionErrors.castingCauseOverflowError(b, ShortType.catalogString)
         }
     case x: NumericType =>
       b => x.numeric.asInstanceOf[Numeric[Any]].toInt(b).toShort
@@ -591,7 +655,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
         if (longValue == longValue.toByte) {
           longValue.toByte
         } else {
-          throw new ArithmeticException(s"Casting $t to byte causes overflow")
+          throw QueryExecutionErrors.castingCauseOverflowError(t, ByteType.catalogString)
         }
       })
     case TimestampType =>
@@ -602,12 +666,12 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
           x.exactNumeric.asInstanceOf[Numeric[Any]].toInt(b)
         } catch {
           case _: ArithmeticException =>
-            throw new ArithmeticException(s"Casting $b to byte causes overflow")
+            throw QueryExecutionErrors.castingCauseOverflowError(b, ByteType.catalogString)
         }
         if (intValue == intValue.toByte) {
           intValue.toByte
         } else {
-          throw new ArithmeticException(s"Casting $b to byte causes overflow")
+          throw QueryExecutionErrors.castingCauseOverflowError(b, ByteType.catalogString)
         }
     case x: NumericType =>
       b => x.numeric.asInstanceOf[Numeric[Any]].toInt(b).toByte
@@ -628,8 +692,8 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
       if (!ansiEnabled) {
         null
       } else {
-        throw new ArithmeticException(s"${value.toDebugString} cannot be represented as " +
-          s"Decimal(${decimalType.precision}, ${decimalType.scale}).")
+        throw QueryExecutionErrors.cannotChangeDecimalPrecisionError(
+          value, decimalType.precision, decimalType.scale)
       }
     }
   }
@@ -645,19 +709,13 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
 
 
   private[this] def castToDecimal(from: DataType, target: DecimalType): Any => Any = from match {
-    case StringType =>
-      buildCast[UTF8String](_, s => try {
-        // According the benchmark test,  `s.toString.trim` is much faster than `s.trim.toString`.
-        // Please refer to https://github.com/apache/spark/pull/26640
-        changePrecision(Decimal(new JavaBigDecimal(s.toString.trim)), target)
-      } catch {
-        case _: NumberFormatException =>
-          if (ansiEnabled) {
-            throw new NumberFormatException(s"invalid input syntax for type numeric: $s")
-          } else {
-            null
-          }
+    case StringType if !ansiEnabled =>
+      buildCast[UTF8String](_, s => {
+        val d = Decimal.fromString(s)
+        if (d == null) null else changePrecision(d, target)
       })
+    case StringType if ansiEnabled =>
+      buildCast[UTF8String](_, s => changePrecision(Decimal.fromStringANSI(s), target))
     case BooleanType =>
       buildCast[Boolean](_, b => toPrecision(if (b) Decimal.ONE else Decimal.ZERO, target))
     case DateType =>
@@ -686,7 +744,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
           case _: NumberFormatException =>
             val d = Cast.processFloatingPointSpecialLiterals(doubleStr, false)
             if(ansiEnabled && d == null) {
-              throw new NumberFormatException(s"invalid input syntax for type numeric: $s")
+              throw QueryExecutionErrors.invalidInputSyntaxForNumericError(s)
             } else {
               d
             }
@@ -711,7 +769,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
           case _: NumberFormatException =>
             val f = Cast.processFloatingPointSpecialLiterals(floatStr, true)
             if (ansiEnabled && f == null) {
-              throw new NumberFormatException(s"invalid input syntax for type numeric: $s")
+              throw QueryExecutionErrors.invalidInputSyntaxForNumericError(s)
             } else {
               f
             }
@@ -770,7 +828,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
     })
   }
 
-  private[this] def cast(from: DataType, to: DataType): Any => Any = {
+  protected[this] def cast(from: DataType, to: DataType): Any => Any = {
     // If the cast does not change the structure, then we don't really need to cast anything.
     // We can return what the children return. Same thing should happen in the codegen path.
     if (DataType.equalsStructurally(from, to)) {
@@ -780,7 +838,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
       // For primitive types, we don't reach here because the guard of `nullSafeEval`.
       // But for nested types like struct, we might reach here for nested null type field.
       // We won't call the returned function actually, but returns a placeholder.
-      _ => throw new SparkException(s"should not directly cast from NullType to $to.")
+      _ => throw QueryExecutionErrors.cannotCastFromNullTypeError(to)
     } else {
       to match {
         case dt if dt == from => identity[Any]
@@ -790,6 +848,8 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
         case decimal: DecimalType => castToDecimal(from, decimal)
         case TimestampType => castToTimestamp(from)
         case CalendarIntervalType => castToInterval(from)
+        case DayTimeIntervalType => castToDayTimeInterval(from)
+        case YearMonthIntervalType => castToYearMonthInterval(from)
         case BooleanType => castToBoolean(from)
         case ByteType => castToByte(from)
         case ShortType => castToShort(from)
@@ -801,16 +861,15 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
           castArray(from.asInstanceOf[ArrayType].elementType, array.elementType)
         case map: MapType => castMap(from.asInstanceOf[MapType], map)
         case struct: StructType => castStruct(from.asInstanceOf[StructType], struct)
-        case udt: UserDefinedType[_]
-          if udt.userClass == from.asInstanceOf[UserDefinedType[_]].userClass =>
+        case udt: UserDefinedType[_] if udt.acceptsType(from) =>
           identity[Any]
         case _: UserDefinedType[_] =>
-          throw new SparkException(s"Cannot cast $from to $to.")
+          throw QueryExecutionErrors.cannotCastError(from, to)
       }
     }
   }
 
-  private[this] lazy val cast: Any => Any = cast(child.dataType, dataType)
+  protected[this] lazy val cast: Any => Any = cast(child.dataType, dataType)
 
   protected override def nullSafeEval(input: Any): Any = cast(input)
 
@@ -834,7 +893,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
 
   // The function arguments are: `input`, `result` and `resultIsNull`. We don't need `inputIsNull`
   // in parameter list, because the returned code will be put in null safe evaluation region.
-  private[this] type CastFunction = (ExprValue, ExprValue, ExprValue) => Block
+  protected[this] type CastFunction = (ExprValue, ExprValue, ExprValue) => Block
 
   private[this] def nullSafeCastFunction(
       from: DataType,
@@ -849,6 +908,8 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
     case decimal: DecimalType => castToDecimalCode(from, decimal, ctx)
     case TimestampType => castToTimestampCode(from, ctx)
     case CalendarIntervalType => castToIntervalCode(from)
+    case DayTimeIntervalType => castToDayTimeIntervalCode(from)
+    case YearMonthIntervalType => castToYearMonthIntervalCode(from)
     case BooleanType => castToBooleanCode(from)
     case ByteType => castToByteCode(from, ctx)
     case ShortType => castToShortCode(from, ctx)
@@ -861,16 +922,15 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
       castArrayCode(from.asInstanceOf[ArrayType].elementType, array.elementType, ctx)
     case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
     case struct: StructType => castStructCode(from.asInstanceOf[StructType], struct, ctx)
-    case udt: UserDefinedType[_]
-      if udt.userClass == from.asInstanceOf[UserDefinedType[_]].userClass =>
+    case udt: UserDefinedType[_] if udt.acceptsType(from) =>
       (c, evPrim, evNull) => code"$evPrim = $c;"
     case _: UserDefinedType[_] =>
-      throw new SparkException(s"Cannot cast $from to $to.")
+      throw QueryExecutionErrors.cannotCastError(from, to)
   }
 
   // Since we need to cast input expressions recursively inside ComplexTypes, such as Map's
   // Key and Value, Struct's field, we need to name out all the variable names involved in a cast.
-  private[this] def castCode(ctx: CodegenContext, input: ExprValue, inputIsNull: ExprValue,
+  protected[this] def castCode(ctx: CodegenContext, input: ExprValue, inputIsNull: ExprValue,
     result: ExprValue, resultIsNull: ExprValue, resultType: DataType, cast: CastFunction): Block = {
     val javaType = JavaCode.javaType(resultType)
     code"""
@@ -882,6 +942,10 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
     """
   }
 
+  private def appendIfNotLegacyCastToStr(buffer: ExprValue, s: String): Block = {
+    if (!legacyCastToStr) code"""$buffer.append("$s");""" else EmptyBlock
+  }
+
   private def writeArrayToStringBuilder(
       et: DataType,
       array: ExprValue,
@@ -904,12 +968,16 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
     code"""
        |$buffer.append("[");
        |if ($array.numElements() > 0) {
-       |  if (!$array.isNullAt(0)) {
+       |  if ($array.isNullAt(0)) {
+       |    ${appendIfNotLegacyCastToStr(buffer, "null")}
+       |  } else {
        |    $buffer.append($elementToStringFunc(${CodeGenerator.getValue(array, et, "0")}));
        |  }
        |  for (int $loopIndex = 1; $loopIndex < $array.numElements(); $loopIndex++) {
        |    $buffer.append(",");
-       |    if (!$array.isNullAt($loopIndex)) {
+       |    if ($array.isNullAt($loopIndex)) {
+       |      ${appendIfNotLegacyCastToStr(buffer, " null")}
+       |    } else {
        |      $buffer.append(" ");
        |      $buffer.append($elementToStringFunc(${CodeGenerator.getValue(array, et, loopIndex)}));
        |    }
@@ -953,11 +1021,13 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
     val getMapKeyArray = CodeGenerator.getValue(mapKeyArray, kt, loopIndex)
     val getMapValueArray = CodeGenerator.getValue(mapValueArray, vt, loopIndex)
     code"""
-       |$buffer.append("[");
+       |$buffer.append("$leftBracket");
        |if ($map.numElements() > 0) {
        |  $buffer.append($keyToStringFunc($getMapFirstKey));
        |  $buffer.append(" ->");
-       |  if (!$map.valueArray().isNullAt(0)) {
+       |  if ($map.valueArray().isNullAt(0)) {
+       |    ${appendIfNotLegacyCastToStr(buffer, " null")}
+       |  } else {
        |    $buffer.append(" ");
        |    $buffer.append($valueToStringFunc($getMapFirstValue));
        |  }
@@ -965,13 +1035,15 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
        |    $buffer.append(", ");
        |    $buffer.append($keyToStringFunc($getMapKeyArray));
        |    $buffer.append(" ->");
-       |    if (!$map.valueArray().isNullAt($loopIndex)) {
+       |    if ($map.valueArray().isNullAt($loopIndex)) {
+       |      ${appendIfNotLegacyCastToStr(buffer, " null")}
+       |    } else {
        |      $buffer.append(" ");
        |      $buffer.append($valueToStringFunc($getMapValueArray));
        |    }
        |  }
        |}
-       |$buffer.append("]");
+       |$buffer.append("$rightBracket");
      """.stripMargin
   }
 
@@ -987,7 +1059,9 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
       val javaType = JavaCode.javaType(ft)
       code"""
          |${if (i != 0) code"""$buffer.append(",");""" else EmptyBlock}
-         |if (!$row.isNullAt($i)) {
+         |if ($row.isNullAt($i)) {
+         |  ${appendIfNotLegacyCastToStr(buffer, if (i == 0) "null" else " null")}
+         |} else {
          |  ${if (i != 0) code"""$buffer.append(" ");""" else EmptyBlock}
          |
          |  // Append $i field into the string buffer
@@ -1006,9 +1080,9 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
         (classOf[UTF8StringBuilder].getName, buffer.code) :: Nil)
 
     code"""
-       |$buffer.append("[");
+       |$buffer.append("$leftBracket");
        |$writeStructCode
-       |$buffer.append("]");
+       |$buffer.append("$rightBracket");
      """.stripMargin
   }
 
@@ -1025,8 +1099,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
         val tf = JavaCode.global(
           ctx.addReferenceObj("timestampFormatter", timestampFormatter),
           timestampFormatter.getClass)
-        (c, evPrim, evNull) => code"""$evPrim = UTF8String.fromString(
-          org.apache.spark.sql.catalyst.util.DateTimeUtils.timestampToString($tf, $c));"""
+        (c, evPrim, evNull) => code"""$evPrim = UTF8String.fromString($tf.format($c));"""
       case CalendarIntervalType =>
         (c, evPrim, _) => code"""$evPrim = UTF8String.fromString($c.toString());"""
       case ArrayType(et, _) =>
@@ -1070,6 +1143,13 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
         (c, evPrim, evNull) => {
           code"$evPrim = UTF8String.fromString($udtRef.deserialize($c).toString());"
         }
+      case i @ (YearMonthIntervalType | DayTimeIntervalType) =>
+        val iu = IntervalUtils.getClass.getName.stripSuffix("$")
+        val iss = IntervalStringStyles.getClass.getName.stripSuffix("$")
+        val subType = if (i.isInstanceOf[YearMonthIntervalType]) "YearMonth" else "DayTime"
+        val f = s"to${subType}IntervalString"
+        val style = s"$iss$$.MODULE$$.ANSI_STYLE()"
+        (c, evPrim, _) => code"""$evPrim = UTF8String.fromString($iu.$f($c, $style));"""
       case _ =>
         (c, evPrim, evNull) => code"$evPrim = UTF8String.fromString(String.valueOf($c));"
     }
@@ -1098,20 +1178,27 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
         val intOpt = ctx.freshVariable("intOpt", classOf[Option[Integer]])
         val zid = getZoneId()
         (c, evPrim, evNull) =>
-          code"""
-          scala.Option<Integer> $intOpt =
-            org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDate($c, $zid);
-          if ($intOpt.isDefined()) {
-            $evPrim = ((Integer) $intOpt.get()).intValue();
+          if (ansiEnabled) {
+            code"""
+              $evPrim = org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDateAnsi($c, $zid);
+            """
           } else {
-            $evNull = true;
+            code"""
+              scala.Option<Integer> $intOpt =
+                org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDate($c, $zid);
+              if ($intOpt.isDefined()) {
+                $evPrim = ((Integer) $intOpt.get()).intValue();
+              } else {
+                $evNull = true;
+              }
+            """
           }
-         """
+
       case TimestampType =>
         val zid = getZoneId()
         (c, evPrim, evNull) =>
           code"""$evPrim =
-            org.apache.spark.sql.catalyst.util.DateTimeUtils.microsToEpochDays($c, $zid);"""
+            org.apache.spark.sql.catalyst.util.DateTimeUtils.microsToDays($c, $zid);"""
       case _ =>
         (c, evPrim, evNull) => code"$evNull = true;"
     }
@@ -1129,8 +1216,8 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
         s"$evNull = true;"
       } else {
         s"""
-           |throw new ArithmeticException($d.toDebugString() + " cannot be represented as " +
-           | "Decimal(${decimalType.precision}, ${decimalType.scale}).");
+           |throw QueryExecutionErrors.cannotChangeDecimalPrecisionError(
+           |  $d, ${decimalType.precision}, ${decimalType.scale});
          """.stripMargin
       }
       code"""
@@ -1150,20 +1237,21 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
     val tmp = ctx.freshVariable("tmpDecimal", classOf[Decimal])
     val canNullSafeCast = Cast.canNullSafeCastToDecimal(from, target)
     from match {
-      case StringType =>
+      case StringType if !ansiEnabled =>
         (c, evPrim, evNull) =>
-          val handleException = if (ansiEnabled) {
-            s"""throw new NumberFormatException("invalid input syntax for type numeric: $c");"""
-          } else {
-            s"$evNull =true;"
-          }
           code"""
-            try {
-              Decimal $tmp = Decimal.apply(new java.math.BigDecimal($c.toString().trim()));
+              Decimal $tmp = Decimal.fromString($c);
+              if ($tmp == null) {
+                $evNull = true;
+              } else {
+                ${changePrecision(tmp, target, evPrim, evNull, canNullSafeCast)}
+              }
+          """
+      case StringType if ansiEnabled =>
+        (c, evPrim, evNull) =>
+          code"""
+              Decimal $tmp = Decimal.fromStringANSI($c);
               ${changePrecision(tmp, target, evPrim, evNull, canNullSafeCast)}
-            } catch (java.lang.NumberFormatException e) {
-              $handleException
-            }
           """
       case BooleanType =>
         (c, evPrim, evNull) =>
@@ -1218,15 +1306,22 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
         zoneIdClass)
       val longOpt = ctx.freshVariable("longOpt", classOf[Option[Long]])
       (c, evPrim, evNull) =>
-        code"""
-          scala.Option<Long> $longOpt =
-            org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToTimestamp($c, $zid);
-          if ($longOpt.isDefined()) {
-            $evPrim = ((Long) $longOpt.get()).longValue();
-          } else {
-            $evNull = true;
-          }
-         """
+        if (ansiEnabled) {
+          code"""
+            $evPrim =
+              org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToTimestampAnsi($c, $zid);
+           """
+        } else {
+          code"""
+            scala.Option<Long> $longOpt =
+              org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToTimestamp($c, $zid);
+            if ($longOpt.isDefined()) {
+              $evPrim = ((Long) $longOpt.get()).longValue();
+            } else {
+              $evNull = true;
+            }
+           """
+        }
     case BooleanType =>
       (c, evPrim, evNull) => code"$evPrim = $c ? 1L : 0L;"
     case _: IntegralType =>
@@ -1238,7 +1333,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
         zoneIdClass)
       (c, evPrim, evNull) =>
         code"""$evPrim =
-          org.apache.spark.sql.catalyst.util.DateTimeUtils.epochDaysToMicros($c, $zid);"""
+          org.apache.spark.sql.catalyst.util.DateTimeUtils.daysToMicros($c, $zid);"""
     case DecimalType() =>
       (c, evPrim, evNull) => code"$evPrim = ${decimalToTimestampCode(c)};"
     case DoubleType =>
@@ -1256,7 +1351,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
           if (Float.isNaN($c) || Float.isInfinite($c)) {
             $evNull = true;
           } else {
-            $evPrim = (long)($c * $MICROS_PER_SECOND);
+            $evPrim = (long)((double)$c * $MICROS_PER_SECOND);
           }
         """
   }
@@ -1273,6 +1368,18 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
 
   }
 
+  private[this] def castToDayTimeIntervalCode(from: DataType): CastFunction = from match {
+    case StringType =>
+      val util = IntervalUtils.getClass.getCanonicalName.stripSuffix("$")
+      (c, evPrim, _) => code"$evPrim = $util.castStringToDTInterval($c);"
+  }
+
+  private[this] def castToYearMonthIntervalCode(from: DataType): CastFunction = from match {
+    case StringType =>
+      val util = IntervalUtils.getClass.getCanonicalName.stripSuffix("$")
+      (c, evPrim, _) => code"$evPrim = $util.castStringToYMInterval($c);"
+  }
+
   private[this] def decimalToTimestampCode(d: ExprValue): Block = {
     val block = inline"new java.math.BigDecimal($MICROS_PER_SECOND)"
     code"($d.toBigDecimal().bigDecimal().multiply($block)).longValue()"
@@ -1287,13 +1394,19 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
     case StringType =>
       val stringUtils = inline"${StringUtils.getClass.getName.stripSuffix("$")}"
       (c, evPrim, evNull) =>
+        val castFailureCode = if (ansiEnabled) {
+          val errorMessage = s""""invalid input syntax for type boolean: " + $c"""
+          s"throw new java.lang.UnsupportedOperationException($errorMessage);"
+        } else {
+          s"$evNull = true;"
+        }
         code"""
           if ($stringUtils.isTrueString($c)) {
             $evPrim = true;
           } else if ($stringUtils.isFalseString($c)) {
             $evPrim = false;
           } else {
-            $evNull = true;
+            $castFailureCode
           }
         """
     case TimestampType =>
@@ -1309,7 +1422,8 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
 
   private[this] def castTimestampToIntegralTypeCode(
       ctx: CodegenContext,
-      integralType: String): CastFunction = {
+      integralType: String,
+      catalogType: String): CastFunction = {
     if (ansiEnabled) {
       val longValue = ctx.freshName("longValue")
       (c, evPrim, evNull) =>
@@ -1318,7 +1432,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
           if ($longValue == ($integralType) $longValue) {
             $evPrim = ($integralType) $longValue;
           } else {
-            throw new ArithmeticException("Casting " + $c + " to $integralType causes overflow");
+            throw QueryExecutionErrors.castingCauseOverflowError($c, "$catalogType");
           }
         """
     } else {
@@ -1328,7 +1442,8 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
 
   private[this] def castDecimalToIntegralTypeCode(
       ctx: CodegenContext,
-      integralType: String): CastFunction = {
+      integralType: String,
+      catalogType: String): CastFunction = {
     if (ansiEnabled) {
       (c, evPrim, evNull) => code"$evPrim = $c.roundTo${integralType.capitalize}();"
     } else {
@@ -1336,37 +1451,35 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
     }
   }
 
-  private[this] def castIntegralTypeToIntegralTypeExactCode(integralType: String): CastFunction = {
+  private[this] def castIntegralTypeToIntegralTypeExactCode(
+      integralType: String,
+      catalogType: String): CastFunction = {
     assert(ansiEnabled)
     (c, evPrim, evNull) =>
       code"""
         if ($c == ($integralType) $c) {
           $evPrim = ($integralType) $c;
         } else {
-          throw new ArithmeticException("Casting " + $c + " to $integralType causes overflow");
+          throw QueryExecutionErrors.castingCauseOverflowError($c, "$catalogType");
         }
       """
   }
 
-  private[this] def lowerAndUpperBound(
-      fractionType: String,
-      integralType: String): (String, String) = {
-    assert(fractionType == "float" || fractionType == "double")
-    val typeIndicator = fractionType.charAt(0)
-    val (min, max) = integralType.toLowerCase(Locale.ROOT) match {
-      case "long" => (Long.MinValue, Long.MaxValue)
-      case "int" => (Int.MinValue, Int.MaxValue)
-      case "short" => (Short.MinValue, Short.MaxValue)
-      case "byte" => (Byte.MinValue, Byte.MaxValue)
+  private[this] def lowerAndUpperBound(integralType: String): (String, String) = {
+    val (min, max, typeIndicator) = integralType.toLowerCase(Locale.ROOT) match {
+      case "long" => (Long.MinValue, Long.MaxValue, "L")
+      case "int" => (Int.MinValue, Int.MaxValue, "")
+      case "short" => (Short.MinValue, Short.MaxValue, "")
+      case "byte" => (Byte.MinValue, Byte.MaxValue, "")
     }
     (min.toString + typeIndicator, max.toString + typeIndicator)
   }
 
   private[this] def castFractionToIntegralTypeCode(
-      fractionType: String,
-      integralType: String): CastFunction = {
+      integralType: String,
+      catalogType: String): CastFunction = {
     assert(ansiEnabled)
-    val (min, max) = lowerAndUpperBound(fractionType, integralType)
+    val (min, max) = lowerAndUpperBound(integralType)
     val mathClass = classOf[Math].getName
     // When casting floating values to integral types, Spark uses the method `Numeric.toInt`
     // Or `Numeric.toLong` directly. For positive floating values, it is equivalent to `Math.floor`;
@@ -1378,7 +1491,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
         if ($mathClass.floor($c) <= $max && $mathClass.ceil($c) >= $min) {
           $evPrim = ($integralType) $c;
         } else {
-          throw new ArithmeticException("Casting " + $c + " to $integralType causes overflow");
+          throw QueryExecutionErrors.castingCauseOverflowError($c, "$catalogType");
         }
       """
   }
@@ -1402,14 +1515,12 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
       (c, evPrim, evNull) => code"$evPrim = $c ? (byte) 1 : (byte) 0;"
     case DateType =>
       (c, evPrim, evNull) => code"$evNull = true;"
-    case TimestampType => castTimestampToIntegralTypeCode(ctx, "byte")
-    case DecimalType() => castDecimalToIntegralTypeCode(ctx, "byte")
-    case _: ShortType | _: IntegerType | _: LongType if ansiEnabled =>
-      castIntegralTypeToIntegralTypeExactCode("byte")
-    case _: FloatType if ansiEnabled =>
-      castFractionToIntegralTypeCode("float", "byte")
-    case _: DoubleType if ansiEnabled =>
-      castFractionToIntegralTypeCode("double", "byte")
+    case TimestampType => castTimestampToIntegralTypeCode(ctx, "byte", ByteType.catalogString)
+    case DecimalType() => castDecimalToIntegralTypeCode(ctx, "byte", ByteType.catalogString)
+    case ShortType | IntegerType | LongType if ansiEnabled =>
+      castIntegralTypeToIntegralTypeExactCode("byte", ByteType.catalogString)
+    case FloatType | DoubleType if ansiEnabled =>
+      castFractionToIntegralTypeCode("byte", ByteType.catalogString)
     case x: NumericType =>
       (c, evPrim, evNull) => code"$evPrim = (byte) $c;"
   }
@@ -1435,14 +1546,12 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
       (c, evPrim, evNull) => code"$evPrim = $c ? (short) 1 : (short) 0;"
     case DateType =>
       (c, evPrim, evNull) => code"$evNull = true;"
-    case TimestampType => castTimestampToIntegralTypeCode(ctx, "short")
-    case DecimalType() => castDecimalToIntegralTypeCode(ctx, "short")
-    case _: IntegerType | _: LongType if ansiEnabled =>
-      castIntegralTypeToIntegralTypeExactCode("short")
-    case _: FloatType if ansiEnabled =>
-      castFractionToIntegralTypeCode("float", "short")
-    case _: DoubleType if ansiEnabled =>
-      castFractionToIntegralTypeCode("double", "short")
+    case TimestampType => castTimestampToIntegralTypeCode(ctx, "short", ShortType.catalogString)
+    case DecimalType() => castDecimalToIntegralTypeCode(ctx, "short", ShortType.catalogString)
+    case IntegerType | LongType if ansiEnabled =>
+      castIntegralTypeToIntegralTypeExactCode("short", ShortType.catalogString)
+    case FloatType | DoubleType if ansiEnabled =>
+      castFractionToIntegralTypeCode("short", ShortType.catalogString)
     case x: NumericType =>
       (c, evPrim, evNull) => code"$evPrim = (short) $c;"
   }
@@ -1466,13 +1575,12 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
       (c, evPrim, evNull) => code"$evPrim = $c ? 1 : 0;"
     case DateType =>
       (c, evPrim, evNull) => code"$evNull = true;"
-    case TimestampType => castTimestampToIntegralTypeCode(ctx, "int")
-    case DecimalType() => castDecimalToIntegralTypeCode(ctx, "int")
-    case _: LongType if ansiEnabled => castIntegralTypeToIntegralTypeExactCode("int")
-    case _: FloatType if ansiEnabled =>
-      castFractionToIntegralTypeCode("float", "int")
-    case _: DoubleType if ansiEnabled =>
-      castFractionToIntegralTypeCode("double", "int")
+    case TimestampType => castTimestampToIntegralTypeCode(ctx, "int", IntegerType.catalogString)
+    case DecimalType() => castDecimalToIntegralTypeCode(ctx, "int", IntegerType.catalogString)
+    case LongType if ansiEnabled =>
+      castIntegralTypeToIntegralTypeExactCode("int", IntegerType.catalogString)
+    case FloatType | DoubleType if ansiEnabled =>
+      castFractionToIntegralTypeCode("int", IntegerType.catalogString)
     case x: NumericType =>
       (c, evPrim, evNull) => code"$evPrim = (int) $c;"
   }
@@ -1498,11 +1606,9 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
       (c, evPrim, evNull) => code"$evNull = true;"
     case TimestampType =>
       (c, evPrim, evNull) => code"$evPrim = (long) ${timestampToLongCode(c)};"
-    case DecimalType() => castDecimalToIntegralTypeCode(ctx, "long")
-    case _: FloatType if ansiEnabled =>
-      castFractionToIntegralTypeCode("float", "long")
-    case _: DoubleType if ansiEnabled =>
-      castFractionToIntegralTypeCode("double", "long")
+    case DecimalType() => castDecimalToIntegralTypeCode(ctx, "long", LongType.catalogString)
+    case FloatType | DoubleType if ansiEnabled =>
+      castFractionToIntegralTypeCode("long", LongType.catalogString)
     case x: NumericType =>
       (c, evPrim, evNull) => code"$evPrim = (long) $c;"
   }
@@ -1513,7 +1619,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
         val floatStr = ctx.freshVariable("floatStr", StringType)
         (c, evPrim, evNull) =>
           val handleNull = if (ansiEnabled) {
-            s"""throw new NumberFormatException("invalid input syntax for type numeric: $c");"""
+            s"throw QueryExecutionErrors.invalidInputSyntaxForNumericError($c);"
           } else {
             s"$evNull = true;"
           }
@@ -1549,7 +1655,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
         val doubleStr = ctx.freshVariable("doubleStr", StringType)
         (c, evPrim, evNull) =>
           val handleNull = if (ansiEnabled) {
-            s"""throw new NumberFormatException("invalid input syntax for type numeric: $c");"""
+            s"throw QueryExecutionErrors.invalidInputSyntaxForNumericError($c);"
           } else {
             s"$evNull = true;"
           }
@@ -1712,14 +1818,33 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
     Examples:
       > SELECT _FUNC_('10' as int);
        10
-  """)
+  """,
+  since = "1.0.0",
+  group = "conversion_funcs")
 case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String] = None)
   extends CastBase {
 
   override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
     copy(timeZoneId = Option(timeZoneId))
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(CAST)
+
   override protected val ansiEnabled: Boolean = SQLConf.get.ansiEnabled
+
+  override def canCast(from: DataType, to: DataType): Boolean = if (ansiEnabled) {
+    AnsiCast.canCast(from, to)
+  } else {
+    Cast.canCast(from, to)
+  }
+
+  override def typeCheckFailureMessage: String = if (ansiEnabled) {
+    AnsiCast.typeCheckFailureMessage(child.dataType, dataType,
+      Some(SQLConf.ANSI_ENABLED.key), Some("false"))
+  } else {
+    s"cannot cast ${child.dataType.catalogString} to ${dataType.catalogString}"
+  }
+
+  override protected def withNewChildInternal(newChild: Expression): Cast = copy(child = newChild)
 }
 
 /**
@@ -1737,13 +1862,179 @@ case class AnsiCast(child: Expression, dataType: DataType, timeZoneId: Option[St
     copy(timeZoneId = Option(timeZoneId))
 
   override protected val ansiEnabled: Boolean = true
+
+  override def canCast(from: DataType, to: DataType): Boolean = AnsiCast.canCast(from, to)
+
+  // For now, this expression is only used in table insertion.
+  // If there are more scenarios for this expression, we should update the error message on type
+  // check failure.
+  override def typeCheckFailureMessage: String =
+    AnsiCast.typeCheckFailureMessage(child.dataType,
+      dataType,
+      Some(SQLConf.STORE_ASSIGNMENT_POLICY.key),
+      Some(SQLConf.StoreAssignmentPolicy.LEGACY.toString))
+
+  override protected def withNewChildInternal(newChild: Expression): AnsiCast =
+    copy(child = newChild)
+}
+
+object AnsiCast {
+  /**
+   * As per section 6.13 "cast specification" in "Information technology — Database languages " +
+   * "- SQL — Part 2: Foundation (SQL/Foundation)":
+   * If the <cast operand> is a <value expression>, then the valid combinations of TD and SD
+   * in a <cast specification> are given by the following table. “Y” indicates that the
+   * combination is syntactically valid without restriction; “M” indicates that the combination
+   * is valid subject to other Syntax Rules in this Sub- clause being satisfied; and “N” indicates
+   * that the combination is not valid:
+   * SD                   TD
+   *     EN AN C D T TS YM DT BO UDT B RT CT RW
+   * EN  Y  Y  Y N N  N  M  M  N   M N  M  N N
+   * AN  Y  Y  Y N N  N  N  N  N   M N  M  N N
+   * C   Y  Y  Y Y Y  Y  Y  Y  Y   M N  M  N N
+   * D   N  N  Y Y N  Y  N  N  N   M N  M  N N
+   * T   N  N  Y N Y  Y  N  N  N   M N  M  N N
+   * TS  N  N  Y Y Y  Y  N  N  N   M N  M  N N
+   * YM  M  N  Y N N  N  Y  N  N   M N  M  N N
+   * DT  M  N  Y N N  N  N  Y  N   M N  M  N N
+   * BO  N  N  Y N N  N  N  N  Y   M N  M  N N
+   * UDT M  M  M M M  M  M  M  M   M M  M  M N
+   * B   N  N  N N N  N  N  N  N   M Y  M  N N
+   * RT  M  M  M M M  M  M  M  M   M M  M  N N
+   * CT  N  N  N N N  N  N  N  N   M N  N  M N
+   * RW  N  N  N N N  N  N  N  N   N N  N  N M
+   *
+   * Where:
+   *   EN  = Exact Numeric
+   *   AN  = Approximate Numeric
+   *   C   = Character (Fixed- or Variable-Length, or Character Large Object)
+   *   D   = Date
+   *   T   = Time
+   *   TS  = Timestamp
+   *   YM  = Year-Month Interval
+   *   DT  = Day-Time Interval
+   *   BO  = Boolean
+   *   UDT  = User-Defined Type
+   *   B   = Binary (Fixed- or Variable-Length or Binary Large Object)
+   *   RT  = Reference type
+   *   CT  = Collection type
+   *   RW  = Row type
+   *
+   * Spark's ANSI mode follows the syntax rules, except it specially allow the following
+   * straightforward type conversions which are disallowed as per the SQL standard:
+   *   - Numeric <=> Boolean
+   *   - String <=> Binary
+   */
+  def canCast(from: DataType, to: DataType): Boolean = (from, to) match {
+    case (fromType, toType) if fromType == toType => true
+
+    case (NullType, _) => true
+
+    case (_, StringType) => true
+
+    case (StringType, _: BinaryType) => true
+
+    case (StringType, BooleanType) => true
+    case (_: NumericType, BooleanType) => true
+
+    case (StringType, TimestampType) => true
+    case (DateType, TimestampType) => true
+
+    case (StringType, _: CalendarIntervalType) => true
+    case (StringType, DayTimeIntervalType) => true
+    case (StringType, YearMonthIntervalType) => true
+
+    case (StringType, DateType) => true
+    case (TimestampType, DateType) => true
+
+    case (_: NumericType, _: NumericType) => true
+    case (StringType, _: NumericType) => true
+    case (BooleanType, _: NumericType) => true
+
+    case (ArrayType(fromType, fn), ArrayType(toType, tn)) =>
+      canCast(fromType, toType) &&
+        resolvableNullability(fn || forceNullable(fromType, toType), tn)
+
+    case (MapType(fromKey, fromValue, fn), MapType(toKey, toValue, tn)) =>
+      canCast(fromKey, toKey) &&
+        (!forceNullable(fromKey, toKey)) &&
+        canCast(fromValue, toValue) &&
+        resolvableNullability(fn || forceNullable(fromValue, toValue), tn)
+
+    case (StructType(fromFields), StructType(toFields)) =>
+      fromFields.length == toFields.length &&
+        fromFields.zip(toFields).forall {
+          case (fromField, toField) =>
+            canCast(fromField.dataType, toField.dataType) &&
+              resolvableNullability(
+                fromField.nullable || forceNullable(fromField.dataType, toField.dataType),
+                toField.nullable)
+        }
+
+    case (udt1: UserDefinedType[_], udt2: UserDefinedType[_]) if udt2.acceptsType(udt1) => true
+
+    case _ => false
+  }
+
+  // Show suggestion on how to complete the disallowed explicit casting with built-in type
+  // conversion functions.
+  private def suggestionOnConversionFunctions (
+      from: DataType,
+      to: DataType,
+      functionNames: String): String = {
+    // scalastyle:off line.size.limit
+    s"""cannot cast ${from.catalogString} to ${to.catalogString}.
+       |To convert values from ${from.catalogString} to ${to.catalogString}, you can use $functionNames instead.
+       |""".stripMargin
+    // scalastyle:on line.size.limit
+  }
+
+  def typeCheckFailureMessage(
+      from: DataType,
+      to: DataType,
+      fallbackConfKey: Option[String],
+      fallbackConfValue: Option[String]): String =
+    (from, to) match {
+      case (_: NumericType, TimestampType) =>
+        suggestionOnConversionFunctions(from, to,
+          "functions TIMESTAMP_SECONDS/TIMESTAMP_MILLIS/TIMESTAMP_MICROS")
+
+      case (TimestampType, _: NumericType) =>
+        suggestionOnConversionFunctions(from, to, "functions UNIX_SECONDS/UNIX_MILLIS/UNIX_MICROS")
+
+      case (_: NumericType, DateType) =>
+        suggestionOnConversionFunctions(from, to, "function DATE_FROM_UNIX_DATE")
+
+      case (DateType, _: NumericType) =>
+        suggestionOnConversionFunctions(from, to, "function UNIX_DATE")
+
+      // scalastyle:off line.size.limit
+      case _ if fallbackConfKey.isDefined && fallbackConfValue.isDefined && Cast.canCast(from, to) =>
+        s"""
+           | cannot cast ${from.catalogString} to ${to.catalogString} with ANSI mode on.
+           | If you have to cast ${from.catalogString} to ${to.catalogString}, you can set ${fallbackConfKey.get} as ${fallbackConfValue.get}.
+           |""".stripMargin
+      // scalastyle:on line.size.limit
+
+      case _ => s"cannot cast ${from.catalogString} to ${to.catalogString}"
+    }
 }
 
 /**
  * Cast the child expression to the target data type, but will throw error if the cast might
  * truncate, e.g. long -> int, timestamp -> data.
+ *
+ * Note: `target` is `AbstractDataType`, so that we can put `object DecimalType`, which means
+ * we accept `DecimalType` with any valid precision/scale.
  */
-case class UpCast(child: Expression, dataType: DataType, walkedTypePath: Seq[String] = Nil)
+case class UpCast(child: Expression, target: AbstractDataType, walkedTypePath: Seq[String] = Nil)
   extends UnaryExpression with Unevaluable {
   override lazy val resolved = false
+
+  def dataType: DataType = target match {
+    case DecimalType => DecimalType.SYSTEM_DEFAULT
+    case _ => target.asInstanceOf[DataType]
+  }
+
+  override protected def withNewChildInternal(newChild: Expression): UpCast = copy(child = newChild)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CodeGeneratorWithInterpretedFallback.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CodeGeneratorWithInterpretedFallback.scala
index 07fa813a98922..3b7219477bd69 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CodeGeneratorWithInterpretedFallback.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CodeGeneratorWithInterpretedFallback.scala
@@ -51,9 +51,8 @@ abstract class CodeGeneratorWithInterpretedFallback[IN, OUT] extends Logging {
         try {
           createCodeGeneratedObject(in)
         } catch {
-          case NonFatal(_) =>
-            // We should have already seen the error message in `CodeGenerator`
-            logWarning("Expr codegen error and falling back to interpreter mode")
+          case NonFatal(e) =>
+            logWarning("Expr codegen error and falling back to interpreter mode", e)
             createInterpretedObject(in)
         }
     }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/DynamicPruning.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/DynamicPruning.scala
index 7065d27517e52..1c185dd316b5b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/DynamicPruning.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/DynamicPruning.scala
@@ -20,6 +20,8 @@ package org.apache.spark.sql.catalyst.expressions
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.trees.TreePattern.{DYNAMIC_PRUNING_SUBQUERY, TreePattern}
+import org.apache.spark.sql.catalyst.trees.UnaryLike
 
 trait DynamicPruning extends Predicate
 
@@ -46,9 +48,10 @@ case class DynamicPruningSubquery(
     exprId: ExprId = NamedExpression.newExprId)
   extends SubqueryExpression(buildQuery, Seq(pruningKey), exprId)
   with DynamicPruning
-  with Unevaluable {
+  with Unevaluable
+  with UnaryLike[Expression] {
 
-  override def children: Seq[Expression] = Seq(pruningKey)
+  override def child: Expression = pruningKey
 
   override def plan: LogicalPlan = buildQuery
 
@@ -67,6 +70,8 @@ case class DynamicPruningSubquery(
       pruningKey.dataType == buildKeys(broadcastKeyIndex).dataType
   }
 
+  final override def nodePatternsInternal: Seq[TreePattern] = Seq(DYNAMIC_PRUNING_SUBQUERY)
+
   override def toString: String = s"dynamicpruning#${exprId.id} $conditionString"
 
   override lazy val canonicalized: DynamicPruning = {
@@ -76,6 +81,9 @@ case class DynamicPruningSubquery(
       buildKeys = buildKeys.map(_.canonicalized),
       exprId = ExprId(0))
   }
+
+  override protected def withNewChildInternal(newChild: Expression): DynamicPruningSubquery =
+    copy(pruningKey = newChild)
 }
 
 /**
@@ -92,4 +100,7 @@ case class DynamicPruningExpression(child: Expression)
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     child.genCode(ctx)
   }
+
+  override protected def withNewChildInternal(newChild: Expression): DynamicPruningExpression =
+    copy(child = newChild)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/EquivalentExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/EquivalentExpressions.scala
index a32052ce121df..1dfff412d9a8e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/EquivalentExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/EquivalentExpressions.scala
@@ -65,11 +65,82 @@ class EquivalentExpressions {
     }
   }
 
+  private def addExprToSet(expr: Expression, set: mutable.Set[Expr]): Boolean = {
+    if (expr.deterministic) {
+      val e = Expr(expr)
+      if (set.contains(e)) {
+        true
+      } else {
+        set.add(e)
+        false
+      }
+    } else {
+      false
+    }
+  }
+
+  /**
+   * Adds only expressions which are common in each of given expressions, in a recursive way.
+   * For example, given two expressions `(a + (b + (c + 1)))` and `(d + (e + (c + 1)))`,
+   * the common expression `(c + 1)` will be added into `equivalenceMap`.
+   */
+  private def addCommonExprs(
+      exprs: Seq[Expression],
+      addFunc: Expression => Boolean = addExpr): Unit = {
+    val exprSetForAll = mutable.Set[Expr]()
+    addExprTree(exprs.head, addExprToSet(_, exprSetForAll))
+
+    val commonExprSet = exprs.tail.foldLeft(exprSetForAll) { (exprSet, expr) =>
+      val otherExprSet = mutable.Set[Expr]()
+      addExprTree(expr, addExprToSet(_, otherExprSet))
+      exprSet.intersect(otherExprSet)
+    }
+
+    commonExprSet.foreach(expr => addFunc(expr.e))
+  }
+
+  // There are some special expressions that we should not recurse into all of its children.
+  //   1. CodegenFallback: it's children will not be used to generate code (call eval() instead)
+  //   2. If: common subexpressions will always be evaluated at the beginning, but the true and
+  //          false expressions in `If` may not get accessed, according to the predicate
+  //          expression. We should only recurse into the predicate expression.
+  //   3. CaseWhen: like `If`, the children of `CaseWhen` only get accessed in a certain
+  //                condition. We should only recurse into the first condition expression as it
+  //                will always get accessed.
+  //   4. Coalesce: it's also a conditional expression, we should only recurse into the first
+  //                children, because others may not get accessed.
+  private def childrenToRecurse(expr: Expression): Seq[Expression] = expr match {
+    case _: CodegenFallback => Nil
+    case i: If => i.predicate :: Nil
+    case c: CaseWhen => c.children.head :: Nil
+    case c: Coalesce => c.children.head :: Nil
+    case other => other.children
+  }
+
+  // For some special expressions we cannot just recurse into all of its children, but we can
+  // recursively add the common expressions shared between all of its children.
+  private def commonChildrenToRecurse(expr: Expression): Seq[Seq[Expression]] = expr match {
+    case i: If => Seq(Seq(i.trueValue, i.falseValue))
+    case c: CaseWhen =>
+      // We look at subexpressions in conditions and values of `CaseWhen` separately. It is
+      // because a subexpression in conditions will be run no matter which condition is matched
+      // if it is shared among conditions, but it doesn't need to be shared in values. Similarly,
+      // a subexpression among values doesn't need to be in conditions because no matter which
+      // condition is true, it will be evaluated.
+      val conditions = c.branches.tail.map(_._1)
+      val values = c.branches.map(_._2) ++ c.elseValue
+      Seq(conditions, values)
+    case c: Coalesce => Seq(c.children.tail)
+    case _ => Nil
+  }
+
   /**
    * Adds the expression to this data structure recursively. Stops if a matching expression
    * is found. That is, if `expr` has already been added, its children are not added.
    */
-  def addExprTree(expr: Expression): Unit = {
+  def addExprTree(
+      expr: Expression,
+      addFunc: Expression => Boolean = addExpr): Unit = {
     val skip = expr.isInstanceOf[LeafExpression] ||
       // `LambdaVariable` is usually used as a loop variable, which can't be evaluated ahead of the
       // loop. So we can't evaluate sub-expressions containing `LambdaVariable` at the beginning.
@@ -78,26 +149,9 @@ class EquivalentExpressions {
       // can cause error like NPE.
       (expr.isInstanceOf[PlanExpression[_]] && TaskContext.get != null)
 
-    // There are some special expressions that we should not recurse into all of its children.
-    //   1. CodegenFallback: it's children will not be used to generate code (call eval() instead)
-    //   2. If: common subexpressions will always be evaluated at the beginning, but the true and
-    //          false expressions in `If` may not get accessed, according to the predicate
-    //          expression. We should only recurse into the predicate expression.
-    //   3. CaseWhen: like `If`, the children of `CaseWhen` only get accessed in a certain
-    //                condition. We should only recurse into the first condition expression as it
-    //                will always get accessed.
-    //   4. Coalesce: it's also a conditional expression, we should only recurse into the first
-    //                children, because others may not get accessed.
-    def childrenToRecurse: Seq[Expression] = expr match {
-      case _: CodegenFallback => Nil
-      case i: If => i.predicate :: Nil
-      case c: CaseWhen => c.children.head :: Nil
-      case c: Coalesce => c.children.head :: Nil
-      case other => other.children
-    }
-
-    if (!skip && !addExpr(expr)) {
-      childrenToRecurse.foreach(addExprTree)
+    if (!skip && !addFunc(expr)) {
+      childrenToRecurse(expr).foreach(addExprTree(_, addFunc))
+      commonChildrenToRecurse(expr).filter(_.nonEmpty).foreach(addCommonExprs(_, addFunc))
     }
   }
 
@@ -106,7 +160,7 @@ class EquivalentExpressions {
    * an empty collection if there are none.
    */
   def getEquivalentExprs(e: Expression): Seq[Expression] = {
-    equivalenceMap.getOrElse(Expr(e), Seq.empty)
+    equivalenceMap.getOrElse(Expr(e), Seq.empty).toSeq
   }
 
   /**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExprUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExprUtils.scala
index 56bd3d7026d52..4f508022855e9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExprUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExprUtils.scala
@@ -20,8 +20,8 @@ package org.apache.spark.sql.catalyst.expressions
 import java.text.{DecimalFormat, DecimalFormatSymbols, ParsePosition}
 import java.util.Locale
 
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.util.ArrayBasedMapData
+import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, CharVarcharUtils}
+import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
 import org.apache.spark.sql.types.{DataType, MapType, StringType, StructType}
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -30,22 +30,21 @@ object ExprUtils {
   def evalTypeExpr(exp: Expression): DataType = {
     if (exp.foldable) {
       exp.eval() match {
-        case s: UTF8String if s != null => DataType.fromDDL(s.toString)
-        case _ => throw new AnalysisException(
-          s"The expression '${exp.sql}' is not a valid schema string.")
+        case s: UTF8String if s != null =>
+          val dataType = DataType.fromDDL(s.toString)
+          CharVarcharUtils.failIfHasCharVarchar(dataType)
+        case _ => throw QueryCompilationErrors.invalidSchemaStringError(exp)
+
       }
     } else {
-      throw new AnalysisException(
-        "Schema should be specified in DDL format as a string literal or output of " +
-          s"the schema_of_json/schema_of_csv functions instead of ${exp.sql}")
+      throw QueryCompilationErrors.schemaNotFoldableError(exp)
     }
   }
 
   def evalSchemaExpr(exp: Expression): StructType = {
     val dataType = evalTypeExpr(exp)
     if (!dataType.isInstanceOf[StructType]) {
-      throw new AnalysisException(
-        s"Schema should be struct type but got ${dataType.sql}.")
+      throw QueryCompilationErrors.schemaIsNotStructTypeError(dataType)
     }
     dataType.asInstanceOf[StructType]
   }
@@ -58,10 +57,9 @@ object ExprUtils {
         key.toString -> value.toString
       }
     case m: CreateMap =>
-      throw new AnalysisException(
-        s"A type of keys and values in map() must be string, but got ${m.dataType.catalogString}")
+      throw QueryCompilationErrors.keyValueInMapNotStringError(m)
     case _ =>
-      throw new AnalysisException("Must use a map() function for options")
+      throw QueryCompilationErrors.nonMapFunctionNotAllowedError
   }
 
   /**
@@ -74,8 +72,7 @@ object ExprUtils {
     schema.getFieldIndex(columnNameOfCorruptRecord).foreach { corruptFieldIndex =>
       val f = schema(corruptFieldIndex)
       if (f.dataType != StringType || !f.nullable) {
-        throw new AnalysisException(
-          "The field for corrupt records must be string type and nullable")
+        throw QueryCompilationErrors.invalidFieldTypeForCorruptRecordError
       }
     }
   }
@@ -90,7 +87,7 @@ object ExprUtils {
         val pos = new ParsePosition(0)
         val result = decimalFormat.parse(s, pos).asInstanceOf[java.math.BigDecimal]
         if (pos.getIndex() != s.length() || pos.getErrorIndex() != -1) {
-          throw new IllegalArgumentException("Cannot parse any decimal");
+          throw QueryExecutionErrors.cannotParseDecimalError
         } else {
           result
         }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
index 18cc648e57d71..002ffcb291458 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
@@ -24,10 +24,10 @@ import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult
 import org.apache.spark.sql.catalyst.expressions.aggregate.DeclarativeAggregate
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.trees.TreeNode
-import org.apache.spark.sql.catalyst.util.toPrettySQL
+import org.apache.spark.sql.catalyst.trees.{BinaryLike, LeafLike, QuaternaryLike, TernaryLike, TreeNode, UnaryLike}
+import org.apache.spark.sql.catalyst.trees.TreePattern.{RUNTIME_REPLACEABLE, TreePattern}
 import org.apache.spark.sql.catalyst.util.truncatedString
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
@@ -63,7 +63,8 @@ import org.apache.spark.sql.types._
  *                            functions.
  * - [[NamedExpression]]: An [[Expression]] that is named.
  * - [[TimeZoneAwareExpression]]: A common base trait for time zone aware expressions.
- * - [[SubqueryExpression]]: A base interface for expressions that contain a [[LogicalPlan]].
+ * - [[SubqueryExpression]]: A base interface for expressions that contain a
+ *                           [[org.apache.spark.sql.catalyst.plans.logical.LogicalPlan]].
  *
  * - [[LeafExpression]]: an expression that has no child.
  * - [[UnaryExpression]]: an expression that has one child.
@@ -286,7 +287,7 @@ abstract class Expression extends TreeNode[Expression] {
   }
 
   override def simpleStringWithNodeId(): String = {
-    throw new UnsupportedOperationException(s"$nodeName does not implement simpleStringWithNodeId")
+    throw QueryExecutionErrors.simpleStringWithNodeIdUnsupportedError(nodeName)
   }
 }
 
@@ -298,11 +299,14 @@ abstract class Expression extends TreeNode[Expression] {
  */
 trait Unevaluable extends Expression {
 
+  /** Unevaluable is not foldable because we don't have an eval for it. */
+  final override def foldable: Boolean = false
+
   final override def eval(input: InternalRow = null): Any =
-    throw new UnsupportedOperationException(s"Cannot evaluate expression: $this")
+    throw QueryExecutionErrors.cannotEvaluateExpressionError(this)
 
   final override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode =
-    throw new UnsupportedOperationException(s"Cannot generate code for expression: $this")
+    throw QueryExecutionErrors.cannotGenerateCodeForExpressionError(this)
 }
 
 
@@ -318,7 +322,6 @@ trait Unevaluable extends Expression {
  */
 trait RuntimeReplaceable extends UnaryExpression with Unevaluable {
   override def nullable: Boolean = child.nullable
-  override def foldable: Boolean = child.foldable
   override def dataType: DataType = child.dataType
   // As this expression gets replaced at optimization with its `child" expression,
   // two `RuntimeReplaceable` are considered to be semantically equal if their "child" expressions
@@ -334,6 +337,8 @@ trait RuntimeReplaceable extends UnaryExpression with Unevaluable {
 
   override def sql: String = mkString(exprsReplaced.map(_.sql))
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(RUNTIME_REPLACEABLE)
+
   def mkString(childrenString: Seq[String]): String = {
     prettyName + childrenString.mkString("(", ", ", ")")
   }
@@ -350,19 +355,24 @@ trait UnevaluableAggregate extends DeclarativeAggregate {
   override def nullable: Boolean = true
 
   override lazy val aggBufferAttributes =
-    throw new UnsupportedOperationException(s"Cannot evaluate aggBufferAttributes: $this")
+    throw QueryExecutionErrors.evaluateUnevaluableAggregateUnsupportedError(
+      "aggBufferAttributes", this)
 
   override lazy val initialValues: Seq[Expression] =
-    throw new UnsupportedOperationException(s"Cannot evaluate initialValues: $this")
+    throw QueryExecutionErrors.evaluateUnevaluableAggregateUnsupportedError(
+      "initialValues", this)
 
   override lazy val updateExpressions: Seq[Expression] =
-    throw new UnsupportedOperationException(s"Cannot evaluate updateExpressions: $this")
+    throw QueryExecutionErrors.evaluateUnevaluableAggregateUnsupportedError(
+      "updateExpressions", this)
 
   override lazy val mergeExpressions: Seq[Expression] =
-    throw new UnsupportedOperationException(s"Cannot evaluate mergeExpressions: $this")
+    throw QueryExecutionErrors.evaluateUnevaluableAggregateUnsupportedError(
+      "mergeExpressions", this)
 
   override lazy val evaluateExpression: Expression =
-    throw new UnsupportedOperationException(s"Cannot evaluate evaluateExpression: $this")
+    throw QueryExecutionErrors.evaluateUnevaluableAggregateUnsupportedError(
+      "evaluateExpression", this)
 }
 
 /**
@@ -444,21 +454,14 @@ trait Stateful extends Nondeterministic {
 /**
  * A leaf expression, i.e. one without any child expressions.
  */
-abstract class LeafExpression extends Expression {
-
-  override final def children: Seq[Expression] = Nil
-}
+abstract class LeafExpression extends Expression with LeafLike[Expression]
 
 
 /**
  * An expression with one input and one output. The output is by default evaluated to null
  * if the input is evaluated to null.
  */
-abstract class UnaryExpression extends Expression {
-
-  def child: Expression
-
-  override final def children: Seq[Expression] = child :: Nil
+abstract class UnaryExpression extends Expression with UnaryLike[Expression] {
 
   override def foldable: Boolean = child.foldable
   override def nullable: Boolean = child.nullable
@@ -482,7 +485,8 @@ abstract class UnaryExpression extends Expression {
    * of evaluation process, we should override [[eval]].
    */
   protected def nullSafeEval(input: Any): Any =
-    sys.error(s"UnaryExpressions must override either eval or nullSafeEval")
+    throw QueryExecutionErrors.notOverrideExpectedMethodsError("UnaryExpressions",
+      "eval", "nullSafeEval")
 
   /**
    * Called by unary expressions to generate a code block that returns null if its parent returns
@@ -535,16 +539,17 @@ abstract class UnaryExpression extends Expression {
   }
 }
 
+
+object UnaryExpression {
+  def unapply(e: UnaryExpression): Option[Expression] = Some(e.child)
+}
+
+
 /**
  * An expression with two inputs and one output. The output is by default evaluated to null
  * if any input is evaluated to null.
  */
-abstract class BinaryExpression extends Expression {
-
-  def left: Expression
-  def right: Expression
-
-  override final def children: Seq[Expression] = Seq(left, right)
+abstract class BinaryExpression extends Expression with BinaryLike[Expression] {
 
   override def foldable: Boolean = left.foldable && right.foldable
 
@@ -574,7 +579,8 @@ abstract class BinaryExpression extends Expression {
    * of evaluation process, we should override [[eval]].
    */
   protected def nullSafeEval(input1: Any, input2: Any): Any =
-    sys.error(s"BinaryExpressions must override either eval or nullSafeEval")
+    throw QueryExecutionErrors.notOverrideExpectedMethodsError("BinaryExpressions",
+      "eval", "nullSafeEval")
 
   /**
    * Short hand for generating binary evaluation code.
@@ -635,6 +641,11 @@ abstract class BinaryExpression extends Expression {
 }
 
 
+object BinaryExpression {
+  def unapply(e: BinaryExpression): Option[(Expression, Expression)] = Some((e.left, e.right))
+}
+
+
 /**
  * A [[BinaryExpression]] that is an operator, with two properties:
  *
@@ -683,7 +694,7 @@ object BinaryOperator {
  * An expression with three inputs and one output. The output is by default evaluated to null
  * if any input is evaluated to null.
  */
-abstract class TernaryExpression extends Expression {
+abstract class TernaryExpression extends Expression with TernaryLike[Expression] {
 
   override def foldable: Boolean = children.forall(_.foldable)
 
@@ -694,12 +705,11 @@ abstract class TernaryExpression extends Expression {
    * If subclass of TernaryExpression override nullable, probably should also override this.
    */
   override def eval(input: InternalRow): Any = {
-    val exprs = children
-    val value1 = exprs(0).eval(input)
+    val value1 = first.eval(input)
     if (value1 != null) {
-      val value2 = exprs(1).eval(input)
+      val value2 = second.eval(input)
       if (value2 != null) {
-        val value3 = exprs(2).eval(input)
+        val value3 = third.eval(input)
         if (value3 != null) {
           return nullSafeEval(value1, value2, value3)
         }
@@ -714,7 +724,8 @@ abstract class TernaryExpression extends Expression {
    * of evaluation process, we should override [[eval]].
    */
   protected def nullSafeEval(input1: Any, input2: Any, input3: Any): Any =
-    sys.error(s"TernaryExpressions must override either eval or nullSafeEval")
+    throw QueryExecutionErrors.notOverrideExpectedMethodsError("TernaryExpressions",
+      "eval", "nullSafeEval")
 
   /**
    * Short hand for generating ternary evaluation code.
@@ -781,7 +792,7 @@ abstract class TernaryExpression extends Expression {
  * An expression with four inputs and one output. The output is by default evaluated to null
  * if any input is evaluated to null.
  */
-abstract class QuaternaryExpression extends Expression {
+abstract class QuaternaryExpression extends Expression with QuaternaryLike[Expression] {
 
   override def foldable: Boolean = children.forall(_.foldable)
 
@@ -792,14 +803,13 @@ abstract class QuaternaryExpression extends Expression {
    * If subclass of QuaternaryExpression override nullable, probably should also override this.
    */
   override def eval(input: InternalRow): Any = {
-    val exprs = children
-    val value1 = exprs(0).eval(input)
+    val value1 = first.eval(input)
     if (value1 != null) {
-      val value2 = exprs(1).eval(input)
+      val value2 = second.eval(input)
       if (value2 != null) {
-        val value3 = exprs(2).eval(input)
+        val value3 = third.eval(input)
         if (value3 != null) {
-          val value4 = exprs(3).eval(input)
+          val value4 = fourth.eval(input)
           if (value4 != null) {
             return nullSafeEval(value1, value2, value3, value4)
           }
@@ -815,7 +825,8 @@ abstract class QuaternaryExpression extends Expression {
    *  full control of evaluation process, we should override [[eval]].
    */
   protected def nullSafeEval(input1: Any, input2: Any, input3: Any, input4: Any): Any =
-    sys.error(s"QuaternaryExpressions must override either eval or nullSafeEval")
+    throw QueryExecutionErrors.notOverrideExpectedMethodsError("QuaternaryExpressions",
+      "eval", "nullSafeEval")
 
   /**
    * Short hand for generating quaternary evaluation code.
@@ -940,7 +951,8 @@ abstract class SeptenaryExpression extends Expression {
       input5: Any,
       input6: Any,
       input7: Option[Any]): Any = {
-    sys.error("SeptenaryExpression must override either eval or nullSafeEval")
+    throw QueryExecutionErrors.notOverrideExpectedMethodsError("SeptenaryExpression",
+      "eval", "nullSafeEval")
   }
 
   /**
@@ -1058,14 +1070,18 @@ trait ComplexTypeMergingExpression extends Expression {
         s" The input types found are\n\t${inputTypesForMerging.mkString("\n\t")}")
   }
 
-  override def dataType: DataType = {
+  private lazy val internalDataType: DataType = {
     dataTypeCheck
     inputTypesForMerging.reduceLeft(TypeCoercion.findCommonTypeDifferentOnlyInNullFlags(_, _).get)
   }
+
+  override def dataType: DataType = internalDataType
 }
 
 /**
  * Common base trait for user-defined functions, including UDF/UDAF/UDTF of different languages
  * and Hive function wrappers.
  */
-trait UserDefinedExpression
+trait UserDefinedExpression {
+  def name: String
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpressionSet.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpressionSet.scala
new file mode 100644
index 0000000000000..5d9b5be311952
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpressionSet.scala
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions
+
+import scala.collection.{mutable, GenTraversableOnce}
+import scala.collection.mutable.ArrayBuffer
+
+object ExpressionSet {
+  /** Constructs a new [[ExpressionSet]] by applying [[Canonicalize]] to `expressions`. */
+  def apply(expressions: TraversableOnce[Expression]): ExpressionSet = {
+    val set = new ExpressionSet()
+    expressions.foreach(set.add)
+    set
+  }
+
+  def apply(): ExpressionSet = {
+    new ExpressionSet()
+  }
+}
+
+/**
+ * A [[Set]] where membership is determined based on determinacy and a canonical representation of
+ * an [[Expression]] (i.e. one that attempts to ignore cosmetic differences).
+ * See [[Canonicalize]] for more details.
+ *
+ * Internally this set uses the canonical representation, but keeps also track of the original
+ * expressions to ease debugging.  Since different expressions can share the same canonical
+ * representation, this means that operations that extract expressions from this set are only
+ * guaranteed to see at least one such expression.  For example:
+ *
+ * {{{
+ *   val set = ExpressionSet(a + 1, 1 + a)
+ *
+ *   set.iterator => Iterator(a + 1)
+ *   set.contains(a + 1) => true
+ *   set.contains(1 + a) => true
+ *   set.contains(a + 2) => false
+ * }}}
+ *
+ * For non-deterministic expressions, they are always considered as not contained in the [[Set]].
+ * On adding a non-deterministic expression, simply append it to the original expressions.
+ * This is consistent with how we define `semanticEquals` between two expressions.
+ */
+class ExpressionSet protected(
+    private val baseSet: mutable.Set[Expression] = new mutable.HashSet,
+    private val originals: mutable.Buffer[Expression] = new ArrayBuffer)
+  extends Iterable[Expression] {
+
+  //  Note: this class supports Scala 2.12. A parallel source tree has a 2.13 implementation.
+
+  protected def add(e: Expression): Unit = {
+    if (!e.deterministic) {
+      originals += e
+    } else if (!baseSet.contains(e.canonicalized)) {
+      baseSet.add(e.canonicalized)
+      originals += e
+    }
+  }
+
+  protected def remove(e: Expression): Unit = {
+    if (e.deterministic) {
+      baseSet --= baseSet.filter(_ == e.canonicalized)
+      originals --= originals.filter(_.canonicalized == e.canonicalized)
+    }
+  }
+
+  def contains(elem: Expression): Boolean = baseSet.contains(elem.canonicalized)
+
+  override def filter(p: Expression => Boolean): ExpressionSet = {
+    val newBaseSet = baseSet.filter(e => p(e.canonicalized))
+    val newOriginals = originals.filter(e => p(e.canonicalized))
+    new ExpressionSet(newBaseSet, newOriginals)
+  }
+
+  override def filterNot(p: Expression => Boolean): ExpressionSet = {
+    val newBaseSet = baseSet.filterNot(e => p(e.canonicalized))
+    val newOriginals = originals.filterNot(e => p(e.canonicalized))
+    new ExpressionSet(newBaseSet, newOriginals)
+  }
+
+  def +(elem: Expression): ExpressionSet = {
+    val newSet = clone()
+    newSet.add(elem)
+    newSet
+  }
+
+  def ++(elems: GenTraversableOnce[Expression]): ExpressionSet = {
+    val newSet = clone()
+    elems.foreach(newSet.add)
+    newSet
+  }
+
+  def -(elem: Expression): ExpressionSet = {
+    val newSet = clone()
+    newSet.remove(elem)
+    newSet
+  }
+
+  def --(elems: GenTraversableOnce[Expression]): ExpressionSet = {
+    val newSet = clone()
+    elems.foreach(newSet.remove)
+    newSet
+  }
+
+  def map(f: Expression => Expression): ExpressionSet = {
+    val newSet = new ExpressionSet()
+    this.iterator.foreach(elem => newSet.add(f(elem)))
+    newSet
+  }
+
+  def flatMap(f: Expression => Iterable[Expression]): ExpressionSet = {
+    val newSet = new ExpressionSet()
+    this.iterator.foreach(f(_).foreach(newSet.add))
+    newSet
+  }
+
+  def iterator: Iterator[Expression] = originals.iterator
+
+  def union(that: ExpressionSet): ExpressionSet = {
+    val newSet = clone()
+    that.iterator.foreach(newSet.add)
+    newSet
+  }
+
+  def subsetOf(that: ExpressionSet): Boolean = this.iterator.forall(that.contains)
+
+  def intersect(that: ExpressionSet): ExpressionSet = this.filter(that.contains)
+
+  def diff(that: ExpressionSet): ExpressionSet = this -- that
+
+  def apply(elem: Expression): Boolean = this.contains(elem)
+
+  override def equals(obj: Any): Boolean = obj match {
+    case other: ExpressionSet => this.baseSet == other.baseSet
+    case _ => false
+  }
+
+  override def hashCode(): Int = baseSet.hashCode()
+
+  override def clone(): ExpressionSet = new ExpressionSet(baseSet.clone(), originals.clone())
+
+  /**
+   * Returns a string containing both the post [[Canonicalize]] expressions and the original
+   * expressions in this set.
+   */
+  def toDebugString: String =
+    s"""
+       |baseSet: ${baseSet.mkString(", ")}
+       |originals: ${originals.mkString(", ")}
+     """.stripMargin
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InterpretedMutableProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InterpretedMutableProjection.scala
index a2daec0b1ade1..91c9457af7de3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InterpretedMutableProjection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InterpretedMutableProjection.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.expressions
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.BindReferences.bindReferences
 import org.apache.spark.sql.catalyst.expressions.aggregate.NoOp
+import org.apache.spark.sql.internal.SQLConf
 
 
 /**
@@ -33,6 +34,15 @@ class InterpretedMutableProjection(expressions: Seq[Expression]) extends Mutable
   def this(expressions: Seq[Expression], inputSchema: Seq[Attribute]) =
     this(bindReferences(expressions, inputSchema))
 
+  private[this] val subExprEliminationEnabled = SQLConf.get.subexpressionEliminationEnabled
+  private[this] lazy val runtime =
+    new SubExprEvaluationRuntime(SQLConf.get.subexpressionEliminationCacheMaxEntries)
+  private[this] val exprs = if (subExprEliminationEnabled) {
+    runtime.proxyExpressions(expressions)
+  } else {
+    expressions
+  }
+
   private[this] val buffer = new Array[Any](expressions.size)
 
   override def initialize(partitionIndex: Int): Unit = {
@@ -76,11 +86,15 @@ class InterpretedMutableProjection(expressions: Seq[Expression]) extends Mutable
   }.toArray
 
   override def apply(input: InternalRow): InternalRow = {
+    if (subExprEliminationEnabled) {
+      runtime.setInput(input)
+    }
+
     var i = 0
     while (i < validExprs.length) {
-      val (expr, ordinal) = validExprs(i)
+      val (_, ordinal) = validExprs(i)
       // Store the result into buffer first, to make the projection atomic (needed by aggregation)
-      buffer(ordinal) = expr.eval(input)
+      buffer(ordinal) = exprs(ordinal).eval(input)
       i += 1
     }
     i = 0
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InterpretedSafeProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InterpretedSafeProjection.scala
index 70789dac1d87a..0e71892db666b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InterpretedSafeProjection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InterpretedSafeProjection.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.catalyst.expressions
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.aggregate.NoOp
 import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, GenericArrayData, MapData}
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 
@@ -30,6 +31,15 @@ import org.apache.spark.sql.types._
  */
 class InterpretedSafeProjection(expressions: Seq[Expression]) extends Projection {
 
+  private[this] val subExprEliminationEnabled = SQLConf.get.subexpressionEliminationEnabled
+  private[this] lazy val runtime =
+    new SubExprEvaluationRuntime(SQLConf.get.subexpressionEliminationCacheMaxEntries)
+  private[this] val exprs = if (subExprEliminationEnabled) {
+    runtime.proxyExpressions(expressions)
+  } else {
+    expressions
+  }
+
   private[this] val mutableRow = new SpecificInternalRow(expressions.map(_.dataType))
 
   private[this] val exprsWithWriters = expressions.zipWithIndex.filter {
@@ -49,7 +59,7 @@ class InterpretedSafeProjection(expressions: Seq[Expression]) extends Projection
         }
       }
     }
-    (e, f)
+    (exprs(i), f)
   }
 
   private def generateSafeValueConverter(dt: DataType): Any => Any = dt match {
@@ -97,6 +107,10 @@ class InterpretedSafeProjection(expressions: Seq[Expression]) extends Projection
   }
 
   override def apply(row: InternalRow): InternalRow = {
+    if (subExprEliminationEnabled) {
+      runtime.setInput(row)
+    }
+
     var i = 0
     while (i < exprsWithWriters.length) {
       val (expr, writer) = exprsWithWriters(i)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InterpretedUnsafeProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InterpretedUnsafeProjection.scala
index 39a16e917c4a5..908b73abadfe6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InterpretedUnsafeProjection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InterpretedUnsafeProjection.scala
@@ -16,10 +16,11 @@
  */
 package org.apache.spark.sql.catalyst.expressions
 
-import org.apache.spark.SparkException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.codegen.{UnsafeArrayWriter, UnsafeRowWriter, UnsafeWriter}
 import org.apache.spark.sql.catalyst.util.ArrayData
+import org.apache.spark.sql.errors.QueryExecutionErrors
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{UserDefinedType, _}
 import org.apache.spark.unsafe.Platform
 
@@ -33,6 +34,15 @@ import org.apache.spark.unsafe.Platform
 class InterpretedUnsafeProjection(expressions: Array[Expression]) extends UnsafeProjection {
   import InterpretedUnsafeProjection._
 
+  private[this] val subExprEliminationEnabled = SQLConf.get.subexpressionEliminationEnabled
+  private[this] lazy val runtime =
+    new SubExprEvaluationRuntime(SQLConf.get.subexpressionEliminationCacheMaxEntries)
+  private[this] val exprs = if (subExprEliminationEnabled) {
+    runtime.proxyExpressions(expressions)
+  } else {
+    expressions.toSeq
+  }
+
   /** Number of (top level) fields in the resulting row. */
   private[this] val numFields = expressions.length
 
@@ -63,17 +73,21 @@ class InterpretedUnsafeProjection(expressions: Array[Expression]) extends Unsafe
   }
 
   override def initialize(partitionIndex: Int): Unit = {
-    expressions.foreach(_.foreach {
+    exprs.foreach(_.foreach {
       case n: Nondeterministic => n.initialize(partitionIndex)
       case _ =>
     })
   }
 
   override def apply(row: InternalRow): UnsafeRow = {
+    if (subExprEliminationEnabled) {
+      runtime.setInput(row)
+    }
+
     // Put the expression results in the intermediate row.
     var i = 0
     while (i < numFields) {
-      values(i) = expressions(i).eval(row)
+      values(i) = exprs(i).eval(row)
       i += 1
     }
 
@@ -143,10 +157,10 @@ object InterpretedUnsafeProjection {
       case ShortType =>
         (v, i) => writer.write(i, v.getShort(i))
 
-      case IntegerType | DateType =>
+      case IntegerType | DateType | YearMonthIntervalType =>
         (v, i) => writer.write(i, v.getInt(i))
 
-      case LongType | TimestampType =>
+      case LongType | TimestampType | DayTimeIntervalType =>
         (v, i) => writer.write(i, v.getLong(i))
 
       case FloatType =>
@@ -240,7 +254,7 @@ object InterpretedUnsafeProjection {
         (_, _) => {}
 
       case _ =>
-        throw new SparkException(s"Unsupported data type $dt")
+        throw QueryExecutionErrors.dataTypeUnsupportedError(dt)
     }
 
     // Always wrap the writer with a null safe version.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/JoinedRow.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/JoinedRow.scala
index 7770684a5b399..86871223d66ad 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/JoinedRow.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/JoinedRow.scala
@@ -55,6 +55,16 @@ class JoinedRow extends InternalRow {
     this
   }
 
+  /** Gets this JoinedRow's left base row. */
+  def getLeft: InternalRow = {
+    row1
+  }
+
+  /** Gets this JoinedRow's right base row. */
+  def getRight: InternalRow = {
+    row2
+  }
+
   override def toSeq(fieldTypes: Seq[DataType]): Seq[Any] = {
     assert(fieldTypes.length == row1.numFields + row2.numFields)
     val (left, right) = fieldTypes.splitAt(row1.numFields)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/MonotonicallyIncreasingID.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/MonotonicallyIncreasingID.scala
index f1da592a76845..f228b36ecd472 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/MonotonicallyIncreasingID.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/MonotonicallyIncreasingID.scala
@@ -40,7 +40,14 @@ import org.apache.spark.sql.types.{DataType, LongType}
       within each partition. The assumption is that the data frame has less than 1 billion
       partitions, and each partition has less than 8 billion records.
       The function is non-deterministic because its result depends on partition IDs.
-  """)
+  """,
+  examples = """
+    Examples:
+      > SELECT _FUNC_();
+       0
+  """,
+  since = "1.4.0",
+  group = "misc_funcs")
 case class MonotonicallyIncreasingID() extends LeafExpression with Stateful {
 
   /**
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/PartitionTransforms.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/PartitionTransforms.scala
similarity index 80%
rename from sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/PartitionTransforms.scala
rename to sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/PartitionTransforms.scala
index e48fd8adaef09..ab390618d4c5c 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/PartitionTransforms.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/PartitionTransforms.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.catalyst.expressions
 
+import org.apache.spark.sql.catalyst.trees.UnaryLike
 import org.apache.spark.sql.types.{DataType, IntegerType}
 
 /**
@@ -32,7 +33,8 @@ import org.apache.spark.sql.types.{DataType, IntegerType}
  *   df.writeTo("catalog.db.table").partitionedBy($"category", days($"timestamp")).create()
  * }}}
  */
-abstract class PartitionTransformExpression extends Expression with Unevaluable {
+abstract class PartitionTransformExpression extends Expression with Unevaluable
+  with UnaryLike[Expression] {
   override def nullable: Boolean = true
 }
 
@@ -41,7 +43,7 @@ abstract class PartitionTransformExpression extends Expression with Unevaluable
  */
 case class Years(child: Expression) extends PartitionTransformExpression {
   override def dataType: DataType = IntegerType
-  override def children: Seq[Expression] = Seq(child)
+  override protected def withNewChildInternal(newChild: Expression): Years = copy(child = newChild)
 }
 
 /**
@@ -49,7 +51,7 @@ case class Years(child: Expression) extends PartitionTransformExpression {
  */
 case class Months(child: Expression) extends PartitionTransformExpression {
   override def dataType: DataType = IntegerType
-  override def children: Seq[Expression] = Seq(child)
+  override protected def withNewChildInternal(newChild: Expression): Months = copy(child = newChild)
 }
 
 /**
@@ -57,7 +59,7 @@ case class Months(child: Expression) extends PartitionTransformExpression {
  */
 case class Days(child: Expression) extends PartitionTransformExpression {
   override def dataType: DataType = IntegerType
-  override def children: Seq[Expression] = Seq(child)
+  override protected def withNewChildInternal(newChild: Expression): Days = copy(child = newChild)
 }
 
 /**
@@ -65,7 +67,7 @@ case class Days(child: Expression) extends PartitionTransformExpression {
  */
 case class Hours(child: Expression) extends PartitionTransformExpression {
   override def dataType: DataType = IntegerType
-  override def children: Seq[Expression] = Seq(child)
+  override protected def withNewChildInternal(newChild: Expression): Hours = copy(child = newChild)
 }
 
 /**
@@ -73,5 +75,5 @@ case class Hours(child: Expression) extends PartitionTransformExpression {
  */
 case class Bucket(numBuckets: Literal, child: Expression) extends PartitionTransformExpression {
   override def dataType: DataType = IntegerType
-  override def children: Seq[Expression] = Seq(numBuckets, child)
+  override protected def withNewChildInternal(newChild: Expression): Bucket = copy(child = newChild)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ProjectionOverSchema.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ProjectionOverSchema.scala
index 13c6f8db7c129..a6be98c8a3aae 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ProjectionOverSchema.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ProjectionOverSchema.scala
@@ -34,14 +34,22 @@ case class ProjectionOverSchema(schema: StructType) {
     expr match {
       case a: AttributeReference if fieldNames.contains(a.name) =>
         Some(a.copy(dataType = schema(a.name).dataType)(a.exprId, a.qualifier))
-      case GetArrayItem(child, arrayItemOrdinal) =>
-        getProjection(child).map { projection => GetArrayItem(projection, arrayItemOrdinal) }
+      case GetArrayItem(child, arrayItemOrdinal, failOnError) =>
+        getProjection(child).map {
+          projection => GetArrayItem(projection, arrayItemOrdinal, failOnError)
+        }
       case a: GetArrayStructFields =>
         getProjection(a.child).map(p => (p, p.dataType)).map {
           case (projection, ArrayType(projSchema @ StructType(_), _)) =>
+            // For case-sensitivity aware field resolution, we should take `ordinal` which
+            // points to correct struct field, because `ExtractValue` actually does column
+            // name resolving correctly.
+            val selectedField = a.child.dataType.asInstanceOf[ArrayType]
+              .elementType.asInstanceOf[StructType](a.ordinal)
+            val prunedField = projSchema(selectedField.name)
             GetArrayStructFields(projection,
-              projSchema(a.field.name),
-              projSchema.fieldIndex(a.field.name),
+              prunedField.copy(name = a.field.name),
+              projSchema.fieldIndex(selectedField.name),
               projSchema.size,
               a.containsNull)
           case (_, projSchema) =>
@@ -53,8 +61,8 @@ case class ProjectionOverSchema(schema: StructType) {
         getProjection(child).map { projection => MapKeys(projection) }
       case MapValues(child) =>
         getProjection(child).map { projection => MapValues(projection) }
-      case GetMapValue(child, key) =>
-        getProjection(child).map { projection => GetMapValue(projection, key) }
+      case GetMapValue(child, key, failOnError) =>
+        getProjection(child).map { projection => GetMapValue(projection, key, failOnError) }
       case GetStructFieldObject(child, field: StructField) =>
         getProjection(child).map(p => (p, p.dataType)).map {
           case (projection, projSchema: StructType) =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/PythonUDF.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/PythonUDF.scala
index da2e1821feb0f..73f8c300b4ae7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/PythonUDF.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/PythonUDF.scala
@@ -73,4 +73,7 @@ case class PythonUDF(
     // `resultId` can be seen as cosmetic variation in PythonUDF, as it doesn't affect the result.
     this.copy(resultId = ExprId(-1)).withNewChildren(canonicalizedChildren)
   }
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): PythonUDF =
+    copy(children = newChildren)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
index e80f03ea84756..375ae95acfc39 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
@@ -17,14 +17,14 @@
 
 package org.apache.spark.sql.catalyst.expressions
 
-import scala.collection.mutable
-
-import org.apache.spark.SparkException
-import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
+import org.apache.spark.sql.catalyst.CatalystTypeConverters.{createToCatalystConverter, createToScalaConverter => catalystCreateToScalaConverter, isPrimitive}
+import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.types.{AbstractDataType, AnyDataType, DataType}
+import org.apache.spark.util.Utils
 
 /**
  * User-defined function.
@@ -37,6 +37,8 @@ import org.apache.spark.sql.types.{AbstractDataType, AnyDataType, DataType}
  * @param inputEncoders ExpressionEncoder for each input parameters. For a input parameter which
  *                      serialized as struct will use encoder instead of CatalystTypeConverters to
  *                      convert internal value to Scala value.
+ * @param outputEncoder ExpressionEncoder for the return type of function. It's only defined when
+ *                      this is a typed Scala UDF.
  * @param udfName  The user-specified name of this UDF.
  * @param nullable  True if the UDF can return null value.
  * @param udfDeterministic  True if the UDF is deterministic. Deterministic UDF returns same result
@@ -47,6 +49,7 @@ case class ScalaUDF(
     dataType: DataType,
     children: Seq[Expression],
     inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Nil,
+    outputEncoder: Option[ExpressionEncoder[_]] = None,
     udfName: Option[String] = None,
     nullable: Boolean = true,
     udfDeterministic: Boolean = true)
@@ -54,7 +57,15 @@ case class ScalaUDF(
 
   override lazy val deterministic: Boolean = udfDeterministic && children.forall(_.deterministic)
 
-  override def toString: String = s"${udfName.getOrElse("UDF")}(${children.mkString(", ")})"
+  override def toString: String = s"$name(${children.mkString(", ")})"
+
+  override def name: String = udfName.getOrElse("UDF")
+
+  override lazy val canonicalized: Expression = {
+    // SPARK-32307: `ExpressionEncoder` can't be canonicalized, and technically we don't
+    // need it to identify a `ScalaUDF`.
+    Canonicalize.execute(copy(children = children.map(_.canonicalized), inputEncoders = Nil))
+  }
 
   /**
    * The analyzer should be aware of Scala primitive types so as to make the
@@ -63,7 +74,7 @@ case class ScalaUDF(
    * Nil(has same effect with all false) and analyzer will skip null-handling
    * on them.
    */
-  def inputPrimitives: Seq[Boolean] = {
+  lazy val inputPrimitives: Seq[Boolean] = {
     inputEncoders.map { encoderOpt =>
       // It's possible that some of the inputs don't have a specific encoder(e.g. `Any`)
       if (encoderOpt.isDefined) {
@@ -103,21 +114,61 @@ case class ScalaUDF(
     }
   }
 
-  private def createToScalaConverter(i: Int, dataType: DataType): Any => Any = {
-    if (inputEncoders.isEmpty) {
-      // for untyped Scala UDF
-      CatalystTypeConverters.createToScalaConverter(dataType)
+  /**
+   * Create the converter which converts the scala data type to the catalyst data type for
+   * the return data type of udf function. We'd use `ExpressionEncoder` to create the
+   * converter for typed ScalaUDF only, since its the only case where we know the type tag
+   * of the return data type of udf function.
+   */
+  private def catalystConverter: Any => Any = outputEncoder.map { enc =>
+    val toRow = enc.createSerializer().asInstanceOf[Any => Any]
+    if (enc.isSerializedAsStructForTopLevel) {
+      value: Any =>
+        if (value == null) null else toRow(value).asInstanceOf[InternalRow]
     } else {
-      val encoder = inputEncoders(i)
-      if (encoder.isDefined && encoder.get.isSerializedAsStructForTopLevel) {
-        val fromRow = encoder.get.resolveAndBind().createDeserializer()
+      value: Any =>
+        if (value == null) null else toRow(value).asInstanceOf[InternalRow].get(0, dataType)
+    }
+  }.getOrElse(createToCatalystConverter(dataType))
+
+  /**
+   * Create the converter which converts the catalyst data type to the scala data type.
+   * We use `CatalystTypeConverters` to create the converter for:
+   *   - UDF which doesn't provide inputEncoders, e.g., untyped Scala UDF and Java UDF
+   *   - type which isn't supported by `ExpressionEncoder`, e.g., Any
+   *   - primitive types, in order to use `identity` for better performance
+   * For other cases like case class, Option[T], we use `ExpressionEncoder` instead since
+   * `CatalystTypeConverters` doesn't support these data types.
+   *
+   * @param i the index of the child
+   * @param dataType the output data type of the i-th child
+   * @return the converter and a boolean value to indicate whether the converter is
+   *         created by using `ExpressionEncoder`.
+   */
+  private def scalaConverter(i: Int, dataType: DataType): (Any => Any, Boolean) = {
+    val useEncoder =
+      !(inputEncoders.isEmpty || // for untyped Scala UDF and Java UDF
+      inputEncoders(i).isEmpty || // for types aren't supported by encoder, e.g. Any
+      inputPrimitives(i)) // for primitive types
+
+    if (useEncoder) {
+      val enc = inputEncoders(i).get
+      val fromRow = enc.createDeserializer()
+      val converter = if (enc.isSerializedAsStructForTopLevel) {
         row: Any => fromRow(row.asInstanceOf[InternalRow])
       } else {
-        CatalystTypeConverters.createToScalaConverter(dataType)
+        val inputRow = new GenericInternalRow(1)
+        value: Any => inputRow.update(0, value); fromRow(inputRow)
       }
+      (converter, true)
+    } else { // use CatalystTypeConverters
+      (catalystCreateToScalaConverter(dataType), false)
     }
   }
 
+  private def createToScalaConverter(i: Int, dataType: DataType): Any => Any =
+    scalaConverter(i, dataType)._1
+
   // scalastyle:off line.size.limit
 
   /** This method has been generated by this script
@@ -1045,12 +1096,12 @@ case class ScalaUDF(
       ev: ExprCode): ExprCode = {
     val converterClassName = classOf[Any => Any].getName
 
-    // The type converters for inputs and the result.
-    val converters: Array[Any => Any] = children.zipWithIndex.map { case (c, i) =>
-      createToScalaConverter(i, c.dataType)
-    }.toArray :+ CatalystTypeConverters.createToCatalystConverter(dataType)
+    // The type converters for inputs and the result
+    val (converters, useEncoders): (Array[Any => Any], Array[Boolean]) =
+      (children.zipWithIndex.map { case (c, i) =>
+        scalaConverter(i, c.dataType)
+      }.toArray :+ (catalystConverter, false)).unzip
     val convertersTerm = ctx.addReferenceObj("converters", converters, s"$converterClassName[]")
-    val errorMsgTerm = ctx.addReferenceObj("errMsg", udfErrorMessage)
     val resultTerm = ctx.freshName("result")
 
     // codegen for children expressions
@@ -1064,12 +1115,26 @@ case class ScalaUDF(
     val (funcArgs, initArgs) = evals.zipWithIndex.zip(children.map(_.dataType)).map {
       case ((eval, i), dt) =>
         val argTerm = ctx.freshName("arg")
-        val initArg = if (CatalystTypeConverters.isPrimitive(dt)) {
+        // Check `inputPrimitives` when it's not empty in order to figure out the Option
+        // type as non primitive type, e.g., Option[Int]. Fall back to `isPrimitive` when
+        // `inputPrimitives` is empty for other cases, e.g., Java UDF, untyped Scala UDF
+        val primitive = (inputPrimitives.isEmpty && isPrimitive(dt)) ||
+          (inputPrimitives.nonEmpty && inputPrimitives(i))
+        val initArg = if (primitive) {
           val convertedTerm = ctx.freshName("conv")
           s"""
              |${CodeGenerator.boxedType(dt)} $convertedTerm = ${eval.value};
              |Object $argTerm = ${eval.isNull} ? null : $convertedTerm;
            """.stripMargin
+        } else if (useEncoders(i)) {
+          s"""
+             |Object $argTerm = null;
+             |if (${eval.isNull}) {
+             |  $argTerm = $convertersTerm[$i].apply(null);
+             |} else {
+             |  $argTerm = $convertersTerm[$i].apply(${eval.value});
+             |}
+          """.stripMargin
         } else {
           s"Object $argTerm = ${eval.isNull} ? null : $convertersTerm[$i].apply(${eval.value});"
         }
@@ -1081,7 +1146,7 @@ case class ScalaUDF(
     val resultConverter = s"$convertersTerm[${children.length}]"
     val boxedType = CodeGenerator.boxedType(dataType)
 
-    val funcInvokation = if (CatalystTypeConverters.isPrimitive(dataType)
+    val funcInvocation = if (isPrimitive(dataType)
         // If the output is nullable, the returned value must be unwrapped from the Option
         && !nullable) {
       s"$resultTerm = ($boxedType)$getFuncResult"
@@ -1092,9 +1157,10 @@ case class ScalaUDF(
       s"""
          |$boxedType $resultTerm = null;
          |try {
-         |  $funcInvokation;
-         |} catch (Exception e) {
-         |  throw new org.apache.spark.SparkException($errorMsgTerm, e);
+         |  $funcInvocation;
+         |} catch (Throwable e) {
+         |  throw QueryExecutionErrors.failedExecuteUserDefinedFunctionError(
+         |    "$funcCls", "$inputTypesString", "$outputType", e);
          |}
        """.stripMargin
 
@@ -1112,23 +1178,24 @@ case class ScalaUDF(
        """.stripMargin)
   }
 
-  private[this] val resultConverter = CatalystTypeConverters.createToCatalystConverter(dataType)
+  private[this] val resultConverter = catalystConverter
 
-  lazy val udfErrorMessage = {
-    val funcCls = function.getClass.getSimpleName
-    val inputTypes = children.map(_.dataType.catalogString).mkString(", ")
-    val outputType = dataType.catalogString
-    s"Failed to execute user defined function($funcCls: ($inputTypes) => $outputType)"
-  }
+  lazy val funcCls = Utils.getSimpleName(function.getClass)
+  lazy val inputTypesString = children.map(_.dataType.catalogString).mkString(", ")
+  lazy val outputType = dataType.catalogString
 
   override def eval(input: InternalRow): Any = {
     val result = try {
       f(input)
     } catch {
       case e: Exception =>
-        throw new SparkException(udfErrorMessage, e)
+        throw QueryExecutionErrors.failedExecuteUserDefinedFunctionError(
+          funcCls, inputTypesString, outputType, e)
     }
 
     resultConverter(result)
   }
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): ScalaUDF =
+    copy(children = newChildren)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SchemaPruning.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SchemaPruning.scala
index 6213267c41c64..9aa2766dd3e8c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SchemaPruning.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SchemaPruning.scala
@@ -17,26 +17,30 @@
 
 package org.apache.spark.sql.catalyst.expressions
 
+import org.apache.spark.sql.catalyst.SQLConfHelper
 import org.apache.spark.sql.types._
 
-object SchemaPruning {
+object SchemaPruning extends SQLConfHelper {
   /**
-   * Filters the schema by the requested fields. For example, if the schema is struct<a:int, b:int>,
-   * and given requested field are "a", the field "b" is pruned in the returned schema.
-   * Note that schema field ordering at original schema is still preserved in pruned schema.
+   * Prunes the nested schema by the requested fields. For example, if the schema is:
+   * `id int, s struct<a:int, b:int>`, and given requested field "s.a", the inner field "b"
+   * is pruned in the returned schema: `id int, s struct<a:int>`.
+   * Note that:
+   *   1. The schema field ordering at original schema is still preserved in pruned schema.
+   *   2. The top-level fields are not pruned here.
    */
   def pruneDataSchema(
       dataSchema: StructType,
       requestedRootFields: Seq[RootField]): StructType = {
+    val resolver = conf.resolver
     // Merge the requested root fields into a single schema. Note the ordering of the fields
     // in the resulting schema may differ from their ordering in the logical relation's
     // original schema
     val mergedSchema = requestedRootFields
-      .map { case root: RootField => StructType(Array(root.field)) }
+      .map { root: RootField => StructType(Array(root.field)) }
       .reduceLeft(_ merge _)
-    val dataSchemaFieldNames = dataSchema.fieldNames.toSet
     val mergedDataSchema =
-      StructType(mergedSchema.filter(f => dataSchemaFieldNames.contains(f.name)))
+      StructType(dataSchema.map(d => mergedSchema.find(m => resolver(m.name, d.name)).getOrElse(d)))
     // Sort the fields of mergedDataSchema according to their order in dataSchema,
     // recursively. This makes mergedDataSchema a pruned schema of dataSchema
     sortLeftFieldsByRight(mergedDataSchema, dataSchema).asInstanceOf[StructType]
@@ -61,12 +65,16 @@ object SchemaPruning {
           sortLeftFieldsByRight(leftValueType, rightValueType),
           containsNull)
       case (leftStruct: StructType, rightStruct: StructType) =>
-        val filteredRightFieldNames = rightStruct.fieldNames.filter(leftStruct.fieldNames.contains)
+        val resolver = conf.resolver
+        val filteredRightFieldNames = rightStruct.fieldNames
+          .filter(name => leftStruct.fieldNames.exists(resolver(_, name)))
         val sortedLeftFields = filteredRightFieldNames.map { fieldName =>
-          val leftFieldType = leftStruct(fieldName).dataType
+          val resolvedLeftStruct = leftStruct.find(p => resolver(p.name, fieldName)).get
+          val leftFieldType = resolvedLeftStruct.dataType
           val rightFieldType = rightStruct(fieldName).dataType
           val sortedLeftFieldType = sortLeftFieldsByRight(leftFieldType, rightFieldType)
-          StructField(fieldName, sortedLeftFieldType, nullable = leftStruct(fieldName).nullable)
+          StructField(fieldName, sortedLeftFieldType, nullable = resolvedLeftStruct.nullable,
+            metadata = resolvedLeftStruct.metadata)
         }
         StructType(sortedLeftFields)
       case _ => left
@@ -117,10 +125,11 @@ object SchemaPruning {
    * When expr is an [[Attribute]], construct a field around it and indicate that that
    * field was derived from an attribute.
    */
-  private def getRootFields(expr: Expression): Seq[RootField] = {
+  private[catalyst] def getRootFields(expr: Expression): Seq[RootField] = {
     expr match {
       case att: Attribute =>
-        RootField(StructField(att.name, att.dataType, att.nullable), derivedFromAtt = true) :: Nil
+        RootField(StructField(att.name, att.dataType, att.nullable, att.metadata),
+          derivedFromAtt = true) :: Nil
       case SelectedField(field) => RootField(field, derivedFromAtt = false) :: Nil
       // Root field accesses by `IsNotNull` and `IsNull` are special cases as the expressions
       // don't actually use any nested fields. These root field accesses might be excluded later
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SelectedField.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SelectedField.scala
index 7ba3d302d553b..4314aad9b46f1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SelectedField.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SelectedField.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.catalyst.expressions
 
-import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.types._
 
 /**
@@ -75,7 +75,11 @@ object SelectedField {
         val field = c.childSchema(c.ordinal)
         val newField = field.copy(dataType = dataTypeOpt.getOrElse(field.dataType))
         selectField(c.child, Option(struct(newField)))
-      case GetArrayStructFields(child, field, _, _, containsNull) =>
+      case GetArrayStructFields(child, _, ordinal, _, containsNull) =>
+        // For case-sensitivity aware field resolution, we should take `ordinal` which
+        // points to correct struct field.
+        val field = child.dataType.asInstanceOf[ArrayType]
+          .elementType.asInstanceOf[StructType](ordinal)
         val newFieldDataType = dataTypeOpt match {
           case None =>
             // GetArrayStructFields is the top level extractor. This means its result is
@@ -87,11 +91,11 @@ object SelectedField {
             dataType
           case Some(x) =>
             // This should not happen.
-            throw new AnalysisException(s"DataType '$x' is not supported by GetArrayStructFields.")
+            throw QueryCompilationErrors.dataTypeUnsupportedByClassError(x, "GetArrayStructFields")
         }
         val newField = StructField(field.name, newFieldDataType, field.nullable)
         selectField(child, Option(ArrayType(struct(newField), containsNull)))
-      case GetMapValue(child, _) =>
+      case GetMapValue(child, _, _) =>
         // GetMapValue does not select a field from a struct (i.e. prune the struct) so it can't be
         // the top-level extractor. However it can be part of an extractor chain.
         val MapType(keyType, _, valueContainsNull) = child.dataType
@@ -105,7 +109,7 @@ object SelectedField {
           case ArrayType(dataType, _) => MapType(keyType, dataType, valueContainsNull)
           case x =>
             // This should not happen.
-            throw new AnalysisException(s"DataType '$x' is not supported by MapValues.")
+            throw QueryCompilationErrors.dataTypeUnsupportedByClassError(x, "MapValues")
         }
         selectField(child, opt)
       case MapKeys(child) =>
@@ -116,10 +120,10 @@ object SelectedField {
           case ArrayType(dataType, _) => MapType(dataType, valueType, valueContainsNull)
           case x =>
             // This should not happen.
-            throw new AnalysisException(s"DataType '$x' is not supported by MapKeys.")
+            throw QueryCompilationErrors.dataTypeUnsupportedByClassError(x, "MapKeys")
         }
         selectField(child, opt)
-      case GetArrayItem(child, _) =>
+      case GetArrayItem(child, _, _) =>
         // GetArrayItem does not select a field from a struct (i.e. prune the struct) so it can't be
         // the top-level extractor. However it can be part of an extractor chain.
         val ArrayType(_, containsNull) = child.dataType
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SortOrder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SortOrder.scala
index 536276b5cb29f..9aef25ce60599 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SortOrder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SortOrder.scala
@@ -63,11 +63,10 @@ case class SortOrder(
     child: Expression,
     direction: SortDirection,
     nullOrdering: NullOrdering,
-    sameOrderExpressions: Set[Expression])
-  extends UnaryExpression with Unevaluable {
+    sameOrderExpressions: Seq[Expression])
+  extends Expression with Unevaluable {
 
-  /** Sort order is not foldable because we don't have an eval for it. */
-  override def foldable: Boolean = false
+  override def children: Seq[Expression] = child +: sameOrderExpressions
 
   override def checkInputDataTypes(): TypeCheckResult = {
     if (RowOrdering.isOrderable(dataType)) {
@@ -86,16 +85,19 @@ case class SortOrder(
   def isAscending: Boolean = direction == Ascending
 
   def satisfies(required: SortOrder): Boolean = {
-    (sameOrderExpressions + child).exists(required.child.semanticEquals) &&
+    children.exists(required.child.semanticEquals) &&
       direction == required.direction && nullOrdering == required.nullOrdering
   }
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): SortOrder =
+    copy(child = newChildren.head, sameOrderExpressions = newChildren.tail)
 }
 
 object SortOrder {
   def apply(
      child: Expression,
      direction: SortDirection,
-     sameOrderExpressions: Set[Expression] = Set.empty): SortOrder = {
+     sameOrderExpressions: Seq[Expression] = Seq.empty): SortOrder = {
     new SortOrder(child, direction, direction.defaultNullOrdering, sameOrderExpressions)
   }
 
@@ -227,4 +229,7 @@ case class SortPrefix(child: SortOrder) extends UnaryExpression {
   }
 
   override def dataType: DataType = LongType
+
+  override protected def withNewChildInternal(newChild: Expression): SortPrefix =
+    copy(child = newChild.asInstanceOf[SortOrder])
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SparkPartitionID.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SparkPartitionID.scala
index 9856b37e53fbc..2de89da2318f7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SparkPartitionID.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SparkPartitionID.scala
@@ -26,7 +26,14 @@ import org.apache.spark.sql.types.{DataType, IntegerType}
  * Expression that returns the current partition id.
  */
 @ExpressionDescription(
-  usage = "_FUNC_() - Returns the current partition id.")
+  usage = "_FUNC_() - Returns the current partition id.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_();
+       0
+  """,
+  since = "1.4.0",
+  group = "misc_funcs")
 case class SparkPartitionID() extends LeafExpression with Nondeterministic {
 
   override def nullable: Boolean = false
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SpecificInternalRow.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SpecificInternalRow.scala
index 159f90995c1fd..0f26192468990 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SpecificInternalRow.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SpecificInternalRow.scala
@@ -192,24 +192,44 @@ final class MutableAny extends MutableValue {
  */
 final class SpecificInternalRow(val values: Array[MutableValue]) extends BaseGenericInternalRow {
 
-  def this(dataTypes: Seq[DataType]) =
-    this(
-      dataTypes.map {
-        case BooleanType => new MutableBoolean
-        case ByteType => new MutableByte
-        case ShortType => new MutableShort
-        // We use INT for DATE internally
-        case IntegerType | DateType => new MutableInt
-        // We use Long for Timestamp internally
-        case LongType | TimestampType => new MutableLong
-        case FloatType => new MutableFloat
-        case DoubleType => new MutableDouble
-        case _ => new MutableAny
-      }.toArray)
+  private[this] def dataTypeToMutableValue(dataType: DataType): MutableValue = dataType match {
+    // We use INT for DATE and YearMonthIntervalType internally
+    case IntegerType | DateType | YearMonthIntervalType => new MutableInt
+    // We use Long for Timestamp and DayTimeInterval internally
+    case LongType | TimestampType | DayTimeIntervalType => new MutableLong
+    case FloatType => new MutableFloat
+    case DoubleType => new MutableDouble
+    case BooleanType => new MutableBoolean
+    case ByteType => new MutableByte
+    case ShortType => new MutableShort
+    case _ => new MutableAny
+  }
+
+  def this(dataTypes: Seq[DataType]) = {
+    // SPARK-32550: use `dataTypes.foreach` instead of `while loop + dataTypes(i)` to ensure
+    // constant-time access of dataTypes `Seq` because it is not necessarily an `IndexSeq` that
+    // support constant-time access.
+    this(new Array[MutableValue](dataTypes.length))
+    var i = 0
+    dataTypes.foreach { dt =>
+      values(i) = dataTypeToMutableValue(dt)
+      i += 1
+    }
+  }
 
   def this() = this(Seq.empty)
 
-  def this(schema: StructType) = this(schema.fields.map(_.dataType))
+  def this(schema: StructType) = {
+    // SPARK-32550: use while loop instead of map
+    this(new Array[MutableValue](schema.fields.length))
+    val length = values.length
+    val fields = schema.fields
+    var i = 0
+    while (i < length) {
+      values(i) = dataTypeToMutableValue(fields(i).dataType)
+      i += 1
+    }
+  }
 
   override def numFields: Int = values.length
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SubExprEvaluationRuntime.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SubExprEvaluationRuntime.scala
new file mode 100644
index 0000000000000..0f224fefe3911
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SubExprEvaluationRuntime.scala
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.catalyst.expressions
+
+import java.util.IdentityHashMap
+
+import com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache}
+import com.google.common.util.concurrent.{ExecutionError, UncheckedExecutionException}
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
+import org.apache.spark.sql.errors.QueryExecutionErrors
+import org.apache.spark.sql.types.DataType
+
+/**
+ * This class helps subexpression elimination for interpreted evaluation
+ * such as `InterpretedUnsafeProjection`. It maintains an evaluation cache.
+ * This class wraps `ExpressionProxy` around given expressions. The `ExpressionProxy`
+ * intercepts expression evaluation and loads from the cache first.
+ */
+class SubExprEvaluationRuntime(cacheMaxEntries: Int) {
+  // The id assigned to `ExpressionProxy`. `SubExprEvaluationRuntime` will use assigned ids of
+  // `ExpressionProxy` to decide the equality when loading from cache. `SubExprEvaluationRuntime`
+  // won't be use by multi-threads so we don't need to consider concurrency here.
+  private var proxyExpressionCurrentId = 0
+
+  private[sql] val cache: LoadingCache[ExpressionProxy, ResultProxy] = CacheBuilder.newBuilder()
+    .maximumSize(cacheMaxEntries)
+    .build(
+      new CacheLoader[ExpressionProxy, ResultProxy]() {
+        override def load(expr: ExpressionProxy): ResultProxy = {
+          ResultProxy(expr.proxyEval(currentInput))
+        }
+      })
+
+  private var currentInput: InternalRow = null
+
+  def getEval(proxy: ExpressionProxy): Any = try {
+    cache.get(proxy).result
+  } catch {
+    // Cache.get() may wrap the original exception. See the following URL
+    // http://google.github.io/guava/releases/14.0/api/docs/com/google/common/cache/
+    //   Cache.html#get(K,%20java.util.concurrent.Callable)
+    case e @ (_: UncheckedExecutionException | _: ExecutionError) =>
+      throw e.getCause
+  }
+
+  /**
+   * Sets given input row as current row for evaluating expressions. This cleans up the cache
+   * too as new input comes.
+   */
+  def setInput(input: InternalRow = null): Unit = {
+    currentInput = input
+    cache.invalidateAll()
+  }
+
+  /**
+   * Recursively replaces expression with its proxy expression in `proxyMap`.
+   */
+  private def replaceWithProxy(
+      expr: Expression,
+      proxyMap: IdentityHashMap[Expression, ExpressionProxy]): Expression = {
+    if (proxyMap.containsKey(expr)) {
+      proxyMap.get(expr)
+    } else {
+      expr.mapChildren(replaceWithProxy(_, proxyMap))
+    }
+  }
+
+  /**
+   * Finds subexpressions and wraps them with `ExpressionProxy`.
+   */
+  def proxyExpressions(expressions: Seq[Expression]): Seq[Expression] = {
+    val equivalentExpressions: EquivalentExpressions = new EquivalentExpressions
+
+    expressions.foreach(equivalentExpressions.addExprTree(_))
+
+    val proxyMap = new IdentityHashMap[Expression, ExpressionProxy]
+
+    val commonExprs = equivalentExpressions.getAllEquivalentExprs.filter(_.size > 1)
+    commonExprs.foreach { e =>
+      val expr = e.head
+      val proxy = ExpressionProxy(expr, proxyExpressionCurrentId, this)
+      proxyExpressionCurrentId += 1
+
+      // We leverage `IdentityHashMap` so we compare expression keys by reference here.
+      // So for example if there are one group of common exprs like Seq(common expr 1,
+      // common expr2, ..., common expr n), we will insert into `proxyMap` some key/value
+      // pairs like Map(common expr 1 -> proxy(common expr 1), ...,
+      // common expr n -> proxy(common expr 1)).
+      e.map(proxyMap.put(_, proxy))
+    }
+
+    // Only adding proxy if we find subexpressions.
+    if (!proxyMap.isEmpty) {
+      expressions.map(replaceWithProxy(_, proxyMap))
+    } else {
+      expressions
+    }
+  }
+}
+
+/**
+ * A proxy for an catalyst `Expression`. Given a runtime object `SubExprEvaluationRuntime`,
+ * when this is asked to evaluate, it will load from the evaluation cache in the runtime first.
+ */
+case class ExpressionProxy(
+    child: Expression,
+    id: Int,
+    runtime: SubExprEvaluationRuntime) extends UnaryExpression {
+
+  final override def dataType: DataType = child.dataType
+  final override def nullable: Boolean = child.nullable
+
+  // `ExpressionProxy` is for interpreted expression evaluation only. So cannot `doGenCode`.
+  final override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode =
+    throw QueryExecutionErrors.cannotGenerateCodeForExpressionError(this)
+
+  def proxyEval(input: InternalRow = null): Any = child.eval(input)
+
+  override def eval(input: InternalRow = null): Any = runtime.getEval(this)
+
+  override def equals(obj: Any): Boolean = obj match {
+    case other: ExpressionProxy => this.id == other.id
+    case _ => false
+  }
+
+  override def hashCode(): Int = this.id.hashCode()
+
+  override protected def withNewChildInternal(newChild: Expression): ExpressionProxy =
+    copy(child = newChild)
+}
+
+/**
+ * A simple wrapper for holding `Any` in the cache of `SubExprEvaluationRuntime`.
+ */
+case class ResultProxy(result: Any)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TimeWindow.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TimeWindow.scala
index 82d689477080d..ed1d77017c120 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TimeWindow.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TimeWindow.scala
@@ -92,6 +92,9 @@ case class TimeWindow(
     }
     dataTypeCheck
   }
+
+  override protected def withNewChildInternal(newChild: Expression): TimeWindow =
+    copy(timeColumn = newChild)
 }
 
 object TimeWindow {
@@ -144,7 +147,7 @@ object TimeWindow {
 case class PreciseTimestampConversion(
     child: Expression,
     fromType: DataType,
-    toType: DataType) extends UnaryExpression with ExpectsInputTypes {
+    toType: DataType) extends UnaryExpression with ExpectsInputTypes with NullIntolerant {
   override def inputTypes: Seq[AbstractDataType] = Seq(fromType)
   override def dataType: DataType = toType
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
@@ -155,4 +158,7 @@ case class PreciseTimestampConversion(
        """.stripMargin)
   }
   override def nullSafeEval(input: Any): Any = input
+
+  override protected def withNewChildInternal(newChild: Expression): PreciseTimestampConversion =
+    copy(child = newChild)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryCast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryCast.scala
new file mode 100644
index 0000000000000..0f63de1bf7e45
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryCast.scala
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions
+
+import org.apache.spark.sql.catalyst.expressions.codegen._
+import org.apache.spark.sql.catalyst.expressions.codegen.Block._
+import org.apache.spark.sql.types.DataType
+
+/**
+ * A special version of [[AnsiCast]]. It performs the same operation (i.e. converts a value of
+ * one data type into another data type), but returns a NULL value instead of raising an error
+ * when the conversion can not be performed.
+ *
+ * When cast from/to timezone related types, we need timeZoneId, which will be resolved with
+ * session local timezone by an analyzer [[ResolveTimeZone]].
+ */
+@ExpressionDescription(
+  usage = """
+    _FUNC_(expr AS type) - Casts the value `expr` to the target data type `type`.
+      This expression is identical to CAST with configuration `spark.sql.ansi.enabled` as
+      true, except it returns NULL instead of raising an error. Note that the behavior of this
+      expression doesn't depend on configuration `spark.sql.ansi.enabled`.
+  """,
+  examples = """
+    Examples:
+      > SELECT _FUNC_('10' as int);
+       10
+      > SELECT _FUNC_(1234567890123L as int);
+       null
+  """,
+  since = "3.2.0",
+  group = "conversion_funcs")
+case class TryCast(child: Expression, dataType: DataType, timeZoneId: Option[String] = None)
+  extends CastBase {
+  override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
+    copy(timeZoneId = Option(timeZoneId))
+
+  // Here we force `ansiEnabled` as true so that we can reuse the evaluation code branches which
+  // throw exceptions on conversion failures.
+  override protected val ansiEnabled: Boolean = true
+
+  override def nullable: Boolean = true
+
+  override def canCast(from: DataType, to: DataType): Boolean = AnsiCast.canCast(from, to)
+
+  override def cast(from: DataType, to: DataType): Any => Any = (input: Any) =>
+    try {
+      super.cast(from, to)(input)
+    } catch {
+      case _: Exception =>
+        null
+    }
+
+  override def castCode(ctx: CodegenContext, input: ExprValue, inputIsNull: ExprValue,
+    result: ExprValue, resultIsNull: ExprValue, resultType: DataType, cast: CastFunction): Block = {
+    val javaType = JavaCode.javaType(resultType)
+    code"""
+      boolean $resultIsNull = $inputIsNull;
+      $javaType $result = ${CodeGenerator.defaultValue(resultType)};
+      if (!$inputIsNull) {
+        try {
+          ${cast(input, result, resultIsNull)}
+        } catch (Exception e) {
+          $resultIsNull = true;
+        }
+      }
+    """
+  }
+
+  override def typeCheckFailureMessage: String =
+    AnsiCast.typeCheckFailureMessage(child.dataType, dataType, None, None)
+
+  override protected def withNewChildInternal(newChild: Expression): TryCast =
+    copy(child = newChild)
+
+  override def toString: String = {
+    s"try_cast($child as ${dataType.simpleString})"
+  }
+
+  override def sql: String = s"TRY_CAST(${child.sql} AS ${dataType.sql})"
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryEval.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryEval.scala
new file mode 100644
index 0000000000000..a75db1bdb3e69
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryEval.scala
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodeGenerator, ExprCode}
+import org.apache.spark.sql.catalyst.expressions.codegen.Block._
+import org.apache.spark.sql.types.DataType
+
+case class TryEval(child: Expression) extends UnaryExpression with NullIntolerant {
+  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
+    val childGen = child.genCode(ctx)
+    ev.copy(code = code"""
+      boolean ${ev.isNull} = true;
+      ${CodeGenerator.javaType(dataType)} ${ev.value} = ${CodeGenerator.defaultValue(dataType)};
+      try {
+        ${childGen.code}
+        ${ev.isNull} = ${childGen.isNull};
+        ${ev.value} = ${childGen.value};
+      } catch (Exception e) {
+      }"""
+    )
+  }
+
+  override def eval(input: InternalRow): Any =
+    try {
+      child.eval(input)
+    } catch {
+      case _: Exception =>
+        null
+    }
+
+  override def dataType: DataType = child.dataType
+
+  override def nullable: Boolean = true
+
+  override protected def withNewChildInternal(newChild: Expression): Expression =
+    copy(child = newChild)
+}
+
+@ExpressionDescription(
+  usage = "_FUNC_(expr1, expr2) - Returns `expr1`+`expr2` and the result is null on overflow.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_(1, 2);
+       3
+      > SELECT _FUNC_(2147483647, 1);
+       NULL
+  """,
+  since = "3.2.0",
+  group = "math_funcs")
+case class TryAdd(left: Expression, right: Expression, child: Expression)
+    extends RuntimeReplaceable {
+  def this(left: Expression, right: Expression) =
+    this(left, right, TryEval(Add(left, right, failOnError = true)))
+
+  override def flatArguments: Iterator[Any] = Iterator(left, right)
+
+  override def exprsReplaced: Seq[Expression] = Seq(left, right)
+
+  override def prettyName: String = "try_add"
+
+  override protected def withNewChildInternal(newChild: Expression): Expression =
+    this.copy(child = newChild)
+}
+
+// scalastyle:off line.size.limit
+@ExpressionDescription(
+  usage = "_FUNC_(expr1, expr2) - Returns `expr1`/`expr2`. It always performs floating point division. Its result is always null if `expr2` is 0.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_(3, 2);
+       1.5
+      > SELECT _FUNC_(2L, 2L);
+       1.0
+      > SELECT _FUNC_(1, 0);
+       NULL
+  """,
+  since = "3.2.0",
+  group = "math_funcs")
+// scalastyle:on line.size.limit
+case class TryDivide(left: Expression, right: Expression, child: Expression)
+    extends RuntimeReplaceable {
+  def this(left: Expression, right: Expression) =
+    this(left, right, TryEval(Divide(left, right, failOnError = true)))
+
+  override def flatArguments: Iterator[Any] = Iterator(left, right)
+
+  override def exprsReplaced: Seq[Expression] = Seq(left, right)
+
+  override def prettyName: String = "try_divide"
+
+  override protected def withNewChildInternal(newChild: Expression): Expression =
+    this.copy(child = newChild)
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxCountDistinctForIntervals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxCountDistinctForIntervals.scala
index f96a087972f1f..19e212d1f9e69 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxCountDistinctForIntervals.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxCountDistinctForIntervals.scala
@@ -23,6 +23,7 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
 import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes, Expression, GenericInternalRow}
+import org.apache.spark.sql.catalyst.trees.BinaryLike
 import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData, HyperLogLogPlusPlusHelper}
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.Platform
@@ -39,7 +40,8 @@ import org.apache.spark.unsafe.Platform
  *                            and its elements should be sorted into ascending order.
  *                            Duplicate endpoints are allowed, e.g. (1, 5, 5, 10), and ndv for
  *                            interval (5, 5] would be 1.
- * @param relativeSD The maximum estimation error allowed in the HyperLogLogPlusPlus algorithm.
+ * @param relativeSD The maximum relative standard deviation allowed
+ *                   in the HyperLogLogPlusPlus algorithm.
  */
 case class ApproxCountDistinctForIntervals(
     child: Expression,
@@ -47,7 +49,7 @@ case class ApproxCountDistinctForIntervals(
     relativeSD: Double = 0.05,
     mutableAggBufferOffset: Int = 0,
     inputAggBufferOffset: Int = 0)
-  extends TypedImperativeAggregate[Array[Long]] with ExpectsInputTypes {
+  extends TypedImperativeAggregate[Array[Long]] with ExpectsInputTypes with BinaryLike[Expression] {
 
   def this(child: Expression, endpointsExpression: Expression, relativeSD: Expression) = {
     this(
@@ -212,7 +214,8 @@ case class ApproxCountDistinctForIntervals(
     copy(inputAggBufferOffset = newInputAggBufferOffset)
   }
 
-  override def children: Seq[Expression] = Seq(child, endpointsExpression)
+  override def left: Expression = child
+  override def right: Expression = endpointsExpression
 
   override def nullable: Boolean = false
 
@@ -246,4 +249,8 @@ case class ApproxCountDistinctForIntervals(
     override def getLong(offset: Int): Long = array(offset)
     override def setLong(offset: Int, value: Long): Unit = { array(offset) = value }
   }
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): ApproxCountDistinctForIntervals =
+    copy(child = newLeft, endpointsExpression = newRight)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala
index d06eeeef23936..38d8d7d71ead8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala
@@ -26,9 +26,11 @@ import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate.ApproximatePercentile.PercentileDigest
+import org.apache.spark.sql.catalyst.trees.TernaryLike
 import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData}
 import org.apache.spark.sql.catalyst.util.QuantileSummaries
 import org.apache.spark.sql.catalyst.util.QuantileSummaries.{defaultCompressThreshold, Stats}
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.types._
 
 /**
@@ -49,21 +51,23 @@ import org.apache.spark.sql.types._
  */
 @ExpressionDescription(
   usage = """
-    _FUNC_(col, percentage [, accuracy]) - Returns the approximate percentile value of numeric
-      column `col` at the given percentage. The value of percentage must be between 0.0
-      and 1.0. The `accuracy` parameter (default: 10000) is a positive numeric literal which
-      controls approximation accuracy at the cost of memory. Higher value of `accuracy` yields
-      better accuracy, `1.0/accuracy` is the relative error of the approximation.
+    _FUNC_(col, percentage [, accuracy]) - Returns the approximate `percentile` of the numeric
+      column `col` which is the smallest value in the ordered `col` values (sorted from least to
+      greatest) such that no more than `percentage` of `col` values is less than the value
+      or equal to that value. The value of percentage must be between 0.0 and 1.0. The `accuracy`
+      parameter (default: 10000) is a positive numeric literal which controls approximation accuracy
+      at the cost of memory. Higher value of `accuracy` yields better accuracy, `1.0/accuracy` is
+      the relative error of the approximation.
       When `percentage` is an array, each value of the percentage array must be between 0.0 and 1.0.
       In this case, returns the approximate percentile array of column `col` at the given
       percentage array.
   """,
   examples = """
     Examples:
-      > SELECT _FUNC_(10.0, array(0.5, 0.4, 0.1), 100);
-       [10.0,10.0,10.0]
-      > SELECT _FUNC_(10.0, 0.5, 100);
-       10.0
+      > SELECT _FUNC_(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col);
+       [1,1,0]
+      > SELECT _FUNC_(col, 0.5, 100) FROM VALUES (0), (6), (7), (9), (10) AS tab(col);
+       7
   """,
   group = "agg_funcs",
   since = "2.1.0")
@@ -73,7 +77,8 @@ case class ApproximatePercentile(
     accuracyExpression: Expression,
     override val mutableAggBufferOffset: Int,
     override val inputAggBufferOffset: Int)
-  extends TypedImperativeAggregate[PercentileDigest] with ImplicitCastInputTypes {
+  extends TypedImperativeAggregate[PercentileDigest] with ImplicitCastInputTypes
+  with TernaryLike[Expression] {
 
   def this(child: Expression, percentageExpression: Expression, accuracyExpression: Expression) = {
     this(child, percentageExpression, accuracyExpression, 0, 0)
@@ -137,7 +142,7 @@ case class ApproximatePercentile(
         case TimestampType => value.asInstanceOf[Long].toDouble
         case n: NumericType => n.numeric.toDouble(value.asInstanceOf[n.InternalType])
         case other: DataType =>
-          throw new UnsupportedOperationException(s"Unexpected data type ${other.catalogString}")
+          throw QueryExecutionErrors.dataTypeUnexpectedError(other)
       }
       buffer.add(doubleValue)
     }
@@ -162,7 +167,7 @@ case class ApproximatePercentile(
       case DoubleType => doubleResult
       case _: DecimalType => doubleResult.map(Decimal(_))
       case other: DataType =>
-        throw new UnsupportedOperationException(s"Unexpected data type ${other.catalogString}")
+        throw QueryExecutionErrors.dataTypeUnexpectedError(other)
     }
     if (result.length == 0) {
       null
@@ -179,16 +184,20 @@ case class ApproximatePercentile(
   override def withNewInputAggBufferOffset(newOffset: Int): ApproximatePercentile =
     copy(inputAggBufferOffset = newOffset)
 
-  override def children: Seq[Expression] = Seq(child, percentageExpression, accuracyExpression)
+  override def first: Expression = child
+  override def second: Expression = percentageExpression
+  override def third: Expression = accuracyExpression
 
   // Returns null for empty inputs
   override def nullable: Boolean = true
 
   // The result type is the same as the input type.
-  override def dataType: DataType = {
+  private lazy val internalDataType: DataType = {
     if (returnPercentileArray) ArrayType(child.dataType, false) else child.dataType
   }
 
+  override def dataType: DataType = internalDataType
+
   override def prettyName: String =
     getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("percentile_approx")
 
@@ -199,6 +208,10 @@ case class ApproximatePercentile(
   override def deserialize(bytes: Array[Byte]): PercentileDigest = {
     ApproximatePercentile.serializer.deserialize(bytes)
   }
+
+  override protected def withNewChildrenInternal(
+      newFirst: Expression, newSecond: Expression, newThird: Expression): ApproximatePercentile =
+    copy(child = newFirst, percentageExpression = newSecond, accuracyExpression = newThird)
 }
 
 object ApproximatePercentile {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala
index d3ce1f8d331ab..82ad2df11a3ec 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala
@@ -20,6 +20,8 @@ package org.apache.spark.sql.catalyst.expressions.aggregate
 import org.apache.spark.sql.catalyst.analysis.{DecimalPrecision, FunctionRegistry, TypeCheckResult}
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.trees.TreePattern.{AVERAGE, TreePattern}
+import org.apache.spark.sql.catalyst.trees.UnaryLike
 import org.apache.spark.sql.catalyst.util.TypeUtils
 import org.apache.spark.sql.types._
 
@@ -34,30 +36,36 @@ import org.apache.spark.sql.types._
   """,
   group = "agg_funcs",
   since = "1.0.0")
-case class Average(child: Expression) extends DeclarativeAggregate with ImplicitCastInputTypes {
+case class Average(child: Expression) extends DeclarativeAggregate with ImplicitCastInputTypes
+  with UnaryLike[Expression] {
 
   override def prettyName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("avg")
 
-  override def children: Seq[Expression] = child :: Nil
-
-  override def inputTypes: Seq[AbstractDataType] = Seq(NumericType)
+  override def inputTypes: Seq[AbstractDataType] =
+    Seq(TypeCollection(NumericType, YearMonthIntervalType, DayTimeIntervalType))
 
   override def checkInputDataTypes(): TypeCheckResult =
-    TypeUtils.checkForNumericExpr(child.dataType, "function average")
+    TypeUtils.checkForAnsiIntervalOrNumericType(child.dataType, "average")
 
   override def nullable: Boolean = true
 
   // Return data type.
   override def dataType: DataType = resultType
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(AVERAGE)
+
   private lazy val resultType = child.dataType match {
     case DecimalType.Fixed(p, s) =>
       DecimalType.bounded(p + 4, s + 4)
+    case _: YearMonthIntervalType => YearMonthIntervalType
+    case _: DayTimeIntervalType => DayTimeIntervalType
     case _ => DoubleType
   }
 
   private lazy val sumDataType = child.dataType match {
     case _ @ DecimalType.Fixed(p, s) => DecimalType.bounded(p + 10, s)
+    case _: YearMonthIntervalType => YearMonthIntervalType
+    case _: DayTimeIntervalType => DayTimeIntervalType
     case _ => DoubleType
   }
 
@@ -77,11 +85,19 @@ case class Average(child: Expression) extends DeclarativeAggregate with Implicit
   )
 
   // If all input are nulls, count will be 0 and we will get null after the division.
+  // We can't directly use `/` as it throws an exception under ansi mode.
   override lazy val evaluateExpression = child.dataType match {
     case _: DecimalType =>
-      DecimalPrecision.decimalAndDecimal(sum / count.cast(DecimalType.LongDecimal)).cast(resultType)
+      DecimalPrecision.decimalAndDecimal()(
+        Divide(sum, count.cast(DecimalType.LongDecimal), failOnError = false)).cast(resultType)
+    case _: YearMonthIntervalType =>
+      If(EqualTo(count, Literal(0L)),
+        Literal(null, YearMonthIntervalType), DivideYMInterval(sum, count))
+    case _: DayTimeIntervalType =>
+      If(EqualTo(count, Literal(0L)),
+        Literal(null, DayTimeIntervalType), DivideDTInterval(sum, count))
     case _ =>
-      sum.cast(resultType) / count.cast(resultType)
+      Divide(sum.cast(resultType), count.cast(resultType), failOnError = false)
   }
 
   override lazy val updateExpressions: Seq[Expression] = Seq(
@@ -91,4 +107,7 @@ case class Average(child: Expression) extends DeclarativeAggregate with Implicit
       coalesce(child.cast(sumDataType), Literal.default(sumDataType))),
     /* count = */ If(child.isNull, count, count + 1L)
   )
+
+  override protected def withNewChildInternal(newChild: Expression): Average =
+    copy(child = newChild)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala
index 53759ca3d9165..c5c78e5062f56 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala
@@ -20,6 +20,8 @@ package org.apache.spark.sql.catalyst.expressions.aggregate
 import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.trees.UnaryLike
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 /**
@@ -43,15 +45,14 @@ import org.apache.spark.sql.types._
  *
  * @param child to compute central moments of.
  */
-abstract class CentralMomentAgg(child: Expression)
-  extends DeclarativeAggregate with ImplicitCastInputTypes {
+abstract class CentralMomentAgg(child: Expression, nullOnDivideByZero: Boolean)
+  extends DeclarativeAggregate with ImplicitCastInputTypes with UnaryLike[Expression] {
 
   /**
    * The central moment order to be computed.
    */
   protected def momentOrder: Int
 
-  override def children: Seq[Expression] = Seq(child)
   override def nullable: Boolean = true
   override def dataType: DataType = DoubleType
   override def inputTypes: Seq[AbstractDataType] = Seq(DoubleType)
@@ -62,6 +63,13 @@ abstract class CentralMomentAgg(child: Expression)
   protected val m3 = AttributeReference("m3", DoubleType, nullable = false)()
   protected val m4 = AttributeReference("m4", DoubleType, nullable = false)()
 
+  protected def divideByZeroEvalResult: Expression = {
+    if (nullOnDivideByZero) Literal.create(null, DoubleType) else Double.NaN
+  }
+
+  override def stringArgs: Iterator[Any] =
+    super.stringArgs.filter(_.isInstanceOf[Expression])
+
   private def trimHigherOrder[T](expressions: Seq[T]) = expressions.take(momentOrder + 1)
 
   override val aggBufferAttributes = trimHigherOrder(Seq(n, avg, m2, m3, m4))
@@ -145,7 +153,12 @@ abstract class CentralMomentAgg(child: Expression)
   group = "agg_funcs",
   since = "1.6.0")
 // scalastyle:on line.size.limit
-case class StddevPop(child: Expression) extends CentralMomentAgg(child) {
+case class StddevPop(
+    child: Expression,
+    nullOnDivideByZero: Boolean = !SQLConf.get.legacyStatisticalAggregate)
+  extends CentralMomentAgg(child, nullOnDivideByZero) {
+
+  def this(child: Expression) = this(child, !SQLConf.get.legacyStatisticalAggregate)
 
   override protected def momentOrder = 2
 
@@ -154,6 +167,9 @@ case class StddevPop(child: Expression) extends CentralMomentAgg(child) {
   }
 
   override def prettyName: String = "stddev_pop"
+
+  override protected def withNewChildInternal(newChild: Expression): StddevPop =
+    copy(child = newChild)
 }
 
 // Compute the sample standard deviation of a column
@@ -168,17 +184,25 @@ case class StddevPop(child: Expression) extends CentralMomentAgg(child) {
   group = "agg_funcs",
   since = "1.6.0")
 // scalastyle:on line.size.limit
-case class StddevSamp(child: Expression) extends CentralMomentAgg(child) {
+case class StddevSamp(
+    child: Expression,
+    nullOnDivideByZero: Boolean = !SQLConf.get.legacyStatisticalAggregate)
+  extends CentralMomentAgg(child, nullOnDivideByZero) {
+
+  def this(child: Expression) = this(child, !SQLConf.get.legacyStatisticalAggregate)
 
   override protected def momentOrder = 2
 
   override val evaluateExpression: Expression = {
     If(n === 0.0, Literal.create(null, DoubleType),
-      If(n === 1.0, Double.NaN, sqrt(m2 / (n - 1.0))))
+      If(n === 1.0, divideByZeroEvalResult, sqrt(m2 / (n - 1.0))))
   }
 
   override def prettyName: String =
     getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("stddev_samp")
+
+  override protected def withNewChildInternal(newChild: Expression): StddevSamp =
+    copy(child = newChild)
 }
 
 // Compute the population variance of a column
@@ -191,7 +215,12 @@ case class StddevSamp(child: Expression) extends CentralMomentAgg(child) {
   """,
   group = "agg_funcs",
   since = "1.6.0")
-case class VariancePop(child: Expression) extends CentralMomentAgg(child) {
+case class VariancePop(
+    child: Expression,
+    nullOnDivideByZero: Boolean = !SQLConf.get.legacyStatisticalAggregate)
+  extends CentralMomentAgg(child, nullOnDivideByZero) {
+
+  def this(child: Expression) = this(child, !SQLConf.get.legacyStatisticalAggregate)
 
   override protected def momentOrder = 2
 
@@ -200,6 +229,9 @@ case class VariancePop(child: Expression) extends CentralMomentAgg(child) {
   }
 
   override def prettyName: String = "var_pop"
+
+  override protected def withNewChildInternal(newChild: Expression): VariancePop =
+    copy(child = newChild)
 }
 
 // Compute the sample variance of a column
@@ -212,16 +244,24 @@ case class VariancePop(child: Expression) extends CentralMomentAgg(child) {
   """,
   group = "agg_funcs",
   since = "1.6.0")
-case class VarianceSamp(child: Expression) extends CentralMomentAgg(child) {
+case class VarianceSamp(
+    child: Expression,
+    nullOnDivideByZero: Boolean = !SQLConf.get.legacyStatisticalAggregate)
+  extends CentralMomentAgg(child, nullOnDivideByZero) {
+
+  def this(child: Expression) = this(child, !SQLConf.get.legacyStatisticalAggregate)
 
   override protected def momentOrder = 2
 
   override val evaluateExpression: Expression = {
     If(n === 0.0, Literal.create(null, DoubleType),
-      If(n === 1.0, Double.NaN, m2 / (n - 1.0)))
+      If(n === 1.0, divideByZeroEvalResult, m2 / (n - 1.0)))
   }
 
   override def prettyName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("var_samp")
+
+  override protected def withNewChildInternal(newChild: Expression): VarianceSamp =
+    copy(child = newChild)
 }
 
 @ExpressionDescription(
@@ -235,7 +275,12 @@ case class VarianceSamp(child: Expression) extends CentralMomentAgg(child) {
   """,
   group = "agg_funcs",
   since = "1.6.0")
-case class Skewness(child: Expression) extends CentralMomentAgg(child) {
+case class Skewness(
+    child: Expression,
+    nullOnDivideByZero: Boolean = !SQLConf.get.legacyStatisticalAggregate)
+  extends CentralMomentAgg(child, nullOnDivideByZero) {
+
+  def this(child: Expression) = this(child, !SQLConf.get.legacyStatisticalAggregate)
 
   override def prettyName: String = "skewness"
 
@@ -243,8 +288,11 @@ case class Skewness(child: Expression) extends CentralMomentAgg(child) {
 
   override val evaluateExpression: Expression = {
     If(n === 0.0, Literal.create(null, DoubleType),
-      If(m2 === 0.0, Double.NaN, sqrt(n) * m3 / sqrt(m2 * m2 * m2)))
+      If(m2 === 0.0, divideByZeroEvalResult, sqrt(n) * m3 / sqrt(m2 * m2 * m2)))
   }
+
+  override protected def withNewChildInternal(newChild: Expression): Skewness =
+    copy(child = newChild)
 }
 
 @ExpressionDescription(
@@ -258,14 +306,22 @@ case class Skewness(child: Expression) extends CentralMomentAgg(child) {
   """,
   group = "agg_funcs",
   since = "1.6.0")
-case class Kurtosis(child: Expression) extends CentralMomentAgg(child) {
+case class Kurtosis(
+    child: Expression,
+    nullOnDivideByZero: Boolean = !SQLConf.get.legacyStatisticalAggregate)
+  extends CentralMomentAgg(child, nullOnDivideByZero) {
+
+  def this(child: Expression) = this(child, !SQLConf.get.legacyStatisticalAggregate)
 
   override protected def momentOrder = 4
 
   override val evaluateExpression: Expression = {
     If(n === 0.0, Literal.create(null, DoubleType),
-      If(m2 === 0.0, Double.NaN, n * m4 / (m2 * m2) - 3.0))
+      If(m2 === 0.0, divideByZeroEvalResult, n * m4 / (m2 * m2) - 3.0))
   }
 
   override def prettyName: String = "kurtosis"
+
+  override protected def withNewChildInternal(newChild: Expression): Kurtosis =
+    copy(child = newChild)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala
index 9ef05bb5d4fec..c798004fe7843 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala
@@ -19,6 +19,8 @@ package org.apache.spark.sql.catalyst.expressions.aggregate
 
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.trees.BinaryLike
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 /**
@@ -28,10 +30,11 @@ import org.apache.spark.sql.types._
  * Definition of Pearson correlation can be found at
  * http://en.wikipedia.org/wiki/Pearson_product-moment_correlation_coefficient
  */
-abstract class PearsonCorrelation(x: Expression, y: Expression)
-  extends DeclarativeAggregate with ImplicitCastInputTypes {
+abstract class PearsonCorrelation(x: Expression, y: Expression, nullOnDivideByZero: Boolean)
+  extends DeclarativeAggregate with ImplicitCastInputTypes with BinaryLike[Expression] {
 
-  override def children: Seq[Expression] = Seq(x, y)
+  override def left: Expression = x
+  override def right: Expression = y
   override def nullable: Boolean = true
   override def dataType: DataType = DoubleType
   override def inputTypes: Seq[AbstractDataType] = Seq(DoubleType, DoubleType)
@@ -43,6 +46,13 @@ abstract class PearsonCorrelation(x: Expression, y: Expression)
   protected val xMk = AttributeReference("xMk", DoubleType, nullable = false)()
   protected val yMk = AttributeReference("yMk", DoubleType, nullable = false)()
 
+  protected def divideByZeroEvalResult: Expression = {
+    if (nullOnDivideByZero) Literal.create(null, DoubleType) else Double.NaN
+  }
+
+  override def stringArgs: Iterator[Any] =
+    super.stringArgs.filter(_.isInstanceOf[Expression])
+
   override val aggBufferAttributes: Seq[AttributeReference] = Seq(n, xAvg, yAvg, ck, xMk, yMk)
 
   override val initialValues: Seq[Expression] = Array.fill(6)(Literal(0.0))
@@ -102,13 +112,22 @@ abstract class PearsonCorrelation(x: Expression, y: Expression)
   group = "agg_funcs",
   since = "1.6.0")
 // scalastyle:on line.size.limit
-case class Corr(x: Expression, y: Expression)
-  extends PearsonCorrelation(x, y) {
+case class Corr(
+    x: Expression,
+    y: Expression,
+    nullOnDivideByZero: Boolean = !SQLConf.get.legacyStatisticalAggregate)
+  extends PearsonCorrelation(x, y, nullOnDivideByZero) {
+
+  def this(x: Expression, y: Expression) =
+    this(x, y, !SQLConf.get.legacyStatisticalAggregate)
 
   override val evaluateExpression: Expression = {
     If(n === 0.0, Literal.create(null, DoubleType),
-      If(n === 1.0, Double.NaN, ck / sqrt(xMk * yMk)))
+      If(n === 1.0, divideByZeroEvalResult, ck / sqrt(xMk * yMk)))
   }
 
   override def prettyName: String = "corr"
+
+  override protected def withNewChildrenInternal(newLeft: Expression, newRight: Expression): Corr =
+    copy(x = newLeft, y = newRight)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala
index e043c81975066..dfdd828d10d03 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala
@@ -17,8 +17,11 @@
 
 package org.apache.spark.sql.catalyst.expressions.aggregate
 
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.trees.TreePattern.{COUNT, TreePattern}
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 // scalastyle:off line.size.limit
@@ -43,11 +46,24 @@ import org.apache.spark.sql.types._
   since = "1.0.0")
 // scalastyle:on line.size.limit
 case class Count(children: Seq[Expression]) extends DeclarativeAggregate {
+
   override def nullable: Boolean = false
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(COUNT)
+
   // Return data type.
   override def dataType: DataType = LongType
 
+  override def checkInputDataTypes(): TypeCheckResult = {
+    if (children.isEmpty && !SQLConf.get.getConf(SQLConf.ALLOW_PARAMETERLESS_COUNT)) {
+      TypeCheckResult.TypeCheckFailure(s"$prettyName requires at least one argument. " +
+        s"If you have to call the function $prettyName without arguments, set the legacy " +
+        s"configuration `${SQLConf.ALLOW_PARAMETERLESS_COUNT.key}` as true")
+    } else {
+      TypeCheckResult.TypeCheckSuccess
+    }
+  }
+
   protected lazy val count = AttributeReference("count", LongType, nullable = false)()
 
   override lazy val aggBufferAttributes = count :: Nil
@@ -76,6 +92,9 @@ case class Count(children: Seq[Expression]) extends DeclarativeAggregate {
       )
     }
   }
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Count =
+    copy(children = newChildren)
 }
 
 object Count {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountIf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountIf.scala
index 5bb95ead3f715..66800b277ffed 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountIf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountIf.scala
@@ -19,6 +19,8 @@ package org.apache.spark.sql.catalyst.expressions.aggregate
 
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription, ImplicitCastInputTypes, UnevaluableAggregate}
+import org.apache.spark.sql.catalyst.trees.TreePattern.{COUNT_IF, TreePattern}
+import org.apache.spark.sql.catalyst.trees.UnaryLike
 import org.apache.spark.sql.types.{AbstractDataType, BooleanType, DataType, LongType}
 
 @ExpressionDescription(
@@ -34,10 +36,12 @@ import org.apache.spark.sql.types.{AbstractDataType, BooleanType, DataType, Long
   """,
   group = "agg_funcs",
   since = "3.0.0")
-case class CountIf(predicate: Expression) extends UnevaluableAggregate with ImplicitCastInputTypes {
+case class CountIf(predicate: Expression) extends UnevaluableAggregate with ImplicitCastInputTypes
+  with UnaryLike[Expression] {
+
   override def prettyName: String = "count_if"
 
-  override def children: Seq[Expression] = Seq(predicate)
+  override def child: Expression = predicate
 
   override def nullable: Boolean = false
 
@@ -45,6 +49,8 @@ case class CountIf(predicate: Expression) extends UnevaluableAggregate with Impl
 
   override def inputTypes: Seq[AbstractDataType] = Seq(BooleanType)
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(COUNT_IF)
+
   override def checkInputDataTypes(): TypeCheckResult = predicate.dataType match {
     case BooleanType =>
       TypeCheckResult.TypeCheckSuccess
@@ -53,4 +59,7 @@ case class CountIf(predicate: Expression) extends UnevaluableAggregate with Impl
         s"function $prettyName requires boolean type, not ${predicate.dataType.catalogString}"
       )
   }
+
+  override protected def withNewChildInternal(newChild: Expression): CountIf =
+    copy(predicate = newChild)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountMinSketchAgg.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountMinSketchAgg.scala
index 787b21859c6da..38d0db1e7610c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountMinSketchAgg.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountMinSketchAgg.scala
@@ -20,7 +20,8 @@ package org.apache.spark.sql.catalyst.expressions.aggregate
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
-import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes, Expression, ExpressionDescription}
+import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes, Expression, ExpressionDescription, Literal}
+import org.apache.spark.sql.catalyst.trees.QuaternaryLike
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 import org.apache.spark.util.sketch.CountMinSketch
@@ -37,6 +38,7 @@ import org.apache.spark.util.sketch.CountMinSketch
  * @param confidenceExpression confidence, must be positive and less than 1.0
  * @param seedExpression random seed
  */
+// scalastyle:off line.size.limit
 @ExpressionDescription(
   usage = """
     _FUNC_(col, eps, confidence, seed) - Returns a count-min sketch of a column with the given esp,
@@ -44,8 +46,14 @@ import org.apache.spark.util.sketch.CountMinSketch
       `CountMinSketch` before usage. Count-min sketch is a probabilistic data structure used for
       cardinality estimation using sub-linear space.
   """,
+  examples = """
+    Examples:
+      > SELECT hex(_FUNC_(col, 0.5d, 0.5d, 1)) FROM VALUES (1), (2), (1) AS tab(col);
+       0000000100000000000000030000000100000004000000005D8D6AB90000000000000000000000000000000200000000000000010000000000000000
+  """,
   group = "agg_funcs",
   since = "2.2.0")
+// scalastyle:on line.size.limit
 case class CountMinSketchAgg(
     child: Expression,
     epsExpression: Expression,
@@ -53,7 +61,9 @@ case class CountMinSketchAgg(
     seedExpression: Expression,
     override val mutableAggBufferOffset: Int,
     override val inputAggBufferOffset: Int)
-  extends TypedImperativeAggregate[CountMinSketch] with ExpectsInputTypes {
+  extends TypedImperativeAggregate[CountMinSketch]
+  with ExpectsInputTypes
+  with QuaternaryLike[Expression] {
 
   def this(
       child: Expression,
@@ -135,8 +145,21 @@ case class CountMinSketchAgg(
 
   override def dataType: DataType = BinaryType
 
-  override def children: Seq[Expression] =
-    Seq(child, epsExpression, confidenceExpression, seedExpression)
+  override def defaultResult: Option[Literal] =
+    Option(Literal.create(eval(createAggregationBuffer()), dataType))
 
   override def prettyName: String = "count_min_sketch"
+
+  override def first: Expression = child
+  override def second: Expression = epsExpression
+  override def third: Expression = confidenceExpression
+  override def fourth: Expression = seedExpression
+
+  override protected def withNewChildrenInternal(first: Expression, second: Expression,
+      third: Expression, fourth: Expression): CountMinSketchAgg =
+    copy(
+      child = first,
+      epsExpression = second,
+      confidenceExpression = third,
+      seedExpression = fourth)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala
index f03c2f2710a04..9ea9b3782032b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala
@@ -19,16 +19,17 @@ package org.apache.spark.sql.catalyst.expressions.aggregate
 
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.trees.BinaryLike
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 /**
  * Compute the covariance between two expressions.
  * When applied on empty data (i.e., count is zero), it returns NULL.
  */
-abstract class Covariance(x: Expression, y: Expression)
-  extends DeclarativeAggregate with ImplicitCastInputTypes {
+abstract class Covariance(val left: Expression, val right: Expression, nullOnDivideByZero: Boolean)
+  extends DeclarativeAggregate with ImplicitCastInputTypes with BinaryLike[Expression] {
 
-  override def children: Seq[Expression] = Seq(x, y)
   override def nullable: Boolean = true
   override def dataType: DataType = DoubleType
   override def inputTypes: Seq[AbstractDataType] = Seq(DoubleType, DoubleType)
@@ -38,6 +39,13 @@ abstract class Covariance(x: Expression, y: Expression)
   protected val yAvg = AttributeReference("yAvg", DoubleType, nullable = false)()
   protected val ck = AttributeReference("ck", DoubleType, nullable = false)()
 
+  protected def divideByZeroEvalResult: Expression = {
+    if (nullOnDivideByZero) Literal.create(null, DoubleType) else Double.NaN
+  }
+
+  override def stringArgs: Iterator[Any] =
+    super.stringArgs.filter(_.isInstanceOf[Expression])
+
   override val aggBufferAttributes: Seq[AttributeReference] = Seq(n, xAvg, yAvg, ck)
 
   override val initialValues: Seq[Expression] = Array.fill(4)(Literal(0.0))
@@ -62,14 +70,14 @@ abstract class Covariance(x: Expression, y: Expression)
 
   protected def updateExpressionsDef: Seq[Expression] = {
     val newN = n + 1.0
-    val dx = x - xAvg
-    val dy = y - yAvg
+    val dx = left - xAvg
+    val dy = right - yAvg
     val dyN = dy / newN
     val newXAvg = xAvg + dx / newN
     val newYAvg = yAvg + dyN
-    val newCk = ck + dx * (y - newYAvg)
+    val newCk = ck + dx * (right - newYAvg)
 
-    val isNull = x.isNull || y.isNull
+    val isNull = left.isNull || right.isNull
     Seq(
       If(isNull, n, newN),
       If(isNull, xAvg, newXAvg),
@@ -88,11 +96,23 @@ abstract class Covariance(x: Expression, y: Expression)
   """,
   group = "agg_funcs",
   since = "2.0.0")
-case class CovPopulation(left: Expression, right: Expression) extends Covariance(left, right) {
+case class CovPopulation(
+    override val left: Expression,
+    override val right: Expression,
+    nullOnDivideByZero: Boolean = !SQLConf.get.legacyStatisticalAggregate)
+  extends Covariance(left, right, nullOnDivideByZero) {
+
+  def this(left: Expression, right: Expression) =
+    this(left, right, !SQLConf.get.legacyStatisticalAggregate)
+
   override val evaluateExpression: Expression = {
     If(n === 0.0, Literal.create(null, DoubleType), ck / n)
   }
   override def prettyName: String = "covar_pop"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): CovPopulation =
+    copy(left = newLeft, right = newRight)
 }
 
 
@@ -105,10 +125,21 @@ case class CovPopulation(left: Expression, right: Expression) extends Covariance
   """,
   group = "agg_funcs",
   since = "2.0.0")
-case class CovSample(left: Expression, right: Expression) extends Covariance(left, right) {
+case class CovSample(
+    override val left: Expression,
+    override val right: Expression,
+    nullOnDivideByZero: Boolean = !SQLConf.get.legacyStatisticalAggregate)
+  extends Covariance(left, right, nullOnDivideByZero) {
+
+  def this(left: Expression, right: Expression) =
+    this(left, right, !SQLConf.get.legacyStatisticalAggregate)
+
   override val evaluateExpression: Expression = {
     If(n === 0.0, Literal.create(null, DoubleType),
-      If(n === 1.0, Double.NaN, ck / (n - 1.0)))
+      If(n === 1.0, divideByZeroEvalResult, ck / (n - 1.0)))
   }
   override def prettyName: String = "covar_samp"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): CovSample = copy(left = newLeft, right = newRight)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala
index df806edbfda05..ea994af0e6168 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala
@@ -17,10 +17,12 @@
 
 package org.apache.spark.sql.catalyst.expressions.aggregate
 
-import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult}
-import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.TypeCheckSuccess
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.trees.UnaryLike
 import org.apache.spark.sql.types._
 
 /**
@@ -49,12 +51,14 @@ import org.apache.spark.sql.types._
   """,
   group = "agg_funcs",
   since = "2.0.0")
-case class First(child: Expression, ignoreNullsExpr: Expression)
-  extends DeclarativeAggregate with ExpectsInputTypes {
+case class First(child: Expression, ignoreNulls: Boolean)
+  extends DeclarativeAggregate with ExpectsInputTypes with UnaryLike[Expression] {
 
-  def this(child: Expression) = this(child, Literal.create(false, BooleanType))
+  def this(child: Expression) = this(child, false)
 
-  override def children: Seq[Expression] = child :: ignoreNullsExpr :: Nil
+  def this(child: Expression, ignoreNullsExpr: Expression) = {
+    this(child, FirstLast.validateIgnoreNullExpr(ignoreNullsExpr, "first"))
+  }
 
   override def nullable: Boolean = true
 
@@ -71,16 +75,11 @@ case class First(child: Expression, ignoreNullsExpr: Expression)
     val defaultCheck = super.checkInputDataTypes()
     if (defaultCheck.isFailure) {
       defaultCheck
-    } else if (!ignoreNullsExpr.foldable) {
-      TypeCheckFailure(
-        s"The second argument of First must be a boolean literal, but got: ${ignoreNullsExpr.sql}")
     } else {
       TypeCheckSuccess
     }
   }
 
-  private def ignoreNulls: Boolean = ignoreNullsExpr.eval().asInstanceOf[Boolean]
-
   private lazy val first = AttributeReference("first", child.dataType)()
 
   private lazy val valueSet = AttributeReference("valueSet", BooleanType)()
@@ -119,4 +118,14 @@ case class First(child: Expression, ignoreNullsExpr: Expression)
   override lazy val evaluateExpression: AttributeReference = first
 
   override def toString: String = s"$prettyName($child)${if (ignoreNulls) " ignore nulls"}"
+
+  override protected def withNewChildInternal(newChild: Expression): First = copy(child = newChild)
+}
+
+object FirstLast {
+  def validateIgnoreNullExpr(exp: Expression, funcName: String): Boolean = exp match {
+    case Literal(b: Boolean, BooleanType) => b
+    case _ => throw new AnalysisException(
+      s"The second argument in $funcName should be a boolean literal.")
+  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala
index aed36902b1567..9b0493f3e68a4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala
@@ -17,10 +17,11 @@
 
 package org.apache.spark.sql.catalyst.expressions.aggregate
 
-import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.trees.UnaryLike
 import org.apache.spark.sql.catalyst.util.HyperLogLogPlusPlusHelper
+import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.types._
 
 // scalastyle:off
@@ -41,13 +42,13 @@ import org.apache.spark.sql.types._
  * https://docs.google.com/document/d/1gyjfMHy43U9OWBXxfaeG-3MjGzejW1dlpyMwEYAAWEI/view?fullscreen#
  *
  * @param child to estimate the cardinality of.
- * @param relativeSD the maximum estimation error allowed.
+ * @param relativeSD the maximum relative standard deviation allowed.
  */
 // scalastyle:on
 @ExpressionDescription(
   usage = """
     _FUNC_(expr[, relativeSD]) - Returns the estimated cardinality by HyperLogLog++.
-      `relativeSD` defines the maximum estimation error allowed.""",
+      `relativeSD` defines the maximum relative standard deviation allowed.""",
   examples = """
     Examples:
       > SELECT _FUNC_(col1) FROM VALUES (1), (1), (2), (2), (3) tab(col1);
@@ -60,7 +61,7 @@ case class HyperLogLogPlusPlus(
     relativeSD: Double = 0.05,
     mutableAggBufferOffset: Int = 0,
     inputAggBufferOffset: Int = 0)
-  extends ImperativeAggregate {
+  extends ImperativeAggregate with UnaryLike[Expression] {
 
   def this(child: Expression) = {
     this(child = child, relativeSD = 0.05, mutableAggBufferOffset = 0, inputAggBufferOffset = 0)
@@ -82,14 +83,14 @@ case class HyperLogLogPlusPlus(
   override def withNewInputAggBufferOffset(newInputAggBufferOffset: Int): ImperativeAggregate =
     copy(inputAggBufferOffset = newInputAggBufferOffset)
 
-  override def children: Seq[Expression] = Seq(child)
-
   override def nullable: Boolean = false
 
   override def dataType: DataType = LongType
 
   override def aggBufferSchema: StructType = StructType.fromAttributes(aggBufferAttributes)
 
+  override def defaultResult: Option[Literal] = Option(Literal.create(0L, dataType))
+
   val hllppHelper = new HyperLogLogPlusPlusHelper(relativeSD)
 
   /** Allocate enough words to store all registers. */
@@ -137,6 +138,9 @@ case class HyperLogLogPlusPlus(
   override def eval(buffer: InternalRow): Any = {
     hllppHelper.query(buffer, mutableAggBufferOffset)
   }
+
+  override protected def withNewChildInternal(newChild: Expression): HyperLogLogPlusPlus =
+    copy(child = newChild)
 }
 
 object HyperLogLogPlusPlus {
@@ -144,6 +148,6 @@ object HyperLogLogPlusPlus {
     case Literal(d: Double, DoubleType) => d
     case Literal(dec: Decimal, _) => dec.toDouble
     case _ =>
-      throw new AnalysisException("The second argument should be a double literal.")
+      throw QueryCompilationErrors.secondArgumentNotDoubleLiteralError
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala
index e55bced192f34..0fe6199cd8c31 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala
@@ -17,10 +17,11 @@
 
 package org.apache.spark.sql.catalyst.expressions.aggregate
 
-import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult}
-import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.TypeCheckSuccess
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.trees.UnaryLike
 import org.apache.spark.sql.types._
 
 /**
@@ -49,12 +50,14 @@ import org.apache.spark.sql.types._
   """,
   group = "agg_funcs",
   since = "2.0.0")
-case class Last(child: Expression, ignoreNullsExpr: Expression)
-  extends DeclarativeAggregate with ExpectsInputTypes {
+case class Last(child: Expression, ignoreNulls: Boolean)
+  extends DeclarativeAggregate with ExpectsInputTypes with UnaryLike[Expression] {
 
-  def this(child: Expression) = this(child, Literal.create(false, BooleanType))
+  def this(child: Expression) = this(child, false)
 
-  override def children: Seq[Expression] = child :: ignoreNullsExpr :: Nil
+  def this(child: Expression, ignoreNullsExpr: Expression) = {
+    this(child, FirstLast.validateIgnoreNullExpr(ignoreNullsExpr, "last"))
+  }
 
   override def nullable: Boolean = true
 
@@ -71,16 +74,11 @@ case class Last(child: Expression, ignoreNullsExpr: Expression)
     val defaultCheck = super.checkInputDataTypes()
     if (defaultCheck.isFailure) {
       defaultCheck
-    } else if (!ignoreNullsExpr.foldable) {
-      TypeCheckFailure(
-        s"The second argument of Last must be a boolean literal, but got: ${ignoreNullsExpr.sql}")
     } else {
       TypeCheckSuccess
     }
   }
 
-  private def ignoreNulls: Boolean = ignoreNullsExpr.eval().asInstanceOf[Boolean]
-
   private lazy val last = AttributeReference("last", child.dataType)()
 
   private lazy val valueSet = AttributeReference("valueSet", BooleanType)()
@@ -117,4 +115,6 @@ case class Last(child: Expression, ignoreNullsExpr: Expression)
   override lazy val evaluateExpression: AttributeReference = last
 
   override def toString: String = s"$prettyName($child)${if (ignoreNulls) " ignore nulls"}"
+
+  override protected def withNewChildInternal(newChild: Expression): Last = copy(child = newChild)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala
index 9bba6604c84ac..b802678ec0468 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.expressions.aggregate
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.trees.UnaryLike
 import org.apache.spark.sql.catalyst.util.TypeUtils
 import org.apache.spark.sql.types._
 
@@ -32,9 +33,7 @@ import org.apache.spark.sql.types._
   """,
   group = "agg_funcs",
   since = "1.0.0")
-case class Max(child: Expression) extends DeclarativeAggregate {
-
-  override def children: Seq[Expression] = child :: Nil
+case class Max(child: Expression) extends DeclarativeAggregate with UnaryLike[Expression] {
 
   override def nullable: Boolean = true
 
@@ -63,4 +62,6 @@ case class Max(child: Expression) extends DeclarativeAggregate {
   }
 
   override lazy val evaluateExpression: AttributeReference = max
+
+  override protected def withNewChildInternal(newChild: Expression): Max = copy(child = newChild)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/MaxByAndMinBy.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/MaxByAndMinBy.scala
index 6d3d3dafe16e4..664bc32ccc464 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/MaxByAndMinBy.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/MaxByAndMinBy.scala
@@ -20,13 +20,14 @@ package org.apache.spark.sql.catalyst.expressions.aggregate
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.trees.BinaryLike
 import org.apache.spark.sql.catalyst.util.TypeUtils
 import org.apache.spark.sql.types._
 
 /**
  * The shared abstract superclass for `MaxBy` and `MinBy` SQL aggregate functions.
  */
-abstract class MaxMinBy extends DeclarativeAggregate {
+abstract class MaxMinBy extends DeclarativeAggregate with BinaryLike[Expression] {
 
   def valueExpr: Expression
   def orderingExpr: Expression
@@ -37,7 +38,8 @@ abstract class MaxMinBy extends DeclarativeAggregate {
   // Used to pick up updated ordering value.
   protected def orderingUpdater(oldExpr: Expression, newExpr: Expression): Expression
 
-  override def children: Seq[Expression] = valueExpr :: orderingExpr :: Nil
+  override def left: Expression = valueExpr
+  override def right: Expression = orderingExpr
 
   override def nullable: Boolean = true
 
@@ -108,6 +110,9 @@ case class MaxBy(valueExpr: Expression, orderingExpr: Expression) extends MaxMin
 
   override protected def orderingUpdater(oldExpr: Expression, newExpr: Expression): Expression =
     greatest(oldExpr, newExpr)
+
+  override protected def withNewChildrenInternal(newLeft: Expression, newRight: Expression): MaxBy =
+    copy(valueExpr = newLeft, orderingExpr = newRight)
 }
 
 @ExpressionDescription(
@@ -128,4 +133,7 @@ case class MinBy(valueExpr: Expression, orderingExpr: Expression) extends MaxMin
 
   override protected def orderingUpdater(oldExpr: Expression, newExpr: Expression): Expression =
     least(oldExpr, newExpr)
+
+  override protected def withNewChildrenInternal(newLeft: Expression, newRight: Expression): MinBy =
+    copy(valueExpr = newLeft, orderingExpr = newRight)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala
index 1d861aa0dd8cf..9c5c7bbda4dc7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.expressions.aggregate
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.trees.UnaryLike
 import org.apache.spark.sql.catalyst.util.TypeUtils
 import org.apache.spark.sql.types._
 
@@ -32,9 +33,7 @@ import org.apache.spark.sql.types._
   """,
   group = "agg_funcs",
   since = "1.0.0")
-case class Min(child: Expression) extends DeclarativeAggregate {
-
-  override def children: Seq[Expression] = child :: Nil
+case class Min(child: Expression) extends DeclarativeAggregate with UnaryLike[Expression] {
 
   override def nullable: Boolean = true
 
@@ -63,4 +62,6 @@ case class Min(child: Expression) extends DeclarativeAggregate {
   }
 
   override lazy val evaluateExpression: AttributeReference = min
+
+  override protected def withNewChildInternal(newChild: Expression): Min = copy(child = newChild)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Percentile.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Percentile.scala
index 0eba61c741133..5bce4d348c726 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Percentile.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Percentile.scala
@@ -20,12 +20,13 @@ package org.apache.spark.sql.catalyst.expressions.aggregate
 import java.io.{ByteArrayInputStream, ByteArrayOutputStream, DataInputStream, DataOutputStream}
 import java.util
 
-import org.apache.spark.SparkException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.trees.TernaryLike
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.types._
 import org.apache.spark.util.collection.OpenHashMap
 
@@ -70,7 +71,8 @@ case class Percentile(
     frequencyExpression : Expression,
     mutableAggBufferOffset: Int = 0,
     inputAggBufferOffset: Int = 0)
-  extends TypedImperativeAggregate[OpenHashMap[AnyRef, Long]] with ImplicitCastInputTypes {
+  extends TypedImperativeAggregate[OpenHashMap[AnyRef, Long]] with ImplicitCastInputTypes
+  with TernaryLike[Expression] {
 
   def this(child: Expression, percentageExpression: Expression) = {
     this(child, percentageExpression, Literal(1L), 0, 0)
@@ -99,9 +101,9 @@ case class Percentile(
     case arrayData: ArrayData => arrayData.toDoubleArray()
   }
 
-  override def children: Seq[Expression] = {
-    child :: percentageExpression :: frequencyExpression :: Nil
-  }
+  override def first: Expression = child
+  override def second: Expression = percentageExpression
+  override def third: Expression = frequencyExpression
 
   // Returns null for empty inputs
   override def nullable: Boolean = true
@@ -165,7 +167,7 @@ case class Percentile(
       if (frqLong > 0) {
         buffer.changeValue(key, frqLong, _ + frqLong)
       } else if (frqLong < 0) {
-        throw new SparkException(s"Negative values found in ${frequencyExpression.sql}")
+        throw QueryExecutionErrors.negativeValueUnexpectedError(frequencyExpression)
       }
     }
     buffer
@@ -191,13 +193,13 @@ case class Percentile(
 
     val sortedCounts = buffer.toSeq.sortBy(_._1)(
       child.dataType.asInstanceOf[NumericType].ordering.asInstanceOf[Ordering[AnyRef]])
-    val accumlatedCounts = sortedCounts.scanLeft((sortedCounts.head._1, 0L)) {
+    val accumulatedCounts = sortedCounts.scanLeft((sortedCounts.head._1, 0L)) {
       case ((key1, count1), (key2, count2)) => (key2, count1 + count2)
     }.tail
-    val maxPosition = accumlatedCounts.last._2 - 1
+    val maxPosition = accumulatedCounts.last._2 - 1
 
     percentages.map { percentile =>
-      getPercentile(accumlatedCounts, maxPosition * percentile)
+      getPercentile(accumulatedCounts, maxPosition * percentile)
     }
   }
 
@@ -302,4 +304,11 @@ case class Percentile(
       bis.close()
     }
   }
+
+  override protected def withNewChildrenInternal(
+      newFirst: Expression, newSecond: Expression, newThird: Expression): Percentile = copy(
+    child = newFirst,
+    percentageExpression = newSecond,
+    frequencyExpression = newThird
+  )
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/PivotFirst.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/PivotFirst.scala
index 17471535873fc..b90e46e1545d8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/PivotFirst.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/PivotFirst.scala
@@ -21,6 +21,7 @@ import scala.collection.immutable.{HashMap, TreeMap}
 
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.trees.BinaryLike
 import org.apache.spark.sql.catalyst.util.{GenericArrayData, TypeUtils}
 import org.apache.spark.sql.types._
 
@@ -73,9 +74,10 @@ case class PivotFirst(
   valueColumn: Expression,
   pivotColumnValues: Seq[Any],
   mutableAggBufferOffset: Int = 0,
-  inputAggBufferOffset: Int = 0) extends ImperativeAggregate {
+  inputAggBufferOffset: Int = 0) extends ImperativeAggregate with BinaryLike[Expression] {
 
-  override val children: Seq[Expression] = pivotColumn :: valueColumn :: Nil
+  override val left: Expression = pivotColumn
+  override val right: Expression = valueColumn
 
   override val nullable: Boolean = false
 
@@ -151,5 +153,9 @@ case class PivotFirst(
 
   override val inputAggBufferAttributes: Seq[AttributeReference] =
     aggBufferAttributes.map(_.newInstance())
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): PivotFirst =
+    copy(pivotColumn = newLeft, valueColumn = newRight)
 }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Product.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Product.scala
new file mode 100644
index 0000000000000..3af3944fd47d7
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Product.scala
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions.aggregate
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, ImplicitCastInputTypes, Literal}
+import org.apache.spark.sql.catalyst.trees.UnaryLike
+import org.apache.spark.sql.types.{AbstractDataType, DataType, DoubleType}
+
+
+/** Multiply numerical values within an aggregation group */
+case class Product(child: Expression)
+    extends DeclarativeAggregate with ImplicitCastInputTypes with UnaryLike[Expression] {
+
+  override def nullable: Boolean = true
+
+  override def dataType: DataType = DoubleType
+
+  override def inputTypes: Seq[AbstractDataType] = Seq(DoubleType)
+
+  private lazy val product = AttributeReference("product", dataType)()
+
+  private lazy val one = Literal(1.0, dataType)
+
+  override lazy val aggBufferAttributes = product :: Nil
+
+  override lazy val initialValues: Seq[Expression] =
+    Seq(Literal(null, dataType))
+
+  override lazy val updateExpressions: Seq[Expression] = {
+    // Treat the result as null until we have seen at least one child value,
+    // whereupon the previous product is promoted to being unity.
+
+    val protoResult = coalesce(product, one) * child
+
+    if (child.nullable) {
+      Seq(coalesce(protoResult, product))
+    } else {
+      Seq(protoResult)
+    }
+  }
+
+  override lazy val mergeExpressions: Seq[Expression] =
+    Seq(coalesce(coalesce(product.left, one) * product.right, product.left))
+
+  override lazy val evaluateExpression: Expression = product
+
+  override protected def withNewChildInternal(newChild: Expression): Product =
+    copy(child = newChild)
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala
index d2daaac72fc85..16cd9d76f7b14 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala
@@ -20,6 +20,8 @@ package org.apache.spark.sql.catalyst.expressions.aggregate
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.trees.TreePattern.{SUM, TreePattern}
+import org.apache.spark.sql.catalyst.trees.UnaryLike
 import org.apache.spark.sql.catalyst.util.TypeUtils
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
@@ -37,63 +39,121 @@ import org.apache.spark.sql.types._
   """,
   group = "agg_funcs",
   since = "1.0.0")
-case class Sum(child: Expression) extends DeclarativeAggregate with ImplicitCastInputTypes {
-
-  override def children: Seq[Expression] = child :: Nil
+case class Sum(child: Expression) extends DeclarativeAggregate with ImplicitCastInputTypes
+  with UnaryLike[Expression] {
 
   override def nullable: Boolean = true
 
   // Return data type.
   override def dataType: DataType = resultType
 
-  override def inputTypes: Seq[AbstractDataType] = Seq(NumericType)
+  override def inputTypes: Seq[AbstractDataType] =
+    Seq(TypeCollection(NumericType, YearMonthIntervalType, DayTimeIntervalType))
 
   override def checkInputDataTypes(): TypeCheckResult =
-    TypeUtils.checkForNumericExpr(child.dataType, "function sum")
+    TypeUtils.checkForAnsiIntervalOrNumericType(child.dataType, "sum")
+
+  final override val nodePatterns: Seq[TreePattern] = Seq(SUM)
 
   private lazy val resultType = child.dataType match {
     case DecimalType.Fixed(precision, scale) =>
       DecimalType.bounded(precision + 10, scale)
     case _: IntegralType => LongType
+    case _: YearMonthIntervalType => YearMonthIntervalType
+    case _: DayTimeIntervalType => DayTimeIntervalType
     case _ => DoubleType
   }
 
-  private lazy val sumDataType = resultType
+  private lazy val sum = AttributeReference("sum", resultType)()
 
-  private lazy val sum = AttributeReference("sum", sumDataType)()
+  private lazy val isEmpty = AttributeReference("isEmpty", BooleanType, nullable = false)()
 
-  private lazy val zero = Literal.default(sumDataType)
+  private lazy val zero = Literal.default(resultType)
 
-  override lazy val aggBufferAttributes = sum :: Nil
+  override lazy val aggBufferAttributes = resultType match {
+    case _: DecimalType => sum :: isEmpty :: Nil
+    case _ => sum :: Nil
+  }
 
-  override lazy val initialValues: Seq[Expression] = Seq(
-    /* sum = */ Literal.create(null, sumDataType)
-  )
+  override lazy val initialValues: Seq[Expression] = resultType match {
+    case _: DecimalType => Seq(zero, Literal(true, BooleanType))
+    case _ => Seq(Literal(null, resultType))
+  }
 
   override lazy val updateExpressions: Seq[Expression] = {
-    if (child.nullable) {
-      Seq(
-        /* sum = */
-        coalesce(coalesce(sum, zero) + child.cast(sumDataType), sum)
-      )
-    } else {
-      Seq(
-        /* sum = */
-        coalesce(sum, zero) + child.cast(sumDataType)
-      )
+    resultType match {
+      case _: DecimalType =>
+        // For decimal type, the initial value of `sum` is 0. We need to keep `sum` unchanged if
+        // the input is null, as SUM function ignores null input. The `sum` can only be null if
+        // overflow happens under non-ansi mode.
+        val sumExpr = if (child.nullable) {
+          If(child.isNull, sum, sum + KnownNotNull(child).cast(resultType))
+        } else {
+          sum + child.cast(resultType)
+        }
+        // The buffer becomes non-empty after seeing the first not-null input.
+        val isEmptyExpr = if (child.nullable) {
+          isEmpty && child.isNull
+        } else {
+          Literal(false, BooleanType)
+        }
+        Seq(sumExpr, isEmptyExpr)
+      case _ =>
+        // For non-decimal type, the initial value of `sum` is null, which indicates no value.
+        // We need `coalesce(sum, zero)` to start summing values. And we need an outer `coalesce`
+        // in case the input is nullable. The `sum` can only be null if there is no value, as
+        // non-decimal type can produce overflowed value under non-ansi mode.
+        if (child.nullable) {
+          Seq(coalesce(coalesce(sum, zero) + child.cast(resultType), sum))
+        } else {
+          Seq(coalesce(sum, zero) + child.cast(resultType))
+        }
     }
   }
 
+  /**
+   * For decimal type:
+   * If isEmpty is false and if sum is null, then it means we have had an overflow.
+   *
+   * update of the sum is as follows:
+   * Check if either portion of the left.sum or right.sum has overflowed
+   * If it has, then the sum value will remain null.
+   * If it did not have overflow, then add the sum.left and sum.right
+   *
+   * isEmpty:  Set to false if either one of the left or right is set to false. This
+   * means we have seen atleast a value that was not null.
+   */
   override lazy val mergeExpressions: Seq[Expression] = {
-    Seq(
-      /* sum = */
-      coalesce(coalesce(sum.left, zero) + sum.right, sum.left)
-    )
+    resultType match {
+      case _: DecimalType =>
+        val bufferOverflow = !isEmpty.left && sum.left.isNull
+        val inputOverflow = !isEmpty.right && sum.right.isNull
+        Seq(
+          If(
+            bufferOverflow || inputOverflow,
+            Literal.create(null, resultType),
+            // If both the buffer and the input do not overflow, just add them, as they can't be
+            // null. See the comments inside `updateExpressions`: `sum` can only be null if
+            // overflow happens.
+            KnownNotNull(sum.left) + KnownNotNull(sum.right)),
+          isEmpty.left && isEmpty.right)
+      case _ => Seq(coalesce(coalesce(sum.left, zero) + sum.right, sum.left))
+    }
   }
 
+  /**
+   * If the isEmpty is true, then it means there were no values to begin with or all the values
+   * were null, so the result will be null.
+   * If the isEmpty is false, then if sum is null that means an overflow has happened.
+   * So now, if ansi is enabled, then throw exception, if not then return null.
+   * If sum is not null, then return the sum.
+   */
   override lazy val evaluateExpression: Expression = resultType match {
-    case d: DecimalType => CheckOverflow(sum, d, !SQLConf.get.ansiEnabled)
+    case d: DecimalType =>
+      If(isEmpty, Literal.create(null, resultType),
+        CheckOverflowInSum(sum, d, !SQLConf.get.ansiEnabled))
     case _ => sum
   }
 
+  override protected def withNewChildInternal(newChild: Expression): Sum = copy(child = newChild)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/UnevaluableAggs.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/UnevaluableAggs.scala
index cb77ded3372a2..244e9d9755752 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/UnevaluableAggs.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/UnevaluableAggs.scala
@@ -19,17 +19,21 @@ package org.apache.spark.sql.catalyst.expressions.aggregate
 
 import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult}
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.trees.TreePattern.{BOOL_AGG, TreePattern}
+import org.apache.spark.sql.catalyst.trees.UnaryLike
 import org.apache.spark.sql.types._
 
 abstract class UnevaluableBooleanAggBase(arg: Expression)
-  extends UnevaluableAggregate with ImplicitCastInputTypes {
+  extends UnevaluableAggregate with ImplicitCastInputTypes with UnaryLike[Expression] {
 
-  override def children: Seq[Expression] = arg :: Nil
+  override def child: Expression = arg
 
   override def dataType: DataType = BooleanType
 
   override def inputTypes: Seq[AbstractDataType] = Seq(BooleanType)
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(BOOL_AGG)
+
   override def checkInputDataTypes(): TypeCheckResult = {
     arg.dataType match {
       case dt if dt != BooleanType =>
@@ -55,6 +59,8 @@ abstract class UnevaluableBooleanAggBase(arg: Expression)
   since = "3.0.0")
 case class BoolAnd(arg: Expression) extends UnevaluableBooleanAggBase(arg) {
   override def nodeName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("bool_and")
+  override protected def withNewChildInternal(newChild: Expression): Expression =
+    copy(arg = newChild)
 }
 
 @ExpressionDescription(
@@ -72,4 +78,6 @@ case class BoolAnd(arg: Expression) extends UnevaluableBooleanAggBase(arg) {
   since = "3.0.0")
 case class BoolOr(arg: Expression) extends UnevaluableBooleanAggBase(arg) {
   override def nodeName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("bool_or")
+  override protected def withNewChildInternal(newChild: Expression): Expression =
+    copy(arg = newChild)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/V2Aggregator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/V2Aggregator.scala
new file mode 100644
index 0000000000000..55e3f504ae2e6
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/V2Aggregator.scala
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions.aggregate
+
+import java.io.{ByteArrayInputStream, ByteArrayOutputStream, ObjectInputStream, ObjectOutputStream}
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.{Expression, UnsafeProjection}
+import org.apache.spark.sql.connector.catalog.functions.{AggregateFunction => V2AggregateFunction}
+import org.apache.spark.sql.types.DataType
+
+case class V2Aggregator[BUF <: java.io.Serializable, OUT](
+    aggrFunc: V2AggregateFunction[BUF, OUT],
+    children: Seq[Expression],
+    mutableAggBufferOffset: Int = 0,
+    inputAggBufferOffset: Int = 0) extends TypedImperativeAggregate[BUF] {
+  private[this] lazy val inputProjection = UnsafeProjection.create(children)
+
+  override def nullable: Boolean = aggrFunc.isResultNullable
+  override def dataType: DataType = aggrFunc.resultType()
+  override def createAggregationBuffer(): BUF = aggrFunc.newAggregationState()
+
+  override def update(buffer: BUF, input: InternalRow): BUF = {
+    aggrFunc.update(buffer, inputProjection(input))
+  }
+
+  override def merge(buffer: BUF, input: BUF): BUF = aggrFunc.merge(buffer, input)
+
+  override def eval(buffer: BUF): Any = {
+    aggrFunc.produceResult(buffer)
+  }
+
+  override def serialize(buffer: BUF): Array[Byte] = {
+    val bos = new ByteArrayOutputStream()
+    val out = new ObjectOutputStream(bos)
+    out.writeObject(buffer)
+    out.close()
+    bos.toByteArray
+  }
+
+  override def deserialize(bytes: Array[Byte]): BUF = {
+    val in = new ObjectInputStream(new ByteArrayInputStream(bytes))
+    in.readObject().asInstanceOf[BUF]
+  }
+
+  def withNewMutableAggBufferOffset(newMutableAggBufferOffset: Int): V2Aggregator[BUF, OUT] =
+    copy(mutableAggBufferOffset = newMutableAggBufferOffset)
+
+  def withNewInputAggBufferOffset(newInputAggBufferOffset: Int): V2Aggregator[BUF, OUT] =
+    copy(inputAggBufferOffset = newInputAggBufferOffset)
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
+    copy(children = newChildren)
+}
+
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala
index b4c1b2c708fb2..86a16ad389b5d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala
@@ -18,16 +18,16 @@
 package org.apache.spark.sql.catalyst.expressions.aggregate
 
 import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BinaryArithmetic, BitwiseAnd, BitwiseOr, BitwiseXor, ExpectsInputTypes, Expression, ExpressionDescription, If, IsNull, Literal}
+import org.apache.spark.sql.catalyst.trees.UnaryLike
 import org.apache.spark.sql.types.{AbstractDataType, DataType, IntegralType}
 
-abstract class BitAggregate extends DeclarativeAggregate with ExpectsInputTypes {
+abstract class BitAggregate extends DeclarativeAggregate with ExpectsInputTypes
+  with UnaryLike[Expression] {
 
   val child: Expression
 
   def bitOperator(left: Expression, right: Expression): BinaryArithmetic
 
-  override def children: Seq[Expression] = child :: Nil
-
   override def nullable: Boolean = true
 
   override def dataType: DataType = child.dataType
@@ -60,6 +60,7 @@ abstract class BitAggregate extends DeclarativeAggregate with ExpectsInputTypes
       > SELECT _FUNC_(col) FROM VALUES (3), (5) AS tab(col);
        1
   """,
+  group = "agg_funcs",
   since = "3.0.0")
 case class BitAndAgg(child: Expression) extends BitAggregate {
 
@@ -68,6 +69,9 @@ case class BitAndAgg(child: Expression) extends BitAggregate {
   override def bitOperator(left: Expression, right: Expression): BinaryArithmetic = {
     BitwiseAnd(left, right)
   }
+
+  override protected def withNewChildInternal(newChild: Expression): BitAndAgg =
+    copy(child = newChild)
 }
 
 @ExpressionDescription(
@@ -86,6 +90,9 @@ case class BitOrAgg(child: Expression) extends BitAggregate {
   override def bitOperator(left: Expression, right: Expression): BinaryArithmetic = {
     BitwiseOr(left, right)
   }
+
+  override protected def withNewChildInternal(newChild: Expression): BitOrAgg =
+    copy(child = newChild)
 }
 
 @ExpressionDescription(
@@ -104,4 +111,7 @@ case class BitXorAgg(child: Expression) extends BitAggregate {
   override def bitOperator(left: Expression, right: Expression): BinaryArithmetic = {
     BitwiseXor(left, right)
   }
+
+  override protected def withNewChildInternal(newChild: Expression): Expression =
+    copy(child = newChild)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala
index 0a3d87623be8b..a8db8211a9e4d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala
@@ -23,6 +23,7 @@ import scala.collection.mutable
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.trees.UnaryLike
 import org.apache.spark.sql.catalyst.util.ArrayData
 import org.apache.spark.sql.catalyst.util.GenericArrayData
 import org.apache.spark.sql.types._
@@ -33,13 +34,12 @@ import org.apache.spark.sql.types._
  * We have to store all the collected elements in memory, and so notice that too many elements
  * can cause GC paused and eventually OutOfMemory Errors.
  */
-abstract class Collect[T <: Growable[Any] with Iterable[Any]] extends TypedImperativeAggregate[T] {
+abstract class Collect[T <: Growable[Any] with Iterable[Any]] extends TypedImperativeAggregate[T]
+  with UnaryLike[Expression] {
 
   val child: Expression
 
-  override def children: Seq[Expression] = child :: Nil
-
-  override def nullable: Boolean = true
+  override def nullable: Boolean = false
 
   override def dataType: DataType = ArrayType(child.dataType, false)
 
@@ -47,6 +47,8 @@ abstract class Collect[T <: Growable[Any] with Iterable[Any]] extends TypedImper
   // actual order of input rows.
   override lazy val deterministic: Boolean = false
 
+  override def defaultResult: Option[Literal] = Option(Literal.create(Array(), dataType))
+
   protected def convertToBufferElement(value: Any): Any
 
   override def update(buffer: T, input: InternalRow): T = {
@@ -123,6 +125,9 @@ case class CollectList(
   override def eval(buffer: mutable.ArrayBuffer[Any]): Any = {
     new GenericArrayData(buffer.toArray)
   }
+
+  override protected def withNewChildInternal(newChild: Expression): CollectList =
+    copy(child = newChild)
 }
 
 /**
@@ -189,4 +194,7 @@ case class CollectSet(
   override def prettyName: String = "collect_set"
 
   override def createAggregationBuffer(): mutable.HashSet[Any] = mutable.HashSet.empty
+
+  override protected def withNewChildInternal(newChild: Expression): CollectSet =
+    copy(child = newChild)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala
index 222ad6fab19e0..8c70c86aa1868 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala
@@ -20,7 +20,8 @@ package org.apache.spark.sql.catalyst.expressions.aggregate
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
 import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
+import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodegenFallback, ExprCode}
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.types._
 
 /** The mode of an [[AggregateFunction]]. */
@@ -61,10 +62,9 @@ case object Complete extends AggregateMode
  * A place holder expressions used in code-gen, it does not change the corresponding value
  * in the row.
  */
-case object NoOp extends Expression with Unevaluable {
+case object NoOp extends LeafExpression with Unevaluable {
   override def nullable: Boolean = true
   override def dataType: DataType = NullType
-  override def children: Seq[Expression] = Nil
 }
 
 object AggregateExpression {
@@ -80,6 +80,14 @@ object AggregateExpression {
       filter,
       NamedExpression.newExprId)
   }
+
+  def containsAggregate(expr: Expression): Boolean = {
+    expr.find(isAggregate).isDefined
+  }
+
+  def isAggregate(expr: Expression): Boolean = {
+    expr.isInstanceOf[AggregateExpression] || PythonUDF.isGroupedAggPandasUDF(expr)
+  }
 }
 
 /**
@@ -133,14 +141,13 @@ case class AggregateExpression(
   override def children: Seq[Expression] = aggregateFunction +: filter.toSeq
 
   override def dataType: DataType = aggregateFunction.dataType
-  override def foldable: Boolean = false
   override def nullable: Boolean = aggregateFunction.nullable
 
   @transient
   override lazy val references: AttributeSet = {
     val aggAttributes = mode match {
       case Partial | Complete => aggregateFunction.references
-      case PartialMerge | Final => AttributeSet(aggregateFunction.aggBufferAttributes)
+      case PartialMerge | Final => AttributeSet(aggregateFunction.inputAggBufferAttributes)
     }
     aggAttributes ++ filterAttributes
   }
@@ -165,6 +172,16 @@ case class AggregateExpression(
       case _ => aggFuncStr
     }
   }
+
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[Expression]): AggregateExpression =
+    if (filter.isDefined) {
+      copy(
+        aggregateFunction = newChildren(0).asInstanceOf[AggregateFunction],
+        filter = Some(newChildren(1)))
+    } else {
+      copy(aggregateFunction = newChildren(0).asInstanceOf[AggregateFunction])
+    }
 }
 
 /**
@@ -203,8 +220,7 @@ abstract class AggregateFunction extends Expression {
   def inputAggBufferAttributes: Seq[AttributeReference]
 
   /**
-   * Result of the aggregate function when the input is empty. This is currently only used for the
-   * proper rewriting of distinct aggregate functions.
+   * Result of the aggregate function when the input is empty.
    */
   def defaultResult: Option[Literal] = None
 
@@ -223,8 +239,14 @@ abstract class AggregateFunction extends Expression {
    * An [[AggregateFunction]] should not be used without being wrapped in
    * an [[AggregateExpression]].
    */
-  def toAggregateExpression(isDistinct: Boolean): AggregateExpression = {
-    AggregateExpression(aggregateFunction = this, mode = Complete, isDistinct = isDistinct)
+  def toAggregateExpression(
+      isDistinct: Boolean,
+      filter: Option[Expression] = None): AggregateExpression = {
+    AggregateExpression(
+      aggregateFunction = this,
+      mode = Complete,
+      isDistinct = isDistinct,
+      filter = filter)
   }
 
   def sql(isDistinct: Boolean): String = {
@@ -368,8 +390,7 @@ abstract class ImperativeAggregate extends AggregateFunction with CodegenFallbac
  */
 abstract class DeclarativeAggregate
   extends AggregateFunction
-  with Serializable
-  with Unevaluable {
+  with Serializable {
 
   /**
    * Expressions for initializing empty aggregation buffers.
@@ -415,6 +436,12 @@ abstract class DeclarativeAggregate
     /** Represents this attribute at the input buffer side (the data value is read-only). */
     def right: AttributeReference = inputAggBufferAttributes(aggBufferAttributes.indexOf(a))
   }
+
+  final override def eval(input: InternalRow = null): Any =
+    throw QueryExecutionErrors.cannotEvaluateExpressionError(this)
+
+  final override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode =
+    throw QueryExecutionErrors.cannotGenerateCodeForExpressionError(this)
 }
 
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
index 354845d5ccd8b..ec220de0582da 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
@@ -21,7 +21,10 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult, TypeCoercion}
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
+import org.apache.spark.sql.catalyst.trees.TreePattern.{BINARY_ARITHMETIC, TreePattern,
+  UNARY_POSITIVE}
 import org.apache.spark.sql.catalyst.util.{IntervalUtils, TypeUtils}
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.CalendarInterval
@@ -32,10 +35,15 @@ import org.apache.spark.unsafe.types.CalendarInterval
     Examples:
       > SELECT _FUNC_(1);
        -1
-  """)
-case class UnaryMinus(child: Expression) extends UnaryExpression
-    with ExpectsInputTypes with NullIntolerant {
-  private val checkOverflow = SQLConf.get.ansiEnabled
+  """,
+  since = "1.0.0",
+  group = "math_funcs")
+case class UnaryMinus(
+    child: Expression,
+    failOnError: Boolean = SQLConf.get.ansiEnabled)
+  extends UnaryExpression with ExpectsInputTypes with NullIntolerant {
+
+  def this(child: Expression) = this(child, SQLConf.get.ansiEnabled)
 
   override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection.NumericAndInterval)
 
@@ -43,11 +51,11 @@ case class UnaryMinus(child: Expression) extends UnaryExpression
 
   override def toString: String = s"-$child"
 
-  private lazy val numeric = TypeUtils.getNumeric(dataType, checkOverflow)
+  private lazy val numeric = TypeUtils.getNumeric(dataType, failOnError)
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = dataType match {
     case _: DecimalType => defineCodeGen(ctx, ev, c => s"$c.unary_$$minus()")
-    case ByteType | ShortType if checkOverflow =>
+    case ByteType | ShortType if failOnError =>
       nullSafeCodeGen(ctx, ev, eval => {
         val javaBoxedType = CodeGenerator.boxedType(dataType)
         val javaType = CodeGenerator.javaType(dataType)
@@ -55,12 +63,12 @@ case class UnaryMinus(child: Expression) extends UnaryExpression
         s"""
            |$javaType $originValue = ($javaType)($eval);
            |if ($originValue == $javaBoxedType.MIN_VALUE) {
-           |  throw new ArithmeticException("- " + $originValue + " caused overflow.");
+           |  throw QueryExecutionErrors.unaryMinusCauseOverflowError($originValue);
            |}
            |${ev.value} = ($javaType)(-($originValue));
            """.stripMargin
       })
-    case IntegerType | LongType if checkOverflow =>
+    case IntegerType | LongType if failOnError =>
       nullSafeCodeGen(ctx, ev, eval => {
         val mathClass = classOf[Math].getName
         s"${ev.value} = $mathClass.negateExact($eval);"
@@ -75,14 +83,21 @@ case class UnaryMinus(child: Expression) extends UnaryExpression
       """})
     case _: CalendarIntervalType =>
       val iu = IntervalUtils.getClass.getCanonicalName.stripSuffix("$")
-      val method = if (checkOverflow) "negateExact" else "negate"
+      val method = if (failOnError) "negateExact" else "negate"
       defineCodeGen(ctx, ev, c => s"$iu.$method($c)")
+    case DayTimeIntervalType | YearMonthIntervalType =>
+      nullSafeCodeGen(ctx, ev, eval => {
+        val mathClass = classOf[Math].getName
+        s"${ev.value} = $mathClass.negateExact($eval);"
+      })
   }
 
   protected override def nullSafeEval(input: Any): Any = dataType match {
-    case CalendarIntervalType if checkOverflow =>
+    case CalendarIntervalType if failOnError =>
       IntervalUtils.negateExact(input.asInstanceOf[CalendarInterval])
     case CalendarIntervalType => IntervalUtils.negate(input.asInstanceOf[CalendarInterval])
+    case DayTimeIntervalType => Math.negateExact(input.asInstanceOf[Long])
+    case YearMonthIntervalType => Math.negateExact(input.asInstanceOf[Int])
     case _ => numeric.negate(input)
   }
 
@@ -92,24 +107,40 @@ case class UnaryMinus(child: Expression) extends UnaryExpression
       case funcName => s"$funcName(${child.sql})"
     }
   }
+
+  override protected def withNewChildInternal(newChild: Expression): UnaryMinus =
+    copy(child = newChild)
 }
 
 @ExpressionDescription(
-  usage = "_FUNC_(expr) - Returns the value of `expr`.")
+  usage = "_FUNC_(expr) - Returns the value of `expr`.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_(1);
+       1
+  """,
+  since = "1.5.0",
+  group = "math_funcs")
 case class UnaryPositive(child: Expression)
-    extends UnaryExpression with ExpectsInputTypes with NullIntolerant {
+  extends UnaryExpression with ExpectsInputTypes with NullIntolerant {
+
   override def prettyName: String = "positive"
 
   override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection.NumericAndInterval)
 
   override def dataType: DataType = child.dataType
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(UNARY_POSITIVE)
+
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode =
     defineCodeGen(ctx, ev, c => c)
 
   protected override def nullSafeEval(input: Any): Any = input
 
   override def sql: String = s"(+ ${child.sql})"
+
+  override protected def withNewChildInternal(newChild: Expression): UnaryPositive =
+    copy(child = newChild)
 }
 
 /**
@@ -121,41 +152,70 @@ case class UnaryPositive(child: Expression)
     Examples:
       > SELECT _FUNC_(-1);
        1
-  """)
-case class Abs(child: Expression)
-    extends UnaryExpression with ExpectsInputTypes with NullIntolerant {
+  """,
+  since = "1.2.0",
+  group = "math_funcs")
+case class Abs(child: Expression, failOnError: Boolean = SQLConf.get.ansiEnabled)
+  extends UnaryExpression with ExpectsInputTypes with NullIntolerant {
+
+  def this(child: Expression) = this(child, SQLConf.get.ansiEnabled)
 
   override def inputTypes: Seq[AbstractDataType] = Seq(NumericType)
 
   override def dataType: DataType = child.dataType
 
-  private lazy val numeric = TypeUtils.getNumeric(dataType)
+  private lazy val numeric = TypeUtils.getNumeric(dataType, failOnError)
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = dataType match {
     case _: DecimalType =>
       defineCodeGen(ctx, ev, c => s"$c.abs()")
+
+    case ByteType | ShortType if failOnError =>
+      val javaBoxedType = CodeGenerator.boxedType(dataType)
+      val javaType = CodeGenerator.javaType(dataType)
+      nullSafeCodeGen(ctx, ev, eval =>
+        s"""
+          |if ($eval == $javaBoxedType.MIN_VALUE) {
+          |  throw QueryExecutionErrors.unaryMinusCauseOverflowError($eval);
+          |} else if ($eval < 0) {
+          |  ${ev.value} = ($javaType)-$eval;
+          |} else {
+          |  ${ev.value} = $eval;
+          |}
+          |""".stripMargin)
+
+    case IntegerType | LongType if failOnError =>
+      defineCodeGen(ctx, ev, c => s"$c < 0 ? java.lang.Math.negateExact($c) : $c")
+
+
     case dt: NumericType =>
       defineCodeGen(ctx, ev, c => s"(${CodeGenerator.javaType(dt)})(java.lang.Math.abs($c))")
   }
 
   protected override def nullSafeEval(input: Any): Any = numeric.abs(input)
+
+  override protected def withNewChildInternal(newChild: Expression): Abs = copy(child = newChild)
 }
 
 abstract class BinaryArithmetic extends BinaryOperator with NullIntolerant {
 
-  protected val checkOverflow = SQLConf.get.ansiEnabled
+  protected val failOnError: Boolean
 
   override def dataType: DataType = left.dataType
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(BINARY_ARITHMETIC)
+
   override lazy val resolved: Boolean = childrenResolved && checkInputDataTypes().isSuccess
 
   /** Name of the function for this expression on a [[Decimal]] type. */
   def decimalMethod: String =
-    sys.error("BinaryArithmetics must override either decimalMethod or genCode")
+    throw QueryExecutionErrors.notOverrideExpectedMethodsError("BinaryArithmetics",
+      "decimalMethod", "genCode")
 
   /** Name of the function for this expression on a [[CalendarInterval]] type. */
   def calendarIntervalMethod: String =
-    sys.error("BinaryArithmetics must override either calendarIntervalMethod or genCode")
+    throw QueryExecutionErrors.notOverrideExpectedMethodsError("BinaryArithmetics",
+      "calendarIntervalMethod", "genCode")
 
   // Name of the function for the exact version of this expression in [[Math]].
   // If the option "spark.sql.ansi.enabled" is enabled and there is corresponding
@@ -169,15 +229,22 @@ abstract class BinaryArithmetic extends BinaryOperator with NullIntolerant {
     case CalendarIntervalType =>
       val iu = IntervalUtils.getClass.getCanonicalName.stripSuffix("$")
       defineCodeGen(ctx, ev, (eval1, eval2) => s"$iu.$calendarIntervalMethod($eval1, $eval2)")
+    case DayTimeIntervalType | YearMonthIntervalType =>
+      assert(exactMathMethod.isDefined,
+        s"The expression '$nodeName' must override the exactMathMethod() method " +
+        "if it is supposed to operate over interval types.")
+      val mathClass = classOf[Math].getName
+      defineCodeGen(ctx, ev, (eval1, eval2) => s"$mathClass.${exactMathMethod.get}($eval1, $eval2)")
     // byte and short are casted into int when add, minus, times or divide
     case ByteType | ShortType =>
       nullSafeCodeGen(ctx, ev, (eval1, eval2) => {
         val tmpResult = ctx.freshName("tmpResult")
-        val overflowCheck = if (checkOverflow) {
+        val overflowCheck = if (failOnError) {
           val javaType = CodeGenerator.boxedType(dataType)
           s"""
              |if ($tmpResult < $javaType.MIN_VALUE || $tmpResult > $javaType.MAX_VALUE) {
-             |  throw new ArithmeticException($eval1 + " $symbol " + $eval2 + " caused overflow.");
+             |  throw QueryExecutionErrors.binaryArithmeticCauseOverflowError(
+             |  $eval1, "$symbol", $eval2);
              |}
            """.stripMargin
         } else {
@@ -191,7 +258,7 @@ abstract class BinaryArithmetic extends BinaryOperator with NullIntolerant {
       })
     case IntegerType | LongType =>
       nullSafeCodeGen(ctx, ev, (eval1, eval2) => {
-        val operation = if (checkOverflow && exactMathMethod.isDefined) {
+        val operation = if (failOnError && exactMathMethod.isDefined) {
           val mathClass = classOf[Math].getName
           s"$mathClass.${exactMathMethod.get}($eval1, $eval2)"
         } else {
@@ -204,7 +271,7 @@ abstract class BinaryArithmetic extends BinaryOperator with NullIntolerant {
     case DoubleType | FloatType =>
       // When Double/Float overflows, there can be 2 cases:
       // - precision loss: according to SQL standard, the number is truncated;
-      // - returns (+/-)Infinite: same behavior also other DBs have (eg. Postgres)
+      // - returns (+/-)Infinite: same behavior also other DBs have (e.g. Postgres)
       nullSafeCodeGen(ctx, ev, (eval1, eval2) => {
         s"""
            |${ev.value} = $eval1 $symbol $eval2;
@@ -223,8 +290,15 @@ object BinaryArithmetic {
     Examples:
       > SELECT 1 _FUNC_ 2;
        3
-  """)
-case class Add(left: Expression, right: Expression) extends BinaryArithmetic {
+  """,
+  since = "1.0.0",
+  group = "math_funcs")
+case class Add(
+    left: Expression,
+    right: Expression,
+    failOnError: Boolean = SQLConf.get.ansiEnabled) extends BinaryArithmetic {
+
+  def this(left: Expression, right: Expression) = this(left, right, SQLConf.get.ansiEnabled)
 
   override def inputType: AbstractDataType = TypeCollection.NumericAndInterval
 
@@ -232,21 +306,28 @@ case class Add(left: Expression, right: Expression) extends BinaryArithmetic {
 
   override def decimalMethod: String = "$plus"
 
-  override def calendarIntervalMethod: String = if (checkOverflow) "addExact" else "add"
+  override def calendarIntervalMethod: String = if (failOnError) "addExact" else "add"
 
-  private lazy val numeric = TypeUtils.getNumeric(dataType, checkOverflow)
+  private lazy val numeric = TypeUtils.getNumeric(dataType, failOnError)
 
   protected override def nullSafeEval(input1: Any, input2: Any): Any = dataType match {
-    case CalendarIntervalType if checkOverflow =>
+    case CalendarIntervalType if failOnError =>
       IntervalUtils.addExact(
         input1.asInstanceOf[CalendarInterval], input2.asInstanceOf[CalendarInterval])
     case CalendarIntervalType =>
       IntervalUtils.add(
         input1.asInstanceOf[CalendarInterval], input2.asInstanceOf[CalendarInterval])
+    case DayTimeIntervalType =>
+      Math.addExact(input1.asInstanceOf[Long], input2.asInstanceOf[Long])
+    case YearMonthIntervalType =>
+      Math.addExact(input1.asInstanceOf[Int], input2.asInstanceOf[Int])
     case _ => numeric.plus(input1, input2)
   }
 
   override def exactMathMethod: Option[String] = Some("addExact")
+
+  override protected def withNewChildrenInternal(newLeft: Expression, newRight: Expression): Add =
+    copy(left = newLeft, right = newRight)
 }
 
 @ExpressionDescription(
@@ -255,8 +336,15 @@ case class Add(left: Expression, right: Expression) extends BinaryArithmetic {
     Examples:
       > SELECT 2 _FUNC_ 1;
        1
-  """)
-case class Subtract(left: Expression, right: Expression) extends BinaryArithmetic {
+  """,
+  since = "1.0.0",
+  group = "math_funcs")
+case class Subtract(
+    left: Expression,
+    right: Expression,
+    failOnError: Boolean = SQLConf.get.ansiEnabled) extends BinaryArithmetic {
+
+  def this(left: Expression, right: Expression) = this(left, right, SQLConf.get.ansiEnabled)
 
   override def inputType: AbstractDataType = TypeCollection.NumericAndInterval
 
@@ -264,21 +352,28 @@ case class Subtract(left: Expression, right: Expression) extends BinaryArithmeti
 
   override def decimalMethod: String = "$minus"
 
-  override def calendarIntervalMethod: String = if (checkOverflow) "subtractExact" else "subtract"
+  override def calendarIntervalMethod: String = if (failOnError) "subtractExact" else "subtract"
 
-  private lazy val numeric = TypeUtils.getNumeric(dataType, checkOverflow)
+  private lazy val numeric = TypeUtils.getNumeric(dataType, failOnError)
 
   protected override def nullSafeEval(input1: Any, input2: Any): Any = dataType match {
-    case CalendarIntervalType if checkOverflow =>
+    case CalendarIntervalType if failOnError =>
       IntervalUtils.subtractExact(
         input1.asInstanceOf[CalendarInterval], input2.asInstanceOf[CalendarInterval])
     case CalendarIntervalType =>
       IntervalUtils.subtract(
         input1.asInstanceOf[CalendarInterval], input2.asInstanceOf[CalendarInterval])
+    case DayTimeIntervalType =>
+      Math.subtractExact(input1.asInstanceOf[Long], input2.asInstanceOf[Long])
+    case YearMonthIntervalType =>
+      Math.subtractExact(input1.asInstanceOf[Int], input2.asInstanceOf[Int])
     case _ => numeric.minus(input1, input2)
   }
 
   override def exactMathMethod: Option[String] = Some("subtractExact")
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): Subtract = copy(left = newLeft, right = newRight)
 }
 
 @ExpressionDescription(
@@ -287,19 +382,29 @@ case class Subtract(left: Expression, right: Expression) extends BinaryArithmeti
     Examples:
       > SELECT 2 _FUNC_ 3;
        6
-  """)
-case class Multiply(left: Expression, right: Expression) extends BinaryArithmetic {
+  """,
+  since = "1.0.0",
+  group = "math_funcs")
+case class Multiply(
+    left: Expression,
+    right: Expression,
+    failOnError: Boolean = SQLConf.get.ansiEnabled) extends BinaryArithmetic {
+
+  def this(left: Expression, right: Expression) = this(left, right, SQLConf.get.ansiEnabled)
 
   override def inputType: AbstractDataType = NumericType
 
   override def symbol: String = "*"
   override def decimalMethod: String = "$times"
 
-  private lazy val numeric = TypeUtils.getNumeric(dataType, checkOverflow)
+  private lazy val numeric = TypeUtils.getNumeric(dataType, failOnError)
 
   protected override def nullSafeEval(input1: Any, input2: Any): Any = numeric.times(input1, input2)
 
   override def exactMathMethod: Option[String] = Some("multiplyExact")
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): Multiply = copy(left = newLeft, right = newRight)
 }
 
 // Common base trait for Divide and Remainder, since these two classes are almost identical
@@ -307,17 +412,33 @@ trait DivModLike extends BinaryArithmetic {
 
   protected def decimalToDataTypeCodeGen(decimalResult: String): String = decimalResult
 
+  // Whether we should check overflow or not in ANSI mode.
+  protected def checkDivideOverflow: Boolean = false
+
   override def nullable: Boolean = true
 
+  private lazy val isZero: Any => Boolean = right.dataType match {
+    case _: DecimalType => x => x.asInstanceOf[Decimal].isZero
+    case _ => x => x == 0
+  }
+
   final override def eval(input: InternalRow): Any = {
+    // evaluate right first as we have a chance to skip left if right is 0
     val input2 = right.eval(input)
-    if (input2 == null || input2 == 0) {
+    if (input2 == null || (!failOnError && isZero(input2))) {
       null
     } else {
       val input1 = left.eval(input)
       if (input1 == null) {
         null
       } else {
+        if (isZero(input2)) {
+          // when we reach here, failOnError must bet true.
+          throw QueryExecutionErrors.divideByZeroError
+        }
+        if (checkDivideOverflow && input1 == Long.MinValue && input2 == -1) {
+          throw QueryExecutionErrors.overflowInIntegralDivideError()
+        }
         evalOperation(input1, input2)
       }
     }
@@ -326,7 +447,7 @@ trait DivModLike extends BinaryArithmetic {
   def evalOperation(left: Any, right: Any): Any
 
   /**
-   * Special case handling due to division/remainder by 0 => null.
+   * Special case handling due to division/remainder by 0 => null or ArithmeticException.
    */
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     val eval1 = left.genCode(ctx)
@@ -343,29 +464,53 @@ trait DivModLike extends BinaryArithmetic {
     } else {
       s"($javaType)(${eval1.value} $symbol ${eval2.value})"
     }
+    val checkIntegralDivideOverflow = if (checkDivideOverflow) {
+      s"""
+        |if (${eval1.value} == ${Long.MinValue}L && ${eval2.value} == -1)
+        |  throw QueryExecutionErrors.overflowInIntegralDivideError();
+        |""".stripMargin
+    } else {
+      ""
+    }
+
+    // evaluate right first as we have a chance to skip left if right is 0
     if (!left.nullable && !right.nullable) {
+      val divByZero = if (failOnError) {
+        s"throw QueryExecutionErrors.divideByZeroError();"
+      } else {
+        s"${ev.isNull} = true;"
+      }
       ev.copy(code = code"""
         ${eval2.code}
         boolean ${ev.isNull} = false;
         $javaType ${ev.value} = ${CodeGenerator.defaultValue(dataType)};
         if ($isZero) {
-          ${ev.isNull} = true;
+          $divByZero
         } else {
           ${eval1.code}
+          $checkIntegralDivideOverflow
           ${ev.value} = $operation;
         }""")
     } else {
+      val nullOnErrorCondition = if (failOnError) "" else s" || $isZero"
+      val failOnErrorBranch = if (failOnError) {
+        s"if ($isZero) throw QueryExecutionErrors.divideByZeroError();"
+      } else {
+        ""
+      }
       ev.copy(code = code"""
         ${eval2.code}
         boolean ${ev.isNull} = false;
         $javaType ${ev.value} = ${CodeGenerator.defaultValue(dataType)};
-        if (${eval2.isNull} || $isZero) {
+        if (${eval2.isNull}$nullOnErrorCondition) {
           ${ev.isNull} = true;
         } else {
           ${eval1.code}
           if (${eval1.isNull}) {
             ${ev.isNull} = true;
           } else {
+            $failOnErrorBranch
+            $checkIntegralDivideOverflow
             ${ev.value} = $operation;
           }
         }""")
@@ -382,9 +527,16 @@ trait DivModLike extends BinaryArithmetic {
        1.5
       > SELECT 2L _FUNC_ 2L;
        1.0
-  """)
+  """,
+  since = "1.0.0",
+  group = "math_funcs")
 // scalastyle:on line.size.limit
-case class Divide(left: Expression, right: Expression) extends DivModLike {
+case class Divide(
+    left: Expression,
+    right: Expression,
+    failOnError: Boolean = SQLConf.get.ansiEnabled) extends DivModLike {
+
+  def this(left: Expression, right: Expression) = this(left, right, SQLConf.get.ansiEnabled)
 
   override def inputType: AbstractDataType = TypeCollection(DoubleType, DecimalType)
 
@@ -396,6 +548,9 @@ case class Divide(left: Expression, right: Expression) extends DivModLike {
   }
 
   override def evalOperation(left: Any, right: Any): Any = div(left, right)
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): Divide = copy(left = newLeft, right = newRight)
 }
 
 // scalastyle:off line.size.limit
@@ -406,13 +561,22 @@ case class Divide(left: Expression, right: Expression) extends DivModLike {
       > SELECT 3 _FUNC_ 2;
        1
   """,
-  since = "3.0.0")
+  since = "3.0.0",
+  group = "math_funcs")
 // scalastyle:on line.size.limit
 case class IntegralDivide(
     left: Expression,
-    right: Expression) extends DivModLike {
+    right: Expression,
+    failOnError: Boolean = SQLConf.get.ansiEnabled) extends DivModLike {
+
+  def this(left: Expression, right: Expression) = this(left, right, SQLConf.get.ansiEnabled)
+
+  override def checkDivideOverflow: Boolean = left.dataType match {
+    case LongType if failOnError => true
+    case _ => false
+  }
 
-  override def inputType: AbstractDataType = TypeCollection(IntegralType, DecimalType)
+  override def inputType: AbstractDataType = TypeCollection(LongType, DecimalType)
 
   override def dataType: DataType = LongType
 
@@ -439,12 +603,10 @@ case class IntegralDivide(
   }
 
   override def evalOperation(left: Any, right: Any): Any = div(left, right)
-}
 
-object IntegralDivide {
-  def apply(left: Expression, right: Expression): IntegralDivide = {
-    new IntegralDivide(left, right)
-  }
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): IntegralDivide =
+    copy(left = newLeft, right = newRight)
 }
 
 @ExpressionDescription(
@@ -455,8 +617,15 @@ object IntegralDivide {
        0.2
       > SELECT MOD(2, 1.8);
        0.2
-  """)
-case class Remainder(left: Expression, right: Expression) extends DivModLike {
+  """,
+  since = "1.0.0",
+  group = "math_funcs")
+case class Remainder(
+    left: Expression,
+    right: Expression,
+    failOnError: Boolean = SQLConf.get.ansiEnabled) extends DivModLike {
+
+  def this(left: Expression, right: Expression) = this(left, right, SQLConf.get.ansiEnabled)
 
   override def inputType: AbstractDataType = NumericType
 
@@ -492,6 +661,9 @@ case class Remainder(left: Expression, right: Expression) extends DivModLike {
   }
 
   override def evalOperation(left: Any, right: Any): Any = mod(left, right)
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): Remainder = copy(left = newLeft, right = newRight)
 }
 
 @ExpressionDescription(
@@ -502,8 +674,15 @@ case class Remainder(left: Expression, right: Expression) extends DivModLike {
        1
       > SELECT _FUNC_(-10, 3);
        2
-  """)
-case class Pmod(left: Expression, right: Expression) extends BinaryArithmetic {
+  """,
+  since = "1.5.0",
+  group = "math_funcs")
+case class Pmod(
+    left: Expression,
+    right: Expression,
+    failOnError: Boolean = SQLConf.get.ansiEnabled) extends BinaryArithmetic {
+
+  def this(left: Expression, right: Expression) = this(left, right, SQLConf.get.ansiEnabled)
 
   override def toString: String = s"pmod($left, $right)"
 
@@ -516,15 +695,25 @@ case class Pmod(left: Expression, right: Expression) extends BinaryArithmetic {
 
   override def nullable: Boolean = true
 
-  override def eval(input: InternalRow): Any = {
+  private lazy val isZero: Any => Boolean = right.dataType match {
+    case _: DecimalType => x => x.asInstanceOf[Decimal].isZero
+    case _ => x => x == 0
+  }
+
+  final override def eval(input: InternalRow): Any = {
+    // evaluate right first as we have a chance to skip left if right is 0
     val input2 = right.eval(input)
-    if (input2 == null || input2 == 0) {
+    if (input2 == null || (!failOnError && isZero(input2))) {
       null
     } else {
       val input1 = left.eval(input)
       if (input1 == null) {
         null
       } else {
+        if (isZero(input2)) {
+          // when we reach here, failOnError must bet true.
+          throw QueryExecutionErrors.divideByZeroError
+        }
         input1 match {
           case i: Integer => pmod(i, input2.asInstanceOf[java.lang.Integer])
           case l: Long => pmod(l, input2.asInstanceOf[java.lang.Long])
@@ -581,29 +770,42 @@ case class Pmod(left: Expression, right: Expression) extends BinaryArithmetic {
         """
     }
 
+    // evaluate right first as we have a chance to skip left if right is 0
     if (!left.nullable && !right.nullable) {
+      val divByZero = if (failOnError) {
+        s"throw QueryExecutionErrors.divideByZeroError();"
+      } else {
+        s"${ev.isNull} = true;"
+      }
       ev.copy(code = code"""
         ${eval2.code}
         boolean ${ev.isNull} = false;
         $javaType ${ev.value} = ${CodeGenerator.defaultValue(dataType)};
         if ($isZero) {
-          ${ev.isNull} = true;
+          $divByZero
         } else {
           ${eval1.code}
           $result
         }""")
     } else {
+      val nullOnErrorCondition = if (failOnError) "" else s" || $isZero"
+      val failOnErrorBranch = if (failOnError) {
+        s"if ($isZero) throw QueryExecutionErrors.divideByZeroError();"
+      } else {
+        ""
+      }
       ev.copy(code = code"""
         ${eval2.code}
         boolean ${ev.isNull} = false;
         $javaType ${ev.value} = ${CodeGenerator.defaultValue(dataType)};
-        if (${eval2.isNull} || $isZero) {
+        if (${eval2.isNull}$nullOnErrorCondition) {
           ${ev.isNull} = true;
         } else {
           ${eval1.code}
           if (${eval1.isNull}) {
             ${ev.isNull} = true;
           } else {
+            $failOnErrorBranch
             $result
           }
         }""")
@@ -646,6 +848,9 @@ case class Pmod(left: Expression, right: Expression) extends BinaryArithmetic {
   }
 
   override def sql: String = s"$prettyName(${left.sql}, ${right.sql})"
+
+  override protected def withNewChildrenInternal(newLeft: Expression, newRight: Expression): Pmod =
+    copy(left = newLeft, right = newRight)
 }
 
 /**
@@ -658,7 +863,9 @@ case class Pmod(left: Expression, right: Expression) extends BinaryArithmetic {
     Examples:
       > SELECT _FUNC_(10, 9, 2, 4, 3);
        2
-  """)
+  """,
+  since = "1.5.0",
+  group = "math_funcs")
 case class Least(children: Seq[Expression]) extends ComplexTypeMergingExpression {
 
   override def nullable: Boolean = children.forall(_.nullable)
@@ -719,6 +926,9 @@ case class Least(children: Seq[Expression]) extends ComplexTypeMergingExpression
          |$codes
       """.stripMargin)
   }
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Least =
+    copy(children = newChildren)
 }
 
 /**
@@ -731,7 +941,9 @@ case class Least(children: Seq[Expression]) extends ComplexTypeMergingExpression
     Examples:
       > SELECT _FUNC_(10, 9, 2, 4, 3);
        10
-  """)
+  """,
+  since = "1.5.0",
+  group = "math_funcs")
 case class Greatest(children: Seq[Expression]) extends ComplexTypeMergingExpression {
 
   override def nullable: Boolean = children.forall(_.nullable)
@@ -792,4 +1004,7 @@ case class Greatest(children: Seq[Expression]) extends ComplexTypeMergingExpress
          |$codes
       """.stripMargin)
   }
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Greatest =
+    copy(children = newChildren)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala
index 7b819db32e425..3940c65593ec5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.catalyst.expressions
 
+import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.types._
 
@@ -32,9 +33,13 @@ import org.apache.spark.sql.types._
     Examples:
       > SELECT 3 _FUNC_ 5;
        1
-  """)
+  """,
+  since = "1.4.0",
+  group = "bitwise_funcs")
 case class BitwiseAnd(left: Expression, right: Expression) extends BinaryArithmetic {
 
+  protected override val failOnError: Boolean = false
+
   override def inputType: AbstractDataType = IntegralType
 
   override def symbol: String = "&"
@@ -51,6 +56,9 @@ case class BitwiseAnd(left: Expression, right: Expression) extends BinaryArithme
   }
 
   protected override def nullSafeEval(input1: Any, input2: Any): Any = and(input1, input2)
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): BitwiseAnd = copy(left = newLeft, right = newRight)
 }
 
 /**
@@ -64,9 +72,13 @@ case class BitwiseAnd(left: Expression, right: Expression) extends BinaryArithme
     Examples:
       > SELECT 3 _FUNC_ 5;
        7
-  """)
+  """,
+  since = "1.4.0",
+  group = "bitwise_funcs")
 case class BitwiseOr(left: Expression, right: Expression) extends BinaryArithmetic {
 
+  protected override val failOnError: Boolean = false
+
   override def inputType: AbstractDataType = IntegralType
 
   override def symbol: String = "|"
@@ -83,6 +95,9 @@ case class BitwiseOr(left: Expression, right: Expression) extends BinaryArithmet
   }
 
   protected override def nullSafeEval(input1: Any, input2: Any): Any = or(input1, input2)
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): BitwiseOr = copy(left = newLeft, right = newRight)
 }
 
 /**
@@ -96,9 +111,13 @@ case class BitwiseOr(left: Expression, right: Expression) extends BinaryArithmet
     Examples:
       > SELECT 3 _FUNC_ 5;
        6
-  """)
+  """,
+  since = "1.4.0",
+  group = "bitwise_funcs")
 case class BitwiseXor(left: Expression, right: Expression) extends BinaryArithmetic {
 
+  protected override val failOnError: Boolean = false
+
   override def inputType: AbstractDataType = IntegralType
 
   override def symbol: String = "^"
@@ -115,6 +134,9 @@ case class BitwiseXor(left: Expression, right: Expression) extends BinaryArithme
   }
 
   protected override def nullSafeEval(input1: Any, input2: Any): Any = xor(input1, input2)
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): BitwiseXor = copy(left = newLeft, right = newRight)
 }
 
 /**
@@ -126,8 +148,11 @@ case class BitwiseXor(left: Expression, right: Expression) extends BinaryArithme
     Examples:
       > SELECT _FUNC_ 0;
        -1
-  """)
-case class BitwiseNot(child: Expression) extends UnaryExpression with ExpectsInputTypes {
+  """,
+  since = "1.4.0",
+  group = "bitwise_funcs")
+case class BitwiseNot(child: Expression)
+  extends UnaryExpression with ExpectsInputTypes with NullIntolerant {
 
   override def inputTypes: Seq[AbstractDataType] = Seq(IntegralType)
 
@@ -153,6 +178,9 @@ case class BitwiseNot(child: Expression) extends UnaryExpression with ExpectsInp
   protected override def nullSafeEval(input: Any): Any = not(input)
 
   override def sql: String = s"~${child.sql}"
+
+  override protected def withNewChildInternal(newChild: Expression): BitwiseNot =
+    copy(child = newChild)
 }
 
 @ExpressionDescription(
@@ -163,8 +191,10 @@ case class BitwiseNot(child: Expression) extends UnaryExpression with ExpectsInp
       > SELECT _FUNC_(0);
        0
   """,
-  since = "3.0.0")
-case class BitwiseCount(child: Expression) extends UnaryExpression with ExpectsInputTypes {
+  since = "3.0.0",
+  group = "bitwise_funcs")
+case class BitwiseCount(child: Expression)
+  extends UnaryExpression with ExpectsInputTypes with NullIntolerant {
 
   override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection(IntegralType, BooleanType))
 
@@ -186,4 +216,68 @@ case class BitwiseCount(child: Expression) extends UnaryExpression with ExpectsI
     case IntegerType => java.lang.Long.bitCount(input.asInstanceOf[Int])
     case LongType => java.lang.Long.bitCount(input.asInstanceOf[Long])
   }
+
+  override protected def withNewChildInternal(newChild: Expression): BitwiseCount =
+    copy(child = newChild)
+}
+
+object BitwiseGetUtil {
+  def checkPosition(pos: Int, size: Int): Unit = {
+    if (pos < 0) {
+      throw new IllegalArgumentException(s"Invalid bit position: $pos is less than zero")
+    } else if (size <= pos) {
+      throw new IllegalArgumentException(s"Invalid bit position: $pos exceeds the bit upper limit")
+    }
+  }
+}
+
+@ExpressionDescription(
+  usage = """
+            |_FUNC_(expr, pos) - Returns the value of the bit (0 or 1) at the specified position.
+            |  The positions are numbered from right to left, starting at zero.
+            |  The position argument cannot be negative.
+          """,
+  examples = """
+    Examples:
+      > SELECT _FUNC_(11, 0);
+       1
+      > SELECT _FUNC_(11, 2);
+       0
+  """,
+  since = "3.2.0",
+  group = "bitwise_funcs")
+case class BitwiseGet(left: Expression, right: Expression)
+  extends BinaryExpression with ImplicitCastInputTypes with NullIntolerant {
+
+  override def inputTypes: Seq[AbstractDataType] = Seq(IntegralType, IntegerType)
+
+  override def dataType: DataType = ByteType
+
+  lazy val bitSize = left.dataType match {
+    case ByteType => java.lang.Byte.SIZE
+    case ShortType => java.lang.Short.SIZE
+    case IntegerType => java.lang.Integer.SIZE
+    case LongType => java.lang.Long.SIZE
+  }
+
+  override def nullSafeEval(target: Any, pos: Any): Any = {
+    val posInt = pos.asInstanceOf[Int]
+    BitwiseGetUtil.checkPosition(posInt, bitSize)
+    ((target.asInstanceOf[Number].longValue() >> posInt) & 1).toByte
+  }
+
+  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
+    nullSafeCodeGen(ctx, ev, (target, pos) => {
+      s"""
+         |org.apache.spark.sql.catalyst.expressions.BitwiseGetUtil.checkPosition($pos, $bitSize);
+         |${ev.value} = (byte) ((((long) $target) >> $pos) & 1);
+       """.stripMargin
+    })
+  }
+
+  override def prettyName: String =
+    getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("bit_get")
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): BitwiseGet = copy(left = newLeft, right = newRight)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
index 1cc7836e93d35..f1fc718432c56 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
@@ -27,8 +27,9 @@ import scala.util.control.NonFatal
 
 import com.google.common.cache.{CacheBuilder, CacheLoader}
 import com.google.common.util.concurrent.{ExecutionError, UncheckedExecutionException}
-import org.codehaus.commons.compiler.CompileException
-import org.codehaus.janino.{ByteArrayClassLoader, ClassBodyEvaluator, InternalCompilerException, SimpleCompiler}
+import org.codehaus.commons.compiler.{CompileException, InternalCompilerException}
+import org.codehaus.commons.compiler.util.reflect.ByteArrayClassLoader
+import org.codehaus.janino.{ClassBodyEvaluator, SimpleCompiler}
 import org.codehaus.janino.util.ClassFile
 
 import org.apache.spark.{TaskContext, TaskKilledException}
@@ -38,9 +39,9 @@ import org.apache.spark.metrics.source.CodegenMetrics
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
-import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData, MapData}
+import org.apache.spark.sql.catalyst.util.{ArrayData, MapData, SQLOrderingUtil}
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.NANOS_PER_MILLIS
-import org.apache.spark.sql.catalyst.util.DateTimeUtils._
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.Platform
@@ -90,8 +91,13 @@ case class SubExprEliminationState(isNull: ExprValue, value: ExprValue)
  * @param codes Strings representing the codes that evaluate common subexpressions.
  * @param states Foreach expression that is participating in subexpression elimination,
  *               the state to use.
+ * @param exprCodesNeedEvaluate Some expression codes that need to be evaluated before
+ *                              calling common subexpressions.
  */
-case class SubExprCodes(codes: Seq[String], states: Map[Expression, SubExprEliminationState])
+case class SubExprCodes(
+  codes: Seq[String],
+  states: Map[Expression, SubExprEliminationState],
+  exprCodesNeedEvaluate: Seq[ExprCode])
 
 /**
  * The main information about a new added function.
@@ -133,7 +139,7 @@ class CodegenContext extends Logging {
   def addReferenceObj(objName: String, obj: Any, className: String = null): String = {
     val idx = references.length
     references += obj
-    val clsName = Option(className).getOrElse(obj.getClass.getName)
+    val clsName = Option(className).getOrElse(CodeGenerator.typeName(obj.getClass))
     s"(($clsName) references[$idx] /* $objName */)"
   }
 
@@ -171,7 +177,7 @@ class CodegenContext extends Logging {
     mutable.ArrayBuffer.empty[(String, String)]
 
   /**
-   * The mapping between mutable state types and corrseponding compacted arrays.
+   * The mapping between mutable state types and corresponding compacted arrays.
    * The keys are java type string. The values are [[MutableStateArrays]] which encapsulates
    * the compacted arrays for the mutable states with the same java type.
    *
@@ -374,7 +380,7 @@ class CodegenContext extends Logging {
 
     // The generated initialization code may exceed 64kb function size limit in JVM if there are too
     // many mutable states, so split it into multiple functions.
-    splitExpressions(expressions = initCodes, funcName = "init", arguments = Nil)
+    splitExpressions(expressions = initCodes.toSeq, funcName = "init", arguments = Nil)
   }
 
   /**
@@ -473,7 +479,7 @@ class CodegenContext extends Logging {
       case NewFunctionSpec(functionName, Some(_), Some(innerClassInstance)) =>
         innerClassInstance + "." + functionName
       case _ =>
-        throw new IllegalArgumentException(s"$funcName is not matched at addNewFunction")
+        throw QueryExecutionErrors.addNewFunctionMismatchedWithFunctionError(funcName)
     }
   }
 
@@ -610,8 +616,8 @@ class CodegenContext extends Logging {
     case udt: UserDefinedType[_] => genEqual(udt.sqlType, c1, c2)
     case NullType => "false"
     case _ =>
-      throw new IllegalArgumentException(
-        "cannot generate equality code for un-comparable type: " + dataType.catalogString)
+      throw QueryExecutionErrors.cannotGenerateCodeForUncomparableTypeError(
+        "equality", dataType)
   }
 
   /**
@@ -624,8 +630,12 @@ class CodegenContext extends Logging {
   def genComp(dataType: DataType, c1: String, c2: String): String = dataType match {
     // java boolean doesn't support > or < operator
     case BooleanType => s"($c1 == $c2 ? 0 : ($c1 ? 1 : -1))"
-    case DoubleType => s"java.lang.Double.compare($c1, $c2)"
-    case FloatType => s"java.lang.Float.compare($c1, $c2)"
+    case DoubleType =>
+      val clsName = SQLOrderingUtil.getClass.getName.stripSuffix("$")
+      s"$clsName.compareDoubles($c1, $c2)"
+    case FloatType =>
+      val clsName = SQLOrderingUtil.getClass.getName.stripSuffix("$")
+      s"$clsName.compareFloats($c1, $c2)"
     // use c1 - c2 may overflow
     case dt: DataType if isPrimitiveType(dt) => s"($c1 > $c2 ? 1 : $c1 < $c2 ? -1 : 0)"
     case BinaryType => s"org.apache.spark.sql.catalyst.util.TypeUtils.compareBinary($c1, $c2)"
@@ -697,8 +707,7 @@ class CodegenContext extends Logging {
     case other if other.isInstanceOf[AtomicType] => s"$c1.compare($c2)"
     case udt: UserDefinedType[_] => genComp(udt.sqlType, c1, c2)
     case _ =>
-      throw new IllegalArgumentException(
-        "cannot generate compare code for un-comparable type: " + dataType.catalogString)
+      throw QueryExecutionErrors.cannotGenerateCodeForUncomparableTypeError("compare", dataType)
   }
 
   /**
@@ -926,6 +935,7 @@ class CodegenContext extends Logging {
       length += CodeFormatter.stripExtraNewLinesAndComments(code).length
     }
     blocks += blockBuilder.toString()
+    blocks.toSeq
   }
 
   /**
@@ -1001,7 +1011,7 @@ class CodegenContext extends Logging {
   def subexprFunctionsCode: String = {
     // Whole-stage codegen's subexpression elimination is handled in another code path
     assert(currentVars == null || subexprFunctions.isEmpty)
-    splitExpressions(subexprFunctions, "subexprFunc_split", Seq("InternalRow" -> INPUT_ROW))
+    splitExpressions(subexprFunctions.toSeq, "subexprFunc_split", Seq("InternalRow" -> INPUT_ROW))
   }
 
   /**
@@ -1034,12 +1044,12 @@ class CodegenContext extends Logging {
     val localSubExprEliminationExprs = mutable.HashMap.empty[Expression, SubExprEliminationState]
 
     // Add each expression tree and compute the common subexpressions.
-    expressions.foreach(equivalentExpressions.addExprTree)
+    expressions.foreach(equivalentExpressions.addExprTree(_))
 
     // Get all the expressions that appear at least twice and set up the state for subexpression
     // elimination.
     val commonExprs = equivalentExpressions.getAllEquivalentExprs.filter(_.size > 1)
-    val commonExprVals = commonExprs.map(_.head.genCode(this))
+    lazy val commonExprVals = commonExprs.map(_.head.genCode(this))
 
     lazy val nonSplitExprCode = {
       commonExprs.zip(commonExprVals).map { case (exprs, eval) =>
@@ -1050,10 +1060,17 @@ class CodegenContext extends Logging {
       }
     }
 
-    val codes = if (commonExprVals.map(_.code.length).sum > SQLConf.get.methodSplitThreshold) {
-      val inputVarsForAllFuncs = commonExprs.map { expr =>
-        getLocalInputVariableValues(this, expr.head).toSeq
-      }
+    // For some operators, they do not require all its child's outputs to be evaluated in advance.
+    // Instead it only early evaluates part of outputs, for example, `ProjectExec` only early
+    // evaluate the outputs used more than twice. So we need to extract these variables used by
+    // subexpressions and evaluate them before subexpressions.
+    val (inputVarsForAllFuncs, exprCodesNeedEvaluate) = commonExprs.map { expr =>
+      val (inputVars, exprCodes) = getLocalInputVariableValues(this, expr.head)
+      (inputVars.toSeq, exprCodes.toSeq)
+    }.unzip
+
+    val splitThreshold = SQLConf.get.methodSplitThreshold
+    val codes = if (commonExprVals.map(_.code.length).sum > splitThreshold) {
       if (inputVarsForAllFuncs.map(calculateParamLengthFromExprValues).forall(isValidParamLength)) {
         commonExprs.zipWithIndex.map { case (exprs, i) =>
           val expr = exprs.head
@@ -1073,7 +1090,8 @@ class CodegenContext extends Logging {
           // Generate the code for this expression tree and wrap it in a function.
           val fnName = freshName("subExpr")
           val inputVars = inputVarsForAllFuncs(i)
-          val argList = inputVars.map(v => s"${v.javaType.getName} ${v.variableName}")
+          val argList =
+            inputVars.map(v => s"${CodeGenerator.typeName(v.javaType)} ${v.variableName}")
           val returnType = javaType(expr.dataType)
           val fn =
             s"""
@@ -1091,20 +1109,17 @@ class CodegenContext extends Logging {
           s"$returnType $value = ${addNewFunction(fnName, fn)}($inputVariables);"
         }
       } else {
-        val errMsg = "Failed to split subexpression code into small functions because the " +
-          "parameter length of at least one split function went over the JVM limit: " +
-          MAX_JVM_METHOD_PARAMS_LENGTH
         if (Utils.isTesting) {
-          throw new IllegalStateException(errMsg)
+          throw QueryExecutionErrors.failedSplitSubExpressionError(MAX_JVM_METHOD_PARAMS_LENGTH)
         } else {
-          logInfo(errMsg)
+          logInfo(QueryExecutionErrors.failedSplitSubExpressionMsg(MAX_JVM_METHOD_PARAMS_LENGTH))
           nonSplitExprCode
         }
       }
     } else {
       nonSplitExprCode
     }
-    SubExprCodes(codes, localSubExprEliminationExprs.toMap)
+    SubExprCodes(codes, localSubExprEliminationExprs.toMap, exprCodesNeedEvaluate.flatten)
   }
 
   /**
@@ -1373,7 +1388,8 @@ object CodeGenerator extends Logging {
       classOf[Expression].getName,
       classOf[TaskContext].getName,
       classOf[TaskKilledException].getName,
-      classOf[InputMetrics].getName
+      classOf[InputMetrics].getName,
+      QueryExecutionErrors.getClass.getName.stripSuffix("$")
     )
     evaluator.setExtendedClass(classOf[GeneratedClass])
 
@@ -1388,15 +1404,15 @@ object CodeGenerator extends Logging {
       updateAndGetCompilationStats(evaluator)
     } catch {
       case e: InternalCompilerException =>
-        val msg = s"failed to compile: $e"
+        val msg = QueryExecutionErrors.failedToCompileMsg(e)
         logError(msg, e)
         logGeneratedCode(code)
-        throw new InternalCompilerException(msg, e)
+        throw QueryExecutionErrors.internalCompilerError(e)
       case e: CompileException =>
-        val msg = s"failed to compile: $e"
+        val msg = QueryExecutionErrors.failedToCompileMsg(e)
         logError(msg, e)
         logGeneratedCode(code)
-        throw new CompileException(msg, e.getLocation)
+        throw QueryExecutionErrors.compilerError(e)
     }
 
     (evaluator.getClazz().getConstructor().newInstance().asInstanceOf[GeneratedClass], codeStats)
@@ -1419,9 +1435,10 @@ object CodeGenerator extends Logging {
   private def updateAndGetCompilationStats(evaluator: ClassBodyEvaluator): ByteCodeStats = {
     // First retrieve the generated classes.
     val classes = {
-      val resultField = classOf[SimpleCompiler].getDeclaredField("result")
-      resultField.setAccessible(true)
-      val loader = resultField.get(evaluator).asInstanceOf[ByteArrayClassLoader]
+      val scField = classOf[ClassBodyEvaluator].getDeclaredField("sc")
+      scField.setAccessible(true)
+      val compiler = scField.get(evaluator).asInstanceOf[SimpleCompiler]
+      val loader = compiler.getClassLoader.asInstanceOf[ByteArrayClassLoader]
       val classesField = loader.getClass.getDeclaredField("classes")
       classesField.setAccessible(true)
       classesField.get(loader).asInstanceOf[JavaMap[String, Array[Byte]]].asScala
@@ -1538,8 +1555,8 @@ object CodeGenerator extends Logging {
   }
 
   /**
-   * Generates code creating a [[UnsafeArrayData]] or [[GenericArrayData]] based on
-   * given parameters.
+   * Generates code creating a [[UnsafeArrayData]] or
+   * [[org.apache.spark.sql.catalyst.util.GenericArrayData]] based on given parameters.
    *
    * @param arrayName name of the array to create
    * @param elementType data type of the elements in source array
@@ -1727,15 +1744,23 @@ object CodeGenerator extends Logging {
   }
 
   /**
-   * Extracts all the input variables from references and subexpression elimination states
-   * for a given `expr`. This result will be used to split the generated code of
-   * expressions into multiple functions.
+   * This methods returns two values in a Tuple.
+   *
+   * First value: Extracts all the input variables from references and subexpression
+   * elimination states for a given `expr`. This result will be used to split the
+   * generated code of expressions into multiple functions.
+   *
+   * Second value: Returns the set of `ExprCodes`s which are necessary codes before
+   * evaluating subexpressions.
    */
   def getLocalInputVariableValues(
       ctx: CodegenContext,
       expr: Expression,
-      subExprs: Map[Expression, SubExprEliminationState] = Map.empty): Set[VariableValue] = {
+      subExprs: Map[Expression, SubExprEliminationState] = Map.empty)
+      : (Set[VariableValue], Set[ExprCode]) = {
     val argSet = mutable.Set[VariableValue]()
+    val exprCodesNeedEvaluate = mutable.Set[ExprCode]()
+
     if (ctx.INPUT_ROW != null) {
       argSet += JavaCode.variable(ctx.INPUT_ROW, classOf[InternalRow])
     }
@@ -1756,16 +1781,21 @@ object CodeGenerator extends Logging {
 
         case ref: BoundReference if ctx.currentVars != null &&
             ctx.currentVars(ref.ordinal) != null =>
-          val ExprCode(_, isNull, value) = ctx.currentVars(ref.ordinal)
-          collectLocalVariable(value)
-          collectLocalVariable(isNull)
+          val exprCode = ctx.currentVars(ref.ordinal)
+          // If the referred variable is not evaluated yet.
+          if (exprCode.code != EmptyBlock) {
+            exprCodesNeedEvaluate += exprCode.copy()
+            exprCode.code = EmptyBlock
+          }
+          collectLocalVariable(exprCode.value)
+          collectLocalVariable(exprCode.isNull)
 
         case e =>
           stack.pushAll(e.children)
       }
     }
 
-    argSet.toSet
+    (argSet.toSet, exprCodesNeedEvaluate.toSet)
   }
 
   /**
@@ -1785,8 +1815,8 @@ object CodeGenerator extends Logging {
     case BooleanType => JAVA_BOOLEAN
     case ByteType => JAVA_BYTE
     case ShortType => JAVA_SHORT
-    case IntegerType | DateType => JAVA_INT
-    case LongType | TimestampType => JAVA_LONG
+    case IntegerType | DateType | YearMonthIntervalType => JAVA_INT
+    case LongType | TimestampType | DayTimeIntervalType => JAVA_LONG
     case FloatType => JAVA_FLOAT
     case DoubleType => JAVA_DOUBLE
     case _: DecimalType => "Decimal"
@@ -1806,8 +1836,8 @@ object CodeGenerator extends Logging {
     case BooleanType => java.lang.Boolean.TYPE
     case ByteType => java.lang.Byte.TYPE
     case ShortType => java.lang.Short.TYPE
-    case IntegerType | DateType => java.lang.Integer.TYPE
-    case LongType | TimestampType => java.lang.Long.TYPE
+    case IntegerType | DateType | YearMonthIntervalType => java.lang.Integer.TYPE
+    case LongType | TimestampType | DayTimeIntervalType => java.lang.Long.TYPE
     case FloatType => java.lang.Float.TYPE
     case DoubleType => java.lang.Double.TYPE
     case _: DecimalType => classOf[Decimal]
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala
index 63bd59e7628b2..5d00519d27c53 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala
@@ -71,7 +71,9 @@ object GenerateOrdering extends CodeGenerator[Seq[SortOrder], BaseOrdering] with
     ctx.INPUT_ROW = row
     // to use INPUT_ROW we must make sure currentVars is null
     ctx.currentVars = null
-    ordering.map(_.child.genCode(ctx))
+    // SPARK-33260: To avoid unpredictable modifications to `ctx` when `ordering` is a Stream, we
+    // use `toIndexedSeq` to make the transformation eager.
+    ordering.toIndexedSeq.map(_.child.genCode(ctx))
   }
 
   /**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GeneratePredicate.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GeneratePredicate.scala
index 6ba646d360d2e..c246d07f189b4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GeneratePredicate.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GeneratePredicate.scala
@@ -17,7 +17,6 @@
 
 package org.apache.spark.sql.catalyst.expressions.codegen
 
-import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 
 /**
@@ -30,9 +29,17 @@ object GeneratePredicate extends CodeGenerator[Expression, BasePredicate] {
   protected def bind(in: Expression, inputSchema: Seq[Attribute]): Expression =
     BindReferences.bindReference(in, inputSchema)
 
-  protected def create(predicate: Expression): BasePredicate = {
+  def generate(expressions: Expression, useSubexprElimination: Boolean): BasePredicate =
+    create(canonicalize(expressions), useSubexprElimination)
+
+  protected def create(predicate: Expression): BasePredicate = create(predicate, false)
+
+  protected def create(predicate: Expression, useSubexprElimination: Boolean): BasePredicate = {
     val ctx = newCodeGenContext()
-    val eval = predicate.genCode(ctx)
+
+    // Do sub-expression elimination for predicates.
+    val eval = ctx.generateExpressions(Seq(predicate), useSubexprElimination).head
+    val evalSubexpr = ctx.subexprFunctionsCode
 
     val codeBody = s"""
       public SpecificPredicate generate(Object[] references) {
@@ -53,6 +60,7 @@ object GeneratePredicate extends CodeGenerator[Expression, BasePredicate] {
         }
 
         public boolean eval(InternalRow ${ctx.INPUT_ROW}) {
+          $evalSubexpr
           ${eval.code}
           return !${eval.isNull} && ${eval.value};
         }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala
index e285398ba1958..4efcca0017eaa 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala
@@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, Generic
 import org.apache.spark.sql.types._
 
 /**
- * Java can not access Projection (in package object)
+ * Java cannot access Projection (in package object)
  */
 abstract class BaseProjection extends Projection {}
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoiner.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoiner.scala
index 070570d8f20b2..27b1f89f70870 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoiner.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoiner.scala
@@ -17,12 +17,8 @@
 
 package org.apache.spark.sql.catalyst.expressions.codegen
 
-import scala.collection.mutable
-import scala.collection.mutable.ArrayBuffer
-
 import org.apache.spark.sql.catalyst.expressions.{Attribute, UnsafeRow}
 import org.apache.spark.sql.types.StructType
-import org.apache.spark.unsafe.Platform
 
 abstract class UnsafeRowJoiner {
   def join(row1: UnsafeRow, row2: UnsafeRow): UnsafeRow
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/javaCode.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/javaCode.scala
index 1c59c3c2ad667..c840cdfd8b2dc 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/javaCode.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/javaCode.scala
@@ -22,7 +22,8 @@ import java.lang.{Boolean => JBool}
 import scala.collection.mutable.ArrayBuffer
 import scala.language.implicitConversions
 
-import org.apache.spark.sql.catalyst.trees.TreeNode
+import org.apache.spark.sql.catalyst.trees.{LeafLike, TreeNode}
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.types.{BooleanType, DataType}
 
 /**
@@ -202,7 +203,7 @@ trait Block extends TreeNode[Block] with JavaCode {
 
   override def verboseString(maxFields: Int): String = toString
   override def simpleStringWithNodeId(): String = {
-    throw new UnsupportedOperationException(s"$nodeName does not implement simpleStringWithNodeId")
+    throw QueryExecutionErrors.simpleStringWithNodeIdUnsupportedError(nodeName)
   }
 }
 
@@ -236,8 +237,7 @@ object Block {
         args.foreach {
           case _: ExprValue | _: Inline | _: Block =>
           case _: Boolean | _: Int | _: Long | _: Float | _: Double | _: String =>
-          case other => throw new IllegalArgumentException(
-            s"Can not interpolate ${other.getClass.getName} into code block.")
+          case other => throw QueryExecutionErrors.cannotInterpolateClassIntoCodeBlockError(other)
         }
 
         val (codeParts, blockInputs) = foldLiteralArgs(sc.parts, args)
@@ -298,11 +298,13 @@ case class CodeBlock(codeParts: Seq[String], blockInputs: Seq[JavaCode]) extends
     }
     buf.toString
   }
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Block]): Block =
+    super.legacyWithNewChildren(newChildren)
 }
 
-case object EmptyBlock extends Block with Serializable {
+case object EmptyBlock extends Block with Serializable with LeafLike[Block] {
   override val code: String = ""
-  override def children: Seq[Block] = Seq.empty
 }
 
 /**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index 4fd68dcfe5156..0c580d1c8e5d8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -16,17 +16,18 @@
  */
 package org.apache.spark.sql.catalyst.expressions
 
-import java.time.ZoneId
+import java.time.{Duration, Period, ZoneId}
 import java.util.Comparator
 
 import scala.collection.mutable
 import scala.reflect.ClassTag
 
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, TypeCoercion}
+import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, TypeCoercion, UnresolvedSeed}
 import org.apache.spark.sql.catalyst.expressions.ArraySortLike.NullOrder
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
+import org.apache.spark.sql.catalyst.trees.TreePattern.{CONCAT, TreePattern}
 import org.apache.spark.sql.catalyst.util._
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils._
@@ -89,7 +90,9 @@ trait BinaryArrayExpressionWithImplicitCast extends BinaryExpression
        2
       > SELECT _FUNC_(NULL);
        -1
-  """)
+  """,
+  since = "1.5.0",
+  group = "collection_funcs")
 case class Size(child: Expression, legacySizeOfNull: Boolean)
   extends UnaryExpression with ExpectsInputTypes {
 
@@ -123,6 +126,8 @@ case class Size(child: Expression, legacySizeOfNull: Boolean)
       defineCodeGen(ctx, ev, c => s"($c).numElements()")
     }
   }
+
+  override protected def withNewChildInternal(newChild: Expression): Size = copy(child = newChild)
 }
 
 object Size {
@@ -139,9 +144,10 @@ object Size {
       > SELECT _FUNC_(map(1, 'a', 2, 'b'));
        [1,2]
   """,
-  group = "map_funcs")
+  group = "map_funcs",
+  since = "2.0.0")
 case class MapKeys(child: Expression)
-  extends UnaryExpression with ExpectsInputTypes {
+  extends UnaryExpression with ExpectsInputTypes with NullIntolerant {
 
   override def inputTypes: Seq[AbstractDataType] = Seq(MapType)
 
@@ -156,6 +162,9 @@ case class MapKeys(child: Expression)
   }
 
   override def prettyName: String = "map_keys"
+
+  override protected def withNewChildInternal(newChild: Expression): MapKeys =
+    copy(child = newChild)
 }
 
 @ExpressionDescription(
@@ -318,6 +327,9 @@ case class ArraysZip(children: Seq[Expression]) extends Expression with ExpectsI
   }
 
   override def prettyName: String = "arrays_zip"
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): ArraysZip =
+    copy(children = newChildren)
 }
 
 /**
@@ -330,9 +342,10 @@ case class ArraysZip(children: Seq[Expression]) extends Expression with ExpectsI
       > SELECT _FUNC_(map(1, 'a', 2, 'b'));
        ["a","b"]
   """,
-  group = "map_funcs")
+  group = "map_funcs",
+  since = "2.0.0")
 case class MapValues(child: Expression)
-  extends UnaryExpression with ExpectsInputTypes {
+  extends UnaryExpression with ExpectsInputTypes with NullIntolerant {
 
   override def inputTypes: Seq[AbstractDataType] = Seq(MapType)
 
@@ -347,6 +360,9 @@ case class MapValues(child: Expression)
   }
 
   override def prettyName: String = "map_values"
+
+  override protected def withNewChildInternal(newChild: Expression): MapValues =
+    copy(child = newChild)
 }
 
 /**
@@ -361,13 +377,14 @@ case class MapValues(child: Expression)
   """,
   group = "map_funcs",
   since = "3.0.0")
-case class MapEntries(child: Expression) extends UnaryExpression with ExpectsInputTypes {
+case class MapEntries(child: Expression)
+  extends UnaryExpression with ExpectsInputTypes with NullIntolerant {
 
   override def inputTypes: Seq[AbstractDataType] = Seq(MapType)
 
   @transient private lazy val childDataType: MapType = child.dataType.asInstanceOf[MapType]
 
-  override def dataType: DataType = {
+  private lazy val internalDataType: DataType = {
     ArrayType(
       StructType(
         StructField("key", childDataType.keyType, false) ::
@@ -376,6 +393,8 @@ case class MapEntries(child: Expression) extends UnaryExpression with ExpectsInp
       false)
   }
 
+  override def dataType: DataType = internalDataType
+
   override protected def nullSafeEval(input: Any): Any = {
     val childMap = input.asInstanceOf[MapData]
     val keys = childMap.keyArray()
@@ -516,6 +535,8 @@ case class MapEntries(child: Expression) extends UnaryExpression with ExpectsInp
   }
 
   override def prettyName: String = "map_entries"
+
+  override def withNewChildInternal(newChild: Expression): MapEntries = copy(child = newChild)
 }
 
 /**
@@ -635,6 +656,9 @@ case class MapConcat(children: Seq[Expression]) extends ComplexTypeMergingExpres
   }
 
   override def prettyName: String = "map_concat"
+
+  override def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): MapConcat =
+    copy(children = newChildren)
 }
 
 /**
@@ -649,7 +673,7 @@ case class MapConcat(children: Seq[Expression]) extends ComplexTypeMergingExpres
   """,
   group = "map_funcs",
   since = "2.4.0")
-case class MapFromEntries(child: Expression) extends UnaryExpression {
+case class MapFromEntries(child: Expression) extends UnaryExpression with NullIntolerant {
 
   @transient
   private lazy val dataTypeDetails: Option[(MapType, Boolean, Boolean)] = child.dataType match {
@@ -713,6 +737,9 @@ case class MapFromEntries(child: Expression) extends UnaryExpression {
   }
 
   override def prettyName: String = "map_from_entries"
+
+  override protected def withNewChildInternal(newChild: Expression): MapFromEntries =
+    copy(child = newChild)
 }
 
 
@@ -870,10 +897,11 @@ object ArraySortLike {
       > SELECT _FUNC_(array('b', 'd', null, 'c', 'a'), true);
        [null,"a","b","c","d"]
   """,
-  group = "array_funcs")
+  group = "array_funcs",
+  since = "1.5.0")
 // scalastyle:on line.size.limit
 case class SortArray(base: Expression, ascendingOrder: Expression)
-  extends BinaryExpression with ArraySortLike {
+  extends BinaryExpression with ArraySortLike with NullIntolerant {
 
   def this(e: Expression) = this(e, Literal(true))
 
@@ -911,6 +939,10 @@ case class SortArray(base: Expression, ascendingOrder: Expression)
   }
 
   override def prettyName: String = "sort_array"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): SortArray =
+    copy(base = newLeft, ascendingOrder = newRight)
 }
 
 /**
@@ -935,6 +967,8 @@ case class Shuffle(child: Expression, randomSeed: Option[Long] = None)
 
   def this(child: Expression) = this(child, None)
 
+  override def seedExpression: Expression = randomSeed.map(Literal.apply).getOrElse(UnresolvedSeed)
+
   override def withNewSeed(seed: Long): Shuffle = copy(randomSeed = Some(seed))
 
   override lazy val resolved: Boolean =
@@ -997,6 +1031,8 @@ case class Shuffle(child: Expression, randomSeed: Option[Long] = None)
   }
 
   override def freshCopy(): Shuffle = Shuffle(child, randomSeed)
+
+  override def withNewChildInternal(newChild: Expression): Shuffle = copy(child = newChild)
 }
 
 /**
@@ -1011,13 +1047,14 @@ case class Shuffle(child: Expression, randomSeed: Option[Long] = None)
       > SELECT _FUNC_(array(2, 1, 4, 3));
        [3,4,1,2]
   """,
-  group = "array_funcs",
+  group = "collection_funcs",
   since = "1.5.0",
   note = """
     Reverse logic for arrays is available since 2.4.0.
   """
 )
-case class Reverse(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
+case class Reverse(child: Expression)
+  extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   // Input types are utilized by type coercion in ImplicitTypeCasts.
   override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection(StringType, ArrayType))
@@ -1072,6 +1109,9 @@ case class Reverse(child: Expression) extends UnaryExpression with ImplicitCastI
   }
 
   override def prettyName: String = "reverse"
+
+  override protected def withNewChildInternal(newChild: Expression): Reverse =
+    copy(child = newChild)
 }
 
 /**
@@ -1084,9 +1124,10 @@ case class Reverse(child: Expression) extends UnaryExpression with ImplicitCastI
       > SELECT _FUNC_(array(1, 2, 3), 2);
        true
   """,
-  group = "array_funcs")
+  group = "array_funcs",
+  since = "1.5.0")
 case class ArrayContains(left: Expression, right: Expression)
-  extends BinaryExpression with ImplicitCastInputTypes {
+  extends BinaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def dataType: DataType = BooleanType
 
@@ -1168,6 +1209,10 @@ case class ArrayContains(left: Expression, right: Expression)
   }
 
   override def prettyName: String = "array_contains"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): ArrayContains =
+    copy(left = newLeft, right = newRight)
 }
 
 /**
@@ -1185,7 +1230,7 @@ case class ArrayContains(left: Expression, right: Expression)
   since = "2.4.0")
 // scalastyle:off line.size.limit
 case class ArraysOverlap(left: Expression, right: Expression)
-  extends BinaryArrayExpressionWithImplicitCast {
+  extends BinaryArrayExpressionWithImplicitCast with NullIntolerant {
 
   override def checkInputDataTypes(): TypeCheckResult = super.checkInputDataTypes() match {
     case TypeCheckResult.TypeCheckSuccess =>
@@ -1391,6 +1436,10 @@ case class ArraysOverlap(left: Expression, right: Expression)
   }
 
   override def prettyName: String = "arrays_overlap"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): ArraysOverlap =
+    copy(left = newLeft, right = newRight)
 }
 
 /**
@@ -1410,13 +1459,15 @@ case class ArraysOverlap(left: Expression, right: Expression)
   since = "2.4.0")
 // scalastyle:on line.size.limit
 case class Slice(x: Expression, start: Expression, length: Expression)
-  extends TernaryExpression with ImplicitCastInputTypes {
+  extends TernaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def dataType: DataType = x.dataType
 
   override def inputTypes: Seq[AbstractDataType] = Seq(ArrayType, IntegerType, IntegerType)
 
-  @transient override lazy val children: Seq[Expression] = Seq(x, start, length) // called from eval
+  override def first: Expression = x
+  override def second: Expression = start
+  override def third: Expression = length
 
   @transient private lazy val elementType: DataType = x.dataType.asInstanceOf[ArrayType].elementType
 
@@ -1502,6 +1553,10 @@ case class Slice(x: Expression, start: Expression, length: Expression)
        |}
      """.stripMargin
   }
+
+  override protected def withNewChildrenInternal(
+      newFirst: Expression, newSecond: Expression, newThird: Expression): Slice =
+    copy(x = newFirst, start = newSecond, length = newThird)
 }
 
 /**
@@ -1545,6 +1600,16 @@ case class ArrayJoin(
     Seq(array, delimiter)
   }
 
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
+    if (nullReplacement.isDefined) {
+      copy(
+        array = newChildren(0),
+        delimiter = newChildren(1),
+        nullReplacement = Some(newChildren(2)))
+    } else {
+      copy(array = newChildren(0), delimiter = newChildren(1))
+    }
+
   override def nullable: Boolean = children.exists(_.nullable)
 
   override def foldable: Boolean = children.forall(_.foldable)
@@ -1688,7 +1753,8 @@ case class ArrayJoin(
   """,
   group = "array_funcs",
   since = "2.4.0")
-case class ArrayMin(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
+case class ArrayMin(child: Expression)
+  extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def nullable: Boolean = true
 
@@ -1741,6 +1807,9 @@ case class ArrayMin(child: Expression) extends UnaryExpression with ImplicitCast
   }
 
   override def prettyName: String = "array_min"
+
+  override protected def withNewChildInternal(newChild: Expression): ArrayMin =
+    copy(child = newChild)
 }
 
 /**
@@ -1755,7 +1824,8 @@ case class ArrayMin(child: Expression) extends UnaryExpression with ImplicitCast
   """,
   group = "array_funcs",
   since = "2.4.0")
-case class ArrayMax(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
+case class ArrayMax(child: Expression)
+  extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def nullable: Boolean = true
 
@@ -1808,6 +1878,9 @@ case class ArrayMax(child: Expression) extends UnaryExpression with ImplicitCast
   }
 
   override def prettyName: String = "array_max"
+
+  override protected def withNewChildInternal(newChild: Expression): ArrayMax =
+    copy(child = newChild)
 }
 
 
@@ -1831,7 +1904,7 @@ case class ArrayMax(child: Expression) extends UnaryExpression with ImplicitCast
   group = "array_funcs",
   since = "2.4.0")
 case class ArrayPosition(left: Expression, right: Expression)
-  extends BinaryExpression with ImplicitCastInputTypes {
+  extends BinaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   @transient private lazy val ordering: Ordering[Any] =
     TypeUtils.getInterpretedOrdering(right.dataType)
@@ -1887,6 +1960,10 @@ case class ArrayPosition(left: Expression, right: Expression)
        """.stripMargin
     })
   }
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): ArrayPosition =
+    copy(left = newLeft, right = newRight)
 }
 
 /**
@@ -1895,10 +1972,14 @@ case class ArrayPosition(left: Expression, right: Expression)
 @ExpressionDescription(
   usage = """
     _FUNC_(array, index) - Returns element of array at given (1-based) index. If index < 0,
-      accesses elements from the last to the first. Returns NULL if the index exceeds the length
-      of the array.
-
-    _FUNC_(map, key) - Returns value for given key, or NULL if the key is not contained in the map
+      accesses elements from the last to the first. The function returns NULL
+      if the index exceeds the length of the array and `spark.sql.ansi.enabled` is set to false.
+      If `spark.sql.ansi.enabled` is set to true, it throws ArrayIndexOutOfBoundsException
+      for invalid indices.
+
+    _FUNC_(map, key) - Returns value for given key. The function returns NULL
+      if the key is not contained in the map and `spark.sql.ansi.enabled` is set to false.
+      If `spark.sql.ansi.enabled` is set to true, it throws NoSuchElementException instead.
   """,
   examples = """
     Examples:
@@ -1907,12 +1988,21 @@ case class ArrayPosition(left: Expression, right: Expression)
       > SELECT _FUNC_(map(1, 'a', 2, 'b'), 2);
        b
   """,
-  since = "2.4.0")
-case class ElementAt(left: Expression, right: Expression)
-  extends GetMapValueUtil with GetArrayItemUtil {
+  since = "2.4.0",
+  group = "map_funcs")
+case class ElementAt(
+    left: Expression,
+    right: Expression,
+    failOnError: Boolean = SQLConf.get.ansiEnabled)
+  extends GetMapValueUtil with GetArrayItemUtil with NullIntolerant {
+
+  def this(left: Expression, right: Expression) = this(left, right, SQLConf.get.ansiEnabled)
 
   @transient private lazy val mapKeyType = left.dataType.asInstanceOf[MapType].keyType
 
+  @transient private lazy val mapValueContainsNull =
+    left.dataType.asInstanceOf[MapType].valueContainsNull
+
   @transient private lazy val arrayContainsNull = left.dataType.asInstanceOf[ArrayType].containsNull
 
   @transient private lazy val ordering: Ordering[Any] = TypeUtils.getInterpretedOrdering(mapKeyType)
@@ -1954,9 +2044,24 @@ case class ElementAt(left: Expression, right: Expression)
     }
   }
 
+  private def nullability(elements: Seq[Expression], ordinal: Int): Boolean = {
+    if (ordinal == 0) {
+      false
+    } else if (elements.length < math.abs(ordinal)) {
+      !failOnError
+    } else {
+      if (ordinal < 0) {
+        elements(elements.length + ordinal).nullable
+      } else {
+        elements(ordinal - 1).nullable
+      }
+    }
+  }
+
   override def nullable: Boolean = left.dataType match {
-    case _: ArrayType => computeNullabilityFromArray(left, right)
-    case _: MapType => true
+    case _: ArrayType =>
+      computeNullabilityFromArray(left, right, failOnError, nullability)
+    case _: MapType => if (failOnError) mapValueContainsNull else true
   }
 
   override def nullSafeEval(value: Any, ordinal: Any): Any = doElementAt(value, ordinal)
@@ -1967,7 +2072,12 @@ case class ElementAt(left: Expression, right: Expression)
         val array = value.asInstanceOf[ArrayData]
         val index = ordinal.asInstanceOf[Int]
         if (array.numElements() < math.abs(index)) {
-          null
+          if (failOnError) {
+            throw new ArrayIndexOutOfBoundsException(
+              s"Invalid index: $index, numElements: ${array.numElements()}")
+          } else {
+            null
+          }
         } else {
           val idx = if (index == 0) {
             throw new ArrayIndexOutOfBoundsException("SQL array indices start at 1")
@@ -1984,7 +2094,7 @@ case class ElementAt(left: Expression, right: Expression)
         }
       }
     case _: MapType =>
-      (value, ordinal) => getValueEval(value, ordinal, mapKeyType, ordering)
+      (value, ordinal) => getValueEval(value, ordinal, mapKeyType, ordering, failOnError)
   }
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
@@ -2001,10 +2111,20 @@ case class ElementAt(left: Expression, right: Expression)
           } else {
             ""
           }
+
+          val indexOutOfBoundBranch = if (failOnError) {
+            s"""throw new ArrayIndexOutOfBoundsException(
+               |  "Invalid index: " + $index + ", numElements: " + $eval1.numElements()
+               |);
+             """.stripMargin
+          } else {
+            s"${ev.isNull} = true;"
+          }
+
           s"""
              |int $index = (int) $eval2;
              |if ($eval1.numElements() < Math.abs($index)) {
-             |  ${ev.isNull} = true;
+             |  $indexOutOfBoundBranch
              |} else {
              |  if ($index == 0) {
              |    throw new ArrayIndexOutOfBoundsException("SQL array indices start at 1");
@@ -2021,11 +2141,14 @@ case class ElementAt(left: Expression, right: Expression)
            """.stripMargin
         })
       case _: MapType =>
-        doGetValueGenCode(ctx, ev, left.dataType.asInstanceOf[MapType])
+        doGetValueGenCode(ctx, ev, left.dataType.asInstanceOf[MapType], failOnError)
     }
   }
 
   override def prettyName: String = "element_at"
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): ElementAt = copy(left = newLeft, right = newRight)
 }
 
 /**
@@ -2044,11 +2167,14 @@ case class ElementAt(left: Expression, right: Expression)
   note = """
     Concat logic for arrays is available since 2.4.0.
   """,
-  group = "array_funcs")
+  group = "collection_funcs",
+  since = "1.5.0")
 case class Concat(children: Seq[Expression]) extends ComplexTypeMergingExpression {
 
   private def allowedTypes: Seq[AbstractDataType] = Seq(StringType, BinaryType, ArrayType)
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(CONCAT)
+
   override def checkInputDataTypes(): TypeCheckResult = {
     if (children.isEmpty) {
       TypeCheckResult.TypeCheckSuccess
@@ -2231,6 +2357,9 @@ case class Concat(children: Seq[Expression]) extends ComplexTypeMergingExpressio
   override def toString: String = s"concat(${children.mkString(", ")})"
 
   override def sql: String = s"concat(${children.map(_.sql).mkString(", ")})"
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Concat =
+    copy(children = newChildren)
 }
 
 /**
@@ -2245,7 +2374,7 @@ case class Concat(children: Seq[Expression]) extends ComplexTypeMergingExpressio
   """,
   group = "array_funcs",
   since = "2.4.0")
-case class Flatten(child: Expression) extends UnaryExpression {
+case class Flatten(child: Expression) extends UnaryExpression with NullIntolerant {
 
   private def childDataType: ArrayType = child.dataType.asInstanceOf[ArrayType]
 
@@ -2343,6 +2472,9 @@ case class Flatten(child: Expression) extends UnaryExpression {
   }
 
   override def prettyName: String = "flatten"
+
+  override protected def withNewChildInternal(newChild: Expression): Flatten =
+    copy(child = newChild)
 }
 
 @ExpressionDescription(
@@ -2355,8 +2487,8 @@ case class Flatten(child: Expression) extends UnaryExpression {
 
       The start and stop expressions must resolve to the same type.
       If start and stop expressions resolve to the 'date' or 'timestamp' type
-      then the step expression must resolve to the 'interval' type, otherwise to the same type
-      as the start and stop expressions.
+      then the step expression must resolve to the 'interval' or 'year-month interval' or
+      'day-time interval' type, otherwise to the same type as the start and stop expressions.
   """,
   arguments = """
     Arguments:
@@ -2375,6 +2507,8 @@ case class Flatten(child: Expression) extends UnaryExpression {
        [5,4,3,2,1]
       > SELECT _FUNC_(to_date('2018-01-01'), to_date('2018-03-01'), interval 1 month);
        [2018-01-01,2018-02-01,2018-03-01]
+      > SELECT _FUNC_(to_date('2018-01-01'), to_date('2018-03-01'), interval '0-1' year to month);
+       [2018-01-01,2018-02-01,2018-03-01]
   """,
   group = "array_funcs",
   since = "2.4.0"
@@ -2400,6 +2534,15 @@ case class Sequence(
 
   override def children: Seq[Expression] = Seq(start, stop) ++ stepOpt
 
+  override def withNewChildrenInternal(
+      newChildren: IndexedSeq[Expression]): TimeZoneAwareExpression = {
+    if (stepOpt.isDefined) {
+      copy(start = newChildren(0), stop = newChildren(1), stepOpt = Some(newChildren(2)))
+    } else {
+      copy(start = newChildren(0), stop = newChildren(1))
+    }
+  }
+
   override def foldable: Boolean = children.forall(_.foldable)
 
   override def nullable: Boolean = children.exists(_.nullable)
@@ -2412,8 +2555,13 @@ case class Sequence(
     val typesCorrect =
       startType.sameType(stop.dataType) &&
         (startType match {
-          case TimestampType | DateType =>
-            stepOpt.isEmpty || CalendarIntervalType.acceptsType(stepType)
+          case TimestampType =>
+            stepOpt.isEmpty || CalendarIntervalType.acceptsType(stepType) ||
+              YearMonthIntervalType.acceptsType(stepType) ||
+              DayTimeIntervalType.acceptsType(stepType)
+          case DateType =>
+            stepOpt.isEmpty || CalendarIntervalType.acceptsType(stepType) ||
+              YearMonthIntervalType.acceptsType(stepType)
           case _: IntegralType =>
             stepOpt.isEmpty || stepType.sameType(startType)
           case _ => false
@@ -2423,29 +2571,51 @@ case class Sequence(
       TypeCheckResult.TypeCheckSuccess
     } else {
       TypeCheckResult.TypeCheckFailure(
-        s"$prettyName only supports integral, timestamp or date types")
+        s"""
+           |$prettyName uses the wrong parameter type. The parameter type must conform to:
+           |1. The start and stop expressions must resolve to the same type.
+           |2. If start and stop expressions resolve to the 'date' or 'timestamp' type
+           |then the step expression must resolve to the 'interval' or
+           |'${YearMonthIntervalType.typeName}' or '${DayTimeIntervalType.typeName}' type,
+           |otherwise to the same type as the start and stop expressions.
+         """.stripMargin)
     }
   }
 
-  def coercibleChildren: Seq[Expression] = children.filter(_.dataType != CalendarIntervalType)
+  private def isNotIntervalType(expr: Expression) = expr.dataType match {
+    case CalendarIntervalType | YearMonthIntervalType | DayTimeIntervalType => false
+    case _ => true
+  }
+
+  def coercibleChildren: Seq[Expression] = children.filter(isNotIntervalType)
 
   def castChildrenTo(widerType: DataType): Expression = Sequence(
     Cast(start, widerType),
     Cast(stop, widerType),
-    stepOpt.map(step => if (step.dataType != CalendarIntervalType) Cast(step, widerType) else step),
+    stepOpt.map(step => if (isNotIntervalType(step)) Cast(step, widerType) else step),
     timeZoneId)
 
-  @transient private lazy val impl: SequenceImpl = dataType.elementType match {
+  @transient private lazy val impl: InternalSequence = dataType.elementType match {
     case iType: IntegralType =>
       type T = iType.InternalType
       val ct = ClassTag[T](iType.tag.mirror.runtimeClass(iType.tag.tpe))
       new IntegralSequenceImpl(iType)(ct, iType.integral)
 
     case TimestampType =>
-      new TemporalSequenceImpl[Long](LongType, 1, identity, zoneId)
+      if (stepOpt.isEmpty || CalendarIntervalType.acceptsType(stepOpt.get.dataType)) {
+        new TemporalSequenceImpl[Long](LongType, 1, identity, zoneId)
+      } else if (YearMonthIntervalType.acceptsType(stepOpt.get.dataType)) {
+        new PeriodSequenceImpl[Long](LongType, 1, identity, zoneId)
+      } else {
+        new DurationSequenceImpl[Long](LongType, 1, identity, zoneId)
+      }
 
     case DateType =>
-      new TemporalSequenceImpl[Int](IntegerType, MICROS_PER_DAY, _.toInt, zoneId)
+      if (stepOpt.isEmpty || CalendarIntervalType.acceptsType(stepOpt.get.dataType)) {
+        new TemporalSequenceImpl[Int](IntegerType, MICROS_PER_DAY, _.toInt, zoneId)
+      } else {
+        new PeriodSequenceImpl[Int](IntegerType, MICROS_PER_DAY, _.toInt, zoneId)
+      }
   }
 
   override def eval(input: InternalRow): Any = {
@@ -2528,7 +2698,7 @@ object Sequence {
     }
   }
 
-  private trait SequenceImpl {
+  private trait InternalSequence {
     def eval(start: Any, stop: Any, step: Any): Any
 
     def genCode(
@@ -2543,7 +2713,7 @@ object Sequence {
   }
 
   private class IntegralSequenceImpl[T: ClassTag]
-    (elemType: IntegralType)(implicit num: Integral[T]) extends SequenceImpl {
+    (elemType: IntegralType)(implicit num: Integral[T]) extends InternalSequence {
 
     override val defaultStep: DefaultStep = new DefaultStep(
       (elemType.ordering.lteq _).asInstanceOf[LessThanOrEqualFn],
@@ -2557,7 +2727,7 @@ object Sequence {
       val stop = input2.asInstanceOf[T]
       val step = input3.asInstanceOf[T]
 
-      var i: Int = getSequenceLength(start, stop, step)
+      var i: Int = getSequenceLength(start, stop, step, step)
       val arr = new Array[T](i)
       while (i > 0) {
         i -= 1
@@ -2575,7 +2745,7 @@ object Sequence {
         elemType: String): String = {
       val i = ctx.freshName("i")
       s"""
-         |${genSequenceLengthCode(ctx, start, stop, step, i)}
+         |${genSequenceLengthCode(ctx, start, stop, step, step, i)}
          |$arr = new $elemType[$i];
          |while ($i > 0) {
          |  $i--;
@@ -2585,46 +2755,127 @@ object Sequence {
     }
   }
 
+  private class PeriodSequenceImpl[T: ClassTag]
+      (dt: IntegralType, scale: Long, fromLong: Long => T, zoneId: ZoneId)
+      (implicit num: Integral[T]) extends InternalSequenceBase(dt, scale, fromLong, zoneId) {
+
+    override val defaultStep: DefaultStep = new DefaultStep(
+      (dt.ordering.lteq _).asInstanceOf[LessThanOrEqualFn],
+      YearMonthIntervalType,
+      Period.of(0, 1, 0))
+
+    val intervalType: DataType = YearMonthIntervalType
+
+    def splitStep(input: Any): (Int, Int, Long) = {
+      (input.asInstanceOf[Int], 0, 0)
+    }
+
+    def stepSplitCode(
+        stepMonths: String, stepDays: String, stepMicros: String, step: String): String = {
+      s"""
+         |final int $stepMonths = $step;
+         |final int $stepDays = 0;
+         |final long $stepMicros = 0L;
+       """.stripMargin
+    }
+  }
+
+  private class DurationSequenceImpl[T: ClassTag]
+      (dt: IntegralType, scale: Long, fromLong: Long => T, zoneId: ZoneId)
+      (implicit num: Integral[T]) extends InternalSequenceBase(dt, scale, fromLong, zoneId) {
+
+    override val defaultStep: DefaultStep = new DefaultStep(
+      (dt.ordering.lteq _).asInstanceOf[LessThanOrEqualFn],
+      DayTimeIntervalType,
+      Duration.ofDays(1))
+
+    val intervalType: DataType = DayTimeIntervalType
+
+    def splitStep(input: Any): (Int, Int, Long) = {
+      (0, 0, input.asInstanceOf[Long])
+    }
+
+    def stepSplitCode(
+        stepMonths: String, stepDays: String, stepMicros: String, step: String): String = {
+      s"""
+         |final int $stepMonths = 0;
+         |final int $stepDays = 0;
+         |final long $stepMicros = $step;
+       """.stripMargin
+    }
+  }
+
   private class TemporalSequenceImpl[T: ClassTag]
       (dt: IntegralType, scale: Long, fromLong: Long => T, zoneId: ZoneId)
-      (implicit num: Integral[T]) extends SequenceImpl {
+      (implicit num: Integral[T]) extends InternalSequenceBase(dt, scale, fromLong, zoneId) {
 
     override val defaultStep: DefaultStep = new DefaultStep(
       (dt.ordering.lteq _).asInstanceOf[LessThanOrEqualFn],
       CalendarIntervalType,
       new CalendarInterval(0, 1, 0))
 
+    val intervalType: DataType = CalendarIntervalType
+
+    def splitStep(input: Any): (Int, Int, Long) = {
+      val step = input.asInstanceOf[CalendarInterval]
+      (step.months, step.days, step.microseconds)
+    }
+
+    def stepSplitCode(
+        stepMonths: String, stepDays: String, stepMicros: String, step: String): String = {
+      s"""
+         |final int $stepMonths = $step.months;
+         |final int $stepDays = $step.days;
+         |final long $stepMicros = $step.microseconds;
+       """.stripMargin
+    }
+  }
+
+  private abstract class InternalSequenceBase[T: ClassTag]
+      (dt: IntegralType, scale: Long, fromLong: Long => T, zoneId: ZoneId)
+      (implicit num: Integral[T]) extends InternalSequence {
+
+    val defaultStep: DefaultStep
+
     private val backedSequenceImpl = new IntegralSequenceImpl[T](dt)
-    private val microsPerDay = HOURS_PER_DAY * MICROS_PER_HOUR
     // We choose a minimum days(28) in one month to calculate the `intervalStepInMicros`
     // in order to make sure the estimated array length is long enough
-    private val microsPerMonth = 28 * microsPerDay
+    private val microsPerMonth = 28 * MICROS_PER_DAY
+
+    protected val intervalType: DataType
+
+    protected def splitStep(input: Any): (Int, Int, Long)
 
     override def eval(input1: Any, input2: Any, input3: Any): Array[T] = {
       val start = input1.asInstanceOf[T]
       val stop = input2.asInstanceOf[T]
-      val step = input3.asInstanceOf[CalendarInterval]
-      val stepMonths = step.months
-      val stepDays = step.days
-      val stepMicros = step.microseconds
+      val (stepMonths, stepDays, stepMicros) = splitStep(input3)
+
+      if (scale == MICROS_PER_DAY && stepMonths == 0 && stepDays == 0) {
+        throw new IllegalArgumentException(s"sequence step must be an ${intervalType.typeName}" +
+          " of day granularity if start and end values are dates")
+      }
 
       if (stepMonths == 0 && stepMicros == 0 && scale == MICROS_PER_DAY) {
+        // Adding pure days to date start/end
         backedSequenceImpl.eval(start, stop, fromLong(stepDays))
 
       } else if (stepMonths == 0 && stepDays == 0 && scale == 1) {
+        // Adding pure microseconds to timestamp start/end
         backedSequenceImpl.eval(start, stop, fromLong(stepMicros))
 
       } else {
         // To estimate the resulted array length we need to make assumptions
         // about a month length in days and a day length in microseconds
         val intervalStepInMicros =
-          stepMicros + stepMonths * microsPerMonth + stepDays * microsPerDay
+          stepMicros + stepMonths * microsPerMonth + stepDays * MICROS_PER_DAY
         val startMicros: Long = num.toLong(start) * scale
         val stopMicros: Long = num.toLong(stop) * scale
+
         val maxEstimatedArrayLength =
-          getSequenceLength(startMicros, stopMicros, intervalStepInMicros)
+          getSequenceLength(startMicros, stopMicros, input3, intervalStepInMicros)
 
-        val stepSign = if (stopMicros > startMicros) +1 else -1
+        val stepSign = if (stopMicros >= startMicros) +1 else -1
         val exclusiveItem = stopMicros + stepSign
         val arr = new Array[T](maxEstimatedArrayLength)
         var t = startMicros
@@ -2642,6 +2893,9 @@ object Sequence {
       }
     }
 
+    protected def stepSplitCode(
+         stepMonths: String, stepDays: String, stepMicros: String, step: String): String
+
     override def genCode(
         ctx: CodegenContext,
         start: String,
@@ -2666,14 +2920,29 @@ object Sequence {
       val sequenceLengthCode =
         s"""
            |final long $intervalInMicros =
-           |  $stepMicros + $stepMonths * ${microsPerMonth}L + $stepDays * ${microsPerDay}L;
-           |${genSequenceLengthCode(ctx, startMicros, stopMicros, intervalInMicros, arrLength)}
-          """.stripMargin
+           |  $stepMicros + $stepMonths * ${microsPerMonth}L + $stepDays * ${MICROS_PER_DAY}L;
+           |${genSequenceLengthCode(
+              ctx, startMicros, stopMicros, step, intervalInMicros, arrLength)}
+         """.stripMargin
+
+      val check = if (scale == MICROS_PER_DAY) {
+        s"""
+           |if ($stepMonths == 0 && $stepDays == 0) {
+           |  throw new IllegalArgumentException(
+           |    "sequence step must be an ${intervalType.typeName} " +
+           |    "of day granularity if start and end values are dates");
+           |}
+         """.stripMargin
+        } else {
+          ""
+        }
+
+      val stepSplits = stepSplitCode(stepMonths, stepDays, stepMicros, step)
 
       s"""
-         |final int $stepMonths = $step.months;
-         |final int $stepDays = $step.days;
-         |final long $stepMicros = $step.microseconds;
+         |$stepSplits
+         |
+         |$check
          |
          |if ($stepMonths == 0 && $stepMicros == 0 && ${scale}L == ${MICROS_PER_DAY}L) {
          |  ${backedSequenceImpl.genCode(ctx, start, stop, stepDays, arr, elemType)};
@@ -2686,7 +2955,7 @@ object Sequence {
          |
          |  $sequenceLengthCode
          |
-         |  final int $stepSign = $stopMicros > $startMicros ? +1 : -1;
+         |  final int $stepSign = $stopMicros >= $startMicros ? +1 : -1;
          |  final long $exclusiveItem = $stopMicros + $stepSign;
          |
          |  $arr = new $elemType[$arrLength];
@@ -2708,15 +2977,16 @@ object Sequence {
     }
   }
 
-  private def getSequenceLength[U](start: U, stop: U, step: U)(implicit num: Integral[U]): Int = {
+  private def getSequenceLength[U](start: U, stop: U, step: Any, estimatedStep: U)
+      (implicit num: Integral[U]): Int = {
     import num._
     require(
-      (step > num.zero && start <= stop)
-        || (step < num.zero && start >= stop)
-        || (step == num.zero && start == stop),
+      (estimatedStep > num.zero && start <= stop)
+        || (estimatedStep < num.zero && start >= stop)
+        || (estimatedStep == num.zero && start == stop),
       s"Illegal sequence boundaries: $start to $stop by $step")
 
-    val len = if (start == stop) 1L else 1L + (stop.toLong - start.toLong) / step.toLong
+    val len = if (start == stop) 1L else 1L + (stop.toLong - start.toLong) / estimatedStep.toLong
 
     require(
       len <= MAX_ROUNDED_ARRAY_LENGTH,
@@ -2730,16 +3000,17 @@ object Sequence {
       start: String,
       stop: String,
       step: String,
+      estimatedStep: String,
       len: String): String = {
     val longLen = ctx.freshName("longLen")
     s"""
-       |if (!(($step > 0 && $start <= $stop) ||
-       |  ($step < 0 && $start >= $stop) ||
-       |  ($step == 0 && $start == $stop))) {
+       |if (!(($estimatedStep > 0 && $start <= $stop) ||
+       |  ($estimatedStep < 0 && $start >= $stop) ||
+       |  ($estimatedStep == 0 && $start == $stop))) {
        |  throw new IllegalArgumentException(
        |    "Illegal sequence boundaries: " + $start + " to " + $stop + " by " + $step);
        |}
-       |long $longLen = $stop == $start ? 1L : 1L + ((long) $stop - $start) / $step;
+       |long $longLen = $stop == $start ? 1L : 1L + ((long) $stop - $start) / $estimatedStep;
        |if ($longLen > $MAX_ROUNDED_ARRAY_LENGTH) {
        |  throw new IllegalArgumentException(
        |    "Too long sequence: " + $longLen + ". Should be <= $MAX_ROUNDED_ARRAY_LENGTH");
@@ -2869,6 +3140,8 @@ case class ArrayRepeat(left: Expression, right: Expression)
      """.stripMargin
   }
 
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): ArrayRepeat = copy(left = newLeft, right = newRight)
 }
 
 /**
@@ -2884,7 +3157,7 @@ case class ArrayRepeat(left: Expression, right: Expression)
   group = "array_funcs",
   since = "2.4.0")
 case class ArrayRemove(left: Expression, right: Expression)
-  extends BinaryExpression with ImplicitCastInputTypes {
+  extends BinaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def dataType: DataType = left.dataType
 
@@ -2983,6 +3256,9 @@ case class ArrayRemove(left: Expression, right: Expression)
   }
 
   override def prettyName: String = "array_remove"
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): ArrayRemove = copy(left = newLeft, right = newRight)
 }
 
 /**
@@ -3026,6 +3302,7 @@ trait ArraySetLike {
   @transient protected lazy val nullValueHolder = et match {
     case ByteType => "(byte) 0"
     case ShortType => "(short) 0"
+    case LongType => "(long) 0"
     case _ => "0"
   }
 
@@ -3081,7 +3358,7 @@ trait ArraySetLike {
   group = "array_funcs",
   since = "2.4.0")
 case class ArrayDistinct(child: Expression)
-  extends UnaryExpression with ArraySetLike with ExpectsInputTypes {
+  extends UnaryExpression with ArraySetLike with ExpectsInputTypes with NullIntolerant {
 
   override def inputTypes: Seq[AbstractDataType] = Seq(ArrayType)
 
@@ -3131,7 +3408,7 @@ case class ArrayDistinct(child: Expression)
           }
         }
       }
-      new GenericArrayData(arrayBuffer)
+      new GenericArrayData(arrayBuffer.toSeq)
     }
   }
 
@@ -3214,12 +3491,16 @@ case class ArrayDistinct(child: Expression)
   }
 
   override def prettyName: String = "array_distinct"
+
+  override protected def withNewChildInternal(newChild: Expression): ArrayDistinct =
+    copy(child = newChild)
 }
 
 /**
  * Will become common base class for [[ArrayUnion]], [[ArrayIntersect]], and [[ArrayExcept]].
  */
-trait ArrayBinaryLike extends BinaryArrayExpressionWithImplicitCast with ArraySetLike {
+trait ArrayBinaryLike
+  extends BinaryArrayExpressionWithImplicitCast with ArraySetLike with NullIntolerant {
   override protected def dt: DataType = dataType
   override protected def et: DataType = elementType
 
@@ -3288,7 +3569,7 @@ case class ArrayUnion(left: Expression, right: Expression) extends ArrayBinaryLi
             i += 1
           }
         }
-        new GenericArrayData(arrayBuffer)
+        new GenericArrayData(arrayBuffer.toSeq)
     } else {
       (array1, array2) =>
         val arrayBuffer = new scala.collection.mutable.ArrayBuffer[Any]
@@ -3319,7 +3600,7 @@ case class ArrayUnion(left: Expression, right: Expression) extends ArrayBinaryLi
             arrayBuffer += elem
           }
         }))
-        new GenericArrayData(arrayBuffer)
+        new GenericArrayData(arrayBuffer.toSeq)
     }
   }
 
@@ -3415,6 +3696,9 @@ case class ArrayUnion(left: Expression, right: Expression) extends ArrayBinaryLi
   }
 
   override def prettyName: String = "array_union"
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): ArrayUnion = copy(left = newLeft, right = newRight)
 }
 
 object ArrayUnion {
@@ -3451,7 +3735,7 @@ object ArrayUnion {
         arrayBuffer += elem
       }
     }))
-    new GenericArrayData(arrayBuffer)
+    new GenericArrayData(arrayBuffer.toSeq)
   }
 }
 
@@ -3472,13 +3756,16 @@ object ArrayUnion {
   since = "2.4.0")
 case class ArrayIntersect(left: Expression, right: Expression) extends ArrayBinaryLike
   with ComplexTypeMergingExpression {
-  override def dataType: DataType = {
+
+  private lazy val internalDataType: DataType = {
     dataTypeCheck
     ArrayType(elementType,
       left.dataType.asInstanceOf[ArrayType].containsNull &&
         right.dataType.asInstanceOf[ArrayType].containsNull)
   }
 
+  override def dataType: DataType = internalDataType
+
   @transient lazy val evalIntersect: (ArrayData, ArrayData) => ArrayData = {
     if (TypeUtils.typeWithProperEquals(elementType)) {
       (array1, array2) =>
@@ -3513,7 +3800,7 @@ case class ArrayIntersect(left: Expression, right: Expression) extends ArrayBina
             }
             i += 1
           }
-          new GenericArrayData(arrayBuffer)
+          new GenericArrayData(arrayBuffer.toSeq)
         } else {
           new GenericArrayData(Array.emptyObjectArray)
         }
@@ -3561,7 +3848,7 @@ case class ArrayIntersect(left: Expression, right: Expression) extends ArrayBina
             }
             i += 1
           }
-          new GenericArrayData(arrayBuffer)
+          new GenericArrayData(arrayBuffer.toSeq)
         } else {
           new GenericArrayData(Array.emptyObjectArray)
         }
@@ -3695,6 +3982,10 @@ case class ArrayIntersect(left: Expression, right: Expression) extends ArrayBina
   }
 
   override def prettyName: String = "array_intersect"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): ArrayIntersect =
+    copy(left = newLeft, right = newRight)
 }
 
 /**
@@ -3715,11 +4006,13 @@ case class ArrayIntersect(left: Expression, right: Expression) extends ArrayBina
 case class ArrayExcept(left: Expression, right: Expression) extends ArrayBinaryLike
   with ComplexTypeMergingExpression {
 
-  override def dataType: DataType = {
+  private lazy val internalDataType: DataType = {
     dataTypeCheck
     left.dataType
   }
 
+  override def dataType: DataType = internalDataType
+
   @transient lazy val evalExcept: (ArrayData, ArrayData) => ArrayData = {
     if (TypeUtils.typeWithProperEquals(elementType)) {
       (array1, array2) =>
@@ -3752,7 +4045,7 @@ case class ArrayExcept(left: Expression, right: Expression) extends ArrayBinaryL
           }
           i += 1
         }
-        new GenericArrayData(arrayBuffer)
+        new GenericArrayData(arrayBuffer.toSeq)
     } else {
       (array1, array2) =>
         val arrayBuffer = new scala.collection.mutable.ArrayBuffer[Any]
@@ -3797,7 +4090,7 @@ case class ArrayExcept(left: Expression, right: Expression) extends ArrayBinaryL
           }
           i += 1
         }
-        new GenericArrayData(arrayBuffer)
+        new GenericArrayData(arrayBuffer.toSeq)
     }
   }
 
@@ -3917,4 +4210,7 @@ case class ArrayExcept(left: Expression, right: Expression) extends ArrayBinaryL
   }
 
   override def prettyName: String = "array_except"
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): ArrayExcept = copy(left = newLeft, right = newRight)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
index 858c91a4d8e86..76974c8229b2e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
@@ -17,16 +17,33 @@
 
 package org.apache.spark.sql.catalyst.expressions
 
+import scala.collection.mutable.ArrayBuffer
+
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, TypeCoercion}
-import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder
+import org.apache.spark.sql.catalyst.analysis.{Resolver, TypeCheckResult, TypeCoercion, UnresolvedAttribute, UnresolvedExtractValue}
+import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.{FUNC_ALIAS, FunctionBuilder}
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
+import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
+import org.apache.spark.sql.catalyst.trees.TreePattern.{CREATE_NAMED_STRUCT, TreePattern}
+import org.apache.spark.sql.catalyst.trees.UnaryLike
 import org.apache.spark.sql.catalyst.util._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
+/**
+ * Trait to indicate the expression does not throw an exception by itself when they are evaluated.
+ * For example, UDFs, [[AssertTrue]], etc can throw an exception when they are executed.
+ * In such case, it is necessary to call [[Expression.eval]], and the optimization rule should
+ * not ignore it.
+ *
+ * This trait can be used in an optimization rule such as
+ * [[org.apache.spark.sql.catalyst.optimizer.ConstantFolding]] to fold the expressions that
+ * do not need to execute, for example, `size(array(c0, c1, c2))`.
+ */
+trait NoThrow
+
 /**
  * Returns an Array containing the evaluation of all children expressions.
  */
@@ -36,9 +53,11 @@ import org.apache.spark.unsafe.types.UTF8String
     Examples:
       > SELECT _FUNC_(1, 2, 3);
        [1,2,3]
-  """)
+  """,
+  since = "1.1.0",
+  group = "array_funcs")
 case class CreateArray(children: Seq[Expression], useStringTypeWhenEmpty: Boolean)
-  extends Expression {
+  extends Expression with NoThrow {
 
   def this(children: Seq[Expression]) = {
     this(children, SQLConf.get.getConf(SQLConf.LEGACY_CREATE_EMPTY_COLLECTION_USING_STRING_TYPE))
@@ -84,6 +103,9 @@ case class CreateArray(children: Seq[Expression], useStringTypeWhenEmpty: Boolea
   }
 
   override def prettyName: String = "array"
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): CreateArray =
+    copy(children = newChildren)
 }
 
 object CreateArray {
@@ -153,9 +175,11 @@ private [sql] object GenArrayData {
     Examples:
       > SELECT _FUNC_(1.0, '2', 3.0, '4');
        {1.0:"2",3.0:"4"}
-  """)
+  """,
+  since = "2.0.0",
+  group = "map_funcs")
 case class CreateMap(children: Seq[Expression], useStringTypeWhenEmpty: Boolean)
-  extends Expression {
+  extends Expression with NoThrow {
 
   def this(children: Seq[Expression]) = {
     this(children, SQLConf.get.getConf(SQLConf.LEGACY_CREATE_EMPTY_COLLECTION_USING_STRING_TYPE))
@@ -234,6 +258,9 @@ case class CreateMap(children: Seq[Expression], useStringTypeWhenEmpty: Boolean)
   }
 
   override def prettyName: String = "map"
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): CreateMap =
+    copy(children = newChildren)
 }
 
 object CreateMap {
@@ -253,9 +280,11 @@ object CreateMap {
     Examples:
       > SELECT _FUNC_(array(1.0, 3.0), array('2', '4'));
        {1.0:"2",3.0:"4"}
-  """, since = "2.4.0")
+  """,
+  since = "2.4.0",
+  group = "map_funcs")
 case class MapFromArrays(left: Expression, right: Expression)
-  extends BinaryExpression with ExpectsInputTypes {
+  extends BinaryExpression with ExpectsInputTypes with NullIntolerant {
 
   override def inputTypes: Seq[AbstractDataType] = Seq(ArrayType, ArrayType)
 
@@ -292,6 +321,10 @@ case class MapFromArrays(left: Expression, right: Expression)
   }
 
   override def prettyName: String = "map_from_arrays"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): MapFromArrays =
+    copy(left = newLeft, right = newRight)
 }
 
 /**
@@ -301,7 +334,6 @@ case class MapFromArrays(left: Expression, right: Expression)
  */
 case object NamePlaceholder extends LeafExpression with Unevaluable {
   override lazy val resolved: Boolean = false
-  override def foldable: Boolean = false
   override def nullable: Boolean = false
   override def dataType: DataType = StringType
   override def prettyName: String = "NamePlaceholder"
@@ -311,31 +343,59 @@ case object NamePlaceholder extends LeafExpression with Unevaluable {
 /**
  * Returns a Row containing the evaluation of all children expressions.
  */
-object CreateStruct extends FunctionBuilder {
+object CreateStruct {
+  /**
+   * Returns a named struct with generated names or using the names when available.
+   * It should not be used for `struct` expressions or functions explicitly called
+   * by users.
+   */
   def apply(children: Seq[Expression]): CreateNamedStruct = {
     CreateNamedStruct(children.zipWithIndex.flatMap {
+      // For multi-part column name like `struct(a.b.c)`, it may be resolved into:
+      //   1. Attribute if `a.b.c` is simply a qualified column name.
+      //   2. GetStructField if `a.b` refers to a struct-type column.
+      //   3. GetArrayStructFields if `a.b` refers to a array-of-struct-type column.
+      //   4. GetMapValue if `a.b` refers to a map-type column.
+      // We should always use the last part of the column name (`c` in the above example) as the
+      // alias name inside CreateNamedStruct.
+      case (u: UnresolvedAttribute, _) => Seq(Literal(u.nameParts.last), u)
       case (e: NamedExpression, _) if e.resolved => Seq(Literal(e.name), e)
       case (e: NamedExpression, _) => Seq(NamePlaceholder, e)
       case (e, index) => Seq(Literal(s"col${index + 1}"), e)
     })
   }
 
+  /**
+   * Returns a named struct with a pretty SQL name. It will show the pretty SQL string
+   * in its output column name as if `struct(...)` was called. Should be
+   * used for `struct` expressions or functions explicitly called by users.
+   */
+  def create(children: Seq[Expression]): CreateNamedStruct = {
+    val expr = CreateStruct(children)
+    expr.setTagValue(FUNC_ALIAS, "struct")
+    expr
+  }
+
   /**
    * Entry to use in the function registry.
    */
   val registryEntry: (String, (ExpressionInfo, FunctionBuilder)) = {
     val info: ExpressionInfo = new ExpressionInfo(
-      "org.apache.spark.sql.catalyst.expressions.NamedStruct",
+      classOf[CreateNamedStruct].getCanonicalName,
       null,
       "struct",
       "_FUNC_(col1, col2, col3, ...) - Creates a struct with the given field values.",
       "",
+      """
+        |    Examples:
+        |      > SELECT _FUNC_(1, 2, 3);
+        |       {"col1":1,"col2":2,"col3":3}
+        |  """.stripMargin,
       "",
-      "",
-      "",
-      "",
+      "struct_funcs",
+      "1.4.0",
       "")
-    ("struct", (info, this))
+    ("struct", (info, this.create))
   }
 }
 
@@ -351,9 +411,11 @@ object CreateStruct extends FunctionBuilder {
     Examples:
       > SELECT _FUNC_("a", 1, "b", 2, "c", 3);
        {"a":1,"b":2,"c":3}
-  """)
+  """,
+  since = "1.5.0",
+  group = "struct_funcs")
 // scalastyle:on line.size.limit
-case class CreateNamedStruct(children: Seq[Expression]) extends Expression {
+case class CreateNamedStruct(children: Seq[Expression]) extends Expression with NoThrow {
   lazy val (nameExprs, valExprs) = children.grouped(2).map {
     case Seq(name, value) => (name, value)
   }.toList.unzip
@@ -364,6 +426,8 @@ case class CreateNamedStruct(children: Seq[Expression]) extends Expression {
 
   override def foldable: Boolean = valExprs.forall(_.foldable)
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(CREATE_NAMED_STRUCT)
+
   override lazy val dataType: StructType = {
     val fields = names.zip(valExprs).map {
       case (name, expr) =>
@@ -433,7 +497,18 @@ case class CreateNamedStruct(children: Seq[Expression]) extends Expression {
        """.stripMargin, isNull = FalseLiteral)
   }
 
-  override def prettyName: String = "named_struct"
+  // There is an alias set at `CreateStruct.create`. If there is an alias,
+  // this is the struct function explicitly called by a user and we should
+  // respect it in the SQL string as `struct(...)`.
+  override def prettyName: String = getTagValue(FUNC_ALIAS).getOrElse("named_struct")
+
+  override def sql: String = getTagValue(FUNC_ALIAS).map { alias =>
+    val childrenSQL = children.indices.filter(_ % 2 == 1).map(children(_).sql).mkString(", ")
+    s"$alias($childrenSQL)"
+  }.getOrElse(super.sql)
+
+  override protected def withNewChildrenInternal(
+    newChildren: IndexedSeq[Expression]): CreateNamedStruct = copy(children = newChildren)
 }
 
 /**
@@ -449,10 +524,11 @@ case class CreateNamedStruct(children: Seq[Expression]) extends Expression {
       > SELECT _FUNC_('a');
        {"a":null}
   """,
-  since = "2.0.1")
+  since = "2.0.1",
+  group = "map_funcs")
 // scalastyle:on line.size.limit
 case class StringToMap(text: Expression, pairDelim: Expression, keyValueDelim: Expression)
-  extends TernaryExpression with ExpectsInputTypes {
+  extends TernaryExpression with ExpectsInputTypes with NullIntolerant {
 
   def this(child: Expression, pairDelim: Expression) = {
     this(child, pairDelim, Literal(":"))
@@ -462,7 +538,9 @@ case class StringToMap(text: Expression, pairDelim: Expression, keyValueDelim: E
     this(child, Literal(","), Literal(":"))
   }
 
-  override def children: Seq[Expression] = Seq(text, pairDelim, keyValueDelim)
+  override def first: Expression = text
+  override def second: Expression = pairDelim
+  override def third: Expression = keyValueDelim
 
   override def inputTypes: Seq[AbstractDataType] = Seq(StringType, StringType, StringType)
 
@@ -514,4 +592,179 @@ case class StringToMap(text: Expression, pairDelim: Expression, keyValueDelim: E
   }
 
   override def prettyName: String = "str_to_map"
+
+  override protected def withNewChildrenInternal(
+      newFirst: Expression, newSecond: Expression, newThird: Expression): Expression = copy(
+    text = newFirst,
+    pairDelim = newSecond,
+    keyValueDelim = newThird
+  )
+}
+
+/**
+ * Represents an operation to be applied to the fields of a struct.
+ */
+trait StructFieldsOperation {
+
+  val resolver: Resolver = SQLConf.get.resolver
+
+  /**
+   * Returns an updated list of StructFields and Expressions that will ultimately be used
+   * as the fields argument for [[StructType]] and as the children argument for
+   * [[CreateNamedStruct]] respectively inside of [[UpdateFields]].
+   */
+  def apply(values: Seq[(StructField, Expression)]): Seq[(StructField, Expression)]
+}
+
+/**
+ * Add or replace a field by name.
+ *
+ * We extend [[Unevaluable]] here to ensure that [[UpdateFields]] can include it as part of its
+ * children, and thereby enable the analyzer to resolve and transform valExpr as necessary.
+ */
+case class WithField(name: String, valExpr: Expression)
+  extends Unevaluable with StructFieldsOperation with UnaryLike[Expression] {
+
+  override def apply(values: Seq[(StructField, Expression)]): Seq[(StructField, Expression)] = {
+    val newFieldExpr = (StructField(name, valExpr.dataType, valExpr.nullable), valExpr)
+    val result = ArrayBuffer.empty[(StructField, Expression)]
+    var hasMatch = false
+    for (existingFieldExpr @ (existingField, _) <- values) {
+      if (resolver(existingField.name, name)) {
+        hasMatch = true
+        result += newFieldExpr
+      } else {
+        result += existingFieldExpr
+      }
+    }
+    if (!hasMatch) result += newFieldExpr
+    result.toSeq
+  }
+
+  override def child: Expression = valExpr
+
+  override def dataType: DataType = throw new IllegalStateException(
+    "WithField.dataType should not be called.")
+
+  override def nullable: Boolean = throw new IllegalStateException(
+    "WithField.nullable should not be called.")
+
+  override def prettyName: String = "WithField"
+
+  override protected def withNewChildInternal(newChild: Expression): WithField =
+    copy(valExpr = newChild)
+}
+
+/**
+ * Drop a field by name.
+ */
+case class DropField(name: String) extends StructFieldsOperation {
+  override def apply(values: Seq[(StructField, Expression)]): Seq[(StructField, Expression)] =
+    values.filterNot { case (field, _) => resolver(field.name, name) }
+}
+
+/**
+ * Updates fields in a struct.
+ */
+case class UpdateFields(structExpr: Expression, fieldOps: Seq[StructFieldsOperation])
+  extends Unevaluable {
+
+  override def checkInputDataTypes(): TypeCheckResult = {
+    val dataType = structExpr.dataType
+    if (!dataType.isInstanceOf[StructType]) {
+      TypeCheckResult.TypeCheckFailure("struct argument should be struct type, got: " +
+        dataType.catalogString)
+    } else if (newExprs.isEmpty) {
+      TypeCheckResult.TypeCheckFailure("cannot drop all fields in struct")
+    } else {
+      TypeCheckResult.TypeCheckSuccess
+    }
+  }
+
+  override def children: Seq[Expression] = structExpr +: fieldOps.collect {
+    case e: Expression => e
+  }
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
+    super.legacyWithNewChildren(newChildren)
+
+  override def dataType: StructType = StructType(newFields)
+
+  override def nullable: Boolean = structExpr.nullable
+
+  override def prettyName: String = "update_fields"
+
+  private lazy val newFieldExprs: Seq[(StructField, Expression)] = {
+    val existingFieldExprs: Seq[(StructField, Expression)] =
+      structExpr.dataType.asInstanceOf[StructType].fields.zipWithIndex.map {
+        case (field, i) => (field, GetStructField(structExpr, i))
+      }
+
+    fieldOps.foldLeft(existingFieldExprs)((exprs, op) => op(exprs))
+  }
+
+  private lazy val newFields: Seq[StructField] = newFieldExprs.map(_._1)
+
+  lazy val newExprs: Seq[Expression] = newFieldExprs.map(_._2)
+
+  lazy val evalExpr: Expression = {
+    val createNamedStructExpr = CreateNamedStruct(newFieldExprs.flatMap {
+      case (field, expr) => Seq(Literal(field.name), expr)
+    })
+
+    if (structExpr.nullable) {
+      If(IsNull(structExpr), Literal(null, dataType), createNamedStructExpr)
+    } else {
+      createNamedStructExpr
+    }
+  }
+}
+
+object UpdateFields {
+  private def nameParts(fieldName: String): Seq[String] = {
+    require(fieldName != null, "fieldName cannot be null")
+
+    if (fieldName.isEmpty) {
+      fieldName :: Nil
+    } else {
+      CatalystSqlParser.parseMultipartIdentifier(fieldName)
+    }
+  }
+
+  /**
+   * Adds/replaces field of `StructType` into `col` expression by name.
+   */
+  def apply(col: Expression, fieldName: String, expr: Expression): UpdateFields = {
+    updateFieldsHelper(col, nameParts(fieldName), name => WithField(name, expr))
+  }
+
+  /**
+   * Drops fields of `StructType` in `col` expression by name.
+   */
+  def apply(col: Expression, fieldName: String): UpdateFields = {
+    updateFieldsHelper(col, nameParts(fieldName), name => DropField(name))
+  }
+
+  private def updateFieldsHelper(
+      structExpr: Expression,
+      namePartsRemaining: Seq[String],
+      valueFunc: String => StructFieldsOperation) : UpdateFields = {
+    val fieldName = namePartsRemaining.head
+    if (namePartsRemaining.length == 1) {
+      UpdateFields(structExpr, valueFunc(fieldName) :: Nil)
+    } else {
+      val newStruct = if (structExpr.resolved) {
+        val resolver = SQLConf.get.resolver
+        ExtractValue(structExpr, Literal(fieldName), resolver)
+      } else {
+        UnresolvedExtractValue(structExpr, Literal(fieldName))
+      }
+
+      val newValue = updateFieldsHelper(
+        structExpr = newStruct,
+        namePartsRemaining = namePartsRemaining.tail,
+        valueFunc = valueFunc)
+      UpdateFields(structExpr, WithField(fieldName, newValue) :: Nil)
+    }
+  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
index 9c600c9d39cf7..7ae4e8ab85b05 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
@@ -21,7 +21,10 @@ import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis._
 import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodeGenerator, ExprCode}
+import org.apache.spark.sql.catalyst.trees.TreePattern.{EXTRACT_VALUE, TreePattern}
 import org.apache.spark.sql.catalyst.util.{quoteIdentifier, ArrayData, GenericArrayData, MapData, TypeUtils}
+import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -57,20 +60,15 @@ object ExtractValue {
         val fieldName = v.toString
         val ordinal = findField(fields, fieldName, resolver)
         GetArrayStructFields(child, fields(ordinal).copy(name = fieldName),
-          ordinal, fields.length, containsNull)
+          ordinal, fields.length, containsNull || fields(ordinal).nullable)
 
       case (_: ArrayType, _) => GetArrayItem(child, extraction)
 
       case (MapType(kt, _, _), _) => GetMapValue(child, extraction)
 
       case (otherType, _) =>
-        val errorMsg = otherType match {
-          case StructType(_) =>
-            s"Field name should be String Literal, but it's $extraction"
-          case other =>
-            s"Can't extract value from $child: need struct type but got ${other.catalogString}"
-        }
-        throw new AnalysisException(errorMsg)
+        throw QueryCompilationErrors.dataTypeUnsupportedByExtractValueError(
+          otherType, extraction, child)
     }
   }
 
@@ -93,7 +91,9 @@ object ExtractValue {
   }
 }
 
-trait ExtractValue extends Expression
+trait ExtractValue extends Expression {
+  final override val nodePatterns: Seq[TreePattern] = Seq(EXTRACT_VALUE)
+}
 
 /**
  * Returns the value of fields in the Struct `child`.
@@ -116,8 +116,10 @@ case class GetStructField(child: Expression, ordinal: Int, name: Option[String]
     s"$child.${name.getOrElse(fieldName)}"
   }
 
+  def extractFieldName: String = name.getOrElse(childSchema(ordinal).name)
+
   override def sql: String =
-    child.sql + s".${quoteIdentifier(name.getOrElse(childSchema(ordinal).name))}"
+    child.sql + s".${quoteIdentifier(extractFieldName)}"
 
   protected override def nullSafeEval(input: Any): Any =
     input.asInstanceOf[InternalRow].get(ordinal, childSchema(ordinal).dataType)
@@ -139,6 +141,9 @@ case class GetStructField(child: Expression, ordinal: Int, name: Option[String]
       }
     })
   }
+
+  override protected def withNewChildInternal(newChild: Expression): GetStructField =
+    copy(child = newChild)
 }
 
 /**
@@ -213,6 +218,9 @@ case class GetArrayStructFields(
       """
     })
   }
+
+  override protected def withNewChildInternal(newChild: Expression): GetArrayStructFields =
+    copy(child = newChild)
 }
 
 /**
@@ -220,10 +228,15 @@ case class GetArrayStructFields(
  *
  * We need to do type checking here as `ordinal` expression maybe unresolved.
  */
-case class GetArrayItem(child: Expression, ordinal: Expression)
+case class GetArrayItem(
+    child: Expression,
+    ordinal: Expression,
+    failOnError: Boolean = SQLConf.get.ansiEnabled)
   extends BinaryExpression with GetArrayItemUtil with ExpectsInputTypes with ExtractValue
   with NullIntolerant {
 
+  def this(child: Expression, ordinal: Expression) = this(child, ordinal, SQLConf.get.ansiEnabled)
+
   // We have done type checking for child in `ExtractValue`, so only need to check the `ordinal`.
   override def inputTypes: Seq[AbstractDataType] = Seq(AnyDataType, IntegralType)
 
@@ -232,13 +245,28 @@ case class GetArrayItem(child: Expression, ordinal: Expression)
 
   override def left: Expression = child
   override def right: Expression = ordinal
-  override def nullable: Boolean = computeNullabilityFromArray(left, right)
+  override def nullable: Boolean =
+    computeNullabilityFromArray(left, right, failOnError, nullability)
   override def dataType: DataType = child.dataType.asInstanceOf[ArrayType].elementType
 
+  private def nullability(elements: Seq[Expression], ordinal: Int): Boolean = {
+    if (ordinal >= 0 && ordinal < elements.length) {
+      elements(ordinal).nullable
+    } else {
+      !failOnError
+    }
+  }
+
   protected override def nullSafeEval(value: Any, ordinal: Any): Any = {
     val baseValue = value.asInstanceOf[ArrayData]
     val index = ordinal.asInstanceOf[Number].intValue()
-    if (index >= baseValue.numElements() || index < 0 || baseValue.isNullAt(index)) {
+    if (index >= baseValue.numElements() || index < 0) {
+      if (failOnError) {
+        throw QueryExecutionErrors.invalidArrayIndexError(index, baseValue.numElements)
+      } else {
+        null
+      }
+    } else if (baseValue.isNullAt(index)) {
       null
     } else {
       baseValue.get(index, dataType)
@@ -249,20 +277,34 @@ case class GetArrayItem(child: Expression, ordinal: Expression)
     nullSafeCodeGen(ctx, ev, (eval1, eval2) => {
       val index = ctx.freshName("index")
       val nullCheck = if (child.dataType.asInstanceOf[ArrayType].containsNull) {
-        s" || $eval1.isNullAt($index)"
+        s"""else if ($eval1.isNullAt($index)) {
+               ${ev.isNull} = true;
+            }
+         """
       } else {
         ""
       }
+
+      val indexOutOfBoundBranch = if (failOnError) {
+        s"throw QueryExecutionErrors.invalidArrayIndexError($index, $eval1.numElements());"
+      } else {
+        s"${ev.isNull} = true;"
+      }
+
       s"""
         final int $index = (int) $eval2;
-        if ($index >= $eval1.numElements() || $index < 0$nullCheck) {
-          ${ev.isNull} = true;
-        } else {
+        if ($index >= $eval1.numElements() || $index < 0) {
+          $indexOutOfBoundBranch
+        } $nullCheck else {
           ${ev.value} = ${CodeGenerator.getValue(eval1, dataType, index)};
         }
       """
     })
   }
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): GetArrayItem =
+    copy(child = newLeft, ordinal = newRight)
 }
 
 /**
@@ -271,20 +313,24 @@ case class GetArrayItem(child: Expression, ordinal: Expression)
 trait GetArrayItemUtil {
 
   /** `Null` is returned for invalid ordinals. */
-  protected def computeNullabilityFromArray(child: Expression, ordinal: Expression): Boolean = {
+  protected def computeNullabilityFromArray(
+      child: Expression,
+      ordinal: Expression,
+      failOnError: Boolean,
+      nullability: (Seq[Expression], Int) => Boolean): Boolean = {
+    val arrayContainsNull = child.dataType.asInstanceOf[ArrayType].containsNull
     if (ordinal.foldable && !ordinal.nullable) {
       val intOrdinal = ordinal.eval().asInstanceOf[Number].intValue()
       child match {
-        case CreateArray(ar, _) if intOrdinal < ar.length =>
-          ar(intOrdinal).nullable
-        case GetArrayStructFields(CreateArray(elements, _), field, _, _, _)
-          if intOrdinal < elements.length =>
-          elements(intOrdinal).nullable || field.nullable
+        case CreateArray(ar, _) =>
+          nullability(ar, intOrdinal)
+        case GetArrayStructFields(CreateArray(elements, _), field, _, _, _) =>
+          nullability(elements, intOrdinal) || field.nullable
         case _ =>
           true
       }
     } else {
-      true
+      if (failOnError) arrayContainsNull else true
     }
   }
 }
@@ -295,7 +341,12 @@ trait GetArrayItemUtil {
 trait GetMapValueUtil extends BinaryExpression with ImplicitCastInputTypes {
 
   // todo: current search is O(n), improve it.
-  def getValueEval(value: Any, ordinal: Any, keyType: DataType, ordering: Ordering[Any]): Any = {
+  def getValueEval(
+      value: Any,
+      ordinal: Any,
+      keyType: DataType,
+      ordering: Ordering[Any],
+      failOnError: Boolean): Any = {
     val map = value.asInstanceOf[MapData]
     val length = map.numElements()
     val keys = map.keyArray()
@@ -311,14 +362,24 @@ trait GetMapValueUtil extends BinaryExpression with ImplicitCastInputTypes {
       }
     }
 
-    if (!found || values.isNullAt(i)) {
+    if (!found) {
+      if (failOnError) {
+        throw QueryExecutionErrors.mapKeyNotExistError(ordinal)
+      } else {
+        null
+      }
+    } else if (values.isNullAt(i)) {
       null
     } else {
       values.get(i, dataType)
     }
   }
 
-  def doGetValueGenCode(ctx: CodegenContext, ev: ExprCode, mapType: MapType): ExprCode = {
+  def doGetValueGenCode(
+      ctx: CodegenContext,
+      ev: ExprCode,
+      mapType: MapType,
+      failOnError: Boolean): ExprCode = {
     val index = ctx.freshName("index")
     val length = ctx.freshName("length")
     val keys = ctx.freshName("keys")
@@ -327,12 +388,22 @@ trait GetMapValueUtil extends BinaryExpression with ImplicitCastInputTypes {
     val values = ctx.freshName("values")
     val keyType = mapType.keyType
     val nullCheck = if (mapType.valueContainsNull) {
-      s" || $values.isNullAt($index)"
+      s"""else if ($values.isNullAt($index)) {
+            ${ev.isNull} = true;
+          }
+       """
     } else {
       ""
     }
+
     val keyJavaType = CodeGenerator.javaType(keyType)
     nullSafeCodeGen(ctx, ev, (eval1, eval2) => {
+      val keyNotFoundBranch = if (failOnError) {
+        s"throw QueryExecutionErrors.mapKeyNotExistError($eval2);"
+      } else {
+        s"${ev.isNull} = true;"
+      }
+
       s"""
         final int $length = $eval1.numElements();
         final ArrayData $keys = $eval1.keyArray();
@@ -349,9 +420,9 @@ trait GetMapValueUtil extends BinaryExpression with ImplicitCastInputTypes {
           }
         }
 
-        if (!$found$nullCheck) {
-          ${ev.isNull} = true;
-        } else {
+        if (!$found) {
+          $keyNotFoundBranch
+        } $nullCheck else {
           ${ev.value} = ${CodeGenerator.getValue(values, dataType, index)};
         }
       """
@@ -364,9 +435,14 @@ trait GetMapValueUtil extends BinaryExpression with ImplicitCastInputTypes {
  *
  * We need to do type checking here as `key` expression maybe unresolved.
  */
-case class GetMapValue(child: Expression, key: Expression)
+case class GetMapValue(
+    child: Expression,
+    key: Expression,
+    failOnError: Boolean = SQLConf.get.ansiEnabled)
   extends GetMapValueUtil with ExtractValue with NullIntolerant {
 
+  def this(child: Expression, key: Expression) = this(child, key, SQLConf.get.ansiEnabled)
+
   @transient private lazy val ordering: Ordering[Any] =
     TypeUtils.getInterpretedOrdering(keyType)
 
@@ -401,10 +477,14 @@ case class GetMapValue(child: Expression, key: Expression)
 
   // todo: current search is O(n), improve it.
   override def nullSafeEval(value: Any, ordinal: Any): Any = {
-    getValueEval(value, ordinal, keyType, ordering)
+    getValueEval(value, ordinal, keyType, ordering, failOnError)
   }
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    doGetValueGenCode(ctx, ev, child.dataType.asInstanceOf[MapType])
+    doGetValueGenCode(ctx, ev, child.dataType.asInstanceOf[MapType], failOnError)
   }
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): GetMapValue =
+    copy(child = newLeft, key = newRight)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala
index 6c6210994954c..3e356f1e8a26c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala
@@ -21,6 +21,8 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, TypeCoercion}
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
+import org.apache.spark.sql.catalyst.trees.TernaryLike
+import org.apache.spark.sql.catalyst.trees.TreePattern.{CASE_WHEN, IF, TreePattern}
 import org.apache.spark.sql.types._
 
 // scalastyle:off line.size.limit
@@ -30,19 +32,25 @@ import org.apache.spark.sql.types._
     Examples:
       > SELECT _FUNC_(1 < 2, 'a', 'b');
        a
-  """)
+  """,
+  since = "1.0.0",
+  group = "conditional_funcs")
 // scalastyle:on line.size.limit
 case class If(predicate: Expression, trueValue: Expression, falseValue: Expression)
-  extends ComplexTypeMergingExpression {
+  extends ComplexTypeMergingExpression with TernaryLike[Expression] {
 
   @transient
   override lazy val inputTypesForMerging: Seq[DataType] = {
     Seq(trueValue.dataType, falseValue.dataType)
   }
 
-  override def children: Seq[Expression] = predicate :: trueValue :: falseValue :: Nil
+  override def first: Expression = predicate
+  override def second: Expression = trueValue
+  override def third: Expression = falseValue
   override def nullable: Boolean = trueValue.nullable || falseValue.nullable
 
+  final override val nodePatterns : Seq[TreePattern] = Seq(IF)
+
   override def checkInputDataTypes(): TypeCheckResult = {
     if (predicate.dataType != BooleanType) {
       TypeCheckResult.TypeCheckFailure(
@@ -90,6 +98,13 @@ case class If(predicate: Expression, trueValue: Expression, falseValue: Expressi
   override def toString: String = s"if ($predicate) $trueValue else $falseValue"
 
   override def sql: String = s"(IF(${predicate.sql}, ${trueValue.sql}, ${falseValue.sql}))"
+
+  override protected def withNewChildrenInternal(
+      newFirst: Expression, newSecond: Expression, newThird: Expression): Expression = copy(
+    predicate = newFirst,
+    trueValue = newSecond,
+    falseValue = newThird
+  )
 }
 
 /**
@@ -116,7 +131,9 @@ case class If(predicate: Expression, trueValue: Expression, falseValue: Expressi
        2.0
       > SELECT CASE WHEN 1 < 0 THEN 1 WHEN 2 < 0 THEN 2.0 END;
        NULL
-  """)
+  """,
+  since = "1.0.1",
+  group = "conditional_funcs")
 // scalastyle:on line.size.limit
 case class CaseWhen(
     branches: Seq[(Expression, Expression)],
@@ -125,6 +142,11 @@ case class CaseWhen(
 
   override def children: Seq[Expression] = branches.flatMap(b => b._1 :: b._2 :: Nil) ++ elseValue
 
+  final override val nodePatterns : Seq[TreePattern] = Seq(CASE_WHEN)
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
+    super.legacyWithNewChildren(newChildren)
+
   // both then and else expressions should be considered.
   @transient
   override lazy val inputTypesForMerging: Seq[DataType] = {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/constraintExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/constraintExpressions.scala
index 5bfae7b77e096..8feaf52ecb134 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/constraintExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/constraintExpressions.scala
@@ -36,6 +36,12 @@ case class KnownNotNull(child: Expression) extends TaggingExpression {
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     child.genCode(ctx).copy(isNull = FalseLiteral)
   }
+
+  override protected def withNewChildInternal(newChild: Expression): KnownNotNull =
+    copy(child = newChild)
 }
 
-case class KnownFloatingPointNormalized(child: Expression) extends TaggingExpression
+case class KnownFloatingPointNormalized(child: Expression) extends TaggingExpression {
+  override protected def withNewChildInternal(newChild: Expression): KnownFloatingPointNormalized =
+    copy(child = newChild)
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/csvExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/csvExpressions.scala
index 5140db90c5954..79bbc103c92d3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/csvExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/csvExpressions.scala
@@ -21,12 +21,12 @@ import java.io.CharArrayWriter
 
 import com.univocity.parsers.csv.CsvParser
 
-import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.csv._
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
@@ -44,13 +44,15 @@ import org.apache.spark.unsafe.types.UTF8String
       > SELECT _FUNC_('26/08/2015', 'time Timestamp', map('timestampFormat', 'dd/MM/yyyy'));
        {"time":2015-08-26 00:00:00}
   """,
-  since = "3.0.0")
+  since = "3.0.0",
+  group = "csv_funcs")
 // scalastyle:on line.size.limit
 case class CsvToStructs(
     schema: StructType,
     options: Map[String, String],
     child: Expression,
-    timeZoneId: Option[String] = None)
+    timeZoneId: Option[String] = None,
+    requiredSchema: Option[StructType] = None)
   extends UnaryExpression
     with TimeZoneAwareExpression
     with CodegenFallback
@@ -89,7 +91,7 @@ case class CsvToStructs(
       assert(!rows.hasNext)
       result
     } else {
-      throw new IllegalArgumentException("Expected one row from CSV parser.")
+      throw QueryExecutionErrors.rowFromCSVParserNotExpectedError
     }
   }
 
@@ -103,8 +105,7 @@ case class CsvToStructs(
       defaultColumnNameOfCorruptRecord = nameOfCorruptRecord)
     val mode = parsedOptions.parseMode
     if (mode != PermissiveMode && mode != FailFastMode) {
-      throw new AnalysisException(s"from_csv() doesn't support the ${mode.name} mode. " +
-        s"Acceptable modes are ${PermissiveMode.name} and ${FailFastMode.name}.")
+      throw QueryCompilationErrors.parseModeUnsupportedError("from_csv", mode)
     }
     ExprUtils.verifyColumnNameOfCorruptRecord(
       nullableSchema,
@@ -112,7 +113,12 @@ case class CsvToStructs(
 
     val actualSchema =
       StructType(nullableSchema.filterNot(_.name == parsedOptions.columnNameOfCorruptRecord))
-    val rawParser = new UnivocityParser(actualSchema, actualSchema, parsedOptions)
+    val actualRequiredSchema =
+      StructType(requiredSchema.map(_.asNullable).getOrElse(nullableSchema)
+        .filterNot(_.name == parsedOptions.columnNameOfCorruptRecord))
+    val rawParser = new UnivocityParser(actualSchema,
+      actualRequiredSchema,
+      parsedOptions)
     new FailureSafeParser[String](
       input => rawParser.parse(input),
       mode,
@@ -120,7 +126,7 @@ case class CsvToStructs(
       parsedOptions.columnNameOfCorruptRecord)
   }
 
-  override def dataType: DataType = nullableSchema
+  override def dataType: DataType = requiredSchema.getOrElse(schema).asNullable
 
   override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression = {
     copy(timeZoneId = Option(timeZoneId))
@@ -134,6 +140,9 @@ case class CsvToStructs(
   override def inputTypes: Seq[AbstractDataType] = StringType :: Nil
 
   override def prettyName: String = "from_csv"
+
+  override protected def withNewChildInternal(newChild: Expression): CsvToStructs =
+    copy(child = newChild)
 }
 
 /**
@@ -144,9 +153,10 @@ case class CsvToStructs(
   examples = """
     Examples:
       > SELECT _FUNC_('1,abc');
-       struct<_c0:int,_c1:string>
+       STRUCT<`_c0`: INT, `_c1`: STRING>
   """,
-  since = "3.0.0")
+  since = "3.0.0",
+  group = "csv_funcs")
 case class SchemaOfCsv(
     child: Expression,
     options: Map[String, String])
@@ -186,10 +196,13 @@ case class SchemaOfCsv(
     val inferSchema = new CSVInferSchema(parsedOptions)
     val fieldTypes = inferSchema.inferRowType(startType, row)
     val st = StructType(inferSchema.toStructFields(fieldTypes, header))
-    UTF8String.fromString(st.catalogString)
+    UTF8String.fromString(st.sql)
   }
 
   override def prettyName: String = "schema_of_csv"
+
+  override protected def withNewChildInternal(newChild: Expression): SchemaOfCsv =
+    copy(child = newChild)
 }
 
 /**
@@ -205,13 +218,15 @@ case class SchemaOfCsv(
       > SELECT _FUNC_(named_struct('time', to_timestamp('2015-08-26', 'yyyy-MM-dd')), map('timestampFormat', 'dd/MM/yyyy'));
        26/08/2015
   """,
-  since = "3.0.0")
+  since = "3.0.0",
+  group = "csv_funcs")
 // scalastyle:on line.size.limit
 case class StructsToCsv(
      options: Map[String, String],
      child: Expression,
      timeZoneId: Option[String] = None)
-  extends UnaryExpression with TimeZoneAwareExpression with CodegenFallback with ExpectsInputTypes {
+  extends UnaryExpression with TimeZoneAwareExpression with CodegenFallback with ExpectsInputTypes
+    with NullIntolerant {
   override def nullable: Boolean = true
 
   def this(options: Map[String, String], child: Expression) = this(options, child, None)
@@ -232,7 +247,7 @@ case class StructsToCsv(
   lazy val inputSchema: StructType = child.dataType match {
     case st: StructType => st
     case other =>
-      throw new IllegalArgumentException(s"Unsupported input type ${other.catalogString}")
+      throw QueryExecutionErrors.inputTypeUnsupportedError(other)
   }
 
   @transient
@@ -255,4 +270,7 @@ case class StructsToCsv(
   override def inputTypes: Seq[AbstractDataType] = StructType :: Nil
 
   override def prettyName: String = "to_csv"
+
+  override protected def withNewChildInternal(newChild: Expression): StructsToCsv =
+    copy(child = newChild)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index ccedcb41fc283..746117e5ff292 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -17,24 +17,22 @@
 
 package org.apache.spark.sql.catalyst.expressions
 
+import java.text.ParseException
 import java.time.{DateTimeException, LocalDate, LocalDateTime, ZoneId}
-import java.time.temporal.IsoFields
+import java.time.format.DateTimeParseException
 import java.util.Locale
 
-import scala.util.control.NonFatal
-
 import org.apache.commons.text.StringEscapeUtils
 
-import org.apache.spark.SparkUpgradeException
-import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
+import org.apache.spark.sql.catalyst.trees.TreePattern.{CURRENT_LIKE, TreePattern}
 import org.apache.spark.sql.catalyst.util.{DateTimeUtils, LegacyDateFormats, TimestampFormatter}
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils._
 import org.apache.spark.sql.catalyst.util.LegacyDateFormats.SIMPLE_DATE_FORMAT
-import org.apache.spark.sql.catalyst.util.toPrettySQL
+import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
@@ -56,15 +54,50 @@ trait TimeZoneAwareExpression extends Expression {
   @transient lazy val zoneId: ZoneId = DateTimeUtils.getZoneId(timeZoneId.get)
 }
 
+trait TimestampFormatterHelper extends TimeZoneAwareExpression {
+
+  protected def formatString: Expression
+
+  protected def isParsing: Boolean
+
+  @transient final protected lazy val formatterOption: Option[TimestampFormatter] =
+    if (formatString.foldable) {
+      Option(formatString.eval()).map(fmt => getFormatter(fmt.toString))
+    } else None
+
+  final protected def getFormatter(fmt: String): TimestampFormatter = {
+    TimestampFormatter(
+      format = fmt,
+      zoneId = zoneId,
+      legacyFormat = SIMPLE_DATE_FORMAT,
+      isParsing = isParsing)
+  }
+}
+
+@ExpressionDescription(
+  usage = "_FUNC_() - Returns the current session local timezone.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_();
+       Asia/Shanghai
+  """,
+  group = "datetime_funcs",
+  since = "3.1.0")
+case class CurrentTimeZone() extends LeafExpression with Unevaluable {
+  override def nullable: Boolean = false
+  override def dataType: DataType = StringType
+  override def prettyName: String = "current_timezone"
+  final override val nodePatterns: Seq[TreePattern] = Seq(CURRENT_LIKE)
+}
+
 /**
  * Returns the current date at the start of query evaluation.
- * All calls of current_date within the same query return the same value.
- *
  * There is no code generation since this expression should get constant folded by the optimizer.
  */
+// scalastyle:off line.size.limit
 @ExpressionDescription(
   usage = """
-    _FUNC_() - Returns the current date at the start of query evaluation.
+    _FUNC_() - Returns the current date at the start of query evaluation. All calls of current_date within the same query return the same value.
 
     _FUNC_ - Returns the current date at the start of query evaluation.
   """,
@@ -80,6 +113,7 @@ trait TimeZoneAwareExpression extends Expression {
   """,
   group = "datetime_funcs",
   since = "1.5.0")
+// scalastyle:on line.size.limit
 case class CurrentDate(timeZoneId: Option[String] = None)
   extends LeafExpression with TimeZoneAwareExpression with CodegenFallback {
 
@@ -90,6 +124,8 @@ case class CurrentDate(timeZoneId: Option[String] = None)
 
   override def dataType: DataType = DateType
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(CURRENT_LIKE)
+
   override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
     copy(timeZoneId = Option(timeZoneId))
 
@@ -103,17 +139,17 @@ abstract class CurrentTimestampLike() extends LeafExpression with CodegenFallbac
   override def nullable: Boolean = false
   override def dataType: DataType = TimestampType
   override def eval(input: InternalRow): Any = currentTimestamp()
+  final override val nodePatterns: Seq[TreePattern] = Seq(CURRENT_LIKE)
 }
 
 /**
  * Returns the current timestamp at the start of query evaluation.
- * All calls of current_timestamp within the same query return the same value.
- *
  * There is no code generation since this expression should get constant folded by the optimizer.
  */
+// scalastyle:off line.size.limit
 @ExpressionDescription(
   usage = """
-    _FUNC_() - Returns the current timestamp at the start of query evaluation.
+    _FUNC_() - Returns the current timestamp at the start of query evaluation. All calls of current_timestamp within the same query return the same value.
 
     _FUNC_ - Returns the current timestamp at the start of query evaluation.
   """,
@@ -129,6 +165,7 @@ abstract class CurrentTimestampLike() extends LeafExpression with CodegenFallbac
   """,
   group = "datetime_funcs",
   since = "1.5.0")
+// scalastyle:on line.size.limit
 case class CurrentTimestamp() extends CurrentTimestampLike {
   override def prettyName: String = "current_timestamp"
 }
@@ -198,7 +235,7 @@ case class CurrentBatchTimestamp(
   group = "datetime_funcs",
   since = "1.5.0")
 case class DateAdd(startDate: Expression, days: Expression)
-  extends BinaryExpression with ExpectsInputTypes {
+  extends BinaryExpression with ExpectsInputTypes with NullIntolerant {
 
   override def left: Expression = startDate
   override def right: Expression = days
@@ -219,6 +256,9 @@ case class DateAdd(startDate: Expression, days: Expression)
   }
 
   override def prettyName: String = "date_add"
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): DateAdd = copy(startDate = newLeft, days = newRight)
 }
 
 /**
@@ -234,7 +274,7 @@ case class DateAdd(startDate: Expression, days: Expression)
   group = "datetime_funcs",
   since = "1.5.0")
 case class DateSub(startDate: Expression, days: Expression)
-  extends BinaryExpression with ExpectsInputTypes {
+  extends BinaryExpression with ExpectsInputTypes with NullIntolerant {
   override def left: Expression = startDate
   override def right: Expression = days
 
@@ -254,40 +294,49 @@ case class DateSub(startDate: Expression, days: Expression)
   }
 
   override def prettyName: String = "date_sub"
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): DateSub = copy(startDate = newLeft, days = newRight)
 }
 
-@ExpressionDescription(
-  usage = "_FUNC_(timestamp) - Returns the hour component of the string/timestamp.",
-  examples = """
-    Examples:
-      > SELECT _FUNC_('2009-07-30 12:58:59');
-       12
-  """,
-  group = "datetime_funcs",
-  since = "1.5.0")
-case class Hour(child: Expression, timeZoneId: Option[String] = None)
-  extends UnaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes {
+trait GetTimeField extends UnaryExpression
+  with TimeZoneAwareExpression with ImplicitCastInputTypes with NullIntolerant {
 
-  def this(child: Expression) = this(child, None)
+  val func: (Long, ZoneId) => Any
+  val funcName: String
 
   override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType)
 
   override def dataType: DataType = IntegerType
 
-  override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
-    copy(timeZoneId = Option(timeZoneId))
-
   override protected def nullSafeEval(timestamp: Any): Any = {
-    DateTimeUtils.getHours(timestamp.asInstanceOf[Long], zoneId)
+    func(timestamp.asInstanceOf[Long], zoneId)
   }
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
     val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getHours($c, $zid)")
+    defineCodeGen(ctx, ev, c => s"$dtu.$funcName($c, $zid)")
   }
 }
 
+@ExpressionDescription(
+  usage = "_FUNC_(timestamp) - Returns the hour component of the string/timestamp.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_('2009-07-30 12:58:59');
+       12
+  """,
+  group = "datetime_funcs",
+  since = "1.5.0")
+case class Hour(child: Expression, timeZoneId: Option[String] = None) extends GetTimeField {
+  def this(child: Expression) = this(child, None)
+  override def withTimeZone(timeZoneId: String): Hour = copy(timeZoneId = Option(timeZoneId))
+  override val func = DateTimeUtils.getHours
+  override val funcName = "getHours"
+  override protected def withNewChildInternal(newChild: Expression): Hour = copy(child = newChild)
+}
+
 @ExpressionDescription(
   usage = "_FUNC_(timestamp) - Returns the minute component of the string/timestamp.",
   examples = """
@@ -297,27 +346,12 @@ case class Hour(child: Expression, timeZoneId: Option[String] = None)
   """,
   group = "datetime_funcs",
   since = "1.5.0")
-case class Minute(child: Expression, timeZoneId: Option[String] = None)
-  extends UnaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes {
-
+case class Minute(child: Expression, timeZoneId: Option[String] = None) extends GetTimeField {
   def this(child: Expression) = this(child, None)
-
-  override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType)
-
-  override def dataType: DataType = IntegerType
-
-  override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
-    copy(timeZoneId = Option(timeZoneId))
-
-  override protected def nullSafeEval(timestamp: Any): Any = {
-    DateTimeUtils.getMinutes(timestamp.asInstanceOf[Long], zoneId)
-  }
-
-  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
-    val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getMinutes($c, $zid)")
-  }
+  override def withTimeZone(timeZoneId: String): Minute = copy(timeZoneId = Option(timeZoneId))
+  override val func = DateTimeUtils.getMinutes
+  override val funcName = "getMinutes"
+  override protected def withNewChildInternal(newChild: Expression): Minute = copy(child = newChild)
 }
 
 @ExpressionDescription(
@@ -329,50 +363,43 @@ case class Minute(child: Expression, timeZoneId: Option[String] = None)
   """,
   group = "datetime_funcs",
   since = "1.5.0")
-case class Second(child: Expression, timeZoneId: Option[String] = None)
-  extends UnaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes {
-
+case class Second(child: Expression, timeZoneId: Option[String] = None) extends GetTimeField {
   def this(child: Expression) = this(child, None)
-
-  override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType)
-
-  override def dataType: DataType = IntegerType
-
-  override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
-    copy(timeZoneId = Option(timeZoneId))
-
-  override protected def nullSafeEval(timestamp: Any): Any = {
-    DateTimeUtils.getSeconds(timestamp.asInstanceOf[Long], zoneId)
-  }
-
-  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
-    val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getSeconds($c, $zid)")
-  }
+  override def withTimeZone(timeZoneId: String): Second = copy(timeZoneId = Option(timeZoneId))
+  override val func = DateTimeUtils.getSeconds
+  override val funcName = "getSeconds"
+  override protected def withNewChildInternal(newChild: Expression): Second =
+    copy(child = newChild)
 }
 
 case class SecondWithFraction(child: Expression, timeZoneId: Option[String] = None)
-  extends UnaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes {
-
+  extends GetTimeField {
   def this(child: Expression) = this(child, None)
-
-  override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType)
-
   // 2 digits for seconds, and 6 digits for the fractional part with microsecond precision.
   override def dataType: DataType = DecimalType(8, 6)
-
-  override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
+  override def withTimeZone(timeZoneId: String): SecondWithFraction =
     copy(timeZoneId = Option(timeZoneId))
+  override val func = DateTimeUtils.getSecondsWithFraction
+  override val funcName = "getSecondsWithFraction"
+  override protected def withNewChildInternal(newChild: Expression): SecondWithFraction =
+    copy(child = newChild)
+}
 
-  override protected def nullSafeEval(timestamp: Any): Any = {
-    DateTimeUtils.getSecondsWithFraction(timestamp.asInstanceOf[Long], zoneId)
+trait GetDateField extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
+  val func: Int => Any
+  val funcName: String
+
+  override def inputTypes: Seq[AbstractDataType] = Seq(DateType)
+
+  override def dataType: DataType = IntegerType
+
+  override protected def nullSafeEval(date: Any): Any = {
+    func(date.asInstanceOf[Int])
   }
 
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
     val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getSecondsWithFraction($c, $zid)")
+    defineCodeGen(ctx, ev, c => s"$dtu.$funcName($c)")
   }
 }
 
@@ -385,111 +412,333 @@ case class SecondWithFraction(child: Expression, timeZoneId: Option[String] = No
   """,
   group = "datetime_funcs",
   since = "1.5.0")
-case class DayOfYear(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
+case class DayOfYear(child: Expression) extends GetDateField {
+  override val func = DateTimeUtils.getDayInYear
+  override val funcName = "getDayInYear"
+  override protected def withNewChildInternal(newChild: Expression): DayOfYear =
+    copy(child = newChild)
+}
 
+@ExpressionDescription(
+  usage = "_FUNC_(days) - Create date from the number of days since 1970-01-01.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_(1);
+       1970-01-02
+  """,
+  group = "datetime_funcs",
+  since = "3.1.0")
+case class DateFromUnixDate(child: Expression) extends UnaryExpression
+  with ImplicitCastInputTypes with NullIntolerant {
+  override def inputTypes: Seq[AbstractDataType] = Seq(IntegerType)
+
+  override def dataType: DataType = DateType
+
+  override def nullSafeEval(input: Any): Any = input.asInstanceOf[Int]
+
+  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode =
+    defineCodeGen(ctx, ev, c => c)
+
+  override def prettyName: String = "date_from_unix_date"
+
+  override protected def withNewChildInternal(newChild: Expression): DateFromUnixDate =
+    copy(child = newChild)
+}
+
+@ExpressionDescription(
+  usage = "_FUNC_(date) - Returns the number of days since 1970-01-01.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_(DATE("1970-01-02"));
+       1
+  """,
+  group = "datetime_funcs",
+  since = "3.1.0")
+case class UnixDate(child: Expression) extends UnaryExpression
+  with ExpectsInputTypes with NullIntolerant {
   override def inputTypes: Seq[AbstractDataType] = Seq(DateType)
 
   override def dataType: DataType = IntegerType
 
-  override protected def nullSafeEval(date: Any): Any = {
-    DateTimeUtils.getDayInYear(date.asInstanceOf[Int])
+  override def nullSafeEval(input: Any): Any = input.asInstanceOf[Int]
+
+  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode =
+    defineCodeGen(ctx, ev, c => c)
+
+  override def prettyName: String = "unix_date"
+
+  override protected def withNewChildInternal(newChild: Expression): UnixDate =
+    copy(child = newChild)
+}
+
+abstract class IntegralToTimestampBase extends UnaryExpression
+  with ExpectsInputTypes with NullIntolerant {
+
+  protected def upScaleFactor: Long
+
+  override def inputTypes: Seq[AbstractDataType] = Seq(IntegralType)
+
+  override def dataType: DataType = TimestampType
+
+  override def nullSafeEval(input: Any): Any = {
+    Math.multiplyExact(input.asInstanceOf[Number].longValue(), upScaleFactor)
   }
 
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getDayInYear($c)")
+    if (upScaleFactor == 1) {
+      defineCodeGen(ctx, ev, c => c)
+    } else {
+      defineCodeGen(ctx, ev, c => s"java.lang.Math.multiplyExact($c, ${upScaleFactor}L)")
+    }
   }
 }
 
+// scalastyle:off line.size.limit
 @ExpressionDescription(
-  usage = "_FUNC_(date) - Returns the year component of the date/timestamp.",
+  usage = "_FUNC_(seconds) - Creates timestamp from the number of seconds (can be fractional) since UTC epoch.",
   examples = """
     Examples:
-      > SELECT _FUNC_('2016-07-30');
-       2016
+      > SELECT _FUNC_(1230219000);
+       2008-12-25 07:30:00
+      > SELECT _FUNC_(1230219000.123);
+       2008-12-25 07:30:00.123
   """,
   group = "datetime_funcs",
-  since = "1.5.0")
-case class Year(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
+  since = "3.1.0")
+// scalastyle:on line.size.limit
+case class SecondsToTimestamp(child: Expression) extends UnaryExpression
+  with ExpectsInputTypes with NullIntolerant {
 
-  override def inputTypes: Seq[AbstractDataType] = Seq(DateType)
+  override def inputTypes: Seq[AbstractDataType] = Seq(NumericType)
 
-  override def dataType: DataType = IntegerType
+  override def dataType: DataType = TimestampType
 
-  override protected def nullSafeEval(date: Any): Any = {
-    DateTimeUtils.getYear(date.asInstanceOf[Int])
+  override def nullable: Boolean = child.dataType match {
+    case _: FloatType | _: DoubleType => true
+    case _ => child.nullable
+  }
+
+  @transient
+  private lazy val evalFunc: Any => Any = child.dataType match {
+    case _: IntegralType => input =>
+      Math.multiplyExact(input.asInstanceOf[Number].longValue(), MICROS_PER_SECOND)
+    case _: DecimalType => input =>
+      val operand = new java.math.BigDecimal(MICROS_PER_SECOND)
+      input.asInstanceOf[Decimal].toJavaBigDecimal.multiply(operand).longValueExact()
+    case _: FloatType => input =>
+      val f = input.asInstanceOf[Float]
+      if (f.isNaN || f.isInfinite) null else (f.toDouble * MICROS_PER_SECOND).toLong
+    case _: DoubleType => input =>
+      val d = input.asInstanceOf[Double]
+      if (d.isNaN || d.isInfinite) null else (d * MICROS_PER_SECOND).toLong
+  }
+
+  override def nullSafeEval(input: Any): Any = evalFunc(input)
+
+  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = child.dataType match {
+    case _: IntegralType =>
+      defineCodeGen(ctx, ev, c => s"java.lang.Math.multiplyExact($c, ${MICROS_PER_SECOND}L)")
+    case _: DecimalType =>
+      val operand = s"new java.math.BigDecimal($MICROS_PER_SECOND)"
+      defineCodeGen(ctx, ev, c => s"$c.toJavaBigDecimal().multiply($operand).longValueExact()")
+    case other =>
+      val castToDouble = if (other.isInstanceOf[FloatType]) "(double)" else ""
+      nullSafeCodeGen(ctx, ev, c => {
+        val typeStr = CodeGenerator.boxedType(other)
+        s"""
+           |if ($typeStr.isNaN($c) || $typeStr.isInfinite($c)) {
+           |  ${ev.isNull} = true;
+           |} else {
+           |  ${ev.value} = (long)($castToDouble$c * $MICROS_PER_SECOND);
+           |}
+           |""".stripMargin
+      })
   }
 
-  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getYear($c)")
-  }
+  override def prettyName: String = "timestamp_seconds"
+
+  override protected def withNewChildInternal(newChild: Expression): SecondsToTimestamp =
+    copy(child = newChild)
 }
 
-case class YearOfWeek(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
+// scalastyle:off line.size.limit
+@ExpressionDescription(
+  usage = "_FUNC_(milliseconds) - Creates timestamp from the number of milliseconds since UTC epoch.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_(1230219000123);
+       2008-12-25 07:30:00.123
+  """,
+  group = "datetime_funcs",
+  since = "3.1.0")
+// scalastyle:on line.size.limit
+case class MillisToTimestamp(child: Expression)
+  extends IntegralToTimestampBase {
 
-  override def inputTypes: Seq[AbstractDataType] = Seq(DateType)
+  override def upScaleFactor: Long = MICROS_PER_MILLIS
 
-  override def dataType: DataType = IntegerType
+  override def prettyName: String = "timestamp_millis"
 
-  override protected def nullSafeEval(date: Any): Any = {
-    DateTimeUtils.getWeekBasedYear(date.asInstanceOf[Int])
+  override protected def withNewChildInternal(newChild: Expression): MillisToTimestamp =
+    copy(child = newChild)
+}
+
+// scalastyle:off line.size.limit
+@ExpressionDescription(
+  usage = "_FUNC_(microseconds) - Creates timestamp from the number of microseconds since UTC epoch.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_(1230219000123123);
+       2008-12-25 07:30:00.123123
+  """,
+  group = "datetime_funcs",
+  since = "3.1.0")
+// scalastyle:on line.size.limit
+case class MicrosToTimestamp(child: Expression)
+  extends IntegralToTimestampBase {
+
+  override def upScaleFactor: Long = 1L
+
+  override def prettyName: String = "timestamp_micros"
+
+  override protected def withNewChildInternal(newChild: Expression): MicrosToTimestamp =
+    copy(child = newChild)
+}
+
+abstract class TimestampToLongBase extends UnaryExpression
+  with ExpectsInputTypes with NullIntolerant {
+
+  protected def scaleFactor: Long
+
+  override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType)
+
+  override def dataType: DataType = LongType
+
+  override def nullSafeEval(input: Any): Any = {
+    Math.floorDiv(input.asInstanceOf[Number].longValue(), scaleFactor)
   }
 
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getWeekBasedYear($c)")
+    if (scaleFactor == 1) {
+      defineCodeGen(ctx, ev, c => c)
+    } else {
+      defineCodeGen(ctx, ev, c => s"java.lang.Math.floorDiv($c, ${scaleFactor}L)")
+    }
   }
 }
 
+// scalastyle:off line.size.limit
 @ExpressionDescription(
-  usage = "_FUNC_(date) - Returns the quarter of the year for date, in the range 1 to 4.",
+  usage = "_FUNC_(timestamp) - Returns the number of seconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of precision.",
   examples = """
     Examples:
-      > SELECT _FUNC_('2016-08-31');
-       3
+      > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01Z'));
+       1
   """,
   group = "datetime_funcs",
-  since = "1.5.0")
-case class Quarter(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
+  since = "3.1.0")
+// scalastyle:on line.size.limit
+case class UnixSeconds(child: Expression) extends TimestampToLongBase {
+  override def scaleFactor: Long = MICROS_PER_SECOND
 
-  override def inputTypes: Seq[AbstractDataType] = Seq(DateType)
+  override def prettyName: String = "unix_seconds"
 
-  override def dataType: DataType = IntegerType
+  override protected def withNewChildInternal(newChild: Expression): UnixSeconds =
+    copy(child = newChild)
+}
 
-  override protected def nullSafeEval(date: Any): Any = {
-    DateTimeUtils.getQuarter(date.asInstanceOf[Int])
-  }
+// scalastyle:off line.size.limit
+@ExpressionDescription(
+  usage = "_FUNC_(timestamp) - Returns the number of milliseconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of precision.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01Z'));
+       1000
+  """,
+  group = "datetime_funcs",
+  since = "3.1.0")
+// scalastyle:on line.size.limit
+case class UnixMillis(child: Expression) extends TimestampToLongBase {
+  override def scaleFactor: Long = MICROS_PER_MILLIS
 
-  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getQuarter($c)")
-  }
+  override def prettyName: String = "unix_millis"
+
+  override protected def withNewChildInternal(newChild: Expression): UnixMillis =
+    copy(child = newChild)
 }
 
+// scalastyle:off line.size.limit
 @ExpressionDescription(
-  usage = "_FUNC_(date) - Returns the month component of the date/timestamp.",
+  usage = "_FUNC_(timestamp) - Returns the number of microseconds since 1970-01-01 00:00:00 UTC.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01Z'));
+       1000000
+  """,
+  group = "datetime_funcs",
+  since = "3.1.0")
+// scalastyle:on line.size.limit
+case class UnixMicros(child: Expression) extends TimestampToLongBase {
+  override def scaleFactor: Long = 1L
+
+  override def prettyName: String = "unix_micros"
+
+  override protected def withNewChildInternal(newChild: Expression): UnixMicros =
+    copy(child = newChild)
+}
+
+@ExpressionDescription(
+  usage = "_FUNC_(date) - Returns the year component of the date/timestamp.",
   examples = """
     Examples:
       > SELECT _FUNC_('2016-07-30');
-       7
+       2016
   """,
   group = "datetime_funcs",
   since = "1.5.0")
-case class Month(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
-
-  override def inputTypes: Seq[AbstractDataType] = Seq(DateType)
+case class Year(child: Expression) extends GetDateField {
+  override val func = DateTimeUtils.getYear
+  override val funcName = "getYear"
+  override protected def withNewChildInternal(newChild: Expression): Year =
+    copy(child = newChild)
+}
 
-  override def dataType: DataType = IntegerType
+case class YearOfWeek(child: Expression) extends GetDateField {
+  override val func = DateTimeUtils.getWeekBasedYear
+  override val funcName = "getWeekBasedYear"
+  override protected def withNewChildInternal(newChild: Expression): YearOfWeek =
+    copy(child = newChild)
+}
 
-  override protected def nullSafeEval(date: Any): Any = {
-    DateTimeUtils.getMonth(date.asInstanceOf[Int])
-  }
+@ExpressionDescription(
+  usage = "_FUNC_(date) - Returns the quarter of the year for date, in the range 1 to 4.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_('2016-08-31');
+       3
+  """,
+  group = "datetime_funcs",
+  since = "1.5.0")
+case class Quarter(child: Expression) extends GetDateField {
+  override val func = DateTimeUtils.getQuarter
+  override val funcName = "getQuarter"
+  override protected def withNewChildInternal(newChild: Expression): Quarter =
+    copy(child = newChild)
+}
 
-  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getMonth($c)")
-  }
+@ExpressionDescription(
+  usage = "_FUNC_(date) - Returns the month component of the date/timestamp.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_('2016-07-30');
+       7
+  """,
+  group = "datetime_funcs",
+  since = "1.5.0")
+case class Month(child: Expression) extends GetDateField {
+  override val func = DateTimeUtils.getMonth
+  override val funcName = "getMonth"
+  override protected def withNewChildInternal(newChild: Expression): Month = copy(child = newChild)
 }
 
 @ExpressionDescription(
@@ -499,21 +748,13 @@ case class Month(child: Expression) extends UnaryExpression with ImplicitCastInp
       > SELECT _FUNC_('2009-07-30');
        30
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
-case class DayOfMonth(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
-
-  override def inputTypes: Seq[AbstractDataType] = Seq(DateType)
-
-  override def dataType: DataType = IntegerType
-
-  override protected def nullSafeEval(date: Any): Any = {
-    DateTimeUtils.getDayOfMonth(date.asInstanceOf[Int])
-  }
-
-  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getDayOfMonth($c)")
-  }
+case class DayOfMonth(child: Expression) extends GetDateField {
+  override val func = DateTimeUtils.getDayOfMonth
+  override val funcName = "getDayOfMonth"
+  override protected def withNewChildInternal(newChild: Expression): DayOfMonth =
+    copy(child = newChild)
 }
 
 // scalastyle:off line.size.limit
@@ -527,20 +768,11 @@ case class DayOfMonth(child: Expression) extends UnaryExpression with ImplicitCa
   group = "datetime_funcs",
   since = "2.3.0")
 // scalastyle:on line.size.limit
-case class DayOfWeek(child: Expression) extends DayWeek {
-
-  override protected def nullSafeEval(date: Any): Any = {
-    val localDate = LocalDate.ofEpochDay(date.asInstanceOf[Int])
-    localDate.getDayOfWeek.plus(1).getValue
-  }
-
-  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    nullSafeCodeGen(ctx, ev, days => {
-      s"""
-        ${ev.value} = java.time.LocalDate.ofEpochDay($days).getDayOfWeek().plus(1).getValue();
-      """
-    })
-  }
+case class DayOfWeek(child: Expression) extends GetDateField {
+  override val func = DateTimeUtils.getDayOfWeek
+  override val funcName = "getDayOfWeek"
+  override protected def withNewChildInternal(newChild: Expression): DayOfWeek =
+    copy(child = newChild)
 }
 
 // scalastyle:off line.size.limit
@@ -554,27 +786,11 @@ case class DayOfWeek(child: Expression) extends DayWeek {
   group = "datetime_funcs",
   since = "2.4.0")
 // scalastyle:on line.size.limit
-case class WeekDay(child: Expression) extends DayWeek {
-
-  override protected def nullSafeEval(date: Any): Any = {
-    val localDate = LocalDate.ofEpochDay(date.asInstanceOf[Int])
-    localDate.getDayOfWeek.ordinal()
-  }
-
-  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    nullSafeCodeGen(ctx, ev, days => {
-      s"""
-         ${ev.value} = java.time.LocalDate.ofEpochDay($days).getDayOfWeek().ordinal();
-      """
-    })
-  }
-}
-
-abstract class DayWeek extends UnaryExpression with ImplicitCastInputTypes {
-
-  override def inputTypes: Seq[AbstractDataType] = Seq(DateType)
-
-  override def dataType: DataType = IntegerType
+case class WeekDay(child: Expression) extends GetDateField {
+  override val func = DateTimeUtils.getWeekDay
+  override val funcName = "getWeekDay"
+  override protected def withNewChildInternal(newChild: Expression): WeekDay =
+    copy(child = newChild)
 }
 
 // scalastyle:off line.size.limit
@@ -588,25 +804,11 @@ abstract class DayWeek extends UnaryExpression with ImplicitCastInputTypes {
   group = "datetime_funcs",
   since = "1.5.0")
 // scalastyle:on line.size.limit
-case class WeekOfYear(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
-
-  override def inputTypes: Seq[AbstractDataType] = Seq(DateType)
-
-  override def dataType: DataType = IntegerType
-
-  override protected def nullSafeEval(date: Any): Any = {
-    val localDate = LocalDate.ofEpochDay(date.asInstanceOf[Int])
-    localDate.get(IsoFields.WEEK_OF_WEEK_BASED_YEAR)
-  }
-
-  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    nullSafeCodeGen(ctx, ev, days => {
-      s"""
-         |${ev.value} = java.time.LocalDate.ofEpochDay($days).get(
-         |  java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR);
-       """.stripMargin
-    })
-  }
+case class WeekOfYear(child: Expression) extends GetDateField {
+  override val func = DateTimeUtils.getWeekOfYear
+  override val funcName = "getWeekOfYear"
+  override protected def withNewChildInternal(newChild: Expression): WeekOfYear =
+    copy(child = newChild)
 }
 
 // scalastyle:off line.size.limit
@@ -627,7 +829,8 @@ case class WeekOfYear(child: Expression) extends UnaryExpression with ImplicitCa
   since = "1.5.0")
 // scalastyle:on line.size.limit
 case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Option[String] = None)
-  extends BinaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes {
+  extends BinaryExpression with TimestampFormatterHelper with ImplicitCastInputTypes
+    with NullIntolerant {
 
   def this(left: Expression, right: Expression) = this(left, right, None)
 
@@ -638,33 +841,13 @@ case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Opti
   override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
     copy(timeZoneId = Option(timeZoneId))
 
-  @transient private lazy val formatter: Option[TimestampFormatter] = {
-    if (right.foldable) {
-      Option(right.eval()).map { format =>
-        TimestampFormatter(
-          format.toString,
-          zoneId,
-          legacyFormat = SIMPLE_DATE_FORMAT,
-          needVarLengthSecondFraction = false)
-      }
-    } else None
-  }
-
   override protected def nullSafeEval(timestamp: Any, format: Any): Any = {
-    val tf = if (formatter.isEmpty) {
-      TimestampFormatter(
-        format.toString,
-        zoneId,
-        legacyFormat = SIMPLE_DATE_FORMAT,
-        needVarLengthSecondFraction = false)
-    } else {
-      formatter.get
-    }
-    UTF8String.fromString(tf.format(timestamp.asInstanceOf[Long]))
+    val formatter = formatterOption.getOrElse(getFormatter(format.toString))
+    UTF8String.fromString(formatter.format(timestamp.asInstanceOf[Long]))
   }
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    formatter.map { tf =>
+    formatterOption.map { tf =>
       val timestampFormatter = ctx.addReferenceObj("timestampFormatter", tf)
       defineCodeGen(ctx, ev, (timestamp, _) => {
         s"""UTF8String.fromString($timestampFormatter.format($timestamp))"""
@@ -685,6 +868,14 @@ case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Opti
   }
 
   override def prettyName: String = "date_format"
+
+  override protected def formatString: Expression = right
+
+  override protected def isParsing: Boolean = false
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): DateFormatClass =
+    copy(left = newLeft, right = newRight)
 }
 
 /**
@@ -693,13 +884,13 @@ case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Opti
  */
 // scalastyle:off line.size.limit
 @ExpressionDescription(
-  usage = "_FUNC_(timeExp[, format]) - Returns the UNIX timestamp of the given time.",
+  usage = "_FUNC_(timeExp[, fmt]) - Returns the UNIX timestamp of the given time.",
   arguments = """
     Arguments:
       * timeExp - A date/timestamp or string which is returned as a UNIX timestamp.
-      * format - Date/time format pattern to follow. Ignored if `timeExp` is not a string.
-                 Default value is "yyyy-MM-dd HH:mm:ss". See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">Datetime Patterns</a>
-                 for valid date and time format patterns.
+      * fmt - Date/time format pattern to follow. Ignored if `timeExp` is not a string.
+              Default value is "yyyy-MM-dd HH:mm:ss". See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">Datetime Patterns</a>
+              for valid date and time format patterns.
   """,
   examples = """
     Examples:
@@ -712,10 +903,12 @@ case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Opti
 case class ToUnixTimestamp(
     timeExp: Expression,
     format: Expression,
-    timeZoneId: Option[String] = None)
+    timeZoneId: Option[String] = None,
+    failOnError: Boolean = SQLConf.get.ansiEnabled)
   extends UnixTime {
 
-  def this(timeExp: Expression, format: Expression) = this(timeExp, format, None)
+  def this(timeExp: Expression, format: Expression) =
+    this(timeExp, format, None, SQLConf.get.ansiEnabled)
 
   override def left: Expression = timeExp
   override def right: Expression = format
@@ -728,6 +921,10 @@ case class ToUnixTimestamp(
   }
 
   override def prettyName: String = "to_unix_timestamp"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): ToUnixTimestamp =
+    copy(timeExp = newLeft, format = newRight)
 }
 
 // scalastyle:off line.size.limit
@@ -741,13 +938,13 @@ case class ToUnixTimestamp(
  * second parameter.
  */
 @ExpressionDescription(
-  usage = "_FUNC_([timeExp[, format]]) - Returns the UNIX timestamp of current or specified time.",
+  usage = "_FUNC_([timeExp[, fmt]]) - Returns the UNIX timestamp of current or specified time.",
   arguments = """
     Arguments:
       * timeExp - A date/timestamp or string. If not provided, this defaults to current time.
-      * format - Date/time format pattern to follow. Ignored if `timeExp` is not a string.
-                 Default value is "yyyy-MM-dd HH:mm:ss". See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html"> Datetime Patterns</a>
-                 for valid date and time format patterns.
+      * fmt - Date/time format pattern to follow. Ignored if `timeExp` is not a string.
+              Default value is "yyyy-MM-dd HH:mm:ss". See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html"> Datetime Patterns</a>
+              for valid date and time format patterns.
   """,
   examples = """
     Examples:
@@ -759,10 +956,15 @@ case class ToUnixTimestamp(
   group = "datetime_funcs",
   since = "1.5.0")
 // scalastyle:on line.size.limit
-case class UnixTimestamp(timeExp: Expression, format: Expression, timeZoneId: Option[String] = None)
+case class UnixTimestamp(
+    timeExp: Expression,
+    format: Expression,
+    timeZoneId: Option[String] = None,
+    failOnError: Boolean = SQLConf.get.ansiEnabled)
   extends UnixTime {
 
-  def this(timeExp: Expression, format: Expression) = this(timeExp, format, None)
+  def this(timeExp: Expression, format: Expression) =
+    this(timeExp, format, None, SQLConf.get.ansiEnabled)
 
   override def left: Expression = timeExp
   override def right: Expression = format
@@ -779,32 +981,36 @@ case class UnixTimestamp(timeExp: Expression, format: Expression, timeZoneId: Op
   }
 
   override def prettyName: String = "unix_timestamp"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): UnixTimestamp =
+    copy(timeExp = newLeft, format = newRight)
 }
 
 abstract class ToTimestamp
-  extends BinaryExpression with TimeZoneAwareExpression with ExpectsInputTypes {
+  extends BinaryExpression with TimestampFormatterHelper with ExpectsInputTypes {
+
+  def failOnError: Boolean
 
   // The result of the conversion to timestamp is microseconds divided by this factor.
   // For example if the factor is 1000000, the result of the expression is in seconds.
   protected def downScaleFactor: Long
 
+  override protected def formatString: Expression = right
+  override protected def isParsing = true
+
   override def inputTypes: Seq[AbstractDataType] =
     Seq(TypeCollection(StringType, DateType, TimestampType), StringType)
 
   override def dataType: DataType = LongType
-  override def nullable: Boolean = true
+  override def nullable: Boolean = if (failOnError) children.exists(_.nullable) else true
 
-  private lazy val constFormat: UTF8String = right.eval().asInstanceOf[UTF8String]
-  private lazy val formatter: TimestampFormatter =
-    try {
-      TimestampFormatter(
-        constFormat.toString,
-        zoneId,
-        legacyFormat = SIMPLE_DATE_FORMAT,
-        needVarLengthSecondFraction = true)
-    } catch {
-      case NonFatal(_) => null
-    }
+  private def isParseError(e: Throwable): Boolean = e match {
+    case _: DateTimeParseException |
+         _: DateTimeException |
+         _: ParseException => true
+    case _ => false
+  }
 
   override def eval(input: InternalRow): Any = {
     val t = left.eval(input)
@@ -813,37 +1019,24 @@ abstract class ToTimestamp
     } else {
       left.dataType match {
         case DateType =>
-          epochDaysToMicros(t.asInstanceOf[Int], zoneId) / downScaleFactor
+          daysToMicros(t.asInstanceOf[Int], zoneId) / downScaleFactor
         case TimestampType =>
           t.asInstanceOf[Long] / downScaleFactor
-        case StringType if right.foldable =>
-          if (constFormat == null || formatter == null) {
-            null
-          } else {
-            try {
-              formatter.parse(
-                t.asInstanceOf[UTF8String].toString) / downScaleFactor
-            } catch {
-              case e: SparkUpgradeException => throw e
-              case NonFatal(_) => null
-            }
-          }
         case StringType =>
-          val f = right.eval(input)
-          if (f == null) {
+          val fmt = right.eval(input)
+          if (fmt == null) {
             null
           } else {
-            val formatString = f.asInstanceOf[UTF8String].toString
+            val formatter = formatterOption.getOrElse(getFormatter(fmt.toString))
             try {
-              TimestampFormatter(
-                formatString,
-                zoneId,
-                legacyFormat = SIMPLE_DATE_FORMAT,
-                needVarLengthSecondFraction = true)
-                .parse(t.asInstanceOf[UTF8String].toString) / downScaleFactor
+              formatter.parse(t.asInstanceOf[UTF8String].toString) / downScaleFactor
             } catch {
-              case e: SparkUpgradeException => throw e
-              case NonFatal(_) => null
+              case e if isParseError(e) =>
+                if (failOnError) {
+                  throw e
+                } else {
+                  null
+                }
             }
           }
       }
@@ -852,56 +1045,46 @@ abstract class ToTimestamp
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     val javaType = CodeGenerator.javaType(dataType)
+    val parseErrorBranch = if (failOnError) "throw e;" else s"${ev.isNull} = true;"
     left.dataType match {
-      case StringType if right.foldable =>
+      case StringType => formatterOption.map { fmt =>
         val df = classOf[TimestampFormatter].getName
-        if (formatter == null) {
-          ExprCode.forNullValue(dataType)
-        } else {
-          val formatterName = ctx.addReferenceObj("formatter", formatter, df)
-          val eval1 = left.genCode(ctx)
-          ev.copy(code = code"""
-            ${eval1.code}
-            boolean ${ev.isNull} = ${eval1.isNull};
-            $javaType ${ev.value} = ${CodeGenerator.defaultValue(dataType)};
-            if (!${ev.isNull}) {
-              try {
-                ${ev.value} = $formatterName.parse(${eval1.value}.toString()) / $downScaleFactor;
-              } catch (java.lang.IllegalArgumentException e) {
-                ${ev.isNull} = true;
-              } catch (java.text.ParseException e) {
-                ${ev.isNull} = true;
-              } catch (java.time.format.DateTimeParseException e) {
-                ${ev.isNull} = true;
-              } catch (java.time.DateTimeException e) {
-                ${ev.isNull} = true;
-              }
-            }""")
-        }
-      case StringType =>
+        val formatterName = ctx.addReferenceObj("formatter", fmt, df)
+        nullSafeCodeGen(ctx, ev, (datetimeStr, _) =>
+          s"""
+             |try {
+             |  ${ev.value} = $formatterName.parse($datetimeStr.toString()) / $downScaleFactor;
+             |} catch (java.time.DateTimeException e) {
+             |  $parseErrorBranch
+             |} catch (java.time.format.DateTimeParseException e) {
+             |  $parseErrorBranch
+             |} catch (java.text.ParseException e) {
+             |  $parseErrorBranch
+             |}
+             |""".stripMargin)
+      }.getOrElse {
         val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
         val tf = TimestampFormatter.getClass.getName.stripSuffix("$")
         val ldf = LegacyDateFormats.getClass.getName.stripSuffix("$")
-        nullSafeCodeGen(ctx, ev, (string, format) => {
+        val timestampFormatter = ctx.freshName("timestampFormatter")
+        nullSafeCodeGen(ctx, ev, (string, format) =>
           s"""
-            try {
-              ${ev.value} = $tf$$.MODULE$$.apply(
-                $format.toString(),
-                $zid,
-                $ldf$$.MODULE$$.SIMPLE_DATE_FORMAT(),
-                true)
-              .parse($string.toString()) / $downScaleFactor;
-            } catch (java.lang.IllegalArgumentException e) {
-              ${ev.isNull} = true;
-            } catch (java.text.ParseException e) {
-              ${ev.isNull} = true;
-            } catch (java.time.format.DateTimeParseException e) {
-              ${ev.isNull} = true;
-            } catch (java.time.DateTimeException e) {
-              ${ev.isNull} = true;
-            }
-          """
-        })
+             |$tf $timestampFormatter = $tf$$.MODULE$$.apply(
+             |  $format.toString(),
+             |  $zid,
+             |  $ldf$$.MODULE$$.SIMPLE_DATE_FORMAT(),
+             |  true);
+             |try {
+             |  ${ev.value} = $timestampFormatter.parse($string.toString()) / $downScaleFactor;
+             |} catch (java.time.format.DateTimeParseException e) {
+             |    $parseErrorBranch
+             |} catch (java.time.DateTimeException e) {
+             |    $parseErrorBranch
+             |} catch (java.text.ParseException e) {
+             |    $parseErrorBranch
+             |}
+             |""".stripMargin)
+      }
       case TimestampType =>
         val eval1 = left.genCode(ctx)
         ev.copy(code = code"""
@@ -920,7 +1103,7 @@ abstract class ToTimestamp
           boolean ${ev.isNull} = ${eval1.isNull};
           $javaType ${ev.value} = ${CodeGenerator.defaultValue(dataType)};
           if (!${ev.isNull}) {
-            ${ev.value} = $dtu.epochDaysToMicros(${eval1.value}, $zid) / $downScaleFactor;
+            ${ev.value} = $dtu.daysToMicros(${eval1.value}, $zid) / $downScaleFactor;
           }""")
     }
   }
@@ -934,27 +1117,31 @@ abstract class UnixTime extends ToTimestamp {
  * Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC) to a string
  * representing the timestamp of that moment in the current system time zone in the given
  * format. If the format is missing, using format like "1970-01-01 00:00:00".
- * Note that hive Language Manual says it returns 0 if fail, but in fact it returns null.
+ * Note that Hive Language Manual says it returns 0 if fail, but in fact it returns null.
  */
 // scalastyle:off line.size.limit
 @ExpressionDescription(
-  usage = "_FUNC_(unix_time, format) - Returns `unix_time` in the specified `format`.",
+  usage = "_FUNC_(unix_time[, fmt]) - Returns `unix_time` in the specified `fmt`.",
   arguments = """
     Arguments:
       * unix_time - UNIX Timestamp to be converted to the provided format.
-      * format - Date/time format pattern to follow. See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">Datetime Patterns</a>
-                 for valid date and time format patterns.
+      * fmt - Date/time format pattern to follow. See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">Datetime Patterns</a>
+              for valid date and time format patterns. The 'yyyy-MM-dd HH:mm:ss' pattern is used if omitted.
   """,
   examples = """
     Examples:
       > SELECT _FUNC_(0, 'yyyy-MM-dd HH:mm:ss');
        1969-12-31 16:00:00
+
+      > SELECT _FUNC_(0);
+       1969-12-31 16:00:00
   """,
   group = "datetime_funcs",
   since = "1.5.0")
 // scalastyle:on line.size.limit
 case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[String] = None)
-  extends BinaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes {
+  extends BinaryExpression with TimestampFormatterHelper with ImplicitCastInputTypes
+    with NullIntolerant {
 
   def this(sec: Expression, format: Expression) = this(sec, format, None)
 
@@ -975,90 +1162,38 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[
   override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
     copy(timeZoneId = Option(timeZoneId))
 
-  private lazy val constFormat: UTF8String = right.eval().asInstanceOf[UTF8String]
-  private lazy val formatter: TimestampFormatter =
-    try {
-      TimestampFormatter(
-        constFormat.toString,
-        zoneId,
-        legacyFormat = SIMPLE_DATE_FORMAT,
-        needVarLengthSecondFraction = false)
-    } catch {
-      case NonFatal(_) => null
-    }
-
-  override def eval(input: InternalRow): Any = {
-    val time = left.eval(input)
-    if (time == null) {
-      null
-    } else {
-      if (format.foldable) {
-        if (constFormat == null || formatter == null) {
-          null
-        } else {
-          try {
-            UTF8String.fromString(formatter.format(time.asInstanceOf[Long] * MICROS_PER_SECOND))
-          } catch {
-            case NonFatal(_) => null
-          }
-        }
-      } else {
-        val f = format.eval(input)
-        if (f == null) {
-          null
-        } else {
-          try {
-            UTF8String.fromString(
-              TimestampFormatter(
-                f.toString,
-                zoneId,
-                legacyFormat = SIMPLE_DATE_FORMAT,
-                needVarLengthSecondFraction = false)
-                .format(time.asInstanceOf[Long] * MICROS_PER_SECOND))
-          } catch {
-            case NonFatal(_) => null
-          }
-        }
-      }
-    }
+  override def nullSafeEval(seconds: Any, format: Any): Any = {
+    val fmt = formatterOption.getOrElse(getFormatter(format.toString))
+    UTF8String.fromString(fmt.format(seconds.asInstanceOf[Long] * MICROS_PER_SECOND))
   }
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val df = classOf[TimestampFormatter].getName
-    if (format.foldable) {
-      if (formatter == null) {
-        ExprCode.forNullValue(StringType)
-      } else {
-        val formatterName = ctx.addReferenceObj("formatter", formatter, df)
-        val t = left.genCode(ctx)
-        ev.copy(code = code"""
-          ${t.code}
-          boolean ${ev.isNull} = ${t.isNull};
-          ${CodeGenerator.javaType(dataType)} ${ev.value} = ${CodeGenerator.defaultValue(dataType)};
-          if (!${ev.isNull}) {
-            try {
-              ${ev.value} = UTF8String.fromString($formatterName.format(${t.value} * 1000000L));
-            } catch (java.lang.IllegalArgumentException e) {
-              ${ev.isNull} = true;
-            }
-          }""")
-      }
-    } else {
-      val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
+    formatterOption.map { f =>
+      val formatterName = ctx.addReferenceObj("formatter", f)
+      defineCodeGen(ctx, ev, (seconds, _) =>
+        s"UTF8String.fromString($formatterName.format($seconds * 1000000L))")
+    }.getOrElse {
       val tf = TimestampFormatter.getClass.getName.stripSuffix("$")
       val ldf = LegacyDateFormats.getClass.getName.stripSuffix("$")
-      nullSafeCodeGen(ctx, ev, (seconds, f) => {
+      val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
+      defineCodeGen(ctx, ev, (seconds, format) =>
         s"""
-        try {
-          ${ev.value} = UTF8String.fromString(
-            $tf$$.MODULE$$.apply($f.toString(), $zid, $ldf$$.MODULE$$.SIMPLE_DATE_FORMAT(), false)
-              .format($seconds * 1000000L));
-        } catch (java.lang.IllegalArgumentException e) {
-          ${ev.isNull} = true;
-        }"""
-      })
+           |UTF8String.fromString(
+           |  $tf$$.MODULE$$.apply($format.toString(),
+           |  $zid,
+           |  $ldf$$.MODULE$$.SIMPLE_DATE_FORMAT(),
+           |  false).format($seconds * 1000000L))
+           |""".stripMargin)
     }
   }
+
+  override protected def formatString: Expression = format
+
+  override protected def isParsing: Boolean = false
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): FromUnixTime =
+    copy(sec = newLeft, format = newRight)
 }
 
 /**
@@ -1073,7 +1208,8 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[
   """,
   group = "datetime_funcs",
   since = "1.5.0")
-case class LastDay(startDate: Expression) extends UnaryExpression with ImplicitCastInputTypes {
+case class LastDay(startDate: Expression)
+  extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
   override def child: Expression = startDate
 
   override def inputTypes: Seq[AbstractDataType] = Seq(DateType)
@@ -1090,6 +1226,9 @@ case class LastDay(startDate: Expression) extends UnaryExpression with ImplicitC
   }
 
   override def prettyName: String = "last_day"
+
+  override protected def withNewChildInternal(newChild: Expression): LastDay =
+    copy(startDate = newChild)
 }
 
 /**
@@ -1101,7 +1240,12 @@ case class LastDay(startDate: Expression) extends UnaryExpression with ImplicitC
  */
 // scalastyle:off line.size.limit
 @ExpressionDescription(
-  usage = "_FUNC_(start_date, day_of_week) - Returns the first date which is later than `start_date` and named as indicated.",
+  usage =
+    """_FUNC_(start_date, day_of_week) - Returns the first date which is later than `start_date` and named as indicated.
+      The function returns NULL if at least one of the input parameters is NULL.
+      When both of the input parameters are not NULL and day_of_week is an invalid input,
+      the function throws IllegalArgumentException if `spark.sql.ansi.enabled` is set to true, otherwise NULL.
+      """,
   examples = """
     Examples:
       > SELECT _FUNC_('2015-01-14', 'TU');
@@ -1110,64 +1254,89 @@ case class LastDay(startDate: Expression) extends UnaryExpression with ImplicitC
   group = "datetime_funcs",
   since = "1.5.0")
 // scalastyle:on line.size.limit
-case class NextDay(startDate: Expression, dayOfWeek: Expression)
-  extends BinaryExpression with ImplicitCastInputTypes {
+case class NextDay(
+    startDate: Expression,
+    dayOfWeek: Expression,
+    failOnError: Boolean = SQLConf.get.ansiEnabled)
+  extends BinaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def left: Expression = startDate
   override def right: Expression = dayOfWeek
 
+  def this(left: Expression, right: Expression) = this(left, right, SQLConf.get.ansiEnabled)
+
   override def inputTypes: Seq[AbstractDataType] = Seq(DateType, StringType)
 
   override def dataType: DataType = DateType
   override def nullable: Boolean = true
 
   override def nullSafeEval(start: Any, dayOfW: Any): Any = {
-    val dow = DateTimeUtils.getDayOfWeekFromString(dayOfW.asInstanceOf[UTF8String])
-    if (dow == -1) {
-      null
-    } else {
+    try {
+      val dow = DateTimeUtils.getDayOfWeekFromString(dayOfW.asInstanceOf[UTF8String])
       val sd = start.asInstanceOf[Int]
       DateTimeUtils.getNextDateForDayOfWeek(sd, dow)
+    } catch {
+      case _: IllegalArgumentException if !failOnError => null
+    }
+  }
+
+  private def dateTimeUtilClass: String = DateTimeUtils.getClass.getName.stripSuffix("$")
+
+  private def nextDayGenCode(
+      ev: ExprCode,
+      dayOfWeekTerm: String,
+      sd: String,
+      dowS: String): String = {
+    if (failOnError) {
+      s"""
+       |int $dayOfWeekTerm = $dateTimeUtilClass.getDayOfWeekFromString($dowS);
+       |${ev.value} = $dateTimeUtilClass.getNextDateForDayOfWeek($sd, $dayOfWeekTerm);
+       |""".stripMargin
+    } else {
+      s"""
+       |try {
+       |  int $dayOfWeekTerm = $dateTimeUtilClass.getDayOfWeekFromString($dowS);
+       |  ${ev.value} = $dateTimeUtilClass.getNextDateForDayOfWeek($sd, $dayOfWeekTerm);
+       |} catch (IllegalArgumentException e) {
+       |  ${ev.isNull} = true;
+       |}
+       |""".stripMargin
     }
   }
 
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     nullSafeCodeGen(ctx, ev, (sd, dowS) => {
-      val dateTimeUtilClass = DateTimeUtils.getClass.getName.stripSuffix("$")
       val dayOfWeekTerm = ctx.freshName("dayOfWeek")
       if (dayOfWeek.foldable) {
         val input = dayOfWeek.eval().asInstanceOf[UTF8String]
-        if ((input eq null) || DateTimeUtils.getDayOfWeekFromString(input) == -1) {
-          s"""
-             |${ev.isNull} = true;
-           """.stripMargin
+        if (input eq null) {
+          s"""${ev.isNull} = true;"""
         } else {
-          val dayOfWeekValue = DateTimeUtils.getDayOfWeekFromString(input)
-          s"""
-             |${ev.value} = $dateTimeUtilClass.getNextDateForDayOfWeek($sd, $dayOfWeekValue);
-           """.stripMargin
+          try {
+            val dayOfWeekValue = DateTimeUtils.getDayOfWeekFromString(input)
+            s"${ev.value} = $dateTimeUtilClass.getNextDateForDayOfWeek($sd, $dayOfWeekValue);"
+          } catch {
+            case _: IllegalArgumentException => nextDayGenCode(ev, dayOfWeekTerm, sd, dowS)
+          }
         }
       } else {
-        s"""
-           |int $dayOfWeekTerm = $dateTimeUtilClass.getDayOfWeekFromString($dowS);
-           |if ($dayOfWeekTerm == -1) {
-           |  ${ev.isNull} = true;
-           |} else {
-           |  ${ev.value} = $dateTimeUtilClass.getNextDateForDayOfWeek($sd, $dayOfWeekTerm);
-           |}
-         """.stripMargin
+        nextDayGenCode(ev, dayOfWeekTerm, sd, dowS)
       }
     })
   }
 
   override def prettyName: String = "next_day"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): NextDay =
+    copy(startDate = newLeft, dayOfWeek = newRight)
 }
 
 /**
  * Adds an interval to timestamp.
  */
 case class TimeAdd(start: Expression, interval: Expression, timeZoneId: Option[String] = None)
-  extends BinaryExpression with TimeZoneAwareExpression with ExpectsInputTypes {
+  extends BinaryExpression with TimeZoneAwareExpression with ExpectsInputTypes with NullIntolerant {
 
   def this(start: Expression, interval: Expression) = this(start, interval, None)
 
@@ -1176,26 +1345,38 @@ case class TimeAdd(start: Expression, interval: Expression, timeZoneId: Option[S
 
   override def toString: String = s"$left + $right"
   override def sql: String = s"${left.sql} + ${right.sql}"
-  override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType, CalendarIntervalType)
+  override def inputTypes: Seq[AbstractDataType] =
+    Seq(TimestampType, TypeCollection(CalendarIntervalType, DayTimeIntervalType))
 
   override def dataType: DataType = TimestampType
 
   override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
     copy(timeZoneId = Option(timeZoneId))
 
-  override def nullSafeEval(start: Any, interval: Any): Any = {
-    val itvl = interval.asInstanceOf[CalendarInterval]
-    DateTimeUtils.timestampAddInterval(
-      start.asInstanceOf[Long], itvl.months, itvl.days, itvl.microseconds, zoneId)
+  override def nullSafeEval(start: Any, interval: Any): Any = right.dataType match {
+    case DayTimeIntervalType =>
+      timestampAddDayTime(start.asInstanceOf[Long], interval.asInstanceOf[Long], zoneId)
+    case CalendarIntervalType =>
+      val i = interval.asInstanceOf[CalendarInterval]
+      timestampAddInterval(start.asInstanceOf[Long], i.months, i.days, i.microseconds, zoneId)
   }
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
     val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, (sd, i) => {
-      s"""$dtu.timestampAddInterval($sd, $i.months, $i.days, $i.microseconds, $zid)"""
-    })
+    interval.dataType match {
+      case DayTimeIntervalType =>
+        defineCodeGen(ctx, ev, (sd, dt) => s"""$dtu.timestampAddDayTime($sd, $dt, $zid)""")
+      case CalendarIntervalType =>
+        defineCodeGen(ctx, ev, (sd, i) => {
+          s"""$dtu.timestampAddInterval($sd, $i.months, $i.days, $i.microseconds, $zid)"""
+        })
+    }
   }
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): TimeAdd =
+    copy(start = newLeft, interval = newRight)
 }
 
 /**
@@ -1209,6 +1390,8 @@ case class DatetimeSub(
   override def exprsReplaced: Seq[Expression] = Seq(start, interval)
   override def toString: String = s"$start - $interval"
   override def mkString(childrenString: Seq[String]): String = childrenString.mkString(" - ")
+  override protected def withNewChildInternal(newChild: Expression): DatetimeSub =
+    copy(child = newChild)
 }
 
 /**
@@ -1225,7 +1408,7 @@ case class DateAddInterval(
     interval: Expression,
     timeZoneId: Option[String] = None,
     ansiEnabled: Boolean = SQLConf.get.ansiEnabled)
-  extends BinaryExpression with ExpectsInputTypes with TimeZoneAwareExpression {
+  extends BinaryExpression with ExpectsInputTypes with TimeZoneAwareExpression with NullIntolerant {
 
   override def left: Expression = start
   override def right: Expression = interval
@@ -1241,10 +1424,10 @@ case class DateAddInterval(
     if (ansiEnabled || itvl.microseconds == 0) {
       DateTimeUtils.dateAddInterval(start.asInstanceOf[Int], itvl)
     } else {
-      val startTs = DateTimeUtils.epochDaysToMicros(start.asInstanceOf[Int], zoneId)
+      val startTs = DateTimeUtils.daysToMicros(start.asInstanceOf[Int], zoneId)
       val resultTs = DateTimeUtils.timestampAddInterval(
         startTs, itvl.months, itvl.days, itvl.microseconds, zoneId)
-      DateTimeUtils.microsToEpochDays(resultTs, zoneId)
+      DateTimeUtils.microsToDays(resultTs, zoneId)
     }
   }
 
@@ -1260,10 +1443,10 @@ case class DateAddInterval(
          |if ($i.microseconds == 0) {
          |  ${ev.value} = $dtu.dateAddInterval($sd, $i);
          |} else {
-         |  long $startTs = $dtu.epochDaysToMicros($sd, $zid);
+         |  long $startTs = $dtu.daysToMicros($sd, $zid);
          |  long $resultTs =
          |    $dtu.timestampAddInterval($startTs, $i.months, $i.days, $i.microseconds, $zid);
-         |  ${ev.value} = $dtu.microsToEpochDays($resultTs, $zid);
+         |  ${ev.value} = $dtu.microsToDays($resultTs, $zid);
          |}
          |""".stripMargin
     })
@@ -1271,43 +1454,21 @@ case class DateAddInterval(
 
   override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
     copy(timeZoneId = Option(timeZoneId))
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): DateAddInterval =
+    copy(start = newLeft, interval = newRight)
 }
 
-/**
- * This is a common function for databases supporting TIMESTAMP WITHOUT TIMEZONE. This function
- * takes a timestamp which is timezone-agnostic, and interprets it as a timestamp in UTC, and
- * renders that timestamp as a timestamp in the given time zone.
- *
- * However, timestamp in Spark represents number of microseconds from the Unix epoch, which is not
- * timezone-agnostic. So in Spark this function just shift the timestamp value from UTC timezone to
- * the given timezone.
- *
- * This function may return confusing result if the input is a string with timezone, e.g.
- * '2018-03-13T06:18:23+00:00'. The reason is that, Spark firstly cast the string to timestamp
- * according to the timezone in the string, and finally display the result by converting the
- * timestamp to string according to the session local timezone.
- */
-// scalastyle:off line.size.limit
-@ExpressionDescription(
-  usage = "_FUNC_(timestamp, timezone) - Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in UTC, and renders that time as a timestamp in the given time zone. For example, 'GMT+1' would yield '2017-07-14 03:40:00.0'.",
-  examples = """
-    Examples:
-      > SELECT _FUNC_('2016-08-31', 'Asia/Seoul');
-       2016-08-31 09:00:00
-  """,
-  group = "datetime_funcs",
-  since = "1.5.0")
-// scalastyle:on line.size.limit
-case class FromUTCTimestamp(left: Expression, right: Expression)
-  extends BinaryExpression with ImplicitCastInputTypes {
+sealed trait UTCTimestamp extends BinaryExpression with ImplicitCastInputTypes with NullIntolerant {
+  val func: (Long, String) => Long
+  val funcName: String
 
   override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType, StringType)
   override def dataType: DataType = TimestampType
-  override def prettyName: String = "from_utc_timestamp"
 
   override def nullSafeEval(time: Any, timezone: Any): Any = {
-    DateTimeUtils.fromUTCTime(time.asInstanceOf[Long],
-      timezone.asInstanceOf[UTF8String].toString)
+    func(time.asInstanceOf[Long], timezone.asInstanceOf[UTF8String].toString)
   }
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
@@ -1326,24 +1487,112 @@ case class FromUTCTimestamp(left: Expression, right: Expression)
         val tzTerm = ctx.addMutableState(tzClass, "tz",
           v => s"""$v = $dtu.getZoneId("$escapedTz");""")
         val utcTerm = "java.time.ZoneOffset.UTC"
+        val (fromTz, toTz) = this match {
+          case _: FromUTCTimestamp => (utcTerm, tzTerm)
+          case _: ToUTCTimestamp => (tzTerm, utcTerm)
+        }
         val eval = left.genCode(ctx)
         ev.copy(code = code"""
            |${eval.code}
            |boolean ${ev.isNull} = ${eval.isNull};
            |long ${ev.value} = 0;
            |if (!${ev.isNull}) {
-           |  ${ev.value} = $dtu.convertTz(${eval.value}, $utcTerm, $tzTerm);
+           |  ${ev.value} = $dtu.convertTz(${eval.value}, $fromTz, $toTz);
            |}
          """.stripMargin)
       }
     } else {
       defineCodeGen(ctx, ev, (timestamp, format) => {
-        s"""$dtu.fromUTCTime($timestamp, $format.toString())"""
+        s"""$dtu.$funcName($timestamp, $format.toString())"""
       })
     }
   }
 }
 
+/**
+ * This is a common function for databases supporting TIMESTAMP WITHOUT TIMEZONE. This function
+ * takes a timestamp which is timezone-agnostic, and interprets it as a timestamp in UTC, and
+ * renders that timestamp as a timestamp in the given time zone.
+ *
+ * However, timestamp in Spark represents number of microseconds from the Unix epoch, which is not
+ * timezone-agnostic. So in Spark this function just shift the timestamp value from UTC timezone to
+ * the given timezone.
+ *
+ * This function may return confusing result if the input is a string with timezone, e.g.
+ * '2018-03-13T06:18:23+00:00'. The reason is that, Spark firstly cast the string to timestamp
+ * according to the timezone in the string, and finally display the result by converting the
+ * timestamp to string according to the session local timezone.
+ */
+// scalastyle:off line.size.limit
+@ExpressionDescription(
+  usage = "_FUNC_(timestamp, timezone) - Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in UTC, and renders that time as a timestamp in the given time zone. For example, 'GMT+1' would yield '2017-07-14 03:40:00.0'.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_('2016-08-31', 'Asia/Seoul');
+       2016-08-31 09:00:00
+  """,
+  group = "datetime_funcs",
+  since = "1.5.0")
+// scalastyle:on line.size.limit
+case class FromUTCTimestamp(left: Expression, right: Expression) extends UTCTimestamp {
+  override val func = DateTimeUtils.fromUTCTime
+  override val funcName: String = "fromUTCTime"
+  override val prettyName: String = "from_utc_timestamp"
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): FromUTCTimestamp =
+    copy(left = newLeft, right = newRight)
+}
+
+/**
+ * This is a common function for databases supporting TIMESTAMP WITHOUT TIMEZONE. This function
+ * takes a timestamp which is timezone-agnostic, and interprets it as a timestamp in the given
+ * timezone, and renders that timestamp as a timestamp in UTC.
+ *
+ * However, timestamp in Spark represents number of microseconds from the Unix epoch, which is not
+ * timezone-agnostic. So in Spark this function just shift the timestamp value from the given
+ * timezone to UTC timezone.
+ *
+ * This function may return confusing result if the input is a string with timezone, e.g.
+ * '2018-03-13T06:18:23+00:00'. The reason is that, Spark firstly cast the string to timestamp
+ * according to the timezone in the string, and finally display the result by converting the
+ * timestamp to string according to the session local timezone.
+ */
+// scalastyle:off line.size.limit
+@ExpressionDescription(
+  usage = "_FUNC_(timestamp, timezone) - Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in the given time zone, and renders that time as a timestamp in UTC. For example, 'GMT+1' would yield '2017-07-14 01:40:00.0'.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_('2016-08-31', 'Asia/Seoul');
+       2016-08-30 15:00:00
+  """,
+  group = "datetime_funcs",
+  since = "1.5.0")
+// scalastyle:on line.size.limit
+case class ToUTCTimestamp(left: Expression, right: Expression) extends UTCTimestamp {
+  override val func = DateTimeUtils.toUTCTime
+  override val funcName: String = "toUTCTime"
+  override val prettyName: String = "to_utc_timestamp"
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): ToUTCTimestamp =
+    copy(left = newLeft, right = newRight)
+}
+
+abstract class AddMonthsBase extends BinaryExpression with ImplicitCastInputTypes
+  with NullIntolerant {
+  override def dataType: DataType = DateType
+
+  override def nullSafeEval(start: Any, months: Any): Any = {
+    DateTimeUtils.dateAddMonths(start.asInstanceOf[Int], months.asInstanceOf[Int])
+  }
+
+  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
+    val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
+    defineCodeGen(ctx, ev, (sd, m) => {
+      s"""$dtu.dateAddMonths($sd, $m)"""
+    })
+  }
+}
+
 /**
  * Returns the date that is num_months after start_date.
  */
@@ -1358,28 +1607,70 @@ case class FromUTCTimestamp(left: Expression, right: Expression)
   group = "datetime_funcs",
   since = "1.5.0")
 // scalastyle:on line.size.limit
-case class AddMonths(startDate: Expression, numMonths: Expression)
-  extends BinaryExpression with ImplicitCastInputTypes {
-
+case class AddMonths(startDate: Expression, numMonths: Expression) extends AddMonthsBase {
   override def left: Expression = startDate
   override def right: Expression = numMonths
 
   override def inputTypes: Seq[AbstractDataType] = Seq(DateType, IntegerType)
 
-  override def dataType: DataType = DateType
+  override def prettyName: String = "add_months"
 
-  override def nullSafeEval(start: Any, months: Any): Any = {
-    DateTimeUtils.dateAddMonths(start.asInstanceOf[Int], months.asInstanceOf[Int])
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): AddMonths =
+    copy(startDate = newLeft, numMonths = newRight)
+}
+
+// Adds the year-month interval to the date
+case class DateAddYMInterval(date: Expression, interval: Expression) extends AddMonthsBase {
+  override def left: Expression = date
+  override def right: Expression = interval
+
+  override def inputTypes: Seq[AbstractDataType] = Seq(DateType, YearMonthIntervalType)
+
+  override def toString: String = s"$left + $right"
+  override def sql: String = s"${left.sql} + ${right.sql}"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): DateAddYMInterval =
+    copy(date = newLeft, interval = newRight)
+}
+
+// Adds the year-month interval to the timestamp
+case class TimestampAddYMInterval(
+    timestamp: Expression,
+    interval: Expression,
+    timeZoneId: Option[String] = None)
+  extends BinaryExpression with TimeZoneAwareExpression with ExpectsInputTypes with NullIntolerant {
+
+  def this(timestamp: Expression, interval: Expression) = this(timestamp, interval, None)
+
+  override def left: Expression = timestamp
+  override def right: Expression = interval
+
+  override def toString: String = s"$left + $right"
+  override def sql: String = s"${left.sql} + ${right.sql}"
+  override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType, YearMonthIntervalType)
+
+  override def dataType: DataType = TimestampType
+
+  override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
+    copy(timeZoneId = Option(timeZoneId))
+
+  override def nullSafeEval(micros: Any, months: Any): Any = {
+    timestampAddMonths(micros.asInstanceOf[Long], months.asInstanceOf[Int], zoneId)
   }
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
+    val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
     val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, (sd, m) => {
-      s"""$dtu.dateAddMonths($sd, $m)"""
+    defineCodeGen(ctx, ev, (micros, months) => {
+      s"""$dtu.timestampAddMonths($micros, $months, $zid)"""
     })
   }
 
-  override def prettyName: String = "add_months"
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): TimestampAddYMInterval =
+    copy(timestamp = newLeft, interval = newRight)
 }
 
 /**
@@ -1413,14 +1704,17 @@ case class MonthsBetween(
     date2: Expression,
     roundOff: Expression,
     timeZoneId: Option[String] = None)
-  extends TernaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes {
+  extends TernaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes
+    with NullIntolerant {
 
   def this(date1: Expression, date2: Expression) = this(date1, date2, Literal.TrueLiteral, None)
 
   def this(date1: Expression, date2: Expression, roundOff: Expression) =
     this(date1, date2, roundOff, None)
 
-  override def children: Seq[Expression] = Seq(date1, date2, roundOff)
+  override def first: Expression = date1
+  override def second: Expression = date2
+  override def third: Expression = roundOff
 
   override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType, TimestampType, BooleanType)
 
@@ -1443,77 +1737,10 @@ case class MonthsBetween(
   }
 
   override def prettyName: String = "months_between"
-}
-
-/**
- * This is a common function for databases supporting TIMESTAMP WITHOUT TIMEZONE. This function
- * takes a timestamp which is timezone-agnostic, and interprets it as a timestamp in the given
- * timezone, and renders that timestamp as a timestamp in UTC.
- *
- * However, timestamp in Spark represents number of microseconds from the Unix epoch, which is not
- * timezone-agnostic. So in Spark this function just shift the timestamp value from the given
- * timezone to UTC timezone.
- *
- * This function may return confusing result if the input is a string with timezone, e.g.
- * '2018-03-13T06:18:23+00:00'. The reason is that, Spark firstly cast the string to timestamp
- * according to the timezone in the string, and finally display the result by converting the
- * timestamp to string according to the session local timezone.
- */
-// scalastyle:off line.size.limit
-@ExpressionDescription(
-  usage = "_FUNC_(timestamp, timezone) - Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in the given time zone, and renders that time as a timestamp in UTC. For example, 'GMT+1' would yield '2017-07-14 01:40:00.0'.",
-  examples = """
-    Examples:
-      > SELECT _FUNC_('2016-08-31', 'Asia/Seoul');
-       2016-08-30 15:00:00
-  """,
-  group = "datetime_funcs",
-  since = "1.5.0")
-// scalastyle:on line.size.limit
-case class ToUTCTimestamp(left: Expression, right: Expression)
-  extends BinaryExpression with ImplicitCastInputTypes {
-
-  override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType, StringType)
-  override def dataType: DataType = TimestampType
-  override def prettyName: String = "to_utc_timestamp"
 
-  override def nullSafeEval(time: Any, timezone: Any): Any = {
-    DateTimeUtils.toUTCTime(time.asInstanceOf[Long],
-      timezone.asInstanceOf[UTF8String].toString)
-  }
-
-  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    if (right.foldable) {
-      val tz = right.eval().asInstanceOf[UTF8String]
-      if (tz == null) {
-        ev.copy(code = code"""
-           |boolean ${ev.isNull} = true;
-           |long ${ev.value} = 0;
-         """.stripMargin)
-      } else {
-        val tzClass = classOf[ZoneId].getName
-        val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-        val escapedTz = StringEscapeUtils.escapeJava(tz.toString)
-        val tzTerm = ctx.addMutableState(tzClass, "tz",
-          v => s"""$v = $dtu.getZoneId("$escapedTz");""")
-        val utcTerm = "java.time.ZoneOffset.UTC"
-        val eval = left.genCode(ctx)
-        ev.copy(code = code"""
-           |${eval.code}
-           |boolean ${ev.isNull} = ${eval.isNull};
-           |long ${ev.value} = 0;
-           |if (!${ev.isNull}) {
-           |  ${ev.value} = $dtu.convertTz(${eval.value}, $tzTerm, $utcTerm);
-           |}
-         """.stripMargin)
-      }
-    } else {
-      defineCodeGen(ctx, ev, (timestamp, format) => {
-        s"""$dtu.toUTCTime($timestamp, $format.toString())"""
-      })
-    }
-  }
+  override protected def withNewChildrenInternal(
+      newFirst: Expression, newSecond: Expression, newThird: Expression): MonthsBetween =
+    copy(date1 = newFirst, date2 = newSecond, roundOff = newThird)
 }
 
 /**
@@ -1545,9 +1772,8 @@ case class ToUTCTimestamp(left: Expression, right: Expression)
 case class ParseToDate(left: Expression, format: Option[Expression], child: Expression)
   extends RuntimeReplaceable {
 
-  def this(left: Expression, format: Expression) {
-      this(left, Option(format),
-        Cast(Cast(UnixTimestamp(left, format), TimestampType), DateType))
+  def this(left: Expression, format: Expression) = {
+    this(left, Option(format), Cast(GetTimestamp(left, format), DateType))
   }
 
   def this(left: Expression) = {
@@ -1559,6 +1785,9 @@ case class ParseToDate(left: Expression, format: Option[Expression], child: Expr
   override def flatArguments: Iterator[Any] = Iterator(left, format)
 
   override def prettyName: String = "to_date"
+
+  override protected def withNewChildInternal(newChild: Expression): ParseToDate =
+    copy(child = newChild)
 }
 
 /**
@@ -1601,6 +1830,9 @@ case class ParseToTimestamp(left: Expression, format: Option[Expression], child:
 
   override def prettyName: String = "to_timestamp"
   override def dataType: DataType = TimestampType
+
+  override protected def withNewChildInternal(newChild: Expression): ParseToTimestamp =
+    copy(child = newChild)
 }
 
 trait TruncInstant extends BinaryExpression with ImplicitCastInputTypes {
@@ -1693,13 +1925,13 @@ trait TruncInstant extends BinaryExpression with ImplicitCastInputTypes {
     _FUNC_(date, fmt) - Returns `date` with the time portion of the day truncated to the unit specified by the format model `fmt`.
   """,
   arguments = """
-     Arguments:
-       * date - date value or valid date string
-       * fmt - the format representing the unit to be truncated to
-           - "YEAR", "YYYY", "YY" - truncate to the first date of the year that the `date` falls in
-           - "QUARTER" - truncate to the first date of the quarter that the `date` falls in
-           - "MONTH", "MM", "MON" - truncate to the first date of the month that the `date` falls in
-           - "WEEK" - truncate to the Monday of the week that the `date` falls in
+    Arguments:
+      * date - date value or valid date string
+      * fmt - the format representing the unit to be truncated to
+          - "YEAR", "YYYY", "YY" - truncate to the first date of the year that the `date` falls in
+          - "QUARTER" - truncate to the first date of the quarter that the `date` falls in
+          - "MONTH", "MM", "MON" - truncate to the first date of the month that the `date` falls in
+          - "WEEK" - truncate to the Monday of the week that the `date` falls in
   """,
   examples = """
     Examples:
@@ -1736,6 +1968,10 @@ case class TruncDate(date: Expression, format: Expression)
       (date: String, fmt: String) => s"truncDate($date, $fmt);"
     }
   }
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): TruncDate =
+    copy(date = newLeft, format = newRight)
 }
 
 /**
@@ -1747,19 +1983,19 @@ case class TruncDate(date: Expression, format: Expression)
     _FUNC_(fmt, ts) - Returns timestamp `ts` truncated to the unit specified by the format model `fmt`.
   """,
   arguments = """
-     Arguments:
-       * fmt - the format representing the unit to be truncated to
-           - "YEAR", "YYYY", "YY" - truncate to the first date of the year that the `ts` falls in, the time part will be zero out
-           - "QUARTER" - truncate to the first date of the quarter that the `ts` falls in, the time part will be zero out
-           - "MONTH", "MM", "MON" - truncate to the first date of the month that the `ts` falls in, the time part will be zero out
-           - "WEEK" - truncate to the Monday of the week that the `ts` falls in, the time part will be zero out
-           - "DAY", "DD" - zero out the time part
-           - "HOUR" - zero out the minute and second with fraction part
-           - "MINUTE"- zero out the second with fraction part
-           - "SECOND" -  zero out the second fraction part
-           - "MILLISECOND" - zero out the microseconds
-           - "MICROSECOND" - everything remains
-       * ts - datetime value or valid timestamp string
+    Arguments:
+      * fmt - the format representing the unit to be truncated to
+          - "YEAR", "YYYY", "YY" - truncate to the first date of the year that the `ts` falls in, the time part will be zero out
+          - "QUARTER" - truncate to the first date of the quarter that the `ts` falls in, the time part will be zero out
+          - "MONTH", "MM", "MON" - truncate to the first date of the month that the `ts` falls in, the time part will be zero out
+          - "WEEK" - truncate to the Monday of the week that the `ts` falls in, the time part will be zero out
+          - "DAY", "DD" - zero out the time part
+          - "HOUR" - zero out the minute and second with fraction part
+          - "MINUTE"- zero out the second with fraction part
+          - "SECOND" -  zero out the second fraction part
+          - "MILLISECOND" - zero out the microseconds
+          - "MICROSECOND" - everything remains
+      * ts - datetime value or valid timestamp string
   """,
   examples = """
     Examples:
@@ -1807,6 +2043,10 @@ case class TruncTimestamp(
         s"truncTimestamp($date, $fmt, $zid);"
     }
   }
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): TruncTimestamp =
+    copy(format = newLeft, timestamp = newRight)
 }
 
 /**
@@ -1825,7 +2065,7 @@ case class TruncTimestamp(
   group = "datetime_funcs",
   since = "1.5.0")
 case class DateDiff(endDate: Expression, startDate: Expression)
-  extends BinaryExpression with ImplicitCastInputTypes {
+  extends BinaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def left: Expression = endDate
   override def right: Expression = startDate
@@ -1839,6 +2079,10 @@ case class DateDiff(endDate: Expression, startDate: Expression)
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     defineCodeGen(ctx, ev, (end, start) => s"$end - $start")
   }
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): DateDiff =
+    copy(endDate = newLeft, startDate = newRight)
 }
 
 /**
@@ -1847,7 +2091,8 @@ case class DateDiff(endDate: Expression, startDate: Expression)
 private case class GetTimestamp(
     left: Expression,
     right: Expression,
-    timeZoneId: Option[String] = None)
+    timeZoneId: Option[String] = None,
+    failOnError: Boolean = SQLConf.get.ansiEnabled)
   extends ToTimestamp {
 
   override val downScaleFactor = 1
@@ -1855,6 +2100,10 @@ private case class GetTimestamp(
 
   override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
     copy(timeZoneId = Option(timeZoneId))
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): GetTimestamp =
+    copy(left = newLeft, right = newRight)
 }
 
 @ExpressionDescription(
@@ -1878,36 +2127,50 @@ private case class GetTimestamp(
   """,
   group = "datetime_funcs",
   since = "3.0.0")
-case class MakeDate(year: Expression, month: Expression, day: Expression)
-  extends TernaryExpression with ImplicitCastInputTypes {
+case class MakeDate(
+    year: Expression,
+    month: Expression,
+    day: Expression,
+    failOnError: Boolean = SQLConf.get.ansiEnabled)
+  extends TernaryExpression with ImplicitCastInputTypes with NullIntolerant {
+
+  def this(year: Expression, month: Expression, day: Expression) =
+    this(year, month, day, SQLConf.get.ansiEnabled)
 
-  override def children: Seq[Expression] = Seq(year, month, day)
+  override def first: Expression = year
+  override def second: Expression = month
+  override def third: Expression = day
   override def inputTypes: Seq[AbstractDataType] = Seq(IntegerType, IntegerType, IntegerType)
   override def dataType: DataType = DateType
-  override def nullable: Boolean = true
+  override def nullable: Boolean = if (failOnError) children.exists(_.nullable) else true
 
   override def nullSafeEval(year: Any, month: Any, day: Any): Any = {
     try {
       val ld = LocalDate.of(year.asInstanceOf[Int], month.asInstanceOf[Int], day.asInstanceOf[Int])
       localDateToDays(ld)
     } catch {
-      case _: java.time.DateTimeException => null
+      case _: java.time.DateTimeException if !failOnError => null
     }
   }
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
+    val failOnErrorBranch = if (failOnError) "throw e;" else s"${ev.isNull} = true;"
     nullSafeCodeGen(ctx, ev, (year, month, day) => {
       s"""
       try {
         ${ev.value} = $dtu.localDateToDays(java.time.LocalDate.of($year, $month, $day));
       } catch (java.time.DateTimeException e) {
-        ${ev.isNull} = true;
+        $failOnErrorBranch
       }"""
     })
   }
 
   override def prettyName: String = "make_date"
+
+  override protected def withNewChildrenInternal(
+      newFirst: Expression, newSecond: Expression, newThird: Expression): MakeDate =
+    copy(year = newFirst, month = newSecond, day = newThird)
 }
 
 // scalastyle:off line.size.limit
@@ -1949,8 +2212,10 @@ case class MakeTimestamp(
     min: Expression,
     sec: Expression,
     timezone: Option[Expression] = None,
-    timeZoneId: Option[String] = None)
-  extends SeptenaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes {
+    timeZoneId: Option[String] = None,
+    failOnError: Boolean = SQLConf.get.ansiEnabled)
+  extends SeptenaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes
+    with NullIntolerant {
 
   def this(
       year: Expression,
@@ -1959,7 +2224,7 @@ case class MakeTimestamp(
       hour: Expression,
       min: Expression,
       sec: Expression) = {
-    this(year, month, day, hour, min, sec, None, None)
+    this(year, month, day, hour, min, sec, None, None, SQLConf.get.ansiEnabled)
   }
 
   def this(
@@ -1970,7 +2235,7 @@ case class MakeTimestamp(
       min: Expression,
       sec: Expression,
       timezone: Expression) = {
-    this(year, month, day, hour, min, sec, Some(timezone), None)
+    this(year, month, day, hour, min, sec, Some(timezone), None, SQLConf.get.ansiEnabled)
   }
 
   override def children: Seq[Expression] = Seq(year, month, day, hour, min, sec) ++ timezone
@@ -1980,7 +2245,7 @@ case class MakeTimestamp(
     Seq(IntegerType, IntegerType, IntegerType, IntegerType, IntegerType, DecimalType(8, 6)) ++
     timezone.map(_ => StringType)
   override def dataType: DataType = TimestampType
-  override def nullable: Boolean = true
+  override def nullable: Boolean = if (failOnError) children.exists(_.nullable) else true
 
   override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
     copy(timeZoneId = Option(timeZoneId))
@@ -1991,26 +2256,30 @@ case class MakeTimestamp(
       day: Int,
       hour: Int,
       min: Int,
-      secAndNanos: Decimal,
+      secAndMicros: Decimal,
       zoneId: ZoneId): Any = {
     try {
-      val secFloor = secAndNanos.floor
-      val nanosPerSec = Decimal(NANOS_PER_SECOND, 10, 0)
-      val nanos = ((secAndNanos - secFloor) * nanosPerSec).toInt
-      val seconds = secFloor.toInt
+      assert(secAndMicros.scale == 6,
+        s"Seconds fraction must have 6 digits for microseconds but got ${secAndMicros.scale}")
+      val unscaledSecFrac = secAndMicros.toUnscaledLong
+      assert(secAndMicros.precision <= 8,
+        s"Seconds and fraction cannot have more than 8 digits but got ${secAndMicros.precision}")
+      val totalMicros = unscaledSecFrac.toInt // 8 digits cannot overflow Int
+      val seconds = Math.floorDiv(totalMicros, MICROS_PER_SECOND.toInt)
+      val nanos = Math.floorMod(totalMicros, MICROS_PER_SECOND.toInt) * NANOS_PER_MICROS.toInt
       val ldt = if (seconds == 60) {
         if (nanos == 0) {
           // This case of sec = 60 and nanos = 0 is supported for compatibility with PostgreSQL
           LocalDateTime.of(year, month, day, hour, min, 0, 0).plusMinutes(1)
         } else {
-          throw new DateTimeException("The fraction of sec must be zero. Valid range is [0, 60].")
+          throw QueryExecutionErrors.invalidFractionOfSecondError
         }
       } else {
         LocalDateTime.of(year, month, day, hour, min, seconds, nanos)
       }
       instantToMicros(ldt.atZone(zoneId).toInstant)
     } catch {
-      case _: DateTimeException => null
+      case _: DateTimeException if !failOnError => null
     }
   }
 
@@ -2039,6 +2308,7 @@ case class MakeTimestamp(
     val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
     val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
     val d = Decimal.getClass.getName.stripSuffix("$")
+    val failOnErrorBranch = if (failOnError) "throw e;" else s"${ev.isNull} = true;"
     nullSafeCodeGen(ctx, ev, (year, month, day, hour, min, secAndNanos, timezone) => {
       val zoneId = timezone.map(tz => s"$dtu.getZoneId(${tz}.toString())").getOrElse(zid)
       s"""
@@ -2053,8 +2323,7 @@ case class MakeTimestamp(
             ldt = java.time.LocalDateTime.of(
               $year, $month, $day, $hour, $min, 0, 0).plusMinutes(1);
           } else {
-            throw new java.time.DateTimeException(
-              "The fraction of sec must be zero. Valid range is [0, 60].");
+            throw QueryExecutionErrors.invalidFractionOfSecondError();
           }
         } else {
           ldt = java.time.LocalDateTime.of($year, $month, $day, $hour, $min, seconds, nanos);
@@ -2062,12 +2331,26 @@ case class MakeTimestamp(
         java.time.Instant instant = ldt.atZone($zoneId).toInstant();
         ${ev.value} = $dtu.instantToMicros(instant);
       } catch (java.time.DateTimeException e) {
-        ${ev.isNull} = true;
+        $failOnErrorBranch
       }"""
     })
   }
 
   override def prettyName: String = "make_timestamp"
+
+//  override def children: Seq[Expression] = Seq(year, month, day, hour, min, sec) ++ timezone
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[Expression]): MakeTimestamp = {
+    val timezoneOpt = if (timezone.isDefined) Some(newChildren(6)) else None
+    copy(
+      year = newChildren(0),
+      month = newChildren(1),
+      day = newChildren(2),
+      hour = newChildren(3),
+      min = newChildren(4),
+      sec = newChildren(5),
+      timezone = timezoneOpt)
+  }
 }
 
 object DatePart {
@@ -2093,22 +2376,22 @@ object DatePart {
 
   def toEquivalentExpr(field: Expression, source: Expression): Expression = {
     if (!field.foldable) {
-      throw new AnalysisException("The field parameter needs to be a foldable string value.")
+      throw QueryCompilationErrors.unfoldableFieldUnsupportedError
     }
     val fieldEval = field.eval()
     if (fieldEval == null) {
       Literal(null, DoubleType)
     } else {
       val fieldStr = fieldEval.asInstanceOf[UTF8String].toString
-      val errMsg = s"Literals of type '$fieldStr' are currently not supported " +
-        s"for the ${source.dataType.catalogString} type."
-      if (source.dataType == CalendarIntervalType) {
-        ExtractIntervalPart.parseExtractField(
-          fieldStr,
-          source,
-          throw new AnalysisException(errMsg))
-      } else {
-        DatePart.parseExtractField(fieldStr, source, throw new AnalysisException(errMsg))
+
+      def analysisException =
+        throw QueryCompilationErrors.literalTypeUnsupportedForSourceTypeError(fieldStr, source)
+
+      source.dataType match {
+        case YearMonthIntervalType | DayTimeIntervalType | CalendarIntervalType =>
+          ExtractIntervalPart.parseExtractField(fieldStr, source, analysisException)
+        case _ =>
+          DatePart.parseExtractField(fieldStr, source, analysisException)
       }
     }
   }
@@ -2136,6 +2419,10 @@ object DatePart {
        5
       > SELECT _FUNC_('seconds', interval 5 hours 30 seconds 1 milliseconds 1 microseconds);
        30.001001
+      > SELECT _FUNC_('MONTH', INTERVAL '2021-11' YEAR TO MONTH);
+       11
+      > SELECT _FUNC_('MINUTE', INTERVAL '123 23:55:59.002001' DAY TO SECOND);
+       55
   """,
   note = """
     The _FUNC_ function is equivalent to the SQL-standard function `EXTRACT(field FROM source)`
@@ -2154,6 +2441,9 @@ case class DatePart(field: Expression, source: Expression, child: Expression)
   override def exprsReplaced: Seq[Expression] = Seq(field, source)
 
   override def prettyName: String = "date_part"
+
+  override protected def withNewChildInternal(newChild: Expression): DatePart =
+    copy(child = newChild)
 }
 
 // scalastyle:off line.size.limit
@@ -2198,10 +2488,15 @@ case class DatePart(field: Expression, source: Expression, child: Expression)
        5
       > SELECT _FUNC_(seconds FROM interval 5 hours 30 seconds 1 milliseconds 1 microseconds);
        30.001001
+      > SELECT _FUNC_(MONTH FROM INTERVAL '2021-11' YEAR TO MONTH);
+       11
+      > SELECT _FUNC_(MINUTE FROM INTERVAL '123 23:55:59.002001' DAY TO SECOND);
+       55
   """,
   note = """
     The _FUNC_ function is equivalent to `date_part(field, source)`.
   """,
+  group = "datetime_funcs",
   since = "3.0.0")
 // scalastyle:on line.size.limit
 case class Extract(field: Expression, source: Expression, child: Expression)
@@ -2218,49 +2513,128 @@ case class Extract(field: Expression, source: Expression, child: Expression)
   override def mkString(childrenString: Seq[String]): String = {
     prettyName + childrenString.mkString("(", " FROM ", ")")
   }
+
+  override protected def withNewChildInternal(newChild: Expression): Extract =
+    copy(child = newChild)
 }
 
 /**
- * Returns the interval from startTimestamp to endTimestamp in which the `months` and `day` field
- * is set to 0 and the `microseconds` field is initialized to the microsecond difference
- * between the given timestamps.
+ * Returns the interval from `right` to `left` timestamps.
+ *   - When the SQL config `spark.sql.legacy.interval.enabled` is `true`,
+ *     it returns `CalendarIntervalType` in which the months` and `day` field is set to 0 and
+ *     the `microseconds` field is initialized to the microsecond difference between
+ *     the given timestamps.
+ *   - Otherwise the expression returns `DayTimeIntervalType` with the difference in microseconds
+ *     between given timestamps.
  */
-case class SubtractTimestamps(endTimestamp: Expression, startTimestamp: Expression)
-  extends BinaryExpression with ExpectsInputTypes {
+case class SubtractTimestamps(
+    left: Expression,
+    right: Expression,
+    legacyInterval: Boolean,
+    timeZoneId: Option[String] = None)
+  extends BinaryExpression
+  with TimeZoneAwareExpression
+  with ExpectsInputTypes
+  with NullIntolerant {
+
+  def this(endTimestamp: Expression, startTimestamp: Expression) =
+    this(endTimestamp, startTimestamp, SQLConf.get.legacyIntervalEnabled)
 
-  override def left: Expression = endTimestamp
-  override def right: Expression = startTimestamp
   override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType, TimestampType)
-  override def dataType: DataType = CalendarIntervalType
+  override def dataType: DataType =
+    if (legacyInterval) CalendarIntervalType else DayTimeIntervalType
 
-  override def nullSafeEval(end: Any, start: Any): Any = {
-    new CalendarInterval(0, 0, end.asInstanceOf[Long] - start.asInstanceOf[Long])
+  override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
+    copy(timeZoneId = Option(timeZoneId))
+
+  @transient
+  private lazy val evalFunc: (Long, Long) => Any = legacyInterval match {
+    case false => (leftMicros, rightMicros) =>
+      subtractTimestamps(leftMicros, rightMicros, zoneId)
+    case true => (leftMicros, rightMicros) =>
+      new CalendarInterval(0, 0, leftMicros - rightMicros)
   }
 
-  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    defineCodeGen(ctx, ev, (end, start) =>
-      s"new org.apache.spark.unsafe.types.CalendarInterval(0, 0, $end - $start)")
+  override def nullSafeEval(leftMicros: Any, rightMicros: Any): Any = {
+    evalFunc(leftMicros.asInstanceOf[Long], rightMicros.asInstanceOf[Long])
+  }
+
+  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = legacyInterval match {
+    case false =>
+      val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
+      val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
+      defineCodeGen(ctx, ev, (l, r) => s"""$dtu.subtractTimestamps($l, $r, $zid)""")
+    case true =>
+      defineCodeGen(ctx, ev, (end, start) =>
+        s"new org.apache.spark.unsafe.types.CalendarInterval(0, 0, $end - $start)")
+  }
+
+  override def toString: String = s"($left - $right)"
+  override def sql: String = s"(${left.sql} - ${right.sql})"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): SubtractTimestamps =
+    copy(left = newLeft, right = newRight)
+}
+
+object SubtractTimestamps {
+  def apply(left: Expression, right: Expression): SubtractTimestamps = {
+    new SubtractTimestamps(left, right)
   }
 }
 
 /**
  * Returns the interval from the `left` date (inclusive) to the `right` date (exclusive).
+ *   - When the SQL config `spark.sql.legacy.interval.enabled` is `true`,
+ *     it returns `CalendarIntervalType` in which the `microseconds` field is set to 0 and
+ *     the `months` and `days` fields are initialized to the difference between the given dates.
+ *   - Otherwise the expression returns `DayTimeIntervalType` with the difference in days
+ *     between the given dates.
  */
-case class SubtractDates(left: Expression, right: Expression)
-  extends BinaryExpression with ImplicitCastInputTypes {
+case class SubtractDates(
+    left: Expression,
+    right: Expression,
+    legacyInterval: Boolean)
+  extends BinaryExpression with ImplicitCastInputTypes with NullIntolerant {
+
+  def this(left: Expression, right: Expression) =
+    this(left, right, SQLConf.get.legacyIntervalEnabled)
 
   override def inputTypes: Seq[AbstractDataType] = Seq(DateType, DateType)
-  override def dataType: DataType = CalendarIntervalType
+  override def dataType: DataType =
+    if (legacyInterval) CalendarIntervalType else DayTimeIntervalType
+
+  @transient
+  private lazy val evalFunc: (Int, Int) => Any = legacyInterval match {
+    case false => (leftDays: Int, rightDays: Int) =>
+      Math.multiplyExact(Math.subtractExact(leftDays, rightDays), MICROS_PER_DAY)
+    case true => (leftDays: Int, rightDays: Int) => subtractDates(leftDays, rightDays)
+  }
 
   override def nullSafeEval(leftDays: Any, rightDays: Any): Any = {
-    DateTimeUtils.subtractDates(leftDays.asInstanceOf[Int], rightDays.asInstanceOf[Int])
+    evalFunc(leftDays.asInstanceOf[Int], rightDays.asInstanceOf[Int])
   }
 
-  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    defineCodeGen(ctx, ev, (leftDays, rightDays) => {
-      val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-      s"$dtu.subtractDates($leftDays, $rightDays)"
-    })
+  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = legacyInterval match {
+    case false =>
+      val m = classOf[Math].getName
+      defineCodeGen(ctx, ev, (leftDays, rightDays) =>
+        s"$m.multiplyExact($m.subtractExact($leftDays, $rightDays), ${MICROS_PER_DAY}L)")
+    case true =>
+      defineCodeGen(ctx, ev, (leftDays, rightDays) => {
+        val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
+        s"$dtu.subtractDates($leftDays, $rightDays)"
+      })
   }
+
+  override def toString: String = s"($left - $right)"
+  override def sql: String = s"(${left.sql} - ${right.sql})"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): SubtractDates =
+    copy(left = newLeft, right = newRight)
 }
 
+object SubtractDates {
+  def apply(left: Expression, right: Expression): SubtractDates = new SubtractDates(left, right)
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalExpressions.scala
index 9014ebfe2f96a..7165bca201a9e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalExpressions.scala
@@ -19,6 +19,8 @@ package org.apache.spark.sql.catalyst.expressions
 
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, EmptyBlock, ExprCode}
+import org.apache.spark.sql.catalyst.expressions.codegen.Block._
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
@@ -27,7 +29,7 @@ import org.apache.spark.sql.types._
  * Note: this expression is internal and created only by the optimizer,
  * we don't need to do type check for it.
  */
-case class UnscaledValue(child: Expression) extends UnaryExpression {
+case class UnscaledValue(child: Expression) extends UnaryExpression with NullIntolerant {
 
   override def dataType: DataType = LongType
   override def toString: String = s"UnscaledValue($child)"
@@ -38,6 +40,9 @@ case class UnscaledValue(child: Expression) extends UnaryExpression {
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     defineCodeGen(ctx, ev, c => s"$c.toUnscaledLong()")
   }
+
+  override protected def withNewChildInternal(newChild: Expression): UnscaledValue =
+    copy(child = newChild)
 }
 
 /**
@@ -49,7 +54,7 @@ case class MakeDecimal(
     child: Expression,
     precision: Int,
     scale: Int,
-    nullOnOverflow: Boolean) extends UnaryExpression {
+    nullOnOverflow: Boolean) extends UnaryExpression with NullIntolerant {
 
   def this(child: Expression, precision: Int, scale: Int) = {
     this(child, precision, scale, !SQLConf.get.ansiEnabled)
@@ -87,6 +92,9 @@ case class MakeDecimal(
          |""".stripMargin
     })
   }
+
+  override protected def withNewChildInternal(newChild: Expression): MakeDecimal =
+    copy(child = newChild)
 }
 
 object MakeDecimal {
@@ -109,6 +117,9 @@ case class PromotePrecision(child: Expression) extends UnaryExpression {
   override def prettyName: String = "promote_precision"
   override def sql: String = child.sql
   override lazy val canonicalized: Expression = child.canonicalized
+
+  override protected def withNewChildInternal(newChild: Expression): Expression =
+    copy(child = newChild)
 }
 
 /**
@@ -143,4 +154,59 @@ case class CheckOverflow(
   override def toString: String = s"CheckOverflow($child, $dataType, $nullOnOverflow)"
 
   override def sql: String = child.sql
+
+  override protected def withNewChildInternal(newChild: Expression): CheckOverflow =
+    copy(child = newChild)
+}
+
+// A variant `CheckOverflow`, which treats null as overflow. This is necessary in `Sum`.
+case class CheckOverflowInSum(
+    child: Expression,
+    dataType: DecimalType,
+    nullOnOverflow: Boolean) extends UnaryExpression {
+
+  override def nullable: Boolean = true
+
+  override def eval(input: InternalRow): Any = {
+    val value = child.eval(input)
+    if (value == null) {
+      if (nullOnOverflow) null else throw QueryExecutionErrors.overflowInSumOfDecimalError
+    } else {
+      value.asInstanceOf[Decimal].toPrecision(
+        dataType.precision,
+        dataType.scale,
+        Decimal.ROUND_HALF_UP,
+        nullOnOverflow)
+    }
+  }
+
+  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
+    val childGen = child.genCode(ctx)
+    val nullHandling = if (nullOnOverflow) {
+      ""
+    } else {
+      s"throw QueryExecutionErrors.overflowInSumOfDecimalError();"
+    }
+    val code = code"""
+       |${childGen.code}
+       |boolean ${ev.isNull} = ${childGen.isNull};
+       |Decimal ${ev.value} = null;
+       |if (${childGen.isNull}) {
+       |  $nullHandling
+       |} else {
+       |  ${ev.value} = ${childGen.value}.toPrecision(
+       |    ${dataType.precision}, ${dataType.scale}, Decimal.ROUND_HALF_UP(), $nullOnOverflow);
+       |  ${ev.isNull} = ${ev.value} == null;
+       |}
+       |""".stripMargin
+
+    ev.copy(code = code)
+  }
+
+  override def toString: String = s"CheckOverflowInSum($child, $dataType, $nullOnOverflow)"
+
+  override def sql: String = child.sql
+
+  override protected def withNewChildInternal(newChild: Expression): CheckOverflowInSum =
+    copy(child = newChild)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
index b0a23c62284d2..fef9bb338d834 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
@@ -25,6 +25,7 @@ import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
 import org.apache.spark.sql.catalyst.util.{ArrayData, MapData}
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
@@ -117,6 +118,9 @@ case class UserDefinedGenerator(
   }
 
   override def toString: String = s"UserDefinedGenerator(${children.mkString(",")})"
+
+  override protected def withNewChildrenInternal(
+    newChildren: IndexedSeq[Expression]): UserDefinedGenerator = copy(children = newChildren)
 }
 
 /**
@@ -135,7 +139,9 @@ case class UserDefinedGenerator(
       > SELECT _FUNC_(2, 1, 2, 3);
        1	2
        3	NULL
-  """)
+  """,
+  since = "2.0.0",
+  group = "generator_funcs")
 // scalastyle:on line.size.limit line.contains.tab
 case class Stack(children: Seq[Expression]) extends Generator {
 
@@ -224,6 +230,9 @@ case class Stack(children: Seq[Expression]) extends Generator {
          |$wrapperClass<InternalRow> ${ev.value} = $wrapperClass$$.MODULE$$.make($rowData);
        """.stripMargin, isNull = FalseLiteral)
   }
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Stack =
+    copy(children = newChildren)
 }
 
 /**
@@ -250,6 +259,9 @@ case class ReplicateRows(children: Seq[Expression]) extends Generator with Codeg
       InternalRow(fields: _*)
     }
   }
+
+  override protected def withNewChildrenInternal(
+    newChildren: IndexedSeq[Expression]): ReplicateRows = copy(children = newChildren)
 }
 
 /**
@@ -258,14 +270,17 @@ case class ReplicateRows(children: Seq[Expression]) extends Generator with Codeg
  */
 case class GeneratorOuter(child: Generator) extends UnaryExpression with Generator {
   final override def eval(input: InternalRow = null): TraversableOnce[InternalRow] =
-    throw new UnsupportedOperationException(s"Cannot evaluate expression: $this")
+    throw QueryExecutionErrors.cannotEvaluateExpressionError(this)
 
   final override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode =
-    throw new UnsupportedOperationException(s"Cannot generate code for expression: $this")
+    throw QueryExecutionErrors.cannotGenerateCodeForExpressionError(this)
 
   override def elementSchema: StructType = child.elementSchema
 
   override lazy val resolved: Boolean = false
+
+  override protected def withNewChildInternal(newChild: Expression): GeneratorOuter =
+    copy(child = newChild.asInstanceOf[Generator])
 }
 
 /**
@@ -360,10 +375,14 @@ abstract class ExplodeBase extends UnaryExpression with CollectionGenerator with
       > SELECT _FUNC_(array(10, 20));
        10
        20
-  """)
+  """,
+  since = "1.0.0",
+  group = "generator_funcs")
 // scalastyle:on line.size.limit
 case class Explode(child: Expression) extends ExplodeBase {
   override val position: Boolean = false
+  override protected def withNewChildInternal(newChild: Expression): Explode =
+    copy(child = newChild)
 }
 
 /**
@@ -383,10 +402,14 @@ case class Explode(child: Expression) extends ExplodeBase {
       > SELECT _FUNC_(array(10,20));
        0	10
        1	20
-  """)
+  """,
+  since = "2.0.0",
+  group = "generator_funcs")
 // scalastyle:on line.size.limit line.contains.tab
 case class PosExplode(child: Expression) extends ExplodeBase {
   override val position = true
+  override protected def withNewChildInternal(newChild: Expression): PosExplode =
+    copy(child = newChild)
 }
 
 /**
@@ -400,7 +423,9 @@ case class PosExplode(child: Expression) extends ExplodeBase {
       > SELECT _FUNC_(array(struct(1, 'a'), struct(2, 'b')));
        1	a
        2	b
-  """)
+  """,
+  since = "2.0.0",
+  group = "generator_funcs")
 // scalastyle:on line.size.limit line.contains.tab
 case class Inline(child: Expression) extends UnaryExpression with CollectionGenerator {
   override val inline: Boolean = true
@@ -436,4 +461,6 @@ case class Inline(child: Expression) extends UnaryExpression with CollectionGene
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     child.genCode(ctx)
   }
+
+  override protected def withNewChildInternal(newChild: Expression): Inline = copy(child = newChild)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/grouping.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/grouping.scala
index ac0f6b86ccd96..aff1806582a30 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/grouping.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/grouping.scala
@@ -19,16 +19,23 @@ package org.apache.spark.sql.catalyst.expressions
 
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
+import org.apache.spark.sql.catalyst.trees.UnaryLike
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 /**
  * A placeholder expression for cube/rollup, which will be replaced by analyzer
  */
-trait GroupingSet extends Expression with CodegenFallback {
+trait BaseGroupingSets extends Expression with CodegenFallback {
 
-  def groupByExprs: Seq[Expression]
-  override def children: Seq[Expression] = groupByExprs
+  def groupingSets: Seq[Seq[Expression]]
+  def selectedGroupByExprs: Seq[Seq[Expression]]
+
+  def groupByExprs: Seq[Expression] = {
+    assert(children.forall(_.resolved),
+      "Cannot call BaseGroupingSets.groupByExprs before the children expressions are all resolved.")
+    BaseGroupingSets.distinctGroupByExprs(children)
+  }
 
   // this should be replaced first
   override lazy val resolved: Boolean = false
@@ -39,45 +46,127 @@ trait GroupingSet extends Expression with CodegenFallback {
   override def eval(input: InternalRow): Any = throw new UnsupportedOperationException
 }
 
-// scalastyle:off line.size.limit line.contains.tab
-@ExpressionDescription(
-  usage = """
-    _FUNC_([col1[, col2 ..]]) - create a multi-dimensional cube using the specified columns
-      so that we can run aggregation on them.
-  """,
-  examples = """
-    Examples:
-      > SELECT name, age, count(*) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY _FUNC_(name, age);
-        Bob	5	1
-        Alice	2	1
-        Alice	NULL	1
-        NULL	2	1
-        NULL	NULL	2
-        Bob	NULL	1
-        NULL	5	1
-  """,
-  since = "2.0.0")
-// scalastyle:on line.size.limit line.contains.tab
-case class Cube(groupByExprs: Seq[Expression]) extends GroupingSet {}
+object BaseGroupingSets {
+  /**
+   * 'GROUP BY a, b, c WITH ROLLUP'
+   * is equivalent to
+   * 'GROUP BY GROUPING SETS ( (a, b, c), (a, b), (a), ( ) )'.
+   * Group Count: N + 1 (N is the number of group expressions)
+   *
+   * We need to get all of its subsets for the rule described above, the subset is
+   * represented as sequence of expressions.
+   */
+  def rollupExprs(exprs: Seq[Seq[Expression]]): Seq[Seq[Expression]] =
+    exprs.inits.map(_.flatten).toIndexedSeq
 
-// scalastyle:off line.size.limit line.contains.tab
-@ExpressionDescription(
-  usage = """
-    _FUNC_([col1[, col2 ..]]) - create a multi-dimensional rollup using the specified columns
-      so that we can run aggregation on them.
-  """,
-  examples = """
-    Examples:
-      > SELECT name, age, count(*) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY _FUNC_(name, age);
-        Bob	5	1
-        Alice	2	1
-        Alice	NULL	1
-        NULL	NULL	2
-        Bob	NULL	1
-  """,
-  since = "2.0.0")
-// scalastyle:on line.size.limit line.contains.tab
-case class Rollup(groupByExprs: Seq[Expression]) extends GroupingSet {}
+  /**
+   * 'GROUP BY a, b, c WITH CUBE'
+   * is equivalent to
+   * 'GROUP BY GROUPING SETS ( (a, b, c), (a, b), (b, c), (a, c), (a), (b), (c), ( ) )'.
+   * Group Count: 2 ^ N (N is the number of group expressions)
+   *
+   * We need to get all of its subsets for a given GROUPBY expression, the subsets are
+   * represented as sequence of expressions.
+   */
+  def cubeExprs(exprs: Seq[Seq[Expression]]): Seq[Seq[Expression]] = {
+    // `cubeExprs0` is recursive and returns a lazy Stream. Here we call `toIndexedSeq` to
+    // materialize it and avoid serialization problems later on.
+    cubeExprs0(exprs).toIndexedSeq
+  }
+
+  def cubeExprs0(exprs: Seq[Seq[Expression]]): Seq[Seq[Expression]] = exprs.toList match {
+    case x :: xs =>
+      val initial = cubeExprs0(xs)
+      initial.map(x ++ _) ++ initial
+    case Nil =>
+      Seq(Seq.empty)
+  }
+
+  /**
+   * This methods converts given grouping sets into the indexes of the flatten grouping sets.
+   * Let's say we have a query below:
+   *   SELECT k1, k2, avg(v) FROM t GROUP BY GROUPING SETS ((k1), (k1, k2), (k2, k1));
+   * In this case, flatten grouping sets are "[k1, k1, k2, k2, k1]" and the method
+   * will return indexes "[[1], [2, 3], [4, 5]]".
+   */
+  def computeGroupingSetIndexes(groupingSets: Seq[Seq[Expression]]): Seq[Seq[Int]] = {
+    val startOffsets = groupingSets.map(_.length).scanLeft(0)(_ + _).init
+    groupingSets.zip(startOffsets).map {
+      case (gs, startOffset) => gs.indices.map(_ + startOffset)
+    }
+  }
+
+  def distinctGroupByExprs(exprs: Seq[Expression]): Seq[Expression] = {
+    exprs.foldLeft(Seq.empty[Expression]) { (result, currentExpr) =>
+      // Only unique expressions are included in the group by expressions and is determined
+      // based on their semantic equality. Example. grouping sets ((a * b), (b * a)) results
+      // in grouping expression (a * b)
+      if (result.exists(_.semanticEquals(currentExpr))) {
+        result
+      } else {
+        result :+ currentExpr
+      }
+    }
+  }
+}
+
+case class Cube(
+    groupingSetIndexes: Seq[Seq[Int]],
+    children: Seq[Expression]) extends BaseGroupingSets {
+  override def groupingSets: Seq[Seq[Expression]] = groupingSetIndexes.map(_.map(children))
+  override def selectedGroupByExprs: Seq[Seq[Expression]] = BaseGroupingSets.cubeExprs(groupingSets)
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Cube =
+    copy(children = newChildren)
+}
+
+object Cube {
+  def apply(groupingSets: Seq[Seq[Expression]]): Cube = {
+    Cube(BaseGroupingSets.computeGroupingSetIndexes(groupingSets), groupingSets.flatten)
+  }
+}
+
+case class Rollup(
+    groupingSetIndexes: Seq[Seq[Int]],
+    children: Seq[Expression]) extends BaseGroupingSets {
+  override def groupingSets: Seq[Seq[Expression]] = groupingSetIndexes.map(_.map(children))
+  override def selectedGroupByExprs: Seq[Seq[Expression]] =
+    BaseGroupingSets.rollupExprs(groupingSets)
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Rollup =
+    copy(children = newChildren)
+}
+
+object Rollup {
+  def apply(groupingSets: Seq[Seq[Expression]]): Rollup = {
+    Rollup(BaseGroupingSets.computeGroupingSetIndexes(groupingSets), groupingSets.flatten)
+  }
+}
+
+case class GroupingSets(
+    groupingSetIndexes: Seq[Seq[Int]],
+    flatGroupingSets: Seq[Expression],
+    userGivenGroupByExprs: Seq[Expression]) extends BaseGroupingSets {
+  override def groupingSets: Seq[Seq[Expression]] = groupingSetIndexes.map(_.map(flatGroupingSets))
+  override def selectedGroupByExprs: Seq[Seq[Expression]] = groupingSets
+  // Includes the `userGivenGroupByExprs` in the children, which will be included in the final
+  // GROUP BY expressions, so that `SELECT c ... GROUP BY (a, b, c) GROUPING SETS (a, b)` works.
+  override def children: Seq[Expression] = flatGroupingSets ++ userGivenGroupByExprs
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[Expression]): GroupingSets =
+    super.legacyWithNewChildren(newChildren).asInstanceOf[GroupingSets]
+}
+
+object GroupingSets {
+  def apply(
+      groupingSets: Seq[Seq[Expression]],
+      userGivenGroupByExprs: Seq[Expression]): GroupingSets = {
+    val groupingSetIndexes = BaseGroupingSets.computeGroupingSetIndexes(groupingSets)
+    GroupingSets(groupingSetIndexes, groupingSets.flatten, userGivenGroupByExprs)
+  }
+
+  def apply(groupingSets: Seq[Seq[Expression]]): GroupingSets = {
+    apply(groupingSets, userGivenGroupByExprs = Nil)
+  }
+}
 
 /**
  * Indicates whether a specified column expression in a GROUP BY list is aggregated or not.
@@ -96,15 +185,18 @@ case class Rollup(groupByExprs: Seq[Expression]) extends GroupingSet {}
         Bob	0	5
         NULL	1	7
   """,
-  since = "2.0.0")
+  since = "2.0.0",
+  group = "agg_funcs")
 // scalastyle:on line.size.limit line.contains.tab
-case class Grouping(child: Expression) extends Expression with Unevaluable {
+case class Grouping(child: Expression) extends Expression with Unevaluable
+  with UnaryLike[Expression] {
   @transient
   override lazy val references: AttributeSet =
     AttributeSet(VirtualColumn.groupingIdAttribute :: Nil)
-  override def children: Seq[Expression] = child :: Nil
   override def dataType: DataType = ByteType
   override def nullable: Boolean = false
+  override protected def withNewChildInternal(newChild: Expression): Grouping =
+    copy(child = newChild)
 }
 
 /**
@@ -122,9 +214,9 @@ case class Grouping(child: Expression) extends Expression with Unevaluable {
     Examples:
       > SELECT name, _FUNC_(), sum(age), avg(height) FROM VALUES (2, 'Alice', 165), (5, 'Bob', 180) people(age, name, height) GROUP BY cube(name, height);
         Alice	0	2	165.0
+        Bob	0	5	180.0
         Alice	1	2	165.0
         NULL	3	7	172.5
-        Bob	0	5	180.0
         Bob	1	5	180.0
         NULL	2	2	165.0
         NULL	2	5	180.0
@@ -133,7 +225,8 @@ case class Grouping(child: Expression) extends Expression with Unevaluable {
     Input columns should match with grouping columns exactly, or empty (means all the grouping
     columns).
   """,
-  since = "2.0.0")
+  since = "2.0.0",
+  group = "agg_funcs")
 // scalastyle:on line.size.limit line.contains.tab
 case class GroupingID(groupByExprs: Seq[Expression]) extends Expression with Unevaluable {
   @transient
@@ -143,6 +236,8 @@ case class GroupingID(groupByExprs: Seq[Expression]) extends Expression with Une
   override def dataType: DataType = GroupingID.dataType
   override def nullable: Boolean = false
   override def prettyName: String = "grouping_id"
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): GroupingID =
+    copy(groupByExprs = newChildren)
 }
 
 object GroupingID {
@@ -151,3 +246,34 @@ object GroupingID {
     if (SQLConf.get.integerGroupingIdEnabled) IntegerType else LongType
   }
 }
+
+object GroupingAnalytics {
+  def unapply(exprs: Seq[Expression])
+  : Option[(Seq[Seq[Expression]], Seq[Expression])] = {
+    if (!exprs.exists(_.isInstanceOf[BaseGroupingSets])) {
+      None
+    } else {
+      val resolved = exprs.forall {
+        case gs: BaseGroupingSets => gs.childrenResolved
+        case other => other.resolved
+      }
+      if (!resolved) {
+        None
+      } else {
+        val groups = exprs.flatMap {
+          case gs: BaseGroupingSets => gs.groupByExprs
+          case other: Expression => other :: Nil
+        }
+        val unmergedSelectedGroupByExprs = exprs.map {
+          case gs: BaseGroupingSets => gs.selectedGroupByExprs
+          case other: Expression => Seq(Seq(other))
+        }
+        val selectedGroupByExprs = unmergedSelectedGroupByExprs.tail
+          .foldLeft(unmergedSelectedGroupByExprs.head) { (x, y) =>
+            for (a <- x; b <- y) yield a ++ b
+          }
+        Some(selectedGroupByExprs, BaseGroupingSets.distinctGroupByExprs(groups))
+      }
+    }
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala
index 4c8c58ae232f4..65e7714a3d8ac 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala
@@ -52,8 +52,11 @@ import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
     Examples:
       > SELECT _FUNC_('Spark');
        8cde774d6f7333752ed72cacddb05126
-  """)
-case class Md5(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
+  """,
+  since = "1.5.0",
+  group = "hash_funcs")
+case class Md5(child: Expression)
+  extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def dataType: DataType = StringType
 
@@ -66,6 +69,8 @@ case class Md5(child: Expression) extends UnaryExpression with ImplicitCastInput
     defineCodeGen(ctx, ev, c =>
       s"UTF8String.fromString(${classOf[DigestUtils].getName}.md5Hex($c))")
   }
+
+  override protected def withNewChildInternal(newChild: Expression): Md5 = copy(child = newChild)
 }
 
 /**
@@ -86,10 +91,12 @@ case class Md5(child: Expression) extends UnaryExpression with ImplicitCastInput
     Examples:
       > SELECT _FUNC_('Spark', 256);
        529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b
-  """)
+  """,
+  since = "1.5.0",
+  group = "hash_funcs")
 // scalastyle:on line.size.limit
 case class Sha2(left: Expression, right: Expression)
-  extends BinaryExpression with Serializable with ImplicitCastInputTypes {
+  extends BinaryExpression with ImplicitCastInputTypes with NullIntolerant with Serializable {
 
   override def dataType: DataType = StringType
   override def nullable: Boolean = true
@@ -147,6 +154,9 @@ case class Sha2(left: Expression, right: Expression)
       """
     })
   }
+
+  override protected def withNewChildrenInternal(newLeft: Expression, newRight: Expression): Sha2 =
+    copy(left = newLeft, right = newRight)
 }
 
 /**
@@ -159,8 +169,11 @@ case class Sha2(left: Expression, right: Expression)
     Examples:
       > SELECT _FUNC_('Spark');
        85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c
-  """)
-case class Sha1(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
+  """,
+  since = "1.5.0",
+  group = "hash_funcs")
+case class Sha1(child: Expression)
+  extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def dataType: DataType = StringType
 
@@ -174,6 +187,8 @@ case class Sha1(child: Expression) extends UnaryExpression with ImplicitCastInpu
       s"UTF8String.fromString(${classOf[DigestUtils].getName}.sha1Hex($c))"
     )
   }
+
+  override protected def withNewChildInternal(newChild: Expression): Sha1 = copy(child = newChild)
 }
 
 /**
@@ -186,8 +201,11 @@ case class Sha1(child: Expression) extends UnaryExpression with ImplicitCastInpu
     Examples:
       > SELECT _FUNC_('Spark');
        1557323817
-  """)
-case class Crc32(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
+  """,
+  since = "1.5.0",
+  group = "hash_funcs")
+case class Crc32(child: Expression)
+  extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def dataType: DataType = LongType
 
@@ -210,6 +228,8 @@ case class Crc32(child: Expression) extends UnaryExpression with ImplicitCastInp
       """
     })
   }
+
+  override protected def withNewChildInternal(newChild: Expression): Crc32 = copy(child = newChild)
 }
 
 
@@ -218,24 +238,30 @@ case class Crc32(child: Expression) extends UnaryExpression with ImplicitCastInp
  * is not exposed to users and should only be set inside spark SQL.
  *
  * The hash value for an expression depends on its type and seed:
- *  - null:               seed
- *  - boolean:            turn boolean into int, 1 for true, 0 for false, and then use murmur3 to
- *                        hash this int with seed.
- *  - byte, short, int:   use murmur3 to hash the input as int with seed.
- *  - long:               use murmur3 to hash the long input with seed.
- *  - float:              turn it into int: java.lang.Float.floatToIntBits(input), and hash it.
- *  - double:             turn it into long: java.lang.Double.doubleToLongBits(input), and hash it.
- *  - decimal:            if it's a small decimal, i.e. precision <= 18, turn it into long and hash
- *                        it. Else, turn it into bytes and hash it.
- *  - calendar interval:  hash `microseconds` first, and use the result as seed to hash `months`.
- *  - binary:             use murmur3 to hash the bytes with seed.
- *  - string:             get the bytes of string and hash it.
- *  - array:              The `result` starts with seed, then use `result` as seed, recursively
- *                        calculate hash value for each element, and assign the element hash value
- *                        to `result`.
- *  - struct:             The `result` starts with seed, then use `result` as seed, recursively
- *                        calculate hash value for each field, and assign the field hash value to
- *                        `result`.
+ *  - null:                    seed
+ *  - boolean:                 turn boolean into int, 1 for true, 0 for false,
+ *                             and then use murmur3 to hash this int with seed.
+ *  - byte, short, int:        use murmur3 to hash the input as int with seed.
+ *  - long:                    use murmur3 to hash the long input with seed.
+ *  - float:                   turn it into int: java.lang.Float.floatToIntBits(input), and hash it.
+ *  - double:                  turn it into long: java.lang.Double.doubleToLongBits(input),
+ *                             and hash it.
+ *  - decimal:                 if it's a small decimal, i.e. precision <= 18, turn it into long
+ *                             and hash it. Else, turn it into bytes and hash it.
+ *  - calendar interval:       hash `microseconds` first, and use the result as seed
+ *                             to hash `months`.
+ *  - interval day to second:  it store long value of `microseconds`, use murmur3 to hash the long
+ *                             input with seed.
+ *  - interval year to month:  it store int value of `months`, use murmur3 to hash the int
+ *                             input with seed.
+ *  - binary:                  use murmur3 to hash the bytes with seed.
+ *  - string:                  get the bytes of string and hash it.
+ *  - array:                   The `result` starts with seed, then use `result` as seed, recursively
+ *                             calculate hash value for each element, and assign the element hash
+ *                             value to `result`.
+ *  - struct:                  The `result` starts with seed, then use `result` as seed, recursively
+ *                             calculate hash value for each field, and assign the field hash value
+ *                             to `result`.
  *
  * Finally we aggregate the hash values for each expression by the same way of struct.
  */
@@ -343,11 +369,25 @@ abstract class HashExpression[E] extends Expression {
   protected def genHashBoolean(input: String, result: String): String =
     genHashInt(s"$input ? 1 : 0", result)
 
-  protected def genHashFloat(input: String, result: String): String =
-    genHashInt(s"Float.floatToIntBits($input)", result)
+  protected def genHashFloat(input: String, result: String): String = {
+    s"""
+       |if($input == -0.0f) {
+       |  ${genHashInt("0", result)}
+       |} else {
+       |  ${genHashInt(s"Float.floatToIntBits($input)", result)}
+       |}
+     """.stripMargin
+  }
 
-  protected def genHashDouble(input: String, result: String): String =
-    genHashLong(s"Double.doubleToLongBits($input)", result)
+  protected def genHashDouble(input: String, result: String): String = {
+    s"""
+      |if($input == -0.0d) {
+      |  ${genHashLong("0L", result)}
+      |} else {
+      |  ${genHashLong(s"Double.doubleToLongBits($input)", result)}
+      |}
+     """.stripMargin
+  }
 
   protected def genHashDecimal(
       ctx: CodegenContext,
@@ -455,6 +495,8 @@ abstract class HashExpression[E] extends Expression {
     case DoubleType => genHashDouble(input, result)
     case d: DecimalType => genHashDecimal(ctx, d, input, result)
     case CalendarIntervalType => genHashCalendarInterval(input, result)
+    case DayTimeIntervalType => genHashLong(input, result)
+    case YearMonthIntervalType => genHashInt(input, result)
     case BinaryType => genHashBytes(input, result)
     case StringType => genHashString(input, result)
     case ArrayType(et, containsNull) => genHashForArray(ctx, input, result, et, containsNull)
@@ -495,7 +537,9 @@ abstract class InterpretedHashFunction {
       case s: Short => hashInt(s, seed)
       case i: Int => hashInt(i, seed)
       case l: Long => hashLong(l, seed)
+      case f: Float if (f == -0.0f) => hashInt(0, seed)
       case f: Float => hashInt(java.lang.Float.floatToIntBits(f), seed)
+      case d: Double if (d == -0.0d) => hashLong(0L, seed)
       case d: Double => hashLong(java.lang.Double.doubleToLongBits(d), seed)
       case d: Decimal =>
         val precision = dataType.asInstanceOf[DecimalType].precision
@@ -572,7 +616,9 @@ abstract class InterpretedHashFunction {
     Examples:
       > SELECT _FUNC_('Spark', array(123), 2);
        -1321691492
-  """)
+  """,
+  since = "2.0.0",
+  group = "hash_funcs")
 case class Murmur3Hash(children: Seq[Expression], seed: Int) extends HashExpression[Int] {
   def this(arguments: Seq[Expression]) = this(arguments, 42)
 
@@ -585,6 +631,9 @@ case class Murmur3Hash(children: Seq[Expression], seed: Int) extends HashExpress
   override protected def computeHash(value: Any, dataType: DataType, seed: Int): Int = {
     Murmur3HashFunction.hash(value, dataType, seed).toInt
   }
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Murmur3Hash =
+    copy(children = newChildren)
 }
 
 object Murmur3HashFunction extends InterpretedHashFunction {
@@ -611,7 +660,8 @@ object Murmur3HashFunction extends InterpretedHashFunction {
       > SELECT _FUNC_('Spark', array(123), 2);
        5602566077635097486
   """,
-  since = "3.0.0")
+  since = "3.0.0",
+  group = "hash_funcs")
 case class XxHash64(children: Seq[Expression], seed: Long) extends HashExpression[Long] {
   def this(arguments: Seq[Expression]) = this(arguments, 42L)
 
@@ -624,6 +674,9 @@ case class XxHash64(children: Seq[Expression], seed: Long) extends HashExpressio
   override protected def computeHash(value: Any, dataType: DataType, seed: Long): Long = {
     XxHash64Function.hash(value, dataType, seed)
   }
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): XxHash64 =
+    copy(children = newChildren)
 }
 
 object XxHash64Function extends InterpretedHashFunction {
@@ -644,7 +697,9 @@ object XxHash64Function extends InterpretedHashFunction {
  * we can guarantee shuffle and bucketing have same data distribution
  */
 @ExpressionDescription(
-  usage = "_FUNC_(expr1, expr2, ...) - Returns a hash value of the arguments.")
+  usage = "_FUNC_(expr1, expr2, ...) - Returns a hash value of the arguments.",
+  since = "2.2.0",
+  group = "hash_funcs")
 case class HiveHash(children: Seq[Expression]) extends HashExpression[Int] {
   override val seed = 0
 
@@ -826,6 +881,9 @@ case class HiveHash(children: Seq[Expression]) extends HashExpression[Int] {
        |$code
      """.stripMargin
   }
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): HiveHash =
+    copy(children = newChildren)
 }
 
 object HiveHashFunction extends InterpretedHashFunction {
@@ -898,7 +956,7 @@ object HiveHashFunction extends InterpretedHashFunction {
    * - year, month (stored as HiveIntervalYearMonth)
    * - day, hour, minute, second, nanosecond (stored as HiveIntervalDayTime)
    *
-   * eg. (INTERVAL '30' YEAR + INTERVAL '-23' DAY) fails in Hive
+   * e.g. (INTERVAL '30' YEAR + INTERVAL '-23' DAY) fails in Hive
    *
    * This method mimics HiveIntervalDayTime.hashCode() in Hive.
    *
@@ -910,7 +968,7 @@ object HiveHashFunction extends InterpretedHashFunction {
    *
    * - Spark's [[CalendarInterval]] has precision upto microseconds but Hive's
    *   HiveIntervalDayTime can store data with precision upto nanoseconds. So, any input intervals
-   *   with nanosecond values will lead to wrong output hashes (ie. non adherent with Hive output)
+   *   with nanosecond values will lead to wrong output hashes (i.e. non adherent with Hive output)
    */
   def hashCalendarInterval(calendarInterval: CalendarInterval): Long = {
     val totalMicroSeconds = calendarInterval.days * MICROS_PER_DAY + calendarInterval.microseconds
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
index e91bd0c7015ee..a6feb02f4d1a0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
@@ -18,14 +18,16 @@
 package org.apache.spark.sql.catalyst.expressions
 
 import java.util.Comparator
-import java.util.concurrent.atomic.AtomicReference
+import java.util.concurrent.atomic.{AtomicInteger, AtomicReference}
 
 import scala.collection.mutable
 
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, TypeCoercion, UnresolvedAttribute, UnresolvedException}
+import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, TypeCoercion, UnresolvedException}
 import org.apache.spark.sql.catalyst.expressions.codegen._
+import org.apache.spark.sql.catalyst.trees.{BinaryLike, QuaternaryLike, TernaryLike}
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.array.ByteArrayMethods
@@ -39,12 +41,12 @@ case class UnresolvedNamedLambdaVariable(nameParts: Seq[String])
   override def name: String =
     nameParts.map(n => if (n.contains(".")) s"`$n`" else n).mkString(".")
 
-  override def exprId: ExprId = throw new UnresolvedException(this, "exprId")
-  override def dataType: DataType = throw new UnresolvedException(this, "dataType")
-  override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
-  override def qualifier: Seq[String] = throw new UnresolvedException(this, "qualifier")
-  override def toAttribute: Attribute = throw new UnresolvedException(this, "toAttribute")
-  override def newInstance(): NamedExpression = throw new UnresolvedException(this, "newInstance")
+  override def exprId: ExprId = throw new UnresolvedException("exprId")
+  override def dataType: DataType = throw new UnresolvedException("dataType")
+  override def nullable: Boolean = throw new UnresolvedException("nullable")
+  override def qualifier: Seq[String] = throw new UnresolvedException("qualifier")
+  override def toAttribute: Attribute = throw new UnresolvedException("toAttribute")
+  override def newInstance(): NamedExpression = throw new UnresolvedException("newInstance")
   override lazy val resolved = false
 
   override def toString: String = s"lambda '$name"
@@ -52,6 +54,16 @@ case class UnresolvedNamedLambdaVariable(nameParts: Seq[String])
   override def sql: String = name
 }
 
+object UnresolvedNamedLambdaVariable {
+
+  // Counter to ensure lambda variable names are unique
+  private val nextVarNameId = new AtomicInteger(0)
+
+  def freshVarName(name: String): String = {
+    s"${name}_${nextVarNameId.getAndIncrement()}"
+  }
+}
+
 /**
  * A named lambda variable.
  */
@@ -101,6 +113,12 @@ case class LambdaFunction(
   lazy val bound: Boolean = arguments.forall(_.resolved)
 
   override def eval(input: InternalRow): Any = function.eval(input)
+
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[Expression]): LambdaFunction =
+    copy(
+      function = newChildren.head,
+      arguments = newChildren.tail.asInstanceOf[Seq[NamedExpression]])
 }
 
 object LambdaFunction {
@@ -118,8 +136,6 @@ trait HigherOrderFunction extends Expression with ExpectsInputTypes {
 
   override def nullable: Boolean = arguments.exists(_.nullable)
 
-  override def children: Seq[Expression] = arguments ++ functions
-
   /**
    * Arguments of the higher ordered function.
    */
@@ -128,7 +144,7 @@ trait HigherOrderFunction extends Expression with ExpectsInputTypes {
   def argumentTypes: Seq[AbstractDataType]
 
   /**
-   * All arguments have been resolved. This means that the types and nullabilty of (most of) the
+   * All arguments have been resolved. This means that the types and nullability of (most of) the
    * lambda function arguments is known, and that we can start binding the lambda functions.
    */
   lazy val argumentsResolved: Boolean = arguments.forall(_.resolved)
@@ -181,7 +197,7 @@ trait HigherOrderFunction extends Expression with ExpectsInputTypes {
 /**
  * Trait for functions having as input one argument and one function.
  */
-trait SimpleHigherOrderFunction extends HigherOrderFunction  {
+trait SimpleHigherOrderFunction extends HigherOrderFunction with BinaryLike[Expression] {
 
   def argument: Expression
 
@@ -201,12 +217,16 @@ trait SimpleHigherOrderFunction extends HigherOrderFunction  {
 
   def functionForEval: Expression = functionsForEval.head
 
+  override def left: Expression = argument
+  override def right: Expression = function
+
   /**
    * Called by [[eval]]. If a subclass keeps the default nullability, it can override this method
    * in order to save null-check code.
    */
   protected def nullSafeEval(inputRow: InternalRow, argumentValue: Any): Any =
-    sys.error(s"UnaryHigherOrderFunction must override either eval or nullSafeEval")
+    throw QueryExecutionErrors.notOverrideExpectedMethodsError("UnaryHigherOrderFunction",
+      "eval", "nullSafeEval")
 
   override def eval(inputRow: InternalRow): Any = {
     val value = argument.eval(inputRow)
@@ -216,6 +236,7 @@ trait SimpleHigherOrderFunction extends HigherOrderFunction  {
       nullSafeEval(inputRow, value)
     }
   }
+
 }
 
 trait ArrayBasedSimpleHigherOrderFunction extends SimpleHigherOrderFunction {
@@ -239,7 +260,8 @@ trait MapBasedSimpleHigherOrderFunction extends SimpleHigherOrderFunction {
       > SELECT _FUNC_(array(1, 2, 3), (x, i) -> x + i);
        [1,3,5]
   """,
-  since = "2.4.0")
+  since = "2.4.0",
+  group = "lambda_funcs")
 case class ArrayTransform(
     argument: Expression,
     function: Expression)
@@ -277,13 +299,18 @@ case class ArrayTransform(
       if (indexVar.isDefined) {
         indexVar.get.value.set(i)
       }
-      result.update(i, f.eval(inputRow))
+      val v = InternalRow.copyValue(f.eval(inputRow))
+      result.update(i, v)
       i += 1
     }
     result
   }
 
   override def prettyName: String = "transform"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): ArrayTransform =
+    copy(argument = newLeft, function = newRight)
 }
 
 /**
@@ -309,7 +336,8 @@ case class ArrayTransform(
       > SELECT _FUNC_(array('b', 'd', null, 'c', 'a'));
        ["a","b","c","d",null]
   """,
-  since = "2.4.0")
+  since = "2.4.0",
+  group = "lambda_funcs")
 // scalastyle:on line.size.limit
 case class ArraySort(
     argument: Expression,
@@ -372,6 +400,10 @@ case class ArraySort(
   }
 
   override def prettyName: String = "array_sort"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): ArraySort =
+    copy(argument = newLeft, function = newRight)
 }
 
 object ArraySort {
@@ -403,7 +435,8 @@ object ArraySort {
       > SELECT _FUNC_(map(1, 0, 2, 2, 3, -1), (k, v) -> k > v);
        {1:0,3:-1}
   """,
-  since = "3.0.0")
+  since = "3.0.0",
+  group = "lambda_funcs")
 case class MapFilter(
     argument: Expression,
     function: Expression)
@@ -441,6 +474,10 @@ case class MapFilter(
   override def functionType: AbstractDataType = BooleanType
 
   override def prettyName: String = "map_filter"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): MapFilter =
+    copy(argument = newLeft, function = newRight)
 }
 
 /**
@@ -454,8 +491,11 @@ case class MapFilter(
        [1,3]
       > SELECT _FUNC_(array(0, 2, 3), (x, i) -> x > i);
        [2,3]
+      > SELECT _FUNC_(array(0, null, 2, 3, null), x -> x IS NOT NULL);
+       [0,2,3]
   """,
   since = "2.4.0",
+  group = "lambda_funcs",
   note = """
     The inner function may use the index argument since 3.0.0.
   """)
@@ -499,10 +539,14 @@ case class ArrayFilter(
       }
       i += 1
     }
-    new GenericArrayData(buffer)
+    new GenericArrayData(buffer.toSeq)
   }
 
   override def prettyName: String = "filter"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): ArrayFilter =
+    copy(argument = newLeft, function = newRight)
 }
 
 /**
@@ -518,8 +562,13 @@ case class ArrayFilter(
        false
       > SELECT _FUNC_(array(1, null, 3), x -> x % 2 == 0);
        NULL
+      > SELECT _FUNC_(array(0, null, 2, 3, null), x -> x IS NULL);
+       true
+      > SELECT _FUNC_(array(1, 2, 3), x -> x IS NULL);
+       false
   """,
-  since = "2.4.0")
+  since = "2.4.0",
+  group = "lambda_funcs")
 case class ArrayExists(
     argument: Expression,
     function: Expression,
@@ -579,6 +628,10 @@ case class ArrayExists(
   }
 
   override def prettyName: String = "exists"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): ArrayExists =
+    copy(argument = newLeft, function = newRight)
 }
 
 object ArrayExists {
@@ -603,7 +656,8 @@ object ArrayExists {
       > SELECT _FUNC_(array(2, null, 8), x -> x % 2 == 0);
        NULL
   """,
-  since = "3.0.0")
+  since = "3.0.0",
+  group = "lambda_funcs")
 case class ArrayForAll(
     argument: Expression,
     function: Expression)
@@ -654,6 +708,10 @@ case class ArrayForAll(
   }
 
   override def prettyName: String = "forall"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): ArrayForAll =
+    copy(argument = newLeft, function = newRight)
 }
 
 /**
@@ -673,13 +731,14 @@ case class ArrayForAll(
       > SELECT _FUNC_(array(1, 2, 3), 0, (acc, x) -> acc + x, acc -> acc * 10);
        60
   """,
-  since = "2.4.0")
+  since = "2.4.0",
+  group = "lambda_funcs")
 case class ArrayAggregate(
     argument: Expression,
     zero: Expression,
     merge: Expression,
     finish: Expression)
-  extends HigherOrderFunction with CodegenFallback {
+  extends HigherOrderFunction with CodegenFallback with QuaternaryLike[Expression] {
 
   def this(argument: Expression, zero: Expression, merge: Expression) = {
     this(argument, zero, merge, LambdaFunction.identity)
@@ -745,6 +804,15 @@ case class ArrayAggregate(
   }
 
   override def prettyName: String = "aggregate"
+
+  override def first: Expression = argument
+  override def second: Expression = zero
+  override def third: Expression = merge
+  override def fourth: Expression = finish
+
+  override protected def withNewChildrenInternal(first: Expression, second: Expression,
+      third: Expression, fourth: Expression): ArrayAggregate =
+    copy(argument = first, zero = second, merge = third, finish = fourth)
 }
 
 /**
@@ -760,7 +828,8 @@ case class ArrayAggregate(
       > SELECT _FUNC_(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + v);
        {2:1,4:2,6:3}
   """,
-  since = "3.0.0")
+  since = "3.0.0",
+  group = "lambda_funcs")
 case class TransformKeys(
     argument: Expression,
     function: Expression)
@@ -779,7 +848,7 @@ case class TransformKeys(
   }
 
   @transient lazy val LambdaFunction(
-    _, (keyVar: NamedLambdaVariable) :: (valueVar: NamedLambdaVariable) :: Nil, _) = function
+    _, Seq(keyVar: NamedLambdaVariable, valueVar: NamedLambdaVariable), _) = function
 
   private lazy val mapBuilder = new ArrayBasedMapBuilder(dataType.keyType, dataType.valueType)
 
@@ -790,7 +859,7 @@ case class TransformKeys(
     while (i < map.numElements) {
       keyVar.value.set(map.keyArray().get(i, keyVar.dataType))
       valueVar.value.set(map.valueArray().get(i, valueVar.dataType))
-      val result = functionForEval.eval(inputRow)
+      val result = InternalRow.copyValue(functionForEval.eval(inputRow))
       resultKeys.update(i, result)
       i += 1
     }
@@ -798,6 +867,10 @@ case class TransformKeys(
   }
 
   override def prettyName: String = "transform_keys"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): TransformKeys =
+    copy(argument = newLeft, function = newRight)
 }
 
 /**
@@ -812,7 +885,8 @@ case class TransformKeys(
       > SELECT _FUNC_(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + v);
        {1:2,2:4,3:6}
   """,
-  since = "3.0.0")
+  since = "3.0.0",
+  group = "lambda_funcs")
 case class TransformValues(
     argument: Expression,
     function: Expression)
@@ -828,7 +902,7 @@ case class TransformValues(
   }
 
   @transient lazy val LambdaFunction(
-    _, (keyVar: NamedLambdaVariable) :: (valueVar: NamedLambdaVariable) :: Nil, _) = function
+    _, Seq(keyVar: NamedLambdaVariable, valueVar: NamedLambdaVariable), _) = function
 
   override def nullSafeEval(inputRow: InternalRow, argumentValue: Any): Any = {
     val map = argumentValue.asInstanceOf[MapData]
@@ -837,13 +911,18 @@ case class TransformValues(
     while (i < map.numElements) {
       keyVar.value.set(map.keyArray().get(i, keyVar.dataType))
       valueVar.value.set(map.valueArray().get(i, valueVar.dataType))
-      resultValues.update(i, functionForEval.eval(inputRow))
+      val v = InternalRow.copyValue(functionForEval.eval(inputRow))
+      resultValues.update(i, v)
       i += 1
     }
     new ArrayBasedMapData(map.keyArray(), resultValues)
   }
 
   override def prettyName: String = "transform_values"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): TransformValues =
+    copy(argument = newLeft, function = newRight)
 }
 
 /**
@@ -863,9 +942,10 @@ case class TransformValues(
       > SELECT _FUNC_(map(1, 'a', 2, 'b'), map(1, 'x', 2, 'y'), (k, v1, v2) -> concat(v1, v2));
        {1:"ax",2:"by"}
   """,
-  since = "3.0.0")
+  since = "3.0.0",
+  group = "lambda_funcs")
 case class MapZipWith(left: Expression, right: Expression, function: Expression)
-  extends HigherOrderFunction with CodegenFallback {
+  extends HigherOrderFunction with CodegenFallback with TernaryLike[Expression] {
 
   def functionForEval: Expression = functionsForEval.head
 
@@ -945,9 +1025,7 @@ case class MapZipWith(left: Expression, right: Expression, function: Expression)
 
   private def assertSizeOfArrayBuffer(size: Int): Unit = {
     if (size > ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH) {
-      throw new RuntimeException(s"Unsuccessful try to zip maps with $size " +
-        s"unique keys due to exceeding the array size limit " +
-        s"${ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH}.")
+      throw QueryExecutionErrors.mapSizeExceedArraySizeWhenZipMapError(size)
     }
   }
 
@@ -1020,13 +1098,25 @@ case class MapZipWith(left: Expression, right: Expression, function: Expression)
       value1Var.value.set(v1)
       value2Var.value.set(v2)
       keys.update(i, key)
-      values.update(i, functionForEval.eval(inputRow))
+      val v = InternalRow.copyValue(functionForEval.eval(inputRow))
+      values.update(i, v)
       i += 1
     }
     new ArrayBasedMapData(keys, values)
   }
 
   override def prettyName: String = "map_zip_with"
+
+  override def first: Expression = left
+  override def second: Expression = right
+  override def third: Expression = function
+
+  override protected def withNewChildrenInternal(
+      newFirst: Expression, newSecond: Expression, newThird: Expression): MapZipWith =
+    copy(
+      left = newFirst,
+      right = newSecond,
+      function = newThird)
 }
 
 // scalastyle:off line.size.limit
@@ -1041,10 +1131,11 @@ case class MapZipWith(left: Expression, right: Expression, function: Expression)
       > SELECT _FUNC_(array('a', 'b', 'c'), array('d', 'e', 'f'), (x, y) -> concat(x, y));
        ["ad","be","cf"]
   """,
-  since = "2.4.0")
+  since = "2.4.0",
+  group = "lambda_funcs")
 // scalastyle:on line.size.limit
 case class ZipWith(left: Expression, right: Expression, function: Expression)
-  extends HigherOrderFunction with CodegenFallback {
+  extends HigherOrderFunction with CodegenFallback with TernaryLike[Expression] {
 
   def functionForEval: Expression = functionsForEval.head
 
@@ -1052,7 +1143,7 @@ case class ZipWith(left: Expression, right: Expression, function: Expression)
 
   override def argumentTypes: Seq[AbstractDataType] = ArrayType :: ArrayType :: Nil
 
-  override def functions: Seq[Expression] = List(function)
+  override def functions: Seq[Expression] = function :: Nil
 
   override def functionTypes: Seq[AbstractDataType] = AnyDataType :: Nil
 
@@ -1092,7 +1183,8 @@ case class ZipWith(left: Expression, right: Expression, function: Expression)
           } else {
             rightElemVar.value.set(null)
           }
-          result.update(i, f.eval(input))
+          val v = InternalRow.copyValue(f.eval(input))
+          result.update(i, v)
           i += 1
         }
         result
@@ -1101,4 +1193,12 @@ case class ZipWith(left: Expression, right: Expression, function: Expression)
   }
 
   override def prettyName: String = "zip_with"
+
+  override def first: Expression = left
+  override def second: Expression = right
+  override def third: Expression = function
+
+  override protected def withNewChildrenInternal(
+      newFirst: Expression, newSecond: Expression, newThird: Expression): ZipWith =
+    copy(left = newFirst, right = newSecond, function = newThird)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/inputFileBlock.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/inputFileBlock.scala
index 3b0141ad52cc7..6cd88367aa9a0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/inputFileBlock.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/inputFileBlock.scala
@@ -24,9 +24,17 @@ import org.apache.spark.sql.catalyst.expressions.codegen.Block._
 import org.apache.spark.sql.types.{DataType, LongType, StringType}
 import org.apache.spark.unsafe.types.UTF8String
 
-
+// scalastyle:off whitespace.end.of.line
 @ExpressionDescription(
-  usage = "_FUNC_() - Returns the name of the file being read, or empty string if not available.")
+  usage = "_FUNC_() - Returns the name of the file being read, or empty string if not available.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_();
+
+  """,
+  since = "1.5.0",
+  group = "misc_funcs")
+// scalastyle:on whitespace.end.of.line
 case class InputFileName() extends LeafExpression with Nondeterministic {
 
   override def nullable: Boolean = false
@@ -51,7 +59,14 @@ case class InputFileName() extends LeafExpression with Nondeterministic {
 
 
 @ExpressionDescription(
-  usage = "_FUNC_() - Returns the start offset of the block being read, or -1 if not available.")
+  usage = "_FUNC_() - Returns the start offset of the block being read, or -1 if not available.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_();
+       -1
+  """,
+  since = "2.2.0",
+  group = "misc_funcs")
 case class InputFileBlockStart() extends LeafExpression with Nondeterministic {
   override def nullable: Boolean = false
 
@@ -74,7 +89,14 @@ case class InputFileBlockStart() extends LeafExpression with Nondeterministic {
 
 
 @ExpressionDescription(
-  usage = "_FUNC_() - Returns the length of the block being read, or -1 if not available.")
+  usage = "_FUNC_() - Returns the length of the block being read, or -1 if not available.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_();
+       -1
+  """,
+  since = "2.2.0",
+  group = "misc_funcs")
 case class InputFileBlockLength() extends LeafExpression with Nondeterministic {
   override def nullable: Boolean = false
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala
index 1a569a7b89fe1..94ca6cc65d856 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala
@@ -17,72 +17,146 @@
 
 package org.apache.spark.sql.catalyst.expressions
 
+import java.math.RoundingMode
 import java.util.Locale
 
-import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
+import com.google.common.math.{DoubleMath, IntMath, LongMath}
+
+import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodeGenerator, ExprCode}
 import org.apache.spark.sql.catalyst.util.IntervalUtils
 import org.apache.spark.sql.catalyst.util.IntervalUtils._
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.CalendarInterval
 
-abstract class ExtractIntervalPart(
-    child: Expression,
+abstract class ExtractIntervalPart[T](
     val dataType: DataType,
-    func: CalendarInterval => Any,
-    funcName: String)
-  extends UnaryExpression with ExpectsInputTypes with Serializable {
-
-  override def inputTypes: Seq[AbstractDataType] = Seq(CalendarIntervalType)
-
-  override protected def nullSafeEval(interval: Any): Any = {
-    func(interval.asInstanceOf[CalendarInterval])
-  }
-
+    func: T => Any,
+    funcName: String) extends UnaryExpression with NullIntolerant with Serializable {
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     val iu = IntervalUtils.getClass.getName.stripSuffix("$")
     defineCodeGen(ctx, ev, c => s"$iu.$funcName($c)")
   }
+
+  override protected def nullSafeEval(interval: Any): Any = {
+    func(interval.asInstanceOf[T])
+  }
 }
 
 case class ExtractIntervalYears(child: Expression)
-  extends ExtractIntervalPart(child, IntegerType, getYears, "getYears")
+  extends ExtractIntervalPart[CalendarInterval](IntegerType, getYears, "getYears") {
+  override protected def withNewChildInternal(newChild: Expression): ExtractIntervalYears =
+    copy(child = newChild)
+}
 
 case class ExtractIntervalMonths(child: Expression)
-  extends ExtractIntervalPart(child, ByteType, getMonths, "getMonths")
+  extends ExtractIntervalPart[CalendarInterval](ByteType, getMonths, "getMonths") {
+  override protected def withNewChildInternal(newChild: Expression): ExtractIntervalMonths =
+    copy(child = newChild)
+}
 
 case class ExtractIntervalDays(child: Expression)
-  extends ExtractIntervalPart(child, IntegerType, getDays, "getDays")
+  extends ExtractIntervalPart[CalendarInterval](IntegerType, getDays, "getDays") {
+  override protected def withNewChildInternal(newChild: Expression): ExtractIntervalDays =
+    copy(child = newChild)
+}
 
 case class ExtractIntervalHours(child: Expression)
-  extends ExtractIntervalPart(child, LongType, getHours, "getHours")
+  extends ExtractIntervalPart[CalendarInterval](ByteType, getHours, "getHours") {
+  override protected def withNewChildInternal(newChild: Expression): ExtractIntervalHours =
+    copy(child = newChild)
+}
 
 case class ExtractIntervalMinutes(child: Expression)
-  extends ExtractIntervalPart(child, ByteType, getMinutes, "getMinutes")
+  extends ExtractIntervalPart[CalendarInterval](ByteType, getMinutes, "getMinutes") {
+  override protected def withNewChildInternal(newChild: Expression): ExtractIntervalMinutes =
+    copy(child = newChild)
+}
 
 case class ExtractIntervalSeconds(child: Expression)
-  extends ExtractIntervalPart(child, DecimalType(8, 6), getSeconds, "getSeconds")
+  extends ExtractIntervalPart[CalendarInterval](DecimalType(8, 6), getSeconds, "getSeconds") {
+  override protected def withNewChildInternal(newChild: Expression): ExtractIntervalSeconds =
+    copy(child = newChild)
+}
+
+case class ExtractANSIIntervalYears(child: Expression)
+    extends ExtractIntervalPart[Int](IntegerType, getYears, "getYears") {
+  override protected def withNewChildInternal(newChild: Expression): ExtractANSIIntervalYears =
+    copy(child = newChild)
+}
+
+case class ExtractANSIIntervalMonths(child: Expression)
+    extends ExtractIntervalPart[Int](ByteType, getMonths, "getMonths") {
+  override protected def withNewChildInternal(newChild: Expression): ExtractANSIIntervalMonths =
+    copy(child = newChild)
+}
+
+case class ExtractANSIIntervalDays(child: Expression)
+    extends ExtractIntervalPart[Long](IntegerType, getDays, "getDays") {
+  override protected def withNewChildInternal(newChild: Expression): ExtractANSIIntervalDays = {
+    copy(child = newChild)
+  }
+}
+
+case class ExtractANSIIntervalHours(child: Expression)
+    extends ExtractIntervalPart[Long](ByteType, getHours, "getHours") {
+  override protected def withNewChildInternal(newChild: Expression): ExtractANSIIntervalHours =
+    copy(child = newChild)
+}
+
+case class ExtractANSIIntervalMinutes(child: Expression)
+    extends ExtractIntervalPart[Long](ByteType, getMinutes, "getMinutes") {
+  override protected def withNewChildInternal(newChild: Expression): ExtractANSIIntervalMinutes =
+    copy(child = newChild)
+}
+
+case class ExtractANSIIntervalSeconds(child: Expression)
+    extends ExtractIntervalPart[Long](DecimalType(8, 6), getSeconds, "getSeconds") {
+  override protected def withNewChildInternal(newChild: Expression): ExtractANSIIntervalSeconds =
+    copy(child = newChild)
+}
 
 object ExtractIntervalPart {
 
   def parseExtractField(
       extractField: String,
       source: Expression,
-      errorHandleFunc: => Nothing): Expression = extractField.toUpperCase(Locale.ROOT) match {
-    case "YEAR" | "Y" | "YEARS" | "YR" | "YRS" => ExtractIntervalYears(source)
-    case "MONTH" | "MON" | "MONS" | "MONTHS" => ExtractIntervalMonths(source)
-    case "DAY" | "D" | "DAYS" => ExtractIntervalDays(source)
-    case "HOUR" | "H" | "HOURS" | "HR" | "HRS" => ExtractIntervalHours(source)
-    case "MINUTE" | "M" | "MIN" | "MINS" | "MINUTES" => ExtractIntervalMinutes(source)
-    case "SECOND" | "S" | "SEC" | "SECONDS" | "SECS" => ExtractIntervalSeconds(source)
-    case _ => errorHandleFunc
+      errorHandleFunc: => Nothing): Expression = {
+    (extractField.toUpperCase(Locale.ROOT), source.dataType) match {
+      case ("YEAR" | "Y" | "YEARS" | "YR" | "YRS", YearMonthIntervalType) =>
+        ExtractANSIIntervalYears(source)
+      case ("YEAR" | "Y" | "YEARS" | "YR" | "YRS", CalendarIntervalType) =>
+        ExtractIntervalYears(source)
+      case ("MONTH" | "MON" | "MONS" | "MONTHS", YearMonthIntervalType) =>
+        ExtractANSIIntervalMonths(source)
+      case ("MONTH" | "MON" | "MONS" | "MONTHS", CalendarIntervalType) =>
+        ExtractIntervalMonths(source)
+      case ("DAY" | "D" | "DAYS", DayTimeIntervalType) =>
+        ExtractANSIIntervalDays(source)
+      case ("DAY" | "D" | "DAYS", CalendarIntervalType) =>
+        ExtractIntervalDays(source)
+      case ("HOUR" | "H" | "HOURS" | "HR" | "HRS", DayTimeIntervalType) =>
+        ExtractANSIIntervalHours(source)
+      case ("HOUR" | "H" | "HOURS" | "HR" | "HRS", CalendarIntervalType) =>
+        ExtractIntervalHours(source)
+      case ("MINUTE" | "M" | "MIN" | "MINS" | "MINUTES", DayTimeIntervalType) =>
+        ExtractANSIIntervalMinutes(source)
+      case ("MINUTE" | "M" | "MIN" | "MINS" | "MINUTES", CalendarIntervalType) =>
+        ExtractIntervalMinutes(source)
+      case ("SECOND" | "S" | "SEC" | "SECONDS" | "SECS", DayTimeIntervalType) =>
+        ExtractANSIIntervalSeconds(source)
+      case ("SECOND" | "S" | "SEC" | "SECONDS" | "SECS", CalendarIntervalType) =>
+        ExtractIntervalSeconds(source)
+      case _ => errorHandleFunc
+    }
   }
 }
 
 abstract class IntervalNumOperation(
     interval: Expression,
     num: Expression)
-  extends BinaryExpression with ImplicitCastInputTypes with Serializable {
+  extends BinaryExpression with ImplicitCastInputTypes with NullIntolerant with Serializable {
   override def left: Expression = interval
   override def right: Expression = num
 
@@ -109,25 +183,33 @@ abstract class IntervalNumOperation(
 case class MultiplyInterval(
     interval: Expression,
     num: Expression,
-    checkOverflow: Boolean = SQLConf.get.ansiEnabled)
+    failOnError: Boolean = SQLConf.get.ansiEnabled)
   extends IntervalNumOperation(interval, num) {
 
   override protected val operation: (CalendarInterval, Double) => CalendarInterval =
-    if (checkOverflow) multiplyExact else multiply
+    if (failOnError) multiplyExact else multiply
+
+  override protected def operationName: String = if (failOnError) "multiplyExact" else "multiply"
 
-  override protected def operationName: String = if (checkOverflow) "multiplyExact" else "multiply"
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): MultiplyInterval =
+    copy(interval = newLeft, num = newRight)
 }
 
 case class DivideInterval(
     interval: Expression,
     num: Expression,
-    checkOverflow: Boolean = SQLConf.get.ansiEnabled)
+    failOnError: Boolean = SQLConf.get.ansiEnabled)
   extends IntervalNumOperation(interval, num) {
 
   override protected val operation: (CalendarInterval, Double) => CalendarInterval =
-    if (checkOverflow) divideExact else divide
+    if (failOnError) divideExact else divide
 
-  override protected def operationName: String = if (checkOverflow) "divideExact" else "divide"
+  override protected def operationName: String = if (failOnError) "divideExact" else "divide"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): DivideInterval =
+    copy(interval = newLeft, num = newRight)
 }
 
 // scalastyle:off line.size.limit
@@ -149,8 +231,11 @@ case class DivideInterval(
        100 years 11 months 8 days 12 hours 30 minutes 1.001001 seconds
       > SELECT _FUNC_(100, null, 3);
        NULL
+      > SELECT _FUNC_(0, 1, 0, 1, 0, 0, 100.000001);
+       1 months 1 days 1 minutes 40.000001 seconds
   """,
-  since = "3.0.0")
+  since = "3.0.0",
+  group = "datetime_funcs")
 // scalastyle:on line.size.limit
 case class MakeInterval(
     years: Expression,
@@ -159,9 +244,20 @@ case class MakeInterval(
     days: Expression,
     hours: Expression,
     mins: Expression,
-    secs: Expression)
-  extends SeptenaryExpression with ImplicitCastInputTypes {
+    secs: Expression,
+    failOnError: Boolean = SQLConf.get.ansiEnabled)
+  extends SeptenaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
+  def this(
+      years: Expression,
+      months: Expression,
+      weeks: Expression,
+      days: Expression,
+      hours: Expression,
+      mins: Expression,
+      sec: Expression) = {
+    this(years, months, weeks, days, hours, mins, sec, SQLConf.get.ansiEnabled)
+  }
   def this(
       years: Expression,
       months: Expression,
@@ -169,7 +265,8 @@ case class MakeInterval(
       days: Expression,
       hours: Expression,
       mins: Expression) = {
-    this(years, months, weeks, days, hours, mins, Literal(Decimal(0, 8, 6)))
+    this(years, months, weeks, days, hours, mins, Literal(Decimal(0, Decimal.MAX_LONG_DIGITS, 6)),
+      SQLConf.get.ansiEnabled)
   }
   def this(
       years: Expression,
@@ -191,9 +288,9 @@ case class MakeInterval(
   // Accept `secs` as DecimalType to avoid loosing precision of microseconds while converting
   // them to the fractional part of `secs`.
   override def inputTypes: Seq[AbstractDataType] = Seq(IntegerType, IntegerType, IntegerType,
-    IntegerType, IntegerType, IntegerType, DecimalType(8, 6))
+    IntegerType, IntegerType, IntegerType, DecimalType(Decimal.MAX_LONG_DIGITS, 6))
   override def dataType: DataType = CalendarIntervalType
-  override def nullable: Boolean = true
+  override def nullable: Boolean = if (failOnError) children.exists(_.nullable) else true
 
   override def nullSafeEval(
       year: Any,
@@ -211,9 +308,9 @@ case class MakeInterval(
         day.asInstanceOf[Int],
         hour.asInstanceOf[Int],
         min.asInstanceOf[Int],
-        sec.map(_.asInstanceOf[Decimal]).getOrElse(Decimal(0, 8, 6)))
+        sec.map(_.asInstanceOf[Decimal]).getOrElse(Decimal(0, Decimal.MAX_LONG_DIGITS, 6)))
     } catch {
-      case _: ArithmeticException => null
+      case _: ArithmeticException if !failOnError => null
     }
   }
 
@@ -221,15 +318,270 @@ case class MakeInterval(
     nullSafeCodeGen(ctx, ev, (year, month, week, day, hour, min, sec) => {
       val iu = IntervalUtils.getClass.getName.stripSuffix("$")
       val secFrac = sec.getOrElse("0")
+      val failOnErrorBranch = if (failOnError) "throw e;" else s"${ev.isNull} = true;"
       s"""
         try {
           ${ev.value} = $iu.makeInterval($year, $month, $week, $day, $hour, $min, $secFrac);
         } catch (java.lang.ArithmeticException e) {
-          ${ev.isNull} = true;
+          $failOnErrorBranch
         }
       """
     })
   }
 
   override def prettyName: String = "make_interval"
+
+  // Seq(years, months, weeks, days, hours, mins, secs)
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[Expression]): MakeInterval =
+    copy(
+      years = newChildren(0),
+      months = newChildren(1),
+      weeks = newChildren(2),
+      days = newChildren(3),
+      hours = newChildren(4),
+      mins = newChildren(5),
+      secs = newChildren(6)
+    )
+}
+
+// Multiply an year-month interval by a numeric
+case class MultiplyYMInterval(
+    interval: Expression,
+    num: Expression)
+  extends BinaryExpression with ImplicitCastInputTypes with NullIntolerant with Serializable {
+  override def left: Expression = interval
+  override def right: Expression = num
+
+  override def inputTypes: Seq[AbstractDataType] = Seq(YearMonthIntervalType, NumericType)
+  override def dataType: DataType = YearMonthIntervalType
+
+  @transient
+  private lazy val evalFunc: (Int, Any) => Any = right.dataType match {
+    case ByteType | ShortType | IntegerType => (months: Int, num) =>
+      Math.multiplyExact(months, num.asInstanceOf[Number].intValue())
+    case LongType => (months: Int, num) =>
+      Math.toIntExact(Math.multiplyExact(months, num.asInstanceOf[Long]))
+    case FloatType | DoubleType => (months: Int, num) =>
+      DoubleMath.roundToInt(months * num.asInstanceOf[Number].doubleValue(), RoundingMode.HALF_UP)
+    case _: DecimalType => (months: Int, num) =>
+      val decimalRes = ((new Decimal).set(months) * num.asInstanceOf[Decimal]).toJavaBigDecimal
+      decimalRes.setScale(0, java.math.RoundingMode.HALF_UP).intValueExact()
+  }
+
+  override def nullSafeEval(interval: Any, num: Any): Any = {
+    evalFunc(interval.asInstanceOf[Int], num)
+  }
+
+  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = right.dataType match {
+    case ByteType | ShortType | IntegerType =>
+      defineCodeGen(ctx, ev, (m, n) => s"java.lang.Math.multiplyExact($m, $n)")
+    case LongType =>
+      val jlm = classOf[Math].getName
+      defineCodeGen(ctx, ev, (m, n) => s"$jlm.toIntExact($jlm.multiplyExact($m, $n))")
+    case FloatType | DoubleType =>
+      val dm = classOf[DoubleMath].getName
+      defineCodeGen(ctx, ev, (m, n) =>
+        s"$dm.roundToInt($m * (double)$n, java.math.RoundingMode.HALF_UP)")
+    case _: DecimalType =>
+      defineCodeGen(ctx, ev, (m, n) =>
+        s"((new Decimal()).set($m).$$times($n)).toJavaBigDecimal()" +
+        ".setScale(0, java.math.RoundingMode.HALF_UP).intValueExact()")
+  }
+
+  override def toString: String = s"($left * $right)"
+  override def sql: String = s"(${left.sql} * ${right.sql})"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): MultiplyYMInterval =
+    copy(interval = newLeft, num = newRight)
+}
+
+// Multiply a day-time interval by a numeric
+case class MultiplyDTInterval(
+    interval: Expression,
+    num: Expression)
+  extends BinaryExpression with ImplicitCastInputTypes with NullIntolerant with Serializable {
+  override def left: Expression = interval
+  override def right: Expression = num
+
+  override def inputTypes: Seq[AbstractDataType] = Seq(DayTimeIntervalType, NumericType)
+  override def dataType: DataType = DayTimeIntervalType
+
+  @transient
+  private lazy val evalFunc: (Long, Any) => Any = right.dataType match {
+    case _: IntegralType => (micros: Long, num) =>
+      Math.multiplyExact(micros, num.asInstanceOf[Number].longValue())
+    case _: DecimalType => (micros: Long, num) =>
+      val decimalRes = ((new Decimal).set(micros) * num.asInstanceOf[Decimal]).toJavaBigDecimal
+      decimalRes.setScale(0, RoundingMode.HALF_UP).longValueExact()
+    case _: FractionalType => (micros: Long, num) =>
+      DoubleMath.roundToLong(micros * num.asInstanceOf[Number].doubleValue(), RoundingMode.HALF_UP)
+  }
+
+  override def nullSafeEval(interval: Any, num: Any): Any = {
+    evalFunc(interval.asInstanceOf[Long], num)
+  }
+
+  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = right.dataType match {
+    case _: IntegralType =>
+      defineCodeGen(ctx, ev, (m, n) => s"java.lang.Math.multiplyExact($m, $n)")
+    case _: DecimalType =>
+      defineCodeGen(ctx, ev, (m, n) =>
+        s"((new Decimal()).set($m).$$times($n)).toJavaBigDecimal()" +
+        ".setScale(0, java.math.RoundingMode.HALF_UP).longValueExact()")
+    case _: FractionalType =>
+      val dm = classOf[DoubleMath].getName
+      defineCodeGen(ctx, ev, (m, n) =>
+        s"$dm.roundToLong($m * (double)$n, java.math.RoundingMode.HALF_UP)")
+  }
+
+  override def toString: String = s"($left * $right)"
+  override def sql: String = s"(${left.sql} * ${right.sql})"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): MultiplyDTInterval =
+    copy(interval = newLeft, num = newRight)
+}
+
+trait IntervalDivide {
+  def checkDivideOverflow(value: Any, minValue: Any, num: Expression, numValue: Any): Unit = {
+    if (value == minValue && num.dataType.isInstanceOf[IntegralType]) {
+      if (numValue.asInstanceOf[Number].longValue() == -1) {
+        throw QueryExecutionErrors.overflowInIntegralDivideError()
+      }
+    }
+  }
+}
+
+// Divide an year-month interval by a numeric
+case class DivideYMInterval(
+    interval: Expression,
+    num: Expression)
+  extends BinaryExpression with ImplicitCastInputTypes with IntervalDivide
+    with NullIntolerant with Serializable {
+  override def left: Expression = interval
+  override def right: Expression = num
+
+  override def inputTypes: Seq[AbstractDataType] = Seq(YearMonthIntervalType, NumericType)
+  override def dataType: DataType = YearMonthIntervalType
+
+  @transient
+  private lazy val evalFunc: (Int, Any) => Any = right.dataType match {
+    case LongType => (months: Int, num) =>
+      // Year-month interval has `Int` as the internal type. The result of the divide operation
+      // of `Int` by `Long` must fit to `Int`. So, the casting to `Int` cannot cause overflow.
+      LongMath.divide(months, num.asInstanceOf[Long], RoundingMode.HALF_UP).toInt
+    case _: IntegralType => (months: Int, num) =>
+      IntMath.divide(months, num.asInstanceOf[Number].intValue(), RoundingMode.HALF_UP)
+    case _: DecimalType => (months: Int, num) =>
+      val decimalRes = ((new Decimal).set(months) / num.asInstanceOf[Decimal]).toJavaBigDecimal
+      decimalRes.setScale(0, java.math.RoundingMode.HALF_UP).intValueExact()
+    case _: FractionalType => (months: Int, num) =>
+      DoubleMath.roundToInt(months / num.asInstanceOf[Number].doubleValue(), RoundingMode.HALF_UP)
+  }
+
+  override def nullSafeEval(interval: Any, num: Any): Any = {
+    checkDivideOverflow(interval.asInstanceOf[Int], Int.MinValue, right, num)
+    evalFunc(interval.asInstanceOf[Int], num)
+  }
+
+  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = right.dataType match {
+    case t: IntegralType =>
+      val math = t match {
+        case LongType => classOf[LongMath].getName
+        case _ => classOf[IntMath].getName
+      }
+      val javaType = CodeGenerator.javaType(dataType)
+      val months = left.genCode(ctx)
+      val num = right.genCode(ctx)
+      val checkIntegralDivideOverflow =
+        s"""
+           |if (${months.value} == ${Int.MinValue} && ${num.value} == -1)
+           |  throw QueryExecutionErrors.overflowInIntegralDivideError();
+           |""".stripMargin
+      nullSafeCodeGen(ctx, ev, (m, n) =>
+        // Similarly to non-codegen code. The result of `divide(Int, Long, ...)` must fit to `Int`.
+        // Casting to `Int` is safe here.
+        s"""
+           |$checkIntegralDivideOverflow
+           |${ev.value} = ($javaType)$math.divide($m, $n, java.math.RoundingMode.HALF_UP);
+        """.stripMargin)
+    case _: DecimalType =>
+      defineCodeGen(ctx, ev, (m, n) =>
+        s"((new Decimal()).set($m).$$div($n)).toJavaBigDecimal()" +
+        ".setScale(0, java.math.RoundingMode.HALF_UP).intValueExact()")
+    case _: FractionalType =>
+      val math = classOf[DoubleMath].getName
+      defineCodeGen(ctx, ev, (m, n) =>
+        s"$math.roundToInt($m / (double)$n, java.math.RoundingMode.HALF_UP)")
+  }
+
+  override def toString: String = s"($left / $right)"
+  override def sql: String = s"(${left.sql} / ${right.sql})"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): DivideYMInterval =
+    copy(interval = newLeft, num = newRight)
+}
+
+// Divide a day-time interval by a numeric
+case class DivideDTInterval(
+    interval: Expression,
+    num: Expression)
+  extends BinaryExpression with ImplicitCastInputTypes with IntervalDivide
+    with NullIntolerant with Serializable {
+  override def left: Expression = interval
+  override def right: Expression = num
+
+  override def inputTypes: Seq[AbstractDataType] = Seq(DayTimeIntervalType, NumericType)
+  override def dataType: DataType = DayTimeIntervalType
+
+  @transient
+  private lazy val evalFunc: (Long, Any) => Any = right.dataType match {
+    case _: IntegralType => (micros: Long, num) =>
+      LongMath.divide(micros, num.asInstanceOf[Number].longValue(), RoundingMode.HALF_UP)
+    case _: DecimalType => (micros: Long, num) =>
+      val decimalRes = ((new Decimal).set(micros) / num.asInstanceOf[Decimal]).toJavaBigDecimal
+      decimalRes.setScale(0, java.math.RoundingMode.HALF_UP).longValueExact()
+    case _: FractionalType => (micros: Long, num) =>
+      DoubleMath.roundToLong(micros / num.asInstanceOf[Number].doubleValue(), RoundingMode.HALF_UP)
+  }
+
+  override def nullSafeEval(interval: Any, num: Any): Any = {
+    checkDivideOverflow(interval.asInstanceOf[Long], Long.MinValue, right, num)
+    evalFunc(interval.asInstanceOf[Long], num)
+  }
+
+  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = right.dataType match {
+    case _: IntegralType =>
+      val math = classOf[LongMath].getName
+      val micros = left.genCode(ctx)
+      val num = right.genCode(ctx)
+      val checkIntegralDivideOverflow =
+        s"""
+           |if (${micros.value} == ${Long.MinValue}L && ${num.value} == -1L)
+           |  throw QueryExecutionErrors.overflowInIntegralDivideError();
+           |""".stripMargin
+      nullSafeCodeGen(ctx, ev, (m, n) =>
+        s"""
+           |$checkIntegralDivideOverflow
+           |${ev.value} = $math.divide($m, $n, java.math.RoundingMode.HALF_UP);
+        """.stripMargin)
+    case _: DecimalType =>
+      defineCodeGen(ctx, ev, (m, n) =>
+        s"((new Decimal()).set($m).$$div($n)).toJavaBigDecimal()" +
+        ".setScale(0, java.math.RoundingMode.HALF_UP).longValueExact()")
+    case _: FractionalType =>
+      val math = classOf[DoubleMath].getName
+      defineCodeGen(ctx, ev, (m, n) =>
+        s"$math.roundToLong($m / (double)$n, java.math.RoundingMode.HALF_UP)")
+  }
+
+  override def toString: String = s"($left / $right)"
+  override def sql: String = s"(${left.sql} / ${right.sql})"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): DivideDTInterval =
+    copy(interval = newLeft, num = newRight)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
index 205e5271517c3..d75b59becb372 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
@@ -29,7 +29,9 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
 import org.apache.spark.sql.catalyst.json._
+import org.apache.spark.sql.catalyst.trees.TreePattern.{JSON_TO_STRUCT, TreePattern}
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
@@ -94,7 +96,7 @@ private[this] object JsonPathParser extends RegexParsers {
       case Success(result, _) =>
         Some(result)
 
-      case NoSuccess(msg, next) =>
+      case _ =>
         None
     }
   }
@@ -119,7 +121,8 @@ private[this] object SharedFactory {
       > SELECT _FUNC_('{"a":"b"}', '$.a');
        b
   """,
-  group = "json_funcs")
+  group = "json_funcs",
+  since = "1.5.0")
 case class GetJsonObject(json: Expression, path: Expression)
   extends BinaryExpression with ExpectsInputTypes with CodegenFallback {
 
@@ -333,6 +336,10 @@ case class GetJsonObject(json: Expression, path: Expression)
         false
     }
   }
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): GetJsonObject =
+    copy(json = newLeft, path = newRight)
 }
 
 // scalastyle:off line.size.limit line.contains.tab
@@ -343,7 +350,8 @@ case class GetJsonObject(json: Expression, path: Expression)
       > SELECT _FUNC_('{"a":1, "b":2}', 'a', 'b');
        1	2
   """,
-  group = "json_funcs")
+  group = "json_funcs",
+  since = "1.6.0")
 // scalastyle:on line.size.limit line.contains.tab
 case class JsonTuple(children: Seq[Expression])
   extends Generator with CodegenFallback {
@@ -488,13 +496,16 @@ case class JsonTuple(children: Seq[Expression])
         // a special case that needs to be handled outside of this method.
         // if a requested field is null, the result must be null. the easiest
         // way to achieve this is just by ignoring null tokens entirely
-        throw new IllegalStateException("Do not attempt to copy a null field")
+        throw QueryExecutionErrors.copyNullFieldNotAllowedError
 
       case _ =>
         // handle other types including objects, arrays, booleans and numbers
         generator.copyCurrentStructure(parser)
     }
   }
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): JsonTuple =
+    copy(children = newChildren)
 }
 
 /**
@@ -519,7 +530,8 @@ case class JsonToStructs(
     options: Map[String, String],
     child: Expression,
     timeZoneId: Option[String] = None)
-  extends UnaryExpression with TimeZoneAwareExpression with CodegenFallback with ExpectsInputTypes {
+  extends UnaryExpression with TimeZoneAwareExpression with CodegenFallback with ExpectsInputTypes
+    with NullIntolerant {
 
   // The JSON input data might be missing certain fields. We force the nullability
   // of the user-provided schema to avoid data corruptions. In particular, the parquet-mr encoder
@@ -528,6 +540,8 @@ case class JsonToStructs(
 
   override def nullable: Boolean = true
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(JSON_TO_STRUCT)
+
   // Used in `FunctionRegistry`
   def this(child: Expression, schema: Expression, options: Map[String, String]) =
     this(
@@ -568,8 +582,7 @@ case class JsonToStructs(
     val parsedOptions = new JSONOptions(options, timeZoneId.get, nameOfCorruptRecord)
     val mode = parsedOptions.parseMode
     if (mode != PermissiveMode && mode != FailFastMode) {
-      throw new IllegalArgumentException(s"from_json() doesn't support the ${mode.name} mode. " +
-        s"Acceptable modes are ${PermissiveMode.name} and ${FailFastMode.name}.")
+      throw QueryCompilationErrors.parseModeUnsupportedError("from_json", mode)
     }
     val (parserSchema, actualSchema) = nullableSchema match {
       case s: StructType =>
@@ -606,6 +619,9 @@ case class JsonToStructs(
   }
 
   override def prettyName: String = "from_json"
+
+  override protected def withNewChildInternal(newChild: Expression): JsonToStructs =
+    copy(child = newChild)
 }
 
 /**
@@ -638,7 +654,8 @@ case class StructsToJson(
     options: Map[String, String],
     child: Expression,
     timeZoneId: Option[String] = None)
-  extends UnaryExpression with TimeZoneAwareExpression with CodegenFallback with ExpectsInputTypes {
+  extends UnaryExpression with TimeZoneAwareExpression with CodegenFallback
+    with ExpectsInputTypes with NullIntolerant {
   override def nullable: Boolean = true
 
   def this(options: Map[String, String], child: Expression) = this(options, child, None)
@@ -727,6 +744,9 @@ case class StructsToJson(
   override def inputTypes: Seq[AbstractDataType] = TypeCollection(ArrayType, StructType) :: Nil
 
   override def prettyName: String = "to_json"
+
+  override protected def withNewChildInternal(newChild: Expression): StructsToJson =
+    copy(child = newChild)
 }
 
 /**
@@ -737,9 +757,9 @@ case class StructsToJson(
   examples = """
     Examples:
       > SELECT _FUNC_('[{"col":0}]');
-       array<struct<col:bigint>>
+       ARRAY<STRUCT<`col`: BIGINT>>
       > SELECT _FUNC_('[{"col":01}]', map('allowNumericLeadingZeros', 'true'));
-       array<struct<col:bigint>>
+       ARRAY<STRUCT<`col`: BIGINT>>
   """,
   group = "json_funcs",
   since = "2.4.0")
@@ -797,17 +817,20 @@ case class SchemaOfJson(
       }
     }
 
-    UTF8String.fromString(dt.catalogString)
+    UTF8String.fromString(dt.sql)
   }
 
   override def prettyName: String = "schema_of_json"
+
+  override protected def withNewChildInternal(newChild: Expression): SchemaOfJson =
+    copy(child = newChild)
 }
 
 /**
- * A function that returns the number of elements in the outmost JSON array.
+ * A function that returns the number of elements in the outermost JSON array.
  */
 @ExpressionDescription(
-  usage = "_FUNC_(jsonArray) - Returns the number of elements in the outmost JSON array.",
+  usage = "_FUNC_(jsonArray) - Returns the number of elements in the outermost JSON array.",
   arguments = """
     Arguments:
       * jsonArray - A JSON array. `NULL` is returned in case of any other valid JSON string,
@@ -870,16 +893,19 @@ case class LengthOfJsonArray(child: Expression) extends UnaryExpression
     }
     length
   }
+
+  override protected def withNewChildInternal(newChild: Expression): LengthOfJsonArray =
+    copy(child = newChild)
 }
 
 /**
- * A function which returns all the keys of the outmost JSON object.
+ * A function which returns all the keys of the outermost JSON object.
  */
 @ExpressionDescription(
-  usage = "_FUNC_(json_object) - Returns all the keys of the outmost JSON object as an array.",
+  usage = "_FUNC_(json_object) - Returns all the keys of the outermost JSON object as an array.",
   arguments = """
     Arguments:
-      * json_object - A JSON object. If a valid JSON object is given, all the keys of the outmost
+      * json_object - A JSON object. If a valid JSON object is given, all the keys of the outermost
           object will be returned as an array. If it is any other valid JSON string, an invalid JSON
           string or an empty string, the function returns null.
   """,
@@ -917,7 +943,7 @@ case class JsonObjectKeys(child: Expression) extends UnaryExpression with Codege
           if (parser.nextToken() == null || parser.currentToken() != JsonToken.START_OBJECT) {
             return null
           }
-          // Parse the JSON string to get all the keys of outmost JSON object
+          // Parse the JSON string to get all the keys of outermost JSON object
           getJsonKeys(parser, input)
         }
       }
@@ -939,4 +965,7 @@ case class JsonObjectKeys(child: Expression) extends UnaryExpression with Codege
     }
     new GenericArrayData(arrayBufferOfKeys.toArray)
   }
+
+  override protected def withNewChildInternal(newChild: Expression): JsonObjectKeys =
+    copy(child = newChild)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
index 213a58a3244e2..1a531e2b91c11 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.catalyst.expressions
 
 import java.lang.{Boolean => JavaBoolean}
 import java.lang.{Byte => JavaByte}
+import java.lang.{Character => JavaChar}
 import java.lang.{Double => JavaDouble}
 import java.lang.{Float => JavaFloat}
 import java.lang.{Integer => JavaInteger}
@@ -27,7 +28,7 @@ import java.lang.{Short => JavaShort}
 import java.math.{BigDecimal => JavaBigDecimal}
 import java.nio.charset.StandardCharsets
 import java.sql.{Date, Timestamp}
-import java.time.{Instant, LocalDate}
+import java.time.{Duration, Instant, LocalDate, Period}
 import java.util
 import java.util.Objects
 import javax.xml.bind.DatatypeConverter
@@ -38,15 +39,21 @@ import scala.util.Try
 
 import org.json4s.JsonAST._
 
-import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow, ScalaReflection}
 import org.apache.spark.sql.catalyst.expressions.codegen._
+import org.apache.spark.sql.catalyst.trees.TreePattern
+import org.apache.spark.sql.catalyst.trees.TreePattern.{LITERAL, NULL_LITERAL, TRUE_OR_FALSE_LITERAL}
 import org.apache.spark.sql.catalyst.util._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils.instantToMicros
+import org.apache.spark.sql.catalyst.util.IntervalStringStyles.ANSI_STYLE
+import org.apache.spark.sql.catalyst.util.IntervalUtils.{durationToMicros, periodToMonths, toDayTimeIntervalString, toYearMonthIntervalString}
+import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types._
 import org.apache.spark.util.Utils
+import org.apache.spark.util.collection.BitSet
+import org.apache.spark.util.collection.ImmutableBitSet
 
 object Literal {
   val TrueLiteral: Literal = Literal(true, BooleanType)
@@ -62,6 +69,7 @@ object Literal {
     case s: Short => Literal(s, ShortType)
     case s: String => Literal(UTF8String.fromString(s), StringType)
     case c: Char => Literal(UTF8String.fromString(c.toString), StringType)
+    case ac: Array[Char] => Literal(UTF8String.fromString(String.valueOf(ac)), StringType)
     case b: Boolean => Literal(b, BooleanType)
     case d: BigDecimal =>
       val decimal = Decimal(d)
@@ -74,6 +82,8 @@ object Literal {
     case t: Timestamp => Literal(DateTimeUtils.fromJavaTimestamp(t), TimestampType)
     case ld: LocalDate => Literal(ld.toEpochDay.toInt, DateType)
     case d: Date => Literal(DateTimeUtils.fromJavaDate(d), DateType)
+    case d: Duration => Literal(durationToMicros(d), DayTimeIntervalType)
+    case p: Period => Literal(periodToMonths(p), YearMonthIntervalType)
     case a: Array[Byte] => Literal(a, BinaryType)
     case a: collection.mutable.WrappedArray[_] => apply(a.array)
     case a: Array[_] =>
@@ -85,7 +95,7 @@ object Literal {
     case null => Literal(null, NullType)
     case v: Literal => v
     case _ =>
-      throw new RuntimeException("Unsupported literal type " + v.getClass + " " + v)
+      throw QueryExecutionErrors.literalTypeUnsupportedError(v)
   }
 
   /**
@@ -102,14 +112,18 @@ object Literal {
     case JavaByte.TYPE => ByteType
     case JavaFloat.TYPE => FloatType
     case JavaBoolean.TYPE => BooleanType
+    case JavaChar.TYPE => StringType
 
     // java classes
     case _ if clz == classOf[LocalDate] => DateType
     case _ if clz == classOf[Date] => DateType
     case _ if clz == classOf[Instant] => TimestampType
     case _ if clz == classOf[Timestamp] => TimestampType
+    case _ if clz == classOf[Duration] => DayTimeIntervalType
+    case _ if clz == classOf[Period] => YearMonthIntervalType
     case _ if clz == classOf[JavaBigDecimal] => DecimalType.SYSTEM_DEFAULT
     case _ if clz == classOf[Array[Byte]] => BinaryType
+    case _ if clz == classOf[Array[Char]] => StringType
     case _ if clz == classOf[JavaShort] => ShortType
     case _ if clz == classOf[JavaInteger] => IntegerType
     case _ if clz == classOf[JavaLong] => LongType
@@ -126,7 +140,7 @@ object Literal {
 
     case _ if clz.isArray => ArrayType(componentTypeToDataType(clz.getComponentType))
 
-    case _ => throw new AnalysisException(s"Unsupported component type $clz in arrays")
+    case _ => throw QueryCompilationErrors.arrayComponentTypeUnsupportedError(clz)
   }
 
   /**
@@ -163,6 +177,8 @@ object Literal {
     case dt: DecimalType => Literal(Decimal(0, dt.precision, dt.scale))
     case DateType => create(0, DateType)
     case TimestampType => create(0L, TimestampType)
+    case DayTimeIntervalType => create(0L, DayTimeIntervalType)
+    case YearMonthIntervalType => create(0, YearMonthIntervalType)
     case StringType => Literal("")
     case BinaryType => Literal("".getBytes(StandardCharsets.UTF_8))
     case CalendarIntervalType => Literal(new CalendarInterval(0, 0, 0))
@@ -172,7 +188,7 @@ object Literal {
       create(InternalRow.fromSeq(struct.fields.map(f => default(f.dataType).value)), struct)
     case udt: UserDefinedType[_] => Literal(default(udt.sqlType).value, udt)
     case other =>
-      throw new RuntimeException(s"no default for type $dataType")
+      throw QueryExecutionErrors.noDefaultForDataTypeError(dataType)
   }
 
   private[expressions] def validateLiteralValue(value: Any, dataType: DataType): Unit = {
@@ -181,8 +197,8 @@ object Literal {
       case BooleanType => v.isInstanceOf[Boolean]
       case ByteType => v.isInstanceOf[Byte]
       case ShortType => v.isInstanceOf[Short]
-      case IntegerType | DateType => v.isInstanceOf[Int]
-      case LongType | TimestampType => v.isInstanceOf[Long]
+      case IntegerType | DateType | YearMonthIntervalType => v.isInstanceOf[Int]
+      case LongType | TimestampType | DayTimeIntervalType => v.isInstanceOf[Long]
       case FloatType => v.isInstanceOf[Float]
       case DoubleType => v.isInstanceOf[Double]
       case _: DecimalType => v.isInstanceOf[Decimal]
@@ -284,6 +300,18 @@ object DecimalLiteral {
   def smallerThanSmallestLong(v: Decimal): Boolean = v < Decimal(Long.MinValue)
 }
 
+object LiteralTreeBits {
+  // Singleton tree pattern BitSet for all Literals that are not true, false, or null.
+  val literalBits: BitSet = new ImmutableBitSet(TreePattern.maxId, LITERAL.id)
+
+  // Singleton tree pattern BitSet for all Literals that are true or false.
+  val booleanLiteralBits: BitSet = new ImmutableBitSet(
+      TreePattern.maxId, LITERAL.id, TRUE_OR_FALSE_LITERAL.id)
+
+  // Singleton tree pattern BitSet for all Literals that are nulls.
+  val nullLiteralBits: BitSet = new ImmutableBitSet(TreePattern.maxId, LITERAL.id, NULL_LITERAL.id)
+}
+
 /**
  * In order to do type checking, use Literal.create() instead of constructor
  */
@@ -294,10 +322,31 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression {
   override def foldable: Boolean = true
   override def nullable: Boolean = value == null
 
+  private def timeZoneId = DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone)
+
+  override lazy val treePatternBits: BitSet = {
+    value match {
+      case null => LiteralTreeBits.nullLiteralBits
+      case true | false => LiteralTreeBits.booleanLiteralBits
+      case _ => LiteralTreeBits.literalBits
+    }
+  }
+
   override def toString: String = value match {
     case null => "null"
     case binary: Array[Byte] => s"0x" + DatatypeConverter.printHexBinary(binary)
-    case other => other.toString
+    case d: ArrayBasedMapData => s"map(${d.toString})"
+    case other =>
+      dataType match {
+        case DateType =>
+          DateFormatter(timeZoneId).format(value.asInstanceOf[Int])
+        case TimestampType =>
+          TimestampFormatter.getFractionFormatter(timeZoneId).format(value.asInstanceOf[Long])
+        case DayTimeIntervalType => toDayTimeIntervalString(value.asInstanceOf[Long], ANSI_STYLE)
+        case YearMonthIntervalType => toYearMonthIntervalString(value.asInstanceOf[Int], ANSI_STYLE)
+        case _ =>
+          other.toString
+      }
   }
 
   override def hashCode(): Int = {
@@ -315,7 +364,11 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression {
       (value, o.value) match {
         case (null, null) => true
         case (a: Array[Byte], b: Array[Byte]) => util.Arrays.equals(a, b)
-        case (a, b) => a != null && a.equals(b)
+        case (a: ArrayBasedMapData, b: ArrayBasedMapData) =>
+          a.keyArray == b.keyArray && a.valueArray == b.valueArray
+        case (a: Double, b: Double) if a.isNaN && b.isNaN => true
+        case (a: Float, b: Float) if a.isNaN && b.isNaN => true
+        case (a, b) => a != null && a == b
       }
     case _ => false
   }
@@ -325,8 +378,8 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression {
     // retain in json format, e.g. {"a": 123} can be an int, or double, or decimal, etc.
     val jsonValue = (value, dataType) match {
       case (null, _) => JNull
-      case (i: Int, DateType) => JString(DateTimeUtils.toJavaDate(i).toString)
-      case (l: Long, TimestampType) => JString(DateTimeUtils.toJavaTimestamp(l).toString)
+      case (i: Int, DateType) => JString(toString)
+      case (l: Long, TimestampType) => JString(toString)
       case (other, _) => JString(other.toString)
     }
     ("value" -> jsonValue) :: ("dataType" -> dataType.jsonValue) :: Nil
@@ -343,7 +396,7 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression {
         ExprCode.forNonNullValue(JavaCode.literal(code, dataType))
       }
       dataType match {
-        case BooleanType | IntegerType | DateType =>
+        case BooleanType | IntegerType | DateType | YearMonthIntervalType =>
           toExprCode(value.toString)
         case FloatType =>
           value.asInstanceOf[Float] match {
@@ -369,7 +422,7 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression {
           }
         case ByteType | ShortType =>
           ExprCode.forNonNullValue(JavaCode.expression(s"($javaType)$value", dataType))
-        case TimestampType | LongType =>
+        case TimestampType | LongType | DayTimeIntervalType =>
           toExprCode(s"${value}L")
         case _ =>
           val constRef = ctx.addReferenceObj("literal", value, javaType)
@@ -405,15 +458,14 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression {
       }
     case (v: Decimal, t: DecimalType) => v + "BD"
     case (v: Int, DateType) =>
-      val formatter = DateFormatter(DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone))
-      s"DATE '${formatter.format(v)}'"
+      s"DATE '$toString'"
     case (v: Long, TimestampType) =>
-      val formatter = TimestampFormatter.getFractionFormatter(
-        DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone))
-      s"TIMESTAMP '${formatter.format(v)}'"
+      s"TIMESTAMP '$toString'"
     case (i: CalendarInterval, CalendarIntervalType) =>
       s"INTERVAL '${i.toString}'"
     case (v: Array[Byte], BinaryType) => s"X'${DatatypeConverter.printHexBinary(v)}'"
+    case (i: Long, DayTimeIntervalType) => toDayTimeIntervalString(i, ANSI_STYLE)
+    case (i: Int, YearMonthIntervalType) => toYearMonthIntervalString(i, ANSI_STYLE)
     case _ => value.toString
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
index 66e6334e3a450..516eeb9929e80 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
@@ -57,7 +57,7 @@ abstract class LeafMathExpression(c: Double, name: String)
  * @param name The short name of the function
  */
 abstract class UnaryMathExpression(val f: Double => Double, name: String)
-  extends UnaryExpression with Serializable with ImplicitCastInputTypes {
+  extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant with Serializable {
 
   override def inputTypes: Seq[AbstractDataType] = Seq(DoubleType)
   override def dataType: DataType = DoubleType
@@ -111,7 +111,7 @@ abstract class UnaryLogExpression(f: Double => Double, name: String)
  * @param name The short name of the function
  */
 abstract class BinaryMathExpression(f: (Double, Double) => Double, name: String)
-  extends BinaryExpression with Serializable with ImplicitCastInputTypes {
+  extends BinaryExpression with ImplicitCastInputTypes with NullIntolerant with Serializable {
 
   override def inputTypes: Seq[DataType] = Seq(DoubleType, DoubleType)
 
@@ -147,7 +147,9 @@ abstract class BinaryMathExpression(f: (Double, Double) => Double, name: String)
     Examples:
       > SELECT _FUNC_();
        2.718281828459045
-  """)
+  """,
+  since = "1.5.0",
+  group = "math_funcs")
 case class EulerNumber() extends LeafMathExpression(math.E, "E")
 
 /**
@@ -160,7 +162,9 @@ case class EulerNumber() extends LeafMathExpression(math.E, "E")
     Examples:
       > SELECT _FUNC_();
        3.141592653589793
-  """)
+  """,
+  since = "1.5.0",
+  group = "math_funcs")
 case class Pi() extends LeafMathExpression(math.Pi, "PI")
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -180,8 +184,12 @@ case class Pi() extends LeafMathExpression(math.Pi, "PI")
        0.0
       > SELECT _FUNC_(2);
        NaN
-  """)
-case class Acos(child: Expression) extends UnaryMathExpression(math.acos, "ACOS")
+  """,
+  since = "1.4.0",
+  group = "math_funcs")
+case class Acos(child: Expression) extends UnaryMathExpression(math.acos, "ACOS") {
+  override protected def withNewChildInternal(newChild: Expression): Acos = copy(child = newChild)
+}
 
 @ExpressionDescription(
   usage = """
@@ -194,8 +202,12 @@ case class Acos(child: Expression) extends UnaryMathExpression(math.acos, "ACOS"
        0.0
       > SELECT _FUNC_(2);
        NaN
-  """)
-case class Asin(child: Expression) extends UnaryMathExpression(math.asin, "ASIN")
+  """,
+  since = "1.4.0",
+  group = "math_funcs")
+case class Asin(child: Expression) extends UnaryMathExpression(math.asin, "ASIN") {
+  override protected def withNewChildInternal(newChild: Expression): Asin = copy(child = newChild)
+}
 
 @ExpressionDescription(
   usage = """
@@ -206,8 +218,12 @@ case class Asin(child: Expression) extends UnaryMathExpression(math.asin, "ASIN"
     Examples:
       > SELECT _FUNC_(0);
        0.0
-  """)
-case class Atan(child: Expression) extends UnaryMathExpression(math.atan, "ATAN")
+  """,
+  since = "1.4.0",
+  group = "math_funcs")
+case class Atan(child: Expression) extends UnaryMathExpression(math.atan, "ATAN") {
+  override protected def withNewChildInternal(newChild: Expression): Atan = copy(child = newChild)
+}
 
 @ExpressionDescription(
   usage = "_FUNC_(expr) - Returns the cube root of `expr`.",
@@ -215,8 +231,12 @@ case class Atan(child: Expression) extends UnaryMathExpression(math.atan, "ATAN"
     Examples:
       > SELECT _FUNC_(27.0);
        3.0
-  """)
-case class Cbrt(child: Expression) extends UnaryMathExpression(math.cbrt, "CBRT")
+  """,
+  since = "1.4.0",
+  group = "math_funcs")
+case class Cbrt(child: Expression) extends UnaryMathExpression(math.cbrt, "CBRT") {
+  override protected def withNewChildInternal(newChild: Expression): Cbrt = copy(child = newChild)
+}
 
 @ExpressionDescription(
   usage = "_FUNC_(expr) - Returns the smallest integer not smaller than `expr`.",
@@ -226,7 +246,9 @@ case class Cbrt(child: Expression) extends UnaryMathExpression(math.cbrt, "CBRT"
        0
       > SELECT _FUNC_(5);
        5
-  """)
+  """,
+  since = "1.4.0",
+  group = "math_funcs")
 case class Ceil(child: Expression) extends UnaryMathExpression(math.ceil, "CEIL") {
   override def dataType: DataType = child.dataType match {
     case dt @ DecimalType.Fixed(_, 0) => dt
@@ -253,6 +275,8 @@ case class Ceil(child: Expression) extends UnaryMathExpression(math.ceil, "CEIL"
       case _ => defineCodeGen(ctx, ev, c => s"(long)(java.lang.Math.${funcName}($c))")
     }
   }
+
+  override protected def withNewChildInternal(newChild: Expression): Ceil = copy(child = newChild)
 }
 
 @ExpressionDescription(
@@ -268,8 +292,12 @@ case class Ceil(child: Expression) extends UnaryMathExpression(math.ceil, "CEIL"
     Examples:
       > SELECT _FUNC_(0);
        1.0
-  """)
-case class Cos(child: Expression) extends UnaryMathExpression(math.cos, "COS")
+  """,
+  since = "1.4.0",
+  group = "math_funcs")
+case class Cos(child: Expression) extends UnaryMathExpression(math.cos, "COS") {
+  override protected def withNewChildInternal(newChild: Expression): Cos = copy(child = newChild)
+}
 
 @ExpressionDescription(
   usage = """
@@ -284,8 +312,12 @@ case class Cos(child: Expression) extends UnaryMathExpression(math.cos, "COS")
     Examples:
       > SELECT _FUNC_(0);
        1.0
-  """)
-case class Cosh(child: Expression) extends UnaryMathExpression(math.cosh, "COSH")
+  """,
+  since = "1.4.0",
+  group = "math_funcs")
+case class Cosh(child: Expression) extends UnaryMathExpression(math.cosh, "COSH") {
+  override protected def withNewChildInternal(newChild: Expression): Cosh = copy(child = newChild)
+}
 
 @ExpressionDescription(
   usage = """
@@ -298,13 +330,15 @@ case class Cosh(child: Expression) extends UnaryMathExpression(math.cosh, "COSH"
       > SELECT _FUNC_(0);
        NaN
   """,
-  since = "3.0.0")
+  since = "3.0.0",
+  group = "math_funcs")
 case class Acosh(child: Expression)
   extends UnaryMathExpression((x: Double) => StrictMath.log(x + math.sqrt(x * x - 1.0)), "ACOSH") {
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     defineCodeGen(ctx, ev,
       c => s"java.lang.StrictMath.log($c + java.lang.Math.sqrt($c * $c - 1.0))")
   }
+  override protected def withNewChildInternal(newChild: Expression): Acosh = copy(child = newChild)
 }
 
 /**
@@ -322,11 +356,15 @@ case class Acosh(child: Expression)
        4
       > SELECT _FUNC_(-10, 16, -10);
        -16
-  """)
+  """,
+  since = "1.5.0",
+  group = "math_funcs")
 case class Conv(numExpr: Expression, fromBaseExpr: Expression, toBaseExpr: Expression)
-  extends TernaryExpression with ImplicitCastInputTypes {
+  extends TernaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
-  override def children: Seq[Expression] = Seq(numExpr, fromBaseExpr, toBaseExpr)
+  override def first: Expression = numExpr
+  override def second: Expression = fromBaseExpr
+  override def third: Expression = toBaseExpr
   override def inputTypes: Seq[AbstractDataType] = Seq(StringType, IntegerType, IntegerType)
   override def dataType: DataType = StringType
   override def nullable: Boolean = true
@@ -349,6 +387,10 @@ case class Conv(numExpr: Expression, fromBaseExpr: Expression, toBaseExpr: Expre
        """
     )
   }
+
+  override protected def withNewChildrenInternal(
+      newFirst: Expression, newSecond: Expression, newThird: Expression): Expression =
+    copy(numExpr = newFirst, fromBaseExpr = newSecond, toBaseExpr = newThird)
 }
 
 @ExpressionDescription(
@@ -357,11 +399,14 @@ case class Conv(numExpr: Expression, fromBaseExpr: Expression, toBaseExpr: Expre
     Examples:
       > SELECT _FUNC_(0);
        1.0
-  """)
+  """,
+  since = "1.4.0",
+  group = "math_funcs")
 case class Exp(child: Expression) extends UnaryMathExpression(StrictMath.exp, "EXP") {
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     defineCodeGen(ctx, ev, c => s"java.lang.StrictMath.exp($c)")
   }
+  override protected def withNewChildInternal(newChild: Expression): Exp = copy(child = newChild)
 }
 
 @ExpressionDescription(
@@ -370,11 +415,14 @@ case class Exp(child: Expression) extends UnaryMathExpression(StrictMath.exp, "E
     Examples:
       > SELECT _FUNC_(0);
        0.0
-  """)
+  """,
+  since = "1.4.0",
+  group = "math_funcs")
 case class Expm1(child: Expression) extends UnaryMathExpression(StrictMath.expm1, "EXPM1") {
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     defineCodeGen(ctx, ev, c => s"java.lang.StrictMath.expm1($c)")
   }
+  override protected def withNewChildInternal(newChild: Expression): Expm1 = copy(child = newChild)
 }
 
 @ExpressionDescription(
@@ -385,7 +433,9 @@ case class Expm1(child: Expression) extends UnaryMathExpression(StrictMath.expm1
        -1
       > SELECT _FUNC_(5);
        5
-  """)
+  """,
+  since = "1.4.0",
+  group = "math_funcs")
 case class Floor(child: Expression) extends UnaryMathExpression(math.floor, "FLOOR") {
   override def dataType: DataType = child.dataType match {
     case dt @ DecimalType.Fixed(_, 0) => dt
@@ -412,6 +462,8 @@ case class Floor(child: Expression) extends UnaryMathExpression(math.floor, "FLO
       case _ => defineCodeGen(ctx, ev, c => s"(long)(java.lang.Math.${funcName}($c))")
     }
   }
+
+  override protected def withNewChildInternal(newChild: Expression): Floor = copy(child = newChild)
 }
 
 object Factorial {
@@ -451,8 +503,11 @@ object Factorial {
     Examples:
       > SELECT _FUNC_(5);
        120
-  """)
-case class Factorial(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
+  """,
+  since = "1.5.0",
+  group = "math_funcs")
+case class Factorial(child: Expression)
+  extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def inputTypes: Seq[DataType] = Seq(IntegerType)
 
@@ -482,6 +537,9 @@ case class Factorial(child: Expression) extends UnaryExpression with ImplicitCas
       """
     })
   }
+
+  override protected def withNewChildInternal(newChild: Expression): Factorial =
+    copy(child = newChild)
 }
 
 @ExpressionDescription(
@@ -490,8 +548,13 @@ case class Factorial(child: Expression) extends UnaryExpression with ImplicitCas
     Examples:
       > SELECT _FUNC_(1);
        0.0
-  """)
-case class Log(child: Expression) extends UnaryLogExpression(StrictMath.log, "LOG")
+  """,
+  since = "1.4.0",
+  group = "math_funcs")
+case class Log(child: Expression) extends UnaryLogExpression(StrictMath.log, "LOG") {
+  override def prettyName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("ln")
+  override protected def withNewChildInternal(newChild: Expression): Log = copy(child = newChild)
+}
 
 @ExpressionDescription(
   usage = "_FUNC_(expr) - Returns the logarithm of `expr` with base 2.",
@@ -499,7 +562,9 @@ case class Log(child: Expression) extends UnaryLogExpression(StrictMath.log, "LO
     Examples:
       > SELECT _FUNC_(2);
        1.0
-  """)
+  """,
+  since = "1.4.0",
+  group = "math_funcs")
 case class Log2(child: Expression)
   extends UnaryLogExpression((x: Double) => StrictMath.log(x) / StrictMath.log(2), "LOG2") {
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
@@ -513,6 +578,7 @@ case class Log2(child: Expression)
       """
     )
   }
+  override protected def withNewChildInternal(newChild: Expression): Log2 = copy(child = newChild)
 }
 
 @ExpressionDescription(
@@ -521,8 +587,12 @@ case class Log2(child: Expression)
     Examples:
       > SELECT _FUNC_(10);
        1.0
-  """)
-case class Log10(child: Expression) extends UnaryLogExpression(StrictMath.log10, "LOG10")
+  """,
+  since = "1.4.0",
+  group = "math_funcs")
+case class Log10(child: Expression) extends UnaryLogExpression(StrictMath.log10, "LOG10") {
+  override protected def withNewChildInternal(newChild: Expression): Log10 = copy(child = newChild)
+}
 
 @ExpressionDescription(
   usage = "_FUNC_(expr) - Returns log(1 + `expr`).",
@@ -530,9 +600,12 @@ case class Log10(child: Expression) extends UnaryLogExpression(StrictMath.log10,
     Examples:
       > SELECT _FUNC_(0);
        0.0
-  """)
+  """,
+  since = "1.4.0",
+  group = "math_funcs")
 case class Log1p(child: Expression) extends UnaryLogExpression(StrictMath.log1p, "LOG1P") {
   protected override val yAsymptote: Double = -1.0
+  override protected def withNewChildInternal(newChild: Expression): Log1p = copy(child = newChild)
 }
 
 // scalastyle:off line.size.limit
@@ -542,10 +615,14 @@ case class Log1p(child: Expression) extends UnaryLogExpression(StrictMath.log1p,
     Examples:
       > SELECT _FUNC_(12.3456);
        12.0
-  """)
+  """,
+  since = "1.4.0",
+  group = "math_funcs")
 // scalastyle:on line.size.limit
 case class Rint(child: Expression) extends UnaryMathExpression(math.rint, "ROUND") {
   override def funcName: String = "rint"
+  override def prettyName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("rint")
+  override protected def withNewChildInternal(newChild: Expression): Rint = copy(child = newChild)
 }
 
 @ExpressionDescription(
@@ -554,8 +631,12 @@ case class Rint(child: Expression) extends UnaryMathExpression(math.rint, "ROUND
     Examples:
       > SELECT _FUNC_(40);
        1.0
-  """)
-case class Signum(child: Expression) extends UnaryMathExpression(math.signum, "SIGNUM")
+  """,
+  since = "1.4.0",
+  group = "math_funcs")
+case class Signum(child: Expression) extends UnaryMathExpression(math.signum, "SIGNUM") {
+  override protected def withNewChildInternal(newChild: Expression): Signum = copy(child = newChild)
+}
 
 @ExpressionDescription(
   usage = "_FUNC_(expr) - Returns the sine of `expr`, as if computed by `java.lang.Math._FUNC_`.",
@@ -567,8 +648,12 @@ case class Signum(child: Expression) extends UnaryMathExpression(math.signum, "S
     Examples:
       > SELECT _FUNC_(0);
        0.0
-  """)
-case class Sin(child: Expression) extends UnaryMathExpression(math.sin, "SIN")
+  """,
+  since = "1.4.0",
+  group = "math_funcs")
+case class Sin(child: Expression) extends UnaryMathExpression(math.sin, "SIN") {
+  override protected def withNewChildInternal(newChild: Expression): Sin = copy(child = newChild)
+}
 
 @ExpressionDescription(
   usage = """
@@ -582,8 +667,12 @@ case class Sin(child: Expression) extends UnaryMathExpression(math.sin, "SIN")
     Examples:
       > SELECT _FUNC_(0);
        0.0
-  """)
-case class Sinh(child: Expression) extends UnaryMathExpression(math.sinh, "SINH")
+  """,
+  since = "1.4.0",
+  group = "math_funcs")
+case class Sinh(child: Expression) extends UnaryMathExpression(math.sinh, "SINH") {
+  override protected def withNewChildInternal(newChild: Expression): Sinh = copy(child = newChild)
+}
 
 @ExpressionDescription(
   usage = """
@@ -594,7 +683,8 @@ case class Sinh(child: Expression) extends UnaryMathExpression(math.sinh, "SINH"
       > SELECT _FUNC_(0);
        0.0
   """,
-  since = "3.0.0")
+  since = "3.0.0",
+  group = "math_funcs")
 case class Asinh(child: Expression)
   extends UnaryMathExpression((x: Double) => x match {
     case Double.NegativeInfinity => Double.NegativeInfinity
@@ -604,6 +694,7 @@ case class Asinh(child: Expression)
       s"$c == Double.NEGATIVE_INFINITY ? Double.NEGATIVE_INFINITY : " +
       s"java.lang.StrictMath.log($c + java.lang.Math.sqrt($c * $c + 1.0))")
   }
+  override protected def withNewChildInternal(newChild: Expression): Asinh = copy(child = newChild)
 }
 
 @ExpressionDescription(
@@ -612,8 +703,12 @@ case class Asinh(child: Expression)
     Examples:
       > SELECT _FUNC_(4);
        2.0
-  """)
-case class Sqrt(child: Expression) extends UnaryMathExpression(math.sqrt, "SQRT")
+  """,
+  since = "1.1.1",
+  group = "math_funcs")
+case class Sqrt(child: Expression) extends UnaryMathExpression(math.sqrt, "SQRT") {
+  override protected def withNewChildInternal(newChild: Expression): Sqrt = copy(child = newChild)
+}
 
 @ExpressionDescription(
   usage = """
@@ -627,8 +722,12 @@ case class Sqrt(child: Expression) extends UnaryMathExpression(math.sqrt, "SQRT"
     Examples:
       > SELECT _FUNC_(0);
        0.0
-  """)
-case class Tan(child: Expression) extends UnaryMathExpression(math.tan, "TAN")
+  """,
+  since = "1.4.0",
+  group = "math_funcs")
+case class Tan(child: Expression) extends UnaryMathExpression(math.tan, "TAN") {
+  override protected def withNewChildInternal(newChild: Expression): Tan = copy(child = newChild)
+}
 
 @ExpressionDescription(
   usage = """
@@ -642,12 +741,15 @@ case class Tan(child: Expression) extends UnaryMathExpression(math.tan, "TAN")
     Examples:
       > SELECT _FUNC_(1);
        0.6420926159343306
-  """)
+  """,
+  since = "2.3.0",
+  group = "math_funcs")
 case class Cot(child: Expression)
   extends UnaryMathExpression((x: Double) => 1 / math.tan(x), "COT") {
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     defineCodeGen(ctx, ev, c => s"${ev.value} = 1 / java.lang.Math.tan($c);")
   }
+  override protected def withNewChildInternal(newChild: Expression): Cot = copy(child = newChild)
 }
 
 @ExpressionDescription(
@@ -663,8 +765,12 @@ case class Cot(child: Expression)
     Examples:
       > SELECT _FUNC_(0);
        0.0
-  """)
-case class Tanh(child: Expression) extends UnaryMathExpression(math.tanh, "TANH")
+  """,
+  since = "1.4.0",
+  group = "math_funcs")
+case class Tanh(child: Expression) extends UnaryMathExpression(math.tanh, "TANH") {
+  override protected def withNewChildInternal(newChild: Expression): Tanh = copy(child = newChild)
+}
 
 @ExpressionDescription(
   usage = """
@@ -677,7 +783,8 @@ case class Tanh(child: Expression) extends UnaryMathExpression(math.tanh, "TANH"
       > SELECT _FUNC_(2);
        NaN
   """,
-  since = "3.0.0")
+  since = "3.0.0",
+  group = "math_funcs")
 case class Atanh(child: Expression)
   // SPARK-28519: more accurate express for 1/2 * ln((1 + x) / (1 - x))
   extends UnaryMathExpression((x: Double) =>
@@ -686,6 +793,7 @@ case class Atanh(child: Expression)
     defineCodeGen(ctx, ev,
       c => s"0.5 * (java.lang.StrictMath.log1p($c) - java.lang.StrictMath.log1p(- $c))")
   }
+  override protected def withNewChildInternal(newChild: Expression): Atanh = copy(child = newChild)
 }
 
 @ExpressionDescription(
@@ -698,9 +806,13 @@ case class Atanh(child: Expression)
     Examples:
       > SELECT _FUNC_(3.141592653589793);
        180.0
-  """)
+  """,
+  since = "1.4.0",
+  group = "math_funcs")
 case class ToDegrees(child: Expression) extends UnaryMathExpression(math.toDegrees, "DEGREES") {
   override def funcName: String = "toDegrees"
+  override protected def withNewChildInternal(newChild: Expression): ToDegrees =
+    copy(child = newChild)
 }
 
 @ExpressionDescription(
@@ -713,9 +825,13 @@ case class ToDegrees(child: Expression) extends UnaryMathExpression(math.toDegre
     Examples:
       > SELECT _FUNC_(180);
        3.141592653589793
-  """)
+  """,
+  since = "1.4.0",
+  group = "math_funcs")
 case class ToRadians(child: Expression) extends UnaryMathExpression(math.toRadians, "RADIANS") {
   override def funcName: String = "toRadians"
+  override protected def withNewChildInternal(newChild: Expression): ToRadians =
+    copy(child = newChild)
 }
 
 // scalastyle:off line.size.limit
@@ -729,10 +845,12 @@ case class ToRadians(child: Expression) extends UnaryMathExpression(math.toRadia
        1111111111111111111111111111111111111111111111111111111111110011
       > SELECT _FUNC_(13.3);
        1101
-  """)
+  """,
+  since = "1.5.0",
+  group = "math_funcs")
 // scalastyle:on line.size.limit
 case class Bin(child: Expression)
-  extends UnaryExpression with Serializable with ImplicitCastInputTypes {
+  extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant with Serializable {
 
   override def inputTypes: Seq[DataType] = Seq(LongType)
   override def dataType: DataType = StringType
@@ -744,6 +862,8 @@ case class Bin(child: Expression)
     defineCodeGen(ctx, ev, (c) =>
       s"UTF8String.fromString(java.lang.Long.toBinaryString($c))")
   }
+
+  override protected def withNewChildInternal(newChild: Expression): Bin = copy(child = newChild)
 }
 
 object Hex {
@@ -830,8 +950,11 @@ object Hex {
        11
       > SELECT _FUNC_('Spark SQL');
        537061726B2053514C
-  """)
-case class Hex(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
+  """,
+  since = "1.5.0",
+  group = "math_funcs")
+case class Hex(child: Expression)
+  extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def inputTypes: Seq[AbstractDataType] =
     Seq(TypeCollection(LongType, BinaryType, StringType))
@@ -853,6 +976,8 @@ case class Hex(child: Expression) extends UnaryExpression with ImplicitCastInput
       })
     })
   }
+
+  override protected def withNewChildInternal(newChild: Expression): Hex = copy(child = newChild)
 }
 
 /**
@@ -865,8 +990,11 @@ case class Hex(child: Expression) extends UnaryExpression with ImplicitCastInput
     Examples:
       > SELECT decode(_FUNC_('537061726B2053514C'), 'UTF-8');
        Spark SQL
-  """)
-case class Unhex(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
+  """,
+  since = "1.5.0",
+  group = "math_funcs")
+case class Unhex(child: Expression)
+  extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def inputTypes: Seq[AbstractDataType] = Seq(StringType)
 
@@ -885,6 +1013,8 @@ case class Unhex(child: Expression) extends UnaryExpression with ImplicitCastInp
        """
     })
   }
+
+  override protected def withNewChildInternal(newChild: Expression): Unhex = copy(child = newChild)
 }
 
 
@@ -909,7 +1039,9 @@ case class Unhex(child: Expression) extends UnaryExpression with ImplicitCastInp
     Examples:
       > SELECT _FUNC_(0, 0);
        0.0
-  """)
+  """,
+  since = "1.4.0",
+  group = "math_funcs")
 case class Atan2(left: Expression, right: Expression)
   extends BinaryMathExpression(math.atan2, "ATAN2") {
 
@@ -921,6 +1053,9 @@ case class Atan2(left: Expression, right: Expression)
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     defineCodeGen(ctx, ev, (c1, c2) => s"java.lang.Math.atan2($c1 + 0.0, $c2 + 0.0)")
   }
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): Expression = copy(left = newLeft, right = newRight)
 }
 
 @ExpressionDescription(
@@ -929,12 +1064,16 @@ case class Atan2(left: Expression, right: Expression)
     Examples:
       > SELECT _FUNC_(2, 3);
        8.0
-  """)
+  """,
+  since = "1.4.0",
+  group = "math_funcs")
 case class Pow(left: Expression, right: Expression)
   extends BinaryMathExpression(StrictMath.pow, "POWER") {
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     defineCodeGen(ctx, ev, (c1, c2) => s"java.lang.StrictMath.pow($c1, $c2)")
   }
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): Expression = copy(left = newLeft, right = newRight)
 }
 
 
@@ -950,9 +1089,11 @@ case class Pow(left: Expression, right: Expression)
     Examples:
       > SELECT _FUNC_(2, 1);
        4
-  """)
+  """,
+  since = "1.5.0",
+  group = "math_funcs")
 case class ShiftLeft(left: Expression, right: Expression)
-  extends BinaryExpression with ImplicitCastInputTypes {
+  extends BinaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def inputTypes: Seq[AbstractDataType] =
     Seq(TypeCollection(IntegerType, LongType), IntegerType)
@@ -969,6 +1110,9 @@ case class ShiftLeft(left: Expression, right: Expression)
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     defineCodeGen(ctx, ev, (left, right) => s"$left << $right")
   }
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): ShiftLeft = copy(left = newLeft, right = newRight)
 }
 
 
@@ -984,9 +1128,11 @@ case class ShiftLeft(left: Expression, right: Expression)
     Examples:
       > SELECT _FUNC_(4, 1);
        2
-  """)
+  """,
+  since = "1.5.0",
+  group = "bitwise_funcs")
 case class ShiftRight(left: Expression, right: Expression)
-  extends BinaryExpression with ImplicitCastInputTypes {
+  extends BinaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def inputTypes: Seq[AbstractDataType] =
     Seq(TypeCollection(IntegerType, LongType), IntegerType)
@@ -1003,6 +1149,9 @@ case class ShiftRight(left: Expression, right: Expression)
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     defineCodeGen(ctx, ev, (left, right) => s"$left >> $right")
   }
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): ShiftRight = copy(left = newLeft, right = newRight)
 }
 
 
@@ -1018,9 +1167,11 @@ case class ShiftRight(left: Expression, right: Expression)
     Examples:
       > SELECT _FUNC_(4, 1);
        2
-  """)
+  """,
+  since = "1.5.0",
+  group = "bitwise_funcs")
 case class ShiftRightUnsigned(left: Expression, right: Expression)
-  extends BinaryExpression with ImplicitCastInputTypes {
+  extends BinaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def inputTypes: Seq[AbstractDataType] =
     Seq(TypeCollection(IntegerType, LongType), IntegerType)
@@ -1037,6 +1188,10 @@ case class ShiftRightUnsigned(left: Expression, right: Expression)
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     defineCodeGen(ctx, ev, (left, right) => s"$left >>> $right")
   }
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): ShiftRightUnsigned =
+    copy(left = newLeft, right = newRight)
 }
 
 @ExpressionDescription(
@@ -1045,9 +1200,14 @@ case class ShiftRightUnsigned(left: Expression, right: Expression)
     Examples:
       > SELECT _FUNC_(3, 4);
        5.0
-  """)
+  """,
+  since = "1.4.0",
+  group = "math_funcs")
 case class Hypot(left: Expression, right: Expression)
-  extends BinaryMathExpression(math.hypot, "HYPOT")
+  extends BinaryMathExpression(math.hypot, "HYPOT") {
+  override protected def withNewChildrenInternal(newLeft: Expression, newRight: Expression): Hypot =
+    copy(left = newLeft, right = newRight)
+}
 
 
 /**
@@ -1062,7 +1222,9 @@ case class Hypot(left: Expression, right: Expression)
     Examples:
       > SELECT _FUNC_(10, 100);
        2.0
-  """)
+  """,
+  since = "1.5.0",
+  group = "math_funcs")
 case class Logarithm(left: Expression, right: Expression)
   extends BinaryMathExpression((c1, c2) => StrictMath.log(c2) / StrictMath.log(c1), "LOG") {
 
@@ -1103,6 +1265,9 @@ case class Logarithm(left: Expression, right: Expression)
         """)
     }
   }
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): Logarithm = copy(left = newLeft, right = newRight)
 }
 
 /**
@@ -1292,12 +1457,16 @@ abstract class RoundBase(child: Expression, scale: Expression,
     Examples:
       > SELECT _FUNC_(2.5, 0);
        3
-  """)
+  """,
+  since = "1.5.0",
+  group = "math_funcs")
 // scalastyle:on line.size.limit
 case class Round(child: Expression, scale: Expression)
   extends RoundBase(child, scale, BigDecimal.RoundingMode.HALF_UP, "ROUND_HALF_UP")
     with Serializable with ImplicitCastInputTypes {
   def this(child: Expression) = this(child, Literal(0))
+  override protected def withNewChildrenInternal(newLeft: Expression, newRight: Expression): Round =
+    copy(child = newLeft, scale = newRight)
 }
 
 /**
@@ -1312,10 +1481,120 @@ case class Round(child: Expression, scale: Expression)
     Examples:
       > SELECT _FUNC_(2.5, 0);
        2
-  """)
+  """,
+  since = "2.0.0",
+  group = "math_funcs")
 // scalastyle:on line.size.limit
 case class BRound(child: Expression, scale: Expression)
   extends RoundBase(child, scale, BigDecimal.RoundingMode.HALF_EVEN, "ROUND_HALF_EVEN")
     with Serializable with ImplicitCastInputTypes {
   def this(child: Expression) = this(child, Literal(0))
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): BRound = copy(child = newLeft, scale = newRight)
+}
+
+object WidthBucket {
+
+  def computeBucketNumber(value: Double, min: Double, max: Double, numBucket: Long): jl.Long = {
+    if (numBucket <= 0 || numBucket == Long.MaxValue || jl.Double.isNaN(value) || min == max ||
+        jl.Double.isNaN(min) || jl.Double.isInfinite(min) ||
+        jl.Double.isNaN(max) || jl.Double.isInfinite(max)) {
+      return null
+    }
+
+    val lower = Math.min(min, max)
+    val upper = Math.max(min, max)
+
+    if (min < max) {
+      if (value < lower) {
+        0L
+      } else if (value >= upper) {
+        numBucket + 1L
+      } else {
+        (numBucket.toDouble * (value - lower) / (upper - lower)).toLong + 1L
+      }
+    } else { // `min > max` case
+      if (value > upper) {
+        0L
+      } else if (value <= lower) {
+        numBucket + 1L
+      } else {
+        (numBucket.toDouble * (upper - value) / (upper - lower)).toLong + 1L
+      }
+    }
+  }
+}
+
+/**
+ * Returns the bucket number into which the value of this expression would fall
+ * after being evaluated. Note that input arguments must follow conditions listed below;
+ * otherwise, the method will return null.
+ *  - `numBucket` must be greater than zero and be less than Long.MaxValue
+ *  - `value`, `min`, and `max` cannot be NaN
+ *  - `min` bound cannot equal `max`
+ *  - `min` and `max` must be finite
+ *
+ * Note: If `minValue` > `maxValue`, a return value is as follows;
+ *  if `value` > `minValue`, it returns 0.
+ *  if `value` <= `maxValue`, it returns `numBucket` + 1.
+ *  otherwise, it returns (`numBucket` * (`minValue` - `value`) / (`minValue` - `maxValue`)) + 1
+ *
+ * @param value is the expression to compute a bucket number in the histogram
+ * @param minValue is the minimum value of the histogram
+ * @param maxValue is the maximum value of the histogram
+ * @param numBucket is the number of buckets
+ */
+@ExpressionDescription(
+  usage = """
+    _FUNC_(value, min_value, max_value, num_bucket) - Returns the bucket number to which
+      `value` would be assigned in an equiwidth histogram with `num_bucket` buckets,
+      in the range `min_value` to `max_value`."
+  """,
+  examples = """
+    Examples:
+      > SELECT _FUNC_(5.3, 0.2, 10.6, 5);
+       3
+      > SELECT _FUNC_(-2.1, 1.3, 3.4, 3);
+       0
+      > SELECT _FUNC_(8.1, 0.0, 5.7, 4);
+       5
+      > SELECT _FUNC_(-0.9, 5.2, 0.5, 2);
+       3
+  """,
+  since = "3.1.0",
+  group = "math_funcs")
+case class WidthBucket(
+    value: Expression,
+    minValue: Expression,
+    maxValue: Expression,
+    numBucket: Expression)
+  extends QuaternaryExpression with ImplicitCastInputTypes with NullIntolerant {
+
+  override def inputTypes: Seq[AbstractDataType] = Seq(DoubleType, DoubleType, DoubleType, LongType)
+  override def dataType: DataType = LongType
+  override def nullable: Boolean = true
+  override def prettyName: String = "width_bucket"
+
+  override protected def nullSafeEval(input: Any, min: Any, max: Any, numBucket: Any): Any = {
+    WidthBucket.computeBucketNumber(
+      input.asInstanceOf[Double],
+      min.asInstanceOf[Double],
+      max.asInstanceOf[Double],
+      numBucket.asInstanceOf[Long])
+  }
+
+  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
+    defineCodeGen(ctx, ev, (input, min, max, numBucket) =>
+      "org.apache.spark.sql.catalyst.expressions.WidthBucket" +
+        s".computeBucketNumber($input, $min, $max, $numBucket)")
+  }
+
+  override def first: Expression = value
+  override def second: Expression = minValue
+  override def third: Expression = maxValue
+  override def fourth: Expression = numBucket
+
+  override protected def withNewChildrenInternal(
+      first: Expression, second: Expression, third: Expression, fourth: Expression): WidthBucket =
+    copy(value = first, minValue = second, maxValue = third, numBucket = fourth)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
index 8ce3ddd30a69e..1318df3d4b3d4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
@@ -19,8 +19,10 @@ package org.apache.spark.sql.catalyst.expressions
 
 import org.apache.spark.{SPARK_REVISION, SPARK_VERSION_SHORT}
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.UnresolvedSeed
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
+import org.apache.spark.sql.catalyst.trees.TreePattern.{CURRENT_LIKE, TreePattern}
 import org.apache.spark.sql.catalyst.util.RandomUUIDGenerator
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
@@ -50,53 +52,101 @@ case class PrintToStderr(child: Expression) extends UnaryExpression {
          | ${ev.value} = $c;
        """.stripMargin)
   }
+
+  override protected def withNewChildInternal(newChild: Expression): PrintToStderr =
+    copy(child = newChild)
 }
 
 /**
- * A function throws an exception if 'condition' is not true.
+ * Throw with the result of an expression (used for debugging).
  */
 @ExpressionDescription(
-  usage = "_FUNC_(expr) - Throws an exception if `expr` is not true.",
+  usage = "_FUNC_(expr) - Throws an exception with `expr`.",
   examples = """
     Examples:
-      > SELECT _FUNC_(0 < 1);
-       NULL
-  """)
-case class AssertTrue(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
-
-  override def nullable: Boolean = true
-
-  override def inputTypes: Seq[DataType] = Seq(BooleanType)
+      > SELECT _FUNC_('custom error message');
+       java.lang.RuntimeException
+       custom error message
+  """,
+  since = "3.1.0",
+  group = "misc_funcs")
+case class RaiseError(child: Expression, dataType: DataType)
+  extends UnaryExpression with ImplicitCastInputTypes {
 
-  override def dataType: DataType = NullType
+  def this(child: Expression) = this(child, NullType)
 
-  override def prettyName: String = "assert_true"
+  override def foldable: Boolean = false
+  override def nullable: Boolean = true
+  override def inputTypes: Seq[AbstractDataType] = Seq(StringType)
 
-  private val errMsg = s"'${child.simpleString(SQLConf.get.maxToStringFields)}' is not true!"
+  override def prettyName: String = "raise_error"
 
-  override def eval(input: InternalRow) : Any = {
-    val v = child.eval(input)
-    if (v == null || java.lang.Boolean.FALSE.equals(v)) {
-      throw new RuntimeException(errMsg)
-    } else {
-      null
+  override def eval(input: InternalRow): Any = {
+    val value = child.eval(input)
+    if (value == null) {
+      throw new RuntimeException()
     }
+    throw new RuntimeException(value.toString)
   }
 
+  // if (true) is to avoid codegen compilation exception that statement is unreachable
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     val eval = child.genCode(ctx)
+    ExprCode(
+      code = code"""${eval.code}
+        |if (true) {
+        |  if (${eval.isNull}) {
+        |    throw new RuntimeException();
+        |  }
+        |  throw new RuntimeException(${eval.value}.toString());
+        |}""".stripMargin,
+      isNull = TrueLiteral,
+      value = JavaCode.defaultLiteral(dataType)
+    )
+  }
+
+  override protected def withNewChildInternal(newChild: Expression): RaiseError =
+    copy(child = newChild)
+}
+
+object RaiseError {
+  def apply(child: Expression): RaiseError = new RaiseError(child)
+}
+
+/**
+ * A function that throws an exception if 'condition' is not true.
+ */
+@ExpressionDescription(
+  usage = "_FUNC_(expr) - Throws an exception if `expr` is not true.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_(0 < 1);
+       NULL
+  """,
+  since = "2.0.0",
+  group = "misc_funcs")
+case class AssertTrue(left: Expression, right: Expression, child: Expression)
+  extends RuntimeReplaceable {
+
+  override def prettyName: String = "assert_true"
+
+  def this(left: Expression, right: Expression) = {
+    this(left, right, If(left, Literal(null), RaiseError(right)))
+  }
 
-    // Use unnamed reference that doesn't create a local field here to reduce the number of fields
-    // because errMsgField is used only when the value is null or false.
-    val errMsgField = ctx.addReferenceObj("errMsg", errMsg)
-    ExprCode(code = code"""${eval.code}
-       |if (${eval.isNull} || !${eval.value}) {
-       |  throw new RuntimeException($errMsgField);
-       |}""".stripMargin, isNull = TrueLiteral,
-      value = JavaCode.defaultLiteral(dataType))
+  def this(left: Expression) = {
+    this(left, Literal(s"'${left.simpleString(SQLConf.get.maxToStringFields)}' is not true!"))
   }
 
-  override def sql: String = s"assert_true(${child.sql})"
+  override def flatArguments: Iterator[Any] = Iterator(left, right)
+  override def exprsReplaced: Seq[Expression] = Seq(left, right)
+
+  override protected def withNewChildInternal(newChild: Expression): AssertTrue =
+    copy(child = newChild)
+}
+
+object AssertTrue {
+  def apply(left: Expression): AssertTrue = new AssertTrue(left)
 }
 
 /**
@@ -108,12 +158,33 @@ case class AssertTrue(child: Expression) extends UnaryExpression with ImplicitCa
     Examples:
       > SELECT _FUNC_();
        default
-  """)
+  """,
+  since = "1.6.0",
+  group = "misc_funcs")
 case class CurrentDatabase() extends LeafExpression with Unevaluable {
   override def dataType: DataType = StringType
-  override def foldable: Boolean = true
   override def nullable: Boolean = false
   override def prettyName: String = "current_database"
+  final override val nodePatterns: Seq[TreePattern] = Seq(CURRENT_LIKE)
+}
+
+/**
+ * Returns the current catalog.
+ */
+@ExpressionDescription(
+  usage = "_FUNC_() - Returns the current catalog.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_();
+       spark_catalog
+  """,
+  since = "3.1.0",
+  group = "misc_funcs")
+case class CurrentCatalog() extends LeafExpression with Unevaluable {
+  override def dataType: DataType = StringType
+  override def nullable: Boolean = false
+  override def prettyName: String = "current_catalog"
+  final override val nodePatterns: Seq[TreePattern] = Seq(CURRENT_LIKE)
 }
 
 // scalastyle:off line.size.limit
@@ -126,13 +197,17 @@ case class CurrentDatabase() extends LeafExpression with Unevaluable {
   """,
   note = """
     The function is non-deterministic.
-  """)
+  """,
+  since = "2.3.0",
+  group = "misc_funcs")
 // scalastyle:on line.size.limit
 case class Uuid(randomSeed: Option[Long] = None) extends LeafExpression with Stateful
     with ExpressionWithRandomSeed {
 
   def this() = this(None)
 
+  override def seedExpression: Expression = randomSeed.map(Literal.apply).getOrElse(UnresolvedSeed)
+
   override def withNewSeed(seed: Long): Uuid = Uuid(Some(seed))
 
   override lazy val resolved: Boolean = randomSeed.isDefined
@@ -167,7 +242,13 @@ case class Uuid(randomSeed: Option[Long] = None) extends LeafExpression with Sta
 // scalastyle:off line.size.limit
 @ExpressionDescription(
   usage = """_FUNC_() - Returns the Spark version. The string contains 2 fields, the first being a release version and the second being a git revision.""",
-  since = "3.0.0")
+  examples = """
+    Examples:
+      > SELECT _FUNC_();
+       3.1.0 a6d6ea3efedbad14d99c24143834cd4e2e52fb40
+  """,
+  since = "3.0.0",
+  group = "misc_funcs")
 // scalastyle:on line.size.limit
 case class SparkVersion() extends LeafExpression with CodegenFallback {
   override def nullable: Boolean = false
@@ -182,13 +263,14 @@ case class SparkVersion() extends LeafExpression with CodegenFallback {
 @ExpressionDescription(
   usage = """_FUNC_(expr) - Return DDL-formatted type string for the data type of the input.""",
   examples = """
-      Examples:
+    Examples:
       > SELECT _FUNC_(1);
        int
       > SELECT _FUNC_(array(1));
        array<int>
   """,
-  since = "3.0.0")
+  since = "3.0.0",
+  group = "misc_funcs")
 case class TypeOf(child: Expression) extends UnaryExpression {
   override def nullable: Boolean = false
   override def foldable: Boolean = true
@@ -198,4 +280,6 @@ case class TypeOf(child: Expression) extends UnaryExpression {
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     defineCodeGen(ctx, ev, _ => s"""UTF8String.fromString(${child.dataType.catalogString})""")
   }
+
+  override protected def withNewChildInternal(newChild: Expression): TypeOf = copy(child = newChild)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
index 77b4cecc263c7..52487d4decb68 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
@@ -23,8 +23,13 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.plans.logical.EventTimeWatermark
-import org.apache.spark.sql.catalyst.util.quoteIdentifier
+import org.apache.spark.sql.catalyst.trees.TreePattern
+import org.apache.spark.sql.catalyst.trees.TreePattern._
+import org.apache.spark.sql.catalyst.util.quoteIfNeeded
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.types._
+import org.apache.spark.util.collection.BitSet
+import org.apache.spark.util.collection.ImmutableBitSet
 
 object NamedExpression {
   private val curId = new java.util.concurrent.atomic.AtomicLong()
@@ -85,6 +90,7 @@ trait NamedExpression extends Expression {
    *    e.g. top level attributes aliased in the SELECT clause, or column from a LocalRelation.
    * 2. Seq with a Single element: either the table name or the alias name of the table.
    * 3. Seq with 2 elements: database name and table name
+   * 4. Seq with 3 elements: catalog name, database name and table name
    */
   def qualifier: Seq[String]
 
@@ -142,13 +148,18 @@ abstract class Attribute extends LeafExpression with NamedExpression with NullIn
  *                  fully qualified way. Consider the examples tableName.name, subQueryAlias.name.
  *                  tableName and subQueryAlias are possible qualifiers.
  * @param explicitMetadata Explicit metadata associated with this alias that overwrites child's.
+ * @param nonInheritableMetadataKeys Keys of metadata entries that are supposed to be removed when
+ *                                   inheriting the metadata from the child.
  */
 case class Alias(child: Expression, name: String)(
     val exprId: ExprId = NamedExpression.newExprId,
     val qualifier: Seq[String] = Seq.empty,
-    val explicitMetadata: Option[Metadata] = None)
+    val explicitMetadata: Option[Metadata] = None,
+    val nonInheritableMetadataKeys: Seq[String] = Seq.empty)
   extends UnaryExpression with NamedExpression {
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(ALIAS)
+
   // Alias(Generator, xx) need to be transformed into Generate(generator, ...)
   override lazy val resolved =
     childrenResolved && checkInputDataTypes().isSuccess && !child.isInstanceOf[Generator]
@@ -158,7 +169,7 @@ case class Alias(child: Expression, name: String)(
   /** Just a simple passthrough for code generation. */
   override def genCode(ctx: CodegenContext): ExprCode = child.genCode(ctx)
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    throw new IllegalStateException("Alias.doGenCode should not be called.")
+    throw QueryExecutionErrors.doGenCodeOfAliasShouldNotBeCalledError
   }
 
   override def dataType: DataType = child.dataType
@@ -166,14 +177,21 @@ case class Alias(child: Expression, name: String)(
   override def metadata: Metadata = {
     explicitMetadata.getOrElse {
       child match {
-        case named: NamedExpression => named.metadata
+        case named: NamedExpression =>
+          val builder = new MetadataBuilder().withMetadata(named.metadata)
+          nonInheritableMetadataKeys.foreach(builder.remove)
+          builder.build()
+
         case _ => Metadata.empty
       }
     }
   }
 
   def newInstance(): NamedExpression =
-    Alias(child, name)(qualifier = qualifier, explicitMetadata = explicitMetadata)
+    Alias(child, name)(
+      qualifier = qualifier,
+      explicitMetadata = explicitMetadata,
+      nonInheritableMetadataKeys = nonInheritableMetadataKeys)
 
   override def toAttribute: Attribute = {
     if (resolved) {
@@ -193,7 +211,7 @@ case class Alias(child: Expression, name: String)(
   override def toString: String = s"$child AS $name#${exprId.id}$typeSuffix$delaySuffix"
 
   override protected final def otherCopyArgs: Seq[AnyRef] = {
-    exprId :: qualifier :: explicitMetadata :: Nil
+    exprId :: qualifier :: explicitMetadata :: nonInheritableMetadataKeys :: Nil
   }
 
   override def hashCode(): Int = {
@@ -204,14 +222,24 @@ case class Alias(child: Expression, name: String)(
   override def equals(other: Any): Boolean = other match {
     case a: Alias =>
       name == a.name && exprId == a.exprId && child == a.child && qualifier == a.qualifier &&
-        explicitMetadata == a.explicitMetadata
+        explicitMetadata == a.explicitMetadata &&
+        nonInheritableMetadataKeys == a.nonInheritableMetadataKeys
     case _ => false
   }
 
   override def sql: String = {
-    val qualifierPrefix = if (qualifier.nonEmpty) qualifier.mkString(".") + "." else ""
-    s"${child.sql} AS $qualifierPrefix${quoteIdentifier(name)}"
+    val qualifierPrefix =
+      if (qualifier.nonEmpty) qualifier.map(quoteIfNeeded).mkString(".") + "." else ""
+    s"${child.sql} AS $qualifierPrefix${quoteIfNeeded(name)}"
   }
+
+  override protected def withNewChildInternal(newChild: Expression): Alias =
+    copy(child = newChild)(exprId, qualifier, explicitMetadata, nonInheritableMetadataKeys)
+}
+
+// Singleton tree pattern BitSet for all AttributeReference instances.
+object AttributeReferenceTreeBits {
+  val bits: BitSet = new ImmutableBitSet(TreePattern.maxId, ATTRIBUTE_REFERENCE.id)
 }
 
 /**
@@ -236,6 +264,8 @@ case class AttributeReference(
     val qualifier: Seq[String] = Seq.empty[String])
   extends Attribute with Unevaluable {
 
+  override lazy val treePatternBits: BitSet = AttributeReferenceTreeBits.bits
+
   /**
    * Returns true iff the expression id is the same for both attributes.
    */
@@ -334,8 +364,9 @@ case class AttributeReference(
   }
 
   override def sql: String = {
-    val qualifierPrefix = if (qualifier.nonEmpty) qualifier.mkString(".") + "." else ""
-    s"$qualifierPrefix${quoteIdentifier(name)}"
+    val qualifierPrefix =
+      if (qualifier.nonEmpty) qualifier.map(quoteIfNeeded).mkString(".") + "." else ""
+    s"$qualifierPrefix${quoteIfNeeded(name)}"
   }
 }
 
@@ -388,6 +419,7 @@ case class OuterReference(e: NamedExpression)
   override def exprId: ExprId = e.exprId
   override def toAttribute: Attribute = e.toAttribute
   override def newInstance(): NamedExpression = OuterReference(e.newInstance())
+  final override val nodePatterns: Seq[TreePattern] = Seq(OUTER_REFERENCE)
 }
 
 object VirtualColumn {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
index 09ae2186b2429..d4a02c7fc80fa 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
@@ -21,6 +21,7 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
+import org.apache.spark.sql.catalyst.trees.TreePattern.{NULL_CHECK, TreePattern}
 import org.apache.spark.sql.catalyst.util.TypeUtils
 import org.apache.spark.sql.types._
 
@@ -43,7 +44,8 @@ import org.apache.spark.sql.types._
       > SELECT _FUNC_(NULL, 1, NULL);
        1
   """,
-  since = "1.0.0")
+  since = "1.0.0",
+  group = "conditional_funcs")
 // scalastyle:on line.size.limit
 case class Coalesce(children: Seq[Expression]) extends ComplexTypeMergingExpression {
 
@@ -119,6 +121,9 @@ case class Coalesce(children: Seq[Expression]) extends ComplexTypeMergingExpress
          |} while (false);
        """.stripMargin)
   }
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Coalesce =
+    copy(children = newChildren)
 }
 
 
@@ -129,7 +134,8 @@ case class Coalesce(children: Seq[Expression]) extends ComplexTypeMergingExpress
       > SELECT _FUNC_(NULL, array('2'));
        ["2"]
   """,
-  since = "2.0.0")
+  since = "2.0.0",
+  group = "conditional_funcs")
 case class IfNull(left: Expression, right: Expression, child: Expression)
   extends RuntimeReplaceable {
 
@@ -139,6 +145,8 @@ case class IfNull(left: Expression, right: Expression, child: Expression)
 
   override def flatArguments: Iterator[Any] = Iterator(left, right)
   override def exprsReplaced: Seq[Expression] = Seq(left, right)
+
+  override protected def withNewChildInternal(newChild: Expression): IfNull = copy(child = newChild)
 }
 
 
@@ -149,7 +157,8 @@ case class IfNull(left: Expression, right: Expression, child: Expression)
       > SELECT _FUNC_(2, 2);
        NULL
   """,
-  since = "2.0.0")
+  since = "2.0.0",
+  group = "conditional_funcs")
 case class NullIf(left: Expression, right: Expression, child: Expression)
   extends RuntimeReplaceable {
 
@@ -159,6 +168,8 @@ case class NullIf(left: Expression, right: Expression, child: Expression)
 
   override def flatArguments: Iterator[Any] = Iterator(left, right)
   override def exprsReplaced: Seq[Expression] = Seq(left, right)
+
+  override protected def withNewChildInternal(newChild: Expression): NullIf = copy(child = newChild)
 }
 
 
@@ -169,7 +180,8 @@ case class NullIf(left: Expression, right: Expression, child: Expression)
       > SELECT _FUNC_(NULL, array('2'));
        ["2"]
   """,
-  since = "2.0.0")
+  since = "2.0.0",
+  group = "conditional_funcs")
 case class Nvl(left: Expression, right: Expression, child: Expression) extends RuntimeReplaceable {
 
   def this(left: Expression, right: Expression) = {
@@ -178,6 +190,8 @@ case class Nvl(left: Expression, right: Expression, child: Expression) extends R
 
   override def flatArguments: Iterator[Any] = Iterator(left, right)
   override def exprsReplaced: Seq[Expression] = Seq(left, right)
+
+  override protected def withNewChildInternal(newChild: Expression): Nvl = copy(child = newChild)
 }
 
 
@@ -189,7 +203,8 @@ case class Nvl(left: Expression, right: Expression, child: Expression) extends R
       > SELECT _FUNC_(NULL, 2, 1);
        1
   """,
-  since = "2.0.0")
+  since = "2.0.0",
+  group = "conditional_funcs")
 // scalastyle:on line.size.limit
 case class Nvl2(expr1: Expression, expr2: Expression, expr3: Expression, child: Expression)
   extends RuntimeReplaceable {
@@ -200,6 +215,8 @@ case class Nvl2(expr1: Expression, expr2: Expression, expr3: Expression, child:
 
   override def flatArguments: Iterator[Any] = Iterator(expr1, expr2, expr3)
   override def exprsReplaced: Seq[Expression] = Seq(expr1, expr2, expr3)
+
+  override protected def withNewChildInternal(newChild: Expression): Nvl2 = copy(child = newChild)
 }
 
 
@@ -213,7 +230,8 @@ case class Nvl2(expr1: Expression, expr2: Expression, expr3: Expression, child:
       > SELECT _FUNC_(cast('NaN' as double));
        true
   """,
-  since = "1.5.0")
+  since = "1.5.0",
+  group = "predicate_funcs")
 case class IsNaN(child: Expression) extends UnaryExpression
   with Predicate with ImplicitCastInputTypes {
 
@@ -243,6 +261,8 @@ case class IsNaN(child: Expression) extends UnaryExpression
           ${ev.value} = !${eval.isNull} && Double.isNaN(${eval.value});""", isNull = FalseLiteral)
     }
   }
+
+  override protected def withNewChildInternal(newChild: Expression): IsNaN = copy(child = newChild)
 }
 
 /**
@@ -256,7 +276,8 @@ case class IsNaN(child: Expression) extends UnaryExpression
       > SELECT _FUNC_(cast('NaN' as double), 123);
        123.0
   """,
-  since = "1.5.0")
+  since = "1.5.0",
+  group = "conditional_funcs")
 case class NaNvl(left: Expression, right: Expression)
     extends BinaryExpression with ImplicitCastInputTypes {
 
@@ -304,6 +325,9 @@ case class NaNvl(left: Expression, right: Expression)
           }""")
     }
   }
+
+  override protected def withNewChildrenInternal(newLeft: Expression, newRight: Expression): NaNvl =
+    copy(left = newLeft, right = newRight)
 }
 
 
@@ -317,10 +341,13 @@ case class NaNvl(left: Expression, right: Expression)
       > SELECT _FUNC_(1);
        false
   """,
-  since = "1.0.0")
+  since = "1.0.0",
+  group = "predicate_funcs")
 case class IsNull(child: Expression) extends UnaryExpression with Predicate {
   override def nullable: Boolean = false
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(NULL_CHECK)
+
   override def eval(input: InternalRow): Any = {
     child.eval(input) == null
   }
@@ -331,6 +358,8 @@ case class IsNull(child: Expression) extends UnaryExpression with Predicate {
   }
 
   override def sql: String = s"(${child.sql} IS NULL)"
+
+  override protected def withNewChildInternal(newChild: Expression): IsNull = copy(child = newChild)
 }
 
 
@@ -344,10 +373,13 @@ case class IsNull(child: Expression) extends UnaryExpression with Predicate {
       > SELECT _FUNC_(1);
        true
   """,
-  since = "1.0.0")
+  since = "1.0.0",
+  group = "predicate_funcs")
 case class IsNotNull(child: Expression) extends UnaryExpression with Predicate {
   override def nullable: Boolean = false
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(NULL_CHECK)
+
   override def eval(input: InternalRow): Any = {
     child.eval(input) != null
   }
@@ -365,6 +397,9 @@ case class IsNotNull(child: Expression) extends UnaryExpression with Predicate {
   }
 
   override def sql: String = s"(${child.sql} IS NOT NULL)"
+
+  override protected def withNewChildInternal(newChild: Expression): IsNotNull =
+    copy(child = newChild)
 }
 
 
@@ -374,7 +409,6 @@ case class IsNotNull(child: Expression) extends UnaryExpression with Predicate {
 case class AtLeastNNonNulls(n: Int, children: Seq[Expression]) extends Predicate {
   override def nullable: Boolean = false
   override def foldable: Boolean = children.forall(_.foldable)
-  override def toString: String = s"AtLeastNNulls(n, ${children.mkString(",")})"
 
   private[this] val childrenArray = children.toArray
 
@@ -458,4 +492,7 @@ case class AtLeastNNonNulls(n: Int, children: Seq[Expression]) extends Predicate
          |${CodeGenerator.JAVA_BOOLEAN} ${ev.value} = $nonnull >= $n;
        """.stripMargin, isNull = FalseLiteral)
   }
+
+  override protected def withNewChildrenInternal(
+    newChildren: IndexedSeq[Expression]): AtLeastNNonNulls = copy(children = newChildren)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
index d5de95c65e49e..e871c307aad3e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
@@ -20,9 +20,11 @@ package org.apache.spark.sql.catalyst.expressions.objects
 import java.lang.reflect.{Method, Modifier}
 
 import scala.collection.JavaConverters._
-import scala.collection.mutable.Builder
+import scala.collection.mutable.{Builder, WrappedArray}
 import scala.reflect.ClassTag
-import scala.util.Try
+import scala.util.{Properties, Try}
+
+import org.apache.commons.lang3.reflect.MethodUtils
 
 import org.apache.spark.{SparkConf, SparkEnv}
 import org.apache.spark.serializer._
@@ -32,7 +34,10 @@ import org.apache.spark.sql.catalyst.encoders.RowEncoder
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
+import org.apache.spark.sql.catalyst.trees.TernaryLike
+import org.apache.spark.sql.catalyst.trees.TreePattern._
 import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, GenericArrayData, MapData}
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.types._
 import org.apache.spark.util.Utils
 
@@ -46,6 +51,13 @@ trait InvokeLike extends Expression with NonSQLExpression {
   def propagateNull: Boolean
 
   protected lazy val needNullCheck: Boolean = propagateNull && arguments.exists(_.nullable)
+  protected lazy val evaluatedArgs: Array[Object] = new Array[Object](arguments.length)
+  private lazy val boxingFn: Any => Any =
+    ScalaReflection.typeBoxedJavaMapping
+      .get(dataType)
+      .map(cls => v => cls.cast(v))
+      .getOrElse(identity)
+
 
   /**
    * Prepares codes for arguments.
@@ -116,24 +128,34 @@ trait InvokeLike extends Expression with NonSQLExpression {
    * @param dataType the data type of the return object
    * @return the return object of a method call
    */
-  def invoke(
-      obj: Any,
-      method: Method,
-      arguments: Seq[Expression],
-      input: InternalRow,
-      dataType: DataType): Any = {
-    val args = arguments.map(e => e.eval(input).asInstanceOf[Object])
-    if (needNullCheck && args.exists(_ == null)) {
+  def invoke(obj: Any, method: Method, input: InternalRow): Any = {
+    var i = 0
+    val len = arguments.length
+    while (i < len) {
+      evaluatedArgs(i) = arguments(i).eval(input).asInstanceOf[Object]
+      i += 1
+    }
+    if (needNullCheck && evaluatedArgs.contains(null)) {
       // return null if one of arguments is null
       null
     } else {
-      val ret = method.invoke(obj, args: _*)
-      val boxedClass = ScalaReflection.typeBoxedJavaMapping.get(dataType)
-      if (boxedClass.isDefined) {
-        boxedClass.get.cast(ret)
-      } else {
-        ret
+      val ret = try {
+        method.invoke(obj, evaluatedArgs: _*)
+      } catch {
+        // Re-throw the original exception.
+        case e: java.lang.reflect.InvocationTargetException if e.getCause != null =>
+          throw e.getCause
       }
+      boxingFn(ret)
+    }
+  }
+
+  final def findMethod(cls: Class[_], functionName: String, argClasses: Seq[Class[_]]): Method = {
+    val method = MethodUtils.getMatchingAccessibleMethod(cls, functionName, argClasses: _*)
+    if (method == null) {
+      throw QueryExecutionErrors.methodNotDeclaredError(functionName)
+    } else {
+      method
     }
   }
 }
@@ -227,10 +249,10 @@ case class StaticInvoke(
   override def children: Seq[Expression] = arguments
 
   lazy val argClasses = ScalaReflection.expressionJavaClasses(arguments)
-  @transient lazy val method = cls.getDeclaredMethod(functionName, argClasses : _*)
+  @transient lazy val method = findMethod(cls, functionName, argClasses)
 
   override def eval(input: InternalRow): Any = {
-    invoke(null, method, arguments, input, dataType)
+    invoke(null, method, input)
   }
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
@@ -247,7 +269,7 @@ case class StaticInvoke(
       ""
     }
 
-    val evaluate = if (returnNullable) {
+    val evaluate = if (returnNullable && !method.getReturnType.isPrimitive) {
       if (CodeGenerator.defaultValue(dataType) == "null") {
         s"""
           ${ev.value} = $callFunc;
@@ -277,6 +299,9 @@ case class StaticInvoke(
      """
     ev.copy(code = code)
   }
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
+    copy(arguments = newChildren)
 }
 
 /**
@@ -314,12 +339,7 @@ case class Invoke(
 
   @transient lazy val method = targetObject.dataType match {
     case ObjectType(cls) =>
-      val m = cls.getMethods.find(_.getName == encodedFunctionName)
-      if (m.isEmpty) {
-        sys.error(s"Couldn't find $encodedFunctionName on $cls")
-      } else {
-        m
-      }
+      Some(findMethod(cls, encodedFunctionName, argClasses))
     case _ => None
   }
 
@@ -332,9 +352,9 @@ case class Invoke(
       val invokeMethod = if (method.isDefined) {
         method.get
       } else {
-        obj.getClass.getDeclaredMethod(functionName, argClasses: _*)
+        obj.getClass.getMethod(functionName, argClasses: _*)
       }
-      invoke(obj, invokeMethod, arguments, input, dataType)
+      invoke(obj, invokeMethod, input)
     }
   }
 
@@ -383,21 +403,37 @@ case class Invoke(
       """
     }
 
+    val mainEvalCode =
+      code"""
+         |$argCode
+         |${ev.isNull} = $resultIsNull;
+         |if (!${ev.isNull}) {
+         |  $evaluate
+         |}
+         |""".stripMargin
+
+    val evalWithNullCheck = if (targetObject.nullable) {
+      code"""
+         |if (!${obj.isNull}) {
+         |  $mainEvalCode
+         |}
+         |""".stripMargin
+    } else {
+      mainEvalCode
+    }
+
     val code = obj.code + code"""
       boolean ${ev.isNull} = true;
       $javaType ${ev.value} = ${CodeGenerator.defaultValue(dataType)};
-      if (!${obj.isNull}) {
-        $argCode
-        ${ev.isNull} = $resultIsNull;
-        if (!${ev.isNull}) {
-          $evaluate
-        }
-      }
+      $evalWithNullCheck
      """
     ev.copy(code = code)
   }
 
   override def toString: String = s"$targetObject.$functionName"
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Invoke =
+    copy(targetObject = newChildren.head, arguments = newChildren.tail)
 }
 
 object NewInstance {
@@ -443,7 +479,7 @@ case class NewInstance(
     // Note that static inner classes (e.g., inner classes within Scala objects) don't need
     // outer pointer registration.
     val needOuterPointer =
-      outerPointer.isEmpty && cls.isMemberClass && !Modifier.isStatic(cls.getModifiers)
+      outerPointer.isEmpty && Utils.isMemberClass(cls) && !Modifier.isStatic(cls.getModifiers)
     childrenResolved && !needOuterPointer
   }
 
@@ -451,12 +487,11 @@ case class NewInstance(
     val paramTypes = ScalaReflection.expressionJavaClasses(arguments)
     val getConstructor = (paramClazz: Seq[Class[_]]) => {
       ScalaReflection.findConstructor(cls, paramClazz).getOrElse {
-        sys.error(s"Couldn't find a valid constructor on $cls")
+        throw QueryExecutionErrors.constructorNotFoundError(cls.toString)
       }
     }
     outerPointer.map { p =>
       val outerObj = p()
-      val d = outerObj.getClass +: paramTypes
       val c = getConstructor(outerObj.getClass +: paramTypes)
       (args: Seq[AnyRef]) => {
         c(outerObj +: args)
@@ -489,7 +524,7 @@ case class NewInstance(
       // that might be defined on the companion object.
       case 0 => s"$className$$.MODULE$$.apply($argString)"
       case _ => outer.map { gen =>
-        s"${gen.value}.new ${cls.getSimpleName}($argString)"
+        s"${gen.value}.new ${Utils.getSimpleName(cls)}($argString)"
       }.getOrElse {
         s"new $className($argString)"
       }
@@ -505,6 +540,9 @@ case class NewInstance(
   }
 
   override def toString: String = s"newInstance($cls)"
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): NewInstance =
+    copy(arguments = newChildren)
 }
 
 /**
@@ -542,6 +580,9 @@ case class UnwrapOption(
     """
     ev.copy(code = code)
   }
+
+  override protected def withNewChildInternal(newChild: Expression): UnwrapOption =
+    copy(child = newChild)
 }
 
 /**
@@ -572,6 +613,9 @@ case class WrapOption(child: Expression, optType: DataType)
     """
     ev.copy(code = code, isNull = FalseLiteral)
   }
+
+  override protected def withNewChildInternal(newChild: Expression): WrapOption =
+    copy(child = newChild)
 }
 
 object LambdaVariable {
@@ -614,6 +658,8 @@ case class LambdaVariable(
 
   private val accessor: (InternalRow, Int) => Any = InternalRow.getAccessor(dataType, nullable)
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(LAMBDA_VARIABLE)
+
   // Interpreted execution of `LambdaVariable` always get the 0-index element from input row.
   override def eval(input: InternalRow): Any = {
     assert(input.numFields == 1,
@@ -656,8 +702,11 @@ case class UnresolvedMapObjects(
   override lazy val resolved = false
 
   override def dataType: DataType = customCollectionCls.map(ObjectType.apply).getOrElse {
-    throw new UnsupportedOperationException("not resolved")
+    throw QueryExecutionErrors.customCollectionClsNotResolvedError
   }
+
+  override protected def withNewChildInternal(newChild: Expression): UnresolvedMapObjects =
+    copy(child = newChild)
 }
 
 object MapObjects {
@@ -678,6 +727,13 @@ object MapObjects {
       elementType: DataType,
       elementNullable: Boolean = true,
       customCollectionCls: Option[Class[_]] = None): MapObjects = {
+    // UnresolvedMapObjects does not serialize its 'function' field.
+    // If an array expression or array Encoder is not correctly resolved before
+    // serialization, this exception condition may occur.
+    require(function != null,
+      "MapObjects applied with a null function. " +
+      "Likely cause is failure to resolve an array expression or encoder. " +
+      "(See UnresolvedMapObjects)")
     val loopVar = LambdaVariable("MapObject", elementType, elementNullable)
     MapObjects(loopVar, function(loopVar), inputData, customCollectionCls)
   }
@@ -707,11 +763,16 @@ case class MapObjects private(
     loopVar: LambdaVariable,
     lambdaFunction: Expression,
     inputData: Expression,
-    customCollectionCls: Option[Class[_]]) extends Expression with NonSQLExpression {
+    customCollectionCls: Option[Class[_]]) extends Expression with NonSQLExpression
+  with TernaryLike[Expression] {
 
   override def nullable: Boolean = inputData.nullable
 
-  override def children: Seq[Expression] = Seq(loopVar, lambdaFunction, inputData)
+  override def first: Expression = loopVar
+  override def second: Expression = lambdaFunction
+  override def third: Expression = inputData
+
+  final override val nodePatterns: Seq[TreePattern] = Seq(MAP_OBJECTS)
 
   // The data with UserDefinedType are actually stored with the data type of its sqlType.
   // When we want to apply MapObjects on it, we have to use it.
@@ -729,12 +790,12 @@ case class MapObjects private(
   }
 
   private lazy val convertToSeq: Any => Seq[_] = inputDataType match {
-    case ObjectType(cls) if classOf[Seq[_]].isAssignableFrom(cls) =>
-      _.asInstanceOf[Seq[_]]
+    case ObjectType(cls) if classOf[scala.collection.Seq[_]].isAssignableFrom(cls) =>
+      _.asInstanceOf[scala.collection.Seq[_]].toSeq
     case ObjectType(cls) if cls.isArray =>
       _.asInstanceOf[Array[_]].toSeq
     case ObjectType(cls) if classOf[java.util.List[_]].isAssignableFrom(cls) =>
-      _.asInstanceOf[java.util.List[_]].asScala
+      _.asInstanceOf[java.util.List[_]].asScala.toSeq
     case ObjectType(cls) if cls == classOf[Object] =>
       (inputCollection) => {
         if (inputCollection.getClass.isArray) {
@@ -748,7 +809,10 @@ case class MapObjects private(
   }
 
   private lazy val mapElements: Seq[_] => Any = customCollectionCls match {
-    case Some(cls) if classOf[Seq[_]].isAssignableFrom(cls) =>
+    case Some(cls) if classOf[WrappedArray[_]].isAssignableFrom(cls) =>
+      // Scala WrappedArray
+      inputCollection => WrappedArray.make(executeFuncOnCollection(inputCollection).toArray)
+    case Some(cls) if classOf[scala.collection.Seq[_]].isAssignableFrom(cls) =>
       // Scala sequence
       executeFuncOnCollection(_).toSeq
     case Some(cls) if classOf[scala.collection.Set[_]].isAssignableFrom(cls) =>
@@ -787,8 +851,7 @@ case class MapObjects private(
       // array
       x => new GenericArrayData(executeFuncOnCollection(x).toArray)
     case Some(cls) =>
-      throw new RuntimeException(s"class `${cls.getName}` is not supported by `MapObjects` as " +
-        "resulting collection.")
+      throw QueryExecutionErrors.classUnsupportedByMapObjectsError(cls)
   }
 
   override def eval(input: InternalRow): Any = {
@@ -832,7 +895,7 @@ case class MapObjects private(
     val array = ctx.freshName("array")
     val determineCollectionType = inputData.dataType match {
       case ObjectType(cls) if cls == classOf[Object] =>
-        val seqClass = classOf[Seq[_]].getName
+        val seqClass = classOf[scala.collection.Seq[_]].getName
         s"""
           $seqClass $seq = null;
           $elementJavaType[] $array = null;
@@ -849,7 +912,7 @@ case class MapObjects private(
     // need to take care of Seq and List because they may have O(n) complexity for indexed accessing
     // like `list.get(1)`. Here we use Iterator to traverse Seq and List.
     val (getLength, prepareLoop, getLoopVar) = inputDataType match {
-      case ObjectType(cls) if classOf[Seq[_]].isAssignableFrom(cls) =>
+      case ObjectType(cls) if classOf[scala.collection.Seq[_]].isAssignableFrom(cls) =>
         val it = ctx.freshName("it")
         (
           s"${genInputData.value}.size()",
@@ -905,7 +968,46 @@ case class MapObjects private(
 
     val (initCollection, addElement, getResult): (String, String => String, String) =
       customCollectionCls match {
-        case Some(cls) if classOf[Seq[_]].isAssignableFrom(cls) ||
+        case Some(cls) if classOf[WrappedArray[_]].isAssignableFrom(cls) =>
+          def doCodeGenForScala212 = {
+            // WrappedArray in Scala 2.12
+            val getBuilder = s"${cls.getName}$$.MODULE$$.newBuilder()"
+            val builder = ctx.freshName("collectionBuilder")
+            (
+              s"""
+                 ${classOf[Builder[_, _]].getName} $builder = $getBuilder;
+                 $builder.sizeHint($dataLength);
+               """,
+              (genValue: String) => s"$builder.$$plus$$eq($genValue);",
+              s"(${cls.getName}) ${classOf[WrappedArray[_]].getName}$$." +
+                s"MODULE$$.make(((${classOf[IndexedSeq[_]].getName})$builder" +
+                s".result()).toArray(scala.reflect.ClassTag$$.MODULE$$.Object()));"
+            )
+          }
+
+          def doCodeGenForScala213 = {
+            // In Scala 2.13, WrappedArray is mutable.ArraySeq and newBuilder method need
+            // a ClassTag type construction parameter
+            val getBuilder = s"${cls.getName}$$.MODULE$$.newBuilder(" +
+              s"scala.reflect.ClassTag$$.MODULE$$.Object())"
+            val builder = ctx.freshName("collectionBuilder")
+            (
+              s"""
+                 ${classOf[Builder[_, _]].getName} $builder = $getBuilder;
+                 $builder.sizeHint($dataLength);
+               """,
+              (genValue: String) => s"$builder.$$plus$$eq($genValue);",
+              s"(${cls.getName})$builder.result();"
+            )
+          }
+
+          val scalaVersion = Properties.versionNumberString
+          if (scalaVersion.startsWith("2.12")) {
+            doCodeGenForScala212
+          } else {
+            doCodeGenForScala213
+          }
+        case Some(cls) if classOf[scala.collection.Seq[_]].isAssignableFrom(cls) ||
           classOf[scala.collection.Set[_]].isAssignableFrom(cls) =>
           // Scala sequence or set
           val getBuilder = s"${cls.getName}$$.MODULE$$.newBuilder()"
@@ -932,7 +1034,7 @@ case class MapObjects private(
             (genValue: String) => s"$builder.add($genValue);",
             s"$builder;"
           )
-        case None =>
+        case _ =>
           // array
           (
             s"""
@@ -973,6 +1075,13 @@ case class MapObjects private(
     """
     ev.copy(code = code, isNull = genInputData.isNull)
   }
+
+  override protected def withNewChildrenInternal(
+      newFirst: Expression, newSecond: Expression, newThird: Expression): Expression =
+    copy(
+      loopVar = newFirst.asInstanceOf[LambdaVariable],
+      lambdaFunction = newSecond,
+      inputData = newThird)
 }
 
 /**
@@ -992,6 +1101,9 @@ case class UnresolvedCatalystToExternalMap(
   override lazy val resolved = false
 
   override def dataType: DataType = ObjectType(collClass)
+
+  override protected def withNewChildInternal(
+    newChild: Expression): UnresolvedCatalystToExternalMap = copy(child = newChild)
 }
 
 object CatalystToExternalMap {
@@ -1162,6 +1274,15 @@ case class CatalystToExternalMap private(
     """
     ev.copy(code = code, isNull = genInputData.isNull)
   }
+
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[Expression]): CatalystToExternalMap =
+    copy(
+      keyLoopVar = newChildren(0).asInstanceOf[LambdaVariable],
+      keyLambdaFunction = newChildren(1),
+      valueLoopVar = newChildren(2).asInstanceOf[LambdaVariable],
+      valueLambdaFunction = newChildren(3),
+      inputData = newChildren(4))
 }
 
 object ExternalMapToCatalyst {
@@ -1238,7 +1359,7 @@ case class ExternalMapToCatalyst private(
             keys(i) = if (key != null) {
               keyConverter.eval(rowWrapper(key))
             } else {
-              throw new RuntimeException("Cannot use null as map key!")
+              throw QueryExecutionErrors.nullAsMapKeyNotAllowedError
             }
             values(i) = if (value != null) {
               valueConverter.eval(rowWrapper(value))
@@ -1260,7 +1381,7 @@ case class ExternalMapToCatalyst private(
             keys(i) = if (key != null) {
               keyConverter.eval(rowWrapper(key))
             } else {
-              throw new RuntimeException("Cannot use null as map key!")
+              throw QueryExecutionErrors.nullAsMapKeyNotAllowedError
             }
             values(i) = if (value != null) {
               valueConverter.eval(rowWrapper(value))
@@ -1365,7 +1486,7 @@ case class ExternalMapToCatalyst private(
 
             ${genKeyConverter.code}
             if (${genKeyConverter.isNull}) {
-              throw new RuntimeException("Cannot use null as map key!");
+              throw QueryExecutionErrors.nullAsMapKeyNotAllowedError();
             } else {
               $convertedKeys[$index] = ($convertedKeyType) ${genKeyConverter.value};
             }
@@ -1385,6 +1506,15 @@ case class ExternalMapToCatalyst private(
       """
     ev.copy(code = code, isNull = inputMap.isNull)
   }
+
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[Expression]): ExternalMapToCatalyst =
+    copy(
+      keyLoopVar = newChildren(0).asInstanceOf[LambdaVariable],
+      keyConverter = newChildren(1),
+      valueLoopVar = newChildren(2).asInstanceOf[LambdaVariable],
+      valueConverter = newChildren(3),
+      inputData = newChildren(4))
 }
 
 /**
@@ -1435,6 +1565,9 @@ case class CreateExternalRow(children: Seq[Expression], schema: StructType)
        """.stripMargin
     ev.copy(code = code, isNull = FalseLiteral)
   }
+
+  override protected def withNewChildrenInternal(
+    newChildren: IndexedSeq[Expression]): CreateExternalRow = copy(children = newChildren)
 }
 
 /**
@@ -1464,6 +1597,9 @@ case class EncodeUsingSerializer(child: Expression, kryo: Boolean)
   }
 
   override def dataType: DataType = BinaryType
+
+  override protected def withNewChildInternal(newChild: Expression): EncodeUsingSerializer =
+    copy(child = newChild)
 }
 
 /**
@@ -1496,6 +1632,9 @@ case class DecodeUsingSerializer[T](child: Expression, tag: ClassTag[T], kryo: B
   }
 
   override def dataType: DataType = ObjectType(tag.runtimeClass)
+
+  override protected def withNewChildInternal(newChild: Expression): DecodeUsingSerializer[T] =
+    copy(child = newChild)
 }
 
 /**
@@ -1525,8 +1664,7 @@ case class InitializeJavaBean(beanInstance: Expression, setters: Map[String, Exp
           }
         }
         if (methods.isEmpty) {
-          throw new NoSuchMethodException(s"""A method named "$name" is not declared """ +
-            "in any enclosing class nor any supertype")
+          throw QueryExecutionErrors.methodNotDeclaredError(name)
         }
         methods.head -> expr
     }
@@ -1578,6 +1716,10 @@ case class InitializeJavaBean(beanInstance: Expression, setters: Map[String, Exp
        """.stripMargin
     ev.copy(code = code, isNull = instanceGen.isNull, value = instanceGen.value)
   }
+
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[Expression]): InitializeJavaBean =
+    super.legacyWithNewChildren(newChildren).asInstanceOf[InitializeJavaBean]
 }
 
 /**
@@ -1595,6 +1737,8 @@ case class AssertNotNull(child: Expression, walkedTypePath: Seq[String] = Nil)
   override def foldable: Boolean = false
   override def nullable: Boolean = false
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(NULL_CHECK)
+
   override def flatArguments: Iterator[Any] = Iterator(child)
 
   private val errMsg = "Null value appeared in non-nullable field:" +
@@ -1625,6 +1769,9 @@ case class AssertNotNull(child: Expression, walkedTypePath: Seq[String] = Nil)
      """
     ev.copy(code = code, isNull = FalseLiteral, value = childGen.value)
   }
+
+  override protected def withNewChildInternal(newChild: Expression): AssertNotNull =
+    copy(child = newChild)
 }
 
 /**
@@ -1643,15 +1790,15 @@ case class GetExternalRowField(
 
   override def dataType: DataType = ObjectType(classOf[Object])
 
-  private val errMsg = s"The ${index}th field '$fieldName' of input row cannot be null."
+  private val errMsg = QueryExecutionErrors.fieldCannotBeNullMsg(index, fieldName)
 
   override def eval(input: InternalRow): Any = {
     val inputRow = child.eval(input).asInstanceOf[Row]
     if (inputRow == null) {
-      throw new RuntimeException("The input external row cannot be null.")
+      throw QueryExecutionErrors.inputExternalRowCannotBeNullError
     }
     if (inputRow.isNullAt(index)) {
-      throw new RuntimeException(errMsg)
+      throw QueryExecutionErrors.fieldCannotBeNullError(index, fieldName)
     }
     inputRow.get(index)
   }
@@ -1665,7 +1812,7 @@ case class GetExternalRowField(
       ${row.code}
 
       if (${row.isNull}) {
-        throw new RuntimeException("The input external row cannot be null.");
+        throw QueryExecutionErrors.inputExternalRowCannotBeNullError();
       }
 
       if (${row.value}.isNullAt($index)) {
@@ -1676,6 +1823,9 @@ case class GetExternalRowField(
      """
     ev.copy(code = code, isNull = FalseLiteral)
   }
+
+  override protected def withNewChildInternal(newChild: Expression): GetExternalRowField =
+    copy(child = newChild)
 }
 
 /**
@@ -1731,7 +1881,7 @@ case class ValidateExternalType(child: Expression, expected: DataType)
         Seq(classOf[java.math.BigDecimal], classOf[scala.math.BigDecimal], classOf[Decimal])
           .map(cls => s"$obj instanceof ${cls.getName}").mkString(" || ")
       case _: ArrayType =>
-        s"$obj.getClass().isArray() || $obj instanceof ${classOf[Seq[_]].getName}"
+        s"$obj.getClass().isArray() || $obj instanceof ${classOf[scala.collection.Seq[_]].getName}"
       case _ =>
         s"$obj instanceof ${CodeGenerator.boxedType(dataType)}"
     }
@@ -1750,4 +1900,7 @@ case class ValidateExternalType(child: Expression, expected: DataType)
     """
     ev.copy(code = code, isNull = input.isNull)
   }
+
+  override protected def withNewChildInternal(newChild: Expression): ValidateExternalType =
+    copy(child = newChild)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ordering.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ordering.scala
index f387e1774edbd..ba3ed02e06ef1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ordering.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ordering.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.expressions
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.BindReferences.bindReferences
 import org.apache.spark.sql.catalyst.expressions.codegen.GenerateOrdering
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.types._
 
 
@@ -69,7 +70,7 @@ class InterpretedOrdering(ordering: Seq[SortOrder]) extends BaseOrdering {
           case s: StructType if order.direction == Descending =>
             - s.interpretedOrdering.asInstanceOf[Ordering[Any]].compare(left, right)
           case other =>
-            throw new IllegalArgumentException(s"Type $other does not support ordered operations")
+            throw QueryExecutionErrors.orderedOperationUnsupportedByDataTypeError(other)
         }
         if (comparison != 0) {
           return comparison
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala
index 8bf1f19844556..d950fef3b26a5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala
@@ -124,7 +124,7 @@ package object expressions  {
     }
 
     private def unique[T](m: Map[T, Seq[Attribute]]): Map[T, Seq[Attribute]] = {
-      m.mapValues(_.distinct).map(identity)
+      m.mapValues(_.distinct).toMap
     }
 
     /** Map to use for direct case insensitive attribute lookups. */
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
index 2c4f41f98ac20..4885f7761ff9a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
@@ -19,14 +19,14 @@ package org.apache.spark.sql.catalyst.expressions
 
 import scala.collection.immutable.TreeSet
 
-import org.apache.spark.sql.catalyst.CatalystTypeConverters.convertToScala
+import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.expressions.BindReferences.bindReference
-import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
 import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LeafNode, LogicalPlan, Project}
+import org.apache.spark.sql.catalyst.trees.TreePattern._
 import org.apache.spark.sql.catalyst.util.TypeUtils
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
@@ -47,11 +47,26 @@ abstract class BasePredicate {
 }
 
 case class InterpretedPredicate(expression: Expression) extends BasePredicate {
-  override def eval(r: InternalRow): Boolean = expression.eval(r).asInstanceOf[Boolean]
+  private[this] val subExprEliminationEnabled = SQLConf.get.subexpressionEliminationEnabled
+  private[this] lazy val runtime =
+    new SubExprEvaluationRuntime(SQLConf.get.subexpressionEliminationCacheMaxEntries)
+  private[this] val expr = if (subExprEliminationEnabled) {
+    runtime.proxyExpressions(Seq(expression)).head
+  } else {
+    expression
+  }
+
+  override def eval(r: InternalRow): Boolean = {
+    if (subExprEliminationEnabled) {
+      runtime.setInput(r)
+    }
+
+    expr.eval(r).asInstanceOf[Boolean]
+  }
 
   override def initialize(partitionIndex: Int): Unit = {
     super.initialize(partitionIndex)
-    expression.foreach {
+    expr.foreach {
       case n: Nondeterministic => n.initialize(partitionIndex)
       case _ =>
     }
@@ -71,7 +86,7 @@ trait Predicate extends Expression {
 object Predicate extends CodeGeneratorWithInterpretedFallback[Expression, BasePredicate] {
 
   override protected def createCodeGeneratedObject(in: Expression): BasePredicate = {
-    GeneratePredicate.generate(in)
+    GeneratePredicate.generate(in, SQLConf.get.subexpressionEliminationEnabled)
   }
 
   override protected def createInterpretedObject(in: Expression): BasePredicate = {
@@ -95,7 +110,7 @@ object Predicate extends CodeGeneratorWithInterpretedFallback[Expression, BasePr
   }
 }
 
-trait PredicateHelper {
+trait PredicateHelper extends AliasHelper with Logging {
   protected def splitConjunctivePredicates(condition: Expression): Seq[Expression] = {
     condition match {
       case And(cond1, cond2) =>
@@ -115,18 +130,13 @@ trait PredicateHelper {
       plan: LogicalPlan): Option[(Expression, LogicalPlan)] = {
 
     plan match {
-      case Project(projectList, child) =>
-        val aliases = AttributeMap(projectList.collect {
-          case a @ Alias(child, _) => (a.toAttribute, child)
-        })
-        findExpressionAndTrackLineageDown(replaceAlias(exp, aliases), child)
+      case p: Project =>
+        val aliases = getAliasMap(p)
+        findExpressionAndTrackLineageDown(replaceAlias(exp, aliases), p.child)
       // we can unwrap only if there are row projections, and no aggregation operation
-      case Aggregate(_, aggregateExpressions, child) =>
-        val aliasMap = AttributeMap(aggregateExpressions.collect {
-          case a: Alias if a.child.find(_.isInstanceOf[AggregateExpression]).isEmpty =>
-            (a.toAttribute, a.child)
-        })
-        findExpressionAndTrackLineageDown(replaceAlias(exp, aliasMap), child)
+      case a: Aggregate =>
+        val aliasMap = getAliasMap(a)
+        findExpressionAndTrackLineageDown(replaceAlias(exp, aliasMap), a.child)
       case l: LeafNode if exp.references.subsetOf(l.outputSet) =>
         Some((exp, l))
       case other =>
@@ -148,16 +158,30 @@ trait PredicateHelper {
     }
   }
 
-  // Substitute any known alias from a map.
-  protected def replaceAlias(
-      condition: Expression,
-      aliases: AttributeMap[Expression]): Expression = {
-    // Use transformUp to prevent infinite recursion when the replacement expression
-    // redefines the same ExprId,
-    condition.transformUp {
-      case a: Attribute =>
-        aliases.getOrElse(a, a)
+  /**
+   * Builds a balanced output predicate in bottom up approach, by applying binary operator op
+   * pair by pair on input predicates exprs recursively.
+   * Example:  exprs = [a, b, c, d], op = And, returns (a And b) And (c And d)
+   * exprs = [a, b, c, d, e, f], op = And, returns ((a And b) And (c And d)) And (e And f)
+   */
+  protected def buildBalancedPredicate(
+      expressions: Seq[Expression], op: (Expression, Expression) => Expression): Expression = {
+    assert(expressions.nonEmpty)
+    var currentResult = expressions
+    while (currentResult.size != 1) {
+      var i = 0
+      val nextResult = new Array[Expression](currentResult.size / 2 + currentResult.size % 2)
+      while (i < currentResult.size) {
+        nextResult(i / 2) = if (i + 1 == currentResult.size) {
+          currentResult(i)
+        } else {
+          op(currentResult(i), currentResult(i + 1))
+        }
+        i += 2
+      }
+      currentResult = nextResult
     }
+    currentResult.head
   }
 
   /**
@@ -198,10 +222,86 @@ trait PredicateHelper {
     case e: Unevaluable => false
     case e => e.children.forall(canEvaluateWithinJoin)
   }
+
+  /**
+   * Returns a filter that its reference is a subset of `outputSet` and it contains the maximum
+   * constraints from `condition`. This is used for predicate pushdown.
+   * When there is no such filter, `None` is returned.
+   */
+  protected def extractPredicatesWithinOutputSet(
+      condition: Expression,
+      outputSet: AttributeSet): Option[Expression] = condition match {
+    case And(left, right) =>
+      val leftResultOptional = extractPredicatesWithinOutputSet(left, outputSet)
+      val rightResultOptional = extractPredicatesWithinOutputSet(right, outputSet)
+      (leftResultOptional, rightResultOptional) match {
+        case (Some(leftResult), Some(rightResult)) => Some(And(leftResult, rightResult))
+        case (Some(leftResult), None) => Some(leftResult)
+        case (None, Some(rightResult)) => Some(rightResult)
+        case _ => None
+      }
+
+    // The Or predicate is convertible when both of its children can be pushed down.
+    // That is to say, if one/both of the children can be partially pushed down, the Or
+    // predicate can be partially pushed down as well.
+    //
+    // Here is an example used to explain the reason.
+    // Let's say we have
+    // condition: (a1 AND a2) OR (b1 AND b2),
+    // outputSet: AttributeSet(a1, b1)
+    // a1 and b1 is convertible, while a2 and b2 is not.
+    // The predicate can be converted as
+    // (a1 OR b1) AND (a1 OR b2) AND (a2 OR b1) AND (a2 OR b2)
+    // As per the logical in And predicate, we can push down (a1 OR b1).
+    case Or(left, right) =>
+      for {
+        lhs <- extractPredicatesWithinOutputSet(left, outputSet)
+        rhs <- extractPredicatesWithinOutputSet(right, outputSet)
+      } yield Or(lhs, rhs)
+
+    // Here we assume all the `Not` operators is already below all the `And` and `Or` operators
+    // after the optimization rule `BooleanSimplification`, so that we don't need to handle the
+    // `Not` operators here.
+    case other =>
+      if (other.references.subsetOf(outputSet)) {
+        Some(other)
+      } else {
+        None
+      }
+  }
+
+  // If one expression and its children are null intolerant, it is null intolerant.
+  protected def isNullIntolerant(expr: Expression): Boolean = expr match {
+    case e: NullIntolerant => e.children.forall(isNullIntolerant)
+    case _ => false
+  }
+
+  protected def outputWithNullability(
+      output: Seq[Attribute],
+      nonNullAttrExprIds: Seq[ExprId]): Seq[Attribute] = {
+    output.map { a =>
+      if (a.nullable && nonNullAttrExprIds.contains(a.exprId)) {
+        a.withNullability(false)
+      } else {
+        a
+      }
+    }
+  }
 }
 
 @ExpressionDescription(
-  usage = "_FUNC_ expr - Logical not.")
+  usage = "_FUNC_ expr - Logical not.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_ true;
+       false
+      > SELECT _FUNC_ false;
+       true
+      > SELECT _FUNC_ NULL;
+       NULL
+  """,
+  since = "1.0.0",
+  group = "predicate_funcs")
 case class Not(child: Expression)
   extends UnaryExpression with Predicate with ImplicitCastInputTypes with NullIntolerant {
 
@@ -209,6 +309,8 @@ case class Not(child: Expression)
 
   override def inputTypes: Seq[DataType] = Seq(BooleanType)
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(NOT)
+
   // +---------+-----------+
   // | CHILD   | NOT CHILD |
   // +---------+-----------+
@@ -223,6 +325,8 @@ case class Not(child: Expression)
   }
 
   override def sql: String = s"(NOT ${child.sql})"
+
+  override protected def withNewChildInternal(newChild: Expression): Not = copy(child = newChild)
 }
 
 /**
@@ -240,6 +344,7 @@ case class InSubquery(values: Seq[Expression], query: ListQuery)
     values.head
   }
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(IN_SUBQUERY)
 
   override def checkInputDataTypes(): TypeCheckResult = {
     if (values.length != query.childOutputs.length) {
@@ -278,9 +383,11 @@ case class InSubquery(values: Seq[Expression], query: ListQuery)
 
   override def children: Seq[Expression] = values :+ query
   override def nullable: Boolean = children.exists(_.nullable)
-  override def foldable: Boolean = children.forall(_.foldable)
   override def toString: String = s"$value IN ($query)"
   override def sql: String = s"(${value.sql} IN (${query.sql}))"
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): InSubquery =
+    copy(values = newChildren.dropRight(1), query = newChildren.last.asInstanceOf[ListQuery])
 }
 
 
@@ -304,7 +411,9 @@ case class InSubquery(values: Seq[Expression], query: ListQuery)
        false
       > SELECT named_struct('a', 1, 'b', 2) _FUNC_(named_struct('a', 1, 'b', 2), named_struct('a', 1, 'b', 3));
        true
-  """)
+  """,
+  since = "1.0.0",
+  group = "predicate_funcs")
 // scalastyle:on line.size.limit
 case class In(value: Expression, list: Seq[Expression]) extends Predicate {
 
@@ -328,6 +437,8 @@ case class In(value: Expression, list: Seq[Expression]) extends Predicate {
   override def nullable: Boolean = children.exists(_.nullable)
   override def foldable: Boolean = children.forall(_.foldable)
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(IN)
+
   override def toString: String = s"$value IN ${list.mkString("(", ",", ")")}"
 
   override def eval(input: InternalRow): Any = {
@@ -420,6 +531,9 @@ case class In(value: Expression, list: Seq[Expression]) extends Predicate {
     val listSQL = list.map(_.sql).mkString(", ")
     s"($valueSQL IN ($listSQL))"
   }
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): In =
+    copy(value = newChildren.head, list = newChildren.tail)
 }
 
 /**
@@ -436,6 +550,8 @@ case class InSet(child: Expression, hset: Set[Any]) extends UnaryExpression with
 
   override def nullable: Boolean = child.nullable || hasNull
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(INSET)
+
   protected override def nullSafeEval(value: Any): Any = {
     if (set.contains(value)) {
       true
@@ -525,10 +641,25 @@ case class InSet(child: Expression, hset: Set[Any]) extends UnaryExpression with
       .mkString(", ")
     s"($valueSQL IN ($listSQL))"
   }
+
+  override protected def withNewChildInternal(newChild: Expression): InSet = copy(child = newChild)
 }
 
 @ExpressionDescription(
-  usage = "expr1 _FUNC_ expr2 - Logical AND.")
+  usage = "expr1 _FUNC_ expr2 - Logical AND.",
+  examples = """
+    Examples:
+      > SELECT true _FUNC_ true;
+       true
+      > SELECT true _FUNC_ false;
+       false
+      > SELECT true _FUNC_ NULL;
+       NULL
+      > SELECT false _FUNC_ NULL;
+       false
+  """,
+  since = "1.0.0",
+  group = "predicate_funcs")
 case class And(left: Expression, right: Expression) extends BinaryOperator with Predicate {
 
   override def inputType: AbstractDataType = BooleanType
@@ -537,6 +668,8 @@ case class And(left: Expression, right: Expression) extends BinaryOperator with
 
   override def sqlOperator: String = "AND"
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(AND_OR)
+
   // +---------+---------+---------+---------+
   // | AND     | TRUE    | FALSE   | UNKNOWN |
   // +---------+---------+---------+---------+
@@ -595,10 +728,26 @@ case class And(left: Expression, right: Expression) extends BinaryOperator with
       """)
     }
   }
+
+  override protected def withNewChildrenInternal(newLeft: Expression, newRight: Expression): And =
+    copy(left = newLeft, right = newRight)
 }
 
 @ExpressionDescription(
-  usage = "expr1 _FUNC_ expr2 - Logical OR.")
+  usage = "expr1 _FUNC_ expr2 - Logical OR.",
+  examples = """
+    Examples:
+      > SELECT true _FUNC_ false;
+       true
+      > SELECT false _FUNC_ false;
+       false
+      > SELECT true _FUNC_ NULL;
+       true
+      > SELECT false _FUNC_ NULL;
+       NULL
+  """,
+  since = "1.0.0",
+  group = "predicate_funcs")
 case class Or(left: Expression, right: Expression) extends BinaryOperator with Predicate {
 
   override def inputType: AbstractDataType = BooleanType
@@ -607,6 +756,8 @@ case class Or(left: Expression, right: Expression) extends BinaryOperator with P
 
   override def sqlOperator: String = "OR"
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(AND_OR)
+
   // +---------+---------+---------+---------+
   // | OR      | TRUE    | FALSE   | UNKNOWN |
   // +---------+---------+---------+---------+
@@ -666,6 +817,9 @@ case class Or(left: Expression, right: Expression) extends BinaryOperator with P
       """)
     }
   }
+
+  override protected def withNewChildrenInternal(newLeft: Expression, newRight: Expression): Or =
+    copy(left = newLeft, right = newRight)
 }
 
 
@@ -675,6 +829,8 @@ abstract class BinaryComparison extends BinaryOperator with Predicate {
   // finitely enumerable. The allowable types are checked below by checkInputDataTypes.
   override def inputType: AbstractDataType = AnyDataType
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(BINARY_COMPARISON)
+
   override def checkInputDataTypes(): TypeCheckResult = super.checkInputDataTypes() match {
     case TypeCheckResult.TypeCheckSuccess =>
       TypeUtils.checkForOrderingExpr(left.dataType, this.getClass.getSimpleName)
@@ -731,7 +887,9 @@ object Equality {
        NULL
       > SELECT NULL _FUNC_ NULL;
        NULL
-  """)
+  """,
+  since = "1.0.0",
+  group = "predicate_funcs")
 case class EqualTo(left: Expression, right: Expression)
     extends BinaryComparison with NullIntolerant {
 
@@ -749,6 +907,9 @@ case class EqualTo(left: Expression, right: Expression)
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     defineCodeGen(ctx, ev, (c1, c2) => ctx.genEqual(left.dataType, c1, c2))
   }
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): EqualTo = copy(left = newLeft, right = newRight)
 }
 
 // TODO: although map type is not orderable, technically map type should be able to be used
@@ -774,7 +935,9 @@ case class EqualTo(left: Expression, right: Expression)
        false
       > SELECT NULL _FUNC_ NULL;
        true
-  """)
+  """,
+  since = "1.1.0",
+  group = "predicate_funcs")
 case class EqualNullSafe(left: Expression, right: Expression) extends BinaryComparison {
 
   override def symbol: String = "<=>"
@@ -808,6 +971,10 @@ case class EqualNullSafe(left: Expression, right: Expression) extends BinaryComp
         boolean ${ev.value} = (${eval1.isNull} && ${eval2.isNull}) ||
            (!${eval1.isNull} && !${eval2.isNull} && $equalCode);""", isNull = FalseLiteral)
   }
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): EqualNullSafe =
+    copy(left = newLeft, right = newRight)
 }
 
 @ExpressionDescription(
@@ -831,13 +998,18 @@ case class EqualNullSafe(left: Expression, right: Expression) extends BinaryComp
        true
       > SELECT 1 _FUNC_ NULL;
        NULL
-  """)
+  """,
+  since = "1.0.0",
+  group = "predicate_funcs")
 case class LessThan(left: Expression, right: Expression)
     extends BinaryComparison with NullIntolerant {
 
   override def symbol: String = "<"
 
   protected override def nullSafeEval(input1: Any, input2: Any): Any = ordering.lt(input1, input2)
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): Expression = copy(left = newLeft, right = newRight)
 }
 
 @ExpressionDescription(
@@ -861,13 +1033,18 @@ case class LessThan(left: Expression, right: Expression)
        true
       > SELECT 1 _FUNC_ NULL;
        NULL
-  """)
+  """,
+  since = "1.0.0",
+  group = "predicate_funcs")
 case class LessThanOrEqual(left: Expression, right: Expression)
     extends BinaryComparison with NullIntolerant {
 
   override def symbol: String = "<="
 
   protected override def nullSafeEval(input1: Any, input2: Any): Any = ordering.lteq(input1, input2)
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): Expression = copy(left = newLeft, right = newRight)
 }
 
 @ExpressionDescription(
@@ -891,13 +1068,18 @@ case class LessThanOrEqual(left: Expression, right: Expression)
        false
       > SELECT 1 _FUNC_ NULL;
        NULL
-  """)
+  """,
+  since = "1.0.0",
+  group = "predicate_funcs")
 case class GreaterThan(left: Expression, right: Expression)
     extends BinaryComparison with NullIntolerant {
 
   override def symbol: String = ">"
 
   protected override def nullSafeEval(input1: Any, input2: Any): Any = ordering.gt(input1, input2)
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): Expression = copy(left = newLeft, right = newRight)
 }
 
 @ExpressionDescription(
@@ -921,13 +1103,19 @@ case class GreaterThan(left: Expression, right: Expression)
        false
       > SELECT 1 _FUNC_ NULL;
        NULL
-  """)
+  """,
+  since = "1.0.0",
+  group = "predicate_funcs")
 case class GreaterThanOrEqual(left: Expression, right: Expression)
     extends BinaryComparison with NullIntolerant {
 
   override def symbol: String = ">="
 
   protected override def nullSafeEval(input1: Any, input2: Any): Any = ordering.gteq(input1, input2)
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): GreaterThanOrEqual =
+    copy(left = newLeft, right = newRight)
 }
 
 /**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala
index 6a945173803b7..dfa5b206ab0a5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala
@@ -17,12 +17,12 @@
 
 package org.apache.spark.sql.catalyst.expressions
 
-import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.UnresolvedSeed
 import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodeGenerator, ExprCode, FalseLiteral}
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
+import org.apache.spark.sql.catalyst.trees.TreePattern.{EXPRESSION_WITH_RANDOM_SEED, TreePattern}
 import org.apache.spark.sql.types._
-import org.apache.spark.util.Utils
 import org.apache.spark.util.random.XORShiftRandom
 
 /**
@@ -32,7 +32,8 @@ import org.apache.spark.util.random.XORShiftRandom
  *
  * Since this expression is stateful, it cannot be a case object.
  */
-abstract class RDG extends UnaryExpression with ExpectsInputTypes with Stateful {
+abstract class RDG extends UnaryExpression with ExpectsInputTypes with Stateful
+  with ExpressionWithRandomSeed {
   /**
    * Record ID within each partition. By being transient, the Random Number Generator is
    * reset every time we serialize and deserialize and initialize it.
@@ -43,11 +44,11 @@ abstract class RDG extends UnaryExpression with ExpectsInputTypes with Stateful
     rng = new XORShiftRandom(seed + partitionIndex)
   }
 
-  @transient protected lazy val seed: Long = child match {
-    case Literal(s, IntegerType) => s.asInstanceOf[Int]
-    case Literal(s, LongType) => s.asInstanceOf[Long]
-    case _ => throw new AnalysisException(
-      s"Input argument to $prettyName must be an integer, long or null literal.")
+  override def seedExpression: Expression = child
+
+  @transient protected lazy val seed: Long = seedExpression match {
+    case e if e.dataType == IntegerType => e.eval().asInstanceOf[Int]
+    case e if e.dataType == LongType => e.eval().asInstanceOf[Long]
   }
 
   override def nullable: Boolean = false
@@ -61,7 +62,10 @@ abstract class RDG extends UnaryExpression with ExpectsInputTypes with Stateful
  * Represents the behavior of expressions which have a random seed and can renew the seed.
  * Usually the random seed needs to be renewed at each execution under streaming queries.
  */
-trait ExpressionWithRandomSeed {
+trait ExpressionWithRandomSeed extends Expression {
+  override val nodePatterns: Seq[TreePattern] = Seq(EXPRESSION_WITH_RANDOM_SEED)
+
+  def seedExpression: Expression
   def withNewSeed(seed: Long): Expression
 }
 
@@ -81,16 +85,16 @@ trait ExpressionWithRandomSeed {
   note = """
     The function is non-deterministic in general case.
   """,
-  since = "1.5.0")
+  since = "1.5.0",
+  group = "math_funcs")
 // scalastyle:on line.size.limit
-case class Rand(child: Expression, hideSeed: Boolean = false)
-  extends RDG with ExpressionWithRandomSeed {
+case class Rand(child: Expression, hideSeed: Boolean = false) extends RDG {
 
-  def this() = this(Literal(Utils.random.nextLong(), LongType), true)
+  def this() = this(UnresolvedSeed, true)
 
   def this(child: Expression) = this(child, false)
 
-  override def withNewSeed(seed: Long): Rand = Rand(Literal(seed, LongType))
+  override def withNewSeed(seed: Long): Rand = Rand(Literal(seed, LongType), hideSeed)
 
   override protected def evalInternal(input: InternalRow): Double = rng.nextDouble()
 
@@ -110,6 +114,8 @@ case class Rand(child: Expression, hideSeed: Boolean = false)
   override def sql: String = {
     s"rand(${if (hideSeed) "" else child.sql})"
   }
+
+  override protected def withNewChildInternal(newChild: Expression): Rand = copy(child = newChild)
 }
 
 object Rand {
@@ -132,16 +138,16 @@ object Rand {
   note = """
     The function is non-deterministic in general case.
   """,
-  since = "1.5.0")
+  since = "1.5.0",
+  group = "math_funcs")
 // scalastyle:on line.size.limit
-case class Randn(child: Expression, hideSeed: Boolean = false)
-  extends RDG with ExpressionWithRandomSeed {
+case class Randn(child: Expression, hideSeed: Boolean = false) extends RDG {
 
-  def this() = this(Literal(Utils.random.nextLong(), LongType), true)
+  def this() = this(UnresolvedSeed, true)
 
   def this(child: Expression) = this(child, false)
 
-  override def withNewSeed(seed: Long): Randn = Randn(Literal(seed, LongType))
+  override def withNewSeed(seed: Long): Randn = Randn(Literal(seed, LongType), hideSeed)
 
   override protected def evalInternal(input: InternalRow): Double = rng.nextGaussian()
 
@@ -161,6 +167,8 @@ case class Randn(child: Expression, hideSeed: Boolean = false)
   override def sql: String = {
     s"randn(${if (hideSeed) "" else child.sql})"
   }
+
+  override protected def withNewChildInternal(newChild: Expression): Randn = copy(child = newChild)
 }
 
 object Randn {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
index 3f60ca388a807..57d7d76268795 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
@@ -18,13 +18,21 @@
 package org.apache.spark.sql.catalyst.expressions
 
 import java.util.Locale
-import java.util.regex.{MatchResult, Pattern}
+import java.util.regex.{Matcher, MatchResult, Pattern}
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
 
 import org.apache.commons.text.StringEscapeUtils
 
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
+import org.apache.spark.sql.catalyst.trees.TreePattern.{LIKE_FAMLIY, TreePattern}
 import org.apache.spark.sql.catalyst.util.{GenericArrayData, StringUtils}
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -62,8 +70,6 @@ abstract class StringRegexExpression extends BinaryExpression
       matches(regex, input1.asInstanceOf[UTF8String].toString)
     }
   }
-
-  override def sql: String = s"${left.sql} ${prettyName.toUpperCase(Locale.ROOT)} ${right.sql}"
 }
 
 // scalastyle:off line.contains.tab
@@ -87,7 +93,7 @@ abstract class StringRegexExpression extends BinaryExpression
           Since Spark 2.0, string literals are unescaped in our SQL parser. For example, in order
           to match "\abc", the pattern should be "\\abc".
 
-          When SQL config 'spark.sql.parser.escapedStringLiterals' is enabled, it fallbacks
+          When SQL config 'spark.sql.parser.escapedStringLiterals' is enabled, it falls back
           to Spark 1.6 behavior regarding string literal parsing. For example, if the config is
           enabled, the pattern to match "\abc" should be "\abc".
       * escape - an character added since Spark 3.0. The default escape character is the '\'.
@@ -112,7 +118,8 @@ abstract class StringRegexExpression extends BinaryExpression
   note = """
     Use RLIKE to match with standard regular expressions.
   """,
-  since = "1.0.0")
+  since = "1.0.0",
+  group = "predicate_funcs")
 // scalastyle:on line.contains.tab
 case class Like(left: Expression, right: Expression, escapeChar: Char)
   extends StringRegexExpression {
@@ -123,11 +130,15 @@ case class Like(left: Expression, right: Expression, escapeChar: Char)
 
   override def matches(regex: Pattern, str: String): Boolean = regex.matcher(str).matches()
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(LIKE_FAMLIY)
+
   override def toString: String = escapeChar match {
     case '\\' => s"$left LIKE $right"
     case c => s"$left LIKE $right ESCAPE '$c'"
   }
 
+  override def sql: String = s"${left.sql} ${prettyName.toUpperCase(Locale.ROOT)} ${right.sql}"
+
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     val patternClass = classOf[Pattern].getName
     val escapeFunc = StringUtils.getClass.getName.stripSuffix("$") + ".escapeLikeRegex"
@@ -172,11 +183,172 @@ case class Like(left: Expression, right: Expression, escapeChar: Char)
       })
     }
   }
+
+  override protected def withNewChildrenInternal(newLeft: Expression, newRight: Expression): Like =
+    copy(left = newLeft, right = newRight)
+}
+
+sealed abstract class MultiLikeBase
+  extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
+
+  protected def patterns: Seq[UTF8String]
+
+  protected def isNotSpecified: Boolean
+
+  override def inputTypes: Seq[DataType] = StringType :: Nil
+
+  override def dataType: DataType = BooleanType
+
+  override def nullable: Boolean = true
+
+  final override val nodePatterns: Seq[TreePattern] = Seq(LIKE_FAMLIY)
+
+  protected lazy val hasNull: Boolean = patterns.contains(null)
+
+  protected lazy val cache = patterns.filterNot(_ == null)
+    .map(s => Pattern.compile(StringUtils.escapeLikeRegex(s.toString, '\\')))
+
+  protected lazy val matchFunc = if (isNotSpecified) {
+    (p: Pattern, inputValue: String) => !p.matcher(inputValue).matches()
+  } else {
+    (p: Pattern, inputValue: String) => p.matcher(inputValue).matches()
+  }
+
+  protected def matches(exprValue: String): Any
+
+  override def eval(input: InternalRow): Any = {
+    val exprValue = child.eval(input)
+    if (exprValue == null) {
+      null
+    } else {
+      matches(exprValue.toString)
+    }
+  }
+}
+
+/**
+ * Optimized version of LIKE ALL, when all pattern values are literal.
+ */
+sealed abstract class LikeAllBase extends MultiLikeBase {
+
+  override def matches(exprValue: String): Any = {
+    if (cache.forall(matchFunc(_, exprValue))) {
+      if (hasNull) null else true
+    } else {
+      false
+    }
+  }
+
+  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
+    val eval = child.genCode(ctx)
+    val patternClass = classOf[Pattern].getName
+    val javaDataType = CodeGenerator.javaType(child.dataType)
+    val pattern = ctx.freshName("pattern")
+    val valueArg = ctx.freshName("valueArg")
+    val patternCache = ctx.addReferenceObj("patternCache", cache.asJava)
+
+    val checkNotMatchCode = if (isNotSpecified) {
+      s"$pattern.matcher($valueArg.toString()).matches()"
+    } else {
+      s"!$pattern.matcher($valueArg.toString()).matches()"
+    }
+
+    ev.copy(code =
+      code"""
+            |${eval.code}
+            |boolean ${ev.isNull} = false;
+            |boolean ${ev.value} = true;
+            |if (${eval.isNull}) {
+            |  ${ev.isNull} = true;
+            |} else {
+            |  $javaDataType $valueArg = ${eval.value};
+            |  for ($patternClass $pattern: $patternCache) {
+            |    if ($checkNotMatchCode) {
+            |      ${ev.value} = false;
+            |      break;
+            |    }
+            |  }
+            |  if (${ev.value} && $hasNull) ${ev.isNull} = true;
+            |}
+      """.stripMargin)
+  }
+}
+
+case class LikeAll(child: Expression, patterns: Seq[UTF8String]) extends LikeAllBase {
+  override def isNotSpecified: Boolean = false
+  override protected def withNewChildInternal(newChild: Expression): LikeAll =
+    copy(child = newChild)
+}
+
+case class NotLikeAll(child: Expression, patterns: Seq[UTF8String]) extends LikeAllBase {
+  override def isNotSpecified: Boolean = true
+  override protected def withNewChildInternal(newChild: Expression): NotLikeAll =
+    copy(child = newChild)
+}
+
+/**
+ * Optimized version of LIKE ANY, when all pattern values are literal.
+ */
+sealed abstract class LikeAnyBase extends MultiLikeBase {
+
+  override def matches(exprValue: String): Any = {
+    if (cache.exists(matchFunc(_, exprValue))) {
+      true
+    } else {
+      if (hasNull) null else false
+    }
+  }
+
+  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
+    val eval = child.genCode(ctx)
+    val patternClass = classOf[Pattern].getName
+    val javaDataType = CodeGenerator.javaType(child.dataType)
+    val pattern = ctx.freshName("pattern")
+    val valueArg = ctx.freshName("valueArg")
+    val patternCache = ctx.addReferenceObj("patternCache", cache.asJava)
+
+    val checkMatchCode = if (isNotSpecified) {
+      s"!$pattern.matcher($valueArg.toString()).matches()"
+    } else {
+      s"$pattern.matcher($valueArg.toString()).matches()"
+    }
+
+    ev.copy(code =
+      code"""
+            |${eval.code}
+            |boolean ${ev.isNull} = false;
+            |boolean ${ev.value} = false;
+            |if (${eval.isNull}) {
+            |  ${ev.isNull} = true;
+            |} else {
+            |  $javaDataType $valueArg = ${eval.value};
+            |  for ($patternClass $pattern: $patternCache) {
+            |    if ($checkMatchCode) {
+            |      ${ev.value} = true;
+            |      break;
+            |    }
+            |  }
+            |  if (!${ev.value} && $hasNull) ${ev.isNull} = true;
+            |}
+      """.stripMargin)
+  }
+}
+
+case class LikeAny(child: Expression, patterns: Seq[UTF8String]) extends LikeAnyBase {
+  override def isNotSpecified: Boolean = false
+  override protected def withNewChildInternal(newChild: Expression): LikeAny =
+    copy(child = newChild)
+}
+
+case class NotLikeAny(child: Expression, patterns: Seq[UTF8String]) extends LikeAnyBase {
+  override def isNotSpecified: Boolean = true
+  override protected def withNewChildInternal(newChild: Expression): NotLikeAny =
+    copy(child = newChild)
 }
 
 // scalastyle:off line.contains.tab
 @ExpressionDescription(
-  usage = "str _FUNC_ regexp - Returns true if `str` matches `regexp`, or false otherwise.",
+  usage = "_FUNC_(str, regexp) - Returns true if `str` matches `regexp`, or false otherwise.",
   arguments = """
     Arguments:
       * str - a string expression
@@ -194,23 +366,25 @@ case class Like(left: Expression, right: Expression, escapeChar: Char)
     Examples:
       > SET spark.sql.parser.escapedStringLiterals=true;
       spark.sql.parser.escapedStringLiterals	true
-      > SELECT '%SystemDrive%\Users\John' _FUNC_ '%SystemDrive%\\Users.*';
+      > SELECT _FUNC_('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*');
       true
       > SET spark.sql.parser.escapedStringLiterals=false;
       spark.sql.parser.escapedStringLiterals	false
-      > SELECT '%SystemDrive%\\Users\\John' _FUNC_ '%SystemDrive%\\\\Users.*';
+      > SELECT _FUNC_('%SystemDrive%\\Users\\John', '%SystemDrive%\\\\Users.*');
       true
   """,
   note = """
     Use LIKE to match with simple string pattern.
   """,
-  since = "1.0.0")
+  since = "1.0.0",
+  group = "predicate_funcs")
 // scalastyle:on line.contains.tab
 case class RLike(left: Expression, right: Expression) extends StringRegexExpression {
 
   override def escape(v: String): String = v
   override def matches(regex: Pattern, str: String): Boolean = regex.matcher(str).find(0)
-  override def toString: String = s"$left RLIKE $right"
+  override def toString: String = s"RLIKE($left, $right)"
+  override def sql: String = s"${prettyName.toUpperCase(Locale.ROOT)}(${left.sql}, ${right.sql})"
 
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     val patternClass = classOf[Pattern].getName
@@ -251,6 +425,9 @@ case class RLike(left: Expression, right: Expression) extends StringRegexExpress
       })
     }
   }
+
+  override protected def withNewChildrenInternal(newLeft: Expression, newRight: Expression): RLike =
+    copy(left = newLeft, right = newRight)
 }
 
 
@@ -281,13 +458,16 @@ case class RLike(left: Expression, right: Expression) extends StringRegexExpress
       > SELECT _FUNC_('oneAtwoBthreeC', '[ABC]', 2);
        ["one","twoBthreeC"]
   """,
-  since = "1.5.0")
+  since = "1.5.0",
+  group = "string_funcs")
 case class StringSplit(str: Expression, regex: Expression, limit: Expression)
-  extends TernaryExpression with ImplicitCastInputTypes {
+  extends TernaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def dataType: DataType = ArrayType(StringType)
   override def inputTypes: Seq[DataType] = Seq(StringType, StringType, IntegerType)
-  override def children: Seq[Expression] = str :: regex :: limit :: Nil
+  override def first: Expression = str
+  override def second: Expression = regex
+  override def third: Expression = limit
 
   def this(exp: Expression, regex: Expression) = this(exp, regex, Literal(-1));
 
@@ -306,6 +486,10 @@ case class StringSplit(str: Expression, regex: Expression, limit: Expression)
   }
 
   override def prettyName: String = "split"
+
+  override protected def withNewChildrenInternal(
+      newFirst: Expression, newSecond: Expression, newThird: Expression): StringSplit =
+    copy(str = newFirst, regex = newSecond, limit = newThird)
 }
 
 
@@ -316,16 +500,50 @@ case class StringSplit(str: Expression, regex: Expression, limit: Expression)
  */
 // scalastyle:off line.size.limit
 @ExpressionDescription(
-  usage = "_FUNC_(str, regexp, rep) - Replaces all substrings of `str` that match `regexp` with `rep`.",
+  usage = "_FUNC_(str, regexp, rep[, position]) - Replaces all substrings of `str` that match `regexp` with `rep`.",
+  arguments = """
+    Arguments:
+      * str - a string expression to search for a regular expression pattern match.
+      * regexp - a string representing a regular expression. The regex string should be a
+          Java regular expression.
+
+          Since Spark 2.0, string literals (including regex patterns) are unescaped in our SQL
+          parser. For example, to match "\abc", a regular expression for `regexp` can be
+          "^\\abc$".
+
+          There is a SQL config 'spark.sql.parser.escapedStringLiterals' that can be used to
+          fallback to the Spark 1.6 behavior regarding string literal parsing. For example,
+          if the config is enabled, the `regexp` that can match "\abc" is "^\abc$".
+      * rep - a string expression to replace matched substrings.
+      * position - a positive integer literal that indicates the position within `str` to begin searching.
+          The default is 1. If position is greater than the number of characters in `str`, the result is `str`.
+  """,
   examples = """
     Examples:
       > SELECT _FUNC_('100-200', '(\\d+)', 'num');
        num-num
   """,
-  since = "1.5.0")
+  since = "1.5.0",
+  group = "string_funcs")
 // scalastyle:on line.size.limit
-case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expression)
-  extends TernaryExpression with ImplicitCastInputTypes {
+case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expression, pos: Expression)
+  extends QuaternaryExpression with ImplicitCastInputTypes with NullIntolerant {
+
+  def this(subject: Expression, regexp: Expression, rep: Expression) =
+    this(subject, regexp, rep, Literal(1))
+
+  override def checkInputDataTypes(): TypeCheckResult = {
+    if (!pos.foldable) {
+      return TypeCheckFailure(s"Position expression must be foldable, but got $pos")
+    }
+
+    val posEval = pos.eval()
+    if (posEval == null || posEval.asInstanceOf[Int] > 0) {
+      TypeCheckSuccess
+    } else {
+      TypeCheckFailure(s"Position expression must be positive, but got: $posEval")
+    }
+  }
 
   // last regex in string, we will update the pattern iff regexp value changed.
   @transient private var lastRegex: UTF8String = _
@@ -337,7 +555,7 @@ case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expressio
   // result buffer write by Matcher
   @transient private lazy val result: StringBuffer = new StringBuffer
 
-  override def nullSafeEval(s: Any, p: Any, r: Any): Any = {
+  override def nullSafeEval(s: Any, p: Any, r: Any, i: Any): Any = {
     if (!p.equals(lastRegex)) {
       // regex value changed
       lastRegex = p.asInstanceOf[UTF8String].clone()
@@ -348,20 +566,25 @@ case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expressio
       lastReplacementInUTF8 = r.asInstanceOf[UTF8String].clone()
       lastReplacement = lastReplacementInUTF8.toString
     }
-    val m = pattern.matcher(s.toString())
-    result.delete(0, result.length())
-
-    while (m.find) {
-      m.appendReplacement(result, lastReplacement)
+    val source = s.toString()
+    val position = i.asInstanceOf[Int] - 1
+    if (position < source.length) {
+      val m = pattern.matcher(source)
+      m.region(position, source.length)
+      result.delete(0, result.length())
+      while (m.find) {
+        m.appendReplacement(result, lastReplacement)
+      }
+      m.appendTail(result)
+      UTF8String.fromString(result.toString)
+    } else {
+      s
     }
-    m.appendTail(result)
-
-    UTF8String.fromString(result.toString)
   }
 
   override def dataType: DataType = StringType
-  override def inputTypes: Seq[AbstractDataType] = Seq(StringType, StringType, StringType)
-  override def children: Seq[Expression] = subject :: regexp :: rep :: Nil
+  override def inputTypes: Seq[AbstractDataType] =
+    Seq(StringType, StringType, StringType, IntegerType)
   override def prettyName: String = "regexp_replace"
 
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
@@ -371,6 +594,8 @@ case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expressio
     val classNameStringBuffer = classOf[java.lang.StringBuffer].getCanonicalName
 
     val matcher = ctx.freshName("matcher")
+    val source = ctx.freshName("source")
+    val position = ctx.freshName("position")
 
     val termLastRegex = ctx.addMutableState("UTF8String", "lastRegex")
     val termPattern = ctx.addMutableState(classNamePattern, "pattern")
@@ -383,7 +608,7 @@ case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expressio
       ""
     }
 
-    nullSafeCodeGen(ctx, ev, (subject, regexp, rep) => {
+    nullSafeCodeGen(ctx, ev, (subject, regexp, rep, pos) => {
     s"""
       if (!$regexp.equals($termLastRegex)) {
         // regex value changed
@@ -395,65 +620,127 @@ case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expressio
         $termLastReplacementInUTF8 = $rep.clone();
         $termLastReplacement = $termLastReplacementInUTF8.toString();
       }
-      $classNameStringBuffer $termResult = new $classNameStringBuffer();
-      java.util.regex.Matcher $matcher = $termPattern.matcher($subject.toString());
-
-      while ($matcher.find()) {
-        $matcher.appendReplacement($termResult, $termLastReplacement);
+      String $source = $subject.toString();
+      int $position = $pos - 1;
+      if ($position < $source.length()) {
+        $classNameStringBuffer $termResult = new $classNameStringBuffer();
+        java.util.regex.Matcher $matcher = $termPattern.matcher($source);
+        $matcher.region($position, $source.length());
+
+        while ($matcher.find()) {
+          $matcher.appendReplacement($termResult, $termLastReplacement);
+        }
+        $matcher.appendTail($termResult);
+        ${ev.value} = UTF8String.fromString($termResult.toString());
+        $termResult = null;
+      } else {
+        ${ev.value} = $subject;
       }
-      $matcher.appendTail($termResult);
-      ${ev.value} = UTF8String.fromString($termResult.toString());
-      $termResult = null;
       $setEvNotNull
     """
     })
   }
+
+  override def first: Expression = subject
+  override def second: Expression = regexp
+  override def third: Expression = rep
+  override def fourth: Expression = pos
+
+  override protected def withNewChildrenInternal(
+      first: Expression, second: Expression, third: Expression, fourth: Expression): RegExpReplace =
+    copy(subject = first, regexp = second, rep = third, pos = fourth)
+}
+
+object RegExpReplace {
+  def apply(subject: Expression, regexp: Expression, rep: Expression): RegExpReplace =
+    new RegExpReplace(subject, regexp, rep)
 }
 
-object RegExpExtract {
+object RegExpExtractBase {
   def checkGroupIndex(groupCount: Int, groupIndex: Int): Unit = {
-    if (groupCount < groupIndex) {
-      throw new IllegalArgumentException(
-        s"Regex group count is $groupCount, but the specified group index is $groupIndex")
+    if (groupIndex < 0) {
+      throw QueryExecutionErrors.regexGroupIndexLessThanZeroError
+    } else if (groupCount < groupIndex) {
+      throw QueryExecutionErrors.regexGroupIndexExceedGroupCountError(
+        groupCount, groupIndex)
     }
   }
 }
 
+abstract class RegExpExtractBase
+  extends TernaryExpression with ImplicitCastInputTypes with NullIntolerant {
+  def subject: Expression
+  def regexp: Expression
+  def idx: Expression
+
+  // last regex in string, we will update the pattern iff regexp value changed.
+  @transient private var lastRegex: UTF8String = _
+  // last regex pattern, we cache it for performance concern
+  @transient private var pattern: Pattern = _
+
+  override def inputTypes: Seq[AbstractDataType] = Seq(StringType, StringType, IntegerType)
+  override def first: Expression = subject
+  override def second: Expression = regexp
+  override def third: Expression = idx
+
+  protected def getLastMatcher(s: Any, p: Any): Matcher = {
+    if (p != lastRegex) {
+      // regex value changed
+      lastRegex = p.asInstanceOf[UTF8String].clone()
+      pattern = Pattern.compile(lastRegex.toString)
+    }
+    pattern.matcher(s.toString)
+  }
+}
+
 /**
  * Extract a specific(idx) group identified by a Java regex.
  *
  * NOTE: this expression is not THREAD-SAFE, as it has some internal mutable status.
  */
 @ExpressionDescription(
-  usage = "_FUNC_(str, regexp[, idx]) - Extracts a group that matches `regexp`.",
+  usage = """
+    _FUNC_(str, regexp[, idx]) - Extract the first string in the `str` that match the `regexp`
+    expression and corresponding to the regex group index.
+  """,
+  arguments = """
+    Arguments:
+      * str - a string expression.
+      * regexp - a string representing a regular expression. The regex string should be a
+          Java regular expression.
+
+          Since Spark 2.0, string literals (including regex patterns) are unescaped in our SQL
+          parser. For example, to match "\abc", a regular expression for `regexp` can be
+          "^\\abc$".
+
+          There is a SQL config 'spark.sql.parser.escapedStringLiterals' that can be used to
+          fallback to the Spark 1.6 behavior regarding string literal parsing. For example,
+          if the config is enabled, the `regexp` that can match "\abc" is "^\abc$".
+      * idx - an integer expression that representing the group index. The regex maybe contains
+          multiple groups. `idx` indicates which regex group to extract. The group index should
+          be non-negative. The minimum value of `idx` is 0, which means matching the entire
+          regular expression. If `idx` is not specified, the default group index value is 1. The
+          `idx` parameter is the Java regex Matcher group() method index.
+  """,
   examples = """
     Examples:
       > SELECT _FUNC_('100-200', '(\\d+)-(\\d+)', 1);
        100
   """,
-  since = "1.5.0")
+  since = "1.5.0",
+  group = "string_funcs")
 case class RegExpExtract(subject: Expression, regexp: Expression, idx: Expression)
-  extends TernaryExpression with ImplicitCastInputTypes {
+  extends RegExpExtractBase {
   def this(s: Expression, r: Expression) = this(s, r, Literal(1))
 
-  // last regex in string, we will update the pattern iff regexp value changed.
-  @transient private var lastRegex: UTF8String = _
-  // last regex pattern, we cache it for performance concern
-  @transient private var pattern: Pattern = _
-
   override def nullSafeEval(s: Any, p: Any, r: Any): Any = {
-    if (!p.equals(lastRegex)) {
-      // regex value changed
-      lastRegex = p.asInstanceOf[UTF8String].clone()
-      pattern = Pattern.compile(lastRegex.toString)
-    }
-    val m = pattern.matcher(s.toString)
+    val m = getLastMatcher(s, p)
     if (m.find) {
       val mr: MatchResult = m.toMatchResult
       val index = r.asInstanceOf[Int]
-      RegExpExtract.checkGroupIndex(mr.groupCount, index)
+      RegExpExtractBase.checkGroupIndex(mr.groupCount, index)
       val group = mr.group(index)
-      if (group == null) { // Pattern matched, but not optional group
+      if (group == null) { // Pattern matched, but it's an optional group
         UTF8String.EMPTY_UTF8
       } else {
         UTF8String.fromString(group)
@@ -464,13 +751,11 @@ case class RegExpExtract(subject: Expression, regexp: Expression, idx: Expressio
   }
 
   override def dataType: DataType = StringType
-  override def inputTypes: Seq[AbstractDataType] = Seq(StringType, StringType, IntegerType)
-  override def children: Seq[Expression] = subject :: regexp :: idx :: Nil
   override def prettyName: String = "regexp_extract"
 
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     val classNamePattern = classOf[Pattern].getCanonicalName
-    val classNameRegExpExtract = classOf[RegExpExtract].getCanonicalName
+    val classNameRegExpExtractBase = classOf[RegExpExtractBase].getCanonicalName
     val matcher = ctx.freshName("matcher")
     val matchResult = ctx.freshName("matchResult")
 
@@ -494,7 +779,7 @@ case class RegExpExtract(subject: Expression, regexp: Expression, idx: Expressio
         $termPattern.matcher($subject.toString());
       if ($matcher.find()) {
         java.util.regex.MatchResult $matchResult = $matcher.toMatchResult();
-        $classNameRegExpExtract.checkGroupIndex($matchResult.groupCount(), $idx);
+        $classNameRegExpExtractBase.checkGroupIndex($matchResult.groupCount(), $idx);
         if ($matchResult.group($idx) == null) {
           ${ev.value} = UTF8String.EMPTY_UTF8;
         } else {
@@ -507,4 +792,115 @@ case class RegExpExtract(subject: Expression, regexp: Expression, idx: Expressio
       }"""
     })
   }
+
+  override protected def withNewChildrenInternal(
+      newFirst: Expression, newSecond: Expression, newThird: Expression): RegExpExtract =
+    copy(subject = newFirst, regexp = newSecond, idx = newThird)
+}
+
+/**
+ * Extract all specific(idx) groups identified by a Java regex.
+ *
+ * NOTE: this expression is not THREAD-SAFE, as it has some internal mutable status.
+ */
+@ExpressionDescription(
+  usage = """
+    _FUNC_(str, regexp[, idx]) - Extract all strings in the `str` that match the `regexp`
+    expression and corresponding to the regex group index.
+  """,
+  arguments = """
+    Arguments:
+      * str - a string expression.
+      * regexp - a string representing a regular expression. The regex string should be a
+          Java regular expression.
+
+          Since Spark 2.0, string literals (including regex patterns) are unescaped in our SQL
+          parser. For example, to match "\abc", a regular expression for `regexp` can be
+          "^\\abc$".
+
+          There is a SQL config 'spark.sql.parser.escapedStringLiterals' that can be used to
+          fallback to the Spark 1.6 behavior regarding string literal parsing. For example,
+          if the config is enabled, the `regexp` that can match "\abc" is "^\abc$".
+      * idx - an integer expression that representing the group index. The regex may contains
+          multiple groups. `idx` indicates which regex group to extract. The group index should
+          be non-negative. The minimum value of `idx` is 0, which means matching the entire
+          regular expression. If `idx` is not specified, the default group index value is 1. The
+          `idx` parameter is the Java regex Matcher group() method index.
+  """,
+  examples = """
+    Examples:
+      > SELECT _FUNC_('100-200, 300-400', '(\\d+)-(\\d+)', 1);
+       ["100","300"]
+  """,
+  since = "3.1.0",
+  group = "string_funcs")
+case class RegExpExtractAll(subject: Expression, regexp: Expression, idx: Expression)
+  extends RegExpExtractBase {
+  def this(s: Expression, r: Expression) = this(s, r, Literal(1))
+
+  override def nullSafeEval(s: Any, p: Any, r: Any): Any = {
+    val m = getLastMatcher(s, p)
+    val matchResults = new ArrayBuffer[UTF8String]()
+    while(m.find) {
+      val mr: MatchResult = m.toMatchResult
+      val index = r.asInstanceOf[Int]
+      RegExpExtractBase.checkGroupIndex(mr.groupCount, index)
+      val group = mr.group(index)
+      if (group == null) { // Pattern matched, but it's an optional group
+        matchResults += UTF8String.EMPTY_UTF8
+      } else {
+        matchResults += UTF8String.fromString(group)
+      }
+    }
+
+    new GenericArrayData(matchResults.toArray.asInstanceOf[Array[Any]])
+  }
+
+  override def dataType: DataType = ArrayType(StringType)
+  override def prettyName: String = "regexp_extract_all"
+
+  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
+    val classNamePattern = classOf[Pattern].getCanonicalName
+    val classNameRegExpExtractBase = classOf[RegExpExtractBase].getCanonicalName
+    val arrayClass = classOf[GenericArrayData].getName
+    val matcher = ctx.freshName("matcher")
+    val matchResult = ctx.freshName("matchResult")
+    val matchResults = ctx.freshName("matchResults")
+
+    val termLastRegex = ctx.addMutableState("UTF8String", "lastRegex")
+    val termPattern = ctx.addMutableState(classNamePattern, "pattern")
+
+    val setEvNotNull = if (nullable) {
+      s"${ev.isNull} = false;"
+    } else {
+      ""
+    }
+    nullSafeCodeGen(ctx, ev, (subject, regexp, idx) => {
+      s"""
+         | if (!$regexp.equals($termLastRegex)) {
+         |   // regex value changed
+         |   $termLastRegex = $regexp.clone();
+         |   $termPattern = $classNamePattern.compile($termLastRegex.toString());
+         | }
+         | java.util.regex.Matcher $matcher = $termPattern.matcher($subject.toString());
+         | java.util.ArrayList $matchResults = new java.util.ArrayList<UTF8String>();
+         | while ($matcher.find()) {
+         |   java.util.regex.MatchResult $matchResult = $matcher.toMatchResult();
+         |   $classNameRegExpExtractBase.checkGroupIndex($matchResult.groupCount(), $idx);
+         |   if ($matchResult.group($idx) == null) {
+         |     $matchResults.add(UTF8String.EMPTY_UTF8);
+         |   } else {
+         |     $matchResults.add(UTF8String.fromString($matchResult.group($idx)));
+         |   }
+         | }
+         | ${ev.value} =
+         |   new $arrayClass($matchResults.toArray(new UTF8String[$matchResults.size()]));
+         | $setEvNotNull
+         """
+    })
+  }
+
+  override protected def withNewChildrenInternal(
+      newFirst: Expression, newSecond: Expression, newThird: Expression): RegExpExtractAll =
+    copy(subject = newFirst, regexp = newSecond, idx = newThird)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
index 0b9fb8f85fe3c..5956c3e882118 100755
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
@@ -30,7 +30,10 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult}
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
+import org.apache.spark.sql.catalyst.trees.TreePattern.{TreePattern, UPPER_OR_LOWER}
 import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData, TypeUtils}
+import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.UTF8StringBuilder
 import org.apache.spark.unsafe.types.{ByteArray, UTF8String}
@@ -48,13 +51,16 @@ import org.apache.spark.unsafe.types.{ByteArray, UTF8String}
  */
 // scalastyle:off line.size.limit
 @ExpressionDescription(
-  usage = "_FUNC_(sep, [str | array(str)]+) - Returns the concatenation of the strings separated by `sep`.",
+  usage = "_FUNC_(sep[, str | array(str)]+) - Returns the concatenation of the strings separated by `sep`.",
   examples = """
     Examples:
       > SELECT _FUNC_(' ', 'Spark', 'SQL');
         Spark SQL
+      > SELECT _FUNC_('s');
+
   """,
-  since = "1.5.0")
+  since = "1.5.0",
+  group = "string_funcs")
 // scalastyle:on line.size.limit
 case class ConcatWs(children: Seq[Expression])
   extends Expression with ImplicitCastInputTypes {
@@ -118,47 +124,72 @@ case class ConcatWs(children: Seq[Expression])
         boolean ${ev.isNull} = ${ev.value} == null;
       """)
     } else {
+      val isNullArgs = ctx.freshName("isNullArgs")
+      val valueArgs = ctx.freshName("valueArgs")
+
       val array = ctx.freshName("array")
       val varargNum = ctx.freshName("varargNum")
       val idxVararg = ctx.freshName("idxInVararg")
 
       val evals = children.map(_.genCode(ctx))
-      val (varargCount, varargBuild) = children.tail.zip(evals.tail).map { case (child, eval) =>
-        child.dataType match {
+      val (argBuild, varargCount, varargBuild) = children.tail.zip(evals.tail)
+        .zipWithIndex.map { case ((child, eval), idx) =>
+        val reprForIsNull = s"$isNullArgs[$idx]"
+        val reprForValue = s"$valueArgs[$idx]"
+
+        val arg =
+          s"""
+           ${eval.code}
+           $reprForIsNull = ${eval.isNull};
+           $reprForValue = ${eval.value};
+           """
+
+        val (varCount, varBuild) = child.dataType match {
           case StringType =>
+            val reprForValueCast = s"((UTF8String) $reprForValue)"
             ("", // we count all the StringType arguments num at once below.
-             if (eval.isNull == TrueLiteral) {
-               ""
-             } else {
-               s"$array[$idxVararg ++] = ${eval.isNull} ? (UTF8String) null : ${eval.value};"
-             })
+              if (eval.isNull == TrueLiteral) {
+                ""
+              } else {
+                s"$array[$idxVararg ++] = $reprForIsNull ? (UTF8String) null : $reprForValueCast;"
+              })
           case _: ArrayType =>
+            val reprForValueCast = s"((ArrayData) $reprForValue)"
             val size = ctx.freshName("n")
             if (eval.isNull == TrueLiteral) {
               ("", "")
             } else {
+              // scalastyle:off line.size.limit
               (s"""
-                if (!${eval.isNull}) {
-                  $varargNum += ${eval.value}.numElements();
+                if (!$reprForIsNull) {
+                  $varargNum += $reprForValueCast.numElements();
                 }
                 """,
-               s"""
-                if (!${eval.isNull}) {
-                  final int $size = ${eval.value}.numElements();
+                s"""
+                if (!$reprForIsNull) {
+                  final int $size = $reprForValueCast.numElements();
                   for (int j = 0; j < $size; j ++) {
-                    $array[$idxVararg ++] = ${CodeGenerator.getValue(eval.value, StringType, "j")};
+                    $array[$idxVararg ++] = ${CodeGenerator.getValue(reprForValueCast, StringType, "j")};
                   }
                 }
                 """)
+              // scalastyle:on line.size.limit
             }
         }
-      }.unzip
 
-      val codes = ctx.splitExpressionsWithCurrentInputs(evals.map(_.code.toString))
+        (arg, varCount, varBuild)
+      }.unzip3
+
+      val argBuilds = ctx.splitExpressionsWithCurrentInputs(
+        expressions = argBuild,
+        funcName = "initializeArgsArrays",
+        extraArguments = ("boolean []", isNullArgs) :: ("Object []", valueArgs) :: Nil
+      )
 
       val varargCounts = ctx.splitExpressionsWithCurrentInputs(
         expressions = varargCount,
         funcName = "varargCountsConcatWs",
+        extraArguments = ("boolean []", isNullArgs) :: ("Object []", valueArgs) :: Nil,
         returnType = "int",
         makeSplitFunction = body =>
           s"""
@@ -171,7 +202,8 @@ case class ConcatWs(children: Seq[Expression])
       val varargBuilds = ctx.splitExpressionsWithCurrentInputs(
         expressions = varargBuild,
         funcName = "varargBuildsConcatWs",
-        extraArguments = ("UTF8String []", array) :: ("int", idxVararg) :: Nil,
+        extraArguments = ("UTF8String []", array) :: ("int", idxVararg) ::
+          ("boolean []", isNullArgs) :: ("Object []", valueArgs) :: Nil,
         returnType = "int",
         makeSplitFunction = body =>
           s"""
@@ -182,17 +214,23 @@ case class ConcatWs(children: Seq[Expression])
 
       ev.copy(
         code"""
-        $codes
+        boolean[] $isNullArgs = new boolean[${children.length - 1}];
+        Object[] $valueArgs = new Object[${children.length - 1}];
+        $argBuilds
         int $varargNum = ${children.count(_.dataType == StringType) - 1};
         int $idxVararg = 0;
         $varargCounts
         UTF8String[] $array = new UTF8String[$varargNum];
         $varargBuilds
+        ${evals.head.code}
         UTF8String ${ev.value} = UTF8String.concatWs(${evals.head.value}, $array);
         boolean ${ev.isNull} = ${ev.value} == null;
       """)
     }
   }
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): ConcatWs =
+    copy(children = newChildren)
 }
 
 /**
@@ -202,15 +240,25 @@ case class ConcatWs(children: Seq[Expression])
  */
 // scalastyle:off line.size.limit
 @ExpressionDescription(
-  usage = "_FUNC_(n, input1, input2, ...) - Returns the `n`-th input, e.g., returns `input2` when `n` is 2.",
+  usage = """
+    _FUNC_(n, input1, input2, ...) - Returns the `n`-th input, e.g., returns `input2` when `n` is 2.
+    The function returns NULL if the index exceeds the length of the array
+    and `spark.sql.ansi.enabled` is set to false. If `spark.sql.ansi.enabled` is set to true,
+    it throws ArrayIndexOutOfBoundsException for invalid indices.
+  """,
   examples = """
     Examples:
       > SELECT _FUNC_(1, 'scala', 'java');
        scala
   """,
-  since = "2.0.0")
+  since = "2.0.0",
+  group = "string_funcs")
 // scalastyle:on line.size.limit
-case class Elt(children: Seq[Expression]) extends Expression {
+case class Elt(
+    children: Seq[Expression],
+    failOnError: Boolean = SQLConf.get.ansiEnabled) extends Expression {
+
+  def this(children: Seq[Expression]) = this(children, SQLConf.get.ansiEnabled)
 
   private lazy val indexExpr = children.head
   private lazy val inputExprs = children.tail.toArray
@@ -246,7 +294,11 @@ case class Elt(children: Seq[Expression]) extends Expression {
     } else {
       val index = indexObj.asInstanceOf[Int]
       if (index <= 0 || index > inputExprs.length) {
-        null
+        if (failOnError) {
+          throw QueryExecutionErrors.invalidArrayIndexError(index, inputExprs.length)
+        } else {
+          null
+        }
       } else {
         inputExprs(index - 1).eval(input)
       }
@@ -294,6 +346,16 @@ case class Elt(children: Seq[Expression]) extends Expression {
          """.stripMargin
       }.mkString)
 
+    val indexOutOfBoundBranch = if (failOnError) {
+      s"""
+         |if (!$indexMatched) {
+         |  throw QueryExecutionErrors.invalidArrayIndexError(${index.value}, ${inputExprs.length});
+         |}
+       """.stripMargin
+    } else {
+      ""
+    }
+
     ev.copy(
       code"""
          |${index.code}
@@ -303,10 +365,14 @@ case class Elt(children: Seq[Expression]) extends Expression {
          |do {
          |  $codes
          |} while (false);
+         |$indexOutOfBoundBranch
          |final ${CodeGenerator.javaType(dataType)} ${ev.value} = $inputVal;
          |final boolean ${ev.isNull} = ${ev.value} == null;
        """.stripMargin)
   }
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Elt =
+    copy(children = newChildren)
 }
 
 
@@ -332,17 +398,22 @@ trait String2StringExpression extends ImplicitCastInputTypes {
       > SELECT _FUNC_('SparkSql');
        SPARKSQL
   """,
-  since = "1.0.1")
+  since = "1.0.1",
+  group = "string_funcs")
 case class Upper(child: Expression)
-  extends UnaryExpression with String2StringExpression {
+  extends UnaryExpression with String2StringExpression with NullIntolerant {
 
   // scalastyle:off caselocale
   override def convert(v: UTF8String): UTF8String = v.toUpperCase
   // scalastyle:on caselocale
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(UPPER_OR_LOWER)
+
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     defineCodeGen(ctx, ev, c => s"($c).toUpperCase()")
   }
+
+  override protected def withNewChildInternal(newChild: Expression): Upper = copy(child = newChild)
 }
 
 /**
@@ -355,16 +426,25 @@ case class Upper(child: Expression)
       > SELECT _FUNC_('SparkSql');
        sparksql
   """,
-  since = "1.0.1")
-case class Lower(child: Expression) extends UnaryExpression with String2StringExpression {
+  since = "1.0.1",
+  group = "string_funcs")
+case class Lower(child: Expression)
+  extends UnaryExpression with String2StringExpression with NullIntolerant {
 
   // scalastyle:off caselocale
   override def convert(v: UTF8String): UTF8String = v.toLowerCase
   // scalastyle:on caselocale
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(UPPER_OR_LOWER)
+
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     defineCodeGen(ctx, ev, c => s"($c).toLowerCase()")
   }
+
+  override def prettyName: String =
+    getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("lower")
+
+  override protected def withNewChildInternal(newChild: Expression): Lower = copy(child = newChild)
 }
 
 /** A base trait for functions that compare two strings, returning a boolean. */
@@ -389,6 +469,8 @@ case class Contains(left: Expression, right: Expression) extends StringPredicate
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     defineCodeGen(ctx, ev, (c1, c2) => s"($c1).contains($c2)")
   }
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): Contains = copy(left = newLeft, right = newRight)
 }
 
 /**
@@ -399,6 +481,8 @@ case class StartsWith(left: Expression, right: Expression) extends StringPredica
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     defineCodeGen(ctx, ev, (c1, c2) => s"($c1).startsWith($c2)")
   }
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): StartsWith = copy(left = newLeft, right = newRight)
 }
 
 /**
@@ -409,6 +493,8 @@ case class EndsWith(left: Expression, right: Expression) extends StringPredicate
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     defineCodeGen(ctx, ev, (c1, c2) => s"($c1).endsWith($c2)")
   }
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): EndsWith = copy(left = newLeft, right = newRight)
 }
 
 /**
@@ -429,10 +515,11 @@ case class EndsWith(left: Expression, right: Expression) extends StringPredicate
       > SELECT _FUNC_('ABCabc', 'abc', 'DEF');
        ABCDEF
   """,
-  since = "2.3.0")
+  since = "2.3.0",
+  group = "string_funcs")
 // scalastyle:on line.size.limit
 case class StringReplace(srcExpr: Expression, searchExpr: Expression, replaceExpr: Expression)
-  extends TernaryExpression with ImplicitCastInputTypes {
+  extends TernaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   def this(srcExpr: Expression, searchExpr: Expression) = {
     this(srcExpr, searchExpr, Literal(""))
@@ -451,8 +538,15 @@ case class StringReplace(srcExpr: Expression, searchExpr: Expression, replaceExp
 
   override def dataType: DataType = StringType
   override def inputTypes: Seq[DataType] = Seq(StringType, StringType, StringType)
-  override def children: Seq[Expression] = srcExpr :: searchExpr :: replaceExpr :: Nil
+  override def first: Expression = srcExpr
+  override def second: Expression = searchExpr
+  override def third: Expression = replaceExpr
+
   override def prettyName: String = "replace"
+
+  override protected def withNewChildrenInternal(
+      newFirst: Expression, newSecond: Expression, newThird: Expression): StringReplace =
+    copy(srcExpr = newFirst, searchExpr = newSecond, replaceExpr = newThird)
 }
 
 object Overlay {
@@ -508,7 +602,9 @@ object Overlay {
        Spark ANSI SQL
       > SELECT _FUNC_(encode('Spark SQL', 'utf-8') PLACING encode('tructured', 'utf-8') FROM 2 FOR 4);
        Structured SQL
-  """)
+  """,
+  since = "3.0.0",
+  group = "string_funcs")
 // scalastyle:on line.size.limit
 case class Overlay(input: Expression, replace: Expression, pos: Expression, len: Expression)
   extends QuaternaryExpression with ImplicitCastInputTypes with NullIntolerant {
@@ -522,8 +618,6 @@ case class Overlay(input: Expression, replace: Expression, pos: Expression, len:
   override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection(StringType, BinaryType),
     TypeCollection(StringType, BinaryType), IntegerType, IntegerType)
 
-  override def children: Seq[Expression] = input :: replace :: pos :: len :: Nil
-
   override def checkInputDataTypes(): TypeCheckResult = {
     val inputTypeCheck = super.checkInputDataTypes()
     if (inputTypeCheck.isSuccess) {
@@ -560,22 +654,43 @@ case class Overlay(input: Expression, replace: Expression, pos: Expression, len:
       "org.apache.spark.sql.catalyst.expressions.Overlay" +
         s".calculate($input, $replace, $pos, $len);")
   }
+
+  override def first: Expression = input
+  override def second: Expression = replace
+  override def third: Expression = pos
+  override def fourth: Expression = len
+
+  override protected def withNewChildrenInternal(
+      first: Expression, second: Expression, third: Expression, fourth: Expression): Overlay =
+    copy(input = first, replace = second, pos = third, len = fourth)
 }
 
 object StringTranslate {
 
   def buildDict(matchingString: UTF8String, replaceString: UTF8String)
-    : JMap[Character, Character] = {
+    : JMap[String, String] = {
     val matching = matchingString.toString()
     val replace = replaceString.toString()
-    val dict = new HashMap[Character, Character]()
+    val dict = new HashMap[String, String]()
     var i = 0
-    while (i < matching.length()) {
-      val rep = if (i < replace.length()) replace.charAt(i) else '\u0000'
-      if (null == dict.get(matching.charAt(i))) {
-        dict.put(matching.charAt(i), rep)
+    var j = 0
+
+    while (i < matching.length) {
+      val rep = if (j < replace.length) {
+        val repCharCount = Character.charCount(replace.codePointAt(j))
+        val repStr = replace.substring(j, j + repCharCount)
+        j += repCharCount
+        repStr
+      } else {
+        "\u0000"
       }
-      i += 1
+
+      val matchCharCount = Character.charCount(matching.codePointAt(i))
+      val matchStr = matching.substring(i, i + matchCharCount)
+      if (null == dict.get(matchStr)) {
+        dict.put(matchStr, rep)
+      }
+      i += matchCharCount
     }
     dict
   }
@@ -595,14 +710,15 @@ object StringTranslate {
       > SELECT _FUNC_('AaBbCc', 'abc', '123');
        A1B2C3
   """,
-  since = "1.5.0")
+  since = "1.5.0",
+  group = "string_funcs")
 // scalastyle:on line.size.limit
 case class StringTranslate(srcExpr: Expression, matchingExpr: Expression, replaceExpr: Expression)
-  extends TernaryExpression with ImplicitCastInputTypes {
+  extends TernaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   @transient private var lastMatching: UTF8String = _
   @transient private var lastReplace: UTF8String = _
-  @transient private var dict: JMap[Character, Character] = _
+  @transient private var dict: JMap[String, String] = _
 
   override def nullSafeEval(srcEval: Any, matchingEval: Any, replaceEval: Any): Any = {
     if (matchingEval != lastMatching || replaceEval != lastReplace) {
@@ -640,8 +756,14 @@ case class StringTranslate(srcExpr: Expression, matchingExpr: Expression, replac
 
   override def dataType: DataType = StringType
   override def inputTypes: Seq[DataType] = Seq(StringType, StringType, StringType)
-  override def children: Seq[Expression] = srcExpr :: matchingExpr :: replaceExpr :: Nil
+  override def first: Expression = srcExpr
+  override def second: Expression = matchingExpr
+  override def third: Expression = replaceExpr
   override def prettyName: String = "translate"
+
+  override protected def withNewChildrenInternal(
+      newFirst: Expression, newSecond: Expression, newThird: Expression): StringTranslate =
+    copy(srcExpr = newFirst, matchingExpr = newSecond, replaceExpr = newThird)
 }
 
 /**
@@ -660,10 +782,11 @@ case class StringTranslate(srcExpr: Expression, matchingExpr: Expression, replac
       > SELECT _FUNC_('ab','abc,b,ab,c,def');
        3
   """,
-  since = "1.5.0")
+  since = "1.5.0",
+  group = "string_funcs")
 // scalastyle:on line.size.limit
 case class FindInSet(left: Expression, right: Expression) extends BinaryExpression
-    with ImplicitCastInputTypes {
+    with ImplicitCastInputTypes with NullIntolerant {
 
   override def inputTypes: Seq[AbstractDataType] = Seq(StringType, StringType)
 
@@ -679,6 +802,9 @@ case class FindInSet(left: Expression, right: Expression) extends BinaryExpressi
   override def dataType: DataType = IntegerType
 
   override def prettyName: String = "find_in_set"
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): FindInSet = copy(left = newLeft, right = newRight)
 }
 
 trait String2TrimExpression extends Expression with ImplicitCastInputTypes {
@@ -694,6 +820,55 @@ trait String2TrimExpression extends Expression with ImplicitCastInputTypes {
   override def nullable: Boolean = children.exists(_.nullable)
   override def foldable: Boolean = children.forall(_.foldable)
 
+  protected def doEval(srcString: UTF8String): UTF8String
+  protected def doEval(srcString: UTF8String, trimString: UTF8String): UTF8String
+
+  override def eval(input: InternalRow): Any = {
+    val srcString = srcStr.eval(input).asInstanceOf[UTF8String]
+    if (srcString == null) {
+      null
+    } else if (trimStr.isDefined) {
+      doEval(srcString, trimStr.get.eval(input).asInstanceOf[UTF8String])
+    } else {
+      doEval(srcString)
+    }
+  }
+
+  protected val trimMethod: String
+
+  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
+    val evals = children.map(_.genCode(ctx))
+    val srcString = evals(0)
+
+    if (evals.length == 1) {
+      ev.copy(code = code"""
+         |${srcString.code}
+         |boolean ${ev.isNull} = false;
+         |UTF8String ${ev.value} = null;
+         |if (${srcString.isNull}) {
+         |  ${ev.isNull} = true;
+         |} else {
+         |  ${ev.value} = ${srcString.value}.$trimMethod();
+         |}""".stripMargin)
+    } else {
+      val trimString = evals(1)
+      ev.copy(code = code"""
+         |${srcString.code}
+         |boolean ${ev.isNull} = false;
+         |UTF8String ${ev.value} = null;
+         |if (${srcString.isNull}) {
+         |  ${ev.isNull} = true;
+         |} else {
+         |  ${trimString.code}
+         |  if (${trimString.isNull}) {
+         |    ${ev.isNull} = true;
+         |  } else {
+         |    ${ev.value} = ${srcString.value}.$trimMethod(${trimString.value});
+         |  }
+         |}""".stripMargin)
+    }
+  }
+
   override def sql: String = if (trimStr.isDefined) {
     s"TRIM($direction ${trimStr.get.sql} FROM ${srcStr.sql})"
   } else {
@@ -768,10 +943,9 @@ object StringTrim {
       > SELECT _FUNC_(TRAILING 'SL' FROM 'SSparkSQLS');
        SSparkSQ
   """,
-  since = "1.5.0")
-case class StringTrim(
-    srcStr: Expression,
-    trimStr: Option[Expression] = None)
+  since = "1.5.0",
+  group = "string_funcs")
+case class StringTrim(srcStr: Expression, trimStr: Option[Expression] = None)
   extends String2TrimExpression {
 
   def this(trimStr: Expression, srcStr: Expression) = this(srcStr, Option(trimStr))
@@ -782,51 +956,68 @@ case class StringTrim(
 
   override protected def direction: String = "BOTH"
 
-  override def eval(input: InternalRow): Any = {
-    val srcString = srcStr.eval(input).asInstanceOf[UTF8String]
-    if (srcString == null) {
-      null
-    } else {
-      if (trimStr.isDefined) {
-        srcString.trim(trimStr.get.eval(input).asInstanceOf[UTF8String])
-      } else {
-        srcString.trim()
-      }
-    }
-  }
+  override def doEval(srcString: UTF8String): UTF8String = srcString.trim()
 
-  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val evals = children.map(_.genCode(ctx))
-    val srcString = evals(0)
+  override def doEval(srcString: UTF8String, trimString: UTF8String): UTF8String =
+    srcString.trim(trimString)
 
-    if (evals.length == 1) {
-      ev.copy(evals.map(_.code) :+ code"""
-        boolean ${ev.isNull} = false;
-        UTF8String ${ev.value} = null;
-        if (${srcString.isNull}) {
-          ${ev.isNull} = true;
-        } else {
-          ${ev.value} = ${srcString.value}.trim();
-        }""")
-    } else {
-      val trimString = evals(1)
-      val getTrimFunction =
-        s"""
-        if (${trimString.isNull}) {
-          ${ev.isNull} = true;
-        } else {
-          ${ev.value} = ${srcString.value}.trim(${trimString.value});
-        }"""
-      ev.copy(evals.map(_.code) :+ code"""
-        boolean ${ev.isNull} = false;
-        UTF8String ${ev.value} = null;
-        if (${srcString.isNull}) {
-          ${ev.isNull} = true;
-        } else {
-          $getTrimFunction
-        }""")
-    }
+  override val trimMethod: String = "trim"
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
+    copy(
+      srcStr = newChildren.head,
+      trimStr = if (trimStr.isDefined) Some(newChildren.last) else None)
+}
+
+/**
+ * A function that takes a character string, removes the leading and trailing characters matching
+ * with any character in the trim string, returns the new string.
+ * trimStr: A character string to be trimmed from the source string, if it has multiple characters,
+ * the function searches for each character in the source string, removes the characters from the
+ * source string until it encounters the first non-match character.
+ */
+@ExpressionDescription(
+  usage = """
+    _FUNC_(str) - Removes the leading and trailing space characters from `str`.
+
+    _FUNC_(str, trimStr) - Remove the leading and trailing `trimStr` characters from `str`.
+  """,
+  arguments = """
+    Arguments:
+      * str - a string expression
+      * trimStr - the trim string characters to trim, the default value is a single space
+  """,
+  examples = """
+    Examples:
+      > SELECT _FUNC_('    SparkSQL   ');
+       SparkSQL
+      > SELECT _FUNC_(encode('    SparkSQL   ', 'utf-8'));
+       SparkSQL
+      > SELECT _FUNC_('SSparkSQLS', 'SL');
+       parkSQ
+      > SELECT _FUNC_(encode('SSparkSQLS', 'utf-8'), encode('SL', 'utf-8'));
+       parkSQ
+  """,
+  since = "3.2.0",
+  group = "string_funcs")
+case class StringTrimBoth(srcStr: Expression, trimStr: Option[Expression], child: Expression)
+  extends RuntimeReplaceable {
+
+  def this(srcStr: Expression, trimStr: Expression) = {
+    this(srcStr, Option(trimStr), StringTrim(srcStr, trimStr))
+  }
+
+  def this(srcStr: Expression) = {
+    this(srcStr, None, StringTrim(srcStr))
   }
+
+  override def exprsReplaced: Seq[Expression] = srcStr +: trimStr.toSeq
+  override def flatArguments: Iterator[Any] = Iterator(srcStr, trimStr)
+
+  override def prettyName: String = "btrim"
+
+  override protected def withNewChildInternal(newChild: Expression): StringTrimBoth =
+    copy(child = newChild)
 }
 
 object StringTrimLeft {
@@ -861,10 +1052,9 @@ object StringTrimLeft {
       > SELECT _FUNC_('    SparkSQL   ');
        SparkSQL
   """,
-  since = "1.5.0")
-case class StringTrimLeft(
-    srcStr: Expression,
-    trimStr: Option[Expression] = None)
+  since = "1.5.0",
+  group = "string_funcs")
+case class StringTrimLeft(srcStr: Expression, trimStr: Option[Expression] = None)
   extends String2TrimExpression {
 
   def this(trimStr: Expression, srcStr: Expression) = this(srcStr, Option(trimStr))
@@ -875,51 +1065,18 @@ case class StringTrimLeft(
 
   override protected def direction: String = "LEADING"
 
-  override def eval(input: InternalRow): Any = {
-    val srcString = srcStr.eval(input).asInstanceOf[UTF8String]
-    if (srcString == null) {
-      null
-    } else {
-      if (trimStr.isDefined) {
-        srcString.trimLeft(trimStr.get.eval(input).asInstanceOf[UTF8String])
-      } else {
-        srcString.trimLeft()
-      }
-    }
-  }
+  override def doEval(srcString: UTF8String): UTF8String = srcString.trimLeft()
 
-  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val evals = children.map(_.genCode(ctx))
-    val srcString = evals(0)
+  override def doEval(srcString: UTF8String, trimString: UTF8String): UTF8String =
+    srcString.trimLeft(trimString)
 
-    if (evals.length == 1) {
-      ev.copy(evals.map(_.code) :+ code"""
-        boolean ${ev.isNull} = false;
-        UTF8String ${ev.value} = null;
-        if (${srcString.isNull}) {
-          ${ev.isNull} = true;
-        } else {
-          ${ev.value} = ${srcString.value}.trimLeft();
-        }""")
-    } else {
-      val trimString = evals(1)
-      val getTrimLeftFunction =
-        s"""
-        if (${trimString.isNull}) {
-          ${ev.isNull} = true;
-        } else {
-          ${ev.value} = ${srcString.value}.trimLeft(${trimString.value});
-        }"""
-      ev.copy(evals.map(_.code) :+ code"""
-        boolean ${ev.isNull} = false;
-        UTF8String ${ev.value} = null;
-        if (${srcString.isNull}) {
-          ${ev.isNull} = true;
-        } else {
-          $getTrimLeftFunction
-        }""")
-    }
-  }
+  override val trimMethod: String = "trimLeft"
+
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[Expression]): StringTrimLeft =
+    copy(
+      srcStr = newChildren.head,
+      trimStr = if (trimStr.isDefined) Some(newChildren.last) else None)
 }
 
 object StringTrimRight {
@@ -955,11 +1112,10 @@ object StringTrimRight {
       > SELECT _FUNC_('    SparkSQL   ');
        SparkSQL
   """,
-  since = "1.5.0")
+  since = "1.5.0",
+  group = "string_funcs")
 // scalastyle:on line.size.limit
-case class StringTrimRight(
-    srcStr: Expression,
-    trimStr: Option[Expression] = None)
+case class StringTrimRight(srcStr: Expression, trimStr: Option[Expression] = None)
   extends String2TrimExpression {
 
   def this(trimStr: Expression, srcStr: Expression) = this(srcStr, Option(trimStr))
@@ -970,51 +1126,18 @@ case class StringTrimRight(
 
   override protected def direction: String = "TRAILING"
 
-  override def eval(input: InternalRow): Any = {
-    val srcString = srcStr.eval(input).asInstanceOf[UTF8String]
-    if (srcString == null) {
-      null
-    } else {
-      if (trimStr.isDefined) {
-        srcString.trimRight(trimStr.get.eval(input).asInstanceOf[UTF8String])
-      } else {
-        srcString.trimRight()
-      }
-    }
-  }
+  override def doEval(srcString: UTF8String): UTF8String = srcString.trimRight()
 
-  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val evals = children.map(_.genCode(ctx))
-    val srcString = evals(0)
+  override def doEval(srcString: UTF8String, trimString: UTF8String): UTF8String =
+    srcString.trimRight(trimString)
 
-    if (evals.length == 1) {
-      ev.copy(evals.map(_.code) :+ code"""
-        boolean ${ev.isNull} = false;
-        UTF8String ${ev.value} = null;
-        if (${srcString.isNull}) {
-          ${ev.isNull} = true;
-        } else {
-          ${ev.value} = ${srcString.value}.trimRight();
-        }""")
-    } else {
-      val trimString = evals(1)
-      val getTrimRightFunction =
-        s"""
-        if (${trimString.isNull}) {
-          ${ev.isNull} = true;
-        } else {
-          ${ev.value} = ${srcString.value}.trimRight(${trimString.value});
-        }"""
-      ev.copy(evals.map(_.code) :+ code"""
-        boolean ${ev.isNull} = false;
-        UTF8String ${ev.value} = null;
-        if (${srcString.isNull}) {
-          ${ev.isNull} = true;
-        } else {
-          $getTrimRightFunction
-        }""")
-    }
-  }
+  override val trimMethod: String = "trimRight"
+
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[Expression]): StringTrimRight =
+    copy(
+      srcStr = newChildren.head,
+      trimStr = if (trimStr.isDefined) Some(newChildren.last) else None)
 }
 
 /**
@@ -1032,10 +1155,11 @@ case class StringTrimRight(
       > SELECT _FUNC_('SparkSQL', 'SQL');
        6
   """,
-  since = "1.5.0")
+  since = "1.5.0",
+  group = "string_funcs")
 // scalastyle:on line.size.limit
 case class StringInstr(str: Expression, substr: Expression)
-  extends BinaryExpression with ImplicitCastInputTypes {
+  extends BinaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def left: Expression = str
   override def right: Expression = substr
@@ -1052,6 +1176,9 @@ case class StringInstr(str: Expression, substr: Expression)
     defineCodeGen(ctx, ev, (l, r) =>
       s"($l).indexOf($r, 0) + 1")
   }
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): StringInstr = copy(str = newLeft, substr = newRight)
 }
 
 /**
@@ -1074,14 +1201,17 @@ case class StringInstr(str: Expression, substr: Expression)
       > SELECT _FUNC_('www.apache.org', '.', 2);
        www.apache
   """,
-  since = "1.5.0")
+  since = "1.5.0",
+  group = "string_funcs")
 // scalastyle:on line.size.limit
 case class SubstringIndex(strExpr: Expression, delimExpr: Expression, countExpr: Expression)
- extends TernaryExpression with ImplicitCastInputTypes {
+ extends TernaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def dataType: DataType = StringType
   override def inputTypes: Seq[DataType] = Seq(StringType, StringType, IntegerType)
-  override def children: Seq[Expression] = Seq(strExpr, delimExpr, countExpr)
+  override def first: Expression = strExpr
+  override def second: Expression = delimExpr
+  override def third: Expression = countExpr
   override def prettyName: String = "substring_index"
 
   override def nullSafeEval(str: Any, delim: Any, count: Any): Any = {
@@ -1093,6 +1223,10 @@ case class SubstringIndex(strExpr: Expression, delimExpr: Expression, countExpr:
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     defineCodeGen(ctx, ev, (str, delim, count) => s"$str.subStringIndex($delim, $count)")
   }
+
+  override protected def withNewChildrenInternal(
+      newFirst: Expression, newSecond: Expression, newThird: Expression): SubstringIndex =
+    copy(strExpr = newFirst, delimExpr = newSecond, countExpr = newThird)
 }
 
 /**
@@ -1114,7 +1248,8 @@ case class SubstringIndex(strExpr: Expression, delimExpr: Expression, countExpr:
       > SELECT POSITION('bar' IN 'foobarbar');
        4
   """,
-  since = "1.5.0")
+  since = "1.5.0",
+  group = "string_funcs")
 // scalastyle:on line.size.limit
 case class StringLocate(substr: Expression, str: Expression, start: Expression)
   extends TernaryExpression with ImplicitCastInputTypes {
@@ -1123,7 +1258,9 @@ case class StringLocate(substr: Expression, str: Expression, start: Expression)
     this(substr, str, Literal(1))
   }
 
-  override def children: Seq[Expression] = substr :: str :: start :: Nil
+  override def first: Expression = substr
+  override def second: Expression = str
+  override def third: Expression = start
   override def nullable: Boolean = substr.nullable || str.nullable
   override def dataType: DataType = IntegerType
   override def inputTypes: Seq[DataType] = Seq(StringType, StringType, IntegerType)
@@ -1182,7 +1319,13 @@ case class StringLocate(substr: Expression, str: Expression, start: Expression)
      """)
   }
 
-  override def prettyName: String = "locate"
+  override def prettyName: String =
+    getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("locate")
+
+  override protected def withNewChildrenInternal(
+      newFirst: Expression, newSecond: Expression, newThird: Expression): StringLocate =
+    copy(substr = newFirst, str = newSecond, start = newThird)
+
 }
 
 /**
@@ -1203,15 +1346,18 @@ case class StringLocate(substr: Expression, str: Expression, start: Expression)
       > SELECT _FUNC_('hi', 5);
           hi
   """,
-  since = "1.5.0")
+  since = "1.5.0",
+  group = "string_funcs")
 case class StringLPad(str: Expression, len: Expression, pad: Expression = Literal(" "))
-  extends TernaryExpression with ImplicitCastInputTypes {
+  extends TernaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   def this(str: Expression, len: Expression) = {
     this(str, len, Literal(" "))
   }
 
-  override def children: Seq[Expression] = str :: len :: pad :: Nil
+  override def first: Expression = str
+  override def second: Expression = len
+  override def third: Expression = pad
   override def dataType: DataType = StringType
   override def inputTypes: Seq[DataType] = Seq(StringType, IntegerType, StringType)
 
@@ -1224,6 +1370,10 @@ case class StringLPad(str: Expression, len: Expression, pad: Expression = Litera
   }
 
   override def prettyName: String = "lpad"
+
+  override protected def withNewChildrenInternal(
+      newFirst: Expression, newSecond: Expression, newThird: Expression): StringLPad =
+    copy(str = newFirst, len = newSecond, pad = newThird)
 }
 
 /**
@@ -1244,15 +1394,19 @@ case class StringLPad(str: Expression, len: Expression, pad: Expression = Litera
       > SELECT _FUNC_('hi', 5);
        hi
   """,
-  since = "1.5.0")
+  since = "1.5.0",
+  group = "string_funcs")
 case class StringRPad(str: Expression, len: Expression, pad: Expression = Literal(" "))
-  extends TernaryExpression with ImplicitCastInputTypes {
+  extends TernaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   def this(str: Expression, len: Expression) = {
     this(str, len, Literal(" "))
   }
 
-  override def children: Seq[Expression] = str :: len :: pad :: Nil
+  override def first: Expression = str
+  override def second: Expression = len
+  override def third: Expression = pad
+
   override def dataType: DataType = StringType
   override def inputTypes: Seq[DataType] = Seq(StringType, IntegerType, StringType)
 
@@ -1265,6 +1419,10 @@ case class StringRPad(str: Expression, len: Expression, pad: Expression = Litera
   }
 
   override def prettyName: String = "rpad"
+
+  override protected def withNewChildrenInternal(
+      newFirst: Expression, newSecond: Expression, newThird: Expression): StringRPad =
+    copy(str = newFirst, len = newSecond, pad = newThird)
 }
 
 object ParseUrl {
@@ -1294,9 +1452,11 @@ object ParseUrl {
       > SELECT _FUNC_('http://spark.apache.org/path?query=1', 'QUERY', 'query');
        1
   """,
-  since = "2.0.0")
-case class ParseUrl(children: Seq[Expression])
+  since = "2.0.0",
+  group = "string_funcs")
+case class ParseUrl(children: Seq[Expression], failOnError: Boolean = SQLConf.get.ansiEnabled)
   extends Expression with ExpectsInputTypes with CodegenFallback {
+  def this(children: Seq[Expression]) = this(children, SQLConf.get.ansiEnabled)
 
   override def nullable: Boolean = true
   override def inputTypes: Seq[DataType] = Seq.fill(children.size)(StringType)
@@ -1342,7 +1502,9 @@ case class ParseUrl(children: Seq[Expression])
     try {
       new URI(url.toString)
     } catch {
-      case e: URISyntaxException => null
+      case e: URISyntaxException if failOnError =>
+        throw QueryExecutionErrors.invalidUrlError(url, e)
+      case _: URISyntaxException => null
     }
   }
 
@@ -1433,6 +1595,9 @@ case class ParseUrl(children: Seq[Expression])
       }
     }
   }
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): ParseUrl =
+    copy(children = newChildren)
 }
 
 /**
@@ -1446,7 +1611,8 @@ case class ParseUrl(children: Seq[Expression])
       > SELECT _FUNC_("Hello World %d %s", 100, "days");
        Hello World 100 days
   """,
-  since = "1.5.0")
+  since = "1.5.0",
+  group = "string_funcs")
 // scalastyle:on line.size.limit
 case class FormatString(children: Expression*) extends Expression with ImplicitCastInputTypes {
 
@@ -1519,6 +1685,9 @@ case class FormatString(children: Expression*) extends Expression with ImplicitC
 
   override def prettyName: String = getTagValue(
     FunctionRegistry.FUNC_ALIAS).getOrElse("format_string")
+
+  override protected def withNewChildrenInternal(
+    newChildren: IndexedSeq[Expression]): FormatString = FormatString(newChildren: _*)
 }
 
 /**
@@ -1535,8 +1704,10 @@ case class FormatString(children: Expression*) extends Expression with ImplicitC
       > SELECT _FUNC_('sPark sql');
        Spark Sql
   """,
-  since = "1.5.0")
-case class InitCap(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
+  since = "1.5.0",
+  group = "string_funcs")
+case class InitCap(child: Expression)
+  extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def inputTypes: Seq[DataType] = Seq(StringType)
   override def dataType: DataType = StringType
@@ -1549,6 +1720,9 @@ case class InitCap(child: Expression) extends UnaryExpression with ImplicitCastI
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     defineCodeGen(ctx, ev, str => s"$str.toLowerCase().toTitleCase()")
   }
+
+  override protected def withNewChildInternal(newChild: Expression): InitCap =
+    copy(child = newChild)
 }
 
 /**
@@ -1561,9 +1735,10 @@ case class InitCap(child: Expression) extends UnaryExpression with ImplicitCastI
       > SELECT _FUNC_('123', 2);
        123123
   """,
-  since = "1.5.0")
+  since = "1.5.0",
+  group = "string_funcs")
 case class StringRepeat(str: Expression, times: Expression)
-  extends BinaryExpression with ImplicitCastInputTypes {
+  extends BinaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def left: Expression = str
   override def right: Expression = times
@@ -1579,6 +1754,9 @@ case class StringRepeat(str: Expression, times: Expression)
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     defineCodeGen(ctx, ev, (l, r) => s"($l).repeat($r)")
   }
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): StringRepeat = copy(str = newLeft, times = newRight)
 }
 
 /**
@@ -1591,9 +1769,10 @@ case class StringRepeat(str: Expression, times: Expression)
       > SELECT concat(_FUNC_(2), '1');
          1
   """,
-  since = "1.5.0")
+  since = "1.5.0",
+  group = "string_funcs")
 case class StringSpace(child: Expression)
-  extends UnaryExpression with ImplicitCastInputTypes {
+  extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def dataType: DataType = StringType
   override def inputTypes: Seq[DataType] = Seq(IntegerType)
@@ -1609,6 +1788,9 @@ case class StringSpace(child: Expression)
   }
 
   override def prettyName: String = "space"
+
+  override protected def withNewChildInternal(newChild: Expression): StringSpace =
+    copy(child = newChild)
 }
 
 /**
@@ -1639,7 +1821,8 @@ case class StringSpace(child: Expression)
       > SELECT _FUNC_('Spark SQL' FROM 5 FOR 1);
        k
   """,
-  since = "1.5.0")
+  since = "1.5.0",
+  group = "string_funcs")
 // scalastyle:on line.size.limit
 case class Substring(str: Expression, pos: Expression, len: Expression)
   extends TernaryExpression with ImplicitCastInputTypes with NullIntolerant {
@@ -1653,7 +1836,9 @@ case class Substring(str: Expression, pos: Expression, len: Expression)
   override def inputTypes: Seq[AbstractDataType] =
     Seq(TypeCollection(StringType, BinaryType), IntegerType, IntegerType)
 
-  override def children: Seq[Expression] = str :: pos :: len :: Nil
+  override def first: Expression = str
+  override def second: Expression = pos
+  override def third: Expression = len
 
   override def nullSafeEval(string: Any, pos: Any, len: Any): Any = {
     str.dataType match {
@@ -1673,6 +1858,11 @@ case class Substring(str: Expression, pos: Expression, len: Expression)
       }
     })
   }
+
+  override protected def withNewChildrenInternal(
+      newFirst: Expression, newSecond: Expression, newThird: Expression): Substring =
+    copy(str = newFirst, pos = newSecond, len = newThird)
+
 }
 
 /**
@@ -1686,7 +1876,8 @@ case class Substring(str: Expression, pos: Expression, len: Expression)
       > SELECT _FUNC_('Spark SQL', 3);
        SQL
   """,
-  since = "2.3.0")
+  since = "2.3.0",
+  group = "string_funcs")
 // scalastyle:on line.size.limit
 case class Right(str: Expression, len: Expression, child: Expression) extends RuntimeReplaceable {
   def this(str: Expression, len: Expression) = {
@@ -1696,6 +1887,8 @@ case class Right(str: Expression, len: Expression, child: Expression) extends Ru
 
   override def flatArguments: Iterator[Any] = Iterator(str, len)
   override def exprsReplaced: Seq[Expression] = Seq(str, len)
+
+  override protected def withNewChildInternal(newChild: Expression): Right = copy(child = newChild)
 }
 
 /**
@@ -1709,7 +1902,8 @@ case class Right(str: Expression, len: Expression, child: Expression) extends Ru
       > SELECT _FUNC_('Spark SQL', 3);
        Spa
   """,
-  since = "2.3.0")
+  since = "2.3.0",
+  group = "string_funcs")
 // scalastyle:on line.size.limit
 case class Left(str: Expression, len: Expression, child: Expression) extends RuntimeReplaceable {
   def this(str: Expression, len: Expression) = {
@@ -1718,6 +1912,7 @@ case class Left(str: Expression, len: Expression, child: Expression) extends Run
 
   override def flatArguments: Iterator[Any] = Iterator(str, len)
   override def exprsReplaced: Seq[Expression] = Seq(str, len)
+  override protected def withNewChildInternal(newChild: Expression): Left = copy(child = newChild)
 }
 
 /**
@@ -1736,9 +1931,11 @@ case class Left(str: Expression, len: Expression, child: Expression) extends Run
       > SELECT CHARACTER_LENGTH('Spark SQL ');
        10
   """,
-  since = "1.5.0")
+  since = "1.5.0",
+  group = "string_funcs")
 // scalastyle:on line.size.limit
-case class Length(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
+case class Length(child: Expression)
+  extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
   override def dataType: DataType = IntegerType
   override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection(StringType, BinaryType))
 
@@ -1753,6 +1950,8 @@ case class Length(child: Expression) extends UnaryExpression with ImplicitCastIn
       case BinaryType => defineCodeGen(ctx, ev, c => s"($c).length")
     }
   }
+
+  override protected def withNewChildInternal(newChild: Expression): Length = copy(child = newChild)
 }
 
 /**
@@ -1765,8 +1964,10 @@ case class Length(child: Expression) extends UnaryExpression with ImplicitCastIn
       > SELECT _FUNC_('Spark SQL');
        72
   """,
-  since = "2.3.0")
-case class BitLength(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
+  since = "2.3.0",
+  group = "string_funcs")
+case class BitLength(child: Expression)
+  extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
   override def dataType: DataType = IntegerType
   override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection(StringType, BinaryType))
 
@@ -1783,6 +1984,9 @@ case class BitLength(child: Expression) extends UnaryExpression with ImplicitCas
   }
 
   override def prettyName: String = "bit_length"
+
+  override protected def withNewChildInternal(newChild: Expression): BitLength =
+    copy(child = newChild)
 }
 
 /**
@@ -1796,8 +2000,10 @@ case class BitLength(child: Expression) extends UnaryExpression with ImplicitCas
       > SELECT _FUNC_('Spark SQL');
        9
   """,
-  since = "2.3.0")
-case class OctetLength(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
+  since = "2.3.0",
+  group = "string_funcs")
+case class OctetLength(child: Expression)
+  extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
   override def dataType: DataType = IntegerType
   override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection(StringType, BinaryType))
 
@@ -1814,6 +2020,9 @@ case class OctetLength(child: Expression) extends UnaryExpression with ImplicitC
   }
 
   override def prettyName: String = "octet_length"
+
+  override protected def withNewChildInternal(newChild: Expression): OctetLength =
+    copy(child = newChild)
 }
 
 /**
@@ -1826,9 +2035,10 @@ case class OctetLength(child: Expression) extends UnaryExpression with ImplicitC
       > SELECT _FUNC_('kitten', 'sitting');
        3
   """,
-  since = "1.5.0")
+  since = "1.5.0",
+  group = "string_funcs")
 case class Levenshtein(left: Expression, right: Expression) extends BinaryExpression
-    with ImplicitCastInputTypes {
+    with ImplicitCastInputTypes with NullIntolerant {
 
   override def inputTypes: Seq[AbstractDataType] = Seq(StringType, StringType)
 
@@ -1840,6 +2050,9 @@ case class Levenshtein(left: Expression, right: Expression) extends BinaryExpres
     nullSafeCodeGen(ctx, ev, (left, right) =>
       s"${ev.value} = $left.levenshteinDistance($right);")
   }
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): Levenshtein = copy(left = newLeft, right = newRight)
 }
 
 /**
@@ -1852,8 +2065,10 @@ case class Levenshtein(left: Expression, right: Expression) extends BinaryExpres
       > SELECT _FUNC_('Miller');
        M460
   """,
-  since = "1.5.0")
-case class SoundEx(child: Expression) extends UnaryExpression with ExpectsInputTypes {
+  since = "1.5.0",
+  group = "string_funcs")
+case class SoundEx(child: Expression)
+  extends UnaryExpression with ExpectsInputTypes with NullIntolerant {
 
   override def dataType: DataType = StringType
 
@@ -1864,6 +2079,9 @@ case class SoundEx(child: Expression) extends UnaryExpression with ExpectsInputT
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     defineCodeGen(ctx, ev, c => s"$c.soundex()")
   }
+
+  override protected def withNewChildInternal(newChild: Expression): SoundEx =
+    copy(child = newChild)
 }
 
 /**
@@ -1878,8 +2096,10 @@ case class SoundEx(child: Expression) extends UnaryExpression with ExpectsInputT
       > SELECT _FUNC_(2);
        50
   """,
-  since = "1.5.0")
-case class Ascii(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
+  since = "1.5.0",
+  group = "string_funcs")
+case class Ascii(child: Expression)
+  extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def dataType: DataType = IntegerType
   override def inputTypes: Seq[DataType] = Seq(StringType)
@@ -1905,6 +2125,8 @@ case class Ascii(child: Expression) extends UnaryExpression with ImplicitCastInp
         }
        """})
   }
+
+  override protected def withNewChildInternal(newChild: Expression): Ascii = copy(child = newChild)
 }
 
 /**
@@ -1919,9 +2141,11 @@ case class Ascii(child: Expression) extends UnaryExpression with ImplicitCastInp
       > SELECT _FUNC_(65);
        A
   """,
-  since = "2.3.0")
+  since = "2.3.0",
+  group = "string_funcs")
 // scalastyle:on line.size.limit
-case class Chr(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
+case class Chr(child: Expression)
+  extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def dataType: DataType = StringType
   override def inputTypes: Seq[DataType] = Seq(LongType)
@@ -1951,6 +2175,8 @@ case class Chr(child: Expression) extends UnaryExpression with ImplicitCastInput
       """
     })
   }
+
+  override protected def withNewChildInternal(newChild: Expression): Chr = copy(child = newChild)
 }
 
 /**
@@ -1963,8 +2189,10 @@ case class Chr(child: Expression) extends UnaryExpression with ImplicitCastInput
       > SELECT _FUNC_('Spark SQL');
        U3BhcmsgU1FM
   """,
-  since = "1.5.0")
-case class Base64(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
+  since = "1.5.0",
+  group = "string_funcs")
+case class Base64(child: Expression)
+  extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def dataType: DataType = StringType
   override def inputTypes: Seq[DataType] = Seq(BinaryType)
@@ -1979,6 +2207,8 @@ case class Base64(child: Expression) extends UnaryExpression with ImplicitCastIn
             ${classOf[CommonsBase64].getName}.encodeBase64($child));
        """})
   }
+
+  override protected def withNewChildInternal(newChild: Expression): Base64 = copy(child = newChild)
 }
 
 /**
@@ -1991,8 +2221,10 @@ case class Base64(child: Expression) extends UnaryExpression with ImplicitCastIn
       > SELECT _FUNC_('U3BhcmsgU1FM');
        Spark SQL
   """,
-  since = "1.5.0")
-case class UnBase64(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
+  since = "1.5.0",
+  group = "string_funcs")
+case class UnBase64(child: Expression)
+  extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def dataType: DataType = BinaryType
   override def inputTypes: Seq[DataType] = Seq(StringType)
@@ -2006,6 +2238,70 @@ case class UnBase64(child: Expression) extends UnaryExpression with ImplicitCast
          ${ev.value} = ${classOf[CommonsBase64].getName}.decodeBase64($child.toString());
        """})
   }
+
+  override protected def withNewChildInternal(newChild: Expression): UnBase64 =
+    copy(child = newChild)
+}
+
+object Decode {
+  def createExpr(params: Seq[Expression]): Expression = {
+    params.length match {
+      case 0 | 1 =>
+        throw QueryCompilationErrors.invalidFunctionArgumentsError("decode", "2", params.length)
+      case 2 => StringDecode(params.head, params.last)
+      case _ =>
+        val input = params.head
+        val other = params.tail
+        val itr = other.iterator
+        var default: Expression = Literal.create(null, StringType)
+        val branches = ArrayBuffer.empty[(Expression, Expression)]
+        while (itr.hasNext) {
+          val search = itr.next
+          if (itr.hasNext) {
+            val condition = EqualTo(input, search)
+            branches += ((condition, itr.next))
+          } else {
+            default = search
+          }
+        }
+        CaseWhen(branches.seq.toSeq, default)
+    }
+  }
+}
+
+// scalastyle:off line.size.limit
+@ExpressionDescription(
+  usage = """
+            |_FUNC_(bin, charset) - Decodes the first argument using the second argument character set.
+            |
+            |_FUNC_(expr, search, result [, search, result ] ... [, default]) - Decode compares expr
+            |  to each search value one by one. If expr is equal to a search, returns the corresponding result.
+            |  If no match is found, then Oracle returns default. If default is omitted, returns null.
+          """,
+  examples = """
+    Examples:
+      > SELECT _FUNC_(encode('abc', 'utf-8'), 'utf-8');
+       abc
+      > SELECT _FUNC_(2, 1, 'Southlake', 2, 'San Francisco', 3, 'New Jersey', 4, 'Seattle', 'Non domestic');
+       San Francisco
+      > SELECT _FUNC_(6, 1, 'Southlake', 2, 'San Francisco', 3, 'New Jersey', 4, 'Seattle', 'Non domestic');
+       Non domestic
+      > SELECT _FUNC_(6, 1, 'Southlake', 2, 'San Francisco', 3, 'New Jersey', 4, 'Seattle');
+       NULL
+  """,
+  since = "3.2.0",
+  group = "string_funcs")
+// scalastyle:on line.size.limit
+case class Decode(params: Seq[Expression], child: Expression) extends RuntimeReplaceable {
+
+  def this(params: Seq[Expression]) = {
+    this(params, Decode.createExpr(params))
+  }
+
+  override def flatArguments: Iterator[Any] = Iterator(params)
+  override def exprsReplaced: Seq[Expression] = params
+
+  override protected def withNewChildInternal(newChild: Expression): Decode = copy(child = newChild)
 }
 
 /**
@@ -2021,10 +2317,11 @@ case class UnBase64(child: Expression) extends UnaryExpression with ImplicitCast
       > SELECT _FUNC_(encode('abc', 'utf-8'), 'utf-8');
        abc
   """,
-  since = "1.5.0")
+  since = "1.5.0",
+  group = "string_funcs")
 // scalastyle:on line.size.limit
-case class Decode(bin: Expression, charset: Expression)
-  extends BinaryExpression with ImplicitCastInputTypes {
+case class StringDecode(bin: Expression, charset: Expression)
+  extends BinaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def left: Expression = bin
   override def right: Expression = charset
@@ -2046,6 +2343,10 @@ case class Decode(bin: Expression, charset: Expression)
         }
       """)
   }
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): StringDecode =
+    copy(bin = newLeft, charset = newRight)
 }
 
 /**
@@ -2061,10 +2362,11 @@ case class Decode(bin: Expression, charset: Expression)
       > SELECT _FUNC_('abc', 'utf-8');
        abc
   """,
-  since = "1.5.0")
+  since = "1.5.0",
+  group = "string_funcs")
 // scalastyle:on line.size.limit
 case class Encode(value: Expression, charset: Expression)
-  extends BinaryExpression with ImplicitCastInputTypes {
+  extends BinaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
   override def left: Expression = value
   override def right: Expression = charset
@@ -2085,6 +2387,9 @@ case class Encode(value: Expression, charset: Expression)
           org.apache.spark.unsafe.Platform.throwException(e);
         }""")
   }
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): Encode = copy(value = newLeft, charset = newRight)
 }
 
 /**
@@ -2106,9 +2411,10 @@ case class Encode(value: Expression, charset: Expression)
       > SELECT _FUNC_(12332.123456, '##################.###');
        12332.123
   """,
-  since = "1.5.0")
+  since = "1.5.0",
+  group = "string_funcs")
 case class FormatNumber(x: Expression, d: Expression)
-  extends BinaryExpression with ExpectsInputTypes {
+  extends BinaryExpression with ExpectsInputTypes with NullIntolerant {
 
   override def left: Expression = x
   override def right: Expression = d
@@ -2264,6 +2570,9 @@ case class FormatNumber(x: Expression, d: Expression)
   }
 
   override def prettyName: String = "format_number"
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): FormatNumber = copy(x = newLeft, d = newRight)
 }
 
 /**
@@ -2277,12 +2586,13 @@ case class FormatNumber(x: Expression, d: Expression)
       > SELECT _FUNC_('Hi there! Good morning.');
        [["Hi","there"],["Good","morning"]]
   """,
-  since = "2.0.0")
+  since = "2.0.0",
+  group = "string_funcs")
 case class Sentences(
     str: Expression,
     language: Expression = Literal(""),
     country: Expression = Literal(""))
-  extends Expression with ImplicitCastInputTypes with CodegenFallback {
+  extends TernaryExpression with ImplicitCastInputTypes with CodegenFallback {
 
   def this(str: Expression) = this(str, Literal(""), Literal(""))
   def this(str: Expression, language: Expression) = this(str, language, Literal(""))
@@ -2291,7 +2601,9 @@ case class Sentences(
   override def dataType: DataType =
     ArrayType(ArrayType(StringType, containsNull = false), containsNull = false)
   override def inputTypes: Seq[AbstractDataType] = Seq(StringType, StringType, StringType)
-  override def children: Seq[Expression] = str :: language :: country :: Nil
+  override def first: Expression = str
+  override def second: Expression = language
+  override def third: Expression = country
 
   override def eval(input: InternalRow): Any = {
     val string = str.eval(input)
@@ -2327,8 +2639,13 @@ case class Sentences(
         widx = wi.current
         if (Character.isLetterOrDigit(word.charAt(0))) words += UTF8String.fromString(word)
       }
-      result += new GenericArrayData(words)
+      result += new GenericArrayData(words.toSeq)
     }
-    new GenericArrayData(result)
+    new GenericArrayData(result.toSeq)
   }
+
+  override protected def withNewChildrenInternal(
+      newFirst: Expression, newSecond: Expression, newThird: Expression): Sentences =
+    copy(str = newFirst, language = newSecond, country = newThird)
+
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/subquery.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/subquery.scala
index f46a1c6836fcf..ac939bf6d13c7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/subquery.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/subquery.scala
@@ -22,12 +22,29 @@ import scala.collection.mutable.ArrayBuffer
 import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
 import org.apache.spark.sql.catalyst.plans.QueryPlan
 import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan}
+import org.apache.spark.sql.catalyst.trees.TreePattern.{EXISTS_SUBQUERY, LIST_SUBQUERY,
+  PLAN_EXPRESSION, SCALAR_SUBQUERY, TreePattern}
 import org.apache.spark.sql.types._
+import org.apache.spark.util.collection.BitSet
 
 /**
  * An interface for expressions that contain a [[QueryPlan]].
  */
 abstract class PlanExpression[T <: QueryPlan[_]] extends Expression {
+
+  // Override `treePatternBits` to propagate bits for its internal plan.
+  override lazy val treePatternBits: BitSet = {
+    val bits: BitSet = getDefaultTreePatternBits
+    // Propagate its query plan's pattern bits
+    bits.union(plan.treePatternBits)
+    bits
+  }
+
+  final override val nodePatterns: Seq[TreePattern] = Seq(PLAN_EXPRESSION) ++ nodePatternsInternal
+
+  // Subclasses can override this function to provide more TreePatterns.
+  def nodePatternsInternal(): Seq[TreePattern] = Seq()
+
   /**  The id of the subquery expression. */
   def exprId: ExprId
 
@@ -130,14 +147,11 @@ object SubExprUtils extends PredicateHelper {
    * Given a logical plan, returns TRUE if it has an outer reference and false otherwise.
    */
   def hasOuterReferences(plan: LogicalPlan): Boolean = {
-    plan.find {
-      case f: Filter => containsOuter(f.condition)
-      case other => false
-    }.isDefined
+    plan.find(_.expressions.exists(containsOuter)).isDefined
   }
 
   /**
-   * Given a list of expressions, returns the expressions which have outer references. Aggregate
+   * Given an expression, returns the expressions which have outer references. Aggregate
    * expressions are treated in a special way. If the children of aggregate expression contains an
    * outer reference, then the entire aggregate expression is marked as an outer reference.
    * Example (SQL):
@@ -173,20 +187,20 @@ object SubExprUtils extends PredicateHelper {
    * }}}
    * The code below needs to change when we support the above cases.
    */
-  def getOuterReferences(conditions: Seq[Expression]): Seq[Expression] = {
+  def getOuterReferences(expr: Expression): Seq[Expression] = {
     val outerExpressions = ArrayBuffer.empty[Expression]
-    conditions foreach { expr =>
-      expr transformDown {
-        case a: AggregateExpression if a.collectLeaves.forall(_.isInstanceOf[OuterReference]) =>
-          val newExpr = stripOuterReference(a)
-          outerExpressions += newExpr
-          newExpr
-        case OuterReference(e) =>
-          outerExpressions += e
-          e
-      }
+    expr transformDown {
+      case a: AggregateExpression if a.collectLeaves.forall(_.isInstanceOf[OuterReference]) =>
+        // Collect and update the sub-tree so that outer references inside this aggregate
+        // expression will not be collected. For example: min(outer(a)) -> min(a).
+        val newExpr = stripOuterReference(a)
+        outerExpressions += newExpr
+        newExpr
+      case OuterReference(e) =>
+        outerExpressions += e
+        e
     }
-    outerExpressions
+    outerExpressions.toSeq
   }
 
   /**
@@ -194,8 +208,7 @@ object SubExprUtils extends PredicateHelper {
    * Filter operator can host outer references.
    */
   def getOuterReferences(plan: LogicalPlan): Seq[Expression] = {
-    val conditions = plan.collect { case Filter(cond, _) => cond }
-    getOuterReferences(conditions)
+    plan.flatMap(_.expressions.flatMap(getOuterReferences))
   }
 
   /**
@@ -238,6 +251,11 @@ case class ScalarSubquery(
       children.map(_.canonicalized),
       ExprId(0))
   }
+
+  override protected def withNewChildrenInternal(
+    newChildren: IndexedSeq[Expression]): ScalarSubquery = copy(children = newChildren)
+
+  final override def nodePatternsInternal: Seq[TreePattern] = Seq(SCALAR_SUBQUERY)
 }
 
 object ScalarSubquery {
@@ -283,6 +301,11 @@ case class ListQuery(
       ExprId(0),
       childOutputs.map(_.canonicalized.asInstanceOf[Attribute]))
   }
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): ListQuery =
+    copy(children = newChildren)
+
+  final override def nodePatternsInternal: Seq[TreePattern] = Seq(LIST_SUBQUERY)
 }
 
 /**
@@ -325,4 +348,9 @@ case class Exists(
       children.map(_.canonicalized),
       ExprId(0))
   }
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Exists =
+    copy(children = newChildren)
+
+  final override def nodePatternsInternal: Seq[TreePattern] = Seq(EXISTS_SUBQUERY)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
index c8b6433207355..fe9c41e387964 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
@@ -23,6 +23,9 @@ import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, UnresolvedExcept
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateFunction, DeclarativeAggregate, NoOp}
+import org.apache.spark.sql.catalyst.trees.{BinaryLike, LeafLike, TernaryLike, UnaryLike}
+import org.apache.spark.sql.catalyst.trees.TreePattern.{TreePattern, WINDOW_EXPRESSION}
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.types._
 
 /**
@@ -45,13 +48,19 @@ case class WindowSpecDefinition(
 
   override def children: Seq[Expression] = partitionSpec ++ orderSpec :+ frameSpecification
 
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[Expression]): WindowSpecDefinition =
+    copy(
+      partitionSpec = newChildren.take(partitionSpec.size),
+      orderSpec = newChildren.drop(partitionSpec.size).dropRight(1).asInstanceOf[Seq[SortOrder]],
+      frameSpecification = newChildren.last.asInstanceOf[WindowFrame])
+
   override lazy val resolved: Boolean =
     childrenResolved && checkInputDataTypes().isSuccess &&
       frameSpecification.isInstanceOf[SpecifiedWindowFrame]
 
   override def nullable: Boolean = true
-  override def foldable: Boolean = false
-  override def dataType: DataType = throw new UnsupportedOperationException("dataType")
+  override def dataType: DataType = throw QueryExecutionErrors.dataTypeOperationUnsupportedError
 
   override def checkInputDataTypes(): TypeCheckResult = {
     frameSpecification match {
@@ -92,7 +101,10 @@ case class WindowSpecDefinition(
 
   private def isValidFrameType(ft: DataType): Boolean = (orderSpec.head.dataType, ft) match {
     case (DateType, IntegerType) => true
+    case (DateType, YearMonthIntervalType) => true
     case (TimestampType, CalendarIntervalType) => true
+    case (TimestampType, YearMonthIntervalType) => true
+    case (TimestampType, DayTimeIntervalType) => true
     case (a, b) => a == b
   }
 }
@@ -141,10 +153,8 @@ case object RangeFrame extends FrameType {
 /**
  * The trait used to represent special boundaries used in a window frame.
  */
-sealed trait SpecialFrameBoundary extends Expression with Unevaluable {
-  override def children: Seq[Expression] = Nil
+sealed trait SpecialFrameBoundary extends LeafExpression with Unevaluable {
   override def dataType: DataType = NullType
-  override def foldable: Boolean = false
   override def nullable: Boolean = false
 }
 
@@ -166,26 +176,25 @@ case object CurrentRow extends SpecialFrameBoundary {
  * Represents a window frame.
  */
 sealed trait WindowFrame extends Expression with Unevaluable {
-  override def children: Seq[Expression] = Nil
-  override def dataType: DataType = throw new UnsupportedOperationException("dataType")
-  override def foldable: Boolean = false
+  override def dataType: DataType = throw QueryExecutionErrors.dataTypeOperationUnsupportedError
   override def nullable: Boolean = false
 }
 
 /** Used as a placeholder when a frame specification is not defined. */
-case object UnspecifiedFrame extends WindowFrame
+case object UnspecifiedFrame extends WindowFrame with LeafLike[Expression]
 
 /**
- * A specified Window Frame. The val lower/uppper can be either a foldable [[Expression]] or a
+ * A specified Window Frame. The val lower/upper can be either a foldable [[Expression]] or a
  * [[SpecialFrameBoundary]].
  */
 case class SpecifiedWindowFrame(
     frameType: FrameType,
     lower: Expression,
     upper: Expression)
-  extends WindowFrame {
+  extends WindowFrame with BinaryLike[Expression] {
 
-  override def children: Seq[Expression] = lower :: upper :: Nil
+  override def left: Expression = lower
+  override def right: Expression = upper
 
   lazy val valueBoundary: Seq[Expression] =
     children.filterNot(_.isInstanceOf[SpecialFrameBoundary])
@@ -238,7 +247,7 @@ case class SpecifiedWindowFrame(
 
   private def boundarySql(expr: Expression): String = expr match {
     case e: SpecialFrameBoundary => e.sql
-    case UnaryMinus(n) => n.sql + " PRECEDING"
+    case UnaryMinus(n, _) => n.sql + " PRECEDING"
     case e: Expression => e.sql + " FOLLOWING"
   }
 
@@ -268,30 +277,43 @@ case class SpecifiedWindowFrame(
       case _ => true
     }
   }
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): SpecifiedWindowFrame =
+    copy(lower = newLeft, upper = newRight)
 }
 
 case class UnresolvedWindowExpression(
     child: Expression,
     windowSpec: WindowSpecReference) extends UnaryExpression with Unevaluable {
 
-  override def dataType: DataType = throw new UnresolvedException(this, "dataType")
-  override def foldable: Boolean = throw new UnresolvedException(this, "foldable")
-  override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
+  override def dataType: DataType = throw new UnresolvedException("dataType")
+  override def nullable: Boolean = throw new UnresolvedException("nullable")
   override lazy val resolved = false
+
+  override protected def withNewChildInternal(newChild: Expression): UnresolvedWindowExpression =
+    copy(child = newChild)
 }
 
 case class WindowExpression(
     windowFunction: Expression,
-    windowSpec: WindowSpecDefinition) extends Expression with Unevaluable {
+    windowSpec: WindowSpecDefinition) extends Expression with Unevaluable
+  with BinaryLike[Expression] {
 
-  override def children: Seq[Expression] = windowFunction :: windowSpec :: Nil
+  override def left: Expression = windowFunction
+  override def right: Expression = windowSpec
 
   override def dataType: DataType = windowFunction.dataType
-  override def foldable: Boolean = windowFunction.foldable
   override def nullable: Boolean = windowFunction.nullable
 
   override def toString: String = s"$windowFunction $windowSpec"
   override def sql: String = windowFunction.sql + " OVER " + windowSpec.sql
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): WindowExpression =
+    copy(windowFunction = newLeft, windowSpec = newRight.asInstanceOf[WindowSpecDefinition])
+
+  override val nodePatterns: Seq[TreePattern] = Seq(WINDOW_EXPRESSION)
 }
 
 /**
@@ -332,59 +354,65 @@ object WindowFunctionType {
   }
 }
 
-
-/**
- * An offset window function is a window function that returns the value of the input column offset
- * by a number of rows within the partition. For instance: an OffsetWindowfunction for value x with
- * offset -2, will get the value of x 2 rows back in the partition.
- */
-abstract class OffsetWindowFunction
-  extends Expression with WindowFunction with Unevaluable with ImplicitCastInputTypes {
+trait OffsetWindowFunction extends WindowFunction {
   /**
    * Input expression to evaluate against a row which a number of rows below or above (depending on
-   * the value and sign of the offset) the current row.
+   * the value and sign of the offset) the starting row (current row if isRelative=true, or the
+   * first row of the window frame otherwise).
    */
   val input: Expression
 
+  /**
+   * (Foldable) expression that contains the number of rows between the current row and the row
+   * where the input expression is evaluated. If `offset` is a positive integer, it means that
+   * the direction of the `offset` is from front to back. If it is a negative integer, the direction
+   * of the `offset` is from back to front. If it is zero, it means that the offset is ignored and
+   * use current row.
+   */
+  val offset: Expression
+
   /**
    * Default result value for the function when the `offset`th row does not exist.
    */
   val default: Expression
 
   /**
-   * (Foldable) expression that contains the number of rows between the current row and the row
-   * where the input expression is evaluated.
+   * An optional specification that indicates the offset window function should skip null values in
+   * the determination of which row to use.
    */
-  val offset: Expression
+  val ignoreNulls: Boolean
 
   /**
-   * Direction of the number of rows between the current row and the row where the input expression
-   * is evaluated.
+   * A fake window frame which is used to hold the offset information. It's used as a key to group
+   * by offset window functions in `WindowExecBase.windowFrameExpressionFactoryPairs`, as offset
+   * window functions with the same offset and same window frame can be evaluated together.
    */
-  val direction: SortDirection
+  lazy val fakeFrame = SpecifiedWindowFrame(RowFrame, offset, offset)
+}
 
-  override def children: Seq[Expression] = Seq(input, offset, default)
+/**
+ * A frameless offset window function is a window function that cannot specify window frame and
+ * returns the value of the input column offset by a number of rows according to the current row
+ * within the partition. For instance: a FrameLessOffsetWindowFunction for value x with offset -2,
+ * will get the value of x 2 rows back from the current row in the partition.
+ */
+sealed abstract class FrameLessOffsetWindowFunction
+  extends OffsetWindowFunction with Unevaluable with ImplicitCastInputTypes {
 
   /*
    * The result of an OffsetWindowFunction is dependent on the frame in which the
    * OffsetWindowFunction is executed, the input expression and the default expression. Even when
    * both the input and the default expression are foldable, the result is still not foldable due to
    * the frame.
+   *
+   * Note, the value of foldable is set to false in the trait Unevaluable
+   *
+   * override def foldable: Boolean = false
    */
-  override def foldable: Boolean = false
 
   override def nullable: Boolean = default == null || default.nullable || input.nullable
 
-  override lazy val frame: WindowFrame = {
-    val boundary = direction match {
-      case Ascending => offset
-      case Descending => UnaryMinus(offset) match {
-          case e: Expression if e.foldable => Literal.create(e.eval(EmptyRow), e.dataType)
-          case o => o
-      }
-    }
-    SpecifiedWindowFrame(RowFrame, boundary, boundary)
-  }
+  override lazy val frame: WindowFrame = fakeFrame
 
   override def checkInputDataTypes(): TypeCheckResult = {
     val check = super.checkInputDataTypes()
@@ -411,6 +439,7 @@ abstract class OffsetWindowFunction
  * integer value. The default offset is 1. When the value of `input` is null at the `offset`th row,
  * null is returned. If there is no such offset row, the `default` expression is evaluated.
  */
+// scalastyle:off line.size.limit line.contains.tab
 @ExpressionDescription(
   usage = """
     _FUNC_(input[, offset[, default]]) - Returns the value of `input` at the `offset`th row
@@ -426,10 +455,23 @@ abstract class OffsetWindowFunction
       * default - a string expression which is to use when the offset is larger than the window.
           The default value is null.
   """,
+  examples = """
+    Examples:
+      > SELECT a, b, _FUNC_(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);
+       A1	1	1
+       A1	1	2
+       A1	2	NULL
+       A2	3	NULL
+  """,
   since = "2.0.0",
   group = "window_funcs")
-case class Lead(input: Expression, offset: Expression, default: Expression)
-    extends OffsetWindowFunction {
+// scalastyle:on line.size.limit line.contains.tab
+case class Lead(
+    input: Expression, offset: Expression, default: Expression, ignoreNulls: Boolean)
+    extends FrameLessOffsetWindowFunction with TernaryLike[Expression] {
+
+  def this(input: Expression, offset: Expression, default: Expression) =
+    this(input, offset, default, false)
 
   def this(input: Expression, offset: Expression) = this(input, offset, Literal(null))
 
@@ -437,7 +479,13 @@ case class Lead(input: Expression, offset: Expression, default: Expression)
 
   def this() = this(Literal(null))
 
-  override val direction = Ascending
+  override def first: Expression = input
+  override def second: Expression = offset
+  override def third: Expression = default
+
+  override protected def withNewChildrenInternal(
+      newFirst: Expression, newSecond: Expression, newThird: Expression): Lead =
+    copy(input = newFirst, offset = newSecond, default = newThird)
 }
 
 /**
@@ -446,6 +494,7 @@ case class Lead(input: Expression, offset: Expression, default: Expression)
  * integer value. The default offset is 1. When the value of `input` is null at the `offset`th row,
  * null is returned. If there is no such offset row, the `default` expression is evaluated.
  */
+// scalastyle:off line.size.limit line.contains.tab
 @ExpressionDescription(
   usage = """
     _FUNC_(input[, offset[, default]]) - Returns the value of `input` at the `offset`th row
@@ -460,37 +509,61 @@ case class Lead(input: Expression, offset: Expression, default: Expression)
       * offset - an int expression which is rows to jump back in the partition.
       * default - a string expression which is to use when the offset row does not exist.
   """,
+  examples = """
+    Examples:
+      > SELECT a, b, _FUNC_(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);
+       A1	1	NULL
+       A1	1	1
+       A1	2	1
+       A2	3	NULL
+  """,
   since = "2.0.0",
   group = "window_funcs")
-case class Lag(input: Expression, offset: Expression, default: Expression)
-    extends OffsetWindowFunction {
+// scalastyle:on line.size.limit line.contains.tab
+case class Lag(
+    input: Expression, inputOffset: Expression, default: Expression, ignoreNulls: Boolean)
+    extends FrameLessOffsetWindowFunction with TernaryLike[Expression] {
 
-  def this(input: Expression, offset: Expression) = this(input, offset, Literal(null))
+  def this(input: Expression, inputOffset: Expression, default: Expression) =
+    this(input, inputOffset, default, false)
+
+  def this(input: Expression, inputOffset: Expression) = this(input, inputOffset, Literal(null))
 
   def this(input: Expression) = this(input, Literal(1))
 
   def this() = this(Literal(null))
 
-  override val direction = Descending
+  override val offset: Expression = UnaryMinus(inputOffset) match {
+    case e: Expression if e.foldable => Literal.create(e.eval(EmptyRow), e.dataType)
+    case o => o
+  }
+
+  override def first: Expression = input
+  override def second: Expression = inputOffset
+  override def third: Expression = default
+
+  override protected def withNewChildrenInternal(
+      newFirst: Expression, newSecond: Expression, newThird: Expression): Lag =
+    copy(input = newFirst, inputOffset = newSecond, default = newThird)
 }
 
 abstract class AggregateWindowFunction extends DeclarativeAggregate with WindowFunction {
   self: Product =>
-  override val frame = SpecifiedWindowFrame(RowFrame, UnboundedPreceding, CurrentRow)
+  override val frame: WindowFrame = SpecifiedWindowFrame(RowFrame, UnboundedPreceding, CurrentRow)
   override def dataType: DataType = IntegerType
   override def nullable: Boolean = true
   override lazy val mergeExpressions =
-    throw new UnsupportedOperationException("Window Functions do not support merging.")
+    throw QueryExecutionErrors.mergeUnsupportedByWindowFunctionError
 }
 
 abstract class RowNumberLike extends AggregateWindowFunction {
-  override def children: Seq[Expression] = Nil
   protected val zero = Literal(0)
   protected val one = Literal(1)
   protected val rowNumber = AttributeReference("rowNumber", IntegerType, nullable = false)()
   override val aggBufferAttributes: Seq[AttributeReference] = rowNumber :: Nil
   override val initialValues: Seq[Expression] = zero :: Nil
   override val updateExpressions: Seq[Expression] = rowNumber + one :: Nil
+  override def nullable: Boolean = false
 }
 
 /**
@@ -514,14 +587,24 @@ object SizeBasedWindowFunction {
  *
  * This documentation has been based upon similar documentation for the Hive and Presto projects.
  */
+// scalastyle:off line.size.limit line.contains.tab
 @ExpressionDescription(
   usage = """
     _FUNC_() - Assigns a unique, sequential number to each row, starting with one,
       according to the ordering of rows within the window partition.
   """,
+  examples = """
+    Examples:
+      > SELECT a, b, _FUNC_() OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);
+       A1	1	1
+       A1	1	2
+       A1	2	3
+       A2	3	1
+  """,
   since = "2.0.0",
   group = "window_funcs")
-case class RowNumber() extends RowNumberLike {
+// scalastyle:on line.size.limit line.contains.tab
+case class RowNumber() extends RowNumberLike with LeafLike[Expression] {
   override val evaluateExpression = rowNumber
   override def prettyName: String = "row_number"
 }
@@ -534,13 +617,23 @@ case class RowNumber() extends RowNumberLike {
  *
  * This documentation has been based upon similar documentation for the Hive and Presto projects.
  */
+// scalastyle:off line.size.limit line.contains.tab
 @ExpressionDescription(
   usage = """
     _FUNC_() - Computes the position of a value relative to all values in the partition.
   """,
+  examples = """
+    Examples:
+      > SELECT a, b, _FUNC_() OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);
+       A1	1	0.6666666666666666
+       A1	1	0.6666666666666666
+       A1	2	1.0
+       A2	3	1.0
+  """,
   since = "2.0.0",
   group = "window_funcs")
-case class CumeDist() extends RowNumberLike with SizeBasedWindowFunction {
+// scalastyle:on line.size.limit line.contains.tab
+case class CumeDist() extends RowNumberLike with SizeBasedWindowFunction with LeafLike[Expression] {
   override def dataType: DataType = DoubleType
   // The frame for CUME_DIST is Range based instead of Row based, because CUME_DIST must
   // return the same value for equal values in the partition.
@@ -549,6 +642,100 @@ case class CumeDist() extends RowNumberLike with SizeBasedWindowFunction {
   override def prettyName: String = "cume_dist"
 }
 
+// scalastyle:off line.size.limit line.contains.tab
+@ExpressionDescription(
+  usage = """
+    _FUNC_(input[, offset]) - Returns the value of `input` at the row that is the `offset`th row
+      from beginning of the window frame. Offset starts at 1. If ignoreNulls=true, we will skip
+      nulls when finding the `offset`th row. Otherwise, every row counts for the `offset`. If
+      there is no such an `offset`th row (e.g., when the offset is 10, size of the window frame
+      is less than 10), null is returned.
+  """,
+  examples = """
+    Examples:
+      > SELECT a, b, _FUNC_(b, 2) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);
+       A1	1	1
+       A1	1	1
+       A1	2	1
+       A2	3	NULL
+  """,
+  arguments = """
+    Arguments:
+      * input - the target column or expression that the function operates on.
+      * offset - a positive int literal to indicate the offset in the window frame. It starts
+          with 1.
+      * ignoreNulls - an optional specification that indicates the NthValue should skip null
+          values in the determination of which row to use.
+  """,
+  since = "3.1.0",
+  group = "window_funcs")
+// scalastyle:on line.size.limit line.contains.tab
+case class NthValue(input: Expression, offset: Expression, ignoreNulls: Boolean)
+    extends AggregateWindowFunction with OffsetWindowFunction with ImplicitCastInputTypes
+    with BinaryLike[Expression] {
+
+  def this(child: Expression, offset: Expression) = this(child, offset, false)
+
+  override lazy val default = Literal.create(null, input.dataType)
+
+  override def left: Expression = input
+  override def right: Expression = offset
+
+  override val frame: WindowFrame = UnspecifiedFrame
+
+  override def dataType: DataType = input.dataType
+
+  override def inputTypes: Seq[AbstractDataType] = Seq(AnyDataType, IntegerType)
+
+  override def checkInputDataTypes(): TypeCheckResult = {
+    val check = super.checkInputDataTypes()
+    if (check.isFailure) {
+      check
+    } else if (!offset.foldable) {
+      TypeCheckFailure(s"Offset expression '$offset' must be a literal.")
+    } else if (offsetVal <= 0) {
+      TypeCheckFailure(
+        s"The 'offset' argument of nth_value must be greater than zero but it is $offsetVal.")
+    } else {
+      TypeCheckSuccess
+    }
+  }
+
+  private lazy val offsetVal = offset.eval().asInstanceOf[Int].toLong
+  private lazy val result = AttributeReference("result", input.dataType)()
+  private lazy val count = AttributeReference("count", LongType)()
+  override lazy val aggBufferAttributes: Seq[AttributeReference] = result :: count :: Nil
+
+  override lazy val initialValues: Seq[Literal] = Seq(
+    /* result = */ default,
+    /* count = */ Literal(1L)
+  )
+
+  override lazy val updateExpressions: Seq[Expression] = {
+    if (ignoreNulls) {
+      Seq(
+        /* result = */ If(count === offsetVal && input.isNotNull, input, result),
+        /* count = */ If(input.isNull, count, count + 1L)
+      )
+    } else {
+      Seq(
+        /* result = */ If(count === offsetVal, input, result),
+        /* count = */ count + 1L
+      )
+    }
+  }
+
+  override lazy val evaluateExpression: AttributeReference = result
+
+  override def prettyName: String = "nth_value"
+  override def sql: String =
+    s"$prettyName(${input.sql}, ${offset.sql})${if (ignoreNulls) " ignore nulls" else ""}"
+
+  override protected def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): NthValue =
+    copy(input = newLeft, offset = newRight)
+}
+
 /**
  * The NTile function divides the rows for each window partition into `n` buckets ranging from 1 to
  * at most `n`. Bucket values will differ by at most 1. If the number of rows in the partition does
@@ -568,6 +755,7 @@ case class CumeDist() extends RowNumberLike with SizeBasedWindowFunction {
  *
  * This documentation has been based upon similar documentation for the Hive and Presto projects.
  */
+// scalastyle:off line.size.limit line.contains.tab
 @ExpressionDescription(
   usage = """
     _FUNC_(n) - Divides the rows for each window partition into `n` buckets ranging
@@ -578,12 +766,23 @@ case class CumeDist() extends RowNumberLike with SizeBasedWindowFunction {
       * buckets - an int expression which is number of buckets to divide the rows in.
           Default value is 1.
   """,
+  examples = """
+    Examples:
+      > SELECT a, b, _FUNC_(2) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);
+       A1	1	1
+       A1	1	1
+       A1	2	2
+       A2	3	1
+  """,
   since = "2.0.0",
   group = "window_funcs")
-case class NTile(buckets: Expression) extends RowNumberLike with SizeBasedWindowFunction {
+// scalastyle:on line.size.limit line.contains.tab
+case class NTile(buckets: Expression) extends RowNumberLike with SizeBasedWindowFunction
+  with UnaryLike[Expression] {
+
   def this() = this(Literal(1))
 
-  override def children: Seq[Expression] = Seq(buckets)
+  override def child: Expression = buckets
 
   // Validate buckets. Note that this could be relaxed, the bucket value only needs to constant
   // for each partition.
@@ -637,6 +836,9 @@ case class NTile(buckets: Expression) extends RowNumberLike with SizeBasedWindow
   )
 
   override val evaluateExpression = bucket
+
+  override protected def withNewChildInternal(
+    newChild: Expression): NTile = copy(buckets = newChild)
 }
 
 /**
@@ -681,6 +883,7 @@ abstract class RankLike extends AggregateWindowFunction {
   override val updateExpressions = increaseRank +: increaseRowNumber +: children
   override val evaluateExpression: Expression = rank
 
+  override def nullable: Boolean = false
   override def sql: String = s"${prettyName.toUpperCase(Locale.ROOT)}()"
 
   def withOrder(order: Seq[Expression]): RankLike
@@ -693,6 +896,7 @@ abstract class RankLike extends AggregateWindowFunction {
  *
  * This documentation has been based upon similar documentation for the Hive and Presto projects.
  */
+// scalastyle:off line.size.limit line.contains.tab
 @ExpressionDescription(
   usage = """
     _FUNC_() - Computes the rank of a value in a group of values. The result is one plus the number
@@ -705,11 +909,22 @@ abstract class RankLike extends AggregateWindowFunction {
           trigger a change in rank. This is an internal parameter and will be assigned by the
           Analyser.
   """,
+  examples = """
+    Examples:
+      > SELECT a, b, _FUNC_(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);
+       A1	1	1
+       A1	1	1
+       A1	2	3
+       A2	3	1
+  """,
   since = "2.0.0",
   group = "window_funcs")
+// scalastyle:on line.size.limit line.contains.tab
 case class Rank(children: Seq[Expression]) extends RankLike {
   def this() = this(Nil)
   override def withOrder(order: Seq[Expression]): Rank = Rank(order)
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Rank =
+    copy(children = newChildren)
 }
 
 /**
@@ -719,6 +934,7 @@ case class Rank(children: Seq[Expression]) extends RankLike {
  *
  * This documentation has been based upon similar documentation for the Hive and Presto projects.
  */
+// scalastyle:off line.size.limit line.contains.tab
 @ExpressionDescription(
   usage = """
     _FUNC_() - Computes the rank of a value in a group of values. The result is one plus the
@@ -731,8 +947,17 @@ case class Rank(children: Seq[Expression]) extends RankLike {
           trigger a change in rank. This is an internal parameter and will be assigned by the
           Analyser.
   """,
+  examples = """
+    Examples:
+      > SELECT a, b, _FUNC_(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);
+       A1	1	1
+       A1	1	1
+       A1	2	2
+       A2	3	1
+  """,
   since = "2.0.0",
   group = "window_funcs")
+// scalastyle:on line.size.limit line.contains.tab
 case class DenseRank(children: Seq[Expression]) extends RankLike {
   def this() = this(Nil)
   override def withOrder(order: Seq[Expression]): DenseRank = DenseRank(order)
@@ -741,6 +966,8 @@ case class DenseRank(children: Seq[Expression]) extends RankLike {
   override val aggBufferAttributes = rank +: orderAttrs
   override val initialValues = zero +: orderInit
   override def prettyName: String = "dense_rank"
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): DenseRank =
+    copy(children = newChildren)
 }
 
 /**
@@ -753,6 +980,7 @@ case class DenseRank(children: Seq[Expression]) extends RankLike {
  *
  * This documentation has been based upon similar documentation for the Hive and Presto projects.
  */
+// scalastyle:off line.size.limit line.contains.tab
 @ExpressionDescription(
   usage = """
     _FUNC_() - Computes the percentage ranking of a value in a group of values.
@@ -763,8 +991,17 @@ case class DenseRank(children: Seq[Expression]) extends RankLike {
           trigger a change in rank. This is an internal parameter and will be assigned by the
           Analyser.
   """,
+  examples = """
+    Examples:
+      > SELECT a, b, _FUNC_(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);
+       A1	1	0.0
+       A1	1	0.0
+       A1	2	1.0
+       A2	3	0.0
+  """,
   since = "2.0.0",
   group = "window_funcs")
+// scalastyle:on line.size.limit line.contains.tab
 case class PercentRank(children: Seq[Expression]) extends RankLike with SizeBasedWindowFunction {
   def this() = this(Nil)
   override def withOrder(order: Seq[Expression]): PercentRank = PercentRank(order)
@@ -772,4 +1009,6 @@ case class PercentRank(children: Seq[Expression]) extends RankLike with SizeBase
   override val evaluateExpression =
     If(n > one, (rank - one).cast(DoubleType) / (n - one).cast(DoubleType), 0.0d)
   override def prettyName: String = "percent_rank"
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): PercentRank =
+    copy(children = newChildren)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala
index 55e06cb9e8471..336dc7a480cff 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala
@@ -30,7 +30,8 @@ import org.apache.spark.unsafe.types.UTF8String
  *
  * This is not the world's most efficient implementation due to type conversion, but works.
  */
-abstract class XPathExtract extends BinaryExpression with ExpectsInputTypes with CodegenFallback {
+abstract class XPathExtract
+  extends BinaryExpression with ExpectsInputTypes with CodegenFallback with NullIntolerant {
   override def left: Expression = xml
   override def right: Expression = path
 
@@ -62,7 +63,9 @@ abstract class XPathExtract extends BinaryExpression with ExpectsInputTypes with
     Examples:
       > SELECT _FUNC_('<a><b>1</b></a>','a/b');
        true
-  """)
+  """,
+  since = "2.0.0",
+  group = "xml_funcs")
 // scalastyle:on line.size.limit
 case class XPathBoolean(xml: Expression, path: Expression) extends XPathExtract {
 
@@ -72,6 +75,9 @@ case class XPathBoolean(xml: Expression, path: Expression) extends XPathExtract
   override def nullSafeEval(xml: Any, path: Any): Any = {
     xpathUtil.evalBoolean(xml.asInstanceOf[UTF8String].toString, pathString)
   }
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): XPathBoolean = copy(xml = newLeft, path = newRight)
 }
 
 // scalastyle:off line.size.limit
@@ -81,7 +87,9 @@ case class XPathBoolean(xml: Expression, path: Expression) extends XPathExtract
     Examples:
       > SELECT _FUNC_('<a><b>1</b><b>2</b></a>', 'sum(a/b)');
        3
-  """)
+  """,
+  since = "2.0.0",
+  group = "xml_funcs")
 // scalastyle:on line.size.limit
 case class XPathShort(xml: Expression, path: Expression) extends XPathExtract {
   override def prettyName: String = "xpath_short"
@@ -91,6 +99,9 @@ case class XPathShort(xml: Expression, path: Expression) extends XPathExtract {
     val ret = xpathUtil.evalNumber(xml.asInstanceOf[UTF8String].toString, pathString)
     if (ret eq null) null else ret.shortValue()
   }
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): XPathShort = copy(xml = newLeft, path = newRight)
 }
 
 // scalastyle:off line.size.limit
@@ -100,7 +111,9 @@ case class XPathShort(xml: Expression, path: Expression) extends XPathExtract {
     Examples:
       > SELECT _FUNC_('<a><b>1</b><b>2</b></a>', 'sum(a/b)');
        3
-  """)
+  """,
+  since = "2.0.0",
+  group = "xml_funcs")
 // scalastyle:on line.size.limit
 case class XPathInt(xml: Expression, path: Expression) extends XPathExtract {
   override def prettyName: String = "xpath_int"
@@ -110,6 +123,9 @@ case class XPathInt(xml: Expression, path: Expression) extends XPathExtract {
     val ret = xpathUtil.evalNumber(xml.asInstanceOf[UTF8String].toString, pathString)
     if (ret eq null) null else ret.intValue()
   }
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): Expression = copy(xml = newLeft, path = newRight)
 }
 
 // scalastyle:off line.size.limit
@@ -119,7 +135,9 @@ case class XPathInt(xml: Expression, path: Expression) extends XPathExtract {
     Examples:
       > SELECT _FUNC_('<a><b>1</b><b>2</b></a>', 'sum(a/b)');
        3
-  """)
+  """,
+  since = "2.0.0",
+  group = "xml_funcs")
 // scalastyle:on line.size.limit
 case class XPathLong(xml: Expression, path: Expression) extends XPathExtract {
   override def prettyName: String = "xpath_long"
@@ -129,6 +147,9 @@ case class XPathLong(xml: Expression, path: Expression) extends XPathExtract {
     val ret = xpathUtil.evalNumber(xml.asInstanceOf[UTF8String].toString, pathString)
     if (ret eq null) null else ret.longValue()
   }
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): XPathLong = copy(xml = newLeft, path = newRight)
 }
 
 // scalastyle:off line.size.limit
@@ -138,7 +159,9 @@ case class XPathLong(xml: Expression, path: Expression) extends XPathExtract {
     Examples:
       > SELECT _FUNC_('<a><b>1</b><b>2</b></a>', 'sum(a/b)');
        3.0
-  """)
+  """,
+  since = "2.0.0",
+  group = "xml_funcs")
 // scalastyle:on line.size.limit
 case class XPathFloat(xml: Expression, path: Expression) extends XPathExtract {
   override def prettyName: String = "xpath_float"
@@ -148,6 +171,9 @@ case class XPathFloat(xml: Expression, path: Expression) extends XPathExtract {
     val ret = xpathUtil.evalNumber(xml.asInstanceOf[UTF8String].toString, pathString)
     if (ret eq null) null else ret.floatValue()
   }
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): XPathFloat = copy(xml = newLeft, path = newRight)
 }
 
 // scalastyle:off line.size.limit
@@ -157,7 +183,9 @@ case class XPathFloat(xml: Expression, path: Expression) extends XPathExtract {
     Examples:
       > SELECT _FUNC_('<a><b>1</b><b>2</b></a>', 'sum(a/b)');
        3.0
-  """)
+  """,
+  since = "2.0.0",
+  group = "xml_funcs")
 // scalastyle:on line.size.limit
 case class XPathDouble(xml: Expression, path: Expression) extends XPathExtract {
   override def prettyName: String =
@@ -168,6 +196,9 @@ case class XPathDouble(xml: Expression, path: Expression) extends XPathExtract {
     val ret = xpathUtil.evalNumber(xml.asInstanceOf[UTF8String].toString, pathString)
     if (ret eq null) null else ret.doubleValue()
   }
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): XPathDouble = copy(xml = newLeft, path = newRight)
 }
 
 // scalastyle:off line.size.limit
@@ -177,7 +208,9 @@ case class XPathDouble(xml: Expression, path: Expression) extends XPathExtract {
     Examples:
       > SELECT _FUNC_('<a><b>b</b><c>cc</c></a>','a/c');
        cc
-  """)
+  """,
+  since = "2.0.0",
+  group = "xml_funcs")
 // scalastyle:on line.size.limit
 case class XPathString(xml: Expression, path: Expression) extends XPathExtract {
   override def prettyName: String = "xpath_string"
@@ -187,6 +220,9 @@ case class XPathString(xml: Expression, path: Expression) extends XPathExtract {
     val ret = xpathUtil.evalString(xml.asInstanceOf[UTF8String].toString, pathString)
     UTF8String.fromString(ret)
   }
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): Expression = copy(xml = newLeft, path = newRight)
 }
 
 // scalastyle:off line.size.limit
@@ -196,7 +232,9 @@ case class XPathString(xml: Expression, path: Expression) extends XPathExtract {
     Examples:
       > SELECT _FUNC_('<a><b>b1</b><b>b2</b><b>b3</b><c>c1</c><c>c2</c></a>','a/b/text()');
        ["b1","b2","b3"]
-  """)
+  """,
+  since = "2.0.0",
+  group = "xml_funcs")
 // scalastyle:on line.size.limit
 case class XPathList(xml: Expression, path: Expression) extends XPathExtract {
   override def prettyName: String = "xpath"
@@ -216,4 +254,7 @@ case class XPathList(xml: Expression, path: Expression) extends XPathExtract {
       null
     }
   }
+
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): XPathList = copy(xml = newLeft, path = newRight)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala
index f9222f5af54da..47be83a41d613 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala
@@ -133,7 +133,13 @@ private[sql] class JSONOptions(
    * Enables inferring of TimestampType from strings matched to the timestamp pattern
    * defined by the timestampFormat option.
    */
-  val inferTimestamp: Boolean = parameters.get("inferTimestamp").map(_.toBoolean).getOrElse(true)
+  val inferTimestamp: Boolean = parameters.get("inferTimestamp").map(_.toBoolean).getOrElse(false)
+
+  /**
+   * Generating \u0000 style codepoints for non-ASCII characters if the parameter is enabled.
+   */
+  val writeNonAsciiCharacterAsCodePoint: Boolean =
+    parameters.get("writeNonAsciiCharacterAsCodePoint").map(_.toBoolean).getOrElse(false)
 
   /** Build a Jackson [[JsonFactory]] using JSON options. */
   def buildJsonFactory(): JsonFactory = {
@@ -168,10 +174,10 @@ private[sql] class JSONOptionsInRead(
   }
 
   protected override def checkedEncoding(enc: String): String = {
-    val isBlacklisted = JSONOptionsInRead.blacklist.contains(Charset.forName(enc))
-    require(multiLine || !isBlacklisted,
-      s"""The ${enc} encoding must not be included in the blacklist when multiLine is disabled:
-         |Blacklist: ${JSONOptionsInRead.blacklist.mkString(", ")}""".stripMargin)
+    val isDenied = JSONOptionsInRead.denyList.contains(Charset.forName(enc))
+    require(multiLine || !isDenied,
+      s"""The $enc encoding must not be included in the denyList when multiLine is disabled:
+         |denylist: ${JSONOptionsInRead.denyList.mkString(", ")}""".stripMargin)
 
     val isLineSepRequired =
         multiLine || Charset.forName(enc) == StandardCharsets.UTF_8 || lineSeparator.nonEmpty
@@ -188,7 +194,7 @@ private[sql] object JSONOptionsInRead {
   // only the first lines will have the BOM which leads to impossibility for reading
   // the rest lines. Besides of that, the lineSep option must have the BOM in such
   // encodings which can never present between lines.
-  val blacklist = Seq(
+  val denyList = Seq(
     Charset.forName("UTF-16"),
     Charset.forName("UTF-32")
   )
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala
index f3938feef0a35..b6014762583b1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala
@@ -20,11 +20,13 @@ package org.apache.spark.sql.catalyst.json
 import java.io.Writer
 
 import com.fasterxml.jackson.core._
+import com.fasterxml.jackson.core.util.DefaultPrettyPrinter
 
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.SpecializedGetters
 import org.apache.spark.sql.catalyst.util._
 import org.apache.spark.sql.catalyst.util.LegacyDateFormats.FAST_DATE_FORMAT
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.types._
 
 /**
@@ -73,7 +75,13 @@ private[sql] class JacksonGenerator(
 
   private val gen = {
     val generator = new JsonFactory().createGenerator(writer).setRootValueSeparator(null)
-    if (options.pretty) generator.useDefaultPrettyPrinter() else generator
+    if (options.pretty) {
+      generator.setPrettyPrinter(new DefaultPrettyPrinter(""))
+    }
+    if (options.writeNonAsciiCharacterAsCodePoint) {
+      generator.setHighestNonEscapedChar(0x7F)
+    }
+    generator
   }
 
   private val lineSeparator: String = options.lineSeparatorInWrite
@@ -83,12 +91,13 @@ private[sql] class JacksonGenerator(
     options.zoneId,
     options.locale,
     legacyFormat = FAST_DATE_FORMAT,
-    needVarLengthSecondFraction = false)
+    isParsing = false)
   private val dateFormatter = DateFormatter(
     options.dateFormat,
     options.zoneId,
     options.locale,
-    legacyFormat = FAST_DATE_FORMAT)
+    legacyFormat = FAST_DATE_FORMAT,
+    isParsing = false)
 
   private def makeWriter(dataType: DataType): ValueWriter = dataType match {
     case NullType =>
@@ -174,8 +183,7 @@ private[sql] class JacksonGenerator(
     case _ =>
       (row: SpecializedGetters, ordinal: Int) =>
         val v = row.get(ordinal, dataType)
-        sys.error(s"Failed to convert value $v (class of ${v.getClass}}) " +
-          s"with the type of $dataType to JSON.")
+        throw QueryExecutionErrors.failToConvertValueToJsonError(v, v.getClass, dataType)
   }
 
   private def writeObject(f: => Unit): Unit = {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
index ef987931e928a..bbcff4949ae87 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
@@ -27,11 +27,12 @@ import com.fasterxml.jackson.core._
 
 import org.apache.spark.SparkUpgradeException
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.{InternalRow, NoopFilters, StructFilters}
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.util._
 import org.apache.spark.sql.catalyst.util.LegacyDateFormats.FAST_DATE_FORMAT
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 import org.apache.spark.util.Utils
@@ -42,7 +43,8 @@ import org.apache.spark.util.Utils
 class JacksonParser(
     schema: DataType,
     val options: JSONOptions,
-    allowArrayAsStructs: Boolean) extends Logging {
+    allowArrayAsStructs: Boolean,
+    filters: Seq[Filter] = Seq.empty) extends Logging {
 
   import JacksonUtils._
   import com.fasterxml.jackson.core.JsonToken._
@@ -56,17 +58,18 @@ class JacksonParser(
 
   private val factory = options.buildJsonFactory()
 
-  private val timestampFormatter = TimestampFormatter(
+  private lazy val timestampFormatter = TimestampFormatter(
     options.timestampFormat,
     options.zoneId,
     options.locale,
     legacyFormat = FAST_DATE_FORMAT,
-    needVarLengthSecondFraction = true)
-  private val dateFormatter = DateFormatter(
+    isParsing = true)
+  private lazy val dateFormatter = DateFormatter(
     options.dateFormat,
     options.zoneId,
     options.locale,
-    legacyFormat = FAST_DATE_FORMAT)
+    legacyFormat = FAST_DATE_FORMAT,
+    isParsing = true)
 
   /**
    * Create a converter which converts the JSON documents held by the `JsonParser`
@@ -84,8 +87,13 @@ class JacksonParser(
   private def makeStructRootConverter(st: StructType): JsonParser => Iterable[InternalRow] = {
     val elementConverter = makeConverter(st)
     val fieldConverters = st.map(_.dataType).map(makeConverter).toArray
+    val jsonFilters = if (SQLConf.get.jsonFilterPushDown) {
+      new JsonFilters(filters, st)
+    } else {
+      new NoopFilters
+    }
     (parser: JsonParser) => parseJsonToken[Iterable[InternalRow]](parser, st) {
-      case START_OBJECT => Some(convertObject(parser, st, fieldConverters))
+      case START_OBJECT => convertObject(parser, st, fieldConverters, jsonFilters, isRoot = true)
         // SPARK-3308: support reading top level JSON arrays and take every element
         // in such an array as a row
         //
@@ -145,7 +153,7 @@ class JacksonParser(
         //
         val st = at.elementType.asInstanceOf[StructType]
         val fieldConverters = st.map(_.dataType).map(makeConverter).toArray
-        Some(InternalRow(new GenericArrayData(Seq(convertObject(parser, st, fieldConverters)))))
+        Some(InternalRow(new GenericArrayData(convertObject(parser, st, fieldConverters).toArray)))
     }
   }
 
@@ -237,7 +245,7 @@ class JacksonParser(
             case NonFatal(e) =>
               // If fails to parse, then tries the way used in 2.0 and 1.x for backwards
               // compatibility.
-              val str = UTF8String.fromString(DateTimeUtils.cleanLegacyTimestampStr(parser.getText))
+              val str = DateTimeUtils.cleanLegacyTimestampStr(UTF8String.fromString(parser.getText))
               DateTimeUtils.stringToTimestamp(str, options.zoneId).getOrElse(throw e)
           }
 
@@ -254,7 +262,7 @@ class JacksonParser(
             case NonFatal(e) =>
               // If fails to parse, then tries the way used in 2.0 and 1.x for backwards
               // compatibility.
-              val str = UTF8String.fromString(DateTimeUtils.cleanLegacyTimestampStr(parser.getText))
+              val str = DateTimeUtils.cleanLegacyTimestampStr(UTF8String.fromString(parser.getText))
               DateTimeUtils.stringToDate(str, options.zoneId).getOrElse {
                 // In Spark 1.5.0, we store the data as number of days since epoch in string.
                 // So, we just convert it to Int.
@@ -290,7 +298,7 @@ class JacksonParser(
     case st: StructType =>
       val fieldConverters = st.map(_.dataType).map(makeConverter).toArray
       (parser: JsonParser) => parseJsonToken[InternalRow](parser, dataType) {
-        case START_OBJECT => convertObject(parser, st, fieldConverters)
+        case START_OBJECT => convertObject(parser, st, fieldConverters).get
       }
 
     case at: ArrayType =>
@@ -374,18 +382,23 @@ class JacksonParser(
   private def convertObject(
       parser: JsonParser,
       schema: StructType,
-      fieldConverters: Array[ValueConverter]): InternalRow = {
+      fieldConverters: Array[ValueConverter],
+      structFilters: StructFilters = new NoopFilters(),
+      isRoot: Boolean = false): Option[InternalRow] = {
     val row = new GenericInternalRow(schema.length)
     var badRecordException: Option[Throwable] = None
+    var skipRow = false
 
-    while (nextUntil(parser, JsonToken.END_OBJECT)) {
+    structFilters.reset()
+    while (!skipRow && nextUntil(parser, JsonToken.END_OBJECT)) {
       schema.getFieldIndex(parser.getCurrentName) match {
         case Some(index) =>
           try {
             row.update(index, fieldConverters(index).apply(parser))
+            skipRow = structFilters.skipRow(row, index)
           } catch {
             case e: SparkUpgradeException => throw e
-            case NonFatal(e) =>
+            case NonFatal(e) if isRoot =>
               badRecordException = badRecordException.orElse(Some(e))
               parser.skipChildren()
           }
@@ -394,8 +407,10 @@ class JacksonParser(
       }
     }
 
-    if (badRecordException.isEmpty) {
-      row
+    if (skipRow) {
+      None
+    } else if (badRecordException.isEmpty) {
+      Some(row)
     } else {
       throw PartialResultException(row, badRecordException.get)
     }
@@ -451,7 +466,7 @@ class JacksonParser(
           case null => None
           case _ => rootConverter.apply(parser) match {
             case null => throw new RuntimeException("Root converter returned null")
-            case rows => rows
+            case rows => rows.toSeq
           }
         }
       }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonFilters.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonFilters.scala
new file mode 100644
index 0000000000000..0d5974af19ac3
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonFilters.scala
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.json
+
+import org.apache.spark.sql.catalyst.{InternalRow, StructFilters}
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.sources
+import org.apache.spark.sql.types.StructType
+
+/**
+ * The class provides API for applying pushed down source filters to rows with
+ * a struct schema parsed from JSON records. The class should be used in this way:
+ * 1. Before processing of the next row, `JacksonParser` (parser for short) resets the internal
+ *    state of `JsonFilters` by calling the `reset()` method.
+ * 2. The parser reads JSON fields one-by-one in streaming fashion. It converts an incoming
+ *    field value to the desired type from the schema. After that, it sets the value to an instance
+ *    of `InternalRow` at the position according to the schema. Order of parsed JSON fields can
+ *    be different from the order in the schema.
+ * 3. Per every JSON field of the top-level JSON object, the parser calls `skipRow` by passing
+ *    an `InternalRow` in which some of fields can be already set, and the position of the JSON
+ *    field according to the schema.
+ *    3.1 `skipRow` finds a group of predicates that refers to this JSON field.
+ *    3.2 Per each predicate from the group, `skipRow` decrements its reference counter.
+ *    3.2.1 If predicate reference counter becomes 0, it means that all predicate attributes have
+ *          been already set in the internal row, and the predicate can be applied to it. `skipRow`
+ *          invokes the predicate for the row.
+ *    3.3 `skipRow` applies predicates until one of them returns `false`. In that case, the method
+ *        returns `true` to the parser.
+ *    3.4 If all predicates with zero reference counter return `true`, the final result of
+ *        the method is `false` which tells the parser to not skip the row.
+ * 4. If the parser gets `true` from `JsonFilters.skipRow`, it must not call the method anymore
+ *    for this internal row, and should go the step 1.
+ *
+ * Besides of `StructFilters` assumptions, `JsonFilters` assumes that:
+ *   - `skipRow()` can be called for any valid index of the struct fields,
+ *      and in any order.
+ *   - After `skipRow()` returns `true`, the internal state of `JsonFilters` can be inconsistent,
+ *     so, `skipRow()` must not be called for the current row anymore without `reset()`.
+ *
+ * @param pushedFilters The pushed down source filters. The filters should refer to
+ *                      the fields of the provided schema.
+ * @param schema The required schema of records from datasource files.
+ */
+class JsonFilters(pushedFilters: Seq[sources.Filter], schema: StructType)
+  extends StructFilters(pushedFilters, schema) {
+
+  /**
+   * Stateful JSON predicate that keeps track of its dependent references in the
+   * current row via `refCount`.
+   *
+   * @param predicate The predicate compiled from pushed down source filters.
+   * @param totalRefs The total amount of all filters references which the predicate
+   *                  compiled from.
+   */
+  case class JsonPredicate(predicate: BasePredicate, totalRefs: Int) {
+    // The current number of predicate references in the row that have been not set yet.
+    // When `refCount` reaches zero, the predicate has all dependencies are set, and can
+    // be applied to the row.
+    var refCount: Int = totalRefs
+
+    def reset(): Unit = {
+      refCount = totalRefs
+    }
+  }
+
+  // Predicates compiled from the pushed down filters. The predicates are grouped by their
+  // attributes. The i-th group contains predicates that refer to the i-th field of the given
+  // schema. A predicates can be placed to many groups if it has many attributes. For example:
+  //   schema: i INTEGER, s STRING
+  //   filters: IsNotNull("i"), AlwaysTrue, Or(EqualTo("i", 0), StringStartsWith("s", "abc"))
+  //   predicates:
+  //     0: Array(IsNotNull("i"), AlwaysTrue, Or(EqualTo("i", 0), StringStartsWith("s", "abc")))
+  //     1: Array(AlwaysTrue, Or(EqualTo("i", 0), StringStartsWith("s", "abc")))
+  private val predicates: Array[Array[JsonPredicate]] = {
+    val groupedPredicates = Array.fill(schema.length)(Array.empty[JsonPredicate])
+    val groupedByRefSet: Map[Set[String], JsonPredicate] = filters
+      // Group filters that have the same set of references. For example:
+      //   IsNotNull("i") -> Set("i"), AlwaysTrue -> Set(),
+      //   Or(EqualTo("i", 0), StringStartsWith("s", "abc")) -> Set("i", "s")
+      // By grouping filters we could avoid tracking their state of references in the
+      // current row separately.
+      .groupBy(_.references.toSet)
+      // Combine all filters from the same group by `And` because all filters should
+      // return `true` to do not skip a row. The result is compiled to a predicate.
+      .map { case (refSet, refsFilters) =>
+        (refSet, JsonPredicate(toPredicate(refsFilters), refSet.size))
+      }
+    // Apply predicates w/o references like `AlwaysTrue` and `AlwaysFalse` to all fields.
+    // We cannot set such predicates to a particular position because skipRow() can
+    // be invoked for any index due to unpredictable order of JSON fields in JSON records.
+    val withLiterals: Map[Set[String], JsonPredicate] = groupedByRefSet.map {
+      case (refSet, pred) if refSet.isEmpty =>
+        (schema.fields.map(_.name).toSet, pred.copy(totalRefs = 1))
+      case others => others
+    }
+    // Build a map where key is only one field and value is seq of predicates refer to the field
+    //   "i" -> Seq(AlwaysTrue, IsNotNull("i"), Or(EqualTo("i", 0), StringStartsWith("s", "abc")))
+    //   "s" -> Seq(AlwaysTrue, Or(EqualTo("i", 0), StringStartsWith("s", "abc")))
+    val groupedByFields: Map[String, Seq[(String, JsonPredicate)]] = withLiterals.toSeq
+      .flatMap { case (refSet, pred) => refSet.map((_, pred)) }
+      .groupBy(_._1)
+    // Build the final array by converting keys of `groupedByFields` to their
+    // indexes in the provided schema.
+    groupedByFields.foreach { case (fieldName, fieldPredicates) =>
+      val fieldIndex = schema.fieldIndex(fieldName)
+      groupedPredicates(fieldIndex) = fieldPredicates.map(_._2).toArray
+    }
+    groupedPredicates
+  }
+
+  /**
+   * Applies predicates (compiled filters) associated with the row field value
+   * at the position `index` only if other predicates dependencies are already
+   * set in the given row.
+   *
+   * Note: If the function returns `true`, `refCount` of some predicates can be not decremented.
+   *
+   * @param row The row with fully or partially set values.
+   * @param index The index of already set value.
+   * @return `true` if at least one of applicable predicates (all dependent row values are set)
+   *         return `false`. It returns `false` if all predicates return `true`.
+   */
+  def skipRow(row: InternalRow, index: Int): Boolean = {
+    assert(0 <= index && index < schema.fields.length,
+      s"The index $index is out of the valid range [0, ${schema.fields.length}). " +
+      s"It must point out to a field of the schema: ${schema.catalogString}.")
+    var skip = false
+    for (pred <- predicates(index) if !skip) {
+      pred.refCount -= 1
+      assert(pred.refCount >= 0,
+        s"Predicate reference counter cannot be negative but got ${pred.refCount}.")
+      skip = pred.refCount == 0 && !pred.predicate.eval(row)
+    }
+    skip
+  }
+
+  /**
+   * Reset states of all predicates by re-initializing reference counters.
+   */
+  override def reset(): Unit = predicates.foreach(_.foreach(_.reset))
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
index 56b12784fd214..a39f06628b9ec 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
@@ -43,7 +43,7 @@ private[sql] class JsonInferSchema(options: JSONOptions) extends Serializable {
     options.zoneId,
     options.locale,
     legacyFormat = FAST_DATE_FORMAT,
-    needVarLengthSecondFraction = true)
+    isParsing = true)
 
   /**
    * Infer the type of a collection of json records in three stages:
@@ -190,6 +190,9 @@ private[sql] class JsonInferSchema(options: JSONOptions) extends Serializable {
         }
 
       case VALUE_TRUE | VALUE_FALSE => BooleanType
+
+      case _ =>
+        throw new SparkException("Malformed JSON")
     }
   }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala
index f79dabf758c14..8f1548a9788af 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.catalyst.optimizer
 
 import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LogicalPlan}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.rules.Rule
 
 /**
@@ -26,20 +26,23 @@ import org.apache.spark.sql.catalyst.rules.Rule
  */
 object SimplifyExtractValueOps extends Rule[LogicalPlan] {
   override def apply(plan: LogicalPlan): LogicalPlan = plan transform {
-    // One place where this optimization is invalid is an aggregation where the select
-    // list expression is a function of a grouping expression:
-    //
-    // SELECT struct(a,b).a FROM tbl GROUP BY struct(a,b)
-    //
-    // cannot be simplified to SELECT a FROM tbl GROUP BY struct(a,b). So just skip this
-    // optimization for Aggregates (although this misses some cases where the optimization
-    // can be made).
-    case a: Aggregate => a
     case p => p.transformExpressionsUp {
       // Remove redundant field extraction.
       case GetStructField(createNamedStruct: CreateNamedStruct, ordinal, _) =>
         createNamedStruct.valExprs(ordinal)
-
+      case GetStructField(u: UpdateFields, ordinal, _)if !u.structExpr.isInstanceOf[UpdateFields] =>
+        val structExpr = u.structExpr
+        u.newExprs(ordinal) match {
+          // if the struct itself is null, then any value extracted from it (expr) will be null
+          // so we don't need to wrap expr in If(IsNull(struct), Literal(null, expr.dataType), expr)
+          case expr: GetStructField if expr.child.semanticEquals(structExpr) => expr
+          case expr =>
+            if (structExpr.nullable) {
+              If(IsNull(structExpr), Literal(null, expr.dataType), expr)
+            } else {
+              expr
+            }
+        }
       // Remove redundant array indexing.
       case GetArrayStructFields(CreateArray(elems, useStringTypeWhenEmpty), field, ordinal, _, _) =>
         // Instead of selecting the field on the entire array, select it from each member
@@ -48,7 +51,7 @@ object SimplifyExtractValueOps extends Rule[LogicalPlan] {
         CreateArray(elems.map(GetStructField(_, ordinal, Some(field.name))), useStringTypeWhenEmpty)
 
       // Remove redundant map lookup.
-      case ga @ GetArrayItem(CreateArray(elems, _), IntegerLiteral(idx)) =>
+      case ga @ GetArrayItem(CreateArray(elems, _), IntegerLiteral(idx), _) =>
         // Instead of creating the array and then selecting one row, remove array creation
         // altogether.
         if (idx >= 0 && idx < elems.size) {
@@ -58,7 +61,7 @@ object SimplifyExtractValueOps extends Rule[LogicalPlan] {
           // out of bounds, mimic the runtime behavior and return null
           Literal(null, ga.dataType)
         }
-      case GetMapValue(CreateMap(elems, _), key) => CaseKeyWhen(key, elems)
+      case GetMapValue(CreateMap(elems, _), key, _) => CaseKeyWhen(key, elems)
     }
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/CostBasedJoinReorder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/CostBasedJoinReorder.scala
index f92d8f5b8e534..659384a507746 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/CostBasedJoinReorder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/CostBasedJoinReorder.scala
@@ -20,10 +20,11 @@ package org.apache.spark.sql.catalyst.optimizer
 import scala.collection.mutable
 
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.catalyst.expressions.{And, Attribute, AttributeSet, Expression, PredicateHelper}
+import org.apache.spark.sql.catalyst.expressions.{And, Attribute, AttributeSet, Expression, ExpressionSet, PredicateHelper}
 import org.apache.spark.sql.catalyst.plans.{Inner, InnerLike, JoinType}
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreePattern.INNER_LIKE_JOIN
 import org.apache.spark.sql.internal.SQLConf
 
 
@@ -34,13 +35,11 @@ import org.apache.spark.sql.internal.SQLConf
  */
 object CostBasedJoinReorder extends Rule[LogicalPlan] with PredicateHelper {
 
-  private def conf = SQLConf.get
-
   def apply(plan: LogicalPlan): LogicalPlan = {
     if (!conf.cboEnabled || !conf.joinReorderEnabled) {
       plan
     } else {
-      val result = plan transformDown {
+      val result = plan.transformDownWithPruning(_.containsPattern(INNER_LIKE_JOIN), ruleId) {
         // Start reordering with a joinable item, which is an InnerLike join with conditions.
         // Avoid reordering if a join hint is present.
         case j @ Join(_, _, _: InnerLike, Some(cond), JoinHint.NONE) =>
@@ -75,18 +74,18 @@ object CostBasedJoinReorder extends Rule[LogicalPlan] with PredicateHelper {
    * Extracts items of consecutive inner joins and join conditions.
    * This method works for bushy trees and left/right deep trees.
    */
-  private def extractInnerJoins(plan: LogicalPlan): (Seq[LogicalPlan], Set[Expression]) = {
+  private def extractInnerJoins(plan: LogicalPlan): (Seq[LogicalPlan], ExpressionSet) = {
     plan match {
       case Join(left, right, _: InnerLike, Some(cond), JoinHint.NONE) =>
         val (leftPlans, leftConditions) = extractInnerJoins(left)
         val (rightPlans, rightConditions) = extractInnerJoins(right)
-        (leftPlans ++ rightPlans, splitConjunctivePredicates(cond).toSet ++
-          leftConditions ++ rightConditions)
+        (leftPlans ++ rightPlans, leftConditions ++ rightConditions ++
+          splitConjunctivePredicates(cond))
       case Project(projectList, j @ Join(_, _, _: InnerLike, Some(cond), JoinHint.NONE))
         if projectList.forall(_.isInstanceOf[Attribute]) =>
         extractInnerJoins(j)
       case _ =>
-        (Seq(plan), Set())
+        (Seq(plan), ExpressionSet())
     }
   }
 
@@ -109,12 +108,15 @@ case class OrderedJoin(
     joinType: JoinType,
     condition: Option[Expression]) extends BinaryNode {
   override def output: Seq[Attribute] = left.output ++ right.output
+  override protected def withNewChildrenInternal(
+      newLeft: LogicalPlan, newRight: LogicalPlan): OrderedJoin =
+    copy(left = newLeft, right = newRight)
 }
 
 /**
  * Reorder the joins using a dynamic programming algorithm. This implementation is based on the
  * paper: Access Path Selection in a Relational Database Management System.
- * http://www.inf.ed.ac.uk/teaching/courses/adbs/AccessPath.pdf
+ * https://dl.acm.org/doi/10.1145/582095.582099
  *
  * First we put all items (basic joined nodes) into level 0, then we build all two-way joins
  * at level 1 from plans at level 0 (single items), then build all 3-way joins from plans
@@ -143,16 +145,23 @@ object JoinReorderDP extends PredicateHelper with Logging {
   def search(
       conf: SQLConf,
       items: Seq[LogicalPlan],
-      conditions: Set[Expression],
+      conditions: ExpressionSet,
       output: Seq[Attribute]): LogicalPlan = {
 
     val startTime = System.nanoTime()
     // Level i maintains all found plans for i + 1 items.
     // Create the initial plans: each plan is a single item with zero cost.
     val itemIndex = items.zipWithIndex
-    val foundPlans = mutable.Buffer[JoinPlanMap](itemIndex.map {
-      case (item, id) => Set(id) -> JoinPlan(Set(id), item, Set.empty, Cost(0, 0))
-    }.toMap)
+    val foundPlans = mutable.Buffer[JoinPlanMap]({
+      // SPARK-32687: Change to use `LinkedHashMap` to make sure that items are
+      // inserted and iterated in the same order.
+      val joinPlanMap = new JoinPlanMap
+      itemIndex.foreach {
+        case (item, id) =>
+          joinPlanMap.put(Set(id), JoinPlan(Set(id), item, ExpressionSet(), Cost(0, 0)))
+      }
+      joinPlanMap
+    })
 
     // Build filters from the join graph to be used by the search algorithm.
     val filters = JoinReorderDPFilters.buildJoinGraphInfo(conf, items, conditions, itemIndex)
@@ -162,7 +171,7 @@ object JoinReorderDP extends PredicateHelper with Logging {
     val topOutputSet = AttributeSet(output)
     while (foundPlans.size < items.length) {
       // Build plans for the next level.
-      foundPlans += searchLevel(foundPlans, conf, conditions, topOutputSet, filters)
+      foundPlans += searchLevel(foundPlans.toSeq, conf, conditions, topOutputSet, filters)
     }
 
     val durationInMs = (System.nanoTime() - startTime) / (1000 * 1000)
@@ -194,11 +203,11 @@ object JoinReorderDP extends PredicateHelper with Logging {
   private def searchLevel(
       existingLevels: Seq[JoinPlanMap],
       conf: SQLConf,
-      conditions: Set[Expression],
+      conditions: ExpressionSet,
       topOutput: AttributeSet,
       filters: Option[JoinGraphInfo]): JoinPlanMap = {
 
-    val nextLevel = mutable.Map.empty[Set[Int], JoinPlan]
+    val nextLevel = new JoinPlanMap
     var k = 0
     val lev = existingLevels.length - 1
     // Build plans for the next level from plans at level k (one side of the join) and level
@@ -231,7 +240,7 @@ object JoinReorderDP extends PredicateHelper with Logging {
       }
       k += 1
     }
-    nextLevel.toMap
+    nextLevel
   }
 
   /**
@@ -255,7 +264,7 @@ object JoinReorderDP extends PredicateHelper with Logging {
       oneJoinPlan: JoinPlan,
       otherJoinPlan: JoinPlan,
       conf: SQLConf,
-      conditions: Set[Expression],
+      conditions: ExpressionSet,
       topOutput: AttributeSet,
       filters: Option[JoinGraphInfo]): Option[JoinPlan] = {
 
@@ -316,7 +325,7 @@ object JoinReorderDP extends PredicateHelper with Logging {
   }
 
   /** Map[set of item ids, join plan for these items] */
-  type JoinPlanMap = Map[Set[Int], JoinPlan]
+  type JoinPlanMap = mutable.LinkedHashMap[Set[Int], JoinPlan]
 
   /**
    * Partial join order in a specific level.
@@ -329,7 +338,7 @@ object JoinReorderDP extends PredicateHelper with Logging {
   case class JoinPlan(
       itemIds: Set[Int],
       plan: LogicalPlan,
-      joinConds: Set[Expression],
+      joinConds: ExpressionSet,
       planCost: Cost) {
 
     /** Get the cost of the root node of this plan tree. */
@@ -343,14 +352,29 @@ object JoinReorderDP extends PredicateHelper with Logging {
       }
     }
 
+    /**
+     * To identify the plan with smaller computational cost,
+     * we use the weighted geometric mean of ratio of rows and the ratio of sizes in bytes.
+     *
+     * There are other ways to combine these values as a cost comparison function.
+     * Some of these, that we have experimented with, but have gotten worse result,
+     * than with the current one:
+     * 1) Weighted arithmetic mean of these two ratios - adding up fractions puts
+     * less emphasis on ratios between 0 and 1. Ratios 10 and 0.1 should be considered
+     * to be just as strong evidences in opposite directions. The arithmetic mean of these
+     * would be heavily biased towards the 10.
+     * 2) Absolute cost (cost = weight * rowCount + (1 - weight) * size) - when adding up
+     * two numeric measurements that have different units we can easily end up with one
+     * overwhelming the other.
+     */
     def betterThan(other: JoinPlan, conf: SQLConf): Boolean = {
       if (other.planCost.card == 0 || other.planCost.size == 0) {
         false
       } else {
         val relativeRows = BigDecimal(this.planCost.card) / BigDecimal(other.planCost.card)
         val relativeSize = BigDecimal(this.planCost.size) / BigDecimal(other.planCost.size)
-        relativeRows * conf.joinReorderCardWeight +
-          relativeSize * (1 - conf.joinReorderCardWeight) < 1
+        Math.pow(relativeRows.doubleValue, conf.joinReorderCardWeight) *
+          Math.pow(relativeSize.doubleValue, 1 - conf.joinReorderCardWeight) < 1
       }
     }
   }
@@ -387,7 +411,7 @@ object JoinReorderDPFilters extends PredicateHelper {
   def buildJoinGraphInfo(
       conf: SQLConf,
       items: Seq[LogicalPlan],
-      conditions: Set[Expression],
+      conditions: ExpressionSet,
       itemIndex: Seq[(LogicalPlan, Int)]): Option[JoinGraphInfo] = {
 
     if (conf.joinReorderDPStarFilter) {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/DecorrelateInnerQuery.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/DecorrelateInnerQuery.scala
new file mode 100644
index 0000000000000..7ba7b350c9c96
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/DecorrelateInnerQuery.scala
@@ -0,0 +1,485 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.expressions.SubExprUtils._
+import org.apache.spark.sql.catalyst.plans._
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.trees.TreePattern.OUTER_REFERENCE
+
+/**
+ * Decorrelate the inner query by eliminating outer references and create domain joins.
+ * The implementation is based on the paper: Unnesting Arbitrary Queries by Thomas Neumann
+ * and Alfons Kemper. https://dl.gi.de/handle/20.500.12116/2418.
+ *
+ * A correlated subquery can be viewed as a "dependent" nested loop join between the outer and
+ * the inner query. For each row produced by the outer query, we bind the [[OuterReference]]s in
+ * in the inner query with the corresponding values in the row, and then evaluate the inner query.
+ *
+ * Dependent Join
+ * :- Outer Query
+ * +- Inner Query
+ *
+ * If the [[OuterReference]]s are bound to the same value, the inner query will return the same
+ * result. Based on this, we can reduce the times to evaluate the inner query by first getting
+ * all distinct values of the [[OuterReference]]s.
+ *
+ * Normal Join
+ * :- Outer Query
+ * +- Dependent Join
+ *    :- Inner Query
+ *    +- Distinct Aggregate (outer_ref1, outer_ref2, ...)
+ *       +- Outer Query
+ *
+ * The distinct aggregate of the outer references is called a "domain", and the dependent join
+ * between the inner query and the domain is called a "domain join". We need to push down the
+ * domain join through the inner query until there is no outer reference in the sub-tree and
+ * the domain join will turn into a normal join.
+ *
+ * The decorrelation function returns a new query plan with optional placeholder [[DomainJoins]]s
+ * added and a list of join conditions with the outer query. [[DomainJoin]]s need to be rewritten
+ * into actual inner join between the inner query sub-tree and the outer query.
+ *
+ * E.g. decorrelate an inner query with equality predicates:
+ *
+ * SELECT (SELECT MIN(b) FROM t1 WHERE t2.c = t1.a) FROM t2
+ *
+ * Aggregate [] [min(b)]            Aggregate [a] [min(b), a]
+ * +- Filter (outer(c) = a)   =>   +- Relation [t1]
+ *    +- Relation [t1]
+ *
+ * Join conditions: [c = a]
+ *
+ * E.g. decorrelate an inner query with non-equality predicates:
+ *
+ * SELECT (SELECT MIN(b) FROM t1 WHERE t2.c > t1.a) FROM t2
+ *
+ * Aggregate [] [min(b)]            Aggregate [c'] [min(b), c']
+ * +- Filter (outer(c) > a)   =>   +- Filter (c' > a)
+ *    +- Relation [t1]                  +- DomainJoin [c']
+ *                                         +- Relation [t1]
+ *
+ * Join conditions: [c <=> c']
+ */
+object DecorrelateInnerQuery extends PredicateHelper {
+
+  /**
+   * Check if an expression contains any attribute. Note OuterReference is a
+   * leaf node and will not be found here.
+   */
+  private def containsAttribute(expression: Expression): Boolean = {
+    expression.find(_.isInstanceOf[Attribute]).isDefined
+  }
+
+  /**
+   * Check if an expression can be pulled up over an [[Aggregate]] without changing the
+   * semantics of the plan. The expression must be an equality predicate that guarantees
+   * one-to-one mapping between inner and outer attributes. More specifically, one side
+   * of the predicate must be an attribute and another side of the predicate must not
+   * contain other attributes from the inner query.
+   * For example:
+   *   (a = outer(c)) -> true
+   *   (a > outer(c)) -> false
+   *   (a + b = outer(c)) -> false
+   *   (a = outer(c) - b) -> false
+   */
+  private def canPullUpOverAgg(expression: Expression): Boolean = expression match {
+    case Equality(_: Attribute, b) => !containsAttribute(b)
+    case Equality(a, _: Attribute) => !containsAttribute(a)
+    case o => !containsAttribute(o)
+  }
+
+  /**
+   * Collect outer references in an expressions that are in the output attributes of the outer plan.
+   */
+  private def collectOuterReferences(expression: Expression): AttributeSet = {
+    AttributeSet(expression.collect { case o: OuterReference => o.toAttribute })
+  }
+
+  /**
+   * Collect outer references in a sequence of expressions that are in the output attributes
+   * of the outer plan.
+   */
+  private def collectOuterReferences(expressions: Seq[Expression]): AttributeSet = {
+    AttributeSet.fromAttributeSets(expressions.map(collectOuterReferences))
+  }
+
+  /**
+   * Build a mapping between outer references with equivalent inner query attributes.
+   * E.g. [outer(a) = x, y = outer(b), outer(c) = z + 1] => {a -> x, b -> y}
+   */
+  private def collectEquivalentOuterReferences(
+      expressions: Seq[Expression]): Map[Attribute, Attribute] = {
+    expressions.collect {
+      case Equality(o: OuterReference, a: Attribute) => (o.toAttribute, a.toAttribute)
+      case Equality(a: Attribute, o: OuterReference) => (o.toAttribute, a.toAttribute)
+    }.toMap
+  }
+
+  /**
+   * Replace all outer references using the expressions in the given outer reference map.
+   */
+  private def replaceOuterReference[E <: Expression](
+      expression: E,
+      outerReferenceMap: Map[Attribute, Attribute]): E = {
+    expression.transformWithPruning(_.containsPattern(OUTER_REFERENCE)) {
+      case o: OuterReference => outerReferenceMap.getOrElse(o.toAttribute, o)
+    }.asInstanceOf[E]
+  }
+
+  /**
+   * Replace all outer references in the given expressions using the expressions in the
+   * outer reference map.
+   */
+  private def replaceOuterReferences[E <: Expression](
+      expressions: Seq[E],
+      outerReferenceMap: Map[Attribute, Attribute]): Seq[E] = {
+    expressions.map(replaceOuterReference(_, outerReferenceMap))
+  }
+
+  /**
+   * Return all references that are presented in the join conditions but not in the output
+   * of the given named expressions.
+   */
+  private def missingReferences(
+      namedExpressions: Seq[NamedExpression],
+      joinCond: Seq[Expression]): AttributeSet = {
+    val output = namedExpressions.map(_.toAttribute)
+    AttributeSet(joinCond.flatMap(_.references)) -- AttributeSet(output)
+  }
+
+  /**
+   * Deduplicate the inner and the outer query attributes and return an aliased
+   * subquery plan and join conditions if duplicates are found. Duplicated attributes
+   * can break the structural integrity when joining the inner and outer plan together.
+   */
+  def deduplicate(
+      innerPlan: LogicalPlan,
+      conditions: Seq[Expression],
+      outerOutputSet: AttributeSet): (LogicalPlan, Seq[Expression]) = {
+    val duplicates = innerPlan.outputSet.intersect(outerOutputSet)
+    if (duplicates.nonEmpty) {
+      val aliasMap = AttributeMap(duplicates.map { dup =>
+        dup -> Alias(dup, dup.toString)()
+      }.toSeq)
+      val aliasedExpressions = innerPlan.output.map { ref =>
+        aliasMap.getOrElse(ref, ref)
+      }
+      val aliasedProjection = Project(aliasedExpressions, innerPlan)
+      val aliasedConditions = conditions.map(_.transform {
+        case ref: Attribute => aliasMap.getOrElse(ref, ref).toAttribute
+      })
+      (aliasedProjection, aliasedConditions)
+    } else {
+      (innerPlan, conditions)
+    }
+  }
+
+  /**
+   * Build a mapping between domain attributes and corresponding outer query expressions
+   * using the join conditions.
+   */
+  private def buildDomainAttrMap(
+      conditions: Seq[Expression],
+      domainAttrs: Seq[Attribute]): Map[Attribute, Expression] = {
+    val domainAttrSet = AttributeSet(domainAttrs)
+    conditions.collect {
+      // When we build the join conditions between the domain attributes and outer references,
+      // the left hand side is always the domain attribute used in the inner query and the right
+      // hand side is the attribute from the outer query. Note here the right hand side of a
+      // condition is not necessarily an attribute, for example it can be a literal (if foldable)
+      // or a cast expression after the optimization.
+      case EqualNullSafe(left: Attribute, right: Expression) if domainAttrSet.contains(left) =>
+        left -> right
+    }.toMap
+  }
+
+  /**
+   * Rewrite all [[DomainJoin]]s in the inner query to actual inner joins with the outer query.
+   */
+  def rewriteDomainJoins(
+      outerPlan: LogicalPlan,
+      innerPlan: LogicalPlan,
+      conditions: Seq[Expression]): LogicalPlan = {
+    innerPlan transform {
+      case d @ DomainJoin(domainAttrs, child) =>
+        val domainAttrMap = buildDomainAttrMap(conditions, domainAttrs)
+        // We should only rewrite a domain join when all corresponding outer plan attributes
+        // can be found from the join condition.
+        if (domainAttrMap.size == domainAttrs.size) {
+          val groupingExprs = domainAttrs.map(domainAttrMap)
+          val aggregateExprs = groupingExprs.zip(domainAttrs).map {
+            // Rebuild the aliases.
+            case (inputAttr, outputAttr) => Alias(inputAttr, outputAttr.name)(outputAttr.exprId)
+          }
+          // Construct a domain with the outer query plan.
+          // DomainJoin [a', b']  =>  Aggregate [a, b] [a AS a', b AS b']
+          //                          +- Relation [a, b]
+          val domain = Aggregate(groupingExprs, aggregateExprs, outerPlan)
+          child match {
+            // A special optimization for OneRowRelation.
+            // TODO: add a more general rule to optimize join with OneRowRelation.
+            case _: OneRowRelation => domain
+            // Construct a domain join.
+            // Join Inner
+            // :- Inner Query
+            // +- Domain
+            case _ => Join(child, domain, Inner, None, JoinHint.NONE)
+          }
+        } else {
+          throw new UnsupportedOperationException(
+            s"Unable to rewrite domain join with conditions: $conditions\n$d")
+        }
+    }
+  }
+
+  def apply(
+      innerPlan: LogicalPlan,
+      outerPlan: LogicalPlan): (LogicalPlan, Seq[Expression]) = {
+    apply(innerPlan, Seq(outerPlan))
+  }
+
+  def apply(
+      innerPlan: LogicalPlan,
+      outerPlans: Seq[LogicalPlan]): (LogicalPlan, Seq[Expression]) = {
+    val outputSet = AttributeSet(outerPlans.flatMap(_.outputSet))
+
+    // The return type of the recursion.
+    // The first parameter is a new logical plan with correlation eliminated.
+    // The second parameter is a list of join conditions with the outer query.
+    // The third parameter is a mapping between the outer references and equivalent
+    // expressions from the inner query that is used to replace outer references.
+    type ReturnType = (LogicalPlan, Seq[Expression], Map[Attribute, Attribute])
+
+    // Decorrelate the input plan with a set of parent outer references and a boolean flag
+    // indicating whether the result of the plan will be aggregated. Steps:
+    // 1. Recursively collects outer references from the inner query until it reaches a node
+    //    that does not contain correlated value.
+    // 2. Inserts an optional [[DomainJoin]] node to indicate whether a domain (inner) join is
+    //    needed between the outer query and the specific sub-tree of the inner query.
+    // 3. Returns a list of join conditions with the outer query and a mapping between outer
+    //    references with references inside the inner query. The parent nodes need to preserve
+    //    the references inside the join conditions and substitute all outer references using
+    //    the mapping.
+    def decorrelate(
+        plan: LogicalPlan,
+        parentOuterReferences: AttributeSet,
+        aggregated: Boolean = false): ReturnType = {
+      val isCorrelated = hasOuterReferences(plan)
+      if (!isCorrelated) {
+        // We have reached a plan without correlation to the outer plan.
+        if (parentOuterReferences.isEmpty) {
+          // If there is no outer references from the parent nodes, it means all outer
+          // attributes can be substituted by attributes from the inner plan. So no
+          // domain join is needed.
+          (plan, Nil, Map.empty[Attribute, Attribute])
+        } else {
+          // Build the domain join with the parent outer references.
+          val attributes = parentOuterReferences.toSeq
+          val domains = attributes.map(_.newInstance())
+          // A placeholder to be rewritten into domain join.
+          val domainJoin = DomainJoin(domains, plan)
+          val outerReferenceMap = attributes.zip(domains).toMap
+          // Build join conditions between domain attributes and outer references.
+          // EqualNullSafe is used to make sure null key can be joined together. Note
+          // outer referenced attributes can be changed during the outer query optimization.
+          // The equality conditions will also serve as an attribute mapping between new
+          // outer references and domain attributes when rewriting the domain joins.
+          // E.g. if the attribute a is changed to a1, the join condition a' <=> outer(a)
+          // will become a' <=> a1, and we can construct the aliases based on the condition:
+          // DomainJoin [a']        Join Inner
+          // +- InnerQuery     =>   :- InnerQuery
+          //                        +- Aggregate [a1] [a1 AS a']
+          //                           +- OuterQuery
+          val conditions = outerReferenceMap.map {
+            case (o, a) => EqualNullSafe(a, OuterReference(o))
+          }
+          (domainJoin, conditions.toSeq, outerReferenceMap)
+        }
+      } else {
+        plan match {
+          case Filter(condition, child) =>
+            val conditions = splitConjunctivePredicates(condition)
+            val (correlated, uncorrelated) = conditions.partition(containsOuter)
+            // Find outer references that can be substituted by attributes from the inner
+            // query using the equality predicates.
+            val equivalences = collectEquivalentOuterReferences(correlated)
+            // Correlated predicates can be removed from the Filter's condition and used as
+            // join conditions with the outer query. However, if the results of the sub-tree
+            // is aggregated, only certain correlated equality predicates can be used, because
+            // the references in the join conditions need to be preserved in both the grouping
+            // and aggregate expressions of an Aggregate, which may change the semantics of the
+            // plan and lead to incorrect results. Here is an example:
+            // Relations:
+            //   t1(a, b): [(1, 1)]
+            //   t2(c, d): [(1, 1), (2, 0)]
+            //
+            // Query:
+            //   SELECT * FROM t1 WHERE a = (SELECT MAX(c) FROM t2 WHERE b >= d)
+            //
+            // Subquery plan transformation if correlated predicates are used as join conditions:
+            //   Aggregate [max(c)]               Aggregate [d] [max(c), d]
+            //   +- Filter (outer(b) >= d)   =>   +- Relation [c, d]
+            //      +- Relation [c, d]
+            //
+            // Plan after rewrite:
+            //   Project [a, b]                                   -- [(1, 1)]
+            //   +- Join LeftOuter (b >= d AND a = max(c))
+            //      :- Relation [a, b]
+            //      +- Aggregate [d] [max(c), d]                  -- [(1, 1), (2, 0)]
+            //         +- Relation [c, d]
+            //
+            // The result of the original query should be an empty set but the transformed
+            // query will output an incorrect result of (1, 1). The correct transformation
+            // with domain join is illustrated below:
+            //   Aggregate [max(c)]               Aggregate [b'] [max(c), b']
+            //   +- Filter (outer(b) >= d)   =>   +- Filter (b' >= d)
+            //      +- Relation [c, d]               +- DomainJoin [b']
+            //                                          +- Relation [c, d]
+            // Plan after rewrite:
+            //   Project [a, b]
+            //   +- Join LeftOuter (b <=> b' AND a = max(c))  -- []
+            //      :- Relation [a, b]
+            //      +- Aggregate [b'] [max(c), b']            -- [(2, 1)]
+            //         +- Join Inner (b' >= d)                -- [(1, 1, 1), (2, 0, 1)] (DomainJoin)
+            //            :- Relation [c, d]
+            //            +- Aggregate [b] [b AS b']          -- [(1)] (Domain)
+            //               +- Relation [a, b]
+            if (aggregated) {
+              // Split the correlated predicates into predicates that can and cannot be directly
+              // used as join conditions with the outer query depending on whether they can
+              // be pulled up over an Aggregate without changing the semantics of the plan.
+              val (equalityCond, predicates) = correlated.partition(canPullUpOverAgg)
+              val outerReferences = collectOuterReferences(predicates)
+              val newOuterReferences =
+                parentOuterReferences ++ outerReferences -- equivalences.keySet
+              val (newChild, joinCond, outerReferenceMap) =
+                decorrelate(child, newOuterReferences, aggregated)
+              // Add the outer references mapping collected from the equality conditions.
+              val newOuterReferenceMap = outerReferenceMap ++ equivalences
+              // Replace all outer references in the non-equality predicates.
+              val newCorrelated = replaceOuterReferences(predicates, newOuterReferenceMap)
+              // The new filter condition is the original filter condition with correlated
+              // equality predicates removed.
+              val newFilterCond = newCorrelated ++ uncorrelated
+              val newFilter = newFilterCond match {
+                case Nil => newChild
+                case conditions => Filter(conditions.reduce(And), newChild)
+              }
+              // Equality predicates are used as join conditions with the outer query.
+              val newJoinCond = joinCond ++ equalityCond
+              (newFilter, newJoinCond, newOuterReferenceMap)
+            } else {
+              // Results of this sub-tree is not aggregated, so all correlated predicates
+              // can be directly used as outer query join conditions.
+              val newOuterReferences = parentOuterReferences -- equivalences.keySet
+              val (newChild, joinCond, outerReferenceMap) =
+                decorrelate(child, newOuterReferences, aggregated)
+              // Add the outer references mapping collected from the equality conditions.
+              val newOuterReferenceMap = outerReferenceMap ++ equivalences
+              val newFilter = uncorrelated match {
+                case Nil => newChild
+                case conditions => Filter(conditions.reduce(And), newChild)
+              }
+              val newJoinCond = joinCond ++ correlated
+              (newFilter, newJoinCond, newOuterReferenceMap)
+            }
+
+          case Project(projectList, child) =>
+            val outerReferences = collectOuterReferences(projectList)
+            val newOuterReferences = parentOuterReferences ++ outerReferences
+            val (newChild, joinCond, outerReferenceMap) =
+              decorrelate(child, newOuterReferences, aggregated)
+            // Replace all outer references in the original project list.
+            val newProjectList = replaceOuterReferences(projectList, outerReferenceMap)
+            // Preserve required domain attributes in the join condition by adding the missing
+            // references to the new project list.
+            val referencesToAdd = missingReferences(newProjectList, joinCond)
+            val newProject = Project(newProjectList ++ referencesToAdd, newChild)
+            (newProject, joinCond, outerReferenceMap)
+
+          case a @ Aggregate(groupingExpressions, aggregateExpressions, child) =>
+            val outerReferences = collectOuterReferences(a.expressions)
+            val newOuterReferences = parentOuterReferences ++ outerReferences
+            val (newChild, joinCond, outerReferenceMap) =
+              decorrelate(child, newOuterReferences, aggregated = true)
+            // Replace all outer references in grouping and aggregate expressions.
+            val newGroupingExpr = replaceOuterReferences(groupingExpressions, outerReferenceMap)
+            val newAggExpr = replaceOuterReferences(aggregateExpressions, outerReferenceMap)
+            // Add all required domain attributes to both grouping and aggregate expressions.
+            val referencesToAdd = missingReferences(newAggExpr, joinCond)
+            val newAggregate = a.copy(
+              groupingExpressions = newGroupingExpr ++ referencesToAdd,
+              aggregateExpressions = newAggExpr ++ referencesToAdd,
+              child = newChild)
+            (newAggregate, joinCond, outerReferenceMap)
+
+          case j @ Join(left, right, joinType, condition, _) =>
+            val outerReferences = collectOuterReferences(j.expressions)
+            // Join condition containing outer references is not supported.
+            assert(outerReferences.isEmpty, s"Correlated column is not allowed in join: $j")
+            val newOuterReferences = parentOuterReferences ++ outerReferences
+            val shouldPushToLeft = joinType match {
+              case LeftOuter | LeftSemiOrAnti(_) | FullOuter => true
+              case _ => hasOuterReferences(left)
+            }
+            val shouldPushToRight = joinType match {
+              case RightOuter | FullOuter => true
+              case _ => hasOuterReferences(right)
+            }
+            val (newLeft, leftJoinCond, leftOuterReferenceMap) = if (shouldPushToLeft) {
+              decorrelate(left, newOuterReferences, aggregated)
+            } else {
+              (left, Nil, Map.empty[Attribute, Attribute])
+            }
+            val (newRight, rightJoinCond, rightOuterReferenceMap) = if (shouldPushToRight) {
+              decorrelate(right, newOuterReferences, aggregated)
+            } else {
+              (right, Nil, Map.empty[Attribute, Attribute])
+            }
+            val newOuterReferenceMap = leftOuterReferenceMap ++ rightOuterReferenceMap
+            val newJoinCond = leftJoinCond ++ rightJoinCond
+            // If we push the dependent join to both sides, we can augment the join condition
+            // such that both sides are matched on the domain attributes. For example,
+            // - Left Map: {outer(c1) = c1}
+            // - Right Map: {outer(c1) = 10 - c1}
+            // Then the join condition can be augmented with (c1 <=> 10 - c1).
+            val augmentedConditions = leftOuterReferenceMap.flatMap {
+              case (outer, inner) => rightOuterReferenceMap.get(outer).map(EqualNullSafe(inner, _))
+            }
+            val newCondition = (condition ++ augmentedConditions).reduceOption(And)
+            val newJoin = j.copy(left = newLeft, right = newRight, condition = newCondition)
+            (newJoin, newJoinCond, newOuterReferenceMap)
+
+          case u: UnaryNode =>
+            val outerReferences = collectOuterReferences(u.expressions)
+            assert(outerReferences.isEmpty, s"Correlated column is not allowed in $u")
+            decorrelate(u.child, parentOuterReferences, aggregated)
+
+          case o =>
+            throw new UnsupportedOperationException(
+              s"Decorrelate inner query through ${o.nodeName} is not supported.")
+        }
+      }
+    }
+    val (newChild, joinCond, _) = decorrelate(BooleanSimplification(innerPlan), AttributeSet.empty)
+    val (plan, conditions) = deduplicate(newChild, joinCond, outputSet)
+    (plan, stripOuterReferences(conditions))
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/EliminateResolvedHint.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/EliminateResolvedHint.scala
index aa0ce5d4ed31a..71cbbbb763c0d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/EliminateResolvedHint.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/EliminateResolvedHint.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.catalyst.optimizer
 
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.internal.SQLConf
 
 /**
  * Replaces [[ResolvedHint]] operators from the plan. Move the [[HintInfo]] to associated [[Join]]
@@ -27,7 +26,7 @@ import org.apache.spark.sql.internal.SQLConf
  */
 object EliminateResolvedHint extends Rule[LogicalPlan] {
 
-  private val hintErrorHandler = SQLConf.get.hintErrorHandler
+  private val hintErrorHandler = conf.hintErrorHandler
 
   // This is also called in the beginning of the optimization phase, and as a result
   // is using transformUp rather than resolveOperators.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/LimitPushDownThroughWindow.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/LimitPushDownThroughWindow.scala
new file mode 100644
index 0000000000000..eaea167ee9ff2
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/LimitPushDownThroughWindow.scala
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.expressions.{Alias, CurrentRow, IntegerLiteral, NamedExpression, RankLike, RowFrame, RowNumberLike, SpecifiedWindowFrame, UnboundedPreceding, WindowExpression, WindowSpecDefinition}
+import org.apache.spark.sql.catalyst.plans.logical.{Limit, LocalLimit, LogicalPlan, Project, Sort, Window}
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreePattern.{LIMIT, WINDOW}
+
+/**
+ * Pushes down [[LocalLimit]] beneath WINDOW. This rule optimizes the following case:
+ * {{{
+ *   SELECT *, ROW_NUMBER() OVER(ORDER BY a) AS rn FROM Tab1 LIMIT 5 ==>
+ *   SELECT *, ROW_NUMBER() OVER(ORDER BY a) AS rn FROM (SELECT * FROM Tab1 ORDER BY a LIMIT 5) t
+ * }}}
+ */
+object LimitPushDownThroughWindow extends Rule[LogicalPlan] {
+  // The window frame of RankLike and RowNumberLike can only be UNBOUNDED PRECEDING to CURRENT ROW.
+  private def supportsPushdownThroughWindow(
+      windowExpressions: Seq[NamedExpression]): Boolean = windowExpressions.forall {
+    case Alias(WindowExpression(_: RankLike | _: RowNumberLike, WindowSpecDefinition(Nil, _,
+        SpecifiedWindowFrame(RowFrame, UnboundedPreceding, CurrentRow))), _) => true
+    case _ => false
+  }
+
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsAllPatterns(WINDOW, LIMIT), ruleId) {
+    // Adding an extra Limit below WINDOW when the partitionSpec of all window functions is empty.
+    case LocalLimit(limitExpr @ IntegerLiteral(limit),
+        window @ Window(windowExpressions, Nil, orderSpec, child))
+      if supportsPushdownThroughWindow(windowExpressions) && child.maxRows.forall(_ > limit) &&
+        limit < conf.topKSortFallbackThreshold =>
+      // Sort is needed here because we need global sort.
+      window.copy(child = Limit(limitExpr, Sort(orderSpec, true, child)))
+    // There is a Project between LocalLimit and Window if they do not have the same output.
+    case LocalLimit(limitExpr @ IntegerLiteral(limit), project @ Project(_,
+        window @ Window(windowExpressions, Nil, orderSpec, child)))
+      if supportsPushdownThroughWindow(windowExpressions) && child.maxRows.forall(_ > limit) &&
+        limit < conf.topKSortFallbackThreshold =>
+      // Sort is needed here because we need global sort.
+      project.copy(child = window.copy(child = Limit(limitExpr, Sort(orderSpec, true, child))))
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala
index 30da902a33cf2..5b12667f4a884 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala
@@ -30,25 +30,54 @@ import org.apache.spark.sql.types._
  */
 object NestedColumnAliasing {
 
-  def unapply(plan: LogicalPlan)
-    : Option[(Map[ExtractValue, Alias], Map[ExprId, Seq[Alias]])] = plan match {
+  def unapply(plan: LogicalPlan): Option[LogicalPlan] = plan match {
+    /**
+     * This pattern is needed to support [[Filter]] plan cases like
+     * [[Project]]->[[Filter]]->listed plan in `canProjectPushThrough` (e.g., [[Window]]).
+     * The reason why we don't simply add [[Filter]] in `canProjectPushThrough` is that
+     * the optimizer can hit an infinite loop during the [[PushDownPredicates]] rule.
+     */
+    case Project(projectList, Filter(condition, child))
+        if SQLConf.get.nestedSchemaPruningEnabled && canProjectPushThrough(child) =>
+      val exprCandidatesToPrune = projectList ++ Seq(condition) ++ child.expressions
+      getAliasSubMap(exprCandidatesToPrune, child.producedAttributes.toSeq).map {
+        case (nestedFieldToAlias, attrToAliases) =>
+          NestedColumnAliasing.replaceToAliases(plan, nestedFieldToAlias, attrToAliases)
+      }
+
     case Project(projectList, child)
         if SQLConf.get.nestedSchemaPruningEnabled && canProjectPushThrough(child) =>
-      getAliasSubMap(projectList)
+      val exprCandidatesToPrune = projectList ++ child.expressions
+      getAliasSubMap(exprCandidatesToPrune, child.producedAttributes.toSeq).map {
+        case (nestedFieldToAlias, attrToAliases) =>
+          NestedColumnAliasing.replaceToAliases(plan, nestedFieldToAlias, attrToAliases)
+      }
+
+    case p if SQLConf.get.nestedSchemaPruningEnabled && canPruneOn(p) =>
+      val exprCandidatesToPrune = p.expressions
+      getAliasSubMap(exprCandidatesToPrune, p.producedAttributes.toSeq).map {
+        case (nestedFieldToAlias, attrToAliases) =>
+          NestedColumnAliasing.replaceToAliases(p, nestedFieldToAlias, attrToAliases)
+      }
+
     case _ => None
   }
 
   /**
    * Replace nested columns to prune unused nested columns later.
    */
-  def replaceToAliases(
+  private def replaceToAliases(
       plan: LogicalPlan,
       nestedFieldToAlias: Map[ExtractValue, Alias],
       attrToAliases: Map[ExprId, Seq[Alias]]): LogicalPlan = plan match {
     case Project(projectList, child) =>
       Project(
         getNewProjectList(projectList, nestedFieldToAlias),
-        replaceChildrenWithAliases(child, attrToAliases))
+        replaceWithAliases(child, nestedFieldToAlias, attrToAliases))
+
+    // The operators reaching here was already guarded by `canPruneOn`.
+    case other =>
+      replaceWithAliases(other, nestedFieldToAlias, attrToAliases)
   }
 
   /**
@@ -64,14 +93,28 @@ object NestedColumnAliasing {
   }
 
   /**
-   * Return a plan with new children replaced with aliases.
+   * Return a plan with new children replaced with aliases, and expressions replaced with
+   * aliased attributes.
    */
-  def replaceChildrenWithAliases(
+  def replaceWithAliases(
       plan: LogicalPlan,
+      nestedFieldToAlias: Map[ExtractValue, Alias],
       attrToAliases: Map[ExprId, Seq[Alias]]): LogicalPlan = {
     plan.withNewChildren(plan.children.map { plan =>
       Project(plan.output.flatMap(a => attrToAliases.getOrElse(a.exprId, Seq(a))), plan)
-    })
+    }).transformExpressions {
+      case f: ExtractValue if nestedFieldToAlias.contains(f) =>
+        nestedFieldToAlias(f).toAttribute
+    }
+  }
+
+  /**
+   * Returns true for those operators that we can prune nested column on it.
+   */
+  private def canPruneOn(plan: LogicalPlan) = plan match {
+    case _: Aggregate => true
+    case _: Expand => true
+    case _ => false
   }
 
   /**
@@ -82,6 +125,10 @@ object NestedColumnAliasing {
     case _: LocalLimit => true
     case _: Repartition => true
     case _: Sample => true
+    case _: RepartitionByExpression => true
+    case _: Join => true
+    case _: Window => true
+    case _: Sort => true
     case _ => false
   }
 
@@ -118,23 +165,39 @@ object NestedColumnAliasing {
         case _ => false
       }
 
+    // Note that when we group by extractors with their references, we should remove
+    // cosmetic variations.
     val exclusiveAttrSet = AttributeSet(exclusiveAttrs ++ otherRootReferences)
     val aliasSub = nestedFieldReferences.asInstanceOf[Seq[ExtractValue]]
       .filter(!_.references.subsetOf(exclusiveAttrSet))
-      .groupBy(_.references.head)
+      .groupBy(_.references.head.canonicalized.asInstanceOf[Attribute])
       .flatMap { case (attr, nestedFields: Seq[ExtractValue]) =>
+        // Remove redundant `ExtractValue`s if they share the same parent nest field.
+        // For example, when `a.b` and `a.b.c` are in project list, we only need to alias `a.b`.
+        // We only need to deal with two `ExtractValue`: `GetArrayStructFields` and
+        // `GetStructField`. Please refer to the method `collectRootReferenceAndExtractValue`.
+        val dedupNestedFields = nestedFields.filter {
+          case e @ (_: GetStructField | _: GetArrayStructFields) =>
+            val child = e.children.head
+            nestedFields.forall(f => child.find(_.semanticEquals(f)).isEmpty)
+          case _ => true
+        }
+
         // Each expression can contain multiple nested fields.
         // Note that we keep the original names to deliver to parquet in a case-sensitive way.
-        val nestedFieldToAlias = nestedFields.distinct.map { f =>
+        val nestedFieldToAlias = dedupNestedFields.distinct.map { f =>
           val exprId = NamedExpression.newExprId
           (f, Alias(f, s"_gen_alias_${exprId.id}")(exprId, Seq.empty, None))
         }
 
         // If all nested fields of `attr` are used, we don't need to introduce new aliases.
         // By default, ColumnPruning rule uses `attr` already.
+        // Note that we need to remove cosmetic variations first, so we only count a
+        // nested field once.
         if (nestedFieldToAlias.nonEmpty &&
-            nestedFieldToAlias
-              .map { case (nestedField, _) => totalFieldNum(nestedField.dataType) }
+            dedupNestedFields.map(_.canonicalized)
+              .distinct
+              .map { nestedField => totalFieldNum(nestedField.dataType) }
               .sum < totalFieldNum(attr.dataType)) {
           Some(attr.exprId -> nestedFieldToAlias)
         } else {
@@ -164,10 +227,31 @@ object NestedColumnAliasing {
 }
 
 /**
- * This prunes unnessary nested columns from `Generate` and optional `Project` on top
+ * This prunes unnecessary nested columns from `Generate` and optional `Project` on top
  * of it.
  */
 object GeneratorNestedColumnAliasing {
+  // Partitions `attrToAliases` based on whether the attribute is in Generator's output.
+  private def aliasesOnGeneratorOutput(
+      attrToAliases: Map[ExprId, Seq[Alias]],
+      generatorOutput: Seq[Attribute]) = {
+    val generatorOutputExprId = generatorOutput.map(_.exprId)
+    attrToAliases.partition { k =>
+      generatorOutputExprId.contains(k._1)
+    }
+  }
+
+  // Partitions `nestedFieldToAlias` based on whether the attribute of nested field extractor
+  // is in Generator's output.
+  private def nestedFieldOnGeneratorOutput(
+      nestedFieldToAlias: Map[ExtractValue, Alias],
+      generatorOutput: Seq[Attribute]) = {
+    val generatorOutputSet = AttributeSet(generatorOutput)
+    nestedFieldToAlias.partition { pair =>
+      pair._1.references.subsetOf(generatorOutputSet)
+    }
+  }
+
   def unapply(plan: LogicalPlan): Option[LogicalPlan] = plan match {
     // Either `nestedPruningOnExpressions` or `nestedSchemaPruningEnabled` is enabled, we
     // need to prune nested columns through Project and under Generate. The difference is
@@ -178,10 +262,81 @@ object GeneratorNestedColumnAliasing {
       // On top on `Generate`, a `Project` that might have nested column accessors.
       // We try to get alias maps for both project list and generator's children expressions.
       val exprsToPrune = projectList ++ g.generator.children
-      NestedColumnAliasing.getAliasSubMap(exprsToPrune, g.qualifiedGeneratorOutput).map {
+      NestedColumnAliasing.getAliasSubMap(exprsToPrune).map {
         case (nestedFieldToAlias, attrToAliases) =>
-          val newChild = pruneGenerate(g, nestedFieldToAlias, attrToAliases)
-          Project(NestedColumnAliasing.getNewProjectList(projectList, nestedFieldToAlias), newChild)
+          val (nestedFieldsOnGenerator, nestedFieldsNotOnGenerator) =
+            nestedFieldOnGeneratorOutput(nestedFieldToAlias, g.qualifiedGeneratorOutput)
+          val (attrToAliasesOnGenerator, attrToAliasesNotOnGenerator) =
+            aliasesOnGeneratorOutput(attrToAliases, g.qualifiedGeneratorOutput)
+
+          // Push nested column accessors through `Generator`.
+          // Defer updating `Generate.unrequiredChildIndex` to next round of `ColumnPruning`.
+          val newChild = NestedColumnAliasing.replaceWithAliases(g,
+            nestedFieldsNotOnGenerator, attrToAliasesNotOnGenerator)
+          val pushedThrough = Project(NestedColumnAliasing
+            .getNewProjectList(projectList, nestedFieldsNotOnGenerator), newChild)
+
+          // If the generator output is `ArrayType`, we cannot push through the extractor.
+          // It is because we don't allow field extractor on two-level array,
+          // i.e., attr.field when attr is a ArrayType(ArrayType(...)).
+          // Similarily, we also cannot push through if the child of generator is `MapType`.
+          g.generator.children.head.dataType match {
+            case _: MapType => return Some(pushedThrough)
+            case ArrayType(_: ArrayType, _) => return Some(pushedThrough)
+            case _ =>
+          }
+
+          // Pruning on `Generator`'s output. We only process single field case.
+          // For multiple field case, we cannot directly move field extractor into
+          // the generator expression. A workaround is to re-construct array of struct
+          // from multiple fields. But it will be more complicated and may not worth.
+          // TODO(SPARK-34956): support multiple fields.
+          if (nestedFieldsOnGenerator.size > 1 || nestedFieldsOnGenerator.isEmpty) {
+            pushedThrough
+          } else {
+            // Only one nested column accessor.
+            // E.g., df.select(explode($"items").as("item")).select($"item.a")
+            pushedThrough match {
+              case p @ Project(_, newG: Generate) =>
+                // Replace the child expression of `ExplodeBase` generator with
+                // nested column accessor.
+                // E.g., df.select(explode($"items").as("item")).select($"item.a") =>
+                //       df.select(explode($"items.a").as("item.a"))
+                val rewrittenG = newG.transformExpressions {
+                  case e: ExplodeBase =>
+                    val extractor = nestedFieldsOnGenerator.head._1.transformUp {
+                      case _: Attribute =>
+                        e.child
+                      case g: GetStructField =>
+                        ExtractValue(g.child, Literal(g.extractFieldName), SQLConf.get.resolver)
+                    }
+                    e.withNewChildren(Seq(extractor))
+                }
+
+                // As we change the child of the generator, its output data type must be updated.
+                val updatedGeneratorOutput = rewrittenG.generatorOutput
+                  .zip(rewrittenG.generator.elementSchema.toAttributes)
+                  .map { case (oldAttr, newAttr) =>
+                  newAttr.withExprId(oldAttr.exprId).withName(oldAttr.name)
+                }
+                assert(updatedGeneratorOutput.length == rewrittenG.generatorOutput.length,
+                  "Updated generator output must have the same length " +
+                    "with original generator output.")
+                val updatedGenerate = rewrittenG.copy(generatorOutput = updatedGeneratorOutput)
+
+                // Replace nested column accessor with generator output.
+                p.withNewChildren(Seq(updatedGenerate)).transformExpressions {
+                  case f: ExtractValue if nestedFieldsOnGenerator.contains(f) =>
+                    updatedGenerate.output
+                      .find(a => attrToAliasesOnGenerator.contains(a.exprId))
+                      .getOrElse(f)
+                }
+
+              case other =>
+                // We should not reach here.
+                throw new IllegalStateException(s"Unreasonable plan after optimization: $other")
+            }
+          }
       }
 
     case g: Generate if SQLConf.get.nestedSchemaPruningEnabled &&
@@ -193,28 +348,14 @@ object GeneratorNestedColumnAliasing {
       NestedColumnAliasing.getAliasSubMap(
         g.generator.children, g.requiredChildOutput).map {
         case (nestedFieldToAlias, attrToAliases) =>
-          pruneGenerate(g, nestedFieldToAlias, attrToAliases)
+          // Defer updating `Generate.unrequiredChildIndex` to next round of `ColumnPruning`.
+          NestedColumnAliasing.replaceWithAliases(g, nestedFieldToAlias, attrToAliases)
       }
 
     case _ =>
       None
   }
 
-  private def pruneGenerate(
-      g: Generate,
-      nestedFieldToAlias: Map[ExtractValue, Alias],
-      attrToAliases: Map[ExprId, Seq[Alias]]): LogicalPlan = {
-    val newGenerator = g.generator.transform {
-      case f: ExtractValue if nestedFieldToAlias.contains(f) =>
-        nestedFieldToAlias(f).toAttribute
-    }.asInstanceOf[Generator]
-
-    // Defer updating `Generate.unrequiredChildIndex` to next round of `ColumnPruning`.
-    val newGenerate = g.copy(generator = newGenerator)
-
-    NestedColumnAliasing.replaceChildrenWithAliases(newGenerate, attrToAliases)
-  }
-
   /**
    * This is a while-list for pruning nested fields at `Generator`.
    */
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala
index 5f94af5ffe636..a6444b13acd02 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala
@@ -17,10 +17,10 @@
 
 package org.apache.spark.sql.catalyst.optimizer
 
-import org.apache.spark.sql.catalyst.expressions.{Alias, And, ArrayTransform, CreateArray, CreateMap, CreateNamedStruct, CreateStruct, EqualTo, ExpectsInputTypes, Expression, GetStructField, KnownFloatingPointNormalized, LambdaFunction, NamedLambdaVariable, UnaryExpression}
+import org.apache.spark.sql.catalyst.expressions.{Alias, And, ArrayTransform, CaseWhen, Coalesce, CreateArray, CreateMap, CreateNamedStruct, EqualTo, ExpectsInputTypes, Expression, GetStructField, If, IsNull, KnownFloatingPointNormalized, LambdaFunction, Literal, NamedLambdaVariable, UnaryExpression}
 import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
 import org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys
-import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Subquery, Window}
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Window}
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.types._
 
@@ -56,10 +56,6 @@ import org.apache.spark.sql.types._
 object NormalizeFloatingNumbers extends Rule[LogicalPlan] {
 
   def apply(plan: LogicalPlan): LogicalPlan = plan match {
-    // A subquery will be rewritten into join later, and will go through this rule
-    // eventually. Here we skip subquery, as we only need to run this rule once.
-    case _: Subquery => plan
-
     case _ => plan transform {
       case w: Window if w.partitionSpec.exists(p => needNormalize(p)) =>
         // Although the `windowExpressions` may refer to `partitionSpec` expressions, we don't need
@@ -123,11 +119,21 @@ object NormalizeFloatingNumbers extends Rule[LogicalPlan] {
     case _ if expr.dataType == FloatType || expr.dataType == DoubleType =>
       KnownFloatingPointNormalized(NormalizeNaNAndZero(expr))
 
+    case If(cond, trueValue, falseValue) =>
+      If(cond, normalize(trueValue), normalize(falseValue))
+
+    case CaseWhen(branches, elseVale) =>
+      CaseWhen(branches.map(br => (br._1, normalize(br._2))), elseVale.map(normalize))
+
+    case Coalesce(children) =>
+      Coalesce(children.map(normalize))
+
     case _ if expr.dataType.isInstanceOf[StructType] =>
-      val fields = expr.dataType.asInstanceOf[StructType].fields.indices.map { i =>
-        normalize(GetStructField(expr, i))
+      val fields = expr.dataType.asInstanceOf[StructType].fieldNames.zipWithIndex.map {
+        case (name, i) => Seq(Literal(name), normalize(GetStructField(expr, i)))
       }
-      CreateStruct(fields)
+      val struct = CreateNamedStruct(fields.flatten.toSeq)
+      KnownFloatingPointNormalized(If(IsNull(expr), Literal(null, struct.dataType), struct))
 
     case _ if expr.dataType.isInstanceOf[ArrayType] =>
       val ArrayType(et, containsNull) = expr.dataType
@@ -137,6 +143,28 @@ object NormalizeFloatingNumbers extends Rule[LogicalPlan] {
 
     case _ => throw new IllegalStateException(s"fail to normalize $expr")
   }
+
+  val FLOAT_NORMALIZER: Any => Any = (input: Any) => {
+    val f = input.asInstanceOf[Float]
+    if (f.isNaN) {
+      Float.NaN
+    } else if (f == -0.0f) {
+      0.0f
+    } else {
+      f
+    }
+  }
+
+  val DOUBLE_NORMALIZER: Any => Any = (input: Any) => {
+    val d = input.asInstanceOf[Double]
+    if (d.isNaN) {
+      Double.NaN
+    } else if (d == -0.0d) {
+      0.0d
+    } else {
+      d
+    }
+  }
 }
 
 case class NormalizeNaNAndZero(child: Expression) extends UnaryExpression with ExpectsInputTypes {
@@ -146,27 +174,8 @@ case class NormalizeNaNAndZero(child: Expression) extends UnaryExpression with E
   override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection(FloatType, DoubleType))
 
   private lazy val normalizer: Any => Any = child.dataType match {
-    case FloatType => (input: Any) => {
-      val f = input.asInstanceOf[Float]
-      if (f.isNaN) {
-        Float.NaN
-      } else if (f == -0.0f) {
-        0.0f
-      } else {
-        f
-      }
-    }
-
-    case DoubleType => (input: Any) => {
-      val d = input.asInstanceOf[Double]
-      if (d.isNaN) {
-        Double.NaN
-      } else if (d == -0.0d) {
-        0.0d
-      } else {
-        d
-      }
-    }
+    case FloatType => NormalizeFloatingNumbers.FLOAT_NORMALIZER
+    case DoubleType => NormalizeFloatingNumbers.DOUBLE_NORMALIZER
   }
 
   override def nullSafeEval(input: Any): Any = {
@@ -202,4 +211,7 @@ case class NormalizeNaNAndZero(child: Expression) extends UnaryExpression with E
 
     nullSafeCodeGen(ctx, ev, codeToNormalize)
   }
+
+  override protected def withNewChildInternal(newChild: Expression): NormalizeNaNAndZero =
+    copy(child = newChild)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeCsvJsonExprs.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeCsvJsonExprs.scala
new file mode 100644
index 0000000000000..670f79209c25b
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeCsvJsonExprs.scala
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreePattern.{CREATE_NAMED_STRUCT, EXTRACT_VALUE,
+  JSON_TO_STRUCT}
+import org.apache.spark.sql.types.{ArrayType, StructType}
+
+/**
+ * Simplify redundant csv/json related expressions.
+ *
+ * The optimization includes:
+ * 1. JsonToStructs(StructsToJson(child)) => child.
+ * 2. Prune unnecessary columns from GetStructField/GetArrayStructFields + JsonToStructs.
+ * 3. CreateNamedStruct(JsonToStructs(json).col1, JsonToStructs(json).col2, ...) =>
+ *      If(IsNull(json), nullStruct, KnownNotNull(JsonToStructs(prunedSchema, ..., json)))
+ *      if JsonToStructs(json) is shared among all fields of CreateNamedStruct. `prunedSchema`
+ *      contains all accessed fields in original CreateNamedStruct.
+ * 4. Prune unnecessary columns from GetStructField + CsvToStructs.
+ */
+object OptimizeCsvJsonExprs extends Rule[LogicalPlan] {
+  private def nameOfCorruptRecord = conf.columnNameOfCorruptRecord
+
+  override def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsAnyPattern(CREATE_NAMED_STRUCT, EXTRACT_VALUE, JSON_TO_STRUCT), ruleId) {
+    case p =>
+      val optimized = if (conf.jsonExpressionOptimization) {
+        p.transformExpressionsWithPruning(
+          _.containsAnyPattern(CREATE_NAMED_STRUCT, EXTRACT_VALUE, JSON_TO_STRUCT)
+          )(jsonOptimization)
+      } else {
+        p
+      }
+
+      if (conf.csvExpressionOptimization) {
+        optimized.transformExpressionsWithPruning(
+          _.containsAnyPattern(EXTRACT_VALUE))(csvOptimization)
+      } else {
+        optimized
+      }
+  }
+
+  private val jsonOptimization: PartialFunction[Expression, Expression] = {
+    case c: CreateNamedStruct
+        // If we create struct from various fields of the same `JsonToStructs`.
+        if c.valExprs.forall { v =>
+          v.isInstanceOf[GetStructField] &&
+            v.asInstanceOf[GetStructField].child.isInstanceOf[JsonToStructs] &&
+            v.children.head.semanticEquals(c.valExprs.head.children.head)
+        } =>
+      val jsonToStructs = c.valExprs.map(_.children.head)
+      val sameFieldName = c.names.zip(c.valExprs).forall {
+        case (name, valExpr: GetStructField) =>
+          name.toString == valExpr.childSchema(valExpr.ordinal).name
+        case _ => false
+      }
+
+      // Although `CreateNamedStruct` allows duplicated field names, e.g. "a int, a int",
+      // `JsonToStructs` does not support parsing json with duplicated field names.
+      val duplicateFields = c.names.map(_.toString).distinct.length != c.names.length
+
+      // If we create struct from various fields of the same `JsonToStructs` and we don't
+      // alias field names and there is no duplicated field in the struct.
+      if (sameFieldName && !duplicateFields) {
+        val fromJson = jsonToStructs.head.asInstanceOf[JsonToStructs].copy(schema = c.dataType)
+        val nullFields = c.children.grouped(2).flatMap {
+          case Seq(name, value) => Seq(name, Literal(null, value.dataType))
+        }.toSeq
+
+        If(IsNull(fromJson.child), c.copy(children = nullFields), KnownNotNull(fromJson))
+      } else {
+        c
+      }
+
+    case jsonToStructs @ JsonToStructs(_, options1,
+      StructsToJson(options2, child, timeZoneId2), timeZoneId1)
+        if options1.isEmpty && options2.isEmpty && timeZoneId1 == timeZoneId2 &&
+          jsonToStructs.dataType == child.dataType =>
+      // `StructsToJson` only fails when `JacksonGenerator` encounters data types it
+      // cannot convert to JSON. But `StructsToJson.checkInputDataTypes` already
+      // verifies its child's data types is convertible to JSON. But in
+      // `StructsToJson(JsonToStructs(...))` case, we cannot verify input json string
+      // so `JsonToStructs` might throw error in runtime. Thus we cannot optimize
+      // this case similarly.
+      child
+
+    case g @ GetStructField(j @ JsonToStructs(schema: StructType, _, _, _), ordinal, _)
+        if schema.length > 1 && j.options.isEmpty =>
+        // Options here should be empty because the optimization should not be enabled
+        // for some options. For example, when the parse mode is failfast it should not
+        // optimize, and should force to parse the whole input JSON with failing fast for
+        // an invalid input.
+        // To be more conservative, it does not optimize when any option is set for now.
+      val prunedSchema = StructType(Seq(schema(ordinal)))
+      g.copy(child = j.copy(schema = prunedSchema), ordinal = 0)
+
+    case g @ GetArrayStructFields(j @ JsonToStructs(schema: ArrayType, _, _, _), _, _, _, _)
+        if schema.elementType.asInstanceOf[StructType].length > 1 && j.options.isEmpty =>
+      val prunedSchema = ArrayType(StructType(Seq(g.field)), g.containsNull)
+      g.copy(child = j.copy(schema = prunedSchema), ordinal = 0, numFields = 1)
+  }
+
+  private val csvOptimization: PartialFunction[Expression, Expression] = {
+    case g @ GetStructField(c @ CsvToStructs(schema: StructType, _, _, _, None), ordinal, _)
+        if schema.length > 1 && c.options.isEmpty && schema(ordinal).name != nameOfCorruptRecord =>
+        // When the parse mode is permissive, and corrupt column is not selected, we can prune here
+        // from `GetStructField`. To be more conservative, it does not optimize when any option
+        // is set.
+      val prunedSchema = StructType(Seq(schema(ordinal)))
+      g.copy(child = c.copy(requiredSchema = Some(prunedSchema)), ordinal = 0)
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index e59e3b999aa7f..19e9312715db0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -27,6 +27,8 @@ import org.apache.spark.sql.catalyst.expressions.aggregate._
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules._
+import org.apache.spark.sql.catalyst.trees.AlwaysProcess
+import org.apache.spark.sql.catalyst.trees.TreePattern._
 import org.apache.spark.sql.connector.catalog.CatalogManager
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
@@ -43,12 +45,14 @@ abstract class Optimizer(catalogManager: CatalogManager)
   // Currently we check after the execution of each rule if a plan:
   // - is still resolved
   // - only host special expressions in supported operators
+  // - has globally-unique attribute IDs
   override protected def isPlanIntegral(plan: LogicalPlan): Boolean = {
     !Utils.isTesting || (plan.resolved &&
-      plan.find(PlanHelper.specialExpressionsInUnsupportedOperator(_).nonEmpty).isEmpty)
+      plan.find(PlanHelper.specialExpressionsInUnsupportedOperator(_).nonEmpty).isEmpty &&
+      LogicalPlanIntegrity.checkIfExprIdsAreGloballyUnique(plan))
   }
 
-  override protected val blacklistedOnceBatches: Set[String] =
+  override protected val excludedOnceBatches: Set[String] =
     Set(
       "PartitionPruning",
       "Extract Python UDFs")
@@ -76,49 +80,62 @@ abstract class Optimizer(catalogManager: CatalogManager)
         PushDownLeftSemiAntiJoin,
         PushLeftSemiLeftAntiThroughJoin,
         LimitPushDown,
+        LimitPushDownThroughWindow,
         ColumnPruning,
-        InferFiltersFromConstraints,
         // Operator combine
         CollapseRepartition,
         CollapseProject,
+        OptimizeWindowFunctions,
         CollapseWindow,
         CombineFilters,
-        CombineLimits,
+        EliminateLimits,
         CombineUnions,
         // Constant folding and strength reduction
+        OptimizeRepartition,
         TransposeWindow,
         NullPropagation,
         ConstantPropagation,
         FoldablePropagation,
         OptimizeIn,
         ConstantFolding,
+        EliminateAggregateFilter,
         ReorderAssociativeOperator,
         LikeSimplification,
         BooleanSimplification,
         SimplifyConditionals,
+        PushFoldableIntoBranches,
         RemoveDispensableExpressions,
         SimplifyBinaryComparison,
         ReplaceNullWithFalseInPredicate,
+        SimplifyConditionalsInPredicate,
         PruneFilters,
         SimplifyCasts,
         SimplifyCaseConversionExpressions,
         RewriteCorrelatedScalarSubquery,
         EliminateSerialization,
         RemoveRedundantAliases,
+        RemoveRedundantAggregates,
+        UnwrapCastInBinaryComparison,
         RemoveNoopOperators,
+        OptimizeUpdateFields,
         SimplifyExtractValueOps,
+        OptimizeCsvJsonExprs,
         CombineConcats) ++
         extendedOperatorOptimizationRules
 
     val operatorOptimizationBatch: Seq[Batch] = {
-      val rulesWithoutInferFiltersFromConstraints =
-        operatorOptimizationRuleSet.filterNot(_ == InferFiltersFromConstraints)
       Batch("Operator Optimization before Inferring Filters", fixedPoint,
-        rulesWithoutInferFiltersFromConstraints: _*) ::
+        operatorOptimizationRuleSet: _*) ::
       Batch("Infer Filters", Once,
+        InferFiltersFromGenerate,
         InferFiltersFromConstraints) ::
       Batch("Operator Optimization after Inferring Filters", fixedPoint,
-        rulesWithoutInferFiltersFromConstraints: _*) :: Nil
+        operatorOptimizationRuleSet: _*) ::
+      // Set strategy to Once to avoid pushing filter every time because we do not change the
+      // join condition.
+      Batch("Push extra predicate through join", fixedPoint,
+        PushExtraPredicateThroughJoin,
+        PushDownPredicates) :: Nil
     }
 
     val batches = (Batch("Eliminate Distinct", Once, EliminateDistinct) ::
@@ -132,10 +149,9 @@ abstract class Optimizer(catalogManager: CatalogManager)
       EliminateView,
       ReplaceExpressions,
       RewriteNonCorrelatedExists,
+      PullOutGroupingExpressions,
       ComputeCurrentTime,
-      GetCurrentDatabase(catalogManager),
-      RewriteDistinctAggregates,
-      ReplaceDeduplicateWithAggregate) ::
+      GetCurrentDatabaseAndCatalog(catalogManager)) ::
     //////////////////////////////////////////////////////////////////////////////////////////
     // Optimizer rules start here
     //////////////////////////////////////////////////////////////////////////////////////////
@@ -145,7 +161,9 @@ abstract class Optimizer(catalogManager: CatalogManager)
     // - Call CombineUnions again in Batch("Operator Optimizations"),
     //   since the other rules might make two separate Unions operators adjacent.
     Batch("Union", Once,
-      CombineUnions) ::
+      RemoveNoopOperators,
+      CombineUnions,
+      RemoveNoopUnion) ::
     Batch("OptimizeLimitZero", Once,
       OptimizeLimitZero) ::
     // Run this once earlier. This might simplify the plan and reduce cost of optimizer.
@@ -155,7 +173,10 @@ abstract class Optimizer(catalogManager: CatalogManager)
     // LocalRelation and does not trigger many rules.
     Batch("LocalRelation early", fixedPoint,
       ConvertToLocalRelation,
-      PropagateEmptyRelation) ::
+      PropagateEmptyRelation,
+      // PropagateEmptyRelation can change the nullability of an attribute from nullable to
+      // non-nullable when an empty relation child of a Union is removed
+      UpdateAttributeNullability) ::
     Batch("Pullup Correlated Expressions", Once,
       PullupCorrelatedPredicates) ::
     // Subquery batch applies the optimizer rules recursively. Therefore, it makes no sense
@@ -168,11 +189,15 @@ abstract class Optimizer(catalogManager: CatalogManager)
       ReplaceIntersectWithSemiJoin,
       ReplaceExceptWithFilter,
       ReplaceExceptWithAntiJoin,
-      ReplaceDistinctWithAggregate) ::
+      ReplaceDistinctWithAggregate,
+      ReplaceDeduplicateWithAggregate) ::
     Batch("Aggregate", fixedPoint,
       RemoveLiteralFromGroupExpressions,
       RemoveRepetitionFromGroupExpressions) :: Nil ++
     operatorOptimizationBatch) :+
+    // This batch rewrites plans after the operator optimization and
+    // before any batches that depend on stats.
+    Batch("Pre CBO Rules", Once, preCBORules: _*) :+
     // This batch pushes filters and projections into scan nodes. Before this batch, the logical
     // plan may contain nodes that do not report stats. Anything that uses stats must run after
     // this batch.
@@ -185,6 +210,10 @@ abstract class Optimizer(catalogManager: CatalogManager)
       EliminateSorts) :+
     Batch("Decimal Optimizations", fixedPoint,
       DecimalAggregates) :+
+    // This batch must run after "Decimal Optimizations", as that one may change the
+    // aggregate distinct column
+    Batch("Distinct Aggregate Rewrite", Once,
+      RewriteDistinctAggregates) :+
     Batch("Object Expressions Optimization", fixedPoint,
       EliminateMapObjects,
       CombineTypedFilters,
@@ -192,7 +221,10 @@ abstract class Optimizer(catalogManager: CatalogManager)
       ReassignLambdaVariableID) :+
     Batch("LocalRelation", fixedPoint,
       ConvertToLocalRelation,
-      PropagateEmptyRelation) :+
+      PropagateEmptyRelation,
+      // PropagateEmptyRelation can change the nullability of an attribute from nullable to
+      // non-nullable when an empty relation child of a Union is removed
+      UpdateAttributeNullability) :+
     // The following batch should be executed after batch "Join Reorder" and "LocalRelation".
     Batch("Check Cartesian Products", Once,
       CheckCartesianProducts) :+
@@ -202,7 +234,8 @@ abstract class Optimizer(catalogManager: CatalogManager)
       CollapseProject,
       RemoveNoopOperators) :+
     // This batch must be executed after the `RewriteSubquery` batch, which creates joins.
-    Batch("NormalizeFloatingNumbers", Once, NormalizeFloatingNumbers)
+    Batch("NormalizeFloatingNumbers", Once, NormalizeFloatingNumbers) :+
+    Batch("ReplaceUpdateFieldsExpression", Once, ReplaceUpdateFieldsExpression)
 
     // remove any batches with no rules. this may happen when subclasses do not add optional rules.
     batches.filter(_.rules.nonEmpty)
@@ -223,7 +256,7 @@ abstract class Optimizer(catalogManager: CatalogManager)
       EliminateView.ruleName ::
       ReplaceExpressions.ruleName ::
       ComputeCurrentTime.ruleName ::
-      GetCurrentDatabase(catalogManager).ruleName ::
+      GetCurrentDatabaseAndCatalog(catalogManager).ruleName ::
       RewriteDistinctAggregates.ruleName ::
       ReplaceDeduplicateWithAggregate.ruleName ::
       ReplaceIntersectWithSemiJoin.ruleName ::
@@ -235,7 +268,9 @@ abstract class Optimizer(catalogManager: CatalogManager)
       PullupCorrelatedPredicates.ruleName ::
       RewriteCorrelatedScalarSubquery.ruleName ::
       RewritePredicateSubquery.ruleName ::
-      NormalizeFloatingNumbers.ruleName :: Nil
+      NormalizeFloatingNumbers.ruleName ::
+      ReplaceUpdateFieldsExpression.ruleName ::
+      PullOutGroupingExpressions.ruleName :: Nil
 
   /**
    * Optimize all the subqueries inside expression.
@@ -248,7 +283,8 @@ abstract class Optimizer(catalogManager: CatalogManager)
         case other => other
       }
     }
-    def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions {
+    def apply(plan: LogicalPlan): LogicalPlan = plan.transformAllExpressionsWithPruning(
+      _.containsPattern(PLAN_EXPRESSION), ruleId) {
       case s: SubqueryExpression =>
         val Subquery(newPlan, _) = Optimizer.this.execute(Subquery.fromExpression(s))
         // At this point we have an optimized subquery plan that we are going to attach
@@ -268,6 +304,12 @@ abstract class Optimizer(catalogManager: CatalogManager)
    */
   def earlyScanPushDownRules: Seq[Rule[LogicalPlan]] = Nil
 
+  /**
+   * Override to provide additional rules for rewriting plans after operator optimization rules and
+   * before any cost-based optimization rules that depend on stats.
+   */
+  def preCBORules: Seq[Rule[LogicalPlan]] = Nil
+
   /**
    * Returns (defaultBatches - (excludedRules - nonExcludableRules)), the rule batches that
    * eventually run in the Optimizer.
@@ -317,11 +359,39 @@ abstract class Optimizer(catalogManager: CatalogManager)
  */
 object EliminateDistinct extends Rule[LogicalPlan] {
   override def apply(plan: LogicalPlan): LogicalPlan = plan transformExpressions  {
-    case ae: AggregateExpression if ae.isDistinct =>
-      ae.aggregateFunction match {
-        case _: Max | _: Min => ae.copy(isDistinct = false)
-        case _ => ae
-      }
+    case ae: AggregateExpression if ae.isDistinct && isDuplicateAgnostic(ae.aggregateFunction) =>
+      ae.copy(isDistinct = false)
+  }
+
+  private def isDuplicateAgnostic(af: AggregateFunction): Boolean = af match {
+    case _: Max => true
+    case _: Min => true
+    case _: BitAndAgg => true
+    case _: BitOrAgg => true
+    case _: CollectSet => true
+    case _ => false
+  }
+}
+
+/**
+ * Remove useless FILTER clause for aggregate expressions.
+ * This rule should be applied before RewriteDistinctAggregates.
+ */
+object EliminateAggregateFilter extends Rule[LogicalPlan] {
+  override def apply(plan: LogicalPlan): LogicalPlan = plan.transformExpressionsWithPruning(
+    _.containsAllPatterns(TRUE_OR_FALSE_LITERAL), ruleId)  {
+    case ae @ AggregateExpression(_, _, _, Some(Literal.TrueLiteral), _) =>
+      ae.copy(filter = None)
+    case AggregateExpression(af: DeclarativeAggregate, _, _, Some(Literal.FalseLiteral), _) =>
+      val initialProject = SafeProjection.create(af.initialValues)
+      val evalProject = SafeProjection.create(af.evaluateExpression :: Nil, af.aggBufferAttributes)
+      val initialBuffer = initialProject(EmptyRow)
+      val internalRow = evalProject(initialBuffer)
+      Literal.create(internalRow.get(0, af.dataType), af.dataType)
+    case AggregateExpression(af: ImperativeAggregate, _, _, Some(Literal.FalseLiteral), _) =>
+      val buffer = new SpecificInternalRow(af.aggBufferAttributes.map(_.dataType))
+      af.initialize(buffer)
+      Literal.create(af.eval(buffer), af.dataType)
   }
 }
 
@@ -335,9 +405,8 @@ object SimpleTestOptimizer extends SimpleTestOptimizer
 
 class SimpleTestOptimizer extends Optimizer(
   new CatalogManager(
-    new SQLConf().copy(SQLConf.CASE_SENSITIVE -> true),
     FakeV2SessionCatalog,
-    new SessionCatalog(new InMemoryCatalog, EmptyFunctionRegistry, new SQLConf())))
+    new SessionCatalog(new InMemoryCatalog, EmptyFunctionRegistry, EmptyTableFunctionRegistry)))
 
 /**
  * Remove redundant aliases from a query plan. A redundant alias is an alias that does not change
@@ -359,38 +428,41 @@ object RemoveRedundantAliases extends Rule[LogicalPlan] {
   /**
    * Remove the top-level alias from an expression when it is redundant.
    */
-  private def removeRedundantAlias(e: Expression, blacklist: AttributeSet): Expression = e match {
+  private def removeRedundantAlias(e: Expression, excludeList: AttributeSet): Expression = e match {
     // Alias with metadata can not be stripped, or the metadata will be lost.
     // If the alias name is different from attribute name, we can't strip it either, or we
     // may accidentally change the output schema name of the root plan.
     case a @ Alias(attr: Attribute, name)
-      if a.metadata == Metadata.empty &&
+      if (a.metadata == Metadata.empty || a.metadata == attr.metadata) &&
         name == attr.name &&
-        !blacklist.contains(attr) &&
-        !blacklist.contains(a) =>
+        !excludeList.contains(attr) &&
+        !excludeList.contains(a) =>
       attr
     case a => a
   }
 
   /**
-   * Remove redundant alias expression from a LogicalPlan and its subtree. A blacklist is used to
-   * prevent the removal of seemingly redundant aliases used to deduplicate the input for a (self)
-   * join or to prevent the removal of top-level subquery attributes.
+   * Remove redundant alias expression from a LogicalPlan and its subtree. A set of excludes is used
+   * to prevent the removal of seemingly redundant aliases used to deduplicate the input for a
+   * (self) join or to prevent the removal of top-level subquery attributes.
    */
-  private def removeRedundantAliases(plan: LogicalPlan, blacklist: AttributeSet): LogicalPlan = {
+  private def removeRedundantAliases(plan: LogicalPlan, excluded: AttributeSet): LogicalPlan = {
+    if (!plan.containsPattern(ALIAS)) {
+      return plan
+    }
     plan match {
       // We want to keep the same output attributes for subqueries. This means we cannot remove
       // the aliases that produce these attributes
       case Subquery(child, correlated) =>
-        Subquery(removeRedundantAliases(child, blacklist ++ child.outputSet), correlated)
+        Subquery(removeRedundantAliases(child, excluded ++ child.outputSet), correlated)
 
       // A join has to be treated differently, because the left and the right side of the join are
-      // not allowed to use the same attributes. We use a blacklist to prevent us from creating a
-      // situation in which this happens; the rule will only remove an alias if its child
+      // not allowed to use the same attributes. We use an exclude list to prevent us from creating
+      // a situation in which this happens; the rule will only remove an alias if its child
       // attribute is not on the black list.
       case Join(left, right, joinType, condition, hint) =>
-        val newLeft = removeRedundantAliases(left, blacklist ++ right.outputSet)
-        val newRight = removeRedundantAliases(right, blacklist ++ newLeft.outputSet)
+        val newLeft = removeRedundantAliases(left, excluded ++ right.outputSet)
+        val newRight = removeRedundantAliases(right, excluded ++ newLeft.outputSet)
         val mapping = AttributeMap(
           createAttributeMapping(left, newLeft) ++
           createAttributeMapping(right, newRight))
@@ -403,7 +475,7 @@ object RemoveRedundantAliases extends Rule[LogicalPlan] {
         // Remove redundant aliases in the subtree(s).
         val currentNextAttrPairs = mutable.Buffer.empty[(Attribute, Attribute)]
         val newNode = plan.mapChildren { child =>
-          val newChild = removeRedundantAliases(child, blacklist)
+          val newChild = removeRedundantAliases(child, excluded)
           currentNextAttrPairs ++= createAttributeMapping(child, newChild)
           newChild
         }
@@ -411,14 +483,14 @@ object RemoveRedundantAliases extends Rule[LogicalPlan] {
         // Create the attribute mapping. Note that the currentNextAttrPairs can contain duplicate
         // keys in case of Union (this is caused by the PushProjectionThroughUnion rule); in this
         // case we use the first mapping (which should be provided by the first child).
-        val mapping = AttributeMap(currentNextAttrPairs)
+        val mapping = AttributeMap(currentNextAttrPairs.toSeq)
 
         // Create a an expression cleaning function for nodes that can actually produce redundant
         // aliases, use identity otherwise.
         val clean: Expression => Expression = plan match {
-          case _: Project => removeRedundantAlias(_, blacklist)
-          case _: Aggregate => removeRedundantAlias(_, blacklist)
-          case _: Window => removeRedundantAlias(_, blacklist)
+          case _: Project => removeRedundantAlias(_, excluded)
+          case _: Aggregate => removeRedundantAlias(_, excluded)
+          case _: Window => removeRedundantAlias(_, excluded)
           case _ => identity[Expression]
         }
 
@@ -434,11 +506,53 @@ object RemoveRedundantAliases extends Rule[LogicalPlan] {
   def apply(plan: LogicalPlan): LogicalPlan = removeRedundantAliases(plan, AttributeSet.empty)
 }
 
+/**
+ * Remove redundant aggregates from a query plan. A redundant aggregate is an aggregate whose
+ * only goal is to keep distinct values, while its parent aggregate would ignore duplicate values.
+ */
+object RemoveRedundantAggregates extends Rule[LogicalPlan] with AliasHelper {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformUpWithPruning(
+    _.containsPattern(AGGREGATE), ruleId) {
+    case upper @ Aggregate(_, _, lower: Aggregate) if lowerIsRedundant(upper, lower) =>
+      val aliasMap = getAliasMap(lower)
+
+      val newAggregate = upper.copy(
+        child = lower.child,
+        groupingExpressions = upper.groupingExpressions.map(replaceAlias(_, aliasMap)),
+        aggregateExpressions = upper.aggregateExpressions.map(
+          replaceAliasButKeepName(_, aliasMap))
+      )
+
+      // We might have introduces non-deterministic grouping expression
+      if (newAggregate.groupingExpressions.exists(!_.deterministic)) {
+        PullOutNondeterministic.applyLocally.applyOrElse(newAggregate, identity[LogicalPlan])
+      } else {
+        newAggregate
+      }
+  }
+
+  private def lowerIsRedundant(upper: Aggregate, lower: Aggregate): Boolean = {
+    val upperHasNoAggregateExpressions =
+      !upper.aggregateExpressions.exists(AggregateExpression.containsAggregate)
+
+    lazy val upperRefsOnlyDeterministicNonAgg = upper.references.subsetOf(AttributeSet(
+      lower
+        .aggregateExpressions
+        .filter(_.deterministic)
+        .filterNot(AggregateExpression.containsAggregate)
+        .map(_.toAttribute)
+    ))
+
+    upperHasNoAggregateExpressions && upperRefsOnlyDeterministicNonAgg
+  }
+}
+
 /**
  * Remove no-op operators from the query plan that do not make any modifications.
  */
 object RemoveNoopOperators extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformUpWithPruning(
+    _.containsAnyPattern(PROJECT, WINDOW), ruleId) {
     // Eliminate no-op Projects
     case p @ Project(_, child) if child.sameOutput(p) => child
 
@@ -448,7 +562,59 @@ object RemoveNoopOperators extends Rule[LogicalPlan] {
 }
 
 /**
- * Pushes down [[LocalLimit]] beneath UNION ALL and beneath the streamed inputs of outer joins.
+ * Smplify the children of `Union` or remove no-op `Union` from the query plan that
+ * do not make any modifications to the query.
+ */
+object RemoveNoopUnion extends Rule[LogicalPlan] {
+  /**
+   * This only removes the `Project` that has only attributes or aliased attributes
+   * from its child.
+   */
+  private def removeAliasOnlyProject(plan: LogicalPlan): LogicalPlan = plan match {
+    case p @ Project(projectList, child) =>
+      val aliasOnly = projectList.length == child.output.length &&
+        projectList.zip(child.output).forall {
+          case (Alias(left: Attribute, _), right) => left.semanticEquals(right)
+          case (left: Attribute, right) => left.semanticEquals(right)
+          case _ => false
+        }
+      if (aliasOnly) {
+        child
+      } else {
+        p
+      }
+    case _ => plan
+  }
+
+  private def simplifyUnion(u: Union): LogicalPlan = {
+    val uniqueChildren = mutable.ArrayBuffer.empty[LogicalPlan]
+    val uniqueChildrenKey = mutable.HashSet.empty[LogicalPlan]
+
+    u.children.foreach { c =>
+      val key = removeAliasOnlyProject(c).canonicalized
+      if (!uniqueChildrenKey.contains(key)) {
+        uniqueChildren += c
+        uniqueChildrenKey += key
+      }
+    }
+    if (uniqueChildren.size == 1) {
+      u.children.head
+    } else {
+      u.copy(children = uniqueChildren.toSeq)
+    }
+  }
+
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformUpWithPruning(
+    _.containsAllPatterns(DISTINCT_LIKE, UNION)) {
+    case d @ Distinct(u: Union) =>
+      d.withNewChildren(Seq(simplifyUnion(u)))
+    case d @ Deduplicate(_, u: Union) =>
+      d.withNewChildren(Seq(simplifyUnion(u)))
+  }
+}
+
+/**
+ * Pushes down [[LocalLimit]] beneath UNION ALL and joins.
  */
 object LimitPushDown extends Rule[LogicalPlan] {
 
@@ -476,29 +642,48 @@ object LimitPushDown extends Rule[LogicalPlan] {
     }
   }
 
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  private def pushLocalLimitThroughJoin(limitExpr: Expression, join: Join): Join = {
+    join.joinType match {
+      case RightOuter => join.copy(right = maybePushLocalLimit(limitExpr, join.right))
+      case LeftOuter => join.copy(left = maybePushLocalLimit(limitExpr, join.left))
+      case _: InnerLike if join.condition.isEmpty =>
+        join.copy(
+          left = maybePushLocalLimit(limitExpr, join.left),
+          right = maybePushLocalLimit(limitExpr, join.right))
+      case LeftSemi | LeftAnti if join.condition.isEmpty =>
+        join.copy(left = maybePushLocalLimit(limitExpr, join.left))
+      case _ => join
+    }
+  }
+
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsPattern(LIMIT), ruleId) {
     // Adding extra Limits below UNION ALL for children which are not Limit or do not have Limit
     // descendants whose maxRow is larger. This heuristic is valid assuming there does not exist any
     // Limit push-down rule that is unable to infer the value of maxRows.
     // Note: right now Union means UNION ALL, which does not de-duplicate rows, so it is safe to
     // pushdown Limit through it. Once we add UNION DISTINCT, however, we will not be able to
     // pushdown Limit.
-    case LocalLimit(exp, Union(children)) =>
-      LocalLimit(exp, Union(children.map(maybePushLocalLimit(exp, _))))
-    // Add extra limits below OUTER JOIN. For LEFT OUTER and RIGHT OUTER JOIN we push limits to
-    // the left and right sides, respectively. It's not safe to push limits below FULL OUTER
-    // JOIN in the general case without a more invasive rewrite.
-    // We also need to ensure that this limit pushdown rule will not eventually introduce limits
-    // on both sides if it is applied multiple times. Therefore:
+    case LocalLimit(exp, u: Union) =>
+      LocalLimit(exp, u.copy(children = u.children.map(maybePushLocalLimit(exp, _))))
+
+    // Add extra limits below JOIN:
+    // 1. For LEFT OUTER and RIGHT OUTER JOIN, we push limits to the left and right sides,
+    //    respectively.
+    // 2. For INNER and CROSS JOIN, we push limits to both the left and right sides if join
+    //    condition is empty.
+    // 3. For LEFT SEMI and LEFT ANTI JOIN, we push limits to the left side if join condition
+    //    is empty.
+    // It's not safe to push limits below FULL OUTER JOIN in the general case without a more
+    // invasive rewrite. We also need to ensure that this limit pushdown rule will not eventually
+    // introduce limits on both sides if it is applied multiple times. Therefore:
     //   - If one side is already limited, stack another limit on top if the new limit is smaller.
     //     The redundant limit will be collapsed by the CombineLimits rule.
-    case LocalLimit(exp, join @ Join(left, right, joinType, _, _)) =>
-      val newJoin = joinType match {
-        case RightOuter => join.copy(right = maybePushLocalLimit(exp, right))
-        case LeftOuter => join.copy(left = maybePushLocalLimit(exp, left))
-        case _ => join
-      }
-      LocalLimit(exp, newJoin)
+    case LocalLimit(exp, join: Join) =>
+      LocalLimit(exp, pushLocalLimitThroughJoin(exp, join))
+    // There is a Project between LocalLimit and Join if they do not have the same output.
+    case LocalLimit(exp, project @ Project(_, join: Join)) =>
+      LocalLimit(exp, project.copy(child = pushLocalLimitThroughJoin(exp, join)))
   }
 }
 
@@ -542,15 +727,15 @@ object PushProjectionThroughUnion extends Rule[LogicalPlan] with PredicateHelper
   def apply(plan: LogicalPlan): LogicalPlan = plan transform {
 
     // Push down deterministic projection through UNION ALL
-    case p @ Project(projectList, Union(children)) =>
-      assert(children.nonEmpty)
+    case p @ Project(projectList, u: Union) =>
+      assert(u.children.nonEmpty)
       if (projectList.forall(_.deterministic)) {
-        val newFirstChild = Project(projectList, children.head)
-        val newOtherChildren = children.tail.map { child =>
-          val rewrites = buildRewrites(children.head, child)
+        val newFirstChild = Project(projectList, u.children.head)
+        val newOtherChildren = u.children.tail.map { child =>
+          val rewrites = buildRewrites(u.children.head, child)
           Project(projectList.map(pushToRight(_, rewrites)), child)
         }
-        Union(newFirstChild +: newOtherChildren)
+        u.copy(children = newFirstChild +: newOtherChildren)
       } else {
         p
       }
@@ -569,7 +754,8 @@ object PushProjectionThroughUnion extends Rule[LogicalPlan] with PredicateHelper
  */
 object ColumnPruning extends Rule[LogicalPlan] {
 
-  def apply(plan: LogicalPlan): LogicalPlan = removeProjectBeforeFilter(plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = removeProjectBeforeFilter(
+    plan.transformWithPruning(AlwaysProcess.fn, ruleId) {
     // Prunes the unused columns from project list of Project/Aggregate/Expand
     case p @ Project(_, p2: Project) if !p2.outputSet.subsetOf(p.references) =>
       p.copy(child = p2.copy(projectList = p2.projectList.filter(p.references.contains)))
@@ -596,9 +782,6 @@ object ColumnPruning extends Rule[LogicalPlan] {
       f.copy(child = prunedChild(child, f.references))
     case e @ Expand(_, _, child) if !child.outputSet.subsetOf(e.references) =>
       e.copy(child = prunedChild(child, e.references))
-    case s @ ScriptTransformation(_, _, _, child, _)
-        if !child.outputSet.subsetOf(s.references) =>
-      s.copy(child = prunedChild(child, s.references))
 
     // prune unrequired references
     case p @ Project(_, g: Generate) if p.references != g.outputSet =>
@@ -644,8 +827,7 @@ object ColumnPruning extends Rule[LogicalPlan] {
     // Can't prune the columns on LeafNode
     case p @ Project(_, _: LeafNode) => p
 
-    case p @ NestedColumnAliasing(nestedFieldToAlias, attrToAliases) =>
-      NestedColumnAliasing.replaceToAliases(p, nestedFieldToAlias, attrToAliases)
+    case NestedColumnAliasing(p) => p
 
     // for all other logical plans that inherits the output from it's children
     // Project over project is handled by the first case, skip it here.
@@ -689,9 +871,10 @@ object ColumnPruning extends Rule[LogicalPlan] {
  *    and the upper project consists of the same number of columns which is equal or aliasing.
  *    `GlobalLimit(LocalLimit)` pattern is also considered.
  */
-object CollapseProject extends Rule[LogicalPlan] {
+object CollapseProject extends Rule[LogicalPlan] with AliasHelper {
 
-  def apply(plan: LogicalPlan): LogicalPlan = plan transformUp {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformUpWithPruning(
+    _.containsPattern(PROJECT), ruleId) {
     case p1 @ Project(_, p2: Project) =>
       if (haveCommonNonDeterministicOutput(p1.projectList, p2.projectList)) {
         p1
@@ -718,17 +901,9 @@ object CollapseProject extends Rule[LogicalPlan] {
       s.copy(child = p2.copy(projectList = buildCleanedProjectList(l1, p2.projectList)))
   }
 
-  private def collectAliases(projectList: Seq[NamedExpression]): AttributeMap[Alias] = {
-    AttributeMap(projectList.collect {
-      case a: Alias => a.toAttribute -> a
-    })
-  }
-
   private def haveCommonNonDeterministicOutput(
       upper: Seq[NamedExpression], lower: Seq[NamedExpression]): Boolean = {
-    // Create a map of Aliases to their values from the lower projection.
-    // e.g., 'SELECT ... FROM (SELECT a + b AS c, d ...)' produces Map(c -> Alias(a + b, c)).
-    val aliases = collectAliases(lower)
+    val aliases = getAliasMap(lower)
 
     // Collapse upper and lower Projects if and only if their overlapped expressions are all
     // deterministic.
@@ -740,21 +915,8 @@ object CollapseProject extends Rule[LogicalPlan] {
   private def buildCleanedProjectList(
       upper: Seq[NamedExpression],
       lower: Seq[NamedExpression]): Seq[NamedExpression] = {
-    // Create a map of Aliases to their values from the lower projection.
-    // e.g., 'SELECT ... FROM (SELECT a + b AS c, d ...)' produces Map(c -> Alias(a + b, c)).
-    val aliases = collectAliases(lower)
-
-    // Substitute any attributes that are produced by the lower projection, so that we safely
-    // eliminate it.
-    // e.g., 'SELECT c + 1 FROM (SELECT a + b AS C ...' produces 'SELECT a + b + 1 ...'
-    // Use transformUp to prevent infinite recursion.
-    val rewrittenUpper = upper.map(_.transformUp {
-      case a: Attribute => aliases.getOrElse(a, a)
-    })
-    // collapse upper and lower Projects may introduce unnecessary Aliases, trim them here.
-    rewrittenUpper.map { p =>
-      CleanupAliases.trimNonTopLevelAliases(p).asInstanceOf[NamedExpression]
-    }
+    val aliases = getAliasMap(lower)
+    upper.map(replaceAliasButKeepName(_, aliases))
   }
 
   private def isRenaming(list1: Seq[NamedExpression], list2: Seq[NamedExpression]): Boolean = {
@@ -770,7 +932,8 @@ object CollapseProject extends Rule[LogicalPlan] {
  * Combines adjacent [[RepartitionOperation]] operators
  */
 object CollapseRepartition extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transformUp {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformUpWithPruning(
+    _.containsPattern(REPARTITION_OPERATION), ruleId) {
     // Case 1: When a Repartition has a child of Repartition or RepartitionByExpression,
     // 1) When the top node does not enable the shuffle (i.e., coalesce API), but the child
     //   enables the shuffle. Returns the child node if the last numPartitions is bigger;
@@ -787,13 +950,44 @@ object CollapseRepartition extends Rule[LogicalPlan] {
   }
 }
 
+/**
+ * Replace RepartitionByExpression numPartitions to 1 if all partition expressions are foldable
+ * and user not specify.
+ */
+object OptimizeRepartition extends Rule[LogicalPlan] {
+  override def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsPattern(REPARTITION_OPERATION), ruleId) {
+    case r @ RepartitionByExpression(partitionExpressions, _, numPartitions)
+      if partitionExpressions.nonEmpty && partitionExpressions.forall(_.foldable) &&
+        numPartitions.isEmpty =>
+      r.copy(optNumPartitions = Some(1))
+  }
+}
+
+/**
+ * Replaces first(col) to nth_value(col, 1) for better performance.
+ */
+object OptimizeWindowFunctions extends Rule[LogicalPlan] {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.resolveExpressionsWithPruning(
+    _.containsPattern(WINDOW_EXPRESSION), ruleId) {
+    case we @ WindowExpression(AggregateExpression(first: First, _, _, _, _),
+        WindowSpecDefinition(_, orderSpec, frameSpecification: SpecifiedWindowFrame))
+        if orderSpec.nonEmpty && frameSpecification.frameType == RowFrame &&
+          frameSpecification.lower == UnboundedPreceding &&
+          (frameSpecification.upper == UnboundedFollowing ||
+            frameSpecification.upper == CurrentRow) =>
+      we.copy(windowFunction = NthValue(first.child, Literal(1), first.ignoreNulls))
+  }
+}
+
 /**
  * Collapse Adjacent Window Expression.
  * - If the partition specs and order specs are the same and the window expression are
  *   independent and are of the same window function type, collapse into the parent.
  */
 object CollapseWindow extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transformUp {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformUpWithPruning(
+    _.containsPattern(WINDOW), ruleId) {
     case w1 @ Window(we1, ps1, os1, w2 @ Window(we2, ps2, os2, grandChild))
         if ps1 == ps2 && os1 == os2 && w1.references.intersect(w2.windowOutputSet).isEmpty &&
           we1.nonEmpty && we2.nonEmpty &&
@@ -810,22 +1004,59 @@ object CollapseWindow extends Rule[LogicalPlan] {
  *   of the child window expression, transpose them.
  */
 object TransposeWindow extends Rule[LogicalPlan] {
-  private def compatibleParititions(ps1 : Seq[Expression], ps2: Seq[Expression]): Boolean = {
+  private def compatiblePartitions(ps1 : Seq[Expression], ps2: Seq[Expression]): Boolean = {
     ps1.length < ps2.length && ps2.take(ps1.length).permutations.exists(ps1.zip(_).forall {
       case (l, r) => l.semanticEquals(r)
     })
   }
 
-  def apply(plan: LogicalPlan): LogicalPlan = plan transformUp {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformUpWithPruning(
+    _.containsPattern(WINDOW), ruleId) {
     case w1 @ Window(we1, ps1, os1, w2 @ Window(we2, ps2, os2, grandChild))
         if w1.references.intersect(w2.windowOutputSet).isEmpty &&
            w1.expressions.forall(_.deterministic) &&
            w2.expressions.forall(_.deterministic) &&
-           compatibleParititions(ps1, ps2) =>
+           compatiblePartitions(ps1, ps2) =>
       Project(w1.output, Window(we2, ps2, os2, Window(we1, ps1, os1, grandChild)))
   }
 }
 
+/**
+ * Infers filters from [[Generate]], such that rows that would have been removed
+ * by this [[Generate]] can be removed earlier - before joins and in data sources.
+ */
+object InferFiltersFromGenerate extends Rule[LogicalPlan] {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformUpWithPruning(
+    _.containsPattern(GENERATE)) {
+    // This rule does not infer filters from foldable expressions to avoid constant filters
+    // like 'size([1, 2, 3]) > 0'. These do not show up in child's constraints and
+    // then the idempotence will break.
+    case generate @ Generate(e, _, _, _, _, _)
+      if !e.deterministic || e.children.forall(_.foldable) => generate
+
+    case generate @ Generate(g, _, false, _, _, _) if canInferFilters(g) =>
+      // Exclude child's constraints to guarantee idempotency
+      val inferredFilters = ExpressionSet(
+        Seq(
+          GreaterThan(Size(g.children.head), Literal(0)),
+          IsNotNull(g.children.head)
+        )
+      ) -- generate.child.constraints
+
+      if (inferredFilters.nonEmpty) {
+        generate.copy(child = Filter(inferredFilters.reduce(And), generate.child))
+      } else {
+        generate
+      }
+  }
+
+  private def canInferFilters(g: Generator): Boolean = g match {
+    case _: ExplodeBase => true
+    case _: Inline => true
+    case _ => false
+  }
+}
+
 /**
  * Generate a list of additional filters from an operator's existing constraint but remove those
  * that are either already part of the operator's condition or are part of the operator's child
@@ -839,14 +1070,15 @@ object InferFiltersFromConstraints extends Rule[LogicalPlan]
   with PredicateHelper with ConstraintHelper {
 
   def apply(plan: LogicalPlan): LogicalPlan = {
-    if (SQLConf.get.constraintPropagationEnabled) {
+    if (conf.constraintPropagationEnabled) {
       inferFilters(plan)
     } else {
       plan
     }
   }
 
-  private def inferFilters(plan: LogicalPlan): LogicalPlan = plan transform {
+  private def inferFilters(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsAnyPattern(FILTER, JOIN)) {
     case filter @ Filter(condition, child) =>
       val newFilters = filter.constraints --
         (child.constraints ++ splitConjunctivePredicates(condition))
@@ -885,13 +1117,13 @@ object InferFiltersFromConstraints extends Rule[LogicalPlan]
   private def getAllConstraints(
       left: LogicalPlan,
       right: LogicalPlan,
-      conditionOpt: Option[Expression]): Set[Expression] = {
+      conditionOpt: Option[Expression]): ExpressionSet = {
     val baseConstraints = left.constraints.union(right.constraints)
-      .union(conditionOpt.map(splitConjunctivePredicates).getOrElse(Nil).toSet)
+      .union(ExpressionSet(conditionOpt.map(splitConjunctivePredicates).getOrElse(Nil)))
     baseConstraints.union(inferAdditionalConstraints(baseConstraints))
   }
 
-  private def inferNewFilter(plan: LogicalPlan, constraints: Set[Expression]): LogicalPlan = {
+  private def inferNewFilter(plan: LogicalPlan, constraints: ExpressionSet): LogicalPlan = {
     val newPredicates = constraints
       .union(constructIsNotNullConstraints(constraints, plan.output))
       .filter { c =>
@@ -909,25 +1141,43 @@ object InferFiltersFromConstraints extends Rule[LogicalPlan]
  * Combines all adjacent [[Union]] operators into a single [[Union]].
  */
 object CombineUnions extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transformDown {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformDownWithPruning(
+    _.containsAnyPattern(UNION, DISTINCT_LIKE), ruleId) {
     case u: Union => flattenUnion(u, false)
     case Distinct(u: Union) => Distinct(flattenUnion(u, true))
+    // Only handle distinct-like 'Deduplicate', where the keys == output
+    case Deduplicate(keys: Seq[Attribute], u: Union) if AttributeSet(keys) == u.outputSet =>
+      Deduplicate(keys, flattenUnion(u, true))
   }
 
   private def flattenUnion(union: Union, flattenDistinct: Boolean): Union = {
+    val topByName = union.byName
+    val topAllowMissingCol = union.allowMissingCol
+
     val stack = mutable.Stack[LogicalPlan](union)
     val flattened = mutable.ArrayBuffer.empty[LogicalPlan]
+    // Note that we should only flatten the unions with same byName and allowMissingCol.
+    // Although we do `UnionCoercion` at analysis phase, we manually run `CombineUnions`
+    // in some places like `Dataset.union`. Flattening unions with different resolution
+    // rules (by position and by name) could cause incorrect results.
     while (stack.nonEmpty) {
       stack.pop() match {
-        case Distinct(Union(children)) if flattenDistinct =>
+        case Distinct(Union(children, byName, allowMissingCol))
+            if flattenDistinct && byName == topByName && allowMissingCol == topAllowMissingCol =>
           stack.pushAll(children.reverse)
-        case Union(children) =>
+        // Only handle distinct-like 'Deduplicate', where the keys == output
+        case Deduplicate(keys: Seq[Attribute], u: Union)
+            if flattenDistinct && u.byName == topByName &&
+              u.allowMissingCol == topAllowMissingCol && AttributeSet(keys) == u.outputSet =>
+          stack.pushAll(u.children.reverse)
+        case Union(children, byName, allowMissingCol)
+            if byName == topByName && allowMissingCol == topAllowMissingCol =>
           stack.pushAll(children.reverse)
         case child =>
           flattened += child
       }
     }
-    Union(flattened)
+    union.copy(children = flattened.toSeq)
   }
 }
 
@@ -936,39 +1186,55 @@ object CombineUnions extends Rule[LogicalPlan] {
  * one conjunctive predicate.
  */
 object CombineFilters extends Rule[LogicalPlan] with PredicateHelper {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform applyLocally
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsPattern(FILTER), ruleId)(applyLocally)
 
   val applyLocally: PartialFunction[LogicalPlan, LogicalPlan] = {
     // The query execution/optimization does not guarantee the expressions are evaluated in order.
     // We only can combine them if and only if both are deterministic.
-    case Filter(fc, nf @ Filter(nc, grandChild)) if fc.deterministic && nc.deterministic =>
-      (ExpressionSet(splitConjunctivePredicates(fc)) --
+    case Filter(fc, nf @ Filter(nc, grandChild)) if nc.deterministic =>
+      val (combineCandidates, nonDeterministic) =
+        splitConjunctivePredicates(fc).partition(_.deterministic)
+      val mergedFilter = (ExpressionSet(combineCandidates) --
         ExpressionSet(splitConjunctivePredicates(nc))).reduceOption(And) match {
         case Some(ac) =>
           Filter(And(nc, ac), grandChild)
         case None =>
           nf
       }
+      nonDeterministic.reduceOption(And).map(c => Filter(c, mergedFilter)).getOrElse(mergedFilter)
   }
 }
 
 /**
- * Removes Sort operation. This can happen:
+ * Removes Sort operations if they don't affect the final output ordering.
+ * Note that changes in the final output ordering may affect the file size (SPARK-32318).
+ * This rule handles the following cases:
  * 1) if the sort order is empty or the sort order does not have any reference
- * 2) if the child is already sorted
- * 3) if there is another Sort operator separated by 0...n Project/Filter operators
- * 4) if the Sort operator is within Join separated by 0...n Project/Filter operators only,
- *    and the Join conditions is deterministic
- * 5) if the Sort operator is within GroupBy separated by 0...n Project/Filter operators only,
- *    and the aggregate function is order irrelevant
+ * 2) if the Sort operator is a local sort and the child is already sorted
+ * 3) if there is another Sort operator separated by 0...n Project, Filter, Repartition or
+ *    RepartitionByExpression (with deterministic expressions) operators
+ * 4) if the Sort operator is within Join separated by 0...n Project, Filter, Repartition or
+ *    RepartitionByExpression (with deterministic expressions) operators only and the Join condition
+ *    is deterministic
+ * 5) if the Sort operator is within GroupBy separated by 0...n Project, Filter, Repartition or
+ *    RepartitionByExpression (with deterministic expressions) operators only and the aggregate
+ *    function is order irrelevant
  */
 object EliminateSorts extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsPattern(SORT))(applyLocally)
+
+  private val applyLocally: PartialFunction[LogicalPlan, LogicalPlan] = {
     case s @ Sort(orders, _, child) if orders.isEmpty || orders.exists(_.child.foldable) =>
       val newOrders = orders.filterNot(_.child.foldable)
-      if (newOrders.isEmpty) child else s.copy(order = newOrders)
-    case Sort(orders, true, child) if SortOrder.orderingSatisfies(child.outputOrdering, orders) =>
-      child
+      if (newOrders.isEmpty) {
+        applyLocally.lift(child).getOrElse(child)
+      } else {
+        s.copy(order = newOrders)
+      }
+    case Sort(orders, false, child) if SortOrder.orderingSatisfies(child.outputOrdering, orders) =>
+      applyLocally.lift(child).getOrElse(child)
     case s @ Sort(_, _, child) => s.copy(child = recursiveRemoveSort(child))
     case j @ Join(originLeft, originRight, _, cond, _) if cond.forall(_.deterministic) =>
       j.copy(left = recursiveRemoveSort(originLeft), right = recursiveRemoveSort(originRight))
@@ -976,22 +1242,29 @@ object EliminateSorts extends Rule[LogicalPlan] {
       g.copy(child = recursiveRemoveSort(originChild))
   }
 
-  private def recursiveRemoveSort(plan: LogicalPlan): LogicalPlan = plan match {
-    case Sort(_, _, child) => recursiveRemoveSort(child)
-    case other if canEliminateSort(other) =>
-      other.withNewChildren(other.children.map(recursiveRemoveSort))
-    case _ => plan
+  private def recursiveRemoveSort(plan: LogicalPlan): LogicalPlan = {
+    if (!plan.containsPattern(SORT)) {
+      return plan
+    }
+    plan match {
+      case Sort(_, _, child) => recursiveRemoveSort(child)
+      case other if canEliminateSort(other) =>
+        other.withNewChildren(other.children.map(recursiveRemoveSort))
+      case _ => plan
+    }
   }
 
   private def canEliminateSort(plan: LogicalPlan): Boolean = plan match {
     case p: Project => p.projectList.forall(_.deterministic)
     case f: Filter => f.condition.deterministic
+    case r: RepartitionByExpression => r.partitionExpressions.forall(_.deterministic)
+    case _: Repartition => true
     case _ => false
   }
 
   private def isOrderIrrelevantAggs(aggs: Seq[NamedExpression]): Boolean = {
     def isOrderIrrelevantAggFunction(func: AggregateFunction): Boolean = func match {
-      case _: Min | _: Max | _: Count => true
+      case _: Min | _: Max | _: Count | _: BitAggregate => true
       // Arithmetic operations for floating-point values are order-sensitive
       // (they are not associative).
       case _: Sum | _: Average | _: CentralMomentAgg =>
@@ -1017,7 +1290,8 @@ object EliminateSorts extends Rule[LogicalPlan] {
  * 3) by eliminating the always-true conditions given the constraints on the child's output.
  */
 object PruneFilters extends Rule[LogicalPlan] with PredicateHelper {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsPattern(FILTER), ruleId) {
     // If the filter condition always evaluate to true, remove the filter.
     case Filter(Literal(true, BooleanType), child) => child
     // If the filter condition always evaluate to null or false,
@@ -1180,23 +1454,6 @@ object PushPredicateThroughNonJoin extends Rule[LogicalPlan] with PredicateHelpe
       }
   }
 
-  def getAliasMap(plan: Project): AttributeMap[Expression] = {
-    // Create a map of Aliases to their values from the child projection.
-    // e.g., 'SELECT a + b AS c, d ...' produces Map(c -> a + b).
-    AttributeMap(plan.projectList.collect { case a: Alias => (a.toAttribute, a.child) })
-  }
-
-  def getAliasMap(plan: Aggregate): AttributeMap[Expression] = {
-    // Find all the aliased expressions in the aggregate list that don't include any actual
-    // AggregateExpression or PythonUDF, and create a map from the alias to the expression
-    val aliasMap = plan.aggregateExpressions.collect {
-      case a: Alias if a.child.find(e => e.isInstanceOf[AggregateExpression] ||
-          PythonUDF.isGroupedAggPandasUDF(e)).isEmpty =>
-        (a.toAttribute, a.child)
-    }
-    AttributeMap(aliasMap)
-  }
-
   def canPushThrough(p: UnaryNode): Boolean = p match {
     // Note that some operators (e.g. project, aggregate, union) are being handled separately
     // (earlier in this rule).
@@ -1210,6 +1467,7 @@ object PushPredicateThroughNonJoin extends Rule[LogicalPlan] with PredicateHelpe
     case _: Sort => true
     case _: BatchEvalPython => true
     case _: ArrowEvalPython => true
+    case _: Expand => true
     case _ => false
   }
 
@@ -1285,11 +1543,17 @@ object PushPredicateThroughJoin extends Rule[LogicalPlan] with PredicateHelper {
     (leftEvaluateCondition, rightEvaluateCondition, commonCondition ++ nonDeterministic)
   }
 
+  private def canPushThrough(joinType: JoinType): Boolean = joinType match {
+    case _: InnerLike | LeftSemi | RightOuter | LeftOuter | LeftAnti | ExistenceJoin(_) => true
+    case _ => false
+  }
+
   def apply(plan: LogicalPlan): LogicalPlan = plan transform applyLocally
 
   val applyLocally: PartialFunction[LogicalPlan, LogicalPlan] = {
     // push the where condition down into join filter
-    case f @ Filter(filterCondition, Join(left, right, joinType, joinCondition, hint)) =>
+    case f @ Filter(filterCondition, Join(left, right, joinType, joinCondition, hint))
+        if canPushThrough(joinType) =>
       val (leftFilterConditions, rightFilterConditions, commonFilterCondition) =
         split(splitConjunctivePredicates(filterCondition), left, right)
       joinType match {
@@ -1329,13 +1593,13 @@ object PushPredicateThroughJoin extends Rule[LogicalPlan] with PredicateHelper {
 
           (rightFilterConditions ++ commonFilterCondition).
             reduceLeftOption(And).map(Filter(_, newJoin)).getOrElse(newJoin)
-        case FullOuter => f // DO Nothing for Full Outer Join
-        case NaturalJoin(_) => sys.error("Untransformed NaturalJoin node")
-        case UsingJoin(_, _) => sys.error("Untransformed Using join node")
+
+        case other =>
+          throw new IllegalStateException(s"Unexpected join type: $other")
       }
 
     // push down the join filter into sub query scanning if applicable
-    case j @ Join(left, right, joinType, joinCondition, hint) =>
+    case j @ Join(left, right, joinType, joinCondition, hint) if canPushThrough(joinType) =>
       val (leftJoinConditions, rightJoinConditions, commonJoinCondition) =
         split(joinCondition.map(splitConjunctivePredicates).getOrElse(Nil), left, right)
 
@@ -1365,19 +1629,31 @@ object PushPredicateThroughJoin extends Rule[LogicalPlan] with PredicateHelper {
           val newJoinCond = (leftJoinConditions ++ commonJoinCondition).reduceLeftOption(And)
 
           Join(newLeft, newRight, joinType, newJoinCond, hint)
-        case FullOuter => j
-        case NaturalJoin(_) => sys.error("Untransformed NaturalJoin node")
-        case UsingJoin(_, _) => sys.error("Untransformed Using join node")
+
+        case other =>
+          throw new IllegalStateException(s"Unexpected join type: $other")
       }
   }
 }
 
 /**
- * Combines two adjacent [[Limit]] operators into one, merging the
- * expressions into one single expression.
+ * This rule optimizes Limit operators by:
+ * 1. Eliminate [[Limit]]/[[GlobalLimit]] operators if it's child max row <= limit.
+ * 2. Combines two adjacent [[Limit]] operators into one, merging the
+ *    expressions into one single expression.
  */
-object CombineLimits extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+object EliminateLimits extends Rule[LogicalPlan] {
+  private def canEliminate(limitExpr: Expression, child: LogicalPlan): Boolean = {
+    limitExpr.foldable && child.maxRows.exists { _ <= limitExpr.eval().asInstanceOf[Int] }
+  }
+
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformDownWithPruning(
+    _.containsPattern(LIMIT), ruleId) {
+    case Limit(l, child) if canEliminate(l, child) =>
+      child
+    case GlobalLimit(l, child) if canEliminate(l, child) =>
+      child
+
     case GlobalLimit(le, GlobalLimit(ne, grandChild)) =>
       GlobalLimit(Least(Seq(ne, le)), grandChild)
     case LocalLimit(le, LocalLimit(ne, grandChild)) =>
@@ -1417,9 +1693,9 @@ object CheckCartesianProducts extends Rule[LogicalPlan] with PredicateHelper {
   }
 
   def apply(plan: LogicalPlan): LogicalPlan =
-    if (SQLConf.get.crossJoinEnabled) {
+    if (conf.crossJoinEnabled) {
       plan
-    } else plan transform {
+    } else plan.transformWithPruning(_.containsAnyPattern(INNER_LIKE_JOIN, OUTER_JOIN))  {
       case j @ Join(left, right, Inner | LeftOuter | RightOuter | FullOuter, _, _)
         if isCartesianProduct(j) =>
           throw new AnalysisException(
@@ -1447,8 +1723,10 @@ object DecimalAggregates extends Rule[LogicalPlan] {
   /** Maximum number of decimal digits representable precisely in a Double */
   private val MAX_DOUBLE_DIGITS = 15
 
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
-    case q: LogicalPlan => q transformExpressionsDown {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsAnyPattern(SUM, AVERAGE), ruleId) {
+    case q: LogicalPlan => q.transformExpressionsDownWithPruning(
+      _.containsAnyPattern(SUM, AVERAGE), ruleId) {
       case we @ WindowExpression(ae @ AggregateExpression(af, _, _, _, _), _) => af match {
         case Sum(e @ DecimalType.Expression(prec, scale)) if prec + 10 <= MAX_LONG_DIGITS =>
           MakeDecimal(we.copy(windowFunction = ae.copy(aggregateFunction = Sum(UnscaledValue(e)))),
@@ -1459,7 +1737,7 @@ object DecimalAggregates extends Rule[LogicalPlan] {
             we.copy(windowFunction = ae.copy(aggregateFunction = Average(UnscaledValue(e))))
           Cast(
             Divide(newAggExpr, Literal.create(math.pow(10.0, scale), DoubleType)),
-            DecimalType(prec + 4, scale + 4), Option(SQLConf.get.sessionLocalTimeZone))
+            DecimalType(prec + 4, scale + 4), Option(conf.sessionLocalTimeZone))
 
         case _ => we
       }
@@ -1471,7 +1749,7 @@ object DecimalAggregates extends Rule[LogicalPlan] {
           val newAggExpr = ae.copy(aggregateFunction = Average(UnscaledValue(e)))
           Cast(
             Divide(newAggExpr, Literal.create(math.pow(10.0, scale), DoubleType)),
-            DecimalType(prec + 4, scale + 4), Option(SQLConf.get.sessionLocalTimeZone))
+            DecimalType(prec + 4, scale + 4), Option(conf.sessionLocalTimeZone))
 
         case _ => ae
       }
@@ -1484,7 +1762,8 @@ object DecimalAggregates extends Rule[LogicalPlan] {
  * another `LocalRelation`.
  */
 object ConvertToLocalRelation extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsPattern(LOCAL_RELATION), ruleId) {
     case Project(projectList, LocalRelation(output, data, isStreaming))
         if !projectList.exists(hasUnevaluableExpr) =>
       val projection = new InterpretedMutableProjection(projectList, output)
@@ -1513,7 +1792,8 @@ object ConvertToLocalRelation extends Rule[LogicalPlan] {
  * }}}
  */
 object ReplaceDistinctWithAggregate extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsPattern(DISTINCT_LIKE), ruleId) {
     case Distinct(child) => Aggregate(child.output, child.output, child)
   }
 }
@@ -1522,14 +1802,14 @@ object ReplaceDistinctWithAggregate extends Rule[LogicalPlan] {
  * Replaces logical [[Deduplicate]] operator with an [[Aggregate]] operator.
  */
 object ReplaceDeduplicateWithAggregate extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
-    case Deduplicate(keys, child) if !child.isStreaming =>
+  def apply(plan: LogicalPlan): LogicalPlan = plan transformUpWithNewOutput {
+    case d @ Deduplicate(keys, child) if !child.isStreaming =>
       val keyExprIds = keys.map(_.exprId)
       val aggCols = child.output.map { attr =>
         if (keyExprIds.contains(attr.exprId)) {
           attr
         } else {
-          Alias(new First(attr).toAggregateExpression(), attr.name)(attr.exprId)
+          Alias(new First(attr).toAggregateExpression(), attr.name)()
         }
       }
       // SPARK-22951: Physical aggregate operators distinguishes global aggregation and grouping
@@ -1538,7 +1818,9 @@ object ReplaceDeduplicateWithAggregate extends Rule[LogicalPlan] {
       // we append a literal when the grouping key list is empty so that the result aggregate
       // operator is properly treated as a grouping aggregation.
       val nonemptyKeys = if (keys.isEmpty) Literal(1) :: Nil else keys
-      Aggregate(nonemptyKeys, aggCols, child)
+      val newAgg = Aggregate(nonemptyKeys, aggCols, child)
+      val attrMapping = d.output.zip(newAgg.output)
+      newAgg -> attrMapping
   }
 }
 
@@ -1555,7 +1837,8 @@ object ReplaceDeduplicateWithAggregate extends Rule[LogicalPlan] {
  *    join conditions will be incorrect.
  */
 object ReplaceIntersectWithSemiJoin extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsPattern(INTERSECT), ruleId) {
     case Intersect(left, right, false) =>
       assert(left.output.size == right.output.size)
       val joinCond = left.output.zip(right.output).map { case (l, r) => EqualNullSafe(l, r) }
@@ -1576,7 +1859,8 @@ object ReplaceIntersectWithSemiJoin extends Rule[LogicalPlan] {
  *    join conditions will be incorrect.
  */
 object ReplaceExceptWithAntiJoin extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsPattern(EXCEPT), ruleId) {
     case Except(left, right, false) =>
       assert(left.output.size == right.output.size)
       val joinCond = left.output.zip(right.output).map { case (l, r) => EqualNullSafe(l, r) }
@@ -1616,7 +1900,8 @@ object ReplaceExceptWithAntiJoin extends Rule[LogicalPlan] {
  */
 
 object RewriteExceptAll extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsPattern(EXCEPT), ruleId) {
     case Except(left, right, true) =>
       assert(left.output.size == right.output.size)
 
@@ -1673,7 +1958,8 @@ object RewriteExceptAll extends Rule[LogicalPlan] {
  * }}}
  */
 object RewriteIntersectAll extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsPattern(INTERSECT), ruleId) {
     case Intersect(left, right, true) =>
       assert(left.output.size == right.output.size)
 
@@ -1725,7 +2011,8 @@ object RewriteIntersectAll extends Rule[LogicalPlan] {
  * but only makes the grouping key bigger.
  */
 object RemoveLiteralFromGroupExpressions extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsPattern(AGGREGATE), ruleId) {
     case a @ Aggregate(grouping, _, _) if grouping.nonEmpty =>
       val newGrouping = grouping.filter(!_.foldable)
       if (newGrouping.nonEmpty) {
@@ -1744,7 +2031,8 @@ object RemoveLiteralFromGroupExpressions extends Rule[LogicalPlan] {
  * but only makes the grouping key bigger.
  */
 object RemoveRepetitionFromGroupExpressions extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsPattern(AGGREGATE), ruleId) {
     case a @ Aggregate(grouping, _, _) if grouping.size > 1 =>
       val newGrouping = ExpressionSet(grouping).toSeq
       if (newGrouping.size == grouping.size) {
@@ -1764,7 +2052,8 @@ object OptimizeLimitZero extends Rule[LogicalPlan] {
   private def empty(plan: LogicalPlan) =
     LocalRelation(plan.output, data = Seq.empty, isStreaming = plan.isStreaming)
 
-  def apply(plan: LogicalPlan): LogicalPlan = plan transformUp {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformUpWithPruning(
+    _.containsAllPatterns(LIMIT, LITERAL)) {
     // Nodes below GlobalLimit or LocalLimit can be pruned if the limit value is zero (0).
     // Any subtree in the logical plan that has GlobalLimit 0 or LocalLimit 0 as its root is
     // semantically equivalent to an empty relation.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelation.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelation.scala
index b19e13870aa65..8455b786bc384 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelation.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelation.scala
@@ -19,10 +19,11 @@ package org.apache.spark.sql.catalyst.optimizer
 
 import org.apache.spark.sql.catalyst.analysis.CastSupport
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.expressions.Literal.FalseLiteral
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules._
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.catalyst.trees.TreePattern.{LOCAL_RELATION, TRUE_OR_FALSE_LITERAL}
 
 /**
  * Collapse plans consisting empty local relations generated by [[PruneFilters]].
@@ -31,6 +32,7 @@ import org.apache.spark.sql.internal.SQLConf
  *    - Join with one or two empty children (including Intersect/Except).
  * 2. Unary-node Logical Plans
  *    - Project/Filter/Sample/Join/Limit/Repartition with all empty children.
+ *    - Join with false condition.
  *    - Aggregate with all empty children and at least one grouping expression.
  *    - Generate(Explode) with all empty children. Others like Hive UDTF may return results.
  */
@@ -47,33 +49,58 @@ object PropagateEmptyRelation extends Rule[LogicalPlan] with PredicateHelper wit
   private def nullValueProjectList(plan: LogicalPlan): Seq[NamedExpression] =
     plan.output.map{ a => Alias(cast(Literal(null), a.dataType), a.name)(a.exprId) }
 
-  override def conf: SQLConf = SQLConf.get
-
-  def apply(plan: LogicalPlan): LogicalPlan = plan transformUp {
-    case p: Union if p.children.forall(isEmptyLocalRelation) =>
-      empty(p)
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformUpWithPruning(
+    _.containsAnyPattern(LOCAL_RELATION, TRUE_OR_FALSE_LITERAL), ruleId) {
+    case p: Union if p.children.exists(isEmptyLocalRelation) =>
+      val newChildren = p.children.filterNot(isEmptyLocalRelation)
+      if (newChildren.isEmpty) {
+        empty(p)
+      } else {
+        val newPlan = if (newChildren.size > 1) Union(newChildren) else newChildren.head
+        val outputs = newPlan.output.zip(p.output)
+        // the original Union may produce different output attributes than the new one so we alias
+        // them if needed
+        if (outputs.forall { case (newAttr, oldAttr) => newAttr.exprId == oldAttr.exprId }) {
+          newPlan
+        } else {
+          val outputAliases = outputs.map { case (newAttr, oldAttr) =>
+            val newExplicitMetadata =
+              if (oldAttr.metadata != newAttr.metadata) Some(oldAttr.metadata) else None
+            Alias(newAttr, oldAttr.name)(oldAttr.exprId, explicitMetadata = newExplicitMetadata)
+          }
+          Project(outputAliases, newPlan)
+        }
+      }
 
     // Joins on empty LocalRelations generated from streaming sources are not eliminated
     // as stateful streaming joins need to perform other state management operations other than
     // just processing the input data.
-    case p @ Join(_, _, joinType, _, _)
+    case p @ Join(_, _, joinType, conditionOpt, _)
         if !p.children.exists(_.isStreaming) =>
       val isLeftEmpty = isEmptyLocalRelation(p.left)
       val isRightEmpty = isEmptyLocalRelation(p.right)
-      if (isLeftEmpty || isRightEmpty) {
+      val isFalseCondition = conditionOpt match {
+        case Some(FalseLiteral) => true
+        case _ => false
+      }
+      if (isLeftEmpty || isRightEmpty || isFalseCondition) {
         joinType match {
           case _: InnerLike => empty(p)
           // Intersect is handled as LeftSemi by `ReplaceIntersectWithSemiJoin` rule.
           // Except is handled as LeftAnti by `ReplaceExceptWithAntiJoin` rule.
           case LeftOuter | LeftSemi | LeftAnti if isLeftEmpty => empty(p)
-          case LeftSemi if isRightEmpty => empty(p)
-          case LeftAnti if isRightEmpty => p.left
+          case LeftSemi if isRightEmpty | isFalseCondition => empty(p)
+          case LeftAnti if isRightEmpty | isFalseCondition => p.left
           case FullOuter if isLeftEmpty && isRightEmpty => empty(p)
           case LeftOuter | FullOuter if isRightEmpty =>
             Project(p.left.output ++ nullValueProjectList(p.right), p.left)
           case RightOuter if isRightEmpty => empty(p)
           case RightOuter | FullOuter if isLeftEmpty =>
             Project(nullValueProjectList(p.left) ++ p.right.output, p.right)
+          case LeftOuter if isFalseCondition =>
+            Project(p.left.output ++ nullValueProjectList(p.right), p.left)
+          case RightOuter if isFalseCondition =>
+            Project(nullValueProjectList(p.left) ++ p.right.output, p.right)
           case _ => p
         }
       } else {
@@ -85,8 +112,8 @@ object PropagateEmptyRelation extends Rule[LogicalPlan] with PredicateHelper wit
       case _: Filter => empty(p)
       case _: Sample => empty(p)
       case _: Sort => empty(p)
-      case _: GlobalLimit => empty(p)
-      case _: LocalLimit => empty(p)
+      case _: GlobalLimit if !p.isStreaming => empty(p)
+      case _: LocalLimit if !p.isStreaming => empty(p)
       case _: Repartition => empty(p)
       case _: RepartitionByExpression => empty(p)
       // An aggregate with non-empty group expression will return one output row per group when the
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PullOutGroupingExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PullOutGroupingExpressions.scala
new file mode 100644
index 0000000000000..859a73a4842f0
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PullOutGroupingExpressions.scala
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import scala.collection.mutable
+
+import org.apache.spark.sql.catalyst.expressions.{Alias, Expression, NamedExpression}
+import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
+import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LogicalPlan, Project}
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreePattern.AGGREGATE
+
+/**
+ * This rule ensures that [[Aggregate]] nodes doesn't contain complex grouping expressions in the
+ * optimization phase.
+ *
+ * Complex grouping expressions are pulled out to a [[Project]] node under [[Aggregate]] and are
+ * referenced in both grouping expressions and aggregate expressions without aggregate functions.
+ * These references ensure that optimization rules don't change the aggregate expressions to invalid
+ * ones that no longer refer to any grouping expressions and also simplify the expression
+ * transformations on the node (need to transform the expression only once).
+ *
+ * For example, in the following query Spark shouldn't optimize the aggregate expression
+ * `Not(IsNull(c))` to `IsNotNull(c)` as the grouping expression is `IsNull(c)`:
+ * SELECT not(c IS NULL)
+ * FROM t
+ * GROUP BY c IS NULL
+ * Instead, the aggregate expression references a `_groupingexpression` attribute:
+ * Aggregate [_groupingexpression#233], [NOT _groupingexpression#233 AS (NOT (c IS NULL))#230]
+ * +- Project [isnull(c#219) AS _groupingexpression#233]
+ *    +- LocalRelation [c#219]
+ */
+object PullOutGroupingExpressions extends Rule[LogicalPlan] {
+  override def apply(plan: LogicalPlan): LogicalPlan = {
+    plan.transformWithPruning(_.containsPattern(AGGREGATE)) {
+      case a: Aggregate if a.resolved =>
+        val complexGroupingExpressionMap = mutable.LinkedHashMap.empty[Expression, NamedExpression]
+        val newGroupingExpressions = a.groupingExpressions.map {
+          case e if !e.foldable && e.children.nonEmpty =>
+            complexGroupingExpressionMap
+              .getOrElseUpdate(e.canonicalized, Alias(e, s"_groupingexpression")())
+              .toAttribute
+          case o => o
+        }
+        if (complexGroupingExpressionMap.nonEmpty) {
+          def replaceComplexGroupingExpressions(e: Expression): Expression = {
+            e match {
+              case _ if AggregateExpression.isAggregate(e) => e
+              case _ if e.foldable => e
+              case _ if complexGroupingExpressionMap.contains(e.canonicalized) =>
+                complexGroupingExpressionMap.get(e.canonicalized).map(_.toAttribute).getOrElse(e)
+              case _ => e.mapChildren(replaceComplexGroupingExpressions)
+            }
+          }
+
+          val newAggregateExpressions = a.aggregateExpressions
+            .map(replaceComplexGroupingExpressions(_).asInstanceOf[NamedExpression])
+          val newChild = Project(a.child.output ++ complexGroupingExpressionMap.values, a.child)
+          Aggregate(newGroupingExpressions, newAggregateExpressions, newChild)
+        } else {
+          a
+        }
+    }
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PushDownLeftSemiAntiJoin.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PushDownLeftSemiAntiJoin.scala
index 606db85fcdea6..31b9d60406095 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PushDownLeftSemiAntiJoin.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PushDownLeftSemiAntiJoin.scala
@@ -21,6 +21,7 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreePattern.LEFT_SEMI_OR_ANTI_JOIN
 
 /**
  * This rule is a variant of [[PushPredicateThroughNonJoin]] which can handle
@@ -31,8 +32,11 @@ import org.apache.spark.sql.catalyst.rules.Rule
  *  4) Aggregate
  *  5) Other permissible unary operators. please see [[PushPredicateThroughNonJoin.canPushThrough]].
  */
-object PushDownLeftSemiAntiJoin extends Rule[LogicalPlan] with PredicateHelper {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+object PushDownLeftSemiAntiJoin extends Rule[LogicalPlan]
+  with PredicateHelper
+  with JoinSelectionHelper {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsPattern(LEFT_SEMI_OR_ANTI_JOIN), ruleId) {
     // LeftSemi/LeftAnti over Project
     case Join(p @ Project(pList, gChild), rightOp, LeftSemiOrAnti(joinType), joinCond, hint)
         if pList.forall(_.deterministic) &&
@@ -42,7 +46,7 @@ object PushDownLeftSemiAntiJoin extends Rule[LogicalPlan] with PredicateHelper {
         // No join condition, just push down the Join below Project
         p.copy(child = Join(gChild, rightOp, joinType, joinCond, hint))
       } else {
-        val aliasMap = PushPredicateThroughNonJoin.getAliasMap(p)
+        val aliasMap = getAliasMap(p)
         val newJoinCond = if (aliasMap.nonEmpty) {
           Option(replaceAlias(joinCond.get, aliasMap))
         } else {
@@ -51,11 +55,12 @@ object PushDownLeftSemiAntiJoin extends Rule[LogicalPlan] with PredicateHelper {
         p.copy(child = Join(gChild, rightOp, joinType, newJoinCond, hint))
       }
 
-    // LeftSemi/LeftAnti over Aggregate
+    // LeftSemi/LeftAnti over Aggregate, only push down if join can be planned as broadcast join.
     case join @ Join(agg: Aggregate, rightOp, LeftSemiOrAnti(_), _, _)
         if agg.aggregateExpressions.forall(_.deterministic) && agg.groupingExpressions.nonEmpty &&
-        !agg.aggregateExpressions.exists(ScalarSubquery.hasCorrelatedScalarSubquery) =>
-      val aliasMap = PushPredicateThroughNonJoin.getAliasMap(agg)
+          !agg.aggregateExpressions.exists(ScalarSubquery.hasCorrelatedScalarSubquery) &&
+          canPlanAsBroadcastHashJoin(join, conf) =>
+      val aliasMap = getAliasMap(agg)
       val canPushDownPredicate = (predicate: Expression) => {
         val replaced = replaceAlias(predicate, aliasMap)
         predicate.references.nonEmpty &&
@@ -172,7 +177,7 @@ object PushDownLeftSemiAntiJoin extends Rule[LogicalPlan] with PredicateHelper {
  * TODO:
  * Currently this rule can push down the left semi or left anti joins to either
  * left or right leg of the child join. This matches the behaviour of `PushPredicateThroughJoin`
- * when the lefi semi or left anti join is in expression form. We need to explore the possibility
+ * when the left semi or left anti join is in expression form. We need to explore the possibility
  * to push the left semi/anti joins to both legs of join if the join condition refers to
  * both left and right legs of the child join.
  */
@@ -237,7 +242,8 @@ object PushLeftSemiLeftAntiThroughJoin extends Rule[LogicalPlan] with PredicateH
     }
   }
 
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsPattern(LEFT_SEMI_OR_ANTI_JOIN), ruleId) {
     // push LeftSemi/LeftAnti down into the join below
     case j @ Join(AllowedJoin(left), right, LeftSemiOrAnti(joinType), joinCond, parentHint) =>
       val (childJoinType, childLeft, childRight, childCondition, childHint) =
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PushExtraPredicateThroughJoin.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PushExtraPredicateThroughJoin.scala
new file mode 100644
index 0000000000000..134a1eba1dd2c
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PushExtraPredicateThroughJoin.scala
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.expressions.{And, Expression, PredicateHelper}
+import org.apache.spark.sql.catalyst.plans._
+import org.apache.spark.sql.catalyst.plans.logical.{Filter, Join, LogicalPlan}
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreeNodeTag
+import org.apache.spark.sql.catalyst.trees.TreePattern.JOIN
+
+/**
+ * Try pushing down disjunctive join condition into left and right child.
+ * To avoid expanding the join condition, the join condition will be kept in the original form even
+ * when predicate pushdown happens.
+ */
+object PushExtraPredicateThroughJoin extends Rule[LogicalPlan] with PredicateHelper {
+
+  private val processedJoinConditionTag = TreeNodeTag[Expression]("processedJoinCondition")
+
+  private def canPushThrough(joinType: JoinType): Boolean = joinType match {
+    case _: InnerLike | LeftSemi | RightOuter | LeftOuter | LeftAnti | ExistenceJoin(_) => true
+    case _ => false
+  }
+
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsPattern(JOIN), ruleId) {
+    case j @ Join(left, right, joinType, Some(joinCondition), hint)
+        if canPushThrough(joinType) =>
+      val alreadyProcessed = j.getTagValue(processedJoinConditionTag).exists { condition =>
+        condition.semanticEquals(joinCondition)
+      }
+
+      lazy val filtersOfBothSide = splitConjunctivePredicates(joinCondition).filter { f =>
+        f.deterministic && f.references.nonEmpty &&
+          !f.references.subsetOf(left.outputSet) && !f.references.subsetOf(right.outputSet)
+      }
+      lazy val leftExtraCondition =
+        filtersOfBothSide.flatMap(extractPredicatesWithinOutputSet(_, left.outputSet))
+      lazy val rightExtraCondition =
+        filtersOfBothSide.flatMap(extractPredicatesWithinOutputSet(_, right.outputSet))
+
+      if (alreadyProcessed || (leftExtraCondition.isEmpty && rightExtraCondition.isEmpty)) {
+        j
+      } else {
+        lazy val newLeft =
+          leftExtraCondition.reduceLeftOption(And).map(Filter(_, left)).getOrElse(left)
+        lazy val newRight =
+          rightExtraCondition.reduceLeftOption(And).map(Filter(_, right)).getOrElse(right)
+
+        val newJoin = joinType match {
+          case _: InnerLike | LeftSemi =>
+            Join(newLeft, newRight, joinType, Some(joinCondition), hint)
+          case RightOuter =>
+            Join(newLeft, right, RightOuter, Some(joinCondition), hint)
+          case LeftOuter | LeftAnti | ExistenceJoin(_) =>
+            Join(left, newRight, joinType, Some(joinCondition), hint)
+          case other =>
+            throw new IllegalStateException(s"Unexpected join type: $other")
+        }
+        newJoin.setTagValue(processedJoinConditionTag, joinCondition)
+        newJoin
+    }
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceExceptWithFilter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceExceptWithFilter.scala
index 4996d24dfd298..8218051c584b3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceExceptWithFilter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceExceptWithFilter.scala
@@ -22,6 +22,7 @@ import scala.annotation.tailrec
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreePattern.EXCEPT
 
 
 /**
@@ -46,7 +47,7 @@ object ReplaceExceptWithFilter extends Rule[LogicalPlan] {
       return plan
     }
 
-    plan.transform {
+    plan.transformWithPruning(_.containsPattern(EXCEPT), ruleId) {
       case e @ Except(left, right, false) if isEligible(left, right) =>
         val filterCondition = combineFilters(skipProject(right)).asInstanceOf[Filter].condition
         if (filterCondition.deterministic) {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicate.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicate.scala
index 33b398e11cde9..3de19afa912aa 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicate.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicate.scala
@@ -17,12 +17,11 @@
 
 package org.apache.spark.sql.catalyst.optimizer
 
-import org.apache.spark.sql.catalyst.expressions.{And, ArrayExists, ArrayFilter, CaseWhen, Expression, If}
-import org.apache.spark.sql.catalyst.expressions.{LambdaFunction, Literal, MapFilter, Or}
-import org.apache.spark.sql.catalyst.expressions.Literal.FalseLiteral
-import org.apache.spark.sql.catalyst.plans.logical.{Filter, Join, LogicalPlan}
+import org.apache.spark.sql.catalyst.expressions.{And, ArrayExists, ArrayFilter, CaseWhen, EqualNullSafe, Expression, If, In, InSet, LambdaFunction, Literal, MapFilter, Not, Or}
+import org.apache.spark.sql.catalyst.expressions.Literal.{FalseLiteral, TrueLiteral}
+import org.apache.spark.sql.catalyst.plans.logical.{DeleteAction, DeleteFromTable, Filter, InsertAction, InsertStarAction, Join, LogicalPlan, MergeAction, MergeIntoTable, UpdateAction, UpdateStarAction, UpdateTable}
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.catalyst.trees.TreePattern.{INSET, NULL_LITERAL, TRUE_OR_FALSE_LITERAL}
 import org.apache.spark.sql.types.BooleanType
 import org.apache.spark.util.Utils
 
@@ -51,10 +50,25 @@ import org.apache.spark.util.Utils
  */
 object ReplaceNullWithFalseInPredicate extends Rule[LogicalPlan] {
 
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsAnyPattern(NULL_LITERAL, TRUE_OR_FALSE_LITERAL, INSET), ruleId) {
     case f @ Filter(cond, _) => f.copy(condition = replaceNullWithFalse(cond))
     case j @ Join(_, _, _, Some(cond), _) => j.copy(condition = Some(replaceNullWithFalse(cond)))
-    case p: LogicalPlan => p transformExpressions {
+    case d @ DeleteFromTable(_, Some(cond)) => d.copy(condition = Some(replaceNullWithFalse(cond)))
+    case u @ UpdateTable(_, _, Some(cond)) => u.copy(condition = Some(replaceNullWithFalse(cond)))
+    case m @ MergeIntoTable(_, _, mergeCond, matchedActions, notMatchedActions) =>
+      m.copy(
+        mergeCondition = replaceNullWithFalse(mergeCond),
+        matchedActions = replaceNullWithFalse(matchedActions),
+        notMatchedActions = replaceNullWithFalse(notMatchedActions))
+    case p: LogicalPlan => p.transformExpressionsWithPruning(
+      _.containsAnyPattern(NULL_LITERAL, TRUE_OR_FALSE_LITERAL), ruleId) {
+      // For `EqualNullSafe` with a `TrueLiteral`, whether the other side is null or false has no
+      // difference, as `null <=> true` and `false <=> true` both return false.
+      case EqualNullSafe(left, TrueLiteral) =>
+        EqualNullSafe(replaceNullWithFalse(left), TrueLiteral)
+      case EqualNullSafe(TrueLiteral, right) =>
+        EqualNullSafe(TrueLiteral, replaceNullWithFalse(right))
       case i @ If(pred, _, _) => i.copy(predicate = replaceNullWithFalse(pred))
       case cw @ CaseWhen(branches, _) =>
         val newBranches = branches.map { case (cond, value) =>
@@ -84,6 +98,17 @@ object ReplaceNullWithFalseInPredicate extends Rule[LogicalPlan] {
   private def replaceNullWithFalse(e: Expression): Expression = e match {
     case Literal(null, BooleanType) =>
       FalseLiteral
+    // In SQL, the `Not(IN)` expression evaluates as follows:
+    // `NULL not in (1)` -> NULL
+    // `NULL not in (1, NULL)` -> NULL
+    // `1 not in (1, NULL)` -> false
+    // `1 not in (2, NULL)` -> NULL
+    // In predicate, NULL is equal to false, so we can simplify them to false directly.
+    case Not(In(value, list)) if (value +: list).exists(isNullLiteral) =>
+      FalseLiteral
+    case Not(InSet(value, list)) if isNullLiteral(value) || list.contains(null) =>
+      FalseLiteral
+
     case And(left, right) =>
       And(replaceNullWithFalse(left), replaceNullWithFalse(right))
     case Or(left, right) =>
@@ -92,7 +117,7 @@ object ReplaceNullWithFalseInPredicate extends Rule[LogicalPlan] {
       val newBranches = cw.branches.map { case (cond, value) =>
         replaceNullWithFalse(cond) -> replaceNullWithFalse(value)
       }
-      val newElseValue = cw.elseValue.map(replaceNullWithFalse)
+      val newElseValue = cw.elseValue.map(replaceNullWithFalse).getOrElse(FalseLiteral)
       CaseWhen(newBranches, newElseValue)
     case i @ If(pred, trueVal, falseVal) if i.dataType == BooleanType =>
       If(replaceNullWithFalse(pred), replaceNullWithFalse(trueVal), replaceNullWithFalse(falseVal))
@@ -108,4 +133,20 @@ object ReplaceNullWithFalseInPredicate extends Rule[LogicalPlan] {
         e
       }
   }
+
+  private def replaceNullWithFalse(mergeActions: Seq[MergeAction]): Seq[MergeAction] = {
+    mergeActions.map {
+      case u @ UpdateAction(Some(cond), _) => u.copy(condition = Some(replaceNullWithFalse(cond)))
+      case u @ UpdateStarAction(Some(cond)) => u.copy(condition = Some(replaceNullWithFalse(cond)))
+      case d @ DeleteAction(Some(cond)) => d.copy(condition = Some(replaceNullWithFalse(cond)))
+      case i @ InsertAction(Some(cond), _) => i.copy(condition = Some(replaceNullWithFalse(cond)))
+      case i @ InsertStarAction(Some(cond)) => i.copy(condition = Some(replaceNullWithFalse(cond)))
+      case other => other
+    }
+  }
+
+  private def isNullLiteral(e: Expression): Boolean = e match {
+    case Literal(null, _) => true
+    case _ => false
+  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregates.scala
index e5571069a7c41..22cde96412567 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregates.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregates.scala
@@ -18,9 +18,10 @@
 package org.apache.spark.sql.catalyst.optimizer
 
 import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, AggregateFunction, Complete}
+import org.apache.spark.sql.catalyst.expressions.aggregate._
 import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Expand, LogicalPlan}
 import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreePattern.AGGREGATE
 import org.apache.spark.sql.types.IntegerType
 
 /**
@@ -39,8 +40,8 @@ import org.apache.spark.sql.types.IntegerType
  *
  *   val agg = data.groupBy($"key")
  *     .agg(
- *       countDistinct($"cat1").as("cat1_cnt"),
- *       countDistinct($"cat2").as("cat2_cnt"),
+ *       count_distinct($"cat1").as("cat1_cnt"),
+ *       count_distinct($"cat2").as("cat2_cnt"),
  *       sum($"value").as("total"))
  * }}}
  *
@@ -59,9 +60,9 @@ import org.apache.spark.sql.types.IntegerType
  * {{{
  * Aggregate(
  *    key = ['key]
- *    functions = [count(if (('gid = 1)) 'cat1 else null),
- *                 count(if (('gid = 2)) 'cat2 else null),
- *                 first(if (('gid = 0)) 'total else null) ignore nulls]
+ *    functions = [count('cat1) FILTER (WHERE 'gid = 1),
+ *                 count('cat2) FILTER (WHERE 'gid = 2),
+ *                 first('total) ignore nulls FILTER (WHERE 'gid = 0)]
  *    output = ['key, 'cat1_cnt, 'cat2_cnt, 'total])
  *   Aggregate(
  *      key = ['key, 'cat1, 'cat2, 'gid]
@@ -81,10 +82,10 @@ import org.apache.spark.sql.types.IntegerType
  *     COUNT(DISTINCT cat1) as cat1_cnt,
  *     COUNT(DISTINCT cat2) as cat2_cnt,
  *     SUM(value) FILTER (WHERE id > 1) AS total
- *  FROM
- *    data
- *  GROUP BY
- *    key
+ *   FROM
+ *     data
+ *   GROUP BY
+ *     key
  * }}}
  *
  * This translates to the following (pseudo) logical plan:
@@ -93,7 +94,7 @@ import org.apache.spark.sql.types.IntegerType
  *    key = ['key]
  *    functions = [COUNT(DISTINCT 'cat1),
  *                 COUNT(DISTINCT 'cat2),
- *                 sum('value) with FILTER('id > 1)]
+ *                 sum('value) FILTER (WHERE 'id > 1)]
  *    output = ['key, 'cat1_cnt, 'cat2_cnt, 'total])
  *   LocalTableScan [...]
  * }}}
@@ -102,13 +103,13 @@ import org.apache.spark.sql.types.IntegerType
  * {{{
  * Aggregate(
  *    key = ['key]
- *    functions = [count(if (('gid = 1)) 'cat1 else null),
- *                 count(if (('gid = 2)) 'cat2 else null),
- *                 first(if (('gid = 0)) 'total else null) ignore nulls]
+ *    functions = [count('cat1) FILTER (WHERE 'gid = 1),
+ *                 count('cat2) FILTER (WHERE 'gid = 2),
+ *                 first('total) ignore nulls FILTER (WHERE 'gid = 0)]
  *    output = ['key, 'cat1_cnt, 'cat2_cnt, 'total])
  *   Aggregate(
  *      key = ['key, 'cat1, 'cat2, 'gid]
- *      functions = [sum('value) with FILTER('id > 1)]
+ *      functions = [sum('value) FILTER (WHERE 'id > 1)]
  *      output = ['key, 'cat1, 'cat2, 'gid, 'total])
  *     Expand(
  *        projections = [('key, null, null, 0, cast('value as bigint), 'id),
@@ -118,6 +119,49 @@ import org.apache.spark.sql.types.IntegerType
  *       LocalTableScan [...]
  * }}}
  *
+ * Third example: aggregate function with distinct and filter clauses (in sql):
+ * {{{
+ *   SELECT
+ *     COUNT(DISTINCT cat1) FILTER (WHERE id > 1) as cat1_cnt,
+ *     COUNT(DISTINCT cat2) FILTER (WHERE id > 2) as cat2_cnt,
+ *     SUM(value) FILTER (WHERE id > 3) AS total
+ *   FROM
+ *     data
+ *   GROUP BY
+ *     key
+ * }}}
+ *
+ * This translates to the following (pseudo) logical plan:
+ * {{{
+ * Aggregate(
+ *    key = ['key]
+ *    functions = [COUNT(DISTINCT 'cat1) FILTER (WHERE 'id > 1),
+ *                 COUNT(DISTINCT 'cat2) FILTER (WHERE 'id > 2),
+ *                 sum('value) FILTER (WHERE 'id > 3)]
+ *    output = ['key, 'cat1_cnt, 'cat2_cnt, 'total])
+ *   LocalTableScan [...]
+ * }}}
+ *
+ * This rule rewrites this logical plan to the following (pseudo) logical plan:
+ * {{{
+ * Aggregate(
+ *    key = ['key]
+ *    functions = [count('cat1) FILTER (WHERE 'gid = 1 and 'max_cond1),
+ *                 count('cat2) FILTER (WHERE 'gid = 2 and 'max_cond2),
+ *                 first('total) ignore nulls FILTER (WHERE 'gid = 0)]
+ *    output = ['key, 'cat1_cnt, 'cat2_cnt, 'total])
+ *   Aggregate(
+ *      key = ['key, 'cat1, 'cat2, 'gid]
+ *      functions = [max('cond1), max('cond2), sum('value) FILTER (WHERE 'id > 3)]
+ *      output = ['key, 'cat1, 'cat2, 'gid, 'max_cond1, 'max_cond2, 'total])
+ *     Expand(
+ *        projections = [('key, null, null, 0, null, null, cast('value as bigint), 'id),
+ *                       ('key, 'cat1, null, 1, 'id > 1, null, null, null),
+ *                       ('key, null, 'cat2, 2, null, 'id > 2, null, null)]
+ *        output = ['key, 'cat1, 'cat2, 'gid, 'cond1, 'cond2, 'value, 'id])
+ *       LocalTableScan [...]
+ * }}}
+ *
  * The rule does the following things here:
  * 1. Expand the data. There are three aggregation groups in this query:
  *    i. the non-distinct group;
@@ -126,15 +170,24 @@ import org.apache.spark.sql.types.IntegerType
  *    An expand operator is inserted to expand the child data for each group. The expand will null
  *    out all unused columns for the given group; this must be done in order to ensure correctness
  *    later on. Groups can by identified by a group id (gid) column added by the expand operator.
+ *    If distinct group exists filter clause, the expand will calculate the filter and output it's
+ *    result (e.g. cond1) which will be used to calculate the global conditions (e.g. max_cond1)
+ *    equivalent to filter clauses.
  * 2. De-duplicate the distinct paths and aggregate the non-aggregate path. The group by clause of
  *    this aggregate consists of the original group by clause, all the requested distinct columns
  *    and the group id. Both de-duplication of distinct column and the aggregation of the
  *    non-distinct group take advantage of the fact that we group by the group id (gid) and that we
- *    have nulled out all non-relevant columns the given group.
+ *    have nulled out all non-relevant columns the given group. If distinct group exists filter
+ *    clause, we will use max to aggregate the results (e.g. cond1) of the filter output in the
+ *    previous step. These aggregate will output the global conditions (e.g. max_cond1) equivalent
+ *    to filter clauses.
  * 3. Aggregating the distinct groups and combining this with the results of the non-distinct
- *    aggregation. In this step we use the group id to filter the inputs for the aggregate
- *    functions. The result of the non-distinct group are 'aggregated' by using the first operator,
- *    it might be more elegant to use the native UDAF merge mechanism for this in the future.
+ *    aggregation. In this step we use the group id and the global condition to filter the inputs
+ *    for the aggregate functions. If the global condition (e.g. max_cond1) is true, it means at
+ *    least one row of a distinct value satisfies the filter. This distinct value should be included
+ *    in the aggregate function. The result of the non-distinct group are 'aggregated' by using
+ *    the first operator, it might be more elegant to use the native UDAF merge mechanism for this
+ *    in the future.
  *
  * This rule duplicates the input data by two or more times (# distinct groups + an optional
  * non-distinct group). This will put quite a bit of memory pressure of the used aggregate and
@@ -144,28 +197,25 @@ import org.apache.spark.sql.types.IntegerType
  */
 object RewriteDistinctAggregates extends Rule[LogicalPlan] {
 
-  private def mayNeedtoRewrite(exprs: Seq[Expression]): Boolean = {
-    val distinctAggs = exprs.flatMap { _.collect {
-      case ae: AggregateExpression if ae.isDistinct => ae
-    }}
-    // We need at least two distinct aggregates for this rule because aggregation
-    // strategy can handle a single distinct group.
+  private def mayNeedtoRewrite(a: Aggregate): Boolean = {
+    val aggExpressions = collectAggregateExprs(a)
+    val distinctAggs = aggExpressions.filter(_.isDistinct)
+    // We need at least two distinct aggregates or the single distinct aggregate group exists filter
+    // clause for this rule because aggregation strategy can handle a single distinct aggregate
+    // group without filter clause.
     // This check can produce false-positives, e.g., SUM(DISTINCT a) & COUNT(DISTINCT a).
-    distinctAggs.size > 1
+    distinctAggs.size > 1 || distinctAggs.exists(_.filter.isDefined)
   }
 
-  def apply(plan: LogicalPlan): LogicalPlan = plan transformUp {
-    case a: Aggregate if mayNeedtoRewrite(a.aggregateExpressions) => rewrite(a)
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformUpWithPruning(
+    _.containsPattern(AGGREGATE)) {
+    case a: Aggregate if mayNeedtoRewrite(a) => rewrite(a)
   }
 
   def rewrite(a: Aggregate): Aggregate = {
 
-    // Collect all aggregate expressions.
-    val aggExpressions = a.aggregateExpressions.flatMap { e =>
-      e.collect {
-        case ae: AggregateExpression => ae
-      }
-    }
+    val aggExpressions = collectAggregateExprs(a)
+    val distinctAggs = aggExpressions.filter(_.isDistinct)
 
     // Extract distinct aggregate expressions.
     val distinctAggGroups = aggExpressions.filter(_.isDistinct).groupBy { e =>
@@ -184,8 +234,8 @@ object RewriteDistinctAggregates extends Rule[LogicalPlan] {
         }
     }
 
-    // Aggregation strategy can handle queries with a single distinct group.
-    if (distinctAggGroups.size > 1) {
+    // Aggregation strategy can handle queries with a single distinct group without filter clause.
+    if (distinctAggGroups.size > 1 || distinctAggs.exists(_.filter.isDefined)) {
       // Create the attributes for the grouping id and the group by clause.
       val gid = AttributeReference("gid", IntegerType, nullable = false)()
       val groupByMap = a.groupingExpressions.collect {
@@ -194,8 +244,6 @@ object RewriteDistinctAggregates extends Rule[LogicalPlan] {
       }
       val groupByAttrs = groupByMap.map(_._2)
 
-      // Functions used to modify aggregate functions and their inputs.
-      def evalWithinGroup(id: Literal, e: Expression) = If(EqualTo(gid, id), e, nullify(e))
       def patchAggregateFunctionChildren(
           af: AggregateFunction)(
           attrs: Expression => Option[Expression]): AggregateFunction = {
@@ -207,13 +255,28 @@ object RewriteDistinctAggregates extends Rule[LogicalPlan] {
       val distinctAggChildren = distinctAggGroups.keySet.flatten.toSeq.distinct
       val distinctAggChildAttrMap = distinctAggChildren.map(expressionAttributePair)
       val distinctAggChildAttrs = distinctAggChildAttrMap.map(_._2)
+      // Setup all the filters in distinct aggregate.
+      val (distinctAggFilters, distinctAggFilterAttrs, maxConds) = distinctAggs.collect {
+        case AggregateExpression(_, _, _, filter, _) if filter.isDefined =>
+          val (e, attr) = expressionAttributePair(filter.get)
+          val aggregateExp = Max(attr).toAggregateExpression()
+          (e, attr, Alias(aggregateExp, attr.name)())
+      }.unzip3
 
       // Setup expand & aggregate operators for distinct aggregate expressions.
       val distinctAggChildAttrLookup = distinctAggChildAttrMap.toMap
+      val distinctAggFilterAttrLookup = distinctAggFilters.zip(maxConds.map(_.toAttribute)).toMap
       val distinctAggOperatorMap = distinctAggGroups.toSeq.zipWithIndex.map {
         case ((group, expressions), i) =>
           val id = Literal(i + 1)
 
+          // Expand projection for filter
+          val filters = expressions.filter(_.filter.isDefined).map(_.filter.get)
+          val filterProjection = distinctAggFilters.map {
+            case e if filters.contains(e) => e
+            case e => nullify(e)
+          }
+
           // Expand projection
           val projection = distinctAggChildren.map {
             case e if group.contains(e) => e
@@ -223,13 +286,24 @@ object RewriteDistinctAggregates extends Rule[LogicalPlan] {
           // Final aggregate
           val operators = expressions.map { e =>
             val af = e.aggregateFunction
-            val naf = patchAggregateFunctionChildren(af) { x =>
-              distinctAggChildAttrLookup.get(x).map(evalWithinGroup(id, _))
+            val condition = e.filter.flatMap(distinctAggFilterAttrLookup.get)
+            val naf = if (af.children.forall(_.foldable)) {
+              af
+            } else {
+              patchAggregateFunctionChildren(af) { x =>
+                distinctAggChildAttrLookup.get(x)
+              }
+            }
+            val newCondition = if (condition.isDefined) {
+              And(EqualTo(gid, id), condition.get)
+            } else {
+              EqualTo(gid, id)
             }
-            (e, e.copy(aggregateFunction = naf, isDistinct = false))
+
+            (e, e.copy(aggregateFunction = naf, isDistinct = false, filter = Some(newCondition)))
           }
 
-          (projection, operators)
+          (projection ++ filterProjection, operators)
       }
 
       // Setup expand for the 'regular' aggregate expressions.
@@ -257,9 +331,10 @@ object RewriteDistinctAggregates extends Rule[LogicalPlan] {
 
         // Select the result of the first aggregate in the last aggregate.
         val result = AggregateExpression(
-          aggregate.First(evalWithinGroup(regularGroupId, operator.toAttribute), Literal(true)),
+          aggregate.First(operator.toAttribute, ignoreNulls = true),
           mode = Complete,
-          isDistinct = false)
+          isDistinct = false,
+          filter = Some(EqualTo(gid, regularGroupId)))
 
         // Some aggregate functions (COUNT) have the special property that they can return a
         // non-null result without any input. We need to make sure we return a result in this case.
@@ -280,6 +355,7 @@ object RewriteDistinctAggregates extends Rule[LogicalPlan] {
         Seq(a.groupingExpressions ++
           distinctAggChildren.map(nullify) ++
           Seq(regularGroupId) ++
+          distinctAggFilters.map(nullify) ++
           regularAggChildren)
       } else {
         Seq.empty[Seq[Expression]]
@@ -297,7 +373,8 @@ object RewriteDistinctAggregates extends Rule[LogicalPlan] {
       // Construct the expand operator.
       val expand = Expand(
         regularAggProjection ++ distinctAggProjections,
-        groupByAttrs ++ distinctAggChildAttrs ++ Seq(gid) ++ regularAggChildAttrMap.map(_._2),
+        groupByAttrs ++ distinctAggChildAttrs ++ Seq(gid) ++ distinctAggFilterAttrs ++
+          regularAggChildAttrMap.map(_._2),
         a.child)
 
       // Construct the first aggregate operator. This de-duplicates all the children of
@@ -305,7 +382,7 @@ object RewriteDistinctAggregates extends Rule[LogicalPlan] {
       val firstAggregateGroupBy = groupByAttrs ++ distinctAggChildAttrs :+ gid
       val firstAggregate = Aggregate(
         firstAggregateGroupBy,
-        firstAggregateGroupBy ++ regularAggOperatorMap.map(_._2),
+        firstAggregateGroupBy ++ maxConds ++ regularAggOperatorMap.map(_._2),
         expand)
 
       // Construct the second aggregate
@@ -331,6 +408,13 @@ object RewriteDistinctAggregates extends Rule[LogicalPlan] {
     }
   }
 
+  private def collectAggregateExprs(a: Aggregate): Seq[AggregateExpression] = {
+    // Collect all aggregate expressions.
+    a.aggregateExpressions.flatMap { _.collect {
+        case ae: AggregateExpression => ae
+    }}
+  }
+
   private def nullify(e: Expression) = Literal.create(null, e.dataType)
 
   private def expressionAttributePair(e: Expression) =
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalsInPredicate.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalsInPredicate.scala
new file mode 100644
index 0000000000000..bc1c2e5d671d2
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalsInPredicate.scala
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.expressions.{And, CaseWhen, Expression, If, Literal, Not, Or}
+import org.apache.spark.sql.catalyst.expressions.Literal.{FalseLiteral, TrueLiteral}
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreePattern.{CASE_WHEN, IF}
+import org.apache.spark.sql.types.BooleanType
+
+/**
+ * A rule that converts conditional expressions to predicate expressions, if possible, in the
+ * search condition of the WHERE/HAVING/ON(JOIN) clauses, which contain an implicit Boolean operator
+ * "(search condition) = TRUE". After this converting, we can potentially push the filter down to
+ * the data source.
+ *
+ * Supported cases are:
+ * - IF(cond, trueVal, false)                   => AND(cond, trueVal)
+ * - IF(cond, trueVal, true)                    => OR(NOT(cond), trueVal)
+ * - IF(cond, false, falseVal)                  => AND(NOT(cond), elseVal)
+ * - IF(cond, true, falseVal)                   => OR(cond, elseVal)
+ * - CASE WHEN cond THEN trueVal ELSE false END => AND(cond, trueVal)
+ * - CASE WHEN cond THEN trueVal END            => AND(cond, trueVal)
+ * - CASE WHEN cond THEN trueVal ELSE null END  => AND(cond, trueVal)
+ * - CASE WHEN cond THEN trueVal ELSE true END  => OR(NOT(cond), trueVal)
+ * - CASE WHEN cond THEN false ELSE elseVal END => AND(NOT(cond), elseVal)
+ * - CASE WHEN cond THEN true ELSE elseVal END  => OR(cond, elseVal)
+ */
+object SimplifyConditionalsInPredicate extends Rule[LogicalPlan] {
+
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsAnyPattern(CASE_WHEN, IF), ruleId) {
+    case f @ Filter(cond, _) => f.copy(condition = simplifyConditional(cond))
+    case j @ Join(_, _, _, Some(cond), _) => j.copy(condition = Some(simplifyConditional(cond)))
+    case d @ DeleteFromTable(_, Some(cond)) => d.copy(condition = Some(simplifyConditional(cond)))
+    case u @ UpdateTable(_, _, Some(cond)) => u.copy(condition = Some(simplifyConditional(cond)))
+  }
+
+  private def simplifyConditional(e: Expression): Expression = e match {
+    case And(left, right) => And(simplifyConditional(left), simplifyConditional(right))
+    case Or(left, right) => Or(simplifyConditional(left), simplifyConditional(right))
+    case If(cond, trueValue, FalseLiteral) => And(cond, trueValue)
+    case If(cond, trueValue, TrueLiteral) => Or(Not(cond), trueValue)
+    case If(cond, FalseLiteral, falseValue) => And(Not(cond), falseValue)
+    case If(cond, TrueLiteral, falseValue) => Or(cond, falseValue)
+    case CaseWhen(Seq((cond, trueValue)),
+        Some(FalseLiteral) | Some(Literal(null, BooleanType)) | None) =>
+      And(cond, trueValue)
+    case CaseWhen(Seq((cond, trueValue)), Some(TrueLiteral)) =>
+      Or(Not(cond), trueValue)
+    case CaseWhen(Seq((cond, FalseLiteral)), Some(elseValue)) =>
+      And(Not(cond), elseValue)
+    case CaseWhen(Seq((cond, TrueLiteral)), Some(elseValue)) =>
+      Or(cond, elseValue)
+    case e if e.dataType == BooleanType => e
+    case e =>
+      assert(e.dataType != BooleanType,
+      "Expected a Boolean type expression in SimplifyConditionalsInPredicate, " +
+        s"but got the type `${e.dataType.catalogString}` in `${e.sql}`.")
+      e
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/StarSchemaDetection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/StarSchemaDetection.scala
index 2aa762e2595ad..bf3fced0ae0fd 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/StarSchemaDetection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/StarSchemaDetection.scala
@@ -19,18 +19,16 @@ package org.apache.spark.sql.catalyst.optimizer
 
 import scala.annotation.tailrec
 
+import org.apache.spark.sql.catalyst.SQLConfHelper
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.planning.PhysicalOperation
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.internal.SQLConf
 
 /**
  * Encapsulates star-schema detection logic.
  */
-object StarSchemaDetection extends PredicateHelper {
-
-  private def conf = SQLConf.get
+object StarSchemaDetection extends PredicateHelper with SQLConfHelper {
 
   /**
    * Star schema consists of one or more fact tables referencing a number of dimension
@@ -199,9 +197,9 @@ object StarSchemaDetection extends PredicateHelper {
               } else {
                 false
               }
-            case None => false
+            case _ => false
           }
-        case None => false
+        case _ => false
       }
     case _ => false
   }
@@ -241,7 +239,7 @@ object StarSchemaDetection extends PredicateHelper {
         case Some(col) if t.outputSet.contains(col) =>
           val stats = t.stats
           stats.attributeStats.nonEmpty && stats.attributeStats.contains(col)
-        case None => false
+        case _ => false
       }
     case _ => false
   }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparison.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparison.scala
new file mode 100644
index 0000000000000..9f72751eba8fe
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparison.scala
@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.expressions.Literal.FalseLiteral
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreePattern.BINARY_COMPARISON
+import org.apache.spark.sql.types._
+
+/**
+ * Unwrap casts in binary comparison operations with patterns like following:
+ *
+ * `BinaryComparison(Cast(fromExp, toType), Literal(value, toType))`
+ *   or
+ * `BinaryComparison(Literal(value, toType), Cast(fromExp, toType))`
+ *
+ * This rule optimizes expressions with the above pattern by either replacing the cast with simpler
+ * constructs, or moving the cast from the expression side to the literal side, which enables them
+ * to be optimized away later and pushed down to data sources.
+ *
+ * Currently this only handles cases where:
+ *   1). `fromType` (of `fromExp`) and `toType` are of numeric types (i.e., short, int, float,
+ *     decimal, etc)
+ *   2). `fromType` can be safely coerced to `toType` without precision loss (e.g., short to int,
+ *     int to long, but not long to int)
+ *
+ * If the above conditions are satisfied, the rule checks to see if the literal `value` is within
+ * range `(min, max)`, where `min` and `max` are the minimum and maximum value of `fromType`,
+ * respectively. If this is true then it means we may safely cast `value` to `fromType` and thus
+ * able to move the cast to the literal side. That is:
+ *
+ *   `cast(fromExp, toType) op value` ==> `fromExp op cast(value, fromType)`
+ *
+ * Note there are some exceptions to the above: if casting from `value` to `fromType` causes
+ * rounding up or down, the above conversion will no longer be valid. Instead, the rule does the
+ * following:
+ *
+ * if casting `value` to `fromType` causes rounding up:
+ *  - `cast(fromExp, toType) > value` ==> `fromExp >= cast(value, fromType)`
+ *  - `cast(fromExp, toType) >= value` ==> `fromExp >= cast(value, fromType)`
+ *  - `cast(fromExp, toType) === value` ==> if(isnull(fromExp), null, false)
+ *  - `cast(fromExp, toType) <=> value` ==> false (if `fromExp` is deterministic)
+ *  - `cast(fromExp, toType) <= value` ==> `fromExp < cast(value, fromType)`
+ *  - `cast(fromExp, toType) < value` ==> `fromExp < cast(value, fromType)`
+ *
+ * Similarly for the case when casting `value` to `fromType` causes rounding down.
+ *
+ * If the `value` is not within range `(min, max)`, the rule breaks the scenario into different
+ * cases and try to replace each with simpler constructs.
+ *
+ * if `value > max`, the cases are of following:
+ *  - `cast(fromExp, toType) > value` ==> if(isnull(fromExp), null, false)
+ *  - `cast(fromExp, toType) >= value` ==> if(isnull(fromExp), null, false)
+ *  - `cast(fromExp, toType) === value` ==> if(isnull(fromExp), null, false)
+ *  - `cast(fromExp, toType) <=> value` ==> false (if `fromExp` is deterministic)
+ *  - `cast(fromExp, toType) <= value` ==> if(isnull(fromExp), null, true)
+ *  - `cast(fromExp, toType) < value` ==> if(isnull(fromExp), null, true)
+ *
+ * if `value == max`, the cases are of following:
+ *  - `cast(fromExp, toType) > value` ==> if(isnull(fromExp), null, false)
+ *  - `cast(fromExp, toType) >= value` ==> fromExp == max
+ *  - `cast(fromExp, toType) === value` ==> fromExp == max
+ *  - `cast(fromExp, toType) <=> value` ==> fromExp <=> max
+ *  - `cast(fromExp, toType) <= value` ==> if(isnull(fromExp), null, true)
+ *  - `cast(fromExp, toType) < value` ==> fromExp =!= max
+ *
+ * Similarly for the cases when `value == min` and `value < min`.
+ *
+ * Further, the above `if(isnull(fromExp), null, false)` is represented using conjunction
+ * `and(isnull(fromExp), null)`, to enable further optimization and filter pushdown to data sources.
+ * Similarly, `if(isnull(fromExp), null, true)` is represented with `or(isnotnull(fromExp), null)`.
+ */
+object UnwrapCastInBinaryComparison extends Rule[LogicalPlan] {
+  override def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsPattern(BINARY_COMPARISON), ruleId) {
+    case l: LogicalPlan =>
+      l.transformExpressionsUpWithPruning(_.containsPattern(BINARY_COMPARISON), ruleId) {
+        case e @ BinaryComparison(_, _) => unwrapCast(e)
+      }
+  }
+
+  private def unwrapCast(exp: Expression): Expression = exp match {
+    // Not a canonical form. In this case we first canonicalize the expression by swapping the
+    // literal and cast side, then process the result and swap the literal and cast again to
+    // restore the original order.
+    case BinaryComparison(Literal(_, literalType), Cast(fromExp, toType, _))
+        if canImplicitlyCast(fromExp, toType, literalType) =>
+      def swap(e: Expression): Expression = e match {
+        case GreaterThan(left, right) => LessThan(right, left)
+        case GreaterThanOrEqual(left, right) => LessThanOrEqual(right, left)
+        case EqualTo(left, right) => EqualTo(right, left)
+        case EqualNullSafe(left, right) => EqualNullSafe(right, left)
+        case LessThanOrEqual(left, right) => GreaterThanOrEqual(right, left)
+        case LessThan(left, right) => GreaterThan(right, left)
+        case _ => e
+      }
+
+      swap(unwrapCast(swap(exp)))
+
+    // In case both sides have numeric type, optimize the comparison by removing casts or
+    // moving cast to the literal side.
+    case be @ BinaryComparison(
+      Cast(fromExp, toType: NumericType, _), Literal(value, literalType))
+        if canImplicitlyCast(fromExp, toType, literalType) =>
+      simplifyNumericComparison(be, fromExp, toType, value)
+
+    case _ => exp
+  }
+
+  /**
+   * Check if the input `value` is within range `(min, max)` of the `fromType`, where `min` and
+   * `max` are the minimum and maximum value of the `fromType`. If the above is true, this
+   * optimizes the expression by moving the cast to the literal side. Otherwise if result is not
+   * true, this replaces the input binary comparison `exp` with simpler expressions.
+   */
+  private def simplifyNumericComparison(
+      exp: BinaryComparison,
+      fromExp: Expression,
+      toType: NumericType,
+      value: Any): Expression = {
+
+    val fromType = fromExp.dataType
+    val ordering = toType.ordering.asInstanceOf[Ordering[Any]]
+    val range = getRange(fromType)
+
+    if (range.isDefined) {
+      val (min, max) = range.get
+      val (minInToType, maxInToType) = {
+        (Cast(Literal(min), toType).eval(), Cast(Literal(max), toType).eval())
+      }
+      val minCmp = ordering.compare(value, minInToType)
+      val maxCmp = ordering.compare(value, maxInToType)
+
+      if (maxCmp >= 0 || minCmp <= 0) {
+        return if (maxCmp > 0) {
+          exp match {
+            case EqualTo(_, _) | GreaterThan(_, _) | GreaterThanOrEqual(_, _) =>
+              falseIfNotNull(fromExp)
+            case LessThan(_, _) | LessThanOrEqual(_, _) =>
+              trueIfNotNull(fromExp)
+            // make sure the expression is evaluated if it is non-deterministic
+            case EqualNullSafe(_, _) if exp.deterministic =>
+              FalseLiteral
+            case _ => exp
+          }
+        } else if (maxCmp == 0) {
+          exp match {
+            case GreaterThan(_, _) =>
+              falseIfNotNull(fromExp)
+            case LessThanOrEqual(_, _) =>
+              trueIfNotNull(fromExp)
+            case LessThan(_, _) =>
+              Not(EqualTo(fromExp, Literal(max, fromType)))
+            case GreaterThanOrEqual(_, _) | EqualTo(_, _) =>
+              EqualTo(fromExp, Literal(max, fromType))
+            case EqualNullSafe(_, _) =>
+              EqualNullSafe(fromExp, Literal(max, fromType))
+            case _ => exp
+          }
+        } else if (minCmp < 0) {
+          exp match {
+            case GreaterThan(_, _) | GreaterThanOrEqual(_, _) =>
+              trueIfNotNull(fromExp)
+            case LessThan(_, _) | LessThanOrEqual(_, _) | EqualTo(_, _) =>
+              falseIfNotNull(fromExp)
+            // make sure the expression is evaluated if it is non-deterministic
+            case EqualNullSafe(_, _) if exp.deterministic =>
+              FalseLiteral
+            case _ => exp
+          }
+        } else { // minCmp == 0
+          exp match {
+            case LessThan(_, _) =>
+              falseIfNotNull(fromExp)
+            case GreaterThanOrEqual(_, _) =>
+              trueIfNotNull(fromExp)
+            case GreaterThan(_, _) =>
+              Not(EqualTo(fromExp, Literal(min, fromType)))
+            case LessThanOrEqual(_, _) | EqualTo(_, _) =>
+              EqualTo(fromExp, Literal(min, fromType))
+            case EqualNullSafe(_, _) =>
+              EqualNullSafe(fromExp, Literal(min, fromType))
+            case _ => exp
+          }
+        }
+      }
+    }
+
+    // When we reach to this point, it means either there is no min/max for the `fromType` (e.g.,
+    // decimal type), or that the literal `value` is within range `(min, max)`. For these, we
+    // optimize by moving the cast to the literal side.
+
+    val newValue = Cast(Literal(value), fromType).eval()
+    if (newValue == null) {
+      // This means the cast failed, for instance, due to the value is not representable in the
+      // narrower type. In this case we simply return the original expression.
+      return exp
+    }
+    val valueRoundTrip = Cast(Literal(newValue, fromType), toType).eval()
+    val lit = Literal(newValue, fromType)
+    val cmp = ordering.compare(value, valueRoundTrip)
+    if (cmp == 0) {
+      exp match {
+        case GreaterThan(_, _) => GreaterThan(fromExp, lit)
+        case GreaterThanOrEqual(_, _) => GreaterThanOrEqual(fromExp, lit)
+        case EqualTo(_, _) => EqualTo(fromExp, lit)
+        case EqualNullSafe(_, _) => EqualNullSafe(fromExp, lit)
+        case LessThan(_, _) => LessThan(fromExp, lit)
+        case LessThanOrEqual(_, _) => LessThanOrEqual(fromExp, lit)
+        case _ => exp
+      }
+    } else if (cmp < 0) {
+      // This means the literal value is rounded up after casting to `fromType`
+      exp match {
+        case EqualTo(_, _) => falseIfNotNull(fromExp)
+        case EqualNullSafe(_, _) if fromExp.deterministic => FalseLiteral
+        case GreaterThan(_, _) | GreaterThanOrEqual(_, _) => GreaterThanOrEqual(fromExp, lit)
+        case LessThan(_, _) | LessThanOrEqual(_, _) => LessThan(fromExp, lit)
+        case _ => exp
+      }
+    } else {
+      // This means the literal value is rounded down after casting to `fromType`
+      exp match {
+        case EqualTo(_, _) => falseIfNotNull(fromExp)
+        case EqualNullSafe(_, _) => FalseLiteral
+        case GreaterThan(_, _) | GreaterThanOrEqual(_, _) => GreaterThan(fromExp, lit)
+        case LessThan(_, _) | LessThanOrEqual(_, _) => LessThanOrEqual(fromExp, lit)
+        case _ => exp
+      }
+    }
+  }
+
+  /**
+   * Check if the input `fromExp` can be safely cast to `toType` without any loss of precision,
+   * i.e., the conversion is injective. Note this only handles the case when both sides are of
+   * numeric type.
+   */
+  private def canImplicitlyCast(
+      fromExp: Expression,
+      toType: DataType,
+      literalType: DataType): Boolean = {
+    toType.sameType(literalType) &&
+      !fromExp.foldable &&
+      fromExp.dataType.isInstanceOf[NumericType] &&
+      toType.isInstanceOf[NumericType] &&
+      Cast.canUpCast(fromExp.dataType, toType)
+  }
+
+  private[optimizer] def getRange(dt: DataType): Option[(Any, Any)] = dt match {
+    case ByteType => Some((Byte.MinValue, Byte.MaxValue))
+    case ShortType => Some((Short.MinValue, Short.MaxValue))
+    case IntegerType => Some((Int.MinValue, Int.MaxValue))
+    case LongType => Some((Long.MinValue, Long.MaxValue))
+    case FloatType => Some((Float.NegativeInfinity, Float.NaN))
+    case DoubleType => Some((Double.NegativeInfinity, Double.NaN))
+    case _ => None
+  }
+
+  /**
+   * Wraps input expression `e` with `if(isnull(e), null, false)`. The if-clause is represented
+   * using `and(isnull(e), null)` which is semantically equivalent by applying 3-valued logic.
+   */
+  private[optimizer] def falseIfNotNull(e: Expression): Expression = {
+    And(IsNull(e), Literal(null, BooleanType))
+  }
+
+  /**
+   * Wraps input expression `e` with `if(isnull(e), null, true)`. The if-clause is represented
+   * using `or(isnotnull(e), null)` which is semantically equivalent by applying 3-valued logic.
+   */
+  private[optimizer] def trueIfNotNull(e: Expression): Expression = {
+    Or(IsNotNull(e), Literal(null, BooleanType))
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/UpdateFields.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/UpdateFields.scala
new file mode 100644
index 0000000000000..820a14a9b7786
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/UpdateFields.scala
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import java.util.Locale
+
+import scala.collection.mutable
+
+import org.apache.spark.sql.catalyst.expressions.{Expression, UpdateFields, WithField}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.AlwaysProcess
+
+
+/**
+ * Optimizes [[UpdateFields]] expression chains.
+ */
+object OptimizeUpdateFields extends Rule[LogicalPlan] {
+  private def canOptimize(names: Seq[String]): Boolean = {
+    if (conf.caseSensitiveAnalysis) {
+      names.distinct.length != names.length
+    } else {
+      names.map(_.toLowerCase(Locale.ROOT)).distinct.length != names.length
+    }
+  }
+
+  val optimizeUpdateFields: PartialFunction[Expression, Expression] = {
+    case UpdateFields(structExpr, fieldOps)
+      if fieldOps.forall(_.isInstanceOf[WithField]) &&
+        canOptimize(fieldOps.map(_.asInstanceOf[WithField].name)) =>
+      val caseSensitive = conf.caseSensitiveAnalysis
+
+      val withFields = fieldOps.map(_.asInstanceOf[WithField])
+      val names = withFields.map(_.name)
+      val values = withFields.map(_.valExpr)
+
+      val newNames = mutable.ArrayBuffer.empty[String]
+      val newValues = mutable.HashMap.empty[String, Expression]
+      // Used to remember the casing of the last instance
+      val nameMap = mutable.HashMap.empty[String, String]
+
+      names.zip(values).foreach { case (name, value) =>
+        val normalizedName = if (caseSensitive) name else name.toLowerCase(Locale.ROOT)
+        if (nameMap.contains(normalizedName)) {
+          newValues += normalizedName -> value
+        } else {
+          newNames += normalizedName
+          newValues += normalizedName -> value
+        }
+        nameMap += normalizedName -> name
+      }
+
+      val newWithFields = newNames.map(n => WithField(nameMap(n), newValues(n)))
+      UpdateFields(structExpr, newWithFields.toSeq)
+
+    case UpdateFields(UpdateFields(struct, fieldOps1), fieldOps2) =>
+      UpdateFields(struct, fieldOps1 ++ fieldOps2)
+  }
+
+  def apply(plan: LogicalPlan): LogicalPlan = plan.resolveExpressionsWithPruning(
+   AlwaysProcess.fn, ruleId)(optimizeUpdateFields)
+}
+
+/**
+ * Replaces [[UpdateFields]] expression with an evaluable expression.
+ */
+object ReplaceUpdateFieldsExpression extends Rule[LogicalPlan] {
+  def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions {
+    case u: UpdateFields => u.evalExpr
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
index bd400f86ea2c1..e9752e046a68f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
@@ -21,15 +21,17 @@ import scala.collection.immutable.HashSet
 import scala.collection.mutable.{ArrayBuffer, Stack}
 
 import org.apache.spark.sql.catalyst.analysis._
-import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, MultiLikeBase, _}
 import org.apache.spark.sql.catalyst.expressions.Literal.{FalseLiteral, TrueLiteral}
 import org.apache.spark.sql.catalyst.expressions.aggregate._
 import org.apache.spark.sql.catalyst.expressions.objects.AssertNotNull
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules._
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.catalyst.trees.AlwaysProcess
+import org.apache.spark.sql.catalyst.trees.TreePattern._
 import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.UTF8String
 
 /*
  * Optimization rules defined in this file should not affect the structure of the logical plan.
@@ -41,13 +43,27 @@ import org.apache.spark.sql.types._
  * equivalent [[Literal]] values.
  */
 object ConstantFolding extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
-    case q: LogicalPlan => q transformExpressionsDown {
+
+  private def hasNoSideEffect(e: Expression): Boolean = e match {
+    case _: Attribute => true
+    case _: Literal => true
+    case _: NoThrow if e.deterministic => e.children.forall(hasNoSideEffect)
+    case _ => false
+  }
+
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(AlwaysProcess.fn, ruleId) {
+    case q: LogicalPlan => q.transformExpressionsDownWithPruning(
+      AlwaysProcess.fn, ruleId) {
       // Skip redundant folding of literals. This rule is technically not necessary. Placing this
       // here avoids running the next rule for Literal values, which would create a new Literal
       // object and running eval unnecessarily.
       case l: Literal => l
 
+      case Size(c: CreateArray, _) if c.children.forall(hasNoSideEffect) =>
+        Literal(c.children.length)
+      case Size(c: CreateMap, _) if c.children.forall(hasNoSideEffect) =>
+        Literal(c.children.length / 2)
+
       // Fold expressions that are foldable.
       case e if e.foldable => Literal.create(e.eval(EmptyRow), e.dataType)
     }
@@ -57,7 +73,7 @@ object ConstantFolding extends Rule[LogicalPlan] {
 /**
  * Substitutes [[Attribute Attributes]] which can be statically evaluated with their corresponding
  * value in conjunctive [[Expression Expressions]]
- * eg.
+ * e.g.
  * {{{
  *   SELECT * FROM table WHERE i = 5 AND j = i + 3
  *   ==>  SELECT * FROM table WHERE i = 5 AND j = 8
@@ -69,7 +85,8 @@ object ConstantFolding extends Rule[LogicalPlan] {
  *   in the AND node.
  */
 object ConstantPropagation extends Rule[LogicalPlan] with PredicateHelper {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformUpWithPruning(
+    _.containsAllPatterns(LITERAL, FILTER), ruleId) {
     case f: Filter =>
       val (newCondition, _) = traverse(f.condition, replaceChildren = true, nullIsFalse = true)
       if (newCondition.isDefined) {
@@ -177,7 +194,7 @@ object ReorderAssociativeOperator extends Rule[LogicalPlan] {
   private def flattenAdd(
     expression: Expression,
     groupSet: ExpressionSet): Seq[Expression] = expression match {
-    case expr @ Add(l, r) if !groupSet.contains(expr) =>
+    case expr @ Add(l, r, _) if !groupSet.contains(expr) =>
       flattenAdd(l, groupSet) ++ flattenAdd(r, groupSet)
     case other => other :: Nil
   }
@@ -185,7 +202,7 @@ object ReorderAssociativeOperator extends Rule[LogicalPlan] {
   private def flattenMultiply(
     expression: Expression,
     groupSet: ExpressionSet): Seq[Expression] = expression match {
-    case expr @ Multiply(l, r) if !groupSet.contains(expr) =>
+    case expr @ Multiply(l, r, _) if !groupSet.contains(expr) =>
       flattenMultiply(l, groupSet) ++ flattenMultiply(r, groupSet)
     case other => other :: Nil
   }
@@ -196,28 +213,30 @@ object ReorderAssociativeOperator extends Rule[LogicalPlan] {
     case _ => ExpressionSet(Seq.empty)
   }
 
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsPattern(BINARY_ARITHMETIC), ruleId) {
     case q: LogicalPlan =>
       // We have to respect aggregate expressions which exists in grouping expressions when plan
       // is an Aggregate operator, otherwise the optimized expression could not be derived from
       // grouping expressions.
+      // TODO: do not reorder consecutive `Add`s or `Multiply`s with different `failOnError` flags
       val groupingExpressionSet = collectGroupingExpressions(q)
-      q transformExpressionsDown {
-      case a: Add if a.deterministic && a.dataType.isInstanceOf[IntegralType] =>
+      q.transformExpressionsDownWithPruning(_.containsPattern(BINARY_ARITHMETIC)) {
+      case a @ Add(_, _, f) if a.deterministic && a.dataType.isInstanceOf[IntegralType] =>
         val (foldables, others) = flattenAdd(a, groupingExpressionSet).partition(_.foldable)
         if (foldables.size > 1) {
-          val foldableExpr = foldables.reduce((x, y) => Add(x, y))
+          val foldableExpr = foldables.reduce((x, y) => Add(x, y, f))
           val c = Literal.create(foldableExpr.eval(EmptyRow), a.dataType)
-          if (others.isEmpty) c else Add(others.reduce((x, y) => Add(x, y)), c)
+          if (others.isEmpty) c else Add(others.reduce((x, y) => Add(x, y, f)), c, f)
         } else {
           a
         }
-      case m: Multiply if m.deterministic && m.dataType.isInstanceOf[IntegralType] =>
+      case m @ Multiply(_, _, f) if m.deterministic && m.dataType.isInstanceOf[IntegralType] =>
         val (foldables, others) = flattenMultiply(m, groupingExpressionSet).partition(_.foldable)
         if (foldables.size > 1) {
-          val foldableExpr = foldables.reduce((x, y) => Multiply(x, y))
+          val foldableExpr = foldables.reduce((x, y) => Multiply(x, y, f))
           val c = Literal.create(foldableExpr.eval(EmptyRow), m.dataType)
-          if (others.isEmpty) c else Multiply(others.reduce((x, y) => Multiply(x, y)), c)
+          if (others.isEmpty) c else Multiply(others.reduce((x, y) => Multiply(x, y, f)), c, f)
         } else {
           m
         }
@@ -235,8 +254,9 @@ object ReorderAssociativeOperator extends Rule[LogicalPlan] {
  *    [[InSet (value, HashSet[Literal])]] which is much faster.
  */
 object OptimizeIn extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
-    case q: LogicalPlan => q transformExpressionsDown {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsPattern(IN), ruleId) {
+    case q: LogicalPlan => q.transformExpressionsDownWithPruning(_.containsPattern(IN), ruleId) {
       case In(v, list) if list.isEmpty =>
         // When v is not nullable, the following expression will be optimized
         // to FalseLiteral which is tested in OptimizeInSuite.scala
@@ -249,7 +269,7 @@ object OptimizeIn extends Rule[LogicalPlan] {
           && !v.isInstanceOf[CreateNamedStruct]
           && !newList.head.isInstanceOf[CreateNamedStruct]) {
           EqualTo(v, newList.head)
-        } else if (newList.length > SQLConf.get.optimizerInSetConversionThreshold) {
+        } else if (newList.length > conf.optimizerInSetConversionThreshold) {
           val hSet = newList.map(e => e.eval(EmptyRow))
           InSet(v, HashSet() ++ hSet)
         } else if (newList.length < list.length) {
@@ -270,8 +290,10 @@ object OptimizeIn extends Rule[LogicalPlan] {
  * 4. Removes `Not` operator.
  */
 object BooleanSimplification extends Rule[LogicalPlan] with PredicateHelper {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
-    case q: LogicalPlan => q transformExpressionsUp {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsAnyPattern(AND_OR, NOT), ruleId) {
+    case q: LogicalPlan => q.transformExpressionsUpWithPruning(
+      _.containsAnyPattern(AND_OR, NOT), ruleId) {
       case TrueLiteral And e => e
       case e And TrueLiteral => e
       case FalseLiteral Or e => e
@@ -330,17 +352,18 @@ object BooleanSimplification extends Rule[LogicalPlan] with PredicateHelper {
       // Common factor elimination for conjunction
       case and @ (left And right) =>
         // 1. Split left and right to get the disjunctive predicates,
-        //   i.e. lhs = (a, b), rhs = (a, c)
+        //    i.e. lhs = (a || b), rhs = (a || c)
         // 2. Find the common predict between lhsSet and rhsSet, i.e. common = (a)
         // 3. Remove common predict from lhsSet and rhsSet, i.e. ldiff = (b), rdiff = (c)
-        // 4. Apply the formula, get the optimized predicate: common || (ldiff && rdiff)
+        // 4. If common is non-empty, apply the formula to get the optimized predicate:
+        //    common || (ldiff && rdiff)
+        // 5. Else if common is empty, split left and right to get the conjunctive predicates.
+        //    for example lhs = (a && b), rhs = (a && c) => all = (a, b, a, c), distinct = (a, b, c)
+        //    optimized predicate: (a && b && c)
         val lhs = splitDisjunctivePredicates(left)
         val rhs = splitDisjunctivePredicates(right)
         val common = lhs.filter(e => rhs.exists(e.semanticEquals))
-        if (common.isEmpty) {
-          // No common factors, return the original predicate
-          and
-        } else {
+        if (common.nonEmpty) {
           val ldiff = lhs.filterNot(e => common.exists(e.semanticEquals))
           val rdiff = rhs.filterNot(e => common.exists(e.semanticEquals))
           if (ldiff.isEmpty || rdiff.isEmpty) {
@@ -348,25 +371,37 @@ object BooleanSimplification extends Rule[LogicalPlan] with PredicateHelper {
             common.reduce(Or)
           } else {
             // (a || b || c || ...) && (a || b || d || ...) =>
-            // ((c || ...) && (d || ...)) || a || b
+            // a || b || ((c || ...) && (d || ...))
             (common :+ And(ldiff.reduce(Or), rdiff.reduce(Or))).reduce(Or)
           }
+        } else {
+          // No common factors from disjunctive predicates, reduce common factor from conjunction
+          val all = splitConjunctivePredicates(left) ++ splitConjunctivePredicates(right)
+          val distinct = ExpressionSet(all)
+          if (all.size == distinct.size) {
+            // No common factors, return the original predicate
+            and
+          } else {
+            // (a && b) && a && (a && c) => a && b && c
+            buildBalancedPredicate(distinct.toSeq, And)
+          }
         }
 
       // Common factor elimination for disjunction
       case or @ (left Or right) =>
         // 1. Split left and right to get the conjunctive predicates,
-        //   i.e.  lhs = (a, b), rhs = (a, c)
+        //    i.e.  lhs = (a && b), rhs = (a && c)
         // 2. Find the common predict between lhsSet and rhsSet, i.e. common = (a)
         // 3. Remove common predict from lhsSet and rhsSet, i.e. ldiff = (b), rdiff = (c)
-        // 4. Apply the formula, get the optimized predicate: common && (ldiff || rdiff)
+        // 4. If common is non-empty, apply the formula to get the optimized predicate:
+        //    common && (ldiff || rdiff)
+        // 5. Else if common is empty, split left and right to get the conjunctive predicates.
+        // for example lhs = (a || b), rhs = (a || c) => all = (a, b, a, c), distinct = (a, b, c)
+        // optimized predicate: (a || b || c)
         val lhs = splitConjunctivePredicates(left)
         val rhs = splitConjunctivePredicates(right)
         val common = lhs.filter(e => rhs.exists(e.semanticEquals))
-        if (common.isEmpty) {
-          // No common factors, return the original predicate
-          or
-        } else {
+        if (common.nonEmpty) {
           val ldiff = lhs.filterNot(e => common.exists(e.semanticEquals))
           val rdiff = rhs.filterNot(e => common.exists(e.semanticEquals))
           if (ldiff.isEmpty || rdiff.isEmpty) {
@@ -374,9 +409,20 @@ object BooleanSimplification extends Rule[LogicalPlan] with PredicateHelper {
             common.reduce(And)
           } else {
             // (a && b && c && ...) || (a && b && d && ...) =>
-            // ((c && ...) || (d && ...)) && a && b
+            // a && b && ((c && ...) || (d && ...))
             (common :+ Or(ldiff.reduce(And), rdiff.reduce(And))).reduce(And)
           }
+        } else {
+          // No common factors in conjunctive predicates, reduce common factor from disjunction
+          val all = splitDisjunctivePredicates(left) ++ splitDisjunctivePredicates(right)
+          val distinct = ExpressionSet(all)
+          if (all.size == distinct.size) {
+            // No common factors, return the original predicate
+            or
+          } else {
+            // (a || b) || a || (a || c) => a || b || c
+            buildBalancedPredicate(distinct.toSeq, Or)
+          }
         }
 
       case Not(TrueLiteral) => FalseLiteral
@@ -420,7 +466,8 @@ object SimplifyBinaryComparison
     }
   }
 
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsPattern(BINARY_COMPARISON), ruleId) {
     case l: LogicalPlan =>
       lazy val notNullExpressions = ExpressionSet(l match {
         case Filter(fc, _) =>
@@ -430,7 +477,7 @@ object SimplifyBinaryComparison
         case _ => Seq.empty
       })
 
-      l transformExpressionsUp {
+      l.transformExpressionsUpWithPruning(_.containsPattern(BINARY_COMPARISON)) {
         // True with equality
         case a EqualNullSafe b if a.semanticEquals(b) => TrueLiteral
         case a EqualTo b if canSimplifyComparison(a, b, notNullExpressions) => TrueLiteral
@@ -456,13 +503,27 @@ object SimplifyConditionals extends Rule[LogicalPlan] with PredicateHelper {
     case _ => false
   }
 
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsAnyPattern(IF, CASE_WHEN), ruleId) {
     case q: LogicalPlan => q transformExpressionsUp {
       case If(TrueLiteral, trueValue, _) => trueValue
       case If(FalseLiteral, _, falseValue) => falseValue
       case If(Literal(null, _), _, falseValue) => falseValue
+      case If(cond, TrueLiteral, FalseLiteral) =>
+        if (cond.nullable) EqualNullSafe(cond, TrueLiteral) else cond
+      case If(cond, FalseLiteral, TrueLiteral) =>
+        if (cond.nullable) Not(EqualNullSafe(cond, TrueLiteral)) else Not(cond)
       case If(cond, trueValue, falseValue)
         if cond.deterministic && trueValue.semanticEquals(falseValue) => trueValue
+      case If(cond, l @ Literal(null, _), FalseLiteral) if !cond.nullable => And(cond, l)
+      case If(cond, l @ Literal(null, _), TrueLiteral) if !cond.nullable => Or(Not(cond), l)
+      case If(cond, FalseLiteral, l @ Literal(null, _)) if !cond.nullable => And(Not(cond), l)
+      case If(cond, TrueLiteral, l @ Literal(null, _)) if !cond.nullable => Or(cond, l)
+
+      case CaseWhen(Seq((cond, TrueLiteral)), Some(FalseLiteral)) =>
+        if (cond.nullable) EqualNullSafe(cond, TrueLiteral) else cond
+      case CaseWhen(Seq((cond, FalseLiteral)), Some(TrueLiteral)) =>
+        if (cond.nullable) Not(EqualNullSafe(cond, TrueLiteral)) else Not(cond)
 
       case e @ CaseWhen(branches, elseValue) if branches.exists(x => falseOrNullLiteral(x._1)) =>
         // If there are branches that are always false, remove them.
@@ -488,8 +549,9 @@ object SimplifyConditionals extends Rule[LogicalPlan] with PredicateHelper {
         val (h, t) = branches.span(_._1 != TrueLiteral)
         CaseWhen( h :+ t.head, None)
 
-      case e @ CaseWhen(branches, Some(elseValue))
-          if branches.forall(_._2.semanticEquals(elseValue)) =>
+      case e @ CaseWhen(branches, elseOpt)
+          if branches.forall(_._2.semanticEquals(elseOpt.getOrElse(Literal(null, e.dataType)))) =>
+        val elseValue = elseOpt.getOrElse(Literal(null, e.dataType))
         // For non-deterministic conditions with side effect, we can not remove it, or change
         // the ordering. As a result, we try to remove the deterministic conditions from the tail.
         var hitNonDeterministicCond = false
@@ -510,6 +572,91 @@ object SimplifyConditionals extends Rule[LogicalPlan] with PredicateHelper {
 }
 
 
+/**
+ * Push the foldable expression into (if / case) branches.
+ */
+object PushFoldableIntoBranches extends Rule[LogicalPlan] with PredicateHelper {
+
+  // To be conservative here: it's only a guaranteed win if all but at most only one branch
+  // end up being not foldable.
+  private def atMostOneUnfoldable(exprs: Seq[Expression]): Boolean = {
+    val (foldables, others) = exprs.partition(_.foldable)
+    foldables.nonEmpty && others.length < 2
+  }
+
+  // Not all UnaryExpression can be pushed into (if / case) branches, e.g. Alias.
+  private def supportedUnaryExpression(e: UnaryExpression): Boolean = e match {
+    case _: IsNull | _: IsNotNull => true
+    case _: UnaryMathExpression | _: Abs | _: Bin | _: Factorial | _: Hex => true
+    case _: String2StringExpression | _: Ascii | _: Base64 | _: BitLength | _: Chr | _: Length =>
+      true
+    case _: CastBase => true
+    case _: GetDateField | _: LastDay => true
+    case _: ExtractIntervalPart[_] => true
+    case _: ArraySetLike => true
+    case _: ExtractValue => true
+    case _ => false
+  }
+
+  // Not all BinaryExpression can be pushed into (if / case) branches.
+  private def supportedBinaryExpression(e: BinaryExpression): Boolean = e match {
+    case _: BinaryComparison | _: StringPredicate | _: StringRegexExpression => true
+    case _: BinaryArithmetic => true
+    case _: BinaryMathExpression => true
+    case _: AddMonths | _: DateAdd | _: DateAddInterval | _: DateDiff | _: DateSub |
+         _: DateAddYMInterval | _: TimestampAddYMInterval | _: TimeAdd => true
+    case _: FindInSet | _: RoundBase => true
+    case _ => false
+  }
+
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsAnyPattern(CASE_WHEN, IF), ruleId) {
+    case q: LogicalPlan => q.transformExpressionsUpWithPruning(
+      _.containsAnyPattern(CASE_WHEN, IF), ruleId) {
+      case u @ UnaryExpression(i @ If(_, trueValue, falseValue))
+          if supportedUnaryExpression(u) && atMostOneUnfoldable(Seq(trueValue, falseValue)) =>
+        i.copy(
+          trueValue = u.withNewChildren(Array(trueValue)),
+          falseValue = u.withNewChildren(Array(falseValue)))
+
+      case u @ UnaryExpression(c @ CaseWhen(branches, elseValue))
+          if supportedUnaryExpression(u) && atMostOneUnfoldable(branches.map(_._2) ++ elseValue) =>
+        c.copy(
+          branches.map(e => e.copy(_2 = u.withNewChildren(Array(e._2)))),
+          elseValue.map(e => u.withNewChildren(Array(e))))
+
+      case b @ BinaryExpression(i @ If(_, trueValue, falseValue), right)
+          if supportedBinaryExpression(b) && right.foldable &&
+            atMostOneUnfoldable(Seq(trueValue, falseValue)) =>
+        i.copy(
+          trueValue = b.withNewChildren(Array(trueValue, right)),
+          falseValue = b.withNewChildren(Array(falseValue, right)))
+
+      case b @ BinaryExpression(left, i @ If(_, trueValue, falseValue))
+          if supportedBinaryExpression(b) && left.foldable &&
+            atMostOneUnfoldable(Seq(trueValue, falseValue)) =>
+        i.copy(
+          trueValue = b.withNewChildren(Array(left, trueValue)),
+          falseValue = b.withNewChildren(Array(left, falseValue)))
+
+      case b @ BinaryExpression(c @ CaseWhen(branches, elseValue), right)
+          if supportedBinaryExpression(b) && right.foldable &&
+            atMostOneUnfoldable(branches.map(_._2) ++ elseValue) =>
+        c.copy(
+          branches.map(e => e.copy(_2 = b.withNewChildren(Array(e._2, right)))),
+          elseValue.map(e => b.withNewChildren(Array(e, right))))
+
+      case b @ BinaryExpression(left, c @ CaseWhen(branches, elseValue))
+          if supportedBinaryExpression(b) && left.foldable &&
+            atMostOneUnfoldable(branches.map(_._2) ++ elseValue) =>
+        c.copy(
+          branches.map(e => e.copy(_2 = b.withNewChildren(Array(left, e._2)))),
+          elseValue.map(e => b.withNewChildren(Array(left, e))))
+    }
+  }
+}
+
+
 /**
  * Simplifies LIKE expressions that do not need full regular expressions to evaluate the condition.
  * For example, when the expression is just checking to see if a string starts with a given
@@ -524,30 +671,71 @@ object LikeSimplification extends Rule[LogicalPlan] {
   private val contains = "%([^_%]+)%".r
   private val equalTo = "([^_%]*)".r
 
-  def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions {
-    case Like(input, Literal(pattern, StringType), escapeChar) =>
+  private def simplifyLike(
+      input: Expression, pattern: String, escapeChar: Char = '\\'): Option[Expression] = {
+    if (pattern.contains(escapeChar)) {
+      // There are three different situations when pattern containing escapeChar:
+      // 1. pattern contains invalid escape sequence, e.g. 'm\aca'
+      // 2. pattern contains escaped wildcard character, e.g. 'ma\%ca'
+      // 3. pattern contains escaped escape character, e.g. 'ma\\ca'
+      // Although there are patterns can be optimized if we handle the escape first, we just
+      // skip this rule if pattern contains any escapeChar for simplicity.
+      None
+    } else {
+      pattern match {
+        case startsWith(prefix) =>
+          Some(StartsWith(input, Literal(prefix)))
+        case endsWith(postfix) =>
+          Some(EndsWith(input, Literal(postfix)))
+        // 'a%a' pattern is basically same with 'a%' && '%a'.
+        // However, the additional `Length` condition is required to prevent 'a' match 'a%a'.
+        case startsAndEndsWith(prefix, postfix) =>
+          Some(And(GreaterThanOrEqual(Length(input), Literal(prefix.length + postfix.length)),
+            And(StartsWith(input, Literal(prefix)), EndsWith(input, Literal(postfix)))))
+        case contains(infix) =>
+          Some(Contains(input, Literal(infix)))
+        case equalTo(str) =>
+          Some(EqualTo(input, Literal(str)))
+        case _ => None
+      }
+    }
+  }
+
+  private def simplifyMultiLike(
+      child: Expression, patterns: Seq[UTF8String], multi: MultiLikeBase): Expression = {
+    val (remainPatternMap, replacementMap) =
+      patterns.map { p =>
+        p -> Option(p).flatMap(p => simplifyLike(child, p.toString))
+      }.partition(_._2.isEmpty)
+    val remainPatterns = remainPatternMap.map(_._1)
+    val replacements = replacementMap.map(_._2.get)
+    if (replacements.isEmpty) {
+      multi
+    } else {
+      multi match {
+        case l: LikeAll => And(replacements.reduceLeft(And), l.copy(patterns = remainPatterns))
+        case l: NotLikeAll =>
+          And(replacements.map(Not(_)).reduceLeft(And), l.copy(patterns = remainPatterns))
+        case l: LikeAny => Or(replacements.reduceLeft(Or), l.copy(patterns = remainPatterns))
+        case l: NotLikeAny =>
+          Or(replacements.map(Not(_)).reduceLeft(Or), l.copy(patterns = remainPatterns))
+      }
+    }
+  }
+
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformAllExpressionsWithPruning(
+    _.containsPattern(LIKE_FAMLIY), ruleId) {
+    case l @ Like(input, Literal(pattern, StringType), escapeChar) =>
       if (pattern == null) {
         // If pattern is null, return null value directly, since "col like null" == null.
         Literal(null, BooleanType)
       } else {
-        val escapeStr = String.valueOf(escapeChar)
-        pattern.toString match {
-          case startsWith(prefix) if !prefix.endsWith(escapeStr) =>
-            StartsWith(input, Literal(prefix))
-          case endsWith(postfix) =>
-            EndsWith(input, Literal(postfix))
-          // 'a%a' pattern is basically same with 'a%' && '%a'.
-          // However, the additional `Length` condition is required to prevent 'a' match 'a%a'.
-          case startsAndEndsWith(prefix, postfix) if !prefix.endsWith(escapeStr) =>
-            And(GreaterThanOrEqual(Length(input), Literal(prefix.length + postfix.length)),
-              And(StartsWith(input, Literal(prefix)), EndsWith(input, Literal(postfix))))
-          case contains(infix) if !infix.endsWith(escapeStr) =>
-            Contains(input, Literal(infix))
-          case equalTo(str) =>
-            EqualTo(input, Literal(str))
-          case _ => Like(input, Literal.create(pattern, StringType), escapeChar)
-        }
+        simplifyLike(input, pattern.toString, escapeChar).getOrElse(l)
       }
+    case l @ LikeAll(child, patterns) => simplifyMultiLike(child, patterns, l)
+    case l @ NotLikeAll(child, patterns) => simplifyMultiLike(child, patterns, l)
+    case l @ LikeAny(child, patterns) => simplifyMultiLike(child, patterns, l)
+    case l @ NotLikeAny(child, patterns) => simplifyMultiLike(child, patterns, l)
   }
 }
 
@@ -563,12 +751,16 @@ object NullPropagation extends Rule[LogicalPlan] {
     case _ => false
   }
 
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
-    case q: LogicalPlan => q transformExpressionsUp {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    t => t.containsAnyPattern(NULL_CHECK, NULL_LITERAL, COUNT)
+      || t.containsAllPatterns(WINDOW_EXPRESSION, CAST, LITERAL), ruleId) {
+    case q: LogicalPlan => q.transformExpressionsUpWithPruning(
+      t => t.containsAnyPattern(NULL_CHECK, NULL_LITERAL, COUNT)
+        || t.containsAllPatterns(WINDOW_EXPRESSION, CAST, LITERAL), ruleId) {
       case e @ WindowExpression(Cast(Literal(0L, _), _, _), _) =>
-        Cast(Literal(0L), e.dataType, Option(SQLConf.get.sessionLocalTimeZone))
+        Cast(Literal(0L), e.dataType, Option(conf.sessionLocalTimeZone))
       case e @ AggregateExpression(Count(exprs), _, _, _, _) if exprs.forall(isNullLiteral) =>
-        Cast(Literal(0L), e.dataType, Option(SQLConf.get.sessionLocalTimeZone))
+        Cast(Literal(0L), e.dataType, Option(conf.sessionLocalTimeZone))
       case ae @ AggregateExpression(Count(exprs), _, false, _, _) if !exprs.exists(_.nullable) =>
         // This rule should be only triggered when isDistinct field is false.
         ae.copy(aggregateFunction = Count(Literal(1)))
@@ -620,70 +812,105 @@ object NullPropagation extends Rule[LogicalPlan] {
  */
 object FoldablePropagation extends Rule[LogicalPlan] {
   def apply(plan: LogicalPlan): LogicalPlan = {
-    var foldableMap = AttributeMap(plan.flatMap {
-      case Project(projectList, _) => projectList.collect {
-        case a: Alias if a.child.foldable => (a.toAttribute, a)
-      }
-      case _ => Nil
-    })
-    val replaceFoldable: PartialFunction[Expression, Expression] = {
-      case a: AttributeReference if foldableMap.contains(a) => foldableMap(a)
+    CleanupAliases(propagateFoldables(plan)._1)
+  }
+
+  private def propagateFoldables(plan: LogicalPlan): (LogicalPlan, AttributeMap[Alias]) = {
+    plan match {
+      case p: Project =>
+        val (newChild, foldableMap) = propagateFoldables(p.child)
+        val newProject =
+          replaceFoldable(p.withNewChildren(Seq(newChild)).asInstanceOf[Project], foldableMap)
+        val newFoldableMap = collectFoldables(newProject.projectList)
+        (newProject, newFoldableMap)
+
+      case a: Aggregate =>
+        val (newChild, foldableMap) = propagateFoldables(a.child)
+        val newAggregate =
+          replaceFoldable(a.withNewChildren(Seq(newChild)).asInstanceOf[Aggregate], foldableMap)
+        val newFoldableMap = collectFoldables(newAggregate.aggregateExpressions)
+        (newAggregate, newFoldableMap)
+
+      // We can not replace the attributes in `Expand.output`. If there are other non-leaf
+      // operators that have the `output` field, we should put them here too.
+      case e: Expand =>
+        val (newChild, foldableMap) = propagateFoldables(e.child)
+        val expandWithNewChildren = e.withNewChildren(Seq(newChild)).asInstanceOf[Expand]
+        val newExpand = if (foldableMap.isEmpty) {
+          expandWithNewChildren
+        } else {
+          val newProjections = expandWithNewChildren.projections.map(_.map(_.transform {
+            case a: AttributeReference if foldableMap.contains(a) => foldableMap(a)
+          }))
+          if (newProjections == expandWithNewChildren.projections) {
+            expandWithNewChildren
+          } else {
+            expandWithNewChildren.copy(projections = newProjections)
+          }
+        }
+        (newExpand, foldableMap)
+
+      case u: UnaryNode if canPropagateFoldables(u) =>
+        val (newChild, foldableMap) = propagateFoldables(u.child)
+        val newU = replaceFoldable(u.withNewChildren(Seq(newChild)), foldableMap)
+        (newU, foldableMap)
+
+      // Join derives the output attributes from its child while they are actually not the
+      // same attributes. For example, the output of outer join is not always picked from its
+      // children, but can also be null. We should exclude these miss-derived attributes when
+      // propagating the foldable expressions.
+      // TODO(cloud-fan): It seems more reasonable to use new attributes as the output attributes
+      // of outer join.
+      case j: Join =>
+        val (newChildren, foldableMaps) = j.children.map(propagateFoldables).unzip
+        val foldableMap = AttributeMap(
+          foldableMaps.foldLeft(Iterable.empty[(Attribute, Alias)])(_ ++ _.baseMap.values).toSeq)
+        val newJoin =
+          replaceFoldable(j.withNewChildren(newChildren).asInstanceOf[Join], foldableMap)
+        val missDerivedAttrsSet: AttributeSet = AttributeSet(newJoin.joinType match {
+          case _: InnerLike | LeftExistence(_) => Nil
+          case LeftOuter => newJoin.right.output
+          case RightOuter => newJoin.left.output
+          case FullOuter => newJoin.left.output ++ newJoin.right.output
+          case _ => Nil
+        })
+        val newFoldableMap = AttributeMap(foldableMap.baseMap.values.filterNot {
+          case (attr, _) => missDerivedAttrsSet.contains(attr)
+        }.toSeq)
+        (newJoin, newFoldableMap)
+
+      // For other plans, they are not safe to apply foldable propagation, and they should not
+      // propagate foldable expressions from children.
+      case o =>
+        val newOther = o.mapChildren(propagateFoldables(_)._1)
+        (newOther, AttributeMap.empty)
     }
+  }
 
+  private def replaceFoldable(plan: LogicalPlan, foldableMap: AttributeMap[Alias]): plan.type = {
     if (foldableMap.isEmpty) {
       plan
     } else {
-      CleanupAliases(plan.transformUp {
-        // We can only propagate foldables for a subset of unary nodes.
-        case u: UnaryNode if foldableMap.nonEmpty && canPropagateFoldables(u) =>
-          u.transformExpressions(replaceFoldable)
-
-        // Join derives the output attributes from its child while they are actually not the
-        // same attributes. For example, the output of outer join is not always picked from its
-        // children, but can also be null. We should exclude these miss-derived attributes when
-        // propagating the foldable expressions.
-        // TODO(cloud-fan): It seems more reasonable to use new attributes as the output attributes
-        // of outer join.
-        case j @ Join(left, right, joinType, _, _) if foldableMap.nonEmpty =>
-          val newJoin = j.transformExpressions(replaceFoldable)
-          val missDerivedAttrsSet: AttributeSet = AttributeSet(joinType match {
-            case _: InnerLike | LeftExistence(_) => Nil
-            case LeftOuter => right.output
-            case RightOuter => left.output
-            case FullOuter => left.output ++ right.output
-          })
-          foldableMap = AttributeMap(foldableMap.baseMap.values.filterNot {
-            case (attr, _) => missDerivedAttrsSet.contains(attr)
-          }.toSeq)
-          newJoin
-
-        // We can not replace the attributes in `Expand.output`. If there are other non-leaf
-        // operators that have the `output` field, we should put them here too.
-        case expand: Expand if foldableMap.nonEmpty =>
-          expand.copy(projections = expand.projections.map { projection =>
-            projection.map(_.transform(replaceFoldable))
-          })
-
-        // For other plans, they are not safe to apply foldable propagation, and they should not
-        // propagate foldable expressions from children.
-        case other if foldableMap.nonEmpty =>
-          val childrenOutputSet = AttributeSet(other.children.flatMap(_.output))
-          foldableMap = AttributeMap(foldableMap.baseMap.values.filterNot {
-            case (attr, _) => childrenOutputSet.contains(attr)
-          }.toSeq)
-          other
-      })
+      plan transformExpressions {
+        case a: AttributeReference if foldableMap.contains(a) => foldableMap(a)
+      }
     }
   }
 
+  private def collectFoldables(expressions: Seq[NamedExpression]) = {
+    AttributeMap(expressions.collect {
+      case a: Alias if a.child.foldable => (a.toAttribute, a)
+    })
+  }
+
   /**
-   * Whitelist of all [[UnaryNode]]s for which allow foldable propagation.
+   * List of all [[UnaryNode]]s which allow foldable propagation.
    */
   private def canPropagateFoldables(u: UnaryNode): Boolean = u match {
-    case _: Project => true
+    // Handling `Project` is moved to `propagateFoldables`.
     case _: Filter => true
     case _: SubqueryAlias => true
-    case _: Aggregate => true
+    // Handling `Aggregate` is moved to `propagateFoldables`.
     case _: Window => true
     case _: Sample => true
     case _: GlobalLimit => true
@@ -705,7 +932,8 @@ object FoldablePropagation extends Rule[LogicalPlan] {
  * Removes [[Cast Casts]] that are unnecessary because the input is already the correct type.
  */
 object SimplifyCasts extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformAllExpressionsWithPruning(
+    _.containsPattern(CAST), ruleId) {
     case Cast(e, dataType, _) if e.dataType == dataType => e
     case c @ Cast(e, dataType, _) => (e.dataType, dataType) match {
       case (ArrayType(from, false), ArrayType(to, true)) if from == to => e
@@ -721,7 +949,8 @@ object SimplifyCasts extends Rule[LogicalPlan] {
  * Removes nodes that are not necessary.
  */
 object RemoveDispensableExpressions extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformAllExpressionsWithPruning(
+    _.containsPattern(UNARY_POSITIVE), ruleId) {
     case UnaryPositive(child) => child
   }
 }
@@ -732,8 +961,10 @@ object RemoveDispensableExpressions extends Rule[LogicalPlan] {
  * the inner conversion is overwritten by the outer one.
  */
 object SimplifyCaseConversionExpressions extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
-    case q: LogicalPlan => q transformExpressionsUp {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsPattern(UPPER_OR_LOWER), ruleId) {
+    case q: LogicalPlan => q.transformExpressionsUpWithPruning(
+      _.containsPattern(UPPER_OR_LOWER), ruleId) {
       case Upper(Upper(child)) => Upper(child)
       case Upper(Lower(child)) => Upper(child)
       case Lower(Upper(child)) => Lower(child)
@@ -765,7 +996,7 @@ object CombineConcats extends Rule[LogicalPlan] {
           flattened += child
       }
     }
-    Concat(flattened)
+    Concat(flattened.toSeq)
   }
 
   private def hasNestedConcats(concat: Concat): Boolean = concat.children.exists {
@@ -774,7 +1005,8 @@ object CombineConcats extends Rule[LogicalPlan] {
     case _ => false
   }
 
-  def apply(plan: LogicalPlan): LogicalPlan = plan.transformExpressionsDown {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformAllExpressionsWithPruning(
+    _.containsPattern(CONCAT), ruleId) {
     case concat: Concat if hasNestedConcats(concat) =>
       flattenConcats(concat)
   }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
index 80d85827657fd..752b316bc6f63 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
@@ -17,14 +17,13 @@
 
 package org.apache.spark.sql.catalyst.optimizer
 
-import java.time.LocalDate
-
 import scala.collection.mutable
 
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate._
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules._
+import org.apache.spark.sql.catalyst.trees.TreePattern._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.connector.catalog.CatalogManager
 import org.apache.spark.sql.types._
@@ -44,7 +43,8 @@ import org.apache.spark.sql.types._
  * how RuntimeReplaceable does.
  */
 object ReplaceExpressions extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformAllExpressionsWithPruning(
+    _.containsAnyPattern(RUNTIME_REPLACEABLE, COUNT_IF, BOOL_AGG)) {
     case e: RuntimeReplaceable => e.child
     case CountIf(predicate) => Count(new NullIf(predicate, Literal.FalseLiteral))
     case BoolOr(arg) => Max(arg)
@@ -59,7 +59,8 @@ object ReplaceExpressions extends Rule[LogicalPlan] {
  *   WHERE (SELECT 1 FROM (SELECT A FROM TABLE B WHERE COL1 > 10) LIMIT 1) IS NOT NULL
  */
 object RewriteNonCorrelatedExists extends Rule[LogicalPlan] {
-  override def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions {
+  override def apply(plan: LogicalPlan): LogicalPlan = plan.transformAllExpressionsWithPruning(
+    _.containsPattern(EXISTS_SUBQUERY)) {
     case exists: Exists if exists.children.isEmpty =>
       IsNotNull(
         ScalarSubquery(
@@ -77,29 +78,37 @@ object ComputeCurrentTime extends Rule[LogicalPlan] {
     val timeExpr = CurrentTimestamp()
     val timestamp = timeExpr.eval(EmptyRow).asInstanceOf[Long]
     val currentTime = Literal.create(timestamp, timeExpr.dataType)
+    val timezone = Literal.create(conf.sessionLocalTimeZone, StringType)
 
-    plan transformAllExpressions {
-      case CurrentDate(Some(timeZoneId)) =>
+    plan.transformAllExpressionsWithPruning(_.containsPattern(CURRENT_LIKE)) {
+      case currentDate @ CurrentDate(Some(timeZoneId)) =>
         currentDates.getOrElseUpdate(timeZoneId, {
           Literal.create(
-            LocalDate.now(DateTimeUtils.getZoneId(timeZoneId)),
+            DateTimeUtils.microsToDays(timestamp, currentDate.zoneId),
             DateType)
         })
       case CurrentTimestamp() | Now() => currentTime
+      case CurrentTimeZone() => timezone
     }
   }
 }
 
 
-/** Replaces the expression of CurrentDatabase with the current database name. */
-case class GetCurrentDatabase(catalogManager: CatalogManager) extends Rule[LogicalPlan] {
+/**
+ * Replaces the expression of CurrentDatabase with the current database name.
+ * Replaces the expression of CurrentCatalog with the current catalog name.
+ */
+case class GetCurrentDatabaseAndCatalog(catalogManager: CatalogManager) extends Rule[LogicalPlan] {
   def apply(plan: LogicalPlan): LogicalPlan = {
     import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
     val currentNamespace = catalogManager.currentNamespace.quoted
+    val currentCatalog = catalogManager.currentCatalog.name()
 
-    plan transformAllExpressions {
+    plan.transformAllExpressionsWithPruning(_.containsPattern(CURRENT_LIKE)) {
       case CurrentDatabase() =>
         Literal.create(currentNamespace, StringType)
+      case CurrentCatalog() =>
+        Literal.create(currentCatalog, StringType)
     }
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/joins.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/joins.scala
index b65221c236bfe..8062b21869511 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/joins.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/joins.scala
@@ -25,6 +25,7 @@ import org.apache.spark.sql.catalyst.planning.ExtractFiltersAndInnerJoins
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules._
+import org.apache.spark.sql.catalyst.trees.TreePattern.{INNER_LIKE_JOIN, OUTER_JOIN}
 import org.apache.spark.sql.internal.SQLConf
 
 /**
@@ -88,10 +89,11 @@ object ReorderJoin extends Rule[LogicalPlan] with PredicateHelper {
     }
   }
 
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsPattern(INNER_LIKE_JOIN), ruleId) {
     case p @ ExtractFiltersAndInnerJoins(input, conditions)
         if input.size > 2 && conditions.nonEmpty =>
-      val reordered = if (SQLConf.get.starSchemaDetection && !SQLConf.get.cboEnabled) {
+      val reordered = if (conf.starSchemaDetection && !conf.cboEnabled) {
         val starJoinPlan = StarSchemaDetection.reorderStarJoins(input, conditions)
         if (starJoinPlan.nonEmpty) {
           val rest = input.filterNot(starJoinPlan.contains(_))
@@ -158,7 +160,8 @@ object EliminateOuterJoin extends Rule[LogicalPlan] with PredicateHelper {
     }
   }
 
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsPattern(OUTER_JOIN), ruleId) {
     case f @ Filter(condition, j @ Join(_, _, RightOuter | LeftOuter | FullOuter, _, _)) =>
       val newJoinType = buildNewJoinType(f, j)
       if (j.joinType == newJoinType) f else Filter(condition, j.copy(joinType = newJoinType))
@@ -208,3 +211,189 @@ object ExtractPythonUDFFromJoinCondition extends Rule[LogicalPlan] with Predicat
       }
   }
 }
+
+sealed abstract class BuildSide
+
+case object BuildRight extends BuildSide
+
+case object BuildLeft extends BuildSide
+
+trait JoinSelectionHelper {
+
+  def getBroadcastBuildSide(
+      left: LogicalPlan,
+      right: LogicalPlan,
+      joinType: JoinType,
+      hint: JoinHint,
+      hintOnly: Boolean,
+      conf: SQLConf): Option[BuildSide] = {
+    val buildLeft = if (hintOnly) {
+      hintToBroadcastLeft(hint)
+    } else {
+      canBroadcastBySize(left, conf) && !hintToNotBroadcastLeft(hint)
+    }
+    val buildRight = if (hintOnly) {
+      hintToBroadcastRight(hint)
+    } else {
+      canBroadcastBySize(right, conf) && !hintToNotBroadcastRight(hint)
+    }
+    getBuildSide(
+      canBuildBroadcastLeft(joinType) && buildLeft,
+      canBuildBroadcastRight(joinType) && buildRight,
+      left,
+      right
+    )
+  }
+
+  def getShuffleHashJoinBuildSide(
+      left: LogicalPlan,
+      right: LogicalPlan,
+      joinType: JoinType,
+      hint: JoinHint,
+      hintOnly: Boolean,
+      conf: SQLConf): Option[BuildSide] = {
+    val buildLeft = if (hintOnly) {
+      hintToShuffleHashJoinLeft(hint)
+    } else {
+      canBuildLocalHashMapBySize(left, conf) && muchSmaller(left, right)
+    }
+    val buildRight = if (hintOnly) {
+      hintToShuffleHashJoinRight(hint)
+    } else {
+      canBuildLocalHashMapBySize(right, conf) && muchSmaller(right, left)
+    }
+    getBuildSide(
+      canBuildShuffledHashJoinLeft(joinType) && buildLeft,
+      canBuildShuffledHashJoinRight(joinType) && buildRight,
+      left,
+      right
+    )
+  }
+
+  def getSmallerSide(left: LogicalPlan, right: LogicalPlan): BuildSide = {
+    if (right.stats.sizeInBytes <= left.stats.sizeInBytes) BuildRight else BuildLeft
+  }
+
+  /**
+   * Matches a plan whose output should be small enough to be used in broadcast join.
+   */
+  def canBroadcastBySize(plan: LogicalPlan, conf: SQLConf): Boolean = {
+    val autoBroadcastJoinThreshold = if (plan.stats.isRuntime) {
+      conf.getConf(SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD)
+        .getOrElse(conf.autoBroadcastJoinThreshold)
+    } else {
+      conf.autoBroadcastJoinThreshold
+    }
+    plan.stats.sizeInBytes >= 0 && plan.stats.sizeInBytes <= autoBroadcastJoinThreshold
+  }
+
+  def canBuildBroadcastLeft(joinType: JoinType): Boolean = {
+    joinType match {
+      case _: InnerLike | RightOuter => true
+      case _ => false
+    }
+  }
+
+  def canBuildBroadcastRight(joinType: JoinType): Boolean = {
+    joinType match {
+      case _: InnerLike | LeftOuter | LeftSemi | LeftAnti | _: ExistenceJoin => true
+      case _ => false
+    }
+  }
+
+  def canBuildShuffledHashJoinLeft(joinType: JoinType): Boolean = {
+    joinType match {
+      case _: InnerLike | RightOuter | FullOuter => true
+      case _ => false
+    }
+  }
+
+  def canBuildShuffledHashJoinRight(joinType: JoinType): Boolean = {
+    joinType match {
+      case _: InnerLike | LeftOuter | FullOuter |
+           LeftSemi | LeftAnti | _: ExistenceJoin => true
+      case _ => false
+    }
+  }
+
+  def canPlanAsBroadcastHashJoin(join: Join, conf: SQLConf): Boolean = {
+    getBroadcastBuildSide(join.left, join.right, join.joinType,
+      join.hint, hintOnly = true, conf).isDefined ||
+      getBroadcastBuildSide(join.left, join.right, join.joinType,
+        join.hint, hintOnly = false, conf).isDefined
+  }
+
+  def hintToBroadcastLeft(hint: JoinHint): Boolean = {
+    hint.leftHint.exists(_.strategy.contains(BROADCAST))
+  }
+
+  def hintToBroadcastRight(hint: JoinHint): Boolean = {
+    hint.rightHint.exists(_.strategy.contains(BROADCAST))
+  }
+
+  def hintToNotBroadcastLeft(hint: JoinHint): Boolean = {
+    hint.leftHint.exists(_.strategy.contains(NO_BROADCAST_HASH))
+  }
+
+  def hintToNotBroadcastRight(hint: JoinHint): Boolean = {
+    hint.rightHint.exists(_.strategy.contains(NO_BROADCAST_HASH))
+  }
+
+  def hintToShuffleHashJoinLeft(hint: JoinHint): Boolean = {
+    hint.leftHint.exists(_.strategy.contains(SHUFFLE_HASH))
+  }
+
+  def hintToShuffleHashJoinRight(hint: JoinHint): Boolean = {
+    hint.rightHint.exists(_.strategy.contains(SHUFFLE_HASH))
+  }
+
+  def hintToSortMergeJoin(hint: JoinHint): Boolean = {
+    hint.leftHint.exists(_.strategy.contains(SHUFFLE_MERGE)) ||
+      hint.rightHint.exists(_.strategy.contains(SHUFFLE_MERGE))
+  }
+
+  def hintToShuffleReplicateNL(hint: JoinHint): Boolean = {
+    hint.leftHint.exists(_.strategy.contains(SHUFFLE_REPLICATE_NL)) ||
+      hint.rightHint.exists(_.strategy.contains(SHUFFLE_REPLICATE_NL))
+  }
+
+  private def getBuildSide(
+      canBuildLeft: Boolean,
+      canBuildRight: Boolean,
+      left: LogicalPlan,
+      right: LogicalPlan): Option[BuildSide] = {
+    if (canBuildLeft && canBuildRight) {
+      // returns the smaller side base on its estimated physical size, if we want to build the
+      // both sides.
+      Some(getSmallerSide(left, right))
+    } else if (canBuildLeft) {
+      Some(BuildLeft)
+    } else if (canBuildRight) {
+      Some(BuildRight)
+    } else {
+      None
+    }
+  }
+
+  /**
+   * Matches a plan whose single partition should be small enough to build a hash table.
+   *
+   * Note: this assume that the number of partition is fixed, requires additional work if it's
+   * dynamic.
+   */
+  private def canBuildLocalHashMapBySize(plan: LogicalPlan, conf: SQLConf): Boolean = {
+    plan.stats.sizeInBytes < conf.autoBroadcastJoinThreshold * conf.numShufflePartitions
+  }
+
+  /**
+   * Returns whether plan a is much smaller (3X) than plan b.
+   *
+   * The cost to build hash map is higher than sorting, we should only build hash map on a table
+   * that is much smaller than other one. Since we does not have the statistic for number of rows,
+   * use the size of bytes here as estimation.
+   */
+  private def muchSmaller(a: LogicalPlan, b: LogicalPlan): Boolean = {
+    a.stats.sizeInBytes * 3 <= b.stats.sizeInBytes
+  }
+}
+
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/objects.scala
index 0fa29e87e5f39..52544ff3e241d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/objects.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/objects.scala
@@ -24,7 +24,7 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.objects._
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules._
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.catalyst.trees.TreePattern._
 import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructType, UserDefinedType}
 
 /*
@@ -36,7 +36,8 @@ import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructType, Use
  * representation of data item.  For example back to back map operations.
  */
 object EliminateSerialization extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsAnyPattern(DESERIALIZE_TO_OBJECT, APPEND_COLUMNS, TYPED_FILTER), ruleId) {
     case d @ DeserializeToObject(_, _, s: SerializeFromObject)
       if d.outputObjAttr.dataType == s.inputObjAttr.dataType =>
       // Adds an extra Project here, to preserve the output expr id of `DeserializeToObject`.
@@ -73,7 +74,8 @@ object EliminateSerialization extends Rule[LogicalPlan] {
  * merging the filter functions into one conjunctive function.
  */
 object CombineTypedFilters extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsPattern(TYPED_FILTER), ruleId) {
     case t1 @ TypedFilter(_, _, _, _, t2 @ TypedFilter(_, _, _, _, child))
         if t1.deserializer.dataType == t2.deserializer.dataType =>
       TypedFilter(
@@ -109,7 +111,8 @@ object CombineTypedFilters extends Rule[LogicalPlan] {
  *   2. no custom collection class specified representation of data item.
  */
 object EliminateMapObjects extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformAllExpressionsWithPruning(
+    _.containsAllPatterns(MAP_OBJECTS, LAMBDA_VARIABLE), ruleId) {
      case MapObjects(_, LambdaVariable(_, _, false, _), inputData, None) => inputData
   }
 }
@@ -147,7 +150,7 @@ object ObjectSerializerPruning extends Rule[LogicalPlan] {
    */
   private def pruneNamedStruct(struct: CreateNamedStruct, prunedType: StructType) = {
     // Filters out the pruned fields.
-    val resolver = SQLConf.get.resolver
+    val resolver = conf.resolver
     val prunedFields = struct.nameExprs.zip(struct.valExprs).filter { case (nameExpr, _) =>
       val name = nameExpr.eval(EmptyRow).toString
       prunedType.fieldNames.exists(resolver(_, name))
@@ -162,10 +165,18 @@ object ObjectSerializerPruning extends Rule[LogicalPlan] {
    * Note: we should do `transformUp` explicitly to change data types.
    */
   private def alignNullTypeInIf(expr: Expression) = expr.transformUp {
-    case i @ If(_: IsNull, Literal(null, dt), ser) if !dt.sameType(ser.dataType) =>
+    case i @ If(IsNullCondition(), Literal(null, dt), ser) if !dt.sameType(ser.dataType) =>
       i.copy(trueValue = Literal(null, ser.dataType))
   }
 
+  object IsNullCondition {
+    def unapply(expr: Expression): Boolean = expr match {
+      case _: IsNull => true
+      case i: Invoke if i.functionName == "isNullAt" => true
+      case _ => false
+    }
+  }
+
   /**
    * This method prunes given serializer expression by given pruned data type. For example,
    * given a serializer creating struct(a int, b int) and pruned data type struct(a int),
@@ -200,7 +211,8 @@ object ObjectSerializerPruning extends Rule[LogicalPlan] {
     }
   }
 
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    _.containsAllPatterns(SERIALIZE_FROM_OBJECT, PROJECT), ruleId) {
     case p @ Project(_, s: SerializeFromObject) =>
       // Prunes individual serializer if it is not used at all by above projection.
       val usedRefs = p.references
@@ -208,7 +220,7 @@ object ObjectSerializerPruning extends Rule[LogicalPlan] {
 
       val rootFields = SchemaPruning.identifyRootFields(p.projectList, Seq.empty)
 
-      if (SQLConf.get.serializerNestedSchemaPruningEnabled && rootFields.nonEmpty) {
+      if (conf.serializerNestedSchemaPruningEnabled && rootFields.nonEmpty) {
         // Prunes nested fields in serializers.
         val prunedSchema = SchemaPruning.pruneDataSchema(
           StructType.fromAttributes(prunedSerializer.map(_.toAttribute)), rootFields)
@@ -245,7 +257,7 @@ object ReassignLambdaVariableID extends Rule[LogicalPlan] {
     var hasNegativeIds = false
     var hasPositiveIds = false
 
-    plan.transformAllExpressions {
+    plan.transformAllExpressionsWithPruning(_.containsPattern(LAMBDA_VARIABLE), ruleId) {
       case lr: LambdaVariable if lr.id == 0 =>
         throw new IllegalStateException("LambdaVariable should never has 0 as its ID.")
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/subquery.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/subquery.scala
index 6fdd2110ab12a..a0da01b10dd64 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/subquery.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/subquery.scala
@@ -20,13 +20,16 @@ package org.apache.spark.sql.catalyst.optimizer
 import scala.collection.mutable.ArrayBuffer
 
 import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.analysis.CleanupAliases
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.expressions.ScalarSubquery._
 import org.apache.spark.sql.catalyst.expressions.SubExprUtils._
 import org.apache.spark.sql.catalyst.expressions.aggregate._
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules._
+import org.apache.spark.sql.catalyst.trees.TreePattern.{EXISTS_SUBQUERY, FILTER, IN_SUBQUERY, LIST_SUBQUERY, SCALAR_SUBQUERY}
+import org.apache.spark.sql.errors.QueryExecutionErrors
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 /*
@@ -64,7 +67,7 @@ object RewritePredicateSubquery extends Rule[LogicalPlan] with PredicateHelper {
     // the produced join then becomes unresolved and break structural integrity. We should
     // de-duplicate conflicting attributes.
     // SPARK-26078: it may also happen that the subquery has conflicting attributes with the outer
-    // values. In this case, the resulting join would contain trivially true conditions (eg.
+    // values. In this case, the resulting join would contain trivially true conditions (e.g.
     // id#3 = id#3) which cannot be de-duplicated after. In this method, if there are conflicting
     // attributes in the join condition, the subquery's conflicting attributes are changed using
     // a projection which aliases them and resolves the problem.
@@ -93,8 +96,10 @@ object RewritePredicateSubquery extends Rule[LogicalPlan] with PredicateHelper {
     }
   }
 
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
-    case Filter(condition, child) =>
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+    t => t.containsAnyPattern(EXISTS_SUBQUERY, LIST_SUBQUERY) && t.containsPattern(FILTER)) {
+    case Filter(condition, child)
+      if SubqueryExpression.hasInOrCorrelatedExistsSubquery(condition) =>
       val (withSubquery, withoutSubquery) =
         splitConjunctivePredicates(condition)
           .partition(SubqueryExpression.hasInOrCorrelatedExistsSubquery)
@@ -162,7 +167,7 @@ object RewritePredicateSubquery extends Rule[LogicalPlan] with PredicateHelper {
       plan: LogicalPlan): (Option[Expression], LogicalPlan) = {
     var newPlan = plan
     val newExprs = exprs.map { e =>
-      e transformDown {
+      e.transformDownWithPruning(_.containsAnyPattern(EXISTS_SUBQUERY, IN_SUBQUERY)) {
         case Exists(sub, conditions, _) =>
           val exists = AttributeReference("exists", BooleanType, nullable = false)()
           newPlan =
@@ -175,7 +180,7 @@ object RewritePredicateSubquery extends Rule[LogicalPlan] with PredicateHelper {
           val inConditions = values.zip(sub.output).map(EqualTo.tupled)
           // To handle a null-aware predicate not-in-subquery in nested conditions
           // (e.g., `v > 0 OR t1.id NOT IN (SELECT id FROM t2)`), we transform
-          // `inConditon` (t1.id=t2.id) into `(inCondition) OR ISNULL(inCondition)`.
+          // `inCondition` (t1.id=t2.id) into `(inCondition) OR ISNULL(inCondition)`.
           //
           // For example, `SELECT * FROM t1 WHERE v > 0 OR t1.id NOT IN (SELECT id FROM t2)`
           // is transformed into a plan below;
@@ -271,22 +276,8 @@ object PullupCorrelatedPredicates extends Rule[LogicalPlan] with PredicateHelper
     val baseConditions = predicateMap.values.flatten.toSeq
     val (newPlan, newCond) = if (outer.nonEmpty) {
       val outputSet = outer.map(_.outputSet).reduce(_ ++ _)
-      val duplicates = transformed.outputSet.intersect(outputSet)
-      val (plan, deDuplicatedConditions) = if (duplicates.nonEmpty) {
-        val aliasMap = AttributeMap(duplicates.map { dup =>
-          dup -> Alias(dup, dup.toString)()
-        }.toSeq)
-        val aliasedExpressions = transformed.output.map { ref =>
-          aliasMap.getOrElse(ref, ref)
-        }
-        val aliasedProjection = Project(aliasedExpressions, transformed)
-        val aliasedConditions = baseConditions.map(_.transform {
-          case ref: Attribute => aliasMap.getOrElse(ref, ref).toAttribute
-        })
-        (aliasedProjection, aliasedConditions)
-      } else {
-        (transformed, baseConditions)
-      }
+      val (plan, deDuplicatedConditions) =
+        DecorrelateInnerQuery.deduplicate(transformed, baseConditions, outputSet)
       (plan, stripOuterReferences(deDuplicatedConditions))
     } else {
       (transformed, stripOuterReferences(baseConditions))
@@ -307,9 +298,18 @@ object PullupCorrelatedPredicates extends Rule[LogicalPlan] with PredicateHelper
       if (newCond.isEmpty) oldCond else newCond
     }
 
-    plan transformExpressions {
+    def decorrelate(sub: LogicalPlan, outer: Seq[LogicalPlan]): (LogicalPlan, Seq[Expression]) = {
+      if (SQLConf.get.decorrelateInnerQueryEnabled) {
+        DecorrelateInnerQuery(sub, outer)
+      } else {
+        pullOutCorrelatedPredicates(sub, outer)
+      }
+    }
+
+    plan.transformExpressionsWithPruning(_.containsAnyPattern(
+      SCALAR_SUBQUERY, EXISTS_SUBQUERY, LIST_SUBQUERY)) {
       case ScalarSubquery(sub, children, exprId) if children.nonEmpty =>
-        val (newPlan, newCond) = pullOutCorrelatedPredicates(sub, outerPlans)
+        val (newPlan, newCond) = decorrelate(sub, outerPlans)
         ScalarSubquery(newPlan, getJoinCondition(newCond, children), exprId)
       case Exists(sub, children, exprId) if children.nonEmpty =>
         val (newPlan, newCond) = pullOutCorrelatedPredicates(sub, outerPlans)
@@ -323,19 +323,22 @@ object PullupCorrelatedPredicates extends Rule[LogicalPlan] with PredicateHelper
   /**
    * Pull up the correlated predicates and rewrite all subqueries in an operator tree..
    */
-  def apply(plan: LogicalPlan): LogicalPlan = plan transformUp {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformUpWithPruning(
+    _.containsAnyPattern(SCALAR_SUBQUERY, EXISTS_SUBQUERY, LIST_SUBQUERY)) {
     case f @ Filter(_, a: Aggregate) =>
       rewriteSubQueries(f, Seq(a, a.child))
     // Only a few unary nodes (Project/Filter/Aggregate) can contain subqueries.
     case q: UnaryNode =>
       rewriteSubQueries(q, q.children)
+    case s: SupportsSubquery =>
+      rewriteSubQueries(s, s.children)
   }
 }
 
 /**
  * This rule rewrites correlated [[ScalarSubquery]] expressions into LEFT OUTER joins.
  */
-object RewriteCorrelatedScalarSubquery extends Rule[LogicalPlan] {
+object RewriteCorrelatedScalarSubquery extends Rule[LogicalPlan] with AliasHelper {
   /**
    * Extract all correlated scalar subqueries from an expression. The subqueries are collected using
    * the given collector. The expression is rewritten and returned.
@@ -343,7 +346,7 @@ object RewriteCorrelatedScalarSubquery extends Rule[LogicalPlan] {
   private def extractCorrelatedScalarSubqueries[E <: Expression](
       expression: E,
       subqueries: ArrayBuffer[ScalarSubquery]): E = {
-    val newExpression = expression transform {
+    val newExpression = expression.transformWithPruning(_.containsPattern(SCALAR_SUBQUERY)) {
       case s: ScalarSubquery if s.children.nonEmpty =>
         subqueries += s
         s.plan.output.head
@@ -357,7 +360,7 @@ object RewriteCorrelatedScalarSubquery extends Rule[LogicalPlan] {
    */
   private def tryEvalExpr(expr: Expression): Expression = {
     // Removes Alias over given expression, because Alias is not foldable.
-    if (!CleanupAliases.trimAliases(expr).foldable) {
+    if (!trimAliases(expr).foldable) {
       // SPARK-28441: Some expressions, like PythonUDF, can't be statically evaluated.
       // Needs to evaluate them on query runtime.
       expr
@@ -376,7 +379,7 @@ object RewriteCorrelatedScalarSubquery extends Rule[LogicalPlan] {
       bindings: Map[ExprId, Expression]): Expression = {
     val rewrittenExpr = expr transform {
       case r: AttributeReference =>
-        bindings.getOrElse(r.exprId, Literal.default(NullType))
+        bindings.getOrElse(r.exprId, Literal.create(null, r.dataType))
     }
 
     tryEvalExpr(rewrittenExpr)
@@ -391,9 +394,9 @@ object RewriteCorrelatedScalarSubquery extends Rule[LogicalPlan] {
     // Also replace attribute refs (for example, for grouping columns) with NULL.
     val rewrittenExpr = expr transform {
       case a @ AggregateExpression(aggFunc, _, _, resultId, _) =>
-        aggFunc.defaultResult.getOrElse(Literal.default(NullType))
+        aggFunc.defaultResult.getOrElse(Literal.create(null, aggFunc.dataType))
 
-      case _: AttributeReference => Literal.default(NullType)
+      case a: AttributeReference => Literal.create(null, a.dataType)
     }
 
     tryEvalExpr(rewrittenExpr)
@@ -451,11 +454,17 @@ object RewriteCorrelatedScalarSubquery extends Rule[LogicalPlan] {
           case ref: AttributeReference => (ref.exprId, Literal.create(null, ref.dataType))
           case alias @ Alias(_: AttributeReference, _) =>
             (alias.exprId, Literal.create(null, alias.dataType))
+          case alias @ Alias(l: Literal, _) =>
+            (alias.exprId, l.copy(value = null))
           case ne => (ne.exprId, evalAggOnZeroTups(ne))
         }.toMap
 
-      case _ =>
-        sys.error(s"Unexpected operator in scalar subquery: $lp")
+      case l: LeafNode =>
+        l.output.map(a => (a.exprId, Literal.create(null, a.dataType))).toMap
+
+      case p: LogicalPlan =>
+        val bindings = p.children.map(evalPlan).reduce(_ ++ _)
+        p.output.map(e => (e.exprId, bindingExpr(e, bindings))).toMap
     }
 
     val resultMap = evalPlan(plan)
@@ -478,11 +487,11 @@ object RewriteCorrelatedScalarSubquery extends Rule[LogicalPlan] {
     while (true) {
       bottomPart match {
         case havingPart @ Filter(_, aggPart: Aggregate) =>
-          return (topPart, Option(havingPart), aggPart)
+          return (topPart.toSeq, Option(havingPart), aggPart)
 
         case aggPart: Aggregate =>
           // No HAVING clause
-          return (topPart, None, aggPart)
+          return (topPart.toSeq, None, aggPart)
 
         case p @ Project(_, child) =>
           topPart += p
@@ -493,13 +502,14 @@ object RewriteCorrelatedScalarSubquery extends Rule[LogicalPlan] {
           bottomPart = child
 
         case Filter(_, op) =>
-          sys.error(s"Correlated subquery has unexpected operator $op below filter")
+          throw QueryExecutionErrors.unexpectedOperatorInCorrelatedSubquery(op, " below filter")
 
-        case op @ _ => sys.error(s"Unexpected operator $op in correlated subquery")
+        case op @ _ => throw QueryExecutionErrors.unexpectedOperatorInCorrelatedSubquery(op)
       }
     }
 
-    sys.error("This line should be unreachable")
+    throw QueryExecutionErrors.unreachableError()
+
   }
 
   // Name of generated column used in rewrite below
@@ -507,12 +517,17 @@ object RewriteCorrelatedScalarSubquery extends Rule[LogicalPlan] {
 
   /**
    * Construct a new child plan by left joining the given subqueries to a base plan.
+   * This method returns the child plan and an attribute mapping
+   * for the updated `ExprId`s of subqueries. If the non-empty mapping returned,
+   * this rule will rewrite subquery references in a parent plan based on it.
    */
   private def constructLeftJoins(
       child: LogicalPlan,
-      subqueries: ArrayBuffer[ScalarSubquery]): LogicalPlan = {
-    subqueries.foldLeft(child) {
-      case (currentChild, ScalarSubquery(query, conditions, _)) =>
+      subqueries: ArrayBuffer[ScalarSubquery]): (LogicalPlan, AttributeMap[Attribute]) = {
+    val subqueryAttrMapping = ArrayBuffer[(Attribute, Attribute)]()
+    val newChild = subqueries.foldLeft(child) {
+      case (currentChild, ScalarSubquery(sub, conditions, _)) =>
+        val query = DecorrelateInnerQuery.rewriteDomainJoins(currentChild, sub, conditions)
         val origOutput = query.output.head
 
         val resultWithZeroTups = evalSubqueryOnZeroTups(query)
@@ -539,12 +554,13 @@ object RewriteCorrelatedScalarSubquery extends Rule[LogicalPlan] {
 
           if (havingNode.isEmpty) {
             // CASE 2: Subquery with no HAVING clause
+            val subqueryResultExpr =
+              Alias(If(IsNull(alwaysTrueRef),
+                resultWithZeroTups.get,
+                aggValRef), origOutput.name)()
+            subqueryAttrMapping += ((origOutput, subqueryResultExpr.toAttribute))
             Project(
-              currentChild.output :+
-                Alias(
-                  If(IsNull(alwaysTrueRef),
-                    resultWithZeroTups.get,
-                    aggValRef), origOutput.name)(exprId = origOutput.exprId),
+              currentChild.output :+ subqueryResultExpr,
               Join(currentChild,
                 Project(query.output :+ alwaysTrueExpr, query),
                 LeftOuter, conditions.reduceOption(And), JoinHint.NONE))
@@ -561,7 +577,7 @@ object RewriteCorrelatedScalarSubquery extends Rule[LogicalPlan] {
                 subqueryRoot = Project(projList ++ havingInputs, subqueryRoot)
               case s @ SubqueryAlias(alias, _) =>
                 subqueryRoot = SubqueryAlias(alias, subqueryRoot)
-              case op => sys.error(s"Unexpected operator $op in corelated subquery")
+              case op => throw QueryExecutionErrors.unexpectedOperatorInCorrelatedSubquery(op)
             }
 
             // CASE WHEN alwaysTrue IS NULL THEN resultOnZeroTups
@@ -571,7 +587,9 @@ object RewriteCorrelatedScalarSubquery extends Rule[LogicalPlan] {
               (IsNull(alwaysTrueRef), resultWithZeroTups.get),
               (Not(havingNode.get.condition), Literal.create(null, aggValRef.dataType))),
               aggValRef),
-              origOutput.name)(exprId = origOutput.exprId)
+              origOutput.name)()
+
+            subqueryAttrMapping += ((origOutput, caseExpr.toAttribute))
 
             Project(
               currentChild.output :+ caseExpr,
@@ -582,16 +600,43 @@ object RewriteCorrelatedScalarSubquery extends Rule[LogicalPlan] {
           }
         }
     }
+    (newChild, AttributeMap(subqueryAttrMapping.toSeq))
+  }
+
+  private def updateAttrs[E <: Expression](
+      exprs: Seq[E],
+      attrMap: AttributeMap[Attribute]): Seq[E] = {
+    if (attrMap.nonEmpty) {
+      val newExprs = exprs.map { _.transform {
+        case a: AttributeReference => attrMap.getOrElse(a, a)
+      }}
+      newExprs.asInstanceOf[Seq[E]]
+    } else {
+      exprs
+    }
+  }
+
+  /**
+   * Check if an [[Aggregate]] has no correlated subquery in aggregate expressions but
+   * still has correlated scalar subqueries in its grouping expressions, which will not
+   * be rewritten.
+   */
+  private def checkScalarSubqueryInAgg(a: Aggregate): Unit = {
+    if (a.groupingExpressions.exists(hasCorrelatedScalarSubquery) &&
+        !a.aggregateExpressions.exists(hasCorrelatedScalarSubquery)) {
+      throw new IllegalStateException(
+        s"Fail to rewrite correlated scalar subqueries in Aggregate:\n$a")
+    }
   }
 
   /**
    * Rewrite [[Filter]], [[Project]] and [[Aggregate]] plans containing correlated scalar
    * subqueries.
    */
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = plan transformUpWithNewOutput {
     case a @ Aggregate(grouping, expressions, child) =>
       val subqueries = ArrayBuffer.empty[ScalarSubquery]
-      val newExpressions = expressions.map(extractCorrelatedScalarSubqueries(_, subqueries))
+      val rewriteExprs = expressions.map(extractCorrelatedScalarSubqueries(_, subqueries))
       if (subqueries.nonEmpty) {
         // We currently only allow correlated subqueries in an aggregate if they are part of the
         // grouping expressions. As a result we need to replace all the scalar subqueries in the
@@ -599,25 +644,38 @@ object RewriteCorrelatedScalarSubquery extends Rule[LogicalPlan] {
         val newGrouping = grouping.map { e =>
           subqueries.find(_.semanticEquals(e)).map(_.plan.output.head).getOrElse(e)
         }
-        Aggregate(newGrouping, newExpressions, constructLeftJoins(child, subqueries))
+        val (newChild, subqueryAttrMapping) = constructLeftJoins(child, subqueries)
+        val newExprs = updateAttrs(rewriteExprs, subqueryAttrMapping)
+        val newAgg = Aggregate(newGrouping, newExprs, newChild)
+        val attrMapping = a.output.zip(newAgg.output)
+        checkScalarSubqueryInAgg(newAgg)
+        newAgg -> attrMapping
       } else {
-        a
+        a -> Nil
       }
     case p @ Project(expressions, child) =>
       val subqueries = ArrayBuffer.empty[ScalarSubquery]
-      val newExpressions = expressions.map(extractCorrelatedScalarSubqueries(_, subqueries))
+      val rewriteExprs = expressions.map(extractCorrelatedScalarSubqueries(_, subqueries))
       if (subqueries.nonEmpty) {
-        Project(newExpressions, constructLeftJoins(child, subqueries))
+        val (newChild, subqueryAttrMapping) = constructLeftJoins(child, subqueries)
+        val newExprs = updateAttrs(rewriteExprs, subqueryAttrMapping)
+        val newProj = Project(newExprs, newChild)
+        val attrMapping = p.output.zip(newProj.output)
+        newProj -> attrMapping
       } else {
-        p
+        p -> Nil
       }
     case f @ Filter(condition, child) =>
       val subqueries = ArrayBuffer.empty[ScalarSubquery]
-      val newCondition = extractCorrelatedScalarSubqueries(condition, subqueries)
+      val rewriteCondition = extractCorrelatedScalarSubqueries(condition, subqueries)
       if (subqueries.nonEmpty) {
-        Project(f.output, Filter(newCondition, constructLeftJoins(child, subqueries)))
+        val (newChild, subqueryAttrMapping) = constructLeftJoins(child, subqueries)
+        val newCondition = updateAttrs(Seq(rewriteCondition), subqueryAttrMapping).head
+        val newProj = Project(f.output, Filter(newCondition, newChild))
+        val attrMapping = f.output.zip(newProj.output)
+        newProj -> attrMapping
       } else {
-        f
+        f -> Nil
       }
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index c0cecf8536c39..6523149bfd968 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.catalyst.parser
 
 import java.util.Locale
+import java.util.concurrent.TimeUnit
 import javax.xml.bind.DatatypeConverter
 
 import scala.collection.JavaConverters._
@@ -28,7 +29,7 @@ import org.antlr.v4.runtime.tree.{ParseTree, RuleNode, TerminalNode}
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
+import org.apache.spark.sql.catalyst.{FunctionIdentifier, SQLConfHelper, TableIdentifier}
 import org.apache.spark.sql.catalyst.analysis._
 import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStorageFormat, FunctionResource, FunctionResourceType}
 import org.apache.spark.sql.catalyst.expressions._
@@ -36,12 +37,14 @@ import org.apache.spark.sql.catalyst.expressions.aggregate.{First, Last}
 import org.apache.spark.sql.catalyst.parser.SqlBaseParser._
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.trees.CurrentOrigin
+import org.apache.spark.sql.catalyst.util.{CharVarcharUtils, IntervalUtils}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils.{getZoneId, stringToDate, stringToTimestamp}
-import org.apache.spark.sql.catalyst.util.IntervalUtils
 import org.apache.spark.sql.catalyst.util.IntervalUtils.IntervalUnit
 import org.apache.spark.sql.connector.catalog.{SupportsNamespaces, TableCatalog}
 import org.apache.spark.sql.connector.catalog.TableChange.ColumnPosition
 import org.apache.spark.sql.connector.expressions.{ApplyTransform, BucketTransform, DaysTransform, Expression => V2Expression, FieldReference, HoursTransform, IdentityTransform, LiteralValue, MonthsTransform, Transform, YearsTransform}
+import org.apache.spark.sql.errors.QueryParsingErrors
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
@@ -51,11 +54,9 @@ import org.apache.spark.util.random.RandomSampler
  * The AstBuilder converts an ANTLR4 ParseTree into a catalyst Expression, LogicalPlan or
  * TableIdentifier.
  */
-class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging {
+class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with SQLConfHelper with Logging {
   import ParserUtils._
 
-  def this() = this(new SQLConf())
-
   protected def typedVisit[T](ctx: ParseTree): T = {
     ctx.accept(this).asInstanceOf[T]
   }
@@ -97,15 +98,12 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
   }
 
   override def visitSingleDataType(ctx: SingleDataTypeContext): DataType = withOrigin(ctx) {
-    visitSparkDataType(ctx.dataType)
+    typedVisit[DataType](ctx.dataType)
   }
 
   override def visitSingleTableSchema(ctx: SingleTableSchemaContext): StructType = {
-    withOrigin(ctx)(StructType(visitColTypeList(ctx.colTypeList)))
-  }
-
-  def parseRawDataType(ctx: SingleDataTypeContext): DataType = withOrigin(ctx) {
-    typedVisit[DataType](ctx.dataType())
+    val schema = StructType(visitColTypeList(ctx.colTypeList))
+    withOrigin(ctx)(schema)
   }
 
   /* ********************************************************************************************
@@ -137,11 +135,10 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     // Check for duplicate names.
     val duplicates = ctes.groupBy(_._1).filter(_._2.size > 1).keys
     if (duplicates.nonEmpty) {
-      throw new ParseException(
-        s"CTE definition can't have duplicate names: ${duplicates.mkString("'", "', '", "'")}.",
-        ctx)
+      throw QueryParsingErrors.duplicateCteDefinitionNamesError(
+        duplicates.mkString("'", "', '", "'"), ctx)
     }
-    With(plan, ctes)
+    With(plan, ctes.toSeq)
   }
 
   /**
@@ -154,7 +151,11 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       withTransformQuerySpecification(
         ctx,
         ctx.transformClause,
+        ctx.lateralView,
         ctx.whereClause,
+        ctx.aggregationClause,
+        ctx.havingClause,
+        ctx.windowClause,
         plan
       )
     } else {
@@ -182,7 +183,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     if (selects.length == 1) {
       selects.head
     } else {
-      Union(selects)
+      Union(selects.toSeq)
     }
   }
 
@@ -229,7 +230,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     if (inserts.length == 1) {
       inserts.head
     } else {
-      Union(inserts)
+      Union(inserts.toSeq)
     }
   }
 
@@ -245,9 +246,9 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
 
   /**
    * Parameters used for writing query to a table:
-   *   (multipartIdentifier, partitionKeys, ifPartitionNotExists).
+   *   (UnresolvedRelation, tableColumnList, partitionKeys, ifPartitionNotExists).
    */
-  type InsertTableParams = (Seq[String], Map[String, Option[String]], Boolean)
+  type InsertTableParams = (UnresolvedRelation, Seq[String], Map[String, Option[String]], Boolean)
 
   /**
    * Parameters used for writing query to a directory: (isLocal, CatalogStorageFormat, provider).
@@ -257,8 +258,8 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
   /**
    * Add an
    * {{{
-   *   INSERT OVERWRITE TABLE tableIdentifier [partitionSpec [IF NOT EXISTS]]?
-   *   INSERT INTO [TABLE] tableIdentifier [partitionSpec]
+   *   INSERT OVERWRITE TABLE tableIdentifier [partitionSpec [IF NOT EXISTS]]? [identifierList]
+   *   INSERT INTO [TABLE] tableIdentifier [partitionSpec]  [identifierList]
    *   INSERT OVERWRITE [LOCAL] DIRECTORY STRING [rowFormat] [createFileFormat]
    *   INSERT OVERWRITE [LOCAL] DIRECTORY [STRING] tableProvider [OPTIONS tablePropertyList]
    * }}}
@@ -269,18 +270,20 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       query: LogicalPlan): LogicalPlan = withOrigin(ctx) {
     ctx match {
       case table: InsertIntoTableContext =>
-        val (tableIdent, partition, ifPartitionNotExists) = visitInsertIntoTable(table)
+        val (relation, cols, partition, ifPartitionNotExists) = visitInsertIntoTable(table)
         InsertIntoStatement(
-          UnresolvedRelation(tableIdent),
+          relation,
           partition,
+          cols,
           query,
           overwrite = false,
           ifPartitionNotExists)
       case table: InsertOverwriteTableContext =>
-        val (tableIdent, partition, ifPartitionNotExists) = visitInsertOverwriteTable(table)
+        val (relation, cols, partition, ifPartitionNotExists) = visitInsertOverwriteTable(table)
         InsertIntoStatement(
-          UnresolvedRelation(tableIdent),
+          relation,
           partition,
+          cols,
           query,
           overwrite = true,
           ifPartitionNotExists)
@@ -291,7 +294,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
         val (isLocal, storage, provider) = visitInsertOverwriteHiveDir(hiveDir)
         InsertIntoDir(isLocal, storage, provider, query, overwrite = true)
       case _ =>
-        throw new ParseException("Invalid InsertIntoContext", ctx)
+        throw QueryParsingErrors.invalidInsertIntoError(ctx)
     }
   }
 
@@ -300,14 +303,14 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   override def visitInsertIntoTable(
       ctx: InsertIntoTableContext): InsertTableParams = withOrigin(ctx) {
-    val tableIdent = visitMultipartIdentifier(ctx.multipartIdentifier)
+    val cols = Option(ctx.identifierList()).map(visitIdentifierList).getOrElse(Nil)
     val partitionKeys = Option(ctx.partitionSpec).map(visitPartitionSpec).getOrElse(Map.empty)
 
     if (ctx.EXISTS != null) {
       operationNotAllowed("INSERT INTO ... IF NOT EXISTS", ctx)
     }
 
-    (tableIdent, partitionKeys, false)
+    (createUnresolvedRelation(ctx.multipartIdentifier), cols, partitionKeys, false)
   }
 
   /**
@@ -316,7 +319,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
   override def visitInsertOverwriteTable(
       ctx: InsertOverwriteTableContext): InsertTableParams = withOrigin(ctx) {
     assert(ctx.OVERWRITE() != null)
-    val tableIdent = visitMultipartIdentifier(ctx.multipartIdentifier)
+    val cols = Option(ctx.identifierList()).map(visitIdentifierList).getOrElse(Nil)
     val partitionKeys = Option(ctx.partitionSpec).map(visitPartitionSpec).getOrElse(Map.empty)
 
     val dynamicPartitionKeys: Map[String, Option[String]] = partitionKeys.filter(_._2.isEmpty)
@@ -325,7 +328,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
         dynamicPartitionKeys.keys.mkString(", "), ctx)
     }
 
-    (tableIdent, partitionKeys, ctx.EXISTS() != null)
+    (createUnresolvedRelation(ctx.multipartIdentifier), cols, partitionKeys, ctx.EXISTS() != null)
   }
 
   /**
@@ -333,7 +336,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   override def visitInsertOverwriteDir(
       ctx: InsertOverwriteDirContext): InsertDirParams = withOrigin(ctx) {
-    throw new ParseException("INSERT OVERWRITE DIRECTORY is not supported", ctx)
+    throw QueryParsingErrors.insertOverwriteDirectoryUnsupportedError(ctx)
   }
 
   /**
@@ -341,7 +344,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   override def visitInsertOverwriteHiveDir(
       ctx: InsertOverwriteHiveDirContext): InsertDirParams = withOrigin(ctx) {
-    throw new ParseException("INSERT OVERWRITE DIRECTORY is not supported", ctx)
+    throw QueryParsingErrors.insertOverwriteDirectoryUnsupportedError(ctx)
   }
 
   private def getTableAliasWithoutColumnAlias(
@@ -351,7 +354,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     } else {
       val ident = ctx.strictIdentifier()
       if (ctx.identifierList() != null) {
-        throw new ParseException(s"Columns aliases are not allowed in $op.", ctx.identifierList())
+        throw QueryParsingErrors.columnAliasInOperationNotAllowedError(op, ctx)
       }
       if (ident != null) Some(ident.getText) else None
     }
@@ -359,7 +362,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
 
   override def visitDeleteFromTable(
       ctx: DeleteFromTableContext): LogicalPlan = withOrigin(ctx) {
-    val table = UnresolvedRelation(visitMultipartIdentifier(ctx.multipartIdentifier()))
+    val table = createUnresolvedRelation(ctx.multipartIdentifier())
     val tableAlias = getTableAliasWithoutColumnAlias(ctx.tableAlias(), "DELETE")
     val aliasedTable = tableAlias.map(SubqueryAlias(_, table)).getOrElse(table)
     val predicate = if (ctx.whereClause() != null) {
@@ -371,7 +374,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
   }
 
   override def visitUpdateTable(ctx: UpdateTableContext): LogicalPlan = withOrigin(ctx) {
-    val table = UnresolvedRelation(visitMultipartIdentifier(ctx.multipartIdentifier()))
+    val table = createUnresolvedRelation(ctx.multipartIdentifier())
     val tableAlias = getTableAliasWithoutColumnAlias(ctx.tableAlias(), "UPDATE")
     val aliasedTable = tableAlias.map(SubqueryAlias(_, table)).getOrElse(table)
     val assignments = withAssignments(ctx.setClause().assignmentList())
@@ -389,21 +392,20 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       assignCtx.assignment().asScala.map { assign =>
         Assignment(UnresolvedAttribute(visitMultipartIdentifier(assign.key)),
           expression(assign.value))
-      }
+      }.toSeq
     }
 
   override def visitMergeIntoTable(ctx: MergeIntoTableContext): LogicalPlan = withOrigin(ctx) {
-    val targetTable = UnresolvedRelation(visitMultipartIdentifier(ctx.target))
+    val targetTable = createUnresolvedRelation(ctx.target)
     val targetTableAlias = getTableAliasWithoutColumnAlias(ctx.targetAlias, "MERGE")
     val aliasedTarget = targetTableAlias.map(SubqueryAlias(_, targetTable)).getOrElse(targetTable)
 
     val sourceTableOrQuery = if (ctx.source != null) {
-      UnresolvedRelation(visitMultipartIdentifier(ctx.source))
+      createUnresolvedRelation(ctx.source)
     } else if (ctx.sourceQuery != null) {
       visitQuery(ctx.sourceQuery)
     } else {
-      throw new ParseException("Empty source for merge: you should specify a source" +
-          " table/subquery in merge.", ctx.source)
+      throw QueryParsingErrors.emptySourceForMergeError(ctx)
     }
     val sourceTableAlias = getTableAliasWithoutColumnAlias(ctx.sourceAlias, "MERGE")
     val aliasedSource =
@@ -411,78 +413,63 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
 
     val mergeCondition = expression(ctx.mergeCondition)
 
-    val matchedClauses = ctx.matchedClause()
-    if (matchedClauses.size() > 2) {
-      throw new ParseException("There should be at most 2 'WHEN MATCHED' clauses.",
-        matchedClauses.get(2))
-    }
-    val matchedActions = matchedClauses.asScala.map {
+    val matchedActions = ctx.matchedClause().asScala.map {
       clause => {
         if (clause.matchedAction().DELETE() != null) {
           DeleteAction(Option(clause.matchedCond).map(expression))
         } else if (clause.matchedAction().UPDATE() != null) {
           val condition = Option(clause.matchedCond).map(expression)
           if (clause.matchedAction().ASTERISK() != null) {
-            UpdateAction(condition, Seq())
+            UpdateStarAction(condition)
           } else {
             UpdateAction(condition, withAssignments(clause.matchedAction().assignmentList()))
           }
         } else {
           // It should not be here.
-          throw new ParseException(
-            s"Unrecognized matched action: ${clause.matchedAction().getText}",
-            clause.matchedAction())
+          throw QueryParsingErrors.unrecognizedMatchedActionError(clause)
         }
       }
     }
-    val notMatchedClauses = ctx.notMatchedClause()
-    if (notMatchedClauses.size() > 1) {
-      throw new ParseException("There should be at most 1 'WHEN NOT MATCHED' clause.",
-        notMatchedClauses.get(1))
-    }
-    val notMatchedActions = notMatchedClauses.asScala.map {
+    val notMatchedActions = ctx.notMatchedClause().asScala.map {
       clause => {
         if (clause.notMatchedAction().INSERT() != null) {
           val condition = Option(clause.notMatchedCond).map(expression)
           if (clause.notMatchedAction().ASTERISK() != null) {
-            InsertAction(condition, Seq())
+            InsertStarAction(condition)
           } else {
             val columns = clause.notMatchedAction().columns.multipartIdentifier()
                 .asScala.map(attr => UnresolvedAttribute(visitMultipartIdentifier(attr)))
             val values = clause.notMatchedAction().expression().asScala.map(expression)
             if (columns.size != values.size) {
-              throw new ParseException("The number of inserted values cannot match the fields.",
-                clause.notMatchedAction())
+              throw QueryParsingErrors.insertedValueNumberNotMatchFieldNumberError(clause)
             }
-            InsertAction(condition, columns.zip(values).map(kv => Assignment(kv._1, kv._2)))
+            InsertAction(condition, columns.zip(values).map(kv => Assignment(kv._1, kv._2)).toSeq)
           }
         } else {
           // It should not be here.
-          throw new ParseException(
-            s"Unrecognized not matched action: ${clause.notMatchedAction().getText}",
-            clause.notMatchedAction())
+          throw QueryParsingErrors.unrecognizedNotMatchedActionError(clause)
         }
       }
     }
     if (matchedActions.isEmpty && notMatchedActions.isEmpty) {
-      throw new ParseException("There must be at least one WHEN clause in a MERGE statement", ctx)
+      throw QueryParsingErrors.mergeStatementWithoutWhenClauseError(ctx)
     }
     // children being empty means that the condition is not set
-    if (matchedActions.length == 2 && matchedActions.head.children.isEmpty) {
-      throw new ParseException("When there are 2 MATCHED clauses in a MERGE statement, " +
-        "the first MATCHED clause must have a condition", ctx)
+    val matchedActionSize = matchedActions.length
+    if (matchedActionSize >= 2 && !matchedActions.init.forall(_.condition.nonEmpty)) {
+      throw QueryParsingErrors.nonLastMatchedClauseOmitConditionError(ctx)
     }
-    if (matchedActions.groupBy(_.getClass).mapValues(_.size).exists(_._2 > 1)) {
-      throw new ParseException(
-        "UPDATE and DELETE can appear at most once in MATCHED clauses in a MERGE statement", ctx)
+    val notMatchedActionSize = notMatchedActions.length
+    if (notMatchedActionSize >= 2 && !notMatchedActions.init.forall(_.condition.nonEmpty)) {
+      throw QueryParsingErrors.nonLastNotMatchedClauseOmitConditionError(ctx)
     }
 
     MergeIntoTable(
       aliasedTarget,
       aliasedSource,
       mergeCondition,
-      matchedActions,
-      notMatchedActions)
+      matchedActions.toSeq,
+      notMatchedActions.toSeq)
   }
 
   /**
@@ -490,15 +477,21 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   override def visitPartitionSpec(
       ctx: PartitionSpecContext): Map[String, Option[String]] = withOrigin(ctx) {
+    val legacyNullAsString =
+      conf.getConf(SQLConf.LEGACY_PARSE_NULL_PARTITION_SPEC_AS_STRING_LITERAL)
     val parts = ctx.partitionVal.asScala.map { pVal =>
       val name = pVal.identifier.getText
-      val value = Option(pVal.constant).map(visitStringConstant)
+      val value = Option(pVal.constant).map(v => visitStringConstant(v, legacyNullAsString))
       name -> value
     }
     // Before calling `toMap`, we check duplicated keys to avoid silently ignore partition values
     // in partition spec like PARTITION(a='1', b='2', a='3'). The real semantical check for
     // partition columns will be done in analyzer.
-    checkDuplicateKeys(parts, ctx)
+    if (conf.caseSensitiveAnalysis) {
+      checkDuplicateKeys(parts.toSeq, ctx)
+    } else {
+      checkDuplicateKeys(parts.map(kv => kv._1.toLowerCase(Locale.ROOT) -> kv._2).toSeq, ctx)
+    }
     parts.toMap
   }
 
@@ -508,7 +501,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
   protected def visitNonOptionalPartitionSpec(
       ctx: PartitionSpecContext): Map[String, String] = withOrigin(ctx) {
     visitPartitionSpec(ctx).map {
-      case (key, None) => throw new ParseException(s"Found an empty partition key '$key'.", ctx)
+      case (key, None) => throw QueryParsingErrors.emptyPartitionKeyError(key, ctx)
       case (key, Some(value)) => key -> value
     }
   }
@@ -518,10 +511,19 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    * main purpose is to prevent slight differences due to back to back conversions i.e.:
    * String -> Literal -> String.
    */
-  protected def visitStringConstant(ctx: ConstantContext): String = withOrigin(ctx) {
-    ctx match {
-      case s: StringLiteralContext => createString(s)
-      case o => o.getText
+  protected def visitStringConstant(
+      ctx: ConstantContext,
+      legacyNullAsString: Boolean): String = withOrigin(ctx) {
+    expression(ctx) match {
+      case Literal(null, _) if !legacyNullAsString => null
+      case l @ Literal(null, _) => l.toString
+      case l: Literal =>
+        // TODO For v2 commands, we will cast the string back to its actual value,
+        //  which is a waste and can be improved in the future.
+        Cast(l, StringType, Some(conf.sessionLocalTimeZone)).eval().toString
+      case other =>
+        throw new IllegalArgumentException(s"Only literals are allowed in the " +
+          s"partition spec, but got ${other.sql}")
     }
   }
 
@@ -538,17 +540,17 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     val withOrder = if (
       !order.isEmpty && sort.isEmpty && distributeBy.isEmpty && clusterBy.isEmpty) {
       // ORDER BY ...
-      Sort(order.asScala.map(visitSortItem), global = true, query)
+      Sort(order.asScala.map(visitSortItem).toSeq, global = true, query)
     } else if (order.isEmpty && !sort.isEmpty && distributeBy.isEmpty && clusterBy.isEmpty) {
       // SORT BY ...
-      Sort(sort.asScala.map(visitSortItem), global = false, query)
+      Sort(sort.asScala.map(visitSortItem).toSeq, global = false, query)
     } else if (order.isEmpty && sort.isEmpty && !distributeBy.isEmpty && clusterBy.isEmpty) {
       // DISTRIBUTE BY ...
       withRepartitionByExpression(ctx, expressionList(distributeBy), query)
     } else if (order.isEmpty && !sort.isEmpty && !distributeBy.isEmpty && clusterBy.isEmpty) {
       // SORT BY ... DISTRIBUTE BY ...
       Sort(
-        sort.asScala.map(visitSortItem),
+        sort.asScala.map(visitSortItem).toSeq,
         global = false,
         withRepartitionByExpression(ctx, expressionList(distributeBy), query))
     } else if (order.isEmpty && sort.isEmpty && distributeBy.isEmpty && !clusterBy.isEmpty) {
@@ -562,8 +564,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       // [EMPTY]
       query
     } else {
-      throw new ParseException(
-        "Combination of ORDER BY/SORT BY/DISTRIBUTE BY/CLUSTER BY is not supported", ctx)
+      throw QueryParsingErrors.combinationQueryResultClausesUnsupportedError(ctx)
     }
 
     // WINDOWS
@@ -583,7 +584,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       ctx: QueryOrganizationContext,
       expressions: Seq[Expression],
       query: LogicalPlan): LogicalPlan = {
-    throw new ParseException("DISTRIBUTE BY is not supported", ctx)
+    throw QueryParsingErrors.distributeByUnsupportedError(ctx)
   }
 
   override def visitTransformQuerySpecification(
@@ -591,7 +592,16 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     val from = OneRowRelation().optional(ctx.fromClause) {
       visitFromClause(ctx.fromClause)
     }
-    withTransformQuerySpecification(ctx, ctx.transformClause, ctx.whereClause, from)
+    withTransformQuerySpecification(
+      ctx,
+      ctx.transformClause,
+      ctx.lateralView,
+      ctx.whereClause,
+      ctx.aggregationClause,
+      ctx.havingClause,
+      ctx.windowClause,
+      from
+    )
   }
 
   override def visitRegularQuerySpecification(
@@ -618,6 +628,12 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       .map(typedVisit[Expression])
   }
 
+  override def visitExpressionSeq(ctx: ExpressionSeqContext): Seq[Expression] = {
+    Option(ctx).toSeq
+      .flatMap(_.expression.asScala)
+      .map(typedVisit[Expression])
+  }
+
   /**
    * Create a logical plan using a having clause.
    */
@@ -645,14 +661,15 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
   private def withTransformQuerySpecification(
       ctx: ParserRuleContext,
       transformClause: TransformClauseContext,
+      lateralView: java.util.List[LateralViewContext],
       whereClause: WhereClauseContext,
-    relation: LogicalPlan): LogicalPlan = withOrigin(ctx) {
-    // Add where.
-    val withFilter = relation.optionalMap(whereClause)(withWhereClause)
-
-    // Create the transform.
-    val expressions = visitNamedExpressionSeq(transformClause.namedExpressionSeq)
-
+      aggregationClause: AggregationClauseContext,
+      havingClause: HavingClauseContext,
+      windowClause: WindowClauseContext,
+      relation: LogicalPlan): LogicalPlan = withOrigin(ctx) {
+    if (transformClause.setQuantifier != null) {
+      throw QueryParsingErrors.transformNotSupportQuantifierError(transformClause.setQuantifier)
+    }
     // Create the attributes.
     val (attributes, schemaLess) = if (transformClause.colTypeList != null) {
       // Typed return columns.
@@ -668,12 +685,20 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
         AttributeReference("value", StringType)()), true)
     }
 
-    // Create the transform.
+    val plan = visitCommonSelectQueryClausePlan(
+      relation,
+      visitExpressionSeq(transformClause.expressionSeq),
+      lateralView,
+      whereClause,
+      aggregationClause,
+      havingClause,
+      windowClause,
+      isDistinct = false)
+
     ScriptTransformation(
-      expressions,
       string(transformClause.script),
       attributes,
-      withFilter,
+      plan,
       withScriptIOSchema(
         ctx,
         transformClause.inRowFormat,
@@ -701,13 +726,38 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       havingClause: HavingClauseContext,
       windowClause: WindowClauseContext,
       relation: LogicalPlan): LogicalPlan = withOrigin(ctx) {
+    val isDistinct = selectClause.setQuantifier() != null &&
+      selectClause.setQuantifier().DISTINCT() != null
+
+    val plan = visitCommonSelectQueryClausePlan(
+      relation,
+      visitNamedExpressionSeq(selectClause.namedExpressionSeq),
+      lateralView,
+      whereClause,
+      aggregationClause,
+      havingClause,
+      windowClause,
+      isDistinct)
+
+    // Hint
+    selectClause.hints.asScala.foldRight(plan)(withHints)
+  }
+
+  def visitCommonSelectQueryClausePlan(
+      relation: LogicalPlan,
+      expressions: Seq[Expression],
+      lateralView: java.util.List[LateralViewContext],
+      whereClause: WhereClauseContext,
+      aggregationClause: AggregationClauseContext,
+      havingClause: HavingClauseContext,
+      windowClause: WindowClauseContext,
+      isDistinct: Boolean): LogicalPlan = {
     // Add lateral views.
     val withLateralView = lateralView.asScala.foldLeft(relation)(withGenerate)
 
     // Add where.
     val withFilter = withLateralView.optionalMap(whereClause)(withWhereClause)
 
-    val expressions = visitNamedExpressionSeq(selectClause.namedExpressionSeq)
     // Add aggregation or a project.
     val namedExpressions = expressions.map {
       case e: NamedExpression => e
@@ -723,7 +773,11 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     val withProject = if (aggregationClause == null && havingClause != null) {
       if (conf.getConf(SQLConf.LEGACY_HAVING_WITHOUT_GROUP_BY_AS_WHERE)) {
         // If the legacy conf is set, treat HAVING without GROUP BY as WHERE.
-        withHavingClause(havingClause, createProject())
+        val predicate = expression(havingClause.booleanExpression) match {
+          case p: Predicate => p
+          case e => Cast(e, BooleanType)
+        }
+        Filter(predicate, createProject())
       } else {
         // According to SQL standard, HAVING without GROUP BY means global aggregate.
         withHavingClause(havingClause, Aggregate(Nil, namedExpressions, withFilter))
@@ -737,9 +791,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     }
 
     // Distinct
-    val withDistinct = if (
-      selectClause.setQuantifier() != null &&
-      selectClause.setQuantifier().DISTINCT() != null) {
+    val withDistinct = if (isDistinct) {
       Distinct(withProject)
     } else {
       withProject
@@ -748,12 +800,36 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     // Window
     val withWindow = withDistinct.optionalMap(windowClause)(withWindowClause)
 
-    // Hint
-    selectClause.hints.asScala.foldRight(withWindow)(withHints)
+    withWindow
+  }
+
+  // Script Transform's input/output format.
+  type ScriptIOFormat =
+    (Seq[(String, String)], Option[String], Seq[(String, String)], Option[String])
+
+  protected def getRowFormatDelimited(ctx: RowFormatDelimitedContext): ScriptIOFormat = {
+    // TODO we should use the visitRowFormatDelimited function here. However HiveScriptIOSchema
+    // expects a seq of pairs in which the old parsers' token names are used as keys.
+    // Transforming the result of visitRowFormatDelimited would be quite a bit messier than
+    // retrieving the key value pairs ourselves.
+    val entries = entry("TOK_TABLEROWFORMATFIELD", ctx.fieldsTerminatedBy) ++
+      entry("TOK_TABLEROWFORMATCOLLITEMS", ctx.collectionItemsTerminatedBy) ++
+      entry("TOK_TABLEROWFORMATMAPKEYS", ctx.keysTerminatedBy) ++
+      entry("TOK_TABLEROWFORMATNULL", ctx.nullDefinedAs) ++
+      Option(ctx.linesSeparatedBy).toSeq.map { token =>
+        val value = string(token)
+        validate(
+          value == "\n",
+          s"LINES TERMINATED BY only supports newline '\\n' right now: $value",
+          ctx)
+        "TOK_TABLEROWFORMATLINES" -> value
+      }
+
+    (entries, None, Seq.empty, None)
   }
 
   /**
-   * Create a (Hive based) [[ScriptInputOutputSchema]].
+   * Create a [[ScriptInputOutputSchema]].
    */
   protected def withScriptIOSchema(
       ctx: ParserRuleContext,
@@ -762,7 +838,30 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       outRowFormat: RowFormatContext,
       recordReader: Token,
       schemaLess: Boolean): ScriptInputOutputSchema = {
-    throw new ParseException("Script Transform is not supported", ctx)
+
+    def format(fmt: RowFormatContext): ScriptIOFormat = fmt match {
+      case c: RowFormatDelimitedContext =>
+        getRowFormatDelimited(c)
+
+      case c: RowFormatSerdeContext =>
+        throw QueryParsingErrors.transformWithSerdeUnsupportedError(ctx)
+
+      // SPARK-32106: When there is no definition about format, we return empty result
+      // to use a built-in default Serde in SparkScriptTransformationExec.
+      case null =>
+        (Nil, None, Seq.empty, None)
+    }
+
+    val (inFormat, inSerdeClass, inSerdeProps, reader) = format(inRowFormat)
+
+    val (outFormat, outSerdeClass, outSerdeProps, writer) = format(outRowFormat)
+
+    ScriptInputOutputSchema(
+      inFormat, outFormat,
+      inSerdeClass, outSerdeClass,
+      inSerdeProps, outSerdeProps,
+      reader, writer,
+      schemaLess)
   }
 
   /**
@@ -777,7 +876,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     }
     if (ctx.pivotClause() != null) {
       if (!ctx.lateralView.isEmpty) {
-        throw new ParseException("LATERAL cannot be used together with PIVOT in FROM clause", ctx)
+        throw QueryParsingErrors.lateralWithPivotInFromClauseNotAllowedError(ctx)
       }
       withPivot(ctx.pivotClause, from)
     } else {
@@ -825,10 +924,16 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       ctx: WindowClauseContext,
       query: LogicalPlan): LogicalPlan = withOrigin(ctx) {
     // Collect all window specifications defined in the WINDOW clause.
-    val baseWindowMap = ctx.namedWindow.asScala.map {
+    val baseWindowTuples = ctx.namedWindow.asScala.map {
       wCtx =>
         (wCtx.name.getText, typedVisit[WindowSpec](wCtx.windowSpec))
-    }.toMap
+    }
+    baseWindowTuples.groupBy(_._1).foreach { kv =>
+      if (kv._2.size > 1) {
+        throw QueryParsingErrors.repetitiveWindowDefinitionError(kv._1, ctx)
+      }
+    }
+    val baseWindowMap = baseWindowTuples.toMap
 
     // Handle cases like
     // window w1 as (partition by p_mfgr order by p_name
@@ -840,42 +945,90 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
           case Some(spec: WindowSpecDefinition) =>
             spec
           case Some(ref) =>
-            throw new ParseException(s"Window reference '$name' is not a window specification", ctx)
+            throw QueryParsingErrors.invalidWindowReferenceError(name, ctx)
           case None =>
-            throw new ParseException(s"Cannot resolve window reference '$name'", ctx)
+            throw QueryParsingErrors.cannotResolveWindowReferenceError(name, ctx)
         }
       case spec: WindowSpecDefinition => spec
     }
 
     // Note that mapValues creates a view instead of materialized map. We force materialization by
     // mapping over identity.
-    WithWindowDefinition(windowMapView.map(identity), query)
+    WithWindowDefinition(windowMapView.map(identity).toMap, query)
   }
 
   /**
-   * Add an [[Aggregate]] or [[GroupingSets]] to a logical plan.
+   * Add an [[Aggregate]] to a logical plan.
    */
   private def withAggregationClause(
       ctx: AggregationClauseContext,
       selectExpressions: Seq[NamedExpression],
       query: LogicalPlan): LogicalPlan = withOrigin(ctx) {
-    val groupByExpressions = expressionList(ctx.groupingExpressions)
+    if (ctx.groupingExpressionsWithGroupingAnalytics.isEmpty) {
+      val groupByExpressions = expressionList(ctx.groupingExpressions)
+      if (ctx.GROUPING != null) {
+        // GROUP BY ... GROUPING SETS (...)
+        // `groupByExpressions` can be non-empty for Hive compatibility. It may add extra grouping
+        // expressions that do not exist in GROUPING SETS (...), and the value is always null.
+        // For example, `SELECT a, b, c FROM ... GROUP BY a, b, c GROUPING SETS (a, b)`, the output
+        // of column `c` is always null.
+        val groupingSets =
+          ctx.groupingSet.asScala.map(_.expression.asScala.map(e => expression(e)).toSeq)
+        Aggregate(Seq(GroupingSets(groupingSets.toSeq, groupByExpressions)),
+          selectExpressions, query)
+      } else {
+        // GROUP BY .... (WITH CUBE | WITH ROLLUP)?
+        val mappedGroupByExpressions = if (ctx.CUBE != null) {
+          Seq(Cube(groupByExpressions.map(Seq(_))))
+        } else if (ctx.ROLLUP != null) {
+          Seq(Rollup(groupByExpressions.map(Seq(_))))
+        } else {
+          groupByExpressions
+        }
+        Aggregate(mappedGroupByExpressions, selectExpressions, query)
+      }
+    } else {
+      val groupByExpressions =
+        ctx.groupingExpressionsWithGroupingAnalytics.asScala
+          .map(groupByExpr => {
+            val groupingAnalytics = groupByExpr.groupingAnalytics
+            if (groupingAnalytics != null) {
+              visitGroupingAnalytics(groupingAnalytics)
+            } else {
+              expression(groupByExpr.expression)
+            }
+          })
+      Aggregate(groupByExpressions.toSeq, selectExpressions, query)
+    }
+  }
 
-    if (ctx.GROUPING != null) {
-      // GROUP BY .... GROUPING SETS (...)
-      val selectedGroupByExprs =
-        ctx.groupingSet.asScala.map(_.expression.asScala.map(e => expression(e)))
-      GroupingSets(selectedGroupByExprs, groupByExpressions, query, selectExpressions)
+  override def visitGroupingAnalytics(
+      groupingAnalytics: GroupingAnalyticsContext): BaseGroupingSets = {
+    val groupingSets = groupingAnalytics.groupingSet.asScala
+      .map(_.expression.asScala.map(e => expression(e)).toSeq)
+    if (groupingAnalytics.CUBE != null) {
+      // CUBE(A, B, (A, B), ()) is not supported.
+      if (groupingSets.exists(_.isEmpty)) {
+        throw QueryParsingErrors.invalidGroupingSetError("CUBE", groupingAnalytics)
+      }
+      Cube(groupingSets.toSeq)
+    } else if (groupingAnalytics.ROLLUP != null) {
+      // ROLLUP(A, B, (A, B), ()) is not supported.
+      if (groupingSets.exists(_.isEmpty)) {
+        throw QueryParsingErrors.invalidGroupingSetError("ROLLUP", groupingAnalytics)
+      }
+      Rollup(groupingSets.toSeq)
     } else {
-      // GROUP BY .... (WITH CUBE | WITH ROLLUP)?
-      val mappedGroupByExpressions = if (ctx.CUBE != null) {
-        Seq(Cube(groupByExpressions))
-      } else if (ctx.ROLLUP != null) {
-        Seq(Rollup(groupByExpressions))
-      } else {
-        groupByExpressions
+      assert(groupingAnalytics.GROUPING != null && groupingAnalytics.SETS != null)
+      val groupingSets = groupingAnalytics.groupingElement.asScala.flatMap { expr =>
+        val groupingAnalytics = expr.groupingAnalytics()
+        if (groupingAnalytics != null) {
+          visitGroupingAnalytics(groupingAnalytics).selectedGroupByExprs
+        } else {
+          Seq(expr.groupingSet().expression().asScala.map(e => expression(e)).toSeq)
+        }
       }
-      Aggregate(mappedGroupByExpressions, selectExpressions, query)
+      GroupingSets(groupingSets.toSeq)
     }
   }
 
@@ -886,8 +1039,9 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       ctx: HintContext,
       query: LogicalPlan): LogicalPlan = withOrigin(ctx) {
     var plan = query
-    ctx.hintStatements.asScala.reverse.foreach { case stmt =>
-      plan = UnresolvedHint(stmt.hintName.getText, stmt.parameters.asScala.map(expression), plan)
+    ctx.hintStatements.asScala.reverse.foreach { stmt =>
+      plan = UnresolvedHint(stmt.hintName.getText,
+        stmt.parameters.asScala.map(expression).toSeq, plan)
     }
     plan
   }
@@ -906,10 +1060,10 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     } else {
       CreateStruct(
         ctx.pivotColumn.identifiers.asScala.map(
-          identifier => UnresolvedAttribute.quoted(identifier.getText)))
+          identifier => UnresolvedAttribute.quoted(identifier.getText)).toSeq)
     }
     val pivotValues = ctx.pivotValues.asScala.map(visitPivotValue)
-    Pivot(None, pivotColumn, pivotValues, aggregates, query)
+    Pivot(None, pivotColumn, pivotValues.toSeq, aggregates, query)
   }
 
   /**
@@ -938,7 +1092,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       // scalastyle:off caselocale
       Some(ctx.tblName.getText.toLowerCase),
       // scalastyle:on caselocale
-      ctx.colName.asScala.map(_.getText).map(UnresolvedAttribute.apply),
+      ctx.colName.asScala.map(_.getText).map(UnresolvedAttribute.quoted).toSeq,
       query)
   }
 
@@ -976,9 +1130,11 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
             (UsingJoin(baseJoinType, visitIdentifierList(c.identifierList)), None)
           case Some(c) if c.booleanExpression != null =>
             (baseJoinType, Option(expression(c.booleanExpression)))
+          case Some(c) =>
+            throw QueryParsingErrors.joinCriteriaUnimplementedError(c, ctx)
           case None if join.NATURAL != null =>
             if (baseJoinType == Cross) {
-              throw new ParseException("NATURAL CROSS JOIN is not supported", ctx)
+              throw QueryParsingErrors.naturalCrossJoinUnsupportedError(ctx)
             }
             (NaturalJoin(baseJoinType), None)
           case None =>
@@ -1012,7 +1168,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     }
 
     if (ctx.sampleMethod() == null) {
-      throw new ParseException("TABLESAMPLE does not accept empty inputs.", ctx)
+      throw QueryParsingErrors.emptyInputForTableSampleError(ctx)
     }
 
     ctx.sampleMethod() match {
@@ -1027,20 +1183,18 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       case ctx: SampleByBytesContext =>
         val bytesStr = ctx.bytes.getText
         if (bytesStr.matches("[0-9]+[bBkKmMgG]")) {
-          throw new ParseException("TABLESAMPLE(byteLengthLiteral) is not supported", ctx)
+          throw QueryParsingErrors.tableSampleByBytesUnsupportedError("byteLengthLiteral", ctx)
         } else {
-          throw new ParseException(
-            bytesStr + " is not a valid byte length literal, " +
-              "expected syntax: DIGIT+ ('B' | 'K' | 'M' | 'G')", ctx)
+          throw QueryParsingErrors.invalidByteLengthLiteralError(bytesStr, ctx)
         }
 
       case ctx: SampleByBucketContext if ctx.ON() != null =>
         if (ctx.identifier != null) {
-          throw new ParseException(
-            "TABLESAMPLE(BUCKET x OUT OF y ON colname) is not supported", ctx)
+          throw QueryParsingErrors.tableSampleByBytesUnsupportedError(
+            "BUCKET x OUT OF y ON colname", ctx)
         } else {
-          throw new ParseException(
-            "TABLESAMPLE(BUCKET x OUT OF y ON function) is not supported", ctx)
+          throw QueryParsingErrors.tableSampleByBytesUnsupportedError(
+            "BUCKET x OUT OF y ON function", ctx)
         }
 
       case ctx: SampleByBucketContext =>
@@ -1087,9 +1241,13 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     } else {
       Seq.empty
     }
+    val name = getFunctionIdentifier(func.functionName)
+    if (name.database.nonEmpty) {
+      operationNotAllowed(s"table valued function cannot specify database name: $name", ctx)
+    }
 
     val tvf = UnresolvedTableValuedFunction(
-      func.funcName.getText, func.expression.asScala.map(expression), aliases)
+      name, func.expression.asScala.map(expression).toSeq, aliases)
     tvf.optionalMap(func.tableAlias.strictIdentifier)(aliasPlan)
   }
 
@@ -1114,7 +1272,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       Seq.tabulate(rows.head.size)(i => s"col${i + 1}")
     }
 
-    val table = UnresolvedInlineTable(aliases, rows)
+    val table = UnresolvedInlineTable(aliases, rows.toSeq)
     table.optionalMap(ctx.tableAlias.strictIdentifier)(aliasPlan)
   }
 
@@ -1165,12 +1323,12 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   private def mayApplyAliasPlan(tableAlias: TableAliasContext, plan: LogicalPlan): LogicalPlan = {
     if (tableAlias.strictIdentifier != null) {
-      val subquery = SubqueryAlias(tableAlias.strictIdentifier.getText, plan)
+      val alias = tableAlias.strictIdentifier.getText
       if (tableAlias.identifierList != null) {
         val columnNames = visitIdentifierList(tableAlias.identifierList)
-        UnresolvedSubqueryColumnAliases(columnNames, subquery)
+        SubqueryAlias(alias, UnresolvedSubqueryColumnAliases(columnNames, plan))
       } else {
-        subquery
+        SubqueryAlias(alias, plan)
       }
     } else {
       plan
@@ -1188,7 +1346,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    * Create a Sequence of Strings for an identifier list.
    */
   override def visitIdentifierSeq(ctx: IdentifierSeqContext): Seq[String] = withOrigin(ctx) {
-    ctx.ident.asScala.map(_.getText)
+    ctx.ident.asScala.map(_.getText).toSeq
   }
 
   /* ********************************************************************************************
@@ -1213,10 +1371,10 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
   /**
    * Create a multi-part identifier.
    */
-  override def visitMultipartIdentifier(
-      ctx: MultipartIdentifierContext): Seq[String] = withOrigin(ctx) {
-    ctx.parts.asScala.map(_.getText)
-  }
+  override def visitMultipartIdentifier(ctx: MultipartIdentifierContext): Seq[String] =
+    withOrigin(ctx) {
+      ctx.parts.asScala.map(_.getText).toSeq
+    }
 
   /* ********************************************************************************************
    * Expression parsing
@@ -1231,7 +1389,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    * Create sequence of expressions from the given sequence of contexts.
    */
   private def expressionList(trees: java.util.List[ExpressionContext]): Seq[Expression] = {
-    trees.asScala.map(expression)
+    trees.asScala.map(expression).toSeq
   }
 
   /**
@@ -1239,7 +1397,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    * Both un-targeted (global) and targeted aliases are supported.
    */
   override def visitStar(ctx: StarContext): Expression = withOrigin(ctx) {
-    UnresolvedStar(Option(ctx.qualifiedName()).map(_.identifier.asScala.map(_.getText)))
+    UnresolvedStar(Option(ctx.qualifiedName()).map(_.identifier.asScala.map(_.getText).toSeq))
   }
 
   /**
@@ -1391,14 +1549,6 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       case other => Seq(other)
     }
 
-    def getLikeQuantifierExprs(expressions: java.util.List[ExpressionContext]): Seq[Expression] = {
-      if (expressions.isEmpty) {
-        throw new ParseException("Expected something between '(' and ')'.", ctx)
-      } else {
-        expressions.asScala.map(expression).map(p => invertIfNotDefined(new Like(e, p)))
-      }
-    }
-
     // Create the predicate.
     ctx.kind.getType match {
       case SqlBaseParser.BETWEEN =>
@@ -1409,18 +1559,43 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       case SqlBaseParser.IN if ctx.query != null =>
         invertIfNotDefined(InSubquery(getValueExpressions(e), ListQuery(plan(ctx.query))))
       case SqlBaseParser.IN =>
-        invertIfNotDefined(In(e, ctx.expression.asScala.map(expression)))
+        invertIfNotDefined(In(e, ctx.expression.asScala.map(expression).toSeq))
       case SqlBaseParser.LIKE =>
         Option(ctx.quantifier).map(_.getType) match {
           case Some(SqlBaseParser.ANY) | Some(SqlBaseParser.SOME) =>
-            getLikeQuantifierExprs(ctx.expression).reduceLeft(Or)
+            validate(!ctx.expression.isEmpty, "Expected something between '(' and ')'.", ctx)
+            val expressions = expressionList(ctx.expression)
+            if (expressions.forall(_.foldable) && expressions.forall(_.dataType == StringType)) {
+              // If there are many pattern expressions, will throw StackOverflowError.
+              // So we use LikeAny or NotLikeAny instead.
+              val patterns = expressions.map(_.eval(EmptyRow).asInstanceOf[UTF8String])
+              ctx.NOT match {
+                case null => LikeAny(e, patterns)
+                case _ => NotLikeAny(e, patterns)
+              }
+            } else {
+              ctx.expression.asScala.map(expression)
+                .map(p => invertIfNotDefined(new Like(e, p))).toSeq.reduceLeft(Or)
+            }
           case Some(SqlBaseParser.ALL) =>
-            getLikeQuantifierExprs(ctx.expression).reduceLeft(And)
+            validate(!ctx.expression.isEmpty, "Expected something between '(' and ')'.", ctx)
+            val expressions = expressionList(ctx.expression)
+            if (expressions.forall(_.foldable) && expressions.forall(_.dataType == StringType)) {
+              // If there are many pattern expressions, will throw StackOverflowError.
+              // So we use LikeAll or NotLikeAll instead.
+              val patterns = expressions.map(_.eval(EmptyRow).asInstanceOf[UTF8String])
+              ctx.NOT match {
+                case null => LikeAll(e, patterns)
+                case _ => NotLikeAll(e, patterns)
+              }
+            } else {
+              ctx.expression.asScala.map(expression)
+                .map(p => invertIfNotDefined(new Like(e, p))).toSeq.reduceLeft(And)
+            }
           case _ =>
             val escapeChar = Option(ctx.escapeChar).map(string).map { str =>
               if (str.length != 1) {
-                throw new ParseException("Invalid escape string." +
-                  "Escape string must contain only one character.", ctx)
+                throw QueryParsingErrors.invalidEscapeStringError(ctx)
               }
               str.charAt(0)
             }.getOrElse('\\')
@@ -1527,14 +1702,24 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    * Create a [[Cast]] expression.
    */
   override def visitCast(ctx: CastContext): Expression = withOrigin(ctx) {
-    Cast(expression(ctx.expression), visitSparkDataType(ctx.dataType))
+    val rawDataType = typedVisit[DataType](ctx.dataType())
+    val dataType = CharVarcharUtils.replaceCharVarcharWithStringForCast(rawDataType)
+    val cast = ctx.name.getType match {
+      case SqlBaseParser.CAST =>
+        Cast(expression(ctx.expression), dataType)
+
+      case SqlBaseParser.TRY_CAST =>
+        TryCast(expression(ctx.expression), dataType)
+    }
+    cast.setTagValue(Cast.USER_SPECIFIED_CAST, true)
+    cast
   }
 
   /**
    * Create a [[CreateStruct]] expression.
    */
   override def visitStruct(ctx: StructContext): Expression = withOrigin(ctx) {
-    CreateStruct(ctx.argument.asScala.map(expression))
+    CreateStruct.create(ctx.argument.asScala.map(expression).toSeq)
   }
 
   /**
@@ -1542,7 +1727,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   override def visitFirst(ctx: FirstContext): Expression = withOrigin(ctx) {
     val ignoreNullsExpr = ctx.IGNORE != null
-    First(expression(ctx.expression), Literal(ignoreNullsExpr)).toAggregateExpression()
+    First(expression(ctx.expression), ignoreNullsExpr).toAggregateExpression()
   }
 
   /**
@@ -1550,7 +1735,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   override def visitLast(ctx: LastContext): Expression = withOrigin(ctx) {
     val ignoreNullsExpr = ctx.IGNORE != null
-    Last(expression(ctx.expression), Literal(ignoreNullsExpr)).toAggregateExpression()
+    Last(expression(ctx.expression), ignoreNullsExpr).toAggregateExpression()
   }
 
   /**
@@ -1593,8 +1778,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       case SqlBaseParser.TRAILING =>
         StringTrimRight(srcStr, trimStr)
       case other =>
-        throw new ParseException("Function trim doesn't support with " +
-          s"type $other. Please use BOTH, LEADING or TRAILING as trim type", ctx)
+        throw QueryParsingErrors.trimOptionUnsupportedError(other, ctx)
     }
   }
 
@@ -1619,7 +1803,8 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     // Create the function call.
     val name = ctx.functionName.getText
     val isDistinct = Option(ctx.setQuantifier()).exists(_.DISTINCT != null)
-    val arguments = ctx.argument.asScala.map(expression) match {
+    // Call `toSeq`, otherwise `ctx.argument.asScala.map(expression)` is `Buffer` in Scala 2.13
+    val arguments = ctx.argument.asScala.map(expression).toSeq match {
       case Seq(UnresolvedStar(None))
         if name.toLowerCase(Locale.ROOT) == "count" && !isDistinct =>
         // Transform COUNT(*) into COUNT(1).
@@ -1628,8 +1813,10 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
         expressions
     }
     val filter = Option(ctx.where).map(expression(_))
+    val ignoreNulls =
+      Option(ctx.nullsOption).map(_.getType == SqlBaseParser.IGNORE).getOrElse(false)
     val function = UnresolvedFunction(
-      getFunctionIdentifier(ctx.functionName), arguments, isDistinct, filter)
+      getFunctionMultiparts(ctx.functionName), arguments, isDistinct, filter, ignoreNulls)
 
     // Check if the function is evaluated in a windowed context.
     ctx.windowSpec match {
@@ -1641,20 +1828,11 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     }
   }
 
-
-  /**
-   * Create a function database (optional) and name pair, for multipartIdentifier.
-   * This is used in CREATE FUNCTION, DROP FUNCTION, SHOWFUNCTIONS.
-   */
-  protected def visitFunctionName(ctx: MultipartIdentifierContext): FunctionIdentifier = {
-    visitFunctionName(ctx, ctx.parts.asScala.map(_.getText))
-  }
-
   /**
    * Create a function database (optional) and name pair.
    */
   protected def visitFunctionName(ctx: QualifiedNameContext): FunctionIdentifier = {
-    visitFunctionName(ctx, ctx.identifier().asScala.map(_.getText))
+    visitFunctionName(ctx, ctx.identifier().asScala.map(_.getText).toSeq)
   }
 
   /**
@@ -1665,7 +1843,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       case Seq(db, fn) => FunctionIdentifier(fn, Option(db))
       case Seq(fn) => FunctionIdentifier(fn, None)
       case other =>
-        throw new ParseException(s"Unsupported function name '${texts.mkString(".")}'", ctx)
+        throw QueryParsingErrors.functionNameUnsupportedError(texts.mkString("."), ctx)
     }
   }
 
@@ -1680,6 +1858,14 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     }
   }
 
+  protected def getFunctionMultiparts(ctx: FunctionNameContext): Seq[String] = {
+    if (ctx.qualifiedName != null) {
+      ctx.qualifiedName().identifier().asScala.map(_.getText).toSeq
+    } else {
+      Seq(ctx.getText)
+    }
+  }
+
   /**
    * Create an [[LambdaFunction]].
    */
@@ -1690,7 +1876,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     val function = expression(ctx.expression).transformUp {
       case a: UnresolvedAttribute => UnresolvedNamedLambdaVariable(a.nameParts)
     }
-    LambdaFunction(function, arguments)
+    LambdaFunction(function, arguments.toSeq)
   }
 
   /**
@@ -1722,8 +1908,8 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     }
 
     WindowSpecDefinition(
-      partition,
-      order,
+      partition.toSeq,
+      order.toSeq,
       frameSpecOption.getOrElse(UnspecifiedFrame))
   }
 
@@ -1755,7 +1941,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    * Create a [[CreateStruct]] expression.
    */
   override def visitRowConstructor(ctx: RowConstructorContext): Expression = withOrigin(ctx) {
-    CreateStruct(ctx.namedExpression().asScala.map(expression))
+    CreateStruct(ctx.namedExpression().asScala.map(expression).toSeq)
   }
 
   /**
@@ -1781,7 +1967,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     val branches = ctx.whenClause.asScala.map { wCtx =>
       (EqualTo(e, expression(wCtx.condition)), expression(wCtx.result))
     }
-    CaseWhen(branches, Option(ctx.elseExpression).map(expression))
+    CaseWhen(branches.toSeq, Option(ctx.elseExpression).map(expression))
   }
 
   /**
@@ -1800,7 +1986,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     val branches = ctx.whenClause.asScala.map { wCtx =>
       (expression(wCtx.condition), expression(wCtx.result))
     }
-    CaseWhen(branches, Option(ctx.elseExpression).map(expression))
+    CaseWhen(branches.toSeq, Option(ctx.elseExpression).map(expression))
   }
 
   /**
@@ -1886,7 +2072,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     } else {
       direction.defaultNullOrdering
     }
-    SortOrder(expression(ctx.expression), direction, nullOrdering, Set.empty)
+    SortOrder(expression(ctx.expression), direction, nullOrdering, Seq.empty)
   }
 
   /**
@@ -1902,22 +2088,22 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
 
     def toLiteral[T](f: UTF8String => Option[T], t: DataType): Literal = {
       f(UTF8String.fromString(value)).map(Literal(_, t)).getOrElse {
-        throw new ParseException(s"Cannot parse the $valueType value: $value", ctx)
+        throw QueryParsingErrors.cannotParseValueTypeError(valueType, value, ctx)
       }
     }
     try {
       valueType match {
         case "DATE" =>
-          toLiteral(stringToDate(_, getZoneId(SQLConf.get.sessionLocalTimeZone)), DateType)
+          toLiteral(stringToDate(_, getZoneId(conf.sessionLocalTimeZone)), DateType)
         case "TIMESTAMP" =>
-          val zoneId = getZoneId(SQLConf.get.sessionLocalTimeZone)
+          val zoneId = getZoneId(conf.sessionLocalTimeZone)
           toLiteral(stringToTimestamp(_, zoneId), TimestampType)
         case "INTERVAL" =>
           val interval = try {
             IntervalUtils.stringToInterval(UTF8String.fromString(value))
           } catch {
             case e: IllegalArgumentException =>
-              val ex = new ParseException("Cannot parse the INTERVAL value: " + value, ctx)
+              val ex = QueryParsingErrors.cannotParseIntervalValueError(value, ctx)
               ex.setStackTrace(e.getStackTrace)
               throw ex
           }
@@ -1926,13 +2112,11 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
           val padding = if (value.length % 2 != 0) "0" else ""
           Literal(DatatypeConverter.parseHexBinary(padding + value))
         case other =>
-          throw new ParseException(s"Literals of type '$other' are currently not" +
-            " supported.", ctx)
+          throw QueryParsingErrors.literalValueTypeUnsupportedError(other, ctx)
       }
     } catch {
       case e: IllegalArgumentException =>
-        val message = Option(e.getMessage).getOrElse(s"Exception parsing $valueType")
-        throw new ParseException(message, ctx)
+        throw QueryParsingErrors.parsingValueTypeError(e, valueType, ctx)
     }
   }
 
@@ -2001,8 +2185,8 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     try {
       val rawBigDecimal = BigDecimal(rawStrippedQualifier)
       if (rawBigDecimal < minValue || rawBigDecimal > maxValue) {
-        throw new ParseException(s"Numeric literal ${rawStrippedQualifier} does not " +
-          s"fit in range [${minValue}, ${maxValue}] for type ${typeName}", ctx)
+        throw QueryParsingErrors.invalidNumericLiteralRangeError(
+          rawStrippedQualifier, minValue, maxValue, typeName, ctx)
       }
       Literal(converter(rawStrippedQualifier))
     } catch {
@@ -2038,6 +2222,15 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       Long.MinValue, Long.MaxValue, LongType.simpleString)(_.toLong)
   }
 
+  /**
+   * Create a Float Literal expression.
+   */
+  override def visitFloatLiteral(ctx: FloatLiteralContext): Literal = {
+    val rawStrippedQualifier = ctx.getText.substring(0, ctx.getText.length - 1)
+    numericLiteral(ctx, rawStrippedQualifier,
+      Float.MinValue, Float.MaxValue, FloatType.simpleString)(_.toFloat)
+  }
+
   /**
    * Create a Double Literal expression.
    */
@@ -2083,30 +2276,91 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
   }
 
   /**
-   * Create a [[CalendarInterval]] literal expression. Two syntaxes are supported:
+   * Create an [[UnresolvedRelation]] from a multi-part identifier context.
+   */
+  private def createUnresolvedRelation(
+      ctx: MultipartIdentifierContext): UnresolvedRelation = withOrigin(ctx) {
+    UnresolvedRelation(visitMultipartIdentifier(ctx))
+  }
+
+  /**
+   * Create an [[UnresolvedTable]] from a multi-part identifier context.
+   */
+  private def createUnresolvedTable(
+      ctx: MultipartIdentifierContext,
+      commandName: String,
+      relationTypeMismatchHint: Option[String] = None): UnresolvedTable = withOrigin(ctx) {
+    UnresolvedTable(visitMultipartIdentifier(ctx), commandName, relationTypeMismatchHint)
+  }
+
+  /**
+   * Create an [[UnresolvedView]] from a multi-part identifier context.
+   */
+  private def createUnresolvedView(
+      ctx: MultipartIdentifierContext,
+      commandName: String,
+      allowTemp: Boolean = true,
+      relationTypeMismatchHint: Option[String] = None): UnresolvedView = withOrigin(ctx) {
+    UnresolvedView(visitMultipartIdentifier(ctx), commandName, allowTemp, relationTypeMismatchHint)
+  }
+
+  /**
+   * Create an [[UnresolvedTableOrView]] from a multi-part identifier context.
+   */
+  private def createUnresolvedTableOrView(
+      ctx: MultipartIdentifierContext,
+      commandName: String,
+      allowTempView: Boolean = true): UnresolvedTableOrView = withOrigin(ctx) {
+    UnresolvedTableOrView(visitMultipartIdentifier(ctx), commandName, allowTempView)
+  }
+
+  /**
+   * Create a [[CalendarInterval]] or ANSI interval literal expression.
+   * Two syntaxes are supported:
    * - multiple unit value pairs, for instance: interval 2 months 2 days.
    * - from-to unit, for instance: interval '1-2' year to month.
    */
   override def visitInterval(ctx: IntervalContext): Literal = withOrigin(ctx) {
+    val calendarInterval = parseIntervalLiteral(ctx)
+    if (ctx.errorCapturingUnitToUnitInterval != null && !conf.legacyIntervalEnabled) {
+      // Check the `to` unit to distinguish year-month and day-time intervals because
+      // `CalendarInterval` doesn't have enough info. For instance, new CalendarInterval(0, 0, 0)
+      // can be derived from INTERVAL '0-0' YEAR TO MONTH as well as from
+      // INTERVAL '0 00:00:00' DAY TO SECOND.
+      val toUnit = ctx.errorCapturingUnitToUnitInterval.body.to.getText.toLowerCase(Locale.ROOT)
+      if (toUnit == "month") {
+        assert(calendarInterval.days == 0 && calendarInterval.microseconds == 0)
+        Literal(calendarInterval.months, YearMonthIntervalType)
+      } else {
+        assert(calendarInterval.months == 0)
+        val micros = IntervalUtils.getDuration(calendarInterval, TimeUnit.MICROSECONDS)
+        Literal(micros, DayTimeIntervalType)
+      }
+    } else {
+      Literal(calendarInterval, CalendarIntervalType)
+    }
+  }
+
+  /**
+   * Create a [[CalendarInterval]] object
+   */
+  protected def parseIntervalLiteral(ctx: IntervalContext): CalendarInterval = withOrigin(ctx) {
     if (ctx.errorCapturingMultiUnitsInterval != null) {
       val innerCtx = ctx.errorCapturingMultiUnitsInterval
       if (innerCtx.unitToUnitInterval != null) {
-        throw new ParseException(
-          "Can only have a single from-to unit in the interval literal syntax",
+        throw QueryParsingErrors.moreThanOneFromToUnitInIntervalLiteralError(
           innerCtx.unitToUnitInterval)
       }
-      Literal(visitMultiUnitsInterval(innerCtx.multiUnitsInterval), CalendarIntervalType)
+      visitMultiUnitsInterval(innerCtx.multiUnitsInterval)
     } else if (ctx.errorCapturingUnitToUnitInterval != null) {
       val innerCtx = ctx.errorCapturingUnitToUnitInterval
       if (innerCtx.error1 != null || innerCtx.error2 != null) {
         val errorCtx = if (innerCtx.error1 != null) innerCtx.error1 else innerCtx.error2
-        throw new ParseException(
-          "Can only have a single from-to unit in the interval literal syntax",
-          errorCtx)
+        throw QueryParsingErrors.moreThanOneFromToUnitInIntervalLiteralError(errorCtx)
       }
-      Literal(visitUnitToUnitInterval(innerCtx.body), CalendarIntervalType)
+      visitUnitToUnitInterval(innerCtx.body)
     } else {
-      throw new ParseException("at least one time unit should be given for interval literal", ctx)
+      throw QueryParsingErrors.invalidIntervalLiteralError(ctx)
     }
   }
 
@@ -2115,14 +2369,22 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   override def visitMultiUnitsInterval(ctx: MultiUnitsIntervalContext): CalendarInterval = {
     withOrigin(ctx) {
-      val units = ctx.intervalUnit().asScala
+      val units = ctx.unit.asScala
       val values = ctx.intervalValue().asScala
       try {
         assert(units.length == values.length)
         val kvs = units.indices.map { i =>
           val u = units(i).getText
           val v = if (values(i).STRING() != null) {
-            string(values(i).STRING())
+            val value = string(values(i).STRING())
+            // SPARK-32840: For invalid cases, e.g. INTERVAL '1 day 2' hour,
+            // INTERVAL 'interval 1' day, we need to check ahead before they are concatenated with
+            // units and become valid ones, e.g. '1 day 2 hour'.
+            // Ideally, we only ensure the value parts don't contain any units here.
+            if (value.exists(Character.isLetter)) {
+              throw QueryParsingErrors.invalidIntervalFormError(value, ctx)
+            }
+            value
           } else {
             values(i).getText
           }
@@ -2143,8 +2405,17 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   override def visitUnitToUnitInterval(ctx: UnitToUnitIntervalContext): CalendarInterval = {
     withOrigin(ctx) {
-      val value = Option(ctx.intervalValue.STRING).map(string).getOrElse {
-        throw new ParseException("The value of from-to unit must be a string", ctx.intervalValue)
+      val value = Option(ctx.intervalValue.STRING).map(string).map { interval =>
+        if (ctx.intervalValue().MINUS() == null) {
+          interval
+        } else {
+          interval.startsWith("-") match {
+            case true => interval.replaceFirst("-", "")
+            case false => s"-$interval"
+          }
+        }
+      }.getOrElse {
+        throw QueryParsingErrors.invalidFromToUnitValueError(ctx.intervalValue)
       }
       try {
         val from = ctx.from.getText.toLowerCase(Locale.ROOT)
@@ -2165,7 +2436,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
           case ("minute", "second") =>
             IntervalUtils.fromDayTimeString(value, IntervalUnit.MINUTE, IntervalUnit.SECOND)
           case _ =>
-            throw new ParseException(s"Intervals FROM $from TO $to are not supported.", ctx)
+            throw QueryParsingErrors.fromToIntervalUnsupportedError(from, to, ctx)
         }
       } catch {
         // Handle Exceptions thrown by CalendarInterval
@@ -2180,12 +2451,6 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
   /* ********************************************************************************************
    * DataType parsing
    * ******************************************************************************************** */
-  /**
-   * Create a Spark DataType.
-   */
-  private def visitSparkDataType(ctx: DataTypeContext): DataType = {
-    HiveStringType.replaceCharType(typedVisit(ctx))
-  }
 
   /**
    * Resolve/create a primitive type.
@@ -2211,13 +2476,22 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
         DecimalType(precision.getText.toInt, 0)
       case ("decimal" | "dec" | "numeric", precision :: scale :: Nil) =>
         DecimalType(precision.getText.toInt, scale.getText.toInt)
+      case ("void", Nil) => NullType
       case ("interval", Nil) => CalendarIntervalType
       case (dt, params) =>
         val dtStr = if (params.nonEmpty) s"$dt(${params.mkString(",")})" else dt
-        throw new ParseException(s"DataType $dtStr is not supported.", ctx)
+        throw QueryParsingErrors.dataTypeUnsupportedError(dtStr, ctx)
     }
   }
 
+  override def visitYearMonthIntervalDataType(ctx: YearMonthIntervalDataTypeContext): DataType = {
+    YearMonthIntervalType
+  }
+
+  override def visitDayTimeIntervalDataType(ctx: DayTimeIntervalDataTypeContext): DataType = {
+    DayTimeIntervalType
+  }
+
   /**
    * Create a complex DataType. Arrays, Maps and Structures are supported.
    */
@@ -2243,7 +2517,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    * Create a [[StructType]] from a number of column definitions.
    */
   override def visitColTypeList(ctx: ColTypeListContext): Seq[StructField] = withOrigin(ctx) {
-    ctx.colType().asScala.map(visitColType)
+    ctx.colType().asScala.map(visitColType).toSeq
   }
 
   /**
@@ -2258,16 +2532,9 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       builder.putString("comment", _)
     }
 
-    // Add Hive type string to metadata.
-    val rawDataType = typedVisit[DataType](ctx.dataType)
-    val cleanedDataType = HiveStringType.replaceCharType(rawDataType)
-    if (rawDataType != cleanedDataType) {
-      builder.putString(HIVE_TYPE_STRING, rawDataType.catalogString)
-    }
-
     StructField(
       name = colName.getText,
-      dataType = cleanedDataType,
+      dataType = typedVisit[DataType](ctx.dataType),
       nullable = NULL == null,
       metadata = builder.build())
   }
@@ -2284,7 +2551,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   override def visitComplexColTypeList(
       ctx: ComplexColTypeListContext): Seq[StructField] = withOrigin(ctx) {
-    ctx.complexColType().asScala.map(visitComplexColType)
+    ctx.complexColType().asScala.map(visitComplexColType).toSeq
   }
 
   /**
@@ -2360,7 +2627,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       key -> value
     }
     // Check for duplicate property names.
-    checkDuplicateKeys(properties, ctx)
+    checkDuplicateKeys(properties.toSeq, ctx)
     properties.toMap
   }
 
@@ -2426,10 +2693,22 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
 
   /**
    * Type to keep track of table clauses:
-   * (partitioning, bucketSpec, properties, options, location, comment).
+   * - partition transforms
+   * - partition columns
+   * - bucketSpec
+   * - properties
+   * - options
+   * - location
+   * - comment
+   * - serde
+   *
+   * Note: Partition transforms are based on existing table schema definition. It can be simple
+   * column names, or functions like `year(date_col)`. Partition columns are column names with data
+   * types like `i INT`, which should be appended to the existing table schema.
    */
-  type TableClauses = (Seq[Transform], Option[BucketSpec], Map[String, String],
-    Map[String, String], Option[String], Option[String])
+  type TableClauses = (
+      Seq[Transform], Seq[StructField], Option[BucketSpec], Map[String, String],
+      Map[String, String], Option[String], Option[String], Option[SerdeInfo])
 
   /**
    * Validate a create table statement and return the [[TableIdentifier]].
@@ -2441,7 +2720,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     if (temporary && ifNotExists) {
       operationNotAllowed("CREATE TEMPORARY TABLE ... IF NOT EXISTS", ctx)
     }
-    val multipartIdentifier = ctx.multipartIdentifier.parts.asScala.map(_.getText)
+    val multipartIdentifier = ctx.multipartIdentifier.parts.asScala.map(_.getText).toSeq
     (multipartIdentifier, temporary, ifNotExists, ctx.EXTERNAL != null)
   }
 
@@ -2450,7 +2729,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   override def visitReplaceTableHeader(
       ctx: ReplaceTableHeaderContext): TableHeader = withOrigin(ctx) {
-    val multipartIdentifier = ctx.multipartIdentifier.parts.asScala.map(_.getText)
+    val multipartIdentifier = ctx.multipartIdentifier.parts.asScala.map(_.getText).toSeq
     (multipartIdentifier, false, false, false)
   }
 
@@ -2458,13 +2737,26 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    * Parse a qualified name to a multipart name.
    */
   override def visitQualifiedName(ctx: QualifiedNameContext): Seq[String] = withOrigin(ctx) {
-    ctx.identifier.asScala.map(_.getText)
+    ctx.identifier.asScala.map(_.getText).toSeq
   }
 
   /**
-   * Parse a list of transforms.
+   * Parse a list of transforms or columns.
    */
-  override def visitTransformList(ctx: TransformListContext): Seq[Transform] = withOrigin(ctx) {
+  override def visitPartitionFieldList(
+      ctx: PartitionFieldListContext): (Seq[Transform], Seq[StructField]) = withOrigin(ctx) {
+    val (transforms, columns) = ctx.fields.asScala.map {
+      case transform: PartitionTransformContext =>
+        (Some(visitPartitionTransform(transform)), None)
+      case field: PartitionColumnContext =>
+        (None, Some(visitColType(field.colType)))
+    }.unzip
+
+    (transforms.flatten.toSeq, columns.flatten.toSeq)
+  }
+
+  override def visitPartitionTransform(
+      ctx: PartitionTransformContext): Transform = withOrigin(ctx) {
     def getFieldReference(
         ctx: ApplyTransformContext,
         arg: V2Expression): FieldReference = {
@@ -2473,8 +2765,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
         case ref: FieldReference =>
           ref
         case nonRef =>
-          throw new ParseException(
-            s"Expected a column reference for transform $name: ${nonRef.describe}", ctx)
+          throw QueryParsingErrors.partitionTransformNotExpectedError(name, nonRef.describe, ctx)
       }
     }
 
@@ -2483,20 +2774,20 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
         arguments: Seq[V2Expression]): FieldReference = {
       lazy val name: String = ctx.identifier.getText
       if (arguments.size > 1) {
-        throw new ParseException(s"Too many arguments for transform $name", ctx)
+        throw QueryParsingErrors.tooManyArgumentsForTransformError(name, ctx)
       } else if (arguments.isEmpty) {
-        throw new ParseException(s"Not enough arguments for transform $name", ctx)
+        throw QueryParsingErrors.notEnoughArgumentsForTransformError(name, ctx)
       } else {
         getFieldReference(ctx, arguments.head)
       }
     }
 
-    ctx.transforms.asScala.map {
+    ctx.transform match {
       case identityCtx: IdentityTransformContext =>
         IdentityTransform(FieldReference(typedVisit[Seq[String]](identityCtx.qualifiedName)))
 
       case applyCtx: ApplyTransformContext =>
-        val arguments = applyCtx.argument.asScala.map(visitTransformArgument)
+        val arguments = applyCtx.argument.asScala.map(visitTransformArgument).toSeq
 
         applyCtx.identifier.getText match {
           case "bucket" =>
@@ -2508,7 +2799,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
               case LiteralValue(longValue, LongType) =>
                 longValue.asInstanceOf[Long].toInt
               case lit =>
-                throw new ParseException(s"Invalid number of buckets: ${lit.describe}", applyCtx)
+                throw QueryParsingErrors.invalidBucketsNumberError(lit.describe, applyCtx)
             }
 
             val fields = arguments.tail.map(arg => getFieldReference(applyCtx, arg))
@@ -2546,7 +2837,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
           .map(typedVisit[Literal])
           .map(lit => LiteralValue(lit.value, lit.dataType))
       reference.orElse(literal)
-          .getOrElse(throw new ParseException(s"Invalid transform argument", ctx))
+          .getOrElse(throw QueryParsingErrors.invalidTransformArgumentError(ctx))
     }
   }
 
@@ -2557,12 +2848,12 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     val legacyOn = conf.getConf(SQLConf.LEGACY_PROPERTY_NON_RESERVED)
     properties.filter {
       case (PROP_LOCATION, _) if !legacyOn =>
-        throw new ParseException(s"$PROP_LOCATION is a reserved namespace property, please use" +
-          s" the LOCATION clause to specify it.", ctx)
+        throw QueryParsingErrors.cannotCleanReservedNamespacePropertyError(
+          PROP_LOCATION, ctx, "please use the LOCATION clause to specify it")
       case (PROP_LOCATION, _) => false
       case (PROP_OWNER, _) if !legacyOn =>
-        throw new ParseException(s"$PROP_OWNER is a reserved namespace property, it will be" +
-          s" set to the current user.", ctx)
+        throw QueryParsingErrors.cannotCleanReservedNamespacePropertyError(
+          PROP_OWNER, ctx, "it will be set to the current user")
       case (PROP_OWNER, _) => false
       case _ => true
     }
@@ -2590,7 +2881,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     checkDuplicateClauses(ctx.DBPROPERTIES, "WITH DBPROPERTIES", ctx)
 
     if (!ctx.PROPERTIES.isEmpty && !ctx.DBPROPERTIES.isEmpty) {
-      throw new ParseException(s"Either PROPERTIES or DBPROPERTIES is allowed.", ctx)
+      throw QueryParsingErrors.propertiesAndDbPropertiesBothSpecifiedError(ctx)
     }
 
     var properties = ctx.tablePropertyList.asScala.headOption
@@ -2629,7 +2920,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
   }
 
   /**
-   * Create an [[AlterNamespaceSetProperties]] logical plan.
+   * Create an [[SetNamespaceProperties]] logical plan.
    *
    * For example:
    * {{{
@@ -2640,14 +2931,14 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
   override def visitSetNamespaceProperties(ctx: SetNamespacePropertiesContext): LogicalPlan = {
     withOrigin(ctx) {
       val properties = cleanNamespaceProperties(visitPropertyKeyValues(ctx.tablePropertyList), ctx)
-      AlterNamespaceSetProperties(
+      SetNamespaceProperties(
         UnresolvedNamespace(visitMultipartIdentifier(ctx.multipartIdentifier)),
         properties)
     }
   }
 
   /**
-   * Create an [[AlterNamespaceSetLocation]] logical plan.
+   * Create an [[SetNamespaceLocation]] logical plan.
    *
    * For example:
    * {{{
@@ -2656,7 +2947,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   override def visitSetNamespaceLocation(ctx: SetNamespaceLocationContext): LogicalPlan = {
     withOrigin(ctx) {
-      AlterNamespaceSetLocation(
+      SetNamespaceLocation(
         UnresolvedNamespace(visitMultipartIdentifier(ctx.multipartIdentifier)),
         visitLocationSpec(ctx.locationSpec))
     }
@@ -2667,7 +2958,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   override def visitShowNamespaces(ctx: ShowNamespacesContext): LogicalPlan = withOrigin(ctx) {
     if (ctx.DATABASES != null && ctx.multipartIdentifier != null) {
-      throw new ParseException(s"FROM/IN operator is not allowed in SHOW DATABASES", ctx)
+      throw QueryParsingErrors.fromOrInNotAllowedInShowDatabasesError(ctx)
     }
 
     val multiPart = Option(ctx.multipartIdentifier).map(visitMultipartIdentifier)
@@ -2697,16 +2988,16 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     val legacyOn = conf.getConf(SQLConf.LEGACY_PROPERTY_NON_RESERVED)
     properties.filter {
       case (PROP_PROVIDER, _) if !legacyOn =>
-        throw new ParseException(s"$PROP_PROVIDER is a reserved table property, please use" +
-          s" the USING clause to specify it.", ctx)
+        throw QueryParsingErrors.cannotCleanReservedTablePropertyError(
+          PROP_PROVIDER, ctx, "please use the USING clause to specify it")
       case (PROP_PROVIDER, _) => false
       case (PROP_LOCATION, _) if !legacyOn =>
-        throw new ParseException(s"$PROP_LOCATION is a reserved table property, please use" +
-          s" the LOCATION clause to specify it.", ctx)
+        throw QueryParsingErrors.cannotCleanReservedTablePropertyError(
+          PROP_LOCATION, ctx, "please use the LOCATION clause to specify it")
       case (PROP_LOCATION, _) => false
       case (PROP_OWNER, _) if !legacyOn =>
-        throw new ParseException(s"$PROP_OWNER is a reserved table property, it will be" +
-          s" set to the current user", ctx)
+        throw QueryParsingErrors.cannotCleanReservedTablePropertyError(
+          PROP_OWNER, ctx, "it will be set to the current user")
       case (PROP_OWNER, _) => false
       case _ => true
     }
@@ -2719,9 +3010,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     var path = location
     val filtered = cleanTableProperties(ctx, options).filter {
       case (k, v) if k.equalsIgnoreCase("path") && path.nonEmpty =>
-        throw new ParseException(s"Duplicated table paths found: '${path.get}' and '$v'. LOCATION" +
-          s" and the case insensitive key 'path' in OPTIONS are all used to indicate the custom" +
-          s" table path, you can only specify one of them.", ctx)
+        throw QueryParsingErrors.duplicatedTablePathsFoundError(path.get, v, ctx)
       case (k, v) if k.equalsIgnoreCase("path") =>
         path = Some(v)
         false
@@ -2730,16 +3019,157 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     (filtered, path)
   }
 
+  /**
+   * Create a [[SerdeInfo]] for creating tables.
+   *
+   * Format: STORED AS (name | INPUTFORMAT input_format OUTPUTFORMAT output_format)
+   */
+  override def visitCreateFileFormat(ctx: CreateFileFormatContext): SerdeInfo = withOrigin(ctx) {
+    (ctx.fileFormat, ctx.storageHandler) match {
+      // Expected format: INPUTFORMAT input_format OUTPUTFORMAT output_format
+      case (c: TableFileFormatContext, null) =>
+        SerdeInfo(formatClasses = Some(FormatClasses(string(c.inFmt), string(c.outFmt))))
+      // Expected format: SEQUENCEFILE | TEXTFILE | RCFILE | ORC | PARQUET | AVRO
+      case (c: GenericFileFormatContext, null) =>
+        SerdeInfo(storedAs = Some(c.identifier.getText))
+      case (null, storageHandler) =>
+        operationNotAllowed("STORED BY", ctx)
+      case _ =>
+        throw QueryParsingErrors.storedAsAndStoredByBothSpecifiedError(ctx)
+    }
+  }
+
+  /**
+   * Create a [[SerdeInfo]] used for creating tables.
+   *
+   * Example format:
+   * {{{
+   *   SERDE serde_name [WITH SERDEPROPERTIES (k1=v1, k2=v2, ...)]
+   * }}}
+   *
+   * OR
+   *
+   * {{{
+   *   DELIMITED [FIELDS TERMINATED BY char [ESCAPED BY char]]
+   *   [COLLECTION ITEMS TERMINATED BY char]
+   *   [MAP KEYS TERMINATED BY char]
+   *   [LINES TERMINATED BY char]
+   *   [NULL DEFINED AS char]
+   * }}}
+   */
+  def visitRowFormat(ctx: RowFormatContext): SerdeInfo = withOrigin(ctx) {
+    ctx match {
+      case serde: RowFormatSerdeContext => visitRowFormatSerde(serde)
+      case delimited: RowFormatDelimitedContext => visitRowFormatDelimited(delimited)
+    }
+  }
+
+  /**
+   * Create SERDE row format name and properties pair.
+   */
+  override def visitRowFormatSerde(ctx: RowFormatSerdeContext): SerdeInfo = withOrigin(ctx) {
+    import ctx._
+    SerdeInfo(
+      serde = Some(string(name)),
+      serdeProperties = Option(tablePropertyList).map(visitPropertyKeyValues).getOrElse(Map.empty))
+  }
+
+  /**
+   * Create a delimited row format properties object.
+   */
+  override def visitRowFormatDelimited(
+      ctx: RowFormatDelimitedContext): SerdeInfo = withOrigin(ctx) {
+    // Collect the entries if any.
+    def entry(key: String, value: Token): Seq[(String, String)] = {
+      Option(value).toSeq.map(x => key -> string(x))
+    }
+    // TODO we need proper support for the NULL format.
+    val entries =
+      entry("field.delim", ctx.fieldsTerminatedBy) ++
+          entry("serialization.format", ctx.fieldsTerminatedBy) ++
+          entry("escape.delim", ctx.escapedBy) ++
+          // The following typo is inherited from Hive...
+          entry("colelction.delim", ctx.collectionItemsTerminatedBy) ++
+          entry("mapkey.delim", ctx.keysTerminatedBy) ++
+          Option(ctx.linesSeparatedBy).toSeq.map { token =>
+            val value = string(token)
+            validate(
+              value == "\n",
+              s"LINES TERMINATED BY only supports newline '\\n' right now: $value",
+              ctx)
+            "line.delim" -> value
+          }
+    SerdeInfo(serdeProperties = entries.toMap)
+  }
+
+  /**
+   * Throw a [[ParseException]] if the user specified incompatible SerDes through ROW FORMAT
+   * and STORED AS.
+   *
+   * The following are allowed. Anything else is not:
+   *   ROW FORMAT SERDE ... STORED AS [SEQUENCEFILE | RCFILE | TEXTFILE]
+   *   ROW FORMAT DELIMITED ... STORED AS TEXTFILE
+   *   ROW FORMAT ... STORED AS INPUTFORMAT ... OUTPUTFORMAT ...
+   */
+  protected def validateRowFormatFileFormat(
+      rowFormatCtx: RowFormatContext,
+      createFileFormatCtx: CreateFileFormatContext,
+      parentCtx: ParserRuleContext): Unit = {
+    if (rowFormatCtx == null || createFileFormatCtx == null) {
+      return
+    }
+    (rowFormatCtx, createFileFormatCtx.fileFormat) match {
+      case (_, ffTable: TableFileFormatContext) => // OK
+      case (rfSerde: RowFormatSerdeContext, ffGeneric: GenericFileFormatContext) =>
+        ffGeneric.identifier.getText.toLowerCase(Locale.ROOT) match {
+          case ("sequencefile" | "textfile" | "rcfile") => // OK
+          case fmt =>
+            operationNotAllowed(
+              s"ROW FORMAT SERDE is incompatible with format '$fmt', which also specifies a serde",
+              parentCtx)
+        }
+      case (rfDelimited: RowFormatDelimitedContext, ffGeneric: GenericFileFormatContext) =>
+        ffGeneric.identifier.getText.toLowerCase(Locale.ROOT) match {
+          case "textfile" => // OK
+          case fmt => operationNotAllowed(
+            s"ROW FORMAT DELIMITED is only compatible with 'textfile', not '$fmt'", parentCtx)
+        }
+      case _ =>
+        // should never happen
+        def str(ctx: ParserRuleContext): String = {
+          (0 until ctx.getChildCount).map { i => ctx.getChild(i).getText }.mkString(" ")
+        }
+        operationNotAllowed(
+          s"Unexpected combination of ${str(rowFormatCtx)} and ${str(createFileFormatCtx)}",
+          parentCtx)
+    }
+  }
+
+  protected def validateRowFormatFileFormat(
+      rowFormatCtx: Seq[RowFormatContext],
+      createFileFormatCtx: Seq[CreateFileFormatContext],
+      parentCtx: ParserRuleContext): Unit = {
+    if (rowFormatCtx.size == 1 && createFileFormatCtx.size == 1) {
+      validateRowFormatFileFormat(rowFormatCtx.head, createFileFormatCtx.head, parentCtx)
+    }
+  }
+
   override def visitCreateTableClauses(ctx: CreateTableClausesContext): TableClauses = {
     checkDuplicateClauses(ctx.TBLPROPERTIES, "TBLPROPERTIES", ctx)
     checkDuplicateClauses(ctx.OPTIONS, "OPTIONS", ctx)
     checkDuplicateClauses(ctx.PARTITIONED, "PARTITIONED BY", ctx)
+    checkDuplicateClauses(ctx.createFileFormat, "STORED AS/BY", ctx)
+    checkDuplicateClauses(ctx.rowFormat, "ROW FORMAT", ctx)
     checkDuplicateClauses(ctx.commentSpec(), "COMMENT", ctx)
     checkDuplicateClauses(ctx.bucketSpec(), "CLUSTERED BY", ctx)
     checkDuplicateClauses(ctx.locationSpec, "LOCATION", ctx)
 
-    val partitioning: Seq[Transform] =
-      Option(ctx.partitioning).map(visitTransformList).getOrElse(Nil)
+    if (ctx.skewSpec.size > 0) {
+      operationNotAllowed("CREATE TABLE ... SKEWED BY", ctx)
+    }
+
+    val (partTransforms, partCols) =
+      Option(ctx.partitioning).map(visitPartitionFieldList).getOrElse((Nil, Nil))
     val bucketSpec = ctx.bucketSpec().asScala.headOption.map(visitBucketSpec)
     val properties = Option(ctx.tableProps).map(visitPropertyKeyValues).getOrElse(Map.empty)
     val cleanedProperties = cleanTableProperties(ctx, properties)
@@ -2747,7 +3177,45 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     val location = visitLocationSpecList(ctx.locationSpec())
     val (cleanedOptions, newLocation) = cleanTableOptions(ctx, options, location)
     val comment = visitCommentSpecList(ctx.commentSpec())
-    (partitioning, bucketSpec, cleanedProperties, cleanedOptions, newLocation, comment)
+    val serdeInfo =
+      getSerdeInfo(ctx.rowFormat.asScala.toSeq, ctx.createFileFormat.asScala.toSeq, ctx)
+    (partTransforms, partCols, bucketSpec, cleanedProperties, cleanedOptions, newLocation, comment,
+      serdeInfo)
+  }
+
+  protected def getSerdeInfo(
+      rowFormatCtx: Seq[RowFormatContext],
+      createFileFormatCtx: Seq[CreateFileFormatContext],
+      ctx: ParserRuleContext): Option[SerdeInfo] = {
+    validateRowFormatFileFormat(rowFormatCtx, createFileFormatCtx, ctx)
+    val rowFormatSerdeInfo = rowFormatCtx.map(visitRowFormat)
+    val fileFormatSerdeInfo = createFileFormatCtx.map(visitCreateFileFormat)
+    (fileFormatSerdeInfo ++ rowFormatSerdeInfo).reduceLeftOption((l, r) => l.merge(r))
+  }
+
+  private def partitionExpressions(
+      partTransforms: Seq[Transform],
+      partCols: Seq[StructField],
+      ctx: ParserRuleContext): Seq[Transform] = {
+    if (partTransforms.nonEmpty) {
+      if (partCols.nonEmpty) {
+        val references = partTransforms.map(_.describe()).mkString(", ")
+        val columns = partCols
+          .map(field => s"${field.name} ${field.dataType.simpleString}")
+          .mkString(", ")
+        operationNotAllowed(
+          s"""PARTITION BY: Cannot mix partition expressions and partition columns:
+             |Expressions: $references
+             |Columns: $columns""".stripMargin, ctx)
+
+      }
+      partTransforms
+    } else {
+      // columns were added to create the schema. convert to column references
+      partCols.map { column =>
+        IdentityTransform(FieldReference(Seq(column.name)))
+      }
+    }
   }
 
   /**
@@ -2756,13 +3224,15 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    * Expected format:
    * {{{
    *   CREATE [TEMPORARY] TABLE [IF NOT EXISTS] [db_name.]table_name
-   *   USING table_provider
+   *   [USING table_provider]
    *   create_table_clauses
    *   [[AS] select_statement];
    *
    *   create_table_clauses (order insensitive):
+   *     [PARTITIONED BY (partition_fields)]
    *     [OPTIONS table_property_list]
-   *     [PARTITIONED BY (col_name, transform(col_name), transform(constant, col_name), ...)]
+   *     [ROW FORMAT row_format]
+   *     [STORED AS file_format]
    *     [CLUSTERED BY (col_name, col_name, ...)
    *       [SORTED BY (col_name [ASC|DESC], ...)]
    *       INTO num_buckets BUCKETS
@@ -2770,40 +3240,55 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    *     [LOCATION path]
    *     [COMMENT table_comment]
    *     [TBLPROPERTIES (property_name=property_value, ...)]
+   *
+   *   partition_fields:
+   *     col_name, transform(col_name), transform(constant, col_name), ... |
+   *     col_name data_type [NOT NULL] [COMMENT col_comment], ...
    * }}}
    */
   override def visitCreateTable(ctx: CreateTableContext): LogicalPlan = withOrigin(ctx) {
     val (table, temp, ifNotExists, external) = visitCreateTableHeader(ctx.createTableHeader)
-    if (external) {
-      operationNotAllowed("CREATE EXTERNAL TABLE ...", ctx)
-    }
-    val schema = Option(ctx.colTypeList()).map(createSchema)
+
+    val columns = Option(ctx.colTypeList()).map(visitColTypeList).getOrElse(Nil)
     val provider = Option(ctx.tableProvider).map(_.multipartIdentifier.getText)
-    val (partitioning, bucketSpec, properties, options, location, comment) =
+    val (partTransforms, partCols, bucketSpec, properties, options, location, comment, serdeInfo) =
       visitCreateTableClauses(ctx.createTableClauses())
 
-    Option(ctx.query).map(plan) match {
-      case Some(_) if temp =>
-        operationNotAllowed("CREATE TEMPORARY TABLE ... USING ... AS query", ctx)
+    if (provider.isDefined && serdeInfo.isDefined) {
+      operationNotAllowed(s"CREATE TABLE ... USING ... ${serdeInfo.get.describe}", ctx)
+    }
+
+    if (temp) {
+      val asSelect = if (ctx.query == null) "" else " AS ..."
+      operationNotAllowed(
+        s"CREATE TEMPORARY TABLE ...$asSelect, use CREATE TEMPORARY VIEW instead", ctx)
+    }
+
+    val partitioning = partitionExpressions(partTransforms, partCols, ctx)
 
-      case Some(_) if schema.isDefined =>
+    Option(ctx.query).map(plan) match {
+      case Some(_) if columns.nonEmpty =>
         operationNotAllowed(
           "Schema may not be specified in a Create Table As Select (CTAS) statement",
           ctx)
 
+      case Some(_) if partCols.nonEmpty =>
+        // non-reference partition columns are not allowed because schema can't be specified
+        operationNotAllowed(
+          "Partition column types may not be specified in Create Table As Select (CTAS)",
+          ctx)
+
       case Some(query) =>
         CreateTableAsSelectStatement(
           table, query, partitioning, bucketSpec, properties, provider, options, location, comment,
-          writeOptions = Map.empty, ifNotExists = ifNotExists)
-
-      case None if temp =>
-        // CREATE TEMPORARY TABLE ... USING ... is not supported by the catalyst parser.
-        // Use CREATE TEMPORARY VIEW ... USING ... instead.
-        operationNotAllowed("CREATE TEMPORARY TABLE IF NOT EXISTS", ctx)
+          writeOptions = Map.empty, serdeInfo, external = external, ifNotExists = ifNotExists)
 
       case _ =>
-        CreateTableStatement(table, schema.getOrElse(new StructType), partitioning, bucketSpec,
-          properties, provider, options, location, comment, ifNotExists = ifNotExists)
+        // Note: table schema includes both the table columns list and the partition columns
+        // with data type.
+        val schema = StructType(columns ++ partCols)
+        CreateTableStatement(table, schema, partitioning, bucketSpec, properties, provider,
+          options, location, comment, serdeInfo, external = external, ifNotExists = ifNotExists)
     }
   }
 
@@ -2813,13 +3298,13 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    * Expected format:
    * {{{
    *   [CREATE OR] REPLACE TABLE [db_name.]table_name
-   *   USING table_provider
+   *   [USING table_provider]
    *   replace_table_clauses
    *   [[AS] select_statement];
    *
    *   replace_table_clauses (order insensitive):
    *     [OPTIONS table_property_list]
-   *     [PARTITIONED BY (col_name, transform(col_name), transform(constant, col_name), ...)]
+   *     [PARTITIONED BY (partition_fields)]
    *     [CLUSTERED BY (col_name, col_name, ...)
    *       [SORTED BY (col_name [ASC|DESC], ...)]
    *       INTO num_buckets BUCKETS
@@ -2827,52 +3312,87 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    *     [LOCATION path]
    *     [COMMENT table_comment]
    *     [TBLPROPERTIES (property_name=property_value, ...)]
+   *
+   *   partition_fields:
+   *     col_name, transform(col_name), transform(constant, col_name), ... |
+   *     col_name data_type [NOT NULL] [COMMENT col_comment], ...
    * }}}
    */
   override def visitReplaceTable(ctx: ReplaceTableContext): LogicalPlan = withOrigin(ctx) {
-    val (table, _, ifNotExists, external) = visitReplaceTableHeader(ctx.replaceTableHeader)
+    val (table, temp, ifNotExists, external) = visitReplaceTableHeader(ctx.replaceTableHeader)
+    val orCreate = ctx.replaceTableHeader().CREATE() != null
+
+    if (temp) {
+      val action = if (orCreate) "CREATE OR REPLACE" else "REPLACE"
+      operationNotAllowed(s"$action TEMPORARY TABLE ..., use $action TEMPORARY VIEW instead.", ctx)
+    }
+
     if (external) {
-      operationNotAllowed("REPLACE EXTERNAL TABLE ... USING", ctx)
+      operationNotAllowed("REPLACE EXTERNAL TABLE ...", ctx)
+    }
+
+    if (ifNotExists) {
+      operationNotAllowed("REPLACE ... IF NOT EXISTS, use CREATE IF NOT EXISTS instead", ctx)
     }
 
-    val (partitioning, bucketSpec, properties, options, location, comment) =
+    val (partTransforms, partCols, bucketSpec, properties, options, location, comment, serdeInfo) =
       visitCreateTableClauses(ctx.createTableClauses())
-    val schema = Option(ctx.colTypeList()).map(createSchema)
+    val columns = Option(ctx.colTypeList()).map(visitColTypeList).getOrElse(Nil)
     val provider = Option(ctx.tableProvider).map(_.multipartIdentifier.getText)
-    val orCreate = ctx.replaceTableHeader().CREATE() != null
+
+    if (provider.isDefined && serdeInfo.isDefined) {
+      operationNotAllowed(s"CREATE TABLE ... USING ... ${serdeInfo.get.describe}", ctx)
+    }
+
+    val partitioning = partitionExpressions(partTransforms, partCols, ctx)
 
     Option(ctx.query).map(plan) match {
-      case Some(_) if schema.isDefined =>
+      case Some(_) if columns.nonEmpty =>
         operationNotAllowed(
           "Schema may not be specified in a Replace Table As Select (RTAS) statement",
           ctx)
 
+      case Some(_) if partCols.nonEmpty =>
+        // non-reference partition columns are not allowed because schema can't be specified
+        operationNotAllowed(
+          "Partition column types may not be specified in Replace Table As Select (RTAS)",
+          ctx)
+
       case Some(query) =>
         ReplaceTableAsSelectStatement(table, query, partitioning, bucketSpec, properties,
-          provider, options, location, comment, writeOptions = Map.empty, orCreate = orCreate)
+          provider, options, location, comment, writeOptions = Map.empty, serdeInfo,
+          orCreate = orCreate)
 
       case _ =>
-        ReplaceTableStatement(table, schema.getOrElse(new StructType), partitioning,
-          bucketSpec, properties, provider, options, location, comment, orCreate = orCreate)
+        // Note: table schema includes both the table columns list and the partition columns
+        // with data type.
+        val schema = StructType(columns ++ partCols)
+        ReplaceTableStatement(table, schema, partitioning, bucketSpec, properties, provider,
+          options, location, comment, serdeInfo, orCreate = orCreate)
     }
   }
 
   /**
-   * Create a [[DropTableStatement]] command.
+   * Create a [[DropTable]] command.
    */
   override def visitDropTable(ctx: DropTableContext): LogicalPlan = withOrigin(ctx) {
-    DropTableStatement(
-      visitMultipartIdentifier(ctx.multipartIdentifier()),
+    // DROP TABLE works with either a table or a temporary view.
+    DropTable(
+      createUnresolvedTableOrView(ctx.multipartIdentifier(), "DROP TABLE"),
       ctx.EXISTS != null,
       ctx.PURGE != null)
   }
 
   /**
-   * Create a [[DropViewStatement]] command.
+   * Create a [[DropView]] command.
    */
   override def visitDropView(ctx: DropViewContext): AnyRef = withOrigin(ctx) {
-    DropViewStatement(
-      visitMultipartIdentifier(ctx.multipartIdentifier()),
+    DropView(
+      createUnresolvedView(
+        ctx.multipartIdentifier(),
+        commandName = "DROP VIEW",
+        allowTemp = true,
+        relationTypeMismatchHint = Some("Please use DROP TABLE instead.")),
       ctx.EXISTS != null)
   }
 
@@ -2903,13 +3423,18 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
   }
 
   /**
-   * Create a [[ShowTableStatement]] command.
+   * Create a [[ShowTableExtended]] command.
    */
-  override def visitShowTable(ctx: ShowTableContext): LogicalPlan = withOrigin(ctx) {
-    ShowTableStatement(
-      Option(ctx.ns).map(visitMultipartIdentifier),
+  override def visitShowTableExtended(
+      ctx: ShowTableExtendedContext): LogicalPlan = withOrigin(ctx) {
+    val multiPart = Option(ctx.multipartIdentifier).map(visitMultipartIdentifier)
+    val partitionKeys = Option(ctx.partitionSpec).map { specCtx =>
+      UnresolvedPartitionSpec(visitNonOptionalPartitionSpec(specCtx), None)
+    }
+    ShowTableExtended(
+      UnresolvedNamespace(multiPart.getOrElse(Seq.empty[String])),
       string(ctx.pattern),
-      Option(ctx.partitionSpec).map(visitNonOptionalPartitionSpec))
+      partitionKeys)
   }
 
   /**
@@ -2954,7 +3479,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
   override def visitAddTableColumns(ctx: AddTableColumnsContext): LogicalPlan = withOrigin(ctx) {
     AlterTableAddColumnsStatement(
       visitMultipartIdentifier(ctx.multipartIdentifier),
-      ctx.columns.qualifiedColTypeWithPosition.asScala.map(typedVisit[QualifiedColType])
+      ctx.columns.qualifiedColTypeWithPosition.asScala.map(typedVisit[QualifiedColType]).toSeq
     )
   }
 
@@ -2970,7 +3495,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       ctx: RenameTableColumnContext): LogicalPlan = withOrigin(ctx) {
     AlterTableRenameColumnStatement(
       visitMultipartIdentifier(ctx.table),
-      ctx.from.parts.asScala.map(_.getText),
+      ctx.from.parts.asScala.map(_.getText).toSeq,
       ctx.to.getText)
   }
 
@@ -3048,12 +3573,13 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     }
     val columnNameParts = typedVisit[Seq[String]](ctx.colName)
     if (!conf.resolver(columnNameParts.last, ctx.colType().colName.getText)) {
-      throw new AnalysisException("Renaming column is not supported in Hive-style ALTER COLUMN, " +
-        "please run RENAME COLUMN instead.")
+      throw QueryParsingErrors.operationInHiveStyleCommandUnsupportedError("Renaming column",
+        "ALTER COLUMN", ctx, Some("please run RENAME COLUMN instead"))
     }
     if (ctx.colType.NULL != null) {
-      throw new AnalysisException("NOT NULL is not supported in Hive-style ALTER COLUMN, " +
-        "please run ALTER COLUMN ... SET/DROP NOT NULL instead.")
+      throw QueryParsingErrors.operationInHiveStyleCommandUnsupportedError(
+        "NOT NULL", "ALTER COLUMN", ctx,
+        Some("please run ALTER COLUMN ... SET/DROP NOT NULL instead"))
     }
 
     AlterTableAlterColumnStatement(
@@ -3074,15 +3600,15 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       visitMultipartIdentifier(ctx.multipartIdentifier),
       ctx.columns.qualifiedColTypeWithPosition.asScala.map { colType =>
         if (colType.NULL != null) {
-          throw new AnalysisException(
-            "NOT NULL is not supported in Hive-style REPLACE COLUMNS")
+          throw QueryParsingErrors.operationInHiveStyleCommandUnsupportedError(
+            "NOT NULL", "REPLACE COLUMNS", ctx)
         }
         if (colType.colPosition != null) {
-          throw new AnalysisException(
-            "Column position is not supported in Hive-style REPLACE COLUMNS")
+          throw QueryParsingErrors.operationInHiveStyleCommandUnsupportedError(
+            "Column position", "REPLACE COLUMNS", ctx)
         }
         typedVisit[QualifiedColType](colType)
-      }
+      }.toSeq
     )
   }
 
@@ -3100,11 +3626,11 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     val columnsToDrop = ctx.columns.multipartIdentifier.asScala.map(typedVisit[Seq[String]])
     AlterTableDropColumnsStatement(
       visitMultipartIdentifier(ctx.multipartIdentifier),
-      columnsToDrop)
+      columnsToDrop.toSeq)
   }
 
   /**
-   * Parse [[AlterViewSetPropertiesStatement]] or [[AlterTableSetPropertiesStatement]] commands.
+   * Parse [[SetViewProperties]] or [[SetTableProperties]] commands.
    *
    * For example:
    * {{{
@@ -3114,18 +3640,28 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   override def visitSetTableProperties(
       ctx: SetTablePropertiesContext): LogicalPlan = withOrigin(ctx) {
-    val identifier = visitMultipartIdentifier(ctx.multipartIdentifier)
     val properties = visitPropertyKeyValues(ctx.tablePropertyList)
     val cleanedTableProperties = cleanTableProperties(ctx, properties)
     if (ctx.VIEW != null) {
-      AlterViewSetPropertiesStatement(identifier, cleanedTableProperties)
+      SetViewProperties(
+        createUnresolvedView(
+          ctx.multipartIdentifier,
+          commandName = "ALTER VIEW ... SET TBLPROPERTIES",
+          allowTemp = false,
+          relationTypeMismatchHint = alterViewTypeMismatchHint),
+        cleanedTableProperties)
     } else {
-      AlterTableSetPropertiesStatement(identifier, cleanedTableProperties)
+      SetTableProperties(
+        createUnresolvedTable(
+          ctx.multipartIdentifier,
+          "ALTER TABLE ... SET TBLPROPERTIES",
+          alterTableTypeMismatchHint),
+        cleanedTableProperties)
     }
   }
 
   /**
-   * Parse [[AlterViewUnsetPropertiesStatement]] or [[AlterTableUnsetPropertiesStatement]] commands.
+   * Parse [[UnsetViewProperties]] or [[UnsetTableProperties]] commands.
    *
    * For example:
    * {{{
@@ -3135,45 +3671,63 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   override def visitUnsetTableProperties(
       ctx: UnsetTablePropertiesContext): LogicalPlan = withOrigin(ctx) {
-    val identifier = visitMultipartIdentifier(ctx.multipartIdentifier)
     val properties = visitPropertyKeys(ctx.tablePropertyList)
     val cleanedProperties = cleanTableProperties(ctx, properties.map(_ -> "").toMap).keys.toSeq
 
     val ifExists = ctx.EXISTS != null
     if (ctx.VIEW != null) {
-      AlterViewUnsetPropertiesStatement(identifier, cleanedProperties, ifExists)
+      UnsetViewProperties(
+        createUnresolvedView(
+          ctx.multipartIdentifier,
+          commandName = "ALTER VIEW ... UNSET TBLPROPERTIES",
+          allowTemp = false,
+          relationTypeMismatchHint = alterViewTypeMismatchHint),
+        cleanedProperties,
+        ifExists)
     } else {
-      AlterTableUnsetPropertiesStatement(identifier, cleanedProperties, ifExists)
+      UnsetTableProperties(
+        createUnresolvedTable(
+          ctx.multipartIdentifier,
+          "ALTER TABLE ... UNSET TBLPROPERTIES",
+          alterTableTypeMismatchHint),
+        cleanedProperties,
+        ifExists)
     }
   }
 
   /**
-   * Create an [[AlterTableSetLocationStatement]] command.
+   * Create an [[SetTableLocation]] command.
    *
    * For example:
    * {{{
-   *   ALTER TABLE table SET LOCATION "loc";
+   *   ALTER TABLE table_name [PARTITION partition_spec] SET LOCATION "loc";
    * }}}
    */
   override def visitSetTableLocation(ctx: SetTableLocationContext): LogicalPlan = withOrigin(ctx) {
-    AlterTableSetLocationStatement(
-      visitMultipartIdentifier(ctx.multipartIdentifier),
+    SetTableLocation(
+      createUnresolvedTable(
+        ctx.multipartIdentifier,
+        "ALTER TABLE ... SET LOCATION ...",
+        alterTableTypeMismatchHint),
       Option(ctx.partitionSpec).map(visitNonOptionalPartitionSpec),
       visitLocationSpec(ctx.locationSpec))
   }
 
   /**
-   * Create a [[DescribeColumnStatement]] or [[DescribeRelation]] commands.
+   * Create a [[DescribeColumn]] or [[DescribeRelation]] commands.
    */
   override def visitDescribeRelation(ctx: DescribeRelationContext): LogicalPlan = withOrigin(ctx) {
     val isExtended = ctx.EXTENDED != null || ctx.FORMATTED != null
+    val relation = createUnresolvedTableOrView(
+      ctx.multipartIdentifier(),
+      "DESCRIBE TABLE")
     if (ctx.describeColName != null) {
       if (ctx.partitionSpec != null) {
-        throw new ParseException("DESC TABLE COLUMN for a specific partition is not supported", ctx)
+        throw QueryParsingErrors.descColumnForPartitionUnsupportedError(ctx)
       } else {
-        DescribeColumnStatement(
-          visitMultipartIdentifier(ctx.multipartIdentifier()),
-          ctx.describeColName.nameParts.asScala.map(_.getText),
+        DescribeColumn(
+          relation,
+          UnresolvedAttribute(ctx.describeColName.nameParts.asScala.map(_.getText).toSeq),
           isExtended)
       }
     } else {
@@ -3182,20 +3736,17 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
         visitPartitionSpec(ctx.partitionSpec).map {
           case (key, Some(value)) => key -> value
           case (key, _) =>
-            throw new ParseException(s"PARTITION specification is incomplete: `$key`", ctx)
+            throw QueryParsingErrors.incompletePartitionSpecificationError(key, ctx)
         }
       } else {
         Map.empty[String, String]
       }
-      DescribeRelation(
-        UnresolvedTableOrView(visitMultipartIdentifier(ctx.multipartIdentifier())),
-        partitionSpec,
-        isExtended)
+      DescribeRelation(relation, partitionSpec, isExtended)
     }
   }
 
   /**
-   * Create an [[AnalyzeTableStatement]], or an [[AnalyzeColumnStatement]].
+   * Create an [[AnalyzeTable]], or an [[AnalyzeColumn]].
    * Example SQL for analyzing a table or a set of partitions :
    * {{{
    *   ANALYZE TABLE multi_part_name [PARTITION (partcol1[=val1], partcol2[=val2], ...)]
@@ -3221,42 +3772,87 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     }
     if (ctx.identifier != null &&
         ctx.identifier.getText.toLowerCase(Locale.ROOT) != "noscan") {
-      throw new ParseException(s"Expected `NOSCAN` instead of `${ctx.identifier.getText}`",
-        ctx.identifier())
+      throw QueryParsingErrors.computeStatisticsNotExpectedError(ctx.identifier())
     }
 
-    val tableName = visitMultipartIdentifier(ctx.multipartIdentifier())
     if (ctx.ALL() != null) {
       checkPartitionSpec()
-      AnalyzeColumnStatement(tableName, None, allColumns = true)
+      AnalyzeColumn(
+        createUnresolvedTableOrView(
+          ctx.multipartIdentifier(),
+          "ANALYZE TABLE ... FOR ALL COLUMNS"),
+        None,
+        allColumns = true)
     } else if (ctx.identifierSeq() == null) {
       val partitionSpec = if (ctx.partitionSpec != null) {
         visitPartitionSpec(ctx.partitionSpec)
       } else {
         Map.empty[String, Option[String]]
       }
-      AnalyzeTableStatement(tableName, partitionSpec, noScan = ctx.identifier != null)
+      AnalyzeTable(
+        createUnresolvedTableOrView(
+          ctx.multipartIdentifier(),
+          "ANALYZE TABLE",
+          allowTempView = false),
+        partitionSpec,
+        noScan = ctx.identifier != null)
     } else {
       checkPartitionSpec()
-      AnalyzeColumnStatement(
-        tableName, Option(visitIdentifierSeq(ctx.identifierSeq())), allColumns = false)
+      AnalyzeColumn(
+        createUnresolvedTableOrView(
+          ctx.multipartIdentifier(),
+          "ANALYZE TABLE ... FOR COLUMNS ..."),
+        Option(visitIdentifierSeq(ctx.identifierSeq())),
+        allColumns = false)
     }
   }
 
   /**
-   * Create a [[RepairTableStatement]].
+   * Create an [[AnalyzeTables]].
+   * Example SQL for analyzing all tables in default database:
+   * {{{
+   *   ANALYZE TABLES IN default COMPUTE STATISTICS;
+   * }}}
+   */
+  override def visitAnalyzeTables(ctx: AnalyzeTablesContext): LogicalPlan = withOrigin(ctx) {
+    if (ctx.identifier != null &&
+      ctx.identifier.getText.toLowerCase(Locale.ROOT) != "noscan") {
+      throw new ParseException(s"Expected `NOSCAN` instead of `${ctx.identifier.getText}`",
+        ctx.identifier())
+    }
+    val multiPart = Option(ctx.multipartIdentifier).map(visitMultipartIdentifier)
+    AnalyzeTables(
+      UnresolvedNamespace(multiPart.getOrElse(Seq.empty[String])),
+      noScan = ctx.identifier != null)
+  }
+
+  /**
+   * Create a [[RepairTable]].
    *
    * For example:
    * {{{
-   *   MSCK REPAIR TABLE multi_part_name
+   *   MSCK REPAIR TABLE multi_part_name [{ADD|DROP|SYNC} PARTITIONS]
    * }}}
    */
   override def visitRepairTable(ctx: RepairTableContext): LogicalPlan = withOrigin(ctx) {
-    RepairTableStatement(visitMultipartIdentifier(ctx.multipartIdentifier()))
+    val (enableAddPartitions, enableDropPartitions, option) =
+      if (ctx.SYNC() != null) {
+        (true, true, " ... SYNC PARTITIONS")
+      } else if (ctx.DROP() != null) {
+        (false, true, " ... DROP PARTITIONS")
+      } else if (ctx.ADD() != null) {
+        (true, false, " ... ADD PARTITIONS")
+      } else {
+        (true, false, "")
+      }
+    RepairTable(
+      createUnresolvedTable(ctx.multipartIdentifier, s"MSCK REPAIR TABLE$option"),
+      enableAddPartitions,
+      enableDropPartitions)
   }
 
   /**
-   * Create a [[LoadDataStatement]].
+   * Create a [[LoadData]].
    *
    * For example:
    * {{{
@@ -3265,8 +3861,8 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    * }}}
    */
   override def visitLoadData(ctx: LoadDataContext): LogicalPlan = withOrigin(ctx) {
-    LoadDataStatement(
-      tableName = visitMultipartIdentifier(ctx.multipartIdentifier),
+    LoadData(
+      child = createUnresolvedTable(ctx.multipartIdentifier, "LOAD DATA"),
       path = string(ctx.path),
       isLocal = ctx.LOCAL != null,
       isOverwrite = ctx.OVERWRITE != null,
@@ -3275,14 +3871,19 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
   }
 
   /**
-   * Creates a [[ShowCreateTableStatement]]
+   * Creates a [[ShowCreateTable]]
    */
   override def visitShowCreateTable(ctx: ShowCreateTableContext): LogicalPlan = withOrigin(ctx) {
-    ShowCreateTableStatement(visitMultipartIdentifier(ctx.multipartIdentifier()), ctx.SERDE != null)
+    ShowCreateTable(
+      createUnresolvedTableOrView(
+        ctx.multipartIdentifier(),
+        "SHOW CREATE TABLE",
+        allowTempView = false),
+      ctx.SERDE != null)
   }
 
   /**
-   * Create a [[CacheTableStatement]].
+   * Create a [[CacheTable]] or [[CacheTableAsSelect]].
    *
    * For example:
    * {{{
@@ -3294,26 +3895,33 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
 
     val query = Option(ctx.query).map(plan)
-    val tableName = visitMultipartIdentifier(ctx.multipartIdentifier)
+    val relation = createUnresolvedRelation(ctx.multipartIdentifier)
+    val tableName = relation.multipartIdentifier
     if (query.isDefined && tableName.length > 1) {
       val catalogAndNamespace = tableName.init
-      throw new ParseException("It is not allowed to add catalog/namespace " +
-        s"prefix ${catalogAndNamespace.quoted} to " +
-        "the table name in CACHE TABLE AS SELECT", ctx)
+      throw QueryParsingErrors.addCatalogInCacheTableAsSelectNotAllowedError(
+        catalogAndNamespace.quoted, ctx)
     }
     val options = Option(ctx.options).map(visitPropertyKeyValues).getOrElse(Map.empty)
-    CacheTableStatement(tableName, query, ctx.LAZY != null, options)
+    val isLazy = ctx.LAZY != null
+    if (query.isDefined) {
+      CacheTableAsSelect(tableName.head, query.get, source(ctx.query()), isLazy, options)
+    } else {
+      CacheTable(relation, tableName, isLazy, options)
+    }
   }
 
   /**
-   * Create an [[UncacheTableStatement]] logical plan.
+   * Create an [[UncacheTable]] logical plan.
    */
   override def visitUncacheTable(ctx: UncacheTableContext): LogicalPlan = withOrigin(ctx) {
-    UncacheTableStatement(visitMultipartIdentifier(ctx.multipartIdentifier), ctx.EXISTS != null)
+    UncacheTable(
+      createUnresolvedRelation(ctx.multipartIdentifier),
+      ctx.EXISTS != null)
   }
 
   /**
-   * Create a [[TruncateTableStatement]] command.
+   * Create a [[TruncateTable]] command.
    *
    * For example:
    * {{{
@@ -3321,9 +3929,10 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    * }}}
    */
   override def visitTruncateTable(ctx: TruncateTableContext): LogicalPlan = withOrigin(ctx) {
-    TruncateTableStatement(
-      visitMultipartIdentifier(ctx.multipartIdentifier),
-      Option(ctx.partitionSpec).map(visitNonOptionalPartitionSpec))
+    val table = createUnresolvedTable(ctx.multipartIdentifier, "TRUNCATE TABLE")
+    Option(ctx.partitionSpec).map { spec =>
+      TruncatePartition(table, UnresolvedPartitionSpec(visitNonOptionalPartitionSpec(spec)))
+    }.getOrElse(TruncateTable(table))
   }
 
   /**
@@ -3338,13 +3947,16 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    * }}}
    */
   override def visitShowPartitions(ctx: ShowPartitionsContext): LogicalPlan = withOrigin(ctx) {
-    val table = visitMultipartIdentifier(ctx.multipartIdentifier)
-    val partitionKeys = Option(ctx.partitionSpec).map(visitNonOptionalPartitionSpec)
-    ShowPartitionsStatement(table, partitionKeys)
+    val partitionKeys = Option(ctx.partitionSpec).map { specCtx =>
+      UnresolvedPartitionSpec(visitNonOptionalPartitionSpec(specCtx), None)
+    }
+    ShowPartitions(
+      createUnresolvedTable(ctx.multipartIdentifier(), "SHOW PARTITIONS"),
+      partitionKeys)
   }
 
   /**
-   * Create a [[RefreshTableStatement]].
+   * Create a [[RefreshTable]].
    *
    * For example:
    * {{{
@@ -3352,12 +3964,15 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    * }}}
    */
   override def visitRefreshTable(ctx: RefreshTableContext): LogicalPlan = withOrigin(ctx) {
-    RefreshTableStatement(visitMultipartIdentifier(ctx.multipartIdentifier()))
+    RefreshTable(
+      createUnresolvedTableOrView(
+        ctx.multipartIdentifier(),
+        "REFRESH TABLE"))
   }
 
   /**
    * A command for users to list the column names for a table.
-   * This function creates a [[ShowColumnsStatement]] logical plan.
+   * This function creates a [[ShowColumns]] logical plan.
    *
    * The syntax of using this command in SQL is:
    * {{{
@@ -3366,13 +3981,22 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    * }}}
    */
   override def visitShowColumns(ctx: ShowColumnsContext): LogicalPlan = withOrigin(ctx) {
-    val table = visitMultipartIdentifier(ctx.table)
+    val table = createUnresolvedTableOrView(ctx.table, "SHOW COLUMNS")
     val namespace = Option(ctx.ns).map(visitMultipartIdentifier)
-    ShowColumnsStatement(table, namespace)
+    // Use namespace only if table name doesn't specify it. If namespace is already specified
+    // in the table name, it's checked against the given namespace after table/view is resolved.
+    val tableWithNamespace = if (namespace.isDefined && table.multipartIdentifier.length == 1) {
+      CurrentOrigin.withOrigin(table.origin) {
+        table.copy(multipartIdentifier = namespace.get ++ table.multipartIdentifier)
+      }
+    } else {
+      table
+    }
+    ShowColumns(tableWithNamespace, namespace)
   }
 
   /**
-   * Create an [[AlterTableRecoverPartitionsStatement]]
+   * Create an [[RecoverPartitions]]
    *
    * For example:
    * {{{
@@ -3381,11 +4005,15 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   override def visitRecoverPartitions(
       ctx: RecoverPartitionsContext): LogicalPlan = withOrigin(ctx) {
-    AlterTableRecoverPartitionsStatement(visitMultipartIdentifier(ctx.multipartIdentifier))
+    RecoverPartitions(
+      createUnresolvedTable(
+        ctx.multipartIdentifier,
+        "ALTER TABLE ... RECOVER PARTITIONS",
+        alterTableTypeMismatchHint))
   }
 
   /**
-   * Create an [[AlterTableAddPartitionStatement]].
+   * Create an [[AddPartitions]].
    *
    * For example:
    * {{{
@@ -3405,16 +4033,19 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     val specsAndLocs = ctx.partitionSpecLocation.asScala.map { splCtx =>
       val spec = visitNonOptionalPartitionSpec(splCtx.partitionSpec)
       val location = Option(splCtx.locationSpec).map(visitLocationSpec)
-      spec -> location
-    }
-    AlterTableAddPartitionStatement(
-      visitMultipartIdentifier(ctx.multipartIdentifier),
-      specsAndLocs,
+      UnresolvedPartitionSpec(spec, location)
+    }
+    AddPartitions(
+      createUnresolvedTable(
+        ctx.multipartIdentifier,
+        "ALTER TABLE ... ADD PARTITION ...",
+        alterTableTypeMismatchHint),
+      specsAndLocs.toSeq,
       ctx.EXISTS != null)
   }
 
   /**
-   * Create an [[AlterTableRenamePartitionStatement]]
+   * Create an [[RenamePartitions]]
    *
    * For example:
    * {{{
@@ -3423,14 +4054,17 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   override def visitRenameTablePartition(
       ctx: RenameTablePartitionContext): LogicalPlan = withOrigin(ctx) {
-    AlterTableRenamePartitionStatement(
-      visitMultipartIdentifier(ctx.multipartIdentifier),
-      visitNonOptionalPartitionSpec(ctx.from),
-      visitNonOptionalPartitionSpec(ctx.to))
+    RenamePartitions(
+      createUnresolvedTable(
+        ctx.multipartIdentifier,
+        "ALTER TABLE ... RENAME TO PARTITION",
+        alterTableTypeMismatchHint),
+      UnresolvedPartitionSpec(visitNonOptionalPartitionSpec(ctx.from)),
+      UnresolvedPartitionSpec(visitNonOptionalPartitionSpec(ctx.to)))
   }
 
   /**
-   * Create an [[AlterTableDropPartitionStatement]]
+   * Create an [[DropPartitions]]
    *
    * For example:
    * {{{
@@ -3447,16 +4081,20 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     if (ctx.VIEW != null) {
       operationNotAllowed("ALTER VIEW ... DROP PARTITION", ctx)
     }
-    AlterTableDropPartitionStatement(
-      visitMultipartIdentifier(ctx.multipartIdentifier),
-      ctx.partitionSpec.asScala.map(visitNonOptionalPartitionSpec),
+    val partSpecs = ctx.partitionSpec.asScala.map(visitNonOptionalPartitionSpec)
+      .map(spec => UnresolvedPartitionSpec(spec))
+    DropPartitions(
+      createUnresolvedTable(
+        ctx.multipartIdentifier,
+        "ALTER TABLE ... DROP PARTITION ...",
+        alterTableTypeMismatchHint),
+      partSpecs.toSeq,
       ifExists = ctx.EXISTS != null,
-      purge = ctx.PURGE != null,
-      retainData = false)
+      purge = ctx.PURGE != null)
   }
 
   /**
-   * Create an [[AlterTableSerDePropertiesStatement]]
+   * Create an [[SetTableSerDeProperties]]
    *
    * For example:
    * {{{
@@ -3466,8 +4104,11 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    * }}}
    */
   override def visitSetTableSerDe(ctx: SetTableSerDeContext): LogicalPlan = withOrigin(ctx) {
-    AlterTableSerDePropertiesStatement(
-      visitMultipartIdentifier(ctx.multipartIdentifier),
+    SetTableSerDeProperties(
+      createUnresolvedTable(
+        ctx.multipartIdentifier,
+        "ALTER TABLE ... SET [SERDE|SERDEPROPERTIES]",
+        alterTableTypeMismatchHint),
       Option(ctx.STRING).map(string),
       Option(ctx.tablePropertyList).map(visitPropertyKeyValues),
       // TODO a partition spec is allowed to have optional values. This is currently violated.
@@ -3505,6 +4146,12 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       }
     }
 
+    val properties = ctx.tablePropertyList.asScala.headOption.map(visitPropertyKeyValues)
+      .getOrElse(Map.empty)
+    if (ctx.TEMPORARY != null && !properties.isEmpty) {
+      operationNotAllowed("TBLPROPERTIES can't coexist with CREATE TEMPORARY VIEW", ctx)
+    }
+
     val viewType = if (ctx.TEMPORARY == null) {
       PersistedView
     } else if (ctx.GLOBAL != null) {
@@ -3516,8 +4163,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       visitMultipartIdentifier(ctx.multipartIdentifier),
       userSpecifiedColumns,
       visitCommentSpecList(ctx.commentSpec()),
-      ctx.tablePropertyList.asScala.headOption.map(visitPropertyKeyValues)
-        .getOrElse(Map.empty),
+      properties,
       Option(source(ctx.query)),
       plan(ctx.query),
       ctx.EXISTS != null,
@@ -3526,7 +4172,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
   }
 
   /**
-   * Alter the query of a view. This creates a [[AlterViewAsStatement]]
+   * Alter the query of a view. This creates a [[AlterViewAs]]
    *
    * For example:
    * {{{
@@ -3534,14 +4180,14 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    * }}}
    */
   override def visitAlterViewQuery(ctx: AlterViewQueryContext): LogicalPlan = withOrigin(ctx) {
-    AlterViewAsStatement(
-      visitMultipartIdentifier(ctx.multipartIdentifier),
+    AlterViewAs(
+      createUnresolvedView(ctx.multipartIdentifier, "ALTER VIEW ... AS"),
       originalText = source(ctx.query),
       query = plan(ctx.query))
   }
 
   /**
-   * Create a [[RenameTableStatement]] command.
+   * Create a [[RenameTable]] command.
    *
    * For example:
    * {{{
@@ -3550,10 +4196,12 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    * }}}
    */
   override def visitRenameTable(ctx: RenameTableContext): LogicalPlan = withOrigin(ctx) {
-    RenameTableStatement(
-      visitMultipartIdentifier(ctx.from),
+    val isView = ctx.VIEW != null
+    val relationStr = if (isView) "VIEW" else "TABLE"
+    RenameTable(
+      createUnresolvedTableOrView(ctx.from, s"ALTER $relationStr ... RENAME TO"),
       visitMultipartIdentifier(ctx.to),
-      ctx.VIEW != null)
+      isView)
   }
 
   /**
@@ -3568,7 +4216,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
   override def visitShowTblProperties(
       ctx: ShowTblPropertiesContext): LogicalPlan = withOrigin(ctx) {
     ShowTableProperties(
-      UnresolvedTableOrView(visitMultipartIdentifier(ctx.table)),
+      createUnresolvedTableOrView(ctx.table, "SHOW TBLPROPERTIES"),
       Option(ctx.key).map(visitTablePropertyKey))
   }
 
@@ -3585,7 +4233,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       } else {
         Seq(describeFuncName.getText)
       }
-    DescribeFunctionStatement(functionName, EXTENDED != null)
+    DescribeFunction(UnresolvedFunc(functionName), EXTENDED != null)
   }
 
   /**
@@ -3597,11 +4245,13 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
         case None | Some("all") => (true, true)
         case Some("system") => (false, true)
         case Some("user") => (true, false)
-        case Some(x) => throw new ParseException(s"SHOW $x FUNCTIONS not supported", ctx)
+        case Some(x) => throw QueryParsingErrors.showFunctionsUnsupportedError(x, ctx.identifier())
     }
     val pattern = Option(ctx.pattern).map(string(_))
-    val functionName = Option(ctx.multipartIdentifier).map(visitMultipartIdentifier)
-    ShowFunctionsStatement(userScope, systemScope, pattern, functionName)
+    val unresolvedFuncOpt = Option(ctx.multipartIdentifier)
+      .map(visitMultipartIdentifier)
+      .map(UnresolvedFunc(_))
+    ShowFunctions(unresolvedFuncOpt, userScope, systemScope, pattern)
   }
 
   /**
@@ -3614,8 +4264,8 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   override def visitDropFunction(ctx: DropFunctionContext): LogicalPlan = withOrigin(ctx) {
     val functionName = visitMultipartIdentifier(ctx.multipartIdentifier)
-    DropFunctionStatement(
-      functionName,
+    DropFunction(
+      UnresolvedFunc(functionName),
       ctx.EXISTS != null,
       ctx.TEMPORARY != null)
   }
@@ -3644,12 +4294,17 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     CreateFunctionStatement(
       functionIdentifier,
       string(ctx.className),
-      resources,
+      resources.toSeq,
       ctx.TEMPORARY != null,
       ctx.EXISTS != null,
       ctx.REPLACE != null)
   }
 
+  override def visitRefreshFunction(ctx: RefreshFunctionContext): LogicalPlan = withOrigin(ctx) {
+    val functionIdentifier = visitMultipartIdentifier(ctx.multipartIdentifier)
+    RefreshFunction(UnresolvedFunc(functionIdentifier))
+  }
+
   override def visitCommentNamespace(ctx: CommentNamespaceContext): LogicalPlan = withOrigin(ctx) {
     val comment = ctx.comment.getType match {
       case SqlBaseParser.NULL => ""
@@ -3664,7 +4319,10 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       case SqlBaseParser.NULL => ""
       case _ => string(ctx.STRING)
     }
-    val nameParts = visitMultipartIdentifier(ctx.multipartIdentifier)
-    CommentOnTable(UnresolvedTable(nameParts), comment)
+    CommentOnTable(createUnresolvedTable(ctx.multipartIdentifier, "COMMENT ON TABLE"), comment)
   }
+
+  private def alterViewTypeMismatchHint: Option[String] = Some("Please use ALTER TABLE instead.")
+
+  private def alterTableTypeMismatchHint: Option[String] = Some("Please use ALTER VIEW instead.")
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/LegacyTypeStringParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/LegacyTypeStringParser.scala
index 058136a213e9e..f36fcade382bf 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/LegacyTypeStringParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/LegacyTypeStringParser.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.catalyst.parser
 
 import scala.util.parsing.combinator.RegexParsers
 
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.types._
 
 /**
@@ -87,6 +88,6 @@ object LegacyTypeStringParser extends RegexParsers {
   def parseString(asString: String): DataType = parseAll(dataType, asString) match {
     case Success(result, _) => result
     case failure: NoSuccess =>
-      throw new IllegalArgumentException(s"Unsupported dataType: $asString, $failure")
+      throw QueryExecutionErrors.dataTypeUnsupportedError(asString, failure.toString)
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala
index fab282f15f215..ae13bf1a0cd70 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala
@@ -23,28 +23,23 @@ import org.antlr.v4.runtime.tree.TerminalNodeImpl
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
+import org.apache.spark.sql.catalyst.{FunctionIdentifier, SQLConfHelper, TableIdentifier}
 import org.apache.spark.sql.catalyst.expressions.Expression
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.trees.Origin
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.errors.QueryParsingErrors
 import org.apache.spark.sql.types.{DataType, StructType}
 
 /**
  * Base SQL parsing infrastructure.
  */
-abstract class AbstractSqlParser(conf: SQLConf) extends ParserInterface with Logging {
+abstract class AbstractSqlParser extends ParserInterface with SQLConfHelper with Logging {
 
   /** Creates/Resolves DataType for a given SQL string. */
   override def parseDataType(sqlText: String): DataType = parse(sqlText) { parser =>
     astBuilder.visitSingleDataType(parser.singleDataType())
   }
 
-  /** Similar to `parseDataType`, but without CHAR/VARCHAR replacement. */
-  override def parseRawDataType(sqlText: String): DataType = parse(sqlText) { parser =>
-    astBuilder.parseRawDataType(parser.singleDataType())
-  }
-
   /** Creates Expression for a given SQL string. */
   override def parseExpression(sqlText: String): Expression = parse(sqlText) { parser =>
     astBuilder.visitSingleExpression(parser.singleExpression())
@@ -83,7 +78,7 @@ abstract class AbstractSqlParser(conf: SQLConf) extends ParserInterface with Log
       case plan: LogicalPlan => plan
       case _ =>
         val position = Origin(None, None)
-        throw new ParseException(Option(sqlText), "Unsupported SQL statement", position, position)
+        throw QueryParsingErrors.sqlStatementUnsupportedError(sqlText, position)
     }
   }
 
@@ -96,16 +91,13 @@ abstract class AbstractSqlParser(conf: SQLConf) extends ParserInterface with Log
     val lexer = new SqlBaseLexer(new UpperCaseCharStream(CharStreams.fromString(command)))
     lexer.removeErrorListeners()
     lexer.addErrorListener(ParseErrorListener)
-    lexer.legacy_setops_precedence_enbled = conf.setOpsPrecedenceEnforced
-    lexer.legacy_exponent_literal_as_decimal_enabled = conf.exponentLiteralAsDecimalEnabled
-    lexer.SQL_standard_keyword_behavior = conf.ansiEnabled
 
     val tokenStream = new CommonTokenStream(lexer)
     val parser = new SqlBaseParser(tokenStream)
     parser.addParseListener(PostProcessor)
     parser.removeErrorListeners()
     parser.addErrorListener(ParseErrorListener)
-    parser.legacy_setops_precedence_enbled = conf.setOpsPrecedenceEnforced
+    parser.legacy_setops_precedence_enabled = conf.setOpsPrecedenceEnforced
     parser.legacy_exponent_literal_as_decimal_enabled = conf.exponentLiteralAsDecimalEnabled
     parser.SQL_standard_keyword_behavior = conf.ansiEnabled
 
@@ -141,14 +133,12 @@ abstract class AbstractSqlParser(conf: SQLConf) extends ParserInterface with Log
 /**
  * Concrete SQL parser for Catalyst-only SQL statements.
  */
-class CatalystSqlParser(conf: SQLConf) extends AbstractSqlParser(conf) {
-  val astBuilder = new AstBuilder(conf)
+class CatalystSqlParser extends AbstractSqlParser {
+  val astBuilder = new AstBuilder
 }
 
 /** For test-only. */
-object CatalystSqlParser extends AbstractSqlParser(SQLConf.get) {
-  val astBuilder = new AstBuilder(SQLConf.get)
-}
+object CatalystSqlParser extends CatalystSqlParser
 
 /**
  * This string stream provides the lexer with upper case characters only. This greatly simplifies
@@ -179,18 +169,7 @@ private[parser] class UpperCaseCharStream(wrapped: CodePointCharStream) extends
   override def seek(where: Int): Unit = wrapped.seek(where)
   override def size(): Int = wrapped.size
 
-  override def getText(interval: Interval): String = {
-    // ANTLR 4.7's CodePointCharStream implementations have bugs when
-    // getText() is called with an empty stream, or intervals where
-    // the start > end. See
-    // https://github.com/antlr/antlr4/commit/ac9f7530 for one fix
-    // that is not yet in a released ANTLR artifact.
-    if (size() > 0 && (interval.b - interval.a >= 0)) {
-      wrapped.getText(interval)
-    } else {
-      ""
-    }
-  }
+  override def getText(interval: Interval): String = wrapped.getText(interval)
 
   override def LA(i: Int): Int = {
     val la = wrapped.LA(i)
@@ -276,8 +255,7 @@ case object PostProcessor extends SqlBaseBaseListener {
   override def exitErrorIdent(ctx: SqlBaseParser.ErrorIdentContext): Unit = {
     val ident = ctx.getParent.getText
 
-    throw new ParseException(s"Possibly unquoted identifier $ident detected. " +
-      s"Please consider quoting it with back-quotes as `$ident`", ctx)
+    throw QueryParsingErrors.unquotedIdentifierError(ident, ctx)
   }
 
   /** Remove the back ticks from an Identifier. */
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserInterface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserInterface.scala
index d724933bc1029..77e357ad073da 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserInterface.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserInterface.scala
@@ -70,10 +70,4 @@ trait ParserInterface {
    */
   @throws[ParseException]("Text cannot be parsed to a DataType")
   def parseDataType(sqlText: String): DataType
-
-  /**
-   * Parse a string to a raw [[DataType]] without CHAR/VARCHAR replacement.
-   */
-  @throws[ParseException]("Text cannot be parsed to a DataType")
-  def parseRawDataType(sqlText: String): DataType
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserUtils.scala
index a3779698a5ac3..f7cf2bada413f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserUtils.scala
@@ -16,6 +16,8 @@
  */
 package org.apache.spark.sql.catalyst.parser
 
+import java.lang.{Long => JLong}
+import java.nio.CharBuffer
 import java.util
 
 import scala.collection.mutable.StringBuilder
@@ -26,11 +28,18 @@ import org.antlr.v4.runtime.tree.TerminalNode
 
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, Origin}
+import org.apache.spark.sql.errors.QueryParsingErrors
 
 /**
  * A collection of utility methods for use during the parsing process.
  */
 object ParserUtils {
+
+  val U16_CHAR_PATTERN = """\\u([a-fA-F0-9]{4})(?s).*""".r
+  val U32_CHAR_PATTERN = """\\U([a-fA-F0-9]{8})(?s).*""".r
+  val OCTAL_CHAR_PATTERN = """\\([01][0-7]{2})(?s).*""".r
+  val ESCAPED_CHAR_PATTERN = """\\((?s).)(?s).*""".r
+
   /** Get the command which created the token. */
   def command(ctx: ParserRuleContext): String = {
     val stream = ctx.getStart.getInputStream
@@ -38,20 +47,20 @@ object ParserUtils {
   }
 
   def operationNotAllowed(message: String, ctx: ParserRuleContext): Nothing = {
-    throw new ParseException(s"Operation not allowed: $message", ctx)
+    throw QueryParsingErrors.operationNotAllowedError(message, ctx)
   }
 
   def checkDuplicateClauses[T](
       nodes: util.List[T], clauseName: String, ctx: ParserRuleContext): Unit = {
     if (nodes.size() > 1) {
-      throw new ParseException(s"Found duplicate clauses: $clauseName", ctx)
+      throw QueryParsingErrors.duplicateClausesError(clauseName, ctx)
     }
   }
 
   /** Check if duplicate keys exist in a set of key-value pairs. */
   def checkDuplicateKeys[T](keyPairs: Seq[(String, T)], ctx: ParserRuleContext): Unit = {
     keyPairs.groupBy(_._1).filter(_._2.size > 1).foreach { case (key, _) =>
-      throw new ParseException(s"Found duplicate keys '$key'.", ctx)
+      throw QueryParsingErrors.duplicateKeysError(key, ctx)
     }
   }
 
@@ -71,6 +80,17 @@ object ParserUtils {
     stream.getText(interval)
   }
 
+  /**
+   * Get all the text which between the given start and end tokens.
+   * When we need to extract everything between two tokens including all spaces we should use
+   * this method instead of defined a named Antlr4 rule for .*?,
+   * which somehow parse "a b" -> "ab" in some cases
+   */
+  def interval(start: Token, end: Token): String = {
+    val interval = Interval.of(start.getStopIndex + 1, end.getStartIndex - 1)
+    start.getInputStream.getText(interval)
+  }
+
   /** Convert a string token into a string. */
   def string(token: Token): String = unescapeSQLString(token.getText)
 
@@ -83,6 +103,11 @@ object ParserUtils {
     node.getText.slice(1, node.getText.size - 1)
   }
 
+  /** Collect the entries if any. */
+  def entry(key: String, value: Token): Seq[(String, String)] = {
+    Option(value).toSeq.map(x => key -> string(x))
+  }
+
   /** Get the origin (line and position) of the token. */
   def position(token: Token): Origin = {
     val opt = Option(token)
@@ -111,9 +136,8 @@ object ParserUtils {
     }
   }
 
-  /** Unescape baskslash-escaped string enclosed by quotes. */
+  /** Unescape backslash-escaped string enclosed by quotes. */
   def unescapeSQLString(b: String): String = {
-    var enclosure: Character = null
     val sb = new StringBuilder(b.length())
 
     def appendEscapedChar(n: Char): Unit = {
@@ -134,63 +158,49 @@ object ParserUtils {
       }
     }
 
-    var i = 0
-    val strLength = b.length
-    while (i < strLength) {
-      val currentChar = b.charAt(i)
-      if (enclosure == null) {
-        if (currentChar == '\'' || currentChar == '\"') {
-          enclosure = currentChar
-        }
-      } else if (enclosure == currentChar) {
-        enclosure = null
-      } else if (currentChar == '\\') {
-
-        if ((i + 6 < strLength) && b.charAt(i + 1) == 'u') {
-          // \u0000 style character literals.
-
-          val base = i + 2
-          val code = (0 until 4).foldLeft(0) { (mid, j) =>
-            val digit = Character.digit(b.charAt(j + base), 16)
-            (mid << 4) + digit
-          }
-          sb.append(code.asInstanceOf[Char])
-          i += 5
-        } else if (i + 4 < strLength) {
-          // \000 style character literals.
-
-          val i1 = b.charAt(i + 1)
-          val i2 = b.charAt(i + 2)
-          val i3 = b.charAt(i + 3)
-
-          if ((i1 >= '0' && i1 <= '1') && (i2 >= '0' && i2 <= '7') && (i3 >= '0' && i3 <= '7')) {
-            val tmp = ((i3 - '0') + ((i2 - '0') << 3) + ((i1 - '0') << 6)).asInstanceOf[Char]
-            sb.append(tmp)
-            i += 3
+    // Skip the first and last quotations enclosing the string literal.
+    val charBuffer = CharBuffer.wrap(b, 1, b.length - 1)
+
+    while (charBuffer.remaining() > 0) {
+      charBuffer match {
+        case U16_CHAR_PATTERN(cp) =>
+          // \u0000 style 16-bit unicode character literals.
+          sb.append(Integer.parseInt(cp, 16).toChar)
+          charBuffer.position(charBuffer.position() + 6)
+        case U32_CHAR_PATTERN(cp) =>
+          // \U00000000 style 32-bit unicode character literals.
+          // Use Long to treat codePoint as unsigned in the range of 32-bit.
+          val codePoint = JLong.parseLong(cp, 16)
+          if (codePoint < 0x10000) {
+            sb.append((codePoint & 0xFFFF).toChar)
           } else {
-            appendEscapedChar(i1)
-            i += 1
+            val highSurrogate = (codePoint - 0x10000) / 0x400 + 0xD800
+            val lowSurrogate = (codePoint - 0x10000) % 0x400 + 0xDC00
+            sb.append(highSurrogate.toChar)
+            sb.append(lowSurrogate.toChar)
           }
-        } else if (i + 2 < strLength) {
+          charBuffer.position(charBuffer.position() + 10)
+        case OCTAL_CHAR_PATTERN(cp) =>
+          // \000 style character literals.
+          sb.append(Integer.parseInt(cp, 8).toChar)
+          charBuffer.position(charBuffer.position() + 4)
+        case ESCAPED_CHAR_PATTERN(c) =>
           // escaped character literals.
-          val n = b.charAt(i + 1)
-          appendEscapedChar(n)
-          i += 1
-        }
-      } else {
-        // non-escaped character literals.
-        sb.append(currentChar)
+          appendEscapedChar(c.charAt(0))
+          charBuffer.position(charBuffer.position() + 2)
+        case _ =>
+          // non-escaped character literals.
+          sb.append(charBuffer.get())
       }
-      i += 1
     }
     sb.toString()
   }
 
   /** the column name pattern in quoted regex without qualifier */
-  val escapedIdentifier = "`(.+)`".r
+  val escapedIdentifier = "`((?s).+)`".r
 
   /** the column name pattern in quoted regex with qualifier */
-  val qualifiedEscapedIdentifier = ("(.+)" + """.""" + "`(.+)`").r
+  val qualifiedEscapedIdentifier = ("((?s).+)" + """.""" + "`((?s).+)`").r
 
   /** Some syntactic sugar which makes it easier to work with optional clauses for LogicalPlans. */
   implicit class EnhancedLogicalPlan(val plan: LogicalPlan) extends AnyVal {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
index 415ce46788119..8def82eea6e7e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
@@ -21,8 +21,10 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
+import org.apache.spark.sql.catalyst.optimizer.JoinSelectionHelper
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.internal.SQLConf
 
 trait OperationHelper {
   type ReturnType = (Seq[NamedExpression], Seq[Expression], LogicalPlan)
@@ -295,11 +297,9 @@ object PhysicalAggregation {
       val aggregateExpressions = resultExpressions.flatMap { expr =>
         expr.collect {
           // addExpr() always returns false for non-deterministic expressions and do not add them.
-          case agg: AggregateExpression
-            if !equivalentAggregateExpressions.addExpr(agg) => agg
-          case udf: PythonUDF
-            if PythonUDF.isGroupedAggPandasUDF(udf) &&
-              !equivalentAggregateExpressions.addExpr(udf) => udf
+          case a
+            if AggregateExpression.isAggregate(a) && !equivalentAggregateExpressions.addExpr(a) =>
+            a
         }
       }
 
@@ -331,7 +331,7 @@ object PhysicalAggregation {
           case ue: PythonUDF if PythonUDF.isGroupedAggPandasUDF(ue) =>
             equivalentAggregateExpressions.getEquivalentExprs(ue).headOption
               .getOrElse(ue).asInstanceOf[PythonUDF].resultAttribute
-          case expression =>
+          case expression if !expression.foldable =>
             // Since we're using `namedGroupingAttributes` to extract the grouping key
             // columns, we need to replace grouping key expressions with their corresponding
             // attributes. We do not rely on the equality check at here since attributes may
@@ -388,3 +388,37 @@ object PhysicalWindow {
     case _ => None
   }
 }
+
+object ExtractSingleColumnNullAwareAntiJoin extends JoinSelectionHelper with PredicateHelper {
+
+  // TODO support multi column NULL-aware anti join in future.
+  // See. http://www.vldb.org/pvldb/vol2/vldb09-423.pdf Section 6
+  // multi-column null aware anti join is much more complicated than single column ones.
+
+  // streamedSideKeys, buildSideKeys
+  private type ReturnType = (Seq[Expression], Seq[Expression])
+
+  /**
+   * See. [SPARK-32290]
+   * LeftAnti(condition: Or(EqualTo(a=b), IsNull(EqualTo(a=b)))
+   * will almost certainly be planned as a Broadcast Nested Loop join,
+   * which is very time consuming because it's an O(M*N) calculation.
+   * But if it's a single column case O(M*N) calculation could be optimized into O(M)
+   * using hash lookup instead of loop lookup.
+   */
+  def unapply(join: Join): Option[ReturnType] = join match {
+    case Join(left, right, LeftAnti,
+      Some(Or(e @ EqualTo(leftAttr: Expression, rightAttr: Expression),
+        IsNull(e2 @ EqualTo(_, _)))), _)
+        if SQLConf.get.optimizeNullAwareAntiJoin &&
+          e.semanticEquals(e2) =>
+      if (canEvaluate(leftAttr, left) && canEvaluate(rightAttr, right)) {
+        Some(Seq(leftAttr), Seq(rightAttr))
+      } else if (canEvaluate(leftAttr, right) && canEvaluate(rightAttr, left)) {
+        Some(Seq(rightAttr), Seq(leftAttr))
+      } else {
+        None
+      }
+    case _ => None
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/DescribeCommandSchema.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/DescribeCommandSchema.scala
new file mode 100644
index 0000000000000..99d2ea7751959
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/DescribeCommandSchema.scala
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.plans
+
+import org.apache.spark.sql.catalyst.expressions.AttributeReference
+import org.apache.spark.sql.types.{MetadataBuilder, StringType}
+
+private[sql] object DescribeCommandSchema {
+  def describeTableAttributes(): Seq[AttributeReference] = Seq(
+    AttributeReference("col_name", StringType, nullable = false,
+      new MetadataBuilder().putString("comment", "name of the column").build())(),
+    AttributeReference("data_type", StringType, nullable = false,
+      new MetadataBuilder().putString("comment", "data type of the column").build())(),
+    AttributeReference("comment", StringType, nullable = true,
+      new MetadataBuilder().putString("comment", "comment of the column").build())())
+
+  def describeColumnAttributes(): Seq[AttributeReference] = Seq(
+    AttributeReference("info_name", StringType, nullable = false,
+      new MetadataBuilder().putString("comment", "name of the column info").build())(),
+    AttributeReference("info_value", StringType, nullable = false,
+      new MetadataBuilder().putString("comment", "value of the column info").build())())
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/DescribeTableSchema.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/DescribeTableSchema.scala
deleted file mode 100644
index ff35972b901f9..0000000000000
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/DescribeTableSchema.scala
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans
-
-import org.apache.spark.sql.catalyst.expressions.AttributeReference
-import org.apache.spark.sql.types.{MetadataBuilder, StringType, StructField, StructType}
-
-private[sql] object DescribeTableSchema {
-  def describeTableAttributes(): Seq[AttributeReference] = Seq(
-    AttributeReference("col_name", StringType, nullable = false,
-      new MetadataBuilder().putString("comment", "name of the column").build())(),
-    AttributeReference("data_type", StringType, nullable = false,
-      new MetadataBuilder().putString("comment", "data type of the column").build())(),
-    AttributeReference("comment", StringType, nullable = true,
-      new MetadataBuilder().putString("comment", "comment of the column").build())())
-}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
index 7133fb231d672..e4e546aa158b3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
@@ -17,11 +17,18 @@
 
 package org.apache.spark.sql.catalyst.plans
 
+import scala.collection.mutable
+
 import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.SQLConfHelper
 import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, TreeNode, TreeNodeTag}
+import org.apache.spark.sql.catalyst.rules.RuleId
+import org.apache.spark.sql.catalyst.rules.UnknownRuleId
+import org.apache.spark.sql.catalyst.trees.{AlwaysProcess, CurrentOrigin, TreeNode, TreeNodeTag}
+import org.apache.spark.sql.catalyst.trees.TreePatternBits
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{DataType, StructType}
+import org.apache.spark.util.collection.BitSet
 
 /**
  * An abstraction of the Spark SQL query plan tree, which can be logical or physical. This class
@@ -33,15 +40,10 @@ import org.apache.spark.sql.types.{DataType, StructType}
  * The tree traverse APIs like `transform`, `foreach`, `collect`, etc. that are
  * inherited from `TreeNode`, do not traverse into query plans inside subqueries.
  */
-abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanType] {
+abstract class QueryPlan[PlanType <: QueryPlan[PlanType]]
+  extends TreeNode[PlanType] with SQLConfHelper {
   self: PlanType =>
 
-  /**
-   * The active config object within the current scope.
-   * See [[SQLConf.get]] for more information.
-   */
-  def conf: SQLConf = SQLConf.get
-
   def output: Seq[Attribute]
 
   /**
@@ -50,6 +52,17 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT
   @transient
   lazy val outputSet: AttributeSet = AttributeSet(output)
 
+  // Override `treePatternBits` to propagate bits for its expressions.
+  override lazy val treePatternBits: BitSet = {
+    val bits: BitSet = getDefaultTreePatternBits
+    // Propagate expressions' pattern bits
+    val exprIterator = expressions.iterator
+    while (exprIterator.hasNext) {
+      bits.union(exprIterator.next.treePatternBits)
+    }
+    bits
+  }
+
   /**
    * The set of all attributes that are input to this operator by its children.
    */
@@ -84,7 +97,29 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT
    * @param rule the rule to be applied to every expression in this operator.
    */
   def transformExpressions(rule: PartialFunction[Expression, Expression]): this.type = {
-    transformExpressionsDown(rule)
+    transformExpressionsWithPruning(AlwaysProcess.fn, UnknownRuleId)(rule)
+  }
+
+  /**
+   * Runs [[transformExpressionsDownWithPruning]] with `rule` on all expressions present
+   * in this query operator.
+   * Users should not expect a specific directionality. If a specific directionality is needed,
+   * transformExpressionsDown or transformExpressionsUp should be used.
+   *
+   * @param rule the rule to be applied to every expression in this operator.
+   * @param cond   a Lambda expression to prune tree traversals. If `cond.apply` returns false
+   *               on an expression T, skips processing T and its subtree; otherwise, processes
+   *               T and its subtree recursively.
+   * @param ruleId is a unique Id for `rule` to prune unnecessary tree traversals. When it is
+   *               UnknownRuleId, no pruning happens. Otherwise, if `rule`(with id `ruleId`)
+   *               has been marked as in effective on an expression T, skips processing T and its
+   *               subtree. Do not pass it if the rule is not purely functional and reads a
+   *               varying initial state for different invocations.
+   */
+  def transformExpressionsWithPruning(cond: TreePatternBits => Boolean,
+    ruleId: RuleId = UnknownRuleId)(rule: PartialFunction[Expression, Expression])
+  : this.type = {
+    transformExpressionsDownWithPruning(cond, ruleId)(rule)
   }
 
   /**
@@ -93,17 +128,56 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT
    * @param rule the rule to be applied to every expression in this operator.
    */
   def transformExpressionsDown(rule: PartialFunction[Expression, Expression]): this.type = {
-    mapExpressions(_.transformDown(rule))
+    transformExpressionsDownWithPruning(AlwaysProcess.fn, UnknownRuleId)(rule)
+  }
+
+  /**
+   * Runs [[transformDownWithPruning]] with `rule` on all expressions present in this query
+   * operator.
+   *
+   * @param rule   the rule to be applied to every expression in this operator.
+   * @param cond   a Lambda expression to prune tree traversals. If `cond.apply` returns false
+   *               on an expression T, skips processing T and its subtree; otherwise, processes
+   *               T and its subtree recursively.
+   * @param ruleId is a unique Id for `rule` to prune unnecessary tree traversals. When it is
+   *               UnknownRuleId, no pruning happens. Otherwise, if `rule` (with id `ruleId`)
+   *               has been marked as in effective on an expression T, skips processing T and its
+   *               subtree. Do not pass it if the rule is not purely functional and reads a
+   *               varying initial state for different invocations.
+   */
+  def transformExpressionsDownWithPruning(cond: TreePatternBits => Boolean,
+    ruleId: RuleId = UnknownRuleId)(rule: PartialFunction[Expression, Expression])
+  : this.type = {
+    mapExpressions(_.transformDownWithPruning(cond, ruleId)(rule))
   }
 
   /**
    * Runs [[transformUp]] with `rule` on all expressions present in this query operator.
    *
    * @param rule the rule to be applied to every expression in this operator.
-   * @return
    */
   def transformExpressionsUp(rule: PartialFunction[Expression, Expression]): this.type = {
-    mapExpressions(_.transformUp(rule))
+    transformExpressionsUpWithPruning(AlwaysProcess.fn, UnknownRuleId)(rule)
+  }
+
+  /**
+   * Runs [[transformExpressionsUpWithPruning]] with `rule` on all expressions present in this
+   * query operator.
+   *
+   * @param rule the rule to be applied to every expression in this operator.
+   * @param cond   a Lambda expression to prune tree traversals. If `cond.apply` returns false
+   *               on an expression T, skips processing T and its subtree; otherwise, processes
+   *               T and its subtree recursively.
+   * @param ruleId is a unique Id for `rule` to prune unnecessary tree traversals. When it is
+   *               UnknownRuleId, no pruning happens. Otherwise, if `rule` (with id `ruleId`)
+   *               has been marked as in effective on an expression T, skips processing T and its
+   *               subtree. Do not pass it if the rule is not purely functional and reads a
+   *               varying initial state for different invocations.
+   */
+  def transformExpressionsUpWithPruning(cond: TreePatternBits => Boolean,
+    ruleId: RuleId = UnknownRuleId)(rule: PartialFunction[Expression, Expression])
+  : this.type = {
+    mapExpressions(_.transformUpWithPruning(cond, ruleId)(rule))
   }
 
   /**
@@ -146,8 +220,19 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT
    * and all its children. Note that this method skips expressions inside subqueries.
    */
   def transformAllExpressions(rule: PartialFunction[Expression, Expression]): this.type = {
-    transform {
-      case q: QueryPlan[_] => q.transformExpressions(rule).asInstanceOf[PlanType]
+    transformAllExpressionsWithPruning(AlwaysProcess.fn, UnknownRuleId)(rule)
+  }
+
+  /**
+   * Returns the result of running [[transformExpressionsWithPruning]] on this node
+   * and all its children. Note that this method skips expressions inside subqueries.
+   */
+  def transformAllExpressionsWithPruning(cond: TreePatternBits => Boolean,
+    ruleId: RuleId = UnknownRuleId)(rule: PartialFunction[Expression, Expression])
+  : this.type = {
+    transformWithPruning(cond, ruleId) {
+      case q: QueryPlan[_] =>
+        q.transformExpressionsWithPruning(cond, ruleId)(rule).asInstanceOf[PlanType]
     }.asInstanceOf[this.type]
   }
 
@@ -168,6 +253,121 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT
     }.toSeq
   }
 
+  /**
+   * A variant of `transformUp`, which takes care of the case that the rule replaces a plan node
+   * with a new one that has different output expr IDs, by updating the attribute references in
+   * the parent nodes accordingly.
+   *
+   * @param rule the function to transform plan nodes, and return new nodes with attributes mapping
+   *             from old attributes to new attributes. The attribute mapping will be used to
+   *             rewrite attribute references in the parent nodes.
+   * @param skipCond a boolean condition to indicate if we can skip transforming a plan node to save
+   *                 time.
+   * @param canGetOutput a boolean condition to indicate if we can get the output of a plan node
+   *                     to prune the attributes mapping to be propagated. The default value is true
+   *                     as only unresolved logical plan can't get output.
+   */
+  def transformUpWithNewOutput(
+      rule: PartialFunction[PlanType, (PlanType, Seq[(Attribute, Attribute)])],
+      skipCond: PlanType => Boolean = _ => false,
+      canGetOutput: PlanType => Boolean = _ => true): PlanType = {
+    def rewrite(plan: PlanType): (PlanType, Seq[(Attribute, Attribute)]) = {
+      if (skipCond(plan)) {
+        plan -> Nil
+      } else {
+        val attrMapping = new mutable.ArrayBuffer[(Attribute, Attribute)]()
+        var newPlan = plan.mapChildren { child =>
+          val (newChild, childAttrMapping) = rewrite(child)
+          attrMapping ++= childAttrMapping
+          newChild
+        }
+
+        val attrMappingForCurrentPlan = attrMapping.filter {
+          // The `attrMappingForCurrentPlan` is used to replace the attributes of the
+          // current `plan`, so the `oldAttr` must be part of `plan.references`.
+          case (oldAttr, _) => plan.references.contains(oldAttr)
+        }
+
+        if (attrMappingForCurrentPlan.nonEmpty) {
+          assert(!attrMappingForCurrentPlan.groupBy(_._1.exprId)
+            .exists(_._2.map(_._2.exprId).distinct.length > 1),
+            "Found duplicate rewrite attributes")
+
+          val attributeRewrites = AttributeMap(attrMappingForCurrentPlan.toSeq)
+          // Using attrMapping from the children plans to rewrite their parent node.
+          // Note that we shouldn't rewrite a node using attrMapping from its sibling nodes.
+          newPlan = newPlan.rewriteAttrs(attributeRewrites)
+        }
+
+        val (planAfterRule, newAttrMapping) = CurrentOrigin.withOrigin(origin) {
+          rule.applyOrElse(newPlan, (plan: PlanType) => plan -> Nil)
+        }
+
+        val newValidAttrMapping = newAttrMapping.filter {
+          case (a1, a2) => a1.exprId != a2.exprId
+        }
+
+        // Updates the `attrMapping` entries that are obsoleted by generated entries in `rule`.
+        // For example, `attrMapping` has a mapping entry 'id#1 -> id#2' and `rule`
+        // generates a new entry 'id#2 -> id#3'. In this case, we need to update
+        // the corresponding old entry from 'id#1 -> id#2' to '#id#1 -> #id#3'.
+        val updatedAttrMap = AttributeMap(newValidAttrMapping)
+        val transferAttrMapping = attrMapping.map {
+          case (a1, a2) => (a1, updatedAttrMap.getOrElse(a2, a2))
+        }
+        val newOtherAttrMapping = {
+          val existingAttrMappingSet = transferAttrMapping.map(_._2).toSet
+          newValidAttrMapping.filterNot { case (_, a) => existingAttrMappingSet.contains(a) }
+        }
+        val resultAttrMapping = if (canGetOutput(plan)) {
+          // We propagate the attributes mapping to the parent plan node to update attributes, so
+          // the `newAttr` must be part of this plan's output.
+          (transferAttrMapping ++ newOtherAttrMapping).filter {
+            case (_, newAttr) => planAfterRule.outputSet.contains(newAttr)
+          }
+        } else {
+          transferAttrMapping ++ newOtherAttrMapping
+        }
+        planAfterRule -> resultAttrMapping.toSeq
+      }
+    }
+    rewrite(this)._1
+  }
+
+  def rewriteAttrs(attrMap: AttributeMap[Attribute]): PlanType = {
+    transformExpressions {
+      case a: AttributeReference =>
+        updateAttr(a, attrMap)
+      case pe: PlanExpression[PlanType] =>
+        pe.withNewPlan(updateOuterReferencesInSubquery(pe.plan, attrMap))
+    }.asInstanceOf[PlanType]
+  }
+
+  private def updateAttr(a: Attribute, attrMap: AttributeMap[Attribute]): Attribute = {
+    attrMap.get(a) match {
+      case Some(b) =>
+        AttributeReference(a.name, b.dataType, b.nullable, a.metadata)(b.exprId, a.qualifier)
+      case None => a
+    }
+  }
+
+  /**
+   * The outer plan may have old references and the function below updates the
+   * outer references to refer to the new attributes.
+   */
+  protected def updateOuterReferencesInSubquery(
+      plan: PlanType,
+      attrMap: AttributeMap[Attribute]): PlanType = {
+    plan.transformDown { case currentFragment =>
+      currentFragment.transformExpressions {
+        case OuterReference(a: AttributeReference) =>
+          OuterReference(updateAttr(a, attrMap))
+        case pe: PlanExpression[PlanType] =>
+          pe.withNewPlan(updateOuterReferencesInSubquery(pe.plan, attrMap))
+      }
+    }
+  }
+
   lazy val schema: StructType = StructType.fromAttributes(output)
 
   /** Returns the output schema in the tree format. */
@@ -195,7 +395,7 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT
   }
 
   def verboseStringWithOperatorId(): String = {
-    val argumentString = argString(SQLConf.get.maxToStringFields)
+    val argumentString = argString(conf.maxToStringFields)
 
     if (argumentString.nonEmpty) {
       s"""
@@ -291,7 +491,7 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT
 
       case ar: AttributeReference if allAttributes.indexOf(ar.exprId) == -1 =>
         // Top level `AttributeReference` may also be used for output like `Alias`, we should
-        // normalize the epxrId too.
+        // normalize the exprId too.
         id += 1
         ar.withExprId(ExprId(id)).canonicalized
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/joinTypes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/joinTypes.scala
index feea1d2177ef0..da3cfb4c9de07 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/joinTypes.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/joinTypes.scala
@@ -102,7 +102,7 @@ case class NaturalJoin(tpe: JoinType) extends JoinType {
 }
 
 case class UsingJoin(tpe: JoinType, usingColumns: Seq[String]) extends JoinType {
-  require(Seq(Inner, LeftOuter, LeftSemi, RightOuter, FullOuter, LeftAnti).contains(tpe),
+  require(Seq(Inner, LeftOuter, LeftSemi, RightOuter, FullOuter, LeftAnti, Cross).contains(tpe),
     "Unsupported using join type " + tpe)
   override def sql: String = "USING " + tpe.sql
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AnalysisHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AnalysisHelper.scala
index 9404a809b453c..9e9bc69a50e00 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AnalysisHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AnalysisHelper.scala
@@ -17,10 +17,11 @@
 
 package org.apache.spark.sql.catalyst.plans.logical
 
-import org.apache.spark.sql.catalyst.analysis.CheckAnalysis
-import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeMap, Expression}
 import org.apache.spark.sql.catalyst.plans.QueryPlan
-import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, TreeNode}
+import org.apache.spark.sql.catalyst.rules.RuleId
+import org.apache.spark.sql.catalyst.rules.UnknownRuleId
+import org.apache.spark.sql.catalyst.trees.{AlwaysProcess, CurrentOrigin, TreePatternBits}
 import org.apache.spark.util.Utils
 
 
@@ -33,7 +34,7 @@ import org.apache.spark.util.Utils
  * analyzed flag set to true.
  *
  * The analyzer rules should use the various resolve methods, in lieu of the various transform
- * methods defined in [[TreeNode]] and [[QueryPlan]].
+ * methods defined in [[org.apache.spark.sql.catalyst.trees.TreeNode]] and [[QueryPlan]].
  *
  * To prevent accidental use of the transform methods, this trait also overrides the transform
  * methods to throw exceptions in test mode, if they are used in the analyzer.
@@ -44,9 +45,10 @@ trait AnalysisHelper extends QueryPlan[LogicalPlan] { self: LogicalPlan =>
 
   /**
    * Recursively marks all nodes in this plan tree as analyzed.
-   * This should only be called by [[CheckAnalysis]].
+   * This should only be called by
+   * [[org.apache.spark.sql.catalyst.analysis.CheckAnalysis]].
    */
-  private[catalyst] def setAnalyzed(): Unit = {
+  private[sql] def setAnalyzed(): Unit = {
     if (!_analyzed) {
       _analyzed = true
       children.foreach(_.setAnalyzed())
@@ -67,10 +69,33 @@ trait AnalysisHelper extends QueryPlan[LogicalPlan] { self: LogicalPlan =>
    * Users should not expect a specific directionality. If a specific directionality is needed,
    * [[resolveOperatorsUp]] or [[resolveOperatorsDown]] should be used.
    *
-   * @param rule the function use to transform this nodes children
+   * @param rule the function used to transform this nodes children.
    */
   def resolveOperators(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan = {
-    resolveOperatorsDown(rule)
+    resolveOperatorsWithPruning(AlwaysProcess.fn, UnknownRuleId)(rule)
+  }
+
+  /**
+   * Returns a copy of this node where `rule` has been recursively applied to the tree. When
+   * `rule` does not apply to a given node, it is left unchanged. This function is similar to
+   * `transform`, but skips sub-trees that have already been marked as analyzed.
+   * Users should not expect a specific directionality. If a specific directionality is needed,
+   * [[resolveOperatorsUpWithPruning]] or [[resolveOperatorsDownWithPruning]] should be used.
+   *
+   * @param rule   the function used to transform this nodes children.
+   * @param cond   a Lambda expression to prune tree traversals. If `cond.apply` returns false
+   *               on an operator T, skips processing T and its subtree; otherwise, processes
+   *               T and its subtree recursively.
+   * @param ruleId is a unique Id for `rule` to prune unnecessary tree traversals. When it is
+   *               UnknownRuleId, no pruning happens. Otherwise, if `rule` (with id `ruleId`)
+   *               has been marked as in effective on an operator T, skips processing T and its
+   *               subtree. Do not pass it if the rule is not purely functional and reads a
+   *               varying initial state for different invocations.
+   */
+  def resolveOperatorsWithPruning(cond: TreePatternBits => Boolean,
+    ruleId: RuleId = UnknownRuleId)(rule: PartialFunction[LogicalPlan, LogicalPlan])
+  : LogicalPlan = {
+    resolveOperatorsDownWithPruning(cond, ruleId)(rule)
   }
 
   /**
@@ -79,13 +104,35 @@ trait AnalysisHelper extends QueryPlan[LogicalPlan] { self: LogicalPlan =>
    * it is left unchanged.  This function is similar to `transformUp`, but skips sub-trees that
    * have already been marked as analyzed.
    *
-   * @param rule the function use to transform this nodes children
+   * @param rule the function used to transform this nodes children.
    */
   def resolveOperatorsUp(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan = {
-    if (!analyzed) {
+    resolveOperatorsUpWithPruning(AlwaysProcess.fn, UnknownRuleId)(rule)
+  }
+
+  /**
+   * Returns a copy of this node where `rule` has been recursively applied first to all of its
+   * children and then itself (post-order, bottom-up). When `rule` does not apply to a given node,
+   * it is left unchanged.  This function is similar to `transformUp`, but skips sub-trees that
+   * have already been marked as analyzed.
+   *
+   * @param rule   the function used to transform this nodes children.
+   * @param cond   a Lambda expression to prune tree traversals. If `cond.apply` returns false
+   *               on an operator T, skips processing T and its subtree; otherwise, processes
+   *               T and its subtree recursively.
+   * @param ruleId is a unique Id for `rule` to prune unnecessary tree traversals. When it is
+   *               UnknownRuleId, no pruning happens. Otherwise, if `rule` (with id `ruleId`)
+   *               has been marked as in effective on an operator T, skips processing T and its
+   *               subtree. Do not pass it if the rule is not purely functional and reads a
+   *               varying initial state for different invocations.
+   */
+  def resolveOperatorsUpWithPruning(cond: TreePatternBits => Boolean,
+    ruleId: RuleId = UnknownRuleId)(rule: PartialFunction[LogicalPlan, LogicalPlan])
+  : LogicalPlan = {
+    if (!analyzed && cond.apply(self) && !isRuleIneffective(ruleId)) {
       AnalysisHelper.allowInvokingTransformsInAnalyzer {
-        val afterRuleOnChildren = mapChildren(_.resolveOperatorsUp(rule))
-        if (self fastEquals afterRuleOnChildren) {
+        val afterRuleOnChildren = mapChildren(_.resolveOperatorsUpWithPruning(cond, ruleId)(rule))
+        val afterRule = if (self fastEquals afterRuleOnChildren) {
           CurrentOrigin.withOrigin(origin) {
             rule.applyOrElse(self, identity[LogicalPlan])
           }
@@ -94,6 +141,13 @@ trait AnalysisHelper extends QueryPlan[LogicalPlan] { self: LogicalPlan =>
             rule.applyOrElse(afterRuleOnChildren, identity[LogicalPlan])
           }
         }
+        if (self eq afterRule) {
+          self.markRuleAsIneffective(ruleId)
+          self
+        } else {
+          afterRule.copyTagsFrom(self)
+          afterRule
+        }
       }
     } else {
       self
@@ -102,7 +156,14 @@ trait AnalysisHelper extends QueryPlan[LogicalPlan] { self: LogicalPlan =>
 
   /** Similar to [[resolveOperatorsUp]], but does it top-down. */
   def resolveOperatorsDown(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan = {
-    if (!analyzed) {
+    resolveOperatorsDownWithPruning(AlwaysProcess.fn, UnknownRuleId)(rule)
+  }
+
+  /** Similar to [[resolveOperatorsUpWithPruning]], but does it top-down. */
+  def resolveOperatorsDownWithPruning(cond: TreePatternBits => Boolean,
+    ruleId: RuleId = UnknownRuleId)(rule: PartialFunction[LogicalPlan, LogicalPlan])
+  : LogicalPlan = {
+    if (!analyzed && cond.apply(self) && !isRuleIneffective(ruleId)) {
       AnalysisHelper.allowInvokingTransformsInAnalyzer {
         val afterRule = CurrentOrigin.withOrigin(origin) {
           rule.applyOrElse(self, identity[LogicalPlan])
@@ -110,9 +171,15 @@ trait AnalysisHelper extends QueryPlan[LogicalPlan] { self: LogicalPlan =>
 
         // Check if unchanged and then possibly return old copy to avoid gc churn.
         if (self fastEquals afterRule) {
-          mapChildren(_.resolveOperatorsDown(rule))
+          val rewritten_plan = mapChildren(_.resolveOperatorsDownWithPruning(cond, ruleId)(rule))
+          if (self eq rewritten_plan) {
+            self.markRuleAsIneffective(ruleId)
+            self
+          } else {
+            rewritten_plan
+          }
         } else {
-          afterRule.mapChildren(_.resolveOperatorsDown(rule))
+          afterRule.mapChildren(_.resolveOperatorsDownWithPruning(cond, ruleId)(rule))
         }
       }
     } else {
@@ -120,13 +187,61 @@ trait AnalysisHelper extends QueryPlan[LogicalPlan] { self: LogicalPlan =>
     }
   }
 
+  /**
+   * A variant of `transformUpWithNewOutput`, which skips touching already analyzed plan.
+   */
+  def resolveOperatorsUpWithNewOutput(
+      rule: PartialFunction[LogicalPlan, (LogicalPlan, Seq[(Attribute, Attribute)])])
+  : LogicalPlan = {
+    if (!analyzed) {
+      transformUpWithNewOutput(rule, skipCond = _.analyzed, canGetOutput = _.resolved)
+    } else {
+      self
+    }
+  }
+
+  override def transformUpWithNewOutput(
+      rule: PartialFunction[LogicalPlan, (LogicalPlan, Seq[(Attribute, Attribute)])],
+      skipCond: LogicalPlan => Boolean,
+      canGetOutput: LogicalPlan => Boolean): LogicalPlan = {
+    AnalysisHelper.allowInvokingTransformsInAnalyzer {
+      super.transformUpWithNewOutput(rule, skipCond, canGetOutput)
+    }
+  }
+
+  override def updateOuterReferencesInSubquery(plan: LogicalPlan, attrMap: AttributeMap[Attribute])
+    : LogicalPlan = {
+    AnalysisHelper.allowInvokingTransformsInAnalyzer {
+      super.updateOuterReferencesInSubquery(plan, attrMap)
+    }
+  }
+
   /**
    * Recursively transforms the expressions of a tree, skipping nodes that have already
    * been analyzed.
    */
   def resolveExpressions(r: PartialFunction[Expression, Expression]): LogicalPlan = {
-    resolveOperators  {
-      case p => p.transformExpressions(r)
+    resolveExpressionsWithPruning(AlwaysProcess.fn, UnknownRuleId)(r)
+  }
+
+  /**
+   * Recursively transforms the expressions of a tree, skipping nodes that have already
+   * been analyzed.
+   *
+   * @param rule   the function used to transform this nodes children.
+   * @param cond   a Lambda expression to prune tree traversals. If `cond.apply` returns false
+   *               on a TreeNode T, skips processing T and its subtree; otherwise, processes
+   *               T and its subtree recursively.
+   * @param ruleId is a unique Id for `rule` to prune unnecessary tree traversals. When it is
+   *               UnknownRuleId, no pruning happens. Otherwise, if `rule` (with id `ruleId`)
+   *               has been marked as in effective on a TreeNode T, skips processing T and its
+   *               subtree. Do not pass it if the rule is not purely functional and reads a
+   *               varying initial state for different invocations.
+   */
+  def resolveExpressionsWithPruning(cond: TreePatternBits => Boolean,
+    ruleId: RuleId = UnknownRuleId)(rule: PartialFunction[Expression, Expression]): LogicalPlan = {
+    resolveOperatorsWithPruning(cond, ruleId) {
+      case p => p.transformExpressionsWithPruning(cond, ruleId)(rule)
     }
   }
 
@@ -142,31 +257,44 @@ trait AnalysisHelper extends QueryPlan[LogicalPlan] { self: LogicalPlan =>
    * In analyzer, use [[resolveOperatorsDown()]] instead. If this is used in the analyzer,
    * an exception will be thrown in test mode. It is however OK to call this function within
    * the scope of a [[resolveOperatorsDown()]] call.
-   * @see [[TreeNode.transformDown()]].
+   * @see [[org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning()]].
    */
-  override def transformDown(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan = {
+  override def transformDownWithPruning(cond: TreePatternBits => Boolean,
+    ruleId: RuleId = UnknownRuleId)(rule: PartialFunction[LogicalPlan, LogicalPlan])
+  : LogicalPlan = {
     assertNotAnalysisRule()
-    super.transformDown(rule)
+    super.transformDownWithPruning(cond, ruleId)(rule)
   }
 
   /**
    * Use [[resolveOperators()]] in the analyzer.
-   * @see [[TreeNode.transformUp()]]
+   *
+   * @see [[org.apache.spark.sql.catalyst.trees.TreeNode.transformUpWithPruning()]]
    */
-  override def transformUp(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan = {
+  override def transformUpWithPruning(cond: TreePatternBits => Boolean,
+    ruleId: RuleId = UnknownRuleId)(rule: PartialFunction[LogicalPlan, LogicalPlan])
+  : LogicalPlan = {
     assertNotAnalysisRule()
-    super.transformUp(rule)
+    super.transformUpWithPruning(cond, ruleId)(rule)
   }
 
   /**
    * Use [[resolveExpressions()]] in the analyzer.
-   * @see [[QueryPlan.transformAllExpressions()]]
+   * @see [[QueryPlan.transformAllExpressionsWithPruning()]]
    */
-  override def transformAllExpressions(rule: PartialFunction[Expression, Expression]): this.type = {
+  override def transformAllExpressionsWithPruning(
+    cond: TreePatternBits => Boolean,
+    ruleId: RuleId = UnknownRuleId)(rule: PartialFunction[Expression, Expression])
+  : this.type = {
     assertNotAnalysisRule()
-    super.transformAllExpressions(rule)
+    super.transformAllExpressionsWithPruning(cond, ruleId)(rule)
   }
 
+  override def clone(): LogicalPlan = {
+    val cloned = super.clone()
+    if (analyzed) cloned.setAnalyzed()
+    cloned
+  }
 }
 
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Command.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Command.scala
index 732c8ce2b5d98..5af1e9c1bb8ea 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Command.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Command.scala
@@ -17,7 +17,9 @@
 
 package org.apache.spark.sql.catalyst.plans.logical
 
-import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet}
+import org.apache.spark.sql.catalyst.plans.QueryPlan
+import org.apache.spark.sql.catalyst.trees.{BinaryLike, LeafLike, UnaryLike}
 
 /**
  * A logical node that represents a non-query command to be executed by the system.  For example,
@@ -26,9 +28,25 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
  */
 trait Command extends LogicalPlan {
   override def output: Seq[Attribute] = Seq.empty
-  override def children: Seq[LogicalPlan] = Seq.empty
+  override def producedAttributes: AttributeSet = outputSet
   // Commands are eagerly executed. They will be converted to LocalRelation after the DataFrame
   // is created. That said, the statistics of a command is useless. Here we just return a dummy
   // statistics to avoid unnecessary statistics calculation of command's children.
   override def stats: Statistics = Statistics.DUMMY
 }
+
+trait LeafCommand extends Command with LeafLike[LogicalPlan]
+trait UnaryCommand extends Command with UnaryLike[LogicalPlan]
+trait BinaryCommand extends Command with BinaryLike[LogicalPlan]
+
+/**
+ * A logical node that can be used for a command that requires its children to be only analyzed,
+ * but not optimized.
+ */
+trait AnalysisOnlyCommand extends Command {
+  val isAnalyzed: Boolean
+  def childrenToAnalyze: Seq[LogicalPlan]
+  override final def children: Seq[LogicalPlan] = if (isAnalyzed) Nil else childrenToAnalyze
+  override def innerChildren: Seq[QueryPlan[_]] = if (isAnalyzed) childrenToAnalyze else Nil
+  def markAsAnalyzed(): LogicalPlan
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/EventTimeWatermark.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/EventTimeWatermark.scala
index b6bf7cd85d472..bf3f93de97f8c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/EventTimeWatermark.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/EventTimeWatermark.scala
@@ -61,4 +61,7 @@ case class EventTimeWatermark(
       a
     }
   }
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): EventTimeWatermark =
+    copy(child = newChild)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LocalRelation.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LocalRelation.scala
index 8c4828a4cef23..e99f433fa6ab9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LocalRelation.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LocalRelation.scala
@@ -22,6 +22,7 @@ import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
 import org.apache.spark.sql.catalyst.analysis
 import org.apache.spark.sql.catalyst.expressions.{Attribute, Literal}
 import org.apache.spark.sql.catalyst.plans.logical.statsEstimation.EstimationUtils
+import org.apache.spark.sql.catalyst.trees.TreePattern.{LOCAL_RELATION, TreePattern}
 import org.apache.spark.sql.types.{StructField, StructType}
 
 object LocalRelation {
@@ -91,4 +92,8 @@ case class LocalRelation(
       " AS " + inlineTableName +
       output.map(_.name).mkString("(", ", ", ")")
   }
+
+  override def maxRows: Option[Long] = Some(data.length.toLong)
+
+  override val nodePatterns: Seq[TreePattern] = Seq(LOCAL_RELATION)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
index ec9bf90247f88..2b3cb3af38467 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
@@ -23,6 +23,8 @@ import org.apache.spark.sql.catalyst.analysis._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.QueryPlan
 import org.apache.spark.sql.catalyst.plans.logical.statsEstimation.LogicalPlanStats
+import org.apache.spark.sql.catalyst.trees.{BinaryLike, LeafLike, UnaryLike}
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.types.StructType
 
 
@@ -33,6 +35,12 @@ abstract class LogicalPlan
   with QueryPlanConstraints
   with Logging {
 
+  /**
+   * Metadata fields that can be projected from this node.
+   * Should be overridden if the plan does not propagate its children's output.
+   */
+  def metadataOutput: Seq[Attribute] = children.flatMap(_.metadataOutput)
+
   /** Returns true if this subtree has data from a streaming data source. */
   def isStreaming: Boolean = children.exists(_.isStreaming)
 
@@ -78,7 +86,7 @@ abstract class LogicalPlan
     schema.map { field =>
       resolve(field.name :: Nil, resolver).map {
         case a: AttributeReference => a
-        case _ => sys.error(s"can not handle nested schema yet...  plan $this")
+        case _ => throw QueryExecutionErrors.resolveCannotHandleNestedSchema(this)
       }.getOrElse {
         throw new AnalysisException(
           s"Unable to resolve ${field.name} given [${output.map(_.name).mkString(", ")}]")
@@ -88,8 +96,12 @@ abstract class LogicalPlan
 
   private[this] lazy val childAttributes = AttributeSeq(children.flatMap(_.output))
 
+  private[this] lazy val childMetadataAttributes = AttributeSeq(children.flatMap(_.metadataOutput))
+
   private[this] lazy val outputAttributes = AttributeSeq(output)
 
+  private[this] lazy val outputMetadataAttributes = AttributeSeq(metadataOutput)
+
   /**
    * Optionally resolves the given strings to a [[NamedExpression]] using the input from all child
    * nodes of this LogicalPlan. The attribute is expressed as
@@ -99,6 +111,7 @@ abstract class LogicalPlan
       nameParts: Seq[String],
       resolver: Resolver): Option[NamedExpression] =
     childAttributes.resolve(nameParts, resolver)
+      .orElse(childMetadataAttributes.resolve(nameParts, resolver))
 
   /**
    * Optionally resolves the given strings to a [[NamedExpression]] based on the output of this
@@ -109,6 +122,7 @@ abstract class LogicalPlan
       nameParts: Seq[String],
       resolver: Resolver): Option[NamedExpression] =
     outputAttributes.resolve(nameParts, resolver)
+      .orElse(outputMetadataAttributes.resolve(nameParts, resolver))
 
   /**
    * Given an attribute name, split it to name parts by dot, but
@@ -118,7 +132,7 @@ abstract class LogicalPlan
   def resolveQuoted(
       name: String,
       resolver: Resolver): Option[NamedExpression] = {
-    outputAttributes.resolve(UnresolvedAttribute.parseAttributeName(name), resolver)
+    resolve(UnresolvedAttribute.parseAttributeName(name), resolver)
   }
 
   /**
@@ -132,7 +146,7 @@ abstract class LogicalPlan
   def outputOrdering: Seq[SortOrder] = Nil
 
   /**
-   * Returns true iff `other`'s output is semantically the same, ie.:
+   * Returns true iff `other`'s output is semantically the same, i.e.:
    *  - it contains the same number of `Attribute`s;
    *  - references are the same;
    *  - the order is equal too.
@@ -149,8 +163,7 @@ abstract class LogicalPlan
 /**
  * A logical plan node with no children.
  */
-abstract class LeafNode extends LogicalPlan {
-  override final def children: Seq[LogicalPlan] = Nil
+trait LeafNode extends LogicalPlan with LeafLike[LogicalPlan] {
   override def producedAttributes: AttributeSet = outputSet
 
   /** Leaf nodes that can survive analysis must define their own statistics. */
@@ -160,17 +173,13 @@ abstract class LeafNode extends LogicalPlan {
 /**
  * A logical plan node with single child.
  */
-abstract class UnaryNode extends LogicalPlan {
-  def child: LogicalPlan
-
-  override final def children: Seq[LogicalPlan] = child :: Nil
-
+trait UnaryNode extends LogicalPlan with UnaryLike[LogicalPlan] {
   /**
    * Generates all valid constraints including an set of aliased constraints by replacing the
    * original constraint expressions with the corresponding alias
    */
-  protected def getAllValidConstraints(projectList: Seq[NamedExpression]): Set[Expression] = {
-    var allConstraints = child.constraints.asInstanceOf[Set[Expression]]
+  protected def getAllValidConstraints(projectList: Seq[NamedExpression]): ExpressionSet = {
+    var allConstraints = child.constraints
     projectList.foreach {
       case a @ Alias(l: Literal, _) =>
         allConstraints += EqualNullSafe(a.toAttribute, l)
@@ -187,19 +196,84 @@ abstract class UnaryNode extends LogicalPlan {
     allConstraints
   }
 
-  override protected lazy val validConstraints: Set[Expression] = child.constraints
+  override protected lazy val validConstraints: ExpressionSet = child.constraints
 }
 
 /**
  * A logical plan node with a left and right child.
  */
-abstract class BinaryNode extends LogicalPlan {
-  def left: LogicalPlan
-  def right: LogicalPlan
-
-  override final def children: Seq[LogicalPlan] = Seq(left, right)
-}
+trait BinaryNode extends LogicalPlan with BinaryLike[LogicalPlan]
 
 abstract class OrderPreservingUnaryNode extends UnaryNode {
   override final def outputOrdering: Seq[SortOrder] = child.outputOrdering
 }
+
+object LogicalPlanIntegrity {
+
+  private def canGetOutputAttrs(p: LogicalPlan): Boolean = {
+    p.resolved && !p.expressions.exists { e =>
+      e.collectFirst {
+        // We cannot call `output` in plans with a `ScalarSubquery` expr having no column,
+        // so, we filter out them in advance.
+        case s: ScalarSubquery if s.plan.schema.fields.isEmpty => true
+      }.isDefined
+    }
+  }
+
+  /**
+   * Since some logical plans (e.g., `Union`) can build `AttributeReference`s in their `output`,
+   * this method checks if the same `ExprId` refers to attributes having the same data type
+   * in plan output.
+   */
+  def hasUniqueExprIdsForOutput(plan: LogicalPlan): Boolean = {
+    val exprIds = plan.collect { case p if canGetOutputAttrs(p) =>
+      // NOTE: we still need to filter resolved expressions here because the output of
+      // some resolved logical plans can have unresolved references,
+      // e.g., outer references in `ExistenceJoin`.
+      p.output.filter(_.resolved).map { a => (a.exprId, a.dataType.asNullable) }
+    }.flatten
+
+    val ignoredExprIds = plan.collect {
+      // NOTE: `Union` currently reuses input `ExprId`s for output references, but we cannot
+      // simply modify the code for assigning new `ExprId`s in `Union#output` because
+      // the modification will make breaking changes (See SPARK-32741(#29585)).
+      // So, this check just ignores the `exprId`s of `Union` output.
+      case u: Union if u.resolved => u.output.map(_.exprId)
+    }.flatten.toSet
+
+    val groupedDataTypesByExprId = exprIds.filterNot { case (exprId, _) =>
+      ignoredExprIds.contains(exprId)
+    }.groupBy(_._1).values.map(_.distinct)
+
+    groupedDataTypesByExprId.forall(_.length == 1)
+  }
+
+  /**
+   * This method checks if reference `ExprId`s are not reused when assigning a new `ExprId`.
+   * For example, it returns false if plan transformers create an alias having the same `ExprId`
+   * with one of reference attributes, e.g., `a#1 + 1 AS a#1`.
+   */
+  def checkIfSameExprIdNotReused(plan: LogicalPlan): Boolean = {
+    plan.collect { case p if p.resolved =>
+      p.expressions.forall {
+        case a: Alias =>
+          // Even if a plan is resolved, `a.references` can return unresolved references,
+          // e.g., in `Grouping`/`GroupingID`, so we need to filter out them and
+          // check if the same `exprId` in `Alias` does not exist
+          // among reference `exprId`s.
+          !a.references.filter(_.resolved).map(_.exprId).exists(_ == a.exprId)
+        case _ =>
+          true
+      }
+    }.forall(identity)
+  }
+
+  /**
+   * This method checks if the same `ExprId` refers to an unique attribute in a plan tree.
+   * Some plan transformers (e.g., `RemoveNoopOperators`) rewrite logical
+   * plans based on this assumption.
+   */
+  def checkIfExprIdsAreGloballyUnique(plan: LogicalPlan): Boolean = {
+    checkIfSameExprIdNotReused(plan) && hasUniqueExprIdsForOutput(plan)
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlanVisitor.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlanVisitor.scala
index 18baced8f3d61..20398dd40b209 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlanVisitor.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlanVisitor.scala
@@ -41,6 +41,8 @@ trait LogicalPlanVisitor[T] {
     case p: ScriptTransformation => visitScriptTransform(p)
     case p: Union => visitUnion(p)
     case p: Window => visitWindow(p)
+    case p: Tail => visitTail(p)
+    case p: Sort => visitSort(p)
     case p: LogicalPlan => default(p)
   }
 
@@ -81,4 +83,8 @@ trait LogicalPlanVisitor[T] {
   def visitUnion(p: Union): T
 
   def visitWindow(p: Window): T
+
+  def visitTail(p: Tail): T
+
+  def visitSort(sort: Sort): T
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/PlanHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/PlanHelper.scala
index 63348f766a5b1..5ec488efc328c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/PlanHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/PlanHelper.scala
@@ -29,7 +29,7 @@ object PlanHelper {
   /**
    * Check if there's any expression in this query plan operator that is
    * - A WindowExpression but the plan is not Window
-   * - An AggregateExpresion but the plan is not Aggregate or Window
+   * - An AggregateExpression but the plan is not Aggregate or Window
    * - A Generator but the plan is not Generate
    * Returns the list of invalid expressions that this operator hosts. This can happen when
    * 1. The input query from users contain invalid expressions.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/QueryPlanConstraints.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/QueryPlanConstraints.scala
index 4c4ec000d0930..c4243da7b9e4b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/QueryPlanConstraints.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/QueryPlanConstraints.scala
@@ -29,16 +29,14 @@ trait QueryPlanConstraints extends ConstraintHelper { self: LogicalPlan =>
    */
   lazy val constraints: ExpressionSet = {
     if (conf.constraintPropagationEnabled) {
-      ExpressionSet(
-        validConstraints
-          .union(inferAdditionalConstraints(validConstraints))
-          .union(constructIsNotNullConstraints(validConstraints, output))
-          .filter { c =>
-            c.references.nonEmpty && c.references.subsetOf(outputSet) && c.deterministic
-          }
-      )
+      validConstraints
+        .union(inferAdditionalConstraints(validConstraints))
+        .union(constructIsNotNullConstraints(validConstraints, output))
+        .filter { c =>
+          c.references.nonEmpty && c.references.subsetOf(outputSet) && c.deterministic
+        }
     } else {
-      ExpressionSet(Set.empty)
+      ExpressionSet()
     }
   }
 
@@ -50,7 +48,7 @@ trait QueryPlanConstraints extends ConstraintHelper { self: LogicalPlan =>
    *
    * See [[Canonicalize]] for more details.
    */
-  protected lazy val validConstraints: Set[Expression] = Set.empty
+  protected lazy val validConstraints: ExpressionSet = ExpressionSet()
 }
 
 trait ConstraintHelper {
@@ -60,8 +58,8 @@ trait ConstraintHelper {
    * For e.g., if an operator has constraints of the form (`a = 5`, `a = b`), this returns an
    * additional constraint of the form `b = 5`.
    */
-  def inferAdditionalConstraints(constraints: Set[Expression]): Set[Expression] = {
-    var inferredConstraints = Set.empty[Expression]
+  def inferAdditionalConstraints(constraints: ExpressionSet): ExpressionSet = {
+    var inferredConstraints = ExpressionSet()
     // IsNotNull should be constructed by `constructIsNotNullConstraints`.
     val predicates = constraints.filterNot(_.isInstanceOf[IsNotNull])
     predicates.foreach {
@@ -79,9 +77,9 @@ trait ConstraintHelper {
   }
 
   private def replaceConstraints(
-      constraints: Set[Expression],
+      constraints: ExpressionSet,
       source: Expression,
-      destination: Expression): Set[Expression] = constraints.map(_ transform {
+      destination: Expression): ExpressionSet = constraints.map(_ transform {
     case e: Expression if e.semanticEquals(source) => destination
   })
 
@@ -91,15 +89,15 @@ trait ConstraintHelper {
    * returns a constraint of the form `isNotNull(a)`
    */
   def constructIsNotNullConstraints(
-      constraints: Set[Expression],
-      output: Seq[Attribute]): Set[Expression] = {
+      constraints: ExpressionSet,
+      output: Seq[Attribute]): ExpressionSet = {
     // First, we propagate constraints from the null intolerant expressions.
-    var isNotNullConstraints: Set[Expression] = constraints.flatMap(inferIsNotNullConstraints)
+    var isNotNullConstraints = constraints.flatMap(inferIsNotNullConstraints(_))
 
     // Second, we infer additional constraints from non-nullable attributes that are part of the
     // operator's output
     val nonNullableAttributes = output.filterNot(_.nullable)
-    isNotNullConstraints ++= nonNullableAttributes.map(IsNotNull).toSet
+    isNotNullConstraints ++= nonNullableAttributes.map(IsNotNull)
 
     isNotNullConstraints -- constraints
   }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ScriptTransformation.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ScriptTransformation.scala
index 30bff884b2249..5fe5dc373718d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ScriptTransformation.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ScriptTransformation.scala
@@ -17,24 +17,25 @@
 
 package org.apache.spark.sql.catalyst.plans.logical
 
-import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, Expression}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet}
 
 /**
  * Transforms the input by forking and running the specified script.
  *
- * @param input the set of expression that should be passed to the script.
  * @param script the command that should be executed.
  * @param output the attributes that are produced by the script.
  * @param ioschema the input and output schema applied in the execution of the script.
  */
 case class ScriptTransformation(
-    input: Seq[Expression],
     script: String,
     output: Seq[Attribute],
     child: LogicalPlan,
     ioschema: ScriptInputOutputSchema) extends UnaryNode {
   @transient
-  override lazy val references: AttributeSet = AttributeSet(input.flatMap(_.references))
+  override lazy val references: AttributeSet = AttributeSet(child.output)
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): ScriptTransformation =
+    copy(child = newChild)
 }
 
 /**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Statistics.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Statistics.scala
index 49f89bed154bb..7db3ee5f7543a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Statistics.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Statistics.scala
@@ -20,18 +20,10 @@ package org.apache.spark.sql.catalyst.plans.logical
 import java.io.{ByteArrayInputStream, ByteArrayOutputStream, DataInputStream, DataOutputStream}
 import java.math.{MathContext, RoundingMode}
 
-import scala.util.control.NonFatal
-
 import net.jpountz.lz4.{LZ4BlockInputStream, LZ4BlockOutputStream}
 
-import org.apache.spark.internal.Logging
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.catalog.CatalogColumnStat
 import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.expressions.aggregate._
-import org.apache.spark.sql.catalyst.util.{ArrayData, DateTimeUtils}
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.util.Utils
 
@@ -55,11 +47,14 @@ object Statistics {
  *                    defaults to the product of children's `sizeInBytes`.
  * @param rowCount Estimated number of rows.
  * @param attributeStats Statistics for Attributes.
+ * @param isRuntime Whether the statistics is inferred from query stage runtime statistics during
+ *                  adaptive query execution.
  */
 case class Statistics(
     sizeInBytes: BigInt,
     rowCount: Option[BigInt] = None,
-    attributeStats: AttributeMap[ColumnStat] = AttributeMap(Nil)) {
+    attributeStats: AttributeMap[ColumnStat] = AttributeMap(Nil),
+    isRuntime: Boolean = false) {
 
   override def toString: String = "Statistics(" + simpleString + ")"
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
index 79a8380826ab3..88a58fda1fa6a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
@@ -17,29 +17,34 @@
 
 package org.apache.spark.sql.catalyst.plans.logical
 
-import scala.collection.mutable
-
 import org.apache.spark.sql.catalyst.AliasIdentifier
-import org.apache.spark.sql.catalyst.analysis.{MultiInstanceRelation}
+import org.apache.spark.sql.catalyst.analysis.{AnsiTypeCoercion, MultiInstanceRelation, TypeCoercion, TypeCoercionBase}
 import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable}
+import org.apache.spark.sql.catalyst.catalog.CatalogTable.VIEW_STORING_ANALYZED_PLAN
 import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, AggregateFunction}
+import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
 import org.apache.spark.sql.catalyst.plans._
-import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partitioning, RangePartitioning, RoundRobinPartitioning}
-import org.apache.spark.sql.catalyst.util.truncatedString
-import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partitioning, RangePartitioning, RoundRobinPartitioning, SinglePartition}
+import org.apache.spark.sql.catalyst.trees.TreeNodeTag
+import org.apache.spark.sql.catalyst.trees.TreePattern._
+import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.errors.QueryCompilationErrors
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.util.random.RandomSampler
 
 /**
- * When planning take() or collect() operations, this special node that is inserted at the top of
+ * When planning take() or collect() operations, this special node is inserted at the top of
  * the logical plan before invoking the query planner.
  *
  * Rules can pattern-match on this node in order to apply transformations that only take effect
  * at the top of the logical query plan.
  */
 case class ReturnAnswer(child: LogicalPlan) extends UnaryNode {
+  override def maxRows: Option[Long] = child.maxRows
   override def output: Seq[Attribute] = child.output
+  override protected def withNewChildInternal(newChild: LogicalPlan): ReturnAnswer =
+    copy(child = newChild)
 }
 
 /**
@@ -51,6 +56,8 @@ case class ReturnAnswer(child: LogicalPlan) extends UnaryNode {
  */
 case class Subquery(child: LogicalPlan, correlated: Boolean) extends OrderPreservingUnaryNode {
   override def output: Seq[Attribute] = child.output
+  override protected def withNewChildInternal(newChild: LogicalPlan): Subquery =
+    copy(child = newChild)
 }
 
 object Subquery {
@@ -62,6 +69,9 @@ case class Project(projectList: Seq[NamedExpression], child: LogicalPlan)
     extends OrderPreservingUnaryNode {
   override def output: Seq[Attribute] = projectList.map(_.toAttribute)
   override def maxRows: Option[Long] = child.maxRows
+  override def maxRowsPerPartition: Option[Long] = child.maxRowsPerPartition
+
+  final override val nodePatterns: Seq[TreePattern] = Seq(PROJECT)
 
   override lazy val resolved: Boolean = {
     val hasSpecialExpressions = projectList.exists ( _.collect {
@@ -74,8 +84,18 @@ case class Project(projectList: Seq[NamedExpression], child: LogicalPlan)
     !expressions.exists(!_.resolved) && childrenResolved && !hasSpecialExpressions
   }
 
-  override lazy val validConstraints: Set[Expression] =
+  override lazy val validConstraints: ExpressionSet =
     getAllValidConstraints(projectList)
+
+  override def metadataOutput: Seq[Attribute] =
+    getTagValue(Project.hiddenOutputTag).getOrElse(Nil)
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): Project =
+    copy(child = newChild)
+}
+
+object Project {
+  val hiddenOutputTag: TreeNodeTag[Seq[Attribute]] = TreeNodeTag[Seq[Attribute]]("hidden_output")
 }
 
 /**
@@ -107,6 +127,8 @@ case class Generate(
     child: LogicalPlan)
   extends UnaryNode {
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(GENERATE)
+
   lazy val requiredChildOutput: Seq[Attribute] = {
     val unrequiredSet = unrequiredChildIndex.toSet
     child.output.zipWithIndex.filterNot(t => unrequiredSet.contains(t._2)).map(_._1)
@@ -134,6 +156,9 @@ case class Generate(
   }
 
   def output: Seq[Attribute] = requiredChildOutput ++ qualifiedGeneratorOutput
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): Generate =
+    copy(child = newChild)
 }
 
 case class Filter(condition: Expression, child: LogicalPlan)
@@ -141,21 +166,27 @@ case class Filter(condition: Expression, child: LogicalPlan)
   override def output: Seq[Attribute] = child.output
 
   override def maxRows: Option[Long] = child.maxRows
+  override def maxRowsPerPartition: Option[Long] = child.maxRowsPerPartition
+
+  final override val nodePatterns: Seq[TreePattern] = Seq(FILTER)
 
-  override protected lazy val validConstraints: Set[Expression] = {
+  override protected lazy val validConstraints: ExpressionSet = {
     val predicates = splitConjunctivePredicates(condition)
       .filterNot(SubqueryExpression.hasCorrelatedSubquery)
-    child.constraints.union(predicates.toSet)
+    child.constraints.union(ExpressionSet(predicates))
   }
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): Filter =
+    copy(child = newChild)
 }
 
 abstract class SetOperation(left: LogicalPlan, right: LogicalPlan) extends BinaryNode {
 
   def duplicateResolved: Boolean = left.outputSet.intersect(right.outputSet).isEmpty
 
-  protected def leftConstraints: Set[Expression] = left.constraints
+  protected def leftConstraints: ExpressionSet = left.constraints
 
-  protected def rightConstraints: Set[Expression] = {
+  protected def rightConstraints: ExpressionSet = {
     require(left.output.size == right.output.size)
     val attributeRewrites = AttributeMap(right.output.zip(left.output))
     right.constraints.map(_ transform {
@@ -182,12 +213,16 @@ case class Intersect(
 
   override def nodeName: String = getClass.getSimpleName + ( if ( isAll ) "All" else "" )
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(INTERSECT)
+
   override def output: Seq[Attribute] =
     left.output.zip(right.output).map { case (leftAttr, rightAttr) =>
       leftAttr.withNullability(leftAttr.nullable && rightAttr.nullable)
     }
 
-  override protected lazy val validConstraints: Set[Expression] =
+  override def metadataOutput: Seq[Attribute] = Nil
+
+  override protected lazy val validConstraints: ExpressionSet =
     leftConstraints.union(rightConstraints)
 
   override def maxRows: Option[Long] = {
@@ -197,6 +232,9 @@ case class Intersect(
       Some(children.flatMap(_.maxRows).min)
     }
   }
+
+  override protected def withNewChildrenInternal(
+    newLeft: LogicalPlan, newRight: LogicalPlan): Intersect = copy(left = newLeft, right = newRight)
 }
 
 case class Except(
@@ -207,7 +245,14 @@ case class Except(
   /** We don't use right.output because those rows get excluded from the set. */
   override def output: Seq[Attribute] = left.output
 
-  override protected lazy val validConstraints: Set[Expression] = leftConstraints
+  override def metadataOutput: Seq[Attribute] = Nil
+
+  final override val nodePatterns : Seq[TreePattern] = Seq(EXCEPT)
+
+  override protected lazy val validConstraints: ExpressionSet = leftConstraints
+
+  override protected def withNewChildrenInternal(
+    newLeft: LogicalPlan, newRight: LogicalPlan): Except = copy(left = newLeft, right = newRight)
 }
 
 /** Factory for constructing new `Union` nodes. */
@@ -218,9 +263,19 @@ object Union {
 }
 
 /**
- * Logical plan for unioning two plans, without a distinct. This is UNION ALL in SQL.
+ * Logical plan for unioning multiple plans, without a distinct. This is UNION ALL in SQL.
+ *
+ * @param byName          Whether resolves columns in the children by column names.
+ * @param allowMissingCol Allows missing columns in children query plans. If it is true,
+ *                        this function allows different set of column names between two Datasets.
+ *                        This can be set to true only if `byName` is true.
  */
-case class Union(children: Seq[LogicalPlan]) extends LogicalPlan {
+case class Union(
+    children: Seq[LogicalPlan],
+    byName: Boolean = false,
+    allowMissingCol: Boolean = false) extends LogicalPlan {
+  assert(!allowMissingCol || byName, "`allowMissingCol` can be true only if `byName` is true.")
+
   override def maxRows: Option[Long] = {
     if (children.exists(_.maxRows.isEmpty)) {
       None
@@ -229,6 +284,8 @@ case class Union(children: Seq[LogicalPlan]) extends LogicalPlan {
     }
   }
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(UNION)
+
   /**
    * Note the definition has assumption about how union is implemented physically.
    */
@@ -260,6 +317,8 @@ case class Union(children: Seq[LogicalPlan]) extends LogicalPlan {
     }
   }
 
+  override def metadataOutput: Seq[Attribute] = Nil
+
   override lazy val resolved: Boolean = {
     // allChildrenCompatible needs to be evaluated after childrenResolved
     def allChildrenCompatible: Boolean =
@@ -270,7 +329,7 @@ case class Union(children: Seq[LogicalPlan]) extends LogicalPlan {
         child.output.zip(children.head.output).forall {
           case (l, r) => l.dataType.sameType(r.dataType)
         })
-    children.length > 1 && childrenResolved && allChildrenCompatible
+    children.length > 1 && !(byName || allowMissingCol) && childrenResolved && allChildrenCompatible
   }
 
   /**
@@ -281,7 +340,7 @@ case class Union(children: Seq[LogicalPlan]) extends LogicalPlan {
   private def rewriteConstraints(
       reference: Seq[Attribute],
       original: Seq[Attribute],
-      constraints: Set[Expression]): Set[Expression] = {
+      constraints: ExpressionSet): ExpressionSet = {
     require(reference.size == original.size)
     val attributeRewrites = AttributeMap(original.zip(reference))
     constraints.map(_ transform {
@@ -289,7 +348,7 @@ case class Union(children: Seq[LogicalPlan]) extends LogicalPlan {
     })
   }
 
-  private def merge(a: Set[Expression], b: Set[Expression]): Set[Expression] = {
+  private def merge(a: ExpressionSet, b: ExpressionSet): ExpressionSet = {
     val common = a.intersect(b)
     // The constraint with only one reference could be easily inferred as predicate
     // Grouping the constraints by it's references so we can combine the constraints with same
@@ -303,11 +362,14 @@ case class Union(children: Seq[LogicalPlan]) extends LogicalPlan {
     common ++ others
   }
 
-  override protected lazy val validConstraints: Set[Expression] = {
+  override protected lazy val validConstraints: ExpressionSet = {
     children
       .map(child => rewriteConstraints(children.head.output, child.output, child.constraints))
       .reduce(merge(_, _))
   }
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[LogicalPlan]): Union =
+    copy(children = newChildren)
 }
 
 case class Join(
@@ -318,6 +380,25 @@ case class Join(
     hint: JoinHint)
   extends BinaryNode with PredicateHelper {
 
+  override def maxRows: Option[Long] = {
+    joinType match {
+      case Inner | Cross | FullOuter | LeftOuter | RightOuter
+        if left.maxRows.isDefined && right.maxRows.isDefined =>
+        val maxRows = BigInt(left.maxRows.get) * BigInt(right.maxRows.get)
+        if (maxRows.isValidLong) {
+          Some(maxRows.toLong)
+        } else {
+          None
+        }
+
+      case LeftSemi | LeftAnti =>
+        left.maxRows
+
+      case _ =>
+        None
+    }
+  }
+
   override def output: Seq[Attribute] = {
     joinType match {
       case j: ExistenceJoin =>
@@ -335,15 +416,26 @@ case class Join(
     }
   }
 
-  override protected lazy val validConstraints: Set[Expression] = {
+  override def metadataOutput: Seq[Attribute] = {
+    joinType match {
+      case ExistenceJoin(_) =>
+        left.metadataOutput
+      case LeftExistence(_) =>
+        left.metadataOutput
+      case _ =>
+        children.flatMap(_.metadataOutput)
+    }
+  }
+
+  override protected lazy val validConstraints: ExpressionSet = {
     joinType match {
       case _: InnerLike if condition.isDefined =>
         left.constraints
           .union(right.constraints)
-          .union(splitConjunctivePredicates(condition.get).toSet)
+          .union(ExpressionSet(splitConjunctivePredicates(condition.get)))
       case LeftSemi if condition.isDefined =>
         left.constraints
-          .union(splitConjunctivePredicates(condition.get).toSet)
+          .union(ExpressionSet(splitConjunctivePredicates(condition.get)))
       case j: ExistenceJoin =>
         left.constraints
       case _: InnerLike =>
@@ -354,8 +446,8 @@ case class Join(
         left.constraints
       case RightOuter =>
         right.constraints
-      case FullOuter =>
-        Set.empty[Expression]
+      case _ =>
+        ExpressionSet()
     }
   }
 
@@ -378,6 +470,18 @@ case class Join(
     case _ => resolvedExceptNatural
   }
 
+  override val nodePatterns : Seq[TreePattern] = {
+    var patterns = Seq(JOIN)
+    joinType match {
+      case _: InnerLike => patterns = patterns :+ INNER_LIKE_JOIN
+      case LeftOuter | FullOuter | RightOuter => patterns = patterns :+ OUTER_JOIN
+      case LeftSemiOrAnti(_) => patterns = patterns :+ LEFT_SEMI_OR_ANTI_JOIN
+      case NaturalJoin(_) | UsingJoin(_, _) => patterns = patterns :+ NATURAL_LIKE_JOIN
+      case _ =>
+    }
+    patterns
+  }
+
   // Ignore hint for canonicalization
   protected override def doCanonicalize(): LogicalPlan =
     super.doCanonicalize().asInstanceOf[Join].copy(hint = JoinHint.NONE)
@@ -388,6 +492,9 @@ case class Join(
       || e.asInstanceOf[JoinHint].leftHint.isDefined
       || e.asInstanceOf[JoinHint].rightHint.isDefined)
   }
+
+  override protected def withNewChildrenInternal(
+    newLeft: LogicalPlan, newRight: LogicalPlan): Join = copy(left = newLeft, right = newRight)
 }
 
 /**
@@ -411,38 +518,82 @@ case class InsertIntoDir(
   extends UnaryNode {
 
   override def output: Seq[Attribute] = Seq.empty
+  override def metadataOutput: Seq[Attribute] = Nil
   override lazy val resolved: Boolean = false
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): InsertIntoDir =
+    copy(child = newChild)
 }
 
 /**
- * A container for holding the view description(CatalogTable), and the output of the view. The
- * child should be a logical plan parsed from the `CatalogTable.viewText`, should throw an error
- * if the `viewText` is not defined.
+ * A container for holding the view description(CatalogTable) and info whether the view is temporary
+ * or not. If it's a SQL (temp) view, the child should be a logical plan parsed from the
+ * `CatalogTable.viewText`. Otherwise, the view is a temporary one created from a dataframe and the
+ * view description should contain a `VIEW_CREATED_FROM_DATAFRAME` property; in this case, the child
+ * must be already resolved.
+ *
  * This operator will be removed at the end of analysis stage.
  *
  * @param desc A view description(CatalogTable) that provides necessary information to resolve the
  *             view.
- * @param output The output of a view operator, this is generated during planning the view, so that
- *               we are able to decouple the output from the underlying structure.
- * @param child The logical plan of a view operator, it should be a logical plan parsed from the
- *              `CatalogTable.viewText`, should throw an error if the `viewText` is not defined.
+ * @param isTempView A flag to indicate whether the view is temporary or not.
+ * @param child The logical plan of a view operator. If the view description is available, it should
+ *              be a logical plan parsed from the `CatalogTable.viewText`.
  */
 case class View(
     desc: CatalogTable,
-    output: Seq[Attribute],
-    child: LogicalPlan) extends LogicalPlan with MultiInstanceRelation {
-
-  override def producedAttributes: AttributeSet = outputSet
-
-  override lazy val resolved: Boolean = child.resolved
+    isTempView: Boolean,
+    child: LogicalPlan) extends UnaryNode {
+  require(!isTempViewStoringAnalyzedPlan || child.resolved)
 
-  override def children: Seq[LogicalPlan] = child :: Nil
+  override def output: Seq[Attribute] = child.output
 
-  override def newInstance(): LogicalPlan = copy(output = output.map(_.newInstance()))
+  override def metadataOutput: Seq[Attribute] = Nil
 
   override def simpleString(maxFields: Int): String = {
     s"View (${desc.identifier}, ${output.mkString("[", ",", "]")})"
   }
+
+  override def doCanonicalize(): LogicalPlan = child match {
+    case p: Project if p.resolved && canRemoveProject(p) => p.child.canonicalized
+    case _ => child.canonicalized
+  }
+
+  def isTempViewStoringAnalyzedPlan: Boolean =
+    isTempView && desc.properties.contains(VIEW_STORING_ANALYZED_PLAN)
+
+  // When resolving a SQL view, we use an extra Project to add cast and alias to make sure the view
+  // output schema doesn't change even if the table referenced by the view is changed after view
+  // creation. We should remove this extra Project during canonicalize if it does nothing.
+  // See more details in `SessionCatalog.fromCatalogTable`.
+  private def canRemoveProject(p: Project): Boolean = {
+    p.output.length == p.child.output.length && p.projectList.zip(p.child.output).forall {
+      case (Alias(cast: CastBase, name), childAttr) =>
+        cast.child match {
+          case a: AttributeReference =>
+            a.dataType == cast.dataType && a.name == name && childAttr.semanticEquals(a)
+          case _ => false
+        }
+      case _ => false
+    }
+  }
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): View =
+    copy(child = newChild)
+}
+
+object View {
+  def effectiveSQLConf(configs: Map[String, String], isTempView: Boolean): SQLConf = {
+    val activeConf = SQLConf.get
+    // For temporary view, we always use captured sql configs
+    if (activeConf.useCurrentSQLConfigsForView && !isTempView) return activeConf
+
+    val sqlConf = new SQLConf()
+    for ((k, v) <- configs) {
+      sqlConf.settings.put(k, v)
+    }
+    sqlConf
+  }
 }
 
 /**
@@ -462,12 +613,16 @@ case class With(child: LogicalPlan, cteRelations: Seq[(String, SubqueryAlias)])
   }
 
   override def innerChildren: Seq[LogicalPlan] = cteRelations.map(_._2)
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): With = copy(child = newChild)
 }
 
 case class WithWindowDefinition(
     windowDefinitions: Map[String, WindowSpecDefinition],
     child: LogicalPlan) extends UnaryNode {
   override def output: Seq[Attribute] = child.output
+  override protected def withNewChildInternal(newChild: LogicalPlan): WithWindowDefinition =
+    copy(child = newChild)
 }
 
 /**
@@ -483,31 +638,90 @@ case class Sort(
   override def output: Seq[Attribute] = child.output
   override def maxRows: Option[Long] = child.maxRows
   override def outputOrdering: Seq[SortOrder] = order
+  final override val nodePatterns: Seq[TreePattern] = Seq(SORT)
+  override protected def withNewChildInternal(newChild: LogicalPlan): Sort = copy(child = newChild)
 }
 
 /** Factory for constructing new `Range` nodes. */
 object Range {
-  def apply(start: Long, end: Long, step: Long,
-            numSlices: Option[Int], isStreaming: Boolean = false): Range = {
-    val output = StructType(StructField("id", LongType, nullable = false) :: Nil).toAttributes
-    new Range(start, end, step, numSlices, output, isStreaming)
-  }
   def apply(start: Long, end: Long, step: Long, numSlices: Int): Range = {
     Range(start, end, step, Some(numSlices))
   }
+
+  def getOutputAttrs: Seq[Attribute] = {
+    StructType(StructField("id", LongType, nullable = false) :: Nil).toAttributes
+  }
+
+  private def typeCoercion: TypeCoercionBase = {
+    if (SQLConf.get.ansiEnabled) AnsiTypeCoercion else TypeCoercion
+  }
+
+  private def castAndEval[T](expression: Expression, dataType: DataType): T = {
+    typeCoercion.implicitCast(expression, dataType)
+      .map(_.eval())
+      .filter(_ != null)
+      .getOrElse {
+        throw QueryCompilationErrors.incompatibleRangeInputDataTypeError(expression, dataType)
+      }.asInstanceOf[T]
+  }
+
+  def toLong(expression: Expression): Long = castAndEval[Long](expression, LongType)
+
+  def toInt(expression: Expression): Int = castAndEval[Int](expression, IntegerType)
 }
 
+@ExpressionDescription(
+  usage = """
+    _FUNC_(start: long, end: long, step: long, numSlices: integer)
+    _FUNC_(start: long, end: long, step: long)
+    _FUNC_(start: long, end: long)
+    _FUNC_(end: long)""",
+  examples = """
+    Examples:
+      > SELECT * FROM _FUNC_(1);
+        +---+
+        | id|
+        +---+
+        |  0|
+        +---+
+      > SELECT * FROM _FUNC_(0, 2);
+        +---+
+        |id |
+        +---+
+        |0  |
+        |1  |
+        +---+
+      > SELECT * FROM _FUNC_(0, 4, 2);
+        +---+
+        |id |
+        +---+
+        |0  |
+        |2  |
+        +---+
+  """,
+  since = "2.0.0",
+  group = "table_funcs")
 case class Range(
     start: Long,
     end: Long,
     step: Long,
     numSlices: Option[Int],
-    output: Seq[Attribute],
-    override val isStreaming: Boolean)
+    override val output: Seq[Attribute] = Range.getOutputAttrs,
+    override val isStreaming: Boolean = false)
   extends LeafNode with MultiInstanceRelation {
 
   require(step != 0, s"step ($step) cannot be 0")
 
+  def this(start: Expression, end: Expression, step: Expression, numSlices: Expression) =
+    this(Range.toLong(start), Range.toLong(end), Range.toLong(step), Some(Range.toInt(numSlices)))
+
+  def this(start: Expression, end: Expression, step: Expression) =
+    this(Range.toLong(start), Range.toLong(end), Range.toLong(step), None)
+
+  def this(start: Expression, end: Expression) = this(start, end, Literal.create(1L, LongType))
+
+  def this(end: Expression) = this(Literal.create(0L, LongType), end)
+
   val numElements: BigInt = {
     val safeStart = BigInt(start)
     val safeEnd = BigInt(end)
@@ -533,8 +747,46 @@ case class Range(
     s"Range ($start, $end, step=$step, splits=$numSlices)"
   }
 
+  override def maxRows: Option[Long] = {
+    if (numElements.isValidLong) {
+      Some(numElements.toLong)
+    } else {
+      None
+    }
+  }
+
+  override def maxRowsPerPartition: Option[Long] = {
+    if (numSlices.isDefined) {
+      var m = numElements / numSlices.get
+      if (numElements % numSlices.get != 0) m += 1
+      if (m.isValidLong) Some(m.toLong) else maxRows
+    } else {
+      maxRows
+    }
+  }
+
   override def computeStats(): Statistics = {
-    Statistics(sizeInBytes = LongType.defaultSize * numElements)
+    if (numElements == 0) {
+      Statistics(sizeInBytes = 0, rowCount = Some(0))
+    } else {
+      val (minVal, maxVal) = if (step > 0) {
+        (start, start + (numElements - 1) * step)
+      } else {
+        (start + (numElements - 1) * step, start)
+      }
+      val colStat = ColumnStat(
+        distinctCount = Some(numElements),
+        max = Some(maxVal),
+        min = Some(minVal),
+        nullCount = Some(0),
+        avgLen = Some(LongType.defaultSize),
+        maxLen = Some(LongType.defaultSize))
+
+      Statistics(
+        sizeInBytes = LongType.defaultSize * numElements,
+        rowCount = Some(numElements),
+        attributeStats = AttributeMap(Seq(output.head -> colStat)))
+    }
   }
 
   override def outputOrdering: Seq[SortOrder] = {
@@ -552,7 +804,7 @@ case class Range(
  *
  * @param groupingExpressions expressions for grouping keys
  * @param aggregateExpressions expressions for a project list, which could contain
- *                             [[AggregateFunction]]s.
+ *                             [[AggregateExpression]]s.
  *
  * Note: Currently, aggregateExpressions is the project list of this Group by operator. Before
  * separating projection from grouping and aggregate, we should avoid expression-level optimization
@@ -575,12 +827,24 @@ case class Aggregate(
   }
 
   override def output: Seq[Attribute] = aggregateExpressions.map(_.toAttribute)
-  override def maxRows: Option[Long] = child.maxRows
+  override def metadataOutput: Seq[Attribute] = Nil
+  override def maxRows: Option[Long] = {
+    if (groupingExpressions.isEmpty) {
+      Some(1L)
+    } else {
+      child.maxRows
+    }
+  }
 
-  override lazy val validConstraints: Set[Expression] = {
+  final override val nodePatterns : Seq[TreePattern] = Seq(AGGREGATE)
+
+  override lazy val validConstraints: ExpressionSet = {
     val nonAgg = aggregateExpressions.filter(_.find(_.isInstanceOf[AggregateExpression]).isEmpty)
     getAllValidConstraints(nonAgg)
   }
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): Aggregate =
+    copy(child = newChild)
 }
 
 case class Window(
@@ -588,11 +852,18 @@ case class Window(
     partitionSpec: Seq[Expression],
     orderSpec: Seq[SortOrder],
     child: LogicalPlan) extends UnaryNode {
-
+  override def maxRows: Option[Long] = child.maxRows
   override def output: Seq[Attribute] =
     child.output ++ windowExpressions.map(_.toAttribute)
 
+  override def producedAttributes: AttributeSet = windowOutputSet
+
+  final override val nodePatterns: Seq[TreePattern] = Seq(WINDOW)
+
   def windowOutputSet: AttributeSet = AttributeSet(windowExpressions.map(_.toAttribute))
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): Window =
+    copy(child = newChild)
 }
 
 object Expand {
@@ -612,7 +883,7 @@ object Expand {
     val numAttributes = attrMap.size
     assert(numAttributes <= GroupingID.dataType.defaultSize * 8)
     val mask = if (numAttributes != 64) (1L << numAttributes) - 1 else 0xFFFFFFFFFFFFFFFFL
-    // Calculate the attrbute masks of selected grouping set. For example, if we have GroupBy
+    // Calculate the attribute masks of selected grouping set. For example, if we have GroupBy
     // attributes (a, b, c, d), grouping set (a, c) will produce the following sequence:
     // (15, 7, 13), whose binary form is (1111, 0111, 1101)
     val masks = (mask +: groupingSetAttrs.map(attrMap).map(index =>
@@ -702,37 +973,16 @@ case class Expand(
   override lazy val references: AttributeSet =
     AttributeSet(projections.flatten.flatMap(_.references))
 
+  override def metadataOutput: Seq[Attribute] = Nil
+
   override def producedAttributes: AttributeSet = AttributeSet(output diff child.output)
 
   // This operator can reuse attributes (for example making them null when doing a roll up) so
   // the constraints of the child may no longer be valid.
-  override protected lazy val validConstraints: Set[Expression] = Set.empty[Expression]
-}
-
-/**
- * A GROUP BY clause with GROUPING SETS can generate a result set equivalent
- * to generated by a UNION ALL of multiple simple GROUP BY clauses.
- *
- * We will transform GROUPING SETS into logical plan Aggregate(.., Expand) in Analyzer
- *
- * @param selectedGroupByExprs A sequence of selected GroupBy expressions, all exprs should
- *                     exist in groupByExprs.
- * @param groupByExprs The Group By expressions candidates.
- * @param child        Child operator
- * @param aggregations The Aggregation expressions, those non selected group by expressions
- *                     will be considered as constant null if it appears in the expressions
- */
-case class GroupingSets(
-    selectedGroupByExprs: Seq[Seq[Expression]],
-    groupByExprs: Seq[Expression],
-    child: LogicalPlan,
-    aggregations: Seq[NamedExpression]) extends UnaryNode {
-
-  override def output: Seq[Attribute] = aggregations.map(_.toAttribute)
+  override protected lazy val validConstraints: ExpressionSet = ExpressionSet()
 
-  // Needs to be unresolved before its translated to Aggregate + Expand because output attributes
-  // will change in analysis.
-  override lazy val resolved: Boolean = false
+  override protected def withNewChildInternal(newChild: LogicalPlan): Expand =
+    copy(child = newChild)
 }
 
 /**
@@ -764,6 +1014,9 @@ case class Pivot(
     }
     groupByExprsOpt.getOrElse(Seq.empty).map(_.toAttribute) ++ pivotAgg
   }
+  override def metadataOutput: Seq[Attribute] = Nil
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): Pivot = copy(child = newChild)
 }
 
 /**
@@ -813,6 +1066,11 @@ case class GlobalLimit(limitExpr: Expression, child: LogicalPlan) extends OrderP
       case _ => None
     }
   }
+
+  final override val nodePatterns: Seq[TreePattern] = Seq(LIMIT)
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): GlobalLimit =
+    copy(child = newChild)
 }
 
 /**
@@ -830,6 +1088,11 @@ case class LocalLimit(limitExpr: Expression, child: LogicalPlan) extends OrderPr
       case _ => None
     }
   }
+
+  final override val nodePatterns: Seq[TreePattern] = Seq(LIMIT)
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): LocalLimit =
+    copy(child = newChild)
 }
 
 /**
@@ -850,6 +1113,8 @@ case class Tail(limitExpr: Expression, child: LogicalPlan) extends OrderPreservi
       case _ => None
     }
   }
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): Tail = copy(child = newChild)
 }
 
 /**
@@ -869,7 +1134,18 @@ case class SubqueryAlias(
     val qualifierList = identifier.qualifier :+ alias
     child.output.map(_.withQualifier(qualifierList))
   }
+
+  override def metadataOutput: Seq[Attribute] = {
+    val qualifierList = identifier.qualifier :+ alias
+    child.metadataOutput.map(_.withQualifier(qualifierList))
+  }
+
+  override def maxRows: Option[Long] = child.maxRows
+
   override def doCanonicalize(): LogicalPlan = child.canonicalized
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): SubqueryAlias =
+    copy(child = newChild)
 }
 
 object SubqueryAlias {
@@ -921,7 +1197,11 @@ case class Sample(
       s"Sampling fraction ($fraction) must be on interval [0, 1] without replacement")
   }
 
+  override def maxRows: Option[Long] = child.maxRows
   override def output: Seq[Attribute] = child.output
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): Sample =
+    copy(child = newChild)
 }
 
 /**
@@ -930,6 +1210,9 @@ case class Sample(
 case class Distinct(child: LogicalPlan) extends UnaryNode {
   override def maxRows: Option[Long] = child.maxRows
   override def output: Seq[Attribute] = child.output
+  final override val nodePatterns: Seq[TreePattern] = Seq(DISTINCT_LIKE)
+  override protected def withNewChildInternal(newChild: LogicalPlan): Distinct =
+    copy(child = newChild)
 }
 
 /**
@@ -938,7 +1221,10 @@ case class Distinct(child: LogicalPlan) extends UnaryNode {
 abstract class RepartitionOperation extends UnaryNode {
   def shuffle: Boolean
   def numPartitions: Int
+  override final def maxRows: Option[Long] = child.maxRows
   override def output: Seq[Attribute] = child.output
+  final override val nodePatterns: Seq[TreePattern] = Seq(REPARTITION_OPERATION)
+  def partitioning: Partitioning
 }
 
 /**
@@ -950,22 +1236,34 @@ abstract class RepartitionOperation extends UnaryNode {
 case class Repartition(numPartitions: Int, shuffle: Boolean, child: LogicalPlan)
   extends RepartitionOperation {
   require(numPartitions > 0, s"Number of partitions ($numPartitions) must be positive.")
+
+  override def partitioning: Partitioning = {
+    require(shuffle, "Partitioning can only be used in shuffle.")
+    numPartitions match {
+      case 1 => SinglePartition
+      case _ => RoundRobinPartitioning(numPartitions)
+    }
+  }
+  override protected def withNewChildInternal(newChild: LogicalPlan): Repartition =
+    copy(child = newChild)
 }
 
 /**
- * This method repartitions data using [[Expression]]s into `numPartitions`, and receives
+ * This method repartitions data using [[Expression]]s into `optNumPartitions`, and receives
  * information about the number of partitions during execution. Used when a specific ordering or
  * distribution is expected by the consumer of the query result. Use [[Repartition]] for RDD-like
- * `coalesce` and `repartition`.
+ * `coalesce` and `repartition`. If no `optNumPartitions` is given, by default it partitions data
+ * into `numShufflePartitions` defined in `SQLConf`, and could be coalesced by AQE.
  */
 case class RepartitionByExpression(
     partitionExpressions: Seq[Expression],
     child: LogicalPlan,
-    numPartitions: Int) extends RepartitionOperation {
+    optNumPartitions: Option[Int]) extends RepartitionOperation {
 
+  val numPartitions = optNumPartitions.getOrElse(conf.numShufflePartitions)
   require(numPartitions > 0, s"Number of partitions ($numPartitions) must be positive.")
 
-  val partitioning: Partitioning = {
+  override val partitioning: Partitioning = {
     val (sortOrder, nonSortOrder) = partitionExpressions.partition(_.isInstanceOf[SortOrder])
 
     require(sortOrder.isEmpty || nonSortOrder.isEmpty,
@@ -977,7 +1275,9 @@ case class RepartitionByExpression(
          |NonSortOrder: $nonSortOrder
        """.stripMargin)
 
-    if (sortOrder.nonEmpty) {
+    if (numPartitions == 1) {
+      SinglePartition
+    } else if (sortOrder.nonEmpty) {
       RangePartitioning(sortOrder.map(_.asInstanceOf[SortOrder]), numPartitions)
     } else if (nonSortOrder.nonEmpty) {
       HashPartitioning(nonSortOrder, numPartitions)
@@ -986,8 +1286,19 @@ case class RepartitionByExpression(
     }
   }
 
-  override def maxRows: Option[Long] = child.maxRows
   override def shuffle: Boolean = true
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): RepartitionByExpression =
+    copy(child = newChild)
+}
+
+object RepartitionByExpression {
+  def apply(
+      partitionExpressions: Seq[Expression],
+      child: LogicalPlan,
+      numPartitions: Int): RepartitionByExpression = {
+    RepartitionByExpression(partitionExpressions, child, Some(numPartitions))
+  }
 }
 
 /**
@@ -1010,13 +1321,16 @@ case class OneRowRelation() extends LeafNode {
 case class Deduplicate(
     keys: Seq[Attribute],
     child: LogicalPlan) extends UnaryNode {
-
+  override def maxRows: Option[Long] = child.maxRows
   override def output: Seq[Attribute] = child.output
+  final override val nodePatterns: Seq[TreePattern] = Seq(DISTINCT_LIKE)
+  override protected def withNewChildInternal(newChild: LogicalPlan): Deduplicate =
+    copy(child = newChild)
 }
 
 /**
  * A trait to represent the commands that support subqueries.
- * This is used to whitelist such commands in the subquery-related checks.
+ * This is used to allow such commands in the subquery-related checks.
  */
 trait SupportsSubquery extends LogicalPlan
 
@@ -1040,4 +1354,18 @@ case class CollectMetrics(
   }
 
   override def output: Seq[Attribute] = child.output
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): CollectMetrics =
+    copy(child = newChild)
+}
+
+/**
+ * A placeholder for domain join that can be added when decorrelating subqueries.
+ * It should be rewritten during the optimization phase.
+ */
+case class DomainJoin(domainAttrs: Seq[Attribute], child: LogicalPlan) extends UnaryNode {
+  override def output: Seq[Attribute] = child.output ++ domainAttrs
+  override def producedAttributes: AttributeSet = AttributeSet(domainAttrs)
+  override protected def withNewChildInternal(newChild: LogicalPlan): DomainJoin =
+    copy(child = newChild)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/hints.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/hints.scala
index a325b61fcc5a9..5bda94cea9527 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/hints.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/hints.scala
@@ -18,7 +18,6 @@
 package org.apache.spark.sql.catalyst.plans.logical
 
 import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.util.Utils
 
 /**
  * A general hint for the child that is not yet resolved. This node is generated by the parser and
@@ -32,6 +31,9 @@ case class UnresolvedHint(name: String, parameters: Seq[Any], child: LogicalPlan
 
   override lazy val resolved: Boolean = false
   override def output: Seq[Attribute] = child.output
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): UnresolvedHint =
+    copy(child = newChild)
 }
 
 /**
@@ -44,6 +46,9 @@ case class ResolvedHint(child: LogicalPlan, hints: HintInfo = HintInfo())
   override def output: Seq[Attribute] = child.output
 
   override def doCanonicalize(): LogicalPlan = child.canonicalized
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): ResolvedHint =
+    copy(child = newChild)
 }
 
 /**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/object.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/object.scala
index d383532cbd3d3..1f7eb67bf1726 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/object.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/object.scala
@@ -24,6 +24,7 @@ import org.apache.spark.sql.catalyst.analysis.UnresolvedDeserializer
 import org.apache.spark.sql.catalyst.encoders._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.objects.Invoke
+import org.apache.spark.sql.catalyst.trees.TreePattern._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.streaming.{GroupStateTimeout, OutputMode}
 import org.apache.spark.sql.types._
@@ -79,7 +80,11 @@ trait ObjectConsumer extends UnaryNode {
 case class DeserializeToObject(
     deserializer: Expression,
     outputObjAttr: Attribute,
-    child: LogicalPlan) extends UnaryNode with ObjectProducer
+    child: LogicalPlan) extends UnaryNode with ObjectProducer {
+  final override val nodePatterns: Seq[TreePattern] = Seq(DESERIALIZE_TO_OBJECT)
+  override protected def withNewChildInternal(newChild: LogicalPlan): DeserializeToObject =
+    copy(child = newChild)
+}
 
 /**
  * Takes the input object from child and turns it into unsafe row using the given serializer
@@ -90,6 +95,11 @@ case class SerializeFromObject(
     child: LogicalPlan) extends ObjectConsumer {
 
   override def output: Seq[Attribute] = serializer.map(_.toAttribute)
+
+  final override val nodePatterns: Seq[TreePattern] = Seq(SERIALIZE_FROM_OBJECT)
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): SerializeFromObject =
+    copy(child = newChild)
 }
 
 object MapPartitions {
@@ -111,7 +121,10 @@ object MapPartitions {
 case class MapPartitions(
     func: Iterator[Any] => Iterator[Any],
     outputObjAttr: Attribute,
-    child: LogicalPlan) extends ObjectConsumer with ObjectProducer
+    child: LogicalPlan) extends ObjectConsumer with ObjectProducer {
+  override protected def withNewChildInternal(newChild: LogicalPlan): MapPartitions =
+    copy(child = newChild)
+}
 
 object MapPartitionsInR {
   def apply(
@@ -159,6 +172,9 @@ case class MapPartitionsInR(
 
   override protected def stringArgs: Iterator[Any] = Iterator(inputSchema, outputSchema,
     outputObjAttr, child)
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): MapPartitionsInR =
+    copy(child = newChild)
 }
 
 /**
@@ -182,6 +198,9 @@ case class MapPartitionsInRWithArrow(
     inputSchema, StructType.fromAttributes(output), child)
 
   override val producedAttributes = AttributeSet(output)
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): MapPartitionsInRWithArrow =
+    copy(child = newChild)
 }
 
 object MapElements {
@@ -207,7 +226,10 @@ case class MapElements(
     argumentClass: Class[_],
     argumentSchema: StructType,
     outputObjAttr: Attribute,
-    child: LogicalPlan) extends ObjectConsumer with ObjectProducer
+    child: LogicalPlan) extends ObjectConsumer with ObjectProducer {
+  override protected def withNewChildInternal(newChild: LogicalPlan): MapElements =
+    copy(child = newChild)
+}
 
 object TypedFilter {
   def apply[T : Encoder](func: AnyRef, child: LogicalPlan): TypedFilter = {
@@ -238,6 +260,8 @@ case class TypedFilter(
 
   override def output: Seq[Attribute] = child.output
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(TYPED_FILTER)
+
   def withObjectProducerChild(obj: LogicalPlan): Filter = {
     assert(obj.output.length == 1)
     Filter(typedCondition(obj.output.head), obj)
@@ -251,6 +275,9 @@ case class TypedFilter(
     val funcObj = Literal.create(func, ObjectType(funcMethod._1))
     Invoke(funcObj, funcMethod._2, BooleanType, input :: Nil)
   }
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): TypedFilter =
+    copy(child = newChild)
 }
 
 object FunctionUtils {
@@ -333,7 +360,12 @@ case class AppendColumns(
 
   override def output: Seq[Attribute] = child.output ++ newColumns
 
+  final override val nodePatterns: Seq[TreePattern] = Seq(APPEND_COLUMNS)
+
   def newColumns: Seq[Attribute] = serializer.map(_.toAttribute)
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): AppendColumns =
+    copy(child = newChild)
 }
 
 /**
@@ -346,6 +378,9 @@ case class AppendColumnsWithObject(
     child: LogicalPlan) extends ObjectConsumer {
 
   override def output: Seq[Attribute] = (childSerializer ++ newColumnsSerializer).map(_.toAttribute)
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): AppendColumnsWithObject =
+    copy(child = newChild)
 }
 
 /** Factory for constructing new `MapGroups` nodes. */
@@ -382,7 +417,10 @@ case class MapGroups(
     groupingAttributes: Seq[Attribute],
     dataAttributes: Seq[Attribute],
     outputObjAttr: Attribute,
-    child: LogicalPlan) extends UnaryNode with ObjectProducer
+    child: LogicalPlan) extends UnaryNode with ObjectProducer {
+  override protected def withNewChildInternal(newChild: LogicalPlan): MapGroups =
+    copy(child = newChild)
+}
 
 /** Internal class representing State */
 trait LogicalGroupState[S]
@@ -453,6 +491,9 @@ case class FlatMapGroupsWithState(
   if (isMapGroupsWithState) {
     assert(outputMode == OutputMode.Update)
   }
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): FlatMapGroupsWithState =
+    copy(child = newChild)
 }
 
 /** Factory for constructing new `FlatMapGroupsInR` nodes. */
@@ -513,6 +554,9 @@ case class FlatMapGroupsInR(
   override protected def stringArgs: Iterator[Any] = Iterator(inputSchema, outputSchema,
     keyDeserializer, valueDeserializer, groupingAttributes, dataAttributes, outputObjAttr,
     child)
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): FlatMapGroupsInR =
+    copy(child = newChild)
 }
 
 /**
@@ -537,6 +581,9 @@ case class FlatMapGroupsInRWithArrow(
     inputSchema, StructType.fromAttributes(output), keyDeserializer, groupingAttributes, child)
 
   override val producedAttributes = AttributeSet(output)
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): FlatMapGroupsInRWithArrow =
+    copy(child = newChild)
 }
 
 /** Factory for constructing new `CoGroup` nodes. */
@@ -584,4 +631,7 @@ case class CoGroup(
     rightAttr: Seq[Attribute],
     outputObjAttr: Attribute,
     left: LogicalPlan,
-    right: LogicalPlan) extends BinaryNode with ObjectProducer
+    right: LogicalPlan) extends BinaryNode with ObjectProducer {
+  override protected def withNewChildrenInternal(
+      newLeft: LogicalPlan, newRight: LogicalPlan): CoGroup = copy(left = newLeft, right = newRight)
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/pythonLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/pythonLogicalOperators.scala
index c4f741cd2cec8..ba8352cf6ac89 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/pythonLogicalOperators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/pythonLogicalOperators.scala
@@ -37,6 +37,9 @@ case class FlatMapGroupsInPandas(
    * from the input.
    */
   override val producedAttributes = AttributeSet(output)
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): FlatMapGroupsInPandas =
+    copy(child = newChild)
 }
 
 /**
@@ -49,6 +52,9 @@ case class MapInPandas(
     child: LogicalPlan) extends UnaryNode {
 
   override val producedAttributes = AttributeSet(output)
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): MapInPandas =
+    copy(child = newChild)
 }
 
 /**
@@ -56,14 +62,24 @@ case class MapInPandas(
  * This is used by DataFrame.groupby().cogroup().apply().
  */
 case class FlatMapCoGroupsInPandas(
-    leftAttributes: Seq[Attribute],
-    rightAttributes: Seq[Attribute],
+    leftGroupingLen: Int,
+    rightGroupingLen: Int,
     functionExpr: Expression,
     output: Seq[Attribute],
     left: LogicalPlan,
     right: LogicalPlan) extends BinaryNode {
 
   override val producedAttributes = AttributeSet(output)
+  override lazy val references: AttributeSet =
+    AttributeSet(leftAttributes ++ rightAttributes ++ functionExpr.references) -- producedAttributes
+
+  def leftAttributes: Seq[Attribute] = left.output.take(leftGroupingLen)
+
+  def rightAttributes: Seq[Attribute] = right.output.take(rightGroupingLen)
+
+  override protected def withNewChildrenInternal(
+      newLeft: LogicalPlan, newRight: LogicalPlan): FlatMapCoGroupsInPandas =
+    copy(left = newLeft, right = newRight)
 }
 
 trait BaseEvalPython extends UnaryNode {
@@ -83,7 +99,10 @@ trait BaseEvalPython extends UnaryNode {
 case class BatchEvalPython(
     udfs: Seq[PythonUDF],
     resultAttrs: Seq[Attribute],
-    child: LogicalPlan) extends BaseEvalPython
+    child: LogicalPlan) extends BaseEvalPython {
+  override protected def withNewChildInternal(newChild: LogicalPlan): BatchEvalPython =
+    copy(child = newChild)
+}
 
 /**
  * A logical plan that evaluates a [[PythonUDF]] with Apache Arrow.
@@ -92,4 +111,7 @@ case class ArrowEvalPython(
     udfs: Seq[PythonUDF],
     resultAttrs: Seq[Attribute],
     child: LogicalPlan,
-    evalType: Int) extends BaseEvalPython
+    evalType: Int) extends BaseEvalPython {
+  override protected def withNewChildInternal(newChild: LogicalPlan): ArrowEvalPython =
+    copy(child = newChild)
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
index b1129e741221b..44550ae2844ca 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
@@ -19,8 +19,8 @@ package org.apache.spark.sql.catalyst.plans.logical
 
 import org.apache.spark.sql.catalyst.analysis.ViewType
 import org.apache.spark.sql.catalyst.catalog.{BucketSpec, FunctionResource}
-import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
 import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.trees.{LeafLike, UnaryLike}
 import org.apache.spark.sql.connector.catalog.TableChange.ColumnPosition
 import org.apache.spark.sql.connector.expressions.Transform
 import org.apache.spark.sql.types.{DataType, StructType}
@@ -48,11 +48,87 @@ abstract class ParsedStatement extends LogicalPlan {
 
   override def output: Seq[Attribute] = Seq.empty
 
-  override def children: Seq[LogicalPlan] = Seq.empty
-
   final override lazy val resolved = false
 }
 
+trait LeafParsedStatement extends ParsedStatement with LeafLike[LogicalPlan]
+trait UnaryParsedStatement extends ParsedStatement with UnaryLike[LogicalPlan]
+
+/**
+ * Type to keep track of Hive serde info
+ */
+case class SerdeInfo(
+    storedAs: Option[String] = None,
+    formatClasses: Option[FormatClasses] = None,
+    serde: Option[String] = None,
+    serdeProperties: Map[String, String] = Map.empty) {
+  // this uses assertions because validation is done in validateRowFormatFileFormat etc.
+  assert(storedAs.isEmpty || formatClasses.isEmpty,
+    "Cannot specify both STORED AS and INPUTFORMAT/OUTPUTFORMAT")
+
+  def describe: String = {
+    val serdeString = if (serde.isDefined || serdeProperties.nonEmpty) {
+      "ROW FORMAT " + serde.map(sd => s"SERDE $sd").getOrElse("DELIMITED")
+    } else {
+      ""
+    }
+
+    this match {
+      case SerdeInfo(Some(storedAs), _, _, _) =>
+        s"STORED AS $storedAs $serdeString"
+      case SerdeInfo(_, Some(formatClasses), _, _) =>
+        s"STORED AS $formatClasses $serdeString"
+      case _ =>
+        serdeString
+    }
+  }
+
+  def merge(other: SerdeInfo): SerdeInfo = {
+    def getOnly[T](desc: String, left: Option[T], right: Option[T]): Option[T] = {
+      (left, right) match {
+        case (Some(l), Some(r)) =>
+          assert(l == r, s"Conflicting $desc values: $l != $r")
+          left
+        case (Some(_), _) =>
+          left
+        case (_, Some(_)) =>
+          right
+        case _ =>
+          None
+      }
+    }
+
+    SerdeInfo.checkSerdePropMerging(serdeProperties, other.serdeProperties)
+    SerdeInfo(
+      getOnly("STORED AS", storedAs, other.storedAs),
+      getOnly("INPUTFORMAT/OUTPUTFORMAT", formatClasses, other.formatClasses),
+      getOnly("SERDE", serde, other.serde),
+      serdeProperties ++ other.serdeProperties)
+  }
+}
+
+case class FormatClasses(input: String, output: String) {
+  override def toString: String = s"INPUTFORMAT $input OUTPUTFORMAT $output"
+}
+
+object SerdeInfo {
+  val empty: SerdeInfo = SerdeInfo(None, None, None, Map.empty)
+
+  def checkSerdePropMerging(
+      props1: Map[String, String], props2: Map[String, String]): Unit = {
+    val conflictKeys = props1.keySet.intersect(props2.keySet)
+    if (conflictKeys.nonEmpty) {
+      throw new UnsupportedOperationException(
+        s"""
+          |Cannot safely merge SERDEPROPERTIES:
+          |${props1.map { case (k, v) => s"$k=$v" }.mkString("{", ",", "}")}
+          |${props2.map { case (k, v) => s"$k=$v" }.mkString("{", ",", "}")}
+          |The conflict keys: ${conflictKeys.mkString(", ")}
+          |""".stripMargin)
+    }
+  }
+}
+
 /**
  * A CREATE TABLE command, as parsed from SQL.
  *
@@ -68,7 +144,9 @@ case class CreateTableStatement(
     options: Map[String, String],
     location: Option[String],
     comment: Option[String],
-    ifNotExists: Boolean) extends ParsedStatement
+    serde: Option[SerdeInfo],
+    external: Boolean,
+    ifNotExists: Boolean) extends LeafParsedStatement
 
 /**
  * A CREATE TABLE AS SELECT command, as parsed from SQL.
@@ -84,9 +162,13 @@ case class CreateTableAsSelectStatement(
     location: Option[String],
     comment: Option[String],
     writeOptions: Map[String, String],
-    ifNotExists: Boolean) extends ParsedStatement {
+    serde: Option[SerdeInfo],
+    external: Boolean,
+    ifNotExists: Boolean) extends UnaryParsedStatement {
 
-  override def children: Seq[LogicalPlan] = Seq(asSelect)
+  override def child: LogicalPlan = asSelect
+  override protected def withNewChildInternal(newChild: LogicalPlan): CreateTableAsSelectStatement =
+    copy(asSelect = newChild)
 }
 
 /**
@@ -101,7 +183,10 @@ case class CreateViewStatement(
     child: LogicalPlan,
     allowExisting: Boolean,
     replace: Boolean,
-    viewType: ViewType) extends ParsedStatement
+    viewType: ViewType) extends UnaryParsedStatement {
+  override protected def withNewChildInternal(newChild: LogicalPlan): CreateViewStatement =
+    copy(child = newChild)
+}
 
 /**
  * A REPLACE TABLE command, as parsed from SQL.
@@ -119,7 +204,8 @@ case class ReplaceTableStatement(
     options: Map[String, String],
     location: Option[String],
     comment: Option[String],
-    orCreate: Boolean) extends ParsedStatement
+    serde: Option[SerdeInfo],
+    orCreate: Boolean) extends LeafParsedStatement
 
 /**
  * A REPLACE TABLE AS SELECT command, as parsed from SQL.
@@ -135,9 +221,12 @@ case class ReplaceTableAsSelectStatement(
     location: Option[String],
     comment: Option[String],
     writeOptions: Map[String, String],
-    orCreate: Boolean) extends ParsedStatement {
+    serde: Option[SerdeInfo],
+    orCreate: Boolean) extends UnaryParsedStatement {
 
-  override def children: Seq[LogicalPlan] = Seq(asSelect)
+  override def child: LogicalPlan = asSelect
+  override protected def withNewChildInternal(
+    newChild: LogicalPlan): ReplaceTableAsSelectStatement = copy(asSelect = newChild)
 }
 
 
@@ -156,11 +245,11 @@ case class QualifiedColType(
  */
 case class AlterTableAddColumnsStatement(
     tableName: Seq[String],
-    columnsToAdd: Seq[QualifiedColType]) extends ParsedStatement
+    columnsToAdd: Seq[QualifiedColType]) extends LeafParsedStatement
 
 case class AlterTableReplaceColumnsStatement(
     tableName: Seq[String],
-    columnsToAdd: Seq[QualifiedColType]) extends ParsedStatement
+    columnsToAdd: Seq[QualifiedColType]) extends LeafParsedStatement
 
 /**
  * ALTER TABLE ... CHANGE COLUMN command, as parsed from SQL.
@@ -171,7 +260,7 @@ case class AlterTableAlterColumnStatement(
     dataType: Option[DataType],
     nullable: Option[Boolean],
     comment: Option[String],
-    position: Option[ColumnPosition]) extends ParsedStatement
+    position: Option[ColumnPosition]) extends LeafParsedStatement
 
 /**
  * ALTER TABLE ... RENAME COLUMN command, as parsed from SQL.
@@ -179,137 +268,20 @@ case class AlterTableAlterColumnStatement(
 case class AlterTableRenameColumnStatement(
     tableName: Seq[String],
     column: Seq[String],
-    newName: String) extends ParsedStatement
+    newName: String) extends LeafParsedStatement
 
 /**
  * ALTER TABLE ... DROP COLUMNS command, as parsed from SQL.
  */
 case class AlterTableDropColumnsStatement(
     tableName: Seq[String],
-    columnsToDrop: Seq[Seq[String]]) extends ParsedStatement
-
-/**
- * ALTER TABLE ... SET TBLPROPERTIES command, as parsed from SQL.
- */
-case class AlterTableSetPropertiesStatement(
-    tableName: Seq[String],
-    properties: Map[String, String]) extends ParsedStatement
-
-/**
- * ALTER TABLE ... UNSET TBLPROPERTIES command, as parsed from SQL.
- */
-case class AlterTableUnsetPropertiesStatement(
-    tableName: Seq[String],
-    propertyKeys: Seq[String],
-    ifExists: Boolean) extends ParsedStatement
-
-/**
- * ALTER TABLE ... SET LOCATION command, as parsed from SQL.
- */
-case class AlterTableSetLocationStatement(
-    tableName: Seq[String],
-    partitionSpec: Option[TablePartitionSpec],
-    location: String) extends ParsedStatement
-
-/**
- * ALTER TABLE ... RECOVER PARTITIONS command, as parsed from SQL.
- */
-case class AlterTableRecoverPartitionsStatement(
-    tableName: Seq[String]) extends ParsedStatement
-
-/**
- * ALTER TABLE ... ADD PARTITION command, as parsed from SQL
- */
-case class AlterTableAddPartitionStatement(
-    tableName: Seq[String],
-    partitionSpecsAndLocs: Seq[(TablePartitionSpec, Option[String])],
-    ifNotExists: Boolean) extends ParsedStatement
-
-/**
- * ALTER TABLE ... RENAME PARTITION command, as parsed from SQL.
- */
-case class AlterTableRenamePartitionStatement(
-    tableName: Seq[String],
-    from: TablePartitionSpec,
-    to: TablePartitionSpec) extends ParsedStatement
-
-/**
- * ALTER TABLE ... DROP PARTITION command, as parsed from SQL
- */
-case class AlterTableDropPartitionStatement(
-    tableName: Seq[String],
-    specs: Seq[TablePartitionSpec],
-    ifExists: Boolean,
-    purge: Boolean,
-    retainData: Boolean) extends ParsedStatement
-
-/**
- * ALTER TABLE ... SERDEPROPERTIES command, as parsed from SQL
- */
-case class AlterTableSerDePropertiesStatement(
-    tableName: Seq[String],
-    serdeClassName: Option[String],
-    serdeProperties: Option[Map[String, String]],
-    partitionSpec: Option[TablePartitionSpec]) extends ParsedStatement
-
-/**
- * ALTER VIEW ... SET TBLPROPERTIES command, as parsed from SQL.
- */
-case class AlterViewSetPropertiesStatement(
-    viewName: Seq[String],
-    properties: Map[String, String]) extends ParsedStatement
-
-/**
- * ALTER VIEW ... UNSET TBLPROPERTIES command, as parsed from SQL.
- */
-case class AlterViewUnsetPropertiesStatement(
-    viewName: Seq[String],
-    propertyKeys: Seq[String],
-    ifExists: Boolean) extends ParsedStatement
-
-/**
- * ALTER VIEW ... Query command, as parsed from SQL.
- */
-case class AlterViewAsStatement(
-    viewName: Seq[String],
-    originalText: String,
-    query: LogicalPlan) extends ParsedStatement
-
-/**
- * ALTER TABLE ... RENAME TO command, as parsed from SQL.
- */
-case class RenameTableStatement(
-    oldName: Seq[String],
-    newName: Seq[String],
-    isView: Boolean) extends ParsedStatement
-
-/**
- * A DROP TABLE statement, as parsed from SQL.
- */
-case class DropTableStatement(
-    tableName: Seq[String],
-    ifExists: Boolean,
-    purge: Boolean) extends ParsedStatement
-
-/**
- * A DROP VIEW statement, as parsed from SQL.
- */
-case class DropViewStatement(
-    viewName: Seq[String],
-    ifExists: Boolean) extends ParsedStatement
-
-/**
- * A DESCRIBE TABLE tbl_name col_name statement, as parsed from SQL.
- */
-case class DescribeColumnStatement(
-    tableName: Seq[String],
-    colNameParts: Seq[String],
-    isExtended: Boolean) extends ParsedStatement
+    columnsToDrop: Seq[Seq[String]]) extends LeafParsedStatement
 
 /**
  * An INSERT INTO statement, as parsed from SQL.
  *
  * @param table                the logical plan representing the table.
+ * @param userSpecifiedCols    the user specified list of columns that belong to the table.
  * @param query                the logical plan representing data to write to.
  * @param overwrite            overwrite existing table or partitions.
  * @param partitionSpec        a map from the partition key to the partition value (optional).
@@ -324,151 +296,38 @@ case class DescribeColumnStatement(
 case class InsertIntoStatement(
     table: LogicalPlan,
     partitionSpec: Map[String, Option[String]],
+    userSpecifiedCols: Seq[String],
     query: LogicalPlan,
     overwrite: Boolean,
-    ifPartitionNotExists: Boolean) extends ParsedStatement {
+    ifPartitionNotExists: Boolean) extends UnaryParsedStatement {
 
   require(overwrite || !ifPartitionNotExists,
     "IF NOT EXISTS is only valid in INSERT OVERWRITE")
   require(partitionSpec.values.forall(_.nonEmpty) || !ifPartitionNotExists,
     "IF NOT EXISTS is only valid with static partitions")
 
-  override def children: Seq[LogicalPlan] = query :: Nil
+  override def child: LogicalPlan = query
+  override protected def withNewChildInternal(newChild: LogicalPlan): InsertIntoStatement =
+    copy(query = newChild)
 }
 
-/**
- * A SHOW TABLE EXTENDED statement, as parsed from SQL.
- */
-case class ShowTableStatement(
-    namespace: Option[Seq[String]],
-    pattern: String,
-    partitionSpec: Option[TablePartitionSpec])
-  extends ParsedStatement
-
 /**
  * A CREATE NAMESPACE statement, as parsed from SQL.
  */
 case class CreateNamespaceStatement(
     namespace: Seq[String],
     ifNotExists: Boolean,
-    properties: Map[String, String]) extends ParsedStatement
+    properties: Map[String, String]) extends LeafParsedStatement
 
 /**
  * A USE statement, as parsed from SQL.
  */
-case class UseStatement(isNamespaceSet: Boolean, nameParts: Seq[String]) extends ParsedStatement
-
-/**
- * An ANALYZE TABLE statement, as parsed from SQL.
- */
-case class AnalyzeTableStatement(
-    tableName: Seq[String],
-    partitionSpec: Map[String, Option[String]],
-    noScan: Boolean) extends ParsedStatement
-
-/**
- * An ANALYZE TABLE FOR COLUMNS statement, as parsed from SQL.
- */
-case class AnalyzeColumnStatement(
-    tableName: Seq[String],
-    columnNames: Option[Seq[String]],
-    allColumns: Boolean) extends ParsedStatement {
-  require(columnNames.isDefined ^ allColumns, "Parameter `columnNames` or `allColumns` are " +
-    "mutually exclusive. Only one of them should be specified.")
-}
-
-/**
- * A REPAIR TABLE statement, as parsed from SQL
- */
-case class RepairTableStatement(tableName: Seq[String]) extends ParsedStatement
-
-/**
- * A LOAD DATA INTO TABLE statement, as parsed from SQL
- */
-case class LoadDataStatement(
-    tableName: Seq[String],
-    path: String,
-    isLocal: Boolean,
-    isOverwrite: Boolean,
-    partition: Option[TablePartitionSpec]) extends ParsedStatement
-
-/**
- * A SHOW CREATE TABLE statement, as parsed from SQL.
- */
-case class ShowCreateTableStatement(
-    tableName: Seq[String],
-    asSerde: Boolean = false) extends ParsedStatement
-
-/**
- * A CACHE TABLE statement, as parsed from SQL
- */
-case class CacheTableStatement(
-    tableName: Seq[String],
-    plan: Option[LogicalPlan],
-    isLazy: Boolean,
-    options: Map[String, String]) extends ParsedStatement
-
-/**
- * An UNCACHE TABLE statement, as parsed from SQL
- */
-case class UncacheTableStatement(
-    tableName: Seq[String],
-    ifExists: Boolean) extends ParsedStatement
-
-/**
- * A TRUNCATE TABLE statement, as parsed from SQL
- */
-case class TruncateTableStatement(
-    tableName: Seq[String],
-    partitionSpec: Option[TablePartitionSpec]) extends ParsedStatement
-
-/**
- * A SHOW PARTITIONS statement, as parsed from SQL
- */
-case class ShowPartitionsStatement(
-    tableName: Seq[String],
-    partitionSpec: Option[TablePartitionSpec]) extends ParsedStatement
-
-/**
- * A REFRESH TABLE statement, as parsed from SQL
- */
-case class RefreshTableStatement(tableName: Seq[String]) extends ParsedStatement
-
-/**
- * A SHOW COLUMNS statement, as parsed from SQL
- */
-case class ShowColumnsStatement(
-    table: Seq[String],
-    namespace: Option[Seq[String]]) extends ParsedStatement
+case class UseStatement(isNamespaceSet: Boolean, nameParts: Seq[String]) extends LeafParsedStatement
 
 /**
  * A SHOW CURRENT NAMESPACE statement, as parsed from SQL
  */
-case class ShowCurrentNamespaceStatement() extends ParsedStatement
-
-/**
- * A DESCRIBE FUNCTION statement, as parsed from SQL
- */
-case class DescribeFunctionStatement(
-    functionName: Seq[String],
-    isExtended: Boolean) extends ParsedStatement
-
-/**
- *  SHOW FUNCTIONS statement, as parsed from SQL
- */
-case class ShowFunctionsStatement(
-    userScope: Boolean,
-    systemScope: Boolean,
-    pattern: Option[String],
-    functionName: Option[Seq[String]]) extends ParsedStatement
-
-/**
- *  DROP FUNCTION statement, as parsed from SQL
- */
-case class DropFunctionStatement(
-    functionName: Seq[String],
-    ifExists: Boolean,
-    isTemp: Boolean) extends ParsedStatement
+case class ShowCurrentNamespaceStatement() extends LeafParsedStatement
 
 /**
  *  CREATE FUNCTION statement, as parsed from SQL
@@ -479,4 +338,4 @@ case class CreateFunctionStatement(
     resources: Seq[FunctionResource],
     isTemp: Boolean,
     ignoreIfExists: Boolean,
-    replace: Boolean) extends ParsedStatement
+    replace: Boolean) extends LeafParsedStatement
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/BasicStatsPlanVisitor.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/BasicStatsPlanVisitor.scala
index b8c652dc8f12e..201277680271f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/BasicStatsPlanVisitor.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/BasicStatsPlanVisitor.scala
@@ -27,13 +27,23 @@ object BasicStatsPlanVisitor extends LogicalPlanVisitor[Statistics] {
   /** Falls back to the estimation computed by [[SizeInBytesOnlyStatsPlanVisitor]]. */
   private def fallback(p: LogicalPlan): Statistics = SizeInBytesOnlyStatsPlanVisitor.visit(p)
 
-  override def default(p: LogicalPlan): Statistics = fallback(p)
+  override def default(p: LogicalPlan): Statistics = p match {
+    case p: LeafNode => p.computeStats()
+    case _: LogicalPlan =>
+      val stats = p.children.map(_.stats)
+      val rowCount = if (stats.exists(_.rowCount.isEmpty)) {
+        None
+      } else {
+        Some(stats.map(_.rowCount.get).filter(_ > 0L).product)
+      }
+      Statistics(sizeInBytes = stats.map(_.sizeInBytes).filter(_ > 0L).product, rowCount = rowCount)
+  }
 
   override def visitAggregate(p: Aggregate): Statistics = {
     AggregateEstimation.estimate(p).getOrElse(fallback(p))
   }
 
-  override def visitDistinct(p: Distinct): Statistics = fallback(p)
+  override def visitDistinct(p: Distinct): Statistics = default(p)
 
   override def visitExcept(p: Except): Statistics = fallback(p)
 
@@ -43,11 +53,21 @@ object BasicStatsPlanVisitor extends LogicalPlanVisitor[Statistics] {
     FilterEstimation(p).estimate.getOrElse(fallback(p))
   }
 
-  override def visitGenerate(p: Generate): Statistics = fallback(p)
+  override def visitGenerate(p: Generate): Statistics = default(p)
 
   override def visitGlobalLimit(p: GlobalLimit): Statistics = fallback(p)
 
-  override def visitIntersect(p: Intersect): Statistics = fallback(p)
+  override def visitIntersect(p: Intersect): Statistics = {
+    val leftStats = p.left.stats
+    val rightStats = p.right.stats
+    val leftSize = leftStats.sizeInBytes
+    val rightSize = rightStats.sizeInBytes
+    if (leftSize < rightSize) {
+      Statistics(sizeInBytes = leftSize, rowCount = leftStats.rowCount)
+    } else {
+      Statistics(sizeInBytes = rightSize, rowCount = rightStats.rowCount)
+    }
+  }
 
   override def visitJoin(p: Join): Statistics = {
     JoinEstimation(p).estimate.getOrElse(fallback(p))
@@ -55,21 +75,31 @@ object BasicStatsPlanVisitor extends LogicalPlanVisitor[Statistics] {
 
   override def visitLocalLimit(p: LocalLimit): Statistics = fallback(p)
 
-  override def visitPivot(p: Pivot): Statistics = fallback(p)
+  override def visitPivot(p: Pivot): Statistics = default(p)
 
   override def visitProject(p: Project): Statistics = {
     ProjectEstimation.estimate(p).getOrElse(fallback(p))
   }
 
-  override def visitRepartition(p: Repartition): Statistics = fallback(p)
+  override def visitRepartition(p: Repartition): Statistics = default(p)
 
-  override def visitRepartitionByExpr(p: RepartitionByExpression): Statistics = fallback(p)
+  override def visitRepartitionByExpr(p: RepartitionByExpression): Statistics = default(p)
 
   override def visitSample(p: Sample): Statistics = fallback(p)
 
-  override def visitScriptTransform(p: ScriptTransformation): Statistics = fallback(p)
+  override def visitScriptTransform(p: ScriptTransformation): Statistics = default(p)
 
-  override def visitUnion(p: Union): Statistics = fallback(p)
+  override def visitUnion(p: Union): Statistics = {
+    UnionEstimation.estimate(p).getOrElse(fallback(p))
+  }
 
   override def visitWindow(p: Window): Statistics = fallback(p)
+
+  override def visitSort(p: Sort): Statistics = {
+    BasicStatsPlanVisitor.visit(p.child)
+  }
+
+  override def visitTail(p: Tail): Statistics = {
+    fallback(p)
+  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/EstimationUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/EstimationUtils.scala
index e1dbef9ebeede..967ccedeeeacb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/EstimationUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/EstimationUtils.scala
@@ -344,7 +344,7 @@ object EstimationUtils {
         }
       }
     }
-    overlappedRanges
+    overlappedRanges.toSeq
   }
 
   /**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/JoinEstimation.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/JoinEstimation.scala
index 19a0d1279cc32..777a4c8291223 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/JoinEstimation.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/JoinEstimation.scala
@@ -323,7 +323,7 @@ case class JoinEstimation(join: Join) extends Logging {
         outputAttrStats += a -> newColStat
       }
     }
-    outputAttrStats
+    outputAttrStats.toSeq
   }
 
   private def extractJoinKeysWithColStats(
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/ProjectEstimation.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/ProjectEstimation.scala
index 6925423f003ba..8e58c4f314df0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/ProjectEstimation.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/ProjectEstimation.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.catalyst.plans.logical.statsEstimation
 
-import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeMap}
+import org.apache.spark.sql.catalyst.expressions.AttributeMap
 import org.apache.spark.sql.catalyst.plans.logical.{Project, Statistics}
 
 object ProjectEstimation {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/SizeInBytesOnlyStatsPlanVisitor.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/SizeInBytesOnlyStatsPlanVisitor.scala
index da36db7ae1f5f..e1349ff0c62d1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/SizeInBytesOnlyStatsPlanVisitor.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/SizeInBytesOnlyStatsPlanVisitor.scala
@@ -53,7 +53,8 @@ object SizeInBytesOnlyStatsPlanVisitor extends LogicalPlanVisitor[Statistics] {
    */
   override def default(p: LogicalPlan): Statistics = p match {
     case p: LeafNode => p.computeStats()
-    case _: LogicalPlan => Statistics(sizeInBytes = p.children.map(_.stats.sizeInBytes).product)
+    case _: LogicalPlan =>
+      Statistics(sizeInBytes = p.children.map(_.stats.sizeInBytes).filter(_ > 0L).product)
   }
 
   override def visitAggregate(p: Aggregate): Statistics = {
@@ -149,4 +150,15 @@ object SizeInBytesOnlyStatsPlanVisitor extends LogicalPlanVisitor[Statistics] {
   }
 
   override def visitWindow(p: Window): Statistics = visitUnaryNode(p)
+
+  override def visitSort(p: Sort): Statistics = default(p)
+
+  override def visitTail(p: Tail): Statistics = {
+    val limit = p.limitExpr.eval().asInstanceOf[Int]
+    val childStats = p.child.stats
+    val rowCount: BigInt = childStats.rowCount.map(_.min(limit)).getOrElse(limit)
+    Statistics(
+      sizeInBytes = EstimationUtils.getOutputSize(p.output, rowCount, childStats.attributeStats),
+      rowCount = Some(rowCount))
+  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/UnionEstimation.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/UnionEstimation.scala
new file mode 100644
index 0000000000000..c89ee1e80d926
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/UnionEstimation.scala
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.plans.logical.statsEstimation
+
+import scala.collection.mutable.ArrayBuffer
+
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeMap}
+import org.apache.spark.sql.catalyst.plans.logical.{ColumnStat, Statistics, Union}
+import org.apache.spark.sql.types._
+
+/**
+ * Estimate the number of output rows by doing the sum of output rows for each child of union,
+ * and estimate min and max stats for each column by finding the overall min and max of that
+ * column coming from its children.
+ */
+object UnionEstimation {
+  import EstimationUtils._
+
+  private def createStatComparator(dt: DataType): (Any, Any) => Boolean = dt match {
+    case ByteType => (a: Any, b: Any) =>
+      ByteType.ordering.lt(a.asInstanceOf[Byte], b.asInstanceOf[Byte])
+    case ShortType => (a: Any, b: Any) =>
+      ShortType.ordering.lt(a.asInstanceOf[Short], b.asInstanceOf[Short])
+    case IntegerType => (a: Any, b: Any) =>
+      IntegerType.ordering.lt(a.asInstanceOf[Int], b.asInstanceOf[Int])
+    case LongType => (a: Any, b: Any) =>
+      LongType.ordering.lt(a.asInstanceOf[Long], b.asInstanceOf[Long])
+    case FloatType => (a: Any, b: Any) =>
+      FloatType.ordering.lt(a.asInstanceOf[Float], b.asInstanceOf[Float])
+    case DoubleType => (a: Any, b: Any) =>
+      DoubleType.ordering.lt(a.asInstanceOf[Double], b.asInstanceOf[Double])
+    case _: DecimalType => (a: Any, b: Any) =>
+      dt.asInstanceOf[DecimalType].ordering.lt(a.asInstanceOf[Decimal], b.asInstanceOf[Decimal])
+    case DateType => (a: Any, b: Any) =>
+      DateType.ordering.lt(a.asInstanceOf[DateType.InternalType],
+        b.asInstanceOf[DateType.InternalType])
+    case TimestampType => (a: Any, b: Any) =>
+      TimestampType.ordering.lt(a.asInstanceOf[TimestampType.InternalType],
+        b.asInstanceOf[TimestampType.InternalType])
+    case _ =>
+      throw new IllegalStateException(s"Unsupported data type: ${dt.catalogString}")
+  }
+
+  private def isTypeSupported(dt: DataType): Boolean = dt match {
+    case ByteType | IntegerType | ShortType | FloatType | LongType |
+         DoubleType | DateType | _: DecimalType | TimestampType => true
+    case _ => false
+  }
+
+  def estimate(union: Union): Option[Statistics] = {
+    val sizeInBytes = union.children.map(_.stats.sizeInBytes).sum
+    val outputRows = if (rowCountsExist(union.children: _*)) {
+      Some(union.children.map(_.stats.rowCount.get).sum)
+    } else {
+      None
+    }
+
+    val unionOutput = union.output
+
+    val attrToComputeMinMaxStats = union.children.map(_.output).transpose.zipWithIndex.filter {
+      case (attrs, outputIndex) => isTypeSupported(unionOutput(outputIndex).dataType) &&
+        // checks if all the children has min/max stats for an attribute
+        attrs.zipWithIndex.forall {
+          case (attr, childIndex) =>
+            val attrStats = union.children(childIndex).stats.attributeStats
+            attrStats.get(attr).isDefined && attrStats(attr).hasMinMaxStats
+        }
+    }
+
+    val newAttrStats = if (attrToComputeMinMaxStats.nonEmpty) {
+      val outputAttrStats = new ArrayBuffer[(Attribute, ColumnStat)]()
+      attrToComputeMinMaxStats.foreach {
+        case (attrs, outputIndex) =>
+          val dataType = unionOutput(outputIndex).dataType
+          val statComparator = createStatComparator(dataType)
+          val minMaxValue = attrs.zipWithIndex.foldLeft[(Option[Any], Option[Any])]((None, None)) {
+              case ((minVal, maxVal), (attr, childIndex)) =>
+                val colStat = union.children(childIndex).stats.attributeStats(attr)
+                val min = if (minVal.isEmpty || statComparator(colStat.min.get, minVal.get)) {
+                  colStat.min
+                } else {
+                  minVal
+                }
+                val max = if (maxVal.isEmpty || statComparator(maxVal.get, colStat.max.get)) {
+                  colStat.max
+                } else {
+                  maxVal
+                }
+                (min, max)
+            }
+          val newStat = ColumnStat(min = minMaxValue._1, max = minMaxValue._2)
+          outputAttrStats += unionOutput(outputIndex) -> newStat
+      }
+      AttributeMap(outputAttrStats.toSeq)
+    } else {
+      AttributeMap.empty[ColumnStat]
+    }
+
+    Some(
+      Statistics(
+        sizeInBytes = sizeInBytes,
+        rowCount = outputRows,
+        attributeStats = newAttrStats))
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
index 579157a6f2f2e..310a4377bf5bc 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
@@ -17,39 +17,53 @@
 
 package org.apache.spark.sql.catalyst.plans.logical
 
-import org.apache.spark.sql.catalyst.analysis.{NamedRelation, UnresolvedException}
+import org.apache.spark.sql.catalyst.analysis.{NamedRelation, PartitionSpec, UnresolvedException}
 import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
-import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression, Unevaluable}
-import org.apache.spark.sql.catalyst.plans.DescribeTableSchema
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, AttributeSet, Expression, Unevaluable}
+import org.apache.spark.sql.catalyst.plans.DescribeCommandSchema
+import org.apache.spark.sql.catalyst.trees.BinaryLike
+import org.apache.spark.sql.catalyst.util.CharVarcharUtils
 import org.apache.spark.sql.connector.catalog._
 import org.apache.spark.sql.connector.catalog.TableChange.{AddColumn, ColumnChange}
 import org.apache.spark.sql.connector.expressions.Transform
-import org.apache.spark.sql.types.{DataType, MetadataBuilder, StringType, StructType}
+import org.apache.spark.sql.connector.write.Write
+import org.apache.spark.sql.types.{BooleanType, DataType, MetadataBuilder, StringType, StructType}
 
 /**
  * Base trait for DataSourceV2 write commands
  */
-trait V2WriteCommand extends Command {
+trait V2WriteCommand extends UnaryCommand {
   def table: NamedRelation
   def query: LogicalPlan
+  def isByName: Boolean
 
-  override def children: Seq[LogicalPlan] = Seq(query)
+  override def child: LogicalPlan = query
 
-  override lazy val resolved: Boolean = outputResolved
+  override lazy val resolved: Boolean = table.resolved && query.resolved && outputResolved
 
   def outputResolved: Boolean = {
+    assert(table.resolved && query.resolved,
+      "`outputResolved` can only be called when `table` and `query` are both resolved.")
     // If the table doesn't require schema match, we don't need to resolve the output columns.
-    table.skipSchemaResolution || {
-      table.resolved && query.resolved && query.output.size == table.output.size &&
-        query.output.zip(table.output).forall {
-          case (inAttr, outAttr) =>
-            // names and types must match, nullability must be compatible
-            inAttr.name == outAttr.name &&
-              DataType.equalsIgnoreCompatibleNullability(outAttr.dataType, inAttr.dataType) &&
-              (outAttr.nullable || !inAttr.nullable)
-        }
-    }
+    table.skipSchemaResolution || (query.output.size == table.output.size &&
+      query.output.zip(table.output).forall {
+        case (inAttr, outAttr) =>
+          val outType = CharVarcharUtils.getRawType(outAttr.metadata).getOrElse(outAttr.dataType)
+          // names and types must match, nullability must be compatible
+          inAttr.name == outAttr.name &&
+            DataType.equalsIgnoreCompatibleNullability(inAttr.dataType, outType) &&
+            (outAttr.nullable || !inAttr.nullable)
+      })
   }
+
+  def withNewQuery(newQuery: LogicalPlan): V2WriteCommand
+  def withNewTable(newTable: NamedRelation): V2WriteCommand
+}
+
+trait V2PartitionCommand extends UnaryCommand {
+  def table: LogicalPlan
+  def allowPartialPartitionSpec: Boolean = false
+  override def child: LogicalPlan = table
 }
 
 /**
@@ -59,7 +73,13 @@ case class AppendData(
     table: NamedRelation,
     query: LogicalPlan,
     writeOptions: Map[String, String],
-    isByName: Boolean) extends V2WriteCommand
+    isByName: Boolean,
+    write: Option[Write] = None) extends V2WriteCommand {
+  override def withNewQuery(newQuery: LogicalPlan): AppendData = copy(query = newQuery)
+  override def withNewTable(newTable: NamedRelation): AppendData = copy(table = newTable)
+  override protected def withNewChildInternal(newChild: LogicalPlan): AppendData =
+    copy(query = newChild)
+}
 
 object AppendData {
   def byName(
@@ -85,8 +105,21 @@ case class OverwriteByExpression(
     deleteExpr: Expression,
     query: LogicalPlan,
     writeOptions: Map[String, String],
-    isByName: Boolean) extends V2WriteCommand {
-  override lazy val resolved: Boolean = outputResolved && deleteExpr.resolved
+    isByName: Boolean,
+    write: Option[Write] = None) extends V2WriteCommand {
+  override lazy val resolved: Boolean = {
+    table.resolved && query.resolved && outputResolved && deleteExpr.resolved
+  }
+  override def inputSet: AttributeSet = AttributeSet(table.output)
+  override def withNewQuery(newQuery: LogicalPlan): OverwriteByExpression = {
+    copy(query = newQuery)
+  }
+  override def withNewTable(newTable: NamedRelation): OverwriteByExpression = {
+    copy(table = newTable)
+  }
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): OverwriteByExpression =
+    copy(query = newChild)
 }
 
 object OverwriteByExpression {
@@ -114,7 +147,18 @@ case class OverwritePartitionsDynamic(
     table: NamedRelation,
     query: LogicalPlan,
     writeOptions: Map[String, String],
-    isByName: Boolean) extends V2WriteCommand
+    isByName: Boolean,
+    write: Option[Write] = None) extends V2WriteCommand {
+  override def withNewQuery(newQuery: LogicalPlan): OverwritePartitionsDynamic = {
+    copy(query = newQuery)
+  }
+  override def withNewTable(newTable: NamedRelation): OverwritePartitionsDynamic = {
+    copy(table = newTable)
+  }
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): OverwritePartitionsDynamic =
+    copy(query = newChild)
+}
 
 object OverwritePartitionsDynamic {
   def byName(
@@ -155,7 +199,7 @@ case class CreateV2Table(
     tableSchema: StructType,
     partitioning: Seq[Transform],
     properties: Map[String, String],
-    ignoreIfExists: Boolean) extends Command with V2CreateTablePlan {
+    ignoreIfExists: Boolean) extends LeafCommand with V2CreateTablePlan {
   override def withPartitioning(rewritten: Seq[Transform]): V2CreateTablePlan = {
     this.copy(partitioning = rewritten)
   }
@@ -171,10 +215,10 @@ case class CreateTableAsSelect(
     query: LogicalPlan,
     properties: Map[String, String],
     writeOptions: Map[String, String],
-    ignoreIfExists: Boolean) extends Command with V2CreateTablePlan {
+    ignoreIfExists: Boolean) extends UnaryCommand with V2CreateTablePlan {
 
   override def tableSchema: StructType = query.schema
-  override def children: Seq[LogicalPlan] = Seq(query)
+  override def child: LogicalPlan = query
 
   override lazy val resolved: Boolean = childrenResolved && {
     // the table schema is created from the query schema, so the only resolution needed is to check
@@ -186,6 +230,9 @@ case class CreateTableAsSelect(
   override def withPartitioning(rewritten: Seq[Transform]): V2CreateTablePlan = {
     this.copy(partitioning = rewritten)
   }
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): CreateTableAsSelect =
+    copy(query = newChild)
 }
 
 /**
@@ -202,7 +249,7 @@ case class ReplaceTable(
     tableSchema: StructType,
     partitioning: Seq[Transform],
     properties: Map[String, String],
-    orCreate: Boolean) extends Command with V2CreateTablePlan {
+    orCreate: Boolean) extends LeafCommand with V2CreateTablePlan {
   override def withPartitioning(rewritten: Seq[Transform]): V2CreateTablePlan = {
     this.copy(partitioning = rewritten)
   }
@@ -221,10 +268,10 @@ case class ReplaceTableAsSelect(
     query: LogicalPlan,
     properties: Map[String, String],
     writeOptions: Map[String, String],
-    orCreate: Boolean) extends Command with V2CreateTablePlan {
+    orCreate: Boolean) extends UnaryCommand with V2CreateTablePlan {
 
   override def tableSchema: StructType = query.schema
-  override def children: Seq[LogicalPlan] = Seq(query)
+  override def child: LogicalPlan = query
 
   override lazy val resolved: Boolean = childrenResolved && {
     // the table schema is created from the query schema, so the only resolution needed is to check
@@ -236,161 +283,263 @@ case class ReplaceTableAsSelect(
   override def withPartitioning(rewritten: Seq[Transform]): V2CreateTablePlan = {
     this.copy(partitioning = rewritten)
   }
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): ReplaceTableAsSelect =
+    copy(query = newChild)
 }
 
 /**
- * The logical plan of the CREATE NAMESPACE command that works for v2 catalogs.
+ * The logical plan of the CREATE NAMESPACE command.
  */
 case class CreateNamespace(
     catalog: SupportsNamespaces,
     namespace: Seq[String],
     ifNotExists: Boolean,
-    properties: Map[String, String]) extends Command
+    properties: Map[String, String]) extends LeafCommand
 
 /**
- * The logical plan of the DROP NAMESPACE command that works for v2 catalogs.
+ * The logical plan of the DROP NAMESPACE command.
  */
 case class DropNamespace(
     namespace: LogicalPlan,
     ifExists: Boolean,
-    cascade: Boolean) extends Command {
-  override def children: Seq[LogicalPlan] = Seq(namespace)
+    cascade: Boolean) extends UnaryCommand {
+  override def child: LogicalPlan = namespace
+  override protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan =
+    copy(namespace = newChild)
 }
 
 /**
- * The logical plan of the DESCRIBE NAMESPACE command that works for v2 catalogs.
+ * The logical plan of the DESCRIBE NAMESPACE command.
  */
 case class DescribeNamespace(
     namespace: LogicalPlan,
-    extended: Boolean) extends Command {
-  override def children: Seq[LogicalPlan] = Seq(namespace)
+    extended: Boolean,
+    override val output: Seq[Attribute] = DescribeNamespace.getOutputAttrs) extends UnaryCommand {
+  override def child: LogicalPlan = namespace
+  override protected def withNewChildInternal(newChild: LogicalPlan): DescribeNamespace =
+    copy(namespace = newChild)
+}
 
-  override def output: Seq[Attribute] = Seq(
-    AttributeReference("name", StringType, nullable = false,
-      new MetadataBuilder().putString("comment", "name of the column").build())(),
-    AttributeReference("value", StringType, nullable = true,
-      new MetadataBuilder().putString("comment", "value of the column").build())())
+object DescribeNamespace {
+  def getOutputAttrs: Seq[Attribute] = Seq(
+    AttributeReference("info_name", StringType, nullable = false,
+      new MetadataBuilder().putString("comment", "name of the namespace info").build())(),
+    AttributeReference("info_value", StringType, nullable = true,
+      new MetadataBuilder().putString("comment", "value of the namespace info").build())())
 }
 
 /**
  * The logical plan of the ALTER (DATABASE|SCHEMA|NAMESPACE) ... SET (DBPROPERTIES|PROPERTIES)
- * command that works for v2 catalogs.
+ * command.
  */
-case class AlterNamespaceSetProperties(
+case class SetNamespaceProperties(
     namespace: LogicalPlan,
-    properties: Map[String, String]) extends Command {
-  override def children: Seq[LogicalPlan] = Seq(namespace)
+    properties: Map[String, String]) extends UnaryCommand {
+  override def child: LogicalPlan = namespace
+  override protected def withNewChildInternal(newChild: LogicalPlan): SetNamespaceProperties =
+    copy(namespace = newChild)
 }
 
 /**
- * The logical plan of the ALTER (DATABASE|SCHEMA|NAMESPACE) ... SET LOCATION
- * command that works for v2 catalogs.
+ * The logical plan of the ALTER (DATABASE|SCHEMA|NAMESPACE) ... SET LOCATION command.
  */
-case class AlterNamespaceSetLocation(
+case class SetNamespaceLocation(
     namespace: LogicalPlan,
-    location: String) extends Command {
-  override def children: Seq[LogicalPlan] = Seq(namespace)
+    location: String) extends UnaryCommand {
+  override def child: LogicalPlan = namespace
+  override protected def withNewChildInternal(newChild: LogicalPlan): SetNamespaceLocation =
+    copy(namespace = newChild)
 }
 
 /**
- * The logical plan of the SHOW NAMESPACES command that works for v2 catalogs.
+ * The logical plan of the SHOW NAMESPACES command.
  */
 case class ShowNamespaces(
     namespace: LogicalPlan,
-    pattern: Option[String]) extends Command {
-  override def children: Seq[LogicalPlan] = Seq(namespace)
+    pattern: Option[String],
+    override val output: Seq[Attribute] = ShowNamespaces.getOutputAttrs) extends UnaryCommand {
+  override def child: LogicalPlan = namespace
+  override protected def withNewChildInternal(newChild: LogicalPlan): ShowNamespaces =
+    copy(namespace = newChild)
+}
 
-  override val output: Seq[Attribute] = Seq(
-    AttributeReference("namespace", StringType, nullable = false)())
+object ShowNamespaces {
+  def getOutputAttrs: Seq[Attribute] = {
+    Seq(AttributeReference("namespace", StringType, nullable = false)())
+  }
 }
 
 /**
- * The logical plan of the DESCRIBE relation_name command that works for v2 tables.
+ * The logical plan of the DESCRIBE relation_name command.
  */
 case class DescribeRelation(
     relation: LogicalPlan,
     partitionSpec: TablePartitionSpec,
-    isExtended: Boolean) extends Command {
-  override def children: Seq[LogicalPlan] = Seq(relation)
-  override def output: Seq[Attribute] = DescribeTableSchema.describeTableAttributes()
+    isExtended: Boolean,
+    override val output: Seq[Attribute] = DescribeRelation.getOutputAttrs) extends UnaryCommand {
+  override def child: LogicalPlan = relation
+  override protected def withNewChildInternal(newChild: LogicalPlan): DescribeRelation =
+    copy(relation = newChild)
+}
+
+object DescribeRelation {
+  def getOutputAttrs: Seq[Attribute] = DescribeCommandSchema.describeTableAttributes()
 }
 
 /**
- * The logical plan of the DELETE FROM command that works for v2 tables.
+ * The logical plan of the DESCRIBE relation_name col_name command.
+ */
+case class DescribeColumn(
+    relation: LogicalPlan,
+    column: Expression,
+    isExtended: Boolean,
+    override val output: Seq[Attribute] = DescribeColumn.getOutputAttrs) extends UnaryCommand {
+  override def child: LogicalPlan = relation
+  override protected def withNewChildInternal(newChild: LogicalPlan): DescribeColumn =
+    copy(relation = newChild)
+}
+
+object DescribeColumn {
+  def getOutputAttrs: Seq[Attribute] = DescribeCommandSchema.describeColumnAttributes()
+}
+
+/**
+ * The logical plan of the DELETE FROM command.
  */
 case class DeleteFromTable(
     table: LogicalPlan,
-    condition: Option[Expression]) extends Command with SupportsSubquery {
-  override def children: Seq[LogicalPlan] = table :: Nil
+    condition: Option[Expression]) extends UnaryCommand with SupportsSubquery {
+  override def child: LogicalPlan = table
+  override protected def withNewChildInternal(newChild: LogicalPlan): DeleteFromTable =
+    copy(table = newChild)
 }
 
 /**
- * The logical plan of the UPDATE TABLE command that works for v2 tables.
+ * The logical plan of the UPDATE TABLE command.
  */
 case class UpdateTable(
     table: LogicalPlan,
     assignments: Seq[Assignment],
-    condition: Option[Expression]) extends Command with SupportsSubquery {
-  override def children: Seq[LogicalPlan] = table :: Nil
+    condition: Option[Expression]) extends UnaryCommand with SupportsSubquery {
+  override def child: LogicalPlan = table
+  override protected def withNewChildInternal(newChild: LogicalPlan): UpdateTable =
+    copy(table = newChild)
 }
 
 /**
- * The logical plan of the MERGE INTO command that works for v2 tables.
+ * The logical plan of the MERGE INTO command.
  */
 case class MergeIntoTable(
     targetTable: LogicalPlan,
     sourceTable: LogicalPlan,
     mergeCondition: Expression,
     matchedActions: Seq[MergeAction],
-    notMatchedActions: Seq[MergeAction]) extends Command with SupportsSubquery {
-  override def children: Seq[LogicalPlan] = Seq(targetTable, sourceTable)
+    notMatchedActions: Seq[MergeAction]) extends BinaryCommand with SupportsSubquery {
+  def duplicateResolved: Boolean = targetTable.outputSet.intersect(sourceTable.outputSet).isEmpty
+  override def left: LogicalPlan = targetTable
+  override def right: LogicalPlan = sourceTable
+  override protected def withNewChildrenInternal(
+      newLeft: LogicalPlan, newRight: LogicalPlan): MergeIntoTable =
+    copy(targetTable = newLeft, sourceTable = newRight)
 }
 
-sealed abstract class MergeAction(
-    condition: Option[Expression]) extends Expression with Unevaluable {
-  override def foldable: Boolean = false
+sealed abstract class MergeAction extends Expression with Unevaluable {
+  def condition: Option[Expression]
   override def nullable: Boolean = false
-  override def dataType: DataType = throw new UnresolvedException(this, "nullable")
+  override def dataType: DataType = throw new UnresolvedException("nullable")
   override def children: Seq[Expression] = condition.toSeq
 }
 
-case class DeleteAction(condition: Option[Expression]) extends MergeAction(condition)
+case class DeleteAction(condition: Option[Expression]) extends MergeAction {
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[Expression]): DeleteAction =
+    copy(condition = if (condition.isDefined) Some(newChildren(0)) else None)
+}
 
 case class UpdateAction(
     condition: Option[Expression],
-    assignments: Seq[Assignment]) extends MergeAction(condition) {
+    assignments: Seq[Assignment]) extends MergeAction {
   override def children: Seq[Expression] = condition.toSeq ++ assignments
+
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[Expression]): UpdateAction =
+    copy(
+      condition = if (condition.isDefined) Some(newChildren.head) else None,
+      assignments = newChildren.tail.asInstanceOf[Seq[Assignment]])
+}
+
+case class UpdateStarAction(condition: Option[Expression]) extends MergeAction {
+  override def children: Seq[Expression] = condition.toSeq
+  override lazy val resolved = false
+  override protected def withNewChildrenInternal(
+    newChildren: IndexedSeq[Expression]): UpdateStarAction =
+  copy(condition = if (condition.isDefined) Some(newChildren(0)) else None)
 }
 
 case class InsertAction(
     condition: Option[Expression],
-    assignments: Seq[Assignment]) extends MergeAction(condition) {
+    assignments: Seq[Assignment]) extends MergeAction {
   override def children: Seq[Expression] = condition.toSeq ++ assignments
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[Expression]): InsertAction =
+    copy(
+      condition = if (condition.isDefined) Some(newChildren.head) else None,
+      assignments = newChildren.tail.asInstanceOf[Seq[Assignment]])
+}
+
+case class InsertStarAction(condition: Option[Expression]) extends MergeAction {
+  override def children: Seq[Expression] = condition.toSeq
+  override lazy val resolved = false
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[Expression]): InsertStarAction =
+    copy(condition = if (condition.isDefined) Some(newChildren(0)) else None)
 }
 
-case class Assignment(key: Expression, value: Expression) extends Expression with Unevaluable {
-  override def foldable: Boolean = false
+case class Assignment(key: Expression, value: Expression) extends Expression
+  with Unevaluable with BinaryLike[Expression] {
   override def nullable: Boolean = false
-  override def dataType: DataType = throw new UnresolvedException(this, "nullable")
-  override def children: Seq[Expression] = key ::  value :: Nil
+  override def dataType: DataType = throw new UnresolvedException("nullable")
+  override def left: Expression = key
+  override def right: Expression = value
+  override protected def withNewChildrenInternal(
+    newLeft: Expression, newRight: Expression): Assignment = copy(key = newLeft, value = newRight)
 }
 
 /**
- * The logical plan of the DROP TABLE command that works for v2 tables.
+ * The logical plan of the DROP TABLE command.
+ *
+ * If the `PURGE` option is set, the table catalog must remove table data by skipping the trash
+ * even when the catalog has configured one. The option is applicable only for managed tables.
+ *
+ * The syntax of this command is:
+ * {{{
+ *     DROP TABLE [IF EXISTS] table [PURGE];
+ * }}}
  */
 case class DropTable(
-    catalog: TableCatalog,
-    ident: Identifier,
-    ifExists: Boolean) extends Command
+    child: LogicalPlan,
+    ifExists: Boolean,
+    purge: Boolean) extends UnaryCommand {
+  override protected def withNewChildInternal(newChild: LogicalPlan): DropTable =
+    copy(child = newChild)
+}
 
 /**
- * The logical plan of the ALTER TABLE command that works for v2 tables.
+ * The logical plan for no-op command handling non-existing table.
+ */
+case class NoopCommand(
+    commandName: String,
+    multipartIdentifier: Seq[String]) extends LeafCommand
+
+/**
+ * The logical plan of the ALTER TABLE command.
  */
 case class AlterTable(
     catalog: TableCatalog,
     ident: Identifier,
     table: NamedRelation,
-    changes: Seq[TableChange]) extends Command {
+    changes: Seq[TableChange]) extends LeafCommand {
 
   override lazy val resolved: Boolean = table.resolved && {
     changes.forall {
@@ -416,75 +565,117 @@ case class AlterTable(
 }
 
 /**
- * The logical plan of the ALTER TABLE RENAME command that works for v2 tables.
+ * The logical plan of the ALTER [TABLE|VIEW] ... RENAME TO command.
  */
 case class RenameTable(
-    catalog: TableCatalog,
-    oldIdent: Identifier,
-    newIdent: Identifier) extends Command
+    child: LogicalPlan,
+    newName: Seq[String],
+    isView: Boolean) extends UnaryCommand {
+  override protected def withNewChildInternal(newChild: LogicalPlan): RenameTable =
+    copy(child = newChild)
+}
 
 /**
- * The logical plan of the SHOW TABLE command that works for v2 catalogs.
+ * The logical plan of the SHOW TABLES command.
  */
 case class ShowTables(
     namespace: LogicalPlan,
-    pattern: Option[String]) extends Command {
-  override def children: Seq[LogicalPlan] = Seq(namespace)
+    pattern: Option[String],
+    override val output: Seq[Attribute] = ShowTables.getOutputAttrs) extends UnaryCommand {
+  override def child: LogicalPlan = namespace
+  override protected def withNewChildInternal(newChild: LogicalPlan): ShowTables =
+    copy(namespace = newChild)
+}
 
-  override val output: Seq[Attribute] = Seq(
+object ShowTables {
+  def getOutputAttrs: Seq[Attribute] = Seq(
     AttributeReference("namespace", StringType, nullable = false)(),
-    AttributeReference("tableName", StringType, nullable = false)())
+    AttributeReference("tableName", StringType, nullable = false)(),
+    AttributeReference("isTemporary", BooleanType, nullable = false)())
 }
 
 /**
- * The logical plan of the SHOW VIEWS command that works for v1 and v2 catalogs.
+ * The logical plan of the SHOW TABLE EXTENDED command.
+ */
+case class ShowTableExtended(
+    namespace: LogicalPlan,
+    pattern: String,
+    partitionSpec: Option[PartitionSpec],
+    override val output: Seq[Attribute] = ShowTableExtended.getOutputAttrs) extends UnaryCommand {
+  override def child: LogicalPlan = namespace
+  override protected def withNewChildInternal(newChild: LogicalPlan): ShowTableExtended =
+    copy(namespace = newChild)
+}
+
+object ShowTableExtended {
+  def getOutputAttrs: Seq[Attribute] = Seq(
+    AttributeReference("namespace", StringType, nullable = false)(),
+    AttributeReference("tableName", StringType, nullable = false)(),
+    AttributeReference("isTemporary", BooleanType, nullable = false)(),
+    AttributeReference("information", StringType, nullable = false)())
+}
+
+/**
+ * The logical plan of the SHOW VIEWS command.
  *
  * Notes: v2 catalogs do not support views API yet, the command will fallback to
  * v1 ShowViewsCommand during ResolveSessionCatalog.
  */
 case class ShowViews(
     namespace: LogicalPlan,
-    pattern: Option[String]) extends Command {
-  override def children: Seq[LogicalPlan] = Seq(namespace)
+    pattern: Option[String],
+    override val output: Seq[Attribute] = ShowViews.getOutputAttrs) extends UnaryCommand {
+  override def child: LogicalPlan = namespace
+  override protected def withNewChildInternal(newChild: LogicalPlan): ShowViews =
+    copy(namespace = newChild)
+}
 
-  override val output: Seq[Attribute] = Seq(
+object ShowViews {
+  def getOutputAttrs: Seq[Attribute] = Seq(
     AttributeReference("namespace", StringType, nullable = false)(),
-    AttributeReference("viewName", StringType, nullable = false)())
+    AttributeReference("viewName", StringType, nullable = false)(),
+    AttributeReference("isTemporary", BooleanType, nullable = false)())
 }
 
 /**
- * The logical plan of the USE/USE NAMESPACE command that works for v2 catalogs.
+ * The logical plan of the USE/USE NAMESPACE command.
  */
 case class SetCatalogAndNamespace(
     catalogManager: CatalogManager,
     catalogName: Option[String],
-    namespace: Option[Seq[String]]) extends Command
+    namespace: Option[Seq[String]]) extends LeafCommand
 
 /**
- * The logical plan of the REFRESH TABLE command that works for v2 catalogs.
+ * The logical plan of the REFRESH TABLE command.
  */
-case class RefreshTable(
-    catalog: TableCatalog,
-    ident: Identifier) extends Command
+case class RefreshTable(child: LogicalPlan) extends UnaryCommand {
+  override protected def withNewChildInternal(newChild: LogicalPlan): RefreshTable =
+    copy(child = newChild)
+}
 
 /**
- * The logical plan of the SHOW CURRENT NAMESPACE command that works for v2 catalogs.
+ * The logical plan of the SHOW CURRENT NAMESPACE command.
  */
-case class ShowCurrentNamespace(catalogManager: CatalogManager) extends Command {
+case class ShowCurrentNamespace(catalogManager: CatalogManager) extends LeafCommand {
   override val output: Seq[Attribute] = Seq(
     AttributeReference("catalog", StringType, nullable = false)(),
     AttributeReference("namespace", StringType, nullable = false)())
 }
 
 /**
- * The logical plan of the SHOW TBLPROPERTIES command that works for v2 catalogs.
+ * The logical plan of the SHOW TBLPROPERTIES command.
  */
 case class ShowTableProperties(
     table: LogicalPlan,
-    propertyKey: Option[String]) extends Command {
-  override def children: Seq[LogicalPlan] = table :: Nil
+    propertyKey: Option[String],
+    override val output: Seq[Attribute] = ShowTableProperties.getOutputAttrs) extends UnaryCommand {
+  override def child: LogicalPlan = table
+  override protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan =
+    copy(table = newChild)
+}
 
-  override val output: Seq[Attribute] = Seq(
+object ShowTableProperties {
+  def getOutputAttrs: Seq[Attribute] = Seq(
     AttributeReference("key", StringType, nullable = false)(),
     AttributeReference("value", StringType, nullable = false)())
 }
@@ -499,8 +690,9 @@ case class ShowTableProperties(
  * where the `text` is the new comment written as a string literal; or `NULL` to drop the comment.
  *
  */
-case class CommentOnNamespace(child: LogicalPlan, comment: String) extends Command {
-  override def children: Seq[LogicalPlan] = child :: Nil
+case class CommentOnNamespace(child: LogicalPlan, comment: String) extends UnaryCommand {
+  override protected def withNewChildInternal(newChild: LogicalPlan): CommentOnNamespace =
+    copy(child = newChild)
 }
 
 /**
@@ -513,6 +705,396 @@ case class CommentOnNamespace(child: LogicalPlan, comment: String) extends Comma
  * where the `text` is the new comment written as a string literal; or `NULL` to drop the comment.
  *
  */
-case class CommentOnTable(child: LogicalPlan, comment: String) extends Command {
-  override def children: Seq[LogicalPlan] = child :: Nil
+case class CommentOnTable(child: LogicalPlan, comment: String) extends UnaryCommand {
+  override protected def withNewChildInternal(newChild: LogicalPlan): CommentOnTable =
+    copy(child = newChild)
+}
+
+/**
+ * The logical plan of the REFRESH FUNCTION command.
+ */
+case class RefreshFunction(child: LogicalPlan) extends UnaryCommand {
+  override protected def withNewChildInternal(newChild: LogicalPlan): RefreshFunction =
+    copy(child = newChild)
+}
+
+/**
+ * The logical plan of the DESCRIBE FUNCTION command.
+ */
+case class DescribeFunction(child: LogicalPlan, isExtended: Boolean) extends UnaryCommand {
+  override protected def withNewChildInternal(newChild: LogicalPlan): DescribeFunction =
+    copy(child = newChild)
+}
+
+/**
+ * The logical plan of the DROP FUNCTION command.
+ */
+case class DropFunction(
+    child: LogicalPlan,
+    ifExists: Boolean,
+    isTemp: Boolean) extends UnaryCommand {
+  override protected def withNewChildInternal(newChild: LogicalPlan): DropFunction =
+    copy(child = newChild)
+}
+
+/**
+ * The logical plan of the SHOW FUNCTIONS command.
+ */
+case class ShowFunctions(
+    child: Option[LogicalPlan],
+    userScope: Boolean,
+    systemScope: Boolean,
+    pattern: Option[String],
+    override val output: Seq[Attribute] = ShowFunctions.getOutputAttrs) extends Command {
+  override def children: Seq[LogicalPlan] = child.toSeq
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[LogicalPlan]): ShowFunctions =
+    copy(child = if (child.isDefined) Some(newChildren.head) else None)
+}
+
+object ShowFunctions {
+  def getOutputAttrs: Seq[Attribute] = {
+    Seq(AttributeReference("function", StringType, nullable = false)())
+  }
+}
+
+/**
+ * The logical plan of the ANALYZE TABLE command.
+ */
+case class AnalyzeTable(
+    child: LogicalPlan,
+    partitionSpec: Map[String, Option[String]],
+    noScan: Boolean) extends UnaryCommand {
+  override protected def withNewChildInternal(newChild: LogicalPlan): AnalyzeTable =
+    copy(child = newChild)
+}
+
+/**
+ * The logical plan of the ANALYZE TABLES command.
+ */
+case class AnalyzeTables(
+    namespace: LogicalPlan,
+    noScan: Boolean) extends UnaryCommand {
+  override def child: LogicalPlan = namespace
+  override protected def withNewChildInternal(newChild: LogicalPlan): AnalyzeTables =
+    copy(namespace = newChild)
+}
+
+/**
+ * The logical plan of the ANALYZE TABLE FOR COLUMNS command.
+ */
+case class AnalyzeColumn(
+    child: LogicalPlan,
+    columnNames: Option[Seq[String]],
+    allColumns: Boolean) extends UnaryCommand {
+  require(columnNames.isDefined ^ allColumns, "Parameter `columnNames` or `allColumns` are " +
+    "mutually exclusive. Only one of them should be specified.")
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): AnalyzeColumn =
+    copy(child = newChild)
+}
+
+/**
+ * The logical plan of the ALTER TABLE ADD PARTITION command.
+ *
+ * The syntax of this command is:
+ * {{{
+ *     ALTER TABLE table ADD [IF NOT EXISTS]
+ *                 PARTITION spec1 [LOCATION 'loc1'][, PARTITION spec2 [LOCATION 'loc2'], ...];
+ * }}}
+ */
+case class AddPartitions(
+    table: LogicalPlan,
+    parts: Seq[PartitionSpec],
+    ifNotExists: Boolean) extends V2PartitionCommand {
+  override protected def withNewChildInternal(newChild: LogicalPlan): AddPartitions =
+    copy(table = newChild)
+}
+
+/**
+ * The logical plan of the ALTER TABLE DROP PARTITION command.
+ * This may remove the data and metadata for this partition.
+ *
+ * If the `PURGE` option is set, the table catalog must remove partition data by skipping the trash
+ * even when the catalog has configured one. The option is applicable only for managed tables.
+ *
+ * The syntax of this command is:
+ * {{{
+ *     ALTER TABLE table DROP [IF EXISTS] PARTITION spec1[, PARTITION spec2, ...] [PURGE];
+ * }}}
+ */
+case class DropPartitions(
+    table: LogicalPlan,
+    parts: Seq[PartitionSpec],
+    ifExists: Boolean,
+    purge: Boolean) extends V2PartitionCommand {
+  override protected def withNewChildInternal(newChild: LogicalPlan): DropPartitions =
+    copy(table = newChild)
+}
+
+/**
+ * The logical plan of the ALTER TABLE ... RENAME TO PARTITION command.
+ */
+case class RenamePartitions(
+    table: LogicalPlan,
+    from: PartitionSpec,
+    to: PartitionSpec) extends V2PartitionCommand {
+  override protected def withNewChildInternal(newChild: LogicalPlan): RenamePartitions =
+    copy(table = newChild)
+}
+
+/**
+ * The logical plan of the ALTER TABLE ... RECOVER PARTITIONS command.
+ */
+case class RecoverPartitions(child: LogicalPlan) extends UnaryCommand {
+  override protected def withNewChildInternal(newChild: LogicalPlan): RecoverPartitions =
+    copy(child = newChild)
+}
+
+/**
+ * The logical plan of the LOAD DATA INTO TABLE command.
+ */
+case class LoadData(
+    child: LogicalPlan,
+    path: String,
+    isLocal: Boolean,
+    isOverwrite: Boolean,
+    partition: Option[TablePartitionSpec]) extends UnaryCommand {
+  override protected def withNewChildInternal(newChild: LogicalPlan): LoadData =
+    copy(child = newChild)
+}
+
+/**
+ * The logical plan of the SHOW CREATE TABLE command.
+ */
+case class ShowCreateTable(
+    child: LogicalPlan,
+    asSerde: Boolean = false,
+    override val output: Seq[Attribute] = ShowCreateTable.getoutputAttrs) extends UnaryCommand {
+  override protected def withNewChildInternal(newChild: LogicalPlan): ShowCreateTable =
+    copy(child = newChild)
+}
+
+object ShowCreateTable {
+  def getoutputAttrs: Seq[Attribute] = {
+    Seq(AttributeReference("createtab_stmt", StringType, nullable = false)())
+  }
+}
+
+/**
+ * The logical plan of the SHOW COLUMN command.
+ */
+case class ShowColumns(
+    child: LogicalPlan,
+    namespace: Option[Seq[String]],
+    override val output: Seq[Attribute] = ShowColumns.getOutputAttrs) extends UnaryCommand {
+  override protected def withNewChildInternal(newChild: LogicalPlan): ShowColumns =
+    copy(child = newChild)
+}
+
+object ShowColumns {
+  def getOutputAttrs: Seq[Attribute] = {
+    Seq(AttributeReference("col_name", StringType, nullable = false)())
+  }
+}
+
+/**
+ * The logical plan of the TRUNCATE TABLE command.
+ */
+case class TruncateTable(table: LogicalPlan) extends UnaryCommand {
+  override def child: LogicalPlan = table
+  override protected def withNewChildInternal(newChild: LogicalPlan): TruncateTable =
+    copy(table = newChild)
+}
+
+/**
+ * The logical plan of the TRUNCATE TABLE ... PARTITION command.
+ */
+case class TruncatePartition(
+    table: LogicalPlan,
+    partitionSpec: PartitionSpec) extends V2PartitionCommand {
+  override def allowPartialPartitionSpec: Boolean = true
+  override protected def withNewChildInternal(newChild: LogicalPlan): TruncatePartition =
+    copy(table = newChild)
+}
+
+/**
+ * The logical plan of the SHOW PARTITIONS command.
+ */
+case class ShowPartitions(
+    table: LogicalPlan,
+    pattern: Option[PartitionSpec],
+    override val output: Seq[Attribute] = ShowPartitions.getOutputAttrs)
+  extends V2PartitionCommand {
+  override def allowPartialPartitionSpec: Boolean = true
+  override protected def withNewChildInternal(newChild: LogicalPlan): ShowPartitions =
+    copy(table = newChild)
+}
+
+object ShowPartitions {
+  def getOutputAttrs: Seq[Attribute] = {
+    Seq(AttributeReference("partition", StringType, nullable = false)())
+  }
+}
+
+/**
+ * The logical plan of the DROP VIEW command.
+ */
+case class DropView(
+    child: LogicalPlan,
+    ifExists: Boolean) extends UnaryCommand {
+  override protected def withNewChildInternal(newChild: LogicalPlan): DropView =
+    copy(child = newChild)
+}
+
+/**
+ * The logical plan of the MSCK REPAIR TABLE command.
+ */
+case class RepairTable(
+    child: LogicalPlan,
+    enableAddPartitions: Boolean,
+    enableDropPartitions: Boolean) extends UnaryCommand {
+  override protected def withNewChildInternal(newChild: LogicalPlan): RepairTable =
+    copy(child = newChild)
+}
+
+/**
+ * The logical plan of the ALTER VIEW ... AS command.
+ */
+case class AlterViewAs(
+    child: LogicalPlan,
+    originalText: String,
+    query: LogicalPlan) extends BinaryCommand {
+  override def left: LogicalPlan = child
+  override def right: LogicalPlan = query
+  override protected def withNewChildrenInternal(
+      newLeft: LogicalPlan, newRight: LogicalPlan): LogicalPlan =
+    copy(child = newLeft, query = newRight)
+}
+
+/**
+ * The logical plan of the ALTER VIEW ... SET TBLPROPERTIES command.
+ */
+case class SetViewProperties(
+    child: LogicalPlan,
+    properties: Map[String, String]) extends UnaryCommand {
+  override protected def withNewChildInternal(newChild: LogicalPlan): SetViewProperties =
+    copy(child = newChild)
+}
+
+/**
+ * The logical plan of the ALTER VIEW ... UNSET TBLPROPERTIES command.
+ */
+case class UnsetViewProperties(
+    child: LogicalPlan,
+    propertyKeys: Seq[String],
+    ifExists: Boolean) extends UnaryCommand {
+  override protected def withNewChildInternal(newChild: LogicalPlan): UnsetViewProperties =
+    copy(child = newChild)
+}
+
+/**
+ * The logical plan of the ALTER TABLE ... SET [SERDE|SERDEPROPERTIES] command.
+ */
+case class SetTableSerDeProperties(
+    child: LogicalPlan,
+    serdeClassName: Option[String],
+    serdeProperties: Option[Map[String, String]],
+    partitionSpec: Option[TablePartitionSpec]) extends UnaryCommand {
+  override protected def withNewChildInternal(newChild: LogicalPlan): SetTableSerDeProperties =
+    copy(child = newChild)
+}
+
+/**
+ * The logical plan of the CACHE TABLE command.
+ */
+case class CacheTable(
+    table: LogicalPlan,
+    multipartIdentifier: Seq[String],
+    isLazy: Boolean,
+    options: Map[String, String],
+    isAnalyzed: Boolean = false) extends AnalysisOnlyCommand {
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[LogicalPlan]): CacheTable = {
+    assert(!isAnalyzed)
+    copy(table = newChildren.head)
+  }
+
+  override def childrenToAnalyze: Seq[LogicalPlan] = table :: Nil
+
+  override def markAsAnalyzed(): LogicalPlan = copy(isAnalyzed = true)
+}
+
+/**
+ * The logical plan of the CACHE TABLE ... AS SELECT command.
+ */
+case class CacheTableAsSelect(
+    tempViewName: String,
+    plan: LogicalPlan,
+    originalText: String,
+    isLazy: Boolean,
+    options: Map[String, String],
+    isAnalyzed: Boolean = false) extends AnalysisOnlyCommand {
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[LogicalPlan]): CacheTableAsSelect = {
+    assert(!isAnalyzed)
+    copy(plan = newChildren.head)
+  }
+
+  override def childrenToAnalyze: Seq[LogicalPlan] = plan :: Nil
+
+  override def markAsAnalyzed(): LogicalPlan = copy(isAnalyzed = true)
+}
+
+/**
+ * The logical plan of the UNCACHE TABLE command.
+ */
+case class UncacheTable(
+    table: LogicalPlan,
+    ifExists: Boolean,
+    isAnalyzed: Boolean = false) extends AnalysisOnlyCommand {
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[LogicalPlan]): UncacheTable = {
+    assert(!isAnalyzed)
+    copy(table = newChildren.head)
+  }
+
+  override def childrenToAnalyze: Seq[LogicalPlan] = table :: Nil
+
+  override def markAsAnalyzed(): LogicalPlan = copy(isAnalyzed = true)
+}
+
+/**
+ * The logical plan of the ALTER TABLE ... SET LOCATION command.
+ */
+case class SetTableLocation(
+    table: LogicalPlan,
+    partitionSpec: Option[TablePartitionSpec],
+    location: String) extends UnaryCommand {
+  override def child: LogicalPlan = table
+  override protected def withNewChildInternal(newChild: LogicalPlan): SetTableLocation =
+    copy(table = newChild)
+}
+
+/**
+ * The logical plan of the ALTER TABLE ... SET TBLPROPERTIES command.
+ */
+case class SetTableProperties(
+    table: LogicalPlan,
+    properties: Map[String, String]) extends UnaryCommand {
+  override def child: LogicalPlan = table
+  override protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan =
+    copy(table = newChild)
+}
+
+/**
+ * The logical plan of the ALTER TABLE ... UNSET TBLPROPERTIES command.
+ */
+case class UnsetTableProperties(
+    table: LogicalPlan,
+    propertyKeys: Seq[String],
+    ifExists: Boolean) extends UnaryCommand {
+  override def child: LogicalPlan = table
+  override protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan =
+    copy(table = newChild)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala
index 17e1cb416fc8a..0f8c7887b2b1c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala
@@ -164,7 +164,7 @@ trait Partitioning {
    * i.e. the current dataset does not need to be re-partitioned for the `required`
    * Distribution (it is possible that tuples within a partition need to be reorganized).
    *
-   * A [[Partitioning]] can never satisfy a [[Distribution]] if its `numPartitions` does't match
+   * A [[Partitioning]] can never satisfy a [[Distribution]] if its `numPartitions` doesn't match
    * [[Distribution.requiredNumPartitions]].
    */
   final def satisfies(required: Distribution): Boolean = {
@@ -235,6 +235,9 @@ case class HashPartitioning(expressions: Seq[Expression], numPartitions: Int)
    * than numPartitions) based on hashing expressions.
    */
   def partitionIdExpression: Expression = Pmod(new Murmur3Hash(expressions), Literal(numPartitions))
+
+  override protected def withNewChildrenInternal(
+    newChildren: IndexedSeq[Expression]): HashPartitioning = copy(expressions = newChildren)
 }
 
 /**
@@ -284,6 +287,10 @@ case class RangePartitioning(ordering: Seq[SortOrder], numPartitions: Int)
       }
     }
   }
+
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[Expression]): RangePartitioning =
+    copy(ordering = newChildren.asInstanceOf[Seq[SortOrder]])
 }
 
 /**
@@ -326,6 +333,10 @@ case class PartitioningCollection(partitionings: Seq[Partitioning])
   override def toString: String = {
     partitionings.map(_.toString).mkString("(", " or ", ")")
   }
+
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[Expression]): PartitioningCollection =
+    super.legacyWithNewChildren(newChildren).asInstanceOf[PartitioningCollection]
 }
 
 /**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/Rule.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/Rule.scala
index 7eb72724d7663..e263f827d2847 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/Rule.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/Rule.scala
@@ -18,9 +18,13 @@
 package org.apache.spark.sql.catalyst.rules
 
 import org.apache.spark.internal.Logging
+import org.apache.spark.sql.catalyst.SQLConfHelper
 import org.apache.spark.sql.catalyst.trees.TreeNode
 
-abstract class Rule[TreeType <: TreeNode[_]] extends Logging {
+abstract class Rule[TreeType <: TreeNode[_]] extends SQLConfHelper with Logging {
+
+  // The integer id of a rule, for pruning unnecessary tree traversals.
+  protected lazy val ruleId = RuleIdCollection.getRuleId(this.ruleName)
 
   /** Name for this rule, automatically inferred based on class name. */
   val ruleName: String = {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala
index bff04d317d4d2..118ed85624d89 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.catalyst.rules
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.QueryPlanningTracker
-import org.apache.spark.sql.catalyst.errors.TreeNodeException
 import org.apache.spark.sql.catalyst.trees.TreeNode
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.NANOS_PER_SECOND
 import org.apache.spark.sql.catalyst.util.sideBySide
@@ -44,6 +43,73 @@ object RuleExecutor {
   }
 }
 
+class PlanChangeLogger[TreeType <: TreeNode[_]] extends Logging {
+
+  private val logLevel = SQLConf.get.planChangeLogLevel
+
+  private val logRules = SQLConf.get.planChangeRules.map(Utils.stringToSeq)
+
+  private val logBatches = SQLConf.get.planChangeBatches.map(Utils.stringToSeq)
+
+  def logRule(ruleName: String, oldPlan: TreeType, newPlan: TreeType): Unit = {
+    if (!newPlan.fastEquals(oldPlan)) {
+      if (logRules.isEmpty || logRules.get.contains(ruleName)) {
+        def message(): String = {
+          s"""
+             |=== Applying Rule $ruleName ===
+             |${sideBySide(oldPlan.treeString, newPlan.treeString).mkString("\n")}
+           """.stripMargin
+        }
+
+        logBasedOnLevel(message)
+      }
+    }
+  }
+
+  def logBatch(batchName: String, oldPlan: TreeType, newPlan: TreeType): Unit = {
+    if (logBatches.isEmpty || logBatches.get.contains(batchName)) {
+      def message(): String = {
+        if (!oldPlan.fastEquals(newPlan)) {
+          s"""
+             |=== Result of Batch $batchName ===
+             |${sideBySide(oldPlan.treeString, newPlan.treeString).mkString("\n")}
+          """.stripMargin
+        } else {
+          s"Batch $batchName has no effect."
+        }
+      }
+
+      logBasedOnLevel(message)
+    }
+  }
+
+  def logMetrics(metrics: QueryExecutionMetrics): Unit = {
+    val totalTime = metrics.time / NANOS_PER_SECOND.toDouble
+    val totalTimeEffective = metrics.timeEffective / NANOS_PER_SECOND.toDouble
+    val message =
+      s"""
+         |=== Metrics of Executed Rules ===
+         |Total number of runs: ${metrics.numRuns}
+         |Total time: $totalTime seconds
+         |Total number of effective runs: ${metrics.numEffectiveRuns}
+         |Total time of effective runs: $totalTimeEffective seconds
+      """.stripMargin
+
+    logBasedOnLevel(message)
+  }
+
+  private def logBasedOnLevel(f: => String): Unit = {
+    logLevel match {
+      case "TRACE" => logTrace(f)
+      case "DEBUG" => logDebug(f)
+      case "INFO" => logInfo(f)
+      case "WARN" => logWarning(f)
+      case "ERROR" => logError(f)
+      case _ => logTrace(f)
+    }
+  }
+}
+
 abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
 
   /**
@@ -80,8 +146,8 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
   /** Defines a sequence of rule batches, to be overridden by the implementation. */
   protected def batches: Seq[Batch]
 
-  /** Once batches that are blacklisted in the idempotence checker */
-  protected val blacklistedOnceBatches: Set[String] = Set.empty
+  /** Once batches that are excluded in the idempotence checker */
+  protected val excludedOnceBatches: Set[String] = Set.empty
 
   /**
    * Defines a check function that checks for structural integrity of the plan after the execution
@@ -102,7 +168,7 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
            |Once strategy's idempotence is broken for batch ${batch.name}
            |${sideBySide(plan.treeString, reOptimized.treeString).mkString("\n")}
           """.stripMargin
-      throw new TreeNodeException(reOptimized, message, null)
+      throw new RuntimeException(message)
     }
   }
 
@@ -124,7 +190,7 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
   def execute(plan: TreeType): TreeType = {
     var curPlan = plan
     val queryExecutionMetrics = RuleExecutor.queryExecutionMeter
-    val planChangeLogger = new PlanChangeLogger()
+    val planChangeLogger = new PlanChangeLogger[TreeType]()
     val tracker: Option[QueryPlanningTracker] = QueryPlanningTracker.get
     val beforeMetrics = RuleExecutor.getCurrentMetrics()
 
@@ -132,7 +198,7 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
     if (!isPlanIntegral(plan)) {
       val message = "The structural integrity of the input plan is broken in " +
         s"${this.getClass.getName.stripSuffix("$")}."
-      throw new TreeNodeException(plan, message, null)
+      throw new RuntimeException(message)
     }
 
     batches.foreach { batch =>
@@ -162,10 +228,10 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
             tracker.foreach(_.recordRuleInvocation(rule.ruleName, runTime, effective))
 
             // Run the structural integrity checker against the plan after each rule.
-            if (!isPlanIntegral(result)) {
+            if (effective && !isPlanIntegral(result)) {
               val message = s"After applying rule ${rule.ruleName} in batch ${batch.name}, " +
                 "the structural integrity of the plan is broken."
-              throw new TreeNodeException(result, message, null)
+              throw new RuntimeException(message)
             }
 
             result
@@ -182,14 +248,14 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
             val message = s"Max iterations (${iteration - 1}) reached for batch ${batch.name}" +
               s"$endingMsg"
             if (Utils.isTesting || batch.strategy.errorOnExceed) {
-              throw new TreeNodeException(curPlan, message, null)
+              throw new RuntimeException(message)
             } else {
               logWarning(message)
             }
           }
           // Check idempotence for Once batches.
           if (batch.strategy == Once &&
-            Utils.isTesting && !blacklistedOnceBatches.contains(batch.name)) {
+            Utils.isTesting && !excludedOnceBatches.contains(batch.name)) {
             checkBatchIdempotence(batch, curPlan)
           }
           continue = false
@@ -209,69 +275,4 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
 
     curPlan
   }
-
-  private class PlanChangeLogger {
-
-    private val logLevel = SQLConf.get.optimizerPlanChangeLogLevel
-
-    private val logRules = SQLConf.get.optimizerPlanChangeRules.map(Utils.stringToSeq)
-
-    private val logBatches = SQLConf.get.optimizerPlanChangeBatches.map(Utils.stringToSeq)
-
-    def logRule(ruleName: String, oldPlan: TreeType, newPlan: TreeType): Unit = {
-      if (logRules.isEmpty || logRules.get.contains(ruleName)) {
-        def message(): String = {
-          s"""
-             |=== Applying Rule ${ruleName} ===
-             |${sideBySide(oldPlan.treeString, newPlan.treeString).mkString("\n")}
-           """.stripMargin
-        }
-
-        logBasedOnLevel(message)
-      }
-    }
-
-    def logBatch(batchName: String, oldPlan: TreeType, newPlan: TreeType): Unit = {
-      if (logBatches.isEmpty || logBatches.get.contains(batchName)) {
-        def message(): String = {
-          if (!oldPlan.fastEquals(newPlan)) {
-            s"""
-               |=== Result of Batch ${batchName} ===
-               |${sideBySide(oldPlan.treeString, newPlan.treeString).mkString("\n")}
-            """.stripMargin
-          } else {
-            s"Batch ${batchName} has no effect."
-          }
-        }
-
-        logBasedOnLevel(message)
-      }
-    }
-
-    def logMetrics(metrics: QueryExecutionMetrics): Unit = {
-      val totalTime = metrics.time / NANOS_PER_SECOND.toDouble
-      val totalTimeEffective = metrics.timeEffective / NANOS_PER_SECOND.toDouble
-      val message =
-        s"""
-           |=== Metrics of Executed Rules ===
-           |Total number of runs: ${metrics.numRuns}
-           |Total time: ${totalTime} seconds
-           |Total number of effective runs: ${metrics.numEffectiveRuns}
-           |Total time of effective runs: ${totalTimeEffective} seconds
-        """.stripMargin
-
-      logBasedOnLevel(message)
-    }
-
-    private def logBasedOnLevel(f: => String): Unit = {
-      logLevel match {
-        case "TRACE" => logTrace(f)
-        case "DEBUG" => logDebug(f)
-        case "INFO" => logInfo(f)
-        case "WARN" => logWarning(f)
-        case "ERROR" => logError(f)
-        case _ => logTrace(f)
-      }
-    }
-  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleIdCollection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleIdCollection.scala
new file mode 100644
index 0000000000000..605b57e46fc10
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleIdCollection.scala
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.rules
+
+import scala.collection.mutable
+
+// Represent unique rule ids for rules that are invoked multiple times.
+case class RuleId(id: Int) {
+  // Currently, there are more than 128 but less than 192 rules needing an id. However, the
+  // requirement can be relaxed when we have more such rules. Note that increasing the max id can
+  // result in increased memory consumption from every TreeNode.
+  require(id >= -1 && id < 192)
+}
+
+// Unknown rule id which does not prune tree traversals. It is used as the default rule id for
+// tree transformation functions.
+object UnknownRuleId extends RuleId(-1)
+
+// A collection of rules that use rule ids to prune tree traversals.
+object RuleIdCollection {
+
+  // The rules listed here need a rule id. Typically, rules that are in a fixed point batch or
+  // invoked multiple times by Analyzer/Optimizer/Planner need a rule id to prune unnecessary
+  // tree traversals in the transform function family. Note that those rules should not depend on
+  // a changing, external state. Rules here are in alphabetical order.
+  private val rulesNeedingIds: Seq[String] = {
+      // Catalyst Analyzer rules
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$AddMetadataColumns" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ExtractGenerator" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ExtractWindowExpressions" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$GlobalAggregates" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveAggAliasInGroupBy" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveAggregateFunctions" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveAliases" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveBinaryArithmetic" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveDeserializer" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveEncodersInUDF" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveFunctions" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveGenerate" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveGroupingAnalytics" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveInsertInto" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveMissingReferences" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveNaturalAndUsingJoin" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveNewInstance" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveOrdinalInOrderByAndGroupBy" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveOutputRelation" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolvePivot" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveRandomSeed" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveReferences" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveRelations" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveSubquery" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveSubqueryColumnAliases" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveTables" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveTempViews" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveUpCast" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveUserSpecifiedColumns" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveWindowFrame" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveWindowOrder" ::
+      "org.apache.spark.sql.catalyst.analysis.Analyzer$WindowsSubstitution" ::
+      "org.apache.spark.sql.catalyst.analysis.ApplyCharTypePadding" ::
+      "org.apache.spark.sql.catalyst.analysis.DeduplicateRelations" ::
+      "org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases" ::
+      "org.apache.spark.sql.catalyst.analysis.EliminateUnions" ::
+      "org.apache.spark.sql.catalyst.analysis.ResolveCreateNamedStruct" ::
+      "org.apache.spark.sql.catalyst.analysis.ResolveHints$ResolveCoalesceHints" ::
+      "org.apache.spark.sql.catalyst.analysis.ResolveHints$ResolveJoinStrategyHints" ::
+      "org.apache.spark.sql.catalyst.analysis.ResolveInlineTables" ::
+      "org.apache.spark.sql.catalyst.analysis.ResolveLambdaVariables" ::
+      "org.apache.spark.sql.catalyst.analysis.ResolveTimeZone" ::
+      "org.apache.spark.sql.catalyst.analysis.ResolveUnion" ::
+      "org.apache.spark.sql.catalyst.analysis.SubstituteUnresolvedOrdinals" ::
+      "org.apache.spark.sql.catalyst.analysis.TimeWindowing" ::
+      "org.apache.spark.sql.catalyst.analysis.UpdateOuterReferences" ::
+      // Catalyst Optimizer rules
+      "org.apache.spark.sql.catalyst.optimizer.BooleanSimplification" ::
+      "org.apache.spark.sql.catalyst.optimizer.CollapseProject" ::
+      "org.apache.spark.sql.catalyst.optimizer.CollapseRepartition" ::
+      "org.apache.spark.sql.catalyst.optimizer.CollapseWindow" ::
+      "org.apache.spark.sql.catalyst.optimizer.ColumnPruning" ::
+      "org.apache.spark.sql.catalyst.optimizer.CombineConcats" ::
+      "org.apache.spark.sql.catalyst.optimizer.CombineFilters" ::
+      "org.apache.spark.sql.catalyst.optimizer.CombineTypedFilters" ::
+      "org.apache.spark.sql.catalyst.optimizer.CombineUnions" ::
+      "org.apache.spark.sql.catalyst.optimizer.ConstantFolding" ::
+      "org.apache.spark.sql.catalyst.optimizer.ConstantPropagation" ::
+      "org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation" ::
+      "org.apache.spark.sql.catalyst.optimizer.CostBasedJoinReorder" ::
+      "org.apache.spark.sql.catalyst.optimizer.DecimalAggregates" ::
+      "org.apache.spark.sql.catalyst.optimizer.EliminateAggregateFilter" ::
+      "org.apache.spark.sql.catalyst.optimizer.EliminateLimits" ::
+      "org.apache.spark.sql.catalyst.optimizer.EliminateMapObjects" ::
+      "org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin" ::
+      "org.apache.spark.sql.catalyst.optimizer.EliminateSerialization" ::
+      "org.apache.spark.sql.catalyst.optimizer.LikeSimplification" ::
+      "org.apache.spark.sql.catalyst.optimizer.LimitPushDown" ::
+      "org.apache.spark.sql.catalyst.optimizer.LimitPushDownThroughWindow" ::
+      "org.apache.spark.sql.catalyst.optimizer.NullPropagation" ::
+      "org.apache.spark.sql.catalyst.optimizer.ObjectSerializerPruning" ::
+      "org.apache.spark.sql.catalyst.optimizer.OptimizeCsvJsonExprs" ::
+      "org.apache.spark.sql.catalyst.optimizer.OptimizeIn" ::
+      "org.apache.spark.sql.catalyst.optimizer.Optimizer$OptimizeSubqueries" ::
+      "org.apache.spark.sql.catalyst.optimizer.OptimizeRepartition" ::
+      "org.apache.spark.sql.catalyst.optimizer.OptimizeWindowFunctions" ::
+      "org.apache.spark.sql.catalyst.optimizer.OptimizeUpdateFields"::
+      "org.apache.spark.sql.catalyst.optimizer.PropagateEmptyRelation" ::
+      "org.apache.spark.sql.catalyst.optimizer.PruneFilters" ::
+      "org.apache.spark.sql.catalyst.optimizer.PushDownLeftSemiAntiJoin" ::
+      "org.apache.spark.sql.catalyst.optimizer.PushExtraPredicateThroughJoin" ::
+      "org.apache.spark.sql.catalyst.optimizer.PushFoldableIntoBranches" ::
+      "org.apache.spark.sql.catalyst.optimizer.PushLeftSemiLeftAntiThroughJoin" ::
+      "org.apache.spark.sql.catalyst.optimizer.ReassignLambdaVariableID" ::
+      "org.apache.spark.sql.catalyst.optimizer.RemoveDispensableExpressions" ::
+      "org.apache.spark.sql.catalyst.optimizer.RemoveLiteralFromGroupExpressions" ::
+      "org.apache.spark.sql.catalyst.optimizer.RemoveNoopOperators" ::
+      "org.apache.spark.sql.catalyst.optimizer.RemoveRedundantAggregates" ::
+      "org.apache.spark.sql.catalyst.optimizer.RemoveRepetitionFromGroupExpressions" ::
+      "org.apache.spark.sql.catalyst.optimizer.ReorderAssociativeOperator" ::
+      "org.apache.spark.sql.catalyst.optimizer.ReorderJoin" ::
+      "org.apache.spark.sql.catalyst.optimizer.ReplaceExceptWithAntiJoin" ::
+      "org.apache.spark.sql.catalyst.optimizer.ReplaceExceptWithFilter" ::
+      "org.apache.spark.sql.catalyst.optimizer.ReplaceDistinctWithAggregate" ::
+      "org.apache.spark.sql.catalyst.optimizer.ReplaceNullWithFalseInPredicate" ::
+      "org.apache.spark.sql.catalyst.optimizer.ReplaceIntersectWithSemiJoin" ::
+      "org.apache.spark.sql.catalyst.optimizer.RewriteExceptAll" ::
+      "org.apache.spark.sql.catalyst.optimizer.RewriteIntersectAll" ::
+      "org.apache.spark.sql.catalyst.optimizer.SimplifyBinaryComparison" ::
+      "org.apache.spark.sql.catalyst.optimizer.SimplifyCaseConversionExpressions" ::
+      "org.apache.spark.sql.catalyst.optimizer.SimplifyCasts" ::
+      "org.apache.spark.sql.catalyst.optimizer.SimplifyConditionals" ::
+      "org.apache.spark.sql.catalyst.optimizer.SimplifyConditionalsInPredicate" ::
+      "org.apache.spark.sql.catalyst.optimizer.TransposeWindow" ::
+      "org.apache.spark.sql.catalyst.optimizer.UnwrapCastInBinaryComparison" ::  Nil
+  }
+
+  // Maps rule names to ids. Rule ids are continuous natural numbers starting from 0.
+  private val ruleToId = new mutable.HashMap[String, RuleId]
+
+  // The total number of rules with ids.
+  val NumRules: Int = {
+    var id = 0
+    rulesNeedingIds.foreach(ruleName => {
+      ruleToId.put(ruleName, RuleId(id))
+      id = id + 1
+    })
+    id
+  }
+
+  // Return the rule id for a rule name.
+  def getRuleId(ruleName: String): RuleId = {
+    val ruleIdOpt = ruleToId.get(ruleName)
+    // Please add the rule name to `rulesWithIds` if rule id is not found.
+    if (!ruleIdOpt.isDefined) {
+      throw new NoSuchElementException(s"Rule id not found for $ruleName")
+    }
+    ruleIdOpt.get
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/streaming/StreamingRelationV2.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/streaming/StreamingRelationV2.scala
new file mode 100644
index 0000000000000..06beb61f9ec93
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/streaming/StreamingRelationV2.scala
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.streaming
+
+import org.apache.spark.sql.catalyst.analysis.MultiInstanceRelation
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, Statistics}
+import org.apache.spark.sql.connector.catalog.{CatalogPlugin, Identifier, Table, TableProvider}
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
+
+// We have to pack in the V1 data source as a shim, for the case when a source implements
+// continuous processing (which is always V2) but only has V1 microbatch support. We don't
+// know at read time whether the query is continuous or not, so we need to be able to
+// swap a V1 relation back in.
+/**
+ * Used to link a [[Table]] into a streaming [[LogicalPlan]].
+ */
+case class StreamingRelationV2(
+    source: Option[TableProvider],
+    sourceName: String,
+    table: Table,
+    extraOptions: CaseInsensitiveStringMap,
+    output: Seq[Attribute],
+    catalog: Option[CatalogPlugin],
+    identifier: Option[Identifier],
+    v1Relation: Option[LogicalPlan])
+  extends LeafNode with MultiInstanceRelation {
+  override lazy val resolved = v1Relation.forall(_.resolved)
+  override def isStreaming: Boolean = true
+  override def toString: String = sourceName
+
+  override def computeStats(): Statistics = Statistics(
+    sizeInBytes = BigInt(conf.defaultSizeInBytes)
+  )
+
+  override def newInstance(): LogicalPlan = this.copy(output = output.map(_.newInstance()))
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/streaming/WriteToStream.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/streaming/WriteToStream.scala
new file mode 100644
index 0000000000000..80c441f184d25
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/streaming/WriteToStream.scala
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.streaming
+
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, UnaryNode}
+import org.apache.spark.sql.connector.catalog.{Identifier, Table, TableCatalog}
+import org.apache.spark.sql.streaming.OutputMode
+
+/**
+ * Used to create a [[StreamExecution]].
+ */
+case class WriteToStream(
+    name: String,
+    resolvedCheckpointLocation: String,
+    sink: Table,
+    outputMode: OutputMode,
+    deleteCheckpointOnStop: Boolean,
+    inputQuery: LogicalPlan,
+    catalogAndIdent: Option[(TableCatalog, Identifier)] = None) extends UnaryNode {
+
+  override def isStreaming: Boolean = true
+
+  override def output: Seq[Attribute] = Nil
+
+  override def child: LogicalPlan = inputQuery
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): WriteToStream =
+    copy(inputQuery = newChild)
+}
+
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/streaming/WriteToStreamStatement.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/streaming/WriteToStreamStatement.scala
new file mode 100644
index 0000000000000..85a018a8f55c3
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/streaming/WriteToStreamStatement.scala
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.streaming
+
+import org.apache.hadoop.conf.Configuration
+
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, UnaryNode}
+import org.apache.spark.sql.connector.catalog.{Identifier, Table, TableCatalog}
+import org.apache.spark.sql.streaming.OutputMode
+
+/**
+ * A statement for Stream writing. It contains all neccessary param and will be resolved in the
+ * rule [[ResolveStreamWrite]].
+ *
+ * @param userSpecifiedName  Query name optionally specified by the user.
+ * @param userSpecifiedCheckpointLocation  Checkpoint location optionally specified by the user.
+ * @param useTempCheckpointLocation  Whether to use a temporary checkpoint location when the user
+ *                                   has not specified one. If false, then error will be thrown.
+ * @param recoverFromCheckpointLocation  Whether to recover query from the checkpoint location.
+ *                                       If false and the checkpoint location exists, then error
+ *                                       will be thrown.
+ * @param sink  Sink to write the streaming outputs.
+ * @param outputMode  Output mode for the sink.
+ * @param hadoopConf  The Hadoop Configuration to get a FileSystem instance
+ * @param isContinuousTrigger  Whether the statement is triggered by a continuous query or not.
+ * @param inputQuery  The analyzed query plan from the streaming DataFrame.
+ * @param catalogAndIdent Catalog and identifier for the sink, set when it is a V2 catalog table
+ */
+case class WriteToStreamStatement(
+    userSpecifiedName: Option[String],
+    userSpecifiedCheckpointLocation: Option[String],
+    useTempCheckpointLocation: Boolean,
+    recoverFromCheckpointLocation: Boolean,
+    sink: Table,
+    outputMode: OutputMode,
+    hadoopConf: Configuration,
+    isContinuousTrigger: Boolean,
+    inputQuery: LogicalPlan,
+    catalogAndIdent: Option[(TableCatalog, Identifier)] = None) extends UnaryNode {
+
+  override def isStreaming: Boolean = true
+
+  override def output: Seq[Attribute] = Nil
+
+  override def child: LogicalPlan = inputQuery
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): WriteToStreamStatement =
+    copy(inputQuery = newChild)
+}
+
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
index c4a106702a515..6780a4601ea08 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.trees
 import java.util.UUID
 
 import scala.collection.{mutable, Map}
+import scala.collection.JavaConverters._
 import scala.reflect.ClassTag
 
 import org.apache.commons.lang3.ClassUtils
@@ -30,16 +31,22 @@ import org.json4s.jackson.JsonMethods._
 import org.apache.spark.sql.catalyst.{AliasIdentifier, IdentifierWithDatabase}
 import org.apache.spark.sql.catalyst.ScalaReflection._
 import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStorageFormat, CatalogTable, CatalogTableType, FunctionResource}
-import org.apache.spark.sql.catalyst.errors._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.JoinType
-import org.apache.spark.sql.catalyst.plans.QueryPlan
 import org.apache.spark.sql.catalyst.plans.physical.{BroadcastMode, Partitioning}
+import org.apache.spark.sql.catalyst.rules.RuleId
+import org.apache.spark.sql.catalyst.rules.RuleIdCollection
+import org.apache.spark.sql.catalyst.rules.UnknownRuleId
+import org.apache.spark.sql.catalyst.trees.TreePattern.TreePattern
 import org.apache.spark.sql.catalyst.util.StringUtils.PlanStringConcat
 import org.apache.spark.sql.catalyst.util.truncatedString
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
 import org.apache.spark.storage.StorageLevel
+import org.apache.spark.util.Utils
+import org.apache.spark.util.collection.BitSet
 
 /** Used by [[TreeNode.getNodeNumbered]] when traversing the tree for a given number */
 private class MutableInt(var i: Int)
@@ -68,8 +75,11 @@ object CurrentOrigin {
   }
 
   def withOrigin[A](o: Origin)(f: => A): A = {
+    // remember the previous one so it can be reset to this
+    // this way withOrigin can be recursive
+    val previous = get
     set(o)
-    val ret = try f finally { reset() }
+    val ret = try f finally { set(previous) }
     ret
   }
 }
@@ -77,8 +87,13 @@ object CurrentOrigin {
 // A tag of a `TreeNode`, which defines name and type
 case class TreeNodeTag[T](name: String)
 
+// A functor that always returns true.
+object AlwaysProcess {
+  val fn: TreePatternBits => Boolean = { _ => true}
+}
+
 // scalastyle:off
-abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
+abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product with TreePatternBits {
 // scalastyle:on
   self: BaseType =>
 
@@ -90,8 +105,79 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
    */
   private val tags: mutable.Map[TreeNodeTag[_], Any] = mutable.Map.empty
 
-  protected def copyTagsFrom(other: BaseType): Unit = {
-    tags ++= other.tags
+  /**
+   * Default tree pattern [[BitSet] for a [[TreeNode]].
+   */
+  protected def getDefaultTreePatternBits: BitSet = {
+    val bits: BitSet = new BitSet(TreePattern.maxId)
+    // Propagate node pattern bits
+    val nodePatternIterator = nodePatterns.iterator
+    while (nodePatternIterator.hasNext) {
+      bits.set(nodePatternIterator.next().id)
+    }
+    // Propagate children's pattern bits
+    val childIterator = children.iterator
+    while (childIterator.hasNext) {
+      bits.union(childIterator.next().treePatternBits)
+    }
+    bits
+  }
+
+  /**
+   * A BitSet of tree patterns for this TreeNode and its subtree. If this TreeNode and its
+   * subtree contains a pattern `P`, the corresponding bit for `P.id` is set in this BitSet.
+   */
+  override lazy val treePatternBits: BitSet = getDefaultTreePatternBits
+
+  /**
+   * A BitSet of rule ids to record ineffective rules for this TreeNode and its subtree.
+   * If a rule R (which does not read a varying, external state for each invocation) is
+   * ineffective in one apply call for this TreeNode and its subtree, R will still be
+   * ineffective for subsequent apply calls on this tree because query plan structures are
+   * immutable.
+   */
+  private val ineffectiveRules: BitSet = new BitSet(RuleIdCollection.NumRules)
+
+  /**
+   * @return a sequence of tree pattern enums in a TreeNode T. It does not include propagated
+   *         patterns in the subtree of T.
+   */
+  protected val nodePatterns: Seq[TreePattern] = Seq()
+
+  /**
+   * Mark that a rule (with id `ruleId`) is ineffective for this TreeNode and its subtree.
+   *
+   * @param ruleId the unique identifier of the rule. If `ruleId` is UnknownId, it is a no-op.
+   */
+  protected def markRuleAsIneffective(ruleId : RuleId): Unit = {
+    if (ruleId eq UnknownRuleId) {
+      return
+    }
+    ineffectiveRules.set(ruleId.id)
+  }
+
+  /**
+   * Whether this TreeNode and its subtree have been marked as ineffective for the rule with id
+   * `ruleId`.
+   *
+   * @param ruleId the unique id of the rule
+   * @return true if the rule has been marked as ineffective; false otherwise. If `ruleId` is
+   *         UnknownId, it returns false.
+   */
+  protected def isRuleIneffective(ruleId : RuleId): Boolean = {
+    if (ruleId eq UnknownRuleId) {
+      return false
+    }
+    ineffectiveRules.get(ruleId.id)
+  }
+
+  def copyTagsFrom(other: BaseType): Unit = {
+    // SPARK-32753: it only makes sense to copy tags to a new node
+    // but it's too expensive to detect other cases likes node removal
+    // so we make a compromise here to copy tags to node with no tags
+    if (tags.isEmpty) {
+      tags ++= other.tags
+    }
   }
 
   def setTagValue[T](tag: TreeNodeTag[T], value: T): Unit = {
@@ -185,7 +271,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
   def map[A](f: BaseType => A): Seq[A] = {
     val ret = new collection.mutable.ArrayBuffer[A]()
     foreach(ret += f(_))
-    ret
+    ret.toSeq
   }
 
   /**
@@ -195,7 +281,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
   def flatMap[A](f: BaseType => TraversableOnce[A]): Seq[A] = {
     val ret = new collection.mutable.ArrayBuffer[A]()
     foreach(ret ++= f(_))
-    ret
+    ret.toSeq
   }
 
   /**
@@ -206,7 +292,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
     val ret = new collection.mutable.ArrayBuffer[B]()
     val lifted = pf.lift
     foreach(node => lifted(node).foreach(ret.+=))
-    ret
+    ret.toSeq
   }
 
   /**
@@ -240,11 +326,50 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
     arr
   }
 
+  private def childrenFastEquals(
+      originalChildren: IndexedSeq[BaseType], newChildren: IndexedSeq[BaseType]): Boolean = {
+    val size = originalChildren.size
+    var i = 0
+    while (i < size) {
+      if (!originalChildren(i).fastEquals(newChildren(i))) return false
+      i += 1
+    }
+    true
+  }
+
+  // This is a temporary solution, we will change the type of children to IndexedSeq in a
+  // followup PR
+  private def asIndexedSeq(seq: Seq[BaseType]): IndexedSeq[BaseType] = {
+    if (seq.isInstanceOf[IndexedSeq[BaseType]]) {
+      seq.asInstanceOf[IndexedSeq[BaseType]]
+    } else {
+      seq.toIndexedSeq
+    }
+  }
+
+  final def withNewChildren(newChildren: Seq[BaseType]): BaseType = {
+    val childrenIndexedSeq = asIndexedSeq(children)
+    val newChildrenIndexedSeq = asIndexedSeq(newChildren)
+    assert(newChildrenIndexedSeq.size == childrenIndexedSeq.size, "Incorrect number of children")
+    if (childrenIndexedSeq.isEmpty ||
+        childrenFastEquals(newChildrenIndexedSeq, childrenIndexedSeq)) {
+      this
+    } else {
+      CurrentOrigin.withOrigin(origin) {
+        val res = withNewChildrenInternal(newChildrenIndexedSeq)
+        res.copyTagsFrom(this)
+        res
+      }
+    }
+  }
+
+  protected def withNewChildrenInternal(newChildren: IndexedSeq[BaseType]): BaseType
+
   /**
    * Returns a copy of this node with the children replaced.
    * TODO: Validate somewhere (in debug mode?) that children are ordered correctly.
    */
-  def withNewChildren(newChildren: Seq[BaseType]): BaseType = {
+  protected final def legacyWithNewChildren(newChildren: Seq[BaseType]): BaseType = {
     assert(newChildren.size == children.size, "Incorrect number of children")
     var changed = false
     val remainingNewChildren = newChildren.toBuffer
@@ -275,8 +400,10 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
       case s: Seq[_] =>
         s.map(mapChild)
       case m: Map[_, _] =>
+        // `map.mapValues().view.force` return `Map` in Scala 2.12 but return `IndexedSeq` in Scala
+        // 2.13, call `toMap` method manually to compatible with Scala 2.12 and Scala 2.13
         // `mapValues` is lazy and we need to force it to materialize
-        m.mapValues(mapChild).view.force
+        m.mapValues(mapChild).view.force.toMap
       case arg: TreeNode[_] if containsChild(arg) => mapTreeNode(arg)
       case Some(child) => Some(mapChild(child))
       case nonChild: AnyRef => nonChild
@@ -292,12 +419,34 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
    * Users should not expect a specific directionality. If a specific directionality is needed,
    * transformDown or transformUp should be used.
    *
-   * @param rule the function use to transform this nodes children
+   * @param rule the function used to transform this nodes children
    */
   def transform(rule: PartialFunction[BaseType, BaseType]): BaseType = {
     transformDown(rule)
   }
 
+  /**
+   * Returns a copy of this node where `rule` has been recursively applied to the tree.
+   * When `rule` does not apply to a given node it is left unchanged.
+   * Users should not expect a specific directionality. If a specific directionality is needed,
+   * transformDown or transformUp should be used.
+   *
+   * @param rule   the function used to transform this nodes children
+   * @param cond   a Lambda expression to prune tree traversals. If `cond.apply` returns false
+   *               on a TreeNode T, skips processing T and its subtree; otherwise, processes
+   *               T and its subtree recursively.
+   * @param ruleId is a unique Id for `rule` to prune unnecessary tree traversals. When it is
+   *               UnknownRuleId, no pruning happens. Otherwise, if `rule` (with id `ruleId`)
+   *               has been marked as in effective on a TreeNode T, skips processing T and its
+   *               subtree. Do not pass it if the rule is not purely functional and reads a
+   *               varying initial state for different invocations.
+   */
+  def transformWithPruning(cond: TreePatternBits => Boolean,
+    ruleId: RuleId = UnknownRuleId)(rule: PartialFunction[BaseType, BaseType])
+  : BaseType = {
+    transformDownWithPruning(cond, ruleId)(rule)
+  }
+
   /**
    * Returns a copy of this node where `rule` has been recursively applied to it and all of its
    * children (pre-order). When `rule` does not apply to a given node it is left unchanged.
@@ -305,17 +454,46 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
    * @param rule the function used to transform this nodes children
    */
   def transformDown(rule: PartialFunction[BaseType, BaseType]): BaseType = {
+    transformDownWithPruning(AlwaysProcess.fn, UnknownRuleId)(rule)
+  }
+
+  /**
+   * Returns a copy of this node where `rule` has been recursively applied to it and all of its
+   * children (pre-order). When `rule` does not apply to a given node it is left unchanged.
+   *
+   * @param rule   the function used to transform this nodes children
+   * @param cond   a Lambda expression to prune tree traversals. If `cond.apply` returns false
+   *               on a TreeNode T, skips processing T and its subtree; otherwise, processes
+   *               T and its subtree recursively.
+   * @param ruleId is a unique Id for `rule` to prune unnecessary tree traversals. When it is
+   *               UnknownRuleId, no pruning happens. Otherwise, if `rule` (with id `ruleId`)
+   *               has been marked as in effective on a TreeNode T, skips processing T and its
+   *               subtree. Do not pass it if the rule is not purely functional and reads a
+   *               varying initial state for different invocations.
+   */
+  def transformDownWithPruning(cond: TreePatternBits => Boolean,
+    ruleId: RuleId = UnknownRuleId)(rule: PartialFunction[BaseType, BaseType])
+  : BaseType = {
+    if (!cond.apply(this) || isRuleIneffective(ruleId)) {
+      return this
+    }
     val afterRule = CurrentOrigin.withOrigin(origin) {
       rule.applyOrElse(this, identity[BaseType])
     }
 
     // Check if unchanged and then possibly return old copy to avoid gc churn.
     if (this fastEquals afterRule) {
-      mapChildren(_.transformDown(rule))
+      val rewritten_plan = mapChildren(_.transformDownWithPruning(cond, ruleId)(rule))
+      if (this eq rewritten_plan) {
+        markRuleAsIneffective(ruleId)
+        this
+      } else {
+        rewritten_plan
+      }
     } else {
       // If the transform function replaces this node with a new one, carry over the tags.
       afterRule.copyTagsFrom(this)
-      afterRule.mapChildren(_.transformDown(rule))
+      afterRule.mapChildren(_.transformDownWithPruning(cond, ruleId)(rule))
     }
   }
 
@@ -324,10 +502,34 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
    * children and then itself (post-order). When `rule` does not apply to a given node, it is left
    * unchanged.
    *
-   * @param rule the function use to transform this nodes children
+   * @param rule   the function used to transform this nodes children
    */
   def transformUp(rule: PartialFunction[BaseType, BaseType]): BaseType = {
-    val afterRuleOnChildren = mapChildren(_.transformUp(rule))
+    transformUpWithPruning(AlwaysProcess.fn, UnknownRuleId)(rule)
+  }
+
+  /**
+   * Returns a copy of this node where `rule` has been recursively applied first to all of its
+   * children and then itself (post-order). When `rule` does not apply to a given node, it is left
+   * unchanged.
+   *
+   * @param rule   the function used to transform this nodes children
+   * @param cond   a Lambda expression to prune tree traversals. If `cond.apply` returns false
+   *               on a TreeNode T, skips processing T and its subtree; otherwise, processes
+   *               T and its subtree recursively.
+   * @param ruleId is a unique Id for `rule` to prune unnecessary tree traversals. When it is
+   *               UnknownRuleId, no pruning happens. Otherwise, if `rule` (with id `ruleId`)
+   *               has been marked as in effective on a TreeNode T, skips processing T and its
+   *               subtree. Do not pass it if the rule is not purely functional and reads a
+   *               varying initial state for different invocations.
+   */
+  def transformUpWithPruning(cond: TreePatternBits => Boolean,
+    ruleId: RuleId = UnknownRuleId)(rule: PartialFunction[BaseType, BaseType])
+  : BaseType = {
+    if (!cond.apply(this) || isRuleIneffective(ruleId)) {
+      return this
+    }
+    val afterRuleOnChildren = mapChildren(_.transformUpWithPruning(cond, ruleId)(rule))
     val newNode = if (this fastEquals afterRuleOnChildren) {
       CurrentOrigin.withOrigin(origin) {
         rule.applyOrElse(this, identity[BaseType])
@@ -337,9 +539,14 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
         rule.applyOrElse(afterRuleOnChildren, identity[BaseType])
       }
     }
-    // If the transform function replaces this node with a new one, carry over the tags.
-    newNode.copyTagsFrom(this)
-    newNode
+    if (this eq newNode) {
+      markRuleAsIneffective(ruleId)
+      this
+    } else {
+      // If the transform function replaces this node with a new one, carry over the tags.
+      newNode.copyTagsFrom(this)
+      newNode
+    }
   }
 
   /**
@@ -347,7 +554,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
    */
   def mapChildren(f: BaseType => BaseType): BaseType = {
     if (containsChild.nonEmpty) {
-      mapChildren(f, forceCopy = false)
+      withNewChildren(children.map(f))
     } else {
       this
     }
@@ -411,6 +618,8 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
         } else {
           Some(arg)
         }
+      // `map.mapValues().view.force` return `Map` in Scala 2.12 but return `IndexedSeq` in Scala
+      // 2.13, call `toMap` method manually to compatible with Scala 2.12 and Scala 2.13
       case m: Map[_, _] => m.mapValues {
         case arg: TreeNode[_] if containsChild(arg) =>
           val newChild = f(arg.asInstanceOf[BaseType])
@@ -421,7 +630,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
             arg
           }
         case other => other
-      }.view.force // `mapValues` is lazy and we need to force it to materialize
+      }.view.force.toMap // `mapValues` is lazy and we need to force it to materialize
       case d: DataType => d // Avoid unpacking Structs
       case args: Stream[_] => args.map(mapChild).force // Force materialization on stream
       case args: Iterable[_] => args.map(mapChild)
@@ -455,7 +664,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
    */
   private def makeCopy(
       newArgs: Array[AnyRef],
-      allowEmptyArgs: Boolean): BaseType = attachTree(this, "makeCopy") {
+      allowEmptyArgs: Boolean): BaseType = {
     val allCtors = getClass.getConstructors
     if (newArgs.isEmpty && allCtors.isEmpty) {
       // This is a singleton object which doesn't have any constructor. Just return `this` as we
@@ -466,7 +675,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
     // Skip no-arg constructors that are just there for kryo.
     val ctors = allCtors.filter(allowEmptyArgs || _.getParameterTypes.size != 0)
     if (ctors.isEmpty) {
-      sys.error(s"No valid constructor for $nodeName")
+      throw QueryExecutionErrors.constructorNotFoundError(nodeName)
     }
     val allArgs: Array[AnyRef] = if (otherCopyArgs.isEmpty) {
       newArgs
@@ -494,8 +703,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
       }
     } catch {
       case e: java.lang.IllegalArgumentException =>
-        throw new TreeNodeException(
-          this,
+        throw new IllegalStateException(
           s"""
              |Failed to copy node.
              |Is otherCopyArgs specified correctly for $nodeName.
@@ -511,11 +719,13 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
     mapChildren(_.clone(), forceCopy = true)
   }
 
+  private def simpleClassName: String = Utils.getSimpleName(this.getClass)
+
   /**
    * Returns the name of this type of TreeNode.  Defaults to the class name.
    * Note that we remove the "Exec" suffix for physical operators here.
    */
-  def nodeName: String = getClass.getSimpleName.replaceAll("Exec$", "")
+  def nodeName: String = simpleClassName.replaceAll("Exec$", "")
 
   /**
    * The arguments that should be included in the arg string.  Defaults to the `productIterator`.
@@ -524,6 +734,16 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
 
   private lazy val allChildren: Set[TreeNode[_]] = (children ++ innerChildren).toSet[TreeNode[_]]
 
+  private def redactMapString[K, V](map: Map[K, V], maxFields: Int): List[String] = {
+    // For security reason, redact the map value if the key is in centain patterns
+    val redactedMap = SQLConf.get.redactOptions(map.toMap)
+    // construct the redacted map as strings of the format "key=value"
+    val keyValuePairs = redactedMap.toSeq.map { item =>
+      item._1 + "=" + item._2
+    }
+    truncatedString(keyValuePairs, "[", ", ", "]", maxFields) :: Nil
+  }
+
   /** Returns a string representing the arguments to this node, minus any children */
   def argString(maxFields: Int): String = stringArgs.flatMap {
     case tn: TreeNode[_] if allChildren.contains(tn) => Nil
@@ -540,6 +760,10 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
     case None => Nil
     case Some(null) => Nil
     case Some(any) => any :: Nil
+    case map: CaseInsensitiveStringMap =>
+      redactMapString(map.asCaseSensitiveMap().asScala, maxFields)
+    case map: Map[_, _] =>
+      redactMapString(map, maxFields)
     case table: CatalogTable =>
       table.storage.serde match {
         case Some(serde) => table.identifier :: serde :: Nil
@@ -588,7 +812,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
       addSuffix: Boolean,
       maxFields: Int,
       printOperatorId: Boolean): Unit = {
-    generateTreeString(0, Nil, append, verbose, "", addSuffix, maxFields, printOperatorId)
+    generateTreeString(0, Nil, append, verbose, "", addSuffix, maxFields, printOperatorId, 0)
   }
 
   /**
@@ -656,8 +880,9 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
       prefix: String = "",
       addSuffix: Boolean = false,
       maxFields: Int,
-      printNodeId: Boolean): Unit = {
-
+      printNodeId: Boolean,
+      indent: Int = 0): Unit = {
+    append("   " * indent)
     if (depth > 0) {
       lastChildren.init.foreach { isLast =>
         append(if (isLast) "   " else ":  ")
@@ -681,20 +906,20 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
     if (innerChildren.nonEmpty) {
       innerChildren.init.foreach(_.generateTreeString(
         depth + 2, lastChildren :+ children.isEmpty :+ false, append, verbose,
-        addSuffix = addSuffix, maxFields = maxFields, printNodeId = printNodeId))
+        addSuffix = addSuffix, maxFields = maxFields, printNodeId = printNodeId, indent = indent))
       innerChildren.last.generateTreeString(
         depth + 2, lastChildren :+ children.isEmpty :+ true, append, verbose,
-        addSuffix = addSuffix, maxFields = maxFields, printNodeId = printNodeId)
+        addSuffix = addSuffix, maxFields = maxFields, printNodeId = printNodeId, indent = indent)
     }
 
     if (children.nonEmpty) {
       children.init.foreach(_.generateTreeString(
         depth + 1, lastChildren :+ false, append, verbose, prefix, addSuffix,
-        maxFields, printNodeId = printNodeId)
+        maxFields, printNodeId = printNodeId, indent = indent)
       )
       children.last.generateTreeString(
         depth + 1, lastChildren :+ true, append, verbose, prefix,
-        addSuffix, maxFields, printNodeId = printNodeId)
+        addSuffix, maxFields, printNodeId = printNodeId, indent = indent)
     }
   }
 
@@ -734,7 +959,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
   protected def jsonFields: List[JField] = {
     val fieldNames = getConstructorParameterNames(getClass)
     val fieldValues = productIterator.toSeq ++ otherCopyArgs
-    assert(fieldNames.length == fieldValues.length, s"${getClass.getSimpleName} fields: " +
+    assert(fieldNames.length == fieldValues.length, s"$simpleClassName fields: " +
       fieldNames.mkString(", ") + s", values: " + fieldValues.mkString(", "))
 
     fieldNames.zip(fieldValues).map {
@@ -788,7 +1013,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
       try {
         val fieldNames = getConstructorParameterNames(p.getClass)
         val fieldValues = p.productIterator.toSeq
-        assert(fieldNames.length == fieldValues.length, s"${getClass.getSimpleName} fields: " +
+        assert(fieldNames.length == fieldValues.length, s"$simpleClassName fields: " +
           fieldNames.mkString(", ") + s", values: " + fieldValues.mkString(", "))
         ("product-class" -> JString(p.getClass.getName)) :: fieldNames.zip(fieldValues).map {
           case (name, value) => name -> parseToJson(value)
@@ -815,3 +1040,133 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
     case _ => false
   }
 }
+
+trait LeafLike[T <: TreeNode[T]] { self: TreeNode[T] =>
+  override final def children: Seq[T] = Nil
+  override final def mapChildren(f: T => T): T = this.asInstanceOf[T]
+  override final def withNewChildrenInternal(newChildren: IndexedSeq[T]): T = this.asInstanceOf[T]
+}
+
+trait UnaryLike[T <: TreeNode[T]] { self: TreeNode[T] =>
+  def child: T
+  @transient override final lazy val children: Seq[T] = IndexedSeq(child)
+
+  override final def mapChildren(f: T => T): T = {
+    val newChild = f(child)
+    if (newChild fastEquals child) {
+      this.asInstanceOf[T]
+    } else {
+      CurrentOrigin.withOrigin(origin) {
+        val res = withNewChildInternal(newChild)
+        res.copyTagsFrom(this.asInstanceOf[T])
+        res
+      }
+    }
+  }
+
+  override final def withNewChildrenInternal(newChildren: IndexedSeq[T]): T = {
+    assert(newChildren.size == 1, "Incorrect number of children")
+    withNewChildInternal(newChildren.head)
+  }
+
+  protected def withNewChildInternal(newChild: T): T
+}
+
+trait BinaryLike[T <: TreeNode[T]] { self: TreeNode[T] =>
+  def left: T
+  def right: T
+  @transient override final lazy val children: Seq[T] = IndexedSeq(left, right)
+
+  override final def mapChildren(f: T => T): T = {
+    var newLeft = f(left)
+    newLeft = if (newLeft fastEquals left) left else newLeft
+    var newRight = f(right)
+    newRight = if (newRight fastEquals right) right else newRight
+
+    if (newLeft.eq(left) && newRight.eq(right)) {
+      this.asInstanceOf[T]
+    } else {
+      CurrentOrigin.withOrigin(origin) {
+        val res = withNewChildrenInternal(newLeft, newRight)
+        res.copyTagsFrom(this.asInstanceOf[T])
+        res
+      }
+    }
+  }
+
+  override final def withNewChildrenInternal(newChildren: IndexedSeq[T]): T = {
+    assert(newChildren.size == 2, "Incorrect number of children")
+    withNewChildrenInternal(newChildren(0), newChildren(1))
+  }
+
+  protected def withNewChildrenInternal(newLeft: T, newRight: T): T
+}
+
+trait TernaryLike[T <: TreeNode[T]] { self: TreeNode[T] =>
+  def first: T
+  def second: T
+  def third: T
+  @transient override final lazy val children: Seq[T] = IndexedSeq(first, second, third)
+
+  override final def mapChildren(f: T => T): T = {
+    var newFirst = f(first)
+    newFirst = if (newFirst fastEquals first) first else newFirst
+    var newSecond = f(second)
+    newSecond = if (newSecond fastEquals second) second else newSecond
+    var newThird = f(third)
+    newThird = if (newThird fastEquals third) third else newThird
+
+    if (newFirst.eq(first) && newSecond.eq(second) && newThird.eq(third)) {
+      this.asInstanceOf[T]
+    } else {
+      CurrentOrigin.withOrigin(origin) {
+        val res = withNewChildrenInternal(newFirst, newSecond, newThird)
+        res.copyTagsFrom(this.asInstanceOf[T])
+        res
+      }
+    }
+  }
+
+  override final def withNewChildrenInternal(newChildren: IndexedSeq[T]): T = {
+    assert(newChildren.size == 3, "Incorrect number of children")
+    withNewChildrenInternal(newChildren(0), newChildren(1), newChildren(2))
+  }
+
+  protected def withNewChildrenInternal(newFirst: T, newSecond: T, newThird: T): T
+}
+
+trait QuaternaryLike[T <: TreeNode[T]] { self: TreeNode[T] =>
+  def first: T
+  def second: T
+  def third: T
+  def fourth: T
+  @transient override final lazy val children: Seq[T] = IndexedSeq(first, second, third, fourth)
+
+  override final def mapChildren(f: T => T): T = {
+    var newFirst = f(first)
+    newFirst = if (newFirst fastEquals first) first else newFirst
+    var newSecond = f(second)
+    newSecond = if (newSecond fastEquals second) second else newSecond
+    var newThird = f(third)
+    newThird = if (newThird fastEquals third) third else newThird
+    var newFourth = f(fourth)
+    newFourth = if (newFourth fastEquals fourth) fourth else newFourth
+
+    if (newFirst.eq(first) && newSecond.eq(second) && newThird.eq(third) && newFourth.eq(fourth)) {
+      this.asInstanceOf[T]
+    } else {
+      CurrentOrigin.withOrigin(origin) {
+        val res = withNewChildrenInternal(newFirst, newSecond, newThird, newFourth)
+        res.copyTagsFrom(this.asInstanceOf[T])
+        res
+      }
+    }
+  }
+
+  override final def withNewChildrenInternal(newChildren: IndexedSeq[T]): T = {
+    assert(newChildren.size == 4, "Incorrect number of children")
+    withNewChildrenInternal(newChildren(0), newChildren(1), newChildren(2), newChildren(3))
+  }
+
+  protected def withNewChildrenInternal(newFirst: T, newSecond: T, newThird: T, newFourth: T): T
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatternBits.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatternBits.scala
new file mode 100644
index 0000000000000..b133630fcba05
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatternBits.scala
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.trees
+
+import org.apache.spark.sql.catalyst.trees.TreePattern.TreePattern
+import org.apache.spark.util.collection.BitSet
+
+// A wrapper of BitSet for pattern enums.
+trait TreePatternBits {
+  protected val treePatternBits: BitSet
+
+  /**
+   * @param t, the tree pattern enum to be tested.
+   * @return true if the bit for `t` is set; false otherwise.
+   */
+  @inline final def containsPattern(t: TreePattern): Boolean = {
+    treePatternBits.get(t.id)
+  }
+
+  /**
+   * @param patterns, a sequence of tree pattern enums to be tested.
+   * @return true if every bit for `patterns` is set; false otherwise.
+   */
+  final def containsAllPatterns(patterns: TreePattern*): Boolean = {
+    val iterator = patterns.iterator
+    while (iterator.hasNext) {
+      if (!containsPattern(iterator.next)) {
+        return false
+      }
+    }
+    true
+  }
+
+  /**
+   * @param patterns, a sequence of tree pattern enums to be tested.
+   * @return true if at least one bit for `patterns` is set; false otherwise.
+   */
+  final def containsAnyPattern(patterns: TreePattern*): Boolean = {
+    val iterator = patterns.iterator
+    while (iterator.hasNext) {
+      if (containsPattern(iterator.next)) {
+        return true
+      }
+    }
+    false
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatterns.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatterns.scala
new file mode 100644
index 0000000000000..40ef7cb592daa
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatterns.scala
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.trees
+
+// Enums for commonly encountered tree patterns in rewrite rules.
+object TreePattern extends Enumeration  {
+  type TreePattern = Value
+
+  // Enum Ids start from 0.
+  // Expression patterns (alphabetically ordered)
+  val ALIAS: Value = Value(0)
+  val AND_OR: Value = Value
+  val ATTRIBUTE_REFERENCE: Value = Value
+  val APPEND_COLUMNS: Value = Value
+  val AVERAGE: Value = Value
+  val BINARY_ARITHMETIC: Value = Value
+  val BINARY_COMPARISON: Value = Value
+  val BOOL_AGG: Value = Value
+  val CASE_WHEN: Value = Value
+  val CAST: Value = Value
+  val CONCAT: Value = Value
+  val COUNT: Value = Value
+  val COUNT_IF: Value = Value
+  val CREATE_NAMED_STRUCT: Value = Value
+  val CURRENT_LIKE: Value = Value
+  val DESERIALIZE_TO_OBJECT: Value = Value
+  val DYNAMIC_PRUNING_SUBQUERY: Value = Value
+  val EXISTS_SUBQUERY = Value
+  val EXPRESSION_WITH_RANDOM_SEED: Value = Value
+  val EXTRACT_VALUE: Value = Value
+  val GENERATE: Value = Value
+  val IF: Value = Value
+  val IN: Value = Value
+  val IN_SUBQUERY: Value = Value
+  val INSET: Value = Value
+  val INTERSECT: Value = Value
+  val JSON_TO_STRUCT: Value = Value
+  val LAMBDA_VARIABLE: Value = Value
+  val LIKE_FAMLIY: Value = Value
+  val LIST_SUBQUERY: Value = Value
+  val LITERAL: Value = Value
+  val MAP_OBJECTS: Value = Value
+  val NOT: Value = Value
+  val NULL_CHECK: Value = Value
+  val NULL_LITERAL: Value = Value
+  val SERIALIZE_FROM_OBJECT: Value = Value
+  val OUTER_REFERENCE: Value = Value
+  val PLAN_EXPRESSION: Value = Value
+  val RUNTIME_REPLACEABLE: Value = Value
+  val SCALAR_SUBQUERY: Value = Value
+  val SORT: Value = Value
+  val SUM: Value = Value
+  val TRUE_OR_FALSE_LITERAL: Value = Value
+  val WINDOW_EXPRESSION: Value = Value
+  val UNARY_POSITIVE: Value = Value
+  val UPPER_OR_LOWER: Value = Value
+
+  // Logical plan patterns (alphabetically ordered)
+  val AGGREGATE: Value = Value
+  val DISTINCT_LIKE: Value = Value
+  val EXCEPT: Value = Value
+  val FILTER: Value = Value
+  val INNER_LIKE_JOIN: Value = Value
+  val JOIN: Value = Value
+  val LEFT_SEMI_OR_ANTI_JOIN: Value = Value
+  val LIMIT: Value = Value
+  val LOCAL_RELATION: Value = Value
+  val NATURAL_LIKE_JOIN: Value = Value
+  val OUTER_JOIN: Value = Value
+  val PROJECT: Value = Value
+  val REPARTITION_OPERATION: Value = Value
+  val UNION: Value = Value
+  val TYPED_FILTER: Value = Value
+  val WINDOW: Value = Value
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilder.scala
index 0185b5743e446..1471972362131 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilder.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.util
 import scala.collection.mutable
 
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.array.ByteArrayMethods
@@ -52,24 +53,20 @@ class ArrayBasedMapBuilder(keyType: DataType, valueType: DataType) extends Seria
 
   def put(key: Any, value: Any): Unit = {
     if (key == null) {
-      throw new RuntimeException("Cannot use null as map key.")
+      throw QueryExecutionErrors.nullAsMapKeyNotAllowedError()
     }
 
     val index = keyToIndex.getOrDefault(key, -1)
     if (index == -1) {
       if (size >= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH) {
-        throw new RuntimeException(s"Unsuccessful attempt to build maps with $size elements " +
-          s"due to exceeding the map size limit ${ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH}.")
+        throw QueryExecutionErrors.exceedMapSizeLimitError(size)
       }
       keyToIndex.put(key, values.length)
       keys.append(key)
       values.append(value)
     } else {
       if (mapKeyDedupPolicy == SQLConf.MapKeyDedupPolicy.EXCEPTION.toString) {
-        throw new RuntimeException(s"Duplicate map key $key was found, please check the input " +
-          "data. If you want to remove the duplicated keys, you can set " +
-          s"${SQLConf.MAP_KEY_DEDUP_POLICY.key} to ${SQLConf.MapKeyDedupPolicy.LAST_WIN} so that " +
-          "the key inserted at last takes precedence.")
+        throw QueryExecutionErrors.duplicateMapKeyFoundError(key)
       } else if (mapKeyDedupPolicy == SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
         // Overwrite the previous value, as the policy is last wins.
         values(index) = value
@@ -82,15 +79,14 @@ class ArrayBasedMapBuilder(keyType: DataType, valueType: DataType) extends Seria
   // write a 2-field row, the first field is key and the second field is value.
   def put(entry: InternalRow): Unit = {
     if (entry.isNullAt(0)) {
-      throw new RuntimeException("Cannot use null as map key.")
+      throw QueryExecutionErrors.nullAsMapKeyNotAllowedError()
     }
     put(keyGetter(entry, 0), valueGetter(entry, 1))
   }
 
   def putAll(keyArray: ArrayData, valueArray: ArrayData): Unit = {
     if (keyArray.numElements() != valueArray.numElements()) {
-      throw new RuntimeException(
-        "The key array and value array of MapData must have the same length.")
+      throw QueryExecutionErrors.mapDataKeyArrayLengthDiffersFromValueArrayLengthError()
     }
 
     var i = 0
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapData.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapData.scala
index 5df2af93b94e0..3768f7a1824f1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapData.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapData.scala
@@ -136,7 +136,8 @@ object ArrayBasedMapData {
     keys.zip(values).toMap
   }
 
-  def toScalaMap(keys: Seq[Any], values: Seq[Any]): Map[Any, Any] = {
+  def toScalaMap(keys: scala.collection.Seq[Any],
+      values: scala.collection.Seq[Any]): Map[Any, Any] = {
     keys.zip(values).toMap
   }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayData.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayData.scala
index ebbf241088f80..44203316edd94 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayData.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayData.scala
@@ -22,7 +22,6 @@ import scala.reflect.ClassTag
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{SpecializedGetters, UnsafeArrayData}
 import org.apache.spark.sql.types._
-import org.apache.spark.unsafe.Platform
 import org.apache.spark.unsafe.array.ByteArrayMethods
 
 object ArrayData {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/CharVarcharUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/CharVarcharUtils.scala
new file mode 100644
index 0000000000000..692de9e1a7269
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/CharVarcharUtils.scala
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import scala.collection.mutable
+
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke
+import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types._
+
+object CharVarcharUtils extends Logging {
+
+  private val CHAR_VARCHAR_TYPE_STRING_METADATA_KEY = "__CHAR_VARCHAR_TYPE_STRING"
+
+  /**
+   * Replaces CharType/VarcharType with StringType recursively in the given struct type. If a
+   * top-level StructField's data type is CharType/VarcharType or has nested CharType/VarcharType,
+   * this method will add the original type string to the StructField's metadata, so that we can
+   * re-construct the original data type with CharType/VarcharType later when needed.
+   */
+  def replaceCharVarcharWithStringInSchema(st: StructType): StructType = {
+    StructType(st.map { field =>
+      if (hasCharVarchar(field.dataType)) {
+        val metadata = new MetadataBuilder().withMetadata(field.metadata)
+          .putString(CHAR_VARCHAR_TYPE_STRING_METADATA_KEY, field.dataType.catalogString).build()
+        field.copy(dataType = replaceCharVarcharWithString(field.dataType), metadata = metadata)
+      } else {
+        field
+      }
+    })
+  }
+
+  /**
+   * Returns true if the given data type is CharType/VarcharType or has nested CharType/VarcharType.
+   */
+  def hasCharVarchar(dt: DataType): Boolean = {
+    dt.existsRecursively(f => f.isInstanceOf[CharType] || f.isInstanceOf[VarcharType])
+  }
+
+  /**
+   * Validate the given [[DataType]] to fail if it is char or varchar types or contains nested ones
+   */
+  def failIfHasCharVarchar(dt: DataType): DataType = {
+    if (!SQLConf.get.charVarcharAsString && hasCharVarchar(dt)) {
+      throw new AnalysisException("char/varchar type can only be used in the table schema. " +
+        s"You can set ${SQLConf.LEGACY_CHAR_VARCHAR_AS_STRING.key} to true, so that Spark" +
+        s" treat them as string type as same as Spark 3.0 and earlier")
+    } else {
+      replaceCharVarcharWithString(dt)
+    }
+  }
+
+  /**
+   * Replaces CharType/VarcharType with StringType recursively in the given data type.
+   */
+  def replaceCharVarcharWithString(dt: DataType): DataType = dt match {
+    case ArrayType(et, nullable) =>
+      ArrayType(replaceCharVarcharWithString(et), nullable)
+    case MapType(kt, vt, nullable) =>
+      MapType(replaceCharVarcharWithString(kt), replaceCharVarcharWithString(vt), nullable)
+    case StructType(fields) =>
+      StructType(fields.map { field =>
+        field.copy(dataType = replaceCharVarcharWithString(field.dataType))
+      })
+    case _: CharType => StringType
+    case _: VarcharType => StringType
+    case _ => dt
+  }
+
+  /**
+   * Replaces CharType/VarcharType with StringType recursively in the given data type, with a
+   * warning message if it has char or varchar types
+   */
+  def replaceCharVarcharWithStringForCast(dt: DataType): DataType = {
+    if (SQLConf.get.charVarcharAsString) {
+      replaceCharVarcharWithString(dt)
+    } else if (hasCharVarchar(dt)) {
+      logWarning("The Spark cast operator does not support char/varchar type and simply treats" +
+        " them as string type. Please use string type directly to avoid confusion. Otherwise," +
+        s" you can set ${SQLConf.LEGACY_CHAR_VARCHAR_AS_STRING.key} to true, so that Spark treat" +
+        s" them as string type as same as Spark 3.0 and earlier")
+      replaceCharVarcharWithString(dt)
+    } else {
+      dt
+    }
+  }
+
+  /**
+   * Removes the metadata entry that contains the original type string of CharType/VarcharType from
+   * the given attribute's metadata.
+   */
+  def cleanAttrMetadata(attr: AttributeReference): AttributeReference = {
+    val cleaned = new MetadataBuilder().withMetadata(attr.metadata)
+      .remove(CHAR_VARCHAR_TYPE_STRING_METADATA_KEY).build()
+    attr.withMetadata(cleaned)
+  }
+
+  def getRawTypeString(metadata: Metadata): Option[String] = {
+    if (metadata.contains(CHAR_VARCHAR_TYPE_STRING_METADATA_KEY)) {
+      Some(metadata.getString(CHAR_VARCHAR_TYPE_STRING_METADATA_KEY))
+    } else {
+      None
+    }
+  }
+
+  /**
+   * Re-construct the original data type from the type string in the given metadata.
+   * This is needed when dealing with char/varchar columns/fields.
+   */
+  def getRawType(metadata: Metadata): Option[DataType] = {
+    getRawTypeString(metadata).map(CatalystSqlParser.parseDataType)
+  }
+
+  /**
+   * Re-construct the original schema from the type string in the given metadata of each field.
+   */
+  def getRawSchema(schema: StructType): StructType = {
+    val fields = schema.map { field =>
+      getRawType(field.metadata).map(dt => field.copy(dataType = dt)).getOrElse(field)
+    }
+    StructType(fields)
+  }
+
+  /**
+   * Returns an expression to apply write-side string length check for the given expression. A
+   * string value can not exceed N characters if it's written into a CHAR(N)/VARCHAR(N)
+   * column/field.
+   */
+  def stringLengthCheck(expr: Expression, targetAttr: Attribute): Expression = {
+    getRawType(targetAttr.metadata).map { rawType =>
+      stringLengthCheck(expr, rawType)
+    }.getOrElse(expr)
+  }
+
+  private def stringLengthCheck(expr: Expression, dt: DataType): Expression = {
+    dt match {
+      case CharType(length) =>
+        StaticInvoke(
+          classOf[CharVarcharCodegenUtils],
+          StringType,
+          "charTypeWriteSideCheck",
+          expr :: Literal(length) :: Nil,
+          returnNullable = false)
+
+      case VarcharType(length) =>
+        StaticInvoke(
+          classOf[CharVarcharCodegenUtils],
+          StringType,
+          "varcharTypeWriteSideCheck",
+          expr :: Literal(length) :: Nil,
+          returnNullable = false)
+
+      case StructType(fields) =>
+        val struct = CreateNamedStruct(fields.zipWithIndex.flatMap { case (f, i) =>
+          Seq(Literal(f.name),
+            stringLengthCheck(GetStructField(expr, i, Some(f.name)), f.dataType))
+        })
+        if (expr.nullable) {
+          If(IsNull(expr), Literal(null, struct.dataType), struct)
+        } else {
+          struct
+        }
+
+      case ArrayType(et, containsNull) => stringLengthCheckInArray(expr, et, containsNull)
+
+      case MapType(kt, vt, valueContainsNull) =>
+        val newKeys = stringLengthCheckInArray(MapKeys(expr), kt, containsNull = false)
+        val newValues = stringLengthCheckInArray(MapValues(expr), vt, valueContainsNull)
+        MapFromArrays(newKeys, newValues)
+
+      case _ => expr
+    }
+  }
+
+  private def stringLengthCheckInArray(
+      arr: Expression, et: DataType, containsNull: Boolean): Expression = {
+    val param = NamedLambdaVariable("x", replaceCharVarcharWithString(et), containsNull)
+    val func = LambdaFunction(stringLengthCheck(param, et), Seq(param))
+    ArrayTransform(arr, func)
+  }
+
+  /**
+   * Return expressions to apply char type padding for the string comparison between the given
+   * attributes. When comparing two char type columns/fields, we need to pad the shorter one to
+   * the longer length.
+   */
+  def addPaddingInStringComparison(attrs: Seq[Attribute]): Seq[Expression] = {
+    val rawTypes = attrs.map(attr => getRawType(attr.metadata))
+    if (rawTypes.exists(_.isEmpty)) {
+      attrs
+    } else {
+      val typeWithTargetCharLength = rawTypes.map(_.get).reduce(typeWithWiderCharLength)
+      attrs.zip(rawTypes.map(_.get)).map { case (attr, rawType) =>
+        padCharToTargetLength(attr, rawType, typeWithTargetCharLength).getOrElse(attr)
+      }
+    }
+  }
+
+  private def typeWithWiderCharLength(type1: DataType, type2: DataType): DataType = {
+    (type1, type2) match {
+      case (CharType(len1), CharType(len2)) =>
+        CharType(math.max(len1, len2))
+      case (StructType(fields1), StructType(fields2)) =>
+        assert(fields1.length == fields2.length)
+        StructType(fields1.zip(fields2).map { case (left, right) =>
+          StructField("", typeWithWiderCharLength(left.dataType, right.dataType))
+        })
+      case (ArrayType(et1, _), ArrayType(et2, _)) =>
+        ArrayType(typeWithWiderCharLength(et1, et2))
+      case _ => NullType
+    }
+  }
+
+  private def padCharToTargetLength(
+      expr: Expression,
+      rawType: DataType,
+      typeWithTargetCharLength: DataType): Option[Expression] = {
+    (rawType, typeWithTargetCharLength) match {
+      case (CharType(len), CharType(target)) if target > len =>
+        Some(StringRPad(expr, Literal(target)))
+
+      case (StructType(fields), StructType(targets)) =>
+        assert(fields.length == targets.length)
+        var i = 0
+        var needPadding = false
+        val createStructExprs = mutable.ArrayBuffer.empty[Expression]
+        while (i < fields.length) {
+          val field = fields(i)
+          val fieldExpr = GetStructField(expr, i, Some(field.name))
+          val padded = padCharToTargetLength(fieldExpr, field.dataType, targets(i).dataType)
+          needPadding = padded.isDefined
+          createStructExprs += Literal(field.name)
+          createStructExprs += padded.getOrElse(fieldExpr)
+          i += 1
+        }
+        if (needPadding) Some(CreateNamedStruct(createStructExprs.toSeq)) else None
+
+      case (ArrayType(et, containsNull), ArrayType(target, _)) =>
+        val param = NamedLambdaVariable("x", replaceCharVarcharWithString(et), containsNull)
+        padCharToTargetLength(param, et, target).map { padded =>
+          val func = LambdaFunction(padded, Seq(param))
+          ArrayTransform(expr, func)
+        }
+
+      // We don't handle MapType here as it's not comparable.
+
+      case _ => None
+    }
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
index 0f79c1a6a751d..da80e629ee31d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
@@ -29,18 +29,24 @@ import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy._
 
 sealed trait DateFormatter extends Serializable {
   def parse(s: String): Int // returns days since epoch
+
   def format(days: Int): String
+  def format(date: Date): String
+  def format(localDate: LocalDate): String
+
+  def validatePatternString(): Unit
 }
 
 class Iso8601DateFormatter(
     pattern: String,
     zoneId: ZoneId,
     locale: Locale,
-    legacyFormat: LegacyDateFormats.LegacyDateFormat)
+    legacyFormat: LegacyDateFormats.LegacyDateFormat,
+    isParsing: Boolean)
   extends DateFormatter with DateTimeFormatterHelper {
 
   @transient
-  private lazy val formatter = getOrCreateFormatter(pattern, locale)
+  private lazy val formatter = getOrCreateFormatter(pattern, locale, isParsing)
 
   @transient
   private lazy val legacyFormatter = DateFormatter.getLegacyFormatter(
@@ -50,43 +56,92 @@ class Iso8601DateFormatter(
     val specialDate = convertSpecialDate(s.trim, zoneId)
     specialDate.getOrElse {
       try {
-        val localDate = LocalDate.parse(s, formatter)
+        val localDate = toLocalDate(formatter.parse(s))
         localDateToDays(localDate)
-      } catch checkDiffResult(s, legacyFormatter.parse)
+      } catch checkParsedDiff(s, legacyFormatter.parse)
     }
   }
 
+  override def format(localDate: LocalDate): String = {
+    try {
+      localDate.format(formatter)
+    } catch checkFormattedDiff(toJavaDate(localDateToDays(localDate)),
+      (d: Date) => format(d))
+  }
+
   override def format(days: Int): String = {
-    LocalDate.ofEpochDay(days).format(formatter)
+    format(LocalDate.ofEpochDay(days))
+  }
+
+  override def format(date: Date): String = {
+    legacyFormatter.format(date)
+  }
+
+  override def validatePatternString(): Unit = {
+    try {
+      formatter
+    } catch checkLegacyFormatter(pattern, legacyFormatter.validatePatternString)
   }
 }
 
 trait LegacyDateFormatter extends DateFormatter {
   def parseToDate(s: String): Date
-  def formatDate(d: Date): String
 
   override def parse(s: String): Int = {
     fromJavaDate(new java.sql.Date(parseToDate(s).getTime))
   }
 
   override def format(days: Int): String = {
-    val date = DateTimeUtils.toJavaDate(days)
-    formatDate(date)
+    format(DateTimeUtils.toJavaDate(days))
+  }
+
+  override def format(localDate: LocalDate): String = {
+    format(localDateToDays(localDate))
   }
 }
 
+/**
+ * The legacy formatter is based on Apache Commons FastDateFormat. The formatter uses the default
+ * JVM time zone intentionally for compatibility with Spark 2.4 and earlier versions.
+ *
+ * Note: Using of the default JVM time zone makes the formatter compatible with the legacy
+ *       `DateTimeUtils` methods `toJavaDate` and `fromJavaDate` that are based on the default
+ *       JVM time zone too.
+ *
+ * @param pattern `java.text.SimpleDateFormat` compatible pattern.
+ * @param locale The locale overrides the system locale and is used in parsing/formatting.
+ */
 class LegacyFastDateFormatter(pattern: String, locale: Locale) extends LegacyDateFormatter {
   @transient
   private lazy val fdf = FastDateFormat.getInstance(pattern, locale)
   override def parseToDate(s: String): Date = fdf.parse(s)
-  override def formatDate(d: Date): String = fdf.format(d)
+  override def format(d: Date): String = fdf.format(d)
+  override def validatePatternString(): Unit = fdf
 }
 
+// scalastyle:off line.size.limit
+/**
+ * The legacy formatter is based on `java.text.SimpleDateFormat`. The formatter uses the default
+ * JVM time zone intentionally for compatibility with Spark 2.4 and earlier versions.
+ *
+ * Note: Using of the default JVM time zone makes the formatter compatible with the legacy
+ *       `DateTimeUtils` methods `toJavaDate` and `fromJavaDate` that are based on the default
+ *       JVM time zone too.
+ *
+ * @param pattern The pattern describing the date and time format.
+ *                See <a href="https://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html">
+ *                Date and Time Patterns</a>
+ * @param locale  The locale whose date format symbols should be used. It overrides the system
+ *                locale in parsing/formatting.
+ */
+// scalastyle:on line.size.limit
 class LegacySimpleDateFormatter(pattern: String, locale: Locale) extends LegacyDateFormatter {
   @transient
   private lazy val sdf = new SimpleDateFormat(pattern, locale)
   override def parseToDate(s: String): Date = sdf.parse(s)
-  override def formatDate(d: Date): String = sdf.format(d)
+  override def format(d: Date): String = sdf.format(d)
+  override def validatePatternString(): Unit = sdf
+
 }
 
 object DateFormatter {
@@ -100,12 +155,15 @@ object DateFormatter {
       format: Option[String],
       zoneId: ZoneId,
       locale: Locale = defaultLocale,
-      legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT): DateFormatter = {
+      legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT,
+      isParsing: Boolean): DateFormatter = {
     val pattern = format.getOrElse(defaultPattern)
     if (SQLConf.get.legacyTimeParserPolicy == LEGACY) {
       getLegacyFormatter(pattern, zoneId, locale, legacyFormat)
     } else {
-      new Iso8601DateFormatter(pattern, zoneId, locale, legacyFormat)
+      val df = new Iso8601DateFormatter(pattern, zoneId, locale, legacyFormat, isParsing)
+      df.validatePatternString()
+      df
     }
   }
 
@@ -126,15 +184,16 @@ object DateFormatter {
       format: String,
       zoneId: ZoneId,
       locale: Locale,
-      legacyFormat: LegacyDateFormat): DateFormatter = {
-    getFormatter(Some(format), zoneId, locale, legacyFormat)
+      legacyFormat: LegacyDateFormat,
+      isParsing: Boolean): DateFormatter = {
+    getFormatter(Some(format), zoneId, locale, legacyFormat, isParsing)
   }
 
-  def apply(format: String, zoneId: ZoneId): DateFormatter = {
-    getFormatter(Some(format), zoneId)
+  def apply(format: String, zoneId: ZoneId, isParsing: Boolean = false): DateFormatter = {
+    getFormatter(Some(format), zoneId, isParsing = isParsing)
   }
 
   def apply(zoneId: ZoneId): DateFormatter = {
-    getFormatter(None, zoneId)
+    getFormatter(None, zoneId, isParsing = false)
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
index 05ec23f7ad479..5dc64f083da2e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
@@ -19,29 +19,92 @@ package org.apache.spark.sql.catalyst.util
 
 import java.time._
 import java.time.chrono.IsoChronology
-import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder, DateTimeParseException, ResolverStyle}
+import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder, ResolverStyle}
 import java.time.temporal.{ChronoField, TemporalAccessor, TemporalQueries}
-import java.util.Locale
+import java.util.{Date, Locale}
 
 import com.google.common.cache.CacheBuilder
 
-import org.apache.spark.SparkUpgradeException
 import org.apache.spark.sql.catalyst.util.DateTimeFormatterHelper._
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy._
 
 trait DateTimeFormatterHelper {
+  private def getOrDefault(accessor: TemporalAccessor, field: ChronoField, default: Int): Int = {
+    if (accessor.isSupported(field)) {
+      accessor.get(field)
+    } else {
+      default
+    }
+  }
+
+  private def verifyLocalDate(
+      accessor: TemporalAccessor, field: ChronoField, candidate: LocalDate): Unit = {
+    if (accessor.isSupported(field)) {
+      val actual = accessor.get(field)
+      val expected = candidate.get(field)
+      if (actual != expected) {
+        throw QueryExecutionErrors.fieldDiffersFromDerivedLocalDateError(
+          field, actual, expected, candidate)
+      }
+    }
+  }
+
+  protected def toLocalDate(accessor: TemporalAccessor): LocalDate = {
+    val localDate = accessor.query(TemporalQueries.localDate())
+    // If all the date fields are specified, return the local date directly.
+    if (localDate != null) return localDate
+
+    // Users may want to parse only a few datetime fields from a string and extract these fields
+    // later, and we should provide default values for missing fields.
+    // To be compatible with Spark 2.4, we pick 1970 as the default value of year.
+    val year = getOrDefault(accessor, ChronoField.YEAR, 1970)
+    if (accessor.isSupported(ChronoField.DAY_OF_YEAR)) {
+      val dayOfYear = accessor.get(ChronoField.DAY_OF_YEAR)
+      val date = LocalDate.ofYearDay(year, dayOfYear)
+      verifyLocalDate(accessor, ChronoField.MONTH_OF_YEAR, date)
+      verifyLocalDate(accessor, ChronoField.DAY_OF_MONTH, date)
+      date
+    } else {
+      val month = getOrDefault(accessor, ChronoField.MONTH_OF_YEAR, 1)
+      val day = getOrDefault(accessor, ChronoField.DAY_OF_MONTH, 1)
+      LocalDate.of(year, month, day)
+    }
+  }
+
+  private def toLocalTime(accessor: TemporalAccessor): LocalTime = {
+    val localTime = accessor.query(TemporalQueries.localTime())
+    // If all the time fields are specified, return the local time directly.
+    if (localTime != null) return localTime
+
+    val hour = if (accessor.isSupported(ChronoField.HOUR_OF_DAY)) {
+      accessor.get(ChronoField.HOUR_OF_DAY)
+    } else if (accessor.isSupported(ChronoField.HOUR_OF_AMPM)) {
+      // When we reach here, it means am/pm is not specified. Here we assume it's am.
+      // All of CLOCK_HOUR_OF_AMPM(h)/HOUR_OF_DAY(H)/CLOCK_HOUR_OF_DAY(k)/HOUR_OF_AMPM(K) will
+      // be resolved to HOUR_OF_AMPM here, we do not need to handle them separately
+      accessor.get(ChronoField.HOUR_OF_AMPM)
+    } else if (accessor.isSupported(ChronoField.AMPM_OF_DAY) &&
+      accessor.get(ChronoField.AMPM_OF_DAY) == 1) {
+      // When reach here, the `hour` part is missing, and PM is specified.
+      // None of CLOCK_HOUR_OF_AMPM(h)/HOUR_OF_DAY(H)/CLOCK_HOUR_OF_DAY(k)/HOUR_OF_AMPM(K) is
+      // specified
+      12
+    } else {
+      0
+    }
+    val minute = getOrDefault(accessor, ChronoField.MINUTE_OF_HOUR, 0)
+    val second = getOrDefault(accessor, ChronoField.SECOND_OF_MINUTE, 0)
+    val nanoSecond = getOrDefault(accessor, ChronoField.NANO_OF_SECOND, 0)
+    LocalTime.of(hour, minute, second, nanoSecond)
+  }
+
   // Converts the parsed temporal object to ZonedDateTime. It sets time components to zeros
   // if they does not exist in the parsed object.
-  protected def toZonedDateTime(
-      temporalAccessor: TemporalAccessor,
-      zoneId: ZoneId): ZonedDateTime = {
-    // Parsed input might not have time related part. In that case, time component is set to zeros.
-    val parsedLocalTime = temporalAccessor.query(TemporalQueries.localTime)
-    val localTime = if (parsedLocalTime == null) LocalTime.MIDNIGHT else parsedLocalTime
-    // Parsed input must have date component. At least, year must present in temporalAccessor.
-    val localDate = temporalAccessor.query(TemporalQueries.localDate)
-
+  protected def toZonedDateTime(accessor: TemporalAccessor, zoneId: ZoneId): ZonedDateTime = {
+    val localDate = toLocalDate(accessor)
+    val localTime = toLocalTime(accessor)
     ZonedDateTime.of(localDate, localTime, zoneId)
   }
 
@@ -54,9 +117,9 @@ trait DateTimeFormatterHelper {
   protected def getOrCreateFormatter(
       pattern: String,
       locale: Locale,
-      needVarLengthSecondFraction: Boolean = false): DateTimeFormatter = {
-    val newPattern = convertIncompatiblePattern(pattern)
-    val useVarLen = needVarLengthSecondFraction && newPattern.contains('S')
+      isParsing: Boolean): DateTimeFormatter = {
+    val newPattern = convertIncompatiblePattern(pattern, isParsing)
+    val useVarLen = isParsing && newPattern.contains('S')
     val key = (newPattern, locale, useVarLen)
     var formatter = cache.getIfPresent(key)
     if (formatter == null) {
@@ -66,25 +129,62 @@ trait DateTimeFormatterHelper {
     formatter
   }
 
+  private def needConvertToSparkUpgradeException(e: Throwable): Boolean = e match {
+    case _: DateTimeException if SQLConf.get.legacyTimeParserPolicy == EXCEPTION => true
+    case _ => false
+  }
   // When legacy time parser policy set to EXCEPTION, check whether we will get different results
   // between legacy parser and new parser. If new parser fails but legacy parser works, throw a
   // SparkUpgradeException. On the contrary, if the legacy policy set to CORRECTED,
   // DateTimeParseException will address by the caller side.
-  protected def checkDiffResult[T](
+  protected def checkParsedDiff[T](
       s: String, legacyParseFunc: String => T): PartialFunction[Throwable, T] = {
-    case e: DateTimeParseException if SQLConf.get.legacyTimeParserPolicy == EXCEPTION =>
-      val res = try {
-        Some(legacyParseFunc(s))
+    case e if needConvertToSparkUpgradeException(e) =>
+      try {
+        legacyParseFunc(s)
       } catch {
-        case _: Throwable => None
+        case _: Throwable => throw e
       }
-      if (res.nonEmpty) {
-        throw new SparkUpgradeException("3.0", s"Fail to parse '$s' in the new parser. You can " +
-          s"set ${SQLConf.LEGACY_TIME_PARSER_POLICY.key} to LEGACY to restore the behavior " +
-          s"before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.", e)
-      } else {
-        throw e
+      throw QueryExecutionErrors.failToParseDateTimeInNewParserError(s, e)
+  }
+
+  // When legacy time parser policy set to EXCEPTION, check whether we will get different results
+  // between legacy formatter and new formatter. If new formatter fails but legacy formatter works,
+  // throw a SparkUpgradeException. On the contrary, if the legacy policy set to CORRECTED,
+  // DateTimeParseException will address by the caller side.
+  protected def checkFormattedDiff[T <: Date](
+      d: T,
+      legacyFormatFunc: T => String): PartialFunction[Throwable, String] = {
+    case e if needConvertToSparkUpgradeException(e) =>
+      val resultCandidate = try {
+        legacyFormatFunc(d)
+      } catch {
+        case _: Throwable => throw e
+      }
+      throw QueryExecutionErrors.failToFormatDateTimeInNewFormatterError(resultCandidate, e)
+  }
+
+  /**
+   * When the new DateTimeFormatter failed to initialize because of invalid datetime pattern, it
+   * will throw IllegalArgumentException. If the pattern can be recognized by the legacy formatter
+   * it will raise SparkUpgradeException to tell users to restore the previous behavior via LEGACY
+   * policy or follow our guide to correct their pattern. Otherwise, the original
+   * IllegalArgumentException will be thrown.
+   *
+   * @param pattern the date time pattern
+   * @param tryLegacyFormatter a func to capture exception, identically which forces a legacy
+   *                           datetime formatter to be initialized
+   */
+  protected def checkLegacyFormatter(
+      pattern: String,
+      tryLegacyFormatter: => Unit): PartialFunction[Throwable, DateTimeFormatter] = {
+    case e: IllegalArgumentException =>
+      try {
+        tryLegacyFormatter
+      } catch {
+        case _: Throwable => throw e
       }
+      throw QueryExecutionErrors.failToRecognizePatternInDateTimeFormatterError(pattern, e)
   }
 }
 
@@ -101,10 +201,6 @@ private object DateTimeFormatterHelper {
 
   def toFormatter(builder: DateTimeFormatterBuilder, locale: Locale): DateTimeFormatter = {
     builder
-      .parseDefaulting(ChronoField.MONTH_OF_YEAR, 1)
-      .parseDefaulting(ChronoField.DAY_OF_MONTH, 1)
-      .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0)
-      .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0)
       .toFormatter(locale)
       .withChronology(IsoChronology.INSTANCE)
       .withResolverStyle(ResolverStyle.STRICT)
@@ -121,8 +217,8 @@ private object DateTimeFormatterHelper {
       // string at res(0). So when the first element here is empty string we do not need append `'`
       // literal to the DateTimeFormatterBuilder.
       case ("", idx) if idx != 0 => builder.appendLiteral("'")
-      case (pattenPart, idx) if idx % 2 == 0 =>
-        var rest = pattenPart
+      case (patternPart, idx) if idx % 2 == 0 =>
+        var rest = patternPart
         while (rest.nonEmpty) {
           rest match {
             case extractor(prefix, secondFraction, suffix) =>
@@ -162,7 +258,39 @@ private object DateTimeFormatterHelper {
     toFormatter(builder, TimestampFormatter.defaultLocale)
   }
 
-  final val unsupportedLetters = Set('A', 'c', 'e', 'n', 'N', 'p')
+  private final val bugInStandAloneForm = {
+    // Java 8 has a bug for stand-alone form. See https://bugs.openjdk.java.net/browse/JDK-8114833
+    // Note: we only check the US locale so that it's a static check. It can produce false-negative
+    // as some locales are not affected by the bug. Since `L`/`q` is rarely used, we choose to not
+    // complicate the check here.
+    // TODO: remove it when we drop Java 8 support.
+    val formatter = DateTimeFormatter.ofPattern("LLL qqq", Locale.US)
+    formatter.format(LocalDate.of(2000, 1, 1)) == "1 1"
+  }
+  // SPARK-31892: The week-based date fields are rarely used and really confusing for parsing values
+  // to datetime, especially when they are mixed with other non-week-based ones;
+  // SPARK-31879: It's also difficult for us to restore the behavior of week-based date fields
+  // formatting, in DateTimeFormatter the first day of week for week-based date fields become
+  // localized, for the default Locale.US, it uses Sunday as the first day of week, while in Spark
+  // 2.4, the SimpleDateFormat uses Monday as the first day of week.
+  final val weekBasedLetters = Set('Y', 'W', 'w', 'u', 'e', 'c')
+  final val unsupportedLetters = Set('A', 'n', 'N', 'p')
+  // The quarter fields will also be parsed strangely, e.g. when the pattern contains `yMd` and can
+  // be directly resolved then the `q` do check for whether the month is valid, but if the date
+  // fields is incomplete, e.g. `yM`, the checking will be bypassed.
+  final val unsupportedLettersForParsing = Set('E', 'F', 'q', 'Q')
+  final val unsupportedPatternLengths = {
+    // SPARK-31771: Disable Narrow-form TextStyle to avoid silent data change, as it is Full-form in
+    // 2.4
+    Seq("G", "M", "L", "E", "Q", "q").map(_ * 5) ++
+      // SPARK-31867: Disable year pattern longer than 10 which will cause Java time library throw
+      // unchecked `ArrayIndexOutOfBoundsException` by the `NumberPrinterParser` for formatting. It
+      // makes the call side difficult to handle exceptions and easily leads to silent data change
+      // because of the exceptions being suppressed.
+      // SPARK-32424: The max year that we can actually handle is 6 digits, otherwise, it will
+      // overflow
+      Seq("y").map(_ * 7)
+  }.toSet
 
   /**
    * In Spark 3.0, we switch to the Proleptic Gregorian calendar and use DateTimeFormatter for
@@ -172,7 +300,7 @@ private object DateTimeFormatterHelper {
    * @param pattern The input pattern.
    * @return The pattern for new parser
    */
-  def convertIncompatiblePattern(pattern: String): String = {
+  def convertIncompatiblePattern(pattern: String, isParsing: Boolean): String = {
     val eraDesignatorContained = pattern.split("'").zipWithIndex.exists {
       case (patternPart, index) =>
         // Text can be quoted using single quotes, we only check the non-quote parts.
@@ -181,23 +309,30 @@ private object DateTimeFormatterHelper {
     (pattern + " ").split("'").zipWithIndex.map {
       case (patternPart, index) =>
         if (index % 2 == 0) {
-          for (c <- patternPart if unsupportedLetters.contains(c)) {
+          for (c <- patternPart if weekBasedLetters.contains(c)) {
+            throw new IllegalArgumentException(s"All week-based patterns are unsupported since" +
+              s" Spark 3.0, detected: $c, Please use the SQL function EXTRACT instead")
+          }
+          for (c <- patternPart if unsupportedLetters.contains(c) ||
+            (isParsing && unsupportedLettersForParsing.contains(c))) {
             throw new IllegalArgumentException(s"Illegal pattern character: $c")
           }
-          // The meaning of 'u' was day number of week in SimpleDateFormat, it was changed to year
-          // in DateTimeFormatter. Substitute 'u' to 'e' and use DateTimeFormatter to parse the
-          // string. If parsable, return the result; otherwise, fall back to 'u', and then use the
-          // legacy SimpleDateFormat parser to parse. When it is successfully parsed, throw an
-          // exception and ask users to change the pattern strings or turn on the legacy mode;
-          // otherwise, return NULL as what Spark 2.4 does.
-          val res = patternPart.replace("u", "e")
+          for (style <- unsupportedPatternLengths if patternPart.contains(style)) {
+            throw new IllegalArgumentException(s"Too many pattern letters: ${style.head}")
+          }
+          if (bugInStandAloneForm && (patternPart.contains("LLL") || patternPart.contains("qqq"))) {
+            throw new IllegalArgumentException("Java 8 has a bug to support stand-alone " +
+              "form (3 or more 'L' or 'q' in the pattern string). Please use 'M' or 'Q' instead, " +
+              "or upgrade your Java version. For more details, please read " +
+              "https://bugs.openjdk.java.net/browse/JDK-8114833")
+          }
           // In DateTimeFormatter, 'u' supports negative years. We substitute 'y' to 'u' here for
           // keeping the support in Spark 3.0. If parse failed in Spark 3.0, fall back to 'y'.
           // We only do this substitution when there is no era designator found in the pattern.
           if (!eraDesignatorContained) {
-            res.replace("y", "u")
+            patternPart.replace("y", "u")
           } else {
-            res
+            patternPart
           }
         } else {
           patternPart
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index 21a478aaf06a6..eeaf086097b66 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -30,7 +30,8 @@ import sun.util.calendar.ZoneInfo
 
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
 import org.apache.spark.sql.catalyst.util.RebaseDateTime._
-import org.apache.spark.sql.types.Decimal
+import org.apache.spark.sql.errors.QueryExecutionErrors
+import org.apache.spark.sql.types.{DateType, Decimal, TimestampType}
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 
 /**
@@ -42,67 +43,52 @@ import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
  */
 object DateTimeUtils {
 
-  // we use Int and Long internally to represent [[DateType]] and [[TimestampType]]
-  type SQLDate = Int
-  type SQLTimestamp = Long
-
-  // see http://stackoverflow.com/questions/466321/convert-unix-timestamp-to-julian
-  // it's 2440587.5, rounding up to compatible with Hive
+  // See http://stackoverflow.com/questions/466321/convert-unix-timestamp-to-julian
+  // It's 2440587.5, rounding up to be compatible with Hive.
   final val JULIAN_DAY_OF_EPOCH = 2440588
 
-  final val julianCommonEraStart = Timestamp.valueOf("0001-01-01 00:00:00")
-
-  final val TimeZoneGMT = TimeZone.getTimeZone("GMT")
   final val TimeZoneUTC = TimeZone.getTimeZone("UTC")
 
   val TIMEZONE_OPTION = "timeZone"
 
-  def defaultTimeZone(): TimeZone = TimeZone.getDefault()
-
-  def getZoneId(timeZoneId: String): ZoneId = ZoneId.of(timeZoneId, ZoneId.SHORT_IDS)
-  def getTimeZone(timeZoneId: String): TimeZone = {
-    TimeZone.getTimeZone(getZoneId(timeZoneId))
+  def getZoneId(timeZoneId: String): ZoneId = {
+    // To support the (+|-)h:mm format because it was supported before Spark 3.0.
+    ZoneId.of(timeZoneId.replaceFirst("(\\+|\\-)(\\d):", "$10$2:"), ZoneId.SHORT_IDS)
   }
+  def getTimeZone(timeZoneId: String): TimeZone = TimeZone.getTimeZone(getZoneId(timeZoneId))
 
-  def microsToDays(timestamp: SQLTimestamp): SQLDate = {
-    microsToDays(timestamp, defaultTimeZone().toZoneId)
-  }
-
-  def microsToDays(timestamp: SQLTimestamp, zoneId: ZoneId): SQLDate = {
-    val instant = microsToInstant(timestamp)
-    localDateToDays(LocalDateTime.ofInstant(instant, zoneId).toLocalDate)
-  }
-
-  def daysToMicros(days: SQLDate): SQLTimestamp = {
-    daysToMicros(days, defaultTimeZone().toZoneId)
+  /**
+   * Converts microseconds since 1970-01-01 00:00:00Z to days since 1970-01-01 at the given zone ID.
+   */
+  def microsToDays(micros: Long, zoneId: ZoneId): Int = {
+    localDateToDays(getLocalDateTime(micros, zoneId).toLocalDate)
   }
 
-  def daysToMicros(days: SQLDate, zoneId: ZoneId): SQLTimestamp = {
+  /**
+   * Converts days since 1970-01-01 at the given zone ID to microseconds since 1970-01-01 00:00:00Z.
+   */
+  def daysToMicros(days: Int, zoneId: ZoneId): Long = {
     val instant = daysToLocalDate(days).atStartOfDay(zoneId).toInstant
     instantToMicros(instant)
   }
 
-  // Converts Timestamp to string according to Hive TimestampWritable convention.
-  def timestampToString(tf: TimestampFormatter, us: SQLTimestamp): String = {
-    tf.format(us)
-  }
-
   /**
-   * Converts an instance of `java.sql.Date` to a number of days since the epoch
-   * 1970-01-01 via extracting date fields `year`, `month`, `days` from the input,
-   * creating a local date in Proleptic Gregorian calendar from the fields, and
-   * getting the number of days from the resulted local date.
+   * Converts a local date at the default JVM time zone to the number of days since 1970-01-01
+   * in the hybrid calendar (Julian + Gregorian) by discarding the time part. The resulted days are
+   * rebased from the hybrid to Proleptic Gregorian calendar. The days rebasing is performed via
+   * UTC time zone for simplicity because the difference between two calendars is the same in
+   * any given time zone and UTC time zone.
    *
-   * This approach was taken to have the same local date as the triple of `year`,
-   * `month`, `day` in the original hybrid calendar used by `java.sql.Date` and
-   * Proleptic Gregorian calendar used by Spark since version 3.0.0, see SPARK-26651.
+   * Note: The date is shifted by the offset of the default JVM time zone for backward compatibility
+   *       with Spark 2.4 and earlier versions. The goal of the shift is to get a local date derived
+   *       from the number of days that has the same date fields (year, month, day) as the original
+   *       `date` at the default JVM time zone.
    *
-   * @param date It represents a specific instant in time based on
-   *             the hybrid calendar which combines Julian and
-   *             Gregorian calendars.
-   * @return The number of days since epoch from java.sql.Date.
+   * @param date It represents a specific instant in time based on the hybrid calendar which
+   *             combines Julian and Gregorian calendars.
+   * @return The number of days since the epoch in Proleptic Gregorian calendar.
    */
-  def fromJavaDate(date: Date): SQLDate = {
+  def fromJavaDate(date: Date): Int = {
     val millisUtc = date.getTime
     val millisLocal = millisUtc + TimeZone.getDefault.getOffset(millisUtc)
     val julianDays = Math.toIntExact(Math.floorDiv(millisLocal, MILLIS_PER_DAY))
@@ -110,20 +96,21 @@ object DateTimeUtils {
   }
 
   /**
-   * The opposite to `fromJavaDate` method which converts a number of days to an
-   * instance of `java.sql.Date`. It builds a local date in Proleptic Gregorian
-   * calendar, extracts date fields `year`, `month`, `day`, and creates a local
-   * date in the hybrid calendar (Julian + Gregorian calendars) from the fields.
+   * Converts days since the epoch 1970-01-01 in Proleptic Gregorian calendar to a local date
+   * at the default JVM time zone in the hybrid calendar (Julian + Gregorian). It rebases the given
+   * days from Proleptic Gregorian to the hybrid calendar at UTC time zone for simplicity because
+   * the difference between two calendars doesn't depend on any time zone. The result is shifted
+   * by the time zone offset in wall clock to have the same date fields (year, month, day)
+   * at the default JVM time zone as the input `daysSinceEpoch` in Proleptic Gregorian calendar.
    *
-   * The purpose of the conversion is to have the same local date as the triple
-   * of `year`, `month`, `day` in the original Proleptic Gregorian calendar and
-   * in the target calender.
+   * Note: The date is shifted by the offset of the default JVM time zone for backward compatibility
+   *       with Spark 2.4 and earlier versions.
    *
-   * @param daysSinceEpoch The number of days since 1970-01-01.
-   * @return A `java.sql.Date` from number of days since epoch.
+   * @param days The number of days since 1970-01-01 in Proleptic Gregorian calendar.
+   * @return A local date in the hybrid calendar as `java.sql.Date` from number of days since epoch.
    */
-  def toJavaDate(daysSinceEpoch: SQLDate): Date = {
-    val rebasedDays = rebaseGregorianToJulianDays(daysSinceEpoch)
+  def toJavaDate(days: Int): Date = {
+    val rebasedDays = rebaseGregorianToJulianDays(days)
     val localMillis = Math.multiplyExact(rebasedDays, MILLIS_PER_DAY)
     val timeZoneOffset = TimeZone.getDefault match {
       case zoneInfo: ZoneInfo => zoneInfo.getOffsetsByWall(localMillis, null)
@@ -145,11 +132,11 @@ object DateTimeUtils {
    * representation as `year`, `month`, `day`, ..., `seconds` in the original calendar
    * and in the target calendar.
    *
-   * @param us The number of microseconds since 1970-01-01T00:00:00.000000Z.
+   * @param micros The number of microseconds since 1970-01-01T00:00:00.000000Z.
    * @return A `java.sql.Timestamp` from number of micros since epoch.
    */
-  def toJavaTimestamp(us: SQLTimestamp): Timestamp = {
-    val rebasedMicros = rebaseGregorianToJulianMicros(us)
+  def toJavaTimestamp(micros: Long): Timestamp = {
+    val rebasedMicros = rebaseGregorianToJulianMicros(micros)
     val seconds = Math.floorDiv(rebasedMicros, MICROS_PER_SECOND)
     val ts = new Timestamp(seconds * MILLIS_PER_SECOND)
     val nanos = (rebasedMicros - seconds * MICROS_PER_SECOND) * NANOS_PER_MICROS
@@ -176,21 +163,17 @@ object DateTimeUtils {
    *          Gregorian calendars.
    * @return The number of micros since epoch from `java.sql.Timestamp`.
    */
-  def fromJavaTimestamp(t: Timestamp): SQLTimestamp = {
+  def fromJavaTimestamp(t: Timestamp): Long = {
     val micros = millisToMicros(t.getTime) + (t.getNanos / NANOS_PER_MICROS) % MICROS_PER_MILLIS
     rebaseJulianToGregorianMicros(micros)
   }
 
   /**
-   * Returns the number of microseconds since epoch from Julian day
-   * and nanoseconds in a day
+   * Returns the number of microseconds since epoch from Julian day and nanoseconds in a day.
    */
-  def fromJulianDay(day: Int, nanoseconds: Long): SQLTimestamp = {
+  def fromJulianDay(days: Int, nanos: Long): Long = {
     // use Long to avoid rounding errors
-    val seconds = (day - JULIAN_DAY_OF_EPOCH).toLong * SECONDS_PER_DAY
-    val micros = SECONDS.toMicros(seconds) + NANOSECONDS.toMicros(nanoseconds)
-    val rebased = rebaseJulianToGregorianMicros(micros)
-    rebased
+    (days - JULIAN_DAY_OF_EPOCH).toLong * MICROS_PER_DAY + nanos / NANOS_PER_MICROS
   }
 
   /**
@@ -198,60 +181,39 @@ object DateTimeUtils {
    *
    * Note: support timestamp since 4717 BC (without negative nanoseconds, compatible with Hive).
    */
-  def toJulianDay(us: SQLTimestamp): (Int, Long) = {
-    val julian_us = rebaseGregorianToJulianMicros(us) + JULIAN_DAY_OF_EPOCH * MICROS_PER_DAY
-    val day = julian_us / MICROS_PER_DAY
-    val micros = julian_us % MICROS_PER_DAY
-    (day.toInt, MICROSECONDS.toNanos(micros))
+  def toJulianDay(micros: Long): (Int, Long) = {
+    val julianUs = micros + JULIAN_DAY_OF_EPOCH * MICROS_PER_DAY
+    val days = julianUs / MICROS_PER_DAY
+    val us = julianUs % MICROS_PER_DAY
+    (days.toInt, MICROSECONDS.toNanos(us))
   }
 
-  /*
-   * Converts the timestamp to milliseconds since epoch. In spark timestamp values have microseconds
+  /**
+   * Converts the timestamp to milliseconds since epoch. In Spark timestamp values have microseconds
    * precision, so this conversion is lossy.
    */
-  def microsToMillis(us: SQLTimestamp): Long = {
-    // When the timestamp is negative i.e before 1970, we need to adjust the millseconds portion.
+  def microsToMillis(micros: Long): Long = {
+    // When the timestamp is negative i.e before 1970, we need to adjust the milliseconds portion.
     // Example - 1965-01-01 10:11:12.123456 is represented as (-157700927876544) in micro precision.
     // In millis precision the above needs to be represented as (-157700927877).
-    Math.floorDiv(us, MICROS_PER_MILLIS)
+    Math.floorDiv(micros, MICROS_PER_MILLIS)
   }
 
-  /*
-   * Converts milliseconds since epoch to SQLTimestamp.
+  /**
+   * Converts milliseconds since the epoch to microseconds.
    */
-  def millisToMicros(millis: Long): SQLTimestamp = {
+  def millisToMicros(millis: Long): Long = {
     Math.multiplyExact(millis, MICROS_PER_MILLIS)
   }
 
-  def microsToEpochDays(epochMicros: SQLTimestamp, zoneId: ZoneId): SQLDate = {
-    localDateToDays(microsToInstant(epochMicros).atZone(zoneId).toLocalDate)
-  }
-
-  def epochDaysToMicros(epochDays: SQLDate, zoneId: ZoneId): SQLTimestamp = {
-    val localDate = LocalDate.ofEpochDay(epochDays)
-    val zeroLocalTime = LocalTime.MIDNIGHT
-    val localDateTime = LocalDateTime.of(localDate, zeroLocalTime)
-    instantToMicros(localDateTime.atZone(zoneId).toInstant)
-  }
-
-  // A method called by JSON/CSV parser to clean up the legacy timestamp string by removing the
-  // "GMT" string.
-  def cleanLegacyTimestampStr(s: String): String = {
-    val indexOfGMT = s.indexOf("GMT")
-    if (indexOfGMT != -1) {
-      // ISO8601 with a weird time zone specifier (2000-01-01T00:00GMT+01:00)
-      val s0 = s.substring(0, indexOfGMT)
-      val s1 = s.substring(indexOfGMT + 3)
-      // Mapped to 2000-01-01T00:00+01:00
-      s0 + s1
-    } else {
-      s
-    }
-  }
+  private final val gmtUtf8 = UTF8String.fromString("GMT")
+  // The method is called by JSON/CSV parser to clean up the legacy timestamp string by removing
+  // the "GMT" string. For example, it returns 2000-01-01T00:00+01:00 for 2000-01-01T00:00GMT+01:00.
+  def cleanLegacyTimestampStr(s: UTF8String): UTF8String = s.replace(gmtUtf8, UTF8String.EMPTY_UTF8)
 
   /**
-   * Trim and parse a given UTF8 date string to the corresponding a corresponding [[Long]] value.
-   * The return type is [[Option]] in order to distinguish between 0L and null. The following
+   * Trims and parses a given UTF8 timestamp string to the corresponding a corresponding [[Long]]
+   * value. The return type is [[Option]] in order to distinguish between 0L and null. The following
    * formats are allowed:
    *
    * `yyyy`
@@ -275,7 +237,7 @@ object DateTimeUtils {
    *     - +|-hhmmss
    *  - Region-based zone IDs in the form `area/city`, such as `Europe/Paris`
    */
-  def stringToTimestamp(s: UTF8String, timeZoneId: ZoneId): Option[SQLTimestamp] = {
+  def stringToTimestamp(s: UTF8String, timeZoneId: ZoneId): Option[Long] = {
     if (s == null) {
       return None
     }
@@ -406,34 +368,47 @@ object DateTimeUtils {
     }
   }
 
+  def stringToTimestampAnsi(s: UTF8String, timeZoneId: ZoneId): Long = {
+    stringToTimestamp(s, timeZoneId).getOrElse {
+      throw QueryExecutionErrors.cannotCastUTF8StringToDataTypeError(s, TimestampType)
+    }
+  }
+
+  /**
+   * Gets the number of microseconds since the epoch of 1970-01-01 00:00:00Z from the given
+   * instance of `java.time.Instant`. The epoch microsecond count is a simple incrementing count of
+   * microseconds where microsecond 0 is 1970-01-01 00:00:00Z.
+   */
   def instantToMicros(instant: Instant): Long = {
     val us = Math.multiplyExact(instant.getEpochSecond, MICROS_PER_SECOND)
     val result = Math.addExact(us, NANOSECONDS.toMicros(instant.getNano))
     result
   }
 
-  def microsToInstant(us: Long): Instant = {
-    val secs = Math.floorDiv(us, MICROS_PER_SECOND)
+  /**
+   * Obtains an instance of `java.time.Instant` using microseconds from
+   * the epoch of 1970-01-01 00:00:00Z.
+   */
+  def microsToInstant(micros: Long): Instant = {
+    val secs = Math.floorDiv(micros, MICROS_PER_SECOND)
     // Unfolded Math.floorMod(us, MICROS_PER_SECOND) to reuse the result of
     // the above calculation of `secs` via `floorDiv`.
-    val mos = us - secs * MICROS_PER_SECOND
+    val mos = micros - secs * MICROS_PER_SECOND
     Instant.ofEpochSecond(secs, mos * NANOS_PER_MICROS)
   }
 
-  def instantToDays(instant: Instant): Int = {
-    val seconds = instant.getEpochSecond
-    val days = Math.floorDiv(seconds, SECONDS_PER_DAY)
-    days.toInt
-  }
-
-  def localDateToDays(localDate: LocalDate): Int = {
-    Math.toIntExact(localDate.toEpochDay)
-  }
+  /**
+   * Converts the local date to the number of days since 1970-01-01.
+   */
+  def localDateToDays(localDate: LocalDate): Int = Math.toIntExact(localDate.toEpochDay)
 
+  /**
+   * Obtains an instance of `java.time.LocalDate` from the epoch day count.
+   */
   def daysToLocalDate(days: Int): LocalDate = LocalDate.ofEpochDay(days)
 
   /**
-   * Trim and parse a given UTF8 date string to a corresponding [[Int]] value.
+   * Trims and parses a given UTF8 date string to a corresponding [[Int]] value.
    * The return type is [[Option]] in order to distinguish between 0 and null. The following
    * formats are allowed:
    *
@@ -444,7 +419,7 @@ object DateTimeUtils {
    * `yyyy-[m]m-[d]d *`
    * `yyyy-[m]m-[d]dT*`
    */
-  def stringToDate(s: UTF8String, zoneId: ZoneId): Option[SQLDate] = {
+  def stringToDate(s: UTF8String, zoneId: ZoneId): Option[Int] = {
     if (s == null) {
       return None
     }
@@ -492,124 +467,164 @@ object DateTimeUtils {
     }
   }
 
-  private def localTimestamp(microsec: SQLTimestamp, zoneId: ZoneId): LocalDateTime = {
-    microsToInstant(microsec).atZone(zoneId).toLocalDateTime
+  def stringToDateAnsi(s: UTF8String, zoneId: ZoneId): Int = {
+    stringToDate(s, zoneId).getOrElse {
+      throw QueryExecutionErrors.cannotCastUTF8StringToDataTypeError(s, DateType)
+    }
+  }
+
+  // Gets the local date-time parts (year, month, day and time) of the instant expressed as the
+  // number of microseconds since the epoch at the given time zone ID.
+  private def getLocalDateTime(micros: Long, zoneId: ZoneId): LocalDateTime = {
+    microsToInstant(micros).atZone(zoneId).toLocalDateTime
   }
 
   /**
    * Returns the hour value of a given timestamp value. The timestamp is expressed in microseconds.
    */
-  def getHours(microsec: SQLTimestamp, zoneId: ZoneId): Int = {
-    localTimestamp(microsec, zoneId).getHour
+  def getHours(micros: Long, zoneId: ZoneId): Int = {
+    getLocalDateTime(micros, zoneId).getHour
   }
 
   /**
    * Returns the minute value of a given timestamp value. The timestamp is expressed in
-   * microseconds.
+   * microseconds since the epoch.
    */
-  def getMinutes(microsec: SQLTimestamp, zoneId: ZoneId): Int = {
-    localTimestamp(microsec, zoneId).getMinute
+  def getMinutes(micros: Long, zoneId: ZoneId): Int = {
+    getLocalDateTime(micros, zoneId).getMinute
   }
 
   /**
    * Returns the second value of a given timestamp value. The timestamp is expressed in
-   * microseconds.
+   * microseconds since the epoch.
    */
-  def getSeconds(microsec: SQLTimestamp, zoneId: ZoneId): Int = {
-    localTimestamp(microsec, zoneId).getSecond
+  def getSeconds(micros: Long, zoneId: ZoneId): Int = {
+    getLocalDateTime(micros, zoneId).getSecond
   }
 
   /**
    * Returns the seconds part and its fractional part with microseconds.
    */
-  def getSecondsWithFraction(microsec: SQLTimestamp, zoneId: ZoneId): Decimal = {
-    Decimal(getMicroseconds(microsec, zoneId), 8, 6)
+  def getSecondsWithFraction(micros: Long, zoneId: ZoneId): Decimal = {
+    Decimal(getMicroseconds(micros, zoneId), 8, 6)
   }
 
   /**
-   * Returns seconds, including fractional parts, multiplied by 1000000. The timestamp
-   * is expressed in microseconds since the epoch.
+   * Returns local seconds, including fractional parts, multiplied by 1000000.
+   *
+   * @param micros The number of microseconds since the epoch.
+   * @param zoneId The time zone id which milliseconds should be obtained in.
    */
-  def getMicroseconds(timestamp: SQLTimestamp, zoneId: ZoneId): Int = {
-    val lt = localTimestamp(timestamp, zoneId)
+  def getMicroseconds(micros: Long, zoneId: ZoneId): Int = {
+    val lt = getLocalDateTime(micros, zoneId)
     (lt.getLong(ChronoField.MICRO_OF_SECOND) + lt.getSecond * MICROS_PER_SECOND).toInt
   }
 
   /**
-   * Returns the 'day in year' value for the given date. The date is expressed in days
-   * since 1.1.1970.
+   * Returns the 'day in year' value for the given number of days since 1970-01-01.
    */
-  def getDayInYear(date: SQLDate): Int = {
-    LocalDate.ofEpochDay(date).getDayOfYear
-  }
+  def getDayInYear(days: Int): Int = daysToLocalDate(days).getDayOfYear
 
   /**
-   * Returns the year value for the given date. The date is expressed in days
-   * since 1.1.1970.
+   * Returns the year value for the given number of days since 1970-01-01.
    */
-  def getYear(date: SQLDate): Int = {
-    LocalDate.ofEpochDay(date).getYear
-  }
+  def getYear(days: Int): Int = daysToLocalDate(days).getYear
 
   /**
    * Returns the year which conforms to ISO 8601. Each ISO 8601 week-numbering
    * year begins with the Monday of the week containing the 4th of January.
    */
-  def getWeekBasedYear(date: SQLDate): Int = {
-    daysToLocalDate(date).get(IsoFields.WEEK_BASED_YEAR)
-  }
+  def getWeekBasedYear(days: Int): Int = daysToLocalDate(days).get(IsoFields.WEEK_BASED_YEAR)
+
+  /** Returns the quarter for the given number of days since 1970-01-01. */
+  def getQuarter(days: Int): Int = daysToLocalDate(days).get(IsoFields.QUARTER_OF_YEAR)
 
   /**
-   * Returns the quarter for the given date. The date is expressed in days
-   * since 1.1.1970.
+   * Returns the month value for the given number of days since 1970-01-01.
+   * January is month 1.
    */
-  def getQuarter(date: SQLDate): Int = {
-    LocalDate.ofEpochDay(date).get(IsoFields.QUARTER_OF_YEAR)
-  }
+  def getMonth(days: Int): Int = daysToLocalDate(days).getMonthValue
 
   /**
-   * Split date (expressed in days since 1.1.1970) into four fields:
-   * year, month (Jan is Month 1), dayInMonth, daysToMonthEnd (0 if it's last day of month).
+   * Returns the 'day of month' value for the given number of days since 1970-01-01.
    */
-  def splitDate(date: SQLDate): (Int, Int, Int, Int) = {
-    val ld = LocalDate.ofEpochDay(date)
-    (ld.getYear, ld.getMonthValue, ld.getDayOfMonth, ld.lengthOfMonth() - ld.getDayOfMonth)
-  }
+  def getDayOfMonth(days: Int): Int = daysToLocalDate(days).getDayOfMonth
+
+  /**
+   * Returns the day of the week for the given number of days since 1970-01-01
+   * (1 = Sunday, 2 = Monday, ..., 7 = Saturday).
+   */
+  def getDayOfWeek(days: Int): Int = LocalDate.ofEpochDay(days).getDayOfWeek.plus(1).getValue
+
+  /**
+   * Returns the day of the week for the given number of days since 1970-01-01
+   * (0 = Monday, 1 = Tuesday, ..., 6 = Sunday).
+   */
+  def getWeekDay(days: Int): Int = LocalDate.ofEpochDay(days).getDayOfWeek.ordinal()
 
   /**
-   * Returns the month value for the given date. The date is expressed in days
-   * since 1.1.1970. January is month 1.
+   * Returns the week of the year of the given date expressed as the number of days from 1970-01-01.
+   * A week is considered to start on a Monday and week 1 is the first week with > 3 days.
    */
-  def getMonth(date: SQLDate): Int = {
-    LocalDate.ofEpochDay(date).getMonthValue
+  def getWeekOfYear(days: Int): Int = {
+    LocalDate.ofEpochDay(days).get(IsoFields.WEEK_OF_WEEK_BASED_YEAR)
   }
 
   /**
-   * Returns the 'day of month' value for the given date. The date is expressed in days
-   * since 1.1.1970.
+   * Adds an year-month interval to a date represented as days since 1970-01-01.
+   * @return a date value, expressed in days since 1970-01-01.
    */
-  def getDayOfMonth(date: SQLDate): Int = {
-    LocalDate.ofEpochDay(date).getDayOfMonth
+  def dateAddMonths(days: Int, months: Int): Int = {
+    localDateToDays(daysToLocalDate(days).plusMonths(months))
   }
 
   /**
-   * Add date and year-month interval.
-   * Returns a date value, expressed in days since 1.1.1970.
+   * Adds months to a timestamp at the given time zone. It converts the input timestamp to a local
+   * timestamp at the given time zone, adds months, and converts the resulted local timestamp
+   * back to a timestamp, expressed in microseconds since 1970-01-01 00:00:00Z.
+   *
+   * @param micros The input timestamp value, expressed in microseconds since 1970-01-01 00:00:00Z
+   * @param months The amount of months to add. It can be positive or negative.
+   * @param zoneId The time zone ID at which the operation is performed.
+   * @return A timestamp value, expressed in microseconds since 1970-01-01 00:00:00Z.
    */
-  def dateAddMonths(days: SQLDate, months: Int): SQLDate = {
-    LocalDate.ofEpochDay(days).plusMonths(months).toEpochDay.toInt
+  def timestampAddMonths(micros: Long, months: Int, zoneId: ZoneId): Long = {
+    instantToMicros(microsToInstant(micros).atZone(zoneId).plusMonths(months).toInstant)
   }
 
   /**
-   * Add timestamp and full interval.
-   * Returns a timestamp value, expressed in microseconds since 1.1.1970 00:00:00.
+   * Adds a day-time interval expressed in microseconds to a timestamp at the given time zone.
+   * It converts the input timestamp to a local timestamp, and adds the interval by:
+   *   - Splitting the interval to days and microsecond adjustment in a day, and
+   *   - First of all, it adds days and then the time part.
+   * The resulted local timestamp is converted back to an instant at the given time zone.
+   *
+   * @param micros The input timestamp value, expressed in microseconds since 1970-01-01 00:00:00Z.
+   * @param dayTime The amount of microseconds to add. It can be positive or negative.
+   * @param zoneId The time zone ID at which the operation is performed.
+   * @return A timestamp value, expressed in microseconds since 1970-01-01 00:00:00Z.
+   */
+  def timestampAddDayTime(micros: Long, dayTime: Long, zoneId: ZoneId): Long = {
+    val days = dayTime / MICROS_PER_DAY
+    val microseconds = dayTime - days * MICROS_PER_DAY
+    val resultTimestamp = microsToInstant(micros)
+      .atZone(zoneId)
+      .plusDays(days)
+      .plus(microseconds, ChronoUnit.MICROS)
+    instantToMicros(resultTimestamp.toInstant)
+  }
+
+  /**
+   * Adds a full interval (months, days, microseconds) a timestamp represented as the number of
+   * microseconds since 1970-01-01 00:00:00Z.
+   * @return A timestamp value, expressed in microseconds since 1970-01-01 00:00:00Z.
    */
   def timestampAddInterval(
-      start: SQLTimestamp,
+      start: Long,
       months: Int,
       days: Int,
       microseconds: Long,
-      zoneId: ZoneId): SQLTimestamp = {
+      zoneId: ZoneId): Long = {
     val resultTimestamp = microsToInstant(start)
       .atZone(zoneId)
       .plusMonths(months)
@@ -619,38 +634,47 @@ object DateTimeUtils {
   }
 
   /**
-   * Add the date and the interval's months and days.
-   * Returns a date value, expressed in days since 1.1.1970.
+   * Adds the interval's months and days to a date expressed as days since the epoch.
+   * @return A date value, expressed in days since 1970-01-01.
    *
    * @throws DateTimeException if the result exceeds the supported date range
    * @throws IllegalArgumentException if the interval has `microseconds` part
    */
   def dateAddInterval(
-     start: SQLDate,
-     interval: CalendarInterval): SQLDate = {
+     start: Int,
+     interval: CalendarInterval): Int = {
     require(interval.microseconds == 0,
       "Cannot add hours, minutes or seconds, milliseconds, microseconds to a date")
-    val ld = LocalDate.ofEpochDay(start).plusMonths(interval.months).plusDays(interval.days)
+    val ld = daysToLocalDate(start).plusMonths(interval.months).plusDays(interval.days)
     localDateToDays(ld)
   }
 
   /**
-   * Returns number of months between time1 and time2. time1 and time2 are expressed in
-   * microseconds since 1.1.1970. If time1 is later than time2, the result is positive.
+   * Splits date (expressed in days since 1970-01-01) into four fields:
+   * year, month (Jan is Month 1), dayInMonth, daysToMonthEnd (0 if it's last day of month).
+   */
+  private def splitDate(days: Int): (Int, Int, Int, Int) = {
+    val ld = daysToLocalDate(days)
+    (ld.getYear, ld.getMonthValue, ld.getDayOfMonth, ld.lengthOfMonth() - ld.getDayOfMonth)
+  }
+
+  /**
+   * Returns number of months between micros1 and micros2. micros1 and micros2 are expressed in
+   * microseconds since 1970-01-01. If micros1 is later than micros2, the result is positive.
    *
-   * If time1 and time2 are on the same day of month, or both are the last day of month,
+   * If micros1 and micros2 are on the same day of month, or both are the last day of month,
    * returns, time of day will be ignored.
    *
    * Otherwise, the difference is calculated based on 31 days per month.
    * The result is rounded to 8 decimal places if `roundOff` is set to true.
    */
   def monthsBetween(
-      time1: SQLTimestamp,
-      time2: SQLTimestamp,
+      micros1: Long,
+      micros2: Long,
       roundOff: Boolean,
       zoneId: ZoneId): Double = {
-    val date1 = microsToDays(time1, zoneId)
-    val date2 = microsToDays(time2, zoneId)
+    val date1 = microsToDays(micros1, zoneId)
+    val date2 = microsToDays(micros2, zoneId)
     val (year1, monthInYear1, dayInMonth1, daysToMonthEnd1) = splitDate(date1)
     val (year2, monthInYear2, dayInMonth2, daysToMonthEnd2) = splitDate(date2)
 
@@ -664,8 +688,8 @@ object DateTimeUtils {
     }
     // using milliseconds can cause precision loss with more than 8 digits
     // we follow Hive's implementation which uses seconds
-    val secondsInDay1 = MICROSECONDS.toSeconds(time1 - daysToMicros(date1, zoneId))
-    val secondsInDay2 = MICROSECONDS.toSeconds(time2 - daysToMicros(date2, zoneId))
+    val secondsInDay1 = MICROSECONDS.toSeconds(micros1 - daysToMicros(date1, zoneId))
+    val secondsInDay2 = MICROSECONDS.toSeconds(micros2 - daysToMicros(date2, zoneId))
     val secondsDiff = (dayInMonth1 - dayInMonth2) * SECONDS_PER_DAY + secondsInDay1 - secondsInDay2
     val secondsInMonth = DAYS.toSeconds(31)
     val diff = monthDiff + secondsDiff / secondsInMonth.toDouble
@@ -686,9 +710,10 @@ object DateTimeUtils {
   private val FRIDAY = 1
   private val SATURDAY = 2
 
-  /*
+  /**
    * Returns day of week from String. Starting from Thursday, marked as 0.
    * (Because 1970-01-01 is Thursday).
+   * @throws IllegalArgumentException if the input is not a valid day of week.
    */
   def getDayOfWeekFromString(string: UTF8String): Int = {
     val dowString = string.toString.toUpperCase(Locale.ROOT)
@@ -700,7 +725,8 @@ object DateTimeUtils {
       case "TH" | "THU" | "THURSDAY" => THURSDAY
       case "FR" | "FRI" | "FRIDAY" => FRIDAY
       case "SA" | "SAT" | "SATURDAY" => SATURDAY
-      case _ => -1
+      case _ =>
+        throw new IllegalArgumentException(s"""Illegal input for day of week: $string""")
     }
   }
 
@@ -708,17 +734,14 @@ object DateTimeUtils {
    * Returns the first date which is later than startDate and is of the given dayOfWeek.
    * dayOfWeek is an integer ranges in [0, 6], and 0 is Thu, 1 is Fri, etc,.
    */
-  def getNextDateForDayOfWeek(startDate: SQLDate, dayOfWeek: Int): SQLDate = {
-    startDate + 1 + ((dayOfWeek - 1 - startDate) % 7 + 7) % 7
+  def getNextDateForDayOfWeek(startDay: Int, dayOfWeek: Int): Int = {
+    startDay + 1 + ((dayOfWeek - 1 - startDay) % 7 + 7) % 7
   }
 
-  /**
-   * Returns last day of the month for the given date. The date is expressed in days
-   * since 1.1.1970.
-   */
-  def getLastDayOfMonth(date: SQLDate): SQLDate = {
-    val localDate = LocalDate.ofEpochDay(date)
-    (date - localDate.getDayOfMonth) + localDate.lengthOfMonth()
+  /** Returns last day of the month for the given number of days since 1970-01-01. */
+  def getLastDayOfMonth(days: Int): Int = {
+    val localDate = daysToLocalDate(days)
+    (days - localDate.getDayOfMonth) + localDate.lengthOfMonth()
   }
 
   // The constants are visible for testing purpose only.
@@ -744,21 +767,21 @@ object DateTimeUtils {
    * Returns the trunc date from original date and trunc level.
    * Trunc level should be generated using `parseTruncLevel()`, should be between 6 and 9.
    */
-  def truncDate(d: SQLDate, level: Int): SQLDate = {
+  def truncDate(days: Int, level: Int): Int = {
     level match {
-      case TRUNC_TO_WEEK => getNextDateForDayOfWeek(d - 7, MONDAY)
-      case TRUNC_TO_MONTH => d - DateTimeUtils.getDayOfMonth(d) + 1
+      case TRUNC_TO_WEEK => getNextDateForDayOfWeek(days - 7, MONDAY)
+      case TRUNC_TO_MONTH => days - getDayOfMonth(days) + 1
       case TRUNC_TO_QUARTER =>
-        localDateToDays(daysToLocalDate(d).`with`(IsoFields.DAY_OF_QUARTER, 1L))
-      case TRUNC_TO_YEAR => d - DateTimeUtils.getDayInYear(d) + 1
+        localDateToDays(daysToLocalDate(days).`with`(IsoFields.DAY_OF_QUARTER, 1L))
+      case TRUNC_TO_YEAR => days - getDayInYear(days) + 1
       case _ =>
         // caller make sure that this should never be reached
-        sys.error(s"Invalid trunc level: $level")
+        throw QueryExecutionErrors.unreachableError(s": Invalid trunc level: $level")
     }
   }
 
-  private def truncToUnit(t: SQLTimestamp, zoneId: ZoneId, unit: ChronoUnit): SQLTimestamp = {
-    val truncated = microsToInstant(t).atZone(zoneId).truncatedTo(unit)
+  private def truncToUnit(micros: Long, zoneId: ZoneId, unit: ChronoUnit): Long = {
+    val truncated = microsToInstant(micros).atZone(zoneId).truncatedTo(unit)
     instantToMicros(truncated.toInstant)
   }
 
@@ -766,19 +789,21 @@ object DateTimeUtils {
    * Returns the trunc date time from original date time and trunc level.
    * Trunc level should be generated using `parseTruncLevel()`, should be between 0 and 9.
    */
-  def truncTimestamp(t: SQLTimestamp, level: Int, zoneId: ZoneId): SQLTimestamp = {
+  def truncTimestamp(micros: Long, level: Int, zoneId: ZoneId): Long = {
+    // Time zone offsets have a maximum precision of seconds (see `java.time.ZoneOffset`). Hence
+    // truncation to microsecond, millisecond, and second can be done
+    // without using time zone information. This results in a performance improvement.
     level match {
-      case TRUNC_TO_MICROSECOND => t
+      case TRUNC_TO_MICROSECOND => micros
       case TRUNC_TO_MILLISECOND =>
-        t - Math.floorMod(t, MICROS_PER_MILLIS)
+        micros - Math.floorMod(micros, MICROS_PER_MILLIS)
       case TRUNC_TO_SECOND =>
-        t - Math.floorMod(t, MICROS_PER_SECOND)
-      case TRUNC_TO_MINUTE =>
-        t - Math.floorMod(t, MICROS_PER_MINUTE)
-      case TRUNC_TO_HOUR => truncToUnit(t, zoneId, ChronoUnit.HOURS)
-      case TRUNC_TO_DAY => truncToUnit(t, zoneId, ChronoUnit.DAYS)
+        micros - Math.floorMod(micros, MICROS_PER_SECOND)
+      case TRUNC_TO_MINUTE => truncToUnit(micros, zoneId, ChronoUnit.MINUTES)
+      case TRUNC_TO_HOUR => truncToUnit(micros, zoneId, ChronoUnit.HOURS)
+      case TRUNC_TO_DAY => truncToUnit(micros, zoneId, ChronoUnit.DAYS)
       case _ => // Try to truncate date levels
-        val dDays = microsToDays(t, zoneId)
+        val dDays = microsToDays(micros, zoneId)
         daysToMicros(truncDate(dDays, level), zoneId)
     }
   }
@@ -808,36 +833,41 @@ object DateTimeUtils {
   }
 
   /**
-   * Convert the timestamp `ts` from one timezone to another.
+   * Converts the timestamp `micros` from one timezone to another.
    *
-   * TODO: Because of DST, the conversion between UTC and human time is not exactly one-to-one
-   * mapping, the conversion here may return wrong result, we should make the timestamp
-   * timezone-aware.
+   * Time-zone rules, such as daylight savings, mean that not every local date-time
+   * is valid for the `toZone` time zone, thus the local date-time may be adjusted.
    */
-  def convertTz(ts: SQLTimestamp, fromZone: ZoneId, toZone: ZoneId): SQLTimestamp = {
-    val rebasedDateTime = microsToInstant(ts).atZone(toZone).toLocalDateTime.atZone(fromZone)
+  def convertTz(micros: Long, fromZone: ZoneId, toZone: ZoneId): Long = {
+    val rebasedDateTime = getLocalDateTime(micros, toZone).atZone(fromZone)
     instantToMicros(rebasedDateTime.toInstant)
   }
 
   /**
-   * Returns a timestamp of given timezone from utc timestamp, with the same string
+   * Returns a timestamp of given timezone from UTC timestamp, with the same string
    * representation in their timezone.
    */
-  def fromUTCTime(time: SQLTimestamp, timeZone: String): SQLTimestamp = {
-    convertTz(time, ZoneOffset.UTC, getZoneId(timeZone))
+  def fromUTCTime(micros: Long, timeZone: String): Long = {
+    convertTz(micros, ZoneOffset.UTC, getZoneId(timeZone))
   }
 
   /**
    * Returns a utc timestamp from a given timestamp from a given timezone, with the same
    * string representation in their timezone.
    */
-  def toUTCTime(time: SQLTimestamp, timeZone: String): SQLTimestamp = {
-    convertTz(time, getZoneId(timeZone), ZoneOffset.UTC)
+  def toUTCTime(micros: Long, timeZone: String): Long = {
+    convertTz(micros, getZoneId(timeZone), ZoneOffset.UTC)
   }
 
-  def currentTimestamp(): SQLTimestamp = instantToMicros(Instant.now())
+  /**
+   * Obtains the current instant as microseconds since the epoch at the UTC time zone.
+   */
+  def currentTimestamp(): Long = instantToMicros(Instant.now())
 
-  def currentDate(zoneId: ZoneId): SQLDate = localDateToDays(LocalDate.now(zoneId))
+  /**
+   * Obtains the current date as days since the epoch in the specified time-zone.
+   */
+  def currentDate(zoneId: ZoneId): Int = localDateToDays(LocalDate.now(zoneId))
 
   private def today(zoneId: ZoneId): ZonedDateTime = {
     Instant.now().atZone(zoneId).`with`(LocalTime.MIDNIGHT)
@@ -847,6 +877,7 @@ object DateTimeUtils {
 
   /**
    * Extracts special values from an input string ignoring case.
+   *
    * @param input A trimmed string
    * @param zoneId Zone identifier used to get the current date.
    * @return Some special value in lower case or None.
@@ -876,12 +907,13 @@ object DateTimeUtils {
 
   /**
    * Converts notational shorthands that are converted to ordinary timestamps.
+   *
    * @param input A trimmed string
    * @param zoneId Zone identifier used to get the current date.
    * @return Some of microseconds since the epoch if the conversion completed
    *         successfully otherwise None.
    */
-  def convertSpecialTimestamp(input: String, zoneId: ZoneId): Option[SQLTimestamp] = {
+  def convertSpecialTimestamp(input: String, zoneId: ZoneId): Option[Long] = {
     extractSpecialValue(input, zoneId).flatMap {
       case "epoch" => Some(0)
       case "now" => Some(currentTimestamp())
@@ -892,7 +924,7 @@ object DateTimeUtils {
     }
   }
 
-  private def convertSpecialTimestamp(bytes: Array[Byte], zoneId: ZoneId): Option[SQLTimestamp] = {
+  private def convertSpecialTimestamp(bytes: Array[Byte], zoneId: ZoneId): Option[Long] = {
     if (bytes.length > 0 && Character.isAlphabetic(bytes(0))) {
       convertSpecialTimestamp(new String(bytes, StandardCharsets.UTF_8), zoneId)
     } else {
@@ -902,11 +934,12 @@ object DateTimeUtils {
 
   /**
    * Converts notational shorthands that are converted to ordinary dates.
+   *
    * @param input A trimmed string
    * @param zoneId Zone identifier used to get the current date.
    * @return Some of days since the epoch if the conversion completed successfully otherwise None.
    */
-  def convertSpecialDate(input: String, zoneId: ZoneId): Option[SQLDate] = {
+  def convertSpecialDate(input: String, zoneId: ZoneId): Option[Int] = {
     extractSpecialValue(input, zoneId).flatMap {
       case "epoch" => Some(0)
       case "now" | "today" => Some(currentDate(zoneId))
@@ -916,7 +949,7 @@ object DateTimeUtils {
     }
   }
 
-  private def convertSpecialDate(bytes: Array[Byte], zoneId: ZoneId): Option[SQLDate] = {
+  private def convertSpecialDate(bytes: Array[Byte], zoneId: ZoneId): Option[Int] = {
     if (bytes.length > 0 && Character.isAlphabetic(bytes(0))) {
       convertSpecialDate(new String(bytes, StandardCharsets.UTF_8), zoneId)
     } else {
@@ -925,18 +958,33 @@ object DateTimeUtils {
   }
 
   /**
-   * Subtracts two dates.
-   * @param endDate The end date, exclusive
-   * @param startDate The start date, inclusive
+   * Subtracts two dates expressed as days since 1970-01-01.
+   *
+   * @param endDay The end date, exclusive
+   * @param startDay The start date, inclusive
    * @return An interval between two dates. The interval can be negative
    *         if the end date is before the start date.
    */
-  def subtractDates(endDate: SQLDate, startDate: SQLDate): CalendarInterval = {
-    val period = Period.between(
-      LocalDate.ofEpochDay(startDate),
-      LocalDate.ofEpochDay(endDate))
-    val months = period.getMonths + 12 * period.getYears
+  def subtractDates(endDay: Int, startDay: Int): CalendarInterval = {
+    val period = Period.between(daysToLocalDate(startDay), daysToLocalDate(endDay))
+    val months = Math.toIntExact(period.toTotalMonths)
     val days = period.getDays
     new CalendarInterval(months, days, 0)
   }
+
+  /**
+   * Subtracts two timestamps expressed as microseconds since 1970-01-01 00:00:00Z, and returns
+   * the difference in microseconds between local timestamps at the given time zone.
+   *
+   * @param endMicros The end timestamp as microseconds since the epoch, exclusive
+   * @param startMicros The end timestamp as microseconds since the epoch, inclusive
+   * @param zoneId The time zone ID in which the subtraction is performed
+   * @return The difference in microseconds between local timestamps corresponded to the input
+   *         instants `end` and `start`.
+   */
+  def subtractTimestamps(endMicros: Long, startMicros: Long, zoneId: ZoneId): Long = {
+    val localEndTs = getLocalDateTime(endMicros, zoneId)
+    val localStartTs = getLocalDateTime(startMicros, zoneId)
+    ChronoUnit.MICROS.between(localStartTs, localEndTs)
+  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/GenericArrayData.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/GenericArrayData.scala
index 1f88a700847de..e46d730afb4a3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/GenericArrayData.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/GenericArrayData.scala
@@ -25,8 +25,10 @@ import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 
 class GenericArrayData(val array: Array[Any]) extends ArrayData {
 
-  def this(seq: Seq[Any]) = this(seq.toArray)
-  def this(list: java.util.List[Any]) = this(list.asScala)
+  // Specified this as`scala.collection.Seq` because seqOrArray can be
+  // `mutable.ArraySeq` in Scala 2.13
+  def this(seq: scala.collection.Seq[Any]) = this(seq.toArray)
+  def this(list: java.util.List[Any]) = this(list.asScala.toSeq)
 
   // TODO: This is boxing.  We should specialize.
   def this(primitiveArray: Array[Int]) = this(primitiveArray.toSeq)
@@ -38,7 +40,9 @@ class GenericArrayData(val array: Array[Any]) extends ArrayData {
   def this(primitiveArray: Array[Boolean]) = this(primitiveArray.toSeq)
 
   def this(seqOrArray: Any) = this(seqOrArray match {
-    case seq: Seq[Any] => seq.toArray
+    // Specified this as`scala.collection.Seq` because seqOrArray can be
+    // `mutable.ArraySeq` in Scala 2.13
+    case seq: scala.collection.Seq[Any] => seq.toArray
     case array: Array[Any] => array  // array of objects, so no need to convert
     case array: Array[_] => array.toSeq.toArray[Any] // array of primitives, so box them
   })
@@ -116,7 +120,7 @@ class GenericArrayData(val array: Array[Any]) extends ArrayData {
             if (!o2.isInstanceOf[Double] || ! java.lang.Double.isNaN(o2.asInstanceOf[Double])) {
               return false
             }
-          case _ => if (!o1.equals(o2)) {
+          case _ => if (o1.getClass != o2.getClass || o1 != o2) {
             return false
           }
         }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/HyperLogLogPlusPlusHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/HyperLogLogPlusPlusHelper.scala
index ea619c6a7666c..6471a746f2edf 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/HyperLogLogPlusPlusHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/HyperLogLogPlusPlusHelper.scala
@@ -22,6 +22,7 @@ import java.util
 
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.XxHash64Function
+import org.apache.spark.sql.catalyst.optimizer.NormalizeFloatingNumbers.{DOUBLE_NORMALIZER, FLOAT_NORMALIZER}
 import org.apache.spark.sql.types._
 
 // A helper class for HyperLogLogPlusPlus.
@@ -88,7 +89,12 @@ class HyperLogLogPlusPlusHelper(relativeSD: Double) extends Serializable {
    *
    * Variable names in the HLL++ paper match variable names in the code.
    */
-  def update(buffer: InternalRow, bufferOffset: Int, value: Any, dataType: DataType): Unit = {
+  def update(buffer: InternalRow, bufferOffset: Int, _value: Any, dataType: DataType): Unit = {
+    val value = dataType match {
+      case FloatType => FLOAT_NORMALIZER.apply(_value)
+      case DoubleType => DOUBLE_NORMALIZER.apply(_value)
+      case _ => _value
+    }
     // Create the hashed value 'x'.
     val x = XxHash64Function.hash(value, dataType, 42L)
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
index 7e5a71e0cb3f7..2a11d5fe2f237 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
@@ -17,16 +17,26 @@
 
 package org.apache.spark.sql.catalyst.util
 
+import java.time.{Duration, Period}
+import java.time.temporal.ChronoUnit
 import java.util.concurrent.TimeUnit
 
 import scala.util.control.NonFatal
 
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToMicros
+import org.apache.spark.sql.catalyst.util.IntervalStringStyles.{ANSI_STYLE, HIVE_STYLE, IntervalStyle}
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.Decimal
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 
+// The style of textual representation of intervals
+object IntervalStringStyles extends Enumeration {
+  type IntervalStyle = Value
+  val ANSI_STYLE, HIVE_STYLE = Value
+}
+
 object IntervalUtils {
 
   object IntervalUnit extends Enumeration {
@@ -45,31 +55,39 @@ object IntervalUtils {
   }
   import IntervalUnit._
 
-  def getYears(interval: CalendarInterval): Int = {
-    interval.months / MONTHS_PER_YEAR
-  }
+  def getYears(months: Int): Int = months / MONTHS_PER_YEAR
 
-  def getMonths(interval: CalendarInterval): Byte = {
-    (interval.months % MONTHS_PER_YEAR).toByte
-  }
+  def getYears(interval: CalendarInterval): Int = getYears(interval.months)
+
+  def getMonths(months: Int): Byte = (months % MONTHS_PER_YEAR).toByte
+
+  def getMonths(interval: CalendarInterval): Byte = getMonths(interval.months)
+
+  def getDays(microseconds: Long): Int = (microseconds / MICROS_PER_DAY).toInt
 
   def getDays(interval: CalendarInterval): Int = {
-    val daysInMicroseconds = (interval.microseconds / MICROS_PER_DAY).toInt
+    val daysInMicroseconds = getDays(interval.microseconds)
     Math.addExact(interval.days, daysInMicroseconds)
   }
 
-  def getHours(interval: CalendarInterval): Long = {
-    (interval.microseconds % MICROS_PER_DAY) / MICROS_PER_HOUR
+  def getHours(microseconds: Long): Byte = {
+    ((microseconds % MICROS_PER_DAY) / MICROS_PER_HOUR).toByte
   }
 
-  def getMinutes(interval: CalendarInterval): Byte = {
-    ((interval.microseconds % MICROS_PER_HOUR) / MICROS_PER_MINUTE).toByte
+  def getHours(interval: CalendarInterval): Byte = getHours(interval.microseconds)
+
+  def getMinutes(microseconds: Long): Byte = {
+    ((microseconds % MICROS_PER_HOUR) / MICROS_PER_MINUTE).toByte
   }
 
-  def getSeconds(interval: CalendarInterval): Decimal = {
-    Decimal(interval.microseconds % MICROS_PER_MINUTE, 8, 6)
+  def getMinutes(interval: CalendarInterval): Byte = getMinutes(interval.microseconds)
+
+  def getSeconds(microseconds: Long): Decimal = {
+    Decimal(microseconds % MICROS_PER_MINUTE, 8, 6)
   }
 
+  def getSeconds(interval: CalendarInterval): Decimal = getSeconds(interval.microseconds)
+
   private def toLongWithRange(
       fieldName: IntervalUnit,
       s: String,
@@ -82,7 +100,27 @@ object IntervalUtils {
     result
   }
 
-  private val yearMonthPattern = "^([+|-])?(\\d+)-(\\d+)$".r
+  private val yearMonthPatternString = "([+|-])?(\\d+)-(\\d+)"
+  private val yearMonthRegex = (s"^$yearMonthPatternString$$").r
+  private val yearMonthLiteralRegex =
+    (s"(?i)^INTERVAL\\s+([+|-])?'$yearMonthPatternString'\\s+YEAR\\s+TO\\s+MONTH$$").r
+
+  def castStringToYMInterval(input: UTF8String): Int = {
+    input.trimAll().toString match {
+      case yearMonthRegex("-", year, month) => toYMInterval(year, month, -1)
+      case yearMonthRegex(_, year, month) => toYMInterval(year, month, 1)
+      case yearMonthLiteralRegex(firstSign, secondSign, year, month) =>
+        (firstSign, secondSign) match {
+          case ("-", "-") => toYMInterval(year, month, 1)
+          case ("-", _) => toYMInterval(year, month, -1)
+          case (_, "-") => toYMInterval(year, month, -1)
+          case (_, _) => toYMInterval(year, month, 1)
+        }
+      case _ => throw new IllegalArgumentException(
+        s"Interval string does not match year-month format of `[+|-]y-m` " +
+          s"or `INTERVAL [+|-]'[+|-]y-m' YEAR TO MONTH`: ${input.toString}")
+    }
+  }
 
   /**
    * Parse YearMonth string in form: [+|-]YYYY-MM
@@ -91,29 +129,81 @@ object IntervalUtils {
    */
   def fromYearMonthString(input: String): CalendarInterval = {
     require(input != null, "Interval year-month string must be not null")
-    def toInterval(yearStr: String, monthStr: String): CalendarInterval = {
-      try {
-        val years = toLongWithRange(YEAR, yearStr, 0, Integer.MAX_VALUE).toInt
-        val months = toLongWithRange(MONTH, monthStr, 0, 11).toInt
-        val totalMonths = Math.addExact(Math.multiplyExact(years, 12), months)
-        new CalendarInterval(totalMonths, 0, 0)
-      } catch {
-        case NonFatal(e) =>
-          throw new IllegalArgumentException(
-            s"Error parsing interval year-month string: ${e.getMessage}", e)
-      }
-    }
     input.trim match {
-      case yearMonthPattern("-", yearStr, monthStr) =>
-        negateExact(toInterval(yearStr, monthStr))
-      case yearMonthPattern(_, yearStr, monthStr) =>
-        toInterval(yearStr, monthStr)
+      case yearMonthRegex("-", yearStr, monthStr) =>
+        new CalendarInterval(toYMInterval(yearStr, monthStr, -1), 0, 0)
+      case yearMonthRegex(_, yearStr, monthStr) =>
+        new CalendarInterval(toYMInterval(yearStr, monthStr, 1), 0, 0)
       case _ =>
         throw new IllegalArgumentException(
           s"Interval string does not match year-month format of 'y-m': $input")
     }
   }
 
+  def toYMInterval(yearStr: String, monthStr: String, sign: Int): Int = {
+    try {
+      val years = toLongWithRange(YEAR, yearStr, 0, Integer.MAX_VALUE / MONTHS_PER_YEAR)
+      val totalMonths = sign * (years * MONTHS_PER_YEAR + toLongWithRange(MONTH, monthStr, 0, 11))
+      Math.toIntExact(totalMonths)
+    } catch {
+      case NonFatal(e) =>
+        throw new IllegalArgumentException(
+          s"Error parsing interval year-month string: ${e.getMessage}", e)
+    }
+  }
+
+  private val daySecondPatternString =
+    "([+|-])?(\\d+) (\\d{1,2}):(\\d{1,2}):(\\d{1,2})(\\.\\d{1,9})?"
+  private val daySecondRegex = (s"^$daySecondPatternString$$").r
+  private val daySecondLiteralRegex =
+    (s"(?i)^INTERVAL\\s+([+|-])?\\'$daySecondPatternString\\'\\s+DAY\\s+TO\\s+SECOND$$").r
+
+  def castStringToDTInterval(input: UTF8String): Long = {
+    def secondAndMicro(second: String, micro: String): String = {
+      if (micro != null) {
+        s"$second$micro"
+      } else {
+        second
+      }
+    }
+
+    input.trimAll().toString match {
+      case daySecondRegex("-", day, hour, minute, second, micro) =>
+        toDTInterval(day, hour, minute, secondAndMicro(second, micro), -1)
+      case daySecondRegex(_, day, hour, minute, second, micro) =>
+        toDTInterval(day, hour, minute, secondAndMicro(second, micro), 1)
+      case daySecondLiteralRegex(firstSign, secondSign, day, hour, minute, second, micro) =>
+        (firstSign, secondSign) match {
+          case ("-", "-") => toDTInterval(day, hour, minute, secondAndMicro(second, micro), 1)
+          case ("-", _) => toDTInterval(day, hour, minute, secondAndMicro(second, micro), -1)
+          case (_, "-") => toDTInterval(day, hour, minute, secondAndMicro(second, micro), -1)
+          case (_, _) => toDTInterval(day, hour, minute, secondAndMicro(second, micro), 1)
+        }
+      case _ =>
+        throw new IllegalArgumentException(
+          s"Interval string must match day-time format of `d h:m:s.n` " +
+            s"or `INTERVAL [+|-]'[+|-]d h:m:s.n' DAY TO SECOND`: ${input.toString}, " +
+            s"$fallbackNotice")
+    }
+  }
+
+  def toDTInterval(
+      dayStr: String,
+      hourStr: String,
+      minuteStr: String,
+      secondStr: String,
+      sign: Int): Long = {
+    var micros = 0L
+    val days = toLongWithRange(DAY, dayStr, 0, Int.MaxValue).toInt
+    micros = Math.addExact(micros, sign * days * MICROS_PER_DAY)
+    val hours = toLongWithRange(HOUR, hourStr, 0, 23)
+    micros = Math.addExact(micros, sign * hours * MICROS_PER_HOUR)
+    val minutes = toLongWithRange(MINUTE, minuteStr, 0, 59)
+    micros = Math.addExact(micros, sign * minutes * MICROS_PER_MINUTE)
+    micros = Math.addExact(micros, sign * parseSecondNano(secondStr))
+    micros
+  }
+
   /**
    * Parse dayTime string in form: [-]d HH:mm:ss.nnnnnnnnn and [-]HH:mm:ss.nnnnnnnnn
    *
@@ -496,7 +586,7 @@ object IntervalUtils {
    * @throws ArithmeticException if the result overflows any field value or divided by zero
    */
   def divideExact(interval: CalendarInterval, num: Double): CalendarInterval = {
-    if (num == 0) throw new ArithmeticException("divide by zero")
+    if (num == 0) throw QueryExecutionErrors.divideByZeroError()
     fromDoubles(interval.months / num, interval.days / num, interval.microseconds / num)
   }
 
@@ -573,7 +663,7 @@ object IntervalUtils {
     var pointPrefixed: Boolean = false
 
     def trimToNextState(b: Byte, next: ParseState): Unit = {
-      if (b <= ' ') {
+      if (Character.isWhitespace(b)) {
         i += 1
       } else {
         state = next
@@ -594,7 +684,7 @@ object IntervalUtils {
           if (s.startsWith(intervalStr)) {
             if (s.numBytes() == intervalStr.numBytes()) {
               throwIAE("interval string cannot be empty")
-            } else if (bytes(i + intervalStr.numBytes()) > ' ') {
+            } else if (!Character.isWhitespace(bytes(i + intervalStr.numBytes()))) {
               throwIAE(s"invalid interval prefix $currentWord")
             } else {
               i += intervalStr.numBytes() + 1
@@ -641,7 +731,7 @@ object IntervalUtils {
               } catch {
                 case e: ArithmeticException => throwIAE(e.getMessage, e)
               }
-            case _ if b <= ' ' => state = TRIM_BEFORE_UNIT
+            case _ if Character.isWhitespace(b) => state = TRIM_BEFORE_UNIT
             case '.' =>
               fractionScale = initialFractionScale
               state = VALUE_FRACTIONAL_PART
@@ -652,7 +742,8 @@ object IntervalUtils {
           if ('0' <= b && b <= '9' && fractionScale > 0) {
             fraction += (b - '0') * fractionScale
             fractionScale /= 10
-          } else if (b <= ' ' && (!pointPrefixed || fractionScale < initialFractionScale)) {
+          } else if (Character.isWhitespace(b) &&
+              (!pointPrefixed || fractionScale < initialFractionScale)) {
             fraction /= NANOS_PER_MICROS.toInt
             state = TRIM_BEFORE_UNIT
           } else if ('0' <= b && b <= '9') {
@@ -718,12 +809,12 @@ object IntervalUtils {
         case UNIT_SUFFIX =>
           b match {
             case 's' => state = UNIT_END
-            case _ if b <= ' ' => state = TRIM_BEFORE_SIGN
+            case _ if Character.isWhitespace(b) => state = TRIM_BEFORE_SIGN
             case _ => throwIAE(s"invalid unit '$currentWord'")
           }
           i += 1
         case UNIT_END =>
-          if (b <= ' ') {
+          if (Character.isWhitespace(b) ) {
             i += 1
             state = TRIM_BEFORE_SIGN
           } else {
@@ -735,7 +826,10 @@ object IntervalUtils {
     val result = state match {
       case UNIT_SUFFIX | UNIT_END | TRIM_BEFORE_SIGN =>
         new CalendarInterval(months, days, microseconds)
-      case _ => null
+      case TRIM_BEFORE_VALUE => throwIAE(s"expect a number after '$currentWord' but hit EOL")
+      case VALUE | VALUE_FRACTIONAL_PART =>
+        throwIAE(s"expect a unit name after '$currentWord' but hit EOL")
+      case _ => throwIAE(s"unknown error when parsing '$currentWord'")
     }
 
     result
@@ -751,10 +845,150 @@ object IntervalUtils {
       secs: Decimal): CalendarInterval = {
     val totalMonths = Math.addExact(months, Math.multiplyExact(years, MONTHS_PER_YEAR))
     val totalDays = Math.addExact(days, Math.multiplyExact(weeks, DAYS_PER_WEEK))
-    var micros = (secs * Decimal(MICROS_PER_SECOND)).toLong
+    assert(secs.scale == 6, "Seconds fractional must have 6 digits for microseconds")
+    var micros = secs.toUnscaledLong
     micros = Math.addExact(micros, Math.multiplyExact(hours, MICROS_PER_HOUR))
     micros = Math.addExact(micros, Math.multiplyExact(mins, MICROS_PER_MINUTE))
 
     new CalendarInterval(totalMonths, totalDays, micros)
   }
+
+  // The amount of seconds that can cause overflow in the conversion to microseconds
+  private final val minDurationSeconds = Math.floorDiv(Long.MinValue, MICROS_PER_SECOND)
+
+  /**
+   * Converts this duration to the total length in microseconds.
+   * <p>
+   * If this duration is too large to fit in a [[Long]] microseconds, then an
+   * exception is thrown.
+   * <p>
+   * If this duration has greater than microsecond precision, then the conversion
+   * will drop any excess precision information as though the amount in nanoseconds
+   * was subject to integer division by one thousand.
+   *
+   * @return The total length of the duration in microseconds
+   * @throws ArithmeticException If numeric overflow occurs
+   */
+  def durationToMicros(duration: Duration): Long = {
+    val seconds = duration.getSeconds
+    if (seconds == minDurationSeconds) {
+      val microsInSeconds = (minDurationSeconds + 1) * MICROS_PER_SECOND
+      val nanoAdjustment = duration.getNano
+      assert(0 <= nanoAdjustment && nanoAdjustment < NANOS_PER_SECOND,
+        "Duration.getNano() must return the adjustment to the seconds field " +
+        "in the range from 0 to 999999999 nanoseconds, inclusive.")
+      Math.addExact(microsInSeconds, (nanoAdjustment - NANOS_PER_SECOND) / NANOS_PER_MICROS)
+    } else {
+      val microsInSeconds = Math.multiplyExact(seconds, MICROS_PER_SECOND)
+      Math.addExact(microsInSeconds, duration.getNano / NANOS_PER_MICROS)
+    }
+  }
+
+  /**
+   * Obtains a [[Duration]] representing a number of microseconds.
+   *
+   * @param micros The number of microseconds, positive or negative
+   * @return A [[Duration]], not null
+   */
+  def microsToDuration(micros: Long): Duration = Duration.of(micros, ChronoUnit.MICROS)
+
+  /**
+   * Gets the total number of months in this period.
+   * <p>
+   * This returns the total number of months in the period by multiplying the
+   * number of years by 12 and adding the number of months.
+   * <p>
+   *
+   * @return The total number of months in the period, may be negative
+   * @throws ArithmeticException If numeric overflow occurs
+   */
+  def periodToMonths(period: Period): Int = {
+    val monthsInYears = Math.multiplyExact(period.getYears, MONTHS_PER_YEAR)
+    Math.addExact(monthsInYears, period.getMonths)
+  }
+
+  /**
+   * Obtains a [[Period]] representing a number of months. The days unit will be zero, and the years
+   * and months units will be normalized.
+   *
+   * <p>
+   * The months unit is adjusted to have an absolute value < 12, with the years unit being adjusted
+   * to compensate. For example, the method returns "2 years and 3 months" for the 27 input months.
+   * <p>
+   * The sign of the years and months units will be the same after normalization.
+   * For example, -13 months will be converted to "-1 year and -1 month".
+   *
+   * @param months The number of months, positive or negative
+   * @return The period of months, not null
+   */
+  def monthsToPeriod(months: Int): Period = Period.ofMonths(months).normalized()
+
+  /**
+   * Converts an year-month interval as a number of months to its textual representation
+   * which conforms to the ANSI SQL standard.
+   *
+   * @param months The number of months, positive or negative
+   * @param style The style of textual representation of the interval
+   * @return Year-month interval string
+   */
+  def toYearMonthIntervalString(months: Int, style: IntervalStyle): String = {
+    var sign = ""
+    var absMonths: Long = months
+    if (months < 0) {
+      sign = "-"
+      absMonths = -absMonths
+    }
+    val payload = s"$sign${absMonths / MONTHS_PER_YEAR}-${absMonths % MONTHS_PER_YEAR}"
+    style match {
+      case ANSI_STYLE => s"INTERVAL '$payload' YEAR TO MONTH"
+      case HIVE_STYLE => payload
+    }
+  }
+
+  /**
+   * Converts a day-time interval as a number of microseconds to its textual representation
+   * which conforms to the ANSI SQL standard.
+   *
+   * @param micros The number of microseconds, positive or negative
+   * @param style The style of textual representation of the interval
+   * @return Day-time interval string
+   */
+  def toDayTimeIntervalString(micros: Long, style: IntervalStyle): String = {
+    var sign = ""
+    var rest = micros
+    if (micros < 0) {
+      if (micros == Long.MinValue) {
+        // Especial handling of minimum `Long` value because negate op overflows `Long`.
+        // seconds = 106751991 * (24 * 60 * 60) + 4 * 60 * 60 + 54 = 9223372036854
+        // microseconds = -9223372036854000000L-775808 == Long.MinValue
+        val minIntervalString = style match {
+          case ANSI_STYLE => "INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND"
+          case HIVE_STYLE => "-106751991 04:00:54.775808000"
+        }
+        return minIntervalString
+      } else {
+        sign = "-"
+        rest = -rest
+      }
+    }
+    val secondsWithFraction = rest % MICROS_PER_MINUTE
+    rest /= MICROS_PER_MINUTE
+    val minutes = rest % MINUTES_PER_HOUR
+    rest /= MINUTES_PER_HOUR
+    val hours = rest % HOURS_PER_DAY
+    val days = rest / HOURS_PER_DAY
+    val leadSecZero = if (secondsWithFraction < 10 * MICROS_PER_SECOND) "0" else ""
+    val intervalString = style match {
+      case ANSI_STYLE =>
+        val secStr = java.math.BigDecimal.valueOf(secondsWithFraction, 6)
+          .stripTrailingZeros()
+          .toPlainString()
+        f"INTERVAL '$sign$days $hours%02d:$minutes%02d:$leadSecZero$secStr' DAY TO SECOND"
+      case HIVE_STYLE =>
+        val seconds = secondsWithFraction / MICROS_PER_SECOND
+        val nanos = (secondsWithFraction % MICROS_PER_SECOND) * NANOS_PER_MICROS
+        f"$sign$days $hours%02d:$minutes%02d:$seconds%02d.$nanos%09d"
+    }
+    intervalString
+  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/NumberConverter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/NumberConverter.scala
index 7dbdd1ef1cdc5..dca75e5083331 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/NumberConverter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/NumberConverter.scala
@@ -21,26 +21,6 @@ import org.apache.spark.unsafe.types.UTF8String
 
 object NumberConverter {
 
-  /**
-   * Divide x by m as if x is an unsigned 64-bit integer. Examples:
-   * unsignedLongDiv(-1, 2) == Long.MAX_VALUE unsignedLongDiv(6, 3) == 2
-   * unsignedLongDiv(0, 5) == 0
-   *
-   * @param x is treated as unsigned
-   * @param m is treated as signed
-   */
-  private def unsignedLongDiv(x: Long, m: Int): Long = {
-    if (x >= 0) {
-      x / m
-    } else {
-      // Let uval be the value of the unsigned long with the same bits as x
-      // Two's complement => x = uval - 2*MAX - 2
-      // => uval = x + 2*MAX + 2
-      // Now, use the fact: (a+b)/c = a/c + b/c + (a%c+b%c)/c
-      x / m + 2 * (Long.MaxValue / m) + 2 / m + (x % m + 2 * (Long.MaxValue % m) + 2 % m) / m
-    }
-  }
-
   /**
    * Decode v into value[].
    *
@@ -52,7 +32,7 @@ object NumberConverter {
     java.util.Arrays.fill(value, 0.asInstanceOf[Byte])
     var i = value.length - 1
     while (tmpV != 0) {
-      val q = unsignedLongDiv(tmpV, radix)
+      val q = java.lang.Long.divideUnsigned(tmpV, radix)
       value(i) = (tmpV - q * radix).asInstanceOf[Byte]
       tmpV = q
       i -= 1
@@ -69,12 +49,12 @@ object NumberConverter {
    */
   private def encode(radix: Int, fromPos: Int, value: Array[Byte]): Long = {
     var v: Long = 0L
-    val bound = unsignedLongDiv(-1 - radix, radix) // Possible overflow once
+    val bound = java.lang.Long.divideUnsigned(-1 - radix, radix) // Possible overflow once
     var i = fromPos
     while (i < value.length && value(i) >= 0) {
       if (v >= bound) {
         // Check for overflow
-        if (unsignedLongDiv(-1 - value(i), radix) < v) {
+        if (java.lang.Long.divideUnsigned(-1 - value(i), radix) < v) {
           return -1
         }
       }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/QuantileSummaries.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/QuantileSummaries.scala
index 3a0490d07733d..addf1408a33a8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/QuantileSummaries.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/QuantileSummaries.scala
@@ -173,13 +173,13 @@ class QuantileSummaries(
       // Take the case of the sample `10` from `b`. In the original stream, it could have appeared
       // right after `0` (as expressed by `g=1`) or right before `20`, so `delta=99+0-1=98`.
       // In the GK algorithm's style of working in terms of maximum bounds, one can observe that the
-      // maximum additional uncertainty over samples comming from `b` is `max(g_a + delta_a) =
+      // maximum additional uncertainty over samples coming from `b` is `max(g_a + delta_a) =
       // floor(2 * eps_a * n_a)`. Likewise, additional uncertainty over samples from `a` is
       // `floor(2 * eps_b * n_b)`.
       // Only samples that interleave the other side are affected. That means that samples from
       // one side that are lesser (or greater) than all samples from the other side are just copied
-      // unmodifed.
-      // If the merging instances have different `relativeError`, the resulting instance will cary
+      // unmodified.
+      // If the merging instances have different `relativeError`, the resulting instance will carry
       // the largest one: `eps_ab = max(eps_a, eps_b)`.
       // The main invariant of the GK algorithm is kept:
       // `max(g_ab + delta_ab) <= floor(2 * eps_ab * (n_a + n_b))` since
@@ -223,7 +223,7 @@ class QuantileSummaries(
         otherIdx += 1
       }
 
-      val comp = compressImmut(mergedSampled, 2 * mergedRelativeError * mergedCount)
+      val comp = compressImmut(mergedSampled.toIndexedSeq, 2 * mergedRelativeError * mergedCount)
       new QuantileSummaries(other.compressThreshold, mergedRelativeError, comp, mergedCount, true)
     }
   }
@@ -254,7 +254,7 @@ class QuantileSummaries(
 
     // Target rank
     val rank = math.ceil(quantile * count).toLong
-    val targetError = relativeError * count
+    val targetError = sampled.map(s => s.delta + s.g).max / 2
     // Minimum rank at current sample
     var minRank = 0L
     var i = 0
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
index cc75340cd8fcd..2999d475fc8f6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
@@ -17,9 +17,10 @@
 
 package org.apache.spark.sql.catalyst.util
 
-import java.time.{LocalDate, LocalDateTime, LocalTime, ZoneId}
+import java.time.{LocalDate, LocalDateTime, LocalTime}
 import java.time.temporal.ChronoField
 import java.util.{Calendar, TimeZone}
+import java.util.Calendar.{DAY_OF_MONTH, DST_OFFSET, ERA, HOUR_OF_DAY, MINUTE, MONTH, SECOND, YEAR, ZONE_OFFSET}
 
 import scala.collection.mutable.AnyRefMap
 
@@ -28,6 +29,7 @@ import com.fasterxml.jackson.module.scala.{DefaultScalaModule, ScalaObjectMapper
 
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils._
+import org.apache.spark.util.Utils
 
 /**
  * The collection of functions for rebasing days and microseconds from/to the hybrid calendar
@@ -102,15 +104,15 @@ object RebaseDateTime {
       .setInstant(Math.multiplyExact(days, MILLIS_PER_DAY))
       .build()
     val localDate = LocalDate.of(
-      utcCal.get(Calendar.YEAR),
-      utcCal.get(Calendar.MONTH) + 1,
+      utcCal.get(YEAR),
+      utcCal.get(MONTH) + 1,
       // The number of days will be added later to handle non-existing
       // Julian dates in Proleptic Gregorian calendar.
       // For example, 1000-02-29 exists in Julian calendar because 1000
       // is a leap year but it is not a leap year in Gregorian calendar.
       1)
-      .`with`(ChronoField.ERA, utcCal.get(Calendar.ERA))
-      .plusDays(utcCal.get(Calendar.DAY_OF_MONTH) - 1)
+      .`with`(ChronoField.ERA, utcCal.get(ERA))
+      .plusDays(utcCal.get(DAY_OF_MONTH) - 1)
     Math.toIntExact(localDate.toEpochDay)
   }
 
@@ -185,7 +187,8 @@ object RebaseDateTime {
       .setTimeZone(TimeZoneUTC)
       .setDate(localDate.getYear, localDate.getMonthValue - 1, localDate.getDayOfMonth)
       .build()
-    Math.toIntExact(Math.floorDiv(utcCal.getTimeInMillis, MILLIS_PER_DAY))
+    assert(utcCal.getTimeInMillis % MILLIS_PER_DAY == 0)
+    Math.toIntExact(utcCal.getTimeInMillis / MILLIS_PER_DAY)
   }
 
   /**
@@ -261,7 +264,7 @@ object RebaseDateTime {
   // `JsonRebaseRecord`. AnyRefMap is used here instead of Scala's immutable map because
   // it is 2 times faster in DateTimeRebaseBenchmark.
   private[sql] def loadRebaseRecords(fileName: String): AnyRefMap[String, RebaseInfo] = {
-    val file = Thread.currentThread().getContextClassLoader.getResource(fileName)
+    val file = Utils.getSparkClassLoader.getResource(fileName)
     val mapper = new ObjectMapper() with ScalaObjectMapper
     mapper.registerModule(DefaultScalaModule)
     val jsonRebaseRecords = mapper.readValue[Seq[JsonRebaseRecord]](file)
@@ -319,27 +322,43 @@ object RebaseDateTime {
    *   Julian calendar: 1582-01-01 00:00:00.123456 -> -12243196799876544
    * The code below converts -12244061221876544 to -12243196799876544.
    *
-   * @param zoneId The time zone ID at which the rebasing should be performed.
+   * @param tz The time zone at which the rebasing should be performed.
    * @param micros The number of microseconds since the epoch '1970-01-01T00:00:00Z'
    *               in Proleptic Gregorian calendar. It can be negative.
    * @return The rebased microseconds since the epoch in Julian calendar.
    */
-  private[sql] def rebaseGregorianToJulianMicros(zoneId: ZoneId, micros: Long): Long = {
+  private[sql] def rebaseGregorianToJulianMicros(tz: TimeZone, micros: Long): Long = {
     val instant = microsToInstant(micros)
-    var ldt = instant.atZone(zoneId).toLocalDateTime
+    val zoneId = tz.toZoneId
+    val zonedDateTime = instant.atZone(zoneId)
+    var ldt = zonedDateTime.toLocalDateTime
     if (ldt.isAfter(julianEndTs) && ldt.isBefore(gregorianStartTs)) {
       ldt = LocalDateTime.of(gregorianStartDate, ldt.toLocalTime)
     }
     val cal = new Calendar.Builder()
-      // `gregory` is a hybrid calendar that supports both
-      // the Julian and Gregorian calendar systems
+      // `gregory` is a hybrid calendar that supports both the Julian and Gregorian calendar systems
       .setCalendarType("gregory")
       .setDate(ldt.getYear, ldt.getMonthValue - 1, ldt.getDayOfMonth)
       .setTimeOfDay(ldt.getHour, ldt.getMinute, ldt.getSecond)
-      // Local time-line can overlaps, such as at an autumn daylight savings cutover.
-      // This setting selects the original local timestamp mapped to the given `micros`.
-      .set(Calendar.DST_OFFSET, zoneId.getRules.getDaylightSavings(instant).toMillis.toInt)
+      .setTimeZone(tz)
       .build()
+    // A local timestamp can have 2 instants in the cases of switching from:
+    //  1. Summer to winter time.
+    //  2. One standard time zone to another one. For example, Asia/Hong_Kong switched from JST
+    //     to HKT on 18 November, 1945 01:59:59 AM.
+    // Below we check that the original `instant` is earlier or later instant. If it is an earlier
+    // instant, we take the standard and DST offsets of the previous day otherwise of the next one.
+    val trans = zoneId.getRules.getTransition(ldt)
+    if (trans != null && trans.isOverlap) {
+      val cloned = cal.clone().asInstanceOf[Calendar]
+      // Does the current offset belong to the offset before the transition.
+      // If so, we will take zone offsets from the previous day otherwise from the next day.
+      // This assumes that transitions cannot happen often than once per 2 days.
+      val shift = if (trans.getOffsetBefore == zonedDateTime.getOffset) -1 else 1
+      cloned.add(DAY_OF_MONTH, shift)
+      cal.set(ZONE_OFFSET, cloned.get(ZONE_OFFSET))
+      cal.set(DST_OFFSET, cloned.get(DST_OFFSET))
+    }
     millisToMicros(cal.getTimeInMillis) + ldt.get(ChronoField.MICRO_OF_SECOND)
   }
 
@@ -364,7 +383,7 @@ object RebaseDateTime {
       val tzId = timeZone.getID
       val rebaseRecord = gregJulianRebaseMap.getOrNull(tzId)
       if (rebaseRecord == null || micros < rebaseRecord.switches(0)) {
-        rebaseGregorianToJulianMicros(timeZone.toZoneId, micros)
+        rebaseGregorianToJulianMicros(timeZone, micros)
       } else {
         rebaseMicros(rebaseRecord, micros)
       }
@@ -386,45 +405,50 @@ object RebaseDateTime {
    *   Proleptic Gregorian calendar: 1582-01-01 00:00:00.123456 -> -12244061221876544
    * The code below converts -12243196799876544 to -12244061221876544.
    *
-   * @param zoneId The time zone ID at which the rebasing should be performed.
+   * @param tz The time zone at which the rebasing should be performed.
    * @param micros The number of microseconds since the epoch '1970-01-01T00:00:00Z'
    *               in the Julian calendar. It can be negative.
    * @return The rebased microseconds since the epoch in Proleptic Gregorian calendar.
    */
-  private[sql] def rebaseJulianToGregorianMicros(zoneId: ZoneId, micros: Long): Long = {
+  private[sql] def rebaseJulianToGregorianMicros(tz: TimeZone, micros: Long): Long = {
     val cal = new Calendar.Builder()
-      // `gregory` is a hybrid calendar that supports both
-      // the Julian and Gregorian calendar systems
+      // `gregory` is a hybrid calendar that supports both the Julian and Gregorian calendar systems
       .setCalendarType("gregory")
       .setInstant(microsToMillis(micros))
+      .setTimeZone(tz)
       .build()
     val localDateTime = LocalDateTime.of(
-      cal.get(Calendar.YEAR),
-      cal.get(Calendar.MONTH) + 1,
+      cal.get(YEAR),
+      cal.get(MONTH) + 1,
       // The number of days will be added later to handle non-existing
       // Julian dates in Proleptic Gregorian calendar.
       // For example, 1000-02-29 exists in Julian calendar because 1000
       // is a leap year but it is not a leap year in Gregorian calendar.
       1,
-      cal.get(Calendar.HOUR_OF_DAY),
-      cal.get(Calendar.MINUTE),
-      cal.get(Calendar.SECOND),
+      cal.get(HOUR_OF_DAY),
+      cal.get(MINUTE),
+      cal.get(SECOND),
       (Math.floorMod(micros, MICROS_PER_SECOND) * NANOS_PER_MICROS).toInt)
-      .`with`(ChronoField.ERA, cal.get(Calendar.ERA))
-      .plusDays(cal.get(Calendar.DAY_OF_MONTH) - 1)
+      .`with`(ChronoField.ERA, cal.get(ERA))
+      .plusDays(cal.get(DAY_OF_MONTH) - 1)
+    val zoneId = tz.toZoneId
     val zonedDateTime = localDateTime.atZone(zoneId)
-    // Assuming the daylight saving switchover time is 2:00, the local clock will go back to
-    // 2:00 after hitting 2:59. This means the local time between [2:00, 3:00) appears twice, and
-    // can map to two different physical times (seconds from the UTC epoch).
-    // Java 8 time API resolves the ambiguity by picking the earlier physical time. This is the same
-    // as Java 7 time API, except for 2:00 where Java 7 picks the later physical time.
-    // Here we detect the "2:00" case and pick the latter physical time, to be compatible with the
-    // Java 7 date-time.
-    val adjustedZdt = if (cal.get(Calendar.DST_OFFSET) == 0) {
-      zonedDateTime.withLaterOffsetAtOverlap()
-    } else {
-      zonedDateTime
-    }
+    // In the case of local timestamp overlapping, we need to choose the correct time instant
+    // which is related to the original local timestamp. We look ahead of 1 day, and if the next
+    // date has the same standard zone and DST offsets, the current local timestamp belongs to
+    // the period after the transition. In that case, we take the later zoned date time otherwise
+    // earlier one. Here, we assume that transitions happen not often than once per day.
+    val trans = zoneId.getRules.getTransition(localDateTime)
+    val adjustedZdt = if (trans != null && trans.isOverlap) {
+      val dstOffset = cal.get(DST_OFFSET)
+      val zoneOffset = cal.get(ZONE_OFFSET)
+      cal.add(DAY_OF_MONTH, 1)
+      if (zoneOffset == cal.get(ZONE_OFFSET) && dstOffset == cal.get(DST_OFFSET)) {
+        zonedDateTime.withLaterOffsetAtOverlap()
+      } else {
+        zonedDateTime.withEarlierOffsetAtOverlap()
+      }
+    } else zonedDateTime
     instantToMicros(adjustedZdt.toInstant)
   }
 
@@ -460,7 +484,7 @@ object RebaseDateTime {
       val tzId = timeZone.getID
       val rebaseRecord = julianGregRebaseMap.getOrNull(tzId)
       if (rebaseRecord == null || micros < rebaseRecord.switches(0)) {
-        rebaseJulianToGregorianMicros(timeZone.toZoneId, micros)
+        rebaseJulianToGregorianMicros(timeZone, micros)
       } else {
         rebaseMicros(rebaseRecord, micros)
       }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/SQLOrderingUtil.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/SQLOrderingUtil.scala
new file mode 100644
index 0000000000000..3b7f748c2817b
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/SQLOrderingUtil.scala
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+object SQLOrderingUtil {
+
+  /**
+   * A special version of double comparison that follows SQL semantic:
+   *  1. NaN == NaN
+   *  2. NaN is greater than any non-NaN double
+   *  3. -0.0 == 0.0
+   */
+  def compareDoubles(x: Double, y: Double): Int = {
+    if (x == y) 0 else java.lang.Double.compare(x, y)
+  }
+
+  /**
+   * A special version of float comparison that follows SQL semantic:
+   *  1. NaN == NaN
+   *  2. NaN is greater than any non-NaN float
+   *  3. -0.0 == 0.0
+   */
+  def compareFloats(x: Float, y: Float): Int = {
+    if (x == y) 0 else java.lang.Float.compare(x, y)
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/StringUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/StringUtils.scala
index b42ae4e45366e..2a416d6c97584 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/StringUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/StringUtils.scala
@@ -123,7 +123,11 @@ object StringUtils extends Logging {
           val stringToAppend = if (available >= sLen) s else s.substring(0, available)
           strings.append(stringToAppend)
         }
-        length += sLen
+
+        // Keeps the total length of appended strings. Note that we need to cap the length at
+        // `ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH`; otherwise, we will overflow
+        // length causing StringIndexOutOfBoundsException in the substring call above.
+        length = Math.min(length.toLong + sLen, ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH).toInt
       }
     }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
index dc06fa9d6f1c4..6832d1a9954fb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
@@ -24,7 +24,6 @@ import java.time.format.{DateTimeFormatter, DateTimeParseException}
 import java.time.temporal.ChronoField.MICRO_OF_SECOND
 import java.time.temporal.TemporalQueries
 import java.util.{Calendar, GregorianCalendar, Locale, TimeZone}
-import java.util.concurrent.TimeUnit.SECONDS
 
 import org.apache.commons.lang3.time.FastDateFormat
 
@@ -50,7 +49,11 @@ sealed trait TimestampFormatter extends Serializable {
   @throws(classOf[DateTimeParseException])
   @throws(classOf[DateTimeException])
   def parse(s: String): Long
+
   def format(us: Long): String
+  def format(ts: Timestamp): String
+  def format(instant: Instant): String
+  def validatePatternString(): Unit
 }
 
 class Iso8601TimestampFormatter(
@@ -58,11 +61,11 @@ class Iso8601TimestampFormatter(
     zoneId: ZoneId,
     locale: Locale,
     legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT,
-    needVarLengthSecondFraction: Boolean)
+    isParsing: Boolean)
   extends TimestampFormatter with DateTimeFormatterHelper {
   @transient
   protected lazy val formatter: DateTimeFormatter =
-    getOrCreateFormatter(pattern, locale, needVarLengthSecondFraction)
+    getOrCreateFormatter(pattern, locale, isParsing)
 
   @transient
   protected lazy val legacyFormatter = TimestampFormatter.getLegacyFormatter(
@@ -79,14 +82,31 @@ class Iso8601TimestampFormatter(
         val epochSeconds = zonedDateTime.toEpochSecond
         val microsOfSecond = zonedDateTime.get(MICRO_OF_SECOND)
 
-        Math.addExact(SECONDS.toMicros(epochSeconds), microsOfSecond)
-      } catch checkDiffResult(s, legacyFormatter.parse)
+        Math.addExact(Math.multiplyExact(epochSeconds, MICROS_PER_SECOND), microsOfSecond)
+      } catch checkParsedDiff(s, legacyFormatter.parse)
     }
   }
 
+  override def format(instant: Instant): String = {
+    try {
+      formatter.withZone(zoneId).format(instant)
+    } catch checkFormattedDiff(toJavaTimestamp(instantToMicros(instant)),
+      (t: Timestamp) => format(t))
+  }
+
   override def format(us: Long): String = {
     val instant = DateTimeUtils.microsToInstant(us)
-    formatter.withZone(zoneId).format(instant)
+    format(instant)
+  }
+
+  override def format(ts: Timestamp): String = {
+    legacyFormatter.format(ts)
+  }
+
+  override def validatePatternString(): Unit = {
+    try {
+      formatter
+    } catch checkLegacyFormatter(pattern, legacyFormatter.validatePatternString)
   }
 }
 
@@ -100,10 +120,49 @@ class Iso8601TimestampFormatter(
  */
 class FractionTimestampFormatter(zoneId: ZoneId)
   extends Iso8601TimestampFormatter(
-    "", zoneId, TimestampFormatter.defaultLocale, needVarLengthSecondFraction = false) {
+    TimestampFormatter.defaultPattern,
+    zoneId,
+    TimestampFormatter.defaultLocale,
+    LegacyDateFormats.FAST_DATE_FORMAT,
+    isParsing = false) {
 
   @transient
   override protected lazy val formatter = DateTimeFormatterHelper.fractionFormatter
+
+  // The new formatter will omit the trailing 0 in the timestamp string, but the legacy formatter
+  // can't. Here we use the legacy formatter to format the given timestamp up to seconds fractions,
+  // and custom implementation to format the fractional part without trailing zeros.
+  override def format(ts: Timestamp): String = {
+    val formatted = legacyFormatter.format(ts)
+    var nanos = ts.getNanos
+    if (nanos == 0) {
+      formatted
+    } else {
+      // Formats non-zero seconds fraction w/o trailing zeros. For example:
+      //   formatted = '2020-05:27 15:55:30'
+      //   nanos = 001234000
+      // Counts the length of the fractional part: 001234000 -> 6
+      var fracLen = 9
+      while (nanos % 10 == 0) {
+        nanos /= 10
+        fracLen -= 1
+      }
+      // Places `nanos` = 1234 after '2020-05:27 15:55:30.'
+      val fracOffset = formatted.length + 1
+      val totalLen = fracOffset + fracLen
+      // The buffer for the final result: '2020-05:27 15:55:30.001234'
+      val buf = new Array[Char](totalLen)
+      formatted.getChars(0, formatted.length, buf, 0)
+      buf(formatted.length) = '.'
+      var i = totalLen
+      do {
+        i -= 1
+        buf(i) = ('0' + (nanos % 10)).toChar
+        nanos /= 10
+      } while (i > fracOffset)
+      new String(buf)
+    }
+  }
 }
 
 /**
@@ -120,7 +179,7 @@ class MicrosCalendar(tz: TimeZone, digitsInFraction: Int)
   // Converts parsed `MILLISECOND` field to seconds fraction in microsecond precision.
   // For example if the fraction pattern is `SSSS` then `digitsInFraction` = 4, and
   // if the `MILLISECOND` field was parsed to `1234`.
-  def getMicros(): SQLTimestamp = {
+  def getMicros(): Long = {
     // Append 6 zeros to the field: 1234 -> 1234000000
     val d = fields(Calendar.MILLISECOND) * MICROS_PER_SECOND
     // Take the first 6 digits from `d`: 1234000000 -> 123400
@@ -149,7 +208,7 @@ class LegacyFastTimestampFormatter(
     fastDateFormat.getTimeZone,
     fastDateFormat.getPattern.count(_ == 'S'))
 
-  def parse(s: String): SQLTimestamp = {
+  override def parse(s: String): Long = {
     cal.clear() // Clear the calendar because it can be re-used many times
     if (!fastDateFormat.parse(s, new ParsePosition(0), cal)) {
       throw new IllegalArgumentException(s"'$s' is an invalid timestamp")
@@ -160,12 +219,26 @@ class LegacyFastTimestampFormatter(
     rebaseJulianToGregorianMicros(julianMicros)
   }
 
-  def format(timestamp: SQLTimestamp): String = {
+  override def format(timestamp: Long): String = {
     val julianMicros = rebaseGregorianToJulianMicros(timestamp)
     cal.setTimeInMillis(Math.floorDiv(julianMicros, MICROS_PER_SECOND) * MILLIS_PER_SECOND)
     cal.setMicros(Math.floorMod(julianMicros, MICROS_PER_SECOND))
     fastDateFormat.format(cal)
   }
+
+  override def format(ts: Timestamp): String = {
+    if (ts.getNanos == 0) {
+      fastDateFormat.format(ts)
+    } else {
+      format(fromJavaTimestamp(ts))
+    }
+  }
+
+  override def format(instant: Instant): String = {
+    format(instantToMicros(instant))
+  }
+
+  override def validatePatternString(): Unit = fastDateFormat
 }
 
 class LegacySimpleTimestampFormatter(
@@ -187,6 +260,16 @@ class LegacySimpleTimestampFormatter(
   override def format(us: Long): String = {
     sdf.format(toJavaTimestamp(us))
   }
+
+  override def format(ts: Timestamp): String = {
+    sdf.format(ts)
+  }
+
+  override def format(instant: Instant): String = {
+    format(instantToMicros(instant))
+  }
+
+  override def validatePatternString(): Unit = sdf
 }
 
 object LegacyDateFormats extends Enumeration {
@@ -206,14 +289,16 @@ object TimestampFormatter {
       zoneId: ZoneId,
       locale: Locale = defaultLocale,
       legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT,
-      needVarLengthSecondFraction: Boolean = false): TimestampFormatter = {
+      isParsing: Boolean): TimestampFormatter = {
     val pattern = format.getOrElse(defaultPattern)
-    if (SQLConf.get.legacyTimeParserPolicy == LEGACY) {
+    val formatter = if (SQLConf.get.legacyTimeParserPolicy == LEGACY) {
       getLegacyFormatter(pattern, zoneId, locale, legacyFormat)
     } else {
       new Iso8601TimestampFormatter(
-        pattern, zoneId, locale, legacyFormat, needVarLengthSecondFraction)
+        pattern, zoneId, locale, legacyFormat, isParsing)
     }
+    formatter.validatePatternString()
+    formatter
   }
 
   def getLegacyFormatter(
@@ -236,27 +321,27 @@ object TimestampFormatter {
       zoneId: ZoneId,
       locale: Locale,
       legacyFormat: LegacyDateFormat,
-      needVarLengthSecondFraction: Boolean): TimestampFormatter = {
-    getFormatter(Some(format), zoneId, locale, legacyFormat, needVarLengthSecondFraction)
+      isParsing: Boolean): TimestampFormatter = {
+    getFormatter(Some(format), zoneId, locale, legacyFormat, isParsing)
   }
 
   def apply(
       format: String,
       zoneId: ZoneId,
       legacyFormat: LegacyDateFormat,
-      needVarLengthSecondFraction: Boolean): TimestampFormatter = {
-    getFormatter(Some(format), zoneId, defaultLocale, legacyFormat, needVarLengthSecondFraction)
+      isParsing: Boolean): TimestampFormatter = {
+    getFormatter(Some(format), zoneId, defaultLocale, legacyFormat, isParsing)
   }
 
   def apply(
       format: String,
       zoneId: ZoneId,
-      needVarLengthSecondFraction: Boolean = false): TimestampFormatter = {
-    getFormatter(Some(format), zoneId, needVarLengthSecondFraction = needVarLengthSecondFraction)
+      isParsing: Boolean): TimestampFormatter = {
+    getFormatter(Some(format), zoneId, isParsing = isParsing)
   }
 
   def apply(zoneId: ZoneId): TimestampFormatter = {
-    getFormatter(None, zoneId)
+    getFormatter(None, zoneId, isParsing = false)
   }
 
   def getFractionFormatter(zoneId: ZoneId): TimestampFormatter = {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TypeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TypeUtils.scala
index e8266dd401362..f020161448104 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TypeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TypeUtils.scala
@@ -17,9 +17,9 @@
 
 package org.apache.spark.sql.catalyst.util
 
-import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, TypeCoercion}
 import org.apache.spark.sql.catalyst.expressions.RowOrdering
+import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.types._
 
 /**
@@ -61,6 +61,14 @@ object TypeUtils {
     }
   }
 
+  def checkForAnsiIntervalOrNumericType(
+      dt: DataType, funcName: String): TypeCheckResult = dt match {
+    case YearMonthIntervalType | DayTimeIntervalType | NullType => TypeCheckResult.TypeCheckSuccess
+    case dt if dt.isInstanceOf[NumericType] => TypeCheckResult.TypeCheckSuccess
+    case other => TypeCheckResult.TypeCheckFailure(
+      s"function $funcName requires numeric or interval types, not ${other.catalogString}")
+  }
+
   def getNumeric(t: DataType, exactNumericRequired: Boolean = false): Numeric[Any] = {
     if (exactNumericRequired) {
       t.asInstanceOf[NumericType].exactNumeric.asInstanceOf[Numeric[Any]]
@@ -101,16 +109,16 @@ object TypeUtils {
   }
 
   def failWithIntervalType(dataType: DataType): Unit = {
-    dataType match {
-      case CalendarIntervalType =>
-        throw new AnalysisException("Cannot use interval type in the table schema.")
-      case ArrayType(et, _) => failWithIntervalType(et)
-      case MapType(kt, vt, _) =>
-        failWithIntervalType(kt)
-        failWithIntervalType(vt)
-      case s: StructType => s.foreach(f => failWithIntervalType(f.dataType))
-      case u: UserDefinedType[_] => failWithIntervalType(u.sqlType)
-      case _ =>
+    invokeOnceForInterval(dataType) {
+      throw QueryCompilationErrors.cannotUseIntervalTypeInTableSchemaError()
+    }
+  }
+
+  def invokeOnceForInterval(dataType: DataType)(f: => Unit): Unit = {
+    def isInterval(dataType: DataType): Boolean = dataType match {
+      case CalendarIntervalType | DayTimeIntervalType | YearMonthIntervalType => true
+      case _ => false
     }
+    if (dataType.existsRecursively(isInterval)) f
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/UnsafeRowUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/UnsafeRowUtils.scala
new file mode 100644
index 0000000000000..37a34fac66364
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/UnsafeRowUtils.scala
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import org.apache.spark.sql.catalyst.expressions.UnsafeRow
+import org.apache.spark.sql.types._
+
+object UnsafeRowUtils {
+
+  /**
+   * Use the following rules to check the integrity of the UnsafeRow:
+   * - schema.fields.length == row.numFields should always be true
+   * - UnsafeRow.calculateBitSetWidthInBytes(row.numFields) < row.getSizeInBytes should always be
+   *   true if the expectedSchema contains at least one field.
+   * - For variable-length fields: if null bit says it's null then don't do anything, else extract
+   *   offset and size:
+   *   1) 0 <= size < row.getSizeInBytes should always be true. We can be even more precise than
+   *      this, where the upper bound of size can only be as big as the variable length part of
+   *      the row.
+   *   2) offset should be >= fixed sized part of the row.
+   *   3) offset + size should be within the row bounds.
+   * - For fixed-length fields that are narrower than 8 bytes (boolean/byte/short/int/float), if
+   *   null bit says it's null then don't do anything, else:
+   *     check if the unused bits in the field are all zeros. The UnsafeRowWriter's write() methods
+   *     make this guarantee.
+   * - Check the total length of the row.
+   */
+  def validateStructuralIntegrity(row: UnsafeRow, expectedSchema: StructType): Boolean = {
+    if (expectedSchema.fields.length != row.numFields) {
+      return false
+    }
+    val bitSetWidthInBytes = UnsafeRow.calculateBitSetWidthInBytes(row.numFields)
+    val rowSizeInBytes = row.getSizeInBytes
+    if (expectedSchema.fields.length > 0 && bitSetWidthInBytes >= rowSizeInBytes) {
+      return false
+    }
+    var varLenFieldsSizeInBytes = 0
+    expectedSchema.fields.zipWithIndex.foreach {
+      case (field, index) if !UnsafeRow.isFixedLength(field.dataType) && !row.isNullAt(index) =>
+        val offsetAndSize = row.getLong(index)
+        val offset = (offsetAndSize >> 32).toInt
+        val size = offsetAndSize.toInt
+        if (size < 0 ||
+            offset < bitSetWidthInBytes + 8 * row.numFields || offset + size > rowSizeInBytes) {
+          return false
+        }
+        varLenFieldsSizeInBytes += size
+      case (field, index) if UnsafeRow.isFixedLength(field.dataType) && !row.isNullAt(index) =>
+        field.dataType match {
+          case BooleanType =>
+            if ((row.getLong(index) >> 1) != 0L) return false
+          case ByteType =>
+            if ((row.getLong(index) >> 8) != 0L) return false
+          case ShortType =>
+            if ((row.getLong(index) >> 16) != 0L) return false
+          case IntegerType =>
+            if ((row.getLong(index) >> 32) != 0L) return false
+          case FloatType =>
+            if ((row.getLong(index) >> 32) != 0L) return false
+          case _ =>
+        }
+      case (_, index) if row.isNullAt(index) =>
+        if (row.getLong(index) != 0L) return false
+      case _ =>
+    }
+    if (bitSetWidthInBytes + 8 * row.numFields + varLenFieldsSizeInBytes > rowSizeInBytes) {
+      return false
+    }
+    true
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala
index a5f0b239d6086..33fe48d44dadb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala
@@ -25,7 +25,7 @@ import java.util.concurrent.atomic.AtomicBoolean
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{NumericType, StringType}
+import org.apache.spark.sql.types.{MetadataBuilder, NumericType, StringType}
 import org.apache.spark.unsafe.types.UTF8String
 import org.apache.spark.util.Utils
 
@@ -129,6 +129,7 @@ package object util extends Logging {
     case a: Attribute => new PrettyAttribute(a)
     case Literal(s: UTF8String, StringType) => PrettyAttribute(s.toString, StringType)
     case Literal(v, t: NumericType) if v != null => PrettyAttribute(v.toString, t)
+    case Literal(null, dataType) => PrettyAttribute("NULL", dataType)
     case e: GetStructField =>
       val name = e.name.getOrElse(e.childSchema(e.ordinal).name)
       PrettyAttribute(usePrettyExpression(e.child).sql + "." + name, e.dataType)
@@ -136,6 +137,8 @@ package object util extends Logging {
       PrettyAttribute(usePrettyExpression(e.child) + "." + e.field.name, e.dataType)
     case r: RuntimeReplaceable =>
       PrettyAttribute(r.mkString(r.exprsReplaced.map(toPrettySQL)), r.dataType)
+    case c: CastBase if !c.getTagValue(Cast.USER_SPECIFIED_CAST).getOrElse(false) =>
+      PrettyAttribute(usePrettyExpression(c.child).sql, c.dataType)
   }
 
   def quoteIdentifier(name: String): String = {
@@ -144,6 +147,14 @@ package object util extends Logging {
     "`" + name.replace("`", "``") + "`"
   }
 
+  def quoteIfNeeded(part: String): String = {
+    if (part.matches("[a-zA-Z0-9_]+") && !part.matches("\\d+")) {
+      part
+    } else {
+      s"`${part.replace("`", "``")}`"
+    }
+  }
+
   def toPrettySQL(e: Expression): String = usePrettyExpression(e).sql
 
   def escapeSingleQuotedString(str: String): String = {
@@ -190,4 +201,28 @@ package object util extends Logging {
   def truncatedString[T](seq: Seq[T], sep: String, maxFields: Int): String = {
     truncatedString(seq, "", sep, "", maxFields)
   }
+
+  val METADATA_COL_ATTR_KEY = "__metadata_col"
+
+  implicit class MetadataColumnHelper(attr: Attribute) {
+    /**
+     * If set, this metadata column is a candidate during qualified star expansions.
+     */
+    val SUPPORTS_QUALIFIED_STAR = "__supports_qualified_star"
+
+    def isMetadataCol: Boolean = attr.metadata.contains(METADATA_COL_ATTR_KEY) &&
+      attr.metadata.getBoolean(METADATA_COL_ATTR_KEY)
+
+    def supportsQualifiedStar: Boolean = attr.isMetadataCol &&
+      attr.metadata.contains(SUPPORTS_QUALIFIED_STAR) &&
+      attr.metadata.getBoolean(SUPPORTS_QUALIFIED_STAR)
+
+    def markAsSupportsQualifiedStar(): Attribute = attr.withMetadata(
+      new MetadataBuilder()
+        .withMetadata(attr.metadata)
+        .putBoolean(METADATA_COL_ATTR_KEY, true)
+        .putBoolean(SUPPORTS_QUALIFIED_STAR, true)
+        .build()
+    )
+  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogManager.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogManager.scala
index a81d8f79d6fcc..0779bf53fe446 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogManager.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogManager.scala
@@ -18,9 +18,9 @@
 package org.apache.spark.sql.connector.catalog
 
 import scala.collection.mutable
-import scala.util.control.NonFatal
 
 import org.apache.spark.internal.Logging
+import org.apache.spark.sql.catalyst.SQLConfHelper
 import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
 import org.apache.spark.sql.catalyst.catalog.SessionCatalog
 import org.apache.spark.sql.internal.SQLConf
@@ -38,10 +38,10 @@ import org.apache.spark.sql.internal.SQLConf
 //       need to track current database at all.
 private[sql]
 class CatalogManager(
-    conf: SQLConf,
     defaultSessionCatalog: CatalogPlugin,
-    val v1SessionCatalog: SessionCatalog) extends Logging {
+    val v1SessionCatalog: SessionCatalog) extends SQLConfHelper with Logging {
   import CatalogManager.SESSION_CATALOG_NAME
+  import CatalogV2Util._
 
   private val catalogs = mutable.HashMap.empty[String, CatalogPlugin]
 
@@ -78,18 +78,11 @@ class CatalogManager(
    * This catalog is a v2 catalog that delegates to the v1 session catalog. it is used when the
    * session catalog is responsible for an identifier, but the source requires the v2 catalog API.
    * This happens when the source implementation extends the v2 TableProvider API and is not listed
-   * in the fallback configuration, spark.sql.sources.write.useV1SourceList
+   * in the fallback configuration, spark.sql.sources.useV1SourceList
    */
   private[sql] def v2SessionCatalog: CatalogPlugin = {
-    conf.getConf(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION).map { customV2SessionCatalog =>
-      try {
-        catalogs.getOrElseUpdate(SESSION_CATALOG_NAME, loadV2SessionCatalog())
-      } catch {
-        case NonFatal(_) =>
-          logError(
-            "Fail to instantiate the custom v2 session catalog: " + customV2SessionCatalog)
-          defaultSessionCatalog
-      }
+    conf.getConf(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION).map { _ =>
+      catalogs.getOrElseUpdate(SESSION_CATALOG_NAME, loadV2SessionCatalog())
     }.getOrElse(defaultSessionCatalog)
   }
 
@@ -106,13 +99,15 @@ class CatalogManager(
   }
 
   def setCurrentNamespace(namespace: Array[String]): Unit = synchronized {
-    if (currentCatalog.name() == SESSION_CATALOG_NAME) {
-      if (namespace.length != 1) {
+    currentCatalog match {
+      case _ if isSessionCatalog(currentCatalog) && namespace.length == 1 =>
+        v1SessionCatalog.setCurrentDatabase(namespace.head)
+      case _ if isSessionCatalog(currentCatalog) =>
         throw new NoSuchNamespaceException(namespace)
-      }
-      v1SessionCatalog.setCurrentDatabase(namespace.head)
-    } else {
-      _currentNamespace = Some(namespace)
+      case catalog: SupportsNamespaces if !catalog.namespaceExists(namespace) =>
+        throw new NoSuchNamespaceException(namespace)
+      case _ =>
+        _currentNamespace = Some(namespace)
     }
   }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala
index d90804f4b6ff6..39642fd541706 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala
@@ -17,12 +17,12 @@
 
 package org.apache.spark.sql.connector.catalog
 
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
 import org.apache.spark.sql.catalyst.catalog.BucketSpec
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
+import org.apache.spark.sql.catalyst.util.quoteIfNeeded
 import org.apache.spark.sql.connector.expressions.{BucketTransform, IdentityTransform, LogicalExpressions, Transform}
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.errors.QueryCompilationErrors
 
 /**
  * Conversion helpers for working with v2 [[CatalogPlugin]].
@@ -39,8 +39,7 @@ private[sql] object CatalogV2Implicits {
   implicit class BucketSpecHelper(spec: BucketSpec) {
     def asTransform: BucketTransform = {
       if (spec.sortColumnNames.nonEmpty) {
-        throw new AnalysisException(
-          s"Cannot convert bucketing with sort columns to a transform: $spec")
+        throw QueryCompilationErrors.cannotConvertBucketWithSortColumnsToTransformError(spec)
       }
 
       val references = spec.bucketColumnNames.map(col => reference(Seq(col)))
@@ -53,14 +52,13 @@ private[sql] object CatalogV2Implicits {
       val (idTransforms, nonIdTransforms) = transforms.partition(_.isInstanceOf[IdentityTransform])
 
       if (nonIdTransforms.nonEmpty) {
-        throw new AnalysisException("Transforms cannot be converted to partition columns: " +
-            nonIdTransforms.map(_.describe).mkString(", "))
+        throw QueryCompilationErrors.cannotConvertTransformsToPartitionColumnsError(nonIdTransforms)
       }
 
       idTransforms.map(_.asInstanceOf[IdentityTransform]).map(_.reference).map { ref =>
         val parts = ref.fieldNames
         if (parts.size > 1) {
-          throw new AnalysisException(s"Cannot partition by nested column: $ref")
+          throw QueryCompilationErrors.cannotPartitionByNestedColumnError(ref)
         } else {
           parts(0)
         }
@@ -73,15 +71,26 @@ private[sql] object CatalogV2Implicits {
       case tableCatalog: TableCatalog =>
         tableCatalog
       case _ =>
-        throw new AnalysisException(s"Cannot use catalog ${plugin.name}: not a TableCatalog")
+        throw QueryCompilationErrors.cannotUseCatalogError(plugin, "not a TableCatalog")
     }
 
     def asNamespaceCatalog: SupportsNamespaces = plugin match {
       case namespaceCatalog: SupportsNamespaces =>
         namespaceCatalog
       case _ =>
-        throw new AnalysisException(
-          s"Cannot use catalog ${plugin.name}: does not support namespaces")
+        throw QueryCompilationErrors.cannotUseCatalogError(plugin, "does not support namespaces")
+    }
+
+    def isFunctionCatalog: Boolean = plugin match {
+      case _: FunctionCatalog => true
+      case _ => false
+    }
+
+    def asFunctionCatalog: FunctionCatalog = plugin match {
+      case functionCatalog: FunctionCatalog =>
+        functionCatalog
+      case _ =>
+        throw QueryCompilationErrors.cannotUseCatalogError(plugin, "not a FunctionCatalog")
     }
   }
 
@@ -89,6 +98,17 @@ private[sql] object CatalogV2Implicits {
     def quoted: String = namespace.map(quoteIfNeeded).mkString(".")
   }
 
+  implicit class FunctionIdentifierHelper(ident: FunctionIdentifier) {
+    def asMultipart: Seq[String] = {
+      ident.database match {
+        case Some(db) =>
+          Seq(db, ident.funcName)
+        case _ =>
+          Seq(ident.funcName)
+      }
+    }
+  }
+
   implicit class IdentifierHelper(ident: Identifier) {
     def quoted: String = {
       if (ident.namespace.nonEmpty) {
@@ -104,14 +124,22 @@ private[sql] object CatalogV2Implicits {
       case ns if ns.isEmpty => TableIdentifier(ident.name)
       case Array(dbName) => TableIdentifier(ident.name, Some(dbName))
       case _ =>
-        throw new AnalysisException(
-          s"$quoted is not a valid TableIdentifier as it has more than 2 name parts.")
+        throw QueryCompilationErrors.identifierHavingMoreThanTwoNamePartsError(
+          quoted, "TableIdentifier")
+    }
+
+    def asFunctionIdentifier: FunctionIdentifier = ident.namespace() match {
+      case ns if ns.isEmpty => FunctionIdentifier(ident.name())
+      case Array(dbName) => FunctionIdentifier(ident.name(), Some(dbName))
+      case _ =>
+        throw QueryCompilationErrors.identifierHavingMoreThanTwoNamePartsError(
+          quoted, "FunctionIdentifier")
     }
   }
 
   implicit class MultipartIdentifierHelper(parts: Seq[String]) {
     if (parts.isEmpty) {
-      throw new AnalysisException("multi-part identifier cannot be empty.")
+      throw QueryCompilationErrors.emptyMultipartIdentifierError()
     }
 
     def asIdentifier: Identifier = Identifier.of(parts.init.toArray, parts.last)
@@ -120,24 +148,22 @@ private[sql] object CatalogV2Implicits {
       case Seq(tblName) => TableIdentifier(tblName)
       case Seq(dbName, tblName) => TableIdentifier(tblName, Some(dbName))
       case _ =>
-        throw new AnalysisException(
-          s"$quoted is not a valid TableIdentifier as it has more than 2 name parts.")
+        throw QueryCompilationErrors.identifierHavingMoreThanTwoNamePartsError(
+          quoted, "TableIdentifier")
     }
 
-    def quoted: String = parts.map(quoteIfNeeded).mkString(".")
-  }
-
-  def quoteIfNeeded(part: String): String = {
-    if (part.contains(".") || part.contains("`")) {
-      s"`${part.replace("`", "``")}`"
-    } else {
-      part
+    def asFunctionIdentifier: FunctionIdentifier = parts match {
+      case Seq(funcName) => FunctionIdentifier(funcName)
+      case Seq(dbName, funcName) => FunctionIdentifier(funcName, Some(dbName))
+      case _ =>
+        throw QueryCompilationErrors.identifierHavingMoreThanTwoNamePartsError(
+          quoted, "FunctionIdentifier")
     }
-  }
 
-  private lazy val catalystSqlParser = new CatalystSqlParser(SQLConf.get)
+    def quoted: String = parts.map(quoteIfNeeded).mkString(".")
+  }
 
   def parseColumnPath(name: String): Seq[String] = {
-    catalystSqlParser.parseMultipartIdentifier(name)
+    CatalystSqlParser.parseMultipartIdentifier(name)
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala
index e1f329352592f..a779e50a1f214 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala
@@ -22,13 +22,12 @@ import java.util.Collections
 
 import scala.collection.JavaConverters._
 
-import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.analysis.{NamedRelation, NoSuchDatabaseException, NoSuchNamespaceException, NoSuchTableException, UnresolvedV2Relation}
-import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
-import org.apache.spark.sql.catalyst.plans.logical.AlterTable
+import org.apache.spark.sql.catalyst.plans.logical.{AlterTable, CreateTableAsSelectStatement, CreateTableStatement, ReplaceTableAsSelectStatement, ReplaceTableStatement, SerdeInfo}
 import org.apache.spark.sql.connector.catalog.TableChange._
+import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
-import org.apache.spark.sql.types.{ArrayType, DataType, HIVE_TYPE_STRING, HiveStringType, MapType, StructField, StructType}
+import org.apache.spark.sql.types.{ArrayType, DataType, MapType, NullType, StructField, StructType}
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 import org.apache.spark.util.Utils
 
@@ -295,18 +294,65 @@ private[sql] object CatalogV2Util {
     catalog.name().equalsIgnoreCase(CatalogManager.SESSION_CATALOG_NAME)
   }
 
-  def convertTableProperties(
+  def convertTableProperties(c: CreateTableStatement): Map[String, String] = {
+    convertTableProperties(
+      c.properties, c.options, c.serde, c.location, c.comment, c.provider, c.external)
+  }
+
+  def convertTableProperties(c: CreateTableAsSelectStatement): Map[String, String] = {
+    convertTableProperties(
+      c.properties, c.options, c.serde, c.location, c.comment, c.provider, c.external)
+  }
+
+  def convertTableProperties(r: ReplaceTableStatement): Map[String, String] = {
+    convertTableProperties(r.properties, r.options, r.serde, r.location, r.comment, r.provider)
+  }
+
+  def convertTableProperties(r: ReplaceTableAsSelectStatement): Map[String, String] = {
+    convertTableProperties(r.properties, r.options, r.serde, r.location, r.comment, r.provider)
+  }
+
+  private def convertTableProperties(
       properties: Map[String, String],
       options: Map[String, String],
+      serdeInfo: Option[SerdeInfo],
       location: Option[String],
       comment: Option[String],
-      provider: Option[String]): Map[String, String] = {
-    properties ++ options ++
+      provider: Option[String],
+      external: Boolean = false): Map[String, String] = {
+    properties ++
+      options ++ // to make the transition to the "option." prefix easier, add both
+      options.map { case (key, value) => TableCatalog.OPTION_PREFIX + key -> value } ++
+      convertToProperties(serdeInfo) ++
+      (if (external) Some(TableCatalog.PROP_EXTERNAL -> "true") else None) ++
       provider.map(TableCatalog.PROP_PROVIDER -> _) ++
       comment.map(TableCatalog.PROP_COMMENT -> _) ++
       location.map(TableCatalog.PROP_LOCATION -> _)
   }
 
+  /**
+   * Converts Hive Serde info to table properties. The mapped property keys are:
+   *  - INPUTFORMAT/OUTPUTFORMAT: hive.input/output-format
+   *  - STORED AS: hive.stored-as
+   *  - ROW FORMAT SERDE: hive.serde
+   *  - SERDEPROPERTIES: add "option." prefix
+   */
+  private def convertToProperties(serdeInfo: Option[SerdeInfo]): Map[String, String] = {
+    serdeInfo match {
+      case Some(s) =>
+        s.formatClasses.map { f =>
+          Map("hive.input-format" -> f.input, "hive.output-format" -> f.output)
+        }.getOrElse(Map.empty) ++
+        s.storedAs.map("hive.stored-as" -> _) ++
+        s.serde.map("hive.serde" -> _) ++
+        s.serdeProperties.map {
+          case (key, value) => TableCatalog.OPTION_PREFIX + key -> value
+        }
+      case None =>
+        Map.empty
+    }
+  }
+
   def withDefaultOwnership(properties: Map[String, String]): Map[String, String] = {
     properties ++ Map(TableCatalog.PROP_OWNER -> Utils.getCurrentUserName())
   }
@@ -332,18 +378,21 @@ private[sql] object CatalogV2Util {
       .asTableCatalog
   }
 
-  def failCharType(dt: DataType): Unit = {
-    if (HiveStringType.containsCharType(dt)) {
-      throw new AnalysisException(
-        "Cannot use CHAR type in non-Hive-Serde tables, please use STRING type instead.")
+  def failNullType(dt: DataType): Unit = {
+    def containsNullType(dt: DataType): Boolean = dt match {
+      case ArrayType(et, _) => containsNullType(et)
+      case MapType(kt, vt, _) => containsNullType(kt) || containsNullType(vt)
+      case StructType(fields) => fields.exists(f => containsNullType(f.dataType))
+      case _ => dt.isInstanceOf[NullType]
+    }
+    if (containsNullType(dt)) {
+      throw QueryCompilationErrors.cannotCreateTablesWithNullTypeError()
     }
   }
 
-  def assertNoCharTypeInSchema(schema: StructType): Unit = {
+  def assertNoNullTypeInSchema(schema: StructType): Unit = {
     schema.foreach { f =>
-      if (f.metadata.contains(HIVE_TYPE_STRING)) {
-        failCharType(CatalystSqlParser.parseRawDataType(f.metadata.getString(HIVE_TYPE_STRING)))
-      }
+      failNullType(f.dataType)
     }
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/LookupCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/LookupCatalog.scala
index 10c15747ec4ce..06358590a1e46 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/LookupCatalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/LookupCatalog.scala
@@ -18,8 +18,8 @@
 package org.apache.spark.sql.connector.catalog
 
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
+import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
 
 /**
@@ -57,14 +57,9 @@ private[sql] trait LookupCatalog extends Logging {
    * Extract session catalog and identifier from a multi-part identifier.
    */
   object SessionCatalogAndIdentifier {
-    import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
 
     def unapply(parts: Seq[String]): Option[(CatalogPlugin, Identifier)] = parts match {
       case CatalogAndIdentifier(catalog, ident) if CatalogV2Util.isSessionCatalog(catalog) =>
-        if (ident.namespace.length != 1) {
-          throw new AnalysisException(
-            s"The namespace in session catalog must have exactly one name part: ${parts.quoted}")
-        }
         Some(catalog, ident)
       case _ => None
     }
@@ -140,19 +135,68 @@ private[sql] trait LookupCatalog extends Logging {
    * For legacy support only. Please use [[CatalogAndIdentifier]] instead on DSv2 code paths.
    */
   object AsTableIdentifier {
-    def unapply(parts: Seq[String]): Option[TableIdentifier] = parts match {
-      case CatalogAndMultipartIdentifier(None, names)
+    def unapply(parts: Seq[String]): Option[TableIdentifier] = {
+      def namesToTableIdentifier(names: Seq[String]): Option[TableIdentifier] = names match {
+        case Seq(name) => Some(TableIdentifier(name))
+        case Seq(database, name) => Some(TableIdentifier(name, Some(database)))
+        case _ => None
+      }
+      parts match {
+        case CatalogAndMultipartIdentifier(None, names)
+          if CatalogV2Util.isSessionCatalog(currentCatalog) =>
+          namesToTableIdentifier(names)
+        case CatalogAndMultipartIdentifier(Some(catalog), names)
+          if CatalogV2Util.isSessionCatalog(catalog) &&
+             CatalogV2Util.isSessionCatalog(currentCatalog) =>
+          namesToTableIdentifier(names)
+        case _ => None
+      }
+    }
+  }
+
+  object AsFunctionIdentifier {
+    def unapply(parts: Seq[String]): Option[FunctionIdentifier] = {
+      def namesToFunctionIdentifier(names: Seq[String]): Option[FunctionIdentifier] = names match {
+        case Seq(name) => Some(FunctionIdentifier(name))
+        case Seq(database, name) => Some(FunctionIdentifier(name, Some(database)))
+        case _ => None
+      }
+      parts match {
+        case Seq(name)
+          if catalogManager.v1SessionCatalog.isRegisteredFunction(FunctionIdentifier(name)) =>
+          Some(FunctionIdentifier(name))
+        case CatalogAndMultipartIdentifier(None, names)
           if CatalogV2Util.isSessionCatalog(currentCatalog) =>
-        names match {
-          case Seq(name) =>
-            Some(TableIdentifier(name))
-          case Seq(database, name) =>
-            Some(TableIdentifier(name, Some(database)))
-          case _ =>
-            None
+          namesToFunctionIdentifier(names)
+        case CatalogAndMultipartIdentifier(Some(catalog), names)
+          if CatalogV2Util.isSessionCatalog(catalog) =>
+          namesToFunctionIdentifier(names)
+        case _ => None
+      }
+    }
+  }
+
+  def parseSessionCatalogFunctionIdentifier(nameParts: Seq[String]): FunctionIdentifier = {
+    if (nameParts.length == 1 && catalogManager.v1SessionCatalog.isTempFunction(nameParts.head)) {
+      return FunctionIdentifier(nameParts.head)
+    }
+
+    nameParts match {
+      case SessionCatalogAndIdentifier(_, ident) =>
+        if (nameParts.length == 1) {
+          // If there is only one name part, it means the current catalog is the session catalog.
+          // Here we don't fill the default database, to keep the error message unchanged for
+          // v1 commands.
+          FunctionIdentifier(nameParts.head, None)
+        } else {
+          ident.namespace match {
+            case Array(db) => FunctionIdentifier(ident.name, Some(db))
+            case _ =>
+              throw QueryCompilationErrors.unsupportedFunctionNameError(ident.toString)
+          }
         }
-      case _ =>
-        None
+
+      case _ => throw QueryCompilationErrors.functionUnsupportedInV2CatalogError()
     }
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/V1Table.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/V1Table.scala
index 70fc9689e6087..5c0d0ceee6945 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/V1Table.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/V1Table.scala
@@ -24,7 +24,7 @@ import scala.collection.mutable
 
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.catalog.CatalogTable
-import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.quoteIfNeeded
+import org.apache.spark.sql.catalyst.util.quoteIfNeeded
 import org.apache.spark.sql.connector.expressions.{LogicalExpressions, Transform}
 import org.apache.spark.sql.types.StructType
 
@@ -80,3 +80,11 @@ private[sql] case class V1Table(v1Table: CatalogTable) extends Table {
 
   override def toString: String = s"V1Table($name)"
 }
+
+/**
+ * A V2 table with V1 fallback support. This is used to fallback to V1 table when the V2 one
+ * doesn't implement specific capabilities but V1 already has.
+ */
+private[sql] trait V2TableWithV1Fallback extends Table {
+  def v1Table: CatalogTable
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/distributions/distributions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/distributions/distributions.scala
new file mode 100644
index 0000000000000..599f82b4dc528
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/distributions/distributions.scala
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.distributions
+
+import org.apache.spark.sql.connector.expressions.{Expression, SortOrder}
+
+private[sql] object LogicalDistributions {
+
+  def unspecified(): UnspecifiedDistribution = {
+    UnspecifiedDistributionImpl
+  }
+
+  def clustered(clustering: Array[Expression]): ClusteredDistribution = {
+    ClusteredDistributionImpl(clustering)
+  }
+
+  def ordered(ordering: Array[SortOrder]): OrderedDistribution = {
+    OrderedDistributionImpl(ordering)
+  }
+}
+
+private[sql] object UnspecifiedDistributionImpl extends UnspecifiedDistribution {
+  override def toString: String = "UnspecifiedDistribution"
+}
+
+private[sql] final case class ClusteredDistributionImpl(
+    clusteringExprs: Seq[Expression]) extends ClusteredDistribution {
+
+  override def clustering: Array[Expression] = clusteringExprs.toArray
+
+  override def toString: String = {
+    s"ClusteredDistribution(${clusteringExprs.map(_.describe).mkString(", ")})"
+  }
+}
+
+private[sql] final case class OrderedDistributionImpl(
+    orderingExprs: Seq[SortOrder]) extends OrderedDistribution {
+
+  override def ordering: Array[SortOrder] = orderingExprs.toArray
+
+  override def toString: String = {
+    s"OrderedDistribution(${orderingExprs.map(_.describe).mkString(", ")})"
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/expressions/expressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/expressions/expressions.scala
index ea26847ac3cef..2863d94d198b2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/expressions/expressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/expressions/expressions.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.connector.expressions
 
 import org.apache.spark.sql.catalyst
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{DataType, IntegerType, StringType}
 
 /**
@@ -29,10 +28,6 @@ import org.apache.spark.sql.types.{DataType, IntegerType, StringType}
  * logical and internal expressions are used.
  */
 private[sql] object LogicalExpressions {
-  // a generic parser that is only used for parsing multi-part field names.
-  // because this is only used for field names, the SQL conf passed in does not matter.
-  private lazy val parser = new CatalystSqlParser(SQLConf.get)
-
   def literal[T](value: T): LiteralValue[T] = {
     val internalLit = catalyst.expressions.Literal(value)
     literal(value, internalLit.dataType)
@@ -41,7 +36,7 @@ private[sql] object LogicalExpressions {
   def literal[T](value: T, dataType: DataType): LiteralValue[T] = LiteralValue(value, dataType)
 
   def parseReference(name: String): NamedReference =
-    FieldReference(parser.parseMultipartIdentifier(name))
+    FieldReference(CatalystSqlParser.parseMultipartIdentifier(name))
 
   def reference(nameParts: Seq[String]): NamedReference = FieldReference(nameParts)
 
@@ -59,6 +54,13 @@ private[sql] object LogicalExpressions {
   def days(reference: NamedReference): DaysTransform = DaysTransform(reference)
 
   def hours(reference: NamedReference): HoursTransform = HoursTransform(reference)
+
+  def sort(
+      reference: Expression,
+      direction: SortDirection,
+      nullOrdering: NullOrdering): SortOrder = {
+    SortValue(reference, direction, nullOrdering)
+  }
 }
 
 /**
@@ -115,6 +117,18 @@ private[sql] final case class BucketTransform(
 }
 
 private[sql] object BucketTransform {
+  def unapply(expr: Expression): Option[(Int, FieldReference)] = expr match {
+    case transform: Transform =>
+      transform match {
+        case BucketTransform(n, FieldReference(parts)) =>
+          Some((n, FieldReference(parts)))
+        case _ =>
+          None
+      }
+    case _ =>
+      None
+  }
+
   def unapply(transform: Transform): Option[(Int, NamedReference)] = transform match {
     case NamedTransform("bucket", Seq(
         Lit(value: Int, IntegerType),
@@ -175,6 +189,18 @@ private[sql] final case class IdentityTransform(
 }
 
 private[sql] object IdentityTransform {
+  def unapply(expr: Expression): Option[FieldReference] = expr match {
+    case transform: Transform =>
+      transform match {
+        case IdentityTransform(ref) =>
+          Some(ref)
+        case _ =>
+          None
+      }
+    case _ =>
+      None
+  }
+
   def unapply(transform: Transform): Option[FieldReference] = transform match {
     case NamedTransform("identity", Seq(Ref(parts))) =>
       Some(FieldReference(parts))
@@ -190,6 +216,18 @@ private[sql] final case class YearsTransform(
 }
 
 private[sql] object YearsTransform {
+  def unapply(expr: Expression): Option[FieldReference] = expr match {
+    case transform: Transform =>
+      transform match {
+        case YearsTransform(ref) =>
+          Some(ref)
+        case _ =>
+          None
+      }
+    case _ =>
+      None
+  }
+
   def unapply(transform: Transform): Option[FieldReference] = transform match {
     case NamedTransform("years", Seq(Ref(parts))) =>
       Some(FieldReference(parts))
@@ -205,6 +243,18 @@ private[sql] final case class MonthsTransform(
 }
 
 private[sql] object MonthsTransform {
+  def unapply(expr: Expression): Option[FieldReference] = expr match {
+    case transform: Transform =>
+      transform match {
+        case MonthsTransform(ref) =>
+          Some(ref)
+        case _ =>
+          None
+      }
+    case _ =>
+      None
+  }
+
   def unapply(transform: Transform): Option[FieldReference] = transform match {
     case NamedTransform("months", Seq(Ref(parts))) =>
       Some(FieldReference(parts))
@@ -220,6 +270,18 @@ private[sql] final case class DaysTransform(
 }
 
 private[sql] object DaysTransform {
+  def unapply(expr: Expression): Option[FieldReference] = expr match {
+    case transform: Transform =>
+      transform match {
+        case DaysTransform(ref) =>
+          Some(ref)
+        case _ =>
+          None
+      }
+    case _ =>
+      None
+  }
+
   def unapply(transform: Transform): Option[FieldReference] = transform match {
     case NamedTransform("days", Seq(Ref(parts))) =>
       Some(FieldReference(parts))
@@ -235,6 +297,18 @@ private[sql] final case class HoursTransform(
 }
 
 private[sql] object HoursTransform {
+  def unapply(expr: Expression): Option[FieldReference] = expr match {
+    case transform: Transform =>
+      transform match {
+        case HoursTransform(ref) =>
+          Some(ref)
+        case _ =>
+          None
+      }
+    case _ =>
+      None
+  }
+
   def unapply(transform: Transform): Option[FieldReference] = transform match {
     case NamedTransform("hours", Seq(Ref(parts))) =>
       Some(FieldReference(parts))
@@ -266,3 +340,20 @@ private[sql] object FieldReference {
     LogicalExpressions.parseReference(column)
   }
 }
+
+private[sql] final case class SortValue(
+    expression: Expression,
+    direction: SortDirection,
+    nullOrdering: NullOrdering) extends SortOrder {
+
+  override def describe(): String = s"$expression $direction $nullOrdering"
+}
+
+private[sql] object SortValue {
+  def unapply(expr: Expression): Option[(Expression, SortDirection, NullOrdering)] = expr match {
+    case sort: SortOrder =>
+      Some((sort.expression, sort.direction, sort.nullOrdering))
+    case _ =>
+      None
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
new file mode 100644
index 0000000000000..9a87917d0e468
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
@@ -0,0 +1,1448 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.errors
+
+import org.apache.hadoop.fs.Path
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.{FunctionIdentifier, QualifiedTableName, TableIdentifier}
+import org.apache.spark.sql.catalyst.analysis.{CannotReplaceMissingTableException, NamespaceAlreadyExistsException, NoSuchNamespaceException, NoSuchTableException, ResolvedNamespace, ResolvedTable, ResolvedView, TableAlreadyExistsException}
+import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTable, InvalidUDFClassException}
+import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference, CreateMap, Expression, GroupingID, NamedExpression, SpecifiedWindowFrame, WindowFrame, WindowFunction, WindowSpecDefinition}
+import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoStatement, LogicalPlan, SerdeInfo}
+import org.apache.spark.sql.catalyst.trees.TreeNode
+import org.apache.spark.sql.catalyst.util.{toPrettySQL, FailFastMode, ParseMode, PermissiveMode}
+import org.apache.spark.sql.connector.catalog.{CatalogPlugin, Identifier, NamespaceChange, Table, TableCapability, TableChange, V1Table}
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+import org.apache.spark.sql.connector.expressions.{NamedReference, Transform}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.sources.Filter
+import org.apache.spark.sql.streaming.OutputMode
+import org.apache.spark.sql.types.{AbstractDataType, DataType, NullType, StructField, StructType}
+
+/**
+ * Object for grouping error messages from exceptions thrown during query compilation.
+ * As commands are executed eagerly, this also includes errors thrown during the execution of
+ * commands, which users can see immediately.
+ */
+private[spark] object QueryCompilationErrors {
+
+  def groupingIDMismatchError(groupingID: GroupingID, groupByExprs: Seq[Expression]): Throwable = {
+    new AnalysisException(
+      s"Columns of grouping_id (${groupingID.groupByExprs.mkString(",")}) " +
+        s"does not match grouping columns (${groupByExprs.mkString(",")})")
+  }
+
+  def groupingColInvalidError(groupingCol: Expression, groupByExprs: Seq[Expression]): Throwable = {
+    new AnalysisException(
+      s"Column of grouping ($groupingCol) can't be found " +
+        s"in grouping columns ${groupByExprs.mkString(",")}")
+  }
+
+  def groupingSizeTooLargeError(sizeLimit: Int): Throwable = {
+    new AnalysisException(
+      s"Grouping sets size cannot be greater than $sizeLimit")
+  }
+
+  def unorderablePivotColError(pivotCol: Expression): Throwable = {
+    new AnalysisException(
+      s"Invalid pivot column '$pivotCol'. Pivot columns must be comparable."
+    )
+  }
+
+  def nonLiteralPivotValError(pivotVal: Expression): Throwable = {
+    new AnalysisException(
+      s"Literal expressions required for pivot values, found '$pivotVal'")
+  }
+
+  def pivotValDataTypeMismatchError(pivotVal: Expression, pivotCol: Expression): Throwable = {
+    new AnalysisException(
+      s"Invalid pivot value '$pivotVal': " +
+        s"value data type ${pivotVal.dataType.simpleString} does not match " +
+        s"pivot column data type ${pivotCol.dataType.catalogString}")
+  }
+
+  def unsupportedIfNotExistsError(tableName: String): Throwable = {
+    new AnalysisException(
+      s"Cannot write, IF NOT EXISTS is not supported for table: $tableName")
+  }
+
+  def nonPartitionColError(partitionName: String): Throwable = {
+    new AnalysisException(
+      s"PARTITION clause cannot contain a non-partition column name: $partitionName")
+  }
+
+  def addStaticValToUnknownColError(staticName: String): Throwable = {
+    new AnalysisException(
+      s"Cannot add static value for unknown column: $staticName")
+  }
+
+  def unknownStaticPartitionColError(name: String): Throwable = {
+    new AnalysisException(s"Unknown static partition column: $name")
+  }
+
+  def nestedGeneratorError(trimmedNestedGenerator: Expression): Throwable = {
+    new AnalysisException(
+      "Generators are not supported when it's nested in " +
+        "expressions, but got: " + toPrettySQL(trimmedNestedGenerator))
+  }
+
+  def moreThanOneGeneratorError(generators: Seq[Expression], clause: String): Throwable = {
+    new AnalysisException(
+      s"Only one generator allowed per $clause clause but found " +
+        generators.size + ": " + generators.map(toPrettySQL).mkString(", "))
+  }
+
+  def generatorOutsideSelectError(plan: LogicalPlan): Throwable = {
+    new AnalysisException(
+      "Generators are not supported outside the SELECT clause, but " +
+        "got: " + plan.simpleString(SQLConf.get.maxToStringFields))
+  }
+
+  def legacyStoreAssignmentPolicyError(): Throwable = {
+    val configKey = SQLConf.STORE_ASSIGNMENT_POLICY.key
+    new AnalysisException(
+      "LEGACY store assignment policy is disallowed in Spark data source V2. " +
+        s"Please set the configuration $configKey to other values.")
+  }
+
+  def unresolvedUsingColForJoinError(
+      colName: String, plan: LogicalPlan, side: String): Throwable = {
+    new AnalysisException(
+      s"USING column `$colName` cannot be resolved on the $side " +
+        s"side of the join. The $side-side columns: [${plan.output.map(_.name).mkString(", ")}]")
+  }
+
+  def dataTypeMismatchForDeserializerError(
+      dataType: DataType, desiredType: String): Throwable = {
+    val quantifier = if (desiredType.equals("array")) "an" else "a"
+    new AnalysisException(
+      s"need $quantifier $desiredType field but got " + dataType.catalogString)
+  }
+
+  def fieldNumberMismatchForDeserializerError(
+      schema: StructType, maxOrdinal: Int): Throwable = {
+    new AnalysisException(
+      s"Try to map ${schema.catalogString} to Tuple${maxOrdinal + 1}, " +
+        "but failed as the number of fields does not line up.")
+  }
+
+  def upCastFailureError(
+      fromStr: String, from: Expression, to: DataType, walkedTypePath: Seq[String]): Throwable = {
+    new AnalysisException(
+      s"Cannot up cast $fromStr from " +
+        s"${from.dataType.catalogString} to ${to.catalogString}.\n" +
+        s"The type path of the target object is:\n" + walkedTypePath.mkString("", "\n", "\n") +
+        "You can either add an explicit cast to the input data or choose a higher precision " +
+        "type of the field in the target object")
+  }
+
+  def unsupportedAbstractDataTypeForUpCastError(gotType: AbstractDataType): Throwable = {
+    new AnalysisException(
+      s"UpCast only support DecimalType as AbstractDataType yet, but got: $gotType")
+  }
+
+  def outerScopeFailureForNewInstanceError(className: String): Throwable = {
+    new AnalysisException(
+      s"Unable to generate an encoder for inner class `$className` without " +
+        "access to the scope that this class was defined in.\n" +
+        "Try moving this class out of its parent class.")
+  }
+
+  def referenceColNotFoundForAlterTableChangesError(
+      after: TableChange.After, parentName: String): Throwable = {
+    new AnalysisException(
+      s"Couldn't find the reference column for $after at $parentName")
+  }
+
+  def windowSpecificationNotDefinedError(windowName: String): Throwable = {
+    new AnalysisException(s"Window specification $windowName is not defined in the WINDOW clause.")
+  }
+
+  def selectExprNotInGroupByError(expr: Expression, groupByAliases: Seq[Alias]): Throwable = {
+    new AnalysisException(s"$expr doesn't show up in the GROUP BY list $groupByAliases")
+  }
+
+  def groupingMustWithGroupingSetsOrCubeOrRollupError(): Throwable = {
+    new AnalysisException("grouping()/grouping_id() can only be used with GroupingSets/Cube/Rollup")
+  }
+
+  def pandasUDFAggregateNotSupportedInPivotError(): Throwable = {
+    new AnalysisException("Pandas UDF aggregate expressions are currently not supported in pivot.")
+  }
+
+  def aggregateExpressionRequiredForPivotError(sql: String): Throwable = {
+    new AnalysisException(s"Aggregate expression required for pivot, but '$sql' " +
+      "did not appear in any aggregate function.")
+  }
+
+  def writeIntoTempViewNotAllowedError(quoted: String): Throwable = {
+    new AnalysisException("Cannot write into temp view " +
+      s"$quoted as it's not a data source v2 relation.")
+  }
+
+  def expectTableOrPermanentViewNotTempViewError(
+      quoted: String, cmd: String, t: TreeNode[_]): Throwable = {
+    new AnalysisException(s"$quoted is a temp view. '$cmd' expects a table or permanent view.",
+      t.origin.line, t.origin.startPosition)
+  }
+
+  def readNonStreamingTempViewError(quoted: String): Throwable = {
+    new AnalysisException(s"$quoted is not a temp view of streaming " +
+      "logical plan, please use batch API such as `DataFrameReader.table` to read it.")
+  }
+
+  def viewDepthExceedsMaxResolutionDepthError(
+      identifier: TableIdentifier, maxNestedViewDepth: Int, t: TreeNode[_]): Throwable = {
+    new AnalysisException(s"The depth of view $identifier exceeds the maximum " +
+      s"view resolution depth ($maxNestedViewDepth). Analysis is aborted to " +
+      s"avoid errors. Increase the value of ${SQLConf.MAX_NESTED_VIEW_DEPTH.key} to work " +
+      "around this.", t.origin.line, t.origin.startPosition)
+  }
+
+  def insertIntoViewNotAllowedError(identifier: TableIdentifier, t: TreeNode[_]): Throwable = {
+    new AnalysisException(s"Inserting into a view is not allowed. View: $identifier.",
+      t.origin.line, t.origin.startPosition)
+  }
+
+  def writeIntoViewNotAllowedError(identifier: TableIdentifier, t: TreeNode[_]): Throwable = {
+    new AnalysisException(s"Writing into a view is not allowed. View: $identifier.",
+      t.origin.line, t.origin.startPosition)
+  }
+
+  def writeIntoV1TableNotAllowedError(identifier: TableIdentifier, t: TreeNode[_]): Throwable = {
+    new AnalysisException(s"Cannot write into v1 table: $identifier.",
+      t.origin.line, t.origin.startPosition)
+  }
+
+  def expectTableNotViewError(
+      v: ResolvedView, cmd: String, mismatchHint: Option[String], t: TreeNode[_]): Throwable = {
+    val viewStr = if (v.isTemp) "temp view" else "view"
+    val hintStr = mismatchHint.map(" " + _).getOrElse("")
+    new AnalysisException(s"${v.identifier.quoted} is a $viewStr. '$cmd' expects a table.$hintStr",
+      t.origin.line, t.origin.startPosition)
+  }
+
+  def expectViewNotTableError(
+      v: ResolvedTable, cmd: String, mismatchHint: Option[String], t: TreeNode[_]): Throwable = {
+    val hintStr = mismatchHint.map(" " + _).getOrElse("")
+    new AnalysisException(s"${v.identifier.quoted} is a table. '$cmd' expects a view.$hintStr",
+      t.origin.line, t.origin.startPosition)
+  }
+
+  def permanentViewNotSupportedByStreamingReadingAPIError(quoted: String): Throwable = {
+    new AnalysisException(s"$quoted is a permanent view, which is not supported by " +
+      "streaming reading API such as `DataStreamReader.table` yet.")
+  }
+
+  def starNotAllowedWhenGroupByOrdinalPositionUsedError(): Throwable = {
+    new AnalysisException(
+      "Star (*) is not allowed in select list when GROUP BY ordinal position is used")
+  }
+
+  def invalidStarUsageError(prettyName: String): Throwable = {
+    new AnalysisException(s"Invalid usage of '*' in $prettyName")
+  }
+
+  def singleTableStarInCountNotAllowedError(targetString: String): Throwable = {
+    new AnalysisException(s"count($targetString.*) is not allowed. " +
+      "Please use count(*) or expand the columns manually, e.g. count(col1, col2)")
+  }
+
+  def orderByPositionRangeError(index: Int, size: Int, t: TreeNode[_]): Throwable = {
+    new AnalysisException(s"ORDER BY position $index is not in select list " +
+      s"(valid range is [1, $size])", t.origin.line, t.origin.startPosition)
+  }
+
+  def groupByPositionRefersToAggregateFunctionError(
+      index: Int,
+      expr: Expression): Throwable = {
+    new AnalysisException(s"GROUP BY $index refers to an expression that is or contains " +
+      "an aggregate function. Aggregate functions are not allowed in GROUP BY, " +
+      s"but got ${expr.sql}")
+  }
+
+  def groupByPositionRangeError(index: Int, size: Int): Throwable = {
+    new AnalysisException(s"GROUP BY position $index is not in select list " +
+      s"(valid range is [1, $size])")
+  }
+
+  def generatorNotExpectedError(name: FunctionIdentifier, classCanonicalName: String): Throwable = {
+    new AnalysisException(s"$name is expected to be a generator. However, " +
+      s"its class is $classCanonicalName, which is not a generator.")
+  }
+
+  def functionWithUnsupportedSyntaxError(prettyName: String, syntax: String): Throwable = {
+    new AnalysisException(s"Function $prettyName does not support $syntax")
+  }
+
+  def nonDeterministicFilterInAggregateError(): Throwable = {
+    new AnalysisException("FILTER expression is non-deterministic, " +
+      "it cannot be used in aggregate functions")
+  }
+
+  def aliasNumberNotMatchColumnNumberError(
+      columnSize: Int, outputSize: Int, t: TreeNode[_]): Throwable = {
+    new AnalysisException("Number of column aliases does not match number of columns. " +
+      s"Number of column aliases: $columnSize; " +
+      s"number of columns: $outputSize.", t.origin.line, t.origin.startPosition)
+  }
+
+  def aliasesNumberNotMatchUDTFOutputError(
+      aliasesSize: Int, aliasesNames: String): Throwable = {
+    new AnalysisException("The number of aliases supplied in the AS clause does not " +
+      s"match the number of columns output by the UDTF expected $aliasesSize " +
+      s"aliases but got $aliasesNames ")
+  }
+
+  def windowAggregateFunctionWithFilterNotSupportedError(): Throwable = {
+    new AnalysisException("window aggregate function with filter predicate is not supported yet.")
+  }
+
+  def windowFunctionInsideAggregateFunctionNotAllowedError(): Throwable = {
+    new AnalysisException("It is not allowed to use a window function inside an aggregate " +
+      "function. Please use the inner window function in a sub-query.")
+  }
+
+  def expressionWithoutWindowExpressionError(expr: NamedExpression): Throwable = {
+    new AnalysisException(s"$expr does not have any WindowExpression.")
+  }
+
+  def expressionWithMultiWindowExpressionsError(
+      expr: NamedExpression, distinctWindowSpec: Seq[WindowSpecDefinition]): Throwable = {
+    new AnalysisException(s"$expr has multiple Window Specifications ($distinctWindowSpec)." +
+      "Please file a bug report with this error message, stack trace, and the query.")
+  }
+
+  def windowFunctionNotAllowedError(clauseName: String): Throwable = {
+    new AnalysisException(s"It is not allowed to use window functions inside $clauseName clause")
+  }
+
+  def cannotSpecifyWindowFrameError(prettyName: String): Throwable = {
+    new AnalysisException(s"Cannot specify window frame for $prettyName function")
+  }
+
+  def windowFrameNotMatchRequiredFrameError(
+      f: SpecifiedWindowFrame, required: WindowFrame): Throwable = {
+    new AnalysisException(s"Window Frame $f must match the required frame $required")
+  }
+
+  def windowFunctionWithWindowFrameNotOrderedError(wf: WindowFunction): Throwable = {
+    new AnalysisException(s"Window function $wf requires window to be ordered, please add " +
+      s"ORDER BY clause. For example SELECT $wf(value_expr) OVER (PARTITION BY window_partition " +
+      "ORDER BY window_ordering) from table")
+  }
+
+  def cannotResolveUserSpecifiedColumnsError(col: String, t: TreeNode[_]): Throwable = {
+    new AnalysisException(s"Cannot resolve column name $col", t.origin.line, t.origin.startPosition)
+  }
+
+  def writeTableWithMismatchedColumnsError(
+      columnSize: Int, outputSize: Int, t: TreeNode[_]): Throwable = {
+    new AnalysisException("Cannot write to table due to mismatched user specified column " +
+      s"size($columnSize) and data column size($outputSize)", t.origin.line, t.origin.startPosition)
+  }
+
+  def multiTimeWindowExpressionsNotSupportedError(t: TreeNode[_]): Throwable = {
+    new AnalysisException("Multiple time window expressions would result in a cartesian product " +
+      "of rows, therefore they are currently not supported.", t.origin.line, t.origin.startPosition)
+  }
+
+  def viewOutputNumberMismatchQueryColumnNamesError(
+      output: Seq[Attribute], queryColumnNames: Seq[String]): Throwable = {
+    new AnalysisException(
+      s"The view output ${output.mkString("[", ",", "]")} doesn't have the same" +
+        "number of columns with the query column names " +
+        s"${queryColumnNames.mkString("[", ",", "]")}")
+  }
+
+  def attributeNotFoundError(colName: String, child: LogicalPlan): Throwable = {
+    new AnalysisException(
+      s"Attribute with name '$colName' is not found in " +
+        s"'${child.output.map(_.name).mkString("(", ",", ")")}'")
+  }
+
+  def cannotUpCastAsAttributeError(
+      fromAttr: Attribute, toAttr: Attribute): Throwable = {
+    new AnalysisException(s"Cannot up cast ${fromAttr.sql} from " +
+      s"${fromAttr.dataType.catalogString} to ${toAttr.dataType.catalogString} " +
+      "as it may truncate")
+  }
+
+  def functionUndefinedError(name: FunctionIdentifier): Throwable = {
+    new AnalysisException(s"undefined function $name")
+  }
+
+  def invalidFunctionArgumentsError(
+      name: String, expectedInfo: String, actualNumber: Int): Throwable = {
+    new AnalysisException(s"Invalid number of arguments for function $name. " +
+      s"Expected: $expectedInfo; Found: $actualNumber")
+  }
+
+  def invalidFunctionArgumentNumberError(
+      validParametersCount: Seq[Int], name: String, params: Seq[Class[Expression]]): Throwable = {
+    if (validParametersCount.length == 0) {
+      new AnalysisException(s"Invalid arguments for function $name")
+    } else {
+      val expectedNumberOfParameters = if (validParametersCount.length == 1) {
+        validParametersCount.head.toString
+      } else {
+        validParametersCount.init.mkString("one of ", ", ", " and ") +
+          validParametersCount.last
+      }
+      invalidFunctionArgumentsError(name, expectedNumberOfParameters, params.length)
+    }
+  }
+
+  def functionAcceptsOnlyOneArgumentError(name: String): Throwable = {
+    new AnalysisException(s"Function $name accepts only one argument")
+  }
+
+  def alterV2TableSetLocationWithPartitionNotSupportedError(): Throwable = {
+    new AnalysisException("ALTER TABLE SET LOCATION does not support partition for v2 tables.")
+  }
+
+  def joinStrategyHintParameterNotSupportedError(unsupported: Any): Throwable = {
+    new AnalysisException("Join strategy hint parameter " +
+      s"should be an identifier or string but was $unsupported (${unsupported.getClass}")
+  }
+
+  def invalidHintParameterError(
+      hintName: String, invalidParams: Seq[Any]): Throwable = {
+    new AnalysisException(s"$hintName Hint parameter should include columns, but " +
+      s"${invalidParams.mkString(", ")} found")
+  }
+
+  def invalidCoalesceHintParameterError(hintName: String): Throwable = {
+    new AnalysisException(s"$hintName Hint expects a partition number as a parameter")
+  }
+
+  def attributeNameSyntaxError(name: String): Throwable = {
+    new AnalysisException(s"syntax error in attribute name: $name")
+  }
+
+  def starExpandDataTypeNotSupportedError(attributes: Seq[String]): Throwable = {
+    new AnalysisException(s"Can only star expand struct data types. Attribute: `$attributes`")
+  }
+
+  def cannotResolveStarExpandGivenInputColumnsError(
+      targetString: String, columns: String): Throwable = {
+    new AnalysisException(s"cannot resolve '$targetString.*' given input columns '$columns'")
+  }
+
+  def addColumnWithV1TableCannotSpecifyNotNullError(): Throwable = {
+    new AnalysisException("ADD COLUMN with v1 tables cannot specify NOT NULL.")
+  }
+
+  def replaceColumnsOnlySupportedWithV2TableError(): Throwable = {
+    new AnalysisException("REPLACE COLUMNS is only supported with v2 tables.")
+  }
+
+  def alterQualifiedColumnOnlySupportedWithV2TableError(): Throwable = {
+    new AnalysisException("ALTER COLUMN with qualified column is only supported with v2 tables.")
+  }
+
+  def alterColumnWithV1TableCannotSpecifyNotNullError(): Throwable = {
+    new AnalysisException("ALTER COLUMN with v1 tables cannot specify NOT NULL.")
+  }
+
+  def alterOnlySupportedWithV2TableError(): Throwable = {
+    new AnalysisException("ALTER COLUMN ... FIRST | ALTER is only supported with v2 tables.")
+  }
+
+  def alterColumnCannotFindColumnInV1TableError(colName: String, v1Table: V1Table): Throwable = {
+    new AnalysisException(
+      s"ALTER COLUMN cannot find column $colName in v1 table. " +
+        s"Available: ${v1Table.schema.fieldNames.mkString(", ")}")
+  }
+
+  def renameColumnOnlySupportedWithV2TableError(): Throwable = {
+    new AnalysisException("RENAME COLUMN is only supported with v2 tables.")
+  }
+
+  def dropColumnOnlySupportedWithV2TableError(): Throwable = {
+    new AnalysisException("DROP COLUMN is only supported with v2 tables.")
+  }
+
+  def invalidDatabaseNameError(quoted: String): Throwable = {
+    new AnalysisException(s"The database name is not valid: $quoted")
+  }
+
+  def replaceTableOnlySupportedWithV2TableError(): Throwable = {
+    new AnalysisException("REPLACE TABLE is only supported with v2 tables.")
+  }
+
+  def replaceTableAsSelectOnlySupportedWithV2TableError(): Throwable = {
+    new AnalysisException("REPLACE TABLE AS SELECT is only supported with v2 tables.")
+  }
+
+  def cannotDropViewWithDropTableError(): Throwable = {
+    new AnalysisException("Cannot drop a view with DROP TABLE. Please use DROP VIEW instead")
+  }
+
+  def showColumnsWithConflictDatabasesError(
+      db: Seq[String], v1TableName: TableIdentifier): Throwable = {
+    new AnalysisException("SHOW COLUMNS with conflicting databases: " +
+        s"'${db.head}' != '${v1TableName.database.get}'")
+  }
+
+  def externalCatalogNotSupportShowViewsError(resolved: ResolvedNamespace): Throwable = {
+    new AnalysisException(s"Catalog ${resolved.catalog.name} doesn't support " +
+      "SHOW VIEWS, only SessionCatalog supports this command.")
+  }
+
+  def unsupportedFunctionNameError(quoted: String): Throwable = {
+    new AnalysisException(s"Unsupported function name '$quoted'")
+  }
+
+  def sqlOnlySupportedWithV1TablesError(sql: String): Throwable = {
+    new AnalysisException(s"$sql is only supported with v1 tables.")
+  }
+
+  def cannotCreateTableWithBothProviderAndSerdeError(
+      provider: Option[String], maybeSerdeInfo: Option[SerdeInfo]): Throwable = {
+    new AnalysisException(
+      s"Cannot create table with both USING $provider and ${maybeSerdeInfo.get.describe}")
+  }
+
+  def invalidFileFormatForStoredAsError(serdeInfo: SerdeInfo): Throwable = {
+    new AnalysisException(
+      s"STORED AS with file format '${serdeInfo.storedAs.get}' is invalid.")
+  }
+
+  def commandNotSupportNestedColumnError(command: String, quoted: String): Throwable = {
+    new AnalysisException(s"$command does not support nested column: $quoted")
+  }
+
+  def columnDoesNotExistError(colName: String): Throwable = {
+    new AnalysisException(s"Column $colName does not exist")
+  }
+
+  def renameTempViewToExistingViewError(oldName: String, newName: String): Throwable = {
+    new AnalysisException(
+      s"rename temporary view from '$oldName' to '$newName': destination view already exists")
+  }
+
+  def databaseNotEmptyError(db: String, details: String): Throwable = {
+    new AnalysisException(s"Database $db is not empty. One or more $details exist.")
+  }
+
+  def invalidNameForTableOrDatabaseError(name: String): Throwable = {
+    new AnalysisException(s"`$name` is not a valid name for tables/databases. " +
+      "Valid names only contain alphabet characters, numbers and _.")
+  }
+
+  def cannotCreateDatabaseWithSameNameAsPreservedDatabaseError(database: String): Throwable = {
+    new AnalysisException(s"$database is a system preserved database, " +
+      "you cannot create a database with this name.")
+  }
+
+  def cannotDropDefaultDatabaseError(): Throwable = {
+    new AnalysisException("Can not drop default database")
+  }
+
+  def cannotUsePreservedDatabaseAsCurrentDatabaseError(database: String): Throwable = {
+    new AnalysisException(s"$database is a system preserved database, you cannot use it as " +
+      "current database. To access global temporary views, you should use qualified name with " +
+      s"the GLOBAL_TEMP_DATABASE, e.g. SELECT * FROM $database.viewName.")
+  }
+
+  def createExternalTableWithoutLocationError(): Throwable = {
+    new AnalysisException("CREATE EXTERNAL TABLE must be accompanied by LOCATION")
+  }
+
+  def cannotOperateManagedTableWithExistingLocationError(
+      methodName: String, tableIdentifier: TableIdentifier, tableLocation: Path): Throwable = {
+    new AnalysisException(s"Can not $methodName the managed table('$tableIdentifier')" +
+      s". The associated location('${tableLocation.toString}') already exists.")
+  }
+
+  def dropNonExistentColumnsNotSupportedError(
+      nonExistentColumnNames: Seq[String]): Throwable = {
+    new AnalysisException(
+      s"""
+         |Some existing schema fields (${nonExistentColumnNames.mkString("[", ",", "]")}) are
+         |not present in the new schema. We don't support dropping columns yet.
+         """.stripMargin)
+  }
+
+  def cannotRetrieveTableOrViewNotInSameDatabaseError(
+      qualifiedTableNames: Seq[QualifiedTableName]): Throwable = {
+    new AnalysisException("Only the tables/views belong to the same database can be retrieved. " +
+      s"Querying tables/views are $qualifiedTableNames")
+  }
+
+  def renameTableSourceAndDestinationMismatchError(db: String, newDb: String): Throwable = {
+    new AnalysisException(
+      s"RENAME TABLE source and destination databases do not match: '$db' != '$newDb'")
+  }
+
+  def cannotRenameTempViewWithDatabaseSpecifiedError(
+      oldName: TableIdentifier, newName: TableIdentifier): Throwable = {
+    new AnalysisException(s"RENAME TEMPORARY VIEW from '$oldName' to '$newName': cannot " +
+      s"specify database name '${newName.database.get}' in the destination table")
+  }
+
+  def cannotRenameTempViewToExistingTableError(
+      oldName: TableIdentifier, newName: TableIdentifier): Throwable = {
+    new AnalysisException(s"RENAME TEMPORARY VIEW from '$oldName' to '$newName': " +
+      "destination table already exists")
+  }
+
+  def invalidPartitionSpecError(details: String): Throwable = {
+    new AnalysisException(s"Partition spec is invalid. $details")
+  }
+
+  def functionAlreadyExistsError(func: FunctionIdentifier): Throwable = {
+    new AnalysisException(s"Function $func already exists")
+  }
+
+  def cannotLoadClassWhenRegisteringFunctionError(
+      className: String, func: FunctionIdentifier): Throwable = {
+    new AnalysisException(s"Can not load class '$className' when registering " +
+      s"the function '$func', please make sure it is on the classpath")
+  }
+
+  def resourceTypeNotSupportedError(resourceType: String): Throwable = {
+    new AnalysisException(s"Resource Type '$resourceType' is not supported.")
+  }
+
+  def tableNotSpecifyDatabaseError(identifier: TableIdentifier): Throwable = {
+    new AnalysisException(s"table $identifier did not specify database")
+  }
+
+  def tableNotSpecifyLocationUriError(identifier: TableIdentifier): Throwable = {
+    new AnalysisException(s"table $identifier did not specify locationUri")
+  }
+
+  def partitionNotSpecifyLocationUriError(specString: String): Throwable = {
+    new AnalysisException(s"Partition [$specString] did not specify locationUri")
+  }
+
+  def invalidBucketNumberError(bucketingMaxBuckets: Int, numBuckets: Int): Throwable = {
+    new AnalysisException(
+      s"Number of buckets should be greater than 0 but less than or equal to " +
+        s"bucketing.maxBuckets (`$bucketingMaxBuckets`). Got `$numBuckets`")
+  }
+
+  def corruptedTableNameContextInCatalogError(numParts: Int, index: Int): Throwable = {
+    new AnalysisException("Corrupted table name context in catalog: " +
+      s"$numParts parts expected, but part $index is missing.")
+  }
+
+  def corruptedViewSQLConfigsInCatalogError(e: Exception): Throwable = {
+    new AnalysisException("Corrupted view SQL configs in catalog", cause = Some(e))
+  }
+
+  def corruptedViewQueryOutputColumnsInCatalogError(numCols: String, index: Int): Throwable = {
+    new AnalysisException("Corrupted view query output column names in catalog: " +
+      s"$numCols parts expected, but part $index is missing.")
+  }
+
+  def corruptedViewReferredTempViewInCatalogError(e: Exception): Throwable = {
+    new AnalysisException("corrupted view referred temp view names in catalog", cause = Some(e))
+  }
+
+  def corruptedViewReferredTempFunctionsInCatalogError(e: Exception): Throwable = {
+    new AnalysisException(
+      "corrupted view referred temp functions names in catalog", cause = Some(e))
+  }
+
+  def columnStatisticsDeserializationNotSupportedError(
+      name: String, dataType: DataType): Throwable = {
+    new AnalysisException("Column statistics deserialization is not supported for " +
+      s"column $name of data type: $dataType.")
+  }
+
+  def columnStatisticsSerializationNotSupportedError(
+      colName: String, dataType: DataType): Throwable = {
+    new AnalysisException("Column statistics serialization is not supported for " +
+      s"column $colName of data type: $dataType.")
+  }
+
+  def cannotReadCorruptedTablePropertyError(key: String, details: String = ""): Throwable = {
+    new AnalysisException(s"Cannot read table property '$key' as it's corrupted.$details")
+  }
+
+  def invalidSchemaStringError(exp: Expression): Throwable = {
+    new AnalysisException(s"The expression '${exp.sql}' is not a valid schema string.")
+  }
+
+  def schemaNotFoldableError(exp: Expression): Throwable = {
+    new AnalysisException(
+      "Schema should be specified in DDL format as a string literal or output of " +
+        s"the schema_of_json/schema_of_csv functions instead of ${exp.sql}")
+  }
+
+  def schemaIsNotStructTypeError(dataType: DataType): Throwable = {
+    new AnalysisException(s"Schema should be struct type but got ${dataType.sql}.")
+  }
+
+  def keyValueInMapNotStringError(m: CreateMap): Throwable = {
+    new AnalysisException(
+      s"A type of keys and values in map() must be string, but got ${m.dataType.catalogString}")
+  }
+
+  def nonMapFunctionNotAllowedError(): Throwable = {
+    new AnalysisException("Must use a map() function for options")
+  }
+
+  def invalidFieldTypeForCorruptRecordError(): Throwable = {
+    new AnalysisException("The field for corrupt records must be string type and nullable")
+  }
+
+  def dataTypeUnsupportedByClassError(x: DataType, className: String): Throwable = {
+    new AnalysisException(s"DataType '$x' is not supported by $className.")
+  }
+
+  def parseModeUnsupportedError(funcName: String, mode: ParseMode): Throwable = {
+    new AnalysisException(s"$funcName() doesn't support the ${mode.name} mode. " +
+      s"Acceptable modes are ${PermissiveMode.name} and ${FailFastMode.name}.")
+  }
+
+  def unfoldableFieldUnsupportedError(): Throwable = {
+    new AnalysisException("The field parameter needs to be a foldable string value.")
+  }
+
+  def literalTypeUnsupportedForSourceTypeError(field: String, source: Expression): Throwable = {
+    new AnalysisException(s"Literals of type '$field' are currently not supported " +
+      s"for the ${source.dataType.catalogString} type.")
+  }
+
+  def arrayComponentTypeUnsupportedError(clz: Class[_]): Throwable = {
+    new AnalysisException(s"Unsupported component type $clz in arrays")
+  }
+
+  def secondArgumentNotDoubleLiteralError(): Throwable = {
+    new AnalysisException("The second argument should be a double literal.")
+  }
+
+  def dataTypeUnsupportedByExtractValueError(
+      dataType: DataType, extraction: Expression, child: Expression): Throwable = {
+    val errorMsg = dataType match {
+      case StructType(_) =>
+        s"Field name should be String Literal, but it's $extraction"
+      case other =>
+        s"Can't extract value from $child: need struct type but got ${other.catalogString}"
+    }
+    new AnalysisException(errorMsg)
+  }
+
+  def noHandlerForUDAFError(name: String): Throwable = {
+    new InvalidUDFClassException(s"No handler for UDAF '$name'. " +
+      "Use sparkSession.udf.register(...) instead.")
+  }
+
+  def batchWriteCapabilityError(
+      table: Table, v2WriteClassName: String, v1WriteClassName: String): Throwable = {
+    new AnalysisException(
+      s"Table ${table.name} declares ${TableCapability.V1_BATCH_WRITE} capability but " +
+        s"$v2WriteClassName is not an instance of $v1WriteClassName")
+  }
+
+  def unsupportedDeleteByConditionWithSubqueryError(condition: Option[Expression]): Throwable = {
+    new AnalysisException(
+      s"Delete by condition with subquery is not supported: $condition")
+  }
+
+  def cannotTranslateExpressionToSourceFilterError(f: Expression): Throwable = {
+    new AnalysisException("Exec update failed:" +
+      s" cannot translate expression to source filter: $f")
+  }
+
+  def cannotDeleteTableWhereFiltersError(table: Table, filters: Array[Filter]): Throwable = {
+    new AnalysisException(
+      s"Cannot delete from table ${table.name} where ${filters.mkString("[", ", ", "]")}")
+  }
+
+  def deleteOnlySupportedWithV2TablesError(): Throwable = {
+    new AnalysisException("DELETE is only supported with v2 tables.")
+  }
+
+  def describeDoesNotSupportPartitionForV2TablesError(): Throwable = {
+    new AnalysisException("DESCRIBE does not support partition for v2 tables.")
+  }
+
+  def cannotReplaceMissingTableError(
+      tableIdentifier: Identifier): Throwable = {
+    new CannotReplaceMissingTableException(tableIdentifier)
+  }
+
+  def cannotReplaceMissingTableError(
+      tableIdentifier: Identifier, cause: Option[Throwable]): Throwable = {
+    new CannotReplaceMissingTableException(tableIdentifier, cause)
+  }
+
+  def unsupportedTableOperationError(table: Table, cmd: String): Throwable = {
+    new AnalysisException(s"Table ${table.name} does not support $cmd.")
+  }
+
+  def unsupportedBatchReadError(table: Table): Throwable = {
+    unsupportedTableOperationError(table, "batch scan")
+  }
+
+  def unsupportedMicroBatchOrContinuousScanError(table: Table): Throwable = {
+    unsupportedTableOperationError(table, "either micro-batch or continuous scan")
+  }
+
+  def unsupportedAppendInBatchModeError(table: Table): Throwable = {
+    unsupportedTableOperationError(table, "append in batch mode")
+  }
+
+  def unsupportedDynamicOverwriteInBatchModeError(table: Table): Throwable = {
+    unsupportedTableOperationError(table, "dynamic overwrite in batch mode")
+  }
+
+  def unsupportedTruncateInBatchModeError(table: Table): Throwable = {
+    unsupportedTableOperationError(table, "truncate in batch mode")
+  }
+
+  def unsupportedOverwriteByFilterInBatchModeError(table: Table): Throwable = {
+    unsupportedTableOperationError(table, "overwrite by filter in batch mode")
+  }
+
+  def streamingSourcesDoNotSupportCommonExecutionModeError(
+      microBatchSources: Seq[String],
+      continuousSources: Seq[String]): Throwable = {
+    new AnalysisException(
+      "The streaming sources in a query do not have a common supported execution mode.\n" +
+        "Sources support micro-batch: " + microBatchSources.mkString(", ") + "\n" +
+        "Sources support continuous: " + continuousSources.mkString(", "))
+  }
+
+  def noSuchTableError(ident: Identifier): Throwable = {
+    new NoSuchTableException(ident)
+  }
+
+  def noSuchNamespaceError(namespace: Array[String]): Throwable = {
+    new NoSuchNamespaceException(namespace)
+  }
+
+  def tableAlreadyExistsError(ident: Identifier): Throwable = {
+    new TableAlreadyExistsException(ident)
+  }
+
+  def requiresSinglePartNamespaceError(ident: Identifier): Throwable = {
+    new NoSuchTableException(
+      s"V2 session catalog requires a single-part namespace: ${ident.quoted}")
+  }
+
+  def namespaceAlreadyExistsError(namespace: Array[String]): Throwable = {
+    new NamespaceAlreadyExistsException(namespace)
+  }
+
+  private def notSupportedInJDBCCatalog(cmd: String): Throwable = {
+    new AnalysisException(s"$cmd is not supported in JDBC catalog.")
+  }
+
+  def cannotCreateJDBCTableUsingProviderError(): Throwable = {
+    notSupportedInJDBCCatalog("CREATE TABLE ... USING ...")
+  }
+
+  def cannotCreateJDBCTableUsingLocationError(): Throwable = {
+    notSupportedInJDBCCatalog("CREATE TABLE ... LOCATION ...")
+  }
+
+  def cannotCreateJDBCNamespaceUsingProviderError(): Throwable = {
+    notSupportedInJDBCCatalog("CREATE NAMESPACE ... LOCATION ...")
+  }
+
+  def cannotCreateJDBCNamespaceWithPropertyError(k: String): Throwable = {
+    notSupportedInJDBCCatalog(s"CREATE NAMESPACE with property $k")
+  }
+
+  def cannotSetJDBCNamespaceWithPropertyError(k: String): Throwable = {
+    notSupportedInJDBCCatalog(s"SET NAMESPACE with property $k")
+  }
+
+  def cannotUnsetJDBCNamespaceWithPropertyError(k: String): Throwable = {
+    notSupportedInJDBCCatalog(s"Remove NAMESPACE property $k")
+  }
+
+  def unsupportedJDBCNamespaceChangeInCatalogError(changes: Seq[NamespaceChange]): Throwable = {
+    new AnalysisException(s"Unsupported NamespaceChange $changes in JDBC catalog.")
+  }
+
+  private def tableDoesNotSupportError(cmd: String, table: Table): Throwable = {
+    new AnalysisException(s"Table does not support $cmd: ${table.name}")
+  }
+
+  def tableDoesNotSupportReadsError(table: Table): Throwable = {
+    tableDoesNotSupportError("reads", table)
+  }
+
+  def tableDoesNotSupportWritesError(table: Table): Throwable = {
+    tableDoesNotSupportError("writes", table)
+  }
+
+  def tableDoesNotSupportDeletesError(table: Table): Throwable = {
+    tableDoesNotSupportError("deletes", table)
+  }
+
+  def tableDoesNotSupportTruncatesError(table: Table): Throwable = {
+    tableDoesNotSupportError("truncates", table)
+  }
+
+  def tableDoesNotSupportPartitionManagementError(table: Table): Throwable = {
+    tableDoesNotSupportError("partition management", table)
+  }
+
+  def tableDoesNotSupportAtomicPartitionManagementError(table: Table): Throwable = {
+    tableDoesNotSupportError("atomic partition management", table)
+  }
+
+  def cannotRenameTableWithAlterViewError(): Throwable = {
+    new AnalysisException(
+      "Cannot rename a table with ALTER VIEW. Please use ALTER TABLE instead.")
+  }
+
+  private def notSupportedForV2TablesError(cmd: String): Throwable = {
+    new AnalysisException(s"$cmd is not supported for v2 tables.")
+  }
+
+  def analyzeTableNotSupportedForV2TablesError(): Throwable = {
+    notSupportedForV2TablesError("ANALYZE TABLE")
+  }
+
+  def alterTableRecoverPartitionsNotSupportedForV2TablesError(): Throwable = {
+    notSupportedForV2TablesError("ALTER TABLE ... RECOVER PARTITIONS")
+  }
+
+  def alterTableSerDePropertiesNotSupportedForV2TablesError(): Throwable = {
+    notSupportedForV2TablesError("ALTER TABLE ... SET [SERDE|SERDEPROPERTIES]")
+  }
+
+  def loadDataNotSupportedForV2TablesError(): Throwable = {
+    notSupportedForV2TablesError("LOAD DATA")
+  }
+
+  def showCreateTableNotSupportedForV2TablesError(): Throwable = {
+    notSupportedForV2TablesError("SHOW CREATE TABLE")
+  }
+
+  def truncateTableNotSupportedForV2TablesError(): Throwable = {
+    notSupportedForV2TablesError("TRUNCATE TABLE")
+  }
+
+  def showColumnsNotSupportedForV2TablesError(): Throwable = {
+    notSupportedForV2TablesError("SHOW COLUMNS")
+  }
+
+  def repairTableNotSupportedForV2TablesError(): Throwable = {
+    notSupportedForV2TablesError("MSCK REPAIR TABLE")
+  }
+
+  def databaseFromV1SessionCatalogNotSpecifiedError(): Throwable = {
+    new AnalysisException("Database from v1 session catalog is not specified")
+  }
+
+  def nestedDatabaseUnsupportedByV1SessionCatalogError(catalog: String): Throwable = {
+    new AnalysisException(s"Nested databases are not supported by v1 session catalog: $catalog")
+  }
+
+  def invalidRepartitionExpressionsError(sortOrders: Seq[Any]): Throwable = {
+    new AnalysisException(s"Invalid partitionExprs specified: $sortOrders For range " +
+      "partitioning use REPARTITION_BY_RANGE instead.")
+  }
+
+  def partitionColumnNotSpecifiedError(format: String, partitionColumn: String): Throwable = {
+    new AnalysisException(s"Failed to resolve the schema for $format for " +
+      s"the partition column: $partitionColumn. It must be specified manually.")
+  }
+
+  def dataSchemaNotSpecifiedError(format: String): Throwable = {
+    new AnalysisException(s"Unable to infer schema for $format. It must be specified manually.")
+  }
+
+  def dataPathNotExistError(path: String): Throwable = {
+    new AnalysisException(s"Path does not exist: $path")
+  }
+
+  def dataSourceOutputModeUnsupportedError(
+      className: String, outputMode: OutputMode): Throwable = {
+    new AnalysisException(s"Data source $className does not support $outputMode output mode")
+  }
+
+  def schemaNotSpecifiedForSchemaRelationProviderError(className: String): Throwable = {
+    new AnalysisException(s"A schema needs to be specified when using $className.")
+  }
+
+  def userSpecifiedSchemaMismatchActualSchemaError(
+      schema: StructType, actualSchema: StructType): Throwable = {
+    new AnalysisException(
+      s"""
+         |The user-specified schema doesn't match the actual schema:
+         |user-specified: ${schema.toDDL}, actual: ${actualSchema.toDDL}. If you're using
+         |DataFrameReader.schema API or creating a table, please do not specify the schema.
+         |Or if you're scanning an existed table, please drop it and re-create it.
+       """.stripMargin)
+  }
+
+  def dataSchemaNotSpecifiedError(format: String, fileCatalog: String): Throwable = {
+    new AnalysisException(
+      s"Unable to infer schema for $format at $fileCatalog. It must be specified manually")
+  }
+
+  def invalidDataSourceError(className: String): Throwable = {
+    new AnalysisException(s"$className is not a valid Spark SQL Data Source.")
+  }
+
+  def cannotSaveIntervalIntoExternalStorageError(): Throwable = {
+    new AnalysisException("Cannot save interval data type into external storage.")
+  }
+
+  def cannotResolveAttributeError(name: String, data: LogicalPlan): Throwable = {
+    new AnalysisException(
+      s"Unable to resolve $name given [${data.output.map(_.name).mkString(", ")}]")
+  }
+
+  def orcNotUsedWithHiveEnabledError(): Throwable = {
+    new AnalysisException(
+      s"""
+         |Hive built-in ORC data source must be used with Hive support enabled.
+         |Please use the native ORC data source by setting 'spark.sql.orc.impl' to 'native'
+       """.stripMargin)
+  }
+
+  def failedToFindAvroDataSourceError(provider: String): Throwable = {
+    new AnalysisException(
+      s"""
+         |Failed to find data source: $provider. Avro is built-in but external data
+         |source module since Spark 2.4. Please deploy the application as per
+         |the deployment section of "Apache Avro Data Source Guide".
+       """.stripMargin.replaceAll("\n", " "))
+  }
+
+  def failedToFindKafkaDataSourceError(provider: String): Throwable = {
+    new AnalysisException(
+      s"""
+         |Failed to find data source: $provider. Please deploy the application as
+         |per the deployment section of "Structured Streaming + Kafka Integration Guide".
+       """.stripMargin.replaceAll("\n", " "))
+  }
+
+  def findMultipleDataSourceError(provider: String, sourceNames: Seq[String]): Throwable = {
+    new AnalysisException(
+      s"""
+         |Multiple sources found for $provider (${sourceNames.mkString(", ")}),
+         | please specify the fully qualified class name.
+       """.stripMargin)
+  }
+
+  def writeEmptySchemasUnsupportedByDataSourceError(): Throwable = {
+    new AnalysisException(
+      s"""
+         |Datasource does not support writing empty or nested empty schemas.
+         |Please make sure the data schema has at least one or more column(s).
+       """.stripMargin)
+  }
+
+  def insertMismatchedColumnNumberError(
+      targetAttributes: Seq[Attribute],
+      sourceAttributes: Seq[Attribute],
+      staticPartitionsSize: Int): Throwable = {
+    new AnalysisException(
+      s"""
+         |The data to be inserted needs to have the same number of columns as the
+         |target table: target table has ${targetAttributes.size} column(s) but the
+         |inserted data has ${sourceAttributes.size + staticPartitionsSize} column(s),
+         |which contain $staticPartitionsSize partition column(s) having assigned
+         |constant values.
+       """.stripMargin)
+  }
+
+  def insertMismatchedPartitionNumberError(
+      targetPartitionSchema: StructType,
+      providedPartitionsSize: Int): Throwable = {
+    new AnalysisException(
+      s"""
+         |The data to be inserted needs to have the same number of partition columns
+         |as the target table: target table has ${targetPartitionSchema.fields.size}
+         |partition column(s) but the inserted data has $providedPartitionsSize
+         |partition columns specified.
+       """.stripMargin.replaceAll("\n", " "))
+  }
+
+  def invalidPartitionColumnError(
+      partKey: String, targetPartitionSchema: StructType): Throwable = {
+    new AnalysisException(
+      s"""
+         |$partKey is not a partition column. Partition columns are
+         |${targetPartitionSchema.fields.map(_.name).mkString("[", ",", "]")}
+       """.stripMargin)
+  }
+
+  def multiplePartitionColumnValuesSpecifiedError(
+      field: StructField, potentialSpecs: Map[String, String]): Throwable = {
+    new AnalysisException(
+      s"""
+         |Partition column ${field.name} have multiple values specified,
+         |${potentialSpecs.mkString("[", ", ", "]")}. Please only specify a single value.
+       """.stripMargin)
+  }
+
+  def invalidOrderingForConstantValuePartitionColumnError(
+      targetPartitionSchema: StructType): Throwable = {
+    new AnalysisException(
+      s"""
+         |The ordering of partition columns is
+         |${targetPartitionSchema.fields.map(_.name).mkString("[", ",", "]")}
+         |All partition columns having constant values need to appear before other
+         |partition columns that do not have an assigned constant value.
+       """.stripMargin)
+  }
+
+  def cannotWriteDataToRelationsWithMultiplePathsError(): Throwable = {
+    new AnalysisException("Can only write data to relations with a single path.")
+  }
+
+  def failedToRebuildExpressionError(filter: Filter): Throwable = {
+    new AnalysisException(
+      s"Fail to rebuild expression: missing key $filter in `translatedFilterToExpr`")
+  }
+
+  def dataTypeUnsupportedByDataSourceError(format: String, field: StructField): Throwable = {
+    new AnalysisException(
+      s"$format data source does not support ${field.dataType.catalogString} data type.")
+  }
+
+  def failToResolveDataSourceForTableError(table: CatalogTable, key: String): Throwable = {
+    new AnalysisException(
+      s"""
+         |Fail to resolve data source for the table ${table.identifier} since the table
+         |serde property has the duplicated key $key with extra options specified for this
+         |scan operation. To fix this, you can rollback to the legacy behavior of ignoring
+         |the extra options by setting the config
+         |${SQLConf.LEGACY_EXTRA_OPTIONS_BEHAVIOR.key} to `false`, or address the
+         |conflicts of the same config.
+       """.stripMargin)
+  }
+
+  def outputPathAlreadyExistsError(outputPath: Path): Throwable = {
+    new AnalysisException(s"path $outputPath already exists.")
+  }
+
+  def cannotUseDataTypeForPartitionColumnError(field: StructField): Throwable = {
+    new AnalysisException(s"Cannot use ${field.dataType} for partition column")
+  }
+
+  def cannotUseAllColumnsForPartitionColumnsError(): Throwable = {
+    new AnalysisException(s"Cannot use all columns for partition columns")
+  }
+
+  def partitionColumnNotFoundInSchemaError(col: String, schemaCatalog: String): Throwable = {
+    new AnalysisException(s"Partition column `$col` not found in schema $schemaCatalog")
+  }
+
+  def columnNotFoundInSchemaError(
+      col: StructField, tableSchema: Option[StructType]): Throwable = {
+    new AnalysisException(s"""Column "${col.name}" not found in schema $tableSchema""")
+  }
+
+  def unsupportedDataSourceTypeForDirectQueryOnFilesError(className: String): Throwable = {
+    new AnalysisException(s"Unsupported data source type for direct query on files: $className")
+  }
+
+  def saveDataIntoViewNotAllowedError(): Throwable = {
+    new AnalysisException("Saving data into a view is not allowed.")
+  }
+
+  def mismatchedTableFormatError(
+      tableName: String, existingProvider: Class[_], specifiedProvider: Class[_]): Throwable = {
+    new AnalysisException(
+      s"""
+         |The format of the existing table $tableName is `${existingProvider.getSimpleName}`.
+         |It doesn't match the specified format `${specifiedProvider.getSimpleName}`.
+       """.stripMargin)
+  }
+
+  def mismatchedTableLocationError(
+      identifier: TableIdentifier,
+      existingTable: CatalogTable,
+      tableDesc: CatalogTable): Throwable = {
+    new AnalysisException(
+      s"""
+         |The location of the existing table ${identifier.quotedString} is
+         |`${existingTable.location}`. It doesn't match the specified location
+         |`${tableDesc.location}`.
+       """.stripMargin)
+  }
+
+  def mismatchedTableColumnNumberError(
+      tableName: String,
+      existingTable: CatalogTable,
+      query: LogicalPlan): Throwable = {
+    new AnalysisException(
+      s"""
+         |The column number of the existing table $tableName
+         |(${existingTable.schema.catalogString}) doesn't match the data schema
+         |(${query.schema.catalogString})
+       """.stripMargin)
+  }
+
+  def cannotResolveColumnGivenInputColumnsError(col: String, inputColumns: String): Throwable = {
+    new AnalysisException(s"cannot resolve '$col' given input columns: [$inputColumns]")
+  }
+
+  def mismatchedTablePartitionColumnError(
+      tableName: String,
+      specifiedPartCols: Seq[String],
+      existingPartCols: String): Throwable = {
+    new AnalysisException(
+      s"""
+         |Specified partitioning does not match that of the existing table $tableName.
+         |Specified partition columns: [${specifiedPartCols.mkString(", ")}]
+         |Existing partition columns: [$existingPartCols]
+       """.stripMargin)
+  }
+
+  def mismatchedTableBucketingError(
+      tableName: String,
+      specifiedBucketString: String,
+      existingBucketString: String): Throwable = {
+    new AnalysisException(
+      s"""
+         |Specified bucketing does not match that of the existing table $tableName.
+         |Specified bucketing: $specifiedBucketString
+         |Existing bucketing: $existingBucketString
+       """.stripMargin)
+  }
+
+  def specifyPartitionNotAllowedWhenTableSchemaNotDefinedError(): Throwable = {
+    new AnalysisException("It is not allowed to specify partitioning when the " +
+      "table schema is not defined.")
+  }
+
+  def bucketingColumnCannotBePartOfPartitionColumnsError(
+      bucketCol: String, normalizedPartCols: Seq[String]): Throwable = {
+    new AnalysisException(s"bucketing column '$bucketCol' should not be part of " +
+      s"partition columns '${normalizedPartCols.mkString(", ")}'")
+  }
+
+  def bucketSortingColumnCannotBePartOfPartitionColumnsError(
+    sortCol: String, normalizedPartCols: Seq[String]): Throwable = {
+    new AnalysisException(s"bucket sorting column '$sortCol' should not be part of " +
+      s"partition columns '${normalizedPartCols.mkString(", ")}'")
+  }
+
+  def mismatchedInsertedDataColumnNumberError(
+      tableName: String, insert: InsertIntoStatement, staticPartCols: Set[String]): Throwable = {
+    new AnalysisException(
+      s"$tableName requires that the data to be inserted have the same number of columns as " +
+        s"the target table: target table has ${insert.table.output.size} column(s) but the " +
+        s"inserted data has ${insert.query.output.length + staticPartCols.size} column(s), " +
+        s"including ${staticPartCols.size} partition column(s) having constant value(s).")
+  }
+
+  def requestedPartitionsMismatchTablePartitionsError(
+      tableName: String,
+      normalizedPartSpec: Map[String, Option[String]],
+      partColNames: StructType): Throwable = {
+    new AnalysisException(
+      s"""
+         |Requested partitioning does not match the table $tableName:
+         |Requested partitions: ${normalizedPartSpec.keys.mkString(",")}
+         |Table partitions: ${partColNames.mkString(",")}
+       """.stripMargin)
+  }
+
+  def ddlWithoutHiveSupportEnabledError(detail: String): Throwable = {
+    new AnalysisException(s"Hive support is required to $detail")
+  }
+
+  def createTableColumnTypesOptionColumnNotFoundInSchemaError(
+      col: String, schema: StructType): Throwable = {
+    new AnalysisException(
+      s"createTableColumnTypes option column $col not found in schema ${schema.catalogString}")
+  }
+
+  def parquetTypeUnsupportedYetError(parquetType: String): Throwable = {
+    new AnalysisException(s"Parquet type not yet supported: $parquetType")
+  }
+
+  def illegalParquetTypeError(parquetType: String): Throwable = {
+    new AnalysisException(s"Illegal Parquet type: $parquetType")
+  }
+
+  def unrecognizedParquetTypeError(field: String): Throwable = {
+    new AnalysisException(s"Unrecognized Parquet type: $field")
+  }
+
+  def cannotConvertDataTypeToParquetTypeError(field: StructField): Throwable = {
+    new AnalysisException(s"Unsupported data type ${field.dataType.catalogString}")
+  }
+
+  def incompatibleViewSchemaChange(
+      viewName: String,
+      colName: String,
+      expectedNum: Int,
+      actualCols: Seq[Attribute]): Throwable = {
+    new AnalysisException(s"The SQL query of view $viewName has an incompatible schema change " +
+      s"and column $colName cannot be resolved. Expected $expectedNum columns named $colName but " +
+      s"got ${actualCols.map(_.name).mkString("[", ",", "]")}")
+  }
+
+  def numberOfPartitionsNotAllowedWithUnspecifiedDistributionError(): Throwable = {
+    throw new AnalysisException("The number of partitions can't be specified with unspecified" +
+      " distribution. Invalid writer requirements detected.")
+  }
+
+  def cannotApplyTableValuedFunctionError(
+      name: String, arguments: String, usage: String, details: String = ""): Throwable = {
+    new AnalysisException(s"Table-valued function $name with alternatives: $usage\n" +
+      s"cannot be applied to ($arguments): $details")
+  }
+
+  def incompatibleRangeInputDataTypeError(
+      expression: Expression, dataType: DataType): Throwable = {
+    new AnalysisException(s"Incompatible input data type. " +
+      s"Expected: ${dataType.typeName}; Found: ${expression.dataType.typeName}")
+  }
+
+  def groupAggPandasUDFUnsupportedByStreamingAggError(): Throwable = {
+    new AnalysisException("Streaming aggregation doesn't support group aggregate pandas UDF")
+  }
+
+  def streamJoinStreamWithoutEqualityPredicateUnsupportedError(plan: LogicalPlan): Throwable = {
+    new AnalysisException(
+      "Stream-stream join without equality predicate is not supported", plan = Some(plan))
+  }
+
+  def cannotUseMixtureOfAggFunctionAndGroupAggPandasUDFError(): Throwable = {
+    new AnalysisException(
+      "Cannot use a mixture of aggregate function and group aggregate pandas UDF")
+  }
+
+  def ambiguousAttributesInSelfJoinError(
+      ambiguousAttrs: Seq[AttributeReference]): Throwable = {
+    new AnalysisException(
+      s"""
+         |Column ${ambiguousAttrs.mkString(", ")} are ambiguous. It's probably because
+         |you joined several Datasets together, and some of these Datasets are the same.
+         |This column points to one of the Datasets but Spark is unable to figure out
+         |which one. Please alias the Datasets with different names via `Dataset.as`
+         |before joining them, and specify the column using qualified name, e.g.
+         |`df.as("a").join(df.as("b"), $$"a.id" > $$"b.id")`. You can also set
+         |${SQLConf.FAIL_AMBIGUOUS_SELF_JOIN_ENABLED.key} to false to disable this check.
+       """.stripMargin.replaceAll("\n", " "))
+  }
+
+  def unexpectedEvalTypesForUDFsError(evalTypes: Set[Int]): Throwable = {
+    new AnalysisException(
+      s"Expected udfs have the same evalType but got different evalTypes: " +
+        s"${evalTypes.mkString(",")}")
+  }
+
+  def ambiguousFieldNameError(fieldName: String, names: String): Throwable = {
+    new AnalysisException(
+      s"Ambiguous field name: $fieldName. Found multiple columns that can match: $names")
+  }
+
+  def cannotUseIntervalTypeInTableSchemaError(): Throwable = {
+    new AnalysisException("Cannot use interval type in the table schema.")
+  }
+
+  def cannotConvertBucketWithSortColumnsToTransformError(spec: BucketSpec): Throwable = {
+    new AnalysisException(
+      s"Cannot convert bucketing with sort columns to a transform: $spec")
+  }
+
+  def cannotConvertTransformsToPartitionColumnsError(nonIdTransforms: Seq[Transform]): Throwable = {
+    new AnalysisException("Transforms cannot be converted to partition columns: " +
+      nonIdTransforms.map(_.describe).mkString(", "))
+  }
+
+  def cannotPartitionByNestedColumnError(reference: NamedReference): Throwable = {
+    new AnalysisException(s"Cannot partition by nested column: $reference")
+  }
+
+  def cannotUseCatalogError(plugin: CatalogPlugin, msg: String): Throwable = {
+    new AnalysisException(s"Cannot use catalog ${plugin.name}: $msg")
+  }
+
+  def identifierHavingMoreThanTwoNamePartsError(
+      quoted: String, identifier: String): Throwable = {
+    new AnalysisException(s"$quoted is not a valid $identifier as it has more than 2 name parts.")
+  }
+
+  def emptyMultipartIdentifierError(): Throwable = {
+    new AnalysisException("multi-part identifier cannot be empty.")
+  }
+
+  def cannotCreateTablesWithNullTypeError(): Throwable = {
+    new AnalysisException(s"Cannot create tables with ${NullType.simpleString} type.")
+  }
+
+  def functionUnsupportedInV2CatalogError(): Throwable = {
+    new AnalysisException("function is only supported in v1 catalog")
+  }
+
+  def cannotOperateOnHiveDataSourceFilesError(operation: String): Throwable = {
+    new AnalysisException("Hive data source can only be used with tables, you can not " +
+      s"$operation files of Hive data source directly.")
+  }
+
+  def setPathOptionAndCallWithPathParameterError(method: String): Throwable = {
+    new AnalysisException(
+      s"""
+         |There is a 'path' option set and $method() is called with a path
+         |parameter. Either remove the path option, or call $method() without the parameter.
+         |To ignore this check, set '${SQLConf.LEGACY_PATH_OPTION_BEHAVIOR.key}' to 'true'.
+       """.stripMargin.replaceAll("\n", " "))
+  }
+
+  def userSpecifiedSchemaWithTextFileError(): Throwable = {
+    new AnalysisException("User specified schema not supported with `textFile`")
+  }
+
+  def tempViewNotSupportStreamingWriteError(viewName: String): Throwable = {
+    new AnalysisException(s"Temporary view $viewName doesn't support streaming write")
+  }
+
+  def streamingIntoViewNotSupportedError(viewName: String): Throwable = {
+    new AnalysisException(s"Streaming into views $viewName is not supported.")
+  }
+
+  def inputSourceDiffersFromDataSourceProviderError(
+      source: String, tableName: String, table: CatalogTable): Throwable = {
+    new AnalysisException(s"The input source($source) is different from the table " +
+      s"$tableName's data source provider(${table.provider.get}).")
+  }
+
+  def tableNotSupportStreamingWriteError(tableName: String, t: Table): Throwable = {
+    new AnalysisException(s"Table $tableName doesn't support streaming write - $t")
+  }
+
+  def queryNameNotSpecifiedForMemorySinkError(): Throwable = {
+    new AnalysisException("queryName must be specified for memory sink")
+  }
+
+  def sourceNotSupportedWithContinuousTriggerError(source: String): Throwable = {
+    new AnalysisException(s"'$source' is not supported with continuous trigger")
+  }
+
+  def columnNotFoundInExistingColumnsError(
+      columnType: String, columnName: String, validColumnNames: Seq[String]): Throwable = {
+    new AnalysisException(s"$columnType column $columnName not found in " +
+      s"existing columns (${validColumnNames.mkString(", ")})")
+  }
+
+  def operationNotSupportPartitioningError(operation: String): Throwable = {
+    new AnalysisException(s"'$operation' does not support partitioning")
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
new file mode 100644
index 0000000000000..a2ea696a2ac29
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
@@ -0,0 +1,934 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.errors
+
+import java.io.{FileNotFoundException, IOException}
+import java.net.URISyntaxException
+import java.sql.{SQLException, SQLFeatureNotSupportedException}
+import java.time.{DateTimeException, LocalDate}
+import java.time.temporal.ChronoField
+import java.util.ConcurrentModificationException
+
+import org.apache.hadoop.fs.{FileStatus, Path}
+import org.codehaus.commons.compiler.{CompileException, InternalCompilerException}
+
+import org.apache.spark.{Partition, SparkException, SparkUpgradeException}
+import org.apache.spark.executor.CommitDeniedException
+import org.apache.spark.memory.SparkOutOfMemoryError
+import org.apache.spark.sql.catalyst.analysis.UnresolvedGenerator
+import org.apache.spark.sql.catalyst.catalog.CatalogDatabase
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, UnevaluableAggregate}
+import org.apache.spark.sql.catalyst.plans.JoinType
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.expressions.Transform
+import org.apache.spark.sql.execution.QueryExecutionException
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types.{DataType, Decimal, StructType}
+import org.apache.spark.unsafe.array.ByteArrayMethods
+import org.apache.spark.unsafe.types.UTF8String
+
+/**
+ * Object for grouping error messages from (most) exceptions thrown during query execution.
+ * This does not include exceptions thrown during the eager execution of commands, which are
+ * grouped into [[QueryCompilationErrors]].
+ */
+object QueryExecutionErrors {
+
+  def columnChangeUnsupportedError(): Throwable = {
+    new UnsupportedOperationException("Please add an implementation for a column change here")
+  }
+
+  def logicalHintOperatorNotRemovedDuringAnalysisError(): Throwable = {
+    new IllegalStateException(
+      "Internal error: logical hint operator should have been removed during analysis")
+  }
+
+  def cannotEvaluateExpressionError(expression: Expression): Throwable = {
+    new UnsupportedOperationException(s"Cannot evaluate expression: $expression")
+  }
+
+  def cannotGenerateCodeForExpressionError(expression: Expression): Throwable = {
+    new UnsupportedOperationException(s"Cannot generate code for expression: $expression")
+  }
+
+  def cannotTerminateGeneratorError(generator: UnresolvedGenerator): Throwable = {
+    new UnsupportedOperationException(s"Cannot terminate expression: $generator")
+  }
+
+  def castingCauseOverflowError(t: Any, targetType: String): ArithmeticException = {
+    new ArithmeticException(s"Casting $t to $targetType causes overflow")
+  }
+
+  def cannotChangeDecimalPrecisionError(
+      value: Decimal, decimalPrecision: Int, decimalScale: Int): ArithmeticException = {
+    new ArithmeticException(s"${value.toDebugString} cannot be represented as " +
+      s"Decimal($decimalPrecision, $decimalScale).")
+  }
+
+  def invalidInputSyntaxForNumericError(s: UTF8String): NumberFormatException = {
+    new NumberFormatException(s"invalid input syntax for type numeric: $s")
+  }
+
+  def cannotCastFromNullTypeError(to: DataType): Throwable = {
+    new SparkException(s"should not directly cast from NullType to $to.")
+  }
+
+  def cannotCastError(from: DataType, to: DataType): Throwable = {
+    new SparkException(s"Cannot cast $from to $to.")
+  }
+
+  def cannotParseDecimalError(): Throwable = {
+    new IllegalArgumentException("Cannot parse any decimal")
+  }
+
+  def simpleStringWithNodeIdUnsupportedError(nodeName: String): Throwable = {
+    new UnsupportedOperationException(s"$nodeName does not implement simpleStringWithNodeId")
+  }
+
+  def evaluateUnevaluableAggregateUnsupportedError(
+      methodName: String, unEvaluable: UnevaluableAggregate): Throwable = {
+    new UnsupportedOperationException(s"Cannot evaluate $methodName: $unEvaluable")
+  }
+
+  def dataTypeUnsupportedError(dt: DataType): Throwable = {
+    new SparkException(s"Unsupported data type $dt")
+  }
+
+  def dataTypeUnsupportedError(dataType: String, failure: String): Throwable = {
+    new IllegalArgumentException(s"Unsupported dataType: $dataType, $failure")
+  }
+
+  def failedExecuteUserDefinedFunctionError(funcCls: String, inputTypes: String,
+      outputType: String, e: Throwable): Throwable = {
+    new SparkException(
+      s"Failed to execute user defined function ($funcCls: ($inputTypes) => $outputType)", e)
+  }
+
+  def divideByZeroError(): ArithmeticException = {
+    new ArithmeticException("divide by zero")
+  }
+
+  def invalidArrayIndexError(index: Int, numElements: Int): ArrayIndexOutOfBoundsException = {
+    new ArrayIndexOutOfBoundsException(s"Invalid index: $index, numElements: $numElements")
+  }
+
+  def mapKeyNotExistError(key: Any): NoSuchElementException = {
+    new NoSuchElementException(s"Key $key does not exist.")
+  }
+
+  def rowFromCSVParserNotExpectedError(): Throwable = {
+    new IllegalArgumentException("Expected one row from CSV parser.")
+  }
+
+  def inputTypeUnsupportedError(dataType: DataType): Throwable = {
+    new IllegalArgumentException(s"Unsupported input type ${dataType.catalogString}")
+  }
+
+  def invalidFractionOfSecondError(): DateTimeException = {
+    new DateTimeException("The fraction of sec must be zero. Valid range is [0, 60].")
+  }
+
+  def overflowInSumOfDecimalError(): ArithmeticException = {
+    new ArithmeticException("Overflow in sum of decimals.")
+  }
+
+  def overflowInIntegralDivideError(): ArithmeticException = {
+    new ArithmeticException("Overflow in integral divide.")
+  }
+
+  def mapSizeExceedArraySizeWhenZipMapError(size: Int): RuntimeException = {
+    new RuntimeException(s"Unsuccessful try to zip maps with $size " +
+      "unique keys due to exceeding the array size limit " +
+      s"${ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH}.")
+  }
+
+  def copyNullFieldNotAllowedError(): Throwable = {
+    new IllegalStateException("Do not attempt to copy a null field")
+  }
+
+  def literalTypeUnsupportedError(v: Any): RuntimeException = {
+    new RuntimeException(s"Unsupported literal type ${v.getClass} $v")
+  }
+
+  def noDefaultForDataTypeError(dataType: DataType): RuntimeException = {
+    new RuntimeException(s"no default for type $dataType")
+  }
+
+  def doGenCodeOfAliasShouldNotBeCalledError(): Throwable = {
+    new IllegalStateException("Alias.doGenCode should not be called.")
+  }
+
+  def orderedOperationUnsupportedByDataTypeError(dataType: DataType): Throwable = {
+    new IllegalArgumentException(s"Type $dataType does not support ordered operations")
+  }
+
+  def regexGroupIndexLessThanZeroError(): Throwable = {
+    new IllegalArgumentException("The specified group index cannot be less than zero")
+  }
+
+  def regexGroupIndexExceedGroupCountError(
+      groupCount: Int, groupIndex: Int): Throwable = {
+    new IllegalArgumentException(
+      s"Regex group count is $groupCount, but the specified group index is $groupIndex")
+  }
+
+  def invalidUrlError(url: UTF8String, e: URISyntaxException): Throwable = {
+    new IllegalArgumentException(s"Find an invaild url string ${url.toString}", e)
+  }
+
+  def dataTypeOperationUnsupportedError(): Throwable = {
+    new UnsupportedOperationException("dataType")
+  }
+
+  def mergeUnsupportedByWindowFunctionError(): Throwable = {
+    new UnsupportedOperationException("Window Functions do not support merging.")
+  }
+
+  def dataTypeUnexpectedError(dataType: DataType): Throwable = {
+    new UnsupportedOperationException(s"Unexpected data type ${dataType.catalogString}")
+  }
+
+  def typeUnsupportedError(dataType: DataType): Throwable = {
+    new IllegalArgumentException(s"Unexpected type $dataType")
+  }
+
+  def negativeValueUnexpectedError(frequencyExpression : Expression): Throwable = {
+    new SparkException(s"Negative values found in ${frequencyExpression.sql}")
+  }
+
+  def addNewFunctionMismatchedWithFunctionError(funcName: String): Throwable = {
+    new IllegalArgumentException(s"$funcName is not matched at addNewFunction")
+  }
+
+  def cannotGenerateCodeForUncomparableTypeError(
+      codeType: String, dataType: DataType): Throwable = {
+    new IllegalArgumentException(
+      s"cannot generate $codeType code for un-comparable type: ${dataType.catalogString}")
+  }
+
+  def cannotGenerateCodeForUnsupportedTypeError(dataType: DataType): Throwable = {
+    new IllegalArgumentException(s"cannot generate code for unsupported type: $dataType")
+  }
+
+  def cannotInterpolateClassIntoCodeBlockError(arg: Any): Throwable = {
+    new IllegalArgumentException(
+      s"Can not interpolate ${arg.getClass.getName} into code block.")
+  }
+
+  def customCollectionClsNotResolvedError(): Throwable = {
+    new UnsupportedOperationException("not resolved")
+  }
+
+  def classUnsupportedByMapObjectsError(cls: Class[_]): RuntimeException = {
+    new RuntimeException(s"class `${cls.getName}` is not supported by `MapObjects` as " +
+      "resulting collection.")
+  }
+
+  def nullAsMapKeyNotAllowedError(): RuntimeException = {
+    new RuntimeException("Cannot use null as map key!")
+  }
+
+  def methodNotDeclaredError(name: String): Throwable = {
+    new NoSuchMethodException(s"""A method named "$name" is not declared """ +
+      "in any enclosing class nor any supertype")
+  }
+
+  def constructorNotFoundError(cls: String): Throwable = {
+    new RuntimeException(s"Couldn't find a valid constructor on $cls")
+  }
+
+  def primaryConstructorNotFoundError(cls: Class[_]): Throwable = {
+    new RuntimeException(s"Couldn't find a primary constructor on $cls")
+  }
+
+  def unsupportedNaturalJoinTypeError(joinType: JoinType): Throwable = {
+    new RuntimeException("Unsupported natural join type " + joinType)
+  }
+
+  def notExpectedUnresolvedEncoderError(attr: AttributeReference): Throwable = {
+    new RuntimeException(s"Unresolved encoder expected, but $attr was found.")
+  }
+
+  def unsupportedEncoderError(): Throwable = {
+    new RuntimeException("Only expression encoders are supported for now.")
+  }
+
+  def notOverrideExpectedMethodsError(className: String, m1: String, m2: String): Throwable = {
+    new RuntimeException(s"$className must override either $m1 or $m2")
+  }
+
+  def failToConvertValueToJsonError(value: AnyRef, cls: Class[_], dataType: DataType): Throwable = {
+    new RuntimeException(s"Failed to convert value $value (class of $cls) " +
+      s"with the type of $dataType to JSON.")
+  }
+
+  def unexpectedOperatorInCorrelatedSubquery(op: LogicalPlan, pos: String = ""): Throwable = {
+    new RuntimeException(s"Unexpected operator $op in correlated subquery" + pos)
+  }
+
+  def unreachableError(err: String = ""): Throwable = {
+    new RuntimeException("This line should be unreachable" + err)
+  }
+
+  def unsupportedRoundingMode(roundMode: BigDecimal.RoundingMode.Value): Throwable = {
+    new RuntimeException(s"Not supported rounding mode: $roundMode")
+  }
+
+  def resolveCannotHandleNestedSchema(plan: LogicalPlan): Throwable = {
+    new RuntimeException(s"Can not handle nested schema yet...  plan $plan")
+  }
+
+  def inputExternalRowCannotBeNullError(): RuntimeException = {
+    new RuntimeException("The input external row cannot be null.")
+  }
+
+  def fieldCannotBeNullMsg(index: Int, fieldName: String): String = {
+    s"The ${index}th field '$fieldName' of input row cannot be null."
+  }
+
+  def fieldCannotBeNullError(index: Int, fieldName: String): RuntimeException = {
+    new RuntimeException(fieldCannotBeNullMsg(index, fieldName))
+  }
+
+  def unableToCreateDatabaseAsFailedToCreateDirectoryError(
+      dbDefinition: CatalogDatabase, e: IOException): Throwable = {
+    new SparkException(s"Unable to create database ${dbDefinition.name} as failed " +
+      s"to create its directory ${dbDefinition.locationUri}", e)
+  }
+
+  def unableToDropDatabaseAsFailedToDeleteDirectoryError(
+      dbDefinition: CatalogDatabase, e: IOException): Throwable = {
+    new SparkException(s"Unable to drop database ${dbDefinition.name} as failed " +
+      s"to delete its directory ${dbDefinition.locationUri}", e)
+  }
+
+  def unableToCreateTableAsFailedToCreateDirectoryError(
+      table: String, defaultTableLocation: Path, e: IOException): Throwable = {
+    new SparkException(s"Unable to create table $table as failed " +
+      s"to create its directory $defaultTableLocation", e)
+  }
+
+  def unableToDeletePartitionPathError(partitionPath: Path, e: IOException): Throwable = {
+    new SparkException(s"Unable to delete partition path $partitionPath", e)
+  }
+
+  def unableToDropTableAsFailedToDeleteDirectoryError(
+      table: String, dir: Path, e: IOException): Throwable = {
+    new SparkException(s"Unable to drop table $table as failed " +
+      s"to delete its directory $dir", e)
+  }
+
+  def unableToRenameTableAsFailedToRenameDirectoryError(
+      oldName: String, newName: String, oldDir: Path, e: IOException): Throwable = {
+    new SparkException(s"Unable to rename table $oldName to $newName as failed " +
+      s"to rename its directory $oldDir", e)
+  }
+
+  def unableToCreatePartitionPathError(partitionPath: Path, e: IOException): Throwable = {
+    new SparkException(s"Unable to create partition path $partitionPath", e)
+  }
+
+  def unableToRenamePartitionPathError(oldPartPath: Path, e: IOException): Throwable = {
+    new SparkException(s"Unable to rename partition path $oldPartPath", e)
+  }
+
+  def methodNotImplementedError(methodName: String): Throwable = {
+    new UnsupportedOperationException(s"$methodName is not implemented")
+  }
+
+  def tableStatsNotSpecifiedError(): Throwable = {
+    new IllegalStateException("table stats must be specified.")
+  }
+
+  def unaryMinusCauseOverflowError(originValue: AnyVal): ArithmeticException = {
+    new ArithmeticException(s"- $originValue caused overflow.")
+  }
+
+  def binaryArithmeticCauseOverflowError(
+      eval1: Short, symbol: String, eval2: Short): ArithmeticException = {
+    new ArithmeticException(s"$eval1 $symbol $eval2 caused overflow.")
+  }
+
+  def failedSplitSubExpressionMsg(length: Int): String = {
+    "Failed to split subexpression code into small functions because " +
+      s"the parameter length of at least one split function went over the JVM limit: $length"
+  }
+
+  def failedSplitSubExpressionError(length: Int): Throwable = {
+    new IllegalStateException(failedSplitSubExpressionMsg(length))
+  }
+
+  def failedToCompileMsg(e: Exception): String = {
+    s"failed to compile: $e"
+  }
+
+  def internalCompilerError(e: InternalCompilerException): Throwable = {
+    new InternalCompilerException(failedToCompileMsg(e), e)
+  }
+
+  def compilerError(e: CompileException): Throwable = {
+    new CompileException(failedToCompileMsg(e), e.getLocation)
+  }
+
+  def unsupportedTableChangeError(e: IllegalArgumentException): Throwable = {
+    new SparkException(s"Unsupported table change: ${e.getMessage}", e)
+  }
+
+  def notADatasourceRDDPartitionError(split: Partition): Throwable = {
+    new SparkException(s"[BUG] Not a DataSourceRDDPartition: $split")
+  }
+
+  def dataPathNotSpecifiedError(): Throwable = {
+    new IllegalArgumentException("'path' is not specified")
+  }
+
+  def createStreamingSourceNotSpecifySchemaError(): Throwable = {
+    new IllegalArgumentException(
+      s"""
+         |Schema must be specified when creating a streaming source DataFrame. If some
+         |files already exist in the directory, then depending on the file format you
+         |may be able to create a static DataFrame on that directory with
+         |'spark.read.load(directory)' and infer schema from it.
+       """.stripMargin)
+  }
+
+  def streamedOperatorUnsupportedByDataSourceError(
+      className: String, operator: String): Throwable = {
+    new UnsupportedOperationException(
+      s"Data source $className does not support streamed $operator")
+  }
+
+  def multiplePathsSpecifiedError(allPaths: Seq[String]): Throwable = {
+    new IllegalArgumentException("Expected exactly one path to be specified, but " +
+      s"got: ${allPaths.mkString(", ")}")
+  }
+
+  def failedToFindDataSourceError(provider: String, error: Throwable): Throwable = {
+    new ClassNotFoundException(
+      s"""
+         |Failed to find data source: $provider. Please find packages at
+         |http://spark.apache.org/third-party-projects.html
+       """.stripMargin, error)
+  }
+
+  def removedClassInSpark2Error(className: String, e: Throwable): Throwable = {
+    new ClassNotFoundException(s"$className was removed in Spark 2.0. " +
+      "Please check if your library is compatible with Spark 2.0", e)
+  }
+
+  def incompatibleDataSourceRegisterError(e: Throwable): Throwable = {
+    new ClassNotFoundException(
+      s"""
+         |Detected an incompatible DataSourceRegister. Please remove the incompatible
+         |library from classpath or upgrade it. Error: ${e.getMessage}
+       """.stripMargin, e)
+  }
+
+  def unrecognizedFileFormatError(format: String): Throwable = {
+    new IllegalStateException(s"unrecognized format $format")
+  }
+
+  def sparkUpgradeInReadingDatesError(
+      format: String, config: String, option: String): SparkUpgradeException = {
+    new SparkUpgradeException("3.0",
+      s"""
+         |reading dates before 1582-10-15 or timestamps before 1900-01-01T00:00:00Z from $format
+         |files can be ambiguous, as the files may be written by Spark 2.x or legacy versions of
+         |Hive, which uses a legacy hybrid calendar that is different from Spark 3.0+'s Proleptic
+         |Gregorian calendar. See more details in SPARK-31404. You can set the SQL config
+         |'$config' or the datasource option '$option' to 'LEGACY' to rebase the datetime values
+         |w.r.t. the calendar difference during reading. To read the datetime values as it is,
+         |set the SQL config '$config' or the datasource option '$option' to 'CORRECTED'.
+       """.stripMargin, null)
+  }
+
+  def sparkUpgradeInWritingDatesError(format: String, config: String): SparkUpgradeException = {
+    new SparkUpgradeException("3.0",
+      s"""
+         |writing dates before 1582-10-15 or timestamps before 1900-01-01T00:00:00Z into $format
+         |files can be dangerous, as the files may be read by Spark 2.x or legacy versions of Hive
+         |later, which uses a legacy hybrid calendar that is different from Spark 3.0+'s Proleptic
+         |Gregorian calendar. See more details in SPARK-31404. You can set $config to 'LEGACY' to
+         |rebase the datetime values w.r.t. the calendar difference during writing, to get maximum
+         |interoperability. Or set $config to 'CORRECTED' to write the datetime values as it is,
+         |if you are 100% sure that the written files will only be read by Spark 3.0+ or other
+         |systems that use Proleptic Gregorian calendar.
+       """.stripMargin, null)
+  }
+
+  def buildReaderUnsupportedForFileFormatError(format: String): Throwable = {
+    new UnsupportedOperationException(s"buildReader is not supported for $format")
+  }
+
+  def jobAbortedError(cause: Throwable): Throwable = {
+    new SparkException("Job aborted.", cause)
+  }
+
+  def taskFailedWhileWritingRowsError(cause: Throwable): Throwable = {
+    new SparkException("Task failed while writing rows.", cause)
+  }
+
+  def readCurrentFileNotFoundError(e: FileNotFoundException): Throwable = {
+    new FileNotFoundException(
+      s"""
+         |${e.getMessage}\n
+         |It is possible the underlying files have been updated. You can explicitly invalidate
+         |the cache in Spark by running 'REFRESH TABLE tableName' command in SQL or by
+         |recreating the Dataset/DataFrame involved.
+       """.stripMargin)
+  }
+
+  def unsupportedSaveModeError(saveMode: String, pathExists: Boolean): Throwable = {
+    new IllegalStateException(s"unsupported save mode $saveMode ($pathExists)")
+  }
+
+  def cannotClearOutputDirectoryError(staticPrefixPath: Path): Throwable = {
+    new IOException(s"Unable to clear output directory $staticPrefixPath prior to writing to it")
+  }
+
+  def cannotClearPartitionDirectoryError(path: Path): Throwable = {
+    new IOException(s"Unable to clear partition directory $path prior to writing to it")
+  }
+
+  def failedToCastValueToDataTypeForPartitionColumnError(
+      value: String, dataType: DataType, columnName: String): Throwable = {
+    new RuntimeException(s"Failed to cast value `$value` to " +
+      s"`$dataType` for partition column `$columnName`")
+  }
+
+  def endOfStreamError(): Throwable = {
+    new NoSuchElementException("End of stream")
+  }
+
+  def fallbackV1RelationReportsInconsistentSchemaError(
+      v2Schema: StructType, v1Schema: StructType): Throwable = {
+    new IllegalArgumentException(
+      "The fallback v1 relation reports inconsistent schema:\n" +
+        "Schema of v2 scan:     " + v2Schema + "\n" +
+        "Schema of v1 relation: " + v1Schema)
+  }
+
+  def cannotDropNonemptyNamespaceError(namespace: Seq[String]): Throwable = {
+    new SparkException(
+      s"Cannot drop a non-empty namespace: ${namespace.quoted}. " +
+        "Use CASCADE option to drop a non-empty namespace.")
+  }
+
+  def noRecordsFromEmptyDataReaderError(): Throwable = {
+    new IOException("No records should be returned from EmptyDataReader")
+  }
+
+  def fileNotFoundError(e: FileNotFoundException): Throwable = {
+    new FileNotFoundException(
+      e.getMessage + "\n" +
+        "It is possible the underlying files have been updated. " +
+        "You can explicitly invalidate the cache in Spark by " +
+        "recreating the Dataset/DataFrame involved.")
+  }
+
+  def unsupportedSchemaColumnConvertError(
+      filePath: String,
+      column: String,
+      logicalType: String,
+      physicalType: String,
+      e: Exception): Throwable = {
+    val message = "Parquet column cannot be converted in " +
+      s"file $filePath. Column: $column, " +
+      s"Expected: $logicalType, Found: $physicalType"
+    new QueryExecutionException(message, e)
+  }
+
+  def cannotReadParquetFilesError(e: Exception): Throwable = {
+    val message = "Encounter error while reading parquet files. " +
+      "One possible cause: Parquet column cannot be converted in the " +
+      "corresponding files. Details: "
+    new QueryExecutionException(message, e)
+  }
+
+  def cannotCreateColumnarReaderError(): Throwable = {
+    new UnsupportedOperationException("Cannot create columnar reader.")
+  }
+
+  def invalidNamespaceNameError(namespace: Array[String]): Throwable = {
+    new IllegalArgumentException(s"Invalid namespace name: ${namespace.quoted}")
+  }
+
+  def unsupportedPartitionTransformError(transform: Transform): Throwable = {
+    new UnsupportedOperationException(
+      s"SessionCatalog does not support partition transform: $transform")
+  }
+
+  def missingDatabaseLocationError(): Throwable = {
+    new IllegalArgumentException("Missing database location")
+  }
+
+  def cannotRemoveReservedPropertyError(property: String): Throwable = {
+    new UnsupportedOperationException(s"Cannot remove reserved property: $property")
+  }
+
+  def namespaceNotEmptyError(namespace: Array[String]): Throwable = {
+    new IllegalStateException(s"Namespace ${namespace.quoted} is not empty")
+  }
+
+  def writingJobFailedError(cause: Throwable): Throwable = {
+    new SparkException("Writing job failed.", cause)
+  }
+
+  def writingJobAbortedError(e: Throwable): Throwable = {
+    new SparkException("Writing job aborted.", e)
+  }
+
+  def commitDeniedError(
+      partId: Int, taskId: Long, attemptId: Int, stageId: Int, stageAttempt: Int): Throwable = {
+    val message = s"Commit denied for partition $partId (task $taskId, attempt $attemptId, " +
+      s"stage $stageId.$stageAttempt)"
+    new CommitDeniedException(message, stageId, partId, attemptId)
+  }
+
+  def unsupportedTableWritesError(ident: Identifier): Throwable = {
+    new SparkException(
+      s"Table implementation does not support writes: ${ident.quoted}")
+  }
+
+  def cannotCreateJDBCTableWithPartitionsError(): Throwable = {
+    new UnsupportedOperationException("Cannot create JDBC table with partition")
+  }
+
+  def unsupportedUserSpecifiedSchemaError(): Throwable = {
+    new UnsupportedOperationException("user-specified schema")
+  }
+
+  def writeUnsupportedForBinaryFileDataSourceError(): Throwable = {
+    new UnsupportedOperationException("Write is not supported for binary file data source")
+  }
+
+  def fileLengthExceedsMaxLengthError(status: FileStatus, maxLength: Int): Throwable = {
+    new SparkException(
+      s"The length of ${status.getPath} is ${status.getLen}, " +
+        s"which exceeds the max length allowed: ${maxLength}.")
+  }
+
+  def unsupportedFieldNameError(fieldName: String): Throwable = {
+    new RuntimeException(s"Unsupported field name: ${fieldName}")
+  }
+
+  def cannotSpecifyBothJdbcTableNameAndQueryError(
+      jdbcTableName: String, jdbcQueryString: String): Throwable = {
+    new IllegalArgumentException(
+      s"Both '$jdbcTableName' and '$jdbcQueryString' can not be specified at the same time.")
+  }
+
+  def missingJdbcTableNameAndQueryError(
+      jdbcTableName: String, jdbcQueryString: String): Throwable = {
+    new IllegalArgumentException(
+      s"Option '$jdbcTableName' or '$jdbcQueryString' is required."
+    )
+  }
+
+  def emptyOptionError(optionName: String): Throwable = {
+    new IllegalArgumentException(s"Option `$optionName` can not be empty.")
+  }
+
+  def invalidJdbcTxnIsolationLevelError(jdbcTxnIsolationLevel: String, value: String): Throwable = {
+    new IllegalArgumentException(
+      s"Invalid value `$value` for parameter `$jdbcTxnIsolationLevel`. This can be " +
+        "`NONE`, `READ_UNCOMMITTED`, `READ_COMMITTED`, `REPEATABLE_READ` or `SERIALIZABLE`.")
+  }
+
+  def cannotGetJdbcTypeError(dt: DataType): Throwable = {
+    new IllegalArgumentException(s"Can't get JDBC type for ${dt.catalogString}")
+  }
+
+  def unrecognizedSqlTypeError(sqlType: Int): Throwable = {
+    new SQLException(s"Unrecognized SQL type $sqlType")
+  }
+
+  def unsupportedJdbcTypeError(content: String): Throwable = {
+    new SQLException(s"Unsupported type $content")
+  }
+
+  def unsupportedArrayElementTypeBasedOnBinaryError(dt: DataType): Throwable = {
+    new IllegalArgumentException(s"Unsupported array element " +
+      s"type ${dt.catalogString} based on binary")
+  }
+
+  def nestedArraysUnsupportedError(): Throwable = {
+    new IllegalArgumentException("Nested arrays unsupported")
+  }
+
+  def cannotTranslateNonNullValueForFieldError(pos: Int): Throwable = {
+    new IllegalArgumentException(s"Can't translate non-null value for field $pos")
+  }
+
+  def invalidJdbcNumPartitionsError(n: Int, jdbcNumPartitions: String): Throwable = {
+    new IllegalArgumentException(
+      s"Invalid value `$n` for parameter `$jdbcNumPartitions` in table writing " +
+        "via JDBC. The minimum value is 1.")
+  }
+
+  def transactionUnsupportedByJdbcServerError(): Throwable = {
+    new SQLFeatureNotSupportedException("The target JDBC server does not support " +
+      "transaction and can only support ALTER TABLE with a single action.")
+  }
+
+  def dataTypeUnsupportedYetError(dataType: DataType): Throwable = {
+    new UnsupportedOperationException(s"$dataType is not supported yet.")
+  }
+
+  def unsupportedOperationForDataTypeError(dataType: DataType): Throwable = {
+    new UnsupportedOperationException(s"DataType: ${dataType.catalogString}")
+  }
+
+  def inputFilterNotFullyConvertibleError(owner: String): Throwable = {
+    new SparkException(s"The input filter of $owner should be fully convertible.")
+  }
+
+  def cannotReadFooterForFileError(file: Path, e: IOException): Throwable = {
+    new SparkException(s"Could not read footer for file: $file", e)
+  }
+
+  def cannotReadFooterForFileError(file: FileStatus, e: RuntimeException): Throwable = {
+    new IOException(s"Could not read footer for file: $file", e)
+  }
+
+  def foundDuplicateFieldInCaseInsensitiveModeError(
+      requiredFieldName: String, matchedOrcFields: String): Throwable = {
+    new RuntimeException(
+      s"""
+         |Found duplicate field(s) "$requiredFieldName": $matchedOrcFields
+         |in case-insensitive mode
+       """.stripMargin.replaceAll("\n", " "))
+  }
+
+  def failedToMergeIncompatibleSchemasError(
+      left: StructType, right: StructType, e: Throwable): Throwable = {
+    new SparkException(s"Failed to merge incompatible schemas $left and $right", e)
+  }
+
+  def ddlUnsupportedTemporarilyError(ddl: String): Throwable = {
+    new UnsupportedOperationException(s"$ddl is not supported temporarily.")
+  }
+
+  def operatingOnCanonicalizationPlanError(): Throwable = {
+    new IllegalStateException("operating on canonicalization plan")
+  }
+
+  def executeBroadcastTimeoutError(timeout: Long): Throwable = {
+    new SparkException(
+      s"""
+         |Could not execute broadcast in $timeout secs. You can increase the timeout
+         |for broadcasts via ${SQLConf.BROADCAST_TIMEOUT.key} or disable broadcast join
+         |by setting ${SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key} to -1
+       """.stripMargin.replaceAll("\n", " "))
+  }
+
+  def cannotCompareCostWithTargetCostError(cost: String): Throwable = {
+    new IllegalArgumentException(s"Could not compare cost with $cost")
+  }
+
+  def unsupportedDataTypeError(dt: DataType): Throwable = {
+    new UnsupportedOperationException(s"Unsupported data type: ${dt.catalogString}")
+  }
+
+  def notSupportTypeError(dataType: DataType): Throwable = {
+    new Exception(s"not support type: $dataType")
+  }
+
+  def notSupportNonPrimitiveTypeError(): Throwable = {
+    new RuntimeException("Not support non-primitive type now")
+  }
+
+  def unsupportedTypeError(dataType: DataType): Throwable = {
+    new Exception(s"Unsupported type: ${dataType.catalogString}")
+  }
+
+  def useDictionaryEncodingWhenDictionaryOverflowError(): Throwable = {
+    new IllegalStateException(
+      "Dictionary encoding should not be used because of dictionary overflow.")
+  }
+
+  def endOfIteratorError(): Throwable = {
+    new NoSuchElementException("End of the iterator")
+  }
+
+  def cannotAllocateMemoryToGrowBytesToBytesMapError(): Throwable = {
+    new IOException("Could not allocate memory to grow BytesToBytesMap")
+  }
+
+  def cannotAcquireMemoryToBuildLongHashedRelationError(size: Long, got: Long): Throwable = {
+    new SparkException(s"Can't acquire $size bytes memory to build hash relation, " +
+      s"got $got bytes")
+  }
+
+  def cannotAcquireMemoryToBuildUnsafeHashedRelationError(): Throwable = {
+    new SparkOutOfMemoryError("There is not enough memory to build hash map")
+  }
+
+  def rowLargerThan256MUnsupportedError(): Throwable = {
+    new UnsupportedOperationException("Does not support row that is larger than 256M")
+  }
+
+  def cannotBuildHashedRelationWithUniqueKeysExceededError(): Throwable = {
+    new UnsupportedOperationException(
+      "Cannot build HashedRelation with more than 1/3 billions unique keys")
+  }
+
+  def cannotBuildHashedRelationLargerThan8GError(): Throwable = {
+    new UnsupportedOperationException(
+      "Can not build a HashedRelation that is larger than 8G")
+  }
+
+  def failedToPushRowIntoRowQueueError(rowQueue: String): Throwable = {
+    new SparkException(s"failed to push a row into $rowQueue")
+  }
+
+  def unexpectedWindowFunctionFrameError(frame: String): Throwable = {
+    new RuntimeException(s"Unexpected window function frame $frame.")
+  }
+
+  def cannotParseStatisticAsPercentileError(
+      stats: String, e: NumberFormatException): Throwable = {
+    new IllegalArgumentException(s"Unable to parse $stats as a percentile", e)
+  }
+
+  def statisticNotRecognizedError(stats: String): Throwable = {
+    new IllegalArgumentException(s"$stats is not a recognised statistic")
+  }
+
+  def unknownColumnError(unknownColumn: String): Throwable = {
+    new IllegalArgumentException(s"Unknown column: $unknownColumn")
+  }
+
+  def unexpectedAccumulableUpdateValueError(o: Any): Throwable = {
+    new IllegalArgumentException(s"Unexpected: $o")
+  }
+
+  def unscaledValueTooLargeForPrecisionError(): Throwable = {
+    new ArithmeticException("Unscaled value too large for precision")
+  }
+
+  def decimalPrecisionExceedsMaxPrecisionError(precision: Int, maxPrecision: Int): Throwable = {
+    new ArithmeticException(
+      s"Decimal precision $precision exceeds max precision $maxPrecision")
+  }
+
+  def outOfDecimalTypeRangeError(str: UTF8String): Throwable = {
+    new ArithmeticException(s"out of decimal type range: $str")
+  }
+
+  def unsupportedArrayTypeError(clazz: Class[_]): Throwable = {
+    new RuntimeException(s"Do not support array of type $clazz.")
+  }
+
+  def unsupportedJavaTypeError(clazz: Class[_]): Throwable = {
+    new RuntimeException(s"Do not support type $clazz.")
+  }
+
+  def failedParsingStructTypeError(raw: String): Throwable = {
+    new RuntimeException(s"Failed parsing ${StructType.simpleString}: $raw")
+  }
+
+  def failedMergingFieldsError(leftName: String, rightName: String, e: Throwable): Throwable = {
+    new SparkException(s"Failed to merge fields '$leftName' and '$rightName'. ${e.getMessage}")
+  }
+
+  def cannotMergeDecimalTypesWithIncompatiblePrecisionAndScaleError(
+      leftPrecision: Int, rightPrecision: Int, leftScale: Int, rightScale: Int): Throwable = {
+    new SparkException("Failed to merge decimal types with incompatible " +
+      s"precision $leftPrecision and $rightPrecision & scale $leftScale and $rightScale")
+  }
+
+  def cannotMergeDecimalTypesWithIncompatiblePrecisionError(
+      leftPrecision: Int, rightPrecision: Int): Throwable = {
+    new SparkException("Failed to merge decimal types with incompatible " +
+      s"precision $leftPrecision and $rightPrecision")
+  }
+
+  def cannotMergeDecimalTypesWithIncompatibleScaleError(
+      leftScale: Int, rightScale: Int): Throwable = {
+    new SparkException("Failed to merge decimal types with incompatible " +
+      s"scala $leftScale and $rightScale")
+  }
+
+  def cannotMergeIncompatibleDataTypesError(left: DataType, right: DataType): Throwable = {
+    new SparkException(s"Failed to merge incompatible data types ${left.catalogString}" +
+      s" and ${right.catalogString}")
+  }
+
+  def exceedMapSizeLimitError(size: Int): Throwable = {
+    new RuntimeException(s"Unsuccessful attempt to build maps with $size elements " +
+      s"due to exceeding the map size limit ${ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH}.")
+  }
+
+  def duplicateMapKeyFoundError(key: Any): Throwable = {
+    new RuntimeException(s"Duplicate map key $key was found, please check the input " +
+      "data. If you want to remove the duplicated keys, you can set " +
+      s"${SQLConf.MAP_KEY_DEDUP_POLICY.key} to ${SQLConf.MapKeyDedupPolicy.LAST_WIN} so that " +
+      "the key inserted at last takes precedence.")
+  }
+
+  def mapDataKeyArrayLengthDiffersFromValueArrayLengthError(): Throwable = {
+    new RuntimeException("The key array and value array of MapData must have the same length.")
+  }
+
+  def fieldDiffersFromDerivedLocalDateError(
+      field: ChronoField, actual: Int, expected: Int, candidate: LocalDate): Throwable = {
+    new DateTimeException(s"Conflict found: Field $field $actual differs from" +
+      s" $field $expected derived from $candidate")
+  }
+
+  def failToParseDateTimeInNewParserError(s: String, e: Throwable): Throwable = {
+    new SparkUpgradeException("3.0", s"Fail to parse '$s' in the new parser. You can " +
+      s"set ${SQLConf.LEGACY_TIME_PARSER_POLICY.key} to LEGACY to restore the behavior " +
+      s"before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.", e)
+  }
+
+  def failToFormatDateTimeInNewFormatterError(
+      resultCandidate: String, e: Throwable): Throwable = {
+    new SparkUpgradeException("3.0",
+      s"""
+         |Fail to format it to '$resultCandidate' in the new formatter. You can set
+         |${SQLConf.LEGACY_TIME_PARSER_POLICY.key} to LEGACY to restore the behavior before
+         |Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+       """.stripMargin.replaceAll("\n", " "), e)
+  }
+
+  def failToRecognizePatternInDateTimeFormatterError(
+      pattern: String, e: Throwable): Throwable = {
+    new SparkUpgradeException("3.0", s"Fail to recognize '$pattern' pattern in the" +
+      s" DateTimeFormatter. 1) You can set ${SQLConf.LEGACY_TIME_PARSER_POLICY.key} to LEGACY" +
+      s" to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern" +
+      s" with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html",
+      e)
+  }
+
+  def cannotCastUTF8StringToDataTypeError(s: UTF8String, to: DataType): Throwable = {
+    new DateTimeException(s"Cannot cast $s to $to.")
+  }
+
+  def registeringStreamingQueryListenerError(e: Exception): Throwable = {
+    new SparkException("Exception when registering StreamingQueryListener", e)
+  }
+
+  def concurrentQueryInstanceError(): Throwable = {
+    new ConcurrentModificationException(
+      "Another instance of this query was just started by a concurrent session.")
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala
new file mode 100644
index 0000000000000..b714f57875275
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala
@@ -0,0 +1,373 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.errors
+
+import org.antlr.v4.runtime.ParserRuleContext
+
+import org.apache.spark.sql.catalyst.parser.ParseException
+import org.apache.spark.sql.catalyst.parser.SqlBaseParser._
+import org.apache.spark.sql.catalyst.trees.Origin
+
+/**
+ * Object for grouping all error messages of the query parsing.
+ * Currently it includes all ParseException.
+ */
+object QueryParsingErrors {
+
+  def invalidInsertIntoError(ctx: InsertIntoContext): Throwable = {
+    new ParseException("Invalid InsertIntoContext", ctx)
+  }
+
+  def insertOverwriteDirectoryUnsupportedError(ctx: InsertIntoContext): Throwable = {
+    new ParseException("INSERT OVERWRITE DIRECTORY is not supported", ctx)
+  }
+
+  def columnAliasInOperationNotAllowedError(op: String, ctx: TableAliasContext): Throwable = {
+    new ParseException(s"Columns aliases are not allowed in $op.", ctx.identifierList())
+  }
+
+  def emptySourceForMergeError(ctx: MergeIntoTableContext): Throwable = {
+    new ParseException("Empty source for merge: you should specify a source" +
+      " table/subquery in merge.", ctx.source)
+  }
+
+  def unrecognizedMatchedActionError(ctx: MatchedClauseContext): Throwable = {
+    new ParseException(s"Unrecognized matched action: ${ctx.matchedAction().getText}",
+      ctx.matchedAction())
+  }
+
+  def insertedValueNumberNotMatchFieldNumberError(ctx: NotMatchedClauseContext): Throwable = {
+    new ParseException("The number of inserted values cannot match the fields.",
+      ctx.notMatchedAction())
+  }
+
+  def unrecognizedNotMatchedActionError(ctx: NotMatchedClauseContext): Throwable = {
+    new ParseException(s"Unrecognized not matched action: ${ctx.notMatchedAction().getText}",
+      ctx.notMatchedAction())
+  }
+
+  def mergeStatementWithoutWhenClauseError(ctx: MergeIntoTableContext): Throwable = {
+    new ParseException("There must be at least one WHEN clause in a MERGE statement", ctx)
+  }
+
+  def nonLastMatchedClauseOmitConditionError(ctx: MergeIntoTableContext): Throwable = {
+    new ParseException("When there are more than one MATCHED clauses in a MERGE " +
+      "statement, only the last MATCHED clause can omit the condition.", ctx)
+  }
+
+  def nonLastNotMatchedClauseOmitConditionError(ctx: MergeIntoTableContext): Throwable = {
+    new ParseException("When there are more than one NOT MATCHED clauses in a MERGE " +
+      "statement, only the last NOT MATCHED clause can omit the condition.", ctx)
+  }
+
+  def emptyPartitionKeyError(key: String, ctx: PartitionSpecContext): Throwable = {
+    new ParseException(s"Found an empty partition key '$key'.", ctx)
+  }
+
+  def combinationQueryResultClausesUnsupportedError(ctx: QueryOrganizationContext): Throwable = {
+    new ParseException(
+      "Combination of ORDER BY/SORT BY/DISTRIBUTE BY/CLUSTER BY is not supported", ctx)
+  }
+
+  def distributeByUnsupportedError(ctx: QueryOrganizationContext): Throwable = {
+    new ParseException("DISTRIBUTE BY is not supported", ctx)
+  }
+
+  def transformNotSupportQuantifierError(ctx: ParserRuleContext): Throwable = {
+    new ParseException("TRANSFORM does not support DISTINCT/ALL in inputs", ctx)
+  }
+
+  def transformWithSerdeUnsupportedError(ctx: ParserRuleContext): Throwable = {
+    new ParseException("TRANSFORM with serde is only supported in hive mode", ctx)
+  }
+
+  def lateralWithPivotInFromClauseNotAllowedError(ctx: FromClauseContext): Throwable = {
+    new ParseException("LATERAL cannot be used together with PIVOT in FROM clause", ctx)
+  }
+
+  def repetitiveWindowDefinitionError(name: String, ctx: WindowClauseContext): Throwable = {
+    new ParseException(s"The definition of window '$name' is repetitive", ctx)
+  }
+
+  def invalidWindowReferenceError(name: String, ctx: WindowClauseContext): Throwable = {
+    new ParseException(s"Window reference '$name' is not a window specification", ctx)
+  }
+
+  def cannotResolveWindowReferenceError(name: String, ctx: WindowClauseContext): Throwable = {
+    new ParseException(s"Cannot resolve window reference '$name'", ctx)
+  }
+
+  def joinCriteriaUnimplementedError(join: JoinCriteriaContext, ctx: RelationContext): Throwable = {
+    new ParseException(s"Unimplemented joinCriteria: $join", ctx)
+  }
+
+  def naturalCrossJoinUnsupportedError(ctx: RelationContext): Throwable = {
+    new ParseException("NATURAL CROSS JOIN is not supported", ctx)
+  }
+
+  def emptyInputForTableSampleError(ctx: ParserRuleContext): Throwable = {
+    new ParseException("TABLESAMPLE does not accept empty inputs.", ctx)
+  }
+
+  def tableSampleByBytesUnsupportedError(msg: String, ctx: SampleMethodContext): Throwable = {
+    new ParseException(s"TABLESAMPLE($msg) is not supported", ctx)
+  }
+
+  def invalidByteLengthLiteralError(bytesStr: String, ctx: SampleByBytesContext): Throwable = {
+    new ParseException(s"$bytesStr is not a valid byte length literal, " +
+        "expected syntax: DIGIT+ ('B' | 'K' | 'M' | 'G')", ctx)
+  }
+
+  def invalidEscapeStringError(ctx: PredicateContext): Throwable = {
+    new ParseException("Invalid escape string. Escape string must contain only one character.", ctx)
+  }
+
+  def trimOptionUnsupportedError(trimOption: Int, ctx: TrimContext): Throwable = {
+    new ParseException("Function trim doesn't support with " +
+      s"type $trimOption. Please use BOTH, LEADING or TRAILING as trim type", ctx)
+  }
+
+  def functionNameUnsupportedError(functionName: String, ctx: ParserRuleContext): Throwable = {
+    new ParseException(s"Unsupported function name '$functionName'", ctx)
+  }
+
+  def cannotParseValueTypeError(
+      valueType: String, value: String, ctx: TypeConstructorContext): Throwable = {
+    new ParseException(s"Cannot parse the $valueType value: $value", ctx)
+  }
+
+  def cannotParseIntervalValueError(value: String, ctx: TypeConstructorContext): Throwable = {
+    new ParseException(s"Cannot parse the INTERVAL value: $value", ctx)
+  }
+
+  def literalValueTypeUnsupportedError(
+      valueType: String, ctx: TypeConstructorContext): Throwable = {
+    new ParseException(s"Literals of type '$valueType' are currently not supported.", ctx)
+  }
+
+  def parsingValueTypeError(
+      e: IllegalArgumentException, valueType: String, ctx: TypeConstructorContext): Throwable = {
+    val message = Option(e.getMessage).getOrElse(s"Exception parsing $valueType")
+    new ParseException(message, ctx)
+  }
+
+  def invalidNumericLiteralRangeError(rawStrippedQualifier: String, minValue: BigDecimal,
+      maxValue: BigDecimal, typeName: String, ctx: NumberContext): Throwable = {
+    new ParseException(s"Numeric literal $rawStrippedQualifier does not " +
+      s"fit in range [$minValue, $maxValue] for type $typeName", ctx)
+  }
+
+  def moreThanOneFromToUnitInIntervalLiteralError(ctx: ParserRuleContext): Throwable = {
+    new ParseException("Can only have a single from-to unit in the interval literal syntax", ctx)
+  }
+
+  def invalidIntervalLiteralError(ctx: IntervalContext): Throwable = {
+    new ParseException("at least one time unit should be given for interval literal", ctx)
+  }
+
+  def invalidIntervalFormError(value: String, ctx: MultiUnitsIntervalContext): Throwable = {
+    new ParseException("Can only use numbers in the interval value part for" +
+      s" multiple unit value pairs interval form, but got invalid value: $value", ctx)
+  }
+
+  def invalidFromToUnitValueError(ctx: IntervalValueContext): Throwable = {
+    new ParseException("The value of from-to unit must be a string", ctx)
+  }
+
+  def fromToIntervalUnsupportedError(
+      from: String, to: String, ctx: UnitToUnitIntervalContext): Throwable = {
+    new ParseException(s"Intervals FROM $from TO $to are not supported.", ctx)
+  }
+
+  def dataTypeUnsupportedError(dataType: String, ctx: PrimitiveDataTypeContext): Throwable = {
+    new ParseException(s"DataType $dataType is not supported.", ctx)
+  }
+
+  def partitionTransformNotExpectedError(
+      name: String, describe: String, ctx: ApplyTransformContext): Throwable = {
+    new ParseException(s"Expected a column reference for transform $name: $describe", ctx)
+  }
+
+  def tooManyArgumentsForTransformError(name: String, ctx: ApplyTransformContext): Throwable = {
+    new ParseException(s"Too many arguments for transform $name", ctx)
+  }
+
+  def notEnoughArgumentsForTransformError(name: String, ctx: ApplyTransformContext): Throwable = {
+    new ParseException(s"Not enough arguments for transform $name", ctx)
+  }
+
+  def invalidBucketsNumberError(describe: String, ctx: ApplyTransformContext): Throwable = {
+    new ParseException(s"Invalid number of buckets: $describe", ctx)
+  }
+
+  def invalidTransformArgumentError(ctx: TransformArgumentContext): Throwable = {
+    new ParseException("Invalid transform argument", ctx)
+  }
+
+  def cannotCleanReservedNamespacePropertyError(
+      property: String, ctx: ParserRuleContext, msg: String): Throwable = {
+    new ParseException(s"$property is a reserved namespace property, $msg.", ctx)
+  }
+
+  def propertiesAndDbPropertiesBothSpecifiedError(ctx: CreateNamespaceContext): Throwable = {
+    new ParseException("Either PROPERTIES or DBPROPERTIES is allowed.", ctx)
+  }
+
+  def fromOrInNotAllowedInShowDatabasesError(ctx: ShowNamespacesContext): Throwable = {
+    new ParseException(s"FROM/IN operator is not allowed in SHOW DATABASES", ctx)
+  }
+
+  def cannotCleanReservedTablePropertyError(
+      property: String, ctx: ParserRuleContext, msg: String): Throwable = {
+    new ParseException(s"$property is a reserved table property, $msg.", ctx)
+  }
+
+  def duplicatedTablePathsFoundError(
+      pathOne: String, pathTwo: String, ctx: ParserRuleContext): Throwable = {
+    new ParseException(s"Duplicated table paths found: '$pathOne' and '$pathTwo'. LOCATION" +
+      s" and the case insensitive key 'path' in OPTIONS are all used to indicate the custom" +
+      s" table path, you can only specify one of them.", ctx)
+  }
+
+  def storedAsAndStoredByBothSpecifiedError(ctx: CreateFileFormatContext): Throwable = {
+    new ParseException("Expected either STORED AS or STORED BY, not both", ctx)
+  }
+
+  def operationInHiveStyleCommandUnsupportedError(operation: String,
+      command: String, ctx: StatementContext, msgOpt: Option[String] = None): Throwable = {
+    val basicError = s"$operation is not supported in Hive-style $command"
+    val msg = if (msgOpt.isDefined) {
+      s"$basicError, ${msgOpt.get}."
+    } else {
+      basicError
+    }
+    new ParseException(msg, ctx)
+  }
+
+  def operationNotAllowedError(message: String, ctx: ParserRuleContext): Throwable = {
+    new ParseException(s"Operation not allowed: $message", ctx)
+  }
+
+  def descColumnForPartitionUnsupportedError(ctx: DescribeRelationContext): Throwable = {
+    new ParseException("DESC TABLE COLUMN for a specific partition is not supported", ctx)
+  }
+
+  def incompletePartitionSpecificationError(
+      key: String, ctx: DescribeRelationContext): Throwable = {
+    new ParseException(s"PARTITION specification is incomplete: `$key`", ctx)
+  }
+
+  def computeStatisticsNotExpectedError(ctx: IdentifierContext): Throwable = {
+    new ParseException(s"Expected `NOSCAN` instead of `${ctx.getText}`", ctx)
+  }
+
+  def addCatalogInCacheTableAsSelectNotAllowedError(
+      quoted: String, ctx: CacheTableContext): Throwable = {
+    new ParseException(s"It is not allowed to add catalog/namespace prefix $quoted to " +
+      "the table name in CACHE TABLE AS SELECT", ctx)
+  }
+
+  def showFunctionsUnsupportedError(identifier: String, ctx: IdentifierContext): Throwable = {
+    new ParseException(s"SHOW $identifier FUNCTIONS not supported", ctx)
+  }
+
+  def duplicateCteDefinitionNamesError(duplicateNames: String, ctx: CtesContext): Throwable = {
+    new ParseException(s"CTE definition can't have duplicate names: $duplicateNames.", ctx)
+  }
+
+  def sqlStatementUnsupportedError(sqlText: String, position: Origin): Throwable = {
+    new ParseException(Option(sqlText), "Unsupported SQL statement", position, position)
+  }
+
+  def unquotedIdentifierError(ident: String, ctx: ErrorIdentContext): Throwable = {
+    new ParseException(s"Possibly unquoted identifier $ident detected. " +
+      s"Please consider quoting it with back-quotes as `$ident`", ctx)
+  }
+
+  def duplicateClausesError(clauseName: String, ctx: ParserRuleContext): Throwable = {
+    new ParseException(s"Found duplicate clauses: $clauseName", ctx)
+  }
+
+  def duplicateKeysError(key: String, ctx: ParserRuleContext): Throwable = {
+    new ParseException(s"Found duplicate keys '$key'.", ctx)
+  }
+
+  def unexpectedFomatForSetConfigurationError(ctx: SetConfigurationContext): Throwable = {
+    new ParseException(
+      s"""
+         |Expected format is 'SET', 'SET key', or 'SET key=value'. If you want to include
+         |special characters in key, or include semicolon in value, please use quotes,
+         |e.g., SET `ke y`=`v;alue`.
+       """.stripMargin.replaceAll("\n", " "), ctx)
+  }
+
+  def invalidPropertyKeyForSetQuotedConfigurationError(
+      keyCandidate: String, valueStr: String, ctx: SetQuotedConfigurationContext): Throwable = {
+    new ParseException(s"'$keyCandidate' is an invalid property key, please " +
+      s"use quotes, e.g. SET `$keyCandidate`=`$valueStr`", ctx)
+  }
+
+  def invalidPropertyValueForSetQuotedConfigurationError(
+      valueCandidate: String, keyStr: String, ctx: SetQuotedConfigurationContext): Throwable = {
+    new ParseException(s"'$valueCandidate' is an invalid property value, please " +
+      s"use quotes, e.g. SET `$keyStr`=`$valueCandidate`", ctx)
+  }
+
+  def unexpectedFormatForResetConfigurationError(ctx: ResetConfigurationContext): Throwable = {
+    new ParseException(
+      s"""
+         |Expected format is 'RESET' or 'RESET key'. If you want to include special characters
+         |in key, please use quotes, e.g., RESET `ke y`.
+       """.stripMargin.replaceAll("\n", " "), ctx)
+  }
+
+  def intervalValueOutOfRangeError(ctx: IntervalContext): Throwable = {
+    new ParseException("The interval value must be in the range of [-18, +18] hours" +
+      " with second precision", ctx)
+  }
+
+  def invalidTimeZoneDisplacementValueError(ctx: SetTimeZoneContext): Throwable = {
+    new ParseException("Invalid time zone displacement value", ctx)
+  }
+
+  def createTempTableNotSpecifyProviderError(ctx: CreateTableContext): Throwable = {
+    new ParseException("CREATE TEMPORARY TABLE without a provider is not allowed.", ctx)
+  }
+
+  def rowFormatNotUsedWithStoredAsError(ctx: CreateTableLikeContext): Throwable = {
+    new ParseException("'ROW FORMAT' must be used with 'STORED AS'", ctx)
+  }
+
+  def useDefinedRecordReaderOrWriterClassesError(ctx: ParserRuleContext): Throwable = {
+    new ParseException(
+      "Unsupported operation: Used defined record reader/writer classes.", ctx)
+  }
+
+  def directoryPathAndOptionsPathBothSpecifiedError(ctx: InsertOverwriteDirContext): Throwable = {
+    new ParseException(
+      "Directory path and 'path' in OPTIONS should be specified one, but not both", ctx)
+  }
+
+  def unsupportedLocalFileSchemeError(ctx: InsertOverwriteDirContext): Throwable = {
+    new ParseException("LOCAL is supported only with file: scheme", ctx)
+  }
+
+  def invalidGroupingSetError(element: String, ctx: GroupingAnalyticsContext): Throwable = {
+    new ParseException(s"Empty set in $element grouping sets is not supported.", ctx)
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecutionException.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/execution/QueryExecutionException.scala
similarity index 100%
rename from sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecutionException.scala
rename to sql/catalyst/src/main/scala/org/apache/spark/sql/execution/QueryExecutionException.scala
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Implicits.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Implicits.scala
index 86ef867eca547..efd3ffebf5c1f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Implicits.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Implicits.scala
@@ -19,8 +19,12 @@ package org.apache.spark.sql.execution.datasources.v2
 
 import scala.collection.JavaConverters._
 
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.connector.catalog.{SupportsDelete, SupportsRead, SupportsWrite, Table, TableCapability}
+import org.apache.spark.sql.catalyst.analysis.{PartitionSpec, ResolvedPartitionSpec, UnresolvedPartitionSpec}
+import org.apache.spark.sql.catalyst.expressions.AttributeReference
+import org.apache.spark.sql.catalyst.util.METADATA_COL_ATTR_KEY
+import org.apache.spark.sql.connector.catalog.{MetadataColumn, SupportsAtomicPartitionManagement, SupportsDelete, SupportsPartitionManagement, SupportsRead, SupportsWrite, Table, TableCapability, TruncatableTable}
+import org.apache.spark.sql.errors.QueryCompilationErrors
+import org.apache.spark.sql.types.{MetadataBuilder, StructField, StructType}
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
 object DataSourceV2Implicits {
@@ -30,7 +34,7 @@ object DataSourceV2Implicits {
         case support: SupportsRead =>
           support
         case _ =>
-          throw new AnalysisException(s"Table does not support reads: ${table.name}")
+          throw QueryCompilationErrors.tableDoesNotSupportReadsError(table)
       }
     }
 
@@ -39,7 +43,7 @@ object DataSourceV2Implicits {
         case support: SupportsWrite =>
           support
         case _ =>
-          throw new AnalysisException(s"Table does not support writes: ${table.name}")
+          throw QueryCompilationErrors.tableDoesNotSupportWritesError(table)
       }
     }
 
@@ -48,7 +52,33 @@ object DataSourceV2Implicits {
         case support: SupportsDelete =>
           support
         case _ =>
-          throw new AnalysisException(s"Table does not support deletes: ${table.name}")
+          throw QueryCompilationErrors.tableDoesNotSupportDeletesError(table)
+      }
+    }
+
+    def asTruncatable: TruncatableTable = {
+      table match {
+        case t: TruncatableTable => t
+        case _ =>
+          throw QueryCompilationErrors.tableDoesNotSupportTruncatesError(table)
+      }
+    }
+
+    def asPartitionable: SupportsPartitionManagement = {
+      table match {
+        case support: SupportsPartitionManagement =>
+          support
+        case _ =>
+          throw QueryCompilationErrors.tableDoesNotSupportPartitionManagementError(table)
+      }
+    }
+
+    def asAtomicPartitionable: SupportsAtomicPartitionManagement = {
+      table match {
+        case support: SupportsAtomicPartitionManagement =>
+          support
+        case _ =>
+          throw QueryCompilationErrors.tableDoesNotSupportAtomicPartitionManagementError(table)
       }
     }
 
@@ -57,9 +87,30 @@ object DataSourceV2Implicits {
     def supportsAny(capabilities: TableCapability*): Boolean = capabilities.exists(supports)
   }
 
+  implicit class MetadataColumnsHelper(metadata: Array[MetadataColumn]) {
+    def asStruct: StructType = {
+      val fields = metadata.map { metaCol =>
+        val fieldMeta = new MetadataBuilder().putBoolean(METADATA_COL_ATTR_KEY, true).build()
+        val field = StructField(metaCol.name, metaCol.dataType, metaCol.isNullable, fieldMeta)
+        Option(metaCol.comment).map(field.withComment).getOrElse(field)
+      }
+      StructType(fields)
+    }
+
+    def toAttributes: Seq[AttributeReference] = asStruct.toAttributes
+  }
+
   implicit class OptionsHelper(options: Map[String, String]) {
     def asOptions: CaseInsensitiveStringMap = {
       new CaseInsensitiveStringMap(options.asJava)
     }
   }
+
+  implicit class PartitionSpecsHelper(partSpecs: Seq[PartitionSpec]) {
+    def asUnresolvedPartitionSpecs: Seq[UnresolvedPartitionSpec] =
+      partSpecs.map(_.asInstanceOf[UnresolvedPartitionSpec])
+
+    def asResolvedPartitionSpecs: Seq[ResolvedPartitionSpec] =
+      partSpecs.map(_.asInstanceOf[ResolvedPartitionSpec])
+  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Relation.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Relation.scala
index 45d89498f5ae9..eff0cd8364a84 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Relation.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Relation.scala
@@ -20,11 +20,10 @@ package org.apache.spark.sql.execution.datasources.v2
 import org.apache.spark.sql.catalyst.analysis.{MultiInstanceRelation, NamedRelation}
 import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
 import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, Statistics}
-import org.apache.spark.sql.catalyst.util.truncatedString
-import org.apache.spark.sql.connector.catalog.{CatalogPlugin, Identifier, Table, TableCapability}
-import org.apache.spark.sql.connector.read.{Scan, ScanBuilder, Statistics => V2Statistics, SupportsReportStatistics}
+import org.apache.spark.sql.catalyst.util.{truncatedString, CharVarcharUtils}
+import org.apache.spark.sql.connector.catalog.{CatalogPlugin, Identifier, MetadataColumn, SupportsMetadataColumns, Table, TableCapability}
+import org.apache.spark.sql.connector.read.{Scan, Statistics => V2Statistics, SupportsReportStatistics}
 import org.apache.spark.sql.connector.read.streaming.{Offset, SparkDataStream}
-import org.apache.spark.sql.connector.write.WriteBuilder
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 import org.apache.spark.util.Utils
 
@@ -35,8 +34,9 @@ import org.apache.spark.util.Utils
  * @param output the output attributes of this relation.
  * @param catalog catalogPlugin for the table. None if no catalog is specified.
  * @param identifier the identifier for the table. None if no identifier is defined.
- * @param options The options for this table operation. It's used to create fresh [[ScanBuilder]]
- *                and [[WriteBuilder]].
+ * @param options The options for this table operation. It's used to create fresh
+ *                [[org.apache.spark.sql.connector.read.ScanBuilder]] and
+ *                [[org.apache.spark.sql.connector.write.WriteBuilder]].
  */
 case class DataSourceV2Relation(
     table: Table,
@@ -48,6 +48,21 @@ case class DataSourceV2Relation(
 
   import DataSourceV2Implicits._
 
+  override lazy val metadataOutput: Seq[AttributeReference] = table match {
+    case hasMeta: SupportsMetadataColumns =>
+      val resolve = conf.resolver
+      val outputNames = outputSet.map(_.name)
+      def isOutputColumn(col: MetadataColumn): Boolean = {
+        outputNames.exists(name => resolve(col.name, name))
+      }
+      // filter out metadata columns that have names conflicting with output columns. if the table
+      // has a column "line" and the table can produce a metadata column called "line", then the
+      // data column should be returned, not the metadata column.
+      hasMeta.metadataColumns.filterNot(isOutputColumn).toAttributes
+    case _ =>
+      Nil
+  }
+
   override def name: String = table.name()
 
   override def skipSchemaResolution: Boolean = table.supports(TableCapability.ACCEPT_ANY_SCHEMA)
@@ -78,6 +93,14 @@ case class DataSourceV2Relation(
   override def newInstance(): DataSourceV2Relation = {
     copy(output = output.map(_.newInstance()))
   }
+
+  def withMetadataColumns(): DataSourceV2Relation = {
+    if (metadataOutput.nonEmpty) {
+      DataSourceV2Relation(table, output ++ metadataOutput, catalog, identifier, options)
+    } else {
+      this
+    }
+  }
 }
 
 /**
@@ -87,16 +110,16 @@ case class DataSourceV2Relation(
  * plan. This ensures that the stats that are used by the optimizer account for the filters and
  * projection that will be pushed down.
  *
- * @param table a DSv2 [[Table]]
+ * @param relation a [[DataSourceV2Relation]]
  * @param scan a DSv2 [[Scan]]
  * @param output the output attributes of this relation
  */
 case class DataSourceV2ScanRelation(
-    table: Table,
+    relation: DataSourceV2Relation,
     scan: Scan,
     output: Seq[AttributeReference]) extends LeafNode with NamedRelation {
 
-  override def name: String = table.name()
+  override def name: String = relation.table.name()
 
   override def simpleString(maxFields: Int): String = {
     s"RelationV2${truncatedString(output, "[", ", ", "]", maxFields)} $name"
@@ -147,8 +170,10 @@ object DataSourceV2Relation {
       catalog: Option[CatalogPlugin],
       identifier: Option[Identifier],
       options: CaseInsensitiveStringMap): DataSourceV2Relation = {
-    val output = table.schema().toAttributes
-    DataSourceV2Relation(table, output, catalog, identifier, options)
+    // The v2 source may return schema containing char/varchar type. We replace char/varchar
+    // with "annotated" string type here as the query engine doesn't support char/varchar yet.
+    val schema = CharVarcharUtils.replaceCharVarcharWithStringInSchema(table.schema)
+    DataSourceV2Relation(table, schema.toAttributes, catalog, identifier, options)
   }
 
   def create(
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index cf0268773c399..6ce74a67025cc 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -18,23 +18,25 @@
 package org.apache.spark.sql.internal
 
 import java.util.{Locale, NoSuchElementException, Properties, TimeZone}
-import java.util.concurrent.TimeUnit
+import java.util.concurrent.{ConcurrentHashMap, TimeUnit}
 import java.util.concurrent.atomic.AtomicReference
 import java.util.zip.Deflater
 
 import scala.collection.JavaConverters._
 import scala.collection.immutable
 import scala.util.Try
+import scala.util.control.NonFatal
 import scala.util.matching.Regex
 
 import org.apache.hadoop.fs.Path
 
-import org.apache.spark.{SparkContext, TaskContext}
+import org.apache.spark.{SparkConf, SparkContext, TaskContext}
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config._
 import org.apache.spark.internal.config.{IGNORE_MISSING_FILES => SPARK_IGNORE_MISSING_FILES}
 import org.apache.spark.network.util.ByteUnit
 import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.ScalaReflection
 import org.apache.spark.sql.catalyst.analysis.{HintErrorLogger, Resolver}
 import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode
 import org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator
@@ -51,22 +53,22 @@ import org.apache.spark.util.Utils
 
 object SQLConf {
 
-  private[sql] val sqlConfEntries = java.util.Collections.synchronizedMap(
-    new java.util.HashMap[String, ConfigEntry[_]]())
+  private[sql] val sqlConfEntries =
+    new ConcurrentHashMap[String, ConfigEntry[_]]()
 
   val staticConfKeys: java.util.Set[String] =
     java.util.Collections.synchronizedSet(new java.util.HashSet[String]())
 
-  private def register(entry: ConfigEntry[_]): Unit = sqlConfEntries.synchronized {
-    require(!sqlConfEntries.containsKey(entry.key),
-      s"Duplicate SQLConfigEntry. ${entry.key} has been registered")
-    sqlConfEntries.put(entry.key, entry)
-  }
+  private def register(entry: ConfigEntry[_]): Unit = sqlConfEntries.merge(entry.key, entry,
+    (existingConfigEntry, newConfigEntry) => {
+      require(existingConfigEntry == null,
+        s"Duplicate SQLConfigEntry. ${newConfigEntry.key} has been registered")
+      newConfigEntry
+    }
+  )
 
   // For testing only
-  private[sql] def unregister(entry: ConfigEntry[_]): Unit = sqlConfEntries.synchronized {
-    sqlConfEntries.remove(entry.key)
-  }
+  private[sql] def unregister(entry: ConfigEntry[_]): Unit = sqlConfEntries.remove(entry.key)
 
   def buildConf(key: String): ConfigBuilder = ConfigBuilder(key).onCreate(register)
 
@@ -77,6 +79,29 @@ object SQLConf {
     }
   }
 
+  /**
+   * Merge all non-static configs to the SQLConf. For example, when the 1st [[SparkSession]] and
+   * the global [[SharedState]] have been initialized, all static configs have taken affect and
+   * should not be set to other values. Other later created sessions should respect all static
+   * configs and only be able to change non-static configs.
+   */
+  private[sql] def mergeNonStaticSQLConfigs(
+      sqlConf: SQLConf,
+      configs: Map[String, String]): Unit = {
+    for ((k, v) <- configs if !staticConfKeys.contains(k)) {
+      sqlConf.setConfString(k, v)
+    }
+  }
+
+  /**
+   * Extract entries from `SparkConf` and put them in the `SQLConf`
+   */
+  private[sql] def mergeSparkConf(sqlConf: SQLConf, sparkConf: SparkConf): Unit = {
+    sparkConf.getAll.foreach { case (k, v) =>
+      sqlConf.setConfString(k, v)
+    }
+  }
+
   /**
    * Default config. Only used when there is no active SparkSession for the thread.
    * See [[get]] for more information.
@@ -181,6 +206,17 @@ object SQLConf {
     .intConf
     .createWithDefault(100)
 
+  val ANSI_ENABLED = buildConf("spark.sql.ansi.enabled")
+    .doc("When true, Spark SQL uses an ANSI compliant dialect instead of being Hive compliant. " +
+      "For example, Spark will throw an exception at runtime instead of returning null results " +
+      "when the inputs to a SQL operator/function are invalid." +
+      "For full details of this dialect, you can find them in the section \"ANSI Compliance\" of " +
+      "Spark's documentation. Some ANSI dialect features may be not from the ANSI SQL " +
+      "standard directly, but their behaviors align with ANSI SQL's style")
+    .version("3.0.0")
+    .booleanConf
+    .createWithDefault(false)
+
   val OPTIMIZER_EXCLUDED_RULES = buildConf("spark.sql.optimizer.excludedRules")
     .doc("Configures a list of rules to be disabled in the optimizer, in which the rules are " +
       "specified by their rule names and separated by comma. It is not guaranteed that all the " +
@@ -216,32 +252,32 @@ object SQLConf {
         "for using switch statements in InSet must be non-negative and less than or equal to 600")
       .createWithDefault(400)
 
-  val OPTIMIZER_PLAN_CHANGE_LOG_LEVEL = buildConf("spark.sql.optimizer.planChangeLog.level")
+  val PLAN_CHANGE_LOG_LEVEL = buildConf("spark.sql.planChangeLog.level")
     .internal()
     .doc("Configures the log level for logging the change from the original plan to the new " +
       "plan after a rule or batch is applied. The value can be 'trace', 'debug', 'info', " +
       "'warn', or 'error'. The default log level is 'trace'.")
-    .version("3.0.0")
+    .version("3.1.0")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValue(logLevel => Set("TRACE", "DEBUG", "INFO", "WARN", "ERROR").contains(logLevel),
-      "Invalid value for 'spark.sql.optimizer.planChangeLog.level'. Valid values are " +
+      "Invalid value for 'spark.sql.planChangeLog.level'. Valid values are " +
         "'trace', 'debug', 'info', 'warn' and 'error'.")
     .createWithDefault("trace")
 
-  val OPTIMIZER_PLAN_CHANGE_LOG_RULES = buildConf("spark.sql.optimizer.planChangeLog.rules")
+  val PLAN_CHANGE_LOG_RULES = buildConf("spark.sql.planChangeLog.rules")
     .internal()
-    .doc("Configures a list of rules to be logged in the optimizer, in which the rules are " +
+    .doc("Configures a list of rules for logging plan changes, in which the rules are " +
       "specified by their rule names and separated by comma.")
-    .version("3.0.0")
+    .version("3.1.0")
     .stringConf
     .createOptional
 
-  val OPTIMIZER_PLAN_CHANGE_LOG_BATCHES = buildConf("spark.sql.optimizer.planChangeLog.batches")
+  val PLAN_CHANGE_LOG_BATCHES = buildConf("spark.sql.planChangeLog.batches")
     .internal()
-    .doc("Configures a list of batches to be logged in the optimizer, in which the batches " +
+    .doc("Configures a list of batches for logging plan changes, in which the batches " +
       "are specified by their batch names and separated by comma.")
-    .version("3.0.0")
+    .version("3.1.0")
     .stringConf
     .createOptional
 
@@ -262,16 +298,16 @@ object SQLConf {
       .booleanConf
       .createWithDefault(true)
 
-  val DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO = buildConf(
-    "spark.sql.optimizer.dynamicPartitionPruning.fallbackFilterRatio")
-    .internal()
-    .doc("When statistics are not available or configured not to be used, this config will be " +
-      "used as the fallback filter ratio for computing the data size of the partitioned table " +
-      "after dynamic partition pruning, in order to evaluate if it is worth adding an extra " +
-      "subquery as the pruning filter if broadcast reuse is not applicable.")
-    .version("3.0.0")
-    .doubleConf
-    .createWithDefault(0.5)
+  val DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO =
+    buildConf("spark.sql.optimizer.dynamicPartitionPruning.fallbackFilterRatio")
+      .internal()
+      .doc("When statistics are not available or configured not to be used, this config will be " +
+        "used as the fallback filter ratio for computing the data size of the partitioned table " +
+        "after dynamic partition pruning, in order to evaluate if it is worth adding an extra " +
+        "subquery as the pruning filter if broadcast reuse is not applicable.")
+      .version("3.0.0")
+      .doubleConf
+      .createWithDefault(0.5)
 
   val DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY =
     buildConf("spark.sql.optimizer.dynamicPartitionPruning.reuseBroadcastOnly")
@@ -282,6 +318,18 @@ object SQLConf {
       .booleanConf
       .createWithDefault(true)
 
+  val DYNAMIC_PARTITION_PRUNING_PRUNING_SIDE_EXTRA_FILTER_RATIO =
+    buildConf("spark.sql.optimizer.dynamicPartitionPruning.pruningSideExtraFilterRatio")
+      .internal()
+      .doc("When filtering side doesn't support broadcast by join type, and doing DPP means " +
+        "running an extra query that may have significant overhead. This config will be used " +
+        "as the extra filter ratio for computing the data size of the pruning side after DPP, " +
+        "in order to evaluate if it is worth adding an extra subquery as the pruning filter.")
+      .version("3.2.0")
+      .doubleConf
+      .checkValue(ratio => ratio > 0.0 && ratio <= 1.0, "The ratio value must be in (0.0, 1.0].")
+      .createWithDefault(0.04)
+
   val COMPRESS_CACHED = buildConf("spark.sql.inMemoryColumnarStorage.compressed")
     .doc("When set to true Spark SQL will automatically select a compression codec for each " +
       "column based on statistics of the data.")
@@ -329,7 +377,10 @@ object SQLConf {
 
   val PREFER_SORTMERGEJOIN = buildConf("spark.sql.join.preferSortMergeJoin")
     .internal()
-    .doc("When true, prefer sort merge join over shuffle hash join.")
+    .doc("When true, prefer sort merge join over shuffled hash join. " +
+      "Sort merge join consumes less memory than shuffled hash join and it works efficiently " +
+      "when both join tables are large. On the other hand, shuffled hash join can improve " +
+      "performance (e.g., of full outer joins) when one of join tables is much smaller.")
     .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
@@ -371,6 +422,15 @@ object SQLConf {
       .booleanConf
       .createWithDefault(true)
 
+  val LEAF_NODE_DEFAULT_PARALLELISM = buildConf("spark.sql.leafNodeDefaultParallelism")
+    .doc("The default parallelism of Spark SQL leaf nodes that produce data, such as the file " +
+      "scan node, the local data scan node, the range node, etc. The default value of this " +
+      "config is 'SparkContext#defaultParallelism'.")
+    .version("3.2.0")
+    .intConf
+    .checkValue(_ > 0, "The value of spark.sql.leafNodeDefaultParallelism must be positive.")
+    .createOptional
+
   val SHUFFLE_PARTITIONS = buildConf("spark.sql.shuffle.partitions")
     .doc("The default number of partitions to use when shuffling data for joins or aggregations. " +
       "Note: For structured streaming, this configuration cannot be changed between query " +
@@ -393,7 +453,7 @@ object SQLConf {
       "middle of query execution, based on accurate runtime statistics.")
     .version("1.6.0")
     .booleanConf
-    .createWithDefault(false)
+    .createWithDefault(true)
 
   val ADAPTIVE_EXECUTION_FORCE_APPLY = buildConf("spark.sql.adaptive.forceApply")
     .internal()
@@ -446,7 +506,7 @@ object SQLConf {
 
   val COALESCE_PARTITIONS_INITIAL_PARTITION_NUM =
     buildConf("spark.sql.adaptive.coalescePartitions.initialPartitionNum")
-      .doc("The initial number of shuffle partitions before coalescing. By default it equals to " +
+      .doc("The initial number of shuffle partitions before coalescing. If not set, it equals to " +
         s"${SHUFFLE_PARTITIONS.key}. This configuration only has an effect when " +
         s"'${ADAPTIVE_EXECUTION_ENABLED.key}' and '${COALESCE_PARTITIONS_ENABLED.key}' " +
         "are both true.")
@@ -463,8 +523,8 @@ object SQLConf {
         "reduce IO and improve performance. Note, multiple contiguous blocks exist in single " +
         s"fetch request only happen when '${ADAPTIVE_EXECUTION_ENABLED.key}' and " +
         s"'${COALESCE_PARTITIONS_ENABLED.key}' are both true. This feature also depends " +
-        "on a relocatable serializer, the concatenation support codec in use and the new version " +
-        "shuffle fetch protocol.")
+        "on a relocatable serializer, the concatenation support codec in use, the new version " +
+        "shuffle fetch protocol and io encryption is disabled.")
       .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
@@ -481,8 +541,8 @@ object SQLConf {
   val SKEW_JOIN_ENABLED =
     buildConf("spark.sql.adaptive.skewJoin.enabled")
       .doc(s"When true and '${ADAPTIVE_EXECUTION_ENABLED.key}' is true, Spark dynamically " +
-        "handles skew in sort-merge join by splitting (and replicating if needed) skewed " +
-        "partitions.")
+        "handles skew in shuffled join (sort-merge and shuffled hash) by splitting (and " +
+        "replicating if needed) skewed partitions.")
       .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
@@ -494,8 +554,8 @@ object SQLConf {
         "'spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes'")
       .version("3.0.0")
       .intConf
-      .checkValue(_ > 0, "The skew factor must be positive.")
-      .createWithDefault(10)
+      .checkValue(_ >= 0, "The skew factor cannot be negative.")
+      .createWithDefault(5)
 
   val SKEW_JOIN_SKEWED_PARTITION_THRESHOLD =
     buildConf("spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes")
@@ -519,6 +579,25 @@ object SQLConf {
       .checkValue(_ >= 0, "The non-empty partition ratio must be positive number.")
       .createWithDefault(0.2)
 
+  val ADAPTIVE_OPTIMIZER_EXCLUDED_RULES =
+    buildConf("spark.sql.adaptive.optimizer.excludedRules")
+      .doc("Configures a list of rules to be disabled in the adaptive optimizer, in which the " +
+        "rules are specified by their rule names and separated by comma. The optimizer will log " +
+        "the rules that have indeed been excluded.")
+      .version("3.1.0")
+      .stringConf
+      .createOptional
+
+  val ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD =
+    buildConf("spark.sql.adaptive.autoBroadcastJoinThreshold")
+      .doc("Configures the maximum size in bytes for a table that will be broadcast to all " +
+        "worker nodes when performing a join. By setting this value to -1 broadcasting can be " +
+        s"disabled. The default value is same with ${AUTO_BROADCASTJOIN_THRESHOLD.key}. " +
+        "Note that, this config is used only in adaptive framework.")
+      .version("3.2.0")
+      .bytesConf(ByteUnit.BYTE)
+      .createOptional
+
   val SUBEXPRESSION_ELIMINATION_ENABLED =
     buildConf("spark.sql.subexpressionElimination.enabled")
       .internal()
@@ -527,6 +606,15 @@ object SQLConf {
       .booleanConf
       .createWithDefault(true)
 
+  val SUBEXPRESSION_ELIMINATION_CACHE_MAX_ENTRIES =
+    buildConf("spark.sql.subexpressionElimination.cache.maxEntries")
+      .internal()
+      .doc("The maximum entries of the cache used for interpreted subexpression elimination.")
+      .version("3.1.0")
+      .intConf
+      .checkValue(_ >= 0, "The maximum must not be negative")
+      .createWithDefault(100)
+
   val CASE_SENSITIVE = buildConf("spark.sql.caseSensitive")
     .internal()
     .doc("Whether the query analyzer should be case sensitive or not. " +
@@ -561,7 +649,7 @@ object SQLConf {
       " a heavily underestimated result.")
     .version("2.3.1")
     .doubleConf
-    .checkValue(_ > 0, "the value of fileDataSizeFactor must be greater than 0")
+    .checkValue(_ > 0, "the value of fileCompressionFactor must be greater than 0")
     .createWithDefault(1.0)
 
   val PARQUET_SCHEMA_MERGING_ENABLED = buildConf("spark.sql.parquet.mergeSchema")
@@ -743,11 +831,11 @@ object SQLConf {
     .doc("Sets the compression codec used when writing ORC files. If either `compression` or " +
       "`orc.compress` is specified in the table-specific options/properties, the precedence " +
       "would be `compression`, `orc.compress`, `spark.sql.orc.compression.codec`." +
-      "Acceptable values include: none, uncompressed, snappy, zlib, lzo.")
+      "Acceptable values include: none, uncompressed, snappy, zlib, lzo, zstd.")
     .version("2.3.0")
     .stringConf
     .transform(_.toLowerCase(Locale.ROOT))
-    .checkValues(Set("none", "uncompressed", "snappy", "zlib", "lzo"))
+    .checkValues(Set("none", "uncompressed", "snappy", "zlib", "lzo", "zstd"))
     .createWithDefault("snappy")
 
   val ORC_IMPLEMENTATION = buildConf("spark.sql.orc.impl")
@@ -772,6 +860,13 @@ object SQLConf {
     .intConf
     .createWithDefault(4096)
 
+  val ORC_VECTORIZED_READER_NESTED_COLUMN_ENABLED =
+    buildConf("spark.sql.orc.enableNestedColumnVectorizedReader")
+      .doc("Enables vectorized orc decoding for nested column.")
+      .version("3.2.0")
+      .booleanConf
+      .createWithDefault(false)
+
   val ORC_FILTER_PUSHDOWN_ENABLED = buildConf("spark.sql.orc.filterPushdown")
     .doc("When true, enable filter pushdown for ORC files.")
     .version("1.4.0")
@@ -803,6 +898,20 @@ object SQLConf {
       .booleanConf
       .createWithDefault(true)
 
+  val HIVE_METASTORE_PARTITION_PRUNING_INSET_THRESHOLD =
+    buildConf("spark.sql.hive.metastorePartitionPruningInSetThreshold")
+      .doc("The threshold of set size for InSet predicate when pruning partitions through Hive " +
+        "Metastore. When the set size exceeds the threshold, we rewrite the InSet predicate " +
+        "to be greater than or equal to the minimum value in set and less than or equal to the " +
+        "maximum value in set. Larger values may cause Hive Metastore stack overflow. But for " +
+        "InSet inside Not with values exceeding the threshold, we won't push it to Hive Metastore."
+      )
+      .version("3.1.0")
+      .internal()
+      .intConf
+      .checkValue(_ > 0, "The value of metastorePartitionPruningInSetThreshold must be positive")
+      .createWithDefault(1000)
+
   val HIVE_MANAGE_FILESOURCE_PARTITIONS =
     buildConf("spark.sql.hive.manageFilesourcePartitions")
       .doc("When true, enable metastore partition management for file source tables as well. " +
@@ -840,6 +949,16 @@ object SQLConf {
     .checkValues(HiveCaseSensitiveInferenceMode.values.map(_.toString))
     .createWithDefault(HiveCaseSensitiveInferenceMode.NEVER_INFER.toString)
 
+  val HIVE_TABLE_PROPERTY_LENGTH_THRESHOLD =
+    buildConf("spark.sql.hive.tablePropertyLengthThreshold")
+      .internal()
+      .doc("The maximum length allowed in a single cell when storing Spark-specific information " +
+        "in Hive's metastore as table properties. Currently it covers 2 things: the schema's " +
+        "JSON string, the histogram of column statistics.")
+      .version("3.2.0")
+      .intConf
+      .createOptional
+
   val OPTIMIZER_METADATA_ONLY = buildConf("spark.sql.optimizer.metadataOnly")
     .internal()
     .doc("When true, enable the metadata-only query optimization that use the table's metadata " +
@@ -881,6 +1000,27 @@ object SQLConf {
       .booleanConf
       .createWithDefault(false)
 
+  val THRIFTSERVER_FORCE_CANCEL =
+    buildConf("spark.sql.thriftServer.interruptOnCancel")
+      .doc("When true, all running tasks will be interrupted if one cancels a query. " +
+        "When false, all running tasks will remain until finished.")
+      .version("3.2.0")
+      .booleanConf
+      .createWithDefault(false)
+
+  val THRIFTSERVER_QUERY_TIMEOUT =
+    buildConf("spark.sql.thriftServer.queryTimeout")
+      .doc("Set a query duration timeout in seconds in Thrift Server. If the timeout is set to " +
+        "a positive value, a running query will be cancelled automatically when the timeout is " +
+        "exceeded, otherwise the query continues to run till completion. If timeout values are " +
+        "set for each statement via `java.sql.Statement.setQueryTimeout` and they are smaller " +
+        "than this configuration value, they take precedence. If you set this timeout and prefer " +
+        "to cancel the queries right away without waiting task to finish, consider enabling " +
+        s"${THRIFTSERVER_FORCE_CANCEL.key} together.")
+      .version("3.1.0")
+      .timeConf(TimeUnit.SECONDS)
+      .createWithDefault(0L)
+
   val THRIFTSERVER_UI_STATEMENT_LIMIT =
     buildConf("spark.sql.thriftserver.ui.retainedStatements")
       .doc("The number of SQL statements kept in the JDBC/ODBC web UI history.")
@@ -939,6 +1079,29 @@ object SQLConf {
     .checkValue(_ > 0, "the value of spark.sql.sources.bucketing.maxBuckets must be greater than 0")
     .createWithDefault(100000)
 
+  val AUTO_BUCKETED_SCAN_ENABLED =
+    buildConf("spark.sql.sources.bucketing.autoBucketedScan.enabled")
+      .doc("When true, decide whether to do bucketed scan on input tables based on query plan " +
+        "automatically. Do not use bucketed scan if 1. query does not have operators to utilize " +
+        "bucketing (e.g. join, group-by, etc), or 2. there's an exchange operator between these " +
+        s"operators and table scan. Note when '${BUCKETING_ENABLED.key}' is set to " +
+        "false, this configuration does not take any effect.")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(true)
+
+  val CAN_CHANGE_CACHED_PLAN_OUTPUT_PARTITIONING =
+    buildConf("spark.sql.optimizer.canChangeCachedPlanOutputPartitioning")
+      .internal()
+      .doc("Whether to forcibly enable some optimization rules that can change the output " +
+        "partitioning of a cached query when executing it for caching. If it is set to true, " +
+        "queries may need an extra shuffle to read the cached data. This configuration is " +
+        "disabled by default. Currently, the optimization rules enabled by this configuration " +
+        s"are ${ADAPTIVE_EXECUTION_ENABLED.key} and ${AUTO_BUCKETED_SCAN_ENABLED.key}.")
+      .version("3.2.0")
+      .booleanConf
+      .createWithDefault(false)
+
   val CROSS_JOINS_ENABLED = buildConf("spark.sql.crossJoin.enabled")
     .internal()
     .doc("When false, we will throw an error if a query contains a cartesian product without " +
@@ -962,8 +1125,9 @@ object SQLConf {
     .createWithDefault(true)
 
   val GROUP_BY_ALIASES = buildConf("spark.sql.groupByAliases")
-    .doc("When true, aliases in a select list can be used in group by clauses. When false, " +
-      "an analysis exception is thrown in the case.")
+    .doc("This configuration is only effective when ANSI mode is disabled. When it is true and " +
+      s"${ANSI_ENABLED.key} is false, aliases in a select list can be used in group by clauses. " +
+      "Otherwise, an analysis exception is thrown in the case.")
     .version("2.2.0")
     .booleanConf
     .createWithDefault(true)
@@ -1082,7 +1246,7 @@ object SQLConf {
 
   val CODEGEN_FACTORY_MODE = buildConf("spark.sql.codegen.factoryMode")
     .doc("This config determines the fallback behavior of several codegen generators " +
-      "during tests. `FALLBACK` means trying codegen first and then fallbacking to " +
+      "during tests. `FALLBACK` means trying codegen first and then falling back to " +
       "interpreted if any compile error happens. Disabling fallback if `CODEGEN_ONLY`. " +
       "`NO_CODEGEN` skips codegen and goes interpreted path always. Note that " +
       "this config works only for tests.")
@@ -1163,6 +1327,15 @@ object SQLConf {
     .longConf
     .createWithDefault(4 * 1024 * 1024)
 
+  val FILES_MIN_PARTITION_NUM = buildConf("spark.sql.files.minPartitionNum")
+    .doc("The suggested (not guaranteed) minimum number of split file partitions. " +
+      "If not set, the default value is `spark.default.parallelism`. This configuration is " +
+      "effective only when using file-based sources such as Parquet, JSON and ORC.")
+    .version("3.1.0")
+    .intConf
+    .checkValue(v => v > 0, "The min partition number must be a positive integer.")
+    .createOptional
+
   val IGNORE_CORRUPT_FILES = buildConf("spark.sql.files.ignoreCorruptFiles")
     .doc("Whether to ignore corrupt files. If true, the Spark jobs will continue to run when " +
       "encountering corrupted files and the contents that have been read will still be returned. " +
@@ -1202,6 +1375,21 @@ object SQLConf {
     .booleanConf
     .createWithDefault(true)
 
+  val REMOVE_REDUNDANT_PROJECTS_ENABLED = buildConf("spark.sql.execution.removeRedundantProjects")
+    .internal()
+    .doc("Whether to remove redundant project exec node based on children's output and " +
+      "ordering requirement.")
+    .version("3.1.0")
+    .booleanConf
+    .createWithDefault(true)
+
+  val REMOVE_REDUNDANT_SORTS_ENABLED = buildConf("spark.sql.execution.removeRedundantSorts")
+    .internal()
+    .doc("Whether to remove redundant physical sort node")
+    .version("2.4.8")
+    .booleanConf
+    .createWithDefault(true)
+
   val STATE_STORE_PROVIDER_CLASS =
     buildConf("spark.sql.streaming.stateStore.providerClass")
       .internal()
@@ -1215,6 +1403,14 @@ object SQLConf {
       .createWithDefault(
         "org.apache.spark.sql.execution.streaming.state.HDFSBackedStateStoreProvider")
 
+  val STATE_SCHEMA_CHECK_ENABLED =
+    buildConf("spark.sql.streaming.stateStore.stateSchemaCheck")
+      .doc("When true, Spark will validate the state schema against schema on existing state and " +
+        "fail query if it's incompatible.")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(true)
+
   val STATE_STORE_MIN_DELTAS_FOR_SNAPSHOT =
     buildConf("spark.sql.streaming.stateStore.minDeltasForSnapshot")
       .internal()
@@ -1224,6 +1420,16 @@ object SQLConf {
       .intConf
       .createWithDefault(10)
 
+  val STATE_STORE_FORMAT_VALIDATION_ENABLED =
+    buildConf("spark.sql.streaming.stateStore.formatValidation.enabled")
+      .internal()
+      .doc("When true, check if the data from state store is valid or not when running streaming " +
+        "queries. This can happen if the state store format has been changed. Note, the feature " +
+        "is only effective in the build-in HDFS state store provider now.")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(true)
+
   val FLATMAPGROUPSWITHSTATE_STATE_FORMAT_VERSION =
     buildConf("spark.sql.streaming.flatMapGroupsWithState.stateFormatVersion")
       .internal()
@@ -1263,6 +1469,27 @@ object SQLConf {
     .intConf
     .createWithDefault(2)
 
+  val STREAMING_MAINTENANCE_INTERVAL =
+    buildConf("spark.sql.streaming.stateStore.maintenanceInterval")
+      .internal()
+      .doc("The interval in milliseconds between triggering maintenance tasks in StateStore. " +
+        "The maintenance task executes background maintenance task in all the loaded store " +
+        "providers if they are still the active instances according to the coordinator. If not, " +
+        "inactive instances of store providers will be closed.")
+      .version("2.0.0")
+      .timeConf(TimeUnit.MILLISECONDS)
+      .createWithDefault(TimeUnit.MINUTES.toMillis(1)) // 1 minute
+
+  val STATE_STORE_COMPRESSION_CODEC =
+    buildConf("spark.sql.streaming.stateStore.compression.codec")
+      .internal()
+      .doc("The codec used to compress delta and snapshot files generated by StateStore. " +
+        "By default, Spark provides four codecs: lz4, lzf, snappy, and zstd. You can also " +
+        "use fully qualified class names to specify the codec. Default codec is lz4.")
+      .version("3.1.0")
+      .stringConf
+      .createWithDefault("lz4")
+
   val STREAMING_AGGREGATION_STATE_FORMAT_VERSION =
     buildConf("spark.sql.streaming.aggregation.stateFormatVersion")
       .internal()
@@ -1304,6 +1531,32 @@ object SQLConf {
       .booleanConf
       .createWithDefault(true)
 
+  val USE_DEPRECATED_KAFKA_OFFSET_FETCHING =
+    buildConf("spark.sql.streaming.kafka.useDeprecatedOffsetFetching")
+      .internal()
+      .doc("When true, the deprecated Consumer based offset fetching used which could cause " +
+        "infinite wait in Spark queries. Such cases query restart is the only workaround. " +
+        "For further details please see Offset Fetching chapter of Structured Streaming Kafka " +
+        "Integration Guide.")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(true)
+
+  val STATEFUL_OPERATOR_CHECK_CORRECTNESS_ENABLED =
+    buildConf("spark.sql.streaming.statefulOperator.checkCorrectness.enabled")
+      .internal()
+      .doc("When true, the stateful operators for streaming query will be checked for possible " +
+        "correctness issue due to global watermark. The correctness issue comes from queries " +
+        "containing stateful operation which can emit rows older than the current watermark " +
+        "plus allowed late record delay, which are \"late rows\" in downstream stateful " +
+        "operations and these rows can be discarded. Please refer the programming guide doc for " +
+        "more details. Once the issue is detected, Spark will throw analysis exception. " +
+        "When this config is disabled, Spark will just print warning message for users. " +
+        "Prior to Spark 3.1.0, the behavior is disabling this config.")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(true)
+
   val VARIABLE_SUBSTITUTE_ENABLED =
     buildConf("spark.sql.variable.substitute")
       .doc("This enables substitution using syntax like `${var}`, `${system:var}`, " +
@@ -1355,6 +1608,40 @@ object SQLConf {
         "must be positive.")
       .createWithDefault(100)
 
+  val ALLOW_PARAMETERLESS_COUNT =
+    buildConf("spark.sql.legacy.allowParameterlessCount")
+      .internal()
+      .doc("When true, the SQL function 'count' is allowed to take no parameters.")
+      .version("3.1.1")
+      .booleanConf
+      .createWithDefault(false)
+
+  val ALLOW_STAR_WITH_SINGLE_TABLE_IDENTIFIER_IN_COUNT =
+    buildConf("spark.sql.legacy.allowStarWithSingleTableIdentifierInCount")
+      .internal()
+      .doc("When true, the SQL function 'count' is allowed to take single 'tblName.*' as parameter")
+      .version("3.2")
+      .booleanConf
+      .createWithDefault(false)
+
+  val USE_CURRENT_SQL_CONFIGS_FOR_VIEW =
+    buildConf("spark.sql.legacy.useCurrentConfigsForView")
+      .internal()
+      .doc("When true, SQL Configs of the current active SparkSession instead of the captured " +
+        "ones will be applied during the parsing and analysis phases of the view resolution.")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(false)
+
+  val STORE_ANALYZED_PLAN_FOR_VIEW =
+    buildConf("spark.sql.legacy.storeAnalyzedPlanForView")
+      .internal()
+      .doc("When true, analyzed plan instead of SQL text will be stored when creating " +
+        "temporary view")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(false)
+
   val STREAMING_FILE_COMMIT_PROTOCOL_CLASS =
     buildConf("spark.sql.streaming.commitProtocolClass")
       .version("2.1.0")
@@ -1408,6 +1695,23 @@ object SQLConf {
       .booleanConf
       .createWithDefault(true)
 
+  val JSON_EXPRESSION_OPTIMIZATION =
+    buildConf("spark.sql.optimizer.enableJsonExpressionOptimization")
+      .doc("Whether to optimize JSON expressions in SQL optimizer. It includes pruning " +
+        "unnecessary columns from from_json, simplifying from_json + to_json, to_json + " +
+        "named_struct(from_json.col1, from_json.col2, ....).")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(true)
+
+  val CSV_EXPRESSION_OPTIMIZATION =
+    buildConf("spark.sql.optimizer.enableCsvExpressionOptimization")
+      .doc("Whether to optimize CSV expressions in SQL optimizer. It includes pruning " +
+        "unnecessary columns from from_csv.")
+      .version("3.2.0")
+      .booleanConf
+      .createWithDefault(true)
+
   val FILE_SINK_LOG_DELETION = buildConf("spark.sql.streaming.fileSink.log.deletion")
     .internal()
     .doc("Whether to delete the expired log files in file stream sink.")
@@ -1530,18 +1834,18 @@ object SQLConf {
 
   val STREAMING_CHECKPOINT_FILE_MANAGER_CLASS =
     buildConf("spark.sql.streaming.checkpointFileManagerClass")
+      .internal()
       .doc("The class used to write checkpoint files atomically. This class must be a subclass " +
         "of the interface CheckpointFileManager.")
       .version("2.4.0")
-      .internal()
       .stringConf
 
   val STREAMING_CHECKPOINT_ESCAPED_PATH_CHECK_ENABLED =
     buildConf("spark.sql.streaming.checkpoint.escapedPathCheck.enabled")
+      .internal()
       .doc("Whether to detect a streaming query may pick up an incorrect checkpoint path due " +
         "to SPARK-26824.")
       .version("3.0.0")
-      .internal()
       .booleanConf
       .createWithDefault(true)
 
@@ -1579,8 +1883,8 @@ object SQLConf {
   val NDV_MAX_ERROR =
     buildConf("spark.sql.statistics.ndv.maxError")
       .internal()
-      .doc("The maximum estimation error allowed in HyperLogLog++ algorithm when generating " +
-        "column level statistics.")
+      .doc("The maximum relative standard deviation allowed in HyperLogLog++ algorithm " +
+        "when generating column level statistics.")
       .version("2.1.1")
       .doubleConf
       .createWithDefault(0.05)
@@ -1656,8 +1960,10 @@ object SQLConf {
   val JOIN_REORDER_CARD_WEIGHT =
     buildConf("spark.sql.cbo.joinReorder.card.weight")
       .internal()
-      .doc("The weight of cardinality (number of rows) for plan cost comparison in join reorder: " +
-        "rows * weight + size * (1 - weight).")
+      .doc("The weight of the ratio of cardinalities (number of rows) " +
+        "in the cost comparison function. The ratio of sizes in bytes has weight " +
+        "1 - this value. The weighted geometric mean of these ratios is used to decide " +
+        "which of the candidate plans will be chosen by the CBO.")
       .version("2.2.0")
       .doubleConf
       .checkValue(weight => weight >= 0 && weight <= 1, "The weight value must be in [0, 1].")
@@ -1691,9 +1997,9 @@ object SQLConf {
   val SESSION_LOCAL_TIMEZONE = buildConf("spark.sql.session.timeZone")
     .doc("The ID of session local timezone in the format of either region-based zone IDs or " +
       "zone offsets. Region IDs must have the form 'area/city', such as 'America/Los_Angeles'. " +
-      "Zone offsets must be in the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. " +
-      "Also 'UTC' and 'Z' are supported as aliases of '+00:00'. Other short names are not " +
-      "recommended to use because they can be ambiguous.")
+      "Zone offsets must be in the format '(+|-)HH', '(+|-)HH:mm' or '(+|-)HH:mm:ss', e.g '-08', " +
+      "'+01:00' or '-13:33:33'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'. Other " +
+      "short names are not recommended to use because they can be ambiguous.")
     .version("2.2.0")
     .stringConf
     .checkValue(isValidTimezone, s"Cannot resolve the given timezone with" +
@@ -1780,10 +2086,30 @@ object SQLConf {
         "1. pyspark.sql.DataFrame.toPandas " +
         "2. pyspark.sql.SparkSession.createDataFrame when its input is a Pandas DataFrame " +
         "The following data types are unsupported: " +
-        "BinaryType, MapType, ArrayType of TimestampType, and nested StructType.")
+        "ArrayType of TimestampType, and nested StructType.")
       .version("3.0.0")
       .fallbackConf(ARROW_EXECUTION_ENABLED)
 
+  val ARROW_PYSPARK_SELF_DESTRUCT_ENABLED =
+    buildConf("spark.sql.execution.arrow.pyspark.selfDestruct.enabled")
+      .doc("(Experimental) When true, make use of Apache Arrow's self-destruct and split-blocks " +
+        "options for columnar data transfers in PySpark, when converting from Arrow to Pandas. " +
+        "This reduces memory usage at the cost of some CPU time. " +
+        "This optimization applies to: pyspark.sql.DataFrame.toPandas " +
+        "when 'spark.sql.execution.arrow.pyspark.enabled' is set.")
+      .version("3.2.0")
+      .booleanConf
+      .createWithDefault(false)
+
+  val PYSPARK_JVM_STACKTRACE_ENABLED =
+    buildConf("spark.sql.pyspark.jvmStacktrace.enabled")
+      .doc("When true, it shows the JVM stacktrace in the user-facing PySpark exception " +
+        "together with Python stacktrace. By default, it is disabled and hides JVM stacktrace " +
+        "and shows a Python-friendly exception only.")
+      .version("3.0.0")
+      .booleanConf
+      .createWithDefault(false)
+
   val ARROW_SPARKR_EXECUTION_ENABLED =
     buildConf("spark.sql.execution.arrow.sparkr.enabled")
       .doc("When true, make use of Apache Arrow for columnar data transfers in SparkR. " +
@@ -1831,6 +2157,16 @@ object SQLConf {
       .version("3.0.0")
       .fallbackConf(BUFFER_SIZE)
 
+  val PYSPARK_SIMPLIFIEID_TRACEBACK =
+    buildConf("spark.sql.execution.pyspark.udf.simplifiedTraceback.enabled")
+      .doc(
+        "When true, the traceback from Python UDFs is simplified. It hides " +
+        "the Python worker, (de)serialization, etc from PySpark in tracebacks, and only " +
+        "shows the exception messages from UDFs. Note that this works only with CPython 3.7+.")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(false)
+
   val PANDAS_GROUPED_MAP_ASSIGN_COLUMNS_BY_NAME =
     buildConf("spark.sql.legacy.execution.pandas.groupedMap.assignColumnsByName")
       .internal()
@@ -1871,7 +2207,7 @@ object SQLConf {
     buildConf("spark.sql.decimalOperations.allowPrecisionLoss")
       .internal()
       .doc("When true (default), establishing the result type of an arithmetic operation " +
-        "happens according to Hive behavior and SQL ANSI 2011 specification, ie. rounding the " +
+        "happens according to Hive behavior and SQL ANSI 2011 specification, i.e. rounding the " +
         "decimal part of the result if an exact representation is not possible. Otherwise, NULL " +
         "is returned in those cases, as previously.")
       .version("2.3.1")
@@ -1984,6 +2320,20 @@ object SQLConf {
       .stringConf
       .createWithDefault("")
 
+  val FASTFAIL_ON_FILEFORMAT_OUTPUT =
+    buildConf("spark.sql.execution.fastFailOnFileFormatOutput")
+      .internal()
+      .doc("Whether to fast fail task execution when writing output to FileFormat datasource. " +
+        "If this is enabled, in `FileFormatWriter` we will catch `FileAlreadyExistsException` " +
+        "and fast fail output task without further task retry. Only enabling this if you know " +
+        "the `FileAlreadyExistsException` of the output task is unrecoverable, i.e., further " +
+        "task attempts won't be able to success. If the `FileAlreadyExistsException` might be " +
+        "recoverable, you should keep this as disabled and let Spark to retry output tasks. " +
+        "This is disabled by default.")
+      .version("3.0.2")
+      .booleanConf
+      .createWithDefault(false)
+
   object PartitionOverwriteMode extends Enumeration {
     val STATIC, DYNAMIC = Value
   }
@@ -2032,15 +2382,6 @@ object SQLConf {
       .checkValues(StoreAssignmentPolicy.values.map(_.toString))
       .createWithDefault(StoreAssignmentPolicy.ANSI.toString)
 
-  val ANSI_ENABLED = buildConf("spark.sql.ansi.enabled")
-    .doc("When true, Spark tries to conform to the ANSI SQL specification: 1. Spark will " +
-      "throw a runtime exception if an overflow occurs in any operation on integral/decimal " +
-      "field. 2. Spark will forbid using the reserved keywords of ANSI SQL as identifiers in " +
-      "the SQL parser.")
-    .version("3.0.0")
-    .booleanConf
-    .createWithDefault(false)
-
   val SORT_BEFORE_REPARTITION =
     buildConf("spark.sql.execution.sortBeforeRepartition")
       .internal()
@@ -2065,15 +2406,24 @@ object SQLConf {
       .booleanConf
       .createWithDefault(true)
 
+  val DISABLE_HINTS =
+    buildConf("spark.sql.optimizer.disableHints")
+      .internal()
+      .doc("When true, the optimizer will disable user-specified hints that are additional " +
+        "directives for better planning of a query.")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(false)
+
   val NESTED_PREDICATE_PUSHDOWN_FILE_SOURCE_LIST =
     buildConf("spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources")
       .internal()
       .doc("A comma-separated list of data source short names or fully qualified data source " +
         "implementation class names for which Spark tries to push down predicates for nested " +
         "columns and/or names containing `dots` to data sources. This configuration is only " +
-        "effective with file-based data source in DSv1. Currently, Parquet implements " +
-        "both optimizations while ORC only supports predicates for names containing `dots`. The " +
-        "other data sources don't support this feature yet. So the default value is 'parquet,orc'.")
+        "effective with file-based data sources in DSv1. Currently, Parquet and ORC implement " +
+        "both optimizations. The other data sources don't support this feature yet. So the " +
+        "default value is 'parquet,orc'.")
       .version("3.0.0")
       .stringConf
       .createWithDefault("parquet,orc")
@@ -2099,6 +2449,14 @@ object SQLConf {
       .booleanConf
       .createWithDefault(true)
 
+  val DECORRELATE_INNER_QUERY_ENABLED =
+    buildConf("spark.sql.optimizer.decorrelateInnerQuery.enabled")
+      .internal()
+      .doc("Decorrelate inner query by eliminating correlated references and build domain joins.")
+      .version("3.2.0")
+      .booleanConf
+      .createWithDefault(true)
+
   val TOP_K_SORT_FALLBACK_THRESHOLD =
     buildConf("spark.sql.execution.topKSortFallbackThreshold")
       .internal()
@@ -2125,6 +2483,16 @@ object SQLConf {
     .booleanConf
     .createWithDefault(true)
 
+  val CSV_INPUT_BUFFER_SIZE = buildConf("spark.sql.csv.parser.inputBufferSize")
+    .internal()
+    .doc("If it is set, it configures the buffer size of CSV input during parsing. " +
+      "It is the same as inputBufferSize option in CSV which has a higher priority. " +
+      "Note that this is a workaround for the parsing library's regression, and this " +
+      "configuration is internal and supposed to be removed in the near future.")
+    .version("3.0.3")
+    .intConf
+    .createOptional
+
   val REPL_EAGER_EVAL_ENABLED = buildConf("spark.sql.repl.eagerEval.enabled")
     .doc("Enables eager evaluation or not. When true, the top K rows of Dataset will be " +
       "displayed if and only if the REPL supports the eager evaluation. Currently, the " +
@@ -2166,10 +2534,10 @@ object SQLConf {
 
   val AVRO_COMPRESSION_CODEC = buildConf("spark.sql.avro.compression.codec")
     .doc("Compression codec used in writing of AVRO files. Supported codecs: " +
-      "uncompressed, deflate, snappy, bzip2 and xz. Default codec is snappy.")
+      "uncompressed, deflate, snappy, bzip2, xz and zstandard. Default codec is snappy.")
     .version("2.4.0")
     .stringConf
-    .checkValues(Set("uncompressed", "deflate", "snappy", "bzip2", "xz"))
+    .checkValues(Set("uncompressed", "deflate", "snappy", "bzip2", "xz", "zstandard"))
     .createWithDefault("snappy")
 
   val AVRO_DEFLATE_LEVEL = buildConf("spark.sql.avro.deflate.level")
@@ -2189,6 +2557,16 @@ object SQLConf {
     .booleanConf
     .createWithDefault(true)
 
+  val LEGACY_PARSE_NULL_PARTITION_SPEC_AS_STRING_LITERAL =
+    buildConf("spark.sql.legacy.parseNullPartitionSpecAsStringLiteral")
+      .internal()
+      .doc("If it is set to true, `PARTITION(col=null)` is parsed as a string literal of its " +
+        "text representation, e.g., string 'null', when the partition column is string type. " +
+        "Otherwise, it is always parsed as a null literal in the partition spec.")
+      .version("3.0.2")
+      .booleanConf
+      .createWithDefault(false)
+
   val LEGACY_REPLACE_DATABRICKS_SPARK_AVRO_ENABLED =
     buildConf("spark.sql.legacy.replaceDatabricksSparkAvro.enabled")
       .internal()
@@ -2274,6 +2652,16 @@ object SQLConf {
       .booleanConf
       .createWithDefault(false)
 
+  val LEGACY_STATISTICAL_AGGREGATE =
+    buildConf("spark.sql.legacy.statisticalAggregate")
+      .internal()
+      .doc("When set to true, statistical aggregate function returns Double.NaN " +
+        "if divide by zero occurred during expression evaluation, otherwise, it returns null. " +
+        "Before version 3.1.0, it returns NaN in divideByZero case by default.")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(false)
+
   val TRUNCATE_TABLE_IGNORE_PERMISSION_ACL =
     buildConf("spark.sql.truncateTable.ignorePermissionAcl.enabled")
       .internal()
@@ -2313,6 +2701,14 @@ object SQLConf {
       "(nonnegative and shorter than the maximum size).")
     .createWithDefaultString(s"${ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH}")
 
+  val MAX_METADATA_STRING_LENGTH = buildConf("spark.sql.maxMetadataStringLength")
+    .doc("Maximum number of characters to output for a metadata string. e.g. " +
+      "file location in `DataSourceScanExec`, every value will be abbreviated if exceed length.")
+    .version("3.1.0")
+    .intConf
+    .checkValue(_ > 3, "This value must be bigger than 3.")
+    .createWithDefault(100)
+
   val SET_COMMAND_REJECTS_SPARK_CORE_CONFS =
     buildConf("spark.sql.legacy.setCommandRejectsSparkCoreConfs")
       .internal()
@@ -2355,7 +2751,7 @@ object SQLConf {
     buildConf("spark.sql.legacy.typeCoercion.datetimeToString.enabled")
       .internal()
       .doc("If it is set to true, date/timestamp will cast to string in binary comparisons " +
-        "with String")
+        s"with String when ${ANSI_ENABLED.key} is false.")
       .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
@@ -2446,7 +2842,8 @@ object SQLConf {
       .version("3.0.0")
       .stringConf
       .createWithDefault(
-        "https://maven-central.storage-download.googleapis.com/maven2/")
+        sys.env.getOrElse("DEFAULT_ARTIFACT_REPOSITORY",
+          "https://maven-central.storage-download.googleapis.com/maven2/"))
 
   val LEGACY_FROM_DAYTIME_STRING =
     buildConf("spark.sql.legacy.fromDayTimeString.enabled")
@@ -2493,18 +2890,32 @@ object SQLConf {
     .booleanConf
     .createWithDefault(true)
 
+  val JSON_FILTER_PUSHDOWN_ENABLED = buildConf("spark.sql.json.filterPushdown.enabled")
+    .doc("When true, enable filter pushdown to JSON datasource.")
+    .version("3.1.0")
+    .booleanConf
+    .createWithDefault(true)
+
+  val AVRO_FILTER_PUSHDOWN_ENABLED = buildConf("spark.sql.avro.filterPushdown.enabled")
+    .doc("When true, enable filter pushdown to Avro datasource.")
+    .version("3.1.0")
+    .booleanConf
+    .createWithDefault(true)
+
   val ADD_PARTITION_BATCH_SIZE =
     buildConf("spark.sql.addPartitionInBatch.size")
       .internal()
       .doc("The number of partitions to be handled in one turn when use " +
-        "`AlterTableAddPartitionCommand` to add partitions into table. The smaller " +
-        "batch size is, the less memory is required for the real handler, e.g. Hive Metastore.")
+        "`AlterTableAddPartitionCommand` or `RepairTableCommand` to add partitions into table. " +
+        "The smaller batch size is, the less memory is required for the real handler, e.g. " +
+        "Hive Metastore.")
       .version("3.0.0")
       .intConf
       .checkValue(_ > 0, "The value of spark.sql.addPartitionInBatch.size must be positive")
       .createWithDefault(100)
 
   val LEGACY_ALLOW_HASH_ON_MAPTYPE = buildConf("spark.sql.legacy.allowHashOnMapType")
+    .internal()
     .doc("When set to true, hash expressions can be applied on elements of MapType. Otherwise, " +
       "an analysis exception will be thrown.")
     .version("3.0.0")
@@ -2519,37 +2930,76 @@ object SQLConf {
       .booleanConf
       .createWithDefault(false)
 
-  val LEGACY_PARQUET_REBASE_MODE_IN_WRITE =
-    buildConf("spark.sql.legacy.parquet.datetimeRebaseModeInWrite")
+  val PARQUET_INT96_REBASE_MODE_IN_WRITE =
+    buildConf("spark.sql.parquet.int96RebaseModeInWrite")
+      .internal()
+      .doc("When LEGACY, Spark will rebase INT96 timestamps from Proleptic Gregorian calendar to " +
+        "the legacy hybrid (Julian + Gregorian) calendar when writing Parquet files. " +
+        "When CORRECTED, Spark will not do rebase and write the timestamps as it is. " +
+        "When EXCEPTION, which is the default, Spark will fail the writing if it sees ancient " +
+        "timestamps that are ambiguous between the two calendars.")
+      .version("3.1.0")
+      .withAlternative("spark.sql.legacy.parquet.int96RebaseModeInWrite")
+      .stringConf
+      .transform(_.toUpperCase(Locale.ROOT))
+      .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
+      .createWithDefault(LegacyBehaviorPolicy.EXCEPTION.toString)
+
+  val PARQUET_REBASE_MODE_IN_WRITE =
+    buildConf("spark.sql.parquet.datetimeRebaseModeInWrite")
       .internal()
       .doc("When LEGACY, Spark will rebase dates/timestamps from Proleptic Gregorian calendar " +
         "to the legacy hybrid (Julian + Gregorian) calendar when writing Parquet files. " +
         "When CORRECTED, Spark will not do rebase and write the dates/timestamps as it is. " +
         "When EXCEPTION, which is the default, Spark will fail the writing if it sees " +
-        "ancient dates/timestamps that are ambiguous between the two calendars.")
+        "ancient dates/timestamps that are ambiguous between the two calendars. " +
+        "This config influences on writes of the following parquet logical types: DATE, " +
+        "TIMESTAMP_MILLIS, TIMESTAMP_MICROS. The INT96 type has the separate config: " +
+        s"${PARQUET_INT96_REBASE_MODE_IN_WRITE.key}.")
       .version("3.0.0")
+      .withAlternative("spark.sql.legacy.parquet.datetimeRebaseModeInWrite")
       .stringConf
       .transform(_.toUpperCase(Locale.ROOT))
       .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
       .createWithDefault(LegacyBehaviorPolicy.EXCEPTION.toString)
 
-  val LEGACY_PARQUET_REBASE_MODE_IN_READ =
-    buildConf("spark.sql.legacy.parquet.datetimeRebaseModeInRead")
+  val PARQUET_INT96_REBASE_MODE_IN_READ =
+    buildConf("spark.sql.parquet.int96RebaseModeInRead")
+      .internal()
+      .doc("When LEGACY, Spark will rebase INT96 timestamps from the legacy hybrid (Julian + " +
+        "Gregorian) calendar to Proleptic Gregorian calendar when reading Parquet files. " +
+        "When CORRECTED, Spark will not do rebase and read the timestamps as it is. " +
+        "When EXCEPTION, which is the default, Spark will fail the reading if it sees ancient " +
+        "timestamps that are ambiguous between the two calendars. This config is only effective " +
+        "if the writer info (like Spark, Hive) of the Parquet files is unknown.")
+      .version("3.1.0")
+      .withAlternative("spark.sql.legacy.parquet.int96RebaseModeInRead")
+      .stringConf
+      .transform(_.toUpperCase(Locale.ROOT))
+      .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
+      .createWithDefault(LegacyBehaviorPolicy.EXCEPTION.toString)
+
+  val PARQUET_REBASE_MODE_IN_READ =
+    buildConf("spark.sql.parquet.datetimeRebaseModeInRead")
       .internal()
       .doc("When LEGACY, Spark will rebase dates/timestamps from the legacy hybrid (Julian + " +
         "Gregorian) calendar to Proleptic Gregorian calendar when reading Parquet files. " +
         "When CORRECTED, Spark will not do rebase and read the dates/timestamps as it is. " +
         "When EXCEPTION, which is the default, Spark will fail the reading if it sees " +
         "ancient dates/timestamps that are ambiguous between the two calendars. This config is " +
-        "only effective if the writer info (like Spark, Hive) of the Parquet files is unknown.")
+        "only effective if the writer info (like Spark, Hive) of the Parquet files is unknown. " +
+        "This config influences on reads of the following parquet logical types: DATE, " +
+        "TIMESTAMP_MILLIS, TIMESTAMP_MICROS. The INT96 type has the separate config: " +
+        s"${PARQUET_INT96_REBASE_MODE_IN_READ.key}.")
       .version("3.0.0")
+      .withAlternative("spark.sql.legacy.parquet.datetimeRebaseModeInRead")
       .stringConf
       .transform(_.toUpperCase(Locale.ROOT))
       .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
       .createWithDefault(LegacyBehaviorPolicy.EXCEPTION.toString)
 
-  val LEGACY_AVRO_REBASE_MODE_IN_WRITE =
-    buildConf("spark.sql.legacy.avro.datetimeRebaseModeInWrite")
+  val AVRO_REBASE_MODE_IN_WRITE =
+    buildConf("spark.sql.avro.datetimeRebaseModeInWrite")
       .internal()
       .doc("When LEGACY, Spark will rebase dates/timestamps from Proleptic Gregorian calendar " +
         "to the legacy hybrid (Julian + Gregorian) calendar when writing Avro files. " +
@@ -2557,13 +3007,14 @@ object SQLConf {
         "When EXCEPTION, which is the default, Spark will fail the writing if it sees " +
         "ancient dates/timestamps that are ambiguous between the two calendars.")
       .version("3.0.0")
+      .withAlternative("spark.sql.legacy.avro.datetimeRebaseModeInWrite")
       .stringConf
       .transform(_.toUpperCase(Locale.ROOT))
       .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
       .createWithDefault(LegacyBehaviorPolicy.EXCEPTION.toString)
 
-  val LEGACY_AVRO_REBASE_MODE_IN_READ =
-    buildConf("spark.sql.legacy.avro.datetimeRebaseModeInRead")
+  val AVRO_REBASE_MODE_IN_READ =
+    buildConf("spark.sql.avro.datetimeRebaseModeInRead")
       .internal()
       .doc("When LEGACY, Spark will rebase dates/timestamps from the legacy hybrid (Julian + " +
         "Gregorian) calendar to Proleptic Gregorian calendar when reading Avro files. " +
@@ -2572,6 +3023,7 @@ object SQLConf {
         "ancient dates/timestamps that are ambiguous between the two calendars. This config is " +
         "only effective if the writer info (like Spark, Hive) of the Avro files is unknown.")
       .version("3.0.0")
+      .withAlternative("spark.sql.legacy.avro.datetimeRebaseModeInRead")
       .stringConf
       .transform(_.toUpperCase(Locale.ROOT))
       .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
@@ -2586,6 +3038,149 @@ object SQLConf {
       .checkValue(_ > 0, "The timeout value must be positive")
       .createWithDefault(10L)
 
+  val COALESCE_BUCKETS_IN_JOIN_ENABLED =
+    buildConf("spark.sql.bucketing.coalesceBucketsInJoin.enabled")
+      .doc("When true, if two bucketed tables with the different number of buckets are joined, " +
+        "the side with a bigger number of buckets will be coalesced to have the same number " +
+        "of buckets as the other side. Bigger number of buckets is divisible by the smaller " +
+        "number of buckets. Bucket coalescing is applied to sort-merge joins and " +
+        "shuffled hash join. Note: Coalescing bucketed table can avoid unnecessary shuffling " +
+        "in join, but it also reduces parallelism and could possibly cause OOM for " +
+        "shuffled hash join.")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(false)
+
+  val COALESCE_BUCKETS_IN_JOIN_MAX_BUCKET_RATIO =
+    buildConf("spark.sql.bucketing.coalesceBucketsInJoin.maxBucketRatio")
+      .doc("The ratio of the number of two buckets being coalesced should be less than or " +
+        "equal to this value for bucket coalescing to be applied. This configuration only " +
+        s"has an effect when '${COALESCE_BUCKETS_IN_JOIN_ENABLED.key}' is set to true.")
+      .version("3.1.0")
+      .intConf
+      .checkValue(_ > 0, "The difference must be positive.")
+      .createWithDefault(4)
+
+  val BROADCAST_HASH_JOIN_OUTPUT_PARTITIONING_EXPAND_LIMIT =
+    buildConf("spark.sql.execution.broadcastHashJoin.outputPartitioningExpandLimit")
+      .internal()
+      .doc("The maximum number of partitionings that a HashPartitioning can be expanded to. " +
+        "This configuration is applicable only for BroadcastHashJoin inner joins and can be " +
+        "set to '0' to disable this feature.")
+      .version("3.1.0")
+      .intConf
+      .checkValue(_ >= 0, "The value must be non-negative.")
+      .createWithDefault(8)
+
+  val OPTIMIZE_NULL_AWARE_ANTI_JOIN =
+    buildConf("spark.sql.optimizeNullAwareAntiJoin")
+      .internal()
+      .doc("When true, NULL-aware anti join execution will be planed into " +
+        "BroadcastHashJoinExec with flag isNullAwareAntiJoin enabled, " +
+        "optimized from O(M*N) calculation into O(M) calculation " +
+        "using Hash lookup instead of Looping lookup." +
+        "Only support for singleColumn NAAJ for now.")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(true)
+
+  val LEGACY_COMPLEX_TYPES_TO_STRING =
+    buildConf("spark.sql.legacy.castComplexTypesToString.enabled")
+      .internal()
+      .doc("When true, maps and structs are wrapped by [] in casting to strings, and " +
+        "NULL elements of structs/maps/arrays will be omitted while converting to strings. " +
+        "Otherwise, if this is false, which is the default, maps and structs are wrapped by {}, " +
+        "and NULL elements will be converted to \"null\".")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(false)
+
+  val LEGACY_PATH_OPTION_BEHAVIOR =
+    buildConf("spark.sql.legacy.pathOptionBehavior.enabled")
+      .internal()
+      .doc("When true, \"path\" option is overwritten if one path parameter is passed to " +
+        "DataFrameReader.load(), DataFrameWriter.save(), DataStreamReader.load(), or " +
+        "DataStreamWriter.start(). Also, \"path\" option is added to the overall paths if " +
+        "multiple path parameters are passed to DataFrameReader.load()")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(false)
+
+  val LEGACY_EXTRA_OPTIONS_BEHAVIOR =
+    buildConf("spark.sql.legacy.extraOptionsBehavior.enabled")
+      .internal()
+      .doc("When true, the extra options will be ignored for DataFrameReader.table(). If set it " +
+        "to false, which is the default, Spark will check if the extra options have the same " +
+        "key, but the value is different with the table serde properties. If the check passes, " +
+        "the extra options will be merged with the serde properties as the scan options. " +
+        "Otherwise, an exception will be thrown.")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(false)
+
+  val DISABLED_JDBC_CONN_PROVIDER_LIST =
+    buildConf("spark.sql.sources.disabledJdbcConnProviderList")
+    .internal()
+    .doc("Configures a list of JDBC connection providers, which are disabled. " +
+      "The list contains the name of the JDBC connection providers separated by comma.")
+    .version("3.1.0")
+    .stringConf
+    .createWithDefault("")
+
+  val LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT =
+    buildConf("spark.sql.legacy.createHiveTableByDefault")
+      .internal()
+      .doc("When set to true, CREATE TABLE syntax without USING or STORED AS will use Hive " +
+        s"instead of the value of ${DEFAULT_DATA_SOURCE_NAME.key} as the table provider.")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(true)
+
+  val LEGACY_CHAR_VARCHAR_AS_STRING =
+    buildConf("spark.sql.legacy.charVarcharAsString")
+      .internal()
+      .doc("When true, Spark will not fail if user uses char and varchar type directly in those" +
+        " APIs that accept or parse data types as parameters, e.g." +
+        " `SparkSession.read.schema(...)`, `SparkSession.udf.register(...)` but treat them as" +
+        " string type as Spark 3.0 and earlier.")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(false)
+
+  val CLI_PRINT_HEADER =
+    buildConf("spark.sql.cli.print.header")
+     .doc("When set to true, spark-sql CLI prints the names of the columns in query output.")
+     .version("3.2.0")
+    .booleanConf
+    .createWithDefault(false)
+
+  val LEGACY_KEEP_COMMAND_OUTPUT_SCHEMA =
+    buildConf("spark.sql.legacy.keepCommandOutputSchema")
+      .internal()
+      .doc("When true, Spark will keep the output schema of commands such as SHOW DATABASES " +
+        "unchanged, for v1 catalog and/or table.")
+      .version("3.0.2")
+      .booleanConf
+      .createWithDefault(false)
+
+  val LEGACY_INTERVAL_ENABLED = buildConf("spark.sql.legacy.interval.enabled")
+    .internal()
+    .doc("When set to true, Spark SQL uses the mixed legacy interval type `CalendarIntervalType` " +
+      "instead of the ANSI compliant interval types `YearMonthIntervalType` and " +
+      "`DayTimeIntervalType`. For instance, the date subtraction expression returns " +
+      "`CalendarIntervalType` when the SQL config is set to `true` otherwise an ANSI interval.")
+    .version("3.2.0")
+    .booleanConf
+    .createWithDefault(false)
+
+  val MAX_CONCURRENT_OUTPUT_FILE_WRITERS = buildConf("spark.sql.maxConcurrentOutputFileWriters")
+    .internal()
+    .doc("Maximum number of output file writers to use concurrently. If number of writers " +
+      "needed reaches this limit, task will sort rest of output then writing them.")
+    .version("3.2.0")
+    .intConf
+    .createWithDefault(0)
+
   /**
    * Holds information about keys that have been deprecated.
    *
@@ -2618,7 +3213,25 @@ object SQLConf {
         s"Use '${ADVISORY_PARTITION_SIZE_IN_BYTES.key}' instead of it."),
       DeprecatedConfig(OPTIMIZER_METADATA_ONLY.key, "3.0",
         "Avoid to depend on this optimization to prevent a potential correctness issue. " +
-          "If you must use, use 'SparkSessionExtensions' instead to inject it as a custom rule.")
+          "If you must use, use 'SparkSessionExtensions' instead to inject it as a custom rule."),
+      DeprecatedConfig(CONVERT_CTAS.key, "3.1",
+        s"Set '${LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT.key}' to false instead."),
+      DeprecatedConfig("spark.sql.sources.schemaStringLengthThreshold", "3.2",
+        s"Use '${HIVE_TABLE_PROPERTY_LENGTH_THRESHOLD.key}' instead."),
+      DeprecatedConfig(PARQUET_INT96_REBASE_MODE_IN_WRITE.alternatives.head, "3.2",
+        s"Use '${PARQUET_INT96_REBASE_MODE_IN_WRITE.key}' instead."),
+      DeprecatedConfig(PARQUET_INT96_REBASE_MODE_IN_READ.alternatives.head, "3.2",
+        s"Use '${PARQUET_INT96_REBASE_MODE_IN_READ.key}' instead."),
+      DeprecatedConfig(PARQUET_REBASE_MODE_IN_WRITE.alternatives.head, "3.2",
+        s"Use '${PARQUET_REBASE_MODE_IN_WRITE.key}' instead."),
+      DeprecatedConfig(PARQUET_REBASE_MODE_IN_READ.alternatives.head, "3.2",
+        s"Use '${PARQUET_REBASE_MODE_IN_READ.key}' instead."),
+      DeprecatedConfig(AVRO_REBASE_MODE_IN_WRITE.alternatives.head, "3.2",
+        s"Use '${AVRO_REBASE_MODE_IN_WRITE.key}' instead."),
+      DeprecatedConfig(AVRO_REBASE_MODE_IN_READ.alternatives.head, "3.2",
+        s"Use '${AVRO_REBASE_MODE_IN_READ.key}' instead."),
+      DeprecatedConfig(LEGACY_REPLACE_DATABRICKS_SPARK_AVRO_ENABLED.key, "3.2",
+        """Use `.format("avro")` in `DataFrameWriter` or `DataFrameReader` instead.""")
     )
 
     Map(configs.map { cfg => cfg.key -> cfg } : _*)
@@ -2658,7 +3271,13 @@ object SQLConf {
         s"Use '${PARQUET_OUTPUT_TIMESTAMP_TYPE.key}' instead of it."),
       RemovedConfig("spark.sql.execution.pandas.respectSessionTimeZone", "3.0.0", "true",
         "The non-default behavior is considered as a bug, see SPARK-22395. " +
-        "The config was deprecated since Spark 2.3.")
+        "The config was deprecated since Spark 2.3."),
+      RemovedConfig("spark.sql.optimizer.planChangeLog.level", "3.1.0", "trace",
+        s"Please use `${PLAN_CHANGE_LOG_LEVEL.key}` instead."),
+      RemovedConfig("spark.sql.optimizer.planChangeLog.rules", "3.1.0", "",
+        s"Please use `${PLAN_CHANGE_LOG_RULES.key}` instead."),
+      RemovedConfig("spark.sql.optimizer.planChangeLog.batches", "3.1.0", "",
+        s"Please use `${PLAN_CHANGE_LOG_BATCHES.key}` instead.")
     )
 
     Map(configs.map { cfg => cfg.key -> cfg } : _*)
@@ -2695,11 +3314,11 @@ class SQLConf extends Serializable with Logging {
 
   def optimizerInSetSwitchThreshold: Int = getConf(OPTIMIZER_INSET_SWITCH_THRESHOLD)
 
-  def optimizerPlanChangeLogLevel: String = getConf(OPTIMIZER_PLAN_CHANGE_LOG_LEVEL)
+  def planChangeLogLevel: String = getConf(PLAN_CHANGE_LOG_LEVEL)
 
-  def optimizerPlanChangeRules: Option[String] = getConf(OPTIMIZER_PLAN_CHANGE_LOG_RULES)
+  def planChangeRules: Option[String] = getConf(PLAN_CHANGE_LOG_RULES)
 
-  def optimizerPlanChangeBatches: Option[String] = getConf(OPTIMIZER_PLAN_CHANGE_LOG_BATCHES)
+  def planChangeBatches: Option[String] = getConf(PLAN_CHANGE_LOG_BATCHES)
 
   def dynamicPartitionPruningEnabled: Boolean = getConf(DYNAMIC_PARTITION_PRUNING_ENABLED)
 
@@ -2711,14 +3330,26 @@ class SQLConf extends Serializable with Logging {
   def dynamicPartitionPruningReuseBroadcastOnly: Boolean =
     getConf(DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY)
 
+  def dynamicPartitionPruningPruningSideExtraFilterRatio: Double =
+    getConf(DYNAMIC_PARTITION_PRUNING_PRUNING_SIDE_EXTRA_FILTER_RATIO)
+
   def stateStoreProviderClass: String = getConf(STATE_STORE_PROVIDER_CLASS)
 
+  def isStateSchemaCheckEnabled: Boolean = getConf(STATE_SCHEMA_CHECK_ENABLED)
+
   def stateStoreMinDeltasForSnapshot: Int = getConf(STATE_STORE_MIN_DELTAS_FOR_SNAPSHOT)
 
+  def stateStoreFormatValidationEnabled: Boolean = getConf(STATE_STORE_FORMAT_VALIDATION_ENABLED)
+
   def checkpointLocation: Option[String] = getConf(CHECKPOINT_LOCATION)
 
   def isUnsupportedOperationCheckEnabled: Boolean = getConf(UNSUPPORTED_OPERATION_CHECK_ENABLED)
 
+  def useDeprecatedKafkaOffsetFetching: Boolean = getConf(USE_DEPRECATED_KAFKA_OFFSET_FETCHING)
+
+  def statefulOperatorCorrectnessCheckEnabled: Boolean =
+    getConf(STATEFUL_OPERATOR_CHECK_CORRECTNESS_ENABLED)
+
   def streamingFileCommitProtocolClass: String = getConf(STREAMING_FILE_COMMIT_PROTOCOL_CLASS)
 
   def fileSinkLogDeletion: Boolean = getConf(FILE_SINK_LOG_DELETION)
@@ -2751,6 +3382,8 @@ class SQLConf extends Serializable with Logging {
 
   def filesOpenCostInBytes: Long = getConf(FILES_OPEN_COST_IN_BYTES)
 
+  def filesMinPartitionNum: Option[Int] = getConf(FILES_MIN_PARTITION_NUM)
+
   def ignoreCorruptFiles: Boolean = getConf(IGNORE_CORRUPT_FILES)
 
   def ignoreMissingFiles: Boolean = getConf(IGNORE_MISSING_FILES)
@@ -2765,6 +3398,9 @@ class SQLConf extends Serializable with Logging {
 
   def orcVectorizedReaderBatchSize: Int = getConf(ORC_VECTORIZED_READER_BATCH_SIZE)
 
+  def orcVectorizedReaderNestedColumnEnabled: Boolean =
+    getConf(ORC_VECTORIZED_READER_NESTED_COLUMN_ENABLED)
+
   def parquetCompressionCodec: String = getConf(PARQUET_COMPRESSION)
 
   def parquetVectorizedReaderEnabled: Boolean = getConf(PARQUET_VECTORIZED_READER_ENABLED)
@@ -2775,7 +3411,15 @@ class SQLConf extends Serializable with Logging {
 
   def cacheVectorizedReaderEnabled: Boolean = getConf(CACHE_VECTORIZED_READER_ENABLED)
 
-  def numShufflePartitions: Int = getConf(SHUFFLE_PARTITIONS)
+  def defaultNumShufflePartitions: Int = getConf(SHUFFLE_PARTITIONS)
+
+  def numShufflePartitions: Int = {
+    if (adaptiveExecutionEnabled && coalesceShufflePartitionsEnabled) {
+      getConf(COALESCE_PARTITIONS_INITIAL_PARTITION_NUM).getOrElse(defaultNumShufflePartitions)
+    } else {
+      defaultNumShufflePartitions
+    }
+  }
 
   def adaptiveExecutionEnabled: Boolean = getConf(ADAPTIVE_EXECUTION_ENABLED)
 
@@ -2788,13 +3432,14 @@ class SQLConf extends Serializable with Logging {
 
   def coalesceShufflePartitionsEnabled: Boolean = getConf(COALESCE_PARTITIONS_ENABLED)
 
-  def initialShufflePartitionNum: Int =
-    getConf(COALESCE_PARTITIONS_INITIAL_PARTITION_NUM).getOrElse(numShufflePartitions)
-
   def minBatchesToRetain: Int = getConf(MIN_BATCHES_TO_RETAIN)
 
   def maxBatchesToRetainInMemory: Int = getConf(MAX_BATCHES_TO_RETAIN_IN_MEMORY)
 
+  def streamingMaintenanceInterval: Long = getConf(STREAMING_MAINTENANCE_INTERVAL)
+
+  def stateStoreCompressionCodec: String = getConf(STATE_STORE_COMPRESSION_CODEC)
+
   def parquetFilterPushDown: Boolean = getConf(PARQUET_FILTER_PUSHDOWN_ENABLED)
 
   def parquetFilterPushDownDate: Boolean = getConf(PARQUET_FILTER_PUSHDOWN_DATE_ENABLED)
@@ -2817,6 +3462,9 @@ class SQLConf extends Serializable with Logging {
 
   def metastorePartitionPruning: Boolean = getConf(HIVE_METASTORE_PARTITION_PRUNING)
 
+  def metastorePartitionPruningInSetThreshold: Int =
+    getConf(HIVE_METASTORE_PARTITION_PRUNING_INSET_THRESHOLD)
+
   def manageFilesourcePartitions: Boolean = getConf(HIVE_MANAGE_FILESOURCE_PARTITIONS)
 
   def filesourcePartitionFileCacheSize: Long = getConf(HIVE_FILESOURCE_PARTITION_FILE_CACHE_SIZE)
@@ -2885,6 +3533,9 @@ class SQLConf extends Serializable with Logging {
     LegacyBehaviorPolicy.withName(getConf(SQLConf.LEGACY_TIME_PARSER_POLICY))
   }
 
+  def broadcastHashJoinOutputPartitioningExpandLimit: Int =
+    getConf(BROADCAST_HASH_JOIN_OUTPUT_PARTITIONING_EXPAND_LIMIT)
+
   /**
    * Returns the [[Resolver]] for the current configuration, which can be used to determine if two
    * identifiers are equal.
@@ -2905,6 +3556,9 @@ class SQLConf extends Serializable with Logging {
   def subexpressionEliminationEnabled: Boolean =
     getConf(SUBEXPRESSION_ELIMINATION_ENABLED)
 
+  def subexpressionEliminationCacheMaxEntries: Int =
+    getConf(SUBEXPRESSION_ELIMINATION_CACHE_MAX_ENTRIES)
+
   def autoBroadcastJoinThreshold: Long = getConf(AUTO_BROADCASTJOIN_THRESHOLD)
 
   def limitScaleUpFactor: Int = getConf(LIMIT_SCALE_UP_FACTOR)
@@ -2968,6 +3622,8 @@ class SQLConf extends Serializable with Logging {
 
   def bucketingMaxBuckets: Int = getConf(SQLConf.BUCKETING_MAX_BUCKETS)
 
+  def autoBucketedScanEnabled: Boolean = getConf(SQLConf.AUTO_BUCKETED_SCAN_ENABLED)
+
   def dataFrameSelfJoinAutoResolveAmbiguity: Boolean =
     getConf(DATAFRAME_SELF_JOIN_AUTO_RESOLVE_AMBIGUITY)
 
@@ -3004,6 +3660,10 @@ class SQLConf extends Serializable with Logging {
 
   def jsonGeneratorIgnoreNullFields: Boolean = getConf(SQLConf.JSON_GENERATOR_IGNORE_NULL_FIELDS)
 
+  def jsonExpressionOptimization: Boolean = getConf(SQLConf.JSON_EXPRESSION_OPTIMIZATION)
+
+  def csvExpressionOptimization: Boolean = getConf(SQLConf.CSV_EXPRESSION_OPTIMIZATION)
+
   def parallelFileListingInStatsComputation: Boolean =
     getConf(SQLConf.PARALLEL_FILE_LISTING_IN_STATS_COMPUTATION)
 
@@ -3053,6 +3713,13 @@ class SQLConf extends Serializable with Logging {
 
   def maxNestedViewDepth: Int = getConf(SQLConf.MAX_NESTED_VIEW_DEPTH)
 
+  def useCurrentSQLConfigsForView: Boolean = getConf(SQLConf.USE_CURRENT_SQL_CONFIGS_FOR_VIEW)
+
+  def storeAnalyzedPlanForView: Boolean = getConf(SQLConf.STORE_ANALYZED_PLAN_FOR_VIEW)
+
+  def allowStarWithSingleTableIdentifierInCount: Boolean =
+    getConf(SQLConf.ALLOW_STAR_WITH_SINGLE_TABLE_IDENTIFIER_IN_COUNT)
+
   def starSchemaDetection: Boolean = getConf(STARSCHEMA_DETECTION)
 
   def starSchemaFTRatio: Double = getConf(STARSCHEMA_FACT_TABLE_RATIO)
@@ -3063,6 +3730,10 @@ class SQLConf extends Serializable with Logging {
 
   def arrowPySparkEnabled: Boolean = getConf(ARROW_PYSPARK_EXECUTION_ENABLED)
 
+  def arrowPySparkSelfDestructEnabled: Boolean = getConf(ARROW_PYSPARK_SELF_DESTRUCT_ENABLED)
+
+  def pysparkJVMStacktraceEnabled: Boolean = getConf(PYSPARK_JVM_STACKTRACE_ENABLED)
+
   def arrowSparkREnabled: Boolean = getConf(ARROW_SPARKR_EXECUTION_ENABLED)
 
   def arrowPySparkFallbackEnabled: Boolean = getConf(ARROW_PYSPARK_FALLBACK_ENABLED)
@@ -3071,6 +3742,8 @@ class SQLConf extends Serializable with Logging {
 
   def pandasUDFBufferSize: Int = getConf(PANDAS_UDF_BUFFER_SIZE)
 
+  def pysparkSimplifiedTraceback: Boolean = getConf(PYSPARK_SIMPLIFIEID_TRACEBACK)
+
   def pandasGroupedMapAssignColumnsByName: Boolean =
     getConf(SQLConf.PANDAS_GROUPED_MAP_ASSIGN_COLUMNS_BY_NAME)
 
@@ -3095,6 +3768,8 @@ class SQLConf extends Serializable with Logging {
   def disabledV2StreamingMicroBatchReaders: String =
     getConf(DISABLED_V2_STREAMING_MICROBATCH_READERS)
 
+  def fastFailFileFormatOutput: Boolean = getConf(FASTFAIL_ON_FILEFORMAT_OUTPUT)
+
   def concatBinaryAsString: Boolean = getConf(CONCAT_BINARY_AS_STRING)
 
   def eltOutputAsString: Boolean = getConf(ELT_OUTPUT_AS_STRING)
@@ -3144,6 +3819,8 @@ class SQLConf extends Serializable with Logging {
   def allowNegativeScaleOfDecimalEnabled: Boolean =
     getConf(SQLConf.LEGACY_ALLOW_NEGATIVE_SCALE_OF_DECIMAL_ENABLED)
 
+  def legacyStatisticalAggregate: Boolean = getConf(SQLConf.LEGACY_STATISTICAL_AGGREGATE)
+
   def truncateTableIgnorePermissionAcl: Boolean =
     getConf(SQLConf.TRUNCATE_TABLE_IGNORE_PERMISSION_ACL)
 
@@ -3154,6 +3831,8 @@ class SQLConf extends Serializable with Logging {
 
   def maxPlanStringLength: Int = getConf(SQLConf.MAX_PLAN_STRING_LENGTH).toInt
 
+  def maxMetadataStringLength: Int = getConf(SQLConf.MAX_METADATA_STRING_LENGTH)
+
   def setCommandRejectsSparkCoreConfs: Boolean =
     getConf(SQLConf.SET_COMMAND_REJECTS_SPARK_CORE_CONFS)
 
@@ -3163,8 +3842,36 @@ class SQLConf extends Serializable with Logging {
 
   def csvFilterPushDown: Boolean = getConf(CSV_FILTER_PUSHDOWN_ENABLED)
 
+  def jsonFilterPushDown: Boolean = getConf(JSON_FILTER_PUSHDOWN_ENABLED)
+
+  def avroFilterPushDown: Boolean = getConf(AVRO_FILTER_PUSHDOWN_ENABLED)
+
   def integerGroupingIdEnabled: Boolean = getConf(SQLConf.LEGACY_INTEGER_GROUPING_ID)
 
+  def metadataCacheTTL: Long = getConf(StaticSQLConf.METADATA_CACHE_TTL_SECONDS)
+
+  def coalesceBucketsInJoinEnabled: Boolean = getConf(SQLConf.COALESCE_BUCKETS_IN_JOIN_ENABLED)
+
+  def coalesceBucketsInJoinMaxBucketRatio: Int =
+    getConf(SQLConf.COALESCE_BUCKETS_IN_JOIN_MAX_BUCKET_RATIO)
+
+  def optimizeNullAwareAntiJoin: Boolean =
+    getConf(SQLConf.OPTIMIZE_NULL_AWARE_ANTI_JOIN)
+
+  def legacyPathOptionBehavior: Boolean = getConf(SQLConf.LEGACY_PATH_OPTION_BEHAVIOR)
+
+  def disabledJdbcConnectionProviders: String = getConf(SQLConf.DISABLED_JDBC_CONN_PROVIDER_LIST)
+
+  def charVarcharAsString: Boolean = getConf(SQLConf.LEGACY_CHAR_VARCHAR_AS_STRING)
+
+  def cliPrintHeader: Boolean = getConf(SQLConf.CLI_PRINT_HEADER)
+
+  def legacyIntervalEnabled: Boolean = getConf(LEGACY_INTERVAL_ENABLED)
+
+  def decorrelateInnerQueryEnabled: Boolean = getConf(SQLConf.DECORRELATE_INNER_QUERY_ENABLED)
+
+  def maxConcurrentOutputFileWriters: Int = getConf(SQLConf.MAX_CONCURRENT_OUTPUT_FILE_WRITERS)
+
   /** ********************** SQLConf functionality methods ************ */
 
   /** Set Spark SQL configuration properties. */
@@ -3253,6 +3960,27 @@ class SQLConf extends Serializable with Logging {
     }
   }
 
+  private var definedConfsLoaded = false
+  /**
+   * Init [[StaticSQLConf]] and [[org.apache.spark.sql.hive.HiveUtils]] so that all the defined
+   * SQL Configurations will be registered to SQLConf
+   */
+  private def loadDefinedConfs(): Unit = {
+    if (!definedConfsLoaded) {
+      definedConfsLoaded = true
+      // Force to register static SQL configurations
+      StaticSQLConf
+      try {
+        // Force to register SQL configurations from Hive module
+        val symbol = ScalaReflection.mirror.staticModule("org.apache.spark.sql.hive.HiveUtils")
+        ScalaReflection.mirror.reflectModule(symbol).instance
+      } catch {
+        case NonFatal(e) =>
+          logWarning("SQL configurations from Hive module is not loaded", e)
+      }
+    }
+  }
+
   /**
    * Return all the configuration properties that have been set (i.e. not the default).
    * This creates a new copy of the config properties in the form of a Map.
@@ -3265,6 +3993,7 @@ class SQLConf extends Serializable with Logging {
    * definition contains key, defaultValue and doc.
    */
   def getAllDefinedConfs: Seq[(String, String, String, String)] = sqlConfEntries.synchronized {
+    loadDefinedConfs()
     sqlConfEntries.values.asScala.filter(_.isPublic).map { entry =>
       val displayValue = Option(getConfString(entry.key, null)).getOrElse(entry.defaultValueString)
       (entry.key, displayValue, entry.doc, entry.version)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
index 9618ff6062635..bfefca4e2ebad 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.internal
 
 import java.util.Locale
+import java.util.concurrent.TimeUnit
 
 import org.apache.spark.util.Utils
 
@@ -126,6 +127,16 @@ object StaticSQLConf {
     .toSequence
     .createOptional
 
+  val SPARK_CACHE_SERIALIZER = buildStaticConf("spark.sql.cache.serializer")
+    .doc("The name of a class that implements " +
+      "org.apache.spark.sql.columnar.CachedBatchSerializer. It will be used to " +
+      "translate SQL data into a format that can more efficiently be cached. The underlying " +
+      "API is subject to change so use with caution. Multiple classes cannot be specified. " +
+      "The class must have a no-arg constructor.")
+    .version("3.1.0")
+    .stringConf
+    .createWithDefault("org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer")
+
   val QUERY_EXECUTION_LISTENERS = buildStaticConf("spark.sql.queryExecutionListeners")
     .doc("List of class names implementing QueryExecutionListener that will be automatically " +
       "added to newly created sessions. The classes should have either a no-arg constructor, " +
@@ -184,6 +195,7 @@ object StaticSQLConf {
 
   val SQL_LEGACY_SESSION_INIT_WITH_DEFAULTS =
     buildStaticConf("spark.sql.legacy.sessionInitWithConfigDefaults")
+      .internal()
       .doc("Flag to revert to legacy behavior where a cloned SparkSession receives SparkConf " +
         "defaults, dropping any overrides in its parent SparkSession.")
       .version("3.0.0")
@@ -226,4 +238,28 @@ object StaticSQLConf {
       .version("3.0.0")
       .intConf
       .createWithDefault(100)
+
+  val METADATA_CACHE_TTL_SECONDS = buildStaticConf("spark.sql.metadataCacheTTLSeconds")
+    .doc("Time-to-live (TTL) value for the metadata caches: partition file metadata cache and " +
+      "session catalog cache. This configuration only has an effect when this value having " +
+      "a positive value (> 0). It also requires setting " +
+      s"'${StaticSQLConf.CATALOG_IMPLEMENTATION.key}' to `hive`, setting " +
+      s"'${SQLConf.HIVE_FILESOURCE_PARTITION_FILE_CACHE_SIZE.key}' > 0 and setting " +
+      s"'${SQLConf.HIVE_MANAGE_FILESOURCE_PARTITIONS.key}' to `true` " +
+      "to be applied to the partition file metadata cache.")
+    .version("3.1.0")
+    .timeConf(TimeUnit.SECONDS)
+    .createWithDefault(-1)
+
+  val ENABLED_STREAMING_UI_CUSTOM_METRIC_LIST =
+    buildStaticConf("spark.sql.streaming.ui.enabledCustomMetricList")
+      .internal()
+      .doc("Configures a list of custom metrics on Structured Streaming UI, which are enabled. " +
+        "The list contains the name of the custom metrics separated by comma. In aggregation" +
+        "only sum used. The list of supported custom metrics is state store provider specific " +
+        "and it can be found out for example from query progress log entry.")
+      .version("3.1.0")
+      .stringConf
+      .toSequence
+      .createWithDefault(Nil)
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/SupportsMetadata.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/SupportsMetadata.scala
new file mode 100644
index 0000000000000..b2cb19b009141
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/SupportsMetadata.scala
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.internal.connector
+
+/**
+ * A mix-in interface for {@link FileScan}. This can be used to report metadata
+ * for a file based scan operator. This is currently used for supporting formatted
+ * explain.
+ */
+trait SupportsMetadata {
+  def getMetaData(): Map[String, String]
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/SupportsStreamingUpdateAsAppend.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/SupportsStreamingUpdateAsAppend.scala
new file mode 100644
index 0000000000000..6cacac7b9de89
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/SupportsStreamingUpdateAsAppend.scala
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.internal.connector
+
+import org.apache.spark.sql.connector.write.WriteBuilder
+
+/**
+ * An internal `WriteBuilder` mixin to support UPDATE streaming output mode. Now there's no good
+ * way to pass the `keys` to upsert or replace (delete -> append), we do the same with append writes
+ * and let end users to deal with.
+ *
+ * This approach may be still valid for streaming writers which can't do the upsert or replace.
+ * We can promote the API to the official API along with the new API for upsert/replace.
+ */
+// TODO: design an official API for streaming output mode UPDATE which can do the upsert
+//  (or delete -> append).
+trait SupportsStreamingUpdateAsAppend extends WriteBuilder {
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/sources/filters.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/sources/filters.scala
index 7533793253513..2b44a3a861ed9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/sources/filters.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/sources/filters.scala
@@ -164,7 +164,7 @@ case class In(attribute: String, values: Array[Any]) extends Filter {
     var h = attribute.hashCode
     values.foreach { v =>
       h *= 41
-      h += v.hashCode()
+      h += (if (v != null) v.hashCode() else 0)
     }
     h
   }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala
index 21ac32adca6e9..02c95b286a215 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala
@@ -82,7 +82,11 @@ private[sql] object TypeCollection {
    * Types that include numeric types and interval type. They are only used in unary_minus,
    * unary_positive, add and subtract operations.
    */
-  val NumericAndInterval = TypeCollection(NumericType, CalendarIntervalType)
+  val NumericAndInterval = TypeCollection(
+    NumericType,
+    CalendarIntervalType,
+    DayTimeIntervalType,
+    YearMonthIntervalType)
 
   def apply(types: AbstractDataType*): TypeCollection = new TypeCollection(types)
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/CalendarIntervalType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/CalendarIntervalType.scala
index 35ad864db0e7d..d506a1521e183 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/CalendarIntervalType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/CalendarIntervalType.scala
@@ -28,7 +28,7 @@ import org.apache.spark.annotation.Stable
  *
  * Please use the singleton `DataTypes.CalendarIntervalType` to refer the type.
  *
- * @note Calendar intervals support comparison and ordering since 3.0.0.
+ * @note Calendar intervals are not comparable.
  *
  * @since 1.5.0
  */
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/CharType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/CharType.scala
new file mode 100644
index 0000000000000..67ab1cc2f3321
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/CharType.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.types
+
+import scala.math.Ordering
+import scala.reflect.runtime.universe.typeTag
+
+import org.apache.spark.annotation.Experimental
+import org.apache.spark.unsafe.types.UTF8String
+
+@Experimental
+case class CharType(length: Int) extends AtomicType {
+  require(length >= 0, "The length of char type cannot be negative.")
+
+  private[sql] type InternalType = UTF8String
+  @transient private[sql] lazy val tag = typeTag[InternalType]
+  private[sql] val ordering = implicitly[Ordering[InternalType]]
+
+  override def defaultSize: Int = length
+  override def typeName: String = s"char($length)"
+  override def toString: String = s"CharType($length)"
+  private[spark] override def asNullable: CharType = this
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
index 7449a28e069d2..5c5742c812e8e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
@@ -28,6 +28,7 @@ import org.json4s.JsonDSL._
 import org.json4s.jackson.JsonMethods._
 
 import org.apache.spark.annotation.Stable
+import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.analysis.Resolver
 import org.apache.spark.sql.catalyst.expressions.{Cast, Expression}
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
@@ -123,20 +124,54 @@ abstract class DataType extends AbstractDataType {
 object DataType {
 
   private val FIXED_DECIMAL = """decimal\(\s*(\d+)\s*,\s*(\-?\d+)\s*\)""".r
+  private val CHAR_TYPE = """char\(\s*(\d+)\s*\)""".r
+  private val VARCHAR_TYPE = """varchar\(\s*(\d+)\s*\)""".r
 
   def fromDDL(ddl: String): DataType = {
+    parseTypeWithFallback(
+      ddl,
+      CatalystSqlParser.parseDataType,
+      "Cannot parse the data type: ",
+      fallbackParser = str => CatalystSqlParser.parseTableSchema(str))
+  }
+
+  /**
+   * Parses data type from a string with schema. It calls `parser` for `schema`.
+   * If it fails, calls `fallbackParser`. If the fallback function fails too, combines error message
+   * from `parser` and `fallbackParser`.
+   *
+   * @param schema The schema string to parse by `parser` or `fallbackParser`.
+   * @param parser The function that should be invoke firstly.
+   * @param errorMsg The error message for `parser`.
+   * @param fallbackParser The function that is called when `parser` fails.
+   * @return The data type parsed from the `schema` schema.
+   */
+  def parseTypeWithFallback(
+      schema: String,
+      parser: String => DataType,
+      errorMsg: String,
+      fallbackParser: String => DataType): DataType = {
     try {
-      CatalystSqlParser.parseDataType(ddl)
+      parser(schema)
     } catch {
-      case NonFatal(_) => CatalystSqlParser.parseTableSchema(ddl)
+      case NonFatal(e1) =>
+        try {
+          fallbackParser(schema)
+        } catch {
+          case NonFatal(e2) =>
+            throw new AnalysisException(
+              message = s"$errorMsg${e1.getMessage}\nFailed fallback parsing: ${e2.getMessage}",
+              cause = Some(e1.getCause))
+        }
     }
   }
 
   def fromJson(json: String): DataType = parseDataType(parse(json))
 
-  private val nonDecimalNameToType = {
+  private val otherTypes = {
     Seq(NullType, DateType, TimestampType, BinaryType, IntegerType, BooleanType, LongType,
-      DoubleType, FloatType, ShortType, ByteType, StringType, CalendarIntervalType)
+      DoubleType, FloatType, ShortType, ByteType, StringType, CalendarIntervalType,
+      DayTimeIntervalType, YearMonthIntervalType)
       .map(t => t.typeName -> t).toMap
   }
 
@@ -145,7 +180,9 @@ object DataType {
     name match {
       case "decimal" => DecimalType.USER_DEFAULT
       case FIXED_DECIMAL(precision, scale) => DecimalType(precision.toInt, scale.toInt)
-      case other => nonDecimalNameToType.getOrElse(
+      case CHAR_TYPE(length) => CharType(length.toInt)
+      case VARCHAR_TYPE(length) => VarcharType(length.toInt)
+      case other => otherTypes.getOrElse(
         other,
         throw new IllegalArgumentException(
           s"Failed to convert the JSON string '$name' to a data type."))
@@ -154,7 +191,7 @@ object DataType {
 
   private object JSortedObject {
     def unapplySeq(value: JValue): Option[List[(String, JValue)]] = value match {
-      case JObject(seq) => Some(seq.toList.sortBy(_._1))
+      case JObject(seq) => Some(seq.sortBy(_._1))
       case _ => None
     }
   }
@@ -271,21 +308,49 @@ object DataType {
    *   of `fromField.nullable` and `toField.nullable` are false.
    */
   private[sql] def equalsIgnoreCompatibleNullability(from: DataType, to: DataType): Boolean = {
+    equalsIgnoreCompatibleNullability(from, to, ignoreName = false)
+  }
+
+  /**
+   * Compares two types, ignoring compatible nullability of ArrayType, MapType, StructType, and
+   * also the field name. It compares based on the position.
+   *
+   * Compatible nullability is defined as follows:
+   *   - If `from` and `to` are ArrayTypes, `from` has a compatible nullability with `to`
+   *   if and only if `to.containsNull` is true, or both of `from.containsNull` and
+   *   `to.containsNull` are false.
+   *   - If `from` and `to` are MapTypes, `from` has a compatible nullability with `to`
+   *   if and only if `to.valueContainsNull` is true, or both of `from.valueContainsNull` and
+   *   `to.valueContainsNull` are false.
+   *   - If `from` and `to` are StructTypes, `from` has a compatible nullability with `to`
+   *   if and only if for all every pair of fields, `to.nullable` is true, or both
+   *   of `fromField.nullable` and `toField.nullable` are false.
+   */
+  private[sql] def equalsIgnoreNameAndCompatibleNullability(
+      from: DataType,
+      to: DataType): Boolean = {
+    equalsIgnoreCompatibleNullability(from, to, ignoreName = true)
+  }
+
+  private def equalsIgnoreCompatibleNullability(
+      from: DataType,
+      to: DataType,
+      ignoreName: Boolean = false): Boolean = {
     (from, to) match {
       case (ArrayType(fromElement, fn), ArrayType(toElement, tn)) =>
-        (tn || !fn) && equalsIgnoreCompatibleNullability(fromElement, toElement)
+        (tn || !fn) && equalsIgnoreCompatibleNullability(fromElement, toElement, ignoreName)
 
       case (MapType(fromKey, fromValue, fn), MapType(toKey, toValue, tn)) =>
         (tn || !fn) &&
-          equalsIgnoreCompatibleNullability(fromKey, toKey) &&
-          equalsIgnoreCompatibleNullability(fromValue, toValue)
+          equalsIgnoreCompatibleNullability(fromKey, toKey, ignoreName) &&
+          equalsIgnoreCompatibleNullability(fromValue, toValue, ignoreName)
 
       case (StructType(fromFields), StructType(toFields)) =>
         fromFields.length == toFields.length &&
           fromFields.zip(toFields).forall { case (fromField, toField) =>
-            fromField.name == toField.name &&
+            (ignoreName || fromField.name == toField.name) &&
               (toField.nullable || !fromField.nullable) &&
-              equalsIgnoreCompatibleNullability(fromField.dataType, toField.dataType)
+              equalsIgnoreCompatibleNullability(fromField.dataType, toField.dataType, ignoreName)
           }
 
       case (fromDataType, toDataType) => fromDataType == toDataType
@@ -328,19 +393,19 @@ object DataType {
       ignoreNullability: Boolean = false): Boolean = {
     (from, to) match {
       case (left: ArrayType, right: ArrayType) =>
-        equalsStructurally(left.elementType, right.elementType) &&
+        equalsStructurally(left.elementType, right.elementType, ignoreNullability) &&
           (ignoreNullability || left.containsNull == right.containsNull)
 
       case (left: MapType, right: MapType) =>
-        equalsStructurally(left.keyType, right.keyType) &&
-          equalsStructurally(left.valueType, right.valueType) &&
+        equalsStructurally(left.keyType, right.keyType, ignoreNullability) &&
+          equalsStructurally(left.valueType, right.valueType, ignoreNullability) &&
           (ignoreNullability || left.valueContainsNull == right.valueContainsNull)
 
       case (StructType(fromFields), StructType(toFields)) =>
         fromFields.length == toFields.length &&
           fromFields.zip(toFields)
             .forall { case (l, r) =>
-              equalsStructurally(l.dataType, r.dataType) &&
+              equalsStructurally(l.dataType, r.dataType, ignoreNullability) &&
                 (ignoreNullability || l.nullable == r.nullable)
             }
 
@@ -457,7 +522,7 @@ object DataType {
 
       case (w: AtomicType, r: AtomicType) if storeAssignmentPolicy == STRICT =>
         if (!Cast.canUpCast(w, r)) {
-          addError(s"Cannot safely cast '$context': $w to $r")
+          addError(s"Cannot safely cast '$context': ${w.catalogString} to ${r.catalogString}")
           false
         } else {
           true
@@ -467,7 +532,7 @@ object DataType {
 
       case (w: AtomicType, r: AtomicType) if storeAssignmentPolicy == ANSI =>
         if (!Cast.canANSIStoreAssign(w, r)) {
-          addError(s"Cannot safely cast '$context': $w to $r")
+          addError(s"Cannot safely cast '$context': ${w.catalogString} to ${r.catalogString}")
           false
         } else {
           true
@@ -477,7 +542,8 @@ object DataType {
         true
 
       case (w, r) =>
-        addError(s"Cannot write '$context': $w is incompatible with $r")
+        addError(s"Cannot write '$context': " +
+          s"${w.catalogString} is incompatible with ${r.catalogString}")
         false
     }
   }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DayTimeIntervalType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DayTimeIntervalType.scala
new file mode 100644
index 0000000000000..0582f2053c372
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DayTimeIntervalType.scala
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.types
+
+import scala.math.Ordering
+import scala.reflect.runtime.universe.typeTag
+
+import org.apache.spark.annotation.Unstable
+
+/**
+ * The type represents day-time intervals of the SQL standard. A day-time interval is made up
+ * of a contiguous subset of the following fields:
+ *   - SECOND, seconds within minutes and possibly fractions of a second [0..59.999999],
+ *   - MINUTE, minutes within hours [0..59],
+ *   - HOUR, hours within days [0..23],
+ *   - DAY, days in the range [0..106751991].
+ *
+ * `DayTimeIntervalType` represents positive as well as negative day-time intervals.
+ *
+ * Please use the singleton `DataTypes.DayTimeIntervalType` to refer the type.
+ *
+ * @since 3.2.0
+ */
+@Unstable
+class DayTimeIntervalType private() extends AtomicType {
+  /**
+   * Internally, values of day-time intervals are stored in `Long` values as amount of time in terms
+   * of microseconds that are calculated by the formula:
+   *   -/+ (24*60*60 * DAY + 60*60 * HOUR + 60 * MINUTE + SECOND) * 1000000
+   */
+  private[sql] type InternalType = Long
+
+  @transient private[sql] lazy val tag = typeTag[InternalType]
+
+  private[sql] val ordering = implicitly[Ordering[InternalType]]
+
+  /**
+   * The day-time interval type has constant precision. A value of the type always occupies 8 bytes.
+   * The DAY field is constrained by the upper bound 106751991 to fit to `Long`.
+   */
+  override def defaultSize: Int = 8
+
+  private[spark] override def asNullable: DayTimeIntervalType = this
+
+  override def typeName: String = "interval day to second"
+}
+
+/**
+ * The companion case object and its class is separated so the companion object also subclasses
+ * the DayTimeIntervalType class. Otherwise, the companion object would be of type
+ * "DayTimeIntervalType$" in byte code. Defined with a private constructor so the companion object
+ * is the only possible instantiation.
+ *
+ * @since 3.2.0
+ */
+@Unstable
+case object DayTimeIntervalType extends DayTimeIntervalType
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
index f32e48e1cc128..bad14791abb0e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
@@ -17,13 +17,14 @@
 
 package org.apache.spark.sql.types
 
-import java.lang.{Long => JLong}
-import java.math.{BigInteger, MathContext, RoundingMode}
+import java.math.{BigDecimal => JavaBigDecimal, BigInteger, MathContext, RoundingMode}
 
 import scala.util.Try
 
 import org.apache.spark.annotation.Unstable
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.unsafe.types.UTF8String
 
 /**
  * A mutable implementation of BigDecimal that can hold a Long if values are small enough.
@@ -80,7 +81,7 @@ final class Decimal extends Ordered[Decimal] with Serializable {
    */
   def set(unscaled: Long, precision: Int, scale: Int): Decimal = {
     if (setOrNull(unscaled, precision, scale) == null) {
-      throw new ArithmeticException("Unscaled value too large for precision")
+      throw QueryExecutionErrors.unscaledValueTooLargeForPrecisionError()
     }
     this
   }
@@ -118,8 +119,8 @@ final class Decimal extends Ordered[Decimal] with Serializable {
     DecimalType.checkNegativeScale(scale)
     this.decimalVal = decimal.setScale(scale, ROUND_HALF_UP)
     if (decimalVal.precision > precision) {
-      throw new ArithmeticException(
-        s"Decimal precision ${decimalVal.precision} exceeds max precision $precision")
+      throw QueryExecutionErrors.decimalPrecisionExceedsMaxPrecisionError(
+        decimalVal.precision, precision)
     }
     this.longVal = 0L
     this._precision = precision
@@ -251,7 +252,7 @@ final class Decimal extends Ordered[Decimal] with Serializable {
   def toByte: Byte = toLong.toByte
 
   private def overflowException(dataType: String) =
-    throw new ArithmeticException(s"Casting $this to $dataType causes overflow")
+    throw QueryExecutionErrors.castingCauseOverflowError(this, dataType)
 
   /**
    * @return the Byte value that is equal to the rounded decimal.
@@ -263,14 +264,14 @@ final class Decimal extends Ordered[Decimal] with Serializable {
       if (actualLongVal == actualLongVal.toByte) {
         actualLongVal.toByte
       } else {
-        overflowException("byte")
+        throw QueryExecutionErrors.castingCauseOverflowError(this, "byte")
       }
     } else {
       val doubleVal = decimalVal.toDouble
       if (Math.floor(doubleVal) <= Byte.MaxValue && Math.ceil(doubleVal) >= Byte.MinValue) {
         doubleVal.toByte
       } else {
-        overflowException("byte")
+        throw QueryExecutionErrors.castingCauseOverflowError(this, "byte")
       }
     }
   }
@@ -285,14 +286,14 @@ final class Decimal extends Ordered[Decimal] with Serializable {
       if (actualLongVal == actualLongVal.toShort) {
         actualLongVal.toShort
       } else {
-        overflowException("short")
+        throw QueryExecutionErrors.castingCauseOverflowError(this, "short")
       }
     } else {
       val doubleVal = decimalVal.toDouble
       if (Math.floor(doubleVal) <= Short.MaxValue && Math.ceil(doubleVal) >= Short.MinValue) {
         doubleVal.toShort
       } else {
-        overflowException("short")
+        throw QueryExecutionErrors.castingCauseOverflowError(this, "short")
       }
     }
   }
@@ -307,14 +308,14 @@ final class Decimal extends Ordered[Decimal] with Serializable {
       if (actualLongVal == actualLongVal.toInt) {
         actualLongVal.toInt
       } else {
-        overflowException("int")
+        throw QueryExecutionErrors.castingCauseOverflowError(this, "int")
       }
     } else {
       val doubleVal = decimalVal.toDouble
       if (Math.floor(doubleVal) <= Int.MaxValue && Math.ceil(doubleVal) >= Int.MinValue) {
         doubleVal.toInt
       } else {
-        overflowException("int")
+        throw QueryExecutionErrors.castingCauseOverflowError(this, "int")
       }
     }
   }
@@ -333,7 +334,8 @@ final class Decimal extends Ordered[Decimal] with Serializable {
         // `longValueExact` to make sure the range check is accurate.
         decimalVal.bigDecimal.toBigInteger.longValueExact()
       } catch {
-        case _: ArithmeticException => overflowException("long")
+        case _: ArithmeticException =>
+          throw QueryExecutionErrors.castingCauseOverflowError(this, "long")
       }
     }
   }
@@ -365,8 +367,7 @@ final class Decimal extends Ordered[Decimal] with Serializable {
       if (nullOnOverflow) {
         null
       } else {
-        throw new ArithmeticException(
-          s"$toDebugString cannot be represented as Decimal($precision, $scale).")
+        throw QueryExecutionErrors.cannotChangeDecimalPrecisionError(this, precision, scale)
       }
     }
   }
@@ -413,7 +414,7 @@ final class Decimal extends Ordered[Decimal] with Serializable {
               longVal += (if (droppedDigits < 0) -1L else 1L)
             }
           case _ =>
-            sys.error(s"Not supported rounding mode: $roundMode")
+            throw QueryExecutionErrors.unsupportedRoundingMode(roundMode)
         }
       } else if (scale > _scale) {
         // We might be able to multiply longVal by a power of 10 and not overflow, but if not,
@@ -478,7 +479,7 @@ final class Decimal extends Ordered[Decimal] with Serializable {
     if (decimalVal.eq(null) && that.decimalVal.eq(null) && scale == that.scale) {
       Decimal(longVal + that.longVal, Math.max(precision, that.precision), scale)
     } else {
-      Decimal(toBigDecimal + that.toBigDecimal)
+      Decimal(toBigDecimal.bigDecimal.add(that.toBigDecimal.bigDecimal))
     }
   }
 
@@ -486,7 +487,7 @@ final class Decimal extends Ordered[Decimal] with Serializable {
     if (decimalVal.eq(null) && that.decimalVal.eq(null) && scale == that.scale) {
       Decimal(longVal - that.longVal, Math.max(precision, that.precision), scale)
     } else {
-      Decimal(toBigDecimal - that.toBigDecimal)
+      Decimal(toBigDecimal.bigDecimal.subtract(that.toBigDecimal.bigDecimal))
     }
   }
 
@@ -550,9 +551,6 @@ object Decimal {
   private[sql] val ZERO = Decimal(0)
   private[sql] val ONE = Decimal(1)
 
-  private val LONG_MAX_BIG_INT = BigInteger.valueOf(JLong.MAX_VALUE)
-  private val LONG_MIN_BIG_INT = BigInteger.valueOf(JLong.MIN_VALUE)
-
   def apply(value: Double): Decimal = new Decimal().set(value)
 
   def apply(value: Long): Decimal = new Decimal().set(value)
@@ -589,6 +587,52 @@ object Decimal {
     }
   }
 
+  private def calculatePrecision(bigDecimal: JavaBigDecimal): Int = {
+    if (bigDecimal.scale < 0) {
+      bigDecimal.precision - bigDecimal.scale
+    } else {
+      bigDecimal.precision
+    }
+  }
+
+  private def stringToJavaBigDecimal(str: UTF8String): JavaBigDecimal = {
+    // According the benchmark test,  `s.toString.trim` is much faster than `s.trim.toString`.
+    // Please refer to https://github.com/apache/spark/pull/26640
+    new JavaBigDecimal(str.toString.trim)
+  }
+
+  def fromString(str: UTF8String): Decimal = {
+    try {
+      val bigDecimal = stringToJavaBigDecimal(str)
+      // We fast fail because constructing a very large JavaBigDecimal to Decimal is very slow.
+      // For example: Decimal("6.0790316E+25569151")
+      if (calculatePrecision(bigDecimal) > DecimalType.MAX_PRECISION) {
+        null
+      } else {
+        Decimal(bigDecimal)
+      }
+    } catch {
+      case _: NumberFormatException =>
+        null
+    }
+  }
+
+  def fromStringANSI(str: UTF8String): Decimal = {
+    try {
+      val bigDecimal = stringToJavaBigDecimal(str)
+      // We fast fail because constructing a very large JavaBigDecimal to Decimal is very slow.
+      // For example: Decimal("6.0790316E+25569151")
+      if (calculatePrecision(bigDecimal) > DecimalType.MAX_PRECISION) {
+        throw QueryExecutionErrors.outOfDecimalTypeRangeError(str)
+      } else {
+        Decimal(bigDecimal)
+      }
+    } catch {
+      case _: NumberFormatException =>
+        throw QueryExecutionErrors.invalidInputSyntaxForNumericError(str)
+    }
+  }
+
   /**
    * Creates a decimal from unscaled, precision and scale without checking the bounds.
    */
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DoubleType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DoubleType.scala
index 01268a9ff1667..ea4f39d4b19d2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DoubleType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DoubleType.scala
@@ -22,6 +22,7 @@ import scala.reflect.runtime.universe.typeTag
 import scala.util.Try
 
 import org.apache.spark.annotation.Stable
+import org.apache.spark.sql.catalyst.util.SQLOrderingUtil
 
 /**
  * The data type representing `Double` values. Please use the singleton `DataTypes.DoubleType`.
@@ -38,7 +39,7 @@ class DoubleType private() extends FractionalType {
   private[sql] val numeric = implicitly[Numeric[Double]]
   private[sql] val fractional = implicitly[Fractional[Double]]
   private[sql] val ordering =
-    (x: Double, y: Double) => java.lang.Double.compare(x, y)
+    (x: Double, y: Double) => SQLOrderingUtil.compareDoubles(x, y)
   private[sql] val asIntegral = DoubleType.DoubleAsIfIntegral
 
   override private[sql] def exactNumeric = DoubleExactNumeric
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/FloatType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/FloatType.scala
index 1491f5904baef..f00046facf693 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/FloatType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/FloatType.scala
@@ -22,6 +22,7 @@ import scala.reflect.runtime.universe.typeTag
 import scala.util.Try
 
 import org.apache.spark.annotation.Stable
+import org.apache.spark.sql.catalyst.util.SQLOrderingUtil
 
 /**
  * The data type representing `Float` values. Please use the singleton `DataTypes.FloatType`.
@@ -38,7 +39,7 @@ class FloatType private() extends FractionalType {
   private[sql] val numeric = implicitly[Numeric[Float]]
   private[sql] val fractional = implicitly[Fractional[Float]]
   private[sql] val ordering =
-    (x: Float, y: Float) => java.lang.Float.compare(x, y)
+    (x: Float, y: Float) => SQLOrderingUtil.compareFloats(x, y)
   private[sql] val asIntegral = FloatType.FloatAsIfIntegral
 
   override private[sql] def exactNumeric = FloatExactNumeric
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/HiveStringType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/HiveStringType.scala
deleted file mode 100644
index a29f49ad14a77..0000000000000
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/HiveStringType.scala
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.types
-
-import scala.math.Ordering
-import scala.reflect.runtime.universe.typeTag
-
-import org.apache.spark.unsafe.types.UTF8String
-
-/**
- * A hive string type for compatibility. These datatypes should only used for parsing,
- * and should NOT be used anywhere else. Any instance of these data types should be
- * replaced by a [[StringType]] before analysis.
- */
-sealed abstract class HiveStringType extends AtomicType {
-  private[sql] type InternalType = UTF8String
-
-  private[sql] val ordering = implicitly[Ordering[InternalType]]
-
-  @transient private[sql] lazy val tag = typeTag[InternalType]
-
-  override def defaultSize: Int = length
-
-  private[spark] override def asNullable: HiveStringType = this
-
-  def length: Int
-}
-
-object HiveStringType {
-  def replaceCharType(dt: DataType): DataType = dt match {
-    case ArrayType(et, nullable) =>
-      ArrayType(replaceCharType(et), nullable)
-    case MapType(kt, vt, nullable) =>
-      MapType(replaceCharType(kt), replaceCharType(vt), nullable)
-    case StructType(fields) =>
-      StructType(fields.map { field =>
-        field.copy(dataType = replaceCharType(field.dataType))
-      })
-    case _: HiveStringType => StringType
-    case _ => dt
-  }
-
-  def containsCharType(dt: DataType): Boolean = dt match {
-    case ArrayType(et, _) => containsCharType(et)
-    case MapType(kt, vt, _) => containsCharType(kt) || containsCharType(vt)
-    case StructType(fields) => fields.exists(f => containsCharType(f.dataType))
-    case _ => dt.isInstanceOf[CharType]
-  }
-}
-
-/**
- * Hive char type. Similar to other HiveStringType's, these datatypes should only used for
- * parsing, and should NOT be used anywhere else. Any instance of these data types should be
- * replaced by a [[StringType]] before analysis.
- */
-case class CharType(length: Int) extends HiveStringType {
-  override def simpleString: String = s"char($length)"
-}
-
-/**
- * Hive varchar type. Similar to other HiveStringType's, these datatypes should only used for
- * parsing, and should NOT be used anywhere else. Any instance of these data types should be
- * replaced by a [[StringType]] before analysis.
- */
-case class VarcharType(length: Int) extends HiveStringType {
-  override def simpleString: String = s"varchar($length)"
-}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Metadata.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Metadata.scala
index 982f6244f8a0d..3e05eda3443d9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Metadata.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Metadata.scala
@@ -23,6 +23,7 @@ import org.json4s._
 import org.json4s.jackson.JsonMethods._
 
 import org.apache.spark.annotation.Stable
+import org.apache.spark.sql.errors.QueryExecutionErrors
 
 
 /**
@@ -162,13 +163,13 @@ object Metadata {
               builder.putMetadataArray(
                 key, value.asInstanceOf[List[JObject]].map(fromJObject).toArray)
             case other =>
-              throw new RuntimeException(s"Do not support array of type ${other.getClass}.")
+              throw QueryExecutionErrors.unsupportedArrayTypeError(other.getClass)
           }
         }
       case (key, JNull) =>
         builder.putNull(key)
       case (key, other) =>
-        throw new RuntimeException(s"Do not support type ${other.getClass}.")
+        throw QueryExecutionErrors.unsupportedJavaTypeError(other.getClass)
     }
     builder.build()
   }
@@ -195,15 +196,17 @@ object Metadata {
       case x: Metadata =>
         toJsonValue(x.map)
       case other =>
-        throw new RuntimeException(s"Do not support type ${other.getClass}.")
+        throw QueryExecutionErrors.unsupportedJavaTypeError(other.getClass)
     }
   }
 
   /** Computes the hash code for the types we support. */
   private def hash(obj: Any): Int = {
     obj match {
+      // `map.mapValues` return `Map` in Scala 2.12 and return `MapView` in Scala 2.13, call
+      // `toMap` for Scala version compatibility.
       case map: Map[_, _] =>
-        map.mapValues(hash).##
+        map.mapValues(hash).toMap.##
       case arr: Array[_] =>
         // Seq.empty[T] has the same hashCode regardless of T.
         arr.toSeq.map(hash).##
@@ -220,7 +223,7 @@ object Metadata {
       case null =>
         0
       case other =>
-        throw new RuntimeException(s"Do not support type ${other.getClass}.")
+        throw QueryExecutionErrors.unsupportedJavaTypeError(other.getClass)
     }
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructField.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructField.scala
index 93478af425955..f4c7370be5175 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructField.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructField.scala
@@ -23,6 +23,7 @@ import org.json4s.JsonDSL._
 import org.apache.spark.annotation.Stable
 import org.apache.spark.sql.catalyst.util.{escapeSingleQuotedString, quoteIdentifier}
 import org.apache.spark.sql.catalyst.util.StringUtils.StringConcat
+import org.apache.spark.sql.util.SchemaUtils
 
 /**
  * A field inside a StructType.
@@ -49,7 +50,8 @@ case class StructField(
       stringConcat: StringConcat,
       maxDepth: Int): Unit = {
     if (maxDepth > 0) {
-      stringConcat.append(s"$prefix-- $name: ${dataType.typeName} (nullable = $nullable)\n")
+      stringConcat.append(s"$prefix-- ${SchemaUtils.escapeMetaCharacters(name)}: " +
+        s"${dataType.typeName} (nullable = $nullable)\n")
       DataType.buildFormattedString(dataType, s"$prefix    |", stringConcat, maxDepth)
     }
   }
@@ -82,17 +84,22 @@ case class StructField(
     if (metadata.contains("comment")) Option(metadata.getString("comment")) else None
   }
 
+  private def getDDLComment = getComment()
+    .map(escapeSingleQuotedString)
+    .map(" COMMENT '" + _ + "'")
+    .getOrElse("")
+
+  /**
+   * Returns a string containing a schema in SQL format. For example the following value:
+   * `StructField("eventId", IntegerType)` will be converted to `eventId`: INT.
+   */
+  private[sql] def sql = s"${quoteIdentifier(name)}: ${dataType.sql}$getDDLComment"
+
   /**
    * Returns a string containing a schema in DDL format. For example, the following value:
    * `StructField("eventId", IntegerType)` will be converted to `eventId` INT.
    *
    * @since 2.4.0
    */
-  def toDDL: String = {
-    val comment = getComment()
-      .map(escapeSingleQuotedString)
-      .map(" COMMENT '" + _ + "'")
-
-    s"${quoteIdentifier(name)} ${dataType.sql}${comment.getOrElse("")}"
-  }
+  def toDDL: String = s"${quoteIdentifier(name)} ${dataType.sql}$getDDLComment"
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
index bd2c1d5c26299..8ff0536c2f3a0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
@@ -23,14 +23,13 @@ import scala.util.control.NonFatal
 
 import org.json4s.JsonDSL._
 
-import org.apache.spark.SparkException
 import org.apache.spark.annotation.Stable
-import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.analysis.Resolver
 import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, InterpretedOrdering}
 import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, LegacyTypeStringParser}
-import org.apache.spark.sql.catalyst.util.{quoteIdentifier, truncatedString, StringUtils}
+import org.apache.spark.sql.catalyst.util.{truncatedString, StringUtils}
 import org.apache.spark.sql.catalyst.util.StringUtils.StringConcat
+import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
 import org.apache.spark.sql.internal.SQLConf
 
 /**
@@ -333,9 +332,8 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
         if (found.length > 1) {
           val names = found.map(f => prettyFieldName(normalizedPath :+ f.name))
             .mkString("[", ", ", " ]")
-          throw new AnalysisException(
-            s"Ambiguous field name: ${prettyFieldName(normalizedPath :+ searchName)}. Found " +
-              s"multiple columns that can match: $names")
+          throw QueryCompilationErrors.ambiguousFieldNameError(
+            prettyFieldName(normalizedPath :+ searchName), names)
         } else if (found.isEmpty) {
           None
         } else {
@@ -423,7 +421,7 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
   override def defaultSize: Int = fields.map(_.dataType.defaultSize).sum
 
   override def simpleString: String = {
-    val fieldTypes = fields.view.map(field => s"${field.name}:${field.dataType.simpleString}")
+    val fieldTypes = fields.view.map(field => s"${field.name}:${field.dataType.simpleString}").toSeq
     truncatedString(
       fieldTypes,
       "struct<", ",", ">",
@@ -445,10 +443,7 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
     stringConcat.toString
   }
 
-  override def sql: String = {
-    val fieldTypes = fields.map(f => s"${quoteIdentifier(f.name)}: ${f.dataType.sql}")
-    s"STRUCT<${fieldTypes.mkString(", ")}>"
-  }
+  override def sql: String = s"STRUCT<${fields.map(_.sql).mkString(", ")}>"
 
   /**
    * Returns a string containing a schema in DDL format. For example, the following value:
@@ -526,7 +521,7 @@ object StructType extends AbstractDataType {
   private[sql] def fromString(raw: String): StructType = {
     Try(DataType.fromJson(raw)).getOrElse(LegacyTypeStringParser.parseString(raw)) match {
       case t: StructType => t
-      case _ => throw new RuntimeException(s"Failed parsing ${StructType.simpleString}: $raw")
+      case _ => throw QueryExecutionErrors.failedParsingStructTypeError(raw)
     }
   }
 
@@ -542,7 +537,7 @@ object StructType extends AbstractDataType {
 
   def apply(fields: java.util.List[StructField]): StructType = {
     import scala.collection.JavaConverters._
-    StructType(fields.asScala)
+    StructType(fields.asScala.toSeq)
   }
 
   private[sql] def fromAttributes(attributes: Seq[Attribute]): StructType =
@@ -589,8 +584,7 @@ object StructType extends AbstractDataType {
                     nullable = leftNullable || rightNullable)
                 } catch {
                   case NonFatal(e) =>
-                    throw new SparkException(s"Failed to merge fields '$leftName' and " +
-                      s"'$rightName'. " + e.getMessage)
+                    throw QueryExecutionErrors.failedMergingFieldsError(leftName, rightName, e)
                 }
               }
               .orElse {
@@ -606,21 +600,21 @@ object StructType extends AbstractDataType {
             newFields += f
           }
 
-        StructType(newFields)
+        StructType(newFields.toSeq)
 
       case (DecimalType.Fixed(leftPrecision, leftScale),
         DecimalType.Fixed(rightPrecision, rightScale)) =>
         if ((leftPrecision == rightPrecision) && (leftScale == rightScale)) {
           DecimalType(leftPrecision, leftScale)
         } else if ((leftPrecision != rightPrecision) && (leftScale != rightScale)) {
-          throw new SparkException("Failed to merge decimal types with incompatible " +
-            s"precision $leftPrecision and $rightPrecision & scale $leftScale and $rightScale")
+          throw QueryExecutionErrors.cannotMergeDecimalTypesWithIncompatiblePrecisionAndScaleError(
+            leftPrecision, rightPrecision, leftScale, rightScale)
         } else if (leftPrecision != rightPrecision) {
-          throw new SparkException("Failed to merge decimal types with incompatible " +
-            s"precision $leftPrecision and $rightPrecision")
+          throw QueryExecutionErrors.cannotMergeDecimalTypesWithIncompatiblePrecisionError(
+            leftPrecision, rightPrecision)
         } else {
-          throw new SparkException("Failed to merge decimal types with incompatible " +
-            s"scala $leftScale and $rightScale")
+          throw QueryExecutionErrors.cannotMergeDecimalTypesWithIncompatibleScaleError(
+            leftScale, rightScale)
         }
 
       case (leftUdt: UserDefinedType[_], rightUdt: UserDefinedType[_])
@@ -630,8 +624,7 @@ object StructType extends AbstractDataType {
         leftType
 
       case _ =>
-        throw new SparkException(s"Failed to merge incompatible data types ${left.catalogString}" +
-          s" and ${right.catalogString}")
+        throw QueryExecutionErrors.cannotMergeIncompatibleDataTypesError(left, right)
     }
 
   private[sql] def fieldsMap(fields: Array[StructField]): Map[String, StructField] = {
@@ -641,4 +634,39 @@ object StructType extends AbstractDataType {
     fields.foreach(s => map.put(s.name, s))
     map
   }
+
+  /**
+   * Returns a `StructType` that contains missing fields recursively from `source` to `target`.
+   * Note that this doesn't support looking into array type and map type recursively.
+   */
+  def findMissingFields(
+      source: StructType,
+      target: StructType,
+      resolver: Resolver): Option[StructType] = {
+    def bothStructType(dt1: DataType, dt2: DataType): Boolean =
+      dt1.isInstanceOf[StructType] && dt2.isInstanceOf[StructType]
+
+    val newFields = mutable.ArrayBuffer.empty[StructField]
+
+    target.fields.foreach { field =>
+      val found = source.fields.find(f => resolver(field.name, f.name))
+      if (found.isEmpty) {
+        // Found a missing field in `source`.
+        newFields += field
+      } else if (bothStructType(found.get.dataType, field.dataType) &&
+          !found.get.dataType.sameType(field.dataType)) {
+        // Found a field with same name, but different data type.
+        findMissingFields(found.get.dataType.asInstanceOf[StructType],
+          field.dataType.asInstanceOf[StructType], resolver).map { missingType =>
+          newFields += found.get.copy(dataType = missingType)
+        }
+      }
+    }
+
+    if (newFields.isEmpty) {
+      None
+    } else {
+      Some(StructType(newFields.toSeq))
+    }
+  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/UDTRegistration.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/UDTRegistration.scala
index 20ec75c70615b..f13651f090253 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/UDTRegistration.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/UDTRegistration.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.types
 import scala.collection.mutable
 
 import org.apache.spark.SparkException
+import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.internal.Logging
 import org.apache.spark.util.Utils
 
@@ -29,7 +30,7 @@ import org.apache.spark.util.Utils
  * However, by doing this, we add SparkSQL dependency on user classes. This object provides
  * alternative approach to register UDTs for user classes.
  */
-private[spark]
+@DeveloperApi
 object UDTRegistration extends Serializable with Logging {
 
   /** The mapping between the Class between UserDefinedType and user classes. */
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/UserDefinedType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/UserDefinedType.scala
index 6af16e2dba105..7a26809415c7b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/UserDefinedType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/UserDefinedType.scala
@@ -22,6 +22,8 @@ import java.util.Objects
 import org.json4s.JsonAST.JValue
 import org.json4s.JsonDSL._
 
+import org.apache.spark.annotation.DeveloperApi
+
 /**
  * The data type for User Defined Types (UDTs).
  *
@@ -34,11 +36,8 @@ import org.json4s.JsonDSL._
  *
  * The conversion via `serialize` occurs when instantiating a `DataFrame` from another RDD.
  * The conversion via `deserialize` occurs when reading from a `DataFrame`.
- *
- * Note: This was previously a developer API in Spark 1.x. We are making this private in Spark 2.0
- * because we will very likely create a new version of this that works better with Datasets.
  */
-private[spark]
+@DeveloperApi
 abstract class UserDefinedType[UserType >: Null] extends DataType with Serializable {
 
   /** Underlying storage type for this UDT */
@@ -78,8 +77,8 @@ abstract class UserDefinedType[UserType >: Null] extends DataType with Serializa
    */
   override private[spark] def asNullable: UserDefinedType[UserType] = this
 
-  override private[sql] def acceptsType(dataType: DataType) = dataType match {
-    case other: UserDefinedType[_] =>
+  override private[sql] def acceptsType(dataType: DataType): Boolean = dataType match {
+    case other: UserDefinedType[_] if this.userClass != null && other.userClass != null =>
       this.getClass == other.getClass ||
         this.userClass.isAssignableFrom(other.userClass)
     case _ => false
@@ -90,7 +89,7 @@ abstract class UserDefinedType[UserType >: Null] extends DataType with Serializa
   override def hashCode(): Int = getClass.hashCode()
 
   override def equals(other: Any): Boolean = other match {
-    case that: UserDefinedType[_] => this.acceptsType(that)
+    case that: UserDefinedType[_] => this.getClass == that.getClass
     case _ => false
   }
 
@@ -131,6 +130,11 @@ private[sql] class PythonUserDefinedType(
       ("sqlType" -> sqlType.jsonValue)
   }
 
+  override private[sql] def acceptsType(dataType: DataType): Boolean = dataType match {
+    case other: PythonUserDefinedType => pyUDT == other.pyUDT
+    case _ => false
+  }
+
   override def equals(other: Any): Boolean = other match {
     case that: PythonUserDefinedType => pyUDT == that.pyUDT
     case _ => false
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/VarcharType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/VarcharType.scala
new file mode 100644
index 0000000000000..2e30820ef0a05
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/VarcharType.scala
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.types
+
+import scala.math.Ordering
+import scala.reflect.runtime.universe.typeTag
+
+import org.apache.spark.annotation.Experimental
+import org.apache.spark.unsafe.types.UTF8String
+
+@Experimental
+case class VarcharType(length: Int) extends AtomicType {
+  require(length >= 0, "The length of varchar type cannot be negative.")
+
+  private[sql] type InternalType = UTF8String
+  @transient private[sql] lazy val tag = typeTag[InternalType]
+  private[sql] val ordering = implicitly[Ordering[InternalType]]
+
+  override def defaultSize: Int = length
+  override def typeName: String = s"varchar($length)"
+  override def toString: String = s"VarcharType($length)"
+  private[spark] override def asNullable: VarcharType = this
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/YearMonthIntervalType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/YearMonthIntervalType.scala
new file mode 100644
index 0000000000000..8ee4bef0be88b
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/YearMonthIntervalType.scala
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.types
+
+import scala.math.Ordering
+import scala.reflect.runtime.universe.typeTag
+
+import org.apache.spark.annotation.Unstable
+
+/**
+ * The type represents year-month intervals of the SQL standard. A year-month interval is made up
+ * of a contiguous subset of the following fields:
+ *   - MONTH, months within years [0..11],
+ *   - YEAR, years in the range [0..178956970].
+ *
+ * `YearMonthIntervalType` represents positive as well as negative year-month intervals.
+ *
+ * Please use the singleton `DataTypes.YearMonthIntervalType` to refer the type.
+ *
+ * @since 3.2.0
+ */
+@Unstable
+class YearMonthIntervalType private() extends AtomicType {
+  /**
+   * Internally, values of year-month intervals are stored in `Int` values as amount of months
+   * that are calculated by the formula:
+   *   -/+ (12 * YEAR + MONTH)
+   */
+  private[sql] type InternalType = Int
+
+  @transient private[sql] lazy val tag = typeTag[InternalType]
+
+  private[sql] val ordering = implicitly[Ordering[InternalType]]
+
+  /**
+   * Year-month interval values always occupy 4 bytes.
+   * The YEAR field is constrained by the upper bound 178956970 to fit to `Int`.
+   */
+  override def defaultSize: Int = 4
+
+  private[spark] override def asNullable: YearMonthIntervalType = this
+
+  override def typeName: String = "interval year to month"
+}
+
+/**
+ * The companion case object and its class is separated so the companion object also subclasses
+ * the YearMonthIntervalType class. Otherwise, the companion object would be of type
+ * "YearMonthIntervalType$" in byte code. Defined with a private constructor so the companion object
+ * is the only possible instantiation.
+ *
+ * @since 3.2.0
+ */
+@Unstable
+case object YearMonthIntervalType extends YearMonthIntervalType
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala
index 3956629cf6a57..a3e76797b6bf2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala
@@ -20,12 +20,14 @@ package org.apache.spark.sql.types
 import scala.math.Numeric._
 import scala.math.Ordering
 
+import org.apache.spark.sql.catalyst.util.SQLOrderingUtil
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.types.Decimal.DecimalIsConflicted
 
 private[sql] object ByteExactNumeric extends ByteIsIntegral with Ordering.ByteOrdering {
   private def checkOverflow(res: Int, x: Byte, y: Byte, op: String): Unit = {
     if (res > Byte.MaxValue || res < Byte.MinValue) {
-      throw new ArithmeticException(s"$x $op $y caused overflow.")
+      throw QueryExecutionErrors.binaryArithmeticCauseOverflowError(x, op, y)
     }
   }
 
@@ -49,7 +51,7 @@ private[sql] object ByteExactNumeric extends ByteIsIntegral with Ordering.ByteOr
 
   override def negate(x: Byte): Byte = {
     if (x == Byte.MinValue) { // if and only if x is Byte.MinValue, overflow can happen
-      throw new ArithmeticException(s"- $x caused overflow.")
+      throw QueryExecutionErrors.unaryMinusCauseOverflowError(x)
     }
     (-x).toByte
   }
@@ -59,7 +61,7 @@ private[sql] object ByteExactNumeric extends ByteIsIntegral with Ordering.ByteOr
 private[sql] object ShortExactNumeric extends ShortIsIntegral with Ordering.ShortOrdering {
   private def checkOverflow(res: Int, x: Short, y: Short, op: String): Unit = {
     if (res > Short.MaxValue || res < Short.MinValue) {
-      throw new ArithmeticException(s"$x $op $y caused overflow.")
+      throw QueryExecutionErrors.binaryArithmeticCauseOverflowError(x, op, y)
     }
   }
 
@@ -83,7 +85,7 @@ private[sql] object ShortExactNumeric extends ShortIsIntegral with Ordering.Shor
 
   override def negate(x: Short): Short = {
     if (x == Short.MinValue) { // if and only if x is Byte.MinValue, overflow can happen
-      throw new ArithmeticException(s"- $x caused overflow.")
+      throw QueryExecutionErrors.unaryMinusCauseOverflowError(x)
     }
     (-x).toShort
   }
@@ -113,14 +115,11 @@ private[sql] object LongExactNumeric extends LongIsIntegral with Ordering.LongOr
     if (x == x.toInt) {
       x.toInt
     } else {
-      throw new ArithmeticException(s"Casting $x to int causes overflow")
+      throw QueryExecutionErrors.castingCauseOverflowError(x, "int")
     }
 }
 
 private[sql] object FloatExactNumeric extends FloatIsFractional {
-  private def overflowException(x: Float, dataType: String) =
-    throw new ArithmeticException(s"Casting $x to $dataType causes overflow")
-
   private val intUpperBound = Int.MaxValue
   private val intLowerBound = Int.MinValue
   private val longUpperBound = Long.MaxValue
@@ -136,7 +135,7 @@ private[sql] object FloatExactNumeric extends FloatIsFractional {
     if (Math.floor(x) <= intUpperBound && Math.ceil(x) >= intLowerBound) {
       x.toInt
     } else {
-      overflowException(x, "int")
+      throw QueryExecutionErrors.castingCauseOverflowError(x, "int")
     }
   }
 
@@ -144,17 +143,14 @@ private[sql] object FloatExactNumeric extends FloatIsFractional {
     if (Math.floor(x) <= longUpperBound && Math.ceil(x) >= longLowerBound) {
       x.toLong
     } else {
-      overflowException(x, "int")
+      throw QueryExecutionErrors.castingCauseOverflowError(x, "int")
     }
   }
 
-  override def compare(x: Float, y: Float): Int = java.lang.Float.compare(x, y)
+  override def compare(x: Float, y: Float): Int = SQLOrderingUtil.compareFloats(x, y)
 }
 
 private[sql] object DoubleExactNumeric extends DoubleIsFractional {
-  private def overflowException(x: Double, dataType: String) =
-    throw new ArithmeticException(s"Casting $x to $dataType causes overflow")
-
   private val intUpperBound = Int.MaxValue
   private val intLowerBound = Int.MinValue
   private val longUpperBound = Long.MaxValue
@@ -164,7 +160,7 @@ private[sql] object DoubleExactNumeric extends DoubleIsFractional {
     if (Math.floor(x) <= intUpperBound && Math.ceil(x) >= intLowerBound) {
       x.toInt
     } else {
-      overflowException(x, "int")
+      throw QueryExecutionErrors.castingCauseOverflowError(x, "int")
     }
   }
 
@@ -172,11 +168,11 @@ private[sql] object DoubleExactNumeric extends DoubleIsFractional {
     if (Math.floor(x) <= longUpperBound && Math.ceil(x) >= longLowerBound) {
       x.toLong
     } else {
-      overflowException(x, "long")
+      throw QueryExecutionErrors.castingCauseOverflowError(x, "long")
     }
   }
 
-  override def compare(x: Double, y: Double): Int = java.lang.Double.compare(x, y)
+  override def compare(x: Double, y: Double): Int = SQLOrderingUtil.compareDoubles(x, y)
 }
 
 private[sql] object DecimalExactNumeric extends DecimalIsConflicted {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/package.scala
index f29cbc2069e39..346a51ea10c82 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/package.scala
@@ -21,12 +21,4 @@ package org.apache.spark.sql
  * Contains a type system for attributes produced by relations, including complex types like
  * structs, arrays and maps.
  */
-package object types {
-  /**
-   * Metadata key used to store the raw hive type string in the metadata of StructField. This
-   * is relevant for datatypes that do not have a direct Spark SQL counterpart, such as CHAR and
-   * VARCHAR. We need to preserve the original type in order to invoke the correct object
-   * inspector in Hive.
-   */
-  val HIVE_TYPE_STRING = "HIVE_TYPE_STRING"
-}
+package object types
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala
index 003ce850c926e..ce8acd1825531 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala
@@ -21,7 +21,7 @@ import scala.collection.JavaConverters._
 
 import org.apache.arrow.memory.RootAllocator
 import org.apache.arrow.vector.complex.MapVector
-import org.apache.arrow.vector.types.{DateUnit, FloatingPointPrecision, TimeUnit}
+import org.apache.arrow.vector.types.{DateUnit, FloatingPointPrecision, IntervalUnit, TimeUnit}
 import org.apache.arrow.vector.types.pojo.{ArrowType, Field, FieldType, Schema}
 
 import org.apache.spark.sql.internal.SQLConf
@@ -53,6 +53,9 @@ private[sql] object ArrowUtils {
       } else {
         new ArrowType.Timestamp(TimeUnit.MICROSECOND, timeZoneId)
       }
+    case NullType => ArrowType.Null.INSTANCE
+    case YearMonthIntervalType => new ArrowType.Interval(IntervalUnit.YEAR_MONTH)
+    case DayTimeIntervalType => new ArrowType.Interval(IntervalUnit.DAY_TIME)
     case _ =>
       throw new UnsupportedOperationException(s"Unsupported data type: ${dt.catalogString}")
   }
@@ -72,6 +75,9 @@ private[sql] object ArrowUtils {
     case d: ArrowType.Decimal => DecimalType(d.getPrecision, d.getScale)
     case date: ArrowType.Date if date.getUnit == DateUnit.DAY => DateType
     case ts: ArrowType.Timestamp if ts.getUnit == TimeUnit.MICROSECOND => TimestampType
+    case ArrowType.Null.INSTANCE => NullType
+    case yi: ArrowType.Interval if yi.getUnit == IntervalUnit.YEAR_MONTH => YearMonthIntervalType
+    case di: ArrowType.Interval if di.getUnit == IntervalUnit.DAY_TIME => DayTimeIntervalType
     case _ => throw new UnsupportedOperationException(s"Unsupported data type: $dt")
   }
 
@@ -121,7 +127,7 @@ private[sql] object ArrowUtils {
           val dt = fromArrowField(child)
           StructField(child.getName, dt, child.isNullable)
         }
-        StructType(fields)
+        StructType(fields.toSeq)
       case arrowType => fromArrowType(arrowType)
     }
   }
@@ -137,7 +143,7 @@ private[sql] object ArrowUtils {
     StructType(schema.getFields.asScala.map { field =>
       val dt = fromArrowField(field)
       StructField(field.getName, dt, field.isNullable)
-    })
+    }.toSeq)
   }
 
   /** Return Map with conf settings to be used in ArrowPythonRunner */
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/util/PartitioningUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/util/PartitioningUtils.scala
new file mode 100644
index 0000000000000..004725487911a
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/util/PartitioningUtils.scala
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.util
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.analysis.Resolver
+import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
+import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils.DEFAULT_PARTITION_NAME
+import org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils
+import org.apache.spark.sql.catalyst.util.CharVarcharUtils
+import org.apache.spark.sql.types.{CharType, StructType, VarcharType}
+import org.apache.spark.unsafe.types.UTF8String
+
+private[sql] object PartitioningUtils {
+  /**
+   * Normalize the column names in partition specification, w.r.t. the real partition column names
+   * and case sensitivity. e.g., if the partition spec has a column named `monTh`, and there is a
+   * partition column named `month`, and it's case insensitive, we will normalize `monTh` to
+   * `month`.
+   */
+  def normalizePartitionSpec[T](
+      partitionSpec: Map[String, T],
+      partCols: StructType,
+      tblName: String,
+      resolver: Resolver): Map[String, T] = {
+    val rawSchema = CharVarcharUtils.getRawSchema(partCols)
+    val normalizedPartSpec = partitionSpec.toSeq.map { case (key, value) =>
+      val normalizedFiled = rawSchema.find(f => resolver(f.name, key)).getOrElse {
+        throw new AnalysisException(s"$key is not a valid partition column in table $tblName.")
+      }
+
+      val normalizedVal = normalizedFiled.dataType match {
+        case CharType(len) if value != null && value != DEFAULT_PARTITION_NAME =>
+          val v = value match {
+            case Some(str: String) => Some(charTypeWriteSideCheck(str, len))
+            case str: String => charTypeWriteSideCheck(str, len)
+            case other => other
+          }
+          v.asInstanceOf[T]
+        case VarcharType(len) if value != null && value != DEFAULT_PARTITION_NAME =>
+          val v = value match {
+            case Some(str: String) => Some(varcharTypeWriteSideCheck(str, len))
+            case str: String => varcharTypeWriteSideCheck(str, len)
+            case other => other
+          }
+          v.asInstanceOf[T]
+        case _ => value
+      }
+      normalizedFiled.name -> normalizedVal
+    }
+
+    SchemaUtils.checkColumnNameDuplication(
+      normalizedPartSpec.map(_._1), "in the partition schema", resolver)
+
+    normalizedPartSpec.toMap
+  }
+
+  private def charTypeWriteSideCheck(inputStr: String, limit: Int): String = {
+    val toUtf8 = UTF8String.fromString(inputStr)
+    CharVarcharCodegenUtils.charTypeWriteSideCheck(toUtf8, limit).toString
+  }
+
+  private def varcharTypeWriteSideCheck(inputStr: String, limit: Int): String = {
+    val toUtf8 = UTF8String.fromString(inputStr)
+    CharVarcharCodegenUtils.varcharTypeWriteSideCheck(toUtf8, limit).toString
+  }
+
+  /**
+   * Verify if the input partition spec exactly matches the existing defined partition spec
+   * The columns must be the same but the orders could be different.
+   */
+  def requireExactMatchedPartitionSpec(
+      tableName: String,
+      spec: TablePartitionSpec,
+      partitionColumnNames: Seq[String]): Unit = {
+    val defined = partitionColumnNames.sorted
+    if (spec.keys.toSeq.sorted != defined) {
+      throw new AnalysisException(
+        s"Partition spec is invalid. The spec (${spec.keys.mkString(", ")}) must match " +
+        s"the partition spec (${partitionColumnNames.mkString(", ")}) defined in " +
+        s"table '$tableName'")
+    }
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/util/SchemaUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/util/SchemaUtils.scala
index 27b5eec27281d..05f9cc53898cd 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/util/SchemaUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/util/SchemaUtils.scala
@@ -22,6 +22,7 @@ import java.util.Locale
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.analysis._
 import org.apache.spark.sql.connector.expressions.{BucketTransform, FieldReference, NamedTransform, Transform}
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructField, StructType}
 
 
@@ -41,8 +42,38 @@ private[spark] object SchemaUtils {
    * @param caseSensitiveAnalysis whether duplication checks should be case sensitive or not
    */
   def checkSchemaColumnNameDuplication(
-      schema: StructType, colType: String, caseSensitiveAnalysis: Boolean = false): Unit = {
-    checkColumnNameDuplication(schema.map(_.name), colType, caseSensitiveAnalysis)
+      schema: DataType,
+      colType: String,
+      caseSensitiveAnalysis: Boolean = false): Unit = {
+    schema match {
+      case ArrayType(elementType, _) =>
+        checkSchemaColumnNameDuplication(elementType, colType, caseSensitiveAnalysis)
+      case MapType(keyType, valueType, _) =>
+        checkSchemaColumnNameDuplication(keyType, colType, caseSensitiveAnalysis)
+        checkSchemaColumnNameDuplication(valueType, colType, caseSensitiveAnalysis)
+      case structType: StructType =>
+        val fields = structType.fields
+        checkColumnNameDuplication(fields.map(_.name), colType, caseSensitiveAnalysis)
+        fields.foreach { field =>
+          checkSchemaColumnNameDuplication(field.dataType, colType, caseSensitiveAnalysis)
+        }
+      case _ =>
+    }
+  }
+
+  /**
+   * Checks if an input schema has duplicate column names. This throws an exception if the
+   * duplication exists.
+   *
+   * @param schema schema to check
+   * @param colType column type name, used in an exception message
+   * @param resolver resolver used to determine if two identifiers are equal
+   */
+  def checkSchemaColumnNameDuplication(
+      schema: StructType,
+      colType: String,
+      resolver: Resolver): Unit = {
+    checkSchemaColumnNameDuplication(schema, colType, isCaseSensitiveAnalysis(resolver))
   }
 
   // Returns true if a given resolver is case-sensitive
@@ -52,7 +83,8 @@ private[spark] object SchemaUtils {
     } else if (resolver == caseInsensitiveResolution) {
       false
     } else {
-      sys.error("A resolver to check if two identifiers are equal must be " +
+      throw QueryExecutionErrors.unreachableError(
+        ": A resolver to check if two identifiers are equal must be " +
         "`caseSensitiveResolution` or `caseInsensitiveResolution` in o.a.s.sql.catalyst.")
     }
   }
@@ -88,7 +120,7 @@ private[spark] object SchemaUtils {
         case (x, ys) if ys.length > 1 => s"`$x`"
       }
       throw new AnalysisException(
-        s"Found duplicate column(s) $colType: ${duplicateColumns.mkString(", ")}")
+        s"Found duplicate column(s) $colType: ${duplicateColumns.toSeq.sorted.mkString(", ")}")
     }
   }
 
@@ -241,4 +273,18 @@ private[spark] object SchemaUtils {
     }
     field._1
   }
+
+  /**
+   * @param str The string to be escaped.
+   * @return The escaped string.
+   */
+  def escapeMetaCharacters(str: String): String = {
+    str.replaceAll("\n", "\\\\n")
+      .replaceAll("\r", "\\\\r")
+      .replaceAll("\t", "\\\\t")
+      .replaceAll("\f", "\\\\f")
+      .replaceAll("\b", "\\\\b")
+      .replaceAll("\u000B", "\\\\v")
+      .replaceAll("\u0007", "\\\\a")
+  }
 }
diff --git a/sql/catalyst/src/test/java/org/apache/spark/sql/catalyst/expressions/XXH64Suite.java b/sql/catalyst/src/test/java/org/apache/spark/sql/catalyst/expressions/XXH64Suite.java
index 1baee91b3439c..1bd4eeeb35aee 100644
--- a/sql/catalyst/src/test/java/org/apache/spark/sql/catalyst/expressions/XXH64Suite.java
+++ b/sql/catalyst/src/test/java/org/apache/spark/sql/catalyst/expressions/XXH64Suite.java
@@ -17,7 +17,6 @@
 
 package org.apache.spark.sql.catalyst.expressions;
 
-import java.nio.ByteOrder;
 import java.nio.charset.StandardCharsets;
 import java.util.HashSet;
 import java.util.Random;
@@ -73,42 +72,62 @@ public void testKnownByteArrayInputs() {
             hasher.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, 1));
     Assert.assertEquals(0x739840CB819FA723L,
             XXH64.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, 1, PRIME));
+    Assert.assertEquals(0x9256E58AA397AEF1L,
+            hasher.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, 4));
+    Assert.assertEquals(0x9D5FFDFB928AB4BL,
+            XXH64.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, 4, PRIME));
+    Assert.assertEquals(0xF74CB1451B32B8CFL,
+            hasher.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, 8));
+    Assert.assertEquals(0x9C44B77FBCC302C5L,
+            XXH64.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, 8, PRIME));
+    Assert.assertEquals(0xCFFA8DB881BC3A3DL,
+            hasher.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, 14));
+    Assert.assertEquals(0x5B9611585EFCC9CBL,
+            XXH64.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, 14, PRIME));
+    Assert.assertEquals(0x0EAB543384F878ADL,
+            hasher.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, SIZE));
+    Assert.assertEquals(0xCAA65939306F1E21L,
+            XXH64.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, SIZE, PRIME));
+  }
 
-    if (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN) {
-      Assert.assertEquals(0x9256E58AA397AEF1L,
-              hasher.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, 4));
-      Assert.assertEquals(0x9D5FFDFB928AB4BL,
-              XXH64.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, 4, PRIME));
-      Assert.assertEquals(0xF74CB1451B32B8CFL,
-              hasher.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, 8));
-      Assert.assertEquals(0x9C44B77FBCC302C5L,
-              XXH64.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, 8, PRIME));
-      Assert.assertEquals(0xCFFA8DB881BC3A3DL,
-              hasher.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, 14));
-      Assert.assertEquals(0x5B9611585EFCC9CBL,
-              XXH64.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, 14, PRIME));
-      Assert.assertEquals(0x0EAB543384F878ADL,
-              hasher.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, SIZE));
-      Assert.assertEquals(0xCAA65939306F1E21L,
-              XXH64.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, SIZE, PRIME));
-    } else {
-      Assert.assertEquals(0x7F875412350ADDDCL,
-              hasher.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, 4));
-      Assert.assertEquals(0x564D279F524D8516L,
-              XXH64.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, 4, PRIME));
-      Assert.assertEquals(0x7D9F07E27E0EB006L,
-              hasher.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, 8));
-      Assert.assertEquals(0x893CEF564CB7858L,
-              XXH64.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, 8, PRIME));
-      Assert.assertEquals(0xC6198C4C9CC49E17L,
-              hasher.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, 14));
-      Assert.assertEquals(0x4E21BEF7164D4BBL,
-              XXH64.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, 14, PRIME));
-      Assert.assertEquals(0xBCF5FAEDEE1F2B5AL,
-              hasher.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, SIZE));
-      Assert.assertEquals(0x6F680C877A358FE5L,
-              XXH64.hashUnsafeBytes(BUFFER, Platform.BYTE_ARRAY_OFFSET, SIZE, PRIME));
-    }
+  @Test
+  public void testKnownWordArrayInputs() {
+    Assert.assertEquals(0XEF46DB3751D8E999L,
+            hasher.hashUnsafeWords(BUFFER, Platform.BYTE_ARRAY_OFFSET, 0));
+    Assert.assertEquals(0XAC75FDA2929B17EFL,
+            XXH64.hashUnsafeWords(BUFFER, Platform.BYTE_ARRAY_OFFSET, 0, PRIME));
+    Assert.assertEquals(0XF74CB1451B32B8CFL,
+            hasher.hashUnsafeWords(BUFFER, Platform.BYTE_ARRAY_OFFSET, 8));
+    Assert.assertEquals(0X9C44B77FBCC302C5L,
+            XXH64.hashUnsafeWords(BUFFER, Platform.BYTE_ARRAY_OFFSET, 8, PRIME));
+    Assert.assertEquals(0X169173A697113B29L,
+            hasher.hashUnsafeWords(BUFFER, Platform.BYTE_ARRAY_OFFSET, 16));
+    Assert.assertEquals(0XA0B652822C1538F6L,
+            XXH64.hashUnsafeWords(BUFFER, Platform.BYTE_ARRAY_OFFSET, 16, PRIME));
+    Assert.assertEquals(0XCEF5D1497F99F246L,
+            hasher.hashUnsafeWords(BUFFER, Platform.BYTE_ARRAY_OFFSET, 24));
+    Assert.assertEquals(0X1FA29EA08AA60D77L,
+            XXH64.hashUnsafeWords(BUFFER, Platform.BYTE_ARRAY_OFFSET, 24, PRIME));
+    Assert.assertEquals(0XAF5753D39159EDEEL,
+            hasher.hashUnsafeWords(BUFFER, Platform.BYTE_ARRAY_OFFSET, 32));
+    Assert.assertEquals(0XDCAB9233B8CA7B0FL,
+            XXH64.hashUnsafeWords(BUFFER, Platform.BYTE_ARRAY_OFFSET, 32, PRIME));
+    Assert.assertEquals(0XBAB04D3F1769013L,
+            hasher.hashUnsafeWords(BUFFER, Platform.BYTE_ARRAY_OFFSET, 40));
+    Assert.assertEquals(0X21273A6B8D344CF0L,
+            XXH64.hashUnsafeWords(BUFFER, Platform.BYTE_ARRAY_OFFSET, 40, PRIME));
+    Assert.assertEquals(0XB3571A0E02A3F4E1L,
+            hasher.hashUnsafeWords(BUFFER, Platform.BYTE_ARRAY_OFFSET, 48));
+    Assert.assertEquals(0X867479AC0EF16154L,
+            XXH64.hashUnsafeWords(BUFFER, Platform.BYTE_ARRAY_OFFSET, 48, PRIME));
+    Assert.assertEquals(0XA3D5C82BD2EE104AL,
+            hasher.hashUnsafeWords(BUFFER, Platform.BYTE_ARRAY_OFFSET, 56));
+    Assert.assertEquals(0X55EF042CF82C04D7L,
+            XXH64.hashUnsafeWords(BUFFER, Platform.BYTE_ARRAY_OFFSET, 56, PRIME));
+    Assert.assertEquals(0X18F5388F1D2BA08CL,
+            hasher.hashUnsafeWords(BUFFER, Platform.BYTE_ARRAY_OFFSET, 64));
+    Assert.assertEquals(0X479E7103CF9AA020L,
+            XXH64.hashUnsafeWords(BUFFER, Platform.BYTE_ARRAY_OFFSET, 64, PRIME));
   }
 
   @Test
diff --git a/sql/catalyst/src/test/resources/ansi-sql-2016-reserved-keywords.txt b/sql/catalyst/src/test/resources/ansi-sql-2016-reserved-keywords.txt
new file mode 100644
index 0000000000000..921491a4a4761
--- /dev/null
+++ b/sql/catalyst/src/test/resources/ansi-sql-2016-reserved-keywords.txt
@@ -0,0 +1,401 @@
+-- This file comes from: https://github.com/postgres/postgres/tree/master/doc/src/sgml/keywords
+ABS
+ACOS
+ALL
+ALLOCATE
+ALTER
+AND
+ANY
+ARE
+ARRAY
+ARRAY_AGG
+ARRAY_MAX_CARDINALITY
+AS
+ASENSITIVE
+ASIN
+ASYMMETRIC
+AT
+ATAN
+ATOMIC
+AUTHORIZATION
+AVG
+BEGIN
+BEGIN_FRAME
+BEGIN_PARTITION
+BETWEEN
+BIGINT
+BINARY
+BLOB
+BOOLEAN
+BOTH
+BY
+CALL
+CALLED
+CARDINALITY
+CASCADED
+CASE
+CAST
+CEIL
+CEILING
+CHAR
+CHAR_LENGTH
+CHARACTER
+CHARACTER_LENGTH
+CHECK
+CLASSIFIER
+CLOB
+CLOSE
+COALESCE
+COLLATE
+COLLECT
+COLUMN
+COMMIT
+CONDITION
+CONNECT
+CONSTRAINT
+CONTAINS
+CONVERT
+COPY
+CORR
+CORRESPONDING
+COS
+COSH
+COUNT
+COVAR_POP
+COVAR_SAMP
+CREATE
+CROSS
+CUBE
+CUME_DIST
+CURRENT
+CURRENT_CATALOG
+CURRENT_DATE
+CURRENT_DEFAULT_TRANSFORM_GROUP
+CURRENT_PATH
+CURRENT_ROLE
+CURRENT_ROW
+CURRENT_SCHEMA
+CURRENT_TIME
+CURRENT_TIMESTAMP
+CURRENT_TRANSFORM_GROUP_FOR_TYPE
+CURRENT_USER
+CURSOR
+CYCLE
+DATE
+DAY
+DEALLOCATE
+DEC
+DECIMAL
+DECFLOAT
+DECLARE
+DEFAULT
+DEFINE
+DELETE
+DENSE_RANK
+DEREF
+DESCRIBE
+DETERMINISTIC
+DISCONNECT
+DISTINCT
+DOUBLE
+DROP
+DYNAMIC
+EACH
+ELEMENT
+ELSE
+EMPTY
+END
+END_FRAME
+END_PARTITION
+END-EXEC
+EQUALS
+ESCAPE
+EVERY
+EXCEPT
+EXEC
+EXECUTE
+EXISTS
+EXP
+EXTERNAL
+EXTRACT
+FALSE
+FETCH
+FILTER
+FIRST_VALUE
+FLOAT
+FLOOR
+FOR
+FOREIGN
+FRAME_ROW
+FREE
+FROM
+FULL
+FUNCTION
+FUSION
+GET
+GLOBAL
+GRANT
+GROUP
+GROUPING
+GROUPS
+HAVING
+HOLD
+HOUR
+IDENTITY
+IN
+INDICATOR
+INITIAL
+INNER
+INOUT
+INSENSITIVE
+INSERT
+INT
+INTEGER
+INTERSECT
+INTERSECTION
+INTERVAL
+INTO
+IS
+JOIN
+JSON_ARRAY
+JSON_ARRAYAGG
+JSON_EXISTS
+JSON_OBJECT
+JSON_OBJECTAGG
+JSON_QUERY
+JSON_TABLE
+JSON_TABLE_PRIMITIVE
+JSON_VALUE
+LAG
+LANGUAGE
+LARGE
+LAST_VALUE
+LATERAL
+LEAD
+LEADING
+LEFT
+LIKE
+LIKE_REGEX
+LISTAGG
+LN
+LOCAL
+LOCALTIME
+LOCALTIMESTAMP
+LOG
+LOG10
+LOWER
+MATCH
+MATCH_NUMBER
+MATCH_RECOGNIZE
+MATCHES
+MAX
+MEASURES
+MEMBER
+MERGE
+METHOD
+MIN
+MINUTE
+MOD
+MODIFIES
+MODULE
+MONTH
+MULTISET
+NATIONAL
+NATURAL
+NCHAR
+NCLOB
+NEW
+NO
+NONE
+NORMALIZE
+NOT
+NTH_VALUE
+NTILE
+NULL
+NULLIF
+NUMERIC
+OCTET_LENGTH
+OCCURRENCES_REGEX
+OF
+OFFSET
+OLD
+OMIT
+ON
+ONE
+ONLY
+OPEN
+OR
+ORDER
+OUT
+OUTER
+OVER
+OVERLAPS
+OVERLAY
+PARAMETER
+PARTITION
+PATTERN
+PER
+PERCENT
+PERCENT_RANK
+PERCENTILE_CONT
+PERCENTILE_DISC
+PERIOD
+PERMUTE
+PORTION
+POSITION
+POSITION_REGEX
+POWER
+PRECEDES
+PRECISION
+PREPARE
+PRIMARY
+PROCEDURE
+PTF
+RANGE
+RANK
+READS
+REAL
+RECURSIVE
+REF
+REFERENCES
+REFERENCING
+REGR_AVGX
+REGR_AVGY
+REGR_COUNT
+REGR_INTERCEPT
+REGR_R2
+REGR_SLOPE
+REGR_SXX
+REGR_SXY
+REGR_SYY
+RELEASE
+RESULT
+RETURN
+RETURNS
+REVOKE
+RIGHT
+ROLLBACK
+ROLLUP
+ROW
+ROW_NUMBER
+ROWS
+RUNNING
+SAVEPOINT
+SCOPE
+SCROLL
+SEARCH
+SECOND
+SEEK
+SELECT
+SENSITIVE
+SESSION_USER
+SET
+SHOW
+SIMILAR
+SIN
+SINH
+SKIP
+SMALLINT
+SOME
+SPECIFIC
+SPECIFICTYPE
+SQL
+SQLEXCEPTION
+SQLSTATE
+SQLWARNING
+SQRT
+START
+STATIC
+STDDEV_POP
+STDDEV_SAMP
+SUBMULTISET
+SUBSET
+SUBSTRING
+SUBSTRING_REGEX
+SUCCEEDS
+SUM
+SYMMETRIC
+SYSTEM
+SYSTEM_TIME
+SYSTEM_USER
+TABLE
+TABLESAMPLE
+TAN
+TANH
+THEN
+TIME
+TIMESTAMP
+TIMEZONE_HOUR
+TIMEZONE_MINUTE
+TO
+TRAILING
+TRANSLATE
+TRANSLATE_REGEX
+TRANSLATION
+TREAT
+TRIGGER
+TRIM
+TRIM_ARRAY
+TRUE
+TRUNCATE
+UESCAPE
+UNION
+UNIQUE
+UNKNOWN
+UNMATCHED
+UNNEST
+UPDATE
+UPPER
+USER
+USING
+VALUE
+VALUES
+VALUE_OF
+VAR_POP
+VAR_SAMP
+VARBINARY
+VARCHAR
+VARYING
+VERSIONING
+WHEN
+WHENEVER
+WHERE
+WIDTH_BUCKET
+WINDOW
+WITH
+WITHIN
+WITHOUT
+YEAR
+DATALINK
+DLNEWCOPY
+DLPREVIOUSCOPY
+DLURLCOMPLETE
+DLURLCOMPLETEWRITE
+DLURLCOMPLETEONLY
+DLURLPATH
+DLURLPATHWRITE
+DLURLPATHONLY
+DLURLSCHEME
+DLURLSERVER
+DLVALUE
+IMPORT
+XML
+XMLAGG
+XMLATTRIBUTES
+XMLBINARY
+XMLCAST
+XMLCOMMENT
+XMLCONCAT
+XMLDOCUMENT
+XMLELEMENT
+XMLEXISTS
+XMLFOREST
+XMLITERATE
+XMLNAMESPACES
+XMLPARSE
+XMLPI
+XMLQUERY
+XMLSERIALIZE
+XMLTABLE
+XMLTEXT
+XMLVALIDATE
diff --git a/sql/catalyst/src/test/scala-2.12/org/apache/spark/sql/catalyst/analysis/ExtractGeneratorSuite.scala b/sql/catalyst/src/test/scala-2.12/org/apache/spark/sql/catalyst/analysis/ExtractGeneratorSuite.scala
new file mode 100644
index 0000000000000..7df96a8c64e8d
--- /dev/null
+++ b/sql/catalyst/src/test/scala-2.12/org/apache/spark/sql/catalyst/analysis/ExtractGeneratorSuite.scala
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.types._
+
+/**
+ * Note: this test supports Scala 2.12. A parallel source tree has a 2.13 implementation.
+ */
+class ExtractGeneratorSuite extends AnalysisTest {
+
+  test("SPARK-34141: ExtractGenerator with lazy project list") {
+    val b = AttributeReference("b", ArrayType(StringType))()
+
+    val columns = AttributeReference("a", StringType)() :: b :: Nil
+    val explode = Alias(Explode(b), "c")()
+
+    // view is a lazy seq
+    val rel = LocalRelation(output = columns.view)
+    val plan = Project(rel.output ++ (explode :: Nil), rel)
+
+    assertAnalysisSuccess(plan)
+  }
+}
diff --git a/sql/catalyst/src/test/scala-2.13/org/apache/spark/sql/catalyst/analysis/ExtractGeneratorSuite.scala b/sql/catalyst/src/test/scala-2.13/org/apache/spark/sql/catalyst/analysis/ExtractGeneratorSuite.scala
new file mode 100644
index 0000000000000..cac66f81101fd
--- /dev/null
+++ b/sql/catalyst/src/test/scala-2.13/org/apache/spark/sql/catalyst/analysis/ExtractGeneratorSuite.scala
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import scala.collection.immutable.LazyList
+
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.types._
+
+/**
+ * Note: this test supports Scala 2.13. A parallel source tree has a 2.12 implementation.
+ */
+class ExtractGeneratorSuite extends AnalysisTest {
+
+  test("SPARK-34141: ExtractGenerator with lazy project list") {
+    val b = AttributeReference("b", ArrayType(StringType))()
+
+    val columns = AttributeReference("a", StringType)() #:: b #:: LazyList.empty
+    val explode = Alias(Explode(b), "c")()
+
+    val rel = LocalRelation(output = columns)
+    val plan = Project(rel.output ++ (explode :: Nil), rel)
+
+    assertAnalysisSuccess(plan)
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala
index 6a5bdc4f6fc3d..6d2ea464ec7bb 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala
@@ -19,7 +19,8 @@ package org.apache.spark.sql
 
 import java.math.MathContext
 import java.sql.{Date, Timestamp}
-import java.time.{Instant, LocalDate, LocalDateTime, ZoneId}
+import java.time.{Duration, Instant, LocalDate, LocalDateTime, Period, ZoneId}
+import java.time.temporal.ChronoUnit
 
 import scala.collection.mutable
 import scala.util.{Random, Try}
@@ -136,7 +137,7 @@ object RandomDataGenerator {
       }
       i += 1
     }
-    StructType(fields)
+    StructType(fields.toSeq)
   }
 
   /**
@@ -149,12 +150,15 @@ object RandomDataGenerator {
    * @param dataType the type to generate values for
    * @param nullable whether null values should be generated
    * @param rand an optional random number generator
+   * @param validJulianDatetime whether to generate dates and timestamps that are valid
+   *                            in the Julian calendar.
    * @return a function which can be called to generate random values.
    */
   def forType(
       dataType: DataType,
       nullable: Boolean = true,
-      rand: Random = new Random): Option[() => Any] = {
+      rand: Random = new Random,
+      validJulianDatetime: Boolean = false): Option[() => Any] = {
     val valueGenerator: Option[() => Any] = dataType match {
       case StringType => Some(() => rand.nextString(rand.nextInt(MAX_STR_LEN)))
       case BinaryType => Some(() => {
@@ -182,29 +186,37 @@ object RandomDataGenerator {
           "1970-01-01", // the epoch date
           "9999-12-31" // the last supported date according to SQL standard
         )
+        def getRandomDate(rand: Random): java.sql.Date = {
+          val date = DateTimeUtils.toJavaDate(uniformDaysRand(rand))
+          // The generated `date` is based on the hybrid calendar Julian + Gregorian since
+          // 1582-10-15 but it should be valid in Proleptic Gregorian calendar too which is used
+          // by Spark SQL since version 3.0 (see SPARK-26651). We try to convert `date` to
+          // a local date in Proleptic Gregorian calendar to satisfy this requirement. Some
+          // years are leap years in Julian calendar but not in Proleptic Gregorian calendar.
+          // As the consequence of that, 29 February of such years might not exist in Proleptic
+          // Gregorian calendar. When this happens, we shift the date by one day.
+          Try { date.toLocalDate; date }.getOrElse(new Date(date.getTime + MILLIS_PER_DAY))
+        }
         if (SQLConf.get.getConf(SQLConf.DATETIME_JAVA8API_ENABLED)) {
           randomNumeric[LocalDate](
             rand,
-            (rand: Random) => LocalDate.ofEpochDay(uniformDaysRand(rand)),
+            (rand: Random) => {
+              val days = if (validJulianDatetime) {
+                DateTimeUtils.fromJavaDate(getRandomDate(rand))
+              } else {
+                uniformDaysRand(rand)
+              }
+              LocalDate.ofEpochDay(days)
+            },
             specialDates.map(LocalDate.parse))
         } else {
           randomNumeric[java.sql.Date](
             rand,
-            (rand: Random) => {
-              val date = DateTimeUtils.toJavaDate(uniformDaysRand(rand))
-              // The generated `date` is based on the hybrid calendar Julian + Gregorian since
-              // 1582-10-15 but it should be valid in Proleptic Gregorian calendar too which is used
-              // by Spark SQL since version 3.0 (see SPARK-26651). We try to convert `date` to
-              // a local date in Proleptic Gregorian calendar to satisfy this requirement. Some
-              // years are leap years in Julian calendar but not in Proleptic Gregorian calendar.
-              // As the consequence of that, 29 February of such years might not exist in Proleptic
-              // Gregorian calendar. When this happens, we shift the date by one day.
-              Try { date.toLocalDate; date }.getOrElse(new Date(date.getTime + MILLIS_PER_DAY))
-            },
+            getRandomDate,
             specialDates.map(java.sql.Date.valueOf))
         }
       case TimestampType =>
-        def uniformMicorsRand(rand: Random): Long = {
+        def uniformMicrosRand(rand: Random): Long = {
           var milliseconds = rand.nextLong() % 253402329599999L
           // -62135740800000L is the number of milliseconds before January 1, 1970, 00:00:00 GMT
           // for "0001-01-01 00:00:00.000000". We need to find a
@@ -222,10 +234,29 @@ object RandomDataGenerator {
           "1970-01-01 00:00:00", // the epoch timestamp
           "9999-12-31 23:59:59"  // the last supported timestamp according to SQL standard
         )
+        def getRandomTimestamp(rand: Random): java.sql.Timestamp = {
+          // DateTimeUtils.toJavaTimestamp takes microsecond.
+          val ts = DateTimeUtils.toJavaTimestamp(uniformMicrosRand(rand))
+          // The generated `ts` is based on the hybrid calendar Julian + Gregorian since
+          // 1582-10-15 but it should be valid in Proleptic Gregorian calendar too which is used
+          // by Spark SQL since version 3.0 (see SPARK-26651). We try to convert `ts` to
+          // a local timestamp in Proleptic Gregorian calendar to satisfy this requirement. Some
+          // years are leap years in Julian calendar but not in Proleptic Gregorian calendar.
+          // As the consequence of that, 29 February of such years might not exist in Proleptic
+          // Gregorian calendar. When this happens, we shift the timestamp `ts` by one day.
+          Try { ts.toLocalDateTime; ts }.getOrElse(new Timestamp(ts.getTime + MILLIS_PER_DAY))
+        }
         if (SQLConf.get.getConf(SQLConf.DATETIME_JAVA8API_ENABLED)) {
           randomNumeric[Instant](
             rand,
-            (rand: Random) => DateTimeUtils.microsToInstant(uniformMicorsRand(rand)),
+            (rand: Random) => {
+              val micros = if (validJulianDatetime) {
+                DateTimeUtils.fromJavaTimestamp(getRandomTimestamp(rand))
+              } else {
+                uniformMicrosRand(rand)
+              }
+              DateTimeUtils.microsToInstant(micros)
+            },
             specialTs.map { s =>
               val ldt = LocalDateTime.parse(s.replace(" ", "T"))
               ldt.atZone(ZoneId.systemDefault()).toInstant
@@ -233,18 +264,7 @@ object RandomDataGenerator {
         } else {
           randomNumeric[java.sql.Timestamp](
             rand,
-            (rand: Random) => {
-              // DateTimeUtils.toJavaTimestamp takes microsecond.
-              val ts = DateTimeUtils.toJavaTimestamp(uniformMicorsRand(rand))
-              // The generated `ts` is based on the hybrid calendar Julian + Gregorian since
-              // 1582-10-15 but it should be valid in Proleptic Gregorian calendar too which is used
-              // by Spark SQL since version 3.0 (see SPARK-26651). We try to convert `ts` to
-              // a local timestamp in Proleptic Gregorian calendar to satisfy this requirement. Some
-              // years are leap years in Julian calendar but not in Proleptic Gregorian calendar.
-              // As the consequence of that, 29 February of such years might not exist in Proleptic
-              // Gregorian calendar. When this happens, we shift the timestamp `ts` by one day.
-              Try { ts.toLocalDateTime; ts }.getOrElse(new Timestamp(ts.getTime + MILLIS_PER_DAY))
-            },
+            getRandomTimestamp,
             specialTs.map(java.sql.Timestamp.valueOf))
         }
       case CalendarIntervalType => Some(() => {
@@ -253,6 +273,8 @@ object RandomDataGenerator {
         val ns = rand.nextLong()
         new CalendarInterval(months, days, ns)
       })
+      case DayTimeIntervalType => Some(() => Duration.of(rand.nextLong(), ChronoUnit.MICROS))
+      case YearMonthIntervalType => Some(() => Period.ofMonths(rand.nextInt()).normalized())
       case DecimalType.Fixed(precision, scale) => Some(
         () => BigDecimal.apply(
           rand.nextLong() % math.pow(10, precision).toLong,
@@ -260,10 +282,10 @@ object RandomDataGenerator {
           new MathContext(precision)).bigDecimal)
       case DoubleType => randomNumeric[Double](
         rand, r => longBitsToDouble(r.nextLong()), Seq(Double.MinValue, Double.MinPositiveValue,
-          Double.MaxValue, Double.PositiveInfinity, Double.NegativeInfinity, Double.NaN, 0.0))
+          Double.MaxValue, Double.PositiveInfinity, Double.NegativeInfinity, Double.NaN, 0.0, -0.0))
       case FloatType => randomNumeric[Float](
         rand, r => intBitsToFloat(r.nextInt()), Seq(Float.MinValue, Float.MinPositiveValue,
-          Float.MaxValue, Float.PositiveInfinity, Float.NegativeInfinity, Float.NaN, 0.0f))
+          Float.MaxValue, Float.PositiveInfinity, Float.NegativeInfinity, Float.NaN, 0.0f, -0.0f))
       case ByteType => randomNumeric[Byte](
         rand, _.nextInt().toByte, Seq(Byte.MinValue, Byte.MaxValue, 0.toByte))
       case IntegerType => randomNumeric[Int](
@@ -360,7 +382,7 @@ object RandomDataGenerator {
               arr += gen()
               i += 1
             }
-            arr
+            arr.toSeq
           }
           fields += data
         case StructType(children) =>
@@ -372,6 +394,6 @@ object RandomDataGenerator {
           fields += gen()
       }
     }
-    Row.fromSeq(fields)
+    Row.fromSeq(fields.toSeq)
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGeneratorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGeneratorSuite.scala
index cb335e5f435a3..69dca2cb7384b 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGeneratorSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGeneratorSuite.scala
@@ -143,4 +143,18 @@ class RandomDataGeneratorSuite extends SparkFunSuite with SQLHelper {
     assert(!Arrays.equals(array1, arrayExpected))
     assert(Arrays.equals(array2, arrayExpected))
   }
+
+  test("SPARK-35116: The generated data fits the precision of DayTimeIntervalType in spark") {
+    Seq(DayTimeIntervalType, YearMonthIntervalType).foreach { dt =>
+      for (seed <- 1 to 1000) {
+        val generator = RandomDataGenerator.forType(dt, false, new Random(seed)).get
+        val toCatalyst = CatalystTypeConverters.createToCatalystConverter(dt)
+        val toScala = CatalystTypeConverters.createToScalaConverter(dt)
+        val data = generator.apply()
+        val catalyst = toCatalyst(data)
+        val convertedBack = toScala(catalyst)
+        assert(data == convertedBack)
+      }
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/RowJsonSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/RowJsonSuite.scala
index ac18b0f79b5f3..1962fca66c059 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/RowJsonSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/RowJsonSuite.scala
@@ -17,14 +17,13 @@
 package org.apache.spark.sql
 
 import java.sql.{Date, Timestamp}
-import java.time.{Instant, LocalDate}
+import java.time.LocalDate
 
 import org.json4s.JsonAST.{JArray, JBool, JDecimal, JDouble, JLong, JNull, JObject, JString, JValue}
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.encoders.{ExamplePoint, ExamplePointUDT}
 import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 /**
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/RowTest.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/RowTest.scala
index 25699de33d717..385f749736846 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/RowTest.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/RowTest.scala
@@ -17,13 +17,15 @@
 
 package org.apache.spark.sql
 
-import org.scalatest.{FunSpec, Matchers}
+import org.scalatest.funspec.AnyFunSpec
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{GenericRow, GenericRowWithSchema}
 import org.apache.spark.sql.types._
 
-class RowTest extends FunSpec with Matchers {
+class RowTest extends AnyFunSpec with Matchers {
 
   val schema = StructType(
     StructField("col1", StringType) ::
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/UnsafeProjectionBenchmark.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/UnsafeProjectionBenchmark.scala
index 950e313fb727a..352afaa2740ce 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/UnsafeProjectionBenchmark.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/UnsafeProjectionBenchmark.scala
@@ -27,7 +27,8 @@ import org.apache.spark.sql.types._
  * Benchmark `UnsafeProjection` for fixed-length/primitive-type fields.
  * {{{
  *   To run this benchmark:
- *   1. without sbt: bin/spark-submit --class <this class> <spark sql test jar>
+ *   1. without sbt:
+ *      bin/spark-submit --class <this class> --jars <spark core test jar> <spark catalyst test jar>
  *   2. build/sbt "sql/test:runMain <this class>"
  *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
  *      Results will be written to "benchmarks/UnsafeProjectionBenchmark-results.txt".
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystTypeConvertersSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystTypeConvertersSuite.scala
index b9e7cf3049896..169c5d6a31973 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystTypeConvertersSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystTypeConvertersSuite.scala
@@ -17,13 +17,13 @@
 
 package org.apache.spark.sql.catalyst
 
-import java.time.{Instant, LocalDate}
+import java.time.{Duration, Instant, LocalDate, Period}
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.expressions.UnsafeArrayData
 import org.apache.spark.sql.catalyst.plans.SQLHelper
-import org.apache.spark.sql.catalyst.util.{DateTimeUtils, GenericArrayData}
+import org.apache.spark.sql.catalyst.util.{DateTimeConstants, DateTimeUtils, GenericArrayData, IntervalUtils}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
@@ -152,6 +152,13 @@ class CatalystTypeConvertersSuite extends SparkFunSuite with SQLHelper {
     assert(converter(chr) === expected)
   }
 
+  test("SPARK-33390: Make Literal support char array") {
+    val ac = Array('a', 'c')
+    val converter = CatalystTypeConverters.createToCatalystConverter(StringType)
+    val expected = UTF8String.fromString(String.valueOf(ac))
+    assert(converter(ac) === expected)
+  }
+
   test("converting java.time.Instant to TimestampType") {
     Seq(
       "0101-02-16T10:11:32Z",
@@ -190,8 +197,8 @@ class CatalystTypeConvertersSuite extends SparkFunSuite with SQLHelper {
       "1970-01-01",
       "1972-12-31",
       "2019-02-16",
-      "2119-03-16").foreach { timestamp =>
-      val input = LocalDate.parse(timestamp)
+      "2119-03-16").foreach { date =>
+      val input = LocalDate.parse(date)
       val result = CatalystTypeConverters.convertToCatalyst(input)
       val expected = DateTimeUtils.localDateToDays(input)
       assert(result === expected)
@@ -216,4 +223,115 @@ class CatalystTypeConvertersSuite extends SparkFunSuite with SQLHelper {
       }
     }
   }
+
+  test("SPARK-34605: converting java.time.Duration to DayTimeIntervalType") {
+    Seq(
+      Duration.ZERO,
+      Duration.ofNanos(1),
+      Duration.ofNanos(-1),
+      Duration.ofSeconds(0, Long.MaxValue),
+      Duration.ofSeconds(0, Long.MinValue),
+      Duration.ofDays(106751991),
+      Duration.ofDays(-106751991)).foreach { input =>
+      val result = CatalystTypeConverters.convertToCatalyst(input)
+      val expected = IntervalUtils.durationToMicros(input)
+      assert(result === expected)
+    }
+
+    val errMsg = intercept[ArithmeticException] {
+      IntervalUtils.durationToMicros(Duration.ofSeconds(Long.MaxValue, Long.MaxValue))
+    }.getMessage
+    assert(errMsg.contains("long overflow"))
+  }
+
+  test("SPARK-34605: converting DayTimeIntervalType to java.time.Duration") {
+    Seq(
+      0L,
+      1L,
+      999999,
+      -1000000,
+      Long.MaxValue).foreach { input =>
+      Seq(1L, -1L).foreach { sign =>
+        val us = sign * input
+        val duration = IntervalUtils.microsToDuration(us)
+        assert(CatalystTypeConverters.createToScalaConverter(DayTimeIntervalType)(us) === duration)
+      }
+    }
+  }
+
+  test("SPARK-34615: converting java.time.Period to YearMonthIntervalType") {
+    Seq(
+      Period.ZERO,
+      Period.ofMonths(1),
+      Period.ofMonths(-1),
+      Period.ofMonths(Int.MaxValue).normalized(),
+      Period.ofMonths(Int.MinValue).normalized(),
+      Period.ofYears(106751991),
+      Period.ofYears(-106751991)).foreach { input =>
+      val result = CatalystTypeConverters.convertToCatalyst(input)
+      val expected = IntervalUtils.periodToMonths(input)
+      assert(result === expected)
+    }
+
+    val errMsg = intercept[ArithmeticException] {
+      IntervalUtils.periodToMonths(Period.of(Int.MaxValue, Int.MaxValue, Int.MaxValue))
+    }.getMessage
+    assert(errMsg.contains("integer overflow"))
+  }
+
+  test("SPARK-34615: converting YearMonthIntervalType to java.time.Period") {
+    Seq(
+      0,
+      1,
+      999999,
+      1000000,
+      Int.MaxValue).foreach { input =>
+      Seq(1, -1).foreach { sign =>
+        val months = sign * input
+        val period = IntervalUtils.monthsToPeriod(months)
+        assert(
+          CatalystTypeConverters.createToScalaConverter(YearMonthIntervalType)(months) === period)
+      }
+    }
+  }
+
+  test("SPARK-35204: createToCatalystConverter for date") {
+    Seq(true, false).foreach { enable =>
+      withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> enable.toString) {
+        Seq(-1234, 0, 1234).foreach { days =>
+          val converter = CatalystTypeConverters.createToCatalystConverter(DateType)
+
+          val ld = LocalDate.ofEpochDay(days)
+          val result1 = converter(ld)
+
+          val d = java.sql.Date.valueOf(ld)
+          val result2 = converter(d)
+
+          val expected = DateTimeUtils.localDateToDays(ld)
+          assert(result1 === expected)
+          assert(result2 === expected)
+        }
+      }
+    }
+  }
+
+  test("SPARK-35204: createToCatalystConverter for timestamp") {
+    Seq(true, false).foreach { enable =>
+      withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> enable.toString) {
+        Seq(-1234, 0, 1234).foreach { seconds =>
+          val converter = CatalystTypeConverters.createToCatalystConverter(TimestampType)
+
+          val i = Instant.ofEpochSecond(seconds)
+          val result1 = converter(i)
+
+          val t = new java.sql.Timestamp(seconds * DateTimeConstants.MILLIS_PER_SECOND)
+          val result2 = converter(t)
+
+          val expected = seconds * DateTimeConstants.MICROS_PER_SECOND
+          assert(result1 === expected)
+          assert(result2 === expected)
+        }
+      }
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/OrderedFiltersSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/OrderedFiltersSuite.scala
new file mode 100644
index 0000000000000..b156cb52e921c
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/OrderedFiltersSuite.scala
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst
+
+import org.apache.spark.sql.sources
+import org.apache.spark.sql.types.StructType
+
+class OrderedFiltersSuite extends StructFiltersSuite {
+  override def createFilters(filters: Seq[sources.Filter], schema: StructType): StructFilters = {
+    new OrderedFilters(filters, schema)
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/SQLKeywordSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/SQLKeywordSuite.scala
new file mode 100644
index 0000000000000..45f88628f3ab3
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/SQLKeywordSuite.scala
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst
+
+import java.io.File
+import java.nio.file.Files
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.plans.SQLHelper
+import org.apache.spark.sql.catalyst.util.fileToString
+
+trait SQLKeywordUtils extends SparkFunSuite with SQLHelper {
+
+  val sqlSyntaxDefs = {
+    val sqlBasePath = {
+      getWorkspaceFilePath("sql", "catalyst", "src", "main", "antlr4", "org",
+        "apache", "spark", "sql", "catalyst", "parser", "SqlBase.g4").toFile
+    }
+    fileToString(sqlBasePath).split("\n")
+  }
+
+  // each element is an array of 4 string: the keyword name, reserve or not in Spark ANSI mode,
+  // Spark default mode, and the SQL standard.
+  val keywordsInDoc: Array[Array[String]] = {
+    val docPath = {
+      getWorkspaceFilePath("docs", "sql-ref-ansi-compliance.md").toFile
+    }
+    fileToString(docPath).split("\n")
+      .dropWhile(!_.startsWith("|Keyword|")).drop(2).takeWhile(_.startsWith("|"))
+      .map(_.stripPrefix("|").split("\\|").map(_.trim))
+  }
+
+  private def parseAntlrGrammars[T](startTag: String, endTag: String)
+      (f: PartialFunction[String, Seq[T]]): Set[T] = {
+    val keywords = new mutable.ArrayBuffer[T]
+    val default = (_: String) => Nil
+    var startTagFound = false
+    var parseFinished = false
+    val lineIter = sqlSyntaxDefs.toIterator
+    while (!parseFinished && lineIter.hasNext) {
+      val line = lineIter.next()
+      if (line.trim.startsWith(startTag)) {
+        startTagFound = true
+      } else if (line.trim.startsWith(endTag)) {
+        parseFinished = true
+      } else if (startTagFound) {
+        f.applyOrElse(line, default).foreach { symbol =>
+          keywords += symbol
+        }
+      }
+    }
+    assert(keywords.nonEmpty && startTagFound && parseFinished, "cannot extract keywords from " +
+      s"the `SqlBase.g4` file, so please check if the start/end tags (`$startTag` and `$endTag`) " +
+      "are placed correctly in the file.")
+    keywords.toSet
+  }
+
+  // If a symbol does not have the same string with its literal (e.g., `SETMINUS: 'MINUS';`),
+  // we need to map a symbol to actual literal strings.
+  val symbolsToExpandIntoDifferentLiterals = {
+    val kwDef = """([A-Z_]+):(.+);""".r
+    val keywords = parseAntlrGrammars(
+        "//--SPARK-KEYWORD-LIST-START", "//--SPARK-KEYWORD-LIST-END") {
+      case kwDef(symbol, literalDef) =>
+        val splitDefs = literalDef.split("""\|""")
+        val hasMultipleLiterals = splitDefs.length > 1
+        // The case where a symbol has multiple literal definitions,
+        // e.g., `DATABASES: 'DATABASES' | 'SCHEMAS';`.
+        if (hasMultipleLiterals) {
+          // Filters out inappropriate entries, e.g., `!` in `NOT: 'NOT' | '!';`
+          val litDef = """([A-Z_]+)""".r
+          val literals = splitDefs.map(_.replaceAll("'", "").trim).toSeq.flatMap {
+            case litDef(lit) => Some(lit)
+            case _ => None
+          }
+          (symbol, literals) :: Nil
+        } else {
+          val literal = literalDef.replaceAll("'", "").trim
+          // The case where a symbol string and its literal string are different,
+          // e.g., `SETMINUS: 'MINUS';`.
+          if (symbol != literal) {
+            (symbol, literal :: Nil) :: Nil
+          } else {
+            Nil
+          }
+        }
+    }
+    keywords.toMap
+  }
+
+  // All the SQL keywords defined in `SqlBase.g4`
+  val allCandidateKeywords: Set[String] = {
+    val kwDef = """([A-Z_]+):.+;""".r
+    parseAntlrGrammars(
+        "//--SPARK-KEYWORD-LIST-START", "//--SPARK-KEYWORD-LIST-END") {
+      // Parses a pattern, e.g., `AFTER: 'AFTER';`
+      case kwDef(symbol) =>
+        if (symbolsToExpandIntoDifferentLiterals.contains(symbol)) {
+          symbolsToExpandIntoDifferentLiterals(symbol)
+        } else {
+          symbol :: Nil
+        }
+    }
+  }
+
+  val nonReservedKeywordsInAnsiMode: Set[String] = {
+    val kwDef = """\s*[\|:]\s*([A-Z_]+)\s*""".r
+    parseAntlrGrammars("//--ANSI-NON-RESERVED-START", "//--ANSI-NON-RESERVED-END") {
+      // Parses a pattern, e.g., `    | AFTER`
+      case kwDef(symbol) =>
+        if (symbolsToExpandIntoDifferentLiterals.contains(symbol)) {
+          symbolsToExpandIntoDifferentLiterals(symbol)
+        } else {
+          symbol :: Nil
+        }
+    }
+  }
+
+  val reservedKeywordsInAnsiMode = allCandidateKeywords -- nonReservedKeywordsInAnsiMode
+
+  val nonReservedKeywordsInDefaultMode: Set[String] = {
+    val kwDef = """\s*[\|:]\s*([A-Z_]+)\s*""".r
+    parseAntlrGrammars("//--DEFAULT-NON-RESERVED-START", "//--DEFAULT-NON-RESERVED-END") {
+      // Parses a pattern, e.g., `    | AFTER`
+      case kwDef(symbol) =>
+        if (symbolsToExpandIntoDifferentLiterals.contains(symbol)) {
+          symbolsToExpandIntoDifferentLiterals(symbol)
+        } else {
+          symbol :: Nil
+        }
+    }
+  }
+}
+
+class SQLKeywordSuite extends SQLKeywordUtils {
+  test("all keywords are documented") {
+    val documentedKeywords = keywordsInDoc.map(_.head).toSet
+    if (allCandidateKeywords != documentedKeywords) {
+      val undocumented = (allCandidateKeywords -- documentedKeywords).toSeq.sorted
+      fail("Some keywords are not documented: " + undocumented.mkString(", "))
+    }
+  }
+
+  test("Spark keywords are documented correctly under ANSI mode") {
+    // keywords under ANSI mode should either be reserved or non-reserved.
+    keywordsInDoc.map(_.apply(1)).foreach { desc =>
+      assert(desc == "reserved" || desc == "non-reserved")
+    }
+
+    val nonReservedInDoc = keywordsInDoc.filter(_.apply(1) == "non-reserved").map(_.head).toSet
+    if (nonReservedKeywordsInAnsiMode != nonReservedInDoc) {
+      val misImplemented = ((nonReservedInDoc -- nonReservedKeywordsInAnsiMode) ++
+        (nonReservedKeywordsInAnsiMode -- nonReservedInDoc)).toSeq.sorted
+      fail("Some keywords are documented and implemented inconsistently: " +
+        misImplemented.mkString(", "))
+    }
+  }
+
+  test("Spark keywords are documented correctly under default mode") {
+    // keywords under default mode should either be strict-non-reserved or non-reserved.
+    keywordsInDoc.map(_.apply(2)).foreach { desc =>
+      assert(desc == "strict-non-reserved" || desc == "non-reserved")
+    }
+
+    val nonReservedInDoc = keywordsInDoc.filter(_.apply(2) == "non-reserved").map(_.head).toSet
+    if (nonReservedKeywordsInDefaultMode != nonReservedInDoc) {
+      val misImplemented = ((nonReservedInDoc -- nonReservedKeywordsInDefaultMode) ++
+        (nonReservedKeywordsInDefaultMode -- nonReservedInDoc)).toSeq.sorted
+      fail("Some keywords are documented and implemented inconsistently: " +
+        misImplemented.mkString(", "))
+    }
+  }
+
+  test("SQL 2016 keywords are documented correctly") {
+    withTempDir { dir =>
+      val tmpFile = new File(dir, "tmp")
+      val is = Thread.currentThread().getContextClassLoader
+        .getResourceAsStream("ansi-sql-2016-reserved-keywords.txt")
+      Files.copy(is, tmpFile.toPath)
+      val reservedKeywordsInSql2016 = Files.readAllLines(tmpFile.toPath)
+        .asScala.filterNot(_.startsWith("--")).map(_.trim).toSet
+      val documented = keywordsInDoc.filter(_.last == "reserved").map(_.head).toSet
+      assert((documented -- reservedKeywordsInSql2016).isEmpty)
+    }
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
index b981a50499bf5..164bbd7f34d04 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
@@ -22,6 +22,7 @@ import java.sql.{Date, Timestamp}
 import scala.reflect.runtime.universe.TypeTag
 
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.FooEnum.FooEnum
 import org.apache.spark.sql.catalyst.analysis.UnresolvedExtractValue
 import org.apache.spark.sql.catalyst.expressions.{CreateNamedStruct, Expression, If, SpecificInternalRow, UpCast}
 import org.apache.spark.sql.catalyst.expressions.objects.{AssertNotNull, NewInstance}
@@ -90,6 +91,13 @@ case class FooWithAnnotation(f1: String @FooAnnotation, f2: Option[String] @FooA
 
 case class SpecialCharAsFieldData(`field.1`: String, `field 2`: String)
 
+object FooEnum extends Enumeration {
+  type FooEnum = Value
+  val E1, E2 = Value
+}
+
+case class FooClassWithEnum(i: Int, e: FooEnum)
+
 object TestingUDT {
   @SQLUserDefinedType(udt = classOf[NestedStructUDT])
   class NestedStruct(val a: Integer, val b: Long, val c: Double)
@@ -126,7 +134,6 @@ object ScroogeLikeExample {
 }
 
 trait ScroogeLikeExample extends Product1[Int] with Serializable {
-  import ScroogeLikeExample._
 
   def x: Int
 
@@ -437,4 +444,11 @@ class ScalaReflectionSuite extends SparkFunSuite {
       StructField("f2", StringType))))
     assert(deserializerFor[FooWithAnnotation].dataType == ObjectType(classOf[FooWithAnnotation]))
   }
+
+  test("SPARK-32585: Support scala enumeration in ScalaReflection") {
+    assert(serializerFor[FooClassWithEnum].dataType == StructType(Seq(
+      StructField("i", IntegerType, false),
+      StructField("e", StringType, true))))
+    assert(deserializerFor[FooClassWithEnum].dataType == ObjectType(classOf[FooClassWithEnum]))
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/StructFiltersSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/StructFiltersSuite.scala
new file mode 100644
index 0000000000000..3893b2b07c519
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/StructFiltersSuite.scala
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.sources
+import org.apache.spark.sql.sources.{AlwaysFalse, AlwaysTrue, Filter}
+import org.apache.spark.sql.types.{IntegerType, StructType}
+import org.apache.spark.unsafe.types.UTF8String
+
+abstract class StructFiltersSuite extends SparkFunSuite {
+
+  protected def createFilters(filters: Seq[sources.Filter], schema: StructType): StructFilters
+
+  test("filter to expression conversion") {
+    val ref = BoundReference(0, IntegerType, true)
+    def check(f: Filter, expr: Expression): Unit = {
+      assert(StructFilters.filterToExpression(f, _ => Some(ref)).get === expr)
+    }
+
+    check(sources.AlwaysTrue, Literal(true))
+    check(sources.AlwaysFalse, Literal(false))
+    check(sources.IsNull("a"), IsNull(ref))
+    check(sources.Not(sources.IsNull("a")), Not(IsNull(ref)))
+    check(sources.IsNotNull("a"), IsNotNull(ref))
+    check(sources.EqualTo("a", "b"), EqualTo(ref, Literal("b")))
+    check(sources.EqualNullSafe("a", "b"), EqualNullSafe(ref, Literal("b")))
+    check(sources.StringStartsWith("a", "b"), StartsWith(ref, Literal("b")))
+    check(sources.StringEndsWith("a", "b"), EndsWith(ref, Literal("b")))
+    check(sources.StringContains("a", "b"), Contains(ref, Literal("b")))
+    check(sources.LessThanOrEqual("a", 1), LessThanOrEqual(ref, Literal(1)))
+    check(sources.LessThan("a", 1), LessThan(ref, Literal(1)))
+    check(sources.GreaterThanOrEqual("a", 1), GreaterThanOrEqual(ref, Literal(1)))
+    check(sources.GreaterThan("a", 1), GreaterThan(ref, Literal(1)))
+    check(sources.And(sources.AlwaysTrue, sources.AlwaysTrue), And(Literal(true), Literal(true)))
+    check(sources.Or(sources.AlwaysTrue, sources.AlwaysTrue), Or(Literal(true), Literal(true)))
+    check(sources.In("a", Array(1)), In(ref, Seq(Literal(1))))
+  }
+
+  private def getSchema(str: String): StructType = str match {
+    case "" => new StructType()
+    case _ => StructType.fromDDL(str)
+  }
+
+  test("skipping rows") {
+    def check(
+      requiredSchema: String = "i INTEGER, d DOUBLE",
+      filters: Seq[Filter],
+      row: InternalRow,
+      pos: Int,
+      skip: Boolean): Unit = {
+      val structFilters = createFilters(filters, getSchema(requiredSchema))
+      structFilters.reset()
+      assert(structFilters.skipRow(row, pos) === skip)
+    }
+
+    check(filters = Seq(), row = InternalRow(3.14), pos = 0, skip = false)
+    check(filters = Seq(AlwaysTrue), row = InternalRow(1), pos = 0, skip = false)
+    check(filters = Seq(AlwaysFalse), row = InternalRow(1), pos = 0, skip = true)
+    check(
+      filters = Seq(sources.EqualTo("i", 1), sources.LessThan("d", 10), sources.AlwaysFalse),
+      row = InternalRow(1, 3.14),
+      pos = 0,
+      skip = true)
+    check(
+      filters = Seq(sources.EqualTo("i", 10)),
+      row = InternalRow(10, 3.14),
+      pos = 0,
+      skip = false)
+    check(
+      filters = Seq(sources.IsNotNull("d"), sources.GreaterThanOrEqual("d", 2.96)),
+      row = InternalRow(3.14),
+      pos = 0,
+      skip = false)
+    check(
+      filters = Seq(sources.In("i", Array(10, 20)), sources.LessThanOrEqual("d", 2.96)),
+      row = InternalRow(10, 3.14),
+      pos = 1,
+      skip = true)
+    val filters1 = Seq(
+      sources.Or(
+        sources.AlwaysTrue,
+        sources.And(
+          sources.Not(sources.IsNull("i")),
+          sources.Not(
+            sources.And(
+              sources.StringEndsWith("s", "ab"),
+              sources.StringEndsWith("s", "cd")
+            )
+          )
+        )
+      ),
+      sources.GreaterThan("d", 0),
+      sources.LessThan("i", 500)
+    )
+    val filters2 = Seq(
+      sources.And(
+        sources.StringContains("s", "abc"),
+        sources.And(
+          sources.Not(sources.IsNull("i")),
+          sources.And(
+            sources.StringEndsWith("s", "ab"),
+            sources.StringEndsWith("s", "bc")
+          )
+        )
+      ),
+      sources.GreaterThan("d", 100),
+      sources.LessThan("i", 0)
+    )
+    Seq(filters1 -> false, filters2 -> true).foreach { case (filters, skip) =>
+      val schema = "i INTEGER, d DOUBLE, s STRING"
+      val row = InternalRow(10, 3.14, UTF8String.fromString("abc"))
+      val structFilters = createFilters(filters, getSchema(schema))
+      structFilters.reset()
+      for { p <- 0 until 3 if !skip } {
+        assert(structFilters.skipRow(row, p) === skip, s"p = $p filters = $filters skip = $skip")
+      }
+    }
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
index 166ffec44a60d..8ea84a484d570 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
@@ -88,6 +88,8 @@ case class TestFunction(
   extends Expression with ImplicitCastInputTypes with Unevaluable {
   override def nullable: Boolean = true
   override def dataType: DataType = StringType
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
+    copy(children = newChildren)
 }
 
 case class UnresolvedTestPlan() extends LeafNode {
@@ -184,22 +186,22 @@ class AnalysisErrorSuite extends AnalysisTest {
   errorTest(
     "distinct function",
     CatalystSqlParser.parsePlan("SELECT hex(DISTINCT a) FROM TaBlE"),
-    "DISTINCT or FILTER specified, but hex is not an aggregate function" :: Nil)
+    "Function hex does not support DISTINCT" :: Nil)
 
   errorTest(
     "non aggregate function with filter predicate",
     CatalystSqlParser.parsePlan("SELECT hex(a) FILTER (WHERE c = 1) FROM TaBlE2"),
-    "DISTINCT or FILTER specified, but hex is not an aggregate function" :: Nil)
+    "Function hex does not support FILTER clause" :: Nil)
 
   errorTest(
     "distinct window function",
     CatalystSqlParser.parsePlan("SELECT percent_rank(DISTINCT a) OVER () FROM TaBlE"),
-    "DISTINCT or FILTER specified, but percent_rank is not an aggregate function" :: Nil)
+    "Function percent_rank does not support DISTINCT" :: Nil)
 
   errorTest(
     "window function with filter predicate",
     CatalystSqlParser.parsePlan("SELECT percent_rank(a) FILTER (WHERE c > 1) OVER () FROM TaBlE2"),
-    "DISTINCT or FILTER specified, but percent_rank is not an aggregate function" :: Nil)
+    "Function percent_rank does not support FILTER clause" :: Nil)
 
   errorTest(
     "higher order function with filter predicate",
@@ -207,16 +209,31 @@ class AnalysisErrorSuite extends AnalysisTest {
       "FILTER (WHERE c > 1)"),
     "FILTER predicate specified, but aggregate is not an aggregate function" :: Nil)
 
-  errorTest(
-    "DISTINCT aggregate function with filter predicate",
-    CatalystSqlParser.parsePlan("SELECT count(DISTINCT a) FILTER (WHERE c > 1) FROM TaBlE2"),
-    "DISTINCT and FILTER cannot be used in aggregate functions at the same time" :: Nil)
-
   errorTest(
     "non-deterministic filter predicate in aggregate functions",
     CatalystSqlParser.parsePlan("SELECT count(a) FILTER (WHERE rand(int(c)) > 1) FROM TaBlE2"),
     "FILTER expression is non-deterministic, it cannot be used in aggregate functions" :: Nil)
 
+  errorTest(
+    "function don't support ignore nulls",
+    CatalystSqlParser.parsePlan("SELECT hex(a) IGNORE NULLS FROM TaBlE2"),
+    "Function hex does not support IGNORE NULLS" :: Nil)
+
+  errorTest(
+    "some window function don't support ignore nulls",
+    CatalystSqlParser.parsePlan("SELECT percent_rank(a) IGNORE NULLS FROM TaBlE2"),
+    "Function percent_rank does not support IGNORE NULLS" :: Nil)
+
+  errorTest(
+    "aggregate function don't support ignore nulls",
+    CatalystSqlParser.parsePlan("SELECT count(a) IGNORE NULLS FROM TaBlE2"),
+    "Function count does not support IGNORE NULLS" :: Nil)
+
+  errorTest(
+    "higher order function don't support ignore nulls",
+    CatalystSqlParser.parsePlan("SELECT aggregate(array(1, 2, 3), 0, (acc, x) -> acc + x) " +
+      "IGNORE NULLS"), "Function aggregate does not support IGNORE NULLS" :: Nil)
+
   errorTest(
     "nested aggregate functions",
     testRelation.groupBy($"a")(
@@ -236,7 +253,29 @@ class AnalysisErrorSuite extends AnalysisTest {
           UnresolvedAttribute("a") :: Nil,
           SortOrder(UnresolvedAttribute("b"), Ascending) :: Nil,
           SpecifiedWindowFrame(RangeFrame, Literal(1), Literal(2)))).as("window")),
-    "window frame" :: "must match the required frame" :: Nil)
+    "Cannot specify window frame for lead function" :: Nil)
+
+  errorTest(
+    "the offset of nth_value window function is negative or zero",
+    testRelation2.select(
+      WindowExpression(
+        new NthValue(AttributeReference("b", IntegerType)(), Literal(0)),
+        WindowSpecDefinition(
+          UnresolvedAttribute("a") :: Nil,
+          SortOrder(UnresolvedAttribute("b"), Ascending) :: Nil,
+          SpecifiedWindowFrame(RowFrame, Literal(0), Literal(0)))).as("window")),
+    "The 'offset' argument of nth_value must be greater than zero but it is 0." :: Nil)
+
+  errorTest(
+    "the offset of nth_value window function is not int literal",
+    testRelation2.select(
+      WindowExpression(
+        new NthValue(AttributeReference("b", IntegerType)(), Literal(true)),
+        WindowSpecDefinition(
+          UnresolvedAttribute("a") :: Nil,
+          SortOrder(UnresolvedAttribute("b"), Ascending) :: Nil,
+          SpecifiedWindowFrame(RowFrame, Literal(0), Literal(0)))).as("window")),
+    "argument 2 requires int type, however, 'true' is of boolean type." :: Nil)
 
   errorTest(
     "too many generators",
@@ -268,7 +307,7 @@ class AnalysisErrorSuite extends AnalysisTest {
   errorTest(
     "sorting by attributes are not from grouping expressions",
     testRelation2.groupBy($"a", $"c")($"a", $"c", count($"a").as("a3")).orderBy($"b".asc),
-    "cannot resolve" :: "'`b`'" :: "given input columns" :: "[a, a3, c]" :: Nil)
+    "cannot resolve" :: "'b'" :: "given input columns" :: "[a, a3, c]" :: Nil)
 
   errorTest(
     "non-boolean filters",
@@ -283,7 +322,7 @@ class AnalysisErrorSuite extends AnalysisTest {
   errorTest(
     "missing group by",
     testRelation2.groupBy($"a")($"b"),
-    "'`b`'" :: "group by" :: Nil
+    "'b'" :: "group by" :: Nil
   )
 
   errorTest(
@@ -361,7 +400,7 @@ class AnalysisErrorSuite extends AnalysisTest {
     "SPARK-9955: correct error message for aggregate",
     // When parse SQL string, we will wrap aggregate expressions with UnresolvedAlias.
     testRelation2.where($"bad_column" > 1).groupBy($"a")(UnresolvedAlias(max($"b"))),
-    "cannot resolve '`bad_column`'" :: Nil)
+    "cannot resolve 'bad_column'" :: Nil)
 
   errorTest(
     "slide duration greater than window in time window",
@@ -538,7 +577,7 @@ class AnalysisErrorSuite extends AnalysisTest {
       if (shouldSuccess) {
         assertAnalysisSuccess(plan, true)
       } else {
-        assertAnalysisError(plan, "expression `a` cannot be used as a grouping expression" :: Nil)
+        assertAnalysisError(plan, "expression a cannot be used as a grouping expression" :: Nil)
       }
     }
 
@@ -683,4 +722,73 @@ class AnalysisErrorSuite extends AnalysisTest {
           UnresolvedRelation(TableIdentifier("t", Option("nonexist")))))))
     assertAnalysisError(plan, "Table or view not found:" :: Nil)
   }
+
+  test("SPARK-33909: Check rand functions seed is legal at analyer side") {
+    Seq(Rand("a".attr), Randn("a".attr)).foreach { r =>
+      val plan = Project(Seq(r.as("r")), testRelation)
+      assertAnalysisError(plan,
+        s"Input argument to ${r.prettyName} must be a constant." :: Nil)
+    }
+    Seq(Rand(1.0), Rand("1"), Randn("a")).foreach { r =>
+      val plan = Project(Seq(r.as("r")), testRelation)
+      assertAnalysisError(plan,
+        s"data type mismatch: argument 1 requires (int or bigint) type" :: Nil)
+    }
+  }
+
+  test("SPARK-34946: correlated scalar subquery in grouping expressions only") {
+    val c1 = AttributeReference("c1", IntegerType)()
+    val c2 = AttributeReference("c2", IntegerType)()
+    val t = LocalRelation(c1, c2)
+    val plan = Aggregate(
+      ScalarSubquery(
+        Aggregate(Nil, sum($"c2").as("sum") :: Nil,
+          Filter($"t1.c1" === $"t2.c1",
+            t.as("t2")))
+      ) :: Nil,
+      sum($"c2").as("sum") :: Nil, t.as("t1"))
+    assertAnalysisError(plan, "Correlated scalar subqueries in the group by clause must also be " +
+      "in the aggregate expressions" :: Nil)
+  }
+
+  test("SPARK-34946: correlated scalar subquery in aggregate expressions only") {
+    val c1 = AttributeReference("c1", IntegerType)()
+    val c2 = AttributeReference("c2", IntegerType)()
+    val t = LocalRelation(c1, c2)
+    val plan = Aggregate(
+      $"c1" :: Nil,
+      ScalarSubquery(
+        Aggregate(Nil, sum($"c2").as("sum") :: Nil,
+          Filter($"t1.c1" === $"t2.c1",
+            t.as("t2")))
+      ).as("sub") :: Nil, t.as("t1"))
+    assertAnalysisError(plan, "Correlated scalar subquery 'scalarsubquery(t1.c1)' is " +
+      "neither present in the group by, nor in an aggregate function. Add it to group by " +
+      "using ordinal position or wrap it in first() (or first_value) if you don't care " +
+      "which value you get." :: Nil)
+  }
+
+  test("SPARK-35080: Unsupported correlated equality predicates in subquery") {
+    val a = AttributeReference("a", IntegerType)()
+    val b = AttributeReference("b", IntegerType)()
+    val c = AttributeReference("c", IntegerType)()
+    val t1 = LocalRelation(a, b)
+    val t2 = LocalRelation(c)
+    val conditions = Seq(
+      (abs($"a") === $"c", "abs(a) = outer(c)"),
+      (abs($"a") <=> $"c", "abs(a) <=> outer(c)"),
+      ($"a" + 1 === $"c", "(a + 1) = outer(c)"),
+      ($"a" + $"b" === $"c", "(a + b) = outer(c)"),
+      ($"a" + $"c" === $"b", "(a + outer(c)) = b"),
+      (And($"a" === $"c", Cast($"a", IntegerType) === $"c"), "CAST(a AS INT) = outer(c)"))
+    conditions.foreach { case (cond, msg) =>
+      val plan = Project(
+        ScalarSubquery(
+          Aggregate(Nil, count(Literal(1)).as("cnt") :: Nil,
+            Filter(cond, t1))
+        ).as("sub") :: Nil,
+        t2)
+      assertAnalysisError(plan, s"Correlated column is not allowed in predicate ($msg)" :: Nil)
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisExceptionPositionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisExceptionPositionSuite.scala
new file mode 100644
index 0000000000000..e50a58f8ce5fe
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisExceptionPositionSuite.scala
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan
+
+class AnalysisExceptionPositionSuite extends AnalysisTest {
+  test("SPARK-33886: UnresolvedTable should retain sql text position") {
+    verifyTablePosition("MSCK REPAIR TABLE unknown", "unknown")
+    verifyTablePosition("LOAD DATA LOCAL INPATH 'filepath' INTO TABLE unknown", "unknown")
+    verifyTablePosition("TRUNCATE TABLE unknown", "unknown")
+    verifyTablePosition("SHOW PARTITIONS unknown", "unknown")
+    verifyTablePosition("ALTER TABLE unknown RECOVER PARTITIONS", "unknown")
+    verifyTablePosition("ALTER TABLE unknown ADD PARTITION (p=1)", "unknown")
+    verifyTablePosition("ALTER TABLE unknown PARTITION (p=1) RENAME TO PARTITION (p=2)", "unknown")
+    verifyTablePosition("ALTER TABLE unknown DROP PARTITION (p=1)", "unknown")
+    verifyTablePosition("ALTER TABLE unknown SET SERDEPROPERTIES ('a'='b')", "unknown")
+    verifyTablePosition("COMMENT ON TABLE unknown IS 'hello'", "unknown")
+  }
+
+  test("SPARK-33918: UnresolvedView should retain sql text position") {
+    verifyViewPosition("DROP VIEW unknown", "unknown")
+    verifyViewPosition("ALTER VIEW unknown SET TBLPROPERTIES ('k'='v')", "unknown")
+    verifyViewPosition("ALTER VIEW unknown UNSET TBLPROPERTIES ('k')", "unknown")
+    verifyViewPosition("ALTER VIEW unknown AS SELECT 1", "unknown")
+  }
+
+  test("SPARK-34057: UnresolvedTableOrView should retain sql text position") {
+    verifyTableOrViewPosition("DROP TABLE unknown", "unknown")
+    verifyTableOrViewPosition("DESCRIBE TABLE unknown", "unknown")
+    verifyTableOrPermanentViewPosition("ANALYZE TABLE unknown COMPUTE STATISTICS", "unknown")
+    verifyTableOrViewPosition("ANALYZE TABLE unknown COMPUTE STATISTICS FOR COLUMNS col", "unknown")
+    verifyTableOrViewPosition("ANALYZE TABLE unknown COMPUTE STATISTICS FOR ALL COLUMNS", "unknown")
+    verifyTableOrPermanentViewPosition("SHOW CREATE TABLE unknown", "unknown")
+    verifyTableOrViewPosition("REFRESH TABLE unknown", "unknown")
+    verifyTableOrViewPosition("SHOW COLUMNS FROM unknown", "unknown")
+    // Special case where namespace is prepended to the table name.
+    assertAnalysisError(
+      parsePlan("SHOW COLUMNS FROM unknown IN db"),
+      Seq(s"Table or view not found: db.unknown; line 1 pos 18"))
+    verifyTableOrViewPosition("ALTER TABLE unknown RENAME TO t", "unknown")
+    verifyTableOrViewPosition("ALTER VIEW unknown RENAME TO v", "unknown")
+  }
+
+  test("SPARK-34139: UnresolvedRelation should retain sql text position") {
+    verifyTableOrViewPosition("CACHE TABLE unknown", "unknown")
+    verifyTableOrViewPosition("UNCACHE TABLE unknown", "unknown")
+    verifyTableOrViewPosition("DELETE FROM unknown", "unknown")
+    verifyTableOrViewPosition("UPDATE unknown SET name='abc'", "unknown")
+    verifyTableOrViewPosition(
+      "MERGE INTO unknownTarget AS target USING TaBlE AS source " +
+        "ON target.col = source.col WHEN MATCHED THEN DELETE",
+      "unknownTarget")
+    verifyTableOrViewPosition(
+      "MERGE INTO TaBlE AS target USING unknownSource AS source " +
+        "ON target.col = source.col WHEN MATCHED THEN DELETE",
+      "unknownSource")
+    verifyTablePosition("INSERT INTO TABLE unknown SELECT 1", "unknown")
+    verifyTablePosition("INSERT OVERWRITE TABLE unknown VALUES (1, 'a')", "unknown")
+  }
+
+  private def verifyTablePosition(sql: String, table: String): Unit = {
+    verifyPosition(sql, table, "Table")
+  }
+
+  private def verifyViewPosition(sql: String, table: String): Unit = {
+    verifyPosition(sql, table, "View")
+  }
+
+  private def verifyTableOrViewPosition(sql: String, table: String): Unit = {
+    verifyPosition(sql, table, "Table or view")
+  }
+
+  private def verifyTableOrPermanentViewPosition(sql: String, table: String): Unit = {
+    verifyPosition(sql, table, "Table or permanent view")
+  }
+
+  private def verifyPosition(sql: String, table: String, msgPrefix: String): Unit = {
+    val expectedPos = sql.indexOf(table)
+    assert(expectedPos != -1)
+    assertAnalysisError(
+      parsePlan(sql),
+      Seq(s"$msgPrefix not found: $table; line 1 pos $expectedPos"))
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisExternalCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisExternalCatalogSuite.scala
index a85ac3fc4d0b7..df99cd851cc3e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisExternalCatalogSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisExternalCatalogSuite.scala
@@ -21,20 +21,17 @@ import java.io.File
 import java.net.URI
 
 import org.mockito.Mockito._
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
 
 import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
 import org.apache.spark.sql.catalyst.catalog.{CatalogDatabase, CatalogStorageFormat, CatalogTable, CatalogTableType, ExternalCatalog, InMemoryCatalog, SessionCatalog}
 import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeReference}
 import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, Project}
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
-import org.apache.spark.util.Utils
 
 class AnalysisExternalCatalogSuite extends AnalysisTest with Matchers {
   private def getAnalyzer(externCatalog: ExternalCatalog, databasePath: File): Analyzer = {
-    val conf = new SQLConf()
-    val catalog = new SessionCatalog(externCatalog, FunctionRegistry.builtin, conf)
+    val catalog = new SessionCatalog(externCatalog, FunctionRegistry.builtin)
     catalog.createDatabase(
       CatalogDatabase("default", "", databasePath.toURI, Map.empty),
       ignoreIfExists = false)
@@ -45,7 +42,7 @@ class AnalysisExternalCatalogSuite extends AnalysisTest with Matchers {
         CatalogStorageFormat.empty,
         StructType(Seq(StructField("a", IntegerType, nullable = true)))),
       ignoreIfExists = false)
-    new Analyzer(catalog, conf)
+    new Analyzer(catalog)
   }
 
   test("query builtin functions don't call the external catalog") {
@@ -67,7 +64,7 @@ class AnalysisExternalCatalogSuite extends AnalysisTest with Matchers {
     withTempDir { tempDir =>
       val inMemoryCatalog = new InMemoryCatalog
       val externCatalog = spy(inMemoryCatalog)
-      val catalog = new SessionCatalog(externCatalog, FunctionRegistry.builtin, conf)
+      val catalog = new SessionCatalog(externCatalog, FunctionRegistry.builtin)
       catalog.createDatabase(
         CatalogDatabase("default", "", new URI(tempDir.toString), Map.empty),
         ignoreIfExists = false)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
index 02472e153b09e..dc9f92d7c642d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
@@ -17,35 +17,58 @@
 
 package org.apache.spark.sql.catalyst.analysis
 
-import java.util.{Locale, TimeZone}
+import java.util.TimeZone
 
+import scala.collection.JavaConverters._
 import scala.reflect.ClassTag
+import scala.reflect.runtime.universe.TypeTag
 
 import org.apache.log4j.Level
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
 
 import org.apache.spark.api.python.PythonEvalType
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType, InMemoryCatalog, SessionCatalog}
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.{AliasIdentifier, TableIdentifier}
+import org.apache.spark.sql.catalyst.catalog.{InMemoryCatalog, SessionCatalog}
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-import org.apache.spark.sql.catalyst.errors.TreeNodeException
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Count, Sum}
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan
 import org.apache.spark.sql.catalyst.plans.{Cross, Inner}
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partitioning, RangePartitioning, RoundRobinPartitioning}
-import org.apache.spark.sql.catalyst.rules.RuleExecutor
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.connector.catalog.InMemoryTable
+import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
-
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
 class AnalysisSuite extends AnalysisTest with Matchers {
   import org.apache.spark.sql.catalyst.analysis.TestRelations._
 
+  test("fail for unresolved plan") {
+    intercept[AnalysisException] {
+      // `testRelation` does not have column `b`.
+      testRelation.select('b).analyze
+    }
+  }
+
+  test("fail if a leaf node has char/varchar type output") {
+    val schema1 = new StructType().add("c", CharType(5))
+    val schema2 = new StructType().add("c", VarcharType(5))
+    val schema3 = new StructType().add("c", ArrayType(CharType(5)))
+    Seq(schema1, schema2, schema3).foreach { schema =>
+      val table = new InMemoryTable("t", schema, Array.empty, Map.empty[String, String].asJava)
+      intercept[IllegalStateException] {
+        DataSourceV2Relation(
+          table, schema.toAttributes, None, None, CaseInsensitiveStringMap.empty()).analyze
+      }
+    }
+  }
+
   test("union project *") {
     val plan = (1 to 120)
       .map(_ => testRelation)
@@ -72,7 +95,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
       Project(Seq(UnresolvedAttribute("a")), testRelation),
       Project(testRelation.output, testRelation))
 
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       Project(Seq(UnresolvedAttribute("TbL.a")),
         SubqueryAlias("TbL", UnresolvedRelation(TableIdentifier("TaBlE")))),
       Project(testRelation.output, testRelation))
@@ -82,13 +105,13 @@ class AnalysisSuite extends AnalysisTest with Matchers {
         SubqueryAlias("TbL", UnresolvedRelation(TableIdentifier("TaBlE")))),
       Seq("cannot resolve"))
 
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       Project(Seq(UnresolvedAttribute("TbL.a")),
         SubqueryAlias("TbL", UnresolvedRelation(TableIdentifier("TaBlE")))),
       Project(testRelation.output, testRelation),
       caseSensitive = false)
 
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       Project(Seq(UnresolvedAttribute("tBl.a")),
         SubqueryAlias("TbL", UnresolvedRelation(TableIdentifier("TaBlE")))),
       Project(testRelation.output, testRelation),
@@ -180,15 +203,15 @@ class AnalysisSuite extends AnalysisTest with Matchers {
 
   test("resolve relations") {
     assertAnalysisError(UnresolvedRelation(TableIdentifier("tAbLe")), Seq())
-    checkAnalysis(UnresolvedRelation(TableIdentifier("TaBlE")), testRelation)
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(UnresolvedRelation(TableIdentifier("TaBlE")), testRelation)
+    checkAnalysisWithoutViewWrapper(
       UnresolvedRelation(TableIdentifier("tAbLe")), testRelation, caseSensitive = false)
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       UnresolvedRelation(TableIdentifier("TaBlE")), testRelation, caseSensitive = false)
   }
 
   test("divide should be casted into fractional types") {
-    val plan = caseInsensitiveAnalyzer.execute(
+    val plan = getAnalyzer.execute(
       testRelation2.select(
         $"a" / Literal(2) as "div1",
         $"a" / $"b" as "div2",
@@ -249,13 +272,13 @@ class AnalysisSuite extends AnalysisTest with Matchers {
       CreateStruct(Seq(att1, ((att1.as("aa")) + 1).as("a_plus_1"))).as("col"),
       att1
     )
-    val prevPlan = getAnalyzer(true).execute(plan)
+    val prevPlan = getAnalyzer.execute(plan)
     plan = prevPlan.select(CreateArray(Seq(
       CreateStruct(Seq(att1, (att1 + 1).as("a_plus_1"))).as("col1"),
       /** alias should be eliminated by [[CleanupAliases]] */
       "col".attr.as("col2")
     )).as("arr"))
-    plan = getAnalyzer(true).execute(plan)
+    plan = getAnalyzer.execute(plan)
 
     val expectedPlan = prevPlan.select(
       CreateArray(Seq(
@@ -307,6 +330,10 @@ class AnalysisSuite extends AnalysisTest with Matchers {
   }
 
   test("SPARK-11725: correctly handle null inputs for ScalaUDF") {
+    def resolvedEncoder[T : TypeTag](): ExpressionEncoder[T] = {
+      ExpressionEncoder[T]().resolveAndBind()
+    }
+
     val testRelation = LocalRelation(
       AttributeReference("a", StringType)(),
       AttributeReference("b", DoubleType)(),
@@ -328,20 +355,20 @@ class AnalysisSuite extends AnalysisTest with Matchers {
 
     // non-primitive parameters do not need special null handling
     val udf1 = ScalaUDF((s: String) => "x", StringType, string :: Nil,
-      Option(ExpressionEncoder[String]()) :: Nil)
+      Option(resolvedEncoder[String]()) :: Nil)
     val expected1 = udf1
     checkUDF(udf1, expected1)
 
     // only primitive parameter needs special null handling
     val udf2 = ScalaUDF((s: String, d: Double) => "x", StringType, string :: double :: Nil,
-      Option(ExpressionEncoder[String]()) :: Option(ExpressionEncoder[Double]()) :: Nil)
+      Option(resolvedEncoder[String]()) :: Option(resolvedEncoder[Double]()) :: Nil)
     val expected2 =
       If(IsNull(double), nullResult, udf2.copy(children = string :: KnownNotNull(double) :: Nil))
     checkUDF(udf2, expected2)
 
     // special null handling should apply to all primitive parameters
     val udf3 = ScalaUDF((s: Short, d: Double) => "x", StringType, short :: double :: Nil,
-      Option(ExpressionEncoder[Short]()) :: Option(ExpressionEncoder[Double]()) :: Nil)
+      Option(resolvedEncoder[Short]()) :: Option(resolvedEncoder[Double]()) :: Nil)
     val expected3 = If(
       IsNull(short) || IsNull(double),
       nullResult,
@@ -353,7 +380,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
       (s: Short, d: Double) => "x",
       StringType,
       short :: nonNullableDouble :: Nil,
-      Option(ExpressionEncoder[Short]()) :: Option(ExpressionEncoder[Double]()) :: Nil)
+      Option(resolvedEncoder[Short]()) :: Option(resolvedEncoder[Double]()) :: Nil)
     val expected4 = If(
       IsNull(short),
       nullResult,
@@ -398,7 +425,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
     checkAnalysis(plan, expected)
   }
 
-  test("SPARK-12102: Ignore nullablity when comparing two sides of case") {
+  test("SPARK-12102: Ignore nullability when comparing two sides of case") {
     val relation = LocalRelation(Symbol("a").struct(Symbol("x").int),
       Symbol("b").struct(Symbol("x").int.withNullability(false)))
     val plan = relation.select(
@@ -602,6 +629,61 @@ class AnalysisSuite extends AnalysisTest with Matchers {
       Project(Seq(UnresolvedAttribute("temp0.a"), UnresolvedAttribute("temp1.a")), join))
   }
 
+  test("SPARK-34319: analysis fails on self-join with FlatMapCoGroupsInPandas") {
+    val pythonUdf = PythonUDF("pyUDF", null,
+      StructType(Seq(StructField("a", LongType))),
+      Seq.empty,
+      PythonEvalType.SQL_COGROUPED_MAP_PANDAS_UDF,
+      true)
+    val output = pythonUdf.dataType.asInstanceOf[StructType].toAttributes
+    val project1 = Project(Seq(UnresolvedAttribute("a")), testRelation)
+    val project2 = Project(Seq(UnresolvedAttribute("a")), testRelation2)
+    val flatMapGroupsInPandas = FlatMapCoGroupsInPandas(
+      1,
+      1,
+      pythonUdf,
+      output,
+      project1,
+      project2)
+    val left = SubqueryAlias("temp0", flatMapGroupsInPandas)
+    val right = SubqueryAlias("temp1", flatMapGroupsInPandas)
+    val join = Join(left, right, Inner, None, JoinHint.NONE)
+    assertAnalysisSuccess(
+      Project(Seq(UnresolvedAttribute("temp0.a"), UnresolvedAttribute("temp1.a")), join))
+  }
+
+  test("SPARK-34319: analysis fails on self-join with MapInPandas") {
+    val pythonUdf = PythonUDF("pyUDF", null,
+      StructType(Seq(StructField("a", LongType))),
+      Seq.empty,
+      PythonEvalType.SQL_MAP_PANDAS_ITER_UDF,
+      true)
+    val output = pythonUdf.dataType.asInstanceOf[StructType].toAttributes
+    val project = Project(Seq(UnresolvedAttribute("a")), testRelation)
+    val mapInPandas = MapInPandas(
+      pythonUdf,
+      output,
+      project)
+    val left = SubqueryAlias("temp0", mapInPandas)
+    val right = SubqueryAlias("temp1", mapInPandas)
+    val join = Join(left, right, Inner, None, JoinHint.NONE)
+    assertAnalysisSuccess(
+      Project(Seq(UnresolvedAttribute("temp0.a"), UnresolvedAttribute("temp1.a")), join))
+  }
+
+  test("SPARK-34741: Avoid ambiguous reference in MergeIntoTable") {
+    val cond = 'a > 1
+    assertAnalysisError(
+      MergeIntoTable(
+        testRelation,
+        testRelation,
+        cond,
+        UpdateAction(Some(cond), Assignment('a, 'a) :: Nil) :: Nil,
+        Nil
+      ),
+      "Reference 'a' is ambiguous" :: Nil)
+  }
+
   test("SPARK-24488 Generator with multiple aliases") {
     assertAnalysisSuccess(
       listRelation.select(Explode($"list").as("first_alias").as("second_alias")))
@@ -626,30 +708,9 @@ class AnalysisSuite extends AnalysisTest with Matchers {
     }
   }
 
-  test("SPARK-25691: AliasViewChild with different nullabilities") {
-    object ViewAnalyzer extends RuleExecutor[LogicalPlan] {
-      val batches = Batch("View", Once, EliminateView) :: Nil
-    }
-    val relation = LocalRelation(Symbol("a").int.notNull, Symbol("b").string)
-    val view = View(CatalogTable(
-        identifier = TableIdentifier("v1"),
-        tableType = CatalogTableType.VIEW,
-        storage = CatalogStorageFormat.empty,
-        schema = StructType(Seq(StructField("a", IntegerType), StructField("b", StringType)))),
-      output = Seq(Symbol("a").int, Symbol("b").string),
-      child = relation)
-    val tz = Option(conf.sessionLocalTimeZone)
-    val expected = Project(Seq(
-        Alias(Cast(Symbol("a").int.notNull, IntegerType, tz), "a")(),
-        Alias(Cast(Symbol("b").string, StringType, tz), "b")()),
-      relation)
-    val res = ViewAnalyzer.execute(view)
-    comparePlans(res, expected)
-  }
-
   test("CTE with non-existing column alias") {
     assertAnalysisError(parsePlan("WITH t(x) AS (SELECT 1) SELECT * FROM t WHERE y = 1"),
-      Seq("cannot resolve '`y`' given input columns: [x]"))
+      Seq("cannot resolve 'y' given input columns: [t.x]"))
   }
 
   test("CTE with non-matching column alias") {
@@ -758,22 +819,23 @@ class AnalysisSuite extends AnalysisTest with Matchers {
     // RuleExecutor only throw exception or log warning when the rule is supposed to run
     // more than once.
     val maxIterations = 2
-    val conf = new SQLConf().copy(SQLConf.ANALYZER_MAX_ITERATIONS -> maxIterations)
-    val testAnalyzer = new Analyzer(
-      new SessionCatalog(new InMemoryCatalog, FunctionRegistry.builtin, conf), conf)
+    withSQLConf(SQLConf.ANALYZER_MAX_ITERATIONS.key -> maxIterations.toString) {
+      val testAnalyzer = new Analyzer(
+        new SessionCatalog(new InMemoryCatalog, FunctionRegistry.builtin))
 
-    val plan = testRelation2.select(
-      $"a" / Literal(2) as "div1",
-      $"a" / $"b" as "div2",
-      $"a" / $"c" as "div3",
-      $"a" / $"d" as "div4",
-      $"e" / $"e" as "div5")
+      val plan = testRelation2.select(
+        $"a" / Literal(2) as "div1",
+        $"a" / $"b" as "div2",
+        $"a" / $"c" as "div3",
+        $"a" / $"d" as "div4",
+        $"e" / $"e" as "div5")
 
-    val message = intercept[TreeNodeException[LogicalPlan]] {
-      testAnalyzer.execute(plan)
-    }.getMessage
-    assert(message.startsWith(s"Max iterations ($maxIterations) reached for batch Resolution, " +
-      s"please set '${SQLConf.ANALYZER_MAX_ITERATIONS.key}' to a larger value."))
+      val message = intercept[RuntimeException] {
+        testAnalyzer.execute(plan)
+      }.getMessage
+      assert(message.startsWith(s"Max iterations ($maxIterations) reached for batch Resolution, " +
+        s"please set '${SQLConf.ANALYZER_MAX_ITERATIONS.key}' to a larger value."))
+    }
   }
 
   test("SPARK-30886 Deprecate two-parameter TRIM/LTRIM/RTRIM") {
@@ -789,7 +851,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
 
       withLogAppender(logAppender) {
         val testAnalyzer1 = new Analyzer(
-          new SessionCatalog(new InMemoryCatalog, FunctionRegistry.builtin, conf), conf)
+          new SessionCatalog(new InMemoryCatalog, FunctionRegistry.builtin))
 
         val plan1 = testRelation2.select(
           UnresolvedFunction(f, $"a" :: Nil, isDistinct = false))
@@ -811,7 +873,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
 
         // New analyzer from new SessionState
         val testAnalyzer2 = new Analyzer(
-          new SessionCatalog(new InMemoryCatalog, FunctionRegistry.builtin, conf), conf)
+          new SessionCatalog(new InMemoryCatalog, FunctionRegistry.builtin))
         val plan4 = testRelation2.select(
           UnresolvedFunction(f, $"c" :: $"d" :: Nil, isDistinct = false))
         testAnalyzer2.execute(plan4)
@@ -826,4 +888,233 @@ class AnalysisSuite extends AnalysisTest with Matchers {
       }
     }
   }
+
+  test("SPARK-32131: Fix wrong column index when we have more than two columns" +
+    " during union and set operations" ) {
+    val firstTable = LocalRelation(
+      AttributeReference("a", StringType)(),
+      AttributeReference("b", DoubleType)(),
+      AttributeReference("c", IntegerType)(),
+      AttributeReference("d", FloatType)())
+
+    val secondTable = LocalRelation(
+      AttributeReference("a", StringType)(),
+      AttributeReference("b", TimestampType)(),
+      AttributeReference("c", IntegerType)(),
+      AttributeReference("d", FloatType)())
+
+    val thirdTable = LocalRelation(
+      AttributeReference("a", StringType)(),
+      AttributeReference("b", DoubleType)(),
+      AttributeReference("c", TimestampType)(),
+      AttributeReference("d", FloatType)())
+
+    val fourthTable = LocalRelation(
+      AttributeReference("a", StringType)(),
+      AttributeReference("b", DoubleType)(),
+      AttributeReference("c", IntegerType)(),
+      AttributeReference("d", TimestampType)())
+
+    val r1 = Union(firstTable, secondTable)
+    val r2 = Union(firstTable, thirdTable)
+    val r3 = Union(firstTable, fourthTable)
+    val r4 = Except(firstTable, secondTable, isAll = false)
+    val r5 = Intersect(firstTable, secondTable, isAll = false)
+
+    assertAnalysisError(r1,
+      Seq("Union can only be performed on tables with the compatible column types. " +
+        "timestamp <> double at the second column of the second table"))
+
+    assertAnalysisError(r2,
+      Seq("Union can only be performed on tables with the compatible column types. " +
+        "timestamp <> int at the third column of the second table"))
+
+    assertAnalysisError(r3,
+      Seq("Union can only be performed on tables with the compatible column types. " +
+        "timestamp <> float at the 4th column of the second table"))
+
+    assertAnalysisError(r4,
+      Seq("Except can only be performed on tables with the compatible column types. " +
+        "timestamp <> double at the second column of the second table"))
+
+    assertAnalysisError(r5,
+      Seq("Intersect can only be performed on tables with the compatible column types. " +
+        "timestamp <> double at the second column of the second table"))
+  }
+
+  test("SPARK-31975: Throw user facing error when use WindowFunction directly") {
+    assertAnalysisError(testRelation2.select(RowNumber()),
+      Seq("Window function row_number() requires an OVER clause."))
+
+    assertAnalysisError(testRelation2.select(Sum(RowNumber())),
+      Seq("Window function row_number() requires an OVER clause."))
+
+    assertAnalysisError(testRelation2.select(RowNumber() + 1),
+      Seq("Window function row_number() requires an OVER clause."))
+  }
+
+  test("SPARK-32237: Hint in CTE") {
+    val plan = With(
+      Project(
+        Seq(UnresolvedAttribute("cte.a")),
+        UnresolvedRelation(TableIdentifier("cte"))
+      ),
+      Seq(
+        (
+          "cte",
+          SubqueryAlias(
+            AliasIdentifier("cte"),
+            UnresolvedHint(
+              "REPARTITION",
+              Seq(Literal(3)),
+              Project(testRelation.output, testRelation)
+            )
+          )
+        )
+      )
+    )
+    assertAnalysisSuccess(plan)
+  }
+
+  test("SPARK-33197: Make sure changes to ANALYZER_MAX_ITERATIONS take effect at runtime") {
+    // RuleExecutor only throw exception or log warning when the rule is supposed to run
+    // more than once.
+    val maxIterations = 2
+    val maxIterationsEnough = 5
+    withSQLConf(SQLConf.ANALYZER_MAX_ITERATIONS.key -> maxIterations.toString) {
+      val testAnalyzer = new Analyzer(
+        new SessionCatalog(new InMemoryCatalog, FunctionRegistry.builtin))
+
+      val plan = testRelation2.select(
+        $"a" / Literal(2) as "div1",
+        $"a" / $"b" as "div2",
+        $"a" / $"c" as "div3",
+        $"a" / $"d" as "div4",
+        $"e" / $"e" as "div5")
+
+      val message1 = intercept[RuntimeException] {
+        testAnalyzer.execute(plan)
+      }.getMessage
+      assert(message1.startsWith(s"Max iterations ($maxIterations) reached for batch Resolution, " +
+        s"please set '${SQLConf.ANALYZER_MAX_ITERATIONS.key}' to a larger value."))
+
+      withSQLConf(SQLConf.ANALYZER_MAX_ITERATIONS.key -> maxIterationsEnough.toString) {
+        try {
+          testAnalyzer.execute(plan)
+        } catch {
+          case ex: AnalysisException
+            if ex.getMessage.contains(SQLConf.ANALYZER_MAX_ITERATIONS.key) =>
+              fail("analyzer.execute should not reach max iterations.")
+        }
+      }
+
+      val message2 = intercept[RuntimeException] {
+        testAnalyzer.execute(plan)
+      }.getMessage
+      assert(message2.startsWith(s"Max iterations ($maxIterations) reached for batch Resolution, " +
+        s"please set '${SQLConf.ANALYZER_MAX_ITERATIONS.key}' to a larger value."))
+    }
+  }
+
+  test("SPARK-33733: PullOutNondeterministic should check and collect deterministic field") {
+    val reflect =
+      CallMethodViaReflection(Seq("java.lang.Math", "abs", testRelation.output.head))
+    val udf = ScalaUDF(
+      (s: String) => s,
+      StringType,
+      Literal.create(null, StringType) :: Nil,
+      Option(ExpressionEncoder[String]().resolveAndBind()) :: Nil,
+      udfDeterministic = false)
+
+    Seq(reflect, udf).foreach { e: Expression =>
+      val plan = Sort(Seq(e.asc), false, testRelation)
+      val projected = Alias(e, "_nondeterministic")()
+      val expect =
+        Project(testRelation.output,
+          Sort(Seq(projected.toAttribute.asc), false,
+            Project(testRelation.output :+ projected,
+              testRelation)))
+      checkAnalysis(plan, expect)
+    }
+  }
+
+  test("SPARK-33857: Unify the default seed of random functions") {
+    Seq(new Rand(), new Randn(), Shuffle(Literal(Array(1))), Uuid()).foreach { r =>
+      assert(r.seedExpression == UnresolvedSeed)
+      val p = getAnalyzer.execute(Project(Seq(r.as("r")), testRelation))
+      assert(
+        p.asInstanceOf[Project].projectList.head.asInstanceOf[Alias]
+          .child.asInstanceOf[ExpressionWithRandomSeed]
+          .seedExpression.isInstanceOf[Literal]
+      )
+    }
+  }
+
+  test("SPARK-22748: Analyze __grouping__id as a literal function") {
+    assertAnalysisSuccess(parsePlan(
+      """
+        |SELECT grouping__id FROM (
+        |  SELECT grouping__id FROM (
+        |    SELECT a, b, count(1), grouping__id FROM TaBlE2
+        |      GROUP BY a, b WITH ROLLUP
+        |  )
+        |)
+      """.stripMargin), false)
+
+
+    assertAnalysisSuccess(parsePlan(
+      """
+        |SELECT grouping__id FROM (
+        |  SELECT a, b, count(1), grouping__id FROM TaBlE2
+        |   GROUP BY a, b WITH CUBE
+        |)
+      """.stripMargin), false)
+
+    assertAnalysisSuccess(parsePlan(
+      """
+        |SELECT grouping__id FROM (
+        |  SELECT a, b, count(1), grouping__id FROM TaBlE2
+        |    GROUP BY a, b GROUPING SETS ((a, b), ())
+        |)
+      """.stripMargin), false)
+
+    assertAnalysisSuccess(parsePlan(
+      """
+        |SELECT a, b, count(1) FROM TaBlE2
+        |  GROUP BY CUBE(a, b) HAVING grouping__id > 0
+      """.stripMargin), false)
+
+    assertAnalysisSuccess(parsePlan(
+      """
+        |SELECT * FROM (
+        |  SELECT a, b, count(1), grouping__id FROM TaBlE2
+        |    GROUP BY a, b GROUPING SETS ((a, b), ())
+        |) WHERE grouping__id > 0
+      """.stripMargin), false)
+
+    assertAnalysisSuccess(parsePlan(
+      """
+        |SELECT * FROM (
+        |  SELECT a, b, count(1), grouping__id FROM TaBlE2
+        |    GROUP BY a, b GROUPING SETS ((a, b), ())
+        |) ORDER BY grouping__id > 0
+      """.stripMargin), false)
+
+    assertAnalysisSuccess(parsePlan(
+      """
+        |SELECT a, b, count(1) FROM TaBlE2
+        |  GROUP BY a, b GROUPING SETS ((a, b), ())
+        |    ORDER BY grouping__id > 0
+      """.stripMargin), false)
+
+    assertAnalysisError(parsePlan(
+      """
+        |SELECT grouping__id FROM (
+        |  SELECT a, b, count(1), grouping__id FROM TaBlE2
+        |    GROUP BY a, b
+        |)
+      """.stripMargin),
+      Seq("grouping_id() can only be used with GroupingSets/Cube/Rollup"),
+      false)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala
index 4473c20b2cca6..8332668a754e1 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala
@@ -21,48 +21,92 @@ import java.net.URI
 import java.util.Locale
 
 import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.QueryPlanningTracker
-import org.apache.spark.sql.catalyst.catalog.{CatalogDatabase, InMemoryCatalog, SessionCatalog}
+import org.apache.spark.sql.catalyst.{QueryPlanningTracker, TableIdentifier}
+import org.apache.spark.sql.catalyst.catalog.{CatalogDatabase, CatalogStorageFormat, CatalogTable, CatalogTableType, InMemoryCatalog, SessionCatalog, TemporaryViewRelation}
+import org.apache.spark.sql.catalyst.catalog.CatalogTable.VIEW_STORING_ANALYZED_PLAN
 import org.apache.spark.sql.catalyst.parser.ParseException
 import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
+import org.apache.spark.sql.types.StructType
 
 trait AnalysisTest extends PlanTest {
 
-  protected lazy val caseSensitiveAnalyzer = makeAnalyzer(caseSensitive = true)
-  protected lazy val caseInsensitiveAnalyzer = makeAnalyzer(caseSensitive = false)
-
   protected def extendedAnalysisRules: Seq[Rule[LogicalPlan]] = Nil
 
-  private def makeAnalyzer(caseSensitive: Boolean): Analyzer = {
-    val conf = new SQLConf().copy(SQLConf.CASE_SENSITIVE -> caseSensitive)
-    val catalog = new SessionCatalog(new InMemoryCatalog, FunctionRegistry.builtin, conf)
+  protected def createTempView(
+      catalog: SessionCatalog,
+      name: String,
+      plan: LogicalPlan,
+      overrideIfExists: Boolean): Unit = {
+    val identifier = TableIdentifier(name)
+    val metadata = createTempViewMetadata(identifier, plan.schema)
+    val viewDefinition = TemporaryViewRelation(metadata, Some(plan))
+    catalog.createTempView(name, viewDefinition, overrideIfExists)
+  }
+
+  protected def createGlobalTempView(
+      catalog: SessionCatalog,
+      name: String,
+      plan: LogicalPlan,
+      overrideIfExists: Boolean): Unit = {
+    val globalDb = Some(SQLConf.get.getConf(StaticSQLConf.GLOBAL_TEMP_DATABASE))
+    val identifier = TableIdentifier(name, globalDb)
+    val metadata = createTempViewMetadata(identifier, plan.schema)
+    val viewDefinition = TemporaryViewRelation(metadata, Some(plan))
+    catalog.createGlobalTempView(name, viewDefinition, overrideIfExists)
+  }
+
+  private def createTempViewMetadata(
+      identifier: TableIdentifier,
+      schema: StructType): CatalogTable = {
+    CatalogTable(
+      identifier = identifier,
+      tableType = CatalogTableType.VIEW,
+      storage = CatalogStorageFormat.empty,
+      schema = schema,
+      properties = Map((VIEW_STORING_ANALYZED_PLAN, "true")))
+  }
+
+  protected def getAnalyzer: Analyzer = {
+    val catalog = new SessionCatalog(
+      new InMemoryCatalog, FunctionRegistry.builtin, TableFunctionRegistry.builtin)
     catalog.createDatabase(
       CatalogDatabase("default", "", new URI("loc"), Map.empty),
       ignoreIfExists = false)
-    catalog.createTempView("TaBlE", TestRelations.testRelation, overrideIfExists = true)
-    catalog.createTempView("TaBlE2", TestRelations.testRelation2, overrideIfExists = true)
-    catalog.createTempView("TaBlE3", TestRelations.testRelation3, overrideIfExists = true)
-    catalog.createGlobalTempView("TaBlE4", TestRelations.testRelation4, overrideIfExists = true)
-    catalog.createGlobalTempView("TaBlE5", TestRelations.testRelation5, overrideIfExists = true)
-    new Analyzer(catalog, conf) {
-      override val extendedResolutionRules = EliminateSubqueryAliases +: extendedAnalysisRules
+    createTempView(catalog, "TaBlE", TestRelations.testRelation, overrideIfExists = true)
+    createTempView(catalog, "TaBlE2", TestRelations.testRelation2, overrideIfExists = true)
+    createTempView(catalog, "TaBlE3", TestRelations.testRelation3, overrideIfExists = true)
+    createGlobalTempView(catalog, "TaBlE4", TestRelations.testRelation4, overrideIfExists = true)
+    createGlobalTempView(catalog, "TaBlE5", TestRelations.testRelation5, overrideIfExists = true)
+    new Analyzer(catalog) {
+      override val extendedResolutionRules = extendedAnalysisRules
     }
   }
 
-  protected def getAnalyzer(caseSensitive: Boolean) = {
-    if (caseSensitive) caseSensitiveAnalyzer else caseInsensitiveAnalyzer
+  protected def checkAnalysis(
+      inputPlan: LogicalPlan,
+      expectedPlan: LogicalPlan,
+      caseSensitive: Boolean = true): Unit = {
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
+      val analyzer = getAnalyzer
+      val actualPlan = analyzer.executeAndCheck(inputPlan, new QueryPlanningTracker)
+      comparePlans(EliminateSubqueryAliases(actualPlan), expectedPlan)
+    }
   }
 
-  protected def checkAnalysis(
+  protected def checkAnalysisWithoutViewWrapper(
       inputPlan: LogicalPlan,
       expectedPlan: LogicalPlan,
       caseSensitive: Boolean = true): Unit = {
-    val analyzer = getAnalyzer(caseSensitive)
-    val actualPlan = analyzer.executeAndCheck(inputPlan, new QueryPlanningTracker)
-    comparePlans(actualPlan, expectedPlan)
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
+      val actualPlan = getAnalyzer.executeAndCheck(inputPlan, new QueryPlanningTracker)
+      val transformed = EliminateSubqueryAliases(actualPlan) transformUp {
+        case v: View if v.isTempViewStoringAnalyzedPlan => v.child
+      }
+      comparePlans(transformed, expectedPlan)
+    }
   }
 
   protected override def comparePlans(
@@ -76,18 +120,20 @@ trait AnalysisTest extends PlanTest {
   protected def assertAnalysisSuccess(
       inputPlan: LogicalPlan,
       caseSensitive: Boolean = true): Unit = {
-    val analyzer = getAnalyzer(caseSensitive)
-    val analysisAttempt = analyzer.execute(inputPlan)
-    try analyzer.checkAnalysis(analysisAttempt) catch {
-      case a: AnalysisException =>
-        fail(
-          s"""
-            |Failed to Analyze Plan
-            |$inputPlan
-            |
-            |Partial Analysis
-            |$analysisAttempt
-          """.stripMargin, a)
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
+      val analyzer = getAnalyzer
+      val analysisAttempt = analyzer.execute(inputPlan)
+      try analyzer.checkAnalysis(analysisAttempt) catch {
+        case a: AnalysisException =>
+          fail(
+            s"""
+              |Failed to Analyze Plan
+              |$inputPlan
+              |
+              |Partial Analysis
+              |$analysisAttempt
+            """.stripMargin, a)
+      }
     }
   }
 
@@ -95,22 +141,24 @@ trait AnalysisTest extends PlanTest {
       inputPlan: LogicalPlan,
       expectedErrors: Seq[String],
       caseSensitive: Boolean = true): Unit = {
-    val analyzer = getAnalyzer(caseSensitive)
-    val e = intercept[AnalysisException] {
-      analyzer.checkAnalysis(analyzer.execute(inputPlan))
-    }
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
+      val analyzer = getAnalyzer
+      val e = intercept[AnalysisException] {
+        analyzer.checkAnalysis(analyzer.execute(inputPlan))
+      }
 
-    if (!expectedErrors.map(_.toLowerCase(Locale.ROOT)).forall(
-        e.getMessage.toLowerCase(Locale.ROOT).contains)) {
-      fail(
-        s"""Exception message should contain the following substrings:
-           |
-           |  ${expectedErrors.mkString("\n  ")}
-           |
-           |Actual exception message:
-           |
-           |  ${e.getMessage}
-         """.stripMargin)
+      if (!expectedErrors.map(_.toLowerCase(Locale.ROOT)).forall(
+          e.getMessage.toLowerCase(Locale.ROOT).contains)) {
+        fail(
+          s"""Exception message should contain the following substrings:
+             |
+             |  ${expectedErrors.mkString("\n  ")}
+             |
+             |Actual exception message:
+             |
+             |  ${e.getMessage}
+           """.stripMargin)
+      }
     }
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercionSuite.scala
new file mode 100644
index 0000000000000..e3e61f022c7b3
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercionSuite.scala
@@ -0,0 +1,1421 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import java.sql.Timestamp
+
+import org.apache.spark.sql.catalyst.analysis.AnsiTypeCoercion._
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.{Rule, RuleExecutor}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types._
+
+class AnsiTypeCoercionSuite extends AnalysisTest {
+  import TypeCoercionSuite._
+
+  // scalastyle:off line.size.limit
+  // The following table shows all implicit data type conversions that are not visible to the user.
+  // +----------------------+----------+-----------+-------------+----------+------------+------------+------------+------------+-------------+------------+----------+---------------+------------+----------+-------------+----------+----------------------+---------------------+-------------+--------------+
+  // | Source Type\CAST TO  | ByteType | ShortType | IntegerType | LongType | FloatType  | DoubleType | Dec(10, 2) | BinaryType | BooleanType | StringType | DateType | TimestampType | ArrayType  | MapType  | StructType  | NullType | CalendarIntervalType |     DecimalType     | NumericType | IntegralType |
+  // +----------------------+----------+-----------+-------------+----------+------------+------------+------------+------------+-------------+------------+----------+---------------+------------+----------+-------------+----------+----------------------+---------------------+-------------+--------------+
+  // | ByteType             | ByteType | ShortType | IntegerType | LongType | DoubleType | DoubleType | Dec(10, 2) | X          | X           | X          | X        | X             | X          | X        | X           | X        | X                    | DecimalType(3, 0)   | ByteType    | ByteType     |
+  // | ShortType            | X        | ShortType | IntegerType | LongType | DoubleType | DoubleType | Dec(10, 2) | X          | X           | X          | X        | X             | X          | X        | X           | X        | X                    | DecimalType(5, 0)   | ShortType   | ShortType    |
+  // | IntegerType          | X        | X         | IntegerType | LongType | DoubleType | DoubleType | X          | X          | X           | X          | X        | X             | X          | X        | X           | X        | X                    | DecimalType(10, 0)  | IntegerType | IntegerType  |
+  // | LongType             | X        | X         | X           | LongType | DoubleType | DoubleType | X          | X          | X           | X          | X        | X             | X          | X        | X           | X        | X                    | DecimalType(20, 0)  | LongType    | LongType     |
+  // | FloatType            | X        | X         | X           | X        | FloatType  | DoubleType | X          | X          | X           | X          | X        | X             | X          | X        | X           | X        | X                    | DecimalType(30, 15) | DoubleType  | X            |
+  // | DoubleType           | X        | X         | X           | X        | X          | DoubleType | X          | X          | X           | X          | X        | X             | X          | X        | X           | X        | X                    | DecimalType(14, 7)  | FloatType   | X            |
+  // | Dec(10, 2)           | X        | X         | X           | X        | DoubleType | DoubleType | Dec(10, 2) | X          | X           | X          | X        | X             | X          | X        | X           | X        | X                    | DecimalType(10, 2)  | Dec(10, 2)  | X            |
+  // | BinaryType           | X        | X         | X           | X        | X          | X          | X          | BinaryType | X           | X          | X        | X             | X          | X        | X           | X        | X                    | X                   | X           | X            |
+  // | BooleanType          | X        | X         | X           | X        | X          | X          | X          | X          | BooleanType | X          | X        | X             | X          | X        | X           | X        | X                    | X                   | X           | X            |
+  // | StringType           | X        | X         | X           | X        | X          | X          | X          | X          | X           | X          | X        | X             | X          | X        | X           | X        | X                    | X                   | X           | X            |
+  // | DateType             | X        | X         | X           | X        | X          | X          | X          | X          | X           | X          | DateType | TimestampType | X          | X        | X           | X        | X                    | X                   | X           | X            |
+  // | TimestampType        | X        | X         | X           | X        | X          | X          | X          | X          | X           | X          | X        | TimestampType | X          | X        | X           | X        | X                    | X                   | X           | X            |
+  // | ArrayType            | X        | X         | X           | X        | X          | X          | X          | X          | X           | X          | X        | X             | ArrayType* | X        | X           | X        | X                    | X                   | X           | X            |
+  // | MapType              | X        | X         | X           | X        | X          | X          | X          | X          | X           | X          | X        | X             | X          | MapType* | X           | X        | X                    | X                   | X           | X            |
+  // | StructType           | X        | X         | X           | X        | X          | X          | X          | X          | X           | X          | X        | X             | X          | X        | StructType* | X        | X                    | X                   | X           | X            |
+  // | NullType             | ByteType | ShortType | IntegerType | LongType | DoubleType | FloatType  | Dec(10, 2) | BinaryType | BooleanType | StringType | DateType | TimestampType | ArrayType  | MapType  | StructType  | NullType | CalendarIntervalType | DecimalType(38, 18) | DoubleType  | IntegerType  |
+  // | CalendarIntervalType | X        | X         | X           | X        | X          | X          | X          | X          | X           | X          | X        | X             | X          | X        | X           | X        | CalendarIntervalType | X                   | X           | X            |
+  // +----------------------+----------+-----------+-------------+----------+------------+------------+------------+------------+-------------+------------+----------+---------------+------------+----------+-------------+----------+----------------------+---------------------+-------------+--------------+
+  // Note: StructType* is castable when all the internal child types are castable according to the table.
+  // Note: ArrayType* is castable when the element type is castable according to the table.
+  // Note: MapType* is castable when both the key type and the value type are castable according to the table.
+  // scalastyle:on line.size.limit
+
+  private def shouldCast(from: DataType, to: AbstractDataType, expected: DataType): Unit = {
+    // Check default value
+    val castDefault = AnsiTypeCoercion.implicitCast(default(from), to)
+    assert(DataType.equalsIgnoreCompatibleNullability(
+      castDefault.map(_.dataType).getOrElse(null), expected),
+      s"Failed to cast $from to $to")
+
+    // Check null value
+    val castNull = AnsiTypeCoercion.implicitCast(createNull(from), to)
+    assert(DataType.equalsIgnoreCaseAndNullability(
+      castNull.map(_.dataType).getOrElse(null), expected),
+      s"Failed to cast $from to $to")
+  }
+
+  private def shouldNotCast(from: DataType, to: AbstractDataType): Unit = {
+    // Check default value
+    val castDefault = AnsiTypeCoercion.implicitCast(default(from), to)
+    assert(castDefault.isEmpty, s"Should not be able to cast $from to $to, but got $castDefault")
+
+    // Check null value
+    val castNull = AnsiTypeCoercion.implicitCast(createNull(from), to)
+    assert(castNull.isEmpty, s"Should not be able to cast $from to $to, but got $castNull")
+  }
+
+  private def shouldCastStringLiteral(to: AbstractDataType, expected: DataType): Unit = {
+    val input = Literal("123")
+    val castResult = AnsiTypeCoercion.implicitCast(input, to)
+    assert(DataType.equalsIgnoreCaseAndNullability(
+      castResult.map(_.dataType).getOrElse(null), expected),
+      s"Failed to cast String literal to $to")
+  }
+
+  private def shouldNotCastStringLiteral(to: AbstractDataType): Unit = {
+    val input = Literal("123")
+    val castResult = AnsiTypeCoercion.implicitCast(input, to)
+    assert(castResult.isEmpty, s"Should not be able to cast String literal to $to")
+  }
+
+  private def shouldNotCastStringInput(to: AbstractDataType): Unit = {
+    val input = AttributeReference("s", StringType)()
+    val castResult = AnsiTypeCoercion.implicitCast(input, to)
+    assert(castResult.isEmpty, s"Should not be able to cast non-foldable String input to $to")
+  }
+
+  private def default(dataType: DataType): Expression = dataType match {
+    case ArrayType(internalType: DataType, _) =>
+      CreateArray(Seq(Literal.default(internalType)))
+    case MapType(keyDataType: DataType, valueDataType: DataType, _) =>
+      CreateMap(Seq(Literal.default(keyDataType), Literal.default(valueDataType)))
+    case _ => Literal.default(dataType)
+  }
+
+  private def createNull(dataType: DataType): Expression = dataType match {
+    case ArrayType(internalType: DataType, _) =>
+      CreateArray(Seq(Literal.create(null, internalType)))
+    case MapType(keyDataType: DataType, valueDataType: DataType, _) =>
+      CreateMap(Seq(Literal.create(null, keyDataType), Literal.create(null, valueDataType)))
+    case _ => Literal.create(null, dataType)
+  }
+
+  // Check whether the type `checkedType` can be cast to all the types in `castableTypes`,
+  // but cannot be cast to the other types in `allTypes`.
+  private def checkTypeCasting(checkedType: DataType, castableTypes: Seq[DataType]): Unit = {
+    val nonCastableTypes = allTypes.filterNot(castableTypes.contains)
+
+    castableTypes.foreach { tpe =>
+      shouldCast(checkedType, tpe, tpe)
+    }
+    nonCastableTypes.foreach { tpe =>
+      shouldNotCast(checkedType, tpe)
+    }
+  }
+
+  private def checkWidenType(
+      widenFunc: (DataType, DataType) => Option[DataType],
+      t1: DataType,
+      t2: DataType,
+      expected: Option[DataType],
+      isSymmetric: Boolean = true): Unit = {
+    var found = widenFunc(t1, t2)
+    assert(found == expected,
+      s"Expected $expected as wider common type for $t1 and $t2, found $found")
+    // Test both directions to make sure the widening is symmetric.
+    if (isSymmetric) {
+      found = widenFunc(t2, t1)
+      assert(found == expected,
+        s"Expected $expected as wider common type for $t2 and $t1, found $found")
+    }
+  }
+
+  test("implicit type cast - ByteType") {
+    val checkedType = ByteType
+    checkTypeCasting(checkedType, castableTypes = numericTypes)
+    shouldCast(checkedType, DecimalType, DecimalType.ByteDecimal)
+    shouldCast(checkedType, NumericType, checkedType)
+    shouldCast(checkedType, IntegralType, checkedType)
+  }
+
+  test("implicit type cast - ShortType") {
+    val checkedType = ShortType
+    checkTypeCasting(checkedType, castableTypes = numericTypes.filterNot(_ == ByteType))
+    shouldCast(checkedType, DecimalType, DecimalType.ShortDecimal)
+    shouldCast(checkedType, NumericType, checkedType)
+    shouldCast(checkedType, IntegralType, checkedType)
+  }
+
+  test("implicit type cast - IntegerType") {
+    val checkedType = IntegerType
+    checkTypeCasting(checkedType, castableTypes =
+      Seq(IntegerType, LongType, FloatType, DoubleType, DecimalType.SYSTEM_DEFAULT))
+    shouldCast(IntegerType, DecimalType, DecimalType.IntDecimal)
+    shouldCast(checkedType, NumericType, checkedType)
+    shouldCast(checkedType, IntegralType, checkedType)
+  }
+
+  test("implicit type cast - LongType") {
+    val checkedType = LongType
+    checkTypeCasting(checkedType, castableTypes =
+      Seq(LongType, FloatType, DoubleType, DecimalType.SYSTEM_DEFAULT))
+    shouldCast(checkedType, DecimalType, DecimalType.LongDecimal)
+    shouldCast(checkedType, NumericType, checkedType)
+    shouldCast(checkedType, IntegralType, checkedType)
+  }
+
+  test("implicit type cast - FloatType") {
+    val checkedType = FloatType
+    checkTypeCasting(checkedType, castableTypes = Seq(FloatType, DoubleType))
+    shouldCast(checkedType, DecimalType, DecimalType.FloatDecimal)
+    shouldCast(checkedType, NumericType, checkedType)
+    shouldNotCast(checkedType, IntegralType)
+  }
+
+  test("implicit type cast - DoubleType") {
+    val checkedType = DoubleType
+    checkTypeCasting(checkedType, castableTypes = Seq(DoubleType))
+    shouldCast(checkedType, DecimalType, DecimalType.DoubleDecimal)
+    shouldCast(checkedType, NumericType, checkedType)
+    shouldNotCast(checkedType, IntegralType)
+  }
+
+  test("implicit type cast - DecimalType(10, 2)") {
+    val checkedType = DecimalType(10, 2)
+    checkTypeCasting(checkedType, castableTypes = fractionalTypes)
+    shouldCast(checkedType, DecimalType, checkedType)
+    shouldCast(checkedType, NumericType, checkedType)
+    shouldNotCast(checkedType, IntegralType)
+  }
+
+  test("implicit type cast - BinaryType") {
+    val checkedType = BinaryType
+    checkTypeCasting(checkedType, castableTypes = Seq(checkedType))
+    shouldNotCast(checkedType, DecimalType)
+    shouldNotCast(checkedType, NumericType)
+    shouldNotCast(checkedType, IntegralType)
+  }
+
+  test("implicit type cast - BooleanType") {
+    val checkedType = BooleanType
+    checkTypeCasting(checkedType, castableTypes = Seq(checkedType))
+    shouldNotCast(checkedType, DecimalType)
+    shouldNotCast(checkedType, NumericType)
+    shouldNotCast(checkedType, IntegralType)
+    shouldNotCast(checkedType, StringType)
+  }
+
+  test("implicit type cast - unfoldable StringType") {
+    val nonCastableTypes = allTypes.filterNot(_ == StringType)
+    nonCastableTypes.foreach { dt =>
+      shouldNotCastStringInput(dt)
+    }
+    shouldNotCastStringInput(DecimalType)
+    shouldNotCastStringInput(NumericType)
+  }
+
+  test("implicit type cast - foldable StringType") {
+    atomicTypes.foreach { dt =>
+      shouldCastStringLiteral(dt, dt)
+    }
+    allTypes.filterNot(atomicTypes.contains).foreach { dt =>
+      shouldNotCastStringLiteral(dt)
+    }
+    shouldCastStringLiteral(DecimalType, DecimalType.defaultConcreteType)
+    shouldCastStringLiteral(NumericType, DoubleType)
+  }
+
+  test("implicit type cast - DateType") {
+    val checkedType = DateType
+    checkTypeCasting(checkedType, castableTypes = Seq(checkedType, TimestampType))
+    shouldNotCast(checkedType, DecimalType)
+    shouldNotCast(checkedType, NumericType)
+    shouldNotCast(checkedType, IntegralType)
+    shouldNotCast(checkedType, StringType)
+  }
+
+  test("implicit type cast - TimestampType") {
+    val checkedType = TimestampType
+    checkTypeCasting(checkedType, castableTypes = Seq(checkedType))
+    shouldNotCast(checkedType, DecimalType)
+    shouldNotCast(checkedType, NumericType)
+    shouldNotCast(checkedType, IntegralType)
+  }
+
+  test("implicit type cast - unfoldable ArrayType(StringType)") {
+    val input = AttributeReference("a", ArrayType(StringType))()
+    val nonCastableTypes = allTypes.filterNot(_ == StringType)
+    nonCastableTypes.map(ArrayType(_)).foreach { dt =>
+      assert(AnsiTypeCoercion.implicitCast(input, dt).isEmpty)
+    }
+    assert(AnsiTypeCoercion.implicitCast(input, DecimalType).isEmpty)
+    assert(AnsiTypeCoercion.implicitCast(input, NumericType).isEmpty)
+  }
+
+  test("implicit type cast - foldable arrayType(StringType)") {
+    val input = Literal(Array("1"))
+    assert(AnsiTypeCoercion.implicitCast(input, ArrayType(StringType)) == Some(input))
+    (numericTypes ++ datetimeTypes ++ Seq(BinaryType)).foreach { dt =>
+      assert(AnsiTypeCoercion.implicitCast(input, ArrayType(dt)) ==
+        Some(Cast(input, ArrayType(dt))))
+    }
+  }
+
+  test("implicit type cast between two Map types") {
+    val sourceType = MapType(IntegerType, IntegerType, true)
+    val castableTypes =
+      Seq(IntegerType, LongType, FloatType, DoubleType, DecimalType.SYSTEM_DEFAULT)
+    val targetTypes = castableTypes.map { t =>
+      MapType(t, sourceType.valueType, valueContainsNull = true)
+    }
+    val nonCastableTargetTypes = allTypes.filterNot(castableTypes.contains(_)).map {t =>
+      MapType(t, sourceType.valueType, valueContainsNull = true)
+    }
+
+    // Tests that its possible to setup implicit casts between two map types when
+    // source map's key type is integer and the target map's key type are either Byte, Short,
+    // Long, Double, Float, Decimal(38, 18) or String.
+    targetTypes.foreach { targetType =>
+      shouldCast(sourceType, targetType, targetType)
+    }
+
+    // Tests that its not possible to setup implicit casts between two map types when
+    // source map's key type is integer and the target map's key type are either Binary,
+    // Boolean, Date, Timestamp, Array, Struct, CalendarIntervalType or NullType
+    nonCastableTargetTypes.foreach { targetType =>
+      shouldNotCast(sourceType, targetType)
+    }
+
+    // Tests that its not possible to cast from nullable map type to not nullable map type.
+    val targetNotNullableTypes = allTypes.filterNot(_ == IntegerType).map { t =>
+      MapType(t, sourceType.valueType, valueContainsNull = false)
+    }
+    val sourceMapExprWithValueNull =
+      CreateMap(Seq(Literal.default(sourceType.keyType),
+        Literal.create(null, sourceType.valueType)))
+    targetNotNullableTypes.foreach { targetType =>
+      val castDefault =
+        AnsiTypeCoercion.implicitCast(sourceMapExprWithValueNull, targetType)
+      assert(castDefault.isEmpty,
+        s"Should not be able to cast $sourceType to $targetType, but got $castDefault")
+    }
+  }
+
+  test("implicit type cast - StructType().add(\"a1\", StringType)") {
+    val checkedType = new StructType().add("a1", StringType)
+    checkTypeCasting(checkedType, castableTypes = Seq(checkedType))
+    shouldNotCast(checkedType, DecimalType)
+    shouldNotCast(checkedType, NumericType)
+    shouldNotCast(checkedType, IntegralType)
+  }
+
+  test("implicit type cast - NullType") {
+    val checkedType = NullType
+    checkTypeCasting(checkedType, castableTypes = allTypes)
+    shouldCast(checkedType, DecimalType, DecimalType.SYSTEM_DEFAULT)
+    shouldCast(checkedType, NumericType, NumericType.defaultConcreteType)
+    shouldCast(checkedType, IntegralType, IntegralType.defaultConcreteType)
+  }
+
+  test("implicit type cast - CalendarIntervalType") {
+    val checkedType = CalendarIntervalType
+    checkTypeCasting(checkedType, castableTypes = Seq(checkedType))
+    shouldNotCast(checkedType, DecimalType)
+    shouldNotCast(checkedType, NumericType)
+    shouldNotCast(checkedType, IntegralType)
+  }
+
+  test("eligible implicit type cast - TypeCollection") {
+    shouldCast(StringType, TypeCollection(StringType, BinaryType), StringType)
+    shouldCast(BinaryType, TypeCollection(StringType, BinaryType), BinaryType)
+    shouldCast(StringType, TypeCollection(BinaryType, StringType), StringType)
+
+    shouldCast(IntegerType, TypeCollection(IntegerType, BinaryType), IntegerType)
+    shouldCast(IntegerType, TypeCollection(BinaryType, IntegerType), IntegerType)
+    shouldCast(BinaryType, TypeCollection(BinaryType, IntegerType), BinaryType)
+    shouldCast(BinaryType, TypeCollection(IntegerType, BinaryType), BinaryType)
+
+    shouldCast(DecimalType.SYSTEM_DEFAULT,
+      TypeCollection(IntegerType, DecimalType), DecimalType.SYSTEM_DEFAULT)
+    shouldCast(DecimalType(10, 2), TypeCollection(IntegerType, DecimalType), DecimalType(10, 2))
+    shouldCast(DecimalType(10, 2), TypeCollection(DecimalType, IntegerType), DecimalType(10, 2))
+
+    shouldCast(
+      ArrayType(StringType, false),
+      TypeCollection(ArrayType(StringType), StringType),
+      ArrayType(StringType, false))
+
+    shouldCast(
+      ArrayType(StringType, true),
+      TypeCollection(ArrayType(StringType), StringType),
+      ArrayType(StringType, true))
+
+    // When there are multiple convertible types in the `TypeCollection`, use the closest
+    // convertible data type among convertible types.
+    shouldCast(IntegerType, TypeCollection(BinaryType, FloatType, LongType), LongType)
+    shouldCast(ShortType, TypeCollection(BinaryType, LongType, IntegerType), IntegerType)
+    shouldCast(ShortType, TypeCollection(DateType, LongType, IntegerType, DoubleType), IntegerType)
+    // If the result is Float type and Double type is also among the convertible target types,
+    // use Double Type instead of Float type.
+    shouldCast(LongType, TypeCollection(FloatType, DoubleType, StringType), DoubleType)
+  }
+
+  test("ineligible implicit type cast - TypeCollection") {
+    shouldNotCast(IntegerType, TypeCollection(StringType, BinaryType))
+    shouldNotCast(IntegerType, TypeCollection(BinaryType, StringType))
+    shouldNotCast(IntegerType, TypeCollection(DateType, TimestampType))
+    shouldNotCast(IntegerType, TypeCollection(DecimalType(10, 2), StringType))
+    shouldNotCastStringInput(TypeCollection(NumericType, BinaryType))
+    // When there are multiple convertible types in the `TypeCollection` and there is no such
+    // a data type that can be implicit cast to all the other convertible types in the collection.
+    Seq(TypeCollection(NumericType, BinaryType),
+      TypeCollection(NumericType, DecimalType, BinaryType),
+      TypeCollection(IntegerType, LongType, BooleanType),
+      TypeCollection(DateType, TimestampType, BooleanType)).foreach { typeCollection =>
+      shouldNotCastStringLiteral(typeCollection)
+      shouldNotCast(NullType, typeCollection)
+    }
+  }
+
+  test("tightest common bound for types") {
+    def widenTest(t1: DataType, t2: DataType, expected: Option[DataType]): Unit =
+      checkWidenType(AnsiTypeCoercion.findTightestCommonType, t1, t2, expected)
+
+    // Null
+    widenTest(NullType, NullType, Some(NullType))
+
+    // Boolean
+    widenTest(NullType, BooleanType, Some(BooleanType))
+    widenTest(BooleanType, BooleanType, Some(BooleanType))
+    widenTest(IntegerType, BooleanType, None)
+    widenTest(LongType, BooleanType, None)
+
+    // Integral
+    widenTest(NullType, ByteType, Some(ByteType))
+    widenTest(NullType, IntegerType, Some(IntegerType))
+    widenTest(NullType, LongType, Some(LongType))
+    widenTest(ShortType, IntegerType, Some(IntegerType))
+    widenTest(ShortType, LongType, Some(LongType))
+    widenTest(IntegerType, LongType, Some(LongType))
+    widenTest(LongType, LongType, Some(LongType))
+
+    // Floating point
+    widenTest(NullType, FloatType, Some(FloatType))
+    widenTest(NullType, DoubleType, Some(DoubleType))
+    widenTest(FloatType, DoubleType, Some(DoubleType))
+    widenTest(FloatType, FloatType, Some(FloatType))
+    widenTest(DoubleType, DoubleType, Some(DoubleType))
+
+    // Integral mixed with floating point.
+    widenTest(IntegerType, FloatType, Some(DoubleType))
+    widenTest(IntegerType, DoubleType, Some(DoubleType))
+    widenTest(IntegerType, DoubleType, Some(DoubleType))
+    widenTest(LongType, FloatType, Some(DoubleType))
+    widenTest(LongType, DoubleType, Some(DoubleType))
+
+    widenTest(DecimalType(2, 1), DecimalType(3, 2), None)
+    widenTest(DecimalType(2, 1), DoubleType, None)
+    widenTest(DecimalType(2, 1), IntegerType, None)
+    widenTest(DoubleType, DecimalType(2, 1), None)
+
+    // StringType
+    widenTest(NullType, StringType, Some(StringType))
+    widenTest(StringType, StringType, Some(StringType))
+    widenTest(IntegerType, StringType, None)
+    widenTest(LongType, StringType, None)
+
+    // TimestampType
+    widenTest(NullType, TimestampType, Some(TimestampType))
+    widenTest(TimestampType, TimestampType, Some(TimestampType))
+    widenTest(DateType, TimestampType, Some(TimestampType))
+    widenTest(IntegerType, TimestampType, None)
+    widenTest(StringType, TimestampType, None)
+
+    // ComplexType
+    widenTest(NullType,
+      MapType(IntegerType, StringType, false),
+      Some(MapType(IntegerType, StringType, false)))
+    widenTest(NullType, StructType(Seq()), Some(StructType(Seq())))
+    widenTest(StringType, MapType(IntegerType, StringType, true), None)
+    widenTest(ArrayType(IntegerType), StructType(Seq()), None)
+
+    widenTest(
+      StructType(Seq(StructField("a", IntegerType))),
+      StructType(Seq(StructField("b", IntegerType))),
+      None)
+    widenTest(
+      StructType(Seq(StructField("a", IntegerType, nullable = false))),
+      StructType(Seq(StructField("a", DoubleType, nullable = false))),
+      Some(StructType(Seq(StructField("a", DoubleType, nullable = false)))))
+
+    widenTest(
+      StructType(Seq(StructField("a", IntegerType, nullable = false))),
+      StructType(Seq(StructField("a", IntegerType, nullable = false))),
+      Some(StructType(Seq(StructField("a", IntegerType, nullable = false)))))
+    widenTest(
+      StructType(Seq(StructField("a", IntegerType, nullable = false))),
+      StructType(Seq(StructField("a", IntegerType, nullable = true))),
+      Some(StructType(Seq(StructField("a", IntegerType, nullable = true)))))
+    widenTest(
+      StructType(Seq(StructField("a", IntegerType, nullable = true))),
+      StructType(Seq(StructField("a", IntegerType, nullable = false))),
+      Some(StructType(Seq(StructField("a", IntegerType, nullable = true)))))
+    widenTest(
+      StructType(Seq(StructField("a", IntegerType, nullable = true))),
+      StructType(Seq(StructField("a", IntegerType, nullable = true))),
+      Some(StructType(Seq(StructField("a", IntegerType, nullable = true)))))
+
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+      widenTest(
+        StructType(Seq(StructField("a", IntegerType))),
+        StructType(Seq(StructField("A", IntegerType))),
+        None)
+    }
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+      checkWidenType(
+        AnsiTypeCoercion.findTightestCommonType,
+        StructType(Seq(StructField("a", IntegerType), StructField("B", IntegerType))),
+        StructType(Seq(StructField("A", IntegerType), StructField("b", IntegerType))),
+        Some(StructType(Seq(StructField("a", IntegerType), StructField("B", IntegerType)))),
+        isSymmetric = false)
+    }
+
+    widenTest(
+      ArrayType(IntegerType, containsNull = true),
+      ArrayType(IntegerType, containsNull = false),
+      Some(ArrayType(IntegerType, containsNull = true)))
+
+    widenTest(
+      ArrayType(NullType, containsNull = true),
+      ArrayType(IntegerType, containsNull = false),
+      Some(ArrayType(IntegerType, containsNull = true)))
+
+    widenTest(
+      MapType(IntegerType, StringType, valueContainsNull = true),
+      MapType(IntegerType, StringType, valueContainsNull = false),
+      Some(MapType(IntegerType, StringType, valueContainsNull = true)))
+
+    widenTest(
+      MapType(NullType, NullType, true),
+      MapType(IntegerType, StringType, false),
+      Some(MapType(IntegerType, StringType, true)))
+
+    widenTest(
+      new StructType()
+        .add("arr", ArrayType(IntegerType, containsNull = true), nullable = false),
+      new StructType()
+        .add("arr", ArrayType(IntegerType, containsNull = false), nullable = true),
+      Some(new StructType()
+        .add("arr", ArrayType(IntegerType, containsNull = true), nullable = true)))
+
+    widenTest(
+      new StructType()
+        .add("null", NullType, nullable = true),
+      new StructType()
+        .add("null", IntegerType, nullable = false),
+      Some(new StructType()
+        .add("null", IntegerType, nullable = true)))
+
+    widenTest(
+      ArrayType(NullType, containsNull = false),
+      ArrayType(IntegerType, containsNull = false),
+      Some(ArrayType(IntegerType, containsNull = false)))
+
+    widenTest(MapType(NullType, NullType, false),
+      MapType(IntegerType, StringType, false),
+      Some(MapType(IntegerType, StringType, false)))
+
+    widenTest(
+      new StructType()
+        .add("null", NullType, nullable = false),
+      new StructType()
+        .add("null", IntegerType, nullable = false),
+      Some(new StructType()
+        .add("null", IntegerType, nullable = false)))
+  }
+
+  test("wider common type for decimal and array") {
+    def widenTestWithoutStringPromotion(
+        t1: DataType,
+        t2: DataType,
+        expected: Option[DataType],
+        isSymmetric: Boolean = true): Unit = {
+      checkWidenType(
+        AnsiTypeCoercion.findWiderTypeWithoutStringPromotionForTwo, t1, t2, expected, isSymmetric)
+    }
+
+    widenTestWithoutStringPromotion(
+      new StructType().add("num", IntegerType),
+      new StructType().add("num", LongType).add("str", StringType),
+      None)
+
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+      widenTestWithoutStringPromotion(
+        new StructType().add("a", IntegerType),
+        new StructType().add("A", LongType),
+        Some(new StructType().add("a", LongType)),
+        isSymmetric = false)
+    }
+
+    // Without string promotion
+    widenTestWithoutStringPromotion(IntegerType, StringType, None)
+    widenTestWithoutStringPromotion(StringType, TimestampType, None)
+    widenTestWithoutStringPromotion(ArrayType(LongType), ArrayType(StringType), None)
+    widenTestWithoutStringPromotion(ArrayType(StringType), ArrayType(TimestampType), None)
+    widenTestWithoutStringPromotion(
+      MapType(LongType, IntegerType), MapType(StringType, IntegerType), None)
+    widenTestWithoutStringPromotion(
+      MapType(IntegerType, LongType), MapType(IntegerType, StringType), None)
+    widenTestWithoutStringPromotion(
+      MapType(StringType, IntegerType), MapType(TimestampType, IntegerType), None)
+    widenTestWithoutStringPromotion(
+      MapType(IntegerType, StringType), MapType(IntegerType, TimestampType), None)
+    widenTestWithoutStringPromotion(
+      new StructType().add("a", IntegerType),
+      new StructType().add("a", StringType),
+      None)
+    widenTestWithoutStringPromotion(
+      new StructType().add("a", StringType),
+      new StructType().add("a", IntegerType),
+      None)
+  }
+
+  private def ruleTest(rule: Rule[LogicalPlan],
+      initial: Expression, transformed: Expression): Unit = {
+    ruleTest(Seq(rule), initial, transformed)
+  }
+
+  private def ruleTest(
+      rules: Seq[Rule[LogicalPlan]],
+      initial: Expression,
+      transformed: Expression): Unit = {
+    val testRelation = LocalRelation(AttributeReference("a", IntegerType)())
+    val analyzer = new RuleExecutor[LogicalPlan] {
+      override val batches = Seq(Batch("Resolution", FixedPoint(3), rules: _*))
+    }
+
+    comparePlans(
+      analyzer.execute(Project(Seq(Alias(initial, "a")()), testRelation)),
+      Project(Seq(Alias(transformed, "a")()), testRelation))
+  }
+
+  test("cast NullType for expressions that implement ExpectsInputTypes") {
+    ruleTest(AnsiTypeCoercion.ImplicitTypeCasts,
+      AnyTypeUnaryExpression(Literal.create(null, NullType)),
+      AnyTypeUnaryExpression(Literal.create(null, NullType)))
+
+    ruleTest(AnsiTypeCoercion.ImplicitTypeCasts,
+      NumericTypeUnaryExpression(Literal.create(null, NullType)),
+      NumericTypeUnaryExpression(Literal.create(null, DoubleType)))
+  }
+
+  test("cast NullType for binary operators") {
+    ruleTest(AnsiTypeCoercion.ImplicitTypeCasts,
+      AnyTypeBinaryOperator(Literal.create(null, NullType), Literal.create(null, NullType)),
+      AnyTypeBinaryOperator(Literal.create(null, NullType), Literal.create(null, NullType)))
+
+    ruleTest(AnsiTypeCoercion.ImplicitTypeCasts,
+      NumericTypeBinaryOperator(Literal.create(null, NullType), Literal.create(null, NullType)),
+      NumericTypeBinaryOperator(Literal.create(null, DoubleType), Literal.create(null, DoubleType)))
+  }
+
+  test("coalesce casts") {
+    val rule = AnsiTypeCoercion.FunctionArgumentConversion
+
+    val intLit = Literal(1)
+    val longLit = Literal.create(1L)
+    val doubleLit = Literal(1.0)
+    val stringLit = Literal.create("c", StringType)
+    val nullLit = Literal.create(null, NullType)
+    val floatNullLit = Literal.create(null, FloatType)
+    val floatLit = Literal.create(1.0f, FloatType)
+    val doubleNullLit = Cast(floatNullLit, DoubleType)
+    val timestampLit = Literal.create(Timestamp.valueOf("2017-04-12 00:00:00"), TimestampType)
+    val decimalLit = Literal(new java.math.BigDecimal("1000000000000000000000"))
+    val tsArrayLit = Literal(Array(new Timestamp(System.currentTimeMillis())))
+    val strArrayLit = Literal(Array("c"))
+    val intArrayLit = Literal(Array(1))
+
+    ruleTest(rule,
+      Coalesce(Seq(doubleLit, intLit, floatLit)),
+      Coalesce(Seq(doubleLit, Cast(intLit, DoubleType), Cast(floatLit, DoubleType))))
+
+    ruleTest(rule,
+      Coalesce(Seq(longLit, intLit, decimalLit)),
+      Coalesce(Seq(Cast(longLit, DecimalType(22, 0)),
+        Cast(intLit, DecimalType(22, 0)), decimalLit)))
+
+    ruleTest(rule,
+      Coalesce(Seq(nullLit, intLit)),
+      Coalesce(Seq(Cast(nullLit, IntegerType), intLit)))
+
+    ruleTest(rule,
+      Coalesce(Seq(timestampLit, stringLit)),
+      Coalesce(Seq(timestampLit, stringLit)))
+
+    ruleTest(rule,
+      Coalesce(Seq(nullLit, floatNullLit, intLit)),
+      Coalesce(Seq(Cast(nullLit, DoubleType), doubleNullLit, Cast(intLit, DoubleType))))
+
+    ruleTest(rule,
+      Coalesce(Seq(nullLit, intLit, decimalLit, doubleLit)),
+      Coalesce(Seq(Cast(nullLit, DoubleType), Cast(intLit, DoubleType),
+        Cast(decimalLit, DoubleType), doubleLit)))
+
+    // There is no a common type among Float/Double/String
+    ruleTest(rule,
+      Coalesce(Seq(nullLit, floatNullLit, doubleLit, stringLit)),
+      Coalesce(Seq(nullLit, floatNullLit, doubleLit, stringLit)))
+
+    // There is no a common type among Timestamp/Int/String
+    ruleTest(rule,
+      Coalesce(Seq(timestampLit, intLit, stringLit)),
+      Coalesce(Seq(timestampLit, intLit, stringLit)))
+
+    ruleTest(rule,
+      Coalesce(Seq(tsArrayLit, intArrayLit, strArrayLit)),
+      Coalesce(Seq(tsArrayLit, intArrayLit, strArrayLit)))
+  }
+
+  test("CreateArray casts") {
+    ruleTest(AnsiTypeCoercion.FunctionArgumentConversion,
+      CreateArray(Literal(1.0)
+        :: Literal(1)
+        :: Literal.create(1.0f, FloatType)
+        :: Nil),
+      CreateArray(Literal(1.0)
+        :: Cast(Literal(1), DoubleType)
+        :: Cast(Literal.create(1.0f, FloatType), DoubleType)
+        :: Nil))
+
+    ruleTest(AnsiTypeCoercion.FunctionArgumentConversion,
+      CreateArray(Literal(1.0)
+        :: Literal(1)
+        :: Literal("a")
+        :: Nil),
+      CreateArray(Literal(1.0)
+        :: Literal(1)
+        :: Literal("a")
+        :: Nil))
+
+    ruleTest(AnsiTypeCoercion.FunctionArgumentConversion,
+      CreateArray(Literal.create(null, DecimalType(5, 3))
+        :: Literal(1)
+        :: Nil),
+      CreateArray(Literal.create(null, DecimalType(5, 3)).cast(DecimalType(13, 3))
+        :: Literal(1).cast(DecimalType(13, 3))
+        :: Nil))
+
+    ruleTest(AnsiTypeCoercion.FunctionArgumentConversion,
+      CreateArray(Literal.create(null, DecimalType(5, 3))
+        :: Literal.create(null, DecimalType(22, 10))
+        :: Literal.create(null, DecimalType(38, 38))
+        :: Nil),
+      CreateArray(Literal.create(null, DecimalType(5, 3)).cast(DecimalType(38, 38))
+        :: Literal.create(null, DecimalType(22, 10)).cast(DecimalType(38, 38))
+        :: Literal.create(null, DecimalType(38, 38))
+        :: Nil))
+  }
+
+  test("CreateMap casts") {
+    // type coercion for map keys
+    ruleTest(AnsiTypeCoercion.FunctionArgumentConversion,
+      CreateMap(Literal(1)
+        :: Literal("a")
+        :: Literal.create(2.0f, FloatType)
+        :: Literal("b")
+        :: Nil),
+      CreateMap(Cast(Literal(1), DoubleType)
+        :: Literal("a")
+        :: Cast(Literal.create(2.0f, FloatType), DoubleType)
+        :: Literal("b")
+        :: Nil))
+    ruleTest(AnsiTypeCoercion.FunctionArgumentConversion,
+      CreateMap(Literal.create(null, DecimalType(5, 3))
+        :: Literal("a")
+        :: Literal.create(2.0f, FloatType)
+        :: Literal("b")
+        :: Nil),
+      CreateMap(Literal.create(null, DecimalType(5, 3)).cast(DoubleType)
+        :: Literal("a")
+        :: Literal.create(2.0f, FloatType).cast(DoubleType)
+        :: Literal("b")
+        :: Nil))
+    // type coercion for map values
+    ruleTest(AnsiTypeCoercion.FunctionArgumentConversion,
+      CreateMap(Literal(1)
+        :: Literal("a")
+        :: Literal(2)
+        :: Literal(3.0)
+        :: Nil),
+      CreateMap(Literal(1)
+        :: Literal("a")
+        :: Literal(2)
+        :: Literal(3.0)
+        :: Nil))
+    ruleTest(AnsiTypeCoercion.FunctionArgumentConversion,
+      CreateMap(Literal(1)
+        :: Literal.create(null, DecimalType(38, 0))
+        :: Literal(2)
+        :: Literal.create(null, DecimalType(38, 38))
+        :: Nil),
+      CreateMap(Literal(1)
+        :: Literal.create(null, DecimalType(38, 0)).cast(DecimalType(38, 38))
+        :: Literal(2)
+        :: Literal.create(null, DecimalType(38, 38))
+        :: Nil))
+    // type coercion for both map keys and values
+    ruleTest(AnsiTypeCoercion.FunctionArgumentConversion,
+      CreateMap(Literal(1)
+        :: Literal("a")
+        :: Literal(2.0)
+        :: Literal(3.0)
+        :: Nil),
+      CreateMap(Cast(Literal(1), DoubleType)
+        :: Literal("a")
+        :: Literal(2.0)
+        :: Literal(3.0)
+        :: Nil))
+  }
+
+  test("greatest/least cast") {
+    for (operator <- Seq[(Seq[Expression] => Expression)](Greatest, Least)) {
+      ruleTest(AnsiTypeCoercion.FunctionArgumentConversion,
+        operator(Literal(1.0)
+          :: Literal(1)
+          :: Literal.create(1.0f, FloatType)
+          :: Nil),
+        operator(Literal(1.0)
+          :: Cast(Literal(1), DoubleType)
+          :: Cast(Literal.create(1.0f, FloatType), DoubleType)
+          :: Nil))
+      ruleTest(AnsiTypeCoercion.FunctionArgumentConversion,
+        operator(Literal(1L)
+          :: Literal(1)
+          :: Literal(new java.math.BigDecimal("1000000000000000000000"))
+          :: Nil),
+        operator(Cast(Literal(1L), DecimalType(22, 0))
+          :: Cast(Literal(1), DecimalType(22, 0))
+          :: Literal(new java.math.BigDecimal("1000000000000000000000"))
+          :: Nil))
+      ruleTest(AnsiTypeCoercion.FunctionArgumentConversion,
+        operator(Literal(1.0)
+          :: Literal.create(null, DecimalType(10, 5))
+          :: Literal(1)
+          :: Nil),
+        operator(Literal(1.0)
+          :: Literal.create(null, DecimalType(10, 5)).cast(DoubleType)
+          :: Literal(1).cast(DoubleType)
+          :: Nil))
+      ruleTest(AnsiTypeCoercion.FunctionArgumentConversion,
+        operator(Literal.create(null, DecimalType(15, 0))
+          :: Literal.create(null, DecimalType(10, 5))
+          :: Literal(1)
+          :: Nil),
+        operator(Literal.create(null, DecimalType(15, 0)).cast(DecimalType(20, 5))
+          :: Literal.create(null, DecimalType(10, 5)).cast(DecimalType(20, 5))
+          :: Literal(1).cast(DecimalType(20, 5))
+          :: Nil))
+      ruleTest(AnsiTypeCoercion.FunctionArgumentConversion,
+        operator(Literal.create(2L, LongType)
+          :: Literal(1)
+          :: Literal.create(null, DecimalType(10, 5))
+          :: Nil),
+        operator(Literal.create(2L, LongType).cast(DecimalType(25, 5))
+          :: Literal(1).cast(DecimalType(25, 5))
+          :: Literal.create(null, DecimalType(10, 5)).cast(DecimalType(25, 5))
+          :: Nil))
+    }
+  }
+
+  test("nanvl casts") {
+    ruleTest(AnsiTypeCoercion.FunctionArgumentConversion,
+      NaNvl(Literal.create(1.0f, FloatType), Literal.create(1.0, DoubleType)),
+      NaNvl(Cast(Literal.create(1.0f, FloatType), DoubleType), Literal.create(1.0, DoubleType)))
+    ruleTest(AnsiTypeCoercion.FunctionArgumentConversion,
+      NaNvl(Literal.create(1.0, DoubleType), Literal.create(1.0f, FloatType)),
+      NaNvl(Literal.create(1.0, DoubleType), Cast(Literal.create(1.0f, FloatType), DoubleType)))
+    ruleTest(AnsiTypeCoercion.FunctionArgumentConversion,
+      NaNvl(Literal.create(1.0, DoubleType), Literal.create(1.0, DoubleType)),
+      NaNvl(Literal.create(1.0, DoubleType), Literal.create(1.0, DoubleType)))
+    ruleTest(AnsiTypeCoercion.FunctionArgumentConversion,
+      NaNvl(Literal.create(1.0f, FloatType), Literal.create(null, NullType)),
+      NaNvl(Literal.create(1.0f, FloatType), Cast(Literal.create(null, NullType), FloatType)))
+    ruleTest(AnsiTypeCoercion.FunctionArgumentConversion,
+      NaNvl(Literal.create(1.0, DoubleType), Literal.create(null, NullType)),
+      NaNvl(Literal.create(1.0, DoubleType), Cast(Literal.create(null, NullType), DoubleType)))
+  }
+
+  test("type coercion for If") {
+    val rule = AnsiTypeCoercion.IfCoercion
+    val intLit = Literal(1)
+    val doubleLit = Literal(1.0)
+    val trueLit = Literal.create(true, BooleanType)
+    val falseLit = Literal.create(false, BooleanType)
+    val stringLit = Literal.create("c", StringType)
+    val floatLit = Literal.create(1.0f, FloatType)
+    val timestampLit = Literal.create(Timestamp.valueOf("2017-04-12 00:00:00"), TimestampType)
+    val decimalLit = Literal(new java.math.BigDecimal("1000000000000000000000"))
+
+    ruleTest(rule,
+      If(Literal(true), Literal(1), Literal(1L)),
+      If(Literal(true), Cast(Literal(1), LongType), Literal(1L)))
+
+    ruleTest(rule,
+      If(Literal.create(null, NullType), Literal(1), Literal(1)),
+      If(Literal.create(null, BooleanType), Literal(1), Literal(1)))
+
+    ruleTest(rule,
+      If(AssertTrue(trueLit), Literal(1), Literal(2)),
+      If(Cast(AssertTrue(trueLit), BooleanType), Literal(1), Literal(2)))
+
+    ruleTest(rule,
+      If(AssertTrue(falseLit), Literal(1), Literal(2)),
+      If(Cast(AssertTrue(falseLit), BooleanType), Literal(1), Literal(2)))
+
+    ruleTest(rule,
+      If(trueLit, intLit, doubleLit),
+      If(trueLit, Cast(intLit, DoubleType), doubleLit))
+
+    ruleTest(rule,
+      If(trueLit, floatLit, doubleLit),
+      If(trueLit, Cast(floatLit, DoubleType), doubleLit))
+
+    ruleTest(rule,
+      If(trueLit, floatLit, decimalLit),
+      If(trueLit, Cast(floatLit, DoubleType), Cast(decimalLit, DoubleType)))
+
+    ruleTest(rule,
+      If(falseLit, stringLit, doubleLit),
+      If(falseLit, stringLit, doubleLit))
+
+    ruleTest(rule,
+      If(trueLit, timestampLit, stringLit),
+      If(trueLit, timestampLit, stringLit))
+  }
+
+  test("type coercion for CaseKeyWhen") {
+    ruleTest(AnsiTypeCoercion.ImplicitTypeCasts,
+      CaseKeyWhen(Literal(1.toShort), Seq(Literal(1), Literal("a"))),
+      CaseKeyWhen(Cast(Literal(1.toShort), IntegerType), Seq(Literal(1), Literal("a")))
+    )
+    ruleTest(AnsiTypeCoercion.CaseWhenCoercion,
+      CaseKeyWhen(Literal(true), Seq(Literal(1), Literal("a"))),
+      CaseKeyWhen(Literal(true), Seq(Literal(1), Literal("a")))
+    )
+    ruleTest(AnsiTypeCoercion.CaseWhenCoercion,
+      CaseWhen(Seq((Literal(true), Literal(1.2))),
+        Literal.create(BigDecimal.valueOf(1), DecimalType(7, 2))),
+      CaseWhen(Seq((Literal(true), Literal(1.2))),
+        Cast(Literal.create(BigDecimal.valueOf(1), DecimalType(7, 2)), DoubleType))
+    )
+    ruleTest(AnsiTypeCoercion.CaseWhenCoercion,
+      CaseWhen(Seq((Literal(true), Literal(100L))),
+        Literal.create(BigDecimal.valueOf(1), DecimalType(7, 2))),
+      CaseWhen(Seq((Literal(true), Cast(Literal(100L), DecimalType(22, 2)))),
+        Cast(Literal.create(BigDecimal.valueOf(1), DecimalType(7, 2)), DecimalType(22, 2)))
+    )
+  }
+
+  test("type coercion for Stack") {
+    val rule = AnsiTypeCoercion.StackCoercion
+
+    ruleTest(rule,
+      Stack(Seq(Literal(3), Literal(1), Literal(2), Literal(null))),
+      Stack(Seq(Literal(3), Literal(1), Literal(2), Literal.create(null, IntegerType))))
+    ruleTest(rule,
+      Stack(Seq(Literal(3), Literal(1.0), Literal(null), Literal(3.0))),
+      Stack(Seq(Literal(3), Literal(1.0), Literal.create(null, DoubleType), Literal(3.0))))
+    ruleTest(rule,
+      Stack(Seq(Literal(3), Literal(null), Literal("2"), Literal("3"))),
+      Stack(Seq(Literal(3), Literal.create(null, StringType), Literal("2"), Literal("3"))))
+    ruleTest(rule,
+      Stack(Seq(Literal(3), Literal(null), Literal(null), Literal(null))),
+      Stack(Seq(Literal(3), Literal(null), Literal(null), Literal(null))))
+
+    ruleTest(rule,
+      Stack(Seq(Literal(2),
+        Literal(1), Literal("2"),
+        Literal(null), Literal(null))),
+      Stack(Seq(Literal(2),
+        Literal(1), Literal("2"),
+        Literal.create(null, IntegerType), Literal.create(null, StringType))))
+
+    ruleTest(rule,
+      Stack(Seq(Literal(2),
+        Literal(1), Literal(null),
+        Literal(null), Literal("2"))),
+      Stack(Seq(Literal(2),
+        Literal(1), Literal.create(null, StringType),
+        Literal.create(null, IntegerType), Literal("2"))))
+
+    ruleTest(rule,
+      Stack(Seq(Literal(2),
+        Literal(null), Literal(1),
+        Literal("2"), Literal(null))),
+      Stack(Seq(Literal(2),
+        Literal.create(null, StringType), Literal(1),
+        Literal("2"), Literal.create(null, IntegerType))))
+
+    ruleTest(rule,
+      Stack(Seq(Literal(2),
+        Literal(null), Literal(null),
+        Literal(1), Literal("2"))),
+      Stack(Seq(Literal(2),
+        Literal.create(null, IntegerType), Literal.create(null, StringType),
+        Literal(1), Literal("2"))))
+
+    ruleTest(rule,
+      Stack(Seq(Subtract(Literal(3), Literal(1)),
+        Literal(1), Literal("2"),
+        Literal(null), Literal(null))),
+      Stack(Seq(Subtract(Literal(3), Literal(1)),
+        Literal(1), Literal("2"),
+        Literal.create(null, IntegerType), Literal.create(null, StringType))))
+  }
+
+  test("type coercion for Concat") {
+    val rule = AnsiTypeCoercion.ConcatCoercion
+
+    ruleTest(rule,
+      Concat(Seq(Literal("ab"), Literal("cde"))),
+      Concat(Seq(Literal("ab"), Literal("cde"))))
+    ruleTest(rule,
+      Concat(Seq(Literal(null), Literal("abc"))),
+      Concat(Seq(Cast(Literal(null), StringType), Literal("abc"))))
+    ruleTest(rule,
+      Concat(Seq(Literal(1), Literal("234"))),
+      Concat(Seq(Literal(1), Literal("234"))))
+    ruleTest(rule,
+      Concat(Seq(Literal("1"), Literal("234".getBytes()))),
+      Concat(Seq(Literal("1"), Literal("234".getBytes()))))
+    ruleTest(rule,
+      Concat(Seq(Literal(1L), Literal(2.toByte), Literal(0.1))),
+      Concat(Seq(Literal(1L), Literal(2.toByte), Literal(0.1))))
+    ruleTest(rule,
+      Concat(Seq(Literal(true), Literal(0.1f), Literal(3.toShort))),
+      Concat(Seq(Literal(true), Literal(0.1f), Literal(3.toShort))))
+    ruleTest(rule,
+      Concat(Seq(Literal(1L), Literal(0.1))),
+      Concat(Seq(Literal(1L), Literal(0.1))))
+    ruleTest(rule,
+      Concat(Seq(Literal(Decimal(10)))),
+      Concat(Seq(Literal(Decimal(10)))))
+    ruleTest(rule,
+      Concat(Seq(Literal(BigDecimal.valueOf(10)))),
+      Concat(Seq(Literal(BigDecimal.valueOf(10)))))
+    ruleTest(rule,
+      Concat(Seq(Literal(java.math.BigDecimal.valueOf(10)))),
+      Concat(Seq(Literal(java.math.BigDecimal.valueOf(10)))))
+    ruleTest(rule,
+      Concat(Seq(Literal(new java.sql.Date(0)), Literal(new Timestamp(0)))),
+      Concat(Seq(Literal(new java.sql.Date(0)), Literal(new Timestamp(0)))))
+
+    ruleTest(rule,
+      Concat(Seq(Literal("123".getBytes), Literal("456".getBytes))),
+      Concat(Seq(Literal("123".getBytes), Literal("456".getBytes))))
+  }
+
+  test("type coercion for Elt") {
+    val rule = AnsiTypeCoercion.EltCoercion
+
+    ruleTest(rule,
+      Elt(Seq(Literal(1), Literal("ab"), Literal("cde"))),
+      Elt(Seq(Literal(1), Literal("ab"), Literal("cde"))))
+    ruleTest(rule,
+      Elt(Seq(Literal(1.toShort), Literal("ab"), Literal("cde"))),
+      Elt(Seq(Cast(Literal(1.toShort), IntegerType), Literal("ab"), Literal("cde"))))
+    ruleTest(rule,
+      Elt(Seq(Literal(2), Literal(null), Literal("abc"))),
+      Elt(Seq(Literal(2), Cast(Literal(null), StringType), Literal("abc"))))
+    ruleTest(rule,
+      Elt(Seq(Literal(2), Literal(1), Literal("234"))),
+      Elt(Seq(Literal(2), Literal(1), Literal("234"))))
+    ruleTest(rule,
+      Elt(Seq(Literal(3), Literal(1L), Literal(2.toByte), Literal(0.1))),
+      Elt(Seq(Literal(3), Literal(1L), Literal(2.toByte), Literal(0.1))))
+    ruleTest(rule,
+      Elt(Seq(Literal(2), Literal(true), Literal(0.1f), Literal(3.toShort))),
+      Elt(Seq(Literal(2), Literal(true), Literal(0.1f), Literal(3.toShort))))
+    ruleTest(rule,
+      Elt(Seq(Literal(1), Literal(1L), Literal(0.1))),
+      Elt(Seq(Literal(1), Literal(1L), Literal(0.1))))
+    ruleTest(rule,
+      Elt(Seq(Literal(1), Literal(Decimal(10)))),
+      Elt(Seq(Literal(1), Literal(Decimal(10)))))
+    ruleTest(rule,
+      Elt(Seq(Literal(1), Literal(BigDecimal.valueOf(10)))),
+      Elt(Seq(Literal(1), Literal(BigDecimal.valueOf(10)))))
+    ruleTest(rule,
+      Elt(Seq(Literal(1), Literal(java.math.BigDecimal.valueOf(10)))),
+      Elt(Seq(Literal(1), Literal(java.math.BigDecimal.valueOf(10)))))
+    ruleTest(rule,
+      Elt(Seq(Literal(2), Literal(new java.sql.Date(0)), Literal(new Timestamp(0)))),
+      Elt(Seq(Literal(2), Literal(new java.sql.Date(0)), Literal(new Timestamp(0)))))
+
+    ruleTest(rule,
+      Elt(Seq(Literal(1), Literal("123".getBytes), Literal("456".getBytes))),
+      Elt(Seq(Literal(1), Literal("123".getBytes), Literal("456".getBytes))))
+  }
+
+  private def checkOutput(logical: LogicalPlan, expectTypes: Seq[DataType]): Unit = {
+    logical.output.zip(expectTypes).foreach { case (attr, dt) =>
+      assert(attr.dataType === dt)
+    }
+  }
+
+  private val timeZoneResolver = ResolveTimeZone
+
+  private def widenSetOperationTypes(plan: LogicalPlan): LogicalPlan = {
+    timeZoneResolver(AnsiTypeCoercion.WidenSetOperationTypes(plan))
+  }
+
+  test("WidenSetOperationTypes for except and intersect") {
+    val firstTable = LocalRelation(
+      AttributeReference("i", IntegerType)(),
+      AttributeReference("u", DecimalType.SYSTEM_DEFAULT)(),
+      AttributeReference("b", ByteType)(),
+      AttributeReference("d", DoubleType)())
+    val secondTable = LocalRelation(
+      AttributeReference("s", LongType)(),
+      AttributeReference("d", DecimalType(2, 1))(),
+      AttributeReference("f", FloatType)(),
+      AttributeReference("l", LongType)())
+
+    val expectedTypes = Seq(LongType, DecimalType.SYSTEM_DEFAULT, DoubleType, DoubleType)
+
+    val r1 = widenSetOperationTypes(
+      Except(firstTable, secondTable, isAll = false)).asInstanceOf[Except]
+    val r2 = widenSetOperationTypes(
+      Intersect(firstTable, secondTable, isAll = false)).asInstanceOf[Intersect]
+    checkOutput(r1.left, expectedTypes)
+    checkOutput(r1.right, expectedTypes)
+    checkOutput(r2.left, expectedTypes)
+    checkOutput(r2.right, expectedTypes)
+
+    // Check if a Project is added
+    assert(r1.left.isInstanceOf[Project])
+    assert(r1.right.isInstanceOf[Project])
+    assert(r2.left.isInstanceOf[Project])
+    assert(r2.right.isInstanceOf[Project])
+  }
+
+  test("WidenSetOperationTypes for union") {
+    val firstTable = LocalRelation(
+      AttributeReference("i", DateType)(),
+      AttributeReference("u", DecimalType.SYSTEM_DEFAULT)(),
+      AttributeReference("b", ByteType)(),
+      AttributeReference("d", DoubleType)())
+    val secondTable = LocalRelation(
+      AttributeReference("s", DateType)(),
+      AttributeReference("d", DecimalType(2, 1))(),
+      AttributeReference("f", FloatType)(),
+      AttributeReference("l", LongType)())
+    val thirdTable = LocalRelation(
+      AttributeReference("m", TimestampType)(),
+      AttributeReference("n", DecimalType.SYSTEM_DEFAULT)(),
+      AttributeReference("p", FloatType)(),
+      AttributeReference("q", DoubleType)())
+    val forthTable = LocalRelation(
+      AttributeReference("m", DateType)(),
+      AttributeReference("n", DecimalType.SYSTEM_DEFAULT)(),
+      AttributeReference("p", ByteType)(),
+      AttributeReference("q", DoubleType)())
+
+    val expectedTypes = Seq(TimestampType, DecimalType.SYSTEM_DEFAULT, DoubleType, DoubleType)
+
+    val unionRelation = widenSetOperationTypes(
+      Union(firstTable :: secondTable :: thirdTable :: forthTable :: Nil)).asInstanceOf[Union]
+    assert(unionRelation.children.length == 4)
+    checkOutput(unionRelation.children.head, expectedTypes)
+    checkOutput(unionRelation.children(1), expectedTypes)
+    checkOutput(unionRelation.children(2), expectedTypes)
+    checkOutput(unionRelation.children(3), expectedTypes)
+
+    assert(unionRelation.children.head.isInstanceOf[Project])
+    assert(unionRelation.children(1).isInstanceOf[Project])
+    assert(unionRelation.children(2).isInstanceOf[Project])
+    assert(unionRelation.children(3).isInstanceOf[Project])
+  }
+
+  test("Transform Decimal precision/scale for union except and intersect") {
+    def checkOutput(logical: LogicalPlan, expectTypes: Seq[DataType]): Unit = {
+      logical.output.zip(expectTypes).foreach { case (attr, dt) =>
+        assert(attr.dataType === dt)
+      }
+    }
+
+    val left1 = LocalRelation(
+      AttributeReference("l", DecimalType(10, 8))())
+    val right1 = LocalRelation(
+      AttributeReference("r", DecimalType(5, 5))())
+    val expectedType1 = Seq(DecimalType(10, 8))
+
+    val r1 = widenSetOperationTypes(Union(left1, right1)).asInstanceOf[Union]
+    val r2 = widenSetOperationTypes(
+      Except(left1, right1, isAll = false)).asInstanceOf[Except]
+    val r3 = widenSetOperationTypes(
+      Intersect(left1, right1, isAll = false)).asInstanceOf[Intersect]
+
+    checkOutput(r1.children.head, expectedType1)
+    checkOutput(r1.children.last, expectedType1)
+    checkOutput(r2.left, expectedType1)
+    checkOutput(r2.right, expectedType1)
+    checkOutput(r3.left, expectedType1)
+    checkOutput(r3.right, expectedType1)
+
+    val plan1 = LocalRelation(AttributeReference("l", DecimalType(10, 5))())
+
+    val rightTypes = Seq(ByteType, ShortType, IntegerType, LongType, FloatType, DoubleType)
+    val expectedTypes = Seq(DecimalType(10, 5), DecimalType(10, 5), DecimalType(15, 5),
+      DecimalType(25, 5), DoubleType, DoubleType)
+
+    rightTypes.zip(expectedTypes).foreach { case (rType, expectedType) =>
+      val plan2 = LocalRelation(
+        AttributeReference("r", rType)())
+
+      val r1 = widenSetOperationTypes(Union(plan1, plan2)).asInstanceOf[Union]
+      val r2 = widenSetOperationTypes(
+        Except(plan1, plan2, isAll = false)).asInstanceOf[Except]
+      val r3 = widenSetOperationTypes(
+        Intersect(plan1, plan2, isAll = false)).asInstanceOf[Intersect]
+
+      checkOutput(r1.children.last, Seq(expectedType))
+      checkOutput(r2.right, Seq(expectedType))
+      checkOutput(r3.right, Seq(expectedType))
+
+      val r4 = widenSetOperationTypes(Union(plan2, plan1)).asInstanceOf[Union]
+      val r5 = widenSetOperationTypes(
+        Except(plan2, plan1, isAll = false)).asInstanceOf[Except]
+      val r6 = widenSetOperationTypes(
+        Intersect(plan2, plan1, isAll = false)).asInstanceOf[Intersect]
+
+      checkOutput(r4.children.last, Seq(expectedType))
+      checkOutput(r5.left, Seq(expectedType))
+      checkOutput(r6.left, Seq(expectedType))
+    }
+  }
+
+  test("SPARK-32638: corrects references when adding aliases in WidenSetOperationTypes") {
+    val t1 = LocalRelation(AttributeReference("v", DecimalType(10, 0))())
+    val t2 = LocalRelation(AttributeReference("v", DecimalType(11, 0))())
+    val p1 = t1.select(t1.output.head).as("p1")
+    val p2 = t2.select(t2.output.head).as("p2")
+    val union = p1.union(p2)
+    val wp1 = widenSetOperationTypes(union.select(p1.output.head, $"p2.v"))
+    assert(wp1.isInstanceOf[Project])
+    // The attribute `p1.output.head` should be replaced in the root `Project`.
+    assert(wp1.expressions.forall(_.find(_ == p1.output.head).isEmpty))
+    val wp2 = widenSetOperationTypes(Aggregate(Nil, sum(p1.output.head).as("v") :: Nil, union))
+    assert(wp2.isInstanceOf[Aggregate])
+    assert(wp2.missingInput.isEmpty)
+  }
+
+  /**
+   * There are rules that need to not fire before child expressions get resolved.
+   * We use this test to make sure those rules do not fire early.
+   */
+  test("make sure rules do not fire early") {
+    // InConversion
+    val inConversion = AnsiTypeCoercion.InConversion
+    ruleTest(inConversion,
+      In(UnresolvedAttribute("a"), Seq(Literal(1))),
+      In(UnresolvedAttribute("a"), Seq(Literal(1)))
+    )
+    ruleTest(inConversion,
+      In(Literal("test"), Seq(UnresolvedAttribute("a"), Literal(1))),
+      In(Literal("test"), Seq(UnresolvedAttribute("a"), Literal(1)))
+    )
+    ruleTest(inConversion,
+      In(Literal("a"), Seq(Literal(1), Literal("b"))),
+      In(Literal("a"), Seq(Literal(1), Literal("b")))
+    )
+  }
+
+  test("SPARK-15776 Divide expression's dataType should be casted to Double or Decimal " +
+    "in aggregation function like sum") {
+    val rules = Seq(FunctionArgumentConversion, Division)
+    // Casts Integer to Double
+    ruleTest(rules, sum(Divide(4, 3)), sum(Divide(Cast(4, DoubleType), Cast(3, DoubleType))))
+    // Left expression is Double, right expression is Int. Another rule ImplicitTypeCasts will
+    // cast the right expression to Double.
+    ruleTest(rules, sum(Divide(4.0, 3)), sum(Divide(4.0, 3)))
+    // Left expression is Int, right expression is Double
+    ruleTest(rules, sum(Divide(4, 3.0)), sum(Divide(Cast(4, DoubleType), Cast(3.0, DoubleType))))
+    // Casts Float to Double
+    ruleTest(
+      rules,
+      sum(Divide(4.0f, 3)),
+      sum(Divide(Cast(4.0f, DoubleType), Cast(3, DoubleType))))
+    // Left expression is Decimal, right expression is Int. Another rule DecimalPrecision will cast
+    // the right expression to Decimal.
+    ruleTest(rules, sum(Divide(Decimal(4.0), 3)), sum(Divide(Decimal(4.0), 3)))
+  }
+
+  test("SPARK-17117 null type coercion in divide") {
+    val rules = Seq(FunctionArgumentConversion, Division, ImplicitTypeCasts)
+    val nullLit = Literal.create(null, NullType)
+    ruleTest(rules, Divide(1L, nullLit), Divide(Cast(1L, DoubleType), Cast(nullLit, DoubleType)))
+    ruleTest(rules, Divide(nullLit, 1L), Divide(Cast(nullLit, DoubleType), Cast(1L, DoubleType)))
+  }
+
+  test("cast WindowFrame boundaries to the type they operate upon") {
+    // Can cast frame boundaries to order dataType.
+    ruleTest(WindowFrameCoercion,
+      windowSpec(
+        Seq(UnresolvedAttribute("a")),
+        Seq(SortOrder(Literal(1L), Ascending)),
+        SpecifiedWindowFrame(RangeFrame, Literal(3), Literal(2147483648L))),
+      windowSpec(
+        Seq(UnresolvedAttribute("a")),
+        Seq(SortOrder(Literal(1L), Ascending)),
+        SpecifiedWindowFrame(RangeFrame, Cast(3, LongType), Literal(2147483648L)))
+    )
+    // Cannot cast frame boundaries to order dataType.
+    ruleTest(WindowFrameCoercion,
+      windowSpec(
+        Seq(UnresolvedAttribute("a")),
+        Seq(SortOrder(Literal.default(DateType), Ascending)),
+        SpecifiedWindowFrame(RangeFrame, Literal(10.0), Literal(2147483648L))),
+      windowSpec(
+        Seq(UnresolvedAttribute("a")),
+        Seq(SortOrder(Literal.default(DateType), Ascending)),
+        SpecifiedWindowFrame(RangeFrame, Literal(10.0), Literal(2147483648L)))
+    )
+    // Should not cast SpecialFrameBoundary.
+    ruleTest(WindowFrameCoercion,
+      windowSpec(
+        Seq(UnresolvedAttribute("a")),
+        Seq(SortOrder(Literal(1L), Ascending)),
+        SpecifiedWindowFrame(RangeFrame, CurrentRow, UnboundedFollowing)),
+      windowSpec(
+        Seq(UnresolvedAttribute("a")),
+        Seq(SortOrder(Literal(1L), Ascending)),
+        SpecifiedWindowFrame(RangeFrame, CurrentRow, UnboundedFollowing))
+    )
+  }
+
+  test("SPARK-29000: skip to handle decimals in ImplicitTypeCasts") {
+    ruleTest(AnsiTypeCoercion.ImplicitTypeCasts,
+      Multiply(CaseWhen(Seq((EqualTo(1, 2), Cast(1, DecimalType(34, 24)))),
+        Cast(100, DecimalType(34, 24))), Literal(1)),
+      Multiply(CaseWhen(Seq((EqualTo(1, 2), Cast(1, DecimalType(34, 24)))),
+        Cast(100, DecimalType(34, 24))), Literal(1)))
+
+    ruleTest(AnsiTypeCoercion.ImplicitTypeCasts,
+      Multiply(CaseWhen(Seq((EqualTo(1, 2), Cast(1, DecimalType(34, 24)))),
+        Cast(100, DecimalType(34, 24))), Cast(1, IntegerType)),
+      Multiply(CaseWhen(Seq((EqualTo(1, 2), Cast(1, DecimalType(34, 24)))),
+        Cast(100, DecimalType(34, 24))), Cast(1, IntegerType)))
+  }
+
+  test("SPARK-31468: null types should be casted to decimal types in ImplicitTypeCasts") {
+    Seq(AnyTypeBinaryOperator(_, _), NumericTypeBinaryOperator(_, _)).foreach { binaryOp =>
+      // binaryOp(decimal, null) case
+      ruleTest(AnsiTypeCoercion.ImplicitTypeCasts,
+        binaryOp(Literal.create(null, DecimalType.SYSTEM_DEFAULT),
+          Literal.create(null, NullType)),
+        binaryOp(Literal.create(null, DecimalType.SYSTEM_DEFAULT),
+          Cast(Literal.create(null, NullType), DecimalType.SYSTEM_DEFAULT)))
+
+      // binaryOp(null, decimal) case
+      ruleTest(AnsiTypeCoercion.ImplicitTypeCasts,
+        binaryOp(Literal.create(null, NullType),
+          Literal.create(null, DecimalType.SYSTEM_DEFAULT)),
+        binaryOp(Cast(Literal.create(null, NullType), DecimalType.SYSTEM_DEFAULT),
+          Literal.create(null, DecimalType.SYSTEM_DEFAULT)))
+    }
+  }
+
+  test("SPARK-31761: byte, short and int should be cast to long for IntegralDivide's datatype") {
+    val rules = Seq(FunctionArgumentConversion, Division, ImplicitTypeCasts)
+    // Casts Byte to Long
+    ruleTest(AnsiTypeCoercion.IntegralDivision, IntegralDivide(2.toByte, 1.toByte),
+      IntegralDivide(Cast(2.toByte, LongType), Cast(1.toByte, LongType)))
+    // Casts Short to Long
+    ruleTest(AnsiTypeCoercion.IntegralDivision, IntegralDivide(2.toShort, 1.toShort),
+      IntegralDivide(Cast(2.toShort, LongType), Cast(1.toShort, LongType)))
+    // Casts Integer to Long
+    ruleTest(AnsiTypeCoercion.IntegralDivision, IntegralDivide(2, 1),
+      IntegralDivide(Cast(2, LongType), Cast(1, LongType)))
+    // should not be any change for Long data types
+    ruleTest(AnsiTypeCoercion.IntegralDivision, IntegralDivide(2L, 1L), IntegralDivide(2L, 1L))
+    // one of the operand is byte
+    ruleTest(AnsiTypeCoercion.IntegralDivision, IntegralDivide(2L, 1.toByte),
+      IntegralDivide(2L, Cast(1.toByte, LongType)))
+    // one of the operand is short
+    ruleTest(AnsiTypeCoercion.IntegralDivision, IntegralDivide(2.toShort, 1L),
+      IntegralDivide(Cast(2.toShort, LongType), 1L))
+    // one of the operand is int
+    ruleTest(AnsiTypeCoercion.IntegralDivision, IntegralDivide(2, 1L),
+      IntegralDivide(Cast(2, LongType), 1L))
+  }
+
+  test("Promote string literals") {
+    val rule = AnsiTypeCoercion.PromoteStringLiterals
+    val stringLiteral = Literal("123")
+    val castStringLiteralAsInt = Cast(stringLiteral, IntegerType)
+    val castStringLiteralAsDouble = Cast(stringLiteral, DoubleType)
+    val castStringLiteralAsDate = Cast(stringLiteral, DateType)
+    val castStringLiteralAsTimestamp = Cast(stringLiteral, TimestampType)
+    ruleTest(rule,
+      GreaterThan(stringLiteral, Literal(1)),
+      GreaterThan(castStringLiteralAsInt, Literal(1)))
+    ruleTest(rule,
+      LessThan(Literal(true), stringLiteral),
+      LessThan(Literal(true), Cast(stringLiteral, BooleanType)))
+    ruleTest(rule,
+      EqualTo(Literal(Array(1, 2)), stringLiteral),
+      EqualTo(Literal(Array(1, 2)), stringLiteral))
+    ruleTest(rule,
+      GreaterThan(stringLiteral, Literal(0.5)),
+      GreaterThan(castStringLiteralAsDouble, Literal(0.5)))
+
+    val dateLiteral = Literal(java.sql.Date.valueOf("2021-01-01"))
+    ruleTest(rule,
+      EqualTo(stringLiteral, dateLiteral),
+      EqualTo(castStringLiteralAsDate, dateLiteral))
+
+    val timestampLiteral = Literal(Timestamp.valueOf("2021-01-01 00:00:00"))
+    ruleTest(rule,
+      EqualTo(stringLiteral, timestampLiteral),
+      EqualTo(castStringLiteralAsTimestamp, timestampLiteral))
+
+    ruleTest(rule, Add(stringLiteral, Literal(1)),
+      Add(castStringLiteralAsInt, Literal(1)))
+    ruleTest(rule, Divide(stringLiteral, Literal(1)),
+      Divide(castStringLiteralAsInt, Literal(1)))
+
+    ruleTest(rule,
+      In(Literal(1), Seq(stringLiteral, Literal(2))),
+      In(Literal(1), Seq(castStringLiteralAsInt, Literal(2))))
+    ruleTest(rule,
+      In(Literal(1.0), Seq(stringLiteral, Literal(2.2))),
+      In(Literal(1.0), Seq(castStringLiteralAsDouble, Literal(2.2))))
+    ruleTest(rule,
+      In(dateLiteral, Seq(stringLiteral)),
+      In(dateLiteral, Seq(castStringLiteralAsDate)))
+    ruleTest(rule,
+      In(timestampLiteral, Seq(stringLiteral)),
+      In(timestampLiteral, Seq(castStringLiteralAsTimestamp)))
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/CreateTablePartitioningValidationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/CreateTablePartitioningValidationSuite.scala
index f433229595e9e..f7e57e3b27b21 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/CreateTablePartitioningValidationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/CreateTablePartitioningValidationSuite.scala
@@ -19,9 +19,8 @@ package org.apache.spark.sql.catalyst.analysis
 
 import org.apache.spark.sql.catalyst.expressions.AttributeReference
 import org.apache.spark.sql.catalyst.plans.logical.{CreateTableAsSelect, LeafNode}
-import org.apache.spark.sql.connector.InMemoryTableCatalog
-import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog}
-import org.apache.spark.sql.connector.expressions.{Expressions, LogicalExpressions}
+import org.apache.spark.sql.connector.catalog.{Identifier, InMemoryTableCatalog, TableCatalog}
+import org.apache.spark.sql.connector.expressions.Expressions
 import org.apache.spark.sql.types.{DoubleType, LongType, StringType, StructType}
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DataSourceV2AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DataSourceV2AnalysisSuite.scala
index c01dea96fe2de..5065276747ad3 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DataSourceV2AnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DataSourceV2AnalysisSuite.scala
@@ -21,7 +21,7 @@ import java.net.URI
 import java.util.Locale
 
 import org.apache.spark.sql.catalyst.catalog.{CatalogDatabase, InMemoryCatalog, SessionCatalog}
-import org.apache.spark.sql.catalyst.expressions.{Alias, AnsiCast, AttributeReference, Cast, Expression, LessThanOrEqual, Literal}
+import org.apache.spark.sql.catalyst.expressions.{Alias, AnsiCast, AttributeReference, Cast, LessThanOrEqual, Literal}
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy
@@ -114,28 +114,50 @@ case class TestRelationAcceptAnySchema(output: Seq[AttributeReference])
 }
 
 abstract class DataSourceV2ANSIAnalysisSuite extends DataSourceV2AnalysisBaseSuite {
-  override def getSQLConf(caseSensitive: Boolean): SQLConf =
-    super.getSQLConf(caseSensitive)
-      .copy(SQLConf.STORE_ASSIGNMENT_POLICY -> StoreAssignmentPolicy.ANSI)
-
 
   // For Ansi store assignment policy, expression `AnsiCast` is used instead of `Cast`.
   override def checkAnalysis(
       inputPlan: LogicalPlan,
       expectedPlan: LogicalPlan,
-      caseSensitive: Boolean): Unit = {
+      caseSensitive: Boolean = true): Unit = {
     val expectedPlanWithAnsiCast = expectedPlan transformAllExpressions {
       case c: Cast => AnsiCast(c.child, c.dataType, c.timeZoneId)
       case other => other
     }
-    super.checkAnalysis(inputPlan, expectedPlanWithAnsiCast, caseSensitive)
+
+    withSQLConf(SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.ANSI.toString) {
+      super.checkAnalysis(inputPlan, expectedPlanWithAnsiCast, caseSensitive)
+    }
+  }
+
+  override def assertAnalysisError(
+      inputPlan: LogicalPlan,
+      expectedErrors: Seq[String],
+      caseSensitive: Boolean = true): Unit = {
+    withSQLConf(SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.ANSI.toString) {
+      super.assertAnalysisError(inputPlan, expectedErrors, caseSensitive)
+    }
   }
 }
 
 abstract class DataSourceV2StrictAnalysisSuite extends DataSourceV2AnalysisBaseSuite {
-  override def getSQLConf(caseSensitive: Boolean): SQLConf =
-    super.getSQLConf(caseSensitive)
-      .copy(SQLConf.STORE_ASSIGNMENT_POLICY -> StoreAssignmentPolicy.STRICT)
+  override def checkAnalysis(
+      inputPlan: LogicalPlan,
+      expectedPlan: LogicalPlan,
+      caseSensitive: Boolean = true): Unit = {
+    withSQLConf(SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.STRICT.toString) {
+      super.checkAnalysis(inputPlan, expectedPlan, caseSensitive)
+    }
+  }
+
+  override def assertAnalysisError(
+      inputPlan: LogicalPlan,
+      expectedErrors: Seq[String],
+      caseSensitive: Boolean = true): Unit = {
+    withSQLConf(SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.STRICT.toString) {
+      super.assertAnalysisError(inputPlan, expectedErrors, caseSensitive)
+    }
+  }
 
   test("byName: fail canWrite check") {
     val parsedPlan = byName(table, widerTable)
@@ -143,7 +165,7 @@ abstract class DataSourceV2StrictAnalysisSuite extends DataSourceV2AnalysisBaseS
     assertNotResolved(parsedPlan)
     assertAnalysisError(parsedPlan, Seq(
       "Cannot write", "'table-name'",
-      "Cannot safely cast", "'x'", "'y'", "DoubleType to FloatType"))
+      "Cannot safely cast", "'x'", "'y'", "double to float"))
   }
 
   test("byName: multiple field errors are reported") {
@@ -160,7 +182,7 @@ abstract class DataSourceV2StrictAnalysisSuite extends DataSourceV2AnalysisBaseS
     assertNotResolved(parsedPlan)
     assertAnalysisError(parsedPlan, Seq(
       "Cannot write incompatible data to table", "'table-name'",
-      "Cannot safely cast", "'x'", "DoubleType to FloatType",
+      "Cannot safely cast", "'x'", "double to float",
       "Cannot write nullable values to non-null column", "'x'",
       "Cannot find data for output column", "'y'"))
   }
@@ -176,7 +198,7 @@ abstract class DataSourceV2StrictAnalysisSuite extends DataSourceV2AnalysisBaseS
     assertNotResolved(parsedPlan)
     assertAnalysisError(parsedPlan, Seq(
       "Cannot write", "'table-name'",
-      "Cannot safely cast", "'x'", "'y'", "DoubleType to FloatType"))
+      "Cannot safely cast", "'x'", "'y'", "double to float"))
   }
 
   test("byPosition: multiple field errors are reported") {
@@ -194,22 +216,18 @@ abstract class DataSourceV2StrictAnalysisSuite extends DataSourceV2AnalysisBaseS
     assertAnalysisError(parsedPlan, Seq(
       "Cannot write incompatible data to table", "'table-name'",
       "Cannot write nullable values to non-null column", "'x'",
-      "Cannot safely cast", "'x'", "DoubleType to FloatType"))
+      "Cannot safely cast", "'x'", "double to float"))
   }
 }
 
 abstract class DataSourceV2AnalysisBaseSuite extends AnalysisTest {
 
-  protected def getSQLConf(caseSensitive: Boolean): SQLConf =
-    new SQLConf().copy(SQLConf.CASE_SENSITIVE -> caseSensitive)
-
-  override def getAnalyzer(caseSensitive: Boolean): Analyzer = {
-    val conf = getSQLConf(caseSensitive)
-    val catalog = new SessionCatalog(new InMemoryCatalog, FunctionRegistry.builtin, conf)
+  override def getAnalyzer: Analyzer = {
+    val catalog = new SessionCatalog(new InMemoryCatalog, FunctionRegistry.builtin)
     catalog.createDatabase(
       CatalogDatabase("default", "", new URI("loc"), Map.empty),
       ignoreIfExists = false)
-    new Analyzer(catalog, conf) {
+    new Analyzer(catalog) {
       override val extendedResolutionRules = EliminateSubqueryAliases :: Nil
     }
   }
@@ -230,6 +248,77 @@ abstract class DataSourceV2AnalysisBaseSuite extends AnalysisTest {
 
   def byPosition(table: NamedRelation, query: LogicalPlan): LogicalPlan
 
+  test("SPARK-33136: output resolved on complex types for V2 write commands") {
+    def assertTypeCompatibility(name: String, fromType: DataType, toType: DataType): Unit = {
+      val table = TestRelation(StructType(Seq(StructField("a", toType))).toAttributes)
+      val query = TestRelation(StructType(Seq(StructField("a", fromType))).toAttributes)
+      val parsedPlan = byName(table, query)
+      assertResolved(parsedPlan)
+      checkAnalysis(parsedPlan, parsedPlan)
+    }
+
+    // The major difference between `from` and `to` is that `from` is a complex type
+    // with non-nullable, whereas `to` is same data type with flipping nullable.
+
+    // nested struct type
+    val fromStructType = StructType(Array(
+      StructField("s", StringType),
+      StructField("i_nonnull", IntegerType, nullable = false),
+      StructField("st", StructType(Array(
+        StructField("l", LongType),
+        StructField("s_nonnull", StringType, nullable = false))))))
+
+    val toStructType = StructType(Array(
+      StructField("s", StringType),
+      StructField("i_nonnull", IntegerType),
+      StructField("st", StructType(Array(
+        StructField("l", LongType),
+        StructField("s_nonnull", StringType))))))
+
+    assertTypeCompatibility("struct", fromStructType, toStructType)
+
+    // array type
+    assertTypeCompatibility("array", ArrayType(LongType, containsNull = false),
+      ArrayType(LongType, containsNull = true))
+
+    // array type with struct type
+    val fromArrayWithStructType = ArrayType(
+      StructType(Array(StructField("s", StringType, nullable = false))),
+      containsNull = false)
+
+    val toArrayWithStructType = ArrayType(
+      StructType(Array(StructField("s", StringType))),
+      containsNull = true)
+
+    assertTypeCompatibility("array_struct", fromArrayWithStructType, toArrayWithStructType)
+
+    // map type
+    assertTypeCompatibility("map", MapType(IntegerType, StringType, valueContainsNull = false),
+      MapType(IntegerType, StringType, valueContainsNull = true))
+
+    // map type with struct type
+    val fromMapWithStructType = MapType(
+      IntegerType,
+      StructType(Array(StructField("s", StringType, nullable = false))),
+      valueContainsNull = false)
+
+    val toMapWithStructType = MapType(
+      IntegerType,
+      StructType(Array(StructField("s", StringType))),
+      valueContainsNull = true)
+
+    assertTypeCompatibility("map_struct", fromMapWithStructType, toMapWithStructType)
+  }
+
+  test("skipSchemaResolution should still require query to be resolved") {
+    val table = TestRelationAcceptAnySchema(StructType(Seq(
+      StructField("a", FloatType),
+      StructField("b", DoubleType))).toAttributes)
+    val query = UnresolvedRelation(Seq("t"))
+    val parsedPlan = byName(table, query)
+    assertNotResolved(parsedPlan)
+  }
+
   test("byName: basic behavior") {
     val query = TestRelation(table.schema.toAttributes)
 
@@ -579,9 +668,7 @@ abstract class DataSourceV2AnalysisBaseSuite extends AnalysisTest {
         Alias(Cast(a, DoubleType, Some(conf.sessionLocalTimeZone)), "x")(),
         Alias(Cast(b, DoubleType, Some(conf.sessionLocalTimeZone)), "y")()),
         query),
-      LessThanOrEqual(
-        AttributeReference("x", DoubleType, nullable = false)(x.exprId),
-        Literal(15.0d)))
+      LessThanOrEqual(x, Literal(15.0d)))
 
     assertNotResolved(parsedPlan)
     checkAnalysis(parsedPlan, expectedPlan)
@@ -589,7 +676,7 @@ abstract class DataSourceV2AnalysisBaseSuite extends AnalysisTest {
   }
 
   protected def testNotResolvedOverwriteByExpression(): Unit = {
-    val xRequiredTable = TestRelation(StructType(Seq(
+    val table = TestRelation(StructType(Seq(
       StructField("x", DoubleType, nullable = false),
       StructField("y", DoubleType))).toAttributes)
 
@@ -598,10 +685,19 @@ abstract class DataSourceV2AnalysisBaseSuite extends AnalysisTest {
       StructField("b", DoubleType))).toAttributes)
 
     // the write is resolved (checked above). this test plan is not because of the expression.
-    val parsedPlan = OverwriteByExpression.byPosition(xRequiredTable, query,
+    val parsedPlan = OverwriteByExpression.byPosition(table, query,
       LessThanOrEqual(UnresolvedAttribute(Seq("a")), Literal(15.0d)))
 
     assertNotResolved(parsedPlan)
-    assertAnalysisError(parsedPlan, Seq("cannot resolve", "`a`", "given input columns", "x, y"))
+    assertAnalysisError(parsedPlan, Seq("cannot resolve", "a", "given input columns", "x, y"))
+
+    val tableAcceptAnySchema = TestRelationAcceptAnySchema(StructType(Seq(
+      StructField("x", DoubleType, nullable = false),
+      StructField("y", DoubleType))).toAttributes)
+
+    val parsedPlan2 = OverwriteByExpression.byPosition(tableAcceptAnySchema, query,
+      LessThanOrEqual(UnresolvedAttribute(Seq("a")), Literal(15.0d)))
+    assertNotResolved(parsedPlan2)
+    assertAnalysisError(parsedPlan2, Seq("cannot resolve", "a", "given input columns", "x, y"))
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecisionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecisionSuite.scala
index c3e18c7f9557f..834f38a3ad190 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecisionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecisionSuite.scala
@@ -24,15 +24,14 @@ import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.Literal.{FalseLiteral, TrueLiteral}
 import org.apache.spark.sql.catalyst.expressions.aggregate._
-import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, Project, Union}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 
 class DecimalPrecisionSuite extends AnalysisTest with BeforeAndAfter {
-  private val catalog = new SessionCatalog(new InMemoryCatalog, EmptyFunctionRegistry, conf)
-  private val analyzer = new Analyzer(catalog, conf)
+  private val catalog = new SessionCatalog(new InMemoryCatalog, EmptyFunctionRegistry)
+  private val analyzer = new Analyzer(catalog)
 
   private val relation = LocalRelation(
     AttributeReference("i", IntegerType)(),
@@ -50,10 +49,6 @@ class DecimalPrecisionSuite extends AnalysisTest with BeforeAndAfter {
   private val f: Expression = UnresolvedAttribute("f")
   private val b: Expression = UnresolvedAttribute("b")
 
-  before {
-    catalog.createTempView("table", relation, overrideIfExists = true)
-  }
-
   private def checkType(expression: Expression, expectedType: DataType): Unit = {
     val plan = Project(Seq(Alias(expression, "c")()), relation)
     assert(analyzer.execute(plan).schema.fields(0).dataType === expectedType)
@@ -73,7 +68,7 @@ class DecimalPrecisionSuite extends AnalysisTest with BeforeAndAfter {
       Union(Project(Seq(Alias(left, "l")()), relation),
         Project(Seq(Alias(right, "r")()), relation))
     val (l, r) = analyzer.execute(plan).collect {
-      case Union(Seq(child1, child2)) => (child1.output.head, child2.output.head)
+      case Union(Seq(child1, child2), _, _) => (child1.output.head, child2.output.head)
     }.head
     assert(l.dataType === expectedType)
     assert(r.dataType === expectedType)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
index 46634c93148b1..9ba03efed1839 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
@@ -78,9 +78,9 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite {
     assertErrorForDifferingTypes(BitwiseXor(Symbol("intField"), Symbol("booleanField")))
 
     assertError(Add(Symbol("booleanField"), Symbol("booleanField")),
-      "requires (numeric or interval) type")
+      "requires (numeric or interval or interval day to second or interval year to month) type")
     assertError(Subtract(Symbol("booleanField"), Symbol("booleanField")),
-      "requires (numeric or interval) type")
+      "requires (numeric or interval or interval day to second or interval year to month) type")
     assertError(Multiply(Symbol("booleanField"), Symbol("booleanField")), "requires numeric type")
     assertError(Divide(Symbol("booleanField"), Symbol("booleanField")),
       "requires (double or decimal) type")
@@ -158,8 +158,9 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite {
 
     assertError(Min(Symbol("mapField")), "min does not support ordering on type")
     assertError(Max(Symbol("mapField")), "max does not support ordering on type")
-    assertError(Sum(Symbol("booleanField")), "function sum requires numeric type")
-    assertError(Average(Symbol("booleanField")), "function average requires numeric type")
+    assertError(Sum(Symbol("booleanField")), "function sum requires numeric or interval types")
+    assertError(Average(Symbol("booleanField")),
+      "function average requires numeric or interval types")
   }
 
   test("check types for others") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/LookupFunctionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/LookupFunctionsSuite.scala
index cea0f2a9cbc97..85e0b1062c81f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/LookupFunctionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/LookupFunctionsSuite.scala
@@ -24,19 +24,18 @@ import org.apache.spark.sql.catalyst.catalog.{CatalogDatabase, InMemoryCatalog,
 import org.apache.spark.sql.catalyst.expressions.Alias
 import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
 
 class LookupFunctionsSuite extends PlanTest {
 
   test("SPARK-23486: the functionExists for the Persistent function check") {
     val externalCatalog = new CustomInMemoryCatalog
-    val conf = new SQLConf()
-    val catalog = new SessionCatalog(externalCatalog, FunctionRegistry.builtin, conf)
+    val catalog = new SessionCatalog(externalCatalog, FunctionRegistry.builtin)
     val analyzer = {
       catalog.createDatabase(
         CatalogDatabase("default", "", new URI("loc"), Map.empty),
         ignoreIfExists = false)
-      new Analyzer(catalog, conf)
+      new Analyzer(catalog)
     }
 
     def table(ref: String): LogicalPlan = UnresolvedRelation(TableIdentifier(ref))
@@ -51,19 +50,18 @@ class LookupFunctionsSuite extends PlanTest {
 
     assert(externalCatalog.getFunctionExistsCalledTimes == 1)
     assert(analyzer.LookupFunctions.normalizeFuncName
-      (unresolvedPersistentFunc.name).database == Some("default"))
+      (unresolvedPersistentFunc.nameParts.asFunctionIdentifier).database == Some("default"))
   }
 
   test("SPARK-23486: the functionExists for the Registered function check") {
     val externalCatalog = new InMemoryCatalog
-    val conf = new SQLConf()
     val customerFunctionReg = new CustomerFunctionRegistry
-    val catalog = new SessionCatalog(externalCatalog, customerFunctionReg, conf)
+    val catalog = new SessionCatalog(externalCatalog, customerFunctionReg)
     val analyzer = {
       catalog.createDatabase(
         CatalogDatabase("default", "", new URI("loc"), Map.empty),
         ignoreIfExists = false)
-      new Analyzer(catalog, conf)
+      new Analyzer(catalog)
     }
 
     def table(ref: String): LogicalPlan = UnresolvedRelation(TableIdentifier(ref))
@@ -73,9 +71,9 @@ class LookupFunctionsSuite extends PlanTest {
       table("TaBlE"))
     analyzer.LookupFunctions.apply(plan)
 
-    assert(customerFunctionReg.getIsRegisteredFunctionCalledTimes == 2)
+    assert(customerFunctionReg.getIsRegisteredFunctionCalledTimes == 4)
     assert(analyzer.LookupFunctions.normalizeFuncName
-      (unresolvedRegisteredFunc.name).database == Some("default"))
+      (unresolvedRegisteredFunc.nameParts.asFunctionIdentifier).database == Some("default"))
   }
 }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveAliasesSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveAliasesSuite.scala
new file mode 100644
index 0000000000000..498aeb28ea42f
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveAliasesSuite.scala
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import java.sql.Date
+
+import scala.collection.mutable.ArrayBuffer
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan, Project}
+import org.apache.spark.sql.types.{DoubleType, IntegerType, LongType, StringType}
+
+class ResolveAliasesSuite extends AnalysisTest {
+
+  private lazy val t1 = LocalRelation("a".attr.int)
+  private lazy val t2 = LocalRelation("b".attr.long)
+
+  private def checkAliasName(plan: LogicalPlan, expected: String): Unit = {
+    val analyzed = getAnalyzer.execute(plan)
+    val actual = analyzed.find(_.isInstanceOf[Project]).get.asInstanceOf[Project]
+      .projectList.head.asInstanceOf[Alias].name
+    assert(actual == expected)
+  }
+
+  private def checkSubqueryAliasName(plan: LogicalPlan, expected: String): Unit = {
+    val analyzed = getAnalyzer.execute(plan)
+    val subqueryExpression = new ArrayBuffer[SubqueryExpression]()
+    analyzed.transformExpressions {
+      case e: SubqueryExpression =>
+        subqueryExpression.append(e)
+        e
+    }
+    assert(subqueryExpression.length == 1)
+    val actual = subqueryExpression.head.plan.find(_.isInstanceOf[Project]).get
+      .asInstanceOf[Project].projectList.head.asInstanceOf[Alias].name
+    assert(actual == expected)
+  }
+
+  test("SPARK-33989: test unary expression") {
+    checkAliasName(t1.select(Floor(Literal(null))), "FLOOR(NULL)")
+    checkAliasName(t1.select(Floor("a".attr)), "FLOOR(a)")
+    checkAliasName(t1.select(Floor("a".attr.cast(DoubleType))), "FLOOR(CAST(a AS DOUBLE))")
+  }
+
+  test("SPARK-33989: test binary expression") {
+    checkAliasName(t1.select(EqualTo("a".attr, Literal(null))), "(a = NULL)")
+    checkAliasName(t1.select(EqualTo("a".attr.cast(LongType), Literal(1))),
+      "(CAST(a AS BIGINT) = 1)")
+    checkAliasName(t1.select(EqualTo("a".attr.cast(LongType), Literal("2").cast(LongType))),
+      "(CAST(a AS BIGINT) = CAST(2 AS BIGINT))")
+  }
+
+  test("SPARK-33989: test nested expression") {
+    checkAliasName(t1.select(StringSplit("a".attr + 1, ",", Literal(-1))),
+      "split((a + 1), ,, -1)")
+    checkAliasName(t1.select(StringSplit(("a".attr + 1).cast(StringType), ",", Literal(-1))),
+      "split(CAST((a + 1) AS STRING), ,, -1)")
+  }
+
+  test("SPARK-33989: test subquery expression") {
+    checkSubqueryAliasName(
+     t1.select(ScalarSubquery(t2.select(EqualTo("b".attr, Literal(null))))),
+     "(b = NULL)")
+    checkSubqueryAliasName(
+      t1.select(ScalarSubquery(t2.select(EqualTo("b".attr.cast(IntegerType), Literal(1))))),
+      "(CAST(b AS INT) = 1)")
+  }
+
+  test("SPARK-34150: Strip Null literal.sql in resolve alias") {
+    checkAliasName(t1.select(Rand(Literal(null))), "rand(NULL)")
+    checkAliasName(t1.select(DateSub(Literal(Date.valueOf("2021-01-18")), Literal(null))),
+      "date_sub(DATE '2021-01-18', NULL)")
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveGroupingAnalyticsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveGroupingAnalyticsSuite.scala
index 7284a6a30ef7e..ae36ab95b6939 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveGroupingAnalyticsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveGroupingAnalyticsSuite.scala
@@ -42,7 +42,7 @@ class ResolveGroupingAnalyticsSuite extends AnalysisTest {
 
   test("rollupExprs") {
     val testRollup = (exprs: Seq[Expression], rollup: Seq[Seq[Expression]]) => {
-      val result = SimpleAnalyzer.ResolveGroupingAnalytics.rollupExprs(exprs)
+      val result = BaseGroupingSets.rollupExprs(exprs.map(Seq(_)))
       assert(result.sortBy(_.hashCode) == rollup.sortBy(_.hashCode))
     }
 
@@ -54,7 +54,7 @@ class ResolveGroupingAnalyticsSuite extends AnalysisTest {
 
   test("cubeExprs") {
     val testCube = (exprs: Seq[Expression], cube: Seq[Seq[Expression]]) => {
-      val result = SimpleAnalyzer.ResolveGroupingAnalytics.cubeExprs(exprs)
+      val result = BaseGroupingSets.cubeExprs(exprs.map(Seq(_)))
       assert(result.sortBy(_.hashCode) == cube.sortBy(_.hashCode))
     }
 
@@ -67,9 +67,9 @@ class ResolveGroupingAnalyticsSuite extends AnalysisTest {
   }
 
   test("grouping sets") {
-    val originalPlan = GroupingSets(Seq(Seq(), Seq(unresolved_a), Seq(unresolved_a, unresolved_b)),
-      Seq(unresolved_a, unresolved_b), r1,
-      Seq(unresolved_a, unresolved_b, UnresolvedAlias(count(unresolved_c))))
+    val originalPlan = Aggregate(
+      Seq(GroupingSets(Seq(Seq(), Seq(unresolved_a), Seq(unresolved_a, unresolved_b)))),
+        Seq(unresolved_a, unresolved_b, UnresolvedAlias(count(unresolved_c))), r1)
     val expected = Aggregate(Seq(a, b, gid), Seq(a, b, count(c).as("count(c)")),
       Expand(
         Seq(Seq(a, b, c, nulInt, nulStr, 3L), Seq(a, b, c, a, nulStr, 1L), Seq(a, b, c, a, b, 0L)),
@@ -77,8 +77,9 @@ class ResolveGroupingAnalyticsSuite extends AnalysisTest {
         Project(Seq(a, b, c, a.as("a"), b.as("b")), r1)))
     checkAnalysis(originalPlan, expected)
 
-    val originalPlan2 = GroupingSets(Seq(), Seq(unresolved_a, unresolved_b), r1,
-      Seq(unresolved_a, unresolved_b, UnresolvedAlias(count(unresolved_c))))
+    val originalPlan2 = Aggregate(
+      Seq(GroupingSets(Seq(), Seq(unresolved_a, unresolved_b))),
+      Seq(unresolved_a, unresolved_b, UnresolvedAlias(count(unresolved_c))), r1)
     val expected2 = Aggregate(Seq(a, b, gid), Seq(a, b, count(c).as("count(c)")),
       Expand(
         Seq(),
@@ -86,31 +87,27 @@ class ResolveGroupingAnalyticsSuite extends AnalysisTest {
         Project(Seq(a, b, c, a.as("a"), b.as("b")), r1)))
     checkAnalysis(originalPlan2, expected2)
 
-    val originalPlan3 = GroupingSets(Seq(Seq(), Seq(unresolved_a), Seq(unresolved_a, unresolved_b),
-      Seq(unresolved_c)), Seq(unresolved_a, unresolved_b), r1,
-      Seq(unresolved_a, unresolved_b, UnresolvedAlias(count(unresolved_c))))
-    assertAnalysisError(originalPlan3, Seq("doesn't show up in the GROUP BY list"))
-  }
-
-  test("grouping sets with no explicit group by expressions") {
-    val originalPlan = GroupingSets(Seq(Seq(), Seq(unresolved_a), Seq(unresolved_a, unresolved_b)),
-      Nil, r1,
-      Seq(unresolved_a, unresolved_b, UnresolvedAlias(count(unresolved_c))))
-    val expected = Aggregate(Seq(a, b, gid), Seq(a, b, count(c).as("count(c)")),
+    // `b` should be included in the GROUP BY expressions even though it's not in the grouping sets.
+    val originalPlan3 = Aggregate(
+      Seq(GroupingSets(Seq(Seq(), Seq(unresolved_a)), Seq(unresolved_a, unresolved_b))),
+      Seq(unresolved_a, unresolved_b, UnresolvedAlias(count(unresolved_c))), r1)
+    val expected3 = Aggregate(Seq(a, b, gid), Seq(a, b, count(c).as("count(c)")),
       Expand(
-        Seq(Seq(a, b, c, nulInt, nulStr, 3L), Seq(a, b, c, a, nulStr, 1L), Seq(a, b, c, a, b, 0L)),
+        Seq(Seq(a, b, c, nulInt, nulStr, 3L), Seq(a, b, c, a, nulStr, 1L)),
         Seq(a, b, c, a, b, gid),
         Project(Seq(a, b, c, a.as("a"), b.as("b")), r1)))
-    checkAnalysis(originalPlan, expected)
+    checkAnalysis(originalPlan3, expected3)
 
     // Computation of grouping expression should remove duplicate expression based on their
     // semantics (semanticEqual).
-    val originalPlan2 = GroupingSets(Seq(Seq(Multiply(unresolved_a, Literal(2))),
-      Seq(Multiply(Literal(2), unresolved_a), unresolved_b)), Nil, r1,
+    val originalPlan4 = Aggregate(
+      Seq(GroupingSets(Seq(
+        Seq(Multiply(unresolved_a, Literal(2))),
+        Seq(Multiply(Literal(2), unresolved_a), unresolved_b)))),
       Seq(UnresolvedAlias(Multiply(unresolved_a, Literal(2))),
-        unresolved_b, UnresolvedAlias(count(unresolved_c))))
+        unresolved_b, UnresolvedAlias(count(unresolved_c))), r1)
 
-    val resultPlan = getAnalyzer(true).executeAndCheck(originalPlan2, new QueryPlanningTracker)
+    val resultPlan = getAnalyzer.executeAndCheck(originalPlan4, new QueryPlanningTracker)
     val gExpressions = resultPlan.asInstanceOf[Aggregate].groupingExpressions
     assert(gExpressions.size == 3)
     val firstGroupingExprAttrName =
@@ -121,7 +118,7 @@ class ResolveGroupingAnalyticsSuite extends AnalysisTest {
   }
 
   test("cube") {
-    val originalPlan = Aggregate(Seq(Cube(Seq(unresolved_a, unresolved_b))),
+    val originalPlan = Aggregate(Seq(Cube(Seq(Seq(unresolved_a), Seq(unresolved_b)))),
       Seq(unresolved_a, unresolved_b, UnresolvedAlias(count(unresolved_c))), r1)
     val expected = Aggregate(Seq(a, b, gid), Seq(a, b, count(c).as("count(c)")),
       Expand(
@@ -141,7 +138,7 @@ class ResolveGroupingAnalyticsSuite extends AnalysisTest {
   }
 
   test("rollup") {
-    val originalPlan = Aggregate(Seq(Rollup(Seq(unresolved_a, unresolved_b))),
+    val originalPlan = Aggregate(Seq(Rollup(Seq(Seq(unresolved_a), Seq(unresolved_b)))),
       Seq(unresolved_a, unresolved_b, UnresolvedAlias(count(unresolved_c))), r1)
     val expected = Aggregate(Seq(a, b, gid), Seq(a, b, count(c).as("count(c)")),
       Expand(
@@ -161,10 +158,10 @@ class ResolveGroupingAnalyticsSuite extends AnalysisTest {
 
   test("grouping function") {
     // GrouingSets
-    val originalPlan = GroupingSets(Seq(Seq(), Seq(unresolved_a), Seq(unresolved_a, unresolved_b)),
-      Seq(unresolved_a, unresolved_b), r1,
+    val originalPlan = Aggregate(
+      Seq(GroupingSets(Seq(Seq(), Seq(unresolved_a), Seq(unresolved_a, unresolved_b)))),
       Seq(unresolved_a, unresolved_b, UnresolvedAlias(count(unresolved_c)),
-        UnresolvedAlias(Grouping(unresolved_a))))
+        UnresolvedAlias(Grouping(unresolved_a))), r1)
     val expected = Aggregate(Seq(a, b, gid),
       Seq(a, b, count(c).as("count(c)"), grouping_a.as("grouping(a)")),
       Expand(
@@ -174,7 +171,7 @@ class ResolveGroupingAnalyticsSuite extends AnalysisTest {
     checkAnalysis(originalPlan, expected)
 
     // Cube
-    val originalPlan2 = Aggregate(Seq(Cube(Seq(unresolved_a, unresolved_b))),
+    val originalPlan2 = Aggregate(Seq(Cube(Seq(Seq(unresolved_a), Seq(unresolved_b)))),
       Seq(unresolved_a, unresolved_b, UnresolvedAlias(count(unresolved_c)),
         UnresolvedAlias(Grouping(unresolved_a))), r1)
     val expected2 = Aggregate(Seq(a, b, gid),
@@ -187,7 +184,7 @@ class ResolveGroupingAnalyticsSuite extends AnalysisTest {
     checkAnalysis(originalPlan2, expected2)
 
     // Rollup
-    val originalPlan3 = Aggregate(Seq(Rollup(Seq(unresolved_a, unresolved_b))),
+    val originalPlan3 = Aggregate(Seq(Rollup(Seq(Seq(unresolved_a), Seq(unresolved_b)))),
       Seq(unresolved_a, unresolved_b, UnresolvedAlias(count(unresolved_c)),
         UnresolvedAlias(Grouping(unresolved_a))), r1)
     val expected3 = Aggregate(Seq(a, b, gid),
@@ -201,10 +198,10 @@ class ResolveGroupingAnalyticsSuite extends AnalysisTest {
 
   test("grouping_id") {
     // GrouingSets
-    val originalPlan = GroupingSets(Seq(Seq(), Seq(unresolved_a), Seq(unresolved_a, unresolved_b)),
-      Seq(unresolved_a, unresolved_b), r1,
+    val originalPlan = Aggregate(
+      Seq(GroupingSets(Seq(Seq(), Seq(unresolved_a), Seq(unresolved_a, unresolved_b)))),
       Seq(unresolved_a, unresolved_b, UnresolvedAlias(count(unresolved_c)),
-        UnresolvedAlias(GroupingID(Seq(unresolved_a, unresolved_b)))))
+        UnresolvedAlias(GroupingID(Seq(unresolved_a, unresolved_b)))), r1)
     val expected = Aggregate(Seq(a, b, gid),
       Seq(a, b, count(c).as("count(c)"), gid.as("grouping_id(a, b)")),
       Expand(
@@ -214,7 +211,7 @@ class ResolveGroupingAnalyticsSuite extends AnalysisTest {
     checkAnalysis(originalPlan, expected)
 
     // Cube
-    val originalPlan2 = Aggregate(Seq(Cube(Seq(unresolved_a, unresolved_b))),
+    val originalPlan2 = Aggregate(Seq(Cube(Seq(Seq(unresolved_a), Seq(unresolved_b)))),
       Seq(unresolved_a, unresolved_b, UnresolvedAlias(count(unresolved_c)),
         UnresolvedAlias(GroupingID(Seq(unresolved_a, unresolved_b)))), r1)
     val expected2 = Aggregate(Seq(a, b, gid),
@@ -227,7 +224,7 @@ class ResolveGroupingAnalyticsSuite extends AnalysisTest {
     checkAnalysis(originalPlan2, expected2)
 
     // Rollup
-    val originalPlan3 = Aggregate(Seq(Rollup(Seq(unresolved_a, unresolved_b))),
+    val originalPlan3 = Aggregate(Seq(Rollup(Seq(Seq(unresolved_a), Seq(unresolved_b)))),
       Seq(unresolved_a, unresolved_b, UnresolvedAlias(count(unresolved_c)),
         UnresolvedAlias(GroupingID(Seq(unresolved_a, unresolved_b)))), r1)
     val expected3 = Aggregate(Seq(a, b, gid),
@@ -242,8 +239,9 @@ class ResolveGroupingAnalyticsSuite extends AnalysisTest {
   test("filter with grouping function") {
     // Filter with Grouping function
     val originalPlan = Filter(Grouping(unresolved_a) === 0,
-      GroupingSets(Seq(Seq(), Seq(unresolved_a), Seq(unresolved_a, unresolved_b)),
-        Seq(unresolved_a, unresolved_b), r1, Seq(unresolved_a, unresolved_b)))
+      Aggregate(
+        Seq(GroupingSets(Seq(Seq(), Seq(unresolved_a), Seq(unresolved_a, unresolved_b)))),
+        Seq(unresolved_a, unresolved_b), r1))
     val expected = Project(Seq(a, b),
       Filter(Cast(grouping_a, IntegerType, Option(TimeZone.getDefault().getID)) === 0,
       Aggregate(Seq(a, b, gid),
@@ -262,8 +260,9 @@ class ResolveGroupingAnalyticsSuite extends AnalysisTest {
 
     // Filter with GroupingID
     val originalPlan3 = Filter(GroupingID(Seq(unresolved_a, unresolved_b)) === 1L,
-      GroupingSets(Seq(Seq(), Seq(unresolved_a), Seq(unresolved_a, unresolved_b)),
-        Seq(unresolved_a, unresolved_b), r1, Seq(unresolved_a, unresolved_b)))
+      Aggregate(
+        Seq(GroupingSets(Seq(Seq(), Seq(unresolved_a), Seq(unresolved_a, unresolved_b)))),
+        Seq(unresolved_a, unresolved_b), r1))
     val expected3 = Project(Seq(a, b), Filter(gid === 1L,
       Aggregate(Seq(a, b, gid),
         Seq(a, b, gid),
@@ -284,8 +283,9 @@ class ResolveGroupingAnalyticsSuite extends AnalysisTest {
     // Sort with Grouping function
     val originalPlan = Sort(
       Seq(SortOrder(Grouping(unresolved_a), Ascending)), true,
-      GroupingSets(Seq(Seq(), Seq(unresolved_a), Seq(unresolved_a, unresolved_b)),
-        Seq(unresolved_a, unresolved_b), r1, Seq(unresolved_a, unresolved_b)))
+      Aggregate(
+        Seq(GroupingSets(Seq(Seq(), Seq(unresolved_a), Seq(unresolved_a, unresolved_b)))),
+        Seq(unresolved_a, unresolved_b), r1))
     val expected = Project(Seq(a, b), Sort(
       Seq(SortOrder('aggOrder.byte.withNullability(false), Ascending)), true,
       Aggregate(Seq(a, b, gid),
@@ -305,8 +305,9 @@ class ResolveGroupingAnalyticsSuite extends AnalysisTest {
     // Sort with GroupingID
     val originalPlan3 = Sort(
       Seq(SortOrder(GroupingID(Seq(unresolved_a, unresolved_b)), Ascending)), true,
-      GroupingSets(Seq(Seq(), Seq(unresolved_a), Seq(unresolved_a, unresolved_b)),
-        Seq(unresolved_a, unresolved_b), r1, Seq(unresolved_a, unresolved_b)))
+      Aggregate(
+        Seq(GroupingSets(Seq(Seq(), Seq(unresolved_a), Seq(unresolved_a, unresolved_b)))),
+        Seq(unresolved_a, unresolved_b), r1))
     val expected3 = Project(Seq(a, b), Sort(
       Seq(SortOrder('aggOrder.long.withNullability(false), Ascending)), true,
       Aggregate(Seq(a, b, gid),
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveHintsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveHintsSuite.scala
index d3bd5d07a0932..4b9a2ca94ea50 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveHintsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveHintsSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.catalyst.analysis
 
 import org.apache.log4j.Level
 
+import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.expressions.{Ascending, AttributeReference, Literal, SortOrder}
@@ -31,36 +32,36 @@ class ResolveHintsSuite extends AnalysisTest {
   import org.apache.spark.sql.catalyst.analysis.TestRelations._
 
   test("invalid hints should be ignored") {
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       UnresolvedHint("some_random_hint_that_does_not_exist", Seq("TaBlE"), table("TaBlE")),
       testRelation,
       caseSensitive = false)
   }
 
   test("case-sensitive or insensitive parameters") {
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       UnresolvedHint("MAPJOIN", Seq("TaBlE"), table("TaBlE")),
       ResolvedHint(testRelation, HintInfo(strategy = Some(BROADCAST))),
       caseSensitive = false)
 
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       UnresolvedHint("MAPJOIN", Seq("table"), table("TaBlE")),
       ResolvedHint(testRelation, HintInfo(strategy = Some(BROADCAST))),
       caseSensitive = false)
 
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       UnresolvedHint("MAPJOIN", Seq("TaBlE"), table("TaBlE")),
       ResolvedHint(testRelation, HintInfo(strategy = Some(BROADCAST))),
       caseSensitive = true)
 
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       UnresolvedHint("MAPJOIN", Seq("table"), table("TaBlE")),
       testRelation,
       caseSensitive = true)
   }
 
   test("multiple broadcast hint aliases") {
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       UnresolvedHint("MAPJOIN", Seq("table", "table2"), table("table").join(table("table2"))),
       Join(ResolvedHint(testRelation, HintInfo(strategy = Some(BROADCAST))),
         ResolvedHint(testRelation2, HintInfo(strategy = Some(BROADCAST))),
@@ -69,7 +70,7 @@ class ResolveHintsSuite extends AnalysisTest {
   }
 
   test("do not traverse past existing broadcast hints") {
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       UnresolvedHint("MAPJOIN", Seq("table"),
         ResolvedHint(table("table").where('a > 1), HintInfo(strategy = Some(BROADCAST)))),
       ResolvedHint(testRelation.where('a > 1), HintInfo(strategy = Some(BROADCAST))).analyze,
@@ -77,32 +78,32 @@ class ResolveHintsSuite extends AnalysisTest {
   }
 
   test("should work for subqueries") {
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       UnresolvedHint("MAPJOIN", Seq("tableAlias"), table("table").as("tableAlias")),
       ResolvedHint(testRelation, HintInfo(strategy = Some(BROADCAST))),
       caseSensitive = false)
 
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       UnresolvedHint("MAPJOIN", Seq("tableAlias"), table("table").subquery('tableAlias)),
       ResolvedHint(testRelation, HintInfo(strategy = Some(BROADCAST))),
       caseSensitive = false)
 
     // Negative case: if the alias doesn't match, don't match the original table name.
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       UnresolvedHint("MAPJOIN", Seq("table"), table("table").as("tableAlias")),
       testRelation,
       caseSensitive = false)
   }
 
   test("do not traverse past subquery alias") {
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       UnresolvedHint("MAPJOIN", Seq("table"), table("table").where('a > 1).subquery('tableAlias)),
       testRelation.where('a > 1).analyze,
       caseSensitive = false)
   }
 
   test("should work for CTE") {
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       CatalystSqlParser.parsePlan(
         """
           |WITH ctetable AS (SELECT * FROM table WHERE a > 1)
@@ -115,7 +116,7 @@ class ResolveHintsSuite extends AnalysisTest {
   }
 
   test("should not traverse down CTE") {
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       CatalystSqlParser.parsePlan(
         """
           |WITH ctetable AS (SELECT * FROM table WHERE a > 1)
@@ -127,16 +128,16 @@ class ResolveHintsSuite extends AnalysisTest {
   }
 
   test("coalesce and repartition hint") {
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       UnresolvedHint("COALESCE", Seq(Literal(10)), table("TaBlE")),
       Repartition(numPartitions = 10, shuffle = false, child = testRelation))
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       UnresolvedHint("coalesce", Seq(Literal(20)), table("TaBlE")),
       Repartition(numPartitions = 20, shuffle = false, child = testRelation))
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       UnresolvedHint("REPARTITION", Seq(Literal(100)), table("TaBlE")),
       Repartition(numPartitions = 100, shuffle = true, child = testRelation))
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       UnresolvedHint("RePARTITion", Seq(Literal(200)), table("TaBlE")),
       Repartition(numPartitions = 200, shuffle = true, child = testRelation))
 
@@ -152,20 +153,20 @@ class ResolveHintsSuite extends AnalysisTest {
       UnresolvedHint("COALESCE", Seq(Literal(1.0)), table("TaBlE")),
       Seq(errMsg))
 
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       UnresolvedHint("RePartition", Seq(Literal(10), UnresolvedAttribute("a")), table("TaBlE")),
       RepartitionByExpression(Seq(AttributeReference("a", IntegerType)()), testRelation, 10))
 
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       UnresolvedHint("REPARTITION", Seq(Literal(10), UnresolvedAttribute("a")), table("TaBlE")),
       RepartitionByExpression(Seq(AttributeReference("a", IntegerType)()), testRelation, 10))
 
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       UnresolvedHint("REPARTITION", Seq(UnresolvedAttribute("a")), table("TaBlE")),
       RepartitionByExpression(
-        Seq(AttributeReference("a", IntegerType)()), testRelation, conf.numShufflePartitions))
+        Seq(AttributeReference("a", IntegerType)()), testRelation, None))
 
-    val e = intercept[IllegalArgumentException] {
+    val e = intercept[AnalysisException] {
       checkAnalysis(
         UnresolvedHint("REPARTITION",
           Seq(SortOrder(AttributeReference("a", IntegerType)(), Ascending)),
@@ -176,18 +177,18 @@ class ResolveHintsSuite extends AnalysisTest {
     }
     e.getMessage.contains("For range partitioning use REPARTITION_BY_RANGE instead")
 
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       UnresolvedHint(
         "REPARTITION_BY_RANGE", Seq(Literal(10), UnresolvedAttribute("a")), table("TaBlE")),
       RepartitionByExpression(
         Seq(SortOrder(AttributeReference("a", IntegerType)(), Ascending)), testRelation, 10))
 
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       UnresolvedHint(
         "REPARTITION_BY_RANGE", Seq(UnresolvedAttribute("a")), table("TaBlE")),
       RepartitionByExpression(
         Seq(SortOrder(AttributeReference("a", IntegerType)(), Ascending)),
-        testRelation, conf.numShufflePartitions))
+        testRelation, None))
 
     val errMsg2 = "REPARTITION Hint parameter should include columns, but"
 
@@ -225,7 +226,7 @@ class ResolveHintsSuite extends AnalysisTest {
   test("log warnings for invalid hints") {
     val logAppender = new LogAppender("invalid hints")
     withLogAppender(logAppender) {
-      checkAnalysis(
+      checkAnalysisWithoutViewWrapper(
         UnresolvedHint("unknown_hint", Seq("TaBlE"), table("TaBlE")),
         testRelation,
         caseSensitive = false)
@@ -236,7 +237,7 @@ class ResolveHintsSuite extends AnalysisTest {
   }
 
   test("SPARK-30003: Do not throw stack overflow exception in non-root unknown hint resolution") {
-    checkAnalysis(
+    checkAnalysisWithoutViewWrapper(
       Project(testRelation.output, UnresolvedHint("unknown_hint", Seq("TaBlE"), table("TaBlE"))),
       Project(testRelation.output, testRelation),
       caseSensitive = false)
@@ -248,7 +249,7 @@ class ResolveHintsSuite extends AnalysisTest {
         ("SHUFFLE_HASH", SHUFFLE_HASH),
         ("SHUFFLE_REPLICATE_NL", SHUFFLE_REPLICATE_NL)).foreach { case (hintName, st) =>
       // local temp table (single-part identifier case)
-      checkAnalysis(
+      checkAnalysisWithoutViewWrapper(
         UnresolvedHint(hintName, Seq("table", "table2"),
           table("TaBlE").join(table("TaBlE2"))),
         Join(
@@ -259,7 +260,7 @@ class ResolveHintsSuite extends AnalysisTest {
           JoinHint.NONE),
         caseSensitive = false)
 
-      checkAnalysis(
+      checkAnalysisWithoutViewWrapper(
         UnresolvedHint(hintName, Seq("TaBlE", "table2"),
           table("TaBlE").join(table("TaBlE2"))),
         Join(
@@ -271,7 +272,7 @@ class ResolveHintsSuite extends AnalysisTest {
         caseSensitive = true)
 
       // global temp table (multi-part identifier case)
-      checkAnalysis(
+      checkAnalysisWithoutViewWrapper(
         UnresolvedHint(hintName, Seq("GlOBal_TeMP.table4", "table5"),
           table("global_temp", "table4").join(table("global_temp", "table5"))),
         Join(
@@ -282,7 +283,7 @@ class ResolveHintsSuite extends AnalysisTest {
           JoinHint.NONE),
         caseSensitive = false)
 
-      checkAnalysis(
+      checkAnalysisWithoutViewWrapper(
         UnresolvedHint(hintName, Seq("global_temp.TaBlE4", "table5"),
           table("global_temp", "TaBlE4").join(table("global_temp", "TaBlE5"))),
         Join(
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveInlineTablesSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveInlineTablesSuite.scala
index 9e99c8e11cdfe..16d23153c1c53 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveInlineTablesSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveInlineTablesSuite.scala
@@ -35,53 +35,53 @@ class ResolveInlineTablesSuite extends AnalysisTest with BeforeAndAfter {
   private def lit(v: Any): Literal = Literal(v)
 
   test("validate inputs are foldable") {
-    ResolveInlineTables(conf).validateInputEvaluable(
+    ResolveInlineTables.validateInputEvaluable(
       UnresolvedInlineTable(Seq("c1", "c2"), Seq(Seq(lit(1)))))
 
     // nondeterministic (rand) should not work
     intercept[AnalysisException] {
-      ResolveInlineTables(conf).validateInputEvaluable(
+      ResolveInlineTables.validateInputEvaluable(
         UnresolvedInlineTable(Seq("c1"), Seq(Seq(Rand(1)))))
     }
 
     // aggregate should not work
     intercept[AnalysisException] {
-      ResolveInlineTables(conf).validateInputEvaluable(
+      ResolveInlineTables.validateInputEvaluable(
         UnresolvedInlineTable(Seq("c1"), Seq(Seq(Count(lit(1))))))
     }
 
     // unresolved attribute should not work
     intercept[AnalysisException] {
-      ResolveInlineTables(conf).validateInputEvaluable(
+      ResolveInlineTables.validateInputEvaluable(
         UnresolvedInlineTable(Seq("c1"), Seq(Seq(UnresolvedAttribute("A")))))
     }
   }
 
   test("validate input dimensions") {
-    ResolveInlineTables(conf).validateInputDimension(
+    ResolveInlineTables.validateInputDimension(
       UnresolvedInlineTable(Seq("c1"), Seq(Seq(lit(1)), Seq(lit(2)))))
 
     // num alias != data dimension
     intercept[AnalysisException] {
-      ResolveInlineTables(conf).validateInputDimension(
+      ResolveInlineTables.validateInputDimension(
         UnresolvedInlineTable(Seq("c1", "c2"), Seq(Seq(lit(1)), Seq(lit(2)))))
     }
 
     // num alias == data dimension, but data themselves are inconsistent
     intercept[AnalysisException] {
-      ResolveInlineTables(conf).validateInputDimension(
+      ResolveInlineTables.validateInputDimension(
         UnresolvedInlineTable(Seq("c1"), Seq(Seq(lit(1)), Seq(lit(21), lit(22)))))
     }
   }
 
   test("do not fire the rule if not all expressions are resolved") {
     val table = UnresolvedInlineTable(Seq("c1", "c2"), Seq(Seq(UnresolvedAttribute("A"))))
-    assert(ResolveInlineTables(conf)(table) == table)
+    assert(ResolveInlineTables(table) == table)
   }
 
   test("convert") {
     val table = UnresolvedInlineTable(Seq("c1"), Seq(Seq(lit(1)), Seq(lit(2L))))
-    val converted = ResolveInlineTables(conf).convert(table)
+    val converted = ResolveInlineTables.convert(table)
 
     assert(converted.output.map(_.dataType) == Seq(LongType))
     assert(converted.data.size == 2)
@@ -92,8 +92,8 @@ class ResolveInlineTablesSuite extends AnalysisTest with BeforeAndAfter {
   test("convert TimeZoneAwareExpression") {
     val table = UnresolvedInlineTable(Seq("c1"),
       Seq(Seq(Cast(lit("1991-12-06 00:00:00.0"), TimestampType))))
-    val withTimeZone = ResolveTimeZone(conf).apply(table)
-    val LocalRelation(output, data, _) = ResolveInlineTables(conf).apply(withTimeZone)
+    val withTimeZone = ResolveTimeZone.apply(table)
+    val LocalRelation(output, data, _) = ResolveInlineTables.apply(withTimeZone)
     val correct = Cast(lit("1991-12-06 00:00:00.0"), TimestampType)
       .withTimeZone(conf.sessionLocalTimeZone).eval().asInstanceOf[Long]
     assert(output.map(_.dataType) == Seq(TimestampType))
@@ -103,11 +103,11 @@ class ResolveInlineTablesSuite extends AnalysisTest with BeforeAndAfter {
 
   test("nullability inference in convert") {
     val table1 = UnresolvedInlineTable(Seq("c1"), Seq(Seq(lit(1)), Seq(lit(2L))))
-    val converted1 = ResolveInlineTables(conf).convert(table1)
+    val converted1 = ResolveInlineTables.convert(table1)
     assert(!converted1.schema.fields(0).nullable)
 
     val table2 = UnresolvedInlineTable(Seq("c1"), Seq(Seq(lit(1)), Seq(Literal(null, NullType))))
-    val converted2 = ResolveInlineTables(conf).convert(table2)
+    val converted2 = ResolveInlineTables.convert(table2)
     assert(converted2.schema.fields(0).nullable)
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveLambdaVariablesSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveLambdaVariablesSuite.scala
index a5847ba7c522d..b9233a27f3d7a 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveLambdaVariablesSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveLambdaVariablesSuite.scala
@@ -32,7 +32,7 @@ class ResolveLambdaVariablesSuite extends PlanTest {
   import org.apache.spark.sql.catalyst.dsl.plans._
 
   object Analyzer extends RuleExecutor[LogicalPlan] {
-    val batches = Batch("Resolution", FixedPoint(4), ResolveLambdaVariables(conf)) :: Nil
+    val batches = Batch("Resolution", FixedPoint(4), ResolveLambdaVariables) :: Nil
   }
 
   private val key = 'key.int
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveNaturalJoinSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveNaturalJoinSuite.scala
index e449b9669cc72..ea2284e5420bd 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveNaturalJoinSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveNaturalJoinSuite.scala
@@ -17,7 +17,6 @@
 
 package org.apache.spark.sql.catalyst.analysis
 
-import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.expressions._
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveUnionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveUnionSuite.scala
new file mode 100644
index 0000000000000..5c7ad0067a456
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveUnionSuite.scala
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.catalyst.analysis
+
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types._
+
+class ResolveUnionSuite extends AnalysisTest {
+  test("Resolve Union") {
+    val table1 = LocalRelation(
+      AttributeReference("i", IntegerType)(),
+      AttributeReference("u", DecimalType.SYSTEM_DEFAULT)(),
+      AttributeReference("b", ByteType)(),
+      AttributeReference("d", DoubleType)())
+    val table2 = LocalRelation(
+      AttributeReference("u", DecimalType.SYSTEM_DEFAULT)(),
+      AttributeReference("b", ByteType)(),
+      AttributeReference("d", DoubleType)(),
+      AttributeReference("i", IntegerType)())
+    val table3 = LocalRelation(
+      AttributeReference("u", DecimalType.SYSTEM_DEFAULT)(),
+      AttributeReference("d", DoubleType)(),
+      AttributeReference("i", IntegerType)())
+    val table4 = LocalRelation(
+      AttributeReference("u", DecimalType.SYSTEM_DEFAULT)(),
+      AttributeReference("i", IntegerType)())
+
+    val rules = Seq(ResolveUnion)
+    val analyzer = new RuleExecutor[LogicalPlan] {
+      override val batches = Seq(Batch("Resolution", Once, rules: _*))
+    }
+
+    // By name resolution
+    val union1 = Union(table1 :: table2 :: Nil, true, false)
+    val analyzed1 = analyzer.execute(union1)
+    val projected1 =
+      Project(Seq(table2.output(3), table2.output(0), table2.output(1), table2.output(2)), table2)
+    val expected1 = Union(table1 :: projected1 :: Nil)
+    comparePlans(analyzed1, expected1)
+
+    // Allow missing column
+    val union2 = Union(table1 :: table3 :: Nil, true, true)
+    val analyzed2 = analyzer.execute(union2)
+    val nullAttr1 = Alias(Literal(null, ByteType), "b")()
+    val projected2 =
+      Project(Seq(table2.output(3), table2.output(0), nullAttr1, table2.output(2)), table3)
+    val expected2 = Union(table1 :: projected2 :: Nil)
+    comparePlans(analyzed2, expected2)
+
+    // Allow missing column + Allow missing column
+    val union3 = Union(union2 :: table4 :: Nil, true, true)
+    val analyzed3 = analyzer.execute(union3)
+    val nullAttr2 = Alias(Literal(null, DoubleType), "d")()
+    val projected3 =
+      Project(Seq(table2.output(3), table2.output(0), nullAttr1, nullAttr2), table4)
+    val expected3 = Union(table1 :: projected2 :: projected3 :: Nil)
+    comparePlans(analyzed3, expected3)
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolvedUuidExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolvedUuidExpressionsSuite.scala
index 64bd07534b19b..5ddfa9f2191e0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolvedUuidExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolvedUuidExpressionsSuite.scala
@@ -36,7 +36,7 @@ class ResolvedUuidExpressionsSuite extends AnalysisTest {
   private lazy val uuid1Ref = uuid1.toAttribute
 
   private val tracker = new QueryPlanningTracker
-  private val analyzer = getAnalyzer(caseSensitive = true)
+  private val analyzer = getAnalyzer
 
   private def getUuidExpressions(plan: LogicalPlan): Seq[Uuid] = {
     plan.flatMap {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/StreamingJoinHelperSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/StreamingJoinHelperSuite.scala
index 8cf41a02320d2..7566545f98355 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/StreamingJoinHelperSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/StreamingJoinHelperSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.analysis
 import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, AttributeSet}
 import org.apache.spark.sql.catalyst.optimizer.SimpleTestOptimizer
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
-import org.apache.spark.sql.catalyst.plans.logical.{EventTimeWatermark, Filter, LeafNode, LocalRelation}
+import org.apache.spark.sql.catalyst.plans.logical.{EventTimeWatermark, Filter, LeafNode}
 import org.apache.spark.sql.types.{IntegerType, MetadataBuilder, TimestampType}
 
 class StreamingJoinHelperSuite extends AnalysisTest {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/SubstituteUnresolvedOrdinalsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/SubstituteUnresolvedOrdinalsSuite.scala
index 2331346f325aa..c0312282c76c8 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/SubstituteUnresolvedOrdinalsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/SubstituteUnresolvedOrdinalsSuite.scala
@@ -36,31 +36,35 @@ class SubstituteUnresolvedOrdinalsSuite extends AnalysisTest {
     // Tests order by ordinal, apply single rule.
     val plan = testRelation2.orderBy(Literal(1).asc, Literal(2).asc)
     comparePlans(
-      new SubstituteUnresolvedOrdinals(conf).apply(plan),
+      SubstituteUnresolvedOrdinals.apply(plan),
       testRelation2.orderBy(UnresolvedOrdinal(1).asc, UnresolvedOrdinal(2).asc))
 
     // Tests order by ordinal, do full analysis
     checkAnalysis(plan, testRelation2.orderBy(a.asc, b.asc))
 
     // order by ordinal can be turned off by config
-    comparePlans(
-      new SubstituteUnresolvedOrdinals(conf.copy(SQLConf.ORDER_BY_ORDINAL -> false)).apply(plan),
-      testRelation2.orderBy(Literal(1).asc, Literal(2).asc))
+    withSQLConf(SQLConf.ORDER_BY_ORDINAL.key -> "false") {
+      comparePlans(
+        SubstituteUnresolvedOrdinals.apply(plan),
+        testRelation2.orderBy(Literal(1).asc, Literal(2).asc))
+    }
   }
 
   test("group by ordinal") {
     // Tests group by ordinal, apply single rule.
     val plan2 = testRelation2.groupBy(Literal(1), Literal(2))('a, 'b)
     comparePlans(
-      new SubstituteUnresolvedOrdinals(conf).apply(plan2),
+      SubstituteUnresolvedOrdinals.apply(plan2),
       testRelation2.groupBy(UnresolvedOrdinal(1), UnresolvedOrdinal(2))('a, 'b))
 
     // Tests group by ordinal, do full analysis
     checkAnalysis(plan2, testRelation2.groupBy(a, b)(a, b))
 
     // group by ordinal can be turned off by config
-    comparePlans(
-      new SubstituteUnresolvedOrdinals(conf.copy(SQLConf.GROUP_BY_ORDINAL -> false)).apply(plan2),
-      testRelation2.groupBy(Literal(1), Literal(2))('a, 'b))
+    withSQLConf(SQLConf.GROUP_BY_ORDINAL.key -> "false") {
+      comparePlans(
+        SubstituteUnresolvedOrdinals.apply(plan2),
+        testRelation2.groupBy(Literal(1), Literal(2))('a, 'b))
+    }
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TableLookupCacheSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TableLookupCacheSuite.scala
index eed962cd0f69d..7d6ad3bc60902 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TableLookupCacheSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TableLookupCacheSuite.scala
@@ -19,23 +19,22 @@ package org.apache.spark.sql.catalyst.analysis
 
 import java.io.File
 
+import scala.collection.JavaConverters._
+
 import org.mockito.ArgumentMatchers.any
 import org.mockito.Mockito._
 import org.mockito.invocation.InvocationOnMock
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
 
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.catalog.{CatalogDatabase, CatalogStorageFormat, CatalogTable, CatalogTableType, ExternalCatalog, InMemoryCatalog, SessionCatalog}
 import org.apache.spark.sql.catalyst.dsl.plans._
-import org.apache.spark.sql.connector.InMemoryTableCatalog
-import org.apache.spark.sql.connector.catalog.{CatalogManager, CatalogNotFoundException, Identifier, Table, V1Table}
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.connector.catalog.{CatalogManager, CatalogNotFoundException, Identifier, InMemoryTable, InMemoryTableCatalog, Table}
 import org.apache.spark.sql.types._
 
 class TableLookupCacheSuite extends AnalysisTest with Matchers {
   private def getAnalyzer(externalCatalog: ExternalCatalog, databasePath: File): Analyzer = {
-    val conf = new SQLConf()
-    val v1Catalog = new SessionCatalog(externalCatalog, FunctionRegistry.builtin, conf)
+    val v1Catalog = new SessionCatalog(externalCatalog, FunctionRegistry.builtin)
     v1Catalog.createDatabase(
       CatalogDatabase("default", "", databasePath.toURI, Map.empty),
       ignoreIfExists = false)
@@ -48,7 +47,12 @@ class TableLookupCacheSuite extends AnalysisTest with Matchers {
       ignoreIfExists = false)
     val v2Catalog = new InMemoryTableCatalog {
       override def loadTable(ident: Identifier): Table = {
-        V1Table(externalCatalog.getTable("default", ident.name))
+        val catalogTable = externalCatalog.getTable("default", ident.name)
+        new InMemoryTable(
+          catalogTable.identifier.table,
+          catalogTable.schema,
+          Array.empty,
+          Map.empty[String, String].asJava)
       }
       override def name: String = CatalogManager.SESSION_CATALOG_NAME
     }
@@ -64,7 +68,7 @@ class TableLookupCacheSuite extends AnalysisTest with Matchers {
     when(catalogManager.currentCatalog).thenReturn(v2Catalog)
     when(catalogManager.currentNamespace).thenReturn(Array("default"))
 
-    new Analyzer(catalogManager, conf)
+    new Analyzer(catalogManager)
   }
 
   test("table lookups to external catalog are cached") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
index e37555f1c0ec3..9058e3eb3f041 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
@@ -21,13 +21,12 @@ import java.sql.Timestamp
 
 import org.apache.spark.sql.catalyst.analysis.TypeCoercion._
 import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules.{Rule, RuleExecutor}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
-import org.apache.spark.unsafe.types.CalendarInterval
 
 class TypeCoercionSuite extends AnalysisTest {
   import TypeCoercionSuite._
@@ -62,13 +61,13 @@ class TypeCoercionSuite extends AnalysisTest {
 
   private def shouldCast(from: DataType, to: AbstractDataType, expected: DataType): Unit = {
     // Check default value
-    val castDefault = TypeCoercion.ImplicitTypeCasts.implicitCast(default(from), to)
+    val castDefault = TypeCoercion.implicitCast(default(from), to)
     assert(DataType.equalsIgnoreCompatibleNullability(
       castDefault.map(_.dataType).getOrElse(null), expected),
       s"Failed to cast $from to $to")
 
     // Check null value
-    val castNull = TypeCoercion.ImplicitTypeCasts.implicitCast(createNull(from), to)
+    val castNull = TypeCoercion.implicitCast(createNull(from), to)
     assert(DataType.equalsIgnoreCaseAndNullability(
       castNull.map(_.dataType).getOrElse(null), expected),
       s"Failed to cast $from to $to")
@@ -76,11 +75,11 @@ class TypeCoercionSuite extends AnalysisTest {
 
   private def shouldNotCast(from: DataType, to: AbstractDataType): Unit = {
     // Check default value
-    val castDefault = TypeCoercion.ImplicitTypeCasts.implicitCast(default(from), to)
+    val castDefault = TypeCoercion.implicitCast(default(from), to)
     assert(castDefault.isEmpty, s"Should not be able to cast $from to $to, but got $castDefault")
 
     // Check null value
-    val castNull = TypeCoercion.ImplicitTypeCasts.implicitCast(createNull(from), to)
+    val castNull = TypeCoercion.implicitCast(createNull(from), to)
     assert(castNull.isEmpty, s"Should not be able to cast $from to $to, but got $castNull")
   }
 
@@ -261,7 +260,7 @@ class TypeCoercionSuite extends AnalysisTest {
 
     // Tests that its not possible to setup implicit casts between two map types when
     // source map's key type is integer and the target map's key type are either Binary,
-    // Boolean, Date, Timestamp, Array, Struct, CaleandarIntervalType or NullType
+    // Boolean, Date, Timestamp, Array, Struct, CalendarIntervalType or NullType
     nonCastableTargetTypes.foreach { targetType =>
       shouldNotCast(sourceType, targetType)
     }
@@ -275,7 +274,7 @@ class TypeCoercionSuite extends AnalysisTest {
         Literal.create(null, sourceType.valueType)))
     targetNotNullableTypes.foreach { targetType =>
       val castDefault =
-        TypeCoercion.ImplicitTypeCasts.implicitCast(sourceMapExprWithValueNull, targetType)
+        TypeCoercion.implicitCast(sourceMapExprWithValueNull, targetType)
       assert(castDefault.isEmpty,
         s"Should not be able to cast $sourceType to $targetType, but got $castDefault")
     }
@@ -1104,7 +1103,7 @@ class TypeCoercionSuite extends AnalysisTest {
   }
 
   test("type coercion for Concat") {
-    val rule = TypeCoercion.ConcatCoercion(conf)
+    val rule = TypeCoercion.ConcatCoercion
 
     ruleTest(rule,
       Concat(Seq(Literal("ab"), Literal("cde"))),
@@ -1158,7 +1157,7 @@ class TypeCoercionSuite extends AnalysisTest {
   }
 
   test("type coercion for Elt") {
-    val rule = TypeCoercion.EltCoercion(conf)
+    val rule = TypeCoercion.EltCoercion
 
     ruleTest(rule,
       Elt(Seq(Literal(1), Literal("ab"), Literal("cde"))),
@@ -1285,7 +1284,7 @@ class TypeCoercionSuite extends AnalysisTest {
     }
   }
 
-  private val timeZoneResolver = ResolveTimeZone(new SQLConf)
+  private val timeZoneResolver = ResolveTimeZone
 
   private def widenSetOperationTypes(plan: LogicalPlan): LogicalPlan = {
     timeZoneResolver(TypeCoercion.WidenSetOperationTypes(plan))
@@ -1417,13 +1416,28 @@ class TypeCoercionSuite extends AnalysisTest {
     }
   }
 
+  test("SPARK-32638: corrects references when adding aliases in WidenSetOperationTypes") {
+    val t1 = LocalRelation(AttributeReference("v", DecimalType(10, 0))())
+    val t2 = LocalRelation(AttributeReference("v", DecimalType(11, 0))())
+    val p1 = t1.select(t1.output.head).as("p1")
+    val p2 = t2.select(t2.output.head).as("p2")
+    val union = p1.union(p2)
+    val wp1 = widenSetOperationTypes(union.select(p1.output.head, $"p2.v"))
+    assert(wp1.isInstanceOf[Project])
+    // The attribute `p1.output.head` should be replaced in the root `Project`.
+    assert(wp1.expressions.forall(_.find(_ == p1.output.head).isEmpty))
+    val wp2 = widenSetOperationTypes(Aggregate(Nil, sum(p1.output.head).as("v") :: Nil, union))
+    assert(wp2.isInstanceOf[Aggregate])
+    assert(wp2.missingInput.isEmpty)
+  }
+
   /**
    * There are rules that need to not fire before child expressions get resolved.
    * We use this test to make sure those rules do not fire early.
    */
   test("make sure rules do not fire early") {
     // InConversion
-    val inConversion = TypeCoercion.InConversion(conf)
+    val inConversion = TypeCoercion.InConversion
     ruleTest(inConversion,
       In(UnresolvedAttribute("a"), Seq(Literal(1))),
       In(UnresolvedAttribute("a"), Seq(Literal(1)))
@@ -1467,7 +1481,7 @@ class TypeCoercionSuite extends AnalysisTest {
   }
 
   test("binary comparison with string promotion") {
-    val rule = TypeCoercion.PromoteStrings(conf)
+    val rule = TypeCoercion.PromoteStrings
     ruleTest(rule,
       GreaterThan(Literal("123"), Literal(1)),
       GreaterThan(Cast(Literal("123"), IntegerType), Literal(1)))
@@ -1559,6 +1573,30 @@ class TypeCoercionSuite extends AnalysisTest {
           Literal.create(null, DecimalType.SYSTEM_DEFAULT)))
     }
   }
+
+  test("SPARK-31761: byte, short and int should be cast to long for IntegralDivide's datatype") {
+    val rules = Seq(FunctionArgumentConversion, Division, ImplicitTypeCasts)
+    // Casts Byte to Long
+    ruleTest(TypeCoercion.IntegralDivision, IntegralDivide(2.toByte, 1.toByte),
+      IntegralDivide(Cast(2.toByte, LongType), Cast(1.toByte, LongType)))
+    // Casts Short to Long
+    ruleTest(TypeCoercion.IntegralDivision, IntegralDivide(2.toShort, 1.toShort),
+      IntegralDivide(Cast(2.toShort, LongType), Cast(1.toShort, LongType)))
+    // Casts Integer to Long
+    ruleTest(TypeCoercion.IntegralDivision, IntegralDivide(2, 1),
+      IntegralDivide(Cast(2, LongType), Cast(1, LongType)))
+    // should not be any change for Long data types
+    ruleTest(TypeCoercion.IntegralDivision, IntegralDivide(2L, 1L), IntegralDivide(2L, 1L))
+    // one of the operand is byte
+    ruleTest(TypeCoercion.IntegralDivision, IntegralDivide(2L, 1.toByte),
+      IntegralDivide(2L, Cast(1.toByte, LongType)))
+    // one of the operand is short
+    ruleTest(TypeCoercion.IntegralDivision, IntegralDivide(2.toShort, 1L),
+      IntegralDivide(Cast(2.toShort, LongType), 1L))
+    // one of the operand is int
+    ruleTest(TypeCoercion.IntegralDivision, IntegralDivide(2, 1L),
+      IntegralDivide(Cast(2, LongType), 1L))
+  }
 }
 
 
@@ -1569,8 +1607,9 @@ object TypeCoercionSuite {
   val fractionalTypes: Seq[DataType] =
     Seq(DoubleType, FloatType, DecimalType.SYSTEM_DEFAULT, DecimalType(10, 2))
   val numericTypes: Seq[DataType] = integralTypes ++ fractionalTypes
+  val datetimeTypes: Seq[DataType] = Seq(DateType, TimestampType)
   val atomicTypes: Seq[DataType] =
-    numericTypes ++ Seq(BinaryType, BooleanType, StringType, DateType, TimestampType)
+    numericTypes ++ datetimeTypes ++ Seq(BinaryType, BooleanType, StringType)
   val complexTypes: Seq[DataType] =
     Seq(ArrayType(IntegerType),
       ArrayType(StringType),
@@ -1584,12 +1623,16 @@ object TypeCoercionSuite {
     extends UnaryExpression with ExpectsInputTypes with Unevaluable {
     override def inputTypes: Seq[AbstractDataType] = Seq(AnyDataType)
     override def dataType: DataType = NullType
+    override protected def withNewChildInternal(newChild: Expression): AnyTypeUnaryExpression =
+      copy(child = newChild)
   }
 
   case class NumericTypeUnaryExpression(child: Expression)
     extends UnaryExpression with ExpectsInputTypes with Unevaluable {
     override def inputTypes: Seq[AbstractDataType] = Seq(NumericType)
     override def dataType: DataType = NullType
+    override protected def withNewChildInternal(newChild: Expression): NumericTypeUnaryExpression =
+      copy(child = newChild)
   }
 
   case class AnyTypeBinaryOperator(left: Expression, right: Expression)
@@ -1597,6 +1640,9 @@ object TypeCoercionSuite {
     override def dataType: DataType = NullType
     override def inputType: AbstractDataType = AnyDataType
     override def symbol: String = "anytype"
+    override protected def withNewChildrenInternal(
+        newLeft: Expression, newRight: Expression): AnyTypeBinaryOperator =
+      copy(left = newLeft, right = newRight)
   }
 
   case class NumericTypeBinaryOperator(left: Expression, right: Expression)
@@ -1604,5 +1650,8 @@ object TypeCoercionSuite {
     override def dataType: DataType = NullType
     override def inputType: AbstractDataType = NumericType
     override def symbol: String = "numerictype"
+    override protected def withNewChildrenInternal(
+        newLeft: Expression, newRight: Expression): NumericTypeBinaryOperator =
+      copy(left = newLeft, right = newRight)
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala
index 3ec6fdeedd4b8..296d0ee8f4af5 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala
@@ -29,14 +29,14 @@ import org.apache.spark.sql.catalyst.expressions.aggregate.Count
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical.{FlatMapGroupsWithState, _}
 import org.apache.spark.sql.catalyst.streaming.InternalOutputModes._
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.streaming.OutputMode
 import org.apache.spark.sql.types.{IntegerType, LongType, MetadataBuilder}
-import org.apache.spark.unsafe.types.CalendarInterval
 
 /** A dummy command for testing unsupported operations. */
-case class DummyCommand() extends Command
+case class DummyCommand() extends LeafCommand
 
-class UnsupportedOperationsSuite extends SparkFunSuite {
+class UnsupportedOperationsSuite extends SparkFunSuite with SQLHelper {
 
   val attribute = AttributeReference("a", IntegerType, nullable = true)()
   val watermarkMetadata = new MetadataBuilder()
@@ -62,7 +62,7 @@ class UnsupportedOperationsSuite extends SparkFunSuite {
 
   assertNotSupportedInBatchPlan(
     "select on streaming source",
-    streamRelation.select($"count(*)"),
+    streamRelation.select($"`count(*)`"),
     Seq("with streaming source", "start"))
 
 
@@ -76,7 +76,7 @@ class UnsupportedOperationsSuite extends SparkFunSuite {
   testError(
     "streaming plan - no streaming source",
     Seq("without streaming source", "start")) {
-    UnsupportedOperationChecker.checkForStreaming(batchRelation.select($"count(*)"), Append)
+    UnsupportedOperationChecker.checkForStreaming(batchRelation.select($"`count(*)`"), Append)
   }
 
   // Commands
@@ -218,6 +218,7 @@ class UnsupportedOperationsSuite extends SparkFunSuite {
     expectedMsgs = Seq("flatMapGroupsWithState in append mode", "update"))
 
   // FlatMapGroupsWithState(Append) in streaming with aggregation
+  // Only supported when `spark.sql.streaming.statefulOperator.correctnessCheck` is disabled.
   for (outputMode <- Seq(Append, Update, Complete)) {
     assertSupportedInStreamingPlan(
       "flatMapGroupsWithState - flatMapGroupsWithState(Append) " +
@@ -228,7 +229,8 @@ class UnsupportedOperationsSuite extends SparkFunSuite {
         FlatMapGroupsWithState(
           null, att, att, Seq(att), Seq(att), att, null, Append, isMapGroupsWithState = false, null,
           streamRelation)),
-      outputMode = outputMode)
+      outputMode = outputMode,
+      SQLConf.STATEFUL_OPERATOR_CHECK_CORRECTNESS_ENABLED.key -> "false")
   }
 
   for (outputMode <- Seq(Append, Update)) {
@@ -268,6 +270,7 @@ class UnsupportedOperationsSuite extends SparkFunSuite {
   }
 
   // multiple FlatMapGroupsWithStates
+  // Only supported when `spark.sql.streaming.statefulOperator.correctnessCheck` is disabled.
   assertSupportedInStreamingPlan(
     "flatMapGroupsWithState - multiple flatMapGroupsWithStates on streaming relation and all are " +
       "in append mode",
@@ -275,7 +278,8 @@ class UnsupportedOperationsSuite extends SparkFunSuite {
       isMapGroupsWithState = false, null,
       FlatMapGroupsWithState(null, att, att, Seq(att), Seq(att), att, null, Append,
         isMapGroupsWithState = false, null, streamRelation)),
-    outputMode = Append)
+    outputMode = Append,
+    SQLConf.STATEFUL_OPERATOR_CHECK_CORRECTNESS_ENABLED.key -> "false")
 
   assertNotSupportedInStreamingPlan(
     "flatMapGroupsWithState -  multiple flatMapGroupsWithStates on s streaming relation but some" +
@@ -388,16 +392,14 @@ class UnsupportedOperationsSuite extends SparkFunSuite {
   testBinaryOperationInStreamingPlan(
     "single inner join in append mode",
     _.join(_, joinType = Inner),
-    outputMode = Append,
-    streamStreamSupported = true)
+    outputMode = Append)
 
   testBinaryOperationInStreamingPlan(
     "multiple inner joins in append mode",
     (x: LogicalPlan, y: LogicalPlan) => {
       x.join(y, joinType = Inner).join(streamRelation, joinType = Inner)
     },
-    outputMode = Append,
-    streamStreamSupported = true)
+    outputMode = Append)
 
   testBinaryOperationInStreamingPlan(
     "inner join in update mode",
@@ -406,155 +408,145 @@ class UnsupportedOperationsSuite extends SparkFunSuite {
     streamStreamSupported = false,
     expectedMsg = "is not supported in Update output mode")
 
-  // Full outer joins: only batch-batch is allowed
+  // Full outer joins: stream-batch/batch-stream join are not allowed,
+  // and stream-stream join is allowed 'conditionally' - see below check
   testBinaryOperationInStreamingPlan(
-    "full outer join",
+    "FullOuter join",
     _.join(_, joinType = FullOuter),
     streamStreamSupported = false,
     batchStreamSupported = false,
-    streamBatchSupported = false)
+    streamBatchSupported = false,
+    expectedMsg = "FullOuter join")
+
+  // Left outer, left semi, left anti join: *-stream not allowed
+  Seq((LeftOuter, "LeftOuter join"), (LeftSemi, "LeftSemi join"), (LeftAnti, "LeftAnti join"))
+    .foreach { case (joinType, name) =>
+      testBinaryOperationInStreamingPlan(
+        name,
+        _.join(_, joinType = joinType),
+        batchStreamSupported = false,
+        streamStreamSupported = false,
+        expectedMsg = name)
+    }
 
-  // Left outer joins: *-stream not allowed
+  // Right outer joins: stream-* not allowed
   testBinaryOperationInStreamingPlan(
-    "left outer join",
-    _.join(_, joinType = LeftOuter),
-    batchStreamSupported = false,
+    "RightOuter join",
+    _.join(_, joinType = RightOuter),
+    streamBatchSupported = false,
     streamStreamSupported = false,
-    expectedMsg = "outer join")
+    expectedMsg = "RightOuter join")
 
-  // Left outer joins: update and complete mode not allowed
-  assertNotSupportedInStreamingPlan(
-    s"left outer join with stream-stream relations and update mode",
-    streamRelation.join(streamRelation, joinType = LeftOuter,
-      condition = Some(attribute === attribute)),
-    OutputMode.Update(),
-    Seq("is not supported in Update output mode"))
-  assertNotSupportedInStreamingPlan(
-    s"left outer join with stream-stream relations and complete mode",
-    Aggregate(Nil, aggExprs("d"), streamRelation.join(streamRelation, joinType = LeftOuter,
-      condition = Some(attribute === attribute))),
-    OutputMode.Complete(),
-    Seq("is not supported in Complete output mode"))
-
-  // Left outer joins: stream-stream allowed with join on watermark attribute
-  // Note that the attribute need not be watermarked on both sides.
-  assertSupportedInStreamingPlan(
-    s"left outer join with stream-stream relations and join on attribute with left watermark",
-    streamRelation.join(streamRelation, joinType = LeftOuter,
-      condition = Some(attributeWithWatermark === attribute)),
-    OutputMode.Append())
-  assertSupportedInStreamingPlan(
-    s"left outer join with stream-stream relations and join on attribute with right watermark",
-    streamRelation.join(streamRelation, joinType = LeftOuter,
-      condition = Some(attribute === attributeWithWatermark)),
-    OutputMode.Append())
-  assertNotSupportedInStreamingPlan(
-    s"left outer join with stream-stream relations and join on non-watermarked attribute",
-    streamRelation.join(streamRelation, joinType = LeftOuter,
-      condition = Some(attribute === attribute)),
-    OutputMode.Append(),
-    Seq("watermark in the join keys"))
+  // Left outer, right outer, full outer, left semi joins
+  Seq(LeftOuter, RightOuter, FullOuter, LeftSemi).foreach { joinType =>
+    // Update mode not allowed
+    assertNotSupportedInStreamingPlan(
+      s"$joinType join with stream-stream relations and update mode",
+      streamRelation.join(streamRelation, joinType = joinType,
+        condition = Some(attribute === attribute)),
+      OutputMode.Update(),
+      Seq("is not supported in Update output mode"))
 
-  // Left outer joins: stream-stream allowed with range condition yielding state value watermark
-  assertSupportedInStreamingPlan(
-    s"left outer join with stream-stream relations and state value watermark", {
-      val leftRelation = streamRelation
-      val rightTimeWithWatermark =
-        AttributeReference("b", IntegerType)().withMetadata(watermarkMetadata)
-      val rightRelation = new TestStreamingRelation(rightTimeWithWatermark)
-      leftRelation.join(
-        rightRelation,
-        joinType = LeftOuter,
-        condition = Some(attribute > rightTimeWithWatermark + 10))
-    },
-    OutputMode.Append())
+    // Complete mode not allowed
+    assertNotSupportedInStreamingPlan(
+      s"$joinType join with stream-stream relations and complete mode",
+      Aggregate(Nil, aggExprs("d"), streamRelation.join(streamRelation, joinType = joinType,
+        condition = Some(attribute === attribute))),
+      OutputMode.Complete(),
+      Seq("is not supported in Complete output mode"))
+
+    // Stream-stream allowed with join on watermark attribute
+    // Note that the attribute need not be watermarked on both sides.
+    assertSupportedInStreamingPlan(
+      s"$joinType join with stream-stream relations and join on attribute with left watermark",
+      streamRelation.join(streamRelation, joinType = joinType,
+        condition = Some(attributeWithWatermark === attribute)),
+      OutputMode.Append())
+    assertSupportedInStreamingPlan(
+      s"$joinType join with stream-stream relations and join on attribute with right watermark",
+      streamRelation.join(streamRelation, joinType = joinType,
+        condition = Some(attribute === attributeWithWatermark)),
+      OutputMode.Append())
+    assertNotSupportedInStreamingPlan(
+      s"$joinType join with stream-stream relations and join on non-watermarked attribute",
+      streamRelation.join(streamRelation, joinType = joinType,
+        condition = Some(attribute === attribute)),
+      OutputMode.Append(),
+      Seq("without a watermark in the join keys"))
+
+    val timeWithWatermark =
+      AttributeReference("b", IntegerType)().withMetadata(watermarkMetadata)
+    val relationWithWatermark = new TestStreamingRelation(timeWithWatermark)
+    val (leftRelation, rightRelation) =
+      if (joinType == RightOuter) {
+        (relationWithWatermark, streamRelation)
+      } else {
+        (streamRelation, relationWithWatermark)
+      }
 
-  // Left outer joins: stream-stream not allowed with insufficient range condition
-  assertNotSupportedInStreamingPlan(
-    s"left outer join with stream-stream relations and state value watermark", {
-      val leftRelation = streamRelation
-      val rightTimeWithWatermark =
-        AttributeReference("b", IntegerType)().withMetadata(watermarkMetadata)
-      val rightRelation = new TestStreamingRelation(rightTimeWithWatermark)
-      leftRelation.join(
-        rightRelation,
-        joinType = LeftOuter,
-        condition = Some(attribute < rightTimeWithWatermark + 10))
-    },
-    OutputMode.Append(),
-    Seq("appropriate range condition"))
+    // stream-stream allowed with range condition yielding state value watermark
+    assertSupportedInStreamingPlan(
+      s"$joinType join with stream-stream relations and state value watermark",
+      leftRelation.join(rightRelation, joinType = joinType,
+        condition = Some(attribute > timeWithWatermark + 10)),
+      OutputMode.Append())
 
-  // Left semi joins: stream-* not allowed
-  testBinaryOperationInStreamingPlan(
-    "left semi join",
-    _.join(_, joinType = LeftSemi),
-    streamStreamSupported = false,
-    batchStreamSupported = false,
-    expectedMsg = "left semi/anti joins")
+    // stream-stream not allowed with insufficient range condition
+    assertNotSupportedInStreamingPlan(
+      s"$joinType join with stream-stream relations and state value watermark",
+      leftRelation.join(rightRelation, joinType = joinType,
+        condition = Some(attribute < timeWithWatermark + 10)),
+      OutputMode.Append(),
+      Seq("is not supported without a watermark in the join keys, or a watermark on " +
+        "the nullable side and an appropriate range condition"))
+  }
 
-  // Left anti joins: stream-* not allowed
-  testBinaryOperationInStreamingPlan(
-    "left anti join",
-    _.join(_, joinType = LeftAnti),
-    streamStreamSupported = false,
-    batchStreamSupported = false,
-    expectedMsg = "left semi/anti joins")
+  // stream-stream inner join doesn't emit late rows, whereas outer joins could
+  Seq((Inner, false), (LeftOuter, true), (RightOuter, true)).map {
+    case (joinType, expectFailure) =>
+      assertPassOnGlobalWatermarkLimit(
+        s"single $joinType join in Append mode",
+        streamRelation.join(streamRelation, joinType = RightOuter,
+          condition = Some(attributeWithWatermark === attribute)),
+        OutputMode.Append())
 
-  // Right outer joins: stream-* not allowed
-  testBinaryOperationInStreamingPlan(
-    "right outer join",
-    _.join(_, joinType = RightOuter),
-    streamBatchSupported = false,
-    streamStreamSupported = false,
-    expectedMsg = "outer join")
+      testGlobalWatermarkLimit(
+        s"streaming aggregation after stream-stream $joinType join in Append mode",
+        streamRelation.join(streamRelation, joinType = joinType,
+          condition = Some(attributeWithWatermark === attribute))
+          .groupBy("a")(count("*")),
+        OutputMode.Append(),
+        expectFailure = expectFailure)
 
-  // Right outer joins: stream-stream allowed with join on watermark attribute
-  // Note that the attribute need not be watermarked on both sides.
-  assertSupportedInStreamingPlan(
-    s"right outer join with stream-stream relations and join on attribute with left watermark",
-    streamRelation.join(streamRelation, joinType = RightOuter,
-      condition = Some(attributeWithWatermark === attribute)),
-    OutputMode.Append())
-  assertSupportedInStreamingPlan(
-    s"right outer join with stream-stream relations and join on attribute with right watermark",
-    streamRelation.join(streamRelation, joinType = RightOuter,
-      condition = Some(attribute === attributeWithWatermark)),
-    OutputMode.Append())
-  assertNotSupportedInStreamingPlan(
-    s"right outer join with stream-stream relations and join on non-watermarked attribute",
-    streamRelation.join(streamRelation, joinType = RightOuter,
-      condition = Some(attribute === attribute)),
-    OutputMode.Append(),
-    Seq("watermark in the join keys"))
+      Seq(Inner, LeftOuter, RightOuter).foreach { joinType2 =>
+        testGlobalWatermarkLimit(
+          s"streaming-stream $joinType2 after stream-stream $joinType join in Append mode",
+          streamRelation.join(
+            streamRelation.join(streamRelation, joinType = joinType,
+              condition = Some(attributeWithWatermark === attribute)),
+            joinType = joinType2,
+            condition = Some(attributeWithWatermark === attribute)),
+          OutputMode.Append(),
+          expectFailure = expectFailure)
+      }
 
-  // Right outer joins: stream-stream allowed with range condition yielding state value watermark
-  assertSupportedInStreamingPlan(
-    s"right outer join with stream-stream relations and state value watermark", {
-      val leftTimeWithWatermark =
-        AttributeReference("b", IntegerType)().withMetadata(watermarkMetadata)
-      val leftRelation = new TestStreamingRelation(leftTimeWithWatermark)
-      val rightRelation = streamRelation
-      leftRelation.join(
-        rightRelation,
-        joinType = RightOuter,
-        condition = Some(leftTimeWithWatermark + 10 < attribute))
-    },
-    OutputMode.Append())
+      testGlobalWatermarkLimit(
+        s"FlatMapGroupsWithState after stream-stream $joinType join in Append mode",
+        FlatMapGroupsWithState(
+          null, att, att, Seq(att), Seq(att), att, null, Append,
+          isMapGroupsWithState = false, null,
+          streamRelation.join(streamRelation, joinType = joinType,
+            condition = Some(attributeWithWatermark === attribute))),
+        OutputMode.Append(),
+        expectFailure = expectFailure)
 
-  // Right outer joins: stream-stream not allowed with insufficient range condition
-  assertNotSupportedInStreamingPlan(
-    s"right outer join with stream-stream relations and state value watermark", {
-      val leftTimeWithWatermark =
-        AttributeReference("b", IntegerType)().withMetadata(watermarkMetadata)
-      val leftRelation = new TestStreamingRelation(leftTimeWithWatermark)
-      val rightRelation = streamRelation
-      leftRelation.join(
-        rightRelation,
-        joinType = RightOuter,
-        condition = Some(leftTimeWithWatermark + 10 > attribute))
-    },
-    OutputMode.Append(),
-    Seq("appropriate range condition"))
+      testGlobalWatermarkLimit(
+        s"deduplicate after stream-stream $joinType join in Append mode",
+        Deduplicate(Seq(attribute), streamRelation.join(streamRelation, joinType = joinType,
+          condition = Some(attributeWithWatermark === attribute))),
+        OutputMode.Append(),
+        expectFailure = expectFailure)
+  }
 
   // Cogroup: only batch-batch is allowed
   testBinaryOperationInStreamingPlan(
@@ -593,10 +585,12 @@ class UnsupportedOperationsSuite extends SparkFunSuite {
     streamStreamSupported = false,
     batchStreamSupported = false)
 
-  // Intersect: stream-stream not supported
+  // Intersect: not supported
   testBinaryOperationInStreamingPlan(
     "intersect",
     _.intersect(_, isAll = false),
+    batchStreamSupported = false,
+    streamBatchSupported = false,
     streamStreamSupported = false)
 
   // Sort: supported only on batch subplans and after aggregation on streaming plan + complete mode
@@ -677,53 +671,6 @@ class UnsupportedOperationsSuite extends SparkFunSuite {
       OutputMode.Append())
   }
 
-  // stream-stream join
-  // stream-stream inner join doesn't emit late rows, whereas outer joins could
-  Seq((Inner, false), (LeftOuter, true), (RightOuter, true)).map { case (joinType, expectFailure) =>
-    assertPassOnGlobalWatermarkLimit(
-      s"single $joinType join in Append mode",
-      streamRelation.join(streamRelation, joinType = RightOuter,
-        condition = Some(attributeWithWatermark === attribute)),
-      OutputMode.Append())
-
-    testGlobalWatermarkLimit(
-      s"streaming aggregation after stream-stream $joinType join in Append mode",
-      streamRelation.join(streamRelation, joinType = joinType,
-        condition = Some(attributeWithWatermark === attribute))
-        .groupBy("a")(count("*")),
-      OutputMode.Append(),
-      expectFailure = expectFailure)
-
-    Seq(Inner, LeftOuter, RightOuter).map { joinType2 =>
-      testGlobalWatermarkLimit(
-        s"streaming-stream $joinType2 after stream-stream $joinType join in Append mode",
-        streamRelation.join(
-          streamRelation.join(streamRelation, joinType = joinType,
-            condition = Some(attributeWithWatermark === attribute)),
-          joinType = joinType2,
-          condition = Some(attributeWithWatermark === attribute)),
-        OutputMode.Append(),
-        expectFailure = expectFailure)
-    }
-
-    testGlobalWatermarkLimit(
-      s"FlatMapGroupsWithState after stream-stream $joinType join in Append mode",
-      FlatMapGroupsWithState(
-        null, att, att, Seq(att), Seq(att), att, null, Append,
-        isMapGroupsWithState = false, null,
-        streamRelation.join(streamRelation, joinType = joinType,
-          condition = Some(attributeWithWatermark === attribute))),
-      OutputMode.Append(),
-      expectFailure = expectFailure)
-
-    testGlobalWatermarkLimit(
-      s"deduplicate after stream-stream $joinType join in Append mode",
-      Deduplicate(Seq(attribute), streamRelation.join(streamRelation, joinType = joinType,
-        condition = Some(attributeWithWatermark === attribute))),
-      OutputMode.Append(),
-      expectFailure = expectFailure)
-  }
-
   // FlatMapGroupsWithState
   {
     assertPassOnGlobalWatermarkLimit(
@@ -740,7 +687,7 @@ class UnsupportedOperationsSuite extends SparkFunSuite {
         isMapGroupsWithState = false, null, streamRelation).groupBy("*")(count("*")),
       OutputMode.Append())
 
-    Seq(Inner, LeftOuter, RightOuter).map { joinType =>
+    Seq(Inner, LeftOuter, RightOuter).foreach { joinType =>
       assertFailOnGlobalWatermarkLimit(
         s"stream-stream $joinType after FlatMapGroupsWithState in Append mode",
         streamRelation.join(
@@ -773,7 +720,7 @@ class UnsupportedOperationsSuite extends SparkFunSuite {
       Deduplicate(Seq(attribute), streamRelation).groupBy("a")(count("*")),
       OutputMode.Append())
 
-    Seq(Inner, LeftOuter, RightOuter).map { joinType =>
+    Seq(Inner, LeftOuter, RightOuter).foreach { joinType =>
       assertPassOnGlobalWatermarkLimit(
         s"$joinType join after deduplicate in Append mode",
         streamRelation.join(Deduplicate(Seq(attribute), streamRelation), joinType = joinType,
@@ -933,13 +880,16 @@ class UnsupportedOperationsSuite extends SparkFunSuite {
   def assertSupportedInStreamingPlan(
       name: String,
       plan: LogicalPlan,
-      outputMode: OutputMode): Unit = {
+      outputMode: OutputMode,
+      configs: (String, String)*): Unit = {
     test(s"streaming plan - $name: supported") {
-      UnsupportedOperationChecker.checkForStreaming(wrapInStreaming(plan), outputMode)
+      withSQLConf(configs: _*) {
+        UnsupportedOperationChecker.checkForStreaming(wrapInStreaming(plan), outputMode)
+      }
     }
   }
 
-  /** Assert that the logical plan is supported for continuous procsssing mode */
+  /** Assert that the logical plan is supported for continuous processing mode */
   def assertSupportedForContinuousProcessing(
     name: String,
     plan: LogicalPlan,
@@ -1008,14 +958,18 @@ class UnsupportedOperationsSuite extends SparkFunSuite {
       expectFailure: Boolean): Unit = {
     test(s"Global watermark limit - $testNamePostfix") {
       if (expectFailure) {
-        val e = intercept[AnalysisException] {
-          UnsupportedOperationChecker.checkStreamingQueryGlobalWatermarkLimit(
-            wrapInStreaming(plan), outputMode, failWhenDetected = true)
+        withSQLConf(SQLConf.STATEFUL_OPERATOR_CHECK_CORRECTNESS_ENABLED.key -> "true") {
+          val e = intercept[AnalysisException] {
+            UnsupportedOperationChecker.checkStreamingQueryGlobalWatermarkLimit(
+              wrapInStreaming(plan), outputMode)
+          }
+          assert(e.message.contains("Detected pattern of possible 'correctness' issue"))
         }
-        assert(e.message.contains("Detected pattern of possible 'correctness' issue"))
       } else {
-        UnsupportedOperationChecker.checkStreamingQueryGlobalWatermarkLimit(
-          wrapInStreaming(plan), outputMode, failWhenDetected = true)
+        withSQLConf(SQLConf.STATEFUL_OPERATOR_CHECK_CORRECTNESS_ENABLED.key -> "false") {
+          UnsupportedOperationChecker.checkStreamingQueryGlobalWatermarkLimit(
+            wrapInStreaming(plan), outputMode)
+        }
       }
     }
   }
@@ -1046,6 +1000,8 @@ class UnsupportedOperationsSuite extends SparkFunSuite {
   case class StreamingPlanWrapper(child: LogicalPlan) extends UnaryNode {
     override def output: Seq[Attribute] = child.output
     override def isStreaming: Boolean = true
+    override protected def withNewChildInternal(newChild: LogicalPlan): StreamingPlanWrapper =
+      copy(child = newChild)
   }
 
   case class TestStreamingRelation(output: Seq[Attribute]) extends LeafNode {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala
index 55712d0da518d..d310538e302de 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala
@@ -408,8 +408,8 @@ abstract class ExternalCatalogSuite extends SparkFunSuite with BeforeAndAfterEac
       partitionColumnNames = Seq("partCol1", "partCol2"))
     catalog.createTable(table, ignoreIfExists = false)
 
-    val newLocationPart1 = newUriForDatabase()
-    val newLocationPart2 = newUriForDatabase()
+    val newLocationPart1 = newUriForPartition(Seq("p1=1", "p2=2"))
+    val newLocationPart2 = newUriForPartition(Seq("p1=3", "p2=4"))
 
     val partition1 =
       CatalogTablePartition(Map("partCol1" -> "1", "partCol2" -> "2"),
@@ -991,6 +991,11 @@ abstract class CatalogTestUtils {
 
   def newUriForDatabase(): URI = new URI(Utils.createTempDir().toURI.toString.stripSuffix("/"))
 
+  def newUriForPartition(parts: Seq[String]): URI = {
+    val path = parts.foldLeft(Utils.createTempDir())(new java.io.File(_, _))
+    new URI(path.toURI.toString.stripSuffix("/"))
+  }
+
   def newDb(name: String): CatalogDatabase = {
     CatalogDatabase(name, name + " description", newUriForDatabase(), Map.empty)
   }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala
index 4d88a8d7ee546..8096062f71a23 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala
@@ -17,15 +17,19 @@
 
 package org.apache.spark.sql.catalyst.catalog
 
+import scala.concurrent.duration._
+
+import org.scalatest.concurrent.Eventually
+
 import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
+import org.apache.spark.sql.catalyst.{AliasIdentifier, FunctionIdentifier, QualifiedTableName, TableIdentifier}
 import org.apache.spark.sql.catalyst.analysis._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
-import org.apache.spark.sql.catalyst.plans.logical.{Range, SubqueryAlias, View}
+import org.apache.spark.sql.catalyst.plans.logical.{LeafCommand, LogicalPlan, Project, Range, SubqueryAlias, View}
 import org.apache.spark.sql.connector.catalog.CatalogManager
 import org.apache.spark.sql.connector.catalog.SupportsNamespaces.PROP_OWNER
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
 import org.apache.spark.sql.types._
 
 class InMemorySessionCatalogSuite extends SessionCatalogSuite {
@@ -45,7 +49,7 @@ class InMemorySessionCatalogSuite extends SessionCatalogSuite {
  * signatures but do not extend a common parent. This is largely by design but
  * unfortunately leads to very similar test code in two places.
  */
-abstract class SessionCatalogSuite extends AnalysisTest {
+abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
   protected val utils: CatalogTestUtils
 
   protected val isHiveExternalCatalog = false
@@ -70,6 +74,22 @@ abstract class SessionCatalogSuite extends AnalysisTest {
       catalog.reset()
     }
   }
+
+  private def withConfAndEmptyCatalog(conf: SQLConf)(f: SessionCatalog => Unit): Unit = {
+    val catalog = new SessionCatalog(newEmptyCatalog(), new SimpleFunctionRegistry(), conf)
+    catalog.createDatabase(newDb("default"), ignoreIfExists = true)
+    try {
+      f(catalog)
+    } finally {
+      catalog.reset()
+    }
+  }
+
+  private def getTempViewRawPlan(plan: Option[LogicalPlan]): Option[LogicalPlan] = plan match {
+    case Some(v: View) if v.isTempViewStoringAnalyzedPlan => Some(v.child)
+    case other => other
+  }
+
   // --------------------------------------------------------------------------
   // Databases
   // --------------------------------------------------------------------------
@@ -285,18 +305,18 @@ abstract class SessionCatalogSuite extends AnalysisTest {
     withBasicCatalog { catalog =>
       val tempTable1 = Range(1, 10, 1, 10)
       val tempTable2 = Range(1, 20, 2, 10)
-      catalog.createTempView("tbl1", tempTable1, overrideIfExists = false)
-      catalog.createTempView("tbl2", tempTable2, overrideIfExists = false)
-      assert(catalog.getTempView("tbl1") == Option(tempTable1))
-      assert(catalog.getTempView("tbl2") == Option(tempTable2))
-      assert(catalog.getTempView("tbl3").isEmpty)
+      createTempView(catalog, "tbl1", tempTable1, overrideIfExists = false)
+      createTempView(catalog, "tbl2", tempTable2, overrideIfExists = false)
+      assert(getTempViewRawPlan(catalog.getTempView("tbl1")) == Option(tempTable1))
+      assert(getTempViewRawPlan(catalog.getTempView("tbl2")) == Option(tempTable2))
+      assert(getTempViewRawPlan(catalog.getTempView("tbl3")).isEmpty)
       // Temporary view already exists
       intercept[TempTableAlreadyExistsException] {
-        catalog.createTempView("tbl1", tempTable1, overrideIfExists = false)
+        createTempView(catalog, "tbl1", tempTable1, overrideIfExists = false)
       }
       // Temporary view already exists but we override it
-      catalog.createTempView("tbl1", tempTable2, overrideIfExists = true)
-      assert(catalog.getTempView("tbl1") == Option(tempTable2))
+      createTempView(catalog, "tbl1", tempTable2, overrideIfExists = true)
+      assert(getTempViewRawPlan(catalog.getTempView("tbl1")) == Option(tempTable2))
     }
   }
 
@@ -336,9 +356,9 @@ abstract class SessionCatalogSuite extends AnalysisTest {
   test("drop temp table") {
     withBasicCatalog { catalog =>
       val tempTable = Range(1, 10, 2, 10)
-      catalog.createTempView("tbl1", tempTable, overrideIfExists = false)
+      createTempView(catalog, "tbl1", tempTable, overrideIfExists = false)
       catalog.setCurrentDatabase("db2")
-      assert(catalog.getTempView("tbl1") == Some(tempTable))
+      assert(getTempViewRawPlan(catalog.getTempView("tbl1")) == Some(tempTable))
       assert(catalog.externalCatalog.listTables("db2").toSet == Set("tbl1", "tbl2"))
       // If database is not specified, temp table should be dropped first
       catalog.dropTable(TableIdentifier("tbl1"), ignoreIfNotExists = false, purge = false)
@@ -348,11 +368,11 @@ abstract class SessionCatalogSuite extends AnalysisTest {
       catalog.dropTable(TableIdentifier("tbl1"), ignoreIfNotExists = false, purge = false)
       assert(catalog.externalCatalog.listTables("db2").toSet == Set("tbl2"))
       // If database is specified, temp tables are never dropped
-      catalog.createTempView("tbl1", tempTable, overrideIfExists = false)
+      createTempView(catalog, "tbl1", tempTable, overrideIfExists = false)
       catalog.createTable(newTable("tbl1", "db2"), ignoreIfExists = false)
       catalog.dropTable(TableIdentifier("tbl1", Some("db2")), ignoreIfNotExists = false,
         purge = false)
-      assert(catalog.getTempView("tbl1") == Some(tempTable))
+      assert(getTempViewRawPlan(catalog.getTempView("tbl1")) == Some(tempTable))
       assert(catalog.externalCatalog.listTables("db2").toSet == Set("tbl2"))
     }
   }
@@ -403,18 +423,18 @@ abstract class SessionCatalogSuite extends AnalysisTest {
   test("rename temp table") {
     withBasicCatalog { catalog =>
       val tempTable = Range(1, 10, 2, 10)
-      catalog.createTempView("tbl1", tempTable, overrideIfExists = false)
+      createTempView(catalog, "tbl1", tempTable, overrideIfExists = false)
       catalog.setCurrentDatabase("db2")
-      assert(catalog.getTempView("tbl1") == Option(tempTable))
+      assert(getTempViewRawPlan(catalog.getTempView("tbl1")) == Option(tempTable))
       assert(catalog.externalCatalog.listTables("db2").toSet == Set("tbl1", "tbl2"))
       // If database is not specified, temp table should be renamed first
       catalog.renameTable(TableIdentifier("tbl1"), TableIdentifier("tbl3"))
       assert(catalog.getTempView("tbl1").isEmpty)
-      assert(catalog.getTempView("tbl3") == Option(tempTable))
+      assert(getTempViewRawPlan(catalog.getTempView("tbl3")) == Option(tempTable))
       assert(catalog.externalCatalog.listTables("db2").toSet == Set("tbl1", "tbl2"))
       // If database is specified, temp tables are never renamed
       catalog.renameTable(TableIdentifier("tbl2", Some("db2")), TableIdentifier("tbl4"))
-      assert(catalog.getTempView("tbl3") == Option(tempTable))
+      assert(getTempViewRawPlan(catalog.getTempView("tbl3")) == Option(tempTable))
       assert(catalog.getTempView("tbl4").isEmpty)
       assert(catalog.externalCatalog.listTables("db2").toSet == Set("tbl1", "tbl4"))
     }
@@ -605,14 +625,15 @@ abstract class SessionCatalogSuite extends AnalysisTest {
     withBasicCatalog { catalog =>
       val tempTable1 = Range(1, 10, 1, 10)
       val metastoreTable1 = catalog.externalCatalog.getTable("db2", "tbl1")
-      catalog.createTempView("tbl1", tempTable1, overrideIfExists = false)
+      createTempView(catalog, "tbl1", tempTable1, overrideIfExists = false)
       catalog.setCurrentDatabase("db2")
       // If we explicitly specify the database, we'll look up the relation in that database
       assert(catalog.lookupRelation(TableIdentifier("tbl1", Some("db2"))).children.head
         .asInstanceOf[UnresolvedCatalogRelation].tableMeta == metastoreTable1)
       // Otherwise, we'll first look up a temporary table with the same name
-      assert(catalog.lookupRelation(TableIdentifier("tbl1"))
-        == SubqueryAlias("tbl1", tempTable1))
+      val tbl1 = catalog.lookupRelation(TableIdentifier("tbl1")).asInstanceOf[SubqueryAlias]
+      assert(tbl1.identifier == AliasIdentifier("tbl1"))
+      assert(getTempViewRawPlan(Some(tbl1.child)).get == tempTable1)
       // Then, if that does not exist, look up the relation in the current database
       catalog.dropTable(TableIdentifier("tbl1"), ignoreIfNotExists = false, purge = false)
       assert(catalog.lookupRelation(TableIdentifier("tbl1")).children.head
@@ -620,6 +641,14 @@ abstract class SessionCatalogSuite extends AnalysisTest {
     }
   }
 
+  private def getViewPlan(metadata: CatalogTable): LogicalPlan = {
+    import org.apache.spark.sql.catalyst.dsl.expressions._
+    val projectList = metadata.schema.map { field =>
+      UpCast(field.name.attr, field.dataType).as(field.name)
+    }
+    Project(projectList, CatalystSqlParser.parsePlan(metadata.viewText.get))
+  }
+
   test("look up view relation") {
     withBasicCatalog { catalog =>
       val props = CatalogTable.catalogAndNamespaceToProps("cat1", Seq("ns1"))
@@ -632,8 +661,7 @@ abstract class SessionCatalogSuite extends AnalysisTest {
 
       // Look up a view.
       catalog.setCurrentDatabase("default")
-      val view = View(desc = metadata, output = metadata.schema.toAttributes,
-        child = CatalystSqlParser.parsePlan(metadata.viewText.get))
+      val view = View(desc = metadata, isTempView = false, child = getViewPlan(metadata))
       comparePlans(catalog.lookupRelation(TableIdentifier("view1", Some("db3"))),
         SubqueryAlias(Seq(CatalogManager.SESSION_CATALOG_NAME, "db3", "view1"), view))
       // Look up a view using current database of the session catalog.
@@ -652,8 +680,7 @@ abstract class SessionCatalogSuite extends AnalysisTest {
       assert(metadata.viewText.isDefined)
       assert(metadata.viewCatalogAndNamespace == Seq(CatalogManager.SESSION_CATALOG_NAME, "db2"))
 
-      val view = View(desc = metadata, output = metadata.schema.toAttributes,
-        child = CatalystSqlParser.parsePlan(metadata.viewText.get))
+      val view = View(desc = metadata, isTempView = false, child = getViewPlan(metadata))
       comparePlans(catalog.lookupRelation(TableIdentifier("view2", Some("db3"))),
         SubqueryAlias(Seq(CatalogManager.SESSION_CATALOG_NAME, "db3", "view2"), view))
     }
@@ -674,9 +701,12 @@ abstract class SessionCatalogSuite extends AnalysisTest {
       assert(catalog.tableExists(TableIdentifier("tbl1")))
       assert(catalog.tableExists(TableIdentifier("tbl2")))
 
-      catalog.createTempView("tbl3", tempTable, overrideIfExists = false)
+      createTempView(catalog, "tbl3", tempTable, overrideIfExists = false)
       // tableExists should not check temp view.
       assert(!catalog.tableExists(TableIdentifier("tbl3")))
+
+      // If database doesn't exist, return false instead of failing.
+      assert(!catalog.tableExists(TableIdentifier("tbl1", Some("non-exist"))))
     }
   }
 
@@ -691,7 +721,7 @@ abstract class SessionCatalogSuite extends AnalysisTest {
         catalog.getTempViewOrPermanentTableMetadata(TableIdentifier("view1", Some("default")))
       }.getMessage
 
-      catalog.createTempView("view1", tempTable, overrideIfExists = false)
+      createTempView(catalog, "view1", tempTable, overrideIfExists = false)
       assert(catalog.getTempViewOrPermanentTableMetadata(
         TableIdentifier("view1")).identifier.table == "view1")
       assert(catalog.getTempViewOrPermanentTableMetadata(
@@ -706,8 +736,8 @@ abstract class SessionCatalogSuite extends AnalysisTest {
   test("list tables without pattern") {
     withBasicCatalog { catalog =>
       val tempTable = Range(1, 10, 2, 10)
-      catalog.createTempView("tbl1", tempTable, overrideIfExists = false)
-      catalog.createTempView("tbl4", tempTable, overrideIfExists = false)
+      createTempView(catalog, "tbl1", tempTable, overrideIfExists = false)
+      createTempView(catalog, "tbl4", tempTable, overrideIfExists = false)
       assert(catalog.listTables("db1").toSet ==
         Set(TableIdentifier("tbl1"), TableIdentifier("tbl4")))
       assert(catalog.listTables("db2").toSet ==
@@ -724,8 +754,8 @@ abstract class SessionCatalogSuite extends AnalysisTest {
   test("list tables with pattern") {
     withBasicCatalog { catalog =>
       val tempTable = Range(1, 10, 2, 10)
-      catalog.createTempView("tbl1", tempTable, overrideIfExists = false)
-      catalog.createTempView("tbl4", tempTable, overrideIfExists = false)
+      createTempView(catalog, "tbl1", tempTable, overrideIfExists = false)
+      createTempView(catalog, "tbl4", tempTable, overrideIfExists = false)
       assert(catalog.listTables("db1", "*").toSet == catalog.listTables("db1").toSet)
       assert(catalog.listTables("db2", "*").toSet == catalog.listTables("db2").toSet)
       assert(catalog.listTables("db2", "tbl*").toSet ==
@@ -747,8 +777,8 @@ abstract class SessionCatalogSuite extends AnalysisTest {
       catalog.createTable(newTable("tbl1", "mydb"), ignoreIfExists = false)
       catalog.createTable(newTable("tbl2", "mydb"), ignoreIfExists = false)
       val tempTable = Range(1, 10, 2, 10)
-      catalog.createTempView("temp_view1", tempTable, overrideIfExists = false)
-      catalog.createTempView("temp_view4", tempTable, overrideIfExists = false)
+      createTempView(catalog, "temp_view1", tempTable, overrideIfExists = false)
+      createTempView(catalog, "temp_view4", tempTable, overrideIfExists = false)
 
       assert(catalog.listTables("mydb").toSet == catalog.listTables("mydb", "*").toSet)
       assert(catalog.listTables("mydb").toSet == catalog.listTables("mydb", "*", true).toSet)
@@ -774,8 +804,8 @@ abstract class SessionCatalogSuite extends AnalysisTest {
   test("list temporary view with pattern") {
     withBasicCatalog { catalog =>
       val tempTable = Range(1, 10, 2, 10)
-      catalog.createTempView("temp_view1", tempTable, overrideIfExists = false)
-      catalog.createTempView("temp_view4", tempTable, overrideIfExists = false)
+      createTempView(catalog, "temp_view1", tempTable, overrideIfExists = false)
+      createTempView(catalog, "temp_view4", tempTable, overrideIfExists = false)
       assert(catalog.listLocalTempViews("*").toSet ==
         Set(TableIdentifier("temp_view1"), TableIdentifier("temp_view4")))
       assert(catalog.listLocalTempViews("temp_view*").toSet ==
@@ -788,10 +818,10 @@ abstract class SessionCatalogSuite extends AnalysisTest {
   test("list global temporary view and local temporary view with pattern") {
     withBasicCatalog { catalog =>
       val tempTable = Range(1, 10, 2, 10)
-      catalog.createTempView("temp_view1", tempTable, overrideIfExists = false)
-      catalog.createTempView("temp_view4", tempTable, overrideIfExists = false)
-      catalog.globalTempViewManager.create("global_temp_view1", tempTable, overrideIfExists = false)
-      catalog.globalTempViewManager.create("global_temp_view2", tempTable, overrideIfExists = false)
+      createTempView(catalog, "temp_view1", tempTable, overrideIfExists = false)
+      createTempView(catalog, "temp_view4", tempTable, overrideIfExists = false)
+      createGlobalTempView(catalog, "global_temp_view1", tempTable, overrideIfExists = false)
+      createGlobalTempView(catalog, "global_temp_view2", tempTable, overrideIfExists = false)
       assert(catalog.listTables(catalog.globalTempViewManager.database, "*").toSet ==
         Set(TableIdentifier("temp_view1"),
           TableIdentifier("temp_view4"),
@@ -1554,21 +1584,21 @@ abstract class SessionCatalogSuite extends AnalysisTest {
   test("copy SessionCatalog state - temp views") {
     withEmptyCatalog { original =>
       val tempTable1 = Range(1, 10, 1, 10)
-      original.createTempView("copytest1", tempTable1, overrideIfExists = false)
+      createTempView(original, "copytest1", tempTable1, overrideIfExists = false)
 
       // check if tables copied over
       val clone = new SessionCatalog(original.externalCatalog)
       original.copyStateTo(clone)
 
       assert(original ne clone)
-      assert(clone.getTempView("copytest1") == Some(tempTable1))
+      assert(getTempViewRawPlan(clone.getTempView("copytest1")) == Some(tempTable1))
 
       // check if clone and original independent
       clone.dropTable(TableIdentifier("copytest1"), ignoreIfNotExists = false, purge = false)
-      assert(original.getTempView("copytest1") == Some(tempTable1))
+      assert(getTempViewRawPlan(original.getTempView("copytest1")) == Some(tempTable1))
 
       val tempTable2 = Range(1, 20, 2, 10)
-      original.createTempView("copytest2", tempTable2, overrideIfExists = false)
+      createTempView(original, "copytest2", tempTable2, overrideIfExists = false)
       assert(clone.getTempView("copytest2").isEmpty)
     }
   }
@@ -1604,26 +1634,28 @@ abstract class SessionCatalogSuite extends AnalysisTest {
     import org.apache.spark.sql.catalyst.dsl.plans._
 
     Seq(true, false) foreach { caseSensitive =>
-      val conf = new SQLConf().copy(SQLConf.CASE_SENSITIVE -> caseSensitive)
-      val catalog = new SessionCatalog(newBasicCatalog(), new SimpleFunctionRegistry, conf)
-      catalog.setCurrentDatabase("db1")
-      try {
-        val analyzer = new Analyzer(catalog, conf)
-
-        // The analyzer should report the undefined function rather than the undefined table first.
-        val cause = intercept[AnalysisException] {
-          analyzer.execute(
-            UnresolvedRelation(TableIdentifier("undefined_table")).select(
-              UnresolvedFunction("undefined_fn", Nil, isDistinct = false)
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
+        val catalog = new SessionCatalog(newBasicCatalog(), new SimpleFunctionRegistry)
+        catalog.setCurrentDatabase("db1")
+        try {
+          val analyzer = new Analyzer(catalog)
+
+          // The analyzer should report the undefined function
+          // rather than the undefined table first.
+          val cause = intercept[AnalysisException] {
+            analyzer.execute(
+              UnresolvedRelation(TableIdentifier("undefined_table")).select(
+                UnresolvedFunction("undefined_fn", Nil, isDistinct = false)
+              )
             )
-          )
-        }
+          }
 
-        assert(cause.getMessage.contains("Undefined function: 'undefined_fn'"))
-        // SPARK-21318: the error message should contains the current database name
-        assert(cause.getMessage.contains("db1"))
-      } finally {
-        catalog.reset()
+          assert(cause.getMessage.contains("Undefined function: 'undefined_fn'"))
+          // SPARK-21318: the error message should contains the current database name
+          assert(cause.getMessage.contains("db1"))
+        } finally {
+          catalog.reset()
+        }
       }
     }
   }
@@ -1641,4 +1673,43 @@ abstract class SessionCatalogSuite extends AnalysisTest {
       assert(cause.cause.get.getMessage.contains("Actual error"))
     }
   }
+
+  test("expire table relation cache if TTL is configured") {
+    case class TestCommand() extends LeafCommand
+
+    val conf = new SQLConf()
+    conf.setConf(StaticSQLConf.METADATA_CACHE_TTL_SECONDS, 1L)
+
+    withConfAndEmptyCatalog(conf) { catalog =>
+      val table = QualifiedTableName(catalog.getCurrentDatabase, "test")
+
+      // First, make sure the test table is not cached.
+      assert(catalog.getCachedTable(table) === null)
+
+      catalog.cacheTable(table, TestCommand())
+      assert(catalog.getCachedTable(table) !== null)
+
+      // Wait until the cache expiration.
+      eventually(timeout(3.seconds)) {
+        // And the cache is gone.
+        assert(catalog.getCachedTable(table) === null)
+      }
+    }
+  }
+
+  test("SPARK-34197: refreshTable should not invalidate the relation cache for temporary views") {
+    withBasicCatalog { catalog =>
+      createTempView(catalog, "tbl1", Range(1, 10, 1, 10), false)
+      val qualifiedName1 = QualifiedTableName("default", "tbl1")
+      catalog.cacheTable(qualifiedName1, Range(1, 10, 1, 10))
+      catalog.refreshTable(TableIdentifier("tbl1"))
+      assert(catalog.getCachedTable(qualifiedName1) != null)
+
+      createGlobalTempView(catalog, "tbl2", Range(2, 10, 1, 10), false)
+      val qualifiedName2 = QualifiedTableName(catalog.globalTempViewManager.database, "tbl2")
+      catalog.cacheTable(qualifiedName2, Range(2, 10, 1, 10))
+      catalog.refreshTable(TableIdentifier("tbl2", Some(catalog.globalTempViewManager.database)))
+      assert(catalog.getCachedTable(qualifiedName2) != null)
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtilsSuite.scala
index 06ffb4231062c..fcb10c98243d9 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtilsSuite.scala
@@ -29,7 +29,7 @@ class CSVExprUtilsSuite extends SparkFunSuite {
     assert(CSVExprUtils.toChar("""\f""") === '\f')
     assert(CSVExprUtils.toChar("""\"""") === '\"')
     assert(CSVExprUtils.toChar("""\'""") === '\'')
-    assert(CSVExprUtils.toChar("""\u0000""") === '\u0000')
+    assert(CSVExprUtils.toChar("\u0000") === '\u0000')
     assert(CSVExprUtils.toChar("""\\""") === '\\')
   }
 
@@ -76,7 +76,7 @@ class CSVExprUtilsSuite extends SparkFunSuite {
     // tab in the middle of some other letters
     ("""ba\tr""", Some("ba\tr"), None),
     // null character, expressed in Unicode literal syntax
-    ("""\u0000""", Some("\u0000"), None),
+    ("\u0000", Some("\u0000"), None),
     // and specified directly
     ("\u0000", Some("\u0000"), None)
   )
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVFiltersSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVFiltersSuite.scala
deleted file mode 100644
index 499bbaf452aee..0000000000000
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVFiltersSuite.scala
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.catalyst.csv
-
-import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.sources
-import org.apache.spark.sql.sources.{AlwaysFalse, AlwaysTrue, Filter}
-import org.apache.spark.sql.types.{IntegerType, StructType}
-import org.apache.spark.unsafe.types.UTF8String
-
-class CSVFiltersSuite extends SparkFunSuite {
-  test("filter to expression conversion") {
-    val ref = BoundReference(0, IntegerType, true)
-    def check(f: Filter, expr: Expression): Unit = {
-      assert(CSVFilters.filterToExpression(f, _ => Some(ref)).get === expr)
-    }
-
-    check(sources.AlwaysTrue, Literal(true))
-    check(sources.AlwaysFalse, Literal(false))
-    check(sources.IsNull("a"), IsNull(ref))
-    check(sources.Not(sources.IsNull("a")), Not(IsNull(ref)))
-    check(sources.IsNotNull("a"), IsNotNull(ref))
-    check(sources.EqualTo("a", "b"), EqualTo(ref, Literal("b")))
-    check(sources.EqualNullSafe("a", "b"), EqualNullSafe(ref, Literal("b")))
-    check(sources.StringStartsWith("a", "b"), StartsWith(ref, Literal("b")))
-    check(sources.StringEndsWith("a", "b"), EndsWith(ref, Literal("b")))
-    check(sources.StringContains("a", "b"), Contains(ref, Literal("b")))
-    check(sources.LessThanOrEqual("a", 1), LessThanOrEqual(ref, Literal(1)))
-    check(sources.LessThan("a", 1), LessThan(ref, Literal(1)))
-    check(sources.GreaterThanOrEqual("a", 1), GreaterThanOrEqual(ref, Literal(1)))
-    check(sources.GreaterThan("a", 1), GreaterThan(ref, Literal(1)))
-    check(sources.And(sources.AlwaysTrue, sources.AlwaysTrue), And(Literal(true), Literal(true)))
-    check(sources.Or(sources.AlwaysTrue, sources.AlwaysTrue), Or(Literal(true), Literal(true)))
-    check(sources.In("a", Array(1)), In(ref, Seq(Literal(1))))
-  }
-
-  private def getSchema(str: String): StructType = str match {
-    case "" => new StructType()
-    case _ => StructType.fromDDL(str)
-  }
-
-  test("skipping rows") {
-    def check(
-        requiredSchema: String = "i INTEGER, d DOUBLE",
-        filters: Seq[Filter],
-        row: InternalRow,
-        pos: Int,
-        skip: Boolean): Unit = {
-      val csvFilters = new CSVFilters(filters, getSchema(requiredSchema))
-      assert(csvFilters.skipRow(row, pos) === skip)
-    }
-
-    check(filters = Seq(), row = InternalRow(3.14), pos = 0, skip = false)
-    check(filters = Seq(AlwaysTrue), row = InternalRow(1), pos = 0, skip = false)
-    check(filters = Seq(AlwaysFalse), row = InternalRow(1), pos = 0, skip = true)
-    check(
-      filters = Seq(sources.EqualTo("i", 1), sources.LessThan("d", 10), sources.AlwaysFalse),
-      row = InternalRow(1, 3.14),
-      pos = 0,
-      skip = true)
-    check(
-      filters = Seq(sources.EqualTo("i", 10)),
-      row = InternalRow(10, 3.14),
-      pos = 0,
-      skip = false)
-    check(
-      filters = Seq(sources.IsNotNull("d"), sources.GreaterThanOrEqual("d", 2.96)),
-      row = InternalRow(3.14),
-      pos = 0,
-      skip = false)
-    check(
-      filters = Seq(sources.In("i", Array(10, 20)), sources.LessThanOrEqual("d", 2.96)),
-      row = InternalRow(10, 3.14),
-      pos = 1,
-      skip = true)
-    val filters1 = Seq(
-      sources.Or(
-        sources.AlwaysTrue,
-        sources.And(
-          sources.Not(sources.IsNull("i")),
-          sources.Not(
-            sources.And(
-              sources.StringEndsWith("s", "ab"),
-              sources.StringEndsWith("s", "cd")
-            )
-          )
-        )
-      ),
-      sources.GreaterThan("d", 0),
-      sources.LessThan("i", 500)
-    )
-    val filters2 = Seq(
-      sources.And(
-        sources.StringContains("s", "abc"),
-        sources.And(
-          sources.Not(sources.IsNull("i")),
-          sources.And(
-            sources.StringEndsWith("s", "ab"),
-            sources.StringEndsWith("s", "bc")
-          )
-        )
-      ),
-      sources.GreaterThan("d", 100),
-      sources.LessThan("i", 0)
-    )
-    Seq(filters1 -> false, filters2 -> true).foreach { case (filters, skip) =>
-      for (p <- 0 until 3) {
-        check(
-          requiredSchema = "i INTEGER, d DOUBLE, s STRING",
-          filters = filters,
-          row = InternalRow(10, 3.14, UTF8String.fromString("abc")),
-          pos = p,
-          skip = skip)
-      }
-    }
-  }
-}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchemaSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchemaSuite.scala
index b014eb92fae50..d268f8c2e7210 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchemaSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchemaSuite.scala
@@ -56,11 +56,11 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
     assert(inferSchema.inferField(IntegerType, "1.0") == DoubleType)
     assert(inferSchema.inferField(DoubleType, null) == DoubleType)
     assert(inferSchema.inferField(DoubleType, "test") == StringType)
-    assert(inferSchema.inferField(LongType, "2015-08-20 14:57:00") == TimestampType)
-    assert(inferSchema.inferField(DoubleType, "2015-08-20 15:57:00") == TimestampType)
-    assert(inferSchema.inferField(LongType, "True") == BooleanType)
-    assert(inferSchema.inferField(IntegerType, "FALSE") == BooleanType)
-    assert(inferSchema.inferField(TimestampType, "FALSE") == BooleanType)
+    assert(inferSchema.inferField(LongType, "2015-08-20 14:57:00") == StringType)
+    assert(inferSchema.inferField(DoubleType, "2015-08-20 15:57:00") == StringType)
+    assert(inferSchema.inferField(LongType, "True") == StringType)
+    assert(inferSchema.inferField(IntegerType, "FALSE") == StringType)
+    assert(inferSchema.inferField(TimestampType, "FALSE") == StringType)
 
     val textValueOne = Long.MaxValue.toString + "0"
     val decimalValueOne = new java.math.BigDecimal(textValueOne)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala
index 0e9fcc980aabb..822008007ebbc 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala
@@ -325,7 +325,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
       assert(parser.makeConverter("t", TimestampType).apply("2020-1-12 12:3:45") ==
         date(2020, 1, 12, 12, 3, 45, 0))
       assert(parser.makeConverter("t", DateType).apply("2020-1-12") ==
-        days(2020, 1, 12, 0, 0, 0))
+        days(2020, 1, 12))
       // The legacy format allows arbitrary length of second fraction.
       assert(parser.makeConverter("t", TimestampType).apply("2020-1-12 12:3:45.1") ==
         date(2020, 1, 12, 12, 3, 45, 100000))
@@ -333,22 +333,22 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
         date(2020, 1, 12, 12, 3, 45, 123400))
       // The legacy format allow date string to end with T or space, with arbitrary string
       assert(parser.makeConverter("t", DateType).apply("2020-1-12T") ==
-        days(2020, 1, 12, 0, 0, 0))
+        days(2020, 1, 12))
       assert(parser.makeConverter("t", DateType).apply("2020-1-12Txyz") ==
-        days(2020, 1, 12, 0, 0, 0))
+        days(2020, 1, 12))
       assert(parser.makeConverter("t", DateType).apply("2020-1-12 ") ==
-        days(2020, 1, 12, 0, 0, 0))
+        days(2020, 1, 12))
       assert(parser.makeConverter("t", DateType).apply("2020-1-12 xyz") ==
-        days(2020, 1, 12, 0, 0, 0))
+        days(2020, 1, 12))
       // The legacy format ignores the "GMT" from the string
       assert(parser.makeConverter("t", TimestampType).apply("2020-1-12 12:3:45GMT") ==
         date(2020, 1, 12, 12, 3, 45, 0))
       assert(parser.makeConverter("t", TimestampType).apply("GMT2020-1-12 12:3:45") ==
         date(2020, 1, 12, 12, 3, 45, 0))
       assert(parser.makeConverter("t", DateType).apply("2020-1-12GMT") ==
-        days(2020, 1, 12, 0, 0, 0))
+        days(2020, 1, 12))
       assert(parser.makeConverter("t", DateType).apply("GMT2020-1-12") ==
-        days(2020, 1, 12, 0, 0, 0))
+        days(2020, 1, 12))
     }
 
     val options = new CSVOptions(Map.empty[String, String], false, "UTC")
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala
index 48f4ef5051fb3..3a02c837aba3c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala
@@ -22,9 +22,9 @@ import scala.reflect.runtime.universe.TypeTag
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.dsl.expressions._
-import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
 import org.apache.spark.sql.catalyst.plans.PlanTest
-import org.apache.spark.sql.catalyst.util.GenericArrayData
+import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData}
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -211,7 +211,7 @@ class EncoderResolutionSuite extends PlanTest {
       val attrs = Seq(attr)
       assert(intercept[AnalysisException](encoder.resolveAndBind(attrs)).message ==
         s"""
-           |Cannot up cast `a` from ${attr.dataType.catalogString} to string.
+           |Cannot up cast a from ${attr.dataType.catalogString} to string.
            |The type path of the target object is:
            |- root class: "java.lang.String"
            |You can either add an explicit cast to the input data or choose a higher precision type
@@ -225,7 +225,7 @@ class EncoderResolutionSuite extends PlanTest {
     }.message
     assert(msg1 ==
       s"""
-         |Cannot up cast `b` from bigint to int.
+         |Cannot up cast b from bigint to int.
          |The type path of the target object is:
          |- field (class: "scala.Int", name: "b")
          |- root class: "org.apache.spark.sql.catalyst.encoders.StringIntClass"
@@ -238,7 +238,7 @@ class EncoderResolutionSuite extends PlanTest {
     }.message
     assert(msg2 ==
       s"""
-         |Cannot up cast `b`.`b` from decimal(38,18) to bigint.
+         |Cannot up cast b.`b` from decimal(38,18) to bigint.
          |The type path of the target object is:
          |- field (class: "scala.Long", name: "b")
          |- field (class: "org.apache.spark.sql.catalyst.encoders.StringLongClass", name: "b")
@@ -247,6 +247,13 @@ class EncoderResolutionSuite extends PlanTest {
        """.stripMargin.trim + " of the field in the target object")
   }
 
+  test("SPARK-31750: eliminate UpCast if child's dataType is DecimalType") {
+    val encoder = ExpressionEncoder[Seq[BigDecimal]]
+    val attr = Seq(AttributeReference("a", ArrayType(DecimalType(38, 0)))())
+    // Before SPARK-31750, it will fail because Decimal(38, 0) can not be casted to Decimal(38, 18)
+    testFromRow(encoder, attr, InternalRow(ArrayData.toArrayData(Array(Decimal(1.0)))))
+  }
+
   // test for leaf types
   castSuccess[Int, Long]
   castSuccess[java.sql.Date, java.sql.Timestamp]
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
index 6a094d4aaddae..7faab4e7aa757 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
@@ -25,7 +25,7 @@ import scala.collection.mutable.ArrayBuffer
 import scala.reflect.runtime.universe.TypeTag
 
 import org.apache.spark.sql.{Encoder, Encoders}
-import org.apache.spark.sql.catalyst.{OptionalData, PrimitiveData}
+import org.apache.spark.sql.catalyst.{FooClassWithEnum, FooEnum, OptionalData, PrimitiveData}
 import org.apache.spark.sql.catalyst.analysis.AnalysisTest
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.expressions.AttributeReference
@@ -195,8 +195,9 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
     encoderFor(Encoders.javaSerialization[JavaSerializable]))
 
   // test product encoders
-  private def productTest[T <: Product : ExpressionEncoder](input: T): Unit = {
-    encodeDecodeTest(input, input.getClass.getSimpleName)
+  private def productTest[T <: Product : ExpressionEncoder](
+      input: T, useFallback: Boolean = false): Unit = {
+    encodeDecodeTest(input, input.getClass.getSimpleName, useFallback)
   }
 
   case class InnerClass(i: Int)
@@ -205,6 +206,90 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
 
   encodeDecodeTest(Array(Option(InnerClass(1))), "array of optional inner class")
 
+  // holder class to trigger Class.getSimpleName issue
+  object MalformedClassObject extends Serializable {
+    case class MalformedNameExample(x: Int)
+  }
+
+  {
+    OuterScopes.addOuterScope(MalformedClassObject)
+    encodeDecodeTest(
+      MalformedClassObject.MalformedNameExample(42),
+      "nested Scala class should work",
+      useFallback = true)
+  }
+
+  object OuterLevelWithVeryVeryVeryLongClassName1 {
+    object OuterLevelWithVeryVeryVeryLongClassName2 {
+      object OuterLevelWithVeryVeryVeryLongClassName3 {
+        object OuterLevelWithVeryVeryVeryLongClassName4 {
+          object OuterLevelWithVeryVeryVeryLongClassName5 {
+            object OuterLevelWithVeryVeryVeryLongClassName6 {
+              object OuterLevelWithVeryVeryVeryLongClassName7 {
+                object OuterLevelWithVeryVeryVeryLongClassName8 {
+                  object OuterLevelWithVeryVeryVeryLongClassName9 {
+                    object OuterLevelWithVeryVeryVeryLongClassName10 {
+                      object OuterLevelWithVeryVeryVeryLongClassName11 {
+                        object OuterLevelWithVeryVeryVeryLongClassName12 {
+                          object OuterLevelWithVeryVeryVeryLongClassName13 {
+                            object OuterLevelWithVeryVeryVeryLongClassName14 {
+                              object OuterLevelWithVeryVeryVeryLongClassName15 {
+                                object OuterLevelWithVeryVeryVeryLongClassName16 {
+                                  object OuterLevelWithVeryVeryVeryLongClassName17 {
+                                    object OuterLevelWithVeryVeryVeryLongClassName18 {
+                                      object OuterLevelWithVeryVeryVeryLongClassName19 {
+                                        object OuterLevelWithVeryVeryVeryLongClassName20 {
+                                          case class MalformedNameExample(x: Int)
+                                        }}}}}}}}}}}}}}}}}}}}
+
+  {
+    OuterScopes.addOuterScope(
+      OuterLevelWithVeryVeryVeryLongClassName1
+        .OuterLevelWithVeryVeryVeryLongClassName2
+        .OuterLevelWithVeryVeryVeryLongClassName3
+        .OuterLevelWithVeryVeryVeryLongClassName4
+        .OuterLevelWithVeryVeryVeryLongClassName5
+        .OuterLevelWithVeryVeryVeryLongClassName6
+        .OuterLevelWithVeryVeryVeryLongClassName7
+        .OuterLevelWithVeryVeryVeryLongClassName8
+        .OuterLevelWithVeryVeryVeryLongClassName9
+        .OuterLevelWithVeryVeryVeryLongClassName10
+        .OuterLevelWithVeryVeryVeryLongClassName11
+        .OuterLevelWithVeryVeryVeryLongClassName12
+        .OuterLevelWithVeryVeryVeryLongClassName13
+        .OuterLevelWithVeryVeryVeryLongClassName14
+        .OuterLevelWithVeryVeryVeryLongClassName15
+        .OuterLevelWithVeryVeryVeryLongClassName16
+        .OuterLevelWithVeryVeryVeryLongClassName17
+        .OuterLevelWithVeryVeryVeryLongClassName18
+        .OuterLevelWithVeryVeryVeryLongClassName19
+        .OuterLevelWithVeryVeryVeryLongClassName20)
+    encodeDecodeTest(
+      OuterLevelWithVeryVeryVeryLongClassName1
+        .OuterLevelWithVeryVeryVeryLongClassName2
+        .OuterLevelWithVeryVeryVeryLongClassName3
+        .OuterLevelWithVeryVeryVeryLongClassName4
+        .OuterLevelWithVeryVeryVeryLongClassName5
+        .OuterLevelWithVeryVeryVeryLongClassName6
+        .OuterLevelWithVeryVeryVeryLongClassName7
+        .OuterLevelWithVeryVeryVeryLongClassName8
+        .OuterLevelWithVeryVeryVeryLongClassName9
+        .OuterLevelWithVeryVeryVeryLongClassName10
+        .OuterLevelWithVeryVeryVeryLongClassName11
+        .OuterLevelWithVeryVeryVeryLongClassName12
+        .OuterLevelWithVeryVeryVeryLongClassName13
+        .OuterLevelWithVeryVeryVeryLongClassName14
+        .OuterLevelWithVeryVeryVeryLongClassName15
+        .OuterLevelWithVeryVeryVeryLongClassName16
+        .OuterLevelWithVeryVeryVeryLongClassName17
+        .OuterLevelWithVeryVeryVeryLongClassName18
+        .OuterLevelWithVeryVeryVeryLongClassName19
+        .OuterLevelWithVeryVeryVeryLongClassName20
+        .MalformedNameExample(42),
+      "deeply nested Scala class should work",
+      useFallback = true)
+  }
+
   productTest(PrimitiveData(1, 1, 1, 1, 1, 1, true))
 
   productTest(
@@ -214,7 +299,8 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
   productTest(OptionalData(None, None, None, None, None, None, None, None, None))
 
   encodeDecodeTest(Seq(Some(1), None), "Option in array")
-  encodeDecodeTest(Map(1 -> Some(10L), 2 -> Some(20L), 3 -> None), "Option in map")
+  encodeDecodeTest(Map(1 -> Some(10L), 2 -> Some(20L), 3 -> None), "Option in map",
+    useFallback = true)
 
   productTest(BoxedData(1, 1L, 1.0, 1.0f, 1.toShort, 1.toByte, true))
 
@@ -232,7 +318,7 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
       Map(1 -> null),
       PrimitiveData(1, 1, 1, 1, 1, 1, true)))
 
-  productTest(NestedArray(Array(Array(1, -2, 3), null, Array(4, 5, -6))))
+  productTest(NestedArray(Array(Array(1, -2, 3), null, Array(4, 5, -6))), useFallback = true)
 
   productTest(("Seq[(String, String)]",
     Seq(("a", "b"))))
@@ -389,6 +475,16 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
     assert(e.getMessage.contains("tuple with more than 22 elements are not supported"))
   }
 
+  encodeDecodeTest((1, FooEnum.E1), "Tuple with Int and scala Enum")
+  encodeDecodeTest((null, FooEnum.E1, FooEnum.E2), "Tuple with Null and scala Enum")
+  encodeDecodeTest(Seq(FooEnum.E1, null), "Seq with scala Enum")
+  encodeDecodeTest(Map("key" -> FooEnum.E1), "Map with String key and scala Enum",
+    useFallback = true)
+  encodeDecodeTest(Map(FooEnum.E1 -> "value"), "Map with scala Enum key and String value",
+    useFallback = true)
+  encodeDecodeTest(FooClassWithEnum(1, FooEnum.E1), "case class with Int and scala Enum")
+  encodeDecodeTest(FooEnum.E1, "scala Enum")
+
   // Scala / Java big decimals ----------------------------------------------------------
 
   encodeDecodeTest(BigDecimal(("9" * 20) + "." + "9" * 18),
@@ -465,8 +561,9 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
 
   private def encodeDecodeTest[T : ExpressionEncoder](
       input: T,
-      testName: String): Unit = {
-    testAndVerifyNotLeakingReflectionObjects(s"encode/decode for $testName: $input") {
+      testName: String,
+      useFallback: Boolean = false): Unit = {
+    testAndVerifyNotLeakingReflectionObjects(s"encode/decode for $testName: $input", useFallback) {
       val encoder = implicitly[ExpressionEncoder[T]]
 
       // Make sure encoder is serializable.
@@ -497,10 +594,8 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
       val isCorrect = (input, convertedBack) match {
         case (b1: Array[Byte], b2: Array[Byte]) => Arrays.equals(b1, b2)
         case (b1: Array[Int], b2: Array[Int]) => Arrays.equals(b1, b2)
-        case (b1: Array[Array[_]], b2: Array[Array[_]]) =>
-          Arrays.deepEquals(b1.asInstanceOf[Array[AnyRef]], b2.asInstanceOf[Array[AnyRef]])
         case (b1: Array[_], b2: Array[_]) =>
-          Arrays.equals(b1.asInstanceOf[Array[AnyRef]], b2.asInstanceOf[Array[AnyRef]])
+          Arrays.deepEquals(b1.asInstanceOf[Array[AnyRef]], b2.asInstanceOf[Array[AnyRef]])
         case (left: Comparable[_], right: Comparable[_]) =>
           left.asInstanceOf[Comparable[Any]].compareTo(right) == 0
         case _ => input == convertedBack
@@ -560,9 +655,16 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
     r
   }
 
-  private def testAndVerifyNotLeakingReflectionObjects(testName: String)(testFun: => Any): Unit = {
-    test(testName) {
-      verifyNotLeakingReflectionObjects(testFun)
+  private def testAndVerifyNotLeakingReflectionObjects(
+      testName: String, useFallback: Boolean = false)(testFun: => Any): Unit = {
+    if (useFallback) {
+      testFallback(testName) {
+        verifyNotLeakingReflectionObjects(testFun)
+      }
+    } else {
+      test(testName) {
+        verifyNotLeakingReflectionObjects(testFun)
+      }
     }
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
index fd24f058f357c..6c22c14870d6d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
@@ -22,7 +22,7 @@ import scala.util.Random
 import org.apache.spark.sql.{RandomDataGenerator, Row}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.plans.CodegenInterpretedPlanTest
-import org.apache.spark.sql.catalyst.util.{ArrayData, DateTimeUtils, GenericArrayData}
+import org.apache.spark.sql.catalyst.util.{ArrayData, DateTimeUtils, GenericArrayData, IntervalUtils}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
@@ -336,12 +336,32 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
       val encoder = RowEncoder(schema).resolveAndBind()
       val localDate = java.time.LocalDate.parse("2019-02-27")
       val row = toRow(encoder, Row(localDate))
-      assert(row.getLong(0) === DateTimeUtils.localDateToDays(localDate))
+      assert(row.getInt(0) === DateTimeUtils.localDateToDays(localDate))
       val readback = fromRow(encoder, row)
       assert(readback.get(0).equals(localDate))
     }
   }
 
+  test("SPARK-34605: encoding/decoding DayTimeIntervalType to/from java.time.Duration") {
+    val schema = new StructType().add("d", DayTimeIntervalType)
+    val encoder = RowEncoder(schema).resolveAndBind()
+    val duration = java.time.Duration.ofDays(1)
+    val row = toRow(encoder, Row(duration))
+    assert(row.getLong(0) === IntervalUtils.durationToMicros(duration))
+    val readback = fromRow(encoder, row)
+    assert(readback.get(0).equals(duration))
+  }
+
+  test("SPARK-34615: encoding/decoding YearMonthIntervalType to/from java.time.Period") {
+    val schema = new StructType().add("p", YearMonthIntervalType)
+    val encoder = RowEncoder(schema).resolveAndBind()
+    val period = java.time.Period.ofMonths(1)
+    val row = toRow(encoder, Row(period))
+    assert(row.getInt(0) === IntervalUtils.periodToMonths(period))
+    val readback = fromRow(encoder, row)
+    assert(readback.get(0).equals(period))
+  }
+
   for {
     elementType <- Seq(IntegerType, StringType)
     containsNull <- Seq(true, false)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala
index 675f85f9e82ea..5d07912c68963 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.catalyst.expressions
 
 import java.sql.{Date, Timestamp}
+import java.time.{Duration, Period}
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.InternalRow
@@ -60,10 +61,10 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
     checkEvaluation(Add(positiveIntLit, negativeIntLit), -1)
     checkEvaluation(Add(positiveLongLit, negativeLongLit), -1L)
 
-    Seq("true", "false").foreach { checkOverflow =>
-      withSQLConf(SQLConf.ANSI_ENABLED.key -> checkOverflow) {
+    Seq("true", "false").foreach { failOnError =>
+      withSQLConf(SQLConf.ANSI_ENABLED.key -> failOnError) {
         DataTypeTestUtils.numericAndInterval.foreach { tpe =>
-          checkConsistencyBetweenInterpretedAndCodegenAllowingException(Add, tpe, tpe)
+          checkConsistencyBetweenInterpretedAndCodegenAllowingException(Add(_, _), tpe, tpe)
         }
       }
     }
@@ -103,8 +104,12 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
     checkEvaluation(UnaryMinus(positiveLongLit), - positiveLong)
     checkEvaluation(UnaryMinus(negativeLongLit), - negativeLong)
 
-    DataTypeTestUtils.numericAndInterval.foreach { tpe =>
-      checkConsistencyBetweenInterpretedAndCodegen(UnaryMinus, tpe)
+    Seq("true", "false").foreach { failOnError =>
+      withSQLConf(SQLConf.ANSI_ENABLED.key -> failOnError) {
+        DataTypeTestUtils.numericAndInterval.foreach { tpe =>
+          checkConsistencyBetweenInterpretedAndCodegenAllowingException(UnaryMinus(_), tpe)
+        }
+      }
     }
   }
 
@@ -121,10 +126,10 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
     checkEvaluation(Subtract(positiveIntLit, negativeIntLit), positiveInt - negativeInt)
     checkEvaluation(Subtract(positiveLongLit, negativeLongLit), positiveLong - negativeLong)
 
-    Seq("true", "false").foreach { checkOverflow =>
-      withSQLConf(SQLConf.ANSI_ENABLED.key -> checkOverflow) {
+    Seq("true", "false").foreach { failOnError =>
+      withSQLConf(SQLConf.ANSI_ENABLED.key -> failOnError) {
         DataTypeTestUtils.numericAndInterval.foreach { tpe =>
-          checkConsistencyBetweenInterpretedAndCodegenAllowingException(Subtract, tpe, tpe)
+          checkConsistencyBetweenInterpretedAndCodegenAllowingException(Subtract(_, _), tpe, tpe)
         }
       }
     }
@@ -143,10 +148,10 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
     checkEvaluation(Multiply(positiveIntLit, negativeIntLit), positiveInt * negativeInt)
     checkEvaluation(Multiply(positiveLongLit, negativeLongLit), positiveLong * negativeLong)
 
-    Seq("true", "false").foreach { checkOverflow =>
-      withSQLConf(SQLConf.ANSI_ENABLED.key -> checkOverflow) {
+    Seq("true", "false").foreach { failOnError =>
+      withSQLConf(SQLConf.ANSI_ENABLED.key -> failOnError) {
         DataTypeTestUtils.numericTypeWithoutDecimal.foreach { tpe =>
-          checkConsistencyBetweenInterpretedAndCodegenAllowingException(Multiply, tpe, tpe)
+          checkConsistencyBetweenInterpretedAndCodegenAllowingException(Multiply(_, _), tpe, tpe)
         }
       }
     }
@@ -161,26 +166,52 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
     testDecimalAndDoubleType { convert =>
       val left = Literal(convert(2))
       val right = Literal(convert(1))
-      val dataType = left.dataType
       checkEvaluation(Divide(left, right), convert(2))
-      checkEvaluation(Divide(Literal.create(null, dataType), right), null)
+      checkEvaluation(Divide(Literal.create(null, left.dataType), right), null)
       checkEvaluation(Divide(left, Literal.create(null, right.dataType)), null)
       checkEvaluation(Divide(left, Literal(convert(0))), null)  // divide by zero
     }
 
-    Seq(DoubleType, DecimalType.SYSTEM_DEFAULT).foreach { tpe =>
-      checkConsistencyBetweenInterpretedAndCodegen(Divide, tpe, tpe)
+    Seq("true", "false").foreach { failOnError =>
+      withSQLConf(SQLConf.ANSI_ENABLED.key -> failOnError) {
+        Seq(DoubleType, DecimalType.SYSTEM_DEFAULT).foreach { tpe =>
+          checkConsistencyBetweenInterpretedAndCodegenAllowingException(Divide(_, _), tpe, tpe)
+        }
+      }
     }
   }
 
-  test("/ (Divide) for integral type") {
-    checkEvaluation(IntegralDivide(Literal(1.toByte), Literal(2.toByte)), 0L)
-    checkEvaluation(IntegralDivide(Literal(1.toShort), Literal(2.toShort)), 0L)
-    checkEvaluation(IntegralDivide(Literal(1), Literal(2)), 0L)
-    checkEvaluation(IntegralDivide(Literal(1.toLong), Literal(2.toLong)), 0L)
-    checkEvaluation(IntegralDivide(positiveShortLit, negativeShortLit), 0L)
-    checkEvaluation(IntegralDivide(positiveIntLit, negativeIntLit), 0L)
+  private def testDecimalAndLongType(testFunc: (Int => Any) => Unit): Unit = {
+    testFunc(_.toLong)
+    testFunc(Decimal(_))
+  }
+
+  test("/ (Divide) for Long and Decimal type") {
+    testDecimalAndLongType { convert =>
+      val left = Literal(convert(1))
+      val right = Literal(convert(2))
+      checkEvaluation(IntegralDivide(left, right), 0L)
+      checkEvaluation(IntegralDivide(Literal.create(null, left.dataType), right), null)
+      checkEvaluation(IntegralDivide(left, Literal.create(null, right.dataType)), null)
+      checkEvaluation(IntegralDivide(left, Literal(convert(0))), null)  // divide by zero
+    }
     checkEvaluation(IntegralDivide(positiveLongLit, negativeLongLit), 0L)
+
+    Seq("true", "false").foreach { failOnError =>
+      withSQLConf(SQLConf.ANSI_ENABLED.key -> failOnError) {
+        Seq(LongType, DecimalType.SYSTEM_DEFAULT).foreach { tpe =>
+          checkConsistencyBetweenInterpretedAndCodegenAllowingException(
+            IntegralDivide(_, _), tpe, tpe)
+        }
+      }
+    }
+  }
+
+  test("IntegralDivide: throw exception on overflow under ANSI mode") {
+    withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
+      checkExceptionInExpression[ArithmeticException](
+        IntegralDivide(Literal(Long.MinValue), Literal(-1L)), "Overflow in integral divide.")
+    }
   }
 
   test("% (Remainder)") {
@@ -199,8 +230,12 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
     checkEvaluation(Remainder(positiveLongLit, positiveLongLit), 0L)
     checkEvaluation(Remainder(negativeLongLit, negativeLongLit), 0L)
 
-    DataTypeTestUtils.numericTypeWithoutDecimal.foreach { tpe =>
-      checkConsistencyBetweenInterpretedAndCodegen(Remainder, tpe, tpe)
+    Seq("true", "false").foreach { failOnError =>
+      withSQLConf(SQLConf.ANSI_ENABLED.key -> failOnError) {
+        DataTypeTestUtils.numericTypeWithoutDecimal.foreach { tpe =>
+          checkConsistencyBetweenInterpretedAndCodegenAllowingException(Remainder(_, _), tpe, tpe)
+        }
+      }
     }
   }
 
@@ -216,13 +251,19 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
   }
 
   test("Abs") {
-    testNumericDataTypes { convert =>
-      val input = Literal(convert(1))
-      val dataType = input.dataType
-      checkEvaluation(Abs(Literal(convert(0))), convert(0))
-      checkEvaluation(Abs(Literal(convert(1))), convert(1))
-      checkEvaluation(Abs(Literal(convert(-1))), convert(1))
-      checkEvaluation(Abs(Literal.create(null, dataType)), null)
+    // SPARK-34742: when the input is not MinValue of integral types, the results of function ABS
+    //              should be the same with/without ANSI mode on.
+    Seq("true", "false").foreach { ansiEnabled =>
+      withSQLConf(SQLConf.ANSI_ENABLED.key -> ansiEnabled) {
+        testNumericDataTypes { convert =>
+          val input = Literal(convert(1))
+          val dataType = input.dataType
+          checkEvaluation(Abs(Literal(convert(0))), convert(0))
+          checkEvaluation(Abs(Literal(convert(1))), convert(1))
+          checkEvaluation(Abs(Literal(convert(-1))), convert(1))
+          checkEvaluation(Abs(Literal.create(null, dataType)), null)
+        }
+      }
     }
     checkEvaluation(Abs(positiveShortLit), positiveShort)
     checkEvaluation(Abs(negativeShortLit), (- negativeShort).toShort)
@@ -232,7 +273,26 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
     checkEvaluation(Abs(negativeLongLit), - negativeLong)
 
     DataTypeTestUtils.numericTypeWithoutDecimal.foreach { tpe =>
-      checkConsistencyBetweenInterpretedAndCodegen(Abs, tpe)
+      checkConsistencyBetweenInterpretedAndCodegen((e: Expression) => Abs(e, false), tpe)
+    }
+  }
+
+  test("SPARK-34742: Abs throws exception when input is out of range in ANSI mode") {
+    val minValues = Seq(
+      Literal(Byte.MinValue, ByteType),
+      Literal(Short.MinValue, ShortType),
+      Literal(Int.MinValue),
+      Literal(Long.MinValue)
+    )
+    withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
+      minValues.foreach { v =>
+        checkExceptionInExpression[ArithmeticException](Abs(v), "overflow")
+      }
+    }
+    withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
+      minValues.foreach { v =>
+        checkEvaluation(Abs(v), v.value)
+      }
     }
   }
 
@@ -253,12 +313,13 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
     checkEvaluation(Pmod(positiveInt, negativeInt), positiveInt)
     checkEvaluation(Pmod(positiveLong, negativeLong), positiveLong)
 
-    // mod by 0
-    checkEvaluation(Pmod(Literal(-7), Literal(0)), null)
-    checkEvaluation(Pmod(Literal(7.2D), Literal(0D)), null)
-    checkEvaluation(Pmod(Literal(7.2F), Literal(0F)), null)
-    checkEvaluation(Pmod(Literal(2.toByte), Literal(0.toByte)), null)
-    checkEvaluation(Pmod(positiveShort, 0.toShort), null)
+    Seq("true", "false").foreach { failOnError =>
+      withSQLConf(SQLConf.ANSI_ENABLED.key -> failOnError) {
+        DataTypeTestUtils.numericTypeWithoutDecimal.foreach { tpe =>
+          checkConsistencyBetweenInterpretedAndCodegenAllowingException(Pmod(_, _), tpe, tpe)
+        }
+      }
+    }
   }
 
   test("function least") {
@@ -413,18 +474,24 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
   test("SPARK-24598: overflow on long returns wrong result") {
     val maxLongLiteral = Literal(Long.MaxValue)
     val minLongLiteral = Literal(Long.MinValue)
-    val e1 = Add(maxLongLiteral, Literal(1L))
-    val e2 = Subtract(maxLongLiteral, Literal(-1L))
-    val e3 = Multiply(maxLongLiteral, Literal(2L))
-    val e4 = Add(minLongLiteral, minLongLiteral)
-    val e5 = Subtract(minLongLiteral, maxLongLiteral)
-    val e6 = Multiply(minLongLiteral, minLongLiteral)
     withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
+      val e1 = Add(maxLongLiteral, Literal(1L))
+      val e2 = Subtract(maxLongLiteral, Literal(-1L))
+      val e3 = Multiply(maxLongLiteral, Literal(2L))
+      val e4 = Add(minLongLiteral, minLongLiteral)
+      val e5 = Subtract(minLongLiteral, maxLongLiteral)
+      val e6 = Multiply(minLongLiteral, minLongLiteral)
       Seq(e1, e2, e3, e4, e5, e6).foreach { e =>
         checkExceptionInExpression[ArithmeticException](e, "overflow")
       }
     }
     withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
+      val e1 = Add(maxLongLiteral, Literal(1L))
+      val e2 = Subtract(maxLongLiteral, Literal(-1L))
+      val e3 = Multiply(maxLongLiteral, Literal(2L))
+      val e4 = Add(minLongLiteral, minLongLiteral)
+      val e5 = Subtract(minLongLiteral, maxLongLiteral)
+      val e6 = Multiply(minLongLiteral, minLongLiteral)
       checkEvaluation(e1, Long.MinValue)
       checkEvaluation(e2, Long.MinValue)
       checkEvaluation(e3, -2L)
@@ -437,18 +504,24 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
   test("SPARK-24598: overflow on integer returns wrong result") {
     val maxIntLiteral = Literal(Int.MaxValue)
     val minIntLiteral = Literal(Int.MinValue)
-    val e1 = Add(maxIntLiteral, Literal(1))
-    val e2 = Subtract(maxIntLiteral, Literal(-1))
-    val e3 = Multiply(maxIntLiteral, Literal(2))
-    val e4 = Add(minIntLiteral, minIntLiteral)
-    val e5 = Subtract(minIntLiteral, maxIntLiteral)
-    val e6 = Multiply(minIntLiteral, minIntLiteral)
     withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
+      val e1 = Add(maxIntLiteral, Literal(1))
+      val e2 = Subtract(maxIntLiteral, Literal(-1))
+      val e3 = Multiply(maxIntLiteral, Literal(2))
+      val e4 = Add(minIntLiteral, minIntLiteral)
+      val e5 = Subtract(minIntLiteral, maxIntLiteral)
+      val e6 = Multiply(minIntLiteral, minIntLiteral)
       Seq(e1, e2, e3, e4, e5, e6).foreach { e =>
         checkExceptionInExpression[ArithmeticException](e, "overflow")
       }
     }
     withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
+      val e1 = Add(maxIntLiteral, Literal(1))
+      val e2 = Subtract(maxIntLiteral, Literal(-1))
+      val e3 = Multiply(maxIntLiteral, Literal(2))
+      val e4 = Add(minIntLiteral, minIntLiteral)
+      val e5 = Subtract(minIntLiteral, maxIntLiteral)
+      val e6 = Multiply(minIntLiteral, minIntLiteral)
       checkEvaluation(e1, Int.MinValue)
       checkEvaluation(e2, Int.MinValue)
       checkEvaluation(e3, -2)
@@ -461,18 +534,24 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
   test("SPARK-24598: overflow on short returns wrong result") {
     val maxShortLiteral = Literal(Short.MaxValue)
     val minShortLiteral = Literal(Short.MinValue)
-    val e1 = Add(maxShortLiteral, Literal(1.toShort))
-    val e2 = Subtract(maxShortLiteral, Literal((-1).toShort))
-    val e3 = Multiply(maxShortLiteral, Literal(2.toShort))
-    val e4 = Add(minShortLiteral, minShortLiteral)
-    val e5 = Subtract(minShortLiteral, maxShortLiteral)
-    val e6 = Multiply(minShortLiteral, minShortLiteral)
     withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
+      val e1 = Add(maxShortLiteral, Literal(1.toShort))
+      val e2 = Subtract(maxShortLiteral, Literal((-1).toShort))
+      val e3 = Multiply(maxShortLiteral, Literal(2.toShort))
+      val e4 = Add(minShortLiteral, minShortLiteral)
+      val e5 = Subtract(minShortLiteral, maxShortLiteral)
+      val e6 = Multiply(minShortLiteral, minShortLiteral)
       Seq(e1, e2, e3, e4, e5, e6).foreach { e =>
         checkExceptionInExpression[ArithmeticException](e, "overflow")
       }
     }
     withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
+      val e1 = Add(maxShortLiteral, Literal(1.toShort))
+      val e2 = Subtract(maxShortLiteral, Literal((-1).toShort))
+      val e3 = Multiply(maxShortLiteral, Literal(2.toShort))
+      val e4 = Add(minShortLiteral, minShortLiteral)
+      val e5 = Subtract(minShortLiteral, maxShortLiteral)
+      val e6 = Multiply(minShortLiteral, minShortLiteral)
       checkEvaluation(e1, Short.MinValue)
       checkEvaluation(e2, Short.MinValue)
       checkEvaluation(e3, (-2).toShort)
@@ -485,18 +564,24 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
   test("SPARK-24598: overflow on byte returns wrong result") {
     val maxByteLiteral = Literal(Byte.MaxValue)
     val minByteLiteral = Literal(Byte.MinValue)
-    val e1 = Add(maxByteLiteral, Literal(1.toByte))
-    val e2 = Subtract(maxByteLiteral, Literal((-1).toByte))
-    val e3 = Multiply(maxByteLiteral, Literal(2.toByte))
-    val e4 = Add(minByteLiteral, minByteLiteral)
-    val e5 = Subtract(minByteLiteral, maxByteLiteral)
-    val e6 = Multiply(minByteLiteral, minByteLiteral)
     withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
+      val e1 = Add(maxByteLiteral, Literal(1.toByte))
+      val e2 = Subtract(maxByteLiteral, Literal((-1).toByte))
+      val e3 = Multiply(maxByteLiteral, Literal(2.toByte))
+      val e4 = Add(minByteLiteral, minByteLiteral)
+      val e5 = Subtract(minByteLiteral, maxByteLiteral)
+      val e6 = Multiply(minByteLiteral, minByteLiteral)
       Seq(e1, e2, e3, e4, e5, e6).foreach { e =>
         checkExceptionInExpression[ArithmeticException](e, "overflow")
       }
     }
     withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
+      val e1 = Add(maxByteLiteral, Literal(1.toByte))
+      val e2 = Subtract(maxByteLiteral, Literal((-1).toByte))
+      val e3 = Multiply(maxByteLiteral, Literal(2.toByte))
+      val e4 = Add(minByteLiteral, minByteLiteral)
+      val e5 = Subtract(minByteLiteral, maxByteLiteral)
+      val e6 = Multiply(minByteLiteral, minByteLiteral)
       checkEvaluation(e1, Byte.MinValue)
       checkEvaluation(e2, Byte.MinValue)
       checkEvaluation(e3, (-2).toByte)
@@ -505,4 +590,62 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
       checkEvaluation(e6, 0.toByte)
     }
   }
+
+  test("SPARK-33008: division by zero on divide-like operations returns incorrect result") {
+    withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
+      val operators: Seq[((Expression, Expression) => Expression, ((Int => Any) => Unit) => Unit)] =
+        Seq((Divide(_, _), testDecimalAndDoubleType),
+          (IntegralDivide(_, _), testDecimalAndLongType),
+          (Remainder(_, _), testNumericDataTypes),
+          (Pmod(_, _), testNumericDataTypes))
+      operators.foreach { case (operator, testTypesFn) =>
+        testTypesFn { convert =>
+          val one = Literal(convert(1))
+          val zero = Literal(convert(0))
+          checkEvaluation(operator(Literal.create(null, one.dataType), zero), null)
+          checkEvaluation(operator(one, Literal.create(null, zero.dataType)), null)
+          checkExceptionInExpression[ArithmeticException](operator(one, zero), "divide by zero")
+        }
+      }
+    }
+  }
+
+  test("SPARK-34677: exact add and subtract of day-time and year-month intervals") {
+    Seq(true, false).foreach { failOnError =>
+      checkExceptionInExpression[ArithmeticException](
+        UnaryMinus(
+          Literal.create(Period.ofMonths(Int.MinValue), YearMonthIntervalType),
+          failOnError),
+        "overflow")
+      checkExceptionInExpression[ArithmeticException](
+        Subtract(
+          Literal.create(Period.ofMonths(Int.MinValue), YearMonthIntervalType),
+          Literal.create(Period.ofMonths(10), YearMonthIntervalType),
+          failOnError
+        ),
+        "overflow")
+      checkExceptionInExpression[ArithmeticException](
+        Add(
+          Literal.create(Period.ofMonths(Int.MaxValue), YearMonthIntervalType),
+          Literal.create(Period.ofMonths(10), YearMonthIntervalType),
+          failOnError
+        ),
+        "overflow")
+
+      checkExceptionInExpression[ArithmeticException](
+        Subtract(
+          Literal.create(Duration.ofDays(-106751991), DayTimeIntervalType),
+          Literal.create(Duration.ofDays(10), DayTimeIntervalType),
+          failOnError
+        ),
+        "overflow")
+      checkExceptionInExpression[ArithmeticException](
+        Add(
+          Literal.create(Duration.ofDays(106751991), DayTimeIntervalType),
+          Literal.create(Duration.ofDays(10), DayTimeIntervalType),
+          failOnError
+        ),
+        "overflow")
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/BitwiseExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/BitwiseExpressionsSuite.scala
index 4188dade3fe65..718d8dd44321b 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/BitwiseExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/BitwiseExpressionsSuite.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.catalyst.expressions
 
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.types._
 
 
@@ -131,4 +132,45 @@ class BitwiseExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
       checkConsistencyBetweenInterpretedAndCodegen(BitwiseXor, dt, dt)
     }
   }
+
+  test("BitGet") {
+    val nullLongLiteral = Literal.create(null, LongType)
+    val nullIntLiteral = Literal.create(null, IntegerType)
+    checkEvaluation(BitwiseGet(nullLongLiteral, Literal(1)), null)
+    checkEvaluation(BitwiseGet(Literal(11L), nullIntLiteral), null)
+    checkEvaluation(BitwiseGet(nullLongLiteral, nullIntLiteral), null)
+    checkEvaluation(BitwiseGet(Literal(11L), Literal(3)), 1.toByte)
+    checkEvaluation(BitwiseGet(Literal(11L), Literal(2)), 0.toByte)
+    checkEvaluation(BitwiseGet(Literal(11L), Literal(1)), 1.toByte)
+    checkEvaluation(BitwiseGet(Literal(11L), Literal(0)), 1.toByte)
+    checkEvaluation(BitwiseGet(Literal(11L), Literal(63)), 0.toByte)
+
+    val row1 = create_row(11L, -1)
+    val row2 = create_row(11L, 64)
+    val row3 = create_row(11, 32)
+    val row4 = create_row(11.toShort, 16)
+    val row5 = create_row(11.toByte, 16)
+
+    val tl = 't.long.at(0)
+    val ti = 't.int.at(0)
+    val ts = 't.short.at(0)
+    val tb = 't.byte.at(0)
+    val p = 'p.int.at(1)
+
+    val expr = BitwiseGet(tl, p)
+    checkExceptionInExpression[IllegalArgumentException](
+      expr, row1, "Invalid bit position: -1 is less than zero")
+    checkExceptionInExpression[IllegalArgumentException](
+      expr, row2, "Invalid bit position: 64 exceeds the bit upper limit")
+    checkExceptionInExpression[IllegalArgumentException](
+      BitwiseGet(ti, p), row3, "Invalid bit position: 32 exceeds the bit upper limit")
+    checkExceptionInExpression[IllegalArgumentException](
+      BitwiseGet(ts, p), row4, "Invalid bit position: 16 exceeds the bit upper limit")
+    checkExceptionInExpression[IllegalArgumentException](
+      BitwiseGet(tb, p), row5, "Invalid bit position: 16 exceeds the bit upper limit")
+
+    DataTypeTestUtils.integralType.foreach { dt =>
+      checkConsistencyBetweenInterpretedAndCodegenAllowingException(BitwiseGet, dt, IntegerType)
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CanonicalizeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CanonicalizeSuite.scala
index a043b4cbed1f1..ac31a68b2b618 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CanonicalizeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CanonicalizeSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.expressions
 import java.util.TimeZone
 
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.plans.logical.Range
 import org.apache.spark.sql.types.{IntegerType, LongType, StructField, StructType}
@@ -95,4 +96,78 @@ class CanonicalizeSuite extends SparkFunSuite {
     val castWithTimeZoneId = Cast(literal, LongType, Some(TimeZone.getDefault.getID))
     assert(castWithTimeZoneId.semanticEquals(cast))
   }
+
+  test("SPARK-32927: Bitwise operations are commutative") {
+    Seq(BitwiseOr(_, _), BitwiseAnd(_, _), BitwiseXor(_, _)).foreach { f =>
+      val e1 = f('a, f('b, 'c))
+      val e2 = f(f('a, 'b), 'c)
+      val e3 = f('a, f('b, 'a))
+
+      assert(e1.canonicalized == e2.canonicalized)
+      assert(e1.canonicalized != e3.canonicalized)
+    }
+  }
+
+  test("SPARK-32927: Bitwise operations are commutative for non-deterministic expressions") {
+    Seq(BitwiseOr(_, _), BitwiseAnd(_, _), BitwiseXor(_, _)).foreach { f =>
+      val e1 = f('a, f(rand(42), 'c))
+      val e2 = f(f('a, rand(42)), 'c)
+      val e3 = f('a, f(rand(42), 'a))
+
+      assert(e1.canonicalized == e2.canonicalized)
+      assert(e1.canonicalized != e3.canonicalized)
+    }
+  }
+
+  test("SPARK-32927: Bitwise operations are commutative for literal expressions") {
+    Seq(BitwiseOr(_, _), BitwiseAnd(_, _), BitwiseXor(_, _)).foreach { f =>
+      val e1 = f('a, f(42, 'c))
+      val e2 = f(f('a, 42), 'c)
+      val e3 = f('a, f(42, 'a))
+
+      assert(e1.canonicalized == e2.canonicalized)
+      assert(e1.canonicalized != e3.canonicalized)
+    }
+  }
+
+  test("SPARK-32927: Bitwise operations are commutative in a complex case") {
+    Seq(BitwiseOr(_, _), BitwiseAnd(_, _), BitwiseXor(_, _)).foreach { f1 =>
+      Seq(BitwiseOr(_, _), BitwiseAnd(_, _), BitwiseXor(_, _)).foreach { f2 =>
+        val e1 = f2(f1('a, f1('b, 'c)), 'a)
+        val e2 = f2(f1(f1('a, 'b), 'c), 'a)
+        val e3 = f2(f1('a, f1('b, 'a)), 'a)
+
+        assert(e1.canonicalized == e2.canonicalized)
+        assert(e1.canonicalized != e3.canonicalized)
+      }
+    }
+  }
+
+  test("SPARK-33421: Support Greatest and Least in Expression Canonicalize") {
+    Seq(Least(_), Greatest(_)).foreach { f =>
+      // test deterministic expr
+      val expr1 = f(Seq(Literal(1), Literal(2), Literal(3)))
+      val expr2 = f(Seq(Literal(3), Literal(1), Literal(2)))
+      val expr3 = f(Seq(Literal(1), Literal(1), Literal(1)))
+      assert(expr1.canonicalized == expr2.canonicalized)
+      assert(expr1.canonicalized != expr3.canonicalized)
+      assert(expr2.canonicalized != expr3.canonicalized)
+
+      // test non-deterministic expr
+      val randExpr1 = f(Seq(Literal(1), rand(1)))
+      val randExpr2 = f(Seq(rand(1), Literal(1)))
+      val randExpr3 = f(Seq(Literal(1), rand(2)))
+      assert(randExpr1.canonicalized == randExpr2.canonicalized)
+      assert(randExpr1.canonicalized != randExpr3.canonicalized)
+      assert(randExpr2.canonicalized != randExpr3.canonicalized)
+
+      // test nested expr
+      val nestedExpr1 = f(Seq(Literal(1), f(Seq(Literal(2), Literal(3)))))
+      val nestedExpr2 = f(Seq(f(Seq(Literal(2), Literal(3))), Literal(1)))
+      val nestedExpr3 = f(Seq(f(Seq(Literal(1), Literal(1))), Literal(1)))
+      assert(nestedExpr1.canonicalized == nestedExpr2.canonicalized)
+      assert(nestedExpr1.canonicalized != nestedExpr3.canonicalized)
+      assert(nestedExpr2.canonicalized != nestedExpr3.canonicalized)
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
index e5bff7f7af007..cf7be470265f8 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
@@ -18,6 +18,8 @@
 package org.apache.spark.sql.catalyst.expressions
 
 import java.sql.{Date, Timestamp}
+import java.time.{DateTimeException, Duration, Period}
+import java.time.temporal.ChronoUnit
 import java.util.{Calendar, TimeZone}
 
 import scala.collection.parallel.immutable.ParVector
@@ -25,23 +27,22 @@ import scala.collection.parallel.immutable.ParVector
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.TypeCheckFailure
 import org.apache.spark.sql.catalyst.analysis.TypeCoercion.numericPrecedence
 import org.apache.spark.sql.catalyst.analysis.TypeCoercionSuite
 import org.apache.spark.sql.catalyst.expressions.aggregate.{CollectList, CollectSet}
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
-import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.catalyst.util.DateTimeUtils._
+import org.apache.spark.sql.catalyst.util.IntervalUtils.microsToDuration
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
 abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
 
-  // Whether it is required to set SQLConf.ANSI_ENABLED as true for testing numeric overflow.
-  protected def requiredAnsiEnabledForOverflowTestCases: Boolean
-
   protected def cast(v: Any, targetType: DataType, timeZoneId: Option[String] = None): CastBase
 
   // expected cannot be null
@@ -53,11 +54,11 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(cast(Literal.create(null, from), to, UTC_OPT), null)
   }
 
+  protected def isAlwaysNullable: Boolean = false
+
   test("null cast") {
     import DataTypeTestUtils._
 
-    // follow [[org.apache.spark.sql.catalyst.expressions.Cast.canCast]] logic
-    // to ensure we test every possible cast situation here
     atomicTypes.zip(atomicTypes).foreach { case (from, to) =>
       checkNullCast(from, to)
     }
@@ -66,14 +67,10 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
     atomicTypes.foreach(dt => checkNullCast(dt, StringType))
     checkNullCast(StringType, BinaryType)
     checkNullCast(StringType, BooleanType)
-    checkNullCast(DateType, BooleanType)
-    checkNullCast(TimestampType, BooleanType)
     numericTypes.foreach(dt => checkNullCast(dt, BooleanType))
 
     checkNullCast(StringType, TimestampType)
-    checkNullCast(BooleanType, TimestampType)
     checkNullCast(DateType, TimestampType)
-    numericTypes.foreach(dt => checkNullCast(dt, TimestampType))
 
     checkNullCast(StringType, DateType)
     checkNullCast(TimestampType, DateType)
@@ -81,8 +78,6 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
     checkNullCast(StringType, CalendarIntervalType)
     numericTypes.foreach(dt => checkNullCast(StringType, dt))
     numericTypes.foreach(dt => checkNullCast(BooleanType, dt))
-    numericTypes.foreach(dt => checkNullCast(DateType, dt))
-    numericTypes.foreach(dt => checkNullCast(TimestampType, dt))
     for (from <- numericTypes; to <- numericTypes) checkNullCast(from, to)
   }
 
@@ -103,12 +98,6 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(Cast(Literal("2015-03-18 123142"), DateType), new Date(c.getTimeInMillis))
     checkEvaluation(Cast(Literal("2015-03-18T123123"), DateType), new Date(c.getTimeInMillis))
     checkEvaluation(Cast(Literal("2015-03-18T"), DateType), new Date(c.getTimeInMillis))
-
-    checkEvaluation(Cast(Literal("2015-03-18X"), DateType), null)
-    checkEvaluation(Cast(Literal("2015/03/18"), DateType), null)
-    checkEvaluation(Cast(Literal("2015.03.18"), DateType), null)
-    checkEvaluation(Cast(Literal("20150318"), DateType), null)
-    checkEvaluation(Cast(Literal("2015-031-8"), DateType), null)
   }
 
   test("cast string to timestamp") {
@@ -117,8 +106,6 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
         checkEvaluation(cast(Literal(str), TimestampType, Option(zid.getId)), expected)
       }
 
-      checkCastStringToTimestamp("123", null)
-
       val tz = TimeZone.getTimeZone(zid)
       var c = Calendar.getInstance(tz)
       c.set(2015, 0, 1, 0, 0, 0)
@@ -195,15 +182,6 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
       c.set(2015, 2, 18, 12, 3, 17)
       c.set(Calendar.MILLISECOND, 123)
       checkCastStringToTimestamp("2015-03-18T12:03:17.123+7:3", new Timestamp(c.getTimeInMillis))
-
-      checkCastStringToTimestamp("2015-03-18 123142", null)
-      checkCastStringToTimestamp("2015-03-18T123123", null)
-      checkCastStringToTimestamp("2015-03-18X", null)
-      checkCastStringToTimestamp("2015/03/18", null)
-      checkCastStringToTimestamp("2015.03.18", null)
-      checkCastStringToTimestamp("20150318", null)
-      checkCastStringToTimestamp("2015-031-8", null)
-      checkCastStringToTimestamp("2015-03-18T12:03:17-0:70", null)
     }
   }
 
@@ -216,6 +194,39 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(cast(cast(0, BooleanType), IntegerType), 0)
   }
 
+  test("cast from int") {
+    checkCast(0, false)
+    checkCast(1, true)
+    checkCast(-5, true)
+    checkCast(1, 1.toByte)
+    checkCast(1, 1.toShort)
+    checkCast(1, 1)
+    checkCast(1, 1.toLong)
+    checkCast(1, 1.0f)
+    checkCast(1, 1.0)
+    checkCast(123, "123")
+
+    checkEvaluation(cast(123, DecimalType.USER_DEFAULT), Decimal(123))
+    checkEvaluation(cast(123, DecimalType(3, 0)), Decimal(123))
+    checkEvaluation(cast(1, LongType), 1.toLong)
+  }
+
+  test("cast from long") {
+    checkCast(0L, false)
+    checkCast(1L, true)
+    checkCast(-5L, true)
+    checkCast(1L, 1.toByte)
+    checkCast(1L, 1.toShort)
+    checkCast(1L, 1)
+    checkCast(1L, 1.toLong)
+    checkCast(1L, 1.0f)
+    checkCast(1L, 1.0)
+    checkCast(123L, "123")
+
+    checkEvaluation(cast(123L, DecimalType.USER_DEFAULT), Decimal(123))
+    checkEvaluation(cast(123L, DecimalType(3, 0)), Decimal(123))
+  }
+
   test("cast from float") {
     checkCast(0.0f, false)
     checkCast(0.5f, true)
@@ -238,14 +249,11 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
     checkCast(1.5, 1.toLong)
     checkCast(1.5, 1.5f)
     checkCast(1.5, "1.5")
-
-    checkEvaluation(cast(cast(1.toDouble, TimestampType), DoubleType), 1.toDouble)
-    checkEvaluation(cast(cast(1.toDouble, TimestampType), DoubleType), 1.toDouble)
   }
 
   test("cast from string") {
-    assert(cast("abcdef", StringType).nullable === false)
-    assert(cast("abcdef", BinaryType).nullable === false)
+    assert(cast("abcdef", StringType).nullable === isAlwaysNullable)
+    assert(cast("abcdef", BinaryType).nullable === isAlwaysNullable)
     assert(cast("abcdef", BooleanType).nullable)
     assert(cast("abcdef", TimestampType).nullable)
     assert(cast("abcdef", LongType).nullable)
@@ -268,13 +276,13 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
 
     for (tz <- ALL_TIMEZONES) {
       val timeZoneId = Option(tz.getId)
-      var c = Calendar.getInstance(TimeZoneGMT)
+      var c = Calendar.getInstance(TimeZoneUTC)
       c.set(2015, 2, 8, 2, 30, 0)
       checkEvaluation(
         cast(cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId),
           TimestampType, timeZoneId),
         millisToMicros(c.getTimeInMillis))
-      c = Calendar.getInstance(TimeZoneGMT)
+      c = Calendar.getInstance(TimeZoneUTC)
       c.set(2015, 10, 1, 2, 30, 0)
       checkEvaluation(
         cast(cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId),
@@ -283,7 +291,6 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
     }
 
     checkEvaluation(cast("abdef", StringType), "abdef")
-    checkEvaluation(cast("abdef", TimestampType, UTC_OPT), null)
     checkEvaluation(cast("12.65", DecimalType.SYSTEM_DEFAULT), Decimal(12.65))
 
     checkEvaluation(cast(cast(sd, DateType), StringType), sd)
@@ -291,7 +298,7 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(cast(cast(nts, TimestampType, UTC_OPT), StringType, UTC_OPT), nts)
     checkEvaluation(
       cast(cast(ts, StringType, UTC_OPT), TimestampType, UTC_OPT),
-      DateTimeUtils.fromJavaTimestamp(ts))
+      fromJavaTimestamp(ts))
 
     // all convert to string type to check
     checkEvaluation(
@@ -306,17 +313,6 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(cast(cast(cast(cast(
       cast(cast("5", ByteType), ShortType), IntegerType), FloatType), DoubleType), LongType),
       5.toLong)
-    checkEvaluation(
-      cast(cast(cast(cast(cast(cast("5", ByteType), TimestampType),
-        DecimalType.SYSTEM_DEFAULT), LongType), StringType), ShortType),
-      5.toShort)
-    checkEvaluation(
-      cast(cast(cast(cast(cast(cast("5", TimestampType, UTC_OPT), ByteType),
-        DecimalType.SYSTEM_DEFAULT), LongType), StringType), ShortType),
-      null)
-    checkEvaluation(cast(cast(cast(cast(cast(cast("5", DecimalType.SYSTEM_DEFAULT),
-      ByteType), TimestampType), LongType), StringType), ShortType),
-      5.toShort)
 
     checkEvaluation(cast("23", DoubleType), 23d)
     checkEvaluation(cast("23", IntegerType), 23)
@@ -351,57 +347,6 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
     checkCast(Decimal(1.5), "1.5")
   }
 
-  test("cast from date") {
-    val d = Date.valueOf("1970-01-01")
-    checkEvaluation(cast(d, ShortType), null)
-    checkEvaluation(cast(d, IntegerType), null)
-    checkEvaluation(cast(d, LongType), null)
-    checkEvaluation(cast(d, FloatType), null)
-    checkEvaluation(cast(d, DoubleType), null)
-    checkEvaluation(cast(d, DecimalType.SYSTEM_DEFAULT), null)
-    checkEvaluation(cast(d, DecimalType(10, 2)), null)
-    checkEvaluation(cast(d, StringType), "1970-01-01")
-
-    checkEvaluation(
-      cast(cast(d, TimestampType, UTC_OPT), StringType, UTC_OPT),
-      "1970-01-01 00:00:00")
-  }
-
-  test("cast from timestamp") {
-    val millis = 15 * 1000 + 3
-    val seconds = millis * 1000 + 3
-    val ts = new Timestamp(millis)
-    val tss = new Timestamp(seconds)
-    checkEvaluation(cast(ts, ShortType), 15.toShort)
-    checkEvaluation(cast(ts, IntegerType), 15)
-    checkEvaluation(cast(ts, LongType), 15.toLong)
-    checkEvaluation(cast(ts, FloatType), 15.003f)
-    checkEvaluation(cast(ts, DoubleType), 15.003)
-    checkEvaluation(cast(cast(tss, ShortType), TimestampType),
-      DateTimeUtils.fromJavaTimestamp(ts) * MILLIS_PER_SECOND)
-    checkEvaluation(cast(cast(tss, IntegerType), TimestampType),
-      DateTimeUtils.fromJavaTimestamp(ts) * MILLIS_PER_SECOND)
-    checkEvaluation(cast(cast(tss, LongType), TimestampType),
-      DateTimeUtils.fromJavaTimestamp(ts) * MILLIS_PER_SECOND)
-    checkEvaluation(
-      cast(cast(millis.toFloat / MILLIS_PER_SECOND, TimestampType), FloatType),
-      millis.toFloat / MILLIS_PER_SECOND)
-    checkEvaluation(
-      cast(cast(millis.toDouble / MILLIS_PER_SECOND, TimestampType), DoubleType),
-      millis.toDouble / MILLIS_PER_SECOND)
-    checkEvaluation(
-      cast(cast(Decimal(1), TimestampType), DecimalType.SYSTEM_DEFAULT),
-      Decimal(1))
-
-    // A test for higher precision than millis
-    checkEvaluation(cast(cast(0.000001, TimestampType), DoubleType), 0.000001)
-
-    checkEvaluation(cast(Double.NaN, TimestampType), null)
-    checkEvaluation(cast(1.0 / 0.0, TimestampType), null)
-    checkEvaluation(cast(Float.NaN, TimestampType), null)
-    checkEvaluation(cast(1.0f / 0.0f, TimestampType), null)
-  }
-
   test("cast from array") {
     val array = Literal.create(Seq("123", "true", "f", null),
       ArrayType(StringType, containsNull = true))
@@ -410,12 +355,6 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
 
     checkNullCast(ArrayType(StringType), ArrayType(IntegerType))
 
-    {
-      val ret = cast(array, ArrayType(BooleanType, containsNull = true))
-      assert(ret.resolved)
-      checkEvaluation(ret, Seq(null, true, false, null))
-    }
-
     {
       val array = Literal.create(Seq.empty, ArrayType(NullType, containsNull = false))
       val ret = cast(array, ArrayType(IntegerType, containsNull = false))
@@ -428,11 +367,6 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
       assert(ret.resolved === false)
     }
 
-    {
-      val ret = cast(array_notNull, ArrayType(BooleanType, containsNull = true))
-      assert(ret.resolved)
-      checkEvaluation(ret, Seq(null, true, false))
-    }
     {
       val ret = cast(array_notNull, ArrayType(BooleanType, containsNull = false))
       assert(ret.resolved === false)
@@ -454,11 +388,6 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
 
     checkNullCast(MapType(StringType, IntegerType), MapType(StringType, StringType))
 
-    {
-      val ret = cast(map, MapType(StringType, BooleanType, valueContainsNull = true))
-      assert(ret.resolved)
-      checkEvaluation(ret, Map("a" -> null, "b" -> true, "c" -> false, "d" -> null))
-    }
     {
       val ret = cast(map, MapType(StringType, BooleanType, valueContainsNull = false))
       assert(ret.resolved === false)
@@ -467,11 +396,6 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
       val ret = cast(map, MapType(IntegerType, StringType, valueContainsNull = true))
       assert(ret.resolved === false)
     }
-    {
-      val ret = cast(map_notNull, MapType(StringType, BooleanType, valueContainsNull = true))
-      assert(ret.resolved)
-      checkEvaluation(ret, Map("a" -> null, "b" -> true, "c" -> false))
-    }
     {
       val ret = cast(map_notNull, MapType(StringType, BooleanType, valueContainsNull = false))
       assert(ret.resolved === false)
@@ -517,15 +441,6 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
         StructField("b", StringType, nullable = false),
         StructField("c", StringType, nullable = false))))
 
-    {
-      val ret = cast(struct, StructType(Seq(
-        StructField("a", BooleanType, nullable = true),
-        StructField("b", BooleanType, nullable = true),
-        StructField("c", BooleanType, nullable = true),
-        StructField("d", BooleanType, nullable = true))))
-      assert(ret.resolved)
-      checkEvaluation(ret, InternalRow(null, true, false, null))
-    }
     {
       val ret = cast(struct, StructType(Seq(
         StructField("a", BooleanType, nullable = true),
@@ -535,14 +450,6 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
       assert(ret.resolved === false)
     }
 
-    {
-      val ret = cast(struct_notNull, StructType(Seq(
-        StructField("a", BooleanType, nullable = true),
-        StructField("b", BooleanType, nullable = true),
-        StructField("c", BooleanType, nullable = true))))
-      assert(ret.resolved)
-      checkEvaluation(ret, InternalRow(null, true, false))
-    }
     {
       val ret = cast(struct_notNull, StructType(Seq(
         StructField("a", BooleanType, nullable = true),
@@ -630,21 +537,22 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
     checkCast("n", false)
     checkCast("no", false)
     checkCast("0", false)
+  }
 
-    checkEvaluation(cast("abc", BooleanType), null)
-    checkEvaluation(cast("", BooleanType), null)
+  protected def checkInvalidCastFromNumericType(to: DataType): Unit = {
+    assert(cast(1.toByte, to).checkInputDataTypes().isFailure)
+    assert(cast(1.toShort, to).checkInputDataTypes().isFailure)
+    assert(cast(1, to).checkInputDataTypes().isFailure)
+    assert(cast(1L, to).checkInputDataTypes().isFailure)
+    assert(cast(1.0.toFloat, to).checkInputDataTypes().isFailure)
+    assert(cast(1.0, to).checkInputDataTypes().isFailure)
   }
 
   test("SPARK-16729 type checking for casting to date type") {
     assert(cast("1234", DateType).checkInputDataTypes().isSuccess)
     assert(cast(new Timestamp(1), DateType).checkInputDataTypes().isSuccess)
     assert(cast(false, DateType).checkInputDataTypes().isFailure)
-    assert(cast(1.toByte, DateType).checkInputDataTypes().isFailure)
-    assert(cast(1.toShort, DateType).checkInputDataTypes().isFailure)
-    assert(cast(1, DateType).checkInputDataTypes().isFailure)
-    assert(cast(1L, DateType).checkInputDataTypes().isFailure)
-    assert(cast(1.0.toFloat, DateType).checkInputDataTypes().isFailure)
-    assert(cast(1.0, DateType).checkInputDataTypes().isFailure)
+    checkInvalidCastFromNumericType(DateType)
   }
 
   test("SPARK-20302 cast with same structure") {
@@ -686,75 +594,6 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
     assert(ctx.inlinedMutableStates.length == 0)
   }
 
-  test("SPARK-22825 Cast array to string") {
-    val ret1 = cast(Literal.create(Array(1, 2, 3, 4, 5)), StringType)
-    checkEvaluation(ret1, "[1, 2, 3, 4, 5]")
-    val ret2 = cast(Literal.create(Array("ab", "cde", "f")), StringType)
-    checkEvaluation(ret2, "[ab, cde, f]")
-    val ret3 = cast(Literal.create(Array("ab", null, "c")), StringType)
-    checkEvaluation(ret3, "[ab,, c]")
-    val ret4 = cast(Literal.create(Array("ab".getBytes, "cde".getBytes, "f".getBytes)), StringType)
-    checkEvaluation(ret4, "[ab, cde, f]")
-    val ret5 = cast(
-      Literal.create(Array("2014-12-03", "2014-12-04", "2014-12-06").map(Date.valueOf)),
-      StringType)
-    checkEvaluation(ret5, "[2014-12-03, 2014-12-04, 2014-12-06]")
-    val ret6 = cast(
-      Literal.create(Array("2014-12-03 13:01:00", "2014-12-04 15:05:00").map(Timestamp.valueOf)),
-      StringType)
-    checkEvaluation(ret6, "[2014-12-03 13:01:00, 2014-12-04 15:05:00]")
-    val ret7 = cast(Literal.create(Array(Array(1, 2, 3), Array(4, 5))), StringType)
-    checkEvaluation(ret7, "[[1, 2, 3], [4, 5]]")
-    val ret8 = cast(
-      Literal.create(Array(Array(Array("a"), Array("b", "c")), Array(Array("d")))),
-      StringType)
-    checkEvaluation(ret8, "[[[a], [b, c]], [[d]]]")
-  }
-
-  test("SPARK-22973 Cast map to string") {
-    val ret1 = cast(Literal.create(Map(1 -> "a", 2 -> "b", 3 -> "c")), StringType)
-    checkEvaluation(ret1, "[1 -> a, 2 -> b, 3 -> c]")
-    val ret2 = cast(
-      Literal.create(Map("1" -> "a".getBytes, "2" -> null, "3" -> "c".getBytes)),
-      StringType)
-    checkEvaluation(ret2, "[1 -> a, 2 ->, 3 -> c]")
-    val ret3 = cast(
-      Literal.create(Map(
-        1 -> Date.valueOf("2014-12-03"),
-        2 -> Date.valueOf("2014-12-04"),
-        3 -> Date.valueOf("2014-12-05"))),
-      StringType)
-    checkEvaluation(ret3, "[1 -> 2014-12-03, 2 -> 2014-12-04, 3 -> 2014-12-05]")
-    val ret4 = cast(
-      Literal.create(Map(
-        1 -> Timestamp.valueOf("2014-12-03 13:01:00"),
-        2 -> Timestamp.valueOf("2014-12-04 15:05:00"))),
-      StringType)
-    checkEvaluation(ret4, "[1 -> 2014-12-03 13:01:00, 2 -> 2014-12-04 15:05:00]")
-    val ret5 = cast(
-      Literal.create(Map(
-        1 -> Array(1, 2, 3),
-        2 -> Array(4, 5, 6))),
-      StringType)
-    checkEvaluation(ret5, "[1 -> [1, 2, 3], 2 -> [4, 5, 6]]")
-  }
-
-  test("SPARK-22981 Cast struct to string") {
-    val ret1 = cast(Literal.create((1, "a", 0.1)), StringType)
-    checkEvaluation(ret1, "[1, a, 0.1]")
-    val ret2 = cast(Literal.create(Tuple3[Int, String, String](1, null, "a")), StringType)
-    checkEvaluation(ret2, "[1,, a]")
-    val ret3 = cast(Literal.create(
-      (Date.valueOf("2014-12-03"), Timestamp.valueOf("2014-12-03 15:05:00"))), StringType)
-    checkEvaluation(ret3, "[2014-12-03, 2014-12-03 15:05:00]")
-    val ret4 = cast(Literal.create(((1, "a"), 5, 0.1)), StringType)
-    checkEvaluation(ret4, "[[1, a], 5, 0.1]")
-    val ret5 = cast(Literal.create((Seq(1, 2, 3), "a", 0.1)), StringType)
-    checkEvaluation(ret5, "[[1, 2, 3], a, 0.1]")
-    val ret6 = cast(Literal.create((1, Map(1 -> "a", 2 -> "b", 3 -> "c"))), StringType)
-    checkEvaluation(ret6, "[1, [1 -> a, 2 -> b, 3 -> c]]")
-  }
-
   test("up-cast") {
     def isCastSafe(from: NumericType, to: NumericType): Boolean = (from, to) match {
       case (_, dt: DecimalType) => dt.isWiderThan(from)
@@ -827,20 +666,6 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
     }
   }
 
-  test("Throw exception on casting out-of-range value to decimal type") {
-    withSQLConf(SQLConf.ANSI_ENABLED.key -> requiredAnsiEnabledForOverflowTestCases.toString) {
-      checkExceptionInExpression[ArithmeticException](
-        cast(Literal("134.12"), DecimalType(3, 2)), "cannot be represented")
-      checkExceptionInExpression[ArithmeticException](
-        cast(Literal(Timestamp.valueOf("2019-07-25 22:04:36")), DecimalType(3, 2)),
-        "cannot be represented")
-      checkExceptionInExpression[ArithmeticException](
-        cast(Literal(BigDecimal(134.12)), DecimalType(3, 2)), "cannot be represented")
-      checkExceptionInExpression[ArithmeticException](
-        cast(Literal(134.12), DecimalType(3, 2)), "cannot be represented")
-    }
-  }
-
   test("Process Infinity, -Infinity, NaN in case insensitive manner") {
     Seq("inf", "+inf", "infinity", "+infiNity", " infinity ").foreach { value =>
       checkEvaluation(cast(value, FloatType), Float.PositiveInfinity)
@@ -862,13 +687,171 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
     }
   }
 
+  test("SPARK-22825 Cast array to string") {
+    val ret1 = cast(Literal.create(Array(1, 2, 3, 4, 5)), StringType)
+    checkEvaluation(ret1, "[1, 2, 3, 4, 5]")
+    val ret2 = cast(Literal.create(Array("ab", "cde", "f")), StringType)
+    checkEvaluation(ret2, "[ab, cde, f]")
+    Seq(false, true).foreach { omitNull =>
+      withSQLConf(SQLConf.LEGACY_COMPLEX_TYPES_TO_STRING.key -> omitNull.toString) {
+        val ret3 = cast(Literal.create(Array("ab", null, "c")), StringType)
+        checkEvaluation(ret3, s"[ab,${if (omitNull) "" else " null"}, c]")
+      }
+    }
+    val ret4 =
+      cast(Literal.create(Array("ab".getBytes, "cde".getBytes, "f".getBytes)), StringType)
+    checkEvaluation(ret4, "[ab, cde, f]")
+    val ret5 = cast(
+      Literal.create(Array("2014-12-03", "2014-12-04", "2014-12-06").map(Date.valueOf)),
+      StringType)
+    checkEvaluation(ret5, "[2014-12-03, 2014-12-04, 2014-12-06]")
+    val ret6 = cast(
+      Literal.create(Array("2014-12-03 13:01:00", "2014-12-04 15:05:00")
+        .map(Timestamp.valueOf)),
+      StringType)
+    checkEvaluation(ret6, "[2014-12-03 13:01:00, 2014-12-04 15:05:00]")
+    val ret7 = cast(Literal.create(Array(Array(1, 2, 3), Array(4, 5))), StringType)
+    checkEvaluation(ret7, "[[1, 2, 3], [4, 5]]")
+    val ret8 = cast(
+      Literal.create(Array(Array(Array("a"), Array("b", "c")), Array(Array("d")))),
+      StringType)
+    checkEvaluation(ret8, "[[[a], [b, c]], [[d]]]")
+  }
+
+  test("SPARK-33291: Cast array with null elements to string") {
+    Seq(false, true).foreach { omitNull =>
+      withSQLConf(SQLConf.LEGACY_COMPLEX_TYPES_TO_STRING.key -> omitNull.toString) {
+        val ret1 = cast(Literal.create(Array(null, null)), StringType)
+        checkEvaluation(
+          ret1,
+          s"[${if (omitNull) "" else "null"},${if (omitNull) "" else " null"}]")
+      }
+    }
+  }
+
+  test("SPARK-22973 Cast map to string") {
+    Seq(
+      false -> ("{", "}"),
+      true -> ("[", "]")).foreach { case (legacyCast, (lb, rb)) =>
+      withSQLConf(SQLConf.LEGACY_COMPLEX_TYPES_TO_STRING.key -> legacyCast.toString) {
+        val ret1 = cast(Literal.create(Map(1 -> "a", 2 -> "b", 3 -> "c")), StringType)
+        checkEvaluation(ret1, s"${lb}1 -> a, 2 -> b, 3 -> c$rb")
+        val ret2 = cast(
+          Literal.create(Map("1" -> "a".getBytes, "2" -> null, "3" -> "c".getBytes)),
+          StringType)
+        checkEvaluation(ret2, s"${lb}1 -> a, 2 ->${if (legacyCast) "" else " null"}, 3 -> c$rb")
+        val ret3 = cast(
+          Literal.create(Map(
+            1 -> Date.valueOf("2014-12-03"),
+            2 -> Date.valueOf("2014-12-04"),
+            3 -> Date.valueOf("2014-12-05"))),
+          StringType)
+        checkEvaluation(ret3, s"${lb}1 -> 2014-12-03, 2 -> 2014-12-04, 3 -> 2014-12-05$rb")
+        val ret4 = cast(
+          Literal.create(Map(
+            1 -> Timestamp.valueOf("2014-12-03 13:01:00"),
+            2 -> Timestamp.valueOf("2014-12-04 15:05:00"))),
+          StringType)
+        checkEvaluation(ret4, s"${lb}1 -> 2014-12-03 13:01:00, 2 -> 2014-12-04 15:05:00$rb")
+        val ret5 = cast(
+          Literal.create(Map(
+            1 -> Array(1, 2, 3),
+            2 -> Array(4, 5, 6))),
+          StringType)
+        checkEvaluation(ret5, s"${lb}1 -> [1, 2, 3], 2 -> [4, 5, 6]$rb")
+      }
+    }
+  }
+
+  test("SPARK-22981 Cast struct to string") {
+    Seq(
+      false -> ("{", "}"),
+      true -> ("[", "]")).foreach { case (legacyCast, (lb, rb)) =>
+      withSQLConf(SQLConf.LEGACY_COMPLEX_TYPES_TO_STRING.key -> legacyCast.toString) {
+        val ret1 = cast(Literal.create((1, "a", 0.1)), StringType)
+        checkEvaluation(ret1, s"${lb}1, a, 0.1$rb")
+        val ret2 = cast(Literal.create(Tuple3[Int, String, String](1, null, "a")), StringType)
+        checkEvaluation(ret2, s"${lb}1,${if (legacyCast) "" else " null"}, a$rb")
+        val ret3 = cast(Literal.create(
+          (Date.valueOf("2014-12-03"), Timestamp.valueOf("2014-12-03 15:05:00"))), StringType)
+        checkEvaluation(ret3, s"${lb}2014-12-03, 2014-12-03 15:05:00$rb")
+        val ret4 = cast(Literal.create(((1, "a"), 5, 0.1)), StringType)
+        checkEvaluation(ret4, s"$lb${lb}1, a$rb, 5, 0.1$rb")
+        val ret5 = cast(Literal.create((Seq(1, 2, 3), "a", 0.1)), StringType)
+        checkEvaluation(ret5, s"$lb[1, 2, 3], a, 0.1$rb")
+        val ret6 = cast(Literal.create((1, Map(1 -> "a", 2 -> "b", 3 -> "c"))), StringType)
+        checkEvaluation(ret6, s"${lb}1, ${lb}1 -> a, 2 -> b, 3 -> c$rb$rb")
+      }
+    }
+  }
+
+  test("SPARK-33291: Cast struct with null elements to string") {
+    Seq(
+      false -> ("{", "}"),
+      true -> ("[", "]")).foreach { case (legacyCast, (lb, rb)) =>
+      withSQLConf(SQLConf.LEGACY_COMPLEX_TYPES_TO_STRING.key -> legacyCast.toString) {
+        val ret1 = cast(Literal.create(Tuple2[String, String](null, null)), StringType)
+        checkEvaluation(
+          ret1,
+          s"$lb${if (legacyCast) "" else "null"},${if (legacyCast) "" else " null"}$rb")
+      }
+    }
+  }
+
+  test("SPARK-34667: cast year-month interval to string") {
+    Seq(
+      Period.ofMonths(0) -> "0-0",
+      Period.ofMonths(1) -> "0-1",
+      Period.ofMonths(-1) -> "-0-1",
+      Period.ofYears(1) -> "1-0",
+      Period.ofYears(-1) -> "-1-0",
+      Period.ofYears(10).plusMonths(10) -> "10-10",
+      Period.ofYears(-123).minusMonths(6) -> "-123-6",
+      Period.ofMonths(Int.MaxValue) -> "178956970-7",
+      Period.ofMonths(Int.MinValue) -> "-178956970-8"
+    ).foreach { case (period, intervalPayload) =>
+      checkEvaluation(
+        Cast(Literal(period), StringType),
+        s"INTERVAL '$intervalPayload' YEAR TO MONTH")
+    }
+
+    checkConsistencyBetweenInterpretedAndCodegen(
+      (child: Expression) => Cast(child, StringType), YearMonthIntervalType)
+  }
+
+  test("SPARK-34668: cast day-time interval to string") {
+    Seq(
+      Duration.ZERO -> "0 00:00:00",
+      Duration.of(1, ChronoUnit.MICROS) -> "0 00:00:00.000001",
+      Duration.ofMillis(-1) -> "-0 00:00:00.001",
+      Duration.ofMillis(1234) -> "0 00:00:01.234",
+      Duration.ofSeconds(-9).minus(999999, ChronoUnit.MICROS) -> "-0 00:00:09.999999",
+      Duration.ofMinutes(30).plusMillis(59010) -> "0 00:30:59.01",
+      Duration.ofHours(-23).minusSeconds(59) -> "-0 23:00:59",
+      Duration.ofDays(1).plus(12345678, ChronoUnit.MICROS) -> "1 00:00:12.345678",
+      Duration.ofDays(-1234).minusHours(23).minusMinutes(59).minusSeconds(59).minusMillis(999) ->
+        "-1234 23:59:59.999",
+      microsToDuration(Long.MaxValue) -> "106751991 04:00:54.775807",
+      microsToDuration(Long.MinValue + 1) -> "-106751991 04:00:54.775807",
+      microsToDuration(Long.MinValue) -> "-106751991 04:00:54.775808"
+    ).foreach { case (period, intervalPayload) =>
+      checkEvaluation(
+        Cast(Literal(period), StringType),
+        s"INTERVAL '$intervalPayload' DAY TO SECOND")
+    }
+
+    checkConsistencyBetweenInterpretedAndCodegen(
+      (child: Expression) => Cast(child, StringType), DayTimeIntervalType)
+  }
+}
+
+abstract class AnsiCastSuiteBase extends CastSuiteBase {
+
   private def testIntMaxAndMin(dt: DataType): Unit = {
     assert(Seq(IntegerType, ShortType, ByteType).contains(dt))
     Seq(Int.MaxValue + 1L, Int.MinValue - 1L).foreach { value =>
       checkExceptionInExpression[ArithmeticException](cast(value, dt), "overflow")
       checkExceptionInExpression[ArithmeticException](cast(Decimal(value.toString), dt), "overflow")
-      checkExceptionInExpression[ArithmeticException](
-        cast(Literal(value * MICROS_PER_SECOND, TimestampType), dt), "overflow")
       checkExceptionInExpression[ArithmeticException](
         cast(Literal(value * 1.5f, FloatType), dt), "overflow")
       checkExceptionInExpression[ArithmeticException](
@@ -888,98 +871,380 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
     }
   }
 
-  test("Throw exception on casting out-of-range value to byte type") {
-    withSQLConf(SQLConf.ANSI_ENABLED.key -> requiredAnsiEnabledForOverflowTestCases.toString) {
-      testIntMaxAndMin(ByteType)
-      Seq(Byte.MaxValue + 1, Byte.MinValue - 1).foreach { value =>
-        checkExceptionInExpression[ArithmeticException](cast(value, ByteType), "overflow")
-        checkExceptionInExpression[ArithmeticException](
-          cast(Literal(value * MICROS_PER_SECOND, TimestampType), ByteType), "overflow")
-        checkExceptionInExpression[ArithmeticException](
-          cast(Literal(value.toFloat, FloatType), ByteType), "overflow")
-        checkExceptionInExpression[ArithmeticException](
-          cast(Literal(value.toDouble, DoubleType), ByteType), "overflow")
-      }
+  test("ANSI mode: Throw exception on casting out-of-range value to byte type") {
+    testIntMaxAndMin(ByteType)
+    Seq(Byte.MaxValue + 1, Byte.MinValue - 1).foreach { value =>
+      checkExceptionInExpression[ArithmeticException](cast(value, ByteType), "overflow")
+      checkExceptionInExpression[ArithmeticException](
+        cast(Literal(value.toFloat, FloatType), ByteType), "overflow")
+      checkExceptionInExpression[ArithmeticException](
+        cast(Literal(value.toDouble, DoubleType), ByteType), "overflow")
+    }
 
-      Seq(Byte.MaxValue, 0.toByte, Byte.MinValue).foreach { value =>
-        checkEvaluation(cast(value, ByteType), value)
-        checkEvaluation(cast(value.toString, ByteType), value)
-        checkEvaluation(cast(Decimal(value.toString), ByteType), value)
-        checkEvaluation(cast(Literal(value * MICROS_PER_SECOND, TimestampType), ByteType), value)
-        checkEvaluation(cast(Literal(value.toInt, DateType), ByteType), null)
-        checkEvaluation(cast(Literal(value.toFloat, FloatType), ByteType), value)
-        checkEvaluation(cast(Literal(value.toDouble, DoubleType), ByteType), value)
-      }
+    Seq(Byte.MaxValue, 0.toByte, Byte.MinValue).foreach { value =>
+      checkEvaluation(cast(value, ByteType), value)
+      checkEvaluation(cast(value.toString, ByteType), value)
+      checkEvaluation(cast(Decimal(value.toString), ByteType), value)
+      checkEvaluation(cast(Literal(value.toFloat, FloatType), ByteType), value)
+      checkEvaluation(cast(Literal(value.toDouble, DoubleType), ByteType), value)
     }
   }
 
-  test("Throw exception on casting out-of-range value to short type") {
-    withSQLConf(SQLConf.ANSI_ENABLED.key -> requiredAnsiEnabledForOverflowTestCases.toString) {
-      testIntMaxAndMin(ShortType)
-      Seq(Short.MaxValue + 1, Short.MinValue - 1).foreach { value =>
-        checkExceptionInExpression[ArithmeticException](cast(value, ShortType), "overflow")
-        checkExceptionInExpression[ArithmeticException](
-          cast(Literal(value * MICROS_PER_SECOND, TimestampType), ShortType), "overflow")
-        checkExceptionInExpression[ArithmeticException](
-          cast(Literal(value.toFloat, FloatType), ShortType), "overflow")
-        checkExceptionInExpression[ArithmeticException](
-          cast(Literal(value.toDouble, DoubleType), ShortType), "overflow")
-      }
+  test("ANSI mode: Throw exception on casting out-of-range value to short type") {
+    testIntMaxAndMin(ShortType)
+    Seq(Short.MaxValue + 1, Short.MinValue - 1).foreach { value =>
+      checkExceptionInExpression[ArithmeticException](cast(value, ShortType), "overflow")
+      checkExceptionInExpression[ArithmeticException](
+        cast(Literal(value.toFloat, FloatType), ShortType), "overflow")
+      checkExceptionInExpression[ArithmeticException](
+        cast(Literal(value.toDouble, DoubleType), ShortType), "overflow")
+    }
 
-      Seq(Short.MaxValue, 0.toShort, Short.MinValue).foreach { value =>
-        checkEvaluation(cast(value, ShortType), value)
-        checkEvaluation(cast(value.toString, ShortType), value)
-        checkEvaluation(cast(Decimal(value.toString), ShortType), value)
-        checkEvaluation(cast(Literal(value * MICROS_PER_SECOND, TimestampType), ShortType), value)
-        checkEvaluation(cast(Literal(value.toInt, DateType), ShortType), null)
-        checkEvaluation(cast(Literal(value.toFloat, FloatType), ShortType), value)
-        checkEvaluation(cast(Literal(value.toDouble, DoubleType), ShortType), value)
-      }
+    Seq(Short.MaxValue, 0.toShort, Short.MinValue).foreach { value =>
+      checkEvaluation(cast(value, ShortType), value)
+      checkEvaluation(cast(value.toString, ShortType), value)
+      checkEvaluation(cast(Decimal(value.toString), ShortType), value)
+      checkEvaluation(cast(Literal(value.toFloat, FloatType), ShortType), value)
+      checkEvaluation(cast(Literal(value.toDouble, DoubleType), ShortType), value)
     }
   }
 
-  test("Throw exception on casting out-of-range value to int type") {
-    withSQLConf(SQLConf.ANSI_ENABLED.key -> requiredAnsiEnabledForOverflowTestCases.toString) {
-      testIntMaxAndMin(IntegerType)
-      testLongMaxAndMin(IntegerType)
+  test("ANSI mode: Throw exception on casting out-of-range value to int type") {
+    testIntMaxAndMin(IntegerType)
+    testLongMaxAndMin(IntegerType)
 
-      Seq(Int.MaxValue, 0, Int.MinValue).foreach { value =>
-        checkEvaluation(cast(value, IntegerType), value)
-        checkEvaluation(cast(value.toString, IntegerType), value)
-        checkEvaluation(cast(Decimal(value.toString), IntegerType), value)
-        checkEvaluation(cast(Literal(value * MICROS_PER_SECOND, TimestampType), IntegerType), value)
-        checkEvaluation(cast(Literal(value * 1.0, DoubleType), IntegerType), value)
-      }
-      checkEvaluation(cast(Int.MaxValue + 0.9D, IntegerType), Int.MaxValue)
-      checkEvaluation(cast(Int.MinValue - 0.9D, IntegerType), Int.MinValue)
+    Seq(Int.MaxValue, 0, Int.MinValue).foreach { value =>
+      checkEvaluation(cast(value, IntegerType), value)
+      checkEvaluation(cast(value.toString, IntegerType), value)
+      checkEvaluation(cast(Decimal(value.toString), IntegerType), value)
+      checkEvaluation(cast(Literal(value * 1.0, DoubleType), IntegerType), value)
     }
+    checkEvaluation(cast(Int.MaxValue + 0.9D, IntegerType), Int.MaxValue)
+    checkEvaluation(cast(Int.MinValue - 0.9D, IntegerType), Int.MinValue)
   }
 
-  test("Throw exception on casting out-of-range value to long type") {
-    withSQLConf(SQLConf.ANSI_ENABLED.key -> requiredAnsiEnabledForOverflowTestCases.toString) {
-      testLongMaxAndMin(LongType)
+  test("ANSI mode: Throw exception on casting out-of-range value to long type") {
+    testLongMaxAndMin(LongType)
 
-      Seq(Long.MaxValue, 0, Long.MinValue).foreach { value =>
-        checkEvaluation(cast(value, LongType), value)
-        checkEvaluation(cast(value.toString, LongType), value)
-        checkEvaluation(cast(Decimal(value.toString), LongType), value)
-        checkEvaluation(cast(Literal(value, TimestampType), LongType),
-          Math.floorDiv(value, MICROS_PER_SECOND))
-      }
-      checkEvaluation(cast(Long.MaxValue + 0.9F, LongType), Long.MaxValue)
-      checkEvaluation(cast(Long.MinValue - 0.9F, LongType), Long.MinValue)
-      checkEvaluation(cast(Long.MaxValue + 0.9D, LongType), Long.MaxValue)
-      checkEvaluation(cast(Long.MinValue - 0.9D, LongType), Long.MinValue)
+    Seq(Long.MaxValue, 0, Long.MinValue).foreach { value =>
+      checkEvaluation(cast(value, LongType), value)
+      checkEvaluation(cast(value.toString, LongType), value)
+      checkEvaluation(cast(Decimal(value.toString), LongType), value)
     }
+    checkEvaluation(cast(Long.MaxValue + 0.9F, LongType), Long.MaxValue)
+    checkEvaluation(cast(Long.MinValue - 0.9F, LongType), Long.MinValue)
+    checkEvaluation(cast(Long.MaxValue + 0.9D, LongType), Long.MaxValue)
+    checkEvaluation(cast(Long.MinValue - 0.9D, LongType), Long.MinValue)
   }
-}
 
-/**
- * Test suite for data type casting expression [[Cast]].
- */
-class CastSuite extends CastSuiteBase {
-  // It is required to set SQLConf.ANSI_ENABLED as true for testing numeric overflow.
-  override protected def requiredAnsiEnabledForOverflowTestCases: Boolean = true
+  test("ANSI mode: Throw exception on casting out-of-range value to decimal type") {
+    checkExceptionInExpression[ArithmeticException](
+      cast(Literal("134.12"), DecimalType(3, 2)), "cannot be represented")
+    checkExceptionInExpression[ArithmeticException](
+      cast(Literal(BigDecimal(134.12)), DecimalType(3, 2)), "cannot be represented")
+    checkExceptionInExpression[ArithmeticException](
+      cast(Literal(134.12), DecimalType(3, 2)), "cannot be represented")
+  }
+
+  protected def setConfigurationHint: String
+
+  private def verifyCastFailure(c: CastBase, optionalExpectedMsg: Option[String] = None): Unit = {
+    val typeCheckResult = c.checkInputDataTypes()
+    assert(typeCheckResult.isFailure)
+    assert(typeCheckResult.isInstanceOf[TypeCheckFailure])
+    val message = typeCheckResult.asInstanceOf[TypeCheckFailure].message
+
+    if (optionalExpectedMsg.isDefined) {
+      assert(message.contains(optionalExpectedMsg.get))
+    } else if (setConfigurationHint.nonEmpty) {
+      assert(message.contains("with ANSI mode on"))
+      assert(message.contains(setConfigurationHint))
+    } else {
+      assert("cannot cast [a-zA-Z]+ to [a-zA-Z]+".r.findFirstIn(message).isDefined)
+    }
+  }
+
+  test("ANSI mode: disallow type conversions between Numeric types and Timestamp type") {
+    import DataTypeTestUtils.numericTypes
+    checkInvalidCastFromNumericType(TimestampType)
+    var errorMsg =
+      "you can use functions TIMESTAMP_SECONDS/TIMESTAMP_MILLIS/TIMESTAMP_MICROS instead"
+    verifyCastFailure(cast(Literal(0L), TimestampType), Some(errorMsg))
+
+    val timestampLiteral = Literal(1L, TimestampType)
+    errorMsg = "you can use functions UNIX_SECONDS/UNIX_MILLIS/UNIX_MICROS instead."
+    numericTypes.foreach { numericType =>
+      verifyCastFailure(cast(timestampLiteral, numericType), Some(errorMsg))
+    }
+  }
+
+  test("ANSI mode: disallow type conversions between Numeric types and Date type") {
+    import DataTypeTestUtils.numericTypes
+    checkInvalidCastFromNumericType(DateType)
+    var errorMsg = "you can use function DATE_FROM_UNIX_DATE instead"
+    verifyCastFailure(cast(Literal(0L), DateType), Some(errorMsg))
+    val dateLiteral = Literal(1, DateType)
+    errorMsg = "you can use function UNIX_DATE instead"
+    numericTypes.foreach { numericType =>
+      verifyCastFailure(cast(dateLiteral, numericType), Some(errorMsg))
+    }
+  }
+
+  test("ANSI mode: disallow type conversions between Numeric types and Binary type") {
+    import DataTypeTestUtils.numericTypes
+    checkInvalidCastFromNumericType(BinaryType)
+    val binaryLiteral = Literal(new Array[Byte](1.toByte), BinaryType)
+    numericTypes.foreach { numericType =>
+      assert(cast(binaryLiteral, numericType).checkInputDataTypes().isFailure)
+    }
+  }
+
+  test("ANSI mode: disallow type conversions between Datatime types and Boolean types") {
+    val timestampLiteral = Literal(1L, TimestampType)
+    assert(cast(timestampLiteral, BooleanType).checkInputDataTypes().isFailure)
+    val dateLiteral = Literal(1, DateType)
+    assert(cast(dateLiteral, BooleanType).checkInputDataTypes().isFailure)
+
+    val booleanLiteral = Literal(true, BooleanType)
+    assert(cast(booleanLiteral, TimestampType).checkInputDataTypes().isFailure)
+    assert(cast(booleanLiteral, DateType).checkInputDataTypes().isFailure)
+  }
+
+  test("cast from invalid string to numeric should throw NumberFormatException") {
+    // cast to IntegerType
+    Seq(IntegerType, ShortType, ByteType, LongType).foreach { dataType =>
+      checkExceptionInExpression[NumberFormatException](
+        cast("string", dataType), "invalid input syntax for type numeric: string")
+      checkExceptionInExpression[NumberFormatException](
+        cast("123-string", dataType), "invalid input syntax for type numeric: 123-string")
+      checkExceptionInExpression[NumberFormatException](
+        cast("2020-07-19", dataType), "invalid input syntax for type numeric: 2020-07-19")
+      checkExceptionInExpression[NumberFormatException](
+        cast("1.23", dataType), "invalid input syntax for type numeric: 1.23")
+    }
+
+    Seq(DoubleType, FloatType, DecimalType.USER_DEFAULT).foreach { dataType =>
+      checkExceptionInExpression[NumberFormatException](
+        cast("string", dataType), "invalid input syntax for type numeric: string")
+      checkExceptionInExpression[NumberFormatException](
+        cast("123.000.00", dataType), "invalid input syntax for type numeric: 123.000.00")
+      checkExceptionInExpression[NumberFormatException](
+        cast("abc.com", dataType), "invalid input syntax for type numeric: abc.com")
+    }
+  }
+
+  protected def checkCastToNumericError(l: Literal, to: DataType, tryCastResult: Any): Unit = {
+    checkExceptionInExpression[NumberFormatException](
+      cast(l, to), "invalid input syntax for type numeric: true")
+  }
+
+  test("cast from invalid string array to numeric array should throw NumberFormatException") {
+    val array = Literal.create(Seq("123", "true", "f", null),
+      ArrayType(StringType, containsNull = true))
+
+    checkCastToNumericError(array, ArrayType(ByteType, containsNull = true),
+      Seq(123.toByte, null, null, null))
+    checkCastToNumericError(array, ArrayType(ShortType, containsNull = true),
+      Seq(123.toShort, null, null, null))
+    checkCastToNumericError(array, ArrayType(IntegerType, containsNull = true),
+      Seq(123, null, null, null))
+    checkCastToNumericError(array, ArrayType(LongType, containsNull = true),
+      Seq(123L, null, null, null))
+  }
+
+  test("Fast fail for cast string type to decimal type in ansi mode") {
+    checkEvaluation(cast("12345678901234567890123456789012345678", DecimalType(38, 0)),
+      Decimal("12345678901234567890123456789012345678"))
+    checkExceptionInExpression[ArithmeticException](
+      cast("123456789012345678901234567890123456789", DecimalType(38, 0)),
+      "out of decimal type range")
+    checkExceptionInExpression[ArithmeticException](
+      cast("12345678901234567890123456789012345678", DecimalType(38, 1)),
+      "cannot be represented as Decimal(38, 1)")
+
+    checkEvaluation(cast("0.00000000000000000000000000000000000001", DecimalType(38, 0)),
+      Decimal("0"))
+    checkEvaluation(cast("0.00000000000000000000000000000000000000000001", DecimalType(38, 0)),
+      Decimal("0"))
+    checkEvaluation(cast("0.00000000000000000000000000000000000001", DecimalType(38, 18)),
+      Decimal("0E-18"))
+    checkEvaluation(cast("6E-120", DecimalType(38, 0)),
+      Decimal("0"))
+
+    checkEvaluation(cast("6E+37", DecimalType(38, 0)),
+      Decimal("60000000000000000000000000000000000000"))
+    checkExceptionInExpression[ArithmeticException](
+      cast("6E+38", DecimalType(38, 0)),
+      "out of decimal type range")
+    checkExceptionInExpression[ArithmeticException](
+      cast("6E+37", DecimalType(38, 1)),
+      "cannot be represented as Decimal(38, 1)")
+
+    checkExceptionInExpression[NumberFormatException](
+      cast("abcd", DecimalType(38, 1)),
+      "invalid input syntax for type numeric")
+  }
+
+  protected def checkCastToBooleanError(l: Literal, to: DataType, tryCastResult: Any): Unit = {
+    checkExceptionInExpression[UnsupportedOperationException](
+      cast(l, to), s"invalid input syntax for type boolean")
+  }
+
+  test("ANSI mode: cast string to boolean with parse error") {
+    checkCastToBooleanError(Literal("abc"), BooleanType, null)
+    checkCastToBooleanError(Literal(""), BooleanType, null)
+  }
+
+  test("cast from array II") {
+    val array = Literal.create(Seq("123", "true", "f", null),
+      ArrayType(StringType, containsNull = true))
+    val array_notNull = Literal.create(Seq("123", "true", "f"),
+      ArrayType(StringType, containsNull = false))
+
+    {
+      val to: DataType = ArrayType(BooleanType, containsNull = true)
+      val ret = cast(array, to)
+      assert(ret.resolved)
+      checkCastToBooleanError(array, to, Seq(null, true, false, null))
+    }
+
+    {
+      val to: DataType = ArrayType(BooleanType, containsNull = true)
+      val ret = cast(array_notNull, to)
+      assert(ret.resolved)
+      checkCastToBooleanError(array_notNull, to, Seq(null, true, false))
+    }
+  }
+
+  test("cast from map II") {
+    val map = Literal.create(
+      Map("a" -> "123", "b" -> "true", "c" -> "f", "d" -> null),
+      MapType(StringType, StringType, valueContainsNull = true))
+    val map_notNull = Literal.create(
+      Map("a" -> "123", "b" -> "true", "c" -> "f"),
+      MapType(StringType, StringType, valueContainsNull = false))
+
+    checkNullCast(MapType(StringType, IntegerType), MapType(StringType, StringType))
+
+    {
+      val to: DataType = MapType(StringType, BooleanType, valueContainsNull = true)
+      val ret = cast(map, to)
+      assert(ret.resolved)
+      checkCastToBooleanError(map, to, Map("a" -> null, "b" -> true, "c" -> false, "d" -> null))
+    }
+
+    {
+      val to: DataType = MapType(StringType, BooleanType, valueContainsNull = true)
+      val ret = cast(map_notNull, to)
+      assert(ret.resolved)
+      checkCastToBooleanError(map_notNull, to, Map("a" -> null, "b" -> true, "c" -> false))
+    }
+  }
+
+  test("cast from struct II") {
+    checkNullCast(
+      StructType(Seq(
+        StructField("a", StringType),
+        StructField("b", IntegerType))),
+      StructType(Seq(
+        StructField("a", StringType),
+        StructField("b", StringType))))
+
+    val struct = Literal.create(
+      InternalRow(
+        UTF8String.fromString("123"),
+        UTF8String.fromString("true"),
+        UTF8String.fromString("f"),
+        null),
+      StructType(Seq(
+        StructField("a", StringType, nullable = true),
+        StructField("b", StringType, nullable = true),
+        StructField("c", StringType, nullable = true),
+        StructField("d", StringType, nullable = true))))
+    val struct_notNull = Literal.create(
+      InternalRow(
+        UTF8String.fromString("123"),
+        UTF8String.fromString("true"),
+        UTF8String.fromString("f")),
+      StructType(Seq(
+        StructField("a", StringType, nullable = false),
+        StructField("b", StringType, nullable = false),
+        StructField("c", StringType, nullable = false))))
+
+    {
+      val to: DataType = StructType(Seq(
+        StructField("a", BooleanType, nullable = true),
+        StructField("b", BooleanType, nullable = true),
+        StructField("c", BooleanType, nullable = true),
+        StructField("d", BooleanType, nullable = true)))
+      val ret = cast(struct, to)
+      assert(ret.resolved)
+      checkCastToBooleanError(struct, to, InternalRow(null, true, false, null))
+    }
+
+    {
+      val to: DataType = StructType(Seq(
+        StructField("a", BooleanType, nullable = true),
+        StructField("b", BooleanType, nullable = true),
+        StructField("c", BooleanType, nullable = true)))
+      val ret = cast(struct_notNull, to)
+      assert(ret.resolved)
+      checkCastToBooleanError(struct_notNull, to, InternalRow(null, true, false))
+    }
+  }
+
+  test("ANSI mode: cast string to timestamp with parse error") {
+    DateTimeTestUtils.outstandingZoneIds.foreach { zid =>
+      def checkCastWithParseError(str: String): Unit = {
+        checkExceptionInExpression[DateTimeException](
+          cast(Literal(str), TimestampType, Option(zid.getId)),
+          s"Cannot cast $str to TimestampType.")
+      }
+
+      checkCastWithParseError("123")
+      checkCastWithParseError("2015-03-18 123142")
+      checkCastWithParseError("2015-03-18T123123")
+      checkCastWithParseError("2015-03-18X")
+      checkCastWithParseError("2015/03/18")
+      checkCastWithParseError("2015.03.18")
+      checkCastWithParseError("20150318")
+      checkCastWithParseError("2015-031-8")
+      checkCastWithParseError("2015-03-18T12:03:17-0:70")
+      checkCastWithParseError("abdef")
+    }
+  }
+
+  test("ANSI mode: cast string to date with parse error") {
+    DateTimeTestUtils.outstandingZoneIds.foreach { zid =>
+      def checkCastWithParseError(str: String): Unit = {
+        checkExceptionInExpression[DateTimeException](
+          cast(Literal(str), DateType, Option(zid.getId)),
+          s"Cannot cast $str to DateType.")
+      }
+
+      checkCastWithParseError("12345")
+      checkCastWithParseError("12345-12-18")
+      checkCastWithParseError("2015-13-18")
+      checkCastWithParseError("2015-03-128")
+      checkCastWithParseError("2015/03/18")
+      checkCastWithParseError("2015.03.18")
+      checkCastWithParseError("20150318")
+      checkCastWithParseError("2015-031-8")
+      checkCastWithParseError("2015-03-18ABC")
+      checkCastWithParseError("abdef")
+    }
+  }
+
+  test("SPARK-26218: Fix the corner case of codegen when casting float to Integer") {
+    checkExceptionInExpression[ArithmeticException](
+      cast(cast(Literal("2147483648"), FloatType), IntegerType), "overflow")
+  }
+}
+
+/**
+ * Test suite for data type casting expression [[Cast]].
+ */
+class CastSuite extends CastSuiteBase {
 
   override def cast(v: Any, targetType: DataType, timeZoneId: Option[String] = None): CastBase = {
     v match {
@@ -988,54 +1253,38 @@ class CastSuite extends CastSuiteBase {
     }
   }
 
-  test("cast from int") {
-    checkCast(0, false)
-    checkCast(1, true)
-    checkCast(-5, true)
-    checkCast(1, 1.toByte)
-    checkCast(1, 1.toShort)
-    checkCast(1, 1)
-    checkCast(1, 1.toLong)
-    checkCast(1, 1.0f)
-    checkCast(1, 1.0)
-    checkCast(123, "123")
+  test("null cast #2") {
+    import DataTypeTestUtils._
 
-    checkEvaluation(cast(123, DecimalType.USER_DEFAULT), Decimal(123))
-    checkEvaluation(cast(123, DecimalType(3, 0)), Decimal(123))
-    checkEvaluation(cast(123, DecimalType(3, 1)), null)
-    checkEvaluation(cast(123, DecimalType(2, 0)), null)
+    checkNullCast(DateType, BooleanType)
+    checkNullCast(TimestampType, BooleanType)
+    checkNullCast(BooleanType, TimestampType)
+    numericTypes.foreach(dt => checkNullCast(dt, TimestampType))
+    numericTypes.foreach(dt => checkNullCast(TimestampType, dt))
+    numericTypes.foreach(dt => checkNullCast(DateType, dt))
   }
 
-  test("cast from long") {
-    checkCast(0L, false)
-    checkCast(1L, true)
-    checkCast(-5L, true)
-    checkCast(1L, 1.toByte)
-    checkCast(1L, 1.toShort)
-    checkCast(1L, 1)
-    checkCast(1L, 1.toLong)
-    checkCast(1L, 1.0f)
-    checkCast(1L, 1.0)
-    checkCast(123L, "123")
-
-    checkEvaluation(cast(123L, DecimalType.USER_DEFAULT), Decimal(123))
-    checkEvaluation(cast(123L, DecimalType(3, 0)), Decimal(123))
+  test("cast from long #2") {
     checkEvaluation(cast(123L, DecimalType(3, 1)), null)
-
     checkEvaluation(cast(123L, DecimalType(2, 0)), null)
   }
 
-  test("cast from int 2") {
-    checkEvaluation(cast(1, LongType), 1.toLong)
+  test("cast from int #2") {
     checkEvaluation(cast(cast(1000, TimestampType), LongType), 1000.toLong)
     checkEvaluation(cast(cast(-1200, TimestampType), LongType), -1200.toLong)
 
-    checkEvaluation(cast(123, DecimalType.USER_DEFAULT), Decimal(123))
-    checkEvaluation(cast(123, DecimalType(3, 0)), Decimal(123))
     checkEvaluation(cast(123, DecimalType(3, 1)), null)
     checkEvaluation(cast(123, DecimalType(2, 0)), null)
   }
 
+  test("cast string to date #2") {
+    checkEvaluation(Cast(Literal("2015-03-18X"), DateType), null)
+    checkEvaluation(Cast(Literal("2015/03/18"), DateType), null)
+    checkEvaluation(Cast(Literal("2015.03.18"), DateType), null)
+    checkEvaluation(Cast(Literal("20150318"), DateType), null)
+    checkEvaluation(Cast(Literal("2015-031-8"), DateType), null)
+  }
+
   test("casting to fixed-precision decimals") {
     assert(cast(123, DecimalType.USER_DEFAULT).nullable === false)
     assert(cast(10.03f, DecimalType.SYSTEM_DEFAULT).nullable)
@@ -1168,6 +1417,101 @@ class CastSuite extends CastSuiteBase {
       StructType(StructField("a", IntegerType, true) :: Nil)))
   }
 
+  test("cast string to boolean II") {
+    checkEvaluation(cast("abc", BooleanType), null)
+    checkEvaluation(cast("", BooleanType), null)
+  }
+
+  test("cast from array II") {
+    val array = Literal.create(Seq("123", "true", "f", null),
+      ArrayType(StringType, containsNull = true))
+    val array_notNull = Literal.create(Seq("123", "true", "f"),
+      ArrayType(StringType, containsNull = false))
+
+    {
+      val ret = cast(array, ArrayType(BooleanType, containsNull = true))
+      assert(ret.resolved)
+      checkEvaluation(ret, Seq(null, true, false, null))
+    }
+
+    {
+      val ret = cast(array_notNull, ArrayType(BooleanType, containsNull = true))
+      assert(ret.resolved)
+      checkEvaluation(ret, Seq(null, true, false))
+    }
+  }
+
+  test("cast from map II") {
+    val map = Literal.create(
+      Map("a" -> "123", "b" -> "true", "c" -> "f", "d" -> null),
+      MapType(StringType, StringType, valueContainsNull = true))
+    val map_notNull = Literal.create(
+      Map("a" -> "123", "b" -> "true", "c" -> "f"),
+      MapType(StringType, StringType, valueContainsNull = false))
+
+    {
+      val ret = cast(map, MapType(StringType, BooleanType, valueContainsNull = true))
+      assert(ret.resolved)
+      checkEvaluation(ret, Map("a" -> null, "b" -> true, "c" -> false, "d" -> null))
+    }
+
+    {
+      val ret = cast(map_notNull, MapType(StringType, BooleanType, valueContainsNull = true))
+      assert(ret.resolved)
+      checkEvaluation(ret, Map("a" -> null, "b" -> true, "c" -> false))
+    }
+  }
+
+  test("cast from struct II") {
+    checkNullCast(
+      StructType(Seq(
+        StructField("a", StringType),
+        StructField("b", IntegerType))),
+      StructType(Seq(
+        StructField("a", StringType),
+        StructField("b", StringType))))
+
+    val struct = Literal.create(
+      InternalRow(
+        UTF8String.fromString("123"),
+        UTF8String.fromString("true"),
+        UTF8String.fromString("f"),
+        null),
+      StructType(Seq(
+        StructField("a", StringType, nullable = true),
+        StructField("b", StringType, nullable = true),
+        StructField("c", StringType, nullable = true),
+        StructField("d", StringType, nullable = true))))
+    val struct_notNull = Literal.create(
+      InternalRow(
+        UTF8String.fromString("123"),
+        UTF8String.fromString("true"),
+        UTF8String.fromString("f")),
+      StructType(Seq(
+        StructField("a", StringType, nullable = false),
+        StructField("b", StringType, nullable = false),
+        StructField("c", StringType, nullable = false))))
+
+    {
+      val ret = cast(struct, StructType(Seq(
+        StructField("a", BooleanType, nullable = true),
+        StructField("b", BooleanType, nullable = true),
+        StructField("c", BooleanType, nullable = true),
+        StructField("d", BooleanType, nullable = true))))
+      assert(ret.resolved)
+      checkEvaluation(ret, InternalRow(null, true, false, null))
+    }
+
+    {
+      val ret = cast(struct_notNull, StructType(Seq(
+        StructField("a", BooleanType, nullable = true),
+        StructField("b", BooleanType, nullable = true),
+        StructField("c", BooleanType, nullable = true))))
+      assert(ret.resolved)
+      checkEvaluation(ret, InternalRow(null, true, false))
+    }
+  }
+
   test("SPARK-31227: Non-nullable null type should not coerce to nullable type") {
     TypeCoercionSuite.allTypes.foreach { t =>
       assert(Cast.canCast(ArrayType(NullType, false), ArrayType(t, false)))
@@ -1300,6 +1644,58 @@ class CastSuite extends CastSuiteBase {
     }
   }
 
+  test("cast from date") {
+    val d = Date.valueOf("1970-01-01")
+    checkEvaluation(cast(d, ShortType), null)
+    checkEvaluation(cast(d, IntegerType), null)
+    checkEvaluation(cast(d, LongType), null)
+    checkEvaluation(cast(d, FloatType), null)
+    checkEvaluation(cast(d, DoubleType), null)
+    checkEvaluation(cast(d, DecimalType.SYSTEM_DEFAULT), null)
+    checkEvaluation(cast(d, DecimalType(10, 2)), null)
+    checkEvaluation(cast(d, StringType), "1970-01-01")
+
+    checkEvaluation(
+      cast(cast(d, TimestampType, UTC_OPT), StringType, UTC_OPT),
+      "1970-01-01 00:00:00")
+  }
+
+  test("cast from timestamp") {
+    val millis = 15 * 1000 + 3
+    val seconds = millis * 1000 + 3
+    val ts = new Timestamp(millis)
+    val tss = new Timestamp(seconds)
+    checkEvaluation(cast(ts, ShortType), 15.toShort)
+    checkEvaluation(cast(ts, IntegerType), 15)
+    checkEvaluation(cast(ts, LongType), 15.toLong)
+    checkEvaluation(cast(ts, FloatType), 15.003f)
+    checkEvaluation(cast(ts, DoubleType), 15.003)
+
+    checkEvaluation(cast(cast(tss, ShortType), TimestampType),
+      fromJavaTimestamp(ts) * MILLIS_PER_SECOND)
+    checkEvaluation(cast(cast(tss, IntegerType), TimestampType),
+      fromJavaTimestamp(ts) * MILLIS_PER_SECOND)
+    checkEvaluation(cast(cast(tss, LongType), TimestampType),
+      fromJavaTimestamp(ts) * MILLIS_PER_SECOND)
+    checkEvaluation(
+      cast(cast(millis.toFloat / MILLIS_PER_SECOND, TimestampType), FloatType),
+      millis.toFloat / MILLIS_PER_SECOND)
+    checkEvaluation(
+      cast(cast(millis.toDouble / MILLIS_PER_SECOND, TimestampType), DoubleType),
+      millis.toDouble / MILLIS_PER_SECOND)
+    checkEvaluation(
+      cast(cast(Decimal(1), TimestampType), DecimalType.SYSTEM_DEFAULT),
+      Decimal(1))
+
+    // A test for higher precision than millis
+    checkEvaluation(cast(cast(0.000001, TimestampType), DoubleType), 0.000001)
+
+    checkEvaluation(cast(Double.NaN, TimestampType), null)
+    checkEvaluation(cast(1.0 / 0.0, TimestampType), null)
+    checkEvaluation(cast(Float.NaN, TimestampType), null)
+    checkEvaluation(cast(1.0f / 0.0f, TimestampType), null)
+  }
+
   test("cast a timestamp before the epoch 1970-01-01 00:00:00Z") {
     withDefaultTimeZone(UTC) {
       val negativeTs = Timestamp.valueOf("1900-05-05 18:34:56.1")
@@ -1311,14 +1707,197 @@ class CastSuite extends CastSuiteBase {
       checkEvaluation(cast(negativeTs, LongType), expectedSecs)
     }
   }
+
+  test("SPARK-32828: cast from a derived user-defined type to a base type") {
+    val v = Literal.create(Row(1), new ExampleSubTypeUDT())
+    checkEvaluation(cast(v, new ExampleBaseTypeUDT), Row(1))
+  }
+
+  test("Fast fail for cast string type to decimal type") {
+    checkEvaluation(cast("12345678901234567890123456789012345678", DecimalType(38, 0)),
+      Decimal("12345678901234567890123456789012345678"))
+    checkEvaluation(cast("123456789012345678901234567890123456789", DecimalType(38, 0)), null)
+    checkEvaluation(cast("12345678901234567890123456789012345678", DecimalType(38, 1)), null)
+
+    checkEvaluation(cast("0.00000000000000000000000000000000000001", DecimalType(38, 0)),
+      Decimal("0"))
+    checkEvaluation(cast("0.00000000000000000000000000000000000000000001", DecimalType(38, 0)),
+      Decimal("0"))
+    checkEvaluation(cast("0.00000000000000000000000000000000000001", DecimalType(38, 18)),
+      Decimal("0E-18"))
+    checkEvaluation(cast("6E-120", DecimalType(38, 0)),
+      Decimal("0"))
+
+    checkEvaluation(cast("6E+37", DecimalType(38, 0)),
+      Decimal("60000000000000000000000000000000000000"))
+    checkEvaluation(cast("6E+38", DecimalType(38, 0)), null)
+    checkEvaluation(cast("6E+37", DecimalType(38, 1)), null)
+
+    checkEvaluation(cast("abcd", DecimalType(38, 1)), null)
+  }
+
+  test("data type casting II") {
+    checkEvaluation(
+      cast(cast(cast(cast(cast(cast("5", ByteType), TimestampType),
+        DecimalType.SYSTEM_DEFAULT), LongType), StringType), ShortType),
+        5.toShort)
+      checkEvaluation(
+        cast(cast(cast(cast(cast(cast("5", TimestampType, UTC_OPT), ByteType),
+          DecimalType.SYSTEM_DEFAULT), LongType), StringType), ShortType),
+        null)
+      checkEvaluation(cast(cast(cast(cast(cast(cast("5", DecimalType.SYSTEM_DEFAULT),
+        ByteType), TimestampType), LongType), StringType), ShortType),
+        5.toShort)
+  }
+
+  test("Cast from double II") {
+    checkEvaluation(cast(cast(1.toDouble, TimestampType), DoubleType), 1.toDouble)
+  }
+
+  test("SPARK-34727: cast from float II") {
+    checkCast(16777215.0f, java.time.Instant.ofEpochSecond(16777215))
+  }
+
+  test("SPARK-34744: Improve error message for casting cause overflow error") {
+    withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
+      val e1 = intercept[ArithmeticException] {
+        Cast(Literal(Byte.MaxValue + 1), ByteType).eval()
+      }.getMessage
+      assert(e1.contains("Casting 128 to tinyint causes overflow"))
+      val e2 = intercept[ArithmeticException] {
+        Cast(Literal(Short.MaxValue + 1), ShortType).eval()
+      }.getMessage
+      assert(e2.contains("Casting 32768 to smallint causes overflow"))
+      val e3 = intercept[ArithmeticException] {
+        Cast(Literal(Int.MaxValue + 1L), IntegerType).eval()
+      }.getMessage
+      assert(e3.contains("Casting 2147483648 to int causes overflow"))
+    }
+  }
+
+  test("SPARK-35112: Cast string to day-time interval") {
+    checkEvaluation(cast(Literal.create("0 0:0:0"), DayTimeIntervalType), 0L)
+    checkEvaluation(cast(Literal.create(" interval '0 0:0:0' Day TO second   "),
+      DayTimeIntervalType), 0L)
+    checkEvaluation(cast(Literal.create("INTERVAL '1 2:03:04' DAY TO SECOND"),
+      DayTimeIntervalType), 93784000000L)
+    checkEvaluation(cast(Literal.create("INTERVAL '1 03:04:00' DAY TO SECOND"),
+      DayTimeIntervalType), 97440000000L)
+    checkEvaluation(cast(Literal.create("INTERVAL '1 03:04:00.0000' DAY TO SECOND"),
+      DayTimeIntervalType), 97440000000L)
+    checkEvaluation(cast(Literal.create("1 2:03:04"), DayTimeIntervalType), 93784000000L)
+    checkEvaluation(cast(Literal.create("INTERVAL '-10 2:03:04' DAY TO SECOND"),
+      DayTimeIntervalType), -871384000000L)
+    checkEvaluation(cast(Literal.create("-10 2:03:04"), DayTimeIntervalType), -871384000000L)
+    checkEvaluation(cast(Literal.create("-106751991 04:00:54.775808"), DayTimeIntervalType),
+      Long.MinValue)
+    checkEvaluation(cast(Literal.create("106751991 04:00:54.775807"), DayTimeIntervalType),
+      Long.MaxValue)
+
+    Seq("-106751991 04:00:54.775808", "106751991 04:00:54.775807").foreach { interval =>
+      val ansiInterval = s"INTERVAL '$interval' DAY TO SECOND"
+      checkEvaluation(
+        cast(cast(Literal.create(interval), DayTimeIntervalType), StringType), ansiInterval)
+      checkEvaluation(cast(cast(Literal.create(ansiInterval),
+        DayTimeIntervalType), StringType), ansiInterval)
+    }
+
+    Seq("INTERVAL '-106751991 04:00:54.775809' YEAR TO MONTH",
+      "INTERVAL '106751991 04:00:54.775808' YEAR TO MONTH").foreach { interval =>
+        val e = intercept[IllegalArgumentException] {
+          cast(Literal.create(interval), DayTimeIntervalType).eval()
+        }.getMessage
+        assert(e.contains("Interval string must match day-time format of"))
+      }
+
+    Seq(Byte.MaxValue, Short.MaxValue, Int.MaxValue, Long.MaxValue, Long.MinValue + 1,
+      Long.MinValue).foreach { duration =>
+        val interval = Literal.create(Duration.of(duration, ChronoUnit.MICROS), DayTimeIntervalType)
+        checkEvaluation(cast(cast(interval, StringType), DayTimeIntervalType), duration)
+      }
+  }
+
+  test("SPARK-35111: Cast string to year-month interval") {
+    checkEvaluation(cast(Literal.create("INTERVAL '1-0' YEAR TO MONTH"),
+      YearMonthIntervalType), 12)
+    checkEvaluation(cast(Literal.create("INTERVAL '-1-0' YEAR TO MONTH"),
+      YearMonthIntervalType), -12)
+    checkEvaluation(cast(Literal.create("INTERVAL -'-1-0' YEAR TO MONTH"),
+      YearMonthIntervalType), 12)
+    checkEvaluation(cast(Literal.create("INTERVAL +'-1-0' YEAR TO MONTH"),
+      YearMonthIntervalType), -12)
+    checkEvaluation(cast(Literal.create("INTERVAL +'+1-0' YEAR TO MONTH"),
+      YearMonthIntervalType), 12)
+    checkEvaluation(cast(Literal.create("INTERVAL +'1-0' YEAR TO MONTH"),
+      YearMonthIntervalType), 12)
+    checkEvaluation(cast(Literal.create(" interval +'1-0' YEAR  TO MONTH "),
+      YearMonthIntervalType), 12)
+    checkEvaluation(cast(Literal.create(" -1-0 "), YearMonthIntervalType), -12)
+    checkEvaluation(cast(Literal.create("-1-0"), YearMonthIntervalType), -12)
+    checkEvaluation(cast(Literal.create(null, StringType), YearMonthIntervalType), null)
+
+    Seq("0-0", "10-1", "-178956970-7", "178956970-7", "-178956970-8").foreach { interval =>
+      val ansiInterval = s"INTERVAL '$interval' YEAR TO MONTH"
+      checkEvaluation(
+        cast(cast(Literal.create(interval), YearMonthIntervalType), StringType), ansiInterval)
+      checkEvaluation(cast(cast(Literal.create(ansiInterval),
+        YearMonthIntervalType), StringType), ansiInterval)
+    }
+
+    Seq("INTERVAL '-178956970-9' YEAR TO MONTH", "INTERVAL '178956970-8' YEAR TO MONTH")
+      .foreach { interval =>
+        val e = intercept[IllegalArgumentException] {
+          cast(Literal.create(interval), YearMonthIntervalType).eval()
+        }.getMessage
+        assert(e.contains("Error parsing interval year-month string: integer overflow"))
+      }
+
+    Seq(Byte.MaxValue, Short.MaxValue, Int.MaxValue, Int.MinValue + 1, Int.MinValue)
+      .foreach { period =>
+        val interval = Literal.create(Period.ofMonths(period), YearMonthIntervalType)
+        checkEvaluation(cast(cast(interval, StringType), YearMonthIntervalType), period)
+      }
+  }
+}
+
+/**
+ * Test suite for data type casting expression [[Cast]] with ANSI mode disabled.
+ */
+class CastSuiteWithAnsiModeOn extends AnsiCastSuiteBase {
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    SQLConf.get.setConf(SQLConf.ANSI_ENABLED, true)
+  }
+
+  override def afterAll(): Unit = {
+    super.afterAll()
+    SQLConf.get.unsetConf(SQLConf.ANSI_ENABLED)
+  }
+
+  override def cast(v: Any, targetType: DataType, timeZoneId: Option[String] = None): CastBase = {
+    v match {
+      case lit: Expression => Cast(lit, targetType, timeZoneId)
+      case _ => Cast(Literal(v), targetType, timeZoneId)
+    }
+  }
+
+  override def setConfigurationHint: String =
+    s"set ${SQLConf.ANSI_ENABLED.key} as false"
 }
 
 /**
- * Test suite for data type casting expression [[AnsiCast]].
+ * Test suite for data type casting expression [[AnsiCast]] with ANSI mode enabled.
  */
-class AnsiCastSuite extends CastSuiteBase {
-  // It is not required to set SQLConf.ANSI_ENABLED as true for testing numeric overflow.
-  override protected def requiredAnsiEnabledForOverflowTestCases: Boolean = false
+class AnsiCastSuiteWithAnsiModeOn extends AnsiCastSuiteBase {
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    SQLConf.get.setConf(SQLConf.ANSI_ENABLED, true)
+  }
+
+  override def afterAll(): Unit = {
+    super.afterAll()
+    SQLConf.get.unsetConf(SQLConf.ANSI_ENABLED)
+  }
 
   override def cast(v: Any, targetType: DataType, timeZoneId: Option[String] = None): CastBase = {
     v match {
@@ -1327,43 +1906,33 @@ class AnsiCastSuite extends CastSuiteBase {
     }
   }
 
-  test("cast from invalid string to numeric should throw NumberFormatException") {
-    // cast to IntegerType
-    Seq(IntegerType, ShortType, ByteType, LongType).foreach { dataType =>
-      val array = Literal.create(Seq("123", "true", "f", null),
-        ArrayType(StringType, containsNull = true))
-      checkExceptionInExpression[NumberFormatException](
-        cast(array, ArrayType(dataType, containsNull = true)), "invalid input")
-      checkExceptionInExpression[NumberFormatException](
-        cast("string", dataType), "invalid input")
-      checkExceptionInExpression[NumberFormatException](
-        cast("123-string", dataType), "invalid input")
-      checkExceptionInExpression[NumberFormatException](
-        cast("2020-07-19", dataType), "invalid input")
-      checkExceptionInExpression[NumberFormatException](
-        cast("1.23", dataType), "invalid input")
-    }
+  override def setConfigurationHint: String =
+    s"set ${SQLConf.STORE_ASSIGNMENT_POLICY.key} as" +
+      s" ${SQLConf.StoreAssignmentPolicy.LEGACY.toString}"
+}
 
-    Seq(DoubleType, FloatType, DecimalType.USER_DEFAULT).foreach { dataType =>
-      checkExceptionInExpression[NumberFormatException](
-        cast("string", dataType), "invalid input")
-      checkExceptionInExpression[NumberFormatException](
-        cast("123.000.00", dataType), "invalid input")
-      checkExceptionInExpression[NumberFormatException](
-        cast("abc.com", dataType), "invalid input")
-    }
+/**
+ * Test suite for data type casting expression [[AnsiCast]] with ANSI mode disabled.
+ */
+class AnsiCastSuiteWithAnsiModeOff extends AnsiCastSuiteBase {
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    SQLConf.get.setConf(SQLConf.ANSI_ENABLED, false)
   }
 
-  test("cast a timestamp before the epoch 1970-01-01 00:00:00Z") {
-    def errMsg(t: String): String = s"Casting -2198208303900000 to $t causes overflow"
-    withDefaultTimeZone(UTC) {
-      val negativeTs = Timestamp.valueOf("1900-05-05 18:34:56.1")
-      assert(negativeTs.getTime < 0)
-      val expectedSecs = Math.floorDiv(negativeTs.getTime, MILLIS_PER_SECOND)
-      checkExceptionInExpression[ArithmeticException](cast(negativeTs, ByteType), errMsg("byte"))
-      checkExceptionInExpression[ArithmeticException](cast(negativeTs, ShortType), errMsg("short"))
-      checkExceptionInExpression[ArithmeticException](cast(negativeTs, IntegerType), errMsg("int"))
-      checkEvaluation(cast(negativeTs, LongType), expectedSecs)
+  override def afterAll(): Unit = {
+    super.afterAll()
+    SQLConf.get.unsetConf(SQLConf.ANSI_ENABLED)
+  }
+
+  override def cast(v: Any, targetType: DataType, timeZoneId: Option[String] = None): CastBase = {
+    v match {
+      case lit: Expression => AnsiCast(lit, targetType, timeZoneId)
+      case _ => AnsiCast(Literal(v), targetType, timeZoneId)
     }
   }
+
+  override def setConfigurationHint: String =
+    s"set ${SQLConf.STORE_ASSIGNMENT_POLICY.key} as" +
+      s" ${SQLConf.StoreAssignmentPolicy.LEGACY.toString}"
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
index db52229f860b4..026d9676f4fba 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
@@ -19,6 +19,8 @@ package org.apache.spark.sql.catalyst.expressions
 
 import java.sql.Timestamp
 
+import scala.math.Ordering
+
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.metrics.source.CodegenMetrics
 import org.apache.spark.sql.Row
@@ -102,7 +104,7 @@ class CodeGenerationSuite extends SparkFunSuite with ExpressionEvalHelper {
   test("SPARK-22543: split large if expressions into blocks due to JVM code size limit") {
     var strExpr: Expression = Literal("abc")
     for (_ <- 1 to 150) {
-      strExpr = Decode(Encode(strExpr, "utf-8"), "utf-8")
+      strExpr = StringDecode(Encode(strExpr, "utf-8"), "utf-8")
     }
 
     val expressions = Seq(If(EqualTo(strExpr, strExpr), strExpr, strExpr))
@@ -330,7 +332,7 @@ class CodeGenerationSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("SPARK-17160: field names are properly escaped by AssertTrue") {
-    GenerateUnsafeProjection.generate(AssertTrue(Cast(Literal("\""), BooleanType)) :: Nil)
+    GenerateUnsafeProjection.generate(AssertTrue(Cast(Literal("\""), BooleanType)).child :: Nil)
   }
 
   test("should not apply common subexpression elimination on conditional expressions") {
@@ -525,7 +527,7 @@ class CodeGenerationSuite extends SparkFunSuite with ExpressionEvalHelper {
       .exists(_.getRenderedMessage().contains("Generated method too long")))
   }
 
-  test("SPARK-28916: subexrepssion elimination can cause 64kb code limit on UnsafeProjection") {
+  test("SPARK-28916: subexpression elimination can cause 64kb code limit on UnsafeProjection") {
     val numOfExprs = 10000
     val exprs = (0 to numOfExprs).flatMap(colIndex =>
       Seq(Add(BoundReference(colIndex, DoubleType, true),
@@ -536,12 +538,31 @@ class CodeGenerationSuite extends SparkFunSuite with ExpressionEvalHelper {
     GenerateUnsafeProjection.generate(exprs, true)
     GenerateMutableProjection.generate(exprs, true)
   }
+
+  test("SPARK-32624: Use CodeGenerator.typeName() to fix byte[] compile issue") {
+    val ctx = new CodegenContext
+    val bytes = new Array[Byte](3)
+    val refTerm = ctx.addReferenceObj("bytes", bytes)
+    assert(refTerm == "((byte[]) references[0] /* bytes */)")
+  }
+
+  test("SPARK-32624: CodegenContext.addReferenceObj should work for nested Scala class") {
+    // emulate TypeUtils.getInterpretedOrdering(StringType)
+    val ctx = new CodegenContext
+    val comparator = implicitly[Ordering[UTF8String]]
+    val refTerm = ctx.addReferenceObj("comparator", comparator)
+
+    // Expecting result:
+    //   "((scala.math.LowPriorityOrderingImplicits$$anon$3) references[0] /* comparator */)"
+    // Using lenient assertions to be resilient to anonymous class numbering changes
+    assert(!refTerm.contains("null"))
+    assert(refTerm.contains("scala.math.LowPriorityOrderingImplicits$$anon$"))
+  }
 }
 
-case class HugeCodeIntExpression(value: Int) extends Expression {
+case class HugeCodeIntExpression(value: Int) extends LeafExpression {
   override def nullable: Boolean = true
   override def dataType: DataType = IntegerType
-  override def children: Seq[Expression] = Nil
   override def eval(input: InternalRow): Any = value
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     // Assuming HugeMethodLimit to be 8000
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
index a4baad0c72444..456ccafa571f9 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.catalyst.expressions
 
 import java.sql.{Date, Timestamp}
+import java.time.{Duration, Period}
 import java.util.TimeZone
 
 import scala.language.implicitConversions
@@ -28,7 +29,6 @@ import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, DateTimeUtils}
-import org.apache.spark.sql.catalyst.util.DateTimeConstants.MICROS_PER_DAY
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.UTC
 import org.apache.spark.sql.catalyst.util.IntervalUtils._
 import org.apache.spark.sql.internal.SQLConf
@@ -932,7 +932,187 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
           Literal(Date.valueOf("1970-02-01")),
           Literal(negateExact(stringToInterval("interval 1 month")))),
         EmptyRow,
-        s"sequence boundaries: 0 to 2678400000000 by -${28 * MICROS_PER_DAY}")
+        s"sequence boundaries: 0 to 2678400000000 by -1 months")
+
+      // SPARK-32133: Sequence step must be a day interval if start and end values are dates
+      checkExceptionInExpression[IllegalArgumentException](Sequence(
+        Cast(Literal("2011-03-01"), DateType),
+        Cast(Literal("2011-04-01"), DateType),
+        Option(Literal(stringToInterval("interval 1 hour")))), null,
+        "sequence step must be an interval of day granularity if start and end values are dates")
+    }
+  }
+
+  test("SPARK-35088: Accept ANSI intervals by the Sequence expression") {
+    checkEvaluation(new Sequence(
+      Literal(Timestamp.valueOf("2018-01-01 00:00:00")),
+      Literal(Timestamp.valueOf("2018-01-02 00:00:00")),
+      Literal(Duration.ofHours(12))),
+      Seq(
+        Timestamp.valueOf("2018-01-01 00:00:00"),
+        Timestamp.valueOf("2018-01-01 12:00:00"),
+        Timestamp.valueOf("2018-01-02 00:00:00")))
+
+    checkEvaluation(new Sequence(
+      Literal(Timestamp.valueOf("2018-01-01 00:00:00")),
+      Literal(Timestamp.valueOf("2018-01-02 00:00:01")),
+      Literal(Duration.ofHours(12))),
+      Seq(
+        Timestamp.valueOf("2018-01-01 00:00:00"),
+        Timestamp.valueOf("2018-01-01 12:00:00"),
+        Timestamp.valueOf("2018-01-02 00:00:00")))
+
+    checkEvaluation(new Sequence(
+      Literal(Timestamp.valueOf("2018-01-02 00:00:00")),
+      Literal(Timestamp.valueOf("2018-01-01 00:00:00")),
+      Literal(Duration.ofHours(-12))),
+      Seq(
+        Timestamp.valueOf("2018-01-02 00:00:00"),
+        Timestamp.valueOf("2018-01-01 12:00:00"),
+        Timestamp.valueOf("2018-01-01 00:00:00")))
+
+    checkEvaluation(new Sequence(
+      Literal(Timestamp.valueOf("2018-01-02 00:00:00")),
+      Literal(Timestamp.valueOf("2017-12-31 23:59:59")),
+      Literal(Duration.ofHours(-12))),
+      Seq(
+        Timestamp.valueOf("2018-01-02 00:00:00"),
+        Timestamp.valueOf("2018-01-01 12:00:00"),
+        Timestamp.valueOf("2018-01-01 00:00:00")))
+
+    checkEvaluation(new Sequence(
+      Literal(Timestamp.valueOf("2018-01-01 00:00:00")),
+      Literal(Timestamp.valueOf("2018-03-01 00:00:00")),
+      Literal(Period.ofMonths(1))),
+      Seq(
+        Timestamp.valueOf("2018-01-01 00:00:00"),
+        Timestamp.valueOf("2018-02-01 00:00:00"),
+        Timestamp.valueOf("2018-03-01 00:00:00")))
+
+    checkEvaluation(new Sequence(
+      Literal(Timestamp.valueOf("2018-03-01 00:00:00")),
+      Literal(Timestamp.valueOf("2018-01-01 00:00:00")),
+      Literal(Period.ofMonths(-1))),
+      Seq(
+        Timestamp.valueOf("2018-03-01 00:00:00"),
+        Timestamp.valueOf("2018-02-01 00:00:00"),
+        Timestamp.valueOf("2018-01-01 00:00:00")))
+
+    checkEvaluation(new Sequence(
+      Literal(Timestamp.valueOf("2018-01-31 00:00:00")),
+      Literal(Timestamp.valueOf("2018-04-30 00:00:00")),
+      Literal(Period.ofMonths(1))),
+      Seq(
+        Timestamp.valueOf("2018-01-31 00:00:00"),
+        Timestamp.valueOf("2018-02-28 00:00:00"),
+        Timestamp.valueOf("2018-03-31 00:00:00"),
+        Timestamp.valueOf("2018-04-30 00:00:00")))
+
+    checkEvaluation(new Sequence(
+      Literal(Timestamp.valueOf("2018-01-01 00:00:00")),
+      Literal(Timestamp.valueOf("2023-01-01 00:00:00")),
+      Literal(Period.of(1, 5, 0))),
+      Seq(
+        Timestamp.valueOf("2018-01-01 00:00:00.000"),
+        Timestamp.valueOf("2019-06-01 00:00:00.000"),
+        Timestamp.valueOf("2020-11-01 00:00:00.000"),
+        Timestamp.valueOf("2022-04-01 00:00:00.000")))
+
+    checkEvaluation(new Sequence(
+      Literal(Timestamp.valueOf("2022-04-01 00:00:00")),
+      Literal(Timestamp.valueOf("2017-01-01 00:00:00")),
+      Literal(Period.of(-1, -5, 0))),
+      Seq(
+        Timestamp.valueOf("2022-04-01 00:00:00.000"),
+        Timestamp.valueOf("2020-11-01 00:00:00.000"),
+        Timestamp.valueOf("2019-06-01 00:00:00.000"),
+        Timestamp.valueOf("2018-01-01 00:00:00.000")))
+
+    checkEvaluation(new Sequence(
+      Literal(Timestamp.valueOf("2018-01-01 00:00:00")),
+      Literal(Timestamp.valueOf("2018-01-04 00:00:00")),
+      Literal(Duration.ofDays(1))),
+      Seq(
+        Timestamp.valueOf("2018-01-01 00:00:00.000"),
+        Timestamp.valueOf("2018-01-02 00:00:00.000"),
+        Timestamp.valueOf("2018-01-03 00:00:00.000"),
+        Timestamp.valueOf("2018-01-04 00:00:00.000")))
+
+    checkEvaluation(new Sequence(
+      Literal(Timestamp.valueOf("2018-01-04 00:00:00")),
+      Literal(Timestamp.valueOf("2018-01-01 00:00:00")),
+      Literal(Duration.ofDays(-1))),
+      Seq(
+        Timestamp.valueOf("2018-01-04 00:00:00.000"),
+        Timestamp.valueOf("2018-01-03 00:00:00.000"),
+        Timestamp.valueOf("2018-01-02 00:00:00.000"),
+        Timestamp.valueOf("2018-01-01 00:00:00.000")))
+
+    checkExceptionInExpression[IllegalArgumentException](
+      new Sequence(
+        Literal(Timestamp.valueOf("2018-01-01 00:00:00")),
+        Literal(Timestamp.valueOf("2018-01-04 00:00:00")),
+        Literal(Period.ofDays(1))),
+      EmptyRow, s"sequence boundaries: 1514793600000000 to 1515052800000000 by 0")
+
+    checkExceptionInExpression[IllegalArgumentException](
+      new Sequence(
+        Literal(Timestamp.valueOf("2018-01-04 00:00:00")),
+        Literal(Timestamp.valueOf("2018-01-01 00:00:00")),
+        Literal(Period.ofDays(-1))),
+      EmptyRow, s"sequence boundaries: 1515052800000000 to 1514793600000000 by 0")
+
+    DateTimeTestUtils.withDefaultTimeZone(UTC) {
+      checkEvaluation(new Sequence(
+        Literal(Date.valueOf("2018-01-01")),
+        Literal(Date.valueOf("2018-03-01")),
+        Literal(Period.ofMonths(1))),
+        Seq(
+          Date.valueOf("2018-01-01"),
+          Date.valueOf("2018-02-01"),
+          Date.valueOf("2018-03-01")))
+
+      checkEvaluation(new Sequence(
+        Literal(Date.valueOf("2018-01-31")),
+        Literal(Date.valueOf("2018-04-30")),
+        Literal(Period.ofMonths(1))),
+        Seq(
+          Date.valueOf("2018-01-31"),
+          Date.valueOf("2018-02-28"),
+          Date.valueOf("2018-03-31"),
+          Date.valueOf("2018-04-30")))
+
+      checkEvaluation(new Sequence(
+        Literal(Date.valueOf("2018-01-01")),
+        Literal(Date.valueOf("2023-01-01")),
+        Literal(Period.of(1, 5, 0))),
+        Seq(
+          Date.valueOf("2018-01-01"),
+          Date.valueOf("2019-06-01"),
+          Date.valueOf("2020-11-01"),
+          Date.valueOf("2022-04-01")))
+
+      checkExceptionInExpression[IllegalArgumentException](
+        new Sequence(
+          Literal(Date.valueOf("2018-01-01")),
+          Literal(Date.valueOf("2018-01-05")),
+          Literal(Period.ofDays(2))),
+        EmptyRow,
+        "sequence step must be an interval year to month of day granularity" +
+          " if start and end values are dates")
+
+      checkExceptionInExpression[IllegalArgumentException](
+        new Sequence(
+          Literal(Date.valueOf("1970-01-01")),
+          Literal(Date.valueOf("1970-02-01")),
+          Literal(Period.ofMonths(-1))),
+        EmptyRow,
+        s"sequence boundaries: 0 to 2678400000000 by -1")
+
+      assert(Sequence(
+        Cast(Literal("2011-03-01"), DateType),
+        Cast(Literal("2011-04-01"), DateType),
+        Option(Literal(Duration.ofHours(1)))).checkInputDataTypes().isFailure)
     }
   }
 
@@ -1111,36 +1291,62 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
   }
 
   test("correctly handles ElementAt nullability for arrays") {
-    // CreateArray case
-    val a = AttributeReference("a", IntegerType, nullable = false)()
-    val b = AttributeReference("b", IntegerType, nullable = true)()
-    val array = CreateArray(a :: b :: Nil)
-    assert(!ElementAt(array, Literal(0)).nullable)
-    assert(ElementAt(array, Literal(1)).nullable)
-    assert(!ElementAt(array, Subtract(Literal(2), Literal(2))).nullable)
-    assert(ElementAt(array, AttributeReference("ordinal", IntegerType)()).nullable)
-
-    // GetArrayStructFields case
-    val f1 = StructField("a", IntegerType, nullable = false)
-    val f2 = StructField("b", IntegerType, nullable = true)
-    val structType = StructType(f1 :: f2 :: Nil)
-    val c = AttributeReference("c", structType, nullable = false)()
-    val inputArray1 = CreateArray(c :: Nil)
-    val inputArray1ContainsNull = c.nullable
-    val stArray1 = GetArrayStructFields(inputArray1, f1, 0, 2, inputArray1ContainsNull)
-    assert(!ElementAt(stArray1, Literal(0)).nullable)
-    val stArray2 = GetArrayStructFields(inputArray1, f2, 1, 2, inputArray1ContainsNull)
-    assert(ElementAt(stArray2, Literal(0)).nullable)
-
-    val d = AttributeReference("d", structType, nullable = true)()
-    val inputArray2 = CreateArray(c :: d :: Nil)
-    val inputArray2ContainsNull = c.nullable || d.nullable
-    val stArray3 = GetArrayStructFields(inputArray2, f1, 0, 2, inputArray2ContainsNull)
-    assert(!ElementAt(stArray3, Literal(0)).nullable)
-    assert(ElementAt(stArray3, Literal(1)).nullable)
-    val stArray4 = GetArrayStructFields(inputArray2, f2, 1, 2, inputArray2ContainsNull)
-    assert(ElementAt(stArray4, Literal(0)).nullable)
-    assert(ElementAt(stArray4, Literal(1)).nullable)
+    Seq(true, false).foreach { ansiEnabled =>
+      withSQLConf(SQLConf.ANSI_ENABLED.key -> ansiEnabled.toString) {
+        // CreateArray case
+        val a = AttributeReference("a", IntegerType, nullable = false)()
+        val b = AttributeReference("b", IntegerType, nullable = true)()
+        val array = CreateArray(a :: b :: Nil)
+        assert(!ElementAt(array, Literal(1)).nullable)
+        assert(!ElementAt(array, Literal(-2)).nullable)
+        assert(ElementAt(array, Literal(2)).nullable)
+        assert(ElementAt(array, Literal(-1)).nullable)
+        assert(!ElementAt(array, Subtract(Literal(2), Literal(1))).nullable)
+        assert(ElementAt(array, AttributeReference("ordinal", IntegerType)()).nullable)
+
+        // CreateArray case invalid indices
+        assert(!ElementAt(array, Literal(0)).nullable)
+        assert(ElementAt(array, Literal(4)).nullable == !ansiEnabled)
+        assert(ElementAt(array, Literal(-4)).nullable == !ansiEnabled)
+
+        // GetArrayStructFields case
+        val f1 = StructField("a", IntegerType, nullable = false)
+        val f2 = StructField("b", IntegerType, nullable = true)
+        val structType = StructType(f1 :: f2 :: Nil)
+        val c = AttributeReference("c", structType, nullable = false)()
+        val inputArray1 = CreateArray(c :: Nil)
+        val inputArray1ContainsNull = c.nullable
+        val stArray1 = GetArrayStructFields(inputArray1, f1, 0, 2, inputArray1ContainsNull)
+        assert(!ElementAt(stArray1, Literal(1)).nullable)
+        assert(!ElementAt(stArray1, Literal(-1)).nullable)
+        val stArray2 = GetArrayStructFields(inputArray1, f2, 1, 2, inputArray1ContainsNull)
+        assert(ElementAt(stArray2, Literal(1)).nullable)
+        assert(ElementAt(stArray2, Literal(-1)).nullable)
+
+        val d = AttributeReference("d", structType, nullable = true)()
+        val inputArray2 = CreateArray(c :: d :: Nil)
+        val inputArray2ContainsNull = c.nullable || d.nullable
+        val stArray3 = GetArrayStructFields(inputArray2, f1, 0, 2, inputArray2ContainsNull)
+        assert(!ElementAt(stArray3, Literal(1)).nullable)
+        assert(!ElementAt(stArray3, Literal(-2)).nullable)
+        assert(ElementAt(stArray3, Literal(2)).nullable)
+        assert(ElementAt(stArray3, Literal(-1)).nullable)
+        val stArray4 = GetArrayStructFields(inputArray2, f2, 1, 2, inputArray2ContainsNull)
+        assert(ElementAt(stArray4, Literal(1)).nullable)
+        assert(ElementAt(stArray4, Literal(-2)).nullable)
+        assert(ElementAt(stArray4, Literal(2)).nullable)
+        assert(ElementAt(stArray4, Literal(-1)).nullable)
+
+        // GetArrayStructFields case invalid indices
+        assert(!ElementAt(stArray3, Literal(0)).nullable)
+        assert(ElementAt(stArray3, Literal(4)).nullable == !ansiEnabled)
+        assert(ElementAt(stArray3, Literal(-4)).nullable == !ansiEnabled)
+
+        assert(ElementAt(stArray4, Literal(0)).nullable)
+        assert(ElementAt(stArray4, Literal(4)).nullable)
+        assert(ElementAt(stArray4, Literal(-4)).nullable)
+      }
+    }
   }
 
   test("Concat") {
@@ -1836,4 +2042,65 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
     checkEvaluation(ArrayIntersect(empty, oneNull), Seq.empty)
     checkEvaluation(ArrayIntersect(oneNull, empty), Seq.empty)
   }
+
+  test("SPARK-31980: Start and end equal in month range") {
+    checkEvaluation(new Sequence(
+      Literal(Date.valueOf("2018-01-01")),
+      Literal(Date.valueOf("2018-01-01")),
+      Literal(stringToInterval("interval 1 day"))),
+      Seq(Date.valueOf("2018-01-01")))
+    checkEvaluation(new Sequence(
+      Literal(Date.valueOf("2018-01-01")),
+      Literal(Date.valueOf("2018-01-01")),
+      Literal(stringToInterval("interval 1 month"))),
+      Seq(Date.valueOf("2018-01-01")))
+    checkEvaluation(new Sequence(
+      Literal(Date.valueOf("2018-01-01")),
+      Literal(Date.valueOf("2018-01-01")),
+      Literal(stringToInterval("interval 1 year"))),
+      Seq(Date.valueOf("2018-01-01")))
+  }
+
+  test("SPARK-33386: element_at ArrayIndexOutOfBoundsException") {
+    Seq(true, false).foreach { ansiEnabled =>
+      withSQLConf(SQLConf.ANSI_ENABLED.key -> ansiEnabled.toString) {
+        val array = Literal.create(Seq(1, 2, 3), ArrayType(IntegerType))
+        var expr: Expression = ElementAt(array, Literal(5))
+        if (ansiEnabled) {
+          val errMsg = "Invalid index: 5, numElements: 3"
+          checkExceptionInExpression[Exception](expr, errMsg)
+        } else {
+          checkEvaluation(expr, null)
+        }
+
+        expr = ElementAt(array, Literal(-5))
+        if (ansiEnabled) {
+          val errMsg = "Invalid index: -5, numElements: 3"
+          checkExceptionInExpression[Exception](expr, errMsg)
+        } else {
+          checkEvaluation(expr, null)
+        }
+
+        // SQL array indices start at 1 exception throws for both mode.
+        expr = ElementAt(array, Literal(0))
+        val errMsg = "SQL array indices start at 1"
+        checkExceptionInExpression[Exception](expr, errMsg)
+      }
+    }
+  }
+
+  test("SPARK-33460: element_at NoSuchElementException") {
+    Seq(true, false).foreach { ansiEnabled =>
+      withSQLConf(SQLConf.ANSI_ENABLED.key -> ansiEnabled.toString) {
+        val map = Literal.create(Map(1 -> "a", 2 -> "b"), MapType(IntegerType, StringType))
+        val expr: Expression = ElementAt(map, Literal(5))
+        if (ansiEnabled) {
+          val errMsg = "Key 5 does not exist."
+          checkExceptionInExpression[Exception](expr, errMsg)
+        } else {
+          checkEvaluation(expr, null)
+        }
+      }
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
index 3df7d02fb6604..57abdb4de229f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
@@ -18,9 +18,11 @@
 package org.apache.spark.sql.catalyst.expressions
 
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, UnresolvedExtractValue}
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
+import org.apache.spark.sql.catalyst.util._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
@@ -60,6 +62,46 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(GetArrayItem(nestedArray, Literal(0)), Seq(1))
   }
 
+  test("SPARK-33386: GetArrayItem ArrayIndexOutOfBoundsException") {
+    Seq(true, false).foreach { ansiEnabled =>
+      withSQLConf(SQLConf.ANSI_ENABLED.key -> ansiEnabled.toString) {
+        val array = Literal.create(Seq("a", "b"), ArrayType(StringType))
+
+        if (ansiEnabled) {
+          checkExceptionInExpression[Exception](
+            GetArrayItem(array, Literal(5)),
+            "Invalid index: 5, numElements: 2"
+          )
+
+          checkExceptionInExpression[Exception](
+            GetArrayItem(array, Literal(-1)),
+            "Invalid index: -1, numElements: 2"
+          )
+        } else {
+          checkEvaluation(GetArrayItem(array, Literal(5)), null)
+          checkEvaluation(GetArrayItem(array, Literal(-1)), null)
+        }
+      }
+    }
+  }
+
+  test("SPARK-33460: GetMapValue NoSuchElementException") {
+    Seq(true, false).foreach { ansiEnabled =>
+      withSQLConf(SQLConf.ANSI_ENABLED.key -> ansiEnabled.toString) {
+        val map = Literal.create(Map(1 -> "a", 2 -> "b"), MapType(IntegerType, StringType))
+
+        if (ansiEnabled) {
+          checkExceptionInExpression[Exception](
+            GetMapValue(map, Literal(5)),
+            "Key 5 does not exist."
+          )
+        } else {
+          checkEvaluation(GetMapValue(map, Literal(5)), null)
+        }
+      }
+    }
+  }
+
   test("SPARK-26637 handles GetArrayItem nullability correctly when input array size is constant") {
     // CreateArray case
     val a = AttributeReference("a", IntegerType, nullable = false)()
@@ -142,21 +184,51 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("GetArrayStructFields") {
-    val typeAS = ArrayType(StructType(StructField("a", IntegerType, false) :: Nil))
-    val typeNullAS = ArrayType(StructType(StructField("a", IntegerType) :: Nil))
-    val arrayStruct = Literal.create(Seq(create_row(1)), typeAS)
-    val nullArrayStruct = Literal.create(null, typeNullAS)
-
-    def getArrayStructFields(expr: Expression, fieldName: String): GetArrayStructFields = {
-      expr.dataType match {
-        case ArrayType(StructType(fields), containsNull) =>
-          val field = fields.find(_.name == fieldName).get
-          GetArrayStructFields(expr, field, fields.indexOf(field), fields.length, containsNull)
-      }
+    // test 4 types: struct field nullability X array element nullability
+    val type1 = ArrayType(StructType(StructField("a", IntegerType) :: Nil))
+    val type2 = ArrayType(StructType(StructField("a", IntegerType, nullable = false) :: Nil))
+    val type3 = ArrayType(StructType(StructField("a", IntegerType) :: Nil), containsNull = false)
+    val type4 = ArrayType(
+      StructType(StructField("a", IntegerType, nullable = false) :: Nil), containsNull = false)
+
+    val input1 = Literal.create(Seq(create_row(1)), type4)
+    val input2 = Literal.create(Seq(create_row(null)), type3)
+    val input3 = Literal.create(Seq(null), type2)
+    val input4 = Literal.create(null, type1)
+
+    def getArrayStructFields(expr: Expression, fieldName: String): Expression = {
+      ExtractValue.apply(expr, Literal.create(fieldName, StringType), _ == _)
     }
 
-    checkEvaluation(getArrayStructFields(arrayStruct, "a"), Seq(1))
-    checkEvaluation(getArrayStructFields(nullArrayStruct, "a"), null)
+    checkEvaluation(getArrayStructFields(input1, "a"), Seq(1))
+    checkEvaluation(getArrayStructFields(input2, "a"), Seq(null))
+    checkEvaluation(getArrayStructFields(input3, "a"), Seq(null))
+    checkEvaluation(getArrayStructFields(input4, "a"), null)
+  }
+
+  test("SPARK-32167: nullability of GetArrayStructFields") {
+    val resolver = SQLConf.get.resolver
+
+    val array1 = ArrayType(
+      new StructType().add("a", "int", nullable = true),
+      containsNull = false)
+    val data1 = Literal.create(Seq(Row(null)), array1)
+    val get1 = ExtractValue(data1, Literal("a"), resolver).asInstanceOf[GetArrayStructFields]
+    assert(get1.containsNull)
+
+    val array2 = ArrayType(
+      new StructType().add("a", "int", nullable = false),
+      containsNull = true)
+    val data2 = Literal.create(Seq(null), array2)
+    val get2 = ExtractValue(data2, Literal("a"), resolver).asInstanceOf[GetArrayStructFields]
+    assert(get2.containsNull)
+
+    val array3 = ArrayType(
+      new StructType().add("a", "int", nullable = false),
+      containsNull = false)
+    val data3 = Literal.create(Seq(Row(1)), array3)
+    val get3 = ExtractValue(data3, Literal("a"), resolver).asInstanceOf[GetArrayStructFields]
+    assert(!get3.containsNull)
   }
 
   test("CreateArray") {
@@ -353,14 +425,14 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {
     def checkErrorMessage(
       childDataType: DataType,
       fieldDataType: DataType,
-      errorMesage: String): Unit = {
+      errorMessage: String): Unit = {
       val e = intercept[org.apache.spark.sql.AnalysisException] {
         ExtractValue(
           Literal.create(null, childDataType),
           Literal.create(null, fieldDataType),
           _ == _)
       }
-      assert(e.getMessage().contains(errorMesage))
+      assert(e.getMessage().contains(errorMessage))
     }
 
     checkErrorMessage(structType, IntegerType, "Field name should be String Literal")
@@ -440,4 +512,18 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {
     CreateNamedStruct(Seq("a", "x", "b", 2.0)).genCode(ctx)
     assert(ctx.inlinedMutableStates.isEmpty)
   }
+
+  test("SPARK-33338: semanticEquals should handle static GetMapValue correctly") {
+    val keys = new Array[UTF8String](1)
+    val values = new Array[UTF8String](1)
+    keys(0) = UTF8String.fromString("key")
+    values(0) = UTF8String.fromString("value")
+
+    val d1 = new ArrayBasedMapData(new GenericArrayData(keys), new GenericArrayData(values))
+    val d2 = new ArrayBasedMapData(new GenericArrayData(keys), new GenericArrayData(values))
+    val m1 = GetMapValue(Literal.create(d1, MapType(StringType, StringType)), Literal("a"))
+    val m2 = GetMapValue(Literal.create(d2, MapType(StringType, StringType)), Literal("a"))
+
+    assert(m1.semanticEquals(m2))
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ConditionalExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ConditionalExpressionSuite.scala
index 87e34aca510f5..ee6f89a155ae0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ConditionalExpressionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ConditionalExpressionSuite.scala
@@ -212,8 +212,8 @@ class ConditionalExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
 
   test("case key when - internal pattern matching expects a List while apply takes a Seq") {
     val indexedSeq = IndexedSeq(Literal(1), Literal(42), Literal(42), Literal(1))
-    val caseKeyWhaen = CaseKeyWhen(Literal(12), indexedSeq)
-    assert(caseKeyWhaen.branches ==
+    val caseKeyWhen = CaseKeyWhen(Literal(12), indexedSeq)
+    assert(caseKeyWhen.branches ==
       IndexedSeq((Literal(12) === Literal(1), Literal(42)),
         (Literal(12) === Literal(42), Literal(1))))
   }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala
index 16d78b0526503..7945974a1f3dc 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala
@@ -73,7 +73,7 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
     val schema = StructType(StructField("t", TimestampType) :: Nil)
 
     val csvData1 = "2016-01-01T00:00:00.123Z"
-    var c = Calendar.getInstance(DateTimeUtils.TimeZoneGMT)
+    var c = Calendar.getInstance(DateTimeUtils.TimeZoneUTC)
     c.set(2016, 0, 1, 0, 0, 0)
     c.set(Calendar.MILLISECOND, 123)
     checkEvaluation(
@@ -158,13 +158,13 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
   }
 
   test("infer schema of CSV strings") {
-    checkEvaluation(new SchemaOfCsv(Literal.create("1,abc")), "struct<_c0:int,_c1:string>")
+    checkEvaluation(new SchemaOfCsv(Literal.create("1,abc")), "STRUCT<`_c0`: INT, `_c1`: STRING>")
   }
 
   test("infer schema of CSV strings by using options") {
     checkEvaluation(
       new SchemaOfCsv(Literal.create("1|abc"), Map("delimiter" -> "|")),
-      "struct<_c0:int,_c1:string>")
+      "STRUCT<`_c0`: INT, `_c1`: STRING>")
   }
 
   test("to_csv - struct") {
@@ -184,7 +184,7 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
 
   test("to_csv with timestamp") {
     val schema = StructType(StructField("t", TimestampType) :: Nil)
-    val c = Calendar.getInstance(DateTimeUtils.TimeZoneGMT)
+    val c = Calendar.getInstance(DateTimeUtils.TimeZoneUTC)
     c.set(2016, 0, 1, 0, 0, 0)
     c.set(Calendar.MILLISECOND, 0)
     val struct = Literal.create(create_row(c.getTimeInMillis * 1000L), schema)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index 6e8397d12da78..74865e37ad064 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -18,17 +18,22 @@
 package org.apache.spark.sql.catalyst.expressions
 
 import java.sql.{Date, Timestamp}
-import java.text.SimpleDateFormat
-import java.time.{Instant, LocalDate, ZoneId}
+import java.text.{ParseException, SimpleDateFormat}
+import java.time.{DateTimeException, Duration, Instant, LocalDate, Period, ZoneId}
+import java.time.format.DateTimeParseException
+import java.time.temporal.ChronoUnit
 import java.util.{Calendar, Locale, TimeZone}
 import java.util.concurrent.TimeUnit._
 
+import scala.reflect.ClassTag
+
 import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
 import org.apache.spark.sql.catalyst.util.{DateTimeUtils, IntervalUtils, TimestampFormatter}
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.{getZoneId, TimeZoneUTC}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
@@ -41,7 +46,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   private val JST_OPT = Option(JST.getId)
 
   def toMillis(timestamp: String): Long = {
-    val tf = TimestampFormatter("yyyy-MM-dd HH:mm:ss", UTC)
+    val tf = TimestampFormatter("yyyy-MM-dd HH:mm:ss", UTC, isParsing = true)
     DateTimeUtils.microsToMillis(tf.parse(timestamp))
   }
   val date = "2015-04-08 13:10:15"
@@ -267,7 +272,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
         // Test escaping of format
         GenerateUnsafeProjection.generate(
-          DateFormatClass(Literal(ts), Literal("\"quote"), JST_OPT) :: Nil)
+          DateFormatClass(Literal(ts), Literal("\""), JST_OPT) :: Nil)
 
         // SPARK-28072 The codegen path should work
         checkEvaluation(
@@ -517,24 +522,34 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     }
   }
 
-  test("add_months") {
-    checkEvaluation(AddMonths(Literal(Date.valueOf("2015-01-30")), Literal(1)),
+  private def testAddMonths(dataType: DataType): Unit = {
+    def addMonths(date: Literal, months: Any): AddMonthsBase = dataType match {
+      case IntegerType => AddMonths(date, Literal.create(months, dataType))
+      case YearMonthIntervalType =>
+        val period = if (months == null) null else Period.ofMonths(months.asInstanceOf[Int])
+        DateAddYMInterval(date, Literal.create(period, dataType))
+    }
+    checkEvaluation(addMonths(Literal(Date.valueOf("2015-01-30")), 1),
       DateTimeUtils.fromJavaDate(Date.valueOf("2015-02-28")))
-    checkEvaluation(AddMonths(Literal(Date.valueOf("2016-03-30")), Literal(-1)),
+    checkEvaluation(addMonths(Literal(Date.valueOf("2016-03-30")), -1),
       DateTimeUtils.fromJavaDate(Date.valueOf("2016-02-29")))
     checkEvaluation(
-      AddMonths(Literal(Date.valueOf("2015-01-30")), Literal.create(null, IntegerType)),
+      addMonths(Literal(Date.valueOf("2015-01-30")), null),
       null)
-    checkEvaluation(AddMonths(Literal.create(null, DateType), Literal(1)), null)
-    checkEvaluation(AddMonths(Literal.create(null, DateType), Literal.create(null, IntegerType)),
+    checkEvaluation(addMonths(Literal.create(null, DateType), 1), null)
+    checkEvaluation(addMonths(Literal.create(null, DateType), null),
       null)
     // Valid range of DateType is [0001-01-01, 9999-12-31]
     val maxMonthInterval = 10000 * 12
     checkEvaluation(
-      AddMonths(Literal(LocalDate.parse("0001-01-01")), Literal(maxMonthInterval)),
+      addMonths(Literal(LocalDate.parse("0001-01-01")), maxMonthInterval),
       LocalDate.of(10001, 1, 1).toEpochDay.toInt)
     checkEvaluation(
-      AddMonths(Literal(Date.valueOf("9999-12-31")), Literal(-1 * maxMonthInterval)), -719529)
+      addMonths(Literal(Date.valueOf("9999-12-31")), -1 * maxMonthInterval), -719529)
+  }
+
+  test("add_months") {
+    testAddMonths(IntegerType)
     // Test evaluation results between Interpreted mode and Codegen mode
     forAll (
       LiteralGenerator.randomGen(DateType),
@@ -544,6 +559,17 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     }
   }
 
+  test("SPARK-34721: add a year-month interval to a date") {
+    testAddMonths(YearMonthIntervalType)
+    // Test evaluation results between Interpreted mode and Codegen mode
+    forAll (
+      LiteralGenerator.randomGen(DateType),
+      LiteralGenerator.yearMonthIntervalLiteralGen
+    ) { (l1: Literal, l2: Literal) =>
+      cmpInterpretWithCodegen(EmptyRow, DateAddYMInterval(l1, l2))
+    }
+  }
+
   test("months_between") {
     val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US)
     for (zid <- outstandingZoneIds) {
@@ -636,13 +662,33 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     testNextDay("2015-07-23", "Fri", "2015-07-24")
     testNextDay("2015-07-23", "fr", "2015-07-24")
 
-    checkEvaluation(NextDay(Literal(Date.valueOf("2015-07-23")), Literal("xx")), null)
-    checkEvaluation(NextDay(Literal.create(null, DateType), Literal("xx")), null)
-    checkEvaluation(
-      NextDay(Literal(Date.valueOf("2015-07-23")), Literal.create(null, StringType)), null)
-    // Test escaping of dayOfWeek
-    GenerateUnsafeProjection.generate(
-      NextDay(Literal(Date.valueOf("2015-07-23")), Literal("\"quote")) :: Nil)
+    Seq(true, false).foreach { ansiEnabled =>
+      withSQLConf(SQLConf.ANSI_ENABLED.key -> ansiEnabled.toString) {
+        var expr: Expression = NextDay(Literal(Date.valueOf("2015-07-23")), Literal("xx"))
+        if (ansiEnabled) {
+          val errMsg = "Illegal input for day of week: xx"
+          checkExceptionInExpression[Exception](expr, errMsg)
+        } else {
+          checkEvaluation(expr, null)
+        }
+
+        expr = NextDay(Literal.create(null, DateType), Literal("xx"))
+        checkEvaluation(expr, null)
+
+        expr = NextDay(Literal(Date.valueOf("2015-07-23")), Literal.create(null, StringType))
+        checkEvaluation(expr, null)
+
+        // Test escaping of dayOfWeek
+        expr = NextDay(Literal(Date.valueOf("2015-07-23")), Literal("\"quote"))
+        GenerateUnsafeProjection.generate(expr :: Nil)
+        if (ansiEnabled) {
+          val errMsg = """Illegal input for day of week: "quote"""
+          checkExceptionInExpression[Exception](expr, errMsg)
+        } else {
+          checkEvaluation(expr, null)
+        }
+      }
+    }
   }
 
   private def testTruncDate(input: Date, fmt: String, expected: Date): Unit = {
@@ -777,8 +823,6 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
           checkEvaluation(
             FromUnixTime(Literal(1000L), Literal.create(null, StringType), timeZoneId),
             null)
-          checkEvaluation(
-            FromUnixTime(Literal(0L), Literal("not a valid format"), timeZoneId), null)
 
           // SPARK-28072 The codegen path for non-literal input should also work
           checkEvaluation(
@@ -792,7 +836,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
       }
     }
     // Test escaping of format
-    GenerateUnsafeProjection.generate(FromUnixTime(Literal(0L), Literal("\"quote")) :: Nil)
+    GenerateUnsafeProjection.generate(FromUnixTime(Literal(0L), Literal("\""), UTC_OPT) :: Nil)
   }
 
   test("unix_timestamp") {
@@ -803,7 +847,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
         val sdf2 = new SimpleDateFormat(fmt2, Locale.US)
         val fmt3 = "yy-MM-dd"
         val sdf3 = new SimpleDateFormat(fmt3, Locale.US)
-        sdf3.setTimeZone(TimeZone.getTimeZone(UTC))
+        sdf3.setTimeZone(TimeZoneUTC)
 
         withDefaultTimeZone(UTC) {
           for (zid <- outstandingZoneIds) {
@@ -854,15 +898,13 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
               UnixTimestamp(Literal(date1), Literal.create(null, StringType), timeZoneId),
               MICROSECONDS.toSeconds(
                 DateTimeUtils.daysToMicros(DateTimeUtils.fromJavaDate(date1), tz.toZoneId)))
-            checkEvaluation(
-              UnixTimestamp(Literal("2015-07-24"), Literal("not a valid format"), timeZoneId), null)
           }
         }
       }
     }
     // Test escaping of format
     GenerateUnsafeProjection.generate(
-      UnixTimestamp(Literal("2015-07-24"), Literal("\"quote")) :: Nil)
+      UnixTimestamp(Literal("2015-07-24"), Literal("\""), UTC_OPT) :: Nil)
   }
 
   test("to_unix_timestamp") {
@@ -874,7 +916,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
         val sdf2 = new SimpleDateFormat(fmt2, Locale.US)
         val fmt3 = "yy-MM-dd"
         val sdf3 = new SimpleDateFormat(fmt3, Locale.US)
-        sdf3.setTimeZone(TimeZone.getTimeZone(UTC))
+        sdf3.setTimeZone(TimeZoneUTC)
 
         withDefaultTimeZone(UTC) {
           for (zid <- outstandingZoneIds) {
@@ -920,10 +962,6 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
               Literal(date1), Literal.create(null, StringType), timeZoneId),
               MICROSECONDS.toSeconds(
                 DateTimeUtils.daysToMicros(DateTimeUtils.fromJavaDate(date1), zid)))
-            checkEvaluation(
-              ToUnixTimestamp(
-                Literal("2015-07-24"),
-                Literal("not a valid format"), timeZoneId), null)
 
             // SPARK-28072 The codegen path for non-literal input should also work
             checkEvaluation(
@@ -940,7 +978,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     }
     // Test escaping of format
     GenerateUnsafeProjection.generate(
-      ToUnixTimestamp(Literal("2015-07-24"), Literal("\"quote")) :: Nil)
+      ToUnixTimestamp(Literal("2015-07-24"), Literal("\""), UTC_OPT) :: Nil)
   }
 
   test("datediff") {
@@ -1018,49 +1056,97 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("creating values of DateType via make_date") {
-    checkEvaluation(MakeDate(Literal(2013), Literal(7), Literal(15)), Date.valueOf("2013-7-15"))
-    checkEvaluation(MakeDate(Literal.create(null, IntegerType), Literal(7), Literal(15)), null)
-    checkEvaluation(MakeDate(Literal(2019), Literal.create(null, IntegerType), Literal(19)), null)
-    checkEvaluation(MakeDate(Literal(2019), Literal(7), Literal.create(null, IntegerType)), null)
-    checkEvaluation(MakeDate(Literal(Int.MaxValue), Literal(13), Literal(19)), null)
-    checkEvaluation(MakeDate(Literal(2019), Literal(13), Literal(19)), null)
-    checkEvaluation(MakeDate(Literal(2019), Literal(7), Literal(32)), null)
+    Seq(true, false).foreach({ ansi =>
+      withSQLConf(SQLConf.ANSI_ENABLED.key -> ansi.toString) {
+        checkEvaluation(MakeDate(Literal(2013), Literal(7), Literal(15)), Date.valueOf("2013-7-15"))
+        checkEvaluation(MakeDate(Literal.create(null, IntegerType), Literal(7), Literal(15)), null)
+        checkEvaluation(MakeDate(Literal(2019), Literal.create(null, IntegerType), Literal(19)),
+          null)
+        checkEvaluation(MakeDate(Literal(2019), Literal(7), Literal.create(null, IntegerType)),
+          null)
+      }
+    })
+
+    // ansi test
+    withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
+      checkExceptionInExpression[DateTimeException](MakeDate(Literal(Int.MaxValue), Literal(13),
+        Literal(19)), EmptyRow, "Invalid value for Year")
+      checkExceptionInExpression[DateTimeException](MakeDate(Literal(2019),
+        Literal(13), Literal(19)), EmptyRow, "Invalid value for Month")
+      checkExceptionInExpression[DateTimeException](MakeDate(Literal(2019), Literal(7),
+        Literal(32)), EmptyRow, "Invalid value for Day")
+    }
+
+    // non-ansi test
+    withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
+      checkEvaluation(MakeDate(Literal(Int.MaxValue), Literal(13), Literal(19)), null)
+      checkEvaluation(MakeDate(Literal(2019), Literal(13), Literal(19)), null)
+      checkEvaluation(MakeDate(Literal(2019), Literal(7), Literal(32)), null)
+    }
   }
 
   test("creating values of TimestampType via make_timestamp") {
-    var makeTimestampExpr = MakeTimestamp(
-      Literal(2013), Literal(7), Literal(15), Literal(8), Literal(15),
-      Literal(Decimal(BigDecimal(23.5), 8, 6)), Some(Literal(ZoneId.systemDefault().getId)))
     val expected = Timestamp.valueOf("2013-7-15 8:15:23.5")
-    checkEvaluation(makeTimestampExpr, expected)
-    checkEvaluation(makeTimestampExpr.copy(timezone = None), expected)
 
-    checkEvaluation(makeTimestampExpr.copy(year = Literal.create(null, IntegerType)), null)
-    checkEvaluation(makeTimestampExpr.copy(year = Literal(Int.MaxValue)), null)
-
-    checkEvaluation(makeTimestampExpr.copy(month = Literal.create(null, IntegerType)), null)
-    checkEvaluation(makeTimestampExpr.copy(month = Literal(13)), null)
-
-    checkEvaluation(makeTimestampExpr.copy(day = Literal.create(null, IntegerType)), null)
-    checkEvaluation(makeTimestampExpr.copy(day = Literal(32)), null)
-
-    checkEvaluation(makeTimestampExpr.copy(hour = Literal.create(null, IntegerType)), null)
-    checkEvaluation(makeTimestampExpr.copy(hour = Literal(25)), null)
+    Seq(true, false).foreach { ansi =>
+      withSQLConf(SQLConf.ANSI_ENABLED.key -> ansi.toString) {
+        var makeTimestampExpr = MakeTimestamp(
+          Literal(2013), Literal(7), Literal(15), Literal(8), Literal(15),
+          Literal(Decimal(BigDecimal(23.5), 8, 6)), Some(Literal(ZoneId.systemDefault().getId)))
+        checkEvaluation(makeTimestampExpr, expected)
+        checkEvaluation(makeTimestampExpr.copy(year = Literal.create(null, IntegerType)), null)
+        checkEvaluation(makeTimestampExpr.copy(month = Literal.create(null, IntegerType)), null)
+        checkEvaluation(makeTimestampExpr.copy(day = Literal.create(null, IntegerType)), null)
+        checkEvaluation(makeTimestampExpr.copy(hour = Literal.create(null, IntegerType)), null)
+        checkEvaluation(makeTimestampExpr.copy(min = Literal.create(null, IntegerType)), null)
+        checkEvaluation(makeTimestampExpr.copy(sec = Literal.create(null, DecimalType(8, 6))), null)
+        checkEvaluation(makeTimestampExpr.copy(timezone = None), expected)
+
+        Seq(
+          (makeTimestampExpr.copy(year = Literal(Int.MaxValue)), "Invalid value for Year"),
+          (makeTimestampExpr.copy(month = Literal(13)), "Invalid value for Month"),
+          (makeTimestampExpr.copy(day = Literal(32)), "Invalid value for Day"),
+          (makeTimestampExpr.copy(hour = Literal(25)), "Invalid value for Hour"),
+          (makeTimestampExpr.copy(min = Literal(65)), "Invalid value for Min"),
+          (makeTimestampExpr.copy(sec = Literal(Decimal(
+            BigDecimal(70.0), 8, 6))), "Invalid value for Second")
+        ).foreach { entry =>
+          if (ansi) {
+            checkExceptionInExpression[DateTimeException](entry._1, EmptyRow, entry._2)
+          } else {
+            checkEvaluation(entry._1, null)
+          }
+        }
 
-    checkEvaluation(makeTimestampExpr.copy(min = Literal.create(null, IntegerType)), null)
-    checkEvaluation(makeTimestampExpr.copy(min = Literal(65)), null)
+        makeTimestampExpr = MakeTimestamp(Literal(2019), Literal(6), Literal(30),
+          Literal(23), Literal(59), Literal(Decimal(BigDecimal(60.0), 8, 6)))
+        if (ansi) {
+          checkExceptionInExpression[DateTimeException](makeTimestampExpr.copy(sec = Literal(
+            Decimal(BigDecimal(60.5), 8, 6))), EmptyRow, "The fraction of sec must be zero")
+        } else {
+          checkEvaluation(makeTimestampExpr, Timestamp.valueOf("2019-07-01 00:00:00"))
+        }
 
-    checkEvaluation(makeTimestampExpr.copy(sec = Literal.create(null, DecimalType(8, 6))), null)
-    checkEvaluation(makeTimestampExpr.copy(sec = Literal(Decimal(BigDecimal(70.0), 8, 6))), null)
+        makeTimestampExpr = MakeTimestamp(Literal(2019), Literal(8), Literal(12), Literal(0),
+          Literal(0), Literal(Decimal(BigDecimal(58.000001), 8, 6)))
+        checkEvaluation(makeTimestampExpr, Timestamp.valueOf("2019-08-12 00:00:58.000001"))
+      }
+    }
 
-    makeTimestampExpr = MakeTimestamp(Literal(2019), Literal(6), Literal(30),
-      Literal(23), Literal(59), Literal(Decimal(BigDecimal(60.0), 8, 6)))
-    checkEvaluation(makeTimestampExpr, Timestamp.valueOf("2019-07-01 00:00:00"))
-    checkEvaluation(makeTimestampExpr.copy(sec = Literal(Decimal(BigDecimal(60.5), 8, 6))), null)
+    // non-ansi test
+    withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
+      val makeTimestampExpr = MakeTimestamp(Literal(2019), Literal(6), Literal(30),
+        Literal(23), Literal(59), Literal(Decimal(BigDecimal(60.0), 8, 6)))
+      checkEvaluation(makeTimestampExpr.copy(sec = Literal(Decimal(BigDecimal(60.5), 8, 6))), null)
+    }
 
-    makeTimestampExpr = MakeTimestamp(Literal(2019), Literal(8), Literal(12),
-      Literal(0), Literal(0), Literal(Decimal(BigDecimal(58.000001), 8, 6)))
-    checkEvaluation(makeTimestampExpr, Timestamp.valueOf("2019-08-12 00:00:58.000001"))
+    Seq(true, false).foreach { ansi =>
+      withSQLConf(SQLConf.ANSI_ENABLED.key -> ansi.toString) {
+        val makeTimestampExpr = MakeTimestamp(Literal(2019), Literal(8), Literal(12),
+          Literal(0), Literal(0), Literal(Decimal(BigDecimal(58.000001), 8, 6)))
+        checkEvaluation(makeTimestampExpr, Timestamp.valueOf("2019-08-12 00:00:58.000001"))
+      }
+    }
   }
 
   test("ISO 8601 week-numbering year") {
@@ -1088,42 +1174,112 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     }
   }
 
-  test("timestamps difference") {
+  test("SPARK-34903: timestamps difference") {
     val end = Instant.parse("2019-10-04T11:04:01.123456Z")
-    checkEvaluation(SubtractTimestamps(Literal(end), Literal(end)),
-      new CalendarInterval(0, 0, 0))
-    checkEvaluation(SubtractTimestamps(Literal(end), Literal(Instant.EPOCH)),
-      IntervalUtils.stringToInterval(UTF8String.fromString("interval " +
-        "436163 hours 4 minutes 1 seconds 123 milliseconds 456 microseconds")))
-    checkEvaluation(SubtractTimestamps(Literal(Instant.EPOCH), Literal(end)),
-      IntervalUtils.stringToInterval(UTF8String.fromString("interval " +
-        "-436163 hours -4 minutes -1 seconds -123 milliseconds -456 microseconds")))
-    checkEvaluation(
-      SubtractTimestamps(
-        Literal(Instant.parse("9999-12-31T23:59:59.999999Z")),
-        Literal(Instant.parse("0001-01-01T00:00:00Z"))),
-      IntervalUtils.stringToInterval(UTF8String.fromString("interval " +
-        "87649415 hours 59 minutes 59 seconds 999 milliseconds 999 microseconds")))
+    outstandingTimezonesIds.foreach { tz =>
+      def sub(left: Instant, right: Instant): Expression = {
+        SubtractTimestamps(
+          Literal(left),
+          Literal(right),
+          legacyInterval = true,
+          timeZoneId = Some(tz))
+      }
+      checkEvaluation(sub(end, end), new CalendarInterval(0, 0, 0))
+      checkEvaluation(sub(end, Instant.EPOCH),
+        IntervalUtils.stringToInterval(UTF8String.fromString("interval " +
+          "436163 hours 4 minutes 1 seconds 123 milliseconds 456 microseconds")))
+      checkEvaluation(sub(Instant.EPOCH, end),
+        IntervalUtils.stringToInterval(UTF8String.fromString("interval " +
+          "-436163 hours -4 minutes -1 seconds -123 milliseconds -456 microseconds")))
+      checkEvaluation(
+        sub(
+          Instant.parse("9999-12-31T23:59:59.999999Z"),
+          Instant.parse("0001-01-01T00:00:00Z")),
+        IntervalUtils.stringToInterval(UTF8String.fromString("interval " +
+          "87649415 hours 59 minutes 59 seconds 999 milliseconds 999 microseconds")))
+    }
+
+    outstandingTimezonesIds.foreach { tz =>
+      def check(left: Instant, right: Instant): Unit = {
+        checkEvaluation(
+          SubtractTimestamps(
+            Literal(left),
+            Literal(right),
+            legacyInterval = false,
+            timeZoneId = Some(tz)),
+          Duration.between(
+            right.atZone(getZoneId(tz)).toLocalDateTime,
+            left.atZone(getZoneId(tz)).toLocalDateTime))
+      }
+
+      check(end, end)
+      check(end, Instant.EPOCH)
+      check(Instant.EPOCH, end)
+      check(Instant.parse("9999-12-31T23:59:59.999999Z"), Instant.parse("0001-01-01T00:00:00Z"))
+
+      val errMsg = intercept[ArithmeticException] {
+        checkEvaluation(
+          SubtractTimestamps(
+            Literal(Instant.MIN),
+            Literal(Instant.MAX),
+            legacyInterval = false,
+            timeZoneId = Some(tz)),
+          Duration.ZERO)
+      }.getMessage
+      assert(errMsg.contains("overflow"))
+
+      Seq(false, true).foreach { legacy =>
+        checkConsistencyBetweenInterpretedAndCodegen(
+          (end: Expression, start: Expression) => SubtractTimestamps(end, start, legacy, Some(tz)),
+          TimestampType, TimestampType)
+      }
+    }
   }
 
-  test("subtract dates") {
+  test("SPARK-34896: subtract dates") {
     val end = LocalDate.of(2019, 10, 5)
-    checkEvaluation(SubtractDates(Literal(end), Literal(end)),
-      new CalendarInterval(0, 0, 0))
-    checkEvaluation(SubtractDates(Literal(end.plusDays(1)), Literal(end)),
-      IntervalUtils.stringToInterval(UTF8String.fromString("interval 1 days")))
-    checkEvaluation(SubtractDates(Literal(end.minusDays(1)), Literal(end)),
-      IntervalUtils.stringToInterval(UTF8String.fromString("interval -1 days")))
     val epochDate = Literal(LocalDate.ofEpochDay(0))
-    checkEvaluation(SubtractDates(Literal(end), epochDate),
-      IntervalUtils.stringToInterval(UTF8String.fromString("interval 49 years 9 months 4 days")))
-    checkEvaluation(SubtractDates(epochDate, Literal(end)),
-      IntervalUtils.stringToInterval(UTF8String.fromString("interval -49 years -9 months -4 days")))
-    checkEvaluation(
-      SubtractDates(
-        Literal(LocalDate.of(10000, 1, 1)),
-        Literal(LocalDate.of(1, 1, 1))),
-      IntervalUtils.stringToInterval(UTF8String.fromString("interval 9999 years")))
+
+    withSQLConf(SQLConf.LEGACY_INTERVAL_ENABLED.key -> "true") {
+      checkEvaluation(SubtractDates(Literal(end), Literal(end)),
+        new CalendarInterval(0, 0, 0))
+      checkEvaluation(SubtractDates(Literal(end.plusDays(1)), Literal(end)),
+        IntervalUtils.stringToInterval(UTF8String.fromString("interval 1 days")))
+      checkEvaluation(SubtractDates(Literal(end.minusDays(1)), Literal(end)),
+        IntervalUtils.stringToInterval(UTF8String.fromString("interval -1 days")))
+      checkEvaluation(SubtractDates(Literal(end), epochDate),
+        IntervalUtils.stringToInterval(UTF8String.fromString("interval 49 years 9 months 4 days")))
+      checkEvaluation(SubtractDates(epochDate, Literal(end)),
+        IntervalUtils.stringToInterval(
+          UTF8String.fromString("interval -49 years -9 months -4 days")))
+      checkEvaluation(
+        SubtractDates(
+          Literal(LocalDate.of(10000, 1, 1)),
+          Literal(LocalDate.of(1, 1, 1))),
+        IntervalUtils.stringToInterval(UTF8String.fromString("interval 9999 years")))
+    }
+
+    withSQLConf(SQLConf.LEGACY_INTERVAL_ENABLED.key -> "false") {
+      checkEvaluation(SubtractDates(Literal(end), Literal(end)), Duration.ZERO)
+      checkEvaluation(SubtractDates(Literal(end.plusDays(1)), Literal(end)), Duration.ofDays(1))
+      checkEvaluation(SubtractDates(Literal(end.minusDays(1)), Literal(end)), Duration.ofDays(-1))
+      checkEvaluation(SubtractDates(Literal(end), epochDate), Duration.ofDays(end.toEpochDay))
+      checkEvaluation(SubtractDates(epochDate, Literal(end)),
+        Duration.ofDays(end.toEpochDay).negated())
+      checkEvaluation(
+        SubtractDates(
+          Literal(LocalDate.of(10000, 1, 1)),
+          Literal(LocalDate.of(1, 1, 1))),
+        Duration.ofDays(ChronoUnit.DAYS.between( LocalDate.of(1, 1, 1), LocalDate.of(10000, 1, 1))))
+      checkExceptionInExpression[ArithmeticException](
+        SubtractDates(Literal(LocalDate.MAX), Literal(LocalDate.MIN)),
+        "overflow")
+    }
+    Seq(false, true).foreach { ansiIntervals =>
+      checkConsistencyBetweenInterpretedAndCodegen(
+        (end: Expression, start: Expression) => SubtractDates(end, start, ansiIntervals),
+        DateType, DateType)
+    }
   }
 
   test("to_timestamp exception mode") {
@@ -1146,4 +1302,366 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
           Literal("yyyy-MM-dd'T'HH:mm:ss.SSSz")), "Fail to parse")
     }
   }
+
+  test("Consistent error handling for datetime formatting and parsing functions") {
+
+    def checkException[T <: Exception : ClassTag](c: String): Unit = {
+      checkExceptionInExpression[T](new ParseToTimestamp(Literal("1"), Literal(c)).child, c)
+      checkExceptionInExpression[T](new ParseToDate(Literal("1"), Literal(c)).child, c)
+      checkExceptionInExpression[T](ToUnixTimestamp(Literal("1"), Literal(c)), c)
+      checkExceptionInExpression[T](UnixTimestamp(Literal("1"), Literal(c)), c)
+      if (!Set("E", "F", "q", "Q").contains(c)) {
+        checkExceptionInExpression[T](DateFormatClass(CurrentTimestamp(), Literal(c)), c)
+        checkExceptionInExpression[T](FromUnixTime(Literal(0L), Literal(c)), c)
+      }
+    }
+
+    Seq('Y', 'W', 'w', 'E', 'u', 'F').foreach { l =>
+      checkException[SparkUpgradeException](l.toString)
+    }
+
+    Seq('q', 'Q', 'e', 'c', 'A', 'n', 'N', 'p').foreach { l =>
+      checkException[IllegalArgumentException](l.toString)
+    }
+  }
+
+  test("SPARK-31896: Handle am-pm timestamp parsing when hour is missing") {
+    checkEvaluation(
+      new ParseToTimestamp(Literal("PM"), Literal("a")).child,
+      Timestamp.valueOf("1970-01-01 12:00:00.0"))
+    checkEvaluation(
+      new ParseToTimestamp(Literal("11:11 PM"), Literal("mm:ss a")).child,
+      Timestamp.valueOf("1970-01-01 12:11:11.0"))
+  }
+
+  def testIntegralInput(testFunc: Number => Unit): Unit = {
+    def checkResult(input: Long): Unit = {
+      if (input.toByte == input) {
+        testFunc(input.toByte)
+      } else if (input.toShort == input) {
+        testFunc(input.toShort)
+      } else if (input.toInt == input) {
+        testFunc(input.toInt)
+      } else {
+        testFunc(input)
+      }
+    }
+    checkResult(0)
+    checkResult(Byte.MaxValue)
+    checkResult(Byte.MinValue)
+    checkResult(Short.MaxValue)
+    checkResult(Short.MinValue)
+    checkResult(Int.MaxValue)
+    checkResult(Int.MinValue)
+    checkResult(Int.MaxValue.toLong + 100)
+    checkResult(Int.MinValue.toLong - 100)
+  }
+
+  test("DATE_FROM_UNIX_DATE") {
+    def testIntegralFunc(value: Number): Unit = {
+      checkEvaluation(
+        DateFromUnixDate(Literal(value.intValue())),
+        LocalDate.ofEpochDay(value.intValue()))
+    }
+    // test null input
+    checkEvaluation(DateFromUnixDate(Literal(null, IntegerType)), null)
+    // test integral input
+    testIntegralInput(testIntegralFunc)
+  }
+
+  test("UNIX_DATE") {
+    def testIntegralFunc(value: Number): Unit = {
+      checkEvaluation(
+        UnixDate(Literal(LocalDate.ofEpochDay(value.intValue()))),
+        value.intValue())
+    }
+    // test null input
+    checkEvaluation(UnixDate(Literal(null, DateType)), null)
+    // test various inputs
+    testIntegralInput(testIntegralFunc)
+  }
+
+  test("UNIX_SECONDS") {
+    checkEvaluation(UnixSeconds(Literal(null, TimestampType)), null)
+    var timestamp = Literal(new Timestamp(0L))
+    checkEvaluation(UnixSeconds(timestamp), 0L)
+    timestamp = Literal(new Timestamp(1000L))
+    checkEvaluation(UnixSeconds(timestamp), 1L)
+    timestamp = Literal(new Timestamp(-1000L))
+    checkEvaluation(UnixSeconds(timestamp), -1L)
+    // -1ms is considered to be in -1st second, as 0-999ms is in 0th second.
+    timestamp = Literal(new Timestamp(-1L))
+    checkEvaluation(UnixSeconds(timestamp), -1L)
+    timestamp = Literal(new Timestamp(-1000L))
+    checkEvaluation(UnixSeconds(timestamp), -1L)
+    // Truncates higher levels of precision
+    timestamp = Literal(new Timestamp(1999L))
+    checkEvaluation(UnixSeconds(timestamp), 1L)
+  }
+
+  test("UNIX_MILLIS") {
+    checkEvaluation(UnixMillis(Literal(null, TimestampType)), null)
+    var timestamp = Literal(new Timestamp(0L))
+    checkEvaluation(UnixMillis(timestamp), 0L)
+    timestamp = Literal(new Timestamp(1000L))
+    checkEvaluation(UnixMillis(timestamp), 1000L)
+    timestamp = Literal(new Timestamp(-1000L))
+    checkEvaluation(UnixMillis(timestamp), -1000L)
+    // Truncates higher levels of precision
+    val timestampWithNanos = new Timestamp(1000L)
+    timestampWithNanos.setNanos(999999)
+    checkEvaluation(UnixMillis(Literal(timestampWithNanos)), 1000L)
+  }
+
+  test("UNIX_MICROS") {
+    checkEvaluation(UnixMicros(Literal(null, TimestampType)), null)
+    var timestamp = Literal(new Timestamp(0L))
+    checkEvaluation(UnixMicros(timestamp), 0L)
+    timestamp = Literal(new Timestamp(1000L))
+    checkEvaluation(UnixMicros(timestamp), 1000000L)
+    timestamp = Literal(new Timestamp(-1000L))
+    checkEvaluation(UnixMicros(timestamp), -1000000L)
+    val timestampWithNanos = new Timestamp(1000L)
+    timestampWithNanos.setNanos(1000) // 1 microsecond
+    checkEvaluation(UnixMicros(Literal(timestampWithNanos)), 1000001L)
+  }
+
+  test("TIMESTAMP_SECONDS") {
+    def testIntegralFunc(value: Number): Unit = {
+      checkEvaluation(
+        SecondsToTimestamp(Literal(value)),
+        Instant.ofEpochSecond(value.longValue()))
+    }
+
+    // test null input
+    checkEvaluation(
+      SecondsToTimestamp(Literal(null, IntegerType)),
+      null)
+
+    // test integral input
+    testIntegralInput(testIntegralFunc)
+    // test overflow
+    checkExceptionInExpression[ArithmeticException](
+      SecondsToTimestamp(Literal(Long.MaxValue, LongType)), EmptyRow, "long overflow")
+
+    def testFractionalInput(input: String): Unit = {
+      Seq(input.toFloat, input.toDouble, Decimal(input)).foreach { value =>
+        checkEvaluation(
+          SecondsToTimestamp(Literal(value)),
+          (input.toDouble * MICROS_PER_SECOND).toLong)
+      }
+    }
+
+    testFractionalInput("1.0")
+    testFractionalInput("-1.0")
+    testFractionalInput("1.234567")
+    testFractionalInput("-1.234567")
+
+    // test overflow for decimal input
+    checkExceptionInExpression[ArithmeticException](
+      SecondsToTimestamp(Literal(Decimal("9" * 38))), "Overflow"
+    )
+    // test truncation error for decimal input
+    checkExceptionInExpression[ArithmeticException](
+      SecondsToTimestamp(Literal(Decimal("0.1234567"))), "Rounding necessary"
+    )
+
+    // test NaN
+    checkEvaluation(
+      SecondsToTimestamp(Literal(Double.NaN)),
+      null)
+    checkEvaluation(
+      SecondsToTimestamp(Literal(Float.NaN)),
+      null)
+    // double input can truncate
+    checkEvaluation(
+      SecondsToTimestamp(Literal(123.456789123)),
+      Instant.ofEpochSecond(123, 456789000))
+    checkEvaluation(SecondsToTimestamp(Literal(16777215.0f)), Instant.ofEpochSecond(16777215))
+  }
+
+  test("TIMESTAMP_MILLIS") {
+    def testIntegralFunc(value: Number): Unit = {
+      checkEvaluation(
+        MillisToTimestamp(Literal(value)),
+        Instant.ofEpochMilli(value.longValue()))
+    }
+
+    // test null input
+    checkEvaluation(
+      MillisToTimestamp(Literal(null, IntegerType)),
+      null)
+
+    // test integral input
+    testIntegralInput(testIntegralFunc)
+    // test overflow
+    checkExceptionInExpression[ArithmeticException](
+      MillisToTimestamp(Literal(Long.MaxValue, LongType)), EmptyRow, "long overflow")
+  }
+
+  test("TIMESTAMP_MICROS") {
+    def testIntegralFunc(value: Number): Unit = {
+      checkEvaluation(
+        MicrosToTimestamp(Literal(value)),
+        value.longValue())
+    }
+
+    // test null input
+    checkEvaluation(
+      MicrosToTimestamp(Literal(null, IntegerType)),
+      null)
+
+    // test integral input
+    testIntegralInput(testIntegralFunc)
+    // test max/min input
+    testIntegralFunc(Long.MaxValue)
+    testIntegralFunc(Long.MinValue)
+  }
+
+  test("SPARK-33498: GetTimestamp,UnixTimestamp,ToUnixTimestamp with parseError") {
+    Seq(true, false).foreach { ansiEnabled =>
+      Seq("LEGACY", "CORRECTED", "EXCEPTION").foreach { policy =>
+        withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> policy,
+          SQLConf.ANSI_ENABLED.key -> ansiEnabled.toString) {
+
+          val exprSeq = Seq[Expression](
+            GetTimestamp(Literal("2020-01-27T20:06:11.847"), Literal("yyyy-MM-dd HH:mm:ss.SSS")),
+            GetTimestamp(Literal("Unparseable"), Literal("yyyy-MM-dd HH:mm:ss.SSS")),
+            UnixTimestamp(Literal("2020-01-27T20:06:11.847"), Literal("yyyy-MM-dd HH:mm:ss.SSS")),
+            UnixTimestamp(Literal("Unparseable"), Literal("yyyy-MM-dd HH:mm:ss.SSS")),
+            ToUnixTimestamp(Literal("2020-01-27T20:06:11.847"), Literal("yyyy-MM-dd HH:mm:ss.SSS")),
+            ToUnixTimestamp(Literal("Unparseable"), Literal("yyyy-MM-dd HH:mm:ss.SSS"))
+          )
+
+          if (!ansiEnabled) {
+            exprSeq.foreach(checkEvaluation(_, null))
+          } else if (policy == "LEGACY") {
+            exprSeq.foreach(checkExceptionInExpression[ParseException](_, "Unparseable"))
+          } else {
+            exprSeq.foreach(
+              checkExceptionInExpression[DateTimeParseException](_, "could not be parsed"))
+          }
+
+          // LEGACY works, CORRECTED failed, EXCEPTION with SparkUpgradeException
+          val exprSeq2 = Seq[(Expression, Long)](
+            (GetTimestamp(Literal("2020-01-27T20:06:11.847!!!"),
+              Literal("yyyy-MM-dd'T'HH:mm:ss.SSS")), 1580184371847000L),
+            (UnixTimestamp(Literal("2020-01-27T20:06:11.847!!!"),
+              Literal("yyyy-MM-dd'T'HH:mm:ss.SSS")), 1580184371L),
+            (ToUnixTimestamp(Literal("2020-01-27T20:06:11.847!!!"),
+              Literal("yyyy-MM-dd'T'HH:mm:ss.SSS")), 1580184371L)
+          )
+
+          if (policy == "LEGACY") {
+            exprSeq2.foreach(pair => checkEvaluation(pair._1, pair._2))
+          } else if (policy == "EXCEPTION") {
+            exprSeq2.foreach(pair =>
+              checkExceptionInExpression[SparkUpgradeException](
+                pair._1,
+                  "You may get a different result due to the upgrading of Spark 3.0"))
+          } else {
+            if (ansiEnabled) {
+              exprSeq2.foreach(pair =>
+                checkExceptionInExpression[DateTimeParseException](pair._1, "could not be parsed"))
+            } else {
+              exprSeq2.foreach(pair => checkEvaluation(pair._1, null))
+            }
+          }
+        }
+      }
+    }
+  }
+
+  test("SPARK-34739: add a year-month interval to a timestamp") {
+    val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US)
+    for (zid <- outstandingZoneIds) {
+      val timeZoneId = Option(zid.getId)
+      sdf.setTimeZone(TimeZone.getTimeZone(zid))
+
+      checkEvaluation(
+        TimestampAddYMInterval(
+          Literal(new Timestamp(sdf.parse("2016-01-29 10:11:12.123").getTime)),
+          Literal(Period.ofMonths(2)),
+          timeZoneId),
+        DateTimeUtils.fromJavaTimestamp(
+          new Timestamp(sdf.parse("2016-03-29 10:11:12.123").getTime)))
+
+      checkEvaluation(
+        TimestampAddYMInterval(
+          Literal.create(null, TimestampType),
+          Literal(Period.ofMonths(1)),
+          timeZoneId),
+        null)
+      checkEvaluation(
+        TimestampAddYMInterval(
+          Literal(new Timestamp(sdf.parse("2016-01-29 10:00:00.000").getTime)),
+          Literal.create(null, YearMonthIntervalType),
+          timeZoneId),
+        null)
+      checkEvaluation(
+        TimestampAddYMInterval(
+          Literal.create(null, TimestampType),
+          Literal.create(null, YearMonthIntervalType),
+          timeZoneId),
+        null)
+      checkConsistencyBetweenInterpretedAndCodegen(
+        (ts: Expression, interval: Expression) => TimestampAddYMInterval(ts, interval, timeZoneId),
+        TimestampType, YearMonthIntervalType)
+    }
+  }
+
+  test("SPARK-34761: add a day-time interval to a timestamp") {
+    val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US)
+    for (zid <- outstandingZoneIds) {
+      val timeZoneId = Option(zid.getId)
+      sdf.setTimeZone(TimeZone.getTimeZone(zid))
+      checkEvaluation(
+        TimeAdd(
+          Literal(new Timestamp(sdf.parse("2021-01-01 00:00:00.123").getTime)),
+          Literal(Duration.ofDays(10).plusMinutes(10).plusMillis(321)),
+          timeZoneId),
+        DateTimeUtils.fromJavaTimestamp(
+          new Timestamp(sdf.parse("2021-01-11 00:10:00.444").getTime)))
+      checkEvaluation(
+        TimeAdd(
+          Literal(new Timestamp(sdf.parse("2021-01-01 00:10:00.123").getTime)),
+          Literal(Duration.ofDays(-10).minusMinutes(9).minusMillis(120)),
+          timeZoneId),
+        DateTimeUtils.fromJavaTimestamp(
+          new Timestamp(sdf.parse("2020-12-22 00:01:00.003").getTime)))
+
+      val e = intercept[Exception] {
+        checkEvaluation(
+          TimeAdd(
+            Literal(new Timestamp(sdf.parse("2021-01-01 00:00:00.123").getTime)),
+            Literal(Duration.of(Long.MaxValue, ChronoUnit.MICROS)),
+            timeZoneId),
+          null)
+      }.getCause
+      assert(e.isInstanceOf[ArithmeticException])
+      assert(e.getMessage.contains("long overflow"))
+
+      checkEvaluation(
+        TimeAdd(
+          Literal.create(null, TimestampType),
+          Literal(Duration.ofDays(1)),
+          timeZoneId),
+        null)
+      checkEvaluation(
+        TimeAdd(
+          Literal(new Timestamp(sdf.parse("2021-01-01 00:00:00.123").getTime)),
+          Literal.create(null, DayTimeIntervalType),
+          timeZoneId),
+        null)
+      checkEvaluation(
+        TimeAdd(
+          Literal.create(null, TimestampType),
+          Literal.create(null, DayTimeIntervalType),
+          timeZoneId),
+        null)
+      checkConsistencyBetweenInterpretedAndCodegen(
+        (ts: Expression, interval: Expression) => TimeAdd(ts, interval, timeZoneId),
+        TimestampType, DayTimeIntervalType)
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala
index 6f73c1b0c04fb..26d98157807cd 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala
@@ -36,10 +36,14 @@ import org.apache.spark.sql.catalyst.plans.logical.{OneRowRelation, Project}
 import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, MapData}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
-import org.apache.spark.util.Utils
 
 /**
  * A few helper functions for expression evaluation testing. Mixin this trait to use them.
+ *
+ * Note: when you write unit test for an expression and call `checkEvaluation` to check the result,
+ *       please make sure that you explore all the cases that can lead to null result (including
+ *       null in struct fields, array elements and map values). The framework will test the
+ *       nullability flag of the expression automatically.
  */
 trait ExpressionEvalHelper extends ScalaCheckDrivenPropertyChecks with PlanTestBase {
   self: SparkFunSuite =>
@@ -69,7 +73,7 @@ trait ExpressionEvalHelper extends ScalaCheckDrivenPropertyChecks with PlanTestB
 
   private def prepareEvaluation(expression: Expression): Expression = {
     val serializer = new JavaSerializer(new SparkConf()).newInstance
-    val resolver = ResolveTimeZone(new SQLConf)
+    val resolver = ResolveTimeZone
     val expr = resolver.resolveTimeZones(expression)
     assert(expr.resolved)
     serializer.deserialize(serializer.serialize(expr))
@@ -155,11 +159,20 @@ trait ExpressionEvalHelper extends ScalaCheckDrivenPropertyChecks with PlanTestB
       expectedErrMsg: String): Unit = {
 
     def checkException(eval: => Unit, testMode: String): Unit = {
+      val modes = Seq(CodegenObjectFactoryMode.CODEGEN_ONLY, CodegenObjectFactoryMode.NO_CODEGEN)
       withClue(s"($testMode)") {
         val errMsg = intercept[T] {
-          eval
+          for (fallbackMode <- modes) {
+            withSQLConf(SQLConf.CODEGEN_FACTORY_MODE.key -> fallbackMode.toString) {
+              eval
+            }
+          }
         }.getMessage
-        if (!errMsg.contains(expectedErrMsg)) {
+        if (errMsg == null) {
+          if (expectedErrMsg != null) {
+            fail(s"Expected null error message, but `$errMsg` found")
+          }
+        } else if (!errMsg.contains(expectedErrMsg)) {
           fail(s"Expected error message is `$expectedErrMsg`, but `$errMsg` found")
         }
       }
@@ -183,22 +196,6 @@ trait ExpressionEvalHelper extends ScalaCheckDrivenPropertyChecks with PlanTestB
     expression.eval(inputRow)
   }
 
-  protected def generateProject(
-      generator: => Projection,
-      expression: Expression): Projection = {
-    try {
-      generator
-    } catch {
-      case e: Throwable =>
-        fail(
-          s"""
-            |Code generation of $expression failed:
-            |$e
-            |${Utils.exceptionString(e)}
-          """.stripMargin)
-    }
-  }
-
   protected def checkEvaluationWithoutCodegen(
       expression: Expression,
       expected: Any,
@@ -235,9 +232,7 @@ trait ExpressionEvalHelper extends ScalaCheckDrivenPropertyChecks with PlanTestB
   protected def evaluateWithMutableProjection(
       expression: => Expression,
       inputRow: InternalRow = EmptyRow): Any = {
-    val plan = generateProject(
-      MutableProjection.create(Alias(expression, s"Optimized($expression)")() :: Nil),
-      expression)
+    val plan = MutableProjection.create(Alias(expression, s"Optimized($expression)")() :: Nil)
     plan.initialize(0)
 
     plan(inputRow).get(0, expression.dataType)
@@ -283,11 +278,9 @@ trait ExpressionEvalHelper extends ScalaCheckDrivenPropertyChecks with PlanTestB
     // SPARK-16489 Explicitly doing code generation twice so code gen will fail if
     // some expression is reusing variable names across different instances.
     // This behavior is tested in ExpressionEvalHelperSuite.
-    val plan = generateProject(
-      UnsafeProjection.create(
-        Alias(expression, s"Optimized($expression)1")() ::
-          Alias(expression, s"Optimized($expression)2")() :: Nil),
-      expression)
+    val plan = UnsafeProjection.create(
+      Alias(expression, s"Optimized($expression)1")() ::
+        Alias(expression, s"Optimized($expression)2")() :: Nil)
 
     plan.initialize(0)
     plan(inputRow)
@@ -310,16 +303,13 @@ trait ExpressionEvalHelper extends ScalaCheckDrivenPropertyChecks with PlanTestB
     checkEvaluationWithMutableProjection(expression, expected)
     checkEvaluationWithOptimization(expression, expected)
 
-    var plan = generateProject(
-      GenerateMutableProjection.generate(Alias(expression, s"Optimized($expression)")() :: Nil),
-      expression)
+    var plan: Projection =
+      GenerateMutableProjection.generate(Alias(expression, s"Optimized($expression)")() :: Nil)
     plan.initialize(0)
     var actual = plan(inputRow).get(0, expression.dataType)
     assert(checkResult(actual, expected, expression))
 
-    plan = generateProject(
-      GenerateUnsafeProjection.generate(Alias(expression, s"Optimized($expression)")() :: Nil),
-      expression)
+    plan = GenerateUnsafeProjection.generate(Alias(expression, s"Optimized($expression)")() :: Nil)
     plan.initialize(0)
     val ref = new BoundReference(0, expression.dataType, nullable = true)
     actual = GenerateSafeProjection.generate(ref :: Nil)(plan(inputRow)).get(0, expression.dataType)
@@ -340,6 +330,21 @@ trait ExpressionEvalHelper extends ScalaCheckDrivenPropertyChecks with PlanTestB
     }
   }
 
+  /**
+   * Test evaluation results between Interpreted mode and Codegen mode, making sure we have
+   * consistent result regardless of the evaluation method we use. If an exception is thrown,
+   * it checks that both modes throw the same exception.
+   *
+   * This method test against unary expressions by feeding them arbitrary literals of `dataType`.
+   */
+  def checkConsistencyBetweenInterpretedAndCodegenAllowingException(
+      c: Expression => Expression,
+      dataType: DataType): Unit = {
+    forAll (LiteralGenerator.randomGen(dataType)) { (l: Literal) =>
+      cmpInterpretWithCodegen(EmptyRow, c(l), true)
+    }
+  }
+
   /**
    * Test evaluation results between Interpreted mode and Codegen mode, making sure we have
    * consistent result regardless of the evaluation method we use.
@@ -432,9 +437,7 @@ trait ExpressionEvalHelper extends ScalaCheckDrivenPropertyChecks with PlanTestB
       }
     }
 
-    val plan = generateProject(
-      GenerateMutableProjection.generate(Alias(expr, s"Optimized($expr)")() :: Nil),
-      expr)
+    val plan = GenerateMutableProjection.generate(Alias(expr, s"Optimized($expr)")() :: Nil)
     val (codegen, codegenExc) = try {
       (Some(plan(inputRow).get(0, expr.dataType)), None)
     } catch {
@@ -448,7 +451,7 @@ trait ExpressionEvalHelper extends ScalaCheckDrivenPropertyChecks with PlanTestB
     if (interpret.isDefined && codegen.isDefined && !compareResults(interpret.get, codegen.get)) {
       fail(s"Incorrect evaluation: $expr, interpret: ${interpret.get}, codegen: ${codegen.get}")
     } else if (interpretExc.isDefined && codegenExc.isEmpty) {
-      fail(s"Incorrect evaluation: $expr, interpet threw exception ${interpretExc.get}")
+      fail(s"Incorrect evaluation: $expr, interpret threw exception ${interpretExc.get}")
     } else if (interpretExc.isEmpty && codegenExc.isDefined) {
       fail(s"Incorrect evaluation: $expr, codegen threw exception ${codegenExc.get}")
     } else if (interpretExc.isDefined && codegenExc.isDefined
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelperSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelperSuite.scala
index 54ef9641bee0d..3cc50da38906e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelperSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelperSuite.scala
@@ -32,8 +32,8 @@ import org.apache.spark.sql.types.{DataType, IntegerType, MapType}
  */
 class ExpressionEvalHelperSuite extends SparkFunSuite with ExpressionEvalHelper {
 
-  test("SPARK-16489 checkEvaluation should fail if expression reuses variable names") {
-    val e = intercept[RuntimeException] { checkEvaluation(BadCodegenExpression(), 10) }
+  test("SPARK-16489: checkEvaluation should fail if expression reuses variable names") {
+    val e = intercept[Exception] { checkEvaluation(BadCodegenExpression(), 10) }
     assert(e.getMessage.contains("some_variable"))
   }
 
@@ -43,6 +43,12 @@ class ExpressionEvalHelperSuite extends SparkFunSuite with ExpressionEvalHelper
     }
     assert(e.getMessage.contains("and exprNullable was"))
   }
+
+  test("SPARK-33619: make sure checkExceptionInExpression work as expected") {
+    checkExceptionInExpression[Exception](
+      BadCodegenAndEvalExpression(),
+      "Cannot determine simple type name \"NoSuchElementException\"")
+  }
 }
 
 /**
@@ -76,3 +82,18 @@ case class MapIncorrectDataTypeExpression() extends LeafExpression with CodegenF
   // since values includes null, valueContainsNull must be true
   override def dataType: DataType = MapType(IntegerType, IntegerType, valueContainsNull = false)
 }
+
+case class BadCodegenAndEvalExpression() extends LeafExpression {
+  override def nullable: Boolean = false
+  override def eval(input: InternalRow): Any =
+    throw new Exception("Cannot determine simple type name \"NoSuchElementException\"")
+  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
+    // it should be java.util.NoSuchElementException in generated code.
+    ev.copy(code =
+      code"""
+            |int ${ev.value} = 10;
+            |throw new NoSuchElementException("compile failed!");
+      """.stripMargin)
+  }
+  override def dataType: DataType = IntegerType
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionSQLBuilderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionSQLBuilderSuite.scala
index c12dd3051d271..a58765d57a511 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionSQLBuilderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionSQLBuilderSuite.scala
@@ -94,45 +94,45 @@ class ExpressionSQLBuilderSuite extends SparkFunSuite {
   }
 
   test("attributes") {
-    checkSQL('a.int, "`a`")
+    checkSQL('a.int, "a")
     checkSQL(Symbol("foo bar").int, "`foo bar`")
     // Keyword
-    checkSQL('int.int, "`int`")
+    checkSQL('int.int, "int")
   }
 
   test("binary comparisons") {
-    checkSQL('a.int === 'b.int, "(`a` = `b`)")
-    checkSQL('a.int <=> 'b.int, "(`a` <=> `b`)")
-    checkSQL('a.int =!= 'b.int, "(NOT (`a` = `b`))")
+    checkSQL('a.int === 'b.int, "(a = b)")
+    checkSQL('a.int <=> 'b.int, "(a <=> b)")
+    checkSQL('a.int =!= 'b.int, "(NOT (a = b))")
 
-    checkSQL('a.int < 'b.int, "(`a` < `b`)")
-    checkSQL('a.int <= 'b.int, "(`a` <= `b`)")
-    checkSQL('a.int > 'b.int, "(`a` > `b`)")
-    checkSQL('a.int >= 'b.int, "(`a` >= `b`)")
+    checkSQL('a.int < 'b.int, "(a < b)")
+    checkSQL('a.int <= 'b.int, "(a <= b)")
+    checkSQL('a.int > 'b.int, "(a > b)")
+    checkSQL('a.int >= 'b.int, "(a >= b)")
 
-    checkSQL('a.int in ('b.int, 'c.int), "(`a` IN (`b`, `c`))")
-    checkSQL('a.int in (1, 2), "(`a` IN (1, 2))")
+    checkSQL('a.int in ('b.int, 'c.int), "(a IN (b, c))")
+    checkSQL('a.int in (1, 2), "(a IN (1, 2))")
 
-    checkSQL('a.int.isNull, "(`a` IS NULL)")
-    checkSQL('a.int.isNotNull, "(`a` IS NOT NULL)")
+    checkSQL('a.int.isNull, "(a IS NULL)")
+    checkSQL('a.int.isNotNull, "(a IS NOT NULL)")
   }
 
   test("logical operators") {
-    checkSQL('a.boolean && 'b.boolean, "(`a` AND `b`)")
-    checkSQL('a.boolean || 'b.boolean, "(`a` OR `b`)")
-    checkSQL(!'a.boolean, "(NOT `a`)")
-    checkSQL(If('a.boolean, 'b.int, 'c.int), "(IF(`a`, `b`, `c`))")
+    checkSQL('a.boolean && 'b.boolean, "(a AND b)")
+    checkSQL('a.boolean || 'b.boolean, "(a OR b)")
+    checkSQL(!'a.boolean, "(NOT a)")
+    checkSQL(If('a.boolean, 'b.int, 'c.int), "(IF(a, b, c))")
   }
 
   test("arithmetic expressions") {
-    checkSQL('a.int + 'b.int, "(`a` + `b`)")
-    checkSQL('a.int - 'b.int, "(`a` - `b`)")
-    checkSQL('a.int * 'b.int, "(`a` * `b`)")
-    checkSQL('a.int / 'b.int, "(`a` / `b`)")
-    checkSQL('a.int % 'b.int, "(`a` % `b`)")
-
-    checkSQL(-'a.int, "(- `a`)")
-    checkSQL(-('a.int + 'b.int), "(- (`a` + `b`))")
+    checkSQL('a.int + 'b.int, "(a + b)")
+    checkSQL('a.int - 'b.int, "(a - b)")
+    checkSQL('a.int * 'b.int, "(a * b)")
+    checkSQL('a.int / 'b.int, "(a / b)")
+    checkSQL('a.int % 'b.int, "(a % b)")
+
+    checkSQL(-'a.int, "(- a)")
+    checkSQL(-('a.int + 'b.int), "(- (a + b))")
   }
 
   test("window specification") {
@@ -140,27 +140,27 @@ class ExpressionSQLBuilderSuite extends SparkFunSuite {
 
     checkSQL(
       WindowSpecDefinition('a.int :: Nil, Nil, frame),
-      s"(PARTITION BY `a` ${frame.sql})"
+      s"(PARTITION BY a ${frame.sql})"
     )
 
     checkSQL(
       WindowSpecDefinition('a.int :: 'b.string :: Nil, Nil, frame),
-      s"(PARTITION BY `a`, `b` ${frame.sql})"
+      s"(PARTITION BY a, b ${frame.sql})"
     )
 
     checkSQL(
       WindowSpecDefinition(Nil, 'a.int.asc :: Nil, frame),
-      s"(ORDER BY `a` ASC NULLS FIRST ${frame.sql})"
+      s"(ORDER BY a ASC NULLS FIRST ${frame.sql})"
     )
 
     checkSQL(
       WindowSpecDefinition(Nil, 'a.int.asc :: 'b.string.desc :: Nil, frame),
-      s"(ORDER BY `a` ASC NULLS FIRST, `b` DESC NULLS LAST ${frame.sql})"
+      s"(ORDER BY a ASC NULLS FIRST, b DESC NULLS LAST ${frame.sql})"
     )
 
     checkSQL(
       WindowSpecDefinition('a.int :: 'b.string :: Nil, 'c.int.asc :: 'd.string.desc :: Nil, frame),
-      s"(PARTITION BY `a`, `b` ORDER BY `c` ASC NULLS FIRST, `d` DESC NULLS LAST ${frame.sql})"
+      s"(PARTITION BY a, b ORDER BY c ASC NULLS FIRST, d DESC NULLS LAST ${frame.sql})"
     )
   }
 
@@ -169,17 +169,17 @@ class ExpressionSQLBuilderSuite extends SparkFunSuite {
 
     checkSQL(
       TimeAdd('a, interval),
-      "`a` + INTERVAL '1 hours'"
+      "a + INTERVAL '1 hours'"
     )
 
     checkSQL(
       DatetimeSub('a, interval, Literal.default(TimestampType)),
-      "`a` - INTERVAL '1 hours'"
+      "a - INTERVAL '1 hours'"
     )
 
     checkSQL(
       DateAddInterval('a, interval),
-      "`a` + INTERVAL '1 hours'"
+      "a + INTERVAL '1 hours'"
     )
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExtractPredicatesWithinOutputSetSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExtractPredicatesWithinOutputSetSuite.scala
new file mode 100644
index 0000000000000..ed141ef923e0a
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExtractPredicatesWithinOutputSetSuite.scala
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.types.BooleanType
+
+class ExtractPredicatesWithinOutputSetSuite
+  extends SparkFunSuite
+  with PredicateHelper
+  with PlanTest {
+  private val a = AttributeReference("A", BooleanType)(exprId = ExprId(1))
+  private val b = AttributeReference("B", BooleanType)(exprId = ExprId(2))
+  private val c = AttributeReference("C", BooleanType)(exprId = ExprId(3))
+  private val d = AttributeReference("D", BooleanType)(exprId = ExprId(4))
+  private val e = AttributeReference("E", BooleanType)(exprId = ExprId(5))
+  private val f = AttributeReference("F", BooleanType)(exprId = ExprId(6))
+  private val g = AttributeReference("G", BooleanType)(exprId = ExprId(7))
+  private val h = AttributeReference("H", BooleanType)(exprId = ExprId(8))
+  private val i = AttributeReference("I", BooleanType)(exprId = ExprId(9))
+
+  private def checkCondition(
+      input: Expression,
+      convertibleAttributes: Seq[Attribute],
+      expected: Option[Expression]): Unit = {
+    val result = extractPredicatesWithinOutputSet(input, AttributeSet(convertibleAttributes))
+    if (expected.isEmpty) {
+      assert(result.isEmpty)
+    } else {
+      assert(result.isDefined && result.get.semanticEquals(expected.get))
+    }
+  }
+
+  test("Convertible conjunctive predicates") {
+    checkCondition(a && b, Seq(a, b), Some(a && b))
+    checkCondition(a && b, Seq(a), Some(a))
+    checkCondition(a && b, Seq(b), Some(b))
+    checkCondition(a && b && c, Seq(a, c), Some(a && c))
+    checkCondition(a && b && c && d, Seq(b, c), Some(b && c))
+  }
+
+  test("Convertible disjunctive predicates") {
+    checkCondition(a || b, Seq(a, b), Some(a || b))
+    checkCondition(a || b, Seq(a), None)
+    checkCondition(a ||  b, Seq(b), None)
+    checkCondition(a || b || c, Seq(a, c), None)
+    checkCondition(a || b || c || d, Seq(a, b, d), None)
+    checkCondition(a || b || c || d, Seq(d, c, b, a), Some(a || b || c || d))
+  }
+
+  test("Convertible complex predicates") {
+    checkCondition((a && b) || (c && d), Seq(a, c), Some(a || c))
+    checkCondition((a && b) || (c && d), Seq(a, b), None)
+    checkCondition((a && b) || (c && d), Seq(a, c, d), Some(a || (c && d)))
+    checkCondition((a && b && c) || (d && e && f), Seq(a, c, d, f), Some((a && c) || (d && f)))
+    checkCondition((a && b) || (c && d) || (e && f) || (g && h), Seq(a, c, e, g),
+      Some(a || c || e || g))
+    checkCondition((a && b) || (c && d) || (e && f) || (g && h), Seq(a, e, g), None)
+    checkCondition((a || b) || (c && d) || (e && f) || (g && h), Seq(a, c, e, g), None)
+    checkCondition((a || b) || (c && d) || (e && f) || (g && h), Seq(a, b, c, e, g),
+      Some(a || b || c || e || g))
+    checkCondition((a && b && c) || (d && e && f) || (g && h && i), Seq(b, e, h), Some(b || e || h))
+    checkCondition((a && b && c) || (d && e && f) || (g && h && i), Seq(b, e, d), None)
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala
index af6e5a3f35ee1..bd981d1633aa6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.catalyst.expressions
 
 import java.nio.charset.StandardCharsets
-import java.time.{ZoneId, ZoneOffset}
+import java.time.{Duration, Period, ZoneId, ZoneOffset}
 
 import scala.collection.mutable.ArrayBuffer
 import scala.language.implicitConversions
@@ -697,6 +697,28 @@ class HashExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(XxHash64(Seq(literal), 100L), XxHash64(Seq(literal), 100).eval())
   }
 
+  test("SPARK-35113: HashExpression support DayTimeIntervalType/YearMonthIntervalType") {
+    val dayTime = Literal.create(Duration.ofSeconds(1237123123), DayTimeIntervalType)
+    val yearMonth = Literal.create(Period.ofMonths(1234), YearMonthIntervalType)
+    checkEvaluation(Murmur3Hash(Seq(dayTime), 10), -428664612)
+    checkEvaluation(Murmur3Hash(Seq(yearMonth), 10), -686520021)
+    checkEvaluation(XxHash64(Seq(dayTime), 10), 8228802290839366895L)
+    checkEvaluation(XxHash64(Seq(yearMonth), 10), -1774215319882784110L)
+    checkEvaluation(HiveHash(Seq(dayTime)), 743331816)
+    checkEvaluation(HiveHash(Seq(yearMonth)), 1234)
+  }
+
+  test("SPARK-35207: Compute hash consistent between -0.0 and 0.0") {
+    def checkResult(exprs1: Expression, exprs2: Expression): Unit = {
+      checkEvaluation(Murmur3Hash(Seq(exprs1), 42), Murmur3Hash(Seq(exprs2), 42).eval())
+      checkEvaluation(XxHash64(Seq(exprs1), 42), XxHash64(Seq(exprs2), 42).eval())
+      checkEvaluation(HiveHash(Seq(exprs1)), HiveHash(Seq(exprs2)).eval())
+    }
+
+    checkResult(Literal.create(-0D, DoubleType), Literal.create(0D, DoubleType))
+    checkResult(Literal.create(-0F, FloatType), Literal.create(0F, FloatType))
+  }
+
   private def testHash(inputSchema: StructType): Unit = {
     val inputGenerator = RandomDataGenerator.forType(inputSchema, nullable = false).get
     val toRow = RowEncoder(inputSchema).createSerializer()
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
index 8c972a9917043..cf2f5057cc40c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
@@ -17,14 +17,18 @@
 
 package org.apache.spark.sql.catalyst.expressions
 
+import java.time.{Duration, Period}
+import java.time.temporal.ChronoUnit
+
 import scala.language.implicitConversions
 
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, IntervalUtils}
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
-import org.apache.spark.sql.catalyst.util.DateTimeTestUtils
 import org.apache.spark.sql.catalyst.util.IntervalUtils.{safeStringToInterval, stringToInterval}
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.Decimal
+import org.apache.spark.sql.types.{DayTimeIntervalType, Decimal, DecimalType, YearMonthIntervalType}
+import org.apache.spark.sql.types.DataTypeTestUtils.numericTypes
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 
 class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
@@ -72,17 +76,17 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("hours") {
-    checkEvaluation(ExtractIntervalHours("0 hours"), 0L)
-    checkEvaluation(ExtractIntervalHours("1 hour"), 1L)
-    checkEvaluation(ExtractIntervalHours("-1 hour"), -1L)
-    checkEvaluation(ExtractIntervalHours("23 hours"), 23L)
-    checkEvaluation(ExtractIntervalHours("-23 hours"), -23L)
+    checkEvaluation(ExtractIntervalHours("0 hours"), 0.toByte)
+    checkEvaluation(ExtractIntervalHours("1 hour"), 1.toByte)
+    checkEvaluation(ExtractIntervalHours("-1 hour"), -1.toByte)
+    checkEvaluation(ExtractIntervalHours("23 hours"), 23.toByte)
+    checkEvaluation(ExtractIntervalHours("-23 hours"), -23.toByte)
     // Years, months and days must not be taken into account
-    checkEvaluation(ExtractIntervalHours("100 year 10 months 10 days 10 hours"), 10L)
+    checkEvaluation(ExtractIntervalHours("100 year 10 months 10 days 10 hours"), 10.toByte)
     // Minutes should be taken into account
-    checkEvaluation(ExtractIntervalHours("10 hours 100 minutes"), 11L)
-    checkEvaluation(ExtractIntervalHours(largeInterval), 11L)
-    checkEvaluation(ExtractIntervalHours("25 hours"), 1L)
+    checkEvaluation(ExtractIntervalHours("10 hours 100 minutes"), 11.toByte)
+    checkEvaluation(ExtractIntervalHours(largeInterval), 11.toByte)
+    checkEvaluation(ExtractIntervalHours("25 hours"), 1.toByte)
 
   }
 
@@ -190,7 +194,8 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
         micros: Int = 0): Unit = {
       val secFrac = DateTimeTestUtils.secFrac(seconds, millis, micros)
       val intervalExpr = MakeInterval(Literal(years), Literal(months), Literal(weeks),
-        Literal(days), Literal(hours), Literal(minutes), Literal(Decimal(secFrac, 8, 6)))
+        Literal(days), Literal(hours), Literal(minutes),
+        Literal(Decimal(secFrac, Decimal.MAX_LONG_DIGITS, 6)))
       val totalMonths = years * MONTHS_PER_YEAR + months
       val totalDays = weeks * DAYS_PER_WEEK + days
       val totalMicros = secFrac + minutes * MICROS_PER_MINUTE + hours * MICROS_PER_HOUR
@@ -206,5 +211,239 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     check(years = 10000, micros = -1)
     check(-9999, -11, 0, -31, -23, -59, -59, -999, -999)
     check(years = -10000, micros = 1)
+    check(
+      hours = Int.MaxValue,
+      minutes = Int.MaxValue,
+      seconds = Int.MaxValue,
+      millis = Int.MaxValue,
+      micros = Int.MaxValue)
+  }
+
+  test("ANSI mode: make interval") {
+    def check(
+        years: Int = 0,
+        months: Int = 0,
+        weeks: Int = 0,
+        days: Int = 0,
+        hours: Int = 0,
+        minutes: Int = 0,
+        seconds: Int = 0,
+        millis: Int = 0,
+        micros: Int = 0): Unit = {
+      val secFrac = DateTimeTestUtils.secFrac(seconds, millis, micros)
+      val intervalExpr = MakeInterval(Literal(years), Literal(months), Literal(weeks),
+        Literal(days), Literal(hours), Literal(minutes),
+        Literal(Decimal(secFrac, Decimal.MAX_LONG_DIGITS, 6)))
+      val totalMonths = years * MONTHS_PER_YEAR + months
+      val totalDays = weeks * DAYS_PER_WEEK + days
+      val totalMicros = secFrac + minutes * MICROS_PER_MINUTE + hours * MICROS_PER_HOUR
+      val expected = new CalendarInterval(totalMonths, totalDays, totalMicros)
+      checkEvaluation(intervalExpr, expected)
+    }
+
+    def checkException(
+        years: Int = 0,
+        months: Int = 0,
+        weeks: Int = 0,
+        days: Int = 0,
+        hours: Int = 0,
+        minutes: Int = 0,
+        seconds: Int = 0,
+        millis: Int = 0,
+        micros: Int = 0): Unit = {
+      val secFrac = DateTimeTestUtils.secFrac(seconds, millis, micros)
+      val intervalExpr = MakeInterval(Literal(years), Literal(months), Literal(weeks),
+        Literal(days), Literal(hours), Literal(minutes),
+        Literal(Decimal(secFrac, Decimal.MAX_LONG_DIGITS, 6)))
+      checkExceptionInExpression[ArithmeticException](intervalExpr, EmptyRow, "")
+    }
+
+    withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
+      check(months = 0, days = 0, micros = 0)
+      check(years = -123)
+      check(weeks = 123)
+      check(millis = -123)
+      check(9999, 11, 0, 31, 23, 59, 59, 999, 999)
+      check(years = 10000, micros = -1)
+      check(-9999, -11, 0, -31, -23, -59, -59, -999, -999)
+      check(years = -10000, micros = 1)
+      check(
+        hours = Int.MaxValue,
+        minutes = Int.MaxValue,
+        seconds = Int.MaxValue,
+        millis = Int.MaxValue,
+        micros = Int.MaxValue)
+
+      checkException(years = Int.MaxValue)
+      checkException(weeks = Int.MaxValue)
+    }
+  }
+
+  test("SPARK-34824: multiply year-month interval by numeric") {
+    Seq(
+      (Period.ofYears(-123), Literal(null, DecimalType.USER_DEFAULT)) -> null,
+      (Period.ofMonths(0), 10) -> Period.ofMonths(0),
+      (Period.ofMonths(10), 0L) -> Period.ofMonths(0),
+      (Period.ofYears(100), -1.toByte) -> Period.ofYears(-100),
+      (Period.ofMonths(12), 0.3f) -> Period.ofMonths(4),
+      (Period.ofYears(-1000), 0.3d) -> Period.ofYears(-300),
+      (Period.ofYears(9999), 0.0001d) -> Period.ofYears(1),
+      (Period.ofYears(9999), BigDecimal(0.0001)) -> Period.ofYears(1)
+    ).foreach { case ((period, num), expected) =>
+      checkEvaluation(MultiplyYMInterval(Literal(period), Literal(num)), expected)
+    }
+
+    Seq(
+      (Period.ofMonths(2), Int.MaxValue) -> "overflow",
+      (Period.ofMonths(Int.MinValue), 10d) -> "not in range",
+      (Period.ofMonths(-100), Float.NaN) -> "input is infinite or NaN",
+      (Period.ofMonths(200), Double.PositiveInfinity) -> "input is infinite or NaN",
+      (Period.ofMonths(-200), Float.NegativeInfinity) -> "input is infinite or NaN"
+    ).foreach { case ((period, num), expectedErrMsg) =>
+      checkExceptionInExpression[ArithmeticException](
+        MultiplyYMInterval(Literal(period), Literal(num)),
+        expectedErrMsg)
+    }
+
+    numericTypes.foreach { numType =>
+      checkConsistencyBetweenInterpretedAndCodegenAllowingException(
+        (interval: Expression, num: Expression) => MultiplyYMInterval(interval, num),
+        YearMonthIntervalType, numType)
+    }
+  }
+
+  test("SPARK-34850: multiply day-time interval by numeric") {
+    Seq(
+      (Duration.ofHours(-123), Literal(null, DecimalType.USER_DEFAULT)) -> null,
+      (Duration.ofMinutes(0), 10) -> Duration.ofMinutes(0),
+      (Duration.ofSeconds(10), 0L) -> Duration.ofSeconds(0),
+      (Duration.ofMillis(100), -1.toByte) -> Duration.ofMillis(-100),
+      (Duration.ofDays(12), 0.3d) -> Duration.ofDays(12).multipliedBy(3).dividedBy(10),
+      (Duration.of(-1000, ChronoUnit.MICROS), 0.3f) -> Duration.of(-300, ChronoUnit.MICROS),
+      (Duration.ofDays(9999), 0.0001d) -> Duration.ofDays(9999).dividedBy(10000),
+      (Duration.ofDays(9999), BigDecimal(0.0001)) -> Duration.ofDays(9999).dividedBy(10000)
+    ).foreach { case ((duration, num), expected) =>
+      checkEvaluation(MultiplyDTInterval(Literal(duration), Literal(num)), expected)
+    }
+
+    Seq(
+      (Duration.ofDays(-100), Float.NaN) -> "input is infinite or NaN",
+      (Duration.ofDays(2), Int.MaxValue) -> "overflow",
+      (Duration.ofHours(Int.MinValue), Short.MinValue) -> "overflow",
+      (Duration.ofDays(10), BigDecimal(Long.MinValue)) -> "Overflow",
+      (Duration.ofDays(200), Double.PositiveInfinity) -> "input is infinite or NaN",
+      (Duration.ofDays(-200), Float.NegativeInfinity) -> "input is infinite or NaN"
+    ).foreach { case ((duration, num), expectedErrMsg) =>
+      checkExceptionInExpression[ArithmeticException](
+        MultiplyDTInterval(Literal(duration), Literal(num)), expectedErrMsg)
+    }
+
+    numericTypes.foreach { numType =>
+      checkConsistencyBetweenInterpretedAndCodegenAllowingException(
+        (interval: Expression, num: Expression) => MultiplyDTInterval(interval, num),
+        DayTimeIntervalType, numType)
+    }
+  }
+
+  test("SPARK-34868: divide year-month interval by numeric") {
+    Seq(
+      (Period.ofYears(-123), Literal(null, DecimalType.USER_DEFAULT)) -> null,
+      (Period.ofMonths(0), 10) -> Period.ofMonths(0),
+      (Period.ofMonths(200), Double.PositiveInfinity) -> Period.ofMonths(0),
+      (Period.ofMonths(-200), Float.NegativeInfinity) -> Period.ofMonths(0),
+      (Period.ofYears(100), -1.toByte) -> Period.ofYears(-100),
+      (Period.ofYears(1), 2.toShort) -> Period.ofMonths(6),
+      (Period.ofYears(-1), -3) -> Period.ofMonths(4),
+      (Period.ofMonths(-1000), 0.5f) -> Period.ofMonths(-2000),
+      (Period.ofYears(1000), 100d) -> Period.ofYears(10),
+      (Period.ofMonths(2), BigDecimal(0.1)) -> Period.ofMonths(20)
+    ).foreach { case ((period, num), expected) =>
+      checkEvaluation(DivideYMInterval(Literal(period), Literal(num)), expected)
+    }
+
+    Seq(
+      (Period.ofMonths(1), 0) -> "/ by zero",
+      (Period.ofMonths(Int.MinValue), 0d) -> "input is infinite or NaN",
+      (Period.ofMonths(-100), Float.NaN) -> "input is infinite or NaN"
+    ).foreach { case ((period, num), expectedErrMsg) =>
+      checkExceptionInExpression[ArithmeticException](
+        DivideYMInterval(Literal(period), Literal(num)),
+        expectedErrMsg)
+    }
+
+    numericTypes.foreach { numType =>
+      checkConsistencyBetweenInterpretedAndCodegenAllowingException(
+        (interval: Expression, num: Expression) => DivideYMInterval(interval, num),
+        YearMonthIntervalType, numType)
+    }
+  }
+
+  test("SPARK-34875: divide day-time interval by numeric") {
+    Seq(
+      (Duration.ofDays(-123), Literal(null, DecimalType.USER_DEFAULT)) -> null,
+      (Duration.ZERO, 10) -> Duration.ZERO,
+      (Duration.ofMillis(200), Double.PositiveInfinity) -> Duration.ZERO,
+      (Duration.ofSeconds(-200), Float.NegativeInfinity) -> Duration.ZERO,
+      (Duration.ofMinutes(100), -1.toByte) -> Duration.ofMinutes(-100),
+      (Duration.ofHours(1), 2.toShort) -> Duration.ofMinutes(30),
+      (Duration.ofDays(-1), -3) -> Duration.ofHours(8),
+      (Duration.of(-1000, ChronoUnit.MICROS), 0.5f) ->Duration.of(-2000, ChronoUnit.MICROS),
+      (Duration.ofDays(10080), 100d) -> Duration.ofDays(10080).dividedBy(100),
+      (Duration.ofMillis(2), BigDecimal(-0.1)) -> Duration.ofMillis(-20)
+    ).foreach { case ((period, num), expected) =>
+      checkEvaluation(DivideDTInterval(Literal(period), Literal(num)), expected)
+    }
+
+    Seq(
+      (Duration.ofDays(1), 0) -> "/ by zero",
+      (Duration.ofMillis(Int.MinValue), 0d) -> "input is infinite or NaN",
+      (Duration.ofSeconds(-100), Float.NaN) -> "input is infinite or NaN"
+    ).foreach { case ((period, num), expectedErrMsg) =>
+      checkExceptionInExpression[ArithmeticException](
+        DivideDTInterval(Literal(period), Literal(num)),
+        expectedErrMsg)
+    }
+
+    numericTypes.foreach { numType =>
+      checkConsistencyBetweenInterpretedAndCodegenAllowingException(
+        (interval: Expression, num: Expression) => DivideDTInterval(interval, num),
+        DayTimeIntervalType, numType)
+    }
+  }
+
+  test("ANSI: extract years and months") {
+    Seq(Period.ZERO,
+      Period.ofMonths(100),
+      Period.ofMonths(-100),
+      Period.ofYears(100),
+      Period.ofYears(-100)).foreach { p =>
+      checkEvaluation(ExtractANSIIntervalYears(Literal(p)),
+        IntervalUtils.getYears(p.toTotalMonths.toInt))
+      checkEvaluation(ExtractANSIIntervalMonths(Literal(p)),
+        IntervalUtils.getMonths(p.toTotalMonths.toInt))
+    }
+    checkEvaluation(ExtractANSIIntervalYears(Literal(null, YearMonthIntervalType)), null)
+    checkEvaluation(ExtractANSIIntervalMonths(Literal(null, YearMonthIntervalType)), null)
+  }
+
+  test("ANSI: extract days, hours, minutes and seconds") {
+    Seq(Duration.ZERO,
+      Duration.ofMillis(1L * MILLIS_PER_DAY + 2 * MILLIS_PER_SECOND),
+      Duration.ofMillis(-1L * MILLIS_PER_DAY + 2 * MILLIS_PER_SECOND),
+      Duration.ofDays(100),
+      Duration.ofDays(-100),
+      Duration.ofHours(-100)).foreach { d =>
+
+      checkEvaluation(ExtractANSIIntervalDays(Literal(d)), d.toDays.toInt)
+      checkEvaluation(ExtractANSIIntervalHours(Literal(d)), (d.toHours % HOURS_PER_DAY).toByte)
+      checkEvaluation(ExtractANSIIntervalMinutes(Literal(d)),
+        (d.toMinutes % MINUTES_PER_HOUR).toByte)
+      checkEvaluation(ExtractANSIIntervalSeconds(Literal(d)),
+        IntervalUtils.getSeconds(IntervalUtils.durationToMicros(d)))
+    }
+    checkEvaluation(ExtractANSIIntervalDays(Literal(null, DayTimeIntervalType)), null)
+    checkEvaluation(ExtractANSIIntervalHours(Literal(null, DayTimeIntervalType)), null)
+    checkEvaluation(ExtractANSIIntervalMinutes(Literal(null, DayTimeIntervalType)), null)
+    checkEvaluation(ExtractANSIIntervalSeconds(Literal(null, DayTimeIntervalType)), null)
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
index 6f062dcc9a1ce..b3666936e5855 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
@@ -735,17 +735,17 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
 
   test("SPARK-24709: infer schema of json strings") {
     checkEvaluation(new SchemaOfJson(Literal.create("""{"col":0}""")),
-      "struct<col:bigint>")
+      "STRUCT<`col`: BIGINT>")
     checkEvaluation(
       new SchemaOfJson(Literal.create("""{"col0":["a"], "col1": {"col2": "b"}}""")),
-      "struct<col0:array<string>,col1:struct<col2:string>>")
+      "STRUCT<`col0`: ARRAY<STRING>, `col1`: STRUCT<`col2`: STRING>>")
   }
 
   test("infer schema of JSON strings by using options") {
     checkEvaluation(
       new SchemaOfJson(Literal.create("""{"col":01}"""),
         CreateMap(Seq(Literal.create("allowNumericLeadingZeros"), Literal.create("true")))),
-      "struct<col:bigint>")
+      "STRUCT<`col`: BIGINT>")
   }
 
   test("parse date with locale") {
@@ -810,7 +810,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
     }
 
     Seq("en-US", "ko-KR", "ru-RU", "de-DE").foreach {
-        checkDecimalInfer(_, """struct<d:decimal(7,3)>""")
+        checkDecimalInfer(_, """STRUCT<`d`: DECIMAL(7,3)>""")
     }
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
index 4714635a3370b..bda43aac97c43 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
@@ -18,7 +18,8 @@
 package org.apache.spark.sql.catalyst.expressions
 
 import java.nio.charset.StandardCharsets
-import java.time.{Instant, LocalDate, LocalDateTime, ZoneOffset}
+import java.time.{Duration, Instant, LocalDate, LocalDateTime, Period, ZoneOffset}
+import java.time.temporal.ChronoUnit
 import java.util.TimeZone
 
 import scala.reflect.runtime.universe.TypeTag
@@ -49,6 +50,8 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(Literal.create(null, DateType), null)
     checkEvaluation(Literal.create(null, TimestampType), null)
     checkEvaluation(Literal.create(null, CalendarIntervalType), null)
+    checkEvaluation(Literal.create(null, YearMonthIntervalType), null)
+    checkEvaluation(Literal.create(null, DayTimeIntervalType), null)
     checkEvaluation(Literal.create(null, ArrayType(ByteType, true)), null)
     checkEvaluation(Literal.create(null, ArrayType(StringType, true)), null)
     checkEvaluation(Literal.create(null, MapType(StringType, IntegerType)), null)
@@ -76,6 +79,8 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
       checkEvaluation(Literal.default(TimestampType), Instant.ofEpochSecond(0))
     }
     checkEvaluation(Literal.default(CalendarIntervalType), new CalendarInterval(0, 0, 0L))
+    checkEvaluation(Literal.default(YearMonthIntervalType), 0)
+    checkEvaluation(Literal.default(DayTimeIntervalType), 0L)
     checkEvaluation(Literal.default(ArrayType(StringType)), Array())
     checkEvaluation(Literal.default(MapType(IntegerType, StringType)), Map())
     checkEvaluation(Literal.default(StructType(StructField("a", StringType) :: Nil)), Row(""))
@@ -187,20 +192,21 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkArrayLiteral(Array(1, 2, 3))
     checkArrayLiteral(Array("a", "b", "c"))
     checkArrayLiteral(Array(1.0, 4.0))
-    checkArrayLiteral(Array(MICROS_PER_DAY, MICROS_PER_HOUR))
+    checkArrayLiteral(Array(new CalendarInterval(1, 0, 0), new CalendarInterval(0, 1, 0)))
     val arr = collection.mutable.WrappedArray.make(Array(1.0, 4.0))
     checkEvaluation(Literal(arr), toCatalyst(arr))
   }
 
   test("seq") {
-    def checkSeqLiteral[T: TypeTag](a: Seq[T], elementType: DataType): Unit = {
+    def checkSeqLiteral[T: TypeTag](a: Seq[T]): Unit = {
       checkEvaluation(Literal.create(a), toCatalyst(a))
     }
-    checkSeqLiteral(Seq(1, 2, 3), IntegerType)
-    checkSeqLiteral(Seq("a", "b", "c"), StringType)
-    checkSeqLiteral(Seq(1.0, 4.0), DoubleType)
-    checkSeqLiteral(Seq(MICROS_PER_DAY, MICROS_PER_HOUR),
-      CalendarIntervalType)
+    checkSeqLiteral(Seq(1, 2, 3))
+    checkSeqLiteral(Seq("a", "b", "c"))
+    checkSeqLiteral(Seq(1.0, 4.0))
+    checkSeqLiteral(Seq(new CalendarInterval(1, 0, 0), new CalendarInterval(0, 1, 0)))
+    checkSeqLiteral(Seq(Period.ZERO, Period.ofMonths(1)))
+    checkSeqLiteral(Seq(Duration.ZERO, Duration.ofDays(1)))
   }
 
   test("map") {
@@ -209,6 +215,8 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
     }
     checkMapLiteral(Map("a" -> 1, "b" -> 2, "c" -> 3))
     checkMapLiteral(Map("1" -> 1.0, "2" -> 2.0, "3" -> 3.0))
+    checkMapLiteral(Map(Period.ofMonths(1) -> Duration.ZERO))
+    assert(Literal.create(Map("a" -> 1)).toString === "map(keys: [a], values: [1])")
   }
 
   test("struct") {
@@ -218,6 +226,7 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkStructLiteral((1, 3.0, "abcde"))
     checkStructLiteral(("de", 1, 2.0f))
     checkStructLiteral((1, ("fgh", 3.0)))
+    checkStructLiteral((Period.ZERO, ("abc", Duration.ofDays(1))))
   }
 
   test("unsupported types (map and struct) in Literal.apply") {
@@ -238,6 +247,15 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(Literal.create('\n'), "\n")
   }
 
+  test("SPARK-33390: Make Literal support char array") {
+    checkEvaluation(Literal(Array('h', 'e', 'l', 'l', 'o')), "hello")
+    checkEvaluation(Literal(Array("hello".toCharArray)), Array("hello"))
+    // scalastyle:off
+    checkEvaluation(Literal(Array('测','试')), "测试")
+    checkEvaluation(Literal(Array('a', '测', 'b', '试', 'c')), "a测b试c")
+    // scalastyle:on
+  }
+
   test("construct literals from java.time.LocalDate") {
     Seq(
       LocalDate.of(1, 1, 1),
@@ -316,4 +334,91 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
       assert(literalStr === expected)
     }
   }
+
+  test("SPARK-33860: Make CatalystTypeConverters.convertToCatalyst match special Array value") {
+    assert(Literal(Array(1, 2, 3)) == Literal.create(Array(1, 2, 3), ArrayType(IntegerType)))
+    assert(Literal(Array(1L, 2L, 3L)) == Literal.create(Array(1L, 2L, 3L), ArrayType(LongType)))
+    assert(Literal(Array(1D, 2D, 3D)) == Literal.create(Array(1D, 2D, 3D), ArrayType(DoubleType)))
+    assert(Literal("123") == Literal.create(Array('1', '2', '3'), StringType))
+    assert(Literal(Array(1.toByte, 2.toByte, 3.toByte)) ==
+      Literal.create(Array(1.toByte, 2.toByte, 3.toByte), BinaryType))
+    assert(Literal(Array("1", "2", "3")) ==
+      Literal.create(Array("1", "2", "3"), ArrayType(StringType)))
+    assert(Literal(Array(Period.ofMonths(1))) ==
+      Literal.create(Array(Period.ofMonths(1)), ArrayType(YearMonthIntervalType)))
+  }
+
+  test("SPARK-34342: Date/Timestamp toString") {
+    assert(Literal.default(DateType).toString === "1970-01-01")
+    assert(Literal.default(TimestampType).toString === "1969-12-31 16:00:00")
+    withTimeZones(sessionTimeZone = "GMT+01:00", systemTimeZone = "GMT-08:00") {
+      val timestamp = LocalDateTime.of(2021, 2, 3, 16, 50, 3, 456000000)
+        .atZone(ZoneOffset.UTC)
+        .toInstant
+      val literalStr = Literal.create(timestamp).toString
+      assert(literalStr === "2021-02-03 17:50:03.456")
+    }
+  }
+
+  test("SPARK-34605: construct literals from java.time.Duration") {
+    Seq(
+      Duration.ofNanos(0),
+      Duration.ofSeconds(-1),
+      Duration.ofNanos(123456000),
+      Duration.ofDays(106751991),
+      Duration.ofDays(-106751991)).foreach { duration =>
+      checkEvaluation(Literal(duration), duration)
+    }
+  }
+
+  test("SPARK-34605: construct literals from arrays of java.time.Duration") {
+    val duration0 = Duration.ofDays(2).plusHours(3).plusMinutes(4)
+    checkEvaluation(Literal(Array(duration0)), Array(duration0))
+    val duration1 = Duration.ofHours(-1024)
+    checkEvaluation(Literal(Array(duration0, duration1)), Array(duration0, duration1))
+  }
+
+  test("SPARK-34615: construct literals from java.time.Period") {
+    Seq(
+      Period.ofYears(0),
+      Period.of(-1, 11, 0),
+      Period.of(1, -11, 0),
+      Period.ofMonths(Int.MaxValue),
+      Period.ofMonths(Int.MinValue)).foreach { period =>
+      checkEvaluation(Literal(period), period)
+    }
+  }
+
+  test("SPARK-34615: construct literals from arrays of java.time.Period") {
+    val period0 = Period.ofYears(123).withMonths(456)
+    checkEvaluation(Literal(Array(period0)), Array(period0))
+    val period1 = Period.ofMonths(-1024)
+    checkEvaluation(Literal(Array(period0, period1)), Array(period0, period1))
+  }
+
+  test("SPARK-35099: convert a literal of day-time interval to SQL string") {
+    Seq(
+      Duration.ofDays(-1) -> "-1 00:00:00",
+      Duration.of(10, ChronoUnit.MICROS) -> "0 00:00:00.00001",
+      Duration.of(MICROS_PER_DAY - 1, ChronoUnit.MICROS) -> "0 23:59:59.999999"
+    ).foreach { case (duration, intervalPayload) =>
+      val literal = Literal.apply(duration)
+      val expected = s"INTERVAL '$intervalPayload' DAY TO SECOND"
+      assert(literal.sql === expected)
+      assert(literal.toString === expected)
+    }
+  }
+
+  test("SPARK-35099: convert a literal of year-month interval to SQL string") {
+    Seq(
+      Period.ofYears(-1) -> "-1-0",
+      Period.of(9999, 11, 0) -> "9999-11",
+      Period.ofMonths(-11) -> "-0-11"
+    ).foreach { case (period, intervalPayload) =>
+      val literal = Literal.apply(period)
+      val expected = s"INTERVAL '$intervalPayload' YEAR TO MONTH"
+      assert(literal.sql === expected)
+      assert(literal.toString === expected)
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralGenerator.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralGenerator.scala
index d92eb01b69bf0..35ddc1d7bd9b9 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralGenerator.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralGenerator.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.catalyst.expressions
 
 import java.sql.{Date, Timestamp}
-import java.time.{Duration, Instant, LocalDate}
+import java.time.{Duration, Instant, LocalDate, Period}
 import java.util.concurrent.TimeUnit
 
 import org.scalacheck.{Arbitrary, Gen}
@@ -68,16 +68,27 @@ object LiteralGenerator {
   lazy val longLiteralGen: Gen[Literal] =
     for { l <- Arbitrary.arbLong.arbitrary } yield Literal.create(l, LongType)
 
+  // The floatLiteralGen and doubleLiteralGen will 50% of the time yield arbitrary values
+  // and 50% of the time will yield some special values that are more likely to reveal
+  // corner cases. This behavior is similar to the integral value generators.
   lazy val floatLiteralGen: Gen[Literal] =
     for {
-      f <- Gen.chooseNum(Float.MinValue / 2, Float.MaxValue / 2,
-        Float.NaN, Float.PositiveInfinity, Float.NegativeInfinity)
+      f <- Gen.oneOf(
+        Gen.oneOf(
+          Float.NaN, Float.PositiveInfinity, Float.NegativeInfinity, Float.MinPositiveValue,
+          Float.MaxValue, -Float.MaxValue, 0.0f, -0.0f, 1.0f, -1.0f),
+        Arbitrary.arbFloat.arbitrary
+      )
     } yield Literal.create(f, FloatType)
 
   lazy val doubleLiteralGen: Gen[Literal] =
     for {
-      f <- Gen.chooseNum(Double.MinValue / 2, Double.MaxValue / 2,
-        Double.NaN, Double.PositiveInfinity, Double.NegativeInfinity)
+      f <- Gen.oneOf(
+        Gen.oneOf(
+          Double.NaN, Double.PositiveInfinity, Double.NegativeInfinity, Double.MinPositiveValue,
+          Double.MaxValue, -Double.MaxValue, 0.0, -0.0, 1.0, -1.0),
+        Arbitrary.arbDouble.arbitrary
+      )
     } yield Literal.create(f, DoubleType)
 
   // TODO cache the generated data
@@ -152,6 +163,19 @@ object LiteralGenerator {
   lazy val limitedIntegerLiteralGen: Gen[Literal] =
     for { i <- Gen.choose(-100, 100) } yield Literal.create(i, IntegerType)
 
+  lazy val dayTimeIntervalLiteralGen: Gen[Literal] = {
+    calendarIntervalLiterGen.map { calendarIntervalLiteral =>
+      Literal.create(
+        calendarIntervalLiteral.value.asInstanceOf[CalendarInterval].extractAsDuration(),
+        DayTimeIntervalType)
+    }
+  }
+
+  lazy val yearMonthIntervalLiteralGen: Gen[Literal] = {
+    for { months <- Gen.choose(-1 * maxIntervalInMonths, maxIntervalInMonths) }
+      yield Literal.create(Period.ofMonths(months), YearMonthIntervalType)
+  }
+
   def randomGen(dt: DataType): Gen[Literal] = {
     dt match {
       case ByteType => byteLiteralGen
@@ -167,6 +191,8 @@ object LiteralGenerator {
       case BinaryType => binaryLiteralGen
       case CalendarIntervalType => calendarIntervalLiterGen
       case DecimalType.Fixed(precision, scale) => decimalLiteralGen(precision, scale)
+      case DayTimeIntervalType => dayTimeIntervalLiteralGen
+      case YearMonthIntervalType => yearMonthIntervalLiteralGen
       case dt => throw new IllegalArgumentException(s"not supported type $dt")
     }
   }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MathExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MathExpressionsSuite.scala
index b4096f21bea3a..a17a8ba22bd63 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MathExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MathExpressionsSuite.scala
@@ -23,7 +23,7 @@ import com.google.common.math.LongMath
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.TypeCoercion.ImplicitTypeCasts.implicitCast
+import org.apache.spark.sql.catalyst.analysis.TypeCoercion.implicitCast
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.GenerateMutableProjection
 import org.apache.spark.sql.catalyst.optimizer.SimpleTestOptimizer
@@ -138,9 +138,8 @@ class MathExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     expression: Expression,
     inputRow: InternalRow = EmptyRow): Unit = {
 
-    val plan = generateProject(
-      GenerateMutableProjection.generate(Alias(expression, s"Optimized($expression)")() :: Nil),
-      expression)
+    val plan =
+      GenerateMutableProjection.generate(Alias(expression, s"Optimized($expression)")() :: Nil)
 
     val actual = plan(inputRow).get(0, expression.dataType)
     if (!actual.asInstanceOf[Double].isNaN) {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MiscExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MiscExpressionsSuite.scala
index 4b2d153a28cc8..d42081024c1dd 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MiscExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MiscExpressionsSuite.scala
@@ -26,21 +26,21 @@ import org.apache.spark.sql.types._
 
 class MiscExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
-  test("assert_true") {
-    intercept[RuntimeException] {
-      checkEvaluation(AssertTrue(Literal.create(false, BooleanType)), null)
-    }
-    intercept[RuntimeException] {
-      checkEvaluation(AssertTrue(Cast(Literal(0), BooleanType)), null)
-    }
-    intercept[RuntimeException] {
-      checkEvaluation(AssertTrue(Literal.create(null, NullType)), null)
-    }
-    intercept[RuntimeException] {
-      checkEvaluation(AssertTrue(Literal.create(null, BooleanType)), null)
-    }
-    checkEvaluation(AssertTrue(Literal.create(true, BooleanType)), null)
-    checkEvaluation(AssertTrue(Cast(Literal(1), BooleanType)), null)
+  test("RaiseError") {
+    checkExceptionInExpression[RuntimeException](
+      RaiseError(Literal("error message")),
+      EmptyRow,
+      "error message"
+    )
+
+    checkExceptionInExpression[RuntimeException](
+      RaiseError(Literal.create(null, StringType)),
+      EmptyRow,
+      null
+    )
+
+    // Expects a string
+    assert(RaiseError(Literal(5)).checkInputDataTypes().isFailure)
   }
 
   test("uuid") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MutableProjectionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MutableProjectionSuite.scala
index c31310bc54023..3d11ff97f6d5c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MutableProjectionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MutableProjectionSuite.scala
@@ -23,13 +23,14 @@ import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
 import org.apache.spark.sql.catalyst.util.IntervalUtils
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.Platform
 import org.apache.spark.unsafe.types.UTF8String
 
 class MutableProjectionSuite extends SparkFunSuite with ExpressionEvalHelper {
 
   val fixedLengthTypes = Array[DataType](
     BooleanType, ByteType, ShortType, IntegerType, LongType, FloatType, DoubleType,
-    DateType, TimestampType)
+    DateType, TimestampType, YearMonthIntervalType, DayTimeIntervalType)
 
   val variableLengthTypes = Array(
     StringType, DecimalType.defaultConcreteType, CalendarIntervalType, BinaryType,
@@ -41,15 +42,19 @@ class MutableProjectionSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   testBothCodegenAndInterpreted("fixed-length types") {
-    val inputRow = InternalRow.fromSeq(Seq(true, 3.toByte, 15.toShort, -83, 129L, 1.0f, 5.0, 1, 2L))
+    val inputRow = InternalRow.fromSeq(Seq(
+      true, 3.toByte, 15.toShort, -83, 129L, 1.0f, 5.0, 1, 2L, Int.MaxValue, Long.MinValue))
     val proj = createMutableProjection(fixedLengthTypes)
     assert(proj(inputRow) === inputRow)
   }
 
   testBothCodegenAndInterpreted("unsafe buffer") {
-    val inputRow = InternalRow.fromSeq(Seq(false, 1.toByte, 9.toShort, -18, 53L, 3.2f, 7.8, 4, 9L))
-    val numBytes = UnsafeRow.calculateBitSetWidthInBytes(fixedLengthTypes.length)
-    val unsafeBuffer = UnsafeRow.createFromByteArray(numBytes, fixedLengthTypes.length)
+    val inputRow = InternalRow.fromSeq(Seq(
+      false, 1.toByte, 9.toShort, -18, 53L, 3.2f, 7.8, 4, 9L, Int.MinValue, Long.MaxValue))
+    val numFields = fixedLengthTypes.length
+    val numBytes = Platform.BYTE_ARRAY_OFFSET + UnsafeRow.calculateBitSetWidthInBytes(numFields) +
+      UnsafeRow.WORD_SIZE * numFields
+    val unsafeBuffer = UnsafeRow.createFromByteArray(numBytes, numFields)
     val proj = createMutableProjection(fixedLengthTypes)
     val projUnsafeRow = proj.target(unsafeBuffer)(inputRow)
     assert(SafeProjection.create(fixedLengthTypes)(projUnsafeRow) === inputRow)
@@ -80,4 +85,50 @@ class MutableProjectionSuite extends SparkFunSuite with ExpressionEvalHelper {
       assert(errMsg.contains("MutableProjection cannot use UnsafeRow for output data types:"))
     }
   }
+
+  test("SPARK-33473: subexpression elimination for interpreted MutableProjection") {
+    Seq("true", "false").foreach { enabled =>
+      withSQLConf(
+        SQLConf.SUBEXPRESSION_ELIMINATION_ENABLED.key -> enabled,
+        SQLConf.CODEGEN_FACTORY_MODE.key -> CodegenObjectFactoryMode.NO_CODEGEN.toString) {
+        val one = BoundReference(0, DoubleType, true)
+        val two = BoundReference(1, DoubleType, true)
+
+        val mul = Multiply(one, two)
+        val mul2 = Multiply(mul, mul)
+        val sqrt = Sqrt(mul2)
+        val sum = Add(mul2, sqrt)
+
+        val proj = MutableProjection.create(Seq(sum))
+        val result = (d1: Double, d2: Double) =>
+          ((d1 * d2) * (d1 * d2)) + Math.sqrt((d1 * d2) * (d1 * d2))
+
+        val inputRows = Seq(
+          InternalRow.fromSeq(Seq(1.0, 2.0)),
+          InternalRow.fromSeq(Seq(2.0, 3.0)),
+          InternalRow.fromSeq(Seq(1.0, null)),
+          InternalRow.fromSeq(Seq(null, 2.0)),
+          InternalRow.fromSeq(Seq(3.0, 4.0)),
+          InternalRow.fromSeq(Seq(null, null))
+        )
+        val expectedResults = Seq(
+          result(1.0, 2.0),
+          result(2.0, 3.0),
+          null,
+          null,
+          result(3.0, 4.0),
+          null
+        )
+
+        inputRows.zip(expectedResults).foreach { case (inputRow, expected) =>
+          val projRow = proj.apply(inputRow)
+          if (expected != null) {
+            assert(projRow.getDouble(0) == expected)
+          } else {
+            assert(projRow.isNullAt(0))
+          }
+        }
+      }
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NamedExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NamedExpressionSuite.scala
new file mode 100644
index 0000000000000..f6cc19abaf9df
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NamedExpressionSuite.scala
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
+import org.apache.spark.sql.types.IntegerType
+
+class NamedExpressionSuite extends SparkFunSuite {
+
+  test("SPARK-34636: sql method should quote qualified names properly") {
+    val attr1 = UnresolvedAttribute("a"::Nil)
+    assert(attr1.sql === "a")
+    val attr2 = UnresolvedAttribute("a"::"b"::Nil)
+    assert(attr2.sql === "a.b")
+    val attr3 = UnresolvedAttribute("a.b"::"c.d"::Nil)
+    assert(attr3.sql === "`a.b`.`c.d`")
+    val attr4 = UnresolvedAttribute("a`b"::"c.d"::Nil)
+    assert(attr4.sql === "`a``b`.`c.d`")
+    val attr5 = AttributeReference("a", IntegerType)()
+    assert(attr5.sql === "a")
+    val attr6 = AttributeReference("a", IntegerType)(qualifier = "b"::Nil)
+    assert(attr6.sql === "b.a")
+    val attr7 = AttributeReference("a.b", IntegerType)(qualifier = "c.d"::Nil)
+    assert(attr7.sql === "`c.d`.`a.b`")
+    val attr8 = AttributeReference("a.b", IntegerType)(qualifier = "c`d"::Nil)
+    assert(attr8.sql === "`c``d`.`a.b`")
+    val attr9 = Alias(attr8, "e")()
+    assert(attr9.sql === "`c``d`.`a.b` AS e")
+    val attr10 = Alias(attr8, "e")(qualifier = "f"::Nil)
+    assert(attr10.sql === "`c``d`.`a.b` AS f.e")
+    val attr11 = Alias(attr8, "e.f")(qualifier = "g.h"::Nil)
+    assert(attr11.sql === "`c``d`.`a.b` AS `g.h`.`e.f`")
+    val attr12 = Alias(attr8, "e`f")(qualifier = "g.h"::Nil)
+    assert(attr12.sql === "`c``d`.`a.b` AS `g.h`.`e``f`")
+    val attr13 = UnresolvedAttribute("`a.b`")
+    assert(attr13.sql === "`a.b`")
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NullExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NullExpressionsSuite.scala
index 49fd59c8694f1..7abea96915d2f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NullExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NullExpressionsSuite.scala
@@ -160,6 +160,12 @@ class NullExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(AtLeastNNonNulls(4, nullOnly), false, EmptyRow)
   }
 
+  test("SPARK-34857: AtLeastNNonNulls toString") {
+    val e = AtLeastNNonNulls(2,
+      Seq(Literal(42), Literal("test"), Literal.create(null, DoubleType)))
+    assert(e.toString == "atleastnnonnulls(2, 42, test, null)")
+  }
+
   test("Coalesce should not throw 64KiB exception") {
     val inputs = (1 to 2500).map(x => Literal(s"x_$x"))
     checkEvaluation(Coalesce(inputs), "x_1")
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala
index c40149368b055..5e4c5bcd0ecc9 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.expressions
 import java.sql.{Date, Timestamp}
 
 import scala.collection.JavaConverters._
+import scala.collection.mutable.WrappedArray
 import scala.reflect.ClassTag
 import scala.reflect.runtime.universe.TypeTag
 import scala.util.Random
@@ -27,18 +28,17 @@ import scala.util.Random
 import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.serializer.{JavaSerializer, KryoSerializer}
 import org.apache.spark.sql.{RandomDataGenerator, Row}
-import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow, JavaTypeInference, ScalaReflection}
+import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
 import org.apache.spark.sql.catalyst.ScroogeLikeExample
 import org.apache.spark.sql.catalyst.analysis.{ResolveTimeZone, SimpleAnalyzer, UnresolvedDeserializer}
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.encoders._
-import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
+import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, GenerateUnsafeProjection}
 import org.apache.spark.sql.catalyst.expressions.objects._
 import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, Project}
 import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, DateTimeUtils, GenericArrayData, IntervalUtils}
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
-import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
+import org.apache.spark.unsafe.types.UTF8String
 
 class InvokeTargetClass extends Serializable {
   def filterInt(e: Any): Any = e.asInstanceOf[Int] > 0
@@ -156,10 +156,10 @@ class ObjectExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
         "fromPrimitiveArray", ObjectType(classOf[Array[Int]]),
         Array[Int](1, 2, 3), UnsafeArrayData.fromPrimitiveArray(Array[Int](1, 2, 3))),
       (DateTimeUtils.getClass, ObjectType(classOf[Date]),
-        "toJavaDate", ObjectType(classOf[DateTimeUtils.SQLDate]), 77777,
+        "toJavaDate", ObjectType(classOf[Int]), 77777,
         DateTimeUtils.toJavaDate(77777)),
       (DateTimeUtils.getClass, ObjectType(classOf[Timestamp]),
-        "toJavaTimestamp", ObjectType(classOf[DateTimeUtils.SQLTimestamp]),
+        "toJavaTimestamp", ObjectType(classOf[Long]),
         88888888.toLong, DateTimeUtils.toJavaTimestamp(88888888))
     ).foreach { case (cls, dataType, methodName, argType, arg, expected) =>
       checkObjectExprEvaluation(StaticInvoke(cls, dataType, methodName,
@@ -212,9 +212,9 @@ class ObjectExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
     val initializeWithNonexistingMethod = InitializeJavaBean(
       Literal.fromObject(new java.util.LinkedList[Int]),
-      Map("nonexisting" -> Literal(1)))
+      Map("nonexistent" -> Literal(1)))
     checkExceptionInExpression[Exception](initializeWithNonexistingMethod,
-      """A method named "nonexisting" is not declared in any enclosing class """ +
+      """A method named "nonexistent" is not declared in any enclosing class """ +
         "nor any supertype")
 
     val initializeWithWrongParamType = InitializeJavaBean(
@@ -269,7 +269,7 @@ class ObjectExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   private def checkObjectExprEvaluation(
       expression: => Expression, expected: Any, inputRow: InternalRow = EmptyRow): Unit = {
     val serializer = new JavaSerializer(new SparkConf()).newInstance
-    val resolver = ResolveTimeZone(new SQLConf)
+    val resolver = ResolveTimeZone
     val expr = resolver.resolveTimeZones(serializer.deserialize(serializer.serialize(expression)))
     checkEvaluationWithoutCodegen(expr, expected, inputRow)
     checkEvaluationWithMutableProjection(expr, expected, inputRow)
@@ -330,6 +330,8 @@ class ObjectExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
           assert(result.asInstanceOf[ArrayData].array.toSeq == expected)
         case l if classOf[java.util.List[_]].isAssignableFrom(l) =>
           assert(result.asInstanceOf[java.util.List[_]].asScala == expected)
+        case a if classOf[WrappedArray[Int]].isAssignableFrom(a) =>
+          assert(result == WrappedArray.make[Int](expected.toArray))
         case s if classOf[Seq[_]].isAssignableFrom(s) =>
           assert(result.asInstanceOf[Seq[_]] == expected)
         case s if classOf[scala.collection.Set[_]].isAssignableFrom(s) =>
@@ -337,7 +339,8 @@ class ObjectExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
       }
     }
 
-    val customCollectionClasses = Seq(classOf[Seq[Int]], classOf[scala.collection.Set[Int]],
+    val customCollectionClasses = Seq(classOf[WrappedArray[Int]],
+      classOf[Seq[Int]], classOf[scala.collection.Set[Int]],
       classOf[java.util.List[Int]], classOf[java.util.AbstractList[Int]],
       classOf[java.util.AbstractSequentialList[Int]], classOf[java.util.Vector[Int]],
       classOf[java.util.Stack[Int]], null)
@@ -357,6 +360,7 @@ class ObjectExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     stack.add(3)
 
     Seq(
+      (Seq(1, 2, 3), ObjectType(classOf[WrappedArray[Int]])),
       (Seq(1, 2, 3), ObjectType(classOf[Seq[Int]])),
       (Array(1, 2, 3), ObjectType(classOf[Array[Int]])),
       (Seq(1, 2, 3), ObjectType(classOf[Object])),
@@ -604,6 +608,60 @@ class ObjectExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkExceptionInExpression[RuntimeException](
       serializer4, EmptyRow, "Cannot use null as map key!")
   }
+
+  test("SPARK-35244: invoke should throw the original exception") {
+    val strClsType = ObjectType(classOf[String])
+    checkExceptionInExpression[StringIndexOutOfBoundsException](
+      Invoke(Literal("a", strClsType), "substring", strClsType, Seq(Literal(3))), "")
+
+    val mathCls = classOf[Math]
+    checkExceptionInExpression[ArithmeticException](
+      StaticInvoke(mathCls, IntegerType, "addExact", Seq(Literal(Int.MaxValue), Literal(1))), "")
+  }
+
+  test("SPARK-35278: invoke should find method with correct number of parameters") {
+    val strClsType = ObjectType(classOf[String])
+    checkExceptionInExpression[StringIndexOutOfBoundsException](
+      Invoke(Literal("a", strClsType), "substring", strClsType, Seq(Literal(3))), "")
+
+    checkObjectExprEvaluation(
+      Invoke(Literal("a", strClsType), "substring", strClsType, Seq(Literal(0))), "a")
+
+    checkExceptionInExpression[StringIndexOutOfBoundsException](
+      Invoke(Literal("a", strClsType), "substring", strClsType, Seq(Literal(0), Literal(3))), "")
+
+    checkObjectExprEvaluation(
+      Invoke(Literal("a", strClsType), "substring", strClsType, Seq(Literal(0), Literal(1))), "a")
+  }
+
+  test("SPARK-35278: invoke should correctly invoke override method") {
+    val clsType = ObjectType(classOf[ConcreteClass])
+    val obj = new ConcreteClass
+
+    val input = (1, 2)
+    checkObjectExprEvaluation(
+      Invoke(Literal(obj, clsType), "testFunc", IntegerType,
+        Seq(Literal(input, ObjectType(input.getClass)))), 2)
+  }
+
+  test("SPARK-35288: static invoke should find method without exact param type match") {
+    val input = (1, 2)
+
+    checkObjectExprEvaluation(
+      StaticInvoke(TestStaticInvoke.getClass, IntegerType, "func",
+        Seq(Literal(input, ObjectType(input.getClass)))), 3)
+
+    checkObjectExprEvaluation(
+      StaticInvoke(TestStaticInvoke.getClass, IntegerType, "func",
+        Seq(Literal(1, IntegerType))), -1)
+  }
+
+  test("SPARK-35281: StaticInvoke shouldn't box primitive when result is nullable") {
+    val ctx = new CodegenContext
+    val arguments = Seq(Literal(0), Literal(1))
+    val genCode = StaticInvoke(TestFun.getClass, IntegerType, "foo", arguments).genCode(ctx)
+    assert(!genCode.code.toString.contains("boxedResult"))
+  }
 }
 
 class TestBean extends Serializable {
@@ -614,3 +672,28 @@ class TestBean extends Serializable {
   def setNonPrimitive(i: AnyRef): Unit =
     assert(i != null, "this setter should not be called with null.")
 }
+
+object TestStaticInvoke {
+  def func(param: Any): Int = param match {
+    case pair: Tuple2[_, _] =>
+      pair.asInstanceOf[Tuple2[Int, Int]]._1 + pair.asInstanceOf[Tuple2[Int, Int]]._2
+    case _ => -1
+  }
+}
+
+abstract class BaseClass[T] {
+  def testFunc(param: T): Int
+}
+
+class ConcreteClass extends BaseClass[Product] with Serializable {
+  override def testFunc(param: Product): Int = param match {
+    case _: Tuple2[_, _] => 2
+    case _: Tuple3[_, _, _] => 3
+    case _ => 4
+  }
+}
+
+case object TestFun {
+  def foo(left: Int, right: Int): Int = left + right
+}
+
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala
index 1ad0a8ed758f4..6f75623dc59ae 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala
@@ -538,4 +538,110 @@ class PredicateSuite extends SparkFunSuite with ExpressionEvalHelper {
     val inSet = InSet(BoundReference(0, IntegerType, true), Set.empty)
     checkEvaluation(inSet, false, row)
   }
+
+  test("SPARK-32764: compare special double/float values") {
+    checkEvaluation(EqualTo(Literal(Double.NaN), Literal(Double.NaN)), true)
+    checkEvaluation(EqualTo(Literal(Double.NaN), Literal(Double.PositiveInfinity)), false)
+    checkEvaluation(EqualTo(Literal(0.0D), Literal(-0.0D)), true)
+    checkEvaluation(GreaterThan(Literal(Double.NaN), Literal(Double.PositiveInfinity)), true)
+    checkEvaluation(GreaterThan(Literal(Double.NaN), Literal(Double.NaN)), false)
+    checkEvaluation(GreaterThan(Literal(0.0D), Literal(-0.0D)), false)
+
+    checkEvaluation(EqualTo(Literal(Float.NaN), Literal(Float.NaN)), true)
+    checkEvaluation(EqualTo(Literal(Float.NaN), Literal(Float.PositiveInfinity)), false)
+    checkEvaluation(EqualTo(Literal(0.0F), Literal(-0.0F)), true)
+    checkEvaluation(GreaterThan(Literal(Float.NaN), Literal(Float.PositiveInfinity)), true)
+    checkEvaluation(GreaterThan(Literal(Float.NaN), Literal(Float.NaN)), false)
+    checkEvaluation(GreaterThan(Literal(0.0F), Literal(-0.0F)), false)
+  }
+
+  test("SPARK-32110: compare special double/float values in array") {
+    def createUnsafeDoubleArray(d: Double): Literal = {
+      Literal(UnsafeArrayData.fromPrimitiveArray(Array(d)), ArrayType(DoubleType))
+    }
+    def createSafeDoubleArray(d: Double): Literal = {
+      Literal(new GenericArrayData(Array(d)), ArrayType(DoubleType))
+    }
+    def createUnsafeFloatArray(d: Double): Literal = {
+      Literal(UnsafeArrayData.fromPrimitiveArray(Array(d.toFloat)), ArrayType(FloatType))
+    }
+    def createSafeFloatArray(d: Double): Literal = {
+      Literal(new GenericArrayData(Array(d.toFloat)), ArrayType(FloatType))
+    }
+    def checkExpr(
+        exprBuilder: (Expression, Expression) => Expression,
+        left: Double,
+        right: Double,
+        expected: Any): Unit = {
+      // test double
+      checkEvaluation(
+        exprBuilder(createUnsafeDoubleArray(left), createUnsafeDoubleArray(right)), expected)
+      checkEvaluation(
+        exprBuilder(createUnsafeDoubleArray(left), createSafeDoubleArray(right)), expected)
+      checkEvaluation(
+        exprBuilder(createSafeDoubleArray(left), createSafeDoubleArray(right)), expected)
+      // test float
+      checkEvaluation(
+        exprBuilder(createUnsafeFloatArray(left), createUnsafeFloatArray(right)), expected)
+      checkEvaluation(
+        exprBuilder(createUnsafeFloatArray(left), createSafeFloatArray(right)), expected)
+      checkEvaluation(
+        exprBuilder(createSafeFloatArray(left), createSafeFloatArray(right)), expected)
+    }
+
+    checkExpr(EqualTo, Double.NaN, Double.NaN, true)
+    checkExpr(EqualTo, Double.NaN, Double.PositiveInfinity, false)
+    checkExpr(EqualTo, 0.0, -0.0, true)
+    checkExpr(GreaterThan, Double.NaN, Double.PositiveInfinity, true)
+    checkExpr(GreaterThan, Double.NaN, Double.NaN, false)
+    checkExpr(GreaterThan, 0.0, -0.0, false)
+  }
+
+  test("SPARK-32110: compare special double/float values in struct") {
+    def createUnsafeDoubleRow(d: Double): Literal = {
+      val dt = new StructType().add("d", "double")
+      val converter = UnsafeProjection.create(dt)
+      val unsafeRow = converter.apply(InternalRow(d))
+      Literal(unsafeRow, dt)
+    }
+    def createSafeDoubleRow(d: Double): Literal = {
+      Literal(InternalRow(d), new StructType().add("d", "double"))
+    }
+    def createUnsafeFloatRow(d: Double): Literal = {
+      val dt = new StructType().add("f", "float")
+      val converter = UnsafeProjection.create(dt)
+      val unsafeRow = converter.apply(InternalRow(d.toFloat))
+      Literal(unsafeRow, dt)
+    }
+    def createSafeFloatRow(d: Double): Literal = {
+      Literal(InternalRow(d.toFloat), new StructType().add("f", "float"))
+    }
+    def checkExpr(
+        exprBuilder: (Expression, Expression) => Expression,
+        left: Double,
+        right: Double,
+        expected: Any): Unit = {
+      // test double
+      checkEvaluation(
+        exprBuilder(createUnsafeDoubleRow(left), createUnsafeDoubleRow(right)), expected)
+      checkEvaluation(
+        exprBuilder(createUnsafeDoubleRow(left), createSafeDoubleRow(right)), expected)
+      checkEvaluation(
+        exprBuilder(createSafeDoubleRow(left), createSafeDoubleRow(right)), expected)
+      // test float
+      checkEvaluation(
+        exprBuilder(createUnsafeFloatRow(left), createUnsafeFloatRow(right)), expected)
+      checkEvaluation(
+        exprBuilder(createUnsafeFloatRow(left), createSafeFloatRow(right)), expected)
+      checkEvaluation(
+        exprBuilder(createSafeFloatRow(left), createSafeFloatRow(right)), expected)
+    }
+
+    checkExpr(EqualTo, Double.NaN, Double.NaN, true)
+    checkExpr(EqualTo, Double.NaN, Double.PositiveInfinity, false)
+    checkExpr(EqualTo, 0.0, -0.0, true)
+    checkExpr(GreaterThan, Double.NaN, Double.PositiveInfinity, true)
+    checkExpr(GreaterThan, Double.NaN, Double.NaN, false)
+    checkExpr(GreaterThan, 0.0, -0.0, false)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
index ad9492a8d3ab1..019857580d077 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
@@ -22,6 +22,8 @@ import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
 import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
+import org.apache.spark.sql.catalyst.optimizer.ConstantFolding
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.StringType
 
 /**
@@ -48,6 +50,56 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(mkExpr(regex), expected, create_row(input)) // check row input
   }
 
+  test("LIKE ALL") {
+    checkEvaluation(Literal.create(null, StringType).likeAll("%foo%", "%oo"), null)
+    checkEvaluation(Literal.create("foo", StringType).likeAll("%foo%", "%oo"), true)
+    checkEvaluation(Literal.create("foo", StringType).likeAll("%foo%", "%bar%"), false)
+    checkEvaluation(Literal.create("foo", StringType)
+      .likeAll("%foo%", Literal.create(null, StringType)), null)
+    checkEvaluation(Literal.create("foo", StringType)
+      .likeAll(Literal.create(null, StringType), "%foo%"), null)
+    checkEvaluation(Literal.create("foo", StringType)
+      .likeAll("%feo%", Literal.create(null, StringType)), false)
+    checkEvaluation(Literal.create("foo", StringType)
+      .likeAll(Literal.create(null, StringType), "%feo%"), false)
+    checkEvaluation(Literal.create("foo", StringType).notLikeAll("tee", "%yoo%"), true)
+    checkEvaluation(Literal.create("foo", StringType).notLikeAll("%oo%", "%yoo%"), false)
+    checkEvaluation(Literal.create("foo", StringType)
+      .notLikeAll("%foo%", Literal.create(null, StringType)), false)
+    checkEvaluation(Literal.create("foo", StringType)
+      .notLikeAll(Literal.create(null, StringType), "%foo%"), false)
+    checkEvaluation(Literal.create("foo", StringType)
+      .notLikeAll("%yoo%", Literal.create(null, StringType)), null)
+    checkEvaluation(Literal.create("foo", StringType)
+      .notLikeAll(Literal.create(null, StringType), "%yoo%"), null)
+  }
+
+  test("LIKE ANY") {
+    checkEvaluation(Literal.create(null, StringType).likeAny("%foo%", "%oo"), null)
+    checkEvaluation(Literal.create("foo", StringType).likeAny("%foo%", "%oo"), true)
+    checkEvaluation(Literal.create("foo", StringType).likeAny("%foo%", "%bar%"), true)
+    checkEvaluation(Literal.create("foo", StringType).likeAny("%fee%", "%bar%"), false)
+    checkEvaluation(Literal.create("foo", StringType)
+      .likeAny("%foo%", Literal.create(null, StringType)), true)
+    checkEvaluation(Literal.create("foo", StringType)
+      .likeAny(Literal.create(null, StringType), "%foo%"), true)
+    checkEvaluation(Literal.create("foo", StringType)
+      .likeAny("%feo%", Literal.create(null, StringType)), null)
+    checkEvaluation(Literal.create("foo", StringType)
+      .likeAny(Literal.create(null, StringType), "%feo%"), null)
+    checkEvaluation(Literal.create("foo", StringType).notLikeAny("tee", "%yoo%"), true)
+    checkEvaluation(Literal.create("foo", StringType).notLikeAny("%oo%", "%yoo%"), true)
+    checkEvaluation(Literal.create("foo", StringType).notLikeAny("%foo%", "%oo"), false)
+    checkEvaluation(Literal.create("foo", StringType)
+      .notLikeAny("%foo%", Literal.create(null, StringType)), null)
+    checkEvaluation(Literal.create("foo", StringType)
+      .notLikeAny(Literal.create(null, StringType), "%foo%"), null)
+    checkEvaluation(Literal.create("foo", StringType)
+      .notLikeAny("%yoo%", Literal.create(null, StringType)), true)
+    checkEvaluation(Literal.create("foo", StringType)
+      .notLikeAny(Literal.create(null, StringType), "%yoo%"), true)
+  }
+
   test("LIKE Pattern") {
 
     // null handling
@@ -253,6 +305,20 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(expr, null, row4)
     checkEvaluation(expr, null, row5)
     checkEvaluation(expr, null, row6)
+    // test position
+    val exprWithPos = RegExpReplace(s, p, r, 4)
+    checkEvaluation(exprWithPos, "100-num", row1)
+    checkEvaluation(exprWithPos, "100-###", row2)
+    checkEvaluation(exprWithPos, "100###200", row3)
+    checkEvaluation(exprWithPos, null, row4)
+    checkEvaluation(exprWithPos, null, row5)
+    checkEvaluation(exprWithPos, null, row6)
+    val exprWithLargePos = RegExpReplace(s, p, r, 7)
+    checkEvaluation(exprWithLargePos, "100-20num", row1)
+    checkEvaluation(exprWithLargePos, "100-20###", row2)
+    val exprWithExceedLength = RegExpReplace(s, p, r, 8)
+    checkEvaluation(exprWithExceedLength, "100-200", row1)
+    checkEvaluation(exprWithExceedLength, "100-200", row2)
 
     val nonNullExpr = RegExpReplace(Literal("100-200"), Literal("(\\d+)"), Literal("num"))
     checkEvaluation(nonNullExpr, "num-num", row1)
@@ -303,6 +369,8 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     val row8 = create_row("100-200", "(\\d+)-(\\d+)", 3)
     val row9 = create_row("100-200", "(\\d+).*", 2)
     val row10 = create_row("100-200", "\\d+", 1)
+    val row11 = create_row("100-200", "(\\d+)-(\\d+)", -1)
+    val row12 = create_row("100-200", "\\d+", -1)
 
     checkExceptionInExpression[IllegalArgumentException](
       expr, row8, "Regex group count is 2, but the specified group index is 3")
@@ -310,12 +378,66 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
       expr, row9, "Regex group count is 1, but the specified group index is 2")
     checkExceptionInExpression[IllegalArgumentException](
       expr, row10, "Regex group count is 0, but the specified group index is 1")
+    checkExceptionInExpression[IllegalArgumentException](
+      expr, row11, "The specified group index cannot be less than zero")
+    checkExceptionInExpression[IllegalArgumentException](
+      expr, row12, "The specified group index cannot be less than zero")
 
     // Test escaping of arguments
     GenerateUnsafeProjection.generate(
       RegExpExtract(Literal("\"quote"), Literal("\"quote"), Literal(1)) :: Nil)
   }
 
+  test("RegexExtractAll") {
+    val row1 = create_row("100-200,300-400,500-600", "(\\d+)-(\\d+)", 0)
+    val row2 = create_row("100-200,300-400,500-600", "(\\d+)-(\\d+)", 1)
+    val row3 = create_row("100-200,300-400,500-600", "(\\d+)-(\\d+)", 2)
+    val row4 = create_row("100-200,300-400,500-600", "(\\d+).*", 1)
+    val row5 = create_row("100-200,300-400,500-600", "([a-z])", 1)
+    val row6 = create_row(null, "([a-z])", 1)
+    val row7 = create_row("100-200,300-400,500-600", null, 1)
+    val row8 = create_row("100-200,300-400,500-600", "([a-z])", null)
+
+    val s = 's.string.at(0)
+    val p = 'p.string.at(1)
+    val r = 'r.int.at(2)
+
+    val expr = RegExpExtractAll(s, p, r)
+    checkEvaluation(expr, Seq("100-200", "300-400", "500-600"), row1)
+    checkEvaluation(expr, Seq("100", "300", "500"), row2)
+    checkEvaluation(expr, Seq("200", "400", "600"), row3)
+    checkEvaluation(expr, Seq("100"), row4)
+    checkEvaluation(expr, Seq(), row5)
+    checkEvaluation(expr, null, row6)
+    checkEvaluation(expr, null, row7)
+    checkEvaluation(expr, null, row8)
+
+    val expr1 = new RegExpExtractAll(s, p)
+    checkEvaluation(expr1, Seq("100", "300", "500"), row2)
+
+    val nonNullExpr = RegExpExtractAll(Literal("100-200,300-400,500-600"),
+      Literal("(\\d+)-(\\d+)"), Literal(1))
+    checkEvaluation(nonNullExpr, Seq("100", "300", "500"), row2)
+
+    // invalid group index
+    val row9 = create_row("100-200,300-400,500-600", "(\\d+)-(\\d+)", 3)
+    val row10 = create_row("100-200,300-400,500-600", "(\\d+).*", 2)
+    val row11 = create_row("100-200,300-400,500-600", "\\d+", 1)
+    val row12 = create_row("100-200,300-400,500-600", "(\\d+)-(\\d+)", -1)
+    val row13 = create_row("100-200,300-400,500-600", "\\d+", -1)
+
+    checkExceptionInExpression[IllegalArgumentException](
+      expr, row9, "Regex group count is 2, but the specified group index is 3")
+    checkExceptionInExpression[IllegalArgumentException](
+      expr, row10, "Regex group count is 1, but the specified group index is 2")
+    checkExceptionInExpression[IllegalArgumentException](
+      expr, row11, "Regex group count is 0, but the specified group index is 1")
+    checkExceptionInExpression[IllegalArgumentException](
+      expr, row12, "The specified group index cannot be less than zero")
+    checkExceptionInExpression[IllegalArgumentException](
+      expr, row13, "The specified group index cannot be less than zero")
+  }
+
   test("SPLIT") {
     val s1 = 'a.string.at(0)
     val s2 = 'b.string.at(1)
@@ -350,4 +472,12 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     cache.setAccessible(true)
     assert(cache.get(expr).asInstanceOf[java.util.regex.Pattern].pattern().contains("a"))
   }
+
+  test("SPARK-34814: LikeSimplification should handle NULL") {
+    withSQLConf(SQLConf.OPTIMIZER_EXCLUDED_RULES.key ->
+      ConstantFolding.getClass.getName.stripSuffix("$")) {
+      checkEvaluation(Literal.create("foo", StringType)
+        .likeAll("%foo%", Literal.create(null, StringType)), null)
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala
index 836b2eaa642a3..1b40e02aa8662 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala
@@ -19,6 +19,8 @@ package org.apache.spark.sql.catalyst.expressions
 
 import java.util.Locale
 
+import scala.reflect.runtime.universe.TypeTag
+
 import org.apache.spark.{SparkException, SparkFunSuite}
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
@@ -27,13 +29,17 @@ import org.apache.spark.sql.types.{DecimalType, IntegerType, StringType}
 
 class ScalaUDFSuite extends SparkFunSuite with ExpressionEvalHelper {
 
+  private def resolvedEncoder[T : TypeTag](): ExpressionEncoder[T] = {
+    ExpressionEncoder[T]().resolveAndBind()
+  }
+
   test("basic") {
     val intUdf = ScalaUDF((i: Int) => i + 1, IntegerType, Literal(1) :: Nil,
-      Option(ExpressionEncoder[Int]()) :: Nil)
+      Option(resolvedEncoder[Int]()) :: Nil)
     checkEvaluation(intUdf, 2)
 
     val stringUdf = ScalaUDF((s: String) => s + "x", StringType, Literal("a") :: Nil,
-      Option(ExpressionEncoder[String]()) :: Nil)
+      Option(resolvedEncoder[String]()) :: Nil)
     checkEvaluation(stringUdf, "ax")
   }
 
@@ -42,7 +48,7 @@ class ScalaUDFSuite extends SparkFunSuite with ExpressionEvalHelper {
       (s: String) => s.toLowerCase(Locale.ROOT),
       StringType,
       Literal.create(null, StringType) :: Nil,
-      Option(ExpressionEncoder[String]()) :: Nil)
+      Option(resolvedEncoder[String]()) :: Nil)
 
     val e1 = intercept[SparkException](udf.eval())
     assert(e1.getMessage.contains("Failed to execute user defined function"))
@@ -56,7 +62,7 @@ class ScalaUDFSuite extends SparkFunSuite with ExpressionEvalHelper {
   test("SPARK-22695: ScalaUDF should not use global variables") {
     val ctx = new CodegenContext
     ScalaUDF((s: String) => s + "x", StringType, Literal("a") :: Nil,
-      Option(ExpressionEncoder[String]()) :: Nil).genCode(ctx)
+      Option(resolvedEncoder[String]()) :: Nil).genCode(ctx)
     assert(ctx.inlinedMutableStates.isEmpty)
   }
 
@@ -66,7 +72,7 @@ class ScalaUDFSuite extends SparkFunSuite with ExpressionEvalHelper {
         (a: java.math.BigDecimal) => a.multiply(new java.math.BigDecimal(100)),
         DecimalType.SYSTEM_DEFAULT,
         Literal(BigDecimal("12345678901234567890.123")) :: Nil,
-        Option(ExpressionEncoder[java.math.BigDecimal]()) :: Nil)
+        Option(resolvedEncoder[java.math.BigDecimal]()) :: Nil)
       val e1 = intercept[ArithmeticException](udf.eval())
       assert(e1.getMessage.contains("cannot be represented as Decimal"))
       val e2 = intercept[SparkException] {
@@ -79,7 +85,7 @@ class ScalaUDFSuite extends SparkFunSuite with ExpressionEvalHelper {
         (a: java.math.BigDecimal) => a.multiply(new java.math.BigDecimal(100)),
         DecimalType.SYSTEM_DEFAULT,
         Literal(BigDecimal("12345678901234567890.123")) :: Nil,
-        Option(ExpressionEncoder[java.math.BigDecimal]()) :: Nil)
+        Option(resolvedEncoder[java.math.BigDecimal]()) :: Nil)
       checkEvaluation(udf, null)
     }
   }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SchemaPruningSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SchemaPruningSuite.scala
index c04f59ebb1b1b..c67a9622b61fd 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SchemaPruningSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SchemaPruningSuite.scala
@@ -18,28 +18,36 @@
 package org.apache.spark.sql.catalyst.expressions
 
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.plans.SQLHelper
+import org.apache.spark.sql.internal.SQLConf.CASE_SENSITIVE
 import org.apache.spark.sql.types._
 
-class SchemaPruningSuite extends SparkFunSuite {
-  test("prune schema by the requested fields") {
-    def testPrunedSchema(
-        schema: StructType,
-        requestedFields: StructField*): Unit = {
-      val requestedRootFields = requestedFields.map { f =>
-        // `derivedFromAtt` doesn't affect the result of pruned schema.
-        SchemaPruning.RootField(field = f, derivedFromAtt = true)
-      }
-      val expectedSchema = SchemaPruning.pruneDataSchema(schema, requestedRootFields)
-      assert(expectedSchema == StructType(requestedFields))
+class SchemaPruningSuite extends SparkFunSuite with SQLHelper {
+  private def testPrunedSchema(
+      schema: StructType,
+      requestedFields: Seq[StructField],
+      expectedSchema: StructType): Unit = {
+    val requestedRootFields = requestedFields.map { f =>
+      // `derivedFromAtt` doesn't affect the result of pruned schema.
+      SchemaPruning.RootField(field = f, derivedFromAtt = true)
     }
+    val prunedSchema = SchemaPruning.pruneDataSchema(schema, requestedRootFields)
+    assert(prunedSchema === expectedSchema)
+  }
 
-    testPrunedSchema(StructType.fromDDL("a int, b int"), StructField("a", IntegerType))
-    testPrunedSchema(StructType.fromDDL("a int, b int"), StructField("b", IntegerType))
+  test("prune schema by the requested fields") {
+    testPrunedSchema(
+      StructType.fromDDL("a int, b int"),
+      Seq(StructField("a", IntegerType)),
+      StructType.fromDDL("a int, b int"))
 
     val structOfStruct = StructType.fromDDL("a struct<a:int, b:int>, b int")
-    testPrunedSchema(structOfStruct, StructField("a", StructType.fromDDL("a int, b int")))
-    testPrunedSchema(structOfStruct, StructField("b", IntegerType))
-    testPrunedSchema(structOfStruct, StructField("a", StructType.fromDDL("b int")))
+    testPrunedSchema(structOfStruct,
+      Seq(StructField("a", StructType.fromDDL("a int")), StructField("b", IntegerType)),
+      StructType.fromDDL("a struct<a:int>, b int"))
+    testPrunedSchema(structOfStruct,
+      Seq(StructField("a", StructType.fromDDL("a int"))),
+      StructType.fromDDL("a struct<a:int>, b int"))
 
     val arrayOfStruct = StructField("a", ArrayType(StructType.fromDDL("a int, b int, c string")))
     val mapOfStruct = StructField("d", MapType(StructType.fromDDL("a int, b int, c string"),
@@ -49,14 +57,90 @@ class SchemaPruningSuite extends SparkFunSuite {
       arrayOfStruct :: StructField("b", structOfStruct) :: StructField("c", IntegerType) ::
         mapOfStruct :: Nil)
 
-    testPrunedSchema(complexStruct, StructField("a", ArrayType(StructType.fromDDL("b int"))),
-      StructField("b", StructType.fromDDL("a int")))
     testPrunedSchema(complexStruct,
-      StructField("a", ArrayType(StructType.fromDDL("b int, c string"))),
-      StructField("b", StructType.fromDDL("b int")))
+      Seq(StructField("a", ArrayType(StructType.fromDDL("b int"))),
+        StructField("b", StructType.fromDDL("a int"))),
+      StructType(
+        StructField("a", ArrayType(StructType.fromDDL("b int"))) ::
+          StructField("b", StructType.fromDDL("a int")) ::
+          StructField("c", IntegerType) ::
+          mapOfStruct :: Nil))
+    testPrunedSchema(complexStruct,
+      Seq(StructField("a", ArrayType(StructType.fromDDL("b int, c string"))),
+        StructField("b", StructType.fromDDL("b int"))),
+      StructType(
+        StructField("a", ArrayType(StructType.fromDDL("b int, c string"))) ::
+          StructField("b", StructType.fromDDL("b int")) ::
+          StructField("c", IntegerType) ::
+          mapOfStruct :: Nil))
 
     val selectFieldInMap = StructField("d", MapType(StructType.fromDDL("a int, b int"),
       StructType.fromDDL("e int, f string")))
-    testPrunedSchema(complexStruct, StructField("c", IntegerType), selectFieldInMap)
+    testPrunedSchema(complexStruct,
+      Seq(StructField("c", IntegerType), selectFieldInMap),
+      StructType(
+        arrayOfStruct ::
+          StructField("b", structOfStruct) ::
+          StructField("c", IntegerType) ::
+          selectFieldInMap :: Nil))
+  }
+
+  test("SPARK-35096: test case insensitivity of pruned schema") {
+    val upperCaseSchema = StructType.fromDDL("A struct<A:int, B:int>, B int")
+    val lowerCaseSchema = StructType.fromDDL("a struct<a:int, b:int>, b int")
+    val upperCaseRequestedFields = Seq(StructField("A", StructType.fromDDL("A int")))
+    val lowerCaseRequestedFields = Seq(StructField("a", StructType.fromDDL("a int")))
+
+    Seq(true, false).foreach { isCaseSensitive =>
+      withSQLConf(CASE_SENSITIVE.key -> isCaseSensitive.toString) {
+        if (isCaseSensitive) {
+          testPrunedSchema(
+            upperCaseSchema,
+            upperCaseRequestedFields,
+            StructType.fromDDL("A struct<A:int>, B int"))
+          testPrunedSchema(
+            upperCaseSchema,
+            lowerCaseRequestedFields,
+            upperCaseSchema)
+
+          testPrunedSchema(
+            lowerCaseSchema,
+            upperCaseRequestedFields,
+            lowerCaseSchema)
+          testPrunedSchema(
+            lowerCaseSchema,
+            lowerCaseRequestedFields,
+            StructType.fromDDL("a struct<a:int>, b int"))
+        } else {
+          Seq(upperCaseRequestedFields, lowerCaseRequestedFields).foreach { requestedFields =>
+            testPrunedSchema(
+              upperCaseSchema,
+              requestedFields,
+              StructType.fromDDL("A struct<A:int>, B int"))
+          }
+
+          Seq(upperCaseRequestedFields, lowerCaseRequestedFields).foreach { requestedFields =>
+            testPrunedSchema(
+              lowerCaseSchema,
+              requestedFields,
+              StructType.fromDDL("a struct<a:int>, b int"))
+          }
+        }
+      }
+    }
+  }
+
+  test("SPARK-35232: getRootFields/pruneDataSchema should retain attribute metadata") {
+    val metadata = new MetadataBuilder().putString("foo", "bar").build()
+    val attr = AttributeReference("my_attr", IntegerType, metadata = metadata)()
+
+    val rootFields = SchemaPruning.getRootFields(attr)
+    assert(rootFields.length == 1)
+    val field = rootFields.head.field
+    assert(field.metadata.getString("foo") == "bar")
+
+    val schema = StructType(Seq(field))
+    val prunedSchema = SchemaPruning.pruneDataSchema(schema, rootFields)
+    assert(prunedSchema.head.metadata.getString("foo") == "bar")
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SelectedFieldSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SelectedFieldSuite.scala
index 3c826e812b5cc..cf5463be1faa1 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SelectedFieldSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SelectedFieldSuite.scala
@@ -254,13 +254,13 @@ class SelectedFieldSuite extends AnalysisTest {
     StructField("col3", ArrayType(StructType(
       StructField("field1", StructType(
         StructField("subfield1", IntegerType, nullable = false) :: Nil))
-        :: Nil), containsNull = false), nullable = false)
+        :: Nil), containsNull = true), nullable = false)
   }
 
   testSelect(arrayWithStructAndMap, "col3.field2['foo'] as foo") {
     StructField("col3", ArrayType(StructType(
       StructField("field2", MapType(StringType, IntegerType, valueContainsNull = false))
-        :: Nil), containsNull = false), nullable = false)
+        :: Nil), containsNull = true), nullable = false)
   }
 
   //  |-- col1: string (nullable = false)
@@ -471,7 +471,7 @@ class SelectedFieldSuite extends AnalysisTest {
   testSelect(mapWithArrayOfStructKey, "map_keys(col2)[0].field1 as foo") {
     StructField("col2", MapType(
       ArrayType(StructType(
-        StructField("field1", StringType) :: Nil), containsNull = false),
+        StructField("field1", StringType) :: Nil), containsNull = true),
       ArrayType(StructType(
         StructField("field3", StructType(
           StructField("subfield3", IntegerType) ::
@@ -482,7 +482,7 @@ class SelectedFieldSuite extends AnalysisTest {
     StructField("col2", MapType(
       ArrayType(StructType(
         StructField("field2", StructType(
-          StructField("subfield1", IntegerType) :: Nil)) :: Nil), containsNull = false),
+          StructField("subfield1", IntegerType) :: Nil)) :: Nil), containsNull = true),
       ArrayType(StructType(
         StructField("field3", StructType(
           StructField("subfield3", IntegerType) ::
@@ -534,7 +534,7 @@ class SelectedFieldSuite extends AnalysisTest {
   private def unapplySelect(expr: String, relation: LocalRelation) = {
     val parsedExpr = parseAsCatalystExpression(Seq(expr)).head
     val select = relation.select(parsedExpr)
-    val analyzed = caseSensitiveAnalyzer.execute(select)
+    val analyzed = getAnalyzer.execute(select)
     SelectedField.unapply(analyzed.expressions.head)
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
index f18364d844ce1..d25240fec13d1 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
@@ -18,9 +18,9 @@
 package org.apache.spark.sql.catalyst.expressions
 
 import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
@@ -28,7 +28,7 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   test("concat") {
     def testConcat(inputs: String*): Unit = {
       val expected = if (inputs.contains(null)) null else inputs.mkString
-      checkEvaluation(Concat(inputs.map(Literal.create(_, StringType))), expected, EmptyRow)
+      checkEvaluation(Concat(inputs.map(Literal.create(_, StringType))), expected)
     }
 
     testConcat()
@@ -50,7 +50,7 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   test("SPARK-22498: Concat should not generate codes beyond 64KB") {
     val N = 5000
     val strs = (1 to N).map(x => s"s$x")
-    checkEvaluation(Concat(strs.map(Literal.create(_, StringType))), strs.mkString, EmptyRow)
+    checkEvaluation(Concat(strs.map(Literal.create(_, StringType))), strs.mkString)
   }
 
   test("SPARK-22771 Check Concat.checkInputDataTypes results") {
@@ -73,7 +73,7 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
         case s: String => Literal.create(s, StringType)
       }
       val sepExpr = Literal.create(sep, StringType)
-      checkEvaluation(ConcatWs(sepExpr +: inputExprs), expected, EmptyRow)
+      checkEvaluation(ConcatWs(sepExpr +: inputExprs), expected)
     }
 
     // scalastyle:off
@@ -99,12 +99,12 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     val sepExpr = Literal.create("#", StringType)
     val strings1 = (1 to N).map(x => s"s$x")
     val inputsExpr1 = strings1.map(Literal.create(_, StringType))
-    checkEvaluation(ConcatWs(sepExpr +: inputsExpr1), strings1.mkString("#"), EmptyRow)
+    checkEvaluation(ConcatWs(sepExpr +: inputsExpr1), strings1.mkString("#"))
 
     val strings2 = (1 to N).map(x => Seq(s"s$x"))
     val inputsExpr2 = strings2.map(Literal.create(_, ArrayType(StringType)))
     checkEvaluation(
-      ConcatWs(sepExpr +: inputsExpr2), strings2.map(s => s(0)).mkString("#"), EmptyRow)
+      ConcatWs(sepExpr +: inputsExpr2), strings2.map(s => s(0)).mkString("#"))
   }
 
   test("elt") {
@@ -118,7 +118,7 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     testElt(null, 1, null, "world")
     testElt(null, null, "hello", "world")
 
-    // Invalid ranages
+    // Invalid ranges
     testElt(null, 3, "hello", "world")
     testElt(null, 0, "hello", "world")
     testElt(null, -1, "hello", "world")
@@ -236,6 +236,10 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
       "xample",
       row)
 
+    // Substring with from negative position with negative length
+    checkEvaluation(Substring(s, Literal.create(-1207959552, IntegerType),
+      Literal.create(-1207959552, IntegerType)), "", row)
+
     val s_notNull = 'a.string.notNull.at(0)
 
     assert(Substring(s, Literal.create(0, IntegerType), Literal.create(2, IntegerType)).nullable)
@@ -345,23 +349,23 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     // scalastyle:off
     // non ascii characters are not allowed in the code, so we disable the scalastyle here.
     checkEvaluation(
-      Decode(Encode(Literal("大千世界"), Literal("UTF-16LE")), Literal("UTF-16LE")), "大千世界")
+      StringDecode(Encode(Literal("大千世界"), Literal("UTF-16LE")), Literal("UTF-16LE")), "大千世界")
     checkEvaluation(
-      Decode(Encode(a, Literal("utf-8")), Literal("utf-8")), "大千世界", create_row("大千世界"))
+      StringDecode(Encode(a, Literal("utf-8")), Literal("utf-8")), "大千世界", create_row("大千世界"))
     checkEvaluation(
-      Decode(Encode(a, Literal("utf-8")), Literal("utf-8")), "", create_row(""))
+      StringDecode(Encode(a, Literal("utf-8")), Literal("utf-8")), "", create_row(""))
     // scalastyle:on
     checkEvaluation(Encode(a, Literal("utf-8")), null, create_row(null))
     checkEvaluation(Encode(Literal.create(null, StringType), Literal("utf-8")), null)
     checkEvaluation(Encode(a, Literal.create(null, StringType)), null, create_row(""))
 
-    checkEvaluation(Decode(b, Literal("utf-8")), null, create_row(null))
-    checkEvaluation(Decode(Literal.create(null, BinaryType), Literal("utf-8")), null)
-    checkEvaluation(Decode(b, Literal.create(null, StringType)), null, create_row(null))
+    checkEvaluation(StringDecode(b, Literal("utf-8")), null, create_row(null))
+    checkEvaluation(StringDecode(Literal.create(null, BinaryType), Literal("utf-8")), null)
+    checkEvaluation(StringDecode(b, Literal.create(null, StringType)), null, create_row(null))
 
     // Test escaping of charset
     GenerateUnsafeProjection.generate(Encode(a, Literal("\"quote")) :: Nil)
-    GenerateUnsafeProjection.generate(Decode(b, Literal("\"quote")) :: Nil)
+    GenerateUnsafeProjection.generate(StringDecode(b, Literal("\"quote")) :: Nil)
   }
 
   test("initcap unit test") {
@@ -546,6 +550,13 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     // scalastyle:off
     // non ascii characters are not allowed in the source code, so we disable the scalastyle.
     checkEvaluation(StringTranslate(Literal("花花世界"), Literal("花界"), Literal("ab")), "aa世b")
+    // test for unicode characters whose code point >= 0x10000
+    checkEvaluation(
+      StringTranslate(
+        Literal("\uD840\uDC0Bxyza\uD867\uDE49c123b\uD842\uDFB7\uD867\uDE3D"),
+        Literal("\uD867\uDE3Da\uD842\uDFB7b\uD840\uDC0Bc\uD867\uDE49c"),
+        Literal("1\uD83C\uDF3B2\uD83C\uDF37\uD83D\uDC15\uD83D\uDC08\uD83C\uDF38")),
+      "\uD83D\uDC15xyz\uD83C\uDF3B\uD83C\uDF38\uD83D\uDC08123\uD83C\uDF3721")
     // scalastyle:on
   }
 
@@ -939,6 +950,20 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     GenerateUnsafeProjection.generate(ParseUrl(Seq(Literal("\"quote"), Literal("\"quote"))) :: Nil)
   }
 
+  test("SPARK-33468: ParseUrl in ANSI mode should fail if input string is not a valid url") {
+    withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
+      val msg = intercept[IllegalArgumentException] {
+        evaluateWithoutCodegen(
+          ParseUrl(Seq("https://a.b.c/index.php?params1=a|b&params2=x", "HOST")))
+      }.getMessage
+      assert(msg.contains("Find an invaild url string"))
+    }
+    withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
+      checkEvaluation(
+        ParseUrl(Seq("https://a.b.c/index.php?params1=a|b&params2=x", "HOST")), null)
+    }
+  }
+
   test("Sentences") {
     val nullString = Literal.create(null, StringType)
     checkEvaluation(Sentences(nullString, nullString, nullString), null)
@@ -964,4 +989,34 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     GenerateUnsafeProjection.generate(
       Sentences(Literal("\"quote"), Literal("\"quote"), Literal("\"quote")) :: Nil)
   }
+
+  test("SPARK-33386: elt ArrayIndexOutOfBoundsException") {
+    Seq(true, false).foreach { ansiEnabled =>
+      withSQLConf(SQLConf.ANSI_ENABLED.key -> ansiEnabled.toString) {
+        var expr: Expression = Elt(Seq(Literal(4), Literal("123"), Literal("456")))
+        if (ansiEnabled) {
+          val errMsg = "Invalid index: 4, numElements: 2"
+          checkExceptionInExpression[Exception](expr, errMsg)
+        } else {
+          checkEvaluation(expr, null)
+        }
+
+        expr = Elt(Seq(Literal(0), Literal("123"), Literal("456")))
+        if (ansiEnabled) {
+          val errMsg = "Invalid index: 0, numElements: 2"
+          checkExceptionInExpression[Exception](expr, errMsg)
+        } else {
+          checkEvaluation(expr, null)
+        }
+
+        expr = Elt(Seq(Literal(-1), Literal("123"), Literal("456")))
+        if (ansiEnabled) {
+          val errMsg = "Invalid index: -1, numElements: 2"
+          checkExceptionInExpression[Exception](expr, errMsg)
+        } else {
+          checkEvaluation(expr, null)
+        }
+      }
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SubExprEvaluationRuntimeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SubExprEvaluationRuntimeSuite.scala
new file mode 100644
index 0000000000000..f8dca266a62d4
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SubExprEvaluationRuntimeSuite.scala
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.catalyst.expressions
+
+import org.apache.spark.SparkFunSuite
+
+class SubExprEvaluationRuntimeSuite extends SparkFunSuite {
+
+  test("Evaluate ExpressionProxy should create cached result") {
+    val runtime = new SubExprEvaluationRuntime(1)
+    val proxy = ExpressionProxy(Literal(1), 0, runtime)
+    assert(runtime.cache.size() == 0)
+    proxy.eval()
+    assert(runtime.cache.size() == 1)
+    assert(runtime.cache.get(proxy) == ResultProxy(1))
+  }
+
+  test("SubExprEvaluationRuntime cannot exceed configured max entries") {
+    val runtime = new SubExprEvaluationRuntime(2)
+    assert(runtime.cache.size() == 0)
+
+    val proxy1 = ExpressionProxy(Literal(1), 0, runtime)
+    proxy1.eval()
+    assert(runtime.cache.size() == 1)
+    assert(runtime.cache.get(proxy1) == ResultProxy(1))
+
+    val proxy2 = ExpressionProxy(Literal(2), 1, runtime)
+    proxy2.eval()
+    assert(runtime.cache.size() == 2)
+    assert(runtime.cache.get(proxy2) == ResultProxy(2))
+
+    val proxy3 = ExpressionProxy(Literal(3), 2, runtime)
+    proxy3.eval()
+    assert(runtime.cache.size() == 2)
+    assert(runtime.cache.get(proxy3) == ResultProxy(3))
+  }
+
+  test("setInput should empty cached result") {
+    val runtime = new SubExprEvaluationRuntime(2)
+    val proxy1 = ExpressionProxy(Literal(1), 0, runtime)
+    assert(runtime.cache.size() == 0)
+    proxy1.eval()
+    assert(runtime.cache.size() == 1)
+    assert(runtime.cache.get(proxy1) == ResultProxy(1))
+
+    val proxy2 = ExpressionProxy(Literal(2), 1, runtime)
+    proxy2.eval()
+    assert(runtime.cache.size() == 2)
+    assert(runtime.cache.get(proxy2) == ResultProxy(2))
+
+    runtime.setInput()
+    assert(runtime.cache.size() == 0)
+  }
+
+  test("Wrap ExpressionProxy on subexpressions") {
+    val runtime = new SubExprEvaluationRuntime(1)
+
+    val one = Literal(1)
+    val two = Literal(2)
+    val mul = Multiply(one, two)
+    val mul2 = Multiply(mul, mul)
+    val sqrt = Sqrt(mul2)
+    val sum = Add(mul2, sqrt)
+
+    //  ( (one * two) * (one * two) ) + sqrt( (one * two) * (one * two) )
+    val proxyExpressions = runtime.proxyExpressions(Seq(sum))
+    val proxys = proxyExpressions.flatMap(_.collect {
+      case p: ExpressionProxy => p
+    })
+    // ( (one * two) * (one * two) )
+    assert(proxys.size == 2)
+    assert(proxys.forall(_.child == mul2))
+  }
+
+  test("ExpressionProxy won't be on non deterministic") {
+    val runtime = new SubExprEvaluationRuntime(1)
+
+    val sum = Add(Rand(0), Rand(0))
+    val proxys = runtime.proxyExpressions(Seq(sum, sum)).flatMap(_.collect {
+      case p: ExpressionProxy => p
+    })
+    assert(proxys.isEmpty)
+  }
+
+  test("SubExprEvaluationRuntime should wrap semantically equal exprs") {
+    val runtime = new SubExprEvaluationRuntime(1)
+
+    val one = Literal(1)
+    val two = Literal(2)
+    def mul: (Literal, Literal) => Expression =
+      (left: Literal, right: Literal) => Multiply(left, right)
+
+    val mul2_1 = Multiply(mul(one, two), mul(one, two))
+    val mul2_2 = Multiply(mul(one, two), mul(one, two))
+
+    val sqrt = Sqrt(mul2_1)
+    val sum = Add(mul2_2, sqrt)
+    val proxyExpressions = runtime.proxyExpressions(Seq(sum))
+    val proxys = proxyExpressions.flatMap(_.collect {
+      case p: ExpressionProxy => p
+    })
+    // ( (one * two) * (one * two) )
+    assert(proxys.size == 2)
+    assert(proxys.forall(_.child.semanticEquals(mul2_1)))
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SubexpressionEliminationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SubexpressionEliminationSuite.scala
index 1fa185cc77ebb..9bfe69b1709d2 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SubexpressionEliminationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SubexpressionEliminationSuite.scala
@@ -17,10 +17,11 @@
 package org.apache.spark.sql.catalyst.expressions
 
 import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
-import org.apache.spark.sql.types.{DataType, IntegerType}
+import org.apache.spark.sql.catalyst.expressions.codegen._
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types.{BinaryType, DataType, IntegerType}
 
-class SubexpressionEliminationSuite extends SparkFunSuite {
+class SubexpressionEliminationSuite extends SparkFunSuite with ExpressionEvalHelper {
   test("Semantic equals and hash") {
     val a: AttributeReference = AttributeReference("name", IntegerType)()
     val id = {
@@ -146,24 +147,173 @@ class SubexpressionEliminationSuite extends SparkFunSuite {
     equivalence.addExprTree(add)
     // the `two` inside `fallback` should not be added
     assert(equivalence.getAllEquivalentExprs.count(_.size > 1) == 0)
-    assert(equivalence.getAllEquivalentExprs.count(_.size == 1) == 3)  // add, two, explode
+    assert(equivalence.getAllEquivalentExprs.count(_.size == 1) == 3) // add, two, explode
   }
 
-  test("Children of conditional expressions") {
-    val condition = And(Literal(true), Literal(false))
+  test("Children of conditional expressions: If") {
     val add = Add(Literal(1), Literal(2))
-    val ifExpr = If(condition, add, add)
+    val condition = GreaterThan(add, Literal(3))
 
-    val equivalence = new EquivalentExpressions
-    equivalence.addExprTree(ifExpr)
-    // the `add` inside `If` should not be added
-    assert(equivalence.getAllEquivalentExprs.count(_.size > 1) == 0)
-    // only ifExpr and its predicate expression
-    assert(equivalence.getAllEquivalentExprs.count(_.size == 1) == 2)
+    val ifExpr1 = If(condition, add, add)
+    val equivalence1 = new EquivalentExpressions
+    equivalence1.addExprTree(ifExpr1)
+
+    // `add` is in both two branches of `If` and predicate.
+    assert(equivalence1.getAllEquivalentExprs.count(_.size == 2) == 1)
+    assert(equivalence1.getAllEquivalentExprs.filter(_.size == 2).head == Seq(add, add))
+    // one-time expressions: only ifExpr and its predicate expression
+    assert(equivalence1.getAllEquivalentExprs.count(_.size == 1) == 2)
+    assert(equivalence1.getAllEquivalentExprs.filter(_.size == 1).contains(Seq(ifExpr1)))
+    assert(equivalence1.getAllEquivalentExprs.filter(_.size == 1).contains(Seq(condition)))
+
+    // Repeated `add` is only in one branch, so we don't count it.
+    val ifExpr2 = If(condition, Add(Literal(1), Literal(3)), Add(add, add))
+    val equivalence2 = new EquivalentExpressions
+    equivalence2.addExprTree(ifExpr2)
+
+    assert(equivalence2.getAllEquivalentExprs.count(_.size > 1) == 0)
+    assert(equivalence2.getAllEquivalentExprs.count(_.size == 1) == 3)
+
+    val ifExpr3 = If(condition, ifExpr1, ifExpr1)
+    val equivalence3 = new EquivalentExpressions
+    equivalence3.addExprTree(ifExpr3)
+
+    // `add`: 2, `condition`: 2
+    assert(equivalence3.getAllEquivalentExprs.count(_.size == 2) == 2)
+    assert(equivalence3.getAllEquivalentExprs.filter(_.size == 2).contains(Seq(add, add)))
+    assert(
+      equivalence3.getAllEquivalentExprs.filter(_.size == 2).contains(Seq(condition, condition)))
+
+    // `ifExpr1`, `ifExpr3`
+    assert(equivalence3.getAllEquivalentExprs.count(_.size == 1) == 2)
+    assert(equivalence3.getAllEquivalentExprs.filter(_.size == 1).contains(Seq(ifExpr1)))
+    assert(equivalence3.getAllEquivalentExprs.filter(_.size == 1).contains(Seq(ifExpr3)))
+  }
+
+  test("Children of conditional expressions: CaseWhen") {
+    val add1 = Add(Literal(1), Literal(2))
+    val add2 = Add(Literal(2), Literal(3))
+    val conditions1 = (GreaterThan(add2, Literal(3)), add1) ::
+      (GreaterThan(add2, Literal(4)), add1) ::
+      (GreaterThan(add2, Literal(5)), add1) :: Nil
+
+    val caseWhenExpr1 = CaseWhen(conditions1, None)
+    val equivalence1 = new EquivalentExpressions
+    equivalence1.addExprTree(caseWhenExpr1)
+
+    // `add2` is repeatedly in all conditions.
+    assert(equivalence1.getAllEquivalentExprs.count(_.size == 2) == 1)
+    assert(equivalence1.getAllEquivalentExprs.filter(_.size == 2).head == Seq(add2, add2))
+
+    val conditions2 = (GreaterThan(add1, Literal(3)), add1) ::
+      (GreaterThan(add2, Literal(4)), add1) ::
+      (GreaterThan(add2, Literal(5)), add1) :: Nil
+
+    val caseWhenExpr2 = CaseWhen(conditions2, None)
+    val equivalence2 = new EquivalentExpressions
+    equivalence2.addExprTree(caseWhenExpr2)
+
+    // `add1` is repeatedly in all branch values, and first predicate.
+    assert(equivalence2.getAllEquivalentExprs.count(_.size == 2) == 1)
+    assert(equivalence2.getAllEquivalentExprs.filter(_.size == 2).head == Seq(add1, add1))
+
+    // Negative case. `add1` or `add2` is not commonly used in all predicates/branch values.
+    val conditions3 = (GreaterThan(add1, Literal(3)), add2) ::
+      (GreaterThan(add2, Literal(4)), add1) ::
+      (GreaterThan(add2, Literal(5)), add1) :: Nil
+
+    val caseWhenExpr3 = CaseWhen(conditions3, None)
+    val equivalence3 = new EquivalentExpressions
+    equivalence3.addExprTree(caseWhenExpr3)
+    assert(equivalence3.getAllEquivalentExprs.count(_.size == 2) == 0)
+  }
+
+  test("Children of conditional expressions: Coalesce") {
+    val add1 = Add(Literal(1), Literal(2))
+    val add2 = Add(Literal(2), Literal(3))
+    val conditions1 = GreaterThan(add2, Literal(3)) ::
+      GreaterThan(add2, Literal(4)) ::
+      GreaterThan(add2, Literal(5)) :: Nil
+
+    val coalesceExpr1 = Coalesce(conditions1)
+    val equivalence1 = new EquivalentExpressions
+    equivalence1.addExprTree(coalesceExpr1)
+
+    // `add2` is repeatedly in all conditions.
+    assert(equivalence1.getAllEquivalentExprs.count(_.size == 2) == 1)
+    assert(equivalence1.getAllEquivalentExprs.filter(_.size == 2).head == Seq(add2, add2))
+
+    // Negative case. `add1` and `add2` both are not used in all branches.
+    val conditions2 = GreaterThan(add1, Literal(3)) ::
+      GreaterThan(add2, Literal(4)) ::
+      GreaterThan(add2, Literal(5)) :: Nil
+
+    val coalesceExpr2 = Coalesce(conditions2)
+    val equivalence2 = new EquivalentExpressions
+    equivalence2.addExprTree(coalesceExpr2)
+
+    assert(equivalence2.getAllEquivalentExprs.count(_.size == 2) == 0)
+  }
+
+  test("SPARK-34723: Correct parameter type for subexpression elimination under whole-stage") {
+    withSQLConf(SQLConf.CODEGEN_METHOD_SPLIT_THRESHOLD.key -> "1") {
+      val str = BoundReference(0, BinaryType, false)
+      val pos = BoundReference(1, IntegerType, false)
+
+      val substr = new Substring(str, pos)
+
+      val add = Add(Length(substr), Literal(1))
+      val add2 = Add(Length(substr), Literal(2))
+
+      val ctx = new CodegenContext()
+      val exprs = Seq(add, add2)
+
+      val oneVar = ctx.freshVariable("str", BinaryType)
+      val twoVar = ctx.freshVariable("pos", IntegerType)
+      ctx.addMutableState("byte[]", oneVar, forceInline = true, useFreshName = false)
+      ctx.addMutableState("int", twoVar, useFreshName = false)
+
+      ctx.INPUT_ROW = null
+      ctx.currentVars = Seq(
+        ExprCode(TrueLiteral, oneVar),
+        ExprCode(TrueLiteral, twoVar))
+
+      val subExprs = ctx.subexpressionEliminationForWholeStageCodegen(exprs)
+      ctx.withSubExprEliminationExprs(subExprs.states) {
+        exprs.map(_.genCode(ctx))
+      }
+      val subExprsCode = subExprs.codes.mkString("\n")
+
+      val codeBody = s"""
+        public java.lang.Object generate(Object[] references) {
+          return new TestCode(references);
+        }
+
+        class TestCode {
+          ${ctx.declareMutableStates()}
+
+          public TestCode(Object[] references) {
+          }
+
+          public void initialize(int partitionIndex) {
+            ${subExprsCode}
+          }
+
+          ${ctx.declareAddedFunctions()}
+        }
+      """
+
+      val code = CodeFormatter.stripOverlappingComments(
+        new CodeAndComment(codeBody, ctx.getPlaceHolderToComments()))
+
+      CodeGenerator.compile(code)
+    }
   }
 }
 
 case class CodegenFallbackExpression(child: Expression)
   extends UnaryExpression with CodegenFallback {
   override def dataType: DataType = child.dataType
+  override protected def withNewChildInternal(newChild: Expression): CodegenFallbackExpression =
+    copy(child = newChild)
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TryCastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TryCastSuite.scala
new file mode 100644
index 0000000000000..76ce96705c44f
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TryCastSuite.scala
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions
+
+import scala.reflect.ClassTag
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.types.{DataType, IntegerType}
+
+class TryCastSuite extends AnsiCastSuiteBase {
+  override protected def cast(v: Any, targetType: DataType, timeZoneId: Option[String]) = {
+    v match {
+      case lit: Expression => TryCast(lit, targetType, timeZoneId)
+      case _ => TryCast(Literal(v), targetType, timeZoneId)
+    }
+  }
+
+  override def isAlwaysNullable: Boolean = true
+
+  override protected def setConfigurationHint: String = ""
+
+  override def checkExceptionInExpression[T <: Throwable : ClassTag](
+      expression: => Expression,
+      inputRow: InternalRow,
+      expectedErrMsg: String): Unit = {
+    checkEvaluation(expression, null, inputRow)
+  }
+
+  override def checkCastToBooleanError(l: Literal, to: DataType, tryCastResult: Any): Unit = {
+    checkEvaluation(cast(l, to), tryCastResult, InternalRow(l.value))
+  }
+
+  override def checkCastToNumericError(l: Literal, to: DataType, tryCastResult: Any): Unit = {
+    checkEvaluation(cast(l, to), tryCastResult, InternalRow(l.value))
+  }
+
+  test("try_cast: to_string") {
+    assert(TryCast(Literal("1"), IntegerType).toString == "try_cast(1 as int)")
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TryEvalSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TryEvalSuite.scala
new file mode 100644
index 0000000000000..928077523d7e3
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TryEvalSuite.scala
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions
+
+import org.apache.spark.SparkFunSuite
+
+class TryEvalSuite extends SparkFunSuite with ExpressionEvalHelper {
+  test("try_add") {
+    Seq(
+      (1, 1, 2),
+      (Int.MaxValue, 1, null),
+      (Int.MinValue, -1, null)
+    ).foreach { case (a, b, expected) =>
+      val left = Literal(a)
+      val right = Literal(b)
+      val input = TryEval(Add(left, right, failOnError = true))
+      checkEvaluation(input, expected)
+    }
+  }
+
+  test("try_divide") {
+    Seq(
+      (3.0, 2.0, 1.5),
+      (1.0, 0.0, null),
+      (-1.0, 0.0, null)
+    ).foreach { case (a, b, expected) =>
+      val left = Literal(a)
+      val right = Literal(b)
+      val input = TryEval(Divide(left, right, failOnError = true))
+      checkEvaluation(input, expected)
+    }
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/UnsafeRowConverterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/UnsafeRowConverterSuite.scala
index 0149f0deb8d2b..220728fcaa2f0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/UnsafeRowConverterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/UnsafeRowConverterSuite.scala
@@ -20,7 +20,8 @@ package org.apache.spark.sql.catalyst.expressions
 import java.nio.charset.StandardCharsets
 import java.sql.{Date, Timestamp}
 
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.InternalRow
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentileSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentileSuite.scala
index 303fa137d8925..a03b5b2b19f27 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentileSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentileSuite.scala
@@ -30,7 +30,7 @@ import org.apache.spark.sql.catalyst.expressions.aggregate.ApproximatePercentile
 import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
 import org.apache.spark.sql.catalyst.util.{ArrayData, QuantileSummaries}
 import org.apache.spark.sql.catalyst.util.QuantileSummaries.Stats
-import org.apache.spark.sql.types.{ArrayType, Decimal, DecimalType, DoubleType, FloatType, IntegerType, IntegralType, LongType}
+import org.apache.spark.sql.types.{ArrayType, Decimal, DecimalType, DoubleType, FloatType, IntegerType, IntegralType}
 import org.apache.spark.util.SizeEstimator
 
 class ApproximatePercentileSuite extends SparkFunSuite {
@@ -170,12 +170,12 @@ class ApproximatePercentileSuite extends SparkFunSuite {
     val defaultAccuracy = ApproximatePercentile.DEFAULT_PERCENTILE_ACCURACY
     // sql, single percentile
     assertEqual(
-      s"percentile_approx(`a`, 0.5D, $defaultAccuracy)",
+      s"percentile_approx(a, 0.5D, $defaultAccuracy)",
       new ApproximatePercentile("a".attr, percentageExpression = Literal(0.5D)).sql: String)
 
     // sql, array of percentile
     assertEqual(
-      s"percentile_approx(`a`, array(0.25D, 0.5D, 0.75D), $defaultAccuracy)",
+      s"percentile_approx(a, array(0.25D, 0.5D, 0.75D), $defaultAccuracy)",
       new ApproximatePercentile(
         "a".attr,
         percentageExpression = CreateArray(Seq(0.25D, 0.5D, 0.75D).map(Literal(_)))
@@ -183,13 +183,13 @@ class ApproximatePercentileSuite extends SparkFunSuite {
 
     // sql(isDistinct = false), single percentile
     assertEqual(
-      s"percentile_approx(`a`, 0.5D, $defaultAccuracy)",
+      s"percentile_approx(a, 0.5D, $defaultAccuracy)",
       new ApproximatePercentile("a".attr, percentageExpression = Literal(0.5D))
         .sql(isDistinct = false))
 
     // sql(isDistinct = false), array of percentile
     assertEqual(
-      s"percentile_approx(`a`, array(0.25D, 0.5D, 0.75D), $defaultAccuracy)",
+      s"percentile_approx(a, array(0.25D, 0.5D, 0.75D), $defaultAccuracy)",
       new ApproximatePercentile(
         "a".attr,
         percentageExpression = CreateArray(Seq(0.25D, 0.5D, 0.75D).map(Literal(_)))
@@ -197,13 +197,13 @@ class ApproximatePercentileSuite extends SparkFunSuite {
 
     // sql(isDistinct = true), single percentile
     assertEqual(
-      s"percentile_approx(DISTINCT `a`, 0.5D, $defaultAccuracy)",
+      s"percentile_approx(DISTINCT a, 0.5D, $defaultAccuracy)",
       new ApproximatePercentile("a".attr, percentageExpression = Literal(0.5D))
         .sql(isDistinct = true))
 
     // sql(isDistinct = true), array of percentile
     assertEqual(
-      s"percentile_approx(DISTINCT `a`, array(0.25D, 0.5D, 0.75D), $defaultAccuracy)",
+      s"percentile_approx(DISTINCT a, array(0.25D, 0.5D, 0.75D), $defaultAccuracy)",
       new ApproximatePercentile(
         "a".attr,
         percentageExpression = CreateArray(Seq(0.25D, 0.5D, 0.75D).map(Literal(_)))
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/FirstLastTestSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/FirstLastTestSuite.scala
new file mode 100644
index 0000000000000..bb6672e1046da
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/FirstLastTestSuite.scala
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.catalyst.expressions.aggregate
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Literal}
+import org.apache.spark.sql.types.IntegerType
+
+class FirstLastTestSuite extends SparkFunSuite {
+  val input = AttributeReference("input", IntegerType, nullable = true)()
+  val evaluator = DeclarativeAggregateEvaluator(Last(input, false), Seq(input))
+  val evaluatorIgnoreNulls = DeclarativeAggregateEvaluator(Last(input, true), Seq(input))
+
+  test("empty buffer") {
+    assert(evaluator.initialize() === InternalRow(null, false))
+  }
+
+  test("update") {
+    val result = evaluator.update(
+      InternalRow(1),
+      InternalRow(9),
+      InternalRow(-1))
+    assert(result === InternalRow(-1, true))
+  }
+
+  test("update - ignore nulls") {
+    val result1 = evaluatorIgnoreNulls.update(
+      InternalRow(null),
+      InternalRow(9),
+      InternalRow(null))
+    assert(result1 === InternalRow(9, true))
+
+    val result2 = evaluatorIgnoreNulls.update(
+      InternalRow(null),
+      InternalRow(null))
+    assert(result2 === InternalRow(null, false))
+  }
+
+  test("merge") {
+    // Empty merge
+    val p0 = evaluator.initialize()
+    assert(evaluator.merge(p0) === InternalRow(null, false))
+
+    // Single merge
+    val p1 = evaluator.update(InternalRow(1), InternalRow(-99))
+    assert(evaluator.merge(p1) === p1)
+
+    // Multiple merges.
+    val p2 = evaluator.update(InternalRow(2), InternalRow(10))
+    assert(evaluator.merge(p1, p2) === p2)
+
+    // Empty partitions (p0 is empty)
+    assert(evaluator.merge(p1, p0, p2) === p2)
+    assert(evaluator.merge(p2, p1, p0) === p1)
+  }
+
+  test("merge - ignore nulls") {
+    // Multi merges
+    val p1 = evaluatorIgnoreNulls.update(InternalRow(1), InternalRow(null))
+    val p2 = evaluatorIgnoreNulls.update(InternalRow(null), InternalRow(null))
+    assert(evaluatorIgnoreNulls.merge(p1, p2) === p1)
+  }
+
+  test("eval") {
+    // Null Eval
+    assert(evaluator.eval(InternalRow(null, true)) === InternalRow(null))
+    assert(evaluator.eval(InternalRow(null, false)) === InternalRow(null))
+
+    // Empty Eval
+    val p0 = evaluator.initialize()
+    assert(evaluator.eval(p0) === InternalRow(null))
+
+    // Update - Eval
+    val p1 = evaluator.update(InternalRow(1), InternalRow(-99))
+    assert(evaluator.eval(p1) === InternalRow(-99))
+
+    // Update - Merge - Eval
+    val p2 = evaluator.update(InternalRow(2), InternalRow(10))
+    val m1 = evaluator.merge(p1, p0, p2)
+    assert(evaluator.eval(m1) === InternalRow(10))
+
+    // Update - Merge - Eval (empty partition at the end)
+    val m2 = evaluator.merge(p2, p1, p0)
+    assert(evaluator.eval(m2) === InternalRow(-99))
+  }
+
+  test("eval - ignore nulls") {
+    // Update - Merge - Eval
+    val p1 = evaluatorIgnoreNulls.update(InternalRow(1), InternalRow(null))
+    val p2 = evaluatorIgnoreNulls.update(InternalRow(null), InternalRow(null))
+    val m1 = evaluatorIgnoreNulls.merge(p1, p2)
+    assert(evaluatorIgnoreNulls.eval(m1) === InternalRow(1))
+  }
+
+  test("SPARK-32344: correct error handling for a type mismatch") {
+    val msg1 = intercept[AnalysisException] {
+      new First(input, Literal(1, IntegerType))
+    }.getMessage
+    assert(msg1.contains("The second argument in first should be a boolean literal"))
+    val msg2 = intercept[AnalysisException] {
+      new Last(input, Literal(1, IntegerType))
+    }.getMessage
+    assert(msg2.contains("The second argument in last should be a boolean literal"))
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlusSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlusSuite.scala
index 98fd04c9cca91..1afccea5aef15 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlusSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlusSuite.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.catalyst.expressions.aggregate
 
+import java.lang.{Double => JDouble}
 import java.util.Random
 
 import scala.collection.mutable
@@ -24,7 +25,7 @@ import scala.collection.mutable
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{BoundReference, SpecificInternalRow}
-import org.apache.spark.sql.types.{DataType, IntegerType}
+import org.apache.spark.sql.types.{DataType, DoubleType, IntegerType}
 
 class HyperLogLogPlusPlusSuite extends SparkFunSuite {
 
@@ -153,4 +154,25 @@ class HyperLogLogPlusPlusSuite extends SparkFunSuite {
     // Check if the buffers are equal.
     assert(buffer2 == buffer1a, "Buffers should be equal")
   }
+
+  test("SPARK-32110: add 0.0 and -0.0") {
+    val (hll, input, buffer) = createEstimator(0.05, DoubleType)
+    input.setDouble(0, 0.0)
+    hll.update(buffer, input)
+    input.setDouble(0, -0.0)
+    hll.update(buffer, input)
+    evaluateEstimate(hll, buffer, 1);
+  }
+
+  test("SPARK-32110: add NaN") {
+    val (hll, input, buffer) = createEstimator(0.05, DoubleType)
+    input.setDouble(0, Double.NaN)
+    hll.update(buffer, input)
+    val specialNaN = JDouble.longBitsToDouble(0x7ff1234512345678L)
+    assert(JDouble.isNaN(specialNaN))
+    assert(JDouble.doubleToRawLongBits(Double.NaN) != JDouble.doubleToRawLongBits(specialNaN))
+    input.setDouble(0, specialNaN)
+    hll.update(buffer, input)
+    evaluateEstimate(hll, buffer, 1);
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/LastTestSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/LastTestSuite.scala
deleted file mode 100644
index ba36bc074e154..0000000000000
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/LastTestSuite.scala
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.catalyst.expressions.aggregate
-
-import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Literal}
-import org.apache.spark.sql.types.IntegerType
-
-class LastTestSuite extends SparkFunSuite {
-  val input = AttributeReference("input", IntegerType, nullable = true)()
-  val evaluator = DeclarativeAggregateEvaluator(Last(input, Literal(false)), Seq(input))
-  val evaluatorIgnoreNulls = DeclarativeAggregateEvaluator(Last(input, Literal(true)), Seq(input))
-
-  test("empty buffer") {
-    assert(evaluator.initialize() === InternalRow(null, false))
-  }
-
-  test("update") {
-    val result = evaluator.update(
-      InternalRow(1),
-      InternalRow(9),
-      InternalRow(-1))
-    assert(result === InternalRow(-1, true))
-  }
-
-  test("update - ignore nulls") {
-    val result1 = evaluatorIgnoreNulls.update(
-      InternalRow(null),
-      InternalRow(9),
-      InternalRow(null))
-    assert(result1 === InternalRow(9, true))
-
-    val result2 = evaluatorIgnoreNulls.update(
-      InternalRow(null),
-      InternalRow(null))
-    assert(result2 === InternalRow(null, false))
-  }
-
-  test("merge") {
-    // Empty merge
-    val p0 = evaluator.initialize()
-    assert(evaluator.merge(p0) === InternalRow(null, false))
-
-    // Single merge
-    val p1 = evaluator.update(InternalRow(1), InternalRow(-99))
-    assert(evaluator.merge(p1) === p1)
-
-    // Multiple merges.
-    val p2 = evaluator.update(InternalRow(2), InternalRow(10))
-    assert(evaluator.merge(p1, p2) === p2)
-
-    // Empty partitions (p0 is empty)
-    assert(evaluator.merge(p1, p0, p2) === p2)
-    assert(evaluator.merge(p2, p1, p0) === p1)
-  }
-
-  test("merge - ignore nulls") {
-    // Multi merges
-    val p1 = evaluatorIgnoreNulls.update(InternalRow(1), InternalRow(null))
-    val p2 = evaluatorIgnoreNulls.update(InternalRow(null), InternalRow(null))
-    assert(evaluatorIgnoreNulls.merge(p1, p2) === p1)
-  }
-
-  test("eval") {
-    // Null Eval
-    assert(evaluator.eval(InternalRow(null, true)) === InternalRow(null))
-    assert(evaluator.eval(InternalRow(null, false)) === InternalRow(null))
-
-    // Empty Eval
-    val p0 = evaluator.initialize()
-    assert(evaluator.eval(p0) === InternalRow(null))
-
-    // Update - Eval
-    val p1 = evaluator.update(InternalRow(1), InternalRow(-99))
-    assert(evaluator.eval(p1) === InternalRow(-99))
-
-    // Update - Merge - Eval
-    val p2 = evaluator.update(InternalRow(2), InternalRow(10))
-    val m1 = evaluator.merge(p1, p0, p2)
-    assert(evaluator.eval(m1) === InternalRow(10))
-
-    // Update - Merge - Eval (empty partition at the end)
-    val m2 = evaluator.merge(p2, p1, p0)
-    assert(evaluator.eval(m2) === InternalRow(-99))
-  }
-
-  test("eval - ignore nulls") {
-    // Update - Merge - Eval
-    val p1 = evaluatorIgnoreNulls.update(InternalRow(1), InternalRow(null))
-    val p2 = evaluatorIgnoreNulls.update(InternalRow(null), InternalRow(null))
-    val m1 = evaluatorIgnoreNulls.merge(p1, p2)
-    assert(evaluatorIgnoreNulls.eval(m1) === InternalRow(1))
-  }
-}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/PercentileSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/PercentileSuite.scala
index 972db7fa30a91..fa874079e6de3 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/PercentileSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/PercentileSuite.scala
@@ -66,20 +66,20 @@ class PercentileSuite extends SparkFunSuite {
     // Test with row with frequency. Second and third columns are frequency in Int and Long
     val countForFrequencyTest = 1000
     val rowsWithFrequency = (1 to countForFrequencyTest).map(x => Seq(x, x):+ x.toLong)
-    val expectedPercentilesWithFrquency = Seq(1.0, 500.0, 707.0, 866.0, 1000.0)
+    val expectedPercentilesWithFrequency = Seq(1.0, 500.0, 707.0, 866.0, 1000.0)
 
     val frequencyExpressionInt = BoundReference(1, IntegerType, nullable = false)
     val aggInt = new Percentile(childExpression, percentageExpression, frequencyExpressionInt)
-    runTest(aggInt, rowsWithFrequency, expectedPercentilesWithFrquency)
+    runTest(aggInt, rowsWithFrequency, expectedPercentilesWithFrequency)
 
     val frequencyExpressionLong = BoundReference(2, LongType, nullable = false)
     val aggLong = new Percentile(childExpression, percentageExpression, frequencyExpressionLong)
-    runTest(aggLong, rowsWithFrequency, expectedPercentilesWithFrquency)
+    runTest(aggLong, rowsWithFrequency, expectedPercentilesWithFrequency)
 
     // Run test with Flatten data
     val flattenRows = (1 to countForFrequencyTest).flatMap(current =>
       (1 to current).map(y => current )).map(Seq(_))
-    runTest(agg, flattenRows, expectedPercentilesWithFrquency)
+    runTest(agg, flattenRows, expectedPercentilesWithFrequency)
   }
 
   private def runTest(agg: Percentile,
@@ -171,7 +171,7 @@ class PercentileSuite extends SparkFunSuite {
       val percentile = new Percentile(child, percentage)
       assertEqual(percentile.checkInputDataTypes(),
         TypeCheckFailure(s"argument 1 requires numeric type, however, " +
-            s"'`a`' is of ${dataType.simpleString} type."))
+            s"'a' is of ${dataType.simpleString} type."))
     }
 
     val invalidFrequencyDataTypes = Seq(FloatType, DoubleType, BooleanType,
@@ -185,7 +185,7 @@ class PercentileSuite extends SparkFunSuite {
       val percentile = new Percentile(child, percentage, frq)
       assertEqual(percentile.checkInputDataTypes(),
         TypeCheckFailure(s"argument 1 requires numeric type, however, " +
-            s"'`a`' is of ${dataType.simpleString} type."))
+            s"'a' is of ${dataType.simpleString} type."))
     }
 
     for(dataType <- validDataTypes;
@@ -195,7 +195,7 @@ class PercentileSuite extends SparkFunSuite {
       val percentile = new Percentile(child, percentage, frq)
       assertEqual(percentile.checkInputDataTypes(),
         TypeCheckFailure(s"argument 3 requires integral type, however, " +
-            s"'`frq`' is of ${frequencyType.simpleString} type."))
+            s"'frq' is of ${frequencyType.simpleString} type."))
     }
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ProductAggSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ProductAggSuite.scala
new file mode 100644
index 0000000000000..e5bbe4f2ff272
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ProductAggSuite.scala
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.catalyst.expressions.aggregate
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.AttributeReference
+import org.apache.spark.sql.types.DoubleType
+
+
+class ProductAggSuite extends SparkFunSuite {
+  val input = AttributeReference("product", DoubleType, nullable = true)()
+  val evaluator = DeclarativeAggregateEvaluator(Product(input), Seq(input))
+
+  test("empty buffer") {
+    assert(evaluator.initialize() === InternalRow(null))
+  }
+
+  test("update") {
+    val result = evaluator.update(
+      InternalRow(-2.0),
+      InternalRow(3.0),
+      InternalRow(-5.0),
+      InternalRow(7.0))
+    assert(result === InternalRow(210.0))
+  }
+
+  test("update - with nulls") {
+    val result1 = evaluator.update(
+      InternalRow(null),
+      InternalRow(11.0),
+      InternalRow(null),
+      InternalRow(13.0))
+    assert(result1 === InternalRow(143.0))
+
+    val result2 = evaluator.update(
+      InternalRow(null),
+      InternalRow(null))
+    assert(result2 === InternalRow(null))
+  }
+
+  test("update - with specials") {
+    val result1 = evaluator.update(
+      InternalRow(Double.NaN),
+      InternalRow(2.0))
+    assert(result1 === InternalRow(Double.NaN))
+
+    val result2 = evaluator.update(
+      InternalRow(3.0),
+      InternalRow(Double.PositiveInfinity))
+    assert(result2 === InternalRow(Double.PositiveInfinity))
+
+    val result3 = evaluator.update(
+      InternalRow(Double.NegativeInfinity),
+      InternalRow(5.0))
+    assert(result3 === InternalRow(Double.NegativeInfinity))
+
+    val result4 = evaluator.update(
+      InternalRow(7.0),
+      InternalRow(Double.PositiveInfinity),
+      InternalRow(null))
+    assert(result4 === InternalRow(Double.PositiveInfinity))
+
+    val result5 = evaluator.update(
+      InternalRow(Double.NaN),
+      InternalRow(Double.PositiveInfinity),
+      InternalRow(1.0))
+    assert(result5 === InternalRow(Double.NaN))
+
+    val result6 = evaluator.update(
+      InternalRow(Double.NegativeInfinity),
+      InternalRow(2.0),
+      InternalRow(Double.NaN))
+    assert(result6 === InternalRow(Double.NaN))
+  }
+
+  test("merge") {
+    // Empty
+    val p0 = evaluator.initialize()
+    assert(evaluator.merge(p0) === InternalRow(null))
+
+    // Singleton
+    val p1 = evaluator.update(InternalRow(7.0), InternalRow(11.0))
+    assert(evaluator.merge(p1) === p1)
+
+    // Pair
+    val p2 = evaluator.update(InternalRow(17.0), InternalRow(19.0))
+    assert(evaluator.merge(p1, p2) === InternalRow((7 * 11 * 17 * 19).toDouble))
+    assert(evaluator.merge(p1, p2) === evaluator.merge(p2, p1))
+
+    // Mixtures with empty
+    assert(evaluator.merge(p1, p0, p2) === evaluator.merge(p1, p2))
+    assert(evaluator.merge(p0, p2, p1) === evaluator.merge(p1, p2))
+  }
+
+  test("merge - with nulls") {
+    val p0 = evaluator.update(InternalRow(null), InternalRow(null))
+    val p1 = evaluator.update(InternalRow(5.0), InternalRow(null))
+    val p2 = evaluator.update(InternalRow(null), InternalRow(7.0))
+
+    assert(evaluator.merge(p0, p0) === p0)
+    assert(evaluator.merge(p0, p1) === p1)
+    assert(evaluator.merge(p2, p0) === p2)
+
+    assert(evaluator.merge(p2, p1, p0) === InternalRow((5 * 7).toDouble))
+  }
+
+  test("merge - with specials") {
+    val p0 = evaluator.update(InternalRow(Double.NaN), InternalRow(1.0))
+    val p1 = evaluator.update(InternalRow(Double.PositiveInfinity), InternalRow(1.0))
+    val p2 = evaluator.update(InternalRow(Double.NegativeInfinity), InternalRow(1.0))
+    val p3 = evaluator.update(InternalRow(null), InternalRow(1.0))
+
+    assert(evaluator.merge(p0, p0) === p0)
+    assert(evaluator.merge(p0, p1) === p0)
+    assert(evaluator.merge(p0, p2) === p0)
+    assert(evaluator.merge(p0, p3) === p0)
+
+    assert(evaluator.merge(p1, p1) === p1)
+    assert(evaluator.merge(p1, p2) === p2)
+    assert(evaluator.merge(p1, p3) === p1)
+
+    assert(evaluator.merge(p2, p2) === p1)
+    assert(evaluator.merge(p2, p3) === p2)
+
+    assert(evaluator.merge(p3, p3) === p3)
+  }
+
+  test("eval") {
+    // Null
+    assert(evaluator.eval(InternalRow(null)) === InternalRow(null))
+
+    // Empty
+    assert(evaluator.eval(evaluator.initialize()) === InternalRow(null))
+
+    // Non-trivial
+    val p1 = evaluator.update(InternalRow(2.0), InternalRow(3.0))
+    val p2 = evaluator.update(InternalRow(5.0), InternalRow(7.0), InternalRow(11.0))
+    val m12 = evaluator.merge(p1, p2)
+    assert(evaluator.eval(m12) === InternalRow(2.0 * 3 * 5 * 7 * 11))
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/BufferHolderSparkSubmitSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/BufferHolderSparkSubmitSuite.scala
index d2862c8f41d1b..972a832255155 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/BufferHolderSparkSubmitSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/BufferHolderSparkSubmitSuite.scala
@@ -17,7 +17,8 @@
 
 package org.apache.spark.sql.catalyst.expressions.codegen
 
-import org.scalatest.{Assertions, BeforeAndAfterEach, Matchers}
+import org.scalatest.{Assertions, BeforeAndAfterEach}
+import org.scalatest.matchers.must.Matchers
 
 import org.apache.spark.{SparkFunSuite, TestUtils}
 import org.apache.spark.deploy.SparkSubmitSuite
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeBlockSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeBlockSuite.scala
index 67e3bc69543e8..9d4c5986300c5 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeBlockSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeBlockSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.expressions.codegen
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
-import org.apache.spark.sql.types.{BooleanType, IntegerType}
+import org.apache.spark.sql.types.IntegerType
 
 class CodeBlockSuite extends SparkFunSuite {
 
@@ -115,7 +115,7 @@ class CodeBlockSuite extends SparkFunSuite {
     assert(exprValues === Set(isNull1, value1, isNull2, value2, literal))
   }
 
-  test("Throws exception when interpolating unexcepted object in code block") {
+  test("Throws exception when interpolating unexpected object in code block") {
     val obj = Tuple2(1, 1)
     val e = intercept[IllegalArgumentException] {
       code"$obj"
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodegenSubexpressionEliminationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodegenSubexpressionEliminationSuite.scala
new file mode 100644
index 0000000000000..471f25356887f
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodegenSubexpressionEliminationSuite.scala
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions.codegen
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.types.{BooleanType, DataType}
+
+/**
+ * A test suite that makes sure code generation handles sub-expression elimination correctly.
+ */
+class CodegenSubexpressionEliminationSuite extends SparkFunSuite {
+
+  test("SPARK-32903: GeneratePredicate should eliminate sub-expressions") {
+    Seq(true, false).foreach { useSubexprElimination =>
+      val leaf1 = ExprWithEvaluatedState()
+      val leaf2 = ExprWithEvaluatedState()
+      val leaf3 = ExprWithEvaluatedState()
+      val leaf4 = ExprWithEvaluatedState()
+
+      val cond = Or(And(leaf1, leaf2), And(leaf3, leaf4))
+      val instance = GeneratePredicate.generate(cond, useSubexprElimination = useSubexprElimination)
+      instance.initialize(0)
+      assert(instance.eval(null) === false)
+
+      if (useSubexprElimination) {
+        // When we do sub-expression elimination, Spark thought left and right side of
+        // the `Or` expression are the same. So only left side was evaluated, and Spark
+        // reused the evaluation for right side.
+        assert(leaf1.evaluated == true)
+        assert(leaf2.evaluated == false)
+        assert(leaf3.evaluated == false)
+        assert(leaf4.evaluated == false)
+      } else {
+        assert(leaf1.evaluated == true)
+        assert(leaf2.evaluated == false)
+        assert(leaf3.evaluated == true)
+        assert(leaf4.evaluated == false)
+      }
+    }
+  }
+
+}
+
+/**
+ * An expression with evaluated state so we can know whether it is evaluated.
+ */
+case class ExprWithEvaluatedState() extends LeafExpression with CodegenFallback {
+  var evaluated: Boolean = false
+  override def eval(input: InternalRow): Any = {
+    evaluated = true
+    false
+  }
+
+  override def nullable: Boolean = false
+  override def dataType: DataType = BooleanType
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/GeneratedProjectionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/GeneratedProjectionSuite.scala
index 4c9bcfe8f93a6..180665e653727 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/GeneratedProjectionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/GeneratedProjectionSuite.scala
@@ -23,13 +23,14 @@ import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.util.GenericArrayData
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
 /**
  * A test suite for generated projections
  */
-class GeneratedProjectionSuite extends SparkFunSuite {
+class GeneratedProjectionSuite extends SparkFunSuite with ExpressionEvalHelper {
 
   test("generated projections on wider table") {
     val N = 1000
@@ -246,4 +247,50 @@ class GeneratedProjectionSuite extends SparkFunSuite {
     val row2 = mutableProj(result)
     assert(result === row2)
   }
+
+  test("SPARK-33473: subexpression elimination for interpreted SafeProjection") {
+    Seq("true", "false").foreach { enabled =>
+      withSQLConf(
+        SQLConf.SUBEXPRESSION_ELIMINATION_ENABLED.key -> enabled,
+        SQLConf.CODEGEN_FACTORY_MODE.key -> CodegenObjectFactoryMode.NO_CODEGEN.toString) {
+        val one = BoundReference(0, DoubleType, true)
+        val two = BoundReference(1, DoubleType, true)
+
+        val mul = Multiply(one, two)
+        val mul2 = Multiply(mul, mul)
+        val sqrt = Sqrt(mul2)
+        val sum = Add(mul2, sqrt)
+
+        val proj = SafeProjection.create(Seq(sum))
+        val result = (d1: Double, d2: Double) =>
+          ((d1 * d2) * (d1 * d2)) + Math.sqrt((d1 * d2) * (d1 * d2))
+
+        val inputRows = Seq(
+          InternalRow.fromSeq(Seq(1.0, 2.0)),
+          InternalRow.fromSeq(Seq(2.0, 3.0)),
+          InternalRow.fromSeq(Seq(1.0, null)),
+          InternalRow.fromSeq(Seq(null, 2.0)),
+          InternalRow.fromSeq(Seq(3.0, 4.0)),
+          InternalRow.fromSeq(Seq(null, null))
+        )
+        val expectedResults = Seq(
+          result(1.0, 2.0),
+          result(2.0, 3.0),
+          null,
+          null,
+          result(3.0, 4.0),
+          null
+        )
+
+        inputRows.zip(expectedResults).foreach { case (inputRow, expected) =>
+          val projRow = proj.apply(inputRow)
+          if (expected != null) {
+            assert(projRow.getDouble(0) == expected)
+          } else {
+            assert(projRow.isNullAt(0))
+          }
+        }
+      }
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JacksonGeneratorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JacksonGeneratorSuite.scala
index 2bb948ec24fb3..4b8693cf7fd53 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JacksonGeneratorSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JacksonGeneratorSuite.scala
@@ -26,8 +26,8 @@ import org.apache.spark.sql.types._
 
 class JacksonGeneratorSuite extends SparkFunSuite {
 
-  val gmtId = DateTimeUtils.TimeZoneGMT.getID
-  val option = new JSONOptions(Map.empty, gmtId)
+  val utcId = DateTimeUtils.TimeZoneUTC.getID
+  val option = new JSONOptions(Map.empty, utcId)
 
   test("initial with StructType and write out a row") {
     val dataType = StructType(StructField("a", IntegerType) :: Nil)
@@ -45,7 +45,7 @@ class JacksonGeneratorSuite extends SparkFunSuite {
     val input = InternalRow(null)
     val writer = new CharArrayWriter()
     val allowNullOption =
-      new JSONOptions(Map("ignoreNullFields" -> "false"), gmtId)
+      new JSONOptions(Map("ignoreNullFields" -> "false"), utcId)
     val gen = new JacksonGenerator(dataType, writer, allowNullOption)
     gen.write(input)
     gen.flush()
@@ -59,7 +59,7 @@ class JacksonGeneratorSuite extends SparkFunSuite {
     val input = InternalRow(InternalRow(null))
     val writer = new CharArrayWriter()
     val allowNullOption =
-      new JSONOptions(Map("ignoreNullFields" -> "false"), gmtId)
+      new JSONOptions(Map("ignoreNullFields" -> "false"), utcId)
     val gen = new JacksonGenerator(dataType, writer, allowNullOption)
     gen.write(input)
     gen.flush()
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JacksonParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JacksonParserSuite.scala
new file mode 100644
index 0000000000000..587e22e787b87
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JacksonParserSuite.scala
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.json
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.sources.{EqualTo, Filter, StringStartsWith}
+import org.apache.spark.sql.types.StructType
+import org.apache.spark.unsafe.types.UTF8String
+
+class JacksonParserSuite extends SparkFunSuite {
+  test("skipping rows using pushdown filters") {
+    def check(
+      input: String = """{"i":1, "s": "a"}""",
+      schema: StructType = StructType.fromDDL("i INTEGER"),
+      filters: Seq[Filter],
+      expected: Seq[InternalRow]): Unit = {
+      val options = new JSONOptions(Map.empty[String, String], "GMT", "")
+      val parser = new JacksonParser(schema, options, false, filters)
+      val createParser = CreateJacksonParser.string _
+      val actual = parser.parse(input, createParser, UTF8String.fromString)
+      assert(actual === expected)
+    }
+
+    check(filters = Seq(), expected = Seq(InternalRow(1)))
+    check(filters = Seq(EqualTo("i", 1)), expected = Seq(InternalRow(1)))
+    check(filters = Seq(EqualTo("i", 2)), expected = Seq.empty)
+    check(
+      schema = StructType.fromDDL("s STRING"),
+      filters = Seq(StringStartsWith("s", "b")),
+      expected = Seq.empty)
+    check(
+      schema = StructType.fromDDL("i INTEGER, s STRING"),
+      filters = Seq(StringStartsWith("s", "a")),
+      expected = Seq(InternalRow(1, UTF8String.fromString("a"))))
+    check(
+      input = """{"i":1,"s": "a", "d": 3.14}""",
+      schema = StructType.fromDDL("i INTEGER, d DOUBLE"),
+      filters = Seq(EqualTo("d", 3.14)),
+      expected = Seq(InternalRow(1, 3.14)))
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonFiltersSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonFiltersSuite.scala
new file mode 100644
index 0000000000000..82f5e71d67964
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonFiltersSuite.scala
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.json
+
+import org.apache.spark.sql.catalyst.{StructFilters, StructFiltersSuite}
+import org.apache.spark.sql.sources
+import org.apache.spark.sql.types.StructType
+
+class JsonFiltersSuite extends StructFiltersSuite {
+  override def createFilters(filters: Seq[sources.Filter], schema: StructType): StructFilters = {
+    new JsonFilters(filters, schema)
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonInferSchemaSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonInferSchemaSuite.scala
index bce917c80f93c..8290b38e33934 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonInferSchemaSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonInferSchemaSuite.scala
@@ -35,22 +35,29 @@ class JsonInferSchemaSuite extends SparkFunSuite with SQLHelper {
     assert(inferSchema.inferField(parser) === expectedType)
   }
 
-  def checkTimestampType(pattern: String, json: String): Unit = {
-    checkType(Map("timestampFormat" -> pattern), json, TimestampType)
+  def checkTimestampType(pattern: String, json: String, inferTimestamp: Boolean): Unit = {
+    checkType(
+      Map("timestampFormat" -> pattern, "inferTimestamp" -> inferTimestamp.toString),
+      json,
+      if (inferTimestamp) TimestampType else StringType)
   }
 
   test("inferring timestamp type") {
-    Seq("legacy", "corrected").foreach { legacyParserPolicy =>
-      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
-        checkTimestampType("yyyy", """{"a": "2018"}""")
-        checkTimestampType("yyyy=MM", """{"a": "2018=12"}""")
-        checkTimestampType("yyyy MM dd", """{"a": "2018 12 02"}""")
-        checkTimestampType(
-          "yyyy-MM-dd'T'HH:mm:ss.SSS",
-          """{"a": "2018-12-02T21:04:00.123"}""")
-        checkTimestampType(
-          "yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX",
-          """{"a": "2018-12-02T21:04:00.123567+01:00"}""")
+    Seq(true, false).foreach { inferTimestamp =>
+      Seq("legacy", "corrected").foreach { legacyParserPolicy =>
+        withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
+          checkTimestampType("yyyy", """{"a": "2018"}""", inferTimestamp)
+          checkTimestampType("yyyy=MM", """{"a": "2018=12"}""", inferTimestamp)
+          checkTimestampType("yyyy MM dd", """{"a": "2018 12 02"}""", inferTimestamp)
+          checkTimestampType(
+            "yyyy-MM-dd'T'HH:mm:ss.SSS",
+            """{"a": "2018-12-02T21:04:00.123"}""",
+            inferTimestamp)
+          checkTimestampType(
+            "yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX",
+            """{"a": "2018-12-02T21:04:00.123567+01:00"}""",
+            inferTimestamp)
+        }
       }
     }
   }
@@ -71,16 +78,19 @@ class JsonInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("skip decimal type inferring") {
-    Seq("legacy", "corrected").foreach { legacyParserPolicy =>
-      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
-        checkType(
-          options = Map(
-            "prefersDecimal" -> "false",
-            "timestampFormat" -> "yyyyMMdd.HHmmssSSS"
-          ),
-          json = """{"a": "20181202.210400123"}""",
-          dt = TimestampType
-        )
+    Seq(true, false).foreach { inferTimestamp =>
+      Seq("legacy", "corrected").foreach { legacyParserPolicy =>
+        withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
+          checkType(
+            options = Map(
+              "prefersDecimal" -> "false",
+              "timestampFormat" -> "yyyyMMdd.HHmmssSSS",
+              "inferTimestamp" -> inferTimestamp.toString
+            ),
+            json = """{"a": "20181202.210400123"}""",
+            dt = if (inferTimestamp) TimestampType else StringType
+          )
+        }
       }
     }
   }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/AggregateOptimizeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/AggregateOptimizeSuite.scala
index f8ddc93597070..8984bad479a6b 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/AggregateOptimizeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/AggregateOptimizeSuite.scala
@@ -17,21 +17,16 @@
 
 package org.apache.spark.sql.catalyst.optimizer
 
-import org.apache.spark.sql.catalyst.analysis.{Analyzer, EmptyFunctionRegistry}
-import org.apache.spark.sql.catalyst.catalog.{InMemoryCatalog, SessionCatalog}
+import org.apache.spark.sql.catalyst.analysis.AnalysisTest
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.expressions.Literal
-import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan}
 import org.apache.spark.sql.catalyst.rules.RuleExecutor
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.{CASE_SENSITIVE, GROUP_BY_ORDINAL}
 
-class AggregateOptimizeSuite extends PlanTest {
-  override val conf = new SQLConf().copy(CASE_SENSITIVE -> false, GROUP_BY_ORDINAL -> false)
-  val catalog = new SessionCatalog(new InMemoryCatalog, EmptyFunctionRegistry, conf)
-  val analyzer = new Analyzer(catalog, conf)
+class AggregateOptimizeSuite extends AnalysisTest {
+  val analyzer = getAnalyzer
 
   object Optimize extends RuleExecutor[LogicalPlan] {
     val batches = Batch("Aggregate", FixedPoint(100),
@@ -51,11 +46,14 @@ class AggregateOptimizeSuite extends PlanTest {
   }
 
   test("do not remove all grouping expressions if they are all literals") {
-    val query = testRelation.groupBy(Literal("1"), Literal(1) + Literal(2))(sum('b))
-    val optimized = Optimize.execute(analyzer.execute(query))
-    val correctAnswer = analyzer.execute(testRelation.groupBy(Literal(0))(sum('b)))
+    withSQLConf(CASE_SENSITIVE.key -> "false", GROUP_BY_ORDINAL.key -> "false") {
+      val analyzer = getAnalyzer
+      val query = testRelation.groupBy(Literal("1"), Literal(1) + Literal(2))(sum('b))
+      val optimized = Optimize.execute(analyzer.execute(query))
+      val correctAnswer = analyzer.execute(testRelation.groupBy(Literal(0))(sum('b)))
 
-    comparePlans(optimized, correctAnswer)
+      comparePlans(optimized, correctAnswer)
+    }
   }
 
   test("Remove aliased literals") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala
index 9c71cc8e0d291..c02691848c8f0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala
@@ -119,9 +119,7 @@ class BinaryComparisonSimplificationSuite extends PlanTest with PredicateHelper
 
     // GetStructField with different names are semantically equal; thus, `EqualTo(fieldA1, fieldA2)`
     // will be optimized to `TrueLiteral` by `SimplifyBinaryComparison`.
-    val originalQuery = nonNullableRelation
-        .where(EqualTo(fieldA1, fieldA2))
-        .analyze
+    val originalQuery = nonNullableRelation.where(EqualTo(fieldA1, fieldA2))
 
     val optimized = Optimize.execute(originalQuery)
     val correctAnswer = nonNullableRelation.analyze
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala
index a8b8417754b00..07f16f438cc56 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala
@@ -26,7 +26,6 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules._
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.BooleanType
 
 class BooleanSimplificationSuite extends PlanTest with ExpressionEvalHelper with PredicateHelper {
@@ -127,6 +126,54 @@ class BooleanSimplificationSuite extends PlanTest with ExpressionEvalHelper with
       'a === 'b || 'b > 3 && 'a > 3 && 'a < 5)
   }
 
+  test("SPARK-34222: simplify conjunctive predicates (a && b) && a && (a && c) => a && b && c") {
+    checkCondition(('a > 1 && 'b > 2) && 'a > 1 && ('a > 1 && 'c > 3),
+      'a > 1 && ('b > 2 && 'c > 3))
+
+    checkCondition(('a > 1 && 'b > 2) && ('a > 4 && 'b > 5) && ('a > 1 && 'c > 3),
+      ('a > 1 && 'b > 2) && ('c > 3 && 'a > 4) && 'b > 5)
+
+    checkCondition(
+      'a > 1 && 'b > 3 && ('a > 1 && 'b > 3 && ('a > 1 && 'b > 3 && 'c > 1)),
+      'a > 1 && 'b > 3 && 'c > 1)
+
+    checkCondition(
+      ('a > 1 || 'b > 3) && (('a > 1 || 'b > 3) && 'd > 0 && (('a > 1 || 'b > 3) && 'c > 1)),
+      ('a > 1 || 'b > 3) && 'd > 0 && 'c > 1)
+
+    checkCondition(
+      'a > 1 && 'b > 2 && 'a > 1 && 'c > 3,
+      'a > 1 && 'b > 2 && 'c > 3)
+
+    checkCondition(
+      ('a > 1 && 'b > 3 && 'a > 1) || ('a > 1 && 'b > 3 && 'a > 1 && 'c > 1),
+      'a > 1 && 'b > 3)
+  }
+
+  test("SPARK-34222: simplify disjunctive predicates (a || b) || a || (a || c) => a || b || c") {
+    checkCondition(('a > 1 || 'b > 2) || 'a > 1 || ('a > 1 || 'c > 3),
+      'a > 1 || 'b > 2 || 'c > 3)
+
+    checkCondition(('a > 1 || 'b > 2) || ('a > 4 || 'b > 5) ||('a > 1 || 'c > 3),
+      ('a > 1 || 'b > 2) || ('a > 4 || 'b > 5) || 'c > 3)
+
+    checkCondition(
+      'a > 1 || 'b > 3 || ('a > 1 || 'b > 3 || ('a > 1 || 'b > 3 || 'c > 1)),
+      'a > 1 || 'b > 3 || 'c > 1)
+
+    checkCondition(
+      ('a > 1 && 'b > 3) || (('a > 1 && 'b > 3) || (('a > 1 && 'b > 3) || 'c > 1)),
+      ('a > 1 && 'b > 3) || 'c > 1)
+
+    checkCondition(
+      'a > 1 || 'b > 2 || 'a > 1 || 'c > 3,
+      'a > 1 || 'b > 2 || 'c > 3)
+
+    checkCondition(
+      ('a > 1 || 'b > 3 || 'a > 1) && ('a > 1 || 'b > 3 || 'a > 1 || 'c > 1 ),
+      'a > 1 || 'b > 3)
+  }
+
   test("e && (!e || f) - not nullable") {
     checkConditionInNotNullableRelation('e && (!'e || 'f ), 'e && 'f)
 
@@ -188,25 +235,23 @@ class BooleanSimplificationSuite extends PlanTest with ExpressionEvalHelper with
     checkCondition(!(('e || 'f) && ('g || 'h)), (!'e && !'f) || (!'g && !'h))
   }
 
-  private val caseInsensitiveConf = new SQLConf().copy(SQLConf.CASE_SENSITIVE -> false)
-  private val caseInsensitiveAnalyzer = new Analyzer(
-    new SessionCatalog(new InMemoryCatalog, EmptyFunctionRegistry, caseInsensitiveConf),
-    caseInsensitiveConf)
+  private val analyzer = new Analyzer(
+    new SessionCatalog(new InMemoryCatalog, EmptyFunctionRegistry))
 
   test("(a && b) || (a && c) => a && (b || c) when case insensitive") {
-    val plan = caseInsensitiveAnalyzer.execute(
+    val plan = analyzer.execute(
       testRelation.where(('a > 2 && 'b > 3) || ('A > 2 && 'b < 5)))
     val actual = Optimize.execute(plan)
-    val expected = caseInsensitiveAnalyzer.execute(
+    val expected = analyzer.execute(
       testRelation.where('a > 2 && ('b > 3 || 'b < 5)))
     comparePlans(actual, expected)
   }
 
   test("(a || b) && (a || c) => a || (b && c) when case insensitive") {
-    val plan = caseInsensitiveAnalyzer.execute(
+    val plan = analyzer.execute(
       testRelation.where(('a > 2 || 'b > 3) && ('A > 2 || 'b < 5)))
     val actual = Optimize.execute(plan)
-    val expected = caseInsensitiveAnalyzer.execute(
+    val expected = analyzer.execute(
       testRelation.where('a > 2 || ('b > 3 && 'b < 5)))
     comparePlans(actual, expected)
   }
@@ -221,14 +266,14 @@ class BooleanSimplificationSuite extends PlanTest with ExpressionEvalHelper with
 
   test("Complementation Laws - null handling") {
     checkCondition('e && !'e,
-      testRelationWithData.where(If('e.isNull, Literal.create(null, BooleanType), false)).analyze)
+      testRelationWithData.where(And(Literal(null, BooleanType), 'e.isNull)).analyze)
     checkCondition(!'e && 'e,
-      testRelationWithData.where(If('e.isNull, Literal.create(null, BooleanType), false)).analyze)
+      testRelationWithData.where(And(Literal(null, BooleanType), 'e.isNull)).analyze)
 
     checkCondition('e || !'e,
-      testRelationWithData.where(If('e.isNull, Literal.create(null, BooleanType), true)).analyze)
+      testRelationWithData.where(Or('e.isNotNull, Literal(null, BooleanType))).analyze)
     checkCondition(!'e || 'e,
-      testRelationWithData.where(If('e.isNull, Literal.create(null, BooleanType), true)).analyze)
+      testRelationWithData.where(Or('e.isNotNull, Literal(null, BooleanType))).analyze)
   }
 
   test("Complementation Laws - negative case") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CheckCartesianProductsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CheckCartesianProductsSuite.scala
index 788fedb3c8e8e..dea2b36ecc844 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CheckCartesianProductsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CheckCartesianProductsSuite.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.catalyst.optimizer
 
-import org.scalatest.Matchers._
+import org.scalatest.matchers.must.Matchers._
 
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.dsl.expressions._
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala
index 5be37318ae6eb..4db58298e1f6c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala
@@ -127,7 +127,7 @@ class ColumnPruningSuite extends PlanTest {
 
         val optimized = Optimize.execute(query)
 
-        val aliases = NestedColumnAliasingSuite.collectGeneratedAliases(optimized)
+        val aliases = NestedColumnAliasingSuite.collectGeneratedAliases(optimized).toSeq
 
         val selectedFields = UnresolvedAttribute("a") +: aliasedExprs(aliases)
         val finalSelectedExprs = Seq(UnresolvedAttribute("a"), $"${aliases(0)}".as("c.d")) ++
@@ -218,30 +218,6 @@ class ColumnPruningSuite extends PlanTest {
     comparePlans(optimized, expected)
   }
 
-  test("Column pruning for ScriptTransformation") {
-    val input = LocalRelation('a.int, 'b.string, 'c.double)
-    val query =
-      ScriptTransformation(
-        Seq('a, 'b),
-        "func",
-        Seq.empty,
-        input,
-        null).analyze
-    val optimized = Optimize.execute(query)
-
-    val expected =
-      ScriptTransformation(
-        Seq('a, 'b),
-        "func",
-        Seq.empty,
-        Project(
-          Seq('a, 'b),
-          input),
-        null).analyze
-
-    comparePlans(optimized, expected)
-  }
-
   test("Column pruning on Filter") {
     val input = LocalRelation('a.int, 'b.string, 'c.double)
     val plan1 = Filter('a > 1, input).analyze
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CombineConcatsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CombineConcatsSuite.scala
index 441c15340a778..997ccb7204965 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CombineConcatsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CombineConcatsSuite.scala
@@ -31,8 +31,10 @@ class CombineConcatsSuite extends PlanTest {
   }
 
   protected def assertEquivalent(e1: Expression, e2: Expression): Unit = {
-    val correctAnswer = Project(Alias(e2, "out")() :: Nil, OneRowRelation()).analyze
-    val actual = Optimize.execute(Project(Alias(e1, "out")() :: Nil, OneRowRelation()).analyze)
+    val correctAnswer = Limit(Literal(1), Project(Alias(e2, "out")() :: Nil, OneRowRelation()))
+      .analyze
+    val actual = Optimize.execute(Limit(Literal(1), Project(Alias(e1, "out")() :: Nil,
+      OneRowRelation())).analyze)
     comparePlans(actual, correctAnswer)
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CombiningLimitsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CombiningLimitsSuite.scala
index b190dd5a7c220..423ff81aaebcc 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CombiningLimitsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CombiningLimitsSuite.scala
@@ -17,9 +17,11 @@
 
 package org.apache.spark.sql.catalyst.optimizer
 
+import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
-import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules._
 
@@ -30,8 +32,8 @@ class CombiningLimitsSuite extends PlanTest {
       Batch("Column Pruning", FixedPoint(100),
         ColumnPruning,
         RemoveNoopOperators) ::
-      Batch("Combine Limit", FixedPoint(10),
-        CombineLimits) ::
+      Batch("Eliminate Limit", FixedPoint(10),
+        EliminateLimits) ::
       Batch("Constant Folding", FixedPoint(10),
         NullPropagation,
         ConstantFolding,
@@ -39,7 +41,16 @@ class CombiningLimitsSuite extends PlanTest {
         SimplifyConditionals) :: Nil
   }
 
-  val testRelation = LocalRelation('a.int, 'b.int, 'c.int)
+  val testRelation = LocalRelation.fromExternalRows(
+    Seq("a".attr.int, "b".attr.int, "c".attr.int),
+    1.to(10).map(_ => Row(1, 2, 3))
+  )
+  val testRelation2 = LocalRelation.fromExternalRows(
+    Seq("x".attr.int, "y".attr.int, "z".attr.int),
+    Seq(Row(1, 2, 3), Row(2, 3, 4))
+  )
+  val testRelation3 = RelationWithoutMaxRows(Seq("i".attr.int))
+  val testRelation4 = LongMaxRelation(Seq("j".attr.int))
 
   test("limits: combines two limits") {
     val originalQuery =
@@ -90,4 +101,151 @@ class CombiningLimitsSuite extends PlanTest {
 
     comparePlans(optimized, correctAnswer)
   }
+
+  test("SPARK-33442: Change Combine Limit to Eliminate limit using max row") {
+    // test child max row <= limit.
+    val query1 = testRelation.select().groupBy()(count(1)).limit(1).analyze
+    val optimized1 = Optimize.execute(query1)
+    val expected1 = testRelation.select().groupBy()(count(1)).analyze
+    comparePlans(optimized1, expected1)
+
+    // test child max row > limit.
+    val query2 = testRelation.select().groupBy()(count(1)).limit(0).analyze
+    val optimized2 = Optimize.execute(query2)
+    comparePlans(optimized2, query2)
+
+    // test child max row is none
+    val query3 = testRelation.select(Symbol("a")).limit(1).analyze
+    val optimized3 = Optimize.execute(query3)
+    comparePlans(optimized3, query3)
+
+    // test sort after limit
+    val query4 = testRelation.select().groupBy()(count(1))
+      .orderBy(count(1).asc).limit(1).analyze
+    val optimized4 = Optimize.execute(query4)
+    // the top project has been removed, so we need optimize expected too
+    val expected4 = Optimize.execute(
+      testRelation.select().groupBy()(count(1)).orderBy(count(1).asc).analyze)
+    comparePlans(optimized4, expected4)
+  }
+
+  test("SPARK-33497: Eliminate Limit if LocalRelation max rows not larger than Limit") {
+    checkPlanAndMaxRow(
+      testRelation.select().limit(10),
+      testRelation.select(),
+      10
+    )
+  }
+
+  test("SPARK-33497: Eliminate Limit if Range max rows not larger than Limit") {
+    checkPlanAndMaxRow(
+      Range(0, 100, 1, None).select().limit(200),
+      Range(0, 100, 1, None).select(),
+      100
+    )
+    checkPlanAndMaxRow(
+      Range(-1, Long.MaxValue, 1, None).select().limit(1),
+      Range(-1, Long.MaxValue, 1, None).select().limit(1),
+      1
+    )
+  }
+
+  test("SPARK-33497: Eliminate Limit if Sample max rows not larger than Limit") {
+    checkPlanAndMaxRow(
+      testRelation.select().sample(0, 0.2, false, 1).limit(10),
+      testRelation.select().sample(0, 0.2, false, 1),
+      10
+    )
+  }
+
+  test("SPARK-33497: Eliminate Limit if Deduplicate max rows not larger than Limit") {
+    checkPlanAndMaxRow(
+      testRelation.deduplicate("a".attr).limit(10),
+      testRelation.deduplicate("a".attr),
+      10
+    )
+  }
+
+  test("SPARK-33497: Eliminate Limit if Repartition max rows not larger than Limit") {
+    checkPlanAndMaxRow(
+      testRelation.repartition(2).limit(10),
+      testRelation.repartition(2),
+      10
+    )
+    checkPlanAndMaxRow(
+      testRelation.distribute("a".attr)(2).limit(10),
+      testRelation.distribute("a".attr)(2),
+      10
+    )
+  }
+
+  test("SPARK-33497: Eliminate Limit if Join max rows not larger than Limit") {
+    Seq(Inner, FullOuter, LeftOuter, RightOuter).foreach { joinType =>
+      checkPlanAndMaxRow(
+        testRelation.join(testRelation2, joinType).limit(20),
+        testRelation.join(testRelation2, joinType),
+        20
+      )
+      checkPlanAndMaxRow(
+        testRelation.join(testRelation2, joinType).limit(10),
+        testRelation.join(testRelation2, joinType).limit(10),
+        10
+      )
+      // without maxRow
+      checkPlanAndMaxRow(
+        testRelation.join(testRelation3, joinType).limit(100),
+        testRelation.join(testRelation3, joinType).limit(100),
+        100
+      )
+      // maxRow is not valid long
+      checkPlanAndMaxRow(
+        testRelation.join(testRelation4, joinType).limit(100),
+        testRelation.join(testRelation4, joinType).limit(100),
+        100
+      )
+    }
+
+    Seq(LeftSemi, LeftAnti).foreach { joinType =>
+      checkPlanAndMaxRow(
+        testRelation.join(testRelation2, joinType).limit(5),
+        testRelation.join(testRelation2.select(), joinType).limit(5),
+        5
+      )
+      checkPlanAndMaxRow(
+        testRelation.join(testRelation2, joinType).limit(10),
+        testRelation.join(testRelation2.select(), joinType),
+        10
+      )
+    }
+  }
+
+  test("SPARK-33497: Eliminate Limit if Window max rows not larger than Limit") {
+    checkPlanAndMaxRow(
+      testRelation.window(
+        Seq(count(1).as("c")), Seq("a".attr), Seq("b".attr.asc)).limit(20),
+      testRelation.window(
+        Seq(count(1).as("c")), Seq("a".attr), Seq("b".attr.asc)),
+      10
+    )
+  }
+
+  test("SPARK-34628: Remove GlobalLimit operator if its child max rows <= limit") {
+    val query = GlobalLimit(100, testRelation)
+    val optimized = Optimize.execute(query.analyze)
+    comparePlans(optimized, testRelation)
+  }
+
+  private def checkPlanAndMaxRow(
+      optimized: LogicalPlan, expected: LogicalPlan, expectedMaxRow: Long): Unit = {
+    comparePlans(Optimize.execute(optimized.analyze), expected.analyze)
+    assert(expected.maxRows.get == expectedMaxRow)
+  }
+}
+
+case class RelationWithoutMaxRows(output: Seq[Attribute]) extends LeafNode {
+  override def maxRows: Option[Long] = None
+}
+
+case class LongMaxRelation(output: Seq[Attribute]) extends LeafNode {
+  override def maxRows: Option[Long] = Some(Long.MaxValue)
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ComputeCurrentTimeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ComputeCurrentTimeSuite.scala
index db0399d2a73ee..82d6757407b51 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ComputeCurrentTimeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ComputeCurrentTimeSuite.scala
@@ -20,11 +20,13 @@ package org.apache.spark.sql.catalyst.optimizer
 import java.time.ZoneId
 
 import org.apache.spark.sql.catalyst.dsl.plans._
-import org.apache.spark.sql.catalyst.expressions.{Alias, CurrentDate, CurrentTimestamp, Literal}
+import org.apache.spark.sql.catalyst.expressions.{Alias, CurrentDate, CurrentTimestamp, CurrentTimeZone, Literal}
 import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan, Project}
 import org.apache.spark.sql.catalyst.rules.RuleExecutor
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.unsafe.types.UTF8String
 
 class ComputeCurrentTimeSuite extends PlanTest {
   object Optimize extends RuleExecutor[LogicalPlan] {
@@ -67,4 +69,16 @@ class ComputeCurrentTimeSuite extends PlanTest {
     assert(lits(1) >= min && lits(1) <= max)
     assert(lits(0) == lits(1))
   }
+
+  test("SPARK-33469: Add current_timezone function") {
+    val in = Project(Seq(Alias(CurrentTimeZone(), "c")()), LocalRelation())
+    val plan = Optimize.execute(in.analyze).asInstanceOf[Project]
+    val lits = new scala.collection.mutable.ArrayBuffer[String]
+    plan.transformAllExpressions { case e: Literal =>
+      lits += e.value.asInstanceOf[UTF8String].toString
+      e
+    }
+    assert(lits.size == 1)
+    assert(lits.head == SQLConf.get.sessionLocalTimeZone)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConstantFoldingSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConstantFoldingSuite.scala
index 23ab6b2df3e64..ae644c1110740 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConstantFoldingSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConstantFoldingSuite.scala
@@ -263,4 +263,40 @@ class ConstantFoldingSuite extends PlanTest {
 
     comparePlans(optimized, correctAnswer)
   }
+
+  test("SPARK-33544: Constant folding test with side effects") {
+    val originalQuery =
+      testRelation
+        .select('a)
+        .where(Size(CreateArray(Seq(AssertTrue(false)))) > 0)
+
+    val optimized = Optimize.execute(originalQuery.analyze)
+    comparePlans(optimized, originalQuery.analyze)
+  }
+
+  object OptimizeForCreate extends RuleExecutor[LogicalPlan] {
+    val batches =
+      Batch("AnalysisNodes", Once,
+        EliminateSubqueryAliases) ::
+      Batch("ConstantFolding", FixedPoint(4),
+        OptimizeIn,
+        ConstantFolding,
+        PruneFilters) :: Nil
+  }
+
+  test("SPARK-33544: Constant folding test CreateArray") {
+    val originalQuery =
+      testRelation
+        .select('a)
+        .where(Size(CreateArray(Seq('a))) > 0)
+
+    val optimized = OptimizeForCreate.execute(originalQuery.analyze)
+
+    val correctAnswer =
+      testRelation
+        .select('a)
+        .analyze
+
+    comparePlans(optimized, correctAnswer)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConvertToLocalRelationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConvertToLocalRelationSuite.scala
index 43579d4c903a1..02b6eed9ed050 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConvertToLocalRelationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConvertToLocalRelationSuite.scala
@@ -104,4 +104,7 @@ case class ExprReuseOutput(child: Expression) extends UnaryExpression {
     row.update(0, child.eval(input))
     row
   }
+
+  override protected def withNewChildInternal(newChild: Expression): ExprReuseOutput =
+    copy(child = newChild)
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/DecorrelateInnerQuerySuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/DecorrelateInnerQuerySuite.scala
new file mode 100644
index 0000000000000..93b27035aca33
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/DecorrelateInnerQuerySuite.scala
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans._
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.types.IntegerType
+
+class DecorrelateInnerQuerySuite extends PlanTest {
+
+  val a = AttributeReference("a", IntegerType)()
+  val b = AttributeReference("b", IntegerType)()
+  val c = AttributeReference("c", IntegerType)()
+  val x = AttributeReference("x", IntegerType)()
+  val y = AttributeReference("y", IntegerType)()
+  val z = AttributeReference("z", IntegerType)()
+  val testRelation = LocalRelation(a, b, c)
+  val testRelation2 = LocalRelation(x, y, z)
+
+  private def hasOuterReferences(plan: LogicalPlan): Boolean = {
+    plan.find(_.expressions.exists(SubExprUtils.containsOuter)).isDefined
+  }
+
+  private def check(
+      innerPlan: LogicalPlan,
+      outerPlan: LogicalPlan,
+      correctAnswer: LogicalPlan,
+      conditions: Seq[Expression]): Unit = {
+    val (outputPlan, joinCond) = DecorrelateInnerQuery(innerPlan, outerPlan)
+    assert(!hasOuterReferences(outputPlan))
+    comparePlans(outputPlan, correctAnswer)
+    assert(joinCond.length == conditions.length)
+    joinCond.zip(conditions).foreach(e => compareExpressions(e._1, e._2))
+  }
+
+  test("filter with correlated equality predicates only") {
+    val outerPlan = testRelation2
+    val innerPlan =
+      Project(Seq(a, b),
+        Filter(OuterReference(x) === a,
+          testRelation))
+    val correctAnswer = Project(Seq(a, b), testRelation)
+    check(innerPlan, outerPlan, correctAnswer, Seq(x === a))
+  }
+
+  test("filter with local and correlated equality predicates") {
+    val outerPlan = testRelation2
+    val innerPlan =
+      Project(Seq(a, b),
+        Filter(And(OuterReference(x) === a, b === 3),
+          testRelation))
+    val correctAnswer =
+      Project(Seq(a, b),
+        Filter(b === 3,
+          testRelation))
+    check(innerPlan, outerPlan, correctAnswer, Seq(x === a))
+  }
+
+  test("filter with correlated non-equality predicates") {
+    val outerPlan = testRelation2
+    val innerPlan =
+      Project(Seq(a, b),
+        Filter(OuterReference(x) > a,
+          testRelation))
+    val correctAnswer = Project(Seq(a, b), testRelation)
+    check(innerPlan, outerPlan, correctAnswer, Seq(x > a))
+  }
+
+  test("duplicated output attributes") {
+    val outerPlan = testRelation
+    val innerPlan =
+      Project(Seq(a),
+        Filter(OuterReference(a) === a,
+          testRelation))
+    val (outputPlan, joinCond) = DecorrelateInnerQuery(innerPlan, outerPlan)
+    val a1 = outputPlan.output.head
+    val correctAnswer =
+      Project(Seq(Alias(a, a1.name)(a1.exprId)),
+        Project(Seq(a),
+          testRelation))
+    comparePlans(outputPlan, correctAnswer)
+    assert(joinCond == Seq(a === a1))
+  }
+
+  test("filter with equality predicates with correlated values on both sides") {
+    val outerPlan = testRelation2
+    val innerPlan =
+      Project(Seq(a),
+        Filter(OuterReference(x) === OuterReference(y) + b,
+          testRelation))
+    val correctAnswer = Project(Seq(a, b), testRelation)
+    check(innerPlan, outerPlan, correctAnswer, Seq(x === y + b))
+  }
+
+  test("aggregate with correlated equality predicates that can be pulled up") {
+    val outerPlan = testRelation2
+    val minB = Alias(min(b), "min_b")()
+    val innerPlan =
+      Aggregate(Nil, Seq(minB),
+        Filter(And(OuterReference(x) === a, b === 3),
+          testRelation))
+    val correctAnswer =
+      Aggregate(Seq(a), Seq(minB, a),
+        Filter(b === 3,
+          testRelation))
+    check(innerPlan, outerPlan, correctAnswer, Seq(x === a))
+  }
+
+  test("aggregate with correlated equality predicates that cannot be pulled up") {
+    val outerPlan = testRelation2
+    val minB = Alias(min(b), "min_b")()
+    val innerPlan =
+      Aggregate(Nil, Seq(minB),
+        Filter(OuterReference(x) === OuterReference(y) + a,
+          testRelation))
+    val correctAnswer =
+      Aggregate(Seq(x, y), Seq(minB, x, y),
+        Filter(x === y + a,
+          DomainJoin(Seq(x, y), testRelation)))
+    check(innerPlan, outerPlan, correctAnswer, Seq(x <=> x, y <=> y))
+  }
+
+  test("aggregate with correlated equality predicates that has no attribute") {
+    val outerPlan = testRelation2
+    val minB = Alias(min(b), "min_b")()
+    val innerPlan =
+      Aggregate(Nil, Seq(minB),
+        Filter(OuterReference(x) === OuterReference(y),
+          testRelation))
+    val correctAnswer =
+      Aggregate(Nil, Seq(minB),
+        testRelation)
+    check(innerPlan, outerPlan, correctAnswer, Seq(x === y))
+  }
+
+  test("aggregate with correlated non-equality predicates") {
+    val outerPlan = testRelation2
+    val minB = Alias(min(b), "min_b")()
+    val innerPlan =
+      Aggregate(Nil, Seq(minB),
+        Filter(OuterReference(x) > a,
+          testRelation))
+    val correctAnswer =
+      Aggregate(Seq(x), Seq(minB, x),
+        Filter(x > a,
+          DomainJoin(Seq(x), testRelation)))
+    check(innerPlan, outerPlan, correctAnswer, Seq(x <=> x))
+  }
+
+  test("join with correlated equality predicates") {
+    val outerPlan = testRelation2
+    val joinCondition = Some($"t1.b" === $"t2.b")
+    val left =
+      Project(Seq(b),
+        Filter(OuterReference(x) === b,
+          testRelation)).as("t1")
+    val right =
+      Project(Seq(b),
+        Filter(OuterReference(x) === a,
+          testRelation)).as("t2")
+    Seq(Inner, LeftOuter, LeftSemi, LeftAnti, RightOuter, FullOuter, Cross).foreach { joinType =>
+      val innerPlan = Join(left, right, joinType, joinCondition, JoinHint.NONE).analyze
+      val newLeft = Project(Seq(b), testRelation).as("t1")
+      val newRight = Project(Seq(b, a), testRelation).as("t2")
+      // Since the left-hand side has outer(x) = b, and the right-hand side has outer(x) = a, the
+      // join condition will be augmented with b <=> a.
+      val newCond = Some(And($"t1.b" <=> $"t2.a", $"t1.b" === $"t2.b"))
+      val correctAnswer = Join(newLeft, newRight, joinType, newCond, JoinHint.NONE).analyze
+      check(innerPlan, outerPlan, correctAnswer, Seq(x === b, x === a))
+    }
+  }
+
+  test("correlated values inside join condition") {
+    val outerPlan = testRelation2
+    val innerPlan =
+      Join(
+        testRelation.as("t1"),
+        Filter(OuterReference(y) === 3, testRelation),
+        Inner,
+        Some(OuterReference(x) === a),
+        JoinHint.NONE)
+    val error = intercept[AssertionError] { DecorrelateInnerQuery(innerPlan, outerPlan) }
+    assert(error.getMessage.contains("Correlated column is not allowed in join"))
+  }
+
+  test("correlated values in project") {
+    val outerPlan = testRelation2
+    val innerPlan = Project(Seq(OuterReference(x), OuterReference(y)), OneRowRelation())
+    val correctAnswer = Project(Seq(x, y), DomainJoin(Seq(x, y), OneRowRelation()))
+    check(innerPlan, outerPlan, correctAnswer, Seq(x <=> x, y <=> y))
+  }
+
+  test("correlated values in project with alias") {
+    val outerPlan = testRelation2
+    val innerPlan =
+      Project(Seq(OuterReference(x), 'y1, 'sum),
+        Project(Seq(
+          OuterReference(x),
+          OuterReference(y).as("y1"),
+          Add(OuterReference(x), OuterReference(y)).as("sum")),
+            testRelation)).analyze
+    val correctAnswer =
+      Project(Seq(x, 'y1, 'sum, y),
+        Project(Seq(x, y.as("y1"), (x + y).as("sum"), y),
+          DomainJoin(Seq(x, y), testRelation))).analyze
+    check(innerPlan, outerPlan, correctAnswer, Seq(x <=> x, y <=> y))
+  }
+
+  test("correlated values in project with correlated equality conditions in filter") {
+    val outerPlan = testRelation2
+    val innerPlan =
+      Project(
+        Seq(OuterReference(x)),
+        Filter(
+          And(OuterReference(x) === a, And(OuterReference(x) + OuterReference(y) === c, b === 1)),
+          testRelation
+        )
+      )
+    val correctAnswer = Project(Seq(a, c), Filter(b === 1, testRelation))
+    check(innerPlan, outerPlan, correctAnswer, Seq(x === a, x + y === c))
+  }
+
+  test("correlated values in project without correlated equality conditions in filter") {
+    val outerPlan = testRelation2
+    val innerPlan =
+      Project(
+        Seq(OuterReference(y)),
+        Filter(
+          And(OuterReference(x) === a, And(OuterReference(x) + OuterReference(y) === c, b === 1)),
+          testRelation
+        )
+      )
+    val correctAnswer =
+      Project(Seq(y, a, c),
+        Filter(b === 1,
+          DomainJoin(Seq(y), testRelation)
+        )
+      )
+    check(innerPlan, outerPlan, correctAnswer, Seq(y <=> y, x === a, x + y === c))
+  }
+
+  test("correlated values in project with aggregate") {
+    val outerPlan = testRelation2
+    val innerPlan =
+      Aggregate(
+        Seq('x1), Seq(min('y1).as("min_y1")),
+        Project(
+          Seq(a, OuterReference(x).as("x1"), OuterReference(y).as("y1")),
+          Filter(
+            And(OuterReference(x) === a, OuterReference(y) === OuterReference(z)),
+            testRelation
+          )
+        )
+      ).analyze
+    val correctAnswer =
+      Aggregate(
+        Seq('x1, y, a), Seq(min('y1).as("min_y1"), y, a),
+        Project(
+          Seq(a, a.as("x1"), y.as("y1"), y),
+          DomainJoin(Seq(y), testRelation)
+        )
+      ).analyze
+    check(innerPlan, outerPlan, correctAnswer, Seq(y <=> y, x === a, y === z))
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateAggregateFilterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateAggregateFilterSuite.scala
new file mode 100644
index 0000000000000..ec9b876f78e1d
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateAggregateFilterSuite.scala
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions.{GreaterThan, Literal}
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan}
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types.LongType
+
+class EliminateAggregateFilterSuite extends PlanTest {
+  object Optimize extends RuleExecutor[LogicalPlan] {
+    val batches =
+      Batch("Operator Optimizations", Once, ConstantFolding, EliminateAggregateFilter) :: Nil
+  }
+
+  val testRelation = LocalRelation('a.int)
+
+  test("Eliminate Filter always is true") {
+    val query = testRelation
+      .select(sumDistinct('a, Some(Literal.TrueLiteral)).as('result))
+      .analyze
+    val answer = testRelation
+      .select(sumDistinct('a).as('result))
+      .analyze
+    comparePlans(Optimize.execute(query), answer)
+  }
+
+  test("Eliminate Filter is foldable and always is true") {
+    val query = testRelation
+      .select(countDistinctWithFilter(GreaterThan(Literal(2), Literal(1)), 'a).as('result))
+      .analyze
+    val answer = testRelation
+      .select(countDistinct('a).as('result))
+      .analyze
+    comparePlans(Optimize.execute(query), answer)
+  }
+
+  test("Eliminate Filter always is false") {
+    val query = testRelation
+      .select(sumDistinct('a, Some(Literal.FalseLiteral)).as('result))
+      .analyze
+    val answer = testRelation
+      .groupBy()(Literal.create(null, LongType).as('result))
+      .analyze
+    comparePlans(Optimize.execute(query), answer)
+  }
+
+  test("Eliminate Filter is foldable and always is false") {
+    val query = testRelation
+      .select(countDistinctWithFilter(GreaterThan(Literal(1), Literal(2)), 'a).as('result))
+      .analyze
+    val answer = testRelation
+      .groupBy()(Literal.create(0L, LongType).as('result))
+      .analyze
+    comparePlans(Optimize.execute(query), answer)
+  }
+
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateDistinctSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateDistinctSuite.scala
index f40691bd1a038..0848d5609ff02 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateDistinctSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateDistinctSuite.scala
@@ -18,8 +18,10 @@ package org.apache.spark.sql.catalyst.optimizer
 
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.aggregate._
 import org.apache.spark.sql.catalyst.plans.PlanTest
-import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Expand, LocalRelation, LogicalPlan}
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan}
 import org.apache.spark.sql.catalyst.rules.RuleExecutor
 
 class EliminateDistinctSuite extends PlanTest {
@@ -32,25 +34,24 @@ class EliminateDistinctSuite extends PlanTest {
 
   val testRelation = LocalRelation('a.int)
 
-  test("Eliminate Distinct in Max") {
-    val query = testRelation
-      .select(maxDistinct('a).as('result))
-      .analyze
-    val answer = testRelation
-      .select(max('a).as('result))
-      .analyze
-    assert(query != answer)
-    comparePlans(Optimize.execute(query), answer)
-  }
-
-  test("Eliminate Distinct in Min") {
-    val query = testRelation
-      .select(minDistinct('a).as('result))
-      .analyze
-    val answer = testRelation
-      .select(min('a).as('result))
-      .analyze
-    assert(query != answer)
-    comparePlans(Optimize.execute(query), answer)
+  Seq(
+    Max(_),
+    Min(_),
+    BitAndAgg(_),
+    BitOrAgg(_),
+    CollectSet(_: Expression)
+  ).foreach {
+    aggBuilder =>
+      val agg = aggBuilder('a)
+      test(s"Eliminate Distinct in ${agg.prettyName}") {
+        val query = testRelation
+          .select(agg.toAggregateExpression(isDistinct = true).as('result))
+          .analyze
+        val answer = testRelation
+          .select(agg.toAggregateExpression(isDistinct = false).as('result))
+          .analyze
+        assert(query != answer)
+        comparePlans(Optimize.execute(query), answer)
+      }
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsBeforeRepartitionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsBeforeRepartitionSuite.scala
new file mode 100644
index 0000000000000..82db174ad41b0
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsBeforeRepartitionSuite.scala
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.analysis.{Analyzer, EmptyFunctionRegistry}
+import org.apache.spark.sql.catalyst.catalog.{InMemoryCatalog, SessionCatalog}
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan}
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+
+class EliminateSortsBeforeRepartitionSuite extends PlanTest {
+
+  val catalog = new SessionCatalog(new InMemoryCatalog, EmptyFunctionRegistry)
+  val analyzer = new Analyzer(catalog)
+
+  val testRelation = LocalRelation('a.int, 'b.int, 'c.int)
+  val anotherTestRelation = LocalRelation('d.int, 'e.int)
+
+  object Optimize extends RuleExecutor[LogicalPlan] {
+    val batches =
+      Batch("Default", FixedPoint(10),
+        FoldablePropagation,
+        LimitPushDown) ::
+      Batch("Eliminate Sorts", Once,
+        EliminateSorts) ::
+      Batch("Collapse Project", Once,
+        CollapseProject) :: Nil
+  }
+
+  def repartition(plan: LogicalPlan): LogicalPlan = plan.repartition(10)
+
+  test("sortBy") {
+    val plan = testRelation.select('a, 'b).sortBy('a.asc, 'b.desc)
+    val optimizedPlan = testRelation.select('a, 'b)
+    checkRepartitionCases(plan, optimizedPlan)
+  }
+
+  test("sortBy with projection") {
+    val plan = testRelation.sortBy('a.asc, 'b.asc).select('a + 1 as "a", 'b + 2 as "b")
+    val optimizedPlan = testRelation.select('a + 1 as "a", 'b + 2 as "b")
+    checkRepartitionCases(plan, optimizedPlan)
+  }
+
+  test("sortBy with projection and filter") {
+    val plan = testRelation.sortBy('a.asc, 'b.asc).select('a, 'b).where('a === 10)
+    val optimizedPlan = testRelation.select('a, 'b).where('a === 10)
+    checkRepartitionCases(plan, optimizedPlan)
+  }
+
+  test("sortBy with limit") {
+    val plan = testRelation.sortBy('a.asc, 'b.asc).limit(10)
+    val optimizedPlan = testRelation.sortBy('a.asc, 'b.asc).limit(10)
+    checkRepartitionCases(plan, optimizedPlan)
+  }
+
+  test("sortBy with non-deterministic projection") {
+    val plan = testRelation.sortBy('a.asc, 'b.asc).select(rand(1), 'a, 'b)
+    val optimizedPlan = testRelation.sortBy('a.asc, 'b.asc).select(rand(1), 'a, 'b)
+    checkRepartitionCases(plan, optimizedPlan)
+  }
+
+  test("orderBy") {
+    val plan = testRelation.select('a, 'b).orderBy('a.asc, 'b.asc)
+    val optimizedPlan = testRelation.select('a, 'b)
+    checkRepartitionCases(plan, optimizedPlan)
+  }
+
+  test("orderBy with projection") {
+    val plan = testRelation.orderBy('a.asc, 'b.asc).select('a + 1 as "a", 'b + 2 as "b")
+    val optimizedPlan = testRelation.select('a + 1 as "a", 'b + 2 as "b")
+    checkRepartitionCases(plan, optimizedPlan)
+  }
+
+  test("orderBy with projection and filter") {
+    val plan = testRelation.orderBy('a.asc, 'b.asc).select('a, 'b).where('a === 10)
+    val optimizedPlan = testRelation.select('a, 'b).where('a === 10)
+    checkRepartitionCases(plan, optimizedPlan)
+  }
+
+  test("orderBy with limit") {
+    val plan = testRelation.orderBy('a.asc, 'b.asc).limit(10)
+    val optimizedPlan = testRelation.orderBy('a.asc, 'b.asc).limit(10)
+    checkRepartitionCases(plan, optimizedPlan)
+  }
+
+  test("orderBy with non-deterministic projection") {
+    val plan = testRelation.orderBy('a.asc, 'b.asc).select(rand(1), 'a, 'b)
+    val optimizedPlan = testRelation.orderBy('a.asc, 'b.asc).select(rand(1), 'a, 'b)
+    checkRepartitionCases(plan, optimizedPlan)
+  }
+
+  test("additional coalesce and sortBy") {
+    val plan = testRelation.sortBy('a.asc, 'b.asc).coalesce(1)
+    val optimizedPlan = testRelation.coalesce(1)
+    checkRepartitionCases(plan, optimizedPlan)
+  }
+
+  test("additional projection, repartition and sortBy") {
+    val plan = testRelation.sortBy('a.asc, 'b.asc).repartition(100).select('a + 1 as "a")
+    val optimizedPlan = testRelation.repartition(100).select('a + 1 as "a")
+    checkRepartitionCases(plan, optimizedPlan)
+  }
+
+  test("additional filter, distribute and sortBy") {
+    val plan = testRelation.sortBy('a.asc, 'b.asc).distribute('a)(2).where('a === 10)
+    val optimizedPlan = testRelation.distribute('a)(2).where('a === 10)
+    checkRepartitionCases(plan, optimizedPlan)
+  }
+
+  test("join") {
+    val plan = testRelation.sortBy('a.asc, 'b.asc).distribute('a)(2).where('a === 10)
+    val optimizedPlan = testRelation.distribute('a)(2).where('a === 10)
+    val anotherPlan = anotherTestRelation.select('d)
+    val joinPlan = plan.join(anotherPlan)
+    val optimizedJoinPlan = optimize(joinPlan)
+    val correctJoinPlan = analyze(optimizedPlan.join(anotherPlan))
+    comparePlans(optimizedJoinPlan, correctJoinPlan)
+  }
+
+  test("aggregate") {
+    val plan = testRelation.sortBy('a.asc, 'b.asc).distribute('a)(2).where('a === 10)
+    val optimizedPlan = testRelation.distribute('a)(2).where('a === 10)
+    val aggPlan = plan.groupBy('a)(sum('b))
+    val optimizedAggPlan = optimize(aggPlan)
+    val correctAggPlan = analyze(optimizedPlan.groupBy('a)(sum('b)))
+    comparePlans(optimizedAggPlan, correctAggPlan)
+  }
+
+  protected def checkRepartitionCases(plan: LogicalPlan, optimizedPlan: LogicalPlan): Unit = {
+    // cannot remove sortBy before repartition without sortBy/orderBy
+    val planWithRepartition = repartition(plan)
+    val optimizedPlanWithRepartition = optimize(planWithRepartition)
+    val correctPlanWithRepartition = analyze(planWithRepartition)
+    comparePlans(optimizedPlanWithRepartition, correctPlanWithRepartition)
+
+    // can remove sortBy before repartition with sortBy
+    val planWithRepartitionAndSortBy = planWithRepartition.sortBy('a.asc)
+    val optimizedPlanWithRepartitionAndSortBy = optimize(planWithRepartitionAndSortBy)
+    val correctPlanWithRepartitionAndSortBy = analyze(repartition(optimizedPlan).sortBy('a.asc))
+    comparePlans(optimizedPlanWithRepartitionAndSortBy, correctPlanWithRepartitionAndSortBy)
+
+    // can remove sortBy before repartition with orderBy
+    val planWithRepartitionAndOrderBy = planWithRepartition.orderBy('a.asc)
+    val optimizedPlanWithRepartitionAndOrderBy = optimize(planWithRepartitionAndOrderBy)
+    val correctPlanWithRepartitionAndOrderBy = analyze(repartition(optimizedPlan).orderBy('a.asc))
+    comparePlans(optimizedPlanWithRepartitionAndOrderBy, correctPlanWithRepartitionAndOrderBy)
+  }
+
+  private def analyze(plan: LogicalPlan): LogicalPlan = {
+    analyzer.execute(plan)
+  }
+
+  private def optimize(plan: LogicalPlan): LogicalPlan = {
+    Optimize.execute(analyzer.execute(plan))
+  }
+}
+
+class EliminateSortsBeforeRepartitionByExprsSuite extends EliminateSortsBeforeRepartitionSuite {
+  override def repartition(plan: LogicalPlan): LogicalPlan = plan.distribute('a)(10)
+
+  test("sortBy before repartition with non-deterministic expressions") {
+    val plan = testRelation.sortBy('a.asc, 'b.asc).limit(10)
+    val planWithRepartition = plan.distribute(rand(1).asc, 'a.asc)(20)
+    checkRepartitionCases(plan = planWithRepartition, optimizedPlan = planWithRepartition)
+  }
+
+  test("orderBy before repartition with non-deterministic expressions") {
+    val plan = testRelation.orderBy('a.asc, 'b.asc).limit(10)
+    val planWithRepartition = plan.distribute(rand(1).asc, 'a.asc)(20)
+    checkRepartitionCases(plan = planWithRepartition, optimizedPlan = planWithRepartition)
+  }
+}
+
+class EliminateSortsBeforeCoalesceSuite extends EliminateSortsBeforeRepartitionSuite {
+  override def repartition(plan: LogicalPlan): LogicalPlan = plan.coalesce(1)
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala
index d7eb048ba8705..01ecbd808c251 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala
@@ -18,8 +18,8 @@
 package org.apache.spark.sql.catalyst.optimizer
 
 import org.apache.spark.api.python.PythonEvalType
-import org.apache.spark.sql.catalyst.analysis.{Analyzer, EmptyFunctionRegistry}
-import org.apache.spark.sql.catalyst.catalog.{InMemoryCatalog, SessionCatalog}
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.analysis.AnalysisTest
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
@@ -27,14 +27,11 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules._
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.{CASE_SENSITIVE, ORDER_BY_ORDINAL}
 import org.apache.spark.sql.types.IntegerType
 
-class EliminateSortsSuite extends PlanTest {
-  override val conf = new SQLConf().copy(CASE_SENSITIVE -> true, ORDER_BY_ORDINAL -> false)
-  val catalog = new SessionCatalog(new InMemoryCatalog, EmptyFunctionRegistry, conf)
-  val analyzer = new Analyzer(catalog, conf)
+class EliminateSortsSuite extends AnalysisTest {
+  val analyzer = getAnalyzer
 
   object Optimize extends RuleExecutor[LogicalPlan] {
     val batches =
@@ -52,8 +49,14 @@ class EliminateSortsSuite extends PlanTest {
       Batch("Limit PushDown", FixedPoint(10), LimitPushDown) :: Nil
   }
 
-  val testRelation = LocalRelation('a.int, 'b.int, 'c.int)
-  val testRelationB = LocalRelation('d.int)
+  val testRelation = LocalRelation.fromExternalRows(
+    Seq("a".attr.int, "b".attr.int, "c".attr.int),
+    1.to(12).map(_ => Row(1, 2, 3))
+  )
+  val testRelationB = LocalRelation.fromExternalRows(
+    Seq("d".attr.int),
+    1.to(12).map(_ => Row(1))
+  )
 
   test("Empty order by clause") {
     val x = testRelation
@@ -66,23 +69,29 @@ class EliminateSortsSuite extends PlanTest {
   }
 
   test("All the SortOrder are no-op") {
-    val x = testRelation
+    withSQLConf(CASE_SENSITIVE.key -> "true", ORDER_BY_ORDINAL.key -> "false") {
+      val x = testRelation
+      val analyzer = getAnalyzer
 
-    val query = x.orderBy(SortOrder(3, Ascending), SortOrder(-1, Ascending))
-    val optimized = Optimize.execute(analyzer.execute(query))
-    val correctAnswer = analyzer.execute(x)
+      val query = x.orderBy(SortOrder(3, Ascending), SortOrder(-1, Ascending))
+      val optimized = Optimize.execute(analyzer.execute(query))
+      val correctAnswer = analyzer.execute(x)
 
-    comparePlans(optimized, correctAnswer)
+      comparePlans(optimized, correctAnswer)
+    }
   }
 
   test("Partial order-by clauses contain no-op SortOrder") {
-    val x = testRelation
+    withSQLConf(CASE_SENSITIVE.key -> "true", ORDER_BY_ORDINAL.key -> "false") {
+      val x = testRelation
+      val analyzer = getAnalyzer
 
-    val query = x.orderBy(SortOrder(3, Ascending), 'a.asc)
-    val optimized = Optimize.execute(analyzer.execute(query))
-    val correctAnswer = analyzer.execute(x.orderBy('a.asc))
+      val query = x.orderBy(SortOrder(3, Ascending), 'a.asc)
+      val optimized = Optimize.execute(analyzer.execute(query))
+      val correctAnswer = analyzer.execute(x.orderBy('a.asc))
 
-    comparePlans(optimized, correctAnswer)
+      comparePlans(optimized, correctAnswer)
+    }
   }
 
   test("Remove no-op alias") {
@@ -97,12 +106,34 @@ class EliminateSortsSuite extends PlanTest {
     comparePlans(optimized, correctAnswer)
   }
 
-  test("remove redundant order by") {
+  test("SPARK-33183: remove consecutive no-op sorts") {
+    val plan = testRelation.orderBy().orderBy().orderBy()
+    val optimized = Optimize.execute(plan.analyze)
+    val correctAnswer = testRelation.analyze
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("SPARK-33183: remove redundant sort by") {
     val orderedPlan = testRelation.select('a, 'b).orderBy('a.asc, 'b.desc_nullsFirst)
-    val unnecessaryReordered = orderedPlan.limit(2).select('a).orderBy('a.asc, 'b.desc_nullsFirst)
+    val unnecessaryReordered = orderedPlan.limit(2).select('a).sortBy('a.asc, 'b.desc_nullsFirst)
     val optimized = Optimize.execute(unnecessaryReordered.analyze)
     val correctAnswer = orderedPlan.limit(2).select('a).analyze
-    comparePlans(Optimize.execute(optimized), correctAnswer)
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("SPARK-33183: remove all redundant local sorts") {
+    val orderedPlan = testRelation.sortBy('a.asc).orderBy('a.asc).sortBy('a.asc)
+    val optimized = Optimize.execute(orderedPlan.analyze)
+    val correctAnswer = testRelation.orderBy('a.asc).analyze
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("SPARK-33183: should not remove global sort") {
+    val orderedPlan = testRelation.select('a, 'b).orderBy('a.asc, 'b.desc_nullsFirst)
+    val reordered = orderedPlan.limit(2).select('a).orderBy('a.asc, 'b.desc_nullsFirst)
+    val optimized = Optimize.execute(reordered.analyze)
+    val correctAnswer = reordered.analyze
+    comparePlans(optimized, correctAnswer)
   }
 
   test("do not remove sort if the order is different") {
@@ -113,22 +144,39 @@ class EliminateSortsSuite extends PlanTest {
     comparePlans(optimized, correctAnswer)
   }
 
-  test("filters don't affect order") {
+  test("SPARK-33183: remove top level local sort with filter operators") {
     val orderedPlan = testRelation.select('a, 'b).orderBy('a.asc, 'b.desc)
-    val filteredAndReordered = orderedPlan.where('a > Literal(10)).orderBy('a.asc, 'b.desc)
+    val filteredAndReordered = orderedPlan.where('a > Literal(10)).sortBy('a.asc, 'b.desc)
     val optimized = Optimize.execute(filteredAndReordered.analyze)
     val correctAnswer = orderedPlan.where('a > Literal(10)).analyze
     comparePlans(optimized, correctAnswer)
   }
 
-  test("limits don't affect order") {
+  test("SPARK-33183: keep top level global sort with filter operators") {
+    val projectPlan = testRelation.select('a, 'b)
+    val orderedPlan = projectPlan.orderBy('a.asc, 'b.desc)
+    val filteredAndReordered = orderedPlan.where('a > Literal(10)).orderBy('a.asc, 'b.desc)
+    val optimized = Optimize.execute(filteredAndReordered.analyze)
+    val correctAnswer = projectPlan.where('a > Literal(10)).orderBy('a.asc, 'b.desc).analyze
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("SPARK-33183: limits should not affect order for local sort") {
     val orderedPlan = testRelation.select('a, 'b).orderBy('a.asc, 'b.desc)
-    val filteredAndReordered = orderedPlan.limit(Literal(10)).orderBy('a.asc, 'b.desc)
+    val filteredAndReordered = orderedPlan.limit(Literal(10)).sortBy('a.asc, 'b.desc)
     val optimized = Optimize.execute(filteredAndReordered.analyze)
     val correctAnswer = orderedPlan.limit(Literal(10)).analyze
     comparePlans(optimized, correctAnswer)
   }
 
+  test("SPARK-33183: should not remove global sort with limit operators") {
+    val orderedPlan = testRelation.select('a, 'b).orderBy('a.asc, 'b.desc)
+    val filteredAndReordered = orderedPlan.limit(Literal(10)).orderBy('a.asc, 'b.desc)
+    val optimized = Optimize.execute(filteredAndReordered.analyze)
+    val correctAnswer = filteredAndReordered.analyze
+    comparePlans(optimized, correctAnswer)
+  }
+
   test("different sorts are not simplified if limit is in between") {
     val orderedPlan = testRelation.select('a, 'b).orderBy('b.desc).limit(Literal(10))
       .orderBy('a.asc)
@@ -137,11 +185,11 @@ class EliminateSortsSuite extends PlanTest {
     comparePlans(optimized, correctAnswer)
   }
 
-  test("range is already sorted") {
+  test("SPARK-33183: should not remove global sort with range operator") {
     val inputPlan = Range(1L, 1000L, 1, 10)
     val orderedPlan = inputPlan.orderBy('id.asc)
     val optimized = Optimize.execute(orderedPlan.analyze)
-    val correctAnswer = inputPlan.analyze
+    val correctAnswer = orderedPlan.analyze
     comparePlans(optimized, correctAnswer)
 
     val reversedPlan = inputPlan.orderBy('id.desc)
@@ -152,10 +200,18 @@ class EliminateSortsSuite extends PlanTest {
     val negativeStepInputPlan = Range(10L, 1L, -1, 10)
     val negativeStepOrderedPlan = negativeStepInputPlan.orderBy('id.desc)
     val negativeStepOptimized = Optimize.execute(negativeStepOrderedPlan.analyze)
-    val negativeStepCorrectAnswer = negativeStepInputPlan.analyze
+    val negativeStepCorrectAnswer = negativeStepOrderedPlan.analyze
     comparePlans(negativeStepOptimized, negativeStepCorrectAnswer)
   }
 
+  test("SPARK-33183: remove local sort with range operator") {
+    val inputPlan = Range(1L, 1000L, 1, 10)
+    val orderedPlan = inputPlan.sortBy('id.asc)
+    val optimized = Optimize.execute(orderedPlan.analyze)
+    val correctAnswer = inputPlan.analyze
+    comparePlans(optimized, correctAnswer)
+  }
+
   test("sort should not be removed when there is a node which doesn't guarantee any order") {
     val orderedPlan = testRelation.select('a, 'b)
     val groupedAndResorted = orderedPlan.groupBy('a)(sum('a)).orderBy('a.asc)
@@ -197,13 +253,25 @@ class EliminateSortsSuite extends PlanTest {
     comparePlans(optimizedThrice, correctAnswerThrice)
   }
 
-  test("remove orderBy in groupBy clause with count aggs") {
-    val projectPlan = testRelation.select('a, 'b)
-    val unnecessaryOrderByPlan = projectPlan.orderBy('a.asc, 'b.desc)
-    val groupByPlan = unnecessaryOrderByPlan.groupBy('a)(count(1))
-    val optimized = Optimize.execute(groupByPlan.analyze)
-    val correctAnswer = projectPlan.groupBy('a)(count(1)).analyze
-    comparePlans(optimized, correctAnswer)
+  test("remove orderBy in groupBy clause with order irrelevant aggs") {
+    Seq(
+      (e : Expression) => min(e),
+      (e : Expression) => minDistinct(e),
+      (e : Expression) => max(e),
+      (e : Expression) => maxDistinct(e),
+      (e : Expression) => count(e),
+      (e : Expression) => countDistinct(e),
+      (e : Expression) => bitAnd(e),
+      (e : Expression) => bitOr(e),
+      (e : Expression) => bitXor(e)
+    ).foreach(agg => {
+      val projectPlan = testRelation.select('a, 'b)
+      val unnecessaryOrderByPlan = projectPlan.orderBy('a.asc, 'b.desc)
+      val groupByPlan = unnecessaryOrderByPlan.groupBy('a)(agg('b))
+      val optimized = Optimize.execute(groupByPlan.analyze)
+      val correctAnswer = projectPlan.groupBy('a)(agg('b)).analyze
+      comparePlans(optimized, correctAnswer)
+    })
   }
 
   test("remove orderBy in groupBy clause with sum aggs") {
@@ -284,6 +352,15 @@ class EliminateSortsSuite extends PlanTest {
     comparePlans(optimized, correctAnswer)
   }
 
+  test("SPARK-32318: should not remove orderBy in distribute statement") {
+    val projectPlan = testRelation.select('a, 'b)
+    val orderByPlan = projectPlan.orderBy('b.desc)
+    val distributedPlan = orderByPlan.distribute('a)(1)
+    val optimized = Optimize.execute(distributedPlan.analyze)
+    val correctAnswer = distributedPlan.analyze
+    comparePlans(optimized, correctAnswer)
+  }
+
   test("should not remove orderBy in left join clause if there is an outer limit") {
     val projectPlan = testRelation.select('a, 'b)
     val orderByPlan = projectPlan.orderBy('a.asc, 'b.desc)
@@ -310,4 +387,39 @@ class EliminateSortsSuite extends PlanTest {
     val correctAnswer = PushDownOptimizer.execute(noOrderByPlan.analyze)
     comparePlans(optimized, correctAnswer)
   }
+
+  test("SPARK-33183: remove consecutive global sorts with the same ordering") {
+    Seq(
+      (testRelation.orderBy('a.asc).orderBy('a.asc), testRelation.orderBy('a.asc)),
+      (testRelation.orderBy('a.asc, 'b.desc).orderBy('a.asc), testRelation.orderBy('a.asc))
+    ).foreach { case (ordered, answer) =>
+      val optimized = Optimize.execute(ordered.analyze)
+      comparePlans(optimized, answer.analyze)
+    }
+  }
+
+  test("SPARK-33183: remove consecutive local sorts with the same ordering") {
+    val orderedPlan = testRelation.sortBy('a.asc).sortBy('a.asc).sortBy('a.asc)
+    val optimized = Optimize.execute(orderedPlan.analyze)
+    val correctAnswer = testRelation.sortBy('a.asc).analyze
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("SPARK-33183: remove consecutive local sorts with different ordering") {
+    val orderedPlan = testRelation.sortBy('b.asc).sortBy('a.desc).sortBy('a.asc)
+    val optimized = Optimize.execute(orderedPlan.analyze)
+    val correctAnswer = testRelation.sortBy('a.asc).analyze
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("SPARK-33183: should keep global sort when child is a local sort with the same ordering") {
+    val correctAnswer = testRelation.orderBy('a.asc).analyze
+    Seq(
+      testRelation.sortBy('a.asc).orderBy('a.asc),
+      testRelation.orderBy('a.asc).sortBy('a.asc).orderBy('a.asc)
+    ).foreach { ordered =>
+      val optimized = Optimize.execute(ordered.analyze)
+      comparePlans(optimized, correctAnswer)
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
index 70e29dca46e9e..acdbff95422fd 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
@@ -25,12 +25,13 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules._
-import org.apache.spark.sql.types.{BooleanType, IntegerType}
+import org.apache.spark.sql.types.{IntegerType, StringType}
 import org.apache.spark.unsafe.types.CalendarInterval
 
 class FilterPushdownSuite extends PlanTest {
 
   object Optimize extends RuleExecutor[LogicalPlan] {
+
     val batches =
       Batch("Subqueries", Once,
         EliminateSubqueryAliases) ::
@@ -39,7 +40,10 @@ class FilterPushdownSuite extends PlanTest {
         PushPredicateThroughNonJoin,
         BooleanSimplification,
         PushPredicateThroughJoin,
-        CollapseProject) :: Nil
+        CollapseProject) ::
+      Batch("Push extra predicate through join", FixedPoint(10),
+        PushExtraPredicateThroughJoin,
+        PushDownPredicates) :: Nil
   }
 
   val attrA = 'a.int
@@ -51,6 +55,15 @@ class FilterPushdownSuite extends PlanTest {
 
   val testRelation1 = LocalRelation(attrD)
 
+  val simpleDisjunctivePredicate =
+    ("x.a".attr > 3) && ("y.a".attr > 13) || ("x.a".attr > 1) && ("y.a".attr > 11)
+  val expectedPredicatePushDownResult = {
+    val left = testRelation.where(('a > 3 || 'a > 1)).subquery('x)
+    val right = testRelation.where('a > 13 || 'a > 11).subquery('y)
+    left.join(right, condition = Some("x.b".attr === "y.b".attr
+      && (("x.a".attr > 3) && ("y.a".attr > 13) || ("x.a".attr > 1) && ("y.a".attr > 11)))).analyze
+  }
+
   // This test already passes.
   test("eliminate subqueries") {
     val originalQuery =
@@ -664,14 +677,14 @@ class FilterPushdownSuite extends PlanTest {
     val generator = Explode('c_arr)
     val originalQuery = {
       testRelationWithArrayType
-        .generate(generator, alias = Some("arr"))
+        .generate(generator, alias = Some("arr"), outputNames = Seq("c"))
         .where(('b >= 5) && ('c > 6))
     }
     val optimized = Optimize.execute(originalQuery.analyze)
     val referenceResult = {
       testRelationWithArrayType
         .where('b >= 5)
-        .generate(generator, alias = Some("arr"))
+        .generate(generator, alias = Some("arr"), outputNames = Seq("c"))
         .where('c > 6).analyze
     }
 
@@ -846,7 +859,7 @@ class FilterPushdownSuite extends PlanTest {
 
   test("expand") {
     val agg = testRelation
-      .groupBy(Cube(Seq('a, 'b)))('a, 'b, sum('c))
+      .groupBy(Cube(Seq(Seq('a), Seq('b))))('a, 'b, sum('c))
       .analyze
       .asInstanceOf[Aggregate]
 
@@ -1135,49 +1148,32 @@ class FilterPushdownSuite extends PlanTest {
     comparePlans(Optimize.execute(originalQuery.analyze), correctAnswer)
   }
 
-  test("join condition pushdown: deterministic and non-deterministic") {
-    val x = testRelation.subquery('x)
-    val y = testRelation.subquery('y)
-
-    // Verify that all conditions except the watermark touching condition are pushed down
-    // by the optimizer and others are not.
-    val originalQuery = x.join(y, condition = Some("x.a".attr === 5 && "y.a".attr === 5 &&
-      "x.a".attr === Rand(10) && "y.b".attr === 5))
-    val correctAnswer =
-      x.where("x.a".attr === 5).join(y.where("y.a".attr === 5 && "y.b".attr === 5),
-        condition = Some("x.a".attr === Rand(10)))
-
-    // CheckAnalysis will ensure nondeterministic expressions not appear in join condition.
-    // TODO support nondeterministic expressions in join condition.
-    comparePlans(Optimize.execute(originalQuery.analyze), correctAnswer.analyze,
-      checkAnalysis = false)
-  }
-
   test("watermark pushdown: no pushdown on watermark attribute #1") {
     val interval = new CalendarInterval(2, 2, 2000L)
+    val relation = LocalRelation(attrA, 'b.timestamp, attrC)
 
     // Verify that all conditions except the watermark touching condition are pushed down
     // by the optimizer and others are not.
-    val originalQuery = EventTimeWatermark('b, interval, testRelation)
-      .where('a === 5 && 'b === 10 && 'c === 5)
+    val originalQuery = EventTimeWatermark('b, interval, relation)
+      .where('a === 5 && 'b === new java.sql.Timestamp(0) && 'c === 5)
     val correctAnswer = EventTimeWatermark(
-      'b, interval, testRelation.where('a === 5 && 'c === 5))
-      .where('b === 10)
+      'b, interval, relation.where('a === 5 && 'c === 5))
+      .where('b === new java.sql.Timestamp(0))
 
-    comparePlans(Optimize.execute(originalQuery.analyze), correctAnswer.analyze,
-      checkAnalysis = false)
+    comparePlans(Optimize.execute(originalQuery.analyze), correctAnswer.analyze)
   }
 
   test("watermark pushdown: no pushdown for nondeterministic filter") {
     val interval = new CalendarInterval(2, 2, 2000L)
+    val relation = LocalRelation(attrA, attrB, 'c.timestamp)
 
     // Verify that all conditions except the watermark touching condition are pushed down
     // by the optimizer and others are not.
-    val originalQuery = EventTimeWatermark('c, interval, testRelation)
-      .where('a === 5 && 'b === Rand(10) && 'c === 5)
+    val originalQuery = EventTimeWatermark('c, interval, relation)
+      .where('a === 5 && 'b === Rand(10) && 'c === new java.sql.Timestamp(0))
     val correctAnswer = EventTimeWatermark(
-      'c, interval, testRelation.where('a === 5))
-      .where('b === Rand(10) && 'c === 5)
+      'c, interval, relation.where('a === 5))
+      .where('b === Rand(10) && 'c === new java.sql.Timestamp(0))
 
     comparePlans(Optimize.execute(originalQuery.analyze), correctAnswer.analyze,
       checkAnalysis = false)
@@ -1185,13 +1181,14 @@ class FilterPushdownSuite extends PlanTest {
 
   test("watermark pushdown: full pushdown") {
     val interval = new CalendarInterval(2, 2, 2000L)
+    val relation = LocalRelation(attrA, attrB, 'c.timestamp)
 
     // Verify that all conditions except the watermark touching condition are pushed down
     // by the optimizer and others are not.
-    val originalQuery = EventTimeWatermark('c, interval, testRelation)
+    val originalQuery = EventTimeWatermark('c, interval, relation)
       .where('a === 5 && 'b === 10)
     val correctAnswer = EventTimeWatermark(
-      'c, interval, testRelation.where('a === 5 && 'b === 10))
+      'c, interval, relation.where('a === 5 && 'b === 10))
 
     comparePlans(Optimize.execute(originalQuery.analyze), correctAnswer.analyze,
       checkAnalysis = false)
@@ -1199,16 +1196,39 @@ class FilterPushdownSuite extends PlanTest {
 
   test("watermark pushdown: no pushdown on watermark attribute #2") {
     val interval = new CalendarInterval(2, 2, 2000L)
+    val relation = LocalRelation('a.timestamp, attrB, attrC)
 
-    val originalQuery = EventTimeWatermark('a, interval, testRelation)
-      .where('a === 5 && 'b === 10)
+    val originalQuery = EventTimeWatermark('a, interval, relation)
+      .where('a === new java.sql.Timestamp(0) && 'b === 10)
     val correctAnswer = EventTimeWatermark(
-      'a, interval, testRelation.where('b === 10)).where('a === 5)
+      'a, interval, relation.where('b === 10)).where('a === new java.sql.Timestamp(0))
 
     comparePlans(Optimize.execute(originalQuery.analyze), correctAnswer.analyze,
       checkAnalysis = false)
   }
 
+  test("push down predicate through expand") {
+    val query =
+        Filter('a > 1,
+          Expand(
+            Seq(
+              Seq('a, 'b, 'c, Literal.create(null, StringType), 1),
+              Seq('a, 'b, 'c, 'a, 2)),
+            Seq('a, 'b, 'c),
+            testRelation)).analyze
+    val optimized = Optimize.execute(query)
+
+    val expected =
+        Expand(
+          Seq(
+            Seq('a, 'b, 'c, Literal.create(null, StringType), 1),
+            Seq('a, 'b, 'c, 'a, 2)),
+          Seq('a, 'b, 'c),
+          Filter('a > 1, testRelation)).analyze
+
+    comparePlans(optimized, expected)
+  }
+
   test("SPARK-28345: PythonUDF predicate should be able to pushdown to join") {
     val pythonUDFJoinCond = {
       val pythonUDF = PythonUDF("pythonUDF", null,
@@ -1230,4 +1250,138 @@ class FilterPushdownSuite extends PlanTest {
 
     comparePlans(Optimize.execute(query.analyze), expected)
   }
+
+  test("push down filter predicates through inner join") {
+    val x = testRelation.subquery('x)
+    val y = testRelation.subquery('y)
+
+    val originalQuery = x.join(y).where(("x.b".attr === "y.b".attr) && (simpleDisjunctivePredicate))
+
+    val optimized = Optimize.execute(originalQuery.analyze)
+    comparePlans(optimized, expectedPredicatePushDownResult)
+  }
+
+  test("push down join predicates through inner join") {
+    val x = testRelation.subquery('x)
+    val y = testRelation.subquery('y)
+
+    val originalQuery =
+      x.join(y, condition = Some(("x.b".attr === "y.b".attr) && (simpleDisjunctivePredicate)))
+
+    val optimized = Optimize.execute(originalQuery.analyze)
+    comparePlans(optimized, expectedPredicatePushDownResult)
+  }
+
+  test("push down complex predicates through inner join") {
+    val x = testRelation.subquery('x)
+    val y = testRelation.subquery('y)
+
+    val joinCondition = (("x.b".attr === "y.b".attr)
+      && ((("x.a".attr === 5) && ("y.a".attr >= 2) && ("y.a".attr <= 3))
+      || (("x.a".attr === 2) && ("y.a".attr >= 1) && ("y.a".attr <= 14))
+      || (("x.a".attr === 1) && ("y.a".attr >= 9) && ("y.a".attr <= 27))))
+
+    val originalQuery = x.join(y, condition = Some(joinCondition))
+    val optimized = Optimize.execute(originalQuery.analyze)
+    val left = testRelation.where(
+      ('a === 5 || 'a === 2 || 'a === 1)).subquery('x)
+    val right = testRelation.where(
+      ('a >= 2 && 'a <= 3) || ('a >= 1 && 'a <= 14) || ('a >= 9 && 'a <= 27)).subquery('y)
+    val correctAnswer = left.join(right, condition = Some(joinCondition)).analyze
+
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("push down predicates(with NOT predicate) through inner join") {
+    val x = testRelation.subquery('x)
+    val y = testRelation.subquery('y)
+
+    val originalQuery =
+      x.join(y, condition = Some(("x.b".attr === "y.b".attr)
+        && Not(("x.a".attr > 3)
+        && ("x.a".attr < 2 || ("y.a".attr > 13)) || ("x.a".attr > 1) && ("y.a".attr > 11))))
+
+    val optimized = Optimize.execute(originalQuery.analyze)
+    val left = testRelation.where('a <= 3 || 'a >= 2).subquery('x)
+    val right = testRelation.subquery('y)
+    val correctAnswer =
+      left.join(right, condition = Some("x.b".attr === "y.b".attr
+        && (("x.a".attr <= 3) || (("x.a".attr >= 2) && ("y.a".attr <= 13)))
+        && (("x.a".attr <= 1) || ("y.a".attr <= 11))))
+        .analyze
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("push down predicates through left join") {
+    val x = testRelation.subquery('x)
+    val y = testRelation.subquery('y)
+
+    val originalQuery =
+      x.join(y, joinType = LeftOuter, condition = Some(("x.b".attr === "y.b".attr)
+        && simpleDisjunctivePredicate))
+
+    val optimized = Optimize.execute(originalQuery.analyze)
+    val left = testRelation.subquery('x)
+    val right = testRelation.where('a > 13 || 'a > 11).subquery('y)
+    val correctAnswer =
+      left.join(right, joinType = LeftOuter, condition = Some("x.b".attr === "y.b".attr
+        && (("x.a".attr > 3) && ("y.a".attr > 13) || ("x.a".attr > 1) && ("y.a".attr > 11))))
+        .analyze
+
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("push down predicates through right join") {
+    val x = testRelation.subquery('x)
+    val y = testRelation.subquery('y)
+
+    val originalQuery =
+      x.join(y, joinType = RightOuter, condition = Some(("x.b".attr === "y.b".attr)
+        && simpleDisjunctivePredicate))
+
+    val optimized = Optimize.execute(originalQuery.analyze)
+    val left = testRelation.where('a > 3 || 'a > 1).subquery('x)
+    val right = testRelation.subquery('y)
+    val correctAnswer =
+      left.join(right, joinType = RightOuter, condition = Some("x.b".attr === "y.b".attr
+        && (("x.a".attr > 3) && ("y.a".attr > 13) || ("x.a".attr > 1) && ("y.a".attr > 11))))
+        .analyze
+
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("SPARK-32302: avoid generating too many predicates") {
+    val x = testRelation.subquery('x)
+    val y = testRelation.subquery('y)
+
+    val originalQuery =
+      x.join(y, condition = Some(("x.b".attr === "y.b".attr) && ((("x.a".attr > 3) &&
+        ("x.a".attr < 13) && ("y.c".attr <= 5)) || (("y.a".attr > 2) && ("y.c".attr < 1)))))
+
+    val optimized = Optimize.execute(originalQuery.analyze)
+    val left = testRelation.subquery('x)
+    val right = testRelation.where('c <= 5 || ('a > 2 && 'c < 1)).subquery('y)
+    val correctAnswer = left.join(right, condition = Some("x.b".attr === "y.b".attr &&
+      ((("x.a".attr > 3) && ("x.a".attr < 13) && ("y.c".attr <= 5)) ||
+        (("y.a".attr > 2) && ("y.c".attr < 1))))).analyze
+
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("push down predicate through multiple joins") {
+    val x = testRelation.subquery('x)
+    val y = testRelation.subquery('y)
+    val z = testRelation.subquery('z)
+    val xJoinY = x.join(y, condition = Some("x.b".attr === "y.b".attr))
+    val originalQuery = z.join(xJoinY,
+      condition = Some("x.a".attr === "z.a".attr && simpleDisjunctivePredicate))
+
+    val optimized = Optimize.execute(originalQuery.analyze)
+    val left = x.where('a > 3 || 'a > 1)
+    val right = y.where('a > 13 || 'a > 11)
+    val correctAnswer = z.join(left.join(right,
+      condition = Some("x.b".attr === "y.b".attr && simpleDisjunctivePredicate)),
+      condition = Some("x.a".attr === "z.a".attr)).analyze
+    comparePlans(optimized, correctAnswer)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FoldablePropagationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FoldablePropagationSuite.scala
index 0d48ecb31cfa4..92e4fa345e2ad 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FoldablePropagationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FoldablePropagationSuite.scala
@@ -156,8 +156,8 @@ class FoldablePropagationSuite extends PlanTest {
     val query = expand.where(a1.isNotNull).select(a1, a2).analyze
     val optimized = Optimize.execute(query)
     val correctExpand = expand.copy(projections = Seq(
-      Seq(Literal(null), c2),
-      Seq(c1, Literal(null))))
+      Seq(Literal(null), Literal(2)),
+      Seq(Literal(1), Literal(null))))
     val correctAnswer = correctExpand.where(a1.isNotNull).select(a1, a2).analyze
     comparePlans(optimized, correctAnswer)
   }
@@ -180,4 +180,28 @@ class FoldablePropagationSuite extends PlanTest {
       .select((Literal(1) + 3).as('res)).analyze
     comparePlans(optimized, correctAnswer)
   }
+
+  test("SPARK-32635: Replace references with foldables coming only from the node's children") {
+    val leftExpression = 'a.int
+    val left = LocalRelation(leftExpression).select('a)
+    val rightExpression = Alias(Literal(2), "a")(leftExpression.exprId)
+    val right = LocalRelation('b.int).select('b, rightExpression).select('b)
+    val join = left.join(right, joinType = LeftOuter, condition = Some('b === 'a))
+
+    val query = join.analyze
+    val optimized = Optimize.execute(query)
+    comparePlans(optimized, query)
+  }
+
+  test("SPARK-32951: Foldable propagation from Aggregate") {
+    val query = testRelation
+      .groupBy('a)('a, sum('b).as('b), Literal(1).as('c))
+      .select('a, 'b, 'c)
+
+    val optimized = Optimize.execute(query.analyze)
+    val correctAnswer = testRelation
+      .groupBy('a)('a, sum('b).as('b), Literal(1).as('c))
+      .select('a, 'b, Literal(1).as('c)).analyze
+    comparePlans(optimized, correctAnswer)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/InferFiltersFromGenerateSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/InferFiltersFromGenerateSuite.scala
new file mode 100644
index 0000000000000..93a1d414ed403
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/InferFiltersFromGenerateSuite.scala
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans._
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
+
+class InferFiltersFromGenerateSuite extends PlanTest {
+  object Optimize extends RuleExecutor[LogicalPlan] {
+    val batches = Batch("Infer Filters", Once, InferFiltersFromGenerate) :: Nil
+  }
+
+  val testRelation = LocalRelation('a.array(StructType(Seq(
+    StructField("x", IntegerType),
+    StructField("y", IntegerType)
+  ))), 'c1.string, 'c2.string)
+
+  Seq(Explode(_), PosExplode(_), Inline(_)).foreach { f =>
+    val generator = f('a)
+    test("Infer filters from " + generator) {
+      val originalQuery = testRelation.generate(generator).analyze
+      val correctAnswer = testRelation
+        .where(IsNotNull('a) && Size('a) > 0)
+        .generate(generator)
+        .analyze
+      val optimized = Optimize.execute(originalQuery)
+      comparePlans(optimized, correctAnswer)
+    }
+
+    test("Don't infer duplicate filters from " + generator) {
+      val originalQuery = testRelation
+        .where(IsNotNull('a) && Size('a) > 0)
+        .generate(generator)
+        .analyze
+      val optimized = Optimize.execute(originalQuery)
+      comparePlans(optimized, originalQuery)
+    }
+
+    test("Don't infer filters from outer " + generator) {
+      val originalQuery = testRelation.generate(generator, outer = true).analyze
+      val optimized = Optimize.execute(originalQuery)
+      comparePlans(optimized, originalQuery)
+    }
+
+    val foldableExplode = f(CreateArray(Seq(
+      CreateStruct(Seq(Literal(0), Literal(1))),
+      CreateStruct(Seq(Literal(2), Literal(3)))
+    )))
+    test("Don't infer filters from " + foldableExplode) {
+      val originalQuery = testRelation.generate(foldableExplode).analyze
+      val optimized = Optimize.execute(originalQuery)
+      comparePlans(optimized, originalQuery)
+    }
+  }
+
+  // setup rules to test inferFilters with ConstantFolding to make sure
+  // the Filter rule added in inferFilters is removed again when doing
+  // explode with CreateArray/CreateMap
+  object OptimizeInferAndConstantFold extends RuleExecutor[LogicalPlan] {
+    val batches =
+      Batch("AnalysisNodes", Once,
+        EliminateSubqueryAliases) ::
+      Batch("Infer Filters", Once, InferFiltersFromGenerate) ::
+      Batch("ConstantFolding after", FixedPoint(4),
+        ConstantFolding,
+        NullPropagation,
+        PruneFilters) :: Nil
+  }
+
+  Seq(Explode(_), PosExplode(_)).foreach { f =>
+     val createArrayExplode = f(CreateArray(Seq('c1)))
+     test("SPARK-33544: Don't infer filters from CreateArray " + createArrayExplode) {
+       val originalQuery = testRelation.generate(createArrayExplode).analyze
+       val optimized = OptimizeInferAndConstantFold.execute(originalQuery)
+       comparePlans(optimized, originalQuery)
+     }
+     val createMapExplode = f(CreateMap(Seq('c1, 'c2)))
+     test("SPARK-33544: Don't infer filters from CreateMap " + createMapExplode) {
+       val originalQuery = testRelation.generate(createMapExplode).analyze
+       val optimized = OptimizeInferAndConstantFold.execute(originalQuery)
+       comparePlans(optimized, originalQuery)
+     }
+   }
+
+   Seq(Inline(_)).foreach { f =>
+     val createArrayStructExplode = f(CreateArray(Seq(CreateStruct(Seq('c1)))))
+     test("SPARK-33544: Don't infer filters from CreateArray " + createArrayStructExplode) {
+       val originalQuery = testRelation.generate(createArrayStructExplode).analyze
+       val optimized = OptimizeInferAndConstantFold.execute(originalQuery)
+       comparePlans(optimized, originalQuery)
+     }
+   }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinSelectionHelperSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinSelectionHelperSuite.scala
new file mode 100644
index 0000000000000..3513cfa14808f
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinSelectionHelperSuite.scala
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.expressions.AttributeMap
+import org.apache.spark.sql.catalyst.plans.{Inner, PlanTest}
+import org.apache.spark.sql.catalyst.plans.logical.{BROADCAST, HintInfo, JoinHint, NO_BROADCAST_HASH, SHUFFLE_HASH}
+import org.apache.spark.sql.catalyst.statsEstimation.StatsTestPlan
+import org.apache.spark.sql.internal.SQLConf
+
+class JoinSelectionHelperSuite extends PlanTest with JoinSelectionHelper {
+
+  private val left = StatsTestPlan(
+    outputList = Seq('a.int, 'b.int, 'c.int),
+    rowCount = 20000000,
+    size = Some(20000000),
+    attributeStats = AttributeMap(Seq()))
+
+  private val right = StatsTestPlan(
+    outputList = Seq('d.int),
+    rowCount = 1000,
+    size = Some(1000),
+    attributeStats = AttributeMap(Seq()))
+
+  private val hintBroadcast = Some(HintInfo(Some(BROADCAST)))
+  private val hintNotToBroadcast = Some(HintInfo(Some(NO_BROADCAST_HASH)))
+  private val hintShuffleHash = Some(HintInfo(Some(SHUFFLE_HASH)))
+
+  test("getBroadcastBuildSide (hintOnly = true) return BuildLeft with only a left hint") {
+    val broadcastSide = getBroadcastBuildSide(
+      left,
+      right,
+      Inner,
+      JoinHint(hintBroadcast, None),
+      hintOnly = true,
+      SQLConf.get
+    )
+    assert(broadcastSide === Some(BuildLeft))
+  }
+
+  test("getBroadcastBuildSide (hintOnly = true) return BuildRight with only a right hint") {
+    val broadcastSide = getBroadcastBuildSide(
+      left,
+      right,
+      Inner,
+      JoinHint(None, hintBroadcast),
+      hintOnly = true,
+      SQLConf.get
+    )
+    assert(broadcastSide === Some(BuildRight))
+  }
+
+  test("getBroadcastBuildSide (hintOnly = true) return smaller side with both having hints") {
+    val broadcastSide = getBroadcastBuildSide(
+      left,
+      right,
+      Inner,
+      JoinHint(hintBroadcast, hintBroadcast),
+      hintOnly = true,
+      SQLConf.get
+    )
+    assert(broadcastSide === Some(BuildRight))
+  }
+
+  test("getBroadcastBuildSide (hintOnly = true) return None when no side has a hint") {
+    val broadcastSide = getBroadcastBuildSide(
+      left,
+      right,
+      Inner,
+      JoinHint(None, None),
+      hintOnly = true,
+      SQLConf.get
+    )
+    assert(broadcastSide === None)
+  }
+
+  test("getBroadcastBuildSide (hintOnly = false) return BuildRight when right is broadcastable") {
+    val broadcastSide = getBroadcastBuildSide(
+      left,
+      right,
+      Inner,
+      JoinHint(None, None),
+      hintOnly = false,
+      SQLConf.get
+    )
+    assert(broadcastSide === Some(BuildRight))
+  }
+
+  test("getBroadcastBuildSide (hintOnly = false) return None when right has no broadcast hint") {
+    val broadcastSide = getBroadcastBuildSide(
+      left,
+      right,
+      Inner,
+      JoinHint(None, hintNotToBroadcast ),
+      hintOnly = false,
+      SQLConf.get
+    )
+    assert(broadcastSide === None)
+  }
+
+  test("getShuffleHashJoinBuildSide (hintOnly = true) return BuildLeft with only a left hint") {
+    val broadcastSide = getShuffleHashJoinBuildSide(
+      left,
+      right,
+      Inner,
+      JoinHint(hintShuffleHash, None),
+      hintOnly = true,
+      SQLConf.get
+    )
+    assert(broadcastSide === Some(BuildLeft))
+  }
+
+  test("getShuffleHashJoinBuildSide (hintOnly = true) return BuildRight with only a right hint") {
+    val broadcastSide = getShuffleHashJoinBuildSide(
+      left,
+      right,
+      Inner,
+      JoinHint(None, hintShuffleHash),
+      hintOnly = true,
+      SQLConf.get
+    )
+    assert(broadcastSide === Some(BuildRight))
+  }
+
+  test("getShuffleHashJoinBuildSide (hintOnly = true) return smaller side when both have hints") {
+    val broadcastSide = getShuffleHashJoinBuildSide(
+      left,
+      right,
+      Inner,
+      JoinHint(hintShuffleHash, hintShuffleHash),
+      hintOnly = true,
+      SQLConf.get
+    )
+    assert(broadcastSide === Some(BuildRight))
+  }
+
+  test("getShuffleHashJoinBuildSide (hintOnly = true) return None when no side has a hint") {
+    val broadcastSide = getShuffleHashJoinBuildSide(
+      left,
+      right,
+      Inner,
+      JoinHint(None, None),
+      hintOnly = true,
+      SQLConf.get
+    )
+    assert(broadcastSide === None)
+  }
+
+  test("getShuffleHashJoinBuildSide (hintOnly = false) return BuildRight when right is smaller") {
+    val broadcastSide = getBroadcastBuildSide(
+      left,
+      right,
+      Inner,
+      JoinHint(None, None),
+      hintOnly = false,
+      SQLConf.get
+    )
+    assert(broadcastSide === Some(BuildRight))
+  }
+
+  test("getSmallerSide should return BuildRight") {
+    assert(getSmallerSide(left, right) === BuildRight)
+  }
+
+  test("canBroadcastBySize should return true if the plan size is less than 10MB") {
+    assert(canBroadcastBySize(left, SQLConf.get) === false)
+    assert(canBroadcastBySize(right, SQLConf.get) === true)
+  }
+
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LeftSemiAntiJoinPushDownSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LeftSemiAntiJoinPushDownSuite.scala
index a3da9f73ebd40..88c29c9274a68 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LeftSemiAntiJoinPushDownSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LeftSemiAntiJoinPushDownSuite.scala
@@ -24,6 +24,7 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules._
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.IntegerType
 
 class LeftSemiPushdownSuite extends PlanTest {
@@ -60,7 +61,7 @@ class LeftSemiPushdownSuite extends PlanTest {
 
   test("Project: LeftSemiAnti join no pushdown because of non-deterministic proj exprs") {
     val originalQuery = testRelation
-      .select(Rand('a), 'b, 'c)
+      .select(Rand(1), 'b, 'c)
       .join(testRelation1, joinType = LeftSemi, condition = Some('b === 'd))
 
     val optimized = Optimize.execute(originalQuery.analyze)
@@ -239,14 +240,23 @@ class LeftSemiPushdownSuite extends PlanTest {
     comparePlans(optimized, correctAnswer)
   }
 
-  test("Union: LeftSemiAnti join no pushdown in self join scenario") {
+  test("Union: LeftSemiAnti join pushdown in self join scenario") {
     val testRelation2 = LocalRelation('x.int, 'y.int, 'z.int)
+    val attrX = testRelation2.output.head
 
     val originalQuery = Union(Seq(testRelation, testRelation2))
-      .join(testRelation2, joinType = LeftSemi, condition = Some('a === 'x))
+      .join(testRelation2, joinType = LeftSemi, condition = Some('a === attrX))
 
     val optimized = Optimize.execute(originalQuery.analyze)
-    comparePlans(optimized, originalQuery.analyze)
+
+    val correctAnswer = Union(Seq(
+      testRelation.join(testRelation2, joinType = LeftSemi, condition = Some('a === 'x)),
+      // We can't construct the actual query, as relations deduplication will create new attribute
+      // IDs. Here we use a fake join condition (always true) to verify the query plan shape.
+      testRelation2.join(testRelation2, joinType = LeftSemi, condition = Some(attrX === attrX))))
+      .analyze
+
+    comparePlans(optimized, correctAnswer)
   }
 
   test("Unary: LeftSemiAnti join pushdown") {
@@ -315,6 +325,21 @@ class LeftSemiPushdownSuite extends PlanTest {
     comparePlans(optimized, originalQuery.analyze)
   }
 
+  test("Unary: LeftSemi join push down through Expand") {
+    val expand = Expand(Seq(Seq('a, 'b, "null"), Seq('a, "null", 'c)),
+      Seq('a, 'b, 'c), testRelation)
+    val originalQuery = expand
+      .join(testRelation1, joinType = LeftSemi, condition = Some('b === 'd && 'b === 1))
+
+    val optimized = Optimize.execute(originalQuery.analyze)
+    val correctAnswer = Expand(Seq(Seq('a, 'b, "null"), Seq('a, "null", 'c)),
+      Seq('a, 'b, 'c), testRelation
+        .join(testRelation1, joinType = LeftSemi, condition = Some('b === 'd && 'b === 1)))
+      .analyze
+
+    comparePlans(optimized, correctAnswer)
+  }
+
   Seq(Some('d === 'e), None).foreach { case innerJoinCond =>
     Seq(LeftSemi, LeftAnti).foreach { case outerJT =>
       Seq(Inner, LeftOuter, Cross, RightOuter).foreach { case innerJT =>
@@ -428,4 +453,28 @@ class LeftSemiPushdownSuite extends PlanTest {
     }
   }
 
+  Seq(LeftSemi, LeftAnti).foreach { jt =>
+    test(s"SPARK-34081: $jt only push down if join can be planned as broadcast join") {
+      Seq(-1, 100000).foreach { threshold =>
+        withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> threshold.toString) {
+          val originalQuery = testRelation
+            .groupBy('b)('b)
+            .join(testRelation1, joinType = jt, condition = Some('b <=> 'd))
+
+          val optimized = Optimize.execute(originalQuery.analyze)
+          val correctAnswer = if (threshold > 0) {
+            testRelation
+              .join(testRelation1, joinType = jt, condition = Some('b <=> 'd))
+              .groupBy('b)('b)
+              .analyze
+          } else {
+            originalQuery.analyze
+          }
+
+          comparePlans(optimized, correctAnswer)
+        }
+      }
+    }
+  }
+
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LikeSimplificationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LikeSimplificationSuite.scala
index 436f62e4225c8..c06c92f9c1511 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LikeSimplificationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LikeSimplificationSuite.scala
@@ -116,4 +116,120 @@ class LikeSimplificationSuite extends PlanTest {
     val optimized2 = Optimize.execute(originalQuery2.analyze)
     comparePlans(optimized2, originalQuery2.analyze)
   }
+
+  test("SPARK-33677: LikeSimplification should be skipped if pattern contains any escapeChar") {
+    val originalQuery1 =
+      testRelation
+        .where(('a like "abc%") || ('a like "\\abc%"))
+    val optimized1 = Optimize.execute(originalQuery1.analyze)
+    val correctAnswer1 = testRelation
+      .where(StartsWith('a, "abc") || ('a like "\\abc%"))
+      .analyze
+    comparePlans(optimized1, correctAnswer1)
+
+    val originalQuery2 =
+      testRelation
+        .where(('a like "%xyz") || ('a like "%xyz\\"))
+    val optimized2 = Optimize.execute(originalQuery2.analyze)
+    val correctAnswer2 = testRelation
+      .where(EndsWith('a, "xyz") || ('a like "%xyz\\"))
+      .analyze
+    comparePlans(optimized2, correctAnswer2)
+
+    val originalQuery3 =
+      testRelation
+        .where(('a like ("@bc%def", '@')) || ('a like "abc%def"))
+    val optimized3 = Optimize.execute(originalQuery3.analyze)
+    val correctAnswer3 = testRelation
+      .where(('a like ("@bc%def", '@')) ||
+        (Length('a) >= 6 && (StartsWith('a, "abc") && EndsWith('a, "def"))))
+      .analyze
+    comparePlans(optimized3, correctAnswer3)
+
+    val originalQuery4 =
+      testRelation
+        .where(('a like "%mn%") || ('a like ("%mn%", '%')))
+    val optimized4 = Optimize.execute(originalQuery4.analyze)
+    val correctAnswer4 = testRelation
+      .where(Contains('a, "mn") || ('a like ("%mn%", '%')))
+      .analyze
+    comparePlans(optimized4, correctAnswer4)
+
+    val originalQuery5 =
+      testRelation
+        .where(('a like "abc") || ('a like ("abbc", 'b')))
+    val optimized5 = Optimize.execute(originalQuery5.analyze)
+    val correctAnswer5 = testRelation
+      .where(('a === "abc") || ('a like ("abbc", 'b')))
+      .analyze
+    comparePlans(optimized5, correctAnswer5)
+  }
+
+  test("simplify LikeAll") {
+    val originalQuery =
+      testRelation
+        .where(('a likeAll(
+    "abc%", "abc\\%", "%xyz", "abc\\%def", "abc%def", "%mn%", "%mn\\%", "", "abc")))
+
+    val optimized = Optimize.execute(originalQuery.analyze)
+    val correctAnswer = testRelation
+      .where((((((StartsWith('a, "abc") && EndsWith('a, "xyz")) &&
+        (Length('a) >= 6 && (StartsWith('a, "abc") && EndsWith('a, "def")))) &&
+        Contains('a, "mn")) && ('a === "")) && ('a === "abc")) &&
+        ('a likeAll("abc\\%", "abc\\%def", "%mn\\%")))
+      .analyze
+
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("simplify NotLikeAll") {
+    val originalQuery =
+      testRelation
+        .where(('a notLikeAll(
+          "abc%", "abc\\%", "%xyz", "abc\\%def", "abc%def", "%mn%", "%mn\\%", "", "abc")))
+
+    val optimized = Optimize.execute(originalQuery.analyze)
+    val correctAnswer = testRelation
+      .where((((((Not(StartsWith('a, "abc")) && Not(EndsWith('a, "xyz"))) &&
+        Not(Length('a) >= 6 && (StartsWith('a, "abc") && EndsWith('a, "def")))) &&
+        Not(Contains('a, "mn"))) && Not('a === "")) && Not('a === "abc")) &&
+        ('a notLikeAll("abc\\%", "abc\\%def", "%mn\\%")))
+      .analyze
+
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("simplify LikeAny") {
+    val originalQuery =
+      testRelation
+        .where(('a likeAny(
+          "abc%", "abc\\%", "%xyz", "abc\\%def", "abc%def", "%mn%", "%mn\\%", "", "abc")))
+
+    val optimized = Optimize.execute(originalQuery.analyze)
+    val correctAnswer = testRelation
+      .where((((((StartsWith('a, "abc") || EndsWith('a, "xyz")) ||
+        (Length('a) >= 6 && (StartsWith('a, "abc") && EndsWith('a, "def")))) ||
+        Contains('a, "mn")) || ('a === "")) || ('a === "abc")) ||
+        ('a likeAny("abc\\%", "abc\\%def", "%mn\\%")))
+      .analyze
+
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("simplify NotLikeAny") {
+    val originalQuery =
+      testRelation
+        .where(('a notLikeAny(
+          "abc%", "abc\\%", "%xyz", "abc\\%def", "abc%def", "%mn%", "%mn\\%", "", "abc")))
+
+    val optimized = Optimize.execute(originalQuery.analyze)
+    val correctAnswer = testRelation
+      .where((((((Not(StartsWith('a, "abc")) || Not(EndsWith('a, "xyz"))) ||
+        Not(Length('a) >= 6 && (StartsWith('a, "abc") && EndsWith('a, "def")))) ||
+        Not(Contains('a, "mn"))) || Not('a === "")) || Not('a === "abc")) ||
+        ('a notLikeAny("abc\\%", "abc\\%def", "%mn\\%")))
+      .analyze
+
+    comparePlans(optimized, correctAnswer)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LimitPushdownSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LimitPushdownSuite.scala
index 17fb9fc5d11e3..c2503e362c8c8 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LimitPushdownSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LimitPushdownSuite.scala
@@ -17,11 +17,12 @@
 
 package org.apache.spark.sql.catalyst.optimizer
 
+import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.expressions.Add
-import org.apache.spark.sql.catalyst.plans.{FullOuter, LeftOuter, PlanTest, RightOuter}
+import org.apache.spark.sql.catalyst.plans.{Cross, FullOuter, Inner, LeftAnti, LeftOuter, LeftSemi, PlanTest, RightOuter}
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules._
 
@@ -33,13 +34,17 @@ class LimitPushdownSuite extends PlanTest {
         EliminateSubqueryAliases) ::
       Batch("Limit pushdown", FixedPoint(100),
         LimitPushDown,
-        CombineLimits,
+        EliminateLimits,
         ConstantFolding,
         BooleanSimplification) :: Nil
   }
 
-  private val testRelation = LocalRelation('a.int, 'b.int, 'c.int)
-  private val testRelation2 = LocalRelation('d.int, 'e.int, 'f.int)
+  private val testRelation = LocalRelation.fromExternalRows(
+    Seq("a".attr.int, "b".attr.int, "c".attr.int),
+    1.to(6).map(_ => Row(1, 2, 3)))
+  private val testRelation2 = LocalRelation.fromExternalRows(
+    Seq("d".attr.int, "e".attr.int, "f".attr.int),
+    1.to(6).map(_ => Row(1, 2, 3)))
   private val x = testRelation.subquery('x)
   private val y = testRelation.subquery('y)
 
@@ -74,7 +79,7 @@ class LimitPushdownSuite extends PlanTest {
       Union(testRelation.limit(1), testRelation2.select('d, 'e, 'f).limit(1)).limit(2)
     val unionOptimized = Optimize.execute(unionQuery.analyze)
     val unionCorrectAnswer =
-      Limit(2, Union(testRelation.limit(1), testRelation2.select('d, 'e, 'f).limit(1))).analyze
+      Union(testRelation.limit(1), testRelation2.select('d, 'e, 'f).limit(1)).analyze
     comparePlans(unionOptimized, unionCorrectAnswer)
   }
 
@@ -148,7 +153,7 @@ class LimitPushdownSuite extends PlanTest {
   }
 
   test("full outer join where neither side is limited and left side has larger statistics") {
-    val xBig = testRelation.copy(data = Seq.fill(2)(null)).subquery('x)
+    val xBig = testRelation.copy(data = Seq.fill(10)(null)).subquery('x)
     assert(xBig.stats.sizeInBytes > y.stats.sizeInBytes)
     val originalQuery = xBig.join(y, FullOuter).limit(1).analyze
     val optimized = Optimize.execute(originalQuery)
@@ -157,7 +162,7 @@ class LimitPushdownSuite extends PlanTest {
   }
 
   test("full outer join where neither side is limited and right side has larger statistics") {
-    val yBig = testRelation.copy(data = Seq.fill(2)(null)).subquery('y)
+    val yBig = testRelation.copy(data = Seq.fill(10)(null)).subquery('y)
     assert(x.stats.sizeInBytes < yBig.stats.sizeInBytes)
     val originalQuery = x.join(yBig, FullOuter).limit(1).analyze
     val optimized = Optimize.execute(originalQuery)
@@ -171,4 +176,67 @@ class LimitPushdownSuite extends PlanTest {
     // No pushdown for FULL OUTER JOINS.
     comparePlans(optimized, originalQuery)
   }
+
+  test("SPARK-33433: Change Aggregate max rows to 1 if grouping is empty") {
+    val analyzed1 = Limit(1, Union(
+      x.groupBy()(count(1)),
+      y.groupBy()(count(1)))).analyze
+    val optimized1 = Optimize.execute(analyzed1)
+    comparePlans(analyzed1, optimized1)
+
+    // test push down
+    val analyzed2 = Limit(1, Union(
+      x.groupBy(Symbol("a"))(count(1)),
+      y.groupBy(Symbol("b"))(count(1)))).analyze
+    val optimized2 = Optimize.execute(analyzed2)
+    val expected2 = Limit(1, Union(
+      LocalLimit(1, x.groupBy(Symbol("a"))(count(1))),
+      LocalLimit(1, y.groupBy(Symbol("b"))(count(1))))).analyze
+    comparePlans(expected2, optimized2)
+  }
+
+  test("SPARK-26138: pushdown limit through InnerLike when condition is empty") {
+    Seq(Cross, Inner).foreach { joinType =>
+      val originalQuery = x.join(y, joinType).limit(1)
+      val optimized = Optimize.execute(originalQuery.analyze)
+      val correctAnswer = Limit(1, LocalLimit(1, x).join(LocalLimit(1, y), joinType)).analyze
+      comparePlans(optimized, correctAnswer)
+    }
+  }
+
+  test("SPARK-26138: Should not pushdown limit through InnerLike when condition is not empty") {
+    Seq(Cross, Inner).foreach { joinType =>
+      val originalQuery = x.join(y, joinType, Some("x.a".attr === "y.b".attr)).limit(1)
+      val optimized = Optimize.execute(originalQuery.analyze)
+      val correctAnswer = Limit(1, x.join(y, joinType, Some("x.a".attr === "y.b".attr))).analyze
+      comparePlans(optimized, correctAnswer)
+    }
+  }
+
+  test("SPARK-34514: Push down limit through LEFT SEMI and LEFT ANTI join") {
+    // Push down when condition is empty
+    Seq(LeftSemi, LeftAnti).foreach { joinType =>
+      val originalQuery = x.join(y, joinType).limit(1)
+      val optimized = Optimize.execute(originalQuery.analyze)
+      val correctAnswer = Limit(1, LocalLimit(1, x).join(y, joinType)).analyze
+      comparePlans(optimized, correctAnswer)
+    }
+
+    // No push down when condition is not empty
+    Seq(LeftSemi, LeftAnti).foreach { joinType =>
+      val originalQuery = x.join(y, joinType, Some("x.a".attr === "y.b".attr)).limit(1)
+      val optimized = Optimize.execute(originalQuery.analyze)
+      val correctAnswer = Limit(1, x.join(y, joinType, Some("x.a".attr === "y.b".attr))).analyze
+      comparePlans(optimized, correctAnswer)
+    }
+  }
+
+  test("SPARK-34622: Fix Push down limit through join if its output is not match the LocalLimit") {
+    val joinCondition = Some("x.a".attr === "y.a".attr && "x.b".attr === "y.b".attr)
+    val originalQuery = x.join(y, LeftOuter, joinCondition).select("x.a".attr).limit(5)
+    val optimized = Optimize.execute(originalQuery.analyze)
+    val correctAnswer =
+      Limit(5, LocalLimit(5, x).join(y, LeftOuter, joinCondition).select("x.a".attr)).analyze
+    comparePlans(optimized, correctAnswer)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LimitPushdownThroughWindowSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LimitPushdownThroughWindowSuite.scala
new file mode 100644
index 0000000000000..f2c1f452d0203
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LimitPushdownThroughWindowSuite.scala
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions.{CurrentRow, Rank, RowFrame, RowNumber, SpecifiedWindowFrame, UnboundedPreceding}
+import org.apache.spark.sql.catalyst.plans._
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules._
+import org.apache.spark.sql.internal.SQLConf
+
+class LimitPushdownThroughWindowSuite extends PlanTest {
+  // CollapseProject and RemoveNoopOperators is needed because we need it to collapse project.
+  private val limitPushdownRules = Seq(
+    CollapseProject,
+    RemoveNoopOperators,
+    LimitPushDownThroughWindow,
+    EliminateLimits,
+    ConstantFolding,
+    BooleanSimplification)
+
+  private object Optimize extends RuleExecutor[LogicalPlan] {
+    val batches =
+      Batch("Limit pushdown through window", FixedPoint(100),
+        limitPushdownRules: _*) :: Nil
+  }
+
+  private object WithoutOptimize extends RuleExecutor[LogicalPlan] {
+    val batches =
+      Batch("Without limit pushdown through window", FixedPoint(100),
+        limitPushdownRules
+          .filterNot(_.ruleName.equals(LimitPushDownThroughWindow.ruleName)): _*) :: Nil
+  }
+
+  private val testRelation = LocalRelation.fromExternalRows(
+    Seq("a".attr.int, "b".attr.int, "c".attr.int),
+    1.to(6).map(_ => Row(1, 2, 3)))
+
+  private val a = testRelation.output(0)
+  private val b = testRelation.output(1)
+  private val c = testRelation.output(2)
+  private val windowFrame = SpecifiedWindowFrame(RowFrame, UnboundedPreceding, CurrentRow)
+
+  test("Push down limit through window when partitionSpec is empty") {
+    val originalQuery = testRelation
+      .select(a, b, c,
+        windowExpr(RowNumber(), windowSpec(Nil, c.desc :: Nil, windowFrame)).as("rn"))
+      .limit(2)
+    val correctAnswer = testRelation
+      .select(a, b, c)
+      .orderBy(c.desc)
+      .limit(2)
+      .select(a, b, c,
+        windowExpr(RowNumber(), windowSpec(Nil, c.desc :: Nil, windowFrame)).as("rn"))
+
+    comparePlans(
+      Optimize.execute(originalQuery.analyze),
+      WithoutOptimize.execute(correctAnswer.analyze))
+  }
+
+  test("Push down limit through window for multiple window functions") {
+    val originalQuery = testRelation
+      .select(a, b, c,
+        windowExpr(RowNumber(), windowSpec(Nil, c.desc :: Nil, windowFrame)).as("rn"),
+        windowExpr(new Rank(), windowSpec(Nil, c.desc :: Nil, windowFrame)).as("rk"))
+      .limit(2)
+    val correctAnswer = testRelation
+      .select(a, b, c)
+      .orderBy(c.desc)
+      .limit(2)
+      .select(a, b, c,
+        windowExpr(RowNumber(), windowSpec(Nil, c.desc :: Nil, windowFrame)).as("rn"),
+        windowExpr(new Rank(), windowSpec(Nil, c.desc :: Nil, windowFrame)).as("rk"))
+
+    comparePlans(
+      Optimize.execute(originalQuery.analyze),
+      WithoutOptimize.execute(correctAnswer.analyze))
+  }
+
+  test("Push down limit through window respect spark.sql.execution.topKSortFallbackThreshold") {
+    Seq(1, 100).foreach { threshold =>
+      withSQLConf(SQLConf.TOP_K_SORT_FALLBACK_THRESHOLD.key -> threshold.toString) {
+        val originalQuery = testRelation
+          .select(a, b, c,
+            windowExpr(RowNumber(), windowSpec(Nil, c.desc :: Nil, windowFrame)).as("rn"))
+          .limit(2)
+        val correctAnswer = if (threshold == 1) {
+          originalQuery
+        } else {
+          testRelation
+            .select(a, b, c)
+            .orderBy(c.desc)
+            .limit(2)
+            .select(a, b, c,
+              windowExpr(RowNumber(), windowSpec(Nil, c.desc :: Nil, windowFrame)).as("rn"))
+        }
+
+        comparePlans(
+          Optimize.execute(originalQuery.analyze),
+          WithoutOptimize.execute(correctAnswer.analyze))
+      }
+    }
+  }
+
+  test("Push down to first window if order column is different") {
+    val originalQuery = testRelation
+      .select(a, b, c,
+        windowExpr(RowNumber(), windowSpec(Nil, b.desc :: Nil, windowFrame)).as("rn"),
+        windowExpr(new Rank(), windowSpec(Nil, c.asc :: Nil, windowFrame)).as("rk"))
+      .limit(2)
+    val correctAnswer = testRelation
+      .select(a, b, c,
+        windowExpr(RowNumber(), windowSpec(Nil, b.desc :: Nil, windowFrame)).as("rn"))
+      .orderBy(c.asc)
+      .limit(2)
+      .select(a, b, c, $"rn".attr,
+        windowExpr(new Rank(), windowSpec(Nil, c.asc :: Nil, windowFrame)).as("rk"))
+
+    comparePlans(
+      Optimize.execute(originalQuery.analyze),
+      WithoutOptimize.execute(correctAnswer.analyze))
+  }
+
+  test("Push down if there is a Project between LocalLimit and Window") {
+    val originalQuery = testRelation
+      .select(a, b,
+        windowExpr(RowNumber(), windowSpec(Nil, b.desc :: Nil, windowFrame)).as("rn"))
+      .select(a, $"rn".attr)
+      .limit(2)
+    val correctAnswer = testRelation
+      .select(a, b)
+      .orderBy(b.desc)
+      .limit(2)
+      .select(a, windowExpr(RowNumber(), windowSpec(Nil, b.desc :: Nil, windowFrame)).as("rn"))
+
+    comparePlans(
+      Optimize.execute(originalQuery.analyze),
+      WithoutOptimize.execute(correctAnswer.analyze))
+  }
+
+  test("Should not push down if partitionSpec is not empty") {
+    val originalQuery = testRelation
+      .select(a, b, c,
+        windowExpr(RowNumber(), windowSpec(a :: Nil, c.desc :: Nil, windowFrame)).as("rn"))
+      .limit(2)
+
+    comparePlans(
+      Optimize.execute(originalQuery.analyze),
+      WithoutOptimize.execute(originalQuery.analyze))
+  }
+
+  test("Should not push down when child's maxRows smaller than limit value") {
+    val originalQuery = testRelation
+      .select(a, b, c,
+        windowExpr(RowNumber(), windowSpec(Nil, c.desc :: Nil, windowFrame)).as("rn"))
+      .limit(20)
+
+    comparePlans(
+      Optimize.execute(originalQuery.analyze),
+      WithoutOptimize.execute(originalQuery.analyze))
+  }
+
+  test("Should not push down if it is not RankLike/RowNumberLike window function") {
+    val originalQuery = testRelation
+      .select(a, b, c,
+        windowExpr(count(b), windowSpec(Nil, c.desc :: Nil, windowFrame)).as("rn"))
+      .limit(2)
+
+    comparePlans(
+      Optimize.execute(originalQuery.analyze),
+      WithoutOptimize.execute(originalQuery.analyze))
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasingSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasingSuite.scala
index d4d6f79d7895e..a856caa6781e8 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasingSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasingSuite.scala
@@ -144,11 +144,8 @@ class NestedColumnAliasingSuite extends SchemaPruningTest {
   test("Pushing a single nested field projection - negative") {
     val ops = Seq(
       (input: LogicalPlan) => input.distribute('name)(1),
-      (input: LogicalPlan) => input.distribute($"name.middle")(1),
       (input: LogicalPlan) => input.orderBy('name.asc),
-      (input: LogicalPlan) => input.orderBy($"name.middle".asc),
       (input: LogicalPlan) => input.sortBy('name.asc),
-      (input: LogicalPlan) => input.sortBy($"name.middle".asc),
       (input: LogicalPlan) => input.union(input)
     )
 
@@ -164,7 +161,7 @@ class NestedColumnAliasingSuite extends SchemaPruningTest {
       comparePlans(optimized, expected)
     }
     val expectedUnion =
-      contact.select('name).union(contact.select('name.as('name)))
+      contact.select('name).union(contact.select('name))
         .select(GetStructField('name, 1, Some("middle"))).analyze
     comparePlans(optimizedUnion, expectedUnion)
   }
@@ -262,6 +259,37 @@ class NestedColumnAliasingSuite extends SchemaPruningTest {
     comparePlans(optimized, expected)
   }
 
+  test("SPARK-27633: Do not generate redundant aliases if parent nested field is aliased too") {
+    val nestedRelation = LocalRelation('a.struct('b.struct('c.int,
+      'd.struct('f.int, 'g.int)), 'e.int))
+
+    // `a.b`
+    val first = 'a.getField("b")
+    // `a.b.c` + 1
+    val second = 'a.getField("b").getField("c") + Literal(1)
+    // `a.b.d.f`
+    val last = 'a.getField("b").getField("d").getField("f")
+
+    val query = nestedRelation
+      .limit(5)
+      .select(first, second, last)
+      .analyze
+
+    val optimized = Optimize.execute(query)
+
+    val aliases = collectGeneratedAliases(optimized)
+
+    val expected = nestedRelation
+      .select(first.as(aliases(0)))
+      .limit(5)
+      .select($"${aliases(0)}".as("a.b"),
+        ($"${aliases(0)}".getField("c") + Literal(1)).as("(a.b.c + 1)"),
+        $"${aliases(0)}".getField("d").getField("f").as("a.b.d.f"))
+      .analyze
+
+    comparePlans(optimized, expected)
+  }
+
   test("Nested field pruning for Project and Generate") {
     val query = contact
       .generate(Explode('friends.getField("first")), outputNames = Seq("explode"))
@@ -301,14 +329,14 @@ class NestedColumnAliasingSuite extends SchemaPruningTest {
     comparePlans(optimized, expected)
   }
 
-  test("Nested field pruning for Project and Generate: not prune on generator output") {
+  test("Nested field pruning for Project and Generate: multiple-field case is not supported") {
     val companies = LocalRelation(
       'id.int,
       'employers.array(employer))
 
     val query = companies
       .generate(Explode('employers.getField("company")), outputNames = Seq("company"))
-      .select('company.getField("name"))
+      .select('company.getField("name"), 'company.getField("address"))
       .analyze
     val optimized = Optimize.execute(query)
 
@@ -319,7 +347,8 @@ class NestedColumnAliasingSuite extends SchemaPruningTest {
       .generate(Explode($"${aliases(0)}"),
         unrequiredChildIndex = Seq(0),
         outputNames = Seq("company"))
-      .select('company.getField("name").as("company.name"))
+      .select('company.getField("name").as("company.name"),
+        'company.getField("address").as("company.address"))
       .analyze
     comparePlans(optimized, expected)
   }
@@ -341,6 +370,344 @@ class NestedColumnAliasingSuite extends SchemaPruningTest {
       .analyze
     comparePlans(optimized, expected)
   }
+
+  test("Nested field pruning through RepartitionByExpression") {
+    val query1 = contact
+      .distribute($"id")(1)
+      .select($"name.middle")
+      .analyze
+    val optimized1 = Optimize.execute(query1)
+
+    val aliases1 = collectGeneratedAliases(optimized1)
+
+    val expected1 = contact
+      .select('id, 'name.getField("middle").as(aliases1(0)))
+      .distribute($"id")(1)
+      .select($"${aliases1(0)}".as("middle"))
+      .analyze
+    comparePlans(optimized1, expected1)
+
+    val query2 = contact
+      .distribute($"name.middle")(1)
+      .select($"name.middle")
+      .analyze
+    val optimized2 = Optimize.execute(query2)
+
+    val aliases2 = collectGeneratedAliases(optimized2)
+
+    val expected2 = contact
+      .select('name.getField("middle").as(aliases2(0)))
+      .distribute($"${aliases2(0)}")(1)
+      .select($"${aliases2(0)}".as("middle"))
+      .analyze
+    comparePlans(optimized2, expected2)
+
+    val query3 = contact
+      .select($"name")
+      .distribute($"name")(1)
+      .select($"name.middle")
+      .analyze
+    val optimized3 = Optimize.execute(query3)
+
+    comparePlans(optimized3, query3)
+  }
+
+  test("Nested field pruning through Join") {
+    val department = LocalRelation(
+      'depID.int,
+      'personID.string)
+
+    val query1 = contact.join(department, condition = Some($"id" === $"depID"))
+      .select($"name.middle")
+      .analyze
+    val optimized1 = Optimize.execute(query1)
+
+    val aliases1 = collectGeneratedAliases(optimized1)
+
+    val expected1 = contact.select('id, 'name.getField("middle").as(aliases1(0)))
+      .join(department.select('depID), condition = Some($"id" === $"depID"))
+      .select($"${aliases1(0)}".as("middle"))
+      .analyze
+    comparePlans(optimized1, expected1)
+
+    val query2 = contact.join(department, condition = Some($"name.middle" === $"personID"))
+      .select($"name.first")
+      .analyze
+    val optimized2 = Optimize.execute(query2)
+
+    val aliases2 = collectGeneratedAliases(optimized2)
+
+    val expected2 = contact.select(
+      'name.getField("first").as(aliases2(0)),
+      'name.getField("middle").as(aliases2(1)))
+      .join(department.select('personID), condition = Some($"${aliases2(1)}" === $"personID"))
+      .select($"${aliases2(0)}".as("first"))
+      .analyze
+    comparePlans(optimized2, expected2)
+
+    val contact2 = LocalRelation('name2.struct(name))
+    val query3 = contact.select('name)
+      .join(contact2, condition = Some($"name" === $"name2"))
+      .select($"name.first")
+      .analyze
+    val optimized3 = Optimize.execute(query3)
+    comparePlans(optimized3, query3)
+  }
+
+  test("Nested field pruning for Aggregate") {
+    def runTest(basePlan: LogicalPlan => LogicalPlan): Unit = {
+      val query1 = basePlan(contact).groupBy($"id")(first($"name.first").as("first")).analyze
+      val optimized1 = Optimize.execute(query1)
+      val aliases1 = collectGeneratedAliases(optimized1)
+
+      val expected1 = basePlan(
+        contact
+        .select($"id", 'name.getField("first").as(aliases1(0)))
+      ).groupBy($"id")(first($"${aliases1(0)}").as("first")).analyze
+      comparePlans(optimized1, expected1)
+
+      val query2 = basePlan(contact).groupBy($"name.last")(first($"name.first").as("first")).analyze
+      val optimized2 = Optimize.execute(query2)
+      val aliases2 = collectGeneratedAliases(optimized2)
+
+      val expected2 = basePlan(
+        contact
+        .select('name.getField("last").as(aliases2(0)), 'name.getField("first").as(aliases2(1)))
+      ).groupBy($"${aliases2(0)}")(first($"${aliases2(1)}").as("first")).analyze
+      comparePlans(optimized2, expected2)
+    }
+
+    Seq(
+      (plan: LogicalPlan) => plan,
+      (plan: LogicalPlan) => plan.limit(100),
+      (plan: LogicalPlan) => plan.repartition(100),
+      (plan: LogicalPlan) => Sample(0.0, 0.6, false, 11L, plan)).foreach {  base =>
+      runTest(base)
+    }
+
+    val query3 = contact.groupBy($"id")(first($"name"), first($"name.first").as("first")).analyze
+    val optimized3 = Optimize.execute(query3)
+    val expected3 = contact.select($"id", $"name")
+      .groupBy($"id")(first($"name"), first($"name.first").as("first")).analyze
+    comparePlans(optimized3, expected3)
+  }
+
+  test("Nested field pruning for Window") {
+    val spec = windowSpec($"address" :: Nil, $"id".asc :: Nil, UnspecifiedFrame)
+    val winExpr = windowExpr(RowNumber(), spec)
+    val query = contact
+      .select($"name.first", winExpr.as('window))
+      .orderBy($"name.last".asc)
+      .analyze
+    val optimized = Optimize.execute(query)
+    val aliases = collectGeneratedAliases(optimized)
+    val expected = contact
+      .select($"name.first", $"address", $"id", $"name.last".as(aliases(1)))
+      .window(Seq(winExpr.as("window")), Seq($"address"), Seq($"id".asc))
+      .select($"first", $"window", $"${aliases(1)}".as(aliases(0)))
+      .orderBy($"${aliases(0)}".asc)
+      .select($"first", $"window")
+      .analyze
+    comparePlans(optimized, expected)
+  }
+
+  test("Nested field pruning for Filter with other supported operators") {
+    val spec = windowSpec($"address" :: Nil, $"id".asc :: Nil, UnspecifiedFrame)
+    val winExpr = windowExpr(RowNumber(), spec)
+    val query1 = contact.select($"name.first", winExpr.as('window))
+      .where($"window" === 1 && $"name.first" === "a")
+      .analyze
+    val optimized1 = Optimize.execute(query1)
+    val aliases1 = collectGeneratedAliases(optimized1)
+    val expected1 = contact
+      .select($"name.first", $"address", $"id", $"name.first".as(aliases1(1)))
+      .window(Seq(winExpr.as("window")), Seq($"address"), Seq($"id".asc))
+      .select($"first", $"${aliases1(1)}".as(aliases1(0)), $"window")
+      .where($"window" === 1 && $"${aliases1(0)}" === "a")
+      .select($"first", $"window")
+      .analyze
+    comparePlans(optimized1, expected1)
+
+    val query2 = contact.sortBy($"name.first".asc)
+      .where($"name.first" === "a")
+      .select($"name.first")
+      .analyze
+    val optimized2 = Optimize.execute(query2)
+    val aliases2 = collectGeneratedAliases(optimized2)
+    val expected2 = contact
+      .select($"name.first".as(aliases2(1)))
+      .sortBy($"${aliases2(1)}".asc)
+      .select($"${aliases2(1)}".as(aliases2(0)))
+      .where($"${aliases2(0)}" === "a")
+      .select($"${aliases2(0)}".as("first"))
+      .analyze
+    comparePlans(optimized2, expected2)
+
+    val query3 = contact.distribute($"name.first")(100)
+      .where($"name.first" === "a")
+      .select($"name.first")
+      .analyze
+    val optimized3 = Optimize.execute(query3)
+    val aliases3 = collectGeneratedAliases(optimized3)
+    val expected3 = contact
+      .select($"name.first".as(aliases3(1)))
+      .distribute($"${aliases3(1)}")(100)
+      .select($"${aliases3(1)}".as(aliases3(0)))
+      .where($"${aliases3(0)}" === "a")
+      .select($"${aliases3(0)}".as("first"))
+      .analyze
+    comparePlans(optimized3, expected3)
+
+    val department = LocalRelation(
+      'depID.int,
+      'personID.string)
+    val query4 = contact.join(department, condition = Some($"id" === $"depID"))
+      .where($"name.first" === "a")
+      .select($"name.first")
+      .analyze
+    val optimized4 = Optimize.execute(query4)
+    val aliases4 = collectGeneratedAliases(optimized4)
+    val expected4 = contact
+      .select($"id", $"name.first".as(aliases4(1)))
+      .join(department.select('depID), condition = Some($"id" === $"depID"))
+      .select($"${aliases4(1)}".as(aliases4(0)))
+      .where($"${aliases4(0)}" === "a")
+      .select($"${aliases4(0)}".as("first"))
+      .analyze
+    comparePlans(optimized4, expected4)
+
+    def runTest(basePlan: LogicalPlan => LogicalPlan): Unit = {
+      val query = basePlan(contact)
+        .where($"name.first" === "a")
+        .select($"name.first")
+        .analyze
+      val optimized = Optimize.execute(query)
+      val aliases = collectGeneratedAliases(optimized)
+      val expected = basePlan(contact
+        .select($"name.first".as(aliases(0))))
+        .where($"${aliases(0)}" === "a")
+        .select($"${aliases(0)}".as("first"))
+        .analyze
+      comparePlans(optimized, expected)
+    }
+    Seq(
+      (plan: LogicalPlan) => plan.limit(100),
+      (plan: LogicalPlan) => plan.repartition(100),
+      (plan: LogicalPlan) => Sample(0.0, 0.6, false, 11L, plan)).foreach {  base =>
+        runTest(base)
+      }
+  }
+
+  test("Nested field pruning for Sort") {
+    val query1 = contact.select($"name.first", $"name.last")
+      .sortBy($"name.first".asc, $"name.last".asc)
+      .analyze
+    val optimized1 = Optimize.execute(query1)
+    val aliases1 = collectGeneratedAliases(optimized1)
+    val expected1 = contact
+      .select($"name.first",
+        $"name.last",
+        $"name.first".as(aliases1(0)),
+        $"name.last".as(aliases1(1)))
+      .sortBy($"${aliases1(0)}".asc, $"${aliases1(1)}".asc)
+      .select($"first", $"last")
+      .analyze
+    comparePlans(optimized1, expected1)
+
+    val query2 = contact.select($"name.first", $"name.last")
+      .orderBy($"name.first".asc, $"name.last".asc)
+      .analyze
+    val optimized2 = Optimize.execute(query2)
+    val aliases2 = collectGeneratedAliases(optimized2)
+    val expected2 = contact
+      .select($"name.first",
+        $"name.last",
+        $"name.first".as(aliases2(0)),
+        $"name.last".as(aliases2(1)))
+      .orderBy($"${aliases2(0)}".asc, $"${aliases2(1)}".asc)
+      .select($"first", $"last")
+      .analyze
+    comparePlans(optimized2, expected2)
+  }
+
+  test("Nested field pruning for Expand") {
+    def runTest(basePlan: LogicalPlan => LogicalPlan): Unit = {
+      val query1 = Expand(
+        Seq(
+          Seq($"name.first", $"name.middle"),
+          Seq(ConcatWs(Seq($"name.first", $"name.middle")),
+            ConcatWs(Seq($"name.middle", $"name.first")))
+        ),
+        Seq('a.string, 'b.string),
+        basePlan(contact)
+      ).analyze
+      val optimized1 = Optimize.execute(query1)
+      val aliases1 = collectGeneratedAliases(optimized1)
+
+      val expected1 = Expand(
+        Seq(
+          Seq($"${aliases1(0)}", $"${aliases1(1)}"),
+          Seq(ConcatWs(Seq($"${aliases1(0)}", $"${aliases1(1)}")),
+            ConcatWs(Seq($"${aliases1(1)}", $"${aliases1(0)}")))
+        ),
+        Seq('a.string, 'b.string),
+        basePlan(contact.select(
+          'name.getField("first").as(aliases1(0)),
+          'name.getField("middle").as(aliases1(1))))
+      ).analyze
+      comparePlans(optimized1, expected1)
+    }
+
+    Seq(
+      (plan: LogicalPlan) => plan,
+      (plan: LogicalPlan) => plan.limit(100),
+      (plan: LogicalPlan) => plan.repartition(100),
+      (plan: LogicalPlan) => Sample(0.0, 0.6, false, 11L, plan)).foreach {  base =>
+      runTest(base)
+    }
+
+    val query2 = Expand(
+      Seq(
+        Seq($"name", $"name.middle"),
+        Seq($"name", ConcatWs(Seq($"name.middle", $"name.first")))
+      ),
+      Seq('a.string, 'b.string),
+      contact
+    ).analyze
+    val optimized2 = Optimize.execute(query2)
+    val expected2 = Expand(
+      Seq(
+        Seq($"name", $"name.middle"),
+        Seq($"name", ConcatWs(Seq($"name.middle", $"name.first")))
+      ),
+      Seq('a.string, 'b.string),
+      contact.select($"name")
+    ).analyze
+    comparePlans(optimized2, expected2)
+  }
+
+  test("SPARK-34638: nested column prune on generator output for one field") {
+    val companies = LocalRelation(
+      'id.int,
+      'employers.array(employer))
+
+    val query = companies
+      .generate(Explode('employers.getField("company")), outputNames = Seq("company"))
+      .select('company.getField("name"))
+      .analyze
+    val optimized = Optimize.execute(query)
+
+    val aliases = collectGeneratedAliases(optimized)
+
+    val expected = companies
+      .select('employers.getField("company").getField("name").as(aliases(0)))
+      .generate(Explode($"${aliases(0)}"),
+        unrequiredChildIndex = Seq(0),
+        outputNames = Seq("company"))
+      .select('company.as("company.name"))
+      .analyze
+    comparePlans(optimized, expected)
+  }
 }
 
 object NestedColumnAliasingSuite {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingPointNumbersSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingPointNumbersSuite.scala
index f5af416602c9d..bb9919f94eef2 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingPointNumbersSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingPointNumbersSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.optimizer
 
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
-import org.apache.spark.sql.catalyst.expressions.{And, IsNull, KnownFloatingPointNormalized}
+import org.apache.spark.sql.catalyst.expressions.{CaseWhen, If, IsNull, KnownFloatingPointNormalized}
 import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules.RuleExecutor
@@ -91,5 +91,38 @@ class NormalizeFloatingPointNumbersSuite extends PlanTest {
 
     comparePlans(doubleOptimized, correctAnswer)
   }
+
+  test("SPARK-32258: normalize the children of If") {
+    val cond = If(a > 0.1D, namedStruct("a", a), namedStruct("a", a + 0.2D)) === namedStruct("a", b)
+    val query = testRelation1.join(testRelation2, condition = Some(cond))
+    val optimized = Optimize.execute(query)
+    val doubleOptimized = Optimize.execute(optimized)
+
+    val joinCond = If(a > 0.1D,
+      namedStruct("a", KnownFloatingPointNormalized(NormalizeNaNAndZero(a))),
+        namedStruct("a", KnownFloatingPointNormalized(NormalizeNaNAndZero(a + 0.2D)))) ===
+          namedStruct("a", KnownFloatingPointNormalized(NormalizeNaNAndZero(b)))
+    val correctAnswer = testRelation1.join(testRelation2, condition = Some(joinCond))
+
+    comparePlans(doubleOptimized, correctAnswer)
+  }
+
+  test("SPARK-32258: normalize the children of CaseWhen") {
+    val cond = CaseWhen(
+      Seq((a > 0.1D, namedStruct("a", a)), (a > 0.2D, namedStruct("a", a + 0.2D))),
+      Some(namedStruct("a", a + 0.3D))) === namedStruct("a", b)
+    val query = testRelation1.join(testRelation2, condition = Some(cond))
+    val optimized = Optimize.execute(query)
+    val doubleOptimized = Optimize.execute(optimized)
+
+    val joinCond = CaseWhen(
+      Seq((a > 0.1D, namedStruct("a", KnownFloatingPointNormalized(NormalizeNaNAndZero(a)))),
+        (a > 0.2D, namedStruct("a", KnownFloatingPointNormalized(NormalizeNaNAndZero(a + 0.2D))))),
+      Some(namedStruct("a", KnownFloatingPointNormalized(NormalizeNaNAndZero(a + 0.3D))))) ===
+      namedStruct("a", KnownFloatingPointNormalized(NormalizeNaNAndZero(b)))
+    val correctAnswer = testRelation1.join(testRelation2, condition = Some(joinCond))
+
+    comparePlans(doubleOptimized, correctAnswer)
+  }
 }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ObjectSerializerPruningSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ObjectSerializerPruningSuite.scala
index 0dd4d6a245f5c..6d7c4c3c7e9d2 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ObjectSerializerPruningSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ObjectSerializerPruningSuite.scala
@@ -22,8 +22,9 @@ import scala.reflect.runtime.universe.TypeTag
 
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
-import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
+import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder}
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.expressions.objects.Invoke
 import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules.RuleExecutor
@@ -107,4 +108,34 @@ class ObjectSerializerPruningSuite extends PlanTest {
       comparePlans(optimized, expected)
     }
   }
+
+  test("SPARK-32652: Prune nested serializers: RowEncoder") {
+    withSQLConf(SQLConf.SERIALIZER_NESTED_SCHEMA_PRUNING_ENABLED.key -> "true") {
+      val testRelation = LocalRelation('i.struct(StructType.fromDDL("a int, b string")), 'j.int)
+      val rowEncoder = RowEncoder(new StructType()
+        .add("i", new StructType().add("a", "int").add("b", "string"))
+        .add("j", "int"))
+      val serializerObject = CatalystSerde.serialize(
+        CatalystSerde.deserialize(testRelation)(rowEncoder))(rowEncoder)
+      val query = serializerObject.select($"i.a")
+      val optimized = Optimize.execute(query.analyze)
+
+      val prunedSerializer = serializerObject.serializer.head.transformDown {
+        case CreateNamedStruct(children) => CreateNamedStruct(children.take(2))
+      }.transformUp {
+        // Aligns null literal in `If` expression to make it resolvable.
+        case i @ If(invoke: Invoke, Literal(null, dt), ser) if invoke.functionName == "isNullAt" &&
+            !dt.sameType(ser.dataType) =>
+          i.copy(trueValue = Literal(null, ser.dataType))
+      }.asInstanceOf[NamedExpression]
+
+      // `name` in `GetStructField` affects `comparePlans`. Maybe we can ignore
+      // `name` in `GetStructField.equals`?
+      val expected = serializerObject.copy(serializer = Seq(prunedSerializer))
+        .select($"i.a").analyze.transformAllExpressions {
+        case g: GetStructField => g.copy(name = None)
+      }
+      comparePlans(optimized, expected)
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeCsvExprsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeCsvExprsSuite.scala
new file mode 100644
index 0000000000000..9b208cf2b57c4
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeCsvExprsSuite.scala
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan}
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types._
+
+class OptimizeCsvExprsSuite extends PlanTest with ExpressionEvalHelper {
+
+  private var csvExpressionOptimizeEnabled: Boolean = _
+  protected override def beforeAll(): Unit = {
+    csvExpressionOptimizeEnabled = SQLConf.get.csvExpressionOptimization
+  }
+
+  protected override def afterAll(): Unit = {
+    SQLConf.get.setConf(SQLConf.CSV_EXPRESSION_OPTIMIZATION, csvExpressionOptimizeEnabled)
+  }
+
+  object Optimizer extends RuleExecutor[LogicalPlan] {
+    val batches = Batch("Csv optimization", FixedPoint(10), OptimizeCsvJsonExprs) :: Nil
+  }
+
+  val schema = StructType.fromDDL("a int, b int")
+
+  private val csvAttr = 'csv.string
+  private val testRelation = LocalRelation(csvAttr)
+
+  test("SPARK-32968: prune unnecessary columns from GetStructField + from_csv") {
+    val options = Map.empty[String, String]
+
+    val query1 = testRelation
+      .select(GetStructField(CsvToStructs(schema, options, 'csv), 0))
+    val optimized1 = Optimizer.execute(query1.analyze)
+
+    val prunedSchema1 = StructType.fromDDL("a int")
+    val expected1 = testRelation
+      .select(GetStructField(CsvToStructs(schema, options, 'csv, None, Some(prunedSchema1)), 0))
+      .analyze
+    comparePlans(optimized1, expected1)
+
+    val query2 = testRelation
+      .select(GetStructField(CsvToStructs(schema, options, 'csv), 1))
+    val optimized2 = Optimizer.execute(query2.analyze)
+
+    val prunedSchema2 = StructType.fromDDL("b int")
+    val expected2 = testRelation
+      .select(GetStructField(CsvToStructs(schema, options, 'csv, None, Some(prunedSchema2)), 0))
+      .analyze
+    comparePlans(optimized2, expected2)
+  }
+
+  test("SPARK-32968: don't prune columns if options is not empty") {
+    val options = Map("mode" -> "failfast")
+
+    val query = testRelation
+      .select(GetStructField(CsvToStructs(schema, options, 'csv), 0))
+    val optimized = Optimizer.execute(query.analyze)
+
+    val expected = query.analyze
+    comparePlans(optimized, expected)
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeInSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeInSuite.scala
index a36083b847043..3d48eb2d6ee3f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeInSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeInSuite.scala
@@ -169,24 +169,37 @@ class OptimizeInSuite extends PlanTest {
   }
 
   test("OptimizedIn test: Setting the threshold for turning Set into InSet.") {
-    val plan =
-      testRelation
-        .where(In(UnresolvedAttribute("a"), Seq(Literal(1), Literal(2), Literal(3))))
-        .analyze
-
-    withSQLConf(OPTIMIZER_INSET_CONVERSION_THRESHOLD.key -> "10") {
-      val notOptimizedPlan = OptimizeIn(plan)
-      comparePlans(notOptimizedPlan, plan)
+    {
+      val plan =
+        testRelation
+          .where(In(UnresolvedAttribute("a"), Seq(Literal(1), Literal(2), Literal(3))))
+          .analyze
+
+      withSQLConf(OPTIMIZER_INSET_CONVERSION_THRESHOLD.key -> "10") {
+        val notOptimizedPlan = OptimizeIn(plan)
+        comparePlans(notOptimizedPlan, plan)
+      }
     }
 
-    // Reduce the threshold to turning into InSet.
-    withSQLConf(OPTIMIZER_INSET_CONVERSION_THRESHOLD.key -> "2") {
-      val optimizedPlan = OptimizeIn(plan)
-      optimizedPlan match {
-        case Filter(cond, _)
-          if cond.isInstanceOf[InSet] && cond.asInstanceOf[InSet].set.size == 3 =>
-        // pass
-        case _ => fail("Unexpected result for OptimizedIn")
+    // Since OptimizeIn has been marked as ineffective for `plan` in the preceding test, we need
+    // a new `plan` to run the next test. Here, OptimizeIn depends on a changing, external state,
+    // i.e., conf OPTIMIZER_INSET_CONVERSION_THRESHOLD, which however cannot happen in the
+    // production code path.
+    {
+      val plan =
+        testRelation
+          .where(In(UnresolvedAttribute("a"), Seq(Literal(1), Literal(2), Literal(3))))
+          .analyze
+
+      // Reduce the threshold to turning into InSet.
+      withSQLConf(OPTIMIZER_INSET_CONVERSION_THRESHOLD.key -> "2") {
+        val optimizedPlan = OptimizeIn(plan)
+        optimizedPlan match {
+          case Filter(cond, _)
+            if cond.isInstanceOf[InSet] && cond.asInstanceOf[InSet].set.size == 3 =>
+          // pass
+          case _ => fail("Unexpected result for OptimizedIn")
+        }
       }
     }
   }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeJsonExprsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeJsonExprsSuite.scala
new file mode 100644
index 0000000000000..ccbc61e8a4987
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeJsonExprsSuite.scala
@@ -0,0 +1,310 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan}
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.getZoneId
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types._
+
+class OptimizeJsonExprsSuite extends PlanTest with ExpressionEvalHelper {
+
+  private var jsonExpressionOptimizeEnabled: Boolean = _
+  protected override def beforeAll(): Unit = {
+    jsonExpressionOptimizeEnabled = SQLConf.get.jsonExpressionOptimization
+  }
+
+  protected override def afterAll(): Unit = {
+    SQLConf.get.setConf(SQLConf.JSON_EXPRESSION_OPTIMIZATION, jsonExpressionOptimizeEnabled)
+  }
+
+  object Optimizer extends RuleExecutor[LogicalPlan] {
+    val batches = Batch("Json optimization", FixedPoint(10), OptimizeCsvJsonExprs) :: Nil
+  }
+
+  val schema = StructType.fromDDL("a int, b int")
+
+  private val structAtt = 'struct.struct(schema).notNull
+  private val jsonAttr = 'json.string
+
+  private val testRelation = LocalRelation(structAtt)
+  private val testRelation2 = LocalRelation(jsonAttr)
+
+  test("SPARK-32948: optimize from_json + to_json") {
+    val options = Map.empty[String, String]
+
+    val query1 = testRelation
+      .select(JsonToStructs(schema, options, StructsToJson(options, 'struct)).as("struct"))
+    val optimized1 = Optimizer.execute(query1.analyze)
+
+    val expected = testRelation.select('struct.as("struct")).analyze
+    comparePlans(optimized1, expected)
+
+    val query2 = testRelation
+      .select(
+        JsonToStructs(schema, options,
+          StructsToJson(options,
+            JsonToStructs(schema, options,
+              StructsToJson(options, 'struct)))).as("struct"))
+    val optimized2 = Optimizer.execute(query2.analyze)
+
+    comparePlans(optimized2, expected)
+  }
+
+  test("SPARK-32948: not optimize from_json + to_json if schema is different") {
+    val options = Map.empty[String, String]
+    val schema = StructType.fromDDL("a int")
+
+    val query = testRelation
+      .select(JsonToStructs(schema, options, StructsToJson(options, 'struct)).as("struct"))
+    val optimized = Optimizer.execute(query.analyze)
+
+    val expected = testRelation.select(
+      JsonToStructs(schema, options, StructsToJson(options, 'struct)).as("struct")).analyze
+    comparePlans(optimized, expected)
+  }
+
+  test("SPARK-32948: if user gives schema with different letter case under case-insensitive") {
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+      val options = Map.empty[String, String]
+      val schema = StructType.fromDDL("a int, B int")
+
+      val query = testRelation
+        .select(JsonToStructs(schema, options, StructsToJson(options, 'struct)).as("struct"))
+      val optimized = Optimizer.execute(query.analyze)
+
+      val expected = testRelation.select(
+        JsonToStructs(schema, options, StructsToJson(options, 'struct)).as("struct")).analyze
+      comparePlans(optimized, expected)
+    }
+  }
+
+  test("SPARK-32948: not optimize from_json + to_json if nullability is different") {
+    val options = Map.empty[String, String]
+    val nonNullSchema = StructType(
+      StructField("a", IntegerType, false) :: StructField("b", IntegerType, false) :: Nil)
+
+    val structAtt = 'struct.struct(nonNullSchema).notNull
+    val testRelationWithNonNullAttr = LocalRelation(structAtt)
+
+    val schema = StructType.fromDDL("a int, b int")
+
+    val query = testRelationWithNonNullAttr
+      .select(JsonToStructs(schema, options, StructsToJson(options, 'struct)).as("struct"))
+    val optimized = Optimizer.execute(query.analyze)
+
+    val expected = testRelationWithNonNullAttr.select(
+      JsonToStructs(schema, options, StructsToJson(options, 'struct)).as("struct")).analyze
+    comparePlans(optimized, expected)
+  }
+
+  test("SPARK-32948: not optimize from_json + to_json if option is not empty") {
+    val options = Map("testOption" -> "test")
+
+    val query = testRelation
+      .select(JsonToStructs(schema, options, StructsToJson(options, 'struct)).as("struct"))
+    val optimized = Optimizer.execute(query.analyze)
+
+    val expected = testRelation.select(
+      JsonToStructs(schema, options, StructsToJson(options, 'struct)).as("struct")).analyze
+    comparePlans(optimized, expected)
+  }
+
+  test("SPARK-32948: not optimize from_json + to_json if timezone is different") {
+    val options = Map.empty[String, String]
+    val UTC_OPT = Option("UTC")
+    val PST = getZoneId("-08:00")
+
+    val query1 = testRelation
+      .select(JsonToStructs(schema, options,
+        StructsToJson(options, 'struct, Option(PST.getId)), UTC_OPT).as("struct"))
+    val optimized1 = Optimizer.execute(query1.analyze)
+
+    val expected1 = testRelation.select(
+      JsonToStructs(schema, options,
+        StructsToJson(options, 'struct, Option(PST.getId)), UTC_OPT).as("struct")).analyze
+    comparePlans(optimized1, expected1)
+
+    val query2 = testRelation
+      .select(JsonToStructs(schema, options,
+        StructsToJson(options, 'struct, UTC_OPT), UTC_OPT).as("struct"))
+    val optimized2 = Optimizer.execute(query2.analyze)
+    val expected2 = testRelation.select('struct.as("struct")).analyze
+    comparePlans(optimized2, expected2)
+  }
+
+  test("SPARK-32958: prune unnecessary columns from GetStructField + from_json") {
+    val options = Map.empty[String, String]
+
+    val query1 = testRelation2
+      .select(GetStructField(JsonToStructs(schema, options, 'json), 0))
+    val optimized1 = Optimizer.execute(query1.analyze)
+
+    val prunedSchema1 = StructType.fromDDL("a int")
+    val expected1 = testRelation2
+      .select(GetStructField(JsonToStructs(prunedSchema1, options, 'json), 0)).analyze
+    comparePlans(optimized1, expected1)
+
+    val query2 = testRelation2
+      .select(GetStructField(JsonToStructs(schema, options, 'json), 1))
+    val optimized2 = Optimizer.execute(query2.analyze)
+
+    val prunedSchema2 = StructType.fromDDL("b int")
+    val expected2 = testRelation2
+      .select(GetStructField(JsonToStructs(prunedSchema2, options, 'json), 0)).analyze
+    comparePlans(optimized2, expected2)
+  }
+
+  test("SPARK-32958: prune unnecessary columns from GetArrayStructFields + from_json") {
+    val options = Map.empty[String, String]
+    val schema1 = ArrayType(StructType.fromDDL("a int, b int"), containsNull = true)
+    val field1 = schema1.elementType.asInstanceOf[StructType](0)
+
+    val query1 = testRelation2
+      .select(GetArrayStructFields(
+        JsonToStructs(schema1, options, 'json), field1, 0, 2, true).as("a"))
+    val optimized1 = Optimizer.execute(query1.analyze)
+
+    val prunedSchema1 = ArrayType(StructType.fromDDL("a int"), containsNull = true)
+    val expected1 = testRelation2
+      .select(GetArrayStructFields(
+        JsonToStructs(prunedSchema1, options, 'json), field1, 0, 1, true).as("a")).analyze
+    comparePlans(optimized1, expected1)
+
+    val schema2 = ArrayType(
+      StructType(
+        StructField("a", IntegerType, false) ::
+          StructField("b", IntegerType, false) :: Nil), containsNull = false)
+    val field2 = schema2.elementType.asInstanceOf[StructType](1)
+    val query2 = testRelation2
+      .select(GetArrayStructFields(
+        JsonToStructs(schema2, options, 'json), field2, 1, 2, false).as("b"))
+    val optimized2 = Optimizer.execute(query2.analyze)
+
+    val prunedSchema2 = ArrayType(
+      StructType(StructField("b", IntegerType, false) :: Nil), containsNull = false)
+    val expected2 = testRelation2
+      .select(GetArrayStructFields(
+        JsonToStructs(prunedSchema2, options, 'json), field2, 0, 1, false).as("b")).analyze
+    comparePlans(optimized2, expected2)
+  }
+
+  test("SPARK-33907: do not prune unnecessary columns if options is not empty") {
+    val options = Map("mode" -> "failfast")
+
+    val query1 = testRelation2
+      .select(GetStructField(JsonToStructs(schema, options, 'json), 0))
+    val optimized1 = Optimizer.execute(query1.analyze)
+
+    comparePlans(optimized1, query1.analyze)
+
+    val schema1 = ArrayType(StructType.fromDDL("a int, b int"), containsNull = true)
+    val field1 = schema1.elementType.asInstanceOf[StructType](0)
+
+    val query2 = testRelation2
+      .select(GetArrayStructFields(
+        JsonToStructs(schema1, options, 'json), field1, 0, 2, true).as("a"))
+    val optimized2 = Optimizer.execute(query2.analyze)
+
+    comparePlans(optimized2, query2.analyze)
+  }
+
+  test("SPARK-33007: simplify named_struct + from_json") {
+    val options = Map.empty[String, String]
+    val schema = StructType.fromDDL("a int, b int, c long, d string")
+
+    val prunedSchema1 = StructType.fromDDL("a int, b int")
+    val nullStruct = namedStruct("a", Literal(null, IntegerType), "b", Literal(null, IntegerType))
+
+    val UTC_OPT = Option("UTC")
+    val json: BoundReference = 'json.string.canBeNull.at(0)
+
+    assertEquivalent(
+      testRelation2,
+      namedStruct(
+        "a", GetStructField(JsonToStructs(schema, options, json, UTC_OPT), 0),
+        "b", GetStructField(JsonToStructs(schema, options, json, UTC_OPT), 1)).as("struct"),
+      If(IsNull(json),
+        nullStruct,
+        KnownNotNull(JsonToStructs(prunedSchema1, options, json, UTC_OPT))).as("struct"))
+
+    val field1 = StructType.fromDDL("a int")
+    val field2 = StructType.fromDDL("b int")
+
+    // Skip optimization if `namedStruct` aliases field name.
+    assertEquivalent(
+      testRelation2,
+      namedStruct(
+        "a1", GetStructField(JsonToStructs(schema, options, json, UTC_OPT), 0),
+        "b", GetStructField(JsonToStructs(schema, options, json, UTC_OPT), 1)).as("struct"),
+      namedStruct(
+        "a1", GetStructField(JsonToStructs(field1, options, json, UTC_OPT), 0),
+        "b", GetStructField(JsonToStructs(field2, options, json, UTC_OPT), 0)).as("struct"))
+
+    assertEquivalent(
+      testRelation2,
+      namedStruct(
+        "a", GetStructField(JsonToStructs(schema, options, json, UTC_OPT), 0),
+        "a", GetStructField(JsonToStructs(schema, options, json, UTC_OPT), 0)).as("struct"),
+      namedStruct(
+        "a", GetStructField(JsonToStructs(field1, options, json, UTC_OPT), 0),
+        "a", GetStructField(JsonToStructs(field1, options, json, UTC_OPT), 0)).as("struct"))
+
+    val PST = getZoneId("-08:00")
+    // Skip optimization if `JsonToStructs`s are not the same.
+    assertEquivalent(
+      testRelation2,
+      namedStruct(
+        "a", GetStructField(JsonToStructs(schema, options, json, UTC_OPT), 0),
+        "b", GetStructField(JsonToStructs(schema, options, json, Option(PST.getId)), 1))
+        .as("struct"),
+      namedStruct(
+        "a", GetStructField(JsonToStructs(field1, options, json, UTC_OPT), 0),
+        "b", GetStructField(JsonToStructs(field2, options, json, Option(PST.getId)), 0))
+        .as("struct"))
+  }
+
+  private def assertEquivalent(relation: LocalRelation, e1: Expression, e2: Expression): Unit = {
+    val plan = relation.select(e1).analyze
+    val actual = Optimizer.execute(plan)
+    val expected = relation.select(e2).analyze
+    comparePlans(actual, expected)
+
+    Seq("""{"a":1, "b":2, "c": 123, "d": "test"}""", null).foreach(v => {
+      val row = create_row(v)
+      checkEvaluation(e1, e2.eval(row), row)
+    })
+  }
+
+  test("SPARK-33078: disable json optimization") {
+    withSQLConf(SQLConf.JSON_EXPRESSION_OPTIMIZATION.key -> "false") {
+      val options = Map.empty[String, String]
+
+      val query = testRelation
+        .select(JsonToStructs(schema, options, StructsToJson(options, 'struct)).as("struct"))
+      val optimized = Optimizer.execute(query.analyze)
+
+      comparePlans(optimized, query.analyze)
+    }
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeLimitZeroSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeLimitZeroSuite.scala
index cf875efc62c98..c8c1ecd7718b0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeLimitZeroSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeLimitZeroSuite.scala
@@ -74,7 +74,7 @@ class OptimizeLimitZeroSuite extends PlanTest {
   ).foreach { case (jt, correctAnswer) =>
       test(s"Limit 0: for join type $jt") {
         val query = testRelation1
-          .join(testRelation2.limit(0), joinType = jt, condition = Some('a.attr == 'b.attr))
+          .join(testRelation2.limit(0), joinType = jt, condition = Some('a.attr === 'b.attr))
 
         val optimized = Optimize.execute(query.analyze)
 
@@ -86,9 +86,9 @@ class OptimizeLimitZeroSuite extends PlanTest {
     val testRelation3 = LocalRelation.fromExternalRows(Seq('c.int), data = Seq(Row(1)))
 
     val subJoinQuery = testRelation1
-      .join(testRelation2, joinType = Inner, condition = Some('a.attr == 'b.attr))
+      .join(testRelation2, joinType = Inner, condition = Some('a.attr === 'b.attr))
     val query = subJoinQuery
-      .join(testRelation3.limit(0), joinType = Inner, condition = Some('a.attr == 'c.attr))
+      .join(testRelation3.limit(0), joinType = Inner, condition = Some('a.attr === 'c.attr))
 
     val optimized = Optimize.execute(query.analyze)
     val correctAnswer = LocalRelation('a.int, 'b.int, 'c.int)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeWindowFunctionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeWindowFunctionsSuite.scala
new file mode 100644
index 0000000000000..cf850bbe21ce6
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeWindowFunctionsSuite.scala
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.expressions.aggregate.First
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan}
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+
+class OptimizeWindowFunctionsSuite extends PlanTest {
+  object Optimize extends RuleExecutor[LogicalPlan] {
+    val batches = Batch("OptimizeWindowFunctions", FixedPoint(10),
+        OptimizeWindowFunctions) :: Nil
+  }
+
+  val testRelation = LocalRelation('a.double, 'b.double, 'c.string)
+  val a = testRelation.output(0)
+  val b = testRelation.output(1)
+  val c = testRelation.output(2)
+
+  test("replace first by nth_value if frame is UNBOUNDED PRECEDING AND CURRENT ROW") {
+    val inputPlan = testRelation.select(
+      WindowExpression(
+        First(a, false).toAggregateExpression(),
+        WindowSpecDefinition(b :: Nil, c.asc :: Nil,
+          SpecifiedWindowFrame(RowFrame, UnboundedPreceding, CurrentRow))))
+    val correctAnswer = testRelation.select(
+      WindowExpression(
+        NthValue(a, Literal(1), false),
+        WindowSpecDefinition(b :: Nil, c.asc :: Nil,
+          SpecifiedWindowFrame(RowFrame, UnboundedPreceding, CurrentRow))))
+
+    val optimized = Optimize.execute(inputPlan)
+    assert(optimized == correctAnswer)
+  }
+
+  test("replace first by nth_value if frame is UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING") {
+    val inputPlan = testRelation.select(
+      WindowExpression(
+        First(a, false).toAggregateExpression(),
+        WindowSpecDefinition(b :: Nil, c.asc :: Nil,
+          SpecifiedWindowFrame(RowFrame, UnboundedPreceding, UnboundedFollowing))))
+    val correctAnswer = testRelation.select(
+      WindowExpression(
+        NthValue(a, Literal(1), false),
+        WindowSpecDefinition(b :: Nil, c.asc :: Nil,
+          SpecifiedWindowFrame(RowFrame, UnboundedPreceding, UnboundedFollowing))))
+
+    val optimized = Optimize.execute(inputPlan)
+    assert(optimized == correctAnswer)
+  }
+
+  test("can't replace first by nth_value if frame is not suitable") {
+    val inputPlan = testRelation.select(
+      WindowExpression(
+        First(a, false).toAggregateExpression(),
+        WindowSpecDefinition(b :: Nil, c.asc :: Nil,
+          SpecifiedWindowFrame(RowFrame, Literal(1), CurrentRow))))
+
+    val optimized = Optimize.execute(inputPlan)
+    assert(optimized == inputPlan)
+  }
+
+  test("can't replace first by nth_value if the window frame type is range") {
+    val inputPlan = testRelation.select(
+      WindowExpression(
+        First(a, false).toAggregateExpression(),
+        WindowSpecDefinition(b :: Nil, c.asc :: Nil,
+          SpecifiedWindowFrame(RangeFrame, UnboundedPreceding, CurrentRow))))
+
+    val optimized = Optimize.execute(inputPlan)
+    assert(optimized == inputPlan)
+  }
+
+  test("can't replace first by nth_value if the window frame isn't ordered") {
+    val inputPlan = testRelation.select(
+      WindowExpression(
+        First(a, false).toAggregateExpression(),
+        WindowSpecDefinition(b :: Nil, Nil,
+          SpecifiedWindowFrame(RowFrame, UnboundedPreceding, CurrentRow))))
+
+    val optimized = Optimize.execute(inputPlan)
+    assert(optimized == inputPlan)
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeWithFieldsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeWithFieldsSuite.scala
new file mode 100644
index 0000000000000..e63742ac0de56
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeWithFieldsSuite.scala
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions.{Alias, GetStructField, Literal, UpdateFields, WithField}
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules._
+import org.apache.spark.sql.internal.SQLConf
+
+class OptimizeWithFieldsSuite extends PlanTest {
+
+  object Optimize extends RuleExecutor[LogicalPlan] {
+    val batches = Batch("OptimizeUpdateFields", FixedPoint(10),
+      OptimizeUpdateFields, SimplifyExtractValueOps) :: Nil
+  }
+
+  private val testRelation = LocalRelation('a.struct('a1.int))
+  private val testRelation2 = LocalRelation('a.struct('a1.int).notNull)
+
+  test("combines two adjacent UpdateFields Expressions") {
+    val originalQuery = testRelation
+      .select(Alias(
+        UpdateFields(
+          UpdateFields(
+            'a,
+            WithField("b1", Literal(4)) :: Nil),
+          WithField("c1", Literal(5)) :: Nil), "out")())
+
+    val optimized = Optimize.execute(originalQuery.analyze)
+    val correctAnswer = testRelation
+      .select(Alias(UpdateFields('a, WithField("b1", Literal(4)) :: WithField("c1", Literal(5)) ::
+        Nil), "out")())
+      .analyze
+
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("combines three adjacent UpdateFields Expressions") {
+    val originalQuery = testRelation
+      .select(Alias(
+        UpdateFields(
+          UpdateFields(
+            UpdateFields(
+              'a,
+              WithField("b1", Literal(4)) :: Nil),
+            WithField("c1", Literal(5)) :: Nil),
+          WithField("d1", Literal(6)) :: Nil), "out")())
+
+    val optimized = Optimize.execute(originalQuery.analyze)
+    val correctAnswer = testRelation
+      .select(Alias(UpdateFields('a, WithField("b1", Literal(4)) :: WithField("c1", Literal(5)) ::
+        WithField("d1", Literal(6)) :: Nil), "out")())
+      .analyze
+
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("SPARK-32941: optimize WithFields followed by GetStructField") {
+    val originalQuery = testRelation2
+      .select(Alias(
+        GetStructField(UpdateFields('a,
+          WithField("b1", Literal(4)) :: Nil), 1), "out")())
+
+    val optimized = Optimize.execute(originalQuery.analyze)
+    val correctAnswer = testRelation2
+      .select(Alias(Literal(4), "out")())
+      .analyze
+
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("SPARK-32941: optimize WithFields chain - case insensitive") {
+    val originalQuery = testRelation
+      .select(
+        Alias(UpdateFields('a,
+          WithField("b1", Literal(4)) :: WithField("b1", Literal(5)) :: Nil), "out1")(),
+        Alias(UpdateFields('a,
+          WithField("b1", Literal(4)) :: WithField("B1", Literal(5)) :: Nil), "out2")())
+
+    val optimized = Optimize.execute(originalQuery.analyze)
+    val correctAnswer = testRelation
+      .select(
+        Alias(UpdateFields('a, WithField("b1", Literal(5)) :: Nil), "out1")(),
+        Alias(UpdateFields('a, WithField("B1", Literal(5)) :: Nil), "out2")())
+      .analyze
+
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("SPARK-32941: optimize WithFields chain - case sensitive") {
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+      val originalQuery = testRelation
+        .select(
+          Alias(UpdateFields('a,
+            WithField("b1", Literal(4)) :: WithField("b1", Literal(5)) :: Nil), "out1")(),
+          Alias(UpdateFields('a,
+              WithField("b1", Literal(4)) :: WithField("B1", Literal(5)) :: Nil), "out2")())
+
+      val optimized = Optimize.execute(originalQuery.analyze)
+      val correctAnswer = testRelation
+        .select(
+          Alias(UpdateFields('a, WithField("b1", Literal(5)) :: Nil), "out1")(),
+          Alias(
+            UpdateFields('a,
+              WithField("b1", Literal(4)) :: WithField("B1", Literal(5)) :: Nil), "out2")())
+        .analyze
+
+      comparePlans(optimized, correctAnswer)
+    }
+  }
+
+  test("SPARK-35213: ensure optimize WithFields maintains correct WithField ordering") {
+    val originalQuery = testRelation
+      .select(
+        Alias(UpdateFields('a,
+          WithField("a1", Literal(3)) ::
+          WithField("b1", Literal(4)) ::
+          WithField("a1", Literal(5)) ::
+          Nil), "out")())
+
+    val optimized = Optimize.execute(originalQuery.analyze)
+    val correctAnswer = testRelation
+      .select(
+        Alias(UpdateFields('a,
+          WithField("a1", Literal(5)) ::
+          WithField("b1", Literal(4)) ::
+          Nil), "out")())
+      .analyze
+
+    comparePlans(optimized, correctAnswer)
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerLoggingSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerLoggingSuite.scala
index db22121e08601..1187950c04240 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerLoggingSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerLoggingSuite.scala
@@ -38,8 +38,7 @@ class OptimizerLoggingSuite extends PlanTest {
 
   private def verifyLog(expectedLevel: Level, expectedRulesOrBatches: Seq[String]): Unit = {
     val logAppender = new LogAppender("optimizer rules")
-    withLogAppender(logAppender,
-        loggerName = Some(Optimize.getClass.getName.dropRight(1)), level = Some(Level.TRACE)) {
+    withLogAppender(logAppender, level = Some(Level.TRACE)) {
       val input = LocalRelation('a.int, 'b.string, 'c.double)
       val query = input.select('a, 'b).select('a).where('a > 1).analyze
       val expected = input.where('a > 1).select('a).analyze
@@ -79,7 +78,7 @@ class OptimizerLoggingSuite extends PlanTest {
       "deBUG" -> Level.DEBUG)
 
     levels.foreach { level =>
-      withSQLConf(SQLConf.OPTIMIZER_PLAN_CHANGE_LOG_LEVEL.key -> level._1) {
+      withSQLConf(SQLConf.PLAN_CHANGE_LOG_LEVEL.key -> level._1) {
         verifyLog(
           level._2,
           Seq(
@@ -98,10 +97,10 @@ class OptimizerLoggingSuite extends PlanTest {
 
     levels.foreach { level =>
       val error = intercept[IllegalArgumentException] {
-        withSQLConf(SQLConf.OPTIMIZER_PLAN_CHANGE_LOG_LEVEL.key -> level) {}
+        withSQLConf(SQLConf.PLAN_CHANGE_LOG_LEVEL.key -> level) {}
       }
       assert(error.getMessage.contains(
-        "Invalid value for 'spark.sql.optimizer.planChangeLog.level'."))
+        "Invalid value for 'spark.sql.planChangeLog.level'."))
     }
   }
 
@@ -128,8 +127,8 @@ class OptimizerLoggingSuite extends PlanTest {
 
     rulesSeq.foreach { case (rulesConf, expectedRules) =>
       withSQLConf(
-        SQLConf.OPTIMIZER_PLAN_CHANGE_LOG_RULES.key -> rulesConf,
-        SQLConf.OPTIMIZER_PLAN_CHANGE_LOG_LEVEL.key -> "INFO") {
+        SQLConf.PLAN_CHANGE_LOG_RULES.key -> rulesConf,
+        SQLConf.PLAN_CHANGE_LOG_LEVEL.key -> "INFO") {
         verifyLog(Level.INFO, expectedRules)
       }
     }
@@ -137,16 +136,16 @@ class OptimizerLoggingSuite extends PlanTest {
 
   test("test log batches which change the plan") {
     withSQLConf(
-      SQLConf.OPTIMIZER_PLAN_CHANGE_LOG_BATCHES.key -> "Optimizer Batch",
-      SQLConf.OPTIMIZER_PLAN_CHANGE_LOG_LEVEL.key -> "INFO") {
+      SQLConf.PLAN_CHANGE_LOG_BATCHES.key -> "Optimizer Batch",
+      SQLConf.PLAN_CHANGE_LOG_LEVEL.key -> "INFO") {
       verifyLog(Level.INFO, Seq("Optimizer Batch"))
     }
   }
 
   test("test log batches which do not change the plan") {
     withSQLConf(
-      SQLConf.OPTIMIZER_PLAN_CHANGE_LOG_BATCHES.key -> "Batch Has No Effect",
-      SQLConf.OPTIMIZER_PLAN_CHANGE_LOG_LEVEL.key -> "INFO") {
+      SQLConf.PLAN_CHANGE_LOG_BATCHES.key -> "Batch Has No Effect",
+      SQLConf.PLAN_CHANGE_LOG_LEVEL.key -> "INFO") {
       verifyLog(Level.INFO, Seq("Batch Has No Effect"))
     }
   }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerStructuralIntegrityCheckerSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerStructuralIntegrityCheckerSuite.scala
index 5998437f11f4d..f4a52180373c0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerStructuralIntegrityCheckerSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerStructuralIntegrityCheckerSuite.scala
@@ -21,13 +21,11 @@ import org.apache.spark.sql.catalyst.analysis.{EmptyFunctionRegistry, FakeV2Sess
 import org.apache.spark.sql.catalyst.catalog.{InMemoryCatalog, SessionCatalog}
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
-import org.apache.spark.sql.catalyst.errors.TreeNodeException
 import org.apache.spark.sql.catalyst.expressions.{Alias, Literal, NamedExpression}
 import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LocalRelation, LogicalPlan, OneRowRelation, Project}
 import org.apache.spark.sql.catalyst.rules._
 import org.apache.spark.sql.connector.catalog.CatalogManager
-import org.apache.spark.sql.internal.SQLConf
 
 
 class OptimizerStructuralIntegrityCheckerSuite extends PlanTest {
@@ -45,9 +43,8 @@ class OptimizerStructuralIntegrityCheckerSuite extends PlanTest {
 
   object Optimize extends Optimizer(
     new CatalogManager(
-      new SQLConf(),
       FakeV2SessionCatalog,
-      new SessionCatalog(new InMemoryCatalog, EmptyFunctionRegistry, new SQLConf()))) {
+      new SessionCatalog(new InMemoryCatalog, EmptyFunctionRegistry))) {
     val newBatch = Batch("OptimizeRuleBreakSI", Once, OptimizeRuleBreakSI)
     override def defaultBatches: Seq[Batch] = Seq(newBatch) ++ super.defaultBatches
   }
@@ -55,7 +52,7 @@ class OptimizerStructuralIntegrityCheckerSuite extends PlanTest {
   test("check for invalid plan after execution of rule - unresolved attribute") {
     val analyzed = Project(Alias(Literal(10), "attr")() :: Nil, OneRowRelation()).analyze
     assert(analyzed.resolved)
-    val message = intercept[TreeNodeException[LogicalPlan]] {
+    val message = intercept[RuntimeException] {
       Optimize.execute(analyzed)
     }.getMessage
     val ruleName = OptimizeRuleBreakSI.ruleName
@@ -70,7 +67,7 @@ class OptimizerStructuralIntegrityCheckerSuite extends PlanTest {
     assert(analyzed.resolved)
 
     // Should fail verification with the OptimizeRuleBreakSI rule
-    val message = intercept[TreeNodeException[LogicalPlan]] {
+    val message = intercept[RuntimeException] {
       Optimize.execute(analyzed)
     }.getMessage
     val ruleName = OptimizeRuleBreakSI.ruleName
@@ -88,7 +85,7 @@ class OptimizerStructuralIntegrityCheckerSuite extends PlanTest {
     val invalidPlan = OptimizeRuleBreakSI.apply(analyzed)
 
     // Should fail verification right at the beginning
-    val message = intercept[TreeNodeException[LogicalPlan]] {
+    val message = intercept[RuntimeException] {
       Optimize.execute(invalidPlan)
     }.getMessage
     assert(message.contains("The structural integrity of the input plan is broken"))
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerSuite.scala
new file mode 100644
index 0000000000000..6b63f860b7da9
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerSuite.scala
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions.{Alias, IntegerLiteral, Literal}
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, OneRowRelation, Project}
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * A dummy optimizer rule for testing that decrements integer literals until 0.
+ */
+object DecrementLiterals extends Rule[LogicalPlan] {
+  def apply(plan: LogicalPlan): LogicalPlan = plan transformExpressions {
+    case IntegerLiteral(i) if i > 0 => Literal(i - 1)
+  }
+}
+
+class OptimizerSuite extends PlanTest {
+  test("Optimizer exceeds max iterations") {
+    val iterations = 5
+    val maxIterationsNotEnough = 3
+    val maxIterationsEnough = 10
+    val analyzed = Project(Alias(Literal(iterations), "attr")() :: Nil, OneRowRelation()).analyze
+
+    withSQLConf(SQLConf.OPTIMIZER_MAX_ITERATIONS.key -> maxIterationsNotEnough.toString) {
+      val optimizer = new SimpleTestOptimizer() {
+        override def defaultBatches: Seq[Batch] =
+          Batch("test", fixedPoint,
+            DecrementLiterals) :: Nil
+      }
+
+      val message1 = intercept[RuntimeException] {
+        optimizer.execute(analyzed)
+      }.getMessage
+      assert(message1.startsWith(s"Max iterations ($maxIterationsNotEnough) reached for batch " +
+        s"test, please set '${SQLConf.OPTIMIZER_MAX_ITERATIONS.key}' to a larger value."))
+
+      withSQLConf(SQLConf.OPTIMIZER_MAX_ITERATIONS.key -> maxIterationsEnough.toString) {
+        try {
+          optimizer.execute(analyzed)
+        } catch {
+          case ex: AnalysisException
+            if ex.getMessage.contains(SQLConf.OPTIMIZER_MAX_ITERATIONS.key) =>
+              fail("optimizer.execute should not reach max iterations.")
+        }
+      }
+
+      val message2 = intercept[RuntimeException] {
+        optimizer.execute(analyzed)
+      }.getMessage
+      assert(message2.startsWith(s"Max iterations ($maxIterationsNotEnough) reached for batch " +
+        s"test, please set '${SQLConf.OPTIMIZER_MAX_ITERATIONS.key}' to a larger value."))
+    }
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala
index 9c7d4c7d8d233..b5dcb8aa67646 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala
@@ -22,10 +22,11 @@ import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.expressions.Literal
+import org.apache.spark.sql.catalyst.expressions.Literal.FalseLiteral
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan, Project}
 import org.apache.spark.sql.catalyst.rules.RuleExecutor
-import org.apache.spark.sql.types.{IntegerType, StructType}
+import org.apache.spark.sql.types.{IntegerType, MetadataBuilder, StructType}
 
 class PropagateEmptyRelationSuite extends PlanTest {
   object Optimize extends RuleExecutor[LogicalPlan] {
@@ -55,6 +56,9 @@ class PropagateEmptyRelationSuite extends PlanTest {
 
   val testRelation1 = LocalRelation.fromExternalRows(Seq('a.int), data = Seq(Row(1)))
   val testRelation2 = LocalRelation.fromExternalRows(Seq('b.int), data = Seq(Row(1)))
+  val metadata = new MetadataBuilder().putLong("test", 1).build()
+  val testRelation3 =
+    LocalRelation.fromExternalRows(Seq('c.int.notNull.withMetadata(metadata)), data = Seq(Row(1)))
 
   test("propagate empty relation through Union") {
     val query = testRelation1
@@ -67,6 +71,39 @@ class PropagateEmptyRelationSuite extends PlanTest {
     comparePlans(optimized, correctAnswer)
   }
 
+  test("SPARK-32241: remove empty relation children from Union") {
+    val query = testRelation1.union(testRelation2.where(false))
+    val optimized = Optimize.execute(query.analyze)
+    val correctAnswer = testRelation1
+    comparePlans(optimized, correctAnswer)
+
+    val query2 = testRelation1.where(false).union(testRelation2)
+    val optimized2 = Optimize.execute(query2.analyze)
+    val correctAnswer2 = testRelation2.select('b.as('a)).analyze
+    comparePlans(optimized2, correctAnswer2)
+
+    val query3 = testRelation1.union(testRelation2.where(false)).union(testRelation3)
+    val optimized3 = Optimize.execute(query3.analyze)
+    val correctAnswer3 = testRelation1.union(testRelation3)
+    comparePlans(optimized3, correctAnswer3)
+
+    val query4 = testRelation1.where(false).union(testRelation2).union(testRelation3)
+    val optimized4 = Optimize.execute(query4.analyze)
+    val correctAnswer4 = testRelation2.union(testRelation3).select('b.as('a)).analyze
+    comparePlans(optimized4, correctAnswer4)
+
+    // Nullability can change from nullable to non-nullable
+    val query5 = testRelation1.where(false).union(testRelation3)
+    val optimized5 = Optimize.execute(query5.analyze)
+    assert(query5.output.head.nullable, "Original output should be nullable")
+    assert(!optimized5.output.head.nullable, "New output should be non-nullable")
+
+    // Keep metadata
+    val query6 = testRelation3.where(false).union(testRelation1)
+    val optimized6 = Optimize.execute(query6.analyze)
+    assert(optimized6.output.head.metadata == metadata, "New output should keep metadata")
+  }
+
   test("propagate empty relation through Join") {
     // Testcases are tuples of (left predicate, right predicate, joinType, correct answer)
     // Note that `None` is used to compare with OptimizeWithoutPropagateEmptyRelation.
@@ -111,7 +148,29 @@ class PropagateEmptyRelationSuite extends PlanTest {
     testcases.foreach { case (left, right, jt, answer) =>
       val query = testRelation1
         .where(left)
-        .join(testRelation2.where(right), joinType = jt, condition = Some('a.attr == 'b.attr))
+        .join(testRelation2.where(right), joinType = jt, condition = Some('a.attr === 'b.attr))
+      val optimized = Optimize.execute(query.analyze)
+      val correctAnswer =
+        answer.getOrElse(OptimizeWithoutPropagateEmptyRelation.execute(query.analyze))
+      comparePlans(optimized, correctAnswer)
+    }
+  }
+
+  test("SPARK-28220: Propagate empty relation through Join if condition is FalseLiteral") {
+    val testcases = Seq(
+      (Inner, Some(LocalRelation('a.int, 'b.int))),
+      (Cross, Some(LocalRelation('a.int, 'b.int))),
+      (LeftOuter,
+        Some(Project(Seq('a, Literal(null).cast(IntegerType).as('b)), testRelation1).analyze)),
+      (RightOuter,
+        Some(Project(Seq(Literal(null).cast(IntegerType).as('a), 'b), testRelation2).analyze)),
+      (FullOuter, None),
+      (LeftAnti, Some(testRelation1)),
+      (LeftSemi, Some(LocalRelation('a.int)))
+    )
+
+    testcases.foreach { case (jt, answer) =>
+      val query = testRelation1.join(testRelation2, joinType = jt, condition = Some(FalseLiteral))
       val optimized = Optimize.execute(query.analyze)
       val correctAnswer =
         answer.getOrElse(OptimizeWithoutPropagateEmptyRelation.execute(query.analyze))
@@ -221,4 +280,10 @@ class PropagateEmptyRelationSuite extends PlanTest {
     val optimized = Optimize.execute(query.analyze)
     assert(optimized.resolved)
   }
+
+  test("should not optimize away limit if streaming") {
+    val query = LocalRelation(Nil, Nil, isStreaming = true).limit(1).analyze
+    val optimized = Optimize.execute(query)
+    comparePlans(optimized, query)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PruneFiltersSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PruneFiltersSuite.scala
index 15fbe3c5b0a17..f2c6b779577ff 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PruneFiltersSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PruneFiltersSuite.scala
@@ -160,4 +160,18 @@ class PruneFiltersSuite extends PlanTest {
       comparePlans(optimized, correctAnswer)
     }
   }
+
+  test("SPARK-35273: CombineFilters support non-deterministic expressions") {
+    val x = testRelation.where(!'a.attr.in(1, 3, 5)).subquery('x)
+
+    comparePlans(
+      Optimize.execute(x.where('a.attr === 7 && Rand(10) > 0.1).analyze),
+      testRelation.where(!'a.attr.in(1, 3, 5) && 'a.attr === 7).where(Rand(10) > 0.1).analyze)
+
+    comparePlans(
+      Optimize.execute(
+        x.where('a.attr === 7 && Rand(10) > 0.1 && 'b.attr === 1 && Rand(10) < 1.1).analyze),
+      testRelation.where(!'a.attr.in(1, 3, 5) && 'a.attr === 7 && 'b.attr === 1)
+        .where(Rand(10) > 0.1 && Rand(10) < 1.1).analyze)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PullupCorrelatedPredicatesSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PullupCorrelatedPredicatesSuite.scala
index 2d86d5a97e769..ae9a694b50444 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PullupCorrelatedPredicatesSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PullupCorrelatedPredicatesSuite.scala
@@ -21,13 +21,13 @@ import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.PlanTest
-import org.apache.spark.sql.catalyst.plans.logical.{Filter, LocalRelation, LogicalPlan}
+import org.apache.spark.sql.catalyst.plans.logical.{Assignment, DeleteAction, DeleteFromTable, InsertAction, LocalRelation, LogicalPlan, MergeIntoTable, UpdateTable}
 import org.apache.spark.sql.catalyst.rules.RuleExecutor
 
 class PullupCorrelatedPredicatesSuite extends PlanTest {
 
   object Optimize extends RuleExecutor[LogicalPlan] {
-    override protected val blacklistedOnceBatches = Set("PullupCorrelatedPredicates")
+    override protected val excludedOnceBatches = Set("PullupCorrelatedPredicates")
 
     val batches =
       Batch("PullupCorrelatedPredicates", Once,
@@ -91,12 +91,73 @@ class PullupCorrelatedPredicatesSuite extends PlanTest {
         .select(max('d))
     val scalarSubquery =
       testRelation
-        .where(ScalarSubquery(subPlan))
+        .where(ScalarSubquery(subPlan) === 1)
         .select('a).analyze
-    assert(scalarSubquery.resolved)
 
     val optimized = Optimize.execute(scalarSubquery)
     val doubleOptimized = Optimize.execute(optimized)
     comparePlans(optimized, doubleOptimized, false)
   }
+
+  test("PullupCorrelatedPredicates should handle deletes") {
+    val subPlan = testRelation2.where('a === 'c).select('c)
+    val cond = InSubquery(Seq('a), ListQuery(subPlan))
+    val deletePlan = DeleteFromTable(testRelation, Some(cond)).analyze
+    assert(deletePlan.resolved)
+
+    val optimized = Optimize.execute(deletePlan)
+    assert(optimized.resolved)
+
+    optimized match {
+      case DeleteFromTable(_, Some(s: InSubquery)) =>
+        val outerRefs = SubExprUtils.getOuterReferences(s.query.plan)
+        assert(outerRefs.isEmpty, "should be no outer refs")
+      case other =>
+        fail(s"unexpected logical plan: $other")
+    }
+  }
+
+  test("PullupCorrelatedPredicates should handle updates") {
+    val subPlan = testRelation2.where('a === 'c).select('c)
+    val cond = InSubquery(Seq('a), ListQuery(subPlan))
+    val updatePlan = UpdateTable(testRelation, Seq.empty, Some(cond)).analyze
+    assert(updatePlan.resolved)
+
+    val optimized = Optimize.execute(updatePlan)
+    assert(optimized.resolved)
+
+    optimized match {
+      case UpdateTable(_, _, Some(s: InSubquery)) =>
+        val outerRefs = SubExprUtils.getOuterReferences(s.query.plan)
+        assert(outerRefs.isEmpty, "should be no outer refs")
+      case other =>
+        fail(s"unexpected logical plan: $other")
+    }
+  }
+
+  test("PullupCorrelatedPredicates should handle merge") {
+    val testRelation3 = LocalRelation('e.int, 'f.double)
+    val subPlan = testRelation3.where('a === 'e).select('e)
+    val cond = InSubquery(Seq('a), ListQuery(subPlan))
+
+    val mergePlan = MergeIntoTable(
+      testRelation,
+      testRelation2,
+      cond,
+      Seq(DeleteAction(None)),
+      Seq(InsertAction(None, Seq(Assignment('a, 'c), Assignment('b, 'd)))))
+    val analyzedMergePlan = mergePlan.analyze
+    assert(analyzedMergePlan.resolved)
+
+    val optimized = Optimize.execute(analyzedMergePlan)
+    assert(optimized.resolved)
+
+    optimized match {
+      case MergeIntoTable(_, _, s: InSubquery, _, _) =>
+        val outerRefs = SubExprUtils.getOuterReferences(s.query.plan)
+        assert(outerRefs.isEmpty, "should be no outer refs")
+      case other =>
+        fail(s"unexpected logical plan: $other")
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PushFoldableIntoBranchesSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PushFoldableIntoBranchesSuite.scala
new file mode 100644
index 0000000000000..f669b91804cb9
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PushFoldableIntoBranchesSuite.scala
@@ -0,0 +1,359 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import java.sql.{Date, Timestamp}
+import java.time.{Duration, Period}
+
+import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.expressions.Literal.{FalseLiteral, TrueLiteral}
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules._
+import org.apache.spark.sql.types.{BooleanType, IntegerType, StringType, TimestampType, YearMonthIntervalType}
+import org.apache.spark.unsafe.types.CalendarInterval
+
+
+class PushFoldableIntoBranchesSuite
+  extends PlanTest with ExpressionEvalHelper with PredicateHelper {
+
+  object Optimize extends RuleExecutor[LogicalPlan] {
+    val batches = Batch("PushFoldableIntoBranches", FixedPoint(50),
+      BooleanSimplification, ConstantFolding, SimplifyConditionals, PushFoldableIntoBranches) :: Nil
+  }
+
+  private val relation = LocalRelation('a.int, 'b.int, 'c.boolean)
+  private val a = EqualTo(UnresolvedAttribute("a"), Literal(100))
+  private val b = UnresolvedAttribute("b")
+  private val c = EqualTo(UnresolvedAttribute("c"), Literal(true))
+  private val ifExp = If(a, Literal(2), Literal(3))
+  private val caseWhen = CaseWhen(Seq((a, Literal(1)), (c, Literal(2))), Some(Literal(3)))
+
+  protected def assertEquivalent(e1: Expression, e2: Expression): Unit = {
+    val correctAnswer = Project(Alias(e2, "out")() :: Nil, relation).analyze
+    val actual = Optimize.execute(Project(Alias(e1, "out")() :: Nil, relation).analyze)
+    comparePlans(actual, correctAnswer)
+  }
+
+  test("Push down EqualTo through If") {
+    assertEquivalent(EqualTo(ifExp, Literal(4)), FalseLiteral)
+    assertEquivalent(EqualTo(ifExp, Literal(3)), Not(a <=> TrueLiteral))
+
+    // Push down at most one not foldable expressions.
+    assertEquivalent(
+      EqualTo(If(a, b, Literal(2)), Literal(2)),
+      If(a, EqualTo(b, Literal(2)), TrueLiteral))
+    assertEquivalent(
+      EqualTo(If(a, b, b + 1), Literal(2)),
+      EqualTo(If(a, b, b + 1), Literal(2)))
+
+    // Push down non-deterministic expressions.
+    val nonDeterministic = If(LessThan(Rand(1), Literal(0.5)), Literal(1), Literal(2))
+    assert(!nonDeterministic.deterministic)
+    assertEquivalent(EqualTo(nonDeterministic, Literal(2)),
+      GreaterThanOrEqual(Rand(1), Literal(0.5)))
+    assertEquivalent(EqualTo(nonDeterministic, Literal(3)),
+      If(LessThan(Rand(1), Literal(0.5)), FalseLiteral, FalseLiteral))
+
+    // Handle Null values.
+    assertEquivalent(
+      EqualTo(If(a, Literal(null, IntegerType), Literal(1)), Literal(1)),
+      If(a, Literal(null, BooleanType), TrueLiteral))
+    assertEquivalent(
+      EqualTo(If(a, Literal(null, IntegerType), Literal(1)), Literal(2)),
+      If(a, Literal(null, BooleanType), FalseLiteral))
+    assertEquivalent(
+      EqualTo(If(a, Literal(1), Literal(2)), Literal(null, IntegerType)),
+      Literal(null, BooleanType))
+    assertEquivalent(
+      EqualTo(If(a, Literal(null, IntegerType), Literal(null, IntegerType)), Literal(1)),
+      Literal(null, BooleanType))
+  }
+
+  test("Push down other BinaryComparison through If") {
+    assertEquivalent(EqualNullSafe(ifExp, Literal(4)), FalseLiteral)
+    assertEquivalent(GreaterThan(ifExp, Literal(4)), FalseLiteral)
+    assertEquivalent(GreaterThanOrEqual(ifExp, Literal(4)), FalseLiteral)
+    assertEquivalent(LessThan(ifExp, Literal(4)), TrueLiteral)
+    assertEquivalent(LessThanOrEqual(ifExp, Literal(4)), TrueLiteral)
+  }
+
+  test("Push down other BinaryOperator through If") {
+    assertEquivalent(Add(ifExp, Literal(4)), If(a, Literal(6), Literal(7)))
+    assertEquivalent(Subtract(ifExp, Literal(4)), If(a, Literal(-2), Literal(-1)))
+    assertEquivalent(Multiply(ifExp, Literal(4)), If(a, Literal(8), Literal(12)))
+    assertEquivalent(Pmod(ifExp, Literal(4)), If(a, Literal(2), Literal(3)))
+    assertEquivalent(Remainder(ifExp, Literal(4)), If(a, Literal(2), Literal(3)))
+    assertEquivalent(Divide(If(a, Literal(2.0), Literal(3.0)), Literal(1.0)),
+      If(a, Literal(2.0), Literal(3.0)))
+    assertEquivalent(And(If(a, FalseLiteral, TrueLiteral), TrueLiteral), Not(a <=> TrueLiteral))
+    assertEquivalent(Or(If(a, FalseLiteral, TrueLiteral), TrueLiteral), TrueLiteral)
+  }
+
+  test("Push down other BinaryExpression through If") {
+    assertEquivalent(BRound(If(a, Literal(1.23), Literal(1.24)), Literal(1)), Literal(1.2))
+    assertEquivalent(StartsWith(If(a, Literal("ab"), Literal("ac")), Literal("a")), TrueLiteral)
+    assertEquivalent(FindInSet(If(a, Literal("ab"), Literal("ac")), Literal("a")), Literal(0))
+    assertEquivalent(
+      AddMonths(If(a, Literal(Date.valueOf("2020-01-01")), Literal(Date.valueOf("2021-01-01"))),
+        Literal(1)),
+      If(a, Literal(Date.valueOf("2020-02-01")), Literal(Date.valueOf("2021-02-01"))))
+  }
+
+  test("Push down EqualTo through CaseWhen") {
+    assertEquivalent(EqualTo(caseWhen, Literal(4)), FalseLiteral)
+    assertEquivalent(EqualTo(caseWhen, Literal(3)),
+      CaseWhen(Seq((a, FalseLiteral), (c, FalseLiteral)), Some(TrueLiteral)))
+    assertEquivalent(
+      EqualTo(CaseWhen(Seq((a, Literal(1)), (c, Literal(2))), None), Literal(4)),
+      CaseWhen(Seq((a, FalseLiteral), (c, FalseLiteral)), None))
+
+    assertEquivalent(
+      And(EqualTo(caseWhen, Literal(5)), EqualTo(caseWhen, Literal(6))),
+      FalseLiteral)
+
+    // Push down at most one branch is not foldable expressions.
+    assertEquivalent(EqualTo(CaseWhen(Seq((a, b), (c, Literal(1))), None), Literal(1)),
+      CaseWhen(Seq((a, EqualTo(b, Literal(1))), (c, TrueLiteral)), None))
+    assertEquivalent(EqualTo(CaseWhen(Seq((a, b), (c, b + 1)), None), Literal(1)),
+      EqualTo(CaseWhen(Seq((a, b), (c, b + 1)), None), Literal(1)))
+    assertEquivalent(EqualTo(CaseWhen(Seq((a, b)), None), Literal(1)),
+      EqualTo(CaseWhen(Seq((a, b)), None), Literal(1)))
+
+    // Push down non-deterministic expressions.
+    val nonDeterministic =
+      CaseWhen(Seq((LessThan(Rand(1), Literal(0.5)), Literal(1))), Some(Literal(2)))
+    assert(!nonDeterministic.deterministic)
+    assertEquivalent(EqualTo(nonDeterministic, Literal(2)),
+      GreaterThanOrEqual(Rand(1), Literal(0.5)))
+    assertEquivalent(EqualTo(nonDeterministic, Literal(3)),
+      CaseWhen(Seq((LessThan(Rand(1), Literal(0.5)), FalseLiteral)), Some(FalseLiteral)))
+
+    // Handle Null values.
+    assertEquivalent(
+      EqualTo(CaseWhen(Seq((a, Literal(null, IntegerType))), Some(Literal(1))), Literal(2)),
+      CaseWhen(Seq((a, Literal(null, BooleanType))), Some(FalseLiteral)))
+    assertEquivalent(
+      EqualTo(CaseWhen(Seq((a, Literal(1))), Some(Literal(2))), Literal(null, IntegerType)),
+      Literal(null, BooleanType))
+    assertEquivalent(
+      EqualTo(CaseWhen(Seq((a, Literal(null, IntegerType))), Some(Literal(1))), Literal(1)),
+      CaseWhen(Seq((a, Literal(null, BooleanType))), Some(TrueLiteral)))
+    assertEquivalent(
+      EqualTo(CaseWhen(Seq((a, Literal(null, IntegerType))), Some(Literal(null, IntegerType))),
+        Literal(1)),
+      Literal(null, BooleanType))
+    assertEquivalent(
+      EqualTo(CaseWhen(Seq((a, Literal(null, IntegerType))), Some(Literal(null, IntegerType))),
+        Literal(null, IntegerType)),
+      Literal(null, BooleanType))
+  }
+
+  test("Push down other BinaryComparison through CaseWhen") {
+    assertEquivalent(EqualNullSafe(caseWhen, Literal(4)), FalseLiteral)
+    assertEquivalent(GreaterThan(caseWhen, Literal(4)), FalseLiteral)
+    assertEquivalent(GreaterThanOrEqual(caseWhen, Literal(4)), FalseLiteral)
+    assertEquivalent(LessThan(caseWhen, Literal(4)), TrueLiteral)
+    assertEquivalent(LessThanOrEqual(caseWhen, Literal(4)), TrueLiteral)
+  }
+
+  test("Push down other BinaryOperator through CaseWhen") {
+    assertEquivalent(Add(caseWhen, Literal(4)),
+      CaseWhen(Seq((a, Literal(5)), (c, Literal(6))), Some(Literal(7))))
+    assertEquivalent(Subtract(caseWhen, Literal(4)),
+      CaseWhen(Seq((a, Literal(-3)), (c, Literal(-2))), Some(Literal(-1))))
+    assertEquivalent(Multiply(caseWhen, Literal(4)),
+      CaseWhen(Seq((a, Literal(4)), (c, Literal(8))), Some(Literal(12))))
+    assertEquivalent(Pmod(caseWhen, Literal(4)),
+      CaseWhen(Seq((a, Literal(1)), (c, Literal(2))), Some(Literal(3))))
+    assertEquivalent(Remainder(caseWhen, Literal(4)),
+      CaseWhen(Seq((a, Literal(1)), (c, Literal(2))), Some(Literal(3))))
+    assertEquivalent(Divide(CaseWhen(Seq((a, Literal(1.0)), (c, Literal(2.0))), Some(Literal(3.0))),
+      Literal(1.0)),
+      CaseWhen(Seq((a, Literal(1.0)), (c, Literal(2.0))), Some(Literal(3.0))))
+    assertEquivalent(And(CaseWhen(Seq((a, FalseLiteral), (c, TrueLiteral)), Some(TrueLiteral)),
+      TrueLiteral),
+      CaseWhen(Seq((a, FalseLiteral), (c, TrueLiteral)), Some(TrueLiteral)))
+    assertEquivalent(Or(CaseWhen(Seq((a, FalseLiteral), (c, TrueLiteral)), Some(TrueLiteral)),
+      TrueLiteral), TrueLiteral)
+  }
+
+  test("Push down other BinaryExpression through CaseWhen") {
+    assertEquivalent(
+      BRound(CaseWhen(Seq((a, Literal(1.23)), (c, Literal(1.24))), Some(Literal(1.25))),
+        Literal(1)),
+      Literal(1.2))
+    assertEquivalent(
+      StartsWith(CaseWhen(Seq((a, Literal("ab")), (c, Literal("ac"))), Some(Literal("ad"))),
+        Literal("a")),
+      TrueLiteral)
+    assertEquivalent(
+      FindInSet(CaseWhen(Seq((a, Literal("ab")), (c, Literal("ac"))), Some(Literal("ad"))),
+        Literal("a")),
+      Literal(0))
+    assertEquivalent(
+      AddMonths(CaseWhen(Seq((a, Literal(Date.valueOf("2020-01-01"))),
+        (c, Literal(Date.valueOf("2021-01-01")))),
+        Some(Literal(Date.valueOf("2022-01-01")))),
+        Literal(1)),
+      CaseWhen(Seq((a, Literal(Date.valueOf("2020-02-01"))),
+        (c, Literal(Date.valueOf("2021-02-01")))),
+        Some(Literal(Date.valueOf("2022-02-01")))))
+  }
+
+  test("Push down BinaryExpression through If/CaseWhen backwards") {
+    assertEquivalent(EqualTo(Literal(4), ifExp), FalseLiteral)
+    assertEquivalent(EqualTo(Literal(4), caseWhen), FalseLiteral)
+  }
+
+  test("SPARK-33848: Push down cast through If/CaseWhen") {
+    assertEquivalent(If(a, Literal(2), Literal(3)).cast(StringType),
+      If(a, Literal("2"), Literal("3")))
+    assertEquivalent(If(a, b, Literal(3)).cast(StringType),
+      If(a, b.cast(StringType), Literal("3")))
+    assertEquivalent(If(a, b, b + 1).cast(StringType),
+      If(a, b, b + 1).cast(StringType))
+
+    assertEquivalent(
+      CaseWhen(Seq((a, Literal(1))), Some(Literal(3))).cast(StringType),
+      CaseWhen(Seq((a, Literal("1"))), Some(Literal("3"))))
+    assertEquivalent(
+      CaseWhen(Seq((a, Literal(1))), Some(b)).cast(StringType),
+      CaseWhen(Seq((a, Literal("1"))), Some(b.cast(StringType))))
+    assertEquivalent(
+      CaseWhen(Seq((a, b)), Some(b + 1)).cast(StringType),
+      CaseWhen(Seq((a, b)), Some(b + 1)).cast(StringType))
+  }
+
+  test("SPARK-33848: Push down abs through If/CaseWhen") {
+    assertEquivalent(Abs(If(a, Literal(-2), Literal(-3))), If(a, Literal(2), Literal(3)))
+    assertEquivalent(
+      Abs(CaseWhen(Seq((a, Literal(-1))), Some(Literal(-3)))),
+      CaseWhen(Seq((a, Literal(1))), Some(Literal(3))))
+  }
+
+  test("SPARK-33848: Push down cast with binary expression through If/CaseWhen") {
+    assertEquivalent(EqualTo(If(a, Literal(2), Literal(3)).cast(StringType), Literal("4")),
+      FalseLiteral)
+    assertEquivalent(
+      EqualTo(CaseWhen(Seq((a, Literal(1))), Some(Literal(3))).cast(StringType), Literal("4")),
+      FalseLiteral)
+    assertEquivalent(
+      EqualTo(CaseWhen(Seq((a, Literal(1)), (c, Literal(2))), None).cast(StringType), Literal("4")),
+      CaseWhen(Seq((a, FalseLiteral), (c, FalseLiteral)), None))
+  }
+
+  test("SPARK-33848: Push down dateTimeExpression with binary expression through If/CaseWhen") {
+    val d = Date.valueOf("2021-01-01")
+    // If
+    assertEquivalent(AddMonths(Literal(d),
+      If(a, Literal(1), Literal(2))),
+      If(a, Literal(Date.valueOf("2021-02-01")), Literal(Date.valueOf("2021-03-01"))))
+    assertEquivalent(DateAdd(Literal(d),
+      If(a, Literal(1), Literal(2))),
+      If(a, Literal(Date.valueOf("2021-01-02")), Literal(Date.valueOf("2021-01-03"))))
+    assertEquivalent(DateAddInterval(Literal(d),
+      If(a, Literal(new CalendarInterval(1, 1, 0)),
+        Literal(new CalendarInterval(1, 2, 0)))),
+      If(a, Literal(Date.valueOf("2021-02-02")), Literal(Date.valueOf("2021-02-03"))))
+    assertEquivalent(DateAddYMInterval(Literal(d),
+      If(a, Literal.create(Period.ofMonths(1), YearMonthIntervalType),
+        Literal.create(Period.ofMonths(2), YearMonthIntervalType))),
+      If(a, Literal(Date.valueOf("2021-02-01")), Literal(Date.valueOf("2021-03-01"))))
+    assertEquivalent(DateDiff(Literal(d),
+      If(a, Literal(Date.valueOf("2021-02-01")), Literal(Date.valueOf("2021-03-01")))),
+      If(a, Literal(-31), Literal(-59)))
+    assertEquivalent(DateSub(Literal(d),
+      If(a, Literal(1), Literal(2))),
+      If(a, Literal(Date.valueOf("2020-12-31")), Literal(Date.valueOf("2020-12-30"))))
+    assertEquivalent(TimestampAddYMInterval(
+      Literal.create(Timestamp.valueOf("2021-01-01 00:00:00.000"), TimestampType),
+      If(a, Literal.create(Period.ofMonths(1), YearMonthIntervalType),
+        Literal.create(Period.ofMonths(2), YearMonthIntervalType))),
+      If(a, Literal.create(Timestamp.valueOf("2021-02-01 00:00:00"), TimestampType),
+        Literal.create(Timestamp.valueOf("2021-03-01 00:00:00"), TimestampType)))
+    assertEquivalent(TimeAdd(
+      Literal.create(Timestamp.valueOf("2021-01-01 00:00:00.000"), TimestampType),
+      If(a, Literal(Duration.ofDays(10).plusMinutes(10).plusMillis(321)),
+        Literal(Duration.ofDays(10).plusMinutes(10).plusMillis(456)))),
+      If(a, Literal.create(Timestamp.valueOf("2021-01-11 00:10:00.321"), TimestampType),
+        Literal.create(Timestamp.valueOf("2021-01-11 00:10:00.456"), TimestampType)))
+
+    // CaseWhen
+    assertEquivalent(AddMonths(Literal(d),
+      CaseWhen(Seq((a, Literal(1)), (c, Literal(2))), None)),
+      CaseWhen(Seq((a, Literal(Date.valueOf("2021-02-01"))),
+        (c, Literal(Date.valueOf("2021-03-01")))), None))
+    assertEquivalent(DateAdd(Literal(d),
+      CaseWhen(Seq((a, Literal(1)), (c, Literal(2))), None)),
+      CaseWhen(Seq((a, Literal(Date.valueOf("2021-01-02"))),
+        (c, Literal(Date.valueOf("2021-01-03")))), None))
+    assertEquivalent(DateAddInterval(Literal(d),
+      CaseWhen(Seq((a, Literal(new CalendarInterval(1, 1, 0))),
+        (c, Literal(new CalendarInterval(1, 2, 0)))), None)),
+      CaseWhen(Seq((a, Literal(Date.valueOf("2021-02-02"))),
+        (c, Literal(Date.valueOf("2021-02-03")))), None))
+    assertEquivalent(DateAddYMInterval(Literal(d),
+      CaseWhen(Seq((a, Literal.create(Period.ofMonths(1), YearMonthIntervalType)),
+        (c, Literal.create(Period.ofMonths(2), YearMonthIntervalType))), None)),
+      CaseWhen(Seq((a, Literal(Date.valueOf("2021-02-01"))),
+        (c, Literal(Date.valueOf("2021-03-01")))), None))
+    assertEquivalent(DateDiff(Literal(d),
+      CaseWhen(Seq((a, Literal(Date.valueOf("2021-02-01"))),
+        (c, Literal(Date.valueOf("2021-03-01")))), None)),
+      CaseWhen(Seq((a, Literal(-31)), (c, Literal(-59))), None))
+    assertEquivalent(DateSub(Literal(d),
+      CaseWhen(Seq((a, Literal(1)), (c, Literal(2))), None)),
+      CaseWhen(Seq((a, Literal(Date.valueOf("2020-12-31"))),
+        (c, Literal(Date.valueOf("2020-12-30")))), None))
+    assertEquivalent(TimestampAddYMInterval(
+      Literal.create(Timestamp.valueOf("2021-01-01 00:00:00.000"), TimestampType),
+      CaseWhen(Seq((a, Literal.create(Period.ofMonths(1), YearMonthIntervalType)),
+        (c, Literal.create(Period.ofMonths(2), YearMonthIntervalType))), None)),
+      CaseWhen(Seq((a, Literal.create(Timestamp.valueOf("2021-02-01 00:00:00"), TimestampType)),
+        (c, Literal.create(Timestamp.valueOf("2021-03-01 00:00:00"), TimestampType))), None))
+    assertEquivalent(TimeAdd(
+      Literal.create(Timestamp.valueOf("2021-01-01 00:00:00.000"), TimestampType),
+      CaseWhen(Seq((a, Literal(Duration.ofDays(10).plusMinutes(10).plusMillis(321))),
+        (c, Literal(Duration.ofDays(10).plusMinutes(10).plusMillis(456)))), None)),
+      CaseWhen(Seq((a, Literal.create(Timestamp.valueOf("2021-01-11 00:10:00.321"), TimestampType)),
+        (c, Literal.create(Timestamp.valueOf("2021-01-11 00:10:00.456"), TimestampType))), None))
+  }
+
+  test("SPARK-33847: Remove the CaseWhen if elseValue is empty and other outputs are null") {
+    assertEquivalent(
+      EqualTo(CaseWhen(Seq((a, Literal.create(null, IntegerType)))), Literal(2)),
+      Literal.create(null, BooleanType))
+    assertEquivalent(
+      EqualTo(CaseWhen(Seq((LessThan(Rand(1), Literal(0.5)), Literal("str")))).cast(IntegerType),
+        Literal(2)),
+      CaseWhen(Seq((LessThan(Rand(1), Literal(0.5)), Literal.create(null, BooleanType)))))
+  }
+
+  test("SPARK-33884: simplify CaseWhen clauses with (true and false) and (false and true)") {
+    assertEquivalent(
+      EqualTo(CaseWhen(Seq(('a > 10, Literal(0))), Literal(1)), Literal(0)),
+      'a > 10 <=> TrueLiteral)
+    assertEquivalent(
+      EqualTo(CaseWhen(Seq(('a > 10, Literal(0))), Literal(1)), Literal(1)),
+      Not('a > 10 <=> TrueLiteral))
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RemoveNoopOperatorsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RemoveNoopOperatorsSuite.scala
new file mode 100644
index 0000000000000..cedd21d2bf522
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RemoveNoopOperatorsSuite.scala
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+
+class RemoveNoopOperatorsSuite extends PlanTest {
+
+  object Optimize extends RuleExecutor[LogicalPlan] {
+    val batches =
+      Batch("RemoveNoopOperators", Once,
+        RemoveNoopOperators) :: Nil
+  }
+
+  val testRelation = LocalRelation('a.int, 'b.int, 'c.int)
+
+  test("Remove all redundant projections in one iteration") {
+    val originalQuery = testRelation
+      .select('a, 'b, 'c)
+      .select('a, 'b, 'c)
+      .analyze
+
+    val optimized = Optimize.execute(originalQuery.analyze)
+
+    comparePlans(optimized, testRelation)
+  }
+
+  test("Remove all redundant windows in one iteration") {
+    val originalQuery = testRelation
+      .window(Nil, Nil, Nil)
+      .window(Nil, Nil, Nil)
+      .analyze
+
+    val optimized = Optimize.execute(originalQuery.analyze)
+
+    comparePlans(optimized, testRelation)
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RemoveNoopUnionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RemoveNoopUnionSuite.scala
new file mode 100644
index 0000000000000..1b40280408e6e
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RemoveNoopUnionSuite.scala
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+
+class RemoveNoopUnionSuite extends PlanTest {
+
+  object Optimize extends RuleExecutor[LogicalPlan] {
+    val batches =
+      Batch("CollapseProject", Once,
+        CollapseProject) ::
+        Batch("RemoveNoopUnion", Once,
+          RemoveNoopUnion) :: Nil
+  }
+
+  val testRelation = LocalRelation('a.int, 'b.int)
+  val testRelation2 = LocalRelation(output = Seq('a.int, 'b.int), data = Seq(InternalRow(1, 2)))
+
+  test("SPARK-34474: Remove redundant Union under Distinct") {
+    val union = Union(testRelation :: testRelation :: Nil)
+    val distinct = Distinct(union)
+    val optimized = Optimize.execute(distinct)
+    comparePlans(optimized, Distinct(testRelation))
+  }
+
+  test("SPARK-34474: Remove redundant Union under Deduplicate") {
+    val union = Union(testRelation :: testRelation :: Nil)
+    val deduplicate = Deduplicate(testRelation.output, union)
+    val optimized = Optimize.execute(deduplicate)
+    comparePlans(optimized, Deduplicate(testRelation.output, testRelation))
+  }
+
+  test("SPARK-34474: Do not remove necessary Project 1") {
+    val child1 = Project(Seq(testRelation.output(0), testRelation.output(1),
+      (testRelation.output(0) + 1).as("expr")), testRelation)
+    val child2 = Project(Seq(testRelation.output(0), testRelation.output(1),
+      (testRelation.output(0) + 2).as("expr")), testRelation)
+    val union = Union(child1 :: child2 :: Nil)
+    val distinct = Distinct(union)
+    val optimized = Optimize.execute(distinct)
+    comparePlans(optimized, distinct)
+  }
+
+  test("SPARK-34474: Do not remove necessary Project 2") {
+    val child1 = Project(Seq(testRelation.output(0), testRelation.output(1)), testRelation)
+    val child2 = Project(Seq(testRelation.output(1), testRelation.output(0)), testRelation)
+    val union = Union(child1 :: child2 :: Nil)
+    val distinct = Distinct(union)
+    val optimized = Optimize.execute(distinct)
+    comparePlans(optimized, distinct)
+  }
+
+  test("SPARK-34548: Remove unnecessary children from Union") {
+    val union = Union(testRelation :: testRelation :: testRelation2 :: Nil)
+    val distinct = Distinct(union)
+    val optimized1 = Optimize.execute(distinct)
+    comparePlans(optimized1, Distinct(Union(testRelation :: testRelation2 :: Nil)))
+
+    val deduplicate = Deduplicate(testRelation.output, union)
+    val optimized2 = Optimize.execute(deduplicate)
+    comparePlans(optimized2,
+      Deduplicate(testRelation.output, Union(testRelation :: testRelation2 :: Nil)))
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RemoveRedundantAggregatesSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RemoveRedundantAggregatesSuite.scala
new file mode 100644
index 0000000000000..d376c31ef965f
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RemoveRedundantAggregatesSuite.scala
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.api.python.PythonEvalType
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions.{Expression, PythonUDF}
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan}
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types.IntegerType
+
+class RemoveRedundantAggregatesSuite extends PlanTest {
+
+  object Optimize extends RuleExecutor[LogicalPlan] {
+    val batches = Batch("RemoveRedundantAggregates", FixedPoint(10),
+      RemoveRedundantAggregates) :: Nil
+  }
+
+  private def aggregates(e: Expression): Seq[Expression] = {
+    Seq(
+      count(e),
+      PythonUDF("pyUDF", null, IntegerType, Seq(e),
+        PythonEvalType.SQL_GROUPED_AGG_PANDAS_UDF, udfDeterministic = true)
+    )
+  }
+
+  test("Remove redundant aggregate") {
+    val relation = LocalRelation('a.int, 'b.int)
+    for (agg <- aggregates('b)) {
+      val query = relation
+        .groupBy('a)('a, agg)
+        .groupBy('a)('a)
+        .analyze
+      val expected = relation
+        .groupBy('a)('a)
+        .analyze
+      val optimized = Optimize.execute(query)
+      comparePlans(optimized, expected)
+    }
+  }
+
+  test("Remove 2 redundant aggregates") {
+    val relation = LocalRelation('a.int, 'b.int)
+    for (agg <- aggregates('b)) {
+      val query = relation
+        .groupBy('a)('a, agg)
+        .groupBy('a)('a)
+        .groupBy('a)('a)
+        .analyze
+      val expected = relation
+        .groupBy('a)('a)
+        .analyze
+      val optimized = Optimize.execute(query)
+      comparePlans(optimized, expected)
+    }
+  }
+
+  test("Remove redundant aggregate with different grouping") {
+    val relation = LocalRelation('a.int, 'b.int)
+    val query = relation
+      .groupBy('a, 'b)('a)
+      .groupBy('a)('a)
+      .analyze
+    val expected = relation
+      .groupBy('a)('a)
+      .analyze
+    val optimized = Optimize.execute(query)
+    comparePlans(optimized, expected)
+  }
+
+  test("Remove redundant aggregate with aliases") {
+    val relation = LocalRelation('a.int, 'b.int)
+    for (agg <- aggregates('b)) {
+      val query = relation
+        .groupBy('a + 'b)(('a + 'b) as 'c, agg)
+        .groupBy('c)('c)
+        .analyze
+      val expected = relation
+        .groupBy('a + 'b)(('a + 'b) as 'c)
+        .analyze
+      val optimized = Optimize.execute(query)
+      comparePlans(optimized, expected)
+    }
+  }
+
+  test("Remove redundant aggregate with non-deterministic upper") {
+    val relation = LocalRelation('a.int, 'b.int)
+    val query = relation
+      .groupBy('a)('a)
+      .groupBy('a)('a, rand(0) as 'c)
+      .analyze
+    val expected = relation
+      .groupBy('a)('a, rand(0) as 'c)
+      .analyze
+    val optimized = Optimize.execute(query)
+    comparePlans(optimized, expected)
+  }
+
+  test("Remove redundant aggregate with non-deterministic lower") {
+    val relation = LocalRelation('a.int, 'b.int)
+    val query = relation
+      .groupBy('a, 'c)('a, rand(0) as 'c)
+      .groupBy('a, 'c)('a, 'c)
+      .analyze
+    val expected = relation
+      .groupBy('a, 'c)('a, rand(0) as 'c)
+      .analyze
+    val optimized = Optimize.execute(query)
+    comparePlans(optimized, expected)
+  }
+
+  test("Keep non-redundant aggregate - upper has agg expression") {
+    val relation = LocalRelation('a.int, 'b.int)
+    for (agg <- aggregates('b)) {
+      val query = relation
+        .groupBy('a, 'b)('a, 'b)
+        // The count would change if we remove the first aggregate
+        .groupBy('a)('a, agg)
+        .analyze
+      val optimized = Optimize.execute(query)
+      comparePlans(optimized, query)
+    }
+  }
+
+  test("Keep non-redundant aggregate - upper references agg expression") {
+    val relation = LocalRelation('a.int, 'b.int)
+    for (agg <- aggregates('b)) {
+      val query = relation
+        .groupBy('a)('a, agg as 'c)
+        .groupBy('c)('c)
+        .analyze
+      val optimized = Optimize.execute(query)
+      comparePlans(optimized, query)
+    }
+  }
+
+  test("Keep non-redundant aggregate - upper references non-deterministic non-grouping") {
+    val relation = LocalRelation('a.int, 'b.int)
+    val query = relation
+      .groupBy('a)('a, ('a + rand(0)) as 'c)
+      .groupBy('a, 'c)('a, 'c)
+      .analyze
+    val optimized = Optimize.execute(query)
+    comparePlans(optimized, query)
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RemoveRedundantAliasAndProjectSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RemoveRedundantAliasAndProjectSuite.scala
index 2e0ab7f64f4d6..4b02a847880f7 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RemoveRedundantAliasAndProjectSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RemoveRedundantAliasAndProjectSuite.scala
@@ -83,12 +83,12 @@ class RemoveRedundantAliasAndProjectSuite extends PlanTest with PredicateHelper
     comparePlans(optimized, query)
   }
 
-  test("retain deduplicating alias in self-join") {
+  test("remove redundant project with self-join") {
     val relation = LocalRelation('a.int)
     val fragment = relation.select('a as 'a)
     val query = fragment.select('a as 'a).join(fragment.select('a as 'a)).analyze
     val optimized = Optimize.execute(query)
-    val expected = relation.join(relation.select('a as 'a)).analyze
+    val expected = relation.join(relation).analyze
     comparePlans(optimized, expected)
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicateSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicateSuite.scala
index c7f42f08406e7..71e2364917c0c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicateSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicateSuite.scala
@@ -17,13 +17,14 @@
 
 package org.apache.spark.sql.catalyst.optimizer
 
+import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
-import org.apache.spark.sql.catalyst.expressions.{And, ArrayExists, ArrayFilter, ArrayTransform, CaseWhen, Expression, GreaterThan, If, LambdaFunction, Literal, MapFilter, NamedExpression, Or, UnresolvedNamedLambdaVariable}
+import org.apache.spark.sql.catalyst.expressions.{And, ArrayExists, ArrayFilter, ArrayTransform, CaseWhen, Expression, GreaterThan, If, In, InSet, LambdaFunction, Literal, MapFilter, NamedExpression, Not, Or, UnresolvedNamedLambdaVariable}
 import org.apache.spark.sql.catalyst.expressions.Literal.{FalseLiteral, TrueLiteral}
 import org.apache.spark.sql.catalyst.plans.{Inner, PlanTest}
-import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan}
+import org.apache.spark.sql.catalyst.plans.logical.{Assignment, DeleteAction, DeleteFromTable, InsertAction, InsertStarAction, LocalRelation, LogicalPlan, MergeIntoTable, UpdateAction, UpdateStarAction, UpdateTable}
 import org.apache.spark.sql.catalyst.rules.RuleExecutor
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{BooleanType, IntegerType}
@@ -47,13 +48,16 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
   test("replace null inside filter and join conditions") {
     testFilter(originalCond = Literal(null, BooleanType), expectedCond = FalseLiteral)
     testJoin(originalCond = Literal(null, BooleanType), expectedCond = FalseLiteral)
+    testDelete(originalCond = Literal(null, BooleanType), expectedCond = FalseLiteral)
+    testUpdate(originalCond = Literal(null, BooleanType), expectedCond = FalseLiteral)
+    testMerge(originalCond = Literal(null, BooleanType), expectedCond = FalseLiteral)
   }
 
   test("Not expected type - replaceNullWithFalse") {
-    val e = intercept[IllegalArgumentException] {
+    val e = intercept[AnalysisException] {
       testFilter(originalCond = Literal(null, IntegerType), expectedCond = FalseLiteral)
     }.getMessage
-    assert(e.contains("but got the type `int` in `CAST(NULL AS INT)"))
+    assert(e.contains("'CAST(NULL AS INT)' of type int is not a boolean"))
   }
 
   test("replace null in branches of If") {
@@ -63,6 +67,9 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
       Literal(null, BooleanType))
     testFilter(originalCond, expectedCond = FalseLiteral)
     testJoin(originalCond, expectedCond = FalseLiteral)
+    testDelete(originalCond, expectedCond = FalseLiteral)
+    testUpdate(originalCond, expectedCond = FalseLiteral)
+    testMerge(originalCond, expectedCond = FalseLiteral)
   }
 
   test("replace nulls in nested expressions in branches of If") {
@@ -72,6 +79,9 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
       UnresolvedAttribute("b") && Literal(null, BooleanType))
     testFilter(originalCond, expectedCond = FalseLiteral)
     testJoin(originalCond, expectedCond = FalseLiteral)
+    testDelete(originalCond, expectedCond = FalseLiteral)
+    testUpdate(originalCond, expectedCond = FalseLiteral)
+    testMerge(originalCond, expectedCond = FalseLiteral)
   }
 
   test("replace null in elseValue of CaseWhen") {
@@ -82,6 +92,9 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
     val expectedCond = CaseWhen(branches, FalseLiteral)
     testFilter(originalCond, expectedCond)
     testJoin(originalCond, expectedCond)
+    testDelete(originalCond, expectedCond)
+    testUpdate(originalCond, expectedCond)
+    testMerge(originalCond, expectedCond)
   }
 
   test("replace null in branch values of CaseWhen") {
@@ -91,6 +104,9 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
     val originalCond = CaseWhen(branches, Literal(null))
     testFilter(originalCond, expectedCond = FalseLiteral)
     testJoin(originalCond, expectedCond = FalseLiteral)
+    testDelete(originalCond, expectedCond = FalseLiteral)
+    testUpdate(originalCond, expectedCond = FalseLiteral)
+    testMerge(originalCond, expectedCond = FalseLiteral)
   }
 
   test("replace null in branches of If inside CaseWhen") {
@@ -103,10 +119,13 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
     val expectedBranches = Seq(
       (UnresolvedAttribute("i") < Literal(10)) -> FalseLiteral,
       (UnresolvedAttribute("i") > Literal(40)) -> TrueLiteral)
-    val expectedCond = CaseWhen(expectedBranches)
+    val expectedCond = CaseWhen(expectedBranches, FalseLiteral)
 
     testFilter(originalCond, expectedCond)
     testJoin(originalCond, expectedCond)
+    testDelete(originalCond, expectedCond)
+    testUpdate(originalCond, expectedCond)
+    testMerge(originalCond, expectedCond)
   }
 
   test("replace null in complex CaseWhen expressions") {
@@ -122,10 +141,13 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
       (UnresolvedAttribute("i") < Literal(10)) -> TrueLiteral,
       (UnresolvedAttribute("i") > Literal(10)) -> FalseLiteral,
       TrueLiteral -> TrueLiteral)
-    val expectedCond = CaseWhen(expectedBranches)
+    val expectedCond = CaseWhen(expectedBranches, FalseLiteral)
 
     testFilter(originalCond, expectedCond)
     testJoin(originalCond, expectedCond)
+    testDelete(originalCond, expectedCond)
+    testUpdate(originalCond, expectedCond)
+    testMerge(originalCond, expectedCond)
   }
 
   test("replace null in Or") {
@@ -133,12 +155,18 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
     val expectedCond = UnresolvedAttribute("b")
     testFilter(originalCond, expectedCond)
     testJoin(originalCond, expectedCond)
+    testDelete(originalCond, expectedCond)
+    testUpdate(originalCond, expectedCond)
+    testMerge(originalCond, expectedCond)
   }
 
   test("replace null in And") {
     val originalCond = And(UnresolvedAttribute("b"), Literal(null))
     testFilter(originalCond, expectedCond = FalseLiteral)
     testJoin(originalCond, expectedCond = FalseLiteral)
+    testDelete(originalCond, expectedCond = FalseLiteral)
+    testUpdate(originalCond, expectedCond = FalseLiteral)
+    testMerge(originalCond, expectedCond = FalseLiteral)
   }
 
   test("replace nulls in nested And/Or expressions") {
@@ -147,6 +175,9 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
       Or(Literal(null), And(Literal(null), And(UnresolvedAttribute("b"), Literal(null)))))
     testFilter(originalCond, expectedCond = FalseLiteral)
     testJoin(originalCond, expectedCond = FalseLiteral)
+    testDelete(originalCond, expectedCond = FalseLiteral)
+    testUpdate(originalCond, expectedCond = FalseLiteral)
+    testMerge(originalCond, expectedCond = FalseLiteral)
   }
 
   test("replace null in And inside branches of If") {
@@ -156,6 +187,9 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
       And(UnresolvedAttribute("b"), Literal(null, BooleanType)))
     testFilter(originalCond, expectedCond = FalseLiteral)
     testJoin(originalCond, expectedCond = FalseLiteral)
+    testDelete(originalCond, expectedCond = FalseLiteral)
+    testUpdate(originalCond, expectedCond = FalseLiteral)
+    testMerge(originalCond, expectedCond = FalseLiteral)
   }
 
   test("replace null in branches of If inside And") {
@@ -167,6 +201,9 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
         And(FalseLiteral, UnresolvedAttribute("b"))))
     testFilter(originalCond, expectedCond = FalseLiteral)
     testJoin(originalCond, expectedCond = FalseLiteral)
+    testDelete(originalCond, expectedCond = FalseLiteral)
+    testUpdate(originalCond, expectedCond = FalseLiteral)
+    testMerge(originalCond, expectedCond = FalseLiteral)
   }
 
   test("replace null in branches of If inside another If") {
@@ -176,6 +213,9 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
       Literal(null))
     testFilter(originalCond, expectedCond = FalseLiteral)
     testJoin(originalCond, expectedCond = FalseLiteral)
+    testDelete(originalCond, expectedCond = FalseLiteral)
+    testUpdate(originalCond, expectedCond = FalseLiteral)
+    testMerge(originalCond, expectedCond = FalseLiteral)
   }
 
   test("replace null in CaseWhen inside another CaseWhen") {
@@ -183,6 +223,9 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
     val originalCond = CaseWhen(Seq(nestedCaseWhen -> TrueLiteral), Literal(null))
     testFilter(originalCond, expectedCond = FalseLiteral)
     testJoin(originalCond, expectedCond = FalseLiteral)
+    testDelete(originalCond, expectedCond = FalseLiteral)
+    testUpdate(originalCond, expectedCond = FalseLiteral)
+    testMerge(originalCond, expectedCond = FalseLiteral)
   }
 
   test("inability to replace null in non-boolean branches of If") {
@@ -195,6 +238,9 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
       FalseLiteral)
     testFilter(originalCond = condition, expectedCond = condition)
     testJoin(originalCond = condition, expectedCond = condition)
+    testDelete(originalCond = condition, expectedCond = condition)
+    testUpdate(originalCond = condition, expectedCond = condition)
+    testMerge(originalCond = condition, expectedCond = condition)
   }
 
   test("inability to replace null in non-boolean values of CaseWhen") {
@@ -205,10 +251,15 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
       Literal(2) === nestedCaseWhen,
       TrueLiteral,
       FalseLiteral)
-    val branches = Seq((UnresolvedAttribute("i") > Literal(10)) -> branchValue)
-    val condition = CaseWhen(branches)
-    testFilter(originalCond = condition, expectedCond = condition)
-    testJoin(originalCond = condition, expectedCond = condition)
+    val condition = CaseWhen(Seq((UnresolvedAttribute("i") > Literal(10)) -> branchValue))
+    val expectedCond = CaseWhen(Seq(
+      (UnresolvedAttribute("i") > Literal(10), (Literal(2) === nestedCaseWhen) <=> TrueLiteral)),
+      FalseLiteral)
+    testFilter(originalCond = condition, expectedCond = expectedCond)
+    testJoin(originalCond = condition, expectedCond = expectedCond)
+    testDelete(originalCond = condition, expectedCond = expectedCond)
+    testUpdate(originalCond = condition, expectedCond = expectedCond)
+    testMerge(originalCond = condition, expectedCond = expectedCond)
   }
 
   test("inability to replace null in non-boolean branches of If inside another If") {
@@ -219,8 +270,15 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
         Literal(3)),
       TrueLiteral,
       FalseLiteral)
-    testFilter(originalCond = condition, expectedCond = condition)
-    testJoin(originalCond = condition, expectedCond = condition)
+    val expectedCond = (Literal(5) > If(
+      UnresolvedAttribute("i") === Literal(15),
+      Literal(null, IntegerType),
+      Literal(3))) <=> TrueLiteral
+    testFilter(originalCond = condition, expectedCond = expectedCond)
+    testJoin(originalCond = condition, expectedCond = expectedCond)
+    testDelete(originalCond = condition, expectedCond = expectedCond)
+    testUpdate(originalCond = condition, expectedCond = expectedCond)
+    testMerge(originalCond = condition, expectedCond = expectedCond)
   }
 
   test("replace null in If used as a join condition") {
@@ -340,6 +398,57 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
     testProjection(originalExpr = column, expectedExpr = column)
   }
 
+  test("replace None of elseValue inside CaseWhen if all branches are FalseLiteral") {
+    val allFalseBranches = Seq(
+      (UnresolvedAttribute("i") < Literal(10)) -> FalseLiteral,
+      (UnresolvedAttribute("i") > Literal(40)) -> FalseLiteral)
+    val allFalseCond = CaseWhen(allFalseBranches)
+
+    val nonAllFalseBranches = Seq(
+      (UnresolvedAttribute("i") < Literal(10)) -> FalseLiteral,
+      (UnresolvedAttribute("i") > Literal(40)) -> TrueLiteral)
+    val nonAllFalseCond = CaseWhen(nonAllFalseBranches, FalseLiteral)
+
+    testFilter(allFalseCond, FalseLiteral)
+    testJoin(allFalseCond, FalseLiteral)
+    testDelete(allFalseCond, FalseLiteral)
+    testUpdate(allFalseCond, FalseLiteral)
+    testMerge(allFalseCond, FalseLiteral)
+
+    testFilter(nonAllFalseCond, nonAllFalseCond)
+    testJoin(nonAllFalseCond, nonAllFalseCond)
+    testDelete(nonAllFalseCond, nonAllFalseCond)
+    testUpdate(nonAllFalseCond, nonAllFalseCond)
+    testMerge(nonAllFalseCond, nonAllFalseCond)
+  }
+
+  test("replace None of elseValue inside CaseWhen if all branches are null") {
+    val allNullBranches = Seq(
+      (UnresolvedAttribute("i") < Literal(10)) -> Literal.create(null, BooleanType),
+      (UnresolvedAttribute("i") > Literal(40)) -> Literal.create(null, BooleanType))
+    val allFalseCond = CaseWhen(allNullBranches)
+    testFilter(allFalseCond, FalseLiteral)
+    testJoin(allFalseCond, FalseLiteral)
+    testDelete(allFalseCond, FalseLiteral)
+    testUpdate(allFalseCond, FalseLiteral)
+    testMerge(allFalseCond, FalseLiteral)
+  }
+
+  test("SPARK-34692: Support Not(Int) and Not(InSet) propagate null") {
+    Seq(
+      Not(In("i".attr, Seq(Literal(1), Literal(2), Literal(null)))),
+      Not(In(Literal(null), Seq(Literal(1), Literal(2)))),
+      Not(InSet("i".attr, Set(1, 2, null))),
+      Not(InSet(Literal(null), Set(1, 2)))
+    ).foreach { condition =>
+      testFilter(condition, FalseLiteral)
+      testJoin(condition, FalseLiteral)
+      testDelete(condition, FalseLiteral)
+      testUpdate(condition, FalseLiteral)
+      testMerge(condition, FalseLiteral)
+    }
+  }
+
   private def testFilter(originalCond: Expression, expectedCond: Expression): Unit = {
     test((rel, exp) => rel.where(exp), originalCond, expectedCond)
   }
@@ -352,6 +461,52 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
     test((rel, exp) => rel.select(exp), originalExpr, expectedExpr)
   }
 
+  private def testDelete(originalCond: Expression, expectedCond: Expression): Unit = {
+    test((rel, expr) => DeleteFromTable(rel, Some(expr)), originalCond, expectedCond)
+  }
+
+  private def testUpdate(originalCond: Expression, expectedCond: Expression): Unit = {
+    test((rel, expr) => UpdateTable(rel, Seq.empty, Some(expr)), originalCond, expectedCond)
+  }
+
+  private def testMerge(originalCond: Expression, expectedCond: Expression): Unit = {
+    val func = (target: LogicalPlan, source: LogicalPlan, expr: Expression) => {
+      val matchedAssignments = Seq(
+        Assignment('i, 'i),
+        Assignment('b, 'b),
+        Assignment('a, 'a),
+        Assignment('m, 'm)
+      )
+      val notMatchedAssignments = Seq(
+        Assignment('i, 'd)
+      )
+      val matchedActions = UpdateAction(Some(expr), matchedAssignments) ::
+        DeleteAction(Some(expr)) :: Nil
+      val notMatchedActions = InsertAction(None, notMatchedAssignments) :: Nil
+      MergeIntoTable(target, source, mergeCondition = expr, matchedActions, notMatchedActions)
+    }
+    val originalPlan = func(testRelation, anotherTestRelation, originalCond).analyze
+    val optimizedPlan = Optimize.execute(originalPlan)
+    val expectedPlan = func(testRelation, anotherTestRelation, expectedCond).analyze
+    comparePlans(optimizedPlan, expectedPlan)
+
+    // Test with star actions
+    def mergePlanWithStar(expr: Expression): MergeIntoTable = {
+      val matchedActions = UpdateStarAction(Some(expr)) :: Nil
+      val notMatchedActions = InsertStarAction(Some(expr)) :: Nil
+      // Between source and target only one should have i and b as those are used for
+      // test expressions and both, source and target, having those columns is ambiguous  .
+      // However, the source must have all the columns present in target for star resolution.
+      val source = LocalRelation('i.int, 'b.boolean, 'a.array(IntegerType))
+      val target = LocalRelation('a.array(IntegerType))
+      MergeIntoTable(target, source, mergeCondition = expr, matchedActions, notMatchedActions)
+    }
+    val originalPlanWithStar = mergePlanWithStar(originalCond).analyze
+    val optimizedPlanWithStar = Optimize.execute(originalPlanWithStar)
+    val expectedPlanWithStar = mergePlanWithStar(expectedCond).analyze
+    comparePlans(optimizedPlanWithStar, expectedPlanWithStar)
+  }
+
   private def testHigherOrderFunc(
       argument: Expression,
       createExpr: (Expression, Expression) => Expression,
@@ -362,9 +517,9 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
     val lambda1 = LambdaFunction(
       function = If(cond, Literal(null, BooleanType), TrueLiteral),
       arguments = lambdaArgs)
-    // the optimized lambda body is: if(arg > 0, false, true)
+    // the optimized lambda body is: if(arg > 0, false, true) => !((arg > 0) <=> true)
     val lambda2 = LambdaFunction(
-      function = If(cond, FalseLiteral, TrueLiteral),
+      function = !(cond <=> TrueLiteral),
       arguments = lambdaArgs)
     testProjection(
       originalExpr = createExpr(argument, lambda1) as 'x,
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregatesSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregatesSuite.scala
index 8cb939e010c68..5d6abf516f288 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregatesSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregatesSuite.scala
@@ -16,23 +16,15 @@
  */
 package org.apache.spark.sql.catalyst.optimizer
 
-import org.apache.spark.sql.catalyst.analysis.{Analyzer, EmptyFunctionRegistry}
-import org.apache.spark.sql.catalyst.catalog.{InMemoryCatalog, SessionCatalog}
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.expressions.Literal
 import org.apache.spark.sql.catalyst.expressions.aggregate.CollectSet
 import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Expand, LocalRelation, LogicalPlan}
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.internal.SQLConf.{CASE_SENSITIVE, GROUP_BY_ORDINAL}
 import org.apache.spark.sql.types.{IntegerType, StringType}
 
 class RewriteDistinctAggregatesSuite extends PlanTest {
-  override val conf = new SQLConf().copy(CASE_SENSITIVE -> false, GROUP_BY_ORDINAL -> false)
-  val catalog = new SessionCatalog(new InMemoryCatalog, EmptyFunctionRegistry, conf)
-  val analyzer = new Analyzer(catalog, conf)
-
   val nullInt = Literal(null, IntegerType)
   val nullString = Literal(null, StringType)
   val testRelation = LocalRelation('a.string, 'b.string, 'c.string, 'd.string, 'e.int)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RewriteSubquerySuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RewriteSubquerySuite.scala
index 0608ded739374..4eff843c12be3 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RewriteSubquerySuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RewriteSubquerySuite.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.catalyst.optimizer
 
+import org.apache.spark.sql.catalyst.QueryPlanningTracker
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.expressions.{IsNull, ListQuery, Not}
@@ -67,4 +68,12 @@ class RewriteSubquerySuite extends PlanTest {
 
     comparePlans(optimized, correctAnswer)
   }
+
+  test("SPARK-34598: Filters without subquery must not be modified by RewritePredicateSubquery") {
+    val relation = LocalRelation('a.int, 'b.int, 'c.int, 'd.int)
+    val query = relation.where(('a === 1 || 'b === 2) && ('c === 3 && 'd === 4)).select('a)
+    val tracker = new QueryPlanningTracker
+    Optimize.executeAndTrack(query.analyze, tracker)
+    assert(tracker.rules(RewritePredicateSubquery.ruleName).numEffectiveInvocations == 0)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SetOperationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SetOperationSuite.scala
index ccc30b1d2f8ce..3fa7df3c94949 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SetOperationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SetOperationSuite.scala
@@ -146,6 +146,94 @@ class SetOperationSuite extends PlanTest {
     comparePlans(distinctUnionCorrectAnswer2, optimized2)
   }
 
+  test("SPARK-34283: Remove unnecessary deduplicate in multiple unions") {
+    val query1 = OneRowRelation()
+      .select(Literal(1).as('a))
+    val query2 = OneRowRelation()
+      .select(Literal(2).as('b))
+    val query3 = OneRowRelation()
+      .select(Literal(3).as('c))
+
+    // D - U - D - U - query1
+    //     |       |
+    //     query3  query2
+    val unionQuery1 = Deduplicate(query1.output, Union(
+      Deduplicate(query1.output, Union(query1, query2)), query3)).analyze
+    val optimized1 = Optimize.execute(unionQuery1)
+    val deduplicateUnionCorrectAnswer1 = Deduplicate(query1.output,
+      Union(query1 :: query2 :: query3 :: Nil))
+    comparePlans(deduplicateUnionCorrectAnswer1, optimized1)
+
+    //         query1
+    //         |
+    // D - U - U - query2
+    //     |
+    //     D - U - query2
+    //         |
+    //         query3
+    val unionQuery2 = Deduplicate(query1.output, Union(Union(query1, query2),
+      Deduplicate(query2.output, Union(query2, query3)))).analyze
+    val optimized2 = Optimize.execute(unionQuery2)
+    val deduplicateUnionCorrectAnswer2 =
+      Deduplicate(query1.output, Union(query1 :: query2 :: query2 :: query3 :: Nil))
+    comparePlans(deduplicateUnionCorrectAnswer2, optimized2)
+
+    // D  -  U  -  D  -  U  -  testRelation
+    //       |           |
+    //  testRelation  testRelation
+    // Union with the same value of 'byName' and 'allowMissingCol'
+    val unionQuery3 = Deduplicate(testRelation.output,
+      Union(Deduplicate(testRelation.output,
+        Union(testRelation :: testRelation :: Nil, true, false)) :: testRelation :: Nil,
+        true, false))
+    val optimized3 = Optimize.execute(unionQuery3)
+    val deduplicateUnionCorrectAnswer3 =
+      Deduplicate(testRelation.output,
+        Union(testRelation :: testRelation :: testRelation :: Nil, true, false))
+    comparePlans(deduplicateUnionCorrectAnswer3, optimized3, false)
+  }
+
+  test("SPARK-34283: Keep necessary deduplicate in multiple unions") {
+    val query1 = OneRowRelation()
+      .select(Literal(1).as('a))
+    val query2 = OneRowRelation()
+      .select(Literal(2).as('b))
+    val query3 = OneRowRelation()
+      .select(Literal(3).as('c))
+    val query4 = OneRowRelation()
+      .select(Literal(4).as('d))
+
+    // U - D - U - query1
+    // |       |
+    // query3  query2
+    val unionQuery1 = Union(Deduplicate(query1.output, Union(query1, query2)), query3).analyze
+    val optimized1 = Optimize.execute(unionQuery1)
+    comparePlans(unionQuery1, optimized1)
+
+    //         query1
+    //         |
+    // U - D - U - query2
+    // |
+    // D - U - query3
+    //     |
+    //     query4
+    val unionQuery2 = Union(Deduplicate(query1.output, Union(query1, query2)),
+      Deduplicate(query3.output, Union(query3, query4))).analyze
+    val optimized2 = Optimize.execute(unionQuery2)
+    comparePlans(unionQuery2, optimized2)
+
+    // D  -  U  -  D  -  U  -  testRelation
+    //       |           |
+    //  testRelation  testRelation
+    // Union with different value of 'byName' and 'allowMissingCol'
+    val unionQuery3 = Deduplicate(testRelation.output,
+      Union(Deduplicate(testRelation.output,
+        Union(testRelation :: testRelation :: Nil, true, false)) :: testRelation :: Nil,
+        true, true))
+    val optimized3 = Optimize.execute(unionQuery3)
+    comparePlans(unionQuery3, optimized3, false)
+  }
+
   test("EXCEPT ALL rewrite") {
     val input = Except(testRelation, testRelation2, isAll = true)
     val rewrittenPlan = RewriteExceptAll(input)
@@ -154,11 +242,11 @@ class SetOperationSuite extends PlanTest {
       .union(testRelation2.select(Literal(-1L).as("vcol"), 'd, 'e, 'f))
       .groupBy('a, 'b, 'c)('a, 'b, 'c, sum('vcol).as("sum"))
       .where(GreaterThan('sum, Literal(0L))).analyze
-    val multiplerAttr = planFragment.output.last
+    val multiplierAttr = planFragment.output.last
     val output = planFragment.output.dropRight(1)
     val expectedPlan = Project(output,
       Generate(
-        ReplicateRows(Seq(multiplerAttr) ++ output),
+        ReplicateRows(Seq(multiplierAttr) ++ output),
         Nil,
         false,
         None,
@@ -183,11 +271,11 @@ class SetOperationSuite extends PlanTest {
       .select('a, 'b, 'c,
         If(GreaterThan('vcol1_count, 'vcol2_count), 'vcol2_count, 'vcol1_count).as("min_count"))
       .analyze
-    val multiplerAttr = planFragment.output.last
+    val multiplierAttr = planFragment.output.last
     val output = planFragment.output.dropRight(1)
     val expectedPlan = Project(output,
       Generate(
-        ReplicateRows(Seq(multiplerAttr) ++ output),
+        ReplicateRows(Seq(multiplierAttr) ++ output),
         Nil,
         false,
         None,
@@ -223,4 +311,21 @@ class SetOperationSuite extends PlanTest {
     val unionCorrectAnswer = unionQuery.analyze
     comparePlans(unionOptimized, unionCorrectAnswer)
   }
+
+  test("CombineUnions only flatten the unions with same byName and allowMissingCol") {
+    val union1 = Union(testRelation :: testRelation :: Nil, true, false)
+    val union2 = Union(testRelation :: testRelation :: Nil, true, true)
+    val union3 = Union(testRelation :: testRelation2 :: Nil, false, false)
+
+    val union4 = Union(union1 :: union2 :: union3 :: Nil)
+    val unionOptimized1 = Optimize.execute(union4)
+    val unionCorrectAnswer1 = Union(union1 :: union2 :: testRelation :: testRelation2 :: Nil)
+    comparePlans(unionOptimized1, unionCorrectAnswer1, false)
+
+    val union5 = Union(union1 :: union1 :: Nil, true, false)
+    val unionOptimized2 = Optimize.execute(union5)
+    val unionCorrectAnswer2 =
+      Union(testRelation :: testRelation :: testRelation :: testRelation :: Nil, true, false)
+    comparePlans(unionOptimized2, unionCorrectAnswer2, false)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
index 7b3f5b084b015..c981cee55d0fa 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
@@ -17,10 +17,8 @@
 
 package org.apache.spark.sql.catalyst.optimizer
 
-import org.apache.spark.sql.catalyst.dsl._
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
-import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules.RuleExecutor
@@ -42,7 +40,8 @@ class SimplifyCastsSuite extends PlanTest {
 
   test("nullable element to non-nullable element array cast") {
     val input = LocalRelation('a.array(ArrayType(IntegerType, true)))
-    val plan = input.select('a.cast(ArrayType(IntegerType, false)).as("casted")).analyze
+    val attr = input.output.head
+    val plan = input.select(attr.cast(ArrayType(IntegerType, false)).as("casted"))
     val optimized = Optimize.execute(plan)
     // Though cast from `ArrayType(IntegerType, true)` to `ArrayType(IntegerType, false)` is not
     // allowed, here we just ensure that `SimplifyCasts` rule respect the plan.
@@ -60,8 +59,9 @@ class SimplifyCastsSuite extends PlanTest {
 
   test("nullable value map to non-nullable value map cast") {
     val input = LocalRelation('m.map(MapType(StringType, StringType, true)))
-    val plan = input.select('m.cast(MapType(StringType, StringType, false))
-      .as("casted")).analyze
+    val attr = input.output.head
+    val plan = input.select(attr.cast(MapType(StringType, StringType, false))
+      .as("casted"))
     val optimized = Optimize.execute(plan)
     // Though cast from `MapType(StringType, StringType, true)` to
     // `MapType(StringType, StringType, false)` is not allowed, here we just ensure that
@@ -69,4 +69,3 @@ class SimplifyCastsSuite extends PlanTest {
     comparePlans(optimized, plan, checkAnalysis = false)
   }
 }
-
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalSuite.scala
index 8ad7c12020b82..2a685bfeefcb2 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalSuite.scala
@@ -18,36 +18,35 @@
 package org.apache.spark.sql.catalyst.optimizer
 
 import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
+import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.Literal.{FalseLiteral, TrueLiteral}
 import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules._
-import org.apache.spark.sql.types.{IntegerType, NullType}
+import org.apache.spark.sql.types.{BooleanType, IntegerType}
 
 
-class SimplifyConditionalSuite extends PlanTest with PredicateHelper {
+class SimplifyConditionalSuite extends PlanTest with ExpressionEvalHelper with PredicateHelper {
 
   object Optimize extends RuleExecutor[LogicalPlan] {
     val batches = Batch("SimplifyConditionals", FixedPoint(50),
       BooleanSimplification, ConstantFolding, SimplifyConditionals) :: Nil
   }
 
+  private val relation = LocalRelation('a.int, 'b.int, 'c.boolean)
+
   protected def assertEquivalent(e1: Expression, e2: Expression): Unit = {
-    val correctAnswer = Project(Alias(e2, "out")() :: Nil, OneRowRelation()).analyze
-    val actual = Optimize.execute(Project(Alias(e1, "out")() :: Nil, OneRowRelation()).analyze)
+    val correctAnswer = Project(Alias(e2, "out")() :: Nil, relation).analyze
+    val actual = Optimize.execute(Project(Alias(e1, "out")() :: Nil, relation).analyze)
     comparePlans(actual, correctAnswer)
   }
 
   private val trueBranch = (TrueLiteral, Literal(5))
   private val normalBranch = (NonFoldableLiteral(true), Literal(10))
   private val unreachableBranch = (FalseLiteral, Literal(20))
-  private val nullBranch = (Literal.create(null, NullType), Literal(30))
-
-  val isNotNullCond = IsNotNull(UnresolvedAttribute(Seq("a")))
-  val isNullCond = IsNull(UnresolvedAttribute("b"))
-  val notCond = Not(UnresolvedAttribute("c"))
+  private val nullBranch = (Literal.create(null, BooleanType), Literal(30))
 
   test("simplify if") {
     assertEquivalent(
@@ -59,7 +58,7 @@ class SimplifyConditionalSuite extends PlanTest with PredicateHelper {
       Literal(20))
 
     assertEquivalent(
-      If(Literal.create(null, NullType), Literal(10), Literal(20)),
+      If(Literal.create(null, BooleanType), Literal(10), Literal(20)),
       Literal(20))
   }
 
@@ -127,9 +126,9 @@ class SimplifyConditionalSuite extends PlanTest with PredicateHelper {
   test("simplify CaseWhen if all the outputs are semantic equivalence") {
     // When the conditions in `CaseWhen` are all deterministic, `CaseWhen` can be removed.
     assertEquivalent(
-      CaseWhen((isNotNullCond, Subtract(Literal(3), Literal(2))) ::
-        (isNullCond, Literal(1)) ::
-        (notCond, Add(Literal(6), Literal(-5))) ::
+      CaseWhen(('a.isNotNull, Subtract(Literal(3), Literal(2))) ::
+        ('b.isNull, Literal(1)) ::
+        (!'c, Add(Literal(6), Literal(-5))) ::
         Nil,
         Add(Literal(2), Literal(-1))),
       Literal(1)
@@ -166,4 +165,120 @@ class SimplifyConditionalSuite extends PlanTest with PredicateHelper {
         Literal(1))
     )
   }
+
+  test("simplify if when one clause is null and another is boolean") {
+    val p = IsNull('a)
+    val nullLiteral = Literal(null, BooleanType)
+    assertEquivalent(If(p, nullLiteral, FalseLiteral), And(p, nullLiteral))
+    assertEquivalent(If(p, nullLiteral, TrueLiteral), Or(IsNotNull('a), nullLiteral))
+    assertEquivalent(If(p, FalseLiteral, nullLiteral), And(IsNotNull('a), nullLiteral))
+    assertEquivalent(If(p, TrueLiteral, nullLiteral), Or(p, nullLiteral))
+
+    // the rule should not apply to nullable predicate
+    Seq(TrueLiteral, FalseLiteral).foreach { b =>
+      assertEquivalent(If(GreaterThan('a, 42), nullLiteral, b),
+        If(GreaterThan('a, 42), nullLiteral, b))
+      assertEquivalent(If(GreaterThan('a, 42), b, nullLiteral),
+        If(GreaterThan('a, 42), b, nullLiteral))
+    }
+
+    // check evaluation also
+    Seq(TrueLiteral, FalseLiteral).foreach { b =>
+      checkEvaluation(If(b, nullLiteral, FalseLiteral), And(b, nullLiteral).eval(EmptyRow))
+      checkEvaluation(If(b, nullLiteral, TrueLiteral), Or(Not(b), nullLiteral).eval(EmptyRow))
+      checkEvaluation(If(b, FalseLiteral, nullLiteral), And(Not(b), nullLiteral).eval(EmptyRow))
+      checkEvaluation(If(b, TrueLiteral, nullLiteral), Or(b, nullLiteral).eval(EmptyRow))
+    }
+
+    // should have no effect on expressions with nullable if condition
+    assert((Factorial(5) > 100L).nullable)
+    Seq(TrueLiteral, FalseLiteral).foreach { b =>
+      checkEvaluation(If(Factorial(5) > 100L, nullLiteral, b),
+        If(Factorial(5) > 100L, nullLiteral, b).eval(EmptyRow))
+      checkEvaluation(If(Factorial(5) > 100L, b, nullLiteral),
+        If(Factorial(5) > 100L, b, nullLiteral).eval(EmptyRow))
+    }
+  }
+
+  test("SPARK-33845: remove unnecessary if when the outputs are boolean type") {
+    // verify the boolean equivalence of all transformations involved
+    val fields = Seq(
+      'cond.boolean.notNull,
+      'cond_nullable.boolean,
+      'a.boolean,
+      'b.boolean
+    )
+    val Seq(cond, cond_nullable, a, b) = fields.zipWithIndex.map { case (f, i) => f.at(i) }
+
+    val exprs = Seq(
+      // actual expressions of the transformations: original -> transformed
+      If(cond, true, false) -> cond,
+      If(cond, false, true) -> !cond,
+      If(cond_nullable, true, false) -> (cond_nullable <=> true),
+      If(cond_nullable, false, true) -> (!(cond_nullable <=> true)))
+
+    // check plans
+    for ((originalExpr, expectedExpr) <- exprs) {
+      assertEquivalent(originalExpr, expectedExpr)
+    }
+
+    // check evaluation
+    val binaryBooleanValues = Seq(true, false)
+    val ternaryBooleanValues = Seq(true, false, null)
+    for (condVal <- binaryBooleanValues;
+         condNullableVal <- ternaryBooleanValues;
+         aVal <- ternaryBooleanValues;
+         bVal <- ternaryBooleanValues;
+         (originalExpr, expectedExpr) <- exprs) {
+      val inputRow = create_row(condVal, condNullableVal, aVal, bVal)
+      val optimizedVal = evaluateWithoutCodegen(expectedExpr, inputRow)
+      checkEvaluation(originalExpr, optimizedVal, inputRow)
+    }
+  }
+
+  test("SPARK-33847: Remove the CaseWhen if elseValue is empty and other outputs are null") {
+    assertEquivalent(
+      CaseWhen((GreaterThan('a, 1), Literal.create(null, IntegerType)) :: Nil, None),
+      Literal.create(null, IntegerType))
+
+    assertEquivalent(
+      CaseWhen((GreaterThan(Rand(0), 0.5), Literal.create(null, IntegerType)) :: Nil, None),
+      CaseWhen((GreaterThan(Rand(0), 0.5), Literal.create(null, IntegerType)) :: Nil, None))
+  }
+
+  test("SPARK-33884: simplify CaseWhen clauses with (true and false) and (false and true)") {
+    // verify the boolean equivalence of all transformations involved
+    val fields = Seq(
+      'cond.boolean.notNull,
+      'cond_nullable.boolean,
+      'a.boolean,
+      'b.boolean
+    )
+    val Seq(cond, cond_nullable, a, b) = fields.zipWithIndex.map { case (f, i) => f.at(i) }
+
+    val exprs = Seq(
+      // actual expressions of the transformations: original -> transformed
+      CaseWhen(Seq((cond, TrueLiteral)), FalseLiteral) -> cond,
+      CaseWhen(Seq((cond, FalseLiteral)), TrueLiteral) -> !cond,
+      CaseWhen(Seq((cond_nullable, TrueLiteral)), FalseLiteral) -> (cond_nullable <=> true),
+      CaseWhen(Seq((cond_nullable, FalseLiteral)), TrueLiteral) -> (!(cond_nullable <=> true)))
+
+    // check plans
+    for ((originalExpr, expectedExpr) <- exprs) {
+      assertEquivalent(originalExpr, expectedExpr)
+    }
+
+    // check evaluation
+    val binaryBooleanValues = Seq(true, false)
+    val ternaryBooleanValues = Seq(true, false, null)
+    for (condVal <- binaryBooleanValues;
+         condNullableVal <- ternaryBooleanValues;
+         aVal <- ternaryBooleanValues;
+         bVal <- ternaryBooleanValues;
+         (originalExpr, expectedExpr) <- exprs) {
+      val inputRow = create_row(condVal, condNullableVal, aVal, bVal)
+      val optimizedVal = evaluateWithoutCodegen(expectedExpr, inputRow)
+      checkEvaluation(originalExpr, optimizedVal, inputRow)
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalsInPredicateSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalsInPredicateSuite.scala
new file mode 100644
index 0000000000000..04ebb4e63c675
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalsInPredicateSuite.scala
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions.{And, CaseWhen, Expression, If, IsNotNull, Literal, Or, Rand}
+import org.apache.spark.sql.catalyst.expressions.Literal.{FalseLiteral, TrueLiteral}
+import org.apache.spark.sql.catalyst.plans.{Inner, PlanTest}
+import org.apache.spark.sql.catalyst.plans.logical.{DeleteFromTable, LocalRelation, LogicalPlan, UpdateTable}
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types.{BooleanType, IntegerType}
+
+class SimplifyConditionalsInPredicateSuite extends PlanTest {
+
+  object Optimize extends RuleExecutor[LogicalPlan] {
+    val batches =
+      Batch("SimplifyConditionalsInPredicate", FixedPoint(10),
+        NullPropagation,
+        ConstantFolding,
+        BooleanSimplification,
+        SimplifyConditionals,
+        SimplifyConditionalsInPredicate) :: Nil
+  }
+
+  private val testRelation =
+    LocalRelation('i.int, 'b.boolean, 'a.array(IntegerType), 'm.map(IntegerType, IntegerType))
+  private val anotherTestRelation = LocalRelation('d.int)
+
+  test("IF(cond, trueVal, false) => AND(cond, trueVal)") {
+    val originalCond = If(
+      UnresolvedAttribute("i") > Literal(10),
+      UnresolvedAttribute("b"),
+      FalseLiteral)
+    val expectedCond = And(
+      UnresolvedAttribute("i") > Literal(10),
+      UnresolvedAttribute("b"))
+    testFilter(originalCond, expectedCond = expectedCond)
+    testJoin(originalCond, expectedCond = expectedCond)
+    testDelete(originalCond, expectedCond = expectedCond)
+    testUpdate(originalCond, expectedCond = expectedCond)
+    testProjection(originalCond, expectedExpr = originalCond)
+  }
+
+  test("IF(cond, trueVal, true) => OR(NOT(cond), trueVal)") {
+    val originalCond = If(
+      UnresolvedAttribute("i") > Literal(10),
+      UnresolvedAttribute("b"),
+      TrueLiteral)
+    val expectedCond = Or(
+      UnresolvedAttribute("i") <= Literal(10),
+      UnresolvedAttribute("b"))
+    testFilter(originalCond, expectedCond = expectedCond)
+    testJoin(originalCond, expectedCond = expectedCond)
+    testDelete(originalCond, expectedCond = expectedCond)
+    testUpdate(originalCond, expectedCond = expectedCond)
+    testProjection(originalCond, expectedExpr = originalCond)
+  }
+
+  test("IF(cond, false, falseVal) => AND(NOT(cond), elseVal)") {
+    val originalCond = If(
+      UnresolvedAttribute("i") > Literal(10),
+      FalseLiteral,
+      UnresolvedAttribute("b"))
+    val expectedCond = And(
+      UnresolvedAttribute("i") <= Literal(10),
+      UnresolvedAttribute("b"))
+    testFilter(originalCond, expectedCond = expectedCond)
+    testJoin(originalCond, expectedCond = expectedCond)
+    testDelete(originalCond, expectedCond = expectedCond)
+    testUpdate(originalCond, expectedCond = expectedCond)
+    testProjection(originalCond, expectedExpr = originalCond)
+  }
+
+  test("IF(cond, true, falseVal) => OR(cond, elseVal)") {
+    val originalCond = If(
+      UnresolvedAttribute("i") > Literal(10),
+      TrueLiteral,
+      UnresolvedAttribute("b"))
+    val expectedCond = Or(
+      UnresolvedAttribute("i") > Literal(10),
+      UnresolvedAttribute("b"))
+    testFilter(originalCond, expectedCond = expectedCond)
+    testJoin(originalCond, expectedCond = expectedCond)
+    testDelete(originalCond, expectedCond = expectedCond)
+    testUpdate(originalCond, expectedCond = expectedCond)
+    testProjection(originalCond, expectedExpr = originalCond)
+  }
+
+  test("CASE WHEN cond THEN trueVal ELSE false END => AND(cond, trueVal)") {
+    Seq(Some(FalseLiteral), None, Some(Literal(null, BooleanType))).foreach { elseExp =>
+      val originalCond = CaseWhen(
+        Seq((UnresolvedAttribute("i") > Literal(10), UnresolvedAttribute("b"))),
+        elseExp)
+      val expectedCond = And(
+        UnresolvedAttribute("i") > Literal(10),
+        UnresolvedAttribute("b"))
+      testFilter(originalCond, expectedCond = expectedCond)
+      testJoin(originalCond, expectedCond = expectedCond)
+      testDelete(originalCond, expectedCond = expectedCond)
+      testUpdate(originalCond, expectedCond = expectedCond)
+      testProjection(originalCond, expectedExpr = originalCond)
+    }
+  }
+
+  test("CASE WHEN cond THEN trueVal ELSE true END => OR(NOT(cond), trueVal)") {
+    val originalCond = CaseWhen(
+      Seq((UnresolvedAttribute("i") > Literal(10), UnresolvedAttribute("b"))),
+      TrueLiteral)
+    val expectedCond = Or(
+      UnresolvedAttribute("i") <= Literal(10),
+      UnresolvedAttribute("b"))
+    testFilter(originalCond, expectedCond = expectedCond)
+    testJoin(originalCond, expectedCond = expectedCond)
+    testDelete(originalCond, expectedCond = expectedCond)
+    testUpdate(originalCond, expectedCond = expectedCond)
+    testProjection(originalCond, expectedExpr = originalCond)
+  }
+
+  test("CASE WHEN cond THEN false ELSE elseVal END => AND(NOT(cond), elseVal)") {
+    val originalCond = CaseWhen(
+      Seq((UnresolvedAttribute("i") > Literal(10), FalseLiteral)),
+      UnresolvedAttribute("b"))
+    val expectedCond = And(
+      UnresolvedAttribute("i") <= Literal(10),
+      UnresolvedAttribute("b"))
+    testFilter(originalCond, expectedCond = expectedCond)
+    testJoin(originalCond, expectedCond = expectedCond)
+    testDelete(originalCond, expectedCond = expectedCond)
+    testUpdate(originalCond, expectedCond = expectedCond)
+    testProjection(originalCond, expectedExpr = originalCond)
+  }
+
+  test("CASE WHEN cond THEN false END => false") {
+    val originalCond = CaseWhen(
+      Seq((UnresolvedAttribute("i") > Literal(10), FalseLiteral)))
+    testFilter(originalCond, expectedCond = FalseLiteral)
+    testJoin(originalCond, expectedCond = FalseLiteral)
+    testDelete(originalCond, expectedCond = FalseLiteral)
+    testUpdate(originalCond, expectedCond = FalseLiteral)
+    testProjection(originalCond, expectedExpr = originalCond)
+  }
+
+  test("CASE WHEN non-deterministic-cond THEN false END") {
+    val originalCond =
+      CaseWhen(Seq((UnresolvedAttribute("i") > Rand(0), FalseLiteral)))
+    val expectedCond = And(UnresolvedAttribute("i") > Rand(0), FalseLiteral)
+    // nondeterministic expressions are only allowed in Project, Filter, Aggregate or Window,
+    testFilter(originalCond, expectedCond = FalseLiteral)
+    testProjection(originalCond, expectedExpr = originalCond)
+  }
+
+  test("CASE WHEN cond THEN true ELSE elseVal END  => OR(cond, elseVal)") {
+    val originalCond = CaseWhen(
+      Seq((UnresolvedAttribute("i") > Literal(10), TrueLiteral)),
+      UnresolvedAttribute("b"))
+    val expectedCond = Or(
+      UnresolvedAttribute("i") > Literal(10),
+      UnresolvedAttribute("b"))
+    testFilter(originalCond, expectedCond = expectedCond)
+    testJoin(originalCond, expectedCond = expectedCond)
+    testDelete(originalCond, expectedCond = expectedCond)
+    testUpdate(originalCond, expectedCond = expectedCond)
+    testProjection(originalCond, expectedExpr = originalCond)
+  }
+
+  test("CASE WHEN cond THEN true END => cond") {
+    val originalCond = CaseWhen(
+      Seq((UnresolvedAttribute("i") > Literal(10), TrueLiteral)))
+    val expectedCond = UnresolvedAttribute("i") > Literal(10)
+    testFilter(originalCond, expectedCond = expectedCond)
+    testJoin(originalCond, expectedCond = expectedCond)
+    testDelete(originalCond, expectedCond = expectedCond)
+    testUpdate(originalCond, expectedCond = expectedCond)
+    testProjection(originalCond, expectedExpr = originalCond)
+  }
+
+  test("Simplify conditional in conditions of CaseWhen inside another CaseWhen") {
+    val nestedCaseWhen = CaseWhen(
+      Seq((UnresolvedAttribute("i") > Literal(10)) -> UnresolvedAttribute("b")),
+      FalseLiteral)
+    val originalCond = CaseWhen(Seq(IsNotNull(nestedCaseWhen) -> FalseLiteral))
+    val expectedCond = FalseLiteral
+
+    testFilter(originalCond, expectedCond = expectedCond)
+    testJoin(originalCond, expectedCond = expectedCond)
+    testDelete(originalCond, expectedCond = expectedCond)
+    testUpdate(originalCond, expectedCond = expectedCond)
+    testProjection(originalCond, expectedExpr = originalCond)
+  }
+
+  test("Not expected type - SimplifyConditionalsInPredicate") {
+    val e = intercept[AnalysisException] {
+      testFilter(originalCond = Literal(null, IntegerType), expectedCond = FalseLiteral)
+    }.getMessage
+    assert(e.contains("'CAST(NULL AS INT)' of type int is not a boolean"))
+  }
+
+  private def testFilter(originalCond: Expression, expectedCond: Expression): Unit = {
+    test((rel, exp) => rel.where(exp), originalCond, expectedCond)
+  }
+
+  private def testJoin(originalCond: Expression, expectedCond: Expression): Unit = {
+    test((rel, exp) => rel.join(anotherTestRelation, Inner, Some(exp)), originalCond, expectedCond)
+  }
+
+  private def testProjection(originalExpr: Expression, expectedExpr: Expression): Unit = {
+    test((rel, exp) => rel.select(exp), originalExpr, expectedExpr)
+  }
+
+  private def testDelete(originalCond: Expression, expectedCond: Expression): Unit = {
+    test((rel, expr) => DeleteFromTable(rel, Some(expr)), originalCond, expectedCond)
+  }
+
+  private def testUpdate(originalCond: Expression, expectedCond: Expression): Unit = {
+    test((rel, expr) => UpdateTable(rel, Seq.empty, Some(expr)), originalCond, expectedCond)
+  }
+
+  private def test(
+      func: (LogicalPlan, Expression) => LogicalPlan,
+      originalExpr: Expression,
+      expectedExpr: Expression): Unit = {
+
+    val originalPlan = func(testRelation, originalExpr).analyze
+    val optimizedPlan = Optimize.execute(originalPlan)
+    val expectedPlan = func(testRelation, expectedExpr).analyze
+    comparePlans(optimizedPlan, expectedPlan)
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala
new file mode 100644
index 0000000000000..0afb166b80ca5
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans.DslLogicalPlan
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.expressions.IntegralLiteralTestUtils._
+import org.apache.spark.sql.catalyst.expressions.aggregate.First
+import org.apache.spark.sql.catalyst.optimizer.UnwrapCastInBinaryComparison._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types._
+
+class UnwrapCastInBinaryComparisonSuite extends PlanTest with ExpressionEvalHelper {
+
+  object Optimize extends RuleExecutor[LogicalPlan] {
+    val batches: List[Batch] =
+      Batch("Unwrap casts in binary comparison", FixedPoint(10),
+        NullPropagation, UnwrapCastInBinaryComparison) :: Nil
+  }
+
+  val testRelation: LocalRelation = LocalRelation('a.short, 'b.float, 'c.decimal(5, 2))
+  val f: BoundReference = 'a.short.canBeNull.at(0)
+  val f2: BoundReference = 'b.float.canBeNull.at(1)
+  val f3: BoundReference = 'c.decimal(5, 2).canBeNull.at(2)
+
+  test("unwrap casts when literal == max") {
+    val v = Short.MaxValue
+    assertEquivalent(castInt(f) > v.toInt, falseIfNotNull(f))
+    assertEquivalent(castInt(f) >= v.toInt, f === v)
+    assertEquivalent(castInt(f) === v.toInt, f === v)
+    assertEquivalent(castInt(f) <=> v.toInt, f <=> v)
+    assertEquivalent(castInt(f) <= v.toInt, trueIfNotNull(f))
+    assertEquivalent(castInt(f) < v.toInt, f =!= v)
+
+    val d = Float.NaN
+    assertEquivalent(castDouble(f2) > d.toDouble, falseIfNotNull(f2))
+    assertEquivalent(castDouble(f2) >= d.toDouble, f2 === d)
+    assertEquivalent(castDouble(f2) === d.toDouble, f2 === d)
+    assertEquivalent(castDouble(f2) <=> d.toDouble, f2 <=> d)
+    assertEquivalent(castDouble(f2) <= d.toDouble, trueIfNotNull(f2))
+    assertEquivalent(castDouble(f2) < d.toDouble, f2 =!= d)
+  }
+
+  test("unwrap casts when literal > max") {
+    val v: Int = positiveInt
+    assertEquivalent(castInt(f) > v, falseIfNotNull(f))
+    assertEquivalent(castInt(f) >= v, falseIfNotNull(f))
+    assertEquivalent(castInt(f) === v, falseIfNotNull(f))
+    assertEquivalent(castInt(f) <=> v, false)
+    assertEquivalent(castInt(f) <= v, trueIfNotNull(f))
+    assertEquivalent(castInt(f) < v, trueIfNotNull(f))
+  }
+
+  test("unwrap casts when literal == min") {
+    val v = Short.MinValue
+    assertEquivalent(castInt(f) > v.toInt, f =!= v)
+    assertEquivalent(castInt(f) >= v.toInt, trueIfNotNull(f))
+    assertEquivalent(castInt(f) === v.toInt, f === v)
+    assertEquivalent(castInt(f) <=> v.toInt, f <=> v)
+    assertEquivalent(castInt(f) <= v.toInt, f === v)
+    assertEquivalent(castInt(f) < v.toInt, falseIfNotNull(f))
+
+    val d = Float.NegativeInfinity
+    assertEquivalent(castDouble(f2) > d.toDouble, f2 =!= d)
+    assertEquivalent(castDouble(f2) >= d.toDouble, trueIfNotNull(f2))
+    assertEquivalent(castDouble(f2) === d.toDouble, f2 === d)
+    assertEquivalent(castDouble(f2) <=> d.toDouble, f2 <=> d)
+    assertEquivalent(castDouble(f2) <= d.toDouble, f2 === d)
+    assertEquivalent(castDouble(f2) < d.toDouble, falseIfNotNull(f2))
+
+    // Double.NegativeInfinity == Float.NegativeInfinity
+    val d2 = Double.NegativeInfinity
+    assertEquivalent(castDouble(f2) > d2, f2 =!= d)
+    assertEquivalent(castDouble(f2) >= d2, trueIfNotNull(f2))
+    assertEquivalent(castDouble(f2) === d2, f2 === d)
+    assertEquivalent(castDouble(f2) <=> d2, f2 <=> d)
+    assertEquivalent(castDouble(f2) <= d2, f2 === d)
+    assertEquivalent(castDouble(f2) < d2, falseIfNotNull(f2))
+  }
+
+  test("unwrap casts when literal < min") {
+    val v: Int = negativeInt
+    assertEquivalent(castInt(f) > v, trueIfNotNull(f))
+    assertEquivalent(castInt(f) >= v, trueIfNotNull(f))
+    assertEquivalent(castInt(f) === v, falseIfNotNull(f))
+    assertEquivalent(castInt(f) <=> v, false)
+    assertEquivalent(castInt(f) <= v, falseIfNotNull(f))
+    assertEquivalent(castInt(f) < v, falseIfNotNull(f))
+  }
+
+  test("unwrap casts when literal is within range (min, max) or fromType has no range") {
+    Seq(300, 500, 32766, -6000, -32767).foreach(v => {
+      assertEquivalent(castInt(f) > v, f > v.toShort)
+      assertEquivalent(castInt(f) >= v, f >= v.toShort)
+      assertEquivalent(castInt(f) === v, f === v.toShort)
+      assertEquivalent(castInt(f) <=> v, f <=> v.toShort)
+      assertEquivalent(castInt(f) <= v, f <= v.toShort)
+      assertEquivalent(castInt(f) < v, f < v.toShort)
+    })
+
+    Seq(3.14.toFloat.toDouble, -1000.0.toFloat.toDouble,
+      20.0.toFloat.toDouble, -2.414.toFloat.toDouble,
+      Float.MinValue.toDouble, Float.MaxValue.toDouble, Float.PositiveInfinity.toDouble
+    ).foreach(v => {
+      assertEquivalent(castDouble(f2) > v, f2 > v.toFloat)
+      assertEquivalent(castDouble(f2) >= v, f2 >= v.toFloat)
+      assertEquivalent(castDouble(f2) === v, f2 === v.toFloat)
+      assertEquivalent(castDouble(f2) <=> v, f2 <=> v.toFloat)
+      assertEquivalent(castDouble(f2) <= v, f2 <= v.toFloat)
+      assertEquivalent(castDouble(f2) < v, f2 < v.toFloat)
+    })
+
+    Seq(decimal2(100.20), decimal2(-200.50)).foreach(v => {
+      assertEquivalent(castDecimal2(f3) > v, f3 > decimal(v))
+      assertEquivalent(castDecimal2(f3) >= v, f3 >= decimal(v))
+      assertEquivalent(castDecimal2(f3) === v, f3 === decimal(v))
+      assertEquivalent(castDecimal2(f3) <=> v, f3 <=> decimal(v))
+      assertEquivalent(castDecimal2(f3) <= v, f3 <= decimal(v))
+      assertEquivalent(castDecimal2(f3) < v, f3 < decimal(v))
+    })
+  }
+
+  test("unwrap cast when literal is within range (min, max) AND has round up or down") {
+    // Cases for rounding down
+    var doubleValue = 100.6
+    assertEquivalent(castDouble(f) > doubleValue, f > doubleValue.toShort)
+    assertEquivalent(castDouble(f) >= doubleValue, f > doubleValue.toShort)
+    assertEquivalent(castDouble(f) === doubleValue, falseIfNotNull(f))
+    assertEquivalent(castDouble(f) <=> doubleValue, false)
+    assertEquivalent(castDouble(f) <= doubleValue, f <= doubleValue.toShort)
+    assertEquivalent(castDouble(f) < doubleValue, f <= doubleValue.toShort)
+
+    // Cases for rounding up: 3.14 will be rounded to 3.14000010... after casting to float
+    doubleValue = 3.14
+    assertEquivalent(castDouble(f2) > doubleValue, f2 >= doubleValue.toFloat)
+    assertEquivalent(castDouble(f2) >= doubleValue, f2 >= doubleValue.toFloat)
+    assertEquivalent(castDouble(f2) === doubleValue, falseIfNotNull(f2))
+    assertEquivalent(castDouble(f2) <=> doubleValue, false)
+    assertEquivalent(castDouble(f2) <= doubleValue, f2 < doubleValue.toFloat)
+    assertEquivalent(castDouble(f2) < doubleValue, f2 < doubleValue.toFloat)
+
+    // Another case: 400.5678 is rounded up to 400.57
+    val decimalValue = decimal2(400.5678)
+    assertEquivalent(castDecimal2(f3) > decimalValue, f3 >= decimal(decimalValue))
+    assertEquivalent(castDecimal2(f3) >= decimalValue, f3 >= decimal(decimalValue))
+    assertEquivalent(castDecimal2(f3) === decimalValue, falseIfNotNull(f3))
+    assertEquivalent(castDecimal2(f3) <=> decimalValue, false)
+    assertEquivalent(castDecimal2(f3) <= decimalValue, f3 < decimal(decimalValue))
+    assertEquivalent(castDecimal2(f3) < decimalValue, f3 < decimal(decimalValue))
+  }
+
+  test("unwrap casts when cast is on rhs") {
+    val v = Short.MaxValue
+    assertEquivalent(Literal(v.toInt) < castInt(f), falseIfNotNull(f))
+    assertEquivalent(Literal(v.toInt) <= castInt(f), Literal(v) === f)
+    assertEquivalent(Literal(v.toInt) === castInt(f), Literal(v) === f)
+    assertEquivalent(Literal(v.toInt) <=> castInt(f), Literal(v) <=> f)
+    assertEquivalent(Literal(v.toInt) >= castInt(f), trueIfNotNull(f))
+    assertEquivalent(Literal(v.toInt) > castInt(f), f =!= v)
+
+    assertEquivalent(Literal(30) <= castInt(f), Literal(30.toShort, ShortType) <= f)
+  }
+
+ test("unwrap cast should skip when expression is non-deterministic or foldable") {
+    Seq(positiveInt, negativeInt).foreach(v => {
+      val e = Cast(First(f, ignoreNulls = true), IntegerType) <=> v
+      assertEquivalent(e, e, evaluate = false)
+      val e2 = Cast(Literal(30.toShort), IntegerType) >= v
+      assertEquivalent(e2, e2, evaluate = false)
+    })
+  }
+
+  test("unwrap casts when literal is null") {
+    val intLit = Literal.create(null, IntegerType)
+    val nullLit = Literal.create(null, BooleanType)
+    assertEquivalent(castInt(f) > intLit, nullLit)
+    assertEquivalent(castInt(f) >= intLit, nullLit)
+    assertEquivalent(castInt(f) === intLit, nullLit)
+    assertEquivalent(castInt(f) <=> intLit, IsNull(castInt(f)))
+    assertEquivalent(castInt(f) <= intLit, nullLit)
+    assertEquivalent(castInt(f) < intLit, nullLit)
+  }
+
+  test("unwrap casts should skip if downcast failed") {
+    val decimalValue = decimal2(123456.1234)
+    assertEquivalent(castDecimal2(f3) === decimalValue, castDecimal2(f3) === decimalValue)
+  }
+
+  test("unwrap cast should skip if cannot coerce type") {
+    assertEquivalent(Cast(f, ByteType) > 100.toByte, Cast(f, ByteType) > 100.toByte)
+  }
+
+  test("test getRange()") {
+    assert(Some((Byte.MinValue, Byte.MaxValue)) === getRange(ByteType))
+    assert(Some((Short.MinValue, Short.MaxValue)) === getRange(ShortType))
+    assert(Some((Int.MinValue, Int.MaxValue)) === getRange(IntegerType))
+    assert(Some((Long.MinValue, Long.MaxValue)) === getRange(LongType))
+
+    val floatRange = getRange(FloatType)
+    assert(floatRange.isDefined)
+    val (floatMin, floatMax) = floatRange.get
+    assert(floatMin.isInstanceOf[Float])
+    assert(floatMin.asInstanceOf[Float].isNegInfinity)
+    assert(floatMax.isInstanceOf[Float])
+    assert(floatMax.asInstanceOf[Float].isNaN)
+
+    val doubleRange = getRange(DoubleType)
+    assert(doubleRange.isDefined)
+    val (doubleMin, doubleMax) = doubleRange.get
+    assert(doubleMin.isInstanceOf[Double])
+    assert(doubleMin.asInstanceOf[Double].isNegInfinity)
+    assert(doubleMax.isInstanceOf[Double])
+    assert(doubleMax.asInstanceOf[Double].isNaN)
+
+    assert(getRange(DecimalType(5, 2)).isEmpty)
+  }
+
+  private def castInt(e: Expression): Expression = Cast(e, IntegerType)
+  private def castDouble(e: Expression): Expression = Cast(e, DoubleType)
+  private def castDecimal2(e: Expression): Expression = Cast(e, DecimalType(10, 4))
+
+  private def decimal(v: Decimal): Decimal = Decimal(v.toJavaBigDecimal, 5, 2)
+  private def decimal2(v: BigDecimal): Decimal = Decimal(v, 10, 4)
+
+  private def assertEquivalent(e1: Expression, e2: Expression, evaluate: Boolean = true): Unit = {
+    val plan = testRelation.where(e1).analyze
+    val actual = Optimize.execute(plan)
+    val expected = testRelation.where(e2).analyze
+    comparePlans(actual, expected)
+
+    if (evaluate) {
+      Seq(
+        (100.toShort, 3.14.toFloat, decimal2(100)),
+        (-300.toShort, 3.1415927.toFloat, decimal2(-3000.50)),
+        (null, Float.NaN, decimal2(12345.6789)),
+        (null, null, null),
+        (Short.MaxValue, Float.PositiveInfinity, decimal2(Short.MaxValue)),
+        (Short.MinValue, Float.NegativeInfinity, decimal2(Short.MinValue)),
+        (0.toShort, Float.MaxValue, decimal2(0)),
+        (0.toShort, Float.MinValue, decimal2(0.01))
+      ).foreach(v => {
+        val row = create_row(v._1, v._2, v._3)
+        checkEvaluation(e1, e2.eval(row), row)
+      })
+    }
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala
index d55746002783a..00a4212f661d9 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala
@@ -22,7 +22,7 @@ import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
 import org.apache.spark.sql.catalyst.plans.PlanTest
-import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan, Range}
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan}
 import org.apache.spark.sql.catalyst.rules.RuleExecutor
 import org.apache.spark.sql.catalyst.util.GenericArrayData
 import org.apache.spark.sql.types._
@@ -36,15 +36,18 @@ class ComplexTypesSuite extends PlanTest with ExpressionEvalHelper {
 
   object Optimizer extends RuleExecutor[LogicalPlan] {
     val batches =
+      Batch("Finish Analysis", Once,
+        PullOutGroupingExpressions) ::
       Batch("collapse projections", FixedPoint(10),
-          CollapseProject) ::
+        CollapseProject) ::
       Batch("Constant Folding", FixedPoint(10),
-          NullPropagation,
-          ConstantFolding,
-          BooleanSimplification,
-          SimplifyConditionals,
-          SimplifyBinaryComparison,
-          SimplifyExtractValueOps) :: Nil
+         NullPropagation,
+         ConstantFolding,
+         BooleanSimplification,
+         SimplifyConditionals,
+         SimplifyBinaryComparison,
+         OptimizeUpdateFields,
+         SimplifyExtractValueOps) :: Nil
   }
 
   private val idAtt = ('id).long.notNull
@@ -56,7 +59,7 @@ class ComplexTypesSuite extends PlanTest with ExpressionEvalHelper {
   private def checkRule(originalQuery: LogicalPlan, correctAnswer: LogicalPlan) = {
     val optimized = Optimizer.execute(originalQuery.analyze)
     assert(optimized.resolved, "optimized plans must be still resolvable")
-    comparePlans(optimized, correctAnswer.analyze)
+    comparePlans(optimized, PullOutGroupingExpressions(correctAnswer.analyze))
   }
 
   test("explicit get from namedStruct") {
@@ -404,14 +407,6 @@ class ComplexTypesSuite extends PlanTest with ExpressionEvalHelper {
     val arrayAggRel = relation.groupBy(
       CreateArray(Seq('nullable_id)))(GetArrayItem(CreateArray(Seq('nullable_id)), 0))
     checkRule(arrayAggRel, arrayAggRel)
-
-    // This could be done if we had a more complex rule that checks that
-    // the CreateMap does not come from key.
-    val originalQuery = relation
-      .groupBy('id)(
-        GetMapValue(CreateMap(Seq('id, 'id + 1L)), 0L) as "a"
-      )
-    checkRule(originalQuery, originalQuery)
   }
 
   test("SPARK-23500: namedStruct and getField in the same Project #1") {
@@ -452,4 +447,353 @@ class ComplexTypesSuite extends PlanTest with ExpressionEvalHelper {
     checkEvaluation(GetMapValue(mb0, Literal(Array[Byte](2, 1), BinaryType)), "2")
     checkEvaluation(GetMapValue(mb0, Literal(Array[Byte](3, 4))), null)
   }
+
+  private val structAttr = 'struct1.struct('a.int, 'b.int).withNullability(false)
+  private val testStructRelation = LocalRelation(structAttr)
+
+  private val nullableStructAttr = 'struct1.struct('a.int, 'b.int)
+  private val testNullableStructRelation = LocalRelation(nullableStructAttr)
+
+  test("simplify GetStructField on basic UpdateFields") {
+    def check(fieldOps: Seq[StructFieldsOperation], ordinal: Int, expected: Expression): Unit = {
+      def query(relation: LocalRelation): LogicalPlan =
+        relation.select(GetStructField(UpdateFields('struct1, fieldOps), ordinal).as("res"))
+
+      checkRule(
+        query(testStructRelation),
+        testStructRelation.select(expected.as("res")))
+
+      checkRule(
+        query(testNullableStructRelation),
+        testNullableStructRelation.select((expected match {
+          case expr: GetStructField => expr
+          case expr => If(IsNull('struct1), Literal(null, expr.dataType), expr)
+        }).as("res")))
+    }
+
+    // scalastyle:off line.size.limit
+
+    // add attribute, extract an attribute from the original struct
+    check(WithField("c", Literal(3)) :: Nil, 0, GetStructField('struct1, 0))
+    check(WithField("c", Literal(3)) :: Nil, 1, GetStructField('struct1, 1))
+    // add attribute, extract added attribute
+    check(WithField("c", Literal(3)) :: Nil, 2, Literal(3))
+
+    // replace attribute, extract an attribute from the original struct
+    check(WithField("a", Literal(1)) :: Nil, 1, GetStructField('struct1, 1))
+    check(WithField("b", Literal(2)) :: Nil, 0, GetStructField('struct1, 0))
+    // replace attribute, extract replaced attribute
+    check(WithField("a", Literal(1)) :: Nil, 0, Literal(1))
+    check(WithField("b", Literal(2)) :: Nil, 1, Literal(2))
+
+    // add multiple attributes, extract an attribute from the original struct
+    check(WithField("c", Literal(3)) :: WithField("c", Literal(4)) :: Nil, 0, GetStructField('struct1, 0))
+    check(WithField("c", Literal(3)) :: WithField("d", Literal(4)) :: Nil, 0, GetStructField('struct1, 0))
+    check(WithField("c", Literal(3)) :: WithField("c", Literal(4)) :: Nil, 1, GetStructField('struct1, 1))
+    check(WithField("c", Literal(3)) :: WithField("d", Literal(4)) :: Nil, 1, GetStructField('struct1, 1))
+    // add multiple attributes, extract newly added attribute
+    check(WithField("c", Literal(3)) :: WithField("c", Literal(4)) :: Nil, 2, Literal(4))
+    check(WithField("c", Literal(4)) :: WithField("c", Literal(3)) :: Nil, 2, Literal(3))
+    check(WithField("c", Literal(3)) :: WithField("d", Literal(4)) :: Nil, 2, Literal(3))
+    check(WithField("c", Literal(3)) :: WithField("d", Literal(4)) :: Nil, 3, Literal(4))
+    check(WithField("d", Literal(4)) :: WithField("c", Literal(3)) :: Nil, 2, Literal(4))
+    check(WithField("d", Literal(4)) :: WithField("c", Literal(3)) :: Nil, 3, Literal(3))
+
+    // drop attribute, extract an attribute from the original struct
+    check(DropField("b") :: Nil, 0, GetStructField('struct1, 0))
+    check(DropField("a") :: Nil, 0, GetStructField('struct1, 1))
+
+    // drop attribute, add attribute, extract an attribute from the original struct
+    check(DropField("b") :: WithField("c", Literal(3)) :: Nil, 0, GetStructField('struct1, 0))
+    check(DropField("a") :: WithField("c", Literal(3)) :: Nil, 0, GetStructField('struct1, 1))
+    // drop attribute, add attribute, extract added attribute
+    check(DropField("b") :: WithField("c", Literal(3)) :: Nil, 1, Literal(3))
+    check(DropField("a") :: WithField("c", Literal(3)) :: Nil, 1, Literal(3))
+
+    // add attribute, drop attribute, extract an attribute from the original struct
+    check(WithField("c", Literal(3)) :: DropField("a") :: Nil, 0, GetStructField('struct1, 1))
+    check(WithField("c", Literal(3)) :: DropField("b") :: Nil, 0, GetStructField('struct1, 0))
+    // add attribute, drop attribute, extract added attribute
+    check(WithField("c", Literal(3)) :: DropField("a") :: Nil, 1, Literal(3))
+    check(WithField("c", Literal(3)) :: DropField("b") :: Nil, 1, Literal(3))
+
+    // replace attribute, drop same attribute, extract an attribute from the original struct
+    check(WithField("b", Literal(3)) :: DropField("b") :: Nil, 0, GetStructField('struct1, 0))
+    check(WithField("a", Literal(3)) :: DropField("a") :: Nil, 0, GetStructField('struct1, 1))
+
+    // add attribute, drop same attribute, extract an attribute from the original struct
+    check(WithField("c", Literal(3)) :: DropField("c") :: Nil, 0, GetStructField('struct1, 0))
+    check(WithField("c", Literal(3)) :: DropField("c") :: Nil, 1, GetStructField('struct1, 1))
+
+    // replace attribute, drop another attribute, extract added attribute
+    check(WithField("b", Literal(3)) :: DropField("a") :: Nil, 0, Literal(3))
+    check(WithField("a", Literal(3)) :: DropField("b") :: Nil, 0, Literal(3))
+
+    // drop attribute, add same attribute, extract attribute from the original struct
+    check(DropField("b") :: WithField("b", Literal(3)) :: Nil, 0, GetStructField('struct1, 0))
+    check(DropField("a") :: WithField("a", Literal(3)) :: Nil, 0, GetStructField('struct1, 1))
+    // drop attribute, add same attribute, extract added attribute
+    check(DropField("b") :: WithField("b", Literal(3)) :: Nil, 1, Literal(3))
+    check(DropField("a") :: WithField("a", Literal(3)) :: Nil, 1, Literal(3))
+
+    // drop non-existent attribute, add same attribute, extract attribute from the original struct
+    check(DropField("c") :: WithField("c", Literal(3)) :: Nil, 0, GetStructField('struct1, 0))
+    check(DropField("c") :: WithField("c", Literal(3)) :: Nil, 1, GetStructField('struct1, 1))
+    // drop non-existent attribute, add same attribute, extract added attribute
+    check(DropField("c") :: WithField("c", Literal(3)) :: Nil, 2, Literal(3))
+
+    // scalastyle:on  line.size.limit
+  }
+
+  test("simplify GetStructField that is extracting a field nested inside a struct") {
+    val struct2 = 'struct2.struct('b.int)
+    val testStructRelation = LocalRelation(structAttr, struct2)
+    val testNullableStructRelation = LocalRelation(nullableStructAttr, struct2)
+
+    // if the field being extracted is from the same struct that UpdateFields is modifying,
+    // we can just return GetStructField in both the non-nullable and nullable struct scenario
+
+    def addFieldFromSameStructAndThenExtractIt(relation: LocalRelation): LogicalPlan =
+      relation.select(GetStructField(
+        UpdateFields('struct1, WithField("b", GetStructField('struct1, 0)) :: Nil), 1).as("res"))
+
+    checkRule(
+      addFieldFromSameStructAndThenExtractIt(testStructRelation),
+      testStructRelation.select(GetStructField('struct1, 0).as("res")))
+
+    checkRule(
+      addFieldFromSameStructAndThenExtractIt(testNullableStructRelation),
+      testNullableStructRelation.select(GetStructField('struct1, 0).as("res")))
+
+    // if the field being extracted is from a different struct than the one UpdateFields is
+    // modifying, we must return GetStructField wrapped in If(IsNull(struct), null, GetStructField)
+    // in the nullable struct scenario
+
+    def addFieldFromAnotherStructAndThenExtractIt(relation: LocalRelation): LogicalPlan =
+      relation.select(GetStructField(
+        UpdateFields('struct1, WithField("b", GetStructField('struct2, 0)) :: Nil), 1).as("res"))
+
+    checkRule(
+      addFieldFromAnotherStructAndThenExtractIt(testStructRelation),
+      testStructRelation.select(GetStructField('struct2, 0).as("res")))
+
+    checkRule(
+      addFieldFromAnotherStructAndThenExtractIt(testNullableStructRelation),
+      testNullableStructRelation.select(
+        If(IsNull('struct1), Literal(null, IntegerType), GetStructField('struct2, 0)).as("res")))
+  }
+
+  test("simplify GetStructField on nested UpdateFields") {
+    def query(relation: LocalRelation, ordinal: Int): LogicalPlan = {
+      val nestedUpdateFields =
+        UpdateFields(
+          UpdateFields(
+            UpdateFields(
+              UpdateFields(
+                'struct1,
+                WithField("c", Literal(1)) :: Nil),
+              WithField("d", Literal(2)) :: Nil),
+            WithField("e", Literal(3)) :: Nil),
+          WithField("f", Literal(4)) :: Nil)
+
+      relation.select(GetStructField(nestedUpdateFields, ordinal) as "res")
+    }
+
+    // extract newly added field
+
+    checkRule(
+      query(testStructRelation, 5),
+      testStructRelation.select(Literal(4) as "res"))
+
+    checkRule(
+      query(testNullableStructRelation, 5),
+      testNullableStructRelation.select(
+        If(IsNull('struct1), Literal(null, IntegerType), Literal(4)) as "res"))
+
+    // extract field from original struct
+
+    checkRule(
+      query(testStructRelation, 0),
+      testStructRelation.select(GetStructField('struct1, 0) as "res"))
+
+    checkRule(
+      query(testNullableStructRelation, 0),
+      testNullableStructRelation.select(GetStructField('struct1, 0) as "res"))
+  }
+
+  test("simplify multiple GetStructField on the same UpdateFields") {
+    def query(relation: LocalRelation): LogicalPlan = relation
+      .select(UpdateFields('struct1, WithField("b", Literal(2)) :: Nil) as "struct2")
+      .select(
+        GetStructField('struct2, 0, Some("a")) as "struct1A",
+        GetStructField('struct2, 1, Some("b")) as "struct1B")
+
+    checkRule(
+      query(testStructRelation),
+      testStructRelation.select(
+        GetStructField('struct1, 0) as "struct1A",
+        Literal(2) as "struct1B"))
+
+    checkRule(
+      query(testNullableStructRelation),
+      testNullableStructRelation.select(
+        GetStructField('struct1, 0) as "struct1A",
+        If(IsNull('struct1), Literal(null, IntegerType), Literal(2)) as "struct1B"))
+  }
+
+  test("simplify multiple GetStructField on different UpdateFields") {
+    def query(relation: LocalRelation): LogicalPlan = relation
+      .select(
+        UpdateFields('struct1, WithField("b", Literal(2)) :: Nil) as "struct2",
+        UpdateFields('struct1, WithField("b", Literal(3)) :: Nil) as "struct3")
+      .select(
+        GetStructField('struct2, 0, Some("a")) as "struct2A",
+        GetStructField('struct2, 1, Some("b")) as "struct2B",
+        GetStructField('struct3, 0, Some("a")) as "struct3A",
+        GetStructField('struct3, 1, Some("b")) as "struct3B")
+
+    checkRule(
+      query(testStructRelation),
+      testStructRelation
+        .select(
+          GetStructField('struct1, 0) as "struct2A",
+          Literal(2) as "struct2B",
+          GetStructField('struct1, 0) as "struct3A",
+          Literal(3) as "struct3B"))
+
+    checkRule(
+      query(testNullableStructRelation),
+      testNullableStructRelation
+        .select(
+          GetStructField('struct1, 0) as "struct2A",
+          If(IsNull('struct1), Literal(null, IntegerType), Literal(2)) as "struct2B",
+          GetStructField('struct1, 0) as "struct3A",
+          If(IsNull('struct1), Literal(null, IntegerType), Literal(3)) as "struct3B"))
+  }
+
+  test("simplify add multiple nested fields to non-nullable struct") {
+    // this scenario is possible if users add multiple nested columns to a non-nullable struct
+    // using the Column.withField API in a non-performant way
+    val structLevel2 = LocalRelation(
+      'a1.struct(
+        'a2.struct('a3.int.notNull)).notNull)
+
+    val query = {
+      val addB3toA1A2 = UpdateFields('a1, Seq(WithField("a2",
+        UpdateFields(GetStructField('a1, 0), Seq(WithField("b3", Literal(2)))))))
+
+      structLevel2.select(
+        UpdateFields(
+          addB3toA1A2,
+          Seq(WithField("a2", UpdateFields(
+            GetStructField(addB3toA1A2, 0), Seq(WithField("c3", Literal(3))))))).as("a1"))
+    }
+
+    val expected = structLevel2.select(
+      UpdateFields('a1, Seq(
+        // scalastyle:off line.size.limit
+        WithField("a2", UpdateFields(GetStructField('a1, 0), WithField("b3", 2) :: WithField("c3", 3) :: Nil))
+        // scalastyle:on line.size.limit
+      )).as("a1"))
+
+    checkRule(query, expected)
+  }
+
+  test("simplify add multiple nested fields to nullable struct") {
+    // this scenario is possible if users add multiple nested columns to a nullable struct
+    // using the Column.withField API in a non-performant way
+    val structLevel2 = LocalRelation(
+      'a1.struct(
+        'a2.struct('a3.int.notNull)))
+
+    val query = {
+      val addB3toA1A2 = UpdateFields('a1, Seq(WithField("a2",
+        UpdateFields(GetStructField('a1, 0), Seq(WithField("b3", Literal(2)))))))
+
+      structLevel2.select(
+        UpdateFields(
+          addB3toA1A2,
+          Seq(WithField("a2", UpdateFields(
+            GetStructField(addB3toA1A2, 0), Seq(WithField("c3", Literal(3))))))).as("a1"))
+    }
+
+    val expected = {
+      val repeatedExpr = UpdateFields(GetStructField('a1, 0), WithField("b3", Literal(2)) :: Nil)
+      val repeatedExprDataType = StructType(Seq(
+        StructField("a3", IntegerType, nullable = false),
+        StructField("b3", IntegerType, nullable = false)))
+
+      structLevel2.select(
+        UpdateFields('a1, Seq(
+          WithField("a2", UpdateFields(
+            If(IsNull('a1), Literal(null, repeatedExprDataType), repeatedExpr),
+            WithField("c3", Literal(3)) :: Nil))
+        )).as("a1"))
+    }
+
+    checkRule(query, expected)
+  }
+
+  test("simplify drop multiple nested fields in non-nullable struct") {
+    // this scenario is possible if users drop multiple nested columns in a non-nullable struct
+    // using the Column.dropFields API in a non-performant way
+    val structLevel2 = LocalRelation(
+      'a1.struct(
+        'a2.struct('a3.int.notNull, 'b3.int.notNull, 'c3.int.notNull).notNull
+      ).notNull)
+
+    val query = {
+      val dropA1A2B = UpdateFields('a1, Seq(WithField("a2", UpdateFields(
+        GetStructField('a1, 0), Seq(DropField("b3"))))))
+
+      structLevel2.select(
+        UpdateFields(
+          dropA1A2B,
+          Seq(WithField("a2", UpdateFields(
+            GetStructField(dropA1A2B, 0), Seq(DropField("c3")))))).as("a1"))
+    }
+
+    val expected = structLevel2.select(
+      UpdateFields('a1, Seq(
+        WithField("a2", UpdateFields(GetStructField('a1, 0), Seq(DropField("b3"), DropField("c3"))))
+      )).as("a1"))
+
+    checkRule(query, expected)
+  }
+
+  test("simplify drop multiple nested fields in nullable struct") {
+    // this scenario is possible if users drop multiple nested columns in a nullable struct
+    // using the Column.dropFields API in a non-performant way
+    val structLevel2 = LocalRelation(
+      'a1.struct(
+        'a2.struct('a3.int.notNull, 'b3.int.notNull, 'c3.int.notNull)
+      ))
+
+    val query = {
+      val dropA1A2B = UpdateFields('a1, Seq(WithField("a2", UpdateFields(
+        GetStructField('a1, 0), Seq(DropField("b3"))))))
+
+      structLevel2.select(
+        UpdateFields(
+          dropA1A2B,
+          Seq(WithField("a2", UpdateFields(
+            GetStructField(dropA1A2B, 0), Seq(DropField("c3")))))).as("a1"))
+    }
+
+    val expected = {
+      val repeatedExpr = UpdateFields(GetStructField('a1, 0), DropField("b3") :: Nil)
+      val repeatedExprDataType = StructType(Seq(
+        StructField("a3", IntegerType, nullable = false),
+        StructField("c3", IntegerType, nullable = false)))
+
+      structLevel2.select(
+        UpdateFields('a1, Seq(
+          WithField("a2", UpdateFields(
+            If(IsNull('a1), Literal(null, repeatedExprDataType), repeatedExpr),
+            DropField("c3") :: Nil))
+        )).as("a1"))
+    }
+
+    checkRule(query, expected)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/joinReorder/JoinReorderPlanTestBase.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/joinReorder/JoinReorderPlanTestBase.scala
new file mode 100644
index 0000000000000..e79c973e750d3
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/joinReorder/JoinReorderPlanTestBase.scala
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer.joinReorder
+
+import org.apache.spark.sql.catalyst.dsl.plans.DslLogicalPlan
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.optimizer.EliminateResolvedHint
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical.{Join, LogicalPlan, Project}
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.catalyst.util.sideBySide
+
+trait JoinReorderPlanTestBase extends PlanTest {
+
+  def outputsOf(plans: LogicalPlan*): Seq[Attribute] = {
+    plans.map(_.output).reduce(_ ++ _)
+  }
+
+  def assertEqualJoinPlans(
+      optimizer: RuleExecutor[LogicalPlan],
+      originalPlan: LogicalPlan,
+      groundTruthBestPlan: LogicalPlan): Unit = {
+    val analyzed = originalPlan.analyze
+    val optimized = optimizer.execute(analyzed)
+    val expected = EliminateResolvedHint.apply(groundTruthBestPlan.analyze)
+
+    // if this fails, the expected plan itself is incorrect
+    assert(equivalentOutput(analyzed, expected))
+    assert(equivalentOutput(analyzed, optimized))
+
+    compareJoinOrder(optimized, expected)
+  }
+
+  private def equivalentOutput(plan1: LogicalPlan, plan2: LogicalPlan): Boolean = {
+    normalizeExprIds(plan1).output == normalizeExprIds(plan2).output
+  }
+
+  /** Fails the test if the join order in the two plans do not match */
+  private def compareJoinOrder(plan1: LogicalPlan, plan2: LogicalPlan): Unit = {
+    val normalized1 = normalizePlan(normalizeExprIds(plan1))
+    val normalized2 = normalizePlan(normalizeExprIds(plan2))
+    if (!sameJoinPlan(normalized1, normalized2)) {
+      fail(
+        s"""
+           |== FAIL: Plans do not match ===
+           |${sideBySide(
+          rewriteNameFromAttrNullability(normalized1).treeString,
+          rewriteNameFromAttrNullability(normalized2).treeString).mkString("\n")}
+         """.stripMargin)
+    }
+  }
+
+  /** Consider symmetry for joins when comparing plans. */
+  private def sameJoinPlan(plan1: LogicalPlan, plan2: LogicalPlan): Boolean = {
+    (plan1, plan2) match {
+      case (j1: Join, j2: Join) =>
+        (sameJoinPlan(j1.left, j2.left) && sameJoinPlan(j1.right, j2.right)
+          && j1.hint.leftHint == j2.hint.leftHint && j1.hint.rightHint == j2.hint.rightHint) ||
+          (sameJoinPlan(j1.left, j2.right) && sameJoinPlan(j1.right, j2.left)
+            && j1.hint.leftHint == j2.hint.rightHint && j1.hint.rightHint == j2.hint.leftHint)
+      case (p1: Project, p2: Project) =>
+        p1.projectList == p2.projectList && sameJoinPlan(p1.child, p2.child)
+      case _ =>
+        plan1 == plan2
+    }
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/joinReorder/JoinReorderSuite.scala
similarity index 87%
rename from sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala
rename to sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/joinReorder/JoinReorderSuite.scala
index f7755000b3f39..f5e2ff4f1b3e8 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/joinReorder/JoinReorderSuite.scala
@@ -15,19 +15,21 @@
  * limitations under the License.
  */
 
-package org.apache.spark.sql.catalyst.optimizer
+package org.apache.spark.sql.catalyst.optimizer.joinReorder
 
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeMap}
-import org.apache.spark.sql.catalyst.plans.{Cross, Inner, PlanTest}
+import org.apache.spark.sql.catalyst.optimizer._
+import org.apache.spark.sql.catalyst.optimizer.JoinReorderDP.JoinPlan
+import org.apache.spark.sql.catalyst.plans.{Cross, Inner}
 import org.apache.spark.sql.catalyst.plans.logical.{ColumnStat, LocalRelation, LogicalPlan}
 import org.apache.spark.sql.catalyst.rules.RuleExecutor
 import org.apache.spark.sql.catalyst.statsEstimation.{StatsEstimationTestBase, StatsTestPlan}
 import org.apache.spark.sql.internal.SQLConf.{CBO_ENABLED, JOIN_REORDER_ENABLED}
 
 
-class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
+class JoinReorderSuite extends JoinReorderPlanTestBase with StatsEstimationTestBase {
 
   object Optimize extends RuleExecutor[LogicalPlan] {
     val batches =
@@ -44,12 +46,6 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
         CostBasedJoinReorder) :: Nil
   }
 
-  object ResolveHints extends RuleExecutor[LogicalPlan] {
-    val batches =
-      Batch("Resolve Hints", Once,
-        EliminateResolvedHint) :: Nil
-  }
-
   var originalConfCBOEnabled = false
   var originalConfJoinReorderEnabled = false
 
@@ -135,7 +131,7 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
         .join(t2, Inner, Some(nameToAttr("t1.k-1-2") === nameToAttr("t2.k-1-5")))
         .select(outputsOf(t1, t2, t3): _*)
 
-    assertEqualPlans(originalPlan, bestPlan)
+    assertEqualJoinPlans(Optimize, originalPlan, bestPlan)
   }
 
   test("put unjoinable item at the end and reorder 3 joinable tables") {
@@ -152,7 +148,7 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
         .join(t4)
         .select(outputsOf(t1, t2, t4, t3): _*)
 
-    assertEqualPlans(originalPlan, bestPlan)
+    assertEqualJoinPlans(Optimize, originalPlan, bestPlan)
   }
 
   test("reorder 3 tables with pure-attribute project") {
@@ -167,7 +163,7 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
         .join(t2, Inner, Some(nameToAttr("t1.k-1-2") === nameToAttr("t2.k-1-5")))
         .select(nameToAttr("t1.v-1-10"))
 
-    assertEqualPlans(originalPlan, bestPlan)
+    assertEqualJoinPlans(Optimize, originalPlan, bestPlan)
   }
 
   test("reorder 3 tables - one of the leaf items is a project") {
@@ -184,7 +180,7 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
           Some(nameToAttr("t1.k-1-2") === nameToAttr("t5.k-1-5")))
         .select(nameToAttr("t1.v-1-10"))
 
-    assertEqualPlans(originalPlan, bestPlan)
+    assertEqualJoinPlans(Optimize, originalPlan, bestPlan)
   }
 
   test("don't reorder if project contains non-attribute") {
@@ -194,7 +190,7 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
         .join(t3, Inner, Some(nameToAttr("t1.v-1-10") === nameToAttr("t3.v-1-100")))
         .select("key".attr)
 
-    assertEqualPlans(originalPlan, originalPlan)
+    assertEqualJoinPlans(Optimize, originalPlan, originalPlan)
   }
 
   test("reorder 4 tables (bushy tree)") {
@@ -215,7 +211,7 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
           Inner, Some(nameToAttr("t1.k-1-2") === nameToAttr("t4.k-1-2")))
         .select(outputsOf(t1, t4, t2, t3): _*)
 
-    assertEqualPlans(originalPlan, bestPlan)
+    assertEqualJoinPlans(Optimize, originalPlan, bestPlan)
   }
 
   test("keep the order of attributes in the final output") {
@@ -227,7 +223,7 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
           .join(t2, Inner, Some(nameToAttr("t1.k-1-2") === nameToAttr("t2.k-1-5")))
           .select(expectedOrder: _*)
       // The plan should not change after optimization
-      assertEqualPlans(expectedPlan, expectedPlan)
+      assertEqualJoinPlans(Optimize, expectedPlan, expectedPlan)
     }
   }
 
@@ -245,7 +241,7 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
           .join(tab2, Cross, Some('b === 'i))
           .select(outputsOf(tab1, tab2, tab3): _*)
 
-    assertEqualPlans(original, expected)
+    assertEqualJoinPlans(Optimize, original, expected)
   }
 
   test("reorder recursively") {
@@ -289,7 +285,7 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
       .union(t4.select(nameToAttr("t4.v-1-10")))
       .join(t5, Inner, Some(nameToAttr("t1.v-1-10") === nameToAttr("t5.v-1-5")))
 
-    assertEqualPlans(originalPlan, bestPlan)
+    assertEqualJoinPlans(Optimize, originalPlan, bestPlan)
   }
 
   test("don't reorder if hints present") {
@@ -302,16 +298,16 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
           Inner,
           Some(nameToAttr("t1.k-1-2") === nameToAttr("t4.k-1-2")))
 
-    assertEqualPlans(originalPlan, originalPlan)
+    assertEqualJoinPlans(Optimize, originalPlan, originalPlan)
 
     val originalPlan2 =
       t1.join(t2, Inner, Some(nameToAttr("t1.k-1-2") === nameToAttr("t2.k-1-5")))
         .hint("broadcast")
-        .join(t4, Inner, Some(nameToAttr("t4.v-1-10") === nameToAttr("t3.v-1-100")))
+        .join(t4, Inner, Some(nameToAttr("t1.k-1-2") === nameToAttr("t4.k-1-2")))
         .hint("broadcast")
-        .join(t3, Inner, Some(nameToAttr("t1.k-1-2") === nameToAttr("t4.k-1-2")))
+        .join(t3, Inner, Some(nameToAttr("t4.v-1-10") === nameToAttr("t3.v-1-100")))
 
-    assertEqualPlans(originalPlan2, originalPlan2)
+    assertEqualJoinPlans(Optimize, originalPlan2, originalPlan2)
 
     val originalPlan3 =
       t1.join(t2, Inner, Some(nameToAttr("t1.k-1-2") === nameToAttr("t2.k-1-5")))
@@ -319,7 +315,7 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
         .join(t3, Inner, Some(nameToAttr("t1.v-1-10") === nameToAttr("t3.v-1-100")))
         .join(t5, Inner, Some(nameToAttr("t5.v-1-5") === nameToAttr("t3.v-1-100")))
 
-    assertEqualPlans(originalPlan3, originalPlan3)
+    assertEqualJoinPlans(Optimize, originalPlan3, originalPlan3)
   }
 
   test("reorder below and above the hint node") {
@@ -335,7 +331,7 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
       .select(outputsOf(t1, t2, t3): _*)
       .hint("broadcast").join(t4)
 
-    assertEqualPlans(originalPlan, bestPlan)
+    assertEqualJoinPlans(Optimize, originalPlan, bestPlan)
 
     val originalPlan2 =
       t1.join(t2).join(t3)
@@ -349,7 +345,7 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
         .select(outputsOf(t1, t2, t3): _*)
         .join(t4.hint("broadcast"))
 
-    assertEqualPlans(originalPlan2, bestPlan2)
+    assertEqualJoinPlans(Optimize, originalPlan2, bestPlan2)
 
     val originalPlan3 =
       t1.join(t2, Inner, Some(nameToAttr("t1.k-1-2") === nameToAttr("t2.k-1-5")))
@@ -366,23 +362,26 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
         .join(t4, Inner, Some(nameToAttr("t4.v-1-10") === nameToAttr("t3.v-1-100")))
         .join(t5, Inner, Some(nameToAttr("t5.v-1-5") === nameToAttr("t3.v-1-100")))
 
-    assertEqualPlans(originalPlan3, bestPlan3)
+    assertEqualJoinPlans(Optimize, originalPlan3, bestPlan3)
   }
 
-  private def assertEqualPlans(
-      originalPlan: LogicalPlan,
-      groundTruthBestPlan: LogicalPlan): Unit = {
-    val analyzed = originalPlan.analyze
-    val optimized = Optimize.execute(analyzed)
-    val expected = ResolveHints.execute(groundTruthBestPlan.analyze)
+  test("SPARK-33935: betterThan should be consistent") {
+    val plan1 = JoinPlan(null, null, null, Cost(300, 80))
+    val plan2 = JoinPlan(null, null, null, Cost(500, 30))
 
-    assert(analyzed.sameOutput(expected)) // if this fails, the expected plan itself is incorrect
-    assert(analyzed.sameOutput(optimized))
+    assert(!plan1.betterThan(plan1, conf))
+    assert(!plan2.betterThan(plan2, conf))
 
-    compareJoinOrder(optimized, expected)
+    assert(plan1.betterThan(plan2, conf))
+    assert(!plan2.betterThan(plan1, conf))
   }
 
-  private def outputsOf(plans: LogicalPlan*): Seq[Attribute] = {
-    plans.map(_.output).reduce(_ ++ _)
+  test("SPARK-34354: join reorder with self-join") {
+    val plan = t2.join(t1, Inner, Some(nameToAttr("t1.k-1-2") === nameToAttr("t2.k-1-5")))
+      .select(nameToAttr("t1.v-1-10"))
+      .join(t2, Inner, Some(nameToAttr("t1.v-1-10") === nameToAttr("t2.k-1-5")))
+
+    // this can fail before the fix
+    Optimize.execute(plan.analyze)
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinCostBasedReorderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/joinReorder/StarJoinCostBasedReorderSuite.scala
similarity index 91%
rename from sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinCostBasedReorderSuite.scala
rename to sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/joinReorder/StarJoinCostBasedReorderSuite.scala
index 5b8e59ae7cb31..c4a660532b207 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinCostBasedReorderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/joinReorder/StarJoinCostBasedReorderSuite.scala
@@ -15,19 +15,20 @@
  * limitations under the License.
  */
 
-package org.apache.spark.sql.catalyst.optimizer
+package org.apache.spark.sql.catalyst.optimizer.joinReorder
 
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeMap}
-import org.apache.spark.sql.catalyst.plans.{Inner, PlanTest}
+import org.apache.spark.sql.catalyst.optimizer._
+import org.apache.spark.sql.catalyst.plans.Inner
 import org.apache.spark.sql.catalyst.plans.logical.{ColumnStat, LogicalPlan}
 import org.apache.spark.sql.catalyst.rules.RuleExecutor
 import org.apache.spark.sql.catalyst.statsEstimation.{StatsEstimationTestBase, StatsTestPlan}
 import org.apache.spark.sql.internal.SQLConf._
 
 
-class StarJoinCostBasedReorderSuite extends PlanTest with StatsEstimationTestBase {
+class StarJoinCostBasedReorderSuite extends JoinReorderPlanTestBase with StatsEstimationTestBase {
 
   object Optimize extends RuleExecutor[LogicalPlan] {
     val batches =
@@ -220,7 +221,7 @@ class StarJoinCostBasedReorderSuite extends PlanTest with StatsEstimationTestBas
         .join(t1, Inner, Some(nameToAttr("f1_c1") === nameToAttr("t1_c1")))
         .select(outputsOf(f1, t1, t2, d1, d2): _*)
 
-    assertEqualPlans(query, expected)
+    assertEqualJoinPlans(Optimize, query, expected)
   }
 
   test("Test 2: Star with a linear branch") {
@@ -259,7 +260,7 @@ class StarJoinCostBasedReorderSuite extends PlanTest with StatsEstimationTestBas
         .join(t1, Inner, Some(nameToAttr("t1_c1") === nameToAttr("f1_c1")))
         .select(outputsOf(d1, t1, t2, f1, d2, t3): _*)
 
-    assertEqualPlans(query, expected)
+    assertEqualJoinPlans(Optimize, query, expected)
   }
 
   test("Test 3: Star with derived branches") {
@@ -293,15 +294,16 @@ class StarJoinCostBasedReorderSuite extends PlanTest with StatsEstimationTestBas
           (nameToAttr("f1_fk2") === nameToAttr("d2_pk")))
 
     val expected =
-      f1.join(d2, Inner, Some(nameToAttr("f1_fk2") === nameToAttr("d2_pk")))
+      f1
+        .join(d2, Inner, Some(nameToAttr("f1_fk2") === nameToAttr("d2_pk")))
         .join(d1, Inner, Some(nameToAttr("f1_fk1") === nameToAttr("d1_pk")))
         .join(t3.join(t4, Inner, Some(nameToAttr("t3_c1") === nameToAttr("t4_c1"))), Inner,
-          Some(nameToAttr("t3_c1") === nameToAttr("t4_c1")))
+          Some(nameToAttr("d1_c2") === nameToAttr("t4_c1")))
         .join(t1.join(t2, Inner, Some(nameToAttr("t1_c1") === nameToAttr("t2_c1"))), Inner,
           Some(nameToAttr("t1_c2") === nameToAttr("t4_c2")))
         .select(outputsOf(d1, t1, t2, t3, t4, f1, d2): _*)
 
-    assertEqualPlans(query, expected)
+    assertEqualJoinPlans(Optimize, query, expected)
   }
 
   test("Test 4: Star with several branches") {
@@ -328,8 +330,10 @@ class StarJoinCostBasedReorderSuite extends PlanTest with StatsEstimationTestBas
     // level 9: {d1 t5 t3 t6 t2 t4 d3 f1 t1 d2 }
     //
     // Number of generated plans: 46 (vs. 82)
+    // TODO(SPARK-32687): find a way to make optimization result of `CostBasedJoinReorder`
+    //  deterministic even if the input order is different.
     val query =
-      d1.join(t3).join(t4).join(f1).join(d2).join(t5).join(t6).join(d3).join(t1).join(t2)
+      d1.join(t3).join(t4).join(f1).join(d3).join(d2).join(t5).join(t6).join(t1).join(t2)
         .where((nameToAttr("d1_c2") === nameToAttr("t3_c1")) &&
           (nameToAttr("t3_c2") === nameToAttr("t4_c2")) &&
           (nameToAttr("d1_pk") === nameToAttr("f1_fk1")) &&
@@ -342,17 +346,17 @@ class StarJoinCostBasedReorderSuite extends PlanTest with StatsEstimationTestBas
 
     val expected =
       f1.join(d3, Inner, Some(nameToAttr("f1_fk3") === nameToAttr("d3_pk")))
-        .join(d1, Inner, Some(nameToAttr("f1_fk1") === nameToAttr("d1_pk")))
         .join(d2, Inner, Some(nameToAttr("f1_fk2") === nameToAttr("d2_pk")))
+        .join(d1, Inner, Some(nameToAttr("f1_fk1") === nameToAttr("d1_pk")))
         .join(t4.join(t3, Inner, Some(nameToAttr("t3_c2") === nameToAttr("t4_c2"))), Inner,
           Some(nameToAttr("d1_c2") === nameToAttr("t3_c1")))
         .join(t2.join(t1, Inner, Some(nameToAttr("t1_c2") === nameToAttr("t2_c2"))), Inner,
           Some(nameToAttr("d3_c2") === nameToAttr("t1_c1")))
         .join(t5.join(t6, Inner, Some(nameToAttr("t5_c2") === nameToAttr("t6_c2"))), Inner,
           Some(nameToAttr("d2_c2") === nameToAttr("t5_c1")))
-        .select(outputsOf(d1, t3, t4, f1, d2, t5, t6, d3, t1, t2): _*)
+        .select(outputsOf(d1, t3, t4, f1, d3, d2, t5, t6, t1, t2): _*)
 
-    assertEqualPlans(query, expected)
+    assertEqualJoinPlans(Optimize, query, expected)
   }
 
   test("Test 5: RI star only") {
@@ -381,7 +385,7 @@ class StarJoinCostBasedReorderSuite extends PlanTest with StatsEstimationTestBas
         .join(d1, Inner, Some(nameToAttr("f1_fk1") === nameToAttr("d1_pk")))
         .select(outputsOf(d1, d2, f1, d3): _*)
 
-    assertEqualPlans(query, expected)
+    assertEqualJoinPlans(Optimize, query, expected)
   }
 
   test("Test 6: No RI star") {
@@ -407,25 +411,6 @@ class StarJoinCostBasedReorderSuite extends PlanTest with StatsEstimationTestBas
         .join(t1, Inner, Some(nameToAttr("f1_fk1") === nameToAttr("t1_c1")))
         .select(outputsOf(t1, f1, t2, t3): _*)
 
-    assertEqualPlans(query, expected)
-  }
-
-  private def assertEqualPlans( plan1: LogicalPlan, plan2: LogicalPlan): Unit = {
-    val analyzed = plan1.analyze
-    val optimized = Optimize.execute(analyzed)
-    val expected = plan2.analyze
-
-    assert(equivalentOutput(analyzed, expected)) // if this fails, the expected itself is incorrect
-    assert(equivalentOutput(analyzed, optimized))
-
-    compareJoinOrder(optimized, expected)
-  }
-
-  private def outputsOf(plans: LogicalPlan*): Seq[Attribute] = {
-    plans.map(_.output).reduce(_ ++ _)
-  }
-
-  private def equivalentOutput(plan1: LogicalPlan, plan2: LogicalPlan): Boolean = {
-    normalizeExprIds(plan1).output == normalizeExprIds(plan2).output
+    assertEqualJoinPlans(Optimize, query, expected)
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinReorderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/joinReorder/StarJoinReorderSuite.scala
similarity index 91%
rename from sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinReorderSuite.scala
rename to sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/joinReorder/StarJoinReorderSuite.scala
index 10e970d534c49..ebc12b1d82cfa 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinReorderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/joinReorder/StarJoinReorderSuite.scala
@@ -15,18 +15,19 @@
  * limitations under the License.
  */
 
-package org.apache.spark.sql.catalyst.optimizer
+package org.apache.spark.sql.catalyst.optimizer.joinReorder
 
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeMap}
-import org.apache.spark.sql.catalyst.plans.{Inner, PlanTest}
+import org.apache.spark.sql.catalyst.optimizer._
+import org.apache.spark.sql.catalyst.plans.Inner
 import org.apache.spark.sql.catalyst.plans.logical.{ColumnStat, LocalRelation, LogicalPlan}
 import org.apache.spark.sql.catalyst.rules.RuleExecutor
 import org.apache.spark.sql.catalyst.statsEstimation.{StatsEstimationTestBase, StatsTestPlan}
 import org.apache.spark.sql.internal.SQLConf._
 
-class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
+class StarJoinReorderSuite extends JoinReorderPlanTestBase with StatsEstimationTestBase {
 
   var originalConfStarSchemaDetection = false
   var originalConfCBOEnabled = true
@@ -184,7 +185,7 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
         .join(s3, Inner, Some(nameToAttr("d3_fk1") === nameToAttr("s3_pk1")))
         .select(outputsOf(d1, d2, f1, d3, s3): _*)
 
-    assertEqualPlans(query, expected)
+    assertEqualJoinPlans(Optimize, query, expected)
   }
 
   test("Test 2: Star join on a subset of dimensions due to inequality joins") {
@@ -223,10 +224,10 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
         .join(s3, Inner, Some(nameToAttr("d3_fk1") === nameToAttr("s3_pk1")))
         .select(outputsOf(d1, f1, d2, s3, d3): _*)
 
-    assertEqualPlans(query, expected)
+    assertEqualJoinPlans(Optimize, query, expected)
   }
 
-  test("Test 3:  Star join on a subset of dimensions since join column is not unique") {
+  test("Test 3: Star join on a subset of dimensions since join column is not unique") {
     // Star join:
     //   (=)  (=)
     // d1 - f1 - d2
@@ -254,12 +255,12 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
     val expected =
       f1.join(d1.where(nameToAttr("d1_c2") === 2), Inner,
           Some(nameToAttr("f1_fk1") === nameToAttr("d1_pk1")))
-        .join(d3, Inner, Some(nameToAttr("d3_fk1") === nameToAttr("s3_pk1")))
-        .join(d2, Inner, Some(nameToAttr("f1_fk2") === nameToAttr("d2_pk1")))
-        .join(s3, Inner, Some(nameToAttr("f1_fk3") === nameToAttr("s3_c2")))
+        .join(d3, Inner, Some(nameToAttr("f1_fk3") === nameToAttr("d3_pk1")))
+        .join(d2, Inner, Some(nameToAttr("f1_fk2") === nameToAttr("d2_c4")))
+        .join(s3, Inner, Some(nameToAttr("d3_fk1") === nameToAttr("s3_pk1")))
         .select(outputsOf(d1, f1, d2, s3, d3): _*)
 
-    assertEqualPlans(query, expected)
+    assertEqualJoinPlans(Optimize, query, expected)
   }
 
   test("Test 4: Star join on a subset of dimensions since join column is nullable") {
@@ -296,7 +297,7 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
         .join(s3, Inner, Some(nameToAttr("d3_fk1") < nameToAttr("s3_pk1")))
         .select(outputsOf(d1, f1, d2, s3, d3): _*)
 
-    assertEqualPlans(query, expected)
+    assertEqualJoinPlans(Optimize, query, expected)
   }
 
   test("Test 5: Table stats not available for some of the joined tables") {
@@ -316,22 +317,25 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
     // Positional join reordering: d3_ns, f1, d1, d2, s3
     // Star join reordering: empty
 
+    val d3_pk1 = d3_ns.output.find(_.name == "d3_pk1").get
+    val d3_fk1 = d3_ns.output.find(_.name == "d3_fk1").get
+
     val query =
       d3_ns.join(f1).join(d1).join(d2).join(s3)
         .where((nameToAttr("f1_fk2") === nameToAttr("d2_pk1")) &&
           (nameToAttr("d2_c2") === 2) &&
           (nameToAttr("f1_fk1") === nameToAttr("d1_pk1")) &&
-          (nameToAttr("f1_fk3") === nameToAttr("d3_pk1")) &&
-          (nameToAttr("d3_fk1") === nameToAttr("s3_pk1")))
+          (nameToAttr("f1_fk3") === d3_pk1) &&
+          (d3_fk1 === nameToAttr("s3_pk1")))
 
     val equivQuery =
-      d3_ns.join(f1, Inner, Some(nameToAttr("f1_fk3") === nameToAttr("d3_pk1")))
+      d3_ns.join(f1, Inner, Some(nameToAttr("f1_fk3") === d3_pk1))
         .join(d1, Inner, Some(nameToAttr("f1_fk1") === nameToAttr("d1_pk1")))
         .join(d2.where(nameToAttr("d2_c2") === 2), Inner,
           Some(nameToAttr("f1_fk2") === nameToAttr("d2_pk1")))
-        .join(s3, Inner, Some(nameToAttr("d3_fk1") === nameToAttr("s3_pk1")))
+        .join(s3, Inner, Some(d3_fk1 === nameToAttr("s3_pk1")))
 
-    assertEqualPlans(query, equivQuery)
+    assertEqualJoinPlans(Optimize, query, equivQuery)
   }
 
   test("Test 6: Join with complex plans") {
@@ -364,7 +368,7 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
         .join(d2.where(nameToAttr("d2_c2") === 2), Inner,
           Some(nameToAttr("f1_fk2") === nameToAttr("d2_pk1")))
 
-    assertEqualPlans(query, expected)
+    assertEqualJoinPlans(Optimize, query, expected)
   }
 
   test("Test 7: Comparable fact table sizes") {
@@ -400,7 +404,7 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
         .join(s3, Inner, Some(nameToAttr("f11_fk1") === nameToAttr("s3_pk1")))
         .select(outputsOf(d1, f11, f1, d2, s3): _*)
 
-    assertEqualPlans(query, equivQuery)
+    assertEqualJoinPlans(Optimize, query, equivQuery)
   }
 
   test("Test 8: No RI joins") {
@@ -436,7 +440,7 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
         .join(s3, Inner, Some(nameToAttr("d3_fk1") === nameToAttr("s3_pk1")))
         .select(outputsOf(d1, d3, f1, d2, s3): _*)
 
-    assertEqualPlans(query, expected)
+    assertEqualJoinPlans(Optimize, query, expected)
   }
 
   test("Test 9: Complex join predicates") {
@@ -472,7 +476,7 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
         .join(s3, Inner, Some(nameToAttr("d3_fk1") === nameToAttr("s3_pk1")))
         .select(outputsOf(d1, d3, f1, d2, s3): _*)
 
-    assertEqualPlans(query, expected)
+    assertEqualJoinPlans(Optimize, query, expected)
   }
 
   test("Test 10: Less than two dimensions") {
@@ -507,7 +511,7 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
         .join(s3, Inner, Some(nameToAttr("d3_fk1") === nameToAttr("s3_pk1")))
         .select(outputsOf(d1, d3, f1, d2, s3): _*)
 
-    assertEqualPlans(query, expected)
+    assertEqualJoinPlans(Optimize, query, expected)
   }
 
   test("Test 11: Expanding star join") {
@@ -541,7 +545,7 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
         .join(s3, Inner, Some(nameToAttr("d3_fk1") < nameToAttr("s3_pk1")))
         .select(outputsOf(d1, d3, f1, d2, s3): _*)
 
-    assertEqualPlans(query, expected)
+    assertEqualJoinPlans(Optimize, query, expected)
   }
 
   test("Test 12: Non selective star join") {
@@ -575,25 +579,6 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
         .join(s3, Inner, Some(nameToAttr("d3_fk1") === nameToAttr("s3_pk1")))
         .select(outputsOf(d1, d3, f1, d2, s3): _*)
 
-    assertEqualPlans(query, expected)
-  }
-
-  private def assertEqualPlans(plan1: LogicalPlan, plan2: LogicalPlan): Unit = {
-    val analyzed = plan1.analyze
-    val optimized = Optimize.execute(analyzed)
-    val expected = plan2.analyze
-
-    assert(equivalentOutput(analyzed, expected)) // if this fails, the expected itself is incorrect
-    assert(equivalentOutput(analyzed, optimized))
-
-    compareJoinOrder(optimized, expected)
-  }
-
-  private def outputsOf(plans: LogicalPlan*): Seq[Attribute] = {
-    plans.map(_.output).reduce(_ ++ _)
-  }
-
-  private def equivalentOutput(plan1: LogicalPlan, plan2: LogicalPlan): Boolean = {
-    normalizeExprIds(plan1).output == normalizeExprIds(plan2).output
+    assertEqualJoinPlans(Optimize, query, expected)
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
index 6499b5d8e7974..c0648919ecc57 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
@@ -20,15 +20,14 @@ package org.apache.spark.sql.catalyst.parser
 import java.util.Locale
 
 import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, GlobalTempView, LocalTempView, PersistedView, UnresolvedAttribute, UnresolvedNamespace, UnresolvedRelation, UnresolvedStar, UnresolvedTable, UnresolvedTableOrView}
-import org.apache.spark.sql.catalyst.catalog.{ArchiveResource, BucketSpec, FileResource, FunctionResource, FunctionResourceType, JarResource}
-import org.apache.spark.sql.catalyst.expressions.{EqualTo, Literal}
+import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, GlobalTempView, LocalTempView, PersistedView, UnresolvedAttribute, UnresolvedFunc, UnresolvedInlineTable, UnresolvedNamespace, UnresolvedRelation, UnresolvedStar, UnresolvedTable, UnresolvedTableOrView, UnresolvedView}
+import org.apache.spark.sql.catalyst.catalog.{ArchiveResource, BucketSpec, FileResource, FunctionResource, JarResource}
+import org.apache.spark.sql.catalyst.expressions.{EqualTo, Hex, Literal}
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.connector.catalog.TableChange.ColumnPosition.{after, first}
 import org.apache.spark.sql.connector.expressions.{ApplyTransform, BucketTransform, DaysTransform, FieldReference, HoursTransform, IdentityTransform, LiteralValue, MonthsTransform, Transform, YearsTransform}
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructType, TimestampType}
-import org.apache.spark.unsafe.types.UTF8String
+import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 
 class DDLParserSuite extends AnalysisTest {
   import CatalystSqlParser._
@@ -64,6 +63,7 @@ class DDLParserSuite extends AnalysisTest {
       Some("parquet"),
       Map.empty[String, String],
       None,
+      None,
       None)
 
     Seq(createSql, replaceSql).foreach { sql =>
@@ -71,7 +71,7 @@ class DDLParserSuite extends AnalysisTest {
     }
 
     intercept("CREATE TABLE my_tab(a: INT COMMENT 'test', b: STRING) USING parquet",
-      "no viable alternative at input")
+      "extraneous input ':'")
   }
 
   test("create/replace table - with IF NOT EXISTS") {
@@ -87,6 +87,7 @@ class DDLParserSuite extends AnalysisTest {
         Some("parquet"),
         Map.empty[String, String],
         None,
+        None,
         None),
       expectedIfNotExists = true)
   }
@@ -107,6 +108,7 @@ class DDLParserSuite extends AnalysisTest {
       Some("parquet"),
       Map.empty[String, String],
       None,
+      None,
       None)
     Seq(createSql, replaceSql).foreach { sql =>
       testCreateOrReplaceDdl(sql, expectedTableSpec, expectedIfNotExists = false)
@@ -161,6 +163,7 @@ class DDLParserSuite extends AnalysisTest {
       Some("parquet"),
       Map.empty[String, String],
       None,
+      None,
       None)
     Seq(createSql, replaceSql).foreach { sql =>
       testCreateOrReplaceDdl(sql, expectedTableSpec, expectedIfNotExists = false)
@@ -183,6 +186,7 @@ class DDLParserSuite extends AnalysisTest {
       Some("parquet"),
       Map.empty[String, String],
       None,
+      None,
       None)
     Seq(createSql, replaceSql).foreach { sql =>
       testCreateOrReplaceDdl(sql, expectedTableSpec, expectedIfNotExists = false)
@@ -201,7 +205,8 @@ class DDLParserSuite extends AnalysisTest {
       Some("parquet"),
       Map.empty[String, String],
       None,
-      Some("abc"))
+      Some("abc"),
+      None)
     Seq(createSql, replaceSql).foreach{ sql =>
       testCreateOrReplaceDdl(sql, expectedTableSpec, expectedIfNotExists = false)
     }
@@ -221,6 +226,7 @@ class DDLParserSuite extends AnalysisTest {
       Some("parquet"),
       Map.empty[String, String],
       None,
+      None,
       None)
     Seq(createSql, replaceSql).foreach { sql =>
       testCreateOrReplaceDdl(sql, expectedTableSpec, expectedIfNotExists = false)
@@ -239,6 +245,7 @@ class DDLParserSuite extends AnalysisTest {
         Some("parquet"),
         Map.empty[String, String],
         Some("/tmp/file"),
+        None,
         None)
     Seq(createSql, replaceSql).foreach { sql =>
       testCreateOrReplaceDdl(sql, expectedTableSpec, expectedIfNotExists = false)
@@ -257,19 +264,309 @@ class DDLParserSuite extends AnalysisTest {
       Some("parquet"),
       Map.empty[String, String],
       None,
+      None,
+      None)
+    Seq(createSql, replaceSql).foreach { sql =>
+      testCreateOrReplaceDdl(sql, expectedTableSpec, expectedIfNotExists = false)
+    }
+  }
+
+  test("create/replace table - partition column definitions") {
+    val createSql = "CREATE TABLE my_tab (id bigint) PARTITIONED BY (part string)"
+    val replaceSql = "REPLACE TABLE my_tab (id bigint) PARTITIONED BY (part string)"
+    val expectedTableSpec = TableSpec(
+      Seq("my_tab"),
+      Some(new StructType().add("id", LongType).add("part", StringType)),
+      Seq(IdentityTransform(FieldReference("part"))),
+      None,
+      Map.empty[String, String],
+      None,
+      Map.empty[String, String],
+      None,
+      None,
+      None)
+    Seq(createSql, replaceSql).foreach { sql =>
+      testCreateOrReplaceDdl(sql, expectedTableSpec, expectedIfNotExists = false)
+    }
+  }
+
+  test("create/replace table - empty columns list") {
+    val createSql = "CREATE TABLE my_tab PARTITIONED BY (part string)"
+    val replaceSql = "REPLACE TABLE my_tab PARTITIONED BY (part string)"
+    val expectedTableSpec = TableSpec(
+      Seq("my_tab"),
+      Some(new StructType().add("part", StringType)),
+      Seq(IdentityTransform(FieldReference("part"))),
+      None,
+      Map.empty[String, String],
+      None,
+      Map.empty[String, String],
+      None,
+      None,
+      None)
+    Seq(createSql, replaceSql).foreach { sql =>
+      testCreateOrReplaceDdl(sql, expectedTableSpec, expectedIfNotExists = false)
+    }
+  }
+
+  test("create/replace table - using with partition column definitions") {
+    val createSql = "CREATE TABLE my_tab (id bigint) USING parquet PARTITIONED BY (part string)"
+    val replaceSql = "REPLACE TABLE my_tab (id bigint) USING parquet PARTITIONED BY (part string)"
+    val expectedTableSpec = TableSpec(
+      Seq("my_tab"),
+      Some(new StructType().add("id", LongType).add("part", StringType)),
+      Seq(IdentityTransform(FieldReference("part"))),
+      None,
+      Map.empty[String, String],
+      Some("parquet"),
+      Map.empty[String, String],
+      None,
+      None,
       None)
     Seq(createSql, replaceSql).foreach { sql =>
       testCreateOrReplaceDdl(sql, expectedTableSpec, expectedIfNotExists = false)
     }
   }
 
+  test("create/replace table - mixed partition references and column definitions") {
+    val createSql = "CREATE TABLE my_tab (id bigint, p1 string) PARTITIONED BY (p1, p2 string)"
+    val replaceSql = createSql.replaceFirst("CREATE", "REPLACE")
+    Seq(createSql, replaceSql).foreach { sql =>
+      assertUnsupported(sql, Seq(
+        "PARTITION BY: Cannot mix partition expressions and partition columns",
+        "Expressions: p1",
+        "Columns: p2 string"))
+    }
+
+    val createSqlWithExpr =
+      "CREATE TABLE my_tab (id bigint, p1 string) PARTITIONED BY (p2 string, truncate(p1, 16))"
+    val replaceSqlWithExpr = createSqlWithExpr.replaceFirst("CREATE", "REPLACE")
+    Seq(createSqlWithExpr, replaceSqlWithExpr).foreach { sql =>
+      assertUnsupported(sql, Seq(
+        "PARTITION BY: Cannot mix partition expressions and partition columns",
+        "Expressions: truncate(p1, 16)",
+        "Columns: p2 string"))
+    }
+  }
+
+  test("create/replace table - stored as") {
+    val createSql =
+      """CREATE TABLE my_tab (id bigint)
+        |PARTITIONED BY (part string)
+        |STORED AS parquet
+        """.stripMargin
+    val replaceSql = createSql.replaceFirst("CREATE", "REPLACE")
+    val expectedTableSpec = TableSpec(
+      Seq("my_tab"),
+      Some(new StructType().add("id", LongType).add("part", StringType)),
+      Seq(IdentityTransform(FieldReference("part"))),
+      None,
+      Map.empty[String, String],
+      None,
+      Map.empty[String, String],
+      None,
+      None,
+      Some(SerdeInfo(storedAs = Some("parquet"))))
+    Seq(createSql, replaceSql).foreach { sql =>
+      testCreateOrReplaceDdl(sql, expectedTableSpec, expectedIfNotExists = false)
+    }
+  }
+
+  test("create/replace table - stored as format with serde") {
+    Seq("sequencefile", "textfile", "rcfile").foreach { format =>
+      val createSql =
+        s"""CREATE TABLE my_tab (id bigint)
+          |PARTITIONED BY (part string)
+          |STORED AS $format
+          |ROW FORMAT SERDE 'customSerde'
+          |WITH SERDEPROPERTIES ('prop'='value')
+        """.stripMargin
+      val replaceSql = createSql.replaceFirst("CREATE", "REPLACE")
+      val expectedTableSpec = TableSpec(
+        Seq("my_tab"),
+        Some(new StructType().add("id", LongType).add("part", StringType)),
+        Seq(IdentityTransform(FieldReference("part"))),
+        None,
+        Map.empty[String, String],
+        None,
+        Map.empty[String, String],
+        None,
+        None,
+        Some(SerdeInfo(storedAs = Some(format), serde = Some("customSerde"), serdeProperties = Map(
+          "prop" -> "value"
+        ))))
+      Seq(createSql, replaceSql).foreach { sql =>
+        testCreateOrReplaceDdl(sql, expectedTableSpec, expectedIfNotExists = false)
+      }
+    }
+
+    val createSql =
+      s"""CREATE TABLE my_tab (id bigint)
+         |PARTITIONED BY (part string)
+         |STORED AS otherFormat
+         |ROW FORMAT SERDE 'customSerde'
+         |WITH SERDEPROPERTIES ('prop'='value')
+         """.stripMargin
+    val replaceSql = createSql.replaceFirst("CREATE", "REPLACE")
+    Seq(createSql, replaceSql).foreach { sql =>
+      assertUnsupported(sql, Seq("ROW FORMAT SERDE is incompatible with format 'otherFormat'"))
+    }
+  }
+
+  test("create/replace table - stored as format with delimited clauses") {
+    val createSql =
+      s"""CREATE TABLE my_tab (id bigint)
+         |PARTITIONED BY (part string)
+         |STORED AS textfile
+         |ROW FORMAT DELIMITED
+         |FIELDS TERMINATED BY ',' ESCAPED BY '\\\\' -- double escape for Scala and for SQL
+         |COLLECTION ITEMS TERMINATED BY '#'
+         |MAP KEYS TERMINATED BY '='
+         |LINES TERMINATED BY '\\n'
+      """.stripMargin
+    val replaceSql = createSql.replaceFirst("CREATE", "REPLACE")
+    val expectedTableSpec = TableSpec(
+      Seq("my_tab"),
+      Some(new StructType().add("id", LongType).add("part", StringType)),
+      Seq(IdentityTransform(FieldReference("part"))),
+      None,
+      Map.empty[String, String],
+      None,
+      Map.empty[String, String],
+      None,
+      None,
+      Some(SerdeInfo(storedAs = Some("textfile"), serdeProperties = Map(
+        "field.delim" -> ",", "serialization.format" -> ",", "escape.delim" -> "\\",
+        "colelction.delim" -> "#", "mapkey.delim" -> "=", "line.delim" -> "\n"
+      ))))
+    Seq(createSql, replaceSql).foreach { sql =>
+      testCreateOrReplaceDdl(sql, expectedTableSpec, expectedIfNotExists = false)
+    }
+
+    val createFailSql =
+      s"""CREATE TABLE my_tab (id bigint)
+         |PARTITIONED BY (part string)
+         |STORED AS otherFormat
+         |ROW FORMAT DELIMITED
+         |FIELDS TERMINATED BY ','
+         """.stripMargin
+    val replaceFailSql = createFailSql.replaceFirst("CREATE", "REPLACE")
+    Seq(createFailSql, replaceFailSql).foreach { sql =>
+      assertUnsupported(sql, Seq(
+        "ROW FORMAT DELIMITED is only compatible with 'textfile', not 'otherFormat'"))
+    }
+  }
+
+  test("create/replace table - stored as inputformat/outputformat") {
+    val createSql =
+      """CREATE TABLE my_tab (id bigint)
+        |PARTITIONED BY (part string)
+        |STORED AS INPUTFORMAT 'inFormat' OUTPUTFORMAT 'outFormat'
+        """.stripMargin
+    val replaceSql = createSql.replaceFirst("CREATE", "REPLACE")
+    val expectedTableSpec = TableSpec(
+      Seq("my_tab"),
+      Some(new StructType().add("id", LongType).add("part", StringType)),
+      Seq(IdentityTransform(FieldReference("part"))),
+      None,
+      Map.empty[String, String],
+      None,
+      Map.empty[String, String],
+      None,
+      None,
+      Some(SerdeInfo(formatClasses = Some(FormatClasses("inFormat", "outFormat")))))
+    Seq(createSql, replaceSql).foreach { sql =>
+      testCreateOrReplaceDdl(sql, expectedTableSpec, expectedIfNotExists = false)
+    }
+  }
+
+  test("create/replace table - stored as inputformat/outputformat with serde") {
+    val createSql =
+      """CREATE TABLE my_tab (id bigint)
+        |PARTITIONED BY (part string)
+        |STORED AS INPUTFORMAT 'inFormat' OUTPUTFORMAT 'outFormat'
+        |ROW FORMAT SERDE 'customSerde'
+        """.stripMargin
+    val replaceSql = createSql.replaceFirst("CREATE", "REPLACE")
+    val expectedTableSpec = TableSpec(
+      Seq("my_tab"),
+      Some(new StructType().add("id", LongType).add("part", StringType)),
+      Seq(IdentityTransform(FieldReference("part"))),
+      None,
+      Map.empty[String, String],
+      None,
+      Map.empty[String, String],
+      None,
+      None,
+      Some(SerdeInfo(
+        formatClasses = Some(FormatClasses("inFormat", "outFormat")),
+        serde = Some("customSerde"))))
+    Seq(createSql, replaceSql).foreach { sql =>
+      testCreateOrReplaceDdl(sql, expectedTableSpec, expectedIfNotExists = false)
+    }
+  }
+
+  test("create/replace table - using with stored as") {
+    val createSql =
+      """CREATE TABLE my_tab (id bigint, part string)
+        |USING parquet
+        |STORED AS parquet
+        """.stripMargin
+    val replaceSql = createSql.replaceFirst("CREATE", "REPLACE")
+    Seq(createSql, replaceSql).foreach { sql =>
+      assertUnsupported(sql, Seq("CREATE TABLE ... USING ... STORED AS"))
+    }
+  }
+
+  test("create/replace table - using with row format serde") {
+    val createSql =
+      """CREATE TABLE my_tab (id bigint, part string)
+        |USING parquet
+        |ROW FORMAT SERDE 'customSerde'
+        """.stripMargin
+    val replaceSql = createSql.replaceFirst("CREATE", "REPLACE")
+    Seq(createSql, replaceSql).foreach { sql =>
+      assertUnsupported(sql, Seq("CREATE TABLE ... USING ... ROW FORMAT SERDE"))
+    }
+  }
+
+  test("create/replace table - using with row format delimited") {
+    val createSql =
+      """CREATE TABLE my_tab (id bigint, part string)
+        |USING parquet
+        |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+        """.stripMargin
+    val replaceSql = createSql.replaceFirst("CREATE", "REPLACE")
+    Seq(createSql, replaceSql).foreach { sql =>
+      assertUnsupported(sql, Seq("CREATE TABLE ... USING ... ROW FORMAT DELIMITED"))
+    }
+  }
+
+  test("create/replace table - stored by") {
+    val createSql =
+      """CREATE TABLE my_tab (id bigint, p1 string)
+        |STORED BY 'handler'
+        """.stripMargin
+    val replaceSql = createSql.replaceFirst("CREATE", "REPLACE")
+    Seq(createSql, replaceSql).foreach { sql =>
+      assertUnsupported(sql, Seq("stored by"))
+    }
+  }
+
+  test("Unsupported skew clause - create/replace table") {
+    intercept("CREATE TABLE my_tab (id bigint) SKEWED BY (id) ON (1,2,3)",
+      "CREATE TABLE ... SKEWED BY")
+    intercept("REPLACE TABLE my_tab (id bigint) SKEWED BY (id) ON (1,2,3)",
+      "CREATE TABLE ... SKEWED BY")
+  }
+
   test("Duplicate clauses - create/replace table") {
     def createTableHeader(duplicateClause: String): String = {
-      s"CREATE TABLE my_tab(a INT, b STRING) USING parquet $duplicateClause $duplicateClause"
+      s"CREATE TABLE my_tab(a INT, b STRING) $duplicateClause $duplicateClause"
     }
 
     def replaceTableHeader(duplicateClause: String): String = {
-      s"CREATE TABLE my_tab(a INT, b STRING) USING parquet $duplicateClause $duplicateClause"
+      s"CREATE TABLE my_tab(a INT, b STRING) $duplicateClause $duplicateClause"
     }
 
     intercept(createTableHeader("TBLPROPERTIES('test' = 'test2')"),
@@ -282,6 +579,14 @@ class DDLParserSuite extends AnalysisTest {
       "Found duplicate clauses: CLUSTERED BY")
     intercept(createTableHeader("PARTITIONED BY (b)"),
       "Found duplicate clauses: PARTITIONED BY")
+    intercept(createTableHeader("PARTITIONED BY (c int)"),
+      "Found duplicate clauses: PARTITIONED BY")
+    intercept(createTableHeader("STORED AS parquet"),
+      "Found duplicate clauses: STORED AS")
+    intercept(createTableHeader("STORED AS INPUTFORMAT 'in' OUTPUTFORMAT 'out'"),
+      "Found duplicate clauses: STORED AS")
+    intercept(createTableHeader("ROW FORMAT SERDE 'serde'"),
+      "Found duplicate clauses: ROW FORMAT")
 
     intercept(replaceTableHeader("TBLPROPERTIES('test' = 'test2')"),
       "Found duplicate clauses: TBLPROPERTIES")
@@ -293,6 +598,14 @@ class DDLParserSuite extends AnalysisTest {
       "Found duplicate clauses: CLUSTERED BY")
     intercept(replaceTableHeader("PARTITIONED BY (b)"),
       "Found duplicate clauses: PARTITIONED BY")
+    intercept(replaceTableHeader("PARTITIONED BY (c int)"),
+      "Found duplicate clauses: PARTITIONED BY")
+    intercept(replaceTableHeader("STORED AS parquet"),
+      "Found duplicate clauses: STORED AS")
+    intercept(replaceTableHeader("STORED AS INPUTFORMAT 'in' OUTPUTFORMAT 'out'"),
+      "Found duplicate clauses: STORED AS")
+    intercept(replaceTableHeader("ROW FORMAT SERDE 'serde'"),
+      "Found duplicate clauses: ROW FORMAT")
   }
 
   test("support for other types in OPTIONS") {
@@ -318,6 +631,7 @@ class DDLParserSuite extends AnalysisTest {
           Some("json"),
           Map("a" -> "1", "b" -> "0.1", "c" -> "true"),
           None,
+          None,
           None),
         expectedIfNotExists = false)
     }
@@ -373,37 +687,26 @@ class DDLParserSuite extends AnalysisTest {
         Some("parquet"),
         Map.empty[String, String],
         Some("/user/external/page_view"),
-        Some("This is the staging page view table"))
+        Some("This is the staging page view table"),
+        None)
     Seq(s1, s2, s3, s4).foreach { sql =>
       testCreateOrReplaceDdl(sql, expectedTableSpec, expectedIfNotExists = true)
     }
   }
 
-  test("drop table") {
-    parseCompare("DROP TABLE testcat.ns1.ns2.tbl",
-      DropTableStatement(Seq("testcat", "ns1", "ns2", "tbl"), ifExists = false, purge = false))
-    parseCompare(s"DROP TABLE db.tab",
-      DropTableStatement(Seq("db", "tab"), ifExists = false, purge = false))
-    parseCompare(s"DROP TABLE IF EXISTS db.tab",
-      DropTableStatement(Seq("db", "tab"), ifExists = true, purge = false))
-    parseCompare(s"DROP TABLE tab",
-      DropTableStatement(Seq("tab"), ifExists = false, purge = false))
-    parseCompare(s"DROP TABLE IF EXISTS tab",
-      DropTableStatement(Seq("tab"), ifExists = true, purge = false))
-    parseCompare(s"DROP TABLE tab PURGE",
-      DropTableStatement(Seq("tab"), ifExists = false, purge = true))
-    parseCompare(s"DROP TABLE IF EXISTS tab PURGE",
-      DropTableStatement(Seq("tab"), ifExists = true, purge = true))
-  }
-
   test("drop view") {
+    val cmd = "DROP VIEW"
+    val hint = Some("Please use DROP TABLE instead.")
     parseCompare(s"DROP VIEW testcat.db.view",
-      DropViewStatement(Seq("testcat", "db", "view"), ifExists = false))
-    parseCompare(s"DROP VIEW db.view", DropViewStatement(Seq("db", "view"), ifExists = false))
+      DropView(UnresolvedView(Seq("testcat", "db", "view"), cmd, true, hint), ifExists = false))
+    parseCompare(s"DROP VIEW db.view",
+      DropView(UnresolvedView(Seq("db", "view"), cmd, true, hint), ifExists = false))
     parseCompare(s"DROP VIEW IF EXISTS db.view",
-      DropViewStatement(Seq("db", "view"), ifExists = true))
-    parseCompare(s"DROP VIEW view", DropViewStatement(Seq("view"), ifExists = false))
-    parseCompare(s"DROP VIEW IF EXISTS view", DropViewStatement(Seq("view"), ifExists = true))
+      DropView(UnresolvedView(Seq("db", "view"), cmd, true, hint), ifExists = true))
+    parseCompare(s"DROP VIEW view",
+      DropView(UnresolvedView(Seq("view"), cmd, true, hint), ifExists = false))
+    parseCompare(s"DROP VIEW IF EXISTS view",
+      DropView(UnresolvedView(Seq("view"), cmd, true, hint), ifExists = true))
   }
 
   private def testCreateOrReplaceDdl(
@@ -435,16 +738,22 @@ class DDLParserSuite extends AnalysisTest {
         "'comment' = 'new_comment')"
     val sql2_view = "ALTER VIEW table_name UNSET TBLPROPERTIES ('comment', 'test')"
     val sql3_view = "ALTER VIEW table_name UNSET TBLPROPERTIES IF EXISTS ('comment', 'test')"
+    val hint = Some("Please use ALTER TABLE instead.")
 
     comparePlans(parsePlan(sql1_view),
-      AlterViewSetPropertiesStatement(
-      Seq("table_name"), Map("test" -> "test", "comment" -> "new_comment")))
+      SetViewProperties(
+        UnresolvedView(Seq("table_name"), "ALTER VIEW ... SET TBLPROPERTIES", false, hint),
+        Map("test" -> "test", "comment" -> "new_comment")))
     comparePlans(parsePlan(sql2_view),
-      AlterViewUnsetPropertiesStatement(
-      Seq("table_name"), Seq("comment", "test"), ifExists = false))
+      UnsetViewProperties(
+        UnresolvedView(Seq("table_name"), "ALTER VIEW ... UNSET TBLPROPERTIES", false, hint),
+        Seq("comment", "test"),
+        ifExists = false))
     comparePlans(parsePlan(sql3_view),
-      AlterViewUnsetPropertiesStatement(
-      Seq("table_name"), Seq("comment", "test"), ifExists = true))
+      UnsetViewProperties(
+        UnresolvedView(Seq("table_name"), "ALTER VIEW ... UNSET TBLPROPERTIES", false, hint),
+        Seq("comment", "test"),
+        ifExists = true))
   }
 
   // ALTER TABLE table_name SET TBLPROPERTIES ('comment' = new_comment);
@@ -454,19 +763,25 @@ class DDLParserSuite extends AnalysisTest {
         "'comment' = 'new_comment')"
     val sql2_table = "ALTER TABLE table_name UNSET TBLPROPERTIES ('comment', 'test')"
     val sql3_table = "ALTER TABLE table_name UNSET TBLPROPERTIES IF EXISTS ('comment', 'test')"
+    val hint = Some("Please use ALTER VIEW instead.")
 
     comparePlans(
       parsePlan(sql1_table),
-      AlterTableSetPropertiesStatement(
-        Seq("table_name"), Map("test" -> "test", "comment" -> "new_comment")))
+      SetTableProperties(
+        UnresolvedTable(Seq("table_name"), "ALTER TABLE ... SET TBLPROPERTIES", hint),
+        Map("test" -> "test", "comment" -> "new_comment")))
     comparePlans(
       parsePlan(sql2_table),
-      AlterTableUnsetPropertiesStatement(
-        Seq("table_name"), Seq("comment", "test"), ifExists = false))
+      UnsetTableProperties(
+        UnresolvedTable(Seq("table_name"), "ALTER TABLE ... UNSET TBLPROPERTIES", hint),
+        Seq("comment", "test"),
+        ifExists = false))
     comparePlans(
       parsePlan(sql3_table),
-      AlterTableUnsetPropertiesStatement(
-        Seq("table_name"), Seq("comment", "test"), ifExists = true))
+      UnsetTableProperties(
+        UnresolvedTable(Seq("table_name"), "ALTER TABLE ... UNSET TBLPROPERTIES", hint),
+        Seq("comment", "test"),
+        ifExists = true))
   }
 
   test("alter table: add column") {
@@ -558,14 +873,18 @@ class DDLParserSuite extends AnalysisTest {
   }
 
   test("alter table: set location") {
+    val hint = Some("Please use ALTER VIEW instead.")
     comparePlans(
       parsePlan("ALTER TABLE a.b.c SET LOCATION 'new location'"),
-      AlterTableSetLocationStatement(Seq("a", "b", "c"), None, "new location"))
+      SetTableLocation(
+        UnresolvedTable(Seq("a", "b", "c"), "ALTER TABLE ... SET LOCATION ...", hint),
+        None,
+        "new location"))
 
     comparePlans(
       parsePlan("ALTER TABLE a.b.c PARTITION(ds='2017-06-10') SET LOCATION 'new location'"),
-      AlterTableSetLocationStatement(
-        Seq("a", "b", "c"),
+      SetTableLocation(
+        UnresolvedTable(Seq("a", "b", "c"), "ALTER TABLE ... SET LOCATION ...", hint),
         Some(Map("ds" -> "2017-06-10")),
         "new location"))
   }
@@ -591,6 +910,13 @@ class DDLParserSuite extends AnalysisTest {
         None))
   }
 
+  test("alter table: update column type invalid type") {
+    val msg = intercept[ParseException] {
+      parsePlan("ALTER TABLE table_name ALTER COLUMN a.b.c TYPE bad_type")
+    }.getMessage
+    assert(msg.contains("DataType bad_type is not supported"))
+  }
+
   test("alter table: update column type") {
     comparePlans(
       parsePlan("ALTER TABLE table_name CHANGE COLUMN a.b.c TYPE bigint"),
@@ -627,7 +953,7 @@ class DDLParserSuite extends AnalysisTest {
         Some(first())))
   }
 
-  test("alter table: mutiple property changes are not allowed") {
+  test("alter table: multiple property changes are not allowed") {
     intercept[ParseException] {
       parsePlan("ALTER TABLE table_name ALTER COLUMN a.b.c " +
         "TYPE bigint COMMENT 'new comment'")}
@@ -769,38 +1095,52 @@ class DDLParserSuite extends AnalysisTest {
       "Column position is not supported in Hive-style REPLACE COLUMNS")
   }
 
-  test("alter table/view: rename table/view") {
-    comparePlans(
-      parsePlan("ALTER TABLE a.b.c RENAME TO x.y.z"),
-      RenameTableStatement(Seq("a", "b", "c"), Seq("x", "y", "z"), isView = false))
+  test("alter view: rename view") {
     comparePlans(
       parsePlan("ALTER VIEW a.b.c RENAME TO x.y.z"),
-      RenameTableStatement(Seq("a", "b", "c"), Seq("x", "y", "z"), isView = true))
+      RenameTable(
+        UnresolvedTableOrView(Seq("a", "b", "c"), "ALTER VIEW ... RENAME TO", true),
+        Seq("x", "y", "z"),
+        isView = true))
   }
 
   test("describe table column") {
     comparePlans(parsePlan("DESCRIBE t col"),
-      DescribeColumnStatement(
-        Seq("t"), Seq("col"), isExtended = false))
+      DescribeColumn(
+        UnresolvedTableOrView(Seq("t"), "DESCRIBE TABLE", true),
+        UnresolvedAttribute(Seq("col")),
+        isExtended = false))
     comparePlans(parsePlan("DESCRIBE t `abc.xyz`"),
-      DescribeColumnStatement(
-        Seq("t"), Seq("abc.xyz"), isExtended = false))
+      DescribeColumn(
+        UnresolvedTableOrView(Seq("t"), "DESCRIBE TABLE", true),
+        UnresolvedAttribute(Seq("abc.xyz")),
+        isExtended = false))
     comparePlans(parsePlan("DESCRIBE t abc.xyz"),
-      DescribeColumnStatement(
-        Seq("t"), Seq("abc", "xyz"), isExtended = false))
+      DescribeColumn(
+        UnresolvedTableOrView(Seq("t"), "DESCRIBE TABLE", true),
+        UnresolvedAttribute(Seq("abc", "xyz")),
+        isExtended = false))
     comparePlans(parsePlan("DESCRIBE t `a.b`.`x.y`"),
-      DescribeColumnStatement(
-        Seq("t"), Seq("a.b", "x.y"), isExtended = false))
+      DescribeColumn(
+        UnresolvedTableOrView(Seq("t"), "DESCRIBE TABLE", true),
+        UnresolvedAttribute(Seq("a.b", "x.y")),
+        isExtended = false))
 
     comparePlans(parsePlan("DESCRIBE TABLE t col"),
-      DescribeColumnStatement(
-        Seq("t"), Seq("col"), isExtended = false))
+      DescribeColumn(
+        UnresolvedTableOrView(Seq("t"), "DESCRIBE TABLE", true),
+        UnresolvedAttribute(Seq("col")),
+        isExtended = false))
     comparePlans(parsePlan("DESCRIBE TABLE EXTENDED t col"),
-      DescribeColumnStatement(
-        Seq("t"), Seq("col"), isExtended = true))
+      DescribeColumn(
+        UnresolvedTableOrView(Seq("t"), "DESCRIBE TABLE", true),
+        UnresolvedAttribute(Seq("col")),
+        isExtended = true))
     comparePlans(parsePlan("DESCRIBE TABLE FORMATTED t col"),
-      DescribeColumnStatement(
-        Seq("t"), Seq("col"), isExtended = true))
+      DescribeColumn(
+        UnresolvedTableOrView(Seq("t"), "DESCRIBE TABLE", true),
+        UnresolvedAttribute(Seq("col")),
+        isExtended = true))
 
     val caught = intercept[AnalysisException](
       parsePlan("DESCRIBE TABLE t PARTITION (ds='1970-01-01') col"))
@@ -819,13 +1159,17 @@ class DDLParserSuite extends AnalysisTest {
 
   test("SPARK-17328 Fix NPE with EXPLAIN DESCRIBE TABLE") {
     comparePlans(parsePlan("describe t"),
-      DescribeRelation(UnresolvedTableOrView(Seq("t")), Map.empty, isExtended = false))
+      DescribeRelation(
+        UnresolvedTableOrView(Seq("t"), "DESCRIBE TABLE", true), Map.empty, isExtended = false))
     comparePlans(parsePlan("describe table t"),
-      DescribeRelation(UnresolvedTableOrView(Seq("t")), Map.empty, isExtended = false))
+      DescribeRelation(
+        UnresolvedTableOrView(Seq("t"), "DESCRIBE TABLE", true), Map.empty, isExtended = false))
     comparePlans(parsePlan("describe table extended t"),
-      DescribeRelation(UnresolvedTableOrView(Seq("t")), Map.empty, isExtended = true))
+      DescribeRelation(
+        UnresolvedTableOrView(Seq("t"), "DESCRIBE TABLE", true), Map.empty, isExtended = true))
     comparePlans(parsePlan("describe table formatted t"),
-      DescribeRelation(UnresolvedTableOrView(Seq("t")), Map.empty, isExtended = true))
+      DescribeRelation(
+        UnresolvedTableOrView(Seq("t"), "DESCRIBE TABLE", true), Map.empty, isExtended = true))
   }
 
   test("insert table: basic append") {
@@ -837,6 +1181,22 @@ class DDLParserSuite extends AnalysisTest {
         InsertIntoStatement(
           UnresolvedRelation(Seq("testcat", "ns1", "ns2", "tbl")),
           Map.empty,
+          Nil,
+          Project(Seq(UnresolvedStar(None)), UnresolvedRelation(Seq("source"))),
+          overwrite = false, ifPartitionNotExists = false))
+    }
+  }
+
+  test("insert table: basic append with a column list") {
+    Seq(
+      "INSERT INTO TABLE testcat.ns1.ns2.tbl (a, b) SELECT * FROM source",
+      "INSERT INTO testcat.ns1.ns2.tbl (a, b) SELECT * FROM source"
+    ).foreach { sql =>
+      parseCompare(sql,
+        InsertIntoStatement(
+          UnresolvedRelation(Seq("testcat", "ns1", "ns2", "tbl")),
+          Map.empty,
+          Seq("a", "b"),
           Project(Seq(UnresolvedStar(None)), UnresolvedRelation(Seq("source"))),
           overwrite = false, ifPartitionNotExists = false))
     }
@@ -847,6 +1207,7 @@ class DDLParserSuite extends AnalysisTest {
       InsertIntoStatement(
         UnresolvedRelation(Seq("testcat", "ns1", "ns2", "tbl")),
         Map.empty,
+        Nil,
         Project(Seq(UnresolvedStar(None)), UnresolvedRelation(Seq("testcat2", "db", "tbl"))),
         overwrite = false, ifPartitionNotExists = false))
   }
@@ -861,6 +1222,22 @@ class DDLParserSuite extends AnalysisTest {
       InsertIntoStatement(
         UnresolvedRelation(Seq("testcat", "ns1", "ns2", "tbl")),
         Map("p1" -> Some("3"), "p2" -> None),
+        Nil,
+        Project(Seq(UnresolvedStar(None)), UnresolvedRelation(Seq("source"))),
+        overwrite = false, ifPartitionNotExists = false))
+  }
+
+  test("insert table: append with partition and a column list") {
+    parseCompare(
+      """
+        |INSERT INTO testcat.ns1.ns2.tbl
+        |PARTITION (p1 = 3, p2) (a, b)
+        |SELECT * FROM source
+      """.stripMargin,
+      InsertIntoStatement(
+        UnresolvedRelation(Seq("testcat", "ns1", "ns2", "tbl")),
+        Map("p1" -> Some("3"), "p2" -> None),
+        Seq("a", "b"),
         Project(Seq(UnresolvedStar(None)), UnresolvedRelation(Seq("source"))),
         overwrite = false, ifPartitionNotExists = false))
   }
@@ -874,6 +1251,22 @@ class DDLParserSuite extends AnalysisTest {
         InsertIntoStatement(
           UnresolvedRelation(Seq("testcat", "ns1", "ns2", "tbl")),
           Map.empty,
+          Nil,
+          Project(Seq(UnresolvedStar(None)), UnresolvedRelation(Seq("source"))),
+          overwrite = true, ifPartitionNotExists = false))
+    }
+  }
+
+  test("insert table: overwrite with column list") {
+    Seq(
+      "INSERT OVERWRITE TABLE testcat.ns1.ns2.tbl (a, b) SELECT * FROM source",
+      "INSERT OVERWRITE testcat.ns1.ns2.tbl (a, b) SELECT * FROM source"
+    ).foreach { sql =>
+      parseCompare(sql,
+        InsertIntoStatement(
+          UnresolvedRelation(Seq("testcat", "ns1", "ns2", "tbl")),
+          Map.empty,
+          Seq("a", "b"),
           Project(Seq(UnresolvedStar(None)), UnresolvedRelation(Seq("source"))),
           overwrite = true, ifPartitionNotExists = false))
     }
@@ -889,6 +1282,22 @@ class DDLParserSuite extends AnalysisTest {
       InsertIntoStatement(
         UnresolvedRelation(Seq("testcat", "ns1", "ns2", "tbl")),
         Map("p1" -> Some("3"), "p2" -> None),
+        Nil,
+        Project(Seq(UnresolvedStar(None)), UnresolvedRelation(Seq("source"))),
+        overwrite = true, ifPartitionNotExists = false))
+  }
+
+  test("insert table: overwrite with partition and column list") {
+    parseCompare(
+      """
+        |INSERT OVERWRITE TABLE testcat.ns1.ns2.tbl
+        |PARTITION (p1 = 3, p2) (a, b)
+        |SELECT * FROM source
+      """.stripMargin,
+      InsertIntoStatement(
+        UnresolvedRelation(Seq("testcat", "ns1", "ns2", "tbl")),
+        Map("p1" -> Some("3"), "p2" -> None),
+        Seq("a", "b"),
         Project(Seq(UnresolvedStar(None)), UnresolvedRelation(Seq("source"))),
         overwrite = true, ifPartitionNotExists = false))
   }
@@ -903,6 +1312,7 @@ class DDLParserSuite extends AnalysisTest {
       InsertIntoStatement(
         UnresolvedRelation(Seq("testcat", "ns1", "ns2", "tbl")),
         Map("p1" -> Some("3")),
+        Nil,
         Project(Seq(UnresolvedStar(None)), UnresolvedRelation(Seq("source"))),
         overwrite = true, ifPartitionNotExists = true))
   }
@@ -1109,9 +1519,8 @@ class DDLParserSuite extends AnalysisTest {
       SubqueryAlias("source", UnresolvedRelation(Seq("testcat2", "ns1", "ns2", "tbl"))),
       EqualTo(UnresolvedAttribute("target.col1"), UnresolvedAttribute("source.col1")),
       Seq(DeleteAction(Some(EqualTo(UnresolvedAttribute("target.col2"), Literal("delete")))),
-        UpdateAction(Some(EqualTo(UnresolvedAttribute("target.col2"), Literal("update"))), Seq())),
-      Seq(InsertAction(Some(EqualTo(UnresolvedAttribute("target.col2"), Literal("insert"))),
-        Seq()))))
+        UpdateStarAction(Some(EqualTo(UnresolvedAttribute("target.col2"), Literal("update"))))),
+      Seq(InsertStarAction(Some(EqualTo(UnresolvedAttribute("target.col2"), Literal("insert")))))))
   }
 
   test("merge into table: columns aliases are not allowed") {
@@ -1134,58 +1543,74 @@ class DDLParserSuite extends AnalysisTest {
     }
   }
 
-  test("merge into table: at most two matched clauses") {
-    val exc = intercept[ParseException] {
-      parsePlan(
-        """
-          |MERGE INTO testcat1.ns1.ns2.tbl AS target
-          |USING testcat2.ns1.ns2.tbl AS source
-          |ON target.col1 = source.col1
-          |WHEN MATCHED AND (target.col2='delete') THEN DELETE
-          |WHEN MATCHED AND (target.col2='update1') THEN UPDATE SET target.col2 = source.col2
-          |WHEN MATCHED AND (target.col2='update2') THEN UPDATE SET target.col2 = source.col2
-          |WHEN NOT MATCHED AND (target.col2='insert')
-          |THEN INSERT (target.col1, target.col2) values (source.col1, source.col2)
-        """.stripMargin)
-    }
-
-    assert(exc.getMessage.contains("There should be at most 2 'WHEN MATCHED' clauses."))
+  test("merge into table: multi matched and not matched clauses") {
+    parseCompare(
+      """
+        |MERGE INTO testcat1.ns1.ns2.tbl AS target
+        |USING testcat2.ns1.ns2.tbl AS source
+        |ON target.col1 = source.col1
+        |WHEN MATCHED AND (target.col2='delete') THEN DELETE
+        |WHEN MATCHED AND (target.col2='update1') THEN UPDATE SET target.col2 = 1
+        |WHEN MATCHED AND (target.col2='update2') THEN UPDATE SET target.col2 = 2
+        |WHEN NOT MATCHED AND (target.col2='insert1')
+        |THEN INSERT (target.col1, target.col2) values (source.col1, 1)
+        |WHEN NOT MATCHED AND (target.col2='insert2')
+        |THEN INSERT (target.col1, target.col2) values (source.col1, 2)
+      """.stripMargin,
+      MergeIntoTable(
+        SubqueryAlias("target", UnresolvedRelation(Seq("testcat1", "ns1", "ns2", "tbl"))),
+        SubqueryAlias("source", UnresolvedRelation(Seq("testcat2", "ns1", "ns2", "tbl"))),
+        EqualTo(UnresolvedAttribute("target.col1"), UnresolvedAttribute("source.col1")),
+        Seq(DeleteAction(Some(EqualTo(UnresolvedAttribute("target.col2"), Literal("delete")))),
+          UpdateAction(Some(EqualTo(UnresolvedAttribute("target.col2"), Literal("update1"))),
+            Seq(Assignment(UnresolvedAttribute("target.col2"), Literal(1)))),
+          UpdateAction(Some(EqualTo(UnresolvedAttribute("target.col2"), Literal("update2"))),
+            Seq(Assignment(UnresolvedAttribute("target.col2"), Literal(2))))),
+        Seq(InsertAction(Some(EqualTo(UnresolvedAttribute("target.col2"), Literal("insert1"))),
+          Seq(Assignment(UnresolvedAttribute("target.col1"), UnresolvedAttribute("source.col1")),
+            Assignment(UnresolvedAttribute("target.col2"), Literal(1)))),
+          InsertAction(Some(EqualTo(UnresolvedAttribute("target.col2"), Literal("insert2"))),
+            Seq(Assignment(UnresolvedAttribute("target.col1"), UnresolvedAttribute("source.col1")),
+              Assignment(UnresolvedAttribute("target.col2"), Literal(2)))))))
   }
 
-  test("merge into table: at most one not matched clause") {
+  test("merge into table: only the last matched clause can omit the condition") {
     val exc = intercept[ParseException] {
       parsePlan(
         """
           |MERGE INTO testcat1.ns1.ns2.tbl AS target
           |USING testcat2.ns1.ns2.tbl AS source
           |ON target.col1 = source.col1
-          |WHEN MATCHED AND (target.col2='delete') THEN DELETE
-          |WHEN MATCHED AND (target.col2='update1') THEN UPDATE SET target.col2 = source.col2
-          |WHEN NOT MATCHED AND (target.col2='insert1')
-          |THEN INSERT (target.col1, target.col2) values (source.col1, source.col2)
-          |WHEN NOT MATCHED AND (target.col2='insert2')
+          |WHEN MATCHED AND (target.col2 == 'update1') THEN UPDATE SET target.col2 = 1
+          |WHEN MATCHED THEN UPDATE SET target.col2 = 2
+          |WHEN MATCHED THEN DELETE
+          |WHEN NOT MATCHED AND (target.col2='insert')
           |THEN INSERT (target.col1, target.col2) values (source.col1, source.col2)
         """.stripMargin)
     }
 
-    assert(exc.getMessage.contains("There should be at most 1 'WHEN NOT MATCHED' clause."))
+    assert(exc.getMessage.contains("only the last MATCHED clause can omit the condition"))
   }
 
-  test("merge into table: the first matched clause must have a condition if there's a second") {
+  test("merge into table: only the last not matched clause can omit the condition") {
     val exc = intercept[ParseException] {
       parsePlan(
         """
           |MERGE INTO testcat1.ns1.ns2.tbl AS target
           |USING testcat2.ns1.ns2.tbl AS source
           |ON target.col1 = source.col1
+          |WHEN MATCHED AND (target.col2 == 'update') THEN UPDATE SET target.col2 = source.col2
           |WHEN MATCHED THEN DELETE
-          |WHEN MATCHED THEN UPDATE SET target.col2 = source.col2
-          |WHEN NOT MATCHED AND (target.col2='insert')
+          |WHEN NOT MATCHED AND (target.col2='insert1')
+          |THEN INSERT (target.col1, target.col2) values (source.col1, 1)
+          |WHEN NOT MATCHED
+          |THEN INSERT (target.col1, target.col2) values (source.col1, 2)
+          |WHEN NOT MATCHED
           |THEN INSERT (target.col1, target.col2) values (source.col1, source.col2)
         """.stripMargin)
     }
 
-    assert(exc.getMessage.contains("the first MATCHED clause must have a condition"))
+    assert(exc.getMessage.contains("only the last NOT MATCHED clause can omit the condition"))
   }
 
   test("merge into table: there must be a when (not) matched condition") {
@@ -1201,75 +1626,6 @@ class DDLParserSuite extends AnalysisTest {
     assert(exc.getMessage.contains("There must be at least one WHEN clause in a MERGE statement"))
   }
 
-  test("merge into table: there can be only a single use DELETE or UPDATE") {
-    Seq("UPDATE SET *", "DELETE").foreach { op =>
-      val exc = intercept[ParseException] {
-        parsePlan(
-          s"""
-             |MERGE INTO testcat1.ns1.ns2.tbl AS target
-             |USING testcat2.ns1.ns2.tbl AS source
-             |ON target.col1 = source.col1
-             |WHEN MATCHED AND (target.col2='delete') THEN $op
-             |WHEN MATCHED THEN $op
-             |WHEN NOT MATCHED AND (target.col2='insert')
-             |THEN INSERT (target.col1, target.col2) values (source.col1, source.col2)
-           """.stripMargin)
-      }
-
-      assert(exc.getMessage.contains(
-        "UPDATE and DELETE can appear at most once in MATCHED clauses"))
-    }
-  }
-
-  test("show tables") {
-    comparePlans(
-      parsePlan("SHOW TABLES"),
-      ShowTables(UnresolvedNamespace(Seq.empty[String]), None))
-    comparePlans(
-      parsePlan("SHOW TABLES '*test*'"),
-      ShowTables(UnresolvedNamespace(Seq.empty[String]), Some("*test*")))
-    comparePlans(
-      parsePlan("SHOW TABLES LIKE '*test*'"),
-      ShowTables(UnresolvedNamespace(Seq.empty[String]), Some("*test*")))
-    comparePlans(
-      parsePlan("SHOW TABLES FROM testcat.ns1.ns2.tbl"),
-      ShowTables(UnresolvedNamespace(Seq("testcat", "ns1", "ns2", "tbl")), None))
-    comparePlans(
-      parsePlan("SHOW TABLES IN testcat.ns1.ns2.tbl"),
-      ShowTables(UnresolvedNamespace(Seq("testcat", "ns1", "ns2", "tbl")), None))
-    comparePlans(
-      parsePlan("SHOW TABLES IN ns1 '*test*'"),
-      ShowTables(UnresolvedNamespace(Seq("ns1")), Some("*test*")))
-    comparePlans(
-      parsePlan("SHOW TABLES IN ns1 LIKE '*test*'"),
-      ShowTables(UnresolvedNamespace(Seq("ns1")), Some("*test*")))
-  }
-
-  test("show table extended") {
-    comparePlans(
-      parsePlan("SHOW TABLE EXTENDED LIKE '*test*'"),
-      ShowTableStatement(None, "*test*", None))
-    comparePlans(
-      parsePlan("SHOW TABLE EXTENDED FROM testcat.ns1.ns2 LIKE '*test*'"),
-      ShowTableStatement(Some(Seq("testcat", "ns1", "ns2")), "*test*", None))
-    comparePlans(
-      parsePlan("SHOW TABLE EXTENDED IN testcat.ns1.ns2 LIKE '*test*'"),
-      ShowTableStatement(Some(Seq("testcat", "ns1", "ns2")), "*test*", None))
-    comparePlans(
-      parsePlan("SHOW TABLE EXTENDED LIKE '*test*' PARTITION(ds='2008-04-09', hr=11)"),
-      ShowTableStatement(None, "*test*", Some(Map("ds" -> "2008-04-09", "hr" -> "11"))))
-    comparePlans(
-      parsePlan("SHOW TABLE EXTENDED FROM testcat.ns1.ns2 LIKE '*test*' " +
-        "PARTITION(ds='2008-04-09')"),
-      ShowTableStatement(Some(Seq("testcat", "ns1", "ns2")), "*test*",
-        Some(Map("ds" -> "2008-04-09"))))
-    comparePlans(
-      parsePlan("SHOW TABLE EXTENDED IN testcat.ns1.ns2 LIKE '*test*' " +
-        "PARTITION(ds='2008-04-09')"),
-      ShowTableStatement(Some(Seq("testcat", "ns1", "ns2")), "*test*",
-        Some(Map("ds" -> "2008-04-09"))))
-  }
-
   test("show views") {
     comparePlans(
       parsePlan("SHOW VIEWS"),
@@ -1408,124 +1764,107 @@ class DDLParserSuite extends AnalysisTest {
   test("set namespace properties") {
     comparePlans(
       parsePlan("ALTER DATABASE a.b.c SET PROPERTIES ('a'='a', 'b'='b', 'c'='c')"),
-      AlterNamespaceSetProperties(
+      SetNamespaceProperties(
         UnresolvedNamespace(Seq("a", "b", "c")), Map("a" -> "a", "b" -> "b", "c" -> "c")))
 
     comparePlans(
       parsePlan("ALTER SCHEMA a.b.c SET PROPERTIES ('a'='a')"),
-      AlterNamespaceSetProperties(
+      SetNamespaceProperties(
         UnresolvedNamespace(Seq("a", "b", "c")), Map("a" -> "a")))
 
     comparePlans(
       parsePlan("ALTER NAMESPACE a.b.c SET PROPERTIES ('b'='b')"),
-      AlterNamespaceSetProperties(
+      SetNamespaceProperties(
         UnresolvedNamespace(Seq("a", "b", "c")), Map("b" -> "b")))
 
     comparePlans(
       parsePlan("ALTER DATABASE a.b.c SET DBPROPERTIES ('a'='a', 'b'='b', 'c'='c')"),
-      AlterNamespaceSetProperties(
+      SetNamespaceProperties(
         UnresolvedNamespace(Seq("a", "b", "c")), Map("a" -> "a", "b" -> "b", "c" -> "c")))
 
     comparePlans(
       parsePlan("ALTER SCHEMA a.b.c SET DBPROPERTIES ('a'='a')"),
-      AlterNamespaceSetProperties(
+      SetNamespaceProperties(
         UnresolvedNamespace(Seq("a", "b", "c")), Map("a" -> "a")))
 
     comparePlans(
       parsePlan("ALTER NAMESPACE a.b.c SET DBPROPERTIES ('b'='b')"),
-      AlterNamespaceSetProperties(
+      SetNamespaceProperties(
         UnresolvedNamespace(Seq("a", "b", "c")), Map("b" -> "b")))
   }
 
   test("set namespace location") {
     comparePlans(
       parsePlan("ALTER DATABASE a.b.c SET LOCATION '/home/user/db'"),
-      AlterNamespaceSetLocation(
+      SetNamespaceLocation(
         UnresolvedNamespace(Seq("a", "b", "c")), "/home/user/db"))
 
     comparePlans(
       parsePlan("ALTER SCHEMA a.b.c SET LOCATION '/home/user/db'"),
-      AlterNamespaceSetLocation(
+      SetNamespaceLocation(
         UnresolvedNamespace(Seq("a", "b", "c")), "/home/user/db"))
 
     comparePlans(
       parsePlan("ALTER NAMESPACE a.b.c SET LOCATION '/home/user/db'"),
-      AlterNamespaceSetLocation(
+      SetNamespaceLocation(
         UnresolvedNamespace(Seq("a", "b", "c")), "/home/user/db"))
   }
 
-  test("show databases: basic") {
-    comparePlans(
-      parsePlan("SHOW DATABASES"),
-      ShowNamespaces(UnresolvedNamespace(Seq.empty[String]), None))
-    comparePlans(
-      parsePlan("SHOW DATABASES LIKE 'defau*'"),
-      ShowNamespaces(UnresolvedNamespace(Seq.empty[String]), Some("defau*")))
-  }
-
-  test("show databases: FROM/IN operator is not allowed") {
-    def verify(sql: String): Unit = {
-      val exc = intercept[ParseException] { parsePlan(sql) }
-      assert(exc.getMessage.contains("FROM/IN operator is not allowed in SHOW DATABASES"))
-    }
-
-    verify("SHOW DATABASES FROM testcat.ns1.ns2")
-    verify("SHOW DATABASES IN testcat.ns1.ns2")
-  }
-
-  test("show namespaces") {
-    comparePlans(
-      parsePlan("SHOW NAMESPACES"),
-      ShowNamespaces(UnresolvedNamespace(Seq.empty[String]), None))
-    comparePlans(
-      parsePlan("SHOW NAMESPACES FROM testcat.ns1.ns2"),
-      ShowNamespaces(UnresolvedNamespace(Seq("testcat", "ns1", "ns2")), None))
-    comparePlans(
-      parsePlan("SHOW NAMESPACES IN testcat.ns1.ns2"),
-      ShowNamespaces(UnresolvedNamespace(Seq("testcat", "ns1", "ns2")), None))
-    comparePlans(
-      parsePlan("SHOW NAMESPACES IN testcat.ns1 LIKE '*pattern*'"),
-      ShowNamespaces(UnresolvedNamespace(Seq("testcat", "ns1")), Some("*pattern*")))
-  }
-
   test("analyze table statistics") {
     comparePlans(parsePlan("analyze table a.b.c compute statistics"),
-      AnalyzeTableStatement(Seq("a", "b", "c"), Map.empty, noScan = false))
+      AnalyzeTable(
+        UnresolvedTableOrView(Seq("a", "b", "c"), "ANALYZE TABLE", allowTempView = false),
+        Map.empty, noScan = false))
     comparePlans(parsePlan("analyze table a.b.c compute statistics noscan"),
-      AnalyzeTableStatement(Seq("a", "b", "c"), Map.empty, noScan = true))
+      AnalyzeTable(
+        UnresolvedTableOrView(Seq("a", "b", "c"), "ANALYZE TABLE", allowTempView = false),
+        Map.empty, noScan = true))
     comparePlans(parsePlan("analyze table a.b.c partition (a) compute statistics nOscAn"),
-      AnalyzeTableStatement(Seq("a", "b", "c"), Map("a" -> None), noScan = true))
+      AnalyzeTable(
+        UnresolvedTableOrView(Seq("a", "b", "c"), "ANALYZE TABLE", allowTempView = false),
+        Map("a" -> None), noScan = true))
 
     // Partitions specified
     comparePlans(
       parsePlan("ANALYZE TABLE a.b.c PARTITION(ds='2008-04-09', hr=11) COMPUTE STATISTICS"),
-      AnalyzeTableStatement(
-        Seq("a", "b", "c"), Map("ds" -> Some("2008-04-09"), "hr" -> Some("11")), noScan = false))
+      AnalyzeTable(
+        UnresolvedTableOrView(Seq("a", "b", "c"), "ANALYZE TABLE", allowTempView = false),
+        Map("ds" -> Some("2008-04-09"), "hr" -> Some("11")), noScan = false))
     comparePlans(
       parsePlan("ANALYZE TABLE a.b.c PARTITION(ds='2008-04-09', hr=11) COMPUTE STATISTICS noscan"),
-      AnalyzeTableStatement(
-        Seq("a", "b", "c"), Map("ds" -> Some("2008-04-09"), "hr" -> Some("11")), noScan = true))
+      AnalyzeTable(
+        UnresolvedTableOrView(Seq("a", "b", "c"), "ANALYZE TABLE", allowTempView = false),
+        Map("ds" -> Some("2008-04-09"), "hr" -> Some("11")), noScan = true))
     comparePlans(
       parsePlan("ANALYZE TABLE a.b.c PARTITION(ds='2008-04-09') COMPUTE STATISTICS noscan"),
-      AnalyzeTableStatement(Seq("a", "b", "c"), Map("ds" -> Some("2008-04-09")), noScan = true))
+      AnalyzeTable(
+        UnresolvedTableOrView(Seq("a", "b", "c"), "ANALYZE TABLE", allowTempView = false),
+        Map("ds" -> Some("2008-04-09")), noScan = true))
     comparePlans(
       parsePlan("ANALYZE TABLE a.b.c PARTITION(ds='2008-04-09', hr) COMPUTE STATISTICS"),
-      AnalyzeTableStatement(
-        Seq("a", "b", "c"), Map("ds" -> Some("2008-04-09"), "hr" -> None), noScan = false))
+      AnalyzeTable(
+        UnresolvedTableOrView(Seq("a", "b", "c"), "ANALYZE TABLE", allowTempView = false),
+        Map("ds" -> Some("2008-04-09"), "hr" -> None), noScan = false))
     comparePlans(
       parsePlan("ANALYZE TABLE a.b.c PARTITION(ds='2008-04-09', hr) COMPUTE STATISTICS noscan"),
-      AnalyzeTableStatement(
-        Seq("a", "b", "c"), Map("ds" -> Some("2008-04-09"), "hr" -> None), noScan = true))
+      AnalyzeTable(
+        UnresolvedTableOrView(Seq("a", "b", "c"), "ANALYZE TABLE", allowTempView = false),
+        Map("ds" -> Some("2008-04-09"), "hr" -> None), noScan = true))
     comparePlans(
       parsePlan("ANALYZE TABLE a.b.c PARTITION(ds, hr=11) COMPUTE STATISTICS noscan"),
-      AnalyzeTableStatement(
-        Seq("a", "b", "c"), Map("ds" -> None, "hr" -> Some("11")), noScan = true))
+      AnalyzeTable(
+        UnresolvedTableOrView(Seq("a", "b", "c"), "ANALYZE TABLE", allowTempView = false),
+        Map("ds" -> None, "hr" -> Some("11")), noScan = true))
     comparePlans(
       parsePlan("ANALYZE TABLE a.b.c PARTITION(ds, hr) COMPUTE STATISTICS"),
-      AnalyzeTableStatement(Seq("a", "b", "c"), Map("ds" -> None, "hr" -> None), noScan = false))
+      AnalyzeTable(
+        UnresolvedTableOrView(Seq("a", "b", "c"), "ANALYZE TABLE", allowTempView = false),
+        Map("ds" -> None, "hr" -> None), noScan = false))
     comparePlans(
       parsePlan("ANALYZE TABLE a.b.c PARTITION(ds, hr) COMPUTE STATISTICS noscan"),
-      AnalyzeTableStatement(Seq("a", "b", "c"), Map("ds" -> None, "hr" -> None), noScan = true))
+      AnalyzeTable(
+        UnresolvedTableOrView(Seq("a", "b", "c"), "ANALYZE TABLE", allowTempView = false),
+        Map("ds" -> None, "hr" -> None), noScan = true))
 
     intercept("analyze table a.b.c compute statistics xxxx",
       "Expected `NOSCAN` instead of `xxxx`")
@@ -1533,12 +1872,24 @@ class DDLParserSuite extends AnalysisTest {
       "Expected `NOSCAN` instead of `xxxx`")
   }
 
+  test("SPARK-33687: analyze tables statistics") {
+    comparePlans(parsePlan("ANALYZE TABLES IN a.b.c COMPUTE STATISTICS"),
+      AnalyzeTables(UnresolvedNamespace(Seq("a", "b", "c")), noScan = false))
+    comparePlans(parsePlan("ANALYZE TABLES FROM a COMPUTE STATISTICS NOSCAN"),
+      AnalyzeTables(UnresolvedNamespace(Seq("a")), noScan = true))
+    intercept("ANALYZE TABLES IN a.b.c COMPUTE STATISTICS xxxx",
+      "Expected `NOSCAN` instead of `xxxx`")
+  }
+
   test("analyze table column statistics") {
     intercept("ANALYZE TABLE a.b.c COMPUTE STATISTICS FOR COLUMNS", "")
 
     comparePlans(
       parsePlan("ANALYZE TABLE a.b.c COMPUTE STATISTICS FOR COLUMNS key, value"),
-      AnalyzeColumnStatement(Seq("a", "b", "c"), Option(Seq("key", "value")), allColumns = false))
+      AnalyzeColumn(
+        UnresolvedTableOrView(Seq("a", "b", "c"), "ANALYZE TABLE ... FOR COLUMNS ...", true),
+        Option(Seq("key", "value")),
+        allColumns = false))
 
     // Partition specified - should be ignored
     comparePlans(
@@ -1547,7 +1898,10 @@ class DDLParserSuite extends AnalysisTest {
            |ANALYZE TABLE a.b.c PARTITION(ds='2017-06-10')
            |COMPUTE STATISTICS FOR COLUMNS key, value
          """.stripMargin),
-      AnalyzeColumnStatement(Seq("a", "b", "c"), Option(Seq("key", "value")), allColumns = false))
+      AnalyzeColumn(
+        UnresolvedTableOrView(Seq("a", "b", "c"), "ANALYZE TABLE ... FOR COLUMNS ...", true),
+        Option(Seq("key", "value")),
+        allColumns = false))
 
     // Partition specified should be ignored in case of COMPUTE STATISTICS FOR ALL COLUMNS
     comparePlans(
@@ -1556,7 +1910,10 @@ class DDLParserSuite extends AnalysisTest {
            |ANALYZE TABLE a.b.c PARTITION(ds='2017-06-10')
            |COMPUTE STATISTICS FOR ALL COLUMNS
          """.stripMargin),
-      AnalyzeColumnStatement(Seq("a", "b", "c"), None, allColumns = true))
+      AnalyzeColumn(
+        UnresolvedTableOrView(Seq("a", "b", "c"), "ANALYZE TABLE ... FOR ALL COLUMNS", true),
+        None,
+        allColumns = true))
 
     intercept("ANALYZE TABLE a.b.c COMPUTE STATISTICS FOR ALL COLUMNS key, value",
       "mismatched input 'key' expecting {<EOF>, ';'}")
@@ -1564,24 +1921,33 @@ class DDLParserSuite extends AnalysisTest {
       "missing 'COLUMNS' at '<EOF>'")
   }
 
-  test("MSCK REPAIR TABLE") {
-    comparePlans(
-      parsePlan("MSCK REPAIR TABLE a.b.c"),
-      RepairTableStatement(Seq("a", "b", "c")))
-  }
-
   test("LOAD DATA INTO table") {
     comparePlans(
       parsePlan("LOAD DATA INPATH 'filepath' INTO TABLE a.b.c"),
-      LoadDataStatement(Seq("a", "b", "c"), "filepath", false, false, None))
+      LoadData(
+        UnresolvedTable(Seq("a", "b", "c"), "LOAD DATA", None),
+        "filepath",
+        false,
+        false,
+        None))
 
     comparePlans(
       parsePlan("LOAD DATA LOCAL INPATH 'filepath' INTO TABLE a.b.c"),
-      LoadDataStatement(Seq("a", "b", "c"), "filepath", true, false, None))
+      LoadData(
+        UnresolvedTable(Seq("a", "b", "c"), "LOAD DATA", None),
+        "filepath",
+        true,
+        false,
+        None))
 
     comparePlans(
       parsePlan("LOAD DATA LOCAL INPATH 'filepath' OVERWRITE INTO TABLE a.b.c"),
-      LoadDataStatement(Seq("a", "b", "c"), "filepath", true, true, None))
+      LoadData(
+        UnresolvedTable(Seq("a", "b", "c"), "LOAD DATA", None),
+        "filepath",
+        true,
+        true,
+        None))
 
     comparePlans(
       parsePlan(
@@ -1589,8 +1955,8 @@ class DDLParserSuite extends AnalysisTest {
            |LOAD DATA LOCAL INPATH 'filepath' OVERWRITE INTO TABLE a.b.c
            |PARTITION(ds='2017-06-10')
          """.stripMargin),
-      LoadDataStatement(
-        Seq("a", "b", "c"),
+      LoadData(
+        UnresolvedTable(Seq("a", "b", "c"), "LOAD DATA", None),
         "filepath",
         true,
         true,
@@ -1600,21 +1966,43 @@ class DDLParserSuite extends AnalysisTest {
   test("SHOW CREATE table") {
     comparePlans(
       parsePlan("SHOW CREATE TABLE a.b.c"),
-      ShowCreateTableStatement(Seq("a", "b", "c")))
+      ShowCreateTable(
+        UnresolvedTableOrView(Seq("a", "b", "c"), "SHOW CREATE TABLE", allowTempView = false)))
+
+    comparePlans(
+      parsePlan("SHOW CREATE TABLE a.b.c AS SERDE"),
+      ShowCreateTable(
+        UnresolvedTableOrView(Seq("a", "b", "c"), "SHOW CREATE TABLE", allowTempView = false),
+        asSerde = true))
   }
 
   test("CACHE TABLE") {
     comparePlans(
       parsePlan("CACHE TABLE a.b.c"),
-      CacheTableStatement(Seq("a", "b", "c"), None, false, Map.empty))
+      CacheTable(
+        UnresolvedRelation(Seq("a", "b", "c")), Seq("a", "b", "c"), false, Map.empty))
+
+    comparePlans(
+      parsePlan("CACHE TABLE t AS SELECT * FROM testData"),
+      CacheTableAsSelect(
+        "t",
+        Project(Seq(UnresolvedStar(None)), UnresolvedRelation(Seq("testData"))),
+        "SELECT * FROM testData",
+        false,
+        Map.empty))
 
     comparePlans(
       parsePlan("CACHE LAZY TABLE a.b.c"),
-      CacheTableStatement(Seq("a", "b", "c"), None, true, Map.empty))
+      CacheTable(
+        UnresolvedRelation(Seq("a", "b", "c")), Seq("a", "b", "c"), true, Map.empty))
 
     comparePlans(
       parsePlan("CACHE LAZY TABLE a.b.c OPTIONS('storageLevel' 'DISK_ONLY')"),
-      CacheTableStatement(Seq("a", "b", "c"), None, true, Map("storageLevel" -> "DISK_ONLY")))
+      CacheTable(
+        UnresolvedRelation(Seq("a", "b", "c")),
+        Seq("a", "b", "c"),
+        true,
+        Map("storageLevel" -> "DISK_ONLY")))
 
     intercept("CACHE TABLE a.b.c AS SELECT * FROM testData",
       "It is not allowed to add catalog/namespace prefix a.b")
@@ -1623,53 +2011,17 @@ class DDLParserSuite extends AnalysisTest {
   test("UNCACHE TABLE") {
     comparePlans(
       parsePlan("UNCACHE TABLE a.b.c"),
-      UncacheTableStatement(Seq("a", "b", "c"), ifExists = false))
+      UncacheTable(UnresolvedRelation(Seq("a", "b", "c")), ifExists = false))
 
     comparePlans(
       parsePlan("UNCACHE TABLE IF EXISTS a.b.c"),
-      UncacheTableStatement(Seq("a", "b", "c"), ifExists = true))
-  }
-
-  test("TRUNCATE table") {
-    comparePlans(
-      parsePlan("TRUNCATE TABLE a.b.c"),
-      TruncateTableStatement(Seq("a", "b", "c"), None))
-
-    comparePlans(
-      parsePlan("TRUNCATE TABLE a.b.c PARTITION(ds='2017-06-10')"),
-      TruncateTableStatement(Seq("a", "b", "c"), Some(Map("ds" -> "2017-06-10"))))
-  }
-
-  test("SHOW PARTITIONS") {
-    val sql1 = "SHOW PARTITIONS t1"
-    val sql2 = "SHOW PARTITIONS db1.t1"
-    val sql3 = "SHOW PARTITIONS t1 PARTITION(partcol1='partvalue', partcol2='partvalue')"
-    val sql4 = "SHOW PARTITIONS a.b.c"
-    val sql5 = "SHOW PARTITIONS a.b.c PARTITION(ds='2017-06-10')"
-
-    val parsed1 = parsePlan(sql1)
-    val expected1 = ShowPartitionsStatement(Seq("t1"), None)
-    val parsed2 = parsePlan(sql2)
-    val expected2 = ShowPartitionsStatement(Seq("db1", "t1"), None)
-    val parsed3 = parsePlan(sql3)
-    val expected3 = ShowPartitionsStatement(Seq("t1"),
-      Some(Map("partcol1" -> "partvalue", "partcol2" -> "partvalue")))
-    val parsed4 = parsePlan(sql4)
-    val expected4 = ShowPartitionsStatement(Seq("a", "b", "c"), None)
-    val parsed5 = parsePlan(sql5)
-    val expected5 = ShowPartitionsStatement(Seq("a", "b", "c"), Some(Map("ds" -> "2017-06-10")))
-
-    comparePlans(parsed1, expected1)
-    comparePlans(parsed2, expected2)
-    comparePlans(parsed3, expected3)
-    comparePlans(parsed4, expected4)
-    comparePlans(parsed5, expected5)
+      UncacheTable(UnresolvedRelation(Seq("a", "b", "c")), ifExists = true))
   }
 
   test("REFRESH TABLE") {
     comparePlans(
       parsePlan("REFRESH TABLE a.b.c"),
-      RefreshTableStatement(Seq("a", "b", "c")))
+      RefreshTable(UnresolvedTableOrView(Seq("a", "b", "c"), "REFRESH TABLE", true)))
   }
 
   test("show columns") {
@@ -1679,13 +2031,15 @@ class DDLParserSuite extends AnalysisTest {
     val sql4 = "SHOW COLUMNS FROM db1.t1 IN db1"
 
     val parsed1 = parsePlan(sql1)
-    val expected1 = ShowColumnsStatement(Seq("t1"), None)
+    val expected1 = ShowColumns(UnresolvedTableOrView(Seq("t1"), "SHOW COLUMNS", true), None)
     val parsed2 = parsePlan(sql2)
-    val expected2 = ShowColumnsStatement(Seq("db1", "t1"), None)
+    val expected2 = ShowColumns(UnresolvedTableOrView(Seq("db1", "t1"), "SHOW COLUMNS", true), None)
     val parsed3 = parsePlan(sql3)
-    val expected3 = ShowColumnsStatement(Seq("t1"), Some(Seq("db1")))
+    val expected3 =
+      ShowColumns(UnresolvedTableOrView(Seq("db1", "t1"), "SHOW COLUMNS", true), Some(Seq("db1")))
     val parsed4 = parsePlan(sql4)
-    val expected4 = ShowColumnsStatement(Seq("db1", "t1"), Some(Seq("db1")))
+    val expected4 =
+      ShowColumns(UnresolvedTableOrView(Seq("db1", "t1"), "SHOW COLUMNS", true), Some(Seq("db1")))
 
     comparePlans(parsed1, expected1)
     comparePlans(parsed2, expected2)
@@ -1693,39 +2047,6 @@ class DDLParserSuite extends AnalysisTest {
     comparePlans(parsed4, expected4)
   }
 
-  test("alter table: recover partitions") {
-    comparePlans(
-      parsePlan("ALTER TABLE a.b.c RECOVER PARTITIONS"),
-      AlterTableRecoverPartitionsStatement(Seq("a", "b", "c")))
-  }
-
-  test("alter table: add partition") {
-    val sql1 =
-      """
-        |ALTER TABLE a.b.c ADD IF NOT EXISTS PARTITION
-        |(dt='2008-08-08', country='us') LOCATION 'location1' PARTITION
-        |(dt='2009-09-09', country='uk')
-      """.stripMargin
-    val sql2 = "ALTER TABLE a.b.c ADD PARTITION (dt='2008-08-08') LOCATION 'loc'"
-
-    val parsed1 = parsePlan(sql1)
-    val parsed2 = parsePlan(sql2)
-
-    val expected1 = AlterTableAddPartitionStatement(
-      Seq("a", "b", "c"),
-      Seq(
-        (Map("dt" -> "2008-08-08", "country" -> "us"), Some("location1")),
-        (Map("dt" -> "2009-09-09", "country" -> "uk"), None)),
-      ifNotExists = true)
-    val expected2 = AlterTableAddPartitionStatement(
-      Seq("a", "b", "c"),
-      Seq((Map("dt" -> "2008-08-08"), Some("loc"))),
-      ifNotExists = false)
-
-    comparePlans(parsed1, expected1)
-    comparePlans(parsed2, expected2)
-  }
-
   test("alter view: add partition (not supported)") {
     assertUnsupported(
       """
@@ -1735,82 +2056,6 @@ class DDLParserSuite extends AnalysisTest {
       """.stripMargin)
   }
 
-  test("alter table: rename partition") {
-    val sql1 =
-      """
-        |ALTER TABLE table_name PARTITION (dt='2008-08-08', country='us')
-        |RENAME TO PARTITION (dt='2008-09-09', country='uk')
-      """.stripMargin
-    val parsed1 = parsePlan(sql1)
-    val expected1 = AlterTableRenamePartitionStatement(
-      Seq("table_name"),
-      Map("dt" -> "2008-08-08", "country" -> "us"),
-      Map("dt" -> "2008-09-09", "country" -> "uk"))
-    comparePlans(parsed1, expected1)
-
-    val sql2 =
-      """
-        |ALTER TABLE a.b.c PARTITION (ds='2017-06-10')
-        |RENAME TO PARTITION (ds='2018-06-10')
-      """.stripMargin
-    val parsed2 = parsePlan(sql2)
-    val expected2 = AlterTableRenamePartitionStatement(
-      Seq("a", "b", "c"),
-      Map("ds" -> "2017-06-10"),
-      Map("ds" -> "2018-06-10"))
-    comparePlans(parsed2, expected2)
-  }
-
-  // ALTER TABLE table_name DROP [IF EXISTS] PARTITION spec1[, PARTITION spec2, ...]
-  // ALTER VIEW table_name DROP [IF EXISTS] PARTITION spec1[, PARTITION spec2, ...]
-  test("alter table: drop partition") {
-    val sql1_table =
-      """
-        |ALTER TABLE table_name DROP IF EXISTS PARTITION
-        |(dt='2008-08-08', country='us'), PARTITION (dt='2009-09-09', country='uk')
-      """.stripMargin
-    val sql2_table =
-      """
-        |ALTER TABLE table_name DROP PARTITION
-        |(dt='2008-08-08', country='us'), PARTITION (dt='2009-09-09', country='uk')
-      """.stripMargin
-    val sql1_view = sql1_table.replace("TABLE", "VIEW")
-    val sql2_view = sql2_table.replace("TABLE", "VIEW")
-
-    val parsed1_table = parsePlan(sql1_table)
-    val parsed2_table = parsePlan(sql2_table)
-    val parsed1_purge = parsePlan(sql1_table + " PURGE")
-
-    assertUnsupported(sql1_view)
-    assertUnsupported(sql2_view)
-
-    val expected1_table = AlterTableDropPartitionStatement(
-      Seq("table_name"),
-      Seq(
-        Map("dt" -> "2008-08-08", "country" -> "us"),
-        Map("dt" -> "2009-09-09", "country" -> "uk")),
-      ifExists = true,
-      purge = false,
-      retainData = false)
-    val expected2_table = expected1_table.copy(ifExists = false)
-    val expected1_purge = expected1_table.copy(purge = true)
-
-    comparePlans(parsed1_table, expected1_table)
-    comparePlans(parsed2_table, expected2_table)
-    comparePlans(parsed1_purge, expected1_purge)
-
-    val sql3_table = "ALTER TABLE a.b.c DROP IF EXISTS PARTITION (ds='2017-06-10')"
-    val expected3_table = AlterTableDropPartitionStatement(
-      Seq("a", "b", "c"),
-      Seq(Map("ds" -> "2017-06-10")),
-      ifExists = true,
-      purge = false,
-      retainData = false)
-
-    val parsed3_table = parsePlan(sql3_table)
-    comparePlans(parsed3_table, expected3_table)
-  }
-
   test("show current namespace") {
     comparePlans(
       parsePlan("SHOW CURRENT NAMESPACE"),
@@ -1819,9 +2064,13 @@ class DDLParserSuite extends AnalysisTest {
 
   test("alter table: SerDe properties") {
     val sql1 = "ALTER TABLE table_name SET SERDE 'org.apache.class'"
+    val hint = Some("Please use ALTER VIEW instead.")
     val parsed1 = parsePlan(sql1)
-    val expected1 = AlterTableSerDePropertiesStatement(
-      Seq("table_name"), Some("org.apache.class"), None, None)
+    val expected1 = SetTableSerDeProperties(
+      UnresolvedTable(Seq("table_name"), "ALTER TABLE ... SET [SERDE|SERDEPROPERTIES]", hint),
+      Some("org.apache.class"),
+      None,
+      None)
     comparePlans(parsed1, expected1)
 
     val sql2 =
@@ -1830,8 +2079,8 @@ class DDLParserSuite extends AnalysisTest {
         |WITH SERDEPROPERTIES ('columns'='foo,bar', 'field.delim' = ',')
       """.stripMargin
     val parsed2 = parsePlan(sql2)
-    val expected2 = AlterTableSerDePropertiesStatement(
-      Seq("table_name"),
+    val expected2 = SetTableSerDeProperties(
+      UnresolvedTable(Seq("table_name"), "ALTER TABLE ... SET [SERDE|SERDEPROPERTIES]", hint),
       Some("org.apache.class"),
       Some(Map("columns" -> "foo,bar", "field.delim" -> ",")),
       None)
@@ -1843,8 +2092,11 @@ class DDLParserSuite extends AnalysisTest {
         |SET SERDEPROPERTIES ('columns'='foo,bar', 'field.delim' = ',')
       """.stripMargin
     val parsed3 = parsePlan(sql3)
-    val expected3 = AlterTableSerDePropertiesStatement(
-      Seq("table_name"), None, Some(Map("columns" -> "foo,bar", "field.delim" -> ",")), None)
+    val expected3 = SetTableSerDeProperties(
+      UnresolvedTable(Seq("table_name"), "ALTER TABLE ... SET [SERDE|SERDEPROPERTIES]", hint),
+      None,
+      Some(Map("columns" -> "foo,bar", "field.delim" -> ",")),
+      None)
     comparePlans(parsed3, expected3)
 
     val sql4 =
@@ -1854,8 +2106,8 @@ class DDLParserSuite extends AnalysisTest {
         |WITH SERDEPROPERTIES ('columns'='foo,bar', 'field.delim' = ',')
       """.stripMargin
     val parsed4 = parsePlan(sql4)
-    val expected4 = AlterTableSerDePropertiesStatement(
-      Seq("table_name"),
+    val expected4 = SetTableSerDeProperties(
+      UnresolvedTable(Seq("table_name"), "ALTER TABLE ... SET [SERDE|SERDEPROPERTIES]", hint),
       Some("org.apache.class"),
       Some(Map("columns" -> "foo,bar", "field.delim" -> ",")),
       Some(Map("test" -> "1", "dt" -> "2008-08-08", "country" -> "us")))
@@ -1867,8 +2119,8 @@ class DDLParserSuite extends AnalysisTest {
         |SET SERDEPROPERTIES ('columns'='foo,bar', 'field.delim' = ',')
       """.stripMargin
     val parsed5 = parsePlan(sql5)
-    val expected5 = AlterTableSerDePropertiesStatement(
-      Seq("table_name"),
+    val expected5 = SetTableSerDeProperties(
+      UnresolvedTable(Seq("table_name"), "ALTER TABLE ... SET [SERDE|SERDEPROPERTIES]", hint),
       None,
       Some(Map("columns" -> "foo,bar", "field.delim" -> ",")),
       Some(Map("test" -> "1", "dt" -> "2008-08-08", "country" -> "us")))
@@ -1880,8 +2132,8 @@ class DDLParserSuite extends AnalysisTest {
         |WITH SERDEPROPERTIES ('columns'='foo,bar', 'field.delim' = ',')
       """.stripMargin
     val parsed6 = parsePlan(sql6)
-    val expected6 = AlterTableSerDePropertiesStatement(
-      Seq("a", "b", "c"),
+    val expected6 = SetTableSerDeProperties(
+      UnresolvedTable(Seq("a", "b", "c"), "ALTER TABLE ... SET [SERDE|SERDEPROPERTIES]", hint),
       Some("org.apache.class"),
       Some(Map("columns" -> "foo,bar", "field.delim" -> ",")),
       None)
@@ -1893,8 +2145,8 @@ class DDLParserSuite extends AnalysisTest {
         |SET SERDEPROPERTIES ('columns'='foo,bar', 'field.delim' = ',')
       """.stripMargin
     val parsed7 = parsePlan(sql7)
-    val expected7 = AlterTableSerDePropertiesStatement(
-      Seq("a", "b", "c"),
+    val expected7 = SetTableSerDeProperties(
+      UnresolvedTable(Seq("a", "b", "c"), "ALTER TABLE ... SET [SERDE|SERDEPROPERTIES]", hint),
       None,
       Some(Map("columns" -> "foo,bar", "field.delim" -> ",")),
       Some(Map("test" -> "1", "dt" -> "2008-08-08", "country" -> "us")))
@@ -1903,8 +2155,10 @@ class DDLParserSuite extends AnalysisTest {
 
   test("alter view: AS Query") {
     val parsed = parsePlan("ALTER VIEW a.b.c AS SELECT 1")
-    val expected = AlterViewAsStatement(
-      Seq("a", "b", "c"), "SELECT 1", parsePlan("SELECT 1"))
+    val expected = AlterViewAs(
+      UnresolvedView(Seq("a", "b", "c"), "ALTER VIEW ... AS", true, None),
+      "SELECT 1",
+      parsePlan("SELECT 1"))
     comparePlans(parsed, expected)
   }
 
@@ -1966,7 +2220,6 @@ class DDLParserSuite extends AnalysisTest {
       """
         |CREATE OR REPLACE GLOBAL TEMPORARY VIEW a.b.c
         |(col1, col3 COMMENT 'hello')
-        |TBLPROPERTIES('prop1Key'="prop1Val")
         |COMMENT 'BLABLA'
         |AS SELECT * FROM tab1
       """.stripMargin
@@ -1975,7 +2228,7 @@ class DDLParserSuite extends AnalysisTest {
       Seq("a", "b", "c"),
       Seq("col1" -> None, "col3" -> Some("hello")),
       Some("BLABLA"),
-      Map("prop1Key" -> "prop1Val"),
+      Map(),
       Some("SELECT * FROM tab1"),
       parsePlan("SELECT * FROM tab1"),
       false,
@@ -2007,53 +2260,67 @@ class DDLParserSuite extends AnalysisTest {
     intercept(sql2, "Found duplicate clauses: TBLPROPERTIES")
   }
 
+  test("SPARK-32374: create temporary view with properties not allowed") {
+    assertUnsupported(
+      sql = """
+        |CREATE OR REPLACE TEMPORARY VIEW a.b.c
+        |(col1, col3 COMMENT 'hello')
+        |TBLPROPERTIES('prop1Key'="prop1Val")
+        |AS SELECT * FROM tab1
+      """.stripMargin,
+      containsThesePhrases = Seq("TBLPROPERTIES can't coexist with CREATE TEMPORARY VIEW"))
+  }
+
   test("SHOW TBLPROPERTIES table") {
     comparePlans(
       parsePlan("SHOW TBLPROPERTIES a.b.c"),
-      ShowTableProperties(UnresolvedTableOrView(Seq("a", "b", "c")), None))
+      ShowTableProperties(
+        UnresolvedTableOrView(Seq("a", "b", "c"), "SHOW TBLPROPERTIES", true),
+        None))
 
     comparePlans(
       parsePlan("SHOW TBLPROPERTIES a.b.c('propKey1')"),
-      ShowTableProperties(UnresolvedTableOrView(Seq("a", "b", "c")), Some("propKey1")))
+      ShowTableProperties(
+        UnresolvedTableOrView(Seq("a", "b", "c"), "SHOW TBLPROPERTIES", true), Some("propKey1")))
   }
 
   test("DESCRIBE FUNCTION") {
     comparePlans(
       parsePlan("DESC FUNCTION a"),
-      DescribeFunctionStatement(Seq("a"), false))
+      DescribeFunction(UnresolvedFunc(Seq("a")), false))
     comparePlans(
       parsePlan("DESCRIBE FUNCTION a"),
-      DescribeFunctionStatement(Seq("a"), false))
+      DescribeFunction(UnresolvedFunc(Seq("a")), false))
     comparePlans(
       parsePlan("DESCRIBE FUNCTION a.b.c"),
-      DescribeFunctionStatement(Seq("a", "b", "c"), false))
+      DescribeFunction(UnresolvedFunc(Seq("a", "b", "c")), false))
     comparePlans(
       parsePlan("DESCRIBE FUNCTION EXTENDED a.b.c"),
-      DescribeFunctionStatement(Seq("a", "b", "c"), true))
+      DescribeFunction(UnresolvedFunc(Seq("a", "b", "c")), true))
   }
 
   test("SHOW FUNCTIONS") {
     comparePlans(
       parsePlan("SHOW FUNCTIONS"),
-      ShowFunctionsStatement(true, true, None, None))
+      ShowFunctions(None, true, true, None))
     comparePlans(
       parsePlan("SHOW USER FUNCTIONS"),
-      ShowFunctionsStatement(true, false, None, None))
+      ShowFunctions(None, true, false, None))
     comparePlans(
       parsePlan("SHOW user FUNCTIONS"),
-      ShowFunctionsStatement(true, false, None, None))
+      ShowFunctions(None, true, false, None))
     comparePlans(
       parsePlan("SHOW SYSTEM FUNCTIONS"),
-      ShowFunctionsStatement(false, true, None, None))
+      ShowFunctions(None, false, true, None))
     comparePlans(
       parsePlan("SHOW ALL FUNCTIONS"),
-      ShowFunctionsStatement(true, true, None, None))
+      ShowFunctions(None, true, true, None))
     comparePlans(
       parsePlan("SHOW FUNCTIONS LIKE 'funct*'"),
-      ShowFunctionsStatement(true, true, Some("funct*"), None))
+      ShowFunctions(None, true, true, Some("funct*")))
     comparePlans(
       parsePlan("SHOW FUNCTIONS LIKE a.b.c"),
-      ShowFunctionsStatement(true, true, None, Some(Seq("a", "b", "c"))))
+      ShowFunctions(Some(UnresolvedFunc(Seq("a", "b", "c"))), true, true, None))
     val sql = "SHOW other FUNCTIONS"
     intercept(sql, s"$sql not supported")
   }
@@ -2061,19 +2328,19 @@ class DDLParserSuite extends AnalysisTest {
   test("DROP FUNCTION") {
     comparePlans(
       parsePlan("DROP FUNCTION a"),
-      DropFunctionStatement(Seq("a"), false, false))
+      DropFunction(UnresolvedFunc(Seq("a")), false, false))
     comparePlans(
       parsePlan("DROP FUNCTION a.b.c"),
-      DropFunctionStatement(Seq("a", "b", "c"), false, false))
+      DropFunction(UnresolvedFunc(Seq("a", "b", "c")), false, false))
     comparePlans(
       parsePlan("DROP TEMPORARY FUNCTION a.b.c"),
-      DropFunctionStatement(Seq("a", "b", "c"), false, true))
+      DropFunction(UnresolvedFunc(Seq("a", "b", "c")), false, true))
     comparePlans(
       parsePlan("DROP FUNCTION IF EXISTS a.b.c"),
-      DropFunctionStatement(Seq("a", "b", "c"), true, false))
+      DropFunction(UnresolvedFunc(Seq("a", "b", "c")), true, false))
     comparePlans(
       parsePlan("DROP TEMPORARY FUNCTION IF EXISTS a.b.c"),
-      DropFunctionStatement(Seq("a", "b", "c"), true, true))
+      DropFunction(UnresolvedFunc(Seq("a", "b", "c")), true, true))
   }
 
   test("CREATE FUNCTION") {
@@ -2113,6 +2380,15 @@ class DDLParserSuite extends AnalysisTest {
       "Operation not allowed: CREATE FUNCTION with resource type 'other'")
   }
 
+  test("REFRESH FUNCTION") {
+    parseCompare("REFRESH FUNCTION c",
+      RefreshFunction(UnresolvedFunc(Seq("c"))))
+    parseCompare("REFRESH FUNCTION b.c",
+      RefreshFunction(UnresolvedFunc(Seq("b", "c"))))
+    parseCompare("REFRESH FUNCTION a.b.c",
+      RefreshFunction(UnresolvedFunc(Seq("a", "b", "c"))))
+  }
+
   private case class TableSpec(
       name: Seq[String],
       schema: Option[StructType],
@@ -2122,7 +2398,9 @@ class DDLParserSuite extends AnalysisTest {
       provider: Option[String],
       options: Map[String, String],
       location: Option[String],
-      comment: Option[String])
+      comment: Option[String],
+      serdeInfo: Option[SerdeInfo],
+      external: Boolean = false)
 
   private object TableSpec {
     def apply(plan: LogicalPlan): TableSpec = {
@@ -2137,7 +2415,9 @@ class DDLParserSuite extends AnalysisTest {
             create.provider,
             create.options,
             create.location,
-            create.comment)
+            create.comment,
+            create.serde,
+            create.external)
         case replace: ReplaceTableStatement =>
           TableSpec(
             replace.tableName,
@@ -2148,7 +2428,8 @@ class DDLParserSuite extends AnalysisTest {
             replace.provider,
             replace.options,
             replace.location,
-            replace.comment)
+            replace.comment,
+            replace.serde)
         case ctas: CreateTableAsSelectStatement =>
           TableSpec(
             ctas.tableName,
@@ -2159,7 +2440,9 @@ class DDLParserSuite extends AnalysisTest {
             ctas.provider,
             ctas.options,
             ctas.location,
-            ctas.comment)
+            ctas.comment,
+            ctas.serde,
+            ctas.external)
         case rtas: ReplaceTableAsSelectStatement =>
           TableSpec(
             rtas.tableName,
@@ -2170,7 +2453,8 @@ class DDLParserSuite extends AnalysisTest {
             rtas.provider,
             rtas.options,
             rtas.location,
-            rtas.comment)
+            rtas.comment,
+            rtas.serde)
         case other =>
           fail(s"Expected to parse Create, CTAS, Replace, or RTAS plan" +
             s" from query, got ${other.getClass.getName}.")
@@ -2193,11 +2477,10 @@ class DDLParserSuite extends AnalysisTest {
 
     comparePlans(
       parsePlan("COMMENT ON TABLE a.b.c IS 'xYz'"),
-      CommentOnTable(UnresolvedTable(Seq("a", "b", "c")), "xYz"))
+      CommentOnTable(UnresolvedTable(Seq("a", "b", "c"), "COMMENT ON TABLE", None), "xYz"))
   }
 
-  // TODO: ignored by SPARK-31707, restore the test after create table syntax unification
-  ignore("create table - without using") {
+  test("create table - without using") {
     val sql = "CREATE TABLE 1m.2g(a INT)"
     val expectedTableSpec = TableSpec(
       Seq("1m", "2g"),
@@ -2208,8 +2491,33 @@ class DDLParserSuite extends AnalysisTest {
       None,
       Map.empty[String, String],
       None,
+      None,
       None)
 
     testCreateOrReplaceDdl(sql, expectedTableSpec, expectedIfNotExists = false)
   }
+
+  test("SPARK-33474: Support typed literals as partition spec values") {
+    def insertPartitionPlan(part: String): InsertIntoStatement = {
+      InsertIntoStatement(
+        UnresolvedRelation(Seq("t")),
+        Map("part" -> Some(part)),
+        Seq.empty[String],
+        UnresolvedInlineTable(Seq("col1"), Seq(Seq(Literal("a")))),
+        overwrite = false, ifPartitionNotExists = false)
+    }
+    val binaryStr = "Spark SQL"
+    val binaryHexStr = Hex.hex(UTF8String.fromString(binaryStr).getBytes).toString
+    val dateTypeSql = "INSERT INTO t PARTITION(part = date'2019-01-02') VALUES('a')"
+    val interval = new CalendarInterval(7, 1, 1000).toString
+    val intervalTypeSql = s"INSERT INTO t PARTITION(part = interval'$interval') VALUES('a')"
+    val timestamp = "2019-01-02 11:11:11"
+    val timestampTypeSql = s"INSERT INTO t PARTITION(part = timestamp'$timestamp') VALUES('a')"
+    val binaryTypeSql = s"INSERT INTO t PARTITION(part = X'$binaryHexStr') VALUES('a')"
+
+    comparePlans(parsePlan(dateTypeSql), insertPartitionPlan("2019-01-02"))
+    comparePlans(parsePlan(intervalTypeSql), insertPartitionPlan(interval))
+    comparePlans(parsePlan(timestampTypeSql), insertPartitionPlan(timestamp))
+    comparePlans(parsePlan(binaryTypeSql), insertPartitionPlan(binaryStr))
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DataTypeParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DataTypeParserSuite.scala
index d519fdf378786..6e432a7759d7f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DataTypeParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DataTypeParserSuite.scala
@@ -56,12 +56,15 @@ class DataTypeParserSuite extends SparkFunSuite {
   checkDataType("DATE", DateType)
   checkDataType("timestamp", TimestampType)
   checkDataType("string", StringType)
-  checkDataType("ChaR(5)", StringType)
-  checkDataType("ChaRacter(5)", StringType)
-  checkDataType("varchAr(20)", StringType)
-  checkDataType("cHaR(27)", StringType)
+  checkDataType("ChaR(5)", CharType(5))
+  checkDataType("ChaRacter(5)", CharType(5))
+  checkDataType("varchAr(20)", VarcharType(20))
+  checkDataType("cHaR(27)", CharType(27))
   checkDataType("BINARY", BinaryType)
+  checkDataType("void", NullType)
   checkDataType("interval", CalendarIntervalType)
+  checkDataType("INTERVAL YEAR TO MONTH", YearMonthIntervalType)
+  checkDataType("interval day to second", DayTimeIntervalType)
 
   checkDataType("array<doublE>", ArrayType(DoubleType, true))
   checkDataType("Array<map<int, tinYint>>", ArrayType(MapType(IntegerType, ByteType, true), true))
@@ -102,9 +105,9 @@ class DataTypeParserSuite extends SparkFunSuite {
         StructType(
           StructField("deciMal", DecimalType.USER_DEFAULT, true) ::
           StructField("anotherDecimal", DecimalType(5, 2), true) :: Nil), true) ::
-      StructField("MAP", MapType(TimestampType, StringType), true) ::
+      StructField("MAP", MapType(TimestampType, VarcharType(10)), true) ::
       StructField("arrAy", ArrayType(DoubleType, true), true) ::
-      StructField("anotherArray", ArrayType(StringType, true), true) :: Nil)
+      StructField("anotherArray", ArrayType(CharType(9), true), true) :: Nil)
   )
   // Use backticks to quote column names having special characters.
   checkDataType(
@@ -112,7 +115,7 @@ class DataTypeParserSuite extends SparkFunSuite {
     StructType(
       StructField("x+y", IntegerType, true) ::
       StructField("!@#$%^&*()", StringType, true) ::
-      StructField("1_2.345<>:\"", StringType, true) :: Nil)
+      StructField("1_2.345<>:\"", VarcharType(20), true) :: Nil)
   )
   // Empty struct.
   checkDataType("strUCt<>", StructType(Nil))
@@ -123,8 +126,8 @@ class DataTypeParserSuite extends SparkFunSuite {
   unsupported("struct<x int, y string>")
 
   test("Do not print empty parentheses for no params") {
-    assert(intercept("unkwon").getMessage.contains("unkwon is not supported"))
-    assert(intercept("unkwon(1,2,3)").getMessage.contains("unkwon(1,2,3) is not supported"))
+    assert(intercept("unknown").getMessage.contains("unknown is not supported"))
+    assert(intercept("unknown(1,2,3)").getMessage.contains("unknown(1,2,3) is not supported"))
   }
 
   // DataType parser accepts certain reserved keywords.
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ErrorParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ErrorParserSuite.scala
index 00b6828c08b38..99051d692451b 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ErrorParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ErrorParserSuite.scala
@@ -77,7 +77,7 @@ class ErrorParserSuite extends AnalysisTest {
   }
 
   test("SPARK-21136: misleading error message due to problematic antlr grammar") {
-    intercept("select * from a left joinn b on a.id = b.id", "missing 'JOIN' at 'joinn'")
+    intercept("select * from a left join_ b on a.id = b.id", "missing 'JOIN' at 'join_'")
     intercept("select * from test where test.t is like 'test'", "mismatched input 'is' expecting")
     intercept("SELECT * FROM test WHERE x NOT NULL", "mismatched input 'NOT' expecting")
   }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
index 41652cab2675b..9a5653e0bf62a 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
@@ -200,23 +200,23 @@ class ExpressionParserSuite extends AnalysisTest {
   }
 
   test("like expressions with ESCAPED_STRING_LITERALS = true") {
-    val conf = new SQLConf()
-    conf.setConfString(SQLConf.ESCAPED_STRING_LITERALS.key, "true")
-    val parser = new CatalystSqlParser(conf)
-    assertEqual("a rlike '^\\x20[\\x20-\\x23]+$'", 'a rlike "^\\x20[\\x20-\\x23]+$", parser)
-    assertEqual("a rlike 'pattern\\\\'", 'a rlike "pattern\\\\", parser)
-    assertEqual("a rlike 'pattern\\t\\n'", 'a rlike "pattern\\t\\n", parser)
+    withSQLConf(SQLConf.ESCAPED_STRING_LITERALS.key -> "true") {
+      val parser = new CatalystSqlParser()
+      assertEqual("a rlike '^\\x20[\\x20-\\x23]+$'", 'a rlike "^\\x20[\\x20-\\x23]+$", parser)
+      assertEqual("a rlike 'pattern\\\\'", 'a rlike "pattern\\\\", parser)
+      assertEqual("a rlike 'pattern\\t\\n'", 'a rlike "pattern\\t\\n", parser)
+    }
   }
 
   test("(NOT) LIKE (ANY | SOME | ALL) expressions") {
     Seq("any", "some").foreach { quantifier =>
-      assertEqual(s"a like $quantifier ('foo%', 'b%')", ('a like "foo%") || ('a like "b%"))
-      assertEqual(s"a not like $quantifier ('foo%', 'b%')", !('a like "foo%") || !('a like "b%"))
-      assertEqual(s"not (a like $quantifier ('foo%', 'b%'))", !(('a like "foo%") || ('a like "b%")))
+      assertEqual(s"a like $quantifier ('foo%', 'b%')", 'a likeAny("foo%", "b%"))
+      assertEqual(s"a not like $quantifier ('foo%', 'b%')", 'a notLikeAny("foo%", "b%"))
+      assertEqual(s"not (a like $quantifier ('foo%', 'b%'))", !('a likeAny("foo%", "b%")))
     }
-    assertEqual("a like all ('foo%', 'b%')", ('a like "foo%") && ('a like "b%"))
-    assertEqual("a not like all ('foo%', 'b%')", !('a like "foo%") && !('a like "b%"))
-    assertEqual("not (a like all ('foo%', 'b%'))", !(('a like "foo%") && ('a like "b%")))
+    assertEqual("a like all ('foo%', 'b%')", 'a likeAll("foo%", "b%"))
+    assertEqual("a not like all ('foo%', 'b%')", 'a notLikeAll("foo%", "b%"))
+    assertEqual("not (a like all ('foo%', 'b%'))", !('a likeAll("foo%", "b%")))
 
     Seq("any", "some", "all").foreach { quantifier =>
       intercept(s"a like $quantifier()", "Expected something between '(' and ')'")
@@ -425,7 +425,7 @@ class ExpressionParserSuite extends AnalysisTest {
     assertEqual("(a + b).b", ('a + 'b).getField("b")) // This will fail analysis.
     assertEqual(
       "struct(a, b).b",
-      namedStruct(NamePlaceholder, 'a, NamePlaceholder, 'b).getField("b"))
+      namedStruct(Literal("a"), 'a, Literal("b"), 'b).getField("b"))
   }
 
   test("reference") {
@@ -557,98 +557,99 @@ class ExpressionParserSuite extends AnalysisTest {
     def testDecimal(value: String, parser: ParserInterface): Unit = {
       assertEqual(value, Literal(BigDecimal(value).underlying), parser)
     }
-    val conf = new SQLConf()
-    conf.setConf(SQLConf.LEGACY_EXPONENT_LITERAL_AS_DECIMAL_ENABLED, true)
-    val parser = new CatalystSqlParser(conf)
-    testDecimal("9e1", parser)
-    testDecimal("9e-1", parser)
-    testDecimal("-9e1", parser)
-    testDecimal("9.0e1", parser)
-    testDecimal(".9e+2", parser)
-    testDecimal("0.9e+2", parser)
+    withSQLConf(SQLConf.LEGACY_EXPONENT_LITERAL_AS_DECIMAL_ENABLED.key -> "true") {
+      val parser = new CatalystSqlParser()
+      testDecimal("9e1", parser)
+      testDecimal("9e-1", parser)
+      testDecimal("-9e1", parser)
+      testDecimal("9.0e1", parser)
+      testDecimal(".9e+2", parser)
+      testDecimal("0.9e+2", parser)
+    }
   }
 
   test("strings") {
     Seq(true, false).foreach { escape =>
-      val conf = new SQLConf()
-      conf.setConfString(SQLConf.ESCAPED_STRING_LITERALS.key, escape.toString)
-      val parser = new CatalystSqlParser(conf)
-
-      // tests that have same result whatever the conf is
-      // Single Strings.
-      assertEqual("\"hello\"", "hello", parser)
-      assertEqual("'hello'", "hello", parser)
-
-      // Multi-Strings.
-      assertEqual("\"hello\" 'world'", "helloworld", parser)
-      assertEqual("'hello' \" \" 'world'", "hello world", parser)
-
-      // 'LIKE' string literals. Notice that an escaped '%' is the same as an escaped '\' and a
-      // regular '%'; to get the correct result you need to add another escaped '\'.
-      // TODO figure out if we shouldn't change the ParseUtils.unescapeSQLString method?
-      assertEqual("'pattern%'", "pattern%", parser)
-      assertEqual("'no-pattern\\%'", "no-pattern\\%", parser)
-
-      // tests that have different result regarding the conf
-      if (escape) {
-        // When SQLConf.ESCAPED_STRING_LITERALS is enabled, string literal parsing fallbacks to
-        // Spark 1.6 behavior.
-
-        // 'LIKE' string literals.
-        assertEqual("'pattern\\\\%'", "pattern\\\\%", parser)
-        assertEqual("'pattern\\\\\\%'", "pattern\\\\\\%", parser)
-
-        // Escaped characters.
-        // Unescape string literal "'\\0'" for ASCII NUL (X'00') doesn't work
-        // when ESCAPED_STRING_LITERALS is enabled.
-        // It is parsed literally.
-        assertEqual("'\\0'", "\\0", parser)
-
-        // Note: Single quote follows 1.6 parsing behavior when ESCAPED_STRING_LITERALS is enabled.
-        val e = intercept[ParseException](parser.parseExpression("'\''"))
-        assert(e.message.contains("extraneous input '''"))
-
-        // The unescape special characters (e.g., "\\t") for 2.0+ don't work
-        // when ESCAPED_STRING_LITERALS is enabled. They are parsed literally.
-        assertEqual("'\\\"'", "\\\"", parser)   // Double quote
-        assertEqual("'\\b'", "\\b", parser)     // Backspace
-        assertEqual("'\\n'", "\\n", parser)     // Newline
-        assertEqual("'\\r'", "\\r", parser)     // Carriage return
-        assertEqual("'\\t'", "\\t", parser)     // Tab character
-
-        // The unescape Octals for 2.0+ don't work when ESCAPED_STRING_LITERALS is enabled.
-        // They are parsed literally.
-        assertEqual("'\\110\\145\\154\\154\\157\\041'", "\\110\\145\\154\\154\\157\\041", parser)
-        // The unescape Unicode for 2.0+ doesn't work when ESCAPED_STRING_LITERALS is enabled.
-        // They are parsed literally.
-        assertEqual("'\\u0057\\u006F\\u0072\\u006C\\u0064\\u0020\\u003A\\u0029'",
-          "\\u0057\\u006F\\u0072\\u006C\\u0064\\u0020\\u003A\\u0029", parser)
-      } else {
-        // Default behavior
-
-        // 'LIKE' string literals.
-        assertEqual("'pattern\\\\%'", "pattern\\%", parser)
-        assertEqual("'pattern\\\\\\%'", "pattern\\\\%", parser)
-
-        // Escaped characters.
-        // See: http://dev.mysql.com/doc/refman/5.7/en/string-literals.html
-        assertEqual("'\\0'", "\u0000", parser) // ASCII NUL (X'00')
-        assertEqual("'\\''", "\'", parser)     // Single quote
-        assertEqual("'\\\"'", "\"", parser)    // Double quote
-        assertEqual("'\\b'", "\b", parser)     // Backspace
-        assertEqual("'\\n'", "\n", parser)     // Newline
-        assertEqual("'\\r'", "\r", parser)     // Carriage return
-        assertEqual("'\\t'", "\t", parser)     // Tab character
-        assertEqual("'\\Z'", "\u001A", parser) // ASCII 26 - CTRL + Z (EOF on windows)
-
-        // Octals
-        assertEqual("'\\110\\145\\154\\154\\157\\041'", "Hello!", parser)
-
-        // Unicode
-        assertEqual("'\\u0057\\u006F\\u0072\\u006C\\u0064\\u0020\\u003A\\u0029'", "World :)",
-          parser)
-      }
+      withSQLConf(SQLConf.ESCAPED_STRING_LITERALS.key -> escape.toString) {
+        val parser = new CatalystSqlParser()
+
+        // tests that have same result whatever the conf is
+        // Single Strings.
+        assertEqual("\"hello\"", "hello", parser)
+        assertEqual("'hello'", "hello", parser)
+
+        // Multi-Strings.
+        assertEqual("\"hello\" 'world'", "helloworld", parser)
+        assertEqual("'hello' \" \" 'world'", "hello world", parser)
+
+        // 'LIKE' string literals. Notice that an escaped '%' is the same as an escaped '\' and a
+        // regular '%'; to get the correct result you need to add another escaped '\'.
+        // TODO figure out if we shouldn't change the ParseUtils.unescapeSQLString method?
+        assertEqual("'pattern%'", "pattern%", parser)
+        assertEqual("'no-pattern\\%'", "no-pattern\\%", parser)
+
+        // tests that have different result regarding the conf
+        if (escape) {
+          // When SQLConf.ESCAPED_STRING_LITERALS is enabled, string literal parsing falls back to
+          // Spark 1.6 behavior.
+
+          // 'LIKE' string literals.
+          assertEqual("'pattern\\\\%'", "pattern\\\\%", parser)
+          assertEqual("'pattern\\\\\\%'", "pattern\\\\\\%", parser)
+
+          // Escaped characters.
+          // Unescape string literal "'\\0'" for ASCII NUL (X'00') doesn't work
+          // when ESCAPED_STRING_LITERALS is enabled.
+          // It is parsed literally.
+          assertEqual("'\\0'", "\\0", parser)
+
+          // Note: Single quote follows 1.6 parsing behavior
+          // when ESCAPED_STRING_LITERALS is enabled.
+          val e = intercept[ParseException](parser.parseExpression("'\''"))
+          assert(e.message.contains("extraneous input '''"))
+
+          // The unescape special characters (e.g., "\\t") for 2.0+ don't work
+          // when ESCAPED_STRING_LITERALS is enabled. They are parsed literally.
+          assertEqual("'\\\"'", "\\\"", parser) // Double quote
+          assertEqual("'\\b'", "\\b", parser) // Backspace
+          assertEqual("'\\n'", "\\n", parser) // Newline
+          assertEqual("'\\r'", "\\r", parser) // Carriage return
+          assertEqual("'\\t'", "\\t", parser) // Tab character
+
+          // The unescape Octals for 2.0+ don't work when ESCAPED_STRING_LITERALS is enabled.
+          // They are parsed literally.
+          assertEqual("'\\110\\145\\154\\154\\157\\041'", "\\110\\145\\154\\154\\157\\041", parser)
+          // The unescape Unicode for 2.0+ doesn't work when ESCAPED_STRING_LITERALS is enabled.
+          // They are parsed literally.
+          assertEqual("'\\u0057\\u006F\\u0072\\u006C\\u0064\\u0020\\u003A\\u0029'",
+            "\\u0057\\u006F\\u0072\\u006C\\u0064\\u0020\\u003A\\u0029", parser)
+        } else {
+          // Default behavior
+
+          // 'LIKE' string literals.
+          assertEqual("'pattern\\\\%'", "pattern\\%", parser)
+          assertEqual("'pattern\\\\\\%'", "pattern\\\\%", parser)
+
+          // Escaped characters.
+          // See: http://dev.mysql.com/doc/refman/5.7/en/string-literals.html
+          assertEqual("'\\0'", "\u0000", parser) // ASCII NUL (X'00')
+          assertEqual("'\\''", "\'", parser) // Single quote
+          assertEqual("'\\\"'", "\"", parser) // Double quote
+          assertEqual("'\\b'", "\b", parser) // Backspace
+          assertEqual("'\\n'", "\n", parser) // Newline
+          assertEqual("'\\r'", "\r", parser) // Carriage return
+          assertEqual("'\\t'", "\t", parser) // Tab character
+          assertEqual("'\\Z'", "\u001A", parser) // ASCII 26 - CTRL + Z (EOF on windows)
+
+          // Octals
+          assertEqual("'\\110\\145\\154\\154\\157\\041'", "Hello!", parser)
+
+          // Unicode
+          assertEqual("'\\u0057\\u006F\\u0072\\u006C\\u0064\\u0020\\u003A\\u0029'", "World :)",
+            parser)
+        }
 
+      }
     }
   }
 
@@ -713,37 +714,39 @@ class ExpressionParserSuite extends AnalysisTest {
     // Non Existing unit
     intercept("interval 10 nanoseconds", "invalid unit 'nanoseconds'")
 
-    // Year-Month intervals.
-    val yearMonthValues = Seq("123-10", "496-0", "-2-3", "-123-0", "\t -1-2\t")
-    yearMonthValues.foreach { value =>
-      val result = Literal(IntervalUtils.fromYearMonthString(value))
-      checkIntervals(s"'$value' year to month", result)
-    }
+    withSQLConf(SQLConf.LEGACY_INTERVAL_ENABLED.key -> "true") {
+      // Year-Month intervals.
+      val yearMonthValues = Seq("123-10", "496-0", "-2-3", "-123-0", "\t -1-2\t")
+      yearMonthValues.foreach { value =>
+        val result = Literal(IntervalUtils.fromYearMonthString(value))
+        checkIntervals(s"'$value' year to month", result)
+      }
 
-    // Day-Time intervals.
-    val datTimeValues = Seq(
-      "99 11:22:33.123456789",
-      "-99 11:22:33.123456789",
-      "10 9:8:7.123456789",
-      "1 0:0:0",
-      "-1 0:0:0",
-      "1 0:0:1",
-      "\t 1 0:0:1 ")
-    datTimeValues.foreach { value =>
-      val result = Literal(IntervalUtils.fromDayTimeString(value))
-      checkIntervals(s"'$value' day to second", result)
-    }
+      // Day-Time intervals.
+      val datTimeValues = Seq(
+        "99 11:22:33.123456789",
+        "-99 11:22:33.123456789",
+        "10 9:8:7.123456789",
+        "1 0:0:0",
+        "-1 0:0:0",
+        "1 0:0:1",
+        "\t 1 0:0:1 ")
+      datTimeValues.foreach { value =>
+        val result = Literal(IntervalUtils.fromDayTimeString(value))
+        checkIntervals(s"'$value' day to second", result)
+      }
 
-    // Hour-Time intervals.
-    val hourTimeValues = Seq(
-      "11:22:33.123456789",
-      "9:8:7.123456789",
-      "-19:18:17.123456789",
-      "0:0:0",
-      "0:0:1")
-    hourTimeValues.foreach { value =>
-      val result = Literal(IntervalUtils.fromDayTimeString(value, HOUR, SECOND))
-      checkIntervals(s"'$value' hour to second", result)
+      // Hour-Time intervals.
+      val hourTimeValues = Seq(
+        "11:22:33.123456789",
+        "9:8:7.123456789",
+        "-19:18:17.123456789",
+        "0:0:0",
+        "0:0:1")
+      hourTimeValues.foreach { value =>
+        val result = Literal(IntervalUtils.fromDayTimeString(value, HOUR, SECOND))
+        checkIntervals(s"'$value' hour to second", result)
+      }
     }
 
     // Unknown FROM TO intervals
@@ -777,18 +780,18 @@ class ExpressionParserSuite extends AnalysisTest {
 
   test("SPARK-17832 function identifier contains backtick") {
     val complexName = FunctionIdentifier("`ba`r", Some("`fo`o"))
-    assertEqual(complexName.quotedString, UnresolvedAttribute("`fo`o.`ba`r"))
+    assertEqual(complexName.quotedString, UnresolvedAttribute(Seq("`fo`o", "`ba`r")))
     intercept(complexName.unquotedString, "mismatched input")
-    // Function identifier contains countious backticks should be treated correctly.
+    // Function identifier contains continuous backticks should be treated correctly.
     val complexName2 = FunctionIdentifier("ba``r", Some("fo``o"))
-    assertEqual(complexName2.quotedString, UnresolvedAttribute("fo``o.ba``r"))
+    assertEqual(complexName2.quotedString, UnresolvedAttribute(Seq("fo``o", "ba``r")))
   }
 
   test("SPARK-19526 Support ignore nulls keywords for first and last") {
-    assertEqual("first(a ignore nulls)", First('a, Literal(true)).toAggregateExpression())
-    assertEqual("first(a)", First('a, Literal(false)).toAggregateExpression())
-    assertEqual("last(a ignore nulls)", Last('a, Literal(true)).toAggregateExpression())
-    assertEqual("last(a)", Last('a, Literal(false)).toAggregateExpression())
+    assertEqual("first(a ignore nulls)", First('a, true).toAggregateExpression())
+    assertEqual("first(a)", First('a, false).toAggregateExpression())
+    assertEqual("last(a ignore nulls)", Last('a, true).toAggregateExpression())
+    assertEqual("last(a)", Last('a, false).toAggregateExpression())
   }
 
   test("timestamp literals") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParserUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParserUtilsSuite.scala
index 6008d091022de..35f09001fd217 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParserUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParserUtilsSuite.scala
@@ -94,12 +94,24 @@ class ParserUtilsSuite extends SparkFunSuite {
     assert(unescapeSQLString(""""\256"""") == "256")
 
     // String including a '\u0000' style literal characters (\u732B is a cat in Kanji).
-    assert(unescapeSQLString(""""How cute \u732B are"""")  == "How cute \u732B are")
+    assert(unescapeSQLString("\"How cute \\u732B are\"")  == "How cute \u732B are")
 
     // String including a surrogate pair character
     // (\uD867\uDE3D is Okhotsk atka mackerel in Kanji).
-    assert(unescapeSQLString(""""\uD867\uDE3D is a fish"""") == "\uD867\uDE3D is a fish")
+    assert(unescapeSQLString("\"\\uD867\\uDE3D is a fish\"") == "\uD867\uDE3D is a fish")
 
+    // String including a '\U00000000' style literal characters (\u732B is a cat in Kanji).
+    assert(unescapeSQLString("\"\\U00000041B\\U000000312\\U0000732B\"") == "AB12\u732B")
+
+    // String including surrogate pair characters (U+1F408 is a cat and U+1F415 is a dog in Emoji).
+    assert(unescapeSQLString("\"\\U0001F408 \\U0001F415\"") == "\uD83D\uDC08 \uD83D\uDC15")
+
+    // String including escaped normal characters.
+    assert(unescapeSQLString(
+      """"ab\
+        |cd\ef"""".stripMargin) ==
+      """ab
+        |cdef""".stripMargin)
     // scalastyle:on nonascii
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
index 88afcb10d9c20..9c82c237f83d2 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
@@ -23,7 +23,7 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.IntegerType
+import org.apache.spark.sql.types.{IntegerType, LongType, StringType}
 
 /**
  * Parser test cases for rules defined in [[CatalystSqlParser]] / [[AstBuilder]].
@@ -295,7 +295,7 @@ class PlanParserSuite extends AnalysisTest {
         partition: Map[String, Option[String]],
         overwrite: Boolean = false,
         ifPartitionNotExists: Boolean = false): LogicalPlan =
-      InsertIntoStatement(table("s"), partition, plan, overwrite, ifPartitionNotExists)
+      InsertIntoStatement(table("s"), partition, Nil, plan, overwrite, ifPartitionNotExists)
 
     // Single inserts
     assertEqual(s"insert overwrite table s $sql",
@@ -315,22 +315,35 @@ class PlanParserSuite extends AnalysisTest {
 
   test("aggregation") {
     val sql = "select a, b, sum(c) as c from d group by a, b"
+    val sqlWithoutGroupBy = "select a, b, sum(c) as c from d"
 
     // Normal
     assertEqual(sql, table("d").groupBy('a, 'b)('a, 'b, 'sum.function('c).as("c")))
 
     // Cube
     assertEqual(s"$sql with cube",
-      table("d").groupBy(Cube(Seq('a, 'b)))('a, 'b, 'sum.function('c).as("c")))
+      table("d").groupBy(Cube(Seq(Seq('a), Seq('b))))('a, 'b, 'sum.function('c).as("c")))
+    assertEqual(s"$sqlWithoutGroupBy group by cube(a, b)",
+      table("d").groupBy(Cube(Seq(Seq('a), Seq('b))))('a, 'b, 'sum.function('c).as("c")))
+    assertEqual(s"$sqlWithoutGroupBy group by cube (a, b)",
+      table("d").groupBy(Cube(Seq(Seq('a), Seq('b))))('a, 'b, 'sum.function('c).as("c")))
 
     // Rollup
     assertEqual(s"$sql with rollup",
-      table("d").groupBy(Rollup(Seq('a, 'b)))('a, 'b, 'sum.function('c).as("c")))
+      table("d").groupBy(Rollup(Seq(Seq('a), Seq('b))))('a, 'b, 'sum.function('c).as("c")))
+    assertEqual(s"$sqlWithoutGroupBy group by rollup(a, b)",
+      table("d").groupBy(Rollup(Seq(Seq('a), Seq('b))))('a, 'b, 'sum.function('c).as("c")))
+    assertEqual(s"$sqlWithoutGroupBy group by rollup (a, b)",
+      table("d").groupBy(Rollup(Seq(Seq('a), Seq('b))))('a, 'b, 'sum.function('c).as("c")))
 
     // Grouping Sets
     assertEqual(s"$sql grouping sets((a, b), (a), ())",
-      GroupingSets(Seq(Seq('a, 'b), Seq('a), Seq()), Seq('a, 'b), table("d"),
-        Seq('a, 'b, 'sum.function('c).as("c"))))
+      Aggregate(Seq(GroupingSets(Seq(Seq('a, 'b), Seq('a), Seq()), Seq('a, 'b))),
+        Seq('a, 'b, 'sum.function('c).as("c")), table("d")))
+
+    assertEqual(s"$sqlWithoutGroupBy group by grouping sets((a, b), (a), ())",
+      Aggregate(Seq(GroupingSets(Seq(Seq('a, 'b), Seq('a), Seq()))),
+        Seq('a, 'b, 'sum.function('c).as("c")), table("d")))
 
     val m = intercept[ParseException] {
       parsePlan("SELECT a, b, count(distinct a, distinct b) as c FROM d GROUP BY a, b")
@@ -591,6 +604,9 @@ class PlanParserSuite extends AnalysisTest {
     assertEqual(
       "select * from range(2)",
       UnresolvedTableValuedFunction("range", Literal(2) :: Nil, Seq.empty).select(star()))
+    // SPARK-34627
+    intercept("select * from default.range(2)",
+      "table valued function cannot specify database name: default.range")
   }
 
   test("SPARK-20311 range(N) as alias") {
@@ -607,20 +623,24 @@ class PlanParserSuite extends AnalysisTest {
   test("SPARK-20841 Support table column aliases in FROM clause") {
     assertEqual(
       "SELECT * FROM testData AS t(col1, col2)",
-      UnresolvedSubqueryColumnAliases(
-        Seq("col1", "col2"),
-        SubqueryAlias("t", UnresolvedRelation(TableIdentifier("testData")))
+      SubqueryAlias(
+        "t",
+        UnresolvedSubqueryColumnAliases(
+          Seq("col1", "col2"),
+          UnresolvedRelation(TableIdentifier("testData"))
+        )
       ).select(star()))
   }
 
   test("SPARK-20962 Support subquery column aliases in FROM clause") {
     assertEqual(
       "SELECT * FROM (SELECT a AS x, b AS y FROM t) t(col1, col2)",
-      UnresolvedSubqueryColumnAliases(
-        Seq("col1", "col2"),
-        SubqueryAlias(
-          "t",
-          UnresolvedRelation(TableIdentifier("t")).select('a.as("x"), 'b.as("y")))
+      SubqueryAlias(
+        "t",
+        UnresolvedSubqueryColumnAliases(
+          Seq("col1", "col2"),
+          UnresolvedRelation(TableIdentifier("t")).select('a.as("x"), 'b.as("y"))
+        )
       ).select(star()))
   }
 
@@ -629,14 +649,27 @@ class PlanParserSuite extends AnalysisTest {
     val src2 = UnresolvedRelation(TableIdentifier("src2")).as("s2")
     assertEqual(
       "SELECT * FROM (src1 s1 INNER JOIN src2 s2 ON s1.id = s2.id) dst(a, b, c, d)",
-      UnresolvedSubqueryColumnAliases(
-        Seq("a", "b", "c", "d"),
-        SubqueryAlias(
-          "dst",
-          src1.join(src2, Inner, Option(Symbol("s1.id") === Symbol("s2.id"))))
+      SubqueryAlias(
+        "dst",
+        UnresolvedSubqueryColumnAliases(
+          Seq("a", "b", "c", "d"),
+          src1.join(src2, Inner, Option(Symbol("s1.id") === Symbol("s2.id")))
+        )
       ).select(star()))
   }
 
+  test("SPARK-34335 Support referencing subquery with column aliases by table alias") {
+    assertEqual(
+      "SELECT t.col1, t.col2 FROM (SELECT a AS x, b AS y FROM t) t(col1, col2)",
+      SubqueryAlias(
+        "t",
+        UnresolvedSubqueryColumnAliases(
+          Seq("col1", "col2"),
+          UnresolvedRelation(TableIdentifier("t")).select('a.as("x"), 'b.as("y")))
+      ).select($"t.col1", $"t.col2")
+    )
+  }
+
   test("inline table") {
     assertEqual("values 1, 2, 3, 4",
       UnresolvedInlineTable(Seq("col1"), Seq(1, 2, 3, 4).map(x => Seq(Literal(x)))))
@@ -713,7 +746,7 @@ class PlanParserSuite extends AnalysisTest {
     comparePlans(
       parsePlan(
         "INSERT INTO s SELECT /*+ REPARTITION(100), COALESCE(500), COALESCE(10) */ * FROM t"),
-      InsertIntoStatement(table("s"), Map.empty,
+      InsertIntoStatement(table("s"), Map.empty, Nil,
         UnresolvedHint("REPARTITION", Seq(Literal(100)),
           UnresolvedHint("COALESCE", Seq(Literal(500)),
             UnresolvedHint("COALESCE", Seq(Literal(10)),
@@ -1031,4 +1064,111 @@ class PlanParserSuite extends AnalysisTest {
     assertEqual("select a, b from db.c;;;", table("db", "c").select('a, 'b))
     assertEqual("select a, b from db.c; ;;  ;", table("db", "c").select('a, 'b))
   }
+
+  test("SPARK-32106: TRANSFORM plan") {
+    // verify schema less
+    assertEqual(
+      """
+        |SELECT TRANSFORM(a, b, c)
+        |USING 'cat'
+        |FROM testData
+      """.stripMargin,
+      ScriptTransformation(
+        "cat",
+        Seq(AttributeReference("key", StringType)(),
+          AttributeReference("value", StringType)()),
+        Project(Seq('a, 'b, 'c), UnresolvedRelation(TableIdentifier("testData"))),
+        ScriptInputOutputSchema(List.empty, List.empty, None, None,
+          List.empty, List.empty, None, None, true))
+    )
+
+    // verify without output schema
+    assertEqual(
+      """
+        |SELECT TRANSFORM(a, b, c)
+        |USING 'cat' AS (a, b, c)
+        |FROM testData
+      """.stripMargin,
+      ScriptTransformation(
+        "cat",
+        Seq(AttributeReference("a", StringType)(),
+          AttributeReference("b", StringType)(),
+          AttributeReference("c", StringType)()),
+        Project(Seq('a, 'b, 'c), UnresolvedRelation(TableIdentifier("testData"))),
+        ScriptInputOutputSchema(List.empty, List.empty, None, None,
+          List.empty, List.empty, None, None, false)))
+
+    // verify with output schema
+    assertEqual(
+      """
+        |SELECT TRANSFORM(a, b, c)
+        |USING 'cat' AS (a int, b string, c long)
+        |FROM testData
+      """.stripMargin,
+      ScriptTransformation(
+        "cat",
+        Seq(AttributeReference("a", IntegerType)(),
+          AttributeReference("b", StringType)(),
+          AttributeReference("c", LongType)()),
+        Project(Seq('a, 'b, 'c), UnresolvedRelation(TableIdentifier("testData"))),
+        ScriptInputOutputSchema(List.empty, List.empty, None, None,
+          List.empty, List.empty, None, None, false)))
+
+    // verify with ROW FORMAT DELIMETED
+    assertEqual(
+      """
+        |SELECT TRANSFORM(a, b, c)
+        |  ROW FORMAT DELIMITED
+        |  FIELDS TERMINATED BY '\t'
+        |  COLLECTION ITEMS TERMINATED BY '\u0002'
+        |  MAP KEYS TERMINATED BY '\u0003'
+        |  LINES TERMINATED BY '\n'
+        |  NULL DEFINED AS 'null'
+        |  USING 'cat' AS (a, b, c)
+        |  ROW FORMAT DELIMITED
+        |  FIELDS TERMINATED BY '\t'
+        |  COLLECTION ITEMS TERMINATED BY '\u0004'
+        |  MAP KEYS TERMINATED BY '\u0005'
+        |  LINES TERMINATED BY '\n'
+        |  NULL DEFINED AS 'NULL'
+        |FROM testData
+      """.stripMargin,
+      ScriptTransformation(
+        "cat",
+        Seq(AttributeReference("a", StringType)(),
+          AttributeReference("b", StringType)(),
+          AttributeReference("c", StringType)()),
+        Project(Seq('a, 'b, 'c), UnresolvedRelation(TableIdentifier("testData"))),
+        ScriptInputOutputSchema(
+          Seq(("TOK_TABLEROWFORMATFIELD", "\t"),
+            ("TOK_TABLEROWFORMATCOLLITEMS", "\u0002"),
+            ("TOK_TABLEROWFORMATMAPKEYS", "\u0003"),
+            ("TOK_TABLEROWFORMATNULL", "null"),
+            ("TOK_TABLEROWFORMATLINES", "\n")),
+          Seq(("TOK_TABLEROWFORMATFIELD", "\t"),
+            ("TOK_TABLEROWFORMATCOLLITEMS", "\u0004"),
+            ("TOK_TABLEROWFORMATMAPKEYS", "\u0005"),
+            ("TOK_TABLEROWFORMATNULL", "NULL"),
+            ("TOK_TABLEROWFORMATLINES", "\n")), None, None,
+          List.empty, List.empty, None, None, false)))
+
+    // verify with ROW FORMAT SERDE
+    intercept(
+      """
+        |SELECT TRANSFORM(a, b, c)
+        |  ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+        |  WITH SERDEPROPERTIES(
+        |    "separatorChar" = "\t",
+        |    "quoteChar" = "'",
+        |    "escapeChar" = "\\")
+        |  USING 'cat' AS (a, b, c)
+        |  ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+        |  WITH SERDEPROPERTIES(
+        |    "separatorChar" = "\t",
+        |    "quoteChar" = "'",
+        |    "escapeChar" = "\\")
+        |FROM testData
+      """.stripMargin,
+      "TRANSFORM with serde is only supported in hive mode")
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
index d5b0885555462..0e1a6df4c3020 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
@@ -16,12 +16,10 @@
  */
 package org.apache.spark.sql.catalyst.parser
 
-import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.plans.SQLHelper
+import org.apache.spark.sql.catalyst.{SQLKeywordUtils, TableIdentifier}
 import org.apache.spark.sql.internal.SQLConf
 
-class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper {
+class TableIdentifierParserSuite extends SQLKeywordUtils {
   import CatalystSqlParser._
 
   // Add "$elem$", "$value$" & "$key$"
@@ -285,333 +283,6 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper {
     "where",
     "with")
 
-  // All the keywords in `docs/sql-keywords.md` are listed below:
-  val allCandidateKeywords = Set(
-    "add",
-    "after",
-    "all",
-    "alter",
-    "analyze",
-    "and",
-    "anti",
-    "any",
-    "archive",
-    "array",
-    "as",
-    "asc",
-    "at",
-    "authorization",
-    "between",
-    "both",
-    "bucket",
-    "buckets",
-    "by",
-    "cache",
-    "cascade",
-    "case",
-    "cast",
-    "change",
-    "check",
-    "clear",
-    "cluster",
-    "clustered",
-    "codegen",
-    "collate",
-    "collection",
-    "column",
-    "columns",
-    "comment",
-    "commit",
-    "compact",
-    "compactions",
-    "compute",
-    "concatenate",
-    "constraint",
-    "cost",
-    "create",
-    "cross",
-    "cube",
-    "current",
-    "current_date",
-    "current_time",
-    "current_timestamp",
-    "current_user",
-    "data",
-    "database",
-    "databases",
-    "day",
-    "dbproperties",
-    "defined",
-    "delete",
-    "delimited",
-    "desc",
-    "describe",
-    "dfs",
-    "directories",
-    "directory",
-    "distinct",
-    "distribute",
-    "div",
-    "drop",
-    "else",
-    "end",
-    "escape",
-    "escaped",
-    "except",
-    "exchange",
-    "exists",
-    "explain",
-    "export",
-    "extended",
-    "external",
-    "extract",
-    "false",
-    "fetch",
-    "fields",
-    "fileformat",
-    "first",
-    "following",
-    "for",
-    "foreign",
-    "format",
-    "formatted",
-    "from",
-    "full",
-    "function",
-    "functions",
-    "global",
-    "grant",
-    "group",
-    "grouping",
-    "having",
-    "hour",
-    "if",
-    "ignore",
-    "import",
-    "in",
-    "index",
-    "indexes",
-    "inner",
-    "inpath",
-    "inputformat",
-    "insert",
-    "intersect",
-    "interval",
-    "into",
-    "is",
-    "items",
-    "join",
-    "keys",
-    "last",
-    "lateral",
-    "lazy",
-    "leading",
-    "left",
-    "like",
-    "limit",
-    "lines",
-    "list",
-    "load",
-    "local",
-    "location",
-    "lock",
-    "locks",
-    "logical",
-    "macro",
-    "map",
-    "minus",
-    "minute",
-    "month",
-    "msck",
-    "namespaces",
-    "natural",
-    "no",
-    "not",
-    "null",
-    "nulls",
-    "of",
-    "on",
-    "only",
-    "option",
-    "options",
-    "or",
-    "order",
-    "out",
-    "outer",
-    "outputformat",
-    "over",
-    "overlaps",
-    "overlay",
-    "overwrite",
-    "partition",
-    "partitioned",
-    "partitions",
-    "percent",
-    "pivot",
-    "placing",
-    "position",
-    "preceding",
-    "primary",
-    "principals",
-    "purge",
-    "query",
-    "range",
-    "recordreader",
-    "recordwriter",
-    "recover",
-    "reduce",
-    "references",
-    "refresh",
-    "rename",
-    "repair",
-    "replace",
-    "reset",
-    "restrict",
-    "revoke",
-    "right",
-    "rlike",
-    "role",
-    "roles",
-    "rollback",
-    "rollup",
-    "row",
-    "rows",
-    "schema",
-    "second",
-    "select",
-    "semi",
-    "separated",
-    "serde",
-    "serdeproperties",
-    "session_user",
-    "set",
-    "sets",
-    "show",
-    "skewed",
-    "some",
-    "sort",
-    "sorted",
-    "start",
-    "statistics",
-    "stored",
-    "stratify",
-    "struct",
-    "substr",
-    "substring",
-    "table",
-    "tables",
-    "tablesample",
-    "tblproperties",
-    "temporary",
-    "terminated",
-    "then",
-    "to",
-    "touch",
-    "trailing",
-    "transaction",
-    "transactions",
-    "transform",
-    "true",
-    "truncate",
-    "unarchive",
-    "unbounded",
-    "uncache",
-    "union",
-    "unique",
-    "unknown",
-    "unlock",
-    "unset",
-    "use",
-    "user",
-    "using",
-    "values",
-    "view",
-    "views",
-    "when",
-    "where",
-    "window",
-    "with",
-    "year")
-
-  val reservedKeywordsInAnsiMode = Set(
-    "all",
-    "and",
-    "anti",
-    "any",
-    "as",
-    "authorization",
-    "both",
-    "case",
-    "cast",
-    "check",
-    "collate",
-    "column",
-    "constraint",
-    "create",
-    "cross",
-    "current_date",
-    "current_time",
-    "current_timestamp",
-    "current_user",
-    "day",
-    "distinct",
-    "else",
-    "end",
-    "escape",
-    "except",
-    "false",
-    "fetch",
-    "for",
-    "foreign",
-    "from",
-    "full",
-    "grant",
-    "group",
-    "having",
-    "hour",
-    "in",
-    "inner",
-    "intersect",
-    "into",
-    "join",
-    "is",
-    "leading",
-    "left",
-    "minute",
-    "month",
-    "natural",
-    "not",
-    "null",
-    "on",
-    "only",
-    "or",
-    "order",
-    "outer",
-    "overlaps",
-    "primary",
-    "references",
-    "right",
-    "select",
-    "semi",
-    "session_user",
-    "minus",
-    "second",
-    "some",
-    "table",
-    "then",
-    "to",
-    "trailing",
-    "union",
-    "unique",
-    "unknown",
-    "user",
-    "using",
-    "when",
-    "where",
-    "with",
-    "year")
-
-  val nonReservedKeywordsInAnsiMode = allCandidateKeywords -- reservedKeywordsInAnsiMode
 
   test("table identifier") {
     // Regular names.
@@ -683,7 +354,7 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper {
     assert(complexName === parseTableIdentifier("```d``b``1`.```weird``table``name`"))
     assert(complexName === parseTableIdentifier(complexName.quotedString))
     intercept[ParseException](parseTableIdentifier(complexName.unquotedString))
-    // Table identifier contains countious backticks should be treated correctly.
+    // Table identifier contains continuous backticks should be treated correctly.
     val complexName2 = TableIdentifier("x``y", Some("d``b"))
     assert(complexName2 === parseTableIdentifier(complexName2.quotedString))
   }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableSchemaParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableSchemaParserSuite.scala
index 6803fc307f919..5519f016e48d3 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableSchemaParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableSchemaParserSuite.scala
@@ -57,11 +57,6 @@ class TableSchemaParserSuite extends SparkFunSuite {
         |anotherArray:Array<char(9)>>
       """.stripMargin.replace("\n", "")
 
-    val builder = new MetadataBuilder
-    builder.putString(HIVE_TYPE_STRING,
-      "struct<struct:struct<deciMal:decimal(10,0),anotherDecimal:decimal(5,2)>," +
-        "MAP:map<timestamp,varchar(10)>,arrAy:array<double>,anotherArray:array<char(9)>>")
-
     val expectedDataType =
       StructType(
         StructField("complexStructCol", StructType(
@@ -69,11 +64,9 @@ class TableSchemaParserSuite extends SparkFunSuite {
             StructType(
               StructField("deciMal", DecimalType.USER_DEFAULT) ::
                 StructField("anotherDecimal", DecimalType(5, 2)) :: Nil)) ::
-            StructField("MAP", MapType(TimestampType, StringType)) ::
+            StructField("MAP", MapType(TimestampType, VarcharType(10))) ::
             StructField("arrAy", ArrayType(DoubleType)) ::
-            StructField("anotherArray", ArrayType(StringType)) :: Nil),
-          nullable = true,
-          builder.build()) :: Nil)
+            StructField("anotherArray", ArrayType(CharType(9))) :: Nil)) :: Nil)
 
     assert(parse(tableSchemaString) === expectedDataType)
   }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/planning/ScanOperationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/planning/ScanOperationSuite.scala
index 7790f467a890b..1290f770349e7 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/planning/ScanOperationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/planning/ScanOperationSuite.scala
@@ -39,6 +39,7 @@ class ScanOperationSuite extends SparkFunSuite {
         assert(projects(0) === colB)
         assert(projects(1) === aliasR)
         assert(filters.size === 1)
+      case _ => assert(false)
     }
   }
 
@@ -50,6 +51,7 @@ class ScanOperationSuite extends SparkFunSuite {
         assert(projects(0) === colA)
         assert(projects(1) === colB)
         assert(filters.size === 1)
+      case _ => assert(false)
     }
   }
 
@@ -65,6 +67,7 @@ class ScanOperationSuite extends SparkFunSuite {
         assert(projects.size === 2)
         assert(projects(0) === colA)
         assert(projects(1) === aliasId)
+      case _ => assert(false)
     }
   }
 
@@ -81,6 +84,7 @@ class ScanOperationSuite extends SparkFunSuite {
         assert(projects(0) === colA)
         assert(projects(1) === aliasR)
         assert(filters.size === 1)
+      case _ => assert(false)
     }
   }
 
@@ -93,6 +97,7 @@ class ScanOperationSuite extends SparkFunSuite {
         assert(projects(0) === colA)
         assert(projects(1) === aliasR)
         assert(filters.size === 1)
+      case _ => assert(false)
     }
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/LogicalPlanSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/LogicalPlanSuite.scala
index 84452399de824..0cd6d8164fe8d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/LogicalPlanSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/LogicalPlanSuite.scala
@@ -18,7 +18,9 @@
 package org.apache.spark.sql.catalyst.plans
 
 import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference, Literal, NamedExpression}
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.types.IntegerType
 
@@ -66,6 +68,9 @@ class LogicalPlanSuite extends SparkFunSuite {
 
     case class TestBinaryRelation(left: LogicalPlan, right: LogicalPlan) extends BinaryNode {
       override def output: Seq[Attribute] = left.output ++ right.output
+      override protected def withNewChildrenInternal(
+          newLeft: LogicalPlan, newRight: LogicalPlan): LogicalPlan =
+        copy(left = newLeft, right = newRight)
     }
 
     require(relation.isStreaming === false)
@@ -93,4 +98,11 @@ class LogicalPlanSuite extends SparkFunSuite {
       OneRowRelation())
     assert(result.sameResult(expected))
   }
+
+  test("SPARK-35231: logical.Range override maxRowsPerPartition") {
+    assert(Range(0, 100, 1, 3).maxRowsPerPartition === Some(34))
+    assert(Range(0, 100, 1, 4).maxRowsPerPartition === Some(25))
+    assert(Range(0, 100, 1, 3).select('id).maxRowsPerPartition === Some(34))
+    assert(Range(0, 100, 1, 3).where('id % 2 === 1).maxRowsPerPartition === Some(34))
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/PlanTest.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/PlanTest.scala
index 9600357f43cc9..207ca9097f640 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/PlanTest.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/PlanTest.scala
@@ -22,6 +22,7 @@ import org.scalatest.Suite
 import org.scalatest.Tag
 
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.SQLConfHelper
 import org.apache.spark.sql.catalyst.analysis.SimpleAnalyzer
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode
@@ -43,12 +44,18 @@ trait CodegenInterpretedPlanTest extends PlanTest {
     val codegenMode = CodegenObjectFactoryMode.CODEGEN_ONLY.toString
     val interpretedMode = CodegenObjectFactoryMode.NO_CODEGEN.toString
 
-    withSQLConf(SQLConf.CODEGEN_FACTORY_MODE.key -> codegenMode) {
-      super.test(testName + " (codegen path)", testTags: _*)(testFun)(pos)
-    }
-    withSQLConf(SQLConf.CODEGEN_FACTORY_MODE.key -> interpretedMode) {
-      super.test(testName + " (interpreted path)", testTags: _*)(testFun)(pos)
-    }
+    super.test(testName + " (codegen path)", testTags: _*)(
+      withSQLConf(SQLConf.CODEGEN_FACTORY_MODE.key -> codegenMode) { testFun })(pos)
+    super.test(testName + " (interpreted path)", testTags: _*)(
+      withSQLConf(SQLConf.CODEGEN_FACTORY_MODE.key -> interpretedMode) { testFun })(pos)
+  }
+
+  protected def testFallback(
+      testName: String,
+      testTags: Tag*)(testFun: => Any)(implicit pos: source.Position): Unit = {
+    val codegenMode = CodegenObjectFactoryMode.FALLBACK.toString
+    super.test(testName + " (codegen fallback mode)", testTags: _*)(
+      withSQLConf(SQLConf.CODEGEN_FACTORY_MODE.key -> codegenMode) { testFun })(pos)
   }
 }
 
@@ -56,19 +63,16 @@ trait CodegenInterpretedPlanTest extends PlanTest {
  * Provides helper methods for comparing plans, but without the overhead of
  * mandating a FunSuite.
  */
-trait PlanTestBase extends PredicateHelper with SQLHelper { self: Suite =>
-
-  // TODO(gatorsmile): remove this from PlanTest and all the analyzer rules
-  protected def conf = SQLConf.get
+trait PlanTestBase extends PredicateHelper with SQLHelper with SQLConfHelper { self: Suite =>
 
   /**
    * Since attribute references are given globally unique ids during analysis,
    * we must normalize them to check if two different queries are identical.
    */
-  protected def normalizeExprIds(plan: LogicalPlan) = {
+  protected def normalizeExprIds(plan: LogicalPlan): LogicalPlan = {
     plan transformAllExpressions {
       case s: ScalarSubquery =>
-        s.copy(exprId = ExprId(0))
+        s.copy(plan = normalizeExprIds(s.plan), exprId = ExprId(0))
       case e: Exists =>
         e.copy(exprId = ExprId(0))
       case l: ListQuery =>
@@ -86,7 +90,7 @@ trait PlanTestBase extends PredicateHelper with SQLHelper { self: Suite =>
     }
   }
 
-  private def rewriteNameFromAttrNullability(plan: LogicalPlan): LogicalPlan = {
+  protected def rewriteNameFromAttrNullability(plan: LogicalPlan): LogicalPlan = {
     plan.transformAllExpressions {
       case a @ AttributeReference(name, _, false, _) =>
         a.copy(name = s"*$name")(exprId = a.exprId, qualifier = a.qualifier)
@@ -104,8 +108,8 @@ trait PlanTestBase extends PredicateHelper with SQLHelper { self: Suite =>
   protected def normalizePlan(plan: LogicalPlan): LogicalPlan = {
     plan transform {
       case Filter(condition: Expression, child: LogicalPlan) =>
-        Filter(splitConjunctivePredicates(condition).map(rewriteEqual).sortBy(_.hashCode())
-          .reduce(And), child)
+        Filter(splitConjunctivePredicates(condition).map(rewriteBinaryComparison)
+          .sortBy(_.hashCode()).reduce(And), child)
       case sample: Sample =>
         sample.copy(seed = 0L)
       case Join(left, right, joinType, condition, hint) if condition.isDefined =>
@@ -117,23 +121,26 @@ trait PlanTestBase extends PredicateHelper with SQLHelper { self: Suite =>
         }
 
         val newCondition =
-          splitConjunctivePredicates(condition.get).map(rewriteEqual).sortBy(_.hashCode())
-            .reduce(And)
+          splitConjunctivePredicates(condition.get).map(rewriteBinaryComparison)
+            .sortBy(_.hashCode()).reduce(And)
         Join(left, right, newJoinType, Some(newCondition), hint)
     }
   }
 
   /**
-   * Rewrite [[EqualTo]] and [[EqualNullSafe]] operator to keep order. The following cases will be
+   * Rewrite [[BinaryComparison]] operator to keep order. The following cases will be
    * equivalent:
    * 1. (a = b), (b = a);
    * 2. (a <=> b), (b <=> a).
+   * 3. (a > b), (b < a)
    */
-  private def rewriteEqual(condition: Expression): Expression = condition match {
-    case eq @ EqualTo(l: Expression, r: Expression) =>
-      Seq(l, r).sortBy(_.hashCode()).reduce(EqualTo)
-    case eq @ EqualNullSafe(l: Expression, r: Expression) =>
-      Seq(l, r).sortBy(_.hashCode()).reduce(EqualNullSafe)
+  private def rewriteBinaryComparison(condition: Expression): Expression = condition match {
+    case EqualTo(l, r) => Seq(l, r).sortBy(_.hashCode()).reduce(EqualTo)
+    case EqualNullSafe(l, r) => Seq(l, r).sortBy(_.hashCode()).reduce(EqualNullSafe)
+    case GreaterThan(l, r) if l.hashCode() > r.hashCode() => LessThan(r, l)
+    case LessThan(l, r) if l.hashCode() > r.hashCode() => GreaterThan(r, l)
+    case GreaterThanOrEqual(l, r) if l.hashCode() > r.hashCode() => LessThanOrEqual(r, l)
+    case LessThanOrEqual(l, r) if l.hashCode() > r.hashCode() => GreaterThanOrEqual(r, l)
     case _ => condition // Don't reorder.
   }
 
@@ -165,34 +172,4 @@ trait PlanTestBase extends PredicateHelper with SQLHelper { self: Suite =>
   protected def compareExpressions(e1: Expression, e2: Expression): Unit = {
     comparePlans(Filter(e1, OneRowRelation()), Filter(e2, OneRowRelation()), checkAnalysis = false)
   }
-
-  /** Fails the test if the join order in the two plans do not match */
-  protected def compareJoinOrder(plan1: LogicalPlan, plan2: LogicalPlan): Unit = {
-    val normalized1 = normalizePlan(normalizeExprIds(plan1))
-    val normalized2 = normalizePlan(normalizeExprIds(plan2))
-    if (!sameJoinPlan(normalized1, normalized2)) {
-      fail(
-        s"""
-           |== FAIL: Plans do not match ===
-           |${sideBySide(
-             rewriteNameFromAttrNullability(normalized1).treeString,
-             rewriteNameFromAttrNullability(normalized2).treeString).mkString("\n")}
-         """.stripMargin)
-    }
-  }
-
-  /** Consider symmetry for joins when comparing plans. */
-  private def sameJoinPlan(plan1: LogicalPlan, plan2: LogicalPlan): Boolean = {
-    (plan1, plan2) match {
-      case (j1: Join, j2: Join) =>
-        (sameJoinPlan(j1.left, j2.left) && sameJoinPlan(j1.right, j2.right)
-          && j1.hint.leftHint == j2.hint.leftHint && j1.hint.rightHint == j2.hint.rightHint) ||
-          (sameJoinPlan(j1.left, j2.right) && sameJoinPlan(j1.right, j2.left)
-            && j1.hint.leftHint == j2.hint.rightHint && j1.hint.rightHint == j2.hint.leftHint)
-      case (p1: Project, p2: Project) =>
-        p1.projectList == p2.projectList && sameJoinPlan(p1.child, p2.child)
-      case _ =>
-        plan1 == plan2
-    }
-  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/QueryPlanSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/QueryPlanSuite.scala
index 91ce187f4d270..404c8895c4d11 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/QueryPlanSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/QueryPlanSuite.scala
@@ -20,9 +20,11 @@ package org.apache.spark.sql.catalyst.plans
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
-import org.apache.spark.sql.catalyst.dsl.plans
-import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, ListQuery, Literal, NamedExpression}
-import org.apache.spark.sql.catalyst.plans.logical.{Filter, Project, Union}
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeReference, Expression, ListQuery, Literal, NamedExpression}
+import org.apache.spark.sql.catalyst.plans.logical.{Filter, LocalRelation, LogicalPlan, Project, Union}
+import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, Origin}
 import org.apache.spark.sql.types.IntegerType
 
@@ -31,7 +33,7 @@ class QueryPlanSuite extends SparkFunSuite {
   test("origin remains the same after mapExpressions (SPARK-23823)") {
     CurrentOrigin.setPosition(0, 0)
     val column = AttributeReference("column", IntegerType)(NamedExpression.newExprId)
-    val query = plans.DslLogicalPlan(plans.table("table")).select(column)
+    val query = DslLogicalPlan(table("table")).select(column)
     CurrentOrigin.reset()
 
     val mappedQuery = query mapExpressions {
@@ -83,4 +85,20 @@ class QueryPlanSuite extends SparkFunSuite {
     assert(countRelationsInPlan == 2)
     assert(countRelationsInPlanAndSubqueries == 5)
   }
+
+  test("SPARK-33035: consecutive attribute updates in parent plan nodes") {
+    val testRule = new Rule[LogicalPlan] {
+      override def apply(plan: LogicalPlan): LogicalPlan = plan.transformUpWithNewOutput {
+        case p @ Project(projList, _) =>
+          // Assigns new `ExprId`s for output references
+          val newPlan = p.copy(projectList = projList.map { ne => Alias(ne, ne.name)() })
+          val attrMapping = p.output.zip(newPlan.output)
+          newPlan -> attrMapping
+      }
+    }
+
+    val t = LocalRelation('a.int, 'b.int)
+    val plan = t.select($"a", $"b").select($"a", $"b").select($"a", $"b").analyze
+    assert(testRule(plan).resolved)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/SQLHelper.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/SQLHelper.scala
index d213743946e76..5deab79b9f67d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/SQLHelper.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/SQLHelper.scala
@@ -21,6 +21,8 @@ import java.time.ZoneId
 
 import scala.util.control.NonFatal
 
+import org.scalatest.Assertions.fail
+
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils
 import org.apache.spark.sql.catalyst.util.DateTimeUtils.getZoneId
@@ -83,4 +85,11 @@ trait SQLHelper {
       }
     }
   }
+
+  protected lazy val sparkHome: String = {
+    if (!(sys.props.contains("spark.test.home") || sys.env.contains("SPARK_HOME"))) {
+      fail("spark.test.home or SPARK_HOME is not set.")
+    }
+    sys.props.getOrElse("spark.test.home", sys.env("SPARK_HOME"))
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlanIntegritySuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlanIntegritySuite.scala
new file mode 100644
index 0000000000000..009e2a731fe41
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlanIntegritySuite.scala
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference}
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.types.LongType
+
+class LogicalPlanIntegritySuite extends PlanTest {
+  import LogicalPlanIntegrity._
+
+  case class OutputTestPlan(child: LogicalPlan, output: Seq[Attribute]) extends UnaryNode {
+    override val analyzed = true
+    override protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan =
+      copy(child = newChild)
+  }
+
+  test("Checks if the same `ExprId` refers to a semantically-equal attribute in a plan output") {
+    val t = LocalRelation('a.int, 'b.int)
+    assert(hasUniqueExprIdsForOutput(OutputTestPlan(t, t.output)))
+    assert(!hasUniqueExprIdsForOutput(OutputTestPlan(t, t.output.zipWithIndex.map {
+      case (a, i) => AttributeReference(s"c$i", LongType)(a.exprId)
+    })))
+  }
+
+  test("Checks if reference ExprIds are not reused when assigning a new ExprId") {
+    val t = LocalRelation('a.int, 'b.int)
+    val Seq(a, b) = t.output
+    assert(checkIfSameExprIdNotReused(t.select(Alias(a + 1, "a")())))
+    assert(!checkIfSameExprIdNotReused(t.select(Alias(a + 1, "a")(exprId = a.exprId))))
+    assert(checkIfSameExprIdNotReused(t.select(Alias(a + 1, "a")(exprId = b.exprId))))
+    assert(checkIfSameExprIdNotReused(t.select(Alias(a + b, "ab")())))
+    assert(!checkIfSameExprIdNotReused(t.select(Alias(a + b, "ab")(exprId = a.exprId))))
+    assert(!checkIfSameExprIdNotReused(t.select(Alias(a + b, "ab")(exprId = b.exprId))))
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/BasicStatsEstimationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/BasicStatsEstimationSuite.scala
index dfe790dca54d8..389c60b65b50d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/BasicStatsEstimationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/BasicStatsEstimationSuite.scala
@@ -22,13 +22,12 @@ import org.mockito.Mockito.mock
 import org.apache.spark.sql.catalyst.analysis.ResolvedNamespace
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
-import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeMap, AttributeReference, Literal}
+import org.apache.spark.sql.catalyst.expressions.{Ascending, Attribute, AttributeMap, AttributeReference, Literal, SortOrder}
 import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.connector.catalog.SupportsNamespaces
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.IntegerType
-
+import org.apache.spark.sql.types.{BooleanType, ByteType, IntegerType, LongType}
 
 class BasicStatsEstimationSuite extends PlanTest with StatsEstimationTestBase {
   val attribute = attr("key")
@@ -42,13 +41,90 @@ class BasicStatsEstimationSuite extends PlanTest with StatsEstimationTestBase {
     // row count * (overhead + column size)
     size = Some(10 * (8 + 4)))
 
-  test("range") {
+  test("range with positive step") {
     val range = Range(1, 5, 1, None)
-    val rangeStats = Statistics(sizeInBytes = 4 * 8)
-    checkStats(
-      range,
-      expectedStatsCboOn = rangeStats,
-      expectedStatsCboOff = rangeStats)
+    val rangeStats = Statistics(
+      sizeInBytes = 4 * 8,
+      rowCount = Some(4),
+      attributeStats = AttributeMap(
+        range.output.map(
+          attr =>
+            (
+              attr,
+              ColumnStat(
+                distinctCount = Some(4),
+                min = Some(1),
+                max = Some(4),
+                nullCount = Some(0),
+                maxLen = Some(LongType.defaultSize),
+                avgLen = Some(LongType.defaultSize))))))
+    checkStats(range, expectedStatsCboOn = rangeStats, expectedStatsCboOff = rangeStats)
+  }
+
+  test("range with positive step where end minus start not divisible by step") {
+    val range = Range(-4, 5, 2, None)
+    val rangeStats = Statistics(
+      sizeInBytes = 5 * 8,
+      rowCount = Some(5),
+      attributeStats = AttributeMap(
+        range.output.map(
+          attr =>
+            (
+              attr,
+              ColumnStat(
+                distinctCount = Some(5),
+                min = Some(-4),
+                max = Some(4),
+                nullCount = Some(0),
+                maxLen = Some(LongType.defaultSize),
+                avgLen = Some(LongType.defaultSize))))))
+    checkStats(range, expectedStatsCboOn = rangeStats, expectedStatsCboOff = rangeStats)
+  }
+
+  test("range with negative step") {
+    val range = Range(-10, -20, -2, None)
+    val rangeStats = Statistics(
+      sizeInBytes = 5 * 8,
+      rowCount = Some(5),
+      attributeStats = AttributeMap(
+        range.output.map(
+          attr =>
+            (
+              attr,
+              ColumnStat(
+                distinctCount = Some(5),
+                min = Some(-18),
+                max = Some(-10),
+                nullCount = Some(0),
+                maxLen = Some(LongType.defaultSize),
+                avgLen = Some(LongType.defaultSize))))))
+    checkStats(range, expectedStatsCboOn = rangeStats, expectedStatsCboOff = rangeStats)
+  }
+
+  test("range with negative step where end minus start not divisible by step") {
+    val range = Range(-10, -20, -3, None)
+    val rangeStats = Statistics(
+      sizeInBytes = 4 * 8,
+      rowCount = Some(4),
+      attributeStats = AttributeMap(
+        range.output.map(
+          attr =>
+            (
+              attr,
+              ColumnStat(
+                distinctCount = Some(4),
+                min = Some(-19),
+                max = Some(-10),
+                nullCount = Some(0),
+                maxLen = Some(LongType.defaultSize),
+                avgLen = Some(LongType.defaultSize))))))
+    checkStats(range, expectedStatsCboOn = rangeStats, expectedStatsCboOff = rangeStats)
+  }
+
+  test("range with empty output") {
+    val range = Range(-10, -10, -1, None)
+    val rangeStats = Statistics(sizeInBytes = 0, rowCount = Some(0))
+    checkStats(range, expectedStatsCboOn = rangeStats, expectedStatsCboOff = rangeStats)
   }
 
   test("windows") {
@@ -84,6 +160,12 @@ class BasicStatsEstimationSuite extends PlanTest with StatsEstimationTestBase {
     checkStats(globalLimit, stats)
   }
 
+  test("tail estimation") {
+    checkStats(Tail(Literal(1), plan), Statistics(sizeInBytes = 12, rowCount = Some(1)))
+    checkStats(Tail(Literal(20), plan), plan.stats.copy(attributeStats = AttributeMap(Nil)))
+    checkStats(Tail(Literal(0), plan), Statistics(sizeInBytes = 1, rowCount = Some(0)))
+  }
+
   test("sample estimation") {
     val sample = Sample(0.0, 0.5, withReplacement = false, (math.random * 1000).toLong, plan)
     checkStats(sample, Statistics(sizeInBytes = 60, rowCount = Some(5)))
@@ -128,6 +210,75 @@ class BasicStatsEstimationSuite extends PlanTest with StatsEstimationTestBase {
       expectedStatsCboOff = Statistics.DUMMY)
   }
 
+  test("SPARK-33954: Some operator missing rowCount when enable CBO") {
+    checkStats(
+      plan.repartition(10),
+      expectedStatsCboOn = Statistics(sizeInBytes = 120, rowCount = Some(10)),
+      expectedStatsCboOff = Statistics(sizeInBytes = 120))
+  }
+
+  test("SPARK-34031: Union operator missing rowCount when enable CBO") {
+    val union = Union(plan :: plan :: plan :: Nil)
+    val childrenSize = union.children.size
+    val sizeInBytes = plan.size.get * childrenSize
+    val rowCount = Some(plan.rowCount * childrenSize)
+    val attributeStats = AttributeMap(
+      Seq(
+        attribute -> ColumnStat(min = Some(1), max = Some(10))))
+    checkStats(
+      union,
+      expectedStatsCboOn = Statistics(sizeInBytes = sizeInBytes,
+        rowCount = rowCount,
+        attributeStats = attributeStats),
+      expectedStatsCboOff = Statistics(sizeInBytes = sizeInBytes))
+  }
+
+  test("SPARK-34121: Intersect operator missing rowCount when enable CBO") {
+    val intersect = Intersect(plan, plan, false)
+    val childrenSize = intersect.children.size
+    val sizeInBytes = plan.size.get
+    val rowCount = Some(plan.rowCount)
+    checkStats(
+      intersect,
+      expectedStatsCboOn = Statistics(sizeInBytes = sizeInBytes, rowCount = rowCount),
+      expectedStatsCboOff = Statistics(sizeInBytes = sizeInBytes))
+  }
+
+  test("row size and column stats estimation for sort") {
+    val columnInfo = AttributeMap(
+      Seq(
+        AttributeReference("cbool", BooleanType)() -> ColumnStat(
+          distinctCount = Some(2),
+          min = Some(false),
+          max = Some(true),
+          nullCount = Some(0),
+          avgLen = Some(1),
+          maxLen = Some(1)),
+        AttributeReference("cbyte", ByteType)() -> ColumnStat(
+          distinctCount = Some(2),
+          min = Some(1),
+          max = Some(2),
+          nullCount = Some(0),
+          avgLen = Some(1),
+          maxLen = Some(1))))
+
+    val expectedSize = 16
+    val child = StatsTestPlan(
+      outputList = columnInfo.keys.toSeq,
+      rowCount = 2,
+      attributeStats = columnInfo,
+      size = Some(expectedSize))
+
+    val sortOrder = SortOrder(columnInfo.keys.head, Ascending)
+    val sort = Sort(order = Seq(sortOrder), global = true, child = child)
+    val expectedSortStats =
+      Statistics(sizeInBytes = expectedSize, rowCount = Some(2), attributeStats = columnInfo)
+    checkStats(
+      sort,
+      expectedStatsCboOn = expectedSortStats,
+      expectedStatsCboOff = Statistics(sizeInBytes = expectedSize))
+  }
+
   /** Check estimated stats when cbo is turned on/off. */
   private def checkStats(
       plan: LogicalPlan,
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/FilterEstimationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/FilterEstimationSuite.scala
index 1cf888519077a..878fae4c547b3 100755
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/FilterEstimationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/FilterEstimationSuite.scala
@@ -23,7 +23,7 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.Literal.{FalseLiteral, TrueLiteral}
 import org.apache.spark.sql.catalyst.plans.LeftOuter
 import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.catalyst.plans.logical.statsEstimation.{ColumnStatsMap, FilterEstimation}
+import org.apache.spark.sql.catalyst.plans.logical.statsEstimation.ColumnStatsMap
 import org.apache.spark.sql.catalyst.plans.logical.statsEstimation.EstimationUtils._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.types._
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/JoinEstimationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/JoinEstimationSuite.scala
index 6c5a2b247fc23..cdfc863cc0212 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/JoinEstimationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/JoinEstimationSuite.scala
@@ -551,4 +551,26 @@ class JoinEstimationSuite extends StatsEstimationTestBase {
       attributeStats = AttributeMap(Nil))
     assert(join.stats == expectedStats)
   }
+
+  test("SPARK-33018 Fix estimate statistics issue if child has 0 bytes") {
+    case class MyStatsTestPlan(
+        outputList: Seq[Attribute],
+        sizeInBytes: BigInt) extends LeafNode {
+      override def output: Seq[Attribute] = outputList
+      override def computeStats(): Statistics = Statistics(sizeInBytes = sizeInBytes)
+    }
+
+    val left = MyStatsTestPlan(
+      outputList = Seq("key-1-2", "key-2-4").map(nameToAttr),
+      sizeInBytes = BigInt(100))
+
+    val right = MyStatsTestPlan(
+      outputList = Seq("key-1-2", "key-2-3").map(nameToAttr),
+      sizeInBytes = BigInt(0))
+
+    val join = Join(left, right, LeftOuter,
+      Some(EqualTo(nameToAttr("key-2-4"), nameToAttr("key-2-3"))), JoinHint.NONE)
+
+    assert(join.stats == Statistics(sizeInBytes = 100))
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/StatsEstimationTestBase.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/StatsEstimationTestBase.scala
index 9dceca59f5b87..17d182ba172ba 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/StatsEstimationTestBase.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/StatsEstimationTestBase.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.catalyst.statsEstimation
 
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.analysis.MultiInstanceRelation
 import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeMap, AttributeReference}
 import org.apache.spark.sql.catalyst.plans.logical.{ColumnStat, LeafNode, LogicalPlan, Statistics}
 import org.apache.spark.sql.internal.SQLConf
@@ -26,17 +27,20 @@ import org.apache.spark.sql.types.{IntegerType, StringType}
 
 trait StatsEstimationTestBase extends SparkFunSuite {
 
-  var originalValue: Boolean = false
+  var originalCBOValue: Boolean = false
+  var originalPlanStatsValue: Boolean = false
 
   override def beforeAll(): Unit = {
     super.beforeAll()
     // Enable stats estimation based on CBO.
-    originalValue = SQLConf.get.getConf(SQLConf.CBO_ENABLED)
+    originalCBOValue = SQLConf.get.getConf(SQLConf.CBO_ENABLED)
+    originalPlanStatsValue = SQLConf.get.getConf(SQLConf.PLAN_STATS_ENABLED)
     SQLConf.get.setConf(SQLConf.CBO_ENABLED, true)
   }
 
   override def afterAll(): Unit = {
-    SQLConf.get.setConf(SQLConf.CBO_ENABLED, originalValue)
+    SQLConf.get.setConf(SQLConf.CBO_ENABLED, originalCBOValue)
+    SQLConf.get.setConf(SQLConf.PLAN_STATS_ENABLED, originalPlanStatsValue)
     super.afterAll()
   }
 
@@ -69,11 +73,12 @@ case class StatsTestPlan(
     outputList: Seq[Attribute],
     rowCount: BigInt,
     attributeStats: AttributeMap[ColumnStat],
-    size: Option[BigInt] = None) extends LeafNode {
+    size: Option[BigInt] = None) extends LeafNode with MultiInstanceRelation {
   override def output: Seq[Attribute] = outputList
   override def computeStats(): Statistics = Statistics(
     // If sizeInBytes is useless in testing, we just use a fake value
     sizeInBytes = size.getOrElse(Int.MaxValue),
     rowCount = Some(rowCount),
     attributeStats = attributeStats)
+  override def newInstance(): LogicalPlan = copy(outputList = outputList.map(_.newInstance()))
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/UnionEstimationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/UnionEstimationSuite.scala
new file mode 100644
index 0000000000000..06bd38fd8ad78
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/UnionEstimationSuite.scala
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.statsEstimation
+
+import org.apache.spark.sql.catalyst.expressions.{AttributeMap, AttributeReference}
+import org.apache.spark.sql.catalyst.plans.logical
+import org.apache.spark.sql.catalyst.plans.logical.{ColumnStat, Union}
+import org.apache.spark.sql.types._
+
+class UnionEstimationSuite extends StatsEstimationTestBase {
+
+  test("test row size estimation") {
+    val attrInt = AttributeReference("cint", IntegerType)()
+
+    val sz = Some(BigInt(1024))
+    val child1 = StatsTestPlan(
+      outputList = Seq(attrInt),
+      rowCount = 2,
+      attributeStats = AttributeMap(Nil),
+      size = sz)
+
+    val child2 = StatsTestPlan(
+      outputList = Seq(attrInt),
+      rowCount = 2,
+      attributeStats = AttributeMap(Nil),
+      size = sz)
+
+    val union = Union(Seq(child1, child2))
+    val expectedStats = logical.Statistics(sizeInBytes = 2 * 1024, rowCount = Some(4))
+    assert(union.stats === expectedStats)
+  }
+
+  test("col stats estimation") {
+    val sz = Some(BigInt(1024))
+
+    val attrInt = AttributeReference("cint", IntegerType)()
+    val attrDouble = AttributeReference("cdouble", DoubleType)()
+    val attrShort = AttributeReference("cshort", ShortType)()
+    val attrLong = AttributeReference("clong", LongType)()
+    val attrByte = AttributeReference("cbyte", ByteType)()
+    val attrFloat = AttributeReference("cfloat", FloatType)()
+    val attrDecimal = AttributeReference("cdecimal", DecimalType(5, 4))()
+    val attrDate = AttributeReference("cdate", DateType)()
+    val attrTimestamp = AttributeReference("ctimestamp", TimestampType)()
+
+    val s1 = 1.toShort
+    val s2 = 4.toShort
+    val b1 = 1.toByte
+    val b2 = 4.toByte
+    val columnInfo = AttributeMap(
+      Seq(
+        attrInt -> ColumnStat(
+          distinctCount = Some(2),
+          min = Some(1),
+          max = Some(4),
+          nullCount = Some(0),
+          avgLen = Some(4),
+          maxLen = Some(4)),
+        attrDouble -> ColumnStat(
+          distinctCount = Some(2),
+          min = Some(5.0),
+          max = Some(4.0),
+          nullCount = Some(0),
+          avgLen = Some(4),
+          maxLen = Some(4)),
+        attrShort -> ColumnStat(min = Some(s1), max = Some(s2)),
+        attrLong -> ColumnStat(min = Some(1L), max = Some(4L)),
+        attrByte -> ColumnStat(min = Some(b1), max = Some(b2)),
+        attrFloat -> ColumnStat(min = Some(1.1f), max = Some(4.1f)),
+        attrDecimal -> ColumnStat(min = Some(Decimal(13.5)), max = Some(Decimal(19.5))),
+        attrDate -> ColumnStat(min = Some(1), max = Some(4)),
+        attrTimestamp -> ColumnStat(min = Some(1L), max = Some(4L))))
+
+    val s3 = 2.toShort
+    val s4 = 6.toShort
+    val b3 = 2.toByte
+    val b4 = 6.toByte
+    val columnInfo1: AttributeMap[ColumnStat] = AttributeMap(
+      Seq(
+        AttributeReference("cint1", IntegerType)() -> ColumnStat(
+          distinctCount = Some(2),
+          min = Some(3),
+          max = Some(6),
+          nullCount = Some(0),
+          avgLen = Some(8),
+          maxLen = Some(8)),
+        AttributeReference("cdouble1", DoubleType)() -> ColumnStat(
+          distinctCount = Some(2),
+          min = Some(2.0),
+          max = Some(7.0),
+          nullCount = Some(0),
+          avgLen = Some(8),
+          maxLen = Some(8)),
+        AttributeReference("cshort1", ShortType)() -> ColumnStat(min = Some(s3), max = Some(s4)),
+        AttributeReference("clong1", LongType)() -> ColumnStat(min = Some(2L), max = Some(6L)),
+        AttributeReference("cbyte1", ByteType)() -> ColumnStat(min = Some(b3), max = Some(b4)),
+        AttributeReference("cfloat1", FloatType)() -> ColumnStat(
+          min = Some(2.2f),
+          max = Some(6.1f)),
+        AttributeReference("cdecimal1", DecimalType(5, 4))() -> ColumnStat(
+          min = Some(Decimal(14.5)),
+          max = Some(Decimal(19.9))),
+        AttributeReference("cdate1", DateType)() -> ColumnStat(min = Some(3), max = Some(6)),
+        AttributeReference("ctimestamp1", TimestampType)() -> ColumnStat(
+          min = Some(3L),
+          max = Some(6L))))
+
+    val child1 = StatsTestPlan(
+      outputList = columnInfo.keys.toSeq.sortWith(_.exprId.id < _.exprId.id),
+      rowCount = 2,
+      attributeStats = columnInfo,
+      size = sz)
+
+    val child2 = StatsTestPlan(
+      outputList = columnInfo1.keys.toSeq.sortWith(_.exprId.id < _.exprId.id),
+      rowCount = 2,
+      attributeStats = columnInfo1,
+      size = sz)
+
+    val union = Union(Seq(child1, child2))
+
+    val expectedStats = logical.Statistics(
+      sizeInBytes = 2 * 1024,
+      rowCount = Some(4),
+      attributeStats = AttributeMap(
+        Seq(
+          attrInt -> ColumnStat(min = Some(1), max = Some(6)),
+          attrDouble -> ColumnStat(min = Some(2.0), max = Some(7.0)),
+          attrShort -> ColumnStat(min = Some(s1), max = Some(s4)),
+          attrLong -> ColumnStat(min = Some(1L), max = Some(6L)),
+          attrByte -> ColumnStat(min = Some(b1), max = Some(b4)),
+          attrFloat -> ColumnStat(min = Some(1.1f), max = Some(6.1f)),
+          attrDecimal -> ColumnStat(min = Some(Decimal(13.5)), max = Some(Decimal(19.9))),
+          attrDate -> ColumnStat(min = Some(1), max = Some(6)),
+          attrTimestamp -> ColumnStat(min = Some(1L), max = Some(6L)))))
+    assert(union.stats === expectedStats)
+  }
+
+  test("col stats estimation when min max stats not present for one child") {
+    val sz = Some(BigInt(1024))
+
+    val attrInt = AttributeReference("cint", IntegerType)()
+
+    val columnInfo = AttributeMap(
+      Seq(
+        attrInt -> ColumnStat(
+          distinctCount = Some(2),
+          min = Some(2),
+          max = Some(2),
+          nullCount = Some(0),
+          avgLen = Some(4),
+          maxLen = Some(4))))
+
+    val columnInfo1 = AttributeMap(
+      Seq(
+        AttributeReference("cint1", IntegerType)() -> ColumnStat(
+          distinctCount = Some(2),
+          nullCount = Some(0),
+          avgLen = Some(8),
+          maxLen = Some(8))))
+
+    val child1 = StatsTestPlan(
+      outputList = columnInfo.keys.toSeq,
+      rowCount = 2,
+      attributeStats = columnInfo,
+      size = sz)
+
+    val child2 = StatsTestPlan(
+      outputList = columnInfo1.keys.toSeq,
+      rowCount = 2,
+      attributeStats = columnInfo1,
+      size = sz)
+
+    val union = Union(Seq(child1, child2))
+
+    val expectedStats = logical.Statistics(sizeInBytes = 2 * 1024, rowCount = Some(4))
+    assert(union.stats === expectedStats)
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/RuleExecutorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/RuleExecutorSuite.scala
index 60dd9a1fe8a12..25352e2d242a1 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/RuleExecutorSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/RuleExecutorSuite.scala
@@ -18,9 +18,7 @@
 package org.apache.spark.sql.catalyst.trees
 
 import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.catalyst.errors.TreeNodeException
 import org.apache.spark.sql.catalyst.expressions.{Expression, IntegerLiteral, Literal}
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.rules.{Rule, RuleExecutor}
 
 class RuleExecutorSuite extends SparkFunSuite {
@@ -67,7 +65,7 @@ class RuleExecutorSuite extends SparkFunSuite {
       val batches = Batch("fixedPoint", FixedPoint(10), DecrementLiterals) :: Nil
     }
 
-    val message = intercept[TreeNodeException[LogicalPlan]] {
+    val message = intercept[RuntimeException] {
       ToFixedPoint.execute(Literal(100))
     }.getMessage
     assert(message.contains("Max iterations (10) reached for batch fixedPoint"))
@@ -84,7 +82,7 @@ class RuleExecutorSuite extends SparkFunSuite {
 
     assert(WithSIChecker.execute(Literal(10)) === Literal(9))
 
-    val message = intercept[TreeNodeException[LogicalPlan]] {
+    val message = intercept[RuntimeException] {
       // The input is already invalid as determined by WithSIChecker.isPlanIntegral
       WithSIChecker.execute(Literal(10.1))
     }.getMessage
@@ -102,7 +100,7 @@ class RuleExecutorSuite extends SparkFunSuite {
 
     assert(WithSICheckerForPositiveLiteral.execute(Literal(2)) === Literal(1))
 
-    val message = intercept[TreeNodeException[LogicalPlan]] {
+    val message = intercept[RuntimeException] {
       WithSICheckerForPositiveLiteral.execute(Literal(1))
     }.getMessage
     assert(message.contains("the structural integrity of the plan is broken"))
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
index f5259706325eb..0d316779d8bcb 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
@@ -47,6 +47,8 @@ case class Dummy(optKey: Option[Expression]) extends Expression with CodegenFall
   override def dataType: NullType = NullType
   override lazy val resolved = true
   override def eval(input: InternalRow): Any = null.asInstanceOf[Any]
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
+    copy(optKey = if (optKey.isDefined) Some(newChildren(0)) else None)
 }
 
 case class ComplexPlan(exprs: Seq[Seq[Expression]])
@@ -59,6 +61,8 @@ case class ExpressionInMap(map: Map[String, Expression]) extends Unevaluable {
   override def nullable: Boolean = true
   override def dataType: NullType = NullType
   override lazy val resolved = true
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
+    super.legacyWithNewChildren(newChildren)
 }
 
 case class SeqTupleExpression(sons: Seq[(Expression, Expression)],
@@ -67,6 +71,9 @@ case class SeqTupleExpression(sons: Seq[(Expression, Expression)],
   override def nullable: Boolean = true
   override def dataType: NullType = NullType
   override lazy val resolved = true
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
+    super.legacyWithNewChildren(newChildren)
 }
 
 case class JsonTestTreeNode(arg: Any) extends LeafNode {
@@ -583,7 +590,9 @@ class TreeNodeSuite extends SparkFunSuite with SQLHelper {
         JObject(
           "class" -> classOf[Union].getName,
           "num-children" -> 2,
-          "children" -> List(0, 1)),
+          "children" -> List(0, 1),
+          "byName" -> JBool(false),
+          "allowMissingCol" -> JBool(false)),
         JObject(
           "class" -> classOf[JsonTestTreeNode].getName,
           "num-children" -> 0,
@@ -734,4 +743,33 @@ class TreeNodeSuite extends SparkFunSuite with SQLHelper {
     assertDifferentInstance(leaf, leafCloned)
     assert(leaf.child.eq(leafCloned.asInstanceOf[FakeLeafPlan].child))
   }
+
+  object MalformedClassObject extends Serializable {
+    case class MalformedNameExpression(child: Expression) extends TaggingExpression {
+      override protected def withNewChildInternal(newChild: Expression): Expression =
+        copy(child = newChild)
+    }
+  }
+
+  test("SPARK-32999: TreeNode.nodeName should not throw malformed class name error") {
+    val testTriggersExpectedError = try {
+      classOf[MalformedClassObject.MalformedNameExpression].getSimpleName
+      false
+    } catch {
+      case ex: java.lang.InternalError if ex.getMessage.contains("Malformed class name") =>
+        true
+      case ex: Throwable => throw ex
+    }
+    // This test case only applies on older JDK versions (e.g. JDK8u), and doesn't trigger the
+    // issue on newer JDK versions (e.g. JDK11u).
+    assume(testTriggersExpectedError, "the test case didn't trigger malformed class name error")
+
+    val expr = MalformedClassObject.MalformedNameExpression(Literal(1))
+    try {
+      expr.nodeName
+    } catch {
+      case ex: java.lang.InternalError if ex.getMessage.contains("Malformed class name") =>
+        fail("TreeNode.nodeName should not throw malformed class name error")
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayDataIndexedSeqSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayDataIndexedSeqSuite.scala
index 1e430351b5137..9c3aaea0f7772 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayDataIndexedSeqSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayDataIndexedSeqSuite.scala
@@ -45,7 +45,7 @@ class ArrayDataIndexedSeqSuite extends SparkFunSuite {
       if (e != null) {
         elementDt match {
           // For Nan, etc.
-          case FloatType | DoubleType => assert(seq(i).equals(e))
+          case FloatType | DoubleType => assert(seq(i) == e)
           case _ => assert(seq(i) === e)
         }
       } else {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/CaseInsensitiveMapSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/CaseInsensitiveMapSuite.scala
index a8bb1d0afdb87..db8b7403cb60e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/CaseInsensitiveMapSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/CaseInsensitiveMapSuite.scala
@@ -44,4 +44,12 @@ class CaseInsensitiveMapSuite extends SparkFunSuite {
     assert(m == Map("a" -> "b", "foo" -> "bar", "x" -> "y"))
     shouldBeSerializable(m)
   }
+
+  test("SPARK-32377: CaseInsensitiveMap should be deterministic for addition") {
+    var m = CaseInsensitiveMap(Map.empty[String, String])
+    Seq(("paTh", "1"), ("PATH", "2"), ("Path", "3"), ("patH", "4"), ("path", "5")).foreach { kv =>
+      m = (m + kv).asInstanceOf[CaseInsensitiveMap[String]]
+      assert(m.get("path").contains(kv._2))
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ComplexDataSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ComplexDataSuite.scala
index 229e32479082c..f921f06537080 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ComplexDataSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ComplexDataSuite.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.sql.catalyst.util
 
-import scala.collection._
-
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{BoundReference, GenericInternalRow, SpecificInternalRow, UnsafeMapData, UnsafeProjection}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateFormatterSuite.scala
new file mode 100644
index 0000000000000..0a29d94dd984d
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateFormatterSuite.scala
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import java.time.{DateTimeException, LocalDate}
+
+import org.apache.spark.SparkUpgradeException
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
+import org.apache.spark.sql.catalyst.util.DateTimeUtils._
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
+
+class DateFormatterSuite extends DatetimeFormatterSuite {
+
+  override def checkFormatterCreation(pattern: String, isParsing: Boolean): Unit = {
+    DateFormatter(pattern, UTC, isParsing)
+  }
+
+  override protected def useDateFormatter: Boolean = true
+
+  test("parsing dates") {
+    outstandingTimezonesIds.foreach { timeZone =>
+      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
+        val formatter = DateFormatter(getZoneId(timeZone))
+        val daysSinceEpoch = formatter.parse("2018-12-02")
+        assert(daysSinceEpoch === 17867)
+      }
+    }
+  }
+
+  test("format dates") {
+    outstandingTimezonesIds.foreach { timeZone =>
+      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
+        val formatter = DateFormatter(getZoneId(timeZone))
+        val (days, expected) = (17867, "2018-12-02")
+        val date = formatter.format(days)
+        assert(date === expected)
+        assert(formatter.format(daysToLocalDate(days)) === expected)
+        assert(formatter.format(toJavaDate(days)) === expected)
+      }
+    }
+  }
+
+  test("roundtrip date -> days -> date") {
+    LegacyBehaviorPolicy.values.foreach { parserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> parserPolicy.toString) {
+        LegacyDateFormats.values.foreach { legacyFormat =>
+          Seq(
+            "0050-01-01",
+            "0953-02-02",
+            "1423-03-08",
+            "1582-10-15",
+            "1969-12-31",
+            "1972-08-25",
+            "1975-09-26",
+            "2018-12-12",
+            "2038-01-01",
+            "5010-11-17").foreach { date =>
+            outstandingTimezonesIds.foreach { timeZone =>
+              withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
+                val formatter = DateFormatter(
+                  DateFormatter.defaultPattern,
+                  getZoneId(timeZone),
+                  DateFormatter.defaultLocale,
+                  legacyFormat,
+                  isParsing = false)
+                val days = formatter.parse(date)
+                assert(date === formatter.format(days))
+                assert(date === formatter.format(daysToLocalDate(days)))
+                assert(date === formatter.format(toJavaDate(days)))
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  test("roundtrip days -> date -> days") {
+    LegacyBehaviorPolicy.values.foreach { parserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> parserPolicy.toString) {
+        LegacyDateFormats.values.foreach { legacyFormat =>
+          Seq(
+            -701265,
+            -371419,
+            -199722,
+            -1,
+            0,
+            967,
+            2094,
+            17877,
+            24837,
+            1110657).foreach { days =>
+            outstandingTimezonesIds.foreach { timeZone =>
+              withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
+                val formatter = DateFormatter(
+                  DateFormatter.defaultPattern,
+                  getZoneId(timeZone),
+                  DateFormatter.defaultLocale,
+                  legacyFormat,
+                  isParsing = false)
+                val date = formatter.format(days)
+                val parsed = formatter.parse(date)
+                assert(days === parsed)
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  test("parsing date without explicit day") {
+    val formatter = DateFormatter("yyyy MMM", UTC)
+    val daysSinceEpoch = formatter.parse("2018 Dec")
+    assert(daysSinceEpoch === days(2018, 12, 1))
+  }
+
+  test("formatting negative years with default pattern") {
+    val epochDays = days(-99, 1, 1)
+    assert(DateFormatter(UTC).format(epochDays) === "-0099-01-01")
+  }
+
+  test("special date values") {
+    testSpecialDatetimeValues { zoneId =>
+      val formatter = DateFormatter(zoneId)
+
+      assert(formatter.parse("EPOCH") === 0)
+      val today = localDateToDays(LocalDate.now(zoneId))
+      assert(formatter.parse("Yesterday") === today - 1)
+      assert(formatter.parse("now") === today)
+      assert(formatter.parse("today ") === today)
+      assert(formatter.parse("tomorrow UTC") === today + 1)
+    }
+  }
+
+  test("SPARK-30958: parse date with negative year") {
+    val formatter1 = DateFormatter("yyyy-MM-dd", UTC)
+    assert(formatter1.parse("-1234-02-22") === days(-1234, 2, 22))
+
+    def assertParsingError(f: => Unit): Unit = {
+      intercept[Exception](f) match {
+        case e: SparkUpgradeException =>
+          assert(e.getCause.isInstanceOf[DateTimeException])
+        case e =>
+          assert(e.isInstanceOf[DateTimeException])
+      }
+    }
+
+    // "yyyy" with "G" can't parse negative year or year 0000.
+    val formatter2 = DateFormatter("G yyyy-MM-dd", UTC)
+    assertParsingError(formatter2.parse("BC -1234-02-22"))
+    assertParsingError(formatter2.parse("AD 0000-02-22"))
+
+    assert(formatter2.parse("BC 1234-02-22") === days(-1233, 2, 22))
+    assert(formatter2.parse("AD 1234-02-22") === days(1234, 2, 22))
+  }
+
+  test("SPARK-31557: rebasing in legacy formatters/parsers") {
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> LegacyBehaviorPolicy.LEGACY.toString) {
+      LegacyDateFormats.values.foreach { legacyFormat =>
+        outstandingTimezonesIds.foreach { timeZone =>
+          withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
+            val formatter = DateFormatter(
+              DateFormatter.defaultPattern,
+              getZoneId(timeZone),
+              DateFormatter.defaultLocale,
+              legacyFormat,
+              isParsing = false)
+            assert(LocalDate.ofEpochDay(formatter.parse("1000-01-01")) === LocalDate.of(1000, 1, 1))
+            assert(formatter.format(LocalDate.of(1000, 1, 1)) === "1000-01-01")
+            assert(formatter.format(localDateToDays(LocalDate.of(1000, 1, 1))) === "1000-01-01")
+            assert(formatter.format(java.sql.Date.valueOf("1000-01-01")) === "1000-01-01")
+          }
+        }
+      }
+    }
+  }
+
+  test("missing date fields") {
+    val formatter = DateFormatter("HH", UTC)
+    val daysSinceEpoch = formatter.parse("20")
+    assert(daysSinceEpoch === days(1970, 1, 1))
+  }
+
+  test("missing year field with invalid date") {
+    val formatter = DateFormatter("MM-dd", UTC)
+    // The date parser in 2.4 accepts 1970-02-29 and turn it into 1970-03-01, so we should get a
+    // SparkUpgradeException here.
+    intercept[SparkUpgradeException](formatter.parse("02-29"))
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala
index 817e503584324..0b15e49af0210 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala
@@ -22,25 +22,46 @@ import org.apache.spark.sql.catalyst.util.DateTimeFormatterHelper._
 
 class DateTimeFormatterHelperSuite extends SparkFunSuite {
 
+  private def convertIncompatiblePattern(pattern: String): String = {
+    DateTimeFormatterHelper.convertIncompatiblePattern(pattern, isParsing = false)
+  }
+
   test("check incompatible pattern") {
-    assert(convertIncompatiblePattern("MM-DD-u") === "MM-DD-e")
     assert(convertIncompatiblePattern("yyyy-MM-dd'T'HH:mm:ss.SSSz")
       === "uuuu-MM-dd'T'HH:mm:ss.SSSz")
     assert(convertIncompatiblePattern("yyyy-MM'y contains in quoted text'HH:mm:ss")
       === "uuuu-MM'y contains in quoted text'HH:mm:ss")
-    assert(convertIncompatiblePattern("yyyy-MM-dd-u'T'HH:mm:ss.SSSz")
-      === "uuuu-MM-dd-e'T'HH:mm:ss.SSSz")
     assert(convertIncompatiblePattern("yyyy-MM'u contains in quoted text'HH:mm:ss")
       === "uuuu-MM'u contains in quoted text'HH:mm:ss")
     assert(convertIncompatiblePattern("yyyy-MM'u contains in quoted text'''''HH:mm:ss")
       === "uuuu-MM'u contains in quoted text'''''HH:mm:ss")
     assert(convertIncompatiblePattern("yyyy-MM-dd'T'HH:mm:ss.SSSz G")
       === "yyyy-MM-dd'T'HH:mm:ss.SSSz G")
+    weekBasedLetters.foreach { l =>
+      val e = intercept[IllegalArgumentException](convertIncompatiblePattern(s"yyyy-MM-dd $l G"))
+      assert(e.getMessage.contains("week-based"))
+    }
     unsupportedLetters.foreach { l =>
       val e = intercept[IllegalArgumentException](convertIncompatiblePattern(s"yyyy-MM-dd $l G"))
       assert(e.getMessage === s"Illegal pattern character: $l")
     }
-    assert(convertIncompatiblePattern("yyyy-MM-dd uuuu") === "uuuu-MM-dd eeee")
+    unsupportedLettersForParsing.foreach { l =>
+      val e = intercept[IllegalArgumentException] {
+        DateTimeFormatterHelper.convertIncompatiblePattern(s"$l", isParsing = true)
+      }
+      assert(e.getMessage === s"Illegal pattern character: $l")
+      assert(convertIncompatiblePattern(s"$l").nonEmpty)
+    }
+    unsupportedPatternLengths.foreach { style =>
+      val e1 = intercept[IllegalArgumentException] {
+        convertIncompatiblePattern(s"yyyy-MM-dd $style")
+      }
+      assert(e1.getMessage === s"Too many pattern letters: ${style.head}")
+      val e2 = intercept[IllegalArgumentException] {
+        convertIncompatiblePattern(s"yyyy-MM-dd $style${style.head}")
+      }
+      assert(e2.getMessage === s"Too many pattern letters: ${style.head}")
+    }
     assert(convertIncompatiblePattern("yyyy-MM-dd EEEE") === "uuuu-MM-dd EEEE")
     assert(convertIncompatiblePattern("yyyy-MM-dd'e'HH:mm:ss") === "uuuu-MM-dd'e'HH:mm:ss")
     assert(convertIncompatiblePattern("yyyy-MM-dd'T'") === "uuuu-MM-dd'T'")
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala
index bf9e8f71ba1c9..66aef1b4b6cb0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala
@@ -88,12 +88,8 @@ object DateTimeTestUtils {
   def days(
       year: Int,
       month: Byte = 1,
-      day: Byte = 1,
-      hour: Byte = 0,
-      minute: Byte = 0,
-      sec: Byte = 0): Int = {
-    val micros = date(year, month, day, hour, minute, sec)
-    TimeUnit.MICROSECONDS.toDays(micros).toInt
+      day: Byte = 1): Int = {
+    LocalDate.of(year, month, day).toEpochDay.toInt
   }
 
   // Returns microseconds since epoch for current date and give time
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index 807ec7dafb568..4de4397c4e41d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -23,13 +23,15 @@ import java.time.{Instant, LocalDate, LocalDateTime, LocalTime, ZoneId}
 import java.util.Locale
 import java.util.concurrent.TimeUnit
 
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils._
+import org.apache.spark.sql.catalyst.util.RebaseDateTime.rebaseJulianToGregorianMicros
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 
 class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
@@ -37,12 +39,12 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
   private def defaultZoneId = ZoneId.systemDefault()
 
   test("nanoseconds truncation") {
-    val tf = TimestampFormatter.getFractionFormatter(DateTimeUtils.defaultTimeZone.toZoneId)
+    val tf = TimestampFormatter.getFractionFormatter(ZoneId.systemDefault())
     def checkStringToTimestamp(originalTime: String, expectedParsedTime: String): Unit = {
       val parsedTimestampOp = DateTimeUtils.stringToTimestamp(
         UTF8String.fromString(originalTime), defaultZoneId)
       assert(parsedTimestampOp.isDefined, "timestamp with nanoseconds was not parsed correctly")
-      assert(DateTimeUtils.timestampToString(tf, parsedTimestampOp.get) === expectedParsedTime)
+      assert(tf.format(parsedTimestampOp.get) === expectedParsedTime)
     }
 
     checkStringToTimestamp("2015-01-02 00:00:00.123456789", "2015-01-02 00:00:00.123456")
@@ -69,17 +71,17 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
   }
 
   test("us and julian day") {
-    val (d, ns) = toJulianDay(0)
+    val (d, ns) = toJulianDay(RebaseDateTime.rebaseGregorianToJulianMicros(0))
     assert(d === JULIAN_DAY_OF_EPOCH)
     assert(ns === 0)
-    assert(fromJulianDay(d, ns) == 0L)
+    assert(rebaseJulianToGregorianMicros(fromJulianDay(d, ns)) == 0L)
 
     Seq(Timestamp.valueOf("2015-06-11 10:10:10.100"),
       Timestamp.valueOf("2015-06-11 20:10:10.100"),
       Timestamp.valueOf("1900-06-11 20:10:10.100")).foreach { t =>
-      val (d, ns) = toJulianDay(fromJavaTimestamp(t))
+      val (d, ns) = toJulianDay(RebaseDateTime.rebaseGregorianToJulianMicros(fromJavaTimestamp(t)))
       assert(ns > 0)
-      val t1 = toJavaTimestamp(fromJulianDay(d, ns))
+      val t1 = toJavaTimestamp(rebaseJulianToGregorianMicros(fromJulianDay(d, ns)))
       assert(t.equals(t1))
     }
   }
@@ -121,7 +123,7 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
     checkFromToJavaDate(new Date(df2.parse("1776-07-04 18:30:00 UTC").getTime))
   }
 
-  private def toDate(s: String, zoneId: ZoneId = UTC): Option[SQLDate] = {
+  private def toDate(s: String, zoneId: ZoneId = UTC): Option[Int] = {
     stringToDate(UTF8String.fromString(s), zoneId)
   }
 
@@ -149,7 +151,7 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
     assert(toDate("1999 08").isEmpty)
   }
 
-  private def toTimestamp(str: String, zoneId: ZoneId): Option[SQLTimestamp] = {
+  private def toTimestamp(str: String, zoneId: ZoneId): Option[Long] = {
     stringToTimestamp(UTF8String.fromString(str), zoneId)
   }
 
@@ -386,20 +388,36 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
   }
 
   test("date add months") {
-    val input = days(1997, 2, 28, 10, 30)
+    val input = days(1997, 2, 28)
     assert(dateAddMonths(input, 36) === days(2000, 2, 28))
     assert(dateAddMonths(input, -13) === days(1996, 1, 28))
   }
 
+  test("SPARK-34739: timestamp add months") {
+    outstandingZoneIds.foreach { zid =>
+      Seq(
+        (date(2021, 3, 13, 21, 28, 13, 123456, zid), 0, date(2021, 3, 13, 21, 28, 13, 123456, zid)),
+        (date(2021, 3, 31, 0, 0, 0, 123, zid), -1, date(2021, 2, 28, 0, 0, 0, 123, zid)),
+        (date(2020, 2, 29, 1, 2, 3, 4, zid), 12, date(2021, 2, 28, 1, 2, 3, 4, zid)),
+        (date(1, 1, 1, 0, 0, 0, 1, zid), 2020 * 12, date(2021, 1, 1, 0, 0, 0, 1, zid)),
+        (date(1581, 10, 7, 23, 59, 59, 999, zid), 12, date(1582, 10, 7, 23, 59, 59, 999, zid)),
+        (date(9999, 12, 31, 23, 59, 59, 999999, zid), -11,
+          date(9999, 1, 31, 23, 59, 59, 999999, zid))
+      ).foreach { case (timestamp, months, expected) =>
+        assert(timestampAddMonths(timestamp, months, zid) === expected)
+      }
+    }
+  }
+
   test("date add interval with day precision") {
-    val input = days(1997, 2, 28, 10, 30)
+    val input = days(1997, 2, 28)
     assert(dateAddInterval(input, new CalendarInterval(36, 0, 0)) === days(2000, 2, 28))
     assert(dateAddInterval(input, new CalendarInterval(36, 47, 0)) === days(2000, 4, 15))
     assert(dateAddInterval(input, new CalendarInterval(-13, 0, 0)) === days(1996, 1, 28))
     intercept[IllegalArgumentException](dateAddInterval(input, new CalendarInterval(36, 47, 1)))
   }
 
-  test("timestamp add months") {
+  test("timestamp add interval") {
     val ts1 = date(1997, 2, 28, 10, 30, 0)
     val ts2 = date(2000, 2, 28, 10, 30, 0, 123000)
     assert(timestampAddInterval(ts1, 36, 0, 123000, defaultZoneId) === ts2)
@@ -469,6 +487,13 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
         test("2011-12-25 09:00:00.123456", JST.getId, "2011-12-25 18:00:00.123456")
         test("2011-12-25 09:00:00.123456", LA.getId, "2011-12-25 01:00:00.123456")
         test("2011-12-25 09:00:00.123456", "Asia/Shanghai", "2011-12-25 17:00:00.123456")
+        test("2011-12-25 09:00:00.123456", "-7", "2011-12-25 02:00:00.123456")
+        test("2011-12-25 09:00:00.123456", "+8:00", "2011-12-25 17:00:00.123456")
+        test("2011-12-25 09:00:00.123456", "+8:00:00", "2011-12-25 17:00:00.123456")
+        test("2011-12-25 09:00:00.123456", "+0800", "2011-12-25 17:00:00.123456")
+        test("2011-12-25 09:00:00.123456", "-071020", "2011-12-25 01:49:40.123456")
+        test("2011-12-25 09:00:00.123456", "-07:10:20", "2011-12-25 01:49:40.123456")
+
       }
     }
 
@@ -494,6 +519,12 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
         test("2011-12-25 18:00:00.123456", JST.getId, "2011-12-25 09:00:00.123456")
         test("2011-12-25 01:00:00.123456", LA.getId, "2011-12-25 09:00:00.123456")
         test("2011-12-25 17:00:00.123456", "Asia/Shanghai", "2011-12-25 09:00:00.123456")
+        test("2011-12-25 02:00:00.123456", "-7", "2011-12-25 09:00:00.123456")
+        test("2011-12-25 17:00:00.123456", "+8:00", "2011-12-25 09:00:00.123456")
+        test("2011-12-25 17:00:00.123456", "+8:00:00", "2011-12-25 09:00:00.123456")
+        test("2011-12-25 17:00:00.123456", "+0800", "2011-12-25 09:00:00.123456")
+        test("2011-12-25 01:49:40.123456", "-071020", "2011-12-25 09:00:00.123456")
+        test("2011-12-25 01:49:40.123456", "-07:10:20", "2011-12-25 09:00:00.123456")
       }
     }
 
@@ -516,18 +547,32 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
     assert(time == None)
   }
 
-  test("truncTimestamp") {
-    def testTrunc(
-        level: Int,
-        expected: String,
-        inputTS: SQLTimestamp,
-        zoneId: ZoneId = defaultZoneId): Unit = {
-      val truncated =
-        DateTimeUtils.truncTimestamp(inputTS, level, zoneId)
-      val expectedTS = toTimestamp(expected, defaultZoneId)
-      assert(truncated === expectedTS.get)
+  def testTrunc(
+      level: Int,
+      expected: String,
+      inputTS: Long,
+      zoneId: ZoneId = defaultZoneId): Unit = {
+    val truncated = DateTimeUtils.truncTimestamp(inputTS, level, zoneId)
+    val expectedTS = toTimestamp(expected, defaultZoneId)
+    assert(truncated === expectedTS.get)
+  }
+
+  test("SPARK-33404: test truncTimestamp when time zone offset from UTC has a " +
+    "granularity of seconds") {
+    for (zid <- ALL_TIMEZONES) {
+      withDefaultTimeZone(zid) {
+        val inputTS = DateTimeUtils.stringToTimestamp(
+          UTF8String.fromString("1769-10-17T17:10:02.123456"), defaultZoneId)
+        testTrunc(DateTimeUtils.TRUNC_TO_MINUTE, "1769-10-17T17:10:00", inputTS.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_SECOND, "1769-10-17T17:10:02", inputTS.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_MILLISECOND, "1769-10-17T17:10:02.123", inputTS.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_MICROSECOND, "1769-10-17T17:10:02.123456",
+          inputTS.get, zid)
+      }
     }
+  }
 
+  test("truncTimestamp") {
     val defaultInputTS = DateTimeUtils.stringToTimestamp(
       UTF8String.fromString("2015-03-05T09:32:05.359123"), defaultZoneId)
     val defaultInputTS1 = DateTimeUtils.stringToTimestamp(
@@ -659,4 +704,87 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       assert(toDate("tomorrow CET ", zoneId).get === today + 1)
     }
   }
+
+  test("parsing day of week") {
+    assert(getDayOfWeekFromString(UTF8String.fromString("THU")) == 0)
+    assert(getDayOfWeekFromString(UTF8String.fromString("MONDAY")) == 4)
+    intercept[IllegalArgumentException](getDayOfWeekFromString(UTF8String.fromString("xx")))
+    intercept[IllegalArgumentException](getDayOfWeekFromString(UTF8String.fromString("\"quote")))
+  }
+
+  test("SPARK-34761: timestamp add day-time interval") {
+    // transit from Pacific Standard Time to Pacific Daylight Time
+    assert(timestampAddDayTime(
+      // 2019-3-9 is the end of Pacific Standard Time
+      date(2019, 3, 9, 12, 0, 0, 123000, LA),
+      MICROS_PER_DAY, LA) ===
+      // 2019-3-10 is the start of Pacific Daylight Time
+      date(2019, 3, 10, 12, 0, 0, 123000, LA))
+    // just normal days
+    outstandingZoneIds.foreach { zid =>
+      assert(timestampAddDayTime(
+        date(2021, 3, 18, 19, 44, 1, 100000, zid), 0, zid) ===
+        date(2021, 3, 18, 19, 44, 1, 100000, zid))
+      assert(timestampAddDayTime(
+        date(2021, 1, 19, 0, 0, 0, 0, zid), -18 * MICROS_PER_DAY, zid) ===
+        date(2021, 1, 1, 0, 0, 0, 0, zid))
+      assert(timestampAddDayTime(
+        date(2021, 3, 18, 19, 44, 1, 999999, zid), 10 * MICROS_PER_MINUTE, zid) ===
+        date(2021, 3, 18, 19, 54, 1, 999999, zid))
+      assert(timestampAddDayTime(
+        date(2021, 3, 18, 19, 44, 1, 1, zid), -MICROS_PER_DAY - 1, zid) ===
+        date(2021, 3, 17, 19, 44, 1, 0, zid))
+      assert(timestampAddDayTime(
+        date(2019, 5, 9, 12, 0, 0, 123456, zid), 2 * MICROS_PER_DAY + 1, zid) ===
+        date(2019, 5, 11, 12, 0, 0, 123457, zid))
+    }
+    // transit from Pacific Daylight Time to Pacific Standard Time
+    assert(timestampAddDayTime(
+      // 2019-11-2 is the end of Pacific Daylight Time
+      date(2019, 11, 2, 12, 0, 0, 123000, LA),
+      MICROS_PER_DAY, LA) ===
+      // 2019-11-3 is the start of Pacific Standard Time
+      date(2019, 11, 3, 12, 0, 0, 123000, LA))
+  }
+
+  test("SPARK-34903: subtract timestamps") {
+    DateTimeTestUtils.outstandingZoneIds.foreach { zid =>
+      Seq(
+        // 1000-02-29 exists in Julian calendar because 1000 is a leap year
+        (LocalDateTime.of(1000, 2, 28, 1, 2, 3, 456789000),
+          LocalDateTime.of(1000, 3, 1, 1, 2, 3, 456789000)) -> TimeUnit.DAYS.toMicros(1),
+        // The range 1582-10-04 .. 1582-10-15 doesn't exist in Julian calendar
+        (LocalDateTime.of(1582, 10, 4, 23, 59, 59, 999999000),
+          LocalDateTime.of(1582, 10, 15, 23, 59, 59, 999999000)) -> TimeUnit.DAYS.toMicros(11),
+        // America/Los_Angeles -08:00 zone offset
+        (LocalDateTime.of(1883, 11, 20, 0, 0, 0, 123456000),
+          // America/Los_Angeles -08:00 zone offset
+          LocalDateTime.of(1883, 11, 10, 0, 0, 0)) -> (TimeUnit.DAYS.toMicros(-10) - 123456),
+        // No difference between Proleptic Gregorian and Julian calendars after 1900-01-01
+        (LocalDateTime.of(1900, 1, 1, 0, 0, 0, 1000),
+          LocalDateTime.of(1899, 12, 31, 23, 59, 59, 999999000)) -> -2,
+        // The 'Asia/Hong_Kong' time zone switched from 'Japan Standard Time' (JST = UTC+9)
+        // to 'Hong Kong Time' (HKT = UTC+8). After Sunday, 18 November, 1945 01:59:59 AM,
+        // clocks were moved backward to become Sunday, 18 November, 1945 01:00:00 AM.
+        // In this way, the overlap happened w/o Daylight Saving Time.
+        (LocalDateTime.of(1945, 11, 18, 0, 30, 30),
+          LocalDateTime.of(1945, 11, 18, 1, 30, 30)) -> TimeUnit.HOURS.toMicros(1),
+        (LocalDateTime.of(1945, 11, 18, 2, 0, 0),
+          LocalDateTime.of(1945, 11, 18, 1, 0, 0)) -> TimeUnit.HOURS.toMicros(-1),
+        // The epoch has zero offset in microseconds
+        (LocalDateTime.of(1970, 1, 1, 0, 0, 0), LocalDateTime.of(1970, 1, 1, 0, 0, 0)) -> 0,
+        // 2020 is a leap year
+        (LocalDateTime.of(2020, 2, 29, 0, 0, 0),
+          LocalDateTime.of(2021, 3, 1, 0, 0, 0)) -> TimeUnit.DAYS.toMicros(366),
+        // Daylight saving in America/Los_Angeles: from winter to summer time
+        (LocalDateTime.of(2021, 3, 14, 1, 0, 0), LocalDateTime.of(2021, 3, 14, 3, 0, 0)) ->
+          TimeUnit.HOURS.toMicros(2)
+      ).foreach { case ((start, end), expected) =>
+        val startMicros = localDateTimeToMicros(start, zid)
+        val endMicros = localDateTimeToMicros(end, zid)
+        val result = subtractTimestamps(endMicros, startMicros, zid)
+        assert(result === expected)
+      }
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DatetimeFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DatetimeFormatterSuite.scala
new file mode 100644
index 0000000000000..54a0aec4cf612
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DatetimeFormatterSuite.scala
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import java.time.DateTimeException
+
+import org.scalatest.matchers.must.Matchers
+
+import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
+import org.apache.spark.sql.catalyst.plans.SQLHelper
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{date, UTC}
+
+trait DatetimeFormatterSuite extends SparkFunSuite with SQLHelper with Matchers {
+  import DateTimeFormatterHelper._
+  import LegacyDateFormats._
+  def checkFormatterCreation(pattern: String, isParsing: Boolean): Unit
+
+  private def dateFormatter(
+      pattern: String, ldf: LegacyDateFormat = FAST_DATE_FORMAT): DateFormatter = {
+    DateFormatter(pattern, UTC, DateFormatter.defaultLocale, ldf, isParsing = true)
+  }
+
+  private def timestampFormatter(
+      pattern: String, ldf: LegacyDateFormat = SIMPLE_DATE_FORMAT): TimestampFormatter = {
+    TimestampFormatter(pattern, UTC, legacyFormat = ldf, isParsing = true)
+  }
+
+  protected def useDateFormatter: Boolean
+
+  private def assertEqual(pattern: String, datetimeStr: String, expected: Long): Unit = {
+    if (useDateFormatter) {
+      assert(dateFormatter(pattern).parse(datetimeStr) ===
+        DateTimeUtils.microsToDays(expected, UTC))
+    } else {
+      assert(timestampFormatter(pattern).parse(datetimeStr) === expected)
+    }
+  }
+
+  private def assertError(pattern: String, datetimeStr: String, expectedMsg: String): Unit = {
+    if (useDateFormatter) {
+      LegacyDateFormats.values.foreach { ldf =>
+        // The legacy DateFormatter is always lenient by default
+        val e = intercept[SparkUpgradeException](dateFormatter(pattern, ldf).parse(datetimeStr))
+        assert(e.getCause.getMessage.contains(expectedMsg))
+      }
+    } else {
+      // In strict mode, the legacy TimestampFormatter fails too
+      val e = intercept[DateTimeException](timestampFormatter(pattern).parse(datetimeStr))
+      assert(e.getMessage.contains(expectedMsg))
+      // In lenient mode, the legacy TimestampFormatter does not fail
+      Seq(FAST_DATE_FORMAT, LENIENT_SIMPLE_DATE_FORMAT).foreach { ldf =>
+        val e = intercept[SparkUpgradeException] {
+          timestampFormatter(pattern, ldf).parse(datetimeStr)
+        }
+        assert(e.getCause.getMessage.contains(expectedMsg))
+      }
+    }
+  }
+
+  test("explicitly forbidden datetime patterns") {
+
+    Seq(true, false).foreach { isParsing =>
+      // not support by the legacy one too
+      val unsupportedBoth = Seq("QQQQQ", "qqqqq", "eeeee", "A", "c", "n", "N", "p", "e")
+      unsupportedBoth.foreach { pattern =>
+        intercept[IllegalArgumentException](checkFormatterCreation(pattern, isParsing))
+      }
+      // supported by the legacy one, then we will suggest users with SparkUpgradeException
+      ((weekBasedLetters ++ unsupportedLetters).map(_.toString)
+        ++ unsupportedPatternLengths -- unsupportedBoth).foreach {
+        pattern => intercept[SparkUpgradeException](checkFormatterCreation(pattern, isParsing))
+      }
+    }
+
+    // not support by the legacy one too
+    val unsupportedBoth = Seq("q", "Q")
+    unsupportedBoth.foreach { pattern =>
+      intercept[IllegalArgumentException](checkFormatterCreation(pattern, true))
+    }
+    // supported by the legacy one, then we will suggest users with SparkUpgradeException
+    (unsupportedLettersForParsing.map(_.toString) -- unsupportedBoth).foreach {
+      pattern => intercept[SparkUpgradeException](checkFormatterCreation(pattern, true))
+    }
+  }
+
+  test("SPARK-31939: Fix Parsing day of year when year field pattern is missing") {
+    // resolved to queryable LocaleDate or fail directly
+    assertEqual("yyyy-dd-DD", "2020-29-60", date(2020, 2, 29))
+    assertError("yyyy-dd-DD", "2020-02-60",
+      "Field DayOfMonth 29 differs from DayOfMonth 2 derived from 2020-02-29")
+    assertEqual("yyyy-MM-DD", "2020-02-60", date(2020, 2, 29))
+    assertError("yyyy-MM-DD", "2020-03-60",
+      "Field MonthOfYear 2 differs from MonthOfYear 3 derived from 2020-02-29")
+    assertEqual("yyyy-MM-dd-DD", "2020-02-29-60", date(2020, 2, 29))
+    assertError("yyyy-MM-dd-DD", "2020-03-01-60",
+      "Field DayOfYear 61 differs from DayOfYear 60 derived from 2020-03-01")
+    assertEqual("yyyy-DDD", "2020-366", date(2020, 12, 31))
+    assertError("yyyy-DDD", "2019-366",
+      "Invalid date 'DayOfYear 366' as '2019' is not a leap year")
+
+    // unresolved and need to check manually(SPARK-31939 fixed)
+    assertEqual("DDD", "365", date(1970, 12, 31))
+    assertError("DDD", "366",
+      "Invalid date 'DayOfYear 366' as '1970' is not a leap year")
+    assertEqual("MM-DD", "03-60", date(1970, 3))
+    assertError("MM-DD", "02-60",
+      "Field MonthOfYear 2 differs from MonthOfYear 3 derived from 1970-03-01")
+    assertEqual("MM-dd-DD", "02-28-59", date(1970, 2, 28))
+    assertError("MM-dd-DD", "02-28-60",
+      "Field MonthOfYear 2 differs from MonthOfYear 3 derived from 1970-03-01")
+    assertError("MM-dd-DD", "02-28-58",
+      "Field DayOfMonth 28 differs from DayOfMonth 27 derived from 1970-02-27")
+    assertEqual("dd-DD", "28-59", date(1970, 2, 28))
+    assertError("dd-DD", "27-59",
+      "Field DayOfMonth 27 differs from DayOfMonth 28 derived from 1970-02-28")
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
index ba738eeebbcdb..87d306a495724 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
@@ -17,12 +17,14 @@
 
 package org.apache.spark.sql.catalyst.util
 
+import java.time.{Duration, Period}
 import java.util.concurrent.TimeUnit
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToMicros
+import org.apache.spark.sql.catalyst.util.IntervalStringStyles.{ANSI_STYLE, HIVE_STYLE}
 import org.apache.spark.sql.catalyst.util.IntervalUtils._
 import org.apache.spark.sql.catalyst.util.IntervalUtils.IntervalUnit._
 import org.apache.spark.sql.internal.SQLConf
@@ -77,6 +79,19 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
     }
   }
 
+  test("string to interval: interval with dangling parts should not results null") {
+    checkFromInvalidString("+", "expect a number after '+' but hit EOL")
+    checkFromInvalidString("-", "expect a number after '-' but hit EOL")
+    checkFromInvalidString("+ 2", "expect a unit name after '2' but hit EOL")
+    checkFromInvalidString("- 1", "expect a unit name after '1' but hit EOL")
+    checkFromInvalidString("1", "expect a unit name after '1' but hit EOL")
+    checkFromInvalidString("1.2", "expect a unit name after '1.2' but hit EOL")
+    checkFromInvalidString("1 day 2", "expect a unit name after '2' but hit EOL")
+    checkFromInvalidString("1 day 2.2", "expect a unit name after '2.2' but hit EOL")
+    checkFromInvalidString("1 day -", "expect a number after '-' but hit EOL")
+    checkFromInvalidString("-.", "expect a unit name after '-.' but hit EOL")
+  }
+
   test("string to interval: multiple units") {
     Seq(
       "-1 MONTH 1 day -1 microseconds" -> new CalendarInterval(-1, 1, -1),
@@ -115,6 +130,14 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
     checkFromString("1 \t day \n 2 \r hour", new CalendarInterval(0, 1, 2 * MICROS_PER_HOUR))
     checkFromInvalidString("interval1 \t day \n 2 \r hour", "invalid interval prefix interval1")
     checkFromString("interval\r1\tday", new CalendarInterval(0, 1, 0))
+    // scalastyle:off nonascii
+    checkFromInvalidString("中国 interval 1 day", "unrecognized number '中国'")
+    checkFromInvalidString("interval浙江 1 day", "invalid interval prefix interval浙江")
+    checkFromInvalidString("interval 1杭州 day", "invalid value '1杭州'")
+    checkFromInvalidString("interval 1 滨江day", "invalid unit '滨江day'")
+    checkFromInvalidString("interval 1 day长河", "invalid unit 'day长河'")
+    checkFromInvalidString("interval 1 day 网商路", "unrecognized number '网商路'")
+    // scalastyle:on nonascii
   }
 
   test("string to interval: seconds with fractional part") {
@@ -146,6 +169,19 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
       fromYearMonthString)
     failFuncWithInvalidInput("-\t99-15", "Interval string does not match year-month format",
       fromYearMonthString)
+
+    assert(fromYearMonthString("178956970-6") == new CalendarInterval(Int.MaxValue - 1, 0, 0))
+    assert(fromYearMonthString("178956970-7") == new CalendarInterval(Int.MaxValue, 0, 0))
+
+    val e1 = intercept[IllegalArgumentException]{
+      assert(fromYearMonthString("178956970-8") == new CalendarInterval(Int.MinValue, 0, 0))
+    }.getMessage
+    assert(e1.contains("integer overflow"))
+    assert(fromYearMonthString("-178956970-8") == new CalendarInterval(Int.MinValue, 0, 0))
+    val e2 = intercept[IllegalArgumentException]{
+      assert(fromYearMonthString("-178956970-9") == new CalendarInterval(Int.MinValue, 0, 0))
+    }.getMessage
+    assert(e2.contains("integer overflow"))
   }
 
   test("from day-time string - legacy") {
@@ -358,4 +394,142 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
     intercept[ArithmeticException](multiplyExact(maxMonth, 2))
     intercept[ArithmeticException](divideExact(maxDay, 0.5))
   }
+
+  test("SPARK-34605: microseconds to duration") {
+    assert(microsToDuration(0).isZero)
+    assert(microsToDuration(-1).toNanos === -1000)
+    assert(microsToDuration(1).toNanos === 1000)
+    assert(microsToDuration(Long.MaxValue).toDays === 106751991)
+    assert(microsToDuration(Long.MinValue).toDays === -106751991)
+  }
+
+  test("SPARK-34605: duration to microseconds") {
+    assert(durationToMicros(Duration.ZERO) === 0)
+    assert(durationToMicros(Duration.ofSeconds(-1)) === -1000000)
+    assert(durationToMicros(Duration.ofNanos(123456)) === 123)
+    assert(durationToMicros(Duration.ofDays(106751991)) ===
+      (Long.MaxValue / MICROS_PER_DAY) * MICROS_PER_DAY)
+
+    val errMsg = intercept[ArithmeticException] {
+      durationToMicros(Duration.ofDays(106751991 + 1))
+    }.getMessage
+    assert(errMsg.contains("long overflow"))
+  }
+
+  test("SPARK-34615: period to months") {
+    assert(periodToMonths(Period.ZERO) === 0)
+    assert(periodToMonths(Period.of(0, -1, 0)) === -1)
+    assert(periodToMonths(Period.of(-1, 0, 10)) === -12) // ignore days
+    assert(periodToMonths(Period.of(178956970, 7, 0)) === Int.MaxValue)
+    assert(periodToMonths(Period.of(-178956970, -8, 123)) === Int.MinValue)
+    assert(periodToMonths(Period.of(0, Int.MaxValue, Int.MaxValue)) === Int.MaxValue)
+
+    val errMsg = intercept[ArithmeticException] {
+      periodToMonths(Period.of(Int.MaxValue, 0, 0))
+    }.getMessage
+    assert(errMsg.contains("integer overflow"))
+  }
+
+  test("SPARK-34615: months to period") {
+    assert(monthsToPeriod(0) === Period.ZERO)
+    assert(monthsToPeriod(-11) === Period.of(0, -11, 0))
+    assert(monthsToPeriod(11) === Period.of(0, 11, 0))
+    assert(monthsToPeriod(27) === Period.of(2, 3, 0))
+    assert(monthsToPeriod(-13) === Period.of(-1, -1, 0))
+    assert(monthsToPeriod(Int.MaxValue) === Period.ofYears(178956970).withMonths(7))
+    assert(monthsToPeriod(Int.MinValue) === Period.ofYears(-178956970).withMonths(-8))
+  }
+
+  test("SPARK-34695: round trip conversion of micros -> duration -> micros") {
+    Seq(
+      0,
+      MICROS_PER_SECOND - 1,
+      -MICROS_PER_SECOND + 1,
+      MICROS_PER_SECOND,
+      -MICROS_PER_SECOND,
+      Long.MaxValue - MICROS_PER_SECOND,
+      Long.MinValue + MICROS_PER_SECOND,
+      Long.MaxValue,
+      Long.MinValue).foreach { micros =>
+      val duration = microsToDuration(micros)
+      assert(durationToMicros(duration) === micros)
+    }
+  }
+
+  test("SPARK-34715: Add round trip tests for period <-> month and duration <-> micros") {
+    // Months -> Period -> Months
+    Seq(
+      0,
+      MONTHS_PER_YEAR - 1,
+      MONTHS_PER_YEAR + 1,
+      MONTHS_PER_YEAR,
+      -MONTHS_PER_YEAR,
+      Int.MaxValue - MONTHS_PER_YEAR,
+      Int.MinValue + MONTHS_PER_YEAR,
+      Int.MaxValue,
+      Int.MinValue).foreach { months =>
+      val period = monthsToPeriod(months)
+      assert(periodToMonths(period) === months)
+    }
+    // Period -> Months -> Period
+    Seq(
+      monthsToPeriod(0),
+      monthsToPeriod(MONTHS_PER_YEAR - 1),
+      monthsToPeriod(MONTHS_PER_YEAR + 1),
+      monthsToPeriod(MONTHS_PER_YEAR),
+      monthsToPeriod(-MONTHS_PER_YEAR),
+      monthsToPeriod(Int.MaxValue - MONTHS_PER_YEAR),
+      monthsToPeriod(Int.MinValue + MONTHS_PER_YEAR),
+      monthsToPeriod(Int.MaxValue),
+      monthsToPeriod(Int.MinValue)).foreach { period =>
+      val months = periodToMonths(period)
+      assert(monthsToPeriod(months) === period)
+    }
+    // Duration -> micros -> Duration
+    Seq(
+      microsToDuration(0),
+      microsToDuration(MICROS_PER_SECOND - 1),
+      microsToDuration(-MICROS_PER_SECOND + 1),
+      microsToDuration(MICROS_PER_SECOND),
+      microsToDuration(-MICROS_PER_SECOND),
+      microsToDuration(Long.MaxValue - MICROS_PER_SECOND),
+      microsToDuration(Long.MinValue + MICROS_PER_SECOND),
+      microsToDuration(Long.MaxValue),
+      microsToDuration(Long.MinValue)).foreach { duration =>
+      val micros = durationToMicros(duration)
+      assert(microsToDuration(micros) === duration)
+    }
+  }
+
+  test("SPARK-35016: format year-month intervals") {
+    Seq(
+      0 -> ("0-0", "INTERVAL '0-0' YEAR TO MONTH"),
+      -11 -> ("-0-11", "INTERVAL '-0-11' YEAR TO MONTH"),
+      11 -> ("0-11", "INTERVAL '0-11' YEAR TO MONTH"),
+      -13 -> ("-1-1", "INTERVAL '-1-1' YEAR TO MONTH"),
+      13 -> ("1-1", "INTERVAL '1-1' YEAR TO MONTH"),
+      -24 -> ("-2-0", "INTERVAL '-2-0' YEAR TO MONTH"),
+      24 -> ("2-0", "INTERVAL '2-0' YEAR TO MONTH"),
+      Int.MinValue -> ("-178956970-8", "INTERVAL '-178956970-8' YEAR TO MONTH"),
+      Int.MaxValue -> ("178956970-7", "INTERVAL '178956970-7' YEAR TO MONTH")
+    ).foreach { case (months, (hiveIntervalStr, ansiIntervalStr)) =>
+      assert(toYearMonthIntervalString(months, ANSI_STYLE) === ansiIntervalStr)
+      assert(toYearMonthIntervalString(months, HIVE_STYLE) === hiveIntervalStr)
+    }
+  }
+
+  test("SPARK-35016: format day-time intervals") {
+    Seq(
+      0L -> ("0 00:00:00.000000000", "INTERVAL '0 00:00:00' DAY TO SECOND"),
+      -1L -> ("-0 00:00:00.000001000", "INTERVAL '-0 00:00:00.000001' DAY TO SECOND"),
+      10 * MICROS_PER_MILLIS -> ("0 00:00:00.010000000", "INTERVAL '0 00:00:00.01' DAY TO SECOND"),
+      (-123 * MICROS_PER_DAY - 3 * MICROS_PER_SECOND) ->
+        ("-123 00:00:03.000000000", "INTERVAL '-123 00:00:03' DAY TO SECOND"),
+      Long.MinValue -> ("-106751991 04:00:54.775808000",
+        "INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND")
+    ).foreach { case (micros, (hiveIntervalStr, ansiIntervalStr)) =>
+      assert(toDayTimeIntervalString(micros, ANSI_STYLE) === ansiIntervalStr)
+      assert(toDayTimeIntervalString(micros, HIVE_STYLE) === hiveIntervalStr)
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/NumberConverterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/NumberConverterSuite.scala
index ec73f4518737d..eb257b7975622 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/NumberConverterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/NumberConverterSuite.scala
@@ -40,6 +40,10 @@ class NumberConverterSuite extends SparkFunSuite {
     checkConv("11abc", 10, 16, "B")
   }
 
+  test("SPARK-34909: convert negative to unsigned") {
+    checkConv("-10", 11, 7, "45012021522523134134555")
+  }
+
   test("byte to binary") {
     checkToBinary(0.toByte)
     checkToBinary(1.toByte)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/PhysicalAggregationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/PhysicalAggregationSuite.scala
new file mode 100644
index 0000000000000..b8c60dfbf4f97
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/PhysicalAggregationSuite.scala
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import org.apache.spark.sql.catalyst.analysis.SimpleAnalyzer
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.planning.PhysicalAggregation
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
+
+class PhysicalAggregationSuite extends PlanTest {
+  val testRelation = LocalRelation('a.int, 'b.int)
+
+  test("SPARK-35014: a foldable expression should not be replaced by an AttributeReference") {
+    val query = testRelation
+      .groupBy('a, Literal.create(1) as 'k)(
+        'a, Round(Literal.create(1.2), Literal.create(1)) as 'r, count('b) as 'c)
+    val analyzedQuery = SimpleAnalyzer.execute(query)
+
+    val PhysicalAggregation(
+      groupingExpressions,
+      aggregateExpressions,
+      resultExpressions,
+      _ /* child */
+    ) = analyzedQuery
+
+    assertResult(2)(groupingExpressions.length)
+    assertResult(1)(aggregateExpressions.length)
+    assertResult(3)(resultExpressions.length)
+
+    // Verify that Round's scale parameter is a Literal.
+    resultExpressions(1) match {
+      case Alias(Round(_, _: Literal), _) =>
+      case other => fail("unexpected result expression: " + other)
+    }
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
index cb5f8e43d762f..428a0c028a5f6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
@@ -21,7 +21,7 @@ import java.sql.{Date, Timestamp}
 import java.time.{Instant, LocalDate, LocalDateTime, ZoneId}
 import java.util.TimeZone
 
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.plans.SQLHelper
@@ -29,6 +29,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeConstants._
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils._
 import org.apache.spark.sql.catalyst.util.RebaseDateTime._
+import org.apache.spark.util.ThreadUtils
 
 class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper {
 
@@ -77,7 +78,7 @@ class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper {
     Math.floor(millisLocal.toDouble / MILLIS_PER_DAY).toInt
   }
   private def fromJavaDateLegacy(date: Date): Int = {
-    millisToDaysLegacy(date.getTime, defaultTimeZone())
+    millisToDaysLegacy(date.getTime, TimeZone.getTimeZone(ZoneId.systemDefault()))
   }
 
   test("rebase gregorian to/from julian days") {
@@ -201,21 +202,21 @@ class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper {
   test("optimization of micros rebasing - Gregorian to Julian") {
     outstandingZoneIds.foreach { zid =>
       withClue(s"zone id = $zid") {
-        withDefaultTimeZone(zid) {
-          val start = instantToMicros(LocalDateTime.of(1, 1, 1, 0, 0, 0)
-            .atZone(zid)
-            .toInstant)
-          val end = instantToMicros(LocalDateTime.of(2100, 1, 1, 0, 0, 0)
-            .atZone(zid)
-            .toInstant)
-          var micros = start
-          do {
-            val rebased = rebaseGregorianToJulianMicros(zid, micros)
-            val rebasedAndOptimized = rebaseGregorianToJulianMicros(micros)
-            assert(rebasedAndOptimized === rebased)
-            micros += (MICROS_PER_DAY * 30 * (0.5 + Math.random())).toLong
-          } while (micros <= end)
-        }
+        val start = instantToMicros(LocalDateTime.of(1, 1, 1, 0, 0, 0)
+          .atZone(zid)
+          .toInstant)
+        val end = instantToMicros(LocalDateTime.of(2100, 1, 1, 0, 0, 0)
+          .atZone(zid)
+          .toInstant)
+        var micros = start
+        do {
+          val rebased = rebaseGregorianToJulianMicros(TimeZone.getTimeZone(zid), micros)
+          val rebasedAndOptimized = withDefaultTimeZone(zid) {
+            rebaseGregorianToJulianMicros(micros)
+          }
+          assert(rebasedAndOptimized === rebased)
+          micros += (MICROS_PER_DAY * 30 * (0.5 + Math.random())).toLong
+        } while (micros <= end)
       }
     }
   }
@@ -223,26 +224,26 @@ class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper {
   test("optimization of micros rebasing - Julian to Gregorian") {
     outstandingZoneIds.foreach { zid =>
       withClue(s"zone id = $zid") {
-        withDefaultTimeZone(zid) {
-          val start = rebaseGregorianToJulianMicros(
-            instantToMicros(LocalDateTime.of(1, 1, 1, 0, 0, 0).atZone(zid).toInstant))
-          val end = rebaseGregorianToJulianMicros(
-            instantToMicros(LocalDateTime.of(2100, 1, 1, 0, 0, 0).atZone(zid).toInstant))
-          var micros = start
-          do {
-            val rebased = rebaseJulianToGregorianMicros(zid, micros)
-            val rebasedAndOptimized = rebaseJulianToGregorianMicros(micros)
-            assert(rebasedAndOptimized === rebased)
-            micros += (MICROS_PER_DAY * 30 * (0.5 + Math.random())).toLong
-          } while (micros <= end)
-        }
+        val start = rebaseGregorianToJulianMicros(
+          instantToMicros(LocalDateTime.of(1, 1, 1, 0, 0, 0).atZone(zid).toInstant))
+        val end = rebaseGregorianToJulianMicros(
+          instantToMicros(LocalDateTime.of(2100, 1, 1, 0, 0, 0).atZone(zid).toInstant))
+        var micros = start
+        do {
+          val rebased = rebaseJulianToGregorianMicros(TimeZone.getTimeZone(zid), micros)
+          val rebasedAndOptimized = withDefaultTimeZone(zid) {
+            rebaseJulianToGregorianMicros(micros)
+          }
+          assert(rebasedAndOptimized === rebased)
+          micros += (MICROS_PER_DAY * 30 * (0.5 + Math.random())).toLong
+        } while (micros <= end)
       }
     }
   }
 
   private def generateRebaseJson(
-      adjustFunc: Long => Long,
-      rebaseFunc: (ZoneId, Long) => Long,
+      adjustFunc: (TimeZone, Long) => Long,
+      rebaseFunc: (TimeZone, Long) => Long,
       dir: String,
       fileName: String): Unit = {
     import java.nio.file.{Files, Paths}
@@ -254,29 +255,26 @@ class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper {
     import com.fasterxml.jackson.module.scala.{DefaultScalaModule, ScalaObjectMapper}
 
     case class RebaseRecord(tz: String, switches: Array[Long], diffs: Array[Long])
-
-    val result = new ArrayBuffer[RebaseRecord]()
-    ALL_TIMEZONES
-      .sortBy(_.getId)
-      .foreach { zid =>
+    val rebaseRecords = ThreadUtils.parmap(ALL_TIMEZONES, "JSON-rebase-gen", 16) { zid =>
       withDefaultTimeZone(zid) {
-        val start = adjustFunc(instantToMicros(LocalDateTime.of(1, 1, 1, 0, 0, 0)
-          .atZone(zid)
-          .toInstant))
+        val tz = TimeZone.getTimeZone(zid)
+        val start = adjustFunc(
+          tz,
+          instantToMicros(LocalDateTime.of(1, 1, 1, 0, 0, 0).atZone(zid).toInstant))
         // sun.util.calendar.ZoneInfo resolves DST after 2037 year incorrectly.
         // See https://bugs.java.com/bugdatabase/view_bug.do?bug_id=8073446
-        val end = adjustFunc(instantToMicros(LocalDateTime.of(2037, 1, 1, 0, 0, 0)
-          .atZone(zid)
-          .toInstant))
+        val end = adjustFunc(
+          tz,
+          instantToMicros(LocalDateTime.of(2037, 1, 1, 0, 0, 0).atZone(zid).toInstant))
 
         var micros = start
         var diff = Long.MaxValue
-        val maxStep = DAYS_PER_WEEK * MICROS_PER_DAY
+        val maxStep = 30 * MICROS_PER_MINUTE
         var step: Long = MICROS_PER_SECOND
         val switches = new ArrayBuffer[Long]()
         val diffs = new ArrayBuffer[Long]()
         while (micros < end) {
-          val rebased = rebaseFunc(zid, micros)
+          val rebased = rebaseFunc(tz, micros)
           val curDiff = rebased - micros
           if (curDiff != diff) {
             if (step > MICROS_PER_SECOND) {
@@ -293,9 +291,11 @@ class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper {
           }
           micros += step
         }
-        result.append(RebaseRecord(zid.getId, switches.toArray, diffs.toArray))
+        RebaseRecord(zid.getId, switches.toArray, diffs.toArray)
       }
     }
+    val result = new ArrayBuffer[RebaseRecord]()
+    rebaseRecords.sortBy(_.tz).foreach(result.append(_))
     val mapper = (new ObjectMapper() with ScalaObjectMapper)
       .registerModule(DefaultScalaModule)
       .writerWithDefaultPrettyPrinter()
@@ -308,7 +308,7 @@ class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper {
 
   ignore("generate 'gregorian-julian-rebase-micros.json'") {
     generateRebaseJson(
-      adjustFunc = identity[Long],
+      adjustFunc = (_: TimeZone, micros: Long) => micros,
       rebaseFunc = rebaseGregorianToJulianMicros,
       dir = "/Users/maximgekk/tmp",
       fileName = "gregorian-julian-rebase-micros.json")
@@ -383,30 +383,73 @@ class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper {
 
   test("rebase not-existed timestamps in the hybrid calendar") {
     outstandingZoneIds.foreach { zid =>
-      withDefaultTimeZone(zid) {
-        Seq(
-          "1582-10-04T23:59:59.999999" -> "1582-10-04 23:59:59.999999",
-          "1582-10-05T00:00:00.000000" -> "1582-10-15 00:00:00.000000",
-          "1582-10-06T01:02:03.000001" -> "1582-10-15 01:02:03.000001",
-          "1582-10-07T00:00:00.000000" -> "1582-10-15 00:00:00.000000",
-          "1582-10-08T23:59:59.999999" -> "1582-10-15 23:59:59.999999",
-          "1582-10-09T23:59:59.001001" -> "1582-10-15 23:59:59.001001",
-          "1582-10-10T00:11:22.334455" -> "1582-10-15 00:11:22.334455",
-          "1582-10-11T11:12:13.111111" -> "1582-10-15 11:12:13.111111",
-          "1582-10-12T10:11:12.131415" -> "1582-10-15 10:11:12.131415",
-          "1582-10-13T00:00:00.123321" -> "1582-10-15 00:00:00.123321",
-          "1582-10-14T23:59:59.999999" -> "1582-10-15 23:59:59.999999",
-          "1582-10-15T00:00:00.000000" -> "1582-10-15 00:00:00.000000"
-        ).foreach { case (gregTs, hybridTs) =>
-          withClue(s"tz = ${zid.getId} greg ts = $gregTs hybrid ts = $hybridTs") {
-            val hybridMicros = parseToJulianMicros(hybridTs)
-            val gregorianMicros = parseToGregMicros(gregTs, zid)
-
-            assert(rebaseGregorianToJulianMicros(zid, gregorianMicros) === hybridMicros)
+      Seq(
+        "1582-10-04T23:59:59.999999" -> "1582-10-04 23:59:59.999999",
+        "1582-10-05T00:00:00.000000" -> "1582-10-15 00:00:00.000000",
+        "1582-10-06T01:02:03.000001" -> "1582-10-15 01:02:03.000001",
+        "1582-10-07T00:00:00.000000" -> "1582-10-15 00:00:00.000000",
+        "1582-10-08T23:59:59.999999" -> "1582-10-15 23:59:59.999999",
+        "1582-10-09T23:59:59.001001" -> "1582-10-15 23:59:59.001001",
+        "1582-10-10T00:11:22.334455" -> "1582-10-15 00:11:22.334455",
+        "1582-10-11T11:12:13.111111" -> "1582-10-15 11:12:13.111111",
+        "1582-10-12T10:11:12.131415" -> "1582-10-15 10:11:12.131415",
+        "1582-10-13T00:00:00.123321" -> "1582-10-15 00:00:00.123321",
+        "1582-10-14T23:59:59.999999" -> "1582-10-15 23:59:59.999999",
+        "1582-10-15T00:00:00.000000" -> "1582-10-15 00:00:00.000000"
+      ).foreach { case (gregTs, hybridTs) =>
+        withClue(s"tz = ${zid.getId} greg ts = $gregTs hybrid ts = $hybridTs") {
+          val hybridMicros = withDefaultTimeZone(zid) { parseToJulianMicros(hybridTs) }
+          val gregorianMicros = parseToGregMicros(gregTs, zid)
+
+          val tz = TimeZone.getTimeZone(zid)
+          assert(rebaseGregorianToJulianMicros(tz, gregorianMicros) === hybridMicros)
+          withDefaultTimeZone(zid) {
             assert(rebaseGregorianToJulianMicros(gregorianMicros) === hybridMicros)
           }
         }
       }
     }
   }
+
+  test("SPARK-31959: JST -> HKT at Asia/Hong_Kong in 1945") {
+    // The 'Asia/Hong_Kong' time zone switched from 'Japan Standard Time' (JST = UTC+9)
+    // to 'Hong Kong Time' (HKT = UTC+8). After Sunday, 18 November, 1945 01:59:59 AM,
+    // clocks were moved backward to become Sunday, 18 November, 1945 01:00:00 AM.
+    // In this way, the overlap happened w/o Daylight Saving Time.
+    val hkZid = getZoneId("Asia/Hong_Kong")
+    var expected = "1945-11-18 01:30:00.0"
+    var ldt = LocalDateTime.of(1945, 11, 18, 1, 30, 0)
+    var earlierMicros = instantToMicros(ldt.atZone(hkZid).withEarlierOffsetAtOverlap().toInstant)
+    var laterMicros = instantToMicros(ldt.atZone(hkZid).withLaterOffsetAtOverlap().toInstant)
+    var overlapInterval = MICROS_PER_HOUR
+    if (earlierMicros + overlapInterval != laterMicros) {
+      // Old JDK might have an outdated time zone database.
+      // See https://bugs.openjdk.java.net/browse/JDK-8228469: "Hong Kong ... Its 1945 transition
+      // from JST to HKT was on 11-18 at 02:00, not 09-15 at 00:00"
+      expected = "1945-09-14 23:30:00.0"
+      ldt = LocalDateTime.of(1945, 9, 14, 23, 30, 0)
+      earlierMicros = instantToMicros(ldt.atZone(hkZid).withEarlierOffsetAtOverlap().toInstant)
+      laterMicros = instantToMicros(ldt.atZone(hkZid).withLaterOffsetAtOverlap().toInstant)
+      // If time zone db doesn't have overlapping at all, set the overlap interval to zero.
+      overlapInterval = laterMicros - earlierMicros
+    }
+    val hkTz = TimeZone.getTimeZone(hkZid)
+    val rebasedEarlierMicros = rebaseGregorianToJulianMicros(hkTz, earlierMicros)
+    val rebasedLaterMicros = rebaseGregorianToJulianMicros(hkTz, laterMicros)
+    assert(rebasedEarlierMicros + overlapInterval === rebasedLaterMicros)
+    withDefaultTimeZone(hkZid) {
+      def toTsStr(micros: Long): String = toJavaTimestamp(micros).toString
+      assert(toTsStr(rebasedEarlierMicros) === expected)
+      assert(toTsStr(rebasedLaterMicros) === expected)
+      // Check optimized rebasing
+      assert(rebaseGregorianToJulianMicros(earlierMicros) === rebasedEarlierMicros)
+      assert(rebaseGregorianToJulianMicros(laterMicros) === rebasedLaterMicros)
+      // Check reverse rebasing
+      assert(rebaseJulianToGregorianMicros(rebasedEarlierMicros) === earlierMicros)
+      assert(rebaseJulianToGregorianMicros(rebasedLaterMicros) === laterMicros)
+    }
+    // Check reverse not-optimized rebasing
+    assert(rebaseJulianToGregorianMicros(hkTz, rebasedEarlierMicros) === earlierMicros)
+    assert(rebaseJulianToGregorianMicros(hkTz, rebasedLaterMicros) === laterMicros)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/SQLOrderingUtilSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/SQLOrderingUtilSuite.scala
new file mode 100644
index 0000000000000..6fe774e8afcb6
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/SQLOrderingUtilSuite.scala
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import java.lang.{Double => JDouble, Float => JFloat}
+
+import org.apache.spark.SparkFunSuite
+
+class SQLOrderingUtilSuite extends SparkFunSuite {
+
+  test("compareDoublesSQL") {
+    def shouldMatchDefaultOrder(a: Double, b: Double): Unit = {
+      assert(SQLOrderingUtil.compareDoubles(a, b) === JDouble.compare(a, b))
+      assert(SQLOrderingUtil.compareDoubles(b, a) === JDouble.compare(b, a))
+    }
+    shouldMatchDefaultOrder(0d, 0d)
+    shouldMatchDefaultOrder(0d, 1d)
+    shouldMatchDefaultOrder(-1d, 1d)
+    shouldMatchDefaultOrder(Double.MinValue, Double.MaxValue)
+
+    val specialNaN = JDouble.longBitsToDouble(0x7ff1234512345678L)
+    assert(JDouble.isNaN(specialNaN))
+    assert(JDouble.doubleToRawLongBits(Double.NaN) != JDouble.doubleToRawLongBits(specialNaN))
+
+    assert(SQLOrderingUtil.compareDoubles(Double.NaN, Double.NaN) === 0)
+    assert(SQLOrderingUtil.compareDoubles(Double.NaN, specialNaN) === 0)
+    assert(SQLOrderingUtil.compareDoubles(Double.NaN, Double.PositiveInfinity) > 0)
+    assert(SQLOrderingUtil.compareDoubles(specialNaN, Double.PositiveInfinity) > 0)
+    assert(SQLOrderingUtil.compareDoubles(Double.NaN, Double.NegativeInfinity) > 0)
+    assert(SQLOrderingUtil.compareDoubles(Double.PositiveInfinity, Double.NaN) < 0)
+    assert(SQLOrderingUtil.compareDoubles(Double.NegativeInfinity, Double.NaN) < 0)
+    assert(SQLOrderingUtil.compareDoubles(0.0d, -0.0d) === 0)
+    assert(SQLOrderingUtil.compareDoubles(-0.0d, 0.0d) === 0)
+  }
+
+  test("compareFloatsSQL") {
+    def shouldMatchDefaultOrder(a: Float, b: Float): Unit = {
+      assert(SQLOrderingUtil.compareFloats(a, b) === JFloat.compare(a, b))
+      assert(SQLOrderingUtil.compareFloats(b, a) === JFloat.compare(b, a))
+    }
+    shouldMatchDefaultOrder(0f, 0f)
+    shouldMatchDefaultOrder(0f, 1f)
+    shouldMatchDefaultOrder(-1f, 1f)
+    shouldMatchDefaultOrder(Float.MinValue, Float.MaxValue)
+
+    val specialNaN = JFloat.intBitsToFloat(-6966608)
+    assert(JFloat.isNaN(specialNaN))
+    assert(JFloat.floatToRawIntBits(Float.NaN) != JFloat.floatToRawIntBits(specialNaN))
+
+    assert(SQLOrderingUtil.compareDoubles(Float.NaN, Float.NaN) === 0)
+    assert(SQLOrderingUtil.compareDoubles(Float.NaN, specialNaN) === 0)
+    assert(SQLOrderingUtil.compareDoubles(Float.NaN, Float.PositiveInfinity) > 0)
+    assert(SQLOrderingUtil.compareDoubles(specialNaN, Float.PositiveInfinity) > 0)
+    assert(SQLOrderingUtil.compareDoubles(Float.NaN, Float.NegativeInfinity) > 0)
+    assert(SQLOrderingUtil.compareDoubles(Float.PositiveInfinity, Float.NaN) < 0)
+    assert(SQLOrderingUtil.compareDoubles(Float.NegativeInfinity, Float.NaN) < 0)
+    assert(SQLOrderingUtil.compareDoubles(0.0f, -0.0f) === 0)
+    assert(SQLOrderingUtil.compareDoubles(-0.0f, 0.0f) === 0)
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/StringUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/StringUtilsSuite.scala
index 67bc4bc81cb92..7ef08fe215c67 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/StringUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/StringUtilsSuite.scala
@@ -18,9 +18,11 @@
 package org.apache.spark.sql.catalyst.util
 
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.catalyst.util.StringUtils._
+import org.apache.spark.sql.internal.SQLConf
 
-class StringUtilsSuite extends SparkFunSuite {
+class StringUtilsSuite extends SparkFunSuite with SQLHelper {
 
   test("escapeLikeRegex") {
     val expectedEscapedStrOne = "(?s)\\Qa\\E\\Qb\\E\\Qd\\E\\Qe\\E\\Qf\\E"
@@ -98,4 +100,42 @@ class StringUtilsSuite extends SparkFunSuite {
     assert(checkLimit("1234567"))
     assert(checkLimit("1234567890"))
   }
+
+  test("SPARK-31916: StringConcat doesn't overflow on many inputs") {
+    val concat = new StringConcat(maxLength = 100)
+    val stringToAppend = "Test internal index of StringConcat does not overflow with many " +
+      "append calls"
+    0.to((Integer.MAX_VALUE / stringToAppend.length) + 1).foreach { _ =>
+      concat.append(stringToAppend)
+    }
+    assert(concat.toString.length === 100)
+  }
+
+  test("SPARK-31916: verify that PlanStringConcat's output shows the actual length of the plan") {
+    withSQLConf(SQLConf.MAX_PLAN_STRING_LENGTH.key -> "0") {
+      val concat = new PlanStringConcat()
+      0.to(3).foreach { i =>
+        concat.append(s"plan fragment $i")
+      }
+      assert(concat.toString === "Truncated plan of 60 characters")
+    }
+
+    withSQLConf(SQLConf.MAX_PLAN_STRING_LENGTH.key -> "60") {
+      val concat = new PlanStringConcat()
+      0.to(2).foreach { i =>
+        concat.append(s"plan fragment $i")
+      }
+      assert(concat.toString === "plan fragment 0plan fragment 1... 15 more characters")
+    }
+  }
+
+  test("SPARK-34872: quoteIfNeeded should quote a string which contains non-word characters") {
+    assert(quoteIfNeeded("a b") === "`a b`")
+    assert(quoteIfNeeded("a*b") === "`a*b`")
+    assert(quoteIfNeeded("123") === "`123`")
+    assert(quoteIfNeeded("1a") === "1a")
+    assert(quoteIfNeeded("_ab_") === "_ab_")
+    assert(quoteIfNeeded("_") === "_")
+    assert(quoteIfNeeded("") === "``")
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
new file mode 100644
index 0000000000000..c65fec29bc6b8
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
@@ -0,0 +1,456 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import java.time.{DateTimeException, Instant, LocalDateTime, LocalTime}
+import java.util.concurrent.TimeUnit
+
+import org.apache.commons.lang3.{JavaVersion, SystemUtils}
+import org.scalatest.matchers.should.Matchers._
+
+import org.apache.spark.SparkUpgradeException
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
+import org.apache.spark.sql.catalyst.util.DateTimeUtils._
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
+import org.apache.spark.unsafe.types.UTF8String
+
+class TimestampFormatterSuite extends DatetimeFormatterSuite {
+
+  override def checkFormatterCreation(pattern: String, isParsing: Boolean): Unit = {
+    TimestampFormatter(pattern, UTC, isParsing)
+  }
+
+  override protected def useDateFormatter: Boolean = false
+
+  test("parsing timestamps using time zones") {
+    val localDate = "2018-12-02T10:11:12.001234"
+    val expectedMicros = Map(
+      "UTC" -> 1543745472001234L,
+      PST.getId -> 1543774272001234L,
+      CET.getId -> 1543741872001234L,
+      "Africa/Dakar" -> 1543745472001234L,
+      "America/Los_Angeles" -> 1543774272001234L,
+      "Antarctica/Vostok" -> 1543723872001234L,
+      "Asia/Hong_Kong" -> 1543716672001234L,
+      "Europe/Amsterdam" -> 1543741872001234L)
+    outstandingTimezonesIds.foreach { zoneId =>
+      val formatter = TimestampFormatter(
+        "yyyy-MM-dd'T'HH:mm:ss.SSSSSS",
+        getZoneId(zoneId),
+        isParsing = true)
+      val microsSinceEpoch = formatter.parse(localDate)
+      assert(microsSinceEpoch === expectedMicros(zoneId))
+    }
+  }
+
+  test("format timestamps using time zones") {
+    val microsSinceEpoch = 1543745472001234L
+    val expectedTimestamp = Map(
+      "UTC" -> "2018-12-02 10:11:12.001234",
+      PST.getId -> "2018-12-02 02:11:12.001234",
+      CET.getId -> "2018-12-02 11:11:12.001234",
+      "Africa/Dakar" -> "2018-12-02 10:11:12.001234",
+      "America/Los_Angeles" -> "2018-12-02 02:11:12.001234",
+      "Antarctica/Vostok" -> "2018-12-02 16:11:12.001234",
+      "Asia/Hong_Kong" -> "2018-12-02 18:11:12.001234",
+      "Europe/Amsterdam" -> "2018-12-02 11:11:12.001234")
+    outstandingTimezonesIds.foreach { zoneId =>
+      Seq(
+        TimestampFormatter(
+          "yyyy-MM-dd HH:mm:ss.SSSSSS",
+          getZoneId(zoneId),
+          // Test only FAST_DATE_FORMAT because other legacy formats don't support formatting
+          // in microsecond precision.
+          LegacyDateFormats.FAST_DATE_FORMAT,
+          isParsing = false),
+        TimestampFormatter.getFractionFormatter(getZoneId(zoneId))).foreach { formatter =>
+        val timestamp = formatter.format(microsSinceEpoch)
+        assert(timestamp === expectedTimestamp(zoneId))
+        assert(formatter.format(microsToInstant(microsSinceEpoch)) === expectedTimestamp(zoneId))
+        assert(formatter.format(toJavaTimestamp(microsSinceEpoch)) === expectedTimestamp(zoneId))
+      }
+    }
+  }
+
+  test("roundtrip micros -> timestamp -> micros using timezones") {
+    Seq("yyyy-MM-dd'T'HH:mm:ss.SSSSSS", "yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXXXX").foreach { pattern =>
+      Seq(
+        -58710115316212000L,
+        -18926315945345679L,
+        -9463427405253013L,
+        -244000001L,
+        0L,
+        99628200102030L,
+        1543749753123456L,
+        2177456523456789L,
+        11858049903010203L).foreach { micros =>
+        outstandingZoneIds.foreach { zoneId =>
+          val timestamp = TimestampFormatter(pattern, zoneId, isParsing = false).format(micros)
+          val parsed = TimestampFormatter(
+            pattern, zoneId, isParsing = true).parse(timestamp)
+          assert(micros === parsed)
+        }
+      }
+    }
+  }
+
+  test("roundtrip timestamp -> micros -> timestamp using timezones") {
+    Seq(
+      "0109-07-20T18:38:03.788000",
+      "1370-04-01T10:00:54.654321",
+      "1670-02-11T14:09:54.746987",
+      "1969-12-31T23:55:55.999999",
+      "1970-01-01T00:00:00.000000",
+      "1973-02-27T02:30:00.102030",
+      "2018-12-02T11:22:33.123456",
+      "2039-01-01T01:02:03.456789",
+      "2345-10-07T22:45:03.010203").foreach { timestamp =>
+      outstandingZoneIds.foreach { zoneId =>
+        val pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSSSS"
+        val micros = TimestampFormatter(
+          pattern, zoneId, isParsing = true).parse(timestamp)
+        val formatted = TimestampFormatter(pattern, zoneId, isParsing = false).format(micros)
+        assert(timestamp === formatted)
+      }
+    }
+  }
+
+  test("case insensitive parsing of am and pm") {
+    val formatter = TimestampFormatter("yyyy MMM dd hh:mm:ss a", UTC, isParsing = false)
+    val micros = formatter.parse("2009 Mar 20 11:30:01 am")
+    assert(micros === date(2009, 3, 20, 11, 30, 1))
+  }
+
+  test("format fraction of second") {
+    val formatter = TimestampFormatter.getFractionFormatter(UTC)
+    Seq(
+      -999999 -> "1969-12-31 23:59:59.000001",
+      -999900 -> "1969-12-31 23:59:59.0001",
+      -1 -> "1969-12-31 23:59:59.999999",
+      0 -> "1970-01-01 00:00:00",
+      1 -> "1970-01-01 00:00:00.000001",
+      1000 -> "1970-01-01 00:00:00.001",
+      900000 -> "1970-01-01 00:00:00.9",
+      1000000 -> "1970-01-01 00:00:01").foreach { case (micros, tsStr) =>
+      assert(formatter.format(micros) === tsStr)
+      assert(formatter.format(microsToInstant(micros)) === tsStr)
+      withDefaultTimeZone(UTC) {
+        assert(formatter.format(toJavaTimestamp(micros)) === tsStr)
+      }
+    }
+  }
+
+  test("formatting negative years with default pattern") {
+    val instant = LocalDateTime.of(-99, 1, 1, 0, 0, 0).atZone(UTC).toInstant
+    val micros = instantToMicros(instant)
+    assert(TimestampFormatter(UTC).format(micros) === "-0099-01-01 00:00:00")
+    assert(TimestampFormatter(UTC).format(instant) === "-0099-01-01 00:00:00")
+    withDefaultTimeZone(UTC) { // toJavaTimestamp depends on the default time zone
+      assert(TimestampFormatter("yyyy-MM-dd HH:mm:SS G", UTC, isParsing = false)
+        .format(toJavaTimestamp(micros)) === "0100-01-01 00:00:00 BC")
+    }
+  }
+
+  test("special timestamp values") {
+    testSpecialDatetimeValues { zoneId =>
+      val formatter = TimestampFormatter(zoneId)
+      val tolerance = TimeUnit.SECONDS.toMicros(30)
+
+      assert(formatter.parse("EPOCH") === 0)
+      val now = instantToMicros(Instant.now())
+      formatter.parse("now") should be(now +- tolerance)
+      val localToday = LocalDateTime.now(zoneId)
+        .`with`(LocalTime.MIDNIGHT)
+        .atZone(zoneId)
+      val yesterday = instantToMicros(localToday.minusDays(1).toInstant)
+      formatter.parse("yesterday CET") should be(yesterday +- tolerance)
+      val today = instantToMicros(localToday.toInstant)
+      formatter.parse(" TODAY ") should be(today +- tolerance)
+      val tomorrow = instantToMicros(localToday.plusDays(1).toInstant)
+      formatter.parse("Tomorrow ") should be(tomorrow +- tolerance)
+    }
+  }
+
+  test("parsing timestamp strings with various seconds fractions") {
+    outstandingZoneIds.foreach { zoneId =>
+      def check(pattern: String, input: String, reference: String): Unit = {
+        val formatter = TimestampFormatter(pattern, zoneId, isParsing = true)
+        val expected = stringToTimestamp(UTF8String.fromString(reference), zoneId).get
+        val actual = formatter.parse(input)
+        assert(actual === expected)
+      }
+
+      check("yyyy-MM-dd'T'HH:mm:ss.SSSSSSSXXX",
+        "2019-10-14T09:39:07.3220000Z", "2019-10-14T09:39:07.322Z")
+      check("yyyy-MM-dd'T'HH:mm:ss.SSSSSS",
+        "2019-10-14T09:39:07.322000", "2019-10-14T09:39:07.322")
+      check("yyyy-MM-dd'T'HH:mm:ss.SSSSSSX",
+        "2019-10-14T09:39:07.123456Z", "2019-10-14T09:39:07.123456Z")
+      check("yyyy-MM-dd'T'HH:mm:ss.SSSSSSX",
+        "2019-10-14T09:39:07.000010Z", "2019-10-14T09:39:07.00001Z")
+      check("yyyy HH:mm:ss.SSSSS", "1970 01:02:03.00004", "1970-01-01 01:02:03.00004")
+      check("yyyy HH:mm:ss.SSSS", "2019 00:00:07.0100", "2019-01-01 00:00:07.0100")
+      check("yyyy-MM-dd'T'HH:mm:ss.SSSX",
+        "2019-10-14T09:39:07.322Z", "2019-10-14T09:39:07.322Z")
+      check("yyyy-MM-dd'T'HH:mm:ss.SS",
+        "2019-10-14T09:39:07.10", "2019-10-14T09:39:07.1")
+      check("yyyy-MM-dd'T'HH:mm:ss.S",
+        "2019-10-14T09:39:07.1", "2019-10-14T09:39:07.1")
+
+      try {
+        TimestampFormatter("yyyy/MM/dd HH_mm_ss.SSSSSS", zoneId, isParsing = true)
+          .parse("2019/11/14 20#25#30.123456")
+        fail("Expected to throw an exception for the invalid input")
+      } catch {
+        case e: java.time.format.DateTimeParseException =>
+          assert(e.getMessage.contains("could not be parsed"))
+      }
+    }
+  }
+
+  test("formatting timestamp strings up to microsecond precision") {
+    outstandingZoneIds.foreach { zoneId =>
+      def check(pattern: String, input: String, expected: String): Unit = {
+        val formatter = TimestampFormatter(pattern, zoneId, isParsing = false)
+        val timestamp = stringToTimestamp(UTF8String.fromString(input), zoneId).get
+        val actual = formatter.format(timestamp)
+        assert(actual === expected)
+      }
+
+      check(
+        "yyyy-MM-dd HH:mm:ss.SSSSSSS", "2019-10-14T09:39:07.123456",
+        "2019-10-14 09:39:07.1234560")
+      check(
+        "yyyy-MM-dd HH:mm:ss.SSSSSS", "1960-01-01T09:39:07.123456",
+        "1960-01-01 09:39:07.123456")
+      check(
+        "yyyy-MM-dd HH:mm:ss.SSSSS", "0001-10-14T09:39:07.1",
+        "0001-10-14 09:39:07.10000")
+      check(
+        "yyyy-MM-dd HH:mm:ss.SSSS", "9999-12-31T23:59:59.999",
+        "9999-12-31 23:59:59.9990")
+      check(
+        "yyyy-MM-dd HH:mm:ss.SSS", "1970-01-01T00:00:00.0101",
+        "1970-01-01 00:00:00.010")
+      check(
+        "yyyy-MM-dd HH:mm:ss.SS", "2019-10-14T09:39:07.09",
+        "2019-10-14 09:39:07.09")
+      check(
+        "yyyy-MM-dd HH:mm:ss.S", "2019-10-14T09:39:07.2",
+        "2019-10-14 09:39:07.2")
+      check(
+        "yyyy-MM-dd HH:mm:ss.S", "2019-10-14T09:39:07",
+        "2019-10-14 09:39:07.0")
+      check(
+        "yyyy-MM-dd HH:mm:ss", "2019-10-14T09:39:07.123456",
+        "2019-10-14 09:39:07")
+    }
+  }
+
+  test("SPARK-30958: parse timestamp with negative year") {
+    val formatter1 = TimestampFormatter("yyyy-MM-dd HH:mm:ss", UTC, isParsing = true)
+    assert(formatter1.parse("-1234-02-22 02:22:22") === date(-1234, 2, 22, 2, 22, 22))
+
+    def assertParsingError(f: => Unit): Unit = {
+      intercept[Exception](f) match {
+        case e: SparkUpgradeException =>
+          assert(e.getCause.isInstanceOf[DateTimeException])
+        case e =>
+          assert(e.isInstanceOf[DateTimeException])
+      }
+    }
+
+    // "yyyy" with "G" can't parse negative year or year 0000.
+    val formatter2 = TimestampFormatter("G yyyy-MM-dd HH:mm:ss", UTC, isParsing = true)
+    assertParsingError(formatter2.parse("BC -1234-02-22 02:22:22"))
+    assertParsingError(formatter2.parse("AC 0000-02-22 02:22:22"))
+
+    assert(formatter2.parse("BC 1234-02-22 02:22:22") === date(-1233, 2, 22, 2, 22, 22))
+    assert(formatter2.parse("AD 1234-02-22 02:22:22") === date(1234, 2, 22, 2, 22, 22))
+  }
+
+  test("SPARK-31557: rebasing in legacy formatters/parsers") {
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> LegacyBehaviorPolicy.LEGACY.toString) {
+      outstandingZoneIds.foreach { zoneId =>
+        withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> zoneId.getId) {
+          withDefaultTimeZone(zoneId) {
+            withClue(s"zoneId = ${zoneId.getId}") {
+              val formatters = LegacyDateFormats.values.toSeq.map { legacyFormat =>
+                TimestampFormatter(
+                  TimestampFormatter.defaultPattern,
+                  zoneId,
+                  TimestampFormatter.defaultLocale,
+                  legacyFormat,
+                  isParsing = false)
+              } :+ TimestampFormatter.getFractionFormatter(zoneId)
+              formatters.foreach { formatter =>
+                assert(microsToInstant(formatter.parse("1000-01-01 01:02:03"))
+                  .atZone(zoneId)
+                  .toLocalDateTime === LocalDateTime.of(1000, 1, 1, 1, 2, 3))
+
+                assert(formatter.format(
+                  LocalDateTime.of(1000, 1, 1, 1, 2, 3).atZone(zoneId).toInstant) ===
+                  "1000-01-01 01:02:03")
+                assert(formatter.format(instantToMicros(
+                  LocalDateTime.of(1000, 1, 1, 1, 2, 3)
+                    .atZone(zoneId).toInstant)) === "1000-01-01 01:02:03")
+                assert(formatter.format(java.sql.Timestamp.valueOf("1000-01-01 01:02:03")) ===
+                  "1000-01-01 01:02:03")
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  test("parsing hour with various patterns") {
+    def createFormatter(pattern: String): TimestampFormatter = {
+      // Use `SIMPLE_DATE_FORMAT`, so that the legacy parser also fails with invalid value range.
+      TimestampFormatter(pattern, UTC, LegacyDateFormats.SIMPLE_DATE_FORMAT, isParsing = true)
+    }
+
+    withClue("HH") {
+      val formatter = createFormatter("yyyy-MM-dd HH")
+
+      val micros1 = formatter.parse("2009-12-12 00")
+      assert(micros1 === date(2009, 12, 12))
+
+      val micros2 = formatter.parse("2009-12-12 15")
+      assert(micros2 === date(2009, 12, 12, 15))
+
+      intercept[DateTimeException](formatter.parse("2009-12-12 24"))
+    }
+
+    withClue("kk") {
+      val formatter = createFormatter("yyyy-MM-dd kk")
+
+      intercept[DateTimeException](formatter.parse("2009-12-12 00"))
+
+      val micros1 = formatter.parse("2009-12-12 15")
+      assert(micros1 === date(2009, 12, 12, 15))
+
+      val micros2 = formatter.parse("2009-12-12 24")
+      assert(micros2 === date(2009, 12, 12))
+    }
+
+    withClue("KK") {
+      val formatter = createFormatter("yyyy-MM-dd KK a")
+
+      val micros1 = formatter.parse("2009-12-12 00 am")
+      assert(micros1 === date(2009, 12, 12))
+
+      // JDK-8223773: DateTimeFormatter Fails to throw an Exception on Invalid HOUR_OF_AMPM
+      // For `KK`, "12:00:00 am" is the same as "00:00:00 pm".
+      if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_13)) {
+        intercept[DateTimeException](formatter.parse("2009-12-12 12 am"))
+      } else {
+        val micros2 = formatter.parse("2009-12-12 12 am")
+        assert(micros2 === date(2009, 12, 12, 12))
+      }
+
+      val micros3 = formatter.parse("2009-12-12 00 pm")
+      assert(micros3 === date(2009, 12, 12, 12))
+
+      intercept[DateTimeException](formatter.parse("2009-12-12 12 pm"))
+    }
+
+    withClue("hh") {
+      val formatter = createFormatter("yyyy-MM-dd hh a")
+
+      intercept[DateTimeException](formatter.parse("2009-12-12 00 am"))
+
+      val micros1 = formatter.parse("2009-12-12 12 am")
+      assert(micros1 === date(2009, 12, 12))
+
+      intercept[DateTimeException](formatter.parse("2009-12-12 00 pm"))
+
+      val micros2 = formatter.parse("2009-12-12 12 pm")
+      assert(micros2 === date(2009, 12, 12, 12))
+    }
+  }
+
+  test("missing date fields") {
+    val formatter = TimestampFormatter("HH:mm:ss", UTC, isParsing = true)
+    val micros = formatter.parse("11:30:01")
+    assert(micros === date(1970, 1, 1, 11, 30, 1))
+  }
+
+  test("missing year field with invalid date") {
+    // Use `SIMPLE_DATE_FORMAT`, so that the legacy parser also fails with invalid date.
+    val formatter =
+      TimestampFormatter("MM-dd", UTC, LegacyDateFormats.SIMPLE_DATE_FORMAT, isParsing = true)
+    withDefaultTimeZone(UTC)(intercept[DateTimeException](formatter.parse("02-29")))
+  }
+
+  test("missing am/pm field") {
+    Seq("HH", "hh", "KK", "kk").foreach { hour =>
+      val formatter = TimestampFormatter(s"yyyy $hour:mm:ss", UTC, isParsing = true)
+      val micros = formatter.parse("2009 11:30:01")
+      assert(micros === date(2009, 1, 1, 11, 30, 1))
+    }
+  }
+
+  test("missing time fields") {
+    val formatter = TimestampFormatter("yyyy HH", UTC, isParsing = true)
+    val micros = formatter.parse("2009 11")
+    assert(micros === date(2009, 1, 1, 11))
+  }
+
+  test("missing hour field") {
+    val f1 = TimestampFormatter("mm:ss a", UTC, isParsing = true)
+    val t1 = f1.parse("30:01 PM")
+    assert(t1 === date(1970, 1, 1, 12, 30, 1))
+    val t2 = f1.parse("30:01 AM")
+    assert(t2 === date(1970, 1, 1, 0, 30, 1))
+    val f2 = TimestampFormatter("mm:ss", UTC, isParsing = true)
+    val t3 = f2.parse("30:01")
+    assert(t3 === date(1970, 1, 1, 0, 30, 1))
+    val f3 = TimestampFormatter("a", UTC, isParsing = true)
+    val t4 = f3.parse("PM")
+    assert(t4 === date(1970, 1, 1, 12))
+    val t5 = f3.parse("AM")
+    assert(t5 === date(1970))
+  }
+
+  test("check result differences for datetime formatting") {
+    val formatter = TimestampFormatter("DD", UTC, isParsing = false)
+    assert(formatter.format(date(1970, 1, 3)) == "03")
+    assert(formatter.format(date(1970, 4, 9)) == "99")
+
+    if (System.getProperty("java.version").split("\\D+")(0).toInt < 9) {
+      // https://bugs.openjdk.java.net/browse/JDK-8079628
+      intercept[SparkUpgradeException] {
+        formatter.format(date(1970, 4, 10))
+      }
+    } else {
+      assert(formatter.format(date(1970, 4, 10)) == "100")
+    }
+  }
+
+  test("SPARK-32424: avoid silent data change when timestamp overflows") {
+    val formatter = TimestampFormatter("y", UTC, isParsing = true)
+    assert(formatter.parse("294247") === date(294247))
+    assert(formatter.parse("-290307") === date(-290307))
+    val e1 = intercept[ArithmeticException](formatter.parse("294248"))
+    assert(e1.getMessage === "long overflow")
+    val e2 = intercept[ArithmeticException](formatter.parse("-290308"))
+    assert(e2.getMessage === "long overflow")
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala
index 6d8ef68473778..5729b02dc4926 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala
@@ -55,7 +55,7 @@ class UnsafeArraySuite extends SparkFunSuite {
     BigDecimal("1.2345678901234567890123456").setScale(21, BigDecimal.RoundingMode.FLOOR),
     BigDecimal("2.3456789012345678901234567").setScale(21, BigDecimal.RoundingMode.FLOOR))
 
-  val calenderintervalArray = Array(
+  val calendarintervalArray = Array(
     new CalendarInterval(3, 2, 321), new CalendarInterval(1, 2, 123))
 
   val intMultiDimArray = Array(Array(1), Array(2, 20), Array(3, 30, 300))
@@ -72,9 +72,8 @@ class UnsafeArraySuite extends SparkFunSuite {
     arrayData
   }
 
-  private def toUnsafeArray[T : TypeTag](array: Array[T]): ArrayData = {
+  private def toUnsafeArray[T: TypeTag](array: Array[T]): ArrayData = {
     val converted = ExpressionEncoder[Array[T]].createSerializer().apply(array).getArray(0)
-    assert(converted.isInstanceOf[T])
     assert(converted.numElements == array.length)
     converted
   }
@@ -143,12 +142,12 @@ class UnsafeArraySuite extends SparkFunSuite {
 
     val schema = new StructType().add("array", ArrayType(CalendarIntervalType))
     val encoder = RowEncoder(schema).resolveAndBind()
-    val externalRow = Row(calenderintervalArray)
+    val externalRow = Row(calendarintervalArray)
     val ir = encoder.createSerializer().apply(externalRow)
     val unsafeCalendar = ir.getArray(0)
     assert(unsafeCalendar.isInstanceOf[UnsafeArrayData])
-    assert(unsafeCalendar.numElements == calenderintervalArray.length)
-    calenderintervalArray.zipWithIndex.map { case (e, i) =>
+    assert(unsafeCalendar.numElements == calendarintervalArray.length)
+    calendarintervalArray.zipWithIndex.map { case (e, i) =>
       assert(unsafeCalendar.getInterval(i) == e)
     }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeMapSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeMapSuite.scala
index ebc88612be22a..443534fd0a06a 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeMapSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeMapSuite.scala
@@ -48,8 +48,8 @@ class UnsafeMapSuite extends SparkFunSuite {
     val ser = new JavaSerializer(new SparkConf).newInstance()
     val mapDataSer = ser.deserialize[UnsafeMapData](ser.serialize(unsafeMapData))
     assert(mapDataSer.numElements() == 1)
-    assert(mapDataSer.keyArray().getInt(0) == 19285)
-    assert(mapDataSer.valueArray().getInt(0) == 19286)
+    assert(mapDataSer.keyArray().getLong(0) == 19285)
+    assert(mapDataSer.valueArray().getLong(0) == 19286)
     assert(mapDataSer.getBaseObject.asInstanceOf[Array[Byte]].length == 1024)
   }
 
@@ -57,8 +57,8 @@ class UnsafeMapSuite extends SparkFunSuite {
     val ser = new KryoSerializer(new SparkConf).newInstance()
     val mapDataSer = ser.deserialize[UnsafeMapData](ser.serialize(unsafeMapData))
     assert(mapDataSer.numElements() == 1)
-    assert(mapDataSer.keyArray().getInt(0) == 19285)
-    assert(mapDataSer.valueArray().getInt(0) == 19286)
+    assert(mapDataSer.keyArray().getLong(0) == 19285)
+    assert(mapDataSer.valueArray().getLong(0) == 19286)
     assert(mapDataSer.getBaseObject.asInstanceOf[Array[Byte]].length == 1024)
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeRowUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeRowUtilsSuite.scala
new file mode 100644
index 0000000000000..4b6a3cfafd894
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeRowUtilsSuite.scala
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.expressions.{SpecificInternalRow, UnsafeProjection, UnsafeRow}
+import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
+
+class UnsafeRowUtilsSuite extends SparkFunSuite {
+
+  val testKeys: Seq[String] = Seq("key1", "key2")
+  val testValues: Seq[String] = Seq("sum(key1)", "sum(key2)")
+
+  val testOutputSchema: StructType = StructType(
+    testKeys.map(createIntegerField) ++ testValues.map(createIntegerField))
+
+  val testRow: UnsafeRow = {
+    val unsafeRowProjection = UnsafeProjection.create(testOutputSchema)
+    val row = unsafeRowProjection(new SpecificInternalRow(testOutputSchema))
+    (testKeys ++ testValues).zipWithIndex.foreach { case (_, index) => row.setInt(index, index) }
+    row
+  }
+
+  private def createIntegerField(name: String): StructField = {
+    StructField(name, IntegerType, nullable = false)
+  }
+
+  test("UnsafeRow format invalidation") {
+    // Pass the checking
+    UnsafeRowUtils.validateStructuralIntegrity(testRow, testOutputSchema)
+    // Fail for fields number not match
+    assert(!UnsafeRowUtils.validateStructuralIntegrity(
+      testRow, StructType(testKeys.map(createIntegerField))))
+    // Fail for invalid schema
+    val invalidSchema = StructType(testKeys.map(createIntegerField) ++
+      Seq(StructField("struct", StructType(Seq(StructField("value1", StringType, true))), true),
+        StructField("value2", IntegerType, false)))
+    assert(!UnsafeRowUtils.validateStructuralIntegrity(testRow, invalidSchema))
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTable.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTable.scala
deleted file mode 100644
index 3d7026e180cd1..0000000000000
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTable.scala
+++ /dev/null
@@ -1,281 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.connector
-
-import java.util
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-
-import org.scalatest.Assertions._
-
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.connector.catalog._
-import org.apache.spark.sql.connector.expressions.{IdentityTransform, NamedReference, Transform}
-import org.apache.spark.sql.connector.read._
-import org.apache.spark.sql.connector.write._
-import org.apache.spark.sql.sources.{And, EqualTo, Filter, IsNotNull}
-import org.apache.spark.sql.types.StructType
-import org.apache.spark.sql.util.CaseInsensitiveStringMap
-
-/**
- * A simple in-memory table. Rows are stored as a buffered group produced by each output task.
- */
-class InMemoryTable(
-    val name: String,
-    val schema: StructType,
-    override val partitioning: Array[Transform],
-    override val properties: util.Map[String, String])
-  extends Table with SupportsRead with SupportsWrite with SupportsDelete {
-
-  private val allowUnsupportedTransforms =
-    properties.getOrDefault("allow-unsupported-transforms", "false").toBoolean
-
-  partitioning.foreach { t =>
-    if (!t.isInstanceOf[IdentityTransform] && !allowUnsupportedTransforms) {
-      throw new IllegalArgumentException(s"Transform $t must be IdentityTransform")
-    }
-  }
-
-  // The key `Seq[Any]` is the partition values.
-  val dataMap: mutable.Map[Seq[Any], BufferedRows] = mutable.Map.empty
-
-  def data: Array[BufferedRows] = dataMap.values.toArray
-
-  def rows: Seq[InternalRow] = dataMap.values.flatMap(_.rows).toSeq
-
-  private val partCols: Array[Array[String]] = partitioning.flatMap(_.references).map { ref =>
-    schema.findNestedField(ref.fieldNames(), includeCollections = false) match {
-      case Some(_) => ref.fieldNames()
-      case None => throw new IllegalArgumentException(s"${ref.describe()} does not exist.")
-    }
-  }
-
-  private def getKey(row: InternalRow): Seq[Any] = {
-    def extractor(fieldNames: Array[String], schema: StructType, row: InternalRow): Any = {
-      val index = schema.fieldIndex(fieldNames(0))
-      val value = row.toSeq(schema).apply(index)
-      if (fieldNames.length > 1) {
-        (value, schema(index).dataType) match {
-          case (row: InternalRow, nestedSchema: StructType) =>
-            extractor(fieldNames.drop(1), nestedSchema, row)
-          case (_, dataType) =>
-            throw new IllegalArgumentException(s"Unsupported type, ${dataType.simpleString}")
-        }
-      } else {
-        value
-      }
-    }
-    partCols.map(fieldNames => extractor(fieldNames, schema, row))
-  }
-
-  def withData(data: Array[BufferedRows]): InMemoryTable = dataMap.synchronized {
-    data.foreach(_.rows.foreach { row =>
-      val key = getKey(row)
-      dataMap += dataMap.get(key)
-        .map(key -> _.withRow(row))
-        .getOrElse(key -> new BufferedRows().withRow(row))
-    })
-    this
-  }
-
-  override def capabilities: util.Set[TableCapability] = Set(
-    TableCapability.BATCH_READ,
-    TableCapability.BATCH_WRITE,
-    TableCapability.OVERWRITE_BY_FILTER,
-    TableCapability.OVERWRITE_DYNAMIC,
-    TableCapability.TRUNCATE).asJava
-
-  override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = {
-    () => new InMemoryBatchScan(data.map(_.asInstanceOf[InputPartition]))
-  }
-
-  class InMemoryBatchScan(data: Array[InputPartition]) extends Scan with Batch {
-    override def readSchema(): StructType = schema
-
-    override def toBatch: Batch = this
-
-    override def planInputPartitions(): Array[InputPartition] = data
-
-    override def createReaderFactory(): PartitionReaderFactory = BufferedRowsReaderFactory
-  }
-
-  override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
-    InMemoryTable.maybeSimulateFailedTableWrite(new CaseInsensitiveStringMap(properties))
-    InMemoryTable.maybeSimulateFailedTableWrite(info.options)
-
-    new WriteBuilder with SupportsTruncate with SupportsOverwrite with SupportsDynamicOverwrite {
-      private var writer: BatchWrite = Append
-
-      override def truncate(): WriteBuilder = {
-        assert(writer == Append)
-        writer = TruncateAndAppend
-        this
-      }
-
-      override def overwrite(filters: Array[Filter]): WriteBuilder = {
-        assert(writer == Append)
-        writer = new Overwrite(filters)
-        this
-      }
-
-      override def overwriteDynamicPartitions(): WriteBuilder = {
-        assert(writer == Append)
-        writer = DynamicOverwrite
-        this
-      }
-
-      override def buildForBatch(): BatchWrite = writer
-    }
-  }
-
-  private abstract class TestBatchWrite extends BatchWrite {
-    override def createBatchWriterFactory(info: PhysicalWriteInfo): DataWriterFactory = {
-      BufferedRowsWriterFactory
-    }
-
-    override def abort(messages: Array[WriterCommitMessage]): Unit = {}
-  }
-
-  private object Append extends TestBatchWrite {
-    override def commit(messages: Array[WriterCommitMessage]): Unit = dataMap.synchronized {
-      withData(messages.map(_.asInstanceOf[BufferedRows]))
-    }
-  }
-
-  private object DynamicOverwrite extends TestBatchWrite {
-    override def commit(messages: Array[WriterCommitMessage]): Unit = dataMap.synchronized {
-      val newData = messages.map(_.asInstanceOf[BufferedRows])
-      dataMap --= newData.flatMap(_.rows.map(getKey))
-      withData(newData)
-    }
-  }
-
-  private class Overwrite(filters: Array[Filter]) extends TestBatchWrite {
-    import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
-    override def commit(messages: Array[WriterCommitMessage]): Unit = dataMap.synchronized {
-      val deleteKeys = InMemoryTable.filtersToKeys(
-        dataMap.keys, partCols.map(_.toSeq.quoted), filters)
-      dataMap --= deleteKeys
-      withData(messages.map(_.asInstanceOf[BufferedRows]))
-    }
-  }
-
-  private object TruncateAndAppend extends TestBatchWrite {
-    override def commit(messages: Array[WriterCommitMessage]): Unit = dataMap.synchronized {
-      dataMap.clear
-      withData(messages.map(_.asInstanceOf[BufferedRows]))
-    }
-  }
-
-  override def deleteWhere(filters: Array[Filter]): Unit = dataMap.synchronized {
-    import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
-    dataMap --= InMemoryTable.filtersToKeys(dataMap.keys, partCols.map(_.toSeq.quoted), filters)
-  }
-}
-
-object InMemoryTable {
-  val SIMULATE_FAILED_WRITE_OPTION = "spark.sql.test.simulateFailedWrite"
-
-  def filtersToKeys(
-      keys: Iterable[Seq[Any]],
-      partitionNames: Seq[String],
-      filters: Array[Filter]): Iterable[Seq[Any]] = {
-    keys.filter { partValues =>
-      filters.flatMap(splitAnd).forall {
-        case EqualTo(attr, value) =>
-          value == extractValue(attr, partitionNames, partValues)
-        case IsNotNull(attr) =>
-          null != extractValue(attr, partitionNames, partValues)
-        case f =>
-          throw new IllegalArgumentException(s"Unsupported filter type: $f")
-      }
-    }
-  }
-
-  private def extractValue(
-      attr: String,
-      partFieldNames: Seq[String],
-      partValues: Seq[Any]): Any = {
-    partFieldNames.zipWithIndex.find(_._1 == attr) match {
-      case Some((_, partIndex)) =>
-        partValues(partIndex)
-      case _ =>
-        throw new IllegalArgumentException(s"Unknown filter attribute: $attr")
-    }
-  }
-
-  private def splitAnd(filter: Filter): Seq[Filter] = {
-    filter match {
-      case And(left, right) => splitAnd(left) ++ splitAnd(right)
-      case _ => filter :: Nil
-    }
-  }
-
-  def maybeSimulateFailedTableWrite(tableOptions: CaseInsensitiveStringMap): Unit = {
-    if (tableOptions.getBoolean(SIMULATE_FAILED_WRITE_OPTION, false)) {
-      throw new IllegalStateException("Manual write to table failure.")
-    }
-  }
-}
-
-class BufferedRows extends WriterCommitMessage with InputPartition with Serializable {
-  val rows = new mutable.ArrayBuffer[InternalRow]()
-
-  def withRow(row: InternalRow): BufferedRows = {
-    rows.append(row)
-    this
-  }
-}
-
-private object BufferedRowsReaderFactory extends PartitionReaderFactory {
-  override def createReader(partition: InputPartition): PartitionReader[InternalRow] = {
-    new BufferedRowsReader(partition.asInstanceOf[BufferedRows])
-  }
-}
-
-private class BufferedRowsReader(partition: BufferedRows) extends PartitionReader[InternalRow] {
-  private var index: Int = -1
-
-  override def next(): Boolean = {
-    index += 1
-    index < partition.rows.length
-  }
-
-  override def get(): InternalRow = partition.rows(index)
-
-  override def close(): Unit = {}
-}
-
-private object BufferedRowsWriterFactory extends DataWriterFactory {
-  override def createWriter(partitionId: Int, taskId: Long): DataWriter[InternalRow] = {
-    new BufferWriter
-  }
-}
-
-private class BufferWriter extends DataWriter[InternalRow] {
-  private val buffer = new BufferedRows
-
-  override def write(row: InternalRow): Unit = buffer.rows.append(row.copy())
-
-  override def commit(): WriterCommitMessage = buffer
-
-  override def abort(): Unit = {}
-
-  override def close(): Unit = {}
-}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/CatalogManagerSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/CatalogManagerSuite.scala
index 17d326019f86b..bfff3ee855e6d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/CatalogManagerSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/CatalogManagerSuite.scala
@@ -19,78 +19,81 @@ package org.apache.spark.sql.connector.catalog
 
 import java.net.URI
 
+import scala.collection.JavaConverters._
+
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.analysis.{EmptyFunctionRegistry, FakeV2SessionCatalog, NoSuchNamespaceException}
-import org.apache.spark.sql.catalyst.catalog.{CatalogDatabase, InMemoryCatalog, SessionCatalog}
+import org.apache.spark.sql.catalyst.catalog.{CatalogDatabase, InMemoryCatalog => V1InMemoryCatalog, SessionCatalog}
+import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
-class CatalogManagerSuite extends SparkFunSuite {
+class CatalogManagerSuite extends SparkFunSuite with SQLHelper {
 
-  private def createSessionCatalog(conf: SQLConf): SessionCatalog = {
-    val catalog = new InMemoryCatalog()
+  private def createSessionCatalog(): SessionCatalog = {
+    val catalog = new V1InMemoryCatalog()
     catalog.createDatabase(
       CatalogDatabase(SessionCatalog.DEFAULT_DATABASE, "", new URI("fake"), Map.empty),
       ignoreIfExists = true)
-    new SessionCatalog(catalog, EmptyFunctionRegistry, conf)
+    new SessionCatalog(catalog, EmptyFunctionRegistry)
   }
 
   test("CatalogManager should reflect the changes of default catalog") {
-    val conf = new SQLConf
-    val catalogManager = new CatalogManager(conf, FakeV2SessionCatalog, createSessionCatalog(conf))
+    val catalogManager = new CatalogManager(FakeV2SessionCatalog, createSessionCatalog())
     assert(catalogManager.currentCatalog.name() == CatalogManager.SESSION_CATALOG_NAME)
     assert(catalogManager.currentNamespace.sameElements(Array("default")))
 
-    conf.setConfString("spark.sql.catalog.dummy", classOf[DummyCatalog].getName)
-    conf.setConfString(SQLConf.DEFAULT_CATALOG.key, "dummy")
-
-    // The current catalog should be changed if the default catalog is set.
-    assert(catalogManager.currentCatalog.name() == "dummy")
-    assert(catalogManager.currentNamespace.sameElements(Array("a", "b")))
+    withSQLConf("spark.sql.catalog.dummy" -> classOf[DummyCatalog].getName,
+      SQLConf.DEFAULT_CATALOG.key -> "dummy") {
+      // The current catalog should be changed if the default catalog is set.
+      assert(catalogManager.currentCatalog.name() == "dummy")
+      assert(catalogManager.currentNamespace.sameElements(Array("a", "b")))
+    }
   }
 
   test("CatalogManager should keep the current catalog once set") {
-    val conf = new SQLConf
-    val catalogManager = new CatalogManager(conf, FakeV2SessionCatalog, createSessionCatalog(conf))
+    val catalogManager = new CatalogManager(FakeV2SessionCatalog, createSessionCatalog())
     assert(catalogManager.currentCatalog.name() == CatalogManager.SESSION_CATALOG_NAME)
-    conf.setConfString("spark.sql.catalog.dummy", classOf[DummyCatalog].getName)
-    catalogManager.setCurrentCatalog("dummy")
-    assert(catalogManager.currentCatalog.name() == "dummy")
-    assert(catalogManager.currentNamespace.sameElements(Array("a", "b")))
-
-    conf.setConfString("spark.sql.catalog.dummy2", classOf[DummyCatalog].getName)
-    conf.setConfString(SQLConf.DEFAULT_CATALOG.key, "dummy2")
-    // The current catalog shouldn't be changed if it's set before.
-    assert(catalogManager.currentCatalog.name() == "dummy")
+    withSQLConf("spark.sql.catalog.dummy" -> classOf[DummyCatalog].getName) {
+      catalogManager.setCurrentCatalog("dummy")
+      assert(catalogManager.currentCatalog.name() == "dummy")
+      assert(catalogManager.currentNamespace.sameElements(Array("a", "b")))
+
+      withSQLConf("spark.sql.catalog.dummy2" -> classOf[DummyCatalog].getName,
+        SQLConf.DEFAULT_CATALOG.key -> "dummy2") {
+        // The current catalog shouldn't be changed if it's set before.
+        assert(catalogManager.currentCatalog.name() == "dummy")
+      }
+    }
   }
 
   test("current namespace should be updated when switching current catalog") {
-    val conf = new SQLConf
-    val catalogManager = new CatalogManager(conf, FakeV2SessionCatalog, createSessionCatalog(conf))
-    conf.setConfString("spark.sql.catalog.dummy", classOf[DummyCatalog].getName)
-    catalogManager.setCurrentCatalog("dummy")
-    assert(catalogManager.currentNamespace.sameElements(Array("a", "b")))
-    catalogManager.setCurrentNamespace(Array("a"))
-    assert(catalogManager.currentNamespace.sameElements(Array("a")))
-
-    // If we set current catalog to the same catalog, current namespace should stay the same.
-    catalogManager.setCurrentCatalog("dummy")
-    assert(catalogManager.currentNamespace.sameElements(Array("a")))
-
-    // If we switch to a different catalog, current namespace should be reset.
-    conf.setConfString("spark.sql.catalog.dummy2", classOf[DummyCatalog].getName)
-    catalogManager.setCurrentCatalog("dummy2")
-    assert(catalogManager.currentNamespace.sameElements(Array("a", "b")))
+    val catalogManager = new CatalogManager(FakeV2SessionCatalog, createSessionCatalog())
+    withSQLConf("spark.sql.catalog.dummy" -> classOf[DummyCatalog].getName) {
+      catalogManager.setCurrentCatalog("dummy")
+      assert(catalogManager.currentNamespace.sameElements(Array("a", "b")))
+      catalogManager.setCurrentNamespace(Array("a"))
+      assert(catalogManager.currentNamespace.sameElements(Array("a")))
+
+      // If we set current catalog to the same catalog, current namespace should stay the same.
+      catalogManager.setCurrentCatalog("dummy")
+      assert(catalogManager.currentNamespace.sameElements(Array("a")))
+
+      // If we switch to a different catalog, current namespace should be reset.
+      withSQLConf("spark.sql.catalog.dummy2" -> classOf[DummyCatalog].getName) {
+        catalogManager.setCurrentCatalog("dummy2")
+        assert(catalogManager.currentNamespace.sameElements(Array("a", "b")))
+      }
+    }
   }
 
   test("set current namespace") {
-    val conf = new SQLConf
-    val v1SessionCatalog = createSessionCatalog(conf)
+    val v1SessionCatalog = createSessionCatalog()
     v1SessionCatalog.createDatabase(
       CatalogDatabase(
         "test", "", v1SessionCatalog.getDefaultDBPath("test"), Map.empty),
       ignoreIfExists = false)
-    val catalogManager = new CatalogManager(conf, FakeV2SessionCatalog, v1SessionCatalog)
+    val catalogManager = new CatalogManager(FakeV2SessionCatalog, v1SessionCatalog)
 
     // If the current catalog is session catalog, setting current namespace actually sets
     // `SessionCatalog.currentDb`.
@@ -103,11 +106,26 @@ class CatalogManagerSuite extends SparkFunSuite {
     }
 
     // when switching current catalog, `SessionCatalog.currentDb` should be reset.
-    conf.setConfString("spark.sql.catalog.dummy", classOf[DummyCatalog].getName)
-    catalogManager.setCurrentCatalog("dummy")
-    assert(v1SessionCatalog.getCurrentDatabase == "default")
-    catalogManager.setCurrentNamespace(Array("test2"))
-    assert(v1SessionCatalog.getCurrentDatabase == "default")
+    withSQLConf("spark.sql.catalog.dummy" -> classOf[DummyCatalog].getName) {
+      catalogManager.setCurrentCatalog("dummy")
+      assert(v1SessionCatalog.getCurrentDatabase == "default")
+      catalogManager.setCurrentNamespace(Array("test2"))
+      assert(v1SessionCatalog.getCurrentDatabase == "default")
+
+      // Check namespace existence if currentCatalog implements SupportsNamespaces.
+      withSQLConf("spark.sql.catalog.testCatalog" -> classOf[InMemoryTableCatalog].getName) {
+        catalogManager.setCurrentCatalog("testCatalog")
+        catalogManager.currentCatalog.asInstanceOf[InMemoryTableCatalog]
+          .createNamespace(Array("test3"), Map.empty[String, String].asJava)
+        assert(v1SessionCatalog.getCurrentDatabase == "default")
+        catalogManager.setCurrentNamespace(Array("test3"))
+        assert(v1SessionCatalog.getCurrentDatabase == "default")
+
+        intercept[NoSuchNamespaceException] {
+          catalogManager.setCurrentNamespace(Array("ns1", "ns2"))
+        }
+      }
+    }
   }
 }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/CatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/CatalogSuite.scala
new file mode 100644
index 0000000000000..0cca1cc9bebf2
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/CatalogSuite.scala
@@ -0,0 +1,970 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.catalog
+
+import java.util
+import java.util.Collections
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.{NamespaceAlreadyExistsException, NoSuchFunctionException, NoSuchNamespaceException, NoSuchTableException, TableAlreadyExistsException}
+import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
+import org.apache.spark.sql.connector.catalog.functions.{BoundFunction, ScalarFunction, UnboundFunction}
+import org.apache.spark.sql.connector.expressions.LogicalExpressions
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types.{DataType, DoubleType, IntegerType, LongType, StringType, StructField, StructType, TimestampType}
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
+
+class CatalogSuite extends SparkFunSuite {
+  import CatalogV2Implicits._
+
+  private val emptyProps: util.Map[String, String] = Collections.emptyMap[String, String]
+  private val schema: StructType = new StructType()
+      .add("id", IntegerType)
+      .add("data", StringType)
+
+  private def newCatalog(): InMemoryCatalog = {
+    val newCatalog = new InMemoryCatalog
+    newCatalog.initialize("test", CaseInsensitiveStringMap.empty())
+    newCatalog
+  }
+
+  private val testNs = Array("`", ".")
+  private val testIdent = Identifier.of(testNs, "test_table")
+  private val testIdentNew = Identifier.of(testNs, "test_table_new")
+
+  test("Catalogs can load the catalog") {
+    val catalog = newCatalog()
+
+    val conf = new SQLConf
+    conf.setConfString("spark.sql.catalog.test", catalog.getClass.getName)
+
+    val loaded = Catalogs.load("test", conf)
+    assert(loaded.getClass == catalog.getClass)
+  }
+
+  test("listTables") {
+    val catalog = newCatalog()
+    val ident1 = Identifier.of(Array("ns"), "test_table_1")
+    val ident2 = Identifier.of(Array("ns"), "test_table_2")
+    val ident3 = Identifier.of(Array("ns2"), "test_table_1")
+
+    intercept[NoSuchNamespaceException](catalog.listTables(Array("ns")))
+
+    catalog.createTable(ident1, schema, Array.empty, emptyProps)
+
+    assert(catalog.listTables(Array("ns")).toSet == Set(ident1))
+    intercept[NoSuchNamespaceException](catalog.listTables(Array("ns2")))
+
+    catalog.createTable(ident3, schema, Array.empty, emptyProps)
+    catalog.createTable(ident2, schema, Array.empty, emptyProps)
+
+    assert(catalog.listTables(Array("ns")).toSet == Set(ident1, ident2))
+    assert(catalog.listTables(Array("ns2")).toSet == Set(ident3))
+
+    catalog.dropTable(ident1)
+
+    assert(catalog.listTables(Array("ns")).toSet == Set(ident2))
+
+    catalog.dropTable(ident2)
+
+    assert(catalog.listTables(Array("ns")).isEmpty)
+    assert(catalog.listTables(Array("ns2")).toSet == Set(ident3))
+  }
+
+  test("createTable") {
+    val catalog = newCatalog()
+
+    assert(!catalog.tableExists(testIdent))
+
+    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    val parsed = CatalystSqlParser.parseMultipartIdentifier(table.name)
+    assert(parsed == Seq("test", "`", ".", "test_table"))
+    assert(table.schema == schema)
+    assert(table.properties.asScala == Map())
+
+    assert(catalog.tableExists(testIdent))
+  }
+
+  test("createTable: with properties") {
+    val catalog = newCatalog()
+
+    val properties = new util.HashMap[String, String]()
+    properties.put("property", "value")
+
+    assert(!catalog.tableExists(testIdent))
+
+    val table = catalog.createTable(testIdent, schema, Array.empty, properties)
+
+    val parsed = CatalystSqlParser.parseMultipartIdentifier(table.name)
+    assert(parsed == Seq("test", "`", ".", "test_table"))
+    assert(table.schema == schema)
+    assert(table.properties == properties)
+
+    assert(catalog.tableExists(testIdent))
+  }
+
+  test("createTable: table already exists") {
+    val catalog = newCatalog()
+
+    assert(!catalog.tableExists(testIdent))
+
+    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    val exc = intercept[TableAlreadyExistsException] {
+      catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+    }
+
+    assert(exc.message.contains(testIdent.quoted))
+    assert(exc.message.contains("already exists"))
+
+    assert(catalog.tableExists(testIdent))
+  }
+
+  test("tableExists") {
+    val catalog = newCatalog()
+
+    assert(!catalog.tableExists(testIdent))
+
+    catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    assert(catalog.tableExists(testIdent))
+
+    catalog.dropTable(testIdent)
+
+    assert(!catalog.tableExists(testIdent))
+  }
+
+  test("loadTable") {
+    val catalog = newCatalog()
+
+    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+    val loaded = catalog.loadTable(testIdent)
+
+    assert(table.name == loaded.name)
+    assert(table.schema == loaded.schema)
+    assert(table.properties == loaded.properties)
+  }
+
+  test("loadTable: table does not exist") {
+    val catalog = newCatalog()
+
+    val exc = intercept[NoSuchTableException] {
+      catalog.loadTable(testIdent)
+    }
+
+    assert(exc.message.contains(testIdent.quoted))
+    assert(exc.message.contains("not found"))
+  }
+
+  test("invalidateTable") {
+    val catalog = newCatalog()
+
+    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+    catalog.invalidateTable(testIdent)
+
+    val loaded = catalog.loadTable(testIdent)
+
+    assert(table.name == loaded.name)
+    assert(table.schema == loaded.schema)
+    assert(table.properties == loaded.properties)
+  }
+
+  test("invalidateTable: table does not exist") {
+    val catalog = newCatalog()
+
+    assert(catalog.tableExists(testIdent) === false)
+
+    catalog.invalidateTable(testIdent)
+  }
+
+  test("alterTable: add property") {
+    val catalog = newCatalog()
+
+    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    assert(table.properties.asScala == Map())
+
+    val updated = catalog.alterTable(testIdent, TableChange.setProperty("prop-1", "1"))
+    assert(updated.properties.asScala == Map("prop-1" -> "1"))
+
+    val loaded = catalog.loadTable(testIdent)
+    assert(loaded.properties.asScala == Map("prop-1" -> "1"))
+
+    assert(table.properties.asScala == Map())
+  }
+
+  test("alterTable: add property to existing") {
+    val catalog = newCatalog()
+
+    val properties = new util.HashMap[String, String]()
+    properties.put("prop-1", "1")
+
+    val table = catalog.createTable(testIdent, schema, Array.empty, properties)
+
+    assert(table.properties.asScala == Map("prop-1" -> "1"))
+
+    val updated = catalog.alterTable(testIdent, TableChange.setProperty("prop-2", "2"))
+    assert(updated.properties.asScala == Map("prop-1" -> "1", "prop-2" -> "2"))
+
+    val loaded = catalog.loadTable(testIdent)
+    assert(loaded.properties.asScala == Map("prop-1" -> "1", "prop-2" -> "2"))
+
+    assert(table.properties.asScala == Map("prop-1" -> "1"))
+  }
+
+  test("alterTable: remove existing property") {
+    val catalog = newCatalog()
+
+    val properties = new util.HashMap[String, String]()
+    properties.put("prop-1", "1")
+
+    val table = catalog.createTable(testIdent, schema, Array.empty, properties)
+
+    assert(table.properties.asScala == Map("prop-1" -> "1"))
+
+    val updated = catalog.alterTable(testIdent, TableChange.removeProperty("prop-1"))
+    assert(updated.properties.asScala == Map())
+
+    val loaded = catalog.loadTable(testIdent)
+    assert(loaded.properties.asScala == Map())
+
+    assert(table.properties.asScala == Map("prop-1" -> "1"))
+  }
+
+  test("alterTable: remove missing property") {
+    val catalog = newCatalog()
+
+    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    assert(table.properties.asScala == Map())
+
+    val updated = catalog.alterTable(testIdent, TableChange.removeProperty("prop-1"))
+    assert(updated.properties.asScala == Map())
+
+    val loaded = catalog.loadTable(testIdent)
+    assert(loaded.properties.asScala == Map())
+
+    assert(table.properties.asScala == Map())
+  }
+
+  test("alterTable: add top-level column") {
+    val catalog = newCatalog()
+
+    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    assert(table.schema == schema)
+
+    val updated = catalog.alterTable(testIdent, TableChange.addColumn(Array("ts"), TimestampType))
+
+    assert(updated.schema == schema.add("ts", TimestampType))
+  }
+
+  test("alterTable: add required column") {
+    val catalog = newCatalog()
+
+    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    assert(table.schema == schema)
+
+    val updated = catalog.alterTable(testIdent,
+      TableChange.addColumn(Array("ts"), TimestampType, false))
+
+    assert(updated.schema == schema.add("ts", TimestampType, nullable = false))
+  }
+
+  test("alterTable: add column with comment") {
+    val catalog = newCatalog()
+
+    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    assert(table.schema == schema)
+
+    val updated = catalog.alterTable(testIdent,
+      TableChange.addColumn(Array("ts"), TimestampType, false, "comment text"))
+
+    val field = StructField("ts", TimestampType, nullable = false).withComment("comment text")
+    assert(updated.schema == schema.add(field))
+  }
+
+  test("alterTable: add nested column") {
+    val catalog = newCatalog()
+
+    val pointStruct = new StructType().add("x", DoubleType).add("y", DoubleType)
+    val tableSchema = schema.add("point", pointStruct)
+
+    val table = catalog.createTable(testIdent, tableSchema, Array.empty, emptyProps)
+
+    assert(table.schema == tableSchema)
+
+    val updated = catalog.alterTable(testIdent,
+      TableChange.addColumn(Array("point", "z"), DoubleType))
+
+    val expectedSchema = schema.add("point", pointStruct.add("z", DoubleType))
+
+    assert(updated.schema == expectedSchema)
+  }
+
+  test("alterTable: add column to primitive field fails") {
+    val catalog = newCatalog()
+
+    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    assert(table.schema == schema)
+
+    val exc = intercept[IllegalArgumentException] {
+      catalog.alterTable(testIdent, TableChange.addColumn(Array("data", "ts"), TimestampType))
+    }
+
+    assert(exc.getMessage.contains("Not a struct"))
+    assert(exc.getMessage.contains("data"))
+
+    // the table has not changed
+    assert(catalog.loadTable(testIdent).schema == schema)
+  }
+
+  test("alterTable: add field to missing column fails") {
+    val catalog = newCatalog()
+
+    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    assert(table.schema == schema)
+
+    val exc = intercept[IllegalArgumentException] {
+      catalog.alterTable(testIdent,
+        TableChange.addColumn(Array("missing_col", "new_field"), StringType))
+    }
+
+    assert(exc.getMessage.contains("missing_col"))
+    assert(exc.getMessage.contains("Cannot find"))
+  }
+
+  test("alterTable: update column data type") {
+    val catalog = newCatalog()
+
+    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    assert(table.schema == schema)
+
+    val updated = catalog.alterTable(testIdent, TableChange.updateColumnType(Array("id"), LongType))
+
+    val expectedSchema = new StructType().add("id", LongType).add("data", StringType)
+    assert(updated.schema == expectedSchema)
+  }
+
+  test("alterTable: update column nullability") {
+    val catalog = newCatalog()
+
+    val originalSchema = new StructType()
+        .add("id", IntegerType, nullable = false)
+        .add("data", StringType)
+    val table = catalog.createTable(testIdent, originalSchema, Array.empty, emptyProps)
+
+    assert(table.schema == originalSchema)
+
+    val updated = catalog.alterTable(testIdent,
+      TableChange.updateColumnNullability(Array("id"), true))
+
+    val expectedSchema = new StructType().add("id", IntegerType).add("data", StringType)
+    assert(updated.schema == expectedSchema)
+  }
+
+  test("alterTable: update missing column fails") {
+    val catalog = newCatalog()
+
+    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    assert(table.schema == schema)
+
+    val exc = intercept[IllegalArgumentException] {
+      catalog.alterTable(testIdent,
+        TableChange.updateColumnType(Array("missing_col"), LongType))
+    }
+
+    assert(exc.getMessage.contains("missing_col"))
+    assert(exc.getMessage.contains("Cannot find"))
+  }
+
+  test("alterTable: add comment") {
+    val catalog = newCatalog()
+
+    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    assert(table.schema == schema)
+
+    val updated = catalog.alterTable(testIdent,
+      TableChange.updateColumnComment(Array("id"), "comment text"))
+
+    val expectedSchema = new StructType()
+        .add("id", IntegerType, nullable = true, "comment text")
+        .add("data", StringType)
+    assert(updated.schema == expectedSchema)
+  }
+
+  test("alterTable: replace comment") {
+    val catalog = newCatalog()
+
+    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    assert(table.schema == schema)
+
+    catalog.alterTable(testIdent, TableChange.updateColumnComment(Array("id"), "comment text"))
+
+    val expectedSchema = new StructType()
+        .add("id", IntegerType, nullable = true, "replacement comment")
+        .add("data", StringType)
+
+    val updated = catalog.alterTable(testIdent,
+      TableChange.updateColumnComment(Array("id"), "replacement comment"))
+
+    assert(updated.schema == expectedSchema)
+  }
+
+  test("alterTable: add comment to missing column fails") {
+    val catalog = newCatalog()
+
+    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    assert(table.schema == schema)
+
+    val exc = intercept[IllegalArgumentException] {
+      catalog.alterTable(testIdent,
+        TableChange.updateColumnComment(Array("missing_col"), "comment"))
+    }
+
+    assert(exc.getMessage.contains("missing_col"))
+    assert(exc.getMessage.contains("Cannot find"))
+  }
+
+  test("alterTable: rename top-level column") {
+    val catalog = newCatalog()
+
+    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    assert(table.schema == schema)
+
+    val updated = catalog.alterTable(testIdent, TableChange.renameColumn(Array("id"), "some_id"))
+
+    val expectedSchema = new StructType().add("some_id", IntegerType).add("data", StringType)
+
+    assert(updated.schema == expectedSchema)
+  }
+
+  test("alterTable: rename nested column") {
+    val catalog = newCatalog()
+
+    val pointStruct = new StructType().add("x", DoubleType).add("y", DoubleType)
+    val tableSchema = schema.add("point", pointStruct)
+
+    val table = catalog.createTable(testIdent, tableSchema, Array.empty, emptyProps)
+
+    assert(table.schema == tableSchema)
+
+    val updated = catalog.alterTable(testIdent,
+      TableChange.renameColumn(Array("point", "x"), "first"))
+
+    val newPointStruct = new StructType().add("first", DoubleType).add("y", DoubleType)
+    val expectedSchema = schema.add("point", newPointStruct)
+
+    assert(updated.schema == expectedSchema)
+  }
+
+  test("alterTable: rename struct column") {
+    val catalog = newCatalog()
+
+    val pointStruct = new StructType().add("x", DoubleType).add("y", DoubleType)
+    val tableSchema = schema.add("point", pointStruct)
+
+    val table = catalog.createTable(testIdent, tableSchema, Array.empty, emptyProps)
+
+    assert(table.schema == tableSchema)
+
+    val updated = catalog.alterTable(testIdent,
+      TableChange.renameColumn(Array("point"), "p"))
+
+    val newPointStruct = new StructType().add("x", DoubleType).add("y", DoubleType)
+    val expectedSchema = schema.add("p", newPointStruct)
+
+    assert(updated.schema == expectedSchema)
+  }
+
+  test("alterTable: rename missing column fails") {
+    val catalog = newCatalog()
+
+    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    assert(table.schema == schema)
+
+    val exc = intercept[IllegalArgumentException] {
+      catalog.alterTable(testIdent,
+        TableChange.renameColumn(Array("missing_col"), "new_name"))
+    }
+
+    assert(exc.getMessage.contains("missing_col"))
+    assert(exc.getMessage.contains("Cannot find"))
+  }
+
+  test("alterTable: multiple changes") {
+    val catalog = newCatalog()
+
+    val pointStruct = new StructType().add("x", DoubleType).add("y", DoubleType)
+    val tableSchema = schema.add("point", pointStruct)
+
+    val table = catalog.createTable(testIdent, tableSchema, Array.empty, emptyProps)
+
+    assert(table.schema == tableSchema)
+
+    val updated = catalog.alterTable(testIdent,
+      TableChange.renameColumn(Array("point", "x"), "first"),
+      TableChange.renameColumn(Array("point", "y"), "second"))
+
+    val newPointStruct = new StructType().add("first", DoubleType).add("second", DoubleType)
+    val expectedSchema = schema.add("point", newPointStruct)
+
+    assert(updated.schema == expectedSchema)
+  }
+
+  test("alterTable: delete top-level column") {
+    val catalog = newCatalog()
+
+    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    assert(table.schema == schema)
+
+    val updated = catalog.alterTable(testIdent,
+      TableChange.deleteColumn(Array("id")))
+
+    val expectedSchema = new StructType().add("data", StringType)
+    assert(updated.schema == expectedSchema)
+  }
+
+  test("alterTable: delete nested column") {
+    val catalog = newCatalog()
+
+    val pointStruct = new StructType().add("x", DoubleType).add("y", DoubleType)
+    val tableSchema = schema.add("point", pointStruct)
+
+    val table = catalog.createTable(testIdent, tableSchema, Array.empty, emptyProps)
+
+    assert(table.schema == tableSchema)
+
+    val updated = catalog.alterTable(testIdent,
+      TableChange.deleteColumn(Array("point", "y")))
+
+    val newPointStruct = new StructType().add("x", DoubleType)
+    val expectedSchema = schema.add("point", newPointStruct)
+
+    assert(updated.schema == expectedSchema)
+  }
+
+  test("alterTable: delete missing column fails") {
+    val catalog = newCatalog()
+
+    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    assert(table.schema == schema)
+
+    val exc = intercept[IllegalArgumentException] {
+      catalog.alterTable(testIdent, TableChange.deleteColumn(Array("missing_col")))
+    }
+
+    assert(exc.getMessage.contains("missing_col"))
+    assert(exc.getMessage.contains("Cannot find"))
+  }
+
+  test("alterTable: delete missing nested column fails") {
+    val catalog = newCatalog()
+
+    val pointStruct = new StructType().add("x", DoubleType).add("y", DoubleType)
+    val tableSchema = schema.add("point", pointStruct)
+
+    val table = catalog.createTable(testIdent, tableSchema, Array.empty, emptyProps)
+
+    assert(table.schema == tableSchema)
+
+    val exc = intercept[IllegalArgumentException] {
+      catalog.alterTable(testIdent, TableChange.deleteColumn(Array("point", "z")))
+    }
+
+    assert(exc.getMessage.contains("z"))
+    assert(exc.getMessage.contains("Cannot find"))
+  }
+
+  test("alterTable: table does not exist") {
+    val catalog = newCatalog()
+
+    val exc = intercept[NoSuchTableException] {
+      catalog.alterTable(testIdent, TableChange.setProperty("prop", "val"))
+    }
+
+    assert(exc.message.contains(testIdent.quoted))
+    assert(exc.message.contains("not found"))
+  }
+
+  test("dropTable") {
+    val catalog = newCatalog()
+
+    assert(!catalog.tableExists(testIdent))
+
+    catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    assert(catalog.tableExists(testIdent))
+
+    val wasDropped = catalog.dropTable(testIdent)
+
+    assert(wasDropped)
+    assert(!catalog.tableExists(testIdent))
+  }
+
+  test("dropTable: table does not exist") {
+    val catalog = newCatalog()
+
+    assert(!catalog.tableExists(testIdent))
+
+    val wasDropped = catalog.dropTable(testIdent)
+
+    assert(!wasDropped)
+    assert(!catalog.tableExists(testIdent))
+  }
+
+  test("purgeTable") {
+    val catalog = newCatalog()
+    intercept[UnsupportedOperationException](catalog.purgeTable(testIdent))
+  }
+
+  test("renameTable") {
+    val catalog = newCatalog()
+
+    assert(!catalog.tableExists(testIdent))
+    assert(!catalog.tableExists(testIdentNew))
+
+    catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    assert(catalog.tableExists(testIdent))
+    catalog.renameTable(testIdent, testIdentNew)
+
+    assert(!catalog.tableExists(testIdent))
+    assert(catalog.tableExists(testIdentNew))
+  }
+
+  test("renameTable: fail if table does not exist") {
+    val catalog = newCatalog()
+
+    val exc = intercept[NoSuchTableException] {
+      catalog.renameTable(testIdent, testIdentNew)
+    }
+
+    assert(exc.message.contains(testIdent.quoted))
+    assert(exc.message.contains("not found"))
+  }
+
+  test("renameTable: fail if new table name already exists") {
+    val catalog = newCatalog()
+
+    assert(!catalog.tableExists(testIdent))
+    assert(!catalog.tableExists(testIdentNew))
+
+    catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+    catalog.createTable(testIdentNew, schema, Array.empty, emptyProps)
+
+    assert(catalog.tableExists(testIdent))
+    assert(catalog.tableExists(testIdentNew))
+
+    val exc = intercept[TableAlreadyExistsException] {
+      catalog.renameTable(testIdent, testIdentNew)
+    }
+
+    assert(exc.message.contains(testIdentNew.quoted))
+    assert(exc.message.contains("already exists"))
+  }
+
+  test("listNamespaces: list namespaces from metadata") {
+    val catalog = newCatalog()
+    catalog.createNamespace(Array("ns1"), Map("property" -> "value").asJava)
+
+    assert(catalog.listNamespaces === Array(Array("ns1")))
+    assert(catalog.listNamespaces(Array()) === Array(Array("ns1")))
+    assert(catalog.listNamespaces(Array("ns1")) === Array())
+  }
+
+  test("listNamespaces: list namespaces from tables") {
+    val catalog = newCatalog()
+    val ident1 = Identifier.of(Array("ns1", "ns2"), "test_table_1")
+    val ident2 = Identifier.of(Array("ns1", "ns2"), "test_table_2")
+
+    catalog.createTable(ident1, schema, Array.empty, emptyProps)
+    catalog.createTable(ident2, schema, Array.empty, emptyProps)
+
+    assert(catalog.listNamespaces === Array(Array("ns1")))
+    assert(catalog.listNamespaces(Array()) === Array(Array("ns1")))
+    assert(catalog.listNamespaces(Array("ns1")) === Array(Array("ns1", "ns2")))
+    assert(catalog.listNamespaces(Array("ns1", "ns2")) === Array())
+  }
+
+  test("listNamespaces: list namespaces from metadata and tables") {
+    val catalog = newCatalog()
+    val ident1 = Identifier.of(Array("ns1", "ns2"), "test_table_1")
+    val ident2 = Identifier.of(Array("ns1", "ns2"), "test_table_2")
+
+    catalog.createNamespace(Array("ns1"), Map("property" -> "value").asJava)
+    catalog.createTable(ident1, schema, Array.empty, emptyProps)
+    catalog.createTable(ident2, schema, Array.empty, emptyProps)
+
+    assert(catalog.listNamespaces === Array(Array("ns1")))
+    assert(catalog.listNamespaces(Array()) === Array(Array("ns1")))
+    assert(catalog.listNamespaces(Array("ns1")) === Array(Array("ns1", "ns2")))
+    assert(catalog.listNamespaces(Array("ns1", "ns2")) === Array())
+  }
+
+  test("loadNamespaceMetadata: fail if no metadata or tables exist") {
+    val catalog = newCatalog()
+
+    val exc = intercept[NoSuchNamespaceException] {
+      catalog.loadNamespaceMetadata(testNs)
+    }
+
+    assert(exc.getMessage.contains(testNs.quoted))
+  }
+
+  test("loadNamespaceMetadata: no metadata, table exists") {
+    val catalog = newCatalog()
+
+    catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    val metadata = catalog.loadNamespaceMetadata(testNs)
+
+    assert(metadata.asScala === Map.empty)
+  }
+
+  test("loadNamespaceMetadata: metadata exists, no tables") {
+    val catalog = newCatalog()
+
+    catalog.createNamespace(testNs, Map("property" -> "value").asJava)
+
+    val metadata = catalog.loadNamespaceMetadata(testNs)
+
+    assert(metadata.asScala === Map("property" -> "value"))
+  }
+
+  test("loadNamespaceMetadata: metadata and table exist") {
+    val catalog = newCatalog()
+
+    catalog.createNamespace(testNs, Map("property" -> "value").asJava)
+    catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    val metadata = catalog.loadNamespaceMetadata(testNs)
+
+    assert(metadata.asScala === Map("property" -> "value"))
+  }
+
+  test("createNamespace: basic behavior") {
+    val catalog = newCatalog()
+
+    catalog.createNamespace(testNs, Map("property" -> "value").asJava)
+
+    assert(catalog.namespaceExists(testNs) === true)
+    assert(catalog.loadNamespaceMetadata(testNs).asScala === Map("property" -> "value"))
+  }
+
+  test("createNamespace: fail if metadata already exists") {
+    val catalog = newCatalog()
+
+    catalog.createNamespace(testNs, Map("property" -> "value").asJava)
+
+    val exc = intercept[NamespaceAlreadyExistsException] {
+      catalog.createNamespace(testNs, Map("property" -> "value").asJava)
+    }
+
+    assert(exc.getMessage.contains(testNs.quoted))
+    assert(catalog.namespaceExists(testNs) === true)
+    assert(catalog.loadNamespaceMetadata(testNs).asScala === Map("property" -> "value"))
+  }
+
+  test("createNamespace: fail if namespace already exists from table") {
+    val catalog = newCatalog()
+
+    catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    assert(catalog.namespaceExists(testNs) === true)
+    assert(catalog.loadNamespaceMetadata(testNs).asScala === Map.empty)
+
+    val exc = intercept[NamespaceAlreadyExistsException] {
+      catalog.createNamespace(testNs, Map("property" -> "value").asJava)
+    }
+
+    assert(exc.getMessage.contains(testNs.quoted))
+    assert(catalog.namespaceExists(testNs) === true)
+    assert(catalog.loadNamespaceMetadata(testNs).asScala === Map.empty)
+  }
+
+  test("dropNamespace: drop missing namespace") {
+    val catalog = newCatalog()
+
+    assert(catalog.namespaceExists(testNs) === false)
+
+    val ret = catalog.dropNamespace(testNs)
+
+    assert(ret === false)
+  }
+
+  test("dropNamespace: drop empty namespace") {
+    val catalog = newCatalog()
+
+    catalog.createNamespace(testNs, Map("property" -> "value").asJava)
+
+    assert(catalog.namespaceExists(testNs) === true)
+    assert(catalog.loadNamespaceMetadata(testNs).asScala === Map("property" -> "value"))
+
+    val ret = catalog.dropNamespace(testNs)
+
+    assert(ret === true)
+    assert(catalog.namespaceExists(testNs) === false)
+  }
+
+  test("dropNamespace: drop even if it's not empty") {
+    val catalog = newCatalog()
+
+    catalog.createNamespace(testNs, Map("property" -> "value").asJava)
+    catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    assert(catalog.dropNamespace(testNs))
+
+    assert(!catalog.namespaceExists(testNs))
+    intercept[NoSuchNamespaceException](catalog.listTables(testNs))
+  }
+
+  test("alterNamespace: basic behavior") {
+    val catalog = newCatalog()
+
+    catalog.createNamespace(testNs, Map("property" -> "value").asJava)
+
+    catalog.alterNamespace(testNs, NamespaceChange.setProperty("property2", "value2"))
+    assert(catalog.loadNamespaceMetadata(testNs).asScala === Map(
+      "property" -> "value", "property2" -> "value2"))
+
+    catalog.alterNamespace(testNs,
+      NamespaceChange.removeProperty("property2"),
+      NamespaceChange.setProperty("property3", "value3"))
+    assert(catalog.loadNamespaceMetadata(testNs).asScala === Map(
+      "property" -> "value", "property3" -> "value3"))
+
+    catalog.alterNamespace(testNs, NamespaceChange.removeProperty("property3"))
+    assert(catalog.loadNamespaceMetadata(testNs).asScala === Map("property" -> "value"))
+  }
+
+  test("alterNamespace: create metadata if missing and table exists") {
+    val catalog = newCatalog()
+
+    catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+
+    catalog.alterNamespace(testNs, NamespaceChange.setProperty("property", "value"))
+
+    assert(catalog.loadNamespaceMetadata(testNs).asScala === Map("property" -> "value"))
+  }
+
+  test("alterNamespace: fail if no metadata or table exists") {
+    val catalog = newCatalog()
+
+    val exc = intercept[NoSuchNamespaceException] {
+      catalog.alterNamespace(testNs, NamespaceChange.setProperty("property", "value"))
+    }
+
+    assert(exc.getMessage.contains(testNs.quoted))
+  }
+
+  test("truncate non-partitioned table") {
+    val catalog = newCatalog()
+
+    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
+      .asInstanceOf[InMemoryTable]
+    table.withData(Array(
+      new BufferedRows("3").withRow(InternalRow(0, "abc", "3")),
+      new BufferedRows("4").withRow(InternalRow(1, "def", "4"))))
+    assert(table.truncateTable())
+    assert(table.rows.isEmpty)
+  }
+
+  test("truncate partitioned table") {
+    val partCatalog = new InMemoryPartitionTableCatalog
+    partCatalog.initialize("test", CaseInsensitiveStringMap.empty())
+
+    val table = partCatalog.createTable(
+      testIdent,
+      new StructType()
+        .add("col0", IntegerType)
+        .add("part0", IntegerType),
+      Array(LogicalExpressions.identity(LogicalExpressions.parseReference("part0"))),
+      util.Collections.emptyMap[String, String])
+    val partTable = table.asInstanceOf[InMemoryPartitionTable]
+    val partIdent = InternalRow.apply(0)
+    val partIdent1 = InternalRow.apply(1)
+    partTable.createPartition(partIdent, new util.HashMap[String, String]())
+    partTable.createPartition(partIdent1, new util.HashMap[String, String]())
+    partTable.withData(Array(
+      new BufferedRows("0").withRow(InternalRow(0, 0)),
+      new BufferedRows("1").withRow(InternalRow(1, 1))
+    ))
+    assert(partTable.listPartitionIdentifiers(Array.empty, InternalRow.empty).length == 2)
+    assert(!partTable.rows.isEmpty)
+    assert(partTable.truncateTable())
+    assert(partTable.listPartitionIdentifiers(Array.empty, InternalRow.empty).length == 2)
+    assert(partTable.rows.isEmpty)
+  }
+
+  val function: UnboundFunction = new UnboundFunction {
+    override def bind(inputType: StructType): BoundFunction = new ScalarFunction[Int] {
+      override def inputTypes(): Array[DataType] = Array(IntegerType)
+      override def resultType(): DataType = IntegerType
+      override def name(): String = "my_bound_function"
+    }
+    override def description(): String = "my_function"
+    override def name(): String = "my_function"
+  }
+
+  test("list functions") {
+    val catalog = newCatalog()
+    val ident1 = Identifier.of(Array("ns1", "ns2"), "func1")
+    val ident2 = Identifier.of(Array("ns1", "ns2"), "func2")
+    val ident3 = Identifier.of(Array("ns1", "ns3"), "func3")
+
+    catalog.createNamespace(Array("ns1", "ns2"), emptyProps)
+    catalog.createNamespace(Array("ns1", "ns3"), emptyProps)
+    catalog.createFunction(ident1, function)
+    catalog.createFunction(ident2, function)
+    catalog.createFunction(ident3, function)
+
+    assert(catalog.listFunctions(Array("ns1", "ns2")).toSet === Set(ident1, ident2))
+    assert(catalog.listFunctions(Array("ns1", "ns3")).toSet === Set(ident3))
+    assert(catalog.listFunctions(Array("ns1")).toSet == Set())
+    intercept[NoSuchNamespaceException](catalog.listFunctions(Array("ns2")))
+  }
+
+  test("lookup function") {
+    val catalog = newCatalog()
+    val ident = Identifier.of(Array("ns"), "func")
+    catalog.createNamespace(Array("ns"), emptyProps)
+    catalog.createFunction(ident, function)
+
+    assert(catalog.loadFunction(ident) == function)
+    intercept[NoSuchFunctionException](catalog.loadFunction(Identifier.of(Array("ns"), "func1")))
+    intercept[NoSuchFunctionException](catalog.loadFunction(Identifier.of(Array("ns1"), "func")))
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/CatalogV2UtilSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/CatalogV2UtilSuite.scala
index 7a9a7f52ff8fd..da5cfab8be3c7 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/CatalogV2UtilSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/CatalogV2UtilSuite.scala
@@ -28,7 +28,7 @@ class CatalogV2UtilSuite extends SparkFunSuite {
     val testCatalog = mock(classOf[TableCatalog])
     val ident = mock(classOf[Identifier])
     val table = mock(classOf[Table])
-    when(table.schema()).thenReturn(mock(classOf[StructType]))
+    when(table.schema()).thenReturn(new StructType().add("i", "int"))
     when(testCatalog.loadTable(ident)).thenReturn(table)
     val r = CatalogV2Util.loadRelation(testCatalog, ident)
     assert(r.isDefined)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryAtomicPartitionTable.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryAtomicPartitionTable.scala
new file mode 100644
index 0000000000000..a48eb04a98806
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryAtomicPartitionTable.scala
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.catalog
+
+import java.util
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionException, PartitionAlreadyExistsException, PartitionsAlreadyExistException}
+import org.apache.spark.sql.connector.expressions.Transform
+import org.apache.spark.sql.types.StructType
+
+/**
+ * This class is used to test SupportsAtomicPartitionManagement API.
+ */
+class InMemoryAtomicPartitionTable (
+    name: String,
+    schema: StructType,
+    partitioning: Array[Transform],
+    properties: util.Map[String, String])
+  extends InMemoryPartitionTable(name, schema, partitioning, properties)
+  with SupportsAtomicPartitionManagement {
+
+  override def createPartition(
+      ident: InternalRow,
+      properties: util.Map[String, String]): Unit = {
+    if (memoryTablePartitions.containsKey(ident)) {
+      throw new PartitionAlreadyExistsException(name, ident, partitionSchema)
+    } else {
+      createPartitionKey(ident.toSeq(schema))
+      memoryTablePartitions.put(ident, properties)
+    }
+  }
+
+  override def dropPartition(ident: InternalRow): Boolean = {
+    if (memoryTablePartitions.containsKey(ident)) {
+      memoryTablePartitions.remove(ident)
+      removePartitionKey(ident.toSeq(schema))
+      true
+    } else {
+      false
+    }
+  }
+
+  override def createPartitions(
+      idents: Array[InternalRow],
+      properties: Array[util.Map[String, String]]): Unit = {
+    if (idents.exists(partitionExists)) {
+      throw new PartitionsAlreadyExistException(
+        name, idents.filter(partitionExists), partitionSchema)
+    }
+    idents.zip(properties).foreach { case (ident, property) =>
+      createPartition(ident, property)
+    }
+  }
+
+  override def dropPartitions(idents: Array[InternalRow]): Boolean = {
+    if (!idents.forall(partitionExists)) {
+      return false;
+    }
+    idents.forall(dropPartition)
+  }
+
+  override def truncatePartitions(idents: Array[InternalRow]): Boolean = {
+    val nonExistent = idents.filterNot(partitionExists)
+    if (nonExistent.isEmpty) {
+      idents.foreach(truncatePartition)
+      true
+    } else {
+      throw new NoSuchPartitionException(name, nonExistent.head, partitionSchema)
+    }
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryCatalog.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryCatalog.scala
new file mode 100644
index 0000000000000..202b03f28f082
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryCatalog.scala
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.catalog
+
+import java.util
+import java.util.concurrent.ConcurrentHashMap
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.sql.catalyst.analysis.{NoSuchFunctionException, NoSuchNamespaceException}
+import org.apache.spark.sql.connector.catalog.functions.UnboundFunction
+
+class InMemoryCatalog extends InMemoryTableCatalog with FunctionCatalog {
+  protected val functions: util.Map[Identifier, UnboundFunction] =
+    new ConcurrentHashMap[Identifier, UnboundFunction]()
+
+  override protected def allNamespaces: Seq[Seq[String]] = {
+    (tables.keySet.asScala.map(_.namespace.toSeq) ++
+      functions.keySet.asScala.map(_.namespace.toSeq) ++
+      namespaces.keySet.asScala).toSeq.distinct
+  }
+
+  override def listFunctions(namespace: Array[String]): Array[Identifier] = {
+    if (namespace.isEmpty || namespaceExists(namespace)) {
+      functions.keySet.asScala.filter(_.namespace.sameElements(namespace)).toArray
+    } else {
+      throw new NoSuchNamespaceException(namespace)
+    }
+  }
+
+  override def loadFunction(ident: Identifier): UnboundFunction = {
+    Option(functions.get(ident)) match {
+      case Some(func) =>
+        func
+      case _ =>
+        throw new NoSuchFunctionException(ident)
+    }
+  }
+
+  def createFunction(ident: Identifier, fn: UnboundFunction): UnboundFunction = {
+    functions.put(ident, fn)
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryPartitionTable.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryPartitionTable.scala
new file mode 100644
index 0000000000000..58dc4847111e2
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryPartitionTable.scala
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.catalog
+
+import java.util
+import java.util.concurrent.ConcurrentHashMap
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionException, PartitionAlreadyExistsException}
+import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
+import org.apache.spark.sql.connector.expressions.Transform
+import org.apache.spark.sql.types.StructType
+
+/**
+ * This class is used to test SupportsPartitionManagement API.
+ */
+class InMemoryPartitionTable(
+    name: String,
+    schema: StructType,
+    partitioning: Array[Transform],
+    properties: util.Map[String, String])
+  extends InMemoryTable(name, schema, partitioning, properties) with SupportsPartitionManagement {
+  import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+  protected val memoryTablePartitions: util.Map[InternalRow, util.Map[String, String]] =
+    new ConcurrentHashMap[InternalRow, util.Map[String, String]]()
+
+  def partitionSchema: StructType = {
+    val partitionColumnNames = partitioning.toSeq.asPartitionColumns
+    new StructType(schema.filter(p => partitionColumnNames.contains(p.name)).toArray)
+  }
+
+  def createPartition(
+      ident: InternalRow,
+      properties: util.Map[String, String]): Unit = {
+    if (memoryTablePartitions.containsKey(ident)) {
+      throw new PartitionAlreadyExistsException(name, ident, partitionSchema)
+    } else {
+      createPartitionKey(ident.toSeq(schema))
+      memoryTablePartitions.put(ident, properties)
+    }
+  }
+
+  def dropPartition(ident: InternalRow): Boolean = {
+    if (memoryTablePartitions.containsKey(ident)) {
+      memoryTablePartitions.remove(ident)
+      removePartitionKey(ident.toSeq(schema))
+      true
+    } else {
+      false
+    }
+  }
+
+  def replacePartitionMetadata(ident: InternalRow, properties: util.Map[String, String]): Unit = {
+    if (memoryTablePartitions.containsKey(ident)) {
+      memoryTablePartitions.put(ident, properties)
+    } else {
+      throw new NoSuchPartitionException(name, ident, partitionSchema)
+    }
+  }
+
+  def loadPartitionMetadata(ident: InternalRow): util.Map[String, String] = {
+    if (memoryTablePartitions.containsKey(ident)) {
+      memoryTablePartitions.get(ident)
+    } else {
+      throw new NoSuchPartitionException(name, ident, partitionSchema)
+    }
+  }
+
+  override protected def addPartitionKey(key: Seq[Any]): Unit = {
+    memoryTablePartitions.putIfAbsent(InternalRow.fromSeq(key), Map.empty[String, String].asJava)
+  }
+
+  override def listPartitionIdentifiers(
+      names: Array[String],
+      ident: InternalRow): Array[InternalRow] = {
+    assert(names.length == ident.numFields,
+      s"Number of partition names (${names.length}) must be equal to " +
+      s"the number of partition values (${ident.numFields}).")
+    val schema = partitionSchema
+    assert(names.forall(fieldName => schema.fieldNames.contains(fieldName)),
+      s"Some partition names ${names.mkString("[", ", ", "]")} don't belong to " +
+      s"the partition schema '${schema.sql}'.")
+    val indexes = names.map(schema.fieldIndex)
+    val dataTypes = names.map(schema(_).dataType)
+    val currentRow = new GenericInternalRow(new Array[Any](names.length))
+    memoryTablePartitions.keySet().asScala.filter { key =>
+      for (i <- 0 until names.length) {
+        currentRow.values(i) = key.get(indexes(i), dataTypes(i))
+      }
+      currentRow == ident
+    }.toArray
+  }
+
+  override def renamePartition(from: InternalRow, to: InternalRow): Boolean = {
+    if (memoryTablePartitions.containsKey(to)) {
+      throw new PartitionAlreadyExistsException(name, to, partitionSchema)
+    } else {
+      val partValue = memoryTablePartitions.remove(from)
+      if (partValue == null) {
+        throw new NoSuchPartitionException(name, from, partitionSchema)
+      }
+      memoryTablePartitions.put(to, partValue) == null &&
+        renamePartitionKey(partitionSchema, from.toSeq(schema), to.toSeq(schema))
+    }
+  }
+
+  override def truncatePartition(ident: InternalRow): Boolean = {
+    if (memoryTablePartitions.containsKey(ident)) {
+      clearPartition(ident.toSeq(schema))
+      true
+    } else {
+      throw new NoSuchPartitionException(name, ident, partitionSchema)
+    }
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryPartitionTableCatalog.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryPartitionTableCatalog.scala
new file mode 100644
index 0000000000000..a24f5c9a0c463
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryPartitionTableCatalog.scala
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.catalog
+
+import java.util
+
+import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
+import org.apache.spark.sql.connector.expressions.Transform
+import org.apache.spark.sql.types.StructType
+
+class InMemoryPartitionTableCatalog extends InMemoryTableCatalog {
+  import CatalogV2Implicits._
+
+  override def createTable(
+      ident: Identifier,
+      schema: StructType,
+      partitions: Array[Transform],
+      properties: util.Map[String, String]): Table = {
+    if (tables.containsKey(ident)) {
+      throw new TableAlreadyExistsException(ident)
+    }
+
+    InMemoryTableCatalog.maybeSimulateFailedTableCreation(properties)
+
+    val table = new InMemoryAtomicPartitionTable(
+      s"$name.${ident.quoted}", schema, partitions, properties)
+    tables.put(ident, table)
+    namespaces.putIfAbsent(ident.namespace.toList, Map())
+    table
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTable.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTable.scala
new file mode 100644
index 0000000000000..9293dbcdce348
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTable.scala
@@ -0,0 +1,569 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.catalog
+
+import java.time.{Instant, ZoneId}
+import java.time.temporal.ChronoUnit
+import java.util
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+import org.scalatest.Assertions._
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.{GenericInternalRow, JoinedRow}
+import org.apache.spark.sql.catalyst.util.{CharVarcharUtils, DateTimeUtils}
+import org.apache.spark.sql.connector.distributions.{Distribution, Distributions}
+import org.apache.spark.sql.connector.expressions._
+import org.apache.spark.sql.connector.read._
+import org.apache.spark.sql.connector.write._
+import org.apache.spark.sql.connector.write.streaming.{StreamingDataWriterFactory, StreamingWrite}
+import org.apache.spark.sql.sources._
+import org.apache.spark.sql.types._
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
+import org.apache.spark.unsafe.types.UTF8String
+
+/**
+ * A simple in-memory table. Rows are stored as a buffered group produced by each output task.
+ */
+class InMemoryTable(
+    val name: String,
+    val schema: StructType,
+    override val partitioning: Array[Transform],
+    override val properties: util.Map[String, String],
+    val distribution: Distribution = Distributions.unspecified(),
+    val ordering: Array[SortOrder] = Array.empty,
+    val numPartitions: Option[Int] = None)
+  extends Table with SupportsRead with SupportsWrite with SupportsDelete
+      with SupportsMetadataColumns {
+
+  private object PartitionKeyColumn extends MetadataColumn {
+    override def name: String = "_partition"
+    override def dataType: DataType = StringType
+    override def comment: String = "Partition key used to store the row"
+  }
+
+  private object IndexColumn extends MetadataColumn {
+    override def name: String = "index"
+    override def dataType: DataType = IntegerType
+    override def comment: String = "Metadata column used to conflict with a data column"
+  }
+
+  // purposely exposes a metadata column that conflicts with a data column in some tests
+  override val metadataColumns: Array[MetadataColumn] = Array(IndexColumn, PartitionKeyColumn)
+  private val metadataColumnNames = metadataColumns.map(_.name).toSet -- schema.map(_.name)
+
+  private val allowUnsupportedTransforms =
+    properties.getOrDefault("allow-unsupported-transforms", "false").toBoolean
+
+  partitioning.foreach {
+    case _: IdentityTransform =>
+    case _: YearsTransform =>
+    case _: MonthsTransform =>
+    case _: DaysTransform =>
+    case _: HoursTransform =>
+    case _: BucketTransform =>
+    case t if !allowUnsupportedTransforms =>
+      throw new IllegalArgumentException(s"Transform $t is not a supported transform")
+  }
+
+  // The key `Seq[Any]` is the partition values.
+  val dataMap: mutable.Map[Seq[Any], BufferedRows] = mutable.Map.empty
+
+  def data: Array[BufferedRows] = dataMap.values.toArray
+
+  def rows: Seq[InternalRow] = dataMap.values.flatMap(_.rows).toSeq
+
+  private val partCols: Array[Array[String]] = partitioning.flatMap(_.references).map { ref =>
+    schema.findNestedField(ref.fieldNames(), includeCollections = false) match {
+      case Some(_) => ref.fieldNames()
+      case None => throw new IllegalArgumentException(s"${ref.describe()} does not exist.")
+    }
+  }
+
+  private val UTC = ZoneId.of("UTC")
+  private val EPOCH_LOCAL_DATE = Instant.EPOCH.atZone(UTC).toLocalDate
+
+  private def getKey(row: InternalRow): Seq[Any] = {
+    def extractor(
+        fieldNames: Array[String],
+        schema: StructType,
+        row: InternalRow): (Any, DataType) = {
+      val index = schema.fieldIndex(fieldNames(0))
+      val value = row.toSeq(schema).apply(index)
+      if (fieldNames.length > 1) {
+        (value, schema(index).dataType) match {
+          case (row: InternalRow, nestedSchema: StructType) =>
+            extractor(fieldNames.drop(1), nestedSchema, row)
+          case (_, dataType) =>
+            throw new IllegalArgumentException(s"Unsupported type, ${dataType.simpleString}")
+        }
+      } else {
+        (value, schema(index).dataType)
+      }
+    }
+
+    val cleanedSchema = CharVarcharUtils.replaceCharVarcharWithStringInSchema(schema)
+    partitioning.map {
+      case IdentityTransform(ref) =>
+        extractor(ref.fieldNames, cleanedSchema, row)._1
+      case YearsTransform(ref) =>
+        extractor(ref.fieldNames, cleanedSchema, row) match {
+          case (days: Int, DateType) =>
+            ChronoUnit.YEARS.between(EPOCH_LOCAL_DATE, DateTimeUtils.daysToLocalDate(days))
+          case (micros: Long, TimestampType) =>
+            val localDate = DateTimeUtils.microsToInstant(micros).atZone(UTC).toLocalDate
+            ChronoUnit.YEARS.between(EPOCH_LOCAL_DATE, localDate)
+          case (v, t) =>
+            throw new IllegalArgumentException(s"Match: unsupported argument(s) type - ($v, $t)")
+        }
+      case MonthsTransform(ref) =>
+        extractor(ref.fieldNames, cleanedSchema, row) match {
+          case (days: Int, DateType) =>
+            ChronoUnit.MONTHS.between(EPOCH_LOCAL_DATE, DateTimeUtils.daysToLocalDate(days))
+          case (micros: Long, TimestampType) =>
+            val localDate = DateTimeUtils.microsToInstant(micros).atZone(UTC).toLocalDate
+            ChronoUnit.MONTHS.between(EPOCH_LOCAL_DATE, localDate)
+          case (v, t) =>
+            throw new IllegalArgumentException(s"Match: unsupported argument(s) type - ($v, $t)")
+        }
+      case DaysTransform(ref) =>
+        extractor(ref.fieldNames, cleanedSchema, row) match {
+          case (days, DateType) =>
+            days
+          case (micros: Long, TimestampType) =>
+            ChronoUnit.DAYS.between(Instant.EPOCH, DateTimeUtils.microsToInstant(micros))
+          case (v, t) =>
+            throw new IllegalArgumentException(s"Match: unsupported argument(s) type - ($v, $t)")
+        }
+      case HoursTransform(ref) =>
+        extractor(ref.fieldNames, cleanedSchema, row) match {
+          case (micros: Long, TimestampType) =>
+            ChronoUnit.HOURS.between(Instant.EPOCH, DateTimeUtils.microsToInstant(micros))
+          case (v, t) =>
+            throw new IllegalArgumentException(s"Match: unsupported argument(s) type - ($v, $t)")
+        }
+      case BucketTransform(numBuckets, ref) =>
+        val (value, dataType) = extractor(ref.fieldNames, cleanedSchema, row)
+        val valueHashCode = if (value == null) 0 else value.hashCode
+        ((valueHashCode + 31 * dataType.hashCode()) & Integer.MAX_VALUE) % numBuckets
+    }
+  }
+
+  protected def addPartitionKey(key: Seq[Any]): Unit = {}
+
+  protected def renamePartitionKey(
+      partitionSchema: StructType,
+      from: Seq[Any],
+      to: Seq[Any]): Boolean = {
+    val rows = dataMap.remove(from).getOrElse(new BufferedRows(from.mkString("/")))
+    val newRows = new BufferedRows(to.mkString("/"))
+    rows.rows.foreach { r =>
+      val newRow = new GenericInternalRow(r.numFields)
+      for (i <- 0 until r.numFields) newRow.update(i, r.get(i, schema(i).dataType))
+      for (i <- 0 until partitionSchema.length) {
+        val j = schema.fieldIndex(partitionSchema(i).name)
+        newRow.update(j, to(i))
+      }
+      newRows.withRow(newRow)
+    }
+    dataMap.put(to, newRows).foreach { _ =>
+      throw new IllegalStateException(
+        s"The ${to.mkString("[", ", ", "]")} partition exists already")
+    }
+    true
+  }
+
+  protected def removePartitionKey(key: Seq[Any]): Unit = dataMap.synchronized {
+    dataMap.remove(key)
+  }
+
+  protected def createPartitionKey(key: Seq[Any]): Unit = dataMap.synchronized {
+    if (!dataMap.contains(key)) {
+      val emptyRows = new BufferedRows(key.toArray.mkString("/"))
+      val rows = if (key.length == schema.length) {
+        emptyRows.withRow(InternalRow.fromSeq(key))
+      } else emptyRows
+      dataMap.put(key, rows)
+    }
+  }
+
+  protected def clearPartition(key: Seq[Any]): Unit = dataMap.synchronized {
+    assert(dataMap.contains(key))
+    dataMap(key).clear()
+  }
+
+  def withData(data: Array[BufferedRows]): InMemoryTable = dataMap.synchronized {
+    data.foreach(_.rows.foreach { row =>
+      val key = getKey(row)
+      dataMap += dataMap.get(key)
+        .map(key -> _.withRow(row))
+        .getOrElse(key -> new BufferedRows(key.toArray.mkString("/")).withRow(row))
+      addPartitionKey(key)
+    })
+    this
+  }
+
+  override def capabilities: util.Set[TableCapability] = Set(
+    TableCapability.BATCH_READ,
+    TableCapability.BATCH_WRITE,
+    TableCapability.STREAMING_WRITE,
+    TableCapability.OVERWRITE_BY_FILTER,
+    TableCapability.OVERWRITE_DYNAMIC,
+    TableCapability.TRUNCATE).asJava
+
+  override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = {
+    new InMemoryScanBuilder(schema)
+  }
+
+  class InMemoryScanBuilder(tableSchema: StructType) extends ScanBuilder
+      with SupportsPushDownRequiredColumns {
+    private var schema: StructType = tableSchema
+
+    override def build: Scan =
+      new InMemoryBatchScan(data.map(_.asInstanceOf[InputPartition]), schema, tableSchema)
+
+    override def pruneColumns(requiredSchema: StructType): Unit = {
+      val schemaNames = metadataColumnNames ++ tableSchema.map(_.name)
+      schema = StructType(requiredSchema.filter(f => schemaNames.contains(f.name)))
+    }
+  }
+
+  class InMemoryBatchScan(
+      data: Array[InputPartition],
+      readSchema: StructType,
+      tableSchema: StructType) extends Scan with Batch {
+    override def readSchema(): StructType = readSchema
+
+    override def toBatch: Batch = this
+
+    override def planInputPartitions(): Array[InputPartition] = data
+
+    override def createReaderFactory(): PartitionReaderFactory = {
+      val metadataColumns = readSchema.map(_.name).filter(metadataColumnNames.contains)
+      val nonMetadataColumns = readSchema.filterNot(f => metadataColumns.contains(f.name))
+      new BufferedRowsReaderFactory(metadataColumns, nonMetadataColumns, tableSchema)
+    }
+  }
+
+  override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
+    InMemoryTable.maybeSimulateFailedTableWrite(new CaseInsensitiveStringMap(properties))
+    InMemoryTable.maybeSimulateFailedTableWrite(info.options)
+
+    new WriteBuilder with SupportsTruncate with SupportsOverwrite with SupportsDynamicOverwrite {
+      private var writer: BatchWrite = Append
+      private var streamingWriter: StreamingWrite = StreamingAppend
+
+      override def truncate(): WriteBuilder = {
+        assert(writer == Append)
+        writer = TruncateAndAppend
+        streamingWriter = StreamingTruncateAndAppend
+        this
+      }
+
+      override def overwrite(filters: Array[Filter]): WriteBuilder = {
+        assert(writer == Append)
+        writer = new Overwrite(filters)
+        streamingWriter = new StreamingNotSupportedOperation(s"overwrite ($filters)")
+        this
+      }
+
+      override def overwriteDynamicPartitions(): WriteBuilder = {
+        assert(writer == Append)
+        writer = DynamicOverwrite
+        streamingWriter = new StreamingNotSupportedOperation("overwriteDynamicPartitions")
+        this
+      }
+
+      override def build(): Write = new Write with RequiresDistributionAndOrdering {
+        override def requiredDistribution: Distribution = distribution
+
+        override def requiredOrdering: Array[SortOrder] = ordering
+
+        override def requiredNumPartitions(): Int = {
+          numPartitions.getOrElse(0)
+        }
+
+        override def toBatch: BatchWrite = writer
+
+        override def toStreaming: StreamingWrite = streamingWriter match {
+          case exc: StreamingNotSupportedOperation => exc.throwsException()
+          case s => s
+        }
+      }
+    }
+  }
+
+  private abstract class TestBatchWrite extends BatchWrite {
+    override def createBatchWriterFactory(info: PhysicalWriteInfo): DataWriterFactory = {
+      BufferedRowsWriterFactory
+    }
+
+    override def abort(messages: Array[WriterCommitMessage]): Unit = {}
+  }
+
+  private object Append extends TestBatchWrite {
+    override def commit(messages: Array[WriterCommitMessage]): Unit = dataMap.synchronized {
+      withData(messages.map(_.asInstanceOf[BufferedRows]))
+    }
+  }
+
+  private object DynamicOverwrite extends TestBatchWrite {
+    override def commit(messages: Array[WriterCommitMessage]): Unit = dataMap.synchronized {
+      val newData = messages.map(_.asInstanceOf[BufferedRows])
+      dataMap --= newData.flatMap(_.rows.map(getKey))
+      withData(newData)
+    }
+  }
+
+  private class Overwrite(filters: Array[Filter]) extends TestBatchWrite {
+    import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
+    override def commit(messages: Array[WriterCommitMessage]): Unit = dataMap.synchronized {
+      val deleteKeys = InMemoryTable.filtersToKeys(
+        dataMap.keys, partCols.map(_.toSeq.quoted), filters)
+      dataMap --= deleteKeys
+      withData(messages.map(_.asInstanceOf[BufferedRows]))
+    }
+  }
+
+  private object TruncateAndAppend extends TestBatchWrite {
+    override def commit(messages: Array[WriterCommitMessage]): Unit = dataMap.synchronized {
+      dataMap.clear
+      withData(messages.map(_.asInstanceOf[BufferedRows]))
+    }
+  }
+
+  private abstract class TestStreamingWrite extends StreamingWrite {
+    def createStreamingWriterFactory(info: PhysicalWriteInfo): StreamingDataWriterFactory = {
+      BufferedRowsWriterFactory
+    }
+
+    def abort(epochId: Long, messages: Array[WriterCommitMessage]): Unit = {}
+  }
+
+  private class StreamingNotSupportedOperation(operation: String) extends TestStreamingWrite {
+    override def createStreamingWriterFactory(info: PhysicalWriteInfo): StreamingDataWriterFactory =
+      throwsException()
+
+    override def commit(epochId: Long, messages: Array[WriterCommitMessage]): Unit =
+      throwsException()
+
+    override def abort(epochId: Long, messages: Array[WriterCommitMessage]): Unit =
+      throwsException()
+
+    def throwsException[T](): T = throw new IllegalStateException("The operation " +
+      s"${operation} isn't supported for streaming query.")
+  }
+
+  private object StreamingAppend extends TestStreamingWrite {
+    override def commit(epochId: Long, messages: Array[WriterCommitMessage]): Unit = {
+      dataMap.synchronized {
+        withData(messages.map(_.asInstanceOf[BufferedRows]))
+      }
+    }
+  }
+
+  private object StreamingTruncateAndAppend extends TestStreamingWrite {
+    override def commit(epochId: Long, messages: Array[WriterCommitMessage]): Unit = {
+      dataMap.synchronized {
+        dataMap.clear
+        withData(messages.map(_.asInstanceOf[BufferedRows]))
+      }
+    }
+  }
+
+  override def canDeleteWhere(filters: Array[Filter]): Boolean = {
+    InMemoryTable.supportsFilters(filters)
+  }
+
+  override def deleteWhere(filters: Array[Filter]): Unit = dataMap.synchronized {
+    import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
+    dataMap --= InMemoryTable.filtersToKeys(dataMap.keys, partCols.map(_.toSeq.quoted), filters)
+  }
+}
+
+object InMemoryTable {
+  val SIMULATE_FAILED_WRITE_OPTION = "spark.sql.test.simulateFailedWrite"
+
+  def filtersToKeys(
+      keys: Iterable[Seq[Any]],
+      partitionNames: Seq[String],
+      filters: Array[Filter]): Iterable[Seq[Any]] = {
+    keys.filter { partValues =>
+      filters.flatMap(splitAnd).forall {
+        case EqualTo(attr, value) =>
+          value == extractValue(attr, partitionNames, partValues)
+        case EqualNullSafe(attr, value) =>
+          val attrVal = extractValue(attr, partitionNames, partValues)
+          if (attrVal == null && value === null) {
+            true
+          } else if (attrVal == null || value === null) {
+            false
+          } else {
+            value == attrVal
+          }
+        case IsNull(attr) =>
+          null == extractValue(attr, partitionNames, partValues)
+        case IsNotNull(attr) =>
+          null != extractValue(attr, partitionNames, partValues)
+        case AlwaysTrue() => true
+        case f =>
+          throw new IllegalArgumentException(s"Unsupported filter type: $f")
+      }
+    }
+  }
+
+  def supportsFilters(filters: Array[Filter]): Boolean = {
+    filters.flatMap(splitAnd).forall {
+      case _: EqualTo => true
+      case _: EqualNullSafe => true
+      case _: IsNull => true
+      case _: IsNotNull => true
+      case _: AlwaysTrue => true
+      case _ => false
+    }
+  }
+
+  private def extractValue(
+      attr: String,
+      partFieldNames: Seq[String],
+      partValues: Seq[Any]): Any = {
+    partFieldNames.zipWithIndex.find(_._1 == attr) match {
+      case Some((_, partIndex)) =>
+        partValues(partIndex)
+      case _ =>
+        throw new IllegalArgumentException(s"Unknown filter attribute: $attr")
+    }
+  }
+
+  private def splitAnd(filter: Filter): Seq[Filter] = {
+    filter match {
+      case And(left, right) => splitAnd(left) ++ splitAnd(right)
+      case _ => filter :: Nil
+    }
+  }
+
+  def maybeSimulateFailedTableWrite(tableOptions: CaseInsensitiveStringMap): Unit = {
+    if (tableOptions.getBoolean(SIMULATE_FAILED_WRITE_OPTION, false)) {
+      throw new IllegalStateException("Manual write to table failure.")
+    }
+  }
+}
+
+class BufferedRows(
+    val key: String = "") extends WriterCommitMessage with InputPartition with Serializable {
+  val rows = new mutable.ArrayBuffer[InternalRow]()
+
+  def withRow(row: InternalRow): BufferedRows = {
+    rows.append(row)
+    this
+  }
+
+  def clear(): Unit = rows.clear()
+}
+
+private class BufferedRowsReaderFactory(
+    metadataColumnNames: Seq[String],
+    nonMetaDataColumns: Seq[StructField],
+    tableSchema: StructType) extends PartitionReaderFactory {
+  override def createReader(partition: InputPartition): PartitionReader[InternalRow] = {
+    new BufferedRowsReader(partition.asInstanceOf[BufferedRows], metadataColumnNames,
+      nonMetaDataColumns, tableSchema)
+  }
+}
+
+private class BufferedRowsReader(
+    partition: BufferedRows,
+    metadataColumnNames: Seq[String],
+    nonMetadataColumns: Seq[StructField],
+    tableSchema: StructType) extends PartitionReader[InternalRow] {
+  private def addMetadata(row: InternalRow): InternalRow = {
+    val metadataRow = new GenericInternalRow(metadataColumnNames.map {
+      case "index" => index
+      case "_partition" => UTF8String.fromString(partition.key)
+    }.toArray)
+    new JoinedRow(row, metadataRow)
+  }
+
+  private var index: Int = -1
+
+  override def next(): Boolean = {
+    index += 1
+    index < partition.rows.length
+  }
+
+  override def get(): InternalRow = {
+    val originalRow = partition.rows(index)
+    val values = new Array[Any](nonMetadataColumns.length)
+    nonMetadataColumns.zipWithIndex.foreach { case (col, idx) =>
+      values(idx) = extractFieldValue(col, tableSchema, originalRow)
+    }
+    addMetadata(new GenericInternalRow(values))
+  }
+
+  override def close(): Unit = {}
+
+  private def extractFieldValue(
+      field: StructField,
+      schema: StructType,
+      row: InternalRow): Any = {
+    val index = schema.fieldIndex(field.name)
+    field.dataType match {
+      case StructType(fields) =>
+        if (row.isNullAt(index)) {
+          return null
+        }
+        val childRow = row.toSeq(schema)(index).asInstanceOf[InternalRow]
+        val childSchema = schema(index).dataType.asInstanceOf[StructType]
+        val resultValue = new Array[Any](fields.length)
+        fields.zipWithIndex.foreach { case (childField, idx) =>
+          val childValue = extractFieldValue(childField, childSchema, childRow)
+          resultValue(idx) = childValue
+        }
+        new GenericInternalRow(resultValue)
+      case dt =>
+        row.get(index, dt)
+    }
+  }
+}
+
+private object BufferedRowsWriterFactory extends DataWriterFactory with StreamingDataWriterFactory {
+  override def createWriter(partitionId: Int, taskId: Long): DataWriter[InternalRow] = {
+    new BufferWriter
+  }
+
+  override def createWriter(
+      partitionId: Int,
+      taskId: Long,
+      epochId: Long): DataWriter[InternalRow] = {
+    new BufferWriter
+  }
+}
+
+private class BufferWriter extends DataWriter[InternalRow] {
+  private val buffer = new BufferedRows
+
+  override def write(row: InternalRow): Unit = buffer.rows.append(row.copy())
+
+  override def commit(): WriterCommitMessage = buffer
+
+  override def abort(): Unit = {}
+
+  override def close(): Unit = {}
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTableCatalog.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableCatalog.scala
similarity index 84%
rename from sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTableCatalog.scala
rename to sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableCatalog.scala
index 6824efd9880a9..0c403baca2113 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTableCatalog.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableCatalog.scala
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.spark.sql.connector
+package org.apache.spark.sql.connector.catalog
 
 import java.util
 import java.util.concurrent.ConcurrentHashMap
@@ -23,8 +23,8 @@ import java.util.concurrent.ConcurrentHashMap
 import scala.collection.JavaConverters._
 
 import org.apache.spark.sql.catalyst.analysis.{NamespaceAlreadyExistsException, NoSuchNamespaceException, NoSuchTableException, TableAlreadyExistsException}
-import org.apache.spark.sql.connector.catalog._
-import org.apache.spark.sql.connector.expressions.Transform
+import org.apache.spark.sql.connector.distributions.{Distribution, Distributions}
+import org.apache.spark.sql.connector.expressions.{SortOrder, Transform}
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
@@ -69,13 +69,27 @@ class BasicInMemoryTableCatalog extends TableCatalog {
       schema: StructType,
       partitions: Array[Transform],
       properties: util.Map[String, String]): Table = {
+    createTable(ident, schema, partitions, properties, Distributions.unspecified(),
+      Array.empty, None)
+  }
+
+  def createTable(
+      ident: Identifier,
+      schema: StructType,
+      partitions: Array[Transform],
+      properties: util.Map[String, String],
+      distribution: Distribution,
+      ordering: Array[SortOrder],
+      requiredNumPartitions: Option[Int]): Table = {
     if (tables.containsKey(ident)) {
       throw new TableAlreadyExistsException(ident)
     }
 
     InMemoryTableCatalog.maybeSimulateFailedTableCreation(properties)
 
-    val table = new InMemoryTable(s"$name.${ident.quoted}", schema, partitions, properties)
+    val tableName = s"$name.${ident.quoted}"
+    val table = new InMemoryTable(tableName, schema, partitions, properties, distribution,
+      ordering, requiredNumPartitions)
     tables.put(ident, table)
     namespaces.putIfAbsent(ident.namespace.toList, Map())
     table
@@ -124,7 +138,7 @@ class BasicInMemoryTableCatalog extends TableCatalog {
 }
 
 class InMemoryTableCatalog extends BasicInMemoryTableCatalog with SupportsNamespaces {
-  private def allNamespaces: Seq[Seq[String]] = {
+  protected def allNamespaces: Seq[Seq[String]] = {
     (tables.keySet.asScala.map(_.namespace.toSeq) ++ namespaces.keySet.asScala).toSeq.distinct
   }
 
@@ -181,9 +195,21 @@ class InMemoryTableCatalog extends BasicInMemoryTableCatalog with SupportsNamesp
 
   override def dropNamespace(namespace: Array[String]): Boolean = {
     listNamespaces(namespace).foreach(dropNamespace)
-    listTables(namespace).foreach(dropTable)
+    try {
+      listTables(namespace).foreach(dropTable)
+    } catch {
+      case _: NoSuchNamespaceException =>
+    }
     Option(namespaces.remove(namespace.toList)).isDefined
   }
+
+  override def listTables(namespace: Array[String]): Array[Identifier] = {
+    if (namespace.isEmpty || namespaceExists(namespace)) {
+      super.listTables(namespace)
+    } else {
+      throw new NoSuchNamespaceException(namespace)
+    }
+  }
 }
 
 object InMemoryTableCatalog {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/LookupCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/LookupCatalogSuite.scala
index b36ded3d9d2ac..0db758d5147f0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/LookupCatalogSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/LookupCatalogSuite.scala
@@ -20,7 +20,7 @@ import org.mockito.ArgumentMatchers.any
 import org.mockito.Mockito.{mock, when}
 import org.mockito.invocation.InvocationOnMock
 import org.scalatest.Inside
-import org.scalatest.Matchers._
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.TableIdentifier
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/StagingInMemoryTableCatalog.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/StagingInMemoryTableCatalog.scala
similarity index 94%
rename from sql/catalyst/src/test/scala/org/apache/spark/sql/connector/StagingInMemoryTableCatalog.scala
rename to sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/StagingInMemoryTableCatalog.scala
index 6d4879d355375..954650ae0eebd 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/StagingInMemoryTableCatalog.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/StagingInMemoryTableCatalog.scala
@@ -15,15 +15,15 @@
  * limitations under the License.
  */
 
-package org.apache.spark.sql.connector
+package org.apache.spark.sql.connector.catalog
 
 import java.util
 
-import org.apache.spark.sql.catalyst.analysis.{CannotReplaceMissingTableException, TableAlreadyExistsException}
-import org.apache.spark.sql.connector.catalog._
+import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
 import org.apache.spark.sql.connector.expressions.Transform
 import org.apache.spark.sql.connector.read.ScanBuilder
 import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder}
+import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
@@ -118,7 +118,7 @@ class StagingInMemoryTableCatalog extends InMemoryTableCatalog with StagingTable
       maybeSimulateDropBeforeCommit()
       val maybePreCommittedTable = tables.replace(ident, delegateTable)
       if (maybePreCommittedTable == null) {
-        throw new CannotReplaceMissingTableException(ident)
+        throw QueryCompilationErrors.cannotReplaceMissingTableError(ident)
       }
     }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/SupportsAtomicPartitionManagementSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/SupportsAtomicPartitionManagementSuite.scala
new file mode 100644
index 0000000000000..df2fbd6d179bb
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/SupportsAtomicPartitionManagementSuite.scala
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.catalog
+
+import java.util
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionException, PartitionsAlreadyExistException}
+import org.apache.spark.sql.connector.expressions.{LogicalExpressions, NamedReference}
+import org.apache.spark.sql.types.{IntegerType, StringType, StructType}
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
+
+class SupportsAtomicPartitionManagementSuite extends SparkFunSuite {
+
+  private val ident: Identifier = Identifier.of(Array("ns"), "test_table")
+
+  def ref(name: String): NamedReference = LogicalExpressions.parseReference(name)
+
+  private val catalog: InMemoryTableCatalog = {
+    val newCatalog = new InMemoryTableCatalog
+    newCatalog.initialize("test", CaseInsensitiveStringMap.empty())
+    newCatalog.createTable(
+      ident,
+      new StructType()
+        .add("id", IntegerType)
+        .add("data", StringType)
+        .add("dt", StringType),
+      Array(LogicalExpressions.identity(ref("dt"))),
+      util.Collections.emptyMap[String, String])
+    newCatalog
+  }
+
+  private def hasPartitions(table: SupportsPartitionManagement): Boolean = {
+    !table.listPartitionIdentifiers(Array.empty, InternalRow.empty).isEmpty
+  }
+
+  test("createPartitions") {
+    val table = catalog.loadTable(ident)
+    val partTable = new InMemoryAtomicPartitionTable(
+      table.name(), table.schema(), table.partitioning(), table.properties())
+    assert(!hasPartitions(partTable))
+
+    val partIdents = Array(InternalRow.apply("3"), InternalRow.apply("4"))
+    partTable.createPartitions(
+      partIdents,
+      Array(new util.HashMap[String, String](), new util.HashMap[String, String]()))
+    assert(hasPartitions(partTable))
+    assert(partTable.partitionExists(InternalRow.apply("3")))
+    assert(partTable.partitionExists(InternalRow.apply("4")))
+
+    partTable.dropPartition(InternalRow.apply("3"))
+    partTable.dropPartition(InternalRow.apply("4"))
+    assert(!hasPartitions(partTable))
+  }
+
+  test("createPartitions failed if partition already exists") {
+    val table = catalog.loadTable(ident)
+    val partTable = new InMemoryAtomicPartitionTable(
+      table.name(), table.schema(), table.partitioning(), table.properties())
+    assert(!hasPartitions(partTable))
+
+    val partIdent = InternalRow.apply("4")
+    partTable.createPartition(partIdent, new util.HashMap[String, String]())
+    assert(hasPartitions(partTable))
+    assert(partTable.partitionExists(partIdent))
+
+    val partIdents = Array(InternalRow.apply("3"), InternalRow.apply("4"))
+    assertThrows[PartitionsAlreadyExistException](
+      partTable.createPartitions(
+        partIdents,
+        Array(new util.HashMap[String, String](), new util.HashMap[String, String]())))
+    assert(!partTable.partitionExists(InternalRow.apply("3")))
+
+    partTable.dropPartition(partIdent)
+    assert(!hasPartitions(partTable))
+  }
+
+  test("dropPartitions") {
+    val table = catalog.loadTable(ident)
+    val partTable = new InMemoryAtomicPartitionTable(
+      table.name(), table.schema(), table.partitioning(), table.properties())
+    assert(!hasPartitions(partTable))
+
+    val partIdents = Array(InternalRow.apply("3"), InternalRow.apply("4"))
+    partTable.createPartitions(
+      partIdents,
+      Array(new util.HashMap[String, String](), new util.HashMap[String, String]()))
+    assert(hasPartitions(partTable))
+    assert(partTable.partitionExists(InternalRow.apply("3")))
+    assert(partTable.partitionExists(InternalRow.apply("4")))
+
+    partTable.dropPartitions(partIdents)
+    assert(!hasPartitions(partTable))
+  }
+
+  test("purgePartitions") {
+    val table = catalog.loadTable(ident)
+    val partTable = new InMemoryAtomicPartitionTable(
+      table.name(), table.schema(), table.partitioning(), table.properties())
+    val partIdents = Array(InternalRow.apply("3"), InternalRow.apply("4"))
+    partTable.createPartitions(
+      partIdents,
+      Array(new util.HashMap[String, String](), new util.HashMap[String, String]()))
+    val errMsg = intercept[UnsupportedOperationException] {
+      partTable.purgePartitions(partIdents)
+    }.getMessage
+    assert(errMsg.contains("purge is not supported"))
+  }
+
+  test("dropPartitions failed if partition not exists") {
+    val table = catalog.loadTable(ident)
+    val partTable = new InMemoryAtomicPartitionTable(
+      table.name(), table.schema(), table.partitioning(), table.properties())
+    assert(!hasPartitions(partTable))
+
+    val partIdent = InternalRow.apply("4")
+    partTable.createPartition(partIdent, new util.HashMap[String, String]())
+    assert(partTable.listPartitionIdentifiers(Array.empty, InternalRow.empty).length == 1)
+
+    val partIdents = Array(InternalRow.apply("3"), InternalRow.apply("4"))
+    assert(!partTable.dropPartitions(partIdents))
+    assert(partTable.partitionExists(partIdent))
+
+    partTable.dropPartition(partIdent)
+    assert(!hasPartitions(partTable))
+  }
+
+  test("truncatePartitions") {
+    val table = catalog.loadTable(ident)
+    val partTable = new InMemoryAtomicPartitionTable(
+      table.name(), table.schema(), table.partitioning(), table.properties())
+    assert(!hasPartitions(partTable))
+
+    partTable.createPartitions(
+      Array(InternalRow("3"), InternalRow("4"), InternalRow("5")),
+      Array.tabulate(3)(_ => new util.HashMap[String, String]()))
+    assert(partTable.listPartitionIdentifiers(Array.empty, InternalRow.empty).length == 3)
+
+    partTable.withData(Array(
+      new BufferedRows("3").withRow(InternalRow(0, "abc", "3")),
+      new BufferedRows("4").withRow(InternalRow(1, "def", "4")),
+      new BufferedRows("5").withRow(InternalRow(2, "zyx", "5"))
+    ))
+
+    partTable.truncatePartitions(Array(InternalRow("3"), InternalRow("4")))
+    assert(partTable.listPartitionIdentifiers(Array.empty, InternalRow.empty).length == 3)
+    assert(partTable.rows === InternalRow(2, "zyx", "5") :: Nil)
+
+    // Truncate non-existing partition
+    val errMsg = intercept[NoSuchPartitionException] {
+      partTable.truncatePartitions(Array(InternalRow("5"), InternalRow("6")))
+    }.getMessage
+    assert(errMsg.contains("Partition not found in table test.ns.test_table: 6 -> dt"))
+    assert(partTable.rows === InternalRow(2, "zyx", "5") :: Nil)
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/SupportsPartitionManagementSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/SupportsPartitionManagementSuite.scala
new file mode 100644
index 0000000000000..e5aeb90b841a6
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/SupportsPartitionManagementSuite.scala
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.catalog
+
+import java.util
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionException, PartitionAlreadyExistsException}
+import org.apache.spark.sql.connector.expressions.{LogicalExpressions, NamedReference}
+import org.apache.spark.sql.types.{IntegerType, StringType, StructType}
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
+
+class SupportsPartitionManagementSuite extends SparkFunSuite {
+
+  private val ident: Identifier = Identifier.of(Array("ns"), "test_table")
+
+  def ref(name: String): NamedReference = LogicalExpressions.parseReference(name)
+
+  private val catalog: InMemoryTableCatalog = {
+    val newCatalog = new InMemoryTableCatalog
+    newCatalog.initialize("test", CaseInsensitiveStringMap.empty())
+    newCatalog.createTable(
+      ident,
+      new StructType()
+        .add("id", IntegerType)
+        .add("data", StringType)
+        .add("dt", StringType),
+      Array(LogicalExpressions.identity(ref("dt"))),
+      util.Collections.emptyMap[String, String])
+    newCatalog
+  }
+
+  private def hasPartitions(table: SupportsPartitionManagement): Boolean = {
+    !table.listPartitionIdentifiers(Array.empty, InternalRow.empty).isEmpty
+  }
+
+  test("createPartition") {
+    val table = catalog.loadTable(ident)
+    val partTable = new InMemoryPartitionTable(
+      table.name(), table.schema(), table.partitioning(), table.properties())
+    assert(!hasPartitions(partTable))
+
+    val partIdent = InternalRow.apply("3")
+    partTable.createPartition(partIdent, new util.HashMap[String, String]())
+    assert(hasPartitions(partTable))
+    assert(partTable.partitionExists(partIdent))
+
+    partTable.dropPartition(partIdent)
+    assert(!hasPartitions(partTable))
+  }
+
+  test("dropPartition") {
+    val table = catalog.loadTable(ident)
+    val partTable = new InMemoryPartitionTable(
+      table.name(), table.schema(), table.partitioning(), table.properties())
+    assert(!hasPartitions(partTable))
+
+    val partIdent = InternalRow.apply("3")
+    val partIdent1 = InternalRow.apply("4")
+    partTable.createPartition(partIdent, new util.HashMap[String, String]())
+    partTable.createPartition(partIdent1, new util.HashMap[String, String]())
+    assert(partTable.listPartitionIdentifiers(Array.empty, InternalRow.empty).length == 2)
+
+    partTable.dropPartition(partIdent)
+    assert(partTable.listPartitionIdentifiers(Array.empty, InternalRow.empty).length == 1)
+    partTable.dropPartition(partIdent1)
+    assert(!hasPartitions(partTable))
+  }
+
+  test("purgePartition") {
+    val table = catalog.loadTable(ident)
+    val partTable = new InMemoryPartitionTable(
+      table.name(), table.schema(), table.partitioning(), table.properties())
+    val errMsg = intercept[UnsupportedOperationException] {
+      partTable.purgePartition(InternalRow.apply("3"))
+    }.getMessage
+    assert(errMsg.contains("purge is not supported"))
+  }
+
+  test("replacePartitionMetadata") {
+    val table = catalog.loadTable(ident)
+    val partTable = new InMemoryPartitionTable(
+      table.name(), table.schema(), table.partitioning(), table.properties())
+    assert(!hasPartitions(partTable))
+
+    val partIdent = InternalRow.apply("3")
+    partTable.createPartition(partIdent, new util.HashMap[String, String]())
+    assert(hasPartitions(partTable))
+    assert(partTable.partitionExists(partIdent))
+    assert(partTable.loadPartitionMetadata(partIdent).isEmpty)
+
+    partTable.replacePartitionMetadata(partIdent, Map("paramKey" -> "paramValue").asJava)
+    assert(hasPartitions(partTable))
+    assert(partTable.partitionExists(partIdent))
+    assert(!partTable.loadPartitionMetadata(partIdent).isEmpty)
+    assert(partTable.loadPartitionMetadata(partIdent).get("paramKey") == "paramValue")
+
+    partTable.dropPartition(partIdent)
+    assert(!hasPartitions(partTable))
+  }
+
+  test("loadPartitionMetadata") {
+    val table = catalog.loadTable(ident)
+    val partTable = new InMemoryPartitionTable(
+      table.name(), table.schema(), table.partitioning(), table.properties())
+    assert(!hasPartitions(partTable))
+
+    val partIdent = InternalRow.apply("3")
+    partTable.createPartition(partIdent, Map("paramKey" -> "paramValue").asJava)
+    assert(hasPartitions(partTable))
+    assert(partTable.partitionExists(partIdent))
+    assert(!partTable.loadPartitionMetadata(partIdent).isEmpty)
+    assert(partTable.loadPartitionMetadata(partIdent).get("paramKey") == "paramValue")
+
+    partTable.dropPartition(partIdent)
+    assert(!hasPartitions(partTable))
+  }
+
+  test("listPartitionIdentifiers") {
+    val table = catalog.loadTable(ident)
+    val partTable = new InMemoryPartitionTable(
+      table.name(), table.schema(), table.partitioning(), table.properties())
+    assert(!hasPartitions(partTable))
+
+    val partIdent = InternalRow.apply("3")
+    partTable.createPartition(partIdent, new util.HashMap[String, String]())
+    assert(partTable.listPartitionIdentifiers(Array.empty, InternalRow.empty).length == 1)
+
+    val partIdent1 = InternalRow.apply("4")
+    partTable.createPartition(partIdent1, new util.HashMap[String, String]())
+    assert(partTable.listPartitionIdentifiers(Array.empty, InternalRow.empty).length == 2)
+    assert(partTable.listPartitionIdentifiers(Array("dt"), partIdent1).length == 1)
+
+    partTable.dropPartition(partIdent)
+    assert(partTable.listPartitionIdentifiers(Array.empty, InternalRow.empty).length == 1)
+    partTable.dropPartition(partIdent1)
+    assert(!hasPartitions(partTable))
+  }
+
+  private def createMultiPartTable(): InMemoryPartitionTable = {
+    val partCatalog = new InMemoryPartitionTableCatalog
+    partCatalog.initialize("test", CaseInsensitiveStringMap.empty())
+    val table = partCatalog.createTable(
+      ident,
+      new StructType()
+        .add("col0", IntegerType)
+        .add("part0", IntegerType)
+        .add("part1", StringType),
+      Array(LogicalExpressions.identity(ref("part0")), LogicalExpressions.identity(ref("part1"))),
+      util.Collections.emptyMap[String, String])
+
+    val partTable = table.asInstanceOf[InMemoryPartitionTable]
+    Seq(
+      InternalRow(0, "abc"),
+      InternalRow(0, "def"),
+      InternalRow(1, "abc")).foreach { partIdent =>
+      partTable.createPartition(partIdent, new util.HashMap[String, String]())
+    }
+
+    partTable
+  }
+
+  test("listPartitionByNames") {
+    val partTable = createMultiPartTable()
+
+    Seq(
+      (Array("part0", "part1"), InternalRow(0, "abc")) -> Set(InternalRow(0, "abc")),
+      (Array("part0"), InternalRow(0)) -> Set(InternalRow(0, "abc"), InternalRow(0, "def")),
+      (Array("part1"), InternalRow("abc")) -> Set(InternalRow(0, "abc"), InternalRow(1, "abc")),
+      (Array.empty[String], InternalRow.empty) ->
+        Set(InternalRow(0, "abc"), InternalRow(0, "def"), InternalRow(1, "abc")),
+      (Array("part0", "part1"), InternalRow(3, "xyz")) -> Set(),
+      (Array("part1"), InternalRow(3.14f)) -> Set()
+    ).foreach { case ((names, idents), expected) =>
+      assert(partTable.listPartitionIdentifiers(names, idents).toSet === expected)
+    }
+    // Check invalid parameters
+    Seq(
+      (Array("part0", "part1"), InternalRow(0)),
+      (Array("col0", "part1"), InternalRow(0, 1)),
+      (Array("wrong"), InternalRow("invalid"))
+    ).foreach { case (names, idents) =>
+      intercept[AssertionError](partTable.listPartitionIdentifiers(names, idents))
+    }
+  }
+
+  test("partitionExists") {
+    val partTable = createMultiPartTable()
+
+    assert(partTable.partitionExists(InternalRow(0, "def")))
+    assert(!partTable.partitionExists(InternalRow(-1, "def")))
+    assert(!partTable.partitionExists(InternalRow("abc", "def")))
+
+    val errMsg = intercept[IllegalArgumentException] {
+      partTable.partitionExists(InternalRow(0))
+    }.getMessage
+    assert(errMsg.contains("The identifier might not refer to one partition"))
+  }
+
+  test("renamePartition") {
+    val partTable = createMultiPartTable()
+
+    val errMsg1 = intercept[PartitionAlreadyExistsException] {
+      partTable.renamePartition(InternalRow(0, "abc"), InternalRow(1, "abc"))
+    }.getMessage
+    assert(errMsg1.contains("Partition already exists"))
+
+    val newPart = InternalRow(2, "xyz")
+    val errMsg2 = intercept[NoSuchPartitionException] {
+      partTable.renamePartition(newPart, InternalRow(3, "abc"))
+    }.getMessage
+    assert(errMsg2.contains("Partition not found"))
+
+    assert(partTable.renamePartition(InternalRow(0, "abc"), newPart))
+    assert(partTable.partitionExists(newPart))
+  }
+
+  test("truncatePartition") {
+    val table = catalog.loadTable(ident)
+    val partTable = new InMemoryPartitionTable(
+      table.name(), table.schema(), table.partitioning(), table.properties())
+    assert(!hasPartitions(partTable))
+
+    val partIdent = InternalRow.apply("3")
+    val partIdent1 = InternalRow.apply("4")
+    partTable.createPartition(partIdent, new util.HashMap[String, String]())
+    partTable.createPartition(partIdent1, new util.HashMap[String, String]())
+    assert(partTable.listPartitionIdentifiers(Array.empty, InternalRow.empty).length == 2)
+
+    partTable.withData(Array(
+      new BufferedRows("3").withRow(InternalRow(0, "abc", "3")),
+      new BufferedRows("4").withRow(InternalRow(1, "def", "4"))
+    ))
+
+    partTable.truncatePartition(partIdent)
+    assert(partTable.listPartitionIdentifiers(Array.empty, InternalRow.empty).length == 2)
+    assert(partTable.rows === InternalRow(1, "def", "4") :: Nil)
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/TableCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/TableCatalogSuite.scala
deleted file mode 100644
index a4c85ec64ecf6..0000000000000
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/TableCatalogSuite.scala
+++ /dev/null
@@ -1,885 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.connector.catalog
-
-import java.util
-import java.util.Collections
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.catalyst.analysis.{NamespaceAlreadyExistsException, NoSuchNamespaceException, NoSuchTableException, TableAlreadyExistsException}
-import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
-import org.apache.spark.sql.connector.InMemoryTableCatalog
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{DoubleType, IntegerType, LongType, StringType, StructField, StructType, TimestampType}
-import org.apache.spark.sql.util.CaseInsensitiveStringMap
-
-class TableCatalogSuite extends SparkFunSuite {
-  import CatalogV2Implicits._
-
-  private val emptyProps: util.Map[String, String] = Collections.emptyMap[String, String]
-  private val schema: StructType = new StructType()
-      .add("id", IntegerType)
-      .add("data", StringType)
-
-  private def newCatalog(): TableCatalog with SupportsNamespaces = {
-    val newCatalog = new InMemoryTableCatalog
-    newCatalog.initialize("test", CaseInsensitiveStringMap.empty())
-    newCatalog
-  }
-
-  private val testNs = Array("`", ".")
-  private val testIdent = Identifier.of(testNs, "test_table")
-  private val testIdentNew = Identifier.of(testNs, "test_table_new")
-
-  test("Catalogs can load the catalog") {
-    val catalog = newCatalog()
-
-    val conf = new SQLConf
-    conf.setConfString("spark.sql.catalog.test", catalog.getClass.getName)
-
-    val loaded = Catalogs.load("test", conf)
-    assert(loaded.getClass == catalog.getClass)
-  }
-
-  test("listTables") {
-    val catalog = newCatalog()
-    val ident1 = Identifier.of(Array("ns"), "test_table_1")
-    val ident2 = Identifier.of(Array("ns"), "test_table_2")
-    val ident3 = Identifier.of(Array("ns2"), "test_table_1")
-
-    assert(catalog.listTables(Array("ns")).isEmpty)
-
-    catalog.createTable(ident1, schema, Array.empty, emptyProps)
-
-    assert(catalog.listTables(Array("ns")).toSet == Set(ident1))
-    assert(catalog.listTables(Array("ns2")).isEmpty)
-
-    catalog.createTable(ident3, schema, Array.empty, emptyProps)
-    catalog.createTable(ident2, schema, Array.empty, emptyProps)
-
-    assert(catalog.listTables(Array("ns")).toSet == Set(ident1, ident2))
-    assert(catalog.listTables(Array("ns2")).toSet == Set(ident3))
-
-    catalog.dropTable(ident1)
-
-    assert(catalog.listTables(Array("ns")).toSet == Set(ident2))
-
-    catalog.dropTable(ident2)
-
-    assert(catalog.listTables(Array("ns")).isEmpty)
-    assert(catalog.listTables(Array("ns2")).toSet == Set(ident3))
-  }
-
-  test("createTable") {
-    val catalog = newCatalog()
-
-    assert(!catalog.tableExists(testIdent))
-
-    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    val parsed = CatalystSqlParser.parseMultipartIdentifier(table.name)
-    assert(parsed == Seq("test", "`", ".", "test_table"))
-    assert(table.schema == schema)
-    assert(table.properties.asScala == Map())
-
-    assert(catalog.tableExists(testIdent))
-  }
-
-  test("createTable: with properties") {
-    val catalog = newCatalog()
-
-    val properties = new util.HashMap[String, String]()
-    properties.put("property", "value")
-
-    assert(!catalog.tableExists(testIdent))
-
-    val table = catalog.createTable(testIdent, schema, Array.empty, properties)
-
-    val parsed = CatalystSqlParser.parseMultipartIdentifier(table.name)
-    assert(parsed == Seq("test", "`", ".", "test_table"))
-    assert(table.schema == schema)
-    assert(table.properties == properties)
-
-    assert(catalog.tableExists(testIdent))
-  }
-
-  test("createTable: table already exists") {
-    val catalog = newCatalog()
-
-    assert(!catalog.tableExists(testIdent))
-
-    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    val exc = intercept[TableAlreadyExistsException] {
-      catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-    }
-
-    assert(exc.message.contains(testIdent.quoted))
-    assert(exc.message.contains("already exists"))
-
-    assert(catalog.tableExists(testIdent))
-  }
-
-  test("tableExists") {
-    val catalog = newCatalog()
-
-    assert(!catalog.tableExists(testIdent))
-
-    catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    assert(catalog.tableExists(testIdent))
-
-    catalog.dropTable(testIdent)
-
-    assert(!catalog.tableExists(testIdent))
-  }
-
-  test("loadTable") {
-    val catalog = newCatalog()
-
-    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-    val loaded = catalog.loadTable(testIdent)
-
-    assert(table.name == loaded.name)
-    assert(table.schema == loaded.schema)
-    assert(table.properties == loaded.properties)
-  }
-
-  test("loadTable: table does not exist") {
-    val catalog = newCatalog()
-
-    val exc = intercept[NoSuchTableException] {
-      catalog.loadTable(testIdent)
-    }
-
-    assert(exc.message.contains(testIdent.quoted))
-    assert(exc.message.contains("not found"))
-  }
-
-  test("invalidateTable") {
-    val catalog = newCatalog()
-
-    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-    catalog.invalidateTable(testIdent)
-
-    val loaded = catalog.loadTable(testIdent)
-
-    assert(table.name == loaded.name)
-    assert(table.schema == loaded.schema)
-    assert(table.properties == loaded.properties)
-  }
-
-  test("invalidateTable: table does not exist") {
-    val catalog = newCatalog()
-
-    assert(catalog.tableExists(testIdent) === false)
-
-    catalog.invalidateTable(testIdent)
-  }
-
-  test("alterTable: add property") {
-    val catalog = newCatalog()
-
-    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    assert(table.properties.asScala == Map())
-
-    val updated = catalog.alterTable(testIdent, TableChange.setProperty("prop-1", "1"))
-    assert(updated.properties.asScala == Map("prop-1" -> "1"))
-
-    val loaded = catalog.loadTable(testIdent)
-    assert(loaded.properties.asScala == Map("prop-1" -> "1"))
-
-    assert(table.properties.asScala == Map())
-  }
-
-  test("alterTable: add property to existing") {
-    val catalog = newCatalog()
-
-    val properties = new util.HashMap[String, String]()
-    properties.put("prop-1", "1")
-
-    val table = catalog.createTable(testIdent, schema, Array.empty, properties)
-
-    assert(table.properties.asScala == Map("prop-1" -> "1"))
-
-    val updated = catalog.alterTable(testIdent, TableChange.setProperty("prop-2", "2"))
-    assert(updated.properties.asScala == Map("prop-1" -> "1", "prop-2" -> "2"))
-
-    val loaded = catalog.loadTable(testIdent)
-    assert(loaded.properties.asScala == Map("prop-1" -> "1", "prop-2" -> "2"))
-
-    assert(table.properties.asScala == Map("prop-1" -> "1"))
-  }
-
-  test("alterTable: remove existing property") {
-    val catalog = newCatalog()
-
-    val properties = new util.HashMap[String, String]()
-    properties.put("prop-1", "1")
-
-    val table = catalog.createTable(testIdent, schema, Array.empty, properties)
-
-    assert(table.properties.asScala == Map("prop-1" -> "1"))
-
-    val updated = catalog.alterTable(testIdent, TableChange.removeProperty("prop-1"))
-    assert(updated.properties.asScala == Map())
-
-    val loaded = catalog.loadTable(testIdent)
-    assert(loaded.properties.asScala == Map())
-
-    assert(table.properties.asScala == Map("prop-1" -> "1"))
-  }
-
-  test("alterTable: remove missing property") {
-    val catalog = newCatalog()
-
-    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    assert(table.properties.asScala == Map())
-
-    val updated = catalog.alterTable(testIdent, TableChange.removeProperty("prop-1"))
-    assert(updated.properties.asScala == Map())
-
-    val loaded = catalog.loadTable(testIdent)
-    assert(loaded.properties.asScala == Map())
-
-    assert(table.properties.asScala == Map())
-  }
-
-  test("alterTable: add top-level column") {
-    val catalog = newCatalog()
-
-    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    assert(table.schema == schema)
-
-    val updated = catalog.alterTable(testIdent, TableChange.addColumn(Array("ts"), TimestampType))
-
-    assert(updated.schema == schema.add("ts", TimestampType))
-  }
-
-  test("alterTable: add required column") {
-    val catalog = newCatalog()
-
-    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    assert(table.schema == schema)
-
-    val updated = catalog.alterTable(testIdent,
-      TableChange.addColumn(Array("ts"), TimestampType, false))
-
-    assert(updated.schema == schema.add("ts", TimestampType, nullable = false))
-  }
-
-  test("alterTable: add column with comment") {
-    val catalog = newCatalog()
-
-    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    assert(table.schema == schema)
-
-    val updated = catalog.alterTable(testIdent,
-      TableChange.addColumn(Array("ts"), TimestampType, false, "comment text"))
-
-    val field = StructField("ts", TimestampType, nullable = false).withComment("comment text")
-    assert(updated.schema == schema.add(field))
-  }
-
-  test("alterTable: add nested column") {
-    val catalog = newCatalog()
-
-    val pointStruct = new StructType().add("x", DoubleType).add("y", DoubleType)
-    val tableSchema = schema.add("point", pointStruct)
-
-    val table = catalog.createTable(testIdent, tableSchema, Array.empty, emptyProps)
-
-    assert(table.schema == tableSchema)
-
-    val updated = catalog.alterTable(testIdent,
-      TableChange.addColumn(Array("point", "z"), DoubleType))
-
-    val expectedSchema = schema.add("point", pointStruct.add("z", DoubleType))
-
-    assert(updated.schema == expectedSchema)
-  }
-
-  test("alterTable: add column to primitive field fails") {
-    val catalog = newCatalog()
-
-    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    assert(table.schema == schema)
-
-    val exc = intercept[IllegalArgumentException] {
-      catalog.alterTable(testIdent, TableChange.addColumn(Array("data", "ts"), TimestampType))
-    }
-
-    assert(exc.getMessage.contains("Not a struct"))
-    assert(exc.getMessage.contains("data"))
-
-    // the table has not changed
-    assert(catalog.loadTable(testIdent).schema == schema)
-  }
-
-  test("alterTable: add field to missing column fails") {
-    val catalog = newCatalog()
-
-    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    assert(table.schema == schema)
-
-    val exc = intercept[IllegalArgumentException] {
-      catalog.alterTable(testIdent,
-        TableChange.addColumn(Array("missing_col", "new_field"), StringType))
-    }
-
-    assert(exc.getMessage.contains("missing_col"))
-    assert(exc.getMessage.contains("Cannot find"))
-  }
-
-  test("alterTable: update column data type") {
-    val catalog = newCatalog()
-
-    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    assert(table.schema == schema)
-
-    val updated = catalog.alterTable(testIdent, TableChange.updateColumnType(Array("id"), LongType))
-
-    val expectedSchema = new StructType().add("id", LongType).add("data", StringType)
-    assert(updated.schema == expectedSchema)
-  }
-
-  test("alterTable: update column nullability") {
-    val catalog = newCatalog()
-
-    val originalSchema = new StructType()
-        .add("id", IntegerType, nullable = false)
-        .add("data", StringType)
-    val table = catalog.createTable(testIdent, originalSchema, Array.empty, emptyProps)
-
-    assert(table.schema == originalSchema)
-
-    val updated = catalog.alterTable(testIdent,
-      TableChange.updateColumnNullability(Array("id"), true))
-
-    val expectedSchema = new StructType().add("id", IntegerType).add("data", StringType)
-    assert(updated.schema == expectedSchema)
-  }
-
-  test("alterTable: update missing column fails") {
-    val catalog = newCatalog()
-
-    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    assert(table.schema == schema)
-
-    val exc = intercept[IllegalArgumentException] {
-      catalog.alterTable(testIdent,
-        TableChange.updateColumnType(Array("missing_col"), LongType))
-    }
-
-    assert(exc.getMessage.contains("missing_col"))
-    assert(exc.getMessage.contains("Cannot find"))
-  }
-
-  test("alterTable: add comment") {
-    val catalog = newCatalog()
-
-    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    assert(table.schema == schema)
-
-    val updated = catalog.alterTable(testIdent,
-      TableChange.updateColumnComment(Array("id"), "comment text"))
-
-    val expectedSchema = new StructType()
-        .add("id", IntegerType, nullable = true, "comment text")
-        .add("data", StringType)
-    assert(updated.schema == expectedSchema)
-  }
-
-  test("alterTable: replace comment") {
-    val catalog = newCatalog()
-
-    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    assert(table.schema == schema)
-
-    catalog.alterTable(testIdent, TableChange.updateColumnComment(Array("id"), "comment text"))
-
-    val expectedSchema = new StructType()
-        .add("id", IntegerType, nullable = true, "replacement comment")
-        .add("data", StringType)
-
-    val updated = catalog.alterTable(testIdent,
-      TableChange.updateColumnComment(Array("id"), "replacement comment"))
-
-    assert(updated.schema == expectedSchema)
-  }
-
-  test("alterTable: add comment to missing column fails") {
-    val catalog = newCatalog()
-
-    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    assert(table.schema == schema)
-
-    val exc = intercept[IllegalArgumentException] {
-      catalog.alterTable(testIdent,
-        TableChange.updateColumnComment(Array("missing_col"), "comment"))
-    }
-
-    assert(exc.getMessage.contains("missing_col"))
-    assert(exc.getMessage.contains("Cannot find"))
-  }
-
-  test("alterTable: rename top-level column") {
-    val catalog = newCatalog()
-
-    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    assert(table.schema == schema)
-
-    val updated = catalog.alterTable(testIdent, TableChange.renameColumn(Array("id"), "some_id"))
-
-    val expectedSchema = new StructType().add("some_id", IntegerType).add("data", StringType)
-
-    assert(updated.schema == expectedSchema)
-  }
-
-  test("alterTable: rename nested column") {
-    val catalog = newCatalog()
-
-    val pointStruct = new StructType().add("x", DoubleType).add("y", DoubleType)
-    val tableSchema = schema.add("point", pointStruct)
-
-    val table = catalog.createTable(testIdent, tableSchema, Array.empty, emptyProps)
-
-    assert(table.schema == tableSchema)
-
-    val updated = catalog.alterTable(testIdent,
-      TableChange.renameColumn(Array("point", "x"), "first"))
-
-    val newPointStruct = new StructType().add("first", DoubleType).add("y", DoubleType)
-    val expectedSchema = schema.add("point", newPointStruct)
-
-    assert(updated.schema == expectedSchema)
-  }
-
-  test("alterTable: rename struct column") {
-    val catalog = newCatalog()
-
-    val pointStruct = new StructType().add("x", DoubleType).add("y", DoubleType)
-    val tableSchema = schema.add("point", pointStruct)
-
-    val table = catalog.createTable(testIdent, tableSchema, Array.empty, emptyProps)
-
-    assert(table.schema == tableSchema)
-
-    val updated = catalog.alterTable(testIdent,
-      TableChange.renameColumn(Array("point"), "p"))
-
-    val newPointStruct = new StructType().add("x", DoubleType).add("y", DoubleType)
-    val expectedSchema = schema.add("p", newPointStruct)
-
-    assert(updated.schema == expectedSchema)
-  }
-
-  test("alterTable: rename missing column fails") {
-    val catalog = newCatalog()
-
-    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    assert(table.schema == schema)
-
-    val exc = intercept[IllegalArgumentException] {
-      catalog.alterTable(testIdent,
-        TableChange.renameColumn(Array("missing_col"), "new_name"))
-    }
-
-    assert(exc.getMessage.contains("missing_col"))
-    assert(exc.getMessage.contains("Cannot find"))
-  }
-
-  test("alterTable: multiple changes") {
-    val catalog = newCatalog()
-
-    val pointStruct = new StructType().add("x", DoubleType).add("y", DoubleType)
-    val tableSchema = schema.add("point", pointStruct)
-
-    val table = catalog.createTable(testIdent, tableSchema, Array.empty, emptyProps)
-
-    assert(table.schema == tableSchema)
-
-    val updated = catalog.alterTable(testIdent,
-      TableChange.renameColumn(Array("point", "x"), "first"),
-      TableChange.renameColumn(Array("point", "y"), "second"))
-
-    val newPointStruct = new StructType().add("first", DoubleType).add("second", DoubleType)
-    val expectedSchema = schema.add("point", newPointStruct)
-
-    assert(updated.schema == expectedSchema)
-  }
-
-  test("alterTable: delete top-level column") {
-    val catalog = newCatalog()
-
-    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    assert(table.schema == schema)
-
-    val updated = catalog.alterTable(testIdent,
-      TableChange.deleteColumn(Array("id")))
-
-    val expectedSchema = new StructType().add("data", StringType)
-    assert(updated.schema == expectedSchema)
-  }
-
-  test("alterTable: delete nested column") {
-    val catalog = newCatalog()
-
-    val pointStruct = new StructType().add("x", DoubleType).add("y", DoubleType)
-    val tableSchema = schema.add("point", pointStruct)
-
-    val table = catalog.createTable(testIdent, tableSchema, Array.empty, emptyProps)
-
-    assert(table.schema == tableSchema)
-
-    val updated = catalog.alterTable(testIdent,
-      TableChange.deleteColumn(Array("point", "y")))
-
-    val newPointStruct = new StructType().add("x", DoubleType)
-    val expectedSchema = schema.add("point", newPointStruct)
-
-    assert(updated.schema == expectedSchema)
-  }
-
-  test("alterTable: delete missing column fails") {
-    val catalog = newCatalog()
-
-    val table = catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    assert(table.schema == schema)
-
-    val exc = intercept[IllegalArgumentException] {
-      catalog.alterTable(testIdent, TableChange.deleteColumn(Array("missing_col")))
-    }
-
-    assert(exc.getMessage.contains("missing_col"))
-    assert(exc.getMessage.contains("Cannot find"))
-  }
-
-  test("alterTable: delete missing nested column fails") {
-    val catalog = newCatalog()
-
-    val pointStruct = new StructType().add("x", DoubleType).add("y", DoubleType)
-    val tableSchema = schema.add("point", pointStruct)
-
-    val table = catalog.createTable(testIdent, tableSchema, Array.empty, emptyProps)
-
-    assert(table.schema == tableSchema)
-
-    val exc = intercept[IllegalArgumentException] {
-      catalog.alterTable(testIdent, TableChange.deleteColumn(Array("point", "z")))
-    }
-
-    assert(exc.getMessage.contains("z"))
-    assert(exc.getMessage.contains("Cannot find"))
-  }
-
-  test("alterTable: table does not exist") {
-    val catalog = newCatalog()
-
-    val exc = intercept[NoSuchTableException] {
-      catalog.alterTable(testIdent, TableChange.setProperty("prop", "val"))
-    }
-
-    assert(exc.message.contains(testIdent.quoted))
-    assert(exc.message.contains("not found"))
-  }
-
-  test("dropTable") {
-    val catalog = newCatalog()
-
-    assert(!catalog.tableExists(testIdent))
-
-    catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    assert(catalog.tableExists(testIdent))
-
-    val wasDropped = catalog.dropTable(testIdent)
-
-    assert(wasDropped)
-    assert(!catalog.tableExists(testIdent))
-  }
-
-  test("dropTable: table does not exist") {
-    val catalog = newCatalog()
-
-    assert(!catalog.tableExists(testIdent))
-
-    val wasDropped = catalog.dropTable(testIdent)
-
-    assert(!wasDropped)
-    assert(!catalog.tableExists(testIdent))
-  }
-
-  test("renameTable") {
-    val catalog = newCatalog()
-
-    assert(!catalog.tableExists(testIdent))
-    assert(!catalog.tableExists(testIdentNew))
-
-    catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    assert(catalog.tableExists(testIdent))
-    catalog.renameTable(testIdent, testIdentNew)
-
-    assert(!catalog.tableExists(testIdent))
-    assert(catalog.tableExists(testIdentNew))
-  }
-
-  test("renameTable: fail if table does not exist") {
-    val catalog = newCatalog()
-
-    val exc = intercept[NoSuchTableException] {
-      catalog.renameTable(testIdent, testIdentNew)
-    }
-
-    assert(exc.message.contains(testIdent.quoted))
-    assert(exc.message.contains("not found"))
-  }
-
-  test("renameTable: fail if new table name already exists") {
-    val catalog = newCatalog()
-
-    assert(!catalog.tableExists(testIdent))
-    assert(!catalog.tableExists(testIdentNew))
-
-    catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-    catalog.createTable(testIdentNew, schema, Array.empty, emptyProps)
-
-    assert(catalog.tableExists(testIdent))
-    assert(catalog.tableExists(testIdentNew))
-
-    val exc = intercept[TableAlreadyExistsException] {
-      catalog.renameTable(testIdent, testIdentNew)
-    }
-
-    assert(exc.message.contains(testIdentNew.quoted))
-    assert(exc.message.contains("already exists"))
-  }
-
-  test("listNamespaces: list namespaces from metadata") {
-    val catalog = newCatalog()
-    catalog.createNamespace(Array("ns1"), Map("property" -> "value").asJava)
-
-    assert(catalog.listNamespaces === Array(Array("ns1")))
-    assert(catalog.listNamespaces(Array()) === Array(Array("ns1")))
-    assert(catalog.listNamespaces(Array("ns1")) === Array())
-  }
-
-  test("listNamespaces: list namespaces from tables") {
-    val catalog = newCatalog()
-    val ident1 = Identifier.of(Array("ns1", "ns2"), "test_table_1")
-    val ident2 = Identifier.of(Array("ns1", "ns2"), "test_table_2")
-
-    catalog.createTable(ident1, schema, Array.empty, emptyProps)
-    catalog.createTable(ident2, schema, Array.empty, emptyProps)
-
-    assert(catalog.listNamespaces === Array(Array("ns1")))
-    assert(catalog.listNamespaces(Array()) === Array(Array("ns1")))
-    assert(catalog.listNamespaces(Array("ns1")) === Array(Array("ns1", "ns2")))
-    assert(catalog.listNamespaces(Array("ns1", "ns2")) === Array())
-  }
-
-  test("listNamespaces: list namespaces from metadata and tables") {
-    val catalog = newCatalog()
-    val ident1 = Identifier.of(Array("ns1", "ns2"), "test_table_1")
-    val ident2 = Identifier.of(Array("ns1", "ns2"), "test_table_2")
-
-    catalog.createNamespace(Array("ns1"), Map("property" -> "value").asJava)
-    catalog.createTable(ident1, schema, Array.empty, emptyProps)
-    catalog.createTable(ident2, schema, Array.empty, emptyProps)
-
-    assert(catalog.listNamespaces === Array(Array("ns1")))
-    assert(catalog.listNamespaces(Array()) === Array(Array("ns1")))
-    assert(catalog.listNamespaces(Array("ns1")) === Array(Array("ns1", "ns2")))
-    assert(catalog.listNamespaces(Array("ns1", "ns2")) === Array())
-  }
-
-  test("loadNamespaceMetadata: fail if no metadata or tables exist") {
-    val catalog = newCatalog()
-
-    val exc = intercept[NoSuchNamespaceException] {
-      catalog.loadNamespaceMetadata(testNs)
-    }
-
-    assert(exc.getMessage.contains(testNs.quoted))
-  }
-
-  test("loadNamespaceMetadata: no metadata, table exists") {
-    val catalog = newCatalog()
-
-    catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    val metadata = catalog.loadNamespaceMetadata(testNs)
-
-    assert(metadata.asScala === Map.empty)
-  }
-
-  test("loadNamespaceMetadata: metadata exists, no tables") {
-    val catalog = newCatalog()
-
-    catalog.createNamespace(testNs, Map("property" -> "value").asJava)
-
-    val metadata = catalog.loadNamespaceMetadata(testNs)
-
-    assert(metadata.asScala === Map("property" -> "value"))
-  }
-
-  test("loadNamespaceMetadata: metadata and table exist") {
-    val catalog = newCatalog()
-
-    catalog.createNamespace(testNs, Map("property" -> "value").asJava)
-    catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    val metadata = catalog.loadNamespaceMetadata(testNs)
-
-    assert(metadata.asScala === Map("property" -> "value"))
-  }
-
-  test("createNamespace: basic behavior") {
-    val catalog = newCatalog()
-
-    catalog.createNamespace(testNs, Map("property" -> "value").asJava)
-
-    assert(catalog.namespaceExists(testNs) === true)
-    assert(catalog.loadNamespaceMetadata(testNs).asScala === Map("property" -> "value"))
-  }
-
-  test("createNamespace: fail if metadata already exists") {
-    val catalog = newCatalog()
-
-    catalog.createNamespace(testNs, Map("property" -> "value").asJava)
-
-    val exc = intercept[NamespaceAlreadyExistsException] {
-      catalog.createNamespace(testNs, Map("property" -> "value").asJava)
-    }
-
-    assert(exc.getMessage.contains(testNs.quoted))
-    assert(catalog.namespaceExists(testNs) === true)
-    assert(catalog.loadNamespaceMetadata(testNs).asScala === Map("property" -> "value"))
-  }
-
-  test("createNamespace: fail if namespace already exists from table") {
-    val catalog = newCatalog()
-
-    catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    assert(catalog.namespaceExists(testNs) === true)
-    assert(catalog.loadNamespaceMetadata(testNs).asScala === Map.empty)
-
-    val exc = intercept[NamespaceAlreadyExistsException] {
-      catalog.createNamespace(testNs, Map("property" -> "value").asJava)
-    }
-
-    assert(exc.getMessage.contains(testNs.quoted))
-    assert(catalog.namespaceExists(testNs) === true)
-    assert(catalog.loadNamespaceMetadata(testNs).asScala === Map.empty)
-  }
-
-  test("dropNamespace: drop missing namespace") {
-    val catalog = newCatalog()
-
-    assert(catalog.namespaceExists(testNs) === false)
-
-    val ret = catalog.dropNamespace(testNs)
-
-    assert(ret === false)
-  }
-
-  test("dropNamespace: drop empty namespace") {
-    val catalog = newCatalog()
-
-    catalog.createNamespace(testNs, Map("property" -> "value").asJava)
-
-    assert(catalog.namespaceExists(testNs) === true)
-    assert(catalog.loadNamespaceMetadata(testNs).asScala === Map("property" -> "value"))
-
-    val ret = catalog.dropNamespace(testNs)
-
-    assert(ret === true)
-    assert(catalog.namespaceExists(testNs) === false)
-  }
-
-  test("dropNamespace: drop even if it's not empty") {
-    val catalog = newCatalog()
-
-    catalog.createNamespace(testNs, Map("property" -> "value").asJava)
-    catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    assert(catalog.dropNamespace(testNs))
-
-    assert(!catalog.namespaceExists(testNs))
-    assert(catalog.listTables(testNs).isEmpty)
-  }
-
-  test("alterNamespace: basic behavior") {
-    val catalog = newCatalog()
-
-    catalog.createNamespace(testNs, Map("property" -> "value").asJava)
-
-    catalog.alterNamespace(testNs, NamespaceChange.setProperty("property2", "value2"))
-    assert(catalog.loadNamespaceMetadata(testNs).asScala === Map(
-      "property" -> "value", "property2" -> "value2"))
-
-    catalog.alterNamespace(testNs,
-      NamespaceChange.removeProperty("property2"),
-      NamespaceChange.setProperty("property3", "value3"))
-    assert(catalog.loadNamespaceMetadata(testNs).asScala === Map(
-      "property" -> "value", "property3" -> "value3"))
-
-    catalog.alterNamespace(testNs, NamespaceChange.removeProperty("property3"))
-    assert(catalog.loadNamespaceMetadata(testNs).asScala === Map("property" -> "value"))
-  }
-
-  test("alterNamespace: create metadata if missing and table exists") {
-    val catalog = newCatalog()
-
-    catalog.createTable(testIdent, schema, Array.empty, emptyProps)
-
-    catalog.alterNamespace(testNs, NamespaceChange.setProperty("property", "value"))
-
-    assert(catalog.loadNamespaceMetadata(testNs).asScala === Map("property" -> "value"))
-  }
-
-  test("alterNamespace: fail if no metadata or table exists") {
-    val catalog = newCatalog()
-
-    val exc = intercept[NoSuchNamespaceException] {
-      catalog.alterNamespace(testNs, NamespaceChange.setProperty("property", "value"))
-    }
-
-    assert(exc.getMessage.contains(testNs.quoted))
-  }
-}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/functions/AggregateFunctionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/functions/AggregateFunctionSuite.scala
new file mode 100644
index 0000000000000..5e5a7d4f05ff9
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/functions/AggregateFunctionSuite.scala
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.catalog.functions
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.types.{DataType, IntegerType, LongType, StructType}
+
+class AggregateFunctionSuite extends SparkFunSuite {
+  test("Test simple iavg(int)") {
+    val rows = Seq(InternalRow(2), InternalRow(2), InternalRow(2))
+
+    val bound = IntegralAverage.bind(new StructType().add("foo", IntegerType, nullable = false))
+    assert(bound.isInstanceOf[AggregateFunction[_, _]])
+    val udaf = bound.asInstanceOf[AggregateFunction[Serializable, _]]
+
+    val finalState = rows.foldLeft(udaf.newAggregationState()) { (state, row) =>
+      udaf.update(state, row)
+    }
+
+    assert(udaf.produceResult(finalState) == 2)
+  }
+
+  test("Test simple iavg(long)") {
+    val bigValue = 9762097370951020L
+    val rows = Seq(InternalRow(bigValue + 2), InternalRow(bigValue), InternalRow(bigValue - 2))
+
+    val bound = IntegralAverage.bind(new StructType().add("foo", LongType, nullable = false))
+    assert(bound.isInstanceOf[AggregateFunction[_, _]])
+    val udaf = bound.asInstanceOf[AggregateFunction[Serializable, _]]
+
+    val finalState = rows.foldLeft(udaf.newAggregationState()) { (state, row) =>
+      udaf.update(state, row)
+    }
+
+    assert(udaf.produceResult(finalState) == bigValue)
+  }
+
+  test("Test associative iavg(long)") {
+    val bigValue = 7620099737951020L
+    val rows = Seq(InternalRow(bigValue + 2), InternalRow(bigValue), InternalRow(bigValue - 2))
+
+    val bound = IntegralAverage.bind(new StructType().add("foo", LongType, nullable = false))
+    assert(bound.isInstanceOf[AggregateFunction[_, _]])
+    val udaf = bound.asInstanceOf[AggregateFunction[Serializable, _]]
+
+    val state1 = rows.foldLeft(udaf.newAggregationState()) { (state, row) =>
+      udaf.update(state, row)
+    }
+    val state2 = rows.foldLeft(udaf.newAggregationState()) { (state, row) =>
+      udaf.update(state, row)
+    }
+    val finalState = udaf.merge(state1, state2)
+
+    assert(udaf.produceResult(finalState) == bigValue)
+  }
+}
+
+object IntegralAverage extends UnboundFunction {
+  override def name(): String = "iavg"
+
+  override def bind(inputType: StructType): BoundFunction = {
+    if (inputType.fields.length > 1) {
+      throw new UnsupportedOperationException("Too many arguments")
+    }
+
+    if (inputType.fields(0).nullable) {
+      throw new UnsupportedOperationException("Nullable values are not supported")
+    }
+
+    inputType.fields(0).dataType match {
+      case _: IntegerType => IntAverage
+      case _: LongType => LongAverage
+      case dataType =>
+        throw new UnsupportedOperationException(s"Unsupported non-integral type: $dataType")
+    }
+  }
+
+  override def description(): String =
+    """iavg: produces an average using integer division
+      |  iavg(int not null) -> int
+      |  iavg(bigint not null) -> bigint""".stripMargin
+}
+
+object IntAverage extends AggregateFunction[(Int, Int), Int] {
+
+  override def inputTypes(): Array[DataType] = Array(IntegerType)
+
+  override def name(): String = "iavg"
+
+  override def newAggregationState(): (Int, Int) = (0, 0)
+
+  override def update(state: (Int, Int), input: InternalRow): (Int, Int) = {
+    val i = input.getInt(0)
+    val (total, count) = state
+    (total + i, count + 1)
+  }
+
+  override def merge(leftState: (Int, Int), rightState: (Int, Int)): (Int, Int) = {
+    (leftState._1 + rightState._1, leftState._2 + rightState._2)
+  }
+
+  override def produceResult(state: (Int, Int)): Int = state._1 / state._2
+
+  override def resultType(): DataType = IntegerType
+}
+
+object LongAverage extends AggregateFunction[(Long, Long), Long] {
+
+  override def inputTypes(): Array[DataType] = Array(LongType)
+
+  override def name(): String = "iavg"
+
+  override def newAggregationState(): (Long, Long) = (0L, 0L)
+
+  override def update(state: (Long, Long), input: InternalRow): (Long, Long) = {
+    val l = input.getLong(0)
+    state match {
+      case (_, 0L) =>
+        (l, 1)
+      case (total, count) =>
+        (total + l, count + 1L)
+    }
+  }
+
+  override def merge(leftState: (Long, Long), rightState: (Long, Long)): (Long, Long) = {
+    (leftState._1 + rightState._1, leftState._2 + rightState._2)
+  }
+
+  override def produceResult(state: (Long, Long)): Long = state._1 / state._2
+
+  override def resultType(): DataType = IntegerType
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
index b71dc91eed4fa..c7bd57ef0448c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
@@ -249,6 +249,15 @@ class DataTypeSuite extends SparkFunSuite {
   checkDataTypeFromJson(MapType(IntegerType, ArrayType(DoubleType), false))
   checkDataTypeFromDDL(MapType(IntegerType, ArrayType(DoubleType), false))
 
+  checkDataTypeFromJson(CharType(1))
+  checkDataTypeFromDDL(CharType(1))
+
+  checkDataTypeFromJson(VarcharType(10))
+  checkDataTypeFromDDL(VarcharType(11))
+
+  checkDataTypeFromDDL(YearMonthIntervalType)
+  checkDataTypeFromDDL(DayTimeIntervalType)
+
   val metadata = new MetadataBuilder()
     .putString("name", "age")
     .build()
@@ -310,6 +319,12 @@ class DataTypeSuite extends SparkFunSuite {
   checkDefaultSize(MapType(IntegerType, StringType, true), 24)
   checkDefaultSize(MapType(IntegerType, ArrayType(DoubleType), false), 12)
   checkDefaultSize(structType, 20)
+  checkDefaultSize(CharType(5), 5)
+  checkDefaultSize(CharType(100), 100)
+  checkDefaultSize(VarcharType(5), 5)
+  checkDefaultSize(VarcharType(10), 10)
+  checkDefaultSize(YearMonthIntervalType, 4)
+  checkDefaultSize(DayTimeIntervalType, 8)
 
   def checkEqualsIgnoreCompatibleNullability(
       from: DataType,
@@ -423,10 +438,15 @@ class DataTypeSuite extends SparkFunSuite {
   checkCatalogString(MapType(IntegerType, StringType))
   checkCatalogString(MapType(IntegerType, createStruct(40)))
 
-  def checkEqualsStructurally(from: DataType, to: DataType, expected: Boolean): Unit = {
-    val testName = s"equalsStructurally: (from: $from, to: $to)"
+  def checkEqualsStructurally(
+      from: DataType,
+      to: DataType,
+      expected: Boolean,
+      ignoreNullability: Boolean = false): Unit = {
+    val testName = s"equalsStructurally: (from: $from, to: $to, " +
+      s"ignoreNullability: $ignoreNullability)"
     test(testName) {
-      assert(DataType.equalsStructurally(from, to) === expected)
+      assert(DataType.equalsStructurally(from, to, ignoreNullability) === expected)
     }
   }
 
@@ -453,6 +473,105 @@ class DataTypeSuite extends SparkFunSuite {
     new StructType().add("f1", IntegerType).add("f", new StructType().add("f2", StringType, false)),
     new StructType().add("f2", IntegerType).add("g", new StructType().add("f1", StringType)),
     false)
+  checkEqualsStructurally(
+    new StructType().add("f1", IntegerType).add("f", new StructType().add("f2", StringType, false)),
+    new StructType().add("f2", IntegerType).add("g", new StructType().add("f1", StringType)),
+    true,
+    ignoreNullability = true)
+  checkEqualsStructurally(
+    new StructType().add("f1", IntegerType).add("f", new StructType().add("f2", StringType)),
+    new StructType().add("f2", IntegerType, nullable = false)
+      .add("g", new StructType().add("f1", StringType)),
+    true,
+    ignoreNullability = true)
+
+  checkEqualsStructurally(
+    ArrayType(
+      ArrayType(IntegerType, true), true),
+    ArrayType(
+      ArrayType(IntegerType, true), true),
+    true,
+     ignoreNullability = false)
+
+  checkEqualsStructurally(
+    ArrayType(
+      ArrayType(IntegerType, true), false),
+    ArrayType(
+      ArrayType(IntegerType, true), true),
+    false,
+    ignoreNullability = false)
+
+  checkEqualsStructurally(
+    ArrayType(
+      ArrayType(IntegerType, true), true),
+    ArrayType(
+      ArrayType(IntegerType, false), true),
+    false,
+    ignoreNullability = false)
+
+  checkEqualsStructurally(
+    ArrayType(
+      ArrayType(IntegerType, true), false),
+    ArrayType(
+      ArrayType(IntegerType, true), true),
+    true,
+    ignoreNullability = true)
+
+  checkEqualsStructurally(
+    ArrayType(
+      ArrayType(IntegerType, true), false),
+    ArrayType(
+      ArrayType(IntegerType, false), true),
+    true,
+    ignoreNullability = true)
+
+  checkEqualsStructurally(
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), true),
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), true),
+    true,
+    ignoreNullability = false)
+
+  checkEqualsStructurally(
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), true),
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), false),
+    false,
+    ignoreNullability = false)
+
+  checkEqualsStructurally(
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), true),
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, false), true),
+    false,
+    ignoreNullability = false)
+
+  checkEqualsStructurally(
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), true),
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), false),
+    true,
+    ignoreNullability = true)
+
+  checkEqualsStructurally(
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), true),
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, false), true),
+    true,
+    ignoreNullability = true)
+
+  checkEqualsStructurally(
+    MapType(
+      ArrayType(IntegerType, false), ArrayType(IntegerType, true), true),
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), true),
+    true,
+    ignoreNullability = true)
 
   test("SPARK-25031: MapType should produce current formatted string for complex types") {
     val keyType: DataType = StructType(Seq(
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeTestUtils.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeTestUtils.scala
index ed2c641d63e25..769de33528893 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeTestUtils.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeTestUtils.scala
@@ -52,13 +52,22 @@ object DataTypeTestUtils {
   /**
    * Instances of all [[NumericType]]s and [[CalendarIntervalType]]
    */
-  val numericAndInterval: Set[DataType] = numericTypeWithoutDecimal + CalendarIntervalType
+  val numericAndInterval: Set[DataType] = numericTypeWithoutDecimal ++ Set(
+    CalendarIntervalType,
+    DayTimeIntervalType,
+    YearMonthIntervalType)
 
   /**
    * All the types that support ordering
    */
-  val ordered: Set[DataType] =
-    numericTypeWithoutDecimal + BooleanType + TimestampType + DateType + StringType + BinaryType
+  val ordered: Set[DataType] = numericTypeWithoutDecimal ++ Set(
+    BooleanType,
+    TimestampType,
+    DateType,
+    StringType,
+    BinaryType,
+    DayTimeIntervalType,
+    YearMonthIntervalType)
 
   /**
    * All the types that we can use in a property check
@@ -73,7 +82,9 @@ object DataTypeTestUtils {
     BooleanType,
     DateType,
     StringType,
-    TimestampType
+    TimestampType,
+    DayTimeIntervalType,
+    YearMonthIntervalType
   )
 
   /**
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeWriteCompatibilitySuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeWriteCompatibilitySuite.scala
index c47332f5d9fcb..9fa016146bbd3 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeWriteCompatibilitySuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeWriteCompatibilitySuite.scala
@@ -80,7 +80,7 @@ class StrictDataTypeWriteCompatibilitySuite extends DataTypeWriteCompatibilityBa
   test("Check NullType is incompatible with all other types") {
     allNonNullTypes.foreach { t =>
       assertSingleError(NullType, t, "nulls", s"Should not allow writing None to type $t") { err =>
-        assert(err.contains(s"incompatible with $t"))
+        assert(err.contains(s"incompatible with ${t.catalogString}"))
       }
     }
   }
@@ -145,12 +145,12 @@ class ANSIDataTypeWriteCompatibilitySuite extends DataTypeWriteCompatibilityBase
   test("Conversions between timestamp and long are not allowed") {
     assertSingleError(LongType, TimestampType, "longToTimestamp",
       "Should not allow long to timestamp") { err =>
-      assert(err.contains("Cannot safely cast 'longToTimestamp': LongType to TimestampType"))
+      assert(err.contains("Cannot safely cast 'longToTimestamp': bigint to timestamp"))
     }
 
     assertSingleError(TimestampType, LongType, "timestampToLong",
       "Should not allow timestamp to long") { err =>
-      assert(err.contains("Cannot safely cast 'timestampToLong': TimestampType to LongType"))
+      assert(err.contains("Cannot safely cast 'timestampToLong': timestamp to bigint"))
     }
   }
 
@@ -209,8 +209,8 @@ abstract class DataTypeWriteCompatibilityBaseSuite extends SparkFunSuite {
             s"Should not allow writing $w to $r because cast is not safe") { err =>
             assert(err.contains("'t'"), "Should include the field name context")
             assert(err.contains("Cannot safely cast"), "Should identify unsafe cast")
-            assert(err.contains(s"$w"), "Should include write type")
-            assert(err.contains(s"$r"), "Should include read type")
+            assert(err.contains(s"${w.catalogString}"), "Should include write type")
+            assert(err.contains(s"${r.catalogString}"), "Should include read type")
           }
         }
       }
@@ -413,7 +413,7 @@ abstract class DataTypeWriteCompatibilityBaseSuite extends SparkFunSuite {
     assertNumErrors(writeType, readType, "top", "Should catch 14 errors", 14) { errs =>
       assert(errs(0).contains("'top.a.element'"), "Should identify bad type")
       assert(errs(0).contains("Cannot safely cast"))
-      assert(errs(0).contains("StringType to DoubleType"))
+      assert(errs(0).contains("string to double"))
 
       assert(errs(1).contains("'top.a'"), "Should identify bad type")
       assert(errs(1).contains("Cannot write nullable elements to array of non-nulls"))
@@ -430,11 +430,11 @@ abstract class DataTypeWriteCompatibilityBaseSuite extends SparkFunSuite {
 
       assert(errs(5).contains("'top.m.key'"), "Should identify bad type")
       assert(errs(5).contains("Cannot safely cast"))
-      assert(errs(5).contains("StringType to LongType"))
+      assert(errs(5).contains("string to bigint"))
 
       assert(errs(6).contains("'top.m.value'"), "Should identify bad type")
       assert(errs(6).contains("Cannot safely cast"))
-      assert(errs(6).contains("BooleanType to FloatType"))
+      assert(errs(6).contains("boolean to float"))
 
       assert(errs(7).contains("'top.m'"), "Should identify bad type")
       assert(errs(7).contains("Cannot write nullable values to map of non-nulls"))
@@ -452,7 +452,7 @@ abstract class DataTypeWriteCompatibilityBaseSuite extends SparkFunSuite {
 
       assert(errs(11).contains("'top.x'"), "Should identify bad type")
       assert(errs(11).contains("Cannot safely cast"))
-      assert(errs(11).contains("StringType to IntegerType"))
+      assert(errs(11).contains("string to int"))
 
       assert(errs(12).contains("'top'"), "Should identify bad type")
       assert(errs(12).contains("expected 'x', found 'y'"), "Should detect name mismatch")
@@ -502,6 +502,6 @@ abstract class DataTypeWriteCompatibilityBaseSuite extends SparkFunSuite {
       DataType.canWrite(writeType, readType, byName, analysis.caseSensitiveResolution, name,
         storeAssignmentPolicy, errMsg => errs += errMsg) === false, desc)
     assert(errs.size === numErrs, s"Should produce $numErrs error messages")
-    checkErrors(errs)
+    checkErrors(errs.toSeq)
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
index 7d0346fc0145e..7ce451ed6d577 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
@@ -24,6 +24,7 @@ import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.Decimal._
+import org.apache.spark.unsafe.types.UTF8String
 
 class DecimalSuite extends SparkFunSuite with PrivateMethodTester with SQLHelper {
   /** Check that a Decimal has the given string representation, precision and scale */
@@ -256,4 +257,33 @@ class DecimalSuite extends SparkFunSuite with PrivateMethodTester with SQLHelper
     assert(decimalLong.toScalaBigInt == scala.math.BigInt("123456789"))
     assert(decimalLong.toJavaBigInteger == new java.math.BigInteger("123456789"))
   }
+
+  test("UTF8String to Decimal") {
+    def checkFromString(string: String): Unit = {
+      assert(Decimal.fromString(UTF8String.fromString(string)) === Decimal(string))
+      assert(Decimal.fromStringANSI(UTF8String.fromString(string)) === Decimal(string))
+    }
+
+    def checkOutOfRangeFromString(string: String): Unit = {
+      assert(Decimal.fromString(UTF8String.fromString(string)) === null)
+      val e = intercept[ArithmeticException](Decimal.fromStringANSI(UTF8String.fromString(string)))
+      assert(e.getMessage.contains("out of decimal type range"))
+    }
+
+    checkFromString("12345678901234567890123456789012345678")
+    checkOutOfRangeFromString("123456789012345678901234567890123456789")
+
+    checkFromString("0.00000000000000000000000000000000000001")
+    checkFromString("0.000000000000000000000000000000000000000000000001")
+
+    checkFromString("6E-640")
+
+    checkFromString("6E+37")
+    checkOutOfRangeFromString("6E+38")
+    checkOutOfRangeFromString("6.0790316E+25569151")
+
+    assert(Decimal.fromString(UTF8String.fromString("str")) === null)
+    val e = intercept[NumberFormatException](Decimal.fromStringANSI(UTF8String.fromString("str")))
+    assert(e.getMessage.contains("invalid input syntax for type numeric"))
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala
index 6824a64badc10..46c38781ce46f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala
@@ -18,9 +18,11 @@
 package org.apache.spark.sql.types
 
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.plans.SQLHelper
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.StructType.fromDDL
 
-class StructTypeSuite extends SparkFunSuite {
+class StructTypeSuite extends SparkFunSuite with SQLHelper {
 
   private val s = StructType.fromDDL("a INT, b STRING")
 
@@ -71,6 +73,44 @@ class StructTypeSuite extends SparkFunSuite {
     assert(struct.toDDL == """`b` BOOLEAN COMMENT 'Field\'s comment'""")
   }
 
+  private val nestedStruct = new StructType()
+    .add(StructField("a", new StructType()
+      .add(StructField("b", new StructType()
+        .add(StructField("c", StringType
+        ).withComment("Deep Nested comment"))
+      ).withComment("Nested comment"))
+    ).withComment("comment"))
+
+  test("SPARK-33846: toDDL should output nested field's comment") {
+    val ddl = "`a` STRUCT<`b`: STRUCT<`c`: STRING COMMENT 'Deep Nested comment'> " +
+      "COMMENT 'Nested comment'> COMMENT 'comment'"
+    assert(nestedStruct.toDDL == ddl)
+  }
+
+  test("SPARK-33846: fromDDL should parse nested field's comment") {
+    val ddl = "`a` STRUCT<`b`: STRUCT<`c`: STRING COMMENT 'Deep Nested comment'> " +
+      "COMMENT 'Nested comment'> COMMENT 'comment'"
+    assert(StructType.fromDDL(ddl) == nestedStruct)
+  }
+
+  test("SPARK-33846: round trip toDDL -> fromDDL - nested struct") {
+    assert(StructType.fromDDL(nestedStruct.toDDL) == nestedStruct)
+  }
+
+  private val structWithEmptyString = new StructType()
+    .add(StructField("a b", StringType).withComment("comment"))
+
+  test("SPARK-33846: empty string in a column's name should be respected by toDDL") {
+    val ddl = "`a b` STRING COMMENT 'comment'"
+
+    assert(structWithEmptyString.toDDL == ddl)
+  }
+
+  test("SPARK-33846: empty string in a column's name should be respected by fromDDL") {
+    val ddl = "`a b` STRING COMMENT 'comment'"
+
+    assert(StructType.fromDDL(ddl) == structWithEmptyString)
+  }
 
   test("Print up to the given level") {
     val schema = StructType.fromDDL(
@@ -103,4 +143,104 @@ class StructTypeSuite extends SparkFunSuite {
     val interval = "`a` INTERVAL"
     assert(fromDDL(interval).toDDL === interval)
   }
+
+  test("find missing (nested) fields") {
+    val schema = StructType.fromDDL("c1 INT, c2 STRUCT<c3: INT, c4: STRUCT<c5: INT, c6: INT>>")
+    val resolver = SQLConf.get.resolver
+
+    val source1 = StructType.fromDDL("c1 INT")
+    val missing1 = StructType.fromDDL("c2 STRUCT<c3: INT, c4: STRUCT<c5: INT, c6: INT>>")
+    assert(StructType.findMissingFields(source1, schema, resolver)
+      .exists(_.sameType(missing1)))
+
+    val source2 = StructType.fromDDL("c1 INT, c3 STRING")
+    val missing2 = StructType.fromDDL("c2 STRUCT<c3: INT, c4: STRUCT<c5: INT, c6: INT>>")
+    assert(StructType.findMissingFields(source2, schema, resolver)
+      .exists(_.sameType(missing2)))
+
+    val source3 = StructType.fromDDL("c1 INT, c2 STRUCT<c3: INT>")
+    val missing3 = StructType.fromDDL("c2 STRUCT<c4: STRUCT<c5: INT, c6: INT>>")
+    assert(StructType.findMissingFields(source3, schema, resolver)
+      .exists(_.sameType(missing3)))
+
+    val source4 = StructType.fromDDL("c1 INT, c2 STRUCT<c3: INT, c4: STRUCT<c6: INT>>")
+    val missing4 = StructType.fromDDL("c2 STRUCT<c4: STRUCT<c5: INT>>")
+    assert(StructType.findMissingFields(source4, schema, resolver)
+      .exists(_.sameType(missing4)))
+  }
+
+  test("find missing (nested) fields: array and map") {
+    val resolver = SQLConf.get.resolver
+
+    val schemaWithArray = StructType.fromDDL("c1 INT, c2 ARRAY<STRUCT<c3: INT, c4: LONG>>")
+    val source5 = StructType.fromDDL("c1 INT")
+    val missing5 = StructType.fromDDL("c2 ARRAY<STRUCT<c3: INT, c4: LONG>>")
+    assert(
+      StructType.findMissingFields(source5, schemaWithArray, resolver)
+        .exists(_.sameType(missing5)))
+
+    val schemaWithMap1 = StructType.fromDDL(
+      "c1 INT, c2 MAP<STRUCT<c3: INT, c4: LONG>, STRING>, c3 LONG")
+    val source6 = StructType.fromDDL("c1 INT, c3 LONG")
+    val missing6 = StructType.fromDDL("c2 MAP<STRUCT<c3: INT, c4: LONG>, STRING>")
+    assert(
+      StructType.findMissingFields(source6, schemaWithMap1, resolver)
+        .exists(_.sameType(missing6)))
+
+    val schemaWithMap2 = StructType.fromDDL(
+      "c1 INT, c2 MAP<STRING, STRUCT<c3: INT, c4: LONG>>, c3 STRING")
+    val source7 = StructType.fromDDL("c1 INT, c3 STRING")
+    val missing7 = StructType.fromDDL("c2 MAP<STRING, STRUCT<c3: INT, c4: LONG>>")
+    assert(
+      StructType.findMissingFields(source7, schemaWithMap2, resolver)
+        .exists(_.sameType(missing7)))
+
+    // Unsupported: nested struct in array, map
+    val source8 = StructType.fromDDL("c1 INT, c2 ARRAY<STRUCT<c3: INT>>")
+    // `findMissingFields` doesn't support looking into nested struct in array type.
+    assert(StructType.findMissingFields(source8, schemaWithArray, resolver).isEmpty)
+
+    val source9 = StructType.fromDDL("c1 INT, c2 MAP<STRUCT<c3: INT>, STRING>, c3 LONG")
+    // `findMissingFields` doesn't support looking into nested struct in map type.
+    assert(StructType.findMissingFields(source9, schemaWithMap1, resolver).isEmpty)
+
+    val source10 = StructType.fromDDL("c1 INT, c2 MAP<STRING, STRUCT<c3: INT>>, c3 STRING")
+    // `findMissingFields` doesn't support looking into nested struct in map type.
+    assert(StructType.findMissingFields(source10, schemaWithMap2, resolver).isEmpty)
+  }
+
+  test("find missing (nested) fields: case sensitive cases") {
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+      val schema = StructType.fromDDL("c1 INT, c2 STRUCT<c3: INT, C4: STRUCT<C5: INT, c6: INT>>")
+      val resolver = SQLConf.get.resolver
+
+      val source1 = StructType.fromDDL("c1 INT, C2 LONG")
+      val missing1 = StructType.fromDDL("c2 STRUCT<c3: INT, C4: STRUCT<C5: INT, c6: INT>>")
+      assert(StructType.findMissingFields(source1, schema, resolver)
+        .exists(_.sameType(missing1)))
+
+      val source2 = StructType.fromDDL("c2 LONG")
+      val missing2 = StructType.fromDDL("c1 INT")
+      assert(StructType.findMissingFields(source2, schema, resolver)
+        .exists(_.sameType(missing2)))
+
+      val source3 = StructType.fromDDL("c1 INT, c2 STRUCT<c3: INT, C4: STRUCT<c5: INT>>")
+      val missing3 = StructType.fromDDL("c2 STRUCT<C4: STRUCT<C5: INT, c6: INT>>")
+      assert(StructType.findMissingFields(source3, schema, resolver)
+        .exists(_.sameType(missing3)))
+
+      val source4 = StructType.fromDDL("c1 INT, c2 STRUCT<c3: INT, C4: STRUCT<C5: Int>>")
+      val missing4 = StructType.fromDDL("c2 STRUCT<C4: STRUCT<c6: INT>>")
+      assert(StructType.findMissingFields(source4, schema, resolver)
+        .exists(_.sameType(missing4)))
+    }
+  }
+
+  test("SPARK-35285: ANSI interval types in schema") {
+    val yearMonthInterval = "`ymi` INTERVAL YEAR TO MONTH"
+    assert(fromDDL(yearMonthInterval).toDDL === yearMonthInterval)
+
+    val dayTimeInterval = "`dti` INTERVAL DAY TO SECOND"
+    assert(fromDDL(dayTimeInterval).toDDL === dayTimeInterval)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/TestUDT.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/TestUDT.scala
index 1be8ee9dfa92b..04b090d7001d1 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/TestUDT.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/TestUDT.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql.types
 
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
 import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData}
 
 
@@ -59,3 +61,74 @@ object TestUDT {
     override def equals(other: Any): Boolean = other.isInstanceOf[MyDenseVectorUDT]
   }
 }
+
+// object and classes to test SPARK-19311
+
+// Trait/Interface for base type
+sealed trait IExampleBaseType extends Serializable {
+  def field: Int
+}
+
+// Trait/Interface for derived type
+sealed trait IExampleSubType extends IExampleBaseType
+
+// a base class
+class ExampleBaseClass(override val field: Int) extends IExampleBaseType
+
+// a derived class
+class ExampleSubClass(override val field: Int)
+  extends ExampleBaseClass(field) with IExampleSubType
+
+// UDT for base class
+class ExampleBaseTypeUDT extends UserDefinedType[IExampleBaseType] {
+
+  override def sqlType: StructType = {
+    StructType(Seq(
+      StructField("intfield", IntegerType, nullable = false)))
+  }
+
+  override def serialize(obj: IExampleBaseType): InternalRow = {
+    val row = new GenericInternalRow(1)
+    row.setInt(0, obj.field)
+    row
+  }
+
+  override def deserialize(datum: Any): IExampleBaseType = {
+    datum match {
+      case row: InternalRow =>
+        require(row.numFields == 1,
+          "ExampleBaseTypeUDT requires row with length == 1")
+        val field = row.getInt(0)
+        new ExampleBaseClass(field)
+    }
+  }
+
+  override def userClass: Class[IExampleBaseType] = classOf[IExampleBaseType]
+}
+
+// UDT for derived class
+private[spark] class ExampleSubTypeUDT extends UserDefinedType[IExampleSubType] {
+
+  override def sqlType: StructType = {
+    StructType(Seq(
+      StructField("intfield", IntegerType, nullable = false)))
+  }
+
+  override def serialize(obj: IExampleSubType): InternalRow = {
+    val row = new GenericInternalRow(1)
+    row.setInt(0, obj.field)
+    row
+  }
+
+  override def deserialize(datum: Any): IExampleSubType = {
+    datum match {
+      case row: InternalRow =>
+        require(row.numFields == 1,
+          "ExampleSubTypeUDT requires row with length == 1")
+        val field = row.getInt(0)
+        new ExampleSubClass(field)
+    }
+  }
+
+  override def userClass: Class[IExampleSubType] = classOf[IExampleSubType]
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrowUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrowUtilsSuite.scala
index 0ffb492e702d1..5ae74c5eaf2be 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrowUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrowUtilsSuite.scala
@@ -17,11 +17,12 @@
 
 package org.apache.spark.sql.util
 
+import java.time.ZoneId
+
 import org.apache.arrow.vector.types.pojo.ArrowType
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.LA
-import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.types._
 
 class ArrowUtilsSuite extends SparkFunSuite {
@@ -47,6 +48,8 @@ class ArrowUtilsSuite extends SparkFunSuite {
     roundtrip(BinaryType)
     roundtrip(DecimalType.SYSTEM_DEFAULT)
     roundtrip(DateType)
+    roundtrip(YearMonthIntervalType)
+    roundtrip(DayTimeIntervalType)
     val tsExMsg = intercept[UnsupportedOperationException] {
       roundtrip(TimestampType)
     }
@@ -63,7 +66,7 @@ class ArrowUtilsSuite extends SparkFunSuite {
       assert(ArrowUtils.fromArrowSchema(arrowSchema) === schema)
     }
 
-    roundtripWithTz(DateTimeUtils.defaultTimeZone().getID)
+    roundtripWithTz(ZoneId.systemDefault().getId)
     roundtripWithTz("Asia/Tokyo")
     roundtripWithTz("UTC")
     roundtripWithTz(LA.getId)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
deleted file mode 100644
index 5e2b6a7c7fafe..0000000000000
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.util
-
-import java.time.{DateTimeException, LocalDate, ZoneOffset}
-
-import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
-import org.apache.spark.sql.catalyst.plans.SQLHelper
-import org.apache.spark.sql.catalyst.util._
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.{getZoneId, localDateToDays}
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
-
-class DateFormatterSuite extends SparkFunSuite with SQLHelper {
-  test("parsing dates") {
-    DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
-      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
-        val formatter = DateFormatter(getZoneId(timeZone))
-        val daysSinceEpoch = formatter.parse("2018-12-02")
-        assert(daysSinceEpoch === 17867)
-      }
-    }
-  }
-
-  test("format dates") {
-    DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
-      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
-        val formatter = DateFormatter(getZoneId(timeZone))
-        val date = formatter.format(17867)
-        assert(date === "2018-12-02")
-      }
-    }
-  }
-
-  test("roundtrip date -> days -> date") {
-    LegacyBehaviorPolicy.values.foreach { parserPolicy =>
-      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> parserPolicy.toString) {
-        LegacyDateFormats.values.foreach { legacyFormat =>
-          Seq(
-            "0050-01-01",
-            "0953-02-02",
-            "1423-03-08",
-            "1582-10-15",
-            "1969-12-31",
-            "1972-08-25",
-            "1975-09-26",
-            "2018-12-12",
-            "2038-01-01",
-            "5010-11-17").foreach { date =>
-            DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
-              withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
-                val formatter = DateFormatter(
-                  DateFormatter.defaultPattern,
-                  getZoneId(timeZone),
-                  DateFormatter.defaultLocale,
-                  legacyFormat)
-                val days = formatter.parse(date)
-                val formatted = formatter.format(days)
-                assert(date === formatted)
-              }
-            }
-          }
-        }
-      }
-    }
-  }
-
-  test("roundtrip days -> date -> days") {
-    LegacyBehaviorPolicy.values.foreach { parserPolicy =>
-      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> parserPolicy.toString) {
-        LegacyDateFormats.values.foreach { legacyFormat =>
-          Seq(
-            -701265,
-            -371419,
-            -199722,
-            -1,
-            0,
-            967,
-            2094,
-            17877,
-            24837,
-            1110657).foreach { days =>
-            DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
-              withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
-                val formatter = DateFormatter(
-                  DateFormatter.defaultPattern,
-                  getZoneId(timeZone),
-                  DateFormatter.defaultLocale,
-                  legacyFormat)
-                val date = formatter.format(days)
-                val parsed = formatter.parse(date)
-                assert(days === parsed)
-              }
-            }
-          }
-        }
-      }
-    }
-  }
-
-  test("parsing date without explicit day") {
-    val formatter = DateFormatter("yyyy MMM", ZoneOffset.UTC)
-    val daysSinceEpoch = formatter.parse("2018 Dec")
-    assert(daysSinceEpoch === LocalDate.of(2018, 12, 1).toEpochDay)
-  }
-
-  test("formatting negative years with default pattern") {
-    val epochDays = LocalDate.of(-99, 1, 1).toEpochDay.toInt
-    assert(DateFormatter(ZoneOffset.UTC).format(epochDays) === "-0099-01-01")
-  }
-
-  test("special date values") {
-    testSpecialDatetimeValues { zoneId =>
-      val formatter = DateFormatter(zoneId)
-
-      assert(formatter.parse("EPOCH") === 0)
-      val today = localDateToDays(LocalDate.now(zoneId))
-      assert(formatter.parse("Yesterday") === today - 1)
-      assert(formatter.parse("now") === today)
-      assert(formatter.parse("today ") === today)
-      assert(formatter.parse("tomorrow UTC") === today + 1)
-    }
-  }
-
-  test("SPARK-30958: parse date with negative year") {
-    val formatter1 = DateFormatter("yyyy-MM-dd", ZoneOffset.UTC)
-    assert(formatter1.parse("-1234-02-22") === localDateToDays(LocalDate.of(-1234, 2, 22)))
-
-    def assertParsingError(f: => Unit): Unit = {
-      intercept[Exception](f) match {
-        case e: SparkUpgradeException =>
-          assert(e.getCause.isInstanceOf[DateTimeException])
-        case e =>
-          assert(e.isInstanceOf[DateTimeException])
-      }
-    }
-
-    // "yyyy" with "G" can't parse negative year or year 0000.
-    val formatter2 = DateFormatter("G yyyy-MM-dd", ZoneOffset.UTC)
-    assertParsingError(formatter2.parse("BC -1234-02-22"))
-    assertParsingError(formatter2.parse("AD 0000-02-22"))
-
-    assert(formatter2.parse("BC 1234-02-22") === localDateToDays(LocalDate.of(-1233, 2, 22)))
-    assert(formatter2.parse("AD 1234-02-22") === localDateToDays(LocalDate.of(1234, 2, 22)))
-  }
-
-  test("SPARK-31557: rebasing in legacy formatters/parsers") {
-    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> LegacyBehaviorPolicy.LEGACY.toString) {
-      LegacyDateFormats.values.foreach { legacyFormat =>
-        DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
-          withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
-            val formatter = DateFormatter(
-              DateFormatter.defaultPattern,
-              getZoneId(timeZone),
-              DateFormatter.defaultLocale,
-              legacyFormat)
-            assert(LocalDate.ofEpochDay(formatter.parse("1000-01-01")) === LocalDate.of(1000, 1, 1))
-            assert(formatter.format(localDateToDays(LocalDate.of(1000, 1, 1))) === "1000-01-01")
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/SchemaUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/SchemaUtilsSuite.scala
index 2f576a4031e92..75caab4145938 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/SchemaUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/SchemaUtilsSuite.scala
@@ -22,7 +22,7 @@ import java.util.Locale
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.analysis._
-import org.apache.spark.sql.types.StructType
+import org.apache.spark.sql.types.{ArrayType, LongType, MapType, StructType}
 
 class SchemaUtilsSuite extends SparkFunSuite {
 
@@ -41,7 +41,7 @@ class SchemaUtilsSuite extends SparkFunSuite {
     test(s"Check column name duplication in $testType cases") {
       def checkExceptionCases(schemaStr: String, duplicatedColumns: Seq[String]): Unit = {
         val expectedErrorMsg = "Found duplicate column(s) in SchemaUtilsSuite: " +
-          duplicatedColumns.map(c => s"`${c.toLowerCase(Locale.ROOT)}`").mkString(", ")
+          duplicatedColumns.sorted.map(c => s"`${c.toLowerCase(Locale.ROOT)}`").mkString(", ")
         val schema = StructType.fromDDL(schemaStr)
         var msg = intercept[AnalysisException] {
           SchemaUtils.checkSchemaColumnNameDuplication(
@@ -82,4 +82,28 @@ class SchemaUtilsSuite extends SparkFunSuite {
 
     checkNoExceptionCases("a INT, b INT, c INT", caseSensitive = false)
   }
+
+  test("SPARK-32431: duplicated fields in nested schemas") {
+    val schemaA = new StructType()
+      .add("LowerCase", LongType)
+      .add("camelcase", LongType)
+      .add("CamelCase", LongType)
+    val schemaB = new StructType()
+      .add("f1", LongType)
+      .add("StructColumn1", schemaA)
+    val schemaC = new StructType()
+      .add("f2", LongType)
+      .add("StructColumn2", schemaB)
+    val schemaD = new StructType()
+      .add("f3", ArrayType(schemaC))
+    val schemaE = MapType(LongType, schemaD)
+    val schemaF = MapType(schemaD, LongType)
+    Seq(schemaA, schemaB, schemaC, schemaD, schemaE, schemaF).foreach { schema =>
+      val msg = intercept[AnalysisException] {
+        SchemaUtils.checkSchemaColumnNameDuplication(
+          schema, "in SchemaUtilsSuite", caseSensitiveAnalysis = false)
+      }.getMessage
+      assert(msg.contains("Found duplicate column(s) in SchemaUtilsSuite: `camelcase`"))
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
deleted file mode 100644
index 5d27a6b8cce1e..0000000000000
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
+++ /dev/null
@@ -1,294 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.util
-
-import java.time.{DateTimeException, Instant, LocalDateTime, LocalTime, ZoneOffset}
-import java.util.concurrent.TimeUnit
-
-import org.scalatest.Matchers
-
-import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
-import org.apache.spark.sql.catalyst.plans.SQLHelper
-import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, DateTimeUtils, LegacyDateFormats, TimestampFormatter}
-import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{CET, PST, UTC}
-import org.apache.spark.sql.catalyst.util.DateTimeUtils._
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
-import org.apache.spark.unsafe.types.UTF8String
-
-class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers {
-
-  test("parsing timestamps using time zones") {
-    val localDate = "2018-12-02T10:11:12.001234"
-    val expectedMicros = Map(
-      "UTC" -> 1543745472001234L,
-      PST.getId -> 1543774272001234L,
-      CET.getId -> 1543741872001234L,
-      "Africa/Dakar" -> 1543745472001234L,
-      "America/Los_Angeles" -> 1543774272001234L,
-      "Antarctica/Vostok" -> 1543723872001234L,
-      "Asia/Hong_Kong" -> 1543716672001234L,
-      "Europe/Amsterdam" -> 1543741872001234L)
-    DateTimeTestUtils.outstandingTimezonesIds.foreach { zoneId =>
-      val formatter = TimestampFormatter(
-        "yyyy-MM-dd'T'HH:mm:ss.SSSSSS",
-        DateTimeUtils.getZoneId(zoneId),
-        needVarLengthSecondFraction = true)
-      val microsSinceEpoch = formatter.parse(localDate)
-      assert(microsSinceEpoch === expectedMicros(zoneId))
-    }
-  }
-
-  test("format timestamps using time zones") {
-    val microsSinceEpoch = 1543745472001234L
-    val expectedTimestamp = Map(
-      "UTC" -> "2018-12-02T10:11:12.001234",
-      PST.getId -> "2018-12-02T02:11:12.001234",
-      CET.getId -> "2018-12-02T11:11:12.001234",
-      "Africa/Dakar" -> "2018-12-02T10:11:12.001234",
-      "America/Los_Angeles" -> "2018-12-02T02:11:12.001234",
-      "Antarctica/Vostok" -> "2018-12-02T16:11:12.001234",
-      "Asia/Hong_Kong" -> "2018-12-02T18:11:12.001234",
-      "Europe/Amsterdam" -> "2018-12-02T11:11:12.001234")
-    DateTimeTestUtils.outstandingTimezonesIds.foreach { zoneId =>
-      val formatter = TimestampFormatter(
-        "yyyy-MM-dd'T'HH:mm:ss.SSSSSS",
-        DateTimeUtils.getZoneId(zoneId))
-      val timestamp = formatter.format(microsSinceEpoch)
-      assert(timestamp === expectedTimestamp(zoneId))
-    }
-  }
-
-  test("roundtrip micros -> timestamp -> micros using timezones") {
-    Seq("yyyy-MM-dd'T'HH:mm:ss.SSSSSS", "yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXXXX").foreach { pattern =>
-      Seq(
-        -58710115316212000L,
-        -18926315945345679L,
-        -9463427405253013L,
-        -244000001L,
-        0L,
-        99628200102030L,
-        1543749753123456L,
-        2177456523456789L,
-        11858049903010203L).foreach { micros =>
-        DateTimeTestUtils.outstandingZoneIds.foreach { zoneId =>
-          val timestamp = TimestampFormatter(pattern, zoneId).format(micros)
-          val parsed = TimestampFormatter(
-            pattern, zoneId, needVarLengthSecondFraction = true).parse(timestamp)
-          assert(micros === parsed)
-        }
-      }
-    }
-  }
-
-  test("roundtrip timestamp -> micros -> timestamp using timezones") {
-    Seq(
-      "0109-07-20T18:38:03.788000",
-      "1370-04-01T10:00:54.654321",
-      "1670-02-11T14:09:54.746987",
-      "1969-12-31T23:55:55.999999",
-      "1970-01-01T00:00:00.000000",
-      "1973-02-27T02:30:00.102030",
-      "2018-12-02T11:22:33.123456",
-      "2039-01-01T01:02:03.456789",
-      "2345-10-07T22:45:03.010203").foreach { timestamp =>
-      DateTimeTestUtils.outstandingZoneIds.foreach { zoneId =>
-        val pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSSSS"
-        val micros = TimestampFormatter(
-          pattern, zoneId, needVarLengthSecondFraction = true).parse(timestamp)
-        val formatted = TimestampFormatter(pattern, zoneId).format(micros)
-        assert(timestamp === formatted)
-      }
-    }
-  }
-
-  test("case insensitive parsing of am and pm") {
-    val formatter = TimestampFormatter("yyyy MMM dd hh:mm:ss a", ZoneOffset.UTC)
-    val micros = formatter.parse("2009 Mar 20 11:30:01 am")
-    assert(micros === TimeUnit.SECONDS.toMicros(
-      LocalDateTime.of(2009, 3, 20, 11, 30, 1).toEpochSecond(ZoneOffset.UTC)))
-  }
-
-  test("format fraction of second") {
-    val formatter = TimestampFormatter.getFractionFormatter(ZoneOffset.UTC)
-    assert(formatter.format(0) === "1970-01-01 00:00:00")
-    assert(formatter.format(1) === "1970-01-01 00:00:00.000001")
-    assert(formatter.format(1000) === "1970-01-01 00:00:00.001")
-    assert(formatter.format(900000) === "1970-01-01 00:00:00.9")
-    assert(formatter.format(1000000) === "1970-01-01 00:00:01")
-  }
-
-  test("formatting negative years with default pattern") {
-    val instant = LocalDateTime.of(-99, 1, 1, 0, 0, 0)
-      .atZone(ZoneOffset.UTC)
-      .toInstant
-    val micros = DateTimeUtils.instantToMicros(instant)
-    assert(TimestampFormatter(ZoneOffset.UTC).format(micros) === "-0099-01-01 00:00:00")
-  }
-
-  test("special timestamp values") {
-    testSpecialDatetimeValues { zoneId =>
-      val formatter = TimestampFormatter(zoneId)
-      val tolerance = TimeUnit.SECONDS.toMicros(30)
-
-      assert(formatter.parse("EPOCH") === 0)
-      val now = instantToMicros(Instant.now())
-      formatter.parse("now") should be(now +- tolerance)
-      val localToday = LocalDateTime.now(zoneId)
-        .`with`(LocalTime.MIDNIGHT)
-        .atZone(zoneId)
-      val yesterday = instantToMicros(localToday.minusDays(1).toInstant)
-      formatter.parse("yesterday CET") should be(yesterday +- tolerance)
-      val today = instantToMicros(localToday.toInstant)
-      formatter.parse(" TODAY ") should be(today +- tolerance)
-      val tomorrow = instantToMicros(localToday.plusDays(1).toInstant)
-      formatter.parse("Tomorrow ") should be(tomorrow +- tolerance)
-    }
-  }
-
-  test("parsing timestamp strings with various seconds fractions") {
-    DateTimeTestUtils.outstandingZoneIds.foreach { zoneId =>
-      def check(pattern: String, input: String, reference: String): Unit = {
-        val formatter = TimestampFormatter(pattern, zoneId, needVarLengthSecondFraction = true)
-        val expected = DateTimeUtils.stringToTimestamp(
-          UTF8String.fromString(reference), zoneId).get
-        val actual = formatter.parse(input)
-        assert(actual === expected)
-      }
-
-      check("yyyy-MM-dd'T'HH:mm:ss.SSSSSSSXXX",
-        "2019-10-14T09:39:07.3220000Z", "2019-10-14T09:39:07.322Z")
-      check("yyyy-MM-dd'T'HH:mm:ss.SSSSSS",
-        "2019-10-14T09:39:07.322000", "2019-10-14T09:39:07.322")
-      check("yyyy-MM-dd'T'HH:mm:ss.SSSSSSX",
-        "2019-10-14T09:39:07.123456Z", "2019-10-14T09:39:07.123456Z")
-      check("yyyy-MM-dd'T'HH:mm:ss.SSSSSSX",
-        "2019-10-14T09:39:07.000010Z", "2019-10-14T09:39:07.00001Z")
-      check("yyyy HH:mm:ss.SSSSS", "1970 01:02:03.00004", "1970-01-01 01:02:03.00004")
-      check("yyyy HH:mm:ss.SSSS", "2019 00:00:07.0100", "2019-01-01 00:00:07.0100")
-      check("yyyy-MM-dd'T'HH:mm:ss.SSSX",
-        "2019-10-14T09:39:07.322Z", "2019-10-14T09:39:07.322Z")
-      check("yyyy-MM-dd'T'HH:mm:ss.SS",
-        "2019-10-14T09:39:07.10", "2019-10-14T09:39:07.1")
-      check("yyyy-MM-dd'T'HH:mm:ss.S",
-        "2019-10-14T09:39:07.1", "2019-10-14T09:39:07.1")
-
-      try {
-        TimestampFormatter("yyyy/MM/dd HH_mm_ss.SSSSSS", zoneId, true)
-          .parse("2019/11/14 20#25#30.123456")
-        fail("Expected to throw an exception for the invalid input")
-      } catch {
-        case e: java.time.format.DateTimeParseException =>
-          assert(e.getMessage.contains("could not be parsed"))
-      }
-    }
-  }
-
-  test("formatting timestamp strings up to microsecond precision") {
-    DateTimeTestUtils.outstandingZoneIds.foreach { zoneId =>
-      def check(pattern: String, input: String, expected: String): Unit = {
-        val formatter = TimestampFormatter(pattern, zoneId)
-        val timestamp = DateTimeUtils.stringToTimestamp(
-          UTF8String.fromString(input), zoneId).get
-        val actual = formatter.format(timestamp)
-        assert(actual === expected)
-      }
-
-      check(
-        "yyyy-MM-dd HH:mm:ss.SSSSSSS", "2019-10-14T09:39:07.123456",
-        "2019-10-14 09:39:07.1234560")
-      check(
-        "yyyy-MM-dd HH:mm:ss.SSSSSS", "1960-01-01T09:39:07.123456",
-        "1960-01-01 09:39:07.123456")
-      check(
-        "yyyy-MM-dd HH:mm:ss.SSSSS", "0001-10-14T09:39:07.1",
-        "0001-10-14 09:39:07.10000")
-      check(
-        "yyyy-MM-dd HH:mm:ss.SSSS", "9999-12-31T23:59:59.999",
-        "9999-12-31 23:59:59.9990")
-      check(
-        "yyyy-MM-dd HH:mm:ss.SSS", "1970-01-01T00:00:00.0101",
-        "1970-01-01 00:00:00.010")
-      check(
-        "yyyy-MM-dd HH:mm:ss.SS", "2019-10-14T09:39:07.09",
-        "2019-10-14 09:39:07.09")
-      check(
-        "yyyy-MM-dd HH:mm:ss.S", "2019-10-14T09:39:07.2",
-        "2019-10-14 09:39:07.2")
-      check(
-        "yyyy-MM-dd HH:mm:ss.S", "2019-10-14T09:39:07",
-        "2019-10-14 09:39:07.0")
-      check(
-        "yyyy-MM-dd HH:mm:ss", "2019-10-14T09:39:07.123456",
-        "2019-10-14 09:39:07")
-    }
-  }
-
-  test("SPARK-30958: parse timestamp with negative year") {
-    val formatter1 = TimestampFormatter("yyyy-MM-dd HH:mm:ss", ZoneOffset.UTC, true)
-    assert(formatter1.parse("-1234-02-22 02:22:22") === instantToMicros(
-      LocalDateTime.of(-1234, 2, 22, 2, 22, 22).toInstant(ZoneOffset.UTC)))
-
-    def assertParsingError(f: => Unit): Unit = {
-      intercept[Exception](f) match {
-        case e: SparkUpgradeException =>
-          assert(e.getCause.isInstanceOf[DateTimeException])
-        case e =>
-          assert(e.isInstanceOf[DateTimeException])
-      }
-    }
-
-    // "yyyy" with "G" can't parse negative year or year 0000.
-    val formatter2 = TimestampFormatter("G yyyy-MM-dd HH:mm:ss", UTC, true)
-    assertParsingError(formatter2.parse("BC -1234-02-22 02:22:22"))
-    assertParsingError(formatter2.parse("AC 0000-02-22 02:22:22"))
-
-    assert(formatter2.parse("BC 1234-02-22 02:22:22") === instantToMicros(
-      LocalDateTime.of(-1233, 2, 22, 2, 22, 22).toInstant(ZoneOffset.UTC)))
-    assert(formatter2.parse("AD 1234-02-22 02:22:22") === instantToMicros(
-      LocalDateTime.of(1234, 2, 22, 2, 22, 22).toInstant(ZoneOffset.UTC)))
-  }
-
-  test("SPARK-31557: rebasing in legacy formatters/parsers") {
-    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> LegacyBehaviorPolicy.LEGACY.toString) {
-      LegacyDateFormats.values.foreach { legacyFormat =>
-        DateTimeTestUtils.outstandingZoneIds.foreach { zoneId =>
-          withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> zoneId.getId) {
-            DateTimeTestUtils.withDefaultTimeZone(zoneId) {
-              withClue(s"${zoneId.getId} legacyFormat = $legacyFormat") {
-                val formatter = TimestampFormatter(
-                  TimestampFormatter.defaultPattern,
-                  zoneId,
-                  TimestampFormatter.defaultLocale,
-                  legacyFormat,
-                  needVarLengthSecondFraction = false)
-                assert(microsToInstant(formatter.parse("1000-01-01 01:02:03"))
-                  .atZone(zoneId)
-                  .toLocalDateTime === LocalDateTime.of(1000, 1, 1, 1, 2, 3))
-
-                assert(formatter.format(instantToMicros(
-                  LocalDateTime.of(1000, 1, 1, 1, 2, 3)
-                    .atZone(zoneId).toInstant)) === "1000-01-01 01:02:03")
-              }
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/sql/core/benchmarks/AggregateBenchmark-jdk11-results.txt b/sql/core/benchmarks/AggregateBenchmark-jdk11-results.txt
index baa237d3142d8..7b1e82d64adf3 100644
--- a/sql/core/benchmarks/AggregateBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/AggregateBenchmark-jdk11-results.txt
@@ -2,142 +2,147 @@
 aggregate without grouping
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 agg w/o group:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-agg w/o group wholestage off                      59499          62969        2375         35.2          28.4       1.0X
-agg w/o group wholestage on                         890            902          10       2357.2           0.4      66.9X
+agg w/o group wholestage off                      82274          82877         853         25.5          39.2       1.0X
+agg w/o group wholestage on                        1322           1358          37       1586.7           0.6      62.2X
 
 
 ================================================================================================
 stat functions
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 stddev:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-stddev wholestage off                              8702           8870         237         12.0          83.0       1.0X
-stddev wholestage on                               1307           1314           9         80.2          12.5       6.7X
+stddev wholestage off                              8975           9129         219         11.7          85.6       1.0X
+stddev wholestage on                               1424           1444          34         73.6          13.6       6.3X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 kurtosis:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-kurtosis wholestage off                           42656          42797         198          2.5         406.8       1.0X
-kurtosis wholestage on                             1440           1466          32         72.8          13.7      29.6X
+kurtosis wholestage off                           42273          42424         213          2.5         403.1       1.0X
+kurtosis wholestage on                             1492           1528          27         70.3          14.2      28.3X
 
 
 ================================================================================================
 aggregate with linear keys
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Aggregate w keys:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-codegen = F                                       10963          11035         101          7.7         130.7       1.0X
-codegen = T hashmap = F                            6852           7080         207         12.2          81.7       1.6X
-codegen = T hashmap = T                            1377           1421          43         60.9          16.4       8.0X
+codegen = F                                       10873          10998         176          7.7         129.6       1.0X
+codegen = T, hashmap = F                           5906           6005          95         14.2          70.4       1.8X
+codegen = T, row-based hashmap = T                 2325           2410          94         36.1          27.7       4.7X
+codegen = T, vectorized hashmap = T                1185           1259          78         70.8          14.1       9.2X
 
 
 ================================================================================================
 aggregate with randomized keys
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Aggregate w keys:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-codegen = F                                       12240          12296          80          6.9         145.9       1.0X
-codegen = T hashmap = F                            8318           8376          87         10.1          99.2       1.5X
-codegen = T hashmap = T                            2551           2617          80         32.9          30.4       4.8X
+codegen = F                                       12385          12470         120          6.8         147.6       1.0X
+codegen = T, hashmap = F                           7734           8110         378         10.8          92.2       1.6X
+codegen = T, row-based hashmap = T                 3663           3702          37         22.9          43.7       3.4X
+codegen = T, vectorized hashmap = T                2532           2621          54         33.1          30.2       4.9X
 
 
 ================================================================================================
 aggregate with string key
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Aggregate w string key:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-codegen = F                                        4442           4504          88          4.7         211.8       1.0X
-codegen = T hashmap = F                            2685           2778          95          7.8         128.0       1.7X
-codegen = T hashmap = T                            1181           1203          15         17.8          56.3       3.8X
+codegen = F                                        4465           4517          73          4.7         212.9       1.0X
+codegen = T, hashmap = F                           2667           2825         208          7.9         127.2       1.7X
+codegen = T, row-based hashmap = T                 1436           1466          21         14.6          68.5       3.1X
+codegen = T, vectorized hashmap = T                1297           1301           5         16.2          61.8       3.4X
 
 
 ================================================================================================
 aggregate with decimal key
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Aggregate w decimal key:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-codegen = F                                        3487           3534          66          6.0         166.3       1.0X
-codegen = T hashmap = F                            2239           2353         161          9.4         106.8       1.6X
-codegen = T hashmap = T                             664            693          26         31.6          31.7       5.3X
+codegen = F                                        3722           3746          34          5.6         177.5       1.0X
+codegen = T, hashmap = F                           2229           2297          96          9.4         106.3       1.7X
+codegen = T, row-based hashmap = T                  927            957          28         22.6          44.2       4.0X
+codegen = T, vectorized hashmap = T                 772            796          22         27.2          36.8       4.8X
 
 
 ================================================================================================
 aggregate with multiple key types
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Aggregate w multiple keys:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-codegen = F                                        6414           6475          86          3.3         305.9       1.0X
-codegen = T hashmap = F                            3776           3800          34          5.6         180.1       1.7X
-codegen = T hashmap = T                            2824           2852          40          7.4         134.7       2.3X
+codegen = F                                        7013           7060          67          3.0         334.4       1.0X
+codegen = T, hashmap = F                           3750           3894         205          5.6         178.8       1.9X
+codegen = T, row-based hashmap = T                 2948           2952           5          7.1         140.6       2.4X
+codegen = T, vectorized hashmap = T                2986           3145         226          7.0         142.4       2.3X
 
 
 ================================================================================================
 max function bytecode size of wholestagecodegen
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 max function bytecode size:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-codegen = F                                         654            696          36          1.0         998.6       1.0X
-codegen = T hugeMethodLimit = 10000                 378            405          24          1.7         577.1       1.7X
-codegen = T hugeMethodLimit = 1500                  383            403          22          1.7         584.3       1.7X
+codegen = F                                         567            620          37          1.2         864.6       1.0X
+codegen = T, hugeMethodLimit = 10000                283            316          26          2.3         431.9       2.0X
+codegen = T, hugeMethodLimit = 1500                 275            324          40          2.4         420.2       2.1X
 
 
 ================================================================================================
 cube
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 cube:                                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cube wholestage off                                3443           3444           1          1.5         656.7       1.0X
-cube wholestage on                                 1797           1815          20          2.9         342.7       1.9X
+cube wholestage off                                3389           3476         123          1.5         646.4       1.0X
+cube wholestage on                                 1692           1726          34          3.1         322.7       2.0X
 
 
 ================================================================================================
 hash and BytesToBytesMap
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 BytesToBytesMap:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-UnsafeRowhash                                       313            314           2         67.1          14.9       1.0X
-murmur3 hash                                        144            145           2        145.9           6.9       2.2X
-fast hash                                            70             72           5        301.5           3.3       4.5X
-arrayEqual                                          189            190           0        111.0           9.0       1.7X
-Java HashMap (Long)                                 129            132           4        162.4           6.2       2.4X
-Java HashMap (two ints)                             148            154           7        142.0           7.0       2.1X
-Java HashMap (UnsafeRow)                            839            840           1         25.0          40.0       0.4X
-LongToUnsafeRowMap (opt=false)                      462            464           1         45.4          22.0       0.7X
-LongToUnsafeRowMap (opt=true)                       108            108           1        194.2           5.2       2.9X
-BytesToBytesMap (off Heap)                         1027           1029           2         20.4          49.0       0.3X
-BytesToBytesMap (on Heap)                           999           1001           3         21.0          47.6       0.3X
-Aggregate HashMap                                    44             44           0        477.8           2.1       7.1X
+UnsafeRowhash                                       302            306           4         69.5          14.4       1.0X
+murmur3 hash                                        125            129           3        167.8           6.0       2.4X
+fast hash                                            69             73           3        304.1           3.3       4.4X
+arrayEqual                                          192            195           3        109.0           9.2       1.6X
+Java HashMap (Long)                                 133            187          53        157.2           6.4       2.3X
+Java HashMap (two ints)                             156            230          62        134.3           7.4       1.9X
+Java HashMap (UnsafeRow)                            807            812           6         26.0          38.5       0.4X
+LongToUnsafeRowMap (opt=false)                      502            529          24         41.8          23.9       0.6X
+LongToUnsafeRowMap (opt=true)                       148            164          20        141.7           7.1       2.0X
+BytesToBytesMap (off Heap)                          936            950          23         22.4          44.6       0.3X
+BytesToBytesMap (on Heap)                           954            956           2         22.0          45.5       0.3X
+Aggregate HashMap                                    46             54          11        455.4           2.2       6.6X
 
 
diff --git a/sql/core/benchmarks/AggregateBenchmark-results.txt b/sql/core/benchmarks/AggregateBenchmark-results.txt
index b1c2e9d6ae9fe..d4de806d03dc9 100644
--- a/sql/core/benchmarks/AggregateBenchmark-results.txt
+++ b/sql/core/benchmarks/AggregateBenchmark-results.txt
@@ -2,142 +2,147 @@
 aggregate without grouping
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 agg w/o group:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-agg w/o group wholestage off                      49902          52257         NaN         42.0          23.8       1.0X
-agg w/o group wholestage on                        1162           1171          10       1805.2           0.6      43.0X
+agg w/o group wholestage off                      53440          63455         NaN         39.2          25.5       1.0X
+agg w/o group wholestage on                        1157           1216          39       1812.5           0.6      46.2X
 
 
 ================================================================================================
 stat functions
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 stddev:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-stddev wholestage off                              8203           8243          56         12.8          78.2       1.0X
-stddev wholestage on                               1287           1303          10         81.5          12.3       6.4X
+stddev wholestage off                              7920           7947          39         13.2          75.5       1.0X
+stddev wholestage on                               1147           1160          11         91.4          10.9       6.9X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 kurtosis:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-kurtosis wholestage off                           39557          39919         511          2.7         377.2       1.0X
-kurtosis wholestage on                             1398           1476         138         75.0          13.3      28.3X
+kurtosis wholestage off                           35143          35319         250          3.0         335.1       1.0X
+kurtosis wholestage on                             1239           1258          20         84.6          11.8      28.4X
 
 
 ================================================================================================
 aggregate with linear keys
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Aggregate w keys:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-codegen = F                                       11236          12182        1337          7.5         133.9       1.0X
-codegen = T hashmap = F                            7079           7337         250         11.9          84.4       1.6X
-codegen = T hashmap = T                            1278           1419         186         65.6          15.2       8.8X
+codegen = F                                        9147           9183          50          9.2         109.0       1.0X
+codegen = T, hashmap = F                           5794           5949         226         14.5          69.1       1.6X
+codegen = T, row-based hashmap = T                 1378           1397          14         60.9          16.4       6.6X
+codegen = T, vectorized hashmap = T                 996           1034          25         84.3          11.9       9.2X
 
 
 ================================================================================================
 aggregate with randomized keys
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Aggregate w keys:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-codegen = F                                       11629          11650          30          7.2         138.6       1.0X
-codegen = T hashmap = F                            7552           7747         169         11.1          90.0       1.5X
-codegen = T hashmap = T                            2414           2662         167         34.7          28.8       4.8X
+codegen = F                                        9356           9425          98          9.0         111.5       1.0X
+codegen = T, hashmap = F                           5787           5912         176         14.5          69.0       1.6X
+codegen = T, row-based hashmap = T                 2569           2602          49         32.7          30.6       3.6X
+codegen = T, vectorized hashmap = T                2094           2128          27         40.1          25.0       4.5X
 
 
 ================================================================================================
 aggregate with string key
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Aggregate w string key:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-codegen = F                                        4790           4904         162          4.4         228.4       1.0X
-codegen = T hashmap = F                            3439           3504         105          6.1         164.0       1.4X
-codegen = T hashmap = T                            2327           2365          39          9.0         111.0       2.1X
+codegen = F                                        4270           4322          75          4.9         203.6       1.0X
+codegen = T, hashmap = F                           3241           3264          30          6.5         154.6       1.3X
+codegen = T, row-based hashmap = T                 2196           2247          32          9.6         104.7       1.9X
+codegen = T, vectorized hashmap = T                2291           2306          14          9.2         109.3       1.9X
 
 
 ================================================================================================
 aggregate with decimal key
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Aggregate w decimal key:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-codegen = F                                        3260           3418         223          6.4         155.5       1.0X
-codegen = T hashmap = F                            2316           2325          14          9.1         110.4       1.4X
-codegen = T hashmap = T                             605            607           2         34.7          28.8       5.4X
+codegen = F                                        2993           3010          23          7.0         142.7       1.0X
+codegen = T, hashmap = F                           1940           1945           7         10.8          92.5       1.5X
+codegen = T, row-based hashmap = T                  738            752          20         28.4          35.2       4.1X
+codegen = T, vectorized hashmap = T                 620            650          21         33.8          29.6       4.8X
 
 
 ================================================================================================
 aggregate with multiple key types
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Aggregate w multiple keys:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-codegen = F                                        7426           7473          67          2.8         354.1       1.0X
-codegen = T hashmap = F                            4685           4723          54          4.5         223.4       1.6X
-codegen = T hashmap = T                            3946           4005          83          5.3         188.2       1.9X
+codegen = F                                        6635           6636           2          3.2         316.4       1.0X
+codegen = T, hashmap = F                           4236           4269          47          5.0         202.0       1.6X
+codegen = T, row-based hashmap = T                 3118           3158          57          6.7         148.7       2.1X
+codegen = T, vectorized hashmap = T                3259           3278          27          6.4         155.4       2.0X
 
 
 ================================================================================================
 max function bytecode size of wholestagecodegen
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 max function bytecode size:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-codegen = F                                         628            672          49          1.0         958.4       1.0X
-codegen = T hugeMethodLimit = 10000                 357            373          12          1.8         545.3       1.8X
-codegen = T hugeMethodLimit = 1500                  344            356           7          1.9         525.6       1.8X
+codegen = F                                         467            492          33          1.4         712.4       1.0X
+codegen = T, hugeMethodLimit = 10000                216            231          19          3.0         329.7       2.2X
+codegen = T, hugeMethodLimit = 1500                 209            221           9          3.1         319.0       2.2X
 
 
 ================================================================================================
 cube
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 cube:                                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cube wholestage off                                3167           3266         140          1.7         604.1       1.0X
-cube wholestage on                                 1549           1576          29          3.4         295.4       2.0X
+cube wholestage off                                2490           2529          56          2.1         474.8       1.0X
+cube wholestage on                                 1401           1416          22          3.7         267.3       1.8X
 
 
 ================================================================================================
 hash and BytesToBytesMap
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 BytesToBytesMap:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-UnsafeRowhash                                       326            328           2         64.3          15.5       1.0X
-murmur3 hash                                        147            147           0        142.7           7.0       2.2X
-fast hash                                            74             75           1        282.3           3.5       4.4X
-arrayEqual                                          175            175           0        119.8           8.3       1.9X
-Java HashMap (Long)                                 138            140           4        152.1           6.6       2.4X
-Java HashMap (two ints)                             148            154           7        141.7           7.1       2.2X
-Java HashMap (UnsafeRow)                           1043           1090          66         20.1          49.8       0.3X
-LongToUnsafeRowMap (opt=false)                      464            466           2         45.2          22.1       0.7X
-LongToUnsafeRowMap (opt=true)                       104            106           8        202.3           4.9       3.1X
-BytesToBytesMap (off Heap)                         1140           1149          12         18.4          54.4       0.3X
-BytesToBytesMap (on Heap)                          1002           1132         183         20.9          47.8       0.3X
-Aggregate HashMap                                    74             74           0        281.9           3.5       4.4X
+UnsafeRowhash                                       259            264           5         81.0          12.3       1.0X
+murmur3 hash                                        113            121           3        185.7           5.4       2.3X
+fast hash                                            84             87           2        249.8           4.0       3.1X
+arrayEqual                                          172            180           4        121.9           8.2       1.5X
+Java HashMap (Long)                                 155            161           5        135.2           7.4       1.7X
+Java HashMap (two ints)                             147            157           8        142.6           7.0       1.8X
+Java HashMap (UnsafeRow)                            739            742           4         28.4          35.2       0.4X
+LongToUnsafeRowMap (opt=false)                      489            491           3         42.9          23.3       0.5X
+LongToUnsafeRowMap (opt=true)                        93            100           6        224.8           4.4       2.8X
+BytesToBytesMap (off Heap)                          882            896          16         23.8          42.1       0.3X
+BytesToBytesMap (on Heap)                           833            863          36         25.2          39.7       0.3X
+Aggregate HashMap                                    66             69           1        317.0           3.2       3.9X
 
 
diff --git a/sql/core/benchmarks/BloomFilterBenchmark-jdk11-results.txt b/sql/core/benchmarks/BloomFilterBenchmark-jdk11-results.txt
index cb1e96a3df236..1e3abebef021f 100644
--- a/sql/core/benchmarks/BloomFilterBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/BloomFilterBenchmark-jdk11-results.txt
@@ -2,23 +2,23 @@
 ORC Write
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Write 100M rows:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Without bloom filter                              19554          20736        1672          5.1         195.5       1.0X
-With bloom filter                                 22112          22203         129          4.5         221.1       0.9X
+Without bloom filter                              19503          19621         166          5.1         195.0       1.0X
+With bloom filter                                 22472          22710         335          4.4         224.7       0.9X
 
 
 ================================================================================================
 ORC Read
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Read a row from 100M rows:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Without bloom filter                               1866           1879          19         53.6          18.7       1.0X
-With bloom filter                                  1523           1544          29         65.6          15.2       1.2X
+Without bloom filter                               1981           2040          82         50.5          19.8       1.0X
+With bloom filter                                  1428           1467          54         70.0          14.3       1.4X
 
 
diff --git a/sql/core/benchmarks/BloomFilterBenchmark-results.txt b/sql/core/benchmarks/BloomFilterBenchmark-results.txt
index ec629129559f4..149d3bf76f770 100644
--- a/sql/core/benchmarks/BloomFilterBenchmark-results.txt
+++ b/sql/core/benchmarks/BloomFilterBenchmark-results.txt
@@ -2,23 +2,23 @@
 ORC Write
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Write 100M rows:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Without bloom filter                              19110          19145          50          5.2         191.1       1.0X
-With bloom filter                                 21890          21908          25          4.6         218.9       0.9X
+Without bloom filter                              14922          15194         384          6.7         149.2       1.0X
+With bloom filter                                 17270          17665         559          5.8         172.7       0.9X
 
 
 ================================================================================================
 ORC Read
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Read a row from 100M rows:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Without bloom filter                               1724           1746          30         58.0          17.2       1.0X
-With bloom filter                                  1364           1376          17         73.3          13.6       1.3X
+Without bloom filter                               1463           1486          32         68.4          14.6       1.0X
+With bloom filter                                  1232           1239           9         81.2          12.3       1.2X
 
 
diff --git a/sql/core/benchmarks/BuiltInDataSourceWriteBenchmark-jdk11-results.txt b/sql/core/benchmarks/BuiltInDataSourceWriteBenchmark-jdk11-results.txt
index bc13f6e9b9424..12153bddb5852 100644
--- a/sql/core/benchmarks/BuiltInDataSourceWriteBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/BuiltInDataSourceWriteBenchmark-jdk11-results.txt
@@ -2,59 +2,59 @@
 Parquet writer benchmark
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Parquet writer benchmark:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Output Single Int Column                           2552           2690         195          6.2         162.2       1.0X
-Output Single Double Column                        2865           2892          38          5.5         182.2       0.9X
-Output Int and String Column                       7876           7885          12          2.0         500.7       0.3X
-Output Partitions                                  5079           5871        1120          3.1         322.9       0.5X
-Output Buckets                                     6980           6994          20          2.3         443.8       0.4X
+Output Single Int Column                           2636           2676          56          6.0         167.6       1.0X
+Output Single Double Column                        2743           2859         164          5.7         174.4       1.0X
+Output Int and String Column                       8811           8822          15          1.8         560.2       0.3X
+Output Partitions                                  5489           5519          43          2.9         349.0       0.5X
+Output Buckets                                     7426           7436          14          2.1         472.1       0.4X
 
 
 ================================================================================================
 ORC writer benchmark
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 ORC writer benchmark:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Output Single Int Column                           1799           1902         146          8.7         114.4       1.0X
-Output Single Double Column                        2268           2276          11          6.9         144.2       0.8X
-Output Int and String Column                       6650           6670          28          2.4         422.8       0.3X
-Output Partitions                                  4697           4719          31          3.3         298.6       0.4X
-Output Buckets                                     6394           6436          60          2.5         406.5       0.3X
+Output Single Int Column                           1931           1944          19          8.1         122.8       1.0X
+Output Single Double Column                        2241           2304          90          7.0         142.5       0.9X
+Output Int and String Column                       7255           7318          89          2.2         461.3       0.3X
+Output Partitions                                  4733           4832         140          3.3         300.9       0.4X
+Output Buckets                                     6496           6547          73          2.4         413.0       0.3X
 
 
 ================================================================================================
 JSON writer benchmark
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 JSON writer benchmark:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Output Single Int Column                           2778           3522        1052          5.7         176.6       1.0X
-Output Single Double Column                        4222           4269          67          3.7         268.4       0.7X
-Output Int and String Column                      10822          10845          33          1.5         688.0       0.3X
-Output Partitions                                  5450           5523         104          2.9         346.5       0.5X
-Output Buckets                                    10827          11622        1123          1.5         688.4       0.3X
+Output Single Int Column                           2521           2552          44          6.2         160.3       1.0X
+Output Single Double Column                        4164           4181          24          3.8         264.7       0.6X
+Output Int and String Column                       6972           7002          43          2.3         443.3       0.4X
+Output Partitions                                  5246           5345         140          3.0         333.5       0.5X
+Output Buckets                                     6860           6935         107          2.3         436.1       0.4X
 
 
 ================================================================================================
 CSV writer benchmark
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 CSV writer benchmark:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Output Single Int Column                           3649           3698          68          4.3         232.0       1.0X
-Output Single Double Column                        4612           4696         120          3.4         293.2       0.8X
-Output Int and String Column                       7334           7517         258          2.1         466.3       0.5X
-Output Partitions                                  6386           6541         220          2.5         406.0       0.6X
-Output Buckets                                     8692           9439        1057          1.8         552.6       0.4X
+Output Single Int Column                           3650           3672          31          4.3         232.1       1.0X
+Output Single Double Column                        4719           4726          10          3.3         300.0       0.8X
+Output Int and String Column                       7842           7863          30          2.0         498.6       0.5X
+Output Partitions                                  6895           6911          23          2.3         438.4       0.5X
+Output Buckets                                     8880           8928          67          1.8         564.6       0.4X
 
 
diff --git a/sql/core/benchmarks/BuiltInDataSourceWriteBenchmark-results.txt b/sql/core/benchmarks/BuiltInDataSourceWriteBenchmark-results.txt
index 1380a6158954a..43674e52a72b7 100644
--- a/sql/core/benchmarks/BuiltInDataSourceWriteBenchmark-results.txt
+++ b/sql/core/benchmarks/BuiltInDataSourceWriteBenchmark-results.txt
@@ -2,59 +2,59 @@
 Parquet writer benchmark
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Parquet writer benchmark:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Output Single Int Column                           2537           2594          81          6.2         161.3       1.0X
-Output Single Double Column                        2615           2645          42          6.0         166.3       1.0X
-Output Int and String Column                       8931           8933           4          1.8         567.8       0.3X
-Output Partitions                                  5011           5321         438          3.1         318.6       0.5X
-Output Buckets                                     6789           6793           6          2.3         431.6       0.4X
+Output Single Int Column                           2067           2087          29          7.6         131.4       1.0X
+Output Single Double Column                        1977           2013          52          8.0         125.7       1.0X
+Output Int and String Column                       7387           7410          33          2.1         469.7       0.3X
+Output Partitions                                  3720           3831         156          4.2         236.5       0.6X
+Output Buckets                                     4987           5028          57          3.2         317.1       0.4X
 
 
 ================================================================================================
 ORC writer benchmark
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 ORC writer benchmark:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Output Single Int Column                           1780           1937         222          8.8         113.2       1.0X
-Output Single Double Column                        2248           2265          23          7.0         142.9       0.8X
-Output Int and String Column                       7996           8031          49          2.0         508.4       0.2X
-Output Partitions                                  4827           4939         159          3.3         306.9       0.4X
-Output Buckets                                     6521           6595         104          2.4         414.6       0.3X
+Output Single Int Column                           1264           1284          29         12.4          80.3       1.0X
+Output Single Double Column                        1708           1711           4          9.2         108.6       0.7X
+Output Int and String Column                       5988           6019          43          2.6         380.7       0.2X
+Output Partitions                                  3192           3269         108          4.9         203.0       0.4X
+Output Buckets                                     4439           4472          47          3.5         282.2       0.3X
 
 
 ================================================================================================
 JSON writer benchmark
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 JSON writer benchmark:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Output Single Int Column                           2791           2823          45          5.6         177.5       1.0X
-Output Single Double Column                        4078           5089        1431          3.9         259.3       0.7X
-Output Int and String Column                      10249          10296          66          1.5         651.6       0.3X
-Output Partitions                                  5247           5296          69          3.0         333.6       0.5X
-Output Buckets                                     8168           9714        2187          1.9         519.3       0.3X
+Output Single Int Column                           2187           2194          10          7.2         139.0       1.0X
+Output Single Double Column                        2808           2833          35          5.6         178.5       0.8X
+Output Int and String Column                       6190           6224          49          2.5         393.5       0.4X
+Output Partitions                                  3646           3705          83          4.3         231.8       0.6X
+Output Buckets                                     4891           5325         614          3.2         311.0       0.4X
 
 
 ================================================================================================
 CSV writer benchmark
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 CSV writer benchmark:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Output Single Int Column                           4055           4200         205          3.9         257.8       1.0X
-Output Single Double Column                        4978           5010          45          3.2         316.5       0.8X
-Output Int and String Column                       9285           9294          13          1.7         590.3       0.4X
-Output Partitions                                  6681           6738          80          2.4         424.8       0.6X
-Output Buckets                                     9495          10052         787          1.7         603.7       0.4X
+Output Single Int Column                           3162           3168           9          5.0         201.0       1.0X
+Output Single Double Column                        3647           3660          19          4.3         231.9       0.9X
+Output Int and String Column                       7266           7271           6          2.2         462.0       0.4X
+Output Partitions                                  4857           4930         103          3.2         308.8       0.7X
+Output Buckets                                     6809           6897         124          2.3         432.9       0.5X
 
 
diff --git a/sql/core/benchmarks/CSVBenchmark-jdk11-results.txt b/sql/core/benchmarks/CSVBenchmark-jdk11-results.txt
index 147a77ff098d0..b9dc68c988f63 100644
--- a/sql/core/benchmarks/CSVBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/CSVBenchmark-jdk11-results.txt
@@ -2,66 +2,66 @@
 Benchmark to measure CSV read/write performance
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Parsing quoted values:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-One quoted string                                 24907          29374         NaN          0.0      498130.5       1.0X
+One quoted string                                 35546          35913         327          0.0      710924.3       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Wide rows with 1000 columns:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 1000 columns                               62811          63690        1416          0.0       62811.4       1.0X
-Select 100 columns                                23839          24064         230          0.0       23839.5       2.6X
-Select one column                                 19936          20641         827          0.1       19936.4       3.2X
-count()                                            4174           4380         206          0.2        4174.4      15.0X
-Select 100 columns, one bad input field           41015          42380        1688          0.0       41015.4       1.5X
-Select 100 columns, corrupt record field          46281          46338          93          0.0       46280.5       1.4X
+Select 1000 columns                               97613          98458        1123          0.0       97613.1       1.0X
+Select 100 columns                                42208          42598         374          0.0       42208.4       2.3X
+Select one column                                 37602          38233         547          0.0       37601.6       2.6X
+count()                                            6343           6432         153          0.2        6343.4      15.4X
+Select 100 columns, one bad input field           65577          66403         829          0.0       65577.2       1.5X
+Select 100 columns, corrupt record field          79049          79718         608          0.0       79048.6       1.2X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Count a dataset with 10 columns:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 10 columns + count()                       10810          10997         163          0.9        1081.0       1.0X
-Select 1 column + count()                          7608           7641          47          1.3         760.8       1.4X
-count()                                            2415           2462          77          4.1         241.5       4.5X
+Select 10 columns + count()                       17730          18004         321          0.6        1773.0       1.0X
+Select 1 column + count()                         12627          12858         292          0.8        1262.7       1.4X
+count()                                            4329           4425         130          2.3         432.9       4.1X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Write dates and timestamps:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Create a dataset of timestamps                      874            914          37         11.4          87.4       1.0X
-to_csv(timestamp)                                  7051           7223         250          1.4         705.1       0.1X
-write timestamps to files                          6712           6741          31          1.5         671.2       0.1X
-Create a dataset of dates                           909            945          35         11.0          90.9       1.0X
-to_csv(date)                                       4222           4231           8          2.4         422.2       0.2X
-write dates to files                               3799           3813          14          2.6         379.9       0.2X
+Create a dataset of timestamps                     1539           1582          47          6.5         153.9       1.0X
+to_csv(timestamp)                                 12782          12980         192          0.8        1278.2       0.1X
+write timestamps to files                         10122          10253         170          1.0        1012.2       0.2X
+Create a dataset of dates                          1646           1765         111          6.1         164.6       0.9X
+to_csv(date)                                       9004           9216         200          1.1         900.4       0.2X
+write dates to files                               6519           6615         148          1.5         651.9       0.2X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Read dates and timestamps:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-read timestamp text from files                     1342           1364          35          7.5         134.2       1.0X
-read timestamps from files                        20300          20473         247          0.5        2030.0       0.1X
-infer timestamps from files                       40705          40744          54          0.2        4070.5       0.0X
-read date text from files                          1146           1151           6          8.7         114.6       1.2X
-read date from files                              12278          12408         117          0.8        1227.8       0.1X
-infer date from files                             12734          12872         220          0.8        1273.4       0.1X
-timestamp strings                                  1467           1482          15          6.8         146.7       0.9X
-parse timestamps from Dataset[String]             21708          22234         477          0.5        2170.8       0.1X
-infer timestamps from Dataset[String]             42357          43253         922          0.2        4235.7       0.0X
-date strings                                       1512           1532          18          6.6         151.2       0.9X
-parse dates from Dataset[String]                  13436          13470          33          0.7        1343.6       0.1X
-from_csv(timestamp)                               20390          20486          95          0.5        2039.0       0.1X
-from_csv(date)                                    12592          12693         139          0.8        1259.2       0.1X
+read timestamp text from files                     2364           2428          90          4.2         236.4       1.0X
+read timestamps from files                        39799          39885          96          0.3        3979.9       0.1X
+infer timestamps from files                       79083          79866         818          0.1        7908.3       0.0X
+read date text from files                          2174           2202          37          4.6         217.4       1.1X
+read date from files                              17997          18249         218          0.6        1799.7       0.1X
+infer date from files                             21635          21893         223          0.5        2163.5       0.1X
+timestamp strings                                  2686           2719          28          3.7         268.6       0.9X
+parse timestamps from Dataset[String]             40089          41277        1071          0.2        4008.9       0.1X
+infer timestamps from Dataset[String]             78144          78581         524          0.1        7814.4       0.0X
+date strings                                       2899           2974          74          3.4         289.9       0.8X
+parse dates from Dataset[String]                  19762          19875          99          0.5        1976.2       0.1X
+from_csv(timestamp)                               39890          40339         725          0.3        3989.0       0.1X
+from_csv(date)                                    18501          18740         214          0.5        1850.1       0.1X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Filters pushdown:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-w/o filters                                       12535          12606          67          0.0      125348.8       1.0X
-pushdown disabled                                 12611          12672          91          0.0      126112.9       1.0X
-w/ filters                                         1093           1099          11          0.1       10928.3      11.5X
+w/o filters                                       25642          25863         202          0.0      256422.1       1.0X
+pushdown disabled                                 24703          25195         476          0.0      247029.0       1.0X
+w/ filters                                         1184           1209          27          0.1       11842.4      21.7X
 
 
diff --git a/sql/core/benchmarks/CSVBenchmark-results.txt b/sql/core/benchmarks/CSVBenchmark-results.txt
index 498ca4caa0e45..caa215762fa9a 100644
--- a/sql/core/benchmarks/CSVBenchmark-results.txt
+++ b/sql/core/benchmarks/CSVBenchmark-results.txt
@@ -2,66 +2,66 @@
 Benchmark to measure CSV read/write performance
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Parsing quoted values:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-One quoted string                                 24073          24109          33          0.0      481463.5       1.0X
+One quoted string                                 43757          44446         765          0.0      875148.4       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Wide rows with 1000 columns:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 1000 columns                               58415          59611        2071          0.0       58414.8       1.0X
-Select 100 columns                                22568          23020         594          0.0       22568.0       2.6X
-Select one column                                 18995          19058          99          0.1       18995.0       3.1X
-count()                                            5301           5332          30          0.2        5300.9      11.0X
-Select 100 columns, one bad input field           39736          40153         361          0.0       39736.1       1.5X
-Select 100 columns, corrupt record field          47195          47826         590          0.0       47195.2       1.2X
+Select 1000 columns                               96330          99161         NaN          0.0       96329.7       1.0X
+Select 100 columns                                41414          42672        1556          0.0       41414.1       2.3X
+Select one column                                 35365          36113         662          0.0       35365.4       2.7X
+count()                                           18845          18867          26          0.1       18845.0       5.1X
+Select 100 columns, one bad input field           68271          68305          51          0.0       68270.7       1.4X
+Select 100 columns, corrupt record field          77700          78165         803          0.0       77699.7       1.2X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Count a dataset with 10 columns:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 10 columns + count()                        9884           9904          25          1.0         988.4       1.0X
-Select 1 column + count()                          6794           6835          46          1.5         679.4       1.5X
-count()                                            2060           2065           5          4.9         206.0       4.8X
+Select 10 columns + count()                       18462          18651         175          0.5        1846.2       1.0X
+Select 1 column + count()                         11897          12075         199          0.8        1189.7       1.6X
+count()                                            4218           4229          10          2.4         421.8       4.4X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Write dates and timestamps:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Create a dataset of timestamps                      717            732          18         14.0          71.7       1.0X
-to_csv(timestamp)                                  6994           7100         121          1.4         699.4       0.1X
-write timestamps to files                          6417           6435          27          1.6         641.7       0.1X
-Create a dataset of dates                           827            855          24         12.1          82.7       0.9X
-to_csv(date)                                       4408           4438          32          2.3         440.8       0.2X
-write dates to files                               3738           3758          28          2.7         373.8       0.2X
+Create a dataset of timestamps                     1680           1699          17          6.0         168.0       1.0X
+to_csv(timestamp)                                 13269          13787         456          0.8        1326.9       0.1X
+write timestamps to files                         10747          10785          48          0.9        1074.7       0.2X
+Create a dataset of dates                          1900           1919          24          5.3         190.0       0.9X
+to_csv(date)                                       9207           9223          23          1.1         920.7       0.2X
+write dates to files                               6331           6339           7          1.6         633.1       0.3X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Read dates and timestamps:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-read timestamp text from files                     1121           1176          52          8.9         112.1       1.0X
-read timestamps from files                        21298          21366         105          0.5        2129.8       0.1X
-infer timestamps from files                       41008          41051          39          0.2        4100.8       0.0X
-read date text from files                           962            967           5         10.4          96.2       1.2X
-read date from files                              11749          11772          22          0.9        1174.9       0.1X
-infer date from files                             12426          12459          29          0.8        1242.6       0.1X
-timestamp strings                                  1508           1519           9          6.6         150.8       0.7X
-parse timestamps from Dataset[String]             21674          21997         455          0.5        2167.4       0.1X
-infer timestamps from Dataset[String]             42141          42230         105          0.2        4214.1       0.0X
-date strings                                       1694           1701           8          5.9         169.4       0.7X
-parse dates from Dataset[String]                  12929          12951          25          0.8        1292.9       0.1X
-from_csv(timestamp)                               20603          20786         166          0.5        2060.3       0.1X
-from_csv(date)                                    12325          12338          12          0.8        1232.5       0.1X
+read timestamp text from files                     2355           2382          24          4.2         235.5       1.0X
+read timestamps from files                        31297          31331          35          0.3        3129.7       0.1X
+infer timestamps from files                       63255          66511         NaN          0.2        6325.5       0.0X
+read date text from files                          2139           2160          18          4.7         213.9       1.1X
+read date from files                              17027          17090          89          0.6        1702.7       0.1X
+infer date from files                             21307          21337          31          0.5        2130.7       0.1X
+timestamp strings                                  3661           3699          35          2.7         366.1       0.6X
+parse timestamps from Dataset[String]             36355          37714        1180          0.3        3635.5       0.1X
+infer timestamps from Dataset[String]             74494          74851         542          0.1        7449.4       0.0X
+date strings                                       3753           3756           5          2.7         375.3       0.6X
+parse dates from Dataset[String]                  21590          21714         126          0.5        2159.0       0.1X
+from_csv(timestamp)                               35419          35459          59          0.3        3541.9       0.1X
+from_csv(date)                                    19081          19124          39          0.5        1908.1       0.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Filters pushdown:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-w/o filters                                       12455          12474          22          0.0      124553.8       1.0X
-pushdown disabled                                 12462          12486          29          0.0      124624.9       1.0X
-w/ filters                                         1073           1092          18          0.1       10727.6      11.6X
+w/o filters                                       22203          22425         192          0.0      222033.1       1.0X
+pushdown disabled                                 22123          22220          89          0.0      221227.6       1.0X
+w/ filters                                         1332           1338           9          0.1       13317.7      16.7X
 
 
diff --git a/sql/core/benchmarks/CharVarcharBenchmark-results.txt b/sql/core/benchmarks/CharVarcharBenchmark-results.txt
new file mode 100644
index 0000000000000..d9d8a307a3469
--- /dev/null
+++ b/sql/core/benchmarks/CharVarcharBenchmark-results.txt
@@ -0,0 +1,122 @@
+================================================================================================
+Char Varchar Write Side Perf w/o Tailing Spaces
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
+Write with length 5:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+write string with length 5                        10387          10439          85          3.9         259.7       1.0X
+write char with length 5                          14181          14200          22          2.8         354.5       0.7X
+write varchar with length 5                       10658          10713          83          3.8         266.5       1.0X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
+Write with length 10:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+write string with length 10                        5292           5305          12          3.8         264.6       1.0X
+write char with length 10                          8539           8549          11          2.3         427.0       0.6X
+write varchar with length 10                       5404           5417          21          3.7         270.2       1.0X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
+Write with length 20:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+write string with length 20                        2680           2701          21          3.7         268.0       1.0X
+write char with length 20                          5768           5793          22          1.7         576.8       0.5X
+write varchar with length 20                       2744           2760          22          3.6         274.4       1.0X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
+Write with length 40:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+write string with length 40                        1401           1409          12          3.6         280.1       1.0X
+write char with length 40                          4306           4316          11          1.2         861.2       0.3X
+write varchar with length 40                       1433           1438           4          3.5         286.5       1.0X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
+Write with length 60:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+write string with length 60                        1123           1127           6          3.0         336.8       1.0X
+write char with length 60                          3807           3811           8          0.9        1142.0       0.3X
+write varchar with length 60                        987            991           3          3.4         296.2       1.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
+Write with length 80:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+write string with length 80                         741            753          11          3.4         296.4       1.0X
+write char with length 80                          3563           3568           6          0.7        1425.1       0.2X
+write varchar with length 80                        751            767          15          3.3         300.3       1.0X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
+Write with length 100:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+write string with length 100                        635            637           2          3.1         317.6       1.0X
+write char with length 100                         3454           3455           1          0.6        1727.0       0.2X
+write varchar with length 100                       634            638           4          3.2         316.9       1.0X
+
+
+================================================================================================
+Char Varchar Write Side Perf w/ Tailing Spaces
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
+Write with length 5:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+write string with length 5                        20381          20394          22          2.0         509.5       1.0X
+write char with length 5                          20172          20180           7          2.0         504.3       1.0X
+write varchar with length 5                       20153          20168          22          2.0         503.8       1.0X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
+Write with length 10:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+write string with length 10                       10697          10714          20          1.9         534.8       1.0X
+write char with length 10                         12411          12426          14          1.6         620.5       0.9X
+write varchar with length 10                      12416          12421           5          1.6         620.8       0.9X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
+Write with length 20:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+write string with length 20                        7370           7381          12          1.4         737.0       1.0X
+write char with length 20                          9090           9099           9          1.1         909.0       0.8X
+write varchar with length 20                       9082           9091           8          1.1         908.2       0.8X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
+Write with length 40:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+write string with length 40                        5606           5608           3          0.9        1121.2       1.0X
+write char with length 40                          7290           7292           3          0.7        1458.0       0.8X
+write varchar with length 40                       7304           7311           8          0.7        1460.8       0.8X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
+Write with length 60:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+write string with length 60                        4898           4906           9          0.7        1469.3       1.0X
+write char with length 60                          6640           6654          12          0.5        1992.1       0.7X
+write varchar with length 60                       6750           6756           7          0.5        2025.0       0.7X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
+Write with length 80:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+write string with length 80                        4720           4726           9          0.5        1887.8       1.0X
+write char with length 80                          6446           6448           1          0.4        2578.6       0.7X
+write varchar with length 80                       6469           6509          35          0.4        2587.7       0.7X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
+Write with length 100:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+write string with length 100                       4436           4450          14          0.5        2217.8       1.0X
+write char with length 100                         6317           6325          12          0.3        3158.4       0.7X
+write varchar with length 100                      6316           6322           7          0.3        3158.1       0.7X
+
+
diff --git a/sql/core/benchmarks/ColumnarBatchBenchmark-jdk11-results.txt b/sql/core/benchmarks/ColumnarBatchBenchmark-jdk11-results.txt
index d5dcdab6b942a..eb13d5b86f218 100644
--- a/sql/core/benchmarks/ColumnarBatchBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/ColumnarBatchBenchmark-jdk11-results.txt
@@ -2,58 +2,58 @@
 Int Read/Write
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Int Read/Write:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Java Array                                          233            234           0       1406.0           0.7       1.0X
-ByteBuffer Unsafe                                   396            397           0        826.5           1.2       0.6X
-ByteBuffer API                                      603            605           3        543.0           1.8       0.4X
-DirectByteBuffer                                   1189           1196           9        275.6           3.6       0.2X
-Unsafe Buffer                                       321            324           3       1022.0           1.0       0.7X
-Column(on heap)                                     235            236           2       1393.1           0.7       1.0X
-Column(off heap)                                    524            526           1        624.8           1.6       0.4X
-Column(off heap direct)                             317            318           1       1034.1           1.0       0.7X
-UnsafeRow (on heap)                                 690            693           3        474.8           2.1       0.3X
-UnsafeRow (off heap)                                642            646           3        510.1           2.0       0.4X
-Column On Heap Append                               522            527           4        628.0           1.6       0.4X
+Java Array                                          263            266           3       1247.5           0.8       1.0X
+ByteBuffer Unsafe                                  1039           1050          15        315.3           3.2       0.3X
+ByteBuffer API                                      923            928           8        355.0           2.8       0.3X
+DirectByteBuffer                                   1370           1371           1        239.1           4.2       0.2X
+Unsafe Buffer                                       350            357           6        937.5           1.1       0.8X
+Column(on heap)                                     265            269           3       1235.8           0.8       1.0X
+Column(off heap)                                    514            519           3        637.2           1.6       0.5X
+Column(off heap direct)                             348            354           5        940.8           1.1       0.8X
+UnsafeRow (on heap)                                 800            819          23        409.6           2.4       0.3X
+UnsafeRow (off heap)                                755            760           5        433.9           2.3       0.3X
+Column On Heap Append                               520            529           9        629.8           1.6       0.5X
 
 
 ================================================================================================
 Boolean Read/Write
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Boolean Read/Write:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Bitset                                              778            780           3        431.1           2.3       1.0X
-Byte Array                                          355            356           1        944.9           1.1       2.2X
+Bitset                                              846            848           1        396.4           2.5       1.0X
+Byte Array                                          386            391           4        869.5           1.2       2.2X
 
 
 ================================================================================================
 String Read/Write
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 String Read/Write:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-On Heap                                             447            453           5         36.6          27.3       1.0X
-Off Heap                                            652            658           4         25.1          39.8       0.7X
+On Heap                                             536            538           1         30.5          32.7       1.0X
+Off Heap                                            751            759           7         21.8          45.8       0.7X
 
 
 ================================================================================================
 Array Vector Read
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Array Vector Read:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-On Heap Read Size Only                              101            102           0       1616.4           0.6       1.0X
-Off Heap Read Size Only                             305            307           2        536.8           1.9       0.3X
-On Heap Read Elements                              4255           4256           1         38.5          26.0       0.0X
-Off Heap Read Elements                             6097           6104          10         26.9          37.2       0.0X
+On Heap Read Size Only                               96            100           2       1698.7           0.6       1.0X
+Off Heap Read Size Only                             356            364           9        459.6           2.2       0.3X
+On Heap Read Elements                              4971           4972           2         33.0          30.3       0.0X
+Off Heap Read Elements                             6833           6880          67         24.0          41.7       0.0X
 
 
diff --git a/sql/core/benchmarks/ColumnarBatchBenchmark-results.txt b/sql/core/benchmarks/ColumnarBatchBenchmark-results.txt
index 8ae21d7a7e653..2559b2c363220 100644
--- a/sql/core/benchmarks/ColumnarBatchBenchmark-results.txt
+++ b/sql/core/benchmarks/ColumnarBatchBenchmark-results.txt
@@ -2,58 +2,58 @@
 Int Read/Write
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Int Read/Write:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Java Array                                          246            246           0       1333.2           0.8       1.0X
-ByteBuffer Unsafe                                   446            447           1        734.6           1.4       0.6X
-ByteBuffer API                                     2141           2144           4        153.0           6.5       0.1X
-DirectByteBuffer                                    745            745           0        439.9           2.3       0.3X
-Unsafe Buffer                                       237            240           1       1379.7           0.7       1.0X
-Column(on heap)                                     247            247           0       1327.3           0.8       1.0X
-Column(off heap)                                    487            488           1        673.1           1.5       0.5X
-Column(off heap direct)                             242            243           1       1354.1           0.7       1.0X
-UnsafeRow (on heap)                                 541            543           1        605.4           1.7       0.5X
-UnsafeRow (off heap)                                581            583           2        563.7           1.8       0.4X
-Column On Heap Append                               520            521           1        629.8           1.6       0.5X
+Java Array                                          274            279           5       1196.8           0.8       1.0X
+ByteBuffer Unsafe                                   476            487           6        687.7           1.5       0.6X
+ByteBuffer API                                     2024           2030           8        161.9           6.2       0.1X
+DirectByteBuffer                                    702            755          47        466.8           2.1       0.4X
+Unsafe Buffer                                       250            266          13       1311.9           0.8       1.1X
+Column(on heap)                                     260            272           7       1258.0           0.8       1.1X
+Column(off heap)                                    528            540          13        620.1           1.6       0.5X
+Column(off heap direct)                             265            272           6       1236.9           0.8       1.0X
+UnsafeRow (on heap)                                 551            556           4        594.8           1.7       0.5X
+UnsafeRow (off heap)                                565            586          19        579.9           1.7       0.5X
+Column On Heap Append                               544            552          10        602.1           1.7       0.5X
 
 
 ================================================================================================
 Boolean Read/Write
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Boolean Read/Write:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Bitset                                              883            883           1        380.1           2.6       1.0X
-Byte Array                                          775            777           2        432.9           2.3       1.1X
+Bitset                                             1009           1026          25        332.6           3.0       1.0X
+Byte Array                                          756            768          11        443.7           2.3       1.3X
 
 
 ================================================================================================
 String Read/Write
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 String Read/Write:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-On Heap                                             463            470          10         35.4          28.3       1.0X
-Off Heap                                            676            677           2         24.2          41.2       0.7X
+On Heap                                             396            408           9         41.3          24.2       1.0X
+Off Heap                                            673            696          20         24.3          41.1       0.6X
 
 
 ================================================================================================
 Array Vector Read
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Array Vector Read:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-On Heap Read Size Only                              422            422           0        388.7           2.6       1.0X
-Off Heap Read Size Only                             259            259           0        633.2           1.6       1.6X
-On Heap Read Elements                              3932           3960          39         41.7          24.0       0.1X
-Off Heap Read Elements                             5143           5173          43         31.9          31.4       0.1X
+On Heap Read Size Only                              458            486          16        357.6           2.8       1.0X
+Off Heap Read Size Only                             260            275           9        630.6           1.6       1.8X
+On Heap Read Elements                              3315           3349          48         49.4          20.2       0.1X
+Off Heap Read Elements                             5265           5304          55         31.1          32.1       0.1X
 
 
diff --git a/sql/core/benchmarks/CompressionSchemeBenchmark-jdk11-results.txt b/sql/core/benchmarks/CompressionSchemeBenchmark-jdk11-results.txt
index 4fd57a9e95560..74b59ef725170 100644
--- a/sql/core/benchmarks/CompressionSchemeBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/CompressionSchemeBenchmark-jdk11-results.txt
@@ -2,136 +2,136 @@
 Compression Scheme Benchmark
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 BOOLEAN Encode:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough(1.000)                                    3              3           0      21087.3           0.0       1.0X
-RunLengthEncoding(2.514)                            739            739           1         90.8          11.0       0.0X
-BooleanBitSet(0.125)                                378            379           1        177.4           5.6       0.0X
+PassThrough(1.000)                                    5              6           0      13175.1           0.1       1.0X
+RunLengthEncoding(2.498)                            907            920          11         74.0          13.5       0.0X
+BooleanBitSet(0.125)                                594            609          11        113.1           8.8       0.0X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 BOOLEAN Decode:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough                                         147            147           1        456.1           2.2       1.0X
-RunLengthEncoding                                   731            732           1         91.8          10.9       0.2X
-BooleanBitSet                                      1410           1411           1         47.6          21.0       0.1X
+PassThrough                                         326            341          20        205.8           4.9       1.0X
+RunLengthEncoding                                  1060           1061           2         63.3          15.8       0.3X
+BooleanBitSet                                      1696           1717          29         39.6          25.3       0.2X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 SHORT Encode (Lower Skew):                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough(1.000)                                    7              7           0       9729.9           0.1       1.0X
-RunLengthEncoding(1.491)                           1576           1576           1         42.6          23.5       0.0X
+PassThrough(1.000)                                    9             10           1       7396.4           0.1       1.0X
+RunLengthEncoding(1.495)                           1412           1412           0         47.5          21.0       0.0X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 SHORT Decode (Lower Skew):                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough                                        1151           1152           1         58.3          17.2       1.0X
-RunLengthEncoding                                  1619           1621           3         41.4          24.1       0.7X
+PassThrough                                        1565           1588          32         42.9          23.3       1.0X
+RunLengthEncoding                                  1841           1861          28         36.5          27.4       0.9X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 SHORT Encode (Higher Skew):               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough(1.000)                                    7              7           0      10135.7           0.1       1.0X
-RunLengthEncoding(2.010)                           1659           1660           0         40.4          24.7       0.0X
+PassThrough(1.000)                                    8              9           1       8316.3           0.1       1.0X
+RunLengthEncoding(2.014)                           1481           1489          10         45.3          22.1       0.0X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 SHORT Decode (Higher Skew):               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough                                        1151           1151           1         58.3          17.2       1.0X
-RunLengthEncoding                                  1655           1655           0         40.5          24.7       0.7X
+PassThrough                                        1537           1538           1         43.7          22.9       1.0X
+RunLengthEncoding                                  1864           1870           9         36.0          27.8       0.8X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 INT Encode (Lower Skew):                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough(1.000)                                   23             23           0       2952.8           0.3       1.0X
-RunLengthEncoding(0.997)                           2356           2356           0         28.5          35.1       0.0X
-DictionaryEncoding(0.500)                          1402           1402           0         47.9          20.9       0.0X
-IntDelta(0.250)                                     213            213           0        315.2           3.2       0.1X
+PassThrough(1.000)                                   17             19           1       3926.8           0.3       1.0X
+RunLengthEncoding(1.002)                           1483           1487           5         45.3          22.1       0.0X
+DictionaryEncoding(0.500)                          1158           1165          10         58.0          17.2       0.0X
+IntDelta(0.250)                                     299            305           6        224.3           4.5       0.1X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 INT Decode (Lower Skew):                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough                                        1319           1319           1         50.9          19.7       1.0X
-RunLengthEncoding                                  1803           1806           5         37.2          26.9       0.7X
-DictionaryEncoding                                  931            931           0         72.1          13.9       1.4X
-IntDelta                                            817            821           4         82.2          12.2       1.6X
+PassThrough                                        1333           1343          14         50.3          19.9       1.0X
+RunLengthEncoding                                  1944           1945           2         34.5          29.0       0.7X
+DictionaryEncoding                                  925            945          18         72.6          13.8       1.4X
+IntDelta                                            784            794          10         85.6          11.7       1.7X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 INT Encode (Higher Skew):                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough(1.000)                                   23             23           0       2976.8           0.3       1.0X
-RunLengthEncoding(1.337)                           2552           2552           1         26.3          38.0       0.0X
-DictionaryEncoding(0.501)                          1377           1377           0         48.7          20.5       0.0X
-IntDelta(0.250)                                     213            214           2        315.3           3.2       0.1X
+PassThrough(1.000)                                   17             18           1       4021.0           0.2       1.0X
+RunLengthEncoding(1.338)                           1554           1576          32         43.2          23.1       0.0X
+DictionaryEncoding(0.501)                          1432           1438           8         46.9          21.3       0.0X
+IntDelta(0.250)                                     303            314          10        221.7           4.5       0.1X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 INT Decode (Higher Skew):                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough                                        1438           1439           1         46.7          21.4       1.0X
-RunLengthEncoding                                  1987           1988           1         33.8          29.6       0.7X
-DictionaryEncoding                                 1249           1250           0         53.7          18.6       1.2X
-IntDelta                                           1135           1136           3         59.2          16.9       1.3X
+PassThrough                                        1414           1434          28         47.5          21.1       1.0X
+RunLengthEncoding                                  2067           2077          14         32.5          30.8       0.7X
+DictionaryEncoding                                 1187           1190           3         56.5          17.7       1.2X
+IntDelta                                           1036           1072          51         64.8          15.4       1.4X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 LONG Encode (Lower Skew):                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough(1.000)                                   45             45           0       1487.9           0.7       1.0X
-RunLengthEncoding(0.750)                           2496           2496           1         26.9          37.2       0.0X
-DictionaryEncoding(0.250)                          1433           1433           1         46.8          21.4       0.0X
-LongDelta(0.125)                                    215            215           0        312.6           3.2       0.2X
+PassThrough(1.000)                                  379            383           6        177.3           5.6       1.0X
+RunLengthEncoding(0.754)                           1618           1623           7         41.5          24.1       0.2X
+DictionaryEncoding(0.250)                          1246           1283          53         53.9          18.6       0.3X
+LongDelta(0.125)                                    197            200           3        340.2           2.9       1.9X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 LONG Decode (Lower Skew):                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough                                        1468           1469           1         45.7          21.9       1.0X
-RunLengthEncoding                                  1977           1981           6         33.9          29.5       0.7X
-DictionaryEncoding                                 1248           1250           3         53.8          18.6       1.2X
-LongDelta                                           838            840           2         80.1          12.5       1.8X
+PassThrough                                        1567           1577          13         42.8          23.4       1.0X
+RunLengthEncoding                                  2064           2080          22         32.5          30.8       0.8X
+DictionaryEncoding                                 1252           1259          10         53.6          18.6       1.3X
+LongDelta                                           922            930           8         72.8          13.7       1.7X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 LONG Encode (Higher Skew):                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough(1.000)                                   47             47           0       1437.2           0.7       1.0X
-RunLengthEncoding(1.002)                           2743           2744           0         24.5          40.9       0.0X
-DictionaryEncoding(0.251)                          2016           2016           0         33.3          30.0       0.0X
-LongDelta(0.125)                                    215            217           5        312.1           3.2       0.2X
+PassThrough(1.000)                                  385            397           8        174.4           5.7       1.0X
+RunLengthEncoding(0.998)                           1687           1689           3         39.8          25.1       0.2X
+DictionaryEncoding(0.251)                          1453           1462          13         46.2          21.7       0.3X
+LongDelta(0.125)                                    190            200           7        353.5           2.8       2.0X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 LONG Decode (Higher Skew):                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough                                        1468           1468           0         45.7          21.9       1.0X
-RunLengthEncoding                                  2020           2021           2         33.2          30.1       0.7X
-DictionaryEncoding                                 1248           1248           0         53.8          18.6       1.2X
-LongDelta                                          1131           1134           4         59.4          16.8       1.3X
+PassThrough                                        1561           1564           3         43.0          23.3       1.0X
+RunLengthEncoding                                  2066           2075          12         32.5          30.8       0.8X
+DictionaryEncoding                                 1244           1252          11         53.9          18.5       1.3X
+LongDelta                                          1114           1117           4         60.3          16.6       1.4X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 STRING Encode:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough(1.000)                                   71             71           0        939.6           1.1       1.0X
-RunLengthEncoding(0.890)                           6050           6052           2         11.1          90.2       0.0X
-DictionaryEncoding(0.167)                          3723           3725           2         18.0          55.5       0.0X
+PassThrough(1.000)                                  584            592           8        114.9           8.7       1.0X
+RunLengthEncoding(0.889)                           2862           2872          14         23.4          42.7       0.2X
+DictionaryEncoding(0.167)                          3034           3053          26         22.1          45.2       0.2X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 STRING Decode:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough                                        2804           2815          16         23.9          41.8       1.0X
-RunLengthEncoding                                  3390           3391           1         19.8          50.5       0.8X
-DictionaryEncoding                                 2901           2905           5         23.1          43.2       1.0X
+PassThrough                                        2937           2939           3         22.8          43.8       1.0X
+RunLengthEncoding                                  3557           3574          24         18.9          53.0       0.8X
+DictionaryEncoding                                 2858           2890          46         23.5          42.6       1.0X
 
 
diff --git a/sql/core/benchmarks/CompressionSchemeBenchmark-results.txt b/sql/core/benchmarks/CompressionSchemeBenchmark-results.txt
index 3f6fbe35a7b86..8bb988a302d1b 100644
--- a/sql/core/benchmarks/CompressionSchemeBenchmark-results.txt
+++ b/sql/core/benchmarks/CompressionSchemeBenchmark-results.txt
@@ -2,136 +2,136 @@
 Compression Scheme Benchmark
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 BOOLEAN Encode:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough(1.000)                                    3              3           0      21114.6           0.0       1.0X
-RunLengthEncoding(2.505)                            694            696           4         96.7          10.3       0.0X
-BooleanBitSet(0.125)                                366            366           0        183.4           5.5       0.0X
+PassThrough(1.000)                                    4              4           0      16463.0           0.1       1.0X
+RunLengthEncoding(2.493)                            732            733           0         91.6          10.9       0.0X
+BooleanBitSet(0.125)                                484            484           0        138.7           7.2       0.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 BOOLEAN Decode:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough                                         145            145           0        464.2           2.2       1.0X
-RunLengthEncoding                                   735            735           0         91.3          10.9       0.2X
-BooleanBitSet                                      1437           1437           1         46.7          21.4       0.1X
+PassThrough                                         135            135           0        497.4           2.0       1.0X
+RunLengthEncoding                                   812            812           0         82.7          12.1       0.2X
+BooleanBitSet                                      1398           1398           0         48.0          20.8       0.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 SHORT Encode (Lower Skew):                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough(1.000)                                    7              7           0       9336.6           0.1       1.0X
-RunLengthEncoding(1.494)                           1912           1917           7         35.1          28.5       0.0X
+PassThrough(1.000)                                   10             10           0       6540.8           0.2       1.0X
+RunLengthEncoding(1.501)                           1497           1497           0         44.8          22.3       0.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 SHORT Decode (Lower Skew):                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough                                        1104           1104           0         60.8          16.4       1.0X
-RunLengthEncoding                                  1627           1628           0         41.2          24.3       0.7X
+PassThrough                                        1225           1227           3         54.8          18.3       1.0X
+RunLengthEncoding                                  1516           1517           0         44.3          22.6       0.8X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 SHORT Encode (Higher Skew):               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough(1.000)                                    7              7           0       9710.6           0.1       1.0X
-RunLengthEncoding(2.003)                           2021           2027           9         33.2          30.1       0.0X
+PassThrough(1.000)                                    8              8           0       8250.5           0.1       1.0X
+RunLengthEncoding(2.000)                           1568           1568           0         42.8          23.4       0.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 SHORT Decode (Higher Skew):               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough                                        1104           1104           0         60.8          16.5       1.0X
-RunLengthEncoding                                  1621           1621           0         41.4          24.1       0.7X
+PassThrough                                        1226           1227           1         54.7          18.3       1.0X
+RunLengthEncoding                                  1505           1505           0         44.6          22.4       0.8X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 INT Encode (Lower Skew):                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough(1.000)                                   24             24           0       2854.3           0.4       1.0X
-RunLengthEncoding(1.005)                           2395           2396           2         28.0          35.7       0.0X
-DictionaryEncoding(0.500)                          1366           1366           0         49.1          20.3       0.0X
-IntDelta(0.250)                                     286            287           0        234.2           4.3       0.1X
+PassThrough(1.000)                                   16             16           0       4116.8           0.2       1.0X
+RunLengthEncoding(1.003)                           1375           1375           0         48.8          20.5       0.0X
+DictionaryEncoding(0.500)                          1221           1222           2         55.0          18.2       0.0X
+IntDelta(0.250)                                     337            337           0        199.0           5.0       0.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 INT Decode (Lower Skew):                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough                                        1248           1248           0         53.8          18.6       1.0X
-RunLengthEncoding                                  1738           1739           2         38.6          25.9       0.7X
-DictionaryEncoding                                  969            970           0         69.2          14.4       1.3X
-IntDelta                                            777            779           1         86.3          11.6       1.6X
+PassThrough                                         925            926           1         72.5          13.8       1.0X
+RunLengthEncoding                                  1575           1576           1         42.6          23.5       0.6X
+DictionaryEncoding                                  834            834           0         80.5          12.4       1.1X
+IntDelta                                            731            731           0         91.8          10.9       1.3X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 INT Encode (Higher Skew):                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough(1.000)                                   23             23           0       2874.4           0.3       1.0X
-RunLengthEncoding(1.334)                           2581           2581           0         26.0          38.5       0.0X
-DictionaryEncoding(0.501)                          1490           1490           0         45.0          22.2       0.0X
-IntDelta(0.250)                                     286            286           0        234.5           4.3       0.1X
+PassThrough(1.000)                                   16             16           0       4162.0           0.2       1.0X
+RunLengthEncoding(1.340)                           1398           1399           1         48.0          20.8       0.0X
+DictionaryEncoding(0.501)                          1731           1731           0         38.8          25.8       0.0X
+IntDelta(0.250)                                     337            337           0        199.1           5.0       0.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 INT Decode (Higher Skew):                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough                                        1389           1389           0         48.3          20.7       1.0X
-RunLengthEncoding                                  1903           1903           0         35.3          28.4       0.7X
-DictionaryEncoding                                 1231           1232           1         54.5          18.3       1.1X
-IntDelta                                           1103           1108           7         60.8          16.4       1.3X
+PassThrough                                         978            979           0         68.6          14.6       1.0X
+RunLengthEncoding                                  1551           1551           1         43.3          23.1       0.6X
+DictionaryEncoding                                 1000           1001           1         67.1          14.9       1.0X
+IntDelta                                            794            796           2         84.5          11.8       1.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 LONG Encode (Lower Skew):                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough(1.000)                                   48             48           0       1405.2           0.7       1.0X
-RunLengthEncoding(0.757)                           2525           2525           1         26.6          37.6       0.0X
-DictionaryEncoding(0.250)                          1380           1381           1         48.6          20.6       0.0X
-LongDelta(0.125)                                    474            474           0        141.7           7.1       0.1X
+PassThrough(1.000)                                   32             32           0       2075.1           0.5       1.0X
+RunLengthEncoding(0.748)                           1477           1477           0         45.4          22.0       0.0X
+DictionaryEncoding(0.250)                          1470           1472           3         45.7          21.9       0.0X
+LongDelta(0.125)                                    496            496           0        135.2           7.4       0.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 LONG Decode (Lower Skew):                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough                                        1348           1349           0         49.8          20.1       1.0X
-RunLengthEncoding                                  1850           1851           2         36.3          27.6       0.7X
-DictionaryEncoding                                 1190           1192           3         56.4          17.7       1.1X
-LongDelta                                           801            801           0         83.8          11.9       1.7X
+PassThrough                                        1014           1015           1         66.2          15.1       1.0X
+RunLengthEncoding                                  1657           1658           1         40.5          24.7       0.6X
+DictionaryEncoding                                 1017           1020           4         66.0          15.2       1.0X
+LongDelta                                           697            697           0         96.3          10.4       1.5X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 LONG Encode (Higher Skew):                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough(1.000)                                   46             46           0       1451.2           0.7       1.0X
-RunLengthEncoding(1.003)                           2742           2743           1         24.5          40.9       0.0X
-DictionaryEncoding(0.251)                          1714           1715           0         39.1          25.5       0.0X
-LongDelta(0.125)                                    476            476           0        140.9           7.1       0.1X
+PassThrough(1.000)                                   32             32           0       2073.3           0.5       1.0X
+RunLengthEncoding(1.004)                           1559           1560           1         43.0          23.2       0.0X
+DictionaryEncoding(0.251)                          1656           1656           0         40.5          24.7       0.0X
+LongDelta(0.125)                                    496            496           0        135.3           7.4       0.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 LONG Decode (Higher Skew):                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough                                        1362           1363           1         49.3          20.3       1.0X
-RunLengthEncoding                                  1862           1863           1         36.0          27.7       0.7X
-DictionaryEncoding                                 1190           1192           3         56.4          17.7       1.1X
-LongDelta                                          1079           1082           4         62.2          16.1       1.3X
+PassThrough                                        1015           1016           1         66.1          15.1       1.0X
+RunLengthEncoding                                  1650           1653           4         40.7          24.6       0.6X
+DictionaryEncoding                                 1024           1025           1         65.5          15.3       1.0X
+LongDelta                                           832            837           5         80.7          12.4       1.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 STRING Encode:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough(1.000)                                   67             67           0        994.8           1.0       1.0X
-RunLengthEncoding(0.888)                           6135           6137           2         10.9          91.4       0.0X
-DictionaryEncoding(0.167)                          3747           3748           0         17.9          55.8       0.0X
+PassThrough(1.000)                                   63             63           0       1067.3           0.9       1.0X
+RunLengthEncoding(0.893)                           2337           2338           1         28.7          34.8       0.0X
+DictionaryEncoding(0.167)                          3541           3543           3         19.0          52.8       0.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 STRING Decode:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-PassThrough                                        3180           3185           8         21.1          47.4       1.0X
-RunLengthEncoding                                  3658           3660           3         18.3          54.5       0.9X
-DictionaryEncoding                                 3292           3295           4         20.4          49.1       1.0X
+PassThrough                                        2628           2635           9         25.5          39.2       1.0X
+RunLengthEncoding                                  3126           3128           2         21.5          46.6       0.8X
+DictionaryEncoding                                 2977           2979           3         22.5          44.4       0.9X
 
 
diff --git a/sql/core/benchmarks/DataSourceReadBenchmark-jdk11-results.txt b/sql/core/benchmarks/DataSourceReadBenchmark-jdk11-results.txt
index 595f69c8e1407..6578d5664cd30 100644
--- a/sql/core/benchmarks/DataSourceReadBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DataSourceReadBenchmark-jdk11-results.txt
@@ -2,251 +2,251 @@
 SQL Single Numeric Column Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 SQL Single TINYINT Column Scan:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                           28338          28589         356          0.6        1801.7       1.0X
-SQL Json                                           9273           9332          83          1.7         589.6       3.1X
-SQL Parquet Vectorized                              186            217          22         84.3          11.9     152.0X
-SQL Parquet MR                                     1951           1972          29          8.1         124.1      14.5X
-SQL ORC Vectorized                                  256            277          22         61.4          16.3     110.6X
-SQL ORC MR                                         1627           1717         127          9.7         103.4      17.4X
+SQL CSV                                           13405          13422          24          1.2         852.3       1.0X
+SQL Json                                          10723          10788          92          1.5         681.7       1.3X
+SQL Parquet Vectorized                              164            217          50         95.9          10.4      81.8X
+SQL Parquet MR                                     2349           2440         129          6.7         149.3       5.7X
+SQL ORC Vectorized                                  312            346          23         50.4          19.8      43.0X
+SQL ORC MR                                         1610           1659          69          9.8         102.4       8.3X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Parquet Reader Single TINYINT Column Scan:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-ParquetReader Vectorized                            208            223          13         75.8          13.2       1.0X
-ParquetReader Vectorized -> Row                      96             97           1        164.1           6.1       2.2X
+-------------------------------------------------------------------------------------------------------------------------
+ParquetReader Vectorized                             187            209          20         84.3          11.9       1.0X
+ParquetReader Vectorized -> Row                       89             95           5        177.6           5.6       2.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 SQL Single SMALLINT Column Scan:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                           28493          28516          33          0.6        1811.5       1.0X
-SQL Json                                          10257          10291          47          1.5         652.1       2.8X
-SQL Parquet Vectorized                              215            233          14         73.2          13.7     132.5X
-SQL Parquet MR                                     2384           2388           7          6.6         151.5      12.0X
-SQL ORC Vectorized                                  298            307           7         52.8          18.9      95.6X
-SQL ORC MR                                         1798           1814          22          8.7         114.3      15.8X
+SQL CSV                                           14214          14549         474          1.1         903.7       1.0X
+SQL Json                                          11866          11934          95          1.3         754.4       1.2X
+SQL Parquet Vectorized                              294            342          53         53.6          18.7      48.4X
+SQL Parquet MR                                     2929           3004         107          5.4         186.2       4.9X
+SQL ORC Vectorized                                  312            328          15         50.4          19.8      45.5X
+SQL ORC MR                                         2037           2097          84          7.7         129.5       7.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Parquet Reader Single SMALLINT Column Scan:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-ParquetReader Vectorized                            286            293           6         54.9          18.2       1.0X
-ParquetReader Vectorized -> Row                     154            179          57        102.3           9.8       1.9X
+--------------------------------------------------------------------------------------------------------------------------
+ParquetReader Vectorized                              249            266          18         63.1          15.8       1.0X
+ParquetReader Vectorized -> Row                       192            247          36         82.1          12.2       1.3X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 SQL Single INT Column Scan:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                           30821          30902         114          0.5        1959.5       1.0X
-SQL Json                                          10935          10944          13          1.4         695.3       2.8X
-SQL Parquet Vectorized                              203            213          12         77.6          12.9     152.1X
-SQL Parquet MR                                     2334           2351          24          6.7         148.4      13.2X
-SQL ORC Vectorized                                  281            286           4         56.0          17.9     109.6X
-SQL ORC MR                                         1943           2022         112          8.1         123.5      15.9X
+SQL CSV                                           15502          15817         446          1.0         985.6       1.0X
+SQL Json                                          12638          12646          11          1.2         803.5       1.2X
+SQL Parquet Vectorized                              193            256          44         81.7          12.2      80.5X
+SQL Parquet MR                                     2943           2953          14          5.3         187.1       5.3X
+SQL ORC Vectorized                                  324            370          34         48.5          20.6      47.8X
+SQL ORC MR                                         2110           2163          75          7.5         134.1       7.3X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Parquet Reader Single INT Column Scan:    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-ParquetReader Vectorized                            284            291           9         55.5          18.0       1.0X
-ParquetReader Vectorized -> Row                     277            281           6         56.8          17.6       1.0X
+ParquetReader Vectorized                            276            287          14         57.0          17.6       1.0X
+ParquetReader Vectorized -> Row                     309            320           9         50.9          19.6       0.9X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 SQL Single BIGINT Column Scan:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                           38264          38306          60          0.4        2432.7       1.0X
-SQL Json                                          14369          14371           3          1.1         913.6       2.7X
-SQL Parquet Vectorized                              313            319           6         50.3          19.9     122.3X
-SQL Parquet MR                                     2581           2602          30          6.1         164.1      14.8X
-SQL ORC Vectorized                                  423            432           9         37.2          26.9      90.4X
-SQL ORC MR                                         2108           2142          49          7.5         134.0      18.2X
+SQL CSV                                           20156          20694         761          0.8        1281.5       1.0X
+SQL Json                                          15228          15380         214          1.0         968.2       1.3X
+SQL Parquet Vectorized                              325            346          20         48.4          20.7      62.0X
+SQL Parquet MR                                     3144           3228         118          5.0         199.9       6.4X
+SQL ORC Vectorized                                  516            526           7         30.5          32.8      39.0X
+SQL ORC MR                                         2353           2367          19          6.7         149.6       8.6X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Parquet Reader Single BIGINT Column Scan:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-ParquetReader Vectorized                            401            409           8         39.2          25.5       1.0X
-ParquetReader Vectorized -> Row                     392            400          15         40.2          24.9       1.0X
+ParquetReader Vectorized                            372            396          24         42.3          23.6       1.0X
+ParquetReader Vectorized -> Row                     437            462          25         36.0          27.8       0.9X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 SQL Single FLOAT Column Scan:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                           36276          36300          34          0.4        2306.4       1.0X
-SQL Json                                          13691          14374         967          1.1         870.4       2.6X
-SQL Parquet Vectorized                              193            198           5         81.6          12.3     188.2X
-SQL Parquet MR                                     2361           2389          40          6.7         150.1      15.4X
-SQL ORC Vectorized                                  430            434           4         36.6          27.3      84.4X
-SQL ORC MR                                         2037           2072          50          7.7         129.5      17.8X
+SQL CSV                                           17413          17599         263          0.9        1107.1       1.0X
+SQL Json                                          14416          14453          53          1.1         916.5       1.2X
+SQL Parquet Vectorized                              181            225          35         86.8          11.5      96.1X
+SQL Parquet MR                                     2940           2996          78          5.3         186.9       5.9X
+SQL ORC Vectorized                                  470            494          29         33.5          29.9      37.1X
+SQL ORC MR                                         2351           2379          39          6.7         149.5       7.4X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Parquet Reader Single FLOAT Column Scan:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-ParquetReader Vectorized                            277            284          10         56.8          17.6       1.0X
-ParquetReader Vectorized -> Row                     274            276           4         57.5          17.4       1.0X
+ParquetReader Vectorized                            268            282          14         58.7          17.0       1.0X
+ParquetReader Vectorized -> Row                     298            321          18         52.8          18.9       0.9X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 SQL Single DOUBLE Column Scan:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                           39757          39761           5          0.4        2527.7       1.0X
-SQL Json                                          20049          20052           5          0.8        1274.7       2.0X
-SQL Parquet Vectorized                              310            318          10         50.7          19.7     128.3X
-SQL Parquet MR                                     2535           2571          52          6.2         161.2      15.7X
-SQL ORC Vectorized                                  537            543           8         29.3          34.1      74.1X
-SQL ORC MR                                         2132           2161          41          7.4         135.6      18.6X
+SQL CSV                                           21666          21697          43          0.7        1377.5       1.0X
+SQL Json                                          18307          18363          79          0.9        1163.9       1.2X
+SQL Parquet Vectorized                              310            337          22         50.7          19.7      69.9X
+SQL Parquet MR                                     3089           3103          19          5.1         196.4       7.0X
+SQL ORC Vectorized                                  589            617          31         26.7          37.5      36.8X
+SQL ORC MR                                         2307           2377          98          6.8         146.7       9.4X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Parquet Reader Single DOUBLE Column Scan:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-ParquetReader Vectorized                            390            394           5         40.3          24.8       1.0X
-ParquetReader Vectorized -> Row                     389            391           5         40.5          24.7       1.0X
+ParquetReader Vectorized                            400            415          18         39.3          25.4       1.0X
+ParquetReader Vectorized -> Row                     393            406          11         40.1          25.0       1.0X
 
 
 ================================================================================================
 Int and String Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Int and String Scan:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                           27215          27240          34          0.4        2595.5       1.0X
-SQL Json                                          12713          12783         100          0.8        1212.4       2.1X
-SQL Parquet Vectorized                             2265           2269           5          4.6         216.0      12.0X
-SQL Parquet MR                                     4477           4544          95          2.3         426.9       6.1X
-SQL ORC Vectorized                                 2388           2404          23          4.4         227.7      11.4X
-SQL ORC MR                                         4295           4305          15          2.4         409.6       6.3X
+SQL CSV                                           17703          17719          22          0.6        1688.3       1.0X
+SQL Json                                          13095          13168         103          0.8        1248.9       1.4X
+SQL Parquet Vectorized                             2253           2266          19          4.7         214.8       7.9X
+SQL Parquet MR                                     4913           4977          91          2.1         468.5       3.6X
+SQL ORC Vectorized                                 2457           2467          14          4.3         234.3       7.2X
+SQL ORC MR                                         4433           4464          44          2.4         422.8       4.0X
 
 
 ================================================================================================
 Repeated String Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Repeated String:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                           17544          17580          51          0.6        1673.1       1.0X
-SQL Json                                           8277           8328          71          1.3         789.4       2.1X
-SQL Parquet Vectorized                              674            682           7         15.6          64.3      26.0X
-SQL Parquet MR                                     1960           1972          17          5.3         187.0       8.9X
-SQL ORC Vectorized                                  551            558          11         19.0          52.6      31.8X
-SQL ORC MR                                         2047           2052           6          5.1         195.2       8.6X
+SQL CSV                                            9741           9804          89          1.1         929.0       1.0X
+SQL Json                                           8230           8401         241          1.3         784.9       1.2X
+SQL Parquet Vectorized                              618            650          31         17.0          58.9      15.8X
+SQL Parquet MR                                     2258           2311          75          4.6         215.4       4.3X
+SQL ORC Vectorized                                  608            629          15         17.3          58.0      16.0X
+SQL ORC MR                                         2466           2479          18          4.3         235.2       4.0X
 
 
 ================================================================================================
 Partitioned Table Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Partitioned Table:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Data column - CSV                                 40273          40290          24          0.4        2560.5       1.0X
-Data column - Json                                14420          14440          28          1.1         916.8       2.8X
-Data column - Parquet Vectorized                    336            342           6         46.8          21.4     119.8X
-Data column - Parquet MR                           2651           2652           2          5.9         168.5      15.2X
-Data column - ORC Vectorized                        444            451           9         35.4          28.2      90.7X
-Data column - ORC MR                               2342           2356          20          6.7         148.9      17.2X
-Partition column - CSV                            11307          11310           4          1.4         718.9       3.6X
-Partition column - Json                           12105          12115          14          1.3         769.6       3.3X
-Partition column - Parquet Vectorized                87             97          13        181.2           5.5     464.0X
-Partition column - Parquet MR                      1364           1368           7         11.5          86.7      29.5X
-Partition column - ORC Vectorized                    83             97          13        189.0           5.3     484.1X
-Partition column - ORC MR                          1424           1437          19         11.0          90.5      28.3X
-Both columns - CSV                                41896          42166         381          0.4        2663.7       1.0X
-Both columns - Json                               15852          15871          27          1.0        1007.8       2.5X
-Both columns - Parquet Vectorized                   379            383           5         41.5          24.1     106.2X
-Both columns - Parquet MR                          2889           2916          38          5.4         183.7      13.9X
-Both columns - ORC Vectorized                       581            582           2         27.1          36.9      69.3X
-Both columns - ORC MR                              2626           2641          22          6.0         166.9      15.3X
+Data column - CSV                                 24195          24573         534          0.7        1538.3       1.0X
+Data column - Json                                14746          14883         194          1.1         937.5       1.6X
+Data column - Parquet Vectorized                    352            385          34         44.7          22.4      68.7X
+Data column - Parquet MR                           3674           3694          27          4.3         233.6       6.6X
+Data column - ORC Vectorized                        480            505          26         32.8          30.5      50.4X
+Data column - ORC MR                               2913           3004         128          5.4         185.2       8.3X
+Partition column - CSV                             7527           7544          23          2.1         478.6       3.2X
+Partition column - Json                           11955          12051         135          1.3         760.1       2.0X
+Partition column - Parquet Vectorized                65             92          29        242.5           4.1     373.0X
+Partition column - Parquet MR                      1614           1628          21          9.7         102.6      15.0X
+Partition column - ORC Vectorized                    71             99          29        220.1           4.5     338.5X
+Partition column - ORC MR                          1761           1769          11          8.9         112.0      13.7X
+Both columns - CSV                                24077          24127          70          0.7        1530.8       1.0X
+Both columns - Json                               15286          15479         273          1.0         971.9       1.6X
+Both columns - Parquet Vectorized                   376            412          40         41.9          23.9      64.4X
+Both columns - Parquet MR                          3808           3826          26          4.1         242.1       6.4X
+Both columns - ORC Vectorized                       560            604          42         28.1          35.6      43.2X
+Both columns - ORC MR                              3046           3080          49          5.2         193.7       7.9X
 
 
 ================================================================================================
 String with Nulls Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 String with Nulls Scan (0.0%):            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                           20831          21141         439          0.5        1986.6       1.0X
-SQL Json                                          11720          11721           1          0.9        1117.7       1.8X
-SQL Parquet Vectorized                             1470           1475           7          7.1         140.2      14.2X
-SQL Parquet MR                                     3902           3902           0          2.7         372.1       5.3X
-ParquetReader Vectorized                           1074           1077           4          9.8         102.5      19.4X
-SQL ORC Vectorized                                 1289           1334          64          8.1         122.9      16.2X
-SQL ORC MR                                         3603           3612          13          2.9         343.6       5.8X
+SQL CSV                                           11805          12021         306          0.9        1125.8       1.0X
+SQL Json                                          12051          12105          77          0.9        1149.3       1.0X
+SQL Parquet Vectorized                             1474           1545         100          7.1         140.6       8.0X
+SQL Parquet MR                                     4488           4492           4          2.3         428.1       2.6X
+ParquetReader Vectorized                           1140           1140           1          9.2         108.7      10.4X
+SQL ORC Vectorized                                 1164           1178          20          9.0         111.0      10.1X
+SQL ORC MR                                         3745           3817         102          2.8         357.1       3.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 String with Nulls Scan (50.0%):           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                           21850          21910          85          0.5        2083.8       1.0X
-SQL Json                                           8651           8668          24          1.2         825.0       2.5X
-SQL Parquet Vectorized                             1079           1090          16          9.7         102.9      20.3X
-SQL Parquet MR                                     2906           2925          27          3.6         277.1       7.5X
-ParquetReader Vectorized                            951            954           4         11.0          90.7      23.0X
-SQL ORC Vectorized                                 1246           1250           5          8.4         118.8      17.5X
-SQL ORC MR                                         3146           3162          22          3.3         300.1       6.9X
+SQL CSV                                            9814           9837          33          1.1         936.0       1.0X
+SQL Json                                           9317           9445         182          1.1         888.5       1.1X
+SQL Parquet Vectorized                             1117           1155          52          9.4         106.6       8.8X
+SQL Parquet MR                                     3463           3538         106          3.0         330.3       2.8X
+ParquetReader Vectorized                           1033           1039           8         10.1          98.6       9.5X
+SQL ORC Vectorized                                 1307           1353          65          8.0         124.7       7.5X
+SQL ORC MR                                         3644           3690          65          2.9         347.5       2.7X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 String with Nulls Scan (95.0%):           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                           18993          19140         209          0.6        1811.3       1.0X
-SQL Json                                           5467           5469           2          1.9         521.4       3.5X
-SQL Parquet Vectorized                              240            248          10         43.8          22.8      79.3X
-SQL Parquet MR                                     1745           1753          12          6.0         166.4      10.9X
-ParquetReader Vectorized                            240            244           5         43.7          22.9      79.1X
-SQL ORC Vectorized                                  496            500           4         21.1          47.3      38.3X
-SQL ORC MR                                         1822           1827           8          5.8         173.7      10.4X
+SQL CSV                                            8145           8270         176          1.3         776.8       1.0X
+SQL Json                                           5714           5764          71          1.8         544.9       1.4X
+SQL Parquet Vectorized                              235            264          15         44.6          22.4      34.7X
+SQL Parquet MR                                     2398           2412          19          4.4         228.7       3.4X
+ParquetReader Vectorized                            248            262          11         42.3          23.6      32.9X
+SQL ORC Vectorized                                  430            462          37         24.4          41.0      18.9X
+SQL ORC MR                                         1983           1993          14          5.3         189.1       4.1X
 
 
 ================================================================================================
 Single Column Scan From Wide Columns
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Single Column Scan from 10 columns:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                            3907           3911           6          0.3        3726.3       1.0X
-SQL Json                                           3755           3763          12          0.3        3581.2       1.0X
-SQL Parquet Vectorized                               68             71           6         15.4          64.8      57.5X
-SQL Parquet MR                                      234            239           5          4.5         223.0      16.7X
-SQL ORC Vectorized                                   74             77           5         14.2          70.4      52.9X
-SQL ORC MR                                          203            204           2          5.2         193.3      19.3X
+SQL CSV                                            2448           2461          18          0.4        2334.3       1.0X
+SQL Json                                           3332           3370          53          0.3        3177.6       0.7X
+SQL Parquet Vectorized                               51             87          25         20.7          48.2      48.4X
+SQL Parquet MR                                      239            278          35          4.4         227.5      10.3X
+SQL ORC Vectorized                                   60             82          19         17.5          57.3      40.8X
+SQL ORC MR                                          197            219          26          5.3         188.3      12.4X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Single Column Scan from 50 columns:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                            7909           7927          25          0.1        7542.9       1.0X
-SQL Json                                          15014          15101         123          0.1       14318.8       0.5X
-SQL Parquet Vectorized                              105            128          22         10.0         100.0      75.4X
-SQL Parquet MR                                      275            283           9          3.8         261.9      28.8X
-SQL ORC Vectorized                                  104            116           9         10.1          98.9      76.3X
-SQL ORC MR                                          234            245          12          4.5         223.0      33.8X
+SQL CSV                                            6034           6061          39          0.2        5754.0       1.0X
+SQL Json                                          12232          12315         118          0.1       11665.4       0.5X
+SQL Parquet Vectorized                               73            120          30         14.4          69.6      82.6X
+SQL Parquet MR                                      316            368          44          3.3         301.1      19.1X
+SQL ORC Vectorized                                   76            122          36         13.7          72.9      79.0X
+SQL ORC MR                                          206            261          47          5.1         196.5      29.3X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Single Column Scan from 100 columns:      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                           13033          13129         136          0.1       12429.1       1.0X
-SQL Json                                          28298          29130        1176          0.0       26987.3       0.5X
-SQL Parquet Vectorized                              139            151           9          7.5         132.7      93.7X
-SQL Parquet MR                                      314            322           7          3.3         299.5      41.5X
-SQL ORC Vectorized                                  123            143          17          8.5         117.3     106.0X
-SQL ORC MR                                          260            272           9          4.0         248.1      50.1X
+SQL CSV                                           10307          10309           4          0.1        9829.0       1.0X
+SQL Json                                          23412          23539         180          0.0       22327.7       0.4X
+SQL Parquet Vectorized                              105            151          23         10.0          99.9      98.4X
+SQL Parquet MR                                      295            325          29          3.6         281.5      34.9X
+SQL ORC Vectorized                                   85            112          31         12.4          81.0     121.4X
+SQL ORC MR                                          212            255          66          4.9         202.3      48.6X
 
 
diff --git a/sql/core/benchmarks/DataSourceReadBenchmark-results.txt b/sql/core/benchmarks/DataSourceReadBenchmark-results.txt
index 40e8dfc77c712..fe083703ae0ea 100644
--- a/sql/core/benchmarks/DataSourceReadBenchmark-results.txt
+++ b/sql/core/benchmarks/DataSourceReadBenchmark-results.txt
@@ -2,251 +2,251 @@
 SQL Single Numeric Column Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 SQL Single TINYINT Column Scan:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                           24716          24743          38          0.6        1571.4       1.0X
-SQL Json                                           9669           9686          25          1.6         614.7       2.6X
-SQL Parquet Vectorized                              172            193          21         91.2          11.0     143.4X
-SQL Parquet MR                                     1929           1942          18          8.2         122.7      12.8X
-SQL ORC Vectorized                                  247            266          19         63.6          15.7      99.9X
-SQL ORC MR                                         1640           1660          29          9.6         104.3      15.1X
+SQL CSV                                           15943          15956          18          1.0        1013.6       1.0X
+SQL Json                                           9109           9158          70          1.7         579.1       1.8X
+SQL Parquet Vectorized                              168            191          16         93.8          10.7      95.1X
+SQL Parquet MR                                     1938           1950          17          8.1         123.2       8.2X
+SQL ORC Vectorized                                  191            199           6         82.2          12.2      83.3X
+SQL ORC MR                                         1523           1537          20         10.3          96.8      10.5X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Parquet Reader Single TINYINT Column Scan:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-ParquetReader Vectorized                            197            200           4         79.9          12.5       1.0X
-ParquetReader Vectorized -> Row                      96             98           3        164.1           6.1       2.1X
+-------------------------------------------------------------------------------------------------------------------------
+ParquetReader Vectorized                             203            206           3         77.5          12.9       1.0X
+ParquetReader Vectorized -> Row                       97            100           2        161.6           6.2       2.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 SQL Single SMALLINT Column Scan:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                           25320          25343          32          0.6        1609.8       1.0X
-SQL Json                                          10460          10465           8          1.5         665.0       2.4X
-SQL Parquet Vectorized                              206            218          13         76.5          13.1     123.2X
-SQL Parquet MR                                     2032           2036           6          7.7         129.2      12.5X
-SQL ORC Vectorized                                  295            301           4         53.4          18.7      85.9X
-SQL ORC MR                                         1867           1885          25          8.4         118.7      13.6X
+SQL CSV                                           17062          17089          38          0.9        1084.8       1.0X
+SQL Json                                           9718           9720           3          1.6         617.9       1.8X
+SQL Parquet Vectorized                              326            333           7         48.2          20.7      52.3X
+SQL Parquet MR                                     2305           2329          34          6.8         146.6       7.4X
+SQL ORC Vectorized                                  201            205           3         78.2          12.8      84.8X
+SQL ORC MR                                         1795           1796           0          8.8         114.1       9.5X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Parquet Reader Single SMALLINT Column Scan:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-ParquetReader Vectorized                            288            294           6         54.6          18.3       1.0X
-ParquetReader Vectorized -> Row                     252            254           4         62.3          16.0       1.1X
+--------------------------------------------------------------------------------------------------------------------------
+ParquetReader Vectorized                              333            339           7         47.2          21.2       1.0X
+ParquetReader Vectorized -> Row                       283            285           3         55.7          18.0       1.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 SQL Single INT Column Scan:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                           27385          27423          54          0.6        1741.1       1.0X
-SQL Json                                          10118          10133          20          1.6         643.3       2.7X
-SQL Parquet Vectorized                              180            189          10         87.4          11.4     152.1X
-SQL Parquet MR                                     2548           2552           6          6.2         162.0      10.7X
-SQL ORC Vectorized                                  306            312           8         51.4          19.4      89.5X
-SQL ORC MR                                         1882           1927          64          8.4         119.6      14.6X
+SQL CSV                                           18722          18809         123          0.8        1190.3       1.0X
+SQL Json                                          10192          10249          80          1.5         648.0       1.8X
+SQL Parquet Vectorized                              155            162           8        101.6           9.8     120.9X
+SQL Parquet MR                                     2348           2360          16          6.7         149.3       8.0X
+SQL ORC Vectorized                                  265            275           7         59.3          16.9      70.5X
+SQL ORC MR                                         1892           1938          65          8.3         120.3       9.9X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Parquet Reader Single INT Column Scan:    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-ParquetReader Vectorized                            255            260           7         61.7          16.2       1.0X
-ParquetReader Vectorized -> Row                     252            257           6         62.4          16.0       1.0X
+ParquetReader Vectorized                            243            251           7         64.8          15.4       1.0X
+ParquetReader Vectorized -> Row                     222            229           5         70.9          14.1       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 SQL Single BIGINT Column Scan:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                           36971          37037          94          0.4        2350.5       1.0X
-SQL Json                                          13285          13300          22          1.2         844.6       2.8X
-SQL Parquet Vectorized                              275            285           5         57.1          17.5     134.3X
-SQL Parquet MR                                     2599           2603           6          6.1         165.3      14.2X
-SQL ORC Vectorized                                  386            395           5         40.7          24.6      95.7X
-SQL ORC MR                                         2059           2075          22          7.6         130.9      18.0X
+SQL CSV                                           24299          24358          84          0.6        1544.9       1.0X
+SQL Json                                          13349          13429         114          1.2         848.7       1.8X
+SQL Parquet Vectorized                              215            241          59         73.3          13.6     113.2X
+SQL Parquet MR                                     2508           2508           0          6.3         159.4       9.7X
+SQL ORC Vectorized                                  323            330           6         48.7          20.5      75.2X
+SQL ORC MR                                         1993           2009          22          7.9         126.7      12.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Parquet Reader Single BIGINT Column Scan:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-ParquetReader Vectorized                            352            361          14         44.7          22.4       1.0X
-ParquetReader Vectorized -> Row                     386            392           8         40.7          24.6       0.9X
+ParquetReader Vectorized                            310            351          74         50.8          19.7       1.0X
+ParquetReader Vectorized -> Row                     281            297           8         55.9          17.9       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 SQL Single FLOAT Column Scan:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                           29272          29322          71          0.5        1861.1       1.0X
-SQL Json                                          15022          15099         108          1.0         955.1       1.9X
-SQL Parquet Vectorized                              172            178           6         91.5          10.9     170.2X
-SQL Parquet MR                                     2184           2206          31          7.2         138.9      13.4X
-SQL ORC Vectorized                                  477            485           6         32.9          30.4      61.3X
-SQL ORC MR                                         2036           2054          26          7.7         129.4      14.4X
+SQL CSV                                           19745          19811          93          0.8        1255.4       1.0X
+SQL Json                                          12523          12760         335          1.3         796.2       1.6X
+SQL Parquet Vectorized                              153            160           6        102.9           9.7     129.2X
+SQL Parquet MR                                     2325           2338          18          6.8         147.8       8.5X
+SQL ORC Vectorized                                  389            401           8         40.5          24.7      50.8X
+SQL ORC MR                                         2009           2009           1          7.8         127.7       9.8X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Parquet Reader Single FLOAT Column Scan:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-ParquetReader Vectorized                            251            255           5         62.6          16.0       1.0X
-ParquetReader Vectorized -> Row                     248            254           7         63.5          15.7       1.0X
+ParquetReader Vectorized                            240            244           4         65.5          15.3       1.0X
+ParquetReader Vectorized -> Row                     223            230           6         70.5          14.2       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 SQL Single DOUBLE Column Scan:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                           38020          38024           6          0.4        2417.2       1.0X
-SQL Json                                          20449          20463          19          0.8        1300.1       1.9X
-SQL Parquet Vectorized                              268            274           8         58.7          17.0     141.8X
-SQL Parquet MR                                     2484           2493          12          6.3         157.9      15.3X
-SQL ORC Vectorized                                  580            582           2         27.1          36.9      65.6X
-SQL ORC MR                                         2179           2199          29          7.2         138.5      17.5X
+SQL CSV                                           27223          27293          99          0.6        1730.8       1.0X
+SQL Json                                          18601          18646          63          0.8        1182.6       1.5X
+SQL Parquet Vectorized                              247            251           3         63.8          15.7     110.4X
+SQL Parquet MR                                     2724           2773          69          5.8         173.2      10.0X
+SQL ORC Vectorized                                  474            484          10         33.2          30.1      57.4X
+SQL ORC MR                                         2342           2368          37          6.7         148.9      11.6X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Parquet Reader Single DOUBLE Column Scan:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-ParquetReader Vectorized                            344            350           7         45.7          21.9       1.0X
-ParquetReader Vectorized -> Row                     346            352          12         45.5          22.0       1.0X
+ParquetReader Vectorized                            326            335          13         48.3          20.7       1.0X
+ParquetReader Vectorized -> Row                     358            365           7         44.0          22.7       0.9X
 
 
 ================================================================================================
 Int and String Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Int and String Scan:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                           27652          28221         804          0.4        2637.1       1.0X
-SQL Json                                          12827          12842          21          0.8        1223.3       2.2X
-SQL Parquet Vectorized                             2297           2311          19          4.6         219.1      12.0X
-SQL Parquet MR                                     4207           4217          15          2.5         401.2       6.6X
-SQL ORC Vectorized                                 2316           2342          36          4.5         220.9      11.9X
-SQL ORC MR                                         4158           4236         110          2.5         396.5       6.7X
+SQL CSV                                           18706          18716          15          0.6        1783.9       1.0X
+SQL Json                                          12665          12762         138          0.8        1207.8       1.5X
+SQL Parquet Vectorized                             2408           2419          15          4.4         229.6       7.8X
+SQL Parquet MR                                     4599           4620          30          2.3         438.6       4.1X
+SQL ORC Vectorized                                 2397           2400           3          4.4         228.6       7.8X
+SQL ORC MR                                         4267           4288          30          2.5         406.9       4.4X
 
 
 ================================================================================================
 Repeated String Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Repeated String:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                           19185          19343         224          0.5        1829.6       1.0X
-SQL Json                                           7682           7692          14          1.4         732.6       2.5X
-SQL Parquet Vectorized                              796            805           9         13.2          75.9      24.1X
-SQL Parquet MR                                     1880           1891          17          5.6         179.2      10.2X
-SQL ORC Vectorized                                  553            558           5         19.0          52.7      34.7X
-SQL ORC MR                                         2105           2128          32          5.0         200.8       9.1X
+SQL CSV                                           10822          10838          23          1.0        1032.0       1.0X
+SQL Json                                           7459           7488          41          1.4         711.4       1.5X
+SQL Parquet Vectorized                              875            895          26         12.0          83.5      12.4X
+SQL Parquet MR                                     1976           2002          37          5.3         188.4       5.5X
+SQL ORC Vectorized                                  533            539           8         19.7          50.9      20.3X
+SQL ORC MR                                         2191           2194           5          4.8         208.9       4.9X
 
 
 ================================================================================================
 Partitioned Table Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Partitioned Table:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Data column - CSV                                 43759          43811          73          0.4        2782.1       1.0X
-Data column - Json                                13866          13874          11          1.1         881.6       3.2X
-Data column - Parquet Vectorized                    292            302          10         53.9          18.5     150.1X
-Data column - Parquet MR                           2681           2697          23          5.9         170.5      16.3X
-Data column - ORC Vectorized                        416            422          12         37.8          26.4     105.2X
-Data column - ORC MR                               2256           2275          27          7.0         143.4      19.4X
-Partition column - CSV                            13909          13949          56          1.1         884.3       3.1X
-Partition column - Json                           11248          11252           7          1.4         715.1       3.9X
-Partition column - Parquet Vectorized                83             95          13        189.4           5.3     526.9X
-Partition column - Parquet MR                      1531           1532           2         10.3          97.3      28.6X
-Partition column - ORC Vectorized                    81             97          17        193.1           5.2     537.3X
-Partition column - ORC MR                          1557           1570          19         10.1          99.0      28.1X
-Both columns - CSV                                48341          48524         259          0.3        3073.4       0.9X
-Both columns - Json                               13636          13652          23          1.2         866.9       3.2X
-Both columns - Parquet Vectorized                   341            354          16         46.1          21.7     128.2X
-Both columns - Parquet MR                          2806           2825          26          5.6         178.4      15.6X
-Both columns - ORC Vectorized                       548            554           8         28.7          34.8      79.8X
-Both columns - ORC MR                              2602           2632          43          6.0         165.4      16.8X
+Data column - CSV                                 31196          31449         359          0.5        1983.4       1.0X
+Data column - Json                                16118          16855        1041          1.0        1024.8       1.9X
+Data column - Parquet Vectorized                    243            251           9         64.8          15.4     128.4X
+Data column - Parquet MR                           4213           4288         106          3.7         267.8       7.4X
+Data column - ORC Vectorized                        335            341           4         46.9          21.3      93.1X
+Data column - ORC MR                               3119           3146          38          5.0         198.3      10.0X
+Partition column - CSV                             9616           9915         423          1.6         611.3       3.2X
+Partition column - Json                           14136          14164          39          1.1         898.8       2.2X
+Partition column - Parquet Vectorized                64             70           6        243.9           4.1     483.8X
+Partition column - Parquet MR                      1954           1980          38          8.1         124.2      16.0X
+Partition column - ORC Vectorized                    67             74           8        233.4           4.3     462.9X
+Partition column - ORC MR                          2461           2479          26          6.4         156.4      12.7X
+Both columns - CSV                                30327          30666         479          0.5        1928.2       1.0X
+Both columns - Json                               18656          18789         188          0.8        1186.1       1.7X
+Both columns - Parquet Vectorized                   291            297           7         54.0          18.5     107.2X
+Both columns - Parquet MR                          4430           4443          19          3.6         281.6       7.0X
+Both columns - ORC Vectorized                       403            411          11         39.0          25.6      77.4X
+Both columns - ORC MR                              3580           3584           5          4.4         227.6       8.7X
 
 
 ================================================================================================
 String with Nulls Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 String with Nulls Scan (0.0%):            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                           22570          22687         166          0.5        2152.4       1.0X
-SQL Json                                          11103          11129          38          0.9        1058.8       2.0X
-SQL Parquet Vectorized                             1508           1516          12          7.0         143.8      15.0X
-SQL Parquet MR                                     3686           3692           9          2.8         351.5       6.1X
-ParquetReader Vectorized                           1117           1133          22          9.4         106.6      20.2X
-SQL ORC Vectorized                                 1195           1212          24          8.8         114.0      18.9X
-SQL ORC MR                                         3617           3618           3          2.9         344.9       6.2X
+SQL CSV                                           15606          15614          11          0.7        1488.3       1.0X
+SQL Json                                          15406          15451          63          0.7        1469.3       1.0X
+SQL Parquet Vectorized                             1555           1573          25          6.7         148.3      10.0X
+SQL Parquet MR                                     5369           5377          11          2.0         512.0       2.9X
+ParquetReader Vectorized                           1145           1150           7          9.2         109.2      13.6X
+SQL ORC Vectorized                                 1023           1027           6         10.2          97.6      15.3X
+SQL ORC MR                                         4421           4542         172          2.4         421.6       3.5X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 String with Nulls Scan (50.0%):           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                           19569          19819         354          0.5        1866.2       1.0X
-SQL Json                                           8292           8308          22          1.3         790.8       2.4X
-SQL Parquet Vectorized                             1107           1136          41          9.5         105.6      17.7X
-SQL Parquet MR                                     2784           2812          39          3.8         265.5       7.0X
-ParquetReader Vectorized                            990            994           5         10.6          94.4      19.8X
-SQL ORC Vectorized                                 1198           1199           2          8.8         114.2      16.3X
-SQL ORC MR                                         3164           3195          44          3.3         301.7       6.2X
+SQL CSV                                           11096          11159          90          0.9        1058.2       1.0X
+SQL Json                                          10797          11304         717          1.0        1029.7       1.0X
+SQL Parquet Vectorized                             1218           1230          16          8.6         116.2       9.1X
+SQL Parquet MR                                     3778           3806          40          2.8         360.3       2.9X
+ParquetReader Vectorized                           1108           1118          14          9.5         105.7      10.0X
+SQL ORC Vectorized                                 1361           1371          13          7.7         129.8       8.2X
+SQL ORC MR                                         4186           4196          14          2.5         399.2       2.7X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 String with Nulls Scan (95.0%):           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                           15940          15969          41          0.7        1520.1       1.0X
-SQL Json                                           4845           4845           0          2.2         462.0       3.3X
-SQL Parquet Vectorized                              243            249           6         43.1          23.2      65.5X
-SQL Parquet MR                                     1732           1751          26          6.1         165.2       9.2X
-ParquetReader Vectorized                            241            243           3         43.4          23.0      66.0X
-SQL ORC Vectorized                                  425            431           7         24.7          40.5      37.5X
-SQL ORC MR                                         1713           1728          20          6.1         163.4       9.3X
+SQL CSV                                            8803           8866          90          1.2         839.5       1.0X
+SQL Json                                           7220           7249          42          1.5         688.5       1.2X
+SQL Parquet Vectorized                              258            265           7         40.6          24.6      34.1X
+SQL Parquet MR                                     2760           2761           0          3.8         263.2       3.2X
+ParquetReader Vectorized                            277            283           5         37.8          26.4      31.7X
+SQL ORC Vectorized                                  514            522           6         20.4          49.1      17.1X
+SQL ORC MR                                         2523           2591          96          4.2         240.6       3.5X
 
 
 ================================================================================================
 Single Column Scan From Wide Columns
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Single Column Scan from 10 columns:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                            3838           3885          66          0.3        3660.4       1.0X
-SQL Json                                           3615           3615           0          0.3        3447.8       1.1X
-SQL Parquet Vectorized                               66             74           8         15.8          63.2      57.9X
-SQL Parquet MR                                      230            237           6          4.6         219.3      16.7X
-SQL ORC Vectorized                                   72             77           9         14.5          68.9      53.1X
-SQL ORC MR                                          194            201           5          5.4         185.3      19.7X
+SQL CSV                                            3022           3032          14          0.3        2881.9       1.0X
+SQL Json                                           4047           4051           5          0.3        3859.5       0.7X
+SQL Parquet Vectorized                               50             54           6         20.8          48.1      59.9X
+SQL Parquet MR                                      299            301           2          3.5         285.0      10.1X
+SQL ORC Vectorized                                   59             63          11         17.9          55.9      51.6X
+SQL ORC MR                                          255            259           5          4.1         243.4      11.8X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Single Column Scan from 50 columns:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                            8711           8754          60          0.1        8307.9       1.0X
-SQL Json                                          14414          14423          12          0.1       13746.5       0.6X
-SQL Parquet Vectorized                               97            106          12         10.8          92.7      89.6X
-SQL Parquet MR                                      267            274           7          3.9         254.2      32.7X
-SQL ORC Vectorized                                  100            104           7         10.5          95.1      87.4X
-SQL ORC MR                                          226            230           6          4.6         215.2      38.6X
+SQL CSV                                            7250           7252           3          0.1        6914.4       1.0X
+SQL Json                                          15641          15718         109          0.1       14916.8       0.5X
+SQL Parquet Vectorized                               66             72           8         15.9          62.9     110.0X
+SQL Parquet MR                                      320            323           3          3.3         305.0      22.7X
+SQL ORC Vectorized                                   72             77          11         14.6          68.6     100.9X
+SQL ORC MR                                          269            273           5          3.9         256.8      26.9X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Single Column Scan from 100 columns:      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-SQL CSV                                           14509          14596         123          0.1       13836.8       1.0X
-SQL Json                                          27545          27909         515          0.0       26269.1       0.5X
-SQL Parquet Vectorized                              141            151          13          7.4         134.8     102.7X
-SQL Parquet MR                                      313            341          23          3.4         298.4      46.4X
-SQL ORC Vectorized                                  121            129          15          8.7         115.4     119.9X
-SQL ORC MR                                          252            269          33          4.2         240.3      57.6X
+SQL CSV                                           10962          11340         535          0.1       10454.1       1.0X
+SQL Json                                          24951          25755        1137          0.0       23795.0       0.4X
+SQL Parquet Vectorized                               84             93           6         12.4          80.5     129.9X
+SQL Parquet MR                                      280            296          14          3.7         266.8      39.2X
+SQL ORC Vectorized                                   70             76           6         15.0          66.6     156.9X
+SQL ORC MR                                          231            242          13          4.5         220.1      47.5X
 
 
diff --git a/sql/core/benchmarks/DatasetBenchmark-jdk11-results.txt b/sql/core/benchmarks/DatasetBenchmark-jdk11-results.txt
index f07fae9121df1..8af16f90c40c2 100644
--- a/sql/core/benchmarks/DatasetBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DatasetBenchmark-jdk11-results.txt
@@ -2,45 +2,45 @@
 Dataset Benchmark
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 back-to-back map long:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-RDD                                               14574          14759         261          6.9         145.7       1.0X
-DataFrame                                          2468           2655         264         40.5          24.7       5.9X
-Dataset                                            3498           3533          50         28.6          35.0       4.2X
+RDD                                               13660          13836         249          7.3         136.6       1.0X
+DataFrame                                          2103           2125          30         47.5          21.0       6.5X
+Dataset                                            2899           2910          16         34.5          29.0       4.7X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 back-to-back map:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-RDD                                               17877          18133         361          5.6         178.8       1.0X
-DataFrame                                          5968           5991          33         16.8          59.7       3.0X
-Dataset                                           12638          12859         313          7.9         126.4       1.4X
+RDD                                               14939          14940           2          6.7         149.4       1.0X
+DataFrame                                          5377           5529         216         18.6          53.8       2.8X
+Dataset                                           15861          15923          88          6.3         158.6       0.9X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 back-to-back filter Long:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-RDD                                                3399           3464          92         29.4          34.0       1.0X
-DataFrame                                          1609           1628          28         62.2          16.1       2.1X
-Dataset                                            3637           3648          16         27.5          36.4       0.9X
+RDD                                                3803           3842          56         26.3          38.0       1.0X
+DataFrame                                          1359           1369          14         73.6          13.6       2.8X
+Dataset                                            3667           3668           1         27.3          36.7       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 back-to-back filter:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-RDD                                                4850           4859          13         20.6          48.5       1.0X
-DataFrame                                           211            244          21        472.9           2.1      22.9X
-Dataset                                            5864           6126         372         17.1          58.6       0.8X
+RDD                                                4572           4595          33         21.9          45.7       1.0X
+DataFrame                                           212            261          45        471.6           2.1      21.6X
+Dataset                                            5629           5776         208         17.8          56.3       0.8X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 aggregate:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-RDD sum                                            4821           4914         131         20.7          48.2       1.0X
-DataFrame sum                                        71             83           8       1412.4           0.7      68.1X
-Dataset sum using Aggregator                       6001           6012          16         16.7          60.0       0.8X
-Dataset complex Aggregator                        10247          10455         294          9.8         102.5       0.5X
+RDD sum                                            3528           3563          50         28.3          35.3       1.0X
+DataFrame sum                                        81            111          23       1240.3           0.8      43.8X
+Dataset sum using Aggregator                       5140           5164          34         19.5          51.4       0.7X
+Dataset complex Aggregator                         9815           9921         150         10.2          98.1       0.4X
 
 
diff --git a/sql/core/benchmarks/DatasetBenchmark-results.txt b/sql/core/benchmarks/DatasetBenchmark-results.txt
index 2774f14664230..965335f07cb8f 100644
--- a/sql/core/benchmarks/DatasetBenchmark-results.txt
+++ b/sql/core/benchmarks/DatasetBenchmark-results.txt
@@ -2,45 +2,45 @@
 Dataset Benchmark
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 back-to-back map long:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-RDD                                               12720          12777          80          7.9         127.2       1.0X
-DataFrame                                          2242           2501         366         44.6          22.4       5.7X
-Dataset                                            3040           3174         189         32.9          30.4       4.2X
+RDD                                               12276          12399         175          8.1         122.8       1.0X
+DataFrame                                          2017           2094         110         49.6          20.2       6.1X
+Dataset                                            3034           3044          15         33.0          30.3       4.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 back-to-back map:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-RDD                                               15865          15922          82          6.3         158.6       1.0X
-DataFrame                                          8423           8476          75         11.9          84.2       1.9X
-Dataset                                           17180          18142        1361          5.8         171.8       0.9X
+RDD                                               16325          16361          50          6.1         163.2       1.0X
+DataFrame                                          8463           8468           6         11.8          84.6       1.9X
+Dataset                                           22525          23091         801          4.4         225.2       0.7X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 back-to-back filter Long:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-RDD                                                2928           3009         114         34.1          29.3       1.0X
-DataFrame                                          1386           1427          59         72.2          13.9       2.1X
-Dataset                                            3448           3451           5         29.0          34.5       0.8X
+RDD                                                3133           3136           3         31.9          31.3       1.0X
+DataFrame                                          1194           1535         482         83.8          11.9       2.6X
+Dataset                                            3146           3156          13         31.8          31.5       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 back-to-back filter:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-RDD                                                5476           5483          10         18.3          54.8       1.0X
-DataFrame                                           209            235          23        479.1           2.1      26.2X
-Dataset                                            9433           9549         163         10.6          94.3       0.6X
+RDD                                                5334           5352          25         18.7          53.3       1.0X
+DataFrame                                           190            221          21        527.1           1.9      28.1X
+Dataset                                           10536          10630         133          9.5         105.4       0.5X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 aggregate:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-RDD sum                                            5146           5239         132         19.4          51.5       1.0X
-DataFrame sum                                        84             99          15       1196.9           0.8      61.6X
-Dataset sum using Aggregator                       8944           9021         109         11.2          89.4       0.6X
-Dataset complex Aggregator                        12832          13141         436          7.8         128.3       0.4X
+RDD sum                                            4970           5099         181         20.1          49.7       1.0X
+DataFrame sum                                        67             81           9       1483.8           0.7      73.8X
+Dataset sum using Aggregator                       9474           9771         420         10.6          94.7       0.5X
+Dataset complex Aggregator                        13975          14701        1028          7.2         139.7       0.4X
 
 
diff --git a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
index 61ca342a0d559..1267cee7e9736 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
@@ -2,456 +2,460 @@
 datetime +/- interval
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 datetime +/- interval:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date + interval(m)                                 1496           1569         104          6.7         149.6       1.0X
-date + interval(m, d)                              1514           1526          17          6.6         151.4       1.0X
-date + interval(m, d, ms)                          6231           6253          30          1.6         623.1       0.2X
-date - interval(m)                                 1481           1487           9          6.8         148.1       1.0X
-date - interval(m, d)                              1550           1552           2          6.5         155.0       1.0X
-date - interval(m, d, ms)                          6269           6272           4          1.6         626.9       0.2X
-timestamp + interval(m)                            3017           3056          54          3.3         301.7       0.5X
-timestamp + interval(m, d)                         3146           3148           3          3.2         314.6       0.5X
-timestamp + interval(m, d, ms)                     3446           3460          20          2.9         344.6       0.4X
-timestamp - interval(m)                            3045           3059          19          3.3         304.5       0.5X
-timestamp - interval(m, d)                         3147           3164          25          3.2         314.7       0.5X
-timestamp - interval(m, d, ms)                     3425           3442          25          2.9         342.5       0.4X
+date + interval(m)                                 1644           1661          24          6.1         164.4       1.0X
+date + interval(m, d)                              1571           1577           9          6.4         157.1       1.0X
+date + interval(m, d, ms)                          6292           6343          72          1.6         629.2       0.3X
+date - interval(m)                                 1490           1493           5          6.7         149.0       1.1X
+date - interval(m, d)                              1551           1563          16          6.4         155.1       1.1X
+date - interval(m, d, ms)                          6432           6458          36          1.6         643.2       0.3X
+timestamp + interval(m)                            3198           3252          76          3.1         319.8       0.5X
+timestamp + interval(m, d)                         3421           3435          20          2.9         342.1       0.5X
+timestamp + interval(m, d, ms)                     3685           3715          43          2.7         368.5       0.4X
+timestamp - interval(m)                            3249           3269          28          3.1         324.9       0.5X
+timestamp - interval(m, d)                         3375           3383          12          3.0         337.5       0.5X
+timestamp - interval(m, d, ms)                     3696           3706          15          2.7         369.6       0.4X
 
 
 ================================================================================================
 Extract components
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    332            336           5         30.1          33.2       1.0X
-cast to timestamp wholestage on                     333            344          10         30.0          33.3       1.0X
+cast to timestamp wholestage off                    279            293          20         35.8          27.9       1.0X
+cast to timestamp wholestage on                     274            286           9         36.4          27.4       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                   1246           1257          16          8.0         124.6       1.0X
-year of timestamp wholestage on                    1209           1218          12          8.3         120.9       1.0X
+year of timestamp wholestage off                   1182           1184           3          8.5         118.2       1.0X
+year of timestamp wholestage on                    1239           1251          10          8.1         123.9       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                1608           1616          11          6.2         160.8       1.0X
-quarter of timestamp wholestage on                 1540           1552          10          6.5         154.0       1.0X
+quarter of timestamp wholestage off                1238           1243           8          8.1         123.8       1.0X
+quarter of timestamp wholestage on                 1269           1291          14          7.9         126.9       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                  1242           1246           6          8.1         124.2       1.0X
-month of timestamp wholestage on                   1202           1212          11          8.3         120.2       1.0X
+month of timestamp wholestage off                  1178           1185          10          8.5         117.8       1.0X
+month of timestamp wholestage on                   1211           1230          12          8.3         121.1       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             1879           1885           8          5.3         187.9       1.0X
-weekofyear of timestamp wholestage on              1832           1845          10          5.5         183.2       1.0X
+weekofyear of timestamp wholestage off             1734           1740          10          5.8         173.4       1.0X
+weekofyear of timestamp wholestage on              1810           1826          16          5.5         181.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                    1236           1239           4          8.1         123.6       1.0X
-day of timestamp wholestage on                     1206           1219          17          8.3         120.6       1.0X
+day of timestamp wholestage off                    1195           1199           4          8.4         119.5       1.0X
+day of timestamp wholestage on                     1214           1231          16          8.2         121.4       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off              1308           1309           1          7.6         130.8       1.0X
-dayofyear of timestamp wholestage on               1239           1255          15          8.1         123.9       1.1X
+dayofyear of timestamp wholestage off              1283           1286           5          7.8         128.3       1.0X
+dayofyear of timestamp wholestage on               1269           1286          11          7.9         126.9       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off             1259           1263           5          7.9         125.9       1.0X
-dayofmonth of timestamp wholestage on              1201           1205           5          8.3         120.1       1.0X
+dayofmonth of timestamp wholestage off             1216           1239          32          8.2         121.6       1.0X
+dayofmonth of timestamp wholestage on              1201           1262          73          8.3         120.1       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off              1406           1410           6          7.1         140.6       1.0X
-dayofweek of timestamp wholestage on               1387           1402          15          7.2         138.7       1.0X
+dayofweek of timestamp wholestage off              1333           1343          14          7.5         133.3       1.0X
+dayofweek of timestamp wholestage on               1380           1399          18          7.2         138.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                1355           1367          18          7.4         135.5       1.0X
-weekday of timestamp wholestage on                 1311           1321          10          7.6         131.1       1.0X
+weekday of timestamp wholestage off                1294           1314          28          7.7         129.4       1.0X
+weekday of timestamp wholestage on                 1361           1379          12          7.3         136.1       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                    996            997           2         10.0          99.6       1.0X
-hour of timestamp wholestage on                     930            936           6         10.7          93.0       1.1X
+hour of timestamp wholestage off                    946           1039         132         10.6          94.6       1.0X
+hour of timestamp wholestage on                     995           1026          18         10.1          99.5       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                 1005           1012          10          9.9         100.5       1.0X
-minute of timestamp wholestage on                   949            952           3         10.5          94.9       1.1X
+minute of timestamp wholestage off                  958            979          28         10.4          95.8       1.0X
+minute of timestamp wholestage on                   952            964          12         10.5          95.2       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                 1013           1014           1          9.9         101.3       1.0X
-second of timestamp wholestage on                   933            934           2         10.7          93.3       1.1X
+second of timestamp wholestage off                  947            977          42         10.6          94.7       1.0X
+second of timestamp wholestage on                   987           1029          62         10.1          98.7       1.0X
 
 
 ================================================================================================
 Current date and time
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         291            293           2         34.3          29.1       1.0X
-current_date wholestage on                          280            284           3         35.7          28.0       1.0X
+current_date wholestage off                         254            254           1         39.4          25.4       1.0X
+current_date wholestage on                          268            275           7         37.3          26.8       0.9X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    311            324          18         32.1          31.1       1.0X
-current_timestamp wholestage on                     275            364          85         36.3          27.5       1.1X
+current_timestamp wholestage off                    263            296          47         38.1          26.3       1.0X
+current_timestamp wholestage on                     274            338          99         36.5          27.4       1.0X
 
 
 ================================================================================================
 Date arithmetic
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                        1077           1079           3          9.3         107.7       1.0X
-cast to date wholestage on                         1018           1030          14          9.8         101.8       1.1X
+cast to date wholestage off                        1053           1068          20          9.5         105.3       1.0X
+cast to date wholestage on                         1133           1151          19          8.8         113.3       0.9X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                            1257           1260           4          8.0         125.7       1.0X
-last_day wholestage on                             1218           1227          14          8.2         121.8       1.0X
+last_day wholestage off                            1178           1179           1          8.5         117.8       1.0X
+last_day wholestage on                             1372           1390          19          7.3         137.2       0.9X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                            1140           1141           1          8.8         114.0       1.0X
-next_day wholestage on                             1067           1076          11          9.4         106.7       1.1X
+next_day wholestage off                            1114           1131          24          9.0         111.4       1.0X
+next_day wholestage on                             1129           1147          13          8.9         112.9       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                            1062           1064           3          9.4         106.2       1.0X
-date_add wholestage on                             1046           1055          11          9.6         104.6       1.0X
+date_add wholestage off                            1039           1041           3          9.6         103.9       1.0X
+date_add wholestage on                             1036           1055          12          9.7         103.6       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                            1082           1083           1          9.2         108.2       1.0X
-date_sub wholestage on                             1047           1056          12          9.6         104.7       1.0X
+date_sub wholestage off                             989           1010          29         10.1          98.9       1.0X
+date_sub wholestage on                             1033           1045          14          9.7         103.3       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                          1430           1431           1          7.0         143.0       1.0X
-add_months wholestage on                           1441           1446           8          6.9         144.1       1.0X
+add_months wholestage off                          1328           1410         116          7.5         132.8       1.0X
+add_months wholestage on                           1439           1478          39          7.0         143.9       0.9X
 
 
 ================================================================================================
 Formatting dates
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         5442           5549         150          1.8         544.2       1.0X
-format date wholestage on                          5529           5655         236          1.8         552.9       1.0X
+format date wholestage off                         5802           5830          40          1.7         580.2       1.0X
+format date wholestage on                          5802           5837          40          1.7         580.2       1.0X
 
 
 ================================================================================================
 Formatting timestamps
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       7416           7440          34          1.3         741.6       1.0X
-from_unixtime wholestage on                        7372           7391          17          1.4         737.2       1.0X
+from_unixtime wholestage off                       6634           6669          49          1.5         663.4       1.0X
+from_unixtime wholestage on                        6724           6806          51          1.5         672.4       1.0X
 
 
 ================================================================================================
 Convert timestamps
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                  1316           1320           6          7.6         131.6       1.0X
-from_utc_timestamp wholestage on                   1268           1272           4          7.9         126.8       1.0X
+from_utc_timestamp wholestage off                  1389           1405          23          7.2         138.9       1.0X
+from_utc_timestamp wholestage on                   1363           1381          16          7.3         136.3       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                    1653           1657           6          6.0         165.3       1.0X
-to_utc_timestamp wholestage on                     1594           1599           4          6.3         159.4       1.0X
+to_utc_timestamp wholestage off                    1828           1837          12          5.5         182.8       1.0X
+to_utc_timestamp wholestage on                     1818           1838          23          5.5         181.8       1.0X
 
 
 ================================================================================================
 Intervals
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        341            343           3         29.4          34.1       1.0X
-cast interval wholestage on                         279            282           1         35.8          27.9       1.2X
+cast interval wholestage off                        288            289           1         34.7          28.8       1.0X
+cast interval wholestage on                         269            281          10         37.2          26.9       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1862           1865           4          5.4         186.2       1.0X
-datediff wholestage on                             1769           1783          15          5.7         176.9       1.1X
+datediff wholestage off                            2024           2028           5          4.9         202.4       1.0X
+datediff wholestage on                             2002           2044          32          5.0         200.2       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      5594           5599           7          1.8         559.4       1.0X
-months_between wholestage on                       5498           5508          11          1.8         549.8       1.0X
+months_between wholestage off                      6378           6398          28          1.6         637.8       1.0X
+months_between wholestage on                       6400           6467          65          1.6         640.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              2044           2127         117          0.5        2044.3       1.0X
-window wholestage on                              48057          48109          54          0.0       48056.9       0.0X
+window wholestage off                              1834           1837           5          0.5        1833.7       1.0X
+window wholestage on                              20763          21129         229          0.0       20763.3       0.1X
 
 
 ================================================================================================
 Truncation
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                     2540           2542           3          3.9         254.0       1.0X
-date_trunc YEAR wholestage on                      2486           2507          29          4.0         248.6       1.0X
+date_trunc YEAR wholestage off                     2786           2817          43          3.6         278.6       1.0X
+date_trunc YEAR wholestage on                      2805           2852          31          3.6         280.5       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                     2542           2543           3          3.9         254.2       1.0X
-date_trunc YYYY wholestage on                      2491           2498           9          4.0         249.1       1.0X
+date_trunc YYYY wholestage off                     2772           2813          58          3.6         277.2       1.0X
+date_trunc YYYY wholestage on                      2860           2894          39          3.5         286.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                       2545           2569          35          3.9         254.5       1.0X
-date_trunc YY wholestage on                        2487           2493           4          4.0         248.7       1.0X
+date_trunc YY wholestage off                       2902           2907           6          3.4         290.2       1.0X
+date_trunc YY wholestage on                        2801           2842          37          3.6         280.1       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                      2590           2590           1          3.9         259.0       1.0X
-date_trunc MON wholestage on                       2506           2520          12          4.0         250.6       1.0X
+date_trunc MON wholestage off                      2863           2876          18          3.5         286.3       1.0X
+date_trunc MON wholestage on                       2870           2905          22          3.5         287.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                    2595           2603          11          3.9         259.5       1.0X
-date_trunc MONTH wholestage on                     2505           2516          12          4.0         250.5       1.0X
+date_trunc MONTH wholestage off                    2847           2866          27          3.5         284.7       1.0X
+date_trunc MONTH wholestage on                     2799           2851          40          3.6         279.9       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                       2605           2612          10          3.8         260.5       1.0X
-date_trunc MM wholestage on                        2501           2515          11          4.0         250.1       1.0X
+date_trunc MM wholestage off                       2833           2857          33          3.5         283.3       1.0X
+date_trunc MM wholestage on                        2890           2907          20          3.5         289.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                      2225           2229           5          4.5         222.5       1.0X
-date_trunc DAY wholestage on                       2184           2196           9          4.6         218.4       1.0X
+date_trunc DAY wholestage off                      2331           2387          79          4.3         233.1       1.0X
+date_trunc DAY wholestage on                       2324           2371          51          4.3         232.4       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                       2232           2236           6          4.5         223.2       1.0X
-date_trunc DD wholestage on                        2183           2190           6          4.6         218.3       1.0X
+date_trunc DD wholestage off                       2314           2352          53          4.3         231.4       1.0X
+date_trunc DD wholestage on                        2351           2392          28          4.3         235.1       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                     2194           2199           7          4.6         219.4       1.0X
-date_trunc HOUR wholestage on                      2160           2166           5          4.6         216.0       1.0X
+date_trunc HOUR wholestage off                     2278           2319          58          4.4         227.8       1.0X
+date_trunc HOUR wholestage on                      2346           2373          32          4.3         234.6       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    390            396           9         25.7          39.0       1.0X
-date_trunc MINUTE wholestage on                     331            337           7         30.2          33.1       1.2X
+date_trunc MINUTE wholestage off                   2326           2346          28          4.3         232.6       1.0X
+date_trunc MINUTE wholestage on                    2310           2348          33          4.3         231.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    375            381           8         26.7          37.5       1.0X
-date_trunc SECOND wholestage on                     332            346          14         30.1          33.2       1.1X
+date_trunc SECOND wholestage off                    368            373           8         27.2          36.8       1.0X
+date_trunc SECOND wholestage on                     370            380           9         27.0          37.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                     2439           2443           6          4.1         243.9       1.0X
-date_trunc WEEK wholestage on                      2390           2409          32          4.2         239.0       1.0X
+date_trunc WEEK wholestage off                     2749           2773          35          3.6         274.9       1.0X
+date_trunc WEEK wholestage on                      2761           2835          49          3.6         276.1       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  3290           3292           4          3.0         329.0       1.0X
-date_trunc QUARTER wholestage on                   3214           3218           3          3.1         321.4       1.0X
+date_trunc QUARTER wholestage off                  3513           3574          86          2.8         351.3       1.0X
+date_trunc QUARTER wholestage on                   3492           3544          38          2.9         349.2       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           308            310           3         32.5          30.8       1.0X
-trunc year wholestage on                            289            293           6         34.7          28.9       1.1X
+trunc year wholestage off                           272            273           2         36.8          27.2       1.0X
+trunc year wholestage on                            314            323           6         31.9          31.4       0.9X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           309            311           3         32.4          30.9       1.0X
-trunc yyyy wholestage on                            289            294           7         34.6          28.9       1.1X
+trunc yyyy wholestage off                           270            272           4         37.1          27.0       1.0X
+trunc yyyy wholestage on                            315            325           8         31.8          31.5       0.9X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             311            311           0         32.2          31.1       1.0X
-trunc yy wholestage on                              288            294           7         34.7          28.8       1.1X
+trunc yy wholestage off                             273            275           3         36.7          27.3       1.0X
+trunc yy wholestage on                              324            327           4         30.9          32.4       0.8X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            313            313           0         32.0          31.3       1.0X
-trunc mon wholestage on                             287            290           2         34.8          28.7       1.1X
+trunc mon wholestage off                            284            285           2         35.2          28.4       1.0X
+trunc mon wholestage on                             318            327           7         31.4          31.8       0.9X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          310            310           0         32.3          31.0       1.0X
-trunc month wholestage on                           287            290           2         34.8          28.7       1.1X
+trunc month wholestage off                          272            274           3         36.8          27.2       1.0X
+trunc month wholestage on                           313            321           5         31.9          31.3       0.9X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             311            312           1         32.1          31.1       1.0X
-trunc mm wholestage on                              287            296           9         34.8          28.7       1.1X
+trunc mm wholestage off                             270            276           8         37.0          27.0       1.0X
+trunc mm wholestage on                              313            320           7         31.9          31.3       0.9X
 
 
 ================================================================================================
 Parsing
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     169            170           1          5.9         168.9       1.0X
-to timestamp str wholestage on                      161            168          11          6.2         161.0       1.0X
+to timestamp str wholestage off                     145            151           8          6.9         145.2       1.0X
+to timestamp str wholestage on                      147            158          19          6.8         146.5       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                        1360           1361           1          0.7        1359.6       1.0X
-to_timestamp wholestage on                         1362           1366           6          0.7        1362.0       1.0X
+to_timestamp wholestage off                        1205           1209           6          0.8        1205.2       1.0X
+to_timestamp wholestage on                         1217           1261          64          0.8        1217.1       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                   1343           1346           4          0.7        1342.6       1.0X
-to_unix_timestamp wholestage on                    1356           1359           2          0.7        1356.2       1.0X
+to_unix_timestamp wholestage off                   1438           1440           2          0.7        1438.0       1.0X
+to_unix_timestamp wholestage on                    1194           1207          12          0.8        1193.6       1.2X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          227            230           4          4.4         227.0       1.0X
-to date str wholestage on                           299            302           3          3.3         299.0       0.8X
+to date str wholestage off                          192            199          10          5.2         191.6       1.0X
+to date str wholestage on                           192            202          16          5.2         191.9       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             3413           3440          38          0.3        3413.0       1.0X
-to_date wholestage on                              3392           3402          12          0.3        3392.3       1.0X
+to_date wholestage off                             2021           2034          18          0.5        2021.5       1.0X
+to_date wholestage on                              1965           1976          15          0.5        1965.1       1.0X
 
 
 ================================================================================================
 Conversion from/to external types
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 To/from Java's date-time:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Date                                  410            415           7         12.2          82.0       1.0X
-From java.time.LocalDate                            332            333           1         15.1          66.4       1.2X
-Collect java.sql.Date                              1891           2542         829          2.6         378.1       0.2X
-Collect java.time.LocalDate                        1630           2138         441          3.1         326.0       0.3X
-From java.sql.Timestamp                             254            259           6         19.7          50.9       1.6X
-From java.time.Instant                              302            306           4         16.6          60.3       1.4X
-Collect longs                                      1134           1265         117          4.4         226.8       0.4X
-Collect java.sql.Timestamp                         1441           1458          16          3.5         288.1       0.3X
-Collect java.time.Instant                          1680           1928         253          3.0         336.0       0.2X
+From java.sql.Date                                  496            516          22         10.1          99.3       1.0X
+From java.time.LocalDate                            380            384           4         13.2          76.0       1.3X
+Collect java.sql.Date                              2476           2837         357          2.0         495.3       0.2X
+Collect java.time.LocalDate                        2281           2314          28          2.2         456.3       0.2X
+From java.sql.Timestamp                             268            289          30         18.7          53.6       1.9X
+From java.time.Instant                              335            351          24         14.9          67.1       1.5X
+Collect longs                                      1981           2184         188          2.5         396.2       0.3X
+Collect java.sql.Timestamp                         2137           2320         171          2.3         427.4       0.2X
+Collect java.time.Instant                          2291           2412         155          2.2         458.1       0.2X
+java.sql.Date to Hive string                       9522           9998         674          0.5        1904.3       0.1X
+java.time.LocalDate to Hive string                 8841           9082         407          0.6        1768.2       0.1X
+java.sql.Timestamp to Hive string                  9385           9733         521          0.5        1876.9       0.1X
+java.time.Instant to Hive string                  11756          11927         252          0.4        2351.2       0.0X
 
 
diff --git a/sql/core/benchmarks/DateTimeBenchmark-results.txt b/sql/core/benchmarks/DateTimeBenchmark-results.txt
index 7586295778bd8..21940b6ca6918 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-results.txt
@@ -2,456 +2,460 @@
 datetime +/- interval
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 datetime +/- interval:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date + interval(m)                                 1638           1701          89          6.1         163.8       1.0X
-date + interval(m, d)                              1785           1790           7          5.6         178.5       0.9X
-date + interval(m, d, ms)                          6229           6270          58          1.6         622.9       0.3X
-date - interval(m)                                 1500           1503           4          6.7         150.0       1.1X
-date - interval(m, d)                              1764           1766           3          5.7         176.4       0.9X
-date - interval(m, d, ms)                          6428           6446          25          1.6         642.8       0.3X
-timestamp + interval(m)                            2719           2722           4          3.7         271.9       0.6X
-timestamp + interval(m, d)                         3011           3021          14          3.3         301.1       0.5X
-timestamp + interval(m, d, ms)                     3405           3412           9          2.9         340.5       0.5X
-timestamp - interval(m)                            2759           2764           7          3.6         275.9       0.6X
-timestamp - interval(m, d)                         3094           3112          25          3.2         309.4       0.5X
-timestamp - interval(m, d, ms)                     3388           3392           5          3.0         338.8       0.5X
+date + interval(m)                                 1640           1688          68          6.1         164.0       1.0X
+date + interval(m, d)                              1840           1845           8          5.4         184.0       0.9X
+date + interval(m, d, ms)                          6767           6819          74          1.5         676.7       0.2X
+date - interval(m)                                 1436           1446          15          7.0         143.6       1.1X
+date - interval(m, d)                              1755           1766          17          5.7         175.5       0.9X
+date - interval(m, d, ms)                          7090           7100          13          1.4         709.0       0.2X
+timestamp + interval(m)                            2791           2793           2          3.6         279.1       0.6X
+timestamp + interval(m, d)                         3058           3068          14          3.3         305.8       0.5X
+timestamp + interval(m, d, ms)                     3414           3425          16          2.9         341.4       0.5X
+timestamp - interval(m)                            2827           2878          72          3.5         282.7       0.6X
+timestamp - interval(m, d)                         3112           3144          45          3.2         311.2       0.5X
+timestamp - interval(m, d, ms)                     3360           3433         103          3.0         336.0       0.5X
 
 
 ================================================================================================
 Extract components
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    319            323           6         31.4          31.9       1.0X
-cast to timestamp wholestage on                     304            311           8         32.9          30.4       1.0X
+cast to timestamp wholestage off                    263            272          13         38.1          26.3       1.0X
+cast to timestamp wholestage on                     256            281          22         39.0          25.6       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                   1234           1239           6          8.1         123.4       1.0X
-year of timestamp wholestage on                    1229           1244          22          8.1         122.9       1.0X
+year of timestamp wholestage off                   1236           1243          10          8.1         123.6       1.0X
+year of timestamp wholestage on                    1206           1221          14          8.3         120.6       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                1440           1445           7          6.9         144.0       1.0X
-quarter of timestamp wholestage on                 1358           1361           3          7.4         135.8       1.1X
+quarter of timestamp wholestage off                1406           1408           3          7.1         140.6       1.0X
+quarter of timestamp wholestage on                 1322           1352          37          7.6         132.2       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                  1239           1240           1          8.1         123.9       1.0X
-month of timestamp wholestage on                   1221           1239          26          8.2         122.1       1.0X
+month of timestamp wholestage off                  1201           1220          27          8.3         120.1       1.0X
+month of timestamp wholestage on                   1192           1212          17          8.4         119.2       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             1926           1934          11          5.2         192.6       1.0X
-weekofyear of timestamp wholestage on              1901           1911          10          5.3         190.1       1.0X
+weekofyear of timestamp wholestage off             1931           1936           8          5.2         193.1       1.0X
+weekofyear of timestamp wholestage on              1923           1959          31          5.2         192.3       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                    1225           1229           6          8.2         122.5       1.0X
-day of timestamp wholestage on                     1217           1225           7          8.2         121.7       1.0X
+day of timestamp wholestage off                    1251           1252           2          8.0         125.1       1.0X
+day of timestamp wholestage on                     1209           1224          17          8.3         120.9       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off              1290           1295           7          7.8         129.0       1.0X
-dayofyear of timestamp wholestage on               1262           1270           7          7.9         126.2       1.0X
+dayofyear of timestamp wholestage off              1290           1310          29          7.8         129.0       1.0X
+dayofyear of timestamp wholestage on               1228           1273          51          8.1         122.8       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off             1239           1239           1          8.1         123.9       1.0X
-dayofmonth of timestamp wholestage on              1215           1222           8          8.2         121.5       1.0X
+dayofmonth of timestamp wholestage off             1224           1283          83          8.2         122.4       1.0X
+dayofmonth of timestamp wholestage on              1211           1228          15          8.3         121.1       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off              1421           1422           2          7.0         142.1       1.0X
-dayofweek of timestamp wholestage on               1379           1388           8          7.3         137.9       1.0X
+dayofweek of timestamp wholestage off              1374           1374           0          7.3         137.4       1.0X
+dayofweek of timestamp wholestage on               1380           1396          10          7.2         138.0       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                1349           1351           2          7.4         134.9       1.0X
-weekday of timestamp wholestage on                 1320           1327           8          7.6         132.0       1.0X
+weekday of timestamp wholestage off                1352           1359          11          7.4         135.2       1.0X
+weekday of timestamp wholestage on                 1318           1336          16          7.6         131.8       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                   1024           1024           0          9.8         102.4       1.0X
-hour of timestamp wholestage on                     921            929          11         10.9          92.1       1.1X
+hour of timestamp wholestage off                    986            999          18         10.1          98.6       1.0X
+hour of timestamp wholestage on                     887            921          31         11.3          88.7       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                  977            982           6         10.2          97.7       1.0X
-minute of timestamp wholestage on                   927            929           2         10.8          92.7       1.1X
+minute of timestamp wholestage off                  936            956          28         10.7          93.6       1.0X
+minute of timestamp wholestage on                   867            899          22         11.5          86.7       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                  987            989           3         10.1          98.7       1.0X
-second of timestamp wholestage on                   923            926           5         10.8          92.3       1.1X
+second of timestamp wholestage off                  915            935          28         10.9          91.5       1.0X
+second of timestamp wholestage on                   917            948          27         10.9          91.7       1.0X
 
 
 ================================================================================================
 Current date and time
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         303            311          12         33.0          30.3       1.0X
-current_date wholestage on                          266            271           5         37.5          26.6       1.1X
+current_date wholestage off                         253            261          10         39.5          25.3       1.0X
+current_date wholestage on                          255            277          24         39.3          25.5       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    297            297           1         33.7          29.7       1.0X
-current_timestamp wholestage on                     264            272           7         37.8          26.4       1.1X
+current_timestamp wholestage off                    292            364         102         34.3          29.2       1.0X
+current_timestamp wholestage on                     265            314          69         37.8          26.5       1.1X
 
 
 ================================================================================================
 Date arithmetic
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                        1062           1063           2          9.4         106.2       1.0X
-cast to date wholestage on                         1007           1021          20          9.9         100.7       1.1X
+cast to date wholestage off                        1053           1054           2          9.5         105.3       1.0X
+cast to date wholestage on                          971            981           8         10.3          97.1       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                            1262           1265           5          7.9         126.2       1.0X
-last_day wholestage on                             1244           1256          14          8.0         124.4       1.0X
+last_day wholestage off                            1268           1280          18          7.9         126.8       1.0X
+last_day wholestage on                             1215           1255          25          8.2         121.5       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                            1119           1121           2          8.9         111.9       1.0X
-next_day wholestage on                             1057           1063           6          9.5         105.7       1.1X
+next_day wholestage off                            1064           1070           8          9.4         106.4       1.0X
+next_day wholestage on                             1026           1047          21          9.7         102.6       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                            1054           1059           7          9.5         105.4       1.0X
-date_add wholestage on                             1037           1069          52          9.6         103.7       1.0X
+date_add wholestage off                            1033           1040          10          9.7         103.3       1.0X
+date_add wholestage on                             1030           1034           4          9.7         103.0       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                            1054           1056           4          9.5         105.4       1.0X
-date_sub wholestage on                             1036           1040           4          9.7         103.6       1.0X
+date_sub wholestage off                            1027           1033           8          9.7         102.7       1.0X
+date_sub wholestage on                             1008           1020           9          9.9         100.8       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                          1408           1421          19          7.1         140.8       1.0X
-add_months wholestage on                           1434           1440           7          7.0         143.4       1.0X
+add_months wholestage off                          1393           1419          37          7.2         139.3       1.0X
+add_months wholestage on                           1390           1433          45          7.2         139.0       1.0X
 
 
 ================================================================================================
 Formatting dates
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         5937           6169         328          1.7         593.7       1.0X
-format date wholestage on                          5836           5878          74          1.7         583.6       1.0X
+format date wholestage off                         6010           6012           3          1.7         601.0       1.0X
+format date wholestage on                          5942           6007          39          1.7         594.2       1.0X
 
 
 ================================================================================================
 Formatting timestamps
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       8904           8914          14          1.1         890.4       1.0X
-from_unixtime wholestage on                        8918           8936          13          1.1         891.8       1.0X
+from_unixtime wholestage off                       7645           7703          82          1.3         764.5       1.0X
+from_unixtime wholestage on                        7748           7800          58          1.3         774.8       1.0X
 
 
 ================================================================================================
 Convert timestamps
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                  1110           1112           3          9.0         111.0       1.0X
-from_utc_timestamp wholestage on                   1115           1119           3          9.0         111.5       1.0X
+from_utc_timestamp wholestage off                  1215           1225          15          8.2         121.5       1.0X
+from_utc_timestamp wholestage on                   1073           1098          22          9.3         107.3       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                    1524           1525           1          6.6         152.4       1.0X
-to_utc_timestamp wholestage on                     1450           1458          14          6.9         145.0       1.1X
+to_utc_timestamp wholestage off                    1538           1547          12          6.5         153.8       1.0X
+to_utc_timestamp wholestage on                     1538           1586          39          6.5         153.8       1.0X
 
 
 ================================================================================================
 Intervals
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        341            342           1         29.3          34.1       1.0X
-cast interval wholestage on                         285            294           7         35.1          28.5       1.2X
+cast interval wholestage off                        300            311          15         33.3          30.0       1.0X
+cast interval wholestage on                         262            274          12         38.2          26.2       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1874           1881          10          5.3         187.4       1.0X
-datediff wholestage on                             1785           1791           3          5.6         178.5       1.0X
+datediff wholestage off                            1806           1815          12          5.5         180.6       1.0X
+datediff wholestage on                             1750           1789          44          5.7         175.0       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      5038           5042           5          2.0         503.8       1.0X
-months_between wholestage on                       4979           4987           8          2.0         497.9       1.0X
+months_between wholestage off                      5164           5208          62          1.9         516.4       1.0X
+months_between wholestage on                       5052           5099          43          2.0         505.2       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              1716           1841         177          0.6        1716.2       1.0X
-window wholestage on                              46024          46063          27          0.0       46024.1       0.0X
+window wholestage off                              1798           1814          23          0.6        1798.3       1.0X
+window wholestage on                              26389          26548         117          0.0       26388.6       0.1X
 
 
 ================================================================================================
 Truncation
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                     2428           2429           2          4.1         242.8       1.0X
-date_trunc YEAR wholestage on                      2451           2469          12          4.1         245.1       1.0X
+date_trunc YEAR wholestage off                     2497           2521          34          4.0         249.7       1.0X
+date_trunc YEAR wholestage on                      2427           2479          39          4.1         242.7       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                     2423           2426           3          4.1         242.3       1.0X
-date_trunc YYYY wholestage on                      2454           2462           8          4.1         245.4       1.0X
+date_trunc YYYY wholestage off                     2531           2545          20          4.0         253.1       1.0X
+date_trunc YYYY wholestage on                      2409           2452          25          4.2         240.9       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                       2421           2441          28          4.1         242.1       1.0X
-date_trunc YY wholestage on                        2453           2461           9          4.1         245.3       1.0X
+date_trunc YY wholestage off                       2531           2552          30          4.0         253.1       1.0X
+date_trunc YY wholestage on                        2419           2480          51          4.1         241.9       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                      2425           2427           3          4.1         242.5       1.0X
-date_trunc MON wholestage on                       2431           2438           9          4.1         243.1       1.0X
+date_trunc MON wholestage off                      2498           2510          17          4.0         249.8       1.0X
+date_trunc MON wholestage on                       2436           2464          20          4.1         243.6       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                    2427           2433           8          4.1         242.7       1.0X
-date_trunc MONTH wholestage on                     2429           2435           4          4.1         242.9       1.0X
+date_trunc MONTH wholestage off                    2505           2533          40          4.0         250.5       1.0X
+date_trunc MONTH wholestage on                     2477           2514          29          4.0         247.7       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                       2425           2431           9          4.1         242.5       1.0X
-date_trunc MM wholestage on                        2430           2435           4          4.1         243.0       1.0X
+date_trunc MM wholestage off                       2522           2525           4          4.0         252.2       1.0X
+date_trunc MM wholestage on                        2472           2499          30          4.0         247.2       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                      2117           2119           4          4.7         211.7       1.0X
-date_trunc DAY wholestage on                       2036           2118         174          4.9         203.6       1.0X
+date_trunc DAY wholestage off                      2111           2122          15          4.7         211.1       1.0X
+date_trunc DAY wholestage on                       2055           2092          33          4.9         205.5       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                       2116           2119           5          4.7         211.6       1.0X
-date_trunc DD wholestage on                        2035           2043          10          4.9         203.5       1.0X
+date_trunc DD wholestage off                       2097           2126          42          4.8         209.7       1.0X
+date_trunc DD wholestage on                        2072           2109          31          4.8         207.2       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                     2013           2014           2          5.0         201.3       1.0X
-date_trunc HOUR wholestage on                      2077           2088          13          4.8         207.7       1.0X
+date_trunc HOUR wholestage off                     2227           2248          30          4.5         222.7       1.0X
+date_trunc HOUR wholestage on                      2119           2146          24          4.7         211.9       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    363            368           8         27.6          36.3       1.0X
-date_trunc MINUTE wholestage on                     321            326           7         31.2          32.1       1.1X
+date_trunc MINUTE wholestage off                   2196           2197           2          4.6         219.6       1.0X
+date_trunc MINUTE wholestage on                    2076           2127          30          4.8         207.6       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    365            366           0         27.4          36.5       1.0X
-date_trunc SECOND wholestage on                     319            332          16         31.4          31.9       1.1X
+date_trunc SECOND wholestage off                    310            313           4         32.3          31.0       1.0X
+date_trunc SECOND wholestage on                     295            308          10         33.9          29.5       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                     2371           2376           7          4.2         237.1       1.0X
-date_trunc WEEK wholestage on                      2314           2322           8          4.3         231.4       1.0X
+date_trunc WEEK wholestage off                     2422           2455          48          4.1         242.2       1.0X
+date_trunc WEEK wholestage on                      2373           2422          33          4.2         237.3       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  3334           3335           1          3.0         333.4       1.0X
-date_trunc QUARTER wholestage on                   3286           3291           7          3.0         328.6       1.0X
+date_trunc QUARTER wholestage off                  3545           3556          15          2.8         354.5       1.0X
+date_trunc QUARTER wholestage on                   3294           3340          39          3.0         329.4       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           303            304           2         33.0          30.3       1.0X
-trunc year wholestage on                            283            291           5         35.3          28.3       1.1X
+trunc year wholestage off                           268            271           4         37.3          26.8       1.0X
+trunc year wholestage on                            279            287           6         35.8          27.9       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           324            330           8         30.9          32.4       1.0X
-trunc yyyy wholestage on                            283            291           9         35.3          28.3       1.1X
+trunc yyyy wholestage off                           289            291           2         34.5          28.9       1.0X
+trunc yyyy wholestage on                            279            291          11         35.9          27.9       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             304            305           3         32.9          30.4       1.0X
-trunc yy wholestage on                              283            302          28         35.3          28.3       1.1X
+trunc yy wholestage off                             276            280           5         36.2          27.6       1.0X
+trunc yy wholestage on                              277            299          20         36.1          27.7       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            315            319           6         31.7          31.5       1.0X
-trunc mon wholestage on                             284            287           5         35.3          28.4       1.1X
+trunc mon wholestage off                            266            268           3         37.6          26.6       1.0X
+trunc mon wholestage on                             285            298           9         35.1          28.5       0.9X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          305            314          13         32.8          30.5       1.0X
-trunc month wholestage on                           283            292          14         35.3          28.3       1.1X
+trunc month wholestage off                          264            271           9         37.8          26.4       1.0X
+trunc month wholestage on                           275            286          13         36.4          27.5       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             301            301           0         33.2          30.1       1.0X
-trunc mm wholestage on                              285            290           7         35.1          28.5       1.1X
+trunc mm wholestage off                             266            279          18         37.6          26.6       1.0X
+trunc mm wholestage on                              288            291           4         34.7          28.8       0.9X
 
 
 ================================================================================================
 Parsing
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     218            220           3          4.6         218.4       1.0X
-to timestamp str wholestage on                      213            216           6          4.7         212.5       1.0X
+to timestamp str wholestage off                     214            221           9          4.7         214.3       1.0X
+to timestamp str wholestage on                      207            212           5          4.8         207.1       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                        1838           1842           5          0.5        1838.1       1.0X
-to_timestamp wholestage on                         1952           1971          11          0.5        1952.2       0.9X
+to_timestamp wholestage off                        1253           1272          27          0.8        1253.2       1.0X
+to_timestamp wholestage on                         1240           1257          13          0.8        1239.5       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                   1987           1988           1          0.5        1986.9       1.0X
-to_unix_timestamp wholestage on                    1944           1948           3          0.5        1944.2       1.0X
+to_unix_timestamp wholestage off                   1281           1288          11          0.8        1280.5       1.0X
+to_unix_timestamp wholestage on                    1270           1314          26          0.8        1270.0       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          263            264           0          3.8         263.5       1.0X
-to date str wholestage on                           263            265           2          3.8         262.6       1.0X
+to date str wholestage off                          247            249           2          4.0         247.5       1.0X
+to date str wholestage on                           245            254           7          4.1         244.9       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             3560           3567          11          0.3        3559.7       1.0X
-to_date wholestage on                              3525           3534          10          0.3        3524.8       1.0X
+to_date wholestage off                             2314           2350          51          0.4        2314.1       1.0X
+to_date wholestage on                              2368           2393          43          0.4        2367.7       1.0X
 
 
 ================================================================================================
 Conversion from/to external types
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 To/from Java's date-time:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Date                                  405            416          16         12.3          81.0       1.0X
-From java.time.LocalDate                            344            352          14         14.5          68.8       1.2X
-Collect java.sql.Date                              1622           2553        1372          3.1         324.4       0.2X
-Collect java.time.LocalDate                        1464           1482          20          3.4         292.8       0.3X
-From java.sql.Timestamp                             248            258          15         20.2          49.6       1.6X
-From java.time.Instant                              237            243           7         21.1          47.4       1.7X
-Collect longs                                      1252           1341         109          4.0         250.5       0.3X
-Collect java.sql.Timestamp                         1515           1516           2          3.3         302.9       0.3X
-Collect java.time.Instant                          1379           1490          96          3.6         275.8       0.3X
+From java.sql.Date                                  403            409           7         12.4          80.6       1.0X
+From java.time.LocalDate                            326            327           2         15.4          65.1       1.2X
+Collect java.sql.Date                              1474           3120         NaN          3.4         294.8       0.3X
+Collect java.time.LocalDate                        1296           1338          39          3.9         259.2       0.3X
+From java.sql.Timestamp                             243            246           3         20.6          48.6       1.7X
+From java.time.Instant                              208            225          15         24.0          41.6       1.9X
+Collect longs                                      1200           1274          79          4.2         240.1       0.3X
+Collect java.sql.Timestamp                         1368           1462          94          3.7         273.6       0.3X
+Collect java.time.Instant                          1312           1432         170          3.8         262.5       0.3X
+java.sql.Date to Hive string                      16614          21580        1798          0.3        3322.8       0.0X
+java.time.LocalDate to Hive string                16907          18295         NaN          0.3        3381.3       0.0X
+java.sql.Timestamp to Hive string                 16926          17914        1288          0.3        3385.3       0.0X
+java.time.Instant to Hive string                  21324          22244        1592          0.2        4264.8       0.0X
 
 
diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
index 5d107c1e50fd8..3a93f70758aec 100644
--- a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
@@ -2,141 +2,153 @@
 Rebasing dates/timestamps in Parquet datasource
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Save DATE to parquet:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  20802          20802           0          4.8         208.0       1.0X
-before 1582, noop                                 10728          10728           0          9.3         107.3       1.9X
-after 1582, rebase off                            32924          32924           0          3.0         329.2       0.6X
-after 1582, rebase on                             32627          32627           0          3.1         326.3       0.6X
-before 1582, rebase off                           21576          21576           0          4.6         215.8       1.0X
-before 1582, rebase on                            23115          23115           0          4.3         231.2       0.9X
-
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+after 1582, noop                                  24957          24957           0          4.0         249.6       1.0X
+before 1582, noop                                 13309          13309           0          7.5         133.1       1.9X
+after 1582, rebase EXCEPTION                      38895          38895           0          2.6         388.9       0.6X
+after 1582, rebase LEGACY                         38477          38477           0          2.6         384.8       0.6X
+after 1582, rebase CORRECTED                      39442          39442           0          2.5         394.4       0.6X
+before 1582, rebase LEGACY                        31573          31573           0          3.2         315.7       0.8X
+before 1582, rebase CORRECTED                     28340          28340           0          3.5         283.4       0.9X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Load DATE from parquet:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   12880          12984         178          7.8         128.8       1.0X
-after 1582, vec off, rebase on                    13118          13255         174          7.6         131.2       1.0X
-after 1582, vec on, rebase off                     3645           3698          76         27.4          36.4       3.5X
-after 1582, vec on, rebase on                      3709           3727          15         27.0          37.1       3.5X
-before 1582, vec off, rebase off                  13014          13051          36          7.7         130.1       1.0X
-before 1582, vec off, rebase on                   14195          14242          48          7.0         142.0       0.9X
-before 1582, vec on, rebase off                    3680           3773          92         27.2          36.8       3.5X
-before 1582, vec on, rebase on                     4310           4381          87         23.2          43.1       3.0X
-
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+after 1582, vec off, rebase EXCEPTION             18452          18602         146          5.4         184.5       1.0X
+after 1582, vec off, rebase LEGACY                18570          18609          36          5.4         185.7       1.0X
+after 1582, vec off, rebase CORRECTED             18432          18646         191          5.4         184.3       1.0X
+after 1582, vec on, rebase EXCEPTION               4473           4581         120         22.4          44.7       4.1X
+after 1582, vec on, rebase LEGACY                  4449           4548          87         22.5          44.5       4.1X
+after 1582, vec on, rebase CORRECTED               4409           4435          32         22.7          44.1       4.2X
+before 1582, vec off, rebase LEGACY               19835          19969         116          5.0         198.4       0.9X
+before 1582, vec off, rebase CORRECTED            19035          19054          29          5.3         190.4       1.0X
+before 1582, vec on, rebase LEGACY                 5843           5929          91         17.1          58.4       3.2X
+before 1582, vec on, rebase CORRECTED              4274           4347          74         23.4          42.7       4.3X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Save TIMESTAMP_INT96 to parquet:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   3026           3026           0         33.1          30.3       1.0X
-before 1900, noop                                  2995           2995           0         33.4          30.0       1.0X
-after 1900, rebase off                            24294          24294           0          4.1         242.9       0.1X
-after 1900, rebase on                             24480          24480           0          4.1         244.8       0.1X
-before 1900, rebase off                           31120          31120           0          3.2         311.2       0.1X
-before 1900, rebase on                            31201          31201           0          3.2         312.0       0.1X
-
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+after 1900, noop                                   3409           3409           0         29.3          34.1       1.0X
+before 1900, noop                                  3664           3664           0         27.3          36.6       0.9X
+after 1900, rebase EXCEPTION                      33008          33008           0          3.0         330.1       0.1X
+after 1900, rebase LEGACY                         32753          32753           0          3.1         327.5       0.1X
+after 1900, rebase CORRECTED                      34472          34472           0          2.9         344.7       0.1X
+before 1900, rebase LEGACY                        38110          38110           0          2.6         381.1       0.1X
+before 1900, rebase CORRECTED                     35285          35285           0          2.8         352.9       0.1X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Load TIMESTAMP_INT96 from parquet:        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off, rebase off                   18283          18309          39          5.5         182.8       1.0X
-after 1900, vec off, rebase on                    18235          18269          53          5.5         182.4       1.0X
-after 1900, vec on, rebase off                     9563           9589          27         10.5          95.6       1.9X
-after 1900, vec on, rebase on                      9463           9554          81         10.6          94.6       1.9X
-before 1900, vec off, rebase off                  21377          21469         118          4.7         213.8       0.9X
-before 1900, vec off, rebase on                   21265          21422         156          4.7         212.7       0.9X
-before 1900, vec on, rebase off                   12481          12524          46          8.0         124.8       1.5X
-before 1900, vec on, rebase on                    12360          12482         105          8.1         123.6       1.5X
-
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+after 1900, vec off, rebase EXCEPTION             27084          27321         216          3.7         270.8       1.0X
+after 1900, vec off, rebase LEGACY                26556          27280         630          3.8         265.6       1.0X
+after 1900, vec off, rebase CORRECTED             26710          27071         319          3.7         267.1       1.0X
+after 1900, vec on, rebase EXCEPTION               8552           8693         168         11.7          85.5       3.2X
+after 1900, vec on, rebase LEGACY                  8853           8866          12         11.3          88.5       3.1X
+after 1900, vec on, rebase CORRECTED               8161           8327         221         12.3          81.6       3.3X
+before 1900, vec off, rebase LEGACY               30156          30951         702          3.3         301.6       0.9X
+before 1900, vec off, rebase CORRECTED            27258          27418         139          3.7         272.6       1.0X
+before 1900, vec on, rebase LEGACY                12023          12086          89          8.3         120.2       2.3X
+before 1900, vec on, rebase CORRECTED              8546           8554           6         11.7          85.5       3.2X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Save TIMESTAMP_MICROS to parquet:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   2984           2984           0         33.5          29.8       1.0X
-before 1900, noop                                  3003           3003           0         33.3          30.0       1.0X
-after 1900, rebase off                            15814          15814           0          6.3         158.1       0.2X
-after 1900, rebase on                             16250          16250           0          6.2         162.5       0.2X
-before 1900, rebase off                           16026          16026           0          6.2         160.3       0.2X
-before 1900, rebase on                            19735          19735           0          5.1         197.3       0.2X
-
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+after 1900, noop                                   3765           3765           0         26.6          37.7       1.0X
+before 1900, noop                                  3563           3563           0         28.1          35.6       1.1X
+after 1900, rebase EXCEPTION                      21884          21884           0          4.6         218.8       0.2X
+after 1900, rebase LEGACY                         23126          23126           0          4.3         231.3       0.2X
+after 1900, rebase CORRECTED                      24592          24592           0          4.1         245.9       0.2X
+before 1900, rebase LEGACY                        27854          27854           0          3.6         278.5       0.1X
+before 1900, rebase CORRECTED                     24708          24708           0          4.0         247.1       0.2X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Load TIMESTAMP_MICROS from parquet:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off, rebase off                   15292          15351          57          6.5         152.9       1.0X
-after 1900, vec off, rebase on                    15753          15886         173          6.3         157.5       1.0X
-after 1900, vec on, rebase off                     4879           4923          52         20.5          48.8       3.1X
-after 1900, vec on, rebase on                      5018           5038          18         19.9          50.2       3.0X
-before 1900, vec off, rebase off                  15257          15311          53          6.6         152.6       1.0X
-before 1900, vec off, rebase on                   18459          18537          90          5.4         184.6       0.8X
-before 1900, vec on, rebase off                    4929           4946          15         20.3          49.3       3.1X
-before 1900, vec on, rebase on                     8254           8339          93         12.1          82.5       1.9X
-
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+after 1900, vec off, rebase EXCEPTION             21394          21679         253          4.7         213.9       1.0X
+after 1900, vec off, rebase LEGACY                21749          21763          12          4.6         217.5       1.0X
+after 1900, vec off, rebase CORRECTED             22065          22265         188          4.5         220.7       1.0X
+after 1900, vec on, rebase EXCEPTION               6613           6766         218         15.1          66.1       3.2X
+after 1900, vec on, rebase LEGACY                  6134           6243         143         16.3          61.3       3.5X
+after 1900, vec on, rebase CORRECTED               6332           6368          54         15.8          63.3       3.4X
+before 1900, vec off, rebase LEGACY               24196          24829         579          4.1         242.0       0.9X
+before 1900, vec off, rebase CORRECTED            21725          22006         286          4.6         217.3       1.0X
+before 1900, vec on, rebase LEGACY                 9491           9774         250         10.5          94.9       2.3X
+before 1900, vec on, rebase CORRECTED              6136           6229          93         16.3          61.4       3.5X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Save TIMESTAMP_MILLIS to parquet:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   2987           2987           0         33.5          29.9       1.0X
-before 1900, noop                                  3002           3002           0         33.3          30.0       1.0X
-after 1900, rebase off                            15215          15215           0          6.6         152.1       0.2X
-after 1900, rebase on                             15577          15577           0          6.4         155.8       0.2X
-before 1900, rebase off                           15505          15505           0          6.4         155.1       0.2X
-before 1900, rebase on                            19143          19143           0          5.2         191.4       0.2X
-
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+after 1900, noop                                   3755           3755           0         26.6          37.5       1.0X
+before 1900, noop                                  3619           3619           0         27.6          36.2       1.0X
+after 1900, rebase EXCEPTION                      21243          21243           0          4.7         212.4       0.2X
+after 1900, rebase LEGACY                         21808          21808           0          4.6         218.1       0.2X
+after 1900, rebase CORRECTED                      22194          22194           0          4.5         221.9       0.2X
+before 1900, rebase LEGACY                        26637          26637           0          3.8         266.4       0.1X
+before 1900, rebase CORRECTED                     22130          22130           0          4.5         221.3       0.2X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Load TIMESTAMP_MILLIS from parquet:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off, rebase off                   15330          15436         113          6.5         153.3       1.0X
-after 1900, vec off, rebase on                    15515          15549          30          6.4         155.1       1.0X
-after 1900, vec on, rebase off                     6056           6074          19         16.5          60.6       2.5X
-after 1900, vec on, rebase on                      6376           6390          14         15.7          63.8       2.4X
-before 1900, vec off, rebase off                  15490          15523          36          6.5         154.9       1.0X
-before 1900, vec off, rebase on                   18613          18685         118          5.4         186.1       0.8X
-before 1900, vec on, rebase off                    6065           6109          41         16.5          60.6       2.5X
-before 1900, vec on, rebase on                     9052           9082          32         11.0          90.5       1.7X
+after 1900, vec off, rebase EXCEPTION             21977          22216         207          4.6         219.8       1.0X
+after 1900, vec off, rebase LEGACY                21929          22352         370          4.6         219.3       1.0X
+after 1900, vec off, rebase CORRECTED             22644          22788         171          4.4         226.4       1.0X
+after 1900, vec on, rebase EXCEPTION               7719           7818         101         13.0          77.2       2.8X
+after 1900, vec on, rebase LEGACY                  7736           7800          88         12.9          77.4       2.8X
+after 1900, vec on, rebase CORRECTED               7896           8002         107         12.7          79.0       2.8X
+before 1900, vec off, rebase LEGACY               25595          25846         218          3.9         255.9       0.9X
+before 1900, vec off, rebase CORRECTED            22795          23167         550          4.4         227.9       1.0X
+before 1900, vec on, rebase LEGACY                10800          10859          60          9.3         108.0       2.0X
+before 1900, vec on, rebase CORRECTED              7546           7635          82         13.3          75.5       2.9X
 
 
 ================================================================================================
 Rebasing dates/timestamps in ORC datasource
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Save DATE to ORC:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  20653          20653           0          4.8         206.5       1.0X
-before 1582, noop                                 10707          10707           0          9.3         107.1       1.9X
-after 1582                                        28288          28288           0          3.5         282.9       0.7X
-before 1582                                       19196          19196           0          5.2         192.0       1.1X
+after 1582, noop                                  25059          25059           0          4.0         250.6       1.0X
+before 1582, noop                                 12893          12893           0          7.8         128.9       1.9X
+after 1582                                        32469          32469           0          3.1         324.7       0.8X
+before 1582                                       25104          25104           0          4.0         251.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Load DATE from ORC:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               10596          10621          37          9.4         106.0       1.0X
-after 1582, vec on                                 3886           3938          61         25.7          38.9       2.7X
-before 1582, vec off                              10955          10984          26          9.1         109.6       1.0X
-before 1582, vec on                                4236           4258          24         23.6          42.4       2.5X
+after 1582, vec off                               11382          11543         194          8.8         113.8       1.0X
+after 1582, vec on                                 4964           5042          99         20.1          49.6       2.3X
+before 1582, vec off                              12443          12478          31          8.0         124.4       0.9X
+before 1582, vec on                                5685           5724          34         17.6          56.9       2.0X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Save TIMESTAMP to ORC:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   2988           2988           0         33.5          29.9       1.0X
-before 1900, noop                                  3007           3007           0         33.3          30.1       1.0X
-after 1900                                        18082          18082           0          5.5         180.8       0.2X
-before 1900                                       22669          22669           0          4.4         226.7       0.1X
+after 1900, noop                                   3528           3528           0         28.3          35.3       1.0X
+before 1900, noop                                  3536           3536           0         28.3          35.4       1.0X
+after 1900                                        19807          19807           0          5.0         198.1       0.2X
+before 1900                                       28731          28731           0          3.5         287.3       0.1X
 
-OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Load TIMESTAMP from ORC:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off                               12029          12035           9          8.3         120.3       1.0X
-after 1900, vec on                                 5194           5197           3         19.3          51.9       2.3X
-before 1900, vec off                              14853          14875          23          6.7         148.5       0.8X
-before 1900, vec on                                7797           7836          60         12.8          78.0       1.5X
+after 1900, vec off                               12936          13032          85          7.7         129.4       1.0X
+after 1900, vec on                                 5957           6006          50         16.8          59.6       2.2X
+before 1900, vec off                              16065          16193         115          6.2         160.7       0.8X
+before 1900, vec on                                8611           8836         196         11.6          86.1       1.5X
 
 
diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
index f2af3cc000f04..fe3efcabd7eb6 100644
--- a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
@@ -2,141 +2,153 @@
 Rebasing dates/timestamps in Parquet datasource
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Save DATE to parquet:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  23567          23567           0          4.2         235.7       1.0X
-before 1582, noop                                 10570          10570           0          9.5         105.7       2.2X
-after 1582, rebase off                            35335          35335           0          2.8         353.3       0.7X
-after 1582, rebase on                             35645          35645           0          2.8         356.5       0.7X
-before 1582, rebase off                           21824          21824           0          4.6         218.2       1.1X
-before 1582, rebase on                            22532          22532           0          4.4         225.3       1.0X
-
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+after 1582, noop                                  24145          24145           0          4.1         241.4       1.0X
+before 1582, noop                                  9863           9863           0         10.1          98.6       2.4X
+after 1582, rebase EXCEPTION                      36123          36123           0          2.8         361.2       0.7X
+after 1582, rebase LEGACY                         36800          36800           0          2.7         368.0       0.7X
+after 1582, rebase CORRECTED                      37083          37083           0          2.7         370.8       0.7X
+before 1582, rebase LEGACY                        27894          27894           0          3.6         278.9       0.9X
+before 1582, rebase CORRECTED                     22720          22720           0          4.4         227.2       1.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Load DATE from parquet:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   13194          13266          81          7.6         131.9       1.0X
-after 1582, vec off, rebase on                    13402          13466          89          7.5         134.0       1.0X
-after 1582, vec on, rebase off                     3627           3657          29         27.6          36.3       3.6X
-after 1582, vec on, rebase on                      3818           3839          26         26.2          38.2       3.5X
-before 1582, vec off, rebase off                  13075          13146         115          7.6         130.7       1.0X
-before 1582, vec off, rebase on                   13794          13804          13          7.2         137.9       1.0X
-before 1582, vec on, rebase off                    3655           3675          21         27.4          36.6       3.6X
-before 1582, vec on, rebase on                     4579           4634          72         21.8          45.8       2.9X
-
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+after 1582, vec off, rebase EXCEPTION             13975          14142         207          7.2         139.7       1.0X
+after 1582, vec off, rebase LEGACY                14288          14464         165          7.0         142.9       1.0X
+after 1582, vec off, rebase CORRECTED             14192          14398         332          7.0         141.9       1.0X
+after 1582, vec on, rebase EXCEPTION               3418           3473          56         29.3          34.2       4.1X
+after 1582, vec on, rebase LEGACY                  3708           3794          91         27.0          37.1       3.8X
+after 1582, vec on, rebase CORRECTED               3474           3542          67         28.8          34.7       4.0X
+before 1582, vec off, rebase LEGACY               14162          14333         198          7.1         141.6       1.0X
+before 1582, vec off, rebase CORRECTED            14201          14254          48          7.0         142.0       1.0X
+before 1582, vec on, rebase LEGACY                 4497           4538          70         22.2          45.0       3.1X
+before 1582, vec on, rebase CORRECTED              3371           3420          46         29.7          33.7       4.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Save TIMESTAMP_INT96 to parquet:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   2671           2671           0         37.4          26.7       1.0X
-before 1900, noop                                  2685           2685           0         37.2          26.8       1.0X
-after 1900, rebase off                            23899          23899           0          4.2         239.0       0.1X
-after 1900, rebase on                             24030          24030           0          4.2         240.3       0.1X
-before 1900, rebase off                           30178          30178           0          3.3         301.8       0.1X
-before 1900, rebase on                            30127          30127           0          3.3         301.3       0.1X
-
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+after 1900, noop                                   2359           2359           0         42.4          23.6       1.0X
+before 1900, noop                                  2500           2500           0         40.0          25.0       0.9X
+after 1900, rebase EXCEPTION                      21442          21442           0          4.7         214.4       0.1X
+after 1900, rebase LEGACY                         21791          21791           0          4.6         217.9       0.1X
+after 1900, rebase CORRECTED                      22458          22458           0          4.5         224.6       0.1X
+before 1900, rebase LEGACY                        26749          26749           0          3.7         267.5       0.1X
+before 1900, rebase CORRECTED                     22908          22908           0          4.4         229.1       0.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Load TIMESTAMP_INT96 from parquet:        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off, rebase off                   16613          16685          75          6.0         166.1       1.0X
-after 1900, vec off, rebase on                    16487          16541          47          6.1         164.9       1.0X
-after 1900, vec on, rebase off                     8840           8870          49         11.3          88.4       1.9X
-after 1900, vec on, rebase on                      8795           8813          20         11.4          87.9       1.9X
-before 1900, vec off, rebase off                  20400          20441          62          4.9         204.0       0.8X
-before 1900, vec off, rebase on                   20430          20481          60          4.9         204.3       0.8X
-before 1900, vec on, rebase off                   12211          12290          73          8.2         122.1       1.4X
-before 1900, vec on, rebase on                    12231          12321          95          8.2         122.3       1.4X
-
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+after 1900, vec off, rebase EXCEPTION             18065          18166          91          5.5         180.7       1.0X
+after 1900, vec off, rebase LEGACY                17358          17602         217          5.8         173.6       1.0X
+after 1900, vec off, rebase CORRECTED             17900          18615        1194          5.6         179.0       1.0X
+after 1900, vec on, rebase EXCEPTION               8235           8301         100         12.1          82.3       2.2X
+after 1900, vec on, rebase LEGACY                  8295           8476         225         12.1          83.0       2.2X
+after 1900, vec on, rebase CORRECTED               8128           8224          98         12.3          81.3       2.2X
+before 1900, vec off, rebase LEGACY               21413          21906         507          4.7         214.1       0.8X
+before 1900, vec off, rebase CORRECTED            18217          18288         103          5.5         182.2       1.0X
+before 1900, vec on, rebase LEGACY                12115          12260         138          8.3         121.2       1.5X
+before 1900, vec on, rebase CORRECTED              8742           8772          42         11.4          87.4       2.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Save TIMESTAMP_MICROS to parquet:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   2836           2836           0         35.3          28.4       1.0X
-before 1900, noop                                  2812           2812           0         35.6          28.1       1.0X
-after 1900, rebase off                            15976          15976           0          6.3         159.8       0.2X
-after 1900, rebase on                             16197          16197           0          6.2         162.0       0.2X
-before 1900, rebase off                           16140          16140           0          6.2         161.4       0.2X
-before 1900, rebase on                            20410          20410           0          4.9         204.1       0.1X
-
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+after 1900, noop                                   2556           2556           0         39.1          25.6       1.0X
+before 1900, noop                                  2619           2619           0         38.2          26.2       1.0X
+after 1900, rebase EXCEPTION                      18377          18377           0          5.4         183.8       0.1X
+after 1900, rebase LEGACY                         18986          18986           0          5.3         189.9       0.1X
+after 1900, rebase CORRECTED                      20329          20329           0          4.9         203.3       0.1X
+before 1900, rebase LEGACY                        24590          24590           0          4.1         245.9       0.1X
+before 1900, rebase CORRECTED                     19737          19737           0          5.1         197.4       0.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Load TIMESTAMP_MICROS from parquet:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off, rebase off                   15297          15324          40          6.5         153.0       1.0X
-after 1900, vec off, rebase on                    15771          15832          59          6.3         157.7       1.0X
-after 1900, vec on, rebase off                     4922           4949          32         20.3          49.2       3.1X
-after 1900, vec on, rebase on                      5392           5411          17         18.5          53.9       2.8X
-before 1900, vec off, rebase off                  15227          15385         141          6.6         152.3       1.0X
-before 1900, vec off, rebase on                   19611          19658          41          5.1         196.1       0.8X
-before 1900, vec on, rebase off                    4965           5013          54         20.1          49.6       3.1X
-before 1900, vec on, rebase on                     9847           9873          43         10.2          98.5       1.6X
-
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+after 1900, vec off, rebase EXCEPTION             16350          16622         272          6.1         163.5       1.0X
+after 1900, vec off, rebase LEGACY                15887          16060         209          6.3         158.9       1.0X
+after 1900, vec off, rebase CORRECTED             16156          16349         175          6.2         161.6       1.0X
+after 1900, vec on, rebase EXCEPTION               4902           4922          32         20.4          49.0       3.3X
+after 1900, vec on, rebase LEGACY                  5245           5354         135         19.1          52.5       3.1X
+after 1900, vec on, rebase CORRECTED               4905           5019          99         20.4          49.1       3.3X
+before 1900, vec off, rebase LEGACY               19924          20162         240          5.0         199.2       0.8X
+before 1900, vec off, rebase CORRECTED            16070          16180         162          6.2         160.7       1.0X
+before 1900, vec on, rebase LEGACY                 9381           9522         127         10.7          93.8       1.7X
+before 1900, vec on, rebase CORRECTED              4942           4990          49         20.2          49.4       3.3X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Save TIMESTAMP_MILLIS to parquet:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   2818           2818           0         35.5          28.2       1.0X
-before 1900, noop                                  2805           2805           0         35.6          28.1       1.0X
-after 1900, rebase off                            15182          15182           0          6.6         151.8       0.2X
-after 1900, rebase on                             15614          15614           0          6.4         156.1       0.2X
-before 1900, rebase off                           15404          15404           0          6.5         154.0       0.2X
-before 1900, rebase on                            19747          19747           0          5.1         197.5       0.1X
-
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+after 1900, noop                                   2583           2583           0         38.7          25.8       1.0X
+before 1900, noop                                  2514           2514           0         39.8          25.1       1.0X
+after 1900, rebase EXCEPTION                      16857          16857           0          5.9         168.6       0.2X
+after 1900, rebase LEGACY                         17211          17211           0          5.8         172.1       0.2X
+after 1900, rebase CORRECTED                      18908          18908           0          5.3         189.1       0.1X
+before 1900, rebase LEGACY                        22901          22901           0          4.4         229.0       0.1X
+before 1900, rebase CORRECTED                     19047          19047           0          5.3         190.5       0.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Load TIMESTAMP_MILLIS from parquet:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off, rebase off                   15622          15649          24          6.4         156.2       1.0X
-after 1900, vec off, rebase on                    15572          15677         119          6.4         155.7       1.0X
-after 1900, vec on, rebase off                     6345           6358          15         15.8          63.5       2.5X
-after 1900, vec on, rebase on                      6780           6834          92         14.8          67.8       2.3X
-before 1900, vec off, rebase off                  15540          15584          38          6.4         155.4       1.0X
-before 1900, vec off, rebase on                   19590          19653          55          5.1         195.9       0.8X
-before 1900, vec on, rebase off                    6374           6381          10         15.7          63.7       2.5X
-before 1900, vec on, rebase on                    10530          10544          25          9.5         105.3       1.5X
+after 1900, vec off, rebase EXCEPTION             16820          16922         116          5.9         168.2       1.0X
+after 1900, vec off, rebase LEGACY                17201          17547         406          5.8         172.0       1.0X
+after 1900, vec off, rebase CORRECTED             17087          17321         220          5.9         170.9       1.0X
+after 1900, vec on, rebase EXCEPTION               6204           6237          55         16.1          62.0       2.7X
+after 1900, vec on, rebase LEGACY                  6853           6887          51         14.6          68.5       2.5X
+after 1900, vec on, rebase CORRECTED               6388           6509         193         15.7          63.9       2.6X
+before 1900, vec off, rebase LEGACY               20223          20522         454          4.9         202.2       0.8X
+before 1900, vec off, rebase CORRECTED            16680          17002         330          6.0         166.8       1.0X
+before 1900, vec on, rebase LEGACY                 9893          10044         162         10.1          98.9       1.7X
+before 1900, vec on, rebase CORRECTED              6381           6486         140         15.7          63.8       2.6X
 
 
 ================================================================================================
 Rebasing dates/timestamps in ORC datasource
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Save DATE to ORC:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  23825          23825           0          4.2         238.2       1.0X
-before 1582, noop                                 10501          10501           0          9.5         105.0       2.3X
-after 1582                                        32134          32134           0          3.1         321.3       0.7X
-before 1582                                       19947          19947           0          5.0         199.5       1.2X
+after 1582, noop                                  24733          24733           0          4.0         247.3       1.0X
+before 1582, noop                                 10658          10658           0          9.4         106.6       2.3X
+after 1582                                        34442          34442           0          2.9         344.4       0.7X
+before 1582                                       20847          20847           0          4.8         208.5       1.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Load DATE from ORC:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               10034          10056          22         10.0         100.3       1.0X
-after 1582, vec on                                 3664           3698          30         27.3          36.6       2.7X
-before 1582, vec off                              10472          10502          30          9.5         104.7       1.0X
-before 1582, vec on                                4052           4098          42         24.7          40.5       2.5X
+after 1582, vec off                               10339          10498         180          9.7         103.4       1.0X
+after 1582, vec on                                 3812           3825          18         26.2          38.1       2.7X
+before 1582, vec off                              11374          11531         150          8.8         113.7       0.9X
+before 1582, vec on                                4387           4641         424         22.8          43.9       2.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Save TIMESTAMP to ORC:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   2812           2812           0         35.6          28.1       1.0X
-before 1900, noop                                  2801           2801           0         35.7          28.0       1.0X
-after 1900                                        18290          18290           0          5.5         182.9       0.2X
-before 1900                                       22344          22344           0          4.5         223.4       0.1X
+after 1900, noop                                   2709           2709           0         36.9          27.1       1.0X
+before 1900, noop                                  2563           2563           0         39.0          25.6       1.1X
+after 1900                                        18967          18967           0          5.3         189.7       0.1X
+before 1900                                       21109          21109           0          4.7         211.1       0.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Load TIMESTAMP from ORC:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off                               11257          11279          32          8.9         112.6       1.0X
-after 1900, vec on                                 5296           5310          15         18.9          53.0       2.1X
-before 1900, vec off                              14700          14758          72          6.8         147.0       0.8X
-before 1900, vec on                                8576           8665         150         11.7          85.8       1.3X
+after 1900, vec off                               11339          11393          86          8.8         113.4       1.0X
+after 1900, vec on                                 5269           5298          47         19.0          52.7       2.2X
+before 1900, vec off                              14399          14551         241          6.9         144.0       0.8X
+before 1900, vec on                                8171           8266          97         12.2          81.7       1.4X
 
 
diff --git a/sql/core/benchmarks/ExternalAppendOnlyUnsafeRowArrayBenchmark-jdk11-results.txt b/sql/core/benchmarks/ExternalAppendOnlyUnsafeRowArrayBenchmark-jdk11-results.txt
index 4f50a894e5c07..373bf2db133c6 100644
--- a/sql/core/benchmarks/ExternalAppendOnlyUnsafeRowArrayBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/ExternalAppendOnlyUnsafeRowArrayBenchmark-jdk11-results.txt
@@ -2,44 +2,44 @@
 WITHOUT SPILL
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Array with 100000 rows:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-ArrayBuffer                                        5371           5392          30         19.1          52.5       1.0X
-ExternalAppendOnlyUnsafeRowArray                   6724           6778          77         15.2          65.7       0.8X
+ArrayBuffer                                        5039           5085          65         20.3          49.2       1.0X
+ExternalAppendOnlyUnsafeRowArray                   7524           7584          85         13.6          73.5       0.7X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Array with 1000 rows:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-ArrayBuffer                                        9120           9121           1         28.7          34.8       1.0X
-ExternalAppendOnlyUnsafeRowArray                  37713          37739          38          7.0         143.9       0.2X
+ArrayBuffer                                       10509          10609         141         24.9          40.1       1.0X
+ExternalAppendOnlyUnsafeRowArray                  29307          29477         239          8.9         111.8       0.4X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Array with 30000 rows:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-ArrayBuffer                                       18037          18052          21         27.3          36.7       1.0X
-ExternalAppendOnlyUnsafeRowArray                  34726          34771          63         14.2          70.7       0.5X
+ArrayBuffer                                       21699          21794         134         22.7          44.1       1.0X
+ExternalAppendOnlyUnsafeRowArray                  38614          38681          94         12.7          78.6       0.6X
 
 
 ================================================================================================
 WITH SPILL
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Spilling with 1000 rows:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-UnsafeExternalSorter                              29668          29676          11          8.8         113.2       1.0X
-ExternalAppendOnlyUnsafeRowArray                  12658          12663           6         20.7          48.3       2.3X
+UnsafeExternalSorter                              26214          26230          22         10.0         100.0       1.0X
+ExternalAppendOnlyUnsafeRowArray                  12670          12678          12         20.7          48.3       2.1X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Spilling with 10000 rows:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-UnsafeExternalSorter                                 12             12           0         13.8          72.7       1.0X
-ExternalAppendOnlyUnsafeRowArray                      8              8           0         19.8          50.6       1.4X
+UnsafeExternalSorter                                 10             11           1         15.9          62.9       1.0X
+ExternalAppendOnlyUnsafeRowArray                      8              8           1         21.3          47.0       1.3X
 
 
diff --git a/sql/core/benchmarks/ExternalAppendOnlyUnsafeRowArrayBenchmark-results.txt b/sql/core/benchmarks/ExternalAppendOnlyUnsafeRowArrayBenchmark-results.txt
index c4be80af1334b..88702951f3353 100644
--- a/sql/core/benchmarks/ExternalAppendOnlyUnsafeRowArrayBenchmark-results.txt
+++ b/sql/core/benchmarks/ExternalAppendOnlyUnsafeRowArrayBenchmark-results.txt
@@ -2,44 +2,44 @@
 WITHOUT SPILL
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Array with 100000 rows:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-ArrayBuffer                                        7626           7776         212         13.4          74.5       1.0X
-ExternalAppendOnlyUnsafeRowArray                   8432           8498          93         12.1          82.3       0.9X
+ArrayBuffer                                        5579           5669         128         18.4          54.5       1.0X
+ExternalAppendOnlyUnsafeRowArray                   5589           5591           2         18.3          54.6       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Array with 1000 rows:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-ArrayBuffer                                       11753          11763          15         22.3          44.8       1.0X
-ExternalAppendOnlyUnsafeRowArray                  38921          39236         446          6.7         148.5       0.3X
+ArrayBuffer                                       12989          13075         122         20.2          49.5       1.0X
+ExternalAppendOnlyUnsafeRowArray                  29432          29595         231          8.9         112.3       0.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Array with 30000 rows:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-ArrayBuffer                                       23820          23829          12         20.6          48.5       1.0X
-ExternalAppendOnlyUnsafeRowArray                  33449          33466          25         14.7          68.1       0.7X
+ArrayBuffer                                       25378          25500         171         19.4          51.6       1.0X
+ExternalAppendOnlyUnsafeRowArray                  29897          30232         474         16.4          60.8       0.8X
 
 
 ================================================================================================
 WITH SPILL
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Spilling with 1000 rows:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-UnsafeExternalSorter                              31384          31479         135          8.4         119.7       1.0X
-ExternalAppendOnlyUnsafeRowArray                  15278          15303          35         17.2          58.3       2.1X
+UnsafeExternalSorter                              26731          27063         469          9.8         102.0       1.0X
+ExternalAppendOnlyUnsafeRowArray                  14471          14653         258         18.1          55.2       1.8X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Spilling with 10000 rows:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-UnsafeExternalSorter                                 11             11           1         14.7          68.0       1.0X
-ExternalAppendOnlyUnsafeRowArray                      9             10           1         17.1          58.5       1.2X
+UnsafeExternalSorter                                  9             11           2         18.1          55.1       1.0X
+ExternalAppendOnlyUnsafeRowArray                      9              9           1         18.7          53.5       1.0X
 
 
diff --git a/sql/core/benchmarks/ExtractBenchmark-jdk11-results.txt b/sql/core/benchmarks/ExtractBenchmark-jdk11-results.txt
index 77f1bdb5eaee0..3466d35680352 100644
--- a/sql/core/benchmarks/ExtractBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/ExtractBenchmark-jdk11-results.txt
@@ -1,104 +1,104 @@
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Invoke extract for timestamp:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp                                   333            345          13         30.0          33.3       1.0X
-YEAR of timestamp                                   826            840          16         12.1          82.6       0.4X
-YEAROFWEEK of timestamp                            1161           1168           7          8.6         116.1       0.3X
-QUARTER of timestamp                                961            966           6         10.4          96.1       0.3X
-MONTH of timestamp                                  831            838           6         12.0          83.1       0.4X
-WEEK of timestamp                                  1155           1159           4          8.7         115.5       0.3X
-DAY of timestamp                                    822            828           6         12.2          82.2       0.4X
-DAYOFWEEK of timestamp                              968            972           6         10.3          96.8       0.3X
-DOW of timestamp                                    960            966           9         10.4          96.0       0.3X
-DOW_ISO of timestamp                                961            967           9         10.4          96.1       0.3X
-DAYOFWEEK_ISO of timestamp                          965            968           5         10.4          96.5       0.3X
-DOY of timestamp                                    864            873           9         11.6          86.4       0.4X
-HOUR of timestamp                                   631            647          14         15.9          63.1       0.5X
-MINUTE of timestamp                                 638            651          12         15.7          63.8       0.5X
-SECOND of timestamp                                 758            760           2         13.2          75.8       0.4X
+cast to timestamp                                   301            311           8         33.2          30.1       1.0X
+YEAR of timestamp                                   930           1003          68         10.8          93.0       0.3X
+YEAROFWEEK of timestamp                            1019           1091          63          9.8         101.9       0.3X
+QUARTER of timestamp                               1151           1228          67          8.7         115.1       0.3X
+MONTH of timestamp                                  987           1033          42         10.1          98.7       0.3X
+WEEK of timestamp                                  1372           1449          67          7.3         137.2       0.2X
+DAY of timestamp                                   1015           1040          22          9.8         101.5       0.3X
+DAYOFWEEK of timestamp                             1155           1223          59          8.7         115.5       0.3X
+DOW of timestamp                                   1090           1170          82          9.2         109.0       0.3X
+DOW_ISO of timestamp                               1059           1143          93          9.4         105.9       0.3X
+DAYOFWEEK_ISO of timestamp                         1060           1118          51          9.4         106.0       0.3X
+DOY of timestamp                                    958           1050          81         10.4          95.8       0.3X
+HOUR of timestamp                                   803            817          25         12.5          80.3       0.4X
+MINUTE of timestamp                                 707            742          30         14.1          70.7       0.4X
+SECOND of timestamp                                 901            934          44         11.1          90.1       0.3X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Invoke date_part for timestamp:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp                                   301            307           8         33.2          30.1       1.0X
-YEAR of timestamp                                   797            805           7         12.6          79.7       0.4X
-YEAROFWEEK of timestamp                            1147           1156           8          8.7         114.7       0.3X
-QUARTER of timestamp                                965            974           9         10.4          96.5       0.3X
-MONTH of timestamp                                  801            815          17         12.5          80.1       0.4X
-WEEK of timestamp                                  1133           1157          21          8.8         113.3       0.3X
-DAY of timestamp                                    825            828           4         12.1          82.5       0.4X
-DAYOFWEEK of timestamp                              929            937           8         10.8          92.9       0.3X
-DOW of timestamp                                    933            940          11         10.7          93.3       0.3X
-DOW_ISO of timestamp                                944            954          11         10.6          94.4       0.3X
-DAYOFWEEK_ISO of timestamp                          948            965          21         10.5          94.8       0.3X
-DOY of timestamp                                    894            898           6         11.2          89.4       0.3X
-HOUR of timestamp                                   678            683           6         14.7          67.8       0.4X
-MINUTE of timestamp                                 669            670           2         15.0          66.9       0.5X
-SECOND of timestamp                                 762            771          12         13.1          76.2       0.4X
+cast to timestamp                                   345            349           5         29.0          34.5       1.0X
+YEAR of timestamp                                  1050           1083          39          9.5         105.0       0.3X
+YEAROFWEEK of timestamp                            1125           1141          14          8.9         112.5       0.3X
+QUARTER of timestamp                               1103           1218         132          9.1         110.3       0.3X
+MONTH of timestamp                                  992           1008          28         10.1          99.2       0.3X
+WEEK of timestamp                                  1334           1403         101          7.5         133.4       0.3X
+DAY of timestamp                                   1001           1059          52         10.0         100.1       0.3X
+DAYOFWEEK of timestamp                             1094           1177          77          9.1         109.4       0.3X
+DOW of timestamp                                   1052           1128          67          9.5         105.2       0.3X
+DOW_ISO of timestamp                               1124           1165          39          8.9         112.4       0.3X
+DAYOFWEEK_ISO of timestamp                         1015           1125         101          9.9         101.5       0.3X
+DOY of timestamp                                   1027           1080          49          9.7         102.7       0.3X
+HOUR of timestamp                                   779            818          34         12.8          77.9       0.4X
+MINUTE of timestamp                                 757            770          19         13.2          75.7       0.5X
+SECOND of timestamp                                 862            898          33         11.6          86.2       0.4X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Invoke extract for date:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date                                        737            751          23         13.6          73.7       1.0X
-YEAR of date                                        804            811          11         12.4          80.4       0.9X
-YEAROFWEEK of date                                 1151           1155           4          8.7         115.1       0.6X
-QUARTER of date                                     964            987          26         10.4          96.4       0.8X
-MONTH of date                                       831            835           6         12.0          83.1       0.9X
-WEEK of date                                       1162           1163           2          8.6         116.2       0.6X
-DAY of date                                         826            833           7         12.1          82.6       0.9X
-DAYOFWEEK of date                                   984           1003          26         10.2          98.4       0.7X
-DOW of date                                         995            997           2         10.0          99.5       0.7X
-DOW_ISO of date                                     985            992           8         10.2          98.5       0.7X
-DAYOFWEEK_ISO of date                               985           1001          19         10.1          98.5       0.7X
-DOY of date                                         884            890           8         11.3          88.4       0.8X
-HOUR of date                                       1630           1640          13          6.1         163.0       0.5X
-MINUTE of date                                     1631           1636           6          6.1         163.1       0.5X
-SECOND of date                                     1737           1739           2          5.8         173.7       0.4X
+cast to date                                        885            922          35         11.3          88.5       1.0X
+YEAR of date                                       1100           1103           3          9.1         110.0       0.8X
+YEAROFWEEK of date                                 1030           1076          56          9.7         103.0       0.9X
+QUARTER of date                                    1065           1160          82          9.4         106.5       0.8X
+MONTH of date                                       937            989          78         10.7          93.7       0.9X
+WEEK of date                                       1389           1479          79          7.2         138.9       0.6X
+DAY of date                                         927            997          82         10.8          92.7       1.0X
+DAYOFWEEK of date                                  1111           1199          76          9.0         111.1       0.8X
+DOW of date                                        1041           1149          94          9.6         104.1       0.8X
+DOW_ISO of date                                    1049           1076          34          9.5         104.9       0.8X
+DAYOFWEEK_ISO of date                              1084           1134          62          9.2         108.4       0.8X
+DOY of date                                        1042           1079          43          9.6         104.2       0.8X
+HOUR of date                                       1792           1829          32          5.6         179.2       0.5X
+MINUTE of date                                     1773           1827          60          5.6         177.3       0.5X
+SECOND of date                                     2066           2085          18          4.8         206.6       0.4X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Invoke date_part for date:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date                                        746            753           8         13.4          74.6       1.0X
-YEAR of date                                        841            843           1         11.9          84.1       0.9X
-YEAROFWEEK of date                                 1169           1191          20          8.6         116.9       0.6X
-QUARTER of date                                    1008           1012           4          9.9         100.8       0.7X
-MONTH of date                                       838            844           6         11.9          83.8       0.9X
-WEEK of date                                       1179           1182           3          8.5         117.9       0.6X
-DAY of date                                         849            850           1         11.8          84.9       0.9X
-DAYOFWEEK of date                                   984            988           4         10.2          98.4       0.8X
-DOW of date                                         986            994           7         10.1          98.6       0.8X
-DOW_ISO of date                                     974            992          17         10.3          97.4       0.8X
-DAYOFWEEK_ISO of date                               979            994          14         10.2          97.9       0.8X
-DOY of date                                         874            878           3         11.4          87.4       0.9X
-HOUR of date                                       1644           1649           6          6.1         164.4       0.5X
-MINUTE of date                                     1610           1626          15          6.2         161.0       0.5X
-SECOND of date                                     1768           1789          30          5.7         176.8       0.4X
+cast to date                                        907            932          23         11.0          90.7       1.0X
+YEAR of date                                        996           1041          55         10.0          99.6       0.9X
+YEAROFWEEK of date                                 1032           1087          81          9.7         103.2       0.9X
+QUARTER of date                                    1143           1193          82          8.7         114.3       0.8X
+MONTH of date                                       953           1014          58         10.5          95.3       1.0X
+WEEK of date                                       1260           1328         102          7.9         126.0       0.7X
+DAY of date                                         907            927          18         11.0          90.7       1.0X
+DAYOFWEEK of date                                  1071           1094          22          9.3         107.1       0.8X
+DOW of date                                        1048           1077          40          9.5         104.8       0.9X
+DOW_ISO of date                                    1024           1049          26          9.8         102.4       0.9X
+DAYOFWEEK_ISO of date                              1072           1082          16          9.3         107.2       0.8X
+DOY of date                                         968            973           5         10.3          96.8       0.9X
+HOUR of date                                       1628           1663          42          6.1         162.8       0.6X
+MINUTE of date                                     1542           1620          69          6.5         154.2       0.6X
+SECOND of date                                     1740           1769          33          5.7         174.0       0.5X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Invoke extract for interval:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to interval                                    989           1003          13         10.1          98.9       1.0X
-YEAR of interval                                    980            999          16         10.2          98.0       1.0X
-MONTH of interval                                   988           1000          15         10.1          98.8       1.0X
-DAY of interval                                     970            983          11         10.3          97.0       1.0X
-HOUR of interval                                    989           1000          13         10.1          98.9       1.0X
-MINUTE of interval                                 1019           1034          13          9.8         101.9       1.0X
-SECOND of interval                                 1084           1095          10          9.2         108.4       0.9X
+cast to interval                                   1413           1431          21          7.1         141.3       1.0X
+YEAR of interval                                   1386           1424          34          7.2         138.6       1.0X
+MONTH of interval                                  1419           1434          17          7.0         141.9       1.0X
+DAY of interval                                    1411           1428          15          7.1         141.1       1.0X
+HOUR of interval                                   1444           1461          24          6.9         144.4       1.0X
+MINUTE of interval                                 1435           1463          27          7.0         143.5       1.0X
+SECOND of interval                                 1537           1565          30          6.5         153.7       0.9X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Invoke date_part for interval:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to interval                                    955            969          15         10.5          95.5       1.0X
-YEAR of interval                                   1004           1007           3         10.0         100.4       1.0X
-MONTH of interval                                   989            990           2         10.1          98.9       1.0X
-DAY of interval                                    1006           1011           6          9.9         100.6       0.9X
-HOUR of interval                                    986            997          10         10.1          98.6       1.0X
-MINUTE of interval                                 1004           1014          15         10.0         100.4       1.0X
-SECOND of interval                                 1083           1088           6          9.2         108.3       0.9X
+cast to interval                                   1434           1450          15          7.0         143.4       1.0X
+YEAR of interval                                   1423           1443          18          7.0         142.3       1.0X
+MONTH of interval                                  1432           1452          22          7.0         143.2       1.0X
+DAY of interval                                    1451           1467          14          6.9         145.1       1.0X
+HOUR of interval                                   1415           1431          23          7.1         141.5       1.0X
+MINUTE of interval                                 1433           1458          36          7.0         143.3       1.0X
+SECOND of interval                                 1543           1556          13          6.5         154.3       0.9X
 
diff --git a/sql/core/benchmarks/ExtractBenchmark-results.txt b/sql/core/benchmarks/ExtractBenchmark-results.txt
index 11144a0547e43..8587ee04564dd 100644
--- a/sql/core/benchmarks/ExtractBenchmark-results.txt
+++ b/sql/core/benchmarks/ExtractBenchmark-results.txt
@@ -1,104 +1,104 @@
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Invoke extract for timestamp:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp                                   292            310          16         34.3          29.2       1.0X
-YEAR of timestamp                                   847            875          26         11.8          84.7       0.3X
-YEAROFWEEK of timestamp                             964            981          24         10.4          96.4       0.3X
-QUARTER of timestamp                               1217           1219           2          8.2         121.7       0.2X
-MONTH of timestamp                                  835            844          10         12.0          83.5       0.3X
-WEEK of timestamp                                  1173           1183          15          8.5         117.3       0.2X
-DAY of timestamp                                    851            878          25         11.7          85.1       0.3X
-DAYOFWEEK of timestamp                              946            970          22         10.6          94.6       0.3X
-DOW of timestamp                                    935            959          21         10.7          93.5       0.3X
-DOW_ISO of timestamp                                947            961          13         10.6          94.7       0.3X
-DAYOFWEEK_ISO of timestamp                          965            992          26         10.4          96.5       0.3X
-DOY of timestamp                                    886            904          26         11.3          88.6       0.3X
-HOUR of timestamp                                   697            700           4         14.3          69.7       0.4X
-MINUTE of timestamp                                 654            665          10         15.3          65.4       0.4X
-SECOND of timestamp                                 770            778           8         13.0          77.0       0.4X
+cast to timestamp                                   400            407           7         25.0          40.0       1.0X
+YEAR of timestamp                                  1031           1099          59          9.7         103.1       0.4X
+YEAROFWEEK of timestamp                            1134           1178          39          8.8         113.4       0.4X
+QUARTER of timestamp                               1179           1225          44          8.5         117.9       0.3X
+MONTH of timestamp                                 1064           1092          33          9.4         106.4       0.4X
+WEEK of timestamp                                  2260           2304          42          4.4         226.0       0.2X
+DAY of timestamp                                   1124           1162          35          8.9         112.4       0.4X
+DAYOFWEEK of timestamp                             1225           1250          30          8.2         122.5       0.3X
+DOW of timestamp                                   1231           1247          27          8.1         123.1       0.3X
+DOW_ISO of timestamp                               1196           1209          15          8.4         119.6       0.3X
+DAYOFWEEK_ISO of timestamp                         1222           1236          12          8.2         122.2       0.3X
+DOY of timestamp                                   1091           1105          16          9.2         109.1       0.4X
+HOUR of timestamp                                   836            854          31         12.0          83.6       0.5X
+MINUTE of timestamp                                 803            837          34         12.5          80.3       0.5X
+SECOND of timestamp                                 990           1025          46         10.1          99.0       0.4X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Invoke date_part for timestamp:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp                                   233            243           9         43.0          23.3       1.0X
-YEAR of timestamp                                   810            826          15         12.3          81.0       0.3X
-YEAROFWEEK of timestamp                             996           1019          21         10.0          99.6       0.2X
-QUARTER of timestamp                               1037           1049          11          9.6         103.7       0.2X
-MONTH of timestamp                                  822            852          30         12.2          82.2       0.3X
-WEEK of timestamp                                  1179           1220          35          8.5         117.9       0.2X
-DAY of timestamp                                    822            825           3         12.2          82.2       0.3X
-DAYOFWEEK of timestamp                              937            941           3         10.7          93.7       0.2X
-DOW of timestamp                                    931            970          34         10.7          93.1       0.2X
-DOW_ISO of timestamp                                927            948          22         10.8          92.7       0.3X
-DAYOFWEEK_ISO of timestamp                          896            918          20         11.2          89.6       0.3X
-DOY of timestamp                                    863            891          25         11.6          86.3       0.3X
-HOUR of timestamp                                   639            645           6         15.7          63.9       0.4X
-MINUTE of timestamp                                 639            647          12         15.7          63.9       0.4X
-SECOND of timestamp                                 785            796          11         12.7          78.5       0.3X
+cast to timestamp                                   378            387          10         26.5          37.8       1.0X
+YEAR of timestamp                                  1089           1110          25          9.2         108.9       0.3X
+YEAROFWEEK of timestamp                            1223           1277          70          8.2         122.3       0.3X
+QUARTER of timestamp                               1322           1365          45          7.6         132.2       0.3X
+MONTH of timestamp                                 1185           1195          10          8.4         118.5       0.3X
+WEEK of timestamp                                  2187           2239          48          4.6         218.7       0.2X
+DAY of timestamp                                   1087           1105          32          9.2         108.7       0.3X
+DAYOFWEEK of timestamp                             1223           1252          28          8.2         122.3       0.3X
+DOW of timestamp                                   1294           1336          37          7.7         129.4       0.3X
+DOW_ISO of timestamp                               1259           1286          25          7.9         125.9       0.3X
+DAYOFWEEK_ISO of timestamp                         1200           1229          25          8.3         120.0       0.3X
+DOY of timestamp                                   1084           1111          30          9.2         108.4       0.3X
+HOUR of timestamp                                   829            878          43         12.1          82.9       0.5X
+MINUTE of timestamp                                 809            839          26         12.4          80.9       0.5X
+SECOND of timestamp                                1069           1102          44          9.4         106.9       0.4X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Invoke extract for date:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date                                        690            693           3         14.5          69.0       1.0X
-YEAR of date                                        822            841          17         12.2          82.2       0.8X
-YEAROFWEEK of date                                  967            974           7         10.3          96.7       0.7X
-QUARTER of date                                    1034           1044          11          9.7         103.4       0.7X
-MONTH of date                                       831            836           5         12.0          83.1       0.8X
-WEEK of date                                       1152           1177          34          8.7         115.2       0.6X
-DAY of date                                         836            873          34         12.0          83.6       0.8X
-DAYOFWEEK of date                                   960            992          30         10.4          96.0       0.7X
-DOW of date                                        1011           1016           4          9.9         101.1       0.7X
-DOW_ISO of date                                     969            984          16         10.3          96.9       0.7X
-DAYOFWEEK_ISO of date                               967            986          19         10.3          96.7       0.7X
-DOY of date                                         901            953          47         11.1          90.1       0.8X
-HOUR of date                                       1581           1586           5          6.3         158.1       0.4X
-MINUTE of date                                     1570           1584          13          6.4         157.0       0.4X
-SECOND of date                                     1713           1740          27          5.8         171.3       0.4X
+cast to date                                        900            912          12         11.1          90.0       1.0X
+YEAR of date                                       1112           1126          12          9.0         111.2       0.8X
+YEAROFWEEK of date                                 1257           1271          17          8.0         125.7       0.7X
+QUARTER of date                                    1177           1202          22          8.5         117.7       0.8X
+MONTH of date                                      1053           1115          59          9.5         105.3       0.9X
+WEEK of date                                       2232           2290          59          4.5         223.2       0.4X
+DAY of date                                        1124           1136          11          8.9         112.4       0.8X
+DAYOFWEEK of date                                  1247           1255          11          8.0         124.7       0.7X
+DOW of date                                        1229           1275          40          8.1         122.9       0.7X
+DOW_ISO of date                                    1175           1192          21          8.5         117.5       0.8X
+DAYOFWEEK_ISO of date                              1147           1172          24          8.7         114.7       0.8X
+DOY of date                                        1087           1123          32          9.2         108.7       0.8X
+HOUR of date                                       1603           1707          90          6.2         160.3       0.6X
+MINUTE of date                                     1647           1704          49          6.1         164.7       0.5X
+SECOND of date                                     1846           1879          39          5.4         184.6       0.5X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Invoke date_part for date:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date                                        756            762           6         13.2          75.6       1.0X
-YEAR of date                                        843            857          13         11.9          84.3       0.9X
-YEAROFWEEK of date                                 1055           1065          16          9.5         105.5       0.7X
-QUARTER of date                                    1066           1073           6          9.4         106.6       0.7X
-MONTH of date                                       856            890          44         11.7          85.6       0.9X
-WEEK of date                                       1155           1204          59          8.7         115.5       0.7X
-DAY of date                                         749            762          12         13.3          74.9       1.0X
-DAYOFWEEK of date                                   850            865          15         11.8          85.0       0.9X
-DOW of date                                         878            893          16         11.4          87.8       0.9X
-DOW_ISO of date                                     865            869           5         11.6          86.5       0.9X
-DAYOFWEEK_ISO of date                               914            967          76         10.9          91.4       0.8X
-DOY of date                                         789            792           4         12.7          78.9       1.0X
-HOUR of date                                       1558           1659         168          6.4         155.8       0.5X
-MINUTE of date                                     1581           1673          80          6.3         158.1       0.5X
-SECOND of date                                     1646           1881         319          6.1         164.6       0.5X
+cast to date                                        846            889          37         11.8          84.6       1.0X
+YEAR of date                                       1020           1071          46          9.8         102.0       0.8X
+YEAROFWEEK of date                                 1288           1303          18          7.8         128.8       0.7X
+QUARTER of date                                    1181           1201          22          8.5         118.1       0.7X
+MONTH of date                                      1021           1076          48          9.8         102.1       0.8X
+WEEK of date                                       2145           2186          42          4.7         214.5       0.4X
+DAY of date                                        1084           1103          26          9.2         108.4       0.8X
+DAYOFWEEK of date                                  1201           1233          41          8.3         120.1       0.7X
+DOW of date                                        1246           1281          34          8.0         124.6       0.7X
+DOW_ISO of date                                    1136           1141           4          8.8         113.6       0.7X
+DAYOFWEEK_ISO of date                              1209           1234          30          8.3         120.9       0.7X
+DOY of date                                        1083           1144          57          9.2         108.3       0.8X
+HOUR of date                                       1632           1654          24          6.1         163.2       0.5X
+MINUTE of date                                     1681           1716          31          6.0         168.1       0.5X
+SECOND of date                                     1838           1902          60          5.4         183.8       0.5X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Invoke extract for interval:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to interval                                    925            941          20         10.8          92.5       1.0X
-YEAR of interval                                    903            919          14         11.1          90.3       1.0X
-MONTH of interval                                   944            958          17         10.6          94.4       1.0X
-DAY of interval                                     917            925           7         10.9          91.7       1.0X
-HOUR of interval                                    925            940          17         10.8          92.5       1.0X
-MINUTE of interval                                  951            962          12         10.5          95.1       1.0X
-SECOND of interval                                 1017           1036          19          9.8         101.7       0.9X
+cast to interval                                   1285           1304          16          7.8         128.5       1.0X
+YEAR of interval                                   1356           1384          24          7.4         135.6       0.9X
+MONTH of interval                                  1349           1370          19          7.4         134.9       1.0X
+DAY of interval                                    1306           1345          51          7.7         130.6       1.0X
+HOUR of interval                                   1351           1400          44          7.4         135.1       1.0X
+MINUTE of interval                                 1339           1410          74          7.5         133.9       1.0X
+SECOND of interval                                 1365           1402          32          7.3         136.5       0.9X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Invoke date_part for interval:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to interval                                    913            920           8         11.0          91.3       1.0X
-YEAR of interval                                    930            935           4         10.8          93.0       1.0X
-MONTH of interval                                   930            943          14         10.7          93.0       1.0X
-DAY of interval                                     933            946          12         10.7          93.3       1.0X
-HOUR of interval                                    951            953           3         10.5          95.1       1.0X
-MINUTE of interval                                  923            958          30         10.8          92.3       1.0X
-SECOND of interval                                  993            995           1         10.1          99.3       0.9X
+cast to interval                                   1274           1334          58          7.9         127.4       1.0X
+YEAR of interval                                   1329           1359          27          7.5         132.9       1.0X
+MONTH of interval                                  1310           1362          45          7.6         131.0       1.0X
+DAY of interval                                    1473           1500          26          6.8         147.3       0.9X
+HOUR of interval                                   1308           1369          57          7.6         130.8       1.0X
+MINUTE of interval                                 1333           1379          76          7.5         133.3       1.0X
+SECOND of interval                                 1423           1448          36          7.0         142.3       0.9X
 
diff --git a/sql/core/benchmarks/FilterPushdownBenchmark-jdk11-results.txt b/sql/core/benchmarks/FilterPushdownBenchmark-jdk11-results.txt
index d292607e2cbcb..9a8422ea88b73 100644
--- a/sql/core/benchmarks/FilterPushdownBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/FilterPushdownBenchmark-jdk11-results.txt
@@ -2,669 +2,669 @@
 Pushdown for many distinct value case
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 0 string row (value IS NULL):      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                11943          12023          69          1.3         759.3       1.0X
-Parquet Vectorized (Pushdown)                       880            934          44         17.9          55.9      13.6X
-Native ORC Vectorized                              7847           7896          41          2.0         498.9       1.5X
-Native ORC Vectorized (Pushdown)                    507            525          22         31.0          32.3      23.5X
+Parquet Vectorized                                10512          10572          58          1.5         668.4       1.0X
+Parquet Vectorized (Pushdown)                       596            621          19         26.4          37.9      17.6X
+Native ORC Vectorized                              8555           8723          97          1.8         543.9       1.2X
+Native ORC Vectorized (Pushdown)                    592            609          11         26.6          37.7      17.8X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 0 string row ('7864320' < value < '7864320'):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                12020          12040          13          1.3         764.2       1.0X
-Parquet Vectorized (Pushdown)                       819            840          24         19.2          52.0      14.7X
-Native ORC Vectorized                              8045           8062          14          2.0         511.5       1.5X
-Native ORC Vectorized (Pushdown)                    498            535          55         31.6          31.7      24.1X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+-----------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                           10406          10461          50          1.5         661.6       1.0X
+Parquet Vectorized (Pushdown)                                  619            641          22         25.4          39.4      16.8X
+Native ORC Vectorized                                         8787           8834          57          1.8         558.6       1.2X
+Native ORC Vectorized (Pushdown)                               592            608          11         26.6          37.6      17.6X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 1 string row (value = '7864320'):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                12006          12028          21          1.3         763.3       1.0X
-Parquet Vectorized (Pushdown)                       772            800          25         20.4          49.1      15.6X
-Native ORC Vectorized                              8074           8091          15          1.9         513.3       1.5X
-Native ORC Vectorized (Pushdown)                    467            481          10         33.7          29.7      25.7X
+Parquet Vectorized                                10632          10694          60          1.5         676.0       1.0X
+Parquet Vectorized (Pushdown)                       608            635          22         25.9          38.6      17.5X
+Native ORC Vectorized                              8790           8838          37          1.8         558.9       1.2X
+Native ORC Vectorized (Pushdown)                    559            584          22         28.1          35.5      19.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 1 string row (value <=> '7864320'):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                11995          12021          30          1.3         762.6       1.0X
-Parquet Vectorized (Pushdown)                       780            792          10         20.2          49.6      15.4X
-Native ORC Vectorized                              8046           8066          12          2.0         511.6       1.5X
-Native ORC Vectorized (Pushdown)                    476            489          13         33.0          30.3      25.2X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+-------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                 10529          10624          74          1.5         669.4       1.0X
+Parquet Vectorized (Pushdown)                        613            631          16         25.7          39.0      17.2X
+Native ORC Vectorized                               8746           8816          63          1.8         556.1       1.2X
+Native ORC Vectorized (Pushdown)                     589            600          11         26.7          37.5      17.9X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 1 string row ('7864320' <= value <= '7864320'):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                11967          11989          12          1.3         760.9       1.0X
-Parquet Vectorized (Pushdown)                       775            788          11         20.3          49.3      15.4X
-Native ORC Vectorized                              8028           8046          25          2.0         510.4       1.5X
-Native ORC Vectorized (Pushdown)                    461            493          29         34.1          29.3      26.0X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+-------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                             10612          10696         110          1.5         674.7       1.0X
+Parquet Vectorized (Pushdown)                                    621            635          13         25.3          39.5      17.1X
+Native ORC Vectorized                                           8856           8901          59          1.8         563.0       1.2X
+Native ORC Vectorized (Pushdown)                                 576            590          14         27.3          36.6      18.4X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select all string rows (value IS NOT NULL):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                21435          21513          46          0.7        1362.8       1.0X
-Parquet Vectorized (Pushdown)                     21710          21742          33          0.7        1380.3       1.0X
-Native ORC Vectorized                             19324          19373          35          0.8        1228.6       1.1X
-Native ORC Vectorized (Pushdown)                  19535          19572          40          0.8        1242.0       1.1X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+--------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                  19970          20121          95          0.8        1269.7       1.0X
+Parquet Vectorized (Pushdown)                       20052          20137          73          0.8        1274.9       1.0X
+Native ORC Vectorized                               18242          18405         147          0.9        1159.8       1.1X
+Native ORC Vectorized (Pushdown)                    18595          18840         167          0.8        1182.2       1.1X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 0 int row (value IS NULL):         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                11008          11034          21          1.4         699.9       1.0X
-Parquet Vectorized (Pushdown)                       762            767           6         20.6          48.4      14.4X
-Native ORC Vectorized                              7156           7186          58          2.2         454.9       1.5X
-Native ORC Vectorized (Pushdown)                    436            447           9         36.1          27.7      25.3X
+Parquet Vectorized                                 9942           9996          56          1.6         632.1       1.0X
+Parquet Vectorized (Pushdown)                       594            613          23         26.5          37.7      16.7X
+Native ORC Vectorized                              7954           8180         173          2.0         505.7       1.2X
+Native ORC Vectorized (Pushdown)                    554            580          17         28.4          35.2      17.9X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 0 int row (7864320 < value < 7864320):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                10987          11033          38          1.4         698.5       1.0X
-Parquet Vectorized (Pushdown)                       766            776          11         20.5          48.7      14.4X
-Native ORC Vectorized                              7160           7192          35          2.2         455.2       1.5X
-Native ORC Vectorized (Pushdown)                    440            454          26         35.8          27.9      25.0X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+----------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                     9987          10077          66          1.6         635.0       1.0X
+Parquet Vectorized (Pushdown)                           617            637          14         25.5          39.2      16.2X
+Native ORC Vectorized                                  7995           8106         100          2.0         508.3       1.2X
+Native ORC Vectorized (Pushdown)                        565            587          18         27.8          36.0      17.7X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 1 int row (value = 7864320):       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                11066          11082          19          1.4         703.6       1.0X
-Parquet Vectorized (Pushdown)                       749            770          15         21.0          47.6      14.8X
-Native ORC Vectorized                              7248           7294          57          2.2         460.8       1.5X
-Native ORC Vectorized (Pushdown)                    437            450          16         36.0          27.8      25.3X
+Parquet Vectorized                                10017          10097          92          1.6         636.9       1.0X
+Parquet Vectorized (Pushdown)                       598            615          13         26.3          38.0      16.8X
+Native ORC Vectorized                              8032           8102          62          2.0         510.7       1.2X
+Native ORC Vectorized (Pushdown)                    562            572          11         28.0          35.7      17.8X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 1 int row (value <=> 7864320):     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                11066          11091          32          1.4         703.5       1.0X
-Parquet Vectorized (Pushdown)                       754            776          28         20.9          47.9      14.7X
-Native ORC Vectorized                              7257           7265           6          2.2         461.4       1.5X
-Native ORC Vectorized (Pushdown)                    449            457           7         35.0          28.6      24.6X
+Parquet Vectorized                                10076          10247         321          1.6         640.6       1.0X
+Parquet Vectorized (Pushdown)                       590            606          15         26.7          37.5      17.1X
+Native ORC Vectorized                              8101           8167          38          1.9         515.0       1.2X
+Native ORC Vectorized (Pushdown)                    566            580          15         27.8          36.0      17.8X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 1 int row (7864320 <= value <= 7864320):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                11067          11079          14          1.4         703.6       1.0X
-Parquet Vectorized (Pushdown)                       753            762           8         20.9          47.9      14.7X
-Native ORC Vectorized                              7251           7267          22          2.2         461.0       1.5X
-Native ORC Vectorized (Pushdown)                    427            435           6         36.8          27.2      25.9X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                      10031          10081          58          1.6         637.8       1.0X
+Parquet Vectorized (Pushdown)                             614            629          19         25.6          39.0      16.3X
+Native ORC Vectorized                                    7980           8095         101          2.0         507.4       1.3X
+Native ORC Vectorized (Pushdown)                          551            561          10         28.5          35.0      18.2X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 1 int row (7864319 < value < 7864321):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                11055          11086          18          1.4         702.8       1.0X
-Parquet Vectorized (Pushdown)                       748            753           6         21.0          47.5      14.8X
-Native ORC Vectorized                              7227           7253          27          2.2         459.5       1.5X
-Native ORC Vectorized (Pushdown)                    429            435           5         36.7          27.3      25.8X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+----------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                    10140          10183          39          1.6         644.7       1.0X
+Parquet Vectorized (Pushdown)                           606            618          13         25.9          38.6      16.7X
+Native ORC Vectorized                                  8013           8104          90          2.0         509.5       1.3X
+Native ORC Vectorized (Pushdown)                        554            572          16         28.4          35.2      18.3X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 10% int rows (value < 1572864):    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                12060          12091          21          1.3         766.7       1.0X
-Parquet Vectorized (Pushdown)                      2799           2821          17          5.6         178.0       4.3X
-Native ORC Vectorized                              8334           8391          55          1.9         529.9       1.4X
-Native ORC Vectorized (Pushdown)                   2197           2209          12          7.2         139.7       5.5X
+Parquet Vectorized                                10876          10962          92          1.4         691.5       1.0X
+Parquet Vectorized (Pushdown)                      2507           2538          29          6.3         159.4       4.3X
+Native ORC Vectorized                              9090           9177          83          1.7         577.9       1.2X
+Native ORC Vectorized (Pushdown)                   2228           2289          65          7.1         141.7       4.9X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 50% int rows (value < 7864320):    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                15593          15619          22          1.0         991.3       1.0X
-Parquet Vectorized (Pushdown)                     10573          10584          10          1.5         672.2       1.5X
-Native ORC Vectorized                             12276          12290          16          1.3         780.5       1.3X
-Native ORC Vectorized (Pushdown)                   8931           8958          31          1.8         567.8       1.7X
+Parquet Vectorized                                14221          14357         129          1.1         904.2       1.0X
+Parquet Vectorized (Pushdown)                      9631           9713          62          1.6         612.4       1.5X
+Native ORC Vectorized                             12437          12586          92          1.3         790.7       1.1X
+Native ORC Vectorized (Pushdown)                   8802           8891          89          1.8         559.6       1.6X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 90% int rows (value < 14155776):   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                19094          19120          23          0.8        1214.0       1.0X
-Parquet Vectorized (Pushdown)                     18327          18342          14          0.9        1165.2       1.0X
-Native ORC Vectorized                             15890          15943          44          1.0        1010.2       1.2X
-Native ORC Vectorized (Pushdown)                  15414          15442          24          1.0         980.0       1.2X
+Parquet Vectorized                                17521          17720         209          0.9        1114.0       1.0X
+Parquet Vectorized (Pushdown)                     16711          16780          66          0.9        1062.5       1.0X
+Native ORC Vectorized                             15743          15909         108          1.0        1000.9       1.1X
+Native ORC Vectorized (Pushdown)                  15332          15452         100          1.0         974.8       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select all int rows (value IS NOT NULL):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                20010          20037          21          0.8        1272.2       1.0X
-Parquet Vectorized (Pushdown)                     20297          20318          21          0.8        1290.4       1.0X
-Native ORC Vectorized                             16994          17029          39          0.9        1080.5       1.2X
-Native ORC Vectorized (Pushdown)                  17180          17224          53          0.9        1092.3       1.2X
+Parquet Vectorized                                18527          18615          69          0.8        1177.9       1.0X
+Parquet Vectorized (Pushdown)                     18761          18898         132          0.8        1192.8       1.0X
+Native ORC Vectorized                             16582          16724         134          0.9        1054.3       1.1X
+Native ORC Vectorized (Pushdown)                  16720          16860          81          0.9        1063.0       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select all int rows (value > -1):         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                19988          20061          96          0.8        1270.8       1.0X
-Parquet Vectorized (Pushdown)                     20343          20389          42          0.8        1293.4       1.0X
-Native ORC Vectorized                             16932          16958          22          0.9        1076.5       1.2X
-Native ORC Vectorized (Pushdown)                  17308          17351          31          0.9        1100.4       1.2X
+Parquet Vectorized                                18667          18837         155          0.8        1186.8       1.0X
+Parquet Vectorized (Pushdown)                     18590          18851         262          0.8        1181.9       1.0X
+Native ORC Vectorized                             16641          16720          48          0.9        1058.0       1.1X
+Native ORC Vectorized (Pushdown)                  16811          16944          96          0.9        1068.8       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select all int rows (value != -1):        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                20167          20210          36          0.8        1282.2       1.0X
-Parquet Vectorized (Pushdown)                     20508          20543          44          0.8        1303.9       1.0X
-Native ORC Vectorized                             17038          17070          32          0.9        1083.3       1.2X
-Native ORC Vectorized (Pushdown)                  17250          17663         538          0.9        1096.7       1.2X
+Parquet Vectorized                                18680          18772          68          0.8        1187.6       1.0X
+Parquet Vectorized (Pushdown)                     18633          18804         152          0.8        1184.7       1.0X
+Native ORC Vectorized                             16587          16714          96          0.9        1054.6       1.1X
+Native ORC Vectorized (Pushdown)                  16810          16983         113          0.9        1068.8       1.1X
 
 
 ================================================================================================
 Pushdown for few distinct value case (use dictionary encoding)
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 0 distinct string row (value IS NULL):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                10641          10674          57          1.5         676.6       1.0X
-Parquet Vectorized (Pushdown)                       661            665           7         23.8          42.0      16.1X
-Native ORC Vectorized                              9194           9227          33          1.7         584.5       1.2X
-Native ORC Vectorized (Pushdown)                    798            808          18         19.7          50.7      13.3X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+----------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                     8687           8807          91          1.8         552.3       1.0X
+Parquet Vectorized (Pushdown)                           516            533          15         30.5          32.8      16.8X
+Native ORC Vectorized                                  9679           9770          76          1.6         615.4       0.9X
+Native ORC Vectorized (Pushdown)                       1011           1023          10         15.6          64.3       8.6X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 0 distinct string row ('100' < value < '100'):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                10785          10810          20          1.5         685.7       1.0X
-Parquet Vectorized (Pushdown)                       656            666          10         24.0          41.7      16.5X
-Native ORC Vectorized                              9435           9457          16          1.7         599.9       1.1X
-Native ORC Vectorized (Pushdown)                    796            822          34         19.8          50.6      13.6X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                             9038           9082          39          1.7         574.6       1.0X
+Parquet Vectorized (Pushdown)                                   510            538          20         30.9          32.4      17.7X
+Native ORC Vectorized                                         10082          10135          80          1.6         641.0       0.9X
+Native ORC Vectorized (Pushdown)                               1005           1033          16         15.6          63.9       9.0X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 1 distinct string row (value = '100'):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                10639          10664          26          1.5         676.4       1.0X
-Parquet Vectorized (Pushdown)                       734            742           5         21.4          46.7      14.5X
-Native ORC Vectorized                              9346           9362          17          1.7         594.2       1.1X
-Native ORC Vectorized (Pushdown)                    863            869           5         18.2          54.9      12.3X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+----------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                     8882           8947          55          1.8         564.7       1.0X
+Parquet Vectorized (Pushdown)                           585            617          30         26.9          37.2      15.2X
+Native ORC Vectorized                                  9998          10016          22          1.6         635.7       0.9X
+Native ORC Vectorized (Pushdown)                       1101           1122          17         14.3          70.0       8.1X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 1 distinct string row (value <=> '100'):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                10617          10672          59          1.5         675.0       1.0X
-Parquet Vectorized (Pushdown)                       736            747           8         21.4          46.8      14.4X
-Native ORC Vectorized                              9345           9445         125          1.7         594.1       1.1X
-Native ORC Vectorized (Pushdown)                    868            877           7         18.1          55.2      12.2X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                       8836           8913          51          1.8         561.8       1.0X
+Parquet Vectorized (Pushdown)                             589            615          22         26.7          37.4      15.0X
+Native ORC Vectorized                                   10005          10088          64          1.6         636.1       0.9X
+Native ORC Vectorized (Pushdown)                         1082           1109          19         14.5          68.8       8.2X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 1 distinct string row ('100' <= value <= '100'):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                10750          10790          35          1.5         683.5       1.0X
-Parquet Vectorized (Pushdown)                       745            748           6         21.1          47.4      14.4X
-Native ORC Vectorized                              9471           9488          16          1.7         602.1       1.1X
-Native ORC Vectorized (Pushdown)                    857            866           6         18.3          54.5      12.5X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+--------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                               9009           9058          37          1.7         572.8       1.0X
+Parquet Vectorized (Pushdown)                                     603            621          17         26.1          38.4      14.9X
+Native ORC Vectorized                                           10175          10251          60          1.5         646.9       0.9X
+Native ORC Vectorized (Pushdown)                                 1102           1111          13         14.3          70.0       8.2X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select all distinct string rows (value IS NOT NULL):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                21173          21253          91          0.7        1346.1       1.0X
-Parquet Vectorized (Pushdown)                     21369          21456          75          0.7        1358.6       1.0X
-Native ORC Vectorized                             20282          20397          83          0.8        1289.5       1.0X
-Native ORC Vectorized (Pushdown)                  20704          20768          48          0.8        1316.3       1.0X
+-----------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                           20070          20183          84          0.8        1276.0       1.0X
+Parquet Vectorized (Pushdown)                                20144          20391         163          0.8        1280.7       1.0X
+Native ORC Vectorized                                        20849          20927          97          0.8        1325.5       1.0X
+Native ORC Vectorized (Pushdown)                             21132          21279         102          0.7        1343.6       0.9X
 
 
 ================================================================================================
 Pushdown benchmark for StringStartsWith
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 StringStartsWith filter: (value like '10%'):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                12426          12474          59          1.3         790.0       1.0X
-Parquet Vectorized (Pushdown)                      1847           1855           8          8.5         117.4       6.7X
-Native ORC Vectorized                              8336           8391          68          1.9         530.0       1.5X
-Native ORC Vectorized (Pushdown)                   8514           8536          14          1.8         541.3       1.5X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+---------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                   11414          11506          79          1.4         725.7       1.0X
+Parquet Vectorized (Pushdown)                         1577           1614          31         10.0         100.3       7.2X
+Native ORC Vectorized                                 8983           9104         148          1.8         571.1       1.3X
+Native ORC Vectorized (Pushdown)                      9175           9273          63          1.7         583.3       1.2X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 StringStartsWith filter: (value like '1000%'):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                12143          12167          23          1.3         772.1       1.0X
-Parquet Vectorized (Pushdown)                       751            758           9         21.0          47.7      16.2X
-Native ORC Vectorized                              8064           8069           8          2.0         512.7       1.5X
-Native ORC Vectorized (Pushdown)                   8226           8254          35          1.9         523.0       1.5X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+-----------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                     11258          11293          31          1.4         715.7       1.0X
+Parquet Vectorized (Pushdown)                            601            613           9         26.2          38.2      18.7X
+Native ORC Vectorized                                   8782           8829          54          1.8         558.3       1.3X
+Native ORC Vectorized (Pushdown)                        8989           9038          56          1.7         571.5       1.3X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 StringStartsWith filter: (value like '786432%'):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                12123          12142          13          1.3         770.8       1.0X
-Parquet Vectorized (Pushdown)                       739            743           4         21.3          47.0      16.4X
-Native ORC Vectorized                              8038           8052          10          2.0         511.0       1.5X
-Native ORC Vectorized (Pushdown)                   8211           8227          12          1.9         522.0       1.5X
+-------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                       11348          11438          75          1.4         721.5       1.0X
+Parquet Vectorized (Pushdown)                              621            635          10         25.3          39.5      18.3X
+Native ORC Vectorized                                     8710           8815          62          1.8         553.8       1.3X
+Native ORC Vectorized (Pushdown)                          8885           9012          86          1.8         564.9       1.3X
 
 
 ================================================================================================
 Pushdown benchmark for decimal
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 1 decimal(9, 2) row (value = 7864320):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 5889           5912          23          2.7         374.4       1.0X
-Parquet Vectorized (Pushdown)                       186            193           7         84.6          11.8      31.7X
-Native ORC Vectorized                              5272           5291          18          3.0         335.2       1.1X
-Native ORC Vectorized (Pushdown)                    156            183          36        100.9           9.9      37.8X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+----------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                     3711           3735          37          4.2         235.9       1.0X
+Parquet Vectorized (Pushdown)                           141            152           8        111.2           9.0      26.2X
+Native ORC Vectorized                                  6095           6143          37          2.6         387.5       0.6X
+Native ORC Vectorized (Pushdown)                        184            193           7         85.5          11.7      20.2X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 10% decimal(9, 2) rows (value < 1572864):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 7738           7804         111          2.0         492.0       1.0X
-Parquet Vectorized (Pushdown)                      3172           3188          27          5.0         201.6       2.4X
-Native ORC Vectorized                              7522           7528           4          2.1         478.2       1.0X
-Native ORC Vectorized (Pushdown)                   3390           3433          35          4.6         215.5       2.3X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+-------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                        5978           6030          36          2.6         380.1       1.0X
+Parquet Vectorized (Pushdown)                             3086           3162          46          5.1         196.2       1.9X
+Native ORC Vectorized                                     8289           8307          13          1.9         527.0       0.7X
+Native ORC Vectorized (Pushdown)                          3608           3618           9          4.4         229.4       1.7X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 50% decimal(9, 2) rows (value < 7864320):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                13990          14000           9          1.1         889.4       1.0X
-Parquet Vectorized (Pushdown)                     13251          13266          21          1.2         842.5       1.1X
-Native ORC Vectorized                             14799          14817          14          1.1         940.9       0.9X
-Native ORC Vectorized (Pushdown)                  14149          14195          64          1.1         899.6       1.0X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+-------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                       13279          13366          80          1.2         844.2       1.0X
+Parquet Vectorized (Pushdown)                            12650          12758         147          1.2         804.3       1.0X
+Native ORC Vectorized                                    15137          15272         113          1.0         962.4       0.9X
+Native ORC Vectorized (Pushdown)                         14389          14472          75          1.1         914.8       0.9X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 90% decimal(9, 2) rows (value < 14155776):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                15114          15165          44          1.0         960.9       1.0X
-Parquet Vectorized (Pushdown)                     15182          15222          23          1.0         965.2       1.0X
-Native ORC Vectorized                             16653          16683          19          0.9        1058.8       0.9X
-Native ORC Vectorized (Pushdown)                  16730          16743           9          0.9        1063.7       0.9X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+--------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                        14918          14969          50          1.1         948.5       1.0X
+Parquet Vectorized (Pushdown)                             15070          15210         115          1.0         958.1       1.0X
+Native ORC Vectorized                                     17081          17453         275          0.9        1086.0       0.9X
+Native ORC Vectorized (Pushdown)                          17298          17452         110          0.9        1099.8       0.9X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 1 decimal(18, 2) row (value = 7864320):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 6112           6127          17          2.6         388.6       1.0X
-Parquet Vectorized (Pushdown)                       186            190           3         84.7          11.8      32.9X
-Native ORC Vectorized                              5292           5359          74          3.0         336.4       1.2X
-Native ORC Vectorized (Pushdown)                    152            159          14        103.3           9.7      40.1X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+-----------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                      3968           4006          31          4.0         252.3       1.0X
+Parquet Vectorized (Pushdown)                            143            153           8        109.9           9.1      27.7X
+Native ORC Vectorized                                   6074           6111          33          2.6         386.2       0.7X
+Native ORC Vectorized (Pushdown)                         178            192           9         88.5          11.3      22.3X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 10% decimal(18, 2) rows (value < 1572864):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 7119           7158          57          2.2         452.6       1.0X
-Parquet Vectorized (Pushdown)                      1765           1784          13          8.9         112.2       4.0X
-Native ORC Vectorized                              6389           6398          11          2.5         406.2       1.1X
-Native ORC Vectorized (Pushdown)                   1732           1743          13          9.1         110.1       4.1X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+--------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                         4949           5032          75          3.2         314.6       1.0X
+Parquet Vectorized (Pushdown)                              1587           1625          27          9.9         100.9       3.1X
+Native ORC Vectorized                                      7049           7143          74          2.2         448.2       0.7X
+Native ORC Vectorized (Pushdown)                           1783           1834          51          8.8         113.3       2.8X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 50% decimal(18, 2) rows (value < 7864320):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                10996          11025          19          1.4         699.1       1.0X
-Parquet Vectorized (Pushdown)                      8047           8074          16          2.0         511.6       1.4X
-Native ORC Vectorized                             10620          10642          23          1.5         675.2       1.0X
-Native ORC Vectorized (Pushdown)                   8086           8102          14          1.9         514.1       1.4X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+--------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                         9178           9253          51          1.7         583.5       1.0X
+Parquet Vectorized (Pushdown)                              7386           7454          58          2.1         469.6       1.2X
+Native ORC Vectorized                                     11201          11271          70          1.4         712.1       0.8X
+Native ORC Vectorized (Pushdown)                           8282           8355          54          1.9         526.6       1.1X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 90% decimal(18, 2) rows (value < 14155776):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                14800          14834          28          1.1         941.0       1.0X
-Parquet Vectorized (Pushdown)                     14261          14268           7          1.1         906.7       1.0X
-Native ORC Vectorized                             14832          14880          44          1.1         943.0       1.0X
-Native ORC Vectorized (Pushdown)                  14402          14428          28          1.1         915.7       1.0X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+---------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                         13453          13543          61          1.2         855.3       1.0X
+Parquet Vectorized (Pushdown)                              13147          13232          65          1.2         835.9       1.0X
+Native ORC Vectorized                                      15464          15648         153          1.0         983.2       0.9X
+Native ORC Vectorized (Pushdown)                           14666          14892         146          1.1         932.4       0.9X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 1 decimal(38, 2) row (value = 7864320):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 8132           8178          36          1.9         517.0       1.0X
-Parquet Vectorized (Pushdown)                       201            209          10         78.3          12.8      40.5X
-Native ORC Vectorized                              5270           5290          16          3.0         335.0       1.5X
-Native ORC Vectorized (Pushdown)                    149            153           2        105.2           9.5      54.4X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+-----------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                      6499           6536          28          2.4         413.2       1.0X
+Parquet Vectorized (Pushdown)                            154            167          10        101.8           9.8      42.1X
+Native ORC Vectorized                                   6092           6134          32          2.6         387.3       1.1X
+Native ORC Vectorized (Pushdown)                         175            189           8         90.0          11.1      37.2X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 10% decimal(38, 2) rows (value < 1572864):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 9422           9465          50          1.7         599.1       1.0X
-Parquet Vectorized (Pushdown)                      2271           2283          13          6.9         144.4       4.1X
-Native ORC Vectorized                              6651           6662          10          2.4         422.9       1.4X
-Native ORC Vectorized (Pushdown)                   2002           2016          17          7.9         127.3       4.7X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+--------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                         7674           7724          36          2.0         487.9       1.0X
+Parquet Vectorized (Pushdown)                              2067           2134          52          7.6         131.4       3.7X
+Native ORC Vectorized                                      7176           7249          52          2.2         456.3       1.1X
+Native ORC Vectorized (Pushdown)                           1880           1948          46          8.4         119.5       4.1X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 50% decimal(38, 2) rows (value < 7864320):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                14781          14814          37          1.1         939.7       1.0X
-Parquet Vectorized (Pushdown)                     10675          10696          24          1.5         678.7       1.4X
-Native ORC Vectorized                             11887          11983         101          1.3         755.8       1.2X
-Native ORC Vectorized (Pushdown)                   9383           9396          11          1.7         596.5       1.6X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+--------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                        13010          13089          89          1.2         827.2       1.0X
+Parquet Vectorized (Pushdown)                              9935          10013          64          1.6         631.6       1.3X
+Native ORC Vectorized                                     12398          12503          65          1.3         788.2       1.0X
+Native ORC Vectorized (Pushdown)                           9483           9594          82          1.7         602.9       1.4X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 90% decimal(38, 2) rows (value < 14155776):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                19753          19777          28          0.8        1255.9       1.0X
-Parquet Vectorized (Pushdown)                     19049          19089          49          0.8        1211.1       1.0X
-Native ORC Vectorized                             17238          17247           7          0.9        1096.0       1.1X
-Native ORC Vectorized (Pushdown)                  16780          16804          21          0.9        1066.9       1.2X
+---------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                         18271          18374          64          0.9        1161.7       1.0X
+Parquet Vectorized (Pushdown)                              17692          17759          79          0.9        1124.8       1.0X
+Native ORC Vectorized                                      17374          17657         324          0.9        1104.6       1.1X
+Native ORC Vectorized (Pushdown)                           16902          16933          23          0.9        1074.6       1.1X
 
 
 ================================================================================================
 Pushdown benchmark for InSet -> InFilters
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 InSet -> InFilters (values count: 5, distribution: 10):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                11101          11144          74          1.4         705.8       1.0X
-Parquet Vectorized (Pushdown)                       773            776           3         20.3          49.1      14.4X
-Native ORC Vectorized                              6973           7001          28          2.3         443.3       1.6X
-Native ORC Vectorized (Pushdown)                    445            451           5         35.4          28.3      25.0X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+--------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                               9945          10030          71          1.6         632.3       1.0X
+Parquet Vectorized (Pushdown)                                     612            628          13         25.7          38.9      16.3X
+Native ORC Vectorized                                            8240           8295          55          1.9         523.9       1.2X
+Native ORC Vectorized (Pushdown)                                  564            593          21         27.9          35.8      17.6X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 InSet -> InFilters (values count: 5, distribution: 50):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                11098          11111          13          1.4         705.6       1.0X
-Parquet Vectorized (Pushdown)                       775            778           4         20.3          49.2      14.3X
-Native ORC Vectorized                              6980           6996           9          2.3         443.8       1.6X
-Native ORC Vectorized (Pushdown)                    441            448           8         35.7          28.0      25.2X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+--------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                              10131          10207          77          1.6         644.1       1.0X
+Parquet Vectorized (Pushdown)                                     608            627          30         25.9          38.7      16.7X
+Native ORC Vectorized                                            8216           8287          49          1.9         522.3       1.2X
+Native ORC Vectorized (Pushdown)                                  561            578          23         28.0          35.7      18.0X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 InSet -> InFilters (values count: 5, distribution: 90):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                11089          11118          26          1.4         705.0       1.0X
-Parquet Vectorized (Pushdown)                       772            777           3         20.4          49.1      14.4X
-Native ORC Vectorized                              6973           6985           9          2.3         443.3       1.6X
-Native ORC Vectorized (Pushdown)                    446            448           3         35.3          28.3      24.9X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+--------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                              10163          10239          64          1.5         646.2       1.0X
+Parquet Vectorized (Pushdown)                                     608            632          17         25.9          38.7      16.7X
+Native ORC Vectorized                                            8122           8166          49          1.9         516.4       1.3X
+Native ORC Vectorized (Pushdown)                                  558            572           9         28.2          35.5      18.2X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 InSet -> InFilters (values count: 10, distribution: 10):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                11113          11137          19          1.4         706.5       1.0X
-Parquet Vectorized (Pushdown)                       802            807           5         19.6          51.0      13.9X
-Native ORC Vectorized                              7034           7080          63          2.2         447.2       1.6X
-Native ORC Vectorized (Pushdown)                    461            469           5         34.1          29.3      24.1X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+---------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                               10098          10246         108          1.6         642.0       1.0X
+Parquet Vectorized (Pushdown)                                      623            654          30         25.3          39.6      16.2X
+Native ORC Vectorized                                             8322           8351          34          1.9         529.1       1.2X
+Native ORC Vectorized (Pushdown)                                   601            615          16         26.2          38.2      16.8X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 InSet -> InFilters (values count: 10, distribution: 50):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                11122          11134          16          1.4         707.1       1.0X
-Parquet Vectorized (Pushdown)                       804            817          16         19.6          51.1      13.8X
-Native ORC Vectorized                              7027           7037          10          2.2         446.8       1.6X
-Native ORC Vectorized (Pushdown)                    463            470           5         34.0          29.4      24.0X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+---------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                               10123          10195          42          1.6         643.6       1.0X
+Parquet Vectorized (Pushdown)                                      651            667          13         24.2          41.4      15.5X
+Native ORC Vectorized                                             8187           8286          83          1.9         520.5       1.2X
+Native ORC Vectorized (Pushdown)                                   565            595          18         27.8          35.9      17.9X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 InSet -> InFilters (values count: 10, distribution: 90):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                11097          11115          13          1.4         705.5       1.0X
-Parquet Vectorized (Pushdown)                       804            810           6         19.6          51.1      13.8X
-Native ORC Vectorized                              7010           7022          15          2.2         445.7       1.6X
-Native ORC Vectorized (Pushdown)                    465            468           4         33.8          29.6      23.9X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+---------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                               10220          10263          38          1.5         649.8       1.0X
+Parquet Vectorized (Pushdown)                                      634            650          17         24.8          40.3      16.1X
+Native ORC Vectorized                                             8290           8335          50          1.9         527.0       1.2X
+Native ORC Vectorized (Pushdown)                                   599            612          10         26.3          38.1      17.1X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 InSet -> InFilters (values count: 50, distribution: 10):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                11378          11387           7          1.4         723.4       1.0X
-Parquet Vectorized (Pushdown)                     11640          11675          37          1.4         740.1       1.0X
-Native ORC Vectorized                              7266           7284          24          2.2         462.0       1.6X
-Native ORC Vectorized (Pushdown)                    612            624          12         25.7          38.9      18.6X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+---------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                               10551          10635          75          1.5         670.8       1.0X
+Parquet Vectorized (Pushdown)                                    10515          10593          59          1.5         668.5       1.0X
+Native ORC Vectorized                                             8460           8547         117          1.9         537.9       1.2X
+Native ORC Vectorized (Pushdown)                                   764            789          22         20.6          48.6      13.8X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 InSet -> InFilters (values count: 50, distribution: 50):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                11411          11448          49          1.4         725.5       1.0X
-Parquet Vectorized (Pushdown)                     11670          11692          20          1.3         741.9       1.0X
-Native ORC Vectorized                              7284           7292          10          2.2         463.1       1.6X
-Native ORC Vectorized (Pushdown)                    652            659           8         24.1          41.4      17.5X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+---------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                               10528          10595          69          1.5         669.4       1.0X
+Parquet Vectorized (Pushdown)                                    10519          10570          65          1.5         668.8       1.0X
+Native ORC Vectorized                                             8550           8688         139          1.8         543.6       1.2X
+Native ORC Vectorized (Pushdown)                                   800            809           7         19.7          50.8      13.2X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 InSet -> InFilters (values count: 50, distribution: 90):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                11399          11447          71          1.4         724.7       1.0X
-Parquet Vectorized (Pushdown)                     11659          11684          15          1.3         741.3       1.0X
-Native ORC Vectorized                              7290           7301          10          2.2         463.5       1.6X
-Native ORC Vectorized (Pushdown)                    655            700          77         24.0          41.7      17.4X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+---------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                               10452          10605         131          1.5         664.5       1.0X
+Parquet Vectorized (Pushdown)                                    10467          10574          94          1.5         665.5       1.0X
+Native ORC Vectorized                                             8575           8646          58          1.8         545.2       1.2X
+Native ORC Vectorized (Pushdown)                                   806            822          13         19.5          51.2      13.0X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 InSet -> InFilters (values count: 100, distribution: 10):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                11398          11424          23          1.4         724.7       1.0X
-Parquet Vectorized (Pushdown)                     11660          11775         100          1.3         741.3       1.0X
-Native ORC Vectorized                              7292           7312          24          2.2         463.6       1.6X
-Native ORC Vectorized (Pushdown)                    789            796           5         19.9          50.2      14.4X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+----------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                10400          10540         117          1.5         661.2       1.0X
+Parquet Vectorized (Pushdown)                                     10487          10535          32          1.5         666.7       1.0X
+Native ORC Vectorized                                              8646           8751         135          1.8         549.7       1.2X
+Native ORC Vectorized (Pushdown)                                    923            942          16         17.0          58.7      11.3X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 InSet -> InFilters (values count: 100, distribution: 50):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                11399          11462          55          1.4         724.7       1.0X
-Parquet Vectorized (Pushdown)                     11648          11670          24          1.4         740.6       1.0X
-Native ORC Vectorized                              7254           7265           8          2.2         461.2       1.6X
-Native ORC Vectorized (Pushdown)                    851            857           5         18.5          54.1      13.4X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+----------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                10514          10592         100          1.5         668.5       1.0X
+Parquet Vectorized (Pushdown)                                     10391          10491         119          1.5         660.6       1.0X
+Native ORC Vectorized                                              8492           8616          94          1.9         539.9       1.2X
+Native ORC Vectorized (Pushdown)                                    999           1031          39         15.7          63.5      10.5X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 InSet -> InFilters (values count: 100, distribution: 90):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                11383          11499          97          1.4         723.7       1.0X
-Parquet Vectorized (Pushdown)                     11694          11731          22          1.3         743.5       1.0X
-Native ORC Vectorized                              7244           7272          22          2.2         460.6       1.6X
-Native ORC Vectorized (Pushdown)                    887            896          12         17.7          56.4      12.8X
+----------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                10396          10447          33          1.5         661.0       1.0X
+Parquet Vectorized (Pushdown)                                     10483          10532          89          1.5         666.5       1.0X
+Native ORC Vectorized                                              8596           8642          35          1.8         546.5       1.2X
+Native ORC Vectorized (Pushdown)                                   1027           1044          13         15.3          65.3      10.1X
 
 
 ================================================================================================
 Pushdown benchmark for tinyint
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 1 tinyint row (value = CAST(63 AS tinyint)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 6301           6337          24          2.5         400.6       1.0X
-Parquet Vectorized (Pushdown)                       254            260           4         61.9          16.2      24.8X
-Native ORC Vectorized                              3257           3273          12          4.8         207.1       1.9X
-Native ORC Vectorized (Pushdown)                    211            217           4         74.4          13.4      29.8X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+----------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                           4181           4231          44          3.8         265.9       1.0X
+Parquet Vectorized (Pushdown)                                 202            209          10         77.9          12.8      20.7X
+Native ORC Vectorized                                        3879           3969          79          4.1         246.6       1.1X
+Native ORC Vectorized (Pushdown)                              253            261           9         62.2          16.1      16.5X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 10% tinyint rows (value < CAST(12 AS tinyint)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 7169           7219          46          2.2         455.8       1.0X
-Parquet Vectorized (Pushdown)                      1665           1672          12          9.4         105.9       4.3X
-Native ORC Vectorized                              4199           4208          12          3.7         267.0       1.7X
-Native ORC Vectorized (Pushdown)                   1376           1394          30         11.4          87.5       5.2X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+-------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                              5054           5147          82          3.1         321.3       1.0X
+Parquet Vectorized (Pushdown)                                   1437           1479          29         10.9          91.3       3.5X
+Native ORC Vectorized                                           4585           4656          54          3.4         291.5       1.1X
+Native ORC Vectorized (Pushdown)                                1356           1400          41         11.6          86.2       3.7X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 50% tinyint rows (value < CAST(63 AS tinyint)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                10703          10739          29          1.5         680.5       1.0X
-Parquet Vectorized (Pushdown)                      7648           7671          20          2.1         486.3       1.4X
-Native ORC Vectorized                              7815           7836          14          2.0         496.9       1.4X
-Native ORC Vectorized (Pushdown)                   6289           6295           9          2.5         399.8       1.7X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+-------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                              8744           8866         117          1.8         555.9       1.0X
+Parquet Vectorized (Pushdown)                                   6744           6844          71          2.3         428.8       1.3X
+Native ORC Vectorized                                           7921           7993          44          2.0         503.6       1.1X
+Native ORC Vectorized (Pushdown)                                6123           6255         101          2.6         389.3       1.4X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 90% tinyint rows (value < CAST(114 AS tinyint)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                14239          14284          35          1.1         905.3       1.0X
-Parquet Vectorized (Pushdown)                     13733          13769          28          1.1         873.1       1.0X
-Native ORC Vectorized                             11432          11481          32          1.4         726.8       1.2X
-Native ORC Vectorized (Pushdown)                  11241          11254          11          1.4         714.7       1.3X
+--------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                              12765          12796          36          1.2         811.6       1.0X
+Parquet Vectorized (Pushdown)                                   12422          12519          88          1.3         789.7       1.0X
+Native ORC Vectorized                                           11353          11452          81          1.4         721.8       1.1X
+Native ORC Vectorized (Pushdown)                                11027          11161         143          1.4         701.1       1.2X
 
 
 ================================================================================================
 Pushdown benchmark for Timestamp
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Select 1 timestamp stored as INT96 row (value = CAST(7864320 AS timestamp)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 6805           6823          17          2.3         432.7       1.0X
-Parquet Vectorized (Pushdown)                      6876           6892          11          2.3         437.2       1.0X
-Native ORC Vectorized                              3186           3208          33          4.9         202.5       2.1X
-Native ORC Vectorized (Pushdown)                    131            140           9        120.4           8.3      52.1X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Select 10% timestamp stored as INT96 rows (value < CAST(1572864 AS timestamp)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 7742           7765          21          2.0         492.2       1.0X
-Parquet Vectorized (Pushdown)                      7798           7853          70          2.0         495.8       1.0X
-Native ORC Vectorized                              4128           4138           8          3.8         262.4       1.9X
-Native ORC Vectorized (Pushdown)                   1392           1402          17         11.3          88.5       5.6X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Select 50% timestamp stored as INT96 rows (value < CAST(7864320 AS timestamp)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                11354          11380          33          1.4         721.9       1.0X
-Parquet Vectorized (Pushdown)                     11408          11449          36          1.4         725.3       1.0X
-Native ORC Vectorized                              7752           7780          26          2.0         492.8       1.5X
-Native ORC Vectorized (Pushdown)                   6233           6246          12          2.5         396.3       1.8X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Select 90% timestamp stored as INT96 rows (value < CAST(14155776 AS timestamp)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                14944          14997          49          1.1         950.1       1.0X
-Parquet Vectorized (Pushdown)                     15066          15120          95          1.0         957.9       1.0X
-Native ORC Vectorized                             11422          11487          45          1.4         726.2       1.3X
-Native ORC Vectorized (Pushdown)                  11169          11211          30          1.4         710.1       1.3X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Select 1 timestamp stored as TIMESTAMP_MICROS row (value = CAST(7864320 AS timestamp)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 6047           6083          28          2.6         384.5       1.0X
-Parquet Vectorized (Pushdown)                       188            192           3         83.5          12.0      32.1X
-Native ORC Vectorized                              3169           3180          11          5.0         201.5       1.9X
-Native ORC Vectorized (Pushdown)                    127            138          15        124.0           8.1      47.7X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Select 10% timestamp stored as TIMESTAMP_MICROS rows (value < CAST(1572864 AS timestamp)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 6967           7009          26          2.3         443.0       1.0X
-Parquet Vectorized (Pushdown)                      1705           1711           7          9.2         108.4       4.1X
-Native ORC Vectorized                              4092           4115          18          3.8         260.2       1.7X
-Native ORC Vectorized (Pushdown)                   1389           1394           9         11.3          88.3       5.0X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Select 50% timestamp stored as TIMESTAMP_MICROS rows (value < CAST(7864320 AS timestamp)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                10658          10695          23          1.5         677.6       1.0X
-Parquet Vectorized (Pushdown)                      7760           7792          35          2.0         493.4       1.4X
-Native ORC Vectorized                              7780           7802          24          2.0         494.6       1.4X
-Native ORC Vectorized (Pushdown)                   6241           6279          24          2.5         396.8       1.7X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Select 90% timestamp stored as TIMESTAMP_MICROS rows (value < CAST(14155776 AS timestamp)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                14209          14253          53          1.1         903.4       1.0X
-Parquet Vectorized (Pushdown)                     13742          13764          15          1.1         873.7       1.0X
-Native ORC Vectorized                             11437          11463          18          1.4         727.1       1.2X
-Native ORC Vectorized (Pushdown)                  11172          11218          43          1.4         710.3       1.3X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Select 1 timestamp stored as TIMESTAMP_MILLIS row (value = CAST(7864320 AS timestamp)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 6124           6173          46          2.6         389.4       1.0X
-Parquet Vectorized (Pushdown)                       187            189           3         84.2          11.9      32.8X
-Native ORC Vectorized                              3161           3174          13          5.0         200.9       1.9X
-Native ORC Vectorized (Pushdown)                    129            144          24        122.4           8.2      47.6X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Select 10% timestamp stored as TIMESTAMP_MILLIS rows (value < CAST(1572864 AS timestamp)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 7068           7094          23          2.2         449.4       1.0X
-Parquet Vectorized (Pushdown)                      1724           1740          19          9.1         109.6       4.1X
-Native ORC Vectorized                              4127           4137          12          3.8         262.4       1.7X
-Native ORC Vectorized (Pushdown)                   1385           1393          10         11.4          88.1       5.1X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Select 50% timestamp stored as TIMESTAMP_MILLIS rows (value < CAST(7864320 AS timestamp)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                10767          10786          21          1.5         684.5       1.0X
-Parquet Vectorized (Pushdown)                      7805           7820          13          2.0         496.2       1.4X
-Native ORC Vectorized                              7714           7775          64          2.0         490.4       1.4X
-Native ORC Vectorized (Pushdown)                   6247           6266          17          2.5         397.2       1.7X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Select 90% timestamp stored as TIMESTAMP_MILLIS rows (value < CAST(14155776 AS timestamp)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                14337          14392          34          1.1         911.5       1.0X
-Parquet Vectorized (Pushdown)                     13774          13791          13          1.1         875.7       1.0X
-Native ORC Vectorized                             11426          11448          16          1.4         726.4       1.3X
-Native ORC Vectorized (Pushdown)                  11149          11181          27          1.4         708.8       1.3X
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+Select 1 timestamp stored as INT96 row (value = timestamp_seconds(7864320)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-----------------------------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                                    4418           4479          39          3.6         280.9       1.0X
+Parquet Vectorized (Pushdown)                                                         4461           4515          57          3.5         283.6       1.0X
+Native ORC Vectorized                                                                 3837           3923          93          4.1         243.9       1.2X
+Native ORC Vectorized (Pushdown)                                                       150            159           6        104.9           9.5      29.5X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+Select 10% timestamp stored as INT96 rows (value < timestamp_seconds(1572864)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+--------------------------------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                                       5516           5557          40          2.9         350.7       1.0X
+Parquet Vectorized (Pushdown)                                                            5394           5511          81          2.9         342.9       1.0X
+Native ORC Vectorized                                                                    4725           4797          57          3.3         300.4       1.2X
+Native ORC Vectorized (Pushdown)                                                         1428           1482          37         11.0          90.8       3.9X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+Select 50% timestamp stored as INT96 rows (value < timestamp_seconds(7864320)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+--------------------------------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                                       9602           9660          46          1.6         610.5       1.0X
+Parquet Vectorized (Pushdown)                                                            9652           9750          74          1.6         613.7       1.0X
+Native ORC Vectorized                                                                    8351           8435         103          1.9         530.9       1.1X
+Native ORC Vectorized (Pushdown)                                                         6625           6711          92          2.4         421.2       1.4X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+Select 90% timestamp stored as INT96 rows (value < timestamp_seconds(14155776)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+---------------------------------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                                       13573          13738         100          1.2         862.9       1.0X
+Parquet Vectorized (Pushdown)                                                            13553          13737         168          1.2         861.7       1.0X
+Native ORC Vectorized                                                                    12094          12204          85          1.3         768.9       1.1X
+Native ORC Vectorized (Pushdown)                                                         11801          11961         102          1.3         750.3       1.2X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+Select 1 timestamp stored as TIMESTAMP_MICROS row (value = timestamp_seconds(7864320)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                                               3810           3885          44          4.1         242.3       1.0X
+Parquet Vectorized (Pushdown)                                                                     137            146          10        114.5           8.7      27.7X
+Native ORC Vectorized                                                                            3879           3936          33          4.1         246.6       1.0X
+Native ORC Vectorized (Pushdown)                                                                  145            156           9        108.1           9.2      26.2X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+Select 10% timestamp stored as TIMESTAMP_MICROS rows (value < timestamp_seconds(1572864)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                                                  4811           4899          70          3.3         305.9       1.0X
+Parquet Vectorized (Pushdown)                                                                       1508           1533          18         10.4          95.9       3.2X
+Native ORC Vectorized                                                                               4738           4815          68          3.3         301.2       1.0X
+Native ORC Vectorized (Pushdown)                                                                    1462           1484          19         10.8          93.0       3.3X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+Select 50% timestamp stored as TIMESTAMP_MICROS rows (value < timestamp_seconds(7864320)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                                                  8971           9010          61          1.8         570.4       1.0X
+Parquet Vectorized (Pushdown)                                                                       7104           7168          74          2.2         451.6       1.3X
+Native ORC Vectorized                                                                               8245           8325          57          1.9         524.2       1.1X
+Native ORC Vectorized (Pushdown)                                                                    6465           6621         101          2.4         411.1       1.4X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+Select 90% timestamp stored as TIMESTAMP_MICROS rows (value < timestamp_seconds(14155776)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                                                  12934          13023          86          1.2         822.3       1.0X
+Parquet Vectorized (Pushdown)                                                                       12688          12766          79          1.2         806.7       1.0X
+Native ORC Vectorized                                                                               12007          12056          62          1.3         763.4       1.1X
+Native ORC Vectorized (Pushdown)                                                                    11717          11865         135          1.3         744.9       1.1X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+Select 1 timestamp stored as TIMESTAMP_MILLIS row (value = timestamp_seconds(7864320)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                                               3832           3911          61          4.1         243.6       1.0X
+Parquet Vectorized (Pushdown)                                                                     137            147           8        114.9           8.7      28.0X
+Native ORC Vectorized                                                                            3861           3894          34          4.1         245.5       1.0X
+Native ORC Vectorized (Pushdown)                                                                  151            162           7        104.4           9.6      25.4X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+Select 10% timestamp stored as TIMESTAMP_MILLIS rows (value < timestamp_seconds(1572864)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                                                  4830           4916          68          3.3         307.1       1.0X
+Parquet Vectorized (Pushdown)                                                                       1538           1567          18         10.2          97.8       3.1X
+Native ORC Vectorized                                                                               4707           4869          99          3.3         299.2       1.0X
+Native ORC Vectorized (Pushdown)                                                                    1437           1476          34         10.9          91.4       3.4X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+Select 50% timestamp stored as TIMESTAMP_MILLIS rows (value < timestamp_seconds(7864320)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                                                  8933           9014          65          1.8         567.9       1.0X
+Parquet Vectorized (Pushdown)                                                                       7178           7269         111          2.2         456.4       1.2X
+Native ORC Vectorized                                                                               8253           8329          65          1.9         524.7       1.1X
+Native ORC Vectorized (Pushdown)                                                                    6499           6575          63          2.4         413.2       1.4X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+Select 90% timestamp stored as TIMESTAMP_MILLIS rows (value < timestamp_seconds(14155776)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                                                  12848          12984          84          1.2         816.8       1.0X
+Parquet Vectorized (Pushdown)                                                                       12504          12702         174          1.3         795.0       1.0X
+Native ORC Vectorized                                                                               11950          12124         113          1.3         759.8       1.1X
+Native ORC Vectorized (Pushdown)                                                                    11713          11794          77          1.3         744.7       1.1X
 
 
 ================================================================================================
 Pushdown benchmark with many filters
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 1 row with 1 filters:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                  509            514           6          0.0   509244147.0       1.0X
-Parquet Vectorized (Pushdown)                       512            519           6          0.0   511810881.0       1.0X
-Native ORC Vectorized                               488            495           8          0.0   488404846.0       1.0X
-Native ORC Vectorized (Pushdown)                    489            500          16          0.0   489064429.0       1.0X
+Parquet Vectorized                                  195            211          10          0.0   195007208.0       1.0X
+Parquet Vectorized (Pushdown)                       189            201          12          0.0   188853012.0       1.0X
+Native ORC Vectorized                               172            185           8          0.0   171902507.0       1.1X
+Native ORC Vectorized (Pushdown)                    178            188          10          0.0   177535267.0       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 1 row with 250 filters:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 1704           1712           7          0.0  1703741457.0       1.0X
-Parquet Vectorized (Pushdown)                      1782           1794           7          0.0  1781727704.0       1.0X
-Native ORC Vectorized                              1668           1674           5          0.0  1668306343.0       1.0X
-Native ORC Vectorized (Pushdown)                   1670           1679           7          0.0  1669592413.0       1.0X
+Parquet Vectorized                                 2024           2103          64          0.0  2024084302.0       1.0X
+Parquet Vectorized (Pushdown)                      2041           2141          82          0.0  2041237030.0       1.0X
+Native ORC Vectorized                              1946           2052          79          0.0  1945760183.0       1.0X
+Native ORC Vectorized (Pushdown)                   1975           2073          61          0.0  1975176024.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select 1 row with 500 filters:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 5011           5044          25          0.0  5011180244.0       1.0X
-Parquet Vectorized (Pushdown)                      5375           5397          17          0.0  5375351637.0       0.9X
-Native ORC Vectorized                              4979           4992          12          0.0  4979326800.0       1.0X
-Native ORC Vectorized (Pushdown)                   4979           5008          26          0.0  4978952429.0       1.0X
+Parquet Vectorized                                11078          11226         162          0.0 11077796980.0       1.0X
+Parquet Vectorized (Pushdown)                     11266          11386         118          0.0 11266306946.0       1.0X
+Native ORC Vectorized                             11183          11310         171          0.0 11183362739.0       1.0X
+Native ORC Vectorized (Pushdown)                  11092          11181          71          0.0 11092193039.0       1.0X
 
 
diff --git a/sql/core/benchmarks/FilterPushdownBenchmark-results.txt b/sql/core/benchmarks/FilterPushdownBenchmark-results.txt
index 1fda7bffc8e82..c4e938716448c 100644
--- a/sql/core/benchmarks/FilterPushdownBenchmark-results.txt
+++ b/sql/core/benchmarks/FilterPushdownBenchmark-results.txt
@@ -2,669 +2,669 @@
 Pushdown for many distinct value case
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 0 string row (value IS NULL):      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 9582           9636          49          1.6         609.2       1.0X
-Parquet Vectorized (Pushdown)                       759            785          40         20.7          48.3      12.6X
-Native ORC Vectorized                              7141           7183          60          2.2         454.0       1.3X
-Native ORC Vectorized (Pushdown)                    513            556          49         30.7          32.6      18.7X
+Parquet Vectorized                                 9604           9646          56          1.6         610.6       1.0X
+Parquet Vectorized (Pushdown)                       615            626           8         25.6          39.1      15.6X
+Native ORC Vectorized                              7129           8305        1477          2.2         453.3       1.3X
+Native ORC Vectorized (Pushdown)                    592            615          27         26.6          37.7      16.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 0 string row ('7864320' < value < '7864320'):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 9657           9675          22          1.6         614.0       1.0X
-Parquet Vectorized (Pushdown)                       707            724          26         22.2          44.9      13.7X
-Native ORC Vectorized                              7250           7296          49          2.2         460.9       1.3X
-Native ORC Vectorized (Pushdown)                    503            537          51         31.3          32.0      19.2X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+-----------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                            9809          10636        1264          1.6         623.7       1.0X
+Parquet Vectorized (Pushdown)                                  599            611          10         26.2          38.1      16.4X
+Native ORC Vectorized                                         7343           7479         267          2.1         466.9       1.3X
+Native ORC Vectorized (Pushdown)                               597            662          85         26.3          38.0      16.4X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 1 string row (value = '7864320'):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 9636           9657          20          1.6         612.7       1.0X
-Parquet Vectorized (Pushdown)                       679            693          13         23.2          43.2      14.2X
-Native ORC Vectorized                              7327           7359          25          2.1         465.8       1.3X
-Native ORC Vectorized (Pushdown)                    483            512          30         32.6          30.7      20.0X
+Parquet Vectorized                                 9816          10707        1227          1.6         624.1       1.0X
+Parquet Vectorized (Pushdown)                       599            610          18         26.2          38.1      16.4X
+Native ORC Vectorized                              7356           8080        1365          2.1         467.7       1.3X
+Native ORC Vectorized (Pushdown)                    583            609          39         27.0          37.0      16.8X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 1 string row (value <=> '7864320'):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 9688           9710          29          1.6         615.9       1.0X
-Parquet Vectorized (Pushdown)                       674            692          14         23.3          42.8      14.4X
-Native ORC Vectorized                              7315           7343          22          2.2         465.1       1.3X
-Native ORC Vectorized (Pushdown)                    483            498          24         32.6          30.7      20.1X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+-------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                  9800          10170         789          1.6         623.1       1.0X
+Parquet Vectorized (Pushdown)                        597            607          14         26.4          37.9      16.4X
+Native ORC Vectorized                               7373           7493         256          2.1         468.8       1.3X
+Native ORC Vectorized (Pushdown)                     587            625          76         26.8          37.3      16.7X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 1 string row ('7864320' <= value <= '7864320'):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 9611           9638          21          1.6         611.1       1.0X
-Parquet Vectorized (Pushdown)                       670            676           4         23.5          42.6      14.3X
-Native ORC Vectorized                              7261           7313          31          2.2         461.6       1.3X
-Native ORC Vectorized (Pushdown)                    480            518          51         32.8          30.5      20.0X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+-------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                              9817           9865          79          1.6         624.1       1.0X
+Parquet Vectorized (Pushdown)                                    595            608          11         26.4          37.8      16.5X
+Native ORC Vectorized                                           7319           7347          22          2.1         465.3       1.3X
+Native ORC Vectorized (Pushdown)                                 584            614          44         26.9          37.1      16.8X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select all string rows (value IS NOT NULL):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                18597          18738          91          0.8        1182.4       1.0X
-Parquet Vectorized (Pushdown)                     18864          18915          30          0.8        1199.4       1.0X
-Native ORC Vectorized                             16865          16968          67          0.9        1072.3       1.1X
-Native ORC Vectorized (Pushdown)                  17056          17168         120          0.9        1084.4       1.1X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+--------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                  19560          19607          56          0.8        1243.6       1.0X
+Parquet Vectorized (Pushdown)                       19579          19657          54          0.8        1244.8       1.0X
+Native ORC Vectorized                               17091          17112          19          0.9        1086.6       1.1X
+Native ORC Vectorized (Pushdown)                    17297          17333          27          0.9        1099.7       1.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 0 int row (value IS NULL):         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 8923           8981          48          1.8         567.3       1.0X
-Parquet Vectorized (Pushdown)                       651            670          25         24.1          41.4      13.7X
-Native ORC Vectorized                              6477           6507          46          2.4         411.8       1.4X
-Native ORC Vectorized (Pushdown)                    453            470          26         34.7          28.8      19.7X
+Parquet Vectorized                                 9173           9193          24          1.7         583.2       1.0X
+Parquet Vectorized (Pushdown)                       585            591           7         26.9          37.2      15.7X
+Native ORC Vectorized                              6644           7830        1272          2.4         422.4       1.4X
+Native ORC Vectorized (Pushdown)                    550            570          38         28.6          34.9      16.7X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 0 int row (7864320 < value < 7864320):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 8979           8989          12          1.8         570.9       1.0X
-Parquet Vectorized (Pushdown)                       659            674          15         23.9          41.9      13.6X
-Native ORC Vectorized                              6502           6522          13          2.4         413.4       1.4X
-Native ORC Vectorized (Pushdown)                    460            479          22         34.2          29.2      19.5X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+----------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                     9648          10861        1284          1.6         613.4       1.0X
+Parquet Vectorized (Pushdown)                           586            601          16         26.8          37.2      16.5X
+Native ORC Vectorized                                  6635           7205         965          2.4         421.8       1.5X
+Native ORC Vectorized (Pushdown)                        560            581          36         28.1          35.6      17.2X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 1 int row (value = 7864320):       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 9046           9065          23          1.7         575.1       1.0X
-Parquet Vectorized (Pushdown)                       654            662           9         24.1          41.6      13.8X
-Native ORC Vectorized                              6592           6617          24          2.4         419.1       1.4X
-Native ORC Vectorized (Pushdown)                    450            468          20         34.9          28.6      20.1X
+Parquet Vectorized                                 9213           9728        1060          1.7         585.7       1.0X
+Parquet Vectorized (Pushdown)                       588            597           9         26.8          37.4      15.7X
+Native ORC Vectorized                              6735           7233         890          2.3         428.2       1.4X
+Native ORC Vectorized (Pushdown)                    559            589          39         28.1          35.6      16.5X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 1 int row (value <=> 7864320):     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 9036           9052          16          1.7         574.5       1.0X
-Parquet Vectorized (Pushdown)                       652            661           9         24.1          41.5      13.9X
-Native ORC Vectorized                              6566           6616          36          2.4         417.4       1.4X
-Native ORC Vectorized (Pushdown)                    456            493          47         34.5          29.0      19.8X
+Parquet Vectorized                                 9202           9441         440          1.7         585.0       1.0X
+Parquet Vectorized (Pushdown)                       584            592          12         26.9          37.2      15.7X
+Native ORC Vectorized                              6760           7297         856          2.3         429.8       1.4X
+Native ORC Vectorized (Pushdown)                    547            572          36         28.7          34.8      16.8X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 1 int row (7864320 <= value <= 7864320):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 9024           9062          52          1.7         573.7       1.0X
-Parquet Vectorized (Pushdown)                       657            662           4         23.9          41.8      13.7X
-Native ORC Vectorized                              6575           6606          19          2.4         418.1       1.4X
-Native ORC Vectorized (Pushdown)                    454            464          15         34.6          28.9      19.9X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                       9248           9934        1130          1.7         588.0       1.0X
+Parquet Vectorized (Pushdown)                             590            596           6         26.7          37.5      15.7X
+Native ORC Vectorized                                    6711           6863         285          2.3         426.6       1.4X
+Native ORC Vectorized (Pushdown)                          558            584          39         28.2          35.5      16.6X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 1 int row (7864319 < value < 7864321):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 8992           9012          18          1.7         571.7       1.0X
-Parquet Vectorized (Pushdown)                       651            656           4         24.2          41.4      13.8X
-Native ORC Vectorized                              6615           6622           5          2.4         420.6       1.4X
-Native ORC Vectorized (Pushdown)                    455            467          22         34.5          29.0      19.7X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+----------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                     9233           9821         891          1.7         587.0       1.0X
+Parquet Vectorized (Pushdown)                           585            596           8         26.9          37.2      15.8X
+Native ORC Vectorized                                  6721           7310         887          2.3         427.3       1.4X
+Native ORC Vectorized (Pushdown)                        556            579          38         28.3          35.3      16.6X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 10% int rows (value < 1572864):    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 9928           9972          49          1.6         631.2       1.0X
-Parquet Vectorized (Pushdown)                      2416           2427          11          6.5         153.6       4.1X
-Native ORC Vectorized                              7620           7650          21          2.1         484.5       1.3X
-Native ORC Vectorized (Pushdown)                   2094           2161         119          7.5         133.1       4.7X
+Parquet Vectorized                                10192          10390         401          1.5         648.0       1.0X
+Parquet Vectorized (Pushdown)                      2422           2434          10          6.5         154.0       4.2X
+Native ORC Vectorized                              7708           8361         964          2.0         490.1       1.3X
+Native ORC Vectorized (Pushdown)                   2185           2209          18          7.2         138.9       4.7X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 50% int rows (value < 7864320):    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                13110          13154          46          1.2         833.5       1.0X
-Parquet Vectorized (Pushdown)                      9096           9130          34          1.7         578.3       1.4X
-Native ORC Vectorized                             11109          11139          21          1.4         706.3       1.2X
-Native ORC Vectorized (Pushdown)                   8100           8158          40          1.9         515.0       1.6X
+Parquet Vectorized                                13724          14226         944          1.1         872.5       1.0X
+Parquet Vectorized (Pushdown)                      9487           9519          27          1.7         603.2       1.4X
+Native ORC Vectorized                             11327          11352          22          1.4         720.1       1.2X
+Native ORC Vectorized (Pushdown)                   8359           8372          17          1.9         531.4       1.6X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 90% int rows (value < 14155776):   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                16372          16424          53          1.0        1040.9       1.0X
-Parquet Vectorized (Pushdown)                     15745          15794          51          1.0        1001.0       1.0X
-Native ORC Vectorized                             14642          14681          31          1.1         930.9       1.1X
-Native ORC Vectorized (Pushdown)                  14209          14239          19          1.1         903.4       1.2X
+Parquet Vectorized                                17332          17598         515          0.9        1102.0       1.0X
+Parquet Vectorized (Pushdown)                     16520          16564          28          1.0        1050.3       1.0X
+Native ORC Vectorized                             14885          15133         477          1.1         946.4       1.2X
+Native ORC Vectorized (Pushdown)                  14434          14499          56          1.1         917.7       1.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select all int rows (value IS NOT NULL):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                17032          17078          45          0.9        1082.9       1.0X
-Parquet Vectorized (Pushdown)                     17495          17509          18          0.9        1112.3       1.0X
-Native ORC Vectorized                             15626          15720          61          1.0         993.5       1.1X
-Native ORC Vectorized (Pushdown)                  15871          15950          71          1.0        1009.1       1.1X
+Parquet Vectorized                                18152          18276         183          0.9        1154.1       1.0X
+Parquet Vectorized (Pushdown)                     18251          18299          35          0.9        1160.4       1.0X
+Native ORC Vectorized                             15640          15675          45          1.0         994.3       1.2X
+Native ORC Vectorized (Pushdown)                  15840          15871          30          1.0        1007.1       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select all int rows (value > -1):         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                17169          17200          20          0.9        1091.6       1.0X
-Parquet Vectorized (Pushdown)                     17450          17490          33          0.9        1109.4       1.0X
-Native ORC Vectorized                             16062          16134          81          1.0        1021.2       1.1X
-Native ORC Vectorized (Pushdown)                  16225          16281          48          1.0        1031.5       1.1X
+Parquet Vectorized                                18139          18190          39          0.9        1153.2       1.0X
+Parquet Vectorized (Pushdown)                     18216          18250          35          0.9        1158.1       1.0X
+Native ORC Vectorized                             15753          15793          29          1.0        1001.6       1.2X
+Native ORC Vectorized (Pushdown)                  15947          15997          38          1.0        1013.9       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select all int rows (value != -1):        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                17091          17199         102          0.9        1086.6       1.0X
-Parquet Vectorized (Pushdown)                     17394          17480          60          0.9        1105.9       1.0X
-Native ORC Vectorized                             16322          16366          35          1.0        1037.7       1.0X
-Native ORC Vectorized (Pushdown)                  16459          16543          52          1.0        1046.4       1.0X
+Parquet Vectorized                                18169          18205          31          0.9        1155.2       1.0X
+Parquet Vectorized (Pushdown)                     18220          18262          45          0.9        1158.4       1.0X
+Native ORC Vectorized                             15797          15840          27          1.0        1004.3       1.2X
+Native ORC Vectorized (Pushdown)                  15980          16030          43          1.0        1015.9       1.1X
 
 
 ================================================================================================
 Pushdown for few distinct value case (use dictionary encoding)
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 0 distinct string row (value IS NULL):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 8671           8722          42          1.8         551.3       1.0X
-Parquet Vectorized (Pushdown)                       567            576           9         27.7          36.1      15.3X
-Native ORC Vectorized                              8567           8600          48          1.8         544.7       1.0X
-Native ORC Vectorized (Pushdown)                    846            870          30         18.6          53.8      10.2X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+----------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                    11129          11136           6          1.4         707.6       1.0X
+Parquet Vectorized (Pushdown)                           507            515          13         31.0          32.2      21.9X
+Native ORC Vectorized                                 11193          11271         130          1.4         711.7       1.0X
+Native ORC Vectorized (Pushdown)                       1043           1078          38         15.1          66.3      10.7X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 0 distinct string row ('100' < value < '100'):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 8852           8890          30          1.8         562.8       1.0X
-Parquet Vectorized (Pushdown)                       563            567           5         27.9          35.8      15.7X
-Native ORC Vectorized                              8816           8847          25          1.8         560.5       1.0X
-Native ORC Vectorized (Pushdown)                    838            869          41         18.8          53.3      10.6X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                            11313          11357          57          1.4         719.3       1.0X
+Parquet Vectorized (Pushdown)                                   515            522          10         30.6          32.7      22.0X
+Native ORC Vectorized                                         11489          11532          27          1.4         730.4       1.0X
+Native ORC Vectorized (Pushdown)                               1043           1082          42         15.1          66.3      10.8X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 1 distinct string row (value = '100'):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 8764           8799          24          1.8         557.2       1.0X
-Parquet Vectorized (Pushdown)                       632            637           6         24.9          40.2      13.9X
-Native ORC Vectorized                              8688           8707          24          1.8         552.4       1.0X
-Native ORC Vectorized (Pushdown)                    907           1073         259         17.3          57.7       9.7X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+----------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                    11181          11222          40          1.4         710.9       1.0X
+Parquet Vectorized (Pushdown)                           578            587          12         27.2          36.7      19.4X
+Native ORC Vectorized                                 11451          11522          45          1.4         728.0       1.0X
+Native ORC Vectorized (Pushdown)                       1093           1133          51         14.4          69.5      10.2X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 1 distinct string row (value <=> '100'):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 8789           8802          11          1.8         558.8       1.0X
-Parquet Vectorized (Pushdown)                       638            646          13         24.6          40.6      13.8X
-Native ORC Vectorized                              8707           8728          16          1.8         553.6       1.0X
-Native ORC Vectorized (Pushdown)                    909            931          26         17.3          57.8       9.7X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                      11156          11216          57          1.4         709.3       1.0X
+Parquet Vectorized (Pushdown)                             579            590          11         27.2          36.8      19.3X
+Native ORC Vectorized                                   11399          11427          24          1.4         724.7       1.0X
+Native ORC Vectorized (Pushdown)                         1089           1126          45         14.4          69.3      10.2X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 1 distinct string row ('100' <= value <= '100'):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 8888           8901          11          1.8         565.1       1.0X
-Parquet Vectorized (Pushdown)                       632            639           5         24.9          40.2      14.1X
-Native ORC Vectorized                              8835           9125         424          1.8         561.7       1.0X
-Native ORC Vectorized (Pushdown)                    909            951          53         17.3          57.8       9.8X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+--------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                              11347          11388          39          1.4         721.4       1.0X
+Parquet Vectorized (Pushdown)                                     582            590           6         27.0          37.0      19.5X
+Native ORC Vectorized                                           11536          11605          98          1.4         733.4       1.0X
+Native ORC Vectorized (Pushdown)                                 1100           1135          43         14.3          69.9      10.3X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select all distinct string rows (value IS NOT NULL):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                18995          19038          52          0.8        1207.7       1.0X
-Parquet Vectorized (Pushdown)                     19265          19339          56          0.8        1224.9       1.0X
-Native ORC Vectorized                             19356          19526         130          0.8        1230.6       1.0X
-Native ORC Vectorized (Pushdown)                  19683          19762         105          0.8        1251.4       1.0X
+-----------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                           22154          22191          23          0.7        1408.5       1.0X
+Parquet Vectorized (Pushdown)                                22123          22263         136          0.7        1406.5       1.0X
+Native ORC Vectorized                                        22058          22098          26          0.7        1402.4       1.0X
+Native ORC Vectorized (Pushdown)                             22367          22441          47          0.7        1422.1       1.0X
 
 
 ================================================================================================
 Pushdown benchmark for StringStartsWith
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 StringStartsWith filter: (value like '10%'):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 9986          10034          42          1.6         634.9       1.0X
-Parquet Vectorized (Pushdown)                      1539           1553          20         10.2          97.8       6.5X
-Native ORC Vectorized                              7520           7560          30          2.1         478.1       1.3X
-Native ORC Vectorized (Pushdown)                   7717           7764          37          2.0         490.6       1.3X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+---------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                   12880          12936          45          1.2         818.9       1.0X
+Parquet Vectorized (Pushdown)                         1490           1498           8         10.6          94.7       8.6X
+Native ORC Vectorized                                10279          10294          20          1.5         653.5       1.3X
+Native ORC Vectorized (Pushdown)                     10468          10507          32          1.5         665.5       1.2X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 StringStartsWith filter: (value like '1000%'):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 9711           9754          31          1.6         617.4       1.0X
-Parquet Vectorized (Pushdown)                       647            657          12         24.3          41.1      15.0X
-Native ORC Vectorized                              7257           7309          42          2.2         461.4       1.3X
-Native ORC Vectorized (Pushdown)                   7424           7497          56          2.1         472.0       1.3X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+-----------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                     12489          12506          21          1.3         794.0       1.0X
+Parquet Vectorized (Pushdown)                            596            607          12         26.4          37.9      21.0X
+Native ORC Vectorized                                  10072          10110          32          1.6         640.4       1.2X
+Native ORC Vectorized (Pushdown)                       10345          10393          61          1.5         657.7       1.2X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 StringStartsWith filter: (value like '786432%'):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 9747           9835          94          1.6         619.7       1.0X
-Parquet Vectorized (Pushdown)                       648            656           8         24.3          41.2      15.0X
-Native ORC Vectorized                              7233           7262          50          2.2         459.8       1.3X
-Native ORC Vectorized (Pushdown)                   7463           7539         115          2.1         474.5       1.3X
+-------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                       12450          12486          24          1.3         791.6       1.0X
+Parquet Vectorized (Pushdown)                              583            594           9         27.0          37.1      21.4X
+Native ORC Vectorized                                    10104          10178         133          1.6         642.4       1.2X
+Native ORC Vectorized (Pushdown)                         10312          10332          19          1.5         655.6       1.2X
 
 
 ================================================================================================
 Pushdown benchmark for decimal
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 1 decimal(9, 2) row (value = 7864320):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 4018           4049          32          3.9         255.4       1.0X
-Parquet Vectorized (Pushdown)                       163            178          19         96.6          10.4      24.7X
-Native ORC Vectorized                              4918           4969          39          3.2         312.7       0.8X
-Native ORC Vectorized (Pushdown)                    160            167          11         98.1          10.2      25.1X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+----------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                     3993           4013          12          3.9         253.9       1.0X
+Parquet Vectorized (Pushdown)                           139            143           7        113.3           8.8      28.8X
+Native ORC Vectorized                                  5123           5176          93          3.1         325.7       0.8X
+Native ORC Vectorized (Pushdown)                        191            201          25         82.4          12.1      20.9X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 10% decimal(9, 2) rows (value < 1572864):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 5800           5837          36          2.7         368.8       1.0X
-Parquet Vectorized (Pushdown)                      2764           2787          19          5.7         175.7       2.1X
-Native ORC Vectorized                              7034           7050          13          2.2         447.2       0.8X
-Native ORC Vectorized (Pushdown)                   3179           3191           9          4.9         202.1       1.8X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+-------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                        5942           5956          13          2.6         377.8       1.0X
+Parquet Vectorized (Pushdown)                             2842           2853          15          5.5         180.7       2.1X
+Native ORC Vectorized                                     7100           7124          22          2.2         451.4       0.8X
+Native ORC Vectorized (Pushdown)                          3141           3154          10          5.0         199.7       1.9X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 50% decimal(9, 2) rows (value < 7864320):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                12173          12215          33          1.3         773.9       1.0X
-Parquet Vectorized (Pushdown)                     11669          11723          31          1.3         741.9       1.0X
-Native ORC Vectorized                             13874          13893          19          1.1         882.1       0.9X
-Native ORC Vectorized (Pushdown)                  13241          13285          33          1.2         841.8       0.9X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+-------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                       12214          12254          30          1.3         776.5       1.0X
+Parquet Vectorized (Pushdown)                            11711          11745          34          1.3         744.6       1.0X
+Native ORC Vectorized                                    13522          13547          19          1.2         859.7       0.9X
+Native ORC Vectorized (Pushdown)                         12929          12946          18          1.2         822.0       0.9X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 90% decimal(9, 2) rows (value < 14155776):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                13290          13316          23          1.2         845.0       1.0X
-Parquet Vectorized (Pushdown)                     13335          13367          23          1.2         847.8       1.0X
-Native ORC Vectorized                             15485          15504          18          1.0         984.5       0.9X
-Native ORC Vectorized (Pushdown)                  15576          15605          18          1.0         990.3       0.9X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+--------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                        13714          13740          24          1.1         871.9       1.0X
+Parquet Vectorized (Pushdown)                             13738          13832         100          1.1         873.4       1.0X
+Native ORC Vectorized                                     15109          15152          37          1.0         960.6       0.9X
+Native ORC Vectorized (Pushdown)                          15140          15159          22          1.0         962.6       0.9X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 1 decimal(18, 2) row (value = 7864320):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 4154           4175          32          3.8         264.1       1.0X
-Parquet Vectorized (Pushdown)                       159            162           7         98.9          10.1      26.1X
-Native ORC Vectorized                              4902           4926          25          3.2         311.7       0.8X
-Native ORC Vectorized (Pushdown)                    153            162          23        102.6           9.7      27.1X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+-----------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                      4207           4232          28          3.7         267.5       1.0X
+Parquet Vectorized (Pushdown)                            140            143           5        112.7           8.9      30.1X
+Native ORC Vectorized                                   5146           5175          47          3.1         327.2       0.8X
+Native ORC Vectorized (Pushdown)                         185            195          21         85.1          11.8      22.8X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 10% decimal(18, 2) rows (value < 1572864):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 5090           5114          46          3.1         323.6       1.0X
-Parquet Vectorized (Pushdown)                      1509           1533          45         10.4          95.9       3.4X
-Native ORC Vectorized                              5943           5952          11          2.6         377.9       0.9X
-Native ORC Vectorized (Pushdown)                   1649           1686          61          9.5         104.8       3.1X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+--------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                         5170           5195          20          3.0         328.7       1.0X
+Parquet Vectorized (Pushdown)                              1524           1537          12         10.3          96.9       3.4X
+Native ORC Vectorized                                      6127           6140          13          2.6         389.5       0.8X
+Native ORC Vectorized (Pushdown)                           1669           1679           7          9.4         106.1       3.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 50% decimal(18, 2) rows (value < 7864320):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 8887           8938          79          1.8         565.0       1.0X
-Parquet Vectorized (Pushdown)                      6901           6917          13          2.3         438.8       1.3X
-Native ORC Vectorized                             10030          10047          15          1.6         637.7       0.9X
-Native ORC Vectorized (Pushdown)                   7672           7689          12          2.1         487.8       1.2X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+--------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                         9075           9104          43          1.7         577.0       1.0X
+Parquet Vectorized (Pushdown)                              7081           7107          27          2.2         450.2       1.3X
+Native ORC Vectorized                                     10127          10154          32          1.6         643.9       0.9X
+Native ORC Vectorized (Pushdown)                           7679           7707          26          2.0         488.2       1.2X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 90% decimal(18, 2) rows (value < 14155776):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                12618          12633          19          1.2         802.2       1.0X
-Parquet Vectorized (Pushdown)                     12296          12315          13          1.3         781.7       1.0X
-Native ORC Vectorized                             14260          14318          43          1.1         906.6       0.9X
-Native ORC Vectorized (Pushdown)                  13661          13678          12          1.2         868.5       0.9X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+---------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                         12918          12944          20          1.2         821.3       1.0X
+Parquet Vectorized (Pushdown)                              12553          12590          37          1.3         798.1       1.0X
+Native ORC Vectorized                                      14060          14076          16          1.1         893.9       0.9X
+Native ORC Vectorized (Pushdown)                           13590          13655          40          1.2         864.1       1.0X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 1 decimal(38, 2) row (value = 7864320):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 6470           6492          29          2.4         411.4       1.0X
-Parquet Vectorized (Pushdown)                       175            178           3         89.8          11.1      36.9X
-Native ORC Vectorized                              4928           4949          17          3.2         313.3       1.3X
-Native ORC Vectorized (Pushdown)                    153            166          23        103.0           9.7      42.4X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+-----------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                      6428           6450          20          2.4         408.7       1.0X
+Parquet Vectorized (Pushdown)                            154            160           6        101.9           9.8      41.6X
+Native ORC Vectorized                                   5146           5156           9          3.1         327.2       1.2X
+Native ORC Vectorized (Pushdown)                         187            195          19         84.2          11.9      34.4X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 10% decimal(38, 2) rows (value < 1572864):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 7674           7709          60          2.0         487.9       1.0X
-Parquet Vectorized (Pushdown)                      1984           1994          10          7.9         126.1       3.9X
-Native ORC Vectorized                              6245           6259          15          2.5         397.1       1.2X
-Native ORC Vectorized (Pushdown)                   1926           1935          11          8.2         122.5       4.0X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+--------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                         7654           7698          26          2.1         486.6       1.0X
+Parquet Vectorized (Pushdown)                              1986           2004          12          7.9         126.3       3.9X
+Native ORC Vectorized                                      6287           6300          14          2.5         399.7       1.2X
+Native ORC Vectorized (Pushdown)                           1835           1849          12          8.6         116.7       4.2X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 50% decimal(38, 2) rows (value < 7864320):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                12485          12497          10          1.3         793.7       1.0X
-Parquet Vectorized (Pushdown)                      9348           9376          26          1.7         594.3       1.3X
-Native ORC Vectorized                             11573          11583          12          1.4         735.8       1.1X
-Native ORC Vectorized (Pushdown)                   9211           9228          10          1.7         585.6       1.4X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+--------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                        12475          12543          55          1.3         793.2       1.0X
+Parquet Vectorized (Pushdown)                              9386           9406          26          1.7         596.7       1.3X
+Native ORC Vectorized                                     10935          10962          19          1.4         695.3       1.1X
+Native ORC Vectorized (Pushdown)                           8498           8560          46          1.9         540.3       1.5X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 90% decimal(38, 2) rows (value < 14155776):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                17198          17206           7          0.9        1093.4       1.0X
-Parquet Vectorized (Pushdown)                     16613          16629          12          0.9        1056.2       1.0X
-Native ORC Vectorized                             17081          17103          31          0.9        1086.0       1.0X
-Native ORC Vectorized (Pushdown)                  16659          16693          28          0.9        1059.2       1.0X
+---------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                         17270          17291          29          0.9        1098.0       1.0X
+Parquet Vectorized (Pushdown)                              16666          16708          43          0.9        1059.6       1.0X
+Native ORC Vectorized                                      15653          15723          69          1.0         995.2       1.1X
+Native ORC Vectorized (Pushdown)                           15229          15258          19          1.0         968.3       1.1X
 
 
 ================================================================================================
 Pushdown benchmark for InSet -> InFilters
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 InSet -> InFilters (values count: 5, distribution: 10):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 8986           9024          59          1.8         571.3       1.0X
-Parquet Vectorized (Pushdown)                       662            676          14         23.7          42.1      13.6X
-Native ORC Vectorized                              6568           6589          27          2.4         417.6       1.4X
-Native ORC Vectorized (Pushdown)                    462            475          18         34.0          29.4      19.4X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+--------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                              12028          12062          37          1.3         764.7       1.0X
+Parquet Vectorized (Pushdown)                                     607            614          12         25.9          38.6      19.8X
+Native ORC Vectorized                                            9373           9399          26          1.7         595.9       1.3X
+Native ORC Vectorized (Pushdown)                                  571            595          39         27.6          36.3      21.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 InSet -> InFilters (values count: 5, distribution: 50):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 8973           8986          21          1.8         570.5       1.0X
-Parquet Vectorized (Pushdown)                       663            666           5         23.7          42.1      13.5X
-Native ORC Vectorized                              6569           6574           6          2.4         417.6       1.4X
-Native ORC Vectorized (Pushdown)                    462            507          57         34.0          29.4      19.4X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+--------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                              12084          12207         222          1.3         768.3       1.0X
+Parquet Vectorized (Pushdown)                                     606            611           7         26.0          38.5      20.0X
+Native ORC Vectorized                                            9413           9452          31          1.7         598.5       1.3X
+Native ORC Vectorized (Pushdown)                                  565            588          36         27.8          35.9      21.4X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 InSet -> InFilters (values count: 5, distribution: 90):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 9003           9037          37          1.7         572.4       1.0X
-Parquet Vectorized (Pushdown)                       666            671           6         23.6          42.4      13.5X
-Native ORC Vectorized                              6571           6581           8          2.4         417.8       1.4X
-Native ORC Vectorized (Pushdown)                    463            473          18         34.0          29.4      19.5X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+--------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                              12085          12129          36          1.3         768.3       1.0X
+Parquet Vectorized (Pushdown)                                     606            612           5         25.9          38.6      19.9X
+Native ORC Vectorized                                            9440           9474          29          1.7         600.2       1.3X
+Native ORC Vectorized (Pushdown)                                  571            594          33         27.5          36.3      21.2X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 InSet -> InFilters (values count: 10, distribution: 10):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 9015           9022           8          1.7         573.2       1.0X
-Parquet Vectorized (Pushdown)                       687            698          13         22.9          43.7      13.1X
-Native ORC Vectorized                              6592           6611          13          2.4         419.1       1.4X
-Native ORC Vectorized (Pushdown)                    485            497          16         32.5          30.8      18.6X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+---------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                               12105          12120          16          1.3         769.6       1.0X
+Parquet Vectorized (Pushdown)                                      627            635           9         25.1          39.8      19.3X
+Native ORC Vectorized                                             9456           9568          99          1.7         601.2       1.3X
+Native ORC Vectorized (Pushdown)                                   590            615          30         26.6          37.5      20.5X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 InSet -> InFilters (values count: 10, distribution: 50):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 9001           9008          14          1.7         572.3       1.0X
-Parquet Vectorized (Pushdown)                       686            692           5         22.9          43.6      13.1X
-Native ORC Vectorized                              6599           6619          32          2.4         419.6       1.4X
-Native ORC Vectorized (Pushdown)                    473            483          17         33.2          30.1      19.0X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+---------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                               12041          12062          23          1.3         765.5       1.0X
+Parquet Vectorized (Pushdown)                                      630            638          10         25.0          40.1      19.1X
+Native ORC Vectorized                                             9371           9417          33          1.7         595.8       1.3X
+Native ORC Vectorized (Pushdown)                                   592            607          31         26.6          37.6      20.3X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 InSet -> InFilters (values count: 10, distribution: 90):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 8996           9002           8          1.7         571.9       1.0X
-Parquet Vectorized (Pushdown)                       690            697           6         22.8          43.9      13.0X
-Native ORC Vectorized                              6579           6587           8          2.4         418.3       1.4X
-Native ORC Vectorized (Pushdown)                    481            490          15         32.7          30.6      18.7X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+---------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                               12063          12262         164          1.3         767.0       1.0X
+Parquet Vectorized (Pushdown)                                      632            640           5         24.9          40.2      19.1X
+Native ORC Vectorized                                             9416           9475          49          1.7         598.6       1.3X
+Native ORC Vectorized (Pushdown)                                   580            602          32         27.1          36.9      20.8X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 InSet -> InFilters (values count: 50, distribution: 10):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 9292           9304           9          1.7         590.8       1.0X
-Parquet Vectorized (Pushdown)                      9558           9567           9          1.6         607.7       1.0X
-Native ORC Vectorized                              6898           6911          10          2.3         438.6       1.3X
-Native ORC Vectorized (Pushdown)                    625            641          15         25.2          39.8      14.9X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+---------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                               12450          12477          22          1.3         791.6       1.0X
+Parquet Vectorized (Pushdown)                                    12456          12510          43          1.3         791.9       1.0X
+Native ORC Vectorized                                             9846           9884          29          1.6         626.0       1.3X
+Native ORC Vectorized (Pushdown)                                   750            764          23         21.0          47.7      16.6X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 InSet -> InFilters (values count: 50, distribution: 50):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 9281           9298          12          1.7         590.1       1.0X
-Parquet Vectorized (Pushdown)                      9546           9561          17          1.6         606.9       1.0X
-Native ORC Vectorized                              6877           6897          18          2.3         437.2       1.3X
-Native ORC Vectorized (Pushdown)                    661            668          15         23.8          42.0      14.0X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+---------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                               12440          12477          44          1.3         790.9       1.0X
+Parquet Vectorized (Pushdown)                                    12473          12522          38          1.3         793.0       1.0X
+Native ORC Vectorized                                             9702           9769          40          1.6         616.8       1.3X
+Native ORC Vectorized (Pushdown)                                   760            781          30         20.7          48.3      16.4X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 InSet -> InFilters (values count: 50, distribution: 90):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 9322           9335          22          1.7         592.7       1.0X
-Parquet Vectorized (Pushdown)                      9551           9573          18          1.6         607.2       1.0X
-Native ORC Vectorized                              6902           6915          13          2.3         438.8       1.4X
-Native ORC Vectorized (Pushdown)                    659            680          25         23.9          41.9      14.1X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+---------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                               12384          12456          52          1.3         787.3       1.0X
+Parquet Vectorized (Pushdown)                                    12448          12480          24          1.3         791.4       1.0X
+Native ORC Vectorized                                             9680           9704          14          1.6         615.4       1.3X
+Native ORC Vectorized (Pushdown)                                   773            805          32         20.3          49.1      16.0X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 InSet -> InFilters (values count: 100, distribution: 10):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 9278           9294          18          1.7         589.9       1.0X
-Parquet Vectorized (Pushdown)                      9520           9560          27          1.7         605.3       1.0X
-Native ORC Vectorized                              6855           6870          16          2.3         435.9       1.4X
-Native ORC Vectorized (Pushdown)                    795            808          16         19.8          50.5      11.7X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+----------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                12410          12697         228          1.3         789.0       1.0X
+Parquet Vectorized (Pushdown)                                     12391          12418          23          1.3         787.8       1.0X
+Native ORC Vectorized                                              9716           9766          37          1.6         617.7       1.3X
+Native ORC Vectorized (Pushdown)                                    883            912          37         17.8          56.1      14.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 InSet -> InFilters (values count: 100, distribution: 50):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 9306           9311           4          1.7         591.6       1.0X
-Parquet Vectorized (Pushdown)                      9529           9551          16          1.7         605.8       1.0X
-Native ORC Vectorized                              6875           6882           7          2.3         437.1       1.4X
-Native ORC Vectorized (Pushdown)                    853            865          15         18.4          54.2      10.9X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+----------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                12334          12353          18          1.3         784.2       1.0X
+Parquet Vectorized (Pushdown)                                     12328          12375          33          1.3         783.8       1.0X
+Native ORC Vectorized                                              9554           9612          39          1.6         607.4       1.3X
+Native ORC Vectorized (Pushdown)                                    974            996          19         16.1          62.0      12.7X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 InSet -> InFilters (values count: 100, distribution: 90):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 9256           9271           9          1.7         588.5       1.0X
-Parquet Vectorized (Pushdown)                      9500           9520          13          1.7         604.0       1.0X
-Native ORC Vectorized                              6843           6857           9          2.3         435.1       1.4X
-Native ORC Vectorized (Pushdown)                    858            870          14         18.3          54.6      10.8X
+----------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                12355          12378          15          1.3         785.5       1.0X
+Parquet Vectorized (Pushdown)                                     12335          12362          42          1.3         784.2       1.0X
+Native ORC Vectorized                                              9644           9701          51          1.6         613.2       1.3X
+Native ORC Vectorized (Pushdown)                                    989           1016          25         15.9          62.9      12.5X
 
 
 ================================================================================================
 Pushdown benchmark for tinyint
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 1 tinyint row (value = CAST(63 AS tinyint)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 4303           4312           8          3.7         273.6       1.0X
-Parquet Vectorized (Pushdown)                       208            213           6         75.5          13.2      20.7X
-Native ORC Vectorized                              2950           2958           9          5.3         187.6       1.5X
-Native ORC Vectorized (Pushdown)                    207            212          10         76.1          13.1      20.8X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+----------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                           5244           5282          30          3.0         333.4       1.0X
+Parquet Vectorized (Pushdown)                                 190            193           4         82.9          12.1      27.6X
+Native ORC Vectorized                                        4019           4040          17          3.9         255.6       1.3X
+Native ORC Vectorized (Pushdown)                              248            260          23         63.5          15.8      21.2X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 10% tinyint rows (value < CAST(12 AS tinyint)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 5125           5162          57          3.1         325.9       1.0X
-Parquet Vectorized (Pushdown)                      1402           1419          16         11.2          89.2       3.7X
-Native ORC Vectorized                              3840           3849           8          4.1         244.1       1.3X
-Native ORC Vectorized (Pushdown)                   1302           1306           3         12.1          82.8       3.9X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+-------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                              6104           6125          16          2.6         388.1       1.0X
+Parquet Vectorized (Pushdown)                                   1416           1432          14         11.1          90.0       4.3X
+Native ORC Vectorized                                           4862           4872          11          3.2         309.1       1.3X
+Native ORC Vectorized (Pushdown)                                1329           1347          12         11.8          84.5       4.6X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 50% tinyint rows (value < CAST(63 AS tinyint)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 8562           8577          12          1.8         544.4       1.0X
-Parquet Vectorized (Pushdown)                      6539           6564          22          2.4         415.8       1.3X
-Native ORC Vectorized                              7300           7320          13          2.2         464.1       1.2X
-Native ORC Vectorized (Pushdown)                   5944           5954          15          2.6         377.9       1.4X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+-------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                              9645           9685          35          1.6         613.2       1.0X
+Parquet Vectorized (Pushdown)                                   6720           6741          20          2.3         427.2       1.4X
+Native ORC Vectorized                                           8318           8339          17          1.9         528.8       1.2X
+Native ORC Vectorized (Pushdown)                                6005           6116         131          2.6         381.8       1.6X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 90% tinyint rows (value < CAST(114 AS tinyint)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                11989          12005          13          1.3         762.2       1.0X
-Parquet Vectorized (Pushdown)                     11637          11686          54          1.4         739.8       1.0X
-Native ORC Vectorized                             10795          10816          18          1.5         686.3       1.1X
-Native ORC Vectorized (Pushdown)                  10592          10599           5          1.5         673.4       1.1X
+--------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                              13318          13363          36          1.2         846.8       1.0X
+Parquet Vectorized (Pushdown)                                   12852          12919          54          1.2         817.1       1.0X
+Native ORC Vectorized                                           11759          11792          21          1.3         747.6       1.1X
+Native ORC Vectorized (Pushdown)                                11404          11457          35          1.4         725.1       1.2X
 
 
 ================================================================================================
 Pushdown benchmark for Timestamp
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Select 1 timestamp stored as INT96 row (value = CAST(7864320 AS timestamp)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 4700           4716          19          3.3         298.8       1.0X
-Parquet Vectorized (Pushdown)                      4745           4758          11          3.3         301.7       1.0X
-Native ORC Vectorized                              2848           2865          22          5.5         181.1       1.7X
-Native ORC Vectorized (Pushdown)                    129            135          12        122.2           8.2      36.5X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Select 10% timestamp stored as INT96 rows (value < CAST(1572864 AS timestamp)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 5575           5590           9          2.8         354.5       1.0X
-Parquet Vectorized (Pushdown)                      5610           5623          10          2.8         356.7       1.0X
-Native ORC Vectorized                              3706           3723          11          4.2         235.6       1.5X
-Native ORC Vectorized (Pushdown)                   1286           1287           2         12.2          81.7       4.3X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Select 50% timestamp stored as INT96 rows (value < CAST(7864320 AS timestamp)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 9872           9887          13          1.6         627.7       1.0X
-Parquet Vectorized (Pushdown)                      9932           9942           8          1.6         631.4       1.0X
-Native ORC Vectorized                              7238           7273          25          2.2         460.2       1.4X
-Native ORC Vectorized (Pushdown)                   5933           5937           6          2.7         377.2       1.7X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Select 90% timestamp stored as INT96 rows (value < CAST(14155776 AS timestamp)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                12660          12677          12          1.2         804.9       1.0X
-Parquet Vectorized (Pushdown)                     12652          12704          31          1.2         804.4       1.0X
-Native ORC Vectorized                             10842          10870          19          1.5         689.3       1.2X
-Native ORC Vectorized (Pushdown)                  10630          10639           6          1.5         675.9       1.2X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Select 1 timestamp stored as TIMESTAMP_MICROS row (value = CAST(7864320 AS timestamp)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 4057           4065           7          3.9         257.9       1.0X
-Parquet Vectorized (Pushdown)                       155            159           7        101.2           9.9      26.1X
-Native ORC Vectorized                              2828           2840           8          5.6         179.8       1.4X
-Native ORC Vectorized (Pushdown)                    126            130           9        124.8           8.0      32.2X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Select 10% timestamp stored as TIMESTAMP_MICROS rows (value < CAST(1572864 AS timestamp)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 4926           4938           9          3.2         313.2       1.0X
-Parquet Vectorized (Pushdown)                      1415           1422           6         11.1          90.0       3.5X
-Native ORC Vectorized                              3705           3714           7          4.2         235.6       1.3X
-Native ORC Vectorized (Pushdown)                   1279           1285           9         12.3          81.3       3.9X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Select 50% timestamp stored as TIMESTAMP_MICROS rows (value < CAST(7864320 AS timestamp)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 8409           8413           4          1.9         534.6       1.0X
-Parquet Vectorized (Pushdown)                      6489           6497           6          2.4         412.5       1.3X
-Native ORC Vectorized                              7248           7255          10          2.2         460.8       1.2X
-Native ORC Vectorized (Pushdown)                   5922           5932           7          2.7         376.5       1.4X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Select 90% timestamp stored as TIMESTAMP_MICROS rows (value < CAST(14155776 AS timestamp)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                11821          11830           8          1.3         751.5       1.0X
-Parquet Vectorized (Pushdown)                     11478          11490          11          1.4         729.8       1.0X
-Native ORC Vectorized                             10851          10856           4          1.4         689.9       1.1X
-Native ORC Vectorized (Pushdown)                  10620          10628          10          1.5         675.2       1.1X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Select 1 timestamp stored as TIMESTAMP_MILLIS row (value = CAST(7864320 AS timestamp)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 4307           4341          75          3.7         273.8       1.0X
-Parquet Vectorized (Pushdown)                       156            161           6        101.1           9.9      27.7X
-Native ORC Vectorized                              2825           2836           8          5.6         179.6       1.5X
-Native ORC Vectorized (Pushdown)                    125            128           9        125.4           8.0      34.3X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Select 10% timestamp stored as TIMESTAMP_MILLIS rows (value < CAST(1572864 AS timestamp)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 5173           5178           3          3.0         328.9       1.0X
-Parquet Vectorized (Pushdown)                      1444           1451           9         10.9          91.8       3.6X
-Native ORC Vectorized                              3707           3715           5          4.2         235.7       1.4X
-Native ORC Vectorized (Pushdown)                   1275           1276           2         12.3          81.0       4.1X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Select 50% timestamp stored as TIMESTAMP_MILLIS rows (value < CAST(7864320 AS timestamp)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 8645           8655           9          1.8         549.6       1.0X
-Parquet Vectorized (Pushdown)                      6588           6603          11          2.4         418.9       1.3X
-Native ORC Vectorized                              7233           7246          13          2.2         459.9       1.2X
-Native ORC Vectorized (Pushdown)                   5904           5912           8          2.7         375.3       1.5X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Select 90% timestamp stored as TIMESTAMP_MILLIS rows (value < CAST(14155776 AS timestamp)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                12057          12069          12          1.3         766.5       1.0X
-Parquet Vectorized (Pushdown)                     11694          11699           5          1.3         743.5       1.0X
-Native ORC Vectorized                             10817          10843          28          1.5         687.7       1.1X
-Native ORC Vectorized (Pushdown)                  10615          10634          11          1.5         674.9       1.1X
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+Select 1 timestamp stored as INT96 row (value = timestamp_seconds(7864320)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-----------------------------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                                    5563           5590          26          2.8         353.7       1.0X
+Parquet Vectorized (Pushdown)                                                         5584           5606          24          2.8         355.0       1.0X
+Native ORC Vectorized                                                                 3268           3704         384          4.8         207.8       1.7X
+Native ORC Vectorized (Pushdown)                                                       152            159          16        103.4           9.7      36.6X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+Select 10% timestamp stored as INT96 rows (value < timestamp_seconds(1572864)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+--------------------------------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                                       6532           6549          21          2.4         415.3       1.0X
+Parquet Vectorized (Pushdown)                                                            6545           6576          19          2.4         416.1       1.0X
+Native ORC Vectorized                                                                    4192           4200           7          3.8         266.5       1.6X
+Native ORC Vectorized (Pushdown)                                                         1368           1394          21         11.5          87.0       4.8X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+Select 50% timestamp stored as INT96 rows (value < timestamp_seconds(7864320)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+--------------------------------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                                      10385          10421          25          1.5         660.2       1.0X
+Parquet Vectorized (Pushdown)                                                           10348          10386          23          1.5         657.9       1.0X
+Native ORC Vectorized                                                                    7724           7768          29          2.0         491.1       1.3X
+Native ORC Vectorized (Pushdown)                                                         6222           6246          17          2.5         395.6       1.7X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+Select 90% timestamp stored as INT96 rows (value < timestamp_seconds(14155776)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+---------------------------------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                                       14117          14146          31          1.1         897.6       1.0X
+Parquet Vectorized (Pushdown)                                                            14106          14128          25          1.1         896.8       1.0X
+Native ORC Vectorized                                                                    11373          11425          59          1.4         723.1       1.2X
+Native ORC Vectorized (Pushdown)                                                         11084          11122          25          1.4         704.7       1.3X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+Select 1 timestamp stored as TIMESTAMP_MICROS row (value = timestamp_seconds(7864320)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                                               4097           4449         441          3.8         260.5       1.0X
+Parquet Vectorized (Pushdown)                                                                     139            143           6        113.4           8.8      29.5X
+Native ORC Vectorized                                                                            3272           3362         179          4.8         208.0       1.3X
+Native ORC Vectorized (Pushdown)                                                                  152            160          17        103.3           9.7      26.9X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+Select 10% timestamp stored as TIMESTAMP_MICROS rows (value < timestamp_seconds(1572864)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                                                  5046           5085          38          3.1         320.8       1.0X
+Parquet Vectorized (Pushdown)                                                                       1469           1487          12         10.7          93.4       3.4X
+Native ORC Vectorized                                                                               4175           4201          25          3.8         265.5       1.2X
+Native ORC Vectorized (Pushdown)                                                                    1359           1382          14         11.6          86.4       3.7X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+Select 50% timestamp stored as TIMESTAMP_MICROS rows (value < timestamp_seconds(7864320)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                                                  8862           8876          11          1.8         563.4       1.0X
+Parquet Vectorized (Pushdown)                                                                       6887           6915          31          2.3         437.9       1.3X
+Native ORC Vectorized                                                                               7759           7781          18          2.0         493.3       1.1X
+Native ORC Vectorized (Pushdown)                                                                    6213           6239          22          2.5         395.0       1.4X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+Select 90% timestamp stored as TIMESTAMP_MICROS rows (value < timestamp_seconds(14155776)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                                                  12582          12603          40          1.3         799.9       1.0X
+Parquet Vectorized (Pushdown)                                                                       12203          12235          24          1.3         775.8       1.0X
+Native ORC Vectorized                                                                               11364          11386          24          1.4         722.5       1.1X
+Native ORC Vectorized (Pushdown)                                                                    11116          11161          50          1.4         706.7       1.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+Select 1 timestamp stored as TIMESTAMP_MILLIS row (value = timestamp_seconds(7864320)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                                               4287           4306          13          3.7         272.6       1.0X
+Parquet Vectorized (Pushdown)                                                                     138            142           7        113.9           8.8      31.0X
+Native ORC Vectorized                                                                            3266           3280          17          4.8         207.7       1.3X
+Native ORC Vectorized (Pushdown)                                                                  151            158          17        104.0           9.6      28.3X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+Select 10% timestamp stored as TIMESTAMP_MILLIS rows (value < timestamp_seconds(1572864)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                                                  5247           5279          24          3.0         333.6       1.0X
+Parquet Vectorized (Pushdown)                                                                       1497           1509          12         10.5          95.2       3.5X
+Native ORC Vectorized                                                                               4185           4197           9          3.8         266.1       1.3X
+Native ORC Vectorized (Pushdown)                                                                    1364           1388          14         11.5          86.8       3.8X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+Select 50% timestamp stored as TIMESTAMP_MILLIS rows (value < timestamp_seconds(7864320)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                                                  9055           9099          47          1.7         575.7       1.0X
+Parquet Vectorized (Pushdown)                                                                       6993           7023          27          2.2         444.6       1.3X
+Native ORC Vectorized                                                                               7740           7787          34          2.0         492.1       1.2X
+Native ORC Vectorized (Pushdown)                                                                    6222           6249          20          2.5         395.6       1.5X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+Select 90% timestamp stored as TIMESTAMP_MILLIS rows (value < timestamp_seconds(14155776)):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+Parquet Vectorized                                                                                  12755          12794          44          1.2         810.9       1.0X
+Parquet Vectorized (Pushdown)                                                                       12408          12441          34          1.3         788.9       1.0X
+Native ORC Vectorized                                                                               11357          11410          52          1.4         722.1       1.1X
+Native ORC Vectorized (Pushdown)                                                                    11089          11144          43          1.4         705.0       1.2X
 
 
 ================================================================================================
 Pushdown benchmark with many filters
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 1 row with 1 filters:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                  463            468           6          0.0   462711905.0       1.0X
-Parquet Vectorized (Pushdown)                       468            470           3          0.0   468018831.0       1.0X
-Native ORC Vectorized                               447            450           5          0.0   446698170.0       1.0X
-Native ORC Vectorized (Pushdown)                    449            458          11          0.0   448997785.0       1.0X
+Parquet Vectorized                                  165            171           5          0.0   164509375.0       1.0X
+Parquet Vectorized (Pushdown)                       167            170           4          0.0   167250773.0       1.0X
+Native ORC Vectorized                               148            152           5          0.0   148320765.0       1.1X
+Native ORC Vectorized (Pushdown)                    154            158           7          0.0   153612969.0       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 1 row with 250 filters:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 1538           1549           9          0.0  1538463215.0       1.0X
-Parquet Vectorized (Pushdown)                      1659           1668           6          0.0  1659315980.0       0.9X
-Native ORC Vectorized                              1513           1517           5          0.0  1512577059.0       1.0X
-Native ORC Vectorized (Pushdown)                   1517           1538          12          0.0  1516938695.0       1.0X
+Parquet Vectorized                                 1973           2021          36          0.0  1972618013.0       1.0X
+Parquet Vectorized (Pushdown)                      2045           2069          21          0.0  2045423860.0       1.0X
+Native ORC Vectorized                              1959           1992          20          0.0  1959045908.0       1.0X
+Native ORC Vectorized (Pushdown)                   1967           2002          26          0.0  1966983577.0       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Select 1 row with 500 filters:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Parquet Vectorized                                 4497           4525          29          0.0  4497410600.0       1.0X
-Parquet Vectorized (Pushdown)                      4945           4955           8          0.0  4945493883.0       0.9X
-Native ORC Vectorized                              4466           4485          23          0.0  4466103057.0       1.0X
-Native ORC Vectorized (Pushdown)                   4477           4496          18          0.0  4476752574.0       1.0X
+Parquet Vectorized                                12096          12477         293          0.0 12096441094.0       1.0X
+Parquet Vectorized (Pushdown)                     12241          12311          54          0.0 12241257148.0       1.0X
+Native ORC Vectorized                             11948          12002          51          0.0 11948392776.0       1.0X
+Native ORC Vectorized (Pushdown)                  11916          12008          61          0.0 11915517617.0       1.0X
 
 
diff --git a/sql/core/benchmarks/GenerateExecBenchmark-jdk11-results.txt b/sql/core/benchmarks/GenerateExecBenchmark-jdk11-results.txt
new file mode 100644
index 0000000000000..52ba08ba5e0f1
--- /dev/null
+++ b/sql/core/benchmarks/GenerateExecBenchmark-jdk11-results.txt
@@ -0,0 +1,12 @@
+================================================================================================
+GenerateExec benchmark
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+GenerateExec Benchmark:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+GenerateExec Benchmark wholestage off            130659         131394        1040          0.8        1306.6       1.0X
+GenerateExec Benchmark wholestage on              64031          64151          90          1.6         640.3       2.0X
+
+
diff --git a/sql/core/benchmarks/GenerateExecBenchmark-results.txt b/sql/core/benchmarks/GenerateExecBenchmark-results.txt
new file mode 100644
index 0000000000000..4dcba78820eaa
--- /dev/null
+++ b/sql/core/benchmarks/GenerateExecBenchmark-results.txt
@@ -0,0 +1,12 @@
+================================================================================================
+GenerateExec benchmark
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
+GenerateExec Benchmark:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+GenerateExec Benchmark wholestage off            102348         105601        1650          1.0        1023.5       1.0X
+GenerateExec Benchmark wholestage on              56423          58047        1028          1.8         564.2       1.8X
+
+
diff --git a/sql/core/benchmarks/HashedRelationMetricsBenchmark-jdk11-results.txt b/sql/core/benchmarks/HashedRelationMetricsBenchmark-jdk11-results.txt
index e01561364b989..fd40ab7799278 100644
--- a/sql/core/benchmarks/HashedRelationMetricsBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/HashedRelationMetricsBenchmark-jdk11-results.txt
@@ -2,10 +2,10 @@
 LongToUnsafeRowMap metrics
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 LongToUnsafeRowMap metrics:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-LongToUnsafeRowMap                                  568            575           5          0.9        1136.4       1.0X
+LongToUnsafeRowMap                                  709            759          51          0.7        1417.4       1.0X
 
 
diff --git a/sql/core/benchmarks/HashedRelationMetricsBenchmark-results.txt b/sql/core/benchmarks/HashedRelationMetricsBenchmark-results.txt
index 7483f517a62e8..5ba6600aad885 100644
--- a/sql/core/benchmarks/HashedRelationMetricsBenchmark-results.txt
+++ b/sql/core/benchmarks/HashedRelationMetricsBenchmark-results.txt
@@ -2,10 +2,10 @@
 LongToUnsafeRowMap metrics
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 LongToUnsafeRowMap metrics:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-LongToUnsafeRowMap                                  513            629         106          1.0        1026.6       1.0X
+LongToUnsafeRowMap                                  613            622          13          0.8        1225.3       1.0X
 
 
diff --git a/sql/core/benchmarks/InExpressionBenchmark-jdk11-results.txt b/sql/core/benchmarks/InExpressionBenchmark-jdk11-results.txt
index 04474566ef43d..b462065fde6b6 100644
--- a/sql/core/benchmarks/InExpressionBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/InExpressionBenchmark-jdk11-results.txt
@@ -2,739 +2,739 @@
 In Expression Benchmark
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 5 bytes:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       163            213          53         61.4          16.3       1.0X
-InSet expression                                    105            130          16         95.5          10.5       1.6X
+In expression                                        56             78          27        179.9           5.6       1.0X
+InSet expression                                     47             63          21        210.5           4.7       1.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 10 bytes:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       112            126           9         89.4          11.2       1.0X
-InSet expression                                     78             92          11        128.4           7.8       1.4X
+In expression                                        68             73           6        146.7           6.8       1.0X
+InSet expression                                     51             66          19        196.8           5.1       1.3X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 25 bytes:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       170            181           8         58.9          17.0       1.0X
-InSet expression                                    109            118           9         91.8          10.9       1.6X
+In expression                                       113            117           4         88.3          11.3       1.0X
+InSet expression                                     69             73           4        145.7           6.9       1.7X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 50 bytes:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       279            283           3         35.8          27.9       1.0X
-InSet expression                                    140            148           8         71.6          14.0       2.0X
+In expression                                       189            194           6         52.9          18.9       1.0X
+InSet expression                                    101            106           4         99.4          10.1       1.9X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 100 bytes:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       461            486          28         21.7          46.1       1.0X
-InSet expression                                    184            187           3         54.5          18.4       2.5X
+In expression                                       343            348           4         29.1          34.3       1.0X
+InSet expression                                    171            179           9         58.5          17.1       2.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 200 bytes:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       735            754          16         13.6          73.5       1.0X
-InSet expression                                    317            323           5         31.6          31.7       2.3X
+In expression                                       556            561           8         18.0          55.6       1.0X
+InSet expression                                    302            311           7         33.1          30.2       1.8X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 5 shorts:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        63             68           5        157.8           6.3       1.0X
-InSet expression                                     56             61           4        177.7           5.6       1.1X
+In expression                                        43             52          16        234.0           4.3       1.0X
+InSet expression                                     36             46          14        274.3           3.6       1.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 10 shorts:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        84             87           2        118.8           8.4       1.0X
-InSet expression                                     58             62           4        171.6           5.8       1.4X
+In expression                                        49             56          11        203.7           4.9       1.0X
+InSet expression                                     35             46          15        281.8           3.5       1.4X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 25 shorts:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       139            146          10         72.0          13.9       1.0X
-InSet expression                                     58             63           7        173.5           5.8       2.4X
+In expression                                        81             90          12        123.3           8.1       1.0X
+InSet expression                                     36             44          10        275.0           3.6       2.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 50 shorts:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       231            236           7         43.3          23.1       1.0X
-InSet expression                                     59             64           8        170.5           5.9       3.9X
+In expression                                       130            138          16         76.9          13.0       1.0X
+InSet expression                                     37             44          13        269.9           3.7       3.5X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 100 shorts:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       411            414           3         24.3          41.1       1.0X
-InSet expression                                     64             72           8        155.4           6.4       6.4X
+In expression                                       228            238          13         43.8          22.8       1.0X
+InSet expression                                     41             50          12        243.9           4.1       5.6X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 200 shorts:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       775            779           6         12.9          77.5       1.0X
-InSet expression                                     70             74           4        142.7           7.0      11.1X
+In expression                                       428            436           7         23.3          42.8       1.0X
+InSet expression                                     48             57          11        206.2           4.8       8.8X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 300 shorts:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1129           1133           3          8.9         112.9       1.0X
-InSet expression                                     86             91           6        116.9           8.6      13.2X
+In expression                                       630            634           4         15.9          63.0       1.0X
+InSet expression                                     57             60           5        176.4           5.7      11.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 400 shorts:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1574           1577           3          6.4         157.4       1.0X
-InSet expression                                     89             92           4        112.9           8.9      17.8X
+In expression                                      1129           1136           5          8.9         112.9       1.0X
+InSet expression                                     65             68           3        153.1           6.5      17.3X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 500 shorts:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1865           1914         100          5.4         186.5       1.0X
-InSet expression                                    275            279           4         36.3          27.5       6.8X
+In expression                                      1598           1641          72          6.3         159.8       1.0X
+InSet expression                                    314            324          11         31.8          31.4       5.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 5 shorts (non-compact):                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        63             66           4        158.5           6.3       1.0X
-InSet expression                                     62             68          14        160.7           6.2       1.0X
+In expression                                        36             43          15        276.7           3.6       1.0X
+InSet expression                                     38             47          14        261.5           3.8       0.9X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 10 shorts (non-compact):                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        81             87          10        124.0           8.1       1.0X
-InSet expression                                     70             82          15        142.4           7.0       1.1X
+In expression                                        48             55          13        209.1           4.8       1.0X
+InSet expression                                     44             51          12        227.8           4.4       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 25 shorts (non-compact):                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       142            145           5         70.4          14.2       1.0X
-InSet expression                                     78             80           3        128.4           7.8       1.8X
+In expression                                        80             89          13        124.4           8.0       1.0X
+InSet expression                                     50             65          23        200.6           5.0       1.6X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 50 shorts (non-compact):                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       229            235          14         43.8          22.9       1.0X
-InSet expression                                     91             94           4        109.5           9.1       2.5X
+In expression                                       132            140          13         75.5          13.2       1.0X
+InSet expression                                     58             65          11        173.6           5.8       2.3X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 100 shorts (non-compact):                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       409            411           2         24.4          40.9       1.0X
-InSet expression                                    108            112           5         92.9          10.8       3.8X
+In expression                                       227            241          23         44.0          22.7       1.0X
+InSet expression                                     65             78          24        153.0           6.5       3.5X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 200 shorts (non-compact):                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       772            779           8         12.9          77.2       1.0X
-InSet expression                                    126            131           4         79.6          12.6       6.1X
+In expression                                       427            430           3         23.4          42.7       1.0X
+InSet expression                                     77             84          11        130.0           7.7       5.6X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 300 shorts (non-compact):                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1138           1144           6          8.8         113.8       1.0X
-InSet expression                                    136            140           6         73.5          13.6       8.4X
+In expression                                       633            636           4         15.8          63.3       1.0X
+InSet expression                                     88             96          12        113.1           8.8       7.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 400 shorts (non-compact):                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1500           1504           3          6.7         150.0       1.0X
-InSet expression                                    148            154           6         67.7          14.8      10.2X
+In expression                                      1260           1264           5          7.9         126.0       1.0X
+InSet expression                                     98            100           2        102.2           9.8      12.9X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 500 shorts (non-compact):                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1932           1969          72          5.2         193.2       1.0X
-InSet expression                                    275            278           3         36.3          27.5       7.0X
+In expression                                      1646           1648           2          6.1         164.6       1.0X
+InSet expression                                    316            322           7         31.7          31.6       5.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 5 ints:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        57             59           2        174.1           5.7       1.0X
-InSet expression                                     53             57           5        187.1           5.3       1.1X
+In expression                                        33             39          11        302.2           3.3       1.0X
+InSet expression                                     29             35          10        343.6           2.9       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 10 ints:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        76             79           7        131.9           7.6       1.0X
-InSet expression                                     54             57           8        186.1           5.4       1.4X
+In expression                                        43             48           9        231.6           4.3       1.0X
+InSet expression                                     29             36          10        339.2           2.9       1.5X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 25 ints:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       128            132           3         77.9          12.8       1.0X
-InSet expression                                     55             58           4        183.2           5.5       2.4X
+In expression                                        75             85          13        132.7           7.5       1.0X
+InSet expression                                     32             39          12        314.4           3.2       2.4X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 50 ints:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       227            237          11         44.1          22.7       1.0X
-InSet expression                                     56             58           2        178.0           5.6       4.0X
+In expression                                       128            134          12         78.0          12.8       1.0X
+InSet expression                                     34             44          15        295.4           3.4       3.8X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 100 ints:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       407            413           9         24.5          40.7       1.0X
-InSet expression                                     59             64           7        169.0           5.9       6.9X
+In expression                                       225            231          14         44.5          22.5       1.0X
+InSet expression                                     37             45          15        269.1           3.7       6.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 200 ints:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       772            775           4         13.0          77.2       1.0X
-InSet expression                                     67             70           5        148.7           6.7      11.5X
+In expression                                       420            422           2         23.8          42.0       1.0X
+InSet expression                                     44             51          10        226.7           4.4       9.5X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 300 ints:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1128           1132           3          8.9         112.8       1.0X
-InSet expression                                     75             77           2        133.6           7.5      15.1X
+In expression                                       618            619           1         16.2          61.8       1.0X
+InSet expression                                     53             61          12        188.0           5.3      11.6X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 400 ints:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1502           1507           6          6.7         150.2       1.0X
-InSet expression                                     82             84           2        121.6           8.2      18.3X
+In expression                                      1154           1158           4          8.7         115.4       1.0X
+InSet expression                                     58             60           2        172.1           5.8      19.9X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 500 ints:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1860           1897          75          5.4         186.0       1.0X
-InSet expression                                    246            249           2         40.7          24.6       7.6X
+In expression                                      1598           1635          73          6.3         159.8       1.0X
+InSet expression                                    307            314           6         32.6          30.7       5.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 5 ints (non-compact):                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        46             48           3        219.3           4.6       1.0X
-InSet expression                                     40             41           2        251.6           4.0       1.1X
+In expression                                        24             30           9        410.7           2.4       1.0X
+InSet expression                                     19             24           9        526.3           1.9       1.3X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 10 ints (non-compact):                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        65             67           2        154.5           6.5       1.0X
-InSet expression                                     45             47           2        221.2           4.5       1.4X
+In expression                                        34             39           9        294.8           3.4       1.0X
+InSet expression                                     21             27           9        470.2           2.1       1.6X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 25 ints (non-compact):                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       121            124           3         82.5          12.1       1.0X
-InSet expression                                     46             49           4        217.2           4.6       2.6X
+In expression                                        69             77          22        145.9           6.9       1.0X
+InSet expression                                     26             34          12        384.8           2.6       2.6X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 50 ints (non-compact):                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       214            216           2         46.7          21.4       1.0X
-InSet expression                                     57             59           3        175.3           5.7       3.8X
+In expression                                       205            209           2         48.7          20.5       1.0X
+InSet expression                                     30             37          10        330.7           3.0       6.8X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 100 ints (non-compact):                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       400            406           6         25.0          40.0       1.0X
-InSet expression                                     58             61           3        173.9           5.8       7.0X
+In expression                                       401            411          18         24.9          40.1       1.0X
+InSet expression                                     37             44          10        271.5           3.7      10.9X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 200 ints (non-compact):                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       762            765           2         13.1          76.2       1.0X
-InSet expression                                     69             71           3        145.5           6.9      11.1X
+In expression                                       792            794           2         12.6          79.2       1.0X
+InSet expression                                     43             51          11        230.5           4.3      18.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 300 ints (non-compact):                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1116           1122           4          9.0         111.6       1.0X
-InSet expression                                     84             87           2        118.5           8.4      13.2X
+In expression                                      1174           1179           3          8.5         117.4       1.0X
+InSet expression                                     51             58          11        195.1           5.1      22.9X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 400 ints (non-compact):                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1491           1498           5          6.7         149.1       1.0X
-InSet expression                                     88             91           2        113.4           8.8      16.9X
+In expression                                      1572           1579           6          6.4         157.2       1.0X
+InSet expression                                     58             61           6        171.4           5.8      27.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 500 ints (non-compact):                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1858           1909         109          5.4         185.8       1.0X
-InSet expression                                    247            249           2         40.5          24.7       7.5X
+In expression                                      1965           2000          70          5.1         196.5       1.0X
+InSet expression                                    312            317           4         32.0          31.2       6.3X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 5 longs:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        53             54           2        190.2           5.3       1.0X
-InSet expression                                    156            162          13         64.3          15.6       0.3X
+In expression                                        31             37           8        318.6           3.1       1.0X
+InSet expression                                    165            175          11         60.7          16.5       0.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 10 longs:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        67             69           3        149.9           6.7       1.0X
-InSet expression                                    171            174           2         58.5          17.1       0.4X
+In expression                                        42             48          11        240.6           4.2       1.0X
+InSet expression                                    183            192          12         54.7          18.3       0.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 25 longs:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       120            122           4         83.0          12.0       1.0X
-InSet expression                                    178            181           1         56.2          17.8       0.7X
+In expression                                        73             79          11        136.3           7.3       1.0X
+InSet expression                                    190            197           8         52.5          19.0       0.4X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 50 longs:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       212            213           2         47.2          21.2       1.0X
-InSet expression                                    222            224           3         45.0          22.2       1.0X
+In expression                                       124            132          14         80.5          12.4       1.0X
+InSet expression                                    237            241           3         42.2          23.7       0.5X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 100 longs:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       393            398           4         25.5          39.3       1.0X
-InSet expression                                    193            197           2         51.8          19.3       2.0X
+In expression                                       223            225           2         44.8          22.3       1.0X
+InSet expression                                    209            216           9         47.9          20.9       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 200 longs:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       756            763           7         13.2          75.6       1.0X
-InSet expression                                    186            188           2         53.8          18.6       4.1X
+In expression                                       419            433          25         23.9          41.9       1.0X
+InSet expression                                    197            206          10         50.8          19.7       2.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 5 floats:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        92             94           4        109.1           9.2       1.0X
-InSet expression                                    205            208           2         48.8          20.5       0.4X
+In expression                                        50             56          10        200.0           5.0       1.0X
+InSet expression                                    200            208           7         49.9          20.0       0.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 10 floats:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       140            141           2         71.6          14.0       1.0X
-InSet expression                                    232            235           2         43.0          23.2       0.6X
+In expression                                        70             76          10        142.5           7.0       1.0X
+InSet expression                                    228            231           3         43.8          22.8       0.3X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 25 floats:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       247            250           5         40.5          24.7       1.0X
-InSet expression                                    240            242           3         41.6          24.0       1.0X
+In expression                                       135            145          14         74.1          13.5       1.0X
+InSet expression                                    229            233           2         43.7          22.9       0.6X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 50 floats:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       426            429           5         23.5          42.6       1.0X
-InSet expression                                    290            293           4         34.5          29.0       1.5X
+In expression                                       239            253          23         41.8          23.9       1.0X
+InSet expression                                    288            291           3         34.7          28.8       0.8X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 100 floats:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       785            787           2         12.7          78.5       1.0X
-InSet expression                                    250            252           2         40.0          25.0       3.1X
+In expression                                       437            441           3         22.9          43.7       1.0X
+InSet expression                                    248            257          10         40.3          24.8       1.8X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 200 floats:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      3101           3190         192          3.2         310.1       1.0X
-InSet expression                                    251            252           1         39.8          25.1      12.3X
+In expression                                      3017           3058          77          3.3         301.7       1.0X
+InSet expression                                    245            252           6         40.8          24.5      12.3X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 5 doubles:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        89             92           4        112.1           8.9       1.0X
-InSet expression                                    171            175           2         58.4          17.1       0.5X
+In expression                                        50             56           9        199.1           5.0       1.0X
+InSet expression                                    203            214           9         49.2          20.3       0.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 10 doubles:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       135            136           2         73.9          13.5       1.0X
-InSet expression                                    191            194           1         52.4          19.1       0.7X
+In expression                                        70             79          14        143.6           7.0       1.0X
+InSet expression                                    229            237          11         43.7          22.9       0.3X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 25 doubles:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       243            245           2         41.2          24.3       1.0X
-InSet expression                                    199            201           1         50.4          19.9       1.2X
+In expression                                       136            144          14         73.8          13.6       1.0X
+InSet expression                                    234            241           8         42.8          23.4       0.6X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 50 doubles:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       420            423           2         23.8          42.0       1.0X
-InSet expression                                    242            244           2         41.3          24.2       1.7X
+In expression                                       234            251          23         42.8          23.4       1.0X
+InSet expression                                    289            298          11         34.6          28.9       0.8X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 100 doubles:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       774            776           2         12.9          77.4       1.0X
-InSet expression                                    210            216           5         47.7          21.0       3.7X
+In expression                                       430            444          26         23.3          43.0       1.0X
+InSet expression                                    255            263          10         39.2          25.5       1.7X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 200 doubles:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      3627           3744         248          2.8         362.7       1.0X
-InSet expression                                    202            206           2         49.4          20.2      17.9X
+In expression                                      3397           3469         129          2.9         339.7       1.0X
+InSet expression                                    241            249           9         41.4          24.1      14.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 5 small decimals:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        52             55           4         19.2          52.1       1.0X
-InSet expression                                    158            172          24          6.3         158.0       0.3X
+In expression                                        36             40           5         27.5          36.4       1.0X
+InSet expression                                    150            156          10          6.7         149.8       0.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 10 small decimals:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        63             64           2         16.0          62.6       1.0X
-InSet expression                                    160            164           4          6.2         160.1       0.4X
+In expression                                        44             47           4         22.7          44.0       1.0X
+InSet expression                                    150            161          13          6.7         150.2       0.3X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 25 small decimals:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        97             99           2         10.3          97.2       1.0X
-InSet expression                                    163            166           2          6.1         163.2       0.6X
+In expression                                        68             71           5         14.7          67.8       1.0X
+InSet expression                                    155            170          25          6.4         155.1       0.4X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 50 small decimals:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       184            186           2          5.4         184.4       1.0X
-InSet expression                                    173            176           2          5.8         173.0       1.1X
+In expression                                       167            169           2          6.0         166.6       1.0X
+InSet expression                                    165            176          15          6.1         164.7       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 100 small decimals:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       472            478           9          2.1         472.4       1.0X
-InSet expression                                    178            181           4          5.6         177.6       2.7X
+In expression                                       465            470           4          2.1         465.5       1.0X
+InSet expression                                    168            173           3          6.0         167.7       2.8X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 200 small decimals:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1054           1184         282          0.9        1053.8       1.0X
-InSet expression                                    188            191           1          5.3         187.9       5.6X
+In expression                                      1042           1131         173          1.0        1042.1       1.0X
+InSet expression                                    176            190          22          5.7         176.1       5.9X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 5 large decimals:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       275            282           8          3.6         274.7       1.0X
-InSet expression                                    195            199           8          5.1         194.7       1.4X
+In expression                                       233            243          22          4.3         233.0       1.0X
+InSet expression                                    174            185           9          5.7         174.1       1.3X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 10 large decimals:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       458            470           8          2.2         458.4       1.0X
-InSet expression                                    195            200           5          5.1         195.5       2.3X
+In expression                                       401            425          25          2.5         401.5       1.0X
+InSet expression                                    177            189          15          5.6         177.3       2.3X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 25 large decimals:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1093           1099           6          0.9        1093.3       1.0X
-InSet expression                                    202            204           1          4.9         202.2       5.4X
+In expression                                       963            978          19          1.0         963.4       1.0X
+InSet expression                                    185            195          11          5.4         184.7       5.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 50 large decimals:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      2091           2101          12          0.5        2090.6       1.0X
-InSet expression                                    207            210           2          4.8         206.7      10.1X
+In expression                                      1876           1905          43          0.5        1875.8       1.0X
+InSet expression                                    196            207          15          5.1         196.0       9.6X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 100 large decimals:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      4164           4187          16          0.2        4164.1       1.0X
-InSet expression                                    239            244           4          4.2         239.1      17.4X
+In expression                                      3784           3805          23          0.3        3783.5       1.0X
+InSet expression                                    220            234          21          4.5         220.1      17.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 200 large decimals:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      8331           8353          21          0.1        8330.7       1.0X
-InSet expression                                    251            252           1          4.0         250.7      33.2X
+In expression                                      7528           7548          31          0.1        7527.8       1.0X
+InSet expression                                    231            243          14          4.3         231.0      32.6X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 5 strings:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        66             67           2         15.2          65.8       1.0X
-InSet expression                                     81             84           6         12.3          81.0       0.8X
+In expression                                        50             54           5         19.9          50.2       1.0X
+InSet expression                                     71             75           6         14.1          71.0       0.7X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 10 strings:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        70             72           2         14.3          70.1       1.0X
-InSet expression                                     84             86           2         12.0          83.6       0.8X
+In expression                                        55             58           5         18.0          55.4       1.0X
+InSet expression                                     75             79           6         13.4          74.6       0.7X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 25 strings:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        89             92           3         11.2          89.3       1.0X
-InSet expression                                     92             94           2         10.9          92.2       1.0X
+In expression                                        88             91           4         11.4          87.6       1.0X
+InSet expression                                     83             88           7         12.0          83.2       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 50 strings:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       123            125           2          8.1         123.0       1.0X
-InSet expression                                     95             97           2         10.6          94.7       1.3X
+In expression                                       195            200           5          5.1         195.1       1.0X
+InSet expression                                     86             92           7         11.7          85.8       2.3X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 100 strings:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       221            223           2          4.5         220.7       1.0X
-InSet expression                                     92             94           3         10.9          91.7       2.4X
+In expression                                       452            454           2          2.2         451.6       1.0X
+InSet expression                                     85             91           6         11.8          85.0       5.3X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 200 strings:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       753            903         327          1.3         752.9       1.0X
-InSet expression                                     97            100           3         10.3          97.2       7.7X
+In expression                                       950           1050         196          1.1         950.4       1.0X
+InSet expression                                     92             98           6         10.8          92.2      10.3X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 5 timestamps:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        48             49           2        209.3           4.8       1.0X
-InSet expression                                    178            181           2         56.1          17.8       0.3X
+In expression                                        25             31           9        402.5           2.5       1.0X
+InSet expression                                    174            183          10         57.5          17.4       0.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 10 timestamps:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        65             67           4        154.9           6.5       1.0X
-InSet expression                                    196            198           1         50.9          19.6       0.3X
+In expression                                        33             38           9        303.9           3.3       1.0X
+InSet expression                                    193            204          12         51.8          19.3       0.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 25 timestamps:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       118            121           6         84.5          11.8       1.0X
-InSet expression                                    259            261           1         38.6          25.9       0.5X
+In expression                                        68             75          12        147.6           6.8       1.0X
+InSet expression                                    243            249          11         41.2          24.3       0.3X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 50 timestamps:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       199            201           1         50.2          19.9       1.0X
-InSet expression                                    280            282           3         35.7          28.0       0.7X
+In expression                                       120            130          14         83.5          12.0       1.0X
+InSet expression                                    262            267           4         38.2          26.2       0.5X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 100 timestamps:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       368            370           2         27.1          36.8       1.0X
-InSet expression                                    252            254           2         39.7          25.2       1.5X
+In expression                                       220            232          27         45.4          22.0       1.0X
+InSet expression                                    240            247           7         41.7          24.0       0.9X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 200 timestamps:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       710            715           3         14.1          71.0       1.0X
-InSet expression                                    251            255           6         39.8          25.1       2.8X
+In expression                                       428            432           4         23.4          42.8       1.0X
+InSet expression                                    237            247          11         42.2          23.7       1.8X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 5 dates:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       699            701           2         14.3          69.9       1.0X
-InSet expression                                    695            695           1         14.4          69.5       1.0X
+In expression                                       681            695          18         14.7          68.1       1.0X
+InSet expression                                    684            694          13         14.6          68.4       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 10 dates:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       700            703           2         14.3          70.0       1.0X
-InSet expression                                    700            701           1         14.3          70.0       1.0X
+In expression                                       712            725          19         14.0          71.2       1.0X
+InSet expression                                    671            680          17         14.9          67.1       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 25 dates:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       742            746           3         13.5          74.2       1.0X
-InSet expression                                    704            706           1         14.2          70.4       1.1X
+In expression                                       753            771          21         13.3          75.3       1.0X
+InSet expression                                    669            688          17         15.0          66.9       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 50 dates:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       839            840           2         11.9          83.9       1.0X
-InSet expression                                    710            713           2         14.1          71.0       1.2X
+In expression                                       835            849          15         12.0          83.5       1.0X
+InSet expression                                    692            713          17         14.4          69.2       1.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 100 dates:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1027           1030           2          9.7         102.7       1.0X
-InSet expression                                    712            714           1         14.0          71.2       1.4X
+In expression                                       977            996          19         10.2          97.7       1.0X
+InSet expression                                    693            705          19         14.4          69.3       1.4X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 200 dates:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1390           1411          43          7.2         139.0       1.0X
-InSet expression                                    727            730           3         13.8          72.7       1.9X
+In expression                                      1306           1312           8          7.7         130.6       1.0X
+InSet expression                                    705            729          23         14.2          70.5       1.9X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 300 dates:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1767           1774           8          5.7         176.7       1.0X
-InSet expression                                    739            741           3         13.5          73.9       2.4X
+In expression                                      1682           1725          72          5.9         168.2       1.0X
+InSet expression                                    716            732          20         14.0          71.6       2.3X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 400 dates:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      2144           2155          12          4.7         214.4       1.0X
-InSet expression                                    758            766           7         13.2          75.8       2.8X
+In expression                                      2094           2109          17          4.8         209.4       1.0X
+InSet expression                                    702            706           3         14.2          70.2       3.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 500 dates:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      2528           2552          22          4.0         252.8       1.0X
-InSet expression                                    865            868           2         11.6          86.5       2.9X
+In expression                                      2465           2476          11          4.1         246.5       1.0X
+InSet expression                                    881            903          18         11.4          88.1       2.8X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 5 arrays:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        58             61           3         17.2          58.3       1.0X
-InSet expression                                    113            115           2          8.9         112.7       0.5X
+In expression                                        46             50           6         21.9          45.7       1.0X
+InSet expression                                    101            109          11          9.9         100.9       0.5X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 10 arrays:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        86             89           3         11.7          85.8       1.0X
-InSet expression                                    113            116           2          8.8         113.3       0.8X
+In expression                                        70             74           5         14.2          70.3       1.0X
+InSet expression                                    101            108           8          9.9         101.4       0.7X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 25 arrays:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       278            280           2          3.6         278.4       1.0X
-InSet expression                                    141            145           5          7.1         140.6       2.0X
+In expression                                       255            261           4          3.9         255.5       1.0X
+InSet expression                                    132            141          10          7.6         132.1       1.9X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 50 arrays:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       577            704          75          1.7         576.9       1.0X
-InSet expression                                    193            196           3          5.2         193.5       3.0X
+In expression                                       549            573          36          1.8         549.2       1.0X
+InSet expression                                    187            197          11          5.3         187.1       2.9X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 100 arrays:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      2462           2643         375          0.4        2461.6       1.0X
-InSet expression                                    234            238           6          4.3         234.1      10.5X
+In expression                                      1109           1113           3          0.9        1109.4       1.0X
+InSet expression                                    229            238          12          4.4         228.9       4.8X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 200 arrays:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      8526           9225         487          0.1        8526.5       1.0X
-InSet expression                                    306            312           5          3.3         306.0      27.9X
+In expression                                      2601           3125         698          0.4        2600.8       1.0X
+InSet expression                                    310            319          10          3.2         310.3       8.4X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 5 structs:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        51             55           7         19.5          51.3       1.0X
-InSet expression                                    169            172           2          5.9         168.8       0.3X
+In expression                                        37             41           6         26.9          37.1       1.0X
+InSet expression                                    159            165           8          6.3         159.1       0.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 10 structs:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        62             66           6         16.2          61.6       1.0X
-InSet expression                                    170            175           7          5.9         169.9       0.4X
+In expression                                        52             57           5         19.2          52.2       1.0X
+InSet expression                                    162            169           4          6.2         162.4       0.3X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 25 structs:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       141            145           3          7.1         141.4       1.0X
-InSet expression                                    213            220           8          4.7         212.7       0.7X
+In expression                                       125            130           4          8.0         124.8       1.0X
+InSet expression                                    212            221           9          4.7         211.6       0.6X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 50 structs:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       335            365          67          3.0         334.9       1.0X
-InSet expression                                    303            309           7          3.3         303.4       1.1X
+In expression                                       317            323           3          3.2         317.2       1.0X
+InSet expression                                    304            312           9          3.3         304.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 100 structs:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       976           1114         286          1.0         976.4       1.0X
-InSet expression                                    358            365          10          2.8         357.5       2.7X
+In expression                                       731            735           3          1.4         730.6       1.0X
+InSet expression                                    372            381           8          2.7         371.6       2.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 200 structs:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      4162           4703         859          0.2        4162.5       1.0X
-InSet expression                                    473            476           3          2.1         473.2       8.8X
+In expression                                      1774           2026         487          0.6        1774.4       1.0X
+InSet expression                                    506            516          12          2.0         505.5       3.5X
 
 
diff --git a/sql/core/benchmarks/InExpressionBenchmark-results.txt b/sql/core/benchmarks/InExpressionBenchmark-results.txt
index 36d38ea9b162b..86c975c2dd5c4 100644
--- a/sql/core/benchmarks/InExpressionBenchmark-results.txt
+++ b/sql/core/benchmarks/InExpressionBenchmark-results.txt
@@ -2,739 +2,739 @@
 In Expression Benchmark
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 5 bytes:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       132            172          32         75.7          13.2       1.0X
-InSet expression                                     79             98          13        125.8           7.9       1.7X
+In expression                                        45             51           5        222.2           4.5       1.0X
+InSet expression                                     42             47           4        239.6           4.2       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 10 bytes:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       100            111           7         99.7          10.0       1.0X
-InSet expression                                     70             78           9        143.0           7.0       1.4X
+In expression                                        56             63           4        177.5           5.6       1.0X
+InSet expression                                     42             49           4        236.3           4.2       1.3X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 25 bytes:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       161            170           7         62.0          16.1       1.0X
-InSet expression                                     88             93           7        113.9           8.8       1.8X
+In expression                                        94            102           6        106.7           9.4       1.0X
+InSet expression                                     56             64           7        178.5           5.6       1.7X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 50 bytes:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       270            277           6         37.0          27.0       1.0X
-InSet expression                                    116            123           9         86.0          11.6       2.3X
+In expression                                       150            156           7         66.9          15.0       1.0X
+InSet expression                                     79             88           8        126.3           7.9       1.9X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 100 bytes:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       450            463          13         22.2          45.0       1.0X
-InSet expression                                    182            189           7         54.9          18.2       2.5X
+In expression                                       263            270           8         38.1          26.3       1.0X
+InSet expression                                    127            137           6         78.9          12.7       2.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 200 bytes:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       706            714           7         14.2          70.6       1.0X
-InSet expression                                    302            311           7         33.1          30.2       2.3X
+In expression                                       443            449           5         22.6          44.3       1.0X
+InSet expression                                    223            236          16         44.8          22.3       2.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 5 shorts:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        62             64           5        162.2           6.2       1.0X
-InSet expression                                     57             59           5        176.7           5.7       1.1X
+In expression                                        34             38           3        293.9           3.4       1.0X
+InSet expression                                     33             38           3        299.4           3.3       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 10 shorts:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        79             82           5        126.7           7.9       1.0X
-InSet expression                                     53             54           3        188.7           5.3       1.5X
+In expression                                        44             47           2        229.4           4.4       1.0X
+InSet expression                                     30             33           2        331.7           3.0       1.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 25 shorts:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       132            135           5         75.8          13.2       1.0X
-InSet expression                                     50             53           6        200.7           5.0       2.6X
+In expression                                        68             73           4        148.1           6.8       1.0X
+InSet expression                                     30             34           3        328.3           3.0       2.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 50 shorts:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       228            231           4         43.8          22.8       1.0X
-InSet expression                                     51             53           3        195.5           5.1       4.5X
+In expression                                       106            114           5         94.3          10.6       1.0X
+InSet expression                                     33             37           4        303.6           3.3       3.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 100 shorts:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       404            404           0         24.8          40.4       1.0X
-InSet expression                                     58             61           4        171.0           5.8       6.9X
+In expression                                       185            194          10         54.1          18.5       1.0X
+InSet expression                                     35             39           4        288.5           3.5       5.3X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 200 shorts:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       766            767           1         13.1          76.6       1.0X
-InSet expression                                     66             68           3        151.0           6.6      11.6X
+In expression                                       346            355           7         28.9          34.6       1.0X
+InSet expression                                     40             46           4        247.9           4.0       8.6X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 300 shorts:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1124           1124           0          8.9         112.4       1.0X
-InSet expression                                     74             77           4        135.3           7.4      15.2X
+In expression                                       508            522          11         19.7          50.8       1.0X
+InSet expression                                     47             51           4        212.8           4.7      10.8X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 400 shorts:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1566           1567           2          6.4         156.6       1.0X
-InSet expression                                     82             84           4        121.9           8.2      19.1X
+In expression                                      1017           1036          20          9.8         101.7       1.0X
+InSet expression                                     53             59           5        188.2           5.3      19.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 500 shorts:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1841           1871          37          5.4         184.1       1.0X
-InSet expression                                    314            318           5         31.8          31.4       5.9X
+In expression                                      1334           1360          19          7.5         133.4       1.0X
+InSet expression                                    256            267           9         39.1          25.6       5.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 5 shorts (non-compact):                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        58             62           5        171.0           5.8       1.0X
-InSet expression                                     53             55           4        187.5           5.3       1.1X
+In expression                                        32             35           2        307.8           3.2       1.0X
+InSet expression                                     32             36           2        312.2           3.2       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 10 shorts (non-compact):                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        76             77           2        131.3           7.6       1.0X
-InSet expression                                     65             66           3        154.3           6.5       1.2X
+In expression                                        41             44           2        241.6           4.1       1.0X
+InSet expression                                     36             39           3        279.9           3.6       1.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 25 shorts (non-compact):                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       138            140           5         72.7          13.8       1.0X
-InSet expression                                     74             78           8        135.2           7.4       1.9X
+In expression                                        67             71           4        149.9           6.7       1.0X
+InSet expression                                     41             44           3        246.3           4.1       1.6X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 50 shorts (non-compact):                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       226            227           1         44.2          22.6       1.0X
-InSet expression                                     83             86           7        120.8           8.3       2.7X
+In expression                                       106            111           4         94.1          10.6       1.0X
+InSet expression                                     46             49           3        216.1           4.6       2.3X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 100 shorts (non-compact):                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       402            403           0         24.9          40.2       1.0X
-InSet expression                                     93             94           3        108.0           9.3       4.3X
+In expression                                       186            195           7         53.8          18.6       1.0X
+InSet expression                                     54             61           5        183.8           5.4       3.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 200 shorts (non-compact):                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       761            762           0         13.1          76.1       1.0X
-InSet expression                                    113            116           7         88.4          11.3       6.7X
+In expression                                       346            363          16         28.9          34.6       1.0X
+InSet expression                                     64             69           4        156.0           6.4       5.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 300 shorts (non-compact):                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1125           1125           0          8.9         112.5       1.0X
-InSet expression                                    136            142          11         73.5          13.6       8.3X
+In expression                                       523            541          19         19.1          52.3       1.0X
+InSet expression                                     72             77           4        138.7           7.2       7.3X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 400 shorts (non-compact):                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1486           1487           1          6.7         148.6       1.0X
-InSet expression                                    141            142           2         70.8          14.1      10.5X
+In expression                                      1042           1053          10          9.6         104.2       1.0X
+InSet expression                                     86             92           5        116.3           8.6      12.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 500 shorts (non-compact):                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1842           1873          67          5.4         184.2       1.0X
-InSet expression                                    315            318           3         31.7          31.5       5.8X
+In expression                                      1350           1369          11          7.4         135.0       1.0X
+InSet expression                                    259            270          10         38.6          25.9       5.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 5 ints:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        54             55           4        186.6           5.4       1.0X
-InSet expression                                     49             51           3        203.0           4.9       1.1X
+In expression                                        32             36           3        311.2           3.2       1.0X
+InSet expression                                     28             31           2        357.8           2.8       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 10 ints:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        72             74           5        139.7           7.2       1.0X
-InSet expression                                     46             48           5        218.2           4.6       1.6X
+In expression                                        41             44           3        245.1           4.1       1.0X
+InSet expression                                     29             32           3        349.0           2.9       1.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 25 ints:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       125            127           5         79.9          12.5       1.0X
-InSet expression                                     47             48           4        212.5           4.7       2.7X
+In expression                                        66             71           4        150.9           6.6       1.0X
+InSet expression                                     29             32           3        339.6           2.9       2.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 50 ints:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       221            224           4         45.2          22.1       1.0X
-InSet expression                                     48             49           3        206.3           4.8       4.6X
+In expression                                       103            109           5         96.7          10.3       1.0X
+InSet expression                                     30             34           3        329.8           3.0       3.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 100 ints:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       401            404           6         25.0          40.1       1.0X
-InSet expression                                     55             56           2        180.5           5.5       7.2X
+In expression                                       184            194           7         54.3          18.4       1.0X
+InSet expression                                     33             37           3        306.6           3.3       5.6X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 200 ints:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       762            763           1         13.1          76.2       1.0X
-InSet expression                                     63             69          14        159.8           6.3      12.2X
+In expression                                       339            350          10         29.5          33.9       1.0X
+InSet expression                                     38             42           3        261.2           3.8       8.9X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 300 ints:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1117           1117           0          9.0         111.7       1.0X
-InSet expression                                     70             71           2        143.3           7.0      16.0X
+In expression                                       506            512           6         19.8          50.6       1.0X
+InSet expression                                     43             47           4        232.6           4.3      11.8X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 400 ints:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1557           1558           1          6.4         155.7       1.0X
-InSet expression                                     77             78           2        129.6           7.7      20.2X
+In expression                                       989           1008          21         10.1          98.9       1.0X
+InSet expression                                     51             57           4        195.1           5.1      19.3X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 500 ints:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1841           1877          79          5.4         184.1       1.0X
-InSet expression                                    320            322           2         31.2          32.0       5.8X
+In expression                                      1398           1422          19          7.2         139.8       1.0X
+InSet expression                                    262            274           8         38.1          26.2       5.3X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 5 ints (non-compact):                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        43             44           3        231.6           4.3       1.0X
-InSet expression                                     40             42           4        252.4           4.0       1.1X
+In expression                                        22             25           2        448.9           2.2       1.0X
+InSet expression                                     20             23           2        510.5           2.0       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 10 ints (non-compact):                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        62             64           4        162.0           6.2       1.0X
-InSet expression                                     45             47           4        222.2           4.5       1.4X
+In expression                                        31             34           2        323.6           3.1       1.0X
+InSet expression                                     22             24           2        454.8           2.2       1.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 25 ints (non-compact):                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       117            119           4         85.4          11.7       1.0X
-InSet expression                                     57             59           5        176.9           5.7       2.1X
+In expression                                        55             61           3        180.5           5.5       1.0X
+InSet expression                                     27             30           2        368.8           2.7       2.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 50 ints (non-compact):                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       207            207           0         48.3          20.7       1.0X
-InSet expression                                     65             66           3        153.3           6.5       3.2X
+In expression                                       172            186          13         58.0          17.2       1.0X
+InSet expression                                     31             34           3        326.2           3.1       5.6X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 100 ints (non-compact):                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       390            394           4         25.7          39.0       1.0X
-InSet expression                                     76             77           3        132.0           7.6       5.1X
+In expression                                       339            358          18         29.5          33.9       1.0X
+InSet expression                                     36             40           4        277.1           3.6       9.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 200 ints (non-compact):                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       752            755           2         13.3          75.2       1.0X
-InSet expression                                    111            112           3         90.2          11.1       6.8X
+In expression                                       653            662           7         15.3          65.3       1.0X
+InSet expression                                     63             67           3        159.6           6.3      10.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 300 ints (non-compact):                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1106           1115           6          9.0         110.6       1.0X
-InSet expression                                    129            130           3         77.7          12.9       8.6X
+In expression                                       967            990          17         10.3          96.7       1.0X
+InSet expression                                     74             81           5        135.1           7.4      13.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 400 ints (non-compact):                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1476           1484           6          6.8         147.6       1.0X
-InSet expression                                    129            130           3         77.3          12.9      11.4X
+In expression                                      1288           1299           9          7.8         128.8       1.0X
+InSet expression                                     80             85           5        125.6           8.0      16.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 500 ints (non-compact):                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1844           1900         124          5.4         184.4       1.0X
-InSet expression                                    321            322           2         31.2          32.1       5.7X
+In expression                                      1600           1647          83          6.3         160.0       1.0X
+InSet expression                                    252            258           4         39.7          25.2       6.3X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 5 longs:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        50             53           5        199.5           5.0       1.0X
-InSet expression                                    166            169           5         60.2          16.6       0.3X
+In expression                                        23             26           2        436.1           2.3       1.0X
+InSet expression                                    133            140           4         75.1          13.3       0.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 10 longs:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        64             68           8        155.5           6.4       1.0X
-InSet expression                                    186            188           4         53.9          18.6       0.3X
+In expression                                        29             33           2        341.4           2.9       1.0X
+InSet expression                                    148            156           6         67.5          14.8       0.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 25 longs:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       118            119           2         84.7          11.8       1.0X
-InSet expression                                    194            208          26         51.4          19.4       0.6X
+In expression                                        56             61           4        179.0           5.6       1.0X
+InSet expression                                    153            160           7         65.3          15.3       0.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 50 longs:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       208            208           0         48.1          20.8       1.0X
-InSet expression                                    240            244           5         41.7          24.0       0.9X
+In expression                                        95            101           4        105.6           9.5       1.0X
+InSet expression                                    189            206          13         52.9          18.9       0.5X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 100 longs:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       390            397          10         25.6          39.0       1.0X
-InSet expression                                    205            207           5         48.8          20.5       1.9X
+In expression                                       179            186           6         55.8          17.9       1.0X
+InSet expression                                    160            176          12         62.7          16.0       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 200 longs:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       745            745           1         13.4          74.5       1.0X
-InSet expression                                    194            197           5         51.5          19.4       3.8X
+In expression                                       339            348           8         29.5          33.9       1.0X
+InSet expression                                    155            163           6         64.6          15.5       2.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 5 floats:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        90             91           1        111.2           9.0       1.0X
-InSet expression                                    199            202           4         50.2          19.9       0.5X
+In expression                                        46             50           3        216.0           4.6       1.0X
+InSet expression                                    156            165           6         64.1          15.6       0.3X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 10 floats:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       131            132           0         76.4          13.1       1.0X
-InSet expression                                    221            223           2         45.2          22.1       0.6X
+In expression                                        62             65           2        160.5           6.2       1.0X
+InSet expression                                    177            185           6         56.6          17.7       0.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 25 floats:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       244            245           0         40.9          24.4       1.0X
-InSet expression                                    235            236           1         42.6          23.5       1.0X
+In expression                                       108            118           8         93.0          10.8       1.0X
+InSet expression                                    182            195           8         54.8          18.2       0.6X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 50 floats:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       423            424           2         23.7          42.3       1.0X
-InSet expression                                    284            285           2         35.3          28.4       1.5X
+In expression                                       190            203          10         52.7          19.0       1.0X
+InSet expression                                    231            239           6         43.3          23.1       0.8X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 100 floats:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       777            778           1         12.9          77.7       1.0X
-InSet expression                                    248            249           2         40.4          24.8       3.1X
+In expression                                       353            358           5         28.4          35.3       1.0X
+InSet expression                                    190            203           7         52.7          19.0       1.9X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 200 floats:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      3032           3125         203          3.3         303.2       1.0X
-InSet expression                                    239            241           2         41.8          23.9      12.7X
+In expression                                      2398           2491         128          4.2         239.8       1.0X
+InSet expression                                    194            203           9         51.4          19.4      12.3X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 5 doubles:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        84             86           3        118.8           8.4       1.0X
-InSet expression                                    167            168           2         59.9          16.7       0.5X
+In expression                                        47             50           3        214.1           4.7       1.0X
+InSet expression                                    142            153          13         70.2          14.2       0.3X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 10 doubles:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       126            127           1         79.1          12.6       1.0X
-InSet expression                                    183            185           2         54.6          18.3       0.7X
+In expression                                        58             62           3        172.9           5.8       1.0X
+InSet expression                                    163            174          11         61.4          16.3       0.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 25 doubles:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       239            240           0         41.8          23.9       1.0X
-InSet expression                                    189            192           4         52.9          18.9       1.3X
+In expression                                       108            126           9         92.5          10.8       1.0X
+InSet expression                                    173            184           9         57.8          17.3       0.6X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 50 doubles:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       417            417           0         24.0          41.7       1.0X
-InSet expression                                    231            234           4         43.3          23.1       1.8X
+In expression                                       188            196           7         53.3          18.8       1.0X
+InSet expression                                    212            218           4         47.1          21.2       0.9X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 100 doubles:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       770            772           3         13.0          77.0       1.0X
-InSet expression                                    201            204           5         49.7          20.1       3.8X
+In expression                                       350            359           7         28.6          35.0       1.0X
+InSet expression                                    178            188          10         56.3          17.8       2.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 200 doubles:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      3587           3686         212          2.8         358.7       1.0X
-InSet expression                                    196            198           3         50.9          19.6      18.3X
+In expression                                      2957           3035          91          3.4         295.7       1.0X
+InSet expression                                    170            180          10         58.8          17.0      17.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 5 small decimals:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        50             51           2         20.1          49.7       1.0X
-InSet expression                                    151            153           3          6.6         150.7       0.3X
+In expression                                        30             33           3         33.4          30.0       1.0X
+InSet expression                                    110            118          10          9.1         109.6       0.3X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 10 small decimals:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        62             62           2         16.2          61.6       1.0X
-InSet expression                                    153            155           2          6.5         153.3       0.4X
+In expression                                        36             40           4         28.1          35.6       1.0X
+InSet expression                                    112            119           5          8.9         112.0       0.3X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 25 small decimals:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        95             96           2         10.5          95.0       1.0X
-InSet expression                                    156            158           2          6.4         156.4       0.6X
+In expression                                        56             61           3         17.9          55.9       1.0X
+InSet expression                                    115            122           6          8.7         114.8       0.5X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 50 small decimals:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       175            177           5          5.7         175.2       1.0X
-InSet expression                                    165            167           3          6.1         164.9       1.1X
+In expression                                       126            132           6          7.9         126.3       1.0X
+InSet expression                                    119            127           5          8.4         119.0       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 100 small decimals:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       440            440           0          2.3         439.7       1.0X
-InSet expression                                    167            169           2          6.0         167.0       2.6X
+In expression                                       323            331           7          3.1         323.4       1.0X
+InSet expression                                    123            131           7          8.1         123.4       2.6X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 200 small decimals:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1005           1154         330          1.0        1004.8       1.0X
-InSet expression                                    180            182           2          5.5         180.2       5.6X
+In expression                                       790            811          16          1.3         790.1       1.0X
+InSet expression                                    132            146           9          7.6         132.1       6.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 5 large decimals:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       251            258           7          4.0         251.1       1.0X
-InSet expression                                    193            197           3          5.2         193.2       1.3X
+In expression                                       183            193           6          5.5         182.7       1.0X
+InSet expression                                    132            139           5          7.6         131.8       1.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 10 large decimals:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       424            437           8          2.4         424.2       1.0X
-InSet expression                                    196            198           2          5.1         195.5       2.2X
+In expression                                       320            340          23          3.1         319.5       1.0X
+InSet expression                                    133            142           5          7.5         133.4       2.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 25 large decimals:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1018           1023           5          1.0        1017.8       1.0X
-InSet expression                                    203            206           3          4.9         202.9       5.0X
+In expression                                       798            816          16          1.3         798.3       1.0X
+InSet expression                                    137            147           7          7.3         137.1       5.8X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 50 large decimals:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1947           1955           7          0.5        1947.5       1.0X
-InSet expression                                    208            211           3          4.8         208.4       9.3X
+In expression                                      1556           1592          35          0.6        1556.1       1.0X
+InSet expression                                    141            157          14          7.1         140.6      11.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 100 large decimals:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      3886           3899          10          0.3        3885.9       1.0X
-InSet expression                                    233            235           4          4.3         232.6      16.7X
+In expression                                      3156           3195          57          0.3        3156.2       1.0X
+InSet expression                                    162            169           5          6.2         161.5      19.5X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 200 large decimals:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      7702           7793          60          0.1        7701.8       1.0X
-InSet expression                                    243            248           6          4.1         243.4      31.6X
+In expression                                      6347           6391          49          0.2        6346.6       1.0X
+InSet expression                                    178            186           5          5.6         178.0      35.6X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 5 strings:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       132            135           4          7.6         132.1       1.0X
-InSet expression                                    147            149           2          6.8         147.1       0.9X
+In expression                                        96            106           8         10.4          96.1       1.0X
+InSet expression                                    110            120           8          9.1         109.9       0.9X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 10 strings:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       139            141           5          7.2         139.0       1.0X
-InSet expression                                    150            151           2          6.7         149.6       0.9X
+In expression                                        99            105           5         10.1          98.6       1.0X
+InSet expression                                    112            118           5          8.9         111.8       0.9X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 25 strings:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       160            161           2          6.3         159.6       1.0X
-InSet expression                                    157            158           2          6.4         157.3       1.0X
+In expression                                       117            123           5          8.6         116.7       1.0X
+InSet expression                                    119            128           9          8.4         118.7       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 50 strings:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       190            192           2          5.3         189.8       1.0X
-InSet expression                                    160            161           2          6.2         160.5       1.2X
+In expression                                       194            202           6          5.1         194.5       1.0X
+InSet expression                                    122            130           6          8.2         121.8       1.6X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 100 strings:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       264            265           2          3.8         263.8       1.0X
-InSet expression                                    159            160           2          6.3         158.5       1.7X
+In expression                                       373            387          12          2.7         372.5       1.0X
+InSet expression                                    124            132           6          8.1         123.9       3.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 200 strings:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       735            884         327          1.4         735.4       1.0X
-InSet expression                                    164            166           3          6.1         163.9       4.5X
+In expression                                       751            764          11          1.3         751.0       1.0X
+InSet expression                                    128            138           9          7.8         127.9       5.9X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 5 timestamps:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        45             46           3        223.6           4.5       1.0X
-InSet expression                                    162            164           3         61.6          16.2       0.3X
+In expression                                        22             25           2        451.1           2.2       1.0X
+InSet expression                                    138            149          12         72.6          13.8       0.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 10 timestamps:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        62             65           7        161.9           6.2       1.0X
-InSet expression                                    179            181           2         56.0          17.9       0.3X
+In expression                                        30             33           3        332.6           3.0       1.0X
+InSet expression                                    148            156           6         67.4          14.8       0.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 25 timestamps:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       115            116           1         86.8          11.5       1.0X
-InSet expression                                    222            225           6         45.1          22.2       0.5X
+In expression                                        53             58           4        187.2           5.3       1.0X
+InSet expression                                    196            204          13         51.1          19.6       0.3X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 50 timestamps:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       198            199           1         50.6          19.8       1.0X
-InSet expression                                    238            239           2         42.0          23.8       0.8X
+In expression                                        94            101           5        106.9           9.4       1.0X
+InSet expression                                    203            214           8         49.3          20.3       0.5X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 100 timestamps:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       363            368           5         27.6          36.3       1.0X
-InSet expression                                    222            224           2         45.0          22.2       1.6X
+In expression                                       168            175           5         59.6          16.8       1.0X
+InSet expression                                    185            197           9         54.0          18.5       0.9X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 200 timestamps:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       701            706           4         14.3          70.1       1.0X
-InSet expression                                    226            228           3         44.2          22.6       3.1X
+In expression                                       328            358          22         30.5          32.8       1.0X
+InSet expression                                    175            182           6         57.0          17.5       1.9X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 5 dates:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       659            665           5         15.2          65.9       1.0X
-InSet expression                                    660            664           3         15.1          66.0       1.0X
+In expression                                       516            527          12         19.4          51.6       1.0X
+InSet expression                                    509            535          20         19.6          50.9       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 10 dates:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       663            667           2         15.1          66.3       1.0X
-InSet expression                                    659            661           2         15.2          65.9       1.0X
+In expression                                       519            535          10         19.3          51.9       1.0X
+InSet expression                                    506            525          14         19.8          50.6       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 25 dates:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       700            705           4         14.3          70.0       1.0X
-InSet expression                                    667            669           2         15.0          66.7       1.0X
+In expression                                       547            572          19         18.3          54.7       1.0X
+InSet expression                                    524            542          17         19.1          52.4       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 50 dates:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       801            805           3         12.5          80.1       1.0X
-InSet expression                                    675            677           1         14.8          67.5       1.2X
+In expression                                       627            664          22         16.0          62.7       1.0X
+InSet expression                                    550            574          20         18.2          55.0       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 100 dates:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       984            987           5         10.2          98.4       1.0X
-InSet expression                                    685            690           4         14.6          68.5       1.4X
+In expression                                       797            812          20         12.5          79.7       1.0X
+InSet expression                                    555            563           6         18.0          55.5       1.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 200 dates:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1350           1356           4          7.4         135.0       1.0X
-InSet expression                                    710            712           3         14.1          71.0       1.9X
+In expression                                      1039           1065          24          9.6         103.9       1.0X
+InSet expression                                    570            599          35         17.5          57.0       1.8X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 300 dates:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1716           1722           6          5.8         171.6       1.0X
-InSet expression                                    716            719           2         14.0          71.6       2.4X
+In expression                                      1375           1392          16          7.3         137.5       1.0X
+InSet expression                                    571            583          11         17.5          57.1       2.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 400 dates:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      2083           2100          13          4.8         208.3       1.0X
-InSet expression                                    742            744           1         13.5          74.2       2.8X
+In expression                                      1650           1713          54          6.1         165.0       1.0X
+InSet expression                                    566            580          11         17.7          56.6       2.9X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 500 dates:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      2469           2486          11          4.1         246.9       1.0X
-InSet expression                                    829            831           2         12.1          82.9       3.0X
+In expression                                      2009           2052          43          5.0         200.9       1.0X
+InSet expression                                    642            683          35         15.6          64.2       3.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 5 arrays:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        56             58           4         17.9          55.9       1.0X
-InSet expression                                    123            124           2          8.1         123.1       0.5X
+In expression                                        36             39           3         27.9          35.9       1.0X
+InSet expression                                     83             88           4         12.0          83.2       0.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 10 arrays:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        83             84           2         12.1          82.7       1.0X
-InSet expression                                    124            127           7          8.1         123.8       0.7X
+In expression                                        59             63           3         16.9          59.2       1.0X
+InSet expression                                     85             90           5         11.8          84.8       0.7X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 25 arrays:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       275            281           9          3.6         275.5       1.0X
-InSet expression                                    155            158           4          6.4         155.2       1.8X
+In expression                                       215            231          11          4.6         215.4       1.0X
+InSet expression                                    108            113           4          9.3         107.6       2.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 50 arrays:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       659            718         109          1.5         659.4       1.0X
-InSet expression                                    217            218           2          4.6         217.0       3.0X
+In expression                                       462            473          10          2.2         462.5       1.0X
+InSet expression                                    158            167           6          6.3         157.7       2.9X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 100 arrays:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      2488           2684         434          0.4        2488.4       1.0X
-InSet expression                                    267            270           3          3.7         266.7       9.3X
+In expression                                       965           1001          24          1.0         964.8       1.0X
+InSet expression                                    184            200          11          5.4         183.9       5.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 200 arrays:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      9462          10091         897          0.1        9462.2       1.0X
-InSet expression                                    347            349           2          2.9         347.4      27.2X
+In expression                                      2142           2516         760          0.5        2142.5       1.0X
+InSet expression                                    251            262           9          4.0         250.7       8.5X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 5 structs:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        51             55           7         19.7          50.8       1.0X
-InSet expression                                    166            168           4          6.0         166.3       0.3X
+In expression                                        33             36           4         30.7          32.6       1.0X
+InSet expression                                    125            132           4          8.0         125.2       0.3X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 10 structs:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                        66             70           5         15.1          66.4       1.0X
-InSet expression                                    167            170           3          6.0         167.3       0.4X
+In expression                                        43             48           5         23.2          43.1       1.0X
+InSet expression                                    122            133           8          8.2         122.0       0.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 25 structs:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       142            144           3          7.0         142.1       1.0X
-InSet expression                                    211            215           6          4.7         211.2       0.7X
+In expression                                       106            113           5          9.5         105.6       1.0X
+InSet expression                                    160            172           9          6.3         160.0       0.7X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 50 structs:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                       375            378           5          2.7         375.5       1.0X
-InSet expression                                    297            298           2          3.4         297.5       1.3X
+In expression                                       294            304           9          3.4         293.6       1.0X
+InSet expression                                    234            258          18          4.3         234.1       1.3X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 100 structs:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      1122           1260         304          0.9        1122.2       1.0X
-InSet expression                                    354            357           4          2.8         353.9       3.2X
+In expression                                       687            700           8          1.5         686.7       1.0X
+InSet expression                                    277            288           9          3.6         277.0       2.5X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 200 structs:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-In expression                                      5651           5989         644          0.2        5651.4       1.0X
-InSet expression                                    471            473           2          2.1         471.0      12.0X
+In expression                                      2864           3173         467          0.3        2864.4       1.0X
+InSet expression                                    362            379          10          2.8         362.1       7.9X
 
 
diff --git a/sql/core/benchmarks/InsertTableWithDynamicPartitionsBenchmark-jdk11-results.txt b/sql/core/benchmarks/InsertTableWithDynamicPartitionsBenchmark-jdk11-results.txt
new file mode 100644
index 0000000000000..33b57463daf76
--- /dev/null
+++ b/sql/core/benchmarks/InsertTableWithDynamicPartitionsBenchmark-jdk11-results.txt
@@ -0,0 +1,8 @@
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+dynamic insert table benchmark, totalRows = 200000:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+----------------------------------------------------------------------------------------------------------------------------------
+one partition column, 100 partitions                         6789           6843          76          0.0       33944.4       1.0X
+two partition columns, 500 partitions                       17469          18429        1358          0.0       87345.0       0.4X
+three partition columns, 2000 partitions                    48564          49213         918          0.0      242817.5       0.1X
+
diff --git a/sql/core/benchmarks/InsertTableWithDynamicPartitionsBenchmark-results.txt b/sql/core/benchmarks/InsertTableWithDynamicPartitionsBenchmark-results.txt
new file mode 100644
index 0000000000000..f8f7e2a446903
--- /dev/null
+++ b/sql/core/benchmarks/InsertTableWithDynamicPartitionsBenchmark-results.txt
@@ -0,0 +1,8 @@
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+dynamic insert table benchmark, totalRows = 200000:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+----------------------------------------------------------------------------------------------------------------------------------
+one partition column, 100 partitions                         5195           5197           3          0.0       25976.7       1.0X
+two partition columns, 500 partitions                       13231          13395         232          0.0       66154.1       0.4X
+three partition columns, 2000 partitions                    36808          37118         437          0.0      184041.2       0.1X
+
diff --git a/sql/core/benchmarks/IntervalBenchmark-jdk11-results.txt b/sql/core/benchmarks/IntervalBenchmark-jdk11-results.txt
index 8958d7c53413f..9343c3dee5145 100644
--- a/sql/core/benchmarks/IntervalBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/IntervalBenchmark-jdk11-results.txt
@@ -1,29 +1,40 @@
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 cast strings to intervals:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-prepare string w/ interval                          448            469          20          2.2         447.6       1.0X
-prepare string w/o interval                         405            409           4          2.5         404.6       1.1X
-1 units w/ interval                                 321            328           6          3.1         321.4       1.4X
-1 units w/o interval                                303            307           4          3.3         303.1       1.5X
-2 units w/ interval                                 445            458          12          2.2         444.6       1.0X
-2 units w/o interval                                416            424          10          2.4         416.2       1.1X
-3 units w/ interval                                1006           1012           8          1.0        1006.4       0.4X
-3 units w/o interval                               1240           1249           8          0.8        1239.6       0.4X
-4 units w/ interval                                1295           1418         106          0.8        1295.4       0.3X
-4 units w/o interval                               1172           1188          15          0.9        1171.6       0.4X
-5 units w/ interval                                1326           1335          11          0.8        1325.6       0.3X
-5 units w/o interval                               1309           1336          44          0.8        1308.7       0.3X
-6 units w/ interval                                1441           1464          29          0.7        1441.0       0.3X
-6 units w/o interval                               1350           1369          17          0.7        1350.1       0.3X
-7 units w/ interval                                1606           1669          99          0.6        1605.6       0.3X
-7 units w/o interval                               1546           1557          12          0.6        1546.3       0.3X
-8 units w/ interval                                1771           1875         120          0.6        1770.6       0.3X
-8 units w/o interval                               1775           1789          13          0.6        1775.2       0.3X
-9 units w/ interval                                2126           2757         849          0.5        2126.4       0.2X
-9 units w/o interval                               2053           2070          21          0.5        2053.3       0.2X
-10 units w/ interval                               2209           2243          30          0.5        2209.1       0.2X
-10 units w/o interval                              2400           2702         365          0.4        2400.2       0.2X
-11 units w/ interval                               2616           2699          72          0.4        2616.5       0.2X
-11 units w/o interval                              3218           3380         195          0.3        3218.4       0.1X
+prepare string w/ interval                          643            653          11          1.6         643.0       1.0X
+prepare string w/o interval                         625            654          44          1.6         624.7       1.0X
+1 units w/ interval                                 542            559          15          1.8         541.6       1.2X
+1 units w/o interval                                474            491          15          2.1         474.0       1.4X
+2 units w/ interval                                 779            788           9          1.3         778.9       0.8X
+2 units w/o interval                                714            733          17          1.4         713.7       0.9X
+3 units w/ interval                                1689           1718          44          0.6        1688.9       0.4X
+3 units w/o interval                               1717           1741          23          0.6        1716.9       0.4X
+4 units w/ interval                                2130           2166          45          0.5        2130.3       0.3X
+4 units w/o interval                               2047           2061          23          0.5        2046.7       0.3X
+5 units w/ interval                                2296           2310          18          0.4        2295.8       0.3X
+5 units w/o interval                               2250           2288          33          0.4        2250.0       0.3X
+6 units w/ interval                                2573           2588          24          0.4        2572.6       0.2X
+6 units w/o interval                               2491           2517          33          0.4        2491.3       0.3X
+7 units w/ interval                                3087           3127          67          0.3        3086.9       0.2X
+7 units w/o interval                               3104           3150          46          0.3        3103.6       0.2X
+8 units w/ interval                                3281           3319          46          0.3        3281.1       0.2X
+8 units w/o interval                               3369           3442          71          0.3        3368.7       0.2X
+9 units w/ interval                                3436           3472          31          0.3        3436.3       0.2X
+9 units w/o interval                               3471           3511          39          0.3        3471.3       0.2X
+10 units w/ interval                               4040           4084          69          0.2        4040.1       0.2X
+10 units w/o interval                              4145           4170          22          0.2        4145.2       0.2X
+11 units w/ interval                               4345           4432          78          0.2        4345.0       0.1X
+11 units w/o interval                              4425           4465          68          0.2        4424.5       0.1X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+make_interval():                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+--------------------------------------------------------------------------------------------------------------------------
+prepare make_interval()                              3911           3961          44          0.3        3910.6       1.0X
+make_interval(0, 1, 2, 3, 4, 5, 50.123456)             66             66           0         15.2          65.8      59.4X
+make_interval(*, *, 2, 3, 4, 5, 50.123456)             95             99           5         10.5          95.2      41.1X
+make_interval(0, 1, *, *, 4, 5, 50.123456)            110            112           2          9.1         110.2      35.5X
+make_interval(0, 1, 2, 3, *, *, *)                   4004           4050          42          0.2        4003.6       1.0X
+make_interval(*, *, *, *, *, *, *)                   3942           3957          13          0.3        3942.4       1.0X
 
diff --git a/sql/core/benchmarks/IntervalBenchmark-results.txt b/sql/core/benchmarks/IntervalBenchmark-results.txt
index 48af333b78ba4..d05429b4dcf75 100644
--- a/sql/core/benchmarks/IntervalBenchmark-results.txt
+++ b/sql/core/benchmarks/IntervalBenchmark-results.txt
@@ -1,29 +1,40 @@
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 cast strings to intervals:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-prepare string w/ interval                          389            410          21          2.6         388.7       1.0X
-prepare string w/o interval                         340            360          18          2.9         340.5       1.1X
-1 units w/ interval                                 378            389          16          2.6         377.8       1.0X
-1 units w/o interval                                346            350           5          2.9         346.2       1.1X
-2 units w/ interval                                 444            457          11          2.3         444.2       0.9X
-2 units w/o interval                                455            464          12          2.2         455.1       0.9X
-3 units w/ interval                                 942            964          20          1.1         941.5       0.4X
-3 units w/o interval                                927           1020          93          1.1         927.3       0.4X
-4 units w/ interval                                1114           1127          17          0.9        1113.9       0.3X
-4 units w/o interval                               1100           1105           4          0.9        1100.3       0.4X
-5 units w/ interval                                1180           1244          57          0.8        1180.1       0.3X
-5 units w/o interval                               1135           1141           6          0.9        1135.2       0.3X
-6 units w/ interval                                1284           1316          48          0.8        1284.0       0.3X
-6 units w/o interval                               1276           1357         122          0.8        1276.1       0.3X
-7 units w/ interval                                1609           1636          32          0.6        1609.1       0.2X
-7 units w/o interval                               1551           1578          36          0.6        1550.9       0.3X
-8 units w/ interval                                1787           1874         129          0.6        1787.1       0.2X
-8 units w/o interval                               1751           1767          15          0.6        1750.6       0.2X
-9 units w/ interval                                1960           2065         141          0.5        1959.7       0.2X
-9 units w/o interval                               1885           1908          39          0.5        1885.1       0.2X
-10 units w/ interval                               2178           2185          11          0.5        2177.9       0.2X
-10 units w/o interval                              2150           2255         164          0.5        2150.1       0.2X
-11 units w/ interval                               2457           2542         139          0.4        2456.7       0.2X
-11 units w/o interval                              2557           2770         188          0.4        2556.7       0.2X
+prepare string w/ interval                          638            642           4          1.6         637.6       1.0X
+prepare string w/o interval                         637            640           5          1.6         636.6       1.0X
+1 units w/ interval                                 652            673          18          1.5         652.4       1.0X
+1 units w/o interval                                610            624          12          1.6         610.4       1.0X
+2 units w/ interval                                 795            800           7          1.3         794.9       0.8X
+2 units w/o interval                                773            812          35          1.3         773.3       0.8X
+3 units w/ interval                                1713           1756          38          0.6        1713.3       0.4X
+3 units w/o interval                               1714           1747          37          0.6        1714.3       0.4X
+4 units w/ interval                                1945           1980          50          0.5        1944.9       0.3X
+4 units w/o interval                               1923           1986          55          0.5        1922.8       0.3X
+5 units w/ interval                                2138           2184          42          0.5        2138.4       0.3X
+5 units w/o interval                               2149           2174          35          0.5        2148.5       0.3X
+6 units w/ interval                                2383           2418          33          0.4        2383.0       0.3X
+6 units w/o interval                               2413           2432          23          0.4        2412.9       0.3X
+7 units w/ interval                                2976           3005          25          0.3        2976.1       0.2X
+7 units w/o interval                               2955           2995          56          0.3        2954.9       0.2X
+8 units w/ interval                                3256           3303          42          0.3        3256.1       0.2X
+8 units w/o interval                               3264           3320          49          0.3        3263.9       0.2X
+9 units w/ interval                                3587           3616          27          0.3        3587.0       0.2X
+9 units w/o interval                               3613           3681          82          0.3        3613.5       0.2X
+10 units w/ interval                               3972           3994          19          0.3        3972.4       0.2X
+10 units w/o interval                              3934           3984          48          0.3        3933.7       0.2X
+11 units w/ interval                               4226           4233           6          0.2        4226.1       0.2X
+11 units w/o interval                              4172           4222          64          0.2        4171.9       0.2X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+make_interval():                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+--------------------------------------------------------------------------------------------------------------------------
+prepare make_interval()                              4533           4626          91          0.2        4532.8       1.0X
+make_interval(0, 1, 2, 3, 4, 5, 50.123456)             68             68           0         14.7          67.9      66.7X
+make_interval(*, *, 2, 3, 4, 5, 50.123456)             92             98           6         10.8          92.3      49.1X
+make_interval(0, 1, *, *, 4, 5, 50.123456)            103            107           6          9.7         102.8      44.1X
+make_interval(0, 1, 2, 3, *, *, *)                   4521           4576          47          0.2        4521.4       1.0X
+make_interval(*, *, *, *, *, *, *)                   4580           4591          11          0.2        4579.9       1.0X
 
diff --git a/sql/core/benchmarks/JoinBenchmark-jdk11-results.txt b/sql/core/benchmarks/JoinBenchmark-jdk11-results.txt
index 1e61017425aa3..7c90d77acf9e1 100644
--- a/sql/core/benchmarks/JoinBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/JoinBenchmark-jdk11-results.txt
@@ -2,74 +2,81 @@
 Join Benchmark
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Join w long:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Join w long wholestage off                         4441           4572         185          4.7         211.8       1.0X
-Join w long wholestage on                          1409           1500          96         14.9          67.2       3.2X
+Join w long wholestage off                         5102           5104           2          4.1         243.3       1.0X
+Join w long wholestage on                          1557           1602          43         13.5          74.2       3.3X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Join w long duplicated:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Join w long duplicated wholestage off              5111           5116           7          4.1         243.7       1.0X
-Join w long duplicated wholestage on               1493           1518          22         14.0          71.2       3.4X
+Join w long duplicated wholestage off              5824           5825           1          3.6         277.7       1.0X
+Join w long duplicated wholestage on               1558           1650          91         13.5          74.3       3.7X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Join w 2 ints:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Join w 2 ints wholestage off                     171821         171906         121          0.1        8193.0       1.0X
-Join w 2 ints wholestage on                      166559         166975         263          0.1        7942.1       1.0X
+Join w 2 ints wholestage off                     253807         254193         546          0.1       12102.4       1.0X
+Join w 2 ints wholestage on                      340317         342234         NaN          0.1       16227.6       0.7X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Join w 2 longs:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Join w 2 longs wholestage off                      7511           7555          62          2.8         358.2       1.0X
-Join w 2 longs wholestage on                       3776           4119         232          5.6         180.1       2.0X
+Join w 2 longs wholestage off                      8169           8222          76          2.6         389.5       1.0X
+Join w 2 longs wholestage on                       4078           4176          80          5.1         194.4       2.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Join w 2 longs duplicated:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Join w 2 longs duplicated wholestage off          13563          13617          77          1.5         646.7       1.0X
-Join w 2 longs duplicated wholestage on            7947           8053          71          2.6         378.9       1.7X
+Join w 2 longs duplicated wholestage off          17448          17625         251          1.2         832.0       1.0X
+Join w 2 longs duplicated wholestage on           10282          10407         106          2.0         490.3       1.7X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 outer join w long:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-outer join w long wholestage off                   3915           3923          12          5.4         186.7       1.0X
-outer join w long wholestage on                    1421           1461          30         14.8          67.8       2.8X
+outer join w long wholestage off                   3053           3102          70          6.9         145.6       1.0X
+outer join w long wholestage on                    1628           1683          71         12.9          77.6       1.9X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 semi join w long:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-semi join w long wholestage off                    2310           2332          30          9.1         110.2       1.0X
-semi join w long wholestage on                      835            860          34         25.1          39.8       2.8X
+semi join w long wholestage off                    1912           1917           6         11.0          91.2       1.0X
+semi join w long wholestage on                      960           1057          88         21.8          45.8       2.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 sort merge join:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-sort merge join wholestage off                     1846           1886          56          1.1         880.5       1.0X
-sort merge join wholestage on                      1402           1654         234          1.5         668.3       1.3X
+sort merge join wholestage off                     1587           1617          43          1.3         756.6       1.0X
+sort merge join wholestage on                      1358           1413          98          1.5         647.7       1.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-sort merge join with duplicates:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-sort merge join with duplicates wholestage off           2852           2879          38          0.7        1360.0       1.0X
-sort merge join with duplicates wholestage on           2645           2742         156          0.8        1261.0       1.1X
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+sort merge join with duplicates:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------------
+sort merge join with duplicates wholestage off           2232           2259          39          0.9        1064.1       1.0X
+sort merge join with duplicates wholestage on            1921           2030          99          1.1         916.1       1.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 shuffle hash join:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-shuffle hash join wholestage off                   1506           1564          82          2.8         359.1       1.0X
-shuffle hash join wholestage on                    1303           1330          23          3.2         310.6       1.2X
+shuffle hash join wholestage off                   1289           1333          62          3.3         307.4       1.0X
+shuffle hash join wholestage on                     813            879          54          5.2         193.9       1.6X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+broadcast nested loop join:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-------------------------------------------------------------------------------------------------------------------------
+broadcast nested loop join wholestage off          63164          63592         606          0.3        3011.9       1.0X
+broadcast nested loop join wholestage on           39833          40527         660          0.5        1899.4       1.6X
 
 
diff --git a/sql/core/benchmarks/JoinBenchmark-results.txt b/sql/core/benchmarks/JoinBenchmark-results.txt
index c19dd4f12bb32..3a11fa8fde8e4 100644
--- a/sql/core/benchmarks/JoinBenchmark-results.txt
+++ b/sql/core/benchmarks/JoinBenchmark-results.txt
@@ -2,74 +2,81 @@
 Join Benchmark
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Join w long:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Join w long wholestage off                         4531           4557          37          4.6         216.1       1.0X
-Join w long wholestage on                          1214           1310          95         17.3          57.9       3.7X
+Join w long wholestage off                         5228           5257          42          4.0         249.3       1.0X
+Join w long wholestage on                          1136           1177          38         18.5          54.1       4.6X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Join w long duplicated:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Join w long duplicated wholestage off              5200           5239          55          4.0         248.0       1.0X
-Join w long duplicated wholestage on               1535           1547          11         13.7          73.2       3.4X
+Join w long duplicated wholestage off              7318           7322           5          2.9         348.9       1.0X
+Join w long duplicated wholestage on               1421           1465          43         14.8          67.8       5.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Join w 2 ints:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Join w 2 ints wholestage off                     170776         170795          27          0.1        8143.2       1.0X
-Join w 2 ints wholestage on                      165134         165183          36          0.1        7874.2       1.0X
+Join w 2 ints wholestage off                     155491         155901         580          0.1        7414.4       1.0X
+Join w 2 ints wholestage on                      146448         147523         646          0.1        6983.2       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Join w 2 longs:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Join w 2 longs wholestage off                      6561           6595          48          3.2         312.8       1.0X
-Join w 2 longs wholestage on                       2999           3070          85          7.0         143.0       2.2X
+Join w 2 longs wholestage off                      7755           7755           0          2.7         369.8       1.0X
+Join w 2 longs wholestage on                       2668           2697          23          7.9         127.2       2.9X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Join w 2 longs duplicated:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Join w 2 longs duplicated wholestage off          15731          15757          38          1.3         750.1       1.0X
-Join w 2 longs duplicated wholestage on            8017           8112          80          2.6         382.3       2.0X
+Join w 2 longs duplicated wholestage off          20840          20879          56          1.0         993.7       1.0X
+Join w 2 longs duplicated wholestage on            8235           8376         163          2.5         392.7       2.5X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 outer join w long:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-outer join w long wholestage off                   3573           3577           6          5.9         170.4       1.0X
-outer join w long wholestage on                    1310           1325          15         16.0          62.5       2.7X
+outer join w long wholestage off                   2492           2529          52          8.4         118.8       1.0X
+outer join w long wholestage on                    1231           1275          27         17.0          58.7       2.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 semi join w long:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-semi join w long wholestage off                    1893           1916          33         11.1          90.3       1.0X
-semi join w long wholestage on                      819            842          30         25.6          39.0       2.3X
+semi join w long wholestage off                    1614           1636          30         13.0          77.0       1.0X
+semi join w long wholestage on                      712            735          35         29.5          33.9       2.3X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 sort merge join:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-sort merge join wholestage off                     1302           1312          13          1.6         620.9       1.0X
-sort merge join wholestage on                      1168           1233          62          1.8         557.0       1.1X
+sort merge join wholestage off                     1313           1313           1          1.6         625.9       1.0X
+sort merge join wholestage on                      1228           1254          21          1.7         585.5       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-sort merge join with duplicates:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-sort merge join with duplicates wholestage off           1996           2005          12          1.1         951.7       1.0X
-sort merge join with duplicates wholestage on           1766           1803          42          1.2         842.0       1.1X
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+sort merge join with duplicates:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------------
+sort merge join with duplicates wholestage off           1837           1840           4          1.1         876.2       1.0X
+sort merge join with duplicates wholestage on            1657           1686          32          1.3         790.1       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 shuffle hash join:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-shuffle hash join wholestage off                   1298           1300           3          3.2         309.6       1.0X
-shuffle hash join wholestage on                    1201           1210          10          3.5         286.4       1.1X
+shuffle hash join wholestage off                   1430           1438          12          2.9         340.8       1.0X
+shuffle hash join wholestage on                     612            628          13          6.9         146.0       2.3X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+broadcast nested loop join:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-------------------------------------------------------------------------------------------------------------------------
+broadcast nested loop join wholestage off          79031          79073          59          0.3        3768.5       1.0X
+broadcast nested loop join wholestage on           30409          30902         383          0.7        1450.0       2.6X
 
 
diff --git a/sql/core/benchmarks/JsonBenchmark-jdk11-results.txt b/sql/core/benchmarks/JsonBenchmark-jdk11-results.txt
index 03bc334471e56..e373d38237202 100644
--- a/sql/core/benchmarks/JsonBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/JsonBenchmark-jdk11-results.txt
@@ -3,110 +3,118 @@ Benchmark for performance of JSON parsing
 ================================================================================================
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 JSON schema inferring:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       46010          46118         113          2.2         460.1       1.0X
-UTF-8 is set                                      54407          55427        1718          1.8         544.1       0.8X
+No encoding                                        4180           4300         122          1.2         836.1       1.0X
+UTF-8 is set                                       5506           5566          70          0.9        1101.3       0.8X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 count a short column:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       26614          28220        1461          3.8         266.1       1.0X
-UTF-8 is set                                      42765          43400         550          2.3         427.6       0.6X
+No encoding                                        2878           2926          58          1.7         575.6       1.0X
+UTF-8 is set                                       4189           4239          43          1.2         837.8       0.7X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 count a wide column:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       35696          35821         113          0.3        3569.6       1.0X
-UTF-8 is set                                      55441          56176        1037          0.2        5544.1       0.6X
+No encoding                                        6729           6876         128          0.1        6728.7       1.0X
+UTF-8 is set                                      10313          10402         126          0.1       10312.6       0.7X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 select wide row:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       61514          62968         NaN          0.0      123027.2       1.0X
-UTF-8 is set                                      72096          72933        1162          0.0      144192.7       0.9X
+No encoding                                       15375          15551         201          0.0      307498.9       1.0X
+UTF-8 is set                                      18257          18476         190          0.0      365135.8       0.8X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Select a subset of 10 columns:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 10 columns                                  9859           9913          79          1.0         985.9       1.0X
-Select 1 column                                   10981          11003          36          0.9        1098.1       0.9X
+Select 10 columns                                  2664           2673          11          0.4        2664.2       1.0X
+Select 1 column                                    2335           2353          16          0.4        2335.3       1.1X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 creation of JSON parser per line:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Short column without encoding                      3555           3579          27          2.8         355.5       1.0X
-Short column with UTF-8                            5204           5227          35          1.9         520.4       0.7X
-Wide column without encoding                      60458          60637         164          0.2        6045.8       0.1X
-Wide column with UTF-8                            77544          78111         551          0.1        7754.4       0.0X
+Short column without encoding                       845            852           7          1.2         845.0       1.0X
+Short column with UTF-8                            1149           1161          12          0.9        1148.8       0.7X
+Wide column without encoding                       9971           9991          29          0.1        9971.1       0.1X
+Wide column with UTF-8                            14047          14059          14          0.1       14047.3       0.1X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 JSON functions:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                           342            346           3         29.2          34.2       1.0X
-from_json                                          7123           7318         179          1.4         712.3       0.0X
-json_tuple                                         9843           9957         132          1.0         984.3       0.0X
-get_json_object                                    7827           8046         194          1.3         782.7       0.0X
+Text read                                            90             91           1         11.1          90.4       1.0X
+from_json                                          2265           2291          25          0.4        2265.3       0.0X
+json_tuple                                         2585           2607          36          0.4        2584.7       0.0X
+get_json_object                                    2381           2388          10          0.4        2381.0       0.0X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Dataset of json strings:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                          1856           1884          32         26.9          37.1       1.0X
-schema inferring                                  16734          16900         153          3.0         334.7       0.1X
-parsing                                           14884          15203         470          3.4         297.7       0.1X
+Text read                                           397            399           2         12.6          79.4       1.0X
+schema inferring                                   3722           3770          43          1.3         744.4       0.1X
+parsing                                            3265           3282          21          1.5         653.0       0.1X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Json files in the per-line mode:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                          5932           6148         228          8.4         118.6       1.0X
-Schema inferring                                  20836          21938        1086          2.4         416.7       0.3X
-Parsing without charset                           18134          18661         457          2.8         362.7       0.3X
-Parsing with UTF-8                                27734          28069         378          1.8         554.7       0.2X
+Text read                                          1030           1037           9          4.9         206.0       1.0X
+Schema inferring                                   4515           4560          78          1.1         902.9       0.2X
+Parsing without charset                            3714           3772          64          1.3         742.7       0.3X
+Parsing with UTF-8                                 5370           5476          97          0.9        1074.1       0.2X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Write dates and timestamps:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Create a dataset of timestamps                      889            914          28         11.2          88.9       1.0X
-to_json(timestamp)                                 7920           8172         353          1.3         792.0       0.1X
-write timestamps to files                          6726           6822         129          1.5         672.6       0.1X
-Create a dataset of dates                           953            963          12         10.5          95.3       0.9X
-to_json(date)                                      5370           5705         320          1.9         537.0       0.2X
-write dates to files                               4109           4166          52          2.4         410.9       0.2X
+Create a dataset of timestamps                      174            178           5          5.7         174.4       1.0X
+to_json(timestamp)                                 1354           1368          12          0.7        1353.8       0.1X
+write timestamps to files                          1215           1226          16          0.8        1214.5       0.1X
+Create a dataset of dates                           184            188           5          5.4         184.0       0.9X
+to_json(date)                                       898            922          24          1.1         898.5       0.2X
+write dates to files                                708            716          10          1.4         708.1       0.2X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Read dates and timestamps:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-read timestamp text from files                     1614           1675          55          6.2         161.4       1.0X
-read timestamps from files                        16640          16858         209          0.6        1664.0       0.1X
-infer timestamps from files                       33239          33388         227          0.3        3323.9       0.0X
-read date text from files                          1310           1340          44          7.6         131.0       1.2X
-read date from files                               9470           9513          41          1.1         947.0       0.2X
-timestamp strings                                  1303           1342          47          7.7         130.3       1.2X
-parse timestamps from Dataset[String]             17650          18073         380          0.6        1765.0       0.1X
-infer timestamps from Dataset[String]             32623          34065        1330          0.3        3262.3       0.0X
-date strings                                       1864           1871           7          5.4         186.4       0.9X
-parse dates from Dataset[String]                  10914          11316         482          0.9        1091.4       0.1X
-from_json(timestamp)                              21102          21990         929          0.5        2110.2       0.1X
-from_json(date)                                   15275          15961         598          0.7        1527.5       0.1X
+read timestamp text from files                      265            285          23          3.8         265.0       1.0X
+read timestamps from files                         3107           3132          23          0.3        3107.1       0.1X
+infer timestamps from files                        6316           6365          43          0.2        6315.5       0.0X
+read date text from files                           241            259          19          4.2         240.6       1.1X
+read date from files                               1259           1278          20          0.8        1259.4       0.2X
+timestamp strings                                   290            293           4          3.4         290.3       0.9X
+parse timestamps from Dataset[String]              3324           3359          34          0.3        3324.4       0.1X
+infer timestamps from Dataset[String]              6868           6979         113          0.1        6867.7       0.0X
+date strings                                        380            384           7          2.6         379.6       0.7X
+parse dates from Dataset[String]                   1650           1672          20          0.6        1649.8       0.2X
+from_json(timestamp)                               4944           4969          33          0.2        4943.7       0.1X
+from_json(date)                                    3188           3251          57          0.3        3188.0       0.1X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+Filters pushdown:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+w/o filters                                       24601          24817         219          0.0      246012.5       1.0X
+pushdown disabled                                 24029          24183         137          0.0      240289.2       1.0X
+w/ filters                                          782            794          12          0.1        7822.7      31.4X
 
 
diff --git a/sql/core/benchmarks/JsonBenchmark-results.txt b/sql/core/benchmarks/JsonBenchmark-results.txt
index 0f188c4cdea56..0877f71ca7c62 100644
--- a/sql/core/benchmarks/JsonBenchmark-results.txt
+++ b/sql/core/benchmarks/JsonBenchmark-results.txt
@@ -3,110 +3,118 @@ Benchmark for performance of JSON parsing
 ================================================================================================
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 JSON schema inferring:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       38998          41002         NaN          2.6         390.0       1.0X
-UTF-8 is set                                      61231          63282        1854          1.6         612.3       0.6X
+No encoding                                        4500           4634         132          1.1         900.1       1.0X
+UTF-8 is set                                       6666           6700          53          0.8        1333.3       0.7X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 count a short column:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       28272          28338          70          3.5         282.7       1.0X
-UTF-8 is set                                      58681          62243        1517          1.7         586.8       0.5X
+No encoding                                        3231           3303          88          1.5         646.2       1.0X
+UTF-8 is set                                       5442           5513          72          0.9        1088.4       0.6X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 count a wide column:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       44026          51829        1329          0.2        4402.6       1.0X
-UTF-8 is set                                      65839          68596         500          0.2        6583.9       0.7X
+No encoding                                       11257          11271          14          0.1       11257.4       1.0X
+UTF-8 is set                                      13220          13469         296          0.1       13219.5       0.9X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 select wide row:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       72144          74820         NaN          0.0      144287.6       1.0X
-UTF-8 is set                                      69571          77888         NaN          0.0      139142.3       1.0X
+No encoding                                       16207          16359         168          0.0      324141.3       1.0X
+UTF-8 is set                                      17577          17811         214          0.0      351539.6       0.9X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Select a subset of 10 columns:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 10 columns                                  9502           9604         106          1.1         950.2       1.0X
-Select 1 column                                   11861          11948         109          0.8        1186.1       0.8X
+Select 10 columns                                  2596           2633          34          0.4        2596.3       1.0X
+Select 1 column                                    2730           2742          12          0.4        2729.6       1.0X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 creation of JSON parser per line:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Short column without encoding                      3830           3846          15          2.6         383.0       1.0X
-Short column with UTF-8                            5538           5543           7          1.8         553.8       0.7X
-Wide column without encoding                      66899          69158         NaN          0.1        6689.9       0.1X
-Wide column with UTF-8                            90052          93235         NaN          0.1        9005.2       0.0X
+Short column without encoding                      1089           1092           4          0.9        1088.9       1.0X
+Short column with UTF-8                            1410           1436          22          0.7        1410.3       0.8X
+Wide column without encoding                      16404          16572         184          0.1       16403.5       0.1X
+Wide column with UTF-8                            19200          19238          57          0.1       19200.0       0.1X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 JSON functions:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                           659            674          13         15.2          65.9       1.0X
-from_json                                          7676           7943         405          1.3         767.6       0.1X
-json_tuple                                         9881          10172         273          1.0         988.1       0.1X
-get_json_object                                    7949           8055         119          1.3         794.9       0.1X
+Text read                                           130            134           4          7.7         130.1       1.0X
+from_json                                          2863           2884          19          0.3        2862.7       0.0X
+json_tuple                                         3325           3374          44          0.3        3324.5       0.0X
+get_json_object                                    2892           2919          25          0.3        2892.2       0.0X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Dataset of json strings:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                          3314           3326          17         15.1          66.3       1.0X
-schema inferring                                  16549          17037         484          3.0         331.0       0.2X
-parsing                                           15138          15283         172          3.3         302.8       0.2X
+Text read                                           564            584          18          8.9         112.7       1.0X
+schema inferring                                   4592           4601          12          1.1         918.3       0.1X
+parsing                                            4032           4109         113          1.2         806.3       0.1X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Json files in the per-line mode:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                          5136           5446         268          9.7         102.7       1.0X
-Schema inferring                                  19864          20568        1191          2.5         397.3       0.3X
-Parsing without charset                           17535          17888         329          2.9         350.7       0.3X
-Parsing with UTF-8                                25609          25758         218          2.0         512.2       0.2X
+Text read                                          1267           1282          18          3.9         253.4       1.0X
+Schema inferring                                   5201           5277          73          1.0        1040.2       0.2X
+Parsing without charset                            5081           5140          64          1.0        1016.3       0.2X
+Parsing with UTF-8                                 6554           6632         116          0.8        1310.8       0.2X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Write dates and timestamps:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Create a dataset of timestamps                      784            790           7         12.8          78.4       1.0X
-to_json(timestamp)                                 8005           8055          50          1.2         800.5       0.1X
-write timestamps to files                          6515           6559          45          1.5         651.5       0.1X
-Create a dataset of dates                           854            881          24         11.7          85.4       0.9X
-to_json(date)                                      5187           5194           7          1.9         518.7       0.2X
-write dates to files                               3663           3684          22          2.7         366.3       0.2X
+Create a dataset of timestamps                      218            223           6          4.6         218.0       1.0X
+to_json(timestamp)                                 1722           1734          12          0.6        1722.5       0.1X
+write timestamps to files                          1490           1503          14          0.7        1489.6       0.1X
+Create a dataset of dates                           241            245           6          4.2         240.6       0.9X
+to_json(date)                                      1102           1128          24          0.9        1102.2       0.2X
+write dates to files                                900            927          24          1.1         899.6       0.2X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Read dates and timestamps:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-read timestamp text from files                     1297           1316          26          7.7         129.7       1.0X
-read timestamps from files                        16915          17723         963          0.6        1691.5       0.1X
-infer timestamps from files                       33967          34304         360          0.3        3396.7       0.0X
-read date text from files                          1095           1100           7          9.1         109.5       1.2X
-read date from files                               8376           8513         209          1.2         837.6       0.2X
-timestamp strings                                  1807           1816           8          5.5         180.7       0.7X
-parse timestamps from Dataset[String]             18189          18242          74          0.5        1818.9       0.1X
-infer timestamps from Dataset[String]             37906          38547         571          0.3        3790.6       0.0X
-date strings                                       2191           2194           4          4.6         219.1       0.6X
-parse dates from Dataset[String]                  11593          11625          33          0.9        1159.3       0.1X
-from_json(timestamp)                              22589          22650         101          0.4        2258.9       0.1X
-from_json(date)                                   16479          16619         159          0.6        1647.9       0.1X
+read timestamp text from files                      323            331           9          3.1         323.2       1.0X
+read timestamps from files                         3456           3465           9          0.3        3456.0       0.1X
+infer timestamps from files                        7237           7264          27          0.1        7237.2       0.0X
+read date text from files                           282            291           8          3.5         282.4       1.1X
+read date from files                               1762           1789          44          0.6        1761.6       0.2X
+timestamp strings                                   455            466          18          2.2         454.5       0.7X
+parse timestamps from Dataset[String]              3870           3937          99          0.3        3869.9       0.1X
+infer timestamps from Dataset[String]              7701           7739          36          0.1        7701.5       0.0X
+date strings                                        513            538          21          1.9         513.3       0.6X
+parse dates from Dataset[String]                   2102           2123          23          0.5        2102.3       0.2X
+from_json(timestamp)                               5846           5950          91          0.2        5846.0       0.1X
+from_json(date)                                    3988           4010          21          0.3        3988.1       0.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+Filters pushdown:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+w/o filters                                       25621          25700         113          0.0      256214.3       1.0X
+pushdown disabled                                 24732          24764          36          0.0      247315.8       1.0X
+w/ filters                                          827            842          13          0.1        8274.2      31.0X
 
 
diff --git a/sql/core/benchmarks/MakeDateTimeBenchmark-jdk11-results.txt b/sql/core/benchmarks/MakeDateTimeBenchmark-jdk11-results.txt
index 65faa752b94cb..c9288a260a751 100644
--- a/sql/core/benchmarks/MakeDateTimeBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/MakeDateTimeBenchmark-jdk11-results.txt
@@ -1,22 +1,22 @@
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 make_date():                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-prepare make_date()                                3204           3323         139         31.2          32.0       1.0X
-make_date(2019, 9, 16)                             2529           2604         126         39.5          25.3       1.3X
-make_date(*, *, *)                                 5102           5113          10         19.6          51.0       0.6X
+prepare make_date()                                4106           4124          28         24.4          41.1       1.0X
+make_date(2019, 9, 16)                             3151           3190          40         31.7          31.5       1.3X
+make_date(*, *, *)                                 4929           4967          44         20.3          49.3       0.8X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-make_timestamp():                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-prepare make_timestamp()                           3484           3513          28          0.3        3484.3       1.0X
-make_timestamp(2019, 1, 2, 3, 4, 50.123456)            112            131          17          9.0         111.5      31.2X
-make_timestamp(2019, 1, 2, 3, 4, 60.000000)             93            102          10         10.8          92.8      37.6X
-make_timestamp(2019, 12, 31, 23, 59, 60.00)             85             88           4         11.8          84.8      41.1X
-make_timestamp(*, *, *, 3, 4, 50.123456)            303            308           8          3.3         302.8      11.5X
-make_timestamp(*, *, *, *, *, 0)                    303            307           3          3.3         302.8      11.5X
-make_timestamp(*, *, *, *, *, 60.0)                 289            297           8          3.5         289.1      12.1X
-make_timestamp(2019, 1, 2, *, *, *)                3576           3585          11          0.3        3576.4       1.0X
-make_timestamp(*, *, *, *, *, *)                   3610           3618          12          0.3        3610.0       1.0X
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+make_timestamp():                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+---------------------------------------------------------------------------------------------------------------------------
+prepare make_timestamp()                              4039           4077          46          0.2        4038.9       1.0X
+make_timestamp(2019, 1, 2, 3, 4, 50.123456)             54             58           6         18.7          53.5      75.5X
+make_timestamp(2019, 1, 2, 3, 4, 60.000000)             51             57           6         19.8          50.6      79.9X
+make_timestamp(2019, 12, 31, 23, 59, 60.00)             50             51           1         19.9          50.4      80.2X
+make_timestamp(*, *, *, 3, 4, 50.123456)               294            309          17          3.4         294.4      13.7X
+make_timestamp(*, *, *, *, *, 0)                       264            291          28          3.8         264.0      15.3X
+make_timestamp(*, *, *, *, *, 60.0)                    272            293          26          3.7         272.1      14.8X
+make_timestamp(2019, 1, 2, *, *, *)                   4472           4498          24          0.2        4471.6       0.9X
+make_timestamp(*, *, *, *, *, *)                      4306           4324          29          0.2        4305.9       0.9X
 
diff --git a/sql/core/benchmarks/MakeDateTimeBenchmark-results.txt b/sql/core/benchmarks/MakeDateTimeBenchmark-results.txt
index 92bcc4444e60a..e68c316246771 100644
--- a/sql/core/benchmarks/MakeDateTimeBenchmark-results.txt
+++ b/sql/core/benchmarks/MakeDateTimeBenchmark-results.txt
@@ -1,22 +1,22 @@
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 make_date():                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-prepare make_date()                                2920           3239         278         34.3          29.2       1.0X
-make_date(2019, 9, 16)                             2322           2371          61         43.1          23.2       1.3X
-make_date(*, *, *)                                 4553           4560           6         22.0          45.5       0.6X
+prepare make_date()                                2468           2609         192         40.5          24.7       1.0X
+make_date(2019, 9, 16)                             2260           2298          44         44.2          22.6       1.1X
+make_date(*, *, *)                                 4163           4247          74         24.0          41.6       0.6X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-make_timestamp():                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-prepare make_timestamp()                           3636           3673          38          0.3        3635.7       1.0X
-make_timestamp(2019, 1, 2, 3, 4, 50.123456)             94             99           4         10.7          93.8      38.8X
-make_timestamp(2019, 1, 2, 3, 4, 60.000000)             68             80          13         14.6          68.3      53.2X
-make_timestamp(2019, 12, 31, 23, 59, 60.00)             65             79          19         15.3          65.3      55.7X
-make_timestamp(*, *, *, 3, 4, 50.123456)            271            280          14          3.7         270.7      13.4X
-make_timestamp(*, *, *, *, *, 0)                    255            263          11          3.9         255.5      14.2X
-make_timestamp(*, *, *, *, *, 60.0)                 254            258           4          3.9         254.2      14.3X
-make_timestamp(2019, 1, 2, *, *, *)                3714           3722           8          0.3        3713.9       1.0X
-make_timestamp(*, *, *, *, *, *)                   3736           3741           6          0.3        3736.3       1.0X
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+make_timestamp():                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+---------------------------------------------------------------------------------------------------------------------------
+prepare make_timestamp()                              3667           3722          54          0.3        3667.0       1.0X
+make_timestamp(2019, 1, 2, 3, 4, 50.123456)             44             44           0         22.9          43.6      84.0X
+make_timestamp(2019, 1, 2, 3, 4, 60.000000)             41             42           1         24.3          41.1      89.1X
+make_timestamp(2019, 12, 31, 23, 59, 60.00)             41             43           3         24.6          40.7      90.0X
+make_timestamp(*, *, *, 3, 4, 50.123456)               244            247           2          4.1         244.4      15.0X
+make_timestamp(*, *, *, *, *, 0)                       241            248           7          4.2         240.6      15.2X
+make_timestamp(*, *, *, *, *, 60.0)                    239            245           6          4.2         239.5      15.3X
+make_timestamp(2019, 1, 2, *, *, *)                   3894           3944          58          0.3        3893.6       0.9X
+make_timestamp(*, *, *, *, *, *)                      3952           3976          38          0.3        3951.8       0.9X
 
diff --git a/sql/core/benchmarks/MetricsAggregationBenchmark-jdk11-results.txt b/sql/core/benchmarks/MetricsAggregationBenchmark-jdk11-results.txt
index e33ed30eaa559..70451fe2e1fed 100644
--- a/sql/core/benchmarks/MetricsAggregationBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/MetricsAggregationBenchmark-jdk11-results.txt
@@ -1,12 +1,12 @@
-OpenJDK 64-Bit Server VM 11.0.4+11 on Linux 4.15.0-66-generic
-Intel(R) Core(TM) i7-6820HQ CPU @ 2.70GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 metrics aggregation (50 metrics, 100000 tasks per stage):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-1 stage(s)                                          672            841         179          0.0   671888474.0       1.0X
-2 stage(s)                                         1700           1842         201          0.0  1699591662.0       0.4X
-3 stage(s)                                         2601           2776         247          0.0  2601465786.0       0.3X
+----------------------------------------------------------------------------------------------------------------------------------------
+1 stage(s)                                                         3172           3298         179          0.0  3171630175.0       1.0X
+2 stage(s)                                                         6668           6766         140          0.0  6667562027.0       0.5X
+3 stage(s)                                                        10079          10517         619          0.0 10078844686.0       0.3X
 
 Stage Count    Stage Proc. Time    Aggreg. Time
-     1              436                164
-     2              537                354
-     3              480                602
+     1              1710                331
+     2              1832                871
+     3              1775                1472
diff --git a/sql/core/benchmarks/MetricsAggregationBenchmark-results.txt b/sql/core/benchmarks/MetricsAggregationBenchmark-results.txt
index 4fae928258d32..9c5b8b4027bf2 100644
--- a/sql/core/benchmarks/MetricsAggregationBenchmark-results.txt
+++ b/sql/core/benchmarks/MetricsAggregationBenchmark-results.txt
@@ -1,12 +1,12 @@
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_181-b13 on Linux 4.15.0-66-generic
-Intel(R) Core(TM) i7-6820HQ CPU @ 2.70GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 metrics aggregation (50 metrics, 100000 tasks per stage):  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-1 stage(s)                                          740            883         147          0.0   740089816.0       1.0X
-2 stage(s)                                         1661           1943         399          0.0  1660649192.0       0.4X
-3 stage(s)                                         2711           2967         362          0.0  2711110178.0       0.3X
+----------------------------------------------------------------------------------------------------------------------------------------
+1 stage(s)                                                         3137           3280         203          0.0  3136824845.0       1.0X
+2 stage(s)                                                         6420           6826         574          0.0  6419861899.0       0.5X
+3 stage(s)                                                        11018          11209         271          0.0 11017607749.0       0.3X
 
 Stage Count    Stage Proc. Time    Aggreg. Time
-     1              405                179
-     2              375                414
-     3              364                644
+     1              2513                326
+     2              2150                1048
+     3              2136                1583
diff --git a/sql/core/benchmarks/MiscBenchmark-jdk11-results.txt b/sql/core/benchmarks/MiscBenchmark-jdk11-results.txt
index 281b0fe28171b..edfbb7ff363a9 100644
--- a/sql/core/benchmarks/MiscBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/MiscBenchmark-jdk11-results.txt
@@ -2,126 +2,126 @@
 filter & aggregate without group
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 range/filter/sum:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-range/filter/sum wholestage off                   54616          54834         309         38.4          26.0       1.0X
-range/filter/sum wholestage on                     3263           3369         129        642.6           1.6      16.7X
+range/filter/sum wholestage off                  116300         116850         778         18.0          55.5       1.0X
+range/filter/sum wholestage on                     4137           4321         381        507.0           2.0      28.1X
 
 
 ================================================================================================
 range/limit/sum
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 range/limit/sum:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-range/limit/sum wholestage off                      239            265          36       2190.0           0.5       1.0X
-range/limit/sum wholestage on                       137            162          15       3814.6           0.3       1.7X
+range/limit/sum wholestage off                      198            200           3       2642.7           0.4       1.0X
+range/limit/sum wholestage on                        46             66          36      11475.6           0.1       4.3X
 
 
 ================================================================================================
 sample
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 sample with replacement:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-sample with replacement wholestage off            13093          13417         458         10.0          99.9       1.0X
-sample with replacement wholestage on              7606           7624          13         17.2          58.0       1.7X
+sample with replacement wholestage off            17080          17302         314          7.7         130.3       1.0X
+sample with replacement wholestage on              8033           8188         101         16.3          61.3       2.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-sample without replacement:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-sample without replacement wholestage off           3031           3038           9         43.2          23.1       1.0X
-sample without replacement wholestage on           1156           1177          16        113.4           8.8       2.6X
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+sample without replacement:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-------------------------------------------------------------------------------------------------------------------------
+sample without replacement wholestage off           5087           5115          38         25.8          38.8       1.0X
+sample without replacement wholestage on            1064           1084          25        123.2           8.1       4.8X
 
 
 ================================================================================================
 collect
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 collect:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-collect 1 million                                   335            360          20          3.1         319.0       1.0X
-collect 2 millions                                  579            633          42          1.8         552.2       0.6X
-collect 4 millions                                 1192           1331         196          0.9        1137.1       0.3X
+collect 1 million                                   391            462          85          2.7         372.8       1.0X
+collect 2 millions                                  789            910         130          1.3         752.1       0.5X
+collect 4 millions                                 1677           1729          75          0.6        1599.0       0.2X
 
 
 ================================================================================================
 collect limit
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 collect limit:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-collect limit 1 million                             378            386          12          2.8         360.8       1.0X
-collect limit 2 millions                            724            733          12          1.4         690.3       0.5X
+collect limit 1 million                             507            545          28          2.1         483.4       1.0X
+collect limit 2 millions                            854           1003         212          1.2         814.1       0.6X
 
 
 ================================================================================================
 generate explode
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 generate explode array:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-generate explode array wholestage off             15880          16159         395          1.1         946.5       1.0X
-generate explode array wholestage on              15690          15783          73          1.1         935.2       1.0X
+generate explode array wholestage off             20533          20699         235          0.8        1223.9       1.0X
+generate explode array wholestage on               5886           6012         109          2.9         350.8       3.5X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 generate explode map:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-generate explode map wholestage off               51293          51311          26          0.3        3057.3       1.0X
-generate explode map wholestage on                50409          50795         388          0.3        3004.6       1.0X
+generate explode map wholestage off               62142          63593        2052          0.3        3703.9       1.0X
+generate explode map wholestage on                38943          39253         439          0.4        2321.2       1.6X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 generate posexplode array:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-generate posexplode array wholestage off          17246          17860         869          1.0        1027.9       1.0X
-generate posexplode array wholestage on           17344          17472          88          1.0        1033.8       1.0X
+generate posexplode array wholestage off          26931          27169         335          0.6        1605.2       1.0X
+generate posexplode array wholestage on            6003           6107          64          2.8         357.8       4.5X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 generate inline array:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-generate inline array wholestage off              13698          13790         130          1.2         816.5       1.0X
-generate inline array wholestage on               12995          13033          38          1.3         774.6       1.1X
+generate inline array wholestage off              19043          19081          55          0.9        1135.0       1.0X
+generate inline array wholestage on                4648           4778         158          3.6         277.0       4.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 generate big struct array:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-generate big struct array wholestage off            476            489          18          0.1        7938.6       1.0X
-generate big struct array wholestage on             424            440          14          0.1        7074.9       1.1X
+generate big struct array wholestage off            486            496          15          0.1        8094.8       1.0X
+generate big struct array wholestage on             396            420          22          0.2        6603.2       1.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-generate big nested struct array:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-generate big nested struct array wholestage off            413            433          28          0.1        6886.8       1.0X
-generate big nested struct array wholestage on            394            420          39          0.2        6560.9       1.0X
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+generate big nested struct array:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-------------------------------------------------------------------------------------------------------------------------------
+generate big nested struct array wholestage off         327380         353982         NaN          0.0     5456328.8       1.0X
+generate big nested struct array wholestage on             391            427          66          0.2        6524.4     836.3X
 
 
 ================================================================================================
 generate regular generator
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 generate stack:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-generate stack wholestage off                     26494          26510          23          0.6        1579.2       1.0X
-generate stack wholestage on                      24205          24339          80          0.7        1442.8       1.1X
+generate stack wholestage off                     31236          31274          54          0.5        1861.8       1.0X
+generate stack wholestage on                       6420           6496         113          2.6         382.7       4.9X
 
 
diff --git a/sql/core/benchmarks/MiscBenchmark-results.txt b/sql/core/benchmarks/MiscBenchmark-results.txt
index 8b1e728ed9cf9..85993a5ba6829 100644
--- a/sql/core/benchmarks/MiscBenchmark-results.txt
+++ b/sql/core/benchmarks/MiscBenchmark-results.txt
@@ -2,126 +2,126 @@
 filter & aggregate without group
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 range/filter/sum:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-range/filter/sum wholestage off                   46812          48110        1836         44.8          22.3       1.0X
-range/filter/sum wholestage on                     3116           3656         309        673.1           1.5      15.0X
+range/filter/sum wholestage off                   98011          99011        1415         21.4          46.7       1.0X
+range/filter/sum wholestage on                     3287           3614         183        638.1           1.6      29.8X
 
 
 ================================================================================================
 range/limit/sum
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 range/limit/sum:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-range/limit/sum wholestage off                      200            219          27       2621.6           0.4       1.0X
-range/limit/sum wholestage on                       117            125           7       4477.8           0.2       1.7X
+range/limit/sum wholestage off                       93             96           4       5625.2           0.2       1.0X
+range/limit/sum wholestage on                        42             49           9      12366.6           0.1       2.2X
 
 
 ================================================================================================
 sample
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 sample with replacement:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-sample with replacement wholestage off            12963          13506         768         10.1          98.9       1.0X
-sample with replacement wholestage on              7397           7742         300         17.7          56.4       1.8X
+sample with replacement wholestage off            13712          13721          13          9.6         104.6       1.0X
+sample with replacement wholestage on              7532           7561          47         17.4          57.5       1.8X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-sample without replacement:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-sample without replacement wholestage off           2977           2977           0         44.0          22.7       1.0X
-sample without replacement wholestage on           1087           1099          13        120.6           8.3       2.7X
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+sample without replacement:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-------------------------------------------------------------------------------------------------------------------------
+sample without replacement wholestage off           4652           4652           1         28.2          35.5       1.0X
+sample without replacement wholestage on            1065           1096          56        123.1           8.1       4.4X
 
 
 ================================================================================================
 collect
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 collect:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-collect 1 million                                   281            315          34          3.7         268.2       1.0X
-collect 2 millions                                  531            564          28          2.0         506.6       0.5X
-collect 4 millions                                 1179           1970        1118          0.9        1124.5       0.2X
+collect 1 million                                   249            253           3          4.2         237.8       1.0X
+collect 2 millions                                  482            500          28          2.2         459.2       0.5X
+collect 4 millions                                  989           1016          38          1.1         943.5       0.3X
 
 
 ================================================================================================
 collect limit
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 collect limit:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-collect limit 1 million                             344            352           5          3.0         328.0       1.0X
-collect limit 2 millions                            656            660           6          1.6         625.2       0.5X
+collect limit 1 million                             297            303           4          3.5         283.5       1.0X
+collect limit 2 millions                            582            593          14          1.8         554.6       0.5X
 
 
 ================================================================================================
 generate explode
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 generate explode array:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-generate explode array wholestage off             14664          14780         164          1.1         874.0       1.0X
-generate explode array wholestage on              14789          14886          87          1.1         881.5       1.0X
+generate explode array wholestage off             17458          17507          70          1.0        1040.6       1.0X
+generate explode array wholestage on               6094           6178          93          2.8         363.2       2.9X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 generate explode map:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-generate explode map wholestage off               50441          51014         811          0.3        3006.5       1.0X
-generate explode map wholestage on                49164          49915         892          0.3        2930.4       1.0X
+generate explode map wholestage off               51077          53221        3032          0.3        3044.4       1.0X
+generate explode map wholestage on                30130          30343         200          0.6        1795.9       1.7X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 generate posexplode array:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-generate posexplode array wholestage off          17101          17130          40          1.0        1019.3       1.0X
-generate posexplode array wholestage on           15625          15675          46          1.1         931.3       1.1X
+generate posexplode array wholestage off          18582          18702         170          0.9        1107.6       1.0X
+generate posexplode array wholestage on            6164           6208          32          2.7         367.4       3.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 generate inline array:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-generate inline array wholestage off              13923          14720        1127          1.2         829.9       1.0X
-generate inline array wholestage on               12246          12591         695          1.4         729.9       1.1X
+generate inline array wholestage off              13275          14431        1635          1.3         791.2       1.0X
+generate inline array wholestage on                4668           4752         118          3.6         278.3       2.8X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 generate big struct array:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-generate big struct array wholestage off            442            456          20          0.1        7368.2       1.0X
-generate big struct array wholestage on             409            429          22          0.1        6823.9       1.1X
+generate big struct array wholestage off            382            385           4          0.2        6367.5       1.0X
+generate big struct array wholestage on             327            349          42          0.2        5446.1       1.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-generate big nested struct array:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-generate big nested struct array wholestage off            409            415           8          0.1        6822.4       1.0X
-generate big nested struct array wholestage on            401            414          23          0.1        6687.0       1.0X
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+generate big nested struct array:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-------------------------------------------------------------------------------------------------------------------------------
+generate big nested struct array wholestage off          45694          45856         229          0.0      761565.1       1.0X
+generate big nested struct array wholestage on             338            356          35          0.2        5631.4     135.2X
 
 
 ================================================================================================
 generate regular generator
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 generate stack:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-generate stack wholestage off                     27043          27118         106          0.6        1611.9       1.0X
-generate stack wholestage on                      24002          24036          34          0.7        1430.6       1.1X
+generate stack wholestage off                     26624          26685          87          0.6        1586.9       1.0X
+generate stack wholestage on                       5948           5971          25          2.8         354.5       4.5X
 
 
diff --git a/sql/core/benchmarks/OrcNestedSchemaPruningBenchmark-jdk11-results.txt b/sql/core/benchmarks/OrcNestedSchemaPruningBenchmark-jdk11-results.txt
index c286dc2da8076..c736d5e0d5089 100644
--- a/sql/core/benchmarks/OrcNestedSchemaPruningBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/OrcNestedSchemaPruningBenchmark-jdk11-results.txt
@@ -2,52 +2,52 @@
 Nested Schema Pruning Benchmark For ORC v1
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Selection:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    155            197          23          6.5         155.0       1.0X
-Nested column                                      1290           1338          43          0.8        1289.9       0.1X
-Nested column in array                             5914           5989          57          0.2        5914.1       0.0X
+Top-level column                                     82            129          39         12.2          82.2       1.0X
+Nested column                                       462            490          23          2.2         461.7       0.2X
+Nested column in array                             2051           2140          74          0.5        2051.2       0.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Limiting:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    463            537          53          2.2         463.2       1.0X
-Nested column                                      1807           1874          62          0.6        1807.3       0.3X
-Nested column in array                             6729           6906         116          0.1        6729.4       0.1X
+Top-level column                                    387            418          36          2.6         387.2       1.0X
+Nested column                                       702            754          41          1.4         702.3       0.6X
+Nested column in array                             2320           2511         108          0.4        2320.4       0.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Repartitioning:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    378            402          18          2.6         377.7       1.0X
-Nested column                                      1765           1794          46          0.6        1765.4       0.2X
-Nested column in array                             6675           6718          42          0.1        6674.6       0.1X
+Top-level column                                    314            363          46          3.2         313.6       1.0X
+Nested column                                       663            712          25          1.5         663.2       0.5X
+Nested column in array                             2174           2278          65          0.5        2174.3       0.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Repartitioning by exprs:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    384            399           8          2.6         383.7       1.0X
-Nested column                                      4724           4820          72          0.2        4724.3       0.1X
-Nested column in array                             9256           9405         133          0.1        9255.7       0.0X
+Top-level column                                    271            312          45          3.7         270.7       1.0X
+Nested column                                       664            695          30          1.5         663.6       0.4X
+Nested column in array                             2228           2331          88          0.4        2227.8       0.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Sample:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    121            137          13          8.3         120.7       1.0X
-Nested column                                      1303           1361          45          0.8        1303.2       0.1X
-Nested column in array                             5901           5978          49          0.2        5901.0       0.0X
+Top-level column                                     99            138          57         10.1          99.5       1.0X
+Nested column                                       395            424          28          2.5         395.4       0.3X
+Nested column in array                             1734           1779          36          0.6        1733.6       0.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Sorting:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    651            675          30          1.5         650.8       1.0X
-Nested column                                      5398           5527         179          0.2        5397.9       0.1X
-Nested column in array                            10413          10685         310          0.1       10413.1       0.1X
+Top-level column                                    458            521          55          2.2         458.0       1.0X
+Nested column                                       818            879          43          1.2         817.5       0.6X
+Nested column in array                             2388           2528          95          0.4        2388.0       0.2X
 
 
diff --git a/sql/core/benchmarks/OrcNestedSchemaPruningBenchmark-results.txt b/sql/core/benchmarks/OrcNestedSchemaPruningBenchmark-results.txt
index 0cbd855ec5e3a..3aae82770da72 100644
--- a/sql/core/benchmarks/OrcNestedSchemaPruningBenchmark-results.txt
+++ b/sql/core/benchmarks/OrcNestedSchemaPruningBenchmark-results.txt
@@ -2,52 +2,52 @@
 Nested Schema Pruning Benchmark For ORC v1
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Selection:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    116            148          22          8.6         115.9       1.0X
-Nested column                                      1200           1221          23          0.8        1200.3       0.1X
-Nested column in array                             4786           4898         202          0.2        4785.6       0.0X
+Top-level column                                     65             76          10         15.4          64.9       1.0X
+Nested column                                       353            368          11          2.8         353.0       0.2X
+Nested column in array                             1630           1665          28          0.6        1629.8       0.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Limiting:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    428            452          18          2.3         427.7       1.0X
-Nested column                                      1681           1725          53          0.6        1680.7       0.3X
-Nested column in array                             5652           5699          47          0.2        5651.9       0.1X
+Top-level column                                    306            332          11          3.3         306.3       1.0X
+Nested column                                       588            629          33          1.7         587.9       0.5X
+Nested column in array                             2064           2110          36          0.5        2064.4       0.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Repartitioning:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    341            351          15          2.9         340.9       1.0X
-Nested column                                      1529           1559          31          0.7        1528.6       0.2X
-Nested column in array                             5468           5517          97          0.2        5468.2       0.1X
+Top-level column                                    247            261           7          4.0         247.0       1.0X
+Nested column                                       550            577          16          1.8         549.9       0.4X
+Nested column in array                             1987           2012          23          0.5        1987.4       0.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Repartitioning by exprs:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    353            365          12          2.8         352.5       1.0X
-Nested column                                      4135           4171          23          0.2        4135.2       0.1X
-Nested column in array                             8766           8816          56          0.1        8766.0       0.0X
+Top-level column                                    260            269           8          3.8         259.7       1.0X
+Nested column                                       664            682          17          1.5         663.6       0.4X
+Nested column in array                             2177           2287          71          0.5        2176.6       0.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Sample:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    115            133          25          8.7         115.3       1.0X
-Nested column                                      1200           1216          13          0.8        1199.7       0.1X
-Nested column in array                             5296           5345          39          0.2        5296.3       0.0X
+Top-level column                                     80            101          26         12.5          80.3       1.0X
+Nested column                                       356            374          21          2.8         356.5       0.2X
+Nested column in array                             1611           1679          39          0.6        1611.0       0.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Sorting:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    545            560          14          1.8         544.7       1.0X
-Nested column                                      4704           4813         184          0.2        4703.8       0.1X
-Nested column in array                             9785           9824          42          0.1        9784.8       0.1X
+Top-level column                                    375            396          12          2.7         375.2       1.0X
+Nested column                                       738            763          12          1.4         738.1       0.5X
+Nested column in array                             2302           2444         129          0.4        2302.2       0.2X
 
 
diff --git a/sql/core/benchmarks/OrcV2NestedSchemaPruningBenchmark-jdk11-results.txt b/sql/core/benchmarks/OrcV2NestedSchemaPruningBenchmark-jdk11-results.txt
index 31970672aacd4..753674ea9ded8 100644
--- a/sql/core/benchmarks/OrcV2NestedSchemaPruningBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/OrcV2NestedSchemaPruningBenchmark-jdk11-results.txt
@@ -2,52 +2,52 @@
 Nested Schema Pruning Benchmark For ORC v2
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Selection:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    176            207          20          5.7         176.2       1.0X
-Nested column                                      1173           1212          61          0.9        1173.2       0.2X
-Nested column in array                             5557           5628          58          0.2        5557.5       0.0X
+Top-level column                                     69             74           6         14.4          69.3       1.0X
+Nested column                                       653            665          11          1.5         652.5       0.1X
+Nested column in array                             3065           3085          22          0.3        3064.7       0.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Limiting:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    143            178          21          7.0         143.5       1.0X
-Nested column                                      1233           1272          31          0.8        1233.0       0.1X
-Nested column in array                             5585           5691          66          0.2        5585.4       0.0X
+Top-level column                                    382            435          53          2.6         382.4       1.0X
+Nested column                                       970           1006          41          1.0         970.3       0.4X
+Nested column in array                             3452           3551          85          0.3        3452.3       0.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Repartitioning:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    395            416          23          2.5         395.4       1.0X
-Nested column                                      1665           1691          16          0.6        1664.7       0.2X
-Nested column in array                             6194           6307         183          0.2        6194.5       0.1X
+Top-level column                                    324            360          40          3.1         323.7       1.0X
+Nested column                                       896            910           8          1.1         896.4       0.4X
+Nested column in array                             3395           3407          11          0.3        3395.5       0.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Repartitioning by exprs:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    394            407          14          2.5         393.7       1.0X
-Nested column                                      4381           4536          97          0.2        4380.6       0.1X
-Nested column in array                             8797           8833          34          0.1        8796.7       0.0X
+Top-level column                                    318            355          42          3.1         318.1       1.0X
+Nested column                                       993           1006          10          1.0         993.3       0.3X
+Nested column in array                             3607           3624          16          0.3        3607.0       0.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Sample:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    136            170          24          7.4         135.6       1.0X
-Nested column                                      1135           1171          30          0.9        1135.0       0.1X
-Nested column in array                             4833           4911          92          0.2        4833.3       0.0X
+Top-level column                                     98            114          27         10.2          97.8       1.0X
+Nested column                                       689            702          12          1.5         688.9       0.1X
+Nested column in array                             3004           3090          55          0.3        3004.5       0.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Sorting:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    327            336           8          3.1         327.0       1.0X
-Nested column                                      3321           3377          56          0.3        3321.0       0.1X
-Nested column in array                             8080           8161          97          0.1        8080.4       0.0X
+Top-level column                                    582            669          59          1.7         582.2       1.0X
+Nested column                                      1265           1300          51          0.8        1264.9       0.5X
+Nested column in array                             4249           4308         129          0.2        4248.7       0.1X
 
 
diff --git a/sql/core/benchmarks/OrcV2NestedSchemaPruningBenchmark-results.txt b/sql/core/benchmarks/OrcV2NestedSchemaPruningBenchmark-results.txt
index c7d8c8a541b59..ae9ac2a89705b 100644
--- a/sql/core/benchmarks/OrcV2NestedSchemaPruningBenchmark-results.txt
+++ b/sql/core/benchmarks/OrcV2NestedSchemaPruningBenchmark-results.txt
@@ -2,52 +2,52 @@
 Nested Schema Pruning Benchmark For ORC v2
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Selection:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    121            156          27          8.3         121.1       1.0X
-Nested column                                      1373           1406          37          0.7        1373.4       0.1X
-Nested column in array                             5545           5579          54          0.2        5544.8       0.0X
+Top-level column                                     51             59           9         19.5          51.2       1.0X
+Nested column                                       472            492          15          2.1         471.7       0.1X
+Nested column in array                             2371           2418          44          0.4        2370.9       0.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Limiting:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    127            147          20          7.9         127.0       1.0X
-Nested column                                      1280           1328          32          0.8        1280.2       0.1X
-Nested column in array                             5617           5696          70          0.2        5617.0       0.0X
+Top-level column                                    303            346          47          3.3         302.8       1.0X
+Nested column                                      1015           1163         136          1.0        1014.8       0.3X
+Nested column in array                             2867           2940         114          0.3        2866.9       0.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Repartitioning:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    343            356          17          2.9         342.6       1.0X
-Nested column                                      1692           1710          14          0.6        1692.3       0.2X
-Nested column in array                             6128           6168          30          0.2        6128.0       0.1X
+Top-level column                                    255            265           5          3.9         255.4       1.0X
+Nested column                                      1026           1047          17          1.0        1026.1       0.2X
+Nested column in array                             2760           2813          37          0.4        2760.2       0.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Repartitioning by exprs:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    348            355          11          2.9         348.1       1.0X
-Nested column                                      4350           4392          35          0.2        4349.8       0.1X
-Nested column in array                             8864           8901          29          0.1        8864.1       0.0X
+Top-level column                                    245            258           8          4.1         245.2       1.0X
+Nested column                                      1085           1124          35          0.9        1084.8       0.2X
+Nested column in array                             2945           2993          36          0.3        2944.9       0.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Sample:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    123            143          27          8.2         122.5       1.0X
-Nested column                                      1233           1295          29          0.8        1233.2       0.1X
-Nested column in array                             5534           5597          53          0.2        5533.7       0.0X
+Top-level column                                     76             86          13         13.1          76.2       1.0X
+Nested column                                       780            824          40          1.3         779.5       0.1X
+Nested column in array                             2450           2530          68          0.4        2449.9       0.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Sorting:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    265            280          20          3.8         264.8       1.0X
-Nested column                                      3211           3263          96          0.3        3211.2       0.1X
-Nested column in array                             8324           8357          42          0.1        8323.6       0.0X
+Top-level column                                    396            425          26          2.5         395.6       1.0X
+Nested column                                      1203           1255          51          0.8        1202.7       0.3X
+Nested column in array                             3077           3159          44          0.3        3076.7       0.1X
 
 
diff --git a/sql/core/benchmarks/ParquetNestedPredicatePushDownBenchmark-jdk11-results.txt b/sql/core/benchmarks/ParquetNestedPredicatePushDownBenchmark-jdk11-results.txt
index c33f8a372d82b..e2e9681fd96b2 100644
--- a/sql/core/benchmarks/ParquetNestedPredicatePushDownBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/ParquetNestedPredicatePushDownBenchmark-jdk11-results.txt
@@ -1,21 +1,21 @@
-OpenJDK 64-Bit Server VM 11.0.2+9 on Mac OS X 10.14.6
-Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Can skip all row groups:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Without nested predicate Pushdown                 34214          35752         NaN          3.1         326.3       1.0X
-With nested predicate Pushdown                       86            102          11       1216.2           0.8     396.8X
+Without nested predicate Pushdown                 51192          53164         953          2.0         488.2       1.0X
+With nested predicate Pushdown                       73            117          48       1443.5           0.7     704.7X
 
-OpenJDK 64-Bit Server VM 11.0.2+9 on Mac OS X 10.14.6
-Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Can skip some row groups:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Without nested predicate Pushdown                 34211          35162         843          3.1         326.3       1.0X
-With nested predicate Pushdown                     3470           3514          36         30.2          33.1       9.9X
+Without nested predicate Pushdown                 53589          55125         932          2.0         511.1       1.0X
+With nested predicate Pushdown                       87            138          36       1201.9           0.8     614.2X
 
-OpenJDK 64-Bit Server VM 11.0.2+9 on Mac OS X 10.14.6
-Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Can skip no row groups:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Without nested predicate Pushdown                 37533          37919         329          2.8         357.9       1.0X
-With nested predicate Pushdown                    37876          39132         536          2.8         361.2       1.0X
+Without nested predicate Pushdown                 63303          65061         NaN          1.7         603.7       1.0X
+With nested predicate Pushdown                    63738          64255         427          1.6         607.8       1.0X
 
diff --git a/sql/core/benchmarks/ParquetNestedPredicatePushDownBenchmark-results.txt b/sql/core/benchmarks/ParquetNestedPredicatePushDownBenchmark-results.txt
index 35dd4f0a5e9cb..1150bc0040f3b 100644
--- a/sql/core/benchmarks/ParquetNestedPredicatePushDownBenchmark-results.txt
+++ b/sql/core/benchmarks/ParquetNestedPredicatePushDownBenchmark-results.txt
@@ -1,21 +1,21 @@
-OpenJDK 64-Bit Server VM 1.8.0_252-b09 on Mac OS X 10.14.6
-Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 Can skip all row groups:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Without nested predicate Pushdown                 30687          31552         NaN          3.4         292.7       1.0X
-With nested predicate Pushdown                      105            150          61        999.3           1.0     292.5X
+Without nested predicate Pushdown                 41268          41398          78          2.5         393.6       1.0X
+With nested predicate Pushdown                       68             81          10       1532.2           0.7     603.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-b09 on Mac OS X 10.14.6
-Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 Can skip some row groups:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Without nested predicate Pushdown                 30505          31828         NaN          3.4         290.9       1.0X
-With nested predicate Pushdown                     3156           3215          77         33.2          30.1       9.7X
+Without nested predicate Pushdown                 41173          41299          80          2.5         392.7       1.0X
+With nested predicate Pushdown                       75             82           9       1404.3           0.7     551.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_252-b09 on Mac OS X 10.14.6
-Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 Can skip no row groups:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Without nested predicate Pushdown                 34475          35302         NaN          3.0         328.8       1.0X
-With nested predicate Pushdown                    34003          34596         567          3.1         324.3       1.0X
+Without nested predicate Pushdown                 46265          46390          91          2.3         441.2       1.0X
+With nested predicate Pushdown                    46263          46383         111          2.3         441.2       1.0X
 
diff --git a/sql/core/benchmarks/ParquetNestedSchemaPruningBenchmark-jdk11-results.txt b/sql/core/benchmarks/ParquetNestedSchemaPruningBenchmark-jdk11-results.txt
index f6135968bb97b..79e249a806c96 100644
--- a/sql/core/benchmarks/ParquetNestedSchemaPruningBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/ParquetNestedSchemaPruningBenchmark-jdk11-results.txt
@@ -2,52 +2,52 @@
 Nested Schema Pruning Benchmark For Parquet
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Selection:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    150            218          36          6.6         150.5       1.0X
-Nested column                                       294            334          39          3.4         293.8       0.5X
-Nested column in array                              994           1024          34          1.0         994.3       0.2X
+Top-level column                                     72            103          23         13.9          71.8       1.0X
+Nested column                                       351            372          22          2.8         351.2       0.2X
+Nested column in array                             1563           1664          57          0.6        1563.3       0.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Limiting:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    161            183          13          6.2         160.9       1.0X
-Nested column                                       296            332          43          3.4         295.6       0.5X
-Nested column in array                             1048           1089          48          1.0        1047.9       0.2X
+Top-level column                                    380            453          68          2.6         379.8       1.0X
+Nested column                                       729            818          70          1.4         728.8       0.5X
+Nested column in array                             2161           2404         173          0.5        2160.9       0.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Repartitioning:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    393            411          16          2.5         393.1       1.0X
-Nested column                                       550            563           9          1.8         549.9       0.7X
-Nested column in array                             1468           1494          20          0.7        1467.5       0.3X
+Top-level column                                    315            358          50          3.2         315.2       1.0X
+Nested column                                       617            678          40          1.6         616.9       0.5X
+Nested column in array                             2032           2149         112          0.5        2031.8       0.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Repartitioning by exprs:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    404            419          15          2.5         404.2       1.0X
-Nested column                                      2833           2875          24          0.4        2832.8       0.1X
-Nested column in array                             3340           3396          53          0.3        3340.3       0.1X
+Top-level column                                    306            348          39          3.3         305.6       1.0X
+Nested column                                       713            762          40          1.4         713.1       0.4X
+Nested column in array                             2364           2452          58          0.4        2364.3       0.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Sample:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    142            159          12          7.0         142.3       1.0X
-Nested column                                       308            318           8          3.2         307.8       0.5X
-Nested column in array                             1119           1154          32          0.9        1118.5       0.1X
+Top-level column                                    106            143          45          9.4         106.2       1.0X
+Nested column                                       427            460          42          2.3         427.3       0.2X
+Nested column in array                             1739           1864          91          0.6        1738.8       0.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Sorting:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    338            342           2          3.0         338.0       1.0X
-Nested column                                      1873           1971         129          0.5        1873.1       0.2X
-Nested column in array                             2708           2760          50          0.4        2707.9       0.1X
+Top-level column                                    520            581          39          1.9         519.6       1.0X
+Nested column                                       983           1063          71          1.0         983.3       0.5X
+Nested column in array                             2538           2687         146          0.4        2538.0       0.2X
 
 
diff --git a/sql/core/benchmarks/ParquetNestedSchemaPruningBenchmark-results.txt b/sql/core/benchmarks/ParquetNestedSchemaPruningBenchmark-results.txt
index 9f64e0425df8d..1dd30ef901f6f 100644
--- a/sql/core/benchmarks/ParquetNestedSchemaPruningBenchmark-results.txt
+++ b/sql/core/benchmarks/ParquetNestedSchemaPruningBenchmark-results.txt
@@ -2,52 +2,52 @@
 Nested Schema Pruning Benchmark For Parquet
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Selection:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    136            157          19          7.3         136.3       1.0X
-Nested column                                       254            267           8          3.9         254.3       0.5X
-Nested column in array                             1071           1089          18          0.9        1071.1       0.1X
+Top-level column                                     56             62           5         17.8          56.1       1.0X
+Nested column                                       291            306           9          3.4         291.0       0.2X
+Nested column in array                             1126           1166          24          0.9        1126.1       0.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Limiting:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    134            147          12          7.5         134.1       1.0X
-Nested column                                       288            295           5          3.5         287.7       0.5X
-Nested column in array                             1104           1135          35          0.9        1104.1       0.1X
+Top-level column                                    372            399          28          2.7         371.9       1.0X
+Nested column                                       686            724          27          1.5         686.0       0.5X
+Nested column in array                             1892           1973          72          0.5        1892.0       0.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Repartitioning:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    361            372          14          2.8         361.1       1.0X
-Nested column                                       522            535          16          1.9         521.8       0.7X
-Nested column in array                             1540           1553          11          0.6        1539.6       0.2X
+Top-level column                                    351            361           6          2.9         350.6       1.0X
+Nested column                                       661            677           8          1.5         660.6       0.5X
+Nested column in array                             1886           1935          41          0.5        1886.3       0.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Repartitioning by exprs:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    375            384          11          2.7         374.6       1.0X
-Nested column                                      2686           2715          24          0.4        2686.2       0.1X
-Nested column in array                             3067           3080          13          0.3        3067.2       0.1X
+Top-level column                                    322            343          15          3.1         322.2       1.0X
+Nested column                                       686            712          23          1.5         686.4       0.5X
+Nested column in array                             1813           1918          67          0.6        1813.1       0.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Sample:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    120            135           8          8.3         120.3       1.0X
-Nested column                                       280            290          13          3.6         279.9       0.4X
-Nested column in array                             1114           1143          29          0.9        1114.2       0.1X
+Top-level column                                     81             92           8         12.3          81.2       1.0X
+Nested column                                       331            393          42          3.0         330.6       0.2X
+Nested column in array                             1209           1324          70          0.8        1209.5       0.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Sorting:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Top-level column                                    263            277          18          3.8         263.0       1.0X
-Nested column                                      1724           1763          38          0.6        1724.1       0.2X
-Nested column in array                             2530           2605          65          0.4        2529.9       0.1X
+Top-level column                                    475            482           4          2.1         475.1       1.0X
+Nested column                                       843            883          17          1.2         843.2       0.6X
+Nested column in array                             1895           1936          33          0.5        1895.1       0.3X
 
 
diff --git a/sql/core/benchmarks/PrimitiveArrayBenchmark-jdk11-results.txt b/sql/core/benchmarks/PrimitiveArrayBenchmark-jdk11-results.txt
index 0e5a051d92c90..01b8509a325c1 100644
--- a/sql/core/benchmarks/PrimitiveArrayBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/PrimitiveArrayBenchmark-jdk11-results.txt
@@ -2,11 +2,11 @@
 Write primitive arrays in dataset
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Write an array in Dataset:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Int                                                 636            681          50         13.2          75.8       1.0X
-Double                                              727            809          71         11.5          86.7       0.9X
+Int                                                 461            490          30         18.2          55.0       1.0X
+Double                                              679            729          52         12.4          80.9       0.7X
 
 
diff --git a/sql/core/benchmarks/PrimitiveArrayBenchmark-results.txt b/sql/core/benchmarks/PrimitiveArrayBenchmark-results.txt
index 837145e7c93a5..2ddce974b986c 100644
--- a/sql/core/benchmarks/PrimitiveArrayBenchmark-results.txt
+++ b/sql/core/benchmarks/PrimitiveArrayBenchmark-results.txt
@@ -2,11 +2,11 @@
 Write primitive arrays in dataset
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 Write an array in Dataset:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Int                                                 631            684          65         13.3          75.2       1.0X
-Double                                              792            854          61         10.6          94.4       0.8X
+Int                                                 324            341          15         25.9          38.6       1.0X
+Double                                              583            599          11         14.4          69.5       0.6X
 
 
diff --git a/sql/core/benchmarks/RangeBenchmark-jdk11-results.txt b/sql/core/benchmarks/RangeBenchmark-jdk11-results.txt
index 44109e406ebb6..358bd76b7100a 100644
--- a/sql/core/benchmarks/RangeBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/RangeBenchmark-jdk11-results.txt
@@ -2,14 +2,14 @@
 range
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 range:                                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-full scan                                         11004          11558         944         47.6          21.0       1.0X
-limit after range                                   121            135          19       4320.4           0.2      90.7X
-filter after range                                 1529           1544          23        342.9           2.9       7.2X
-count after range                                   123            143          21       4268.5           0.2      89.6X
-count after limit after range                       115            123           6       4547.6           0.2      95.4X
+full scan                                         28020          28353         518         18.7          53.4       1.0X
+limit after range                                    34             58          44      15423.8           0.1     824.3X
+filter after range                                 1848           1863          12        283.7           3.5      15.2X
+count after range                                    34             44          15      15406.4           0.1     823.4X
+count after limit after range                        50             79          36      10485.2           0.1     560.4X
 
 
diff --git a/sql/core/benchmarks/RangeBenchmark-results.txt b/sql/core/benchmarks/RangeBenchmark-results.txt
index e006785e7a2ca..273968f781f89 100644
--- a/sql/core/benchmarks/RangeBenchmark-results.txt
+++ b/sql/core/benchmarks/RangeBenchmark-results.txt
@@ -2,14 +2,14 @@
 range
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 range:                                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-full scan                                         11565          11703         217         45.3          22.1       1.0X
-limit after range                                    96            100           4       5455.9           0.2     120.3X
-filter after range                                 1426           1432          10        367.7           2.7       8.1X
-count after range                                    82             84           2       6412.8           0.2     141.5X
-count after limit after range                        72             76           3       7264.9           0.1     160.2X
+full scan                                         18617          18864         257         28.2          35.5       1.0X
+limit after range                                    23             25           2      23167.0           0.0     822.7X
+filter after range                                 1288           1331          47        407.1           2.5      14.5X
+count after range                                    56             62           4       9281.9           0.1     329.6X
+count after limit after range                        34             35           3      15645.0           0.1     555.6X
 
 
diff --git a/sql/core/benchmarks/SortBenchmark-jdk11-results.txt b/sql/core/benchmarks/SortBenchmark-jdk11-results.txt
index a61989b174bf5..7fdc446a792d3 100644
--- a/sql/core/benchmarks/SortBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/SortBenchmark-jdk11-results.txt
@@ -2,15 +2,15 @@
 radix sort
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 radix sort 25000000:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-reference TimSort key prefix array                15588          15592           6          1.6         623.5       1.0X
-reference Arrays.sort                              3367           3377          14          7.4         134.7       4.6X
-radix sort one byte                                 436            447          10         57.3          17.5      35.7X
-radix sort two bytes                                875            885           9         28.6          35.0      17.8X
-radix sort eight bytes                             3409           3429          27          7.3         136.4       4.6X
-radix sort key prefix array                        5985           6058         104          4.2         239.4       2.6X
+reference TimSort key prefix array                13491          13527          52          1.9         539.6       1.0X
+reference Arrays.sort                              3526           3541          21          7.1         141.0       3.8X
+radix sort one byte                                 550            561           9         45.5          22.0      24.5X
+radix sort two bytes                               1038           1058          28         24.1          41.5      13.0X
+radix sort eight bytes                             4205           4214          13          5.9         168.2       3.2X
+radix sort key prefix array                        7674           7713          55          3.3         307.0       1.8X
 
 
diff --git a/sql/core/benchmarks/SortBenchmark-results.txt b/sql/core/benchmarks/SortBenchmark-results.txt
index b321d1115b727..8866158fe727c 100644
--- a/sql/core/benchmarks/SortBenchmark-results.txt
+++ b/sql/core/benchmarks/SortBenchmark-results.txt
@@ -2,15 +2,15 @@
 radix sort
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 radix sort 25000000:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-reference TimSort key prefix array                15736          15778          59          1.6         629.4       1.0X
-reference Arrays.sort                              3051           3057          10          8.2         122.0       5.2X
-radix sort one byte                                 442            453          10         56.6          17.7      35.6X
-radix sort two bytes                                883            885           1         28.3          35.3      17.8X
-radix sort eight bytes                             3422           3440          26          7.3         136.9       4.6X
-radix sort key prefix array                        6025           6138         159          4.1         241.0       2.6X
+reference TimSort key prefix array                13590          13887         420          1.8         543.6       1.0X
+reference Arrays.sort                              3615           3644          40          6.9         144.6       3.8X
+radix sort one byte                                 501            505           3         49.9          20.1      27.1X
+radix sort two bytes                                998           1011          21         25.1          39.9      13.6X
+radix sort eight bytes                             3894           3912          26          6.4         155.8       3.5X
+radix sort key prefix array                        7299           7316          23          3.4         292.0       1.9X
 
 
diff --git a/sql/core/benchmarks/SubExprEliminationBenchmark-jdk11-results.txt b/sql/core/benchmarks/SubExprEliminationBenchmark-jdk11-results.txt
new file mode 100644
index 0000000000000..5d117e1c9d09a
--- /dev/null
+++ b/sql/core/benchmarks/SubExprEliminationBenchmark-jdk11-results.txt
@@ -0,0 +1,25 @@
+================================================================================================
+Benchmark for performance of subexpression elimination
+================================================================================================
+
+Preparing data for benchmarking ...
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+from_json as subExpr in Project:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+subExprElimination false, codegen: true            9888          10006         160          0.0    98884264.7       1.0X
+subExprElimination false, codegen: false           9007           9149         129          0.0    90070910.8       1.1X
+subExprElimination true, codegen: true             1305           1346          53          0.0    13048714.1       7.6X
+subExprElimination true, codegen: false            1316           1323           9          0.0    13162352.5       7.5X
+
+Preparing data for benchmarking ...
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+from_json as subExpr in Filter:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+subExprElimination false, codegen: true           14332          15051         631          0.0   143320683.0       1.0X
+subExprElimination false, codegen: false          15083          15663         666          0.0   150832091.2       1.0X
+subExprElimination true, codegen: true             7865           8153         261          0.0    78653514.4       1.8X
+subExprElimination true, codegen: false            8078           8663         605          0.0    80780867.2       1.8X
+
+
diff --git a/sql/core/benchmarks/SubExprEliminationBenchmark-results.txt b/sql/core/benchmarks/SubExprEliminationBenchmark-results.txt
new file mode 100644
index 0000000000000..68c911f99c863
--- /dev/null
+++ b/sql/core/benchmarks/SubExprEliminationBenchmark-results.txt
@@ -0,0 +1,25 @@
+================================================================================================
+Benchmark for performance of subexpression elimination
+================================================================================================
+
+Preparing data for benchmarking ...
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+from_json as subExpr in Project:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+subExprElimination false, codegen: true            8466           8859         497          0.0    84660198.0       1.0X
+subExprElimination false, codegen: false           8176           8357         159          0.0    81759220.7       1.0X
+subExprElimination true, codegen: true              962           1020          86          0.0     9623297.9       8.8X
+subExprElimination true, codegen: false             948            980          31          0.0     9476038.9       8.9X
+
+Preparing data for benchmarking ...
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+from_json as subExpr in Filter:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+subExprElimination false, codegen: true           12971          13362         348          0.0   129707796.2       1.0X
+subExprElimination false, codegen: false          13796          14144         309          0.0   137961780.5       0.9X
+subExprElimination true, codegen: true             6611           7128         449          0.0    66106385.8       2.0X
+subExprElimination true, codegen: false            7068           7369         323          0.0    70675302.7       1.8X
+
+
diff --git a/sql/core/benchmarks/UDFBenchmark-jdk11-results.txt b/sql/core/benchmarks/UDFBenchmark-jdk11-results.txt
index 79daf73a95bb4..5f31d536a8af3 100644
--- a/sql/core/benchmarks/UDFBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/UDFBenchmark-jdk11-results.txt
@@ -2,58 +2,58 @@
 UDF with mixed input types
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-long/nullable int/string to string:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-long/nullable int/string to string wholestage off            279            320          58          0.4        2789.3       1.0X
-long/nullable int/string to string wholestage on            182            188           7          0.6        1818.0       1.5X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-long/nullable int/string to option:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-long/nullable int/string to option wholestage off            133            137           5          0.8        1332.3       1.0X
-long/nullable int/string to option wholestage on            110            118           8          0.9        1097.3       1.2X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-long/nullable int/string to primitive:    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-long/nullable int/string to primitive wholestage off             69             83          19          1.4         694.7       1.0X
-long/nullable int/string to primitive wholestage on             66             81          18          1.5         657.5       1.1X
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+long/nullable int/string to string:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+---------------------------------------------------------------------------------------------------------------------------------
+long/nullable int/string to string wholestage off             85             85           1          1.2         846.6       1.0X
+long/nullable int/string to string wholestage on              76             81           4          1.3         762.9       1.1X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+long/nullable int/string to option:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+---------------------------------------------------------------------------------------------------------------------------------
+long/nullable int/string to option wholestage off             53             56           4          1.9         529.6       1.0X
+long/nullable int/string to option wholestage on              52             54           2          1.9         518.2       1.0X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+long/nullable int/string to primitive:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------------------
+long/nullable int/string to primitive wholestage off             50             51           1          2.0         502.0       1.0X
+long/nullable int/string to primitive wholestage on              40             43           4          2.5         404.2       1.2X
 
 
 ================================================================================================
 UDF with primitive types
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-long/nullable int to string:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-long/nullable int to string wholestage off             76             80           5          1.3         761.5       1.0X
-long/nullable int to string wholestage on             65             67           2          1.5         654.2       1.2X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-long/nullable int to option:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-long/nullable int to option wholestage off             49             50           2          2.0         487.9       1.0X
-long/nullable int to option wholestage on             50             66          12          2.0         503.6       1.0X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-long/nullable int to primitive:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-long/nullable int to primitive wholestage off             48             58          14          2.1         483.5       1.0X
-long/nullable int to primitive wholestage on             46             52           6          2.2         456.8       1.1X
-
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+long/nullable int to string:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+--------------------------------------------------------------------------------------------------------------------------
+long/nullable int to string wholestage off             62             65           4          1.6         617.7       1.0X
+long/nullable int to string wholestage on              55             60           9          1.8         547.9       1.1X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+long/nullable int to option:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+--------------------------------------------------------------------------------------------------------------------------
+long/nullable int to option wholestage off             43             46           4          2.3         431.5       1.0X
+long/nullable int to option wholestage on              32             34           2          3.1         324.9       1.3X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+long/nullable int to primitive:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-----------------------------------------------------------------------------------------------------------------------------
+long/nullable int to primitive wholestage off             30             31           2          3.4         295.1       1.0X
+long/nullable int to primitive wholestage on              27             28           1          3.7         273.7       1.1X
+
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 UDF identity overhead:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Baseline                                             39             41           3          2.6         385.8       1.0X
-With identity UDF                                    44             46           3          2.3         444.4       0.9X
+Baseline                                             18             19           1          5.6         179.1       1.0X
+With identity UDF                                    27             30           6          3.7         267.9       0.7X
 
 
diff --git a/sql/core/benchmarks/UDFBenchmark-results.txt b/sql/core/benchmarks/UDFBenchmark-results.txt
index 3f17a999094a3..f105b390a8e39 100644
--- a/sql/core/benchmarks/UDFBenchmark-results.txt
+++ b/sql/core/benchmarks/UDFBenchmark-results.txt
@@ -2,58 +2,58 @@
 UDF with mixed input types
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-long/nullable int/string to string:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-long/nullable int/string to string wholestage off            250            327         108          0.4        2500.6       1.0X
-long/nullable int/string to string wholestage on            142            157          16          0.7        1421.2       1.8X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-long/nullable int/string to option:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-long/nullable int/string to option wholestage off            124            125           2          0.8        1237.8       1.0X
-long/nullable int/string to option wholestage on             73             93          27          1.4         730.1       1.7X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-long/nullable int/string to primitive:    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-long/nullable int/string to primitive wholestage off             66             69           4          1.5         658.8       1.0X
-long/nullable int/string to primitive wholestage on             61             67          11          1.6         611.7       1.1X
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+long/nullable int/string to string:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+---------------------------------------------------------------------------------------------------------------------------------
+long/nullable int/string to string wholestage off             88             92           6          1.1         877.7       1.0X
+long/nullable int/string to string wholestage on              83             84           1          1.2         832.5       1.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+long/nullable int/string to option:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+---------------------------------------------------------------------------------------------------------------------------------
+long/nullable int/string to option wholestage off             56             56           0          1.8         559.1       1.0X
+long/nullable int/string to option wholestage on              53             55           2          1.9         532.6       1.0X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+long/nullable int/string to primitive:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------------------
+long/nullable int/string to primitive wholestage off             52             58           8          1.9         523.8       1.0X
+long/nullable int/string to primitive wholestage on              50             50           1          2.0         498.3       1.1X
 
 
 ================================================================================================
 UDF with primitive types
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-long/nullable int to string:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-long/nullable int to string wholestage off             66             67           0          1.5         663.9       1.0X
-long/nullable int to string wholestage on             66             68           2          1.5         664.6       1.0X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-long/nullable int to option:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-long/nullable int to option wholestage off             40             42           3          2.5         402.6       1.0X
-long/nullable int to option wholestage on             40             42           2          2.5         401.3       1.0X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-long/nullable int to primitive:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-long/nullable int to primitive wholestage off             38             39           0          2.6         384.8       1.0X
-long/nullable int to primitive wholestage on             39             45           5          2.5         392.6       1.0X
-
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+long/nullable int to string:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+--------------------------------------------------------------------------------------------------------------------------
+long/nullable int to string wholestage off             59             59           0          1.7         594.7       1.0X
+long/nullable int to string wholestage on              57             64          13          1.7         574.7       1.0X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+long/nullable int to option:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+--------------------------------------------------------------------------------------------------------------------------
+long/nullable int to option wholestage off             33             33           1          3.1         326.0       1.0X
+long/nullable int to option wholestage on              31             32           1          3.2         308.1       1.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+long/nullable int to primitive:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-----------------------------------------------------------------------------------------------------------------------------
+long/nullable int to primitive wholestage off             30             32           1          3.3         304.9       1.0X
+long/nullable int to primitive wholestage on              28             28           1          3.6         276.3       1.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 UDF identity overhead:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Baseline                                             32             33           1          3.1         320.8       1.0X
-With identity UDF                                    37             40           6          2.7         369.1       0.9X
+Baseline                                             18             19           1          5.7         176.1       1.0X
+With identity UDF                                    24             29           7          4.1         243.2       0.7X
 
 
diff --git a/sql/core/benchmarks/UnsafeArrayDataBenchmark-jdk11-results.txt b/sql/core/benchmarks/UnsafeArrayDataBenchmark-jdk11-results.txt
index 577cdb0db62b3..872845f28fc11 100644
--- a/sql/core/benchmarks/UnsafeArrayDataBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/UnsafeArrayDataBenchmark-jdk11-results.txt
@@ -2,32 +2,32 @@
 Benchmark UnsafeArrayData
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Read UnsafeArrayData:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Int                                                 183            184           0        914.7           1.1       1.0X
-Double                                              188            189           0        891.1           1.1       1.0X
+Int                                                 131            133           1       1279.4           0.8       1.0X
+Double                                              280            286           3        599.6           1.7       0.5X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Write UnsafeArrayData:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Int                                                  31             32           1        670.0           1.5       1.0X
-Double                                               70             71           1        300.3           3.3       0.4X
+Int                                                  53             71          14        392.2           2.5       1.0X
+Double                                              114            143          12        183.8           5.4       0.5X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Get primitive array from UnsafeArrayData:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Int                                                  71             73           3        887.3           1.1       1.0X
-Double                                              144            151           6        438.2           2.3       0.5X
+Int                                                  87            108          16        723.4           1.4       1.0X
+Double                                              204            220          19        307.7           3.3       0.4X
 
-OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Create UnsafeArrayData from primitive array:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Int                                                  69             70           1        906.7           1.1       1.0X
-Double                                              140            142           1        448.8           2.2       0.5X
+---------------------------------------------------------------------------------------------------------------------------
+Int                                                     87            108          19        725.0           1.4       1.0X
+Double                                                 203            217          16        309.2           3.2       0.4X
 
 
diff --git a/sql/core/benchmarks/UnsafeArrayDataBenchmark-results.txt b/sql/core/benchmarks/UnsafeArrayDataBenchmark-results.txt
index 6c4669d7ea442..58cc59e016beb 100644
--- a/sql/core/benchmarks/UnsafeArrayDataBenchmark-results.txt
+++ b/sql/core/benchmarks/UnsafeArrayDataBenchmark-results.txt
@@ -2,32 +2,32 @@
 Benchmark UnsafeArrayData
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 Read UnsafeArrayData:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Int                                                 183            183           0        919.0           1.1       1.0X
-Double                                              192            196           4        872.4           1.1       0.9X
+Int                                                   4              4           0      47832.8           0.0       1.0X
+Double                                                3              3           0      54217.6           0.0       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 Write UnsafeArrayData:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Int                                                  33             36           3        639.3           1.6       1.0X
-Double                                               76             79           2        274.6           3.6       0.4X
+Int                                                  45             46           1        470.8           2.1       1.0X
+Double                                               96             99           2        218.1           4.6       0.5X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 Get primitive array from UnsafeArrayData:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Int                                                  71             76           4        890.8           1.1       1.0X
-Double                                              147            151           6        427.4           2.3       0.5X
+Int                                                  83             86           2        754.1           1.3       1.0X
+Double                                              165            170           5        381.9           2.6       0.5X
 
-OpenJDK 64-Bit Server VM 1.8.0_222-b10 on Linux 3.10.0-862.3.2.el7.x86_64
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 Create UnsafeArrayData from primitive array:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-Int                                                  73             75           3        866.8           1.2       1.0X
-Double                                              148            150           1        424.8           2.4       0.5X
+---------------------------------------------------------------------------------------------------------------------------
+Int                                                     85             87           2        738.6           1.4       1.0X
+Double                                                 165            173           6        381.8           2.6       0.5X
 
 
diff --git a/sql/core/benchmarks/UpdateFieldsBenchmark-results.txt b/sql/core/benchmarks/UpdateFieldsBenchmark-results.txt
new file mode 100644
index 0000000000000..ce0e806f59c68
--- /dev/null
+++ b/sql/core/benchmarks/UpdateFieldsBenchmark-results.txt
@@ -0,0 +1,26 @@
+================================================================================================
+Add 2 columns and drop 2 columns at 3 different depths of nesting
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
+Add 2 columns and drop 2 columns at 3 different depths of nesting:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-------------------------------------------------------------------------------------------------------------------------------------------------
+To non-nullable StructTypes using performant method                            4              4           1          0.0      Infinity       1.0X
+To nullable StructTypes using performant method                                3              3           0          0.0      Infinity       1.1X
+To non-nullable StructTypes using non-performant method                       79             80           1          0.0      Infinity       0.0X
+To nullable StructTypes using non-performant method                         2263           2290          39          0.0      Infinity       0.0X
+
+
+================================================================================================
+Add 50 columns and drop 50 columns at 100 different depths of nesting
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
+Add 50 columns and drop 50 columns at 100 different depths of nesting:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-----------------------------------------------------------------------------------------------------------------------------------------------------
+To non-nullable StructTypes using performant method                             3243           3258          21          0.0      Infinity       1.0X
+To nullable StructTypes using performant method                                 3184           3192          10          0.0      Infinity       1.0X
+
+
diff --git a/sql/core/benchmarks/V2FunctionBenchmark-jdk11-results.txt b/sql/core/benchmarks/V2FunctionBenchmark-jdk11-results.txt
new file mode 100644
index 0000000000000..fc95416f658e7
--- /dev/null
+++ b/sql/core/benchmarks/V2FunctionBenchmark-jdk11-results.txt
@@ -0,0 +1,44 @@
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1047-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+scalar function (long + long) -> long, result_nullable = true codegen = true:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------------------------------------------
+native_long_add                                                                       14079          14697         555         35.5          28.2       1.0X
+java_long_add_default                                                                 36350          38220        1620         13.8          72.7       0.4X
+java_long_add_magic                                                                   14910          15251         336         33.5          29.8       0.9X
+java_long_add_static_magic                                                            14863          14962         161         33.6          29.7       0.9X
+scala_long_add_default                                                                41715          42786        1040         12.0          83.4       0.3X
+scala_long_add_magic                                                                  15712          15775          58         31.8          31.4       0.9X
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1047-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+scalar function (long + long) -> long, result_nullable = false codegen = true:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-------------------------------------------------------------------------------------------------------------------------------------------------------------
+native_long_add                                                                        13959          14048          80         35.8          27.9       1.0X
+java_long_add_default                                                                  40773          41318         580         12.3          81.5       0.3X
+java_long_add_magic                                                                    15929          16145         205         31.4          31.9       0.9X
+java_long_add_static_magic                                                             13384          13948         496         37.4          26.8       1.0X
+scala_long_add_default                                                                 37782          39099        1141         13.2          75.6       0.4X
+scala_long_add_magic                                                                   14553          14982         372         34.4          29.1       1.0X
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1047-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+scalar function (long + long) -> long, result_nullable = true codegen = false:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-------------------------------------------------------------------------------------------------------------------------------------------------------------
+native_long_add                                                                        31564          32912        1167         15.8          63.1       1.0X
+java_long_add_default                                                                  45392          46662        1821         11.0          90.8       0.7X
+java_long_add_magic                                                                    44650          45705        1230         11.2          89.3       0.7X
+java_long_add_static_magic                                                             46391          47033         573         10.8          92.8       0.7X
+scala_long_add_default                                                                 42915          44688        1654         11.7          85.8       0.7X
+scala_long_add_magic                                                                   45617          46073         644         11.0          91.2       0.7X
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1047-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+scalar function (long + long) -> long, result_nullable = false codegen = false:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+--------------------------------------------------------------------------------------------------------------------------------------------------------------
+native_long_add                                                                         30192          30399         186         16.6          60.4       1.0X
+java_long_add_default                                                                   41940          42698         679         11.9          83.9       0.7X
+java_long_add_magic                                                                     45087          45760         628         11.1          90.2       0.7X
+java_long_add_static_magic                                                              44109          45979        1726         11.3          88.2       0.7X
+scala_long_add_default                                                                  41676          42064         375         12.0          83.4       0.7X
+scala_long_add_magic                                                                    44886          45825         858         11.1          89.8       0.7X
+
diff --git a/sql/core/benchmarks/V2FunctionBenchmark-results.txt b/sql/core/benchmarks/V2FunctionBenchmark-results.txt
new file mode 100644
index 0000000000000..b442056d24d73
--- /dev/null
+++ b/sql/core/benchmarks/V2FunctionBenchmark-results.txt
@@ -0,0 +1,44 @@
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1047-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+scalar function (long + long) -> long, result_nullable = true codegen = true:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------------------------------------------
+native_long_add                                                                       14269          14469         331         35.0          28.5       1.0X
+java_long_add_default                                                                 38687          38861         198         12.9          77.4       0.4X
+java_long_add_magic                                                                   15328          15717         595         32.6          30.7       0.9X
+java_long_add_static_magic                                                            12795          12966         158         39.1          25.6       1.1X
+scala_long_add_default                                                                42538          42627          84         11.8          85.1       0.3X
+scala_long_add_magic                                                                  14953          15029          82         33.4          29.9       1.0X
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1047-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+scalar function (long + long) -> long, result_nullable = false codegen = true:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-------------------------------------------------------------------------------------------------------------------------------------------------------------
+native_long_add                                                                        12298          12318          18         40.7          24.6       1.0X
+java_long_add_default                                                                  38181          38606         432         13.1          76.4       0.3X
+java_long_add_magic                                                                    15255          15308          58         32.8          30.5       0.8X
+java_long_add_static_magic                                                             12157          12280         117         41.1          24.3       1.0X
+scala_long_add_default                                                                 38249          38530         282         13.1          76.5       0.3X
+scala_long_add_magic                                                                   15037          15314         265         33.3          30.1       0.8X
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1047-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+scalar function (long + long) -> long, result_nullable = true codegen = false:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-------------------------------------------------------------------------------------------------------------------------------------------------------------
+native_long_add                                                                        37620          37724         122         13.3          75.2       1.0X
+java_long_add_default                                                                  48087          48170          73         10.4          96.2       0.8X
+java_long_add_magic                                                                    56373          56402          48          8.9         112.7       0.7X
+java_long_add_static_magic                                                             56752          57235         578          8.8         113.5       0.7X
+scala_long_add_default                                                                 48198          48350         199         10.4          96.4       0.8X
+scala_long_add_magic                                                                   57988          58093          99          8.6         116.0       0.6X
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1047-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+scalar function (long + long) -> long, result_nullable = false codegen = false:  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+--------------------------------------------------------------------------------------------------------------------------------------------------------------
+native_long_add                                                                         34925          34976          75         14.3          69.9       1.0X
+java_long_add_default                                                                   45836          45857          33         10.9          91.7       0.8X
+java_long_add_magic                                                                     55190          55364         158          9.1         110.4       0.6X
+java_long_add_static_magic                                                              54906          54977          73          9.1         109.8       0.6X
+scala_long_add_default                                                                  46960          47204         265         10.6          93.9       0.7X
+scala_long_add_magic                                                                    54993          55016          33          9.1         110.0       0.6X
+
diff --git a/sql/core/benchmarks/WideSchemaBenchmark-jdk11-results.txt b/sql/core/benchmarks/WideSchemaBenchmark-jdk11-results.txt
index 19ce183820c05..b99bd448af594 100644
--- a/sql/core/benchmarks/WideSchemaBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/WideSchemaBenchmark-jdk11-results.txt
@@ -2,144 +2,144 @@
 parsing large select expressions
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 parsing large select:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-1 select expressions                                  8             15           8          0.0     8028037.0       1.0X
-100 select expressions                               15             18           3          0.0    14899892.0       0.5X
-2500 select expressions                             237            243           8          0.0   237252523.0       0.0X
+1 select expressions                                  1              2           1          0.0     1421891.0       1.0X
+100 select expressions                                9             11           2          0.0     8906443.0       0.2X
+2500 select expressions                             400            444          46          0.0   400078084.0       0.0X
 
 
 ================================================================================================
 many column field read and write
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 many column field r/w:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-1 cols x 100000 rows (read in-mem)                   59             72           8          1.7         591.0       1.0X
-1 cols x 100000 rows (exec in-mem)                   57             81          15          1.8         566.0       1.0X
-1 cols x 100000 rows (read parquet)                  61             78          13          1.6         614.8       1.0X
-1 cols x 100000 rows (write parquet)                147            158          10          0.7        1468.5       0.4X
-100 cols x 1000 rows (read in-mem)                   57             62           6          1.8         565.8       1.0X
-100 cols x 1000 rows (exec in-mem)                   76             83          10          1.3         758.7       0.8X
-100 cols x 1000 rows (read parquet)                  70             79          10          1.4         700.8       0.8X
-100 cols x 1000 rows (write parquet)                150            162          11          0.7        1498.8       0.4X
-2500 cols x 40 rows (read in-mem)                   413            424          15          0.2        4134.4       0.1X
-2500 cols x 40 rows (exec in-mem)                   753            772          23          0.1        7528.2       0.1X
-2500 cols x 40 rows (read parquet)                  304            312           8          0.3        3044.6       0.2X
-2500 cols x 40 rows (write parquet)                 507            520          11          0.2        5069.3       0.1X
+1 cols x 100000 rows (read in-mem)                   19             28          10          5.2         190.6       1.0X
+1 cols x 100000 rows (exec in-mem)                   23             40          17          4.4         226.2       0.8X
+1 cols x 100000 rows (read parquet)                  37             54          18          2.7         371.8       0.5X
+1 cols x 100000 rows (write parquet)                116            166          62          0.9        1157.6       0.2X
+100 cols x 1000 rows (read in-mem)                   23             39          20          4.3         231.5       0.8X
+100 cols x 1000 rows (exec in-mem)                   35             56          24          2.8         352.5       0.5X
+100 cols x 1000 rows (read parquet)                  36             64          21          2.8         358.0       0.5X
+100 cols x 1000 rows (write parquet)                121            148          36          0.8        1211.1       0.2X
+2500 cols x 40 rows (read in-mem)                   147            169          18          0.7        1474.8       0.1X
+2500 cols x 40 rows (exec in-mem)                   691            730          41          0.1        6912.5       0.0X
+2500 cols x 40 rows (read parquet)                  650            679          44          0.2        6495.8       0.0X
+2500 cols x 40 rows (write parquet)                 265            285          13          0.4        2651.2       0.1X
 
 
 ================================================================================================
 wide shallowly nested struct field read and write
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 wide shallowly nested struct field r/w:   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-1 wide x 100000 rows (read in-mem)                   54             63           8          1.8         540.7       1.0X
-1 wide x 100000 rows (exec in-mem)                   67             77          11          1.5         671.8       0.8X
-1 wide x 100000 rows (read parquet)                  90             97           6          1.1         901.2       0.6X
-1 wide x 100000 rows (write parquet)                150            163          11          0.7        1503.9       0.4X
-100 wide x 1000 rows (read in-mem)                   69             75           8          1.4         689.8       0.8X
-100 wide x 1000 rows (exec in-mem)                  111            148          96          0.9        1111.5       0.5X
-100 wide x 1000 rows (read parquet)                 181            241          35          0.6        1808.7       0.3X
-100 wide x 1000 rows (write parquet)                164            180          27          0.6        1636.1       0.3X
-2500 wide x 40 rows (read in-mem)                    78            101          84          1.3         781.0       0.7X
-2500 wide x 40 rows (exec in-mem)                   943            966          37          0.1        9430.9       0.1X
-2500 wide x 40 rows (read parquet)                 1385           1453          95          0.1       13853.3       0.0X
-2500 wide x 40 rows (write parquet)                 175            190          19          0.6        1745.5       0.3X
+1 wide x 100000 rows (read in-mem)                   29             47          16          3.4         290.5       1.0X
+1 wide x 100000 rows (exec in-mem)                   35             56          19          2.9         350.3       0.8X
+1 wide x 100000 rows (read parquet)                  57             82          28          1.8         570.1       0.5X
+1 wide x 100000 rows (write parquet)                124            171          44          0.8        1239.2       0.2X
+100 wide x 1000 rows (read in-mem)                   25             37          13          4.1         245.8       1.2X
+100 wide x 1000 rows (exec in-mem)                   40             56          16          2.5         404.5       0.7X
+100 wide x 1000 rows (read parquet)                  37             62          16          2.7         368.6       0.8X
+100 wide x 1000 rows (write parquet)                134            156          22          0.7        1341.0       0.2X
+2500 wide x 40 rows (read in-mem)                    34             49          17          3.0         338.1       0.9X
+2500 wide x 40 rows (exec in-mem)                   433            454          16          0.2        4334.6       0.1X
+2500 wide x 40 rows (read parquet)                  163            233          53          0.6        1634.9       0.2X
+2500 wide x 40 rows (write parquet)                 126            177          45          0.8        1262.7       0.2X
 
 
 ================================================================================================
 deeply nested struct field read and write
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 deeply nested struct field r/w:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-1 deep x 100000 rows (read in-mem)                   44             49           6          2.3         441.1       1.0X
-1 deep x 100000 rows (exec in-mem)                   54             59           6          1.9         536.4       0.8X
-1 deep x 100000 rows (read parquet)                  65             68           6          1.5         646.1       0.7X
-1 deep x 100000 rows (write parquet)                141            147           9          0.7        1413.9       0.3X
-100 deep x 1000 rows (read in-mem)                  459            470          11          0.2        4592.9       0.1X
-100 deep x 1000 rows (exec in-mem)                 1736           1740           6          0.1       17355.1       0.0X
-100 deep x 1000 rows (read parquet)                1638           1643           6          0.1       16382.2       0.0X
-100 deep x 1000 rows (write parquet)                555            567          12          0.2        5548.4       0.1X
-250 deep x 400 rows (read in-mem)                  2556           2556           1          0.0       25558.5       0.0X
-250 deep x 400 rows (exec in-mem)                 10410          10416           8          0.0      104102.6       0.0X
-250 deep x 400 rows (read parquet)                 9670           9688          26          0.0       96699.1       0.0X
-250 deep x 400 rows (write parquet)                2638           2642           5          0.0       26379.7       0.0X
+1 deep x 100000 rows (read in-mem)                   27             40          15          3.7         273.6       1.0X
+1 deep x 100000 rows (exec in-mem)                   34             46          15          3.0         336.3       0.8X
+1 deep x 100000 rows (read parquet)                  53             74          17          1.9         527.0       0.5X
+1 deep x 100000 rows (write parquet)                134            174          37          0.7        1339.3       0.2X
+100 deep x 1000 rows (read in-mem)                   87             99           9          1.1         870.2       0.3X
+100 deep x 1000 rows (exec in-mem)                  917            954          34          0.1        9171.4       0.0X
+100 deep x 1000 rows (read parquet)                 765            793          28          0.1        7645.5       0.0X
+100 deep x 1000 rows (write parquet)                200            222          22          0.5        2003.2       0.1X
+250 deep x 400 rows (read in-mem)                   394            419          18          0.3        3939.3       0.1X
+250 deep x 400 rows (exec in-mem)                  6189           6210          29          0.0       61892.7       0.0X
+250 deep x 400 rows (read parquet)                 4454           4608         218          0.0       44539.5       0.0X
+250 deep x 400 rows (write parquet)                 448            455           4          0.2        4482.4       0.1X
 
 
 ================================================================================================
 bushy struct field read and write
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-bushy struct field r/w:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-1 x 1 deep x 100000 rows (read in-mem)               39             44           6          2.6         388.2       1.0X
-1 x 1 deep x 100000 rows (exec in-mem)               48             50           4          2.1         477.4       0.8X
-1 x 1 deep x 100000 rows (read parquet)              47             54           9          2.1         466.1       0.8X
-1 x 1 deep x 100000 rows (write parquet)            135            141           5          0.7        1350.5       0.3X
-128 x 8 deep x 1000 rows (read in-mem)               45             53           9          2.2         445.2       0.9X
-128 x 8 deep x 1000 rows (exec in-mem)              155            160           4          0.6        1553.0       0.2X
-128 x 8 deep x 1000 rows (read parquet)             173            217          31          0.6        1729.8       0.2X
-128 x 8 deep x 1000 rows (write parquet)            139            154          10          0.7        1389.9       0.3X
-1024 x 11 deep x 100 rows (read in-mem)              73             77           4          1.4         730.2       0.5X
-1024 x 11 deep x 100 rows (exec in-mem)             733            738           8          0.1        7326.1       0.1X
-1024 x 11 deep x 100 rows (read parquet)            652            660           8          0.2        6517.6       0.1X
-1024 x 11 deep x 100 rows (write parquet)            171            186          20          0.6        1706.4       0.2X
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+bushy struct field r/w:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-------------------------------------------------------------------------------------------------------------------------
+1 x 1 deep x 100000 rows (read in-mem)                22             29           9          4.6         217.1       1.0X
+1 x 1 deep x 100000 rows (exec in-mem)                29             36           9          3.4         292.1       0.7X
+1 x 1 deep x 100000 rows (read parquet)               30             39          11          3.3         301.9       0.7X
+1 x 1 deep x 100000 rows (write parquet)             139            181          49          0.7        1388.8       0.2X
+128 x 8 deep x 1000 rows (read in-mem)                27             36          11          3.7         267.1       0.8X
+128 x 8 deep x 1000 rows (exec in-mem)                62             85          22          1.6         618.8       0.4X
+128 x 8 deep x 1000 rows (read parquet)               42             62          19          2.4         422.3       0.5X
+128 x 8 deep x 1000 rows (write parquet)             126            141          11          0.8        1258.9       0.2X
+1024 x 11 deep x 100 rows (read in-mem)               31             44          12          3.2         313.7       0.7X
+1024 x 11 deep x 100 rows (exec in-mem)              363            392          24          0.3        3628.3       0.1X
+1024 x 11 deep x 100 rows (read parquet)              79            107          28          1.3         792.5       0.3X
+1024 x 11 deep x 100 rows (write parquet)            142            160          17          0.7        1424.1       0.2X
 
 
 ================================================================================================
 wide array field read and write
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 wide array field r/w:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-1 wide x 100000 rows (read in-mem)                   43             46           4          2.3         429.7       1.0X
-1 wide x 100000 rows (exec in-mem)                   54             57           4          1.8         542.4       0.8X
-1 wide x 100000 rows (read parquet)                  82             87           8          1.2         816.6       0.5X
-1 wide x 100000 rows (write parquet)                137            159          19          0.7        1374.9       0.3X
-100 wide x 1000 rows (read in-mem)                   37             39           4          2.7         367.1       1.2X
-100 wide x 1000 rows (exec in-mem)                   45             50           6          2.2         451.6       1.0X
-100 wide x 1000 rows (read parquet)                  52             57           5          1.9         520.8       0.8X
-100 wide x 1000 rows (write parquet)                125            131           8          0.8        1247.0       0.3X
-2500 wide x 40 rows (read in-mem)                    35             39           4          2.9         348.8       1.2X
-2500 wide x 40 rows (exec in-mem)                    46             49           5          2.2         456.0       0.9X
-2500 wide x 40 rows (read parquet)                   51             55           6          2.0         508.3       0.8X
-2500 wide x 40 rows (write parquet)                 129            135           6          0.8        1287.3       0.3X
+1 wide x 100000 rows (read in-mem)                   27             44          17          3.6         274.0       1.0X
+1 wide x 100000 rows (exec in-mem)                   31             45          17          3.3         305.4       0.9X
+1 wide x 100000 rows (read parquet)                  74             94          30          1.3         743.7       0.4X
+1 wide x 100000 rows (write parquet)                127            156          29          0.8        1268.3       0.2X
+100 wide x 1000 rows (read in-mem)                   21             32          11          4.8         208.2       1.3X
+100 wide x 1000 rows (exec in-mem)                   26             37          12          3.8         260.0       1.1X
+100 wide x 1000 rows (read parquet)                  36             49          14          2.8         361.9       0.8X
+100 wide x 1000 rows (write parquet)                128            172          42          0.8        1283.4       0.2X
+2500 wide x 40 rows (read in-mem)                    22             32          14          4.5         221.6       1.2X
+2500 wide x 40 rows (exec in-mem)                    26             37          13          3.9         257.0       1.1X
+2500 wide x 40 rows (read parquet)                   35             48          14          2.8         351.8       0.8X
+2500 wide x 40 rows (write parquet)                 124            142          16          0.8        1236.5       0.2X
 
 
 ================================================================================================
 wide map field read and write
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 wide map field r/w:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-1 wide x 100000 rows (read in-mem)                   39             48           9          2.5         394.2       1.0X
-1 wide x 100000 rows (exec in-mem)                   51             56           9          1.9         514.4       0.8X
-1 wide x 100000 rows (read parquet)                 119            124           7          0.8        1195.0       0.3X
-1 wide x 100000 rows (write parquet)                130            138           8          0.8        1299.8       0.3X
-100 wide x 1000 rows (read in-mem)                   31             32           3          3.3         306.5       1.3X
-100 wide x 1000 rows (exec in-mem)                   40             42           3          2.5         402.7       1.0X
-100 wide x 1000 rows (read parquet)                  65             70           6          1.5         651.8       0.6X
-100 wide x 1000 rows (write parquet)                123            129           6          0.8        1228.5       0.3X
-2500 wide x 40 rows (read in-mem)                    33             37           6          3.0         330.1       1.2X
-2500 wide x 40 rows (exec in-mem)                    43             44           3          2.3         426.6       0.9X
-2500 wide x 40 rows (read parquet)                   66             69           9          1.5         657.8       0.6X
-2500 wide x 40 rows (write parquet)                 123            127           2          0.8        1234.4       0.3X
+1 wide x 100000 rows (read in-mem)                   22             35          15          4.5         220.3       1.0X
+1 wide x 100000 rows (exec in-mem)                   31             41          14          3.2         312.3       0.7X
+1 wide x 100000 rows (read parquet)                 103            116          19          1.0        1032.1       0.2X
+1 wide x 100000 rows (write parquet)                128            156          37          0.8        1277.7       0.2X
+100 wide x 1000 rows (read in-mem)                   14             20           9          7.1         140.0       1.6X
+100 wide x 1000 rows (exec in-mem)                   20             25           9          5.1         197.8       1.1X
+100 wide x 1000 rows (read parquet)                  45             60          14          2.2         445.1       0.5X
+100 wide x 1000 rows (write parquet)                116            138          17          0.9        1162.6       0.2X
+2500 wide x 40 rows (read in-mem)                    16             21           9          6.2         160.4       1.4X
+2500 wide x 40 rows (exec in-mem)                    21             30          12          4.8         210.0       1.0X
+2500 wide x 40 rows (read parquet)                   47             59          19          2.1         467.4       0.5X
+2500 wide x 40 rows (write parquet)                 119            125           7          0.8        1186.4       0.2X
 
 
diff --git a/sql/core/benchmarks/WideSchemaBenchmark-results.txt b/sql/core/benchmarks/WideSchemaBenchmark-results.txt
index 59eb181f6b115..8b91921809cae 100644
--- a/sql/core/benchmarks/WideSchemaBenchmark-results.txt
+++ b/sql/core/benchmarks/WideSchemaBenchmark-results.txt
@@ -2,144 +2,144 @@
 parsing large select expressions
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 parsing large select:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-1 select expressions                                  5             13           8          0.0     5370143.0       1.0X
-100 select expressions                               12             16           6          0.0    11995425.0       0.4X
-2500 select expressions                             211            214           4          0.0   210927791.0       0.0X
+1 select expressions                                  1              2           0          0.0     1296117.0       1.0X
+100 select expressions                                9             11           1          0.0     8808690.0       0.1X
+2500 select expressions                             422            426           5          0.0   421632363.0       0.0X
 
 
 ================================================================================================
 many column field read and write
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 many column field r/w:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-1 cols x 100000 rows (read in-mem)                   44             53           6          2.3         440.3       1.0X
-1 cols x 100000 rows (exec in-mem)                   44             54           9          2.3         437.0       1.0X
-1 cols x 100000 rows (read parquet)                  53             61          10          1.9         532.4       0.8X
-1 cols x 100000 rows (write parquet)                129            142          36          0.8        1291.6       0.3X
-100 cols x 1000 rows (read in-mem)                   49             55           7          2.0         494.9       0.9X
-100 cols x 1000 rows (exec in-mem)                   69             73           5          1.4         693.2       0.6X
-100 cols x 1000 rows (read parquet)                  60             67           8          1.7         596.3       0.7X
-100 cols x 1000 rows (write parquet)                142            156          31          0.7        1417.8       0.3X
-2500 cols x 40 rows (read in-mem)                   391            399          13          0.3        3912.6       0.1X
-2500 cols x 40 rows (exec in-mem)                   743            749           8          0.1        7432.5       0.1X
-2500 cols x 40 rows (read parquet)                  297            310          10          0.3        2972.8       0.1X
-2500 cols x 40 rows (write parquet)                 485            492          16          0.2        4848.1       0.1X
+1 cols x 100000 rows (read in-mem)                   15             18           3          6.5         154.0       1.0X
+1 cols x 100000 rows (exec in-mem)                   20             23           4          4.9         202.1       0.8X
+1 cols x 100000 rows (read parquet)                  34             41          12          3.0         338.8       0.5X
+1 cols x 100000 rows (write parquet)                123            132           6          0.8        1226.7       0.1X
+100 cols x 1000 rows (read in-mem)                   22             25           4          4.5         221.5       0.7X
+100 cols x 1000 rows (exec in-mem)                   36             40           5          2.8         357.2       0.4X
+100 cols x 1000 rows (read parquet)                  33             41          15          3.0         331.2       0.5X
+100 cols x 1000 rows (write parquet)                120            133          10          0.8        1199.6       0.1X
+2500 cols x 40 rows (read in-mem)                   165            179           9          0.6        1645.1       0.1X
+2500 cols x 40 rows (exec in-mem)                   734            784          51          0.1        7338.1       0.0X
+2500 cols x 40 rows (read parquet)                  621            634          18          0.2        6210.9       0.0X
+2500 cols x 40 rows (write parquet)                 268            283          18          0.4        2675.3       0.1X
 
 
 ================================================================================================
 wide shallowly nested struct field read and write
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 wide shallowly nested struct field r/w:   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-1 wide x 100000 rows (read in-mem)                   43             48           6          2.3         427.0       1.0X
-1 wide x 100000 rows (exec in-mem)                   56             63           8          1.8         557.8       0.8X
-1 wide x 100000 rows (read parquet)                  82             88          10          1.2         818.6       0.5X
-1 wide x 100000 rows (write parquet)                134            145          21          0.7        1344.6       0.3X
-100 wide x 1000 rows (read in-mem)                   55             61          16          1.8         553.1       0.8X
-100 wide x 1000 rows (exec in-mem)                   94            101          17          1.1         941.4       0.5X
-100 wide x 1000 rows (read parquet)                 151            179          29          0.7        1511.7       0.3X
-100 wide x 1000 rows (write parquet)                147            157           9          0.7        1470.0       0.3X
-2500 wide x 40 rows (read in-mem)                    66             69           9          1.5         658.9       0.6X
-2500 wide x 40 rows (exec in-mem)                   853            871          30          0.1        8525.7       0.1X
-2500 wide x 40 rows (read parquet)                 1158           1296         195          0.1       11577.8       0.0X
-2500 wide x 40 rows (write parquet)                 157            173          23          0.6        1569.6       0.3X
+1 wide x 100000 rows (read in-mem)                   27             30           5          3.8         265.0       1.0X
+1 wide x 100000 rows (exec in-mem)                   33             37           5          3.0         331.8       0.8X
+1 wide x 100000 rows (read parquet)                  54             60           9          1.9         536.1       0.5X
+1 wide x 100000 rows (write parquet)                129            140           6          0.8        1293.8       0.2X
+100 wide x 1000 rows (read in-mem)                   21             26           6          4.8         210.5       1.3X
+100 wide x 1000 rows (exec in-mem)                   43             51           6          2.3         434.4       0.6X
+100 wide x 1000 rows (read parquet)                  34             39           8          2.9         340.2       0.8X
+100 wide x 1000 rows (write parquet)                119            128           4          0.8        1188.9       0.2X
+2500 wide x 40 rows (read in-mem)                    31             37           7          3.2         310.6       0.9X
+2500 wide x 40 rows (exec in-mem)                   524            559          37          0.2        5242.4       0.1X
+2500 wide x 40 rows (read parquet)                  139            152           6          0.7        1393.4       0.2X
+2500 wide x 40 rows (write parquet)                 130            151          42          0.8        1300.7       0.2X
 
 
 ================================================================================================
 deeply nested struct field read and write
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 deeply nested struct field r/w:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-1 deep x 100000 rows (read in-mem)                   37             41           6          2.7         374.5       1.0X
-1 deep x 100000 rows (exec in-mem)                   47             50           6          2.1         466.9       0.8X
-1 deep x 100000 rows (read parquet)                  58             61           7          1.7         577.7       0.6X
-1 deep x 100000 rows (write parquet)                128            134          18          0.8        1282.2       0.3X
-100 deep x 1000 rows (read in-mem)                  345            350           5          0.3        3447.8       0.1X
-100 deep x 1000 rows (exec in-mem)                 1283           1283           0          0.1       12830.5       0.0X
-100 deep x 1000 rows (read parquet)                1201           1205           7          0.1       12005.2       0.0X
-100 deep x 1000 rows (write parquet)                436            443           9          0.2        4361.4       0.1X
-250 deep x 400 rows (read in-mem)                  1882           1883           1          0.1       18819.9       0.0X
-250 deep x 400 rows (exec in-mem)                  7705           7709           5          0.0       77054.4       0.0X
-250 deep x 400 rows (read parquet)                 7052           7087          50          0.0       70517.1       0.0X
-250 deep x 400 rows (write parquet)                1978           1979           1          0.1       19780.3       0.0X
+1 deep x 100000 rows (read in-mem)                   21             25           4          4.7         214.6       1.0X
+1 deep x 100000 rows (exec in-mem)                   27             31           6          3.7         266.7       0.8X
+1 deep x 100000 rows (read parquet)                  43             51          10          2.3         431.2       0.5X
+1 deep x 100000 rows (write parquet)                121            125           4          0.8        1205.3       0.2X
+100 deep x 1000 rows (read in-mem)                   71             77           4          1.4         707.6       0.3X
+100 deep x 1000 rows (exec in-mem)                  675            685          13          0.1        6746.2       0.0X
+100 deep x 1000 rows (read parquet)                 583            588           9          0.2        5832.6       0.0X
+100 deep x 1000 rows (write parquet)                169            183          15          0.6        1687.8       0.1X
+250 deep x 400 rows (read in-mem)                   264            276          11          0.4        2642.9       0.1X
+250 deep x 400 rows (exec in-mem)                  4254           4289          50          0.0       42538.9       0.0X
+250 deep x 400 rows (read parquet)                 3280           3348          96          0.0       32796.7       0.0X
+250 deep x 400 rows (write parquet)                 392            406          14          0.3        3915.9       0.1X
 
 
 ================================================================================================
 bushy struct field read and write
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-bushy struct field r/w:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-1 x 1 deep x 100000 rows (read in-mem)               34             39           7          2.9         341.5       1.0X
-1 x 1 deep x 100000 rows (exec in-mem)               42             45           5          2.4         423.4       0.8X
-1 x 1 deep x 100000 rows (read parquet)              42             45           6          2.4         423.8       0.8X
-1 x 1 deep x 100000 rows (write parquet)            124            132          19          0.8        1240.4       0.3X
-128 x 8 deep x 1000 rows (read in-mem)               39             42           6          2.6         387.3       0.9X
-128 x 8 deep x 1000 rows (exec in-mem)              134            138           6          0.7        1342.5       0.3X
-128 x 8 deep x 1000 rows (read parquet)             147            164          27          0.7        1468.2       0.2X
-128 x 8 deep x 1000 rows (write parquet)            130            142          34          0.8        1297.7       0.3X
-1024 x 11 deep x 100 rows (read in-mem)              64             68          11          1.6         639.3       0.5X
-1024 x 11 deep x 100 rows (exec in-mem)             642            652          14          0.2        6416.9       0.1X
-1024 x 11 deep x 100 rows (read parquet)            527            531           5          0.2        5268.1       0.1X
-1024 x 11 deep x 100 rows (write parquet)            155            166          28          0.6        1545.0       0.2X
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+bushy struct field r/w:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+-------------------------------------------------------------------------------------------------------------------------
+1 x 1 deep x 100000 rows (read in-mem)                19             22           5          5.3         189.0       1.0X
+1 x 1 deep x 100000 rows (exec in-mem)                23             29           5          4.3         231.4       0.8X
+1 x 1 deep x 100000 rows (read parquet)               26             31           7          3.8         264.7       0.7X
+1 x 1 deep x 100000 rows (write parquet)             119            136          44          0.8        1189.3       0.2X
+128 x 8 deep x 1000 rows (read in-mem)                21             24           4          4.8         208.8       0.9X
+128 x 8 deep x 1000 rows (exec in-mem)                56             62           4          1.8         561.8       0.3X
+128 x 8 deep x 1000 rows (read parquet)               35             40           7          2.8         353.0       0.5X
+128 x 8 deep x 1000 rows (write parquet)             119            128           6          0.8        1193.9       0.2X
+1024 x 11 deep x 100 rows (read in-mem)               27             31           4          3.7         269.0       0.7X
+1024 x 11 deep x 100 rows (exec in-mem)              366            377           9          0.3        3661.4       0.1X
+1024 x 11 deep x 100 rows (read parquet)              70             74           6          1.4         698.0       0.3X
+1024 x 11 deep x 100 rows (write parquet)            131            141           6          0.8        1314.8       0.1X
 
 
 ================================================================================================
 wide array field read and write
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 wide array field r/w:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-1 wide x 100000 rows (read in-mem)                   36             39           5          2.7         364.2       1.0X
-1 wide x 100000 rows (exec in-mem)                   46             50           7          2.2         460.4       0.8X
-1 wide x 100000 rows (read parquet)                  75             78           8          1.3         749.8       0.5X
-1 wide x 100000 rows (write parquet)                127            133          19          0.8        1266.0       0.3X
-100 wide x 1000 rows (read in-mem)                   31             33           4          3.2         309.9       1.2X
-100 wide x 1000 rows (exec in-mem)                   40             42           4          2.5         397.3       0.9X
-100 wide x 1000 rows (read parquet)                  49             52           7          2.0         488.6       0.7X
-100 wide x 1000 rows (write parquet)                122            135          23          0.8        1216.2       0.3X
-2500 wide x 40 rows (read in-mem)                    31             32           3          3.3         305.7       1.2X
-2500 wide x 40 rows (exec in-mem)                    39             42           5          2.6         391.9       0.9X
-2500 wide x 40 rows (read parquet)                   48             51           7          2.1         482.9       0.8X
-2500 wide x 40 rows (write parquet)                 120            130          22          0.8        1203.6       0.3X
+1 wide x 100000 rows (read in-mem)                   20             24           4          4.9         204.1       1.0X
+1 wide x 100000 rows (exec in-mem)                   26             31           4          3.8         263.6       0.8X
+1 wide x 100000 rows (read parquet)                  57             64           8          1.8         565.8       0.4X
+1 wide x 100000 rows (write parquet)                115            129           6          0.9        1148.2       0.2X
+100 wide x 1000 rows (read in-mem)                   17             20           6          5.9         170.0       1.2X
+100 wide x 1000 rows (exec in-mem)                   21             24           4          4.7         214.3       1.0X
+100 wide x 1000 rows (read parquet)                  30             36           7          3.3         303.9       0.7X
+100 wide x 1000 rows (write parquet)                113            120           4          0.9        1127.2       0.2X
+2500 wide x 40 rows (read in-mem)                    17             21           4          5.9         169.9       1.2X
+2500 wide x 40 rows (exec in-mem)                    21             26           5          4.7         214.3       1.0X
+2500 wide x 40 rows (read parquet)                   30             35           7          3.3         300.8       0.7X
+2500 wide x 40 rows (write parquet)                 116            122           4          0.9        1161.4       0.2X
 
 
 ================================================================================================
 wide map field read and write
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 wide map field r/w:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-1 wide x 100000 rows (read in-mem)                   35             40           8          2.9         348.8       1.0X
-1 wide x 100000 rows (exec in-mem)                   46             47           2          2.2         461.8       0.8X
-1 wide x 100000 rows (read parquet)                 124            127           7          0.8        1236.1       0.3X
-1 wide x 100000 rows (write parquet)                125            138          26          0.8        1245.4       0.3X
-100 wide x 1000 rows (read in-mem)                   26             35           8          3.8         263.1       1.3X
-100 wide x 1000 rows (exec in-mem)                   35             41          10          2.8         351.8       1.0X
-100 wide x 1000 rows (read parquet)                  59             62           8          1.7         586.7       0.6X
-100 wide x 1000 rows (write parquet)                116            125          32          0.9        1158.2       0.3X
-2500 wide x 40 rows (read in-mem)                    27             30           5          3.7         270.2       1.3X
-2500 wide x 40 rows (exec in-mem)                    37             38           3          2.7         366.4       1.0X
-2500 wide x 40 rows (read parquet)                   58             62           8          1.7         584.3       0.6X
-2500 wide x 40 rows (write parquet)                 118            126          24          0.9        1176.1       0.3X
+1 wide x 100000 rows (read in-mem)                   19             23           3          5.1         194.5       1.0X
+1 wide x 100000 rows (exec in-mem)                   26             29           2          3.8         260.1       0.7X
+1 wide x 100000 rows (read parquet)                  79             88           7          1.3         785.1       0.2X
+1 wide x 100000 rows (write parquet)                120            125           5          0.8        1195.5       0.2X
+100 wide x 1000 rows (read in-mem)                   13             15           3          7.7         130.1       1.5X
+100 wide x 1000 rows (exec in-mem)                   18             20           2          5.6         177.7       1.1X
+100 wide x 1000 rows (read parquet)                  38             43           7          2.6         383.8       0.5X
+100 wide x 1000 rows (write parquet)                113            121           6          0.9        1130.0       0.2X
+2500 wide x 40 rows (read in-mem)                    15             17           4          6.9         145.0       1.3X
+2500 wide x 40 rows (exec in-mem)                    20             22           3          5.1         195.5       1.0X
+2500 wide x 40 rows (read parquet)                   38             44           8          2.6         383.8       0.5X
+2500 wide x 40 rows (write parquet)                 114            120           4          0.9        1137.3       0.2X
 
 
diff --git a/sql/core/benchmarks/WideTableBenchmark-jdk11-results.txt b/sql/core/benchmarks/WideTableBenchmark-jdk11-results.txt
index 8f3920db0dcd9..4a29255009faa 100644
--- a/sql/core/benchmarks/WideTableBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/WideTableBenchmark-jdk11-results.txt
@@ -2,16 +2,16 @@
 projection on wide table
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 projection on wide table:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-split threshold 10                                 8915           9048         180          0.1        8501.7       1.0X
-split threshold 100                                4419           4465          31          0.2        4214.2       2.0X
-split threshold 1024                               2477           2559          57          0.4        2362.4       3.6X
-split threshold 2048                               2314           2391          74          0.5        2206.7       3.9X
-split threshold 4096                               2374           2399          18          0.4        2264.2       3.8X
-split threshold 8192                               2831           2846          11          0.4        2699.7       3.1X
-split threshold 65536                             26886          26944          55          0.0       25640.7       0.3X
+split threshold 10                                 3882           3963          54          0.3        3702.2       1.0X
+split threshold 100                                1810           1821           8          0.6        1726.1       2.1X
+split threshold 1024                               1435           1457          47          0.7        1368.3       2.7X
+split threshold 2048                               1357           1369          13          0.8        1294.3       2.9X
+split threshold 4096                               1902           1921          24          0.6        1814.1       2.0X
+split threshold 8192                               2700           2719          32          0.4        2574.7       1.4X
+split threshold 65536                             26623          26688         125          0.0       25389.7       0.1X
 
 
diff --git a/sql/core/benchmarks/WideTableBenchmark-results.txt b/sql/core/benchmarks/WideTableBenchmark-results.txt
index 4f64f877ac0c8..45aaad3817121 100644
--- a/sql/core/benchmarks/WideTableBenchmark-results.txt
+++ b/sql/core/benchmarks/WideTableBenchmark-results.txt
@@ -2,16 +2,16 @@
 projection on wide table
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
 projection on wide table:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-split threshold 10                                 6461           6535          84          0.2        6161.9       1.0X
-split threshold 100                                3643           3725          74          0.3        3474.5       1.8X
-split threshold 1024                               2217           2255          26          0.5        2113.9       2.9X
-split threshold 2048                               1941           2003          60          0.5        1851.5       3.3X
-split threshold 4096                               2195           2220          20          0.5        2093.4       2.9X
-split threshold 8192                               2592           2652          39          0.4        2472.4       2.5X
-split threshold 65536                             26324          26365          66          0.0       25104.1       0.2X
+split threshold 10                                 1897           1937          46          0.6        1809.2       1.0X
+split threshold 100                                1461           1493          29          0.7        1393.2       1.3X
+split threshold 1024                               1106           1126          14          0.9        1054.3       1.7X
+split threshold 2048                               1064           1104          62          1.0        1014.4       1.8X
+split threshold 4096                               1370           1401          49          0.8        1306.9       1.4X
+split threshold 8192                               1887           1966          56          0.6        1799.4       1.0X
+split threshold 65536                             20021          20148         133          0.1       19093.4       0.1X
 
 
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 7c5fcba9c2131..5ab66bd5aac8a 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
@@ -88,12 +88,10 @@
     <dependency>
       <groupId>org.apache.orc</groupId>
       <artifactId>orc-core</artifactId>
-      <classifier>${orc.classifier}</classifier>
     </dependency>
     <dependency>
       <groupId>org.apache.orc</groupId>
       <artifactId>orc-mapreduce</artifactId>
-      <classifier>${orc.classifier}</classifier>
     </dependency>
     <dependency>
       <groupId>org.apache.hive</groupId>
@@ -145,6 +143,16 @@
       <artifactId>jcc</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.microsoft.sqlserver</groupId>
+      <artifactId>mssql-jdbc</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.oracle.database.jdbc</groupId>
+      <artifactId>ojdbc8</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.parquet</groupId>
       <artifactId>parquet-avro</artifactId>
@@ -162,7 +170,7 @@
     </dependency>
     <dependency>
       <groupId>org.seleniumhq.selenium</groupId>
-      <artifactId>selenium-htmlunit-driver</artifactId>
+      <artifactId>htmlunit-driver</artifactId>
       <scope>test</scope>
     </dependency>
   </dependencies>
@@ -211,8 +219,6 @@
             </goals>
             <configuration>
               <sources>
-                <source>v${hive.version.short}/src/main/scala</source>
-                <source>v${hive.version.short}/src/main/java</source>
                 <source>src/main/scala-${scala.binary.version}</source>
               </sources>
             </configuration>
@@ -225,7 +231,6 @@
             </goals>
             <configuration>
               <sources>
-                <source>v${hive.version.short}/src/test/scala</source>
                 <source>src/test/gen-java</source>
               </sources>
             </configuration>
diff --git a/sql/core/src/main/java/org/apache/spark/sql/connector/write/V1Write.java b/sql/core/src/main/java/org/apache/spark/sql/connector/write/V1Write.java
new file mode 100644
index 0000000000000..a299967ee8bcf
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/connector/write/V1Write.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.write;
+
+import org.apache.spark.annotation.Unstable;
+import org.apache.spark.sql.connector.catalog.TableCapability;
+import org.apache.spark.sql.sources.InsertableRelation;
+
+/**
+ * A logical write that should be executed using V1 InsertableRelation interface.
+ * <p>
+ * Tables that have {@link TableCapability#V1_BATCH_WRITE} in the list of their capabilities
+ * must build {@link V1Write}.
+ */
+@Unstable
+public interface V1Write extends Write {
+  InsertableRelation toInsertableRelation();
+}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/connector/write/V1WriteBuilder.java b/sql/core/src/main/java/org/apache/spark/sql/connector/write/V1WriteBuilder.java
deleted file mode 100644
index 89b567b5231ac..0000000000000
--- a/sql/core/src/main/java/org/apache/spark/sql/connector/write/V1WriteBuilder.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.connector.write;
-
-import org.apache.spark.annotation.Unstable;
-import org.apache.spark.sql.sources.InsertableRelation;
-
-/**
- * A trait that should be implemented by V1 DataSources that would like to leverage the DataSource
- * V2 write code paths. The InsertableRelation will be used only to Append data. Other
- * instances of the [[WriteBuilder]] interface such as [[SupportsOverwrite]], [[SupportsTruncate]]
- * should be extended as well to support additional operations other than data appends.
- *
- * This interface is designed to provide Spark DataSources time to migrate to DataSource V2 and
- * will be removed in a future Spark release.
- *
- * @since 3.0.0
- */
-@Unstable
-public interface V1WriteBuilder extends WriteBuilder {
-  /**
-   * Creates an InsertableRelation that allows appending a DataFrame to a
-   * a destination (using data source-specific parameters). The insert method will only be
-   * called with `overwrite=false`. The DataSource should implement the overwrite behavior as
-   * part of the [[SupportsOverwrite]], and [[SupportsTruncate]] interfaces.
-   *
-   * @since 3.0.0
-   */
-  InsertableRelation buildForV1Write();
-}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java b/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java
index 7a9f61a2cc33d..42ceebccdd1ee 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java
@@ -165,7 +165,8 @@ public UnsafeKVExternalSorter(
         (int) (long) SparkEnv.get().conf().get(package$.MODULE$.SHUFFLE_SORT_INIT_BUFFER_SIZE()),
         pageSizeBytes,
         numElementsForSpillThreshold,
-        inMemSorter);
+        inMemSorter,
+        map.getTotalMemoryConsumption());
 
       // reset the map, so we can re-use it to insert new records. the inMemSorter will not used
       // anymore, so the underline array could be used by map again.
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcArrayColumnVector.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcArrayColumnVector.java
new file mode 100644
index 0000000000000..6e13e97b4cbcc
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcArrayColumnVector.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.orc;
+
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+
+import org.apache.spark.sql.types.ArrayType;
+import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.types.Decimal;
+import org.apache.spark.sql.vectorized.ColumnarArray;
+import org.apache.spark.sql.vectorized.ColumnarMap;
+import org.apache.spark.unsafe.types.UTF8String;
+
+/**
+ * A column vector implementation for Spark's {@link ArrayType}.
+ */
+public class OrcArrayColumnVector extends OrcColumnVector {
+  private final OrcColumnVector data;
+  private final long[] offsets;
+  private final long[] lengths;
+
+  OrcArrayColumnVector(
+      DataType type,
+      ColumnVector vector,
+      OrcColumnVector data,
+      long[] offsets,
+      long[] lengths) {
+
+    super(type, vector);
+
+    this.data = data;
+    this.offsets = offsets;
+    this.lengths = lengths;
+  }
+
+  @Override
+  public ColumnarArray getArray(int rowId) {
+    return new ColumnarArray(data, (int) offsets[rowId], (int) lengths[rowId]);
+  }
+
+  @Override
+  public boolean getBoolean(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public byte getByte(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public short getShort(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public int getInt(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public long getLong(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public float getFloat(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public double getDouble(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public Decimal getDecimal(int rowId, int precision, int scale) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public UTF8String getUTF8String(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public byte[] getBinary(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ColumnarMap getMap(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public org.apache.spark.sql.vectorized.ColumnVector getChild(int ordinal) {
+    throw new UnsupportedOperationException();
+  }
+}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcAtomicColumnVector.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcAtomicColumnVector.java
new file mode 100644
index 0000000000000..c2d8334d928c0
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcAtomicColumnVector.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.orc;
+
+import java.math.BigDecimal;
+
+import org.apache.hadoop.hive.ql.exec.vector.*;
+
+import org.apache.spark.sql.catalyst.util.DateTimeUtils;
+import org.apache.spark.sql.catalyst.util.RebaseDateTime;
+import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.types.DateType;
+import org.apache.spark.sql.types.Decimal;
+import org.apache.spark.sql.types.TimestampType;
+import org.apache.spark.sql.vectorized.ColumnarArray;
+import org.apache.spark.sql.vectorized.ColumnarMap;
+import org.apache.spark.unsafe.types.UTF8String;
+
+/**
+ * A column vector implementation for Spark's AtomicType.
+ */
+public class OrcAtomicColumnVector extends OrcColumnVector {
+  private final boolean isTimestamp;
+  private final boolean isDate;
+
+  // Column vector for each type. Only 1 is populated for any type.
+  private LongColumnVector longData;
+  private DoubleColumnVector doubleData;
+  private BytesColumnVector bytesData;
+  private DecimalColumnVector decimalData;
+  private TimestampColumnVector timestampData;
+
+  OrcAtomicColumnVector(DataType type, ColumnVector vector) {
+    super(type, vector);
+
+    if (type instanceof TimestampType) {
+      isTimestamp = true;
+    } else {
+      isTimestamp = false;
+    }
+
+    if (type instanceof DateType) {
+      isDate = true;
+    } else {
+      isDate = false;
+    }
+
+    if (vector instanceof LongColumnVector) {
+      longData = (LongColumnVector) vector;
+    } else if (vector instanceof DoubleColumnVector) {
+      doubleData = (DoubleColumnVector) vector;
+    } else if (vector instanceof BytesColumnVector) {
+      bytesData = (BytesColumnVector) vector;
+    } else if (vector instanceof DecimalColumnVector) {
+      decimalData = (DecimalColumnVector) vector;
+    } else if (vector instanceof TimestampColumnVector) {
+      timestampData = (TimestampColumnVector) vector;
+    } else {
+      throw new UnsupportedOperationException();
+    }
+  }
+
+  @Override
+  public boolean getBoolean(int rowId) {
+    return longData.vector[getRowIndex(rowId)] == 1;
+  }
+
+  @Override
+  public byte getByte(int rowId) {
+    return (byte) longData.vector[getRowIndex(rowId)];
+  }
+
+  @Override
+  public short getShort(int rowId) {
+    return (short) longData.vector[getRowIndex(rowId)];
+  }
+
+  @Override
+  public int getInt(int rowId) {
+    int value = (int) longData.vector[getRowIndex(rowId)];
+    if (isDate) {
+      return RebaseDateTime.rebaseJulianToGregorianDays(value);
+    } else {
+      return value;
+    }
+  }
+
+  @Override
+  public long getLong(int rowId) {
+    int index = getRowIndex(rowId);
+    if (isTimestamp) {
+      return DateTimeUtils.fromJavaTimestamp(timestampData.asScratchTimestamp(index));
+    } else {
+      return longData.vector[index];
+    }
+  }
+
+  @Override
+  public float getFloat(int rowId) {
+    return (float) doubleData.vector[getRowIndex(rowId)];
+  }
+
+  @Override
+  public double getDouble(int rowId) {
+    return doubleData.vector[getRowIndex(rowId)];
+  }
+
+  @Override
+  public Decimal getDecimal(int rowId, int precision, int scale) {
+    if (isNullAt(rowId)) return null;
+    BigDecimal data = decimalData.vector[getRowIndex(rowId)].getHiveDecimal().bigDecimalValue();
+    return Decimal.apply(data, precision, scale);
+  }
+
+  @Override
+  public UTF8String getUTF8String(int rowId) {
+    if (isNullAt(rowId)) return null;
+    int index = getRowIndex(rowId);
+    BytesColumnVector col = bytesData;
+    return UTF8String.fromBytes(col.vector[index], col.start[index], col.length[index]);
+  }
+
+  @Override
+  public byte[] getBinary(int rowId) {
+    if (isNullAt(rowId)) return null;
+    int index = getRowIndex(rowId);
+    byte[] binary = new byte[bytesData.length[index]];
+    System.arraycopy(bytesData.vector[index], bytesData.start[index], binary, 0, binary.length);
+    return binary;
+  }
+
+  @Override
+  public ColumnarArray getArray(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ColumnarMap getMap(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public org.apache.spark.sql.vectorized.ColumnVector getChild(int ordinal) {
+    throw new UnsupportedOperationException();
+  }
+}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
new file mode 100644
index 0000000000000..0becd2572f99c
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.orc;
+
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+
+import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.vectorized.ColumnarBatch;
+
+/**
+ * A column vector interface wrapping Hive's {@link ColumnVector}.
+ *
+ * Because Spark {@link ColumnarBatch} only accepts Spark's vectorized.ColumnVector,
+ * this column vector is used to adapt Hive ColumnVector with Spark ColumnarVector.
+ */
+public abstract class OrcColumnVector extends org.apache.spark.sql.vectorized.ColumnVector {
+  private final ColumnVector baseData;
+  private int batchSize;
+
+  OrcColumnVector(DataType type, ColumnVector vector) {
+    super(type);
+
+    baseData = vector;
+  }
+
+  @Override
+  public void close() {
+  }
+
+  @Override
+  public boolean hasNull() {
+    return !baseData.noNulls;
+  }
+
+  @Override
+  public int numNulls() {
+    if (baseData.isRepeating) {
+      if (baseData.isNull[0]) {
+        return batchSize;
+      } else {
+        return 0;
+      }
+    } else if (baseData.noNulls) {
+      return 0;
+    } else {
+      int count = 0;
+      for (int i = 0; i < batchSize; i++) {
+        if (baseData.isNull[i]) count++;
+      }
+      return count;
+    }
+  }
+
+  @Override
+  public boolean isNullAt(int rowId) {
+    return baseData.isNull[getRowIndex(rowId)];
+  }
+
+
+  public void setBatchSize(int batchSize) {
+    this.batchSize = batchSize;
+  }
+
+  /* A helper method to get the row index in a column. */
+  protected int getRowIndex(int rowId) {
+    return baseData.isRepeating ? 0 : rowId;
+  }
+}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVectorUtils.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVectorUtils.java
new file mode 100644
index 0000000000000..3bc7cc8f80142
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVectorUtils.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.orc;
+
+import org.apache.hadoop.hive.ql.exec.vector.*;
+
+import org.apache.spark.sql.types.*;
+
+/**
+ * Utility class for {@link OrcColumnVector}.
+ */
+class OrcColumnVectorUtils {
+
+  /**
+   * Convert a Hive's {@link ColumnVector} to a Spark's {@link OrcColumnVector}.
+   *
+   * @param type The data type of column vector
+   * @param vector Hive's column vector
+   * @return Spark's column vector
+   */
+  static OrcColumnVector toOrcColumnVector(DataType type, ColumnVector vector) {
+    if (vector instanceof LongColumnVector ||
+      vector instanceof DoubleColumnVector ||
+      vector instanceof BytesColumnVector ||
+      vector instanceof DecimalColumnVector ||
+      vector instanceof TimestampColumnVector) {
+      return new OrcAtomicColumnVector(type, vector);
+    } else if (vector instanceof StructColumnVector) {
+      StructColumnVector structVector = (StructColumnVector) vector;
+      OrcColumnVector[] fields = new OrcColumnVector[structVector.fields.length];
+      int ordinal = 0;
+      for (StructField f : ((StructType) type).fields()) {
+        fields[ordinal] = toOrcColumnVector(f.dataType(), structVector.fields[ordinal]);
+        ordinal++;
+      }
+      return new OrcStructColumnVector(type, vector, fields);
+    } else if (vector instanceof ListColumnVector) {
+      ListColumnVector listVector = (ListColumnVector) vector;
+      OrcColumnVector dataVector = toOrcColumnVector(
+        ((ArrayType) type).elementType(), listVector.child);
+      return new OrcArrayColumnVector(
+        type, vector, dataVector, listVector.offsets, listVector.lengths);
+    } else if (vector instanceof MapColumnVector) {
+      MapColumnVector mapVector = (MapColumnVector) vector;
+      MapType mapType = (MapType) type;
+      OrcColumnVector keysVector = toOrcColumnVector(mapType.keyType(), mapVector.keys);
+      OrcColumnVector valuesVector = toOrcColumnVector(mapType.valueType(), mapVector.values);
+      return new OrcMapColumnVector(
+        type, vector, keysVector, valuesVector, mapVector.offsets, mapVector.lengths);
+    } else {
+      throw new IllegalArgumentException(
+        String.format("OrcColumnVectorUtils.toOrcColumnVector should not take %s as type " +
+          "and %s as vector", type, vector));
+    }
+  }
+}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReader.java
index 6a4b116cdef0b..40ed0b2454c12 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReader.java
@@ -180,7 +180,8 @@ public void initBatch(
           missingCol.setIsConstant();
           orcVectorWrappers[i] = missingCol;
         } else {
-          orcVectorWrappers[i] = new OrcColumnVector(dt, wrap.batch().cols[colId]);
+          orcVectorWrappers[i] = OrcColumnVectorUtils.toOrcColumnVector(
+            dt, wrap.batch().cols[colId]);
         }
       }
     }
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcMapColumnVector.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcMapColumnVector.java
new file mode 100644
index 0000000000000..ace8d157792dc
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcMapColumnVector.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.orc;
+
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+
+import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.types.Decimal;
+import org.apache.spark.sql.types.MapType;
+import org.apache.spark.sql.vectorized.ColumnarArray;
+import org.apache.spark.sql.vectorized.ColumnarMap;
+import org.apache.spark.unsafe.types.UTF8String;
+
+/**
+ * A column vector implementation for Spark's {@link MapType}.
+ */
+public class OrcMapColumnVector extends OrcColumnVector {
+  private final OrcColumnVector keys;
+  private final OrcColumnVector values;
+  private final long[] offsets;
+  private final long[] lengths;
+
+  OrcMapColumnVector(
+      DataType type,
+      ColumnVector vector,
+      OrcColumnVector keys,
+      OrcColumnVector values,
+      long[] offsets,
+      long[] lengths) {
+
+    super(type, vector);
+
+    this.keys = keys;
+    this.values = values;
+    this.offsets = offsets;
+    this.lengths = lengths;
+  }
+
+  @Override
+  public ColumnarMap getMap(int ordinal) {
+    return new ColumnarMap(keys, values, (int) offsets[ordinal], (int) lengths[ordinal]);
+  }
+
+  @Override
+  public boolean getBoolean(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public byte getByte(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public short getShort(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public int getInt(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public long getLong(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public float getFloat(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public double getDouble(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public Decimal getDecimal(int rowId, int precision, int scale) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public UTF8String getUTF8String(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public byte[] getBinary(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ColumnarArray getArray(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public org.apache.spark.sql.vectorized.ColumnVector getChild(int ordinal) {
+    throw new UnsupportedOperationException();
+  }
+}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcStructColumnVector.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcStructColumnVector.java
new file mode 100644
index 0000000000000..48e540d22095e
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcStructColumnVector.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.orc;
+
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+
+import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.types.Decimal;
+import org.apache.spark.sql.types.StructType;
+import org.apache.spark.sql.vectorized.ColumnarArray;
+import org.apache.spark.sql.vectorized.ColumnarMap;
+import org.apache.spark.unsafe.types.UTF8String;
+
+/**
+ * A column vector implementation for Spark's {@link StructType}.
+ */
+public class OrcStructColumnVector extends OrcColumnVector {
+  private final OrcColumnVector[] fields;
+
+  OrcStructColumnVector(DataType type, ColumnVector vector, OrcColumnVector[] fields) {
+    super(type, vector);
+
+    this.fields = fields;
+  }
+
+  @Override
+  public org.apache.spark.sql.vectorized.ColumnVector getChild(int ordinal) {
+    return fields[ordinal];
+  }
+
+  @Override
+  public boolean getBoolean(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public byte getByte(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public short getShort(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public int getInt(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public long getLong(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public float getFloat(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public double getDouble(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public Decimal getDecimal(int rowId, int precision, int scale) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public UTF8String getUTF8String(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public byte[] getBinary(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ColumnarArray getArray(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ColumnarMap getMap(int rowId) {
+    throw new UnsupportedOperationException();
+  }
+}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/ParquetDictionary.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/ParquetDictionary.java
index 0930edeb352dc..6626f3fee982c 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/ParquetDictionary.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/ParquetDictionary.java
@@ -19,21 +19,36 @@
 
 import org.apache.spark.sql.execution.vectorized.Dictionary;
 
+import java.math.BigInteger;
+
 public final class ParquetDictionary implements Dictionary {
   private org.apache.parquet.column.Dictionary dictionary;
+  private boolean needTransform = false;
 
-  public ParquetDictionary(org.apache.parquet.column.Dictionary dictionary) {
+  public ParquetDictionary(org.apache.parquet.column.Dictionary dictionary, boolean needTransform) {
     this.dictionary = dictionary;
+    this.needTransform = needTransform;
   }
 
   @Override
   public int decodeToInt(int id) {
-    return dictionary.decodeToInt(id);
+    if (needTransform) {
+      return (int) dictionary.decodeToLong(id);
+    } else {
+      return dictionary.decodeToInt(id);
+    }
   }
 
   @Override
   public long decodeToLong(int id) {
-    return dictionary.decodeToLong(id);
+    if (needTransform) {
+      // For unsigned int32, it stores as dictionary encoded signed int32 in Parquet
+      // whenever dictionary is available.
+      // Here we lazily decode it to the original signed int value then convert to long(unit32).
+      return Integer.toUnsignedLong(dictionary.decodeToInt(id));
+    } else {
+      return dictionary.decodeToLong(id);
+    }
   }
 
   @Override
@@ -48,6 +63,14 @@ public double decodeToDouble(int id) {
 
   @Override
   public byte[] decodeToBinary(int id) {
-    return dictionary.decodeToBinary(id).getBytes();
+    if (needTransform) {
+      // For unsigned int64, it stores as dictionary encoded signed int64 in Parquet
+      // whenever dictionary is available.
+      // Here we lazily decode it to the original signed long value then convert to decimal(20, 0).
+      long signed = dictionary.decodeToLong(id);
+      return new BigInteger(Long.toUnsignedString(signed)).toByteArray();
+    } else {
+      return dictionary.decodeToBinary(id).getBytes();
+    }
   }
 }
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/ParquetFooterReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/ParquetFooterReader.java
new file mode 100644
index 0000000000000..ab07b19d08250
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/ParquetFooterReader.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.parquet;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.parquet.HadoopReadOptions;
+import org.apache.parquet.ParquetReadOptions;
+import org.apache.parquet.format.converter.ParquetMetadataConverter;
+import org.apache.parquet.hadoop.ParquetFileReader;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
+import org.apache.parquet.hadoop.util.HadoopInputFile;
+
+import java.io.IOException;
+
+/**
+ * `ParquetFooterReader` is a util class which encapsulates the helper
+ * methods of reading parquet file footer
+ */
+public class ParquetFooterReader {
+  public static ParquetMetadata readFooter(Configuration configuration,
+      Path file, ParquetMetadataConverter.MetadataFilter filter) throws IOException {
+    return readFooter(HadoopInputFile.fromPath(file, configuration), filter);
+  }
+
+  public static ParquetMetadata readFooter(Configuration configuration,
+      FileStatus fileStatus, ParquetMetadataConverter.MetadataFilter filter) throws IOException {
+    return readFooter(HadoopInputFile.fromStatus(fileStatus, configuration), filter);
+  }
+
+  private static ParquetMetadata readFooter(HadoopInputFile inputFile,
+      ParquetMetadataConverter.MetadataFilter filter) throws IOException {
+    ParquetReadOptions readOptions =
+      HadoopReadOptions.builder(inputFile.getConfiguration()).withMetadataFilter(filter).build();
+    // Use try-with-resources to ensure fd is closed.
+    try (ParquetFileReader fileReader = ParquetFileReader.open(inputFile, readOptions)) {
+      return fileReader.getFooter();
+    }
+  }
+}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java
index c975e52734e01..6264d6341c65a 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java
@@ -22,7 +22,6 @@
 import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -32,36 +31,26 @@
 
 import scala.Option;
 
-import static org.apache.parquet.filter2.compat.RowGroupFilter.filterRowGroups;
-import static org.apache.parquet.format.converter.ParquetMetadataConverter.NO_FILTER;
-import static org.apache.parquet.format.converter.ParquetMetadataConverter.range;
-import static org.apache.parquet.hadoop.ParquetFileReader.readFooter;
-import static org.apache.parquet.hadoop.ParquetInputFormat.getFilter;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.FileSplit;
 import org.apache.hadoop.mapreduce.InputSplit;
 import org.apache.hadoop.mapreduce.RecordReader;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.parquet.bytes.BytesInput;
-import org.apache.parquet.bytes.BytesUtils;
-import org.apache.parquet.column.ColumnDescriptor;
-import org.apache.parquet.column.values.ValuesReader;
-import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridDecoder;
-import org.apache.parquet.filter2.compat.FilterCompat;
+import org.apache.parquet.HadoopReadOptions;
+import org.apache.parquet.ParquetReadOptions;
 import org.apache.parquet.hadoop.BadConfigurationException;
 import org.apache.parquet.hadoop.ParquetFileReader;
 import org.apache.parquet.hadoop.ParquetInputFormat;
-import org.apache.parquet.hadoop.ParquetInputSplit;
 import org.apache.parquet.hadoop.api.InitContext;
 import org.apache.parquet.hadoop.api.ReadSupport;
-import org.apache.parquet.hadoop.metadata.BlockMetaData;
-import org.apache.parquet.hadoop.metadata.ParquetMetadata;
 import org.apache.parquet.hadoop.util.ConfigurationUtil;
+import org.apache.parquet.hadoop.util.HadoopInputFile;
 import org.apache.parquet.schema.MessageType;
 import org.apache.parquet.schema.Types;
 import org.apache.spark.TaskContext;
 import org.apache.spark.TaskContext$;
+import org.apache.spark.sql.internal.SQLConf;
 import org.apache.spark.sql.types.StructType;
 import org.apache.spark.sql.types.StructType$;
 import org.apache.spark.util.AccumulatorV2;
@@ -92,64 +81,25 @@ public abstract class SpecificParquetRecordReaderBase<T> extends RecordReader<Vo
   public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
       throws IOException, InterruptedException {
     Configuration configuration = taskAttemptContext.getConfiguration();
-    ParquetInputSplit split = (ParquetInputSplit)inputSplit;
+    FileSplit split = (FileSplit) inputSplit;
     this.file = split.getPath();
-    long[] rowGroupOffsets = split.getRowGroupOffsets();
-
-    ParquetMetadata footer;
-    List<BlockMetaData> blocks;
 
-    // if task.side.metadata is set, rowGroupOffsets is null
-    if (rowGroupOffsets == null) {
-      // then we need to apply the predicate push down filter
-      footer = readFooter(configuration, file, range(split.getStart(), split.getEnd()));
-      MessageType fileSchema = footer.getFileMetaData().getSchema();
-      FilterCompat.Filter filter = getFilter(configuration);
-      blocks = filterRowGroups(filter, footer.getBlocks(), fileSchema);
-    } else {
-      // otherwise we find the row groups that were selected on the client
-      footer = readFooter(configuration, file, NO_FILTER);
-      Set<Long> offsets = new HashSet<>();
-      for (long offset : rowGroupOffsets) {
-        offsets.add(offset);
-      }
-      blocks = new ArrayList<>();
-      for (BlockMetaData block : footer.getBlocks()) {
-        if (offsets.contains(block.getStartingPos())) {
-          blocks.add(block);
-        }
-      }
-      // verify we found them all
-      if (blocks.size() != rowGroupOffsets.length) {
-        long[] foundRowGroupOffsets = new long[footer.getBlocks().size()];
-        for (int i = 0; i < foundRowGroupOffsets.length; i++) {
-          foundRowGroupOffsets[i] = footer.getBlocks().get(i).getStartingPos();
-        }
-        // this should never happen.
-        // provide a good error message in case there's a bug
-        throw new IllegalStateException(
-            "All the offsets listed in the split should be found in the file."
-                + " expected: " + Arrays.toString(rowGroupOffsets)
-                + " found: " + blocks
-                + " out of: " + Arrays.toString(foundRowGroupOffsets)
-                + " in range " + split.getStart() + ", " + split.getEnd());
-      }
-    }
-    this.fileSchema = footer.getFileMetaData().getSchema();
-    Map<String, String> fileMetadata = footer.getFileMetaData().getKeyValueMetaData();
+    ParquetReadOptions options = HadoopReadOptions
+      .builder(configuration)
+      .withRange(split.getStart(), split.getStart() + split.getLength())
+      .build();
+    this.reader = new ParquetFileReader(HadoopInputFile.fromPath(file, configuration), options);
+    this.fileSchema = reader.getFileMetaData().getSchema();
+    Map<String, String> fileMetadata = reader.getFileMetaData().getKeyValueMetaData();
     ReadSupport<T> readSupport = getReadSupportInstance(getReadSupportClass(configuration));
     ReadSupport.ReadContext readContext = readSupport.init(new InitContext(
         taskAttemptContext.getConfiguration(), toSetMultiMap(fileMetadata), fileSchema));
     this.requestedSchema = readContext.getRequestedSchema();
+    reader.setRequestedSchema(requestedSchema);
     String sparkRequestedSchemaString =
         configuration.get(ParquetReadSupport$.MODULE$.SPARK_ROW_REQUESTED_SCHEMA());
     this.sparkSchema = StructType$.MODULE$.fromString(sparkRequestedSchemaString);
-    this.reader = new ParquetFileReader(
-        configuration, footer.getFileMetaData(), file, blocks, requestedSchema.getColumns());
-    // use the blocks from the reader in case some do not match filters and will not be read
-    for (BlockMetaData block : reader.getRowGroups()) {
-      this.totalRowCount += block.getRowCount();
-    }
+    this.totalRowCount = reader.getFilteredRecordCount();
 
     // For test purpose.
     // If the last external accumulator is `NumRowGroupsAccumulator`, the row group number to read
@@ -161,7 +111,7 @@ public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptCont
       if (accu.isDefined() && accu.get().getClass().getSimpleName().equals("NumRowGroupsAcc")) {
         @SuppressWarnings("unchecked")
         AccumulatorV2<Integer, Integer> intAccum = (AccumulatorV2<Integer, Integer>) accu.get();
-        intAccum.add(blocks.size());
+        intAccum.add(reader.getRowGroups().size());
       }
     }
   }
@@ -195,15 +145,18 @@ public static List<String> listDirectory(File path) {
    */
   protected void initialize(String path, List<String> columns) throws IOException {
     Configuration config = new Configuration();
-    config.set("spark.sql.parquet.binaryAsString", "false");
-    config.set("spark.sql.parquet.int96AsTimestamp", "false");
+    config.setBoolean(SQLConf.PARQUET_BINARY_AS_STRING().key() , false);
+    config.setBoolean(SQLConf.PARQUET_INT96_AS_TIMESTAMP().key(), false);
 
     this.file = new Path(path);
     long length = this.file.getFileSystem(config).getFileStatus(this.file).getLen();
-    ParquetMetadata footer = readFooter(config, file, range(0, length));
 
-    List<BlockMetaData> blocks = footer.getBlocks();
-    this.fileSchema = footer.getFileMetaData().getSchema();
+    ParquetReadOptions options = HadoopReadOptions
+      .builder(config)
+      .withRange(0, length)
+      .build();
+    this.reader = ParquetFileReader.open(HadoopInputFile.fromPath(file, config), options);
+    this.fileSchema = reader.getFooter().getFileMetaData().getSchema();
 
     if (columns == null) {
       this.requestedSchema = fileSchema;
@@ -222,13 +175,9 @@ protected void initialize(String path, List<String> columns) throws IOException
         this.requestedSchema = ParquetSchemaConverter.EMPTY_MESSAGE();
       }
     }
+    reader.setRequestedSchema(requestedSchema);
     this.sparkSchema = new ParquetToSparkSchemaConverter(config).convert(requestedSchema);
-    this.reader = new ParquetFileReader(
-        config, footer.getFileMetaData(), file, blocks, requestedSchema.getColumns());
-    // use the blocks from the reader in case some do not match filters and will not be read
-    for (BlockMetaData block : reader.getRowGroups()) {
-      this.totalRowCount += block.getRowCount();
-    }
+    this.totalRowCount = reader.getFilteredRecordCount();
   }
 
   @Override
@@ -244,62 +193,6 @@ public void close() throws IOException {
     }
   }
 
-  /**
-   * Utility classes to abstract over different way to read ints with different encodings.
-   * TODO: remove this layer of abstraction?
-   */
-  abstract static class IntIterator {
-    abstract int nextInt() throws IOException;
-  }
-
-  protected static final class ValuesReaderIntIterator extends IntIterator {
-    ValuesReader delegate;
-
-    public ValuesReaderIntIterator(ValuesReader delegate) {
-      this.delegate = delegate;
-    }
-
-    @Override
-    int nextInt() {
-      return delegate.readInteger();
-    }
-  }
-
-  protected static final class RLEIntIterator extends IntIterator {
-    RunLengthBitPackingHybridDecoder delegate;
-
-    public RLEIntIterator(RunLengthBitPackingHybridDecoder delegate) {
-      this.delegate = delegate;
-    }
-
-    @Override
-    int nextInt() throws IOException {
-      return delegate.readInt();
-    }
-  }
-
-  protected static final class NullIntIterator extends IntIterator {
-    @Override
-    int nextInt() { return 0; }
-  }
-
-  /**
-   * Creates a reader for definition and repetition levels, returning an optimized one if
-   * the levels are not needed.
-   */
-  protected static IntIterator createRLEIterator(
-      int maxLevel, BytesInput bytes, ColumnDescriptor descriptor) throws IOException {
-    try {
-      if (maxLevel == 0) return new NullIntIterator();
-      return new RLEIntIterator(
-          new RunLengthBitPackingHybridDecoder(
-              BytesUtils.getWidthFromMaxInt(maxLevel),
-              bytes.toInputStream()));
-    } catch (IOException e) {
-      throw new IOException("could not read levels in page for col " + descriptor, e);
-    }
-  }
-
   private static <K, V> Map<K, Set<V>> toSetMultiMap(Map<K, V> map) {
     Map<K, Set<V>> setMultiMap = new HashMap<>();
     for (Map.Entry<K, V> entry : map.entrySet()) {
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
index 3e409ab9a50a1..52620b0740851 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.execution.datasources.parquet;
 
 import java.io.IOException;
+import java.math.BigInteger;
 import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.util.Arrays;
@@ -31,6 +32,7 @@
 import org.apache.parquet.column.page.*;
 import org.apache.parquet.column.values.ValuesReader;
 import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.DecimalMetadata;
 import org.apache.parquet.schema.OriginalType;
 import org.apache.parquet.schema.PrimitiveType;
 
@@ -39,12 +41,13 @@
 import org.apache.spark.sql.execution.datasources.DataSourceUtils;
 import org.apache.spark.sql.execution.datasources.SchemaColumnConvertNotSupportedException;
 import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
+import org.apache.spark.sql.types.DataType;
 import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.Decimal;
 import org.apache.spark.sql.types.DecimalType;
 
 import static org.apache.parquet.column.ValuesType.REPETITION_LEVEL;
-import static org.apache.spark.sql.execution.datasources.parquet.SpecificParquetRecordReaderBase.ValuesReaderIntIterator;
-import static org.apache.spark.sql.execution.datasources.parquet.SpecificParquetRecordReaderBase.createRLEIterator;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64;
 
 /**
  * Decoder to return values from a single column.
@@ -77,14 +80,13 @@ public class VectorizedColumnReader {
   private final int maxDefLevel;
 
   /**
-   * Repetition/Definition/Value readers.
+   * Value readers.
    */
-  private SpecificParquetRecordReaderBase.IntIterator repetitionLevelColumn;
-  private SpecificParquetRecordReaderBase.IntIterator definitionLevelColumn;
   private ValuesReader dataColumn;
 
-  // Only set if vectorized decoding is true. This is used instead of the row by row decoding
-  // with `definitionLevelColumn`.
+  /**
+   * Vectorized RLE decoder for definition levels
+   */
   private VectorizedRleValuesReader defColumn;
 
   /**
@@ -104,13 +106,38 @@ public class VectorizedColumnReader {
   private final ZoneId convertTz;
   private static final ZoneId UTC = ZoneOffset.UTC;
   private final String datetimeRebaseMode;
+  private final String int96RebaseMode;
+
+  private boolean isDecimalTypeMatched(DataType dt) {
+    DecimalType d = (DecimalType) dt;
+    DecimalMetadata dm = descriptor.getPrimitiveType().getDecimalMetadata();
+    // It's OK if the required decimal precision is larger than or equal to the physical decimal
+    // precision in the Parquet metadata, as long as the decimal scale is the same.
+    return dm != null && dm.getPrecision() <= d.precision() && dm.getScale() == d.scale();
+  }
+
+  private boolean canReadAsIntDecimal(DataType dt) {
+    if (!DecimalType.is32BitDecimalType(dt)) return false;
+    return isDecimalTypeMatched(dt);
+  }
+
+  private boolean canReadAsLongDecimal(DataType dt) {
+    if (!DecimalType.is64BitDecimalType(dt)) return false;
+    return isDecimalTypeMatched(dt);
+  }
+
+  private boolean canReadAsBinaryDecimal(DataType dt) {
+    if (!DecimalType.isByteArrayDecimalType(dt)) return false;
+    return isDecimalTypeMatched(dt);
+  }
 
   public VectorizedColumnReader(
       ColumnDescriptor descriptor,
       OriginalType originalType,
       PageReader pageReader,
       ZoneId convertTz,
-      String datetimeRebaseMode) throws IOException {
+      String datetimeRebaseMode,
+      String int96RebaseMode) throws IOException {
     this.descriptor = descriptor;
     this.pageReader = pageReader;
     this.convertTz = convertTz;
@@ -136,23 +163,9 @@ public VectorizedColumnReader(
     assert "LEGACY".equals(datetimeRebaseMode) || "EXCEPTION".equals(datetimeRebaseMode) ||
       "CORRECTED".equals(datetimeRebaseMode);
     this.datetimeRebaseMode = datetimeRebaseMode;
-  }
-
-  /**
-   * Advances to the next value. Returns true if the value is non-null.
-   */
-  private boolean next() throws IOException {
-    if (valuesRead >= endOfPageValueCount) {
-      if (valuesRead >= totalValueCount) {
-        // How do we get here? Throw end of stream exception?
-        return false;
-      }
-      readPage();
-    }
-    ++valuesRead;
-    // TODO: Don't read for flat schemas
-    //repetitionLevel = repetitionLevelColumn.nextInt();
-    return definitionLevelColumn.nextInt() == maxDefLevel;
+    assert "LEGACY".equals(int96RebaseMode) || "EXCEPTION".equals(int96RebaseMode) ||
+      "CORRECTED".equals(int96RebaseMode);
+    this.int96RebaseMode = int96RebaseMode;
   }
 
   private boolean isLazyDecodingSupported(PrimitiveType.PrimitiveTypeName typeName) {
@@ -189,10 +202,13 @@ static int rebaseDays(int julianDays, final boolean failIfRebase) {
     }
   }
 
-  static long rebaseMicros(long julianMicros, final boolean failIfRebase) {
+  private static long rebaseTimestamp(
+      long julianMicros,
+      final boolean failIfRebase,
+      final String format) {
     if (failIfRebase) {
       if (julianMicros < RebaseDateTime.lastSwitchJulianTs()) {
-        throw DataSourceUtils.newRebaseExceptionInRead("Parquet");
+        throw DataSourceUtils.newRebaseExceptionInRead(format);
       } else {
         return julianMicros;
       }
@@ -201,6 +217,14 @@ static long rebaseMicros(long julianMicros, final boolean failIfRebase) {
     }
   }
 
+  static long rebaseMicros(long julianMicros, final boolean failIfRebase) {
+    return rebaseTimestamp(julianMicros, failIfRebase, "Parquet");
+  }
+
+  static long rebaseInt96(long julianMicros, final boolean failIfRebase) {
+    return rebaseTimestamp(julianMicros, failIfRebase, "Parquet INT96");
+  }
+
   /**
    * Reads `total` values from this columnReader into column.
    */
@@ -234,7 +258,25 @@ void readBatch(int total, WritableColumnVector column) throws IOException {
           // Column vector supports lazy decoding of dictionary values so just set the dictionary.
           // We can't do this if rowId != 0 AND the column doesn't have a dictionary (i.e. some
           // non-dictionary encoded values have already been added).
-          column.setDictionary(new ParquetDictionary(dictionary));
+          PrimitiveType primitiveType = descriptor.getPrimitiveType();
+
+          // We need to make sure that we initialize the right type for the dictionary otherwise
+          // WritableColumnVector will throw an exception when trying to decode to an Int when the
+          // dictionary is in fact initialized as Long
+          boolean castLongToInt = primitiveType.getOriginalType() == OriginalType.DECIMAL &&
+            primitiveType.getDecimalMetadata().getPrecision() <= Decimal.MAX_INT_DIGITS() &&
+            primitiveType.getPrimitiveTypeName() == INT64;
+
+          // We require a long value, but we need to use dictionary to decode the original
+          // signed int first
+          boolean isUnsignedInt32 = primitiveType.getOriginalType() == OriginalType.UINT_32;
+
+          // We require a decimal value, but we need to use dictionary to decode the original
+          // signed long first
+          boolean isUnsignedInt64 = primitiveType.getOriginalType() == OriginalType.UINT_64;
+
+          boolean needTransform = castLongToInt || isUnsignedInt32 || isUnsignedInt64;
+          column.setDictionary(new ParquetDictionary(dictionary, needTransform));
         } else {
           decodeDictionaryIds(rowId, num, column, dictionaryIds);
         }
@@ -309,13 +351,25 @@ private void decodeDictionaryIds(
     switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) {
       case INT32:
         if (column.dataType() == DataTypes.IntegerType ||
-            DecimalType.is32BitDecimalType(column.dataType()) ||
+            canReadAsIntDecimal(column.dataType()) ||
             (column.dataType() == DataTypes.DateType && "CORRECTED".equals(datetimeRebaseMode))) {
           for (int i = rowId; i < rowId + num; ++i) {
             if (!column.isNullAt(i)) {
               column.putInt(i, dictionary.decodeToInt(dictionaryIds.getDictId(i)));
             }
           }
+        } else if (column.dataType() == DataTypes.LongType) {
+          // In `ParquetToSparkSchemaConverter`, we map parquet UINT32 to our LongType.
+          // For unsigned int32, it stores as dictionary encoded signed int32 in Parquet
+          // whenever dictionary is available.
+          // Here we eagerly decode it to the original signed int value then convert to
+          // long(unit32).
+          for (int i = rowId; i < rowId + num; ++i) {
+            if (!column.isNullAt(i)) {
+              column.putLong(i,
+                Integer.toUnsignedLong(dictionary.decodeToInt(dictionaryIds.getDictId(i))));
+            }
+          }
         } else if (column.dataType() == DataTypes.ByteType) {
           for (int i = rowId; i < rowId + num; ++i) {
             if (!column.isNullAt(i)) {
@@ -343,7 +397,7 @@ private void decodeDictionaryIds(
 
       case INT64:
         if (column.dataType() == DataTypes.LongType ||
-            DecimalType.is64BitDecimalType(column.dataType()) ||
+            canReadAsLongDecimal(column.dataType()) ||
             (originalType == OriginalType.TIMESTAMP_MICROS &&
               "CORRECTED".equals(datetimeRebaseMode))) {
           for (int i = rowId; i < rowId + num; ++i) {
@@ -351,6 +405,19 @@ private void decodeDictionaryIds(
               column.putLong(i, dictionary.decodeToLong(dictionaryIds.getDictId(i)));
             }
           }
+        } else if (originalType == OriginalType.UINT_64) {
+          // In `ParquetToSparkSchemaConverter`, we map parquet UINT64 to our Decimal(20, 0).
+          // For unsigned int64, it stores as dictionary encoded signed int64 in Parquet
+          // whenever dictionary is available.
+          // Here we eagerly decode it to the original signed int64(long) value then convert to
+          // BigInteger.
+          for (int i = rowId; i < rowId + num; ++i) {
+            if (!column.isNullAt(i)) {
+              long signed = dictionary.decodeToLong(dictionaryIds.getDictId(i));
+              byte[] unsigned = new BigInteger(Long.toUnsignedString(signed)).toByteArray();
+              column.putByteArray(i, unsigned);
+            }
+          }
         } else if (originalType == OriginalType.TIMESTAMP_MILLIS) {
           if ("CORRECTED".equals(datetimeRebaseMode)) {
             for (int i = rowId; i < rowId + num; ++i) {
@@ -399,20 +466,44 @@ private void decodeDictionaryIds(
         break;
       case INT96:
         if (column.dataType() == DataTypes.TimestampType) {
+          final boolean failIfRebase = "EXCEPTION".equals(int96RebaseMode);
           if (!shouldConvertTimestamps()) {
-            for (int i = rowId; i < rowId + num; ++i) {
-              if (!column.isNullAt(i)) {
-                Binary v = dictionary.decodeToBinary(dictionaryIds.getDictId(i));
-                column.putLong(i, ParquetRowConverter.binaryToSQLTimestamp(v));
+            if ("CORRECTED".equals(int96RebaseMode)) {
+              for (int i = rowId; i < rowId + num; ++i) {
+                if (!column.isNullAt(i)) {
+                  Binary v = dictionary.decodeToBinary(dictionaryIds.getDictId(i));
+                  column.putLong(i, ParquetRowConverter.binaryToSQLTimestamp(v));
+                }
+              }
+            } else {
+              for (int i = rowId; i < rowId + num; ++i) {
+                if (!column.isNullAt(i)) {
+                  Binary v = dictionary.decodeToBinary(dictionaryIds.getDictId(i));
+                  long julianMicros = ParquetRowConverter.binaryToSQLTimestamp(v);
+                  long gregorianMicros = rebaseInt96(julianMicros, failIfRebase);
+                  column.putLong(i, gregorianMicros);
+                }
               }
             }
           } else {
-            for (int i = rowId; i < rowId + num; ++i) {
-              if (!column.isNullAt(i)) {
-                Binary v = dictionary.decodeToBinary(dictionaryIds.getDictId(i));
-                long rawTime = ParquetRowConverter.binaryToSQLTimestamp(v);
-                long adjTime = DateTimeUtils.convertTz(rawTime, convertTz, UTC);
-                column.putLong(i, adjTime);
+            if ("CORRECTED".equals(int96RebaseMode)) {
+              for (int i = rowId; i < rowId + num; ++i) {
+                if (!column.isNullAt(i)) {
+                  Binary v = dictionary.decodeToBinary(dictionaryIds.getDictId(i));
+                  long gregorianMicros = ParquetRowConverter.binaryToSQLTimestamp(v);
+                  long adjTime = DateTimeUtils.convertTz(gregorianMicros, convertTz, UTC);
+                  column.putLong(i, adjTime);
+                }
+              }
+            } else {
+              for (int i = rowId; i < rowId + num; ++i) {
+                if (!column.isNullAt(i)) {
+                  Binary v = dictionary.decodeToBinary(dictionaryIds.getDictId(i));
+                  long julianMicros = ParquetRowConverter.binaryToSQLTimestamp(v);
+                  long gregorianMicros = rebaseInt96(julianMicros, failIfRebase);
+                  long adjTime = DateTimeUtils.convertTz(gregorianMicros, convertTz, UTC);
+                  column.putLong(i, adjTime);
+                }
               }
             }
           }
@@ -434,21 +525,21 @@ private void decodeDictionaryIds(
         break;
       case FIXED_LEN_BYTE_ARRAY:
         // DecimalType written in the legacy mode
-        if (DecimalType.is32BitDecimalType(column.dataType())) {
+        if (canReadAsIntDecimal(column.dataType())) {
           for (int i = rowId; i < rowId + num; ++i) {
             if (!column.isNullAt(i)) {
               Binary v = dictionary.decodeToBinary(dictionaryIds.getDictId(i));
               column.putInt(i, (int) ParquetRowConverter.binaryToUnscaledLong(v));
             }
           }
-        } else if (DecimalType.is64BitDecimalType(column.dataType())) {
+        } else if (canReadAsLongDecimal(column.dataType())) {
           for (int i = rowId; i < rowId + num; ++i) {
             if (!column.isNullAt(i)) {
               Binary v = dictionary.decodeToBinary(dictionaryIds.getDictId(i));
               column.putLong(i, ParquetRowConverter.binaryToUnscaledLong(v));
             }
           }
-        } else if (DecimalType.isByteArrayDecimalType(column.dataType())) {
+        } else if (canReadAsBinaryDecimal(column.dataType())) {
           for (int i = rowId; i < rowId + num; ++i) {
             if (!column.isNullAt(i)) {
               Binary v = dictionary.decodeToBinary(dictionaryIds.getDictId(i));
@@ -484,9 +575,15 @@ private void readIntBatch(int rowId, int num, WritableColumnVector column) throw
     // This is where we implement support for the valid type conversions.
     // TODO: implement remaining type conversions
     if (column.dataType() == DataTypes.IntegerType ||
-        DecimalType.is32BitDecimalType(column.dataType())) {
+        canReadAsIntDecimal(column.dataType())) {
       defColumn.readIntegers(
           num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
+    } else if (column.dataType() == DataTypes.LongType) {
+      // In `ParquetToSparkSchemaConverter`, we map parquet UINT32 to our LongType.
+      // For unsigned int32, it stores as plain signed int32 in Parquet when dictionary fallbacks.
+      // We read them as long values.
+      defColumn.readUnsignedIntegers(
+          num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
     } else if (column.dataType() == DataTypes.ByteType) {
       defColumn.readBytes(
           num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
@@ -510,13 +607,20 @@ private void readIntBatch(int rowId, int num, WritableColumnVector column) throw
   private void readLongBatch(int rowId, int num, WritableColumnVector column) throws IOException {
     // This is where we implement support for the valid type conversions.
     if (column.dataType() == DataTypes.LongType ||
-        DecimalType.is64BitDecimalType(column.dataType())) {
+        canReadAsLongDecimal(column.dataType())) {
       defColumn.readLongs(
+        num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn,
+        DecimalType.is32BitDecimalType(column.dataType()));
+    } else if (originalType == OriginalType.UINT_64) {
+      // In `ParquetToSparkSchemaConverter`, we map parquet UINT64 to our Decimal(20, 0).
+      // For unsigned int64, it stores as plain signed int64 in Parquet when dictionary fallbacks.
+      // We read them as decimal values.
+      defColumn.readUnsignedLongs(
         num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
     } else if (originalType == OriginalType.TIMESTAMP_MICROS) {
       if ("CORRECTED".equals(datetimeRebaseMode)) {
         defColumn.readLongs(
-          num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
+          num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn, false);
       } else {
         boolean failIfRebase = "EXCEPTION".equals(datetimeRebaseMode);
         defColumn.readLongsWithRebase(
@@ -574,28 +678,56 @@ private void readBinaryBatch(int rowId, int num, WritableColumnVector column) th
     // TODO: implement remaining type conversions
     VectorizedValuesReader data = (VectorizedValuesReader) dataColumn;
     if (column.dataType() == DataTypes.StringType || column.dataType() == DataTypes.BinaryType
-            || DecimalType.isByteArrayDecimalType(column.dataType())) {
+            || canReadAsBinaryDecimal(column.dataType())) {
       defColumn.readBinarys(num, column, rowId, maxDefLevel, data);
     } else if (column.dataType() == DataTypes.TimestampType) {
+      final boolean failIfRebase = "EXCEPTION".equals(int96RebaseMode);
       if (!shouldConvertTimestamps()) {
-        for (int i = 0; i < num; i++) {
-          if (defColumn.readInteger() == maxDefLevel) {
-            // Read 12 bytes for INT96
-            long rawTime = ParquetRowConverter.binaryToSQLTimestamp(data.readBinary(12));
-            column.putLong(rowId + i, rawTime);
-          } else {
-            column.putNull(rowId + i);
+        if ("CORRECTED".equals(int96RebaseMode)) {
+          for (int i = 0; i < num; i++) {
+            if (defColumn.readInteger() == maxDefLevel) {
+              // Read 12 bytes for INT96
+              long gregorianMicros = ParquetRowConverter.binaryToSQLTimestamp(data.readBinary(12));
+              column.putLong(rowId + i, gregorianMicros);
+            } else {
+              column.putNull(rowId + i);
+            }
+          }
+        } else {
+          for (int i = 0; i < num; i++) {
+            if (defColumn.readInteger() == maxDefLevel) {
+              // Read 12 bytes for INT96
+              long julianMicros = ParquetRowConverter.binaryToSQLTimestamp(data.readBinary(12));
+              long gregorianMicros = rebaseInt96(julianMicros, failIfRebase);
+              column.putLong(rowId + i, gregorianMicros);
+            } else {
+              column.putNull(rowId + i);
+            }
           }
         }
       } else {
-        for (int i = 0; i < num; i++) {
-          if (defColumn.readInteger() == maxDefLevel) {
-            // Read 12 bytes for INT96
-            long rawTime = ParquetRowConverter.binaryToSQLTimestamp(data.readBinary(12));
-            long adjTime = DateTimeUtils.convertTz(rawTime, convertTz, UTC);
-            column.putLong(rowId + i, adjTime);
-          } else {
-            column.putNull(rowId + i);
+        if ("CORRECTED".equals(int96RebaseMode)) {
+          for (int i = 0; i < num; i++) {
+            if (defColumn.readInteger() == maxDefLevel) {
+              // Read 12 bytes for INT96
+              long gregorianMicros = ParquetRowConverter.binaryToSQLTimestamp(data.readBinary(12));
+              long adjTime = DateTimeUtils.convertTz(gregorianMicros, convertTz, UTC);
+              column.putLong(rowId + i, adjTime);
+            } else {
+              column.putNull(rowId + i);
+            }
+          }
+        } else {
+          for (int i = 0; i < num; i++) {
+            if (defColumn.readInteger() == maxDefLevel) {
+              // Read 12 bytes for INT96
+              long julianMicros = ParquetRowConverter.binaryToSQLTimestamp(data.readBinary(12));
+              long gregorianMicros = rebaseInt96(julianMicros, failIfRebase);
+              long adjTime = DateTimeUtils.convertTz(gregorianMicros, convertTz, UTC);
+              column.putLong(rowId + i, adjTime);
+            } else {
+              column.putNull(rowId + i);
+            }
           }
         }
       }
@@ -612,7 +744,7 @@ private void readFixedLenByteArrayBatch(
     VectorizedValuesReader data = (VectorizedValuesReader) dataColumn;
     // This is where we implement support for the valid type conversions.
     // TODO: implement remaining type conversions
-    if (DecimalType.is32BitDecimalType(column.dataType())) {
+    if (canReadAsIntDecimal(column.dataType())) {
       for (int i = 0; i < num; i++) {
         if (defColumn.readInteger() == maxDefLevel) {
           column.putInt(rowId + i,
@@ -621,7 +753,7 @@ private void readFixedLenByteArrayBatch(
           column.putNull(rowId + i);
         }
       }
-    } else if (DecimalType.is64BitDecimalType(column.dataType())) {
+    } else if (canReadAsLongDecimal(column.dataType())) {
       for (int i = 0; i < num; i++) {
         if (defColumn.readInteger() == maxDefLevel) {
           column.putLong(rowId + i,
@@ -630,7 +762,7 @@ private void readFixedLenByteArrayBatch(
           column.putNull(rowId + i);
         }
       }
-    } else if (DecimalType.isByteArrayDecimalType(column.dataType())) {
+    } else if (canReadAsBinaryDecimal(column.dataType())) {
       for (int i = 0; i < num; i++) {
         if (defColumn.readInteger() == maxDefLevel) {
           column.putByteArray(rowId + i, data.readBinary(arrayLen).getBytes());
@@ -702,23 +834,24 @@ private void initDataReader(Encoding dataEncoding, ByteBufferInputStream in) thr
 
   private void readPageV1(DataPageV1 page) throws IOException {
     this.pageValueCount = page.getValueCount();
-    ValuesReader rlReader = page.getRlEncoding().getValuesReader(descriptor, REPETITION_LEVEL);
-    ValuesReader dlReader;
 
     // Initialize the decoders.
     if (page.getDlEncoding() != Encoding.RLE && descriptor.getMaxDefinitionLevel() != 0) {
       throw new UnsupportedOperationException("Unsupported encoding: " + page.getDlEncoding());
     }
+
     int bitWidth = BytesUtils.getWidthFromMaxInt(descriptor.getMaxDefinitionLevel());
     this.defColumn = new VectorizedRleValuesReader(bitWidth);
-    dlReader = this.defColumn;
-    this.repetitionLevelColumn = new ValuesReaderIntIterator(rlReader);
-    this.definitionLevelColumn = new ValuesReaderIntIterator(dlReader);
     try {
       BytesInput bytes = page.getBytes();
       ByteBufferInputStream in = bytes.toInputStream();
-      rlReader.initFromPage(pageValueCount, in);
-      dlReader.initFromPage(pageValueCount, in);
+
+      // only used now to consume the repetition level data
+      page.getRlEncoding()
+          .getValuesReader(descriptor, REPETITION_LEVEL)
+          .initFromPage(pageValueCount, in);
+
+      defColumn.initFromPage(pageValueCount, in);
       initDataReader(page.getValueEncoding(), in);
     } catch (IOException e) {
       throw new IOException("could not read page " + page + " in col " + descriptor, e);
@@ -727,15 +860,11 @@ private void readPageV1(DataPageV1 page) throws IOException {
 
   private void readPageV2(DataPageV2 page) throws IOException {
     this.pageValueCount = page.getValueCount();
-    this.repetitionLevelColumn = createRLEIterator(descriptor.getMaxRepetitionLevel(),
-        page.getRepetitionLevels(), descriptor);
 
     int bitWidth = BytesUtils.getWidthFromMaxInt(descriptor.getMaxDefinitionLevel());
     // do not read the length from the stream. v2 pages handle dividing the page bytes.
-    this.defColumn = new VectorizedRleValuesReader(bitWidth, false);
-    this.definitionLevelColumn = new ValuesReaderIntIterator(this.defColumn);
-    this.defColumn.initFromPage(
-        this.pageValueCount, page.getDefinitionLevels().toInputStream());
+    defColumn = new VectorizedRleValuesReader(bitWidth, false);
+    defColumn.initFromPage(this.pageValueCount, page.getDefinitionLevels().toInputStream());
     try {
       initDataReader(page.getDataEncoding(), page.getData().toInputStream());
     } catch (IOException e) {
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java
index b40cc154d76fe..1b159534c8a4f 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java
@@ -93,6 +93,11 @@ public class VectorizedParquetRecordReader extends SpecificParquetRecordReaderBa
    */
   private final String datetimeRebaseMode;
 
+  /**
+   * The mode of rebasing INT96 timestamp from Julian to Proleptic Gregorian calendar.
+   */
+  private final String int96RebaseMode;
+
   /**
    * columnBatch object that is used for batch decoding. This is created on first use and triggers
    * batched decoding. It is not valid to interleave calls to the batched interface with the row
@@ -122,16 +127,21 @@ public class VectorizedParquetRecordReader extends SpecificParquetRecordReaderBa
   private final MemoryMode MEMORY_MODE;
 
   public VectorizedParquetRecordReader(
-    ZoneId convertTz, String datetimeRebaseMode, boolean useOffHeap, int capacity) {
+      ZoneId convertTz,
+      String datetimeRebaseMode,
+      String int96RebaseMode,
+      boolean useOffHeap,
+      int capacity) {
     this.convertTz = convertTz;
     this.datetimeRebaseMode = datetimeRebaseMode;
+    this.int96RebaseMode = int96RebaseMode;
     MEMORY_MODE = useOffHeap ? MemoryMode.OFF_HEAP : MemoryMode.ON_HEAP;
     this.capacity = capacity;
   }
 
   // For test only.
   public VectorizedParquetRecordReader(boolean useOffHeap, int capacity) {
-    this(null, "CORRECTED", useOffHeap, capacity);
+    this(null, "CORRECTED", "LEGACY", useOffHeap, capacity);
   }
 
   /**
@@ -310,7 +320,7 @@ private void initializeInternal() throws IOException, UnsupportedOperationExcept
 
   private void checkEndOfRowGroup() throws IOException {
     if (rowsReturned != totalCountLoadedSoFar) return;
-    PageReadStore pages = reader.readNextRowGroup();
+    PageReadStore pages = reader.readNextFilteredRowGroup();
     if (pages == null) {
       throw new IOException("expecting more rows but reached last block. Read "
           + rowsReturned + " out of " + totalRowCount);
@@ -320,8 +330,13 @@ private void checkEndOfRowGroup() throws IOException {
     columnReaders = new VectorizedColumnReader[columns.size()];
     for (int i = 0; i < columns.size(); ++i) {
       if (missingColumns[i]) continue;
-      columnReaders[i] = new VectorizedColumnReader(columns.get(i), types.get(i).getOriginalType(),
-        pages.getPageReader(columns.get(i)), convertTz, datetimeRebaseMode);
+      columnReaders[i] = new VectorizedColumnReader(
+        columns.get(i),
+        types.get(i).getOriginalType(),
+        pages.getPageReader(columns.get(i)),
+        convertTz,
+        datetimeRebaseMode,
+        int96RebaseMode);
     }
     totalCountLoadedSoFar += pages.getRowCount();
   }
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedPlainValuesReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedPlainValuesReader.java
index eddbf39178e9a..6a0038dbdc44c 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedPlainValuesReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedPlainValuesReader.java
@@ -17,6 +17,7 @@
 package org.apache.spark.sql.execution.datasources.parquet;
 
 import java.io.IOException;
+import java.math.BigInteger;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 
@@ -83,6 +84,15 @@ public final void readIntegers(int total, WritableColumnVector c, int rowId) {
     }
   }
 
+  @Override
+  public final void readUnsignedIntegers(int total, WritableColumnVector c, int rowId) {
+    int requiredBytes = total * 4;
+    ByteBuffer buffer = getBuffer(requiredBytes);
+    for (int i = 0; i < total; i += 1) {
+      c.putLong(rowId + i, Integer.toUnsignedLong(buffer.getInt()));
+    }
+  }
+
   // A fork of `readIntegers` to rebase the date values. For performance reasons, this method
   // iterates the values twice: check if we need to rebase first, then go to the optimized branch
   // if rebase is not needed.
@@ -130,6 +140,16 @@ public final void readLongs(int total, WritableColumnVector c, int rowId) {
     }
   }
 
+  @Override
+  public final void readUnsignedLongs(int total, WritableColumnVector c, int rowId) {
+    int requiredBytes = total * 8;
+    ByteBuffer buffer = getBuffer(requiredBytes);
+    for (int i = 0; i < total; i += 1) {
+      c.putByteArray(
+        rowId + i, new BigInteger(Long.toUnsignedString(buffer.getLong())).toByteArray());
+    }
+  }
+
   // A fork of `readLongs` to rebase the timestamp values. For performance reasons, this method
   // iterates the values twice: check if we need to rebase first, then go to the optimized branch
   // if rebase is not needed.
@@ -169,7 +189,7 @@ public final void readFloats(int total, WritableColumnVector c, int rowId) {
 
     if (buffer.hasArray()) {
       int offset = buffer.arrayOffset() + buffer.position();
-      c.putFloats(rowId, total, buffer.array(), offset);
+      c.putFloatsLittleEndian(rowId, total, buffer.array(), offset);
     } else {
       for (int i = 0; i < total; i += 1) {
         c.putFloat(rowId + i, buffer.getFloat());
@@ -184,7 +204,7 @@ public final void readDoubles(int total, WritableColumnVector c, int rowId) {
 
     if (buffer.hasArray()) {
       int offset = buffer.arrayOffset() + buffer.position();
-      c.putDoubles(rowId, total, buffer.array(), offset);
+      c.putDoublesLittleEndian(rowId, total, buffer.array(), offset);
     } else {
       for (int i = 0; i < total; i += 1) {
         c.putDouble(rowId + i, buffer.getDouble());
@@ -206,6 +226,16 @@ public final void readBytes(int total, WritableColumnVector c, int rowId) {
     }
   }
 
+  @Override
+  public final void readShorts(int total, WritableColumnVector c, int rowId) {
+    int requiredBytes = total * 4;
+    ByteBuffer buffer = getBuffer(requiredBytes);
+
+    for (int i = 0; i < total; i += 1) {
+      c.putShort(rowId + i, (short) buffer.getInt());
+    }
+  }
+
   @Override
   public final boolean readBoolean() {
     // TODO: vectorize decoding and keep boolean[] instead of currentByte
@@ -240,6 +270,11 @@ public final byte readByte() {
     return (byte) readInteger();
   }
 
+  @Override
+  public short readShort() {
+    return (short) readInteger();
+  }
+
   @Override
   public final float readFloat() {
     return getBuffer(4).getFloat();
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java
index 24347a4e3a0c5..125506d4d5013 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.execution.datasources.parquet;
 
 import java.io.IOException;
+import java.math.BigInteger;
 import java.nio.ByteBuffer;
 
 import org.apache.parquet.Preconditions;
@@ -46,7 +47,7 @@ public final class VectorizedRleValuesReader extends ValuesReader
   // Current decoding mode. The encoded data contains groups of either run length encoded data
   // (RLE) or bit packed data. Each group contains a header that indicates which group it is and
   // the number of values in the group.
-  // More details here: https://github.com/Parquet/parquet-format/blob/master/Encodings.md
+  // More details here: https://github.com/apache/parquet-format/blob/master/Encodings.md
   private enum MODE {
     RLE,
     PACKED
@@ -203,6 +204,41 @@ public void readIntegers(
     }
   }
 
+  // A fork of `readIntegers`, reading the signed integers as unsigned in long type
+  public void readUnsignedIntegers(
+      int total,
+      WritableColumnVector c,
+      int rowId,
+      int level,
+      VectorizedValuesReader data) throws IOException {
+    int left = total;
+    while (left > 0) {
+      if (this.currentCount == 0) this.readNextGroup();
+      int n = Math.min(left, this.currentCount);
+      switch (mode) {
+        case RLE:
+          if (currentValue == level) {
+            data.readUnsignedIntegers(n, c, rowId);
+          } else {
+            c.putNulls(rowId, n);
+          }
+          break;
+        case PACKED:
+          for (int i = 0; i < n; ++i) {
+            if (currentBuffer[currentBufferIdx++] == level) {
+              c.putLong(rowId + i, Integer.toUnsignedLong(data.readInteger()));
+            } else {
+              c.putNull(rowId + i);
+            }
+          }
+          break;
+      }
+      rowId += n;
+      left -= n;
+      currentCount -= n;
+    }
+  }
+
   // A fork of `readIntegers`, which rebases the date int value (days) before filling
   // the Spark column vector.
   public void readIntegersWithRebase(
@@ -323,9 +359,7 @@ public void readShorts(
       switch (mode) {
         case RLE:
           if (currentValue == level) {
-            for (int i = 0; i < n; i++) {
-              c.putShort(rowId + i, (short)data.readInteger());
-            }
+            data.readShorts(n, c, rowId);
           } else {
             c.putNulls(rowId, n);
           }
@@ -333,7 +367,7 @@ public void readShorts(
         case PACKED:
           for (int i = 0; i < n; ++i) {
             if (currentBuffer[currentBufferIdx++] == level) {
-              c.putShort(rowId + i, (short)data.readInteger());
+              c.putShort(rowId + i, data.readShort());
             } else {
               c.putNull(rowId + i);
             }
@@ -347,6 +381,58 @@ public void readShorts(
   }
 
   public void readLongs(
+      int total,
+      WritableColumnVector c,
+      int rowId,
+      int level,
+      VectorizedValuesReader data,
+      boolean downCastLongToInt) throws IOException {
+    int left = total;
+    while (left > 0) {
+      if (this.currentCount == 0) this.readNextGroup();
+      int n = Math.min(left, this.currentCount);
+      switch (mode) {
+        case RLE:
+          if (currentValue == level) {
+            if (downCastLongToInt) {
+              for (int i = 0; i < n; ++i) {
+                c.putInt(rowId + i, (int) data.readLong());
+              }
+            } else {
+              data.readLongs(n, c, rowId);
+            }
+          } else {
+            c.putNulls(rowId, n);
+          }
+          break;
+        case PACKED:
+          // code repeated for performance
+          if (downCastLongToInt) {
+            for (int i = 0; i < n; ++i) {
+              if (currentBuffer[currentBufferIdx++] == level) {
+                c.putInt(rowId + i, (int) data.readLong());
+              } else {
+                c.putNull(rowId + i);
+              }
+            }
+          } else {
+            for (int i = 0; i < n; ++i) {
+              if (currentBuffer[currentBufferIdx++] == level) {
+                c.putLong(rowId + i, data.readLong());
+              } else {
+                c.putNull(rowId + i);
+              }
+            }
+          }
+          break;
+      }
+      rowId += n;
+      left -= n;
+      currentCount -= n;
+    }
+  }
+
+  public void readUnsignedLongs(
       int total,
       WritableColumnVector c,
       int rowId,
@@ -359,7 +445,7 @@ public void readLongs(
       switch (mode) {
         case RLE:
           if (currentValue == level) {
-            data.readLongs(n, c, rowId);
+            data.readUnsignedLongs(n, c, rowId);
           } else {
             c.putNulls(rowId, n);
           }
@@ -367,7 +453,8 @@ public void readLongs(
         case PACKED:
           for (int i = 0; i < n; ++i) {
             if (currentBuffer[currentBufferIdx++] == level) {
-              c.putLong(rowId + i, data.readLong());
+              byte[] bytes = new BigInteger(Long.toUnsignedString(data.readLong())).toByteArray();
+              c.putByteArray(rowId + i, bytes);
             } else {
               c.putNull(rowId + i);
             }
@@ -584,6 +671,16 @@ public void readIntegers(int total, WritableColumnVector c, int rowId) {
     }
   }
 
+  @Override
+  public void readUnsignedIntegers(int total, WritableColumnVector c, int rowId) {
+    throw new UnsupportedOperationException("only readInts is valid.");
+  }
+
+  @Override
+  public void readUnsignedLongs(int total, WritableColumnVector c, int rowId) {
+    throw new UnsupportedOperationException("only readInts is valid.");
+  }
+
   @Override
   public void readIntegersWithRebase(
       int total, WritableColumnVector c, int rowId, boolean failIfRebase) {
@@ -595,11 +692,21 @@ public byte readByte() {
     throw new UnsupportedOperationException("only readInts is valid.");
   }
 
+  @Override
+  public short readShort() {
+    throw new UnsupportedOperationException("only readInts is valid.");
+  }
+
   @Override
   public void readBytes(int total, WritableColumnVector c, int rowId) {
     throw new UnsupportedOperationException("only readInts is valid.");
   }
 
+  @Override
+  public void readShorts(int total, WritableColumnVector c, int rowId) {
+    throw new UnsupportedOperationException("only readInts is valid.");
+  }
+
   @Override
   public void readLongs(int total, WritableColumnVector c, int rowId) {
     throw new UnsupportedOperationException("only readInts is valid.");
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedValuesReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedValuesReader.java
index 35db8f235ed60..a2d663fd8c8b6 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedValuesReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedValuesReader.java
@@ -28,6 +28,7 @@
 public interface VectorizedValuesReader {
   boolean readBoolean();
   byte readByte();
+  short readShort();
   int readInteger();
   long readLong();
   float readFloat();
@@ -39,8 +40,11 @@ public interface VectorizedValuesReader {
    */
   void readBooleans(int total, WritableColumnVector c, int rowId);
   void readBytes(int total, WritableColumnVector c, int rowId);
+  void readShorts(int total, WritableColumnVector c, int rowId);
   void readIntegers(int total, WritableColumnVector c, int rowId);
   void readIntegersWithRebase(int total, WritableColumnVector c, int rowId, boolean failIfRebase);
+  void readUnsignedIntegers(int total, WritableColumnVector c, int rowId);
+  void readUnsignedLongs(int total, WritableColumnVector c, int rowId);
   void readLongs(int total, WritableColumnVector c, int rowId);
   void readLongsWithRebase(int total, WritableColumnVector c, int rowId, boolean failIfRebase);
   void readFloats(int total, WritableColumnVector c, int rowId);
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java
index bce6aa28c42a1..25aabcd086289 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java
@@ -54,6 +54,8 @@ public static void populate(WritableColumnVector col, InternalRow row, int field
     } else {
       if (t == DataTypes.BooleanType) {
         col.putBooleans(0, capacity, row.getBoolean(fieldIdx));
+      } else if (t == DataTypes.BinaryType) {
+        col.putByteArray(0, row.getBinary(fieldIdx));
       } else if (t == DataTypes.ByteType) {
         col.putBytes(0, capacity, row.getByte(fieldIdx));
       } else if (t == DataTypes.ShortType) {
@@ -94,6 +96,9 @@ public static void populate(WritableColumnVector col, InternalRow row, int field
         col.putInts(0, capacity, row.getInt(fieldIdx));
       } else if (t instanceof TimestampType) {
         col.putLongs(0, capacity, row.getLong(fieldIdx));
+      } else {
+        throw new RuntimeException(String.format("DataType %s is not supported" +
+            " in column vectorized reader.", t.sql()));
       }
     }
   }
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java
index 3b919c7d471f4..7da5a287710eb 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java
@@ -413,11 +413,16 @@ public void putFloats(int rowId, int count, float[] src, int srcIndex) {
 
   @Override
   public void putFloats(int rowId, int count, byte[] src, int srcIndex) {
+    Platform.copyMemory(src, Platform.BYTE_ARRAY_OFFSET + srcIndex,
+        null, data + rowId * 4L, count * 4L);
+  }
+
+  @Override
+  public void putFloatsLittleEndian(int rowId, int count, byte[] src, int srcIndex) {
     if (!bigEndianPlatform) {
-      Platform.copyMemory(src, Platform.BYTE_ARRAY_OFFSET + srcIndex,
-          null, data + rowId * 4L, count * 4L);
+      putFloats(rowId, count, src, srcIndex);
     } else {
-      ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.BIG_ENDIAN);
+      ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN);
       long offset = data + 4L * rowId;
       for (int i = 0; i < count; ++i, offset += 4) {
         Platform.putFloat(null, offset, bb.getFloat(srcIndex + (4 * i)));
@@ -468,11 +473,16 @@ public void putDoubles(int rowId, int count, double[] src, int srcIndex) {
 
   @Override
   public void putDoubles(int rowId, int count, byte[] src, int srcIndex) {
+    Platform.copyMemory(src, Platform.BYTE_ARRAY_OFFSET + srcIndex,
+      null, data + rowId * 8L, count * 8L);
+  }
+
+  @Override
+  public void putDoublesLittleEndian(int rowId, int count, byte[] src, int srcIndex) {
     if (!bigEndianPlatform) {
-      Platform.copyMemory(src, Platform.BYTE_ARRAY_OFFSET + srcIndex,
-        null, data + rowId * 8L, count * 8L);
+      putDoubles(rowId, count, src, srcIndex);
     } else {
-      ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.BIG_ENDIAN);
+      ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN);
       long offset = data + 8L * rowId;
       for (int i = 0; i < count; ++i, offset += 8) {
         Platform.putDouble(null, offset, bb.getDouble(srcIndex + (8 * i)));
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java
index 1bf3126664177..5942c5f00a710 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java
@@ -392,11 +392,16 @@ public void putFloats(int rowId, int count, float[] src, int srcIndex) {
 
   @Override
   public void putFloats(int rowId, int count, byte[] src, int srcIndex) {
+    Platform.copyMemory(src, Platform.BYTE_ARRAY_OFFSET + srcIndex, floatData,
+        Platform.FLOAT_ARRAY_OFFSET + rowId * 4L, count * 4L);
+  }
+
+  @Override
+  public void putFloatsLittleEndian(int rowId, int count, byte[] src, int srcIndex) {
     if (!bigEndianPlatform) {
-      Platform.copyMemory(src, Platform.BYTE_ARRAY_OFFSET + srcIndex, floatData,
-          Platform.DOUBLE_ARRAY_OFFSET + rowId * 4L, count * 4L);
+      putFloats(rowId, count, src, srcIndex);
     } else {
-      ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.BIG_ENDIAN);
+      ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN);
       for (int i = 0; i < count; ++i) {
         floatData[i + rowId] = bb.getFloat(srcIndex + (4 * i));
       }
@@ -441,11 +446,16 @@ public void putDoubles(int rowId, int count, double[] src, int srcIndex) {
 
   @Override
   public void putDoubles(int rowId, int count, byte[] src, int srcIndex) {
+    Platform.copyMemory(src, Platform.BYTE_ARRAY_OFFSET + srcIndex, doubleData,
+        Platform.DOUBLE_ARRAY_OFFSET + rowId * 8L, count * 8L);
+  }
+
+  @Override
+  public void putDoublesLittleEndian(int rowId, int count, byte[] src, int srcIndex) {
     if (!bigEndianPlatform) {
-      Platform.copyMemory(src, Platform.BYTE_ARRAY_OFFSET + srcIndex, doubleData,
-          Platform.DOUBLE_ARRAY_OFFSET + rowId * 8L, count * 8L);
+      putDoubles(rowId, count, src, srcIndex);
     } else {
-      ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.BIG_ENDIAN);
+      ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN);
       for (int i = 0; i < count; ++i) {
         doubleData[i + rowId] = bb.getDouble(srcIndex + (8 * i));
       }
@@ -531,14 +541,14 @@ protected void reserveInternal(int newCapacity) {
         shortData = newData;
       }
     } else if (type instanceof IntegerType || type instanceof DateType ||
-      DecimalType.is32BitDecimalType(type)) {
+      DecimalType.is32BitDecimalType(type) || type instanceof YearMonthIntervalType) {
       if (intData == null || intData.length < newCapacity) {
         int[] newData = new int[newCapacity];
         if (intData != null) System.arraycopy(intData, 0, newData, 0, capacity);
         intData = newData;
       }
     } else if (type instanceof LongType || type instanceof TimestampType ||
-        DecimalType.is64BitDecimalType(type)) {
+        DecimalType.is64BitDecimalType(type) || type instanceof DayTimeIntervalType) {
       if (longData == null || longData.length < newCapacity) {
         long[] newData = new long[newCapacity];
         if (longData != null) System.arraycopy(longData, 0, newData, 0, capacity);
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVector.java b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVector.java
index f04b0707ff7b5..8c0f1e1257503 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVector.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVector.java
@@ -312,6 +312,12 @@ public WritableColumnVector reserveDictionaryIds(int capacity) {
    */
   public abstract void putFloats(int rowId, int count, byte[] src, int srcIndex);
 
+  /**
+   * Sets values from [src[srcIndex], src[srcIndex + count * 4]) to [rowId, rowId + count)
+   * The data in src must be ieee formatted floats in little endian.
+   */
+  public abstract void putFloatsLittleEndian(int rowId, int count, byte[] src, int srcIndex);
+
   /**
    * Sets `value` to the value at rowId.
    */
@@ -333,6 +339,12 @@ public WritableColumnVector reserveDictionaryIds(int capacity) {
    */
   public abstract void putDoubles(int rowId, int count, byte[] src, int srcIndex);
 
+  /**
+   * Sets values from [src[srcIndex], src[srcIndex + count * 8]) to [rowId, rowId + count)
+   * The data in src must be ieee formatted doubles in little endian.
+   */
+  public abstract void putDoublesLittleEndian(int rowId, int count, byte[] src, int srcIndex);
+
   /**
    * Puts a byte array that already exists in this column.
    */
diff --git a/sql/core/src/main/resources/META-INF/services/org.apache.spark.sql.jdbc.JdbcConnectionProvider b/sql/core/src/main/resources/META-INF/services/org.apache.spark.sql.jdbc.JdbcConnectionProvider
new file mode 100644
index 0000000000000..6e42517a6d40c
--- /dev/null
+++ b/sql/core/src/main/resources/META-INF/services/org.apache.spark.sql.jdbc.JdbcConnectionProvider
@@ -0,0 +1,6 @@
+org.apache.spark.sql.execution.datasources.jdbc.connection.BasicConnectionProvider
+org.apache.spark.sql.execution.datasources.jdbc.connection.DB2ConnectionProvider
+org.apache.spark.sql.execution.datasources.jdbc.connection.MariaDBConnectionProvider
+org.apache.spark.sql.execution.datasources.jdbc.connection.MSSQLConnectionProvider
+org.apache.spark.sql.execution.datasources.jdbc.connection.PostgresConnectionProvider
+org.apache.spark.sql.execution.datasources.jdbc.connection.OracleConnectionProvider
\ No newline at end of file
diff --git a/sql/core/src/main/resources/META-INF/services/org.apache.spark.status.AppHistoryServerPlugin b/sql/core/src/main/resources/META-INF/services/org.apache.spark.status.AppHistoryServerPlugin
index 0bba2f88b92a5..6771eef525307 100644
--- a/sql/core/src/main/resources/META-INF/services/org.apache.spark.status.AppHistoryServerPlugin
+++ b/sql/core/src/main/resources/META-INF/services/org.apache.spark.status.AppHistoryServerPlugin
@@ -1 +1,2 @@
 org.apache.spark.sql.execution.ui.SQLHistoryServerPlugin
+org.apache.spark.sql.execution.ui.StreamingQueryHistoryServerPlugin
diff --git a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
index 9a32b79cd070f..dbdbf9fbf57b1 100644
--- a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
+++ b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
@@ -52,3 +52,8 @@
 .tooltip-inner {
   word-wrap:break-word;
 }
+
+/* Breaks the long job url list when showing Details for Query in SQL */
+.job-url {
+  word-wrap: break-word;
+}
diff --git a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
index 301183f749a84..d1def1b0a42ff 100644
--- a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
+++ b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
@@ -87,14 +87,14 @@ function preprocessGraphLayout(g) {
     var node = g.node(nodes[i]);
     node.padding = "5";
 
-    var firstSearator;
+    var firstSeparator;
     var secondSeparator;
     var splitter;
     if (node.isCluster) {
-      firstSearator = secondSeparator = labelSeparator;
+      firstSeparator = secondSeparator = labelSeparator;
       splitter = "\\n";
     } else {
-      firstSearator = "<span class='stageId-and-taskId-metrics'>";
+      firstSeparator = "<span class='stageId-and-taskId-metrics'>";
       secondSeparator = "</span>";
       splitter = "<br>";
     }
@@ -104,7 +104,7 @@ function preprocessGraphLayout(g) {
       if (newTexts) {
         node.label = node.label.replace(
             newTexts[0],
-            newTexts[1] + firstSearator + newTexts[2] + secondSeparator + newTexts[3]);
+            newTexts[1] + firstSeparator + newTexts[2] + secondSeparator + newTexts[3]);
       }
     });
   }
diff --git a/sql/core/src/main/scala-2.13/org/apache/spark/sql/execution/streaming/StreamProgress.scala b/sql/core/src/main/scala-2.13/org/apache/spark/sql/execution/streaming/StreamProgress.scala
index 0aa29640899c6..6aa1b46cbb94a 100644
--- a/sql/core/src/main/scala-2.13/org/apache/spark/sql/execution/streaming/StreamProgress.scala
+++ b/sql/core/src/main/scala-2.13/org/apache/spark/sql/execution/streaming/StreamProgress.scala
@@ -26,7 +26,7 @@ import org.apache.spark.sql.connector.read.streaming.{Offset => OffsetV2, SparkD
  */
 class StreamProgress(
     val baseMap: immutable.Map[SparkDataStream, OffsetV2] =
-        new immutable.HashMap[SparkDataStream, OffsetV2])                   
+        new immutable.HashMap[SparkDataStream, OffsetV2])
   extends scala.collection.immutable.Map[SparkDataStream, OffsetV2] {
 
   //  Note: this class supports Scala 2.13. A parallel source tree has a 2.12 implementation.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
index 2144472937f9b..db008821584dd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
@@ -18,7 +18,6 @@
 package org.apache.spark.sql
 
 import scala.collection.JavaConverters._
-import scala.language.implicitConversions
 
 import org.apache.spark.annotation.Stable
 import org.apache.spark.internal.Logging
@@ -27,7 +26,7 @@ import org.apache.spark.sql.catalyst.encoders.{encoderFor, ExpressionEncoder}
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
-import org.apache.spark.sql.catalyst.util.toPrettySQL
+import org.apache.spark.sql.catalyst.util.{toPrettySQL, CharVarcharUtils}
 import org.apache.spark.sql.execution.aggregate.TypedAggregateExpression
 import org.apache.spark.sql.expressions.Window
 import org.apache.spark.sql.functions.lit
@@ -170,21 +169,12 @@ class Column(val expr: Expression) extends Logging {
    * Returns the expression for this column either with an existing or auto assigned name.
    */
   private[sql] def named: NamedExpression = expr match {
-    // Wrap UnresolvedAttribute with UnresolvedAlias, as when we resolve UnresolvedAttribute, we
-    // will remove intermediate Alias for ExtractValue chain, and we need to alias it again to
-    // make it a NamedExpression.
-    case u: UnresolvedAttribute => UnresolvedAlias(u)
-
-    case u: UnresolvedExtractValue => UnresolvedAlias(u)
-
     case expr: NamedExpression => expr
 
     // Leave an unaliased generator with an empty list of names since the analyzer will generate
     // the correct defaults after the nested expression's type has been resolved.
     case g: Generator => MultiAlias(g, Nil)
 
-    case func: UnresolvedFunction => UnresolvedAlias(func, Some(Column.generateAlias))
-
     // If we have a top level Cast, there is a chance to give it a better alias, if there is a
     // NamedExpression under this Cast.
     case c: Cast =>
@@ -192,16 +182,10 @@ class Column(val expr: Expression) extends Logging {
         case c @ Cast(_: NamedExpression, _, _) => UnresolvedAlias(c)
       } match {
         case ne: NamedExpression => ne
-        case _ => Alias(expr, toPrettySQL(expr))()
+        case _ => UnresolvedAlias(expr, Some(Column.generateAlias))
       }
 
-    case a: AggregateExpression if a.aggregateFunction.isInstanceOf[TypedAggregateExpression] =>
-      UnresolvedAlias(a, Some(Column.generateAlias))
-
-    // Wait until the struct is resolved. This will generate a nicer looking alias.
-    case struct: CreateNamedStruct => UnresolvedAlias(struct)
-
-    case expr: Expression => Alias(expr, toPrettySQL(expr))()
+    case expr: Expression => UnresolvedAlias(expr, Some(Column.generateAlias))
   }
 
   /**
@@ -871,6 +855,129 @@ class Column(val expr: Expression) extends Logging {
    */
   def getItem(key: Any): Column = withExpr { UnresolvedExtractValue(expr, Literal(key)) }
 
+  // scalastyle:off line.size.limit
+  /**
+   * An expression that adds/replaces field in `StructType` by name.
+   *
+   * {{{
+   *   val df = sql("SELECT named_struct('a', 1, 'b', 2) struct_col")
+   *   df.select($"struct_col".withField("c", lit(3)))
+   *   // result: {"a":1,"b":2,"c":3}
+   *
+   *   val df = sql("SELECT named_struct('a', 1, 'b', 2) struct_col")
+   *   df.select($"struct_col".withField("b", lit(3)))
+   *   // result: {"a":1,"b":3}
+   *
+   *   val df = sql("SELECT CAST(NULL AS struct<a:int,b:int>) struct_col")
+   *   df.select($"struct_col".withField("c", lit(3)))
+   *   // result: null of type struct<a:int,b:int,c:int>
+   *
+   *   val df = sql("SELECT named_struct('a', 1, 'b', 2, 'b', 3) struct_col")
+   *   df.select($"struct_col".withField("b", lit(100)))
+   *   // result: {"a":1,"b":100,"b":100}
+   *
+   *   val df = sql("SELECT named_struct('a', named_struct('a', 1, 'b', 2)) struct_col")
+   *   df.select($"struct_col".withField("a.c", lit(3)))
+   *   // result: {"a":{"a":1,"b":2,"c":3}}
+   *
+   *   val df = sql("SELECT named_struct('a', named_struct('b', 1), 'a', named_struct('c', 2)) struct_col")
+   *   df.select($"struct_col".withField("a.c", lit(3)))
+   *   // result: org.apache.spark.sql.AnalysisException: Ambiguous reference to fields
+   * }}}
+   *
+   * This method supports adding/replacing nested fields directly e.g.
+   *
+   * {{{
+   *   val df = sql("SELECT named_struct('a', named_struct('a', 1, 'b', 2)) struct_col")
+   *   df.select($"struct_col".withField("a.c", lit(3)).withField("a.d", lit(4)))
+   *   // result: {"a":{"a":1,"b":2,"c":3,"d":4}}
+   * }}}
+   *
+   * However, if you are going to add/replace multiple nested fields, it is more optimal to extract
+   * out the nested struct before adding/replacing multiple fields e.g.
+   *
+   * {{{
+   *   val df = sql("SELECT named_struct('a', named_struct('a', 1, 'b', 2)) struct_col")
+   *   df.select($"struct_col".withField("a", $"struct_col.a".withField("c", lit(3)).withField("d", lit(4))))
+   *   // result: {"a":{"a":1,"b":2,"c":3,"d":4}}
+   * }}}
+   *
+   * @group expr_ops
+   * @since 3.1.0
+   */
+  // scalastyle:on line.size.limit
+  def withField(fieldName: String, col: Column): Column = withExpr {
+    require(fieldName != null, "fieldName cannot be null")
+    require(col != null, "col cannot be null")
+    UpdateFields(expr, fieldName, col.expr)
+  }
+
+  // scalastyle:off line.size.limit
+  /**
+   * An expression that drops fields in `StructType` by name.
+   * This is a no-op if schema doesn't contain field name(s).
+   *
+   * {{{
+   *   val df = sql("SELECT named_struct('a', 1, 'b', 2) struct_col")
+   *   df.select($"struct_col".dropFields("b"))
+   *   // result: {"a":1}
+   *
+   *   val df = sql("SELECT named_struct('a', 1, 'b', 2) struct_col")
+   *   df.select($"struct_col".dropFields("c"))
+   *   // result: {"a":1,"b":2}
+   *
+   *   val df = sql("SELECT named_struct('a', 1, 'b', 2, 'c', 3) struct_col")
+   *   df.select($"struct_col".dropFields("b", "c"))
+   *   // result: {"a":1}
+   *
+   *   val df = sql("SELECT named_struct('a', 1, 'b', 2) struct_col")
+   *   df.select($"struct_col".dropFields("a", "b"))
+   *   // result: org.apache.spark.sql.AnalysisException: cannot resolve 'update_fields(update_fields(`struct_col`))' due to data type mismatch: cannot drop all fields in struct
+   *
+   *   val df = sql("SELECT CAST(NULL AS struct<a:int,b:int>) struct_col")
+   *   df.select($"struct_col".dropFields("b"))
+   *   // result: null of type struct<a:int>
+   *
+   *   val df = sql("SELECT named_struct('a', 1, 'b', 2, 'b', 3) struct_col")
+   *   df.select($"struct_col".dropFields("b"))
+   *   // result: {"a":1}
+   *
+   *   val df = sql("SELECT named_struct('a', named_struct('a', 1, 'b', 2)) struct_col")
+   *   df.select($"struct_col".dropFields("a.b"))
+   *   // result: {"a":{"a":1}}
+   *
+   *   val df = sql("SELECT named_struct('a', named_struct('b', 1), 'a', named_struct('c', 2)) struct_col")
+   *   df.select($"struct_col".dropFields("a.c"))
+   *   // result: org.apache.spark.sql.AnalysisException: Ambiguous reference to fields
+   * }}}
+   *
+   * This method supports dropping multiple nested fields directly e.g.
+   *
+   * {{{
+   *   val df = sql("SELECT named_struct('a', named_struct('a', 1, 'b', 2)) struct_col")
+   *   df.select($"struct_col".dropFields("a.b", "a.c"))
+   *   // result: {"a":{"a":1}}
+   * }}}
+   *
+   * However, if you are going to drop multiple nested fields, it is more optimal to extract
+   * out the nested struct before dropping multiple fields from it e.g.
+   *
+   * {{{
+   *   val df = sql("SELECT named_struct('a', named_struct('a', 1, 'b', 2)) struct_col")
+   *   df.select($"struct_col".withField("a", $"struct_col.a".dropFields("b", "c")))
+   *   // result: {"a":{"a":1}}
+   * }}}
+   *
+   * @group expr_ops
+   * @since 3.1.0
+   */
+  // scalastyle:on line.size.limit
+  def dropFields(fieldNames: String*): Column = withExpr {
+    fieldNames.tail.foldLeft(UpdateFields(expr, fieldNames.head)) {
+      (resExpr, fieldName) => UpdateFields(resExpr, fieldName)
+    }
+  }
+
   /**
    * An expression that gets a field by name in a `StructType`.
    *
@@ -1001,7 +1108,7 @@ class Column(val expr: Expression) extends Logging {
    * Gives the column an alias.
    * {{{
    *   // Renames colA to colB in select output.
-   *   df.select($"colA".as('colB))
+   *   df.select($"colA".as("colB"))
    * }}}
    *
    * If the current column has metadata associated with it, this metadata will be propagated
@@ -1042,7 +1149,11 @@ class Column(val expr: Expression) extends Logging {
    * @since 2.0.0
    */
   def name(alias: String): Column = withExpr {
-    Alias(expr, alias)()
+    // SPARK-33536: an alias is no longer a column reference. Therefore,
+    // we should not inherit the column reference related metadata in an alias
+    // so that it is not caught as a column reference in DetectAmbiguousSelfJoin.
+    Alias(expr, alias)(
+      nonInheritableMetadataKeys = Seq(Dataset.DATASET_ID_KEY, Dataset.COL_POS_KEY))
   }
 
   /**
@@ -1059,7 +1170,11 @@ class Column(val expr: Expression) extends Logging {
    * @group expr_ops
    * @since 1.3.0
    */
-  def cast(to: DataType): Column = withExpr { Cast(expr, to) }
+  def cast(to: DataType): Column = withExpr {
+    val cast = Cast(expr, CharVarcharUtils.replaceCharVarcharWithStringForCast(to))
+    cast.setTagValue(Cast.USER_SPECIFIED_CAST, true)
+    cast
+  }
 
   /**
    * Casts the column to a different data type, using the canonical string representation
@@ -1104,7 +1219,7 @@ class Column(val expr: Expression) extends Logging {
    * @group expr_ops
    * @since 2.1.0
    */
-  def desc_nulls_first: Column = withExpr { SortOrder(expr, Descending, NullsFirst, Set.empty) }
+  def desc_nulls_first: Column = withExpr { SortOrder(expr, Descending, NullsFirst, Seq.empty) }
 
   /**
    * Returns a sort expression based on the descending order of the column,
@@ -1120,7 +1235,7 @@ class Column(val expr: Expression) extends Logging {
    * @group expr_ops
    * @since 2.1.0
    */
-  def desc_nulls_last: Column = withExpr { SortOrder(expr, Descending, NullsLast, Set.empty) }
+  def desc_nulls_last: Column = withExpr { SortOrder(expr, Descending, NullsLast, Seq.empty) }
 
   /**
    * Returns a sort expression based on ascending order of the column.
@@ -1151,7 +1266,7 @@ class Column(val expr: Expression) extends Logging {
    * @group expr_ops
    * @since 2.1.0
    */
-  def asc_nulls_first: Column = withExpr { SortOrder(expr, Ascending, NullsFirst, Set.empty) }
+  def asc_nulls_first: Column = withExpr { SortOrder(expr, Ascending, NullsFirst, Seq.empty) }
 
   /**
    * Returns a sort expression based on ascending order of the column,
@@ -1167,7 +1282,7 @@ class Column(val expr: Expression) extends Logging {
    * @group expr_ops
    * @since 2.1.0
    */
-  def asc_nulls_last: Column = withExpr { SortOrder(expr, Ascending, NullsLast, Set.empty) }
+  def asc_nulls_last: Column = withExpr { SortOrder(expr, Ascending, NullsLast, Seq.empty) }
 
   /**
    * Prints the expression to the console for debugging purposes.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
index bbf0ac1dd85e9..91905f259c1e1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
@@ -327,9 +327,9 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
    */
   def replace[T](col: String, replacement: Map[T, T]): DataFrame = {
     if (col == "*") {
-      replace0(df.columns, replacement)
+      replace0(df.logicalPlan.output, replacement)
     } else {
-      replace0(Seq(col), replacement)
+      replace(Seq(col), replacement)
     }
   }
 
@@ -352,10 +352,21 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
    *
    * @since 1.3.1
    */
-  def replace[T](cols: Seq[String], replacement: Map[T, T]): DataFrame = replace0(cols, replacement)
+  def replace[T](cols: Seq[String], replacement: Map[T, T]): DataFrame = {
+    val attrs = cols.map { colName =>
+      // Check column name exists
+      val attr = df.resolve(colName) match {
+        case a: Attribute => a
+        case _ => throw new UnsupportedOperationException(
+          s"Nested field ${colName} is not supported.")
+      }
+      attr
+    }
+    replace0(attrs, replacement)
+  }
 
-  private def replace0[T](cols: Seq[String], replacement: Map[T, T]): DataFrame = {
-    if (replacement.isEmpty || cols.isEmpty) {
+  private def replace0[T](attrs: Seq[Attribute], replacement: Map[T, T]): DataFrame = {
+    if (replacement.isEmpty || attrs.isEmpty) {
       return df
     }
 
@@ -379,15 +390,13 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
       case _: String => StringType
     }
 
-    val columnEquals = df.sparkSession.sessionState.analyzer.resolver
-    val projections = df.schema.fields.map { f =>
-      val shouldReplace = cols.exists(colName => columnEquals(colName, f.name))
-      if (f.dataType.isInstanceOf[NumericType] && targetColumnType == DoubleType && shouldReplace) {
-        replaceCol(f, replacementMap)
-      } else if (f.dataType == targetColumnType && shouldReplace) {
-        replaceCol(f, replacementMap)
+    val output = df.queryExecution.analyzed.output
+    val projections = output.map { attr =>
+      if (attrs.contains(attr) && (attr.dataType == targetColumnType ||
+        (attr.dataType.isInstanceOf[NumericType] && targetColumnType == DoubleType))) {
+        replaceCol(attr, replacementMap)
       } else {
-        df.col(f.name)
+        Column(attr)
       }
     }
     df.select(projections : _*)
@@ -395,10 +404,13 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
 
   private def fillMap(values: Seq[(String, Any)]): DataFrame = {
     // Error handling
-    values.foreach { case (colName, replaceValue) =>
+    val attrToValue = AttributeMap(values.map { case (colName, replaceValue) =>
       // Check column name exists
-      df.resolve(colName)
-
+      val attr = df.resolve(colName) match {
+        case a: Attribute => a
+        case _ => throw new UnsupportedOperationException(
+          s"Nested field ${colName} is not supported.")
+      }
       // Check data type
       replaceValue match {
         case _: jl.Double | _: jl.Float | _: jl.Integer | _: jl.Long | _: jl.Boolean | _: String =>
@@ -406,31 +418,29 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
         case _ => throw new IllegalArgumentException(
           s"Unsupported value type ${replaceValue.getClass.getName} ($replaceValue).")
       }
-    }
-
-    val columnEquals = df.sparkSession.sessionState.analyzer.resolver
-    val projections = df.schema.fields.map { f =>
-      values.find { case (k, _) => columnEquals(k, f.name) }.map { case (_, v) =>
-        v match {
-          case v: jl.Float => fillCol[Float](f, v)
-          case v: jl.Double => fillCol[Double](f, v)
-          case v: jl.Long => fillCol[Long](f, v)
-          case v: jl.Integer => fillCol[Integer](f, v)
-          case v: jl.Boolean => fillCol[Boolean](f, v.booleanValue())
-          case v: String => fillCol[String](f, v)
-        }
-      }.getOrElse(df.col(f.name))
+      attr -> replaceValue
+    })
+
+    val output = df.queryExecution.analyzed.output
+    val projections = output.map {
+      attr => attrToValue.get(attr).map {
+        case v: jl.Float => fillCol[Float](attr, v)
+        case v: jl.Double => fillCol[Double](attr, v)
+        case v: jl.Long => fillCol[Long](attr, v)
+        case v: jl.Integer => fillCol[Integer](attr, v)
+        case v: jl.Boolean => fillCol[Boolean](attr, v.booleanValue())
+        case v: String => fillCol[String](attr, v)
+      }.getOrElse(Column(attr))
     }
     df.select(projections : _*)
   }
 
   /**
-   * Returns a [[Column]] expression that replaces null value in `col` with `replacement`.
-   * It selects a column based on its name.
+   * Returns a [[Column]] expression that replaces null value in column defined by `attr`
+   * with `replacement`.
    */
-  private def fillCol[T](col: StructField, replacement: T): Column = {
-    val quotedColName = "`" + col.name + "`"
-    fillCol(col.dataType, col.name, df.col(quotedColName), replacement)
+  private def fillCol[T](attr: Attribute, replacement: T): Column = {
+    fillCol(attr.dataType, attr.name, Column(attr), replacement)
   }
 
   /**
@@ -452,13 +462,12 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
    *
    * TODO: This can be optimized to use broadcast join when replacementMap is large.
    */
-  private def replaceCol[K, V](col: StructField, replacementMap: Map[K, V]): Column = {
-    val keyExpr = df.col(col.name).expr
-    def buildExpr(v: Any) = Cast(Literal(v), keyExpr.dataType)
+  private def replaceCol[K, V](attr: Attribute, replacementMap: Map[K, V]): Column = {
+    def buildExpr(v: Any) = Cast(Literal(v), attr.dataType)
     val branches = replacementMap.flatMap { case (source, target) =>
       Seq(Literal(source), buildExpr(target))
     }.toSeq
-    new Column(CaseKeyWhen(keyExpr, branches :+ keyExpr)).as(col.name)
+    new Column(CaseKeyWhen(attr, branches :+ attr)).as(attr.name)
   }
 
   private def convertToDouble(v: Any): Double = v match {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
index a7b3d08ec4df7..f7e1903da6877 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
@@ -28,10 +28,11 @@ import org.apache.spark.annotation.Stable
 import org.apache.spark.api.java.JavaRDD
 import org.apache.spark.internal.Logging
 import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
 import org.apache.spark.sql.catalyst.csv.{CSVHeaderChecker, CSVOptions, UnivocityParser}
 import org.apache.spark.sql.catalyst.expressions.ExprUtils
 import org.apache.spark.sql.catalyst.json.{CreateJacksonParser, JacksonParser, JSONOptions}
-import org.apache.spark.sql.catalyst.util.FailureSafeParser
+import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, CharVarcharUtils, FailureSafeParser}
 import org.apache.spark.sql.connector.catalog.{CatalogV2Util, SupportsCatalogOptions, SupportsRead}
 import org.apache.spark.sql.connector.catalog.TableCapability._
 import org.apache.spark.sql.execution.command.DDLUtils
@@ -40,6 +41,7 @@ import org.apache.spark.sql.execution.datasources.csv._
 import org.apache.spark.sql.execution.datasources.jdbc._
 import org.apache.spark.sql.execution.datasources.json.TextInputJsonDataSource
 import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Relation, DataSourceV2Utils}
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 import org.apache.spark.unsafe.types.UTF8String
@@ -71,7 +73,10 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
    * @since 1.4.0
    */
   def schema(schema: StructType): DataFrameReader = {
-    this.userSpecifiedSchema = Option(schema)
+    if (schema != null) {
+      val replaced = CharVarcharUtils.failIfHasCharVarchar(schema).asInstanceOf[StructType]
+      this.userSpecifiedSchema = Option(replaced)
+    }
     this
   }
 
@@ -87,13 +92,15 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
    * @since 2.3.0
    */
   def schema(schemaString: String): DataFrameReader = {
-    this.userSpecifiedSchema = Option(StructType.fromDDL(schemaString))
-    this
+    schema(StructType.fromDDL(schemaString))
   }
 
   /**
    * Adds an input option for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * You can set the following option(s):
    * <ul>
    * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
@@ -114,13 +121,16 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
    * @since 1.4.0
    */
   def option(key: String, value: String): DataFrameReader = {
-    this.extraOptions += (key -> value)
+    this.extraOptions = this.extraOptions + (key -> value)
     this
   }
 
   /**
    * Adds an input option for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * @since 2.0.0
    */
   def option(key: String, value: Boolean): DataFrameReader = option(key, value.toString)
@@ -128,6 +138,9 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
   /**
    * Adds an input option for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * @since 2.0.0
    */
   def option(key: String, value: Long): DataFrameReader = option(key, value.toString)
@@ -135,6 +148,9 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
   /**
    * Adds an input option for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * @since 2.0.0
    */
   def option(key: String, value: Double): DataFrameReader = option(key, value.toString)
@@ -142,6 +158,9 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
   /**
    * (Scala-specific) Adds input options for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * You can set the following option(s):
    * <ul>
    * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
@@ -169,6 +188,9 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
   /**
    * Adds input options for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * You can set the following option(s):
    * <ul>
    * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
@@ -211,7 +233,11 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
    */
   def load(path: String): DataFrame = {
     // force invocation of `load(...varargs...)`
-    option("path", path).load(Seq.empty: _*)
+    if (sparkSession.sessionState.conf.legacyPathOptionBehavior) {
+      option("path", path).load(Seq.empty: _*)
+    } else {
+      load(Seq(path): _*)
+    }
   }
 
   /**
@@ -227,18 +253,24 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
         "read files of Hive data source directly.")
     }
 
+    val legacyPathOptionBehavior = sparkSession.sessionState.conf.legacyPathOptionBehavior
+    if (!legacyPathOptionBehavior &&
+        (extraOptions.contains("path") || extraOptions.contains("paths")) && paths.nonEmpty) {
+      throw new AnalysisException("There is a 'path' or 'paths' option set and load() is called " +
+        "with path parameters. Either remove the path option if it's the same as the path " +
+        "parameter, or add it to the load() parameter if you do want to read multiple paths. " +
+        s"To ignore this check, set '${SQLConf.LEGACY_PATH_OPTION_BEHAVIOR.key}' to 'true'.")
+    }
+
     DataSource.lookupDataSourceV2(source, sparkSession.sessionState.conf).map { provider =>
       val catalogManager = sparkSession.sessionState.catalogManager
       val sessionOptions = DataSourceV2Utils.extractSessionConfigs(
         source = provider, conf = sparkSession.sessionState.conf)
-      val pathsOption = if (paths.isEmpty) {
-        None
-      } else {
-        val objectMapper = new ObjectMapper()
-        Some("paths" -> objectMapper.writeValueAsString(paths.toArray))
-      }
 
-      val finalOptions = sessionOptions ++ extraOptions.toMap ++ pathsOption
+      val optionsWithPath = getOptionsWithPaths(paths: _*)
+
+      val finalOptions = sessionOptions.filterKeys(!optionsWithPath.contains(_)).toMap ++
+        optionsWithPath.originalMap
       val dsOptions = new CaseInsensitiveStringMap(finalOptions.asJava)
       val (table, catalog, ident) = provider match {
         case _: SupportsCatalogOptions if userSpecifiedSchema.nonEmpty =>
@@ -268,15 +300,33 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
     }.getOrElse(loadV1Source(paths: _*))
   }
 
+  private def getOptionsWithPaths(paths: String*): CaseInsensitiveMap[String] = {
+    if (paths.isEmpty) {
+      extraOptions
+    } else if (paths.length == 1) {
+      extraOptions + ("path" -> paths.head)
+    } else {
+      val objectMapper = new ObjectMapper()
+      extraOptions + ("paths" -> objectMapper.writeValueAsString(paths.toArray))
+    }
+  }
+
   private def loadV1Source(paths: String*) = {
+    val legacyPathOptionBehavior = sparkSession.sessionState.conf.legacyPathOptionBehavior
+    val (finalPaths, finalOptions) = if (!legacyPathOptionBehavior && paths.length == 1) {
+      (Nil, extraOptions + ("path" -> paths.head))
+    } else {
+      (paths, extraOptions)
+    }
+
     // Code path for data source v1.
     sparkSession.baseRelationToDataFrame(
       DataSource.apply(
         sparkSession,
-        paths = paths,
+        paths = finalPaths,
         userSpecifiedSchema = userSpecifiedSchema,
         className = source,
-        options = extraOptions.toMap).resolveRelation())
+        options = finalOptions.originalMap).resolveRelation())
   }
 
   /**
@@ -290,7 +340,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
     // properties should override settings in extraOptions.
     this.extraOptions ++= properties.asScala
     // explicit url and dbtable should override all
-    this.extraOptions += (JDBCOptions.JDBC_URL -> url, JDBCOptions.JDBC_TABLE_NAME -> table)
+    this.extraOptions ++= Seq(JDBCOptions.JDBC_URL -> url, JDBCOptions.JDBC_TABLE_NAME -> table)
     format("jdbc").load()
   }
 
@@ -361,7 +411,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
       connectionProperties: Properties): DataFrame = {
     assertNoSpecifiedSchema("jdbc")
     // connectionProperties should override settings in extraOptions.
-    val params = extraOptions.toMap ++ connectionProperties.asScala.toMap
+    val params = extraOptions ++ connectionProperties.asScala
     val options = new JDBCOptions(url, table, params)
     val parts: Array[Partition] = predicates.zipWithIndex.map { case (part, i) =>
       JDBCPartition(part, i) : Partition
@@ -449,8 +499,22 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
    * <li>`pathGlobFilter`: an optional glob pattern to only include files with paths matching
    * the pattern. The syntax follows <code>org.apache.hadoop.fs.GlobFilter</code>.
    * It does not change the behavior of partition discovery.</li>
+   * <li>`modifiedBefore` (batch only): an optional timestamp to only include files with
+   * modification times  occurring before the specified Time. The provided timestamp
+   * must be in the following form: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)</li>
+   * <li>`modifiedAfter` (batch only): an optional timestamp to only include files with
+   * modification times occurring after the specified Time. The provided timestamp
+   * must be in the following form: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)</li>
    * <li>`recursiveFileLookup`: recursively scan a directory for files. Using this option
    * disables partition discovery</li>
+   * <li>`allowNonNumericNumbers` (default `true`): allows JSON parser to recognize set of
+   * "Not-a-Number" (NaN) tokens as legal floating number values:
+   *   <ul>
+   *     <li>`+INF` for positive infinity, as well as alias of `+Infinity` and `Infinity`.
+   *     <li>`-INF` for negative infinity), alias `-Infinity`.
+   *     <li>`NaN` for other not-a-numbers, like result of division by zero.
+   *   </ul>
+   * </li>
    * </ul>
    *
    * @since 2.0.0
@@ -548,6 +612,9 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
    * If the enforceSchema is set to `false`, only the CSV header in the first line is checked
    * to conform specified or inferred schema.
    *
+   * @note if `header` option is set to `true` when calling this API, all lines same with
+   * the header will be removed if exists.
+   *
    * @param csvDataset input Dataset with one CSV row per record
    * @since 2.2.0
    */
@@ -666,6 +733,27 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
    * a record can have.</li>
    * <li>`maxCharsPerColumn` (default `-1`): defines the maximum number of characters allowed
    * for any given value being read. By default, it is -1 meaning unlimited length</li>
+   * <li>`unescapedQuoteHandling` (default `STOP_AT_DELIMITER`): defines how the CsvParser
+   * will handle values with unescaped quotes.
+   *   <ul>
+   *     <li>`STOP_AT_CLOSING_QUOTE`: If unescaped quotes are found in the input, accumulate
+   *     the quote character and proceed parsing the value as a quoted value, until a closing
+   *     quote is found.</li>
+   *     <li>`BACK_TO_DELIMITER`: If unescaped quotes are found in the input, consider the value
+   *     as an unquoted value. This will make the parser accumulate all characters of the current
+   *     parsed value until the delimiter is found. If no
+   *     delimiter is found in the value, the parser will continue accumulating characters from
+   *     the input until a delimiter or line ending is found.</li>
+   *     <li>`STOP_AT_DELIMITER`: If unescaped quotes are found in the input, consider the value
+   *     as an unquoted value. This will make the parser accumulate all characters until the
+   *     delimiter or a line ending is found in the input.</li>
+   *     <li>`SKIP_VALUE`: If unescaped quotes are found in the input, the content parsed
+   *     for the given value will be skipped and the value set in nullValue will be produced
+   *     instead.</li>
+   *     <li>`RAISE_ERROR`: If unescaped quotes are found in the input, a TextParsingException
+   *     will be thrown.</li>
+   *   </ul>
+   * </li>
    * <li>`mode` (default `PERMISSIVE`): allows a mode for dealing with corrupt records
    *    during parsing. It supports the following case-insensitive modes. Note that Spark tries
    *    to parse only required columns in CSV under column pruning. Therefore, corrupt records
@@ -695,6 +783,12 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
    * <li>`pathGlobFilter`: an optional glob pattern to only include files with paths matching
    * the pattern. The syntax follows <code>org.apache.hadoop.fs.GlobFilter</code>.
    * It does not change the behavior of partition discovery.</li>
+   * <li>`modifiedBefore` (batch only): an optional timestamp to only include files with
+   * modification times  occurring before the specified Time. The provided timestamp
+   * must be in the following form: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)</li>
+   * <li>`modifiedAfter` (batch only): an optional timestamp to only include files with
+   * modification times occurring after the specified Time. The provided timestamp
+   * must be in the following form: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)</li>
    * <li>`recursiveFileLookup`: recursively scan a directory for files. Using this option
    * disables partition discovery</li>
    * </ul>
@@ -726,8 +820,37 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
    * <li>`pathGlobFilter`: an optional glob pattern to only include files with paths matching
    * the pattern. The syntax follows <code>org.apache.hadoop.fs.GlobFilter</code>.
    * It does not change the behavior of partition discovery.</li>
+   * <li>`modifiedBefore` (batch only): an optional timestamp to only include files with
+   * modification times  occurring before the specified Time. The provided timestamp
+   * must be in the following form: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)</li>
+   * <li>`modifiedAfter` (batch only): an optional timestamp to only include files with
+   * modification times occurring after the specified Time. The provided timestamp
+   * must be in the following form: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)</li>
    * <li>`recursiveFileLookup`: recursively scan a directory for files. Using this option
    * disables partition discovery</li>
+   * <li>`datetimeRebaseMode` (default is the value specified in the SQL config
+   * `spark.sql.parquet.datetimeRebaseModeInRead`): the rebasing mode for the values
+   * of the `DATE`, `TIMESTAMP_MICROS`, `TIMESTAMP_MILLIS` logical types from the Julian to
+   * Proleptic Gregorian calendar:
+   *   <ul>
+   *     <li>`EXCEPTION` : Spark fails in reads of ancient dates/timestamps that are ambiguous
+   *     between the two calendars</li>
+   *     <li>`CORRECTED` : loading of dates/timestamps without rebasing</li>
+   *     <li>`LEGACY` : perform rebasing of ancient dates/timestamps from the Julian to Proleptic
+   *     Gregorian calendar</li>
+   *   </ul>
+   * </li>
+   * <li>`int96RebaseMode` (default is the value specified in the SQL config
+   * `spark.sql.parquet.int96RebaseModeInRead`): the rebasing mode for `INT96` timestamps
+   * from the Julian to Proleptic Gregorian calendar:
+   *   <ul>
+   *     <li>`EXCEPTION` : Spark fails in reads of ancient `INT96` timestamps that are ambiguous
+   *     between the two calendars</li>
+   *     <li>`CORRECTED` : loading of timestamps without rebasing</li>
+   *     <li>`LEGACY` : perform rebasing of ancient `INT96` timestamps from the Julian to Proleptic
+   *     Gregorian calendar</li>
+   *   </ul>
+   * </li>
    * </ul>
    *
    * @since 1.4.0
@@ -759,6 +882,12 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
    * <li>`pathGlobFilter`: an optional glob pattern to only include files with paths matching
    * the pattern. The syntax follows <code>org.apache.hadoop.fs.GlobFilter</code>.
    * It does not change the behavior of partition discovery.</li>
+   * <li>`modifiedBefore` (batch only): an optional timestamp to only include files with
+   * modification times  occurring before the specified Time. The provided timestamp
+   * must be in the following form: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)</li>
+   * <li>`modifiedAfter` (batch only): an optional timestamp to only include files with
+   * modification times occurring after the specified Time. The provided timestamp
+   * must be in the following form: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)</li>
    * <li>`recursiveFileLookup`: recursively scan a directory for files. Using this option
    * disables partition discovery</li>
    * </ul>
@@ -770,13 +899,24 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
   def orc(paths: String*): DataFrame = format("orc").load(paths: _*)
 
   /**
-   * Returns the specified table as a `DataFrame`.
+   * Returns the specified table/view as a `DataFrame`. If it's a table, it must support batch
+   * reading and the returned DataFrame is the batch scan query plan of this table. If it's a view,
+   * the returned DataFrame is simply the query plan of the view, which can either be a batch or
+   * streaming query plan.
    *
+   * @param tableName is either a qualified or unqualified name that designates a table or view.
+   *                  If a database is specified, it identifies the table/view from the database.
+   *                  Otherwise, it first attempts to find a temporary view with the given name
+   *                  and then match the table/view from the current database.
+   *                  Note that, the global temporary view database is also valid here.
    * @since 1.4.0
    */
   def table(tableName: String): DataFrame = {
     assertNoSpecifiedSchema("table")
-    sparkSession.table(tableName)
+    val multipartIdentifier =
+      sparkSession.sessionState.sqlParser.parseMultipartIdentifier(tableName)
+    Dataset.ofRows(sparkSession, UnresolvedRelation(multipartIdentifier,
+      new CaseInsensitiveStringMap(extraOptions.toMap.asJava)))
   }
 
   /**
@@ -814,6 +954,12 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
    * <li>`pathGlobFilter`: an optional glob pattern to only include files with paths matching
    * the pattern. The syntax follows <code>org.apache.hadoop.fs.GlobFilter</code>.
    * It does not change the behavior of partition discovery.</li>
+   * <li>`modifiedBefore` (batch only): an optional timestamp to only include files with
+   * modification times  occurring before the specified Time. The provided timestamp
+   * must be in the following form: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)</li>
+   * <li>`modifiedAfter` (batch only): an optional timestamp to only include files with
+   * modification times occurring after the specified Time. The provided timestamp
+   * must be in the following form: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)</li>
    * <li>`recursiveFileLookup`: recursively scan a directory for files. Using this option
    * disables partition discovery</li>
    * </ul>
@@ -879,6 +1025,6 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
 
   private var userSpecifiedSchema: Option[StructType] = None
 
-  private val extraOptions = new scala.collection.mutable.HashMap[String, String]
+  private var extraOptions = CaseInsensitiveMap[String](Map.empty)
 
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
index 9aef382470bcc..fe6572cff5de2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
@@ -27,6 +27,7 @@ import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, NoSuchT
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.expressions.Literal
 import org.apache.spark.sql.catalyst.plans.logical.{AppendData, CreateTableAsSelect, CreateTableAsSelectStatement, InsertIntoStatement, LogicalPlan, OverwriteByExpression, OverwritePartitionsDynamic, ReplaceTableAsSelectStatement}
+import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
 import org.apache.spark.sql.connector.catalog.{CatalogPlugin, CatalogV2Implicits, CatalogV2Util, Identifier, SupportsCatalogOptions, Table, TableCatalog, TableProvider, V1Table}
 import org.apache.spark.sql.connector.catalog.TableCapability._
 import org.apache.spark.sql.connector.expressions.{FieldReference, IdentityTransform, Transform}
@@ -34,6 +35,7 @@ import org.apache.spark.sql.execution.SQLExecution
 import org.apache.spark.sql.execution.command.DDLUtils
 import org.apache.spark.sql.execution.datasources.{CreateTable, DataSource, DataSourceUtils, LogicalRelation}
 import org.apache.spark.sql.execution.datasources.v2._
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.PartitionOverwriteMode
 import org.apache.spark.sql.sources.BaseRelation
 import org.apache.spark.sql.types.StructType
@@ -59,8 +61,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
    * <li>`SaveMode.ErrorIfExists`: throw an exception at runtime.</li>
    * </ul>
    * <p>
-   * When writing to data source v1, the default option is `ErrorIfExists`. When writing to data
-   * source v2, the default option is `Append`.
+   * The default option is `ErrorIfExists`.
    *
    * @since 1.4.0
    */
@@ -104,6 +105,9 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   /**
    * Adds an output option for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * You can set the following option(s):
    * <ul>
    * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
@@ -124,13 +128,16 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
    * @since 1.4.0
    */
   def option(key: String, value: String): DataFrameWriter[T] = {
-    this.extraOptions += (key -> value)
+    this.extraOptions = this.extraOptions + (key -> value)
     this
   }
 
   /**
    * Adds an output option for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * @since 2.0.0
    */
   def option(key: String, value: Boolean): DataFrameWriter[T] = option(key, value.toString)
@@ -138,6 +145,9 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   /**
    * Adds an output option for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * @since 2.0.0
    */
   def option(key: String, value: Long): DataFrameWriter[T] = option(key, value.toString)
@@ -145,6 +155,9 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   /**
    * Adds an output option for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * @since 2.0.0
    */
   def option(key: String, value: Double): DataFrameWriter[T] = option(key, value.toString)
@@ -152,6 +165,9 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   /**
    * (Scala-specific) Adds output options for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * You can set the following option(s):
    * <ul>
    * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
@@ -179,6 +195,9 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   /**
    * Adds output options for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * You can set the following option(s):
    * <ul>
    * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
@@ -265,8 +284,13 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
    * @since 1.4.0
    */
   def save(path: String): Unit = {
-    this.extraOptions += ("path" -> path)
-    save()
+    if (!df.sparkSession.sessionState.conf.legacyPathOptionBehavior &&
+        extraOptions.contains("path")) {
+      throw new AnalysisException("There is a 'path' option set and save() is called with a path " +
+        "parameter. Either remove the path option, or call save() without the parameter. " +
+        s"To ignore this check, set '${SQLConf.LEGACY_PATH_OPTION_BEHAVIOR.key}' to 'true'.")
+    }
+    saveInternal(Some(path))
   }
 
   /**
@@ -274,7 +298,9 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
    *
    * @since 1.4.0
    */
-  def save(): Unit = {
+  def save(): Unit = saveInternal(None)
+
+  private def saveInternal(path: Option[String]): Unit = {
     if (source.toLowerCase(Locale.ROOT) == DDLUtils.HIVE_PROVIDER) {
       throw new AnalysisException("Hive data source can only be used with tables, you can not " +
         "write files of Hive data source directly.")
@@ -287,15 +313,20 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
       val provider = maybeV2Provider.get
       val sessionOptions = DataSourceV2Utils.extractSessionConfigs(
         provider, df.sparkSession.sessionState.conf)
-      val options = sessionOptions ++ extraOptions
-      val dsOptions = new CaseInsensitiveStringMap(options.asJava)
+
+      val optionsWithPath = getOptionsWithPath(path)
+
+      val finalOptions = sessionOptions.filterKeys(!optionsWithPath.contains(_)).toMap ++
+        optionsWithPath.originalMap
+      val dsOptions = new CaseInsensitiveStringMap(finalOptions.asJava)
 
       def getTable: Table = {
-        // For file source, it's expensive to infer schema/partition at each write. Here we pass
-        // the schema of input query and the user-specified partitioning to `getTable`. If the
-        // query schema is not compatible with the existing data, the write can still success but
-        // following reads would fail.
-        if (provider.isInstanceOf[FileDataSourceV2]) {
+        // If the source accepts external table metadata, here we pass the schema of input query
+        // and the user-specified partitioning to `getTable`. This is for avoiding
+        // schema/partitioning inference, which can be very expensive.
+        // If the query schema is not compatible with the existing data, the behavior is undefined.
+        // For example, writing file source will success but the following reads will fail.
+        if (provider.supportsExternalMetadata()) {
           provider.getTable(
             df.schema.asNullable,
             partitioningAsV2.toArray,
@@ -324,7 +355,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
                 // Streaming also uses the data source V2 API. So it may be that the data source
                 // implements v2, but has no v2 implementation for batch writes. In that case, we
                 // fall back to saving as though it's a V1 source.
-                return saveToV1Source()
+                return saveToV1Source(path)
               }
           }
 
@@ -332,14 +363,14 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
           checkPartitioningMatchesV2Table(table)
           if (mode == SaveMode.Append) {
             runCommand(df.sparkSession, "save") {
-              AppendData.byName(relation, df.logicalPlan, extraOptions.toMap)
+              AppendData.byName(relation, df.logicalPlan, finalOptions)
             }
           } else {
             // Truncate the table. TableCapabilityCheck will throw a nice exception if this
             // isn't supported
             runCommand(df.sparkSession, "save") {
               OverwriteByExpression.byName(
-                relation, df.logicalPlan, Literal(true), extraOptions.toMap)
+                relation, df.logicalPlan, Literal(true), finalOptions)
             }
           }
 
@@ -359,7 +390,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
                   partitioningAsV2,
                   df.queryExecution.analyzed,
                   Map(TableCatalog.PROP_PROVIDER -> source) ++ location,
-                  extraOptions.toMap,
+                  finalOptions,
                   ignoreIfExists = createMode == SaveMode.Ignore)
               }
             case _: TableProvider =>
@@ -371,29 +402,40 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
                 // Streaming also uses the data source V2 API. So it may be that the data source
                 // implements v2, but has no v2 implementation for batch writes. In that case, we
                 // fallback to saving as though it's a V1 source.
-                saveToV1Source()
+                saveToV1Source(path)
               }
           }
       }
 
     } else {
-      saveToV1Source()
+      saveToV1Source(path)
+    }
+  }
+
+  private def getOptionsWithPath(path: Option[String]): CaseInsensitiveMap[String] = {
+    if (path.isEmpty) {
+      extraOptions
+    } else {
+      extraOptions + ("path" -> path.get)
     }
   }
 
-  private def saveToV1Source(): Unit = {
+  private def saveToV1Source(path: Option[String]): Unit = {
     partitioningColumns.foreach { columns =>
-      extraOptions += (DataSourceUtils.PARTITIONING_COLUMNS_KEY ->
+      extraOptions = extraOptions + (
+        DataSourceUtils.PARTITIONING_COLUMNS_KEY ->
         DataSourceUtils.encodePartitioningColumns(columns))
     }
 
+    val optionsWithPath = getOptionsWithPath(path)
+
     // Code path for data source v1.
     runCommand(df.sparkSession, "save") {
       DataSource(
         sparkSession = df.sparkSession,
         className = source,
         partitionColumns = partitioningColumns.getOrElse(Nil),
-        options = extraOptions.toMap).planForWriting(mode, df.logicalPlan)
+        options = optionsWithPath.originalMap).planForWriting(mode, df.logicalPlan)
     }
   }
 
@@ -428,7 +470,6 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   def insertInto(tableName: String): Unit = {
     import df.sparkSession.sessionState.analyzer.{AsTableIdentifier, NonSessionCatalogAndIdentifier, SessionCatalogAndIdentifier}
     import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-    import org.apache.spark.sql.connector.catalog.CatalogV2Util._
 
     assertNotBucketed("insertInto")
 
@@ -495,6 +536,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
       InsertIntoStatement(
         table = UnresolvedRelation(tableIdent),
         partitionSpec = Map.empty[String, Option[String]],
+        Nil,
         query = df.logicalPlan,
         overwrite = mode == SaveMode.Overwrite,
         ifPartitionNotExists = false)
@@ -617,6 +659,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
           extraOptions.get("path"),
           extraOptions.get(TableCatalog.PROP_COMMENT),
           extraOptions.toMap,
+          None,
           orCreate = true)      // Create the table if it doesn't exist
 
       case (other, _) =>
@@ -634,7 +677,9 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
           extraOptions.get("path"),
           extraOptions.get(TableCatalog.PROP_COMMENT),
           extraOptions.toMap,
-          ifNotExists = other == SaveMode.Ignore)
+          None,
+          ifNotExists = other == SaveMode.Ignore,
+          external = false)
     }
 
     runCommand(df.sparkSession, "saveAsTable") {
@@ -768,7 +813,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
     // connectionProperties should override settings in extraOptions.
     this.extraOptions ++= connectionProperties.asScala
     // explicit url and dbtable should override all
-    this.extraOptions += ("url" -> url, "dbtable" -> table)
+    this.extraOptions ++= Seq("url" -> url, "dbtable" -> table)
     format("jdbc").save()
   }
 
@@ -840,7 +885,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
    * <ul>
    * <li>`compression` (default is the value specified in `spark.sql.orc.compression.codec`):
    * compression codec to use when saving to file. This can be one of the known case-insensitive
-   * shorten names(`none`, `snappy`, `zlib`, and `lzo`). This will override
+   * shorten names(`none`, `snappy`, `zlib`, `lzo`, and `zstd`). This will override
    * `orc.compress` and `spark.sql.orc.compression.codec`. If `orc.compress` is given,
    * it overrides `spark.sql.orc.compression.codec`.</li>
    * </ul>
@@ -960,7 +1005,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
 
   private var mode: SaveMode = SaveMode.ErrorIfExists
 
-  private val extraOptions = new scala.collection.mutable.HashMap[String, String]
+  private var extraOptions = CaseInsensitiveMap[String](Map.empty)
 
   private var partitioningColumns: Option[Seq[String]] = None
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriterV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriterV2.scala
index 15ff7fa0530cc..9a49fc3d74780 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriterV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriterV2.scala
@@ -21,12 +21,11 @@ import scala.collection.JavaConverters._
 import scala.collection.mutable
 
 import org.apache.spark.annotation.Experimental
-import org.apache.spark.sql.catalyst.analysis.{CannotReplaceMissingTableException, NoSuchTableException, TableAlreadyExistsException}
+import org.apache.spark.sql.catalyst.analysis.{CannotReplaceMissingTableException, NoSuchTableException, TableAlreadyExistsException, UnresolvedRelation}
 import org.apache.spark.sql.catalyst.expressions.{Attribute, Bucket, Days, Hours, Literal, Months, Years}
 import org.apache.spark.sql.catalyst.plans.logical.{AppendData, CreateTableAsSelectStatement, LogicalPlan, OverwriteByExpression, OverwritePartitionsDynamic, ReplaceTableAsSelectStatement}
 import org.apache.spark.sql.connector.expressions.{LogicalExpressions, NamedReference, Transform}
 import org.apache.spark.sql.execution.SQLExecution
-import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
 import org.apache.spark.sql.types.IntegerType
 
 /**
@@ -38,21 +37,12 @@ import org.apache.spark.sql.types.IntegerType
 final class DataFrameWriterV2[T] private[sql](table: String, ds: Dataset[T])
     extends CreateTableWriter[T] {
 
-  import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-  import org.apache.spark.sql.connector.catalog.CatalogV2Util._
-  import df.sparkSession.sessionState.analyzer.CatalogAndIdentifier
-
   private val df: DataFrame = ds.toDF()
 
   private val sparkSession = ds.sparkSession
 
   private val tableName = sparkSession.sessionState.sqlParser.parseMultipartIdentifier(table)
 
-  private val (catalog, identifier) = {
-    val CatalogAndIdentifier(catalog, identifier) = tableName
-    (catalog.asTableCatalog, identifier)
-  }
-
   private val logicalPlan = df.queryExecution.logical
 
   private var provider: Option[String] = None
@@ -129,7 +119,9 @@ final class DataFrameWriterV2[T] private[sql](table: String, ds: Dataset[T])
         None,
         None,
         options.toMap,
-        ifNotExists = false)
+        None,
+        ifNotExists = false,
+        external = false)
     }
   }
 
@@ -153,15 +145,7 @@ final class DataFrameWriterV2[T] private[sql](table: String, ds: Dataset[T])
    */
   @throws(classOf[NoSuchTableException])
   def append(): Unit = {
-    val append = loadTable(catalog, identifier) match {
-      case Some(t) =>
-        AppendData.byName(
-          DataSourceV2Relation.create(t, Some(catalog), Some(identifier)),
-          logicalPlan, options.toMap)
-      case _ =>
-        throw new NoSuchTableException(identifier)
-    }
-
+    val append = AppendData.byName(UnresolvedRelation(tableName), logicalPlan, options.toMap)
     runCommand("append")(append)
   }
 
@@ -177,15 +161,8 @@ final class DataFrameWriterV2[T] private[sql](table: String, ds: Dataset[T])
    */
   @throws(classOf[NoSuchTableException])
   def overwrite(condition: Column): Unit = {
-    val overwrite = loadTable(catalog, identifier) match {
-      case Some(t) =>
-        OverwriteByExpression.byName(
-          DataSourceV2Relation.create(t, Some(catalog), Some(identifier)),
-          logicalPlan, condition.expr, options.toMap)
-      case _ =>
-        throw new NoSuchTableException(identifier)
-    }
-
+    val overwrite = OverwriteByExpression.byName(
+      UnresolvedRelation(tableName), logicalPlan, condition.expr, options.toMap)
     runCommand("overwrite")(overwrite)
   }
 
@@ -204,15 +181,8 @@ final class DataFrameWriterV2[T] private[sql](table: String, ds: Dataset[T])
    */
   @throws(classOf[NoSuchTableException])
   def overwritePartitions(): Unit = {
-    val dynamicOverwrite = loadTable(catalog, identifier) match {
-      case Some(t) =>
-        OverwritePartitionsDynamic.byName(
-          DataSourceV2Relation.create(t, Some(catalog), Some(identifier)),
-          logicalPlan, options.toMap)
-      case _ =>
-        throw new NoSuchTableException(identifier)
-    }
-
+    val dynamicOverwrite = OverwritePartitionsDynamic.byName(
+      UnresolvedRelation(tableName), logicalPlan, options.toMap)
     runCommand("overwritePartitions")(dynamicOverwrite)
   }
 
@@ -239,6 +209,7 @@ final class DataFrameWriterV2[T] private[sql](table: String, ds: Dataset[T])
         None,
         None,
         options.toMap,
+        None,
         orCreate = orCreate)
     }
   }
@@ -324,7 +295,7 @@ trait CreateTableWriter[T] extends WriteConfigMethods[CreateTableWriter[T]] {
    * [[org.apache.spark.sql.catalyst.analysis.CannotReplaceMissingTableException]].
    *
    * @throws org.apache.spark.sql.catalyst.analysis.CannotReplaceMissingTableException
-   *         If the table already exists
+   *         If the table does not exist
    */
   @throws(classOf[CannotReplaceMissingTableException])
   def replace(): Unit
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index 12160c9f4c192..540115b919a44 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -21,7 +21,6 @@ import java.io.{ByteArrayOutputStream, CharArrayWriter, DataOutputStream}
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable.ArrayBuffer
-import scala.language.implicitConversions
 import scala.reflect.runtime.universe.TypeTag
 import scala.util.control.NonFatal
 
@@ -54,7 +53,7 @@ import org.apache.spark.sql.execution.aggregate.TypedAggregateExpression
 import org.apache.spark.sql.execution.arrow.{ArrowBatchStreamWriter, ArrowConverters}
 import org.apache.spark.sql.execution.command._
 import org.apache.spark.sql.execution.datasources.LogicalRelation
-import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2ScanRelation, FileTable}
+import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Relation, DataSourceV2ScanRelation, FileTable}
 import org.apache.spark.sql.execution.python.EvaluatePython
 import org.apache.spark.sql.execution.stat.StatFunctions
 import org.apache.spark.sql.internal.SQLConf
@@ -63,7 +62,7 @@ import org.apache.spark.sql.types._
 import org.apache.spark.sql.util.SchemaUtils
 import org.apache.spark.storage.StorageLevel
 import org.apache.spark.unsafe.array.ByteArrayMethods
-import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
+import org.apache.spark.unsafe.types.UTF8String
 import org.apache.spark.util.Utils
 
 private[sql] object Dataset {
@@ -227,12 +226,13 @@ class Dataset[T] private[sql](
     val plan = queryExecution.analyzed match {
       case c: Command =>
         LocalRelation(c.output, withAction("command", queryExecution)(_.executeCollect()))
-      case u @ Union(children) if children.forall(_.isInstanceOf[Command]) =>
+      case u @ Union(children, _, _) if children.forall(_.isInstanceOf[Command]) =>
         LocalRelation(u.output, withAction("command", queryExecution)(_.executeCollect()))
       case _ =>
         queryExecution.analyzed
     }
-    if (sparkSession.sessionState.conf.getConf(SQLConf.FAIL_AMBIGUOUS_SELF_JOIN_ENABLED)) {
+    if (sparkSession.sessionState.conf.getConf(SQLConf.FAIL_AMBIGUOUS_SELF_JOIN_ENABLED) &&
+        plan.getTagValue(Dataset.DATASET_ID_TAG).isEmpty) {
       plan.setTagValue(Dataset.DATASET_ID_TAG, id)
     }
     plan
@@ -260,15 +260,16 @@ class Dataset[T] private[sql](
   private[sql] def resolve(colName: String): NamedExpression = {
     val resolver = sparkSession.sessionState.analyzer.resolver
     queryExecution.analyzed.resolveQuoted(colName, resolver)
-      .getOrElse {
-        val fields = schema.fieldNames
-        val extraMsg = if (fields.exists(resolver(_, colName))) {
-          s"; did you mean to quote the `$colName` column?"
-        } else ""
-        val fieldsStr = fields.mkString(", ")
-        val errorMsg = s"""Cannot resolve column name "$colName" among (${fieldsStr})${extraMsg}"""
-        throw new AnalysisException(errorMsg)
-      }
+      .getOrElse(throw resolveException(colName, schema.fieldNames))
+  }
+
+  private def resolveException(colName: String, fields: Array[String]): AnalysisException = {
+    val extraMsg = if (fields.exists(sparkSession.sessionState.analyzer.resolver(_, colName))) {
+      s"; did you mean to quote the `$colName` column?"
+    } else ""
+    val fieldsStr = fields.mkString(", ")
+    val errorMsg = s"""Cannot resolve column name "$colName" among (${fieldsStr})${extraMsg}"""
+    new AnalysisException(errorMsg)
   }
 
   private[sql] def numericColumns: Seq[Expression] = {
@@ -302,12 +303,14 @@ class Dataset[T] private[sql](
     // For array values, replace Seq and Array with square brackets
     // For cells that are beyond `truncate` characters, replace it with the
     // first `truncate-3` and "..."
-    schema.fieldNames.toSeq +: data.map { row =>
+    schema.fieldNames.map(SchemaUtils.escapeMetaCharacters).toSeq +: data.map { row =>
       row.toSeq.map { cell =>
         val str = cell match {
           case null => "null"
           case binary: Array[Byte] => binary.map("%02X".format(_)).mkString("[", " ", "]")
-          case _ => cell.toString
+          case _ =>
+            // Escapes meta-characters not to break the `showString` format
+            SchemaUtils.escapeMetaCharacters(cell.toString)
         }
         if (truncate > 0 && str.length > truncate) {
           // do not show ellipses for strings shorter than 4 characters.
@@ -1084,8 +1087,8 @@ class Dataset[T] private[sql](
     }
 
     // If left/right have no output set intersection, return the plan.
-    val lanalyzed = withPlan(this.logicalPlan).queryExecution.analyzed
-    val ranalyzed = withPlan(right.logicalPlan).queryExecution.analyzed
+    val lanalyzed = this.queryExecution.analyzed
+    val ranalyzed = right.queryExecution.analyzed
     if (lanalyzed.outputSet.intersect(ranalyzed.outputSet).isEmpty) {
       return withPlan(plan)
     }
@@ -1093,17 +1096,22 @@ class Dataset[T] private[sql](
     // Otherwise, find the trivially true predicates and automatically resolves them to both sides.
     // By the time we get here, since we have already run analysis, all attributes should've been
     // resolved and become AttributeReference.
+    val resolver = sparkSession.sessionState.analyzer.resolver
     val cond = plan.condition.map { _.transform {
       case catalyst.expressions.EqualTo(a: AttributeReference, b: AttributeReference)
           if a.sameRef(b) =>
         catalyst.expressions.EqualTo(
-          withPlan(plan.left).resolve(a.name),
-          withPlan(plan.right).resolve(b.name))
+          plan.left.resolveQuoted(a.name, resolver)
+            .getOrElse(throw resolveException(a.name, plan.left.schema.fieldNames)),
+          plan.right.resolveQuoted(b.name, resolver)
+            .getOrElse(throw resolveException(b.name, plan.right.schema.fieldNames)))
       case catalyst.expressions.EqualNullSafe(a: AttributeReference, b: AttributeReference)
         if a.sameRef(b) =>
         catalyst.expressions.EqualNullSafe(
-          withPlan(plan.left).resolve(a.name),
-          withPlan(plan.right).resolve(b.name))
+          plan.left.resolveQuoted(a.name, resolver)
+            .getOrElse(throw resolveException(a.name, plan.left.schema.fieldNames)),
+          plan.right.resolveQuoted(b.name, resolver)
+            .getOrElse(throw resolveException(b.name, plan.right.schema.fieldNames)))
     }}
 
     withPlan {
@@ -1357,7 +1365,7 @@ class Dataset[T] private[sql](
   // Attach the dataset id and column position to the column reference, so that we can detect
   // ambiguous self-join correctly. See the rule `DetectAmbiguousSelfJoin`.
   // This must be called before we return a `Column` that contains `AttributeReference`.
-  // Note that, the metadata added here are only avaiable in the analyzer, as the analyzer rule
+  // Note that, the metadata added here are only available in the analyzer, as the analyzer rule
   // `DetectAmbiguousSelfJoin` will remove it.
   private def addDataFrameIdToCol(expr: NamedExpression): NamedExpression = {
     val newExpr = expr transform {
@@ -1659,10 +1667,10 @@ class Dataset[T] private[sql](
    * See [[RelationalGroupedDataset]] for all the available aggregate functions.
    *
    * {{{
-   *   // Compute the average for all numeric columns rolluped by department and group.
+   *   // Compute the average for all numeric columns rolled up by department and group.
    *   ds.rollup($"department", $"group").avg()
    *
-   *   // Compute the max age and average salary, rolluped by department and gender.
+   *   // Compute the max age and average salary, rolled up by department and gender.
    *   ds.rollup($"department", $"gender").agg(Map(
    *     "salary" -> "avg",
    *     "age" -> "max"
@@ -1788,10 +1796,10 @@ class Dataset[T] private[sql](
    * (i.e. cannot construct expressions).
    *
    * {{{
-   *   // Compute the average for all numeric columns rolluped by department and group.
+   *   // Compute the average for all numeric columns rolled up by department and group.
    *   ds.rollup("department", "group").avg()
    *
-   *   // Compute the max age and average salary, rolluped by department and gender.
+   *   // Compute the max age and average salary, rolled up by department and gender.
    *   ds.rollup($"department", $"gender").agg(Map(
    *     "salary" -> "avg",
    *     "age" -> "max"
@@ -2030,37 +2038,56 @@ class Dataset[T] private[sql](
    * @group typedrel
    * @since 2.3.0
    */
-  def unionByName(other: Dataset[T]): Dataset[T] = withSetOperator {
-    // Check column name duplication
-    val resolver = sparkSession.sessionState.analyzer.resolver
-    val leftOutputAttrs = logicalPlan.output
-    val rightOutputAttrs = other.logicalPlan.output
-
-    SchemaUtils.checkColumnNameDuplication(
-      leftOutputAttrs.map(_.name),
-      "in the left attributes",
-      sparkSession.sessionState.conf.caseSensitiveAnalysis)
-    SchemaUtils.checkColumnNameDuplication(
-      rightOutputAttrs.map(_.name),
-      "in the right attributes",
-      sparkSession.sessionState.conf.caseSensitiveAnalysis)
-
-    // Builds a project list for `other` based on `logicalPlan` output names
-    val rightProjectList = leftOutputAttrs.map { lattr =>
-      rightOutputAttrs.find { rattr => resolver(lattr.name, rattr.name) }.getOrElse {
-        throw new AnalysisException(
-          s"""Cannot resolve column name "${lattr.name}" among """ +
-            s"""(${rightOutputAttrs.map(_.name).mkString(", ")})""")
-      }
-    }
-
-    // Delegates failure checks to `CheckAnalysis`
-    val notFoundAttrs = rightOutputAttrs.diff(rightProjectList)
-    val rightChild = Project(rightProjectList ++ notFoundAttrs, other.logicalPlan)
+  def unionByName(other: Dataset[T]): Dataset[T] = unionByName(other, false)
 
+  /**
+   * Returns a new Dataset containing union of rows in this Dataset and another Dataset.
+   *
+   * The difference between this function and [[union]] is that this function
+   * resolves columns by name (not by position).
+   *
+   * When the parameter `allowMissingColumns` is `true`, the set of column names
+   * in this and other `Dataset` can differ; missing columns will be filled with null.
+   * Further, the missing columns of this `Dataset` will be added at the end
+   * in the schema of the union result:
+   *
+   * {{{
+   *   val df1 = Seq((1, 2, 3)).toDF("col0", "col1", "col2")
+   *   val df2 = Seq((4, 5, 6)).toDF("col1", "col0", "col3")
+   *   df1.unionByName(df2, true).show
+   *
+   *   // output: "col3" is missing at left df1 and added at the end of schema.
+   *   // +----+----+----+----+
+   *   // |col0|col1|col2|col3|
+   *   // +----+----+----+----+
+   *   // |   1|   2|   3|null|
+   *   // |   5|   4|null|   6|
+   *   // +----+----+----+----+
+   *
+   *   df2.unionByName(df1, true).show
+   *
+   *   // output: "col2" is missing at left df2 and added at the end of schema.
+   *   // +----+----+----+----+
+   *   // |col1|col0|col3|col2|
+   *   // +----+----+----+----+
+   *   // |   4|   5|   6|null|
+   *   // |   2|   1|null|   3|
+   *   // +----+----+----+----+
+   * }}}
+   *
+   * Note that `allowMissingColumns` supports nested column in struct types. Missing nested columns
+   * of struct columns with same name will also be filled with null values. This currently does not
+   * support nested columns in array and map types. Note that if there is any missing nested columns
+   * to be filled, in order to make consistent schema between two sides of union, the nested fields
+   * of structs will be sorted after merging schema.
+   *
+   * @group typedrel
+   * @since 3.1.0
+   */
+  def unionByName(other: Dataset[T], allowMissingColumns: Boolean): Dataset[T] = withSetOperator {
     // This breaks caching, but it's usually ok because it addresses a very specific use case:
     // using union to union many files or partitions.
-    CombineUnions(Union(logicalPlan, rightChild))
+    CombineUnions(Union(logicalPlan :: other.logicalPlan :: Nil, true, allowMissingColumns))
   }
 
   /**
@@ -2280,9 +2307,9 @@ class Dataset[T] private[sql](
    *   case class Book(title: String, words: String)
    *   val ds: Dataset[Book]
    *
-   *   val allWords = ds.select('title, explode(split('words, " ")).as("word"))
+   *   val allWords = ds.select($"title", explode(split($"words", " ")).as("word"))
    *
-   *   val bookCountPerWord = allWords.groupBy("word").agg(countDistinct("title"))
+   *   val bookCountPerWord = allWords.groupBy("word").agg(count_distinct("title"))
    * }}}
    *
    * Using `flatMap()` this can similarly be exploded as:
@@ -2319,7 +2346,7 @@ class Dataset[T] private[sql](
    * `functions.explode()`:
    *
    * {{{
-   *   ds.select(explode(split('words, " ")).as("word"))
+   *   ds.select(explode(split($"words", " ")).as("word"))
    * }}}
    *
    * or `flatMap()`:
@@ -2541,7 +2568,9 @@ class Dataset[T] private[sql](
   def dropDuplicates(colNames: Seq[String]): Dataset[T] = withTypedPlan {
     val resolver = sparkSession.sessionState.analyzer.resolver
     val allColumns = queryExecution.analyzed.output
-    val groupCols = colNames.distinct.flatMap { (colName: String) =>
+    // SPARK-31990: We must keep `toSet.toSeq` here because of the backward compatibility issue
+    // (the Streaming's state store depends on the `groupCols` order).
+    val groupCols = colNames.toSet.toSeq.flatMap { (colName: String) =>
       // It is possibly there are more than one columns with the same name,
       // so we call filter instead of find.
       val cols = allColumns.filter(col => resolver(col.name, colName))
@@ -2631,6 +2660,8 @@ class Dataset[T] private[sql](
    *   <li>min</li>
    *   <li>max</li>
    *   <li>arbitrary approximate percentiles specified as a percentage (e.g. 75%)</li>
+   *   <li>count_distinct</li>
+   *   <li>approx_count_distinct</li>
    * </ul>
    *
    * If no statistics are given, this function computes count, mean, stddev, min,
@@ -2673,6 +2704,20 @@ class Dataset[T] private[sql](
    *   ds.select("age", "height").summary().show()
    * }}}
    *
+   * Specify statistics to output custom summaries:
+   *
+   * {{{
+   *   ds.summary("count", "count_distinct").show()
+   * }}}
+   *
+   * The distinct count isn't included by default.
+   *
+   * You can also run approximate distinct counts which are faster:
+   *
+   * {{{
+   *   ds.summary("count", "approx_count_distinct").show()
+   * }}}
+   *
    * See also [[describe]] for basic statistics.
    *
    * @param statistics Statistics from above list to be computed.
@@ -2989,17 +3034,9 @@ class Dataset[T] private[sql](
     Repartition(numPartitions, shuffle = true, logicalPlan)
   }
 
-  /**
-   * Returns a new Dataset partitioned by the given partitioning expressions into
-   * `numPartitions`. The resulting Dataset is hash partitioned.
-   *
-   * This is the same operation as "DISTRIBUTE BY" in SQL (Hive QL).
-   *
-   * @group typedrel
-   * @since 2.0.0
-   */
-  @scala.annotation.varargs
-  def repartition(numPartitions: Int, partitionExprs: Column*): Dataset[T] = {
+  private def repartitionByExpression(
+      numPartitions: Option[Int],
+      partitionExprs: Seq[Column]): Dataset[T] = {
     // The underlying `LogicalPlan` operator special-cases all-`SortOrder` arguments.
     // However, we don't want to complicate the semantics of this API method.
     // Instead, let's give users a friendly error message, pointing them to the new method.
@@ -3013,6 +3050,20 @@ class Dataset[T] private[sql](
     }
   }
 
+  /**
+   * Returns a new Dataset partitioned by the given partitioning expressions into
+   * `numPartitions`. The resulting Dataset is hash partitioned.
+   *
+   * This is the same operation as "DISTRIBUTE BY" in SQL (Hive QL).
+   *
+   * @group typedrel
+   * @since 2.0.0
+   */
+  @scala.annotation.varargs
+  def repartition(numPartitions: Int, partitionExprs: Column*): Dataset[T] = {
+    repartitionByExpression(Some(numPartitions), partitionExprs)
+  }
+
   /**
    * Returns a new Dataset partitioned by the given partitioning expressions, using
    * `spark.sql.shuffle.partitions` as number of partitions.
@@ -3025,7 +3076,20 @@ class Dataset[T] private[sql](
    */
   @scala.annotation.varargs
   def repartition(partitionExprs: Column*): Dataset[T] = {
-    repartition(sparkSession.sessionState.conf.numShufflePartitions, partitionExprs: _*)
+    repartitionByExpression(None, partitionExprs)
+  }
+
+  private def repartitionByRange(
+      numPartitions: Option[Int],
+      partitionExprs: Seq[Column]): Dataset[T] = {
+    require(partitionExprs.nonEmpty, "At least one partition-by expression must be specified.")
+    val sortOrder: Seq[SortOrder] = partitionExprs.map(_.expr match {
+      case expr: SortOrder => expr
+      case expr: Expression => SortOrder(expr, Ascending)
+    })
+    withTypedPlan {
+      RepartitionByExpression(sortOrder, logicalPlan, numPartitions)
+    }
   }
 
   /**
@@ -3047,14 +3111,7 @@ class Dataset[T] private[sql](
    */
   @scala.annotation.varargs
   def repartitionByRange(numPartitions: Int, partitionExprs: Column*): Dataset[T] = {
-    require(partitionExprs.nonEmpty, "At least one partition-by expression must be specified.")
-    val sortOrder: Seq[SortOrder] = partitionExprs.map(_.expr match {
-      case expr: SortOrder => expr
-      case expr: Expression => SortOrder(expr, Ascending)
-    })
-    withTypedPlan {
-      RepartitionByExpression(sortOrder, logicalPlan, numPartitions)
-    }
+    repartitionByRange(Some(numPartitions), partitionExprs)
   }
 
   /**
@@ -3076,7 +3133,7 @@ class Dataset[T] private[sql](
    */
   @scala.annotation.varargs
   def repartitionByRange(partitionExprs: Column*): Dataset[T] = {
-    repartitionByRange(sparkSession.sessionState.conf.numShufflePartitions, partitionExprs: _*)
+    repartitionByRange(None, partitionExprs)
   }
 
   /**
@@ -3104,6 +3161,10 @@ class Dataset[T] private[sql](
    * Returns a new Dataset that contains only the unique rows from this Dataset.
    * This is an alias for `dropDuplicates`.
    *
+   * Note that for a streaming [[Dataset]], this method returns distinct rows only once
+   * regardless of the output mode, which the behavior may not be same with `DISTINCT` in SQL
+   * against streaming [[Dataset]].
+   *
    * @note Equality checking is performed directly on the encoded representation of the data
    * and thus is not affected by a custom `equals` function defined on `T`.
    *
@@ -3309,10 +3370,11 @@ class Dataset[T] private[sql](
       comment = None,
       properties = Map.empty,
       originalText = None,
-      child = logicalPlan,
+      plan = logicalPlan,
       allowExisting = false,
       replace = replace,
-      viewType = viewType)
+      viewType = viewType,
+      isAnalyzed = true)
   }
 
   /**
@@ -3421,7 +3483,7 @@ class Dataset[T] private[sql](
         fr.inputFiles
       case r: HiveTableRelation =>
         r.tableMeta.storage.locationUri.map(_.toString).toArray
-      case DataSourceV2ScanRelation(table: FileTable, _, _) =>
+      case DataSourceV2ScanRelation(DataSourceV2Relation(table: FileTable, _, _, _, _), _, _) =>
         table.fileIndex.inputFiles
     }.flatten
     files.toSet.toArray
@@ -3509,8 +3571,8 @@ class Dataset[T] private[sql](
   private[sql] def collectAsArrowToR(): Array[Any] = {
     val timeZoneId = sparkSession.sessionState.conf.sessionLocalTimeZone
 
-    withAction("collectAsArrowToR", queryExecution) { plan =>
-      RRDD.serveToStream("serve-Arrow") { outputStream =>
+    RRDD.serveToStream("serve-Arrow") { outputStream =>
+      withAction("collectAsArrowToR", queryExecution) { plan =>
         val buffer = new ByteArrayOutputStream()
         val out = new DataOutputStream(outputStream)
         val batchWriter = new ArrowBatchStreamWriter(schema, buffer, timeZoneId)
@@ -3518,7 +3580,7 @@ class Dataset[T] private[sql](
         val numPartitions = arrowBatchRdd.partitions.length
 
         // Store collection results for worst case of 1 to N-1 partitions
-        val results = new Array[Array[Array[Byte]]](numPartitions - 1)
+        val results = new Array[Array[Array[Byte]]](Math.max(0, numPartitions - 1))
         var lastIndex = -1  // index of last partition written
 
         // Handler to eagerly write partitions to Python in order
@@ -3563,8 +3625,8 @@ class Dataset[T] private[sql](
   private[sql] def collectAsArrowToPython: Array[Any] = {
     val timeZoneId = sparkSession.sessionState.conf.sessionLocalTimeZone
 
-    withAction("collectAsArrowToPython", queryExecution) { plan =>
-      PythonRDD.serveToStream("serve-Arrow") { outputStream =>
+    PythonRDD.serveToStream("serve-Arrow") { outputStream =>
+      withAction("collectAsArrowToPython", queryExecution) { plan =>
         val out = new DataOutputStream(outputStream)
         val batchWriter = new ArrowBatchStreamWriter(schema, out, timeZoneId)
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala
index b1ba7d4538732..bf3f7058b9bdd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala
@@ -20,12 +20,11 @@ package org.apache.spark.sql
 import java.util.Locale
 
 import scala.collection.JavaConverters._
-import scala.language.implicitConversions
 
 import org.apache.spark.annotation.Stable
 import org.apache.spark.api.python.PythonEvalType
 import org.apache.spark.broadcast.Broadcast
-import org.apache.spark.sql.catalyst.analysis.{Star, UnresolvedAlias, UnresolvedAttribute, UnresolvedFunction}
+import org.apache.spark.sql.catalyst.analysis.{Star, UnresolvedAlias, UnresolvedFunction}
 import org.apache.spark.sql.catalyst.encoders.encoderFor
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate._
@@ -54,7 +53,11 @@ class RelationalGroupedDataset protected[sql](
 
   private[this] def toDF(aggExprs: Seq[Expression]): DataFrame = {
     val aggregates = if (df.sparkSession.sessionState.conf.dataFrameRetainGroupColumns) {
-      groupingExprs ++ aggExprs
+      groupingExprs match {
+        // call `toList` because `Stream` can't serialize in scala 2.13
+        case s: Stream[Expression] => s.toList ++ aggExprs
+        case other => other ++ aggExprs
+      }
     } else {
       aggExprs
     }
@@ -66,10 +69,12 @@ class RelationalGroupedDataset protected[sql](
         Dataset.ofRows(df.sparkSession, Aggregate(groupingExprs, aliasedAgg, df.logicalPlan))
       case RelationalGroupedDataset.RollupType =>
         Dataset.ofRows(
-          df.sparkSession, Aggregate(Seq(Rollup(groupingExprs)), aliasedAgg, df.logicalPlan))
+          df.sparkSession, Aggregate(Seq(Rollup(groupingExprs.map(Seq(_)))),
+            aliasedAgg, df.logicalPlan))
       case RelationalGroupedDataset.CubeType =>
         Dataset.ofRows(
-          df.sparkSession, Aggregate(Seq(Cube(groupingExprs)), aliasedAgg, df.logicalPlan))
+          df.sparkSession, Aggregate(Seq(Cube(groupingExprs.map(Seq(_)))),
+            aliasedAgg, df.logicalPlan))
       case RelationalGroupedDataset.PivotType(pivotCol, values) =>
         val aliasedGrps = groupingExprs.map(alias)
         Dataset.ofRows(
@@ -77,11 +82,7 @@ class RelationalGroupedDataset protected[sql](
     }
   }
 
-  // Wrap UnresolvedAttribute with UnresolvedAlias, as when we resolve UnresolvedAttribute, we
-  // will remove intermediate Alias for ExtractValue chain, and we need to alias it again to
-  // make it a NamedExpression.
   private[this] def alias(expr: Expression): NamedExpression = expr match {
-    case u: UnresolvedAttribute => UnresolvedAlias(u)
     case expr: NamedExpression => expr
     case a: AggregateExpression if a.aggregateFunction.isInstanceOf[TypedAggregateExpression] =>
       UnresolvedAlias(a, Some(Column.generateAlias))
@@ -479,7 +480,7 @@ class RelationalGroupedDataset protected[sql](
    * @since 2.4.0
    */
   def pivot(pivotColumn: Column, values: java.util.List[Any]): RelationalGroupedDataset = {
-    pivot(pivotColumn, values.asScala)
+    pivot(pivotColumn, values.asScala.toSeq)
   }
 
   /**
@@ -546,9 +547,10 @@ class RelationalGroupedDataset protected[sql](
       case ne: NamedExpression => ne
       case other => Alias(other, other.toString)()
     }
-    val groupingAttributes = groupingNamedExpressions.map(_.toAttribute)
     val child = df.logicalPlan
-    val project = Project(groupingNamedExpressions ++ child.output, child)
+    val project = df.sparkSession.sessionState.executePlan(
+      Project(groupingNamedExpressions ++ child.output, child)).analyzed
+    val groupingAttributes = project.output.take(groupingNamedExpressions.length)
     val output = expr.dataType.asInstanceOf[StructType].toAttributes
     val plan = FlatMapGroupsInPandas(groupingAttributes, expr, output, project)
 
@@ -583,17 +585,18 @@ class RelationalGroupedDataset protected[sql](
       case other => Alias(other, other.toString)()
     }
 
-    val leftAttributes = leftGroupingNamedExpressions.map(_.toAttribute)
-    val rightAttributes = rightGroupingNamedExpressions.map(_.toAttribute)
-
     val leftChild = df.logicalPlan
     val rightChild = r.df.logicalPlan
 
-    val left = Project(leftGroupingNamedExpressions ++ leftChild.output, leftChild)
-    val right = Project(rightGroupingNamedExpressions ++ rightChild.output, rightChild)
+    val left = df.sparkSession.sessionState.executePlan(
+      Project(leftGroupingNamedExpressions ++ leftChild.output, leftChild)).analyzed
+    val right = r.df.sparkSession.sessionState.executePlan(
+      Project(rightGroupingNamedExpressions ++ rightChild.output, rightChild)).analyzed
 
     val output = expr.dataType.asInstanceOf[StructType].toAttributes
-    val plan = FlatMapCoGroupsInPandas(leftAttributes, rightAttributes, expr, output, left, right)
+    val plan = FlatMapCoGroupsInPandas(
+      leftGroupingNamedExpressions.length, rightGroupingNamedExpressions.length,
+      expr, output, left, right)
     Dataset.ofRows(df.sparkSession, plan)
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
index e9bc25d489718..2f46fa8073bbc 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
@@ -18,10 +18,8 @@
 package org.apache.spark.sql
 
 import org.apache.spark.annotation.Stable
-import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.{ConfigEntry, OptionalConfigEntry}
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.internal.SQLConf.{DeprecatedConfig, RemovedConfig}
 
 /**
  * Runtime configuration interface for Spark. To access this, use `SparkSession.conf`.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index 68ce82d5badda..b6ebaf86cb4df 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -29,8 +29,9 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.ConfigEntry
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst._
+import org.apache.spark.sql.catalyst.analysis.UnresolvedNamespace
 import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.execution.command.ShowTablesCommand
+import org.apache.spark.sql.catalyst.plans.logical.ShowTables
 import org.apache.spark.sql.internal.{SessionState, SharedState, SQLConf}
 import org.apache.spark.sql.sources.BaseRelation
 import org.apache.spark.sql.streaming.{DataStreamReader, StreamingQueryManager}
@@ -641,8 +642,8 @@ class SQLContext private[sql](val sparkSession: SparkSession)
   }
 
   /**
-   * Executes a SQL query using Spark, returning the result as a `DataFrame`. The dialect that is
-   * used for SQL parsing can be configured with 'spark.sql.dialect'.
+   * Executes a SQL query using Spark, returning the result as a `DataFrame`.
+   * This API eagerly runs DDL/DML commands, but not for SELECT queries.
    *
    * @group basic
    * @since 1.3.0
@@ -661,26 +662,26 @@ class SQLContext private[sql](val sparkSession: SparkSession)
 
   /**
    * Returns a `DataFrame` containing names of existing tables in the current database.
-   * The returned DataFrame has two columns, tableName and isTemporary (a Boolean
+   * The returned DataFrame has three columns, database, tableName and isTemporary (a Boolean
    * indicating if a table is a temporary one or not).
    *
    * @group ddl_ops
    * @since 1.3.0
    */
   def tables(): DataFrame = {
-    Dataset.ofRows(sparkSession, ShowTablesCommand(None, None))
+    Dataset.ofRows(sparkSession, ShowTables(UnresolvedNamespace(Nil), None))
   }
 
   /**
    * Returns a `DataFrame` containing names of existing tables in the given database.
-   * The returned DataFrame has two columns, tableName and isTemporary (a Boolean
+   * The returned DataFrame has three columns, database, tableName and isTemporary (a Boolean
    * indicating if a table is a temporary one or not).
    *
    * @group ddl_ops
    * @since 1.3.0
    */
   def tables(databaseName: String): DataFrame = {
-    Dataset.ofRows(sparkSession, ShowTablesCommand(Some(databaseName), None))
+    Dataset.ofRows(sparkSession, ShowTables(UnresolvedNamespace(Seq(databaseName)), None))
   }
 
   /**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala
index 71cbc3ab14d97..90188cadfd3c3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala
@@ -88,6 +88,16 @@ abstract class SQLImplicits extends LowPrioritySQLImplicits {
   /** @since 3.0.0 */
   implicit def newInstantEncoder: Encoder[java.time.Instant] = Encoders.INSTANT
 
+  /** @since 3.2.0 */
+  implicit def newDurationEncoder: Encoder[java.time.Duration] = Encoders.DURATION
+
+  /** @since 3.2.0 */
+  implicit def newPeriodEncoder: Encoder[java.time.Period] = Encoders.PERIOD
+
+  /** @since 3.2.0 */
+  implicit def newJavaEnumEncoder[A <: java.lang.Enum[_] : TypeTag]: Encoder[A] =
+    ExpressionEncoder()
+
   // Boxed primitives
 
   /** @since 2.0.0 */
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index be597edecba98..62852fe941690 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -18,8 +18,9 @@
 package org.apache.spark.sql
 
 import java.io.Closeable
+import java.util.{ServiceLoader, UUID}
 import java.util.concurrent.TimeUnit._
-import java.util.concurrent.atomic.AtomicReference
+import java.util.concurrent.atomic.{AtomicBoolean, AtomicReference}
 
 import scala.collection.JavaConverters._
 import scala.reflect.runtime.universe.TypeTag
@@ -29,6 +30,7 @@ import org.apache.spark.{SPARK_VERSION, SparkConf, SparkContext, TaskContext}
 import org.apache.spark.annotation.{DeveloperApi, Experimental, Stable, Unstable}
 import org.apache.spark.api.java.JavaRDD
 import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config.{ConfigEntry, EXECUTOR_ALLOW_SPARK_CONTEXT}
 import org.apache.spark.rdd.RDD
 import org.apache.spark.scheduler.{SparkListener, SparkListenerApplicationEnd}
 import org.apache.spark.sql.catalog.Catalog
@@ -37,6 +39,7 @@ import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
 import org.apache.spark.sql.catalyst.encoders._
 import org.apache.spark.sql.catalyst.expressions.AttributeReference
 import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, Range}
+import org.apache.spark.sql.catalyst.util.CharVarcharUtils
 import org.apache.spark.sql.connector.ExternalCommandRunner
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.execution.command.ExternalCommandExecutor
@@ -49,7 +52,6 @@ import org.apache.spark.sql.types.{DataType, StructType}
 import org.apache.spark.sql.util.ExecutionListenerManager
 import org.apache.spark.util.{CallSite, Utils}
 
-
 /**
  * The entry point to programming Spark with the Dataset and DataFrame API.
  *
@@ -81,7 +83,8 @@ class SparkSession private(
     @transient val sparkContext: SparkContext,
     @transient private val existingSharedState: Option[SharedState],
     @transient private val parentSessionState: Option[SessionState],
-    @transient private[sql] val extensions: SparkSessionExtensions)
+    @transient private[sql] val extensions: SparkSessionExtensions,
+    @transient private[sql] val initialSessionOptions: Map[String, String])
   extends Serializable with Closeable with Logging { self =>
 
   // The call site where this SparkSession was constructed.
@@ -93,13 +96,15 @@ class SparkSession private(
    * since that would cause every new session to reinvoke Spark Session Extensions on the currently
    * running extensions.
    */
-  private[sql] def this(sc: SparkContext) {
+  private[sql] def this(sc: SparkContext) = {
     this(sc, None, None,
       SparkSession.applyExtensions(
         sc.getConf.get(StaticSQLConf.SPARK_SESSION_EXTENSIONS).getOrElse(Seq.empty),
-        new SparkSessionExtensions))
+        new SparkSessionExtensions), Map.empty)
   }
 
+  private[sql] val sessionUUID: String = UUID.randomUUID.toString
+
   sparkContext.assertNotStopped()
 
   // If there is no active SparkSession, uses the default SQL conf. Otherwise, use the session's.
@@ -133,12 +138,6 @@ class SparkSession private(
     existingSharedState.getOrElse(new SharedState(sparkContext, initialSessionOptions))
   }
 
-  /**
-   * Initial options for session. This options are applied once when sessionState is created.
-   */
-  @transient
-  private[sql] val initialSessionOptions = new scala.collection.mutable.HashMap[String, String]
-
   /**
    * State isolated across sessions, including SQL configurations, temporary tables, registered
    * functions, and everything else that accepts a [[org.apache.spark.sql.internal.SQLConf]].
@@ -155,9 +154,8 @@ class SparkSession private(
       .map(_.clone(this))
       .getOrElse {
         val state = SparkSession.instantiateSessionState(
-          SparkSession.sessionStateClassName(sparkContext.conf),
+          SparkSession.sessionStateClassName(sharedState.conf),
           self)
-        initialSessionOptions.foreach { case (k, v) => state.conf.setConfString(k, v) }
         state
       }
   }
@@ -244,7 +242,12 @@ class SparkSession private(
    * @since 2.0.0
    */
   def newSession(): SparkSession = {
-    new SparkSession(sparkContext, Some(sharedState), parentSessionState = None, extensions)
+    new SparkSession(
+      sparkContext,
+      Some(sharedState),
+      parentSessionState = None,
+      extensions,
+      initialSessionOptions)
   }
 
   /**
@@ -260,7 +263,12 @@ class SparkSession private(
    * implementation is Hive, this will initialize the metastore, which may take some time.
    */
   private[sql] def cloneSession(): SparkSession = {
-    val result = new SparkSession(sparkContext, Some(sharedState), Some(sessionState), extensions)
+    val result = new SparkSession(
+      sparkContext,
+      Some(sharedState),
+      Some(sessionState),
+      extensions,
+      Map.empty)
     result.sessionState // force copy of SessionState
     result
   }
@@ -342,9 +350,10 @@ class SparkSession private(
    */
   @DeveloperApi
   def createDataFrame(rowRDD: RDD[Row], schema: StructType): DataFrame = withActive {
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(schema).asInstanceOf[StructType]
     // TODO: use MutableProjection when rowRDD is another DataFrame and the applied
     // schema differs from the existing schema on any field data type.
-    val encoder = RowEncoder(schema)
+    val encoder = RowEncoder(replaced)
     val toRow = encoder.createSerializer()
     val catalystRows = rowRDD.map(toRow)
     internalCreateDataFrame(catalystRows.setName(rowRDD.name), schema)
@@ -360,7 +369,8 @@ class SparkSession private(
    */
   @DeveloperApi
   def createDataFrame(rowRDD: JavaRDD[Row], schema: StructType): DataFrame = {
-    createDataFrame(rowRDD.rdd, schema)
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(schema).asInstanceOf[StructType]
+    createDataFrame(rowRDD.rdd, replaced)
   }
 
   /**
@@ -373,7 +383,8 @@ class SparkSession private(
    */
   @DeveloperApi
   def createDataFrame(rows: java.util.List[Row], schema: StructType): DataFrame = withActive {
-    Dataset.ofRows(self, LocalRelation.fromExternalRows(schema.toAttributes, rows.asScala))
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(schema).asInstanceOf[StructType]
+    Dataset.ofRows(self, LocalRelation.fromExternalRows(replaced.toAttributes, rows.asScala.toSeq))
   }
 
   /**
@@ -496,7 +507,7 @@ class SparkSession private(
    * @since 2.0.0
    */
   def createDataset[T : Encoder](data: java.util.List[T]): Dataset[T] = {
-    createDataset(data.asScala)
+    createDataset(data.asScala.toSeq)
   }
 
   /**
@@ -514,7 +525,7 @@ class SparkSession private(
    * @since 2.0.0
    */
   def range(start: Long, end: Long): Dataset[java.lang.Long] = {
-    range(start, end, step = 1, numPartitions = sparkContext.defaultParallelism)
+    range(start, end, step = 1, numPartitions = leafNodeDefaultParallelism)
   }
 
   /**
@@ -524,7 +535,7 @@ class SparkSession private(
    * @since 2.0.0
    */
   def range(start: Long, end: Long, step: Long): Dataset[java.lang.Long] = {
-    range(start, end, step, numPartitions = sparkContext.defaultParallelism)
+    range(start, end, step, numPartitions = leafNodeDefaultParallelism)
   }
 
   /**
@@ -568,7 +579,10 @@ class SparkSession private(
   @transient lazy val catalog: Catalog = new CatalogImpl(self)
 
   /**
-   * Returns the specified table/view as a `DataFrame`.
+   * Returns the specified table/view as a `DataFrame`. If it's a table, it must support batch
+   * reading and the returned DataFrame is the batch scan query plan of this table. If it's a view,
+   * the returned DataFrame is simply the query plan of the view, which can either be a batch or
+   * streaming query plan.
    *
    * @param tableName is either a qualified or unqualified name that designates a table or view.
    *                  If a database is specified, it identifies the table/view from the database.
@@ -578,11 +592,7 @@ class SparkSession private(
    * @since 2.0.0
    */
   def table(tableName: String): DataFrame = {
-    table(sessionState.sqlParser.parseMultipartIdentifier(tableName))
-  }
-
-  private[sql] def table(multipartIdentifier: Seq[String]): DataFrame = {
-    Dataset.ofRows(self, UnresolvedRelation(multipartIdentifier))
+    read.table(tableName)
   }
 
   private[sql] def table(tableIdent: TableIdentifier): DataFrame = {
@@ -595,7 +605,7 @@ class SparkSession private(
 
   /**
    * Executes a SQL query using Spark, returning the result as a `DataFrame`.
-   * The dialect that is used for SQL parsing can be configured with 'spark.sql.dialect'.
+   * This API eagerly runs DDL/DML commands, but not for SELECT queries.
    *
    * @since 2.0.0
    */
@@ -765,6 +775,10 @@ class SparkSession private(
       SparkSession.setActiveSession(old)
     }
   }
+
+  private[sql] def leafNodeDefaultParallelism: Int = {
+    conf.get(SQLConf.LEAF_NODE_DEFAULT_PARALLELISM).getOrElse(sparkContext.defaultParallelism)
+  }
 }
 
 
@@ -901,7 +915,13 @@ object SparkSession extends Logging {
      * @since 2.0.0
      */
     def getOrCreate(): SparkSession = synchronized {
-      assertOnDriver()
+      val sparkConf = new SparkConf()
+      options.foreach { case (k, v) => sparkConf.set(k, v) }
+
+      if (!sparkConf.get(EXECUTOR_ALLOW_SPARK_CONTEXT)) {
+        assertOnDriver()
+      }
+
       // Get the session from current thread's active session.
       var session = activeThreadSession.get()
       if ((session ne null) && !session.sparkContext.isStopped) {
@@ -920,9 +940,6 @@ object SparkSession extends Logging {
 
         // No active nor global default session. Create a new one.
         val sparkContext = userSuppliedContext.getOrElse {
-          val sparkConf = new SparkConf()
-          options.foreach { case (k, v) => sparkConf.set(k, v) }
-
           // set a random app name if not given.
           if (!sparkConf.contains("spark.app.name")) {
             sparkConf.setAppName(java.util.UUID.randomUUID().toString)
@@ -932,23 +949,15 @@ object SparkSession extends Logging {
           // Do not update `SparkConf` for existing `SparkContext`, as it's shared by all sessions.
         }
 
+        loadExtensions(extensions)
         applyExtensions(
           sparkContext.getConf.get(StaticSQLConf.SPARK_SESSION_EXTENSIONS).getOrElse(Seq.empty),
           extensions)
 
-        session = new SparkSession(sparkContext, None, None, extensions)
-        options.foreach { case (k, v) => session.initialSessionOptions.put(k, v) }
+        session = new SparkSession(sparkContext, None, None, extensions, options.toMap)
         setDefaultSession(session)
         setActiveSession(session)
-
-        // Register a successfully instantiated context to the singleton. This should be at the
-        // end of the class definition so that the singleton is updated only if there is no
-        // exception in the construction of the instance.
-        sparkContext.addSparkListener(new SparkListener {
-          override def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd): Unit = {
-            defaultSession.set(null)
-          }
-        })
+        registerContextListener(sparkContext)
       }
 
       return session
@@ -1060,10 +1069,44 @@ object SparkSession extends Logging {
       throw new IllegalStateException("No active or default Spark session found")))
   }
 
+  /**
+   * Returns a cloned SparkSession with all specified configurations disabled, or
+   * the original SparkSession if all configurations are already disabled.
+   */
+  private[sql] def getOrCloneSessionWithConfigsOff(
+      session: SparkSession,
+      configurations: Seq[ConfigEntry[Boolean]]): SparkSession = {
+    val configsEnabled = configurations.filter(session.sessionState.conf.getConf(_))
+    if (configsEnabled.isEmpty) {
+      session
+    } else {
+      val newSession = session.cloneSession()
+      configsEnabled.foreach(conf => {
+        newSession.sessionState.conf.setConf(conf, false)
+      })
+      newSession
+    }
+  }
+
   ////////////////////////////////////////////////////////////////////////////////////////
   // Private methods from now on
   ////////////////////////////////////////////////////////////////////////////////////////
 
+  private val listenerRegistered: AtomicBoolean = new AtomicBoolean(false)
+
+  /** Register the AppEnd listener onto the Context  */
+  private def registerContextListener(sparkContext: SparkContext): Unit = {
+    if (!listenerRegistered.get()) {
+      sparkContext.addSparkListener(new SparkListener {
+        override def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd): Unit = {
+          defaultSession.set(null)
+          listenerRegistered.set(false)
+        }
+      })
+      listenerRegistered.set(true)
+    }
+  }
+
   /** The active SparkSession for the current thread. */
   private val activeThreadSession = new InheritableThreadLocal[SparkSession]
 
@@ -1081,7 +1124,7 @@ object SparkSession extends Logging {
   }
 
   private def assertOnDriver(): Unit = {
-    if (Utils.isTesting && TaskContext.get != null) {
+    if (TaskContext.get != null) {
       // we're accessing it during task execution, fail.
       throw new IllegalStateException(
         "SparkSession should only be created and accessed on the driver.")
@@ -1096,7 +1139,9 @@ object SparkSession extends Logging {
       className: String,
       sparkSession: SparkSession): SessionState = {
     try {
-      // invoke `new [Hive]SessionStateBuilder(SparkSession, Option[SessionState])`
+      // invoke new [Hive]SessionStateBuilder(
+      //   SparkSession,
+      //   Option[SessionState])
       val clazz = Utils.classForName(className)
       val ctor = clazz.getConstructors.head
       ctor.newInstance(sparkSession, None).asInstanceOf[BaseSessionStateBuilder].build()
@@ -1159,4 +1204,22 @@ object SparkSession extends Logging {
     }
     extensions
   }
+
+  /**
+   * Load extensions from [[ServiceLoader]] and use them
+   */
+  private def loadExtensions(extensions: SparkSessionExtensions): Unit = {
+    val loader = ServiceLoader.load(classOf[SparkSessionExtensionsProvider],
+      Utils.getContextOrSparkClassLoader)
+    val loadedExts = loader.iterator()
+
+    while (loadedExts.hasNext) {
+      try {
+        val ext = loadedExts.next()
+        ext(extensions)
+      } catch {
+        case e: Throwable => logWarning("Failed to load session extension", e)
+      }
+    }
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSessionExtensions.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSessionExtensions.scala
index 1c2bf9e7c2a57..b14dce64f5aa2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSessionExtensions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSessionExtensions.scala
@@ -21,13 +21,14 @@ import scala.collection.mutable
 
 import org.apache.spark.annotation.{DeveloperApi, Experimental, Unstable}
 import org.apache.spark.sql.catalyst.FunctionIdentifier
-import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
+import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TableFunctionRegistry}
 import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder
+import org.apache.spark.sql.catalyst.analysis.TableFunctionRegistry.TableFunctionBuilder
 import org.apache.spark.sql.catalyst.expressions.ExpressionInfo
 import org.apache.spark.sql.catalyst.parser.ParserInterface
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.execution.ColumnarRule
+import org.apache.spark.sql.execution.{ColumnarRule, SparkPlan}
 
 /**
  * :: Experimental ::
@@ -40,10 +41,12 @@ import org.apache.spark.sql.execution.ColumnarRule
  * <li>Analyzer Rules.</li>
  * <li>Check Analysis Rules.</li>
  * <li>Optimizer Rules.</li>
+ * <li>Pre CBO Rules.</li>
  * <li>Planning Strategies.</li>
  * <li>Customized Parser.</li>
  * <li>(External) Catalog listeners.</li>
  * <li>Columnar Rules.</li>
+ * <li>Adaptive Query Stage Preparation Rules.</li>
  * </ul>
  *
  * The extensions can be used by calling `withExtensions` on the [[SparkSession.Builder]], for
@@ -68,7 +71,7 @@ import org.apache.spark.sql.execution.ColumnarRule
  * {{{
  *   SparkSession.builder()
  *     .master("...")
- *     .config("spark.sql.extensions", "org.example.MyExtensions")
+ *     .config("spark.sql.extensions", "org.example.MyExtensions,org.example.YourExtensions")
  *     .getOrCreate()
  *
  *   class MyExtensions extends Function1[SparkSessionExtensions, Unit] {
@@ -81,6 +84,15 @@ import org.apache.spark.sql.execution.ColumnarRule
  *       }
  *     }
  *   }
+ *
+ *   class YourExtensions extends SparkSessionExtensionsProvider {
+ *     override def apply(extensions: SparkSessionExtensions): Unit = {
+ *       extensions.injectResolutionRule { session =>
+ *         ...
+ *       }
+ *       extensions.injectFunction(...)
+ *     }
+ *   }
  * }}}
  *
  * Note that none of the injected builders should assume that the [[SparkSession]] is fully
@@ -95,15 +107,25 @@ class SparkSessionExtensions {
   type StrategyBuilder = SparkSession => Strategy
   type ParserBuilder = (SparkSession, ParserInterface) => ParserInterface
   type FunctionDescription = (FunctionIdentifier, ExpressionInfo, FunctionBuilder)
+  type TableFunctionDescription = (FunctionIdentifier, ExpressionInfo, TableFunctionBuilder)
   type ColumnarRuleBuilder = SparkSession => ColumnarRule
+  type QueryStagePrepRuleBuilder = SparkSession => Rule[SparkPlan]
 
   private[this] val columnarRuleBuilders = mutable.Buffer.empty[ColumnarRuleBuilder]
+  private[this] val queryStagePrepRuleBuilders = mutable.Buffer.empty[QueryStagePrepRuleBuilder]
 
   /**
    * Build the override rules for columnar execution.
    */
   private[sql] def buildColumnarRules(session: SparkSession): Seq[ColumnarRule] = {
-    columnarRuleBuilders.map(_.apply(session))
+    columnarRuleBuilders.map(_.apply(session)).toSeq
+  }
+
+  /**
+   * Build the override rules for the query stage preparation phase of adaptive query execution.
+   */
+  private[sql] def buildQueryStagePrepRules(session: SparkSession): Seq[Rule[SparkPlan]] = {
+    queryStagePrepRuleBuilders.map(_.apply(session)).toSeq
   }
 
   /**
@@ -113,13 +135,21 @@ class SparkSessionExtensions {
     columnarRuleBuilders += builder
   }
 
+  /**
+   * Inject a rule that can override the the query stage preparation phase of adaptive query
+   * execution.
+   */
+  def injectQueryStagePrepRule(builder: QueryStagePrepRuleBuilder): Unit = {
+    queryStagePrepRuleBuilders += builder
+  }
+
   private[this] val resolutionRuleBuilders = mutable.Buffer.empty[RuleBuilder]
 
   /**
    * Build the analyzer resolution `Rule`s using the given [[SparkSession]].
    */
   private[sql] def buildResolutionRules(session: SparkSession): Seq[Rule[LogicalPlan]] = {
-    resolutionRuleBuilders.map(_.apply(session))
+    resolutionRuleBuilders.map(_.apply(session)).toSeq
   }
 
   /**
@@ -136,7 +166,7 @@ class SparkSessionExtensions {
    * Build the analyzer post-hoc resolution `Rule`s using the given [[SparkSession]].
    */
   private[sql] def buildPostHocResolutionRules(session: SparkSession): Seq[Rule[LogicalPlan]] = {
-    postHocResolutionRuleBuilders.map(_.apply(session))
+    postHocResolutionRuleBuilders.map(_.apply(session)).toSeq
   }
 
   /**
@@ -153,7 +183,7 @@ class SparkSessionExtensions {
    * Build the check analysis `Rule`s using the given [[SparkSession]].
    */
   private[sql] def buildCheckRules(session: SparkSession): Seq[LogicalPlan => Unit] = {
-    checkRuleBuilders.map(_.apply(session))
+    checkRuleBuilders.map(_.apply(session)).toSeq
   }
 
   /**
@@ -168,7 +198,7 @@ class SparkSessionExtensions {
   private[this] val optimizerRules = mutable.Buffer.empty[RuleBuilder]
 
   private[sql] def buildOptimizerRules(session: SparkSession): Seq[Rule[LogicalPlan]] = {
-    optimizerRules.map(_.apply(session))
+    optimizerRules.map(_.apply(session)).toSeq
   }
 
   /**
@@ -181,10 +211,25 @@ class SparkSessionExtensions {
     optimizerRules += builder
   }
 
+  private[this] val preCBORules = mutable.Buffer.empty[RuleBuilder]
+
+  private[sql] def buildPreCBORules(session: SparkSession): Seq[Rule[LogicalPlan]] = {
+    preCBORules.map(_.apply(session)).toSeq
+  }
+
+  /**
+   * Inject an optimizer `Rule` builder that rewrites logical plans into the [[SparkSession]].
+   * The injected rules will be executed once after the operator optimization batch and
+   * before any cost-based optimization rules that depend on stats.
+   */
+  def injectPreCBORule(builder: RuleBuilder): Unit = {
+    preCBORules += builder
+  }
+
   private[this] val plannerStrategyBuilders = mutable.Buffer.empty[StrategyBuilder]
 
   private[sql] def buildPlannerStrategies(session: SparkSession): Seq[Strategy] = {
-    plannerStrategyBuilders.map(_.apply(session))
+    plannerStrategyBuilders.map(_.apply(session)).toSeq
   }
 
   /**
@@ -218,6 +263,8 @@ class SparkSessionExtensions {
 
   private[this] val injectedFunctions = mutable.Buffer.empty[FunctionDescription]
 
+  private[this] val injectedTableFunctions = mutable.Buffer.empty[TableFunctionDescription]
+
   private[sql] def registerFunctions(functionRegistry: FunctionRegistry) = {
     for ((name, expressionInfo, function) <- injectedFunctions) {
       functionRegistry.registerFunction(name, expressionInfo, function)
@@ -225,6 +272,13 @@ class SparkSessionExtensions {
     functionRegistry
   }
 
+  private[sql] def registerTableFunctions(tableFunctionRegistry: TableFunctionRegistry) = {
+    for ((name, expressionInfo, function) <- injectedTableFunctions) {
+      tableFunctionRegistry.registerFunction(name, expressionInfo, function)
+    }
+    tableFunctionRegistry
+  }
+
   /**
   * Injects a custom function into the [[org.apache.spark.sql.catalyst.analysis.FunctionRegistry]]
   * at runtime for all sessions.
@@ -232,4 +286,12 @@ class SparkSessionExtensions {
   def injectFunction(functionDescription: FunctionDescription): Unit = {
     injectedFunctions += functionDescription
   }
+
+  /**
+   * Injects a custom function into the
+   * [[org.apache.spark.sql.catalyst.analysis.TableFunctionRegistry]] at runtime for all sessions.
+   */
+  def injectTableFunction(functionDescription: TableFunctionDescription): Unit = {
+    injectedTableFunctions += functionDescription
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSessionExtensionsProvider.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSessionExtensionsProvider.scala
new file mode 100644
index 0000000000000..23f4faaa0be60
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSessionExtensionsProvider.scala
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import org.apache.spark.annotation.{DeveloperApi, Since, Unstable}
+
+// scalastyle:off line.size.limit
+/**
+ * :: Unstable ::
+ *
+ * Base trait for implementations used by [[SparkSessionExtensions]]
+ *
+ *
+ * For example, now we have an external function named `Age` to register as an extension for SparkSession:
+ *
+ *
+ * {{{
+ *   package org.apache.spark.examples.extensions
+ *
+ *   import org.apache.spark.sql.catalyst.expressions.{CurrentDate, Expression, RuntimeReplaceable, SubtractDates}
+ *
+ *   case class Age(birthday: Expression, child: Expression) extends RuntimeReplaceable {
+ *
+ *     def this(birthday: Expression) = this(birthday, SubtractDates(CurrentDate(), birthday))
+ *     override def exprsReplaced: Seq[Expression] = Seq(birthday)
+ *     override protected def withNewChildInternal(newChild: Expression): Expression = copy(newChild)
+ *   }
+ * }}}
+ *
+ * We need to create our extension which inherits [[SparkSessionExtensionsProvider]]
+ * Example:
+ *
+ * {{{
+ *   package org.apache.spark.examples.extensions
+ *
+ *   import org.apache.spark.sql.{SparkSessionExtensions, SparkSessionExtensionsProvider}
+ *   import org.apache.spark.sql.catalyst.FunctionIdentifier
+ *   import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionInfo}
+ *
+ *   class MyExtensions extends SparkSessionExtensionsProvider {
+ *     override def apply(v1: SparkSessionExtensions): Unit = {
+ *       v1.injectFunction(
+ *         (new FunctionIdentifier("age"),
+ *           new ExpressionInfo(classOf[Age].getName, "age"),
+ *           (children: Seq[Expression]) => new Age(children.head)))
+ *     }
+ *   }
+ * }}}
+ *
+ * Then, we can inject `MyExtensions` in three ways,
+ * <ul>
+ *   <li>withExtensions of [[SparkSession.Builder]]</li>
+ *   <li>Config - spark.sql.extensions</li>
+ *   <li>[[java.util.ServiceLoader]] - Add to src/main/resources/META-INF/services/org.apache.spark.sql.SparkSessionExtensionsProvider</li>
+ * </ul>
+ *
+ * @see [[SparkSessionExtensions]]
+ * @see [[SparkSession.Builder]]
+ * @see [[java.util.ServiceLoader]]
+ *
+ * @since 3.2.0
+ */
+@DeveloperApi
+@Unstable
+@Since("3.2.0")
+trait SparkSessionExtensionsProvider extends Function1[SparkSessionExtensions, Unit]
+// scalastyle:on line.size.limit
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
index ced4af46c3f30..4c165680d428b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
@@ -30,9 +30,10 @@ import org.apache.spark.sql.catalyst.{JavaTypeInference, ScalaReflection}
 import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.catalyst.expressions.{Expression, ScalaUDF}
-import org.apache.spark.sql.execution.aggregate.{ScalaAggregator, ScalaUDAF}
+import org.apache.spark.sql.catalyst.util.CharVarcharUtils
+import org.apache.spark.sql.execution.aggregate.ScalaUDAF
 import org.apache.spark.sql.execution.python.UserDefinedPythonFunction
-import org.apache.spark.sql.expressions.{Aggregator, SparkUserDefinedFunction, UserDefinedAggregateFunction, UserDefinedAggregator, UserDefinedFunction}
+import org.apache.spark.sql.expressions.{SparkUserDefinedFunction, UserDefinedAggregateFunction, UserDefinedAggregator, UserDefinedFunction}
 import org.apache.spark.sql.types.DataType
 import org.apache.spark.util.Utils
 
@@ -48,6 +49,8 @@ import org.apache.spark.util.Utils
 @Stable
 class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends Logging {
 
+  import UDFRegistration._
+
   protected[sql] def registerPython(name: String, udf: UserDefinedPythonFunction): Unit = {
     log.debug(
       s"""
@@ -79,7 +82,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   @deprecated("Aggregator[IN, BUF, OUT] should now be registered as a UDF" +
     " via the functions.udaf(agg) method.", "3.0.0")
   def register(name: String, udaf: UserDefinedAggregateFunction): UserDefinedAggregateFunction = {
-    def builder(children: Seq[Expression]) = ScalaUDAF(children, udaf)
+    def builder(children: Seq[Expression]) = ScalaUDAF(children, udaf, udafName = Some(name))
     functionRegistry.createOrReplaceTempFunction(name, builder)
     udaf
   }
@@ -106,15 +109,15 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 2.2.0
    */
   def register(name: String, udf: UserDefinedFunction): UserDefinedFunction = {
-    udf match {
+    udf.withName(name) match {
       case udaf: UserDefinedAggregator[_, _, _] =>
         def builder(children: Seq[Expression]) = udaf.scalaAggregator(children)
         functionRegistry.createOrReplaceTempFunction(name, builder)
-        udf
-      case _ =>
-        def builder(children: Seq[Expression]) = udf.apply(children.map(Column.apply) : _*).expr
+        udaf
+      case other =>
+        def builder(children: Seq[Expression]) = other.apply(children.map(Column.apply) : _*).expr
         functionRegistry.createOrReplaceTempFunction(name, builder)
-        udf
+        other
     }
   }
 
@@ -133,9 +136,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
         | * @since 1.3.0
         | */
         |def register[$typeTags](name: String, func: Function$x[$types]): UserDefinedFunction = {
-        |  val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+        |  val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+        |  val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
         |  val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = $inputEncoders
-        |  val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
+        |  val udf = SparkUserDefinedFunction(func, dataType, inputEncoders, outputEncoder).withName(name)
         |  val finalUdf = if (nullable) udf else udf.asNonNullable()
         |  def builder(e: Seq[Expression]) = if (e.length == $x) {
         |    finalUdf.createScalaUDF(e)
@@ -161,9 +165,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
         | * @since $version
         | */
         |def register(name: String, f: UDF$i[$extTypeArgs], returnType: DataType): Unit = {
+        |  val replaced = CharVarcharUtils.failIfHasCharVarchar(returnType)
         |  val func = $funcCall
         |  def builder(e: Seq[Expression]) = if (e.length == $i) {
-        |    ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
+        |    ScalaUDF(func, replaced, e, Nil, udfName = Some(name))
         |  } else {
         |    throw new AnalysisException("Invalid number of arguments for function " + name +
         |      ". Expected: $i; Found: " + e.length)
@@ -179,9 +184,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register[RT: TypeTag](name: String, func: Function0[RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders, outputEncoder).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 0) {
       finalUdf.createScalaUDF(e)
@@ -199,9 +205,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register[RT: TypeTag, A1: TypeTag](name: String, func: Function1[A1, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders, outputEncoder).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 1) {
       finalUdf.createScalaUDF(e)
@@ -219,9 +226,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag](name: String, func: Function2[A1, A2, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders, outputEncoder).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 2) {
       finalUdf.createScalaUDF(e)
@@ -239,9 +247,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag](name: String, func: Function3[A1, A2, A3, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders, outputEncoder).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 3) {
       finalUdf.createScalaUDF(e)
@@ -259,9 +268,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag](name: String, func: Function4[A1, A2, A3, A4, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders, outputEncoder).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 4) {
       finalUdf.createScalaUDF(e)
@@ -279,9 +289,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag](name: String, func: Function5[A1, A2, A3, A4, A5, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders, outputEncoder).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 5) {
       finalUdf.createScalaUDF(e)
@@ -299,9 +310,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag](name: String, func: Function6[A1, A2, A3, A4, A5, A6, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders, outputEncoder).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 6) {
       finalUdf.createScalaUDF(e)
@@ -319,9 +331,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag](name: String, func: Function7[A1, A2, A3, A4, A5, A6, A7, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders, outputEncoder).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 7) {
       finalUdf.createScalaUDF(e)
@@ -339,9 +352,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag](name: String, func: Function8[A1, A2, A3, A4, A5, A6, A7, A8, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders, outputEncoder).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 8) {
       finalUdf.createScalaUDF(e)
@@ -359,9 +373,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag](name: String, func: Function9[A1, A2, A3, A4, A5, A6, A7, A8, A9, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders, outputEncoder).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 9) {
       finalUdf.createScalaUDF(e)
@@ -379,9 +394,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag](name: String, func: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders, outputEncoder).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 10) {
       finalUdf.createScalaUDF(e)
@@ -399,9 +415,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag](name: String, func: Function11[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders, outputEncoder).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 11) {
       finalUdf.createScalaUDF(e)
@@ -419,9 +436,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag](name: String, func: Function12[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders, outputEncoder).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 12) {
       finalUdf.createScalaUDF(e)
@@ -439,9 +457,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag](name: String, func: Function13[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders, outputEncoder).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 13) {
       finalUdf.createScalaUDF(e)
@@ -459,9 +478,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag](name: String, func: Function14[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders, outputEncoder).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 14) {
       finalUdf.createScalaUDF(e)
@@ -479,9 +499,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag](name: String, func: Function15[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Try(ExpressionEncoder[A15]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders, outputEncoder).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 15) {
       finalUdf.createScalaUDF(e)
@@ -499,9 +520,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag](name: String, func: Function16[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Try(ExpressionEncoder[A15]()).toOption :: Try(ExpressionEncoder[A16]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders, outputEncoder).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 16) {
       finalUdf.createScalaUDF(e)
@@ -519,9 +541,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag](name: String, func: Function17[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Try(ExpressionEncoder[A15]()).toOption :: Try(ExpressionEncoder[A16]()).toOption :: Try(ExpressionEncoder[A17]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders, outputEncoder).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 17) {
       finalUdf.createScalaUDF(e)
@@ -539,9 +562,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag](name: String, func: Function18[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Try(ExpressionEncoder[A15]()).toOption :: Try(ExpressionEncoder[A16]()).toOption :: Try(ExpressionEncoder[A17]()).toOption :: Try(ExpressionEncoder[A18]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders, outputEncoder).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 18) {
       finalUdf.createScalaUDF(e)
@@ -559,9 +583,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag](name: String, func: Function19[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Try(ExpressionEncoder[A15]()).toOption :: Try(ExpressionEncoder[A16]()).toOption :: Try(ExpressionEncoder[A17]()).toOption :: Try(ExpressionEncoder[A18]()).toOption :: Try(ExpressionEncoder[A19]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders, outputEncoder).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 19) {
       finalUdf.createScalaUDF(e)
@@ -579,9 +604,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag](name: String, func: Function20[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Try(ExpressionEncoder[A15]()).toOption :: Try(ExpressionEncoder[A16]()).toOption :: Try(ExpressionEncoder[A17]()).toOption :: Try(ExpressionEncoder[A18]()).toOption :: Try(ExpressionEncoder[A19]()).toOption :: Try(ExpressionEncoder[A20]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders, outputEncoder).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 20) {
       finalUdf.createScalaUDF(e)
@@ -599,9 +625,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag](name: String, func: Function21[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Try(ExpressionEncoder[A15]()).toOption :: Try(ExpressionEncoder[A16]()).toOption :: Try(ExpressionEncoder[A17]()).toOption :: Try(ExpressionEncoder[A18]()).toOption :: Try(ExpressionEncoder[A19]()).toOption :: Try(ExpressionEncoder[A20]()).toOption :: Try(ExpressionEncoder[A21]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders, outputEncoder).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 21) {
       finalUdf.createScalaUDF(e)
@@ -619,9 +646,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag, A22: TypeTag](name: String, func: Function22[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, A22, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Try(ExpressionEncoder[A15]()).toOption :: Try(ExpressionEncoder[A16]()).toOption :: Try(ExpressionEncoder[A17]()).toOption :: Try(ExpressionEncoder[A18]()).toOption :: Try(ExpressionEncoder[A19]()).toOption :: Try(ExpressionEncoder[A20]()).toOption :: Try(ExpressionEncoder[A21]()).toOption :: Try(ExpressionEncoder[A22]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders, outputEncoder).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 22) {
       finalUdf.createScalaUDF(e)
@@ -729,9 +757,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 2.3.0
    */
   def register(name: String, f: UDF0[_], returnType: DataType): Unit = {
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(returnType)
     val func = () => f.asInstanceOf[UDF0[Any]].call()
     def builder(e: Seq[Expression]) = if (e.length == 0) {
-      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
+      ScalaUDF(func, replaced, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 0; Found: " + e.length)
@@ -744,9 +773,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register(name: String, f: UDF1[_, _], returnType: DataType): Unit = {
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(returnType)
     val func = f.asInstanceOf[UDF1[Any, Any]].call(_: Any)
     def builder(e: Seq[Expression]) = if (e.length == 1) {
-      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
+      ScalaUDF(func, replaced, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 1; Found: " + e.length)
@@ -759,9 +789,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register(name: String, f: UDF2[_, _, _], returnType: DataType): Unit = {
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(returnType)
     val func = f.asInstanceOf[UDF2[Any, Any, Any]].call(_: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 2) {
-      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
+      ScalaUDF(func, replaced, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 2; Found: " + e.length)
@@ -774,9 +805,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register(name: String, f: UDF3[_, _, _, _], returnType: DataType): Unit = {
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(returnType)
     val func = f.asInstanceOf[UDF3[Any, Any, Any, Any]].call(_: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 3) {
-      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
+      ScalaUDF(func, replaced, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 3; Found: " + e.length)
@@ -789,9 +821,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register(name: String, f: UDF4[_, _, _, _, _], returnType: DataType): Unit = {
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(returnType)
     val func = f.asInstanceOf[UDF4[Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 4) {
-      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
+      ScalaUDF(func, replaced, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 4; Found: " + e.length)
@@ -804,9 +837,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register(name: String, f: UDF5[_, _, _, _, _, _], returnType: DataType): Unit = {
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(returnType)
     val func = f.asInstanceOf[UDF5[Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 5) {
-      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
+      ScalaUDF(func, replaced, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 5; Found: " + e.length)
@@ -819,9 +853,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register(name: String, f: UDF6[_, _, _, _, _, _, _], returnType: DataType): Unit = {
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(returnType)
     val func = f.asInstanceOf[UDF6[Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 6) {
-      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
+      ScalaUDF(func, replaced, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 6; Found: " + e.length)
@@ -834,9 +869,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register(name: String, f: UDF7[_, _, _, _, _, _, _, _], returnType: DataType): Unit = {
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(returnType)
     val func = f.asInstanceOf[UDF7[Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 7) {
-      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
+      ScalaUDF(func, replaced, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 7; Found: " + e.length)
@@ -849,9 +885,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register(name: String, f: UDF8[_, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(returnType)
     val func = f.asInstanceOf[UDF8[Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 8) {
-      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
+      ScalaUDF(func, replaced, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 8; Found: " + e.length)
@@ -864,9 +901,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register(name: String, f: UDF9[_, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(returnType)
     val func = f.asInstanceOf[UDF9[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 9) {
-      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
+      ScalaUDF(func, replaced, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 9; Found: " + e.length)
@@ -879,9 +917,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register(name: String, f: UDF10[_, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(returnType)
     val func = f.asInstanceOf[UDF10[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 10) {
-      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
+      ScalaUDF(func, replaced, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 10; Found: " + e.length)
@@ -894,9 +933,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register(name: String, f: UDF11[_, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(returnType)
     val func = f.asInstanceOf[UDF11[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 11) {
-      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
+      ScalaUDF(func, replaced, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 11; Found: " + e.length)
@@ -909,9 +949,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register(name: String, f: UDF12[_, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(returnType)
     val func = f.asInstanceOf[UDF12[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 12) {
-      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
+      ScalaUDF(func, replaced, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 12; Found: " + e.length)
@@ -924,9 +965,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register(name: String, f: UDF13[_, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(returnType)
     val func = f.asInstanceOf[UDF13[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 13) {
-      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
+      ScalaUDF(func, replaced, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 13; Found: " + e.length)
@@ -939,9 +981,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register(name: String, f: UDF14[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(returnType)
     val func = f.asInstanceOf[UDF14[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 14) {
-      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
+      ScalaUDF(func, replaced, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 14; Found: " + e.length)
@@ -954,9 +997,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register(name: String, f: UDF15[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(returnType)
     val func = f.asInstanceOf[UDF15[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 15) {
-      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
+      ScalaUDF(func, replaced, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 15; Found: " + e.length)
@@ -969,9 +1013,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register(name: String, f: UDF16[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(returnType)
     val func = f.asInstanceOf[UDF16[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 16) {
-      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
+      ScalaUDF(func, replaced, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 16; Found: " + e.length)
@@ -984,9 +1029,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register(name: String, f: UDF17[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(returnType)
     val func = f.asInstanceOf[UDF17[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 17) {
-      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
+      ScalaUDF(func, replaced, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 17; Found: " + e.length)
@@ -999,9 +1045,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register(name: String, f: UDF18[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(returnType)
     val func = f.asInstanceOf[UDF18[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 18) {
-      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
+      ScalaUDF(func, replaced, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 18; Found: " + e.length)
@@ -1014,9 +1061,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register(name: String, f: UDF19[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(returnType)
     val func = f.asInstanceOf[UDF19[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 19) {
-      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
+      ScalaUDF(func, replaced, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 19; Found: " + e.length)
@@ -1029,9 +1077,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register(name: String, f: UDF20[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(returnType)
     val func = f.asInstanceOf[UDF20[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 20) {
-      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
+      ScalaUDF(func, replaced, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 20; Found: " + e.length)
@@ -1044,9 +1093,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register(name: String, f: UDF21[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(returnType)
     val func = f.asInstanceOf[UDF21[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 21) {
-      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
+      ScalaUDF(func, replaced, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 21; Found: " + e.length)
@@ -1059,9 +1109,10 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    * @since 1.3.0
    */
   def register(name: String, f: UDF22[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(returnType)
     val func = f.asInstanceOf[UDF22[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 22) {
-      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
+      ScalaUDF(func, replaced, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 22; Found: " + e.length)
@@ -1072,3 +1123,17 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   // scalastyle:on line.size.limit
 
 }
+
+private[sql] object UDFRegistration {
+  /**
+   * Obtaining the schema of output encoder for `ScalaUDF`.
+   *
+   * As the serialization in `ScalaUDF` is for individual column, not the whole row,
+   * we just take the data type of vanilla object serializer, not `serializer` which
+   * is transformed somehow for top-level row.
+   */
+  def outputSchema(outputEncoder: ExpressionEncoder[_]): ScalaReflection.Schema = {
+    ScalaReflection.Schema(outputEncoder.objSerializer.dataType,
+      outputEncoder.objSerializer.nullable)
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
index 38254609e8f67..2cd26e2d42ed6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
@@ -20,20 +20,17 @@ package org.apache.spark.sql.api.python
 import java.io.InputStream
 import java.nio.channels.Channels
 
-import scala.util.control.NonFatal
-
 import org.apache.spark.api.java.JavaRDD
 import org.apache.spark.api.python.PythonRDDServer
 import org.apache.spark.internal.Logging
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{DataFrame, SQLContext}
-import org.apache.spark.sql.catalyst.ScalaReflection
 import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
 import org.apache.spark.sql.catalyst.expressions.ExpressionInfo
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
 import org.apache.spark.sql.execution.{ExplainMode, QueryExecution}
 import org.apache.spark.sql.execution.arrow.ArrowConverters
-import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.DataType
 
 private[sql] object PythonSQLUtils extends Logging {
@@ -46,16 +43,6 @@ private[sql] object PythonSQLUtils extends Logging {
 
   private def listAllSQLConfigs(): Seq[(String, String, String, String)] = {
     val conf = new SQLConf()
-    // Force to build static SQL configurations
-    StaticSQLConf
-    // Force to build SQL configurations from Hive module
-    try {
-      val symbol = ScalaReflection.mirror.staticModule("org.apache.spark.sql.hive.HiveUtils")
-      ScalaReflection.mirror.reflectModule(symbol).instance
-    } catch {
-      case NonFatal(e) =>
-        logWarning("Cannot generated sql configurations from hive module", e)
-    }
     conf.getAllDefinedConfs
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala
index 693be99d47495..a50752c8a5df5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala
@@ -23,7 +23,6 @@ import java.util.{Locale, Map => JMap}
 import scala.collection.JavaConverters._
 import scala.util.matching.Regex
 
-import org.apache.spark.SparkContext
 import org.apache.spark.api.java.{JavaRDD, JavaSparkContext}
 import org.apache.spark.api.r.SerDe
 import org.apache.spark.broadcast.Broadcast
@@ -33,18 +32,12 @@ import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.expressions.{ExprUtils, GenericRowWithSchema}
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
 import org.apache.spark.sql.execution.arrow.ArrowConverters
-import org.apache.spark.sql.execution.command.ShowTablesCommand
 import org.apache.spark.sql.internal.StaticSQLConf.CATALOG_IMPLEMENTATION
 import org.apache.spark.sql.types._
 
 private[sql] object SQLUtils extends Logging {
   SerDe.setSQLReadObject(readSqlObject).setSQLWriteObject(writeSqlObject)
 
-  private[this] def withHiveExternalCatalog(sc: SparkContext): SparkContext = {
-    sc.conf.set(CATALOG_IMPLEMENTATION.key, "hive")
-    sc
-  }
-
   def getOrCreateSparkSession(
       jsc: JavaSparkContext,
       sparkConfigMap: JMap[Object, Object],
@@ -61,7 +54,7 @@ private[sql] object SQLUtils extends Logging {
           // So, we intentionally check if Hive classes are loadable or not only when
           // Hive support is explicitly enabled by short-circuiting. See also SPARK-26422.
           SparkSession.hiveClassesArePresent) {
-        SparkSession.builder().sparkContext(withHiveExternalCatalog(jsc.sc)).getOrCreate()
+        SparkSession.builder().enableHiveSupport().sparkContext(jsc.sc).getOrCreate()
       } else {
         if (enableHiveSupport) {
           logWarning("SparkR: enableHiveSupport is requested for SparkSession but " +
@@ -216,15 +209,6 @@ private[sql] object SQLUtils extends Logging {
     }
   }
 
-  def getTables(sparkSession: SparkSession, databaseName: String): DataFrame = {
-    databaseName match {
-      case n: String if n != null && n.trim.nonEmpty =>
-        Dataset.ofRows(sparkSession, ShowTablesCommand(Some(n), None))
-      case _ =>
-        Dataset.ofRows(sparkSession, ShowTablesCommand(None, None))
-    }
-  }
-
   def getTableNames(sparkSession: SparkSession, databaseName: String): Array[String] = {
     val db = databaseName match {
       case _ if databaseName != null && databaseName.trim.nonEmpty =>
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalog/Catalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalog/Catalog.scala
index 60738e6d4ef9e..1436574c0d90a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/catalog/Catalog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/catalog/Catalog.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalog
 
 import scala.collection.JavaConverters._
 
-import org.apache.spark.annotation.{Evolving, Experimental, Stable}
+import org.apache.spark.annotation.Stable
 import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset}
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.storage.StorageLevel
@@ -342,6 +342,44 @@ abstract class Catalog {
     createTable(tableName, source, schema, options)
   }
 
+  /**
+   * Creates a table based on the dataset in a data source and a set of options.
+   * Then, returns the corresponding DataFrame.
+   *
+   * @param tableName is either a qualified or unqualified name that designates a table.
+   *                  If no database identifier is provided, it refers to a table in
+   *                  the current database.
+   * @since 3.1.0
+   */
+  def createTable(
+      tableName: String,
+      source: String,
+      description: String,
+      options: java.util.Map[String, String]): DataFrame = {
+    createTable(
+      tableName,
+      source = source,
+      description = description,
+      options = options.asScala.toMap
+    )
+  }
+
+  /**
+   * (Scala-specific)
+   * Creates a table based on the dataset in a data source and a set of options.
+   * Then, returns the corresponding DataFrame.
+   *
+   * @param tableName is either a qualified or unqualified name that designates a table.
+   *                  If no database identifier is provided, it refers to a table in
+   *                  the current database.
+   * @since 3.1.0
+   */
+  def createTable(
+      tableName: String,
+      source: String,
+      description: String,
+      options: Map[String, String]): DataFrame
+
   /**
    * Create a table based on the dataset in a data source, a schema and a set of options.
    * Then, returns the corresponding DataFrame.
@@ -394,6 +432,47 @@ abstract class Catalog {
       schema: StructType,
       options: Map[String, String]): DataFrame
 
+  /**
+   * Create a table based on the dataset in a data source, a schema and a set of options.
+   * Then, returns the corresponding DataFrame.
+   *
+   * @param tableName is either a qualified or unqualified name that designates a table.
+   *                  If no database identifier is provided, it refers to a table in
+   *                  the current database.
+   * @since 3.1.0
+   */
+  def createTable(
+      tableName: String,
+      source: String,
+      schema: StructType,
+      description: String,
+      options: java.util.Map[String, String]): DataFrame = {
+    createTable(
+      tableName,
+      source = source,
+      schema = schema,
+      description = description,
+      options = options.asScala.toMap
+    )
+  }
+
+  /**
+   * (Scala-specific)
+   * Create a table based on the dataset in a data source, a schema and a set of options.
+   * Then, returns the corresponding DataFrame.
+   *
+   * @param tableName is either a qualified or unqualified name that designates a table.
+   *                  If no database identifier is provided, it refers to a table in
+   *                  the current database.
+   * @since 3.1.0
+   */
+  def createTable(
+      tableName: String,
+      source: String,
+      schema: StructType,
+      description: String,
+      options: Map[String, String]): DataFrame
+
   /**
    * Drops the local temporary view with the given view name in the catalog.
    * If the view has been cached before, then it will also be uncached.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
index bf90875e511f8..7f3d0c621147e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
@@ -17,18 +17,21 @@
 
 package org.apache.spark.sql.catalyst.analysis
 
-import org.apache.spark.sql.{AnalysisException, SaveMode}
+import org.apache.spark.sql.SaveMode
 import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
 import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStorageFormat, CatalogTable, CatalogTableType, CatalogUtils}
+import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute}
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.connector.catalog.{CatalogManager, CatalogPlugin, CatalogV2Util, LookupCatalog, SupportsNamespaces, TableCatalog, TableChange, V1Table}
+import org.apache.spark.sql.catalyst.util.{quoteIfNeeded, toPrettySQL}
+import org.apache.spark.sql.connector.catalog.{CatalogManager, CatalogPlugin, CatalogV2Util, Identifier, LookupCatalog, SupportsNamespaces, TableChange, V1Table}
 import org.apache.spark.sql.connector.expressions.Transform
+import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.execution.command._
-import org.apache.spark.sql.execution.datasources.{CreateTable, DataSource, RefreshTable}
+import org.apache.spark.sql.execution.datasources.{CreateTable, DataSource}
 import org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{HIVE_TYPE_STRING, HiveStringType, MetadataBuilder, StructField, StructType}
+import org.apache.spark.sql.internal.{HiveSerDe, SQLConf}
+import org.apache.spark.sql.types.{MetadataBuilder, StructField, StructType}
 
 /**
  * Resolves catalogs from the multi-part identifiers in SQL statements, and convert the statements
@@ -36,33 +39,26 @@ import org.apache.spark.sql.types.{HIVE_TYPE_STRING, HiveStringType, MetadataBui
  *
  * We can remove this rule once we implement all the catalog functionality in `V2SessionCatalog`.
  */
-class ResolveSessionCatalog(
-    val catalogManager: CatalogManager,
-    conf: SQLConf,
-    isTempView: Seq[String] => Boolean,
-    isTempFunction: String => Boolean)
+class ResolveSessionCatalog(val catalogManager: CatalogManager)
   extends Rule[LogicalPlan] with LookupCatalog {
   import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
   import org.apache.spark.sql.connector.catalog.CatalogV2Util._
+  import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Implicits._
 
   override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
     case AlterTableAddColumnsStatement(
          nameParts @ SessionCatalogAndTable(catalog, tbl), cols) =>
+      cols.foreach(c => failNullType(c.dataType))
       loadTable(catalog, tbl.asIdentifier).collect {
         case v1Table: V1Table =>
-          if (!DDLUtils.isHiveTable(v1Table.v1Table)) {
-            cols.foreach(c => failCharType(c.dataType))
-          }
           cols.foreach { c =>
             assertTopLevelColumn(c.name, "AlterTableAddColumnsCommand")
             if (!c.nullable) {
-              throw new AnalysisException(
-                "ADD COLUMN with v1 tables cannot specify NOT NULL.")
+              throw QueryCompilationErrors.addColumnWithV1TableCannotSpecifyNotNullError
             }
           }
           AlterTableAddColumnsCommand(tbl.asTableIdentifier, cols.map(convertToStructField))
       }.getOrElse {
-        cols.foreach(c => failCharType(c.dataType))
         val changes = cols.map { col =>
           TableChange.addColumn(
             col.name.toArray,
@@ -76,11 +72,11 @@ class ResolveSessionCatalog(
 
     case AlterTableReplaceColumnsStatement(
         nameParts @ SessionCatalogAndTable(catalog, tbl), cols) =>
+      cols.foreach(c => failNullType(c.dataType))
       val changes: Seq[TableChange] = loadTable(catalog, tbl.asIdentifier) match {
         case Some(_: V1Table) =>
-          throw new AnalysisException("REPLACE COLUMNS is only supported with v2 tables.")
+          throw QueryCompilationErrors.replaceColumnsOnlySupportedWithV2TableError
         case Some(table) =>
-          cols.foreach(c => failCharType(c.dataType))
           // REPLACE COLUMNS deletes all the existing columns and adds new columns specified.
           val deleteChanges = table.schema.fieldNames.map { name =>
             TableChange.deleteColumn(Array(name))
@@ -100,23 +96,17 @@ class ResolveSessionCatalog(
 
     case a @ AlterTableAlterColumnStatement(
          nameParts @ SessionCatalogAndTable(catalog, tbl), _, _, _, _, _) =>
+      a.dataType.foreach(failNullType)
       loadTable(catalog, tbl.asIdentifier).collect {
         case v1Table: V1Table =>
-          if (!DDLUtils.isHiveTable(v1Table.v1Table)) {
-            a.dataType.foreach(failCharType)
-          }
-
           if (a.column.length > 1) {
-            throw new AnalysisException(
-              "ALTER COLUMN with qualified column is only supported with v2 tables.")
+            throw QueryCompilationErrors.alterQualifiedColumnOnlySupportedWithV2TableError
           }
           if (a.nullable.isDefined) {
-            throw new AnalysisException(
-              "ALTER COLUMN with v1 tables cannot specify NOT NULL.")
+            throw QueryCompilationErrors.alterColumnWithV1TableCannotSpecifyNotNullError
           }
           if (a.position.isDefined) {
-            throw new AnalysisException("" +
-              "ALTER COLUMN ... FIRST | ALTER is only supported with v2 tables.")
+            throw QueryCompilationErrors.alterOnlySupportedWithV2TableError
           }
           val builder = new MetadataBuilder
           // Add comment to metadata
@@ -126,24 +116,17 @@ class ResolveSessionCatalog(
             v1Table.schema.findNestedField(Seq(colName), resolver = conf.resolver)
               .map(_._2.dataType)
               .getOrElse {
-                throw new AnalysisException(
-                  s"ALTER COLUMN cannot find column ${quoteIfNeeded(colName)} in v1 table. " +
-                    s"Available: ${v1Table.schema.fieldNames.mkString(", ")}")
+                throw QueryCompilationErrors.alterColumnCannotFindColumnInV1TableError(
+                  quoteIfNeeded(colName), v1Table)
               }
           }
-          // Add Hive type string to metadata.
-          val cleanedDataType = HiveStringType.replaceCharType(dataType)
-          if (dataType != cleanedDataType) {
-            builder.putString(HIVE_TYPE_STRING, dataType.catalogString)
-          }
           val newColumn = StructField(
             colName,
-            cleanedDataType,
+            dataType,
             nullable = true,
             builder.build())
           AlterTableChangeColumnCommand(tbl.asTableIdentifier, colName, newColumn)
       }.getOrElse {
-        a.dataType.foreach(failCharType)
         val colName = a.column.toArray
         val typeChange = a.dataType.map { newDataType =>
           TableChange.updateColumnType(colName, newDataType)
@@ -168,7 +151,7 @@ class ResolveSessionCatalog(
          nameParts @ SessionCatalogAndTable(catalog, tbl), col, newName) =>
       loadTable(catalog, tbl.asIdentifier).collect {
         case v1Table: V1Table =>
-          throw new AnalysisException("RENAME COLUMN is only supported with v2 tables.")
+          throw QueryCompilationErrors.renameColumnOnlySupportedWithV2TableError
       }.getOrElse {
         val changes = Seq(TableChange.renameColumn(col.toArray, newName))
         createAlterTable(nameParts, catalog, tbl, changes)
@@ -178,125 +161,110 @@ class ResolveSessionCatalog(
          nameParts @ SessionCatalogAndTable(catalog, tbl), cols) =>
       loadTable(catalog, tbl.asIdentifier).collect {
         case v1Table: V1Table =>
-          throw new AnalysisException("DROP COLUMN is only supported with v2 tables.")
+          throw QueryCompilationErrors.dropColumnOnlySupportedWithV2TableError
       }.getOrElse {
         val changes = cols.map(col => TableChange.deleteColumn(col.toArray))
         createAlterTable(nameParts, catalog, tbl, changes)
       }
 
-    case AlterTableSetPropertiesStatement(
-         nameParts @ SessionCatalogAndTable(catalog, tbl), props) =>
-      loadTable(catalog, tbl.asIdentifier).collect {
-        case v1Table: V1Table =>
-          AlterTableSetPropertiesCommand(tbl.asTableIdentifier, props, isView = false)
-      }.getOrElse {
-        val changes = props.map { case (key, value) =>
-          TableChange.setProperty(key, value)
-        }.toSeq
-        createAlterTable(nameParts, catalog, tbl, changes)
-      }
+    case SetTableProperties(ResolvedV1TableIdentifier(ident), props) =>
+      AlterTableSetPropertiesCommand(ident.asTableIdentifier, props, isView = false)
 
-    case AlterTableUnsetPropertiesStatement(
-         nameParts @ SessionCatalogAndTable(catalog, tbl), keys, ifExists) =>
-      loadTable(catalog, tbl.asIdentifier).collect {
-        case v1Table: V1Table =>
-          AlterTableUnsetPropertiesCommand(
-            tbl.asTableIdentifier, keys, ifExists, isView = false)
-      }.getOrElse {
-        val changes = keys.map(key => TableChange.removeProperty(key))
-        createAlterTable(nameParts, catalog, tbl, changes)
-      }
+    case UnsetTableProperties(ResolvedV1TableIdentifier(ident), keys, ifExists) =>
+      AlterTableUnsetPropertiesCommand(ident.asTableIdentifier, keys, ifExists, isView = false)
 
-    case AlterTableSetLocationStatement(
-         nameParts @ SessionCatalogAndTable(catalog, tbl), partitionSpec, newLoc) =>
-      loadTable(catalog, tbl.asIdentifier).collect {
-        case v1Table: V1Table =>
-          AlterTableSetLocationCommand(tbl.asTableIdentifier, partitionSpec, newLoc)
-      }.getOrElse {
-        if (partitionSpec.nonEmpty) {
-          throw new AnalysisException(
-            "ALTER TABLE SET LOCATION does not support partition for v2 tables.")
-        }
-        val changes = Seq(TableChange.setProperty(TableCatalog.PROP_LOCATION, newLoc))
-        createAlterTable(nameParts, catalog, tbl, changes)
-      }
-
-    // ALTER VIEW should always use v1 command if the resolved catalog is session catalog.
-    case AlterViewSetPropertiesStatement(SessionCatalogAndTable(_, tbl), props) =>
-      AlterTableSetPropertiesCommand(tbl.asTableIdentifier, props, isView = true)
+    case SetViewProperties(ResolvedView(ident, _), props) =>
+      AlterTableSetPropertiesCommand(ident.asTableIdentifier, props, isView = true)
 
-    case AlterViewUnsetPropertiesStatement(SessionCatalogAndTable(_, tbl), keys, ifExists) =>
-      AlterTableUnsetPropertiesCommand(tbl.asTableIdentifier, keys, ifExists, isView = true)
+    case UnsetViewProperties(ResolvedView(ident, _), keys, ifExists) =>
+      AlterTableUnsetPropertiesCommand(ident.asTableIdentifier, keys, ifExists, isView = true)
 
-    case d @ DescribeNamespace(SessionCatalogAndNamespace(_, ns), _) =>
-      if (ns.length != 1) {
-        throw new AnalysisException(
-          s"The database name is not valid: ${ns.quoted}")
+    case DescribeNamespace(DatabaseInSessionCatalog(db), extended, output) =>
+      val newOutput = if (conf.getConf(SQLConf.LEGACY_KEEP_COMMAND_OUTPUT_SCHEMA)) {
+        assert(output.length == 2)
+        Seq(output.head.withName("database_description_item"),
+          output.last.withName("database_description_value"))
+      } else {
+        output
       }
-      DescribeDatabaseCommand(ns.head, d.extended)
+      DescribeDatabaseCommand(db, extended, newOutput)
 
-    case AlterNamespaceSetProperties(SessionCatalogAndNamespace(_, ns), properties) =>
-      if (ns.length != 1) {
-        throw new AnalysisException(
-          s"The database name is not valid: ${ns.quoted}")
-      }
-      AlterDatabasePropertiesCommand(ns.head, properties)
+    case SetNamespaceProperties(DatabaseInSessionCatalog(db), properties) =>
+      AlterDatabasePropertiesCommand(db, properties)
 
-    case AlterNamespaceSetLocation(SessionCatalogAndNamespace(_, ns), location) =>
-      if (ns.length != 1) {
-        throw new AnalysisException(
-          s"The database name is not valid: ${ns.quoted}")
+    case SetNamespaceLocation(DatabaseInSessionCatalog(db), location) =>
+      AlterDatabaseSetLocationCommand(db, location)
+
+    case s @ ShowNamespaces(ResolvedNamespace(cata, _), _, output) if isSessionCatalog(cata) =>
+      if (conf.getConf(SQLConf.LEGACY_KEEP_COMMAND_OUTPUT_SCHEMA)) {
+        assert(output.length == 1)
+        s.copy(output = Seq(output.head.withName("databaseName")))
+      } else {
+        s
       }
-      AlterDatabaseSetLocationCommand(ns.head, location)
 
-    // v1 RENAME TABLE supports temp view.
-    case RenameTableStatement(TempViewOrV1Table(oldName), newName, isView) =>
+    case RenameTable(ResolvedV1TableOrViewIdentifier(oldName), newName, isView) =>
       AlterTableRenameCommand(oldName.asTableIdentifier, newName.asTableIdentifier, isView)
 
-    case DescribeRelation(ResolvedTable(_, ident, _: V1Table), partitionSpec, isExtended) =>
-      DescribeTableCommand(ident.asTableIdentifier, partitionSpec, isExtended)
-
     // Use v1 command to describe (temp) view, as v2 catalog doesn't support view yet.
-    case DescribeRelation(ResolvedView(ident), partitionSpec, isExtended) =>
-      DescribeTableCommand(ident.asTableIdentifier, partitionSpec, isExtended)
-
-    case DescribeColumnStatement(tbl, colNameParts, isExtended) =>
-      val name = parseTempViewOrV1Table(tbl, "Describing columns")
-      DescribeColumnCommand(name.asTableIdentifier, colNameParts, isExtended)
+    case DescribeRelation(
+         ResolvedV1TableOrViewIdentifier(ident), partitionSpec, isExtended, output) =>
+      DescribeTableCommand(ident.asTableIdentifier, partitionSpec, isExtended, output)
+
+    case DescribeColumn(
+         ResolvedViewIdentifier(ident), column: UnresolvedAttribute, isExtended, output) =>
+      // For views, the column will not be resolved by `ResolveReferences` because
+      // `ResolvedView` stores only the identifier.
+      DescribeColumnCommand(ident.asTableIdentifier, column.nameParts, isExtended, output)
+
+    case DescribeColumn(ResolvedV1TableIdentifier(ident), column, isExtended, output) =>
+      column match {
+        case u: UnresolvedAttribute =>
+          throw QueryCompilationErrors.columnDoesNotExistError(u.name)
+        case a: Attribute =>
+          DescribeColumnCommand(ident.asTableIdentifier, a.qualifier :+ a.name, isExtended, output)
+        case Alias(child, _) =>
+          throw QueryCompilationErrors.commandNotSupportNestedColumnError(
+            "DESC TABLE COLUMN", toPrettySQL(child))
+        case _ =>
+          throw new IllegalStateException(s"[BUG] unexpected column expression: $column")
+      }
 
     // For CREATE TABLE [AS SELECT], we should use the v1 command if the catalog is resolved to the
     // session catalog and the table provider is not v2.
     case c @ CreateTableStatement(
-         SessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
-      val provider = c.provider.getOrElse(conf.defaultDataSourceName)
+         SessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _, _) =>
+      assertNoNullTypeInSchema(c.tableSchema)
+      val (storageFormat, provider) = getStorageFormatAndProvider(
+        c.provider, c.options, c.location, c.serde, ctas = false)
       if (!isV2Provider(provider)) {
-        if (!DDLUtils.isHiveTable(Some(provider))) {
-          assertNoCharTypeInSchema(c.tableSchema)
-        }
         val tableDesc = buildCatalogTable(tbl.asTableIdentifier, c.tableSchema,
-          c.partitioning, c.bucketSpec, c.properties, provider, c.options, c.location,
-          c.comment, c.ifNotExists)
+          c.partitioning, c.bucketSpec, c.properties, provider, c.location,
+          c.comment, storageFormat, c.external)
         val mode = if (c.ifNotExists) SaveMode.Ignore else SaveMode.ErrorIfExists
         CreateTable(tableDesc, mode, None)
       } else {
-        assertNoCharTypeInSchema(c.tableSchema)
         CreateV2Table(
           catalog.asTableCatalog,
           tbl.asIdentifier,
           c.tableSchema,
           // convert the bucket spec and add it as a transform
           c.partitioning ++ c.bucketSpec.map(_.asTransform),
-          convertTableProperties(c.properties, c.options, c.location, c.comment, Some(provider)),
+          convertTableProperties(c),
           ignoreIfExists = c.ifNotExists)
       }
 
     case c @ CreateTableAsSelectStatement(
-         SessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _) =>
-      val provider = c.provider.getOrElse(conf.defaultDataSourceName)
+         SessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _, _, _) =>
+      if (c.asSelect.resolved) {
+        assertNoNullTypeInSchema(c.asSelect.schema)
+      }
+      val (storageFormat, provider) = getStorageFormatAndProvider(
+        c.provider, c.options, c.location, c.serde, ctas = true)
       if (!isV2Provider(provider)) {
         val tableDesc = buildCatalogTable(tbl.asTableIdentifier, new StructType,
-          c.partitioning, c.bucketSpec, c.properties, provider, c.options, c.location,
-          c.comment, c.ifNotExists)
+          c.partitioning, c.bucketSpec, c.properties, provider, c.location,
+          c.comment, storageFormat, c.external)
         val mode = if (c.ifNotExists) SaveMode.Ignore else SaveMode.ErrorIfExists
         CreateTable(tableDesc, mode, Some(c.asSelect))
       } else {
@@ -306,39 +274,44 @@ class ResolveSessionCatalog(
           // convert the bucket spec and add it as a transform
           c.partitioning ++ c.bucketSpec.map(_.asTransform),
           c.asSelect,
-          convertTableProperties(c.properties, c.options, c.location, c.comment, Some(provider)),
+          convertTableProperties(c),
           writeOptions = c.writeOptions,
           ignoreIfExists = c.ifNotExists)
       }
 
-    // v1 REFRESH TABLE supports temp view.
-    case RefreshTableStatement(TempViewOrV1Table(name)) =>
-      RefreshTable(name.asTableIdentifier)
+    case RefreshTable(ResolvedV1TableIdentifier(ident)) =>
+      RefreshTableCommand(ident.asTableIdentifier)
+
+    case RefreshTable(r: ResolvedView) =>
+      RefreshTableCommand(r.identifier.asTableIdentifier)
 
     // For REPLACE TABLE [AS SELECT], we should fail if the catalog is resolved to the
     // session catalog and the table provider is not v2.
     case c @ ReplaceTableStatement(
-         SessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
+         SessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _) =>
+      assertNoNullTypeInSchema(c.tableSchema)
       val provider = c.provider.getOrElse(conf.defaultDataSourceName)
       if (!isV2Provider(provider)) {
-        throw new AnalysisException("REPLACE TABLE is only supported with v2 tables.")
+        throw QueryCompilationErrors.replaceTableOnlySupportedWithV2TableError
       } else {
-        assertNoCharTypeInSchema(c.tableSchema)
         ReplaceTable(
           catalog.asTableCatalog,
           tbl.asIdentifier,
           c.tableSchema,
           // convert the bucket spec and add it as a transform
           c.partitioning ++ c.bucketSpec.map(_.asTransform),
-          convertTableProperties(c.properties, c.options, c.location, c.comment, Some(provider)),
+          convertTableProperties(c),
           orCreate = c.orCreate)
       }
 
     case c @ ReplaceTableAsSelectStatement(
-         SessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _) =>
+         SessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _, _) =>
+      if (c.asSelect.resolved) {
+        assertNoNullTypeInSchema(c.asSelect.schema)
+      }
       val provider = c.provider.getOrElse(conf.defaultDataSourceName)
       if (!isV2Provider(provider)) {
-        throw new AnalysisException("REPLACE TABLE AS SELECT is only supported with v2 tables.")
+        throw QueryCompilationErrors.replaceTableAsSelectOnlySupportedWithV2TableError
       } else {
         ReplaceTableAsSelect(
           catalog.asTableCatalog,
@@ -346,24 +319,28 @@ class ResolveSessionCatalog(
           // convert the bucket spec and add it as a transform
           c.partitioning ++ c.bucketSpec.map(_.asTransform),
           c.asSelect,
-          convertTableProperties(c.properties, c.options, c.location, c.comment, Some(provider)),
+          convertTableProperties(c),
           writeOptions = c.writeOptions,
           orCreate = c.orCreate)
       }
 
-    // v1 DROP TABLE supports temp view.
-    case DropTableStatement(TempViewOrV1Table(name), ifExists, purge) =>
-      DropTableCommand(name.asTableIdentifier, ifExists, isView = false, purge = purge)
+    case DropTable(ResolvedV1TableIdentifier(ident), ifExists, purge) =>
+      DropTableCommand(ident.asTableIdentifier, ifExists, isView = false, purge = purge)
 
     // v1 DROP TABLE supports temp view.
-    case DropViewStatement(TempViewOrV1Table(name), ifExists) =>
-      DropTableCommand(name.asTableIdentifier, ifExists, isView = true, purge = false)
+    case DropTable(r: ResolvedView, ifExists, purge) =>
+      if (!r.isTemp) {
+        throw QueryCompilationErrors.cannotDropViewWithDropTableError
+      }
+      DropTableCommand(r.identifier.asTableIdentifier, ifExists, isView = false, purge = purge)
+
+    case DropView(r: ResolvedView, ifExists) =>
+      DropTableCommand(r.identifier.asTableIdentifier, ifExists, isView = true, purge = false)
 
     case c @ CreateNamespaceStatement(CatalogAndNamespace(catalog, ns), _, _)
         if isSessionCatalog(catalog) =>
       if (ns.length != 1) {
-        throw new AnalysisException(
-          s"The database name is not valid: ${ns.quoted}")
+        throw QueryCompilationErrors.invalidDatabaseNameError(ns.quoted)
       }
 
       val comment = c.properties.get(SupportsNamespaces.PROP_COMMENT)
@@ -371,156 +348,135 @@ class ResolveSessionCatalog(
       val newProperties = c.properties -- CatalogV2Util.NAMESPACE_RESERVED_PROPERTIES
       CreateDatabaseCommand(ns.head, c.ifNotExists, location, comment, newProperties)
 
-    case d @ DropNamespace(SessionCatalogAndNamespace(_, ns), _, _) =>
-      if (ns.length != 1) {
-        throw new AnalysisException(
-          s"The database name is not valid: ${ns.quoted}")
-      }
-      DropDatabaseCommand(ns.head, d.ifExists, d.cascade)
+    case d @ DropNamespace(DatabaseInSessionCatalog(db), _, _) =>
+      DropDatabaseCommand(db, d.ifExists, d.cascade)
 
-    case ShowTables(SessionCatalogAndNamespace(_, ns), pattern) =>
-      assert(ns.nonEmpty)
-      if (ns.length != 1) {
-          throw new AnalysisException(
-            s"The database name is not valid: ${ns.quoted}")
+    case ShowTables(DatabaseInSessionCatalog(db), pattern, output) =>
+      val newOutput = if (conf.getConf(SQLConf.LEGACY_KEEP_COMMAND_OUTPUT_SCHEMA)) {
+        assert(output.length == 3)
+        output.head.withName("database") +: output.tail
+      } else {
+        output
       }
-      ShowTablesCommand(Some(ns.head), pattern)
-
-    case ShowTableStatement(ns, pattern, partitionsSpec) =>
-      val db = ns match {
-        case Some(ns) if ns.length != 1 =>
-          throw new AnalysisException(
-            s"The database name is not valid: ${ns.quoted}")
-        case _ => ns.map(_.head)
+      ShowTablesCommand(Some(db), pattern, newOutput)
+
+    case ShowTableExtended(
+        DatabaseInSessionCatalog(db),
+        pattern,
+        partitionSpec @ (None | Some(UnresolvedPartitionSpec(_, _))),
+        output) =>
+      val newOutput = if (conf.getConf(SQLConf.LEGACY_KEEP_COMMAND_OUTPUT_SCHEMA)) {
+        assert(output.length == 4)
+        output.head.withName("database") +: output.tail
+      } else {
+        output
       }
-      ShowTablesCommand(db, Some(pattern), true, partitionsSpec)
+      val tablePartitionSpec = partitionSpec.map(_.asInstanceOf[UnresolvedPartitionSpec].spec)
+      ShowTablesCommand(Some(db), Some(pattern), newOutput, true, tablePartitionSpec)
 
-    case AnalyzeTableStatement(tbl, partitionSpec, noScan) =>
-      val v1TableName = parseV1Table(tbl, "ANALYZE TABLE")
+    // ANALYZE TABLE works on permanent views if the views are cached.
+    case AnalyzeTable(ResolvedV1TableOrViewIdentifier(ident), partitionSpec, noScan) =>
       if (partitionSpec.isEmpty) {
-        AnalyzeTableCommand(v1TableName.asTableIdentifier, noScan)
+        AnalyzeTableCommand(ident.asTableIdentifier, noScan)
       } else {
-        AnalyzePartitionCommand(v1TableName.asTableIdentifier, partitionSpec, noScan)
+        AnalyzePartitionCommand(ident.asTableIdentifier, partitionSpec, noScan)
       }
 
-    case AnalyzeColumnStatement(tbl, columnNames, allColumns) =>
-      val v1TableName = parseTempViewOrV1Table(tbl, "ANALYZE TABLE")
-      AnalyzeColumnCommand(v1TableName.asTableIdentifier, columnNames, allColumns)
+    case AnalyzeTables(DatabaseInSessionCatalog(db), noScan) =>
+      AnalyzeTablesCommand(Some(db), noScan)
+
+    case AnalyzeColumn(ResolvedV1TableOrViewIdentifier(ident), columnNames, allColumns) =>
+      AnalyzeColumnCommand(ident.asTableIdentifier, columnNames, allColumns)
 
-    case RepairTableStatement(tbl) =>
-      val v1TableName = parseV1Table(tbl, "MSCK REPAIR TABLE")
-      AlterTableRecoverPartitionsCommand(
-        v1TableName.asTableIdentifier,
-        "MSCK REPAIR TABLE")
+    case RepairTable(ResolvedV1TableIdentifier(ident), addPartitions, dropPartitions) =>
+      RepairTableCommand(ident.asTableIdentifier, addPartitions, dropPartitions)
 
-    case LoadDataStatement(tbl, path, isLocal, isOverwrite, partition) =>
-      val v1TableName = parseV1Table(tbl, "LOAD DATA")
+    case LoadData(ResolvedV1TableIdentifier(ident), path, isLocal, isOverwrite, partition) =>
       LoadDataCommand(
-        v1TableName.asTableIdentifier,
+        ident.asTableIdentifier,
         path,
         isLocal,
         isOverwrite,
         partition)
 
-    case ShowCreateTableStatement(tbl, asSerde) if !asSerde =>
-      val name = parseTempViewOrV1Table(tbl, "SHOW CREATE TABLE")
-      ShowCreateTableCommand(name.asTableIdentifier)
-
-    case ShowCreateTableStatement(tbl, asSerde) if asSerde =>
-      val v1TableName = parseV1Table(tbl, "SHOW CREATE TABLE AS SERDE")
-      ShowCreateTableAsSerdeCommand(v1TableName.asTableIdentifier)
-
-    case CacheTableStatement(tbl, plan, isLazy, options) =>
-      val name = if (plan.isDefined) {
-        // CACHE TABLE ... AS SELECT creates a temp view with the input query.
-        // Temp view doesn't belong to any catalog and we shouldn't resolve catalog in the name.
-        tbl
+    case ShowCreateTable(ResolvedV1TableOrViewIdentifier(ident), asSerde, output) =>
+      if (asSerde) {
+        ShowCreateTableAsSerdeCommand(ident.asTableIdentifier, output)
       } else {
-        parseTempViewOrV1Table(tbl, "CACHE TABLE")
+        ShowCreateTableCommand(ident.asTableIdentifier, output)
       }
-      CacheTableCommand(name.asTableIdentifier, plan, isLazy, options)
 
-    case UncacheTableStatement(tbl, ifExists) =>
-      val name = parseTempViewOrV1Table(tbl, "UNCACHE TABLE")
-      UncacheTableCommand(name.asTableIdentifier, ifExists)
+    case TruncateTable(ResolvedV1TableIdentifier(ident)) =>
+      TruncateTableCommand(ident.asTableIdentifier, None)
 
-    case TruncateTableStatement(tbl, partitionSpec) =>
-      val v1TableName = parseV1Table(tbl, "TRUNCATE TABLE")
+    case TruncatePartition(ResolvedV1TableIdentifier(ident), partitionSpec) =>
       TruncateTableCommand(
-        v1TableName.asTableIdentifier,
-        partitionSpec)
+        ident.asTableIdentifier,
+        Seq(partitionSpec).asUnresolvedPartitionSpecs.map(_.spec).headOption)
 
-    case ShowPartitionsStatement(tbl, partitionSpec) =>
-      val v1TableName = parseV1Table(tbl, "SHOW PARTITIONS")
+    case s @ ShowPartitions(
+        ResolvedV1TableOrViewIdentifier(ident),
+        pattern @ (None | Some(UnresolvedPartitionSpec(_, _))), output) =>
       ShowPartitionsCommand(
-        v1TableName.asTableIdentifier,
-        partitionSpec)
+        ident.asTableIdentifier,
+        output,
+        pattern.map(_.asInstanceOf[UnresolvedPartitionSpec].spec))
 
-    case ShowColumnsStatement(tbl, ns) =>
-      if (ns.isDefined && ns.get.length > 1) {
-        throw new AnalysisException(
-          s"Namespace name should have only one part if specified: ${ns.get.quoted}")
-      }
-      // Use namespace only if table name doesn't specify it. If namespace is already specified
-      // in the table name, it's checked against the given namespace below.
-      val nameParts = if (ns.isDefined && tbl.length == 1) {
-        ns.get ++ tbl
-      } else {
-        tbl
-      }
-      val sql = "SHOW COLUMNS"
-      val v1TableName = parseTempViewOrV1Table(nameParts, sql).asTableIdentifier
+    case s @ ShowColumns(ResolvedV1TableOrViewIdentifier(ident), ns, output) =>
+      val v1TableName = ident.asTableIdentifier
       val resolver = conf.resolver
       val db = ns match {
         case Some(db) if v1TableName.database.exists(!resolver(_, db.head)) =>
-          throw new AnalysisException(
-            s"SHOW COLUMNS with conflicting databases: " +
-              s"'${db.head}' != '${v1TableName.database.get}'")
+          throw QueryCompilationErrors.showColumnsWithConflictDatabasesError(db, v1TableName)
         case _ => ns.map(_.head)
       }
-      ShowColumnsCommand(db, v1TableName)
+      ShowColumnsCommand(db, v1TableName, output)
 
-    case AlterTableRecoverPartitionsStatement(tbl) =>
-      val v1TableName = parseV1Table(tbl, "ALTER TABLE RECOVER PARTITIONS")
-      AlterTableRecoverPartitionsCommand(
-        v1TableName.asTableIdentifier,
+    case RecoverPartitions(ResolvedV1TableIdentifier(ident)) =>
+      RepairTableCommand(
+        ident.asTableIdentifier,
+        enableAddPartitions = true,
+        enableDropPartitions = false,
         "ALTER TABLE RECOVER PARTITIONS")
 
-    case AlterTableAddPartitionStatement(tbl, partitionSpecsAndLocs, ifNotExists) =>
-      val v1TableName = parseV1Table(tbl, "ALTER TABLE ADD PARTITION")
+    case AddPartitions(ResolvedV1TableIdentifier(ident), partSpecsAndLocs, ifNotExists) =>
       AlterTableAddPartitionCommand(
-        v1TableName.asTableIdentifier,
-        partitionSpecsAndLocs,
+        ident.asTableIdentifier,
+        partSpecsAndLocs.asUnresolvedPartitionSpecs.map(spec => (spec.spec, spec.location)),
         ifNotExists)
 
-    case AlterTableRenamePartitionStatement(tbl, from, to) =>
-      val v1TableName = parseV1Table(tbl, "ALTER TABLE RENAME PARTITION")
-      AlterTableRenamePartitionCommand(
-        v1TableName.asTableIdentifier,
-        from,
-        to)
+    case RenamePartitions(
+        ResolvedV1TableIdentifier(ident),
+        UnresolvedPartitionSpec(from, _),
+        UnresolvedPartitionSpec(to, _)) =>
+      AlterTableRenamePartitionCommand(ident.asTableIdentifier, from, to)
 
-    case AlterTableDropPartitionStatement(tbl, specs, ifExists, purge, retainData) =>
-      val v1TableName = parseV1Table(tbl, "ALTER TABLE DROP PARTITION")
+    case DropPartitions(
+        ResolvedV1TableIdentifier(ident), specs, ifExists, purge) =>
       AlterTableDropPartitionCommand(
-        v1TableName.asTableIdentifier,
-        specs,
+        ident.asTableIdentifier,
+        specs.asUnresolvedPartitionSpecs.map(_.spec),
         ifExists,
         purge,
-        retainData)
+        retainData = false)
 
-    case AlterTableSerDePropertiesStatement(tbl, serdeClassName, serdeProperties, partitionSpec) =>
-      val v1TableName = parseV1Table(tbl, "ALTER TABLE SerDe Properties")
+    case SetTableSerDeProperties(
+        ResolvedV1TableIdentifier(ident),
+        serdeClassName,
+        serdeProperties,
+        partitionSpec) =>
       AlterTableSerDePropertiesCommand(
-        v1TableName.asTableIdentifier,
+        ident.asTableIdentifier,
         serdeClassName,
         serdeProperties,
         partitionSpec)
 
-    case AlterViewAsStatement(name, originalText, query) =>
-      val viewName = parseTempViewOrV1Table(name, "ALTER VIEW QUERY")
+    case SetTableLocation(ResolvedV1TableIdentifier(ident), partitionSpec, location) =>
+      AlterTableSetLocationCommand(ident.asTableIdentifier, partitionSpec, location)
+
+    case AlterViewAs(ResolvedView(ident, _), originalText, query) =>
       AlterViewAsCommand(
-        viewName.asTableIdentifier,
+        ident.asTableIdentifier,
         originalText,
         query)
 
@@ -535,63 +491,54 @@ class ResolveSessionCatalog(
         parseV1Table(tbl, "CREATE VIEW")
       }
       CreateViewCommand(
-        v1TableName.asTableIdentifier,
-        userSpecifiedColumns,
-        comment,
-        properties,
-        originalText,
-        child,
-        allowExisting,
-        replace,
-        viewType)
-
-    case ShowViews(resolved: ResolvedNamespace, pattern) =>
+        name = v1TableName.asTableIdentifier,
+        userSpecifiedColumns = userSpecifiedColumns,
+        comment = comment,
+        properties = properties,
+        originalText = originalText,
+        plan = child,
+        allowExisting = allowExisting,
+        replace = replace,
+        viewType = viewType)
+
+    case ShowViews(resolved: ResolvedNamespace, pattern, output) =>
       resolved match {
-        case SessionCatalogAndNamespace(_, ns) =>
-          // Fallback to v1 ShowViewsCommand since there is no view API in v2 catalog
-          assert(ns.nonEmpty)
-          if (ns.length != 1) {
-            throw new AnalysisException(s"The database name is not valid: ${ns.quoted}")
-          }
-          ShowViewsCommand(ns.head, pattern)
+        case DatabaseInSessionCatalog(db) => ShowViewsCommand(db, pattern, output)
         case _ =>
-          throw new AnalysisException(s"Catalog ${resolved.catalog.name} doesn't support " +
-            "SHOW VIEWS, only SessionCatalog supports this command.")
+          throw QueryCompilationErrors.externalCatalogNotSupportShowViewsError(resolved)
       }
 
-    case ShowTableProperties(
-        r @ ResolvedTable(_, _, _: V1Table), propertyKey) if isSessionCatalog(r.catalog) =>
-      ShowTablePropertiesCommand(r.identifier.asTableIdentifier, propertyKey)
-
-    case ShowTableProperties(r: ResolvedView, propertyKey) =>
-      ShowTablePropertiesCommand(r.identifier.asTableIdentifier, propertyKey)
-
-    case DescribeFunctionStatement(nameParts, extended) =>
-      val functionIdent =
-        parseSessionCatalogFunctionIdentifier(nameParts, "DESCRIBE FUNCTION")
-      DescribeFunctionCommand(functionIdent, extended)
-
-    case ShowFunctionsStatement(userScope, systemScope, pattern, fun) =>
-      val (database, function) = fun match {
-        case Some(nameParts) =>
-          val FunctionIdentifier(fn, db) =
-            parseSessionCatalogFunctionIdentifier(nameParts, "SHOW FUNCTIONS")
-          (db, Some(fn))
-        case None => (None, pattern)
-      }
-      ShowFunctionsCommand(database, function, userScope, systemScope)
+    case s @ ShowTableProperties(ResolvedV1TableOrViewIdentifier(ident), propertyKey, output) =>
+      val newOutput =
+        if (conf.getConf(SQLConf.LEGACY_KEEP_COMMAND_OUTPUT_SCHEMA) && propertyKey.isDefined) {
+          assert(output.length == 2)
+          output.tail
+        } else {
+          output
+        }
+      ShowTablePropertiesCommand(ident.asTableIdentifier, propertyKey, newOutput)
+
+    case DescribeFunction(ResolvedFunc(identifier), extended) =>
+      DescribeFunctionCommand(identifier.asFunctionIdentifier, extended)
+
+    case ShowFunctions(None, userScope, systemScope, pattern, output) =>
+      ShowFunctionsCommand(None, pattern, userScope, systemScope, output)
 
-    case DropFunctionStatement(nameParts, ifExists, isTemp) =>
-      val FunctionIdentifier(function, database) =
-        parseSessionCatalogFunctionIdentifier(nameParts, "DROP FUNCTION")
-      DropFunctionCommand(database, function, ifExists, isTemp)
+    case ShowFunctions(Some(ResolvedFunc(identifier)), userScope, systemScope, _, output) =>
+      val funcIdentifier = identifier.asFunctionIdentifier
+      ShowFunctionsCommand(
+        funcIdentifier.database, Some(funcIdentifier.funcName), userScope, systemScope, output)
+
+    case DropFunction(ResolvedFunc(identifier), ifExists, isTemp) =>
+      val funcIdentifier = identifier.asFunctionIdentifier
+      DropFunctionCommand(funcIdentifier.database, funcIdentifier.funcName, ifExists, isTemp)
 
     case CreateFunctionStatement(nameParts,
       className, resources, isTemp, ignoreIfExists, replace) =>
       if (isTemp) {
         // temp func doesn't belong to any catalog and we shouldn't resolve catalog in the name.
         val database = if (nameParts.length > 2) {
-          throw new AnalysisException(s"Unsupported function name '${nameParts.quoted}'")
+          throw QueryCompilationErrors.unsupportedFunctionNameError(nameParts.quoted)
         } else if (nameParts.length == 2) {
           Some(nameParts.head)
         } else {
@@ -607,48 +554,82 @@ class ResolveSessionCatalog(
           replace)
       } else {
         val FunctionIdentifier(function, database) =
-          parseSessionCatalogFunctionIdentifier(nameParts, "CREATE FUNCTION")
+          parseSessionCatalogFunctionIdentifier(nameParts)
         CreateFunctionCommand(database, function, className, resources, isTemp, ignoreIfExists,
           replace)
       }
-  }
-
-  // TODO: move function related v2 statements to the new framework.
-  private def parseSessionCatalogFunctionIdentifier(
-      nameParts: Seq[String],
-      sql: String): FunctionIdentifier = {
-    if (nameParts.length == 1 && isTempFunction(nameParts.head)) {
-      return FunctionIdentifier(nameParts.head)
-    }
-
-    nameParts match {
-      case SessionCatalogAndIdentifier(_, ident) =>
-        if (nameParts.length == 1) {
-          // If there is only one name part, it means the current catalog is the session catalog.
-          // Here we don't fill the default database, to keep the error message unchanged for
-          // v1 commands.
-          FunctionIdentifier(nameParts.head, None)
-        } else {
-          ident.namespace match {
-            case Array(db) => FunctionIdentifier(ident.name, Some(db))
-            case _ =>
-              throw new AnalysisException(s"Unsupported function name '$ident'")
-          }
-        }
 
-      case _ => throw new AnalysisException(s"$sql is only supported in v1 catalog")
-    }
+    case RefreshFunction(ResolvedFunc(identifier)) =>
+      // Fallback to v1 command
+      val funcIdentifier = identifier.asFunctionIdentifier
+      RefreshFunctionCommand(funcIdentifier.database, funcIdentifier.funcName)
   }
 
   private def parseV1Table(tableName: Seq[String], sql: String): Seq[String] = tableName match {
     case SessionCatalogAndTable(_, tbl) => tbl
-    case _ => throw new AnalysisException(s"$sql is only supported with v1 tables.")
+    case _ => throw QueryCompilationErrors.sqlOnlySupportedWithV1TablesError(sql)
   }
 
-  private def parseTempViewOrV1Table(
-      nameParts: Seq[String], sql: String): Seq[String] = nameParts match {
-    case TempViewOrV1Table(name) => name
-    case _ => throw new AnalysisException(s"$sql is only supported with temp views or v1 tables.")
+  private def getStorageFormatAndProvider(
+      provider: Option[String],
+      options: Map[String, String],
+      location: Option[String],
+      maybeSerdeInfo: Option[SerdeInfo],
+      ctas: Boolean): (CatalogStorageFormat, String) = {
+    val nonHiveStorageFormat = CatalogStorageFormat.empty.copy(
+      locationUri = location.map(CatalogUtils.stringToURI),
+      properties = options)
+    val defaultHiveStorage = HiveSerDe.getDefaultStorage(conf).copy(
+      locationUri = location.map(CatalogUtils.stringToURI),
+      properties = options)
+
+    if (provider.isDefined) {
+      // The parser guarantees that USING and STORED AS/ROW FORMAT won't co-exist.
+      if (maybeSerdeInfo.isDefined) {
+        throw QueryCompilationErrors.cannotCreateTableWithBothProviderAndSerdeError(
+          provider, maybeSerdeInfo)
+      }
+      (nonHiveStorageFormat, provider.get)
+    } else if (maybeSerdeInfo.isDefined) {
+      val serdeInfo = maybeSerdeInfo.get
+      SerdeInfo.checkSerdePropMerging(serdeInfo.serdeProperties, defaultHiveStorage.properties)
+      val storageFormat = if (serdeInfo.storedAs.isDefined) {
+        // If `STORED AS fileFormat` is used, infer inputFormat, outputFormat and serde from it.
+        HiveSerDe.sourceToSerDe(serdeInfo.storedAs.get) match {
+          case Some(hiveSerde) =>
+            defaultHiveStorage.copy(
+              inputFormat = hiveSerde.inputFormat.orElse(defaultHiveStorage.inputFormat),
+              outputFormat = hiveSerde.outputFormat.orElse(defaultHiveStorage.outputFormat),
+              // User specified serde takes precedence over the one inferred from file format.
+              serde = serdeInfo.serde.orElse(hiveSerde.serde).orElse(defaultHiveStorage.serde),
+              properties = serdeInfo.serdeProperties ++ defaultHiveStorage.properties)
+          case _ => throw QueryCompilationErrors.invalidFileFormatForStoredAsError(serdeInfo)
+        }
+      } else {
+        defaultHiveStorage.copy(
+          inputFormat =
+            serdeInfo.formatClasses.map(_.input).orElse(defaultHiveStorage.inputFormat),
+          outputFormat =
+            serdeInfo.formatClasses.map(_.output).orElse(defaultHiveStorage.outputFormat),
+          serde = serdeInfo.serde.orElse(defaultHiveStorage.serde),
+          properties = serdeInfo.serdeProperties ++ defaultHiveStorage.properties)
+      }
+      (storageFormat, DDLUtils.HIVE_PROVIDER)
+    } else {
+      // If neither USING nor STORED AS/ROW FORMAT is specified, we create native data source
+      // tables if:
+      //   1. `LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT` is false, or
+      //   2. It's a CTAS and `conf.convertCTAS` is true.
+      val createHiveTableByDefault = conf.getConf(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT)
+      if (!createHiveTableByDefault || (ctas && conf.convertCTAS)) {
+        (nonHiveStorageFormat, conf.defaultDataSourceName)
+      } else {
+        logWarning("A Hive serde table will be created as there is no table provider " +
+          s"specified. You can set ${SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT.key} to false " +
+          "so that native data source table will be created instead.")
+        (defaultHiveStorage, DDLUtils.HIVE_PROVIDER)
+      }
+    }
   }
 
   private def buildCatalogTable(
@@ -658,15 +639,11 @@ class ResolveSessionCatalog(
       bucketSpec: Option[BucketSpec],
       properties: Map[String, String],
       provider: String,
-      options: Map[String, String],
       location: Option[String],
       comment: Option[String],
-      ifNotExists: Boolean): CatalogTable = {
-    val storage = CatalogStorageFormat.empty.copy(
-      locationUri = location.map(CatalogUtils.stringToURI),
-      properties = options)
-
-    val tableType = if (location.isDefined) {
+      storageFormat: CatalogStorageFormat,
+      external: Boolean): CatalogTable = {
+    val tableType = if (external || location.isDefined) {
       CatalogTableType.EXTERNAL
     } else {
       CatalogTableType.MANAGED
@@ -675,7 +652,7 @@ class ResolveSessionCatalog(
     CatalogTable(
       identifier = table,
       tableType = tableType,
-      storage = storage,
+      storage = storageFormat,
       schema = schema,
       provider = Some(provider),
       partitionColumnNames = partitioning.asPartitionColumns,
@@ -692,46 +669,44 @@ class ResolveSessionCatalog(
     }
   }
 
-  object TempViewOrV1Table {
-    def unapply(nameParts: Seq[String]): Option[Seq[String]] = nameParts match {
-      case _ if isTempView(nameParts) => Some(nameParts)
-      case SessionCatalogAndIdentifier(_, tbl) => Some(tbl.asMultipartIdentifier)
+  object ResolvedViewIdentifier {
+    def unapply(resolved: LogicalPlan): Option[Identifier] = resolved match {
+      case ResolvedView(ident, _) => Some(ident)
       case _ => None
     }
   }
 
-  object SessionCatalogAndNamespace {
-    def unapply(resolved: ResolvedNamespace): Option[(CatalogPlugin, Seq[String])] =
-      if (isSessionCatalog(resolved.catalog)) {
-        Some(resolved.catalog -> resolved.namespace)
-      } else {
-        None
-      }
+  object ResolvedV1TableIdentifier {
+    def unapply(resolved: LogicalPlan): Option[Identifier] = resolved match {
+      case ResolvedTable(catalog, ident, _: V1Table, _) if isSessionCatalog(catalog) => Some(ident)
+      case _ => None
+    }
+  }
+
+  object ResolvedV1TableOrViewIdentifier {
+    def unapply(resolved: LogicalPlan): Option[Identifier] = resolved match {
+      case ResolvedV1TableIdentifier(ident) => Some(ident)
+      case ResolvedViewIdentifier(ident) => Some(ident)
+      case _ => None
+    }
   }
 
   private def assertTopLevelColumn(colName: Seq[String], command: String): Unit = {
     if (colName.length > 1) {
-      throw new AnalysisException(s"$command does not support nested column: ${colName.quoted}")
+      throw QueryCompilationErrors.commandNotSupportNestedColumnError(command, colName.quoted)
     }
   }
 
   private def convertToStructField(col: QualifiedColType): StructField = {
     val builder = new MetadataBuilder
     col.comment.foreach(builder.putString("comment", _))
-
-    val cleanedDataType = HiveStringType.replaceCharType(col.dataType)
-    if (col.dataType != cleanedDataType) {
-      builder.putString(HIVE_TYPE_STRING, col.dataType.catalogString)
-    }
-
-    StructField(
-      col.name.head,
-      cleanedDataType,
-      nullable = true,
-      builder.build())
+    StructField(col.name.head, col.dataType, nullable = true, builder.build())
   }
 
   private def isV2Provider(provider: String): Boolean = {
+    // Return earlier since `lookupDataSourceV2` may fail to resolve provider "hive" to
+    // `HiveFileFormat`, when running tests in sql/core.
+    if (DDLUtils.isHiveTable(Some(provider))) return false
     DataSource.lookupDataSourceV2(provider, conf) match {
       // TODO(SPARK-28396): Currently file source v2 can't work with tables.
       case Some(_: FileDataSourceV2) => false
@@ -739,4 +714,17 @@ class ResolveSessionCatalog(
       case _ => false
     }
   }
+
+  private object DatabaseInSessionCatalog {
+    def unapply(resolved: ResolvedNamespace): Option[String] = resolved match {
+      case ResolvedNamespace(catalog, _) if !isSessionCatalog(catalog) => None
+      case ResolvedNamespace(_, Seq()) =>
+        throw QueryCompilationErrors.databaseFromV1SessionCatalogNotSpecifiedError()
+      case ResolvedNamespace(_, Seq(dbName)) => Some(dbName)
+      case _ =>
+        assert(resolved.namespace.length > 1)
+        throw QueryCompilationErrors.nestedDatabaseUnsupportedByV1SessionCatalogError(
+          resolved.namespace.map(quoteIfNeeded).mkString("."))
+    }
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/columnar/CachedBatchSerializer.scala b/sql/core/src/main/scala/org/apache/spark/sql/columnar/CachedBatchSerializer.scala
new file mode 100644
index 0000000000000..1113e63cab332
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/columnar/CachedBatchSerializer.scala
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.columnar
+
+import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.internal.Logging
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.expressions.{And, Attribute, AttributeReference, BindReferences, EqualNullSafe, EqualTo, Expression, GreaterThan, GreaterThanOrEqual, In, IsNotNull, IsNull, Length, LessThan, LessThanOrEqual, Literal, Or, Predicate, StartsWith}
+import org.apache.spark.sql.execution.columnar.{ColumnStatisticsSchema, PartitionStatistics}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types.{AtomicType, BinaryType, StructType}
+import org.apache.spark.sql.vectorized.ColumnarBatch
+import org.apache.spark.storage.StorageLevel
+
+/**
+ * Basic interface that all cached batches of data must support. This is primarily to allow
+ * for metrics to be handled outside of the encoding and decoding steps in a standard way.
+ */
+@DeveloperApi
+@Since("3.1.0")
+trait CachedBatch {
+  def numRows: Int
+  def sizeInBytes: Long
+}
+
+/**
+ * Provides APIs that handle transformations of SQL data associated with the cache/persist APIs.
+ */
+@DeveloperApi
+@Since("3.1.0")
+trait CachedBatchSerializer extends Serializable {
+  /**
+   * Can `convertColumnarBatchToCachedBatch()` be called instead of
+   * `convertInternalRowToCachedBatch()` for this given schema? True if it can and false if it
+   * cannot. Columnar input is only supported if the plan could produce columnar output. Currently
+   * this is mostly supported by input formats like parquet and orc, but more operations are likely
+   * to be supported soon.
+   * @param schema the schema of the data being stored.
+   * @return True if columnar input can be supported, else false.
+   */
+  def supportsColumnarInput(schema: Seq[Attribute]): Boolean
+
+  /**
+   * Convert an `RDD[InternalRow]` into an `RDD[CachedBatch]` in preparation for caching the data.
+   * @param input the input `RDD` to be converted.
+   * @param schema the schema of the data being stored.
+   * @param storageLevel where the data will be stored.
+   * @param conf the config for the query.
+   * @return The data converted into a format more suitable for caching.
+   */
+  def convertInternalRowToCachedBatch(
+      input: RDD[InternalRow],
+      schema: Seq[Attribute],
+      storageLevel: StorageLevel,
+      conf: SQLConf): RDD[CachedBatch]
+
+  /**
+   * Convert an `RDD[ColumnarBatch]` into an `RDD[CachedBatch]` in preparation for caching the data.
+   * This will only be called if `supportsColumnarInput()` returned true for the given schema and
+   * the plan up to this point would could produce columnar output without modifying it.
+   * @param input the input `RDD` to be converted.
+   * @param schema the schema of the data being stored.
+   * @param storageLevel where the data will be stored.
+   * @param conf the config for the query.
+   * @return The data converted into a format more suitable for caching.
+   */
+  def convertColumnarBatchToCachedBatch(
+      input: RDD[ColumnarBatch],
+      schema: Seq[Attribute],
+      storageLevel: StorageLevel,
+      conf: SQLConf): RDD[CachedBatch]
+
+  /**
+   * Builds a function that can be used to filter batches prior to being decompressed.
+   * In most cases extending [[SimpleMetricsCachedBatchSerializer]] will provide the filter logic
+   * necessary. You will need to provide metrics for this to work. [[SimpleMetricsCachedBatch]]
+   * provides the APIs to hold those metrics and explains the metrics used, really just min and max.
+   * Note that this is intended to skip batches that are not needed, and the actual filtering of
+   * individual rows is handled later.
+   * @param predicates the set of expressions to use for filtering.
+   * @param cachedAttributes the schema/attributes of the data that is cached. This can be helpful
+   *                         if you don't store it with the data.
+   * @return a function that takes the partition id and the iterator of batches in the partition.
+   *         It returns an iterator of batches that should be decompressed.
+   */
+  def buildFilter(
+      predicates: Seq[Expression],
+      cachedAttributes: Seq[Attribute]): (Int, Iterator[CachedBatch]) => Iterator[CachedBatch]
+
+  /**
+   * Can `convertCachedBatchToColumnarBatch()` be called instead of
+   * `convertCachedBatchToInternalRow()` for this given schema? True if it can and false if it
+   * cannot. Columnar output is typically preferred because it is more efficient. Note that
+   * `convertCachedBatchToInternalRow()` must always be supported as there are other checks that
+   * can force row based output.
+   * @param schema the schema of the data being checked.
+   * @return true if columnar output should be used for this schema, else false.
+   */
+  def supportsColumnarOutput(schema: StructType): Boolean
+
+  /**
+   * The exact java types of the columns that are output in columnar processing mode. This
+   * is a performance optimization for code generation and is optional.
+   * @param attributes the attributes to be output.
+   * @param conf the config for the query that will read the data.
+   */
+  def vectorTypes(attributes: Seq[Attribute], conf: SQLConf): Option[Seq[String]] = None
+
+  /**
+   * Convert the cached data into a ColumnarBatch. This currently is only used if
+   * `supportsColumnarOutput()` returns true for the associated schema, but there are other checks
+   * that can force row based output. One of the main advantages of doing columnar output over row
+   * based output is that the code generation is more standard and can be combined with code
+   * generation for downstream operations.
+   * @param input the cached batches that should be converted.
+   * @param cacheAttributes the attributes of the data in the batch.
+   * @param selectedAttributes the fields that should be loaded from the data and the order they
+   *                           should appear in the output batch.
+   * @param conf the configuration for the job.
+   * @return an RDD of the input cached batches transformed into the ColumnarBatch format.
+   */
+  def convertCachedBatchToColumnarBatch(
+      input: RDD[CachedBatch],
+      cacheAttributes: Seq[Attribute],
+      selectedAttributes: Seq[Attribute],
+      conf: SQLConf): RDD[ColumnarBatch]
+
+  /**
+   * Convert the cached batch into `InternalRow`s. If you want this to be performant, code
+   * generation is advised.
+   * @param input the cached batches that should be converted.
+   * @param cacheAttributes the attributes of the data in the batch.
+   * @param selectedAttributes the field that should be loaded from the data and the order they
+   *                           should appear in the output rows.
+   * @param conf the configuration for the job.
+   * @return RDD of the rows that were stored in the cached batches.
+   */
+  def convertCachedBatchToInternalRow(
+      input: RDD[CachedBatch],
+      cacheAttributes: Seq[Attribute],
+      selectedAttributes: Seq[Attribute],
+      conf: SQLConf): RDD[InternalRow]
+}
+
+/**
+ * A [[CachedBatch]] that stores some simple metrics that can be used for filtering of batches with
+ * the [[SimpleMetricsCachedBatchSerializer]].
+ * The metrics are returned by the stats value. For each column in the batch 5 columns of metadata
+ * are needed in the row.
+ */
+@DeveloperApi
+@Since("3.1.0")
+trait SimpleMetricsCachedBatch extends CachedBatch {
+  /**
+   * Holds stats for each cached column. The optional `upperBound` and `lowerBound` should be
+   * of the same type as the original column. If they are null, then it is assumed that they
+   * are not provided, and will not be used for filtering.
+   * <ul>
+   *   <li>`upperBound` (optional)</li>
+   *   <li>`lowerBound` (Optional)</li>
+   *   <li>`nullCount`: `Int`</li>
+   *   <li>`rowCount`: `Int`</li>
+   *   <li>`sizeInBytes`: `Long`</li>
+   * </ul>
+   * These are repeated for each column in the original cached data.
+   */
+  val stats: InternalRow
+  override def sizeInBytes: Long =
+    Range.apply(4, stats.numFields, 5).map(stats.getLong).sum
+}
+
+// Currently, uses statistics for all atomic types that are not `BinaryType`.
+private object ExtractableLiteral {
+  def unapply(expr: Expression): Option[Literal] = expr match {
+    case lit: Literal => lit.dataType match {
+      case BinaryType => None
+      case _: AtomicType => Some(lit)
+      case _ => None
+    }
+    case _ => None
+  }
+}
+
+/**
+ * Provides basic filtering for [[CachedBatchSerializer]] implementations.
+ * The requirement to extend this is that all of the batches produced by your serializer are
+ * instances of [[SimpleMetricsCachedBatch]].
+ * This does not calculate the metrics needed to be stored in the batches. That is up to each
+ * implementation. The metrics required are really just min and max values and those are optional
+ * especially for complex types. Because those metrics are simple and it is likely that compression
+ * will also be done on the data we thought it best to let each implementation decide on the most
+ * efficient way to calculate the metrics, possibly combining them with compression passes that
+ * might also be done across the data.
+ */
+@DeveloperApi
+@Since("3.1.0")
+abstract class SimpleMetricsCachedBatchSerializer extends CachedBatchSerializer with Logging {
+  override def buildFilter(
+      predicates: Seq[Expression],
+      cachedAttributes: Seq[Attribute]): (Int, Iterator[CachedBatch]) => Iterator[CachedBatch] = {
+    val stats = new PartitionStatistics(cachedAttributes)
+    val statsSchema = stats.schema
+
+    def statsFor(a: Attribute): ColumnStatisticsSchema = {
+      stats.forAttribute(a)
+    }
+
+    // Returned filter predicate should return false iff it is impossible for the input expression
+    // to evaluate to `true` based on statistics collected about this partition batch.
+    @transient lazy val buildFilter: PartialFunction[Expression, Expression] = {
+      case And(lhs: Expression, rhs: Expression)
+        if buildFilter.isDefinedAt(lhs) || buildFilter.isDefinedAt(rhs) =>
+        (buildFilter.lift(lhs) ++ buildFilter.lift(rhs)).reduce(_ && _)
+
+      case Or(lhs: Expression, rhs: Expression)
+        if buildFilter.isDefinedAt(lhs) && buildFilter.isDefinedAt(rhs) =>
+        buildFilter(lhs) || buildFilter(rhs)
+
+      case EqualTo(a: AttributeReference, ExtractableLiteral(l)) =>
+        statsFor(a).lowerBound <= l && l <= statsFor(a).upperBound
+      case EqualTo(ExtractableLiteral(l), a: AttributeReference) =>
+        statsFor(a).lowerBound <= l && l <= statsFor(a).upperBound
+
+      case EqualNullSafe(a: AttributeReference, ExtractableLiteral(l)) =>
+        statsFor(a).lowerBound <= l && l <= statsFor(a).upperBound
+      case EqualNullSafe(ExtractableLiteral(l), a: AttributeReference) =>
+        statsFor(a).lowerBound <= l && l <= statsFor(a).upperBound
+
+      case LessThan(a: AttributeReference, ExtractableLiteral(l)) => statsFor(a).lowerBound < l
+      case LessThan(ExtractableLiteral(l), a: AttributeReference) => l < statsFor(a).upperBound
+
+      case LessThanOrEqual(a: AttributeReference, ExtractableLiteral(l)) =>
+        statsFor(a).lowerBound <= l
+      case LessThanOrEqual(ExtractableLiteral(l), a: AttributeReference) =>
+        l <= statsFor(a).upperBound
+
+      case GreaterThan(a: AttributeReference, ExtractableLiteral(l)) => l < statsFor(a).upperBound
+      case GreaterThan(ExtractableLiteral(l), a: AttributeReference) => statsFor(a).lowerBound < l
+
+      case GreaterThanOrEqual(a: AttributeReference, ExtractableLiteral(l)) =>
+        l <= statsFor(a).upperBound
+      case GreaterThanOrEqual(ExtractableLiteral(l), a: AttributeReference) =>
+        statsFor(a).lowerBound <= l
+
+      case IsNull(a: Attribute) => statsFor(a).nullCount > 0
+      case IsNotNull(a: Attribute) => statsFor(a).count - statsFor(a).nullCount > 0
+
+      case In(a: AttributeReference, list: Seq[Expression])
+        if list.forall(ExtractableLiteral.unapply(_).isDefined) && list.nonEmpty =>
+        list.map(l => statsFor(a).lowerBound <= l.asInstanceOf[Literal] &&
+            l.asInstanceOf[Literal] <= statsFor(a).upperBound).reduce(_ || _)
+      // This is an example to explain how it works, imagine that the id column stored as follows:
+      // __________________________________________
+      // | Partition ID | lowerBound | upperBound |
+      // |--------------|------------|------------|
+      // |      p1      |    '1'     |    '9'     |
+      // |      p2      |    '10'    |    '19'    |
+      // |      p3      |    '20'    |    '29'    |
+      // |      p4      |    '30'    |    '39'    |
+      // |      p5      |    '40'    |    '49'    |
+      // |______________|____________|____________|
+      //
+      // A filter: df.filter($"id".startsWith("2")).
+      // In this case it substr lowerBound and upperBound:
+      // ________________________________________________________________________________________
+      // | Partition ID | lowerBound.substr(0, Length("2")) | upperBound.substr(0, Length("2")) |
+      // |--------------|-----------------------------------|-----------------------------------|
+      // |      p1      |    '1'                            |    '9'                            |
+      // |      p2      |    '1'                            |    '1'                            |
+      // |      p3      |    '2'                            |    '2'                            |
+      // |      p4      |    '3'                            |    '3'                            |
+      // |      p5      |    '4'                            |    '4'                            |
+      // |______________|___________________________________|___________________________________|
+      //
+      // We can see that we only need to read p1 and p3.
+      case StartsWith(a: AttributeReference, ExtractableLiteral(l)) =>
+        statsFor(a).lowerBound.substr(0, Length(l)) <= l &&
+            l <= statsFor(a).upperBound.substr(0, Length(l))
+    }
+
+    // When we bind the filters we need to do it against the stats schema
+    val partitionFilters: Seq[Expression] = {
+      predicates.flatMap { p =>
+        val filter = buildFilter.lift(p)
+        val boundFilter =
+          filter.map(
+            BindReferences.bindReference(
+              _,
+              statsSchema,
+              allowFailures = true))
+
+        boundFilter.foreach(_ =>
+          filter.foreach(f => logInfo(s"Predicate $p generates partition filter: $f")))
+
+        // If the filter can't be resolved then we are missing required statistics.
+        boundFilter.filter(_.resolved)
+      }
+    }
+
+    def ret(index: Int, cachedBatchIterator: Iterator[CachedBatch]): Iterator[CachedBatch] = {
+      val partitionFilter = Predicate.create(
+        partitionFilters.reduceOption(And).getOrElse(Literal(true)),
+        cachedAttributes)
+
+      partitionFilter.initialize(index)
+      val schemaIndex = cachedAttributes.zipWithIndex
+
+      cachedBatchIterator.filter { cb =>
+        val cachedBatch = cb.asInstanceOf[SimpleMetricsCachedBatch]
+        if (!partitionFilter.eval(cachedBatch.stats)) {
+          logDebug {
+            val statsString = schemaIndex.map { case (a, i) =>
+              val value = cachedBatch.stats.get(i, a.dataType)
+              s"${a.name}: $value"
+            }.mkString(", ")
+            s"Skipping partition based on stats $statsString"
+          }
+          false
+        } else {
+          true
+        }
+      }
+    }
+    ret
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/AggregatingAccumulator.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/AggregatingAccumulator.scala
index 9807b5dbe9348..94e159c562e31 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/AggregatingAccumulator.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/AggregatingAccumulator.scala
@@ -257,16 +257,16 @@ object AggregatingAccumulator {
         imperative
     })
 
-    val updateAttrSeq: AttributeSeq = aggBufferAttributes ++ inputAttributes
-    val mergeAttrSeq: AttributeSeq = aggBufferAttributes ++ inputAggBufferAttributes
-    val aggBufferAttributesSeq: AttributeSeq = aggBufferAttributes
+    val updateAttrSeq: AttributeSeq = (aggBufferAttributes ++ inputAttributes).toSeq
+    val mergeAttrSeq: AttributeSeq = (aggBufferAttributes ++ inputAggBufferAttributes).toSeq
+    val aggBufferAttributesSeq: AttributeSeq = aggBufferAttributes.toSeq
 
     // Create the accumulator.
     new AggregatingAccumulator(
-      aggBufferAttributes.map(_.dataType),
-      initialValues,
-      updateExpressions.map(BindReferences.bindReference(_, updateAttrSeq)),
-      mergeExpressions.map(BindReferences.bindReference(_, mergeAttrSeq)),
+      aggBufferAttributes.map(_.dataType).toSeq,
+      initialValues.toSeq,
+      updateExpressions.map(BindReferences.bindReference(_, updateAttrSeq)).toSeq,
+      mergeExpressions.map(BindReferences.bindReference(_, mergeAttrSeq)).toSeq,
       resultExpressions.map(BindReferences.bindReference(_, aggBufferAttributesSeq)),
       imperatives.toArray,
       typedImperatives.toArray,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/AliasAwareOutputExpression.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/AliasAwareOutputExpression.scala
index fa41e865444da..23a9527a1b349 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/AliasAwareOutputExpression.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/AliasAwareOutputExpression.scala
@@ -16,8 +16,8 @@
  */
 package org.apache.spark.sql.execution
 
-import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference, Expression, NamedExpression, SortOrder}
-import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partitioning}
+import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeMap, AttributeReference, Expression, NamedExpression, SortOrder}
+import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partitioning, PartitioningCollection, UnknownPartitioning}
 
 /**
  * A trait that provides functionality to handle aliases in the `outputExpressions`.
@@ -25,19 +25,15 @@ import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partition
 trait AliasAwareOutputExpression extends UnaryExecNode {
   protected def outputExpressions: Seq[NamedExpression]
 
-  protected def hasAlias: Boolean = outputExpressions.collectFirst { case _: Alias => }.isDefined
+  private lazy val aliasMap = AttributeMap(outputExpressions.collect {
+    case a @ Alias(child: AttributeReference, _) => (child, a.toAttribute)
+  })
 
-  protected def replaceAliases(exprs: Seq[Expression]): Seq[Expression] = {
-    exprs.map {
-      case a: AttributeReference => replaceAlias(a).getOrElse(a)
-      case other => other
-    }
-  }
+  protected def hasAlias: Boolean = aliasMap.nonEmpty
 
-  protected def replaceAlias(attr: AttributeReference): Option[Attribute] = {
-    outputExpressions.collectFirst {
-      case a @ Alias(child: AttributeReference, _) if child.semanticEquals(attr) =>
-        a.toAttribute
+  protected def normalizeExpression(exp: Expression): Expression = {
+    exp.transform {
+      case attr: AttributeReference => aliasMap.getOrElse(attr, attr)
     }
   }
 }
@@ -48,14 +44,33 @@ trait AliasAwareOutputExpression extends UnaryExecNode {
  */
 trait AliasAwareOutputPartitioning extends AliasAwareOutputExpression {
   final override def outputPartitioning: Partitioning = {
-    if (hasAlias) {
+    val normalizedOutputPartitioning = if (hasAlias) {
       child.outputPartitioning match {
-        case h: HashPartitioning => h.copy(expressions = replaceAliases(h.expressions))
+        case e: Expression =>
+          normalizeExpression(e).asInstanceOf[Partitioning]
         case other => other
       }
     } else {
       child.outputPartitioning
     }
+
+    flattenPartitioning(normalizedOutputPartitioning).filter {
+      case hashPartitioning: HashPartitioning => hashPartitioning.references.subsetOf(outputSet)
+      case _ => true
+    } match {
+      case Seq() => UnknownPartitioning(child.outputPartitioning.numPartitions)
+      case Seq(singlePartitioning) => singlePartitioning
+      case seqWithMultiplePartitionings => PartitioningCollection(seqWithMultiplePartitionings)
+    }
+  }
+
+  private def flattenPartitioning(partitioning: Partitioning): Seq[Partitioning] = {
+    partitioning match {
+      case PartitioningCollection(childPartitionings) =>
+        childPartitionings.flatMap(flattenPartitioning)
+      case rest =>
+        rest +: Nil
+    }
   }
 }
 
@@ -68,12 +83,7 @@ trait AliasAwareOutputOrdering extends AliasAwareOutputExpression {
 
   final override def outputOrdering: Seq[SortOrder] = {
     if (hasAlias) {
-      orderingExpressions.map { s =>
-        s.child match {
-          case a: AttributeReference => s.copy(child = replaceAlias(a).getOrElse(a))
-          case _ => s
-        }
-      }
+      orderingExpressions.map(normalizeExpression(_).asInstanceOf[SortOrder])
     } else {
       orderingExpressions
     }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
new file mode 100644
index 0000000000000..da5c6f0d95ca3
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
@@ -0,0 +1,381 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import java.io.{BufferedReader, File, InputStream, InputStreamReader, OutputStream}
+import java.nio.charset.StandardCharsets
+import java.util.concurrent.TimeUnit
+
+import scala.collection.JavaConverters._
+import scala.util.control.NonFatal
+
+import org.apache.hadoop.conf.Configuration
+
+import org.apache.spark.{SparkException, SparkFiles, TaskContext}
+import org.apache.spark.internal.Logging
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, Cast, Expression, GenericInternalRow, JsonToStructs, Literal, StructsToJson, UnsafeProjection}
+import org.apache.spark.sql.catalyst.plans.logical.ScriptInputOutputSchema
+import org.apache.spark.sql.catalyst.plans.physical.Partitioning
+import org.apache.spark.sql.catalyst.util.{DateTimeUtils, IntervalUtils}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.UTF8String
+import org.apache.spark.util.{CircularBuffer, RedirectThread, SerializableConfiguration, Utils}
+
+trait BaseScriptTransformationExec extends UnaryExecNode {
+  def script: String
+  def output: Seq[Attribute]
+  def child: SparkPlan
+  def ioschema: ScriptTransformationIOSchema
+
+  protected lazy val inputExpressionsWithoutSerde: Seq[Expression] = {
+    child.output.map { in =>
+      in.dataType match {
+        case _: ArrayType | _: MapType | _: StructType =>
+          new StructsToJson(ioschema.inputSerdeProps.toMap, in)
+            .withTimeZone(conf.sessionLocalTimeZone)
+        case _ => Cast(in, StringType).withTimeZone(conf.sessionLocalTimeZone)
+      }
+    }
+  }
+
+  override def producedAttributes: AttributeSet = outputSet -- inputSet
+
+  override def outputPartitioning: Partitioning = child.outputPartitioning
+
+  override def doExecute(): RDD[InternalRow] = {
+    val broadcastedHadoopConf =
+      new SerializableConfiguration(sqlContext.sessionState.newHadoopConf())
+
+    child.execute().mapPartitions { iter =>
+      if (iter.hasNext) {
+        val proj = UnsafeProjection.create(schema)
+        processIterator(iter, broadcastedHadoopConf.value).map(proj)
+      } else {
+        // If the input iterator has no rows then do not launch the external script.
+        Iterator.empty
+      }
+    }
+  }
+
+  protected def initProc: (OutputStream, Process, InputStream, CircularBuffer) = {
+    val cmd = List("/bin/bash", "-c", script)
+    val builder = new ProcessBuilder(cmd.asJava)
+      .directory(new File(SparkFiles.getRootDirectory()))
+    val path = System.getenv("PATH") + File.pathSeparator +
+      SparkFiles.getRootDirectory()
+    builder.environment().put("PATH", path)
+
+    val proc = builder.start()
+    val inputStream = proc.getInputStream
+    val outputStream = proc.getOutputStream
+    val errorStream = proc.getErrorStream
+
+    // In order to avoid deadlocks, we need to consume the error output of the child process.
+    // To avoid issues caused by large error output, we use a circular buffer to limit the amount
+    // of error output that we retain. See SPARK-7862 for more discussion of the deadlock / hang
+    // that motivates this.
+    val stderrBuffer = new CircularBuffer(2048)
+    new RedirectThread(
+      errorStream,
+      stderrBuffer,
+      s"Thread-${this.getClass.getSimpleName}-STDERR-Consumer").start()
+    (outputStream, proc, inputStream, stderrBuffer)
+  }
+
+  protected def processIterator(
+      inputIterator: Iterator[InternalRow],
+      hadoopConf: Configuration): Iterator[InternalRow]
+
+  protected def createOutputIteratorWithoutSerde(
+      writerThread: BaseScriptTransformationWriterThread,
+      inputStream: InputStream,
+      proc: Process,
+      stderrBuffer: CircularBuffer): Iterator[InternalRow] = {
+    new Iterator[InternalRow] {
+      var curLine: String = null
+      val reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))
+
+      val outputRowFormat = ioschema.outputRowFormatMap("TOK_TABLEROWFORMATFIELD")
+      val processRowWithoutSerde = if (!ioschema.schemaLess) {
+        prevLine: String =>
+          new GenericInternalRow(
+            prevLine.split(outputRowFormat).padTo(outputFieldWriters.size, null)
+              .zip(outputFieldWriters)
+              .map { case (data, writer) => writer(data) })
+      } else {
+        // In schema less mode, hive will choose first two output column as output.
+        // If output column size less then 2, it will return NULL for columns with missing values.
+        // Here we split row string and choose first 2 values, if values's size less then 2,
+        // we pad NULL value until 2 to make behavior same with hive.
+        val kvWriter = CatalystTypeConverters.createToCatalystConverter(StringType)
+        prevLine: String =>
+          new GenericInternalRow(
+            prevLine.split(outputRowFormat).slice(0, 2).padTo(2, null)
+              .map(kvWriter))
+      }
+
+      override def hasNext: Boolean = {
+        try {
+          if (curLine == null) {
+            curLine = reader.readLine()
+            if (curLine == null) {
+              checkFailureAndPropagate(writerThread, null, proc, stderrBuffer)
+              return false
+            }
+          }
+          true
+        } catch {
+          case NonFatal(e) =>
+            // If this exception is due to abrupt / unclean termination of `proc`,
+            // then detect it and propagate a better exception message for end users
+            checkFailureAndPropagate(writerThread, e, proc, stderrBuffer)
+
+            throw e
+        }
+      }
+
+      override def next(): InternalRow = {
+        if (!hasNext) {
+          throw new NoSuchElementException
+        }
+        val prevLine = curLine
+        curLine = reader.readLine()
+        processRowWithoutSerde(prevLine)
+      }
+    }
+  }
+
+  protected def checkFailureAndPropagate(
+      writerThread: BaseScriptTransformationWriterThread,
+      cause: Throwable = null,
+      proc: Process,
+      stderrBuffer: CircularBuffer): Unit = {
+    if (writerThread.exception.isDefined) {
+      throw writerThread.exception.get
+    }
+
+    // There can be a lag between reader read EOF and the process termination.
+    // If the script fails to startup, this kind of error may be missed.
+    // So explicitly waiting for the process termination.
+    val timeout = conf.getConf(SQLConf.SCRIPT_TRANSFORMATION_EXIT_TIMEOUT)
+    val exitRes = proc.waitFor(timeout, TimeUnit.SECONDS)
+    if (!exitRes) {
+      log.warn(s"Transformation script process exits timeout in $timeout seconds")
+    }
+
+    if (!proc.isAlive) {
+      val exitCode = proc.exitValue()
+      if (exitCode != 0) {
+        logError(stderrBuffer.toString) // log the stderr circular buffer
+        throw new SparkException(s"Subprocess exited with status $exitCode. " +
+          s"Error: ${stderrBuffer.toString}", cause)
+      }
+    }
+  }
+
+  private lazy val outputFieldWriters: Seq[String => Any] = output.map { attr =>
+    val converter = CatalystTypeConverters.createToCatalystConverter(attr.dataType)
+    attr.dataType match {
+      case StringType => wrapperConvertException(data => data, converter)
+      case BooleanType => wrapperConvertException(data => data.toBoolean, converter)
+      case ByteType => wrapperConvertException(data => data.toByte, converter)
+      case BinaryType =>
+        wrapperConvertException(data => UTF8String.fromString(data).getBytes, converter)
+      case IntegerType => wrapperConvertException(data => data.toInt, converter)
+      case ShortType => wrapperConvertException(data => data.toShort, converter)
+      case LongType => wrapperConvertException(data => data.toLong, converter)
+      case FloatType => wrapperConvertException(data => data.toFloat, converter)
+      case DoubleType => wrapperConvertException(data => data.toDouble, converter)
+      case _: DecimalType => wrapperConvertException(data => BigDecimal(data), converter)
+      case DateType if conf.datetimeJava8ApiEnabled =>
+        wrapperConvertException(data => DateTimeUtils.stringToDate(
+          UTF8String.fromString(data),
+          DateTimeUtils.getZoneId(conf.sessionLocalTimeZone))
+          .map(DateTimeUtils.daysToLocalDate).orNull, converter)
+      case DateType => wrapperConvertException(data => DateTimeUtils.stringToDate(
+        UTF8String.fromString(data),
+        DateTimeUtils.getZoneId(conf.sessionLocalTimeZone))
+        .map(DateTimeUtils.toJavaDate).orNull, converter)
+      case TimestampType if conf.datetimeJava8ApiEnabled =>
+        wrapperConvertException(data => DateTimeUtils.stringToTimestamp(
+          UTF8String.fromString(data),
+          DateTimeUtils.getZoneId(conf.sessionLocalTimeZone))
+          .map(DateTimeUtils.microsToInstant).orNull, converter)
+      case TimestampType => wrapperConvertException(data => DateTimeUtils.stringToTimestamp(
+        UTF8String.fromString(data),
+        DateTimeUtils.getZoneId(conf.sessionLocalTimeZone))
+        .map(DateTimeUtils.toJavaTimestamp).orNull, converter)
+      case CalendarIntervalType => wrapperConvertException(
+        data => IntervalUtils.stringToInterval(UTF8String.fromString(data)),
+        converter)
+      case _: ArrayType | _: MapType | _: StructType =>
+        val complexTypeFactory = JsonToStructs(attr.dataType,
+          ioschema.outputSerdeProps.toMap, Literal(null), Some(conf.sessionLocalTimeZone))
+        wrapperConvertException(data =>
+          complexTypeFactory.nullSafeEval(UTF8String.fromString(data)), any => any)
+      case udt: UserDefinedType[_] =>
+        wrapperConvertException(data => udt.deserialize(data), converter)
+      case dt =>
+        throw new SparkException(s"${nodeName} without serde does not support " +
+          s"${dt.getClass.getSimpleName} as output data type")
+    }
+  }
+
+  // Keep consistent with Hive `LazySimpleSerde`, when there is a type case error, return null
+  private val wrapperConvertException: (String => Any, Any => Any) => String => Any =
+    (f: String => Any, converter: Any => Any) =>
+      (data: String) => converter {
+        try {
+          f(data)
+        } catch {
+          case NonFatal(_) => null
+        }
+      }
+}
+
+abstract class BaseScriptTransformationWriterThread extends Thread with Logging {
+
+  def iter: Iterator[InternalRow]
+  def inputSchema: Seq[DataType]
+  def ioSchema: ScriptTransformationIOSchema
+  def outputStream: OutputStream
+  def proc: Process
+  def stderrBuffer: CircularBuffer
+  def taskContext: TaskContext
+  def conf: Configuration
+
+  setDaemon(true)
+
+  @volatile protected var _exception: Throwable = null
+
+  /** Contains the exception thrown while writing the parent iterator to the external process. */
+  def exception: Option[Throwable] = Option(_exception)
+
+  protected def processRows(): Unit
+
+  protected def processRowsWithoutSerde(): Unit = {
+    val len = inputSchema.length
+    iter.foreach { row =>
+      val data = if (len == 0) {
+        ioSchema.inputRowFormatMap("TOK_TABLEROWFORMATLINES")
+      } else {
+        val sb = new StringBuilder
+        sb.append(row.get(0, inputSchema(0)))
+        var i = 1
+        while (i < len) {
+          sb.append(ioSchema.inputRowFormatMap("TOK_TABLEROWFORMATFIELD"))
+          sb.append(row.get(i, inputSchema(i)))
+          i += 1
+        }
+        sb.append(ioSchema.inputRowFormatMap("TOK_TABLEROWFORMATLINES"))
+        sb.toString()
+      }
+      outputStream.write(data.getBytes(StandardCharsets.UTF_8))
+    }
+  }
+
+  override def run(): Unit = Utils.logUncaughtExceptions {
+    TaskContext.setTaskContext(taskContext)
+
+    // We can't use Utils.tryWithSafeFinally here because we also need a `catch` block, so
+    // let's use a variable to record whether the `finally` block was hit due to an exception
+    var threwException: Boolean = true
+    try {
+      processRows()
+      threwException = false
+    } catch {
+      // SPARK-25158 Exception should not be thrown again, otherwise it will be captured by
+      // SparkUncaughtExceptionHandler, then Executor will exit because of this Uncaught Exception,
+      // so pass the exception to `ScriptTransformationExec` is enough.
+      case t: Throwable =>
+        // An error occurred while writing input, so kill the child process. According to the
+        // Javadoc this call will not throw an exception:
+        _exception = t
+        proc.destroy()
+        logError("Thread-ScriptTransformation-Feed exit cause by: ", t)
+    } finally {
+      try {
+        Utils.tryLogNonFatalError(outputStream.close())
+        if (proc.waitFor() != 0) {
+          logError(stderrBuffer.toString) // log the stderr circular buffer
+        }
+      } catch {
+        case NonFatal(exceptionFromFinallyBlock) =>
+          if (!threwException) {
+            throw exceptionFromFinallyBlock
+          } else {
+            log.error("Exception in finally block", exceptionFromFinallyBlock)
+          }
+      }
+    }
+  }
+}
+
+/**
+ * The wrapper class of input and output schema properties
+ */
+case class ScriptTransformationIOSchema(
+    inputRowFormat: Seq[(String, String)],
+    outputRowFormat: Seq[(String, String)],
+    inputSerdeClass: Option[String],
+    outputSerdeClass: Option[String],
+    inputSerdeProps: Seq[(String, String)],
+    outputSerdeProps: Seq[(String, String)],
+    recordReaderClass: Option[String],
+    recordWriterClass: Option[String],
+    schemaLess: Boolean) extends Serializable {
+  import ScriptTransformationIOSchema._
+
+  val inputRowFormatMap = inputRowFormat.toMap.withDefault((k) => defaultFormat(k))
+  val outputRowFormatMap = outputRowFormat.toMap.withDefault((k) => defaultFormat(k))
+}
+
+object ScriptTransformationIOSchema {
+  val defaultFormat = Map(
+    ("TOK_TABLEROWFORMATFIELD", "\u0001"),
+    ("TOK_TABLEROWFORMATLINES", "\n")
+  )
+
+  val defaultIOSchema = ScriptTransformationIOSchema(
+    inputRowFormat = Seq.empty,
+    outputRowFormat = Seq.empty,
+    inputSerdeClass = None,
+    outputSerdeClass = None,
+    inputSerdeProps = Seq.empty,
+    outputSerdeProps = Seq.empty,
+    recordReaderClass = None,
+    recordWriterClass = None,
+    schemaLess = false
+  )
+
+  def apply(input: ScriptInputOutputSchema): ScriptTransformationIOSchema = {
+    ScriptTransformationIOSchema(
+      input.inputRowFormat,
+      input.outputRowFormat,
+      input.inputSerdeClass,
+      input.outputSerdeClass,
+      input.inputSerdeProps,
+      input.outputSerdeProps,
+      input.recordReaderClass,
+      input.recordWriterClass,
+      input.schemaLess)
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala
index 52cec8b202885..e11463c6d616e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala
@@ -22,6 +22,7 @@ import scala.collection.immutable.IndexedSeq
 import org.apache.hadoop.fs.{FileSystem, Path}
 
 import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config.ConfigEntry
 import org.apache.spark.sql.{Dataset, SparkSession}
 import org.apache.spark.sql.catalyst.expressions.{Attribute, SubqueryExpression}
 import org.apache.spark.sql.catalyst.optimizer.EliminateResolvedHint
@@ -56,6 +57,17 @@ class CacheManager extends Logging with AdaptiveSparkPlanHelper {
   @transient @volatile
   private var cachedData = IndexedSeq[CachedData]()
 
+  /**
+   * Configurations needs to be turned off, to avoid regression for cached query, so that the
+   * outputPartitioning of the underlying cached query plan can be leveraged later.
+   * Configurations include:
+   * 1. AQE
+   * 2. Automatic bucketed table scan
+   */
+  private val forceDisableConfigs: Seq[ConfigEntry[Boolean]] = Seq(
+    SQLConf.ADAPTIVE_EXECUTION_ENABLED,
+    SQLConf.AUTO_BUCKETED_SCAN_ENABLED)
+
   /** Clears all cached tables. */
   def clearCache(): Unit = this.synchronized {
     cachedData.foreach(_.cachedRepresentation.cacheBuilder.clearCache())
@@ -76,20 +88,39 @@ class CacheManager extends Logging with AdaptiveSparkPlanHelper {
       query: Dataset[_],
       tableName: Option[String] = None,
       storageLevel: StorageLevel = MEMORY_AND_DISK): Unit = {
-    val planToCache = query.logicalPlan
+    cacheQuery(query.sparkSession, query.logicalPlan, tableName, storageLevel)
+  }
+
+  /**
+   * Caches the data produced by the given [[LogicalPlan]].
+   * Unlike `RDD.cache()`, the default storage level is set to be `MEMORY_AND_DISK` because
+   * recomputing the in-memory columnar representation of the underlying table is expensive.
+   */
+  def cacheQuery(
+      spark: SparkSession,
+      planToCache: LogicalPlan,
+      tableName: Option[String]): Unit = {
+    cacheQuery(spark, planToCache, tableName, MEMORY_AND_DISK)
+  }
+
+  /**
+   * Caches the data produced by the given [[LogicalPlan]].
+   */
+  def cacheQuery(
+      spark: SparkSession,
+      planToCache: LogicalPlan,
+      tableName: Option[String],
+      storageLevel: StorageLevel): Unit = {
     if (lookupCachedData(planToCache).nonEmpty) {
       logWarning("Asked to cache already cached data.")
     } else {
-      // Turn off AQE so that the outputPartitioning of the underlying plan can be leveraged.
-      val sessionWithAqeOff = getOrCloneSessionWithAqeOff(query.sparkSession)
-      val inMemoryRelation = sessionWithAqeOff.withActive {
-        val qe = sessionWithAqeOff.sessionState.executePlan(planToCache)
+      val sessionWithConfigsOff = getOrCloneSessionWithConfigsOff(spark)
+      val inMemoryRelation = sessionWithConfigsOff.withActive {
+        val qe = sessionWithConfigsOff.sessionState.executePlan(planToCache)
         InMemoryRelation(
-          sessionWithAqeOff.sessionState.conf.useCompression,
-          sessionWithAqeOff.sessionState.conf.columnBatchSize, storageLevel,
-          qe.executedPlan,
-          tableName,
-          optimizedPlan = qe.optimizedPlan)
+          storageLevel,
+          qe,
+          tableName)
       }
 
       this.synchronized {
@@ -189,15 +220,12 @@ class CacheManager extends Logging with AdaptiveSparkPlanHelper {
       // Remove the cache entry before creating a new ones.
       cachedData = cachedData.filterNot(cd => needToRecache.exists(_ eq cd))
     }
-    needToRecache.map { cd =>
+    needToRecache.foreach { cd =>
       cd.cachedRepresentation.cacheBuilder.clearCache()
-      // Turn off AQE so that the outputPartitioning of the underlying plan can be leveraged.
-      val sessionWithAqeOff = getOrCloneSessionWithAqeOff(spark)
-      val newCache = sessionWithAqeOff.withActive {
-        val qe = sessionWithAqeOff.sessionState.executePlan(cd.plan)
-        InMemoryRelation(
-          cacheBuilder = cd.cachedRepresentation.cacheBuilder.copy(cachedPlan = qe.executedPlan),
-          optimizedPlan = qe.optimizedPlan)
+      val sessionWithConfigsOff = getOrCloneSessionWithConfigsOff(spark)
+      val newCache = sessionWithConfigsOff.withActive {
+        val qe = sessionWithConfigsOff.sessionState.executePlan(cd.plan)
+        InMemoryRelation(cd.cachedRepresentation.cacheBuilder, qe)
       }
       val recomputedPlan = cd.copy(cachedRepresentation = newCache)
       this.synchronized {
@@ -248,12 +276,17 @@ class CacheManager extends Logging with AdaptiveSparkPlanHelper {
    * `HadoopFsRelation` node(s) as part of its logical plan.
    */
   def recacheByPath(spark: SparkSession, resourcePath: String): Unit = {
-    val (fs, qualifiedPath) = {
-      val path = new Path(resourcePath)
-      val fs = path.getFileSystem(spark.sessionState.newHadoopConf())
-      (fs, fs.makeQualified(path))
-    }
+    val path = new Path(resourcePath)
+    val fs = path.getFileSystem(spark.sessionState.newHadoopConf())
+    recacheByPath(spark, path, fs)
+  }
 
+  /**
+   * Tries to re-cache all the cache entries that contain `resourcePath` in one or more
+   * `HadoopFsRelation` node(s) as part of its logical plan.
+   */
+  def recacheByPath(spark: SparkSession, resourcePath: Path, fs: FileSystem): Unit = {
+    val qualifiedPath = fs.makeQualified(resourcePath)
     recacheByCondition(spark, _.plan.find(lookupAndRefresh(_, fs, qualifiedPath)).isDefined)
   }
 
@@ -293,4 +326,15 @@ class CacheManager extends Logging with AdaptiveSparkPlanHelper {
     if (needToRefresh) fileIndex.refresh()
     needToRefresh
   }
+
+  /**
+   * If CAN_CHANGE_CACHED_PLAN_OUTPUT_PARTITIONING is enabled, just return original session.
+   */
+  private def getOrCloneSessionWithConfigsOff(session: SparkSession): SparkSession = {
+    if (session.sessionState.conf.getConf(SQLConf.CAN_CHANGE_CACHED_PLAN_OUTPUT_PARTITIONING)) {
+      session
+    } else {
+      SparkSession.getOrCloneSessionWithConfigsOff(session, forceDisableConfigs)
+    }
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/CollectMetricsExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/CollectMetricsExec.scala
index e1b9c8f430c56..500425e4809e0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/CollectMetricsExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/CollectMetricsExec.scala
@@ -16,8 +16,6 @@
  */
 package org.apache.spark.sql.execution
 
-import scala.collection.mutable
-
 import org.apache.spark.TaskContext
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.Row
@@ -80,6 +78,9 @@ case class CollectMetricsExec(
       }
     }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): CollectMetricsExec =
+    copy(child = newChild)
 }
 
 object CollectMetricsExec {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/Columnar.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/Columnar.scala
index e01cd8598db0c..6bdd93e9230b1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/Columnar.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/Columnar.scala
@@ -29,7 +29,6 @@ import org.apache.spark.sql.catalyst.plans.physical.Partitioning
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics}
 import org.apache.spark.sql.execution.vectorized.{OffHeapColumnVector, OnHeapColumnVector, WritableColumnVector}
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector}
 
@@ -49,6 +48,13 @@ class ColumnarRule {
   def postColumnarTransitions: Rule[SparkPlan] = plan => plan
 }
 
+/**
+ * A trait that is used as a tag to indicate a transition from columns to rows. This allows plugins
+ * to replace the current [[ColumnarToRowExec]] with an optimized version and still have operations
+ * that walk a spark plan looking for this type of transition properly match it.
+ */
+trait ColumnarToRowTransition extends UnaryExecNode
+
 /**
  * Provides a common executor to translate an [[RDD]] of [[ColumnarBatch]] into an [[RDD]] of
  * [[InternalRow]]. This is inserted whenever such a transition is determined to be needed.
@@ -57,7 +63,7 @@ class ColumnarRule {
  * [[org.apache.spark.sql.execution.python.ArrowEvalPythonExec]] and
  * [[MapPartitionsInRWithArrowExec]]. Eventually this should replace those implementations.
  */
-case class ColumnarToRowExec(child: SparkPlan) extends UnaryExecNode with CodegenSupport {
+case class ColumnarToRowExec(child: SparkPlan) extends ColumnarToRowTransition with CodegenSupport {
   assert(child.supportsColumnar)
 
   override def output: Seq[Attribute] = child.output
@@ -195,6 +201,9 @@ case class ColumnarToRowExec(child: SparkPlan) extends UnaryExecNode with Codege
   override def inputRDDs(): Seq[RDD[InternalRow]] = {
     Seq(child.executeColumnar().asInstanceOf[RDD[InternalRow]]) // Hack because of type erasure
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): ColumnarToRowExec =
+    copy(child = newChild)
 }
 
 /**
@@ -385,6 +394,13 @@ private object RowToColumnConverter {
   }
 }
 
+/**
+ * A trait that is used as a tag to indicate a transition from rows to columns. This allows plugins
+ * to replace the current [[RowToColumnarExec]] with an optimized version and still have operations
+ * that walk a spark plan looking for this type of transition properly match it.
+ */
+trait RowToColumnarTransition extends UnaryExecNode
+
 /**
  * Provides a common executor to translate an [[RDD]] of [[InternalRow]] into an [[RDD]] of
  * [[ColumnarBatch]]. This is inserted whenever such a transition is determined to be needed.
@@ -402,7 +418,7 @@ private object RowToColumnConverter {
  * populate with [[RowToColumnConverter]], but the performance requirements are different and it
  * would only be to reduce code.
  */
-case class RowToColumnarExec(child: SparkPlan) extends UnaryExecNode {
+case class RowToColumnarExec(child: SparkPlan) extends RowToColumnarTransition {
   override def output: Seq[Attribute] = child.output
 
   override def outputPartitioning: Partitioning = child.outputPartitioning
@@ -473,13 +489,17 @@ case class RowToColumnarExec(child: SparkPlan) extends UnaryExecNode {
       }
     }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): RowToColumnarExec =
+    copy(child = newChild)
 }
 
 /**
  * Apply any user defined [[ColumnarRule]]s and find the correct place to insert transitions
  * to/from columnar formatted data.
  */
-case class ApplyColumnarRulesAndInsertTransitions(conf: SQLConf, columnarRules: Seq[ColumnarRule])
+case class ApplyColumnarRulesAndInsertTransitions(
+    columnarRules: Seq[ColumnarRule])
   extends Rule[SparkPlan] {
 
   /**
@@ -490,8 +510,10 @@ case class ApplyColumnarRulesAndInsertTransitions(conf: SQLConf, columnarRules:
       // The tree feels kind of backwards
       // Columnar Processing will start here, so transition from row to columnar
       RowToColumnarExec(insertTransitions(plan))
-    } else {
+    } else if (!plan.isInstanceOf[RowToColumnarTransition]) {
       plan.withNewChildren(plan.children.map(insertRowToColumnar))
+    } else {
+      plan
     }
   }
 
@@ -503,8 +525,10 @@ case class ApplyColumnarRulesAndInsertTransitions(conf: SQLConf, columnarRules:
       // The tree feels kind of backwards
       // This is the end of the columnar processing so go back to rows
       ColumnarToRowExec(insertRowToColumnar(plan))
-    } else {
+    } else if (!plan.isInstanceOf[ColumnarToRowTransition]) {
       plan.withNewChildren(plan.children.map(insertTransitions))
+    } else {
+      plan
     }
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
index 66996498ffd3b..6fa4167384925 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
@@ -25,7 +25,6 @@ import org.apache.commons.lang3.StringUtils
 import org.apache.hadoop.fs.Path
 
 import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier}
 import org.apache.spark.sql.catalyst.catalog.BucketSpec
 import org.apache.spark.sql.catalyst.expressions._
@@ -55,10 +54,12 @@ trait DataSourceScanExec extends LeafExecNode {
   // Metadata that describes more details of this scan.
   protected def metadata: Map[String, String]
 
+  protected val maxMetadataValueLength = sqlContext.sessionState.conf.maxMetadataStringLength
+
   override def simpleString(maxFields: Int): String = {
     val metadataEntries = metadata.toSeq.sorted.map {
       case (key, value) =>
-        key + ": " + StringUtils.abbreviate(redact(value), 100)
+        key + ": " + StringUtils.abbreviate(redact(value), maxMetadataValueLength)
     }
     val metadataStr = truncatedString(metadataEntries, " ", ", ", "", maxFields)
     redact(
@@ -97,8 +98,8 @@ trait DataSourceScanExec extends LeafExecNode {
 
 /** Physical plan node for scanning data from a relation. */
 case class RowDataSourceScanExec(
-    fullOutput: Seq[Attribute],
-    requiredColumnsIndex: Seq[Int],
+    output: Seq[Attribute],
+    requiredSchema: StructType,
     filters: Set[Filter],
     handledFilters: Set[Filter],
     rdd: RDD[InternalRow],
@@ -106,8 +107,6 @@ case class RowDataSourceScanExec(
     tableIdentifier: Option[TableIdentifier])
   extends DataSourceScanExec with InputRDDCodegen {
 
-  def output: Seq[Attribute] = requiredColumnsIndex.map(fullOutput)
-
   override lazy val metrics =
     Map("numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"))
 
@@ -134,14 +133,14 @@ case class RowDataSourceScanExec(
       if (handledFilters.contains(filter)) s"*$filter" else s"$filter"
     }
     Map(
-      "ReadSchema" -> output.toStructType.catalogString,
+      "ReadSchema" -> requiredSchema.catalogString,
       "PushedFilters" -> markedFilters.mkString("[", ", ", "]"))
   }
 
   // Don't care about `rdd` and `tableIdentifier` when canonicalizing.
   override def doCanonicalize(): SparkPlan =
     copy(
-      fullOutput.map(QueryPlan.normalizeExpressions(_, fullOutput)),
+      output.map(QueryPlan.normalizeExpressions(_, output)),
       rdd = null,
       tableIdentifier = None)
 }
@@ -153,9 +152,12 @@ case class RowDataSourceScanExec(
  * @param output Output attributes of the scan, including data attributes and partition attributes.
  * @param requiredSchema Required schema of the underlying relation, excluding partition columns.
  * @param partitionFilters Predicates to use for partition pruning.
- * @param optionalBucketSet Bucket ids for bucket pruning
+ * @param optionalBucketSet Bucket ids for bucket pruning.
+ * @param optionalNumCoalescedBuckets Number of coalesced buckets.
  * @param dataFilters Filters on non-partition columns.
- * @param tableIdentifier identifier for the table in the metastore.
+ * @param tableIdentifier Identifier for the table in the metastore.
+ * @param disableBucketedScan Disable bucketed scan based on physical query plan, see rule
+ *                            [[DisableUnnecessaryBucketedScan]] for details.
  */
 case class FileSourceScanExec(
     @transient relation: HadoopFsRelation,
@@ -163,8 +165,10 @@ case class FileSourceScanExec(
     requiredSchema: StructType,
     partitionFilters: Seq[Expression],
     optionalBucketSet: Option[BitSet],
+    optionalNumCoalescedBuckets: Option[Int],
     dataFilters: Seq[Expression],
-    tableIdentifier: Option[TableIdentifier])
+    tableIdentifier: Option[TableIdentifier],
+    disableBucketedScan: Boolean = false)
   extends DataSourceScanExec {
 
   // Note that some vals referring the file-based relation are lazy intentionally
@@ -175,7 +179,7 @@ case class FileSourceScanExec(
 
   private lazy val needsUnsafeRowConversion: Boolean = {
     if (relation.fileFormat.isInstanceOf[ParquetSource]) {
-      SparkSession.getActiveSession.get.sessionState.conf.parquetVectorizedReaderEnabled
+      sqlContext.conf.parquetVectorizedReaderEnabled
     } else {
       false
     }
@@ -203,15 +207,12 @@ case class FileSourceScanExec(
   private def isDynamicPruningFilter(e: Expression): Boolean =
     e.find(_.isInstanceOf[PlanExpression[_]]).isDefined
 
-  @transient private lazy val selectedPartitions: Array[PartitionDirectory] = {
+  @transient lazy val selectedPartitions: Array[PartitionDirectory] = {
     val optimizerMetadataTimeNs = relation.location.metadataOpsTimeNs.getOrElse(0L)
     val startTime = System.nanoTime()
     val ret =
       relation.location.listFiles(
         partitionFilters.filterNot(isDynamicPruningFilter), dataFilters)
-    if (relation.partitionSchemaOption.isDefined) {
-      driverMetrics("numPartitions") = ret.length
-    }
     setFilesNumAndSizeMetric(ret, true)
     val timeTakenMs = NANOSECONDS.toMillis(
       (System.nanoTime() - startTime) + optimizerMetadataTimeNs)
@@ -258,7 +259,8 @@ case class FileSourceScanExec(
 
   // exposed for testing
   lazy val bucketedScan: Boolean = {
-    if (relation.sparkSession.sessionState.conf.bucketingEnabled && relation.bucketSpec.isDefined) {
+    if (relation.sparkSession.sessionState.conf.bucketingEnabled && relation.bucketSpec.isDefined
+      && !disableBucketedScan) {
       val spec = relation.bucketSpec.get
       val bucketColumns = spec.bucketColumnNames.flatMap(n => toAttribute(n))
       bucketColumns.size == spec.bucketColumnNames.size
@@ -282,14 +284,15 @@ case class FileSourceScanExec(
       //
       // Sort ordering would be over the prefix subset of `sort columns` being read
       // from the table.
-      // eg.
+      // e.g.
       // Assume (col0, col2, col3) are the columns read from the table
       // If sort columns are (col0, col1), then sort ordering would be considered as (col0)
       // If sort columns are (col1, col0), then sort ordering would be empty as per rule #2
       // above
       val spec = relation.bucketSpec.get
       val bucketColumns = spec.bucketColumnNames.flatMap(n => toAttribute(n))
-      val partitioning = HashPartitioning(bucketColumns, spec.numBuckets)
+      val numPartitions = optionalNumCoalescedBuckets.getOrElse(spec.numBuckets)
+      val partitioning = HashPartitioning(bucketColumns, numPartitions)
       val sortColumns =
         spec.sortColumnNames.map(x => toAttribute(x)).takeWhile(x => x.isDefined).map(_.get)
       val shouldCalculateSortOrder =
@@ -309,7 +312,8 @@ case class FileSourceScanExec(
           files.map(_.getPath.getName).groupBy(file => BucketingUtils.getBucketId(file))
         val singleFilePartitions = bucketToFilesGrouping.forall(p => p._2.length <= 1)
 
-        if (singleFilePartitions) {
+        // TODO SPARK-24528 Sort order is currently ignored if buckets are coalesced.
+        if (singleFilePartitions && optionalNumCoalescedBuckets.isEmpty) {
           // TODO Currently Spark does not support writing columns sorting in descending order
           // so using Ascending order. This can be fixed in future
           sortColumns.map(attribute => SortOrder(attribute, Ascending))
@@ -335,7 +339,8 @@ case class FileSourceScanExec(
     def seqToString(seq: Seq[Any]) = seq.mkString("[", ", ", "]")
     val location = relation.location
     val locationDesc =
-      location.getClass.getSimpleName + seqToString(location.rootPaths)
+      location.getClass.getSimpleName +
+        Utils.buildLocationMetadata(location.rootPaths, maxMetadataValueLength)
     val metadata =
       Map(
         "Format" -> relation.fileFormat.toString,
@@ -346,19 +351,23 @@ case class FileSourceScanExec(
         "DataFilters" -> seqToString(dataFilters),
         "Location" -> locationDesc)
 
-    val withSelectedBucketsCount = relation.bucketSpec.map { spec =>
-      val numSelectedBuckets = optionalBucketSet.map { b =>
-        b.cardinality()
+    // TODO(SPARK-32986): Add bucketed scan info in explain output of FileSourceScanExec
+    if (bucketedScan) {
+      relation.bucketSpec.map { spec =>
+        val numSelectedBuckets = optionalBucketSet.map { b =>
+          b.cardinality()
+        } getOrElse {
+          spec.numBuckets
+        }
+        metadata + ("SelectedBucketsCount" ->
+          (s"$numSelectedBuckets out of ${spec.numBuckets}" +
+            optionalNumCoalescedBuckets.map { b => s" (Coalesced to $b)"}.getOrElse("")))
       } getOrElse {
-        spec.numBuckets
+        metadata
       }
-      metadata + ("SelectedBucketsCount" ->
-        s"$numSelectedBuckets out of ${spec.numBuckets}")
-    } getOrElse {
+    } else {
       metadata
     }
-
-    withSelectedBucketsCount
   }
 
   override def verboseStringWithOperatorId(): String = {
@@ -370,12 +379,12 @@ case class FileSourceScanExec(
       case (key, _) if (key.equals("Location")) =>
         val location = relation.location
         val numPaths = location.rootPaths.length
-        val abbreviatedLoaction = if (numPaths <= 1) {
+        val abbreviatedLocation = if (numPaths <= 1) {
           location.rootPaths.mkString("[", ", ", "]")
         } else {
           "[" + location.rootPaths.head + s", ... ${numPaths - 1} entries]"
         }
-        s"$key: ${location.getClass.getSimpleName} ${redact(abbreviatedLoaction)}"
+        s"$key: ${location.getClass.getSimpleName} ${redact(abbreviatedLocation)}"
       case (key, value) => s"$key: ${redact(value)}"
     }
 
@@ -401,7 +410,7 @@ case class FileSourceScanExec(
       createBucketedReadRDD(relation.bucketSpec.get, readFile, dynamicallySelectedPartitions,
         relation)
     } else {
-      createNonBucketedReadRDD(readFile, dynamicallySelectedPartitions, relation)
+      createReadRDD(readFile, dynamicallySelectedPartitions, relation)
     }
     sendDriverMetrics()
     readRDD
@@ -412,7 +421,7 @@ case class FileSourceScanExec(
   }
 
   /** SQL metrics generated only for scans using dynamic partition pruning. */
-  private lazy val staticMetrics = if (partitionFilters.filter(isDynamicPruningFilter).nonEmpty) {
+  private lazy val staticMetrics = if (partitionFilters.exists(isDynamicPruningFilter)) {
     Map("staticFilesNum" -> SQLMetrics.createMetric(sparkContext, "static number of files read"),
       "staticFilesSize" -> SQLMetrics.createSizeMetric(sparkContext, "static size of files read"))
   } else {
@@ -425,22 +434,23 @@ case class FileSourceScanExec(
       static: Boolean): Unit = {
     val filesNum = partitions.map(_.files.size.toLong).sum
     val filesSize = partitions.map(_.files.map(_.getLen).sum).sum
-    if (!static || partitionFilters.filter(isDynamicPruningFilter).isEmpty) {
+    if (!static || !partitionFilters.exists(isDynamicPruningFilter)) {
       driverMetrics("numFiles") = filesNum
       driverMetrics("filesSize") = filesSize
     } else {
       driverMetrics("staticFilesNum") = filesNum
       driverMetrics("staticFilesSize") = filesSize
     }
+    if (relation.partitionSchemaOption.isDefined) {
+      driverMetrics("numPartitions") = partitions.length
+    }
   }
 
   override lazy val metrics = Map(
     "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
     "numFiles" -> SQLMetrics.createMetric(sparkContext, "number of files read"),
     "metadataTime" -> SQLMetrics.createTimingMetric(sparkContext, "metadata time"),
-    "filesSize" -> SQLMetrics.createSizeMetric(sparkContext, "size of files read"),
-    "pruningTime" ->
-      SQLMetrics.createTimingMetric(sparkContext, "dynamic partition pruning time")
+    "filesSize" -> SQLMetrics.createSizeMetric(sparkContext, "size of files read")
   ) ++ {
     // Tracking scan time has overhead, we can't afford to do it for each row, and can only do
     // it for each batch.
@@ -451,9 +461,12 @@ case class FileSourceScanExec(
     }
   } ++ {
     if (relation.partitionSchemaOption.isDefined) {
-      Some("numPartitions" -> SQLMetrics.createMetric(sparkContext, "number of partitions read"))
+      Map(
+        "numPartitions" -> SQLMetrics.createMetric(sparkContext, "number of partitions read"),
+        "pruningTime" ->
+          SQLMetrics.createTimingMetric(sparkContext, "dynamic partition pruning time"))
     } else {
-      None
+      Map.empty[String, SQLMetric]
     }
   } ++ staticMetrics
 
@@ -505,7 +518,7 @@ case class FileSourceScanExec(
 
   /**
    * Create an RDD for bucketed reads.
-   * The non-bucketed variant of this function is [[createNonBucketedReadRDD]].
+   * The non-bucketed variant of this function is [[createReadRDD]].
    *
    * The algorithm is pretty simple: each RDD partition being returned should include all the files
    * with the same bucket id from all the given Hive partitions.
@@ -529,7 +542,7 @@ case class FileSourceScanExec(
       }.groupBy { f =>
         BucketingUtils
           .getBucketId(new Path(f.filePath).getName)
-          .getOrElse(sys.error(s"Invalid bucket file ${f.filePath}"))
+          .getOrElse(throw new IllegalStateException(s"Invalid bucket file ${f.filePath}"))
       }
 
     val prunedFilesGroupedToBuckets = if (optionalBucketSet.isDefined) {
@@ -541,8 +554,19 @@ case class FileSourceScanExec(
       filesGroupedToBuckets
     }
 
-    val filePartitions = Seq.tabulate(bucketSpec.numBuckets) { bucketId =>
-      FilePartition(bucketId, prunedFilesGroupedToBuckets.getOrElse(bucketId, Array.empty))
+    val filePartitions = optionalNumCoalescedBuckets.map { numCoalescedBuckets =>
+      logInfo(s"Coalescing to ${numCoalescedBuckets} buckets")
+      val coalescedBuckets = prunedFilesGroupedToBuckets.groupBy(_._1 % numCoalescedBuckets)
+      Seq.tabulate(numCoalescedBuckets) { bucketId =>
+        val partitionedFiles = coalescedBuckets.get(bucketId).map {
+          _.values.flatten.toArray
+        }.getOrElse(Array.empty)
+        FilePartition(bucketId, partitionedFiles)
+      }
+    }.getOrElse {
+      Seq.tabulate(bucketSpec.numBuckets) { bucketId =>
+        FilePartition(bucketId, prunedFilesGroupedToBuckets.getOrElse(bucketId, Array.empty))
+      }
     }
 
     new FileScanRDD(fsRelation.sparkSession, readFile, filePartitions)
@@ -556,7 +580,7 @@ case class FileSourceScanExec(
    * @param selectedPartitions Hive-style partition that are part of the read.
    * @param fsRelation [[HadoopFsRelation]] associated with the read.
    */
-  private def createNonBucketedReadRDD(
+  private def createReadRDD(
       readFile: (PartitionedFile) => Iterator[InternalRow],
       selectedPartitions: Array[PartitionDirectory],
       fsRelation: HadoopFsRelation): RDD[InternalRow] = {
@@ -566,20 +590,35 @@ case class FileSourceScanExec(
     logInfo(s"Planning scan with bin packing, max size: $maxSplitBytes bytes, " +
       s"open cost is considered as scanning $openCostInBytes bytes.")
 
+    // Filter files with bucket pruning if possible
+    val bucketingEnabled = fsRelation.sparkSession.sessionState.conf.bucketingEnabled
+    val shouldProcess: Path => Boolean = optionalBucketSet match {
+      case Some(bucketSet) if bucketingEnabled =>
+        // Do not prune the file if bucket file name is invalid
+        filePath => BucketingUtils.getBucketId(filePath.getName).forall(bucketSet.get)
+      case _ =>
+        _ => true
+    }
+
     val splitFiles = selectedPartitions.flatMap { partition =>
       partition.files.flatMap { file =>
         // getPath() is very expensive so we only want to call it once in this block:
         val filePath = file.getPath
-        val isSplitable = relation.fileFormat.isSplitable(
-          relation.sparkSession, relation.options, filePath)
-        PartitionedFileUtil.splitFiles(
-          sparkSession = relation.sparkSession,
-          file = file,
-          filePath = filePath,
-          isSplitable = isSplitable,
-          maxSplitBytes = maxSplitBytes,
-          partitionValues = partition.values
-        )
+
+        if (shouldProcess(filePath)) {
+          val isSplitable = relation.fileFormat.isSplitable(
+            relation.sparkSession, relation.options, filePath)
+          PartitionedFileUtil.splitFiles(
+            sparkSession = relation.sparkSession,
+            file = file,
+            filePath = filePath,
+            isSplitable = isSplitable,
+            maxSplitBytes = maxSplitBytes,
+            partitionValues = partition.values
+          )
+        } else {
+          Seq.empty
+        }
       }
     }.sortBy(_.length)(implicitly[Ordering[Long]].reverse)
 
@@ -589,14 +628,24 @@ case class FileSourceScanExec(
     new FileScanRDD(fsRelation.sparkSession, readFile, partitions)
   }
 
+  // Filters unused DynamicPruningExpression expressions - one which has been replaced
+  // with DynamicPruningExpression(Literal.TrueLiteral) during Physical Planning
+  private def filterUnusedDynamicPruningExpressions(
+      predicates: Seq[Expression]): Seq[Expression] = {
+    predicates.filterNot(_ == DynamicPruningExpression(Literal.TrueLiteral))
+  }
+
   override def doCanonicalize(): FileSourceScanExec = {
     FileSourceScanExec(
       relation,
       output.map(QueryPlan.normalizeExpressions(_, output)),
       requiredSchema,
-      QueryPlan.normalizePredicates(partitionFilters, output),
+      QueryPlan.normalizePredicates(
+        filterUnusedDynamicPruningExpressions(partitionFilters), output),
       optionalBucketSet,
+      optionalNumCoalescedBuckets,
       QueryPlan.normalizePredicates(dataFilters, output),
-      None)
+      None,
+      disableBucketedScan)
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ExpandExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ExpandExec.scala
index 0615324b84305..c087fdf5f962b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ExpandExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ExpandExec.scala
@@ -19,12 +19,11 @@ package org.apache.spark.sql.execution
 
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.errors._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen._
-import org.apache.spark.sql.catalyst.expressions.codegen.Block._
 import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, UnknownPartitioning}
 import org.apache.spark.sql.execution.metric.SQLMetrics
+import org.apache.spark.sql.internal.SQLConf
 
 /**
  * Apply all of the GroupExpressions to every input row, hence we will get
@@ -54,7 +53,7 @@ case class ExpandExec(
   private[this] val projection =
     (exprs: Seq[Expression]) => UnsafeProjection.create(exprs, child.output)
 
-  protected override def doExecute(): RDD[InternalRow] = attachTree(this, "execute") {
+  protected override def doExecute(): RDD[InternalRow] = {
     val numOutputRows = longMetric("numOutputRows")
 
     child.execute().mapPartitions { iter =>
@@ -153,40 +152,82 @@ case class ExpandExec(
         // This column is the same across all output rows. Just generate code for it here.
         BindReferences.bindReference(firstExpr, attributeSeq).genCode(ctx)
       } else {
-        val isNull = ctx.freshName("isNull")
-        val value = ctx.freshName("value")
-        val code = code"""
-          |boolean $isNull = true;
-          |${CodeGenerator.javaType(firstExpr.dataType)} $value =
-          |  ${CodeGenerator.defaultValue(firstExpr.dataType)};
-         """.stripMargin
+        val isNull = ctx.addMutableState(
+          CodeGenerator.JAVA_BOOLEAN,
+          "resultIsNull",
+          v => s"$v = true;")
+        val value = ctx.addMutableState(
+          CodeGenerator.javaType(firstExpr.dataType),
+          "resultValue",
+          v => s"$v = ${CodeGenerator.defaultValue(firstExpr.dataType)};")
+
         ExprCode(
-          code,
           JavaCode.isNullVariable(isNull),
           JavaCode.variable(value, firstExpr.dataType))
       }
     }
 
     // Part 2: switch/case statements
-    val cases = projections.zipWithIndex.map { case (exprs, row) =>
-      var updateCode = ""
-      for (col <- exprs.indices) {
+    val switchCaseExprs = projections.zipWithIndex.map { case (exprs, row) =>
+      val (exprCodesWithIndices, inputVarSets) = exprs.indices.flatMap { col =>
         if (!sameOutput(col)) {
-          val ev = BindReferences.bindReference(exprs(col), attributeSeq).genCode(ctx)
-          updateCode +=
+          val boundExpr = BindReferences.bindReference(exprs(col), attributeSeq)
+          val exprCode = boundExpr.genCode(ctx)
+          val inputVars = CodeGenerator.getLocalInputVariableValues(ctx, boundExpr)._1
+          Some(((col, exprCode), inputVars))
+        } else {
+          None
+        }
+      }.unzip
+
+      val inputVars = inputVarSets.foldLeft(Set.empty[VariableValue])(_ ++ _)
+      (row, exprCodesWithIndices, inputVars.toSeq)
+    }
+
+    val updateCodes = switchCaseExprs.map { case (_, exprCodes, _) =>
+      exprCodes.map { case (col, ev) =>
+        s"""
+           |${ev.code}
+           |${outputColumns(col).isNull} = ${ev.isNull};
+           |${outputColumns(col).value} = ${ev.value};
+         """.stripMargin
+      }.mkString("\n")
+    }
+
+    val splitThreshold = SQLConf.get.methodSplitThreshold
+    val cases = if (switchCaseExprs.flatMap(_._2.map(_._2.code.length)).sum > splitThreshold) {
+      switchCaseExprs.zip(updateCodes).map { case ((row, _, inputVars), updateCode) =>
+        val paramLength = CodeGenerator.calculateParamLengthFromExprValues(inputVars)
+        val maybeSplitUpdateCode = if (CodeGenerator.isValidParamLength(paramLength)) {
+          val switchCaseFunc = ctx.freshName("switchCaseCode")
+          val argList = inputVars.map { v =>
+            s"${CodeGenerator.typeName(v.javaType)} ${v.variableName}"
+          }
+          ctx.addNewFunction(switchCaseFunc,
             s"""
-               |${ev.code}
-               |${outputColumns(col).isNull} = ${ev.isNull};
-               |${outputColumns(col).value} = ${ev.value};
-            """.stripMargin
+               |private void $switchCaseFunc(${argList.mkString(", ")}) {
+               |  $updateCode
+               |}
+             """.stripMargin)
+
+          s"$switchCaseFunc(${inputVars.map(_.variableName).mkString(", ")});"
+        } else {
+          updateCode
         }
+        s"""
+           |case $row:
+           |  $maybeSplitUpdateCode
+           |  break;
+         """.stripMargin
+      }
+    } else {
+      switchCaseExprs.map(_._1).zip(updateCodes).map { case (row, updateCode) =>
+        s"""
+           |case $row:
+           |  $updateCode
+           |  break;
+         """.stripMargin
       }
-
-      s"""
-         |case $row:
-         |  ${updateCode.trim}
-         |  break;
-       """.stripMargin
     }
 
     val numOutput = metricTerm(ctx, "numOutputRows")
@@ -204,4 +245,7 @@ case class ExpandExec(
        |}
      """.stripMargin
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): ExpandExec =
+    copy(child = newChild)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala
index b54bd6a579b66..f47542ca59bc7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala
@@ -28,14 +28,14 @@ import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AdaptiveS
 object ExplainUtils extends AdaptiveSparkPlanHelper {
   /**
    * Given a input physical plan, performs the following tasks.
-   *   1. Computes the operator id for current operator and records it in the operaror
+   *   1. Computes the operator id for current operator and records it in the operator
    *      by setting a tag.
    *   2. Computes the whole stage codegen id for current operator and records it in the
    *      operator by setting a tag.
    *   3. Generate the two part explain output for this plan.
    *      1. First part explains the operator tree with each operator tagged with an unique
    *         identifier.
-   *      2. Second part explans each operator in a verbose manner.
+   *      2. Second part explains each operator in a verbose manner.
    *
    * Note : This function skips over subqueries. They are handled by its caller.
    *
@@ -117,7 +117,7 @@ object ExplainUtils extends AdaptiveSparkPlanHelper {
   }
 
   /**
-   * Traverses the supplied input plan in a bottem-up fashion does the following :
+   * Traverses the supplied input plan in a bottom-up fashion does the following :
    *    1. produces a map : operator identifier -> operator
    *    2. Records the operator id via setting a tag in the operator.
    * Note :
@@ -210,7 +210,7 @@ object ExplainUtils extends AdaptiveSparkPlanHelper {
 
   /**
    * Given a input plan, returns an array of tuples comprising of :
-   *  1. Hosting opeator id.
+   *  1. Hosting operator id.
    *  2. Hosting expression
    *  3. Subquery plan
    */
@@ -218,6 +218,8 @@ object ExplainUtils extends AdaptiveSparkPlanHelper {
       plan: => QueryPlan[_],
       subqueries: ArrayBuffer[(SparkPlan, Expression, BaseSubqueryExec)]): Unit = {
     plan.foreach {
+      case a: AdaptiveSparkPlanExec =>
+        getSubqueries(a.executedPlan, subqueries)
       case p: SparkPlan =>
         p.expressions.foreach (_.collect {
           case e: PlanExpression[_] =>
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArray.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArray.scala
index ac282ea2e94f5..c5e5de588ba9d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArray.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArray.scala
@@ -52,7 +52,7 @@ private[sql] class ExternalAppendOnlyUnsafeRowArray(
     numRowsInMemoryBufferThreshold: Int,
     numRowsSpillThreshold: Int) extends Logging {
 
-  def this(numRowsInMemoryBufferThreshold: Int, numRowsSpillThreshold: Int) {
+  def this(numRowsInMemoryBufferThreshold: Int, numRowsSpillThreshold: Int) = {
     this(
       TaskContext.get().taskMemoryManager(),
       SparkEnv.get.blockManager,
@@ -87,7 +87,7 @@ private[sql] class ExternalAppendOnlyUnsafeRowArray(
   def isEmpty: Boolean = numRows == 0
 
   /**
-   * Clears up resources (eg. memory) held by the backing storage
+   * Clears up resources (e.g. memory) held by the backing storage
    */
   def clear(): Unit = {
     if (spillableArray != null) {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/GenerateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/GenerateExec.scala
index 4c9efdbf2ba62..6c7929437ffdd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/GenerateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/GenerateExec.scala
@@ -96,7 +96,7 @@ case class GenerateExec(
           if (outer && outputRows.isEmpty) {
             joinedRow.withRight(generatorNullRow) :: Nil
           } else {
-            outputRows.map(joinedRow.withRight)
+            outputRows.toIterator.map(joinedRow.withRight)
           }
         } ++ LazyIterator(() => boundGenerator.terminate()).map { row =>
           // we leave the left side as the last element of its child output
@@ -124,7 +124,7 @@ case class GenerateExec(
     }
   }
 
-  override def supportCodegen: Boolean = false
+  override def supportCodegen: Boolean = generator.supportCodegen
 
   override def inputRDDs(): Seq[RDD[InternalRow]] = {
     child.asInstanceOf[CodegenSupport].inputRDDs()
@@ -137,16 +137,13 @@ case class GenerateExec(
   override def needCopyResult: Boolean = true
 
   override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = {
-    // Add input rows to the values when we are joining
-    val values = if (requiredChildOutput.nonEmpty) {
-      input
-    } else {
-      Seq.empty
-    }
-
+    val requiredAttrSet = AttributeSet(requiredChildOutput)
+    val requiredInput = child.output.zip(input).filter {
+      case (attr, _) => requiredAttrSet.contains(attr)
+    }.map(_._2)
     boundGenerator match {
-      case e: CollectionGenerator => codeGenCollection(ctx, e, values, row)
-      case g => codeGenTraversableOnce(ctx, g, values, row)
+      case e: CollectionGenerator => codeGenCollection(ctx, e, requiredInput, row)
+      case g => codeGenTraversableOnce(ctx, g, requiredInput, row)
     }
   }
 
@@ -244,7 +241,7 @@ case class GenerateExec(
   private def codeGenTraversableOnce(
       ctx: CodegenContext,
       e: Expression,
-      input: Seq[ExprCode],
+      requiredInput: Seq[ExprCode],
       row: ExprCode): String = {
 
     // Generate the code for the generator
@@ -280,7 +277,7 @@ case class GenerateExec(
          |  boolean $hasNext = $iterator.hasNext();
          |  InternalRow $current = (InternalRow)($hasNext? $iterator.next() : null);
          |  $outerVal = false;
-         |  ${consume(ctx, input ++ values)}
+         |  ${consume(ctx, requiredInput ++ values)}
          |}
       """.stripMargin
     } else {
@@ -290,7 +287,7 @@ case class GenerateExec(
          |while ($iterator.hasNext()) {
          |  $numOutput.add(1);
          |  InternalRow $current = (InternalRow)($iterator.next());
-         |  ${consume(ctx, input ++ values)}
+         |  ${consume(ctx, requiredInput ++ values)}
          |}
       """.stripMargin
     }
@@ -328,4 +325,7 @@ case class GenerateExec(
     if (condition) Seq(code)
     else Seq.empty
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): GenerateExec =
+    copy(child = newChild)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
index 1a84db1970449..52394c1b7f7f4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
@@ -19,10 +19,12 @@ package org.apache.spark.sql.execution
 
 import java.nio.charset.StandardCharsets
 import java.sql.{Date, Timestamp}
-import java.time.{Instant, LocalDate}
+import java.time.{Duration, Instant, LocalDate, Period, ZoneOffset}
 
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.util.{DateFormatter, DateTimeUtils, TimestampFormatter}
+import org.apache.spark.sql.catalyst.util.IntervalStringStyles.HIVE_STYLE
+import org.apache.spark.sql.catalyst.util.IntervalUtils.{durationToMicros, periodToMonths, toDayTimeIntervalString, toYearMonthIntervalString}
 import org.apache.spark.sql.execution.command.{DescribeCommandBase, ExecutedCommandExec, ShowTablesCommand, ShowViewsCommand}
 import org.apache.spark.sql.execution.datasources.v2.{DescribeTableExec, ShowTablesExec}
 import org.apache.spark.sql.internal.SQLConf
@@ -33,6 +35,23 @@ import org.apache.spark.unsafe.types.CalendarInterval
  * Runs a query returning the result in Hive compatible form.
  */
 object HiveResult {
+  case class TimeFormatters(date: DateFormatter, timestamp: TimestampFormatter)
+
+  def getTimeFormatters: TimeFormatters = {
+    // The date formatter does not depend on Spark's session time zone controlled by
+    // the SQL config `spark.sql.session.timeZone`. The `zoneId` parameter is used only in
+    // parsing of special date values like `now`, `yesterday` and etc. but not in date formatting.
+    // While formatting of:
+    // - `java.time.LocalDate`, zone id is not used by `DateTimeFormatter` at all.
+    // - `java.sql.Date`, the date formatter delegates formatting to the legacy formatter
+    //   which uses the default system time zone `TimeZone.getDefault`. This works correctly
+    //   due to `DateTimeUtils.toJavaDate` which is based on the system time zone too.
+    val dateFormatter = DateFormatter(ZoneOffset.UTC)
+    val timestampFormatter = TimestampFormatter.getFractionFormatter(
+      DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone))
+    TimeFormatters(dateFormatter, timestampFormatter)
+  }
+
   /**
    * Returns the result as a hive compatible sequence of strings. This is used in tests and
    * `SparkSQLDriver` for CLI applications.
@@ -55,11 +74,12 @@ object HiveResult {
     case command @ ExecutedCommandExec(_: ShowViewsCommand) =>
       command.executeCollect().map(_.getString(1))
     case other =>
+      val timeFormatters = getTimeFormatters
       val result: Seq[Seq[Any]] = other.executeCollectPublic().map(_.toSeq).toSeq
       // We need the types so we can output struct field names
       val types = executedPlan.output.map(_.dataType)
       // Reformat to match hive tab delimited output.
-      result.map(_.zip(types).map(e => toHiveString(e)))
+      result.map(_.zip(types).map(e => toHiveString(e, false, timeFormatters)))
         .map(_.mkString("\t"))
   }
 
@@ -72,36 +92,37 @@ object HiveResult {
     }
   }
 
-  private def zoneId = DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone)
-  private def dateFormatter = DateFormatter(zoneId)
-  private def timestampFormatter = TimestampFormatter.getFractionFormatter(zoneId)
-
   /** Formats a datum (based on the given data type) and returns the string representation. */
-  def toHiveString(a: (Any, DataType), nested: Boolean = false): String = a match {
+  def toHiveString(
+      a: (Any, DataType),
+      nested: Boolean,
+      formatters: TimeFormatters): String = a match {
     case (null, _) => if (nested) "null" else "NULL"
     case (b, BooleanType) => b.toString
-    case (d: Date, DateType) => dateFormatter.format(DateTimeUtils.fromJavaDate(d))
-    case (ld: LocalDate, DateType) =>
-      dateFormatter.format(DateTimeUtils.localDateToDays(ld))
-    case (t: Timestamp, TimestampType) =>
-      timestampFormatter.format(DateTimeUtils.fromJavaTimestamp(t))
-    case (i: Instant, TimestampType) =>
-      timestampFormatter.format(DateTimeUtils.instantToMicros(i))
+    case (d: Date, DateType) => formatters.date.format(d)
+    case (ld: LocalDate, DateType) => formatters.date.format(ld)
+    case (t: Timestamp, TimestampType) => formatters.timestamp.format(t)
+    case (i: Instant, TimestampType) => formatters.timestamp.format(i)
     case (bin: Array[Byte], BinaryType) => new String(bin, StandardCharsets.UTF_8)
     case (decimal: java.math.BigDecimal, DecimalType()) => decimal.toPlainString
     case (n, _: NumericType) => n.toString
     case (s: String, StringType) => if (nested) "\"" + s + "\"" else s
     case (interval: CalendarInterval, CalendarIntervalType) => interval.toString
-    case (seq: Seq[_], ArrayType(typ, _)) =>
-      seq.map(v => (v, typ)).map(e => toHiveString(e, true)).mkString("[", ",", "]")
+    case (seq: scala.collection.Seq[_], ArrayType(typ, _)) =>
+      seq.map(v => (v, typ)).map(e => toHiveString(e, true, formatters)).mkString("[", ",", "]")
     case (m: Map[_, _], MapType(kType, vType, _)) =>
       m.map { case (key, value) =>
-        toHiveString((key, kType), true) + ":" + toHiveString((value, vType), true)
+        toHiveString((key, kType), true, formatters) + ":" +
+          toHiveString((value, vType), true, formatters)
       }.toSeq.sorted.mkString("{", ",", "}")
     case (struct: Row, StructType(fields)) =>
       struct.toSeq.zip(fields).map { case (v, t) =>
-        s""""${t.name}":${toHiveString((v, t.dataType), true)}"""
+        s""""${t.name}":${toHiveString((v, t.dataType), true, formatters)}"""
       }.mkString("{", ",", "}")
+    case (period: Period, YearMonthIntervalType) =>
+      toYearMonthIntervalString(periodToMonths(period), HIVE_STYLE)
+    case (duration: Duration, DayTimeIntervalType) =>
+      toDayTimeIntervalString(durationToMicros(duration), HIVE_STYLE)
     case (other, _: UserDefinedType[_]) => other.toString
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/LocalTableScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/LocalTableScanExec.scala
index b452213cd6cc7..054daa54d1153 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/LocalTableScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/LocalTableScanExec.scala
@@ -49,7 +49,8 @@ case class LocalTableScanExec(
     if (rows.isEmpty) {
       sqlContext.sparkContext.emptyRDD
     } else {
-      val numSlices = math.min(unsafeRows.length, sqlContext.sparkContext.defaultParallelism)
+      val numSlices = math.min(
+        unsafeRows.length, sqlContext.sparkSession.leafNodeDefaultParallelism)
       sqlContext.sparkContext.parallelize(unsafeRows, numSlices)
     }
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala
index 492d177c7c773..3c6c0bc0557fe 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala
@@ -45,7 +45,7 @@ import org.apache.spark.sql.internal.SQLConf
 case class OptimizeMetadataOnlyQuery(catalog: SessionCatalog) extends Rule[LogicalPlan] {
 
   def apply(plan: LogicalPlan): LogicalPlan = {
-    if (!SQLConf.get.optimizerMetadataOnly) {
+    if (!conf.optimizerMetadataOnly) {
       return plan
     }
 
@@ -132,7 +132,7 @@ case class OptimizeMetadataOnlyQuery(catalog: SessionCatalog) extends Rule[Logic
             val caseInsensitiveProperties =
               CaseInsensitiveMap(relation.tableMeta.storage.properties)
             val timeZoneId = caseInsensitiveProperties.get(DateTimeUtils.TIMEZONE_OPTION)
-              .getOrElse(SQLConf.get.sessionLocalTimeZone)
+              .getOrElse(conf.sessionLocalTimeZone)
             val partitions = relation.prunedPartitions match {
               // for the case where partitions have already been pruned by PruneHiveTablePartitions
               case Some(parts) => parts
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
index 1df812d1aa809..0795776eb3f99 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
@@ -19,20 +19,24 @@ package org.apache.spark.sql.execution
 
 import java.io.{BufferedWriter, OutputStreamWriter}
 import java.util.UUID
+import java.util.concurrent.atomic.AtomicLong
 
 import org.apache.hadoop.fs.Path
 
+import org.apache.spark.internal.Logging
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{AnalysisException, Row, SparkSession}
 import org.apache.spark.sql.catalyst.{InternalRow, QueryPlanningTracker}
 import org.apache.spark.sql.catalyst.analysis.UnsupportedOperationChecker
+import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
 import org.apache.spark.sql.catalyst.expressions.codegen.ByteCodeStats
 import org.apache.spark.sql.catalyst.plans.QueryPlan
 import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, ReturnAnswer}
-import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.rules.{PlanChangeLogger, Rule}
 import org.apache.spark.sql.catalyst.util.StringUtils.PlanStringConcat
 import org.apache.spark.sql.catalyst.util.truncatedString
 import org.apache.spark.sql.execution.adaptive.{AdaptiveExecutionContext, InsertAdaptiveSparkPlan}
+import org.apache.spark.sql.execution.bucketing.{CoalesceBucketsInJoin, DisableUnnecessaryBucketedScan}
 import org.apache.spark.sql.execution.dynamicpruning.PlanDynamicPruningFilters
 import org.apache.spark.sql.execution.exchange.{EnsureRequirements, ReuseExchange}
 import org.apache.spark.sql.execution.streaming.{IncrementalExecution, OffsetSeqMetadata}
@@ -50,7 +54,9 @@ import org.apache.spark.util.Utils
 class QueryExecution(
     val sparkSession: SparkSession,
     val logical: LogicalPlan,
-    val tracker: QueryPlanningTracker = new QueryPlanningTracker) {
+    val tracker: QueryPlanningTracker = new QueryPlanningTracker) extends Logging {
+
+  val id: Long = QueryExecution.nextExecutionId
 
   // TODO: Move the planner an optimizer into here from SessionState.
   protected def planner = sparkSession.sessionState.planner
@@ -79,7 +85,12 @@ class QueryExecution(
   lazy val optimizedPlan: LogicalPlan = executePhase(QueryPlanningTracker.OPTIMIZATION) {
     // clone the plan to avoid sharing the plan instance between different stages like analyzing,
     // optimizing and planning.
-    sparkSession.sessionState.optimizer.executeAndTrack(withCachedData.clone(), tracker)
+    val plan = sparkSession.sessionState.optimizer.executeAndTrack(withCachedData.clone(), tracker)
+    // We do not want optimized plans to be re-analyzed as literals that have been constant folded
+    // and such can cause issues during analysis. While `clone` should maintain the `analyzed` state
+    // of the LogicalPlan, we set the plan as analyzed here as well out of paranoia.
+    plan.setAnalyzed()
+    plan
   }
 
   private def assertOptimized(): Unit = optimizedPlan
@@ -129,30 +140,46 @@ class QueryExecution(
       Option(InsertAdaptiveSparkPlan(AdaptiveExecutionContext(sparkSession, this))))
   }
 
-  private def executePhase[T](phase: String)(block: => T): T = sparkSession.withActive {
+  protected def executePhase[T](phase: String)(block: => T): T = sparkSession.withActive {
     tracker.measurePhase(phase)(block)
   }
 
-  def simpleString: String = simpleString(false)
-
-  def simpleString(formatted: Boolean): String = withRedaction {
+  def simpleString: String = {
     val concat = new PlanStringConcat()
-    concat.append("== Physical Plan ==\n")
+    simpleString(false, SQLConf.get.maxToStringFields, concat.append)
+    withRedaction {
+      concat.toString
+    }
+  }
+
+  private def simpleString(
+      formatted: Boolean,
+      maxFields: Int,
+      append: String => Unit): Unit = {
+    append("== Physical Plan ==\n")
     if (formatted) {
       try {
-        ExplainUtils.processPlan(executedPlan, concat.append)
+        ExplainUtils.processPlan(executedPlan, append)
       } catch {
-        case e: AnalysisException => concat.append(e.toString)
-        case e: IllegalArgumentException => concat.append(e.toString)
+        case e: AnalysisException => append(e.toString)
+        case e: IllegalArgumentException => append(e.toString)
       }
     } else {
-      QueryPlan.append(executedPlan, concat.append, verbose = false, addSuffix = false)
+      QueryPlan.append(executedPlan,
+        append, verbose = false, addSuffix = false, maxFields = maxFields)
     }
-    concat.append("\n")
-    concat.toString
+    append("\n")
   }
 
   def explainString(mode: ExplainMode): String = {
+    val concat = new PlanStringConcat()
+    explainString(mode, SQLConf.get.maxToStringFields, concat.append)
+    withRedaction {
+      concat.toString
+    }
+  }
+
+  private def explainString(mode: ExplainMode, maxFields: Int, append: String => Unit): Unit = {
     val queryExecution = if (logical.isStreaming) {
       // This is used only by explaining `Dataset/DataFrame` created by `spark.readStream`, so the
       // output mode does not matter since there is no `Sink`.
@@ -165,19 +192,19 @@ class QueryExecution(
 
     mode match {
       case SimpleMode =>
-        queryExecution.simpleString
+        queryExecution.simpleString(false, maxFields, append)
       case ExtendedMode =>
-        queryExecution.toString
+        queryExecution.toString(maxFields, append)
       case CodegenMode =>
         try {
-          org.apache.spark.sql.execution.debug.codegenString(queryExecution.executedPlan)
+          org.apache.spark.sql.execution.debug.writeCodegen(append, queryExecution.executedPlan)
         } catch {
-          case e: AnalysisException => e.toString
+          case e: AnalysisException => append(e.toString)
         }
       case CostMode =>
-        queryExecution.stringWithStats
+        queryExecution.stringWithStats(maxFields, append)
       case FormattedMode =>
-        queryExecution.simpleString(formatted = true)
+        queryExecution.simpleString(formatted = true, maxFields = maxFields, append)
     }
   }
 
@@ -187,11 +214,13 @@ class QueryExecution(
     QueryPlan.append(logical, append, verbose, addSuffix, maxFields)
     append("\n== Analyzed Logical Plan ==\n")
     try {
-      append(
-        truncatedString(
-          analyzed.output.map(o => s"${o.name}: ${o.dataType.simpleString}"), ", ", maxFields)
-      )
-      append("\n")
+      if (analyzed.output.nonEmpty) {
+        append(
+          truncatedString(
+            analyzed.output.map(o => s"${o.name}: ${o.dataType.simpleString}"), ", ", maxFields)
+        )
+        append("\n")
+      }
       QueryPlan.append(analyzed, append, verbose, addSuffix, maxFields)
       append("\n== Optimized Logical Plan ==\n")
       QueryPlan.append(optimizedPlan, append, verbose, addSuffix, maxFields)
@@ -204,27 +233,45 @@ class QueryExecution(
 
   override def toString: String = withRedaction {
     val concat = new PlanStringConcat()
-    writePlans(concat.append, SQLConf.get.maxToStringFields)
-    concat.toString
+    toString(SQLConf.get.maxToStringFields, concat.append)
+    withRedaction {
+      concat.toString
+    }
   }
 
-  def stringWithStats: String = withRedaction {
+  private def toString(maxFields: Int, append: String => Unit): Unit = {
+    writePlans(append, maxFields)
+  }
+
+  def stringWithStats: String = {
     val concat = new PlanStringConcat()
+    stringWithStats(SQLConf.get.maxToStringFields, concat.append)
+    withRedaction {
+      concat.toString
+    }
+  }
+
+  private def stringWithStats(maxFields: Int, append: String => Unit): Unit = {
     val maxFields = SQLConf.get.maxToStringFields
 
     // trigger to compute stats for logical plans
     try {
+      optimizedPlan.foreach(_.expressions.foreach(_.foreach {
+        case subqueryExpression: SubqueryExpression =>
+          // trigger subquery's child plan stats propagation
+          subqueryExpression.plan.stats
+        case _ =>
+      }))
       optimizedPlan.stats
     } catch {
-      case e: AnalysisException => concat.append(e.toString + "\n")
+      case e: AnalysisException => append(e.toString + "\n")
     }
     // only show optimized logical plan and physical plan
-    concat.append("== Optimized Logical Plan ==\n")
-    QueryPlan.append(optimizedPlan, concat.append, verbose = true, addSuffix = true, maxFields)
-    concat.append("\n== Physical Plan ==\n")
-    QueryPlan.append(executedPlan, concat.append, verbose = true, addSuffix = false, maxFields)
-    concat.append("\n")
-    concat.toString
+    append("== Optimized Logical Plan ==\n")
+    QueryPlan.append(optimizedPlan, append, verbose = true, addSuffix = true, maxFields)
+    append("\n== Physical Plan ==\n")
+    QueryPlan.append(executedPlan, append, verbose = true, addSuffix = false, maxFields)
+    append("\n")
   }
 
   /**
@@ -261,19 +308,26 @@ class QueryExecution(
     /**
      * Dumps debug information about query execution into the specified file.
      *
+     * @param path path of the file the debug info is written to.
      * @param maxFields maximum number of fields converted to string representation.
+     * @param explainMode the explain mode to be used to generate the string
+     *                    representation of the plan.
      */
-    def toFile(path: String, maxFields: Int = Int.MaxValue): Unit = {
+    def toFile(
+        path: String,
+        maxFields: Int = Int.MaxValue,
+        explainMode: Option[String] = None): Unit = {
       val filePath = new Path(path)
       val fs = filePath.getFileSystem(sparkSession.sessionState.newHadoopConf())
       val writer = new BufferedWriter(new OutputStreamWriter(fs.create(filePath)))
-      val append = (s: String) => {
-        writer.write(s)
-      }
       try {
-        writePlans(append, maxFields)
-        writer.write("\n== Whole Stage Codegen ==\n")
-        org.apache.spark.sql.execution.debug.writeCodegen(writer.write, executedPlan)
+        val mode = explainMode.map(ExplainMode.fromString(_)).getOrElse(ExtendedMode)
+        explainString(mode, maxFields, writer.write)
+        if (mode != CodegenMode) {
+          writer.write("\n== Whole Stage Codegen ==\n")
+          org.apache.spark.sql.execution.debug.writeCodegen(writer.write, executedPlan)
+        }
+        log.info(s"Debug information was written at: $filePath")
       } finally {
         writer.close()
       }
@@ -282,6 +336,10 @@ class QueryExecution(
 }
 
 object QueryExecution {
+  private val _nextExecutionId = new AtomicLong(0)
+
+  private def nextExecutionId: Long = _nextExecutionId.getAndIncrement
+
   /**
    * Construct a sequence of rules that are used to prepare a planned [[SparkPlan]] for execution.
    * These rules will make sure subqueries are planned, make use the data partitioning and ordering
@@ -295,14 +353,19 @@ object QueryExecution {
     // as the original plan is hidden behind `AdaptiveSparkPlanExec`.
     adaptiveExecutionRule.toSeq ++
     Seq(
+      CoalesceBucketsInJoin,
       PlanDynamicPruningFilters(sparkSession),
       PlanSubqueries(sparkSession),
-      EnsureRequirements(sparkSession.sessionState.conf),
-      ApplyColumnarRulesAndInsertTransitions(sparkSession.sessionState.conf,
-        sparkSession.sessionState.columnarRules),
-      CollapseCodegenStages(sparkSession.sessionState.conf),
-      ReuseExchange(sparkSession.sessionState.conf),
-      ReuseSubquery(sparkSession.sessionState.conf)
+      RemoveRedundantProjects,
+      EnsureRequirements,
+      // `RemoveRedundantSorts` needs to be added after `EnsureRequirements` to guarantee the same
+      // number of partitions when instantiating PartitioningCollection.
+      RemoveRedundantSorts,
+      DisableUnnecessaryBucketedScan,
+      ApplyColumnarRulesAndInsertTransitions(sparkSession.sessionState.columnarRules),
+      CollapseCodegenStages(),
+      ReuseExchange,
+      ReuseSubquery
     )
   }
 
@@ -313,7 +376,14 @@ object QueryExecution {
   private[execution] def prepareForExecution(
       preparations: Seq[Rule[SparkPlan]],
       plan: SparkPlan): SparkPlan = {
-    preparations.foldLeft(plan) { case (sp, rule) => rule.apply(sp) }
+    val planChangeLogger = new PlanChangeLogger[SparkPlan]()
+    val preparedPlan = preparations.foldLeft(plan) { case (sp, rule) =>
+      val result = rule.apply(sp)
+      planChangeLogger.logRule(rule.ruleName, sp, result)
+      result
+    }
+    planChangeLogger.logBatch("Preparations", plan, preparedPlan)
+    preparedPlan
   }
 
   /**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/RemoveRedundantProjects.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/RemoveRedundantProjects.scala
new file mode 100644
index 0000000000000..bfb6e805c0541
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/RemoveRedundantProjects.scala
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.expressions.aggregate.{Final, PartialMerge}
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.execution.aggregate.BaseAggregateExec
+import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase
+import org.apache.spark.sql.execution.joins.BaseJoinExec
+import org.apache.spark.sql.execution.window.WindowExec
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * Remove redundant ProjectExec node from the spark plan. A ProjectExec node is redundant when
+ * - It has the same output attributes and orders as its child's output and the ordering of
+ *   the attributes is required.
+ * - It has the same output attributes as its child's output when attribute output ordering
+ *   is not required.
+ * This rule needs to be a physical rule because project nodes are useful during logical
+ * optimization to prune data. During physical planning, redundant project nodes can be removed
+ * to simplify the query plan.
+ */
+object RemoveRedundantProjects extends Rule[SparkPlan] {
+  def apply(plan: SparkPlan): SparkPlan = {
+    if (!conf.getConf(SQLConf.REMOVE_REDUNDANT_PROJECTS_ENABLED)) {
+      plan
+    } else {
+      removeProject(plan, true)
+    }
+  }
+
+  private def removeProject(plan: SparkPlan, requireOrdering: Boolean): SparkPlan = {
+    plan match {
+      case p @ ProjectExec(_, child) =>
+        if (isRedundant(p, child, requireOrdering)) {
+          val newPlan = removeProject(child, requireOrdering)
+          newPlan.setLogicalLink(child.logicalLink.get)
+          newPlan
+        } else {
+          p.mapChildren(removeProject(_, false))
+        }
+      case op: TakeOrderedAndProjectExec =>
+        op.mapChildren(removeProject(_, false))
+      case a: BaseAggregateExec =>
+        // BaseAggregateExec require specific column ordering when mode is Final or PartialMerge.
+        // See comments in BaseAggregateExec inputAttributes method.
+        val keepOrdering = a.aggregateExpressions
+          .exists(ae => ae.mode.equals(Final) || ae.mode.equals(PartialMerge))
+        a.mapChildren(removeProject(_, keepOrdering))
+      case o =>
+        val required = if (canPassThrough(o)) requireOrdering else true
+        o.mapChildren(removeProject(_, requireOrdering = required))
+    }
+  }
+
+  /**
+   * Check if the given node can pass the ordering requirement from its parent.
+   */
+  private def canPassThrough(plan: SparkPlan): Boolean = plan match {
+    case _: FilterExec => true
+    // JoinExec ordering requirement should inherit from its parent. If there is no ProjectExec in
+    // its ancestors, JoinExec should require output columns to be ordered, and vice versa.
+    case _: BaseJoinExec => true
+    case _: WindowExec => true
+    case _: ExpandExec => true
+    case _ => false
+  }
+
+  /**
+   * Check if the nullability change is positive. It catches the case when the project output
+   * attribute is not nullable, but the child output attribute is nullable.
+   */
+  private def checkNullability(output: Seq[Attribute], childOutput: Seq[Attribute]): Boolean =
+    output.zip(childOutput).forall { case (attr1, attr2) => attr1.nullable || !attr2.nullable }
+
+  private def isRedundant(
+      project: ProjectExec,
+      child: SparkPlan,
+      requireOrdering: Boolean): Boolean = {
+    child match {
+      // If a DataSourceV2ScanExec node does not support columnar, a ProjectExec node is required
+      // to convert the rows to UnsafeRow. See DataSourceV2Strategy for more details.
+      case d: DataSourceV2ScanExecBase if !d.supportsColumnar => false
+      case _ =>
+        if (requireOrdering) {
+          project.output.map(_.exprId.id) == child.output.map(_.exprId.id) &&
+            checkNullability(project.output, child.output)
+        } else {
+          val orderedProjectOutput = project.output.sortBy(_.exprId.id)
+          val orderedChildOutput = child.output.sortBy(_.exprId.id)
+          orderedProjectOutput.map(_.exprId.id) == orderedChildOutput.map(_.exprId.id) &&
+            checkNullability(orderedProjectOutput, orderedChildOutput)
+        }
+    }
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/RemoveRedundantSorts.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/RemoveRedundantSorts.scala
new file mode 100644
index 0000000000000..87c08ec865fe9
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/RemoveRedundantSorts.scala
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import org.apache.spark.sql.catalyst.expressions.SortOrder
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * Remove redundant SortExec node from the spark plan. A sort node is redundant when
+ * its child satisfies both its sort orders and its required child distribution. Note
+ * this rule differs from the Optimizer rule EliminateSorts in that this rule also checks
+ * if the child satisfies the required distribution so that it is safe to remove not only a
+ * local sort but also a global sort when its child already satisfies required sort orders.
+ */
+object RemoveRedundantSorts extends Rule[SparkPlan] {
+  def apply(plan: SparkPlan): SparkPlan = {
+    if (!conf.getConf(SQLConf.REMOVE_REDUNDANT_SORTS_ENABLED)) {
+      plan
+    } else {
+      removeSorts(plan)
+    }
+  }
+
+  private def removeSorts(plan: SparkPlan): SparkPlan = plan transform {
+    case s @ SortExec(orders, _, child, _)
+        if SortOrder.orderingSatisfies(child.outputOrdering, orders) &&
+          child.outputPartitioning.satisfies(s.requiredChildDistribution.head) =>
+      child
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ShuffledRowRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ShuffledRowRDD.scala
index 5936492dd819c..ef84cd27a3d0b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ShuffledRowRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ShuffledRowRDD.scala
@@ -133,10 +133,9 @@ class ShuffledRowRDD(
       Array.tabulate(dependency.partitioner.numPartitions)(i => CoalescedPartitionSpec(i, i + 1)))
   }
 
-  if (SQLConf.get.fetchShuffleBlocksInBatch) {
-    dependency.rdd.context.setLocalProperty(
-      SortShuffleManager.FETCH_SHUFFLE_BLOCKS_IN_BATCH_ENABLED_KEY, "true")
-  }
+  dependency.rdd.context.setLocalProperty(
+    SortShuffleManager.FETCH_SHUFFLE_BLOCKS_IN_BATCH_ENABLED_KEY,
+    SQLConf.get.fetchShuffleBlocksInBatch.toString)
 
   override def getDependencies: Seq[Dependency[_]] = List(dependency)
 
@@ -191,7 +190,7 @@ class ShuffledRowRDD(
           sqlMetricsReporter)
 
       case PartialReducerPartitionSpec(reducerIndex, startMapIndex, endMapIndex, _) =>
-        SparkEnv.get.shuffleManager.getReaderForRange(
+        SparkEnv.get.shuffleManager.getReader(
           dependency.shuffleHandle,
           startMapIndex,
           endMapIndex,
@@ -201,7 +200,7 @@ class ShuffledRowRDD(
           sqlMetricsReporter)
 
       case PartialMapperPartitionSpec(mapIndex, startReducerIndex, endReducerIndex) =>
-        SparkEnv.get.shuffleManager.getReaderForRange(
+        SparkEnv.get.shuffleManager.getReader(
           dependency.shuffleHandle,
           mapIndex,
           mapIndex + 1,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SortExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SortExec.scala
index 6b6ca531c6d3b..984a45cd058ab 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SortExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SortExec.scala
@@ -202,4 +202,7 @@ case class SortExec(
     }
     super.cleanupResources()
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): SortExec =
+    copy(child = newChild)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala
index 33b86a2b5340c..dde5dc2be0556 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.connector.catalog.CatalogManager
 import org.apache.spark.sql.execution.datasources.PruneFileSourcePartitions
 import org.apache.spark.sql.execution.datasources.SchemaPruning
-import org.apache.spark.sql.execution.datasources.v2.V2ScanRelationPushDown
+import org.apache.spark.sql.execution.datasources.v2.{V2ScanRelationPushDown, V2Writes}
 import org.apache.spark.sql.execution.dynamicpruning.{CleanupDynamicPruningFilters, PartitionPruning}
 import org.apache.spark.sql.execution.python.{ExtractGroupingPythonUDFFromAggregate, ExtractPythonUDFFromAggregate, ExtractPythonUDFs}
 
@@ -37,7 +37,7 @@ class SparkOptimizer(
 
   override def earlyScanPushDownRules: Seq[Rule[LogicalPlan]] =
     // TODO: move SchemaPruning into catalyst
-    SchemaPruning :: V2ScanRelationPushDown :: PruneFileSourcePartitions :: Nil
+    SchemaPruning :: V2ScanRelationPushDown :: V2Writes :: PruneFileSourcePartitions :: Nil
 
   override def defaultBatches: Seq[Batch] = (preOptimizationBatches ++ super.defaultBatches :+
     Batch("Optimize Metadata Only Query", Once, OptimizeMetadataOnlyQuery(catalog)) :+
@@ -70,7 +70,8 @@ class SparkOptimizer(
     ExtractPythonUDFFromJoinCondition.ruleName :+
     ExtractPythonUDFFromAggregate.ruleName :+ ExtractGroupingPythonUDFFromAggregate.ruleName :+
     ExtractPythonUDFs.ruleName :+
-    V2ScanRelationPushDown.ruleName
+    V2ScanRelationPushDown.ruleName :+
+    V2Writes.ruleName
 
   /**
    * Optimization batches that are executed before the regular optimization batches (also before
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala
index ead8c00031112..40bf094856bca 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala
@@ -32,9 +32,8 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.QueryPlan
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.plans.physical._
-import org.apache.spark.sql.catalyst.trees.TreeNodeTag
+import org.apache.spark.sql.catalyst.trees.{BinaryLike, LeafLike, TreeNodeTag, UnaryLike}
 import org.apache.spark.sql.execution.metric.SQLMetric
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.vectorized.ColumnarBatch
 
 object SparkPlan {
@@ -135,7 +134,12 @@ abstract class SparkPlan extends QueryPlan[SparkPlan] with Logging with Serializ
   def longMetric(name: String): SQLMetric = metrics(name)
 
   // TODO: Move to `DistributedPlan`
-  /** Specifies how data is partitioned across different nodes in the cluster. */
+  /**
+   * Specifies how data is partitioned across different nodes in the cluster.
+   * Note this method may fail if it is invoked before `EnsureRequirements` is applied
+   * since `PartitioningCollection` requires all its partitionings to have
+   * the same number of partitions.
+   */
   def outputPartitioning: Partitioning = UnknownPartitioning(0) // TODO: WRONG WIDTH!
 
   /**
@@ -509,11 +513,11 @@ abstract class SparkPlan extends QueryPlan[SparkPlan] with Logging with Serializ
   }
 }
 
-trait LeafExecNode extends SparkPlan {
-  override final def children: Seq[SparkPlan] = Nil
+trait LeafExecNode extends SparkPlan with LeafLike[SparkPlan] {
+
   override def producedAttributes: AttributeSet = outputSet
   override def verboseStringWithOperatorId(): String = {
-    val argumentString = argString(SQLConf.get.maxToStringFields)
+    val argumentString = argString(conf.maxToStringFields)
     val outputStr = s"${ExplainUtils.generateFieldString("Output", output)}"
 
     if (argumentString.nonEmpty) {
@@ -538,12 +542,10 @@ object UnaryExecNode {
   }
 }
 
-trait UnaryExecNode extends SparkPlan {
-  def child: SparkPlan
+trait UnaryExecNode extends SparkPlan with UnaryLike[SparkPlan] {
 
-  override final def children: Seq[SparkPlan] = child :: Nil
   override def verboseStringWithOperatorId(): String = {
-    val argumentString = argString(SQLConf.get.maxToStringFields)
+    val argumentString = argString(conf.maxToStringFields)
     val inputStr = s"${ExplainUtils.generateFieldString("Input", child.output)}"
 
     if (argumentString.nonEmpty) {
@@ -561,13 +563,10 @@ trait UnaryExecNode extends SparkPlan {
   }
 }
 
-trait BinaryExecNode extends SparkPlan {
-  def left: SparkPlan
-  def right: SparkPlan
+trait BinaryExecNode extends SparkPlan with BinaryLike[SparkPlan] {
 
-  override final def children: Seq[SparkPlan] = Seq(left, right)
   override def verboseStringWithOperatorId(): String = {
-    val argumentString = argString(SQLConf.get.maxToStringFields)
+    val argumentString = argString(conf.maxToStringFields)
     val leftOutputStr = s"${ExplainUtils.generateFieldString("Left output", left.output)}"
     val rightOutputStr = s"${ExplainUtils.generateFieldString("Right output", right.output)}"
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala
index 357820a9d63d0..db587dd98685e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.execution
 
 import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, QueryStageExec}
+import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
 import org.apache.spark.sql.execution.exchange.ReusedExchangeExec
 import org.apache.spark.sql.execution.metric.SQLMetricInfo
 import org.apache.spark.sql.internal.SQLConf
@@ -56,6 +57,7 @@ private[execution] object SparkPlanInfo {
       case ReusedSubqueryExec(child) => child :: Nil
       case a: AdaptiveSparkPlanExec => a.executedPlan :: Nil
       case stage: QueryStageExec => stage.plan :: Nil
+      case inMemTab: InMemoryTableScanExec => inMemTab.relation.cachedPlan :: Nil
       case _ => plan.children ++ plan.subqueries
     }
     val metrics = plan.metrics.toSeq.map { case (key, metric) =>
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanner.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanner.scala
index 895eeedd86b8b..6994aaf47dfba 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanner.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanner.scala
@@ -18,18 +18,15 @@
 package org.apache.spark.sql.execution
 
 import org.apache.spark.sql._
+import org.apache.spark.sql.catalyst.SQLConfHelper
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.execution.adaptive.LogicalQueryStageStrategy
 import org.apache.spark.sql.execution.datasources.{DataSourceStrategy, FileSourceStrategy}
 import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Strategy
-import org.apache.spark.sql.internal.SQLConf
 
-class SparkPlanner(
-    val session: SparkSession,
-    val conf: SQLConf,
-    val experimentalMethods: ExperimentalMethods)
-  extends SparkStrategies {
+class SparkPlanner(val session: SparkSession, val experimentalMethods: ExperimentalMethods)
+  extends SparkStrategies with SQLConfHelper {
 
   def numPartitions: Int = conf.numShufflePartitions
 
@@ -40,12 +37,13 @@ class SparkPlanner(
       PythonEvals ::
       new DataSourceV2Strategy(session) ::
       FileSourceStrategy ::
-      DataSourceStrategy(conf) ::
+      DataSourceStrategy ::
       SpecialLimits ::
       Aggregation ::
       Window ::
       JoinSelection ::
       InMemoryScans ::
+      SparkScripts ::
       BasicOperators :: Nil)
 
   /**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkScriptTransformationExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkScriptTransformationExec.scala
new file mode 100644
index 0000000000000..c79cea40ab023
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkScriptTransformationExec.scala
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import java.io._
+
+import org.apache.hadoop.conf.Configuration
+
+import org.apache.spark.TaskContext
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.types._
+import org.apache.spark.util.CircularBuffer
+
+/**
+ * Transforms the input by forking and running the specified script.
+ *
+ * @param script the command that should be executed.
+ * @param output the attributes that are produced by the script.
+ * @param child logical plan whose output is transformed.
+ * @param ioschema the class set that defines how to handle input/output data.
+ */
+case class SparkScriptTransformationExec(
+    script: String,
+    output: Seq[Attribute],
+    child: SparkPlan,
+    ioschema: ScriptTransformationIOSchema)
+  extends BaseScriptTransformationExec {
+
+  override def processIterator(
+      inputIterator: Iterator[InternalRow],
+      hadoopConf: Configuration): Iterator[InternalRow] = {
+
+    val (outputStream, proc, inputStream, stderrBuffer) = initProc
+
+    val outputProjection = new InterpretedProjection(inputExpressionsWithoutSerde, child.output)
+
+    // This new thread will consume the ScriptTransformation's input rows and write them to the
+    // external process. That process's output will be read by this current thread.
+    val writerThread = SparkScriptTransformationWriterThread(
+      inputIterator.map(outputProjection),
+      inputExpressionsWithoutSerde.map(_.dataType),
+      ioschema,
+      outputStream,
+      proc,
+      stderrBuffer,
+      TaskContext.get(),
+      hadoopConf
+    )
+
+    val outputIterator =
+      createOutputIteratorWithoutSerde(writerThread, inputStream, proc, stderrBuffer)
+
+    writerThread.start()
+
+    outputIterator
+  }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): SparkScriptTransformationExec =
+    copy(child = newChild)
+}
+
+case class SparkScriptTransformationWriterThread(
+    iter: Iterator[InternalRow],
+    inputSchema: Seq[DataType],
+    ioSchema: ScriptTransformationIOSchema,
+    outputStream: OutputStream,
+    proc: Process,
+    stderrBuffer: CircularBuffer,
+    taskContext: TaskContext,
+    conf: Configuration)
+  extends BaseScriptTransformationWriterThread {
+
+  override def processRows(): Unit = {
+    processRowsWithoutSerde()
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
index 078813b7d631d..ea74ce6143a7b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
@@ -17,7 +17,8 @@
 
 package org.apache.spark.sql.execution
 
-import java.util.Locale
+import java.time.ZoneOffset
+import java.util.{Locale, TimeZone}
 import javax.ws.rs.core.UriBuilder
 
 import scala.collection.JavaConverters._
@@ -25,25 +26,26 @@ import scala.collection.JavaConverters._
 import org.antlr.v4.runtime.{ParserRuleContext, Token}
 import org.antlr.v4.runtime.tree.TerminalNode
 
-import org.apache.spark.sql.SaveMode
-import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
+import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.expressions.Expression
 import org.apache.spark.sql.catalyst.parser._
 import org.apache.spark.sql.catalyst.parser.SqlBaseParser._
 import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.util.DateTimeConstants
+import org.apache.spark.sql.errors.QueryParsingErrors
 import org.apache.spark.sql.execution.command._
 import org.apache.spark.sql.execution.datasources._
 import org.apache.spark.sql.internal.{HiveSerDe, SQLConf, VariableSubstitution}
-import org.apache.spark.sql.types.StructType
+import org.apache.spark.sql.internal.StaticSQLConf.CATALOG_IMPLEMENTATION
 
 /**
  * Concrete parser for Spark SQL statements.
  */
-class SparkSqlParser(conf: SQLConf) extends AbstractSqlParser(conf) {
-  val astBuilder = new SparkSqlAstBuilder(conf)
+class SparkSqlParser extends AbstractSqlParser {
+  val astBuilder = new SparkSqlAstBuilder()
 
-  private val substitutor = new VariableSubstitution(conf)
+  private val substitutor = new VariableSubstitution()
 
   protected override def parse[T](command: String)(toResult: SqlBaseParser => T): T = {
     super.parse(substitutor.substitute(command))(toResult)
@@ -53,9 +55,14 @@ class SparkSqlParser(conf: SQLConf) extends AbstractSqlParser(conf) {
 /**
  * Builder that converts an ANTLR ParseTree into a LogicalPlan/Expression/TableIdentifier.
  */
-class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) {
+class SparkSqlAstBuilder extends AstBuilder {
   import org.apache.spark.sql.catalyst.parser.ParserUtils._
 
+  private val configKeyValueDef = """([a-zA-Z_\d\\.:]+)\s*=([^;]*);*""".r
+  private val configKeyDef = """([a-zA-Z_\d\\.:]+)$""".r
+  private val configValueDef = """([^;]*);*""".r
+  private val strLiteralDef = """(".*?[^\\]"|'.*?[^\\]'|[^ \n\r\t"']+)""".r
+
   /**
    * Create a [[SetCommand]] logical plan.
    *
@@ -64,17 +71,42 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) {
    * character in the raw string.
    */
   override def visitSetConfiguration(ctx: SetConfigurationContext): LogicalPlan = withOrigin(ctx) {
-    // Construct the command.
-    val raw = remainder(ctx.SET.getSymbol)
-    val keyValueSeparatorIndex = raw.indexOf('=')
-    if (keyValueSeparatorIndex >= 0) {
-      val key = raw.substring(0, keyValueSeparatorIndex).trim
-      val value = raw.substring(keyValueSeparatorIndex + 1).trim
-      SetCommand(Some(key -> Option(value)))
-    } else if (raw.nonEmpty) {
-      SetCommand(Some(raw.trim -> None))
+    remainder(ctx.SET.getSymbol).trim match {
+      case configKeyValueDef(key, value) =>
+        SetCommand(Some(key -> Option(value.trim)))
+      case configKeyDef(key) =>
+        SetCommand(Some(key -> None))
+      case s if s == "-v" =>
+        SetCommand(Some("-v" -> None))
+      case s if s.isEmpty =>
+        SetCommand(None)
+      case _ => throw QueryParsingErrors.unexpectedFomatForSetConfigurationError(ctx)
+    }
+  }
+
+  override def visitSetQuotedConfiguration(
+      ctx: SetQuotedConfigurationContext): LogicalPlan = withOrigin(ctx) {
+    if (ctx.configValue() != null && ctx.configKey() != null) {
+      SetCommand(Some(ctx.configKey().getText -> Option(ctx.configValue().getText)))
+    } else if (ctx.configValue() != null) {
+      val valueStr = ctx.configValue().getText
+      val keyCandidate = interval(ctx.SET().getSymbol, ctx.EQ().getSymbol).trim
+      keyCandidate match {
+        case configKeyDef(key) => SetCommand(Some(key -> Option(valueStr)))
+        case _ => throw QueryParsingErrors.invalidPropertyKeyForSetQuotedConfigurationError(
+          keyCandidate, valueStr, ctx)
+      }
     } else {
-      SetCommand(None)
+      val keyStr = ctx.configKey().getText
+      if (ctx.EQ() != null) {
+        remainder(ctx.EQ().getSymbol).trim match {
+          case configValueDef(valueStr) => SetCommand(Some(keyStr -> Option(valueStr)))
+          case other => throw QueryParsingErrors.invalidPropertyValueForSetQuotedConfigurationError(
+            other, keyStr, ctx)
+        }
+      } else {
+        SetCommand(Some(keyStr -> None))
+      }
     }
   }
 
@@ -83,11 +115,56 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) {
    * Example SQL :
    * {{{
    *   RESET;
+   *   RESET spark.sql.session.timeZone;
    * }}}
    */
   override def visitResetConfiguration(
       ctx: ResetConfigurationContext): LogicalPlan = withOrigin(ctx) {
-    ResetCommand
+    remainder(ctx.RESET.getSymbol).trim match {
+      case configKeyDef(key) =>
+        ResetCommand(Some(key))
+      case s if s.trim.isEmpty =>
+        ResetCommand(None)
+      case _ => throw QueryParsingErrors.unexpectedFormatForResetConfigurationError(ctx)
+    }
+  }
+
+  override def visitResetQuotedConfiguration(
+      ctx: ResetQuotedConfigurationContext): LogicalPlan = withOrigin(ctx) {
+    ResetCommand(Some(ctx.configKey().getText))
+  }
+
+  /**
+   * Create a [[SetCommand]] logical plan to set [[SQLConf.SESSION_LOCAL_TIMEZONE]]
+   * Example SQL :
+   * {{{
+   *   SET TIME ZONE LOCAL;
+   *   SET TIME ZONE 'Asia/Shanghai';
+   *   SET TIME ZONE INTERVAL 10 HOURS;
+   * }}}
+   */
+  override def visitSetTimeZone(ctx: SetTimeZoneContext): LogicalPlan = withOrigin(ctx) {
+    val key = SQLConf.SESSION_LOCAL_TIMEZONE.key
+    if (ctx.interval != null) {
+      val interval = parseIntervalLiteral(ctx.interval)
+      if (interval.months != 0 || interval.days != 0 ||
+        math.abs(interval.microseconds) > 18 * DateTimeConstants.MICROS_PER_HOUR ||
+        interval.microseconds % DateTimeConstants.MICROS_PER_SECOND != 0) {
+        throw QueryParsingErrors.intervalValueOutOfRangeError(ctx.interval())
+      } else {
+        val seconds = (interval.microseconds / DateTimeConstants.MICROS_PER_SECOND).toInt
+        SetCommand(Some(key -> Some(ZoneOffset.ofTotalSeconds(seconds).toString)))
+      }
+    } else if (ctx.timezone != null) {
+      ctx.timezone.getType match {
+        case SqlBaseParser.LOCAL =>
+          SetCommand(Some(key -> Some(TimeZone.getDefault.getID)))
+        case _ =>
+          SetCommand(Some(key -> Some(string(ctx.STRING))))
+      }
+    } else {
+      throw QueryParsingErrors.invalidTimeZoneDisplacementValueError(ctx)
+    }
   }
 
   /**
@@ -197,16 +274,18 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) {
         operationNotAllowed("CREATE TEMPORARY TABLE IF NOT EXISTS", ctx)
       }
 
-      val (_, _, _, options, _, _) = visitCreateTableClauses(ctx.createTableClauses())
+      val (_, _, _, _, options, location, _, _) = visitCreateTableClauses(ctx.createTableClauses())
       val provider = Option(ctx.tableProvider).map(_.multipartIdentifier.getText).getOrElse(
-        throw new ParseException("CREATE TEMPORARY TABLE without a provider is not allowed.", ctx))
+        throw QueryParsingErrors.createTempTableNotSpecifyProviderError(ctx))
       val schema = Option(ctx.colTypeList()).map(createSchema)
 
       logWarning(s"CREATE TEMPORARY TABLE ... USING ... is deprecated, please use " +
           "CREATE TEMPORARY VIEW ... USING ... instead")
 
       val table = tableIdentifier(ident, "CREATE TEMPORARY VIEW", ctx)
-      CreateTempViewUsing(table, schema, replace = false, global = false, provider, options)
+      val optionsWithLocation = location.map(l => options + ("path" -> l)).getOrElse(options)
+      CreateTempViewUsing(table, schema, replace = false, global = false, provider,
+        optionsWithLocation)
     }
   }
 
@@ -229,14 +308,14 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) {
    */
   override def visitNestedConstantList(
       ctx: NestedConstantListContext): Seq[Seq[String]] = withOrigin(ctx) {
-    ctx.constantList.asScala.map(visitConstantList)
+    ctx.constantList.asScala.map(visitConstantList).toSeq
   }
 
   /**
    * Convert a constants list into a String sequence.
    */
   override def visitConstantList(ctx: ConstantListContext): Seq[String] = withOrigin(ctx) {
-    ctx.constant.asScala.map(visitStringConstant)
+    ctx.constant.asScala.map(v => visitStringConstant(v, legacyNullAsString = false)).toSeq
   }
 
   /**
@@ -256,7 +335,8 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) {
   }
 
   /**
-   * Create a [[AddFileCommand]], [[AddJarCommand]], [[ListFilesCommand]] or [[ListJarsCommand]]
+   * Create a [[AddFilesCommand]], [[AddJarsCommand]], [[AddArchivesCommand]],
+   * [[ListFilesCommand]], [[ListJarsCommand]] or [[ListArchivesCommand]]
    * command depending on the requested operation on resources.
    * Expected format:
    * {{{
@@ -270,180 +350,74 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) {
    *  - '/path/to/fileOrJar'
    */
   override def visitManageResource(ctx: ManageResourceContext): LogicalPlan = withOrigin(ctx) {
-    val mayebePaths = if (ctx.STRING != null) string(ctx.STRING) else remainder(ctx.identifier).trim
+    val rawArg = remainder(ctx.identifier).trim
+    val maybePaths = strLiteralDef.findAllIn(rawArg).toSeq.map {
+      case p if p.startsWith("\"") || p.startsWith("'") => unescapeSQLString(p)
+      case p => p
+    }
+
     ctx.op.getType match {
       case SqlBaseParser.ADD =>
         ctx.identifier.getText.toLowerCase(Locale.ROOT) match {
-          case "file" => AddFileCommand(mayebePaths)
-          case "jar" => AddJarCommand(mayebePaths)
+          case "files" | "file" => AddFilesCommand(maybePaths)
+          case "jars" | "jar" => AddJarsCommand(maybePaths)
+          case "archives" | "archive" => AddArchivesCommand(maybePaths)
           case other => operationNotAllowed(s"ADD with resource type '$other'", ctx)
         }
       case SqlBaseParser.LIST =>
         ctx.identifier.getText.toLowerCase(Locale.ROOT) match {
           case "files" | "file" =>
-            if (mayebePaths.length > 0) {
-              ListFilesCommand(mayebePaths.split("\\s+"))
+            if (maybePaths.length > 0) {
+              ListFilesCommand(maybePaths)
             } else {
               ListFilesCommand()
             }
           case "jars" | "jar" =>
-            if (mayebePaths.length > 0) {
-              ListJarsCommand(mayebePaths.split("\\s+"))
+            if (maybePaths.length > 0) {
+              ListJarsCommand(maybePaths)
             } else {
               ListJarsCommand()
             }
+          case "archives" | "archive" =>
+            if (maybePaths.length > 0) {
+              ListArchivesCommand(maybePaths)
+            } else {
+              ListArchivesCommand()
+            }
           case other => operationNotAllowed(s"LIST with resource type '$other'", ctx)
         }
       case _ => operationNotAllowed(s"Other types of operation on resources", ctx)
     }
   }
 
-  /**
-   * Create a Hive serde table, returning a [[CreateTable]] logical plan.
-   *
-   * This is a legacy syntax for Hive compatibility, we recommend users to use the Spark SQL
-   * CREATE TABLE syntax to create Hive serde table, e.g. "CREATE TABLE ... USING hive ..."
-   *
-   * Note: several features are currently not supported - temporary tables, bucketing,
-   * skewed columns and storage handlers (STORED BY).
-   *
-   * Expected format:
-   * {{{
-   *   CREATE [EXTERNAL] TABLE [IF NOT EXISTS] [db_name.]table_name
-   *   [(col1[:] data_type [COMMENT col_comment], ...)]
-   *   create_table_clauses
-   *   [AS select_statement];
-   *
-   *   create_table_clauses (order insensitive):
-   *     [COMMENT table_comment]
-   *     [PARTITIONED BY (col2[:] data_type [COMMENT col_comment], ...)]
-   *     [ROW FORMAT row_format]
-   *     [STORED AS file_format]
-   *     [LOCATION path]
-   *     [TBLPROPERTIES (property_name=property_value, ...)]
-   * }}}
-   */
-  override def visitCreateHiveTable(ctx: CreateHiveTableContext): LogicalPlan = withOrigin(ctx) {
-    val (ident, temp, ifNotExists, external) = visitCreateTableHeader(ctx.createTableHeader)
-    // TODO: implement temporary tables
-    if (temp) {
-      throw new ParseException(
-        "CREATE TEMPORARY TABLE is not supported yet. " +
-          "Please use CREATE TEMPORARY VIEW as an alternative.", ctx)
-    }
-    if (ctx.skewSpec.size > 0) {
-      operationNotAllowed("CREATE TABLE ... SKEWED BY", ctx)
-    }
-
-    checkDuplicateClauses(ctx.TBLPROPERTIES, "TBLPROPERTIES", ctx)
-    checkDuplicateClauses(ctx.PARTITIONED, "PARTITIONED BY", ctx)
-    checkDuplicateClauses(ctx.commentSpec(), "COMMENT", ctx)
-    checkDuplicateClauses(ctx.bucketSpec(), "CLUSTERED BY", ctx)
-    checkDuplicateClauses(ctx.createFileFormat, "STORED AS/BY", ctx)
-    checkDuplicateClauses(ctx.rowFormat, "ROW FORMAT", ctx)
-    checkDuplicateClauses(ctx.locationSpec, "LOCATION", ctx)
-
-    val dataCols = Option(ctx.columns).map(visitColTypeList).getOrElse(Nil)
-    val partitionCols = Option(ctx.partitionColumns).map(visitColTypeList).getOrElse(Nil)
-    val properties = Option(ctx.tableProps).map(visitPropertyKeyValues).getOrElse(Map.empty)
-    val selectQuery = Option(ctx.query).map(plan)
-    val bucketSpec = ctx.bucketSpec().asScala.headOption.map(visitBucketSpec)
-
-    // Note: Hive requires partition columns to be distinct from the schema, so we need
-    // to include the partition columns here explicitly
-    val schema = StructType(dataCols ++ partitionCols)
-
-    // Storage format
-    val defaultStorage = HiveSerDe.getDefaultStorage(conf)
-    validateRowFormatFileFormat(ctx.rowFormat.asScala, ctx.createFileFormat.asScala, ctx)
-    val fileStorage = ctx.createFileFormat.asScala.headOption.map(visitCreateFileFormat)
-      .getOrElse(CatalogStorageFormat.empty)
-    val rowStorage = ctx.rowFormat.asScala.headOption.map(visitRowFormat)
-      .getOrElse(CatalogStorageFormat.empty)
-    val location = visitLocationSpecList(ctx.locationSpec())
-    // If we are creating an EXTERNAL table, then the LOCATION field is required
-    if (external && location.isEmpty) {
-      operationNotAllowed("CREATE EXTERNAL TABLE must be accompanied by LOCATION", ctx)
-    }
-
-    val locUri = location.map(CatalogUtils.stringToURI(_))
-    val storage = CatalogStorageFormat(
-      locationUri = locUri,
-      inputFormat = fileStorage.inputFormat.orElse(defaultStorage.inputFormat),
-      outputFormat = fileStorage.outputFormat.orElse(defaultStorage.outputFormat),
-      serde = rowStorage.serde.orElse(fileStorage.serde).orElse(defaultStorage.serde),
-      compressed = false,
-      properties = rowStorage.properties ++ fileStorage.properties)
-    // If location is defined, we'll assume this is an external table.
-    // Otherwise, we may accidentally delete existing data.
-    val tableType = if (external || location.isDefined) {
-      CatalogTableType.EXTERNAL
+  private def toStorageFormat(
+      location: Option[String],
+      maybeSerdeInfo: Option[SerdeInfo],
+      ctx: ParserRuleContext): CatalogStorageFormat = {
+    if (maybeSerdeInfo.isEmpty) {
+      CatalogStorageFormat.empty.copy(locationUri = location.map(CatalogUtils.stringToURI))
     } else {
-      CatalogTableType.MANAGED
-    }
-
-    val name = tableIdentifier(ident, "CREATE TABLE ... STORED AS ...", ctx)
-
-    // TODO support the sql text - have a proper location for this!
-    val tableDesc = CatalogTable(
-      identifier = name,
-      tableType = tableType,
-      storage = storage,
-      schema = schema,
-      bucketSpec = bucketSpec,
-      provider = Some(DDLUtils.HIVE_PROVIDER),
-      partitionColumnNames = partitionCols.map(_.name),
-      properties = properties,
-      comment = visitCommentSpecList(ctx.commentSpec()))
-
-    val mode = if (ifNotExists) SaveMode.Ignore else SaveMode.ErrorIfExists
-
-    selectQuery match {
-      case Some(q) =>
-        // Don't allow explicit specification of schema for CTAS.
-        if (dataCols.nonEmpty) {
-          operationNotAllowed(
-            "Schema may not be specified in a Create Table As Select (CTAS) statement",
-            ctx)
-        }
-
-        // When creating partitioned table with CTAS statement, we can't specify data type for the
-        // partition columns.
-        if (partitionCols.nonEmpty) {
-          val errorMessage = "Create Partitioned Table As Select cannot specify data type for " +
-            "the partition columns of the target table."
-          operationNotAllowed(errorMessage, ctx)
-        }
-
-        // Hive CTAS supports dynamic partition by specifying partition column names.
-        val partitionColumnNames =
-          Option(ctx.partitionColumnNames)
-            .map(visitIdentifierList(_).toArray)
-            .getOrElse(Array.empty[String])
-
-        val tableDescWithPartitionColNames =
-          tableDesc.copy(partitionColumnNames = partitionColumnNames)
-
-        val hasStorageProperties = (ctx.createFileFormat.size != 0) || (ctx.rowFormat.size != 0)
-        if (conf.convertCTAS && !hasStorageProperties) {
-          // At here, both rowStorage.serdeProperties and fileStorage.serdeProperties
-          // are empty Maps.
-          val newTableDesc = tableDescWithPartitionColNames.copy(
-            storage = CatalogStorageFormat.empty.copy(locationUri = locUri),
-            provider = Some(conf.defaultDataSourceName))
-          CreateTable(newTableDesc, mode, Some(q))
-        } else {
-          CreateTable(tableDescWithPartitionColNames, mode, Some(q))
-        }
-      case None =>
-        // When creating partitioned table, we must specify data type for the partition columns.
-        if (Option(ctx.partitionColumnNames).isDefined) {
-          val errorMessage = "Must specify a data type for each partition column while creating " +
-            "Hive partitioned table."
-          operationNotAllowed(errorMessage, ctx)
+      val serdeInfo = maybeSerdeInfo.get
+      if (serdeInfo.storedAs.isEmpty) {
+        CatalogStorageFormat.empty.copy(
+          locationUri = location.map(CatalogUtils.stringToURI),
+          inputFormat = serdeInfo.formatClasses.map(_.input),
+          outputFormat = serdeInfo.formatClasses.map(_.output),
+          serde = serdeInfo.serde,
+          properties = serdeInfo.serdeProperties)
+      } else {
+        HiveSerDe.sourceToSerDe(serdeInfo.storedAs.get) match {
+          case Some(hiveSerde) =>
+            CatalogStorageFormat.empty.copy(
+              locationUri = location.map(CatalogUtils.stringToURI),
+              inputFormat = hiveSerde.inputFormat,
+              outputFormat = hiveSerde.outputFormat,
+              serde = serdeInfo.serde.orElse(hiveSerde.serde),
+              properties = serdeInfo.serdeProperties)
+          case _ =>
+            operationNotAllowed(s"STORED AS with file format '${serdeInfo.storedAs.get}'", ctx)
         }
-
-        CreateTable(tableDesc, mode, None)
+      }
     }
   }
 
@@ -474,193 +448,29 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) {
     checkDuplicateClauses(ctx.TBLPROPERTIES, "TBLPROPERTIES", ctx)
     val provider = ctx.tableProvider.asScala.headOption.map(_.multipartIdentifier.getText)
     val location = visitLocationSpecList(ctx.locationSpec())
-    // rowStorage used to determine CatalogStorageFormat.serde and
-    // CatalogStorageFormat.properties in STORED AS clause.
-    val rowStorage = ctx.rowFormat.asScala.headOption.map(visitRowFormat)
-      .getOrElse(CatalogStorageFormat.empty)
-    val fileFormat = ctx.createFileFormat.asScala.headOption.map(visitCreateFileFormat) match {
-      case Some(f) =>
-        if (provider.isDefined) {
-          throw new ParseException("'STORED AS hiveFormats' and 'USING provider' " +
-            "should not be specified both", ctx)
-        }
-        f.copy(
-          locationUri = location.map(CatalogUtils.stringToURI),
-          serde = rowStorage.serde.orElse(f.serde),
-          properties = rowStorage.properties ++ f.properties)
-      case None =>
-        if (rowStorage.serde.isDefined) {
-          throw new ParseException("'ROW FORMAT' must be used with 'STORED AS'", ctx)
-        }
-        CatalogStorageFormat.empty.copy(locationUri = location.map(CatalogUtils.stringToURI))
+    val serdeInfo = getSerdeInfo(
+      ctx.rowFormat.asScala.toSeq, ctx.createFileFormat.asScala.toSeq, ctx)
+    if (provider.isDefined && serdeInfo.isDefined) {
+      operationNotAllowed(s"CREATE TABLE LIKE ... USING ... ${serdeInfo.get.describe}", ctx)
     }
-    val properties = Option(ctx.tableProps).map(visitPropertyKeyValues).getOrElse(Map.empty)
-    CreateTableLikeCommand(
-      targetTable, sourceTable, fileFormat, provider, properties, ctx.EXISTS != null)
-  }
-
-  /**
-   * Create a [[CatalogStorageFormat]] for creating tables.
-   *
-   * Format: STORED AS ...
-   */
-  override def visitCreateFileFormat(
-      ctx: CreateFileFormatContext): CatalogStorageFormat = withOrigin(ctx) {
-    (ctx.fileFormat, ctx.storageHandler) match {
-      // Expected format: INPUTFORMAT input_format OUTPUTFORMAT output_format
-      case (c: TableFileFormatContext, null) =>
-        visitTableFileFormat(c)
-      // Expected format: SEQUENCEFILE | TEXTFILE | RCFILE | ORC | PARQUET | AVRO
-      case (c: GenericFileFormatContext, null) =>
-        visitGenericFileFormat(c)
-      case (null, storageHandler) =>
-        operationNotAllowed("STORED BY", ctx)
-      case _ =>
-        throw new ParseException("Expected either STORED AS or STORED BY, not both", ctx)
-    }
-  }
-
-  /**
-   * Create a [[CatalogStorageFormat]].
-   */
-  override def visitTableFileFormat(
-      ctx: TableFileFormatContext): CatalogStorageFormat = withOrigin(ctx) {
-    CatalogStorageFormat.empty.copy(
-      inputFormat = Option(string(ctx.inFmt)),
-      outputFormat = Option(string(ctx.outFmt)))
-  }
-
-  /**
-   * Resolve a [[HiveSerDe]] based on the name given and return it as a [[CatalogStorageFormat]].
-   */
-  override def visitGenericFileFormat(
-      ctx: GenericFileFormatContext): CatalogStorageFormat = withOrigin(ctx) {
-    val source = ctx.identifier.getText
-    HiveSerDe.sourceToSerDe(source) match {
-      case Some(s) =>
-        CatalogStorageFormat.empty.copy(
-          inputFormat = s.inputFormat,
-          outputFormat = s.outputFormat,
-          serde = s.serde)
-      case None =>
-        operationNotAllowed(s"STORED AS with file format '$source'", ctx)
-    }
-  }
-
-  /**
-   * Create a [[CatalogStorageFormat]] used for creating tables.
-   *
-   * Example format:
-   * {{{
-   *   SERDE serde_name [WITH SERDEPROPERTIES (k1=v1, k2=v2, ...)]
-   * }}}
-   *
-   * OR
-   *
-   * {{{
-   *   DELIMITED [FIELDS TERMINATED BY char [ESCAPED BY char]]
-   *   [COLLECTION ITEMS TERMINATED BY char]
-   *   [MAP KEYS TERMINATED BY char]
-   *   [LINES TERMINATED BY char]
-   *   [NULL DEFINED AS char]
-   * }}}
-   */
-  private def visitRowFormat(ctx: RowFormatContext): CatalogStorageFormat = withOrigin(ctx) {
-    ctx match {
-      case serde: RowFormatSerdeContext => visitRowFormatSerde(serde)
-      case delimited: RowFormatDelimitedContext => visitRowFormatDelimited(delimited)
-    }
-  }
-
-  /**
-   * Create SERDE row format name and properties pair.
-   */
-  override def visitRowFormatSerde(
-      ctx: RowFormatSerdeContext): CatalogStorageFormat = withOrigin(ctx) {
-    import ctx._
-    CatalogStorageFormat.empty.copy(
-      serde = Option(string(name)),
-      properties = Option(tablePropertyList).map(visitPropertyKeyValues).getOrElse(Map.empty))
-  }
-
-  /**
-   * Create a delimited row format properties object.
-   */
-  override def visitRowFormatDelimited(
-      ctx: RowFormatDelimitedContext): CatalogStorageFormat = withOrigin(ctx) {
-    // Collect the entries if any.
-    def entry(key: String, value: Token): Seq[(String, String)] = {
-      Option(value).toSeq.map(x => key -> string(x))
-    }
-    // TODO we need proper support for the NULL format.
-    val entries =
-      entry("field.delim", ctx.fieldsTerminatedBy) ++
-        entry("serialization.format", ctx.fieldsTerminatedBy) ++
-        entry("escape.delim", ctx.escapedBy) ++
-        // The following typo is inherited from Hive...
-        entry("colelction.delim", ctx.collectionItemsTerminatedBy) ++
-        entry("mapkey.delim", ctx.keysTerminatedBy) ++
-        Option(ctx.linesSeparatedBy).toSeq.map { token =>
-          val value = string(token)
-          validate(
-            value == "\n",
-            s"LINES TERMINATED BY only supports newline '\\n' right now: $value",
-            ctx)
-          "line.delim" -> value
-        }
-    CatalogStorageFormat.empty.copy(properties = entries.toMap)
-  }
 
-  /**
-   * Throw a [[ParseException]] if the user specified incompatible SerDes through ROW FORMAT
-   * and STORED AS.
-   *
-   * The following are allowed. Anything else is not:
-   *   ROW FORMAT SERDE ... STORED AS [SEQUENCEFILE | RCFILE | TEXTFILE]
-   *   ROW FORMAT DELIMITED ... STORED AS TEXTFILE
-   *   ROW FORMAT ... STORED AS INPUTFORMAT ... OUTPUTFORMAT ...
-   */
-  private def validateRowFormatFileFormat(
-      rowFormatCtx: RowFormatContext,
-      createFileFormatCtx: CreateFileFormatContext,
-      parentCtx: ParserRuleContext): Unit = {
-    if (rowFormatCtx == null || createFileFormatCtx == null) {
-      return
-    }
-    (rowFormatCtx, createFileFormatCtx.fileFormat) match {
-      case (_, ffTable: TableFileFormatContext) => // OK
-      case (rfSerde: RowFormatSerdeContext, ffGeneric: GenericFileFormatContext) =>
-        ffGeneric.identifier.getText.toLowerCase(Locale.ROOT) match {
-          case ("sequencefile" | "textfile" | "rcfile") => // OK
-          case fmt =>
-            operationNotAllowed(
-              s"ROW FORMAT SERDE is incompatible with format '$fmt', which also specifies a serde",
-              parentCtx)
-        }
-      case (rfDelimited: RowFormatDelimitedContext, ffGeneric: GenericFileFormatContext) =>
-        ffGeneric.identifier.getText.toLowerCase(Locale.ROOT) match {
-          case "textfile" => // OK
-          case fmt => operationNotAllowed(
-            s"ROW FORMAT DELIMITED is only compatible with 'textfile', not '$fmt'", parentCtx)
+    // For "CREATE TABLE dst LIKE src ROW FORMAT SERDE xxx" which doesn't specify the file format,
+    // it's a bit weird to use the default file format, but it's also weird to get file format
+    // from the source table while the serde class is user-specified.
+    // Here we require both serde and format to be specified, to avoid confusion.
+    serdeInfo match {
+      case Some(SerdeInfo(storedAs, formatClasses, serde, _)) =>
+        if (storedAs.isEmpty && formatClasses.isEmpty && serde.isDefined) {
+          throw QueryParsingErrors.rowFormatNotUsedWithStoredAsError(ctx)
         }
       case _ =>
-        // should never happen
-        def str(ctx: ParserRuleContext): String = {
-          (0 until ctx.getChildCount).map { i => ctx.getChild(i).getText }.mkString(" ")
-        }
-        operationNotAllowed(
-          s"Unexpected combination of ${str(rowFormatCtx)} and ${str(createFileFormatCtx)}",
-          parentCtx)
     }
-  }
 
-  private def validateRowFormatFileFormat(
-      rowFormatCtx: Seq[RowFormatContext],
-      createFileFormatCtx: Seq[CreateFileFormatContext],
-      parentCtx: ParserRuleContext): Unit = {
-    if (rowFormatCtx.size == 1 && createFileFormatCtx.size == 1) {
-      validateRowFormatFileFormat(rowFormatCtx.head, createFileFormatCtx.head, parentCtx)
-    }
+    val storage = toStorageFormat(location, serdeInfo, ctx)
+    val properties = Option(ctx.tableProps).map(visitPropertyKeyValues).getOrElse(Map.empty)
+    val cleanedProperties = cleanTableProperties(ctx, properties)
+    CreateTableLikeCommand(
+      targetTable, sourceTable, storage, provider, cleanedProperties, ctx.EXISTS != null)
   }
 
   /**
@@ -675,68 +485,66 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) {
       schemaLess: Boolean): ScriptInputOutputSchema = {
     if (recordWriter != null || recordReader != null) {
       // TODO: what does this message mean?
-      throw new ParseException(
-        "Unsupported operation: Used defined record reader/writer classes.", ctx)
+      throw QueryParsingErrors.useDefinedRecordReaderOrWriterClassesError(ctx)
     }
 
-    // Decode and input/output format.
-    type Format = (Seq[(String, String)], Option[String], Seq[(String, String)], Option[String])
-    def format(
-        fmt: RowFormatContext,
-        configKey: String,
-        defaultConfigValue: String): Format = fmt match {
-      case c: RowFormatDelimitedContext =>
-        // TODO we should use the visitRowFormatDelimited function here. However HiveScriptIOSchema
-        // expects a seq of pairs in which the old parsers' token names are used as keys.
-        // Transforming the result of visitRowFormatDelimited would be quite a bit messier than
-        // retrieving the key value pairs ourselves.
-        def entry(key: String, value: Token): Seq[(String, String)] = {
-          Option(value).map(t => key -> t.getText).toSeq
-        }
-        val entries = entry("TOK_TABLEROWFORMATFIELD", c.fieldsTerminatedBy) ++
-          entry("TOK_TABLEROWFORMATCOLLITEMS", c.collectionItemsTerminatedBy) ++
-          entry("TOK_TABLEROWFORMATMAPKEYS", c.keysTerminatedBy) ++
-          entry("TOK_TABLEROWFORMATLINES", c.linesSeparatedBy) ++
-          entry("TOK_TABLEROWFORMATNULL", c.nullDefinedAs)
-
-        (entries, None, Seq.empty, None)
-
-      case c: RowFormatSerdeContext =>
-        // Use a serde format.
-        val CatalogStorageFormat(None, None, None, Some(name), _, props) = visitRowFormatSerde(c)
-
-        // SPARK-10310: Special cases LazySimpleSerDe
-        val recordHandler = if (name == "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe") {
-          Option(conf.getConfString(configKey, defaultConfigValue))
-        } else {
-          None
-        }
-        (Seq.empty, Option(name), props.toSeq, recordHandler)
-
-      case null =>
-        // Use default (serde) format.
-        val name = conf.getConfString("hive.script.serde",
-          "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe")
-        val props = Seq("field.delim" -> "\t")
-        val recordHandler = Option(conf.getConfString(configKey, defaultConfigValue))
-        (Nil, Option(name), props, recordHandler)
-    }
+    if (!conf.getConf(CATALOG_IMPLEMENTATION).equals("hive")) {
+      super.withScriptIOSchema(
+        ctx,
+        inRowFormat,
+        recordWriter,
+        outRowFormat,
+        recordReader,
+        schemaLess)
+    } else {
+      def format(
+          fmt: RowFormatContext,
+          configKey: String,
+          defaultConfigValue: String): ScriptIOFormat = fmt match {
+        case c: RowFormatDelimitedContext =>
+          getRowFormatDelimited(c)
+
+        case c: RowFormatSerdeContext =>
+          // Use a serde format.
+          val SerdeInfo(None, None, Some(name), props) = visitRowFormatSerde(c)
+
+          // SPARK-10310: Special cases LazySimpleSerDe
+          val recordHandler = if (name == "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe") {
+            Option(conf.getConfString(configKey, defaultConfigValue))
+          } else {
+            None
+          }
+          val finalProps = props ++ Seq("field.delim" -> props.getOrElse("field.delim", "\t"))
+          (Seq.empty, Option(name), finalProps.toSeq, recordHandler)
+
+        case null =>
+          // Use default (serde) format.
+          val name = conf.getConfString("hive.script.serde",
+            "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe")
+          val props = Seq(
+            "field.delim" -> "\t",
+            "serialization.last.column.takes.rest" -> "true")
+          val recordHandler = Option(conf.getConfString(configKey, defaultConfigValue))
+          (Nil, Option(name), props, recordHandler)
+      }
 
-    val (inFormat, inSerdeClass, inSerdeProps, reader) =
-      format(
-        inRowFormat, "hive.script.recordreader", "org.apache.hadoop.hive.ql.exec.TextRecordReader")
-
-    val (outFormat, outSerdeClass, outSerdeProps, writer) =
-      format(
-        outRowFormat, "hive.script.recordwriter",
-        "org.apache.hadoop.hive.ql.exec.TextRecordWriter")
-
-    ScriptInputOutputSchema(
-      inFormat, outFormat,
-      inSerdeClass, outSerdeClass,
-      inSerdeProps, outSerdeProps,
-      reader, writer,
-      schemaLess)
+      val (inFormat, inSerdeClass, inSerdeProps, reader) =
+        format(
+          inRowFormat, "hive.script.recordreader",
+          "org.apache.hadoop.hive.ql.exec.TextRecordReader")
+
+      val (outFormat, outSerdeClass, outSerdeProps, writer) =
+        format(
+          outRowFormat, "hive.script.recordwriter",
+          "org.apache.hadoop.hive.ql.exec.TextRecordWriter")
+
+      ScriptInputOutputSchema(
+        inFormat, outFormat,
+        inSerdeClass, outSerdeClass,
+        inSerdeProps, outSerdeProps,
+        reader, writer,
+        schemaLess)
+    }
   }
 
   /**
@@ -746,7 +554,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) {
       ctx: QueryOrganizationContext,
       expressions: Seq[Expression],
       query: LogicalPlan): LogicalPlan = {
-    RepartitionByExpression(expressions, query, conf.numShufflePartitions)
+    RepartitionByExpression(expressions, query, None)
   }
 
   /**
@@ -768,8 +576,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) {
     val path = Option(ctx.path).map(string).getOrElse("")
 
     if (!(path.isEmpty ^ storage.locationUri.isEmpty)) {
-      throw new ParseException(
-        "Directory path and 'path' in OPTIONS should be specified one, but not both", ctx)
+      throw QueryParsingErrors.directoryPathAndOptionsPathBothSpecifiedError(ctx)
     }
 
     if (!path.isEmpty) {
@@ -781,12 +588,12 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) {
       // assert if directory is local when LOCAL keyword is mentioned
       val scheme = Option(storage.locationUri.get.getScheme)
       scheme match {
-        case None =>
+        case Some(pathScheme) if (!pathScheme.equals("file")) =>
+          throw QueryParsingErrors.unsupportedLocalFileSchemeError(ctx)
+        case _ =>
           // force scheme to be file rather than fs.default.name
           val loc = Some(UriBuilder.fromUri(CatalogUtils.stringToURI(path)).scheme("file").build())
           storage = storage.copy(locationUri = loc)
-        case Some(pathScheme) if (!pathScheme.equals("file")) =>
-          throw new ParseException("LOCAL is supported only with file: scheme", ctx)
       }
     }
 
@@ -809,28 +616,21 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) {
    */
   override def visitInsertOverwriteHiveDir(
       ctx: InsertOverwriteHiveDirContext): InsertDirParams = withOrigin(ctx) {
-    validateRowFormatFileFormat(ctx.rowFormat, ctx.createFileFormat, ctx)
-    val rowStorage = Option(ctx.rowFormat).map(visitRowFormat)
-      .getOrElse(CatalogStorageFormat.empty)
-    val fileStorage = Option(ctx.createFileFormat).map(visitCreateFileFormat)
-      .getOrElse(CatalogStorageFormat.empty)
-
+    val serdeInfo = getSerdeInfo(
+      Option(ctx.rowFormat).toSeq, Option(ctx.createFileFormat).toSeq, ctx)
     val path = string(ctx.path)
     // The path field is required
     if (path.isEmpty) {
       operationNotAllowed("INSERT OVERWRITE DIRECTORY must be accompanied by path", ctx)
     }
 
-    val defaultStorage = HiveSerDe.getDefaultStorage(conf)
-
-    val storage = CatalogStorageFormat(
-      locationUri = Some(CatalogUtils.stringToURI(path)),
-      inputFormat = fileStorage.inputFormat.orElse(defaultStorage.inputFormat),
-      outputFormat = fileStorage.outputFormat.orElse(defaultStorage.outputFormat),
-      serde = rowStorage.serde.orElse(fileStorage.serde).orElse(defaultStorage.serde),
-      compressed = false,
-      properties = rowStorage.properties ++ fileStorage.properties)
+    val default = HiveSerDe.getDefaultStorage(conf)
+    val storage = toStorageFormat(Some(path), serdeInfo, ctx)
+    val finalStorage = storage.copy(
+      inputFormat = storage.inputFormat.orElse(default.inputFormat),
+      outputFormat = storage.outputFormat.orElse(default.outputFormat),
+      serde = storage.serde.orElse(default.serde))
 
-    (ctx.LOCAL != null, storage, Some(DDLUtils.HIVE_PROVIDER))
+    (ctx.LOCAL != null, finalStorage, Some(DDLUtils.HIVE_PROVIDER))
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
index 12a1a1e7fc16e..b9e44b3e5b2e2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
@@ -18,27 +18,26 @@
 package org.apache.spark.sql.execution
 
 import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.{execution, AnalysisException, Strategy}
+import org.apache.spark.sql.{execution, Strategy}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.encoders.RowEncoder
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
-import org.apache.spark.sql.catalyst.optimizer.NormalizeFloatingNumbers
+import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, JoinSelectionHelper, NormalizeFloatingNumbers}
 import org.apache.spark.sql.catalyst.planning._
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.catalyst.plans.physical._
-import org.apache.spark.sql.catalyst.streaming.InternalOutputModes
+import org.apache.spark.sql.catalyst.streaming.{InternalOutputModes, StreamingRelationV2}
+import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
 import org.apache.spark.sql.execution.aggregate.AggUtils
 import org.apache.spark.sql.execution.columnar.{InMemoryRelation, InMemoryTableScanExec}
 import org.apache.spark.sql.execution.command._
-import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
-import org.apache.spark.sql.execution.joins.{BuildLeft, BuildRight, BuildSide}
+import org.apache.spark.sql.execution.exchange.{REPARTITION, REPARTITION_WITH_NUM, ShuffleExchangeExec}
 import org.apache.spark.sql.execution.python._
 import org.apache.spark.sql.execution.streaming._
 import org.apache.spark.sql.execution.streaming.sources.MemoryPlan
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.streaming.{OutputMode, StreamingQuery}
+import org.apache.spark.sql.streaming.OutputMode
 import org.apache.spark.sql.types.StructType
 
 /**
@@ -117,7 +116,9 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
    *
    * - Shuffle hash join:
    *     Only supported for equi-joins, while the join keys do not need to be sortable.
-   *     Supported for all join types except full outer joins.
+   *     Supported for all join types.
+   *     Building hash map from table is a memory-intensive operation and it could cause OOM
+   *     when the build side is big.
    *
    * - Shuffle sort merge join (SMJ):
    *     Only supported for equi-joins and the join keys have to be sortable.
@@ -135,93 +136,9 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
    *     Supports both equi-joins and non-equi-joins.
    *     Supports only inner like joins.
    */
-  object JoinSelection extends Strategy with PredicateHelper {
-
-    /**
-     * Matches a plan whose output should be small enough to be used in broadcast join.
-     */
-    private def canBroadcast(plan: LogicalPlan): Boolean = {
-      plan.stats.sizeInBytes >= 0 && plan.stats.sizeInBytes <= conf.autoBroadcastJoinThreshold
-    }
-
-    /**
-     * Matches a plan whose single partition should be small enough to build a hash table.
-     *
-     * Note: this assume that the number of partition is fixed, requires additional work if it's
-     * dynamic.
-     */
-    private def canBuildLocalHashMap(plan: LogicalPlan): Boolean = {
-      plan.stats.sizeInBytes < conf.autoBroadcastJoinThreshold * conf.numShufflePartitions
-    }
-
-    /**
-     * Returns whether plan a is much smaller (3X) than plan b.
-     *
-     * The cost to build hash map is higher than sorting, we should only build hash map on a table
-     * that is much smaller than other one. Since we does not have the statistic for number of rows,
-     * use the size of bytes here as estimation.
-     */
-    private def muchSmaller(a: LogicalPlan, b: LogicalPlan): Boolean = {
-      a.stats.sizeInBytes * 3 <= b.stats.sizeInBytes
-    }
-
-    private def canBuildRight(joinType: JoinType): Boolean = joinType match {
-      case _: InnerLike | LeftOuter | LeftSemi | LeftAnti | _: ExistenceJoin => true
-      case _ => false
-    }
-
-    private def canBuildLeft(joinType: JoinType): Boolean = joinType match {
-      case _: InnerLike | RightOuter => true
-      case _ => false
-    }
-
-    private def getBuildSide(
-        wantToBuildLeft: Boolean,
-        wantToBuildRight: Boolean,
-        left: LogicalPlan,
-        right: LogicalPlan): Option[BuildSide] = {
-      if (wantToBuildLeft && wantToBuildRight) {
-        // returns the smaller side base on its estimated physical size, if we want to build the
-        // both sides.
-        Some(getSmallerSide(left, right))
-      } else if (wantToBuildLeft) {
-        Some(BuildLeft)
-      } else if (wantToBuildRight) {
-        Some(BuildRight)
-      } else {
-        None
-      }
-    }
-
-    private def getSmallerSide(left: LogicalPlan, right: LogicalPlan) = {
-      if (right.stats.sizeInBytes <= left.stats.sizeInBytes) BuildRight else BuildLeft
-    }
-
-    private def hintToBroadcastLeft(hint: JoinHint): Boolean = {
-      hint.leftHint.exists(_.strategy.contains(BROADCAST))
-    }
-
-    private def hintToBroadcastRight(hint: JoinHint): Boolean = {
-      hint.rightHint.exists(_.strategy.contains(BROADCAST))
-    }
-
-    private def hintToShuffleHashLeft(hint: JoinHint): Boolean = {
-      hint.leftHint.exists(_.strategy.contains(SHUFFLE_HASH))
-    }
-
-    private def hintToShuffleHashRight(hint: JoinHint): Boolean = {
-      hint.rightHint.exists(_.strategy.contains(SHUFFLE_HASH))
-    }
-
-    private def hintToSortMergeJoin(hint: JoinHint): Boolean = {
-      hint.leftHint.exists(_.strategy.contains(SHUFFLE_MERGE)) ||
-        hint.rightHint.exists(_.strategy.contains(SHUFFLE_MERGE))
-    }
-
-    private def hintToShuffleReplicateNL(hint: JoinHint): Boolean = {
-      hint.leftHint.exists(_.strategy.contains(SHUFFLE_REPLICATE_NL)) ||
-        hint.rightHint.exists(_.strategy.contains(SHUFFLE_REPLICATE_NL))
-    }
+  object JoinSelection extends Strategy
+    with PredicateHelper
+    with JoinSelectionHelper {
 
     def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match {
 
@@ -244,41 +161,39 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
       //   4. Pick cartesian product if join type is inner like.
       //   5. Pick broadcast nested loop join as the final solution. It may OOM but we don't have
       //      other choice.
-      case ExtractEquiJoinKeys(joinType, leftKeys, rightKeys, condition, left, right, hint) =>
-        def createBroadcastHashJoin(buildLeft: Boolean, buildRight: Boolean) = {
-          val wantToBuildLeft = canBuildLeft(joinType) && buildLeft
-          val wantToBuildRight = canBuildRight(joinType) && buildRight
-          getBuildSide(wantToBuildLeft, wantToBuildRight, left, right).map { buildSide =>
-            Seq(joins.BroadcastHashJoinExec(
-              leftKeys,
-              rightKeys,
-              joinType,
-              buildSide,
-              condition,
-              planLater(left),
-              planLater(right)))
+      case j @ ExtractEquiJoinKeys(joinType, leftKeys, rightKeys, nonEquiCond, left, right, hint) =>
+        def createBroadcastHashJoin(onlyLookingAtHint: Boolean) = {
+          getBroadcastBuildSide(left, right, joinType, hint, onlyLookingAtHint, conf).map {
+            buildSide =>
+              Seq(joins.BroadcastHashJoinExec(
+                leftKeys,
+                rightKeys,
+                joinType,
+                buildSide,
+                nonEquiCond,
+                planLater(left),
+                planLater(right)))
           }
         }
 
-        def createShuffleHashJoin(buildLeft: Boolean, buildRight: Boolean) = {
-          val wantToBuildLeft = canBuildLeft(joinType) && buildLeft
-          val wantToBuildRight = canBuildRight(joinType) && buildRight
-          getBuildSide(wantToBuildLeft, wantToBuildRight, left, right).map { buildSide =>
-            Seq(joins.ShuffledHashJoinExec(
-              leftKeys,
-              rightKeys,
-              joinType,
-              buildSide,
-              condition,
-              planLater(left),
-              planLater(right)))
+        def createShuffleHashJoin(onlyLookingAtHint: Boolean) = {
+          getShuffleHashJoinBuildSide(left, right, joinType, hint, onlyLookingAtHint, conf).map {
+            buildSide =>
+              Seq(joins.ShuffledHashJoinExec(
+                leftKeys,
+                rightKeys,
+                joinType,
+                buildSide,
+                nonEquiCond,
+                planLater(left),
+                planLater(right)))
           }
         }
 
         def createSortMergeJoin() = {
           if (RowOrdering.isOrderable(leftKeys)) {
             Some(Seq(joins.SortMergeJoinExec(
-              leftKeys, rightKeys, joinType, condition, planLater(left), planLater(right))))
+              leftKeys, rightKeys, joinType, nonEquiCond, planLater(left), planLater(right))))
           } else {
             None
           }
@@ -286,21 +201,19 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
 
         def createCartesianProduct() = {
           if (joinType.isInstanceOf[InnerLike]) {
-            Some(Seq(joins.CartesianProductExec(planLater(left), planLater(right), condition)))
+            // `CartesianProductExec` can't implicitly evaluate equal join condition, here we should
+            // pass the original condition which includes both equal and non-equal conditions.
+            Some(Seq(joins.CartesianProductExec(planLater(left), planLater(right), j.condition)))
           } else {
             None
           }
         }
 
         def createJoinWithoutHint() = {
-          createBroadcastHashJoin(
-            canBroadcast(left) && !hint.leftHint.exists(_.strategy.contains(NO_BROADCAST_HASH)),
-            canBroadcast(right) && !hint.rightHint.exists(_.strategy.contains(NO_BROADCAST_HASH)))
+          createBroadcastHashJoin(false)
             .orElse {
               if (!conf.preferSortMergeJoin) {
-                createShuffleHashJoin(
-                  canBuildLocalHashMap(left) && muchSmaller(left, right),
-                  canBuildLocalHashMap(right) && muchSmaller(right, left))
+                createShuffleHashJoin(false)
               } else {
                 None
               }
@@ -311,16 +224,20 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
               // This join could be very slow or OOM
               val buildSide = getSmallerSide(left, right)
               Seq(joins.BroadcastNestedLoopJoinExec(
-                planLater(left), planLater(right), buildSide, joinType, condition))
+                planLater(left), planLater(right), buildSide, joinType, nonEquiCond))
             }
         }
 
-        createBroadcastHashJoin(hintToBroadcastLeft(hint), hintToBroadcastRight(hint))
+        createBroadcastHashJoin(true)
           .orElse { if (hintToSortMergeJoin(hint)) createSortMergeJoin() else None }
-          .orElse(createShuffleHashJoin(hintToShuffleHashLeft(hint), hintToShuffleHashRight(hint)))
+          .orElse(createShuffleHashJoin(true))
           .orElse { if (hintToShuffleReplicateNL(hint)) createCartesianProduct() else None }
           .getOrElse(createJoinWithoutHint())
 
+      case j @ ExtractSingleColumnNullAwareAntiJoin(leftKeys, rightKeys) =>
+        Seq(joins.BroadcastHashJoinExec(leftKeys, rightKeys, LeftAnti, BuildRight,
+          None, planLater(j.left), planLater(j.right), isNullAwareAntiJoin = true))
+
       // If it is not an equi-join, we first look at the join hints w.r.t. the following order:
       //   1. broadcast hint: pick broadcast nested loop join. If both sides have the broadcast
       //      hints, choose the smaller side (based on stats) to broadcast for inner and full joins,
@@ -345,7 +262,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
           // it's a right join, and broadcast right side if it's a left join.
           // TODO: revisit it. If left side is much smaller than the right side, it may be better
           // to broadcast the left side even if it's a left join.
-          if (canBuildLeft(joinType)) BuildLeft else BuildRight
+          if (canBuildBroadcastLeft(joinType)) BuildLeft else BuildRight
         }
 
         def createBroadcastNLJoin(buildLeft: Boolean, buildRight: Boolean) = {
@@ -374,7 +291,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
         }
 
         def createJoinWithoutHint() = {
-          createBroadcastNLJoin(canBroadcast(left), canBroadcast(right))
+          createBroadcastNLJoin(canBroadcastBySize(left, conf), canBroadcastBySize(right, conf))
             .orElse(createCartesianProduct())
             .getOrElse {
               // This join could be very slow or OOM
@@ -395,8 +312,9 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
 
   /**
    * Used to plan streaming aggregation queries that are computed incrementally as part of a
-   * [[StreamingQuery]]. Currently this rule is injected into the planner
-   * on-demand, only when planning in a [[org.apache.spark.sql.execution.streaming.StreamExecution]]
+   * [[org.apache.spark.sql.streaming.StreamingQuery]]. Currently this rule is injected into the
+   * planner on-demand, only when planning in a
+   * [[org.apache.spark.sql.execution.streaming.StreamExecution]]
    */
   object StatefulAggregationStrategy extends Strategy {
     override def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match {
@@ -409,8 +327,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
         namedGroupingExpressions, aggregateExpressions, rewrittenResultExpressions, child) =>
 
         if (aggregateExpressions.exists(PythonUDF.isGroupedAggPandasUDF)) {
-          throw new AnalysisException(
-            "Streaming aggregation doesn't support group aggregate pandas UDF")
+          throw QueryCompilationErrors.groupAggPandasUDFUnsupportedByStreamingAggError()
         }
 
         val stateVersion = conf.getConf(SQLConf.STREAMING_AGGREGATION_STATE_FORMAT_VERSION)
@@ -495,8 +412,8 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
             stateVersion, planLater(left), planLater(right)) :: Nil
 
         case Join(left, right, _, _, _) if left.isStreaming && right.isStreaming =>
-          throw new AnalysisException(
-            "Stream-stream join without equality predicate is not supported", plan = Some(plan))
+          throw QueryCompilationErrors.streamJoinStreamWithoutEqualityPredicateUnsupportedError(
+            plan)
 
         case _ => Nil
       }
@@ -515,7 +432,8 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
 
         val (functionsWithDistinct, functionsWithoutDistinct) =
           aggregateExpressions.partition(_.isDistinct)
-        if (functionsWithDistinct.map(_.aggregateFunction.children.toSet).distinct.length > 1) {
+        if (functionsWithDistinct.map(
+          _.aggregateFunction.children.filterNot(_.foldable).toSet).distinct.length > 1) {
           // This is a sanity check. We should not reach here when we have multiple distinct
           // column sets. Our `RewriteDistinctAggregates` should take care this case.
           sys.error("You hit a query analyzer bug. Please report your query to " +
@@ -527,6 +445,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
         val normalizedGroupingExpressions = groupingExpressions.map { e =>
           NormalizeFloatingNumbers.normalize(e) match {
             case n: NamedExpression => n
+            // Keep the name of the original expression.
             case other => Alias(other, e.name)(exprId = e.exprId)
           }
         }
@@ -539,10 +458,35 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
               resultExpressions,
               planLater(child))
           } else {
+            // functionsWithDistinct is guaranteed to be non-empty. Even though it may contain
+            // more than one DISTINCT aggregate function, all of those functions will have the
+            // same column expressions. For example, it would be valid for functionsWithDistinct
+            // to be [COUNT(DISTINCT foo), MAX(DISTINCT foo)], but
+            // [COUNT(DISTINCT bar), COUNT(DISTINCT foo)] is disallowed because those two distinct
+            // aggregates have different column expressions.
+            val distinctExpressions =
+              functionsWithDistinct.head.aggregateFunction.children.filterNot(_.foldable)
+            val normalizedNamedDistinctExpressions = distinctExpressions.map { e =>
+              // Ideally this should be done in `NormalizeFloatingNumbers`, but we do it here
+              // because `distinctExpressions` is not extracted during logical phase.
+              NormalizeFloatingNumbers.normalize(e) match {
+                case ne: NamedExpression => ne
+                case other =>
+                  // Keep the name of the original expression.
+                  val name = e match {
+                    case ne: NamedExpression => ne.name
+                    case _ => e.toString
+                  }
+                  Alias(other, name)()
+              }
+            }
+
             AggUtils.planAggregateWithOneDistinct(
               normalizedGroupingExpressions,
               functionsWithDistinct,
               functionsWithoutDistinct,
+              distinctExpressions,
+              normalizedNamedDistinctExpressions,
               resultExpressions,
               planLater(child))
           }
@@ -561,8 +505,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
 
       case PhysicalAggregation(_, _, _, _) =>
         // If cannot match the two cases above, then it's an error
-        throw new AnalysisException(
-          "Cannot use a mixture of aggregate function and group aggregate pandas UDF")
+        throw QueryCompilationErrors.cannotUseMixtureOfAggFunctionAndGroupAggPandasUDFError()
 
       case _ => Nil
     }
@@ -649,6 +592,19 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
     }
   }
 
+  object SparkScripts extends Strategy {
+    def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match {
+      case logical.ScriptTransformation(script, output, child, ioschema) =>
+        SparkScriptTransformationExec(
+          script,
+          output,
+          planLater(child),
+          ScriptTransformationIOSchema(ioschema)
+        ) :: Nil
+      case _ => Nil
+    }
+  }
+
   object BasicOperators extends Strategy {
     def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match {
       case d: DataWritingCommand => DataWritingCommandExec(d, planLater(d.query)) :: Nil
@@ -679,6 +635,10 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
       case logical.ResolvedHint(child, hints) =>
         throw new IllegalStateException(
           "ResolvedHint operator should have been replaced by join hint in the optimizer")
+      case Deduplicate(_, child) if !child.isStreaming =>
+        throw new IllegalStateException(
+          "Deduplicate operator for non streaming data source should have been replaced " +
+            "by aggregate in the optimizer")
 
       case logical.DeserializeToObject(deserializer, objAttr, child) =>
         execution.DeserializeToObjectExec(deserializer, objAttr, planLater(child)) :: Nil
@@ -700,9 +660,10 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
           f, p, b, is, ot, planLater(child)) :: Nil
       case logical.FlatMapGroupsInPandas(grouping, func, output, child) =>
         execution.python.FlatMapGroupsInPandasExec(grouping, func, output, planLater(child)) :: Nil
-      case logical.FlatMapCoGroupsInPandas(leftGroup, rightGroup, func, output, left, right) =>
+      case f @ logical.FlatMapCoGroupsInPandas(_, _, func, output, left, right) =>
         execution.python.FlatMapCoGroupsInPandasExec(
-          leftGroup, rightGroup, func, output, planLater(left), planLater(right)) :: Nil
+          f.leftAttributes, f.rightAttributes,
+          func, output, planLater(left), planLater(right)) :: Nil
       case logical.MapInPandas(func, output, child) =>
         execution.python.MapInPandasExec(func, output, planLater(child)) :: Nil
       case logical.MapElements(f, _, _, objAttr, child) =>
@@ -722,10 +683,9 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
           f, key, lObj, rObj, lGroup, rGroup, lAttr, rAttr, oAttr,
           planLater(left), planLater(right)) :: Nil
 
-      case logical.Repartition(numPartitions, shuffle, child) =>
+      case r @ logical.Repartition(numPartitions, shuffle, child) =>
         if (shuffle) {
-          ShuffleExchangeExec(RoundRobinPartitioning(numPartitions),
-            planLater(child), canChangeNumPartitions = false) :: Nil
+          ShuffleExchangeExec(r.partitioning, planLater(child), REPARTITION_WITH_NUM) :: Nil
         } else {
           execution.CoalesceExec(numPartitions, planLater(child)) :: Nil
         }
@@ -747,8 +707,8 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
         execution.LocalLimitExec(limit, planLater(child)) :: Nil
       case logical.GlobalLimit(IntegerLiteral(limit), child) =>
         execution.GlobalLimitExec(limit, planLater(child)) :: Nil
-      case logical.Union(unionChildren) =>
-        execution.UnionExec(unionChildren.map(planLater)) :: Nil
+      case union: logical.Union =>
+        execution.UnionExec(union.children.map(planLater)) :: Nil
       case g @ logical.Generate(generator, _, outer, _, _, child) =>
         execution.GenerateExec(
           generator, g.requiredChildOutput, outer,
@@ -758,15 +718,19 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
       case r: logical.Range =>
         execution.RangeExec(r) :: Nil
       case r: logical.RepartitionByExpression =>
-        exchange.ShuffleExchangeExec(
-          r.partitioning, planLater(r.child), canChangeNumPartitions = false) :: Nil
+        val shuffleOrigin = if (r.optNumPartitions.isEmpty) {
+          REPARTITION
+        } else {
+          REPARTITION_WITH_NUM
+        }
+        exchange.ShuffleExchangeExec(r.partitioning, planLater(r.child), shuffleOrigin) :: Nil
       case ExternalRDD(outputObjAttr, rdd) => ExternalRDDScanExec(outputObjAttr, rdd) :: Nil
       case r: LogicalRDD =>
         RDDScanExec(r.output, r.rdd, "ExistingRDD", r.outputPartitioning, r.outputOrdering) :: Nil
       case _: UpdateTable =>
-        throw new UnsupportedOperationException(s"UPDATE TABLE is not supported temporarily.")
+        throw QueryExecutionErrors.ddlUnsupportedTemporarilyError("UPDATE TABLE")
       case _: MergeIntoTable =>
-        throw new UnsupportedOperationException(s"MERGE INTO TABLE is not supported temporarily.")
+        throw QueryExecutionErrors.ddlUnsupportedTemporarilyError("MERGE INTO TABLE")
       case logical.CollectMetrics(name, metrics, child) =>
         execution.CollectMetricsExec(name, metrics, planLater(child)) :: Nil
       case _ => Nil
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SubqueryAdaptiveBroadcastExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SubqueryAdaptiveBroadcastExec.scala
new file mode 100644
index 0000000000000..a735d913c953a
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SubqueryAdaptiveBroadcastExec.scala
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions._
+
+/**
+ * Similar to [[SubqueryBroadcastExec]], this node is used to store the
+ * initial physical plan of DPP subquery filters when enabling both AQE and DPP.
+ * It is intermediate physical plan and not executable.
+ * After the build side is executed, this node will be replaced with the
+ * [[SubqueryBroadcastExec]] and the child will be optimized with the ReusedExchange
+ * from the build side.
+ */
+case class SubqueryAdaptiveBroadcastExec(
+    name: String,
+    index: Int,
+    buildKeys: Seq[Expression],
+    child: SparkPlan) extends BaseSubqueryExec with UnaryExecNode {
+
+  protected override def doExecute(): RDD[InternalRow] = {
+    throw new UnsupportedOperationException(
+      "SubqueryAdaptiveBroadcastExec does not support the execute() code path.")
+  }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): SubqueryAdaptiveBroadcastExec =
+    copy(child = newChild)
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SubqueryBroadcastExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SubqueryBroadcastExec.scala
index e9f2f6a2cdfaf..a8c7b9bfbdeea 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SubqueryBroadcastExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SubqueryBroadcastExec.scala
@@ -36,7 +36,8 @@ import org.apache.spark.util.ThreadUtils
  *
  * @param index the index of the join key in the list of keys from the build side
  * @param buildKeys the join keys from the build side of the join used
- * @param child the BroadcastExchange from the build side of the join
+ * @param child the BroadcastExchange or the AdaptiveSparkPlan with BroadcastQueryStageExec
+ *              from the build side of the join
  */
 case class SubqueryBroadcastExec(
     name: String,
@@ -44,12 +45,25 @@ case class SubqueryBroadcastExec(
     buildKeys: Seq[Expression],
     child: SparkPlan) extends BaseSubqueryExec with UnaryExecNode {
 
+  // `SubqueryBroadcastExec` is only used with `InSubqueryExec`. No one would reference this output,
+  // so the exprId doesn't matter here. But it's important to correctly report the output length, so
+  // that `InSubqueryExec` can know it's the single-column execution mode, not multi-column.
+  override def output: Seq[Attribute] = {
+    val key = buildKeys(index)
+    val name = key match {
+      case n: NamedExpression => n.name
+      case Cast(n: NamedExpression, _, _) => n.name
+      case _ => "key"
+    }
+    Seq(AttributeReference(name, key.dataType, key.nullable)())
+  }
+
   override lazy val metrics = Map(
     "dataSize" -> SQLMetrics.createMetric(sparkContext, "data size (bytes)"),
     "collectTime" -> SQLMetrics.createMetric(sparkContext, "time to collect (ms)"))
 
   override def doCanonicalize(): SparkPlan = {
-    val keys = buildKeys.map(k => QueryPlan.normalizeExpressions(k, output))
+    val keys = buildKeys.map(k => QueryPlan.normalizeExpressions(k, child.output))
     SubqueryBroadcastExec("dpp", index, keys, child.canonicalized)
   }
 
@@ -100,6 +114,9 @@ case class SubqueryBroadcastExec(
   }
 
   override def stringArgs: Iterator[Any] = super.stringArgs ++ Iterator(s"[id=#$id]")
+
+  override protected def withNewChildInternal(newChild: SparkPlan): SubqueryBroadcastExec =
+    copy(child = newChild)
 }
 
 object SubqueryBroadcastExec {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala
index 0244542054611..85bc98d194fee 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala
@@ -29,12 +29,11 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
-import org.apache.spark.sql.catalyst.plans.QueryPlan
 import org.apache.spark.sql.catalyst.plans.physical.Partitioning
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution.aggregate.HashAggregateExec
 import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
-import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, SortMergeJoinExec}
+import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BroadcastNestedLoopJoinExec, ShuffledHashJoinExec, SortMergeJoinExec}
 import org.apache.spark.sql.execution.metric.SQLMetrics
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
@@ -50,7 +49,9 @@ trait CodegenSupport extends SparkPlan {
   private def variablePrefix: String = this match {
     case _: HashAggregateExec => "agg"
     case _: BroadcastHashJoinExec => "bhj"
+    case _: ShuffledHashJoinExec => "shj"
     case _: SortMergeJoinExec => "smj"
+    case _: BroadcastNestedLoopJoinExec => "bnlj"
     case _: RDDScanExec => "rdd"
     case _: DataSourceScanExec => "scan"
     case _: InMemoryTableScanExec => "memoryScan"
@@ -91,7 +92,7 @@ trait CodegenSupport extends SparkPlan {
     this.parent = parent
     ctx.freshNamePrefix = variablePrefix
     s"""
-       |${ctx.registerComment(s"PRODUCE: ${this.simpleString(SQLConf.get.maxToStringFields)}")}
+       |${ctx.registerComment(s"PRODUCE: ${this.simpleString(conf.maxToStringFields)}")}
        |${doProduce(ctx)}
      """.stripMargin
   }
@@ -184,7 +185,7 @@ trait CodegenSupport extends SparkPlan {
     //    all variables in output (see `requireAllOutput`).
     // 3. The number of output variables must less than maximum number of parameters in Java method
     //    declaration.
-    val confEnabled = SQLConf.get.wholeStageSplitConsumeFuncByOperator
+    val confEnabled = conf.wholeStageSplitConsumeFuncByOperator
     val requireAllOutput = output.forall(parent.usedInputs.contains(_))
     val paramLength = CodeGenerator.calculateParamLength(output) + (if (row != null) 1 else 0)
     val consumeFunc = if (confEnabled && requireAllOutput
@@ -194,7 +195,7 @@ trait CodegenSupport extends SparkPlan {
       parent.doConsume(ctx, inputVars, rowVar)
     }
     s"""
-       |${ctx.registerComment(s"CONSUME: ${parent.simpleString(SQLConf.get.maxToStringFields)}")}
+       |${ctx.registerComment(s"CONSUME: ${parent.simpleString(conf.maxToStringFields)}")}
        |$evaluated
        |$consumeFunc
      """.stripMargin
@@ -263,7 +264,7 @@ trait CodegenSupport extends SparkPlan {
 
       paramVars += ExprCode(paramIsNull, JavaCode.variable(paramName, attributes(i).dataType))
     }
-    (arguments, parameters, paramVars)
+    (arguments.toSeq, parameters.toSeq, paramVars.toSeq)
   }
 
   /**
@@ -538,7 +539,8 @@ case class InputAdapter(child: SparkPlan) extends UnaryExecNode with InputRDDCod
       prefix: String = "",
       addSuffix: Boolean = false,
       maxFields: Int,
-      printNodeId: Boolean): Unit = {
+      printNodeId: Boolean,
+      indent: Int = 0): Unit = {
     child.generateTreeString(
       depth,
       lastChildren,
@@ -547,10 +549,14 @@ case class InputAdapter(child: SparkPlan) extends UnaryExecNode with InputRDDCod
       prefix = "",
       addSuffix = false,
       maxFields,
-      printNodeId)
+      printNodeId,
+      indent)
   }
 
   override def needCopyResult: Boolean = false
+
+  override protected def withNewChildInternal(newChild: SparkPlan): InputAdapter =
+    copy(child = newChild)
 }
 
 object WholeStageCodegenExec {
@@ -668,7 +674,7 @@ case class WholeStageCodegenExec(child: SparkPlan)(val codegenStageId: Int)
       }
 
       ${ctx.registerComment(
-        s"""Codegend pipeline for stage (id=$codegenStageId)
+        s"""Codegened pipeline for stage (id=$codegenStageId)
            |${this.treeString.trim}""".stripMargin,
          "wsc_codegenPipeline")}
       ${ctx.registerComment(s"codegenStageId=$codegenStageId", "wsc_codegenStageId", true)}
@@ -807,7 +813,8 @@ case class WholeStageCodegenExec(child: SparkPlan)(val codegenStageId: Int)
       prefix: String = "",
       addSuffix: Boolean = false,
       maxFields: Int,
-      printNodeId: Boolean): Unit = {
+      printNodeId: Boolean,
+      indent: Int = 0): Unit = {
     child.generateTreeString(
       depth,
       lastChildren,
@@ -816,7 +823,8 @@ case class WholeStageCodegenExec(child: SparkPlan)(val codegenStageId: Int)
       if (printNodeId) "* " else s"*($codegenStageId) ",
       false,
       maxFields,
-      printNodeId)
+      printNodeId,
+      indent)
   }
 
   override def needStopCheck: Boolean = true
@@ -824,6 +832,9 @@ case class WholeStageCodegenExec(child: SparkPlan)(val codegenStageId: Int)
   override def limitNotReachedChecks: Seq[String] = Nil
 
   override protected def otherCopyArgs: Seq[AnyRef] = Seq(codegenStageId.asInstanceOf[Integer])
+
+  override protected def withNewChildInternal(newChild: SparkPlan): WholeStageCodegenExec =
+    copy(child = newChild)(codegenStageId)
 }
 
 
@@ -868,7 +879,6 @@ case class WholeStageCodegenExec(child: SparkPlan)(val codegenStageId: Int)
  * failed to generate/compile code.
  */
 case class CollapseCodegenStages(
-    conf: SQLConf,
     codegenStageCounter: AtomicInteger = new AtomicInteger(0))
   extends Rule[SparkPlan] {
 
@@ -903,6 +913,10 @@ case class CollapseCodegenStages(
         // The children of SortMergeJoin should do codegen separately.
         j.withNewChildren(j.children.map(
           child => InputAdapter(insertWholeStageCodegen(child))))
+      case j: ShuffledHashJoinExec =>
+        // The children of ShuffledHashJoin should do codegen separately.
+        j.withNewChildren(j.children.map(
+          child => InputAdapter(insertWholeStageCodegen(child))))
       case p => p.withNewChildren(p.children.map(insertInputAdapter))
     }
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AQEOptimizer.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AQEOptimizer.scala
new file mode 100644
index 0000000000000..a1c35ad27b811
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AQEOptimizer.scala
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.adaptive
+
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, LogicalPlanIntegrity, PlanHelper}
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.util.Utils
+
+/**
+ * The optimizer for re-optimizing the logical plan used by AdaptiveSparkPlanExec.
+ */
+class AQEOptimizer(conf: SQLConf) extends RuleExecutor[LogicalPlan] {
+  private val defaultBatches = Seq(
+    Batch("Eliminate Unnecessary Join", Once, EliminateUnnecessaryJoin),
+    Batch("Demote BroadcastHashJoin", Once, DemoteBroadcastHashJoin)
+  )
+
+  final override protected def batches: Seq[Batch] = {
+    val excludedRules = conf.getConf(SQLConf.ADAPTIVE_OPTIMIZER_EXCLUDED_RULES)
+      .toSeq.flatMap(Utils.stringToSeq)
+    defaultBatches.flatMap { batch =>
+      val filteredRules = batch.rules.filter { rule =>
+        val exclude = excludedRules.contains(rule.ruleName)
+        if (exclude) {
+          logInfo(s"Optimization rule '${rule.ruleName}' is excluded from the optimizer.")
+        }
+        !exclude
+      }
+      if (batch.rules == filteredRules) {
+        Some(batch)
+      } else if (filteredRules.nonEmpty) {
+        Some(Batch(batch.name, batch.strategy, filteredRules: _*))
+      } else {
+        logInfo(s"Optimization batch '${batch.name}' is excluded from the optimizer " +
+          s"as all enclosed rules have been excluded.")
+        None
+      }
+    }
+  }
+
+  override protected def isPlanIntegral(plan: LogicalPlan): Boolean = {
+    !Utils.isTesting || (plan.resolved &&
+      plan.find(PlanHelper.specialExpressionsInUnsupportedOperator(_).nonEmpty).isEmpty &&
+      LogicalPlanIntegrity.checkIfExprIdsAreGloballyUnique(plan))
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
index eb5fcd3b24227..256aacb9049d4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
@@ -26,16 +26,17 @@ import scala.collection.mutable
 import scala.concurrent.ExecutionContext
 import scala.util.control.NonFatal
 
-import org.apache.spark.SparkException
+import org.apache.spark.{broadcast, SparkException}
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, ReturnAnswer}
-import org.apache.spark.sql.catalyst.rules.{Rule, RuleExecutor}
+import org.apache.spark.sql.catalyst.rules.{PlanChangeLogger, Rule}
 import org.apache.spark.sql.catalyst.trees.TreeNodeTag
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec._
+import org.apache.spark.sql.execution.bucketing.DisableUnnecessaryBucketedScan
 import org.apache.spark.sql.execution.exchange._
 import org.apache.spark.sql.execution.ui.{SparkListenerSQLAdaptiveExecutionUpdate, SparkListenerSQLAdaptiveSQLMetricUpdates, SQLPlanMetric}
 import org.apache.spark.sql.internal.SQLConf
@@ -58,7 +59,7 @@ import org.apache.spark.util.ThreadUtils
  * the rest of the plan.
  */
 case class AdaptiveSparkPlanExec(
-    initialPlan: SparkPlan,
+    inputPlan: SparkPlan,
     @transient context: AdaptiveExecutionContext,
     @transient preprocessingRules: Seq[Rule[SparkPlan]],
     @transient isSubquery: Boolean)
@@ -75,40 +76,65 @@ case class AdaptiveSparkPlanExec(
     case _ => logDebug(_)
   }
 
-  // The logical plan optimizer for re-optimizing the current logical plan.
-  @transient private val optimizer = new RuleExecutor[LogicalPlan] {
-    // TODO add more optimization rules
-    override protected def batches: Seq[Batch] = Seq(
-      Batch("Demote BroadcastHashJoin", Once, DemoteBroadcastHashJoin(conf))
-    )
-  }
+  @transient private val planChangeLogger = new PlanChangeLogger[SparkPlan]()
 
-  @transient private val ensureRequirements = EnsureRequirements(conf)
+  // The logical plan optimizer for re-optimizing the current logical plan.
+  @transient private val optimizer = new AQEOptimizer(conf)
 
   // A list of physical plan rules to be applied before creation of query stages. The physical
   // plan should reach a final status of query stages (i.e., no more addition or removal of
   // Exchange nodes) after running these rules.
   private def queryStagePreparationRules: Seq[Rule[SparkPlan]] = Seq(
-    ensureRequirements
-  )
+    RemoveRedundantProjects,
+    EnsureRequirements,
+    RemoveRedundantSorts,
+    DisableUnnecessaryBucketedScan
+  ) ++ context.session.sessionState.queryStagePrepRules
 
   // A list of physical optimizer rules to be applied to a new stage before its execution. These
   // optimizations should be stage-independent.
   @transient private val queryStageOptimizerRules: Seq[Rule[SparkPlan]] = Seq(
-    ReuseAdaptiveSubquery(conf, context.subqueryCache),
+    PlanAdaptiveDynamicPruningFilters(this),
+    ReuseAdaptiveSubquery(context.subqueryCache),
     CoalesceShufflePartitions(context.session),
     // The following two rules need to make use of 'CustomShuffleReaderExec.partitionSpecs'
     // added by `CoalesceShufflePartitions`. So they must be executed after it.
-    OptimizeSkewedJoin(conf),
-    OptimizeLocalShuffleReader(conf),
-    ApplyColumnarRulesAndInsertTransitions(conf, context.session.sessionState.columnarRules),
-    CollapseCodegenStages(conf)
+    OptimizeSkewedJoin,
+    OptimizeLocalShuffleReader
+  )
+
+  // A list of physical optimizer rules to be applied right after a new stage is created. The input
+  // plan to these rules has exchange as its root node.
+  @transient private val postStageCreationRules = Seq(
+    ApplyColumnarRulesAndInsertTransitions(context.session.sessionState.columnarRules),
+    CollapseCodegenStages()
   )
 
+  // The partitioning of the query output depends on the shuffle(s) in the final stage. If the
+  // original plan contains a repartition operator, we need to preserve the specified partitioning,
+  // whether or not the repartition-introduced shuffle is optimized out because of an underlying
+  // shuffle of the same partitioning. Thus, we need to exclude some `CustomShuffleReaderRule`s
+  // from the final stage, depending on the presence and properties of repartition operators.
+  private def finalStageOptimizerRules: Seq[Rule[SparkPlan]] = {
+    val origins = inputPlan.collect {
+      case s: ShuffleExchangeLike => s.shuffleOrigin
+    }
+    val allRules = queryStageOptimizerRules ++ postStageCreationRules
+    allRules.filter {
+      case c: CustomShuffleReaderRule =>
+        origins.forall(c.supportedShuffleOrigins.contains)
+      case _ => true
+    }
+  }
+
   @transient private val costEvaluator = SimpleCostEvaluator
 
-  @volatile private var currentPhysicalPlan =
-    applyPhysicalRules(initialPlan, queryStagePreparationRules)
+  @transient private val initialPlan = context.session.withActive {
+    applyPhysicalRules(
+      inputPlan, queryStagePreparationRules, Some((planChangeLogger, "AQE Preparations")))
+  }
+
+  @volatile private var currentPhysicalPlan = initialPlan
 
   private var isFinalPlan = false
 
@@ -129,9 +155,9 @@ case class AdaptiveSparkPlanExec(
 
   override def conf: SQLConf = context.session.sessionState.conf
 
-  override def output: Seq[Attribute] = initialPlan.output
+  override def output: Seq[Attribute] = inputPlan.output
 
-  override def doCanonicalize(): SparkPlan = initialPlan.canonicalized
+  override def doCanonicalize(): SparkPlan = inputPlan.canonicalized
 
   override def resetMetrics(): Unit = {
     metrics.valuesIterator.foreach(_.reset())
@@ -156,7 +182,7 @@ case class AdaptiveSparkPlanExec(
       var currentLogicalPlan = currentPhysicalPlan.logicalLink.get
       var result = createQueryStages(currentPhysicalPlan)
       val events = new LinkedBlockingQueue[StageMaterializationEvent]()
-      val errors = new mutable.ArrayBuffer[SparkException]()
+      val errors = new mutable.ArrayBuffer[Throwable]()
       var stagesToReplace = Seq.empty[QueryStageExec]
       while (!result.allChildStagesMaterialized) {
         currentPhysicalPlan = result.newPlan
@@ -164,8 +190,20 @@ case class AdaptiveSparkPlanExec(
           stagesToReplace = result.newStages ++ stagesToReplace
           executionId.foreach(onUpdatePlan(_, result.newStages.map(_.plan)))
 
+          // SPARK-33933: we should submit tasks of broadcast stages first, to avoid waiting
+          // for tasks to be scheduled and leading to broadcast timeout.
+          // This partial fix only guarantees the start of materialization for BroadcastQueryStage
+          // is prior to others, but because the submission of collect job for broadcasting is
+          // running in another thread, the issue is not completely resolved.
+          val reorderedNewStages = result.newStages
+            .sortWith {
+              case (_: BroadcastQueryStageExec, _: BroadcastQueryStageExec) => false
+              case (_: BroadcastQueryStageExec, _) => true
+              case _ => false
+            }
+
           // Start materialization of all new stages and fail fast if any stages failed eagerly
-          result.newStages.foreach { stage =>
+          reorderedNewStages.foreach { stage =>
             try {
               stage.materialize().onComplete { res =>
                 if (res.isSuccess) {
@@ -176,9 +214,7 @@ case class AdaptiveSparkPlanExec(
               }(AdaptiveSparkPlanExec.executionContext)
             } catch {
               case e: Throwable =>
-                val ex = new SparkException(
-                  s"Early failed query stage found: ${stage.treeString}", e)
-                cleanUpAndThrowException(Seq(ex), Some(stage.id))
+                cleanUpAndThrowException(Seq(e), Some(stage.id))
             }
           }
         }
@@ -191,15 +227,14 @@ case class AdaptiveSparkPlanExec(
         events.drainTo(rem)
         (Seq(nextMsg) ++ rem.asScala).foreach {
           case StageSuccess(stage, res) =>
-            stage.resultOption = Some(res)
+            stage.resultOption.set(Some(res))
           case StageFailure(stage, ex) =>
-            errors.append(
-              new SparkException(s"Failed to materialize query stage: ${stage.treeString}.", ex))
+            errors.append(ex)
         }
 
         // In case of errors, we cancel all running stages and throw exception.
         if (errors.nonEmpty) {
-          cleanUpAndThrowException(errors, None)
+          cleanUpAndThrowException(errors.toSeq, None)
         }
 
         // Try re-optimizing and re-planning. Adopt the new plan if its cost is equal to or less
@@ -230,7 +265,10 @@ case class AdaptiveSparkPlanExec(
       }
 
       // Run the final plan when there's no more unfinished stages.
-      currentPhysicalPlan = applyPhysicalRules(result.newPlan, queryStageOptimizerRules)
+      currentPhysicalPlan = applyPhysicalRules(
+        result.newPlan,
+        finalStageOptimizerRules,
+        Some((planChangeLogger, "AQE Final Query Stage Optimization")))
       isFinalPlan = true
       executionId.foreach(onUpdatePlan(_, Seq(currentPhysicalPlan)))
       currentPhysicalPlan
@@ -272,6 +310,10 @@ case class AdaptiveSparkPlanExec(
     rdd
   }
 
+  override def doExecuteBroadcast[T](): broadcast.Broadcast[T] = {
+    getFinalPhysicalPlan().doExecuteBroadcast()
+  }
+
   protected override def stringArgs: Iterator[Any] = Iterator(s"isFinalPlan=$isFinalPlan")
 
   override def generateTreeString(
@@ -282,34 +324,83 @@ case class AdaptiveSparkPlanExec(
       prefix: String = "",
       addSuffix: Boolean = false,
       maxFields: Int,
-      printNodeId: Boolean): Unit = {
-    super.generateTreeString(depth,
+      printNodeId: Boolean,
+      indent: Int = 0): Unit = {
+    super.generateTreeString(
+      depth,
       lastChildren,
       append,
       verbose,
       prefix,
       addSuffix,
       maxFields,
-      printNodeId)
-    currentPhysicalPlan.generateTreeString(
-      depth + 1,
-      lastChildren :+ true,
+      printNodeId,
+      indent)
+    if (currentPhysicalPlan.fastEquals(initialPlan)) {
+      currentPhysicalPlan.generateTreeString(
+        depth + 1,
+        lastChildren :+ true,
+        append,
+        verbose,
+        prefix = "",
+        addSuffix = false,
+        maxFields,
+        printNodeId,
+        indent)
+    } else {
+      generateTreeStringWithHeader(
+        if (isFinalPlan) "Final Plan" else "Current Plan",
+        currentPhysicalPlan,
+        depth,
+        lastChildren,
+        append,
+        verbose,
+        maxFields,
+        printNodeId)
+      generateTreeStringWithHeader(
+        "Initial Plan",
+        initialPlan,
+        depth,
+        lastChildren,
+        append,
+        verbose,
+        maxFields,
+        printNodeId)
+    }
+  }
+
+
+  private def generateTreeStringWithHeader(
+      header: String,
+      plan: SparkPlan,
+      depth: Int,
+      lastChildren: Seq[Boolean],
+      append: String => Unit,
+      verbose: Boolean,
+      maxFields: Int,
+      printNodeId: Boolean): Unit = {
+    append("   " * depth)
+    append(s"+- == $header ==\n")
+    plan.generateTreeString(
+      0,
+      Nil,
       append,
       verbose,
-      "",
+      prefix = "",
       addSuffix = false,
       maxFields,
-      printNodeId)
+      printNodeId,
+      indent = depth + 1)
   }
 
-  override def hashCode(): Int = initialPlan.hashCode()
+  override def hashCode(): Int = inputPlan.hashCode()
 
   override def equals(obj: Any): Boolean = {
     if (!obj.isInstanceOf[AdaptiveSparkPlanExec]) {
       return false
     }
 
-    this.initialPlan == obj.asInstanceOf[AdaptiveSparkPlanExec].initialPlan
+    this.inputPlan == obj.asInstanceOf[AdaptiveSparkPlanExec].inputPlan
   }
 
   /**
@@ -328,11 +419,11 @@ case class AdaptiveSparkPlanExec(
       context.stageCache.get(e.canonicalized) match {
         case Some(existingStage) if conf.exchangeReuseEnabled =>
           val stage = reuseQueryStage(existingStage, e)
-          // This is a leaf stage and is not materialized yet even if the reused exchange may has
-          // been completed. It will trigger re-optimization later and stage materialization will
-          // finish in instant if the underlying exchange is already completed.
+          val isMaterialized = stage.resultOption.get().isDefined
           CreateStageResult(
-            newPlan = stage, allChildStagesMaterialized = false, newStages = Seq(stage))
+            newPlan = stage,
+            allChildStagesMaterialized = isMaterialized,
+            newStages = if (isMaterialized) Seq.empty else Seq(stage))
 
         case _ =>
           val result = createQueryStages(e.child)
@@ -349,10 +440,11 @@ case class AdaptiveSparkPlanExec(
                 newStage = reuseQueryStage(queryStage, e)
               }
             }
-
-            // We've created a new stage, which is obviously not ready yet.
-            CreateStageResult(newPlan = newStage,
-              allChildStagesMaterialized = false, newStages = Seq(newStage))
+            val isMaterialized = newStage.resultOption.get().isDefined
+            CreateStageResult(
+              newPlan = newStage,
+              allChildStagesMaterialized = isMaterialized,
+              newStages = if (isMaterialized) Seq.empty else Seq(newStage))
           } else {
             CreateStageResult(newPlan = newPlan,
               allChildStagesMaterialized = false, newStages = result.newStages)
@@ -361,7 +453,7 @@ case class AdaptiveSparkPlanExec(
 
     case q: QueryStageExec =>
       CreateStageResult(newPlan = q,
-        allChildStagesMaterialized = q.resultOption.isDefined, newStages = Seq.empty)
+        allChildStagesMaterialized = q.resultOption.get().isDefined, newStages = Seq.empty)
 
     case _ =>
       if (plan.children.isEmpty) {
@@ -376,12 +468,29 @@ case class AdaptiveSparkPlanExec(
   }
 
   private def newQueryStage(e: Exchange): QueryStageExec = {
-    val optimizedPlan = applyPhysicalRules(e.child, queryStageOptimizerRules)
+    val optimizedPlan = applyPhysicalRules(
+      e.child, queryStageOptimizerRules, Some((planChangeLogger, "AQE Query Stage Optimization")))
     val queryStage = e match {
-      case s: ShuffleExchangeExec =>
-        ShuffleQueryStageExec(currentStageId, s.copy(child = optimizedPlan))
-      case b: BroadcastExchangeExec =>
-        BroadcastQueryStageExec(currentStageId, b.copy(child = optimizedPlan))
+      case s: ShuffleExchangeLike =>
+        val newShuffle = applyPhysicalRules(
+          s.withNewChildren(Seq(optimizedPlan)),
+          postStageCreationRules,
+          Some((planChangeLogger, "AQE Post Stage Creation")))
+        if (!newShuffle.isInstanceOf[ShuffleExchangeLike]) {
+          throw new IllegalStateException(
+            "Custom columnar rules cannot transform shuffle node to something else.")
+        }
+        ShuffleQueryStageExec(currentStageId, newShuffle, s.canonicalized)
+      case b: BroadcastExchangeLike =>
+        val newBroadcast = applyPhysicalRules(
+          b.withNewChildren(Seq(optimizedPlan)),
+          postStageCreationRules,
+          Some((planChangeLogger, "AQE Post Stage Creation")))
+        if (!newBroadcast.isInstanceOf[BroadcastExchangeLike]) {
+          throw new IllegalStateException(
+            "Custom columnar rules cannot transform broadcast node to something else.")
+        }
+        BroadcastQueryStageExec(currentStageId, newBroadcast, b.canonicalized)
     }
     currentStageId += 1
     setLogicalLinkForNewQueryStage(queryStage, e)
@@ -485,7 +594,10 @@ case class AdaptiveSparkPlanExec(
     logicalPlan.invalidateStatsCache()
     val optimized = optimizer.execute(logicalPlan)
     val sparkPlan = context.session.sessionState.planner.plan(ReturnAnswer(optimized)).next()
-    val newPlan = applyPhysicalRules(sparkPlan, preprocessingRules ++ queryStagePreparationRules)
+    val newPlan = applyPhysicalRules(
+      sparkPlan,
+      preprocessingRules ++ queryStagePreparationRules,
+      Some((planChangeLogger, "AQE Replanning")))
     (newPlan, optimized)
   }
 
@@ -537,31 +649,28 @@ case class AdaptiveSparkPlanExec(
    * materialization errors and stage cancellation errors.
    */
   private def cleanUpAndThrowException(
-       errors: Seq[SparkException],
+       errors: Seq[Throwable],
        earlyFailedStage: Option[Int]): Unit = {
-    val runningStages = currentPhysicalPlan.collect {
+    currentPhysicalPlan.foreach {
       // earlyFailedStage is the stage which failed before calling doMaterialize,
       // so we should avoid calling cancel on it to re-trigger the failure again.
-      case s: QueryStageExec if !earlyFailedStage.contains(s.id) => s
-    }
-    val cancelErrors = new mutable.ArrayBuffer[SparkException]()
-    try {
-      runningStages.foreach { s =>
+      case s: QueryStageExec if !earlyFailedStage.contains(s.id) =>
         try {
           s.cancel()
         } catch {
           case NonFatal(t) =>
-            cancelErrors.append(
-              new SparkException(s"Failed to cancel query stage: ${s.treeString}", t))
+            logError(s"Exception in cancelling query stage: ${s.treeString}", t)
         }
-      }
-    } finally {
-      val ex = new SparkException(
-        "Adaptive execution failed due to stage materialization failures.", errors.head)
-      errors.tail.foreach(ex.addSuppressed)
-      cancelErrors.foreach(ex.addSuppressed)
-      throw ex
+      case _ =>
     }
+    val e = if (errors.size == 1) {
+      errors.head
+    } else {
+      val se = new SparkException("Multiple failures in stage materialization.", errors.head)
+      errors.tail.foreach(se.addSuppressed)
+      se
+    }
+    throw e
   }
 }
 
@@ -584,8 +693,22 @@ object AdaptiveSparkPlanExec {
   /**
    * Apply a list of physical operator rules on a [[SparkPlan]].
    */
-  def applyPhysicalRules(plan: SparkPlan, rules: Seq[Rule[SparkPlan]]): SparkPlan = {
-    rules.foldLeft(plan) { case (sp, rule) => rule.apply(sp) }
+  def applyPhysicalRules(
+      plan: SparkPlan,
+      rules: Seq[Rule[SparkPlan]],
+      loggerAndBatchName: Option[(PlanChangeLogger[SparkPlan], String)] = None): SparkPlan = {
+    if (loggerAndBatchName.isEmpty) {
+      rules.foldLeft(plan) { case (sp, rule) => rule.apply(sp) }
+    } else {
+      val (logger, batchName) = loggerAndBatchName.get
+      val newPlan = rules.foldLeft(plan) { case (sp, rule) =>
+        val result = rule.apply(sp)
+        logger.logRule(rule.ruleName, sp, result)
+        result
+      }
+      logger.logBatch(batchName, plan, newPlan)
+      newPlan
+    }
   }
 }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanHelper.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanHelper.scala
index 3cf6a13a4a892..eecfa40e8d0bd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanHelper.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanHelper.scala
@@ -17,9 +17,7 @@
 
 package org.apache.spark.sql.execution.adaptive
 
-import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.execution.SparkPlan
-import org.apache.spark.sql.internal.SQLConf
 
 /**
  * This class provides utility methods related to tree traversal of an [[AdaptiveSparkPlanExec]]
@@ -65,7 +63,7 @@ trait AdaptiveSparkPlanHelper {
   def mapPlans[A](p: SparkPlan)(f: SparkPlan => A): Seq[A] = {
     val ret = new collection.mutable.ArrayBuffer[A]()
     foreach(p)(ret += f(_))
-    ret
+    ret.toSeq
   }
 
   /**
@@ -75,7 +73,7 @@ trait AdaptiveSparkPlanHelper {
   def flatMap[A](p: SparkPlan)(f: SparkPlan => TraversableOnce[A]): Seq[A] = {
     val ret = new collection.mutable.ArrayBuffer[A]()
     foreach(p)(ret ++= f(_))
-    ret
+    ret.toSeq
   }
 
   /**
@@ -86,7 +84,7 @@ trait AdaptiveSparkPlanHelper {
     val ret = new collection.mutable.ArrayBuffer[B]()
     val lifted = pf.lift
     foreach(p)(node => lifted(node).foreach(ret.+=))
-    ret
+    ret.toSeq
   }
 
   /**
@@ -117,7 +115,7 @@ trait AdaptiveSparkPlanHelper {
 
   /**
    * Returns a sequence containing the subqueries in this plan, also including the (nested)
-   * subquries in its children
+   * subqueries in its children
    */
   def subqueriesAll(p: SparkPlan): Seq[SparkPlan] = {
     val subqueries = flatMap(p)(_.subqueries)
@@ -137,18 +135,4 @@ trait AdaptiveSparkPlanHelper {
     case a: AdaptiveSparkPlanExec => a.executedPlan
     case other => other
   }
-
-  /**
-   * Returns a cloned [[SparkSession]] with adaptive execution disabled, or the original
-   * [[SparkSession]] if its adaptive execution is already disabled.
-   */
-  def getOrCloneSessionWithAqeOff[T](session: SparkSession): SparkSession = {
-    if (!session.sessionState.conf.adaptiveExecutionEnabled) {
-      session
-    } else {
-      val newSession = session.cloneSession()
-      newSession.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED, false)
-      newSession
-    }
-  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala
index 6aa34497c9ea6..d50e32c8b310f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala
@@ -18,17 +18,18 @@
 package org.apache.spark.sql.execution.adaptive
 
 import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.execution.SparkPlan
+import org.apache.spark.sql.catalyst.plans.physical.SinglePartition
+import org.apache.spark.sql.execution.{ShufflePartitionSpec, SparkPlan}
+import org.apache.spark.sql.execution.exchange.{ENSURE_REQUIREMENTS, REPARTITION, ShuffleExchangeLike, ShuffleOrigin}
 import org.apache.spark.sql.internal.SQLConf
 
 /**
  * A rule to coalesce the shuffle partitions based on the map output statistics, which can
  * avoid many small reduce tasks that hurt performance.
  */
-case class CoalesceShufflePartitions(session: SparkSession) extends Rule[SparkPlan] {
-  import CoalesceShufflePartitions._
-  private def conf = session.sessionState.conf
+case class CoalesceShufflePartitions(session: SparkSession) extends CustomShuffleReaderRule {
+
+  override val supportedShuffleOrigins: Seq[ShuffleOrigin] = Seq(ENSURE_REQUIREMENTS, REPARTITION)
 
   override def apply(plan: SparkPlan): SparkPlan = {
     if (!conf.coalesceShufflePartitionsEnabled) {
@@ -50,11 +51,24 @@ case class CoalesceShufflePartitions(session: SparkSession) extends Rule[SparkPl
     val shuffleStages = collectShuffleStages(plan)
     // ShuffleExchanges introduced by repartition do not support changing the number of partitions.
     // We change the number of partitions in the stage only if all the ShuffleExchanges support it.
-    if (!shuffleStages.forall(_.shuffle.canChangeNumPartitions)) {
+    if (!shuffleStages.forall(s => supportCoalesce(s.shuffle))) {
       plan
     } else {
+      def insertCustomShuffleReader(partitionSpecs: Seq[ShufflePartitionSpec]): SparkPlan = {
+        // This transformation adds new nodes, so we must use `transformUp` here.
+        val stageIds = shuffleStages.map(_.id).toSet
+        plan.transformUp {
+          // even for shuffle exchange whose input RDD has 0 partition, we should still update its
+          // `partitionStartIndices`, so that all the leaf shuffles in a stage have the same
+          // number of output partitions.
+          case stage: ShuffleQueryStageExec if stageIds.contains(stage.id) =>
+            CustomShuffleReaderExec(stage, partitionSpecs)
+        }
+      }
+
       // `ShuffleQueryStageExec#mapStats` returns None when the input RDD has 0 partitions,
       // we should skip it when calculating the `partitionStartIndices`.
+      // If all input RDDs have 0 partition, we create empty partition for every shuffle reader.
       val validMetrics = shuffleStages.flatMap(_.mapStats)
 
       // We may have different pre-shuffle partition numbers, don't reduce shuffle partition number
@@ -62,7 +76,9 @@ case class CoalesceShufflePartitions(session: SparkSession) extends Rule[SparkPl
       // partition) and a result of a SortMergeJoin (multiple partitions).
       val distinctNumPreShufflePartitions =
         validMetrics.map(stats => stats.bytesByPartitionId.length).distinct
-      if (validMetrics.nonEmpty && distinctNumPreShufflePartitions.length == 1) {
+      if (validMetrics.isEmpty) {
+        insertCustomShuffleReader(ShufflePartitionsUtil.createEmptyPartition() :: Nil)
+      } else if (distinctNumPreShufflePartitions.length == 1) {
         // We fall back to Spark default parallelism if the minimum number of coalesced partitions
         // is not set, so to avoid perf regressions compared to no coalescing.
         val minPartitionNum = conf.getConf(SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM)
@@ -71,18 +87,20 @@ case class CoalesceShufflePartitions(session: SparkSession) extends Rule[SparkPl
           validMetrics.toArray,
           advisoryTargetSize = conf.getConf(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES),
           minNumPartitions = minPartitionNum)
-        // This transformation adds new nodes, so we must use `transformUp` here.
-        val stageIds = shuffleStages.map(_.id).toSet
-        plan.transformUp {
-          // even for shuffle exchange whose input RDD has 0 partition, we should still update its
-          // `partitionStartIndices`, so that all the leaf shuffles in a stage have the same
-          // number of output partitions.
-          case stage: ShuffleQueryStageExec if stageIds.contains(stage.id) =>
-            CustomShuffleReaderExec(stage, partitionSpecs)
+        // We can never extend the shuffle partition number, so if we get the same number here,
+        // that means we can not coalesce shuffle partition. Just return the origin plan.
+        if (partitionSpecs.length == distinctNumPreShufflePartitions.head) {
+          plan
+        } else {
+          insertCustomShuffleReader(partitionSpecs)
         }
       } else {
         plan
       }
     }
   }
+
+  private def supportCoalesce(s: ShuffleExchangeLike): Boolean = {
+    s.outputPartitioning != SinglePartition && supportedShuffleOrigins.contains(s.shuffleOrigin)
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffleReaderExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffleReaderExec.scala
index b2633c774f532..1e2d772b4f5c2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffleReaderExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffleReaderExec.scala
@@ -17,7 +17,6 @@
 
 package org.apache.spark.sql.execution.adaptive
 
-import scala.collection.mutable
 import scala.collection.mutable.ArrayBuffer
 
 import org.apache.spark.rdd.RDD
@@ -25,8 +24,9 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression}
 import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, UnknownPartitioning}
 import org.apache.spark.sql.execution._
-import org.apache.spark.sql.execution.exchange.{ReusedExchangeExec, ShuffleExchangeExec}
+import org.apache.spark.sql.execution.exchange.{ReusedExchangeExec, ShuffleExchangeLike}
 import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics}
+import org.apache.spark.sql.vectorized.ColumnarBatch
 
 
 /**
@@ -45,6 +45,8 @@ case class CustomShuffleReaderExec private(
     assert(partitionSpecs.forall(_.isInstanceOf[PartialMapperPartitionSpec]))
   }
 
+  override def supportsColumnar: Boolean = child.supportsColumnar
+
   override def output: Seq[Attribute] = child.output
   override lazy val outputPartitioning: Partitioning = {
     // If it is a local shuffle reader with one mapper per task, then the output partitioning is
@@ -55,9 +57,9 @@ case class CustomShuffleReaderExec private(
         partitionSpecs.map(_.asInstanceOf[PartialMapperPartitionSpec].mapIndex).toSet.size ==
           partitionSpecs.length) {
       child match {
-        case ShuffleQueryStageExec(_, s: ShuffleExchangeExec) =>
+        case ShuffleQueryStageExec(_, s: ShuffleExchangeLike, _) =>
           s.child.outputPartitioning
-        case ShuffleQueryStageExec(_, r @ ReusedExchangeExec(_, s: ShuffleExchangeExec)) =>
+        case ShuffleQueryStageExec(_, r @ ReusedExchangeExec(_, s: ShuffleExchangeLike), _) =>
           s.child.outputPartitioning match {
             case e: Expression => r.updateAttr(e).asInstanceOf[Partitioning]
             case other => other
@@ -106,7 +108,7 @@ case class CustomShuffleReaderExec private(
         case CoalescedPartitionSpec(startReducerIndex, endReducerIndex) =>
           startReducerIndex.until(endReducerIndex).map(bytesByPartitionId).sum
         case p: PartialReducerPartitionSpec => p.dataSize
-        case p => throw new IllegalStateException("unexpected " + p)
+        case p => throw new IllegalStateException(s"unexpected $p")
       })
     } else {
       None
@@ -146,7 +148,7 @@ case class CustomShuffleReaderExec private(
       partitionDataSizeMetrics.set(dataSizes.sum)
     }
 
-    SQLMetrics.postDriverMetricsUpdatedByValue(sparkContext, executionId, driverAccumUpdates)
+    SQLMetrics.postDriverMetricsUpdatedByValue(sparkContext, executionId, driverAccumUpdates.toSeq)
   }
 
   @transient override lazy val metrics: Map[String, SQLMetric] = {
@@ -176,18 +178,24 @@ case class CustomShuffleReaderExec private(
     }
   }
 
-  private lazy val cachedShuffleRDD: RDD[InternalRow] = {
-    sendDriverMetrics()
-
-    shuffleStage.map { stage =>
-      new ShuffledRowRDD(
-        stage.shuffle.shuffleDependency, stage.shuffle.readMetrics, partitionSpecs.toArray)
-    }.getOrElse {
-      throw new IllegalStateException("operating on canonicalized plan")
+  private lazy val shuffleRDD: RDD[_] = {
+    shuffleStage match {
+      case Some(stage) =>
+        sendDriverMetrics()
+        stage.shuffle.getShuffleRDD(partitionSpecs.toArray)
+      case _ =>
+        throw new IllegalStateException("operating on canonicalized plan")
     }
   }
 
   override protected def doExecute(): RDD[InternalRow] = {
-    cachedShuffleRDD
+    shuffleRDD.asInstanceOf[RDD[InternalRow]]
   }
+
+  override protected def doExecuteColumnar(): RDD[ColumnarBatch] = {
+    shuffleRDD.asInstanceOf[RDD[ColumnarBatch]]
+  }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): CustomShuffleReaderExec =
+    copy(child = newChild)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffleReaderRule.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffleReaderRule.scala
new file mode 100644
index 0000000000000..c5b8f73ea59d3
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffleReaderRule.scala
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.adaptive
+
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.execution.SparkPlan
+import org.apache.spark.sql.execution.exchange.ShuffleOrigin
+
+/**
+ * Adaptive Query Execution rule that may create [[CustomShuffleReaderExec]] on top of query stages.
+ */
+trait CustomShuffleReaderRule extends Rule[SparkPlan] {
+
+  /**
+   * Returns the list of [[ShuffleOrigin]]s supported by this rule.
+   */
+  def supportedShuffleOrigins: Seq[ShuffleOrigin]
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/DemoteBroadcastHashJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/DemoteBroadcastHashJoin.scala
index 0f2868e41cc39..3760782515e97 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/DemoteBroadcastHashJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/DemoteBroadcastHashJoin.scala
@@ -19,16 +19,15 @@ package org.apache.spark.sql.execution.adaptive
 
 import org.apache.spark.sql.catalyst.plans.logical.{HintInfo, Join, LogicalPlan, NO_BROADCAST_HASH}
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.internal.SQLConf
 
 /**
  * This optimization rule detects a join child that has a high ratio of empty partitions and
  * adds a no-broadcast-hash-join hint to avoid it being broadcast.
  */
-case class DemoteBroadcastHashJoin(conf: SQLConf) extends Rule[LogicalPlan] {
+object DemoteBroadcastHashJoin extends Rule[LogicalPlan] {
 
   private def shouldDemote(plan: LogicalPlan): Boolean = plan match {
-    case LogicalQueryStage(_, stage: ShuffleQueryStageExec) if stage.resultOption.isDefined
+    case LogicalQueryStage(_, stage: ShuffleQueryStageExec) if stage.resultOption.get().isDefined
       && stage.mapStats.isDefined =>
       val mapStats = stage.mapStats.get
       val partitionCnt = mapStats.bytesByPartitionId.length
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/EliminateUnnecessaryJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/EliminateUnnecessaryJoin.scala
new file mode 100644
index 0000000000000..fca56988e520d
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/EliminateUnnecessaryJoin.scala
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.adaptive
+
+import org.apache.spark.sql.catalyst.planning.ExtractSingleColumnNullAwareAntiJoin
+import org.apache.spark.sql.catalyst.plans.{Inner, LeftAnti, LeftSemi}
+import org.apache.spark.sql.catalyst.plans.logical.{Join, LocalRelation, LogicalPlan}
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.execution.joins.HashedRelationWithAllNullKeys
+
+/**
+ * This optimization rule detects and eliminates unnecessary Join:
+ * 1. Join is single column NULL-aware anti join (NAAJ), and broadcasted [[HashedRelation]]
+ *    is [[HashedRelationWithAllNullKeys]]. Eliminate join to an empty [[LocalRelation]].
+ *
+ * 2. Join is inner join, and either side of join is empty. Eliminate join to an empty
+ *    [[LocalRelation]].
+ *
+ * 3. Join is left semi join
+ *    3.1. Join right side is empty. Eliminate join to an empty [[LocalRelation]].
+ *    3.2. Join right side is non-empty and condition is empty. Eliminate join to its left side.
+ *
+ * 4. Join is left anti join
+ *    4.1. Join right side is empty. Eliminate join to its left side.
+ *    4.2. Join right side is non-empty and condition is empty. Eliminate join to an empty
+ *         [[LocalRelation]].
+ *
+ * This applies to all joins (sort merge join, shuffled hash join, broadcast hash join, and
+ * broadcast nested loop join), because sort merge join and shuffled hash join will be changed
+ * to broadcast hash join with AQE at the first place.
+ */
+object EliminateUnnecessaryJoin extends Rule[LogicalPlan] {
+
+  private def isRelationWithAllNullKeys(plan: LogicalPlan) = plan match {
+    case LogicalQueryStage(_, stage: BroadcastQueryStageExec)
+      if stage.resultOption.get().isDefined =>
+      stage.broadcast.relationFuture.get().value == HashedRelationWithAllNullKeys
+    case _ => false
+  }
+
+  private def checkRowCount(plan: LogicalPlan, hasRow: Boolean): Boolean = plan match {
+    case LogicalQueryStage(_, stage: QueryStageExec) if stage.resultOption.get().isDefined =>
+      stage.getRuntimeStatistics.rowCount match {
+        case Some(count) => hasRow == (count > 0)
+        case _ => false
+      }
+    case _ => false
+  }
+
+  def apply(plan: LogicalPlan): LogicalPlan = plan.transformDown {
+    case j @ ExtractSingleColumnNullAwareAntiJoin(_, _) if isRelationWithAllNullKeys(j.right) =>
+      LocalRelation(j.output, data = Seq.empty, isStreaming = j.isStreaming)
+
+    case j @ Join(_, _, Inner, _, _) if checkRowCount(j.left, hasRow = false) ||
+      checkRowCount(j.right, hasRow = false) =>
+      LocalRelation(j.output, data = Seq.empty, isStreaming = j.isStreaming)
+
+    case j @ Join(_, _, LeftSemi, condition, _) =>
+      if (checkRowCount(j.right, hasRow = false)) {
+        LocalRelation(j.output, data = Seq.empty, isStreaming = j.isStreaming)
+      } else if (condition.isEmpty && checkRowCount(j.right, hasRow = true)) {
+        j.left
+      } else {
+        j
+      }
+
+    case j @ Join(_, _, LeftAnti, condition, _) =>
+      if (checkRowCount(j.right, hasRow = false)) {
+        j.left
+      } else if (condition.isEmpty && checkRowCount(j.right, hasRow = true)) {
+        LocalRelation(j.output, data = Seq.empty, isStreaming = j.isStreaming)
+      } else {
+        j
+      }
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/InsertAdaptiveSparkPlan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/InsertAdaptiveSparkPlan.scala
index 754225dd3fe95..10655192561a1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/InsertAdaptiveSparkPlan.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/InsertAdaptiveSparkPlan.scala
@@ -20,10 +20,11 @@ package org.apache.spark.sql.execution.adaptive
 import scala.collection.mutable
 
 import org.apache.spark.sql.catalyst.expressions
-import org.apache.spark.sql.catalyst.expressions.{DynamicPruningSubquery, ListQuery, SubqueryExpression}
+import org.apache.spark.sql.catalyst.expressions.{ListQuery, SubqueryExpression}
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.plans.physical.UnspecifiedDistribution
 import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreePattern.{DYNAMIC_PRUNING_SUBQUERY, IN_SUBQUERY, SCALAR_SUBQUERY}
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.execution.command.{DataWritingCommandExec, ExecutedCommandExec}
 import org.apache.spark.sql.execution.datasources.v2.V2CommandExec
@@ -39,8 +40,6 @@ import org.apache.spark.sql.internal.SQLConf
 case class InsertAdaptiveSparkPlan(
     adaptiveExecutionContext: AdaptiveExecutionContext) extends Rule[SparkPlan] {
 
-  private val conf = adaptiveExecutionContext.session.sessionState.conf
-
   override def apply(plan: SparkPlan): SparkPlan = applyInternal(plan, false)
 
   private def applyInternal(plan: SparkPlan, isSubquery: Boolean): SparkPlan = plan match {
@@ -100,10 +99,8 @@ case class InsertAdaptiveSparkPlan(
   }
 
   private def supportAdaptive(plan: SparkPlan): Boolean = {
-    // TODO migrate dynamic-partition-pruning onto adaptive execution.
     sanityCheck(plan) &&
       !plan.logicalLink.exists(_.isStreaming) &&
-      !plan.expressions.exists(_.find(_.isInstanceOf[DynamicPruningSubquery]).isDefined) &&
     plan.children.forall(supportAdaptive)
   }
 
@@ -115,14 +112,18 @@ case class InsertAdaptiveSparkPlan(
    * For each sub-query, generate the adaptive execution plan for each sub-query by applying this
    * rule, or reuse the execution plan from another sub-query of the same semantics if possible.
    */
-  private def buildSubqueryMap(plan: SparkPlan): Map[Long, SubqueryExec] = {
-    val subqueryMap = mutable.HashMap.empty[Long, SubqueryExec]
+  private def buildSubqueryMap(plan: SparkPlan): Map[Long, BaseSubqueryExec] = {
+    val subqueryMap = mutable.HashMap.empty[Long, BaseSubqueryExec]
+    if (!plan.containsAnyPattern(SCALAR_SUBQUERY, IN_SUBQUERY, DYNAMIC_PRUNING_SUBQUERY)) {
+      return subqueryMap.toMap
+    }
     plan.foreach(_.expressions.foreach(_.foreach {
       case expressions.ScalarSubquery(p, _, exprId)
           if !subqueryMap.contains(exprId.id) =>
         val executedPlan = compileSubquery(p)
         verifyAdaptivePlan(executedPlan, p)
-        val subquery = SubqueryExec(s"subquery#${exprId.id}", executedPlan)
+        val subquery = SubqueryExec.createForScalarSubquery(
+          s"subquery#${exprId.id}", executedPlan)
         subqueryMap.put(exprId.id, subquery)
       case expressions.InSubquery(_, ListQuery(query, _, exprId, _))
           if !subqueryMap.contains(exprId.id) =>
@@ -130,6 +131,16 @@ case class InsertAdaptiveSparkPlan(
         verifyAdaptivePlan(executedPlan, query)
         val subquery = SubqueryExec(s"subquery#${exprId.id}", executedPlan)
         subqueryMap.put(exprId.id, subquery)
+      case expressions.DynamicPruningSubquery(value, buildPlan,
+          buildKeys, broadcastKeyIndex, onlyInBroadcast, exprId)
+          if !subqueryMap.contains(exprId.id) =>
+        val executedPlan = compileSubquery(buildPlan)
+        verifyAdaptivePlan(executedPlan, buildPlan)
+
+        val name = s"dynamicpruning#${exprId.id}"
+        val subquery = SubqueryAdaptiveBroadcastExec(
+          name, broadcastKeyIndex, buildKeys, executedPlan)
+        subqueryMap.put(exprId.id, subquery)
       case _ =>
     }))
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStage.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStage.scala
index 9914eddd53a3d..bff142315f8ff 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStage.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStage.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.execution.adaptive
 
 import org.apache.spark.sql.catalyst.expressions.{Attribute, SortOrder}
 import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, Statistics}
-import org.apache.spark.sql.catalyst.plans.physical.Partitioning
 import org.apache.spark.sql.execution.SparkPlan
 
 /**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStageStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStageStrategy.scala
index d60c3ca72f6f6..bcf9dc1544ce3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStageStrategy.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStageStrategy.scala
@@ -19,10 +19,12 @@ package org.apache.spark.sql.execution.adaptive
 
 import org.apache.spark.sql.Strategy
 import org.apache.spark.sql.catalyst.expressions.PredicateHelper
-import org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys
+import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight}
+import org.apache.spark.sql.catalyst.planning.{ExtractEquiJoinKeys, ExtractSingleColumnNullAwareAntiJoin}
+import org.apache.spark.sql.catalyst.plans.LeftAnti
 import org.apache.spark.sql.catalyst.plans.logical.{Join, LogicalPlan}
-import org.apache.spark.sql.execution.SparkPlan
-import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BroadcastNestedLoopJoinExec, BuildLeft, BuildRight}
+import org.apache.spark.sql.execution.{joins, SparkPlan}
+import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BroadcastNestedLoopJoinExec}
 
 /**
  * Strategy for plans containing [[LogicalQueryStage]] nodes:
@@ -47,6 +49,11 @@ object LogicalQueryStageStrategy extends Strategy with PredicateHelper {
       Seq(BroadcastHashJoinExec(
         leftKeys, rightKeys, joinType, buildSide, condition, planLater(left), planLater(right)))
 
+    case j @ ExtractSingleColumnNullAwareAntiJoin(leftKeys, rightKeys)
+        if isBroadcastStage(j.right) =>
+      Seq(joins.BroadcastHashJoinExec(leftKeys, rightKeys, LeftAnti, BuildRight,
+        None, planLater(j.left), planLater(j.right), isNullAwareAntiJoin = true))
+
     case j @ Join(left, right, joinType, condition, _)
         if isBroadcastStage(left) || isBroadcastStage(right) =>
       val buildSide = if (isBroadcastStage(left)) BuildLeft else BuildRight
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala
index 5416fde222cb6..4dc982d666d18 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala
@@ -17,10 +17,11 @@
 
 package org.apache.spark.sql.execution.adaptive
 
-import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide}
+import org.apache.spark.sql.catalyst.plans.physical.SinglePartition
 import org.apache.spark.sql.execution._
-import org.apache.spark.sql.execution.exchange.{EnsureRequirements, ShuffleExchangeExec}
-import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BuildLeft, BuildRight, BuildSide}
+import org.apache.spark.sql.execution.exchange.{ENSURE_REQUIREMENTS, EnsureRequirements, ShuffleExchangeExec, ShuffleExchangeLike, ShuffleOrigin}
+import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec
 import org.apache.spark.sql.internal.SQLConf
 
 /**
@@ -32,10 +33,11 @@ import org.apache.spark.sql.internal.SQLConf
  * then run `EnsureRequirements` to check whether additional shuffle introduced.
  * If introduced, we will revert all the local readers.
  */
-case class OptimizeLocalShuffleReader(conf: SQLConf) extends Rule[SparkPlan] {
-  import OptimizeLocalShuffleReader._
+object OptimizeLocalShuffleReader extends CustomShuffleReaderRule {
 
-  private val ensureRequirements = EnsureRequirements(conf)
+  override val supportedShuffleOrigins: Seq[ShuffleOrigin] = Seq(ENSURE_REQUIREMENTS)
+
+  private val ensureRequirements = EnsureRequirements
 
   // The build side is a broadcast query stage which should have been optimized using local reader
   // already. So we only need to deal with probe side here.
@@ -77,10 +79,9 @@ case class OptimizeLocalShuffleReader(conf: SQLConf) extends Rule[SparkPlan] {
   private def getPartitionSpecs(
       shuffleStage: ShuffleQueryStageExec,
       advisoryParallelism: Option[Int]): Seq[ShufflePartitionSpec] = {
-    val shuffleDep = shuffleStage.shuffle.shuffleDependency
-    val numReducers = shuffleDep.partitioner.numPartitions
+    val numMappers = shuffleStage.shuffle.numMappers
+    val numReducers = shuffleStage.shuffle.numPartitions
     val expectedParallelism = advisoryParallelism.getOrElse(numReducers)
-    val numMappers = shuffleDep.rdd.getNumPartitions
     val splitPoints = if (numMappers == 0) {
       Seq.empty
     } else {
@@ -118,9 +119,6 @@ case class OptimizeLocalShuffleReader(conf: SQLConf) extends Rule[SparkPlan] {
         createProbeSideLocalReader(s)
     }
   }
-}
-
-object OptimizeLocalShuffleReader {
 
   object BroadcastJoinWithShuffleLeft {
     def unapply(plan: SparkPlan): Option[(SparkPlan, BuildSide)] = plan match {
@@ -140,9 +138,13 @@ object OptimizeLocalShuffleReader {
 
   def canUseLocalShuffleReader(plan: SparkPlan): Boolean = plan match {
     case s: ShuffleQueryStageExec =>
-      s.shuffle.canChangeNumPartitions
+      s.mapStats.isDefined && supportLocalReader(s.shuffle)
     case CustomShuffleReaderExec(s: ShuffleQueryStageExec, partitionSpecs) =>
-      s.shuffle.canChangeNumPartitions && partitionSpecs.nonEmpty
+      s.mapStats.isDefined && partitionSpecs.nonEmpty && supportLocalReader(s.shuffle)
     case _ => false
   }
+
+  private def supportLocalReader(s: ShuffleExchangeLike): Boolean = {
+    s.outputPartitioning != SinglePartition && supportedShuffleOrigins.contains(s.shuffleOrigin)
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
index 396c9c9d6b4e5..173739bc8e673 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
@@ -23,10 +23,9 @@ import org.apache.commons.io.FileUtils
 
 import org.apache.spark.{MapOutputStatistics, MapOutputTrackerMaster, SparkEnv}
 import org.apache.spark.sql.catalyst.plans._
-import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution._
-import org.apache.spark.sql.execution.exchange.{EnsureRequirements, ShuffleExchangeExec}
-import org.apache.spark.sql.execution.joins.SortMergeJoinExec
+import org.apache.spark.sql.execution.exchange.{ENSURE_REQUIREMENTS, EnsureRequirements, ShuffleExchangeExec, ShuffleOrigin}
+import org.apache.spark.sql.execution.joins.{ShuffledHashJoinExec, SortMergeJoinExec}
 import org.apache.spark.sql.internal.SQLConf
 
 /**
@@ -53,17 +52,19 @@ import org.apache.spark.sql.internal.SQLConf
  * Note that, when this rule is enabled, it also coalesces non-skewed partitions like
  * `CoalesceShufflePartitions` does.
  */
-case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
+object OptimizeSkewedJoin extends CustomShuffleReaderRule {
 
-  private val ensureRequirements = EnsureRequirements(conf)
+  override val supportedShuffleOrigins: Seq[ShuffleOrigin] = Seq(ENSURE_REQUIREMENTS)
+
+  private val ensureRequirements = EnsureRequirements
 
   private val supportedJoinTypes =
     Inner :: Cross :: LeftSemi :: LeftAnti :: LeftOuter :: RightOuter :: Nil
 
   /**
    * A partition is considered as a skewed partition if its size is larger than the median
-   * partition size * ADAPTIVE_EXECUTION_SKEWED_PARTITION_FACTOR and also larger than
-   * ADVISORY_PARTITION_SIZE_IN_BYTES.
+   * partition size * SKEW_JOIN_SKEWED_PARTITION_FACTOR and also larger than
+   * SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.
    */
   private def isSkewed(size: Long, medianSize: Long): Boolean = {
     size > medianSize * conf.getConf(SQLConf.SKEW_JOIN_SKEWED_PARTITION_FACTOR) &&
@@ -147,7 +148,7 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
   /*
    * This method aim to optimize the skewed join with the following steps:
    * 1. Check whether the shuffle partition is skewed based on the median size
-   *    and the skewed partition threshold in origin smj.
+   *    and the skewed partition threshold in origin shuffled join (smj and shj).
    * 2. Assuming partition0 is skewed in left side, and it has 5 mappers (Map0, Map1...Map4).
    *    And we may split the 5 Mappers into 3 mapper ranges [(Map0, Map1), (Map2, Map3), (Map4)]
    *    based on the map size and the max split number.
@@ -156,98 +157,121 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
    * 4. Wrap the join right child with a special shuffle reader that reads partition0 3 times by
    *    3 tasks separately.
    */
-  def optimizeSkewJoin(plan: SparkPlan): SparkPlan = plan.transformUp {
-    case smj @ SortMergeJoinExec(_, _, joinType, _,
-        s1 @ SortExec(_, _, ShuffleStage(left: ShuffleStageInfo), _),
-        s2 @ SortExec(_, _, ShuffleStage(right: ShuffleStageInfo), _), _)
-        if supportedJoinTypes.contains(joinType) =>
-      assert(left.partitionsWithSizes.length == right.partitionsWithSizes.length)
-      val numPartitions = left.partitionsWithSizes.length
-      // We use the median size of the original shuffle partitions to detect skewed partitions.
-      val leftMedSize = medianSize(left.mapStats)
-      val rightMedSize = medianSize(right.mapStats)
-      logDebug(
-        s"""
-          |Optimizing skewed join.
-          |Left side partitions size info:
-          |${getSizeInfo(leftMedSize, left.mapStats.bytesByPartitionId)}
-          |Right side partitions size info:
-          |${getSizeInfo(rightMedSize, right.mapStats.bytesByPartitionId)}
-        """.stripMargin)
-      val canSplitLeft = canSplitLeftSide(joinType)
-      val canSplitRight = canSplitRightSide(joinType)
-      // We use the actual partition sizes (may be coalesced) to calculate target size, so that
-      // the final data distribution is even (coalesced partitions + split partitions).
-      val leftActualSizes = left.partitionsWithSizes.map(_._2)
-      val rightActualSizes = right.partitionsWithSizes.map(_._2)
-      val leftTargetSize = targetSize(leftActualSizes, leftMedSize)
-      val rightTargetSize = targetSize(rightActualSizes, rightMedSize)
-
-      val leftSidePartitions = mutable.ArrayBuffer.empty[ShufflePartitionSpec]
-      val rightSidePartitions = mutable.ArrayBuffer.empty[ShufflePartitionSpec]
-      var numSkewedLeft = 0
-      var numSkewedRight = 0
-      for (partitionIndex <- 0 until numPartitions) {
-        val leftActualSize = leftActualSizes(partitionIndex)
-        val isLeftSkew = isSkewed(leftActualSize, leftMedSize) && canSplitLeft
-        val leftPartSpec = left.partitionsWithSizes(partitionIndex)._1
-        val isLeftCoalesced = leftPartSpec.startReducerIndex + 1 < leftPartSpec.endReducerIndex
-
-        val rightActualSize = rightActualSizes(partitionIndex)
-        val isRightSkew = isSkewed(rightActualSize, rightMedSize) && canSplitRight
-        val rightPartSpec = right.partitionsWithSizes(partitionIndex)._1
-        val isRightCoalesced = rightPartSpec.startReducerIndex + 1 < rightPartSpec.endReducerIndex
-
-        // A skewed partition should never be coalesced, but skip it here just to be safe.
-        val leftParts = if (isLeftSkew && !isLeftCoalesced) {
-          val reducerId = leftPartSpec.startReducerIndex
-          val skewSpecs = createSkewPartitionSpecs(
-            left.shuffleStage.shuffle.shuffleDependency.shuffleId, reducerId, leftTargetSize)
-          if (skewSpecs.isDefined) {
-            logDebug(s"Left side partition $partitionIndex " +
-              s"(${FileUtils.byteCountToDisplaySize(leftActualSize)}) is skewed, " +
-              s"split it into ${skewSpecs.get.length} parts.")
-            numSkewedLeft += 1
-          }
-          skewSpecs.getOrElse(Seq(leftPartSpec))
-        } else {
-          Seq(leftPartSpec)
-        }
+  private def tryOptimizeJoinChildren(
+      left: ShuffleStageInfo,
+      right: ShuffleStageInfo,
+      joinType: JoinType): Option[(SparkPlan, SparkPlan)] = {
+    assert(left.partitionsWithSizes.length == right.partitionsWithSizes.length)
+    val numPartitions = left.partitionsWithSizes.length
+    // We use the median size of the original shuffle partitions to detect skewed partitions.
+    val leftMedSize = medianSize(left.mapStats)
+    val rightMedSize = medianSize(right.mapStats)
+    logDebug(
+      s"""
+         |Optimizing skewed join.
+         |Left side partitions size info:
+         |${getSizeInfo(leftMedSize, left.mapStats.bytesByPartitionId)}
+         |Right side partitions size info:
+         |${getSizeInfo(rightMedSize, right.mapStats.bytesByPartitionId)}
+      """.stripMargin)
+    val canSplitLeft = canSplitLeftSide(joinType)
+    val canSplitRight = canSplitRightSide(joinType)
+    // We use the actual partition sizes (may be coalesced) to calculate target size, so that
+    // the final data distribution is even (coalesced partitions + split partitions).
+    val leftActualSizes = left.partitionsWithSizes.map(_._2)
+    val rightActualSizes = right.partitionsWithSizes.map(_._2)
+    val leftTargetSize = targetSize(leftActualSizes, leftMedSize)
+    val rightTargetSize = targetSize(rightActualSizes, rightMedSize)
 
-        // A skewed partition should never be coalesced, but skip it here just to be safe.
-        val rightParts = if (isRightSkew && !isRightCoalesced) {
-          val reducerId = rightPartSpec.startReducerIndex
-          val skewSpecs = createSkewPartitionSpecs(
-            right.shuffleStage.shuffle.shuffleDependency.shuffleId, reducerId, rightTargetSize)
-          if (skewSpecs.isDefined) {
-            logDebug(s"Right side partition $partitionIndex " +
-              s"(${FileUtils.byteCountToDisplaySize(rightActualSize)}) is skewed, " +
-              s"split it into ${skewSpecs.get.length} parts.")
-            numSkewedRight += 1
-          }
-          skewSpecs.getOrElse(Seq(rightPartSpec))
-        } else {
-          Seq(rightPartSpec)
+    val leftSidePartitions = mutable.ArrayBuffer.empty[ShufflePartitionSpec]
+    val rightSidePartitions = mutable.ArrayBuffer.empty[ShufflePartitionSpec]
+    var numSkewedLeft = 0
+    var numSkewedRight = 0
+    for (partitionIndex <- 0 until numPartitions) {
+      val leftActualSize = leftActualSizes(partitionIndex)
+      val isLeftSkew = isSkewed(leftActualSize, leftMedSize) && canSplitLeft
+      val leftPartSpec = left.partitionsWithSizes(partitionIndex)._1
+      val isLeftCoalesced = leftPartSpec.startReducerIndex + 1 < leftPartSpec.endReducerIndex
+
+      val rightActualSize = rightActualSizes(partitionIndex)
+      val isRightSkew = isSkewed(rightActualSize, rightMedSize) && canSplitRight
+      val rightPartSpec = right.partitionsWithSizes(partitionIndex)._1
+      val isRightCoalesced = rightPartSpec.startReducerIndex + 1 < rightPartSpec.endReducerIndex
+
+      // A skewed partition should never be coalesced, but skip it here just to be safe.
+      val leftParts = if (isLeftSkew && !isLeftCoalesced) {
+        val reducerId = leftPartSpec.startReducerIndex
+        val skewSpecs = createSkewPartitionSpecs(
+          left.mapStats.shuffleId, reducerId, leftTargetSize)
+        if (skewSpecs.isDefined) {
+          logDebug(s"Left side partition $partitionIndex " +
+            s"(${FileUtils.byteCountToDisplaySize(leftActualSize)}) is skewed, " +
+            s"split it into ${skewSpecs.get.length} parts.")
+          numSkewedLeft += 1
         }
+        skewSpecs.getOrElse(Seq(leftPartSpec))
+      } else {
+        Seq(leftPartSpec)
+      }
 
-        for {
-          leftSidePartition <- leftParts
-          rightSidePartition <- rightParts
-        } {
-          leftSidePartitions += leftSidePartition
-          rightSidePartitions += rightSidePartition
+      // A skewed partition should never be coalesced, but skip it here just to be safe.
+      val rightParts = if (isRightSkew && !isRightCoalesced) {
+        val reducerId = rightPartSpec.startReducerIndex
+        val skewSpecs = createSkewPartitionSpecs(
+          right.mapStats.shuffleId, reducerId, rightTargetSize)
+        if (skewSpecs.isDefined) {
+          logDebug(s"Right side partition $partitionIndex " +
+            s"(${FileUtils.byteCountToDisplaySize(rightActualSize)}) is skewed, " +
+            s"split it into ${skewSpecs.get.length} parts.")
+          numSkewedRight += 1
         }
+        skewSpecs.getOrElse(Seq(rightPartSpec))
+      } else {
+        Seq(rightPartSpec)
       }
 
-      logDebug(s"number of skewed partitions: left $numSkewedLeft, right $numSkewedRight")
-      if (numSkewedLeft > 0 || numSkewedRight > 0) {
-        val newLeft = CustomShuffleReaderExec(left.shuffleStage, leftSidePartitions)
-        val newRight = CustomShuffleReaderExec(right.shuffleStage, rightSidePartitions)
+      for {
+        leftSidePartition <- leftParts
+        rightSidePartition <- rightParts
+      } {
+        leftSidePartitions += leftSidePartition
+        rightSidePartitions += rightSidePartition
+      }
+    }
+    logDebug(s"number of skewed partitions: left $numSkewedLeft, right $numSkewedRight")
+    if (numSkewedLeft > 0 || numSkewedRight > 0) {
+      Some((CustomShuffleReaderExec(left.shuffleStage, leftSidePartitions.toSeq),
+        CustomShuffleReaderExec(right.shuffleStage, rightSidePartitions.toSeq)))
+    } else {
+      None
+    }
+  }
+
+  def optimizeSkewJoin(plan: SparkPlan): SparkPlan = plan.transformUp {
+    case smj @ SortMergeJoinExec(_, _, joinType, _,
+        s1 @ SortExec(_, _, ShuffleStage(left: ShuffleStageInfo), _),
+        s2 @ SortExec(_, _, ShuffleStage(right: ShuffleStageInfo), _), isSkewJoin)
+        if !isSkewJoin && supportedJoinTypes.contains(joinType) =>
+      val newChildren = tryOptimizeJoinChildren(left, right, joinType)
+      if (newChildren.isDefined) {
+        val (newLeft, newRight) = newChildren.get
         smj.copy(
           left = s1.copy(child = newLeft), right = s2.copy(child = newRight), isSkewJoin = true)
       } else {
         smj
       }
+
+    case shj @ ShuffledHashJoinExec(_, _, joinType, _, _,
+        ShuffleStage(left: ShuffleStageInfo),
+        ShuffleStage(right: ShuffleStageInfo), isSkewJoin)
+        if !isSkewJoin && supportedJoinTypes.contains(joinType) =>
+      val newChildren = tryOptimizeJoinChildren(left, right, joinType)
+      if (newChildren.isDefined) {
+        val (newLeft, newRight) = newChildren.get
+        shj.copy(left = newLeft, right = newRight, isSkewJoin = true)
+      } else {
+        shj
+      }
   }
 
   override def apply(plan: SparkPlan): SparkPlan = {
@@ -290,7 +314,9 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
 
 private object ShuffleStage {
   def unapply(plan: SparkPlan): Option[ShuffleStageInfo] = plan match {
-    case s: ShuffleQueryStageExec if s.mapStats.isDefined =>
+    case s: ShuffleQueryStageExec
+        if s.mapStats.isDefined &&
+          OptimizeSkewedJoin.supportedShuffleOrigins.contains(s.shuffle.shuffleOrigin) =>
       val mapStats = s.mapStats.get
       val sizes = mapStats.bytesByPartitionId
       val partitions = sizes.zipWithIndex.map {
@@ -299,7 +325,8 @@ private object ShuffleStage {
       Some(ShuffleStageInfo(s, mapStats, partitions))
 
     case CustomShuffleReaderExec(s: ShuffleQueryStageExec, partitionSpecs)
-      if s.mapStats.isDefined && partitionSpecs.nonEmpty =>
+        if s.mapStats.isDefined && partitionSpecs.nonEmpty &&
+          OptimizeSkewedJoin.supportedShuffleOrigins.contains(s.shuffle.shuffleOrigin) =>
       val mapStats = s.mapStats.get
       val sizes = mapStats.bytesByPartitionId
       val partitions = partitionSpecs.map {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/PlanAdaptiveDynamicPruningFilters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/PlanAdaptiveDynamicPruningFilters.scala
new file mode 100644
index 0000000000000..3ef18108d2b30
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/PlanAdaptiveDynamicPruningFilters.scala
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.adaptive
+
+import org.apache.spark.sql.catalyst.expressions.{BindReferences, DynamicPruningExpression, Literal}
+import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight}
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.execution._
+import org.apache.spark.sql.execution.exchange.BroadcastExchangeExec
+import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, HashedRelationBroadcastMode, HashJoin}
+
+/**
+ * A rule to insert dynamic pruning predicates in order to reuse the results of broadcast.
+ */
+case class PlanAdaptiveDynamicPruningFilters(
+    rootPlan: AdaptiveSparkPlanExec) extends Rule[SparkPlan] with AdaptiveSparkPlanHelper {
+  def apply(plan: SparkPlan): SparkPlan = {
+    if (!conf.dynamicPartitionPruningEnabled) {
+      return plan
+    }
+
+    plan transformAllExpressions {
+      case DynamicPruningExpression(InSubqueryExec(
+          value, SubqueryAdaptiveBroadcastExec(name, index, buildKeys,
+          adaptivePlan: AdaptiveSparkPlanExec), exprId, _)) =>
+        val packedKeys = BindReferences.bindReferences(
+          HashJoin.rewriteKeyExpr(buildKeys), adaptivePlan.executedPlan.output)
+        val mode = HashedRelationBroadcastMode(packedKeys)
+        // plan a broadcast exchange of the build side of the join
+        val exchange = BroadcastExchangeExec(mode, adaptivePlan.executedPlan)
+
+        val canReuseExchange = conf.exchangeReuseEnabled && buildKeys.nonEmpty &&
+          find(rootPlan) {
+            case BroadcastHashJoinExec(_, _, _, BuildLeft, _, left, _, _) =>
+              left.sameResult(exchange)
+            case BroadcastHashJoinExec(_, _, _, BuildRight, _, _, right, _) =>
+              right.sameResult(exchange)
+            case _ => false
+          }.isDefined
+
+        if (canReuseExchange) {
+          exchange.setLogicalLink(adaptivePlan.executedPlan.logicalLink.get)
+          val newAdaptivePlan = adaptivePlan.copy(inputPlan = exchange)
+
+          val broadcastValues = SubqueryBroadcastExec(
+            name, index, buildKeys, newAdaptivePlan)
+          DynamicPruningExpression(InSubqueryExec(value, broadcastValues, exprId))
+        } else {
+          DynamicPruningExpression(Literal.TrueLiteral)
+        }
+    }
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/PlanAdaptiveSubqueries.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/PlanAdaptiveSubqueries.scala
index f845b6b16ee3a..a2e4397a3697a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/PlanAdaptiveSubqueries.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/PlanAdaptiveSubqueries.scala
@@ -18,15 +18,19 @@
 package org.apache.spark.sql.execution.adaptive
 
 import org.apache.spark.sql.catalyst.expressions
-import org.apache.spark.sql.catalyst.expressions.{CreateNamedStruct, ListQuery, Literal}
+import org.apache.spark.sql.catalyst.expressions.{CreateNamedStruct, DynamicPruningExpression, ListQuery, Literal}
 import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreePattern.{DYNAMIC_PRUNING_SUBQUERY, IN_SUBQUERY,
+  SCALAR_SUBQUERY}
 import org.apache.spark.sql.execution
-import org.apache.spark.sql.execution.{InSubqueryExec, SparkPlan, SubqueryExec}
+import org.apache.spark.sql.execution.{BaseSubqueryExec, InSubqueryExec, SparkPlan}
 
-case class PlanAdaptiveSubqueries(subqueryMap: Map[Long, SubqueryExec]) extends Rule[SparkPlan] {
+case class PlanAdaptiveSubqueries(
+    subqueryMap: Map[Long, BaseSubqueryExec]) extends Rule[SparkPlan] {
 
   def apply(plan: SparkPlan): SparkPlan = {
-    plan.transformAllExpressions {
+    plan.transformAllExpressionsWithPruning(
+      _.containsAnyPattern(SCALAR_SUBQUERY, IN_SUBQUERY, DYNAMIC_PRUNING_SUBQUERY)) {
       case expressions.ScalarSubquery(_, _, exprId) =>
         execution.ScalarSubquery(subqueryMap(exprId.id), exprId)
       case expressions.InSubquery(values, ListQuery(_, _, exprId, _)) =>
@@ -40,6 +44,8 @@ case class PlanAdaptiveSubqueries(subqueryMap: Map[Long, SubqueryExec]) extends
           )
         }
         InSubqueryExec(expr, subqueryMap(exprId.id), exprId)
+      case expressions.DynamicPruningSubquery(value, _, _, _, _, exprId) =>
+        DynamicPruningExpression(InSubqueryExec(value, subqueryMap(exprId.id), exprId))
     }
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala
index f414f854b92ae..a4ec4f1c22727 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala
@@ -18,19 +18,21 @@
 package org.apache.spark.sql.execution.adaptive
 
 import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicReference
 
 import scala.concurrent.{Future, Promise}
 
-import org.apache.spark.{FutureAction, MapOutputStatistics, SparkException}
+import org.apache.spark.{FutureAction, MapOutputStatistics}
 import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.logical.Statistics
 import org.apache.spark.sql.catalyst.plans.physical.Partitioning
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.execution.exchange._
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.vectorized.ColumnarBatch
 import org.apache.spark.util.ThreadUtils
 
 /**
@@ -56,6 +58,11 @@ abstract class QueryStageExec extends LeafExecNode {
    */
   val plan: SparkPlan
 
+  /**
+   * The canonicalized plan before applying query stage optimizer rules.
+   */
+  val _canonicalized: SparkPlan
+
   /**
    * Materialize this query stage, to prepare for the execution, like submitting map stages,
    * broadcasting data, etc. The caller side can use the returned [[Future]] to wait until this
@@ -74,27 +81,32 @@ abstract class QueryStageExec extends LeafExecNode {
    * stage is ready.
    */
   final def materialize(): Future[Any] = executeQuery {
+    logDebug(s"Materialize query stage ${this.getClass.getSimpleName}: $id")
     doMaterialize()
   }
 
   def newReuseInstance(newStageId: Int, newOutput: Seq[Attribute]): QueryStageExec
 
+  /**
+   * Returns the runtime statistics after stage materialization.
+   */
+  def getRuntimeStatistics: Statistics
+
   /**
    * Compute the statistics of the query stage if executed, otherwise None.
    */
-  def computeStats(): Option[Statistics] = resultOption.map { _ =>
-    // Metrics `dataSize` are available in both `ShuffleExchangeExec` and `BroadcastExchangeExec`.
-    val exchange = plan match {
-      case r: ReusedExchangeExec => r.child
-      case e: Exchange => e
-      case _ => throw new IllegalStateException("wrong plan for query stage:\n " + plan.treeString)
-    }
-    Statistics(sizeInBytes = exchange.metrics("dataSize").value)
+  def computeStats(): Option[Statistics] = resultOption.get().map { _ =>
+    val runtimeStats = getRuntimeStatistics
+    val dataSize = runtimeStats.sizeInBytes.max(0)
+    val numOutputRows = runtimeStats.rowCount.map(_.max(0))
+    Statistics(dataSize, numOutputRows, isRuntime = true)
   }
 
   @transient
   @volatile
-  private[adaptive] var resultOption: Option[Any] = None
+  protected var _resultOption = new AtomicReference[Option[Any]](None)
+
+  private[adaptive] def resultOption: AtomicReference[Option[Any]] = _resultOption
 
   override def output: Seq[Attribute] = plan.output
   override def outputPartitioning: Partitioning = plan.outputPartitioning
@@ -106,8 +118,10 @@ abstract class QueryStageExec extends LeafExecNode {
 
   protected override def doPrepare(): Unit = plan.prepare()
   protected override def doExecute(): RDD[InternalRow] = plan.execute()
+  override def supportsColumnar: Boolean = plan.supportsColumnar
+  protected override def doExecuteColumnar(): RDD[ColumnarBatch] = plan.executeColumnar()
   override def doExecuteBroadcast[T](): Broadcast[T] = plan.executeBroadcast()
-  override def doCanonicalize(): SparkPlan = plan.canonicalized
+  override def doCanonicalize(): SparkPlan = _canonicalized
 
   protected override def stringArgs: Iterator[Any] = Iterator.single(id)
 
@@ -119,7 +133,8 @@ abstract class QueryStageExec extends LeafExecNode {
       prefix: String = "",
       addSuffix: Boolean = false,
       maxFields: Int,
-      printNodeId: Boolean): Unit = {
+      printNodeId: Boolean,
+      indent: Int = 0): Unit = {
     super.generateTreeString(depth,
       lastChildren,
       append,
@@ -127,34 +142,41 @@ abstract class QueryStageExec extends LeafExecNode {
       prefix,
       addSuffix,
       maxFields,
-      printNodeId)
+      printNodeId,
+      indent)
     plan.generateTreeString(
-      depth + 1, lastChildren :+ true, append, verbose, "", false, maxFields, printNodeId)
+      depth + 1, lastChildren :+ true, append, verbose, "", false, maxFields, printNodeId, indent)
   }
 }
 
 /**
- * A shuffle query stage whose child is a [[ShuffleExchangeExec]] or [[ReusedExchangeExec]].
+ * A shuffle query stage whose child is a [[ShuffleExchangeLike]] or [[ReusedExchangeExec]].
+ *
+ * @param id the query stage id.
+ * @param plan the underlying plan.
+ * @param _canonicalized the canonicalized plan before applying query stage optimizer rules.
  */
 case class ShuffleQueryStageExec(
     override val id: Int,
-    override val plan: SparkPlan) extends QueryStageExec {
+    override val plan: SparkPlan,
+    override val _canonicalized: SparkPlan) extends QueryStageExec {
 
   @transient val shuffle = plan match {
-    case s: ShuffleExchangeExec => s
-    case ReusedExchangeExec(_, s: ShuffleExchangeExec) => s
+    case s: ShuffleExchangeLike => s
+    case ReusedExchangeExec(_, s: ShuffleExchangeLike) => s
     case _ =>
-      throw new IllegalStateException("wrong plan for shuffle stage:\n " + plan.treeString)
+      throw new IllegalStateException(s"wrong plan for shuffle stage:\n ${plan.treeString}")
   }
 
-  override def doMaterialize(): Future[Any] = {
-    shuffle.mapOutputStatisticsFuture
-  }
+  override def doMaterialize(): Future[Any] = shuffle.mapOutputStatisticsFuture
 
   override def newReuseInstance(newStageId: Int, newOutput: Seq[Attribute]): QueryStageExec = {
-    ShuffleQueryStageExec(
+    val reuse = ShuffleQueryStageExec(
       newStageId,
-      ReusedExchangeExec(newOutput, shuffle))
+      ReusedExchangeExec(newOutput, shuffle),
+      _canonicalized)
+    reuse._resultOption = this._resultOption
+    reuse
   }
 
   override def cancel(): Unit = {
@@ -171,35 +193,40 @@ case class ShuffleQueryStageExec(
    * this method returns None, as there is no map statistics.
    */
   def mapStats: Option[MapOutputStatistics] = {
-    assert(resultOption.isDefined, "ShuffleQueryStageExec should already be ready")
-    val stats = resultOption.get.asInstanceOf[MapOutputStatistics]
+    assert(resultOption.get().isDefined, s"${getClass.getSimpleName} should already be ready")
+    val stats = resultOption.get().get.asInstanceOf[MapOutputStatistics]
     Option(stats)
   }
+
+  override def getRuntimeStatistics: Statistics = shuffle.runtimeStatistics
 }
 
 /**
- * A broadcast query stage whose child is a [[BroadcastExchangeExec]] or [[ReusedExchangeExec]].
+ * A broadcast query stage whose child is a [[BroadcastExchangeLike]] or [[ReusedExchangeExec]].
+ *
+ * @param id the query stage id.
+ * @param plan the underlying plan.
+ * @param _canonicalized the canonicalized plan before applying query stage optimizer rules.
  */
 case class BroadcastQueryStageExec(
     override val id: Int,
-    override val plan: SparkPlan) extends QueryStageExec {
+    override val plan: SparkPlan,
+    override val _canonicalized: SparkPlan) extends QueryStageExec {
 
   @transient val broadcast = plan match {
-    case b: BroadcastExchangeExec => b
-    case ReusedExchangeExec(_, b: BroadcastExchangeExec) => b
+    case b: BroadcastExchangeLike => b
+    case ReusedExchangeExec(_, b: BroadcastExchangeLike) => b
     case _ =>
-      throw new IllegalStateException("wrong plan for broadcast stage:\n " + plan.treeString)
+      throw new IllegalStateException(s"wrong plan for broadcast stage:\n ${plan.treeString}")
   }
 
   @transient private lazy val materializeWithTimeout = {
     val broadcastFuture = broadcast.completionFuture
-    val timeout = SQLConf.get.broadcastTimeout
+    val timeout = conf.broadcastTimeout
     val promise = Promise[Any]()
     val fail = BroadcastQueryStageExec.scheduledExecutor.schedule(new Runnable() {
       override def run(): Unit = {
-        promise.tryFailure(new SparkException(s"Could not execute broadcast in $timeout secs. " +
-          s"You can increase the timeout for broadcasts via ${SQLConf.BROADCAST_TIMEOUT.key} or " +
-          s"disable broadcast join by setting ${SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key} to -1"))
+        promise.tryFailure(QueryExecutionErrors.executeBroadcastTimeoutError(timeout))
       }
     }, timeout, TimeUnit.SECONDS)
     broadcastFuture.onComplete(_ => fail.cancel(false))(AdaptiveSparkPlanExec.executionContext)
@@ -212,9 +239,12 @@ case class BroadcastQueryStageExec(
   }
 
   override def newReuseInstance(newStageId: Int, newOutput: Seq[Attribute]): QueryStageExec = {
-    BroadcastQueryStageExec(
+    val reuse = BroadcastQueryStageExec(
       newStageId,
-      ReusedExchangeExec(newOutput, broadcast))
+      ReusedExchangeExec(newOutput, broadcast),
+      _canonicalized)
+    reuse._resultOption = this._resultOption
+    reuse
   }
 
   override def cancel(): Unit = {
@@ -223,6 +253,8 @@ case class BroadcastQueryStageExec(
       broadcast.relationFuture.cancel(true)
     }
   }
+
+  override def getRuntimeStatistics: Statistics = broadcast.runtimeStatistics
 }
 
 object BroadcastQueryStageExec {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ReuseAdaptiveSubquery.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ReuseAdaptiveSubquery.scala
index 432f7e204791b..71540dbd39f95 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ReuseAdaptiveSubquery.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ReuseAdaptiveSubquery.scala
@@ -21,10 +21,8 @@ import scala.collection.concurrent.TrieMap
 
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution.{BaseSubqueryExec, ExecSubqueryExpression, ReusedSubqueryExec, SparkPlan}
-import org.apache.spark.sql.internal.SQLConf
 
 case class ReuseAdaptiveSubquery(
-    conf: SQLConf,
     reuseMap: TrieMap[SparkPlan, BaseSubqueryExec]) extends Rule[SparkPlan] {
 
   def apply(plan: SparkPlan): SparkPlan = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsUtil.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsUtil.scala
index d6e44b780d772..a70a5322a6243 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsUtil.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsUtil.scala
@@ -92,9 +92,9 @@ object ShufflePartitionsUtil extends Logging {
     var coalescedSize = 0L
     var i = 0
 
-    def createPartitionSpec(): Unit = {
+    def createPartitionSpec(forceCreate: Boolean = false): Unit = {
       // Skip empty inputs, as it is a waste to launch an empty task.
-      if (coalescedSize > 0) {
+      if (coalescedSize > 0 || forceCreate) {
         partitionSpecs += CoalescedPartitionSpec(latestSplitPoint, i)
       }
     }
@@ -120,8 +120,13 @@ object ShufflePartitionsUtil extends Logging {
       }
       i += 1
     }
-    createPartitionSpec()
-    partitionSpecs
+    // Create at least one partition if all partitions are empty.
+    createPartitionSpec(partitionSpecs.isEmpty)
+    partitionSpecs.toSeq
+  }
+
+  def createEmptyPartition(): ShufflePartitionSpec = {
+    CoalescedPartitionSpec(0, 0)
   }
 
   /**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/simpleCosting.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/simpleCosting.scala
index 67cd720bb5b33..7f026835dcf48 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/simpleCosting.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/simpleCosting.scala
@@ -17,8 +17,9 @@
 
 package org.apache.spark.sql.execution.adaptive
 
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.execution.SparkPlan
-import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
+import org.apache.spark.sql.execution.exchange.ShuffleExchangeLike
 
 /**
  * A simple implementation of [[Cost]], which takes a number of [[Long]] as the cost value.
@@ -29,19 +30,19 @@ case class SimpleCost(value: Long) extends Cost {
     case SimpleCost(thatValue) =>
       if (value < thatValue) -1 else if (value > thatValue) 1 else 0
     case _ =>
-      throw new IllegalArgumentException(s"Could not compare cost with $that")
+      throw QueryExecutionErrors.cannotCompareCostWithTargetCostError(that.toString)
   }
 }
 
 /**
  * A simple implementation of [[CostEvaluator]], which counts the number of
- * [[ShuffleExchangeExec]] nodes in the plan.
+ * [[ShuffleExchangeLike]] nodes in the plan.
  */
 object SimpleCostEvaluator extends CostEvaluator {
 
   override def evaluateCost(plan: SparkPlan): Cost = {
     val cost = plan.collect {
-      case s: ShuffleExchangeExec => s
+      case s: ShuffleExchangeLike => s
     }.size
     SimpleCost(cost)
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala
index 56a287d4d0279..761ac20e84744 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala
@@ -135,20 +135,12 @@ object AggUtils {
       groupingExpressions: Seq[NamedExpression],
       functionsWithDistinct: Seq[AggregateExpression],
       functionsWithoutDistinct: Seq[AggregateExpression],
+      distinctExpressions: Seq[Expression],
+      normalizedNamedDistinctExpressions: Seq[NamedExpression],
       resultExpressions: Seq[NamedExpression],
       child: SparkPlan): Seq[SparkPlan] = {
 
-    // functionsWithDistinct is guaranteed to be non-empty. Even though it may contain more than one
-    // DISTINCT aggregate function, all of those functions will have the same column expressions.
-    // For example, it would be valid for functionsWithDistinct to be
-    // [COUNT(DISTINCT foo), MAX(DISTINCT foo)], but [COUNT(DISTINCT bar), COUNT(DISTINCT foo)] is
-    // disallowed because those two distinct aggregates have different column expressions.
-    val distinctExpressions = functionsWithDistinct.head.aggregateFunction.children
-    val namedDistinctExpressions = distinctExpressions.map {
-      case ne: NamedExpression => ne
-      case other => Alias(other, other.toString)()
-    }
-    val distinctAttributes = namedDistinctExpressions.map(_.toAttribute)
+    val distinctAttributes = normalizedNamedDistinctExpressions.map(_.toAttribute)
     val groupingAttributes = groupingExpressions.map(_.toAttribute)
 
     // 1. Create an Aggregate Operator for partial aggregations.
@@ -159,7 +151,7 @@ object AggUtils {
       // DISTINCT column. For example, for AVG(DISTINCT value) GROUP BY key, the grouping
       // expressions will be [key, value].
       createAggregate(
-        groupingExpressions = groupingExpressions ++ namedDistinctExpressions,
+        groupingExpressions = groupingExpressions ++ normalizedNamedDistinctExpressions,
         aggregateExpressions = aggregateExpressions,
         aggregateAttributes = aggregateAttributes,
         resultExpressions = groupingAttributes ++ distinctAttributes ++
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggregationIterator.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggregationIterator.scala
index 527a9eac9948e..52bee7ecf3bae 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggregationIterator.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggregationIterator.scala
@@ -25,7 +25,8 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate._
 
 /**
- * The base class of [[SortBasedAggregationIterator]] and [[TungstenAggregationIterator]].
+ * The base class of [[SortBasedAggregationIterator]], [[TungstenAggregationIterator]] and
+ * [[ObjectAggregationIterator]].
  * It mainly contains two parts:
  * 1. It initializes aggregate functions.
  * 2. It creates two functions, `processRow` and `generateOutput` based on [[AggregateMode]] of
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/BaseAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/BaseAggregateExec.scala
index f1e053f7fb2a5..c676609bc37e4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/BaseAggregateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/BaseAggregateExec.scala
@@ -17,14 +17,16 @@
 
 package org.apache.spark.sql.execution.aggregate
 
-import org.apache.spark.sql.catalyst.expressions.{Attribute, NamedExpression}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, AttributeSet, Expression, NamedExpression}
 import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Final, PartialMerge}
-import org.apache.spark.sql.execution.{ExplainUtils, UnaryExecNode}
+import org.apache.spark.sql.catalyst.plans.physical.{AllTuples, ClusteredDistribution, Distribution, UnspecifiedDistribution}
+import org.apache.spark.sql.execution.{AliasAwareOutputPartitioning, ExplainUtils, UnaryExecNode}
 
 /**
  * Holds common logic for aggregate operators
  */
-trait BaseAggregateExec extends UnaryExecNode {
+trait BaseAggregateExec extends UnaryExecNode with AliasAwareOutputPartitioning {
+  def requiredChildDistributionExpressions: Option[Seq[Expression]]
   def groupingExpressions: Seq[NamedExpression]
   def aggregateExpressions: Seq[AggregateExpression]
   def aggregateAttributes: Seq[Attribute]
@@ -53,15 +55,44 @@ trait BaseAggregateExec extends UnaryExecNode {
       // can't bind the `mergeExpressions` with the output of the partial aggregate, as they use
       // the `inputAggBufferAttributes` of the original `DeclarativeAggregate` before copy. Instead,
       // we shall use `inputAggBufferAttributes` after copy to match the new `mergeExpressions`.
-      val aggAttrs = aggregateExpressions
-        // there're exactly four cases needs `inputAggBufferAttributes` from child according to the
-        // agg planning in `AggUtils`: Partial -> Final, PartialMerge -> Final,
-        // Partial -> PartialMerge, PartialMerge -> PartialMerge.
-        .filter(a => a.mode == Final || a.mode == PartialMerge).map(_.aggregateFunction)
-        .flatMap(_.inputAggBufferAttributes)
+      val aggAttrs = inputAggBufferAttributes
       child.output.dropRight(aggAttrs.length) ++ aggAttrs
     } else {
       child.output
     }
   }
+
+  private val inputAggBufferAttributes: Seq[Attribute] = {
+    aggregateExpressions
+      // there're exactly four cases needs `inputAggBufferAttributes` from child according to the
+      // agg planning in `AggUtils`: Partial -> Final, PartialMerge -> Final,
+      // Partial -> PartialMerge, PartialMerge -> PartialMerge.
+      .filter(a => a.mode == Final || a.mode == PartialMerge)
+      .flatMap(_.aggregateFunction.inputAggBufferAttributes)
+  }
+
+  protected val aggregateBufferAttributes: Seq[AttributeReference] = {
+    aggregateExpressions.flatMap(_.aggregateFunction.aggBufferAttributes)
+  }
+
+  override def producedAttributes: AttributeSet =
+    AttributeSet(aggregateAttributes) ++
+    AttributeSet(resultExpressions.diff(groupingExpressions).map(_.toAttribute)) ++
+    AttributeSet(aggregateBufferAttributes) ++
+    // it's not empty when the inputAggBufferAttributes is not equal to the aggregate buffer
+    // attributes of the child Aggregate, when the child Aggregate contains the subquery in
+    // AggregateFunction. See SPARK-31620 for more details.
+    AttributeSet(inputAggBufferAttributes.filterNot(child.output.contains))
+
+  override def output: Seq[Attribute] = resultExpressions.map(_.toAttribute)
+
+  override protected def outputExpressions: Seq[NamedExpression] = resultExpressions
+
+  override def requiredChildDistribution: List[Distribution] = {
+    requiredChildDistributionExpressions match {
+      case Some(exprs) if exprs.isEmpty => AllTuples :: Nil
+      case Some(exprs) => ClusteredDistribution(exprs) :: Nil
+      case None => UnspecifiedDistribution :: Nil
+    }
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
index 9c07ea10a87e7..3c1304e9cdad8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
@@ -22,16 +22,14 @@ import java.util.concurrent.TimeUnit._
 import scala.collection.mutable
 
 import org.apache.spark.TaskContext
-import org.apache.spark.memory.{SparkOutOfMemoryError, TaskMemoryManager}
+import org.apache.spark.memory.SparkOutOfMemoryError
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.errors._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.BindReferences.bindReferences
 import org.apache.spark.sql.catalyst.expressions.aggregate._
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
-import org.apache.spark.sql.catalyst.plans.physical._
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.NANOS_PER_MILLIS
 import org.apache.spark.sql.catalyst.util.truncatedString
 import org.apache.spark.sql.execution._
@@ -54,12 +52,8 @@ case class HashAggregateExec(
     resultExpressions: Seq[NamedExpression],
     child: SparkPlan)
   extends BaseAggregateExec
-    with BlockingOperatorWithCodegen
-    with AliasAwareOutputPartitioning {
-
-  private[this] val aggregateBufferAttributes = {
-    aggregateExpressions.flatMap(_.aggregateFunction.aggBufferAttributes)
-  }
+  with BlockingOperatorWithCodegen
+  with GeneratePredicateHelper {
 
   require(HashAggregateExec.supportsAggregate(aggregateBufferAttributes))
 
@@ -75,23 +69,6 @@ case class HashAggregateExec(
     "avgHashProbe" ->
       SQLMetrics.createAverageMetric(sparkContext, "avg hash probe bucket list iters"))
 
-  override def output: Seq[Attribute] = resultExpressions.map(_.toAttribute)
-
-  override protected def outputExpressions: Seq[NamedExpression] = resultExpressions
-
-  override def producedAttributes: AttributeSet =
-    AttributeSet(aggregateAttributes) ++
-    AttributeSet(resultExpressions.diff(groupingExpressions).map(_.toAttribute)) ++
-    AttributeSet(aggregateBufferAttributes)
-
-  override def requiredChildDistribution: List[Distribution] = {
-    requiredChildDistributionExpressions match {
-      case Some(exprs) if exprs.isEmpty => AllTuples :: Nil
-      case Some(exprs) if exprs.nonEmpty => ClusteredDistribution(exprs) :: Nil
-      case None => UnspecifiedDistribution :: Nil
-    }
-  }
-
   // This is for testing. We force TungstenAggregationIterator to fall back to the unsafe row hash
   // map and/or the sort-based aggregation once it has processed a given number of input rows.
   private val testFallbackStartsAt: Option[(Int, Int)] = {
@@ -103,7 +80,7 @@ case class HashAggregateExec(
     }
   }
 
-  protected override def doExecute(): RDD[InternalRow] = attachTree(this, "execute") {
+  protected override def doExecute(): RDD[InternalRow] = {
     val numOutputRows = longMetric("numOutputRows")
     val peakMemory = longMetric("peakMemory")
     val spillSize = longMetric("spillSize")
@@ -154,10 +131,8 @@ case class HashAggregateExec(
   override def usedInputs: AttributeSet = inputSet
 
   override def supportCodegen: Boolean = {
-    // ImperativeAggregate and filter predicate are not supported right now
-    // TODO: SPARK-30027 Support codegen for filter exprs in HashAggregateExec
-    !(aggregateExpressions.exists(_.aggregateFunction.isInstanceOf[ImperativeAggregate]) ||
-        aggregateExpressions.exists(_.filter.isDefined))
+    // ImperativeAggregate are not supported right now
+    !aggregateExpressions.exists(_.aggregateFunction.isInstanceOf[ImperativeAggregate])
   }
 
   override def inputRDDs(): Seq[RDD[InternalRow]] = {
@@ -277,7 +252,7 @@ case class HashAggregateExec(
       aggNames: Seq[String],
       aggBufferUpdatingExprs: Seq[Seq[Expression]],
       aggCodeBlocks: Seq[Block],
-      subExprs: Map[Expression, SubExprEliminationState]): Option[String] = {
+      subExprs: Map[Expression, SubExprEliminationState]): Option[Seq[String]] = {
     val exprValsInSubExprs = subExprs.flatMap { case (_, s) => s.value :: s.isNull :: Nil }
     if (exprValsInSubExprs.exists(_.isInstanceOf[SimpleExprValue])) {
       // `SimpleExprValue`s cannot be used as an input variable for split functions, so
@@ -286,7 +261,7 @@ case class HashAggregateExec(
     } else {
       val inputVars = aggBufferUpdatingExprs.map { aggExprsForOneFunc =>
         val inputVarsForOneFunc = aggExprsForOneFunc.map(
-          CodeGenerator.getLocalInputVariableValues(ctx, _, subExprs)).reduce(_ ++ _).toSeq
+          CodeGenerator.getLocalInputVariableValues(ctx, _, subExprs)._1).reduce(_ ++ _).toSeq
         val paramLength = CodeGenerator.calculateParamLengthFromExprValues(inputVarsForOneFunc)
 
         // Checks if a parameter length for the `aggExprsForOneFunc` does not go over the JVM limit
@@ -316,7 +291,7 @@ case class HashAggregateExec(
           val inputVariables = args.map(_.variableName).mkString(", ")
           s"$doAggFuncName($inputVariables);"
         }
-        Some(splitCodes.mkString("\n").trim)
+        Some(splitCodes)
       } else {
         val errMsg = "Failed to split aggregate code into small functions because the parameter " +
           "length of at least one split function went over the JVM limit: " +
@@ -331,6 +306,39 @@ case class HashAggregateExec(
     }
   }
 
+  private def generateEvalCodeForAggFuncs(
+      ctx: CodegenContext,
+      input: Seq[ExprCode],
+      inputAttrs: Seq[Attribute],
+      boundUpdateExprs: Seq[Seq[Expression]],
+      aggNames: Seq[String],
+      aggCodeBlocks: Seq[Block],
+      subExprs: SubExprCodes): String = {
+    val aggCodes = if (conf.codegenSplitAggregateFunc &&
+      aggCodeBlocks.map(_.length).sum > conf.methodSplitThreshold) {
+      val maybeSplitCodes = splitAggregateExpressions(
+        ctx, aggNames, boundUpdateExprs, aggCodeBlocks, subExprs.states)
+
+      maybeSplitCodes.getOrElse(aggCodeBlocks.map(_.code))
+    } else {
+      aggCodeBlocks.map(_.code)
+    }
+
+    aggCodes.zip(aggregateExpressions.map(ae => (ae.mode, ae.filter))).map {
+      case (aggCode, (Partial | Complete, Some(condition))) =>
+        // Note: wrap in "do { } while(false);", so the generated checks can jump out
+        // with "continue;"
+        s"""
+           |do {
+           |  ${generatePredicateCode(ctx, condition, inputAttrs, input)}
+           |  $aggCode
+           |} while(false);
+         """.stripMargin
+      case (aggCode, _) =>
+        aggCode
+    }.mkString("\n")
+  }
+
   private def doConsumeWithoutKeys(ctx: CodegenContext, input: Seq[ExprCode]): String = {
     // only have DeclarativeAggregate
     val functions = aggregateExpressions.map(_.aggregateFunction.asInstanceOf[DeclarativeAggregate])
@@ -377,24 +385,14 @@ case class HashAggregateExec(
        """.stripMargin
     }
 
-    val codeToEvalAggFunc = if (conf.codegenSplitAggregateFunc &&
-        aggCodeBlocks.map(_.length).sum > conf.methodSplitThreshold) {
-      val maybeSplitCode = splitAggregateExpressions(
-        ctx, aggNames, boundUpdateExprs, aggCodeBlocks, subExprs.states)
-
-      maybeSplitCode.getOrElse {
-        aggCodeBlocks.fold(EmptyBlock)(_ + _).code
-      }
-    } else {
-      aggCodeBlocks.fold(EmptyBlock)(_ + _).code
-    }
-
+    val codeToEvalAggFuncs = generateEvalCodeForAggFuncs(
+      ctx, input, inputAttrs, boundUpdateExprs, aggNames, aggCodeBlocks, subExprs)
     s"""
        |// do aggregate
        |// common sub-expressions
        |$effectiveCodes
        |// evaluate aggregate functions and update aggregation buffers
-       |$codeToEvalAggFunc
+       |$codeToEvalAggFuncs
      """.stripMargin
   }
 
@@ -437,8 +435,8 @@ case class HashAggregateExec(
     )
   }
 
-  def getTaskMemoryManager(): TaskMemoryManager = {
-    TaskContext.get().taskMemoryManager()
+  def getTaskContext(): TaskContext = {
+    TaskContext.get()
   }
 
   def getEmptyAggregationBuffer(): InternalRow = {
@@ -649,7 +647,7 @@ case class HashAggregateExec(
       (groupingKeySchema ++ bufferSchema).forall(f => CodeGenerator.isPrimitiveType(f.dataType) ||
         f.dataType.isInstanceOf[DecimalType] || f.dataType.isInstanceOf[StringType] ||
         f.dataType.isInstanceOf[CalendarIntervalType]) &&
-        bufferSchema.nonEmpty && modes.forall(mode => mode == Partial || mode == PartialMerge)
+        bufferSchema.nonEmpty
 
     // For vectorized hash map, We do not support byte array based decimal type for aggregate values
     // as ColumnVector.putDecimal for high-precision decimals doesn't currently support in-place
@@ -665,7 +663,7 @@ case class HashAggregateExec(
 
   private def enableTwoLevelHashMap(ctx: CodegenContext): Unit = {
     if (!checkIfFastHashMapSupported(ctx)) {
-      if (modes.forall(mode => mode == Partial || mode == PartialMerge) && !Utils.isTesting) {
+      if (!Utils.isTesting) {
         logInfo(s"${SQLConf.ENABLE_TWOLEVEL_AGG_MAP.key} is set to true, but"
           + " current version of codegened fast hashmap does not support this aggregate.")
       }
@@ -685,7 +683,18 @@ case class HashAggregateExec(
     } else if (sqlContext.conf.enableVectorizedHashMap) {
       logWarning("Two level hashmap is disabled but vectorized hashmap is enabled.")
     }
-    val bitMaxCapacity = sqlContext.conf.fastHashAggregateRowMaxCapacityBit
+    val bitMaxCapacity = testFallbackStartsAt match {
+      case Some((fastMapCounter, _)) =>
+        // In testing, with fall back counter of fast hash map (`fastMapCounter`), set the max bit
+        // of map to be no more than log2(`fastMapCounter`). This helps control the number of keys
+        // in map to mimic fall back.
+        if (fastMapCounter <= 1) {
+          0
+        } else {
+          (math.log10(fastMapCounter) / math.log10(2)).floor.toInt
+        }
+      case _ => sqlContext.conf.fastHashAggregateRowMaxCapacityBit
+    }
 
     val thisPlan = ctx.addReferenceObj("plan", this)
 
@@ -719,11 +728,28 @@ case class HashAggregateExec(
           "org.apache.spark.unsafe.KVIterator<UnsafeRow, UnsafeRow>",
           "fastHashMapIter", forceInline = true)
         val create = s"$fastHashMapTerm = new $fastHashMapClassName(" +
-          s"$thisPlan.getTaskMemoryManager(), $thisPlan.getEmptyAggregationBuffer());"
+          s"$thisPlan.getTaskContext().taskMemoryManager(), " +
+          s"$thisPlan.getEmptyAggregationBuffer());"
         (iter, create)
       }
     } else ("", "")
 
+    // Generates the code to register a cleanup task with TaskContext to ensure that memory
+    // is guaranteed to be freed at the end of the task. This is necessary to avoid memory
+    // leaks in when the downstream operator does not fully consume the aggregation map's
+    // output (e.g. aggregate followed by limit).
+    val addHookToCloseFastHashMap = if (isFastHashMapEnabled) {
+      s"""
+         |$thisPlan.getTaskContext().addTaskCompletionListener(
+         |  new org.apache.spark.util.TaskCompletionListener() {
+         |    @Override
+         |    public void onTaskCompletion(org.apache.spark.TaskContext context) {
+         |      $fastHashMapTerm.close();
+         |    }
+         |});
+       """.stripMargin
+    } else ""
+
     // Create a name for the iterator from the regular hash map.
     // Inline mutable state since not many aggregation operations in a task
     val iterTerm = ctx.addMutableState(classOf[KVIterator[UnsafeRow, UnsafeRow]].getName,
@@ -763,6 +789,8 @@ case class HashAggregateExec(
     val bufferTerm = ctx.freshName("aggBuffer")
     val outputFunc = generateResultFunction(ctx)
 
+    val limitNotReachedCondition = limitNotReachedCond
+
     def outputFromFastHashMap: String = {
       if (isFastHashMapEnabled) {
         if (isVectorizedHashMapEnabled) {
@@ -775,7 +803,7 @@ case class HashAggregateExec(
 
     def outputFromRowBasedMap: String = {
       s"""
-         |while ($iterTermForFastHashMap.next()) {
+         |while ($limitNotReachedCondition $iterTermForFastHashMap.next()) {
          |  UnsafeRow $keyTerm = (UnsafeRow) $iterTermForFastHashMap.getKey();
          |  UnsafeRow $bufferTerm = (UnsafeRow) $iterTermForFastHashMap.getValue();
          |  $outputFunc($keyTerm, $bufferTerm);
@@ -800,7 +828,7 @@ case class HashAggregateExec(
           BoundReference(groupingKeySchema.length + i, attr.dataType, attr.nullable)
         })
       s"""
-         |while ($iterTermForFastHashMap.hasNext()) {
+         |while ($limitNotReachedCondition $iterTermForFastHashMap.hasNext()) {
          |  InternalRow $row = (InternalRow) $iterTermForFastHashMap.next();
          |  ${generateKeyRow.code}
          |  ${generateBufferRow.code}
@@ -815,7 +843,7 @@ case class HashAggregateExec(
 
     def outputFromRegularHashMap: String = {
       s"""
-         |while ($limitNotReachedCond $iterTerm.next()) {
+         |while ($limitNotReachedCondition $iterTerm.next()) {
          |  UnsafeRow $keyTerm = (UnsafeRow) $iterTerm.getKey();
          |  UnsafeRow $bufferTerm = (UnsafeRow) $iterTerm.getValue();
          |  $outputFunc($keyTerm, $bufferTerm);
@@ -834,6 +862,7 @@ case class HashAggregateExec(
        |if (!$initAgg) {
        |  $initAgg = true;
        |  $createFastHashMap
+       |  $addHookToCloseFastHashMap
        |  $hashMapTerm = $thisPlan.createHashMap();
        |  long $beforeAgg = System.nanoTime();
        |  $doAggFuncName();
@@ -868,13 +897,11 @@ case class HashAggregateExec(
       }
     }
 
-    val (checkFallbackForGeneratedHashMap, checkFallbackForBytesToBytesMap, resetCounter,
-    incCounter) = if (testFallbackStartsAt.isDefined) {
-      val countTerm = ctx.addMutableState(CodeGenerator.JAVA_INT, "fallbackCounter")
-      (s"$countTerm < ${testFallbackStartsAt.get._1}",
-        s"$countTerm < ${testFallbackStartsAt.get._2}", s"$countTerm = 0;", s"$countTerm += 1;")
-    } else {
-      ("true", "true", "", "")
+    val (checkFallbackForBytesToBytesMap, resetCounter, incCounter) = testFallbackStartsAt match {
+      case Some((_, regularMapCounter)) =>
+        val countTerm = ctx.addMutableState(CodeGenerator.JAVA_INT, "fallbackCounter")
+        (s"$countTerm < $regularMapCounter", s"$countTerm = 0;", s"$countTerm += 1;")
+      case _ => ("true", "", "")
     }
 
     val oomeClassName = classOf[SparkOutOfMemoryError].getName
@@ -914,12 +941,10 @@ case class HashAggregateExec(
         // If fast hash map is on, we first generate code to probe and update the fast hash map.
         // If the probe is successful the corresponding fast row buffer will hold the mutable row.
         s"""
-           |if ($checkFallbackForGeneratedHashMap) {
-           |  ${fastRowKeys.map(_.code).mkString("\n")}
-           |  if (${fastRowKeys.map("!" + _.isNull).mkString(" && ")}) {
-           |    $fastRowBuffer = $fastHashMapTerm.findOrInsert(
-           |      ${fastRowKeys.map(_.value).mkString(", ")});
-           |  }
+           |${fastRowKeys.map(_.code).mkString("\n")}
+           |if (${fastRowKeys.map("!" + _.isNull).mkString(" && ")}) {
+           |  $fastRowBuffer = $fastHashMapTerm.findOrInsert(
+           |    ${fastRowKeys.map(_.value).mkString(", ")});
            |}
            |// Cannot find the key in fast hash map, try regular hash map.
            |if ($fastRowBuffer == null) {
@@ -931,7 +956,7 @@ case class HashAggregateExec(
       }
     }
 
-    val inputAttr = aggregateBufferAttributes ++ inputAttributes
+    val inputAttrs = aggregateBufferAttributes ++ inputAttributes
     // Here we set `currentVars(0)` to `currentVars(numBufferSlots)` to null, so that when
     // generating code for buffer columns, we use `INPUT_ROW`(will be the buffer row), while
     // generating input columns, we use `currentVars`.
@@ -953,7 +978,7 @@ case class HashAggregateExec(
     val updateRowInRegularHashMap: String = {
       ctx.INPUT_ROW = unsafeRowBuffer
       val boundUpdateExprs = updateExprs.map { updateExprsForOneFunc =>
-        bindReferences(updateExprsForOneFunc, inputAttr)
+        bindReferences(updateExprsForOneFunc, inputAttrs)
       }
       val subExprs = ctx.subexpressionEliminationForWholeStageCodegen(boundUpdateExprs.flatten)
       val effectiveCodes = subExprs.codes.mkString("\n")
@@ -984,23 +1009,13 @@ case class HashAggregateExec(
          """.stripMargin
       }
 
-      val codeToEvalAggFunc = if (conf.codegenSplitAggregateFunc &&
-          aggCodeBlocks.map(_.length).sum > conf.methodSplitThreshold) {
-        val maybeSplitCode = splitAggregateExpressions(
-          ctx, aggNames, boundUpdateExprs, aggCodeBlocks, subExprs.states)
-
-        maybeSplitCode.getOrElse {
-          aggCodeBlocks.fold(EmptyBlock)(_ + _).code
-        }
-      } else {
-        aggCodeBlocks.fold(EmptyBlock)(_ + _).code
-      }
-
+      val codeToEvalAggFuncs = generateEvalCodeForAggFuncs(
+        ctx, input, inputAttrs, boundUpdateExprs, aggNames, aggCodeBlocks, subExprs)
       s"""
          |// common sub-expressions
          |$effectiveCodes
          |// evaluate aggregate functions and update aggregation buffers
-         |$codeToEvalAggFunc
+         |$codeToEvalAggFuncs
        """.stripMargin
     }
 
@@ -1009,7 +1024,7 @@ case class HashAggregateExec(
         if (isVectorizedHashMapEnabled) {
           ctx.INPUT_ROW = fastRowBuffer
           val boundUpdateExprs = updateExprs.map { updateExprsForOneFunc =>
-            bindReferences(updateExprsForOneFunc, inputAttr)
+            bindReferences(updateExprsForOneFunc, inputAttrs)
           }
           val subExprs = ctx.subexpressionEliminationForWholeStageCodegen(boundUpdateExprs.flatten)
           val effectiveCodes = subExprs.codes.mkString("\n")
@@ -1039,18 +1054,8 @@ case class HashAggregateExec(
              """.stripMargin
           }
 
-
-          val codeToEvalAggFunc = if (conf.codegenSplitAggregateFunc &&
-              aggCodeBlocks.map(_.length).sum > conf.methodSplitThreshold) {
-            val maybeSplitCode = splitAggregateExpressions(
-              ctx, aggNames, boundUpdateExprs, aggCodeBlocks, subExprs.states)
-
-            maybeSplitCode.getOrElse {
-              aggCodeBlocks.fold(EmptyBlock)(_ + _).code
-            }
-          } else {
-            aggCodeBlocks.fold(EmptyBlock)(_ + _).code
-          }
+          val codeToEvalAggFuncs = generateEvalCodeForAggFuncs(
+            ctx, input, inputAttrs, boundUpdateExprs, aggNames, aggCodeBlocks, subExprs)
 
           // If vectorized fast hash map is on, we first generate code to update row
           // in vectorized fast hash map, if the previous loop up hit vectorized fast hash map.
@@ -1060,7 +1065,7 @@ case class HashAggregateExec(
              |  // common sub-expressions
              |  $effectiveCodes
              |  // evaluate aggregate functions and update aggregation buffers
-             |  $codeToEvalAggFunc
+             |  $codeToEvalAggFuncs
              |} else {
              |  $updateRowInRegularHashMap
              |}
@@ -1130,6 +1135,9 @@ case class HashAggregateExec(
           s"$allAggregateExpressions $resultExpressions fallbackStartsAt=$fallbackStartsAt"
     }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): HashAggregateExec =
+    copy(child = newChild)
 }
 
 object HashAggregateExec {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectAggregationIterator.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectAggregationIterator.scala
index 75651500954cf..ebb6ee3852471 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectAggregationIterator.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectAggregationIterator.scala
@@ -28,7 +28,6 @@ import org.apache.spark.sql.execution.metric.SQLMetric
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.unsafe.KVIterator
-import org.apache.spark.util.collection.unsafe.sort.UnsafeExternalSorter
 
 class ObjectAggregationIterator(
     partIndex: Int,
@@ -42,7 +41,9 @@ class ObjectAggregationIterator(
     originalInputAttributes: Seq[Attribute],
     inputRows: Iterator[InternalRow],
     fallbackCountThreshold: Int,
-    numOutputRows: SQLMetric)
+    numOutputRows: SQLMetric,
+    spillSize: SQLMetric,
+    numTasksFallBacked: SQLMetric)
   extends AggregationIterator(
     partIndex,
     groupingExpressions,
@@ -58,6 +59,10 @@ class ObjectAggregationIterator(
 
   private[this] var aggBufferIterator: Iterator[AggregationBufferEntry] = _
 
+  // Remember spill data size of this task before execute this operator so that we can
+  // figure out how many bytes we spilled for this operator.
+  private val spillSizeBefore = TaskContext.get().taskMetrics().memoryBytesSpilled
+
   // Hacking the aggregation mode to call AggregateFunction.merge to merge two aggregation buffers
   private val mergeAggregationBuffers: (InternalRow, InternalRow) => Unit = {
     val newExpressions = aggregateExpressions.map {
@@ -77,6 +82,11 @@ class ObjectAggregationIterator(
    */
   processInputs()
 
+  TaskContext.get().addTaskCompletionListener[Unit](_ => {
+    // At the end of the task, update the task's spill size.
+    spillSize.set(TaskContext.get().taskMetrics().memoryBytesSpilled - spillSizeBefore)
+  })
+
   override final def hasNext: Boolean = {
     aggBufferIterator.hasNext
   }
@@ -159,7 +169,7 @@ class ObjectAggregationIterator(
         processRow(buffer, newInput)
 
         // The hash map gets too large, makes a sorted spill and clear the map.
-        if (hashMap.size >= fallbackCountThreshold) {
+        if (hashMap.size >= fallbackCountThreshold && inputRows.hasNext) {
           logInfo(
             s"Aggregation hash map size ${hashMap.size} reaches threshold " +
               s"capacity ($fallbackCountThreshold entries), spilling and falling back to sort" +
@@ -169,7 +179,7 @@ class ObjectAggregationIterator(
 
           // Falls back to sort-based aggregation
           sortBased = true
-
+          numTasksFallBacked += 1
         }
       }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectAggregationMap.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectAggregationMap.scala
index b5372bcca89dd..9f2cf84a6d7e6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectAggregationMap.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectAggregationMap.scala
@@ -26,7 +26,6 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, UnsafeProjection, U
 import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateFunction, TypedImperativeAggregate}
 import org.apache.spark.sql.execution.UnsafeKVExternalSorter
 import org.apache.spark.sql.types.StructType
-import org.apache.spark.util.collection.unsafe.sort.UnsafeExternalSorter
 
 /**
  * An aggregation map that supports using safe `SpecificInternalRow`s aggregation buffers, so that
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectHashAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectHashAggregateExec.scala
index f1c0719ff8948..559f545dc05ad 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectHashAggregateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectHashAggregateExec.scala
@@ -21,10 +21,8 @@ import java.util.concurrent.TimeUnit._
 
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.errors._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate._
-import org.apache.spark.sql.catalyst.plans.physical._
 import org.apache.spark.sql.catalyst.util.truncatedString
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.execution.metric.SQLMetrics
@@ -67,11 +65,7 @@ case class ObjectHashAggregateExec(
     initialInputBufferOffset: Int,
     resultExpressions: Seq[NamedExpression],
     child: SparkPlan)
-  extends BaseAggregateExec with AliasAwareOutputPartitioning {
-
-  private[this] val aggregateBufferAttributes = {
-    aggregateExpressions.flatMap(_.aggregateFunction.aggBufferAttributes)
-  }
+  extends BaseAggregateExec {
 
   override lazy val allAttributes: AttributeSeq =
     child.output ++ aggregateBufferAttributes ++ aggregateAttributes ++
@@ -79,29 +73,17 @@ case class ObjectHashAggregateExec(
 
   override lazy val metrics = Map(
     "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
-    "aggTime" -> SQLMetrics.createTimingMetric(sparkContext, "time in aggregation build")
+    "aggTime" -> SQLMetrics.createTimingMetric(sparkContext, "time in aggregation build"),
+    "spillSize" -> SQLMetrics.createSizeMetric(sparkContext, "spill size"),
+    "numTasksFallBacked" -> SQLMetrics.createMetric(sparkContext,
+      "number of tasks fall-backed to sort-based aggregation")
   )
 
-  override def output: Seq[Attribute] = resultExpressions.map(_.toAttribute)
-
-  override def producedAttributes: AttributeSet =
-    AttributeSet(aggregateAttributes) ++
-    AttributeSet(resultExpressions.diff(groupingExpressions).map(_.toAttribute)) ++
-    AttributeSet(aggregateBufferAttributes)
-
-  override def requiredChildDistribution: List[Distribution] = {
-    requiredChildDistributionExpressions match {
-      case Some(exprs) if exprs.isEmpty => AllTuples :: Nil
-      case Some(exprs) if exprs.nonEmpty => ClusteredDistribution(exprs) :: Nil
-      case None => UnspecifiedDistribution :: Nil
-    }
-  }
-
-  override protected def outputExpressions: Seq[NamedExpression] = resultExpressions
-
-  protected override def doExecute(): RDD[InternalRow] = attachTree(this, "execute") {
+  protected override def doExecute(): RDD[InternalRow] = {
     val numOutputRows = longMetric("numOutputRows")
     val aggTime = longMetric("aggTime")
+    val spillSize = longMetric("spillSize")
+    val numTasksFallBacked = longMetric("numTasksFallBacked")
     val fallbackCountThreshold = sqlContext.conf.objectAggSortBasedFallbackThreshold
 
     child.execute().mapPartitionsWithIndexInternal { (partIndex, iter) =>
@@ -126,7 +108,9 @@ case class ObjectHashAggregateExec(
             inputAttributes,
             iter,
             fallbackCountThreshold,
-            numOutputRows)
+            numOutputRows,
+            spillSize,
+            numTasksFallBacked)
         if (!hasInput && groupingExpressions.isEmpty) {
           numOutputRows += 1
           Iterator.single[UnsafeRow](aggregationIterator.outputForEmptyGroupingKeyWithoutInput())
@@ -154,6 +138,9 @@ case class ObjectHashAggregateExec(
       s"ObjectHashAggregate(keys=$keyString, functions=$functionString)"
     }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): ObjectHashAggregateExec =
+    copy(child = newChild)
 }
 
 object ObjectHashAggregateExec {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala
index ba0c3517a1a14..4fb0f44db81c7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala
@@ -19,12 +19,10 @@ package org.apache.spark.sql.execution.aggregate
 
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.errors._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate._
-import org.apache.spark.sql.catalyst.plans.physical._
 import org.apache.spark.sql.catalyst.util.truncatedString
-import org.apache.spark.sql.execution.{AliasAwareOutputOrdering, AliasAwareOutputPartitioning, SparkPlan}
+import org.apache.spark.sql.execution.{AliasAwareOutputOrdering, SparkPlan}
 import org.apache.spark.sql.execution.metric.SQLMetrics
 
 /**
@@ -39,42 +37,20 @@ case class SortAggregateExec(
     resultExpressions: Seq[NamedExpression],
     child: SparkPlan)
   extends BaseAggregateExec
-    with AliasAwareOutputPartitioning
-    with AliasAwareOutputOrdering {
-
-  private[this] val aggregateBufferAttributes = {
-    aggregateExpressions.flatMap(_.aggregateFunction.aggBufferAttributes)
-  }
-
-  override def producedAttributes: AttributeSet =
-    AttributeSet(aggregateAttributes) ++
-      AttributeSet(resultExpressions.diff(groupingExpressions).map(_.toAttribute)) ++
-      AttributeSet(aggregateBufferAttributes)
+  with AliasAwareOutputOrdering {
 
   override lazy val metrics = Map(
     "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"))
 
-  override def output: Seq[Attribute] = resultExpressions.map(_.toAttribute)
-
-  override def requiredChildDistribution: List[Distribution] = {
-    requiredChildDistributionExpressions match {
-      case Some(exprs) if exprs.isEmpty => AllTuples :: Nil
-      case Some(exprs) if exprs.nonEmpty => ClusteredDistribution(exprs) :: Nil
-      case None => UnspecifiedDistribution :: Nil
-    }
-  }
-
   override def requiredChildOrdering: Seq[Seq[SortOrder]] = {
     groupingExpressions.map(SortOrder(_, Ascending)) :: Nil
   }
 
-  override protected def outputExpressions: Seq[NamedExpression] = resultExpressions
-
   override protected def orderingExpressions: Seq[SortOrder] = {
     groupingExpressions.map(SortOrder(_, Ascending))
   }
 
-  protected override def doExecute(): RDD[InternalRow] = attachTree(this, "execute") {
+  protected override def doExecute(): RDD[InternalRow] = {
     val numOutputRows = longMetric("numOutputRows")
     child.execute().mapPartitionsWithIndexInternal { (partIndex, iter) =>
       // Because the constructor of an aggregation iterator will read at least the first row,
@@ -125,4 +101,7 @@ case class SortAggregateExec(
       s"SortAggregate(key=$keyString, functions=$functionString)"
     }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): SortAggregateExec =
+    copy(child = newChild)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortBasedAggregationIterator.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortBasedAggregationIterator.scala
index 492b0f2da77cb..deb9e76c51760 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortBasedAggregationIterator.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortBasedAggregationIterator.scala
@@ -19,12 +19,13 @@ package org.apache.spark.sql.execution.aggregate
 
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, AggregateFunction}
+import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
 import org.apache.spark.sql.execution.metric.SQLMetric
 
 /**
- * An iterator used to evaluate [[AggregateFunction]]. It assumes the input rows have been
- * sorted by values of [[groupingExpressions]].
+ * An iterator used to evaluate
+ * [[org.apache.spark.sql.catalyst.expressions.aggregate.AggregateFunction]].
+ * It assumes the input rows have been sorted by values of [[groupingExpressions]].
  */
 class SortBasedAggregationIterator(
     partIndex: Int,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/TypedAggregateExpression.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/TypedAggregateExpression.scala
index ea44c6013b7d9..d958790dd09b1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/TypedAggregateExpression.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/TypedAggregateExpression.scala
@@ -203,6 +203,10 @@ case class SimpleTypedAggregateExpression(
       schema: StructType): TypedAggregateExpression = {
     copy(inputDeserializer = Some(deser), inputClass = Some(cls), inputSchema = Some(schema))
   }
+
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[Expression]): SimpleTypedAggregateExpression =
+    super.legacyWithNewChildren(newChildren).asInstanceOf[SimpleTypedAggregateExpression]
 }
 
 case class ComplexTypedAggregateExpression(
@@ -285,4 +289,8 @@ case class ComplexTypedAggregateExpression(
       schema: StructType): TypedAggregateExpression = {
     copy(inputDeserializer = Some(deser), inputClass = Some(cls), inputSchema = Some(schema))
   }
+
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[Expression]): ComplexTypedAggregateExpression =
+    super.legacyWithNewChildren(newChildren).asInstanceOf[ComplexTypedAggregateExpression]
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/udaf.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/udaf.scala
index 544b90a736071..33cff7ff2b801 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/udaf.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/udaf.scala
@@ -17,16 +17,14 @@
 
 package org.apache.spark.sql.execution.aggregate
 
-import scala.reflect.runtime.universe.TypeTag
-
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.{Column, Row}
+import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, _}
-import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Complete}
 import org.apache.spark.sql.catalyst.expressions.aggregate.{ImperativeAggregate, TypedImperativeAggregate}
-import org.apache.spark.sql.catalyst.expressions.codegen.{GenerateMutableProjection, GenerateSafeProjection}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.expressions.{Aggregator, MutableAggregationBuffer, UserDefinedAggregateFunction}
 import org.apache.spark.sql.types._
 
@@ -89,6 +87,14 @@ sealed trait BufferSetterGetterUtils {
           (row: InternalRow, ordinal: Int) =>
             if (row.isNullAt(ordinal)) null else row.getLong(ordinal)
 
+        case YearMonthIntervalType =>
+          (row: InternalRow, ordinal: Int) =>
+            if (row.isNullAt(ordinal)) null else row.getInt(ordinal)
+
+        case DayTimeIntervalType =>
+          (row: InternalRow, ordinal: Int) =>
+            if (row.isNullAt(ordinal)) null else row.getLong(ordinal)
+
         case other =>
           (row: InternalRow, ordinal: Int) =>
             if (row.isNullAt(ordinal)) null else row.get(ordinal, other)
@@ -189,6 +195,22 @@ sealed trait BufferSetterGetterUtils {
               row.setNullAt(ordinal)
             }
 
+        case YearMonthIntervalType =>
+          (row: InternalRow, ordinal: Int, value: Any) =>
+            if (value != null) {
+              row.setInt(ordinal, value.asInstanceOf[Int])
+            } else {
+              row.setNullAt(ordinal)
+            }
+
+        case DayTimeIntervalType =>
+          (row: InternalRow, ordinal: Int, value: Any) =>
+            if (value != null) {
+              row.setLong(ordinal, value.asInstanceOf[Long])
+            } else {
+              row.setNullAt(ordinal)
+            }
+
         case other =>
           (row: InternalRow, ordinal: Int, value: Any) =>
             if (value != null) {
@@ -327,7 +349,8 @@ case class ScalaUDAF(
     children: Seq[Expression],
     udaf: UserDefinedAggregateFunction,
     mutableAggBufferOffset: Int = 0,
-    inputAggBufferOffset: Int = 0)
+    inputAggBufferOffset: Int = 0,
+    udafName: Option[String] = None)
   extends ImperativeAggregate
   with NonSQLExpression
   with Logging
@@ -449,29 +472,35 @@ case class ScalaUDAF(
   }
 
   override def toString: String = {
-    s"""${udaf.getClass.getSimpleName}(${children.mkString(",")})"""
+    s"""$nodeName(${children.mkString(",")})"""
   }
 
-  override def nodeName: String = udaf.getClass.getSimpleName
+  override def nodeName: String = name
+
+  override def name: String = udafName.getOrElse(udaf.getClass.getSimpleName)
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): ScalaUDAF =
+    copy(children = newChildren)
 }
 
 case class ScalaAggregator[IN, BUF, OUT](
     children: Seq[Expression],
     agg: Aggregator[IN, BUF, OUT],
-    inputEncoderNR: ExpressionEncoder[IN],
+    inputEncoder: ExpressionEncoder[IN],
+    bufferEncoder: ExpressionEncoder[BUF],
     nullable: Boolean = true,
     isDeterministic: Boolean = true,
     mutableAggBufferOffset: Int = 0,
-    inputAggBufferOffset: Int = 0)
+    inputAggBufferOffset: Int = 0,
+    aggregatorName: Option[String] = None)
   extends TypedImperativeAggregate[BUF]
   with NonSQLExpression
   with UserDefinedExpression
   with ImplicitCastInputTypes
   with Logging {
 
-  private[this] lazy val inputDeserializer = inputEncoderNR.resolveAndBind().createDeserializer()
-  private[this] lazy val bufferEncoder =
-    agg.bufferEncoder.asInstanceOf[ExpressionEncoder[BUF]].resolveAndBind()
+  // input and buffer encoders are resolved by ResolveEncodersInScalaAgg
+  private[this] lazy val inputDeserializer = inputEncoder.createDeserializer()
   private[this] lazy val bufferSerializer = bufferEncoder.createSerializer()
   private[this] lazy val bufferDeserializer = bufferEncoder.createDeserializer()
   private[this] lazy val outputEncoder = agg.outputEncoder.asInstanceOf[ExpressionEncoder[OUT]]
@@ -479,7 +508,7 @@ case class ScalaAggregator[IN, BUF, OUT](
 
   def dataType: DataType = outputEncoder.objSerializer.dataType
 
-  def inputTypes: Seq[DataType] = inputEncoderNR.schema.map(_.dataType)
+  def inputTypes: Seq[DataType] = inputEncoder.schema.map(_.dataType)
 
   override lazy val deterministic: Boolean = isDeterministic
 
@@ -515,5 +544,26 @@ case class ScalaAggregator[IN, BUF, OUT](
 
   override def toString: String = s"""${nodeName}(${children.mkString(",")})"""
 
-  override def nodeName: String = agg.getClass.getSimpleName
+  override def nodeName: String = name
+
+  override def name: String = aggregatorName.getOrElse(agg.getClass.getSimpleName)
+
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[Expression]): ScalaAggregator[IN, BUF, OUT] =
+    copy(children = newChildren)
+}
+
+/**
+ * An extension rule to resolve encoder expressions from a [[ScalaAggregator]]
+ */
+object ResolveEncodersInScalaAgg extends Rule[LogicalPlan] {
+  override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
+    case p if !p.resolved => p
+    case p => p.transformExpressionsUp {
+      case agg: ScalaAggregator[_, _, _] =>
+        agg.copy(
+          inputEncoder = agg.inputEncoder.resolveAndBind(),
+          bufferEncoder = agg.bufferEncoder.resolveAndBind())
+    }
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/analysis/DetectAmbiguousSelfJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/analysis/DetectAmbiguousSelfJoin.scala
index 614d6c2846bfa..30ed747217b34 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/analysis/DetectAmbiguousSelfJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/analysis/DetectAmbiguousSelfJoin.scala
@@ -19,10 +19,11 @@ package org.apache.spark.sql.execution.analysis
 
 import scala.collection.mutable
 
-import org.apache.spark.sql.{AnalysisException, Column, Dataset}
-import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Cast, Equality, Expression, ExprId}
+import org.apache.spark.sql.{Column, Dataset}
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, AttributeSet, Cast, Equality, Expression, ExprId}
 import org.apache.spark.sql.catalyst.plans.logical.{Join, LogicalPlan}
 import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.internal.SQLConf
 
 /**
@@ -40,7 +41,7 @@ import org.apache.spark.sql.internal.SQLConf
  * Note that, this rule removes all the Dataset id related metadata from `AttributeReference`, so
  * that they don't exist after analyzer.
  */
-class DetectAmbiguousSelfJoin(conf: SQLConf) extends Rule[LogicalPlan] {
+object DetectAmbiguousSelfJoin extends Rule[LogicalPlan] {
 
   // Dataset column reference is an `AttributeReference` with 2 special metadata.
   private def isColumnReference(a: AttributeReference): Boolean = {
@@ -76,6 +77,8 @@ class DetectAmbiguousSelfJoin(conf: SQLConf) extends Rule[LogicalPlan] {
     // We always remove the special metadata from `AttributeReference` at the end of this rule, so
     // Dataset column reference only exists in the root node via Dataset transformations like
     // `Dataset#select`.
+    if (plan.find(_.isInstanceOf[Join]).isEmpty) return stripColumnReferenceMetadataInPlan(plan)
+
     val colRefAttrs = plan.expressions.flatMap(_.collect {
       case a: AttributeReference if isColumnReference(a) => a
     })
@@ -84,6 +87,7 @@ class DetectAmbiguousSelfJoin(conf: SQLConf) extends Rule[LogicalPlan] {
       val colRefs = colRefAttrs.map(toColumnReference).distinct
       val ambiguousColRefs = mutable.HashSet.empty[ColumnReference]
       val dsIdSet = colRefs.map(_.datasetId).toSet
+      val inputAttrs = AttributeSet(plan.children.flatMap(_.output))
 
       plan.foreach {
         case LogicalPlanWithDatasetId(p, id) if dsIdSet.contains(id) =>
@@ -99,7 +103,15 @@ class DetectAmbiguousSelfJoin(conf: SQLConf) extends Rule[LogicalPlan] {
                 // the attribute of column reference, then the column reference is ambiguous, as it
                 // refers to a column that gets regenerated by self-join.
                 val actualAttr = p.output(ref.colPos).asInstanceOf[AttributeReference]
-                if (actualAttr.exprId != ref.exprId) {
+                // We should only count ambiguous column references if the attribute is available as
+                // the input attributes of the root node. For example:
+                //   Join(b#1 = 3)
+                //     TableScan(t, [a#0, b#1])
+                //     Project(a#2)
+                //       TableScan(t, [a#2, b#3])
+                // This query is a self-join. The column 'b' in the join condition is not ambiguous,
+                // as it can't come from the right side, which only has column 'a'.
+                if (actualAttr.exprId != ref.exprId && inputAttrs.contains(actualAttr)) {
                   ambiguousColRefs += ref
                 }
               }
@@ -143,16 +155,14 @@ class DetectAmbiguousSelfJoin(conf: SQLConf) extends Rule[LogicalPlan] {
       }
 
       if (ambiguousAttrs.nonEmpty) {
-        throw new AnalysisException(s"Column ${ambiguousAttrs.mkString(", ")} are ambiguous. " +
-          "It's probably because you joined several Datasets together, and some of these " +
-          "Datasets are the same. This column points to one of the Datasets but Spark is unable " +
-          "to figure out which one. Please alias the Datasets with different names via " +
-          "`Dataset.as` before joining them, and specify the column using qualified name, e.g. " +
-          """`df.as("a").join(df.as("b"), $"a.id" > $"b.id")`. You can also set """ +
-          s"${SQLConf.FAIL_AMBIGUOUS_SELF_JOIN_ENABLED.key} to false to disable this check.")
+        throw QueryCompilationErrors.ambiguousAttributesInSelfJoinError(ambiguousAttrs)
       }
     }
 
+    stripColumnReferenceMetadataInPlan(plan)
+  }
+
+  private def stripColumnReferenceMetadataInPlan(plan: LogicalPlan): LogicalPlan = {
     plan.transformExpressions {
       case a: AttributeReference if isColumnReference(a) =>
         // Remove the special metadata from this `AttributeReference`, as the detection is done.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowWriter.scala
index 501e1c460f9c9..fcae7ac32b681 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowWriter.scala
@@ -24,6 +24,8 @@ import org.apache.arrow.vector.complex._
 
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.SpecializedGetters
+import org.apache.spark.sql.catalyst.util.DateTimeConstants.{MICROS_PER_DAY, MICROS_PER_MILLIS}
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.util.ArrowUtils
 
@@ -63,26 +65,27 @@ object ArrowWriter {
         val elementVector = createFieldWriter(vector.getDataVector())
         new ArrayWriter(vector, elementVector)
       case (MapType(_, _, _), vector: MapVector) =>
-        val entryWriter = createFieldWriter(vector.getDataVector).asInstanceOf[StructWriter]
-        val keyWriter = createFieldWriter(entryWriter.valueVector.getChild(MapVector.KEY_NAME))
-        val valueWriter = createFieldWriter(entryWriter.valueVector.getChild(MapVector.VALUE_NAME))
-        new MapWriter(vector, keyWriter, valueWriter)
+        val structVector = vector.getDataVector.asInstanceOf[StructVector]
+        val keyWriter = createFieldWriter(structVector.getChild(MapVector.KEY_NAME))
+        val valueWriter = createFieldWriter(structVector.getChild(MapVector.VALUE_NAME))
+        new MapWriter(vector, structVector, keyWriter, valueWriter)
       case (StructType(_), vector: StructVector) =>
         val children = (0 until vector.size()).map { ordinal =>
           createFieldWriter(vector.getChildByOrdinal(ordinal))
         }
         new StructWriter(vector, children.toArray)
+      case (NullType, vector: NullVector) => new NullWriter(vector)
+      case (YearMonthIntervalType, vector: IntervalYearVector) => new IntervalYearWriter(vector)
+      case (DayTimeIntervalType, vector: IntervalDayVector) => new IntervalDayWriter(vector)
       case (dt, _) =>
-        throw new UnsupportedOperationException(s"Unsupported data type: ${dt.catalogString}")
+        throw QueryExecutionErrors.unsupportedDataTypeError(dt)
     }
   }
 }
 
 class ArrowWriter(val root: VectorSchemaRoot, fields: Array[ArrowFieldWriter]) {
 
-  def schema: StructType = StructType(fields.map { f =>
-    StructField(f.name, f.dataType, f.nullable)
-  })
+  def schema: StructType = ArrowUtils.fromArrowSchema(root.getSchema())
 
   private var count: Int = 0
 
@@ -331,11 +334,11 @@ private[arrow] class StructWriter(
   override def setValue(input: SpecializedGetters, ordinal: Int): Unit = {
     val struct = input.getStruct(ordinal, children.length)
     var i = 0
+    valueVector.setIndexDefined(count)
     while (i < struct.numFields) {
       children(i).write(struct, i)
       i += 1
     }
-    valueVector.setIndexDefined(count)
   }
 
   override def finish(): Unit = {
@@ -351,6 +354,7 @@ private[arrow] class StructWriter(
 
 private[arrow] class MapWriter(
     val valueVector: MapVector,
+    val structVector: StructVector,
     val keyWriter: ArrowFieldWriter,
     val valueWriter: ArrowFieldWriter) extends ArrowFieldWriter {
 
@@ -363,6 +367,7 @@ private[arrow] class MapWriter(
     val values = map.valueArray()
     var i = 0
     while (i <  map.numElements()) {
+      structVector.setIndexDefined(keyWriter.count)
       keyWriter.write(keys, i)
       valueWriter.write(values, i)
       i += 1
@@ -383,3 +388,37 @@ private[arrow] class MapWriter(
     valueWriter.reset()
   }
 }
+
+private[arrow] class NullWriter(val valueVector: NullVector) extends ArrowFieldWriter {
+
+  override def setNull(): Unit = {
+  }
+
+  override def setValue(input: SpecializedGetters, ordinal: Int): Unit = {
+  }
+}
+
+private[arrow] class IntervalYearWriter(val valueVector: IntervalYearVector)
+  extends ArrowFieldWriter {
+  override def setNull(): Unit = {
+    valueVector.setNull(count)
+  }
+
+  override def setValue(input: SpecializedGetters, ordinal: Int): Unit = {
+    valueVector.setSafe(count, input.getInt(ordinal));
+  }
+}
+
+private[arrow] class IntervalDayWriter(val valueVector: IntervalDayVector)
+  extends ArrowFieldWriter {
+  override def setNull(): Unit = {
+    valueVector.setNull(count)
+  }
+
+  override def setValue(input: SpecializedGetters, ordinal: Int): Unit = {
+    val totalMicroseconds = input.getLong(ordinal)
+    val days = totalMicroseconds / MICROS_PER_DAY
+    val millis = (totalMicroseconds % MICROS_PER_DAY) / MICROS_PER_MILLIS
+    valueVector.set(count, days.toInt, millis.toInt)
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala
index 4b376b94566b8..b537040fe71df 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala
@@ -21,7 +21,7 @@ import java.util.concurrent.{Future => JFuture}
 import java.util.concurrent.TimeUnit._
 
 import scala.collection.mutable
-import scala.concurrent.{ExecutionContext}
+import scala.concurrent.ExecutionContext
 import scala.concurrent.duration.Duration
 
 import org.apache.spark.{InterruptibleIterator, Partition, SparkContext, TaskContext}
@@ -34,7 +34,7 @@ import org.apache.spark.sql.catalyst.plans.physical._
 import org.apache.spark.sql.execution.metric.SQLMetrics
 import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
 import org.apache.spark.sql.types.{LongType, StructType}
-import org.apache.spark.util.{ThreadUtils, Utils}
+import org.apache.spark.util.ThreadUtils
 import org.apache.spark.util.random.{BernoulliCellSampler, PoissonSampler}
 
 /** Physical plan for Project. */
@@ -66,10 +66,23 @@ case class ProjectExec(projectList: Seq[NamedExpression], child: SparkPlan)
 
   override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = {
     val exprs = bindReferences[Expression](projectList, child.output)
-    val resultVars = exprs.map(_.genCode(ctx))
+    val (subExprsCode, resultVars, localValInputs) = if (conf.subexpressionEliminationEnabled) {
+      // subexpression elimination
+      val subExprs = ctx.subexpressionEliminationForWholeStageCodegen(exprs)
+      val genVars = ctx.withSubExprEliminationExprs(subExprs.states) {
+        exprs.map(_.genCode(ctx))
+      }
+      (subExprs.codes.mkString("\n"), genVars, subExprs.exprCodesNeedEvaluate)
+    } else {
+      ("", exprs.map(_.genCode(ctx)), Seq.empty)
+    }
+
     // Evaluation of non-deterministic expressions can't be deferred.
     val nonDeterministicAttrs = projectList.filterNot(_.deterministic).map(_.toAttribute)
     s"""
+       |// common sub-expressions
+       |${evaluateVariables(localValInputs)}
+       |$subExprsCode
        |${evaluateRequiredVariables(output, resultVars, AttributeSet(nonDeterministicAttrs))}
        |${consume(ctx, resultVars)}
      """.stripMargin
@@ -94,61 +107,44 @@ case class ProjectExec(projectList: Seq[NamedExpression], child: SparkPlan)
        |${ExplainUtils.generateFieldString("Input", child.output)}
        |""".stripMargin
   }
-}
-
-/** Physical plan for Filter. */
-case class FilterExec(condition: Expression, child: SparkPlan)
-  extends UnaryExecNode with CodegenSupport with PredicateHelper {
-
-  // Split out all the IsNotNulls from condition.
-  private val (notNullPreds, otherPreds) = splitConjunctivePredicates(condition).partition {
-    case IsNotNull(a) => isNullIntolerant(a) && a.references.subsetOf(child.outputSet)
-    case _ => false
-  }
 
-  // If one expression and its children are null intolerant, it is null intolerant.
-  private def isNullIntolerant(expr: Expression): Boolean = expr match {
-    case e: NullIntolerant => e.children.forall(isNullIntolerant)
-    case _ => false
-  }
-
-  // The columns that will filtered out by `IsNotNull` could be considered as not nullable.
-  private val notNullAttributes = notNullPreds.flatMap(_.references).distinct.map(_.exprId)
-
-  // Mark this as empty. We'll evaluate the input during doConsume(). We don't want to evaluate
-  // all the variables at the beginning to take advantage of short circuiting.
-  override def usedInputs: AttributeSet = AttributeSet.empty
+  override protected def withNewChildInternal(newChild: SparkPlan): ProjectExec =
+    copy(child = newChild)
+}
 
-  override def output: Seq[Attribute] = {
-    child.output.map { a =>
-      if (a.nullable && notNullAttributes.contains(a.exprId)) {
-        a.withNullability(false)
-      } else {
-        a
-      }
+trait GeneratePredicateHelper extends PredicateHelper {
+  self: CodegenSupport =>
+
+  protected def generatePredicateCode(
+      ctx: CodegenContext,
+      condition: Expression,
+      inputAttrs: Seq[Attribute],
+      inputExprCode: Seq[ExprCode]): String = {
+    val (notNullPreds, otherPreds) = splitConjunctivePredicates(condition).partition {
+      case IsNotNull(a) => isNullIntolerant(a) && a.references.subsetOf(AttributeSet(inputAttrs))
+      case _ => false
     }
-  }
-
-  override lazy val metrics = Map(
-    "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"))
-
-  override def inputRDDs(): Seq[RDD[InternalRow]] = {
-    child.asInstanceOf[CodegenSupport].inputRDDs()
-  }
-
-  protected override def doProduce(ctx: CodegenContext): String = {
-    child.asInstanceOf[CodegenSupport].produce(ctx, this)
-  }
-
-  override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = {
-    val numOutput = metricTerm(ctx, "numOutputRows")
-
+    val nonNullAttrExprIds = notNullPreds.flatMap(_.references).distinct.map(_.exprId)
+    val outputAttrs = outputWithNullability(inputAttrs, nonNullAttrExprIds)
+    generatePredicateCode(
+      ctx, inputAttrs, inputExprCode, outputAttrs, notNullPreds, otherPreds,
+      nonNullAttrExprIds)
+  }
+
+  protected def generatePredicateCode(
+      ctx: CodegenContext,
+      inputAttrs: Seq[Attribute],
+      inputExprCode: Seq[ExprCode],
+      outputAttrs: Seq[Attribute],
+      notNullPreds: Seq[Expression],
+      otherPreds: Seq[Expression],
+      nonNullAttrExprIds: Seq[ExprId]): String = {
     /**
      * Generates code for `c`, using `in` for input attributes and `attrs` for nullability.
      */
     def genPredicate(c: Expression, in: Seq[ExprCode], attrs: Seq[Attribute]): String = {
       val bound = BindReferences.bindReference(c, attrs)
-      val evaluated = evaluateRequiredVariables(child.output, in, c.references)
+      val evaluated = evaluateRequiredVariables(inputAttrs, in, c.references)
 
       // Generate the code for the predicate.
       val ev = ExpressionCanonicalizer.execute(bound).genCode(ctx)
@@ -182,10 +178,10 @@ case class FilterExec(condition: Expression, child: SparkPlan)
         if (idx != -1 && !generatedIsNotNullChecks(idx)) {
           generatedIsNotNullChecks(idx) = true
           // Use the child's output. The nullability is what the child produced.
-          genPredicate(notNullPreds(idx), input, child.output)
-        } else if (notNullAttributes.contains(r.exprId) && !extraIsNotNullAttrs.contains(r)) {
+          genPredicate(notNullPreds(idx), inputExprCode, inputAttrs)
+        } else if (nonNullAttrExprIds.contains(r.exprId) && !extraIsNotNullAttrs.contains(r)) {
           extraIsNotNullAttrs += r
-          genPredicate(IsNotNull(r), input, child.output)
+          genPredicate(IsNotNull(r), inputExprCode, inputAttrs)
         } else {
           ""
         }
@@ -195,18 +191,61 @@ case class FilterExec(condition: Expression, child: SparkPlan)
       // enforced them with the IsNotNull checks above.
       s"""
          |$nullChecks
-         |${genPredicate(c, input, output)}
+         |${genPredicate(c, inputExprCode, outputAttrs)}
        """.stripMargin.trim
     }.mkString("\n")
 
     val nullChecks = notNullPreds.zipWithIndex.map { case (c, idx) =>
       if (!generatedIsNotNullChecks(idx)) {
-        genPredicate(c, input, child.output)
+        genPredicate(c, inputExprCode, inputAttrs)
       } else {
         ""
       }
     }.mkString("\n")
 
+    s"""
+       |$generated
+       |$nullChecks
+     """.stripMargin
+  }
+}
+
+/** Physical plan for Filter. */
+case class FilterExec(condition: Expression, child: SparkPlan)
+  extends UnaryExecNode with CodegenSupport with GeneratePredicateHelper {
+
+  // Split out all the IsNotNulls from condition.
+  private val (notNullPreds, otherPreds) = splitConjunctivePredicates(condition).partition {
+    case IsNotNull(a) => isNullIntolerant(a) && a.references.subsetOf(child.outputSet)
+    case _ => false
+  }
+
+  // The columns that will filtered out by `IsNotNull` could be considered as not nullable.
+  private val notNullAttributes = notNullPreds.flatMap(_.references).distinct.map(_.exprId)
+
+  // Mark this as empty. We'll evaluate the input during doConsume(). We don't want to evaluate
+  // all the variables at the beginning to take advantage of short circuiting.
+  override def usedInputs: AttributeSet = AttributeSet.empty
+
+  override def output: Seq[Attribute] = outputWithNullability(child.output, notNullAttributes)
+
+  override lazy val metrics = Map(
+    "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"))
+
+  override def inputRDDs(): Seq[RDD[InternalRow]] = {
+    child.asInstanceOf[CodegenSupport].inputRDDs()
+  }
+
+  protected override def doProduce(ctx: CodegenContext): String = {
+    child.asInstanceOf[CodegenSupport].produce(ctx, this)
+  }
+
+  override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = {
+    val numOutput = metricTerm(ctx, "numOutputRows")
+
+    val predicateCode = generatePredicateCode(
+      ctx, child.output, input, output, notNullPreds, otherPreds, notNullAttributes)
+
     // Reset the isNull to false for the not-null columns, then the followed operators could
     // generate better code (remove dead branches).
     val resultVars = input.zipWithIndex.map { case (ev, i) =>
@@ -219,8 +258,7 @@ case class FilterExec(condition: Expression, child: SparkPlan)
     // Note: wrap in "do { } while(false);", so the generated checks can jump out with "continue;"
     s"""
        |do {
-       |  $generated
-       |  $nullChecks
+       |  $predicateCode
        |  $numOutput.add(1);
        |  ${consume(ctx, resultVars)}
        |} while(false);
@@ -251,6 +289,9 @@ case class FilterExec(condition: Expression, child: SparkPlan)
        |Condition : ${condition}
        |""".stripMargin
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): FilterExec =
+    copy(child = newChild)
 }
 
 /**
@@ -357,6 +398,9 @@ case class SampleExec(
        """.stripMargin.trim
     }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): SampleExec =
+    copy(child = newChild)
 }
 
 
@@ -369,8 +413,9 @@ case class RangeExec(range: org.apache.spark.sql.catalyst.plans.logical.Range)
   val start: Long = range.start
   val end: Long = range.end
   val step: Long = range.step
-  val numSlices: Int = range.numSlices.getOrElse(sparkContext.defaultParallelism)
+  val numSlices: Int = range.numSlices.getOrElse(sqlContext.sparkSession.leafNodeDefaultParallelism)
   val numElements: BigInt = range.numElements
+  val isEmptyRange: Boolean = start == end || (start < end ^ 0 < step)
 
   override val output: Seq[Attribute] = range.output
 
@@ -396,7 +441,7 @@ case class RangeExec(range: org.apache.spark.sql.catalyst.plans.logical.Range)
   }
 
   override def inputRDDs(): Seq[RDD[InternalRow]] = {
-    val rdd = if (start == end || (start < end ^ 0 < step)) {
+    val rdd = if (isEmptyRange) {
       new EmptyRDD[InternalRow](sqlContext.sparkContext)
     } else {
       sqlContext.sparkContext.parallelize(0 until numSlices, numSlices).map(i => InternalRow(i))
@@ -562,58 +607,64 @@ case class RangeExec(range: org.apache.spark.sql.catalyst.plans.logical.Range)
 
   protected override def doExecute(): RDD[InternalRow] = {
     val numOutputRows = longMetric("numOutputRows")
-    sqlContext
-      .sparkContext
-      .parallelize(0 until numSlices, numSlices)
-      .mapPartitionsWithIndex { (i, _) =>
-        val partitionStart = (i * numElements) / numSlices * step + start
-        val partitionEnd = (((i + 1) * numElements) / numSlices) * step + start
-        def getSafeMargin(bi: BigInt): Long =
-          if (bi.isValidLong) {
-            bi.toLong
-          } else if (bi > 0) {
-            Long.MaxValue
-          } else {
-            Long.MinValue
-          }
-        val safePartitionStart = getSafeMargin(partitionStart)
-        val safePartitionEnd = getSafeMargin(partitionEnd)
-        val rowSize = UnsafeRow.calculateBitSetWidthInBytes(1) + LongType.defaultSize
-        val unsafeRow = UnsafeRow.createFromByteArray(rowSize, 1)
-        val taskContext = TaskContext.get()
-
-        val iter = new Iterator[InternalRow] {
-          private[this] var number: Long = safePartitionStart
-          private[this] var overflow: Boolean = false
-          private[this] val inputMetrics = taskContext.taskMetrics().inputMetrics
-
-          override def hasNext =
-            if (!overflow) {
-              if (step > 0) {
-                number < safePartitionEnd
-              } else {
-                number > safePartitionEnd
-              }
-            } else false
-
-          override def next() = {
-            val ret = number
-            number += step
-            if (number < ret ^ step < 0) {
-              // we have Long.MaxValue + Long.MaxValue < Long.MaxValue
-              // and Long.MinValue + Long.MinValue > Long.MinValue, so iff the step causes a step
-              // back, we are pretty sure that we have an overflow.
-              overflow = true
+    if (isEmptyRange) {
+      new EmptyRDD[InternalRow](sqlContext.sparkContext)
+    } else {
+      sqlContext
+        .sparkContext
+        .parallelize(0 until numSlices, numSlices)
+        .mapPartitionsWithIndex { (i, _) =>
+          val partitionStart = (i * numElements) / numSlices * step + start
+          val partitionEnd = (((i + 1) * numElements) / numSlices) * step + start
+
+          def getSafeMargin(bi: BigInt): Long =
+            if (bi.isValidLong) {
+              bi.toLong
+            } else if (bi > 0) {
+              Long.MaxValue
+            } else {
+              Long.MinValue
             }
 
-            numOutputRows += 1
-            inputMetrics.incRecordsRead(1)
-            unsafeRow.setLong(0, ret)
-            unsafeRow
+          val safePartitionStart = getSafeMargin(partitionStart)
+          val safePartitionEnd = getSafeMargin(partitionEnd)
+          val rowSize = UnsafeRow.calculateBitSetWidthInBytes(1) + LongType.defaultSize
+          val unsafeRow = UnsafeRow.createFromByteArray(rowSize, 1)
+          val taskContext = TaskContext.get()
+
+          val iter = new Iterator[InternalRow] {
+            private[this] var number: Long = safePartitionStart
+            private[this] var overflow: Boolean = false
+            private[this] val inputMetrics = taskContext.taskMetrics().inputMetrics
+
+            override def hasNext =
+              if (!overflow) {
+                if (step > 0) {
+                  number < safePartitionEnd
+                } else {
+                  number > safePartitionEnd
+                }
+              } else false
+
+            override def next() = {
+              val ret = number
+              number += step
+              if (number < ret ^ step < 0) {
+                // we have Long.MaxValue + Long.MaxValue < Long.MaxValue
+                // and Long.MinValue + Long.MinValue > Long.MinValue, so iff the step causes a step
+                // back, we are pretty sure that we have an overflow.
+                overflow = true
+              }
+
+              numOutputRows += 1
+              inputMetrics.incRecordsRead(1)
+              unsafeRow.setLong(0, ret)
+              unsafeRow
+            }
           }
+          new InterruptibleIterator(taskContext, iter)
         }
-        new InterruptibleIterator(taskContext, iter)
-      }
+    }
   }
 
   override def simpleString(maxFields: Int): String = {
@@ -645,6 +696,9 @@ case class UnionExec(children: Seq[SparkPlan]) extends SparkPlan {
 
   protected override def doExecute(): RDD[InternalRow] =
     sparkContext.union(children.map(_.execute()))
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[SparkPlan]): UnionExec =
+    copy(children = newChildren)
 }
 
 /**
@@ -678,6 +732,9 @@ case class CoalesceExec(numPartitions: Int, child: SparkPlan) extends UnaryExecN
       child.execute().coalesce(numPartitions, shuffle = false)
     }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): CoalesceExec =
+    copy(child = newChild)
 }
 
 object CoalesceExec {
@@ -718,7 +775,8 @@ abstract class BaseSubqueryExec extends SparkPlan {
       prefix: String = "",
       addSuffix: Boolean = false,
       maxFields: Int,
-      printNodeId: Boolean): Unit = {
+      printNodeId: Boolean,
+      indent: Int = 0): Unit = {
     /**
      * In the new explain mode `EXPLAIN FORMATTED`, the subqueries are not shown in the
      * main plan and are printed separately along with correlation information with
@@ -734,7 +792,8 @@ abstract class BaseSubqueryExec extends SparkPlan {
         "",
         false,
         maxFields,
-        printNodeId)
+        printNodeId,
+        indent)
     }
   }
 }
@@ -742,7 +801,7 @@ abstract class BaseSubqueryExec extends SparkPlan {
 /**
  * Physical plan for a subquery.
  */
-case class SubqueryExec(name: String, child: SparkPlan)
+case class SubqueryExec(name: String, child: SparkPlan, maxNumRows: Option[Int] = None)
   extends BaseSubqueryExec with UnaryExecNode {
 
   override lazy val metrics = Map(
@@ -761,7 +820,11 @@ case class SubqueryExec(name: String, child: SparkPlan)
       SQLExecution.withExecutionId(sqlContext.sparkSession, executionId) {
         val beforeCollect = System.nanoTime()
         // Note that we use .executeCollect() because we don't want to convert data to Scala types
-        val rows: Array[InternalRow] = child.executeCollect()
+        val rows: Array[InternalRow] = if (maxNumRows.isDefined) {
+          child.executeTake(maxNumRows.get)
+        } else {
+          child.executeCollect()
+        }
         val beforeBuild = System.nanoTime()
         longMetric("collectTime") += NANOSECONDS.toMillis(beforeBuild - beforeCollect)
         val dataSize = rows.map(_.asInstanceOf[UnsafeRow].getSizeInBytes.toLong).sum
@@ -774,28 +837,48 @@ case class SubqueryExec(name: String, child: SparkPlan)
   }
 
   protected override def doCanonicalize(): SparkPlan = {
-    SubqueryExec("Subquery", child.canonicalized)
+    SubqueryExec("Subquery", child.canonicalized, maxNumRows)
   }
 
   protected override def doPrepare(): Unit = {
     relationFuture
   }
 
+  // `SubqueryExec` should only be used by calling `executeCollect`. It launches a new thread to
+  // collect the result of `child`. We should not trigger codegen of `child` again in other threads,
+  // as generating code is not thread-safe.
+  override def executeCollect(): Array[InternalRow] = {
+    ThreadUtils.awaitResult(relationFuture, Duration.Inf)
+  }
+
   protected override def doExecute(): RDD[InternalRow] = {
-    child.execute()
+    throw new IllegalStateException("SubqueryExec.doExecute should never be called")
   }
 
-  override def executeCollect(): Array[InternalRow] = {
-    ThreadUtils.awaitResult(relationFuture, Duration.Inf)
+  override def executeTake(n: Int): Array[InternalRow] = {
+    throw new IllegalStateException("SubqueryExec.executeTake should never be called")
+  }
+
+  override def executeTail(n: Int): Array[InternalRow] = {
+    throw new IllegalStateException("SubqueryExec.executeTail should never be called")
   }
 
-  override def stringArgs: Iterator[Any] = super.stringArgs ++ Iterator(s"[id=#$id]")
+  override def stringArgs: Iterator[Any] = Iterator(name, child) ++ Iterator(s"[id=#$id]")
+
+  override protected def withNewChildInternal(newChild: SparkPlan): SubqueryExec =
+    copy(child = newChild)
 }
 
 object SubqueryExec {
   private[execution] val executionContext = ExecutionContext.fromExecutorService(
     ThreadUtils.newDaemonCachedThreadPool("subquery",
       SQLConf.get.getConf(StaticSQLConf.SUBQUERY_MAX_THREAD_THRESHOLD)))
+
+  def createForScalarSubquery(name: String, child: SparkPlan): SubqueryExec = {
+    // Scalar subquery needs only one row. We require 2 rows here to validate if the scalar query is
+    // invalid(return more than one row). We don't need all the rows as it may OOM.
+    SubqueryExec(name, child, maxNumRows = Some(2))
+  }
 }
 
 /**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/bucketing/CoalesceBucketsInJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/bucketing/CoalesceBucketsInJoin.scala
new file mode 100644
index 0000000000000..7a31e3437661d
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/bucketing/CoalesceBucketsInJoin.scala
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.bucketing
+
+import scala.annotation.tailrec
+
+import org.apache.spark.sql.catalyst.catalog.BucketSpec
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight}
+import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partitioning}
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.execution.{FileSourceScanExec, FilterExec, ProjectExec, SparkPlan}
+import org.apache.spark.sql.execution.joins.{ShuffledHashJoinExec, ShuffledJoin, SortMergeJoinExec}
+
+/**
+ * This rule coalesces one side of the `SortMergeJoin` and `ShuffledHashJoin`
+ * if the following conditions are met:
+ *   - Two bucketed tables are joined.
+ *   - Join keys match with output partition expressions on their respective sides.
+ *   - The larger bucket number is divisible by the smaller bucket number.
+ *   - COALESCE_BUCKETS_IN_JOIN_ENABLED is set to true.
+ *   - The ratio of the number of buckets is less than the value set in
+ *     COALESCE_BUCKETS_IN_JOIN_MAX_BUCKET_RATIO.
+ */
+object CoalesceBucketsInJoin extends Rule[SparkPlan] {
+  private def updateNumCoalescedBucketsInScan(
+      plan: SparkPlan,
+      numCoalescedBuckets: Int): SparkPlan = {
+    plan transformUp {
+      case f: FileSourceScanExec =>
+        f.copy(optionalNumCoalescedBuckets = Some(numCoalescedBuckets))
+    }
+  }
+
+  private def updateNumCoalescedBuckets(
+      join: ShuffledJoin,
+      numLeftBuckets: Int,
+      numCoalescedBuckets: Int): ShuffledJoin = {
+    if (numCoalescedBuckets != numLeftBuckets) {
+      val leftCoalescedChild =
+        updateNumCoalescedBucketsInScan(join.left, numCoalescedBuckets)
+      join match {
+        case j: SortMergeJoinExec => j.copy(left = leftCoalescedChild)
+        case j: ShuffledHashJoinExec => j.copy(left = leftCoalescedChild)
+      }
+    } else {
+      val rightCoalescedChild =
+        updateNumCoalescedBucketsInScan(join.right, numCoalescedBuckets)
+      join match {
+        case j: SortMergeJoinExec => j.copy(right = rightCoalescedChild)
+        case j: ShuffledHashJoinExec => j.copy(right = rightCoalescedChild)
+      }
+    }
+  }
+
+  private def isCoalesceSHJStreamSide(
+      join: ShuffledHashJoinExec,
+      numLeftBuckets: Int,
+      numCoalescedBuckets: Int): Boolean = {
+    if (numCoalescedBuckets == numLeftBuckets) {
+      join.buildSide != BuildRight
+    } else {
+      join.buildSide != BuildLeft
+    }
+  }
+
+  def apply(plan: SparkPlan): SparkPlan = {
+    if (!conf.coalesceBucketsInJoinEnabled) {
+      return plan
+    }
+
+    plan transform {
+      case ExtractJoinWithBuckets(join, numLeftBuckets, numRightBuckets)
+        if math.max(numLeftBuckets, numRightBuckets) / math.min(numLeftBuckets, numRightBuckets) <=
+          conf.coalesceBucketsInJoinMaxBucketRatio =>
+        val numCoalescedBuckets = math.min(numLeftBuckets, numRightBuckets)
+        join match {
+          case j: SortMergeJoinExec =>
+            updateNumCoalescedBuckets(j, numLeftBuckets, numCoalescedBuckets)
+          case j: ShuffledHashJoinExec
+            // Only coalesce the buckets for shuffled hash join stream side,
+            // to avoid OOM for build side.
+            if isCoalesceSHJStreamSide(j, numLeftBuckets, numCoalescedBuckets) =>
+            updateNumCoalescedBuckets(j, numLeftBuckets, numCoalescedBuckets)
+          case other => other
+        }
+      case other => other
+    }
+  }
+}
+
+/**
+ * An extractor that extracts `SortMergeJoinExec` and `ShuffledHashJoin`,
+ * where both sides of the join have the bucketed tables,
+ * are consisted of only the scan operation,
+ * and numbers of buckets are not equal but divisible.
+ */
+object ExtractJoinWithBuckets {
+  @tailrec
+  private def hasScanOperation(plan: SparkPlan): Boolean = plan match {
+    case f: FilterExec => hasScanOperation(f.child)
+    case p: ProjectExec => hasScanOperation(p.child)
+    case _: FileSourceScanExec => true
+    case _ => false
+  }
+
+  private def getBucketSpec(plan: SparkPlan): Option[BucketSpec] = {
+    plan.collectFirst {
+      case f: FileSourceScanExec if f.relation.bucketSpec.nonEmpty &&
+          f.optionalNumCoalescedBuckets.isEmpty =>
+        f.relation.bucketSpec.get
+    }
+  }
+
+  /**
+   * The join keys should match with expressions for output partitioning. Note that
+   * the ordering does not matter because it will be handled in `EnsureRequirements`.
+   */
+  private def satisfiesOutputPartitioning(
+      keys: Seq[Expression],
+      partitioning: Partitioning): Boolean = {
+    partitioning match {
+      case HashPartitioning(exprs, _) if exprs.length == keys.length =>
+        exprs.forall(e => keys.exists(_.semanticEquals(e)))
+      case _ => false
+    }
+  }
+
+  private def isApplicable(j: ShuffledJoin): Boolean = {
+    (j.isInstanceOf[SortMergeJoinExec] ||
+      j.isInstanceOf[ShuffledHashJoinExec]) &&
+      hasScanOperation(j.left) &&
+      hasScanOperation(j.right) &&
+      satisfiesOutputPartitioning(j.leftKeys, j.left.outputPartitioning) &&
+      satisfiesOutputPartitioning(j.rightKeys, j.right.outputPartitioning)
+  }
+
+  private def isDivisible(numBuckets1: Int, numBuckets2: Int): Boolean = {
+    val (small, large) = (math.min(numBuckets1, numBuckets2), math.max(numBuckets1, numBuckets2))
+    // A bucket can be coalesced only if the bigger number of buckets is divisible by the smaller
+    // number of buckets because bucket id is calculated by modding the total number of buckets.
+    numBuckets1 != numBuckets2 && large % small == 0
+  }
+
+  def unapply(plan: SparkPlan): Option[(ShuffledJoin, Int, Int)] = {
+    plan match {
+      case j: ShuffledJoin if isApplicable(j) =>
+        val leftBucket = getBucketSpec(j.left)
+        val rightBucket = getBucketSpec(j.right)
+        if (leftBucket.isDefined && rightBucket.isDefined &&
+            isDivisible(leftBucket.get.numBuckets, rightBucket.get.numBuckets)) {
+          Some(j, leftBucket.get.numBuckets, rightBucket.get.numBuckets)
+        } else {
+          None
+        }
+      case _ => None
+    }
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/bucketing/DisableUnnecessaryBucketedScan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/bucketing/DisableUnnecessaryBucketedScan.scala
new file mode 100644
index 0000000000000..98bcab2a839af
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/bucketing/DisableUnnecessaryBucketedScan.scala
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.bucketing
+
+import org.apache.spark.sql.catalyst.plans.physical.{ClusteredDistribution, HashClusteredDistribution}
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.execution.{FileSourceScanExec, FilterExec, ProjectExec, SortExec, SparkPlan}
+import org.apache.spark.sql.execution.aggregate.BaseAggregateExec
+import org.apache.spark.sql.execution.exchange.Exchange
+
+/**
+ * Disable unnecessary bucketed table scan based on actual physical query plan.
+ * NOTE: this rule is designed to be applied right after [[EnsureRequirements]],
+ * where all [[ShuffleExchangeExec]] and [[SortExec]] have been added to plan properly.
+ *
+ * When BUCKETING_ENABLED and AUTO_BUCKETED_SCAN_ENABLED are set to true, go through
+ * query plan to check where bucketed table scan is unnecessary, and disable bucketed table
+ * scan if:
+ *
+ * 1. The sub-plan from root to bucketed table scan, does not contain
+ *    [[hasInterestingPartition]] operator.
+ *
+ * 2. The sub-plan from the nearest downstream [[hasInterestingPartition]] operator
+ *    to the bucketed table scan, contains only [[isAllowedUnaryExecNode]] operators
+ *    and at least one [[Exchange]].
+ *
+ * Examples:
+ * 1. no [[hasInterestingPartition]] operator:
+ *                Project
+ *                   |
+ *                 Filter
+ *                   |
+ *             Scan(t1: i, j)
+ *  (bucketed on column j, DISABLE bucketed scan)
+ *
+ * 2. join:
+ *         SortMergeJoin(t1.i = t2.j)
+ *            /            \
+ *        Sort(i)        Sort(j)
+ *          /               \
+ *      Shuffle(i)       Scan(t2: i, j)
+ *        /         (bucketed on column j, enable bucketed scan)
+ *   Scan(t1: i, j)
+ * (bucketed on column j, DISABLE bucketed scan)
+ *
+ * 3. aggregate:
+ *         HashAggregate(i, ..., Final)
+ *                      |
+ *                  Shuffle(i)
+ *                      |
+ *         HashAggregate(i, ..., Partial)
+ *                      |
+ *                    Filter
+ *                      |
+ *                  Scan(t1: i, j)
+ *  (bucketed on column j, DISABLE bucketed scan)
+ *
+ * The idea of [[hasInterestingPartition]] is inspired from "interesting order" in
+ * the paper "Access Path Selection in a Relational Database Management System"
+ * (https://dl.acm.org/doi/10.1145/582095.582099).
+ */
+object DisableUnnecessaryBucketedScan extends Rule[SparkPlan] {
+
+  /**
+   * Disable bucketed table scan with pre-order traversal of plan.
+   *
+   * @param withInterestingPartition The traversed plan has operator with interesting partition.
+   * @param withExchange The traversed plan has [[Exchange]] operator.
+   * @param withAllowedNode The traversed plan has only [[isAllowedUnaryExecNode]] operators.
+   */
+  private def disableBucketWithInterestingPartition(
+      plan: SparkPlan,
+      withInterestingPartition: Boolean,
+      withExchange: Boolean,
+      withAllowedNode: Boolean): SparkPlan = {
+    plan match {
+      case p if hasInterestingPartition(p) =>
+        // Operator with interesting partition, propagates `withInterestingPartition` as true
+        // to its children, and resets `withExchange` and `withAllowedNode`.
+        p.mapChildren(disableBucketWithInterestingPartition(_, true, false, true))
+      case exchange: Exchange =>
+        // Exchange operator propagates `withExchange` as true to its child.
+        exchange.mapChildren(disableBucketWithInterestingPartition(
+          _, withInterestingPartition, true, withAllowedNode))
+      case scan: FileSourceScanExec =>
+        if (scan.bucketedScan) {
+          if (!withInterestingPartition || (withExchange && withAllowedNode)) {
+            val nonBucketedScan = scan.copy(disableBucketedScan = true)
+            scan.logicalLink.foreach(nonBucketedScan.setLogicalLink)
+            nonBucketedScan
+          } else {
+            scan
+          }
+        } else {
+          scan
+        }
+      case o =>
+        o.mapChildren(disableBucketWithInterestingPartition(
+          _,
+          withInterestingPartition,
+          withExchange,
+          withAllowedNode && isAllowedUnaryExecNode(o)))
+    }
+  }
+
+  private def hasInterestingPartition(plan: SparkPlan): Boolean = {
+    plan.requiredChildDistribution.exists {
+      case _: ClusteredDistribution | _: HashClusteredDistribution => true
+      case _ => false
+    }
+  }
+
+  /**
+   * Check if the operator is allowed single-child operator.
+   * We may revisit this method later as we probably can
+   * remove this restriction to allow arbitrary operator between
+   * bucketed table scan and operator with interesting partition.
+   */
+  private def isAllowedUnaryExecNode(plan: SparkPlan): Boolean = {
+    plan match {
+      case _: SortExec | _: ProjectExec | _: FilterExec => true
+      case partialAgg: BaseAggregateExec =>
+        partialAgg.requiredChildDistributionExpressions.isEmpty
+      case _ => false
+    }
+  }
+
+  def apply(plan: SparkPlan): SparkPlan = {
+    lazy val hasBucketedScan = plan.find {
+      case scan: FileSourceScanExec => scan.bucketedScan
+      case _ => false
+    }.isDefined
+
+    if (!conf.bucketingEnabled || !conf.autoBucketedScanEnabled || !hasBucketedScan) {
+      plan
+    } else {
+      disableBucketWithInterestingPartition(plan, false, false, true)
+    }
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnAccessor.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnAccessor.scala
index 06f411dec158d..62835287d9a3a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnAccessor.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnAccessor.scala
@@ -23,6 +23,7 @@ import scala.annotation.tailrec
 
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{UnsafeArrayData, UnsafeMapData, UnsafeRow}
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.execution.columnar.compression.CompressibleColumnAccessor
 import org.apache.spark.sql.execution.vectorized.WritableColumnVector
 import org.apache.spark.sql.types._
@@ -138,8 +139,8 @@ private[sql] object ColumnAccessor {
       case BooleanType => new BooleanColumnAccessor(buf)
       case ByteType => new ByteColumnAccessor(buf)
       case ShortType => new ShortColumnAccessor(buf)
-      case IntegerType | DateType => new IntColumnAccessor(buf)
-      case LongType | TimestampType => new LongColumnAccessor(buf)
+      case IntegerType | DateType | YearMonthIntervalType => new IntColumnAccessor(buf)
+      case LongType | TimestampType | DayTimeIntervalType => new LongColumnAccessor(buf)
       case FloatType => new FloatColumnAccessor(buf)
       case DoubleType => new DoubleColumnAccessor(buf)
       case StringType => new StringColumnAccessor(buf)
@@ -151,8 +152,7 @@ private[sql] object ColumnAccessor {
       case array: ArrayType => new ArrayColumnAccessor(buf, array)
       case map: MapType => new MapColumnAccessor(buf, map)
       case udt: UserDefinedType[_] => ColumnAccessor(udt.sqlType, buffer)
-      case other =>
-        throw new Exception(s"not support type: $other")
+      case other => throw QueryExecutionErrors.notSupportTypeError(other)
     }
   }
 
@@ -162,7 +162,7 @@ private[sql] object ColumnAccessor {
       val nativeAccessor = columnAccessor.asInstanceOf[NativeColumnAccessor[_]]
       nativeAccessor.decompress(columnVector, numRows)
     } else {
-      throw new RuntimeException("Not support non-primitive type now")
+      throw QueryExecutionErrors.notSupportNonPrimitiveTypeError()
     }
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnBuilder.scala
index 3d94681a2fb31..261e5dd6efde1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnBuilder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnBuilder.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.execution.columnar
 import java.nio.{ByteBuffer, ByteOrder}
 
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.execution.columnar.ColumnBuilder._
 import org.apache.spark.sql.execution.columnar.compression.{AllCompressionSchemes, CompressibleColumnBuilder}
 import org.apache.spark.sql.types._
@@ -173,8 +174,8 @@ private[columnar] object ColumnBuilder {
       case BooleanType => new BooleanColumnBuilder
       case ByteType => new ByteColumnBuilder
       case ShortType => new ShortColumnBuilder
-      case IntegerType | DateType => new IntColumnBuilder
-      case LongType | TimestampType => new LongColumnBuilder
+      case IntegerType | DateType | YearMonthIntervalType => new IntColumnBuilder
+      case LongType | TimestampType | DayTimeIntervalType => new LongColumnBuilder
       case FloatType => new FloatColumnBuilder
       case DoubleType => new DoubleColumnBuilder
       case StringType => new StringColumnBuilder
@@ -189,7 +190,7 @@ private[columnar] object ColumnBuilder {
       case udt: UserDefinedType[_] =>
         return apply(udt.sqlType, initialSize, columnName, useCompression)
       case other =>
-        throw new Exception(s"not supported type: $other")
+        throw QueryExecutionErrors.notSupportTypeError(other)
     }
 
     builder.initialize(initialSize, columnName, useCompression)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnStats.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnStats.scala
index 20ecc57c49e75..d2f65b745f35a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnStats.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnStats.scala
@@ -20,9 +20,9 @@ package org.apache.spark.sql.execution.columnar
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeMap, AttributeReference}
 import org.apache.spark.sql.types._
-import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
+import org.apache.spark.unsafe.types.UTF8String
 
-private[columnar] class ColumnStatisticsSchema(a: Attribute) extends Serializable {
+class ColumnStatisticsSchema(a: Attribute) extends Serializable {
   val upperBound = AttributeReference(a.name + ".upperBound", a.dataType, nullable = true)()
   val lowerBound = AttributeReference(a.name + ".lowerBound", a.dataType, nullable = true)()
   val nullCount = AttributeReference(a.name + ".nullCount", IntegerType, nullable = false)()
@@ -32,7 +32,7 @@ private[columnar] class ColumnStatisticsSchema(a: Attribute) extends Serializabl
   val schema = Seq(lowerBound, upperBound, nullCount, count, sizeInBytes)
 }
 
-private[columnar] class PartitionStatistics(tableSchema: Seq[Attribute]) extends Serializable {
+class PartitionStatistics(tableSchema: Seq[Attribute]) extends Serializable {
   val (forAttribute: AttributeMap[ColumnStatisticsSchema], schema: Seq[AttributeReference]) = {
     val allStats = tableSchema.map(a => a -> new ColumnStatisticsSchema(a))
     (AttributeMap(allStats), allStats.flatMap(_._2.schema))
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnType.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnType.scala
index d3c8e9251cefd..2f74fab270f8f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnType.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnType.scala
@@ -25,6 +25,7 @@ import scala.reflect.runtime.universe.TypeTag
 
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.Platform
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
@@ -256,6 +257,71 @@ private[columnar] object LONG extends NativeColumnType(LongType, 8) {
   }
 }
 
+private[columnar] object YEAR_MONTH_INTERVAL extends NativeColumnType(YearMonthIntervalType, 4) {
+  override def append(v: Int, buffer: ByteBuffer): Unit = {
+    buffer.putInt(v)
+  }
+
+  override def append(row: InternalRow, ordinal: Int, buffer: ByteBuffer): Unit = {
+    buffer.putInt(row.getInt(ordinal))
+  }
+
+  override def extract(buffer: ByteBuffer): Int = {
+    ByteBufferHelper.getInt(buffer)
+  }
+
+  override def extract(buffer: ByteBuffer, row: InternalRow, ordinal: Int): Unit = {
+    row.setInt(ordinal, ByteBufferHelper.getInt(buffer))
+  }
+
+  override def setField(row: InternalRow, ordinal: Int, value: Int): Unit = {
+    row.setInt(ordinal, value)
+  }
+
+  override def getField(row: InternalRow, ordinal: Int): Int = row.getInt(ordinal)
+
+
+  override def copyField(
+      from: InternalRow,
+      fromOrdinal: Int,
+      to: InternalRow,
+      toOrdinal: Int): Unit = {
+    to.setInt(toOrdinal, from.getInt(fromOrdinal))
+  }
+}
+
+private[columnar] object DAY_TIME_INTERVAL extends NativeColumnType(DayTimeIntervalType, 8) {
+  override def append(v: Long, buffer: ByteBuffer): Unit = {
+    buffer.putLong(v)
+  }
+
+  override def append(row: InternalRow, ordinal: Int, buffer: ByteBuffer): Unit = {
+    buffer.putLong(row.getLong(ordinal))
+  }
+
+  override def extract(buffer: ByteBuffer): Long = {
+    ByteBufferHelper.getLong(buffer)
+  }
+
+  override def extract(buffer: ByteBuffer, row: InternalRow, ordinal: Int): Unit = {
+    row.setLong(ordinal, ByteBufferHelper.getLong(buffer))
+  }
+
+  override def setField(row: InternalRow, ordinal: Int, value: Long): Unit = {
+    row.setLong(ordinal, value)
+  }
+
+  override def getField(row: InternalRow, ordinal: Int): Long = row.getLong(ordinal)
+
+  override def copyField(
+      from: InternalRow,
+      fromOrdinal: Int,
+      to: InternalRow,
+      toOrdinal: Int): Unit = {
+    to.setLong(toOrdinal, from.getLong(fromOrdinal))
+  }
+}
+
 private[columnar] object FLOAT extends NativeColumnType(FloatType, 4) {
   override def append(v: Float, buffer: ByteBuffer): Unit = {
     buffer.putFloat(v)
@@ -751,8 +817,8 @@ private[columnar] object ColumnType {
       case BooleanType => BOOLEAN
       case ByteType => BYTE
       case ShortType => SHORT
-      case IntegerType | DateType => INT
-      case LongType | TimestampType => LONG
+      case IntegerType | DateType | YearMonthIntervalType => INT
+      case LongType | TimestampType | DayTimeIntervalType => LONG
       case FloatType => FLOAT
       case DoubleType => DOUBLE
       case StringType => STRING
@@ -764,8 +830,7 @@ private[columnar] object ColumnType {
       case map: MapType => MAP(map)
       case struct: StructType => STRUCT(struct)
       case udt: UserDefinedType[_] => apply(udt.sqlType)
-      case other =>
-        throw new Exception(s"Unsupported type: ${other.catalogString}")
+      case other => throw QueryExecutionErrors.unsupportedTypeError(other)
     }
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/GenerateColumnAccessor.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/GenerateColumnAccessor.scala
index bd2d06665a910..376aa53ca8588 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/GenerateColumnAccessor.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/GenerateColumnAccessor.scala
@@ -28,7 +28,7 @@ import org.apache.spark.unsafe.types.CalendarInterval
  * An Iterator to walk through the InternalRows from a CachedBatch
  */
 abstract class ColumnarIterator extends Iterator[InternalRow] {
-  def initialize(input: Iterator[CachedBatch], columnTypes: Array[DataType],
+  def initialize(input: Iterator[DefaultCachedBatch], columnTypes: Array[DataType],
     columnIndexes: Array[Int]): Unit
 }
 
@@ -80,8 +80,8 @@ object GenerateColumnAccessor extends CodeGenerator[Seq[DataType], ColumnarItera
         case BooleanType => classOf[BooleanColumnAccessor].getName
         case ByteType => classOf[ByteColumnAccessor].getName
         case ShortType => classOf[ShortColumnAccessor].getName
-        case IntegerType | DateType => classOf[IntColumnAccessor].getName
-        case LongType | TimestampType => classOf[LongColumnAccessor].getName
+        case IntegerType | DateType | YearMonthIntervalType => classOf[IntColumnAccessor].getName
+        case LongType | TimestampType | DayTimeIntervalType => classOf[LongColumnAccessor].getName
         case FloatType => classOf[FloatColumnAccessor].getName
         case DoubleType => classOf[DoubleColumnAccessor].getName
         case StringType => classOf[StringColumnAccessor].getName
@@ -203,7 +203,8 @@ object GenerateColumnAccessor extends CodeGenerator[Seq[DataType], ColumnarItera
             return false;
           }
 
-          ${classOf[CachedBatch].getName} batch = (${classOf[CachedBatch].getName}) input.next();
+          ${classOf[DefaultCachedBatch].getName} batch =
+              (${classOf[DefaultCachedBatch].getName}) input.next();
           currentRow = 0;
           numRowsInBatch = batch.numRows();
           for (int i = 0; i < columnIndexes.length; i ++) {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala
index c8fa07941af87..07411c0d3803c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala
@@ -19,33 +19,189 @@ package org.apache.spark.sql.execution.columnar
 
 import org.apache.commons.lang3.StringUtils
 
+import org.apache.spark.TaskContext
 import org.apache.spark.network.util.JavaUtils
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.MultiInstanceRelation
 import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.plans.QueryPlan
-import org.apache.spark.sql.catalyst.plans.logical
+import org.apache.spark.sql.catalyst.plans.{logical, QueryPlan}
 import org.apache.spark.sql.catalyst.plans.logical.{ColumnStat, LogicalPlan, Statistics}
 import org.apache.spark.sql.catalyst.util.truncatedString
-import org.apache.spark.sql.execution.SparkPlan
+import org.apache.spark.sql.columnar.{CachedBatch, CachedBatchSerializer, SimpleMetricsCachedBatch, SimpleMetricsCachedBatchSerializer}
+import org.apache.spark.sql.execution.{ColumnarToRowTransition, InputAdapter, QueryExecution, SparkPlan, WholeStageCodegenExec}
+import org.apache.spark.sql.execution.vectorized.{OffHeapColumnVector, OnHeapColumnVector, WritableColumnVector}
+import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
+import org.apache.spark.sql.types.{BooleanType, ByteType, DoubleType, FloatType, IntegerType, LongType, ShortType, StructType, UserDefinedType}
+import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector}
 import org.apache.spark.storage.StorageLevel
-import org.apache.spark.util.LongAccumulator
-
+import org.apache.spark.util.{LongAccumulator, Utils}
 
 /**
- * CachedBatch is a cached batch of rows.
+ * The default implementation of CachedBatch.
  *
  * @param numRows The total number of rows in this batch
  * @param buffers The buffers for serialized columns
  * @param stats The stat of columns
  */
-private[columnar]
-case class CachedBatch(numRows: Int, buffers: Array[Array[Byte]], stats: InternalRow)
+case class DefaultCachedBatch(numRows: Int, buffers: Array[Array[Byte]], stats: InternalRow)
+  extends SimpleMetricsCachedBatch
+
+/**
+ * The default implementation of CachedBatchSerializer.
+ */
+class DefaultCachedBatchSerializer extends SimpleMetricsCachedBatchSerializer {
+  override def supportsColumnarInput(schema: Seq[Attribute]): Boolean = false
+
+  override def convertColumnarBatchToCachedBatch(
+      input: RDD[ColumnarBatch],
+      schema: Seq[Attribute],
+      storageLevel: StorageLevel,
+      conf: SQLConf): RDD[CachedBatch] =
+    throw new IllegalStateException("Columnar input is not supported")
+
+  override def convertInternalRowToCachedBatch(
+      input: RDD[InternalRow],
+      schema: Seq[Attribute],
+      storageLevel: StorageLevel,
+      conf: SQLConf): RDD[CachedBatch] = {
+    val batchSize = conf.columnBatchSize
+    val useCompression = conf.useCompression
+    convertForCacheInternal(input, schema, batchSize, useCompression)
+  }
+
+  def convertForCacheInternal(
+      input: RDD[InternalRow],
+      output: Seq[Attribute],
+      batchSize: Int,
+      useCompression: Boolean): RDD[CachedBatch] = {
+    input.mapPartitionsInternal { rowIterator =>
+      new Iterator[DefaultCachedBatch] {
+        def next(): DefaultCachedBatch = {
+          val columnBuilders = output.map { attribute =>
+            ColumnBuilder(attribute.dataType, batchSize, attribute.name, useCompression)
+          }.toArray
+
+          var rowCount = 0
+          var totalSize = 0L
+          while (rowIterator.hasNext && rowCount < batchSize
+              && totalSize < ColumnBuilder.MAX_BATCH_SIZE_IN_BYTE) {
+            val row = rowIterator.next()
+
+            // Added for SPARK-6082. This assertion can be useful for scenarios when something
+            // like Hive TRANSFORM is used. The external data generation script used in TRANSFORM
+            // may result malformed rows, causing ArrayIndexOutOfBoundsException, which is somewhat
+            // hard to decipher.
+            assert(
+              row.numFields == columnBuilders.length,
+              s"Row column number mismatch, expected ${output.size} columns, " +
+                  s"but got ${row.numFields}." +
+                  s"\nRow content: $row")
+
+            var i = 0
+            totalSize = 0
+            while (i < row.numFields) {
+              columnBuilders(i).appendFrom(row, i)
+              totalSize += columnBuilders(i).columnStats.sizeInBytes
+              i += 1
+            }
+            rowCount += 1
+          }
+
+          val stats = InternalRow.fromSeq(
+            columnBuilders.flatMap(_.columnStats.collectedStatistics).toSeq)
+          DefaultCachedBatch(rowCount, columnBuilders.map { builder =>
+            JavaUtils.bufferToArray(builder.build())
+          }, stats)
+        }
+
+        def hasNext: Boolean = rowIterator.hasNext
+      }
+    }
+  }
+
+  override def supportsColumnarOutput(schema: StructType): Boolean = schema.fields.forall(f =>
+    f.dataType match {
+      // More types can be supported, but this is to match the original implementation that
+      // only supported primitive types "for ease of review"
+      case BooleanType | ByteType | ShortType | IntegerType | LongType |
+           FloatType | DoubleType => true
+      case _ => false
+    })
+
+  override def vectorTypes(attributes: Seq[Attribute], conf: SQLConf): Option[Seq[String]] =
+    Option(Seq.fill(attributes.length)(
+      if (!conf.offHeapColumnVectorEnabled) {
+        classOf[OnHeapColumnVector].getName
+      } else {
+        classOf[OffHeapColumnVector].getName
+      }
+    ))
+
+  override def convertCachedBatchToColumnarBatch(
+      input: RDD[CachedBatch],
+      cacheAttributes: Seq[Attribute],
+      selectedAttributes: Seq[Attribute],
+      conf: SQLConf): RDD[ColumnarBatch] = {
+    val offHeapColumnVectorEnabled = conf.offHeapColumnVectorEnabled
+    val outputSchema = StructType.fromAttributes(selectedAttributes)
+    val columnIndices =
+      selectedAttributes.map(a => cacheAttributes.map(o => o.exprId).indexOf(a.exprId)).toArray
+
+    def createAndDecompressColumn(cb: CachedBatch): ColumnarBatch = {
+      val cachedColumnarBatch = cb.asInstanceOf[DefaultCachedBatch]
+      val rowCount = cachedColumnarBatch.numRows
+      val taskContext = Option(TaskContext.get())
+      val columnVectors = if (!offHeapColumnVectorEnabled || taskContext.isEmpty) {
+        OnHeapColumnVector.allocateColumns(rowCount, outputSchema)
+      } else {
+        OffHeapColumnVector.allocateColumns(rowCount, outputSchema)
+      }
+      val columnarBatch = new ColumnarBatch(columnVectors.asInstanceOf[Array[ColumnVector]])
+      columnarBatch.setNumRows(rowCount)
+
+      for (i <- selectedAttributes.indices) {
+        ColumnAccessor.decompress(
+          cachedColumnarBatch.buffers(columnIndices(i)),
+          columnarBatch.column(i).asInstanceOf[WritableColumnVector],
+          outputSchema.fields(i).dataType, rowCount)
+      }
+      taskContext.foreach(_.addTaskCompletionListener[Unit](_ => columnarBatch.close()))
+      columnarBatch
+    }
+
+    input.map(createAndDecompressColumn)
+  }
+
+  override def convertCachedBatchToInternalRow(
+      input: RDD[CachedBatch],
+      cacheAttributes: Seq[Attribute],
+      selectedAttributes: Seq[Attribute],
+      conf: SQLConf): RDD[InternalRow] = {
+    // Find the ordinals and data types of the requested columns.
+    val (requestedColumnIndices, requestedColumnDataTypes) =
+      selectedAttributes.map { a =>
+        cacheAttributes.map(_.exprId).indexOf(a.exprId) -> a.dataType
+      }.unzip
+
+    val columnTypes = requestedColumnDataTypes.map {
+      case udt: UserDefinedType[_] => udt.sqlType
+      case other => other
+    }.toArray
+
+    input.mapPartitionsInternal { cachedBatchIterator =>
+      val columnarIterator = GenerateColumnAccessor.generate(columnTypes)
+      columnarIterator.initialize(cachedBatchIterator.asInstanceOf[Iterator[DefaultCachedBatch]],
+        columnTypes,
+        requestedColumnIndices.toArray)
+      columnarIterator
+    }
+  }
+}
 
+private[sql]
 case class CachedRDDBuilder(
-    useCompression: Boolean,
-    batchSize: Int,
+    serializer: CachedBatchSerializer,
     storageLevel: StorageLevel,
     @transient cachedPlan: SparkPlan,
     tableName: Option[String]) {
@@ -85,54 +241,24 @@ case class CachedRDDBuilder(
   }
 
   private def buildBuffers(): RDD[CachedBatch] = {
-    val output = cachedPlan.output
-    val cached = cachedPlan.execute().mapPartitionsInternal { rowIterator =>
-      new Iterator[CachedBatch] {
-        def next(): CachedBatch = {
-          val columnBuilders = output.map { attribute =>
-            ColumnBuilder(attribute.dataType, batchSize, attribute.name, useCompression)
-          }.toArray
-
-          var rowCount = 0
-          var totalSize = 0L
-          while (rowIterator.hasNext && rowCount < batchSize
-            && totalSize < ColumnBuilder.MAX_BATCH_SIZE_IN_BYTE) {
-            val row = rowIterator.next()
-
-            // Added for SPARK-6082. This assertion can be useful for scenarios when something
-            // like Hive TRANSFORM is used. The external data generation script used in TRANSFORM
-            // may result malformed rows, causing ArrayIndexOutOfBoundsException, which is somewhat
-            // hard to decipher.
-            assert(
-              row.numFields == columnBuilders.length,
-              s"Row column number mismatch, expected ${output.size} columns, " +
-                s"but got ${row.numFields}." +
-                s"\nRow content: $row")
-
-            var i = 0
-            totalSize = 0
-            while (i < row.numFields) {
-              columnBuilders(i).appendFrom(row, i)
-              totalSize += columnBuilders(i).columnStats.sizeInBytes
-              i += 1
-            }
-            rowCount += 1
-          }
-
-          sizeInBytesStats.add(totalSize)
-          rowCountStats.add(rowCount)
-
-          val stats = InternalRow.fromSeq(
-            columnBuilders.flatMap(_.columnStats.collectedStatistics))
-          CachedBatch(rowCount, columnBuilders.map { builder =>
-            JavaUtils.bufferToArray(builder.build())
-          }, stats)
-        }
-
-        def hasNext: Boolean = rowIterator.hasNext
-      }
+    val cb = if (cachedPlan.supportsColumnar) {
+      serializer.convertColumnarBatchToCachedBatch(
+        cachedPlan.executeColumnar(),
+        cachedPlan.output,
+        storageLevel,
+        cachedPlan.conf)
+    } else {
+      serializer.convertInternalRowToCachedBatch(
+        cachedPlan.execute(),
+        cachedPlan.output,
+        storageLevel,
+        cachedPlan.conf)
+    }
+    val cached = cb.map { batch =>
+      sizeInBytesStats.add(batch.sizeInBytes)
+      rowCountStats.add(batch.numRows)
+      batch
     }.persist(storageLevel)
-
     cached.setName(cachedName)
     cached
   }
@@ -140,22 +266,74 @@ case class CachedRDDBuilder(
 
 object InMemoryRelation {
 
+  private[this] var ser: Option[CachedBatchSerializer] = None
+  private[this] def getSerializer(sqlConf: SQLConf): CachedBatchSerializer = synchronized {
+    if (ser.isEmpty) {
+      val serName = sqlConf.getConf(StaticSQLConf.SPARK_CACHE_SERIALIZER)
+      val serClass = Utils.classForName(serName)
+      val instance = serClass.getConstructor().newInstance().asInstanceOf[CachedBatchSerializer]
+      ser = Some(instance)
+    }
+    ser.get
+  }
+
+  /* Visible for testing */
+  private[columnar] def clearSerializer(): Unit = synchronized { ser = None }
+
+  def convertToColumnarIfPossible(plan: SparkPlan): SparkPlan = plan match {
+    case gen: WholeStageCodegenExec => gen.child match {
+      case c2r: ColumnarToRowTransition => c2r.child match {
+        case ia: InputAdapter => ia.child
+        case _ => plan
+      }
+      case _ => plan
+    }
+    case c2r: ColumnarToRowTransition => // This matches when whole stage code gen is disabled.
+      c2r.child
+    case _ => plan
+  }
+
   def apply(
-      useCompression: Boolean,
-      batchSize: Int,
+      storageLevel: StorageLevel,
+      qe: QueryExecution,
+      tableName: Option[String]): InMemoryRelation = {
+    val optimizedPlan = qe.optimizedPlan
+    val serializer = getSerializer(optimizedPlan.conf)
+    val child = if (serializer.supportsColumnarInput(optimizedPlan.output)) {
+      convertToColumnarIfPossible(qe.executedPlan)
+    } else {
+      qe.executedPlan
+    }
+    val cacheBuilder = CachedRDDBuilder(serializer, storageLevel, child, tableName)
+    val relation = new InMemoryRelation(child.output, cacheBuilder, optimizedPlan.outputOrdering)
+    relation.statsOfPlanToCache = optimizedPlan.stats
+    relation
+  }
+
+  /**
+   * This API is intended only to be used for testing.
+   */
+  def apply(
+      serializer: CachedBatchSerializer,
       storageLevel: StorageLevel,
       child: SparkPlan,
       tableName: Option[String],
       optimizedPlan: LogicalPlan): InMemoryRelation = {
-    val cacheBuilder = CachedRDDBuilder(useCompression, batchSize, storageLevel, child, tableName)
+    val cacheBuilder = CachedRDDBuilder(serializer, storageLevel, child, tableName)
     val relation = new InMemoryRelation(child.output, cacheBuilder, optimizedPlan.outputOrdering)
     relation.statsOfPlanToCache = optimizedPlan.stats
     relation
   }
 
-  def apply(cacheBuilder: CachedRDDBuilder, optimizedPlan: LogicalPlan): InMemoryRelation = {
+  def apply(cacheBuilder: CachedRDDBuilder, qe: QueryExecution): InMemoryRelation = {
+    val optimizedPlan = qe.optimizedPlan
+    val newBuilder = if (cacheBuilder.serializer.supportsColumnarInput(optimizedPlan.output)) {
+      cacheBuilder.copy(cachedPlan = convertToColumnarIfPossible(qe.executedPlan))
+    } else {
+      cacheBuilder.copy(cachedPlan = qe.executedPlan)
+    }
     val relation = new InMemoryRelation(
-      cacheBuilder.cachedPlan.output, cacheBuilder, optimizedPlan.outputOrdering)
+      newBuilder.cachedPlan.output, newBuilder, optimizedPlan.outputOrdering)
     relation.statsOfPlanToCache = optimizedPlan.stats
     relation
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala
index f03c2586048bd..e4194562b7a5d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala
@@ -17,19 +17,15 @@
 
 package org.apache.spark.sql.execution.columnar
 
-import org.apache.spark.TaskContext
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.QueryPlan
 import org.apache.spark.sql.catalyst.plans.physical.Partitioning
+import org.apache.spark.sql.columnar.CachedBatch
 import org.apache.spark.sql.execution.{LeafExecNode, SparkPlan, WholeStageCodegenExec}
 import org.apache.spark.sql.execution.metric.SQLMetrics
-import org.apache.spark.sql.execution.vectorized._
-import org.apache.spark.sql.types._
-import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector}
-
+import org.apache.spark.sql.vectorized.ColumnarBatch
 
 case class InMemoryTableScanExec(
     attributes: Seq[Attribute],
@@ -57,68 +53,29 @@ case class InMemoryTableScanExec(
       relation = relation.canonicalized.asInstanceOf[InMemoryRelation])
 
   override def vectorTypes: Option[Seq[String]] =
-    Option(Seq.fill(attributes.length)(
-      if (!conf.offHeapColumnVectorEnabled) {
-        classOf[OnHeapColumnVector].getName
-      } else {
-        classOf[OffHeapColumnVector].getName
-      }
-    ))
+    relation.cacheBuilder.serializer.vectorTypes(attributes, conf)
 
   /**
    * If true, get data from ColumnVector in ColumnarBatch, which are generally faster.
    * If false, get data from UnsafeRow build from CachedBatch
    */
   override val supportsColumnar: Boolean = {
-    // In the initial implementation, for ease of review
-    // support only primitive data types and # of fields is less than wholeStageMaxNumFields
-    conf.cacheVectorizedReaderEnabled && relation.schema.fields.forall(f => f.dataType match {
-      case BooleanType | ByteType | ShortType | IntegerType | LongType |
-           FloatType | DoubleType => true
-      case _ => false
-    }) && !WholeStageCodegenExec.isTooManyFields(conf, relation.schema)
-  }
-
-  private val columnIndices =
-    attributes.map(a => relation.output.map(o => o.exprId).indexOf(a.exprId)).toArray
-
-  private val relationSchema = relation.schema.toArray
-
-  private lazy val columnarBatchSchema = new StructType(columnIndices.map(i => relationSchema(i)))
-
-  private def createAndDecompressColumn(
-      cachedColumnarBatch: CachedBatch,
-      offHeapColumnVectorEnabled: Boolean): ColumnarBatch = {
-    val rowCount = cachedColumnarBatch.numRows
-    val taskContext = Option(TaskContext.get())
-    val columnVectors = if (!offHeapColumnVectorEnabled || taskContext.isEmpty) {
-      OnHeapColumnVector.allocateColumns(rowCount, columnarBatchSchema)
-    } else {
-      OffHeapColumnVector.allocateColumns(rowCount, columnarBatchSchema)
-    }
-    val columnarBatch = new ColumnarBatch(columnVectors.asInstanceOf[Array[ColumnVector]])
-    columnarBatch.setNumRows(rowCount)
-
-    for (i <- attributes.indices) {
-      ColumnAccessor.decompress(
-        cachedColumnarBatch.buffers(columnIndices(i)),
-        columnarBatch.column(i).asInstanceOf[WritableColumnVector],
-        columnarBatchSchema.fields(i).dataType, rowCount)
-    }
-    taskContext.foreach(_.addTaskCompletionListener[Unit](_ => columnarBatch.close()))
-    columnarBatch
+    conf.cacheVectorizedReaderEnabled  &&
+        !WholeStageCodegenExec.isTooManyFields(conf, relation.schema) &&
+        relation.cacheBuilder.serializer.supportsColumnarOutput(relation.schema)
   }
 
   private lazy val columnarInputRDD: RDD[ColumnarBatch] = {
     val numOutputRows = longMetric("numOutputRows")
     val buffers = filteredCachedBatches()
-    val offHeapColumnVectorEnabled = conf.offHeapColumnVectorEnabled
-    buffers
-      .map(createAndDecompressColumn(_, offHeapColumnVectorEnabled))
-      .map { buffer =>
-        numOutputRows += buffer.numRows()
-        buffer
-      }
+    relation.cacheBuilder.serializer.convertCachedBatchToColumnarBatch(
+      buffers,
+      relation.output,
+      attributes,
+      conf).map { cb =>
+      numOutputRows += cb.numRows()
+      cb
+    }
   }
 
   private lazy val inputRDD: RDD[InternalRow] = {
@@ -130,35 +87,24 @@ case class InMemoryTableScanExec(
     val numOutputRows = longMetric("numOutputRows")
     // Using these variables here to avoid serialization of entire objects (if referenced
     // directly) within the map Partitions closure.
-    val relOutput: AttributeSeq = relation.output
-
-    filteredCachedBatches().mapPartitionsInternal { cachedBatchIterator =>
-      // Find the ordinals and data types of the requested columns.
-      val (requestedColumnIndices, requestedColumnDataTypes) =
-        attributes.map { a =>
-          relOutput.indexOf(a.exprId) -> a.dataType
-        }.unzip
-
-      // update SQL metrics
-      val withMetrics = cachedBatchIterator.map { batch =>
-        if (enableAccumulatorsForTest) {
-          readBatches.add(1)
+    val relOutput = relation.output
+    val serializer = relation.cacheBuilder.serializer
+
+    // update SQL metrics
+    val withMetrics =
+      filteredCachedBatches().mapPartitionsInternal { iter =>
+        if (enableAccumulatorsForTest && iter.hasNext) {
+          readPartitions.add(1)
+        }
+        iter.map { batch =>
+          if (enableAccumulatorsForTest) {
+            readBatches.add(1)
+          }
+          numOutputRows += batch.numRows
+          batch
         }
-        numOutputRows += batch.numRows
-        batch
-      }
-
-      val columnTypes = requestedColumnDataTypes.map {
-        case udt: UserDefinedType[_] => udt.sqlType
-        case other => other
-      }.toArray
-      val columnarIterator = GenerateColumnAccessor.generate(columnTypes)
-      columnarIterator.initialize(withMetrics, columnTypes, requestedColumnIndices.toArray)
-      if (enableAccumulatorsForTest && columnarIterator.hasNext) {
-        readPartitions.add(1)
       }
-      columnarIterator
-    }
+    serializer.convertCachedBatchToInternalRow(withMetrics, relOutput, attributes, conf)
   }
 
   override def output: Seq[Attribute] = attributes
@@ -186,114 +132,6 @@ case class InMemoryTableScanExec(
   override def outputOrdering: Seq[SortOrder] =
     relation.cachedPlan.outputOrdering.map(updateAttribute(_).asInstanceOf[SortOrder])
 
-  // Keeps relation's partition statistics because we don't serialize relation.
-  private val stats = relation.partitionStatistics
-  private def statsFor(a: Attribute) = stats.forAttribute(a)
-
-  // Currently, only use statistics from atomic types except binary type only.
-  private object ExtractableLiteral {
-    def unapply(expr: Expression): Option[Literal] = expr match {
-      case lit: Literal => lit.dataType match {
-        case BinaryType => None
-        case _: AtomicType => Some(lit)
-        case _ => None
-      }
-      case _ => None
-    }
-  }
-
-  // Returned filter predicate should return false iff it is impossible for the input expression
-  // to evaluate to `true` based on statistics collected about this partition batch.
-  @transient lazy val buildFilter: PartialFunction[Expression, Expression] = {
-    case And(lhs: Expression, rhs: Expression)
-      if buildFilter.isDefinedAt(lhs) || buildFilter.isDefinedAt(rhs) =>
-      (buildFilter.lift(lhs) ++ buildFilter.lift(rhs)).reduce(_ && _)
-
-    case Or(lhs: Expression, rhs: Expression)
-      if buildFilter.isDefinedAt(lhs) && buildFilter.isDefinedAt(rhs) =>
-      buildFilter(lhs) || buildFilter(rhs)
-
-    case EqualTo(a: AttributeReference, ExtractableLiteral(l)) =>
-      statsFor(a).lowerBound <= l && l <= statsFor(a).upperBound
-    case EqualTo(ExtractableLiteral(l), a: AttributeReference) =>
-      statsFor(a).lowerBound <= l && l <= statsFor(a).upperBound
-
-    case EqualNullSafe(a: AttributeReference, ExtractableLiteral(l)) =>
-      statsFor(a).lowerBound <= l && l <= statsFor(a).upperBound
-    case EqualNullSafe(ExtractableLiteral(l), a: AttributeReference) =>
-      statsFor(a).lowerBound <= l && l <= statsFor(a).upperBound
-
-    case LessThan(a: AttributeReference, ExtractableLiteral(l)) => statsFor(a).lowerBound < l
-    case LessThan(ExtractableLiteral(l), a: AttributeReference) => l < statsFor(a).upperBound
-
-    case LessThanOrEqual(a: AttributeReference, ExtractableLiteral(l)) =>
-      statsFor(a).lowerBound <= l
-    case LessThanOrEqual(ExtractableLiteral(l), a: AttributeReference) =>
-      l <= statsFor(a).upperBound
-
-    case GreaterThan(a: AttributeReference, ExtractableLiteral(l)) => l < statsFor(a).upperBound
-    case GreaterThan(ExtractableLiteral(l), a: AttributeReference) => statsFor(a).lowerBound < l
-
-    case GreaterThanOrEqual(a: AttributeReference, ExtractableLiteral(l)) =>
-      l <= statsFor(a).upperBound
-    case GreaterThanOrEqual(ExtractableLiteral(l), a: AttributeReference) =>
-      statsFor(a).lowerBound <= l
-
-    case IsNull(a: Attribute) => statsFor(a).nullCount > 0
-    case IsNotNull(a: Attribute) => statsFor(a).count - statsFor(a).nullCount > 0
-
-    case In(a: AttributeReference, list: Seq[Expression])
-      if list.forall(ExtractableLiteral.unapply(_).isDefined) && list.nonEmpty =>
-      list.map(l => statsFor(a).lowerBound <= l.asInstanceOf[Literal] &&
-        l.asInstanceOf[Literal] <= statsFor(a).upperBound).reduce(_ || _)
-
-    // This is an example to explain how it works, imagine that the id column stored as follows:
-    // __________________________________________
-    // | Partition ID | lowerBound | upperBound |
-    // |--------------|------------|------------|
-    // |      p1      |    '1'     |    '9'     |
-    // |      p2      |    '10'    |    '19'    |
-    // |      p3      |    '20'    |    '29'    |
-    // |      p4      |    '30'    |    '39'    |
-    // |      p5      |    '40'    |    '49'    |
-    // |______________|____________|____________|
-    //
-    // A filter: df.filter($"id".startsWith("2")).
-    // In this case it substr lowerBound and upperBound:
-    // ________________________________________________________________________________________
-    // | Partition ID | lowerBound.substr(0, Length("2")) | upperBound.substr(0, Length("2")) |
-    // |--------------|-----------------------------------|-----------------------------------|
-    // |      p1      |    '1'                            |    '9'                            |
-    // |      p2      |    '1'                            |    '1'                            |
-    // |      p3      |    '2'                            |    '2'                            |
-    // |      p4      |    '3'                            |    '3'                            |
-    // |      p5      |    '4'                            |    '4'                            |
-    // |______________|___________________________________|___________________________________|
-    //
-    // We can see that we only need to read p1 and p3.
-    case StartsWith(a: AttributeReference, ExtractableLiteral(l)) =>
-      statsFor(a).lowerBound.substr(0, Length(l)) <= l &&
-        l <= statsFor(a).upperBound.substr(0, Length(l))
-  }
-
-  lazy val partitionFilters: Seq[Expression] = {
-    predicates.flatMap { p =>
-      val filter = buildFilter.lift(p)
-      val boundFilter =
-        filter.map(
-          BindReferences.bindReference(
-            _,
-            stats.schema,
-            allowFailures = true))
-
-      boundFilter.foreach(_ =>
-        filter.foreach(f => logInfo(s"Predicate $p generates partition filter: $f")))
-
-      // If the filter can't be resolved then we are missing required statistics.
-      boundFilter.filter(_.resolved)
-    }
-  }
-
   lazy val enableAccumulatorsForTest: Boolean = sqlContext.conf.inMemoryTableScanStatisticsEnabled
 
   // Accumulators used for testing purposes
@@ -303,37 +141,13 @@ case class InMemoryTableScanExec(
   private val inMemoryPartitionPruningEnabled = sqlContext.conf.inMemoryPartitionPruning
 
   private def filteredCachedBatches(): RDD[CachedBatch] = {
-    // Using these variables here to avoid serialization of entire objects (if referenced directly)
-    // within the map Partitions closure.
-    val schema = stats.schema
-    val schemaIndex = schema.zipWithIndex
     val buffers = relation.cacheBuilder.cachedColumnBuffers
 
-    buffers.mapPartitionsWithIndexInternal { (index, cachedBatchIterator) =>
-      val partitionFilter = Predicate.create(
-        partitionFilters.reduceOption(And).getOrElse(Literal(true)),
-        schema)
-      partitionFilter.initialize(index)
-
-      // Do partition batch pruning if enabled
-      if (inMemoryPartitionPruningEnabled) {
-        cachedBatchIterator.filter { cachedBatch =>
-          if (!partitionFilter.eval(cachedBatch.stats)) {
-            logDebug {
-              val statsString = schemaIndex.map { case (a, i) =>
-                val value = cachedBatch.stats.get(i, a.dataType)
-                s"${a.name}: $value"
-              }.mkString(", ")
-              s"Skipping partition based on stats $statsString"
-            }
-            false
-          } else {
-            true
-          }
-        }
-      } else {
-        cachedBatchIterator
-      }
+    if (inMemoryPartitionPruningEnabled) {
+      val filterFunc = relation.cacheBuilder.serializer.buildFilter(predicates, relation.output)
+      buffers.mapPartitionsWithIndexInternal(filterFunc)
+    } else {
+      buffers
     }
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/compression/compressionSchemes.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/compression/compressionSchemes.scala
index 00a1d54b41709..6dbe6c191dbd9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/compression/compressionSchemes.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/compression/compressionSchemes.scala
@@ -23,7 +23,7 @@ import java.nio.ByteOrder
 import scala.collection.mutable
 
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.SpecificInternalRow
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.execution.columnar._
 import org.apache.spark.sql.execution.vectorized.WritableColumnVector
 import org.apache.spark.sql.types._
@@ -182,8 +182,7 @@ private[columnar] case object RunLengthEncoding extends CompressionScheme {
     private var _uncompressedSize = 0
     private var _compressedSize = 0
 
-    // Using `MutableRow` to store the last value to avoid boxing/unboxing cost.
-    private val lastValue = new SpecificInternalRow(Seq(columnType.dataType))
+    private var lastValue: T#InternalType = _
     private var lastRun = 0
 
     override def uncompressedSize: Int = _uncompressedSize
@@ -195,16 +194,16 @@ private[columnar] case object RunLengthEncoding extends CompressionScheme {
       val actualSize = columnType.actualSize(row, ordinal)
       _uncompressedSize += actualSize
 
-      if (lastValue.isNullAt(0)) {
-        columnType.copyField(row, ordinal, lastValue, 0)
+      if (lastValue == null) {
+        lastValue = columnType.clone(value)
         lastRun = 1
         _compressedSize += actualSize + 4
       } else {
-        if (columnType.getField(lastValue, 0) == value) {
+        if (lastValue == value) {
           lastRun += 1
         } else {
           _compressedSize += actualSize + 4
-          columnType.copyField(row, ordinal, lastValue, 0)
+          lastValue = columnType.clone(value)
           lastRun = 1
         }
       }
@@ -214,30 +213,27 @@ private[columnar] case object RunLengthEncoding extends CompressionScheme {
       to.putInt(RunLengthEncoding.typeId)
 
       if (from.hasRemaining) {
-        val currentValue = new SpecificInternalRow(Seq(columnType.dataType))
         var currentRun = 1
-        val value = new SpecificInternalRow(Seq(columnType.dataType))
-
-        columnType.extract(from, currentValue, 0)
+        var currentValue = columnType.extract(from)
 
         while (from.hasRemaining) {
-          columnType.extract(from, value, 0)
+          val value = columnType.extract(from)
 
-          if (value.get(0, columnType.dataType) == currentValue.get(0, columnType.dataType)) {
+          if (value == currentValue) {
             currentRun += 1
           } else {
             // Writes current run
-            columnType.append(currentValue, 0, to)
+            columnType.append(currentValue, to)
             to.putInt(currentRun)
 
             // Resets current run
-            columnType.copyField(value, 0, currentValue, 0)
+            currentValue = value
             currentRun = 1
           }
         }
 
         // Writes the last run
-        columnType.append(currentValue, 0, to)
+        columnType.append(currentValue, to)
         to.putInt(currentRun)
       }
 
@@ -318,7 +314,8 @@ private[columnar] case object RunLengthEncoding extends CompressionScheme {
       var valueCountLocal = 0
       var currentValueLocal: Long = 0
 
-      while (valueCountLocal < runLocal || (pos < capacity)) {
+      while (pos < capacity) {
+        assert(valueCountLocal <= runLocal)
         if (pos != nextNullIndex) {
           if (valueCountLocal == runLocal) {
             currentValueLocal = getFunction(buffer)
@@ -424,8 +421,7 @@ private[columnar] case object DictionaryEncoding extends CompressionScheme {
 
     override def compress(from: ByteBuffer, to: ByteBuffer): ByteBuffer = {
       if (overflow) {
-        throw new IllegalStateException(
-          "Dictionary encoding should not be used because of dictionary overflow.")
+        throw QueryExecutionErrors.useDictionaryEncodingWhenDictionaryOverflowError()
       }
 
       to.putInt(DictionaryEncoding.typeId)
@@ -616,7 +612,6 @@ private[columnar] case object BooleanBitSet extends CompressionScheme {
     override def hasNext: Boolean = visited < count
 
     override def decompress(columnVector: WritableColumnVector, capacity: Int): Unit = {
-      val countLocal = count
       var currentWordLocal: Long = 0
       var visitedLocal: Int = 0
       val nullsBuffer = buffer.duplicate().order(ByteOrder.nativeOrder())
@@ -626,7 +621,7 @@ private[columnar] case object BooleanBitSet extends CompressionScheme {
       var pos = 0
       var seenNulls = 0
 
-      while (visitedLocal < countLocal) {
+      while (pos < capacity) {
         if (pos != nextNullIndex) {
           val bit = visitedLocal % BITS_PER_LONG
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AnalyzeColumnCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AnalyzeColumnCommand.scala
index 5017893077922..e3c2e90a42dec 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AnalyzeColumnCommand.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AnalyzeColumnCommand.scala
@@ -34,7 +34,7 @@ import org.apache.spark.sql.types._
 case class AnalyzeColumnCommand(
     tableIdent: TableIdentifier,
     columnNames: Option[Seq[String]],
-    allColumns: Boolean) extends RunnableCommand {
+    allColumns: Boolean) extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     require(columnNames.isDefined ^ allColumns, "Parameter `columnNames` or `allColumns` are " +
@@ -61,9 +61,10 @@ case class AnalyzeColumnCommand(
 
   private def analyzeColumnInCachedData(plan: LogicalPlan, sparkSession: SparkSession): Boolean = {
     val cacheManager = sparkSession.sharedState.cacheManager
-    cacheManager.lookupCachedData(plan).map { cachedData =>
+    val planToLookup = sparkSession.sessionState.executePlan(plan).analyzed
+    cacheManager.lookupCachedData(planToLookup).map { cachedData =>
       val columnsToAnalyze = getColumnsToAnalyze(
-        tableIdent, cachedData.plan, columnNames, allColumns)
+        tableIdent, cachedData.cachedRepresentation, columnNames, allColumns)
       cacheManager.analyzeColumnCacheQuery(sparkSession, cachedData, columnsToAnalyze)
       cachedData
     }.isDefined
@@ -71,9 +72,8 @@ case class AnalyzeColumnCommand(
 
   private def analyzeColumnInTempView(plan: LogicalPlan, sparkSession: SparkSession): Unit = {
     if (!analyzeColumnInCachedData(plan, sparkSession)) {
-      val catalog = sparkSession.sessionState.catalog
-      val db = tableIdent.database.getOrElse(catalog.getCurrentDatabase)
-      throw new NoSuchTableException(db = db, table = tableIdent.identifier)
+      throw new AnalysisException(
+        s"Temporary view $tableIdent is not cached for analyzing columns.")
     }
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AnalyzePartitionCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AnalyzePartitionCommand.scala
index 33b29bde93ee5..5b3cb7476608b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AnalyzePartitionCommand.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AnalyzePartitionCommand.scala
@@ -23,7 +23,7 @@ import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionException, Unresol
 import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType, ExternalCatalogUtils}
 import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
 import org.apache.spark.sql.catalyst.expressions.{And, EqualTo, Literal}
-import org.apache.spark.sql.execution.datasources.PartitioningUtils
+import org.apache.spark.sql.util.PartitioningUtils
 
 /**
  * Analyzes a given set of partitions to generate per-partition statistics, which will be used in
@@ -43,11 +43,11 @@ import org.apache.spark.sql.execution.datasources.PartitioningUtils
 case class AnalyzePartitionCommand(
     tableIdent: TableIdentifier,
     partitionSpec: Map[String, Option[String]],
-    noscan: Boolean = true) extends RunnableCommand {
+    noscan: Boolean = true) extends LeafRunnableCommand {
 
   private def getPartitionSpec(table: CatalogTable): Option[TablePartitionSpec] = {
     val normalizedPartitionSpec =
-      PartitioningUtils.normalizePartitionSpec(partitionSpec, table.partitionColumnNames,
+      PartitioningUtils.normalizePartitionSpec(partitionSpec, table.partitionSchema,
         table.identifier.quotedString, conf.resolver)
 
     // Report an error if partition columns in partition specification do not form
@@ -69,7 +69,7 @@ case class AnalyzePartitionCommand(
     if (filteredSpec.isEmpty) {
       None
     } else {
-      Some(filteredSpec)
+      Some(filteredSpec.toMap)
     }
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AnalyzeTableCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AnalyzeTableCommand.scala
index 67cfcebec187c..157554e821811 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AnalyzeTableCommand.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AnalyzeTableCommand.scala
@@ -17,9 +17,8 @@
 
 package org.apache.spark.sql.execution.command
 
-import org.apache.spark.sql.{AnalysisException, Row, SparkSession}
+import org.apache.spark.sql.{Row, SparkSession}
 import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.catalog.CatalogTableType
 
 
 /**
@@ -27,39 +26,10 @@ import org.apache.spark.sql.catalyst.catalog.CatalogTableType
  */
 case class AnalyzeTableCommand(
     tableIdent: TableIdentifier,
-    noscan: Boolean = true) extends RunnableCommand {
+    noScan: Boolean = true) extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
-    val sessionState = sparkSession.sessionState
-    val db = tableIdent.database.getOrElse(sessionState.catalog.getCurrentDatabase)
-    val tableIdentWithDB = TableIdentifier(tableIdent.table, Some(db))
-    val tableMeta = sessionState.catalog.getTableMetadata(tableIdentWithDB)
-    if (tableMeta.tableType == CatalogTableType.VIEW) {
-      // Analyzes a catalog view if the view is cached
-      val table = sparkSession.table(tableIdent.quotedString)
-      val cacheManager = sparkSession.sharedState.cacheManager
-      if (cacheManager.lookupCachedData(table.logicalPlan).isDefined) {
-        if (!noscan) {
-          // To collect table stats, materializes an underlying columnar RDD
-          table.count()
-        }
-      } else {
-        throw new AnalysisException("ANALYZE TABLE is not supported on views.")
-      }
-    } else {
-      // Compute stats for the whole table
-      val newTotalSize = CommandUtils.calculateTotalSize(sparkSession, tableMeta)
-      val newRowCount =
-        if (noscan) None else Some(BigInt(sparkSession.table(tableIdentWithDB).count()))
-
-      // Update the metastore if the above statistics of the table are different from those
-      // recorded in the metastore.
-      val newStats = CommandUtils.compareAndGetNewStats(tableMeta.stats, newTotalSize, newRowCount)
-      if (newStats.isDefined) {
-        sessionState.catalog.alterTableStats(tableIdentWithDB, newStats)
-      }
-    }
-
+    CommandUtils.analyzeTable(sparkSession, tableIdent, noScan)
     Seq.empty[Row]
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AnalyzeTablesCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AnalyzeTablesCommand.scala
new file mode 100644
index 0000000000000..c9b22a7d1b258
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AnalyzeTablesCommand.scala
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import scala.util.control.NonFatal
+
+import org.apache.spark.sql.{Row, SparkSession}
+
+
+/**
+ * Analyzes all tables in the given database to generate statistics.
+ */
+case class AnalyzeTablesCommand(
+    databaseName: Option[String],
+    noScan: Boolean) extends LeafRunnableCommand {
+
+  override def run(sparkSession: SparkSession): Seq[Row] = {
+    val catalog = sparkSession.sessionState.catalog
+    val db = databaseName.getOrElse(catalog.getCurrentDatabase)
+    catalog.listTables(db).foreach { tbl =>
+      try {
+        CommandUtils.analyzeTable(sparkSession, tbl, noScan)
+      } catch {
+        case NonFatal(e) =>
+          logWarning(s"Failed to analyze table ${tbl.table} in the " +
+            s"database $db because of ${e.toString}", e)
+      }
+    }
+    Seq.empty[Row]
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandCheck.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandCheck.scala
index dedace4af4d14..216636c7ea14f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandCheck.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandCheck.scala
@@ -17,14 +17,14 @@
 
 package org.apache.spark.sql.execution.command
 
+import org.apache.spark.sql.catalyst.SQLConfHelper
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.util.SchemaUtils
 
 /**
  * Checks legitimization of various execution commands.
  */
-case class CommandCheck(conf: SQLConf) extends (LogicalPlan => Unit) {
+object CommandCheck extends (LogicalPlan => Unit) with SQLConfHelper {
 
   override def apply(plan: LogicalPlan): Unit = {
     plan.foreach {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandUtils.scala
index c047be774d99a..da5d00c595cb7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandUtils.scala
@@ -27,7 +27,7 @@ import org.apache.hadoop.fs.{FileSystem, Path, PathFilter}
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.{AnalysisException, SparkSession}
 import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier}
-import org.apache.spark.sql.catalyst.catalog.{CatalogColumnStat, CatalogStatistics, CatalogTable}
+import org.apache.spark.sql.catalyst.catalog.{CatalogStatistics, CatalogTable, CatalogTableType}
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate._
 import org.apache.spark.sql.catalyst.plans.logical._
@@ -57,8 +57,11 @@ object CommandUtils extends Logging {
     if (sparkSession.sessionState.conf.autoSizeUpdateEnabled) {
       val newTable = catalog.getTableMetadata(table.identifier)
       val newSize = CommandUtils.calculateTotalSize(sparkSession, newTable)
-      val newStats = CatalogStatistics(sizeInBytes = newSize)
-      catalog.alterTableStats(table.identifier, Some(newStats))
+      val isNewStats = newTable.stats.map(newSize != _.sizeInBytes).getOrElse(true)
+      if (isNewStats) {
+        val newStats = CatalogStatistics(sizeInBytes = newSize)
+        catalog.alterTableStats(table.identifier, Some(newStats))
+      }
     } else if (table.stats.nonEmpty) {
       catalog.alterTableStats(table.identifier, None)
     } else {
@@ -163,7 +166,7 @@ object CommandUtils extends Logging {
       .getConfString("hive.exec.stagingdir", ".hive-staging")
     val filter = new PathFilterIgnoreNonData(stagingDir)
     val sizes = InMemoryFileIndex.bulkListLeafFiles(paths.flatten,
-      sparkSession.sessionState.newHadoopConf(), filter, sparkSession, areRootPaths = true).map {
+      sparkSession.sessionState.newHadoopConf(), filter, sparkSession).map {
       case (_, files) => files.map(_.getLen).sum
     }
     // the size is 0 where paths(i) is not defined and sizes(i) where it is defined
@@ -196,6 +199,41 @@ object CommandUtils extends Logging {
     newStats
   }
 
+  def analyzeTable(
+      sparkSession: SparkSession,
+      tableIdent: TableIdentifier,
+      noScan: Boolean): Unit = {
+    val sessionState = sparkSession.sessionState
+    val db = tableIdent.database.getOrElse(sessionState.catalog.getCurrentDatabase)
+    val tableIdentWithDB = TableIdentifier(tableIdent.table, Some(db))
+    val tableMeta = sessionState.catalog.getTableMetadata(tableIdentWithDB)
+    if (tableMeta.tableType == CatalogTableType.VIEW) {
+      // Analyzes a catalog view if the view is cached
+      val table = sparkSession.table(tableIdent.quotedString)
+      val cacheManager = sparkSession.sharedState.cacheManager
+      if (cacheManager.lookupCachedData(table.logicalPlan).isDefined) {
+        if (!noScan) {
+          // To collect table stats, materializes an underlying columnar RDD
+          table.count()
+        }
+      } else {
+        throw new AnalysisException("ANALYZE TABLE is not supported on views.")
+      }
+    } else {
+      // Compute stats for the whole table
+      val newTotalSize = CommandUtils.calculateTotalSize(sparkSession, tableMeta)
+      val newRowCount =
+        if (noScan) None else Some(BigInt(sparkSession.table(tableIdentWithDB).count()))
+
+      // Update the metastore if the above statistics of the table are different from those
+      // recorded in the metastore.
+      val newStats = CommandUtils.compareAndGetNewStats(tableMeta.stats, newTotalSize, newRowCount)
+      if (newStats.isDefined) {
+        sessionState.catalog.alterTableStats(tableIdentWithDB, newStats)
+      }
+    }
+  }
+
   /**
    * Compute stats for the given columns.
    * @return (row count, map from column name to CatalogColumnStats)
@@ -391,7 +429,7 @@ object CommandUtils extends Logging {
     try {
       sparkSession.catalog.uncacheTable(name)
     } catch {
-      case NonFatal(e) => logWarning("Exception when attempting to uncache $name", e)
+      case NonFatal(e) => logWarning(s"Exception when attempting to uncache $name", e)
     }
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/DataWritingCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/DataWritingCommand.scala
index a1bb5af1ab723..e1f070a8b66cb 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/DataWritingCommand.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/DataWritingCommand.scala
@@ -19,27 +19,27 @@ package org.apache.spark.sql.execution.command
 
 import org.apache.hadoop.conf.Configuration
 
+import org.apache.spark.SparkContext
 import org.apache.spark.sql.{Row, SparkSession}
 import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.plans.logical.{Command, LogicalPlan}
-import org.apache.spark.sql.execution.SparkPlan
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, UnaryCommand}
+import org.apache.spark.sql.execution.{SparkPlan, SQLExecution}
 import org.apache.spark.sql.execution.datasources.BasicWriteJobStatsTracker
-import org.apache.spark.sql.execution.datasources.FileFormatWriter
-import org.apache.spark.sql.execution.metric.SQLMetric
+import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics}
 import org.apache.spark.util.SerializableConfiguration
 
 /**
  * A special `Command` which writes data out and updates metrics.
  */
-trait DataWritingCommand extends Command {
+trait DataWritingCommand extends UnaryCommand {
   /**
    * The input query plan that produces the data to be written.
    * IMPORTANT: the input query plan MUST be analyzed, so that we can carry its output columns
-   *            to [[FileFormatWriter]].
+   *            to [[org.apache.spark.sql.execution.datasources.FileFormatWriter]].
    */
   def query: LogicalPlan
 
-  override final def children: Seq[LogicalPlan] = query :: Nil
+  override final def child: LogicalPlan = query
 
   // Output column names of the analyzed input query plan.
   def outputColumnNames: Seq[String]
@@ -74,4 +74,26 @@ object DataWritingCommand {
       attr.withName(outputName)
     }
   }
+
+  /**
+   * When execute CTAS operators, Spark will use [[InsertIntoHadoopFsRelationCommand]]
+   * or [[InsertIntoHiveTable]] command to write data, they both inherit metrics from
+   * [[DataWritingCommand]], but after running [[InsertIntoHadoopFsRelationCommand]]
+   * or [[InsertIntoHiveTable]], we only update metrics in these two command through
+   * [[BasicWriteJobStatsTracker]], we also need to propogate metrics to the command
+   * that actually calls [[InsertIntoHadoopFsRelationCommand]] or [[InsertIntoHiveTable]].
+   *
+   * @param sparkContext Current SparkContext.
+   * @param command Command to execute writing data.
+   * @param metrics Metrics of real DataWritingCommand.
+   */
+  def propogateMetrics(
+      sparkContext: SparkContext,
+      command: DataWritingCommand,
+      metrics: Map[String, SQLMetric]): Unit = {
+    command.metrics.foreach { case (key, metric) => metrics(key).set(metric.value) }
+    SQLMetrics.postDriverMetricUpdates(sparkContext,
+      sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY),
+      metrics.values.toSeq)
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/InsertIntoDataSourceDirCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/InsertIntoDataSourceDirCommand.scala
index 08d31fdda2dc8..be680a733eac9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/InsertIntoDataSourceDirCommand.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/InsertIntoDataSourceDirCommand.scala
@@ -36,13 +36,13 @@ import org.apache.spark.sql.execution.datasources._
  * @param storage storage format used to describe how the query result is stored.
  * @param provider the data source type to be used
  * @param query the logical plan representing data to write to
- * @param overwrite whthere overwrites existing directory
+ * @param overwrite whether overwrites existing directory
  */
 case class InsertIntoDataSourceDirCommand(
     storage: CatalogStorageFormat,
     provider: String,
     query: LogicalPlan,
-    overwrite: Boolean) extends RunnableCommand {
+    overwrite: Boolean) extends LeafRunnableCommand {
 
   override def innerChildren: Seq[LogicalPlan] = query :: Nil
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala
index 3dc1d52697714..c32d1d74c71f1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.execution.command
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.{Row, SparkSession}
 import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.plans.logical.{IgnoreCachedData, LogicalPlan}
+import org.apache.spark.sql.catalyst.plans.logical.IgnoreCachedData
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.StaticSQLConf.CATALOG_IMPLEMENTATION
 import org.apache.spark.sql.types.{StringType, StructField, StructType}
@@ -34,7 +34,8 @@ import org.apache.spark.sql.types.{StringType, StructField, StructType}
  *   set;
  * }}}
  */
-case class SetCommand(kv: Option[(String, Option[String])]) extends RunnableCommand with Logging {
+case class SetCommand(kv: Option[(String, Option[String])])
+  extends LeafRunnableCommand with Logging {
 
   private def keyValueOutput: Seq[Attribute] = {
     val schema = StructType(
@@ -138,14 +139,29 @@ case class SetCommand(kv: Option[(String, Option[String])]) extends RunnableComm
             s"showing ${SQLConf.SHUFFLE_PARTITIONS.key} instead.")
         Seq(Row(
           SQLConf.SHUFFLE_PARTITIONS.key,
-          sparkSession.sessionState.conf.numShufflePartitions.toString))
+          sparkSession.sessionState.conf.defaultNumShufflePartitions.toString))
       }
       (keyValueOutput, runFunc)
 
     // Queries a single property.
     case Some((key, None)) =>
       val runFunc = (sparkSession: SparkSession) => {
-        val value = sparkSession.conf.getOption(key).getOrElse("<undefined>")
+        val value = sparkSession.conf.getOption(key).getOrElse {
+          // Also lookup the `sharedState.hadoopConf` to display default value for hadoop conf
+          // correctly. It completes all the session-level configs with `sparkSession.conf`
+          // together.
+          //
+          // Note that, as the write-side does not prohibit to set static hadoop/hive to SQLConf
+          // yet, users may get wrong results before reaching here,
+          // e.g. 'SET hive.metastore.uris=abc', where 'hive.metastore.uris' is static and 'abc' is
+          // of no effect, but will show 'abc' via 'SET hive.metastore.uris' wrongly.
+          //
+          // Instead of showing incorrect `<undefined>` to users, it's more reasonable to show the
+          // effective default values. For example, the hadoop output codec/compression configs
+          // take affect from table to table, file to file, so they are not static and users are
+          // very likely to change them based the default value they see.
+          sparkSession.sharedState.hadoopConf.get(key, "<undefined>")
+        }
         Seq(Row(key, value))
       }
       (keyValueOutput, runFunc)
@@ -166,15 +182,24 @@ object SetCommand {
  * via [[SetCommand]] will get reset to default value. Command that runs
  * {{{
  *   reset;
+ *   reset spark.sql.session.timeZone;
  * }}}
  */
-case object ResetCommand extends RunnableCommand with IgnoreCachedData {
+case class ResetCommand(config: Option[String]) extends LeafRunnableCommand with IgnoreCachedData {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
-    val conf = sparkSession.sessionState.conf
-    conf.clear()
-    sparkSession.sparkContext.conf.getAll.foreach { case (k, v) =>
-      conf.setConfString(k, v)
+    val globalInitialConfigs = sparkSession.sharedState.conf
+    config match {
+      case Some(key) =>
+        sparkSession.conf.unset(key)
+        sparkSession.initialSessionOptions.get(key)
+          .orElse(globalInitialConfigs.getOption(key))
+          .foreach(sparkSession.conf.set(key, _))
+      case None =>
+        sparkSession.sessionState.conf.clear()
+        SQLConf.mergeSparkConf(sparkSession.sessionState.conf, globalInitialConfigs)
+        SQLConf.mergeNonStaticSQLConfigs(sparkSession.sessionState.conf,
+          sparkSession.initialSessionOptions)
     }
     Seq.empty[Row]
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/cache.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/cache.scala
index ef6b0bba1628e..de5dbddbfa146 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/cache.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/cache.scala
@@ -17,73 +17,13 @@
 
 package org.apache.spark.sql.execution.command
 
-import java.util.Locale
-
-import org.apache.spark.sql.{Dataset, Row, SparkSession}
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
-import org.apache.spark.sql.catalyst.plans.QueryPlan
-import org.apache.spark.sql.catalyst.plans.logical.{IgnoreCachedData, LogicalPlan}
-import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
-import org.apache.spark.storage.StorageLevel
-
-case class CacheTableCommand(
-    tableIdent: TableIdentifier,
-    plan: Option[LogicalPlan],
-    isLazy: Boolean,
-    options: Map[String, String]) extends RunnableCommand {
-  require(plan.isEmpty || tableIdent.database.isEmpty,
-    "Database name is not allowed in CACHE TABLE AS SELECT")
-
-  override def innerChildren: Seq[QueryPlan[_]] = plan.toSeq
-
-  override def run(sparkSession: SparkSession): Seq[Row] = {
-    plan.foreach { logicalPlan =>
-      Dataset.ofRows(sparkSession, logicalPlan).createTempView(tableIdent.quotedString)
-    }
-
-    val storageLevelKey = "storagelevel"
-    val storageLevelValue =
-      CaseInsensitiveMap(options).get(storageLevelKey).map(_.toUpperCase(Locale.ROOT))
-    val withoutStorageLevel = options.filterKeys(_.toLowerCase(Locale.ROOT) != storageLevelKey)
-    if (withoutStorageLevel.nonEmpty) {
-      logWarning(s"Invalid options: ${withoutStorageLevel.mkString(", ")}")
-    }
-
-    if (storageLevelValue.nonEmpty) {
-      sparkSession.catalog.cacheTable(
-        tableIdent.quotedString, StorageLevel.fromString(storageLevelValue.get))
-    } else {
-      sparkSession.catalog.cacheTable(tableIdent.quotedString)
-    }
-
-    if (!isLazy) {
-      // Performs eager caching
-      sparkSession.table(tableIdent).count()
-    }
-
-    Seq.empty[Row]
-  }
-}
-
-
-case class UncacheTableCommand(
-    tableIdent: TableIdentifier,
-    ifExists: Boolean) extends RunnableCommand {
-
-  override def run(sparkSession: SparkSession): Seq[Row] = {
-    val tableId = tableIdent.quotedString
-    if (!ifExists || sparkSession.catalog.tableExists(tableId)) {
-      sparkSession.catalog.uncacheTable(tableId)
-    }
-    Seq.empty[Row]
-  }
-}
+import org.apache.spark.sql.{Row, SparkSession}
+import org.apache.spark.sql.catalyst.plans.logical.IgnoreCachedData
 
 /**
  * Clear all cached data from the in-memory cache.
  */
-case object ClearCacheCommand extends RunnableCommand with IgnoreCachedData {
+case object ClearCacheCommand extends LeafRunnableCommand with IgnoreCachedData {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     sparkSession.catalog.clearCache()
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/commands.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/commands.scala
index 18fd2a5ac2330..7f4f816d328da 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/commands.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/commands.scala
@@ -18,14 +18,15 @@
 package org.apache.spark.sql.execution.command
 
 import scala.collection.JavaConverters._
+import scala.util.control.NonFatal
 
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{Row, SparkSession}
 import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
-import org.apache.spark.sql.catalyst.errors.TreeNodeException
 import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
 import org.apache.spark.sql.catalyst.plans.QueryPlan
 import org.apache.spark.sql.catalyst.plans.logical.{Command, LogicalPlan}
+import org.apache.spark.sql.catalyst.trees.LeafLike
 import org.apache.spark.sql.connector.ExternalCommandRunner
 import org.apache.spark.sql.execution.{ExplainMode, LeafExecNode, SparkPlan, UnaryExecNode}
 import org.apache.spark.sql.execution.metric.SQLMetric
@@ -39,6 +40,8 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap
  */
 trait RunnableCommand extends Command {
 
+  override def children: Seq[LogicalPlan] = Nil
+
   // The map used to record the metrics of running the command. This will be passed to
   // `ExecutedCommand` during query planning.
   lazy val metrics: Map[String, SQLMetric] = Map.empty
@@ -46,6 +49,8 @@ trait RunnableCommand extends Command {
   def run(sparkSession: SparkSession): Seq[Row]
 }
 
+trait LeafRunnableCommand extends RunnableCommand with LeafLike[LogicalPlan]
+
 /**
  * A physical operator that executes the run method of a `RunnableCommand` and
  * saves the result to prevent multiple executions.
@@ -78,7 +83,7 @@ case class ExecutedCommandExec(cmd: RunnableCommand) extends LeafExecNode {
 
   override def executeCollect(): Array[InternalRow] = sideEffectResult.toArray
 
-  override def executeToIterator: Iterator[InternalRow] = sideEffectResult.toIterator
+  override def executeToIterator(): Iterator[InternalRow] = sideEffectResult.toIterator
 
   override def executeTake(limit: Int): Array[InternalRow] = sideEffectResult.take(limit).toArray
 
@@ -119,7 +124,7 @@ case class DataWritingCommandExec(cmd: DataWritingCommand, child: SparkPlan)
 
   override def executeCollect(): Array[InternalRow] = sideEffectResult.toArray
 
-  override def executeToIterator: Iterator[InternalRow] = sideEffectResult.toIterator
+  override def executeToIterator(): Iterator[InternalRow] = sideEffectResult.toIterator
 
   override def executeTake(limit: Int): Array[InternalRow] = sideEffectResult.take(limit).toArray
 
@@ -130,6 +135,9 @@ case class DataWritingCommandExec(cmd: DataWritingCommand, child: SparkPlan)
   protected override def doExecute(): RDD[InternalRow] = {
     sqlContext.sparkContext.parallelize(sideEffectResult, 1)
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): DataWritingCommandExec =
+    copy(child = newChild)
 }
 
 /**
@@ -148,7 +156,7 @@ case class DataWritingCommandExec(cmd: DataWritingCommand, child: SparkPlan)
 case class ExplainCommand(
     logicalPlan: LogicalPlan,
     mode: ExplainMode)
-  extends RunnableCommand {
+  extends LeafRunnableCommand {
 
   override val output: Seq[Attribute] =
     Seq(AttributeReference("plan", StringType, nullable = true)())
@@ -157,7 +165,7 @@ case class ExplainCommand(
   override def run(sparkSession: SparkSession): Seq[Row] = try {
     val outputString = sparkSession.sessionState.executePlan(logicalPlan).explainString(mode)
     Seq(Row(outputString))
-  } catch { case cause: TreeNodeException[_] =>
+  } catch { case NonFatal(cause) =>
     ("Error occurred during query planning: \n" + cause.getMessage).split("\n").map(Row(_))
   }
 }
@@ -165,7 +173,7 @@ case class ExplainCommand(
 /** An explain command for users to see how a streaming batch is executed. */
 case class StreamingExplainCommand(
     queryExecution: IncrementalExecution,
-    extended: Boolean) extends RunnableCommand {
+    extended: Boolean) extends LeafRunnableCommand {
 
   override val output: Seq[Attribute] =
     Seq(AttributeReference("plan", StringType, nullable = true)())
@@ -179,7 +187,7 @@ case class StreamingExplainCommand(
         queryExecution.simpleString
       }
     Seq(Row(outputString))
-  } catch { case cause: TreeNodeException[_] =>
+  } catch { case NonFatal(cause) =>
     ("Error occurred during query planning: \n" + cause.getMessage).split("\n").map(Row(_))
   }
 }
@@ -191,7 +199,7 @@ case class StreamingExplainCommand(
 case class ExternalCommandExecutor(
     runner: ExternalCommandRunner,
     command: String,
-    options: Map[String, String]) extends RunnableCommand {
+    options: Map[String, String]) extends LeafRunnableCommand {
 
   override def output: Seq[Attribute] =
     Seq(AttributeReference("command_output", StringType)())
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/createDataSourceTables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/createDataSourceTables.scala
index 68c47d6a6dfaa..bb3869ddf811e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/createDataSourceTables.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/createDataSourceTables.scala
@@ -21,8 +21,8 @@ import java.net.URI
 
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.catalog._
-import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.util.CharVarcharUtils
 import org.apache.spark.sql.execution.SparkPlan
 import org.apache.spark.sql.execution.datasources._
 import org.apache.spark.sql.sources.BaseRelation
@@ -42,7 +42,7 @@ import org.apache.spark.sql.types.StructType
  * }}}
  */
 case class CreateDataSourceTableCommand(table: CatalogTable, ignoreIfExists: Boolean)
-  extends RunnableCommand {
+  extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     assert(table.tableType != CatalogTableType.VIEW)
@@ -175,12 +175,13 @@ case class CreateDataSourceTableAsSelectCommand(
       }
       val result = saveDataIntoTable(
         sparkSession, table, tableLocation, child, SaveMode.Overwrite, tableExists = false)
+      val tableSchema = CharVarcharUtils.getRawSchema(result.schema)
       val newTable = table.copy(
         storage = table.storage.copy(locationUri = tableLocation),
         // We will use the schema of resolved.relation as the schema of the table (instead of
         // the schema of df). It is important since the nullability may be changed by the relation
         // provider (for example, see org.apache.spark.sql.parquet.DefaultSource).
-        schema = result.schema)
+        schema = tableSchema)
       // Table location is already validated. No need to check it again during table creation.
       sessionState.catalog.createTable(newTable, ignoreIfExists = false, validateLocation = false)
 
@@ -188,7 +189,10 @@ case class CreateDataSourceTableAsSelectCommand(
         case fs: HadoopFsRelation if table.partitionColumnNames.nonEmpty &&
             sparkSession.sqlContext.conf.manageFilesourcePartitions =>
           // Need to recover partitions into the metastore so our saved data is visible.
-          sessionState.executePlan(AlterTableRecoverPartitionsCommand(table.identifier)).toRdd
+          sessionState.executePlan(RepairTableCommand(
+            table.identifier,
+            enableAddPartitions = true,
+            enableDropPartitions = false)).toRdd
         case _ =>
       }
     }
@@ -216,11 +220,14 @@ case class CreateDataSourceTableAsSelectCommand(
       catalogTable = if (tableExists) Some(table) else None)
 
     try {
-      dataSource.writeAndRead(mode, query, outputColumnNames, physicalPlan)
+      dataSource.writeAndRead(mode, query, outputColumnNames, physicalPlan, metrics)
     } catch {
       case ex: AnalysisException =>
         logError(s"Failed to write to table ${table.identifier.unquotedString}", ex)
         throw ex
     }
   }
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan =
+    copy(query = newChild)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
index 47b213fc2d83b..0876b5f058762 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
@@ -35,15 +35,16 @@ import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.analysis.Resolver
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
-import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
+import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.connector.catalog.{CatalogV2Util, TableCatalog}
 import org.apache.spark.sql.connector.catalog.SupportsNamespaces._
-import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation, PartitioningUtils}
+import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation}
 import org.apache.spark.sql.execution.datasources.orc.OrcFileFormat
 import org.apache.spark.sql.execution.datasources.parquet.ParquetSchemaConverter
 import org.apache.spark.sql.internal.{HiveSerDe, SQLConf}
 import org.apache.spark.sql.types._
+import org.apache.spark.sql.util.PartitioningUtils
 import org.apache.spark.util.{SerializableConfiguration, ThreadUtils}
 
 // Note: The definition of these commands are based on the ones described in
@@ -68,7 +69,7 @@ case class CreateDatabaseCommand(
     path: Option[String],
     comment: Option[String],
     props: Map[String, String])
-  extends RunnableCommand {
+  extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val catalog = sparkSession.sessionState.catalog
@@ -88,8 +89,8 @@ case class CreateDatabaseCommand(
  * A command for users to remove a database from the system.
  *
  * 'ifExists':
- * - true, if database_name does't exist, no action
- * - false (default), if database_name does't exist, a warning message will be issued
+ * - true, if database_name doesn't exist, no action
+ * - false (default), if database_name doesn't exist, a warning message will be issued
  * 'cascade':
  * - true, the dependent objects are automatically dropped before dropping database.
  * - false (default), it is in the Restrict mode. The database cannot be dropped if
@@ -104,7 +105,7 @@ case class DropDatabaseCommand(
     databaseName: String,
     ifExists: Boolean,
     cascade: Boolean)
-  extends RunnableCommand {
+  extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     sparkSession.sessionState.catalog.dropDatabase(databaseName, ifExists, cascade)
@@ -124,7 +125,7 @@ case class DropDatabaseCommand(
 case class AlterDatabasePropertiesCommand(
     databaseName: String,
     props: Map[String, String])
-  extends RunnableCommand {
+  extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val catalog = sparkSession.sessionState.catalog
@@ -145,7 +146,7 @@ case class AlterDatabasePropertiesCommand(
  * }}}
  */
 case class AlterDatabaseSetLocationCommand(databaseName: String, location: String)
-  extends RunnableCommand {
+  extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val catalog = sparkSession.sessionState.catalog
@@ -168,8 +169,9 @@ case class AlterDatabaseSetLocationCommand(databaseName: String, location: Strin
  */
 case class DescribeDatabaseCommand(
     databaseName: String,
-    extended: Boolean)
-  extends RunnableCommand {
+    extended: Boolean,
+    override val output: Seq[Attribute])
+  extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val dbMetadata: CatalogDatabase =
@@ -194,11 +196,6 @@ case class DescribeDatabaseCommand(
       result
     }
   }
-
-  override val output: Seq[Attribute] = {
-    AttributeReference("database_description_item", StringType, nullable = false)() ::
-      AttributeReference("database_description_value", StringType, nullable = false)() :: Nil
-  }
 }
 
 /**
@@ -214,11 +211,11 @@ case class DropTableCommand(
     tableName: TableIdentifier,
     ifExists: Boolean,
     isView: Boolean,
-    purge: Boolean) extends RunnableCommand {
+    purge: Boolean) extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val catalog = sparkSession.sessionState.catalog
-    val isTempView = catalog.isTemporaryTable(tableName)
+    val isTempView = catalog.isTempView(tableName)
 
     if (!isTempView && catalog.tableExists(tableName)) {
       // If the command DROP VIEW is to drop a table or DROP TABLE is to drop a view
@@ -236,8 +233,10 @@ case class DropTableCommand(
 
     if (isTempView || catalog.tableExists(tableName)) {
       try {
+        val hasViewText = isTempView &&
+          catalog.getTempViewOrPermanentTableMetadata(tableName).viewText.isDefined
         sparkSession.sharedState.cacheManager.uncacheQuery(
-          sparkSession.table(tableName), cascade = !isTempView)
+          sparkSession.table(tableName), cascade = !isTempView || hasViewText)
       } catch {
         case NonFatal(e) => log.warn(e.toString, e)
       }
@@ -265,12 +264,11 @@ case class AlterTableSetPropertiesCommand(
     tableName: TableIdentifier,
     properties: Map[String, String],
     isView: Boolean)
-  extends RunnableCommand {
+  extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val catalog = sparkSession.sessionState.catalog
-    val table = catalog.getTableMetadata(tableName)
-    DDLUtils.verifyAlterTableType(catalog, table, isView)
+    val table = catalog.getTableRawMetadata(tableName)
     // This overrides old properties and update the comment parameter of CatalogTable
     // with the newly added/modified comment since CatalogTable also holds comment as its
     // direct property.
@@ -297,12 +295,11 @@ case class AlterTableUnsetPropertiesCommand(
     propKeys: Seq[String],
     ifExists: Boolean,
     isView: Boolean)
-  extends RunnableCommand {
+  extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val catalog = sparkSession.sessionState.catalog
-    val table = catalog.getTableMetadata(tableName)
-    DDLUtils.verifyAlterTableType(catalog, table, isView)
+    val table = catalog.getTableRawMetadata(tableName)
     if (!ifExists) {
       propKeys.foreach { k =>
         if (!table.properties.contains(k) && k != TableCatalog.PROP_COMMENT) {
@@ -336,12 +333,12 @@ case class AlterTableUnsetPropertiesCommand(
 case class AlterTableChangeColumnCommand(
     tableName: TableIdentifier,
     columnName: String,
-    newColumn: StructField) extends RunnableCommand {
+    newColumn: StructField) extends LeafRunnableCommand {
 
   // TODO: support change column name/dataType/metadata/position.
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val catalog = sparkSession.sessionState.catalog
-    val table = catalog.getTableMetadata(tableName)
+    val table = catalog.getTableRawMetadata(tableName)
     val resolver = sparkSession.sessionState.conf.resolver
     DDLUtils.verifyAlterTableType(catalog, table, isView = false)
 
@@ -405,7 +402,7 @@ case class AlterTableSerDePropertiesCommand(
     serdeClassName: Option[String],
     serdeProperties: Option[Map[String, String]],
     partSpec: Option[TablePartitionSpec])
-  extends RunnableCommand {
+  extends LeafRunnableCommand {
 
   // should never happen if we parsed things correctly
   require(serdeClassName.isDefined || serdeProperties.isDefined,
@@ -413,8 +410,7 @@ case class AlterTableSerDePropertiesCommand(
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val catalog = sparkSession.sessionState.catalog
-    val table = catalog.getTableMetadata(tableName)
-    DDLUtils.verifyAlterTableType(catalog, table, isView = false)
+    val table = catalog.getTableRawMetadata(tableName)
     // For datasource tables, disallow setting serde or specifying partition
     if (partSpec.isDefined && DDLUtils.isDatasourceTable(table)) {
       throw new AnalysisException("Operation not allowed: ALTER TABLE SET " +
@@ -458,17 +454,16 @@ case class AlterTableAddPartitionCommand(
     tableName: TableIdentifier,
     partitionSpecsAndLocs: Seq[(TablePartitionSpec, Option[String])],
     ifNotExists: Boolean)
-  extends RunnableCommand {
+  extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val catalog = sparkSession.sessionState.catalog
     val table = catalog.getTableMetadata(tableName)
-    DDLUtils.verifyAlterTableType(catalog, table, isView = false)
     DDLUtils.verifyPartitionProviderIsHive(sparkSession, table, "ALTER TABLE ADD PARTITION")
     val parts = partitionSpecsAndLocs.map { case (spec, location) =>
       val normalizedSpec = PartitioningUtils.normalizePartitionSpec(
         spec,
-        table.partitionColumnNames,
+        table.partitionSchema,
         table.identifier.quotedString,
         sparkSession.sessionState.conf.resolver)
       // inherit table storage format (possibly except for location)
@@ -484,17 +479,18 @@ case class AlterTableAddPartitionCommand(
       catalog.createPartitions(table.identifier, batch, ignoreIfExists = ifNotExists)
     }
 
-    if (table.stats.nonEmpty) {
-      if (sparkSession.sessionState.conf.autoSizeUpdateEnabled) {
-        val addedSize = CommandUtils.calculateMultipleLocationSizes(sparkSession, table.identifier,
-          parts.map(_.storage.locationUri)).sum
-        if (addedSize > 0) {
-          val newStats = CatalogStatistics(sizeInBytes = table.stats.get.sizeInBytes + addedSize)
-          catalog.alterTableStats(table.identifier, Some(newStats))
-        }
-      } else {
-        catalog.alterTableStats(table.identifier, None)
+    sparkSession.catalog.refreshTable(table.identifier.quotedString)
+    if (table.stats.nonEmpty && sparkSession.sessionState.conf.autoSizeUpdateEnabled) {
+      // Updating table stats only if new partition is not empty
+      val addedSize = CommandUtils.calculateMultipleLocationSizes(sparkSession, table.identifier,
+        parts.map(_.storage.locationUri)).sum
+      if (addedSize > 0) {
+        val newStats = CatalogStatistics(sizeInBytes = table.stats.get.sizeInBytes + addedSize)
+        catalog.alterTableStats(table.identifier, Some(newStats))
       }
+    } else {
+      // Re-calculating of table size including all partitions
+      CommandUtils.updateTableStats(sparkSession, table)
     }
     Seq.empty[Row]
   }
@@ -513,28 +509,28 @@ case class AlterTableRenamePartitionCommand(
     tableName: TableIdentifier,
     oldPartition: TablePartitionSpec,
     newPartition: TablePartitionSpec)
-  extends RunnableCommand {
+  extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val catalog = sparkSession.sessionState.catalog
     val table = catalog.getTableMetadata(tableName)
-    DDLUtils.verifyAlterTableType(catalog, table, isView = false)
     DDLUtils.verifyPartitionProviderIsHive(sparkSession, table, "ALTER TABLE RENAME PARTITION")
 
     val normalizedOldPartition = PartitioningUtils.normalizePartitionSpec(
       oldPartition,
-      table.partitionColumnNames,
+      table.partitionSchema,
       table.identifier.quotedString,
       sparkSession.sessionState.conf.resolver)
 
     val normalizedNewPartition = PartitioningUtils.normalizePartitionSpec(
       newPartition,
-      table.partitionColumnNames,
+      table.partitionSchema,
       table.identifier.quotedString,
       sparkSession.sessionState.conf.resolver)
 
     catalog.renamePartitions(
       tableName, Seq(normalizedOldPartition), Seq(normalizedNewPartition))
+    sparkSession.catalog.refreshTable(table.identifier.quotedString)
     Seq.empty[Row]
   }
 
@@ -560,18 +556,17 @@ case class AlterTableDropPartitionCommand(
     ifExists: Boolean,
     purge: Boolean,
     retainData: Boolean)
-  extends RunnableCommand {
+  extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val catalog = sparkSession.sessionState.catalog
     val table = catalog.getTableMetadata(tableName)
-    DDLUtils.verifyAlterTableType(catalog, table, isView = false)
     DDLUtils.verifyPartitionProviderIsHive(sparkSession, table, "ALTER TABLE DROP PARTITION")
 
     val normalizedSpecs = specs.map { spec =>
       PartitioningUtils.normalizePartitionSpec(
         spec,
-        table.partitionColumnNames,
+        table.partitionSchema,
         table.identifier.quotedString,
         sparkSession.sessionState.conf.resolver)
     }
@@ -580,6 +575,7 @@ case class AlterTableDropPartitionCommand(
       table.identifier, normalizedSpecs, ignoreIfNotExists = ifExists, purge = purge,
       retainData = retainData)
 
+    sparkSession.catalog.refreshTable(table.identifier.quotedString)
     CommandUtils.updateTableStats(sparkSession, table)
 
     Seq.empty[Row]
@@ -591,18 +587,20 @@ case class AlterTableDropPartitionCommand(
 case class PartitionStatistics(numFiles: Int, totalSize: Long)
 
 /**
- * Recover Partitions in ALTER TABLE: recover all the partition in the directory of a table and
+ * Repair a table by recovering all the partition in the directory of the table and
  * update the catalog.
  *
  * The syntax of this command is:
  * {{{
  *   ALTER TABLE table RECOVER PARTITIONS;
- *   MSCK REPAIR TABLE table;
+ *   MSCK REPAIR TABLE table [{ADD|DROP|SYNC} PARTITIONS];
  * }}}
  */
-case class AlterTableRecoverPartitionsCommand(
+case class RepairTableCommand(
     tableName: TableIdentifier,
-    cmd: String = "ALTER TABLE RECOVER PARTITIONS") extends RunnableCommand {
+    enableAddPartitions: Boolean,
+    enableDropPartitions: Boolean,
+    cmd: String = "MSCK REPAIR TABLE") extends LeafRunnableCommand {
 
   // These are list of statistics that can be collected quickly without requiring a scan of the data
   // see https://github.com/apache/hive/blob/master/
@@ -628,9 +626,8 @@ case class AlterTableRecoverPartitionsCommand(
 
   override def run(spark: SparkSession): Seq[Row] = {
     val catalog = spark.sessionState.catalog
-    val table = catalog.getTableMetadata(tableName)
+    val table = catalog.getTableRawMetadata(tableName)
     val tableIdentWithDB = table.identifier.quotedString
-    DDLUtils.verifyAlterTableType(catalog, table, isView = false)
     if (table.partitionColumnNames.isEmpty) {
       throw new AnalysisException(
         s"Operation not allowed: $cmd only works on partitioned tables: $tableIdentWithDB")
@@ -646,34 +643,40 @@ case class AlterTableRecoverPartitionsCommand(
     val hadoopConf = spark.sessionState.newHadoopConf()
     val fs = root.getFileSystem(hadoopConf)
 
-    val threshold = spark.sparkContext.conf.get(RDD_PARALLEL_LISTING_THRESHOLD)
-    val pathFilter = getPathFilter(hadoopConf)
+    val droppedAmount = if (enableDropPartitions) {
+      dropPartitions(catalog, fs)
+    } else 0
+    val addedAmount = if (enableAddPartitions) {
+      val threshold = spark.sparkContext.conf.get(RDD_PARALLEL_LISTING_THRESHOLD)
+      val pathFilter = getPathFilter(hadoopConf)
+
+      val evalPool = ThreadUtils.newForkJoinPool("RepairTableCommand", 8)
+      val partitionSpecsAndLocs: GenSeq[(TablePartitionSpec, Path)] =
+        try {
+          scanPartitions(spark, fs, pathFilter, root, Map(), table.partitionColumnNames, threshold,
+            spark.sessionState.conf.resolver, new ForkJoinTaskSupport(evalPool)).seq
+        } finally {
+          evalPool.shutdown()
+        }
+      val total = partitionSpecsAndLocs.length
+      logInfo(s"Found $total partitions in $root")
 
-    val evalPool = ThreadUtils.newForkJoinPool("AlterTableRecoverPartitionsCommand", 8)
-    val partitionSpecsAndLocs: Seq[(TablePartitionSpec, Path)] =
-      try {
-        scanPartitions(spark, fs, pathFilter, root, Map(), table.partitionColumnNames, threshold,
-          spark.sessionState.conf.resolver, new ForkJoinTaskSupport(evalPool)).seq
-      } finally {
-        evalPool.shutdown()
+      val partitionStats = if (spark.sqlContext.conf.gatherFastStats) {
+        gatherPartitionStats(spark, partitionSpecsAndLocs, fs, pathFilter, threshold)
+      } else {
+        GenMap.empty[String, PartitionStatistics]
       }
-    val total = partitionSpecsAndLocs.length
-    logInfo(s"Found $total partitions in $root")
-
-    val partitionStats = if (spark.sqlContext.conf.gatherFastStats) {
-      gatherPartitionStats(spark, partitionSpecsAndLocs, fs, pathFilter, threshold)
-    } else {
-      GenMap.empty[String, PartitionStatistics]
-    }
-    logInfo(s"Finished to gather the fast stats for all $total partitions.")
+      logInfo(s"Finished to gather the fast stats for all $total partitions.")
 
-    addPartitions(spark, table, partitionSpecsAndLocs, partitionStats)
+      addPartitions(spark, table, partitionSpecsAndLocs, partitionStats)
+      total
+    } else 0
     // Updates the table to indicate that its partition metadata is stored in the Hive metastore.
     // This is always the case for Hive format tables, but is not true for Datasource tables created
     // before Spark 2.1 unless they are converted via `msck repair table`.
     spark.sessionState.catalog.alterTable(table.copy(tracksPartitionsInCatalog = true))
-    catalog.refreshTable(tableName)
-    logInfo(s"Recovered all partitions ($total).")
+    spark.catalog.refreshTable(tableIdentWithDB)
+    logInfo(s"Recovered all partitions: added ($addedAmount), dropped ($droppedAmount).")
     Seq.empty[Row]
   }
 
@@ -697,7 +700,7 @@ case class AlterTableRecoverPartitionsCommand(
         // parallelize the list of partitions here, then we can have better parallelism later.
         val parArray = new ParVector(statuses.toVector)
         parArray.tasksupport = evalTaskSupport
-        parArray
+        parArray.seq
       } else {
         statuses
       }
@@ -768,7 +771,7 @@ case class AlterTableRecoverPartitionsCommand(
     // Hive metastore may not have enough memory to handle millions of partitions in single RPC,
     // we should split them into smaller batches. Since Hive client is not thread safe, we cannot
     // do this in parallel.
-    val batchSize = 100
+    val batchSize = spark.conf.get(SQLConf.ADD_PARTITION_BATCH_SIZE)
     partitionSpecsAndLocs.toIterator.grouped(batchSize).foreach { batch =>
       val now = MILLISECONDS.toSeconds(System.currentTimeMillis())
       val parts = batch.map { case (spec, location) =>
@@ -792,8 +795,28 @@ case class AlterTableRecoverPartitionsCommand(
       logDebug(s"Recovered ${parts.length} partitions ($done/$total so far)")
     }
   }
-}
 
+  // Drops the partitions that do not exist in the file system
+  private def dropPartitions(catalog: SessionCatalog, fs: FileSystem): Int = {
+    val dropPartSpecs = ThreadUtils.parmap(
+      catalog.listPartitions(tableName),
+      "RepairTableCommand: non-existing partitions",
+      maxThreads = 8) { partition =>
+      partition.storage.locationUri.flatMap { uri =>
+        if (fs.exists(new Path(uri))) None else Some(partition.spec)
+      }
+    }.flatten
+    catalog.dropPartitions(
+      tableName,
+      dropPartSpecs,
+      ignoreIfNotExists = true,
+      purge = false,
+      // Since we have already checked that partition directories do not exist, we can avoid
+      // additional calls to the file system at the catalog side by setting this flag.
+      retainData = true)
+    dropPartSpecs.length
+  }
+}
 
 /**
  * A command that sets the location of a table or a partition.
@@ -810,26 +833,30 @@ case class AlterTableSetLocationCommand(
     tableName: TableIdentifier,
     partitionSpec: Option[TablePartitionSpec],
     location: String)
-  extends RunnableCommand {
+  extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val catalog = sparkSession.sessionState.catalog
     val table = catalog.getTableMetadata(tableName)
     val locUri = CatalogUtils.stringToURI(location)
-    DDLUtils.verifyAlterTableType(catalog, table, isView = false)
     partitionSpec match {
       case Some(spec) =>
         DDLUtils.verifyPartitionProviderIsHive(
           sparkSession, table, "ALTER TABLE ... SET LOCATION")
         // Partition spec is specified, so we set the location only for this partition
-        val part = catalog.getPartition(table.identifier, spec)
+        val normalizedSpec = PartitioningUtils.normalizePartitionSpec(
+          spec,
+          table.partitionSchema,
+          table.identifier.quotedString,
+          sparkSession.sessionState.conf.resolver)
+        val part = catalog.getPartition(table.identifier, normalizedSpec)
         val newPart = part.copy(storage = part.storage.copy(locationUri = Some(locUri)))
         catalog.alterPartitions(table.identifier, Seq(newPart))
       case None =>
         // No partition spec is specified, so we set the location for the table itself
         catalog.alterTable(table.withNewStorage(locationUri = Some(locUri)))
     }
-
+    sparkSession.catalog.refreshTable(table.identifier.quotedString)
     CommandUtils.updateTableStats(sparkSession, table)
     Seq.empty[Row]
   }
@@ -891,7 +918,7 @@ object DDLUtils {
       catalog: SessionCatalog,
       tableMetadata: CatalogTable,
       isView: Boolean): Unit = {
-    if (!catalog.isTemporaryTable(tableMetadata.identifier)) {
+    if (!catalog.isTempView(tableMetadata.identifier)) {
       tableMetadata.tableType match {
         case CatalogTableType.VIEW if !isView =>
           throw new AnalysisException(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala
index 6fdc7f4a58195..0eda90a596999 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala
@@ -23,8 +23,9 @@ import org.apache.spark.sql.{AnalysisException, Row, SparkSession}
 import org.apache.spark.sql.catalyst.FunctionIdentifier
 import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, NoSuchFunctionException}
 import org.apache.spark.sql.catalyst.catalog.{CatalogFunction, FunctionResource}
-import org.apache.spark.sql.catalyst.expressions.{Attribute, ExpressionInfo}
+import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.util.StringUtils
+import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.types.{StringType, StructField, StructType}
 
 
@@ -54,7 +55,7 @@ case class CreateFunctionCommand(
     isTemp: Boolean,
     ignoreIfExists: Boolean,
     replace: Boolean)
-  extends RunnableCommand {
+  extends LeafRunnableCommand {
 
   if (ignoreIfExists && replace) {
     throw new AnalysisException("CREATE FUNCTION with both IF NOT EXISTS and REPLACE" +
@@ -77,6 +78,9 @@ case class CreateFunctionCommand(
     val catalog = sparkSession.sessionState.catalog
     val func = CatalogFunction(FunctionIdentifier(functionName, databaseName), className, resources)
     if (isTemp) {
+      if (!replace && catalog.isRegisteredFunction(func.identifier)) {
+        throw QueryCompilationErrors.functionAlreadyExistsError(func.identifier)
+      }
       // We first load resources and then put the builder in the function registry.
       catalog.loadFunctionResources(resources)
       catalog.registerFunction(func, overrideIfExists = replace)
@@ -88,7 +92,9 @@ case class CreateFunctionCommand(
       } else {
         // For a permanent, we will store the metadata into underlying external catalog.
         // This function will be loaded into the FunctionRegistry when a query uses it.
-        // We do not load it into FunctionRegistry right now.
+        // We do not load it into FunctionRegistry right now, to avoid loading the resource and
+        // UDF class immediately, as the Spark application to create the function may not have
+        // access to the resource and/or UDF class.
         catalog.createFunction(func, ignoreIfExists)
       }
     }
@@ -106,7 +112,7 @@ case class CreateFunctionCommand(
  */
 case class DescribeFunctionCommand(
     functionName: FunctionIdentifier,
-    isExtended: Boolean) extends RunnableCommand {
+    isExtended: Boolean) extends LeafRunnableCommand {
 
   override val output: Seq[Attribute] = {
     val schema = StructType(StructField("function_desc", StringType, nullable = false) :: Nil)
@@ -114,7 +120,8 @@ case class DescribeFunctionCommand(
   }
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
-    // Hard code "<>", "!=", "between", and "case" for now as there is no corresponding functions.
+    // Hard code "<>", "!=", "between", "case", and "||"
+    // for now as there is no corresponding functions.
     functionName.funcName.toLowerCase(Locale.ROOT) match {
       case "<>" =>
         Row(s"Function: $functionName") ::
@@ -134,6 +141,9 @@ case class DescribeFunctionCommand(
             "[WHEN expr4 THEN expr5]* [ELSE expr6] END - " +
             "When `expr1` = `expr2`, returns `expr3`; " +
             "when `expr1` = `expr4`, return `expr5`; else return `expr6`.") :: Nil
+      case "||" =>
+        Row("Function: ||") ::
+          Row("Usage: expr1 || expr2 - Returns the concatenation of `expr1` and `expr2`.") :: Nil
       case _ =>
         try {
           val info = sparkSession.sessionState.catalog.lookupFunctionInfo(functionName)
@@ -167,7 +177,7 @@ case class DropFunctionCommand(
     functionName: String,
     ifExists: Boolean,
     isTemp: Boolean)
-  extends RunnableCommand {
+  extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val catalog = sparkSession.sessionState.catalog
@@ -205,12 +215,8 @@ case class ShowFunctionsCommand(
     db: Option[String],
     pattern: Option[String],
     showUserFunctions: Boolean,
-    showSystemFunctions: Boolean) extends RunnableCommand {
-
-  override val output: Seq[Attribute] = {
-    val schema = StructType(StructField("function", StringType, nullable = false) :: Nil)
-    schema.toAttributes
-  }
+    showSystemFunctions: Boolean,
+    override val output: Seq[Attribute]) extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val dbName = db.getOrElse(sparkSession.sessionState.catalog.getCurrentDatabase)
@@ -223,8 +229,10 @@ case class ShowFunctionsCommand(
           case (f, "USER") if showUserFunctions => f.unquotedString
           case (f, "SYSTEM") if showSystemFunctions => f.unquotedString
         }
-    // Hard code "<>", "!=", "between", and "case" for now as there is no corresponding functions.
-    // "<>", "!=", "between", and "case" is SystemFunctions, only show when showSystemFunctions=true
+    // Hard code "<>", "!=", "between", "case", and "||"
+    // for now as there is no corresponding functions.
+    // "<>", "!=", "between", "case", and "||" is SystemFunctions,
+    // only show when showSystemFunctions=true
     if (showSystemFunctions) {
       (functionNames ++
         StringUtils.filterPattern(FunctionsCommand.virtualOperators, pattern.getOrElse("*")))
@@ -236,8 +244,47 @@ case class ShowFunctionsCommand(
   }
 }
 
+
+/**
+ * A command for users to refresh the persistent function.
+ * The syntax of using this command in SQL is:
+ * {{{
+ *    REFRESH FUNCTION functionName
+ * }}}
+ */
+case class RefreshFunctionCommand(
+    databaseName: Option[String],
+    functionName: String)
+  extends LeafRunnableCommand {
+
+  override def run(sparkSession: SparkSession): Seq[Row] = {
+    val catalog = sparkSession.sessionState.catalog
+    if (FunctionRegistry.builtin.functionExists(FunctionIdentifier(functionName, databaseName))) {
+      throw new AnalysisException(s"Cannot refresh built-in function $functionName")
+    }
+    if (catalog.isTemporaryFunction(FunctionIdentifier(functionName, databaseName))) {
+      throw new AnalysisException(s"Cannot refresh temporary function $functionName")
+    }
+
+    val identifier = FunctionIdentifier(
+      functionName, Some(databaseName.getOrElse(catalog.getCurrentDatabase)))
+    // we only refresh the permanent function.
+    if (catalog.isPersistentFunction(identifier)) {
+      // register overwrite function.
+      val func = catalog.getFunctionMetadata(identifier)
+      catalog.registerFunction(func, true)
+    } else {
+      // clear cached function and throw exception
+      catalog.unregisterFunction(identifier)
+      throw new NoSuchFunctionException(identifier.database.get, identifier.funcName)
+    }
+
+    Seq.empty[Row]
+  }
+}
+
 object FunctionsCommand {
   // operators that do not have corresponding functions.
   // They should be handled `DescribeFunctionCommand`, `ShowFunctionsCommand`
-  val virtualOperators = Seq("!=", "<>", "between", "case")
+  val virtualOperators = Seq("!=", "<>", "between", "case", "||")
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/resources.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/resources.scala
index 549477dbae6ba..5d3f46a1a43ea 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/resources.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/resources.scala
@@ -18,26 +18,18 @@
 package org.apache.spark.sql.execution.command
 
 import java.io.File
-import java.net.URI
-
-import org.apache.hadoop.fs.Path
 
 import org.apache.spark.sql.{Row, SparkSession}
 import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
-import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
+import org.apache.spark.sql.types.StringType
+import org.apache.spark.util.Utils
 
 /**
  * Adds a jar to the current session so it can be used (for UDFs or serdes).
  */
-case class AddJarCommand(path: String) extends RunnableCommand {
-  override val output: Seq[Attribute] = {
-    val schema = StructType(
-      StructField("result", IntegerType, nullable = false) :: Nil)
-    schema.toAttributes
-  }
-
+case class AddJarsCommand(paths: Seq[String]) extends LeafRunnableCommand {
   override def run(sparkSession: SparkSession): Seq[Row] = {
-    sparkSession.sessionState.resourceLoader.addJar(path)
+    paths.foreach(sparkSession.sessionState.resourceLoader.addJar(_))
     Seq.empty[Row]
   }
 }
@@ -45,10 +37,20 @@ case class AddJarCommand(path: String) extends RunnableCommand {
 /**
  * Adds a file to the current session so it can be used.
  */
-case class AddFileCommand(path: String) extends RunnableCommand {
+case class AddFilesCommand(paths: Seq[String]) extends LeafRunnableCommand {
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val recursive = !sparkSession.sessionState.conf.addSingleFileInAddFile
-    sparkSession.sparkContext.addFile(path, recursive)
+    paths.foreach(sparkSession.sparkContext.addFile(_, recursive))
+    Seq.empty[Row]
+  }
+}
+
+/**
+ * Adds an archive to the current session so it can be used.
+ */
+case class AddArchivesCommand(paths: Seq[String]) extends LeafRunnableCommand {
+  override def run(sparkSession: SparkSession): Seq[Row] = {
+    paths.foreach(sparkSession.sparkContext.addArchive(_))
     Seq.empty[Row]
   }
 }
@@ -57,7 +59,7 @@ case class AddFileCommand(path: String) extends RunnableCommand {
  * Returns a list of file paths that are added to resources.
  * If file paths are provided, return the ones that are added to resources.
  */
-case class ListFilesCommand(files: Seq[String] = Seq.empty[String]) extends RunnableCommand {
+case class ListFilesCommand(files: Seq[String] = Seq.empty[String]) extends LeafRunnableCommand {
   override val output: Seq[Attribute] = {
     AttributeReference("Results", StringType, nullable = false)() :: Nil
   }
@@ -65,12 +67,11 @@ case class ListFilesCommand(files: Seq[String] = Seq.empty[String]) extends Runn
     val fileList = sparkSession.sparkContext.listFiles()
     if (files.size > 0) {
       files.map { f =>
-        val uri = new URI(f)
-        val schemeCorrectedPath = uri.getScheme match {
-          case null | "local" => new File(f).getCanonicalFile.toURI.toString
+        val uri = Utils.resolveURI(f)
+        uri.getScheme match {
+          case null | "local" | "file" => new File(uri).getCanonicalFile.toURI.toString
           case _ => f
         }
-        new Path(schemeCorrectedPath).toUri.toString
       }.collect {
         case f if fileList.contains(f) => f
       }.map(Row(_))
@@ -84,7 +85,7 @@ case class ListFilesCommand(files: Seq[String] = Seq.empty[String]) extends Runn
  * Returns a list of jar files that are added to resources.
  * If jar files are provided, return the ones that are added to resources.
  */
-case class ListJarsCommand(jars: Seq[String] = Seq.empty[String]) extends RunnableCommand {
+case class ListJarsCommand(jars: Seq[String] = Seq.empty[String]) extends LeafRunnableCommand {
   override val output: Seq[Attribute] = {
     AttributeReference("Results", StringType, nullable = false)() :: Nil
   }
@@ -92,7 +93,7 @@ case class ListJarsCommand(jars: Seq[String] = Seq.empty[String]) extends Runnab
     val jarList = sparkSession.sparkContext.listJars()
     if (jars.nonEmpty) {
       for {
-        jarName <- jars.map(f => new Path(f).getName)
+        jarName <- jars.map(f => Utils.resolveURI(f).toString.split("/").last)
         jarPath <- jarList if jarPath.contains(jarName)
       } yield Row(jarPath)
     } else {
@@ -100,3 +101,25 @@ case class ListJarsCommand(jars: Seq[String] = Seq.empty[String]) extends Runnab
     }
   }
 }
+
+/**
+ * Returns a list of archive paths that are added to resources.
+ * If archive paths are provided, return the ones that are added to resources.
+ */
+case class ListArchivesCommand(archives: Seq[String] = Seq.empty[String])
+  extends LeafRunnableCommand {
+  override val output: Seq[Attribute] = {
+    AttributeReference("Results", StringType, nullable = false)() :: Nil
+  }
+  override def run(sparkSession: SparkSession): Seq[Row] = {
+    val archiveList = sparkSession.sparkContext.listArchives()
+    if (archives.nonEmpty) {
+      for {
+        archiveName <- archives.map(f => Utils.resolveURI(f).toString.split("/").last)
+        archivePath <- archiveList if archivePath.contains(archiveName)
+      } yield Row(archivePath)
+    } else {
+      archiveList.map(Row(_))
+    }
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
index fc8cc11bb1067..72168f243900f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
@@ -21,7 +21,6 @@ import java.net.{URI, URISyntaxException}
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable.ArrayBuffer
-import scala.util.Try
 import scala.util.control.NonFatal
 
 import org.apache.hadoop.fs.{FileContext, FsConstants, Path}
@@ -33,11 +32,12 @@ import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionException, Unresol
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.catalog.CatalogTableType._
 import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
-import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
-import org.apache.spark.sql.catalyst.plans.DescribeTableSchema
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.plans.DescribeCommandSchema
 import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.catalyst.util.{escapeSingleQuotedString, quoteIdentifier, CaseInsensitiveMap}
-import org.apache.spark.sql.execution.datasources.{DataSource, PartitioningUtils}
+import org.apache.spark.sql.catalyst.util.{escapeSingleQuotedString, quoteIdentifier, CaseInsensitiveMap, CharVarcharUtils}
+import org.apache.spark.sql.errors.QueryCompilationErrors
+import org.apache.spark.sql.execution.datasources.DataSource
 import org.apache.spark.sql.execution.datasources.csv.CSVFileFormat
 import org.apache.spark.sql.execution.datasources.json.JsonFileFormat
 import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
@@ -47,6 +47,7 @@ import org.apache.spark.sql.execution.datasources.v2.orc.OrcDataSourceV2
 import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetDataSourceV2
 import org.apache.spark.sql.internal.{HiveSerDe, SQLConf}
 import org.apache.spark.sql.types._
+import org.apache.spark.sql.util.PartitioningUtils
 import org.apache.spark.sql.util.SchemaUtils
 
 /**
@@ -81,7 +82,7 @@ case class CreateTableLikeCommand(
     fileFormat: CatalogStorageFormat,
     provider: Option[String],
     properties: Map[String, String] = Map.empty,
-    ifNotExists: Boolean) extends RunnableCommand {
+    ifNotExists: Boolean) extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val catalog = sparkSession.sessionState.catalog
@@ -115,12 +116,13 @@ case class CreateTableLikeCommand(
       CatalogTableType.EXTERNAL
     }
 
+    val newTableSchema = CharVarcharUtils.getRawSchema(sourceTableDesc.schema)
     val newTableDesc =
       CatalogTable(
         identifier = targetTable,
         tableType = tblType,
         storage = newStorage,
-        schema = sourceTableDesc.schema,
+        schema = newTableSchema,
         provider = newProvider,
         partitionColumnNames = sourceTableDesc.partitionColumnNames,
         bucketSpec = sourceTableDesc.bucketSpec,
@@ -159,7 +161,7 @@ case class CreateTableLikeCommand(
  */
 case class CreateTableCommand(
     table: CatalogTable,
-    ignoreIfExists: Boolean) extends RunnableCommand {
+    ignoreIfExists: Boolean) extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     sparkSession.sessionState.catalog.createTable(table, ignoreIfExists)
@@ -181,29 +183,30 @@ case class AlterTableRenameCommand(
     oldName: TableIdentifier,
     newName: TableIdentifier,
     isView: Boolean)
-  extends RunnableCommand {
+  extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val catalog = sparkSession.sessionState.catalog
     // If this is a temp view, just rename the view.
     // Otherwise, if this is a real table, we also need to uncache and invalidate the table.
-    if (catalog.isTemporaryTable(oldName)) {
+    if (catalog.isTempView(oldName)) {
       catalog.renameTable(oldName, newName)
     } else {
       val table = catalog.getTableMetadata(oldName)
       DDLUtils.verifyAlterTableType(catalog, table, isView)
-      // If an exception is thrown here we can just assume the table is uncached;
-      // this can happen with Hive tables when the underlying catalog is in-memory.
-      val wasCached = Try(sparkSession.catalog.isCached(oldName.unquotedString)).getOrElse(false)
-      if (wasCached) {
+      // If `optStorageLevel` is defined, the old table was cached.
+      val optCachedData = sparkSession.sharedState.cacheManager.lookupCachedData(
+        sparkSession.table(oldName.unquotedString))
+      val optStorageLevel = optCachedData.map(_.cachedRepresentation.cacheBuilder.storageLevel)
+      if (optStorageLevel.isDefined) {
         CommandUtils.uncacheTableOrView(sparkSession, oldName.unquotedString)
       }
       // Invalidate the table last, otherwise uncaching the table would load the logical plan
       // back into the hive metastore cache
       catalog.refreshTable(oldName)
       catalog.renameTable(oldName, newName)
-      if (wasCached) {
-        sparkSession.catalog.cacheTable(newName.unquotedString)
+      optStorageLevel.foreach { storageLevel =>
+        sparkSession.catalog.cacheTable(newName.unquotedString, storageLevel)
       }
     }
     Seq.empty[Row]
@@ -221,7 +224,7 @@ case class AlterTableRenameCommand(
 */
 case class AlterTableAddColumnsCommand(
     table: TableIdentifier,
-    colsToAdd: Seq[StructField]) extends RunnableCommand {
+    colsToAdd: Seq[StructField]) extends LeafRunnableCommand {
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val catalog = sparkSession.sessionState.catalog
     val catalogTable = verifyAlterTableAddColumn(sparkSession.sessionState.conf, catalog, table)
@@ -235,7 +238,8 @@ case class AlterTableAddColumnsCommand(
       conf.caseSensitiveAnalysis)
     DDLUtils.checkDataColNames(catalogTable, colsToAdd.map(_.name))
 
-    catalog.alterTableDataSchema(table, StructType(catalogTable.dataSchema ++ colsToAdd))
+    val existingSchema = CharVarcharUtils.getRawSchema(catalogTable.dataSchema)
+    catalog.alterTableDataSchema(table, StructType(existingSchema ++ colsToAdd))
     Seq.empty[Row]
   }
 
@@ -296,7 +300,7 @@ case class LoadDataCommand(
     path: String,
     isLocal: Boolean,
     isOverwrite: Boolean,
-    partition: Option[TablePartitionSpec]) extends RunnableCommand {
+    partition: Option[TablePartitionSpec]) extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val catalog = sparkSession.sessionState.catalog
@@ -305,14 +309,11 @@ case class LoadDataCommand(
     val normalizedSpec = partition.map { spec =>
       PartitioningUtils.normalizePartitionSpec(
         spec,
-        targetTable.partitionColumnNames,
+        targetTable.partitionSchema,
         tableIdentwithDB,
         sparkSession.sessionState.conf.resolver)
     }
 
-    if (targetTable.tableType == CatalogTableType.VIEW) {
-      throw new AnalysisException(s"Target table in LOAD DATA cannot be a view: $tableIdentwithDB")
-    }
     if (DDLUtils.isDatasourceTable(targetTable)) {
       throw new AnalysisException(
         s"LOAD DATA is not supported for datasource tables: $tableIdentwithDB")
@@ -354,7 +355,7 @@ case class LoadDataCommand(
         // entire  string will be considered while making a Path instance,this is mainly done
         // by considering the wild card scenario in mind.as per old logic query param  is
         // been considered while creating URI instance and if path contains wild card char '?'
-        // the remaining charecters after '?' will be removed while forming URI instance
+        // the remaining characters after '?' will be removed while forming URI instance
         LoadDataCommand.makeQualified(defaultFS, uriPath, loadPath)
       }
     }
@@ -389,8 +390,8 @@ case class LoadDataCommand(
         isSrcLocal = isLocal)
     }
 
-    // Refresh the metadata cache to ensure the data visible to the users
-    catalog.refreshTable(targetTable.identifier)
+    // Refresh the data and metadata cache to ensure the data visible to the users
+    sparkSession.catalog.refreshTable(tableIdentwithDB)
 
     CommandUtils.updateTableStats(sparkSession, targetTable)
     Seq.empty[Row]
@@ -440,7 +441,7 @@ object LoadDataCommand {
  */
 case class TruncateTableCommand(
     tableName: TableIdentifier,
-    partitionSpec: Option[TablePartitionSpec]) extends RunnableCommand {
+    partitionSpec: Option[TablePartitionSpec]) extends LeafRunnableCommand {
 
   override def run(spark: SparkSession): Seq[Row] = {
     val catalog = spark.sessionState.catalog
@@ -451,10 +452,6 @@ case class TruncateTableCommand(
       throw new AnalysisException(
         s"Operation not allowed: TRUNCATE TABLE on external tables: $tableIdentWithDB")
     }
-    if (table.tableType == CatalogTableType.VIEW) {
-      throw new AnalysisException(
-        s"Operation not allowed: TRUNCATE TABLE on views: $tableIdentWithDB")
-    }
     if (table.partitionColumnNames.isEmpty && partitionSpec.isDefined) {
       throw new AnalysisException(
         s"Operation not allowed: TRUNCATE TABLE ... PARTITION is not supported " +
@@ -472,7 +469,7 @@ case class TruncateTableCommand(
         val normalizedSpec = partitionSpec.map { spec =>
           PartitioningUtils.normalizePartitionSpec(
             spec,
-            partCols,
+            table.partitionSchema,
             table.identifier.quotedString,
             spark.sessionState.conf.resolver)
         }
@@ -488,7 +485,7 @@ case class TruncateTableCommand(
         partLocations
       }
     val hadoopConf = spark.sessionState.newHadoopConf()
-    val ignorePermissionAcl = SQLConf.get.truncateTableIgnorePermissionAcl
+    val ignorePermissionAcl = conf.truncateTableIgnorePermissionAcl
     locations.foreach { location =>
       if (location.isDefined) {
         val path = new Path(location.get)
@@ -564,22 +561,11 @@ case class TruncateTableCommand(
         }
       }
     }
-    // After deleting the data, invalidate the table to make sure we don't keep around a stale
-    // file relation in the metastore cache.
-    spark.sessionState.refreshTable(tableName.unquotedString)
-    // Also try to drop the contents of the table from the columnar cache
-    try {
-      spark.sharedState.cacheManager.uncacheQuery(spark.table(table.identifier), cascade = true)
-    } catch {
-      case NonFatal(e) =>
-        log.warn(s"Exception when attempting to uncache table $tableIdentWithDB", e)
-    }
+    // After deleting the data, refresh the table to make sure we don't keep around a stale
+    // file relation in the metastore cache and cached table data in the cache manager.
+    spark.catalog.refreshTable(tableIdentWithDB)
 
-    if (table.stats.nonEmpty) {
-      // empty table after truncation
-      val newStats = CatalogStatistics(sizeInBytes = 0, rowCount = Some(0))
-      catalog.alterTableStats(tableName, Some(newStats))
-    }
+    CommandUtils.updateTableStats(spark, table)
     Seq.empty[Row]
   }
 
@@ -594,9 +580,7 @@ case class TruncateTableCommand(
   }
 }
 
-abstract class DescribeCommandBase extends RunnableCommand {
-  override val output = DescribeTableSchema.describeTableAttributes()
-
+abstract class DescribeCommandBase extends LeafRunnableCommand {
   protected def describeSchema(
       schema: StructType,
       buffer: ArrayBuffer[Row],
@@ -623,21 +607,23 @@ abstract class DescribeCommandBase extends RunnableCommand {
 case class DescribeTableCommand(
     table: TableIdentifier,
     partitionSpec: TablePartitionSpec,
-    isExtended: Boolean)
+    isExtended: Boolean,
+    override val output: Seq[Attribute])
   extends DescribeCommandBase {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val result = new ArrayBuffer[Row]
     val catalog = sparkSession.sessionState.catalog
 
-    if (catalog.isTemporaryTable(table)) {
+    if (catalog.isTempView(table)) {
       if (partitionSpec.nonEmpty) {
         throw new AnalysisException(
           s"DESC PARTITION is not allowed on a temporary view: ${table.identifier}")
       }
-      describeSchema(catalog.lookupRelation(table).schema, result, header = false)
+      val schema = catalog.getTempViewOrPermanentTableMetadata(table).schema
+      describeSchema(schema, result, header = false)
     } else {
-      val metadata = catalog.getTableMetadata(table)
+      val metadata = catalog.getTableRawMetadata(table)
       if (metadata.schema.isEmpty) {
         // In older version(prior to 2.1) of Spark, the table schema can be empty and should be
         // inferred at runtime. We should still support it.
@@ -657,7 +643,7 @@ case class DescribeTableCommand(
       }
     }
 
-    result
+    result.toSeq
   }
 
   private def describePartitionInfo(table: CatalogTable, buffer: ArrayBuffer[Row]): Unit = {
@@ -734,13 +720,15 @@ case class DescribeTableCommand(
 case class DescribeQueryCommand(queryText: String, plan: LogicalPlan)
   extends DescribeCommandBase {
 
+  override val output = DescribeCommandSchema.describeTableAttributes()
+
   override def simpleString(maxFields: Int): String = s"$nodeName $queryText".trim
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val result = new ArrayBuffer[Row]
     val queryExecution = sparkSession.sessionState.executePlan(plan)
     describeSchema(queryExecution.analyzed.schema, result, header = false)
-    result
+    result.toSeq
   }
 }
 
@@ -755,17 +743,10 @@ case class DescribeQueryCommand(queryText: String, plan: LogicalPlan)
 case class DescribeColumnCommand(
     table: TableIdentifier,
     colNameParts: Seq[String],
-    isExtended: Boolean)
-  extends RunnableCommand {
-
-  override val output: Seq[Attribute] = {
-    Seq(
-      AttributeReference("info_name", StringType, nullable = false,
-        new MetadataBuilder().putString("comment", "name of the column info").build())(),
-      AttributeReference("info_value", StringType, nullable = false,
-        new MetadataBuilder().putString("comment", "value of the column info").build())()
-    )
-  }
+    isExtended: Boolean,
+    override val output: Seq[Attribute])
+  extends LeafRunnableCommand {
+
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val catalog = sparkSession.sessionState.catalog
@@ -775,13 +756,13 @@ case class DescribeColumnCommand(
     val colName = UnresolvedAttribute(colNameParts).name
     val field = {
       relation.resolve(colNameParts, resolver).getOrElse {
-        throw new AnalysisException(s"Column $colName does not exist")
+        throw QueryCompilationErrors.columnDoesNotExistError(colName)
       }
     }
     if (!field.isInstanceOf[Attribute]) {
       // If the field is not an attribute after `resolve`, then it's a nested field.
-      throw new AnalysisException(
-        s"DESC TABLE COLUMN command does not support nested data types: $colName")
+      throw QueryCompilationErrors.commandNotSupportNestedColumnError(
+        "DESC TABLE COLUMN", colName)
     }
 
     val catalogTable = catalog.getTempViewOrPermanentTableMetadata(table)
@@ -795,9 +776,11 @@ case class DescribeColumnCommand(
       None
     }
 
+    val dataType = CharVarcharUtils.getRawType(field.metadata)
+      .getOrElse(field.dataType).catalogString
     val buffer = ArrayBuffer[Row](
       Row("col_name", field.name),
-      Row("data_type", field.dataType.catalogString),
+      Row("data_type", dataType),
       Row("comment", comment.getOrElse("NULL"))
     )
     if (isExtended) {
@@ -815,7 +798,7 @@ case class DescribeColumnCommand(
       } yield histogramDescription(hist)
       buffer ++= histDesc.getOrElse(Seq(Row("histogram", "NULL")))
     }
-    buffer
+    buffer.toSeq
   }
 
   private def histogramDescription(histogram: Histogram): Seq[Row] = {
@@ -843,21 +826,9 @@ case class DescribeColumnCommand(
 case class ShowTablesCommand(
     databaseName: Option[String],
     tableIdentifierPattern: Option[String],
+    override val output: Seq[Attribute],
     isExtended: Boolean = false,
-    partitionSpec: Option[TablePartitionSpec] = None) extends RunnableCommand {
-
-  // The result of SHOW TABLES/SHOW TABLE has three basic columns: database, tableName and
-  // isTemporary. If `isExtended` is true, append column `information` to the output columns.
-  override val output: Seq[Attribute] = {
-    val tableExtendedInfo = if (isExtended) {
-      AttributeReference("information", StringType, nullable = false)() :: Nil
-    } else {
-      Nil
-    }
-    AttributeReference("database", StringType, nullable = false)() ::
-      AttributeReference("tableName", StringType, nullable = false)() ::
-      AttributeReference("isTemporary", BooleanType, nullable = false)() :: tableExtendedInfo
-  }
+    partitionSpec: Option[TablePartitionSpec] = None) extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     // Since we need to return a Seq of rows, we will call getTables directly
@@ -871,7 +842,7 @@ case class ShowTablesCommand(
       tables.map { tableIdent =>
         val database = tableIdent.database.getOrElse("")
         val tableName = tableIdent.table
-        val isTemp = catalog.isTemporaryTable(tableIdent)
+        val isTemp = catalog.isTempView(tableIdent)
         if (isExtended) {
           val information = catalog.getTempViewOrPermanentTableMetadata(tableIdent).simpleString
           Row(database, tableName, isTemp, s"$information\n")
@@ -884,12 +855,20 @@ case class ShowTablesCommand(
       //
       // Note: tableIdentifierPattern should be non-empty, otherwise a [[ParseException]]
       // should have been thrown by the sql parser.
-      val tableIdent = TableIdentifier(tableIdentifierPattern.get, Some(db))
-      val table = catalog.getTableMetadata(tableIdent).identifier
-      val partition = catalog.getPartition(tableIdent, partitionSpec.get)
-      val database = table.database.getOrElse("")
-      val tableName = table.table
-      val isTemp = catalog.isTemporaryTable(table)
+      val table = catalog.getTableMetadata(TableIdentifier(tableIdentifierPattern.get, Some(db)))
+
+      DDLUtils.verifyPartitionProviderIsHive(sparkSession, table, "SHOW TABLE EXTENDED")
+
+      val tableIdent = table.identifier
+      val normalizedSpec = PartitioningUtils.normalizePartitionSpec(
+        partitionSpec.get,
+        table.partitionSchema,
+        tableIdent.quotedString,
+        sparkSession.sessionState.conf.resolver)
+      val partition = catalog.getPartition(tableIdent, normalizedSpec)
+      val database = tableIdent.database.getOrElse("")
+      val tableName = tableIdent.table
+      val isTemp = catalog.isTempView(tableIdent)
       val information = partition.simpleString
       Seq(Row(database, tableName, isTemp, s"$information\n"))
     }
@@ -906,20 +885,14 @@ case class ShowTablesCommand(
  *   SHOW TBLPROPERTIES table_name[('propertyKey')];
  * }}}
  */
-case class ShowTablePropertiesCommand(table: TableIdentifier, propertyKey: Option[String])
-  extends RunnableCommand {
-
-  override val output: Seq[Attribute] = {
-    val schema = AttributeReference("value", StringType, nullable = false)() :: Nil
-    propertyKey match {
-      case None => AttributeReference("key", StringType, nullable = false)() :: schema
-      case _ => schema
-    }
-  }
+case class ShowTablePropertiesCommand(
+    table: TableIdentifier,
+    propertyKey: Option[String],
+    override val output: Seq[Attribute]) extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val catalog = sparkSession.sessionState.catalog
-    if (catalog.isTemporaryTable(table)) {
+    if (catalog.isTempView(table)) {
       Seq.empty[Row]
     } else {
       val catalogTable = catalog.getTableMetadata(table)
@@ -928,7 +901,11 @@ case class ShowTablePropertiesCommand(table: TableIdentifier, propertyKey: Optio
           val propValue = catalogTable
             .properties
             .getOrElse(p, s"Table ${catalogTable.qualifiedName} does not have property: $p")
-          Seq(Row(propValue))
+          if (output.length == 1) {
+            Seq(Row(propValue))
+          } else {
+            Seq(Row(p, propValue))
+          }
         case None =>
           catalogTable.properties.map(p => Row(p._1, p._2)).toSeq
       }
@@ -946,10 +923,8 @@ case class ShowTablePropertiesCommand(table: TableIdentifier, propertyKey: Optio
  */
 case class ShowColumnsCommand(
     databaseName: Option[String],
-    tableName: TableIdentifier) extends RunnableCommand {
-  override val output: Seq[Attribute] = {
-    AttributeReference("col_name", StringType, nullable = false)() :: Nil
-  }
+    tableName: TableIdentifier,
+    override val output: Seq[Attribute]) extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val catalog = sparkSession.sessionState.catalog
@@ -979,10 +954,8 @@ case class ShowColumnsCommand(
  */
 case class ShowPartitionsCommand(
     tableName: TableIdentifier,
-    spec: Option[TablePartitionSpec]) extends RunnableCommand {
-  override val output: Seq[Attribute] = {
-    AttributeReference("partition", StringType, nullable = false)() :: Nil
-  }
+    override val output: Seq[Attribute],
+    spec: Option[TablePartitionSpec]) extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val catalog = sparkSession.sessionState.catalog
@@ -993,11 +966,7 @@ case class ShowPartitionsCommand(
      * Validate and throws an [[AnalysisException]] exception under the following conditions:
      * 1. If the table is not partitioned.
      * 2. If it is a datasource table.
-     * 3. If it is a view.
      */
-    if (table.tableType == VIEW) {
-      throw new AnalysisException(s"SHOW PARTITIONS is not allowed on a view: $tableIdentWithDB")
-    }
 
     if (table.partitionColumnNames.isEmpty) {
       throw new AnalysisException(
@@ -1007,20 +976,18 @@ case class ShowPartitionsCommand(
     DDLUtils.verifyPartitionProviderIsHive(sparkSession, table, "SHOW PARTITIONS")
 
     /**
-     * Validate the partitioning spec by making sure all the referenced columns are
+     * Normalizes the partition spec w.r.t the partition columns and case sensitivity settings,
+     * and validates the spec by making sure all the referenced columns are
      * defined as partitioning columns in table definition. An AnalysisException exception is
      * thrown if the partitioning spec is invalid.
      */
-    if (spec.isDefined) {
-      val badColumns = spec.get.keySet.filterNot(table.partitionColumnNames.contains)
-      if (badColumns.nonEmpty) {
-        val badCols = badColumns.mkString("[", ", ", "]")
-        throw new AnalysisException(
-          s"Non-partitioning column(s) $badCols are specified for SHOW PARTITIONS")
-      }
-    }
+    val normalizedSpec = spec.map(partitionSpec => PartitioningUtils.normalizePartitionSpec(
+      partitionSpec,
+      table.partitionSchema,
+      table.identifier.quotedString,
+      sparkSession.sessionState.conf.resolver))
 
-    val partNames = catalog.listPartitionNames(tableName, spec)
+    val partNames = catalog.listPartitionNames(tableName, normalizedSpec)
     partNames.map(Row(_))
   }
 }
@@ -1110,19 +1077,18 @@ trait ShowCreateTableCommandBase {
  *   SHOW CREATE TABLE [db_name.]table_name
  * }}}
  */
-case class ShowCreateTableCommand(table: TableIdentifier)
-    extends RunnableCommand with ShowCreateTableCommandBase {
-  override val output: Seq[Attribute] = Seq(
-    AttributeReference("createtab_stmt", StringType, nullable = false)()
-  )
+case class ShowCreateTableCommand(
+    table: TableIdentifier,
+    override val output: Seq[Attribute])
+    extends LeafRunnableCommand with ShowCreateTableCommandBase {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val catalog = sparkSession.sessionState.catalog
-    if (catalog.isTemporaryTable(table)) {
+    if (catalog.isTempView(table)) {
       throw new AnalysisException(
         s"SHOW CREATE TABLE is not supported on a temporary view: ${table.identifier}")
     } else {
-      val tableMetadata = catalog.getTableMetadata(table)
+      val tableMetadata = catalog.getTableRawMetadata(table)
 
       // TODO: [SPARK-28692] unify this after we unify the
       //  CREATE TABLE syntax for hive serde and data source table.
@@ -1217,7 +1183,7 @@ case class ShowCreateTableCommand(table: TableIdentifier)
     // If it is a Hive table, we already convert its metadata and fill in a provider.
     builder ++= s"USING ${metadata.provider.get}\n"
 
-    val dataSourceOptions = SQLConf.get.redactOptions(metadata.storage.properties).map {
+    val dataSourceOptions = conf.redactOptions(metadata.storage.properties).map {
       case (key, value) => s"${quoteIdentifier(key)} '${escapeSingleQuotedString(value)}'"
     }
 
@@ -1265,15 +1231,14 @@ case class ShowCreateTableCommand(table: TableIdentifier)
  *   SHOW CREATE TABLE table_identifier AS SERDE;
  * }}}
  */
-case class ShowCreateTableAsSerdeCommand(table: TableIdentifier)
-    extends RunnableCommand with ShowCreateTableCommandBase {
-  override val output: Seq[Attribute] = Seq(
-    AttributeReference("createtab_stmt", StringType, nullable = false)()
-  )
+case class ShowCreateTableAsSerdeCommand(
+    table: TableIdentifier,
+    override val output: Seq[Attribute])
+    extends LeafRunnableCommand with ShowCreateTableCommandBase {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val catalog = sparkSession.sessionState.catalog
-    val tableMetadata = catalog.getTableMetadata(table)
+    val tableMetadata = catalog.getTableRawMetadata(table)
 
     val stmt = if (DDLUtils.isDatasourceTable(tableMetadata)) {
       throw new AnalysisException(
@@ -1358,7 +1323,7 @@ case class ShowCreateTableAsSerdeCommand(table: TableIdentifier)
     storage.serde.foreach { serde =>
       builder ++= s"ROW FORMAT SERDE '$serde'\n"
 
-      val serdeProps = SQLConf.get.redactOptions(metadata.storage.properties).map {
+      val serdeProps = conf.redactOptions(metadata.storage.properties).map {
         case (key, value) =>
           s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'"
       }
@@ -1379,3 +1344,22 @@ case class ShowCreateTableAsSerdeCommand(table: TableIdentifier)
     }
   }
 }
+
+/**
+ * A command to refresh all cached entries associated with the table.
+ *
+ * The syntax of using this command in SQL is:
+ * {{{
+ *   REFRESH TABLE [db_name.]table_name
+ * }}}
+ */
+case class RefreshTableCommand(tableIdent: TableIdentifier)
+  extends LeafRunnableCommand {
+
+  override def run(sparkSession: SparkSession): Seq[Row] = {
+    // Refresh the given table's metadata. If this table is cached as an InMemoryRelation,
+    // drop the original cached version and make the new version cached lazily.
+    sparkSession.catalog.refreshTable(tableIdent.quotedString)
+    Seq.empty[Row]
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
index 23f1d6c983413..dd164d48e5e7b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
@@ -19,16 +19,20 @@ package org.apache.spark.sql.execution.command
 
 import scala.collection.mutable
 
+import org.json4s.JsonAST.{JArray, JString}
+import org.json4s.jackson.JsonMethods._
+
+import org.apache.spark.internal.Logging
 import org.apache.spark.sql.{AnalysisException, Row, SparkSession}
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.{GlobalTempView, LocalTempView, PersistedView, UnresolvedFunction, UnresolvedRelation, ViewType}
-import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType, SessionCatalog}
-import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference, SubqueryExpression}
-import org.apache.spark.sql.catalyst.plans.QueryPlan
-import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project, View}
+import org.apache.spark.sql.catalyst.{FunctionIdentifier, SQLConfHelper, TableIdentifier}
+import org.apache.spark.sql.catalyst.analysis.{GlobalTempView, LocalTempView, PersistedView, ViewType}
+import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType, SessionCatalog, TemporaryViewRelation}
+import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, SubqueryExpression, UserDefinedExpression}
+import org.apache.spark.sql.catalyst.plans.logical.{AnalysisOnlyCommand, LogicalPlan, Project, View}
+import org.apache.spark.sql.catalyst.util.CharVarcharUtils
 import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.NamespaceHelper
-import org.apache.spark.sql.internal.StaticSQLConf
-import org.apache.spark.sql.types.{BooleanType, MetadataBuilder, StringType}
+import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
+import org.apache.spark.sql.types.{MetadataBuilder, StructType}
 import org.apache.spark.sql.util.SchemaUtils
 
 /**
@@ -43,13 +47,14 @@ import org.apache.spark.sql.util.SchemaUtils
  * @param properties the properties of this view.
  * @param originalText the original SQL text of this view, can be None if this view is created via
  *                     Dataset API.
- * @param child the logical plan that represents the view; this is used to generate the logical
- *              plan for temporary view and the view schema.
+ * @param plan the logical plan that represents the view; this is used to generate the logical
+ *             plan for temporary view and the view schema.
  * @param allowExisting if true, and if the view already exists, noop; if false, and if the view
  *                already exists, throws analysis exception.
  * @param replace if true, and if the view already exists, updates it; if false, and if the view
  *                already exists, throws analysis exception.
  * @param viewType the expected view type to be created with this command.
+ * @param isAnalyzed whether this command is analyzed or not.
  */
 case class CreateViewCommand(
     name: TableIdentifier,
@@ -57,15 +62,24 @@ case class CreateViewCommand(
     comment: Option[String],
     properties: Map[String, String],
     originalText: Option[String],
-    child: LogicalPlan,
+    plan: LogicalPlan,
     allowExisting: Boolean,
     replace: Boolean,
-    viewType: ViewType)
-  extends RunnableCommand {
+    viewType: ViewType,
+    isAnalyzed: Boolean = false) extends RunnableCommand with AnalysisOnlyCommand {
 
   import ViewHelper._
 
-  override def innerChildren: Seq[QueryPlan[_]] = Seq(child)
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[LogicalPlan]): CreateViewCommand = {
+    assert(!isAnalyzed)
+    copy(plan = newChildren.head)
+  }
+
+  // `plan` needs to be analyzed, but shouldn't be optimized so that caching works correctly.
+  override def childrenToAnalyze: Seq[LogicalPlan] = plan :: Nil
+
+  def markAsAnalyzed(): LogicalPlan = copy(isAnalyzed = true)
 
   if (viewType == PersistedView) {
     require(originalText.isDefined, "'originalText' must be provided to create permanent view")
@@ -91,10 +105,10 @@ case class CreateViewCommand(
   }
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
-    // If the plan cannot be analyzed, throw an exception and don't proceed.
-    val qe = sparkSession.sessionState.executePlan(child)
-    qe.assertAnalyzed()
-    val analyzedPlan = qe.analyzed
+    if (!isAnalyzed) {
+      throw new AnalysisException("The logical plan that represents the view is not analyzed.")
+    }
+    val analyzedPlan = plan
 
     if (userSpecifiedColumns.nonEmpty &&
         userSpecifiedColumns.length != analyzedPlan.output.length) {
@@ -107,24 +121,32 @@ case class CreateViewCommand(
 
     // When creating a permanent view, not allowed to reference temporary objects.
     // This should be called after `qe.assertAnalyzed()` (i.e., `child` can be resolved)
-    verifyTemporaryObjectsNotExists(catalog)
+    verifyTemporaryObjectsNotExists(catalog, isTemporary, name, analyzedPlan)
 
     if (viewType == LocalTempView) {
-      if (replace && catalog.getTempView(name.table).isDefined) {
-        logDebug(s"Try to uncache ${name.quotedString} before replacing.")
-        CommandUtils.uncacheTableOrView(sparkSession, name.quotedString)
-      }
       val aliasedPlan = aliasPlan(sparkSession, analyzedPlan)
-      catalog.createTempView(name.table, aliasedPlan, overrideIfExists = replace)
+      val tableDefinition = createTemporaryViewRelation(
+        name,
+        sparkSession,
+        replace,
+        catalog.getRawTempView,
+        originalText,
+        analyzedPlan,
+        aliasedPlan)
+      catalog.createTempView(name.table, tableDefinition, overrideIfExists = replace)
     } else if (viewType == GlobalTempView) {
-      if (replace && catalog.getGlobalTempView(name.table).isDefined) {
-        val db = sparkSession.sessionState.conf.getConf(StaticSQLConf.GLOBAL_TEMP_DATABASE)
-        val globalTempView = TableIdentifier(name.table, Option(db))
-        logDebug(s"Try to uncache ${globalTempView.quotedString} before replacing.")
-        CommandUtils.uncacheTableOrView(sparkSession, globalTempView.quotedString)
-      }
+      val db = sparkSession.sessionState.conf.getConf(StaticSQLConf.GLOBAL_TEMP_DATABASE)
+      val viewIdent = TableIdentifier(name.table, Option(db))
       val aliasedPlan = aliasPlan(sparkSession, analyzedPlan)
-      catalog.createGlobalTempView(name.table, aliasedPlan, overrideIfExists = replace)
+      val tableDefinition = createTemporaryViewRelation(
+        viewIdent,
+        sparkSession,
+        replace,
+        catalog.getRawGlobalTempView,
+        originalText,
+        analyzedPlan,
+        aliasedPlan)
+      catalog.createGlobalTempView(name.table, tableDefinition, overrideIfExists = replace)
     } else if (catalog.tableExists(name)) {
       val tableMetadata = catalog.getTableMetadata(name)
       if (allowExisting) {
@@ -159,39 +181,6 @@ case class CreateViewCommand(
     Seq.empty[Row]
   }
 
-  /**
-   * Permanent views are not allowed to reference temp objects, including temp function and views
-   */
-  private def verifyTemporaryObjectsNotExists(catalog: SessionCatalog): Unit = {
-    import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-    if (!isTemporary) {
-      // This func traverses the unresolved plan `child`. Below are the reasons:
-      // 1) Analyzer replaces unresolved temporary views by a SubqueryAlias with the corresponding
-      // logical plan. After replacement, it is impossible to detect whether the SubqueryAlias is
-      // added/generated from a temporary view.
-      // 2) The temp functions are represented by multiple classes. Most are inaccessible from this
-      // package (e.g., HiveGenericUDF).
-      def verify(child: LogicalPlan) {
-        child.collect {
-          // Disallow creating permanent views based on temporary views.
-          case UnresolvedRelation(nameParts) if catalog.isTempView(nameParts) =>
-            throw new AnalysisException(s"Not allowed to create a permanent view $name by " +
-              s"referencing a temporary view ${nameParts.quoted}. " +
-              "Please create a temp view instead by CREATE TEMP VIEW")
-          case other if !other.resolved => other.expressions.flatMap(_.collect {
-            // Traverse subquery plan for any unresolved relations.
-            case e: SubqueryExpression => verify(e.plan)
-            // Disallow creating permanent views based on temporary UDFs.
-            case e: UnresolvedFunction if catalog.isTemporaryFunction(e.name) =>
-              throw new AnalysisException(s"Not allowed to create a permanent view $name by " +
-                s"referencing a temporary function `${e.name}`")
-          })
-        }
-      }
-      verify(child)
-    }
-  }
-
   /**
    * If `userSpecifiedColumns` is defined, alias the analyzed plan to the user specified columns,
    * else return the analyzed plan directly.
@@ -220,7 +209,8 @@ case class CreateViewCommand(
       throw new AnalysisException(
         "It is not allowed to create a persisted view from the Dataset API")
     }
-    val aliasedSchema = aliasPlan(session, analyzedPlan).schema
+    val aliasedSchema = CharVarcharUtils.getRawSchema(
+      aliasPlan(session, analyzedPlan).schema)
     val newProperties = generateViewProperties(
       properties, session, analyzedPlan, aliasedSchema.fieldNames)
 
@@ -252,42 +242,64 @@ case class CreateViewCommand(
 case class AlterViewAsCommand(
     name: TableIdentifier,
     originalText: String,
-    query: LogicalPlan) extends RunnableCommand {
+    query: LogicalPlan,
+    isAnalyzed: Boolean = false) extends RunnableCommand with AnalysisOnlyCommand {
 
   import ViewHelper._
 
-  override def innerChildren: Seq[QueryPlan[_]] = Seq(query)
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[LogicalPlan]): AlterViewAsCommand = {
+    assert(!isAnalyzed)
+    copy(query = newChildren.head)
+  }
 
-  override def run(session: SparkSession): Seq[Row] = {
-    // If the plan cannot be analyzed, throw an exception and don't proceed.
-    val qe = session.sessionState.executePlan(query)
-    qe.assertAnalyzed()
-    val analyzedPlan = qe.analyzed
+  override def childrenToAnalyze: Seq[LogicalPlan] = query :: Nil
 
-    if (session.sessionState.catalog.alterTempViewDefinition(name, analyzedPlan)) {
-      // a local/global temp view has been altered, we are done.
+  def markAsAnalyzed(): LogicalPlan = copy(isAnalyzed = true)
+
+  override def run(session: SparkSession): Seq[Row] = {
+    if (session.sessionState.catalog.isTempView(name)) {
+      alterTemporaryView(session, query)
     } else {
-      alterPermanentView(session, analyzedPlan)
+      alterPermanentView(session, query)
     }
-
     Seq.empty[Row]
   }
 
+  private def alterTemporaryView(session: SparkSession, analyzedPlan: LogicalPlan): Unit = {
+    val catalog = session.sessionState.catalog
+    val getRawTempView: String => Option[TemporaryViewRelation] = if (name.database.isEmpty) {
+      catalog.getRawTempView
+    } else {
+      catalog.getRawGlobalTempView
+    }
+    val tableDefinition = createTemporaryViewRelation(
+      name,
+      session,
+      replace = true,
+      getRawTempView,
+      Some(originalText),
+      analyzedPlan,
+      aliasedPlan = analyzedPlan)
+    session.sessionState.catalog.alterTempViewDefinition(name, tableDefinition)
+  }
+
   private def alterPermanentView(session: SparkSession, analyzedPlan: LogicalPlan): Unit = {
     val viewMeta = session.sessionState.catalog.getTableMetadata(name)
-    if (viewMeta.tableType != CatalogTableType.VIEW) {
-      throw new AnalysisException(s"${viewMeta.identifier} is not a view.")
-    }
 
     // Detect cyclic view reference on ALTER VIEW.
     val viewIdent = viewMeta.identifier
     checkCyclicViewReference(analyzedPlan, Seq(viewIdent), viewIdent)
 
+    logDebug(s"Try to uncache ${viewIdent.quotedString} before replacing.")
+    CommandUtils.uncacheTableOrView(session, viewIdent.quotedString)
+
     val newProperties = generateViewProperties(
       viewMeta.properties, session, analyzedPlan, analyzedPlan.schema.fieldNames)
 
+    val newSchema = CharVarcharUtils.getRawSchema(analyzedPlan.schema)
     val updatedViewMeta = viewMeta.copy(
-      schema = analyzedPlan.schema,
+      schema = newSchema,
       properties = newProperties,
       viewOriginalText = Some(originalText),
       viewText = Some(originalText))
@@ -306,13 +318,8 @@ case class AlterViewAsCommand(
  */
 case class ShowViewsCommand(
     databaseName: String,
-    tableIdentifierPattern: Option[String]) extends RunnableCommand {
-
-  // The result of SHOW VIEWS has three basic columns: namespace, viewName and isTemporary.
-  override val output: Seq[Attribute] = Seq(
-    AttributeReference("namespace", StringType, nullable = false)(),
-    AttributeReference("viewName", StringType, nullable = false)(),
-    AttributeReference("isTemporary", BooleanType, nullable = false)())
+    tableIdentifierPattern: Option[String],
+    override val output: Seq[Attribute]) extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     val catalog = sparkSession.sessionState.catalog
@@ -323,14 +330,37 @@ case class ShowViewsCommand(
     views.map { tableIdent =>
       val namespace = tableIdent.database.toArray.quoted
       val tableName = tableIdent.table
-      val isTemp = catalog.isTemporaryTable(tableIdent)
+      val isTemp = catalog.isTempView(tableIdent)
 
       Row(namespace, tableName, isTemp)
     }
   }
 }
 
-object ViewHelper {
+object ViewHelper extends SQLConfHelper with Logging {
+
+  private val configPrefixDenyList = Seq(
+    SQLConf.MAX_NESTED_VIEW_DEPTH.key,
+    "spark.sql.optimizer.",
+    "spark.sql.codegen.",
+    "spark.sql.execution.",
+    "spark.sql.shuffle.",
+    "spark.sql.adaptive.",
+    SQLConf.ADDITIONAL_REMOTE_REPOSITORIES.key)
+
+  private val configAllowList = Seq(
+    SQLConf.DISABLE_HINTS.key
+  )
+
+  /**
+   * Capture view config either of:
+   * 1. exists in allowList
+   * 2. do not exists in denyList
+   */
+  private def shouldCaptureConfig(key: String): Boolean = {
+    configAllowList.exists(prefix => key.equals(prefix)) ||
+      !configPrefixDenyList.exists(prefix => key.startsWith(prefix))
+  }
 
   import CatalogTable._
 
@@ -359,11 +389,65 @@ object ViewHelper {
     }
   }
 
+  /**
+   * Convert the view SQL configs to `properties`.
+   */
+  private def sqlConfigsToProps(conf: SQLConf): Map[String, String] = {
+    val modifiedConfs = conf.getAllConfs.filter { case (k, _) =>
+      conf.isModifiable(k) && shouldCaptureConfig(k)
+    }
+    val props = new mutable.HashMap[String, String]
+    for ((key, value) <- modifiedConfs) {
+      props.put(s"$VIEW_SQL_CONFIG_PREFIX$key", value)
+    }
+    props.toMap
+  }
+
+  /**
+   * Remove the view SQL configs in `properties`.
+   */
+  private def removeSQLConfigs(properties: Map[String, String]): Map[String, String] = {
+    // We can't use `filterKeys` here, as the map returned by `filterKeys` is not serializable,
+    // while `CatalogTable` should be serializable.
+    properties.filterNot { case (key, _) =>
+      key.startsWith(VIEW_SQL_CONFIG_PREFIX)
+    }
+  }
+
+  /**
+   * Convert the temporary object names to `properties`.
+   */
+  private def referredTempNamesToProps(
+      viewNames: Seq[Seq[String]], functionsNames: Seq[String]): Map[String, String] = {
+    val viewNamesJson =
+      JArray(viewNames.map(nameParts => JArray(nameParts.map(JString).toList)).toList)
+    val functionsNamesJson = JArray(functionsNames.map(JString).toList)
+
+    val props = new mutable.HashMap[String, String]
+    props.put(VIEW_REFERRED_TEMP_VIEW_NAMES, compact(render(viewNamesJson)))
+    props.put(VIEW_REFERRED_TEMP_FUNCTION_NAMES, compact(render(functionsNamesJson)))
+    props.toMap
+  }
+
+  /**
+   * Remove the temporary object names in `properties`.
+   */
+  private def removeReferredTempNames(properties: Map[String, String]): Map[String, String] = {
+    // We can't use `filterKeys` here, as the map returned by `filterKeys` is not serializable,
+    // while `CatalogTable` should be serializable.
+    properties.filterNot { case (key, _) =>
+      key.startsWith(VIEW_REFERRED_TEMP_VIEW_NAMES) ||
+        key.startsWith(VIEW_REFERRED_TEMP_FUNCTION_NAMES)
+    }
+  }
+
+
   /**
    * Generate the view properties in CatalogTable, including:
    * 1. view default database that is used to provide the default database name on view resolution.
    * 2. the output column names of the query that creates a view, this is used to map the output of
    *    the view child to the view output during view resolution.
+   * 3. the SQL configs when creating the view.
    *
    * @param properties the `properties` in CatalogTable.
    * @param session the spark session.
@@ -374,20 +458,26 @@ object ViewHelper {
       properties: Map[String, String],
       session: SparkSession,
       analyzedPlan: LogicalPlan,
-      fieldNames: Array[String]): Map[String, String] = {
+      fieldNames: Array[String],
+      tempViewNames: Seq[Seq[String]] = Seq.empty,
+      tempFunctionNames: Seq[String] = Seq.empty): Map[String, String] = {
     // for createViewCommand queryOutput may be different from fieldNames
     val queryOutput = analyzedPlan.schema.fieldNames
 
+    val conf = session.sessionState.conf
+
     // Generate the query column names, throw an AnalysisException if there exists duplicate column
     // names.
     SchemaUtils.checkColumnNameDuplication(
-      fieldNames, "in the view definition", session.sessionState.conf.resolver)
+      fieldNames, "in the view definition", conf.resolver)
 
-    // Generate the view default catalog and namespace.
+    // Generate the view default catalog and namespace, as well as captured SQL configs.
     val manager = session.sessionState.catalogManager
-    removeQueryColumnNames(properties) ++
+    removeReferredTempNames(removeSQLConfigs(removeQueryColumnNames(properties))) ++
       catalogAndNamespaceToProps(manager.currentCatalog.name, manager.currentNamespace) ++
-      generateQueryColumnNames(queryOutput)
+      sqlConfigsToProps(conf) ++
+      generateQueryColumnNames(queryOutput) ++
+      referredTempNamesToProps(tempViewNames, tempFunctionNames)
   }
 
   /**
@@ -438,4 +528,165 @@ object ViewHelper {
       }
     }
   }
+
+
+  /**
+   * Permanent views are not allowed to reference temp objects, including temp function and views
+   */
+  def verifyTemporaryObjectsNotExists(
+      catalog: SessionCatalog,
+      isTemporary: Boolean,
+      name: TableIdentifier,
+      child: LogicalPlan): Unit = {
+    import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+    if (!isTemporary) {
+      val (tempViews, tempFunctions) = collectTemporaryObjects(catalog, child)
+      tempViews.foreach { nameParts =>
+        throw new AnalysisException(s"Not allowed to create a permanent view $name by " +
+          s"referencing a temporary view ${nameParts.quoted}. " +
+          "Please create a temp view instead by CREATE TEMP VIEW")
+      }
+      tempFunctions.foreach { funcName =>
+        throw new AnalysisException(s"Not allowed to create a permanent view $name by " +
+          s"referencing a temporary function `${funcName}`")
+      }
+    }
+  }
+
+  /**
+   * Collect all temporary views and functions and return the identifiers separately.
+   */
+  private def collectTemporaryObjects(
+      catalog: SessionCatalog, child: LogicalPlan): (Seq[Seq[String]], Seq[String]) = {
+    def collectTempViews(child: LogicalPlan): Seq[Seq[String]] = {
+      child.flatMap {
+        case view: View if view.isTempView =>
+          val ident = view.desc.identifier
+          Seq(ident.database.toSeq :+ ident.table)
+        case plan => plan.expressions.flatMap(_.flatMap {
+          case e: SubqueryExpression => collectTempViews(e.plan)
+          case _ => Seq.empty
+        })
+      }.distinct
+    }
+
+    def collectTempFunctions(child: LogicalPlan): Seq[String] = {
+      child.flatMap {
+        case plan =>
+          plan.expressions.flatMap(_.flatMap {
+            case e: SubqueryExpression => collectTempFunctions(e.plan)
+            case e: UserDefinedExpression
+                if catalog.isTemporaryFunction(FunctionIdentifier(e.name)) =>
+              Seq(e.name)
+            case _ => Seq.empty
+          })
+      }.distinct
+    }
+    (collectTempViews(child), collectTempFunctions(child))
+  }
+
+  /**
+   * Returns a [[TemporaryViewRelation]] that contains information about a temporary view
+   * to create, given an analyzed plan of the view. If a temp view is to be replaced and it is
+   * cached, it will be uncached before being replaced.
+   *
+   * @param name the name of the temporary view to create/replace.
+   * @param session the spark session.
+   * @param replace if true and the existing view is cached, it will be uncached.
+   * @param getRawTempView the function that returns an optional raw plan of the local or
+   *                       global temporary view.
+   * @param originalText the original SQL text of this view, can be None if this view is created via
+   *                     Dataset API or spark.sql.legacy.storeAnalyzedPlanForView is set to true.
+   * @param analyzedPlan the logical plan that represents the view; this is used to generate the
+   *                     logical plan for temporary view and the view schema.
+   * @param aliasedPlan the aliased logical plan based on the user specified columns. If there are
+   *                    no user specified plans, this should be same as `analyzedPlan`.
+   */
+  def createTemporaryViewRelation(
+      name: TableIdentifier,
+      session: SparkSession,
+      replace: Boolean,
+      getRawTempView: String => Option[TemporaryViewRelation],
+      originalText: Option[String],
+      analyzedPlan: LogicalPlan,
+      aliasedPlan: LogicalPlan): TemporaryViewRelation = {
+    val uncache = getRawTempView(name.table).map { r =>
+      needsToUncache(r, aliasedPlan)
+    }.getOrElse(false)
+    if (replace && uncache) {
+      logDebug(s"Try to uncache ${name.quotedString} before replacing.")
+      checkCyclicViewReference(analyzedPlan, Seq(name), name)
+      CommandUtils.uncacheTableOrView(session, name.quotedString)
+    }
+    if (!conf.storeAnalyzedPlanForView && originalText.nonEmpty) {
+      TemporaryViewRelation(
+        prepareTemporaryView(
+          name,
+          session,
+          analyzedPlan,
+          aliasedPlan.schema,
+          originalText.get))
+    } else {
+      TemporaryViewRelation(
+        prepareTemporaryViewStoringAnalyzedPlan(name, aliasedPlan),
+        Some(aliasedPlan))
+    }
+  }
+
+  /**
+   * Checks if need to uncache the temp view being replaced.
+   */
+  private def needsToUncache(
+      rawTempView: TemporaryViewRelation,
+      aliasedPlan: LogicalPlan): Boolean = rawTempView.plan match {
+    // Do not need to uncache if the to-be-replaced temp view plan and the new plan are the
+    // same-result plans.
+    case Some(p) => !p.sameResult(aliasedPlan)
+    // If TemporaryViewRelation doesn't store the analyzed view, always uncache.
+    case None => true
+  }
+
+  /**
+   * Returns a [[CatalogTable]] that contains information for temporary view.
+   * Generate the view-specific properties(e.g. view default database, view query output
+   * column names) and store them as properties in the CatalogTable, and also creates
+   * the proper schema for the view.
+   */
+  private def prepareTemporaryView(
+      viewName: TableIdentifier,
+      session: SparkSession,
+      analyzedPlan: LogicalPlan,
+      viewSchema: StructType,
+      originalText: String): CatalogTable = {
+
+    val catalog = session.sessionState.catalog
+    val (tempViews, tempFunctions) = collectTemporaryObjects(catalog, analyzedPlan)
+    // TBLPROPERTIES is not allowed for temporary view, so we don't use it for
+    // generating temporary view properties
+    val newProperties = generateViewProperties(
+      Map.empty, session, analyzedPlan, viewSchema.fieldNames, tempViews, tempFunctions)
+
+    CatalogTable(
+      identifier = viewName,
+      tableType = CatalogTableType.VIEW,
+      storage = CatalogStorageFormat.empty,
+      schema = viewSchema,
+      viewText = Some(originalText),
+      properties = newProperties)
+  }
+
+  /**
+   * Returns a [[CatalogTable]] that contains information for the temporary view storing
+   * an analyzed plan.
+   */
+  private def prepareTemporaryViewStoringAnalyzedPlan(
+      viewName: TableIdentifier,
+      analyzedPlan: LogicalPlan): CatalogTable = {
+    CatalogTable(
+      identifier = viewName,
+      tableType = CatalogTableType.VIEW,
+      storage = CatalogStorageFormat.empty,
+      schema = analyzedPlan.schema,
+      properties = Map((VIEW_STORING_ANALYZED_PLAN, "true")))
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/BasicWriteStatsTracker.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/BasicWriteStatsTracker.scala
index b71c2d12f02b8..160ee6dc8d555 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/BasicWriteStatsTracker.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/BasicWriteStatsTracker.scala
@@ -18,9 +18,12 @@
 package org.apache.spark.sql.execution.datasources
 
 import java.io.FileNotFoundException
+import java.nio.charset.StandardCharsets
+
+import scala.collection.mutable
 
 import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
+import org.apache.hadoop.fs.{FileSystem, Path}
 
 import org.apache.spark.{SparkContext, TaskContext}
 import org.apache.spark.internal.Logging
@@ -35,7 +38,7 @@ import org.apache.spark.util.SerializableConfiguration
  * These were first introduced in https://github.com/apache/spark/pull/18159 (SPARK-20703).
  */
 case class BasicWriteTaskStats(
-    numPartitions: Int,
+    partitions: Seq[InternalRow],
     numFiles: Int,
     numBytes: Long,
     numRows: Long)
@@ -48,13 +51,13 @@ case class BasicWriteTaskStats(
 class BasicWriteTaskStatsTracker(hadoopConf: Configuration)
   extends WriteTaskStatsTracker with Logging {
 
-  private[this] var numPartitions: Int = 0
+  private[this] val partitions: mutable.ArrayBuffer[InternalRow] = mutable.ArrayBuffer.empty
   private[this] var numFiles: Int = 0
-  private[this] var submittedFiles: Int = 0
+  private[this] var numSubmittedFiles: Int = 0
   private[this] var numBytes: Long = 0L
   private[this] var numRows: Long = 0L
 
-  private[this] var curFile: Option[String] = None
+  private[this] val submittedFiles = mutable.HashSet[String]()
 
   /**
    * Get the size of the file expected to have been written by a worker.
@@ -64,47 +67,97 @@ class BasicWriteTaskStatsTracker(hadoopConf: Configuration)
   private def getFileSize(filePath: String): Option[Long] = {
     val path = new Path(filePath)
     val fs = path.getFileSystem(hadoopConf)
+    getFileSize(fs, path)
+  }
+
+  /**
+   * Get the size of the file expected to have been written by a worker.
+   * This supports the XAttr in HADOOP-17414 when the "magic committer" adds
+   * a custom HTTP header to the a zero byte marker.
+   * If the output file as returned by getFileStatus > 0 then the length if
+   * returned. For zero-byte files, the (optional) Hadoop FS API getXAttr() is
+   * invoked. If a parseable, non-negative length can be retrieved, this
+   * is returned instead of the length.
+   * @return the file size or None if the file was not found.
+   */
+  private [datasources] def getFileSize(fs: FileSystem, path: Path): Option[Long] = {
+    // the normal file status probe.
     try {
-      Some(fs.getFileStatus(path).getLen())
+      val len = fs.getFileStatus(path).getLen
+      if (len > 0) {
+        return Some(len)
+      }
     } catch {
       case e: FileNotFoundException =>
-        // may arise against eventually consistent object stores
+        // may arise against eventually consistent object stores.
         logDebug(s"File $path is not yet visible", e)
-        None
+        return None
     }
+
+    // Output File Size is 0. Look to see if it has an attribute
+    // declaring a future-file-length.
+    // Failure of API call, parsing, invalid value all return the
+    // 0 byte length.
+
+    var len = 0L
+    try {
+      val attr = fs.getXAttr(path, BasicWriteJobStatsTracker.FILE_LENGTH_XATTR)
+      if (attr != null && attr.nonEmpty) {
+        val str = new String(attr, StandardCharsets.UTF_8)
+        logDebug(s"File Length statistics for $path retrieved from XAttr: $str")
+        // a non-empty header was found. parse to a long via the java class
+        val l = java.lang.Long.parseLong(str)
+        if (l > 0) {
+          len = l
+        } else {
+          logDebug("Ignoring negative value in XAttr file length")
+        }
+      }
+    } catch {
+      case e: NumberFormatException =>
+        // warn but don't dump the whole stack
+        logInfo(s"Failed to parse" +
+          s" ${BasicWriteJobStatsTracker.FILE_LENGTH_XATTR}:$e;" +
+          s" bytes written may be under-reported");
+      case e: UnsupportedOperationException =>
+        // this is not unusual; ignore
+        logDebug(s"XAttr not supported on path $path", e);
+      case e: Exception =>
+        // Something else. Log at debug and continue.
+        logDebug(s"XAttr processing failure on $path", e);
+    }
+    Some(len)
   }
 
 
   override def newPartition(partitionValues: InternalRow): Unit = {
-    numPartitions += 1
+    partitions.append(partitionValues)
   }
 
-  override def newBucket(bucketId: Int): Unit = {
-    // currently unhandled
+  override def newFile(filePath: String): Unit = {
+    submittedFiles += filePath
+    numSubmittedFiles += 1
   }
 
-  override def newFile(filePath: String): Unit = {
-    statCurrentFile()
-    curFile = Some(filePath)
-    submittedFiles += 1
+  override def closeFile(filePath: String): Unit = {
+    updateFileStats(filePath)
+    submittedFiles.remove(filePath)
   }
 
-  private def statCurrentFile(): Unit = {
-    curFile.foreach { path =>
-      getFileSize(path).foreach { len =>
-        numBytes += len
-        numFiles += 1
-      }
-      curFile = None
+  private def updateFileStats(filePath: String): Unit = {
+    getFileSize(filePath).foreach { len =>
+      numBytes += len
+      numFiles += 1
     }
   }
 
-  override def newRow(row: InternalRow): Unit = {
+  override def newRow(filePath: String, row: InternalRow): Unit = {
     numRows += 1
   }
 
   override def getFinalStats(): WriteTaskStats = {
-    statCurrentFile()
+    submittedFiles.foreach(updateFileStats)
+    submittedFiles.clear()
 
     // Reports bytesWritten and recordsWritten to the Spark output metrics.
     Option(TaskContext.get()).map(_.taskMetrics().outputMetrics).foreach { outputMetrics =>
@@ -112,12 +165,12 @@ class BasicWriteTaskStatsTracker(hadoopConf: Configuration)
       outputMetrics.setRecordsWritten(numRows)
     }
 
-    if (submittedFiles != numFiles) {
-      logInfo(s"Expected $submittedFiles files, but only saw $numFiles. " +
+    if (numSubmittedFiles != numFiles) {
+      logInfo(s"Expected $numSubmittedFiles files, but only saw $numFiles. " +
         "This could be due to the output format not writing empty files, " +
         "or files being not immediately visible in the filesystem.")
     }
-    BasicWriteTaskStats(numPartitions, numFiles, numBytes, numRows)
+    BasicWriteTaskStats(partitions.toSeq, numFiles, numBytes, numRows)
   }
 }
 
@@ -139,7 +192,7 @@ class BasicWriteJobStatsTracker(
 
   override def processStats(stats: Seq[WriteTaskStats]): Unit = {
     val sparkContext = SparkContext.getActive.get
-    var numPartitions: Long = 0L
+    var partitionsSet: mutable.Set[InternalRow] = mutable.HashSet.empty
     var numFiles: Long = 0L
     var totalNumBytes: Long = 0L
     var totalNumOutput: Long = 0L
@@ -147,7 +200,7 @@ class BasicWriteJobStatsTracker(
     val basicStats = stats.map(_.asInstanceOf[BasicWriteTaskStats])
 
     basicStats.foreach { summary =>
-      numPartitions += summary.numPartitions
+      partitionsSet ++= summary.partitions
       numFiles += summary.numFiles
       totalNumBytes += summary.numBytes
       totalNumOutput += summary.numRows
@@ -156,7 +209,7 @@ class BasicWriteJobStatsTracker(
     metrics(BasicWriteJobStatsTracker.NUM_FILES_KEY).add(numFiles)
     metrics(BasicWriteJobStatsTracker.NUM_OUTPUT_BYTES_KEY).add(totalNumBytes)
     metrics(BasicWriteJobStatsTracker.NUM_OUTPUT_ROWS_KEY).add(totalNumOutput)
-    metrics(BasicWriteJobStatsTracker.NUM_PARTS_KEY).add(numPartitions)
+    metrics(BasicWriteJobStatsTracker.NUM_PARTS_KEY).add(partitionsSet.size)
 
     val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY)
     SQLMetrics.postDriverMetricUpdates(sparkContext, executionId, metrics.values.toList)
@@ -168,6 +221,8 @@ object BasicWriteJobStatsTracker {
   private val NUM_OUTPUT_BYTES_KEY = "numOutputBytes"
   private val NUM_OUTPUT_ROWS_KEY = "numOutputRows"
   private val NUM_PARTS_KEY = "numParts"
+  /** XAttr key of the data length header added in HADOOP-17414. */
+  val FILE_LENGTH_XATTR = "header.x-hadoop-s3a-magic-data-length"
 
   def metrics: Map[String, SQLMetric] = {
     val sparkContext = SparkContext.getActive.get
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala
index 222fea6528261..b3de98dd0dd0c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala
@@ -20,7 +20,6 @@ package org.apache.spark.sql.execution.datasources
 import java.util.{Locale, ServiceConfigurationError, ServiceLoader}
 
 import scala.collection.JavaConverters._
-import scala.language.implicitConversions
 import scala.util.{Failure, Success, Try}
 
 import org.apache.hadoop.conf.Configuration
@@ -33,8 +32,9 @@ import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
 import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStorageFormat, CatalogTable, CatalogUtils}
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
+import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, TypeUtils}
 import org.apache.spark.sql.connector.catalog.TableProvider
+import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
 import org.apache.spark.sql.execution.SparkPlan
 import org.apache.spark.sql.execution.command.DataWritingCommand
 import org.apache.spark.sql.execution.datasources.csv.CSVFileFormat
@@ -44,14 +44,15 @@ import org.apache.spark.sql.execution.datasources.orc.OrcFileFormat
 import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
 import org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2
 import org.apache.spark.sql.execution.datasources.v2.orc.OrcDataSourceV2
+import org.apache.spark.sql.execution.metric.SQLMetric
 import org.apache.spark.sql.execution.streaming._
 import org.apache.spark.sql.execution.streaming.sources.{RateStreamProvider, TextSocketSourceProvider}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.sources._
 import org.apache.spark.sql.streaming.OutputMode
-import org.apache.spark.sql.types.{CalendarIntervalType, StructField, StructType}
+import org.apache.spark.sql.types.{DataType, StructField, StructType}
 import org.apache.spark.sql.util.SchemaUtils
-import org.apache.spark.util.{ThreadUtils, Utils}
+import org.apache.spark.util.{HadoopFSUtils, ThreadUtils, Utils}
 
 /**
  * The main class responsible for representing a pluggable Data Source in Spark SQL. In addition to
@@ -68,8 +69,9 @@ import org.apache.spark.util.{ThreadUtils, Utils}
  * metadata.  For example, when reading a partitioned table from a file system, partition columns
  * will be inferred from the directory layout even if they are not specified.
  *
- * @param paths A list of file system paths that hold data.  These will be globbed before and
- *              qualified. This option only works when reading from a [[FileFormat]].
+ * @param paths A list of file system paths that hold data. These will be globbed before if
+ *              the "__globPaths__" option is true, and will be qualified. This option only works
+ *              when reading from a [[FileFormat]].
  * @param userSpecifiedSchema An optional specification of the schema of the data. When present
  *                            we skip attempting to infer the schema.
  * @param partitionColumns A list of column names that the relation is partitioned by. This list is
@@ -110,10 +112,22 @@ case class DataSource(
 
   private def providingInstance() = providingClass.getConstructor().newInstance()
 
+  private def newHadoopConfiguration(): Configuration =
+    sparkSession.sessionState.newHadoopConfWithOptions(options)
+
   lazy val sourceInfo: SourceInfo = sourceSchema()
   private val caseInsensitiveOptions = CaseInsensitiveMap(options)
   private val equality = sparkSession.sessionState.conf.resolver
 
+  /**
+   * Whether or not paths should be globbed before being used to access files.
+   */
+  def globPaths: Boolean = {
+    options.get(DataSource.GLOB_PATHS_KEY)
+      .map(_ == "true")
+      .getOrElse(true)
+  }
+
   bucketSpec.map { bucket =>
     SchemaUtils.checkColumnNameDuplication(
       bucket.bucketColumnNames, "in the bucket definition", equality)
@@ -177,8 +191,8 @@ case class DataSource(
             }
             inferredOpt
           }.getOrElse {
-            throw new AnalysisException(s"Failed to resolve the schema for $format for " +
-              s"the partition column: $partitionColumn. It must be specified manually.")
+            throw QueryCompilationErrors.partitionColumnNotSpecifiedError(
+              format.toString, partitionColumn)
           }
         }
         StructType(partitionFields)
@@ -188,16 +202,17 @@ case class DataSource(
     val dataSchema = userSpecifiedSchema.map { schema =>
       StructType(schema.filterNot(f => partitionSchema.exists(p => equality(p.name, f.name))))
     }.orElse {
+      // Remove "path" option so that it is not added to the paths returned by
+      // `tempFileIndex.allFiles()`.
       format.inferSchema(
         sparkSession,
-        caseInsensitiveOptions,
+        caseInsensitiveOptions - "path",
         tempFileIndex.allFiles())
     }.getOrElse {
-      throw new AnalysisException(
-        s"Unable to infer schema for $format. It must be specified manually.")
+      throw QueryCompilationErrors.dataSchemaNotSpecifiedError(format.toString)
     }
 
-    // We just print a waring message if the data schema and partition schema have the duplicate
+    // We just print a warning message if the data schema and partition schema have the duplicate
     // columns. This is because we allow users to do so in the previous Spark releases and
     // we have the existing tests for the cases (e.g., `ParquetHadoopFsRelationSuite`).
     // See SPARK-18108 and SPARK-21144 for related discussions.
@@ -223,17 +238,17 @@ case class DataSource(
 
       case format: FileFormat =>
         val path = caseInsensitiveOptions.getOrElse("path", {
-          throw new IllegalArgumentException("'path' is not specified")
+          throw QueryExecutionErrors.dataPathNotSpecifiedError()
         })
 
         // Check whether the path exists if it is not a glob pattern.
         // For glob pattern, we do not check it because the glob pattern might only make sense
         // once the streaming job starts and some upstream source starts dropping data.
         val hdfsPath = new Path(path)
-        if (!SparkHadoopUtil.get.isGlobPath(hdfsPath)) {
-          val fs = hdfsPath.getFileSystem(sparkSession.sessionState.newHadoopConf())
+        if (!globPaths || !SparkHadoopUtil.get.isGlobPath(hdfsPath)) {
+          val fs = hdfsPath.getFileSystem(newHadoopConfiguration())
           if (!fs.exists(hdfsPath)) {
-            throw new AnalysisException(s"Path does not exist: $path")
+            throw QueryCompilationErrors.dataPathNotExistError(path)
           }
         }
 
@@ -241,11 +256,7 @@ case class DataSource(
         val isTextSource = providingClass == classOf[text.TextFileFormat]
         // If the schema inference is disabled, only text sources require schema to be specified
         if (!isSchemaInferenceEnabled && !isTextSource && userSpecifiedSchema.isEmpty) {
-          throw new IllegalArgumentException(
-            "Schema must be specified when creating a streaming source DataFrame. " +
-              "If some files already exist in the directory, then depending on the file format " +
-              "you may be able to create a static DataFrame on that directory with " +
-              "'spark.read.load(directory)' and infer schema from it.")
+          throw QueryExecutionErrors.createStreamingSourceNotSpecifySchemaError()
         }
 
         val (dataSchema, partitionSchema) = getOrInferFileFormatSchema(format, () => {
@@ -265,8 +276,8 @@ case class DataSource(
           partitionSchema.fieldNames)
 
       case _ =>
-        throw new UnsupportedOperationException(
-          s"Data source $className does not support streamed reading")
+        throw QueryExecutionErrors.streamedOperatorUnsupportedByDataSourceError(
+          className, "reading")
     }
   }
 
@@ -283,7 +294,7 @@ case class DataSource(
 
       case format: FileFormat =>
         val path = caseInsensitiveOptions.getOrElse("path", {
-          throw new IllegalArgumentException("'path' is not specified")
+          throw QueryExecutionErrors.dataPathNotSpecifiedError()
         })
         new FileStreamSource(
           sparkSession = sparkSession,
@@ -294,8 +305,8 @@ case class DataSource(
           metadataPath = metadataPath,
           options = caseInsensitiveOptions)
       case _ =>
-        throw new UnsupportedOperationException(
-          s"Data source $className does not support streamed reading")
+        throw QueryExecutionErrors.streamedOperatorUnsupportedByDataSourceError(
+          className, "reading")
     }
   }
 
@@ -307,17 +318,16 @@ case class DataSource(
 
       case fileFormat: FileFormat =>
         val path = caseInsensitiveOptions.getOrElse("path", {
-          throw new IllegalArgumentException("'path' is not specified")
+          throw QueryExecutionErrors.dataPathNotSpecifiedError()
         })
         if (outputMode != OutputMode.Append) {
-          throw new AnalysisException(
-            s"Data source $className does not support $outputMode output mode")
+          throw QueryCompilationErrors.dataSourceOutputModeUnsupportedError(className, outputMode)
         }
         new FileStreamSink(sparkSession, path, fileFormat, partitionColumns, caseInsensitiveOptions)
 
       case _ =>
-        throw new UnsupportedOperationException(
-          s"Data source $className does not support streamed writing")
+        throw QueryExecutionErrors.streamedOperatorUnsupportedByDataSourceError(
+          className, "writing")
     }
   }
 
@@ -339,17 +349,13 @@ case class DataSource(
       case (dataSource: RelationProvider, None) =>
         dataSource.createRelation(sparkSession.sqlContext, caseInsensitiveOptions)
       case (_: SchemaRelationProvider, None) =>
-        throw new AnalysisException(s"A schema needs to be specified when using $className.")
+        throw QueryCompilationErrors.schemaNotSpecifiedForSchemaRelationProviderError(className)
       case (dataSource: RelationProvider, Some(schema)) =>
         val baseRelation =
           dataSource.createRelation(sparkSession.sqlContext, caseInsensitiveOptions)
         if (baseRelation.schema != schema) {
-          throw new AnalysisException(
-            "The user-specified schema doesn't match the actual schema: " +
-            s"user-specified: ${schema.toDDL}, actual: ${baseRelation.schema.toDDL}. If " +
-            "you're using DataFrameReader.schema API or creating a table, please do not " +
-            "specify the schema. Or if you're scanning an existed table, please drop " +
-            "it and re-create it.")
+          throw QueryCompilationErrors.userSpecifiedSchemaMismatchActualSchemaError(
+            schema, baseRelation.schema)
         }
         baseRelation
 
@@ -358,20 +364,21 @@ case class DataSource(
       case (format: FileFormat, _)
           if FileStreamSink.hasMetadata(
             caseInsensitiveOptions.get("path").toSeq ++ paths,
-            sparkSession.sessionState.newHadoopConf(),
+            newHadoopConfiguration(),
             sparkSession.sessionState.conf) =>
         val basePath = new Path((caseInsensitiveOptions.get("path").toSeq ++ paths).head)
         val fileCatalog = new MetadataLogFileIndex(sparkSession, basePath,
           caseInsensitiveOptions, userSpecifiedSchema)
         val dataSchema = userSpecifiedSchema.orElse {
+          // Remove "path" option so that it is not added to the paths returned by
+          // `fileCatalog.allFiles()`.
           format.inferSchema(
             sparkSession,
-            caseInsensitiveOptions,
+            caseInsensitiveOptions - "path",
             fileCatalog.allFiles())
         }.getOrElse {
-          throw new AnalysisException(
-            s"Unable to infer schema for $format at ${fileCatalog.allFiles().mkString(",")}. " +
-                "It must be specified manually")
+          throw QueryCompilationErrors.dataSchemaNotSpecifiedError(
+            format.toString, fileCatalog.allFiles().mkString(","))
         }
 
         HadoopFsRelation(
@@ -412,24 +419,23 @@ case class DataSource(
           caseInsensitiveOptions)(sparkSession)
 
       case _ =>
-        throw new AnalysisException(
-          s"$className is not a valid Spark SQL Data Source.")
+        throw QueryCompilationErrors.invalidDataSourceError(className)
     }
 
     relation match {
       case hs: HadoopFsRelation =>
-        SchemaUtils.checkColumnNameDuplication(
-          hs.dataSchema.map(_.name),
+        SchemaUtils.checkSchemaColumnNameDuplication(
+          hs.dataSchema,
           "in the data schema",
           equality)
-        SchemaUtils.checkColumnNameDuplication(
-          hs.partitionSchema.map(_.name),
+        SchemaUtils.checkSchemaColumnNameDuplication(
+          hs.partitionSchema,
           "in the partition schema",
           equality)
         DataSourceUtils.verifySchema(hs.fileFormat, hs.dataSchema)
       case _ =>
-        SchemaUtils.checkColumnNameDuplication(
-          relation.schema.map(_.name),
+        SchemaUtils.checkSchemaColumnNameDuplication(
+          relation.schema,
           "in the data schema",
           equality)
     }
@@ -450,11 +456,10 @@ case class DataSource(
     val allPaths = paths ++ caseInsensitiveOptions.get("path")
     val outputPath = if (allPaths.length == 1) {
       val path = new Path(allPaths.head)
-      val fs = path.getFileSystem(sparkSession.sessionState.newHadoopConf())
+      val fs = path.getFileSystem(newHadoopConfiguration())
       path.makeQualified(fs.getUri, fs.getWorkingDirectory)
     } else {
-      throw new IllegalArgumentException("Expected exactly one path to be specified, but " +
-        s"got: ${allPaths.mkString(", ")}")
+      throw QueryExecutionErrors.multiplePathsSpecifiedError(allPaths)
     }
 
     val caseSensitive = sparkSession.sessionState.conf.caseSensitiveAnalysis
@@ -502,12 +507,10 @@ case class DataSource(
       mode: SaveMode,
       data: LogicalPlan,
       outputColumnNames: Seq[String],
-      physicalPlan: SparkPlan): BaseRelation = {
+      physicalPlan: SparkPlan,
+      metrics: Map[String, SQLMetric]): BaseRelation = {
     val outputColumns = DataWritingCommand.logicalPlanOutputWithNames(data, outputColumnNames)
-    if (outputColumns.map(_.dataType).exists(_.isInstanceOf[CalendarIntervalType])) {
-      throw new AnalysisException("Cannot save interval data type into external storage.")
-    }
-
+    disallowWritingIntervals(outputColumns.map(_.dataType))
     providingInstance() match {
       case dataSource: CreatableRelationProvider =>
         dataSource.createRelation(
@@ -522,14 +525,14 @@ case class DataSource(
           assert(unresolved.nameParts.length == 1)
           val name = unresolved.nameParts.head
           outputColumns.find(a => equality(a.name, name)).getOrElse {
-            throw new AnalysisException(
-              s"Unable to resolve $name given [${data.output.map(_.name).mkString(", ")}]")
+            throw QueryCompilationErrors.cannotResolveAttributeError(name, data)
           }
         }
         val resolved = cmd.copy(
           partitionColumns = resolvedPartCols,
           outputColumnNames = outputColumnNames)
         resolved.run(sparkSession, physicalPlan)
+        DataWritingCommand.propogateMetrics(sparkSession.sparkContext, resolved, metrics)
         // Replace the schema with that of the DataFrame we just wrote out to avoid re-inferring
         copy(userSpecifiedSchema = Some(outputColumns.toStructType.asNullable)).resolveRelation()
       case _ =>
@@ -541,10 +544,7 @@ case class DataSource(
    * Returns a logical plan to write the given [[LogicalPlan]] out to this [[DataSource]].
    */
   def planForWriting(mode: SaveMode, data: LogicalPlan): LogicalPlan = {
-    if (data.schema.map(_.dataType).exists(_.isInstanceOf[CalendarIntervalType])) {
-      throw new AnalysisException("Cannot save interval data type into external storage.")
-    }
-
+    disallowWritingIntervals(data.schema.map(_.dataType))
     providingInstance() match {
       case dataSource: CreatableRelationProvider =>
         SaveIntoDataSourceCommand(data, dataSource, caseInsensitiveOptions, mode)
@@ -570,10 +570,14 @@ case class DataSource(
       checkEmptyGlobPath: Boolean,
       checkFilesExist: Boolean): Seq[Path] = {
     val allPaths = caseInsensitiveOptions.get("path") ++ paths
-    val hadoopConf = sparkSession.sessionState.newHadoopConf()
+    DataSource.checkAndGlobPathIfNecessary(allPaths.toSeq, newHadoopConfiguration(),
+      checkEmptyGlobPath, checkFilesExist, enableGlobbing = globPaths)
+  }
 
-    DataSource.checkAndGlobPathIfNecessary(allPaths.toSeq, hadoopConf,
-      checkEmptyGlobPath, checkFilesExist)
+  private def disallowWritingIntervals(dataTypes: Seq[DataType]): Unit = {
+    dataTypes.foreach(TypeUtils.invokeOnceForInterval(_) {
+      throw QueryCompilationErrors.cannotSaveIntervalIntoExternalStorageError()
+    })
   }
 }
 
@@ -652,27 +656,15 @@ object DataSource extends Logging {
                 dataSource
               case Failure(error) =>
                 if (provider1.startsWith("org.apache.spark.sql.hive.orc")) {
-                  throw new AnalysisException(
-                    "Hive built-in ORC data source must be used with Hive support enabled. " +
-                    "Please use the native ORC data source by setting 'spark.sql.orc.impl' to " +
-                    "'native'")
+                  throw QueryCompilationErrors.orcNotUsedWithHiveEnabledError()
                 } else if (provider1.toLowerCase(Locale.ROOT) == "avro" ||
                   provider1 == "com.databricks.spark.avro" ||
                   provider1 == "org.apache.spark.sql.avro") {
-                  throw new AnalysisException(
-                    s"Failed to find data source: $provider1. Avro is built-in but external data " +
-                    "source module since Spark 2.4. Please deploy the application as per " +
-                    "the deployment section of \"Apache Avro Data Source Guide\".")
+                  throw QueryCompilationErrors.failedToFindAvroDataSourceError(provider1)
                 } else if (provider1.toLowerCase(Locale.ROOT) == "kafka") {
-                  throw new AnalysisException(
-                    s"Failed to find data source: $provider1. Please deploy the application as " +
-                    "per the deployment section of " +
-                    "\"Structured Streaming + Kafka Integration Guide\".")
+                  throw QueryCompilationErrors.failedToFindKafkaDataSourceError(provider1)
                 } else {
-                  throw new ClassNotFoundException(
-                    s"Failed to find data source: $provider1. Please find packages at " +
-                      "http://spark.apache.org/third-party-projects.html",
-                    error)
+                  throw QueryExecutionErrors.failedToFindDataSourceError(provider1, error)
                 }
             }
           } catch {
@@ -680,8 +672,7 @@ object DataSource extends Logging {
               // NoClassDefFoundError's class name uses "/" rather than "." for packages
               val className = e.getMessage.replaceAll("/", ".")
               if (spark2RemovedClasses.contains(className)) {
-                throw new ClassNotFoundException(s"$className was removed in Spark 2.0. " +
-                  "Please check if your library is compatible with Spark 2.0", e)
+                throw QueryExecutionErrors.removedClassInSpark2Error(className, e)
               } else {
                 throw e
               }
@@ -700,8 +691,7 @@ object DataSource extends Logging {
               s"defaulting to the internal datasource (${internalSources.head.getClass.getName}).")
             internalSources.head.getClass
           } else {
-            throw new AnalysisException(s"Multiple sources found for $provider1 " +
-              s"(${sourceNames.mkString(", ")}), please specify the fully qualified class name.")
+            throw QueryCompilationErrors.findMultipleDataSourceError(provider1, sourceNames)
           }
       }
     } catch {
@@ -709,9 +699,7 @@ object DataSource extends Logging {
         // NoClassDefFoundError's class name uses "/" rather than "." for packages
         val className = e.getCause.getMessage.replaceAll("/", ".")
         if (spark2RemovedClasses.contains(className)) {
-          throw new ClassNotFoundException(s"Detected an incompatible DataSourceRegister. " +
-            "Please remove the incompatible library from classpath or upgrade it. " +
-            s"Error: ${e.getMessage}", e)
+          throw QueryExecutionErrors.incompatibleDataSourceRegisterError(e)
         } else {
           throw e
         }
@@ -736,6 +724,11 @@ object DataSource extends Logging {
     }
   }
 
+  /**
+   * The key in the "options" map for deciding whether or not to glob paths before use.
+   */
+  val GLOB_PATHS_KEY = "__globPaths__"
+
   /**
    * Checks and returns files in all the paths.
    */
@@ -744,7 +737,8 @@ object DataSource extends Logging {
       hadoopConf: Configuration,
       checkEmptyGlobPath: Boolean,
       checkFilesExist: Boolean,
-      numThreads: Integer = 40): Seq[Path] = {
+      numThreads: Integer = 40,
+      enableGlobbing: Boolean): Seq[Path] = {
     val qualifiedPaths = pathStrings.map { pathString =>
       val path = new Path(pathString)
       val fs = path.getFileSystem(hadoopConf)
@@ -759,10 +753,14 @@ object DataSource extends Logging {
       try {
         ThreadUtils.parmap(globPaths, "globPath", numThreads) { globPath =>
           val fs = globPath.getFileSystem(hadoopConf)
-          val globResult = SparkHadoopUtil.get.globPath(fs, globPath)
+          val globResult = if (enableGlobbing) {
+            SparkHadoopUtil.get.globPath(fs, globPath)
+          } else {
+            qualifiedPaths
+          }
 
           if (checkEmptyGlobPath && globResult.isEmpty) {
-            throw new AnalysisException(s"Path does not exist: $globPath")
+            throw QueryCompilationErrors.dataPathNotExistError(globPath.toString)
           }
 
           globResult
@@ -776,7 +774,7 @@ object DataSource extends Logging {
         ThreadUtils.parmap(nonGlobPaths, "checkPathsExist", numThreads) { path =>
           val fs = path.getFileSystem(hadoopConf)
           if (!fs.exists(path)) {
-            throw new AnalysisException(s"Path does not exist: $path")
+            throw QueryCompilationErrors.dataPathNotExistError(path.toString)
           }
         }
       } catch {
@@ -787,7 +785,7 @@ object DataSource extends Logging {
     val allPaths = globbedPaths ++ nonGlobPaths
     if (checkFilesExist) {
       val (filteredOut, filteredIn) = allPaths.partition { path =>
-        InMemoryFileIndex.shouldFilterOut(path.getName)
+        HadoopFSUtils.shouldFilterOutPathName(path.getName)
       }
       if (filteredIn.isEmpty) {
         logWarning(
@@ -798,7 +796,7 @@ object DataSource extends Logging {
       }
     }
 
-    allPaths.toSeq
+    allPaths
   }
 
   /**
@@ -810,7 +808,7 @@ object DataSource extends Logging {
     val path = CaseInsensitiveMap(options).get("path")
     val optionsWithoutPath = options.filterKeys(_.toLowerCase(Locale.ROOT) != "path")
     CatalogStorageFormat.empty.copy(
-      locationUri = path.map(CatalogUtils.stringToURI), properties = optionsWithoutPath)
+      locationUri = path.map(CatalogUtils.stringToURI), properties = optionsWithoutPath.toMap)
   }
 
   /**
@@ -820,19 +818,15 @@ object DataSource extends Logging {
    */
   def validateSchema(schema: StructType): Unit = {
     def hasEmptySchema(schema: StructType): Boolean = {
-      schema.size == 0 || schema.find {
+      schema.size == 0 || schema.exists {
         case StructField(_, b: StructType, _, _) => hasEmptySchema(b)
         case _ => false
-      }.isDefined
+      }
     }
 
 
     if (hasEmptySchema(schema)) {
-      throw new AnalysisException(
-        s"""
-           |Datasource does not support writing empty or nested empty schemas.
-           |Please make sure the data schema has at least one or more column(s).
-         """.stripMargin)
+      throw QueryCompilationErrors.writeEmptySchemasUnsupportedByDataSourceError()
     }
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
index 23454d7d5e7f3..58ac924a1d36c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
@@ -26,7 +26,7 @@ import org.apache.hadoop.fs.Path
 import org.apache.spark.internal.Logging
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql._
-import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow, QualifiedTableName}
+import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow, QualifiedTableName, SQLConfHelper}
 import org.apache.spark.sql.catalyst.CatalystTypeConverters.convertToScala
 import org.apache.spark.sql.catalyst.analysis._
 import org.apache.spark.sql.catalyst.catalog._
@@ -36,12 +36,17 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.planning.ScanOperation
 import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoDir, InsertIntoStatement, LogicalPlan, Project}
 import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.streaming.StreamingRelationV2
+import org.apache.spark.sql.connector.catalog.SupportsRead
+import org.apache.spark.sql.connector.catalog.TableCapability._
+import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.execution.{RowDataSourceScanExec, SparkPlan}
 import org.apache.spark.sql.execution.command._
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.execution.streaming.StreamingRelation
 import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy
 import org.apache.spark.sql.sources._
 import org.apache.spark.sql.types._
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
 import org.apache.spark.unsafe.types.UTF8String
 
 /**
@@ -51,7 +56,7 @@ import org.apache.spark.unsafe.types.UTF8String
  * Note that, this rule must be run after `PreprocessTableCreation` and
  * `PreprocessTableInsertion`.
  */
-case class DataSourceAnalysis(conf: SQLConf) extends Rule[LogicalPlan] with CastSupport {
+object DataSourceAnalysis extends Rule[LogicalPlan] with CastSupport {
 
   def resolver: Resolver = conf.resolver
 
@@ -72,28 +77,19 @@ case class DataSourceAnalysis(conf: SQLConf) extends Rule[LogicalPlan] with Cast
     // The sum of the number of static partition columns and columns provided in the SELECT
     // clause needs to match the number of columns of the target table.
     if (staticPartitions.size + sourceAttributes.size != targetAttributes.size) {
-      throw new AnalysisException(
-        s"The data to be inserted needs to have the same number of " +
-          s"columns as the target table: target table has ${targetAttributes.size} " +
-          s"column(s) but the inserted data has ${sourceAttributes.size + staticPartitions.size} " +
-          s"column(s), which contain ${staticPartitions.size} partition column(s) having " +
-          s"assigned constant values.")
+      throw QueryCompilationErrors.insertMismatchedColumnNumberError(
+        targetAttributes, sourceAttributes, staticPartitions.size)
     }
 
     if (providedPartitions.size != targetPartitionSchema.fields.size) {
-      throw new AnalysisException(
-        s"The data to be inserted needs to have the same number of " +
-          s"partition columns as the target table: target table " +
-          s"has ${targetPartitionSchema.fields.size} partition column(s) but the inserted " +
-          s"data has ${providedPartitions.size} partition columns specified.")
+      throw QueryCompilationErrors.insertMismatchedPartitionNumberError(
+        targetPartitionSchema, providedPartitions.size)
     }
 
     staticPartitions.foreach {
-      case (partKey, partValue) =>
+      case (partKey, _) =>
         if (!targetPartitionSchema.fields.exists(field => resolver(field.name, partKey))) {
-          throw new AnalysisException(
-            s"$partKey is not a partition column. Partition columns are " +
-              s"${targetPartitionSchema.fields.map(_.name).mkString("[", ",", "]")}")
+          throw QueryCompilationErrors.invalidPartitionColumnError(partKey, targetPartitionSchema)
         }
     }
 
@@ -117,9 +113,8 @@ case class DataSourceAnalysis(conf: SQLConf) extends Rule[LogicalPlan] with Cast
             Some(Alias(cast(Literal(partValue), field.dataType), field.name)())
         }
       } else {
-        throw new AnalysisException(
-          s"Partition column ${field.name} have multiple values specified, " +
-            s"${potentialSpecs.mkString("[", ", ", "]")}. Please only specify a single value.")
+        throw QueryCompilationErrors.multiplePartitionColumnValuesSpecifiedError(
+          field, potentialSpecs)
       }
     }
 
@@ -127,11 +122,8 @@ case class DataSourceAnalysis(conf: SQLConf) extends Rule[LogicalPlan] with Cast
     // any static partition appear after dynamic partitions.
     partitionList.dropWhile(_.isDefined).collectFirst {
       case Some(_) =>
-        throw new AnalysisException(
-          s"The ordering of partition columns is " +
-            s"${targetPartitionSchema.fields.map(_.name).mkString("[", ",", "]")}. " +
-            "All partition columns having constant values need to appear before other " +
-            "partition columns that do not have an assigned constant value.")
+        throw QueryCompilationErrors.invalidOrderingForConstantValuePartitionColumnError(
+          targetPartitionSchema)
     }
 
     assert(partitionList.take(staticPartitions.size).forall(_.isDefined))
@@ -152,19 +144,19 @@ case class DataSourceAnalysis(conf: SQLConf) extends Rule[LogicalPlan] with Cast
       CreateDataSourceTableAsSelectCommand(tableDesc, mode, query, query.output.map(_.name))
 
     case InsertIntoStatement(l @ LogicalRelation(_: InsertableRelation, _, _, _),
-        parts, query, overwrite, false) if parts.isEmpty =>
+        parts, _, query, overwrite, false) if parts.isEmpty =>
       InsertIntoDataSourceCommand(l, query, overwrite)
 
     case InsertIntoDir(_, storage, provider, query, overwrite)
-      if provider.isDefined && provider.get.toLowerCase(Locale.ROOT) != DDLUtils.HIVE_PROVIDER =>
-
+      if query.resolved && provider.isDefined &&
+        provider.get.toLowerCase(Locale.ROOT) != DDLUtils.HIVE_PROVIDER =>
       val outputPath = new Path(storage.locationUri.get)
       if (overwrite) DDLUtils.verifyNotReadPath(query, outputPath)
 
       InsertIntoDataSourceDirCommand(storage, provider.get, query, overwrite)
 
     case i @ InsertIntoStatement(
-        l @ LogicalRelation(t: HadoopFsRelation, _, table, _), parts, query, overwrite, _) =>
+        l @ LogicalRelation(t: HadoopFsRelation, _, table, _), parts, _, query, overwrite, _) =>
       // If the InsertIntoTable command is for a partitioned HadoopFsRelation and
       // the user has specified static partitions, we add a Project operator on top of the query
       // to include those constant column values in the query result.
@@ -195,7 +187,7 @@ case class DataSourceAnalysis(conf: SQLConf) extends Rule[LogicalPlan] with Cast
 
       // Sanity check
       if (t.location.rootPaths.size != 1) {
-        throw new AnalysisException("Can only write data to relations with a single path.")
+        throw QueryCompilationErrors.cannotWriteDataToRelationsWithMultiplePathsError()
       }
 
       val outputPath = t.location.rootPaths.head
@@ -237,11 +229,12 @@ case class DataSourceAnalysis(conf: SQLConf) extends Rule[LogicalPlan] with Cast
  * data source.
  */
 class FindDataSourceTable(sparkSession: SparkSession) extends Rule[LogicalPlan] {
-  private def readDataSourceTable(table: CatalogTable): LogicalPlan = {
+  private def readDataSourceTable(
+      table: CatalogTable, extraOptions: CaseInsensitiveStringMap): LogicalPlan = {
     val qualifiedTableName = QualifiedTableName(table.database, table.identifier.table)
     val catalog = sparkSession.sessionState.catalog
+    val dsOptions = DataSourceUtils.generateDatasourceOptions(extraOptions, table)
     catalog.getCachedPlan(qualifiedTableName, () => {
-      val pathOption = table.storage.locationUri.map("path" -> CatalogUtils.URIToString(_))
       val dataSource =
         DataSource(
           sparkSession,
@@ -251,25 +244,54 @@ class FindDataSourceTable(sparkSession: SparkSession) extends Rule[LogicalPlan]
           partitionColumns = table.partitionColumnNames,
           bucketSpec = table.bucketSpec,
           className = table.provider.get,
-          options = table.storage.properties ++ pathOption,
+          options = dsOptions,
           catalogTable = Some(table))
       LogicalRelation(dataSource.resolveRelation(checkFilesExist = false), table)
     })
   }
 
+  private def getStreamingRelation(
+      table: CatalogTable,
+      extraOptions: CaseInsensitiveStringMap): StreamingRelation = {
+    val dsOptions = DataSourceUtils.generateDatasourceOptions(extraOptions, table)
+    val dataSource = DataSource(
+      SparkSession.active,
+      className = table.provider.get,
+      userSpecifiedSchema = Some(table.schema),
+      options = dsOptions)
+    StreamingRelation(dataSource)
+  }
+
+
   override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
-    case i @ InsertIntoStatement(UnresolvedCatalogRelation(tableMeta), _, _, _, _)
-        if DDLUtils.isDatasourceTable(tableMeta) =>
-      i.copy(table = readDataSourceTable(tableMeta))
+    case i @ InsertIntoStatement(UnresolvedCatalogRelation(tableMeta, options, false),
+        _, _, _, _, _) if DDLUtils.isDatasourceTable(tableMeta) =>
+      i.copy(table = readDataSourceTable(tableMeta, options))
 
-    case i @ InsertIntoStatement(UnresolvedCatalogRelation(tableMeta), _, _, _, _) =>
+    case i @ InsertIntoStatement(UnresolvedCatalogRelation(tableMeta, _, false), _, _, _, _, _) =>
       i.copy(table = DDLUtils.readHiveTable(tableMeta))
 
-    case UnresolvedCatalogRelation(tableMeta) if DDLUtils.isDatasourceTable(tableMeta) =>
-      readDataSourceTable(tableMeta)
+    case UnresolvedCatalogRelation(tableMeta, options, false)
+        if DDLUtils.isDatasourceTable(tableMeta) =>
+      readDataSourceTable(tableMeta, options)
 
-    case UnresolvedCatalogRelation(tableMeta) =>
+    case UnresolvedCatalogRelation(tableMeta, _, false) =>
       DDLUtils.readHiveTable(tableMeta)
+
+    case UnresolvedCatalogRelation(tableMeta, extraOptions, true) =>
+      getStreamingRelation(tableMeta, extraOptions)
+
+    case s @ StreamingRelationV2(
+        _, _, table, extraOptions, _, _, _, Some(UnresolvedCatalogRelation(tableMeta, _, true))) =>
+      import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Implicits._
+      val v1Relation = getStreamingRelation(tableMeta, extraOptions)
+      if (table.isInstanceOf[SupportsRead]
+          && table.supportsAny(MICRO_BATCH_READ, CONTINUOUS_READ)) {
+        s.copy(v1Relation = Some(v1Relation))
+      } else {
+        // Fallback to V1 relation
+        v1Relation
+      }
   }
 }
 
@@ -277,8 +299,8 @@ class FindDataSourceTable(sparkSession: SparkSession) extends Rule[LogicalPlan]
 /**
  * A Strategy for planning scans over data sources defined using the sources API.
  */
-case class DataSourceStrategy(conf: SQLConf) extends Strategy with Logging with CastSupport {
-  import DataSourceStrategy._
+object DataSourceStrategy
+  extends Strategy with Logging with CastSupport with PredicateHelper with SQLConfHelper {
 
   def apply(plan: LogicalPlan): Seq[execution.SparkPlan] = plan match {
     case ScanOperation(projects, filters, l @ LogicalRelation(t: CatalystScan, _, _, _)) =>
@@ -307,7 +329,7 @@ case class DataSourceStrategy(conf: SQLConf) extends Strategy with Logging with
     case l @ LogicalRelation(baseRelation: TableScan, _, _, _) =>
       RowDataSourceScanExec(
         l.output,
-        l.output.indices,
+        l.output.toStructType,
         Set.empty,
         Set.empty,
         toCatalystRDD(l, baseRelation.buildScan()),
@@ -379,8 +401,8 @@ case class DataSourceStrategy(conf: SQLConf) extends Strategy with Logging with
         .map(relation.attributeMap)
 
       val scan = RowDataSourceScanExec(
-        relation.output,
-        requestedColumns.map(relation.output.indexOf),
+        requestedColumns,
+        requestedColumns.toStructType,
         pushedFilters.toSet,
         handledFilters,
         scanBuilder(requestedColumns, candidatePredicates, pushedFilters),
@@ -401,8 +423,8 @@ case class DataSourceStrategy(conf: SQLConf) extends Strategy with Logging with
         (projectSet ++ filterSet -- handledSet).map(relation.attributeMap).toSeq
 
       val scan = RowDataSourceScanExec(
-        relation.output,
-        requestedColumns.map(relation.output.indexOf),
+        requestedColumns,
+        requestedColumns.toStructType,
         pushedFilters.toSet,
         handledFilters,
         scanBuilder(requestedColumns, candidatePredicates, pushedFilters),
@@ -429,9 +451,7 @@ case class DataSourceStrategy(conf: SQLConf) extends Strategy with Logging with
   private[this] def toCatalystRDD(relation: LogicalRelation, rdd: RDD[Row]): RDD[InternalRow] = {
     toCatalystRDD(relation, relation.output, rdd)
   }
-}
 
-object DataSourceStrategy {
   /**
    * The attribute name may differ from the one in the schema if the query analyzer
    * is case insensitive. We should change attribute names to match the ones in the schema,
@@ -448,6 +468,20 @@ object DataSourceStrategy {
     }
   }
 
+  def getPushedDownFilters(
+      partitionColumns: Seq[Expression],
+      normalizedFilters: Seq[Expression]): ExpressionSet = {
+    if (partitionColumns.isEmpty) {
+      ExpressionSet(Nil)
+    } else {
+      val partitionSet = AttributeSet(partitionColumns)
+      val predicates = ExpressionSet(normalizedFilters
+        .flatMap(extractPredicatesWithinOutputSet(_, partitionSet)))
+      logInfo(s"Pruning directories with: ${predicates.mkString(",")}")
+      predicates
+    }
+  }
+
   private def translateLeafNodeFilter(
       predicate: Expression,
       pushableColumn: PushableColumnBase): Option[Filter] = predicate match {
@@ -593,8 +627,7 @@ object DataSourceStrategy {
         expressions.Not(rebuildExpressionFromFilter(pred, translatedFilterToExpr))
       case other =>
         translatedFilterToExpr.getOrElse(other,
-          throw new AnalysisException(
-            s"Fail to rebuild expression: missing key $filter in `translatedFilterToExpr`"))
+          throw QueryCompilationErrors.failedToRebuildExpressionError(filter))
     }
   }
 
@@ -668,6 +701,8 @@ abstract class PushableColumnBase {
     import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
     def helper(e: Expression): Option[Seq[String]] = e match {
       case a: Attribute =>
+        // Attribute that contains dot "." in name is supported only when
+        // nested predicate pushdown is enabled.
         if (nestedPredicatePushdownEnabled || !a.name.contains(".")) {
           Some(Seq(a.name))
         } else {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala
index abb74d8d09ec6..2b10e4efd9ab8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala
@@ -19,18 +19,23 @@ package org.apache.spark.sql.execution.datasources
 
 import java.util.Locale
 
+import scala.collection.JavaConverters._
+
 import org.apache.hadoop.fs.Path
 import org.json4s.NoTypeHints
 import org.json4s.jackson.Serialization
 
 import org.apache.spark.SparkUpgradeException
-import org.apache.spark.sql.{SPARK_LEGACY_DATETIME, SPARK_VERSION_METADATA_KEY}
-import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.{SPARK_LEGACY_DATETIME, SPARK_LEGACY_INT96, SPARK_VERSION_METADATA_KEY}
+import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogUtils}
 import org.apache.spark.sql.catalyst.util.RebaseDateTime
+import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
+import org.apache.spark.sql.execution.datasources.parquet.ParquetOptions
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 import org.apache.spark.sql.sources.BaseRelation
 import org.apache.spark.sql.types._
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
 import org.apache.spark.util.Utils
 
 
@@ -60,8 +65,7 @@ object DataSourceUtils {
   def verifySchema(format: FileFormat, schema: StructType): Unit = {
     schema.foreach { field =>
       if (!format.supportDataType(field.dataType)) {
-        throw new AnalysisException(
-          s"$format data source does not support ${field.dataType.catalogString} data type.")
+        throw QueryCompilationErrors.dataTypeUnsupportedByDataSourceError(format.toString, field)
       }
     }
   }
@@ -107,40 +111,47 @@ object DataSourceUtils {
     }.getOrElse(LegacyBehaviorPolicy.withName(modeByConfig))
   }
 
+  def int96RebaseMode(
+      lookupFileMeta: String => String,
+      modeByConfig: String): LegacyBehaviorPolicy.Value = {
+    if (Utils.isTesting && SQLConf.get.getConfString("spark.test.forceNoRebase", "") == "true") {
+      return LegacyBehaviorPolicy.CORRECTED
+    }
+    // If there is no version, we return the mode specified by the config.
+    Option(lookupFileMeta(SPARK_VERSION_METADATA_KEY)).map { version =>
+      // Files written by Spark 3.0 and earlier follow the legacy hybrid calendar and we need to
+      // rebase the INT96 timestamp values.
+      // Files written by Spark 3.1 and latter may also need the rebase if they were written with
+      // the "LEGACY" rebase mode.
+      if (version < "3.1.0" || lookupFileMeta(SPARK_LEGACY_INT96) != null) {
+        LegacyBehaviorPolicy.LEGACY
+      } else {
+        LegacyBehaviorPolicy.CORRECTED
+      }
+    }.getOrElse(LegacyBehaviorPolicy.withName(modeByConfig))
+  }
+
   def newRebaseExceptionInRead(format: String): SparkUpgradeException = {
-    val config = if (format == "Parquet") {
-      SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ.key
-    } else if (format == "Avro") {
-      SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ.key
-    } else {
-      throw new IllegalStateException("unrecognized format " + format)
+    val (config, option) = format match {
+      case "Parquet INT96" =>
+        (SQLConf.PARQUET_INT96_REBASE_MODE_IN_READ.key, ParquetOptions.INT96_REBASE_MODE)
+      case "Parquet" =>
+        (SQLConf.PARQUET_REBASE_MODE_IN_READ.key, ParquetOptions.DATETIME_REBASE_MODE)
+      case "Avro" =>
+        (SQLConf.AVRO_REBASE_MODE_IN_READ.key, "datetimeRebaseMode")
+      case _ => throw QueryExecutionErrors.unrecognizedFileFormatError(format)
     }
-    new SparkUpgradeException("3.0", "reading dates before 1582-10-15 or timestamps before " +
-      s"1900-01-01T00:00:00Z from $format files can be ambiguous, as the files may be written by " +
-      "Spark 2.x or legacy versions of Hive, which uses a legacy hybrid calendar that is " +
-      "different from Spark 3.0+'s Proleptic Gregorian calendar. See more details in " +
-      s"SPARK-31404. You can set $config to 'LEGACY' to rebase the datetime values w.r.t. " +
-      s"the calendar difference during reading. Or set $config to 'CORRECTED' to read the " +
-      "datetime values as it is.", null)
+    QueryExecutionErrors.sparkUpgradeInReadingDatesError(format, config, option)
   }
 
   def newRebaseExceptionInWrite(format: String): SparkUpgradeException = {
-    val config = if (format == "Parquet") {
-      SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key
-    } else if (format == "Avro") {
-      SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key
-    } else {
-      throw new IllegalStateException("unrecognized format " + format)
+    val config = format match {
+      case "Parquet INT96" => SQLConf.PARQUET_INT96_REBASE_MODE_IN_WRITE.key
+      case "Parquet" => SQLConf.PARQUET_REBASE_MODE_IN_WRITE.key
+      case "Avro" => SQLConf.AVRO_REBASE_MODE_IN_WRITE.key
+      case _ => throw QueryExecutionErrors.unrecognizedFileFormatError(format)
     }
-    new SparkUpgradeException("3.0", "writing dates before 1582-10-15 or timestamps before " +
-      s"1900-01-01T00:00:00Z into $format files can be dangerous, as the files may be read by " +
-      "Spark 2.x or legacy versions of Hive later, which uses a legacy hybrid calendar that is " +
-      "different from Spark 3.0+'s Proleptic Gregorian calendar. See more details in " +
-      s"SPARK-31404. You can set $config to 'LEGACY' to rebase the datetime values w.r.t. " +
-      "the calendar difference during writing, to get maximum interoperability. Or set " +
-      s"$config to 'CORRECTED' to write the datetime values as it is, if you are 100% sure that " +
-      "the written files will only be read by Spark 3.0+ or other systems that use Proleptic " +
-      "Gregorian calendar.", null)
+    QueryExecutionErrors.sparkUpgradeInWritingDatesError(format, config)
   }
 
   def creteDateRebaseFuncInRead(
@@ -190,4 +201,28 @@ object DataSourceUtils {
     case LegacyBehaviorPolicy.LEGACY => RebaseDateTime.rebaseGregorianToJulianMicros
     case LegacyBehaviorPolicy.CORRECTED => identity[Long]
   }
+
+  def generateDatasourceOptions(
+      extraOptions: CaseInsensitiveStringMap, table: CatalogTable): Map[String, String] = {
+    val pathOption = table.storage.locationUri.map("path" -> CatalogUtils.URIToString(_))
+    val options = table.storage.properties ++ pathOption
+    if (!SQLConf.get.getConf(SQLConf.LEGACY_EXTRA_OPTIONS_BEHAVIOR)) {
+      // Check the same key with different values
+      table.storage.properties.foreach { case (k, v) =>
+        if (extraOptions.containsKey(k) && extraOptions.get(k) != v) {
+          throw QueryCompilationErrors.failToResolveDataSourceForTableError(table, k)
+        }
+      }
+      // To keep the original key from table properties, here we filter all case insensitive
+      // duplicate keys out from extra options.
+      val lowerCasedDuplicatedKeys =
+        table.storage.properties.keySet.map(_.toLowerCase(Locale.ROOT))
+          .intersect(extraOptions.keySet.asScala)
+      extraOptions.asCaseSensitiveMap().asScala.filterNot {
+        case (k, _) => lowerCasedDuplicatedKeys.contains(k.toLowerCase(Locale.ROOT))
+      }.toMap ++ options
+    } else {
+      options
+    }
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DaysWritable.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DaysWritable.scala
index 486f678e49d7d..a04c2fcbbac12 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DaysWritable.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DaysWritable.scala
@@ -54,7 +54,12 @@ class DaysWritable(
   }
 
   override def getDays: Int = julianDays
-  override def get(): Date = new Date(DateWritable.daysToMillis(julianDays))
+  override def get: Date = {
+    new Date(DateWritable.daysToMillis(julianDays))
+  }
+  override def get(doesTimeMatter: Boolean): Date = {
+    new Date(DateWritable.daysToMillis(julianDays, doesTimeMatter))
+  }
 
   override def set(d: Int): Unit = {
     gregorianDays = d
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FallBackFileSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FallBackFileSourceV2.scala
index 28a63c26604ec..b5d06db024112 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FallBackFileSourceV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FallBackFileSourceV2.scala
@@ -22,19 +22,20 @@ import scala.collection.JavaConverters._
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoStatement, LogicalPlan}
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Relation, FileDataSourceV2, FileTable}
+import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Relation, FileTable}
 
 /**
  * Replace the File source V2 table in [[InsertIntoStatement]] to V1 [[FileFormat]].
- * E.g, with temporary view `t` using [[FileDataSourceV2]], inserting into  view `t` fails
+ * E.g, with temporary view `t` using
+ * [[org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2]], inserting into view `t` fails
  * since there is no corresponding physical plan.
  * This is a temporary hack for making current data source V2 work. It should be
  * removed when Catalog support of file data source v2 is finished.
  */
 class FallBackFileSourceV2(sparkSession: SparkSession) extends Rule[LogicalPlan] {
   override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
-    case i @
-        InsertIntoStatement(d @ DataSourceV2Relation(table: FileTable, _, _, _, _), _, _, _, _) =>
+    case i @ InsertIntoStatement(
+        d @ DataSourceV2Relation(table: FileTable, _, _, _, _), _, _, _, _, _) =>
       val v1FileFormat = table.fallbackFileFormat.newInstance()
       val relation = HadoopFsRelation(
         table.fileIndex,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormat.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormat.scala
index a72a9c27a9894..7fd48caa656ee 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormat.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormat.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types.{DataType, StructType}
@@ -110,7 +111,7 @@ trait FileFormat {
       filters: Seq[Filter],
       options: Map[String, String],
       hadoopConf: Configuration): PartitionedFile => Iterator[InternalRow] = {
-    throw new UnsupportedOperationException(s"buildReader is not supported for $this")
+    throw QueryExecutionErrors.buildReaderUnsupportedForFileFormatError(this.toString)
   }
 
   /**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatDataWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatDataWriter.scala
index edb49d3f90ca3..7e5a8cce27831 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatDataWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatDataWriter.scala
@@ -21,6 +21,7 @@ import scala.collection.mutable
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.mapreduce.TaskAttemptContext
 
+import org.apache.spark.internal.Logging
 import org.apache.spark.internal.io.FileCommitProtocol
 import org.apache.spark.internal.io.FileCommitProtocol.TaskCommitMessage
 import org.apache.spark.sql.catalyst.InternalRow
@@ -28,6 +29,8 @@ import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
 import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.connector.write.{DataWriter, WriterCommitMessage}
+import org.apache.spark.sql.execution.datasources.FileFormatWriter.ConcurrentOutputWriterSpec
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.StringType
 import org.apache.spark.util.SerializableConfiguration
 
@@ -52,19 +55,35 @@ abstract class FileFormatDataWriter(
   protected val statsTrackers: Seq[WriteTaskStatsTracker] =
     description.statsTrackers.map(_.newTaskInstance())
 
-  protected def releaseResources(): Unit = {
+  /** Release resources of `currentWriter`. */
+  protected def releaseCurrentWriter(): Unit = {
     if (currentWriter != null) {
       try {
         currentWriter.close()
+        statsTrackers.foreach(_.closeFile(currentWriter.path()))
       } finally {
         currentWriter = null
       }
     }
   }
 
-  /** Writes a record */
+  /** Release all resources. */
+  protected def releaseResources(): Unit = {
+    // Call `releaseCurrentWriter()` by default, as this is the only resource to be released.
+    releaseCurrentWriter()
+  }
+
+  /** Writes a record. */
   def write(record: InternalRow): Unit
 
+
+  /** Write an iterator of records. */
+  def writeWithIterator(iterator: Iterator[InternalRow]): Unit = {
+    while (iterator.hasNext) {
+      write(iterator.next())
+    }
+  }
+
   /**
    * Returns the summary of relative information which
    * includes the list of partition strings written out. The list of partitions is sent back
@@ -138,40 +157,44 @@ class SingleDirectoryDataWriter(
     }
 
     currentWriter.write(record)
-    statsTrackers.foreach(_.newRow(record))
+    statsTrackers.foreach(_.newRow(currentWriter.path, record))
     recordsInFile += 1
   }
 }
 
 /**
- * Writes data to using dynamic partition writes, meaning this single function can write to
+ * Holds common logic for writing data with dynamic partition writes, meaning it can write to
  * multiple directories (partitions) or files (bucketing).
  */
-class DynamicPartitionDataWriter(
+abstract class BaseDynamicPartitionDataWriter(
     description: WriteJobDescription,
     taskAttemptContext: TaskAttemptContext,
     committer: FileCommitProtocol)
   extends FileFormatDataWriter(description, taskAttemptContext, committer) {
 
   /** Flag saying whether or not the data to be written out is partitioned. */
-  private val isPartitioned = description.partitionColumns.nonEmpty
+  protected val isPartitioned = description.partitionColumns.nonEmpty
 
   /** Flag saying whether or not the data to be written out is bucketed. */
-  private val isBucketed = description.bucketIdExpression.isDefined
+  protected val isBucketed = description.bucketIdExpression.isDefined
 
   assert(isPartitioned || isBucketed,
     s"""DynamicPartitionWriteTask should be used for writing out data that's either
-         |partitioned or bucketed. In this case neither is true.
-         |WriteJobDescription: $description
+       |partitioned or bucketed. In this case neither is true.
+       |WriteJobDescription: $description
        """.stripMargin)
 
-  private var fileCounter: Int = _
-  private var recordsInFile: Long = _
-  private var currentPartionValues: Option[UnsafeRow] = None
-  private var currentBucketId: Option[Int] = None
+  /** Number of records in current file. */
+  protected var recordsInFile: Long = _
+
+  /**
+   * File counter for writing current partition or bucket. For same partition or bucket,
+   * we may have more than one file, due to number of records limit per file.
+   */
+  protected var fileCounter: Int = _
 
   /** Extracts the partition values out of an input row. */
-  private lazy val getPartitionValues: InternalRow => UnsafeRow = {
+  protected lazy val getPartitionValues: InternalRow => UnsafeRow = {
     val proj = UnsafeProjection.create(description.partitionColumns, description.allColumns)
     row => proj(row)
   }
@@ -186,22 +209,24 @@ class DynamicPartitionDataWriter(
       if (i == 0) Seq(partitionName) else Seq(Literal(Path.SEPARATOR), partitionName)
     })
 
-  /** Evaluates the `partitionPathExpression` above on a row of `partitionValues` and returns
-   * the partition string. */
+  /**
+   * Evaluates the `partitionPathExpression` above on a row of `partitionValues` and returns
+   * the partition string.
+   */
   private lazy val getPartitionPath: InternalRow => String = {
     val proj = UnsafeProjection.create(Seq(partitionPathExpression), description.partitionColumns)
     row => proj(row).getString(0)
   }
 
   /** Given an input row, returns the corresponding `bucketId` */
-  private lazy val getBucketId: InternalRow => Int = {
+  protected lazy val getBucketId: InternalRow => Int = {
     val proj =
       UnsafeProjection.create(description.bucketIdExpression.toSeq, description.allColumns)
     row => proj(row).getInt(0)
   }
 
   /** Returns the data columns to be written given an input row */
-  private val getOutputRow =
+  protected val getOutputRow =
     UnsafeProjection.create(description.dataColumns, description.allColumns)
 
   /**
@@ -209,13 +234,20 @@ class DynamicPartitionDataWriter(
    * If bucket id is specified, we will append it to the end of the file name, but before the
    * file extension, e.g. part-r-00009-ea518ad4-455a-4431-b471-d24e03814677-00002.gz.parquet
    *
-   * @param partitionValues the partition which all tuples being written by this `OutputWriter`
+   * @param partitionValues the partition which all tuples being written by this OutputWriter
    *                        belong to
-   * @param bucketId the bucket which all tuples being written by this `OutputWriter` belong to
+   * @param bucketId the bucket which all tuples being written by this OutputWriter belong to
+   * @param closeCurrentWriter close and release resource for current writer
    */
-  private def newOutputWriter(partitionValues: Option[InternalRow], bucketId: Option[Int]): Unit = {
+  protected def renewCurrentWriter(
+      partitionValues: Option[InternalRow],
+      bucketId: Option[Int],
+      closeCurrentWriter: Boolean): Unit = {
+
     recordsInFile = 0
-    releaseResources()
+    if (closeCurrentWriter) {
+      releaseCurrentWriter()
+    }
 
     val partDir = partitionValues.map(getPartitionPath(_))
     partDir.foreach(updatedPartitions.add)
@@ -243,37 +275,256 @@ class DynamicPartitionDataWriter(
     statsTrackers.foreach(_.newFile(currentPath))
   }
 
+  /**
+   * Open a new output writer when number of records exceeding limit.
+   *
+   * @param partitionValues the partition which all tuples being written by this `OutputWriter`
+   *                        belong to
+   * @param bucketId the bucket which all tuples being written by this `OutputWriter` belong to
+   */
+  protected def renewCurrentWriterIfTooManyRecords(
+      partitionValues: Option[InternalRow],
+      bucketId: Option[Int]): Unit = {
+    // Exceeded the threshold in terms of the number of records per file.
+    // Create a new file by increasing the file counter.
+    fileCounter += 1
+    assert(fileCounter < MAX_FILE_COUNTER,
+      s"File counter $fileCounter is beyond max value $MAX_FILE_COUNTER")
+    renewCurrentWriter(partitionValues, bucketId, closeCurrentWriter = true)
+  }
+
+  /**
+   * Writes the given record with current writer.
+   *
+   * @param record The record to write
+   */
+  protected def writeRecord(record: InternalRow): Unit = {
+    val outputRow = getOutputRow(record)
+    currentWriter.write(outputRow)
+    statsTrackers.foreach(_.newRow(currentWriter.path, outputRow))
+    recordsInFile += 1
+  }
+}
+
+/**
+ * Dynamic partition writer with single writer, meaning only one writer is opened at any time for
+ * writing. The records to be written are required to be sorted on partition and/or bucket
+ * column(s) before writing.
+ */
+class DynamicPartitionDataSingleWriter(
+    description: WriteJobDescription,
+    taskAttemptContext: TaskAttemptContext,
+    committer: FileCommitProtocol)
+  extends BaseDynamicPartitionDataWriter(description, taskAttemptContext, committer) {
+
+  private var currentPartitionValues: Option[UnsafeRow] = None
+  private var currentBucketId: Option[Int] = None
+
   override def write(record: InternalRow): Unit = {
     val nextPartitionValues = if (isPartitioned) Some(getPartitionValues(record)) else None
     val nextBucketId = if (isBucketed) Some(getBucketId(record)) else None
 
-    if (currentPartionValues != nextPartitionValues || currentBucketId != nextBucketId) {
+    if (currentPartitionValues != nextPartitionValues || currentBucketId != nextBucketId) {
       // See a new partition or bucket - write to a new partition dir (or a new bucket file).
-      if (isPartitioned && currentPartionValues != nextPartitionValues) {
-        currentPartionValues = Some(nextPartitionValues.get.copy())
-        statsTrackers.foreach(_.newPartition(currentPartionValues.get))
+      if (isPartitioned && currentPartitionValues != nextPartitionValues) {
+        currentPartitionValues = Some(nextPartitionValues.get.copy())
+        statsTrackers.foreach(_.newPartition(currentPartitionValues.get))
       }
       if (isBucketed) {
         currentBucketId = nextBucketId
-        statsTrackers.foreach(_.newBucket(currentBucketId.get))
       }
 
       fileCounter = 0
-      newOutputWriter(currentPartionValues, currentBucketId)
+      renewCurrentWriter(currentPartitionValues, currentBucketId, closeCurrentWriter = true)
     } else if (description.maxRecordsPerFile > 0 &&
       recordsInFile >= description.maxRecordsPerFile) {
-      // Exceeded the threshold in terms of the number of records per file.
-      // Create a new file by increasing the file counter.
-      fileCounter += 1
-      assert(fileCounter < MAX_FILE_COUNTER,
-        s"File counter $fileCounter is beyond max value $MAX_FILE_COUNTER")
+      renewCurrentWriterIfTooManyRecords(currentPartitionValues, currentBucketId)
+    }
+    writeRecord(record)
+  }
+}
+
+/**
+ * Dynamic partition writer with concurrent writers, meaning multiple concurrent writers are opened
+ * for writing.
+ *
+ * The process has the following steps:
+ *  - Step 1: Maintain a map of output writers per each partition and/or bucket columns. Keep all
+ *            writers opened and write rows one by one.
+ *  - Step 2: If number of concurrent writers exceeds limit, sort rest of rows on partition and/or
+ *            bucket column(s). Write rows one by one, and eagerly close the writer when finishing
+ *            each partition and/or bucket.
+ *
+ * Caller is expected to call `writeWithIterator()` instead of `write()` to write records.
+ */
+class DynamicPartitionDataConcurrentWriter(
+    description: WriteJobDescription,
+    taskAttemptContext: TaskAttemptContext,
+    committer: FileCommitProtocol,
+    concurrentOutputWriterSpec: ConcurrentOutputWriterSpec)
+  extends BaseDynamicPartitionDataWriter(description, taskAttemptContext, committer)
+  with Logging {
+
+  /** Wrapper class to index a unique concurrent output writer. */
+  private case class WriterIndex(
+    var partitionValues: Option[UnsafeRow],
+    var bucketId: Option[Int])
+
+  /** Wrapper class for status of a unique concurrent output writer. */
+  private class WriterStatus(
+    var outputWriter: OutputWriter,
+    var recordsInFile: Long,
+    var fileCounter: Int)
+
+  /**
+   * State to indicate if we are falling back to sort-based writer.
+   * Because we first try to use concurrent writers, its initial value is false.
+   */
+  private var sorted: Boolean = false
+  private val concurrentWriters = mutable.HashMap[WriterIndex, WriterStatus]()
+
+  /**
+   * The index for current writer. Intentionally make the index mutable and reusable.
+   * Avoid JVM GC issue when many short-living `WriterIndex` objects are created
+   * if switching between concurrent writers frequently.
+   */
+  private val currentWriterId = WriterIndex(None, None)
 
-      newOutputWriter(currentPartionValues, currentBucketId)
+  /**
+   * Release resources for all concurrent output writers.
+   */
+  override protected def releaseResources(): Unit = {
+    currentWriter = null
+    concurrentWriters.values.foreach(status => {
+      if (status.outputWriter != null) {
+        try {
+          status.outputWriter.close()
+        } finally {
+          status.outputWriter = null
+        }
+      }
+    })
+    concurrentWriters.clear()
+  }
+
+  override def write(record: InternalRow): Unit = {
+    val nextPartitionValues = if (isPartitioned) Some(getPartitionValues(record)) else None
+    val nextBucketId = if (isBucketed) Some(getBucketId(record)) else None
+
+    if (currentWriterId.partitionValues != nextPartitionValues ||
+      currentWriterId.bucketId != nextBucketId) {
+      // See a new partition or bucket - write to a new partition dir (or a new bucket file).
+      if (currentWriter != null) {
+        if (!sorted) {
+          // Update writer status in concurrent writers map, because the writer is probably needed
+          // again later for writing other rows.
+          updateCurrentWriterStatusInMap()
+        } else {
+          // Remove writer status in concurrent writers map and release current writer resource,
+          // because the writer is not needed any more.
+          concurrentWriters.remove(currentWriterId)
+          releaseCurrentWriter()
+        }
+      }
+
+      if (isBucketed) {
+        currentWriterId.bucketId = nextBucketId
+      }
+      if (isPartitioned && currentWriterId.partitionValues != nextPartitionValues) {
+        currentWriterId.partitionValues = Some(nextPartitionValues.get.copy())
+        if (!concurrentWriters.contains(currentWriterId)) {
+          statsTrackers.foreach(_.newPartition(currentWriterId.partitionValues.get))
+        }
+      }
+      setupCurrentWriterUsingMap()
     }
-    val outputRow = getOutputRow(record)
-    currentWriter.write(outputRow)
-    statsTrackers.foreach(_.newRow(outputRow))
-    recordsInFile += 1
+
+    if (description.maxRecordsPerFile > 0 &&
+      recordsInFile >= description.maxRecordsPerFile) {
+      renewCurrentWriterIfTooManyRecords(currentWriterId.partitionValues, currentWriterId.bucketId)
+      // Update writer status in concurrent writers map, as a new writer is created.
+      updateCurrentWriterStatusInMap()
+    }
+    writeRecord(record)
+  }
+
+  /**
+   * Write iterator of records with concurrent writers.
+   */
+  override def writeWithIterator(iterator: Iterator[InternalRow]): Unit = {
+    while (iterator.hasNext && !sorted) {
+      write(iterator.next())
+    }
+
+    if (iterator.hasNext) {
+      clearCurrentWriterStatus()
+      val sorter = concurrentOutputWriterSpec.createSorter()
+      val sortIterator = sorter.sort(iterator.asInstanceOf[Iterator[UnsafeRow]])
+      while (sortIterator.hasNext) {
+        write(sortIterator.next())
+      }
+    }
+  }
+
+  /**
+   * Update current writer status in map.
+   */
+  private def updateCurrentWriterStatusInMap(): Unit = {
+    val status = concurrentWriters(currentWriterId)
+    status.outputWriter = currentWriter
+    status.recordsInFile = recordsInFile
+    status.fileCounter = fileCounter
+  }
+
+  /**
+   * Retrieve writer in map, or create a new writer if not exists.
+   */
+  private def setupCurrentWriterUsingMap(): Unit = {
+    if (concurrentWriters.contains(currentWriterId)) {
+      val status = concurrentWriters(currentWriterId)
+      currentWriter = status.outputWriter
+      recordsInFile = status.recordsInFile
+      fileCounter = status.fileCounter
+    } else {
+      fileCounter = 0
+      renewCurrentWriter(
+        currentWriterId.partitionValues,
+        currentWriterId.bucketId,
+        closeCurrentWriter = false)
+      if (!sorted) {
+        assert(concurrentWriters.size <= concurrentOutputWriterSpec.maxWriters,
+          s"Number of concurrent output file writers is ${concurrentWriters.size} " +
+            s" which is beyond max value ${concurrentOutputWriterSpec.maxWriters}")
+      } else {
+        assert(concurrentWriters.size <= concurrentOutputWriterSpec.maxWriters + 1,
+          s"Number of output file writers after sort is ${concurrentWriters.size} " +
+            s" which is beyond max value ${concurrentOutputWriterSpec.maxWriters + 1}")
+      }
+      concurrentWriters.put(
+        currentWriterId.copy(),
+        new WriterStatus(currentWriter, recordsInFile, fileCounter))
+      if (concurrentWriters.size >= concurrentOutputWriterSpec.maxWriters && !sorted) {
+        // Fall back to sort-based sequential writer mode.
+        logInfo(s"Number of concurrent writers ${concurrentWriters.size} reaches the threshold. " +
+          "Fall back from concurrent writers to sort-based sequential writer. You may change " +
+          s"threshold with configuration ${SQLConf.MAX_CONCURRENT_OUTPUT_FILE_WRITERS.key}")
+        sorted = true
+      }
+    }
+  }
+
+  /**
+   * Clear the current writer status in map.
+   */
+  private def clearCurrentWriterStatus(): Unit = {
+    if (currentWriterId.partitionValues.isDefined || currentWriterId.bucketId.isDefined) {
+      updateCurrentWriterStatusInMap()
+    }
+    currentWriterId.partitionValues = None
+    currentWriterId.bucketId = None
+    currentWriter = null
+    recordsInFile = 0
+    fileCounter = 0
   }
 }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala
index 219c778b9164a..6839a4db0bc28 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala
@@ -38,7 +38,9 @@ import org.apache.spark.sql.catalyst.expressions.BindReferences.bindReferences
 import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
 import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning
 import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateTimeUtils}
-import org.apache.spark.sql.execution.{ProjectExec, SortExec, SparkPlan, SQLExecution}
+import org.apache.spark.sql.errors.QueryExecutionErrors
+import org.apache.spark.sql.execution.{ProjectExec, SortExec, SparkPlan, SQLExecution, UnsafeExternalRowSorter}
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.StringType
 import org.apache.spark.unsafe.types.UTF8String
 import org.apache.spark.util.{SerializableConfiguration, Utils}
@@ -66,8 +68,16 @@ object FileFormatWriter extends Logging {
            |}""".stripMargin
       })
     }
+
+    override protected def withNewChildInternal(newChild: Expression): Empty2Null =
+      copy(child = newChild)
   }
 
+  /** Describes how concurrent output writers should be executed. */
+  case class ConcurrentOutputWriterSpec(
+      maxWriters: Int,
+      createSorter: () => UnsafeExternalRowSorter)
+
   /**
    * Basic work flow of this command is:
    * 1. Driver side setup, including output committer initialization and data source specific
@@ -132,7 +142,7 @@ object FileFormatWriter extends Logging {
       fileFormat.prepareWrite(sparkSession, job, caseInsensitiveOptions, dataSchema)
 
     val description = new WriteJobDescription(
-      uuid = UUID.randomUUID().toString,
+      uuid = UUID.randomUUID.toString,
       serializableHadoopConf = new SerializableConfiguration(job.getConfiguration),
       outputWriterFactory = outputWriterFactory,
       allColumns = outputSpec.outputColumns,
@@ -163,23 +173,36 @@ object FileFormatWriter extends Logging {
 
     SQLExecution.checkSQLExecutionId(sparkSession)
 
+    // propagate the description UUID into the jobs, so that committers
+    // get an ID guaranteed to be unique.
+    job.getConfiguration.set("spark.sql.sources.writeJobUUID", description.uuid)
+
     // This call shouldn't be put into the `try` block below because it only initializes and
     // prepares the job, any exception thrown from here shouldn't cause abortJob() to be called.
     committer.setupJob(job)
 
     try {
-      val rdd = if (orderingMatched) {
-        empty2NullPlan.execute()
+      val (rdd, concurrentOutputWriterSpec) = if (orderingMatched) {
+        (empty2NullPlan.execute(), None)
       } else {
         // SPARK-21165: the `requiredOrdering` is based on the attributes from analyzed plan, and
         // the physical plan may have different attribute ids due to optimizer removing some
         // aliases. Here we bind the expression ahead to avoid potential attribute ids mismatch.
         val orderingExpr = bindReferences(
           requiredOrdering.map(SortOrder(_, Ascending)), outputSpec.outputColumns)
-        SortExec(
+        val sortPlan = SortExec(
           orderingExpr,
           global = false,
-          child = empty2NullPlan).execute()
+          child = empty2NullPlan)
+
+        val maxWriters = sparkSession.sessionState.conf.maxConcurrentOutputFileWriters
+        val concurrentWritersEnabled = maxWriters > 0 && sortColumns.isEmpty
+        if (concurrentWritersEnabled) {
+          (empty2NullPlan.execute(),
+            Some(ConcurrentOutputWriterSpec(maxWriters, () => sortPlan.createSorter())))
+        } else {
+          (sortPlan.execute(), None)
+        }
       }
 
       // SPARK-23271 If we are attempting to write a zero partition rdd, create a dummy single
@@ -202,7 +225,8 @@ object FileFormatWriter extends Logging {
             sparkPartitionId = taskContext.partitionId(),
             sparkAttemptNumber = taskContext.taskAttemptId().toInt & Integer.MAX_VALUE,
             committer,
-            iterator = iter)
+            iterator = iter,
+            concurrentOutputWriterSpec = concurrentOutputWriterSpec)
         },
         rddWithNonEmptyPartitions.partitions.indices,
         (index, res: WriteTaskResult) => {
@@ -212,8 +236,9 @@ object FileFormatWriter extends Logging {
 
       val commitMsgs = ret.map(_.commitMsg)
 
-      committer.commitJob(job, commitMsgs)
-      logInfo(s"Write Job ${description.uuid} committed.")
+      logInfo(s"Start to commit write Job ${description.uuid}.")
+      val (_, duration) = Utils.timeTakenMs { committer.commitJob(job, commitMsgs) }
+      logInfo(s"Write Job ${description.uuid} committed. Elapsed time: $duration ms.")
 
       processStats(description.statsTrackers, ret.map(_.summary.stats))
       logInfo(s"Finished processing stats for write job ${description.uuid}.")
@@ -223,7 +248,7 @@ object FileFormatWriter extends Logging {
     } catch { case cause: Throwable =>
       logError(s"Aborting job ${description.uuid}.", cause)
       committer.abortJob(job)
-      throw new SparkException("Job aborted.", cause)
+      throw QueryExecutionErrors.jobAbortedError(cause)
     }
   }
 
@@ -235,7 +260,8 @@ object FileFormatWriter extends Logging {
       sparkPartitionId: Int,
       sparkAttemptNumber: Int,
       committer: FileCommitProtocol,
-      iterator: Iterator[InternalRow]): WriteTaskResult = {
+      iterator: Iterator[InternalRow],
+      concurrentOutputWriterSpec: Option[ConcurrentOutputWriterSpec]): WriteTaskResult = {
 
     val jobId = SparkHadoopWriterUtils.createJobID(new Date(jobIdInstant), sparkStageId)
     val taskId = new TaskID(jobId, TaskType.MAP, sparkPartitionId)
@@ -263,15 +289,19 @@ object FileFormatWriter extends Logging {
       } else if (description.partitionColumns.isEmpty && description.bucketIdExpression.isEmpty) {
         new SingleDirectoryDataWriter(description, taskAttemptContext, committer)
       } else {
-        new DynamicPartitionDataWriter(description, taskAttemptContext, committer)
+        concurrentOutputWriterSpec match {
+          case Some(spec) =>
+            new DynamicPartitionDataConcurrentWriter(
+              description, taskAttemptContext, committer, spec)
+          case _ =>
+            new DynamicPartitionDataSingleWriter(description, taskAttemptContext, committer)
+        }
       }
 
     try {
       Utils.tryWithSafeFinallyAndFailureCallbacks(block = {
         // Execute the task to write rows out and commit the task.
-        while (iterator.hasNext) {
-          dataWriter.write(iterator.next())
-        }
+        dataWriter.writeWithIterator(iterator)
         dataWriter.commit()
       })(catchBlock = {
         // If there is an error, abort the task
@@ -283,12 +313,12 @@ object FileFormatWriter extends Logging {
     } catch {
       case e: FetchFailedException =>
         throw e
-      case f: FileAlreadyExistsException =>
+      case f: FileAlreadyExistsException if SQLConf.get.fastFailFileFormatOutput =>
         // If any output file to write already exists, it does not make sense to re-run this task.
         // We throw the exception and let Executor throw ExceptionFailure to abort the job.
         throw new TaskOutputFileAlreadyExistException(f)
       case t: Throwable =>
-        throw new SparkException("Task failed while writing rows.", t)
+        throw QueryExecutionErrors.taskFailedWhileWritingRowsError(t)
     }
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FilePartition.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FilePartition.scala
index b4fc94e097aa8..a4d16a0fd2bb2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FilePartition.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FilePartition.scala
@@ -80,7 +80,7 @@ object FilePartition extends Logging {
       currentFiles += file
     }
     closePartition()
-    partitions
+    partitions.toSeq
   }
 
   def maxSplitBytes(
@@ -88,9 +88,10 @@ object FilePartition extends Logging {
       selectedPartitions: Seq[PartitionDirectory]): Long = {
     val defaultMaxSplitBytes = sparkSession.sessionState.conf.filesMaxPartitionBytes
     val openCostInBytes = sparkSession.sessionState.conf.filesOpenCostInBytes
-    val defaultParallelism = sparkSession.sparkContext.defaultParallelism
+    val minPartitionNum = sparkSession.sessionState.conf.filesMinPartitionNum
+      .getOrElse(sparkSession.leafNodeDefaultParallelism)
     val totalBytes = selectedPartitions.flatMap(_.files.map(_.getLen + openCostInBytes)).sum
-    val bytesPerCore = totalBytes / defaultParallelism
+    val bytesPerCore = totalBytes / minPartitionNum
 
     Math.min(defaultMaxSplitBytes, Math.max(openCostInBytes, bytesPerCore))
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileScanRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileScanRDD.scala
index fc59336d6107c..64d16c6c91812 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileScanRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileScanRDD.scala
@@ -26,6 +26,7 @@ import org.apache.spark.deploy.SparkHadoopUtil
 import org.apache.spark.rdd.{InputFileBlockHolder, RDD}
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.execution.QueryExecutionException
 import org.apache.spark.sql.vectorized.ColumnarBatch
 import org.apache.spark.util.NextIterator
@@ -116,12 +117,7 @@ class FileScanRDD(
           readFunction(currentFile)
         } catch {
           case e: FileNotFoundException =>
-            throw new FileNotFoundException(
-              e.getMessage + "\n" +
-                "It is possible the underlying files have been updated. " +
-                "You can explicitly invalidate the cache in Spark by " +
-                "running 'REFRESH TABLE tableName' command in SQL or " +
-                "by recreating the Dataset/DataFrame involved.")
+            throw QueryExecutionErrors.readCurrentFileNotFoundError(e)
         }
       }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
index 477937d66ad9b..1bfde7515dc92 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
@@ -25,6 +25,7 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.planning.ScanOperation
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.execution.{FileSourceScanExec, SparkPlan}
+import org.apache.spark.sql.types.{DoubleType, FloatType}
 import org.apache.spark.util.collection.BitSet
 
 /**
@@ -50,7 +51,7 @@ import org.apache.spark.util.collection.BitSet
  *     is under the threshold with the addition of the next file, add it.  If not, open a new bucket
  *     and add it.  Proceed to the next file.
  */
-object FileSourceStrategy extends Strategy with Logging {
+object FileSourceStrategy extends Strategy with PredicateHelper with Logging {
 
   // should prune buckets iff num buckets is greater than 1 and there is only one bucket column
   private def shouldPruneBuckets(bucketSpec: Option[BucketSpec]): Boolean = {
@@ -89,11 +90,16 @@ object FileSourceStrategy extends Strategy with Logging {
       case expressions.In(a: Attribute, list)
         if list.forall(_.isInstanceOf[Literal]) && a.name == bucketColumnName =>
         getBucketSetFromIterable(a, list.map(e => e.eval(EmptyRow)))
-      case expressions.InSet(a: Attribute, hset)
-        if hset.forall(_.isInstanceOf[Literal]) && a.name == bucketColumnName =>
-        getBucketSetFromIterable(a, hset.map(e => expressions.Literal(e).eval(EmptyRow)))
+      case expressions.InSet(a: Attribute, hset) if a.name == bucketColumnName =>
+        getBucketSetFromIterable(a, hset)
       case expressions.IsNull(a: Attribute) if a.name == bucketColumnName =>
         getBucketSetFromValue(a, null)
+      case expressions.IsNaN(a: Attribute)
+        if a.name == bucketColumnName && a.dataType == FloatType =>
+        getBucketSetFromValue(a, Float.NaN)
+      case expressions.IsNaN(a: Attribute)
+        if a.name == bucketColumnName && a.dataType == DoubleType =>
+        getBucketSetFromValue(a, Double.NaN)
       case expressions.And(left, right) =>
         getExpressionBuckets(left, bucketColumnName, numBuckets) &
           getExpressionBuckets(right, bucketColumnName, numBuckets)
@@ -154,11 +160,11 @@ object FileSourceStrategy extends Strategy with Logging {
         l.resolve(
           fsRelation.partitionSchema, fsRelation.sparkSession.sessionState.analyzer.resolver)
       val partitionSet = AttributeSet(partitionColumns)
-      val partitionKeyFilters =
-        ExpressionSet(normalizedFilters
-          .filter(_.references.subsetOf(partitionSet)))
 
-      logInfo(s"Pruning directories with: ${partitionKeyFilters.mkString(",")}")
+      // this partitionKeyFilters should be the same with the ones being executed in
+      // PruneFileSourcePartitions
+      val partitionKeyFilters = DataSourceStrategy.getPushedDownFilters(partitionColumns,
+        normalizedFilters)
 
       // subquery expressions are filtered out because they can't be used to prune buckets or pushed
       // down as data filters, yet they would be executed
@@ -176,8 +182,15 @@ object FileSourceStrategy extends Strategy with Logging {
         l.resolve(fsRelation.dataSchema, fsRelation.sparkSession.sessionState.analyzer.resolver)
 
       // Partition keys are not available in the statistics of the files.
-      val dataFilters =
-        normalizedFiltersWithoutSubqueries.filter(_.references.intersect(partitionSet).isEmpty)
+      // `dataColumns` might have partition columns, we need to filter them out.
+      val dataColumnsWithoutPartitionCols = dataColumns.filterNot(partitionColumns.contains)
+      val dataFilters = normalizedFiltersWithoutSubqueries.flatMap { f =>
+        if (f.references.intersect(partitionSet).nonEmpty) {
+          extractPredicatesWithinOutputSet(f, AttributeSet(dataColumnsWithoutPartitionCols))
+        } else {
+          Some(f)
+        }
+      }
       val supportNestedPredicatePushdown =
         DataSourceUtils.supportNestedPredicatePushdown(fsRelation)
       val pushedFilters = dataFilters
@@ -208,6 +221,7 @@ object FileSourceStrategy extends Strategy with Logging {
           outputSchema,
           partitionKeyFilters.toSeq,
           bucketSet,
+          None,
           dataFilters,
           table.map(_.identifier))
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileStatusCache.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileStatusCache.scala
index aea27bd4c4d7f..b5d800f02862e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileStatusCache.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileStatusCache.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.execution.datasources
 
+import java.util.concurrent.TimeUnit
 import java.util.concurrent.atomic.AtomicBoolean
 
 import scala.collection.JavaConverters._
@@ -44,7 +45,9 @@ object FileStatusCache {
       session.sqlContext.conf.filesourcePartitionFileCacheSize > 0) {
       if (sharedCache == null) {
         sharedCache = new SharedInMemoryCache(
-          session.sqlContext.conf.filesourcePartitionFileCacheSize)
+          session.sqlContext.conf.filesourcePartitionFileCacheSize,
+          session.sqlContext.conf.metadataCacheTTL
+        )
       }
       sharedCache.createForNewClient()
     } else {
@@ -89,7 +92,7 @@ abstract class FileStatusCache {
  *
  * @param maxSizeInBytes max allowable cache size before entries start getting evicted
  */
-private class SharedInMemoryCache(maxSizeInBytes: Long) extends Logging {
+private class SharedInMemoryCache(maxSizeInBytes: Long, cacheTTL: Long) extends Logging {
 
   // Opaque object that uniquely identifies a shared cache user
   private type ClientId = Object
@@ -129,11 +132,17 @@ private class SharedInMemoryCache(maxSizeInBytes: Long) extends Logging {
         }
       }
     }
-    CacheBuilder.newBuilder()
+
+    var builder = CacheBuilder.newBuilder()
       .weigher(weigher)
       .removalListener(removalListener)
       .maximumWeight(maxSizeInBytes / weightScale)
-      .build[(ClientId, Path), Array[FileStatus]]()
+
+    if (cacheTTL > 0) {
+      builder = builder.expireAfterWrite(cacheTTL, TimeUnit.SECONDS)
+    }
+
+    builder.build[(ClientId, Path), Array[FileStatus]]()
   }
 
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/HadoopFileLinesReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/HadoopFileLinesReader.scala
index 57082b40e1132..b5e276bd421a8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/HadoopFileLinesReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/HadoopFileLinesReader.scala
@@ -46,7 +46,7 @@ class HadoopFileLinesReader(
 
   def this(file: PartitionedFile, conf: Configuration) = this(file, None, conf)
 
-  private val iterator = {
+  private val _iterator = {
     val fileSplit = new FileSplit(
       new Path(new URI(file.filePath)),
       file.start,
@@ -66,9 +66,9 @@ class HadoopFileLinesReader(
     new RecordReaderIterator(reader)
   }
 
-  override def hasNext: Boolean = iterator.hasNext
+  override def hasNext: Boolean = _iterator.hasNext
 
-  override def next(): Text = iterator.next()
+  override def next(): Text = _iterator.next()
 
-  override def close(): Unit = iterator.close()
+  override def close(): Unit = _iterator.close()
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/HadoopFileWholeTextReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/HadoopFileWholeTextReader.scala
index 0e6d803f02d4d..a48001f04a9bb 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/HadoopFileWholeTextReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/HadoopFileWholeTextReader.scala
@@ -35,7 +35,7 @@ import org.apache.spark.input.WholeTextFileRecordReader
  */
 class HadoopFileWholeTextReader(file: PartitionedFile, conf: Configuration)
   extends Iterator[Text] with Closeable {
-  private val iterator = {
+  private val _iterator = {
     val fileSplit = new CombineFileSplit(
       Array(new Path(new URI(file.filePath))),
       Array(file.start),
@@ -50,9 +50,9 @@ class HadoopFileWholeTextReader(file: PartitionedFile, conf: Configuration)
     new RecordReaderIterator(reader)
   }
 
-  override def hasNext: Boolean = iterator.hasNext
+  override def hasNext: Boolean = _iterator.hasNext
 
-  override def next(): Text = iterator.next()
+  override def next(): Text = _iterator.next()
 
-  override def close(): Unit = iterator.close()
+  override def close(): Unit = _iterator.close()
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/HadoopFsRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/HadoopFsRelation.scala
index d278802e6c9f2..4ed8943ef46f4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/HadoopFsRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/HadoopFsRelation.scala
@@ -17,10 +17,6 @@
 
 package org.apache.spark.sql.execution.datasources
 
-import java.util.Locale
-
-import scala.collection.mutable
-
 import org.apache.spark.sql.{SparkSession, SQLContext}
 import org.apache.spark.sql.catalyst.catalog.BucketSpec
 import org.apache.spark.sql.execution.FileRelation
@@ -44,6 +40,8 @@ import org.apache.spark.sql.types.{StructField, StructType}
 case class HadoopFsRelation(
     location: FileIndex,
     partitionSchema: StructType,
+    // The top-level columns in `dataSchema` should match the actual physical file schema, otherwise
+    // the ORC data source may not work with the by-ordinal mode.
     dataSchema: StructType,
     bucketSpec: Option[BucketSpec],
     fileFormat: FileFormat,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala
index 84160f35540df..6c3deee2c3173 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala
@@ -17,23 +17,18 @@
 
 package org.apache.spark.sql.execution.datasources
 
-import java.io.FileNotFoundException
-
 import scala.collection.mutable
 
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs._
-import org.apache.hadoop.fs.viewfs.ViewFileSystem
-import org.apache.hadoop.hdfs.DistributedFileSystem
 import org.apache.hadoop.mapred.{FileInputFormat, JobConf}
 
-import org.apache.spark.SparkContext
 import org.apache.spark.internal.Logging
 import org.apache.spark.metrics.source.HiveCatalogMetrics
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.execution.streaming.FileStreamSink
 import org.apache.spark.sql.types.StructType
-import org.apache.spark.util.SerializableConfiguration
+import org.apache.spark.util.HadoopFSUtils
 
 
 /**
@@ -133,7 +128,7 @@ class InMemoryFileIndex(
     }
     val filter = FileInputFormat.getInputPathFilter(new JobConf(hadoopConf, this.getClass))
     val discovered = InMemoryFileIndex.bulkListLeafFiles(
-      pathsToFetch, hadoopConf, filter, sparkSession, areRootPaths = true)
+      pathsToFetch.toSeq, hadoopConf, filter, sparkSession)
     discovered.foreach { case (path, leafFiles) =>
       HiveCatalogMetrics.incrementFilesDiscovered(leafFiles.size)
       fileStatusCache.putLeafFiles(path, leafFiles.toArray)
@@ -147,298 +142,26 @@ class InMemoryFileIndex(
 
 object InMemoryFileIndex extends Logging {
 
-  /** A serializable variant of HDFS's BlockLocation. */
-  private case class SerializableBlockLocation(
-      names: Array[String],
-      hosts: Array[String],
-      offset: Long,
-      length: Long)
-
-  /** A serializable variant of HDFS's FileStatus. */
-  private case class SerializableFileStatus(
-      path: String,
-      length: Long,
-      isDir: Boolean,
-      blockReplication: Short,
-      blockSize: Long,
-      modificationTime: Long,
-      accessTime: Long,
-      blockLocations: Array[SerializableBlockLocation])
-
-  /**
-   * Lists a collection of paths recursively. Picks the listing strategy adaptively depending
-   * on the number of paths to list.
-   *
-   * This may only be called on the driver.
-   *
-   * @return for each input path, the set of discovered files for the path
-   */
   private[sql] def bulkListLeafFiles(
       paths: Seq[Path],
       hadoopConf: Configuration,
       filter: PathFilter,
-      sparkSession: SparkSession,
-      areRootPaths: Boolean): Seq[(Path, Seq[FileStatus])] = {
-
-    val ignoreMissingFiles = sparkSession.sessionState.conf.ignoreMissingFiles
-    val ignoreLocality = sparkSession.sessionState.conf.ignoreDataLocality
-
-    // Short-circuits parallel listing when serial listing is likely to be faster.
-    if (paths.size <= sparkSession.sessionState.conf.parallelPartitionDiscoveryThreshold) {
-      return paths.map { path =>
-        val leafFiles = listLeafFiles(
-          path,
-          hadoopConf,
-          filter,
-          Some(sparkSession),
-          ignoreMissingFiles = ignoreMissingFiles,
-          ignoreLocality = ignoreLocality,
-          isRootPath = areRootPaths)
-        (path, leafFiles)
-      }
-    }
-
-    logInfo(s"Listing leaf files and directories in parallel under ${paths.length} paths." +
-      s" The first several paths are: ${paths.take(10).mkString(", ")}.")
-    HiveCatalogMetrics.incrementParallelListingJobCount(1)
-
-    val sparkContext = sparkSession.sparkContext
-    val serializableConfiguration = new SerializableConfiguration(hadoopConf)
-    val serializedPaths = paths.map(_.toString)
-    val parallelPartitionDiscoveryParallelism =
-      sparkSession.sessionState.conf.parallelPartitionDiscoveryParallelism
-
-    // Set the number of parallelism to prevent following file listing from generating many tasks
-    // in case of large #defaultParallelism.
-    val numParallelism = Math.min(paths.size, parallelPartitionDiscoveryParallelism)
-
-    val previousJobDescription = sparkContext.getLocalProperty(SparkContext.SPARK_JOB_DESCRIPTION)
-    val statusMap = try {
-      val description = paths.size match {
-        case 0 =>
-          s"Listing leaf files and directories 0 paths"
-        case 1 =>
-          s"Listing leaf files and directories for 1 path:<br/>${paths(0)}"
-        case s =>
-          s"Listing leaf files and directories for $s paths:<br/>${paths(0)}, ..."
-      }
-      sparkContext.setJobDescription(description)
-      sparkContext
-        .parallelize(serializedPaths, numParallelism)
-        .mapPartitions { pathStrings =>
-          val hadoopConf = serializableConfiguration.value
-          pathStrings.map(new Path(_)).toSeq.map { path =>
-            val leafFiles = listLeafFiles(
-              path,
-              hadoopConf,
-              filter,
-              None,
-              ignoreMissingFiles = ignoreMissingFiles,
-              ignoreLocality = ignoreLocality,
-              isRootPath = areRootPaths)
-            (path, leafFiles)
-          }.iterator
-        }.map { case (path, statuses) =>
-        val serializableStatuses = statuses.map { status =>
-          // Turn FileStatus into SerializableFileStatus so we can send it back to the driver
-          val blockLocations = status match {
-            case f: LocatedFileStatus =>
-              f.getBlockLocations.map { loc =>
-                SerializableBlockLocation(
-                  loc.getNames,
-                  loc.getHosts,
-                  loc.getOffset,
-                  loc.getLength)
-              }
-
-            case _ =>
-              Array.empty[SerializableBlockLocation]
-          }
-
-          SerializableFileStatus(
-            status.getPath.toString,
-            status.getLen,
-            status.isDirectory,
-            status.getReplication,
-            status.getBlockSize,
-            status.getModificationTime,
-            status.getAccessTime,
-            blockLocations)
-        }
-        (path.toString, serializableStatuses)
-      }.collect()
-    } finally {
-      sparkContext.setJobDescription(previousJobDescription)
-    }
-
-    // turn SerializableFileStatus back to Status
-    statusMap.map { case (path, serializableStatuses) =>
-      val statuses = serializableStatuses.map { f =>
-        val blockLocations = f.blockLocations.map { loc =>
-          new BlockLocation(loc.names, loc.hosts, loc.offset, loc.length)
-        }
-        new LocatedFileStatus(
-          new FileStatus(
-            f.length, f.isDir, f.blockReplication, f.blockSize, f.modificationTime,
-            new Path(f.path)),
-          blockLocations)
-      }
-      (new Path(path), statuses)
-    }
+      sparkSession: SparkSession): Seq[(Path, Seq[FileStatus])] = {
+    HadoopFSUtils.parallelListLeafFiles(
+      sc = sparkSession.sparkContext,
+      paths = paths,
+      hadoopConf = hadoopConf,
+      filter = new PathFilterWrapper(filter),
+      ignoreMissingFiles = sparkSession.sessionState.conf.ignoreMissingFiles,
+      ignoreLocality = sparkSession.sessionState.conf.ignoreDataLocality,
+      parallelismThreshold = sparkSession.sessionState.conf.parallelPartitionDiscoveryThreshold,
+      parallelismMax = sparkSession.sessionState.conf.parallelPartitionDiscoveryParallelism)
   }
 
-  /**
-   * Lists a single filesystem path recursively. If a SparkSession object is specified, this
-   * function may launch Spark jobs to parallelize listing.
-   *
-   * If sessionOpt is None, this may be called on executors.
-   *
-   * @return all children of path that match the specified filter.
-   */
-  private def listLeafFiles(
-      path: Path,
-      hadoopConf: Configuration,
-      filter: PathFilter,
-      sessionOpt: Option[SparkSession],
-      ignoreMissingFiles: Boolean,
-      ignoreLocality: Boolean,
-      isRootPath: Boolean): Seq[FileStatus] = {
-    logTrace(s"Listing $path")
-    val fs = path.getFileSystem(hadoopConf)
-
-    // Note that statuses only include FileStatus for the files and dirs directly under path,
-    // and does not include anything else recursively.
-    val statuses: Array[FileStatus] = try {
-      fs match {
-        // DistributedFileSystem overrides listLocatedStatus to make 1 single call to namenode
-        // to retrieve the file status with the file block location. The reason to still fallback
-        // to listStatus is because the default implementation would potentially throw a
-        // FileNotFoundException which is better handled by doing the lookups manually below.
-        case (_: DistributedFileSystem | _: ViewFileSystem) if !ignoreLocality =>
-          val remoteIter = fs.listLocatedStatus(path)
-          new Iterator[LocatedFileStatus]() {
-            def next(): LocatedFileStatus = remoteIter.next
-            def hasNext(): Boolean = remoteIter.hasNext
-          }.toArray
-        case _ => fs.listStatus(path)
-      }
-    } catch {
-      // If we are listing a root path (e.g. a top level directory of a table), we need to
-      // ignore FileNotFoundExceptions during this root level of the listing because
-      //
-      //  (a) certain code paths might construct an InMemoryFileIndex with root paths that
-      //      might not exist (i.e. not all callers are guaranteed to have checked
-      //      path existence prior to constructing InMemoryFileIndex) and,
-      //  (b) we need to ignore deleted root paths during REFRESH TABLE, otherwise we break
-      //      existing behavior and break the ability drop SessionCatalog tables when tables'
-      //      root directories have been deleted (which breaks a number of Spark's own tests).
-      //
-      // If we are NOT listing a root path then a FileNotFoundException here means that the
-      // directory was present in a previous level of file listing but is absent in this
-      // listing, likely indicating a race condition (e.g. concurrent table overwrite or S3
-      // list inconsistency).
-      //
-      // The trade-off in supporting existing behaviors / use-cases is that we won't be
-      // able to detect race conditions involving root paths being deleted during
-      // InMemoryFileIndex construction. However, it's still a net improvement to detect and
-      // fail-fast on the non-root cases. For more info see the SPARK-27676 review discussion.
-      case _: FileNotFoundException if isRootPath || ignoreMissingFiles =>
-        logWarning(s"The directory $path was not found. Was it deleted very recently?")
-        Array.empty[FileStatus]
-    }
-
-    val filteredStatuses = statuses.filterNot(status => shouldFilterOut(status.getPath.getName))
-
-    val allLeafStatuses = {
-      val (dirs, topLevelFiles) = filteredStatuses.partition(_.isDirectory)
-      val nestedFiles: Seq[FileStatus] = sessionOpt match {
-        case Some(session) =>
-          bulkListLeafFiles(
-            dirs.map(_.getPath),
-            hadoopConf,
-            filter,
-            session,
-            areRootPaths = false
-          ).flatMap(_._2)
-        case _ =>
-          dirs.flatMap { dir =>
-            listLeafFiles(
-              dir.getPath,
-              hadoopConf,
-              filter,
-              sessionOpt,
-              ignoreMissingFiles = ignoreMissingFiles,
-              ignoreLocality = ignoreLocality,
-              isRootPath = false)
-          }
-      }
-      val allFiles = topLevelFiles ++ nestedFiles
-      if (filter != null) allFiles.filter(f => filter.accept(f.getPath)) else allFiles
-    }
-
-    val missingFiles = mutable.ArrayBuffer.empty[String]
-    val filteredLeafStatuses = allLeafStatuses.filterNot(
-      status => shouldFilterOut(status.getPath.getName))
-    val resolvedLeafStatuses = filteredLeafStatuses.flatMap {
-      case f: LocatedFileStatus =>
-        Some(f)
-
-      // NOTE:
-      //
-      // - Although S3/S3A/S3N file system can be quite slow for remote file metadata
-      //   operations, calling `getFileBlockLocations` does no harm here since these file system
-      //   implementations don't actually issue RPC for this method.
-      //
-      // - Here we are calling `getFileBlockLocations` in a sequential manner, but it should not
-      //   be a big deal since we always use to `bulkListLeafFiles` when the number of
-      //   paths exceeds threshold.
-      case f if !ignoreLocality =>
-        // The other constructor of LocatedFileStatus will call FileStatus.getPermission(),
-        // which is very slow on some file system (RawLocalFileSystem, which is launch a
-        // subprocess and parse the stdout).
-        try {
-          val locations = fs.getFileBlockLocations(f, 0, f.getLen).map { loc =>
-            // Store BlockLocation objects to consume less memory
-            if (loc.getClass == classOf[BlockLocation]) {
-              loc
-            } else {
-              new BlockLocation(loc.getNames, loc.getHosts, loc.getOffset, loc.getLength)
-            }
-          }
-          val lfs = new LocatedFileStatus(f.getLen, f.isDirectory, f.getReplication, f.getBlockSize,
-            f.getModificationTime, 0, null, null, null, null, f.getPath, locations)
-          if (f.isSymlink) {
-            lfs.setSymlink(f.getSymlink)
-          }
-          Some(lfs)
-        } catch {
-          case _: FileNotFoundException if ignoreMissingFiles =>
-            missingFiles += f.getPath.toString
-            None
-        }
-
-      case f => Some(f)
-    }
-
-    if (missingFiles.nonEmpty) {
-      logWarning(
-        s"the following files were missing during file scan:\n  ${missingFiles.mkString("\n  ")}")
-    }
-
-    resolvedLeafStatuses
-  }
+}
 
-  /** Checks if we should filter out this path name. */
-  def shouldFilterOut(pathName: String): Boolean = {
-    // We filter follow paths:
-    // 1. everything that starts with _ and ., except _common_metadata and _metadata
-    // because Parquet needs to find those metadata files from leaf files returned by this method.
-    // We should refactor this logic to not mix metadata files with data files.
-    // 2. everything that ends with `._COPYING_`, because this is a intermediate state of file. we
-    // should skip this file in case of double reading.
-    val exclude = (pathName.startsWith("_") && !pathName.contains("=")) ||
-      pathName.startsWith(".") || pathName.endsWith("._COPYING_")
-    val include = pathName.startsWith("_common_metadata") || pathName.startsWith("_metadata")
-    exclude && !include
+private class PathFilterWrapper(val filter: PathFilter) extends PathFilter with Serializable {
+  override def accept(path: Path): Boolean = {
+    (filter == null || filter.accept(path)) && !HadoopFSUtils.shouldFilterOutPathName(path.getName)
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoDataSourceCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoDataSourceCommand.scala
index bd9cc0e44fca3..789b1d714fcb6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoDataSourceCommand.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoDataSourceCommand.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.execution.datasources
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.plans.QueryPlan
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.execution.command.RunnableCommand
+import org.apache.spark.sql.execution.command.LeafRunnableCommand
 import org.apache.spark.sql.sources.InsertableRelation
 
 
@@ -31,7 +31,7 @@ case class InsertIntoDataSourceCommand(
     logicalRelation: LogicalRelation,
     query: LogicalPlan,
     overwrite: Boolean)
-  extends RunnableCommand {
+  extends LeafRunnableCommand {
 
   override def innerChildren: Seq[QueryPlan[_]] = Seq(query)
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelationCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelationCommand.scala
index f11972115e09f..267b360b474ca 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelationCommand.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelationCommand.scala
@@ -17,20 +17,19 @@
 
 package org.apache.spark.sql.execution.datasources
 
-import java.io.IOException
-
 import org.apache.hadoop.fs.{FileSystem, Path}
 
 import org.apache.spark.internal.io.FileCommitProtocol
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTable, CatalogTablePartition}
 import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
+import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils._
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
+import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
 import org.apache.spark.sql.execution.SparkPlan
 import org.apache.spark.sql.execution.command._
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.PartitionOverwriteMode
 import org.apache.spark.sql.util.SchemaUtils
 
@@ -59,7 +58,6 @@ case class InsertIntoHadoopFsRelationCommand(
     fileIndex: Option[FileIndex],
     outputColumnNames: Seq[String])
   extends DataWritingCommand {
-  import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils.escapePathName
 
   private lazy val parameters = CaseInsensitiveMap(options)
 
@@ -68,7 +66,7 @@ case class InsertIntoHadoopFsRelationCommand(
       // scalastyle:off caselocale
       .map(mode => PartitionOverwriteMode.withName(mode.toUpperCase))
       // scalastyle:on caselocale
-      .getOrElse(SQLConf.get.partitionOverwriteMode)
+      .getOrElse(conf.partitionOverwriteMode)
     val enableDynamicOverwrite = partitionOverwriteMode == PartitionOverwriteMode.DYNAMIC
     // This config only makes sense when we are overwriting a partitioned dataset with dynamic
     // partition columns.
@@ -106,9 +104,10 @@ case class InsertIntoHadoopFsRelationCommand(
         fs, catalogTable.get, qualifiedOutputPath, matchingPartitions)
     }
 
+    val jobId = java.util.UUID.randomUUID().toString
     val committer = FileCommitProtocol.instantiate(
       sparkSession.sessionState.conf.fileCommitProtocolClass,
-      jobId = java.util.UUID.randomUUID().toString,
+      jobId = jobId,
       outputPath = outputPath.toString,
       dynamicPartitionOverwrite = dynamicPartitionOverwrite)
 
@@ -118,7 +117,7 @@ case class InsertIntoHadoopFsRelationCommand(
       val pathExists = fs.exists(qualifiedOutputPath)
       (mode, pathExists) match {
         case (SaveMode.ErrorIfExists, true) =>
-          throw new AnalysisException(s"path $qualifiedOutputPath already exists.")
+          throw QueryCompilationErrors.outputPathAlreadyExistsError(qualifiedOutputPath)
         case (SaveMode.Overwrite, true) =>
           if (ifPartitionNotExists && matchingPartitions.nonEmpty) {
             false
@@ -134,7 +133,7 @@ case class InsertIntoHadoopFsRelationCommand(
         case (SaveMode.Ignore, exists) =>
           !exists
         case (s, exists) =>
-          throw new IllegalStateException(s"unsupported save mode $s ($exists)")
+          throw QueryExecutionErrors.unsupportedSaveModeError(s.toString, exists)
       }
     }
 
@@ -163,6 +162,15 @@ case class InsertIntoHadoopFsRelationCommand(
         }
       }
 
+      // For dynamic partition overwrite, FileOutputCommitter's output path is staging path, files
+      // will be renamed from staging path to final output path during commit job
+      val committerOutputPath = if (dynamicPartitionOverwrite) {
+        FileCommitProtocol.getStagingDir(outputPath.toString, jobId)
+          .makeQualified(fs.getUri, fs.getWorkingDirectory)
+      } else {
+        qualifiedOutputPath
+      }
+
       val updatedPartitionPaths =
         FileFormatWriter.write(
           sparkSession = sparkSession,
@@ -170,7 +178,7 @@ case class InsertIntoHadoopFsRelationCommand(
           fileFormat = fileFormat,
           committer = committer,
           outputSpec = FileFormatWriter.OutputSpec(
-            qualifiedOutputPath.toString, customPartitionLocations, outputColumns),
+            committerOutputPath.toString, customPartitionLocations, outputColumns),
           hadoopConf = hadoopConf,
           partitionColumns = partitionColumns,
           bucketSpec = bucketSpec,
@@ -192,7 +200,7 @@ case class InsertIntoHadoopFsRelationCommand(
       // refresh cached files in FileIndex
       fileIndex.foreach(_.refresh())
       // refresh data cache if table is cached
-      sparkSession.catalog.refreshByPath(outputPath.toString)
+      sparkSession.sharedState.cacheManager.recacheByPath(sparkSession, outputPath, fs)
 
       if (catalogTable.nonEmpty) {
         CommandUtils.updateTableStats(sparkSession, catalogTable.get)
@@ -216,12 +224,7 @@ case class InsertIntoHadoopFsRelationCommand(
       committer: FileCommitProtocol): Unit = {
     val staticPartitionPrefix = if (staticPartitions.nonEmpty) {
       "/" + partitionColumns.flatMap { p =>
-        staticPartitions.get(p.name) match {
-          case Some(value) =>
-            Some(escapePathName(p.name) + "=" + escapePathName(value))
-          case None =>
-            None
-        }
+        staticPartitions.get(p.name).map(getPartitionPathString(p.name, _))
       }.mkString("/")
     } else {
       ""
@@ -229,8 +232,7 @@ case class InsertIntoHadoopFsRelationCommand(
     // first clear the path determined by the static partition keys (e.g. /table/foo=1)
     val staticPrefixPath = qualifiedOutputPath.suffix(staticPartitionPrefix)
     if (fs.exists(staticPrefixPath) && !committer.deleteWithJob(fs, staticPrefixPath, true)) {
-      throw new IOException(s"Unable to clear output " +
-        s"directory $staticPrefixPath prior to writing to it")
+      throw QueryExecutionErrors.cannotClearOutputDirectoryError(staticPrefixPath)
     }
     // now clear all custom partition locations (e.g. /custom/dir/where/foo=2/bar=4)
     for ((spec, customLoc) <- customPartitionLocations) {
@@ -239,8 +241,7 @@ case class InsertIntoHadoopFsRelationCommand(
         "Custom partition location did not match static partitioning keys")
       val path = new Path(customLoc)
       if (fs.exists(path) && !committer.deleteWithJob(fs, path, true)) {
-        throw new IOException(s"Unable to clear partition " +
-          s"directory $path prior to writing to it")
+        throw QueryExecutionErrors.cannotClearPartitionDirectoryError(path)
       }
     }
   }
@@ -269,4 +270,7 @@ case class InsertIntoHadoopFsRelationCommand(
       }
     }.toMap
   }
+
+  override protected def withNewChildInternal(
+    newChild: LogicalPlan): InsertIntoHadoopFsRelationCommand = copy(query = newChild)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/LogicalRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/LogicalRelation.scala
index 33a3486bf6f67..cfe5f046bd7b4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/LogicalRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/LogicalRelation.scala
@@ -21,7 +21,7 @@ import org.apache.spark.sql.catalyst.catalog.CatalogTable
 import org.apache.spark.sql.catalyst.expressions.{AttributeMap, AttributeReference}
 import org.apache.spark.sql.catalyst.plans.QueryPlan
 import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, Statistics}
-import org.apache.spark.sql.catalyst.util.truncatedString
+import org.apache.spark.sql.catalyst.util.{truncatedString, CharVarcharUtils}
 import org.apache.spark.sql.sources.BaseRelation
 
 /**
@@ -64,14 +64,23 @@ case class LogicalRelation(
   }
 
   override def simpleString(maxFields: Int): String = {
-    s"Relation[${truncatedString(output, ",", maxFields)}] $relation"
+    s"Relation ${catalogTable.map(_.identifier.unquotedString).getOrElse("")}" +
+      s"[${truncatedString(output, ",", maxFields)}] $relation"
   }
 }
 
 object LogicalRelation {
-  def apply(relation: BaseRelation, isStreaming: Boolean = false): LogicalRelation =
-    LogicalRelation(relation, relation.schema.toAttributes, None, isStreaming)
+  def apply(relation: BaseRelation, isStreaming: Boolean = false): LogicalRelation = {
+    // The v1 source may return schema containing char/varchar type. We replace char/varchar
+    // with "annotated" string type here as the query engine doesn't support char/varchar yet.
+    val schema = CharVarcharUtils.replaceCharVarcharWithStringInSchema(relation.schema)
+    LogicalRelation(relation, schema.toAttributes, None, isStreaming)
+  }
 
-  def apply(relation: BaseRelation, table: CatalogTable): LogicalRelation =
-    LogicalRelation(relation, relation.schema.toAttributes, Some(table), false)
+  def apply(relation: BaseRelation, table: CatalogTable): LogicalRelation = {
+    // The v1 source may return schema containing char/varchar type. We replace char/varchar
+    // with "annotated" string type here as the query engine doesn't support char/varchar yet.
+    val schema = CharVarcharUtils.replaceCharVarcharWithStringInSchema(relation.schema)
+    LogicalRelation(relation, schema.toAttributes, Some(table), false)
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/OutputWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/OutputWriter.scala
index 868e5371426c0..7c479d986f3e9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/OutputWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/OutputWriter.scala
@@ -19,8 +19,7 @@ package org.apache.spark.sql.execution.datasources
 
 import org.apache.hadoop.mapreduce.TaskAttemptContext
 
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
+import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.types.StructType
 
 
@@ -58,7 +57,7 @@ abstract class OutputWriterFactory extends Serializable {
  */
 abstract class OutputWriter {
   /**
-   * Persists a single row.  Invoked on the executor side.  When writing to dynamically partitioned
+   * Persists a single row. Invoked on the executor side. When writing to dynamically partitioned
    * tables, dynamic partition columns are not included in rows to be written.
    */
   def write(row: InternalRow): Unit
@@ -68,4 +67,9 @@ abstract class OutputWriter {
    * the task output is committed.
    */
   def close(): Unit
+
+  /**
+   * The file path to write. Invoked on the executor side.
+   */
+  def path(): String
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala
index 2e09c729529a6..5b0d0606da093 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala
@@ -27,7 +27,7 @@ import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.{expressions, InternalRow}
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateTimeUtils}
-import org.apache.spark.sql.types.{StringType, StructType}
+import org.apache.spark.sql.types.StructType
 
 /**
  * An abstract class that represents [[FileIndex]]s that are aware of partitioned tables.
@@ -56,14 +56,14 @@ abstract class PartitioningAwareFileIndex(
 
   protected def leafDirToChildrenFiles: Map[Path, Array[FileStatus]]
 
-  protected lazy val pathGlobFilter = parameters.get("pathGlobFilter").map(new GlobFilter(_))
+  private val caseInsensitiveMap = CaseInsensitiveMap(parameters)
+  private val pathFilters = PathFilterFactory.create(caseInsensitiveMap)
 
-  protected def matchGlobPattern(file: FileStatus): Boolean = {
-    pathGlobFilter.forall(_.accept(file.getPath))
-  }
+  protected def matchPathPattern(file: FileStatus): Boolean =
+    pathFilters.forall(_.accept(file))
 
-  protected lazy val recursiveFileLookup = {
-    parameters.getOrElse("recursiveFileLookup", "false").toBoolean
+  protected lazy val recursiveFileLookup: Boolean = {
+    caseInsensitiveMap.getOrElse("recursiveFileLookup", "false").toBoolean
   }
 
   override def listFiles(
@@ -83,7 +83,7 @@ abstract class PartitioningAwareFileIndex(
           val files: Seq[FileStatus] = leafDirToChildrenFiles.get(path) match {
             case Some(existingDir) =>
               // Directory has children files in it, return them
-              existingDir.filter(f => matchGlobPattern(f) && isNonEmptyFile(f))
+              existingDir.filter(f => matchPathPattern(f) && isNonEmptyFile(f))
 
             case None =>
               // Directory does not exist, or has no children files
@@ -132,7 +132,7 @@ abstract class PartitioningAwareFileIndex(
     } else {
       leafFiles.values.toSeq
     }
-    files.filter(matchGlobPattern)
+    files.filter(matchPathPattern)
   }
 
   protected def inferPartitioning(): PartitionSpec = {
@@ -215,7 +215,7 @@ abstract class PartitioningAwareFileIndex(
    * and the returned DataFrame will have the column of `something`.
    */
   private def basePaths: Set[Path] = {
-    parameters.get(BASE_PATH_PARAM).map(new Path(_)) match {
+    caseInsensitiveMap.get(BASE_PATH_PARAM).map(new Path(_)) match {
       case Some(userDefinedBasePath) =>
         val fs = userDefinedBasePath.getFileSystem(hadoopConf)
         if (!fs.isDirectory(userDefinedBasePath)) {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
index 292ac6db04baf..1cd5e14264723 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
@@ -25,17 +25,20 @@ import java.util.Locale
 import scala.collection.mutable
 import scala.collection.mutable.ArrayBuffer
 import scala.util.Try
+import scala.util.control.NonFatal
 
 import org.apache.hadoop.fs.Path
 
-import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.{Resolver, TypeCoercion}
+import org.apache.spark.sql.catalyst.analysis.TypeCoercion
 import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
-import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Cast, Literal}
+import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils.getPartitionValueString
+import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, Literal}
 import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateFormatter, DateTimeUtils, TimestampFormatter}
+import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.util.SchemaUtils
+import org.apache.spark.unsafe.types.UTF8String
 
 // TODO: We should tighten up visibility of the classes here once we clean up Hive coupling.
 
@@ -62,9 +65,11 @@ object PartitioningUtils {
 
   val timestampPartitionPattern = "yyyy-MM-dd HH:mm:ss[.S]"
 
-  private[datasources] case class PartitionValues(columnNames: Seq[String], literals: Seq[Literal])
+  case class TypedPartValue(value: String, dataType: DataType)
+
+  case class PartitionValues(columnNames: Seq[String], typedValues: Seq[TypedPartValue])
   {
-    require(columnNames.size == literals.size)
+    require(columnNames.size == typedValues.size)
   }
 
   import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils.{escapePathName, unescapePathName, DEFAULT_PARTITION_NAME}
@@ -134,7 +139,7 @@ object PartitioningUtils {
     val timestampFormatter = TimestampFormatter(
       timestampPartitionPattern,
       zoneId,
-      needVarLengthSecondFraction = true)
+      isParsing = true)
     // First, we need to parse every partition's path and see if we can find partition values.
     val (partitionValues, optDiscoveredBasePaths) = paths.map { path =>
       parsePartition(path, typeInference, basePaths, userSpecifiedDataTypes,
@@ -172,13 +177,12 @@ object PartitioningUtils {
           "root directory of the table. If there are multiple root directories, " +
           "please load them separately and then union them.")
 
-      val resolvedPartitionValues =
-        resolvePartitions(pathsWithPartitionValues, caseSensitive, zoneId)
+      val resolvedPartitionValues = resolvePartitions(pathsWithPartitionValues, caseSensitive)
 
       // Creates the StructType which represents the partition columns.
       val fields = {
-        val PartitionValues(columnNames, literals) = resolvedPartitionValues.head
-        columnNames.zip(literals).map { case (name, Literal(_, dataType)) =>
+        val PartitionValues(columnNames, typedValues) = resolvedPartitionValues.head
+        columnNames.zip(typedValues).map { case (name, TypedPartValue(_, dataType)) =>
           // We always assume partition columns are nullable since we've no idea whether null values
           // will be appended in the future.
           val resultName = userSpecifiedNames.getOrElse(name, name)
@@ -189,8 +193,19 @@ object PartitioningUtils {
 
       // Finally, we create `Partition`s based on paths and resolved partition values.
       val partitions = resolvedPartitionValues.zip(pathsWithPartitionValues).map {
-        case (PartitionValues(_, literals), (path, _)) =>
-          PartitionPath(InternalRow.fromSeq(literals.map(_.value)), path)
+        case (PartitionValues(columnNames, typedValues), (path, _)) =>
+          val rowValues = columnNames.zip(typedValues).map { case (columnName, typedValue) =>
+            try {
+              castPartValueToDesiredType(typedValue.dataType, typedValue.value, zoneId)
+            } catch {
+              case NonFatal(_) =>
+                if (validatePartitionColumns) {
+                  throw QueryExecutionErrors.failedToCastValueToDataTypeForPartitionColumnError(
+                    typedValue.value, typedValue.dataType, columnName)
+                } else null
+            }
+          }
+          PartitionPath(InternalRow.fromSeq(rowValues), path)
       }
 
       PartitionSpec(StructType(fields), partitions)
@@ -226,7 +241,7 @@ object PartitioningUtils {
       zoneId: ZoneId,
       dateFormatter: DateFormatter,
       timestampFormatter: TimestampFormatter): (Option[PartitionValues], Option[Path]) = {
-    val columns = ArrayBuffer.empty[(String, Literal)]
+    val columns = ArrayBuffer.empty[(String, TypedPartValue)]
     // Old Hadoop versions don't have `Path.isRoot`
     var finished = path.getParent == null
     // currentPath is the current path that we will use to parse partition column value.
@@ -273,7 +288,7 @@ object PartitioningUtils {
       (None, Some(path))
     } else {
       val (columnNames, values) = columns.reverse.unzip
-      (Some(PartitionValues(columnNames, values)), Some(currentPath))
+      (Some(PartitionValues(columnNames.toSeq, values.toSeq)), Some(currentPath))
     }
   }
 
@@ -284,7 +299,7 @@ object PartitioningUtils {
       validatePartitionColumns: Boolean,
       zoneId: ZoneId,
       dateFormatter: DateFormatter,
-      timestampFormatter: TimestampFormatter): Option[(String, Literal)] = {
+      timestampFormatter: TimestampFormatter): Option[(String, TypedPartValue)] = {
     val equalSignIndex = columnSpec.indexOf('=')
     if (equalSignIndex == -1) {
       None
@@ -295,23 +310,10 @@ object PartitioningUtils {
       val rawColumnValue = columnSpec.drop(equalSignIndex + 1)
       assert(rawColumnValue.nonEmpty, s"Empty partition column value in '$columnSpec'")
 
-      val literal = if (userSpecifiedDataTypes.contains(columnName)) {
+      val dataType = if (userSpecifiedDataTypes.contains(columnName)) {
         // SPARK-26188: if user provides corresponding column schema, get the column value without
         //              inference, and then cast it as user specified data type.
-        val dataType = userSpecifiedDataTypes(columnName)
-        val columnValueLiteral = inferPartitionColumnValue(
-          rawColumnValue,
-          false,
-          zoneId,
-          dateFormatter,
-          timestampFormatter)
-        val columnValue = columnValueLiteral.eval()
-        val castedValue = Cast(columnValueLiteral, dataType, Option(zoneId.getId)).eval()
-        if (validatePartitionColumns && columnValue != null && castedValue == null) {
-          throw new RuntimeException(s"Failed to cast value `$columnValue` to `$dataType` " +
-            s"for partition column `$columnName`")
-        }
-        Literal.create(castedValue, dataType)
+        userSpecifiedDataTypes(columnName)
       } else {
         inferPartitionColumnValue(
           rawColumnValue,
@@ -320,7 +322,7 @@ object PartitioningUtils {
           dateFormatter,
           timestampFormatter)
       }
-      Some(columnName -> literal)
+      Some(columnName -> TypedPartValue(rawColumnValue, dataType))
     }
   }
 
@@ -349,7 +351,7 @@ object PartitioningUtils {
    */
   def getPathFragment(spec: TablePartitionSpec, partitionSchema: StructType): String = {
     partitionSchema.map { field =>
-      escapePathName(field.name) + "=" + escapePathName(spec(field.name))
+      escapePathName(field.name) + "=" + getPartitionValueString(spec(field.name))
     }.mkString("/")
   }
 
@@ -357,38 +359,13 @@ object PartitioningUtils {
     getPathFragment(spec, StructType.fromAttributes(partitionColumns))
   }
 
-  /**
-   * Normalize the column names in partition specification, w.r.t. the real partition column names
-   * and case sensitivity. e.g., if the partition spec has a column named `monTh`, and there is a
-   * partition column named `month`, and it's case insensitive, we will normalize `monTh` to
-   * `month`.
-   */
-  def normalizePartitionSpec[T](
-      partitionSpec: Map[String, T],
-      partColNames: Seq[String],
-      tblName: String,
-      resolver: Resolver): Map[String, T] = {
-    val normalizedPartSpec = partitionSpec.toSeq.map { case (key, value) =>
-      val normalizedKey = partColNames.find(resolver(_, key)).getOrElse {
-        throw new AnalysisException(s"$key is not a valid partition column in table $tblName.")
-      }
-      normalizedKey -> value
-    }
-
-    SchemaUtils.checkColumnNameDuplication(
-      normalizedPartSpec.map(_._1), "in the partition schema", resolver)
-
-    normalizedPartSpec.toMap
-  }
-
   /**
    * Resolves possible type conflicts between partitions by up-casting "lower" types using
    * [[findWiderTypeForPartitionColumn]].
    */
   def resolvePartitions(
       pathsWithPartitionValues: Seq[(Path, PartitionValues)],
-      caseSensitive: Boolean,
-      zoneId: ZoneId): Seq[PartitionValues] = {
+      caseSensitive: Boolean): Seq[PartitionValues] = {
     if (pathsWithPartitionValues.isEmpty) {
       Seq.empty
     } else {
@@ -405,12 +382,12 @@ object PartitioningUtils {
       val values = pathsWithPartitionValues.map(_._2)
       val columnCount = values.head.columnNames.size
       val resolvedValues = (0 until columnCount).map { i =>
-        resolveTypeConflicts(values.map(_.literals(i)), zoneId)
+        resolveTypeConflicts(values.map(_.typedValues(i)))
       }
 
       // Fills resolved literals back to each partition
       values.zipWithIndex.map { case (d, index) =>
-        d.copy(literals = resolvedValues.map(_(index)))
+        d.copy(typedValues = resolvedValues.map(_(index)))
       }
     }
   }
@@ -420,7 +397,7 @@ object PartitioningUtils {
     val distinctPartColNames = pathWithPartitionValues.map(_._2.columnNames).distinct
 
     def groupByKey[K, V](seq: Seq[(K, V)]): Map[K, Iterable[V]] =
-      seq.groupBy { case (key, _) => key }.mapValues(_.map { case (_, value) => value })
+      seq.groupBy { case (key, _) => key }.mapValues(_.map { case (_, value) => value }).toMap
 
     val partColNamesToPaths = groupByKey(pathWithPartitionValues.map {
       case (path, partValues) => partValues.columnNames -> path
@@ -473,16 +450,16 @@ object PartitioningUtils {
       typeInference: Boolean,
       zoneId: ZoneId,
       dateFormatter: DateFormatter,
-      timestampFormatter: TimestampFormatter): Literal = {
+      timestampFormatter: TimestampFormatter): DataType = {
     val decimalTry = Try {
       // `BigDecimal` conversion can fail when the `field` is not a form of number.
       val bigDecimal = new JBigDecimal(raw)
-      // It reduces the cases for decimals by disallowing values having scale (eg. `1.1`).
+      // It reduces the cases for decimals by disallowing values having scale (e.g. `1.1`).
       require(bigDecimal.scale <= 0)
       // `DecimalType` conversion can fail when
       //   1. The precision is bigger than 38.
       //   2. scale is bigger than precision.
-      Literal(bigDecimal)
+      DecimalType.fromDecimal(Decimal(bigDecimal))
     }
 
     val dateTry = Try {
@@ -498,7 +475,7 @@ object PartitioningUtils {
       val dateValue = Cast(Literal(raw), DateType, Some(zoneId.getId)).eval()
       // Disallow DateType if the cast returned null
       require(dateValue != null)
-      Literal.create(dateValue, DateType)
+      DateType
     }
 
     val timestampTry = Try {
@@ -510,50 +487,67 @@ object PartitioningUtils {
       val timestampValue = Cast(Literal(unescapedRaw), TimestampType, Some(zoneId.getId)).eval()
       // Disallow TimestampType if the cast returned null
       require(timestampValue != null)
-      Literal.create(timestampValue, TimestampType)
+      TimestampType
     }
 
     if (typeInference) {
       // First tries integral types
-      Try(Literal.create(Integer.parseInt(raw), IntegerType))
-        .orElse(Try(Literal.create(JLong.parseLong(raw), LongType)))
+      Try({ Integer.parseInt(raw); IntegerType })
+        .orElse(Try { JLong.parseLong(raw); LongType })
         .orElse(decimalTry)
         // Then falls back to fractional types
-        .orElse(Try(Literal.create(JDouble.parseDouble(raw), DoubleType)))
+        .orElse(Try { JDouble.parseDouble(raw); DoubleType })
         // Then falls back to date/timestamp types
         .orElse(timestampTry)
         .orElse(dateTry)
         // Then falls back to string
         .getOrElse {
-          if (raw == DEFAULT_PARTITION_NAME) {
-            Literal.create(null, NullType)
-          } else {
-            Literal.create(unescapePathName(raw), StringType)
-          }
+          if (raw == DEFAULT_PARTITION_NAME) NullType else StringType
         }
     } else {
-      if (raw == DEFAULT_PARTITION_NAME) {
-        Literal.create(null, NullType)
-      } else {
-        Literal.create(unescapePathName(raw), StringType)
-      }
+      if (raw == DEFAULT_PARTITION_NAME) NullType else StringType
     }
   }
 
+  def castPartValueToDesiredType(
+      desiredType: DataType,
+      value: String,
+      zoneId: ZoneId): Any = desiredType match {
+    case _ if value == DEFAULT_PARTITION_NAME => null
+    case NullType => null
+    case StringType => UTF8String.fromString(unescapePathName(value))
+    case IntegerType => Integer.parseInt(value)
+    case LongType => JLong.parseLong(value)
+    case DoubleType => JDouble.parseDouble(value)
+    case _: DecimalType => Literal(new JBigDecimal(value)).value
+    case DateType =>
+      Cast(Literal(value), DateType, Some(zoneId.getId)).eval()
+    case TimestampType =>
+      Try {
+        Cast(Literal(unescapePathName(value)), TimestampType, Some(zoneId.getId)).eval()
+      }.getOrElse {
+        Cast(Cast(Literal(value), DateType, Some(zoneId.getId)), TimestampType).eval()
+      }
+    case dt => throw QueryExecutionErrors.typeUnsupportedError(dt)
+  }
+
   def validatePartitionColumn(
       schema: StructType,
       partitionColumns: Seq[String],
       caseSensitive: Boolean): Unit = {
 
+    SchemaUtils.checkColumnNameDuplication(
+      partitionColumns, partitionColumns.mkString(", "), caseSensitive)
+
     partitionColumnsSchema(schema, partitionColumns, caseSensitive).foreach {
       field => field.dataType match {
         case _: AtomicType => // OK
-        case _ => throw new AnalysisException(s"Cannot use ${field.dataType} for partition column")
+        case _ => throw QueryCompilationErrors.cannotUseDataTypeForPartitionColumnError(field)
       }
     }
 
     if (partitionColumns.nonEmpty && partitionColumns.size == schema.fields.length) {
-      throw new AnalysisException(s"Cannot use all columns for partition columns")
+      throw QueryCompilationErrors.cannotUseAllColumnsForPartitionColumnsError()
     }
   }
 
@@ -565,7 +559,7 @@ object PartitioningUtils {
     StructType(partitionColumns.map { col =>
       schema.find(f => equality(f.name, col)).getOrElse {
         val schemaCatalog = schema.catalogString
-        throw new AnalysisException(s"Partition column `$col` not found in schema $schemaCatalog")
+        throw QueryCompilationErrors.partitionColumnNotFoundInSchemaError(col, schemaCatalog)
       }
     }).asNullable
   }
@@ -611,13 +605,11 @@ object PartitioningUtils {
    * Given a collection of [[Literal]]s, resolves possible type conflicts by
    * [[findWiderTypeForPartitionColumn]].
    */
-  private def resolveTypeConflicts(literals: Seq[Literal], zoneId: ZoneId): Seq[Literal] = {
-    val litTypes = literals.map(_.dataType)
-    val desiredType = litTypes.reduce(findWiderTypeForPartitionColumn)
+  private def resolveTypeConflicts(typedValues: Seq[TypedPartValue]): Seq[TypedPartValue] = {
+    val dataTypes = typedValues.map(_.dataType)
+    val desiredType = dataTypes.reduce(findWiderTypeForPartitionColumn)
 
-    literals.map { case l @ Literal(_, dataType) =>
-      Literal.create(Cast(l, desiredType, Some(zoneId.getId)).eval(), desiredType)
-    }
+    typedValues.map(tv => tv.copy(dataType = desiredType))
   }
 
   /**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PruneFileSourcePartitions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PruneFileSourcePartitions.scala
index a7129fb14d1a6..a88dfb29a12be 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PruneFileSourcePartitions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PruneFileSourcePartitions.scala
@@ -22,6 +22,7 @@ import org.apache.spark.sql.catalyst.catalog.CatalogStatistics
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.planning.PhysicalOperation
 import org.apache.spark.sql.catalyst.plans.logical.{Filter, LeafNode, LogicalPlan, Project}
+import org.apache.spark.sql.catalyst.plans.logical.statsEstimation.FilterEstimation
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2ScanRelation, FileScan}
 import org.apache.spark.sql.types.StructType
@@ -39,7 +40,8 @@ import org.apache.spark.sql.types.StructType
  * its underlying [[FileScan]]. And the partition filters will be removed in the filters of
  * returned logical plan.
  */
-private[sql] object PruneFileSourcePartitions extends Rule[LogicalPlan] {
+private[sql] object PruneFileSourcePartitions
+  extends Rule[LogicalPlan] with PredicateHelper {
 
   private def getPartitionKeyFiltersAndDataFilters(
       sparkSession: SparkSession,
@@ -55,8 +57,10 @@ private[sql] object PruneFileSourcePartitions extends Rule[LogicalPlan] {
     val (partitionFilters, dataFilters) = normalizedFilters.partition(f =>
       f.references.subsetOf(partitionSet)
     )
+    val extraPartitionFilter =
+      dataFilters.flatMap(extractPredicatesWithinOutputSet(_, partitionSet))
 
-    (ExpressionSet(partitionFilters), dataFilters)
+    (ExpressionSet(partitionFilters ++ extraPartitionFilter), dataFilters)
   }
 
   private def rebuildPhysicalOperation(
@@ -88,14 +92,24 @@ private[sql] object PruneFileSourcePartitions extends Rule[LogicalPlan] {
             _))
         if filters.nonEmpty && fsRelation.partitionSchemaOption.isDefined =>
       val (partitionKeyFilters, _) = getPartitionKeyFiltersAndDataFilters(
-        fsRelation.sparkSession, logicalRelation, partitionSchema, filters, logicalRelation.output)
+        fsRelation.sparkSession, logicalRelation, partitionSchema, filters,
+        logicalRelation.output)
+
       if (partitionKeyFilters.nonEmpty) {
         val prunedFileIndex = catalogFileIndex.filterPartitions(partitionKeyFilters.toSeq)
         val prunedFsRelation =
           fsRelation.copy(location = prunedFileIndex)(fsRelation.sparkSession)
         // Change table stats based on the sizeInBytes of pruned files
+        val filteredStats =
+          FilterEstimation(Filter(partitionKeyFilters.reduce(And), logicalRelation)).estimate
+        val colStats = filteredStats.map(_.attributeStats.map { case (attr, colStat) =>
+          (attr.name, colStat.toCatalogColumnStat(attr.name, attr.dataType))
+        })
         val withStats = logicalRelation.catalogTable.map(_.copy(
-          stats = Some(CatalogStatistics(sizeInBytes = BigInt(prunedFileIndex.sizeInBytes)))))
+          stats = Some(CatalogStatistics(
+            sizeInBytes = BigInt(prunedFileIndex.sizeInBytes),
+            rowCount = filteredStats.flatMap(_.rowCount),
+            colStats = colStats.getOrElse(Map.empty)))))
         val prunedLogicalRelation = logicalRelation.copy(
           relation = prunedFsRelation, catalogTable = withStats)
         // Keep partition-pruning predicates so that they are visible in physical planning
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/RecordReaderIterator.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/RecordReaderIterator.scala
index c3dd6939ec5bd..d8e30e600098d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/RecordReaderIterator.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/RecordReaderIterator.scala
@@ -21,7 +21,7 @@ import java.io.Closeable
 
 import org.apache.hadoop.mapreduce.RecordReader
 
-import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.errors.QueryExecutionErrors
 
 /**
  * An adaptor from a Hadoop [[RecordReader]] to an [[Iterator]] over the values returned.
@@ -50,7 +50,7 @@ class RecordReaderIterator[T](
 
   override def next(): T = {
     if (!hasNext) {
-      throw new java.util.NoSuchElementException("End of stream")
+      throw QueryExecutionErrors.endOfStreamError()
     }
     havePair = false
     rowReader.getCurrentValue
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SQLHadoopMapReduceCommitProtocol.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SQLHadoopMapReduceCommitProtocol.scala
index 39c594a9bc618..144be2316f091 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SQLHadoopMapReduceCommitProtocol.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SQLHadoopMapReduceCommitProtocol.scala
@@ -55,7 +55,8 @@ class SQLHadoopMapReduceCommitProtocol(
         // The specified output committer is a FileOutputCommitter.
         // So, we will use the FileOutputCommitter-specified constructor.
         val ctor = clazz.getDeclaredConstructor(classOf[Path], classOf[TaskAttemptContext])
-        committer = ctor.newInstance(new Path(path), context)
+        val committerOutputPath = if (dynamicPartitionOverwrite) stagingDir else new Path(path)
+        committer = ctor.newInstance(committerOutputPath, context)
       } else {
         // The specified output committer is just an OutputCommitter.
         // So, we will use the no-argument constructor.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SaveIntoDataSourceCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SaveIntoDataSourceCommand.scala
index 49e77f618f2a4..486f73cab44f7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SaveIntoDataSourceCommand.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SaveIntoDataSourceCommand.scala
@@ -20,8 +20,7 @@ package org.apache.spark.sql.execution.datasources
 import org.apache.spark.sql.{Dataset, Row, SaveMode, SparkSession}
 import org.apache.spark.sql.catalyst.plans.QueryPlan
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.execution.command.RunnableCommand
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.execution.command.LeafRunnableCommand
 import org.apache.spark.sql.sources.CreatableRelationProvider
 
 /**
@@ -37,7 +36,7 @@ case class SaveIntoDataSourceCommand(
     query: LogicalPlan,
     dataSource: CreatableRelationProvider,
     options: Map[String, String],
-    mode: SaveMode) extends RunnableCommand {
+    mode: SaveMode) extends LeafRunnableCommand {
 
   override def innerChildren: Seq[QueryPlan[_]] = Seq(query)
 
@@ -49,7 +48,7 @@ case class SaveIntoDataSourceCommand(
   }
 
   override def simpleString(maxFields: Int): String = {
-    val redacted = SQLConf.get.redactOptions(options)
+    val redacted = conf.redactOptions(options)
     s"SaveIntoDataSourceCommand ${dataSource}, ${redacted}, ${mode}"
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SchemaMergeUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SchemaMergeUtils.scala
index 99882b0f7c7b0..28097c35401c9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SchemaMergeUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SchemaMergeUtils.scala
@@ -32,10 +32,12 @@ object SchemaMergeUtils extends Logging {
    */
   def mergeSchemasInParallel(
       sparkSession: SparkSession,
+      parameters: Map[String, String],
       files: Seq[FileStatus],
       schemaReader: (Seq[FileStatus], Configuration, Boolean) => Seq[StructType])
       : Option[StructType] = {
-    val serializedConf = new SerializableConfiguration(sparkSession.sessionState.newHadoopConf())
+    val serializedConf = new SerializableConfiguration(
+      sparkSession.sessionState.newHadoopConfWithOptions(parameters))
 
     // !! HACK ALERT !!
     // Here is a hack for Parquet, but it can be used by Orc as well.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SchemaPruning.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SchemaPruning.scala
index 61e0154a0ffe8..a1974455fb78d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SchemaPruning.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SchemaPruning.scala
@@ -23,8 +23,7 @@ import org.apache.spark.sql.catalyst.plans.logical.{Filter, LeafNode, LogicalPla
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution.datasources.orc.OrcFileFormat
 import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructField, StructType}
+import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructType}
 
 /**
  * Prunes unnecessary physical columns given a [[PhysicalOperation]] over a data source relation.
@@ -36,7 +35,7 @@ object SchemaPruning extends Rule[LogicalPlan] {
   import org.apache.spark.sql.catalyst.expressions.SchemaPruning._
 
   override def apply(plan: LogicalPlan): LogicalPlan =
-    if (SQLConf.get.nestedSchemaPruningEnabled) {
+    if (conf.nestedSchemaPruningEnabled) {
       apply0(plan)
     } else {
       plan
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/WriteStatsTracker.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/WriteStatsTracker.scala
index c39a82ee037bc..f58aa33be8695 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/WriteStatsTracker.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/WriteStatsTracker.scala
@@ -32,20 +32,7 @@ trait WriteTaskStats extends Serializable
  * A trait for classes that are capable of collecting statistics on data that's being processed by
  * a single write task in [[FileFormatWriter]] - i.e. there should be one instance per executor.
  *
- * This trait is coupled with the way [[FileFormatWriter]] works, in the sense that its methods
- * will be called according to how tuples are being written out to disk, namely in sorted order
- * according to partitionValue(s), then bucketId.
- *
- * As such, a typical call scenario is:
- *
- * newPartition -> newBucket -> newFile -> newRow -.
- *    ^        |______^___________^ ^         ^____|
- *    |               |             |______________|
- *    |               |____________________________|
- *    |____________________________________________|
- *
- * newPartition and newBucket events are only triggered if the relation to be written out is
- * partitioned and/or bucketed, respectively.
+ * newPartition event is only triggered if the relation to be written out is partitioned.
  */
 trait WriteTaskStatsTracker {
 
@@ -56,27 +43,26 @@ trait WriteTaskStatsTracker {
    */
   def newPartition(partitionValues: InternalRow): Unit
 
-  /**
-   * Process the fact that a new bucket is about to written.
-   * Only triggered when the relation is bucketed by a (non-empty) sequence of columns.
-   * @param bucketId The bucket number.
-   */
-  def newBucket(bucketId: Int): Unit
-
   /**
    * Process the fact that a new file is about to be written.
    * @param filePath Path of the file into which future rows will be written.
    */
   def newFile(filePath: String): Unit
 
+  /**
+   * Process the fact that a file is finished to be written and closed.
+   * @param filePath Path of the file.
+   */
+  def closeFile(filePath: String): Unit
+
   /**
    * Process the fact that a new row to update the tracked statistics accordingly.
-   * The row will be written to the most recently witnessed file (via `newFile`).
    * @note Keep in mind that any overhead here is per-row, obviously,
    *       so implementations should be as lightweight as possible.
+   * @param filePath Path of the file which the row is written to.
    * @param row Current data row to be processed.
    */
-  def newRow(row: InternalRow): Unit
+  def newRow(filePath: String, row: InternalRow): Unit
 
   /**
    * Returns the final statistics computed so far.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormat.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormat.scala
index 637ce68ec05a2..4b500aa9637bc 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormat.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormat.scala
@@ -22,14 +22,14 @@ import java.sql.Timestamp
 
 import com.google.common.io.{ByteStreams, Closeables}
 import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.{FileStatus, GlobFilter, Path}
+import org.apache.hadoop.fs.{FileStatus, Path}
 import org.apache.hadoop.mapreduce.Job
 
-import org.apache.spark.SparkException
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter
-import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateTimeUtils}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.execution.datasources.{FileFormat, OutputWriterFactory, PartitionedFile}
 import org.apache.spark.sql.internal.SQLConf.SOURCES_BINARY_FILE_MAX_LENGTH
 import org.apache.spark.sql.sources.{And, DataSourceRegister, EqualTo, Filter, GreaterThan, GreaterThanOrEqual, LessThan, LessThanOrEqual, Not, Or}
@@ -69,7 +69,7 @@ class BinaryFileFormat extends FileFormat with DataSourceRegister {
       job: Job,
       options: Map[String, String],
       dataSchema: StructType): OutputWriterFactory = {
-    throw new UnsupportedOperationException("Write is not supported for binary file data source")
+    throw QueryExecutionErrors.writeUnsupportedForBinaryFileDataSourceError()
   }
 
   override def isSplitable(
@@ -114,9 +114,7 @@ class BinaryFileFormat extends FileFormat with DataSourceRegister {
             writer.write(i, DateTimeUtils.millisToMicros(status.getModificationTime))
           case (CONTENT, i) =>
             if (status.getLen > maxLength) {
-              throw new SparkException(
-                s"The length of ${status.getPath} is ${status.getLen}, " +
-                  s"which exceeds the max length allowed: ${maxLength}.")
+              throw QueryExecutionErrors.fileLengthExceedsMaxLengthError(status, maxLength)
             }
             val stream = fs.open(status.getPath)
             try {
@@ -125,7 +123,7 @@ class BinaryFileFormat extends FileFormat with DataSourceRegister {
               Closeables.close(stream, true)
             }
           case (other, _) =>
-            throw new RuntimeException(s"Unsupported field name: ${other}")
+            throw QueryExecutionErrors.unsupportedFieldNameError(other)
         }
         Iterator.single(writer.getRow)
       } else {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVDataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVDataSource.scala
index 375cec597166c..d8fa768a604f4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVDataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVDataSource.scala
@@ -23,8 +23,6 @@ import java.nio.charset.{Charset, StandardCharsets}
 import com.univocity.parsers.csv.CsvParser
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.{FileStatus, Path}
-import org.apache.hadoop.io.{LongWritable, Text}
-import org.apache.hadoop.mapred.TextInputFormat
 import org.apache.hadoop.mapreduce.Job
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat
 
@@ -150,21 +148,23 @@ object TextInputCSVDataSource extends CSVDataSource {
       inputPaths: Seq[FileStatus],
       options: CSVOptions): Dataset[String] = {
     val paths = inputPaths.map(_.getPath.toString)
+    val df = sparkSession.baseRelationToDataFrame(
+      DataSource.apply(
+        sparkSession,
+        paths = paths,
+        className = classOf[TextFileFormat].getName,
+        options = options.parameters ++ Map(DataSource.GLOB_PATHS_KEY -> "false")
+      ).resolveRelation(checkFilesExist = false))
+      .select("value").as[String](Encoders.STRING)
+
     if (Charset.forName(options.charset) == StandardCharsets.UTF_8) {
-      sparkSession.baseRelationToDataFrame(
-        DataSource.apply(
-          sparkSession,
-          paths = paths,
-          className = classOf[TextFileFormat].getName,
-          options = options.parameters
-        ).resolveRelation(checkFilesExist = false))
-        .select("value").as[String](Encoders.STRING)
+      df
     } else {
       val charset = options.charset
-      val rdd = sparkSession.sparkContext
-        .hadoopFile[LongWritable, Text, TextInputFormat](paths.mkString(","))
-        .mapPartitions(_.map(pair => new String(pair._2.getBytes, 0, pair._2.getLength, charset)))
-      sparkSession.createDataset(rdd)(Encoders.STRING)
+      sparkSession.createDataset(df.queryExecution.toRdd.map { row =>
+        val bytes = row.getBinary(0)
+        new String(bytes, 0, bytes.length, charset)
+      })(Encoders.STRING)
     }
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CsvOutputWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CsvOutputWriter.scala
index 2b549536ae486..35d0e098b19e9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CsvOutputWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CsvOutputWriter.scala
@@ -28,7 +28,7 @@ import org.apache.spark.sql.execution.datasources.{CodecStreams, OutputWriter}
 import org.apache.spark.sql.types.StructType
 
 class CsvOutputWriter(
-    path: String,
+    val path: String,
     dataSchema: StructType,
     context: TaskAttemptContext,
     params: CSVOptions) extends OutputWriter with Logging {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ddl.scala
index 4022640224424..221db208bc629 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ddl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ddl.scala
@@ -24,8 +24,9 @@ import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.catalog.CatalogTable
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.execution.command.{DDLUtils, RunnableCommand}
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.execution.command.{DDLUtils, LeafRunnableCommand}
+import org.apache.spark.sql.execution.command.ViewHelper.createTemporaryViewRelation
+import org.apache.spark.sql.internal.StaticSQLConf
 import org.apache.spark.sql.types._
 
 /**
@@ -51,6 +52,10 @@ case class CreateTable(
   override def children: Seq[LogicalPlan] = query.toSeq
   override def output: Seq[Attribute] = Seq.empty
   override lazy val resolved: Boolean = false
+
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[LogicalPlan]): LogicalPlan =
+    copy(query = if (query.isDefined) Some(newChildren.head) else None)
 }
 
 /**
@@ -62,7 +67,7 @@ case class CreateTempViewUsing(
     replace: Boolean,
     global: Boolean,
     provider: String,
-    options: Map[String, String]) extends RunnableCommand {
+    options: Map[String, String]) extends LeafRunnableCommand {
 
   if (tableIdent.database.isDefined) {
     throw new AnalysisException(
@@ -74,7 +79,7 @@ case class CreateTempViewUsing(
       userSpecifiedSchema.map(_ + " ").getOrElse("") +
       s"replace:$replace " +
       s"provider:$provider " +
-      SQLConf.get.redactOptions(options)
+      conf.redactOptions(options)
   }
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
@@ -90,12 +95,30 @@ case class CreateTempViewUsing(
       options = options)
 
     val catalog = sparkSession.sessionState.catalog
-    val viewDefinition = Dataset.ofRows(
+    val analyzedPlan = Dataset.ofRows(
       sparkSession, LogicalRelation(dataSource.resolveRelation())).logicalPlan
 
     if (global) {
+      val db = sparkSession.sessionState.conf.getConf(StaticSQLConf.GLOBAL_TEMP_DATABASE)
+      val viewIdent = TableIdentifier(tableIdent.table, Option(db))
+      val viewDefinition = createTemporaryViewRelation(
+        viewIdent,
+        sparkSession,
+        replace,
+        catalog.getRawGlobalTempView,
+        originalText = None,
+        analyzedPlan,
+        aliasedPlan = analyzedPlan)
       catalog.createGlobalTempView(tableIdent.table, viewDefinition, replace)
     } else {
+      val viewDefinition = createTemporaryViewRelation(
+        tableIdent,
+        sparkSession,
+        replace,
+        catalog.getRawTempView,
+        originalText = None,
+        analyzedPlan,
+        aliasedPlan = analyzedPlan)
       catalog.createTempView(tableIdent.table, viewDefinition, replace)
     }
 
@@ -103,19 +126,8 @@ case class CreateTempViewUsing(
   }
 }
 
-case class RefreshTable(tableIdent: TableIdentifier)
-  extends RunnableCommand {
-
-  override def run(sparkSession: SparkSession): Seq[Row] = {
-    // Refresh the given table's metadata. If this table is cached as an InMemoryRelation,
-    // drop the original cached version and make the new version cached lazily.
-    sparkSession.catalog.refreshTable(tableIdent.quotedString)
-    Seq.empty[Row]
-  }
-}
-
 case class RefreshResource(path: String)
-  extends RunnableCommand {
+  extends LeafRunnableCommand {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
     sparkSession.catalog.refreshByPath(path)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/DriverRegistry.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/DriverRegistry.scala
index 530d836d9fde3..3444d03beff5d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/DriverRegistry.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/DriverRegistry.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.execution.datasources.jdbc
 
 import java.sql.{Driver, DriverManager}
 
+import scala.collection.JavaConverters._
 import scala.collection.mutable
 
 import org.apache.spark.internal.Logging
@@ -58,5 +59,15 @@ object DriverRegistry extends Logging {
       }
     }
   }
+
+  def get(className: String): Driver = {
+    DriverManager.getDrivers.asScala.collectFirst {
+      case d: DriverWrapper if d.wrapped.getClass.getCanonicalName == className => d.wrapped
+      case d if d.getClass.getCanonicalName == className => d
+    }.getOrElse {
+      throw new IllegalStateException(
+        s"Did not find registered driver with class $className")
+    }
+  }
 }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala
index 9e0438c0016bd..97d4f2d97622e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala
@@ -25,12 +25,13 @@ import org.apache.commons.io.FilenameUtils
 import org.apache.spark.SparkFiles
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
+import org.apache.spark.sql.errors.QueryExecutionErrors
 
 /**
  * Options for the JDBC data source.
  */
 class JDBCOptions(
-    @transient val parameters: CaseInsensitiveMap[String])
+    val parameters: CaseInsensitiveMap[String])
   extends Serializable with Logging {
 
   import JDBCOptions._
@@ -73,22 +74,20 @@ class JDBCOptions(
   // table name or a table subquery.
   val tableOrQuery = (parameters.get(JDBC_TABLE_NAME), parameters.get(JDBC_QUERY_STRING)) match {
     case (Some(name), Some(subquery)) =>
-      throw new IllegalArgumentException(
-        s"Both '$JDBC_TABLE_NAME' and '$JDBC_QUERY_STRING' can not be specified at the same time."
-      )
+      throw QueryExecutionErrors.cannotSpecifyBothJdbcTableNameAndQueryError(
+        JDBC_TABLE_NAME, JDBC_QUERY_STRING)
     case (None, None) =>
-      throw new IllegalArgumentException(
-        s"Option '$JDBC_TABLE_NAME' or '$JDBC_QUERY_STRING' is required."
-      )
+      throw QueryExecutionErrors.missingJdbcTableNameAndQueryError(
+        JDBC_TABLE_NAME, JDBC_QUERY_STRING)
     case (Some(name), None) =>
       if (name.isEmpty) {
-        throw new IllegalArgumentException(s"Option '$JDBC_TABLE_NAME' can not be empty.")
+        throw QueryExecutionErrors.emptyOptionError(JDBC_TABLE_NAME)
       } else {
         name.trim
       }
     case (None, Some(subquery)) =>
       if (subquery.isEmpty) {
-        throw new IllegalArgumentException(s"Option `$JDBC_QUERY_STRING` can not be empty.")
+        throw QueryExecutionErrors.emptyOptionError(JDBC_QUERY_STRING)
       } else {
         s"(${subquery}) SPARK_GEN_SUBQ_${curId.getAndIncrement()}"
       }
@@ -180,10 +179,8 @@ class JDBCOptions(
       case "READ_COMMITTED" => Connection.TRANSACTION_READ_COMMITTED
       case "REPEATABLE_READ" => Connection.TRANSACTION_REPEATABLE_READ
       case "SERIALIZABLE" => Connection.TRANSACTION_SERIALIZABLE
-      case other => throw new IllegalArgumentException(
-        s"Invalid value `$other` for parameter `$JDBC_TXN_ISOLATION_LEVEL`. This can be " +
-          "`NONE`, `READ_UNCOMMITTED`, `READ_COMMITTED`, `REPEATABLE_READ` or `SERIALIZABLE`."
-      )
+      case other => throw QueryExecutionErrors.invalidJdbcTxnIsolationLevelError(
+        JDBC_TXN_ISOLATION_LEVEL, other)
     }
   // An option to execute custom SQL before fetching data from the remote DB
   val sessionInitStatement = parameters.get(JDBC_SESSION_INIT_STATEMENT)
@@ -206,10 +203,14 @@ class JDBCOptions(
   }
   // The principal name of user's keytab file
   val principal = parameters.getOrElse(JDBC_PRINCIPAL, null)
+
+  val tableComment = parameters.getOrElse(JDBC_TABLE_COMMENT, "").toString
+
+  val refreshKrb5Config = parameters.getOrElse(JDBC_REFRESH_KRB5_CONFIG, "false").toBoolean
 }
 
 class JdbcOptionsInWrite(
-    @transient override val parameters: CaseInsensitiveMap[String])
+    override val parameters: CaseInsensitiveMap[String])
   extends JDBCOptions(parameters) {
 
   import JDBCOptions._
@@ -260,4 +261,6 @@ object JDBCOptions {
   val JDBC_PUSHDOWN_PREDICATE = newOption("pushDownPredicate")
   val JDBC_KEYTAB = newOption("keytab")
   val JDBC_PRINCIPAL = newOption("principal")
+  val JDBC_TABLE_COMMENT = newOption("tableComment")
+  val JDBC_REFRESH_KRB5_CONFIG = newOption("refreshKrb5Config")
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
index e25ce53941ff6..87ca78db59b29 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.execution.datasources.jdbc
 
-import java.sql.{Connection, PreparedStatement, ResultSet, SQLException}
+import java.sql.{Connection, PreparedStatement, ResultSet}
 
 import scala.util.control.NonFatal
 
@@ -46,8 +46,8 @@ object JDBCRDD extends Logging {
    * @param options - JDBC options that contains url, table and other information.
    *
    * @return A StructType giving the table's Catalyst schema.
-   * @throws SQLException if the table specification is garbage.
-   * @throws SQLException if the table contains an unsupported type.
+   * @throws java.sql.SQLException if the table specification is garbage.
+   * @throws java.sql.SQLException if the table contains an unsupported type.
    */
   def resolveTable(options: JDBCOptions): StructType = {
     val url = options.url
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala
index f5a474ddf3904..e6d8819ac29f3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.execution.datasources.jdbc
 
 import scala.collection.mutable.ArrayBuffer
+import scala.math.BigDecimal.RoundingMode
 
 import org.apache.spark.Partition
 import org.apache.spark.internal.Logging
@@ -118,13 +119,28 @@ private[sql] object JDBCRelation extends Logging {
           s"Upper bound: ${boundValueToString(upperBound)}.")
         upperBound - lowerBound
       }
-    // Overflow and silliness can happen if you subtract then divide.
-    // Here we get a little roundoff, but that's (hopefully) OK.
-    val stride: Long = upperBound / numPartitions - lowerBound / numPartitions
+
+    // Overflow can happen if you subtract then divide. For example:
+    // (Long.MaxValue - Long.MinValue) / (numPartitions - 2).
+    // Also, using fixed-point decimals here to avoid possible inaccuracy from floating point.
+    val upperStride = (upperBound / BigDecimal(numPartitions))
+      .setScale(18, RoundingMode.HALF_EVEN)
+    val lowerStride = (lowerBound / BigDecimal(numPartitions))
+      .setScale(18, RoundingMode.HALF_EVEN)
+
+    val preciseStride = upperStride - lowerStride
+    val stride = preciseStride.toLong
+
+    // Determine the number of strides the last partition will fall short of compared to the
+    // supplied upper bound. Take half of those strides, and then add them to the lower bound
+    // for better distribution of the first and last partitions.
+    val lostNumOfStrides = (preciseStride - stride) * numPartitions / stride
+    val lowerBoundWithStrideAlignment = lowerBound +
+      ((lostNumOfStrides / 2) * stride).setScale(0, RoundingMode.HALF_UP).toLong
 
     var i: Int = 0
     val column = partitioning.column
-    var currentValue = lowerBound
+    var currentValue = lowerBoundWithStrideAlignment
     val ans = new ArrayBuffer[Partition]()
     while (i < numPartitions) {
       val lBoundValue = boundValueToString(currentValue)
@@ -203,7 +219,7 @@ private[sql] object JDBCRelation extends Logging {
         case TimestampType =>
           val timestampFormatter = TimestampFormatter.getFractionFormatter(
             DateTimeUtils.getZoneId(timeZoneId))
-          DateTimeUtils.timestampToString(timestampFormatter, value)
+          timestampFormatter.format(value)
       }
       s"'$dateTimeStr'"
     }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcRelationProvider.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcRelationProvider.scala
index e7456f9c8ed0b..314012feef4c4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcRelationProvider.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcRelationProvider.scala
@@ -59,7 +59,7 @@ class JdbcRelationProvider extends CreatableRelationProvider
             } else {
               // Otherwise, do not truncate the table, instead drop and recreate it
               dropTable(conn, options.table, options)
-              createTable(conn, df, options)
+              createTable(conn, options.table, df.schema, isCaseSensitive, options)
               saveTable(df, Some(df.schema), isCaseSensitive, options)
             }
 
@@ -78,7 +78,7 @@ class JdbcRelationProvider extends CreatableRelationProvider
             // Therefore, it is okay to do nothing here and then just return the relation below.
         }
       } else {
-        createTable(conn, df, options)
+        createTable(conn, options.table, df.schema, isCaseSensitive, options)
         saveTable(df, Some(df.schema), isCaseSensitive, options)
       }
     } finally {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
index db4715ef068b6..60fcaf94e1986 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
@@ -17,24 +17,29 @@
 
 package org.apache.spark.sql.execution.datasources.jdbc
 
-import java.sql.{Connection, Driver, DriverManager, JDBCType, PreparedStatement, ResultSet, ResultSetMetaData, SQLException}
+import java.sql.{Connection, Driver, JDBCType, PreparedStatement, ResultSet, ResultSetMetaData, SQLException}
+import java.time.{Instant, LocalDate}
 import java.util.Locale
+import java.util.concurrent.TimeUnit
 
-import scala.collection.JavaConverters._
 import scala.util.Try
 import scala.util.control.NonFatal
 
 import org.apache.spark.TaskContext
 import org.apache.spark.executor.InputMetrics
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.{AnalysisException, DataFrame, Row}
+import org.apache.spark.sql.{DataFrame, Row}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.Resolver
 import org.apache.spark.sql.catalyst.encoders.RowEncoder
 import org.apache.spark.sql.catalyst.expressions.SpecificInternalRow
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
 import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateTimeUtils, GenericArrayData}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.{instantToMicros, localDateToDays, toJavaDate, toJavaTimestamp}
+import org.apache.spark.sql.connector.catalog.TableChange
+import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
 import org.apache.spark.sql.execution.datasources.jdbc.connection.ConnectionProvider
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.jdbc.{JdbcDialect, JdbcDialects, JdbcType}
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.util.SchemaUtils
@@ -55,17 +60,10 @@ object JdbcUtils extends Logging {
     val driverClass: String = options.driverClass
     () => {
       DriverRegistry.register(driverClass)
-      val driver: Driver = DriverManager.getDrivers.asScala.collectFirst {
-        case d: DriverWrapper if d.wrapped.getClass.getCanonicalName == driverClass => d
-        case d if d.getClass.getCanonicalName == driverClass => d
-      }.getOrElse {
-        throw new IllegalStateException(
-          s"Did not find registered driver with class $driverClass")
-      }
-      val connection = ConnectionProvider.create(driver, options).getConnection()
+      val driver: Driver = DriverRegistry.get(driverClass)
+      val connection = ConnectionProvider.create(driver, options.parameters)
       require(connection != null,
         s"The driver could not open a JDBC connection. Check the URL: ${options.url}")
-
       connection
     }
   }
@@ -94,13 +92,7 @@ object JdbcUtils extends Logging {
    * Drops a table from the JDBC database.
    */
   def dropTable(conn: Connection, table: String, options: JDBCOptions): Unit = {
-    val statement = conn.createStatement
-    try {
-      statement.setQueryTimeout(options.queryTimeout)
-      statement.executeUpdate(s"DROP TABLE $table")
-    } finally {
-      statement.close()
-    }
+    executeStatement(conn, options, s"DROP TABLE $table")
   }
 
   /**
@@ -150,7 +142,7 @@ object JdbcUtils extends Logging {
       val tableColumnNames = tableSchema.get.fieldNames
       rddSchema.fields.map { col =>
         val normalizedName = tableColumnNames.find(f => columnNameEquality(f, col.name)).getOrElse {
-          throw new AnalysisException(s"""Column "${col.name}" not found in schema $tableSchema""")
+          throw QueryCompilationErrors.columnNotFoundInSchemaError(col, tableSchema)
         }
         dialect.quoteIdentifier(normalizedName)
       }.mkString(",")
@@ -184,9 +176,9 @@ object JdbcUtils extends Logging {
     }
   }
 
-  private def getJdbcType(dt: DataType, dialect: JdbcDialect): JdbcType = {
+  def getJdbcType(dt: DataType, dialect: JdbcDialect): JdbcType = {
     dialect.getJDBCType(dt).orElse(getCommonJDBCType(dt)).getOrElse(
-      throw new IllegalArgumentException(s"Can't get JDBC type for ${dt.catalogString}"))
+      throw QueryExecutionErrors.cannotGetJdbcTypeError(dt))
   }
 
   /**
@@ -235,7 +227,7 @@ object JdbcUtils extends Logging {
       case java.sql.Types.REAL          => DoubleType
       case java.sql.Types.REF           => StringType
       case java.sql.Types.REF_CURSOR    => null
-      case java.sql.Types.ROWID         => LongType
+      case java.sql.Types.ROWID         => StringType
       case java.sql.Types.SMALLINT      => IntegerType
       case java.sql.Types.SQLXML        => StringType
       case java.sql.Types.STRUCT        => StringType
@@ -249,12 +241,12 @@ object JdbcUtils extends Logging {
       case java.sql.Types.VARBINARY     => BinaryType
       case java.sql.Types.VARCHAR       => StringType
       case _                            =>
-        throw new SQLException("Unrecognized SQL type " + sqlType)
+        throw QueryExecutionErrors.unrecognizedSqlTypeError(sqlType)
       // scalastyle:on
     }
 
     if (answer == null) {
-      throw new SQLException("Unsupported type " + JDBCType.valueOf(sqlType).getName)
+      throw QueryExecutionErrors.unsupportedJdbcTypeError(JDBCType.valueOf(sqlType).getName)
     }
     answer
   }
@@ -316,11 +308,24 @@ object JdbcUtils extends Logging {
       } else {
         rsmd.isNullable(i + 1) != ResultSetMetaData.columnNoNulls
       }
-      val metadata = new MetadataBuilder().putLong("scale", fieldScale)
+      val metadata = new MetadataBuilder()
+      metadata.putLong("scale", fieldScale)
+
+      dataType match {
+        case java.sql.Types.TIME =>
+          // SPARK-33888
+          // - include TIME type metadata
+          // - always build the metadata
+          metadata.putBoolean("logical_time_type", true)
+        case java.sql.Types.ROWID =>
+          metadata.putBoolean("rowid", true)
+        case _ =>
+      }
+
       val columnType =
         dialect.getCatalystType(dataType, typeName, fieldSize, metadata).getOrElse(
           getCatalystType(dataType, fieldSize, fieldScale, isSigned))
-      fields(i) = StructField(columnName, columnType, nullable)
+      fields(i) = StructField(columnName, columnType, nullable, metadata.build())
       i = i + 1
     }
     new StructType(fields)
@@ -448,11 +453,34 @@ object JdbcUtils extends Logging {
       (rs: ResultSet, row: InternalRow, pos: Int) =>
         row.setByte(pos, rs.getByte(pos + 1))
 
+    case StringType if metadata.contains("rowid") =>
+      (rs: ResultSet, row: InternalRow, pos: Int) =>
+        row.update(pos, UTF8String.fromString(rs.getRowId(pos + 1).toString))
+
     case StringType =>
       (rs: ResultSet, row: InternalRow, pos: Int) =>
         // TODO(davies): use getBytes for better performance, if the encoding is UTF-8
         row.update(pos, UTF8String.fromString(rs.getString(pos + 1)))
 
+    // SPARK-34357 - sql TIME type represents as zero epoch timestamp.
+    // It is mapped as Spark TimestampType but fixed at 1970-01-01 for day,
+    // time portion is time of day, with no reference to a particular calendar,
+    // time zone or date, with a precision till microseconds.
+    // It stores the number of milliseconds after midnight, 00:00:00.000000
+    case TimestampType if metadata.contains("logical_time_type") =>
+      (rs: ResultSet, row: InternalRow, pos: Int) => {
+        val rawTime = rs.getTime(pos + 1)
+        if (rawTime != null) {
+          val localTimeMicro = TimeUnit.NANOSECONDS.toMicros(
+            rawTime.toLocalTime().toNanoOfDay())
+          val utcTimeMicro = DateTimeUtils.toUTCTime(
+            localTimeMicro, SQLConf.get.sessionLocalTimeZone)
+          row.setLong(pos, utcTimeMicro)
+        } else {
+          row.update(pos, null)
+        }
+      }
+
     case TimestampType =>
       (rs: ResultSet, row: InternalRow, pos: Int) =>
         val t = rs.getTimestamp(pos + 1)
@@ -494,11 +522,10 @@ object JdbcUtils extends Logging {
             }
 
         case LongType if metadata.contains("binarylong") =>
-          throw new IllegalArgumentException(s"Unsupported array element " +
-            s"type ${dt.catalogString} based on binary")
+          throw QueryExecutionErrors.unsupportedArrayElementTypeBasedOnBinaryError(dt)
 
         case ArrayType(_, _) =>
-          throw new IllegalArgumentException("Nested arrays unsupported")
+          throw QueryExecutionErrors.nestedArraysUnsupportedError()
 
         case _ => (array: Object) => array.asInstanceOf[Array[Any]]
       }
@@ -509,7 +536,7 @@ object JdbcUtils extends Logging {
           array => new GenericArrayData(elementConversion.apply(array.getArray)))
         row.update(pos, array)
 
-    case _ => throw new IllegalArgumentException(s"Unsupported type ${dt.catalogString}")
+    case _ => throw QueryExecutionErrors.unsupportedJdbcTypeError(dt.catalogString)
   }
 
   private def nullSafeConvert[T](input: T, f: T => Any): Any = {
@@ -566,12 +593,22 @@ object JdbcUtils extends Logging {
         stmt.setBytes(pos + 1, row.getAs[Array[Byte]](pos))
 
     case TimestampType =>
-      (stmt: PreparedStatement, row: Row, pos: Int) =>
-        stmt.setTimestamp(pos + 1, row.getAs[java.sql.Timestamp](pos))
+      if (SQLConf.get.datetimeJava8ApiEnabled) {
+        (stmt: PreparedStatement, row: Row, pos: Int) =>
+          stmt.setTimestamp(pos + 1, toJavaTimestamp(instantToMicros(row.getAs[Instant](pos))))
+      } else {
+        (stmt: PreparedStatement, row: Row, pos: Int) =>
+          stmt.setTimestamp(pos + 1, row.getAs[java.sql.Timestamp](pos))
+      }
 
     case DateType =>
-      (stmt: PreparedStatement, row: Row, pos: Int) =>
-        stmt.setDate(pos + 1, row.getAs[java.sql.Date](pos))
+      if (SQLConf.get.datetimeJava8ApiEnabled) {
+        (stmt: PreparedStatement, row: Row, pos: Int) =>
+          stmt.setDate(pos + 1, toJavaDate(localDateToDays(row.getAs[LocalDate](pos))))
+      } else {
+        (stmt: PreparedStatement, row: Row, pos: Int) =>
+          stmt.setDate(pos + 1, row.getAs[java.sql.Date](pos))
+      }
 
     case t: DecimalType =>
       (stmt: PreparedStatement, row: Row, pos: Int) =>
@@ -589,8 +626,7 @@ object JdbcUtils extends Logging {
 
     case _ =>
       (_: PreparedStatement, _: Row, pos: Int) =>
-        throw new IllegalArgumentException(
-          s"Can't translate non-null value for field $pos")
+        throw QueryExecutionErrors.cannotTranslateNonNullValueForFieldError(pos)
   }
 
   /**
@@ -746,15 +782,16 @@ object JdbcUtils extends Logging {
    * Compute the schema string for this RDD.
    */
   def schemaString(
-      df: DataFrame,
+      schema: StructType,
+      caseSensitive: Boolean,
       url: String,
       createTableColumnTypes: Option[String] = None): String = {
     val sb = new StringBuilder()
     val dialect = JdbcDialects.get(url)
     val userSpecifiedColTypesMap = createTableColumnTypes
-      .map(parseUserSpecifiedCreateTableColumnTypes(df, _))
+      .map(parseUserSpecifiedCreateTableColumnTypes(schema, caseSensitive, _))
       .getOrElse(Map.empty[String, String])
-    df.schema.fields.foreach { field =>
+    schema.fields.foreach { field =>
       val name = dialect.quoteIdentifier(field.name)
       val typ = userSpecifiedColTypesMap
         .getOrElse(field.name, getJdbcType(field.dataType, dialect).databaseTypeDefinition)
@@ -770,20 +807,15 @@ object JdbcUtils extends Logging {
    * use in-place of the default data type.
    */
   private def parseUserSpecifiedCreateTableColumnTypes(
-      df: DataFrame,
+      schema: StructType,
+      caseSensitive: Boolean,
       createTableColumnTypes: String): Map[String, String] = {
-    def typeName(f: StructField): String = {
-      // char/varchar gets translated to string type. Real data type specified by the user
-      // is available in the field metadata as HIVE_TYPE_STRING
-      if (f.metadata.contains(HIVE_TYPE_STRING)) {
-        f.metadata.getString(HIVE_TYPE_STRING)
-      } else {
-        f.dataType.catalogString
-      }
-    }
-
     val userSchema = CatalystSqlParser.parseTableSchema(createTableColumnTypes)
-    val nameEquality = df.sparkSession.sessionState.conf.resolver
+    val nameEquality = if (caseSensitive) {
+      org.apache.spark.sql.catalyst.analysis.caseSensitiveResolution
+    } else {
+      org.apache.spark.sql.catalyst.analysis.caseInsensitiveResolution
+    }
 
     // checks duplicate columns in the user specified column types.
     SchemaUtils.checkColumnNameDuplication(
@@ -791,16 +823,14 @@ object JdbcUtils extends Logging {
 
     // checks if user specified column names exist in the DataFrame schema
     userSchema.fieldNames.foreach { col =>
-      df.schema.find(f => nameEquality(f.name, col)).getOrElse {
-        throw new AnalysisException(
-          s"createTableColumnTypes option column $col not found in schema " +
-            df.schema.catalogString)
+      schema.find(f => nameEquality(f.name, col)).getOrElse {
+        throw QueryCompilationErrors.createTableColumnTypesOptionColumnNotFoundInSchemaError(
+          col, schema)
       }
     }
 
-    val userSchemaMap = userSchema.fields.map(f => f.name -> typeName(f)).toMap
-    val isCaseSensitive = df.sparkSession.sessionState.conf.caseSensitiveAnalysis
-    if (isCaseSensitive) userSchemaMap else CaseInsensitiveMap(userSchemaMap)
+    val userSchemaMap = userSchema.fields.map(f => f.name -> f.dataType.catalogString).toMap
+    if (caseSensitive) userSchemaMap else CaseInsensitiveMap(userSchemaMap)
   }
 
   /**
@@ -814,8 +844,10 @@ object JdbcUtils extends Logging {
     if (null != customSchema && customSchema.nonEmpty) {
       val userSchema = CatalystSqlParser.parseTableSchema(customSchema)
 
-      SchemaUtils.checkColumnNameDuplication(
-        userSchema.map(_.name), "in the customSchema option value", nameEquality)
+      SchemaUtils.checkSchemaColumnNameDuplication(
+        userSchema,
+        "in the customSchema option value",
+        nameEquality)
 
       // This is resolved by names, use the custom filed dataType to replace the default dataType.
       val newSchema = tableSchema.map { col =>
@@ -848,9 +880,8 @@ object JdbcUtils extends Logging {
 
     val insertStmt = getInsertStatement(table, rddSchema, tableSchema, isCaseSensitive, dialect)
     val repartitionedDF = options.numPartitions match {
-      case Some(n) if n <= 0 => throw new IllegalArgumentException(
-        s"Invalid value `$n` for parameter `${JDBCOptions.JDBC_NUM_PARTITIONS}` in table writing " +
-          "via JDBC. The minimum value is 1.")
+      case Some(n) if n <= 0 => throw QueryExecutionErrors.invalidJdbcNumPartitionsError(
+        n, JDBCOptions.JDBC_NUM_PARTITIONS)
       case Some(n) if n < df.rdd.getNumPartitions => df.coalesce(n)
       case _ => df
     }
@@ -865,17 +896,130 @@ object JdbcUtils extends Logging {
    */
   def createTable(
       conn: Connection,
-      df: DataFrame,
+      tableName: String,
+      schema: StructType,
+      caseSensitive: Boolean,
       options: JdbcOptionsInWrite): Unit = {
+    val dialect = JdbcDialects.get(options.url)
     val strSchema = schemaString(
-      df, options.url, options.createTableColumnTypes)
-    val table = options.table
+      schema, caseSensitive, options.url, options.createTableColumnTypes)
     val createTableOptions = options.createTableOptions
     // Create the table if the table does not exist.
     // To allow certain options to append when create a new table, which can be
     // table_options or partition_options.
     // E.g., "CREATE TABLE t (name string) ENGINE=InnoDB DEFAULT CHARSET=utf8"
-    val sql = s"CREATE TABLE $table ($strSchema) $createTableOptions"
+    val sql = s"CREATE TABLE $tableName ($strSchema) $createTableOptions"
+    executeStatement(conn, options, sql)
+    if (options.tableComment.nonEmpty) {
+      try {
+        executeStatement(
+          conn, options, dialect.getTableCommentQuery(tableName, options.tableComment))
+      } catch {
+        case e: Exception =>
+          logWarning("Cannot create JDBC table comment. The table comment will be ignored.")
+      }
+    }
+  }
+
+  /**
+   * Rename a table from the JDBC database.
+   */
+  def renameTable(
+      conn: Connection,
+      oldTable: String,
+      newTable: String,
+      options: JDBCOptions): Unit = {
+    val dialect = JdbcDialects.get(options.url)
+    executeStatement(conn, options, dialect.renameTable(oldTable, newTable))
+  }
+
+  /**
+   * Update a table from the JDBC database.
+   */
+  def alterTable(
+      conn: Connection,
+      tableName: String,
+      changes: Seq[TableChange],
+      options: JDBCOptions): Unit = {
+    val dialect = JdbcDialects.get(options.url)
+    val metaData = conn.getMetaData
+    if (changes.length == 1) {
+      executeStatement(conn, options, dialect.alterTable(tableName, changes,
+        metaData.getDatabaseMajorVersion)(0))
+    } else {
+      if (!metaData.supportsTransactions) {
+        throw QueryExecutionErrors.transactionUnsupportedByJdbcServerError()
+      } else {
+        conn.setAutoCommit(false)
+        val statement = conn.createStatement
+        try {
+          statement.setQueryTimeout(options.queryTimeout)
+          for (sql <- dialect.alterTable(tableName, changes, metaData.getDatabaseMajorVersion)) {
+            statement.executeUpdate(sql)
+          }
+          conn.commit()
+        } catch {
+          case e: Exception =>
+            if (conn != null) conn.rollback()
+            throw e
+        } finally {
+          statement.close()
+          conn.setAutoCommit(true)
+        }
+      }
+    }
+  }
+
+  /**
+   * Creates a namespace.
+   */
+  def createNamespace(
+      conn: Connection,
+      options: JDBCOptions,
+      namespace: String,
+      comment: String): Unit = {
+    val dialect = JdbcDialects.get(options.url)
+    executeStatement(conn, options, s"CREATE SCHEMA ${dialect.quoteIdentifier(namespace)}")
+    if (!comment.isEmpty) createNamespaceComment(conn, options, namespace, comment)
+  }
+
+  def createNamespaceComment(
+      conn: Connection,
+      options: JDBCOptions,
+      namespace: String,
+      comment: String): Unit = {
+    val dialect = JdbcDialects.get(options.url)
+    try {
+      executeStatement(
+        conn, options, dialect.getSchemaCommentQuery(namespace, comment))
+    } catch {
+      case e: Exception =>
+        logWarning("Cannot create JDBC catalog comment. The catalog comment will be ignored.")
+    }
+  }
+
+  def removeNamespaceComment(
+      conn: Connection,
+      options: JDBCOptions,
+      namespace: String): Unit = {
+    val dialect = JdbcDialects.get(options.url)
+    try {
+      executeStatement(conn, options, dialect.removeSchemaCommentQuery(namespace))
+    } catch {
+      case e: Exception =>
+        logWarning("Cannot drop JDBC catalog comment.")
+    }
+  }
+
+  /**
+   * Drops a namespace from the JDBC database.
+   */
+  def dropNamespace(conn: Connection, options: JDBCOptions, namespace: String): Unit = {
+    val dialect = JdbcDialects.get(options.url)
+    executeStatement(conn, options, s"DROP SCHEMA ${dialect.quoteIdentifier(namespace)}")
+  }
+
+  private def executeStatement(conn: Connection, options: JDBCOptions, sql: String): Unit = {
     val statement = conn.createStatement
     try {
       statement.setQueryTimeout(options.queryTimeout)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/BasicConnectionProvider.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/BasicConnectionProvider.scala
index c21e16bcf1280..890205f2f6826 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/BasicConnectionProvider.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/BasicConnectionProvider.scala
@@ -18,18 +18,34 @@
 package org.apache.spark.sql.execution.datasources.jdbc.connection
 
 import java.sql.{Connection, Driver}
+import java.util.Properties
 
 import scala.collection.JavaConverters._
 
+import org.apache.spark.internal.Logging
 import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions
+import org.apache.spark.sql.jdbc.JdbcConnectionProvider
 
-private[jdbc] class BasicConnectionProvider(driver: Driver, options: JDBCOptions)
-    extends ConnectionProvider {
-  def getConnection(): Connection = {
-    val properties = getAdditionalProperties()
-    options.asConnectionProperties.entrySet().asScala.foreach { e =>
-      properties.put(e.getKey(), e.getValue())
+private[jdbc] class BasicConnectionProvider extends JdbcConnectionProvider with Logging {
+  /**
+   * Additional properties for data connection (Data source property takes precedence).
+   */
+  def getAdditionalProperties(options: JDBCOptions): Properties = new Properties()
+
+  override val name: String = "basic"
+
+  override def canHandle(driver: Driver, options: Map[String, String]): Boolean = {
+    val jdbcOptions = new JDBCOptions(options)
+    jdbcOptions.keytab == null || jdbcOptions.principal == null
+  }
+
+  override def getConnection(driver: Driver, options: Map[String, String]): Connection = {
+    val jdbcOptions = new JDBCOptions(options)
+    val properties = getAdditionalProperties(jdbcOptions)
+    jdbcOptions.asProperties.asScala.foreach { case(k, v) =>
+      properties.put(k, v)
     }
-    driver.connect(options.url, properties)
+    logDebug(s"JDBC connection initiated with URL: ${jdbcOptions.url} and properties: $properties")
+    driver.connect(jdbcOptions.url, properties)
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/ConnectionProvider.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/ConnectionProvider.scala
index 73e73e59be574..fbc69704f1479 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/ConnectionProvider.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/ConnectionProvider.scala
@@ -18,51 +18,57 @@
 package org.apache.spark.sql.execution.datasources.jdbc.connection
 
 import java.sql.{Connection, Driver}
-import java.util.Properties
+import java.util.ServiceLoader
+import javax.security.auth.login.Configuration
 
-import org.apache.spark.internal.Logging
-import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions
-
-/**
- * Connection provider which opens connection toward various databases (database specific instance
- * needed). If kerberos authentication required then it's the provider's responsibility to set all
- * the parameters.
- */
-private[jdbc] trait ConnectionProvider {
-  /**
-   * Additional properties for data connection (Data source property takes precedence).
-   */
-  def getAdditionalProperties(): Properties = new Properties()
+import scala.collection.mutable
 
-  /**
-   * Opens connection toward the database.
-   */
-  def getConnection(): Connection
-}
+import org.apache.spark.internal.Logging
+import org.apache.spark.security.SecurityConfigurationLock
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.jdbc.JdbcConnectionProvider
+import org.apache.spark.util.Utils
 
 private[jdbc] object ConnectionProvider extends Logging {
-  def create(driver: Driver, options: JDBCOptions): ConnectionProvider = {
-    if (options.keytab == null || options.principal == null) {
-      logDebug("No authentication configuration found, using basic connection provider")
-      new BasicConnectionProvider(driver, options)
-    } else {
-      logDebug("Authentication configuration found, using database specific connection provider")
-      options.driverClass match {
-        case PostgresConnectionProvider.driverClass =>
-          logDebug("Postgres connection provider found")
-          new PostgresConnectionProvider(driver, options)
+  private val providers = loadProviders()
 
-        case MariaDBConnectionProvider.driverClass =>
-          logDebug("MariaDB connection provider found")
-          new MariaDBConnectionProvider(driver, options)
+  def loadProviders(): Seq[JdbcConnectionProvider] = {
+    val loader = ServiceLoader.load(classOf[JdbcConnectionProvider],
+      Utils.getContextOrSparkClassLoader)
+    val providers = mutable.ArrayBuffer[JdbcConnectionProvider]()
 
-        case DB2ConnectionProvider.driverClass =>
-          logDebug("DB2 connection provider found")
-          new DB2ConnectionProvider(driver, options)
+    val iterator = loader.iterator
+    while (iterator.hasNext) {
+      try {
+        val provider = iterator.next
+        logDebug(s"Loaded built-in provider: $provider")
+        providers += provider
+      } catch {
+        case t: Throwable =>
+          logError("Failed to load built-in provider.")
+          logInfo("Loading of the provider failed with the exception:", t)
+      }
+    }
+
+    val disabledProviders = Utils.stringToSeq(SQLConf.get.disabledJdbcConnectionProviders)
+    // toSeq seems duplicate but it's needed for Scala 2.13
+    providers.filterNot(p => disabledProviders.contains(p.name)).toSeq
+  }
 
-        case _ =>
-          throw new IllegalArgumentException(s"Driver ${options.driverClass} does not support " +
-            "Kerberos authentication")
+  def create(driver: Driver, options: Map[String, String]): Connection = {
+    val filteredProviders = providers.filter(_.canHandle(driver, options))
+    require(filteredProviders.size == 1,
+      "JDBC connection initiated but not exactly one connection provider found which can handle " +
+        s"it. Found active providers: ${filteredProviders.mkString(", ")}")
+    SecurityConfigurationLock.synchronized {
+      // Inside getConnection it's safe to get parent again because SecurityConfigurationLock
+      // makes sure it's untouched
+      val parent = Configuration.getConfiguration
+      try {
+        filteredProviders.head.getConnection(driver, options)
+      } finally {
+        logDebug("Restoring original security configuration")
+        Configuration.setConfiguration(parent)
       }
     }
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/DB2ConnectionProvider.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/DB2ConnectionProvider.scala
index cf9729639c03c..060653c5a8b79 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/DB2ConnectionProvider.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/DB2ConnectionProvider.scala
@@ -25,37 +25,31 @@ import org.apache.hadoop.security.UserGroupInformation
 
 import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions
 
-private[sql] class DB2ConnectionProvider(driver: Driver, options: JDBCOptions)
-    extends SecureConnectionProvider(driver, options) {
-  override val appEntry: String = "JaasClient"
-
-  override def getConnection(): Connection = {
-    setAuthenticationConfigIfNeeded()
-    UserGroupInformation.loginUserFromKeytabAndReturnUGI(options.principal, options.keytab).doAs(
-      new PrivilegedExceptionAction[Connection]() {
-        override def run(): Connection = {
-          DB2ConnectionProvider.super.getConnection()
+private[sql] class DB2ConnectionProvider extends SecureConnectionProvider {
+  override val driverClass = "com.ibm.db2.jcc.DB2Driver"
+
+  override val name: String = "db2"
+
+  override def appEntry(driver: Driver, options: JDBCOptions): String = "JaasClient"
+
+  override def getConnection(driver: Driver, options: Map[String, String]): Connection = {
+    val jdbcOptions = new JDBCOptions(options)
+    setAuthenticationConfig(driver, jdbcOptions)
+    UserGroupInformation.loginUserFromKeytabAndReturnUGI(jdbcOptions.principal, jdbcOptions.keytab)
+      .doAs(
+        new PrivilegedExceptionAction[Connection]() {
+          override def run(): Connection = {
+            DB2ConnectionProvider.super.getConnection(driver, options)
+          }
         }
-      }
-    )
+      )
   }
 
-  override def getAdditionalProperties(): Properties = {
+  override def getAdditionalProperties(options: JDBCOptions): Properties = {
     val result = new Properties()
     // 11 is the integer value for kerberos
     result.put("securityMechanism", new String("11"))
     result.put("KerberosServerPrincipal", options.principal)
     result
   }
-
-  override def setAuthenticationConfigIfNeeded(): Unit = SecurityConfigurationLock.synchronized {
-    val (parent, configEntry) = getConfigWithAppEntry()
-    if (configEntry == null || configEntry.isEmpty) {
-      setAuthenticationConfig(parent)
-    }
-  }
-}
-
-private[sql] object DB2ConnectionProvider {
-  val driverClass = "com.ibm.db2.jcc.DB2Driver"
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/MSSQLConnectionProvider.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/MSSQLConnectionProvider.scala
new file mode 100644
index 0000000000000..aa8c9227377c1
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/MSSQLConnectionProvider.scala
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.jdbc.connection
+
+import java.security.PrivilegedExceptionAction
+import java.sql.{Connection, Driver}
+import java.util.Properties
+
+import org.apache.hadoop.security.UserGroupInformation
+
+import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions
+
+private[sql] class MSSQLConnectionProvider extends SecureConnectionProvider {
+  override val driverClass = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
+  val parserMethod: String = "parseAndMergeProperties"
+
+  override val name: String = "mssql"
+
+  override def appEntry(driver: Driver, options: JDBCOptions): String = {
+    val configName = "jaasConfigurationName"
+    val appEntryDefault = "SQLJDBCDriver"
+
+    val parseURL = try {
+      // The default parser method signature is the following:
+      // private Properties parseAndMergeProperties(String Url, Properties suppliedProperties)
+      val m = driver.getClass.getDeclaredMethod(parserMethod, classOf[String], classOf[Properties])
+      m.setAccessible(true)
+      Some(m)
+    } catch {
+      case _: NoSuchMethodException => None
+    }
+
+    parseURL match {
+      case Some(m) =>
+        logDebug("Property parser method found, using it")
+        m.invoke(driver, options.url, null).asInstanceOf[Properties]
+          .getProperty(configName, appEntryDefault)
+
+      case None =>
+        logDebug("Property parser method not found, using custom parsing mechanism")
+        options.url.split(';').map(_.split('='))
+          .find(kv => kv.length == 2 && kv(0) == configName)
+          .getOrElse(Array(configName, appEntryDefault))(1)
+    }
+  }
+
+  override def getConnection(driver: Driver, options: Map[String, String]): Connection = {
+    val jdbcOptions = new JDBCOptions(options)
+    setAuthenticationConfig(driver, jdbcOptions)
+    UserGroupInformation.loginUserFromKeytabAndReturnUGI(jdbcOptions.principal, jdbcOptions.keytab)
+      .doAs(
+        new PrivilegedExceptionAction[Connection]() {
+          override def run(): Connection = {
+            MSSQLConnectionProvider.super.getConnection(driver, options)
+          }
+        }
+      )
+  }
+
+  override def getAdditionalProperties(options: JDBCOptions): Properties = {
+    val result = new Properties()
+    // These props needed to reach internal kerberos authentication in the JDBC driver
+    result.put("integratedSecurity", "true")
+    result.put("authenticationScheme", "JavaKerberos")
+    result
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/MariaDBConnectionProvider.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/MariaDBConnectionProvider.scala
index 8e3381077cbbf..6a53c663a2773 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/MariaDBConnectionProvider.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/MariaDBConnectionProvider.scala
@@ -21,27 +21,10 @@ import java.sql.Driver
 
 import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions
 
-private[jdbc] class MariaDBConnectionProvider(driver: Driver, options: JDBCOptions)
-    extends SecureConnectionProvider(driver, options) {
-  override val appEntry: String = {
-    "Krb5ConnectorContext"
-  }
+private[jdbc] class MariaDBConnectionProvider extends SecureConnectionProvider {
+  override val driverClass = "org.mariadb.jdbc.Driver"
 
-  override def setAuthenticationConfigIfNeeded(): Unit = SecurityConfigurationLock.synchronized {
-    val (parent, configEntry) = getConfigWithAppEntry()
-    /**
-     * Couple of things to mention here:
-     * 1. MariaDB doesn't support JAAS application name configuration
-     * 2. MariaDB sets a default JAAS config if "java.security.auth.login.config" is not set
-     */
-    val entryUsesKeytab = configEntry != null &&
-      configEntry.exists(_.getOptions().get("useKeyTab") == "true")
-    if (configEntry == null || configEntry.isEmpty || !entryUsesKeytab) {
-      setAuthenticationConfig(parent)
-    }
-  }
-}
+  override val name: String = "mariadb"
 
-private[sql] object MariaDBConnectionProvider {
-  val driverClass = "org.mariadb.jdbc.Driver"
+  override def appEntry(driver: Driver, options: JDBCOptions): String = "Krb5ConnectorContext"
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/OracleConnectionProvider.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/OracleConnectionProvider.scala
new file mode 100644
index 0000000000000..ef8d91b5aa8f1
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/OracleConnectionProvider.scala
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.jdbc.connection
+
+import java.security.PrivilegedExceptionAction
+import java.sql.{Connection, Driver}
+import java.util.Properties
+
+import org.apache.hadoop.security.UserGroupInformation
+
+import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions
+
+private[sql] class OracleConnectionProvider extends SecureConnectionProvider {
+  override val driverClass = "oracle.jdbc.OracleDriver"
+
+  override val name: String = "oracle"
+
+  override def appEntry(driver: Driver, options: JDBCOptions): String = "kprb5module"
+
+  override def getConnection(driver: Driver, options: Map[String, String]): Connection = {
+    val jdbcOptions = new JDBCOptions(options)
+    setAuthenticationConfig(driver, jdbcOptions)
+    UserGroupInformation.loginUserFromKeytabAndReturnUGI(jdbcOptions.principal, jdbcOptions.keytab)
+      .doAs(
+        new PrivilegedExceptionAction[Connection]() {
+          override def run(): Connection = {
+            OracleConnectionProvider.super.getConnection(driver, options)
+          }
+        }
+      )
+  }
+
+  override def getAdditionalProperties(options: JDBCOptions): Properties = {
+    val result = new Properties()
+    // This prop is needed to turn on kerberos authentication in the JDBC driver.
+    // The possible values can be found in AnoServices public interface
+    // The value is coming from AUTHENTICATION_KERBEROS5 final String in driver version 19.6.0.0
+    result.put("oracle.net.authentication_services", "(KERBEROS5)");
+    result
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/PostgresConnectionProvider.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/PostgresConnectionProvider.scala
index 73034dcb9c2e0..ec9ac66147e99 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/PostgresConnectionProvider.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/PostgresConnectionProvider.scala
@@ -22,22 +22,14 @@ import java.util.Properties
 
 import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions
 
-private[jdbc] class PostgresConnectionProvider(driver: Driver, options: JDBCOptions)
-    extends SecureConnectionProvider(driver, options) {
-  override val appEntry: String = {
+private[jdbc] class PostgresConnectionProvider extends SecureConnectionProvider {
+  override val driverClass = "org.postgresql.Driver"
+
+  override val name: String = "postgres"
+
+  override def appEntry(driver: Driver, options: JDBCOptions): String = {
     val parseURL = driver.getClass.getMethod("parseURL", classOf[String], classOf[Properties])
     val properties = parseURL.invoke(driver, options.url, null).asInstanceOf[Properties]
     properties.getProperty("jaasApplicationName", "pgjdbc")
   }
-
-  override def setAuthenticationConfigIfNeeded(): Unit = SecurityConfigurationLock.synchronized {
-    val (parent, configEntry) = getConfigWithAppEntry()
-    if (configEntry == null || configEntry.isEmpty) {
-      setAuthenticationConfig(parent)
-    }
-  }
-}
-
-private[sql] object PostgresConnectionProvider {
-  val driverClass = "org.postgresql.Driver"
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/SecureConnectionProvider.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/SecureConnectionProvider.scala
index fa75fc8c28fbf..71c20e11f9274 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/SecureConnectionProvider.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/SecureConnectionProvider.scala
@@ -26,39 +26,34 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions
 import org.apache.spark.util.SecurityUtils
 
-/**
- * Some of the secure connection providers modify global JVM security configuration.
- * In order to avoid race the modification must be synchronized with this.
- */
-private[connection] object SecurityConfigurationLock
+private[jdbc] abstract class SecureConnectionProvider extends BasicConnectionProvider with Logging {
+  /**
+   * Returns the driver canonical class name which the connection provider supports.
+   */
+  protected val driverClass: String
 
-private[jdbc] abstract class SecureConnectionProvider(driver: Driver, options: JDBCOptions)
-    extends BasicConnectionProvider(driver, options) with Logging {
-  override def getConnection(): Connection = {
-    setAuthenticationConfigIfNeeded()
-    super.getConnection()
+  override def canHandle(driver: Driver, options: Map[String, String]): Boolean = {
+    val jdbcOptions = new JDBCOptions(options)
+    jdbcOptions.keytab != null && jdbcOptions.principal != null &&
+      driverClass.equalsIgnoreCase(jdbcOptions.driverClass)
   }
 
-  /**
-   * Returns JAAS application name. This is sometimes configurable on the JDBC driver level.
-   */
-  val appEntry: String
+  override def getConnection(driver: Driver, options: Map[String, String]): Connection = {
+    val jdbcOptions = new JDBCOptions(options)
+    setAuthenticationConfig(driver, jdbcOptions)
+    super.getConnection(driver: Driver, options: Map[String, String])
+  }
 
   /**
-   * Sets database specific authentication configuration when needed. If configuration already set
-   * then later calls must be no op. When the global JVM security configuration changed then the
-   * related code parts must be synchronized properly.
+   * Returns JAAS application name. This is sometimes configurable on the JDBC driver level.
    */
-  def setAuthenticationConfigIfNeeded(): Unit
+  def appEntry(driver: Driver, options: JDBCOptions): String
 
-  protected def getConfigWithAppEntry(): (Configuration, Array[AppConfigurationEntry]) = {
+  private[connection] def setAuthenticationConfig(driver: Driver, options: JDBCOptions) = {
     val parent = Configuration.getConfiguration
-    (parent, parent.getAppConfigurationEntry(appEntry))
-  }
-
-  protected def setAuthenticationConfig(parent: Configuration) = {
     val config = new SecureConnectionProvider.JDBCConfiguration(
-      parent, appEntry, options.keytab, options.principal)
+      parent, appEntry(driver, options), options.keytab,
+      options.principal, options.refreshKrb5Config)
     logDebug("Adding database specific security configuration")
     Configuration.setConfiguration(config)
   }
@@ -69,7 +64,8 @@ object SecureConnectionProvider {
     parent: Configuration,
     appEntry: String,
     keytab: String,
-    principal: String) extends Configuration {
+    principal: String,
+    refreshKrb5Config: Boolean) extends Configuration {
   val entry =
     new AppConfigurationEntry(
       SecurityUtils.getKrb5LoginModuleName(),
@@ -79,7 +75,8 @@ object SecureConnectionProvider {
         "useKeyTab" -> "true",
         "keyTab" -> keytab,
         "principal" -> principal,
-        "debug" -> "true"
+        "debug" -> "true",
+        "refreshKrb5Config" -> refreshKrb5Config.toString
       ).asJava
     )
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonDataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonDataSource.scala
index 7ec2267e3461f..2f4cd4684578a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonDataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonDataSource.scala
@@ -120,7 +120,7 @@ object TextInputJsonDataSource extends JsonDataSource {
         sparkSession,
         paths = inputPaths.map(_.getPath.toString),
         className = classOf[TextFileFormat].getName,
-        options = parsedOptions.parameters
+        options = parsedOptions.parameters ++ Map(DataSource.GLOB_PATHS_KEY -> "false")
       ).resolveRelation(checkFilesExist = false))
       .select("value").as(Encoders.STRING)
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonFileFormat.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonFileFormat.scala
index 95a63c3d1e2d9..f2f6f60cb1dde 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonFileFormat.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonFileFormat.scala
@@ -17,13 +17,10 @@
 
 package org.apache.spark.sql.execution.datasources.json
 
-import java.nio.charset.{Charset, StandardCharsets}
-
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.{FileStatus, Path}
 import org.apache.hadoop.mapreduce.{Job, TaskAttemptContext}
 
-import org.apache.spark.internal.Logging
 import org.apache.spark.sql.{AnalysisException, SparkSession}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.ExprUtils
@@ -125,7 +122,11 @@ class JsonFileFormat extends TextBasedFileFormat with DataSourceRegister {
     }
 
     (file: PartitionedFile) => {
-      val parser = new JacksonParser(actualSchema, parsedOptions, allowArrayAsStructs = true)
+      val parser = new JacksonParser(
+        actualSchema,
+        parsedOptions,
+        allowArrayAsStructs = true,
+        filters)
       JsonDataSource(parsedOptions).readFile(
         broadcastedHadoopConf.value.value,
         file,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonOutputWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonOutputWriter.scala
index dfd84e344eb2a..55602ce2ed9b4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonOutputWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonOutputWriter.scala
@@ -28,7 +28,7 @@ import org.apache.spark.sql.execution.datasources.{CodecStreams, OutputWriter}
 import org.apache.spark.sql.types.StructType
 
 class JsonOutputWriter(
-    path: String,
+    val path: String,
     options: JSONOptions,
     dataSchema: StructType,
     context: TaskAttemptContext)
@@ -39,7 +39,7 @@ class JsonOutputWriter(
     case None => StandardCharsets.UTF_8
   }
 
-  if (JSONOptionsInRead.blacklist.contains(encoding)) {
+  if (JSONOptionsInRead.denyList.contains(encoding)) {
     logWarning(s"The JSON file ($path) was written in the encoding ${encoding.displayName()}" +
       " which can be read back by Spark only if multiLine is enabled.")
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/noop/NoopDataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/noop/NoopDataSource.scala
index 851cc51466a91..79e4150fe6b17 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/noop/NoopDataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/noop/NoopDataSource.scala
@@ -23,9 +23,9 @@ import scala.collection.JavaConverters._
 
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.connector.catalog.{SupportsWrite, Table, TableCapability}
-import org.apache.spark.sql.connector.write.{BatchWrite, DataWriter, DataWriterFactory, LogicalWriteInfo, PhysicalWriteInfo, SupportsTruncate, WriteBuilder, WriterCommitMessage}
+import org.apache.spark.sql.connector.write.{BatchWrite, DataWriter, DataWriterFactory, LogicalWriteInfo, PhysicalWriteInfo, SupportsTruncate, Write, WriteBuilder, WriterCommitMessage}
 import org.apache.spark.sql.connector.write.streaming.{StreamingDataWriterFactory, StreamingWrite}
-import org.apache.spark.sql.internal.connector.SimpleTableProvider
+import org.apache.spark.sql.internal.connector.{SimpleTableProvider, SupportsStreamingUpdateAsAppend}
 import org.apache.spark.sql.sources.DataSourceRegister
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
@@ -52,10 +52,15 @@ private[noop] object NoopTable extends Table with SupportsWrite {
   }
 }
 
-private[noop] object NoopWriteBuilder extends WriteBuilder with SupportsTruncate {
+private[noop] object NoopWriteBuilder extends WriteBuilder
+  with SupportsTruncate with SupportsStreamingUpdateAsAppend {
   override def truncate(): WriteBuilder = this
-  override def buildForBatch(): BatchWrite = NoopBatchWrite
-  override def buildForStreaming(): StreamingWrite = NoopStreamingWrite
+  override def build(): Write = NoopWrite
+}
+
+private[noop] object NoopWrite extends Write {
+  override def toBatch: BatchWrite = NoopBatchWrite
+  override def toStreaming: StreamingWrite = NoopStreamingWrite
 }
 
 private[noop] object NoopBatchWrite extends BatchWrite {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcDeserializer.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcDeserializer.scala
index 4ab009c6bd014..fa8977f239164 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcDeserializer.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcDeserializer.scala
@@ -23,7 +23,7 @@ import org.apache.orc.mapred.{OrcList, OrcMap, OrcStruct, OrcTimestamp}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{SpecificInternalRow, UnsafeArrayData}
 import org.apache.spark.sql.catalyst.util._
-import org.apache.spark.sql.catalyst.util.RebaseDateTime.rebaseJulianToGregorianDays
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -31,7 +31,6 @@ import org.apache.spark.unsafe.types.UTF8String
  * A deserializer to deserialize ORC structs to Spark rows.
  */
 class OrcDeserializer(
-    dataSchema: StructType,
     requiredSchema: StructType,
     requestedColIds: Array[Int]) {
 
@@ -191,7 +190,7 @@ class OrcDeserializer(
       case udt: UserDefinedType[_] => newWriter(udt.sqlType, updater)
 
       case _ =>
-        throw new UnsupportedOperationException(s"$dataType is not supported yet.")
+        throw QueryExecutionErrors.dataTypeUnsupportedYetError(dataType)
     }
 
   private def createArrayData(elementType: DataType, length: Int): ArrayData = elementType match {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala
index fd791ce7c5e19..3a5097441ab37 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala
@@ -45,7 +45,7 @@ import org.apache.spark.util.{SerializableConfiguration, Utils}
 private[sql] object OrcFileFormat {
   private def checkFieldName(name: String): Unit = {
     try {
-      TypeDescription.fromString(s"struct<$name:int>")
+      TypeDescription.fromString(s"struct<`$name`:int>")
     } catch {
       case _: IllegalArgumentException =>
         throw new AnalysisException(
@@ -131,11 +131,27 @@ class OrcFileFormat
     }
   }
 
+  private def supportBatchForNestedColumn(
+      sparkSession: SparkSession,
+      schema: StructType): Boolean = {
+    val hasNestedColumn = schema.map(_.dataType).exists {
+      case _: ArrayType | _: MapType | _: StructType => true
+      case _ => false
+    }
+    if (hasNestedColumn) {
+      sparkSession.sessionState.conf.orcVectorizedReaderNestedColumnEnabled
+    } else {
+      true
+    }
+  }
+
   override def supportBatch(sparkSession: SparkSession, schema: StructType): Boolean = {
     val conf = sparkSession.sessionState.conf
     conf.orcVectorizedReaderEnabled && conf.wholeStageEnabled &&
       schema.length <= conf.wholeStageMaxNumFields &&
-      schema.forall(_.dataType.isInstanceOf[AtomicType])
+      schema.forall(s => supportDataType(s.dataType) &&
+        !s.dataType.isInstanceOf[UserDefinedType[_]]) &&
+      supportBatchForNestedColumn(sparkSession, schema)
   }
 
   override def isSplitable(
@@ -153,24 +169,19 @@ class OrcFileFormat
       filters: Seq[Filter],
       options: Map[String, String],
       hadoopConf: Configuration): (PartitionedFile) => Iterator[InternalRow] = {
-    if (sparkSession.sessionState.conf.orcFilterPushDown) {
-      OrcFilters.createFilter(dataSchema, filters).foreach { f =>
-        OrcInputFormat.setSearchArgument(hadoopConf, f, dataSchema.fieldNames)
-      }
-    }
 
     val resultSchema = StructType(requiredSchema.fields ++ partitionSchema.fields)
     val sqlConf = sparkSession.sessionState.conf
     val enableVectorizedReader = supportBatch(sparkSession, resultSchema)
     val capacity = sqlConf.orcVectorizedReaderBatchSize
 
-    val resultSchemaString = OrcUtils.orcTypeDescriptionString(resultSchema)
-    OrcConf.MAPRED_INPUT_SCHEMA.setString(hadoopConf, resultSchemaString)
     OrcConf.IS_SCHEMA_EVOLUTION_CASE_SENSITIVE.setBoolean(hadoopConf, sqlConf.caseSensitiveAnalysis)
 
     val broadcastedConf =
       sparkSession.sparkContext.broadcast(new SerializableConfiguration(hadoopConf))
     val isCaseSensitive = sparkSession.sessionState.conf.caseSensitiveAnalysis
+    val orcFilterPushDown = sparkSession.sessionState.conf.orcFilterPushDown
+    val ignoreCorruptFiles = sparkSession.sessionState.conf.ignoreCorruptFiles
 
     (file: PartitionedFile) => {
       val conf = broadcastedConf.value.value
@@ -179,16 +190,27 @@ class OrcFileFormat
 
       val fs = filePath.getFileSystem(conf)
       val readerOptions = OrcFile.readerOptions(conf).filesystem(fs)
-      val requestedColIdsOrEmptyFile =
+      val resultedColPruneInfo =
         Utils.tryWithResource(OrcFile.createReader(filePath, readerOptions)) { reader =>
           OrcUtils.requestedColumnIds(
             isCaseSensitive, dataSchema, requiredSchema, reader, conf)
         }
 
-      if (requestedColIdsOrEmptyFile.isEmpty) {
+      if (resultedColPruneInfo.isEmpty) {
         Iterator.empty
       } else {
-        val requestedColIds = requestedColIdsOrEmptyFile.get
+        // ORC predicate pushdown
+        if (orcFilterPushDown && filters.nonEmpty) {
+          OrcUtils.readCatalystSchema(filePath, conf, ignoreCorruptFiles).foreach { fileSchema =>
+            OrcFilters.createFilter(fileSchema, filters).foreach { f =>
+              OrcInputFormat.setSearchArgument(conf, f, fileSchema.fieldNames)
+            }
+          }
+        }
+
+        val (requestedColIds, canPruneCols) = resultedColPruneInfo.get
+        val resultSchemaString = OrcUtils.orcResultSchemaString(canPruneCols,
+          dataSchema, resultSchema, partitionSchema, conf)
         assert(requestedColIds.length == requiredSchema.length,
           "[BUG] requested column IDs do not match required schema")
         val taskConf = new Configuration(conf)
@@ -224,7 +246,7 @@ class OrcFileFormat
 
           val fullSchema = requiredSchema.toAttributes ++ partitionSchema.toAttributes
           val unsafeProjection = GenerateUnsafeProjection.generate(fullSchema, fullSchema)
-          val deserializer = new OrcDeserializer(dataSchema, requiredSchema, requestedColIds)
+          val deserializer = new OrcDeserializer(requiredSchema, requestedColIds)
 
           if (partitionSchema.length == 0) {
             iter.map(value => unsafeProjection(deserializer.deserialize(value)))
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
new file mode 100644
index 0000000000000..5abfa4cc9ef0d
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.orc
+
+import java.time.{Instant, LocalDate}
+
+import org.apache.hadoop.hive.common.`type`.HiveDecimal
+import org.apache.hadoop.hive.ql.io.sarg.{PredicateLeaf, SearchArgument}
+import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.Builder
+import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory.newBuilder
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable
+
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.{instantToMicros, localDateToDays, toJavaDate, toJavaTimestamp}
+import org.apache.spark.sql.errors.QueryExecutionErrors
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.sources.Filter
+import org.apache.spark.sql.types._
+
+/**
+ * Helper object for building ORC `SearchArgument`s, which are used for ORC predicate push-down.
+ *
+ * Due to limitation of ORC `SearchArgument` builder, we had to implement separate checking and
+ * conversion passes through the Filter to make sure we only convert predicates that are known
+ * to be convertible.
+ *
+ * An ORC `SearchArgument` must be built in one pass using a single builder.  For example, you can't
+ * build `a = 1` and `b = 2` first, and then combine them into `a = 1 AND b = 2`.  This is quite
+ * different from the cases in Spark SQL or Parquet, where complex filters can be easily built using
+ * existing simpler ones.
+ *
+ * The annoying part is that, `SearchArgument` builder methods like `startAnd()`, `startOr()`, and
+ * `startNot()` mutate internal state of the builder instance.  This forces us to translate all
+ * convertible filters with a single builder instance. However, if we try to translate a filter
+ * before checking whether it can be converted or not, we may end up with a builder whose internal
+ * state is inconsistent in the case of an inconvertible filter.
+ *
+ * For example, to convert an `And` filter with builder `b`, we call `b.startAnd()` first, and then
+ * try to convert its children.  Say we convert `left` child successfully, but find that `right`
+ * child is inconvertible.  Alas, `b.startAnd()` call can't be rolled back, and `b` is inconsistent
+ * now.
+ *
+ * The workaround employed here is to trim the Spark filters before trying to convert them. This
+ * way, we can only do the actual conversion on the part of the Filter that is known to be
+ * convertible.
+ *
+ * P.S.: Hive seems to use `SearchArgument` together with `ExprNodeGenericFuncDesc` only.  Usage of
+ * builder methods mentioned above can only be found in test code, where all tested filters are
+ * known to be convertible.
+ */
+private[sql] object OrcFilters extends OrcFiltersBase {
+
+  /**
+   * Create ORC filter as a SearchArgument instance.
+   */
+  def createFilter(schema: StructType, filters: Seq[Filter]): Option[SearchArgument] = {
+    val dataTypeMap = OrcFilters.getSearchableTypeMap(schema, SQLConf.get.caseSensitiveAnalysis)
+    // Combines all convertible filters using `And` to produce a single conjunction
+    val conjunctionOptional = buildTree(convertibleFilters(dataTypeMap, filters))
+    conjunctionOptional.map { conjunction =>
+      // Then tries to build a single ORC `SearchArgument` for the conjunction predicate.
+      // The input predicate is fully convertible. There should not be any empty result in the
+      // following recursive method call `buildSearchArgument`.
+      buildSearchArgument(dataTypeMap, conjunction, newBuilder).build()
+    }
+  }
+
+  def convertibleFilters(
+      dataTypeMap: Map[String, OrcPrimitiveField],
+      filters: Seq[Filter]): Seq[Filter] = {
+    import org.apache.spark.sql.sources._
+
+    def convertibleFiltersHelper(
+        filter: Filter,
+        canPartialPushDown: Boolean): Option[Filter] = filter match {
+      // At here, it is not safe to just convert one side and remove the other side
+      // if we do not understand what the parent filters are.
+      //
+      // Here is an example used to explain the reason.
+      // Let's say we have NOT(a = 2 AND b in ('1')) and we do not understand how to
+      // convert b in ('1'). If we only convert a = 2, we will end up with a filter
+      // NOT(a = 2), which will generate wrong results.
+      //
+      // Pushing one side of AND down is only safe to do at the top level or in the child
+      // AND before hitting NOT or OR conditions, and in this case, the unsupported predicate
+      // can be safely removed.
+      case And(left, right) =>
+        val leftResultOptional = convertibleFiltersHelper(left, canPartialPushDown)
+        val rightResultOptional = convertibleFiltersHelper(right, canPartialPushDown)
+        (leftResultOptional, rightResultOptional) match {
+          case (Some(leftResult), Some(rightResult)) => Some(And(leftResult, rightResult))
+          case (Some(leftResult), None) if canPartialPushDown => Some(leftResult)
+          case (None, Some(rightResult)) if canPartialPushDown => Some(rightResult)
+          case _ => None
+        }
+
+      // The Or predicate is convertible when both of its children can be pushed down.
+      // That is to say, if one/both of the children can be partially pushed down, the Or
+      // predicate can be partially pushed down as well.
+      //
+      // Here is an example used to explain the reason.
+      // Let's say we have
+      // (a1 AND a2) OR (b1 AND b2),
+      // a1 and b1 is convertible, while a2 and b2 is not.
+      // The predicate can be converted as
+      // (a1 OR b1) AND (a1 OR b2) AND (a2 OR b1) AND (a2 OR b2)
+      // As per the logical in And predicate, we can push down (a1 OR b1).
+      case Or(left, right) =>
+        for {
+          lhs <- convertibleFiltersHelper(left, canPartialPushDown)
+          rhs <- convertibleFiltersHelper(right, canPartialPushDown)
+        } yield Or(lhs, rhs)
+      case Not(pred) =>
+        val childResultOptional = convertibleFiltersHelper(pred, canPartialPushDown = false)
+        childResultOptional.map(Not)
+      case other =>
+        for (_ <- buildLeafSearchArgument(dataTypeMap, other, newBuilder())) yield other
+    }
+    filters.flatMap { filter =>
+      convertibleFiltersHelper(filter, true)
+    }
+  }
+
+  /**
+   * Get PredicateLeafType which is corresponding to the given DataType.
+   */
+  def getPredicateLeafType(dataType: DataType): PredicateLeaf.Type = dataType match {
+    case BooleanType => PredicateLeaf.Type.BOOLEAN
+    case ByteType | ShortType | IntegerType | LongType => PredicateLeaf.Type.LONG
+    case FloatType | DoubleType => PredicateLeaf.Type.FLOAT
+    case StringType => PredicateLeaf.Type.STRING
+    case DateType => PredicateLeaf.Type.DATE
+    case TimestampType => PredicateLeaf.Type.TIMESTAMP
+    case _: DecimalType => PredicateLeaf.Type.DECIMAL
+    case _ => throw QueryExecutionErrors.unsupportedOperationForDataTypeError(dataType)
+  }
+
+  /**
+   * Cast literal values for filters.
+   *
+   * We need to cast to long because ORC raises exceptions
+   * at 'checkLiteralType' of SearchArgumentImpl.java.
+   */
+  private def castLiteralValue(value: Any, dataType: DataType): Any = dataType match {
+    case ByteType | ShortType | IntegerType | LongType =>
+      value.asInstanceOf[Number].longValue
+    case FloatType | DoubleType =>
+      value.asInstanceOf[Number].doubleValue()
+    case _: DecimalType =>
+      new HiveDecimalWritable(HiveDecimal.create(value.asInstanceOf[java.math.BigDecimal]))
+    case _: DateType if value.isInstanceOf[LocalDate] =>
+      toJavaDate(localDateToDays(value.asInstanceOf[LocalDate]))
+    case _: TimestampType if value.isInstanceOf[Instant] =>
+      toJavaTimestamp(instantToMicros(value.asInstanceOf[Instant]))
+    case _ => value
+  }
+
+  /**
+   * Build a SearchArgument and return the builder so far.
+   *
+   * @param dataTypeMap a map from the attribute name to its data type.
+   * @param expression the input predicates, which should be fully convertible to SearchArgument.
+   * @param builder the input SearchArgument.Builder.
+   * @return the builder so far.
+   */
+  private def buildSearchArgument(
+      dataTypeMap: Map[String, OrcPrimitiveField],
+      expression: Filter,
+      builder: Builder): Builder = {
+    import org.apache.spark.sql.sources._
+
+    expression match {
+      case And(left, right) =>
+        val lhs = buildSearchArgument(dataTypeMap, left, builder.startAnd())
+        val rhs = buildSearchArgument(dataTypeMap, right, lhs)
+        rhs.end()
+
+      case Or(left, right) =>
+        val lhs = buildSearchArgument(dataTypeMap, left, builder.startOr())
+        val rhs = buildSearchArgument(dataTypeMap, right, lhs)
+        rhs.end()
+
+      case Not(child) =>
+        buildSearchArgument(dataTypeMap, child, builder.startNot()).end()
+
+      case other =>
+        buildLeafSearchArgument(dataTypeMap, other, builder).getOrElse {
+          throw QueryExecutionErrors.inputFilterNotFullyConvertibleError(
+            "OrcFilters.buildSearchArgument")
+        }
+    }
+  }
+
+  /**
+   * Build a SearchArgument for a leaf predicate and return the builder so far.
+   *
+   * @param dataTypeMap a map from the attribute name to its data type.
+   * @param expression the input filter predicates.
+   * @param builder the input SearchArgument.Builder.
+   * @return the builder so far.
+   */
+  private def buildLeafSearchArgument(
+      dataTypeMap: Map[String, OrcPrimitiveField],
+      expression: Filter,
+      builder: Builder): Option[Builder] = {
+    def getType(attribute: String): PredicateLeaf.Type =
+      getPredicateLeafType(dataTypeMap(attribute).fieldType)
+
+    import org.apache.spark.sql.sources._
+
+    // NOTE: For all case branches dealing with leaf predicates below, the additional `startAnd()`
+    // call is mandatory. ORC `SearchArgument` builder requires that all leaf predicates must be
+    // wrapped by a "parent" predicate (`And`, `Or`, or `Not`).
+    expression match {
+      case EqualTo(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startAnd()
+          .equals(dataTypeMap(name).fieldName, getType(name), castedValue).end())
+
+      case EqualNullSafe(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startAnd()
+          .nullSafeEquals(dataTypeMap(name).fieldName, getType(name), castedValue).end())
+
+      case LessThan(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startAnd()
+          .lessThan(dataTypeMap(name).fieldName, getType(name), castedValue).end())
+
+      case LessThanOrEqual(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startAnd()
+          .lessThanEquals(dataTypeMap(name).fieldName, getType(name), castedValue).end())
+
+      case GreaterThan(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startNot()
+          .lessThanEquals(dataTypeMap(name).fieldName, getType(name), castedValue).end())
+
+      case GreaterThanOrEqual(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startNot()
+          .lessThan(dataTypeMap(name).fieldName, getType(name), castedValue).end())
+
+      case IsNull(name) if dataTypeMap.contains(name) =>
+        Some(builder.startAnd()
+          .isNull(dataTypeMap(name).fieldName, getType(name)).end())
+
+      case IsNotNull(name) if dataTypeMap.contains(name) =>
+        Some(builder.startNot()
+          .isNull(dataTypeMap(name).fieldName, getType(name)).end())
+
+      case In(name, values) if dataTypeMap.contains(name) =>
+        val castedValues = values.map(v => castLiteralValue(v, dataTypeMap(name).fieldType))
+        Some(builder.startAnd().in(dataTypeMap(name).fieldName, getType(name),
+          castedValues.map(_.asInstanceOf[AnyRef]): _*).end())
+
+      case _ => None
+    }
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFiltersBase.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFiltersBase.scala
index e673309188756..b7de20ae29349 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFiltersBase.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFiltersBase.scala
@@ -17,8 +17,11 @@
 
 package org.apache.spark.sql.execution.datasources.orc
 
+import java.util.Locale
+
+import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
 import org.apache.spark.sql.sources.{And, Filter}
-import org.apache.spark.sql.types.{AtomicType, BinaryType, DataType}
+import org.apache.spark.sql.types.{AtomicType, BinaryType, DataType, StructField, StructType}
 
 /**
  * Methods that can be shared when upgrading the built-in Hive.
@@ -36,13 +39,50 @@ trait OrcFiltersBase {
     }
   }
 
+  case class OrcPrimitiveField(fieldName: String, fieldType: DataType)
+
   /**
-   * Return true if this is a searchable type in ORC.
-   * Both CharType and VarcharType are cleaned at AstBuilder.
+   * This method returns a map which contains ORC field name and data type. Each key
+   * represents a column; `dots` are used as separators for nested columns. If any part
+   * of the names contains `dots`, it is quoted to avoid confusion. See
+   * `org.apache.spark.sql.connector.catalog.quoted` for implementation details.
+   *
+   * BinaryType, UserDefinedType, ArrayType and MapType are ignored.
    */
-  protected[sql] def isSearchableType(dataType: DataType) = dataType match {
-    case BinaryType => false
-    case _: AtomicType => true
-    case _ => false
+  protected[sql] def getSearchableTypeMap(
+      schema: StructType,
+      caseSensitive: Boolean): Map[String, OrcPrimitiveField] = {
+    import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
+
+    def getPrimitiveFields(
+        fields: Seq[StructField],
+        parentFieldNames: Seq[String] = Seq.empty): Seq[(String, OrcPrimitiveField)] = {
+      fields.flatMap { f =>
+        f.dataType match {
+          case st: StructType =>
+            getPrimitiveFields(st.fields, parentFieldNames :+ f.name)
+          case BinaryType => None
+          case _: AtomicType =>
+            val fieldName = (parentFieldNames :+ f.name).quoted
+            val orcField = OrcPrimitiveField(fieldName, f.dataType)
+            Some((fieldName, orcField))
+          case _ => None
+        }
+      }
+    }
+
+    val primitiveFields = getPrimitiveFields(schema.fields)
+    if (caseSensitive) {
+      primitiveFields.toMap
+    } else {
+      // Don't consider ambiguity here, i.e. more than one field are matched in case insensitive
+      // mode, just skip pushdown for these fields, they will trigger Exception when reading,
+      // See: SPARK-25175.
+      val dedupPrimitiveFields = primitiveFields
+        .groupBy(_._1.toLowerCase(Locale.ROOT))
+        .filter(_._2.size == 1)
+        .mapValues(_.head._2)
+      CaseInsensitiveMap(dedupPrimitiveFields.toMap)
+    }
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcOptions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcOptions.scala
index 25f022bcdde89..af92d94d68be9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcOptions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcOptions.scala
@@ -77,7 +77,8 @@ object OrcOptions {
     "uncompressed" -> "NONE",
     "snappy" -> "SNAPPY",
     "zlib" -> "ZLIB",
-    "lzo" -> "LZO")
+    "lzo" -> "LZO",
+    "zstd" -> "ZSTD")
 
   def getORCCompressionCodecName(name: String): String = shortOrcCompressionCodecNames(name)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcOutputWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcOutputWriter.scala
index 08086bcd91f6e..6f215737f5703 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcOutputWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcOutputWriter.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.execution.datasources.OutputWriter
 import org.apache.spark.sql.types._
 
 private[sql] class OrcOutputWriter(
-    path: String,
+    val path: String,
     dataSchema: StructType,
     context: TaskAttemptContext)
   extends OutputWriter {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcSerializer.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcSerializer.scala
index 0b9cbecd0d329..ac32be2fe12b1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcSerializer.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcSerializer.scala
@@ -24,6 +24,7 @@ import org.apache.orc.mapred.{OrcList, OrcMap, OrcStruct, OrcTimestamp}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.SpecializedGetters
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.types._
 
 /**
@@ -206,7 +207,7 @@ class OrcSerializer(dataSchema: StructType) {
     case udt: UserDefinedType[_] => newConverter(udt.sqlType)
 
     case _ =>
-      throw new UnsupportedOperationException(s"$dataType is not supported yet.")
+      throw QueryExecutionErrors.dataTypeUnsupportedYetError(dataType)
   }
 
   /**
diff --git a/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala
similarity index 100%
rename from sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala
rename to sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
index eea9b2a8f9613..c81ac3c940c30 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
@@ -24,15 +24,16 @@ import scala.collection.JavaConverters._
 
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.{FileStatus, Path}
-import org.apache.orc.{OrcFile, Reader, TypeDescription, Writer}
+import org.apache.orc.{OrcConf, OrcFile, Reader, TypeDescription, Writer}
 
-import org.apache.spark.{SPARK_VERSION_SHORT, SparkException}
+import org.apache.spark.SPARK_VERSION_SHORT
 import org.apache.spark.deploy.SparkHadoopUtil
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.{SPARK_VERSION_METADATA_KEY, SparkSession}
 import org.apache.spark.sql.catalyst.analysis.caseSensitiveResolution
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
 import org.apache.spark.sql.catalyst.util.quoteIdentifier
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.execution.datasources.SchemaMergeUtils
 import org.apache.spark.sql.types._
 import org.apache.spark.util.{ThreadUtils, Utils}
@@ -44,6 +45,7 @@ object OrcUtils extends Logging {
     "NONE" -> "",
     "SNAPPY" -> ".snappy",
     "ZLIB" -> ".zlib",
+    "ZSTD" -> ".zstd",
     "LZO" -> ".lzo")
 
   def listOrcFiles(pathStr: String, conf: Configuration): Seq[Path] = {
@@ -76,15 +78,15 @@ object OrcUtils extends Logging {
           logWarning(s"Skipped the footer in the corrupted file: $file", e)
           None
         } else {
-          throw new SparkException(s"Could not read footer for file: $file", e)
+          throw QueryExecutionErrors.cannotReadFooterForFileError(file, e)
         }
     }
   }
 
-  def readSchema(sparkSession: SparkSession, files: Seq[FileStatus])
+  def readSchema(sparkSession: SparkSession, files: Seq[FileStatus], options: Map[String, String])
       : Option[StructType] = {
     val ignoreCorruptFiles = sparkSession.sessionState.conf.ignoreCorruptFiles
-    val conf = sparkSession.sessionState.newHadoopConf()
+    val conf = sparkSession.sessionState.newHadoopConfWithOptions(options)
     files.toIterator.map(file => readSchema(file.getPath, conf, ignoreCorruptFiles)).collectFirst {
       case Some(schema) =>
         logDebug(s"Reading schema from file $files, got Hive schema string: $schema")
@@ -92,6 +94,20 @@ object OrcUtils extends Logging {
     }
   }
 
+  def readCatalystSchema(
+      file: Path,
+      conf: Configuration,
+      ignoreCorruptFiles: Boolean): Option[StructType] = {
+    readSchema(file, conf, ignoreCorruptFiles) match {
+      case Some(schema) =>
+        Some(CatalystSqlParser.parseDataType(schema.toString).asInstanceOf[StructType])
+
+      case None =>
+        // Field names is empty or `FileFormatException` was thrown but ignoreCorruptFiles is true.
+        None
+    }
+  }
+
   /**
    * Reads ORC file schemas in multi-threaded manner, using native version of ORC.
    * This is visible for testing.
@@ -109,33 +125,44 @@ object OrcUtils extends Logging {
     val orcOptions = new OrcOptions(options, sparkSession.sessionState.conf)
     if (orcOptions.mergeSchema) {
       SchemaMergeUtils.mergeSchemasInParallel(
-        sparkSession, files, OrcUtils.readOrcSchemasInParallel)
+        sparkSession, options, files, OrcUtils.readOrcSchemasInParallel)
     } else {
-      OrcUtils.readSchema(sparkSession, files)
+      OrcUtils.readSchema(sparkSession, files, options)
     }
   }
 
   /**
-   * Returns the requested column ids from the given ORC file. Column id can be -1, which means the
-   * requested column doesn't exist in the ORC file. Returns None if the given ORC file is empty.
+   * @return Returns the combination of requested column ids from the given ORC file and
+   *         boolean flag to find if the pruneCols is allowed or not. Requested Column id can be
+   *         -1, which means the requested column doesn't exist in the ORC file. Returns None
+   *         if the given ORC file is empty.
    */
   def requestedColumnIds(
       isCaseSensitive: Boolean,
       dataSchema: StructType,
       requiredSchema: StructType,
       reader: Reader,
-      conf: Configuration): Option[Array[Int]] = {
+      conf: Configuration): Option[(Array[Int], Boolean)] = {
     val orcFieldNames = reader.getSchema.getFieldNames.asScala
+    val forcePositionalEvolution = OrcConf.FORCE_POSITIONAL_EVOLUTION.getBoolean(conf)
     if (orcFieldNames.isEmpty) {
       // SPARK-8501: Some old empty ORC files always have an empty schema stored in their footer.
       None
     } else {
-      if (orcFieldNames.forall(_.startsWith("_col"))) {
-        // This is a ORC file written by Hive, no field names in the physical schema, assume the
-        // physical schema maps to the data scheme by index.
+      if (forcePositionalEvolution || orcFieldNames.forall(_.startsWith("_col"))) {
+        // This is either an ORC file written by an old version of Hive and there are no field
+        // names in the physical schema, or `orc.force.positional.evolution=true` is forced because
+        // the file was written by a newer version of Hive where
+        // `orc.force.positional.evolution=true` was set (possibly because columns were renamed so
+        // the physical schema doesn't match the data schema).
+        // In these cases we map the physical schema to the data schema by index.
         assert(orcFieldNames.length <= dataSchema.length, "The given data schema " +
           s"${dataSchema.catalogString} has less fields than the actual ORC physical schema, " +
           "no idea which columns were dropped, fail to read.")
+        // for ORC file written by Hive, no field names
+        // in the physical schema, there is a need to send the
+        // entire dataSchema instead of required schema.
+        // So pruneCols is not done in this case
         Some(requiredSchema.fieldNames.map { name =>
           val index = dataSchema.fieldIndex(name)
           if (index < orcFieldNames.length) {
@@ -143,7 +170,7 @@ object OrcUtils extends Logging {
           } else {
             -1
           }
-        })
+        }, false)
       } else {
         if (isCaseSensitive) {
           Some(requiredSchema.fieldNames.zipWithIndex.map { case (name, idx) =>
@@ -152,7 +179,7 @@ object OrcUtils extends Logging {
             } else {
               -1
             }
-          })
+          }, true)
         } else {
           // Do case-insensitive resolution only if in case-insensitive mode
           val caseInsensitiveOrcFieldMap = orcFieldNames.groupBy(_.toLowerCase(Locale.ROOT))
@@ -164,13 +191,13 @@ object OrcUtils extends Logging {
                   // Need to fail if there is ambiguity, i.e. more than one field is matched.
                   val matchedOrcFieldsString = matchedOrcFields.mkString("[", ", ", "]")
                   reader.close()
-                  throw new RuntimeException(s"""Found duplicate field(s) "$requiredFieldName": """
-                    + s"$matchedOrcFieldsString in case-insensitive mode")
+                  throw QueryExecutionErrors.foundDuplicateFieldInCaseInsensitiveModeError(
+                    requiredFieldName, matchedOrcFieldsString)
                 } else {
                   idx
                 }
               }.getOrElse(-1)
-          })
+          }, true)
         }
       }
     }
@@ -199,4 +226,31 @@ object OrcUtils extends Logging {
       s"map<${orcTypeDescriptionString(m.keyType)},${orcTypeDescriptionString(m.valueType)}>"
     case _ => dt.catalogString
   }
+
+  /**
+   * Returns the result schema to read from ORC file. In addition, It sets
+   * the schema string to 'orc.mapred.input.schema' so ORC reader can use later.
+   *
+   * @param canPruneCols Flag to decide whether pruned cols schema is send to resultSchema
+   *                     or to send the entire dataSchema to resultSchema.
+   * @param dataSchema   Schema of the orc files.
+   * @param resultSchema Result data schema created after pruning cols.
+   * @param partitionSchema Schema of partitions.
+   * @param conf Hadoop Configuration.
+   * @return Returns the result schema as string.
+   */
+  def orcResultSchemaString(
+      canPruneCols: Boolean,
+      dataSchema: StructType,
+      resultSchema: StructType,
+      partitionSchema: StructType,
+      conf: Configuration): String = {
+    val resultSchemaString = if (canPruneCols) {
+      OrcUtils.orcTypeDescriptionString(resultSchema)
+    } else {
+      OrcUtils.orcTypeDescriptionString(StructType(dataSchema.fields ++ partitionSchema.fields))
+    }
+    OrcConf.MAPRED_INPUT_SCHEMA.setString(conf, resultSchemaString)
+    resultSchemaString
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala
index 71874104fcf4f..48e2e6e57d838 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala
@@ -17,7 +17,6 @@
 
 package org.apache.spark.sql.execution.datasources.parquet
 
-import java.io.IOException
 import java.net.URI
 
 import scala.collection.JavaConverters._
@@ -26,6 +25,7 @@ import scala.util.{Failure, Try}
 
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.{FileStatus, Path}
+import org.apache.hadoop.mapred.FileSplit
 import org.apache.hadoop.mapreduce._
 import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
 import org.apache.parquet.filter2.compat.FilterCompat
@@ -35,9 +35,8 @@ import org.apache.parquet.hadoop._
 import org.apache.parquet.hadoop.ParquetOutputFormat.JobSummaryLevel
 import org.apache.parquet.hadoop.codec.CodecConfig
 import org.apache.parquet.hadoop.util.ContextUtil
-import org.apache.parquet.schema.MessageType
 
-import org.apache.spark.{SparkException, TaskContext}
+import org.apache.spark.TaskContext
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.InternalRow
@@ -45,6 +44,7 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
 import org.apache.spark.sql.catalyst.parser.LegacyTypeStringParser
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.execution.datasources._
 import org.apache.spark.sql.execution.vectorized.{OffHeapColumnVector, OnHeapColumnVector}
 import org.apache.spark.sql.internal.SQLConf
@@ -252,24 +252,20 @@ class ParquetFileFormat
     val pushDownStringStartWith = sqlConf.parquetFilterPushDownStringStartWith
     val pushDownInFilterThreshold = sqlConf.parquetFilterPushDownInFilterThreshold
     val isCaseSensitive = sqlConf.caseSensitiveAnalysis
+    val parquetOptions = new ParquetOptions(options, sparkSession.sessionState.conf)
+    val datetimeRebaseModeInRead = parquetOptions.datetimeRebaseModeInRead
+    val int96RebaseModeInRead = parquetOptions.int96RebaseModeInRead
 
     (file: PartitionedFile) => {
       assert(file.partitionValues.numFields == partitionSchema.size)
 
       val filePath = new Path(new URI(file.filePath))
-      val split =
-        new org.apache.parquet.hadoop.ParquetInputSplit(
-          filePath,
-          file.start,
-          file.start + file.length,
-          file.length,
-          Array.empty,
-          null)
+      val split = new FileSplit(filePath, file.start, file.length, Array.empty[String])
 
       val sharedConf = broadcastedHadoopConf.value.value
 
       lazy val footerFileMetaData =
-        ParquetFileReader.readFooter(sharedConf, filePath, SKIP_ROW_GROUPS).getFileMetaData
+        ParquetFooterReader.readFooter(sharedConf, filePath, SKIP_ROW_GROUPS).getFileMetaData
       // Try to push down filters when filter push-down is enabled.
       val pushed = if (enableParquetFilterPushDown) {
         val parquetSchema = footerFileMetaData.getSchema
@@ -302,7 +298,10 @@ class ParquetFileFormat
 
       val datetimeRebaseMode = DataSourceUtils.datetimeRebaseMode(
         footerFileMetaData.getKeyValueMetaData.get,
-        SQLConf.get.getConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ))
+        datetimeRebaseModeInRead)
+      val int96RebaseMode = DataSourceUtils.int96RebaseMode(
+        footerFileMetaData.getKeyValueMetaData.get,
+        int96RebaseModeInRead)
 
       val attemptId = new TaskAttemptID(new TaskID(new JobID(), TaskType.MAP, 0), 0)
       val hadoopAttemptContext =
@@ -318,6 +317,7 @@ class ParquetFileFormat
         val vectorizedReader = new VectorizedParquetRecordReader(
           convertTz.orNull,
           datetimeRebaseMode.toString,
+          int96RebaseMode.toString,
           enableOffHeapColumnVector && taskContext.isDefined,
           capacity)
         val iter = new RecordReaderIterator(vectorizedReader)
@@ -336,7 +336,10 @@ class ParquetFileFormat
         logDebug(s"Falling back to parquet-mr")
         // ParquetRecordReader returns InternalRow
         val readSupport = new ParquetReadSupport(
-          convertTz, enableVectorizedReader = false, datetimeRebaseMode)
+          convertTz,
+          enableVectorizedReader = false,
+          datetimeRebaseMode,
+          int96RebaseMode)
         val reader = if (pushed.isDefined && enableRecordFilter) {
           val parquetFilter = FilterCompat.get(pushed.get, null)
           new ParquetRecordReader[InternalRow](readSupport, parquetFilter)
@@ -427,7 +430,7 @@ object ParquetFileFormat extends Logging {
 
     finalSchemas.reduceOption { (left, right) =>
       try left.merge(right) catch { case e: Throwable =>
-        throw new SparkException(s"Failed to merge incompatible schemas $left and $right", e)
+        throw QueryExecutionErrors.failedToMergeIncompatibleSchemasError(left, right, e)
       }
     }
   }
@@ -447,14 +450,14 @@ object ParquetFileFormat extends Logging {
         // ParquetFileReader.readFooter throws RuntimeException, instead of IOException,
         // when it can't read the footer.
         Some(new Footer(currentFile.getPath(),
-          ParquetFileReader.readFooter(
+          ParquetFooterReader.readFooter(
             conf, currentFile, SKIP_ROW_GROUPS)))
       } catch { case e: RuntimeException =>
         if (ignoreCorruptFiles) {
           logWarning(s"Skipped the footer in the corrupted file: $currentFile", e)
           None
         } else {
-          throw new IOException(s"Could not read footer for file: $currentFile", e)
+          throw QueryExecutionErrors.cannotReadFooterForFileError(currentFile, e)
         }
       }
     }.flatten
@@ -475,6 +478,7 @@ object ParquetFileFormat extends Logging {
    *     S3 nodes).
    */
   def mergeSchemasInParallel(
+      parameters: Map[String, String],
       filesToTouch: Seq[FileStatus],
       sparkSession: SparkSession): Option[StructType] = {
     val assumeBinaryIsString = sparkSession.sessionState.conf.isParquetBinaryAsString
@@ -490,13 +494,14 @@ object ParquetFileFormat extends Logging {
         .map(ParquetFileFormat.readSchemaFromFooter(_, converter))
     }
 
-    SchemaMergeUtils.mergeSchemasInParallel(sparkSession, filesToTouch, reader)
+    SchemaMergeUtils.mergeSchemasInParallel(sparkSession, parameters, filesToTouch, reader)
   }
 
   /**
    * Reads Spark SQL schema from a Parquet footer.  If a valid serialized Spark SQL schema string
    * can be found in the file metadata, returns the deserialized [[StructType]], otherwise, returns
-   * a [[StructType]] converted from the [[MessageType]] stored in this footer.
+   * a [[StructType]] converted from the [[org.apache.parquet.schema.MessageType]] stored in this
+   * footer.
    */
   def readSchemaFromFooter(
       footer: Footer, converter: ParquetToSparkSchemaConverter): StructType = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
index d89186af8c8e5..73910c3943e9a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.execution.datasources.parquet
 import java.lang.{Boolean => JBoolean, Double => JDouble, Float => JFloat, Long => JLong}
 import java.math.{BigDecimal => JBigDecimal}
 import java.sql.{Date, Timestamp}
-import java.time.LocalDate
+import java.time.{Instant, LocalDate}
 import java.util.Locale
 
 import scala.collection.JavaConverters.asScalaBufferConverter
@@ -34,7 +34,6 @@ import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName
 import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName._
 
 import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateTimeUtils}
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.SQLDate
 import org.apache.spark.sql.sources
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -69,14 +68,14 @@ class ParquetFilters(
         // When g is a `Map`, `g.getOriginalType` is `MAP`.
         // When g is a `List`, `g.getOriginalType` is `LIST`.
         case g: GroupType if g.getOriginalType == null =>
-          getPrimitiveFields(g.getFields.asScala, parentFieldNames :+ g.getName)
+          getPrimitiveFields(g.getFields.asScala.toSeq, parentFieldNames :+ g.getName)
         // Parquet only supports push-down for primitive types; as a result, Map and List types
         // are removed.
         case _ => None
       }
     }
 
-    val primitiveFields = getPrimitiveFields(schema.getFields.asScala).map { field =>
+    val primitiveFields = getPrimitiveFields(schema.getFields.asScala.toSeq).map { field =>
       import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
       (field.fieldNames.toSeq.quoted, field)
     }
@@ -91,7 +90,7 @@ class ParquetFilters(
         .groupBy(_._1.toLowerCase(Locale.ROOT))
         .filter(_._2.size == 1)
         .mapValues(_.head._2)
-      CaseInsensitiveMap(dedupPrimitiveFields)
+      CaseInsensitiveMap(dedupPrimitiveFields.toMap)
     }
   }
 
@@ -124,11 +123,16 @@ class ParquetFilters(
   private val ParquetTimestampMicrosType = ParquetSchemaType(TIMESTAMP_MICROS, INT64, 0, null)
   private val ParquetTimestampMillisType = ParquetSchemaType(TIMESTAMP_MILLIS, INT64, 0, null)
 
-  private def dateToDays(date: Any): SQLDate = date match {
+  private def dateToDays(date: Any): Int = date match {
     case d: Date => DateTimeUtils.fromJavaDate(d)
     case ld: LocalDate => DateTimeUtils.localDateToDays(ld)
   }
 
+  private def timestampToMicros(v: Any): JLong = v match {
+    case i: Instant => DateTimeUtils.instantToMicros(i)
+    case t: Timestamp => DateTimeUtils.fromJavaTimestamp(t)
+  }
+
   private def decimalToInt32(decimal: JBigDecimal): Integer = decimal.unscaledValue().intValue()
 
   private def decimalToInt64(decimal: JBigDecimal): JLong = decimal.unscaledValue().longValue()
@@ -148,6 +152,12 @@ class ParquetFilters(
     Binary.fromConstantByteArray(fixedLengthBytes, 0, numBytes)
   }
 
+  private def timestampToMillis(v: Any): JLong = {
+    val micros = timestampToMicros(v)
+    val millis = DateTimeUtils.microsToMillis(micros)
+    millis.asInstanceOf[JLong]
+  }
+
   private val makeEq:
     PartialFunction[ParquetSchemaType, (Array[String], Any) => FilterPredicate] = {
     case ParquetBooleanType =>
@@ -179,12 +189,11 @@ class ParquetFilters(
     case ParquetTimestampMicrosType if pushDownTimestamp =>
       (n: Array[String], v: Any) => FilterApi.eq(
         longColumn(n),
-        Option(v).map(t => DateTimeUtils.fromJavaTimestamp(t.asInstanceOf[Timestamp])
-          .asInstanceOf[JLong]).orNull)
+        Option(v).map(timestampToMicros).orNull)
     case ParquetTimestampMillisType if pushDownTimestamp =>
       (n: Array[String], v: Any) => FilterApi.eq(
         longColumn(n),
-        Option(v).map(_.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong]).orNull)
+        Option(v).map(timestampToMillis).orNull)
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
       (n: Array[String], v: Any) => FilterApi.eq(
@@ -230,12 +239,11 @@ class ParquetFilters(
     case ParquetTimestampMicrosType if pushDownTimestamp =>
       (n: Array[String], v: Any) => FilterApi.notEq(
         longColumn(n),
-        Option(v).map(t => DateTimeUtils.fromJavaTimestamp(t.asInstanceOf[Timestamp])
-          .asInstanceOf[JLong]).orNull)
+        Option(v).map(timestampToMicros).orNull)
     case ParquetTimestampMillisType if pushDownTimestamp =>
       (n: Array[String], v: Any) => FilterApi.notEq(
         longColumn(n),
-        Option(v).map(_.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong]).orNull)
+        Option(v).map(timestampToMillis).orNull)
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
       (n: Array[String], v: Any) => FilterApi.notEq(
@@ -273,13 +281,9 @@ class ParquetFilters(
       (n: Array[String], v: Any) =>
         FilterApi.lt(intColumn(n), dateToDays(v).asInstanceOf[Integer])
     case ParquetTimestampMicrosType if pushDownTimestamp =>
-      (n: Array[String], v: Any) => FilterApi.lt(
-        longColumn(n),
-        DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.lt(longColumn(n), timestampToMicros(v))
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: Array[String], v: Any) => FilterApi.lt(
-        longColumn(n),
-        v.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.lt(longColumn(n), timestampToMillis(v))
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
       (n: Array[String], v: Any) =>
@@ -314,13 +318,9 @@ class ParquetFilters(
       (n: Array[String], v: Any) =>
         FilterApi.ltEq(intColumn(n), dateToDays(v).asInstanceOf[Integer])
     case ParquetTimestampMicrosType if pushDownTimestamp =>
-      (n: Array[String], v: Any) => FilterApi.ltEq(
-        longColumn(n),
-        DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.ltEq(longColumn(n), timestampToMicros(v))
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: Array[String], v: Any) => FilterApi.ltEq(
-        longColumn(n),
-        v.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.ltEq(longColumn(n), timestampToMillis(v))
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
       (n: Array[String], v: Any) =>
@@ -355,13 +355,9 @@ class ParquetFilters(
       (n: Array[String], v: Any) =>
         FilterApi.gt(intColumn(n), dateToDays(v).asInstanceOf[Integer])
     case ParquetTimestampMicrosType if pushDownTimestamp =>
-      (n: Array[String], v: Any) => FilterApi.gt(
-        longColumn(n),
-        DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.gt(longColumn(n), timestampToMicros(v))
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: Array[String], v: Any) => FilterApi.gt(
-        longColumn(n),
-        v.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.gt(longColumn(n), timestampToMillis(v))
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
       (n: Array[String], v: Any) =>
@@ -396,13 +392,9 @@ class ParquetFilters(
       (n: Array[String], v: Any) =>
         FilterApi.gtEq(intColumn(n), dateToDays(v).asInstanceOf[Integer])
     case ParquetTimestampMicrosType if pushDownTimestamp =>
-      (n: Array[String], v: Any) => FilterApi.gtEq(
-        longColumn(n),
-        DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.gtEq(longColumn(n), timestampToMicros(v))
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: Array[String], v: Any) => FilterApi.gtEq(
-        longColumn(n),
-        v.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.gtEq(longColumn(n), timestampToMillis(v))
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
       (n: Array[String], v: Any) =>
@@ -476,7 +468,7 @@ class ParquetFilters(
       case ParquetDateType =>
         value.isInstanceOf[Date] || value.isInstanceOf[LocalDate]
       case ParquetTimestampMicrosType | ParquetTimestampMillisType =>
-        value.isInstanceOf[Timestamp]
+        value.isInstanceOf[Timestamp] || value.isInstanceOf[Instant]
       case ParquetSchemaType(DECIMAL, INT32, _, decimalMeta) =>
         isDecimalMatched(value, decimalMeta)
       case ParquetSchemaType(DECIMAL, INT64, _, decimalMeta) =>
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetOptions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetOptions.scala
index 9cfc30725f03a..da0c163dd85b5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetOptions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetOptions.scala
@@ -69,6 +69,19 @@ class ParquetOptions(
     .get(MERGE_SCHEMA)
     .map(_.toBoolean)
     .getOrElse(sqlConf.isParquetSchemaMergingEnabled)
+
+  /**
+   * The rebasing mode for the DATE and TIMESTAMP_MICROS, TIMESTAMP_MILLIS values in reads.
+   */
+  def datetimeRebaseModeInRead: String = parameters
+    .get(DATETIME_REBASE_MODE)
+    .getOrElse(sqlConf.getConf(SQLConf.PARQUET_REBASE_MODE_IN_READ))
+  /**
+   * The rebasing mode for INT96 timestamp values in reads.
+   */
+  def int96RebaseModeInRead: String = parameters
+    .get(INT96_REBASE_MODE)
+    .getOrElse(sqlConf.getConf(SQLConf.PARQUET_INT96_REBASE_MODE_IN_READ))
 }
 
 
@@ -89,4 +102,16 @@ object ParquetOptions {
   def getParquetCompressionCodecName(name: String): String = {
     shortParquetCompressionCodecNames(name).name()
   }
+
+  // The option controls rebasing of the DATE and TIMESTAMP values between
+  // Julian and Proleptic Gregorian calendars. It impacts on the behaviour of the Parquet
+  // datasource similarly to the SQL config `spark.sql.parquet.datetimeRebaseModeInRead`,
+  // and can be set to the same values: `EXCEPTION`, `LEGACY` or `CORRECTED`.
+  val DATETIME_REBASE_MODE = "datetimeRebaseMode"
+
+  // The option controls rebasing of the INT96 timestamp values between Julian and Proleptic
+  // Gregorian calendars. It impacts on the behaviour of the Parquet datasource similarly to
+  // the SQL config `spark.sql.parquet.int96RebaseModeInRead`.
+  // The valid option values are: `EXCEPTION`, `LEGACY` or `CORRECTED`.
+  val INT96_REBASE_MODE = "int96RebaseMode"
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetOutputWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetOutputWriter.scala
index e7753cec681cf..efb322f3fc906 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetOutputWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetOutputWriter.scala
@@ -21,12 +21,11 @@ import org.apache.hadoop.fs.Path
 import org.apache.hadoop.mapreduce._
 import org.apache.parquet.hadoop.ParquetOutputFormat
 
-import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.execution.datasources.OutputWriter
 
 // NOTE: This class is instantiated and used on executor side only, no need to be serializable.
-class ParquetOutputWriter(path: String, context: TaskAttemptContext)
+class ParquetOutputWriter(val path: String, context: TaskAttemptContext)
   extends OutputWriter {
 
   private val recordWriter: RecordWriter[Void, InternalRow] = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala
index a30d1c26b3b2d..ce06620a62cb2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala
@@ -31,6 +31,7 @@ import org.apache.parquet.schema.Type.Repetition
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 import org.apache.spark.sql.types._
@@ -54,15 +55,20 @@ import org.apache.spark.sql.types._
 class ParquetReadSupport(
     val convertTz: Option[ZoneId],
     enableVectorizedReader: Boolean,
-    datetimeRebaseMode: LegacyBehaviorPolicy.Value)
+    datetimeRebaseMode: LegacyBehaviorPolicy.Value,
+    int96RebaseMode: LegacyBehaviorPolicy.Value)
   extends ReadSupport[InternalRow] with Logging {
   private var catalystRequestedSchema: StructType = _
 
-  def this() {
+  def this() = {
     // We need a zero-arg constructor for SpecificParquetRecordReaderBase.  But that is only
     // used in the vectorized reader, where we get the convertTz/rebaseDateTime value directly,
     // and the values here are ignored.
-    this(None, enableVectorizedReader = true, datetimeRebaseMode = LegacyBehaviorPolicy.CORRECTED)
+    this(
+      None,
+      enableVectorizedReader = true,
+      datetimeRebaseMode = LegacyBehaviorPolicy.CORRECTED,
+      int96RebaseMode = LegacyBehaviorPolicy.LEGACY)
   }
 
   /**
@@ -131,7 +137,8 @@ class ParquetReadSupport(
       ParquetReadSupport.expandUDT(catalystRequestedSchema),
       new ParquetToSparkSchemaConverter(conf),
       convertTz,
-      datetimeRebaseMode)
+      datetimeRebaseMode,
+      int96RebaseMode)
   }
 }
 
@@ -336,8 +343,8 @@ object ParquetReadSupport {
             if (parquetTypes.size > 1) {
               // Need to fail if there is ambiguity, i.e. more than one field is matched
               val parquetTypesString = parquetTypes.map(_.getName).mkString("[", ", ", "]")
-              throw new RuntimeException(s"""Found duplicate field(s) "${f.name}": """ +
-                s"$parquetTypesString in case-insensitive mode")
+              throw QueryExecutionErrors.foundDuplicateFieldInCaseInsensitiveModeError(
+                f.name, parquetTypesString)
             } else {
               clipParquetType(parquetTypes.head, f.dataType, caseSensitive)
             }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRecordMaterializer.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRecordMaterializer.scala
index bb528d548b6ef..80763ef019b01 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRecordMaterializer.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRecordMaterializer.scala
@@ -35,17 +35,26 @@ import org.apache.spark.sql.types.StructType
  * @param convertTz the optional time zone to convert to int96 data
  * @param datetimeRebaseMode the mode of rebasing date/timestamp from Julian to Proleptic Gregorian
  *                           calendar
+ * @param int96RebaseMode the mode of rebasing INT96 timestamp from Julian to Proleptic Gregorian
+ *                           calendar
  */
 private[parquet] class ParquetRecordMaterializer(
     parquetSchema: MessageType,
     catalystSchema: StructType,
     schemaConverter: ParquetToSparkSchemaConverter,
     convertTz: Option[ZoneId],
-    datetimeRebaseMode: LegacyBehaviorPolicy.Value)
+    datetimeRebaseMode: LegacyBehaviorPolicy.Value,
+    int96RebaseMode: LegacyBehaviorPolicy.Value)
   extends RecordMaterializer[InternalRow] {
 
   private val rootConverter = new ParquetRowConverter(
-    schemaConverter, parquetSchema, catalystSchema, convertTz, datetimeRebaseMode, NoopUpdater)
+    schemaConverter,
+    parquetSchema,
+    catalystSchema,
+    convertTz,
+    datetimeRebaseMode,
+    int96RebaseMode,
+    NoopUpdater)
 
   override def getCurrentRecord: InternalRow = rootConverter.currentRecord
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
index 201ee16faeb08..0a1cca7ed0f3f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
@@ -26,15 +26,14 @@ import scala.collection.mutable.ArrayBuffer
 
 import org.apache.parquet.column.Dictionary
 import org.apache.parquet.io.api.{Binary, Converter, GroupConverter, PrimitiveConverter}
-import org.apache.parquet.schema.{GroupType, MessageType, OriginalType, Type}
-import org.apache.parquet.schema.OriginalType.{INT_32, LIST, UTF8}
-import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.{BINARY, DOUBLE, FIXED_LEN_BYTE_ARRAY, INT32, INT64, INT96}
+import org.apache.parquet.schema.{GroupType, OriginalType, Type}
+import org.apache.parquet.schema.OriginalType.LIST
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.{BINARY, FIXED_LEN_BYTE_ARRAY, INT32, INT64, INT96}
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, CaseInsensitiveMap, DateTimeUtils, GenericArrayData}
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.SQLTimestamp
 import org.apache.spark.sql.execution.datasources.DataSourceUtils
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
@@ -108,11 +107,15 @@ private[parquet] class ParquetPrimitiveConverter(val updater: ParentContainerUpd
  * }}}
  * 5 converters will be created:
  *
- * - a root [[ParquetRowConverter]] for [[MessageType]] `root`, which contains:
- *   - a [[ParquetPrimitiveConverter]] for required [[INT_32]] field `f1`, and
+ * - a root [[ParquetRowConverter]] for [[org.apache.parquet.schema.MessageType]] `root`,
+ * which contains:
+ *   - a [[ParquetPrimitiveConverter]] for required
+ *   [[org.apache.parquet.schema.OriginalType.INT_32]] field `f1`, and
  *   - a nested [[ParquetRowConverter]] for optional [[GroupType]] `f2`, which contains:
- *     - a [[ParquetPrimitiveConverter]] for required [[DOUBLE]] field `f21`, and
- *     - a [[ParquetStringConverter]] for optional [[UTF8]] string field `f22`
+ *     - a [[ParquetPrimitiveConverter]] for required
+ *     [[org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE]] field `f21`, and
+ *     - a [[ParquetStringConverter]] for optional [[org.apache.parquet.schema.OriginalType.UTF8]]
+ *     string field `f22`
  *
  * When used as a root converter, [[NoopUpdater]] should be used since root converters don't have
  * any "parent" container.
@@ -124,6 +127,8 @@ private[parquet] class ParquetPrimitiveConverter(val updater: ParentContainerUpd
  * @param convertTz the optional time zone to convert to int96 data
  * @param datetimeRebaseMode the mode of rebasing date/timestamp from Julian to Proleptic Gregorian
  *                           calendar
+ * @param int96RebaseMode the mode of rebasing INT96 timestamp from Julian to Proleptic Gregorian
+ *                           calendar
  * @param updater An updater which propagates converted field values to the parent container
  */
 private[parquet] class ParquetRowConverter(
@@ -132,6 +137,7 @@ private[parquet] class ParquetRowConverter(
     catalystType: StructType,
     convertTz: Option[ZoneId],
     datetimeRebaseMode: LegacyBehaviorPolicy.Value,
+    int96RebaseMode: LegacyBehaviorPolicy.Value,
     updater: ParentContainerUpdater)
   extends ParquetGroupConverter(updater) with Logging {
 
@@ -188,6 +194,9 @@ private[parquet] class ParquetRowConverter(
   private val timestampRebaseFunc = DataSourceUtils.creteTimestampRebaseFuncInRead(
     datetimeRebaseMode, "Parquet")
 
+  private val int96RebaseFunc = DataSourceUtils.creteTimestampRebaseFuncInRead(
+    int96RebaseMode, "Parquet INT96")
+
   // Converters for each field.
   private[this] val fieldConverters: Array[Converter with HasParentContainerUpdater] = {
     // (SPARK-31116) Use case insensitive map if spark.sql.caseSensitive is false
@@ -243,6 +252,11 @@ private[parquet] class ParquetRowConverter(
       updater: ParentContainerUpdater): Converter with HasParentContainerUpdater = {
 
     catalystType match {
+      case LongType if parquetType.getOriginalType == OriginalType.UINT_32 =>
+        new ParquetPrimitiveConverter(updater) {
+          override def addInt(value: Int): Unit =
+            updater.setLong(Integer.toUnsignedLong(value))
+        }
       case BooleanType | IntegerType | LongType | FloatType | DoubleType | BinaryType =>
         new ParquetPrimitiveConverter(updater)
 
@@ -260,17 +274,51 @@ private[parquet] class ParquetRowConverter(
 
       // For INT32 backed decimals
       case t: DecimalType if parquetType.asPrimitiveType().getPrimitiveTypeName == INT32 =>
-        new ParquetIntDictionaryAwareDecimalConverter(t.precision, t.scale, updater)
+        val metadata = parquetType.asPrimitiveType().getDecimalMetadata
+        if (metadata == null) {
+          // If the column is a plain INT32, we should pick the precision that can host the largest
+          // INT32 value.
+          new ParquetIntDictionaryAwareDecimalConverter(
+            DecimalType.IntDecimal.precision, 0, updater)
+        } else {
+          new ParquetIntDictionaryAwareDecimalConverter(
+            metadata.getPrecision, metadata.getScale, updater)
+        }
+
+      // For unsigned int64
+      case _: DecimalType if parquetType.getOriginalType == OriginalType.UINT_64 =>
+        new ParquetPrimitiveConverter(updater) {
+          override def addLong(value: Long): Unit = {
+            updater.set(Decimal(java.lang.Long.toUnsignedString(value)))
+          }
+        }
 
       // For INT64 backed decimals
       case t: DecimalType if parquetType.asPrimitiveType().getPrimitiveTypeName == INT64 =>
-        new ParquetLongDictionaryAwareDecimalConverter(t.precision, t.scale, updater)
+        val metadata = parquetType.asPrimitiveType().getDecimalMetadata
+        if (metadata == null) {
+          // If the column is a plain INT64, we should pick the precision that can host the largest
+          // INT64 value.
+          new ParquetLongDictionaryAwareDecimalConverter(
+            DecimalType.LongDecimal.precision, 0, updater)
+        } else {
+          new ParquetLongDictionaryAwareDecimalConverter(
+            metadata.getPrecision, metadata.getScale, updater)
+        }
 
       // For BINARY and FIXED_LEN_BYTE_ARRAY backed decimals
       case t: DecimalType
         if parquetType.asPrimitiveType().getPrimitiveTypeName == FIXED_LEN_BYTE_ARRAY ||
            parquetType.asPrimitiveType().getPrimitiveTypeName == BINARY =>
-        new ParquetBinaryDictionaryAwareDecimalConverter(t.precision, t.scale, updater)
+        val metadata = parquetType.asPrimitiveType().getDecimalMetadata
+        if (metadata == null) {
+          throw new RuntimeException(s"Unable to create Parquet converter for ${t.typeName} " +
+            s"whose Parquet type is $parquetType without decimal metadata. Please read this " +
+            "column/field as Spark BINARY type." )
+        } else {
+          new ParquetBinaryDictionaryAwareDecimalConverter(
+            metadata.getPrecision, metadata.getScale, updater)
+        }
 
       case t: DecimalType =>
         throw new RuntimeException(
@@ -301,17 +349,10 @@ private[parquet] class ParquetRowConverter(
         new ParquetPrimitiveConverter(updater) {
           // Converts nanosecond timestamps stored as INT96
           override def addBinary(value: Binary): Unit = {
-            assert(
-              value.length() == 12,
-              "Timestamps (with nanoseconds) are expected to be stored in 12-byte long binaries, " +
-              s"but got a ${value.length()}-byte binary.")
-
-            val buf = value.toByteBuffer.order(ByteOrder.LITTLE_ENDIAN)
-            val timeOfDayNanos = buf.getLong
-            val julianDay = buf.getInt
-            val rawTime = DateTimeUtils.fromJulianDay(julianDay, timeOfDayNanos)
-            val adjTime = convertTz.map(DateTimeUtils.convertTz(rawTime, _, ZoneOffset.UTC))
-              .getOrElse(rawTime)
+            val julianMicros = ParquetRowConverter.binaryToSQLTimestamp(value)
+            val gregorianMicros = int96RebaseFunc(julianMicros)
+            val adjTime = convertTz.map(DateTimeUtils.convertTz(gregorianMicros, _, ZoneOffset.UTC))
+              .getOrElse(gregorianMicros)
             updater.setLong(adjTime)
           }
         }
@@ -372,6 +413,7 @@ private[parquet] class ParquetRowConverter(
           t,
           convertTz,
           datetimeRebaseMode,
+          int96RebaseMode,
           wrappedUpdater)
 
       case t =>
@@ -743,7 +785,7 @@ private[parquet] object ParquetRowConverter {
     unscaled
   }
 
-  def binaryToSQLTimestamp(binary: Binary): SQLTimestamp = {
+  def binaryToSQLTimestamp(binary: Binary): Long = {
     assert(binary.length() == 12, s"Timestamps (with nanoseconds) are expected to be stored in" +
       s" 12-byte long binaries. Found a ${binary.length()}-byte binary instead.")
     val buffer = binary.toByteBuffer.order(ByteOrder.LITTLE_ENDIAN)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala
index 8ce8a86d2f026..e751c97a60c08 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala
@@ -26,6 +26,7 @@ import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName._
 import org.apache.parquet.schema.Type.Repetition._
 
 import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
@@ -79,7 +80,7 @@ class ParquetToSparkSchemaConverter(
       }
     }
 
-    StructType(fields)
+    StructType(fields.toSeq)
   }
 
   /**
@@ -97,14 +98,11 @@ class ParquetToSparkSchemaConverter(
     def typeString =
       if (originalType == null) s"$typeName" else s"$typeName ($originalType)"
 
-    def typeNotSupported() =
-      throw new AnalysisException(s"Parquet type not supported: $typeString")
-
     def typeNotImplemented() =
-      throw new AnalysisException(s"Parquet type not yet supported: $typeString")
+      throw QueryCompilationErrors.parquetTypeUnsupportedYetError(typeString)
 
     def illegalType() =
-      throw new AnalysisException(s"Illegal Parquet type: $typeString")
+      throw QueryCompilationErrors.illegalParquetTypeError(typeString)
 
     // When maxPrecision = -1, we skip precision range check, and always respect the precision
     // specified in field.getDecimalMetadata.  This is useful when interpreting decimal types stored
@@ -130,13 +128,11 @@ class ParquetToSparkSchemaConverter(
       case INT32 =>
         originalType match {
           case INT_8 => ByteType
-          case INT_16 => ShortType
-          case INT_32 | null => IntegerType
+          case INT_16 | UINT_8 => ShortType
+          case INT_32 | UINT_16 | null => IntegerType
           case DATE => DateType
           case DECIMAL => makeDecimalType(Decimal.MAX_INT_DIGITS)
-          case UINT_8 => typeNotSupported()
-          case UINT_16 => typeNotSupported()
-          case UINT_32 => typeNotSupported()
+          case UINT_32 => LongType
           case TIME_MILLIS => typeNotImplemented()
           case _ => illegalType()
         }
@@ -145,7 +141,9 @@ class ParquetToSparkSchemaConverter(
         originalType match {
           case INT_64 | null => LongType
           case DECIMAL => makeDecimalType(Decimal.MAX_LONG_DIGITS)
-          case UINT_64 => typeNotSupported()
+          // The precision to hold the largest unsigned long is:
+          // `java.lang.Long.toUnsignedString(-1).length` = 20
+          case UINT_64 => DecimalType(20, 0)
           case TIMESTAMP_MICROS => TimestampType
           case TIMESTAMP_MILLIS => TimestampType
           case _ => illegalType()
@@ -225,10 +223,6 @@ class ParquetToSparkSchemaConverter(
           s"Invalid map type: $field")
 
         val keyType = keyValueType.getType(0)
-        ParquetSchemaConverter.checkConversionRequirement(
-          keyType.isPrimitive,
-          s"Map key type is expected to be a primitive type, but found: $keyType")
-
         val valueType = keyValueType.getType(1)
         val valueOptional = valueType.isRepetition(OPTIONAL)
         MapType(
@@ -237,7 +231,7 @@ class ParquetToSparkSchemaConverter(
           valueContainsNull = valueOptional)
 
       case _ =>
-        throw new AnalysisException(s"Unrecognized Parquet type: $field")
+        throw QueryCompilationErrors.unrecognizedParquetTypeError(field.toString)
     }
   }
 
@@ -554,7 +548,7 @@ class SparkToParquetSchemaConverter(
         convertField(field.copy(dataType = udt.sqlType))
 
       case _ =>
-        throw new AnalysisException(s"Unsupported data type ${field.dataType.catalogString}")
+        throw QueryCompilationErrors.cannotConvertDataTypeToParquetTypeError(field)
     }
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetUtils.scala
index 7e7dba92f37b5..b91d75c55c513 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetUtils.scala
@@ -104,7 +104,7 @@ object ParquetUtils {
           .orElse(filesByType.data.headOption)
           .toSeq
       }
-    ParquetFileFormat.mergeSchemasInParallel(filesToTouch, sparkSession)
+    ParquetFileFormat.mergeSchemasInParallel(parameters, filesToTouch, sparkSession)
   }
 
   case class FileTypes(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
index 6c333671d59cb..cd0e63938638f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
@@ -31,7 +31,7 @@ import org.apache.parquet.io.api.{Binary, RecordConsumer}
 
 import org.apache.spark.SPARK_VERSION_SHORT
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.{SPARK_LEGACY_DATETIME, SPARK_VERSION_METADATA_KEY}
+import org.apache.spark.sql.{SPARK_LEGACY_DATETIME, SPARK_LEGACY_INT96, SPARK_VERSION_METADATA_KEY}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.SpecializedGetters
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
@@ -80,7 +80,7 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging {
     new Array[Byte](Decimal.minBytesForPrecision(DecimalType.MAX_PRECISION))
 
   private val datetimeRebaseMode = LegacyBehaviorPolicy.withName(
-    SQLConf.get.getConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE))
+    SQLConf.get.getConf(SQLConf.PARQUET_REBASE_MODE_IN_WRITE))
 
   private val dateRebaseFunc = DataSourceUtils.creteDateRebaseFuncInWrite(
     datetimeRebaseMode, "Parquet")
@@ -88,6 +88,12 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging {
   private val timestampRebaseFunc = DataSourceUtils.creteTimestampRebaseFuncInWrite(
     datetimeRebaseMode, "Parquet")
 
+  private val int96RebaseMode = LegacyBehaviorPolicy.withName(
+    SQLConf.get.getConf(SQLConf.PARQUET_INT96_REBASE_MODE_IN_WRITE))
+
+  private val int96RebaseFunc = DataSourceUtils.creteTimestampRebaseFuncInWrite(
+    int96RebaseMode, "Parquet INT96")
+
   override def init(configuration: Configuration): WriteContext = {
     val schemaString = configuration.get(ParquetWriteSupport.SPARK_ROW_SCHEMA)
     this.schema = StructType.fromString(schemaString)
@@ -115,6 +121,12 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging {
       } else {
         None
       }
+    } ++ {
+      if (int96RebaseMode == LegacyBehaviorPolicy.LEGACY) {
+        Some(SPARK_LEGACY_INT96 -> "")
+      } else {
+        None
+      }
     }
 
     logInfo(
@@ -193,7 +205,8 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging {
         outputTimestampType match {
           case SQLConf.ParquetOutputTimestampType.INT96 =>
             (row: SpecializedGetters, ordinal: Int) =>
-              val (julianDay, timeOfDayNanos) = DateTimeUtils.toJulianDay(row.getLong(ordinal))
+              val micros = int96RebaseFunc(row.getLong(ordinal))
+              val (julianDay, timeOfDayNanos) = DateTimeUtils.toJulianDay(micros)
               val buf = ByteBuffer.wrap(timestampBuffer)
               buf.order(ByteOrder.LITTLE_ENDIAN).putLong(timeOfDayNanos).putInt(julianDay)
               recordConsumer.addBinary(Binary.fromReusedByteArray(timestampBuffer))
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/pathFilters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/pathFilters.scala
new file mode 100644
index 0000000000000..c8f23988f93c6
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/pathFilters.scala
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources
+
+import java.util.{Locale, TimeZone}
+
+import org.apache.hadoop.fs.{FileStatus, GlobFilter}
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateTimeUtils}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.unsafe.types.UTF8String
+
+trait PathFilterStrategy extends Serializable {
+  def accept(fileStatus: FileStatus): Boolean
+}
+
+trait StrategyBuilder {
+  def create(parameters: CaseInsensitiveMap[String]): Option[PathFilterStrategy]
+}
+
+class PathGlobFilter(filePatten: String) extends PathFilterStrategy {
+
+  private val globFilter = new GlobFilter(filePatten)
+
+  override def accept(fileStatus: FileStatus): Boolean =
+    globFilter.accept(fileStatus.getPath)
+}
+
+object PathGlobFilter extends StrategyBuilder {
+  val PARAM_NAME = "pathglobfilter"
+
+  override def create(parameters: CaseInsensitiveMap[String]): Option[PathFilterStrategy] = {
+    parameters.get(PARAM_NAME).map(new PathGlobFilter(_))
+  }
+}
+
+/**
+ * Provide modifiedAfter and modifiedBefore options when
+ * filtering from a batch-based file data source.
+ *
+ * Example Usages
+ * Load all CSV files modified after date:
+ * {{{
+ *   spark.read.format("csv").option("modifiedAfter","2020-06-15T05:00:00").load()
+ * }}}
+ *
+ * Load all CSV files modified before date:
+ * {{{
+ *   spark.read.format("csv").option("modifiedBefore","2020-06-15T05:00:00").load()
+ * }}}
+ *
+ * Load all CSV files modified between two dates:
+ * {{{
+ *   spark.read.format("csv").option("modifiedAfter","2019-01-15T05:00:00")
+ *     .option("modifiedBefore","2020-06-15T05:00:00").load()
+ * }}}
+ */
+abstract class ModifiedDateFilter extends PathFilterStrategy {
+
+  def timeZoneId: String
+
+  protected def localTime(micros: Long): Long =
+    DateTimeUtils.fromUTCTime(micros, timeZoneId)
+}
+
+object ModifiedDateFilter {
+
+  def getTimeZoneId(options: CaseInsensitiveMap[String]): String = {
+    options.getOrElse(
+      DateTimeUtils.TIMEZONE_OPTION.toLowerCase(Locale.ROOT),
+      SQLConf.get.sessionLocalTimeZone)
+  }
+
+  def toThreshold(timeString: String, timeZoneId: String, strategy: String): Long = {
+    val timeZone: TimeZone = DateTimeUtils.getTimeZone(timeZoneId)
+    val ts = UTF8String.fromString(timeString)
+    DateTimeUtils.stringToTimestamp(ts, timeZone.toZoneId).getOrElse {
+      throw new AnalysisException(
+        s"The timestamp provided for the '$strategy' option is invalid. The expected format " +
+          s"is 'YYYY-MM-DDTHH:mm:ss', but the provided timestamp: $timeString")
+    }
+  }
+}
+
+/**
+ * Filter used to determine whether file was modified before the provided timestamp.
+ */
+class ModifiedBeforeFilter(thresholdTime: Long, val timeZoneId: String)
+    extends ModifiedDateFilter {
+
+  override def accept(fileStatus: FileStatus): Boolean =
+    // We standardize on microseconds wherever possible
+    // getModificationTime returns in milliseconds
+    thresholdTime - localTime(DateTimeUtils.millisToMicros(fileStatus.getModificationTime)) > 0
+}
+
+object ModifiedBeforeFilter extends StrategyBuilder {
+  import ModifiedDateFilter._
+
+  val PARAM_NAME = "modifiedbefore"
+
+  override def create(parameters: CaseInsensitiveMap[String]): Option[PathFilterStrategy] = {
+    parameters.get(PARAM_NAME).map { value =>
+      val timeZoneId = getTimeZoneId(parameters)
+      val thresholdTime = toThreshold(value, timeZoneId, PARAM_NAME)
+      new ModifiedBeforeFilter(thresholdTime, timeZoneId)
+    }
+  }
+}
+
+/**
+ * Filter used to determine whether file was modified after the provided timestamp.
+ */
+class ModifiedAfterFilter(thresholdTime: Long, val timeZoneId: String)
+    extends ModifiedDateFilter {
+
+  override def accept(fileStatus: FileStatus): Boolean =
+    // getModificationTime returns in milliseconds
+    // We standardize on microseconds wherever possible
+    localTime(DateTimeUtils.millisToMicros(fileStatus.getModificationTime)) - thresholdTime > 0
+}
+
+object ModifiedAfterFilter extends StrategyBuilder {
+  import ModifiedDateFilter._
+
+  val PARAM_NAME = "modifiedafter"
+
+  override def create(parameters: CaseInsensitiveMap[String]): Option[PathFilterStrategy] = {
+    parameters.get(PARAM_NAME).map { value =>
+      val timeZoneId = getTimeZoneId(parameters)
+      val thresholdTime = toThreshold(value, timeZoneId, PARAM_NAME)
+      new ModifiedAfterFilter(thresholdTime, timeZoneId)
+    }
+  }
+}
+
+object PathFilterFactory {
+
+  private val strategies =
+    Seq(PathGlobFilter, ModifiedBeforeFilter, ModifiedAfterFilter)
+
+  def create(parameters: CaseInsensitiveMap[String]): Seq[PathFilterStrategy] = {
+    strategies.flatMap { _.create(parameters) }
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
index 95343e2872def..c22bb3ff6f60f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
@@ -25,13 +25,14 @@ import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.expressions.{Expression, InputFileBlockLength, InputFileBlockStart, InputFileName, RowOrdering}
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.connector.catalog.CatalogV2Util.assertNoNullTypeInSchema
 import org.apache.spark.sql.connector.expressions.{FieldReference, RewritableTransform}
+import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.execution.command.DDLUtils
 import org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy
 import org.apache.spark.sql.sources.InsertableRelation
 import org.apache.spark.sql.types.{AtomicType, StructType}
+import org.apache.spark.sql.util.PartitioningUtils.normalizePartitionSpec
 import org.apache.spark.sql.util.SchemaUtils
 
 /**
@@ -39,7 +40,7 @@ import org.apache.spark.sql.util.SchemaUtils
  */
 class ResolveSQLOnFile(sparkSession: SparkSession) extends Rule[LogicalPlan] {
   private def maybeSQLFile(u: UnresolvedRelation): Boolean = {
-    sparkSession.sessionState.conf.runSQLonFile && u.multipartIdentifier.size == 2
+    conf.runSQLonFile && u.multipartIdentifier.size == 2
   }
 
   def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
@@ -56,8 +57,8 @@ class ResolveSQLOnFile(sparkSession: SparkSession) extends Rule[LogicalPlan] {
         val isFileFormat = classOf[FileFormat].isAssignableFrom(dataSource.providingClass)
         if (!isFileFormat ||
             dataSource.className.toLowerCase(Locale.ROOT) == DDLUtils.HIVE_PROVIDER) {
-          throw new AnalysisException("Unsupported data source type for direct query on files: " +
-            s"${dataSource.className}")
+          throw QueryCompilationErrors.unsupportedDataSourceTypeForDirectQueryOnFilesError(
+            dataSource.className)
         }
         LogicalRelation(dataSource.resolveRelation())
       } catch {
@@ -107,11 +108,10 @@ case class PreprocessTableCreation(sparkSession: SparkSession) extends Rule[Logi
       val existingTable = catalog.getTableMetadata(tableIdentWithDB)
 
       if (existingTable.tableType == CatalogTableType.VIEW) {
-        throw new AnalysisException("Saving data into a view is not allowed.")
+        throw QueryCompilationErrors.saveDataIntoViewNotAllowedError()
       }
 
       // Check if the specified data source match the data source of the existing table.
-      val conf = sparkSession.sessionState.conf
       val existingProvider = DataSource.lookupDataSource(existingTable.provider.get, conf)
       val specifiedProvider = DataSource.lookupDataSource(tableDesc.provider.get, conf)
       // TODO: Check that options from the resolved relation match the relation that we are
@@ -119,27 +119,22 @@ case class PreprocessTableCreation(sparkSession: SparkSession) extends Rule[Logi
       // If the one of the provider is [[FileDataSourceV2]] and the other one is its corresponding
       // [[FileFormat]], the two providers are considered compatible.
       if (fallBackV2ToV1(existingProvider) != fallBackV2ToV1(specifiedProvider)) {
-        throw new AnalysisException(s"The format of the existing table $tableName is " +
-          s"`${existingProvider.getSimpleName}`. It doesn't match the specified format " +
-          s"`${specifiedProvider.getSimpleName}`.")
+        throw QueryCompilationErrors.mismatchedTableFormatError(
+          tableName, existingProvider, specifiedProvider)
       }
       tableDesc.storage.locationUri match {
         case Some(location) if location.getPath != existingTable.location.getPath =>
-          throw new AnalysisException(
-            s"The location of the existing table ${tableIdentWithDB.quotedString} is " +
-              s"`${existingTable.location}`. It doesn't match the specified location " +
-              s"`${tableDesc.location}`.")
+          throw QueryCompilationErrors.mismatchedTableLocationError(
+            tableIdentWithDB, existingTable, tableDesc)
         case _ =>
       }
 
       if (query.schema.length != existingTable.schema.length) {
-        throw new AnalysisException(
-          s"The column number of the existing table $tableName" +
-            s"(${existingTable.schema.catalogString}) doesn't match the data schema" +
-            s"(${query.schema.catalogString})")
+        throw QueryCompilationErrors.mismatchedTableColumnNumberError(
+          tableName, existingTable, query)
       }
 
-      val resolver = sparkSession.sessionState.conf.resolver
+      val resolver = conf.resolver
       val tableCols = existingTable.schema.map(_.name)
 
       // As we are inserting into an existing table, we should respect the existing schema and
@@ -148,8 +143,7 @@ case class PreprocessTableCreation(sparkSession: SparkSession) extends Rule[Logi
       val adjustedColumns = tableCols.map { col =>
         query.resolve(Seq(col), resolver).getOrElse {
           val inputColumns = query.schema.map(_.name).mkString(", ")
-          throw new AnalysisException(
-            s"cannot resolve '$col' given input columns: [$inputColumns]")
+          throw QueryCompilationErrors.cannotResolveColumnGivenInputColumnsError(col, inputColumns)
         }
       }
 
@@ -158,12 +152,8 @@ case class PreprocessTableCreation(sparkSession: SparkSession) extends Rule[Logi
         tableName, tableCols, tableDesc.partitionColumnNames, resolver)
       if (specifiedPartCols != existingTable.partitionColumnNames) {
         val existingPartCols = existingTable.partitionColumnNames.mkString(", ")
-        throw new AnalysisException(
-          s"""
-             |Specified partitioning does not match that of the existing table $tableName.
-             |Specified partition columns: [${specifiedPartCols.mkString(", ")}]
-             |Existing partition columns: [$existingPartCols]
-          """.stripMargin)
+        throw QueryCompilationErrors.mismatchedTablePartitionColumnError(
+          tableName, specifiedPartCols, existingPartCols)
       }
 
       // Check if the specified bucketing match the existing table.
@@ -175,12 +165,8 @@ case class PreprocessTableCreation(sparkSession: SparkSession) extends Rule[Logi
           specifiedBucketSpec.map(_.toString).getOrElse("not bucketed")
         val existingBucketString =
           existingTable.bucketSpec.map(_.toString).getOrElse("not bucketed")
-        throw new AnalysisException(
-          s"""
-             |Specified bucketing does not match that of the existing table $tableName.
-             |Specified bucketing: $specifiedBucketString
-             |Existing bucketing: $existingBucketString
-          """.stripMargin)
+        throw QueryCompilationErrors.mismatchedTableBucketingError(
+          tableName, specifiedBucketString, existingBucketString)
       }
 
       val newQuery = if (adjustedColumns != query.output) {
@@ -240,11 +226,11 @@ case class PreprocessTableCreation(sparkSession: SparkSession) extends Rule[Logi
         c.copy(tableDesc = normalizedTable.copy(schema = reorderedSchema))
       }
 
-    case create: V2CreateTablePlan =>
+    case create: V2CreateTablePlan if create.childrenResolved =>
       val schema = create.tableSchema
       val partitioning = create.partitioning
       val identifier = create.tableName
-      val isCaseSensitive = sparkSession.sessionState.conf.caseSensitiveAnalysis
+      val isCaseSensitive = conf.caseSensitiveAnalysis
       // Check that columns are not duplicated in the schema
       val flattenedSchema = SchemaUtils.explodeNestedFieldNames(schema)
       SchemaUtils.checkColumnNameDuplication(
@@ -258,14 +244,13 @@ case class PreprocessTableCreation(sparkSession: SparkSession) extends Rule[Logi
 
       if (schema.isEmpty) {
         if (partitioning.nonEmpty) {
-          throw new AnalysisException("It is not allowed to specify partitioning when the " +
-            "table schema is not defined.")
+          throw QueryCompilationErrors.specifyPartitionNotAllowedWhenTableSchemaNotDefinedError()
         }
 
         create
       } else {
         // Resolve and normalize partition columns as necessary
-        val resolver = sparkSession.sessionState.conf.resolver
+        val resolver = conf.resolver
         val normalizedPartitions = partitioning.map {
           case transform: RewritableTransform =>
             val rewritten = transform.references().map { ref =>
@@ -290,19 +275,21 @@ case class PreprocessTableCreation(sparkSession: SparkSession) extends Rule[Logi
     SchemaUtils.checkSchemaColumnNameDuplication(
       schema,
       "in the table definition of " + table.identifier,
-      sparkSession.sessionState.conf.caseSensitiveAnalysis)
+      conf.caseSensitiveAnalysis)
+
+    assertNoNullTypeInSchema(schema)
 
     val normalizedPartCols = normalizePartitionColumns(schema, table)
     val normalizedBucketSpec = normalizeBucketSpec(schema, table)
 
     normalizedBucketSpec.foreach { spec =>
       for (bucketCol <- spec.bucketColumnNames if normalizedPartCols.contains(bucketCol)) {
-        throw new AnalysisException(s"bucketing column '$bucketCol' should not be part of " +
-          s"partition columns '${normalizedPartCols.mkString(", ")}'")
+        throw QueryCompilationErrors.bucketingColumnCannotBePartOfPartitionColumnsError(
+          bucketCol, normalizedPartCols)
       }
       for (sortCol <- spec.sortColumnNames if normalizedPartCols.contains(sortCol)) {
-        throw new AnalysisException(s"bucket sorting column '$sortCol' should not be part of " +
-          s"partition columns '${normalizedPartCols.mkString(", ")}'")
+        throw QueryCompilationErrors.bucketSortingColumnCannotBePartOfPartitionColumnsError(
+          sortCol, normalizedPartCols)
       }
     }
 
@@ -314,12 +301,12 @@ case class PreprocessTableCreation(sparkSession: SparkSession) extends Rule[Logi
       tableName = table.identifier.unquotedString,
       tableCols = schema.map(_.name),
       partCols = table.partitionColumnNames,
-      resolver = sparkSession.sessionState.conf.resolver)
+      resolver = conf.resolver)
 
     SchemaUtils.checkColumnNameDuplication(
       normalizedPartitionCols,
       "in the partition schema",
-      sparkSession.sessionState.conf.resolver)
+      conf.resolver)
 
     if (schema.nonEmpty && normalizedPartitionCols.length == schema.length) {
       if (DDLUtils.isHiveTable(table)) {
@@ -348,16 +335,16 @@ case class PreprocessTableCreation(sparkSession: SparkSession) extends Rule[Logi
           tableName = table.identifier.unquotedString,
           tableCols = schema.map(_.name),
           bucketSpec = bucketSpec,
-          resolver = sparkSession.sessionState.conf.resolver)
+          resolver = conf.resolver)
 
         SchemaUtils.checkColumnNameDuplication(
           normalizedBucketSpec.bucketColumnNames,
           "in the bucket definition",
-          sparkSession.sessionState.conf.resolver)
+          conf.resolver)
         SchemaUtils.checkColumnNameDuplication(
           normalizedBucketSpec.sortColumnNames,
           "in the sort definition",
-          sparkSession.sessionState.conf.resolver)
+          conf.resolver)
 
         normalizedBucketSpec.sortColumnNames.map(schema(_)).map(_.dataType).foreach {
           case dt if RowOrdering.isOrderable(dt) => // OK
@@ -379,58 +366,65 @@ case class PreprocessTableCreation(sparkSession: SparkSession) extends Rule[Logi
  * table. It also does data type casting and field renaming, to make sure that the columns to be
  * inserted have the correct data type and fields have the correct names.
  */
-case class PreprocessTableInsertion(conf: SQLConf) extends Rule[LogicalPlan] {
+object PreprocessTableInsertion extends Rule[LogicalPlan] {
   private def preprocess(
       insert: InsertIntoStatement,
       tblName: String,
-      partColNames: Seq[String]): InsertIntoStatement = {
+      partColNames: StructType,
+      catalogTable: Option[CatalogTable]): InsertIntoStatement = {
 
-    val normalizedPartSpec = PartitioningUtils.normalizePartitionSpec(
+    val normalizedPartSpec = normalizePartitionSpec(
       insert.partitionSpec, partColNames, tblName, conf.resolver)
 
     val staticPartCols = normalizedPartSpec.filter(_._2.isDefined).keySet
     val expectedColumns = insert.table.output.filterNot(a => staticPartCols.contains(a.name))
 
     if (expectedColumns.length != insert.query.schema.length) {
-      throw new AnalysisException(
-        s"$tblName requires that the data to be inserted have the same number of columns as the " +
-          s"target table: target table has ${insert.table.output.size} column(s) but the " +
-          s"inserted data has ${insert.query.output.length + staticPartCols.size} column(s), " +
-          s"including ${staticPartCols.size} partition column(s) having constant value(s).")
+      throw QueryCompilationErrors.mismatchedInsertedDataColumnNumberError(
+        tblName, insert, staticPartCols)
+    }
+
+    val partitionsTrackedByCatalog = catalogTable.isDefined &&
+      catalogTable.get.partitionColumnNames.nonEmpty &&
+      catalogTable.get.tracksPartitionsInCatalog
+    if (partitionsTrackedByCatalog && normalizedPartSpec.nonEmpty) {
+      // empty partition column value
+      if (normalizedPartSpec.values.flatten.exists(v => v != null && v.isEmpty)) {
+        val spec = normalizedPartSpec.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]")
+        throw QueryCompilationErrors.invalidPartitionSpecError(
+          s"The spec ($spec) contains an empty partition column value")
+      }
     }
 
     val newQuery = TableOutputResolver.resolveOutputColumns(
       tblName, expectedColumns, insert.query, byName = false, conf)
     if (normalizedPartSpec.nonEmpty) {
       if (normalizedPartSpec.size != partColNames.length) {
-        throw new AnalysisException(
-          s"""
-             |Requested partitioning does not match the table $tblName:
-             |Requested partitions: ${normalizedPartSpec.keys.mkString(",")}
-             |Table partitions: ${partColNames.mkString(",")}
-           """.stripMargin)
+        throw QueryCompilationErrors.requestedPartitionsMismatchTablePartitionsError(
+          tblName, normalizedPartSpec, partColNames)
       }
 
       insert.copy(query = newQuery, partitionSpec = normalizedPartSpec)
     } else {
       // All partition columns are dynamic because the InsertIntoTable command does
       // not explicitly specify partitioning columns.
-      insert.copy(query = newQuery, partitionSpec = partColNames.map(_ -> None).toMap)
+      insert.copy(query = newQuery, partitionSpec = partColNames.map(_.name).map(_ -> None).toMap)
     }
   }
 
   def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
-    case i @ InsertIntoStatement(table, _, query, _, _) if table.resolved && query.resolved =>
+    case i @ InsertIntoStatement(table, _, _, query, _, _) if table.resolved && query.resolved =>
       table match {
         case relation: HiveTableRelation =>
           val metadata = relation.tableMeta
-          preprocess(i, metadata.identifier.quotedString, metadata.partitionColumnNames)
+          preprocess(i, metadata.identifier.quotedString, metadata.partitionSchema,
+            Some(metadata))
         case LogicalRelation(h: HadoopFsRelation, _, catalogTable, _) =>
           val tblName = catalogTable.map(_.identifier.quotedString).getOrElse("unknown")
-          preprocess(i, tblName, h.partitionSchema.map(_.name))
+          preprocess(i, tblName, h.partitionSchema, catalogTable)
         case LogicalRelation(_: InsertableRelation, _, catalogTable, _) =>
           val tblName = catalogTable.map(_.identifier.quotedString).getOrElse("unknown")
-          preprocess(i, tblName, Nil)
+          preprocess(i, tblName, new StructType(), catalogTable)
         case _ => i
       }
   }
@@ -443,10 +437,11 @@ object HiveOnlyCheck extends (LogicalPlan => Unit) {
   def apply(plan: LogicalPlan): Unit = {
     plan.foreach {
       case CreateTable(tableDesc, _, _) if DDLUtils.isHiveTable(tableDesc) =>
-        throw new AnalysisException("Hive support is required to CREATE Hive TABLE (AS SELECT)")
+        throw QueryCompilationErrors.ddlWithoutHiveSupportEnabledError(
+          "CREATE Hive TABLE (AS SELECT)")
       case i: InsertIntoDir if DDLUtils.isHiveTable(i.provider) =>
-        throw new AnalysisException(
-          "Hive support is required to INSERT OVERWRITE DIRECTORY with the Hive format")
+        throw QueryCompilationErrors.ddlWithoutHiveSupportEnabledError(
+          "INSERT OVERWRITE DIRECTORY with the Hive format")
       case _ => // OK
     }
   }
@@ -497,7 +492,7 @@ object PreWriteCheck extends (LogicalPlan => Unit) {
 
   def apply(plan: LogicalPlan): Unit = {
     plan.foreach {
-      case InsertIntoStatement(l @ LogicalRelation(relation, _, _, _), partition, query, _, _) =>
+      case InsertIntoStatement(l @ LogicalRelation(relation, _, _, _), partition, _, query, _, _) =>
         // Get all input data source relations of the query.
         val srcRelations = query.collect {
           case LogicalRelation(src, _, _, _) => src
@@ -519,7 +514,7 @@ object PreWriteCheck extends (LogicalPlan => Unit) {
           case _ => failAnalysis(s"$relation does not allow insertion.")
         }
 
-      case InsertIntoStatement(t, _, _, _, _)
+      case InsertIntoStatement(t, _, _, _, _, _)
         if !t.isInstanceOf[LeafNode] ||
           t.isInstanceOf[Range] ||
           t.isInstanceOf[OneRowRelation] ||
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/text/TextOutputWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/text/TextOutputWriter.scala
index 2b1b81f60ceb4..2fb37c0dc0359 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/text/TextOutputWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/text/TextOutputWriter.scala
@@ -24,7 +24,7 @@ import org.apache.spark.sql.execution.datasources.{CodecStreams, OutputWriter}
 import org.apache.spark.sql.types.StructType
 
 class TextOutputWriter(
-    path: String,
+    val path: String,
     dataSchema: StructType,
     lineSeparator: Array[Byte],
     context: TaskAttemptContext)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionExec.scala
new file mode 100644
index 0000000000000..f3fba93b08eb6
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionExec.scala
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.{PartitionsAlreadyExistException, ResolvedPartitionSpec}
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.{SupportsAtomicPartitionManagement, SupportsPartitionManagement}
+
+/**
+ * Physical plan node for adding partitions of table.
+ */
+case class AddPartitionExec(
+    table: SupportsPartitionManagement,
+    partSpecs: Seq[ResolvedPartitionSpec],
+    ignoreIfExists: Boolean,
+    refreshCache: () => Unit) extends LeafV2CommandExec {
+  import DataSourceV2Implicits._
+
+  override def output: Seq[Attribute] = Seq.empty
+
+  override protected def run(): Seq[InternalRow] = {
+    val (existsParts, notExistsParts) =
+      partSpecs.partition(p => table.partitionExists(p.ident))
+
+    if (existsParts.nonEmpty && !ignoreIfExists) {
+      throw new PartitionsAlreadyExistException(
+        table.name(), existsParts.map(_.ident), table.partitionSchema())
+    }
+
+    val isTableAltered = notExistsParts match {
+      case Seq() => false // Nothing will be done
+      case Seq(partitionSpec) =>
+        val partProp = partitionSpec.location.map(loc => "location" -> loc).toMap
+        table.createPartition(partitionSpec.ident, partProp.asJava)
+        true
+      case _ if table.isInstanceOf[SupportsAtomicPartitionManagement] =>
+        val partIdents = notExistsParts.map(_.ident)
+        val partProps = notExistsParts.map(_.location.map(loc => "location" -> loc).toMap)
+        table.asAtomicPartitionable
+          .createPartitions(
+            partIdents.toArray,
+            partProps.map(_.asJava).toArray)
+        true
+      case _ =>
+        throw new UnsupportedOperationException(
+          s"Nonatomic partition table ${table.name()} can not add multiple partitions.")
+    }
+    if (isTableAltered) refreshCache()
+    Seq.empty
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterNamespaceSetPropertiesExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterNamespaceSetPropertiesExec.scala
index 1eebe4cdb6a86..4bde31abfc0d4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterNamespaceSetPropertiesExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterNamespaceSetPropertiesExec.scala
@@ -27,7 +27,7 @@ import org.apache.spark.sql.connector.catalog.{NamespaceChange, SupportsNamespac
 case class AlterNamespaceSetPropertiesExec(
     catalog: SupportsNamespaces,
     namespace: Seq[String],
-    props: Map[String, String]) extends V2CommandExec {
+    props: Map[String, String]) extends LeafV2CommandExec {
   override protected def run(): Seq[InternalRow] = {
     val changes = props.map{ case (k, v) =>
       NamespaceChange.setProperty(k, v)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterTableExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterTableExec.scala
index 8b2930cca841d..cc1c73b020d68 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterTableExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterTableExec.scala
@@ -17,10 +17,10 @@
 
 package org.apache.spark.sql.execution.datasources.v2
 
-import org.apache.spark.SparkException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog, TableChange}
+import org.apache.spark.sql.errors.QueryExecutionErrors
 
 /**
  * Physical plan node for altering a table.
@@ -28,7 +28,7 @@ import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog, TableCh
 case class AlterTableExec(
     catalog: TableCatalog,
     ident: Identifier,
-    changes: Seq[TableChange]) extends V2CommandExec {
+    changes: Seq[TableChange]) extends LeafV2CommandExec {
 
   override def output: Seq[Attribute] = Seq.empty
 
@@ -37,7 +37,7 @@ case class AlterTableExec(
       catalog.alterTable(ident, changes: _*)
     } catch {
       case e: IllegalArgumentException =>
-        throw new SparkException(s"Unsupported table change: ${e.getMessage}", e)
+        throw QueryExecutionErrors.unsupportedTableChangeError(e)
     }
 
     Seq.empty
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExec.scala
index e4e7887017a1d..1987c9e63a649 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExec.scala
@@ -40,12 +40,12 @@ case class BatchScanExec(
 
   override def hashCode(): Int = batch.hashCode()
 
-  override lazy val partitions: Seq[InputPartition] = batch.planInputPartitions()
+  @transient override lazy val partitions: Seq[InputPartition] = batch.planInputPartitions()
 
   override lazy val readerFactory: PartitionReaderFactory = batch.createReaderFactory()
 
   override lazy val inputRDD: RDD[InternalRow] = {
-    new DataSourceRDD(sparkContext, partitions, readerFactory, supportsColumnar)
+    new DataSourceRDD(sparkContext, partitions, readerFactory, supportsColumnar, customMetrics)
   }
 
   override def doCanonicalize(): BatchScanExec = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CacheTableExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CacheTableExec.scala
new file mode 100644
index 0000000000000..ac97e573f76b9
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CacheTableExec.scala
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import java.util.Locale
+
+import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}
+import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier}
+import org.apache.spark.sql.catalyst.analysis.LocalTempView
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
+import org.apache.spark.sql.execution.command.CreateViewCommand
+import org.apache.spark.storage.StorageLevel
+
+trait BaseCacheTableExec extends LeafV2CommandExec {
+  def relationName: String
+  def planToCache: LogicalPlan
+  def dataFrameForCachedPlan: DataFrame
+  def isLazy: Boolean
+  def options: Map[String, String]
+
+  protected val sparkSession: SparkSession = sqlContext.sparkSession
+
+  override def run(): Seq[InternalRow] = {
+    val storageLevelKey = "storagelevel"
+    val storageLevelValue =
+      CaseInsensitiveMap(options).get(storageLevelKey).map(_.toUpperCase(Locale.ROOT))
+    val withoutStorageLevel = options.filterKeys(_.toLowerCase(Locale.ROOT) != storageLevelKey)
+    if (withoutStorageLevel.nonEmpty) {
+      logWarning(s"Invalid options: ${withoutStorageLevel.mkString(", ")}")
+    }
+
+    if (storageLevelValue.nonEmpty) {
+      sparkSession.sharedState.cacheManager.cacheQuery(
+        sparkSession,
+        planToCache,
+        Some(relationName),
+        StorageLevel.fromString(storageLevelValue.get))
+    } else {
+      sparkSession.sharedState.cacheManager.cacheQuery(
+        sparkSession,
+        planToCache,
+        Some(relationName))
+    }
+
+    if (!isLazy) {
+      // Performs eager caching.
+      dataFrameForCachedPlan.count()
+    }
+
+    Seq.empty
+  }
+
+  override def output: Seq[Attribute] = Seq.empty
+}
+
+case class CacheTableExec(
+    relation: LogicalPlan,
+    multipartIdentifier: Seq[String],
+    override val isLazy: Boolean,
+    override val options: Map[String, String]) extends BaseCacheTableExec {
+  override lazy val relationName: String = multipartIdentifier.quoted
+
+  override lazy val planToCache: LogicalPlan = relation
+
+  override lazy val dataFrameForCachedPlan: DataFrame = {
+    Dataset.ofRows(sparkSession, planToCache)
+  }
+}
+
+case class CacheTableAsSelectExec(
+    tempViewName: String,
+    query: LogicalPlan,
+    originalText: String,
+    override val isLazy: Boolean,
+    override val options: Map[String, String]) extends BaseCacheTableExec {
+  override lazy val relationName: String = tempViewName
+
+  override lazy val planToCache: LogicalPlan = {
+    CreateViewCommand(
+      name = TableIdentifier(tempViewName),
+      userSpecifiedColumns = Nil,
+      comment = None,
+      properties = Map.empty,
+      originalText = Some(originalText),
+      plan = query,
+      allowExisting = false,
+      replace = false,
+      viewType = LocalTempView,
+      isAnalyzed = true
+    ).run(sparkSession)
+
+    dataFrameForCachedPlan.logicalPlan
+  }
+
+  override lazy val dataFrameForCachedPlan: DataFrame = {
+    sparkSession.table(tempViewName)
+  }
+}
+
+case class UncacheTableExec(
+    relation: LogicalPlan,
+    cascade: Boolean) extends LeafV2CommandExec {
+  override def run(): Seq[InternalRow] = {
+    val sparkSession = sqlContext.sparkSession
+    sparkSession.sharedState.cacheManager.uncacheQuery(sparkSession, relation, cascade)
+    Seq.empty
+  }
+
+  override def output: Seq[Attribute] = Seq.empty
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ContinuousScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ContinuousScanExec.scala
index dc95d157e40fa..fea89c581e183 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ContinuousScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ContinuousScanExec.scala
@@ -58,6 +58,7 @@ case class ContinuousScanExec(
       sqlContext.conf.continuousStreamingExecutorPollIntervalMs,
       partitions,
       schema,
-      readerFactory.asInstanceOf[ContinuousPartitionReaderFactory])
+      readerFactory.asInstanceOf[ContinuousPartitionReaderFactory],
+      customMetrics)
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateNamespaceExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateNamespaceExec.scala
index d5b81d13a7cc4..dba84d2385aec 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateNamespaceExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateNamespaceExec.scala
@@ -23,6 +23,7 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.NamespaceAlreadyExistsException
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.connector.catalog.SupportsNamespaces
+import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.util.Utils
 
 /**
@@ -33,7 +34,7 @@ case class CreateNamespaceExec(
     namespace: Seq[String],
     ifNotExists: Boolean,
     private var properties: Map[String, String])
-    extends V2CommandExec {
+    extends LeafV2CommandExec {
   override protected def run(): Seq[InternalRow] = {
     import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
     import org.apache.spark.sql.connector.catalog.SupportsNamespaces._
@@ -49,7 +50,7 @@ case class CreateNamespaceExec(
           logWarning(s"Namespace ${namespace.quoted} was created concurrently. Ignoring.")
       }
     } else if (!ifNotExists) {
-      throw new NamespaceAlreadyExistsException(ns)
+      throw QueryCompilationErrors.namespaceAlreadyExistsError(ns)
     }
 
     Seq.empty
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateTableExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateTableExec.scala
index 511cd8a9a438f..be7331b0d7dc4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateTableExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateTableExec.scala
@@ -24,6 +24,7 @@ import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog}
 import org.apache.spark.sql.connector.expressions.Transform
+import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.types.StructType
 
 case class CreateTableExec(
@@ -32,7 +33,7 @@ case class CreateTableExec(
     tableSchema: StructType,
     partitioning: Seq[Transform],
     tableProperties: Map[String, String],
-    ignoreIfExists: Boolean) extends V2CommandExec {
+    ignoreIfExists: Boolean) extends LeafV2CommandExec {
   import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
 
   override protected def run(): Seq[InternalRow] = {
@@ -44,7 +45,7 @@ case class CreateTableExec(
           logWarning(s"Table ${identifier.quoted} was created concurrently. Ignoring.")
       }
     } else if (!ignoreIfExists) {
-      throw new TableAlreadyExistsException(identifier)
+      throw QueryCompilationErrors.tableAlreadyExistsError(identifier)
     }
 
     Seq.empty
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceRDD.scala
index 63403b9577237..217a1d5750d42 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceRDD.scala
@@ -25,6 +25,8 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.connector.read.{InputPartition, PartitionReader, PartitionReaderFactory}
+import org.apache.spark.sql.errors.QueryExecutionErrors
+import org.apache.spark.sql.execution.metric.{CustomMetrics, SQLMetric}
 import org.apache.spark.sql.vectorized.ColumnarBatch
 
 class DataSourceRDDPartition(val index: Int, val inputPartition: InputPartition)
@@ -36,7 +38,8 @@ class DataSourceRDD(
     sc: SparkContext,
     @transient private val inputPartitions: Seq[InputPartition],
     partitionReaderFactory: PartitionReaderFactory,
-    columnarReads: Boolean)
+    columnarReads: Boolean,
+    customMetrics: Map[String, SQLMetric])
   extends RDD[InternalRow](sc, Nil) {
 
   override protected def getPartitions: Array[Partition] = {
@@ -47,21 +50,28 @@ class DataSourceRDD(
 
   private def castPartition(split: Partition): DataSourceRDDPartition = split match {
     case p: DataSourceRDDPartition => p
-    case _ => throw new SparkException(s"[BUG] Not a DataSourceRDDPartition: $split")
+    case _ => throw QueryExecutionErrors.notADatasourceRDDPartitionError(split)
   }
 
   override def compute(split: Partition, context: TaskContext): Iterator[InternalRow] = {
     val inputPartition = castPartition(split).inputPartition
     val (iter, reader) = if (columnarReads) {
       val batchReader = partitionReaderFactory.createColumnarReader(inputPartition)
-      val iter = new MetricsBatchIterator(new PartitionIterator[ColumnarBatch](batchReader))
+      val iter = new MetricsBatchIterator(
+        new PartitionIterator[ColumnarBatch](batchReader, customMetrics))
       (iter, batchReader)
     } else {
       val rowReader = partitionReaderFactory.createReader(inputPartition)
-      val iter = new MetricsRowIterator(new PartitionIterator[InternalRow](rowReader))
+      val iter = new MetricsRowIterator(
+        new PartitionIterator[InternalRow](rowReader, customMetrics))
       (iter, rowReader)
     }
-    context.addTaskCompletionListener[Unit](_ => reader.close())
+    context.addTaskCompletionListener[Unit] { _ =>
+      // In case of early stopping before consuming the entire iterator,
+      // we need to do one more metric update at the end of the task.
+      CustomMetrics.updateMetrics(reader.currentMetricsValues, customMetrics)
+      reader.close()
+    }
     // TODO: SPARK-25083 remove the type erasure hack in data source scan
     new InterruptibleIterator(context, iter.asInstanceOf[Iterator[InternalRow]])
   }
@@ -71,9 +81,13 @@ class DataSourceRDD(
   }
 }
 
-private class PartitionIterator[T](reader: PartitionReader[T]) extends Iterator[T] {
+private class PartitionIterator[T](
+    reader: PartitionReader[T],
+    customMetrics: Map[String, SQLMetric]) extends Iterator[T] {
   private[this] var valuePrepared = false
 
+  private var numRow = 0L
+
   override def hasNext: Boolean = {
     if (!valuePrepared) {
       valuePrepared = reader.next()
@@ -83,8 +97,12 @@ private class PartitionIterator[T](reader: PartitionReader[T]) extends Iterator[
 
   override def next(): T = {
     if (!hasNext) {
-      throw new java.util.NoSuchElementException("End of stream")
+      throw QueryExecutionErrors.endOfStreamError()
+    }
+    if (numRow % CustomMetrics.NUM_ROWS_PER_UPDATE == 0) {
+      CustomMetrics.updateMetrics(reader.currentMetricsValues, customMetrics)
     }
+    numRow += 1
     valuePrepared = false
     reader.get()
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2ScanExecBase.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2ScanExecBase.scala
index 211f61279ddd5..1248f89b2bdd7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2ScanExecBase.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2ScanExecBase.scala
@@ -24,15 +24,22 @@ import org.apache.spark.sql.catalyst.plans.physical
 import org.apache.spark.sql.catalyst.plans.physical.SinglePartition
 import org.apache.spark.sql.catalyst.util.truncatedString
 import org.apache.spark.sql.connector.read.{InputPartition, PartitionReaderFactory, Scan, SupportsReportPartitioning}
-import org.apache.spark.sql.execution.LeafExecNode
+import org.apache.spark.sql.execution.{ExplainUtils, LeafExecNode}
 import org.apache.spark.sql.execution.metric.SQLMetrics
+import org.apache.spark.sql.internal.connector.SupportsMetadata
 import org.apache.spark.sql.vectorized.ColumnarBatch
 import org.apache.spark.util.Utils
 
 trait DataSourceV2ScanExecBase extends LeafExecNode {
 
-  override lazy val metrics = Map(
-    "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"))
+  lazy val customMetrics = scan.supportedCustomMetrics().map { customMetric =>
+    customMetric.name() -> SQLMetrics.createV2CustomMetric(sparkContext, customMetric)
+  }.toMap
+
+  override lazy val metrics = {
+    Map("numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows")) ++
+      customMetrics
+  }
 
   def scan: Scan
 
@@ -43,7 +50,32 @@ trait DataSourceV2ScanExecBase extends LeafExecNode {
   override def simpleString(maxFields: Int): String = {
     val result =
       s"$nodeName${truncatedString(output, "[", ", ", "]", maxFields)} ${scan.description()}"
-    Utils.redact(sqlContext.sessionState.conf.stringRedactionPattern, result)
+    redact(result)
+  }
+
+  /**
+   * Shorthand for calling redact() without specifying redacting rules
+   */
+  protected def redact(text: String): String = {
+    Utils.redact(sqlContext.sessionState.conf.stringRedactionPattern, text)
+  }
+
+  override def verboseStringWithOperatorId(): String = {
+    val metaDataStr = scan match {
+      case s: SupportsMetadata =>
+        s.getMetaData().toSeq.sorted.flatMap {
+          case (_, value) if value.isEmpty || value.equals("[]") => None
+          case (key, value) => Some(s"$key: ${redact(value)}")
+          case _ => None
+        }
+      case _ =>
+        Seq(scan.description())
+    }
+    s"""
+       |$formattedNodeName
+       |${ExplainUtils.generateFieldString("Output", output)}
+       |${metaDataStr.mkString("\n")}
+       |""".stripMargin
   }
 
   override def outputPartitioning: physical.Partitioning = scan match {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
index cca80c0cb6d57..811f41832d159 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
@@ -19,18 +19,22 @@ package org.apache.spark.sql.execution.datasources.v2
 
 import scala.collection.JavaConverters._
 
-import org.apache.spark.sql.{AnalysisException, SparkSession, Strategy}
-import org.apache.spark.sql.catalyst.analysis.{ResolvedNamespace, ResolvedTable}
-import org.apache.spark.sql.catalyst.expressions.{And, Expression, NamedExpression, PredicateHelper, SubqueryExpression}
+import org.apache.spark.sql.{SparkSession, Strategy}
+import org.apache.spark.sql.catalyst.analysis.{ResolvedNamespace, ResolvedPartitionSpec, ResolvedTable}
+import org.apache.spark.sql.catalyst.expressions.{And, Attribute, Expression, NamedExpression, PredicateHelper, SubqueryExpression}
 import org.apache.spark.sql.catalyst.planning.PhysicalOperation
 import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.connector.catalog.{CatalogV2Util, StagingTableCatalog, SupportsNamespaces, TableCapability, TableCatalog, TableChange}
+import org.apache.spark.sql.catalyst.util.toPrettySQL
+import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Identifier, StagingTableCatalog, SupportsNamespaces, SupportsPartitionManagement, SupportsWrite, Table, TableCapability, TableCatalog, TableChange}
 import org.apache.spark.sql.connector.read.streaming.{ContinuousStream, MicroBatchStream}
-import org.apache.spark.sql.execution.{FilterExec, LeafExecNode, ProjectExec, RowDataSourceScanExec, SparkPlan}
+import org.apache.spark.sql.connector.write.V1Write
+import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
+import org.apache.spark.sql.execution.{FilterExec, LeafExecNode, LocalTableScanExec, ProjectExec, RowDataSourceScanExec, SparkPlan}
 import org.apache.spark.sql.execution.datasources.DataSourceStrategy
-import org.apache.spark.sql.execution.streaming.continuous.{ContinuousCoalesceExec, WriteToContinuousDataSource, WriteToContinuousDataSourceExec}
+import org.apache.spark.sql.execution.streaming.continuous.{WriteToContinuousDataSource, WriteToContinuousDataSourceExec}
 import org.apache.spark.sql.sources.{BaseRelation, TableScan}
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
+import org.apache.spark.storage.StorageLevel
 
 class DataSourceV2Strategy(session: SparkSession) extends Strategy with PredicateHelper {
 
@@ -52,23 +56,47 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
     }
   }
 
+  private def refreshCache(r: DataSourceV2Relation)(): Unit = {
+    session.sharedState.cacheManager.recacheByPlan(session, r)
+  }
+
+  private def recacheTable(r: ResolvedTable)(): Unit = {
+    val v2Relation = DataSourceV2Relation.create(r.table, Some(r.catalog), Some(r.identifier))
+    session.sharedState.cacheManager.recacheByPlan(session, v2Relation)
+  }
+
+  // Invalidates the cache associated with the given table. If the invalidated cache matches the
+  // given table, the cache's storage level is returned.
+  private def invalidateTableCache(r: ResolvedTable)(): Option[StorageLevel] = {
+    val v2Relation = DataSourceV2Relation.create(r.table, Some(r.catalog), Some(r.identifier))
+    val cache = session.sharedState.cacheManager.lookupCachedData(v2Relation)
+    session.sharedState.cacheManager.uncacheQuery(session, v2Relation, cascade = true)
+    if (cache.isDefined) {
+      val cacheLevel = cache.get.cachedRepresentation.cacheBuilder.storageLevel
+      Some(cacheLevel)
+    } else {
+      None
+    }
+  }
+
+  private def invalidateCache(catalog: TableCatalog, table: Table, ident: Identifier): Unit = {
+    val v2Relation = DataSourceV2Relation.create(table, Some(catalog), Some(ident))
+    session.sharedState.cacheManager.uncacheQuery(session, v2Relation, cascade = true)
+  }
+
   override def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match {
     case PhysicalOperation(project, filters,
         relation @ DataSourceV2ScanRelation(_, V1ScanWrapper(scan, translated, pushed), output)) =>
       val v1Relation = scan.toV1TableScan[BaseRelation with TableScan](session.sqlContext)
       if (v1Relation.schema != scan.readSchema()) {
-        throw new IllegalArgumentException(
-          "The fallback v1 relation reports inconsistent schema:\n" +
-            "Schema of v2 scan:     " + scan.readSchema() + "\n" +
-            "Schema of v1 relation: " + v1Relation.schema)
+        throw QueryExecutionErrors.fallbackV1RelationReportsInconsistentSchemaError(
+          scan.readSchema(), v1Relation.schema)
       }
       val rdd = v1Relation.buildScan()
       val unsafeRowRDD = DataSourceStrategy.toCatalystRDD(v1Relation, output, rdd)
-      val originalOutputNames = relation.table.schema().map(_.name)
-      val requiredColumnsIndex = output.map(_.name).map(originalOutputNames.indexOf)
       val dsScan = RowDataSourceScanExec(
         output,
-        requiredColumnsIndex,
+        output.toStructType,
         translated.toSet,
         pushed.toSet,
         unsafeRowRDD,
@@ -110,8 +138,12 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
 
       withProjection :: Nil
 
-    case WriteToDataSourceV2(writer, query) =>
-      WriteToDataSourceV2Exec(writer, planLater(query)) :: Nil
+    case WriteToDataSourceV2(relationOpt, writer, query) =>
+      val invalidateCacheFunc: () => Unit = () => relationOpt match {
+        case Some(r) => session.sharedState.cacheManager.uncacheQuery(session, r, cascade = true)
+        case None => ()
+      }
+      WriteToDataSourceV2Exec(writer, invalidateCacheFunc, planLater(query)) :: Nil
 
     case CreateV2Table(catalog, ident, schema, parts, props, ifNotExists) =>
       val propsWithOwner = CatalogV2Util.withDefaultOwnership(props)
@@ -129,18 +161,20 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
             propsWithOwner, writeOptions, ifNotExists) :: Nil
       }
 
-    case RefreshTable(catalog, ident) =>
-      RefreshTableExec(catalog, ident) :: Nil
+    case RefreshTable(r: ResolvedTable) =>
+      RefreshTableExec(r.catalog, r.identifier, recacheTable(r)) :: Nil
 
     case ReplaceTable(catalog, ident, schema, parts, props, orCreate) =>
       val propsWithOwner = CatalogV2Util.withDefaultOwnership(props)
       catalog match {
         case staging: StagingTableCatalog =>
           AtomicReplaceTableExec(
-            staging, ident, schema, parts, propsWithOwner, orCreate = orCreate) :: Nil
+            staging, ident, schema, parts, propsWithOwner, orCreate = orCreate,
+            invalidateCache) :: Nil
         case _ =>
           ReplaceTableExec(
-            catalog, ident, schema, parts, propsWithOwner, orCreate = orCreate) :: Nil
+            catalog, ident, schema, parts, propsWithOwner, orCreate = orCreate,
+            invalidateCache) :: Nil
       }
 
     case ReplaceTableAsSelect(catalog, ident, parts, query, props, options, orCreate) =>
@@ -156,7 +190,8 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
             planLater(query),
             propsWithOwner,
             writeOptions,
-            orCreate = orCreate) :: Nil
+            orCreate = orCreate,
+            invalidateCache) :: Nil
         case _ =>
           ReplaceTableAsSelectExec(
             catalog,
@@ -166,92 +201,108 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
             planLater(query),
             propsWithOwner,
             writeOptions,
-            orCreate = orCreate) :: Nil
+            orCreate = orCreate,
+            invalidateCache) :: Nil
       }
 
-    case AppendData(r: DataSourceV2Relation, query, writeOptions, _) =>
-      r.table.asWritable match {
-        case v1 if v1.supports(TableCapability.V1_BATCH_WRITE) =>
-          AppendDataExecV1(v1, writeOptions.asOptions, query) :: Nil
-        case v2 =>
-          AppendDataExec(v2, writeOptions.asOptions, planLater(query)) :: Nil
+    case AppendData(r @ DataSourceV2Relation(v1: SupportsWrite, _, _, _, _), query, _,
+        _, Some(write)) if v1.supports(TableCapability.V1_BATCH_WRITE) =>
+      write match {
+        case v1Write: V1Write =>
+          AppendDataExecV1(v1, query, refreshCache(r), v1Write) :: Nil
+        case v2Write =>
+          throw QueryCompilationErrors.batchWriteCapabilityError(
+            v1, v2Write.getClass.getName, classOf[V1Write].getName)
       }
 
-    case OverwriteByExpression(r: DataSourceV2Relation, deleteExpr, query, writeOptions, _) =>
-      // fail if any filter cannot be converted. correctness depends on removing all matching data.
-      val filters = splitConjunctivePredicates(deleteExpr).map {
-        filter => DataSourceStrategy.translateFilter(deleteExpr,
-          supportNestedPredicatePushdown = true).getOrElse(
-            throw new AnalysisException(s"Cannot translate expression to source filter: $filter"))
-      }.toArray
-      r.table.asWritable match {
-        case v1 if v1.supports(TableCapability.V1_BATCH_WRITE) =>
-          OverwriteByExpressionExecV1(v1, filters, writeOptions.asOptions, query) :: Nil
-        case v2 =>
-          OverwriteByExpressionExec(v2, filters, writeOptions.asOptions, planLater(query)) :: Nil
+    case AppendData(r: DataSourceV2Relation, query, _, _, Some(write)) =>
+      AppendDataExec(planLater(query), refreshCache(r), write) :: Nil
+
+    case OverwriteByExpression(r @ DataSourceV2Relation(v1: SupportsWrite, _, _, _, _), _, query,
+        _, _, Some(write)) if v1.supports(TableCapability.V1_BATCH_WRITE) =>
+      write match {
+        case v1Write: V1Write =>
+          OverwriteByExpressionExecV1(v1, query, refreshCache(r), v1Write) :: Nil
+        case v2Write =>
+          throw QueryCompilationErrors.batchWriteCapabilityError(
+            v1, v2Write.getClass.getName, classOf[V1Write].getName)
       }
 
-    case OverwritePartitionsDynamic(r: DataSourceV2Relation, query, writeOptions, _) =>
-      OverwritePartitionsDynamicExec(
-        r.table.asWritable, writeOptions.asOptions, planLater(query)) :: Nil
+    case OverwriteByExpression(r: DataSourceV2Relation, _, query, _, _, Some(write)) =>
+      OverwriteByExpressionExec(planLater(query), refreshCache(r), write) :: Nil
+
+    case OverwritePartitionsDynamic(r: DataSourceV2Relation, query, _, _, Some(write)) =>
+      OverwritePartitionsDynamicExec(planLater(query), refreshCache(r), write) :: Nil
 
     case DeleteFromTable(relation, condition) =>
       relation match {
-        case DataSourceV2ScanRelation(table, _, output) =>
+        case DataSourceV2ScanRelation(r, _, output) =>
+          val table = r.table
           if (condition.exists(SubqueryExpression.hasSubquery)) {
-            throw new AnalysisException(
-              s"Delete by condition with subquery is not supported: $condition")
+            throw QueryCompilationErrors.unsupportedDeleteByConditionWithSubqueryError(condition)
           }
           // fail if any filter cannot be converted.
           // correctness depends on removing all matching data.
           val filters = DataSourceStrategy.normalizeExprs(condition.toSeq, output)
               .flatMap(splitConjunctivePredicates(_).map {
                 f => DataSourceStrategy.translateFilter(f, true).getOrElse(
-                  throw new AnalysisException(s"Exec update failed:" +
-                      s" cannot translate expression to source filter: $f"))
+                  throw QueryCompilationErrors.cannotTranslateExpressionToSourceFilterError(f))
               }).toArray
-          DeleteFromTableExec(table.asDeletable, filters) :: Nil
+
+          if (!table.asDeletable.canDeleteWhere(filters)) {
+            throw QueryCompilationErrors.cannotDeleteTableWhereFiltersError(table, filters)
+          }
+
+          DeleteFromTableExec(table.asDeletable, filters, refreshCache(r)) :: Nil
         case _ =>
-          throw new AnalysisException("DELETE is only supported with v2 tables.")
+          throw QueryCompilationErrors.deleteOnlySupportedWithV2TablesError()
       }
 
     case WriteToContinuousDataSource(writer, query) =>
       WriteToContinuousDataSourceExec(writer, planLater(query)) :: Nil
 
-    case Repartition(1, false, child) =>
-      val isContinuous = child.find {
-        case r: StreamingDataSourceV2Relation => r.stream.isInstanceOf[ContinuousStream]
-        case _ => false
-      }.isDefined
-
-      if (isContinuous) {
-        ContinuousCoalesceExec(1, planLater(child)) :: Nil
-      } else {
-        Nil
-      }
-
-    case desc @ DescribeNamespace(ResolvedNamespace(catalog, ns), extended) =>
-      DescribeNamespaceExec(desc.output, catalog.asNamespaceCatalog, ns, extended) :: Nil
+    case DescribeNamespace(ResolvedNamespace(catalog, ns), extended, output) =>
+      DescribeNamespaceExec(output, catalog.asNamespaceCatalog, ns, extended) :: Nil
 
-    case desc @ DescribeRelation(r: ResolvedTable, partitionSpec, isExtended) =>
+    case DescribeRelation(r: ResolvedTable, partitionSpec, isExtended, output) =>
       if (partitionSpec.nonEmpty) {
-        throw new AnalysisException("DESCRIBE does not support partition for v2 tables.")
+        throw QueryCompilationErrors.describeDoesNotSupportPartitionForV2TablesError()
       }
-      DescribeTableExec(desc.output, r.table, isExtended) :: Nil
+      DescribeTableExec(output, r.table, isExtended) :: Nil
+
+    case DescribeColumn(_: ResolvedTable, column, isExtended, output) =>
+      column match {
+        case c: Attribute =>
+          DescribeColumnExec(output, c, isExtended) :: Nil
+        case nested =>
+          throw QueryCompilationErrors.commandNotSupportNestedColumnError(
+            "DESC TABLE COLUMN", toPrettySQL(nested))
+      }
+
+    case DropTable(r: ResolvedTable, ifExists, purge) =>
+      DropTableExec(r.catalog, r.identifier, ifExists, purge, invalidateTableCache(r)) :: Nil
 
-    case DropTable(catalog, ident, ifExists) =>
-      DropTableExec(catalog, ident, ifExists) :: Nil
+    case _: NoopCommand =>
+      LocalTableScanExec(Nil, Nil) :: Nil
 
     case AlterTable(catalog, ident, _, changes) =>
       AlterTableExec(catalog, ident, changes) :: Nil
 
-    case RenameTable(catalog, oldIdent, newIdent) =>
-      RenameTableExec(catalog, oldIdent, newIdent) :: Nil
-
-    case AlterNamespaceSetProperties(ResolvedNamespace(catalog, ns), properties) =>
+    case RenameTable(r @ ResolvedTable(catalog, oldIdent, _, _), newIdent, isView) =>
+      if (isView) {
+        throw QueryCompilationErrors.cannotRenameTableWithAlterViewError()
+      }
+      RenameTableExec(
+        catalog,
+        oldIdent,
+        newIdent.asIdentifier,
+        invalidateTableCache(r),
+        session.sharedState.cacheManager.cacheQuery) :: Nil
+
+    case SetNamespaceProperties(ResolvedNamespace(catalog, ns), properties) =>
       AlterNamespaceSetPropertiesExec(catalog.asNamespaceCatalog, ns, properties) :: Nil
 
-    case AlterNamespaceSetLocation(ResolvedNamespace(catalog, ns), location) =>
+    case SetNamespaceLocation(ResolvedNamespace(catalog, ns), location) =>
       AlterNamespaceSetPropertiesExec(
         catalog.asNamespaceCatalog,
         ns,
@@ -263,7 +314,7 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
         ns,
         Map(SupportsNamespaces.PROP_COMMENT -> comment)) :: Nil
 
-    case CommentOnTable(ResolvedTable(catalog, identifier, _), comment) =>
+    case CommentOnTable(ResolvedTable(catalog, identifier, _, _), comment) =>
       val changes = TableChange.setProperty(TableCatalog.PROP_COMMENT, comment)
       AlterTableExec(catalog, identifier, Seq(changes)) :: Nil
 
@@ -273,11 +324,11 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
     case DropNamespace(ResolvedNamespace(catalog, ns), ifExists, cascade) =>
       DropNamespaceExec(catalog, ns, ifExists, cascade) :: Nil
 
-    case r @ ShowNamespaces(ResolvedNamespace(catalog, ns), pattern) =>
-      ShowNamespacesExec(r.output, catalog.asNamespaceCatalog, ns, pattern) :: Nil
+    case ShowNamespaces(ResolvedNamespace(catalog, ns), pattern, output) =>
+      ShowNamespacesExec(output, catalog.asNamespaceCatalog, ns, pattern) :: Nil
 
-    case r @ ShowTables(ResolvedNamespace(catalog, ns), pattern) =>
-      ShowTablesExec(r.output, catalog.asTableCatalog, ns, pattern) :: Nil
+    case ShowTables(ResolvedNamespace(catalog, ns), pattern, output) =>
+      ShowTablesExec(output, catalog.asTableCatalog, ns, pattern) :: Nil
 
     case SetCatalogAndNamespace(catalogManager, catalogName, ns) =>
       SetCatalogAndNamespaceExec(catalogManager, catalogName, ns) :: Nil
@@ -285,8 +336,108 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
     case r: ShowCurrentNamespace =>
       ShowCurrentNamespaceExec(r.output, r.catalogManager) :: Nil
 
-    case r @ ShowTableProperties(rt: ResolvedTable, propertyKey) =>
-      ShowTablePropertiesExec(r.output, rt.table, propertyKey) :: Nil
+    case r @ ShowTableProperties(rt: ResolvedTable, propertyKey, output) =>
+      ShowTablePropertiesExec(output, rt.table, propertyKey) :: Nil
+
+    case AnalyzeTable(_: ResolvedTable, _, _) | AnalyzeColumn(_: ResolvedTable, _, _) =>
+      throw QueryCompilationErrors.analyzeTableNotSupportedForV2TablesError()
+
+    case AddPartitions(
+        r @ ResolvedTable(_, _, table: SupportsPartitionManagement, _), parts, ignoreIfExists) =>
+      AddPartitionExec(
+        table,
+        parts.asResolvedPartitionSpecs,
+        ignoreIfExists,
+        recacheTable(r)) :: Nil
+
+    case DropPartitions(
+        r @ ResolvedTable(_, _, table: SupportsPartitionManagement, _),
+        parts,
+        ignoreIfNotExists,
+        purge) =>
+      DropPartitionExec(
+        table,
+        parts.asResolvedPartitionSpecs,
+        ignoreIfNotExists,
+        purge,
+        recacheTable(r)) :: Nil
+
+    case RenamePartitions(
+        r @ ResolvedTable(_, _, table: SupportsPartitionManagement, _), from, to) =>
+      RenamePartitionExec(
+        table,
+        Seq(from).asResolvedPartitionSpecs.head,
+        Seq(to).asResolvedPartitionSpecs.head,
+        recacheTable(r)) :: Nil
+
+    case RecoverPartitions(_: ResolvedTable) =>
+      throw QueryCompilationErrors.alterTableRecoverPartitionsNotSupportedForV2TablesError()
+
+    case SetTableSerDeProperties(_: ResolvedTable, _, _, _) =>
+      throw QueryCompilationErrors.alterTableSerDePropertiesNotSupportedForV2TablesError()
+
+    case LoadData(_: ResolvedTable, _, _, _, _) =>
+      throw QueryCompilationErrors.loadDataNotSupportedForV2TablesError()
+
+    case ShowCreateTable(_: ResolvedTable, _, _) =>
+      throw QueryCompilationErrors.showCreateTableNotSupportedForV2TablesError()
+
+    case TruncateTable(r: ResolvedTable) =>
+      TruncateTableExec(
+        r.table.asTruncatable,
+        recacheTable(r)) :: Nil
+
+    case TruncatePartition(r: ResolvedTable, part) =>
+      TruncatePartitionExec(
+        r.table.asPartitionable,
+        Seq(part).asResolvedPartitionSpecs.head,
+        recacheTable(r)) :: Nil
+
+    case ShowColumns(_: ResolvedTable, _, _) =>
+      throw QueryCompilationErrors.showColumnsNotSupportedForV2TablesError()
+
+    case r @ ShowPartitions(
+        ResolvedTable(catalog, _, table: SupportsPartitionManagement, _),
+        pattern @ (None | Some(_: ResolvedPartitionSpec)), output) =>
+      ShowPartitionsExec(
+        output,
+        catalog,
+        table,
+        pattern.map(_.asInstanceOf[ResolvedPartitionSpec])) :: Nil
+
+    case RepairTable(_: ResolvedTable, _, _) =>
+      throw QueryCompilationErrors.repairTableNotSupportedForV2TablesError()
+
+    case r: CacheTable =>
+      CacheTableExec(r.table, r.multipartIdentifier, r.isLazy, r.options) :: Nil
+
+    case r: CacheTableAsSelect =>
+      CacheTableAsSelectExec(r.tempViewName, r.plan, r.originalText, r.isLazy, r.options) :: Nil
+
+    case r: UncacheTable =>
+      def isTempView(table: LogicalPlan): Boolean = table match {
+        case SubqueryAlias(_, v: View) => v.isTempView
+        case _ => false
+      }
+      UncacheTableExec(r.table, cascade = !isTempView(r.table)) :: Nil
+
+    case SetTableLocation(table: ResolvedTable, partitionSpec, location) =>
+      if (partitionSpec.nonEmpty) {
+        throw QueryCompilationErrors.alterV2TableSetLocationWithPartitionNotSupportedError()
+      }
+      val changes = Seq(TableChange.setProperty(TableCatalog.PROP_LOCATION, location))
+      AlterTableExec(table.catalog, table.identifier, changes) :: Nil
+
+    case SetTableProperties(table: ResolvedTable, props) =>
+      val changes = props.map { case (key, value) =>
+        TableChange.setProperty(key, value)
+      }.toSeq
+      AlterTableExec(table.catalog, table.identifier, changes) :: Nil
+
+    // TODO: v2 `UNSET TBLPROPERTIES` should respect the ifExists flag.
+    case UnsetTableProperties(table: ResolvedTable, keys, _) =>
+      val changes = keys.map(key => TableChange.removeProperty(key))
+      AlterTableExec(table.catalog, table.identifier, changes) :: Nil
 
     case _ => Nil
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DeleteFromTableExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DeleteFromTableExec.scala
index afebbfd01db22..05893a67b3728 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DeleteFromTableExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DeleteFromTableExec.scala
@@ -24,10 +24,12 @@ import org.apache.spark.sql.sources.Filter
 
 case class DeleteFromTableExec(
     table: SupportsDelete,
-    condition: Array[Filter]) extends V2CommandExec {
+    condition: Array[Filter],
+    refreshCache: () => Unit) extends LeafV2CommandExec {
 
   override protected def run(): Seq[InternalRow] = {
     table.deleteWhere(condition)
+    refreshCache()
     Seq.empty
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeColumnExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeColumnExec.scala
new file mode 100644
index 0000000000000..f7d79a1259ea9
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeColumnExec.scala
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import scala.collection.mutable.ArrayBuffer
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+
+case class DescribeColumnExec(
+    override val output: Seq[Attribute],
+    column: Attribute,
+    isExtended: Boolean) extends LeafV2CommandExec {
+
+  override protected def run(): Seq[InternalRow] = {
+    val rows = new ArrayBuffer[InternalRow]()
+
+    val comment = if (column.metadata.contains("comment")) {
+      column.metadata.getString("comment")
+    } else {
+      "NULL"
+    }
+
+    rows += toCatalystRow("col_name", column.name)
+    rows += toCatalystRow("data_type", column.dataType.catalogString)
+    rows += toCatalystRow("comment", comment)
+
+    // TODO: The extended description (isExtended = true) can be added here.
+
+    rows.toSeq
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeNamespaceExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeNamespaceExec.scala
index b4a14c6face31..bd8a4f06fe114 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeNamespaceExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeNamespaceExec.scala
@@ -21,10 +21,8 @@ import scala.collection.JavaConverters._
 import scala.collection.mutable.ArrayBuffer
 
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.encoders.RowEncoder
-import org.apache.spark.sql.catalyst.expressions.{Attribute, GenericRowWithSchema}
+import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.connector.catalog.{CatalogV2Util, SupportsNamespaces}
-import org.apache.spark.sql.types.StructType
 
 /**
  * Physical plan node for describing a namespace.
@@ -33,11 +31,7 @@ case class DescribeNamespaceExec(
     output: Seq[Attribute],
     catalog: SupportsNamespaces,
     namespace: Seq[String],
-    isExtended: Boolean) extends V2CommandExec {
-  private val toRow = {
-    RowEncoder(StructType.fromAttributes(output)).resolveAndBind().createSerializer()
-  }
-
+    isExtended: Boolean) extends LeafV2CommandExec {
   override protected def run(): Seq[InternalRow] = {
     val rows = new ArrayBuffer[InternalRow]()
     val ns = namespace.toArray
@@ -55,10 +49,6 @@ case class DescribeNamespaceExec(
         rows += toCatalystRow("Properties", properties.toSeq.mkString("(", ",", ")"))
       }
     }
-    rows
-  }
-
-  private def toCatalystRow(strs: String*): InternalRow = {
-    toRow(new GenericRowWithSchema(strs.toArray, schema)).copy()
+    rows.toSeq
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeTableExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeTableExec.scala
index bc6bb175f979e..c20189efc91fb 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeTableExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeTableExec.scala
@@ -21,29 +21,23 @@ import scala.collection.JavaConverters._
 import scala.collection.mutable.ArrayBuffer
 
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.encoders.RowEncoder
-import org.apache.spark.sql.catalyst.expressions.{Attribute, GenericRowWithSchema}
-import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Table, TableCatalog}
-import org.apache.spark.sql.types.StructType
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.{CatalogV2Util, SupportsMetadataColumns, Table}
 
 case class DescribeTableExec(
     output: Seq[Attribute],
     table: Table,
-    isExtended: Boolean) extends V2CommandExec {
-
-  private val toRow = {
-    RowEncoder(StructType.fromAttributes(output)).resolveAndBind().createSerializer()
-  }
-
+    isExtended: Boolean) extends LeafV2CommandExec {
   override protected def run(): Seq[InternalRow] = {
     val rows = new ArrayBuffer[InternalRow]()
     addSchema(rows)
     addPartitioning(rows)
 
     if (isExtended) {
+      addMetadataColumns(rows)
       addTableDetails(rows)
     }
-    rows
+    rows.toSeq
   }
 
   private def addTableDetails(rows: ArrayBuffer[InternalRow]): Unit = {
@@ -72,6 +66,19 @@ case class DescribeTableExec(
     }
   }
 
+  private def addMetadataColumns(rows: ArrayBuffer[InternalRow]): Unit = table match {
+    case hasMeta: SupportsMetadataColumns if hasMeta.metadataColumns.nonEmpty =>
+      rows += emptyRow()
+      rows += toCatalystRow("# Metadata Columns", "", "")
+      rows ++= hasMeta.metadataColumns.map { column =>
+        toCatalystRow(
+          column.name,
+          column.dataType.simpleString,
+          Option(column.comment()).getOrElse(""))
+      }
+    case _ =>
+  }
+
   private def addPartitioning(rows: ArrayBuffer[InternalRow]): Unit = {
     rows += emptyRow()
     rows += toCatalystRow("# Partitioning", "", "")
@@ -85,8 +92,4 @@ case class DescribeTableExec(
   }
 
   private def emptyRow(): InternalRow = toCatalystRow("", "", "")
-
-  private def toCatalystRow(strs: String*): InternalRow = {
-    toRow(new GenericRowWithSchema(strs.toArray, schema)).copy()
-  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DistributionAndOrderingUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DistributionAndOrderingUtils.scala
new file mode 100644
index 0000000000000..2e6ce3967a769
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DistributionAndOrderingUtils.scala
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.analysis.Resolver
+import org.apache.spark.sql.catalyst.expressions.{Ascending, Descending, Expression, NamedExpression, NullOrdering, NullsFirst, NullsLast, SortDirection, SortOrder}
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, RepartitionByExpression, Sort}
+import org.apache.spark.sql.connector.distributions.{ClusteredDistribution, OrderedDistribution, UnspecifiedDistribution}
+import org.apache.spark.sql.connector.expressions.{Expression => V2Expression, FieldReference, IdentityTransform, NullOrdering => V2NullOrdering, SortDirection => V2SortDirection, SortValue}
+import org.apache.spark.sql.connector.write.{RequiresDistributionAndOrdering, Write}
+import org.apache.spark.sql.errors.QueryCompilationErrors
+import org.apache.spark.sql.internal.SQLConf
+
+object DistributionAndOrderingUtils {
+
+  def prepareQuery(write: Write, query: LogicalPlan, conf: SQLConf): LogicalPlan = write match {
+    case write: RequiresDistributionAndOrdering =>
+      val resolver = conf.resolver
+
+      val numPartitions = write.requiredNumPartitions()
+      val distribution = write.requiredDistribution match {
+        case d: OrderedDistribution => d.ordering.map(e => toCatalyst(e, query, resolver))
+        case d: ClusteredDistribution => d.clustering.map(e => toCatalyst(e, query, resolver))
+        case _: UnspecifiedDistribution => Array.empty[Expression]
+      }
+
+      val queryWithDistribution = if (distribution.nonEmpty) {
+        val finalNumPartitions = if (numPartitions > 0) {
+          numPartitions
+        } else {
+          conf.numShufflePartitions
+        }
+        // the conversion to catalyst expressions above produces SortOrder expressions
+        // for OrderedDistribution and generic expressions for ClusteredDistribution
+        // this allows RepartitionByExpression to pick either range or hash partitioning
+        RepartitionByExpression(distribution, query, finalNumPartitions)
+      } else if (numPartitions > 0) {
+        throw QueryCompilationErrors.numberOfPartitionsNotAllowedWithUnspecifiedDistributionError()
+      } else {
+        query
+      }
+
+      val ordering = write.requiredOrdering.toSeq
+        .map(e => toCatalyst(e, query, resolver))
+        .asInstanceOf[Seq[SortOrder]]
+
+      val queryWithDistributionAndOrdering = if (ordering.nonEmpty) {
+        Sort(ordering, global = false, queryWithDistribution)
+      } else {
+        queryWithDistribution
+      }
+
+      queryWithDistributionAndOrdering
+
+    case _ =>
+      query
+  }
+
+  private def toCatalyst(
+      expr: V2Expression,
+      query: LogicalPlan,
+      resolver: Resolver): Expression = {
+
+    // we cannot perform the resolution in the analyzer since we need to optimize expressions
+    // in nodes like OverwriteByExpression before constructing a logical write
+    def resolve(ref: FieldReference): NamedExpression = {
+      query.resolve(ref.parts, resolver) match {
+        case Some(attr) => attr
+        case None => throw new AnalysisException(s"Cannot resolve '$ref' using ${query.output}")
+      }
+    }
+
+    expr match {
+      case SortValue(child, direction, nullOrdering) =>
+        val catalystChild = toCatalyst(child, query, resolver)
+        SortOrder(catalystChild, toCatalyst(direction), toCatalyst(nullOrdering), Seq.empty)
+      case IdentityTransform(ref) =>
+        resolve(ref)
+      case ref: FieldReference =>
+        resolve(ref)
+      case _ =>
+        throw new AnalysisException(s"$expr is not currently supported")
+    }
+  }
+
+  private def toCatalyst(direction: V2SortDirection): SortDirection = direction match {
+    case V2SortDirection.ASCENDING => Ascending
+    case V2SortDirection.DESCENDING => Descending
+  }
+
+  private def toCatalyst(nullOrdering: V2NullOrdering): NullOrdering = nullOrdering match {
+    case V2NullOrdering.NULLS_FIRST => NullsFirst
+    case V2NullOrdering.NULLS_LAST => NullsLast
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropNamespaceExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropNamespaceExec.scala
index f7b4317ad65e2..dbd5cbd874945 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropNamespaceExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropNamespaceExec.scala
@@ -17,11 +17,10 @@
 
 package org.apache.spark.sql.execution.datasources.v2
 
-import org.apache.spark.SparkException
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
 import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.{CatalogPlugin, SupportsNamespaces}
+import org.apache.spark.sql.connector.catalog.CatalogPlugin
+import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
 
 /**
  * Physical plan node for dropping a namespace.
@@ -31,7 +30,7 @@ case class DropNamespaceExec(
     namespace: Seq[String],
     ifExists: Boolean,
     cascade: Boolean)
-  extends V2CommandExec {
+  extends LeafV2CommandExec {
   override protected def run(): Seq[InternalRow] = {
     import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
 
@@ -43,17 +42,15 @@ case class DropNamespaceExec(
       if (!cascade) {
         if (catalog.asTableCatalog.listTables(ns).nonEmpty
           || nsCatalog.listNamespaces(ns).nonEmpty) {
-          throw new SparkException(
-            s"Cannot drop a non-empty namespace: ${namespace.quoted}. " +
-              "Use CASCADE option to drop a non-empty namespace.")
+          throw QueryExecutionErrors.cannotDropNonemptyNamespaceError(namespace)
         }
       }
 
       if (!nsCatalog.dropNamespace(ns)) {
-        throw new SparkException(s"Failed to drop a namespace: ${namespace.quoted}.")
+        throw QueryExecutionErrors.cannotDropNonemptyNamespaceError(namespace)
       }
     } else if (!ifExists) {
-      throw new NoSuchNamespaceException(ns)
+      throw QueryCompilationErrors.noSuchNamespaceError(ns)
     }
 
     Seq.empty
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionExec.scala
new file mode 100644
index 0000000000000..8dea5d66e4bfa
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionExec.scala
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionsException, ResolvedPartitionSpec}
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.{SupportsAtomicPartitionManagement, SupportsPartitionManagement}
+
+/**
+ * Physical plan node for dropping partitions of table.
+ */
+case class DropPartitionExec(
+    table: SupportsPartitionManagement,
+    partSpecs: Seq[ResolvedPartitionSpec],
+    ignoreIfNotExists: Boolean,
+    purge: Boolean,
+    refreshCache: () => Unit) extends LeafV2CommandExec {
+  import DataSourceV2Implicits._
+
+  override def output: Seq[Attribute] = Seq.empty
+
+  override protected def run(): Seq[InternalRow] = {
+    val (existsPartIdents, notExistsPartIdents) =
+      partSpecs.map(_.ident).partition(table.partitionExists)
+
+    if (notExistsPartIdents.nonEmpty && !ignoreIfNotExists) {
+      throw new NoSuchPartitionsException(
+        table.name(), notExistsPartIdents, table.partitionSchema())
+    }
+
+    val isTableAltered = existsPartIdents match {
+      case Seq() => false // Nothing will be done
+      case Seq(partIdent) =>
+        if (purge) table.purgePartition(partIdent) else table.dropPartition(partIdent)
+      case _ if table.isInstanceOf[SupportsAtomicPartitionManagement] =>
+        val idents = existsPartIdents.toArray
+        val atomicTable = table.asAtomicPartitionable
+        if (purge) atomicTable.purgePartitions(idents) else atomicTable.dropPartitions(idents)
+      case _ =>
+        throw new UnsupportedOperationException(
+          s"Nonatomic partition table ${table.name()} can not drop multiple partitions.")
+    }
+    if (isTableAltered) refreshCache()
+    Seq.empty
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTableExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTableExec.scala
index 967613f77577c..1e0627fb6dfdd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTableExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTableExec.scala
@@ -18,21 +18,26 @@
 package org.apache.spark.sql.execution.datasources.v2
 
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog}
+import org.apache.spark.sql.errors.QueryCompilationErrors
 
 /**
  * Physical plan node for dropping a table.
  */
-case class DropTableExec(catalog: TableCatalog, ident: Identifier, ifExists: Boolean)
-  extends V2CommandExec {
+case class DropTableExec(
+    catalog: TableCatalog,
+    ident: Identifier,
+    ifExists: Boolean,
+    purge: Boolean,
+    invalidateCache: () => Unit) extends LeafV2CommandExec {
 
   override def run(): Seq[InternalRow] = {
     if (catalog.tableExists(ident)) {
-      catalog.dropTable(ident)
+      invalidateCache()
+      if (purge) catalog.purgeTable(ident) else catalog.dropTable(ident)
     } else if (!ifExists) {
-      throw new NoSuchTableException(ident)
+      throw QueryCompilationErrors.noSuchTableError(ident)
     }
 
     Seq.empty
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/EmptyPartitionReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/EmptyPartitionReader.scala
index 711bd41e1db24..de5c786989f03 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/EmptyPartitionReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/EmptyPartitionReader.scala
@@ -17,9 +17,8 @@
 
 package org.apache.spark.sql.execution.datasources.v2
 
-import java.io.IOException
-
 import org.apache.spark.sql.connector.read.PartitionReader
+import org.apache.spark.sql.errors.QueryExecutionErrors
 
 /**
  * A [[PartitionReader]] with empty output.
@@ -28,7 +27,7 @@ class EmptyPartitionReader[T] extends PartitionReader[T] {
   override def next(): Boolean = false
 
   override def get(): T =
-    throw new IOException("No records should be returned from EmptyDataReader")
+    throw QueryExecutionErrors.noRecordsFromEmptyDataReaderError()
 
   override def close(): Unit = {}
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileBatchWrite.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileBatchWrite.scala
index 266c834909363..7227e48bc9a11 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileBatchWrite.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileBatchWrite.scala
@@ -23,6 +23,7 @@ import org.apache.spark.internal.io.FileCommitProtocol
 import org.apache.spark.sql.connector.write.{BatchWrite, DataWriterFactory, PhysicalWriteInfo, WriterCommitMessage}
 import org.apache.spark.sql.execution.datasources.{WriteJobDescription, WriteTaskResult}
 import org.apache.spark.sql.execution.datasources.FileFormatWriter.processStats
+import org.apache.spark.util.Utils
 
 class FileBatchWrite(
     job: Job,
@@ -31,8 +32,9 @@ class FileBatchWrite(
   extends BatchWrite with Logging {
   override def commit(messages: Array[WriterCommitMessage]): Unit = {
     val results = messages.map(_.asInstanceOf[WriteTaskResult])
-    committer.commitJob(job, results.map(_.commitMsg))
-    logInfo(s"Write Job ${description.uuid} committed.")
+    logInfo(s"Start to commit write Job ${description.uuid}.")
+    val (_, duration) = Utils.timeTakenMs { committer.commitJob(job, results.map(_.commitMsg)) }
+    logInfo(s"Write Job ${description.uuid} committed. Elapsed time: $duration ms.")
 
     processStats(description.statsTrackers, results.map(_.summary.stats))
     logInfo(s"Finished processing stats for write job ${description.uuid}.")
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileDataSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileDataSourceV2.scala
index 30a964d7e643f..08635d51172cf 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileDataSourceV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileDataSourceV2.scala
@@ -18,12 +18,16 @@ package org.apache.spark.sql.execution.datasources.v2
 
 import java.util
 
+import scala.collection.JavaConverters._
+
 import com.fasterxml.jackson.databind.ObjectMapper
+import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.Path
 
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.connector.catalog.{Table, TableProvider}
 import org.apache.spark.sql.connector.expressions.Transform
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.execution.datasources._
 import org.apache.spark.sql.sources.DataSourceRegister
 import org.apache.spark.sql.types.StructType
@@ -53,14 +57,23 @@ trait FileDataSourceV2 extends TableProvider with DataSourceRegister {
     paths ++ Option(map.get("path")).toSeq
   }
 
-  protected def getTableName(paths: Seq[String]): String = {
-    val name = shortName() + " " + paths.map(qualifiedPathName).mkString(",")
+  protected def getOptionsWithoutPaths(map: CaseInsensitiveStringMap): CaseInsensitiveStringMap = {
+    val withoutPath = map.asCaseSensitiveMap().asScala.filterKeys { k =>
+      !k.equalsIgnoreCase("path") && !k.equalsIgnoreCase("paths")
+    }
+    new CaseInsensitiveStringMap(withoutPath.toMap.asJava)
+  }
+
+  protected def getTableName(map: CaseInsensitiveStringMap, paths: Seq[String]): String = {
+    val hadoopConf = sparkSession.sessionState.newHadoopConfWithOptions(
+      map.asCaseSensitiveMap().asScala.toMap)
+    val name = shortName() + " " + paths.map(qualifiedPathName(_, hadoopConf)).mkString(",")
     Utils.redact(sparkSession.sessionState.conf.stringRedactionPattern, name)
   }
 
-  private def qualifiedPathName(path: String): String = {
+  private def qualifiedPathName(path: String, hadoopConf: Configuration): String = {
     val hdfsPath = new Path(path)
-    val fs = hdfsPath.getFileSystem(sparkSession.sessionState.newHadoopConf())
+    val fs = hdfsPath.getFileSystem(hadoopConf)
     hdfsPath.makeQualified(fs.getUri, fs.getWorkingDirectory).toString
   }
 
@@ -69,7 +82,7 @@ trait FileDataSourceV2 extends TableProvider with DataSourceRegister {
   //       implementation and directly implement the TableProvider APIs.
   protected def getTable(options: CaseInsensitiveStringMap): Table
   protected def getTable(options: CaseInsensitiveStringMap, schema: StructType): Table = {
-    throw new UnsupportedOperationException("user-specified schema")
+    throw QueryExecutionErrors.unsupportedUserSpecifiedSchemaError()
   }
 
   override def supportsExternalMetadata(): Boolean = true
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FilePartitionReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FilePartitionReader.scala
index 8f51d454b1434..697a1f341dd1b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FilePartitionReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FilePartitionReader.scala
@@ -20,10 +20,11 @@ import java.io.{FileNotFoundException, IOException}
 
 import org.apache.parquet.io.ParquetDecodingException
 
+import org.apache.spark.SparkUpgradeException
 import org.apache.spark.internal.Logging
 import org.apache.spark.rdd.InputFileBlockHolder
 import org.apache.spark.sql.connector.read.PartitionReader
-import org.apache.spark.sql.execution.QueryExecutionException
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.execution.datasources.SchemaColumnConvertNotSupportedException
 import org.apache.spark.sql.internal.SQLConf
 
@@ -46,11 +47,7 @@ class FilePartitionReader[T](readers: Iterator[PartitionedFileReader[T]])
             currentReader = null
           // Throw FileNotFoundException even if `ignoreCorruptFiles` is true
           case e: FileNotFoundException if !ignoreMissingFiles =>
-            throw new FileNotFoundException(
-              e.getMessage + "\n" +
-                "It is possible the underlying files have been updated. " +
-                "You can explicitly invalidate the cache in Spark by " +
-                "recreating the Dataset/DataFrame involved.")
+            throw QueryExecutionErrors.fileNotFoundError(e)
           case e @ (_: RuntimeException | _: IOException) if ignoreCorruptFiles =>
             logWarning(
               s"Skipped the rest of the content in the corrupted file.", e)
@@ -67,16 +64,13 @@ class FilePartitionReader[T](readers: Iterator[PartitionedFileReader[T]])
       currentReader != null && currentReader.next()
     } catch {
       case e: SchemaColumnConvertNotSupportedException =>
-        val message = "Parquet column cannot be converted in " +
-          s"file ${currentReader.file.filePath}. Column: ${e.getColumn}, " +
-          s"Expected: ${e.getLogicalType}, Found: ${e.getPhysicalType}"
-        throw new QueryExecutionException(message, e)
+        throw QueryExecutionErrors.unsupportedSchemaColumnConvertError(
+          currentReader.file.filePath, e.getColumn, e.getLogicalType, e.getPhysicalType, e)
       case e: ParquetDecodingException =>
-        if (e.getMessage.contains("Can not read value at")) {
-          val message = "Encounter error while reading parquet files. " +
-            "One possible cause: Parquet column cannot be converted in the " +
-            "corresponding files. Details: "
-          throw new QueryExecutionException(message, e)
+        if (e.getCause.isInstanceOf[SparkUpgradeException]) {
+          throw e.getCause
+        } else if (e.getMessage.contains("Can not read value at")) {
+          throw QueryExecutionErrors.cannotReadParquetFilesError(e)
         }
         throw e
       case e @ (_: RuntimeException | _: IOException) if ignoreCorruptFiles =>
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FilePartitionReaderFactory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FilePartitionReaderFactory.scala
index c1d91736a8b8e..5e160228c60e3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FilePartitionReaderFactory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FilePartitionReaderFactory.scala
@@ -18,6 +18,7 @@ package org.apache.spark.sql.execution.datasources.v2
 
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.connector.read.{InputPartition, PartitionReader, PartitionReaderFactory}
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.execution.datasources.{FilePartition, PartitionedFile}
 import org.apache.spark.sql.vectorized.ColumnarBatch
 
@@ -43,7 +44,7 @@ abstract class FilePartitionReaderFactory extends PartitionReaderFactory {
   def buildReader(partitionedFile: PartitionedFile): PartitionReader[InternalRow]
 
   def buildColumnarReader(partitionedFile: PartitionedFile): PartitionReader[ColumnarBatch] = {
-    throw new UnsupportedOperationException("Cannot create columnar reader.")
+    throw QueryExecutionErrors.cannotCreateColumnarReaderError()
   }
 }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileScan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileScan.scala
index 6e05aa56f4f72..ac63725b774d8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileScan.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileScan.scala
@@ -24,16 +24,19 @@ import org.apache.hadoop.fs.Path
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.IO_WARNING_LARGEFILETHRESHOLD
 import org.apache.spark.sql.{AnalysisException, SparkSession}
-import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionSet}
+import org.apache.spark.sql.catalyst.expressions.{AttributeSet, Expression, ExpressionSet}
 import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
+import org.apache.spark.sql.catalyst.plans.QueryPlan
 import org.apache.spark.sql.connector.read.{Batch, InputPartition, Scan, Statistics, SupportsReportStatistics}
 import org.apache.spark.sql.execution.PartitionedFileUtil
 import org.apache.spark.sql.execution.datasources._
+import org.apache.spark.sql.internal.connector.SupportsMetadata
 import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.util.Utils
 
-trait FileScan extends Scan with Batch with SupportsReportStatistics with Logging {
+trait FileScan extends Scan
+  with Batch with SupportsReportStatistics with SupportsMetadata with Logging {
   /**
    * Returns whether a file with `path` could be split or not.
    */
@@ -82,34 +85,54 @@ trait FileScan extends Scan with Batch with SupportsReportStatistics with Loggin
 
   protected def seqToString(seq: Seq[Any]): String = seq.mkString("[", ", ", "]")
 
+  private lazy val (normalizedPartitionFilters, normalizedDataFilters) = {
+    val output = readSchema().toAttributes
+    val partitionFilterAttributes = AttributeSet(partitionFilters).map(a => a.name -> a).toMap
+    val dataFiltersAttributes = AttributeSet(dataFilters).map(a => a.name -> a).toMap
+    val normalizedPartitionFilters = ExpressionSet(partitionFilters.map(
+      QueryPlan.normalizeExpressions(_,
+        output.map(a => partitionFilterAttributes.getOrElse(a.name, a)))))
+    val normalizedDataFilters = ExpressionSet(dataFilters.map(
+      QueryPlan.normalizeExpressions(_,
+        output.map(a => dataFiltersAttributes.getOrElse(a.name, a)))))
+    (normalizedPartitionFilters, normalizedDataFilters)
+  }
+
   override def equals(obj: Any): Boolean = obj match {
     case f: FileScan =>
-      fileIndex == f.fileIndex && readSchema == f.readSchema
-        ExpressionSet(partitionFilters) == ExpressionSet(f.partitionFilters) &&
-        ExpressionSet(dataFilters) == ExpressionSet(f.dataFilters)
+      fileIndex == f.fileIndex && readSchema == f.readSchema &&
+        normalizedPartitionFilters == f.normalizedPartitionFilters &&
+        normalizedDataFilters == f.normalizedDataFilters
 
     case _ => false
   }
 
   override def hashCode(): Int = getClass.hashCode()
 
+  val maxMetadataValueLength = sparkSession.sessionState.conf.maxMetadataStringLength
+
   override def description(): String = {
-    val locationDesc =
-      fileIndex.getClass.getSimpleName + fileIndex.rootPaths.mkString("[", ", ", "]")
-    val metadata: Map[String, String] = Map(
-      "ReadSchema" -> readDataSchema.catalogString,
-      "PartitionFilters" -> seqToString(partitionFilters),
-      "DataFilters" -> seqToString(dataFilters),
-      "Location" -> locationDesc)
-    val metadataStr = metadata.toSeq.sorted.map {
+    val metadataStr = getMetaData().toSeq.sorted.map {
       case (key, value) =>
         val redactedValue =
           Utils.redact(sparkSession.sessionState.conf.stringRedactionPattern, value)
-        key + ": " + StringUtils.abbreviate(redactedValue, 100)
+        key + ": " + StringUtils.abbreviate(redactedValue, maxMetadataValueLength)
     }.mkString(", ")
     s"${this.getClass.getSimpleName} $metadataStr"
   }
 
+  override def getMetaData(): Map[String, String] = {
+    val locationDesc =
+      fileIndex.getClass.getSimpleName +
+        Utils.buildLocationMetadata(fileIndex.rootPaths, maxMetadataValueLength)
+    Map(
+      "Format" -> s"${this.getClass.getSimpleName.replace("Scan", "").toLowerCase(Locale.ROOT)}",
+      "ReadSchema" -> readDataSchema.catalogString,
+      "PartitionFilters" -> seqToString(partitionFilters),
+      "DataFilters" -> seqToString(dataFilters),
+      "Location" -> locationDesc)
+  }
+
   protected def partitions: Seq[FilePartition] = {
     val selectedPartitions = fileIndex.listFiles(partitionFilters, dataFilters)
     val maxSplitBytes = FilePartition.maxSplitBytes(sparkSession, selectedPartitions)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileTable.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileTable.scala
index 59dc3ae56bf25..68d8d3c47e536 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileTable.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileTable.scala
@@ -53,7 +53,7 @@ abstract class FileTable(
     } else {
       // This is a non-streaming file based datasource.
       val rootPathsSpecified = DataSource.checkAndGlobPathIfNecessary(paths, hadoopConf,
-        checkEmptyGlobPath = true, checkFilesExist = true)
+        checkEmptyGlobPath = true, checkFilesExist = true, enableGlobbing = globPaths)
       val fileStatusCache = FileStatusCache.getOrCreate(sparkSession)
       new InMemoryFileIndex(
         sparkSession, rootPathsSpecified, caseSensitiveMap, userSpecifiedSchema, fileStatusCache)
@@ -79,7 +79,7 @@ abstract class FileTable(
 
   override lazy val schema: StructType = {
     val caseSensitive = sparkSession.sessionState.conf.caseSensitiveAnalysis
-    SchemaUtils.checkColumnNameDuplication(dataSchema.fieldNames,
+    SchemaUtils.checkSchemaColumnNameDuplication(dataSchema,
       "in the data schema", caseSensitive)
     dataSchema.foreach { field =>
       if (!supportsDataType(field.dataType)) {
@@ -88,7 +88,7 @@ abstract class FileTable(
       }
     }
     val partitionSchema = fileIndex.partitionSchema
-    SchemaUtils.checkColumnNameDuplication(partitionSchema.fieldNames,
+    SchemaUtils.checkSchemaColumnNameDuplication(partitionSchema,
       "in the partition schema", caseSensitive)
     val partitionNameSet: Set[String] =
       partitionSchema.fields.map(PartitioningUtils.getColName(_, caseSensitive)).toSet
@@ -139,8 +139,16 @@ abstract class FileTable(
    * 2. Catalog support is required, which is still under development for data source V2.
    */
   def fallbackFileFormat: Class[_ <: FileFormat]
+
+  /**
+   * Whether or not paths should be globbed before being used to access files.
+   */
+  private def globPaths: Boolean = {
+    val entry = options.get(DataSource.GLOB_PATHS_KEY)
+    Option(entry).map(_ == "true").getOrElse(true)
+  }
 }
 
 object FileTable {
-  private val CAPABILITIES = Set(BATCH_READ, BATCH_WRITE, TRUNCATE).asJava
+  private val CAPABILITIES = Set(BATCH_READ, BATCH_WRITE).asJava
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileWrite.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileWrite.scala
new file mode 100644
index 0000000000000..4f736cbd89706
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileWrite.scala
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2
+
+import java.util.UUID
+
+import scala.collection.JavaConverters._
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.mapreduce.Job
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat
+
+import org.apache.spark.internal.io.FileCommitProtocol
+import org.apache.spark.sql.{AnalysisException, SparkSession}
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateTimeUtils}
+import org.apache.spark.sql.connector.write.{BatchWrite, LogicalWriteInfo, Write}
+import org.apache.spark.sql.execution.datasources.{BasicWriteJobStatsTracker, DataSource, OutputWriterFactory, WriteJobDescription}
+import org.apache.spark.sql.execution.metric.SQLMetric
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types.{DataType, StructType}
+import org.apache.spark.sql.util.SchemaUtils
+import org.apache.spark.util.SerializableConfiguration
+
+trait FileWrite extends Write {
+  def paths: Seq[String]
+  def formatName: String
+  def supportsDataType: DataType => Boolean
+  def info: LogicalWriteInfo
+
+  private val schema = info.schema()
+  private val queryId = info.queryId()
+  private val options = info.options()
+
+  override def description(): String = formatName
+
+  override def toBatch: BatchWrite = {
+    val sparkSession = SparkSession.active
+    validateInputs(sparkSession.sessionState.conf.caseSensitiveAnalysis)
+    val path = new Path(paths.head)
+    val caseSensitiveMap = options.asCaseSensitiveMap.asScala.toMap
+    // Hadoop Configurations are case sensitive.
+    val hadoopConf = sparkSession.sessionState.newHadoopConfWithOptions(caseSensitiveMap)
+    val job = getJobInstance(hadoopConf, path)
+    val committer = FileCommitProtocol.instantiate(
+      sparkSession.sessionState.conf.fileCommitProtocolClass,
+      jobId = java.util.UUID.randomUUID().toString,
+      outputPath = paths.head)
+    lazy val description =
+      createWriteJobDescription(sparkSession, hadoopConf, job, paths.head, options.asScala.toMap)
+
+    committer.setupJob(job)
+    new FileBatchWrite(job, description, committer)
+  }
+
+  /**
+   * Prepares a write job and returns an [[OutputWriterFactory]].  Client side job preparation can
+   * be put here.  For example, user defined output committer can be configured here
+   * by setting the output committer class in the conf of spark.sql.sources.outputCommitterClass.
+   */
+  def prepareWrite(
+      sqlConf: SQLConf,
+      job: Job,
+      options: Map[String, String],
+      dataSchema: StructType): OutputWriterFactory
+
+  private def validateInputs(caseSensitiveAnalysis: Boolean): Unit = {
+    assert(schema != null, "Missing input data schema")
+    assert(queryId != null, "Missing query ID")
+
+    if (paths.length != 1) {
+      throw new IllegalArgumentException("Expected exactly one path to be specified, but " +
+        s"got: ${paths.mkString(", ")}")
+    }
+    val pathName = paths.head
+    SchemaUtils.checkColumnNameDuplication(schema.fields.map(_.name),
+      s"when inserting into $pathName", caseSensitiveAnalysis)
+    DataSource.validateSchema(schema)
+
+    schema.foreach { field =>
+      if (!supportsDataType(field.dataType)) {
+        throw new AnalysisException(
+          s"$formatName data source does not support ${field.dataType.catalogString} data type.")
+      }
+    }
+  }
+
+  private def getJobInstance(hadoopConf: Configuration, path: Path): Job = {
+    val job = Job.getInstance(hadoopConf)
+    job.setOutputKeyClass(classOf[Void])
+    job.setOutputValueClass(classOf[InternalRow])
+    FileOutputFormat.setOutputPath(job, path)
+    job
+  }
+
+  private def createWriteJobDescription(
+      sparkSession: SparkSession,
+      hadoopConf: Configuration,
+      job: Job,
+      pathName: String,
+      options: Map[String, String]): WriteJobDescription = {
+    val caseInsensitiveOptions = CaseInsensitiveMap(options)
+    // Note: prepareWrite has side effect. It sets "job".
+    val outputWriterFactory =
+      prepareWrite(sparkSession.sessionState.conf, job, caseInsensitiveOptions, schema)
+    val allColumns = schema.toAttributes
+    val metrics: Map[String, SQLMetric] = BasicWriteJobStatsTracker.metrics
+    val serializableHadoopConf = new SerializableConfiguration(hadoopConf)
+    val statsTracker = new BasicWriteJobStatsTracker(serializableHadoopConf, metrics)
+    // TODO: after partitioning is supported in V2:
+    //       1. filter out partition columns in `dataColumns`.
+    //       2. Don't use Seq.empty for `partitionColumns`.
+    new WriteJobDescription(
+      uuid = UUID.randomUUID().toString,
+      serializableHadoopConf = new SerializableConfiguration(job.getConfiguration),
+      outputWriterFactory = outputWriterFactory,
+      allColumns = allColumns,
+      dataColumns = allColumns,
+      partitionColumns = Seq.empty,
+      bucketIdExpression = None,
+      path = pathName,
+      customPartitionLocations = Map.empty,
+      maxRecordsPerFile = caseInsensitiveOptions.get("maxRecordsPerFile").map(_.toLong)
+        .getOrElse(sparkSession.sessionState.conf.maxRecordsPerFile),
+      timeZoneId = caseInsensitiveOptions.get(DateTimeUtils.TIMEZONE_OPTION)
+        .getOrElse(sparkSession.sessionState.conf.sessionLocalTimeZone),
+      statsTrackers = Seq(statsTracker)
+    )
+  }
+}
+
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileWriteBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileWriteBuilder.scala
deleted file mode 100644
index cd62ee7814bf2..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileWriteBuilder.scala
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.execution.datasources.v2
-
-import java.util.UUID
-
-import scala.collection.JavaConverters._
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.mapreduce.Job
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat
-
-import org.apache.spark.internal.io.FileCommitProtocol
-import org.apache.spark.sql.{AnalysisException, SparkSession}
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateTimeUtils}
-import org.apache.spark.sql.connector.write.{BatchWrite, LogicalWriteInfo, WriteBuilder}
-import org.apache.spark.sql.execution.datasources.{BasicWriteJobStatsTracker, DataSource, OutputWriterFactory, WriteJobDescription}
-import org.apache.spark.sql.execution.metric.SQLMetric
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{DataType, StructType}
-import org.apache.spark.sql.util.SchemaUtils
-import org.apache.spark.util.SerializableConfiguration
-
-abstract class FileWriteBuilder(
-    paths: Seq[String],
-    formatName: String,
-    supportsDataType: DataType => Boolean,
-    info: LogicalWriteInfo) extends WriteBuilder {
-  private val schema = info.schema()
-  private val queryId = info.queryId()
-  private val options = info.options()
-
-  override def buildForBatch(): BatchWrite = {
-    val sparkSession = SparkSession.active
-    validateInputs(sparkSession.sessionState.conf.caseSensitiveAnalysis)
-    val path = new Path(paths.head)
-    val caseSensitiveMap = options.asCaseSensitiveMap.asScala.toMap
-    // Hadoop Configurations are case sensitive.
-    val hadoopConf = sparkSession.sessionState.newHadoopConfWithOptions(caseSensitiveMap)
-    val job = getJobInstance(hadoopConf, path)
-    val committer = FileCommitProtocol.instantiate(
-      sparkSession.sessionState.conf.fileCommitProtocolClass,
-      jobId = java.util.UUID.randomUUID().toString,
-      outputPath = paths.head)
-    lazy val description =
-      createWriteJobDescription(sparkSession, hadoopConf, job, paths.head, options.asScala.toMap)
-
-    committer.setupJob(job)
-    new FileBatchWrite(job, description, committer)
-  }
-
-  /**
-   * Prepares a write job and returns an [[OutputWriterFactory]].  Client side job preparation can
-   * be put here.  For example, user defined output committer can be configured here
-   * by setting the output committer class in the conf of spark.sql.sources.outputCommitterClass.
-   */
-  def prepareWrite(
-      sqlConf: SQLConf,
-      job: Job,
-      options: Map[String, String],
-      dataSchema: StructType): OutputWriterFactory
-
-  private def validateInputs(caseSensitiveAnalysis: Boolean): Unit = {
-    assert(schema != null, "Missing input data schema")
-    assert(queryId != null, "Missing query ID")
-
-    if (paths.length != 1) {
-      throw new IllegalArgumentException("Expected exactly one path to be specified, but " +
-        s"got: ${paths.mkString(", ")}")
-    }
-    val pathName = paths.head
-    SchemaUtils.checkColumnNameDuplication(schema.fields.map(_.name),
-      s"when inserting into $pathName", caseSensitiveAnalysis)
-    DataSource.validateSchema(schema)
-
-    schema.foreach { field =>
-      if (!supportsDataType(field.dataType)) {
-        throw new AnalysisException(
-          s"$formatName data source does not support ${field.dataType.catalogString} data type.")
-      }
-    }
-  }
-
-  private def getJobInstance(hadoopConf: Configuration, path: Path): Job = {
-    val job = Job.getInstance(hadoopConf)
-    job.setOutputKeyClass(classOf[Void])
-    job.setOutputValueClass(classOf[InternalRow])
-    FileOutputFormat.setOutputPath(job, path)
-    job
-  }
-
-  private def createWriteJobDescription(
-      sparkSession: SparkSession,
-      hadoopConf: Configuration,
-      job: Job,
-      pathName: String,
-      options: Map[String, String]): WriteJobDescription = {
-    val caseInsensitiveOptions = CaseInsensitiveMap(options)
-    // Note: prepareWrite has side effect. It sets "job".
-    val outputWriterFactory =
-      prepareWrite(sparkSession.sessionState.conf, job, caseInsensitiveOptions, schema)
-    val allColumns = schema.toAttributes
-    val metrics: Map[String, SQLMetric] = BasicWriteJobStatsTracker.metrics
-    val serializableHadoopConf = new SerializableConfiguration(hadoopConf)
-    val statsTracker = new BasicWriteJobStatsTracker(serializableHadoopConf, metrics)
-    // TODO: after partitioning is supported in V2:
-    //       1. filter out partition columns in `dataColumns`.
-    //       2. Don't use Seq.empty for `partitionColumns`.
-    new WriteJobDescription(
-      uuid = UUID.randomUUID().toString,
-      serializableHadoopConf = new SerializableConfiguration(job.getConfiguration),
-      outputWriterFactory = outputWriterFactory,
-      allColumns = allColumns,
-      dataColumns = allColumns,
-      partitionColumns = Seq.empty,
-      bucketIdExpression = None,
-      path = pathName,
-      customPartitionLocations = Map.empty,
-      maxRecordsPerFile = caseInsensitiveOptions.get("maxRecordsPerFile").map(_.toLong)
-        .getOrElse(sparkSession.sessionState.conf.maxRecordsPerFile),
-      timeZoneId = caseInsensitiveOptions.get(DateTimeUtils.TIMEZONE_OPTION)
-        .getOrElse(sparkSession.sessionState.conf.sessionLocalTimeZone),
-      statsTrackers = Seq(statsTracker)
-    )
-  }
-}
-
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileWriterFactory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileWriterFactory.scala
index 1f25fed3000b2..d827e83623570 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileWriterFactory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileWriterFactory.scala
@@ -24,7 +24,7 @@ import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
 import org.apache.spark.internal.io.{FileCommitProtocol, SparkHadoopWriterUtils}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.connector.write.{DataWriter, DataWriterFactory}
-import org.apache.spark.sql.execution.datasources.{DynamicPartitionDataWriter, SingleDirectoryDataWriter, WriteJobDescription}
+import org.apache.spark.sql.execution.datasources.{DynamicPartitionDataSingleWriter, SingleDirectoryDataWriter, WriteJobDescription}
 
 case class FileWriterFactory (
     description: WriteJobDescription,
@@ -35,7 +35,7 @@ case class FileWriterFactory (
     if (description.partitionColumns.isEmpty) {
       new SingleDirectoryDataWriter(description, taskAttemptContext, committer)
     } else {
-      new DynamicPartitionDataWriter(description, taskAttemptContext, committer)
+      new DynamicPartitionDataSingleWriter(description, taskAttemptContext, committer)
     }
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MicroBatchScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MicroBatchScanExec.scala
index bca28e3cacb62..1430a32c8e81a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MicroBatchScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MicroBatchScanExec.scala
@@ -46,6 +46,6 @@ case class MicroBatchScanExec(
   override lazy val readerFactory: PartitionReaderFactory = stream.createReaderFactory()
 
   override lazy val inputRDD: RDD[InternalRow] = {
-    new DataSourceRDD(sparkContext, partitions, readerFactory, supportsColumnar)
+    new DataSourceRDD(sparkContext, partitions, readerFactory, supportsColumnar, customMetrics)
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/PushDownUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/PushDownUtils.scala
index 1a6f03f54f2e9..1f57f17911457 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/PushDownUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/PushDownUtils.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.execution.datasources.v2
 import scala.collection.mutable
 
 import org.apache.spark.sql.catalyst.expressions.{AttributeReference, AttributeSet, Expression, NamedExpression, PredicateHelper, SchemaPruning}
+import org.apache.spark.sql.catalyst.util.CharVarcharUtils
 import org.apache.spark.sql.connector.read.{Scan, ScanBuilder, SupportsPushDownFilters, SupportsPushDownRequiredColumns}
 import org.apache.spark.sql.execution.datasources.DataSourceStrategy
 import org.apache.spark.sql.internal.SQLConf
@@ -63,7 +64,7 @@ object PushDownUtils extends PredicateHelper {
         val postScanFilters = r.pushFilters(translatedFilters.toArray).map { filter =>
           DataSourceStrategy.rebuildExpressionFromFilter(filter, translatedFilterToExpr)
         }
-        (r.pushedFilters(), untranslatableExprs ++ postScanFilters)
+        (r.pushedFilters(), (untranslatableExprs ++ postScanFilters).toSeq)
 
       case _ => (Nil, filters)
     }
@@ -80,6 +81,10 @@ object PushDownUtils extends PredicateHelper {
       relation: DataSourceV2Relation,
       projects: Seq[NamedExpression],
       filters: Seq[Expression]): (Scan, Seq[AttributeReference]) = {
+    val exprs = projects ++ filters
+    val requiredColumns = AttributeSet(exprs.flatMap(_.references))
+    val neededOutput = relation.output.filter(requiredColumns.contains)
+
     scanBuilder match {
       case r: SupportsPushDownRequiredColumns if SQLConf.get.nestedSchemaPruningEnabled =>
         val rootFields = SchemaPruning.identifyRootFields(projects, filters)
@@ -88,21 +93,17 @@ object PushDownUtils extends PredicateHelper {
         } else {
           new StructType()
         }
-        r.pruneColumns(prunedSchema)
+        val neededFieldNames = neededOutput.map(_.name).toSet
+        r.pruneColumns(StructType(prunedSchema.filter(f => neededFieldNames.contains(f.name))))
         val scan = r.build()
         scan -> toOutputAttrs(scan.readSchema(), relation)
 
       case r: SupportsPushDownRequiredColumns =>
-        val exprs = projects ++ filters
-        val requiredColumns = AttributeSet(exprs.flatMap(_.references))
-        val neededOutput = relation.output.filter(requiredColumns.contains)
-        if (neededOutput != relation.output) {
-          r.pruneColumns(neededOutput.toStructType)
-          val scan = r.build()
-          scan -> toOutputAttrs(scan.readSchema(), relation)
-        } else {
-          r.build() -> relation.output
-        }
+        r.pruneColumns(neededOutput.toStructType)
+        val scan = r.build()
+        // always project, in case the relation's output has been updated and doesn't match
+        // the underlying table schema
+        scan -> toOutputAttrs(scan.readSchema(), relation)
 
       case _ => scanBuilder.build() -> relation.output
     }
@@ -112,7 +113,8 @@ object PushDownUtils extends PredicateHelper {
       schema: StructType,
       relation: DataSourceV2Relation): Seq[AttributeReference] = {
     val nameToAttr = relation.output.map(_.name).zip(relation.output).toMap
-    schema.toAttributes.map {
+    val cleaned = CharVarcharUtils.replaceCharVarcharWithStringInSchema(schema)
+    cleaned.toAttributes.map {
       // we have to keep the attribute id during transformation
       a => a.withExprId(nameToAttr(a.name).exprId)
     }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshTableExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshTableExec.scala
index 2a19ff304a9e0..05aa52decd21a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshTableExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshTableExec.scala
@@ -23,9 +23,14 @@ import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog}
 
 case class RefreshTableExec(
     catalog: TableCatalog,
-    ident: Identifier) extends V2CommandExec {
+    ident: Identifier,
+    refreshCache: () => Unit) extends LeafV2CommandExec {
   override protected def run(): Seq[InternalRow] = {
     catalog.invalidateTable(ident)
+
+    // Refresh all caches referencing the given table
+    refreshCache()
+
     Seq.empty
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RenamePartitionExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RenamePartitionExec.scala
new file mode 100644
index 0000000000000..1db29c80739ad
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RenamePartitionExec.scala
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.ResolvedPartitionSpec
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.SupportsPartitionManagement
+
+/**
+ * Physical plan node for renaming a table partition.
+ */
+case class RenamePartitionExec(
+    table: SupportsPartitionManagement,
+    from: ResolvedPartitionSpec,
+    to: ResolvedPartitionSpec,
+    refreshCache: () => Unit) extends LeafV2CommandExec {
+
+  override def output: Seq[Attribute] = Seq.empty
+
+  override protected def run(): Seq[InternalRow] = {
+    if (table.renamePartition(from.ident, to.ident)) {
+      refreshCache()
+    }
+    Seq.empty
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RenameTableExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RenameTableExec.scala
index a650607d5f129..f5ea355182a70 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RenameTableExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RenameTableExec.scala
@@ -17,9 +17,12 @@
 
 package org.apache.spark.sql.execution.datasources.v2
 
+import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog}
+import org.apache.spark.storage.StorageLevel
 
 /**
  * Physical plan node for renaming a table.
@@ -27,14 +30,34 @@ import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog}
 case class RenameTableExec(
     catalog: TableCatalog,
     oldIdent: Identifier,
-    newIdent: Identifier) extends V2CommandExec {
+    newIdent: Identifier,
+    invalidateCache: () => Option[StorageLevel],
+    cacheTable: (SparkSession, LogicalPlan, Option[String], StorageLevel) => Unit)
+  extends LeafV2CommandExec {
 
   override def output: Seq[Attribute] = Seq.empty
 
   override protected def run(): Seq[InternalRow] = {
+    import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.IdentifierHelper
+
+    val optOldStorageLevel = invalidateCache()
     catalog.invalidateTable(oldIdent)
-    catalog.renameTable(oldIdent, newIdent)
 
+    // If new identifier consists of a table name only, the table should be renamed in place.
+    // Such behavior matches to the v1 implementation of table renaming in Spark and other DBMSs.
+    val qualifiedNewIdent = if (newIdent.namespace.isEmpty) {
+      Identifier.of(oldIdent.namespace, newIdent.name)
+    } else newIdent
+    catalog.renameTable(oldIdent, qualifiedNewIdent)
+
+    optOldStorageLevel.foreach { oldStorageLevel =>
+      val tbl = catalog.loadTable(qualifiedNewIdent)
+      val newRelation = DataSourceV2Relation.create(tbl, Some(catalog), Some(qualifiedNewIdent))
+      cacheTable(
+        sqlContext.sparkSession,
+        newRelation,
+        Some(qualifiedNewIdent.quoted), oldStorageLevel)
+    }
     Seq.empty
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceTableExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceTableExec.scala
index 1f3bcf2e3fe57..749cbf631b03e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceTableExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceTableExec.scala
@@ -20,10 +20,11 @@ package org.apache.spark.sql.execution.datasources.v2
 import scala.collection.JavaConverters._
 
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.{CannotReplaceMissingTableException, NoSuchTableException}
+import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
 import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.{Identifier, StagedTable, StagingTableCatalog, TableCatalog}
+import org.apache.spark.sql.connector.catalog.{Identifier, StagedTable, StagingTableCatalog, Table, TableCatalog}
 import org.apache.spark.sql.connector.expressions.Transform
+import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.util.Utils
 
@@ -33,13 +34,16 @@ case class ReplaceTableExec(
     tableSchema: StructType,
     partitioning: Seq[Transform],
     tableProperties: Map[String, String],
-    orCreate: Boolean) extends V2CommandExec {
+    orCreate: Boolean,
+    invalidateCache: (TableCatalog, Table, Identifier) => Unit) extends LeafV2CommandExec {
 
   override protected def run(): Seq[InternalRow] = {
     if (catalog.tableExists(ident)) {
+      val table = catalog.loadTable(ident)
+      invalidateCache(catalog, table, ident)
       catalog.dropTable(ident)
     } else if (!orCreate) {
-      throw new CannotReplaceMissingTableException(ident)
+      throw QueryCompilationErrors.cannotReplaceMissingTableError(ident)
     }
     catalog.createTable(ident, tableSchema, partitioning.toArray, tableProperties.asJava)
     Seq.empty
@@ -54,9 +58,14 @@ case class AtomicReplaceTableExec(
     tableSchema: StructType,
     partitioning: Seq[Transform],
     tableProperties: Map[String, String],
-    orCreate: Boolean) extends V2CommandExec {
+    orCreate: Boolean,
+    invalidateCache: (TableCatalog, Table, Identifier) => Unit) extends LeafV2CommandExec {
 
   override protected def run(): Seq[InternalRow] = {
+    if (catalog.tableExists(identifier)) {
+      val table = catalog.loadTable(identifier)
+      invalidateCache(catalog, table, identifier)
+    }
     val staged = if (orCreate) {
       catalog.stageCreateOrReplace(
         identifier, tableSchema, partitioning.toArray, tableProperties.asJava)
@@ -66,10 +75,10 @@ case class AtomicReplaceTableExec(
           identifier, tableSchema, partitioning.toArray, tableProperties.asJava)
       } catch {
         case e: NoSuchTableException =>
-          throw new CannotReplaceMissingTableException(identifier, Some(e))
+          throw QueryCompilationErrors.cannotReplaceMissingTableError(identifier, Some(e))
       }
     } else {
-      throw new CannotReplaceMissingTableException(identifier)
+      throw QueryCompilationErrors.cannotReplaceMissingTableError(identifier)
     }
     commitOrAbortStagedChanges(staged)
     Seq.empty
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetCatalogAndNamespaceExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetCatalogAndNamespaceExec.scala
index 9e6f00e0923ea..fab95bffac25d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetCatalogAndNamespaceExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetCatalogAndNamespaceExec.scala
@@ -28,12 +28,12 @@ case class SetCatalogAndNamespaceExec(
     catalogManager: CatalogManager,
     catalogName: Option[String],
     namespace: Option[Seq[String]])
-    extends V2CommandExec {
+    extends LeafV2CommandExec {
   override protected def run(): Seq[InternalRow] = {
     // The catalog is updated first because CatalogManager resets the current namespace
     // when the current catalog is set.
-    catalogName.map(catalogManager.setCurrentCatalog)
-    namespace.map(ns => catalogManager.setCurrentNamespace(ns.toArray))
+    catalogName.foreach(catalogManager.setCurrentCatalog)
+    namespace.foreach(ns => catalogManager.setCurrentNamespace(ns.toArray))
 
     Seq.empty
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCurrentNamespaceExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCurrentNamespaceExec.scala
index 5f7b6f4061467..0977452a6ca08 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCurrentNamespaceExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCurrentNamespaceExec.scala
@@ -18,8 +18,7 @@
 package org.apache.spark.sql.execution.datasources.v2
 
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.encoders.RowEncoder
-import org.apache.spark.sql.catalyst.expressions.{Attribute, GenericRowWithSchema}
+import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.connector.catalog.CatalogManager
 import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.NamespaceHelper
 
@@ -29,13 +28,8 @@ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.NamespaceHelper
 case class ShowCurrentNamespaceExec(
     output: Seq[Attribute],
     catalogManager: CatalogManager)
-  extends V2CommandExec {
+  extends LeafV2CommandExec {
   override protected def run(): Seq[InternalRow] = {
-    val toRow = RowEncoder(schema).resolveAndBind().createSerializer()
-    val result = new GenericRowWithSchema(Array[Any](
-      catalogManager.currentCatalog.name,
-      catalogManager.currentNamespace.quoted),
-      schema)
-    Seq(toRow(result).copy())
+    Seq(toCatalystRow(catalogManager.currentCatalog.name, catalogManager.currentNamespace.quoted))
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowNamespacesExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowNamespacesExec.scala
index 9188f4eb60d56..9dafbd79a527e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowNamespacesExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowNamespacesExec.scala
@@ -20,8 +20,7 @@ package org.apache.spark.sql.execution.datasources.v2
 import scala.collection.mutable.ArrayBuffer
 
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.encoders.RowEncoder
-import org.apache.spark.sql.catalyst.expressions.{Attribute, GenericRowWithSchema}
+import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.util.StringUtils
 import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.NamespaceHelper
 import org.apache.spark.sql.connector.catalog.SupportsNamespaces
@@ -44,14 +43,12 @@ case class ShowNamespacesExec(
     }
 
     val rows = new ArrayBuffer[InternalRow]()
-    val toRow = RowEncoder(schema).resolveAndBind().createSerializer()
-
     namespaces.map(_.quoted).map { ns =>
       if (pattern.map(StringUtils.filterPattern(Seq(ns), _).nonEmpty).getOrElse(true)) {
-        rows += toRow(new GenericRowWithSchema(Array(ns), schema)).copy()
+        rows += toCatalystRow(ns)
       }
     }
 
-    rows
+    rows.toSeq
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowPartitionsExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowPartitionsExec.scala
new file mode 100644
index 0000000000000..f298d042b1c31
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowPartitionsExec.scala
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.ResolvedPartitionSpec
+import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils.escapePathName
+import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, Literal}
+import org.apache.spark.sql.connector.catalog.{SupportsPartitionManagement, TableCatalog}
+import org.apache.spark.sql.execution.LeafExecNode
+import org.apache.spark.sql.types.StringType
+import org.apache.spark.unsafe.types.UTF8String
+
+/**
+ * Physical plan node for showing partitions.
+ */
+case class ShowPartitionsExec(
+    output: Seq[Attribute],
+    catalog: TableCatalog,
+    table: SupportsPartitionManagement,
+    partitionSpec: Option[ResolvedPartitionSpec]) extends V2CommandExec with LeafExecNode {
+  override protected def run(): Seq[InternalRow] = {
+    val (names, ident) = partitionSpec
+      .map(spec => (spec.names, spec.ident))
+      // listPartitionByNames() should return all partitions if the partition spec
+      // does not specify any partition names.
+      .getOrElse((Seq.empty[String], InternalRow.empty))
+    val partitionIdentifiers = table.listPartitionIdentifiers(names.toArray, ident)
+    // Converting partition identifiers as `InternalRow` of partition values,
+    // for instance InternalRow(value0, value1, ..., valueN), to `InternalRow`s
+    // with a string in the format: "col0=value0/col1=value1/.../colN=valueN".
+    val schema = table.partitionSchema()
+    val len = schema.length
+    val partitions = new Array[String](len)
+    val timeZoneId = conf.sessionLocalTimeZone
+    val output = partitionIdentifiers.map { row =>
+      var i = 0
+      while (i < len) {
+        val dataType = schema(i).dataType
+        val partValueUTF8String =
+          Cast(Literal(row.get(i, dataType), dataType), StringType, Some(timeZoneId)).eval()
+        val partValueStr = if (partValueUTF8String == null) "null" else partValueUTF8String.toString
+        partitions(i) = escapePathName(schema(i).name) + "=" + escapePathName(partValueStr)
+        i += 1
+      }
+      partitions.mkString("/")
+    }
+    output.sorted.map(p => InternalRow(UTF8String.fromString(p)))
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablePropertiesExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablePropertiesExec.scala
index fef63cb8253ca..33d7337fab635 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablePropertiesExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablePropertiesExec.scala
@@ -18,9 +18,8 @@
 package org.apache.spark.sql.execution.datasources.v2
 
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.encoders.RowEncoder
-import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, GenericRowWithSchema}
-import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Table, TableCatalog}
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Table}
 
 /**
  * Physical plan node for showing table properties.
@@ -28,25 +27,22 @@ import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Table, TableCatalo
 case class ShowTablePropertiesExec(
     output: Seq[Attribute],
     catalogTable: Table,
-    propertyKey: Option[String]) extends V2CommandExec {
-
-  override def producedAttributes: AttributeSet = AttributeSet(output)
+    propertyKey: Option[String]) extends LeafV2CommandExec {
 
   override protected def run(): Seq[InternalRow] = {
     import scala.collection.JavaConverters._
-    val toRow = RowEncoder(schema).resolveAndBind().createSerializer()
 
-    // The reservered properties are accessible through DESCRIBE
+    // The reserved properties are accessible through DESCRIBE
     val properties = catalogTable.properties.asScala
-      .filter { case (k, v) => !CatalogV2Util.TABLE_RESERVED_PROPERTIES.contains(k) }
+      .filter { case (k, _) => !CatalogV2Util.TABLE_RESERVED_PROPERTIES.contains(k) }
     propertyKey match {
       case Some(p) =>
         val propValue = properties
           .getOrElse(p, s"Table ${catalogTable.name} does not have property: $p")
-        Seq(toRow(new GenericRowWithSchema(Array(p, propValue), schema)).copy())
+        Seq(toCatalystRow(p, propValue))
       case None =>
         properties.keys.map(k =>
-          toRow(new GenericRowWithSchema(Array(k, properties(k)), schema)).copy()).toSeq
+          toCatalystRow(k, properties(k))).toSeq
     }
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala
index 820f5ae8f1b12..b624e621f1f3d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala
@@ -20,8 +20,7 @@ package org.apache.spark.sql.execution.datasources.v2
 import scala.collection.mutable.ArrayBuffer
 
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.encoders.RowEncoder
-import org.apache.spark.sql.catalyst.expressions.{Attribute, GenericRowWithSchema}
+import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.util.StringUtils
 import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.NamespaceHelper
 import org.apache.spark.sql.connector.catalog.TableCatalog
@@ -37,18 +36,14 @@ case class ShowTablesExec(
     pattern: Option[String]) extends V2CommandExec with LeafExecNode {
   override protected def run(): Seq[InternalRow] = {
     val rows = new ArrayBuffer[InternalRow]()
-    val toRow = RowEncoder(schema).resolveAndBind().createSerializer()
 
     val tables = catalog.listTables(namespace.toArray)
     tables.map { table =>
       if (pattern.map(StringUtils.filterPattern(Seq(table.name()), _).nonEmpty).getOrElse(true)) {
-        val result = new GenericRowWithSchema(
-          Array(table.namespace().quoted, table.name()),
-          schema)
-        rows += toRow(result).copy()
+        rows += toCatalystRow(table.namespace().quoted, table.name(), false)
       }
     }
 
-    rows
+    rows.toSeq
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/TableCapabilityCheck.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/TableCapabilityCheck.scala
index 509a5f7139cca..acea306679403 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/TableCapabilityCheck.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/TableCapabilityCheck.scala
@@ -17,12 +17,13 @@
 
 package org.apache.spark.sql.execution.datasources.v2
 
-import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.expressions.Literal
 import org.apache.spark.sql.catalyst.plans.logical.{AppendData, LogicalPlan, OverwriteByExpression, OverwritePartitionsDynamic}
-import org.apache.spark.sql.connector.catalog.{SupportsWrite, Table}
+import org.apache.spark.sql.catalyst.streaming.StreamingRelationV2
+import org.apache.spark.sql.connector.catalog.Table
 import org.apache.spark.sql.connector.catalog.TableCapability._
-import org.apache.spark.sql.execution.streaming.{StreamingRelation, StreamingRelationV2}
+import org.apache.spark.sql.errors.QueryCompilationErrors
+import org.apache.spark.sql.execution.streaming.StreamingRelation
 import org.apache.spark.sql.types.BooleanType
 
 /**
@@ -31,8 +32,6 @@ import org.apache.spark.sql.types.BooleanType
 object TableCapabilityCheck extends (LogicalPlan => Unit) {
   import DataSourceV2Implicits._
 
-  private def failAnalysis(msg: String): Unit = throw new AnalysisException(msg)
-
   private def supportsBatchWrite(table: Table): Boolean = {
     table.supportsAny(BATCH_WRITE, V1_BATCH_WRITE)
   }
@@ -40,33 +39,31 @@ object TableCapabilityCheck extends (LogicalPlan => Unit) {
   override def apply(plan: LogicalPlan): Unit = {
     plan foreach {
       case r: DataSourceV2Relation if !r.table.supports(BATCH_READ) =>
-        failAnalysis(s"Table ${r.table.name()} does not support batch scan.")
+        throw QueryCompilationErrors.unsupportedBatchReadError(r.table)
 
       case r: StreamingRelationV2 if !r.table.supportsAny(MICRO_BATCH_READ, CONTINUOUS_READ) =>
-        throw new AnalysisException(s"Table ${r.table.name()} does not support either " +
-          "micro-batch or continuous scan.")
+        throw QueryCompilationErrors.unsupportedMicroBatchOrContinuousScanError(r.table)
 
       // TODO: check STREAMING_WRITE capability. It's not doable now because we don't have a
       //       a logical plan for streaming write.
-      case AppendData(r: DataSourceV2Relation, _, _, _) if !supportsBatchWrite(r.table) =>
-        failAnalysis(s"Table ${r.table.name()} does not support append in batch mode.")
+      case AppendData(r: DataSourceV2Relation, _, _, _, _) if !supportsBatchWrite(r.table) =>
+        throw QueryCompilationErrors.unsupportedAppendInBatchModeError(r.table)
 
-      case OverwritePartitionsDynamic(r: DataSourceV2Relation, _, _, _)
+      case OverwritePartitionsDynamic(r: DataSourceV2Relation, _, _, _, _)
         if !r.table.supports(BATCH_WRITE) || !r.table.supports(OVERWRITE_DYNAMIC) =>
-        failAnalysis(s"Table ${r.table.name()} does not support dynamic overwrite in batch mode.")
+        throw QueryCompilationErrors.unsupportedDynamicOverwriteInBatchModeError(r.table)
 
-      case OverwriteByExpression(r: DataSourceV2Relation, expr, _, _, _) =>
+      case OverwriteByExpression(r: DataSourceV2Relation, expr, _, _, _, _) =>
         expr match {
           case Literal(true, BooleanType) =>
             if (!supportsBatchWrite(r.table) ||
                 !r.table.supportsAny(TRUNCATE, OVERWRITE_BY_FILTER)) {
-              failAnalysis(
-                s"Table ${r.table.name()} does not support truncate in batch mode.")
+              throw QueryCompilationErrors.unsupportedTruncateInBatchModeError(r.table)
             }
           case _ =>
             if (!supportsBatchWrite(r.table) || !r.table.supports(OVERWRITE_BY_FILTER)) {
-              failAnalysis(s"Table ${r.table.name()} does not support " +
-                "overwrite by filter in batch mode.")
+              throw QueryCompilationErrors.unsupportedOverwriteByFilterInBatchModeError(
+                r.table)
             }
         }
 
@@ -92,10 +89,8 @@ object TableCapabilityCheck extends (LogicalPlan => Unit) {
           streamingSources.filter(_.supports(MICRO_BATCH_READ)).map(_.name()) ++
             v1StreamingRelations.map(_.sourceName)
         val continuousSources = streamingSources.filter(_.supports(CONTINUOUS_READ)).map(_.name())
-        throw new AnalysisException(
-          "The streaming sources in a query do not have a common supported execution mode.\n" +
-            "Sources support micro-batch: " + microBatchSources.mkString(", ") + "\n" +
-            "Sources support continuous: " + continuousSources.mkString(", "))
+        throw QueryCompilationErrors.streamingSourcesDoNotSupportCommonExecutionModeError(
+          microBatchSources, continuousSources)
       }
     }
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/TextBasedFileScan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/TextBasedFileScan.scala
index 1ca3fd42c0597..f24fb95acb922 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/TextBasedFileScan.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/TextBasedFileScan.scala
@@ -22,8 +22,6 @@ import org.apache.hadoop.fs.Path
 import org.apache.hadoop.io.compress.CompressionCodecFactory
 
 import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.execution.datasources.PartitioningAwareFileIndex
-import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 import org.apache.spark.util.Utils
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/TruncatePartitionExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/TruncatePartitionExec.scala
new file mode 100644
index 0000000000000..6ebfbc1e55959
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/TruncatePartitionExec.scala
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.ResolvedPartitionSpec
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.{SupportsAtomicPartitionManagement, SupportsPartitionManagement}
+
+/**
+ * Physical plan node for table partition truncation.
+ */
+case class TruncatePartitionExec(
+    table: SupportsPartitionManagement,
+    partSpec: ResolvedPartitionSpec,
+    refreshCache: () => Unit) extends LeafV2CommandExec {
+
+  override def output: Seq[Attribute] = Seq.empty
+
+  override protected def run(): Seq[InternalRow] = {
+    val isTableAltered = if (table.partitionSchema.length != partSpec.names.length) {
+      table match {
+        case atomicPartTable: SupportsAtomicPartitionManagement =>
+          val partitionIdentifiers = atomicPartTable.listPartitionIdentifiers(
+            partSpec.names.toArray, partSpec.ident)
+          atomicPartTable.truncatePartitions(partitionIdentifiers)
+        case _ =>
+          throw new UnsupportedOperationException(
+            s"The table ${table.name()} does not support truncation of multiple partition.")
+      }
+    } else {
+      table.truncatePartition(partSpec.ident)
+    }
+    if (isTableAltered) refreshCache()
+    Seq.empty
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/TruncateTableExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/TruncateTableExec.scala
new file mode 100644
index 0000000000000..948dc1bc8c87c
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/TruncateTableExec.scala
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.TruncatableTable
+
+/**
+ * Physical plan node for table truncation.
+ */
+case class TruncateTableExec(
+    table: TruncatableTable,
+    refreshCache: () => Unit) extends LeafV2CommandExec {
+
+  override def output: Seq[Attribute] = Seq.empty
+
+  override protected def run(): Seq[InternalRow] = {
+    if (table.truncateTable()) refreshCache()
+    Seq.empty
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V1FallbackWriters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V1FallbackWriters.scala
index 7502a8750aee4..f99a4db5e39f8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V1FallbackWriters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V1FallbackWriters.scala
@@ -17,19 +17,14 @@
 
 package org.apache.spark.sql.execution.datasources.v2
 
-import java.util.UUID
-
-import org.apache.spark.SparkException
-import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.Dataset
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.connector.catalog.SupportsWrite
-import org.apache.spark.sql.connector.write.{LogicalWriteInfoImpl, SupportsOverwrite, SupportsTruncate, V1WriteBuilder, WriteBuilder}
+import org.apache.spark.sql.connector.write.V1Write
 import org.apache.spark.sql.execution.SparkPlan
-import org.apache.spark.sql.sources.{AlwaysTrue, Filter, InsertableRelation}
-import org.apache.spark.sql.util.CaseInsensitiveStringMap
+import org.apache.spark.sql.sources.InsertableRelation
 
 /**
  * Physical plan node for append into a v2 table using V1 write interfaces.
@@ -38,13 +33,9 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap
  */
 case class AppendDataExecV1(
     table: SupportsWrite,
-    writeOptions: CaseInsensitiveStringMap,
-    plan: LogicalPlan) extends V1FallbackWriters {
-
-  override protected def run(): Seq[InternalRow] = {
-    writeWithV1(newWriteBuilder().buildForV1Write())
-  }
-}
+    plan: LogicalPlan,
+    refreshCache: () => Unit,
+    write: V1Write) extends V1FallbackWriters
 
 /**
  * Physical plan node for overwrite into a v2 table with V1 write interfaces. Note that when this
@@ -59,52 +50,22 @@ case class AppendDataExecV1(
  */
 case class OverwriteByExpressionExecV1(
     table: SupportsWrite,
-    deleteWhere: Array[Filter],
-    writeOptions: CaseInsensitiveStringMap,
-    plan: LogicalPlan) extends V1FallbackWriters {
-
-  private def isTruncate(filters: Array[Filter]): Boolean = {
-    filters.length == 1 && filters(0).isInstanceOf[AlwaysTrue]
-  }
-
-  override protected def run(): Seq[InternalRow] = {
-    newWriteBuilder() match {
-      case builder: SupportsTruncate if isTruncate(deleteWhere) =>
-        writeWithV1(builder.truncate().asV1Builder.buildForV1Write())
-
-      case builder: SupportsOverwrite =>
-        writeWithV1(builder.overwrite(deleteWhere).asV1Builder.buildForV1Write())
-
-      case _ =>
-        throw new SparkException(s"Table does not support overwrite by expression: $table")
-    }
-  }
-}
+    plan: LogicalPlan,
+    refreshCache: () => Unit,
+    write: V1Write) extends V1FallbackWriters
 
 /** Some helper interfaces that use V2 write semantics through the V1 writer interface. */
-sealed trait V1FallbackWriters extends V2CommandExec with SupportsV1Write {
+sealed trait V1FallbackWriters extends LeafV2CommandExec with SupportsV1Write {
   override def output: Seq[Attribute] = Nil
-  override final def children: Seq[SparkPlan] = Nil
 
   def table: SupportsWrite
-  def writeOptions: CaseInsensitiveStringMap
-
-  protected implicit class toV1WriteBuilder(builder: WriteBuilder) {
-    def asV1Builder: V1WriteBuilder = builder match {
-      case v1: V1WriteBuilder => v1
-      case other => throw new IllegalStateException(
-        s"The returned writer ${other} was no longer a V1WriteBuilder.")
-    }
-  }
-
-  protected def newWriteBuilder(): V1WriteBuilder = {
-    val info = LogicalWriteInfoImpl(
-      queryId = UUID.randomUUID().toString,
-      schema = plan.schema,
-      options = writeOptions)
-    val writeBuilder = table.newWriteBuilder(info)
+  def refreshCache: () => Unit
+  def write: V1Write
 
-    writeBuilder.asV1Builder
+  override def run(): Seq[InternalRow] = {
+    val writtenRows = writeWithV1(write.toInsertableRelation)
+    refreshCache()
+    writtenRows
   }
 }
 
@@ -112,7 +73,6 @@ sealed trait V1FallbackWriters extends V2CommandExec with SupportsV1Write {
  * A trait that allows Tables that use V1 Writer interfaces to append data.
  */
 trait SupportsV1Write extends SparkPlan {
-  // TODO: We should be able to work on SparkPlans at this point.
   def plan: LogicalPlan
 
   protected def writeWithV1(relation: InsertableRelation): Seq[InternalRow] = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2CommandExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2CommandExec.scala
index 4be4a6b30edcd..3c67a3d968fe6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2CommandExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2CommandExec.scala
@@ -19,7 +19,11 @@ package org.apache.spark.sql.execution.datasources.v2
 
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.encoders.RowEncoder
+import org.apache.spark.sql.catalyst.expressions.{AttributeSet, GenericRowWithSchema}
+import org.apache.spark.sql.catalyst.trees.LeafLike
 import org.apache.spark.sql.execution.SparkPlan
+import org.apache.spark.sql.types.StructType
 
 /**
  * A physical operator that executes run() and saves the result to prevent multiple executions.
@@ -44,7 +48,7 @@ abstract class V2CommandExec extends SparkPlan {
    */
   override def executeCollect(): Array[InternalRow] = result.toArray
 
-  override def executeToIterator: Iterator[InternalRow] = result.toIterator
+  override def executeToIterator(): Iterator[InternalRow] = result.toIterator
 
   override def executeTake(limit: Int): Array[InternalRow] = result.take(limit).toArray
 
@@ -54,5 +58,15 @@ abstract class V2CommandExec extends SparkPlan {
     sqlContext.sparkContext.parallelize(result, 1)
   }
 
-  override def children: Seq[SparkPlan] = Nil
+  override def producedAttributes: AttributeSet = outputSet
+
+  protected def toCatalystRow(values: Any*): InternalRow = {
+    rowSerializer(new GenericRowWithSchema(values.toArray, schema)).copy()
+  }
+
+  private lazy val rowSerializer = {
+    RowEncoder(StructType.fromAttributes(output)).resolveAndBind().createSerializer()
+  }
 }
+
+trait LeafV2CommandExec extends V2CommandExec with LeafLike[SparkPlan]
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2ScanRelationPushDown.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2ScanRelationPushDown.scala
index b168e848f0b6f..d2180566790ac 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2ScanRelationPushDown.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2ScanRelationPushDown.scala
@@ -64,7 +64,7 @@ object V2ScanRelationPushDown extends Rule[LogicalPlan] {
         case _ => scan
       }
 
-      val scanRelation = DataSourceV2ScanRelation(relation.table, wrappedScan, output)
+      val scanRelation = DataSourceV2ScanRelation(relation, wrappedScan, output)
 
       val projectionOverSchema = ProjectionOverSchema(output.toStructType)
       val projectionFunc = (expr: Expression) => expr transformDown {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalog.scala
index 2ed33b867183b..33b8f22e3f88a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalog.scala
@@ -23,23 +23,22 @@ import java.util
 import scala.collection.JavaConverters._
 import scala.collection.mutable
 
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.{NamespaceAlreadyExistsException, NoSuchNamespaceException, NoSuchTableException, TableAlreadyExistsException}
+import org.apache.spark.sql.catalyst.{SQLConfHelper, TableIdentifier}
+import org.apache.spark.sql.catalyst.analysis.{NoSuchTableException, TableAlreadyExistsException}
 import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogDatabase, CatalogTable, CatalogTableType, CatalogUtils, SessionCatalog}
 import org.apache.spark.sql.connector.catalog.{CatalogManager, CatalogV2Util, Identifier, NamespaceChange, SupportsNamespaces, Table, TableCatalog, TableChange, V1Table}
 import org.apache.spark.sql.connector.catalog.NamespaceChange.RemoveProperty
 import org.apache.spark.sql.connector.expressions.{BucketTransform, FieldReference, IdentityTransform, Transform}
+import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
 import org.apache.spark.sql.execution.datasources.DataSource
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
 /**
  * A [[TableCatalog]] that translates calls to the v1 SessionCatalog.
  */
-class V2SessionCatalog(catalog: SessionCatalog, conf: SQLConf)
-  extends TableCatalog with SupportsNamespaces {
-  import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.NamespaceHelper
+class V2SessionCatalog(catalog: SessionCatalog)
+  extends TableCatalog with SupportsNamespaces with SQLConfHelper {
   import V2SessionCatalog._
 
   override val defaultNamespace: Array[String] = Array("default")
@@ -57,7 +56,7 @@ class V2SessionCatalog(catalog: SessionCatalog, conf: SQLConf)
           .map(ident => Identifier.of(Array(ident.database.getOrElse("")), ident.table))
           .toArray
       case _ =>
-        throw new NoSuchNamespaceException(namespace)
+        throw QueryCompilationErrors.noSuchNamespaceError(namespace)
     }
   }
 
@@ -66,7 +65,7 @@ class V2SessionCatalog(catalog: SessionCatalog, conf: SQLConf)
       catalog.getTableMetadata(ident.asTableIdentifier)
     } catch {
       case _: NoSuchTableException =>
-        throw new NoSuchTableException(ident)
+        throw QueryCompilationErrors.noSuchTableError(ident)
     }
 
     V1Table(catalogTable)
@@ -86,9 +85,14 @@ class V2SessionCatalog(catalog: SessionCatalog, conf: SQLConf)
     val provider = properties.getOrDefault(TableCatalog.PROP_PROVIDER, conf.defaultDataSourceName)
     val tableProperties = properties.asScala
     val location = Option(properties.get(TableCatalog.PROP_LOCATION))
-    val storage = DataSource.buildStorageFormatFromOptions(tableProperties.toMap)
+    val storage = DataSource.buildStorageFormatFromOptions(toOptions(tableProperties.toMap))
         .copy(locationUri = location.map(CatalogUtils.stringToURI))
-    val tableType = if (location.isDefined) CatalogTableType.EXTERNAL else CatalogTableType.MANAGED
+    val isExternal = properties.containsKey(TableCatalog.PROP_EXTERNAL)
+    val tableType = if (isExternal || location.isDefined) {
+      CatalogTableType.EXTERNAL
+    } else {
+      CatalogTableType.MANAGED
+    }
 
     val tableDesc = CatalogTable(
       identifier = ident.asTableIdentifier,
@@ -106,12 +110,18 @@ class V2SessionCatalog(catalog: SessionCatalog, conf: SQLConf)
       catalog.createTable(tableDesc, ignoreIfExists = false)
     } catch {
       case _: TableAlreadyExistsException =>
-        throw new TableAlreadyExistsException(ident)
+        throw QueryCompilationErrors.tableAlreadyExistsError(ident)
     }
 
     loadTable(ident)
   }
 
+  private def toOptions(properties: Map[String, String]): Map[String, String] = {
+    properties.filterKeys(_.startsWith(TableCatalog.OPTION_PREFIX)).map {
+      case (key, value) => key.drop(TableCatalog.OPTION_PREFIX.length) -> value
+    }.toMap
+  }
+
   override def alterTable(
       ident: Identifier,
       changes: TableChange*): Table = {
@@ -119,21 +129,28 @@ class V2SessionCatalog(catalog: SessionCatalog, conf: SQLConf)
       catalog.getTableMetadata(ident.asTableIdentifier)
     } catch {
       case _: NoSuchTableException =>
-        throw new NoSuchTableException(ident)
+        throw QueryCompilationErrors.noSuchTableError(ident)
     }
 
     val properties = CatalogV2Util.applyPropertiesChanges(catalogTable.properties, changes)
     val schema = CatalogV2Util.applySchemaChanges(catalogTable.schema, changes)
     val comment = properties.get(TableCatalog.PROP_COMMENT)
     val owner = properties.getOrElse(TableCatalog.PROP_OWNER, catalogTable.owner)
+    val location = properties.get(TableCatalog.PROP_LOCATION).map(CatalogUtils.stringToURI)
+    val storage = if (location.isDefined) {
+      catalogTable.storage.copy(locationUri = location)
+    } else {
+      catalogTable.storage
+    }
 
     try {
       catalog.alterTable(
-        catalogTable
-          .copy(properties = properties, schema = schema, owner = owner, comment = comment))
+        catalogTable.copy(
+          properties = properties, schema = schema, owner = owner, comment = comment,
+          storage = storage))
     } catch {
       case _: NoSuchTableException =>
-        throw new NoSuchTableException(ident)
+        throw QueryCompilationErrors.noSuchTableError(ident)
     }
 
     loadTable(ident)
@@ -158,7 +175,7 @@ class V2SessionCatalog(catalog: SessionCatalog, conf: SQLConf)
 
   override def renameTable(oldIdent: Identifier, newIdent: Identifier): Unit = {
     if (tableExists(newIdent)) {
-      throw new TableAlreadyExistsException(newIdent)
+      throw QueryCompilationErrors.tableAlreadyExistsError(newIdent)
     }
 
     // Load table to make sure the table exists
@@ -167,15 +184,12 @@ class V2SessionCatalog(catalog: SessionCatalog, conf: SQLConf)
   }
 
   implicit class TableIdentifierHelper(ident: Identifier) {
-    import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.IdentifierHelper
-
     def asTableIdentifier: TableIdentifier = {
       ident.namespace match {
         case Array(db) =>
           TableIdentifier(ident.name, Some(db))
         case _ =>
-          throw new NoSuchTableException(
-            s"V2 session catalog requires a single-part namespace: ${ident.quoted}")
+          throw QueryCompilationErrors.requiresSinglePartNamespaceError(ident)
       }
     }
   }
@@ -198,7 +212,7 @@ class V2SessionCatalog(catalog: SessionCatalog, conf: SQLConf)
       case Array(db) if catalog.databaseExists(db) =>
         Array()
       case _ =>
-        throw new NoSuchNamespaceException(namespace)
+        throw QueryCompilationErrors.noSuchNamespaceError(namespace)
     }
   }
 
@@ -208,7 +222,7 @@ class V2SessionCatalog(catalog: SessionCatalog, conf: SQLConf)
         catalog.getDatabaseMetadata(db).toMetadata
 
       case _ =>
-        throw new NoSuchNamespaceException(namespace)
+        throw QueryCompilationErrors.noSuchNamespaceError(namespace)
     }
   }
 
@@ -221,10 +235,10 @@ class V2SessionCatalog(catalog: SessionCatalog, conf: SQLConf)
         ignoreIfExists = false)
 
     case Array(_) =>
-      throw new NamespaceAlreadyExistsException(namespace)
+      throw QueryCompilationErrors.namespaceAlreadyExistsError(namespace)
 
     case _ =>
-      throw new IllegalArgumentException(s"Invalid namespace name: ${namespace.quoted}")
+      throw QueryExecutionErrors.invalidNamespaceNameError(namespace)
   }
 
   override def alterNamespace(namespace: Array[String], changes: NamespaceChange*): Unit = {
@@ -234,8 +248,7 @@ class V2SessionCatalog(catalog: SessionCatalog, conf: SQLConf)
         changes.foreach {
           case remove: RemoveProperty
             if CatalogV2Util.NAMESPACE_RESERVED_PROPERTIES.contains(remove.property) =>
-            throw new UnsupportedOperationException(
-              s"Cannot remove reserved property: ${remove.property}")
+            throw QueryExecutionErrors.cannotRemoveReservedPropertyError(remove.property)
           case _ =>
         }
 
@@ -244,14 +257,14 @@ class V2SessionCatalog(catalog: SessionCatalog, conf: SQLConf)
           toCatalogDatabase(db, CatalogV2Util.applyNamespaceChanges(metadata, changes)))
 
       case _ =>
-        throw new NoSuchNamespaceException(namespace)
+        throw QueryCompilationErrors.noSuchNamespaceError(namespace)
     }
   }
 
   override def dropNamespace(namespace: Array[String]): Boolean = namespace match {
     case Array(db) if catalog.databaseExists(db) =>
       if (catalog.listTables(db).nonEmpty) {
-        throw new IllegalStateException(s"Namespace ${namespace.quoted} is not empty")
+        throw QueryExecutionErrors.namespaceNotEmptyError(namespace)
       }
       catalog.dropDatabase(db, ignoreIfNotExists = false, cascade = false)
       true
@@ -261,7 +274,7 @@ class V2SessionCatalog(catalog: SessionCatalog, conf: SQLConf)
       false
 
     case _ =>
-      throw new NoSuchNamespaceException(namespace)
+      throw QueryCompilationErrors.noSuchNamespaceError(namespace)
   }
 
   override def toString: String = s"V2SessionCatalog($name)"
@@ -284,11 +297,10 @@ private[sql] object V2SessionCatalog {
         bucketSpec = Some(BucketSpec(numBuckets, col :: Nil, Nil))
 
       case transform =>
-        throw new UnsupportedOperationException(
-          s"SessionCatalog does not support partition transform: $transform")
+        throw QueryExecutionErrors.unsupportedPartitionTransformError(transform)
     }
 
-    (identityCols, bucketSpec)
+    (identityCols.toSeq, bucketSpec)
   }
 
   private def toCatalogDatabase(
@@ -301,7 +313,7 @@ private[sql] object V2SessionCatalog {
       locationUri = Option(metadata.get(SupportsNamespaces.PROP_LOCATION))
           .map(CatalogUtils.stringToURI)
           .orElse(defaultLocation)
-          .getOrElse(throw new IllegalArgumentException("Missing database location")),
+          .getOrElse(throw QueryExecutionErrors.missingDatabaseLocationError()),
       properties = metadata.asScala.toMap --
         Seq(SupportsNamespaces.PROP_COMMENT, SupportsNamespaces.PROP_LOCATION))
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2Writes.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2Writes.scala
new file mode 100644
index 0000000000000..6efc64087e865
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2Writes.scala
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import java.util.UUID
+
+import org.apache.spark.SparkException
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.expressions.PredicateHelper
+import org.apache.spark.sql.catalyst.plans.logical.{AppendData, LogicalPlan, OverwriteByExpression, OverwritePartitionsDynamic}
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.connector.catalog.Table
+import org.apache.spark.sql.connector.write.{LogicalWriteInfoImpl, SupportsDynamicOverwrite, SupportsOverwrite, SupportsTruncate, WriteBuilder}
+import org.apache.spark.sql.execution.datasources.DataSourceStrategy
+import org.apache.spark.sql.sources.{AlwaysTrue, Filter}
+
+/**
+ * A rule that constructs logical writes.
+ */
+object V2Writes extends Rule[LogicalPlan] with PredicateHelper {
+
+  import DataSourceV2Implicits._
+
+  override def apply(plan: LogicalPlan): LogicalPlan = plan transformDown {
+    case a @ AppendData(r: DataSourceV2Relation, query, options, _, None) =>
+      val writeBuilder = newWriteBuilder(r.table, query, options)
+      val write = writeBuilder.build()
+      val newQuery = DistributionAndOrderingUtils.prepareQuery(write, query, conf)
+      a.copy(write = Some(write), query = newQuery)
+
+    case o @ OverwriteByExpression(r: DataSourceV2Relation, deleteExpr, query, options, _, None) =>
+      // fail if any filter cannot be converted. correctness depends on removing all matching data.
+      val filters = splitConjunctivePredicates(deleteExpr).flatMap { pred =>
+        val filter = DataSourceStrategy.translateFilter(pred, supportNestedPredicatePushdown = true)
+        if (filter.isEmpty) {
+          throw new AnalysisException(s"Cannot translate expression to source filter: $pred")
+        }
+        filter
+      }.toArray
+
+      val table = r.table
+      val writeBuilder = newWriteBuilder(table, query, options)
+      val write = writeBuilder match {
+        case builder: SupportsTruncate if isTruncate(filters) =>
+          builder.truncate().build()
+        case builder: SupportsOverwrite =>
+          builder.overwrite(filters).build()
+        case _ =>
+          throw new SparkException(s"Table does not support overwrite by expression: $table")
+      }
+
+      val newQuery = DistributionAndOrderingUtils.prepareQuery(write, query, conf)
+      o.copy(write = Some(write), query = newQuery)
+
+    case o @ OverwritePartitionsDynamic(r: DataSourceV2Relation, query, options, _, None) =>
+      val table = r.table
+      val writeBuilder = newWriteBuilder(table, query, options)
+      val write = writeBuilder match {
+        case builder: SupportsDynamicOverwrite =>
+          builder.overwriteDynamicPartitions().build()
+        case _ =>
+          throw new SparkException(s"Table does not support dynamic partition overwrite: $table")
+      }
+      val newQuery = DistributionAndOrderingUtils.prepareQuery(write, query, conf)
+      o.copy(write = Some(write), query = newQuery)
+  }
+
+  private def isTruncate(filters: Array[Filter]): Boolean = {
+    filters.length == 1 && filters(0).isInstanceOf[AlwaysTrue]
+  }
+
+  private def newWriteBuilder(
+      table: Table,
+      query: LogicalPlan,
+      writeOptions: Map[String, String]): WriteBuilder = {
+
+    val info = LogicalWriteInfoImpl(
+      queryId = UUID.randomUUID().toString,
+      query.schema,
+      writeOptions.asOptions)
+    table.asWritable.newWriteBuilder(info)
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala
index 616e18ee85a6b..7179eebbd4480 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala
@@ -22,19 +22,19 @@ import java.util.UUID
 import scala.collection.JavaConverters._
 import scala.util.control.NonFatal
 
-import org.apache.spark.{SparkEnv, SparkException, TaskContext}
-import org.apache.spark.executor.CommitDeniedException
+import org.apache.spark.{SparkEnv, TaskContext}
 import org.apache.spark.internal.Logging
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.{CannotReplaceMissingTableException, NoSuchTableException, TableAlreadyExistsException}
+import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
 import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.connector.catalog.{Identifier, StagedTable, StagingTableCatalog, SupportsWrite, TableCatalog}
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, UnaryNode}
+import org.apache.spark.sql.catalyst.util.CharVarcharUtils
+import org.apache.spark.sql.connector.catalog.{Identifier, StagedTable, StagingTableCatalog, SupportsWrite, Table, TableCatalog}
 import org.apache.spark.sql.connector.expressions.Transform
-import org.apache.spark.sql.connector.write.{BatchWrite, DataWriterFactory, LogicalWriteInfoImpl, PhysicalWriteInfoImpl, SupportsDynamicOverwrite, SupportsOverwrite, SupportsTruncate, V1WriteBuilder, WriteBuilder, WriterCommitMessage}
+import org.apache.spark.sql.connector.write.{BatchWrite, DataWriterFactory, LogicalWriteInfoImpl, PhysicalWriteInfoImpl, V1Write, Write, WriterCommitMessage}
+import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
 import org.apache.spark.sql.execution.{SparkPlan, UnaryExecNode}
-import org.apache.spark.sql.sources.{AlwaysTrue, Filter}
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 import org.apache.spark.util.{LongAccumulator, Utils}
 
@@ -43,10 +43,14 @@ import org.apache.spark.util.{LongAccumulator, Utils}
  * specific logical plans, like [[org.apache.spark.sql.catalyst.plans.logical.AppendData]].
  */
 @deprecated("Use specific logical plans like AppendData instead", "2.4.0")
-case class WriteToDataSourceV2(batchWrite: BatchWrite, query: LogicalPlan)
-  extends LogicalPlan {
-  override def children: Seq[LogicalPlan] = Seq(query)
+case class WriteToDataSourceV2(
+    relation: Option[DataSourceV2Relation],
+    batchWrite: BatchWrite,
+    query: LogicalPlan) extends UnaryNode {
+  override def child: LogicalPlan = query
   override def output: Seq[Attribute] = Nil
+  override protected def withNewChildInternal(newChild: LogicalPlan): WriteToDataSourceV2 =
+    copy(query = newChild)
 }
 
 /**
@@ -66,9 +70,7 @@ case class CreateTableAsSelectExec(
     query: SparkPlan,
     properties: Map[String, String],
     writeOptions: CaseInsensitiveStringMap,
-    ifNotExists: Boolean) extends V2TableWriteExec with SupportsV1Write {
-
-  import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.IdentifierHelper
+    ifNotExists: Boolean) extends TableWriteExecHelper {
 
   override protected def run(): Seq[InternalRow] = {
     if (catalog.tableExists(ident)) {
@@ -76,34 +78,17 @@ case class CreateTableAsSelectExec(
         return Nil
       }
 
-      throw new TableAlreadyExistsException(ident)
+      throw QueryCompilationErrors.tableAlreadyExistsError(ident)
     }
 
-    Utils.tryWithSafeFinallyAndFailureCallbacks({
-      val schema = query.schema.asNullable
-      catalog.createTable(
-        ident, schema, partitioning.toArray, properties.asJava) match {
-        case table: SupportsWrite =>
-          val info = LogicalWriteInfoImpl(
-            queryId = UUID.randomUUID().toString,
-            schema,
-            writeOptions)
-          val writeBuilder = table.newWriteBuilder(info)
-
-          writeBuilder match {
-            case v1: V1WriteBuilder => writeWithV1(v1.buildForV1Write())
-            case v2 => writeWithV2(v2.buildForBatch())
-          }
-
-        case _ =>
-          // table does not support writes
-          throw new SparkException(
-            s"Table implementation does not support writes: ${ident.quoted}")
-      }
-    })(catchBlock = {
-      catalog.dropTable(ident)
-    })
+    val schema = CharVarcharUtils.getRawSchema(query.schema).asNullable
+    val table = catalog.createTable(ident, schema,
+      partitioning.toArray, properties.asJava)
+    writeToTable(catalog, table, writeOptions, ident)
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): CreateTableAsSelectExec =
+    copy(query = newChild)
 }
 
 /**
@@ -123,7 +108,7 @@ case class AtomicCreateTableAsSelectExec(
     query: SparkPlan,
     properties: Map[String, String],
     writeOptions: CaseInsensitiveStringMap,
-    ifNotExists: Boolean) extends AtomicTableWriteExec {
+    ifNotExists: Boolean) extends TableWriteExecHelper {
 
   override protected def run(): Seq[InternalRow] = {
     if (catalog.tableExists(ident)) {
@@ -131,12 +116,16 @@ case class AtomicCreateTableAsSelectExec(
         return Nil
       }
 
-      throw new TableAlreadyExistsException(ident)
+      throw QueryCompilationErrors.tableAlreadyExistsError(ident)
     }
+    val schema = CharVarcharUtils.getRawSchema(query.schema).asNullable
     val stagedTable = catalog.stageCreate(
-      ident, query.schema.asNullable, partitioning.toArray, properties.asJava)
-    writeToStagedTable(stagedTable, writeOptions, ident)
+      ident, schema, partitioning.toArray, properties.asJava)
+    writeToTable(catalog, stagedTable, writeOptions, ident)
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): AtomicCreateTableAsSelectExec =
+    copy(query = newChild)
 }
 
 /**
@@ -157,9 +146,8 @@ case class ReplaceTableAsSelectExec(
     query: SparkPlan,
     properties: Map[String, String],
     writeOptions: CaseInsensitiveStringMap,
-    orCreate: Boolean) extends V2TableWriteExec with SupportsV1Write {
-
-  import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.IdentifierHelper
+    orCreate: Boolean,
+    invalidateCache: (TableCatalog, Table, Identifier) => Unit) extends TableWriteExecHelper {
 
   override protected def run(): Seq[InternalRow] = {
     // Note that this operation is potentially unsafe, but these are the strict semantics of
@@ -171,36 +159,20 @@ case class ReplaceTableAsSelectExec(
     // 2. Writing to the new table fails,
     // 3. The table returned by catalog.createTable doesn't support writing.
     if (catalog.tableExists(ident)) {
+      val table = catalog.loadTable(ident)
+      invalidateCache(catalog, table, ident)
       catalog.dropTable(ident)
     } else if (!orCreate) {
-      throw new CannotReplaceMissingTableException(ident)
+      throw QueryCompilationErrors.cannotReplaceMissingTableError(ident)
     }
-    val schema = query.schema.asNullable
-    val createdTable = catalog.createTable(
+    val schema = CharVarcharUtils.getRawSchema(query.schema).asNullable
+    val table = catalog.createTable(
       ident, schema, partitioning.toArray, properties.asJava)
-    Utils.tryWithSafeFinallyAndFailureCallbacks({
-      createdTable match {
-        case table: SupportsWrite =>
-          val info = LogicalWriteInfoImpl(
-            queryId = UUID.randomUUID().toString,
-            schema,
-            writeOptions)
-          val writeBuilder = table.newWriteBuilder(info)
-
-          writeBuilder match {
-            case v1: V1WriteBuilder => writeWithV1(v1.buildForV1Write())
-            case v2 => writeWithV2(v2.buildForBatch())
-          }
-
-        case _ =>
-          // table does not support writes
-          throw new SparkException(
-            s"Table implementation does not support writes: ${ident.quoted}")
-      }
-    })(catchBlock = {
-      catalog.dropTable(ident)
-    })
+    writeToTable(catalog, table, writeOptions, ident)
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): ReplaceTableAsSelectExec =
+    copy(query = newChild)
 }
 
 /**
@@ -211,7 +183,7 @@ case class ReplaceTableAsSelectExec(
  * A new table will be created using the schema of the query, and rows from the query are appended.
  * If the table exists, its contents and schema should be replaced with the schema and the contents
  * of the query. This implementation is atomic. The table replacement is staged, and the commit
- * operation at the end should perform tne replacement of the table's metadata and contents. If the
+ * operation at the end should perform the replacement of the table's metadata and contents. If the
  * write fails, the table is instructed to roll back staged changes and any previously written table
  * is left untouched.
  */
@@ -223,10 +195,15 @@ case class AtomicReplaceTableAsSelectExec(
     query: SparkPlan,
     properties: Map[String, String],
     writeOptions: CaseInsensitiveStringMap,
-    orCreate: Boolean) extends AtomicTableWriteExec {
+    orCreate: Boolean,
+    invalidateCache: (TableCatalog, Table, Identifier) => Unit) extends TableWriteExecHelper {
 
   override protected def run(): Seq[InternalRow] = {
-    val schema = query.schema.asNullable
+    val schema = CharVarcharUtils.getRawSchema(query.schema).asNullable
+    if (catalog.tableExists(ident)) {
+      val table = catalog.loadTable(ident)
+      invalidateCache(catalog, table, ident)
+    }
     val staged = if (orCreate) {
       catalog.stageCreateOrReplace(
         ident, schema, partitioning.toArray, properties.asJava)
@@ -236,13 +213,16 @@ case class AtomicReplaceTableAsSelectExec(
           ident, schema, partitioning.toArray, properties.asJava)
       } catch {
         case e: NoSuchTableException =>
-          throw new CannotReplaceMissingTableException(ident, Some(e))
+          throw QueryCompilationErrors.cannotReplaceMissingTableError(ident, Some(e))
       }
     } else {
-      throw new CannotReplaceMissingTableException(ident)
+      throw QueryCompilationErrors.cannotReplaceMissingTableError(ident)
     }
-    writeToStagedTable(staged, writeOptions, ident)
+    writeToTable(catalog, staged, writeOptions, ident)
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): AtomicReplaceTableAsSelectExec =
+    copy(query = newChild)
 }
 
 /**
@@ -251,13 +231,11 @@ case class AtomicReplaceTableAsSelectExec(
  * Rows in the output data set are appended.
  */
 case class AppendDataExec(
-    table: SupportsWrite,
-    writeOptions: CaseInsensitiveStringMap,
-    query: SparkPlan) extends V2TableWriteExec with BatchWriteHelper {
-
-  override protected def run(): Seq[InternalRow] = {
-    writeWithV2(newWriteBuilder().buildForBatch())
-  }
+    query: SparkPlan,
+    refreshCache: () => Unit,
+    write: Write) extends V2ExistingTableWriteExec {
+  override protected def withNewChildInternal(newChild: SparkPlan): AppendDataExec =
+    copy(query = newChild)
 }
 
 /**
@@ -271,27 +249,11 @@ case class AppendDataExec(
  * AlwaysTrue to delete all rows.
  */
 case class OverwriteByExpressionExec(
-    table: SupportsWrite,
-    deleteWhere: Array[Filter],
-    writeOptions: CaseInsensitiveStringMap,
-    query: SparkPlan) extends V2TableWriteExec with BatchWriteHelper {
-
-  private def isTruncate(filters: Array[Filter]): Boolean = {
-    filters.length == 1 && filters(0).isInstanceOf[AlwaysTrue]
-  }
-
-  override protected def run(): Seq[InternalRow] = {
-    newWriteBuilder() match {
-      case builder: SupportsTruncate if isTruncate(deleteWhere) =>
-        writeWithV2(builder.truncate().buildForBatch())
-
-      case builder: SupportsOverwrite =>
-        writeWithV2(builder.overwrite(deleteWhere).buildForBatch())
-
-      case _ =>
-        throw new SparkException(s"Table does not support overwrite by expression: $table")
-    }
-  }
+    query: SparkPlan,
+    refreshCache: () => Unit,
+    write: Write) extends V2ExistingTableWriteExec {
+  override protected def withNewChildInternal(newChild: SparkPlan): OverwriteByExpressionExec =
+    copy(query = newChild)
 }
 
 /**
@@ -304,46 +266,36 @@ case class OverwriteByExpressionExec(
  * are not modified.
  */
 case class OverwritePartitionsDynamicExec(
-    table: SupportsWrite,
-    writeOptions: CaseInsensitiveStringMap,
-    query: SparkPlan) extends V2TableWriteExec with BatchWriteHelper {
-
-  override protected def run(): Seq[InternalRow] = {
-    newWriteBuilder() match {
-      case builder: SupportsDynamicOverwrite =>
-        writeWithV2(builder.overwriteDynamicPartitions().buildForBatch())
-
-      case _ =>
-        throw new SparkException(s"Table does not support dynamic partition overwrite: $table")
-    }
-  }
+    query: SparkPlan,
+    refreshCache: () => Unit,
+    write: Write) extends V2ExistingTableWriteExec {
+  override protected def withNewChildInternal(newChild: SparkPlan): OverwritePartitionsDynamicExec =
+    copy(query = newChild)
 }
 
 case class WriteToDataSourceV2Exec(
     batchWrite: BatchWrite,
+    refreshCache: () => Unit,
     query: SparkPlan) extends V2TableWriteExec {
 
-  def writeOptions: CaseInsensitiveStringMap = CaseInsensitiveStringMap.empty()
-
   override protected def run(): Seq[InternalRow] = {
-    writeWithV2(batchWrite)
+    val writtenRows = writeWithV2(batchWrite)
+    refreshCache()
+    writtenRows
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): WriteToDataSourceV2Exec =
+    copy(query = newChild)
 }
 
-/**
- * Helper for physical plans that build batch writes.
- */
-trait BatchWriteHelper {
-  def table: SupportsWrite
-  def query: SparkPlan
-  def writeOptions: CaseInsensitiveStringMap
-
-  def newWriteBuilder(): WriteBuilder = {
-    val info = LogicalWriteInfoImpl(
-      queryId = UUID.randomUUID().toString,
-      query.schema,
-      writeOptions)
-    table.newWriteBuilder(info)
+trait V2ExistingTableWriteExec extends V2TableWriteExec {
+  def refreshCache: () => Unit
+  def write: Write
+
+  override protected def run(): Seq[InternalRow] = {
+    val writtenRows = writeWithV2(write.toBatch)
+    refreshCache()
+    writtenRows
   }
 }
 
@@ -405,12 +357,12 @@ trait V2TableWriteExec extends V2CommandExec with UnaryExecNode {
           case t: Throwable =>
             logError(s"Data source write support $batchWrite failed to abort.")
             cause.addSuppressed(t)
-            throw new SparkException("Writing job failed.", cause)
+            throw QueryExecutionErrors.writingJobFailedError(cause)
         }
         logError(s"Data source write support $batchWrite aborted.")
         cause match {
           // Only wrap non fatal exceptions.
-          case NonFatal(e) => throw new SparkException("Writing job aborted.", e)
+          case NonFatal(e) => throw QueryExecutionErrors.writingJobAbortedError(e)
           case _ => throw cause
         }
     }
@@ -449,11 +401,11 @@ object DataWritingSparkTask extends Logging {
             s"stage $stageId.$stageAttempt)")
           dataWriter.commit()
         } else {
-          val message = s"Commit denied for partition $partId (task $taskId, attempt $attemptId, " +
-            s"stage $stageId.$stageAttempt)"
-          logInfo(message)
+          val commitDeniedException = QueryExecutionErrors.commitDeniedError(
+            partId, taskId, attemptId, stageId, stageAttempt)
+          logInfo(commitDeniedException.getMessage)
           // throwing CommitDeniedException will trigger the catch block for abort
-          throw new CommitDeniedException(message, stageId, partId, attemptId)
+          throw commitDeniedException
         }
 
       } else {
@@ -479,15 +431,14 @@ object DataWritingSparkTask extends Logging {
   }
 }
 
-private[v2] trait AtomicTableWriteExec extends V2TableWriteExec with SupportsV1Write {
-  import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.IdentifierHelper
-
-  protected def writeToStagedTable(
-      stagedTable: StagedTable,
+private[v2] trait TableWriteExecHelper extends V2TableWriteExec with SupportsV1Write {
+  protected def writeToTable(
+      catalog: TableCatalog,
+      table: Table,
       writeOptions: CaseInsensitiveStringMap,
       ident: Identifier): Seq[InternalRow] = {
     Utils.tryWithSafeFinallyAndFailureCallbacks({
-      stagedTable match {
+      table match {
         case table: SupportsWrite =>
           val info = LogicalWriteInfoImpl(
             queryId = UUID.randomUUID().toString,
@@ -495,21 +446,29 @@ private[v2] trait AtomicTableWriteExec extends V2TableWriteExec with SupportsV1W
             writeOptions)
           val writeBuilder = table.newWriteBuilder(info)
 
-          val writtenRows = writeBuilder match {
-            case v1: V1WriteBuilder => writeWithV1(v1.buildForV1Write())
-            case v2 => writeWithV2(v2.buildForBatch())
+          val write = writeBuilder.build()
+          val writtenRows = write match {
+            case v1: V1Write => writeWithV1(v1.toInsertableRelation)
+            case v2 => writeWithV2(v2.toBatch)
+          }
+
+          table match {
+            case st: StagedTable => st.commitStagedChanges()
+            case _ =>
           }
-          stagedTable.commitStagedChanges()
           writtenRows
 
         case _ =>
-          // Table does not support writes - staged changes are also rolled back below.
-          throw new SparkException(
-            s"Table implementation does not support writes: ${ident.quoted}")
+          // Table does not support writes - staged changes are also rolled back below if table
+          // is staging.
+          throw QueryExecutionErrors.unsupportedTableWritesError(ident)
       }
     })(catchBlock = {
-      // Failure rolls back the staged writes and metadata changes.
-      stagedTable.abortStagedChanges()
+      table match {
+        // Failure rolls back the staged writes and metadata changes.
+        case st: StagedTable => st.abortStagedChanges()
+        case _ => catalog.dropTable(ident)
+      }
     })
   }
 }
@@ -522,3 +481,4 @@ private[v2] case class DataWritingSparkTaskResult(
  * Sink progress information collected after commit.
  */
 private[sql] case class StreamWriterCommitProgress(numOutputRows: Long)
+
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVDataSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVDataSourceV2.scala
index 1f99d4282f6da..c577cbf8d756a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVDataSourceV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVDataSourceV2.scala
@@ -31,13 +31,15 @@ class CSVDataSourceV2 extends FileDataSourceV2 {
 
   override def getTable(options: CaseInsensitiveStringMap): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
-    CSVTable(tableName, sparkSession, options, paths, None, fallbackFileFormat)
+    val tableName = getTableName(options, paths)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    CSVTable(tableName, sparkSession, optionsWithoutPaths, paths, None, fallbackFileFormat)
   }
 
   override def getTable(options: CaseInsensitiveStringMap, schema: StructType): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
-    CSVTable(tableName, sparkSession, options, paths, Some(schema), fallbackFileFormat)
+    val tableName = getTableName(options, paths)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    CSVTable(tableName, sparkSession, optionsWithoutPaths, paths, Some(schema), fallbackFileFormat)
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVScan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVScan.scala
index 4f510322815ef..43d2b8a97245a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVScan.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVScan.scala
@@ -107,4 +107,8 @@ case class CSVScan(
   override def description(): String = {
     super.description() + ", PushedFilters: " + pushedFilters.mkString("[", ", ", "]")
   }
+
+  override def getMetaData(): Map[String, String] = {
+    super.getMetaData() ++ Map("PushedFilters" -> seqToString(pushedFilters))
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVScanBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVScanBuilder.scala
index 81a234e254000..f7a79bf31948e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVScanBuilder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVScanBuilder.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.execution.datasources.v2.csv
 
 import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.csv.CSVFilters
+import org.apache.spark.sql.catalyst.StructFilters
 import org.apache.spark.sql.connector.read.{Scan, SupportsPushDownFilters}
 import org.apache.spark.sql.execution.datasources.PartitioningAwareFileIndex
 import org.apache.spark.sql.execution.datasources.v2.FileScanBuilder
@@ -49,7 +49,7 @@ case class CSVScanBuilder(
 
   override def pushFilters(filters: Array[Filter]): Array[Filter] = {
     if (sparkSession.sessionState.conf.csvFilterPushDown) {
-      _pushedFilters = CSVFilters.pushedFilters(filters, dataSchema)
+      _pushedFilters = StructFilters.pushedFilters(filters, dataSchema)
     }
     filters
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVTable.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVTable.scala
index 3cafe37b743f3..839cd01be75a5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVTable.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVTable.scala
@@ -22,7 +22,7 @@ import org.apache.hadoop.fs.FileStatus
 
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.csv.CSVOptions
-import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder}
+import org.apache.spark.sql.connector.write.{LogicalWriteInfo, Write, WriteBuilder}
 import org.apache.spark.sql.execution.datasources.FileFormat
 import org.apache.spark.sql.execution.datasources.csv.CSVDataSource
 import org.apache.spark.sql.execution.datasources.v2.FileTable
@@ -50,7 +50,9 @@ case class CSVTable(
   }
 
   override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder =
-    new CSVWriteBuilder(paths, formatName, supportsDataType, info)
+    new WriteBuilder {
+      override def build(): Write = CSVWrite(paths, formatName, supportsDataType, info)
+    }
 
   override def supportsDataType(dataType: DataType): Boolean = dataType match {
     case _: AtomicType => true
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVWrite.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVWrite.scala
new file mode 100644
index 0000000000000..3a1848f544c45
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVWrite.scala
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2.csv
+
+import org.apache.hadoop.mapreduce.{Job, TaskAttemptContext}
+
+import org.apache.spark.sql.catalyst.csv.CSVOptions
+import org.apache.spark.sql.catalyst.util.CompressionCodecs
+import org.apache.spark.sql.connector.write.LogicalWriteInfo
+import org.apache.spark.sql.execution.datasources.{CodecStreams, OutputWriter, OutputWriterFactory}
+import org.apache.spark.sql.execution.datasources.csv.CsvOutputWriter
+import org.apache.spark.sql.execution.datasources.v2.FileWrite
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types.{DataType, StructType}
+
+case class CSVWrite(
+    paths: Seq[String],
+    formatName: String,
+    supportsDataType: DataType => Boolean,
+    info: LogicalWriteInfo) extends FileWrite {
+  override def prepareWrite(
+      sqlConf: SQLConf,
+      job: Job,
+      options: Map[String, String],
+      dataSchema: StructType): OutputWriterFactory = {
+    val conf = job.getConfiguration
+    val csvOptions = new CSVOptions(
+      options,
+      columnPruning = sqlConf.csvColumnPruning,
+      sqlConf.sessionLocalTimeZone)
+    csvOptions.compressionCodec.foreach { codec =>
+      CompressionCodecs.setCodecConfiguration(conf, codec)
+    }
+
+    new OutputWriterFactory {
+      override def newInstance(
+          path: String,
+          dataSchema: StructType,
+          context: TaskAttemptContext): OutputWriter = {
+        new CsvOutputWriter(path, dataSchema, context, csvOptions)
+      }
+
+      override def getFileExtension(context: TaskAttemptContext): String = {
+        ".csv" + CodecStreams.getCompressionExtension(context)
+      }
+    }
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVWriteBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVWriteBuilder.scala
deleted file mode 100644
index bfbb1831aa63d..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVWriteBuilder.scala
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.execution.datasources.v2.csv
-
-import org.apache.hadoop.mapreduce.{Job, TaskAttemptContext}
-
-import org.apache.spark.sql.catalyst.csv.CSVOptions
-import org.apache.spark.sql.catalyst.util.CompressionCodecs
-import org.apache.spark.sql.connector.write.LogicalWriteInfo
-import org.apache.spark.sql.execution.datasources.{CodecStreams, OutputWriter, OutputWriterFactory}
-import org.apache.spark.sql.execution.datasources.csv.CsvOutputWriter
-import org.apache.spark.sql.execution.datasources.v2.FileWriteBuilder
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{DataType, StructType}
-
-class CSVWriteBuilder(
-    paths: Seq[String],
-    formatName: String,
-    supportsDataType: DataType => Boolean,
-    info: LogicalWriteInfo)
-  extends FileWriteBuilder(paths, formatName, supportsDataType, info) {
-  override def prepareWrite(
-      sqlConf: SQLConf,
-      job: Job,
-      options: Map[String, String],
-      dataSchema: StructType): OutputWriterFactory = {
-    val conf = job.getConfiguration
-    val csvOptions = new CSVOptions(
-      options,
-      columnPruning = sqlConf.csvColumnPruning,
-      sqlConf.sessionLocalTimeZone)
-    csvOptions.compressionCodec.foreach { codec =>
-      CompressionCodecs.setCodecConfiguration(conf, codec)
-    }
-
-    new OutputWriterFactory {
-      override def newInstance(
-          path: String,
-          dataSchema: StructType,
-          context: TaskAttemptContext): OutputWriter = {
-        new CsvOutputWriter(path, dataSchema, context, csvOptions)
-      }
-
-      override def getFileExtension(context: TaskAttemptContext): String = {
-        ".csv" + CodecStreams.getCompressionExtension(context)
-      }
-    }
-  }
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCScan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCScan.scala
new file mode 100644
index 0000000000000..860232ba84f39
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCScan.scala
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2.jdbc
+
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{Row, SQLContext}
+import org.apache.spark.sql.connector.read.V1Scan
+import org.apache.spark.sql.execution.datasources.jdbc.JDBCRelation
+import org.apache.spark.sql.sources.{BaseRelation, Filter, TableScan}
+import org.apache.spark.sql.types.StructType
+
+case class JDBCScan(
+    relation: JDBCRelation,
+    prunedSchema: StructType,
+    pushedFilters: Array[Filter]) extends V1Scan {
+
+  override def readSchema(): StructType = prunedSchema
+
+  override def toV1TableScan[T <: BaseRelation with TableScan](context: SQLContext): T = {
+    new BaseRelation with TableScan {
+      override def sqlContext: SQLContext = context
+      override def schema: StructType = prunedSchema
+      override def needConversion: Boolean = relation.needConversion
+      override def buildScan(): RDD[Row] = {
+        relation.buildScan(prunedSchema.map(_.name).toArray, pushedFilters)
+      }
+    }.asInstanceOf[T]
+  }
+
+  override def description(): String = {
+    super.description()  + ", prunedSchema: " + seqToString(prunedSchema) +
+      ", PushedFilters: " + seqToString(pushedFilters)
+  }
+
+  private def seqToString(seq: Seq[Any]): String = seq.mkString("[", ", ", "]")
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCScanBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCScanBuilder.scala
new file mode 100644
index 0000000000000..270c5b6d92e32
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCScanBuilder.scala
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2.jdbc
+
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.connector.read.{Scan, ScanBuilder, SupportsPushDownFilters, SupportsPushDownRequiredColumns}
+import org.apache.spark.sql.execution.datasources.PartitioningUtils
+import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions, JDBCRDD, JDBCRelation}
+import org.apache.spark.sql.jdbc.JdbcDialects
+import org.apache.spark.sql.sources.Filter
+import org.apache.spark.sql.types.StructType
+
+case class JDBCScanBuilder(
+    session: SparkSession,
+    schema: StructType,
+    jdbcOptions: JDBCOptions)
+  extends ScanBuilder with SupportsPushDownFilters with SupportsPushDownRequiredColumns {
+
+  private val isCaseSensitive = session.sessionState.conf.caseSensitiveAnalysis
+
+  private var pushedFilter = Array.empty[Filter]
+
+  private var prunedSchema = schema
+
+  override def pushFilters(filters: Array[Filter]): Array[Filter] = {
+    if (jdbcOptions.pushDownPredicate) {
+      val dialect = JdbcDialects.get(jdbcOptions.url)
+      val (pushed, unSupported) = filters.partition(JDBCRDD.compileFilter(_, dialect).isDefined)
+      this.pushedFilter = pushed
+      unSupported
+    } else {
+      filters
+    }
+  }
+
+  override def pushedFilters(): Array[Filter] = pushedFilter
+
+  override def pruneColumns(requiredSchema: StructType): Unit = {
+    // JDBC doesn't support nested column pruning.
+    // TODO (SPARK-32593): JDBC support nested column and nested column pruning.
+    val requiredCols = requiredSchema.fields.map(PartitioningUtils.getColName(_, isCaseSensitive))
+      .toSet
+    val fields = schema.fields.filter { field =>
+      val colName = PartitioningUtils.getColName(field, isCaseSensitive)
+      requiredCols.contains(colName)
+    }
+    prunedSchema = StructType(fields)
+  }
+
+  override def build(): Scan = {
+    val resolver = session.sessionState.conf.resolver
+    val timeZoneId = session.sessionState.conf.sessionLocalTimeZone
+    val parts = JDBCRelation.columnPartition(schema, resolver, timeZoneId, jdbcOptions)
+    JDBCScan(JDBCRelation(schema, parts, jdbcOptions)(session), prunedSchema, pushedFilter)
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTable.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTable.scala
new file mode 100644
index 0000000000000..5e11ea66be4c6
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTable.scala
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2.jdbc
+
+import java.util
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.connector.catalog._
+import org.apache.spark.sql.connector.catalog.TableCapability._
+import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder}
+import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions, JdbcOptionsInWrite}
+import org.apache.spark.sql.types.StructType
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
+
+case class JDBCTable(ident: Identifier, schema: StructType, jdbcOptions: JDBCOptions)
+  extends Table with SupportsRead with SupportsWrite {
+
+  override def name(): String = ident.toString
+
+  override def capabilities(): util.Set[TableCapability] = {
+    Set(BATCH_READ, V1_BATCH_WRITE, TRUNCATE).asJava
+  }
+
+  override def newScanBuilder(options: CaseInsensitiveStringMap): JDBCScanBuilder = {
+    val mergedOptions = new JDBCOptions(
+      jdbcOptions.parameters.originalMap ++ options.asCaseSensitiveMap().asScala)
+    JDBCScanBuilder(SparkSession.active, schema, mergedOptions)
+  }
+
+  override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
+    val mergedOptions = new JdbcOptionsInWrite(
+      jdbcOptions.parameters.originalMap ++ info.options.asCaseSensitiveMap().asScala)
+    JDBCWriteBuilder(schema, mergedOptions)
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalog.scala
new file mode 100644
index 0000000000000..a90ab564ddb50
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalog.scala
@@ -0,0 +1,324 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2.jdbc
+
+import java.sql.{Connection, SQLException}
+import java.util
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+import scala.collection.mutable.ArrayBuilder
+
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.connector.catalog.{Identifier, NamespaceChange, SupportsNamespaces, Table, TableCatalog, TableChange}
+import org.apache.spark.sql.connector.expressions.Transform
+import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
+import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions, JdbcOptionsInWrite, JDBCRDD, JdbcUtils}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.jdbc.{JdbcDialect, JdbcDialects}
+import org.apache.spark.sql.types.StructType
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
+
+class JDBCTableCatalog extends TableCatalog with SupportsNamespaces with Logging {
+  private var catalogName: String = null
+  private var options: JDBCOptions = _
+  private var dialect: JdbcDialect = _
+
+  override def name(): String = {
+    require(catalogName != null, "The JDBC table catalog is not initialed")
+    catalogName
+  }
+
+  override def initialize(name: String, options: CaseInsensitiveStringMap): Unit = {
+    assert(catalogName == null, "The JDBC table catalog is already initialed")
+    catalogName = name
+
+    val map = options.asCaseSensitiveMap().asScala.toMap
+    // The `JDBCOptions` checks the existence of the table option. This is required by JDBC v1, but
+    // JDBC V2 only knows the table option when loading a table. Here we put a table option with a
+    // fake value, so that it can pass the check of `JDBCOptions`.
+    this.options = new JDBCOptions(map + (JDBCOptions.JDBC_TABLE_NAME -> "__invalid_dbtable"))
+    dialect = JdbcDialects.get(this.options.url)
+  }
+
+  override def listTables(namespace: Array[String]): Array[Identifier] = {
+    checkNamespace(namespace)
+    withConnection { conn =>
+      val schemaPattern = if (namespace.length == 1) namespace.head else null
+      val rs = conn.getMetaData
+        .getTables(null, schemaPattern, "%", Array("TABLE"));
+      new Iterator[Identifier] {
+        def hasNext = rs.next()
+        def next() = Identifier.of(namespace, rs.getString("TABLE_NAME"))
+      }.toArray
+    }
+  }
+
+  override def tableExists(ident: Identifier): Boolean = {
+    checkNamespace(ident.namespace())
+    val writeOptions = new JdbcOptionsInWrite(
+      options.parameters + (JDBCOptions.JDBC_TABLE_NAME -> getTableName(ident)))
+    classifyException(s"Failed table existence check: $ident") {
+      withConnection(JdbcUtils.tableExists(_, writeOptions))
+    }
+  }
+
+  override def dropTable(ident: Identifier): Boolean = {
+    checkNamespace(ident.namespace())
+    withConnection { conn =>
+      try {
+        JdbcUtils.dropTable(conn, getTableName(ident), options)
+        true
+      } catch {
+        case _: SQLException => false
+      }
+    }
+  }
+
+  override def renameTable(oldIdent: Identifier, newIdent: Identifier): Unit = {
+    checkNamespace(oldIdent.namespace())
+    withConnection { conn =>
+      classifyException(s"Failed table renaming from $oldIdent to $newIdent") {
+        JdbcUtils.renameTable(conn, getTableName(oldIdent), getTableName(newIdent), options)
+      }
+    }
+  }
+
+  override def loadTable(ident: Identifier): Table = {
+    checkNamespace(ident.namespace())
+    val optionsWithTableName = new JDBCOptions(
+      options.parameters + (JDBCOptions.JDBC_TABLE_NAME -> getTableName(ident)))
+    try {
+      val schema = JDBCRDD.resolveTable(optionsWithTableName)
+      JDBCTable(ident, schema, optionsWithTableName)
+    } catch {
+      case _: SQLException => throw QueryCompilationErrors.noSuchTableError(ident)
+    }
+  }
+
+  override def createTable(
+      ident: Identifier,
+      schema: StructType,
+      partitions: Array[Transform],
+      properties: java.util.Map[String, String]): Table = {
+    checkNamespace(ident.namespace())
+    if (partitions.nonEmpty) {
+      throw QueryExecutionErrors.cannotCreateJDBCTableWithPartitionsError()
+    }
+
+    var tableOptions = options.parameters + (JDBCOptions.JDBC_TABLE_NAME -> getTableName(ident))
+    var tableComment: String = ""
+    var tableProperties: String = ""
+    if (!properties.isEmpty) {
+      properties.asScala.foreach {
+        case (k, v) => k match {
+          case TableCatalog.PROP_COMMENT => tableComment = v
+          case TableCatalog.PROP_PROVIDER =>
+            throw QueryCompilationErrors.cannotCreateJDBCTableUsingProviderError()
+          case TableCatalog.PROP_OWNER => // owner is ignored. It is default to current user name.
+          case TableCatalog.PROP_LOCATION =>
+            throw QueryCompilationErrors.cannotCreateJDBCTableUsingLocationError()
+          case _ => tableProperties = tableProperties + " " + s"$k $v"
+        }
+      }
+    }
+
+    if (tableComment != "") {
+      tableOptions = tableOptions + (JDBCOptions.JDBC_TABLE_COMMENT -> tableComment)
+    }
+    if (tableProperties != "") {
+      // table property is set in JDBC_CREATE_TABLE_OPTIONS, which will be appended
+      // to CREATE TABLE statement.
+      // E.g., "CREATE TABLE t (name string) ENGINE InnoDB DEFAULT CHARACTER SET utf8"
+      // Spark doesn't check if these table properties are supported by databases. If
+      // table property is invalid, database will fail the table creation.
+      tableOptions = tableOptions + (JDBCOptions.JDBC_CREATE_TABLE_OPTIONS -> tableProperties)
+    }
+
+    val writeOptions = new JdbcOptionsInWrite(tableOptions)
+    val caseSensitive = SQLConf.get.caseSensitiveAnalysis
+    withConnection { conn =>
+      classifyException(s"Failed table creation: $ident") {
+        JdbcUtils.createTable(conn, getTableName(ident), schema, caseSensitive, writeOptions)
+      }
+    }
+
+    JDBCTable(ident, schema, writeOptions)
+  }
+
+  override def alterTable(ident: Identifier, changes: TableChange*): Table = {
+    checkNamespace(ident.namespace())
+    withConnection { conn =>
+      classifyException(s"Failed table altering: $ident") {
+        JdbcUtils.alterTable(conn, getTableName(ident), changes, options)
+      }
+      loadTable(ident)
+    }
+  }
+
+  override def namespaceExists(namespace: Array[String]): Boolean = namespace match {
+    case Array(db) =>
+      withConnection { conn =>
+        val rs = conn.getMetaData.getSchemas(null, db)
+        while (rs.next()) {
+          if (rs.getString(1) == db) return true;
+        }
+        false
+      }
+    case _ => false
+  }
+
+  override def listNamespaces(): Array[Array[String]] = {
+    withConnection { conn =>
+      val schemaBuilder = ArrayBuilder.make[Array[String]]
+      val rs = conn.getMetaData.getSchemas()
+      while (rs.next()) {
+        schemaBuilder += Array(rs.getString(1))
+      }
+      schemaBuilder.result
+    }
+  }
+
+  override def listNamespaces(namespace: Array[String]): Array[Array[String]] = {
+    namespace match {
+      case Array() =>
+        listNamespaces()
+      case Array(_) if namespaceExists(namespace) =>
+        Array()
+      case _ =>
+        throw QueryCompilationErrors.noSuchNamespaceError(namespace)
+    }
+  }
+
+  override def loadNamespaceMetadata(namespace: Array[String]): util.Map[String, String] = {
+    namespace match {
+      case Array(db) =>
+        if (!namespaceExists(namespace)) {
+          throw QueryCompilationErrors.noSuchNamespaceError(Array(db))
+        }
+        mutable.HashMap[String, String]().asJava
+
+      case _ =>
+        throw QueryCompilationErrors.noSuchNamespaceError(namespace)
+    }
+  }
+
+  override def createNamespace(
+      namespace: Array[String],
+      metadata: util.Map[String, String]): Unit = namespace match {
+    case Array(db) if !namespaceExists(namespace) =>
+      var comment = ""
+      if (!metadata.isEmpty) {
+        metadata.asScala.foreach {
+          case (k, v) => k match {
+            case SupportsNamespaces.PROP_COMMENT => comment = v
+            case SupportsNamespaces.PROP_OWNER => // ignore
+            case SupportsNamespaces.PROP_LOCATION =>
+              throw QueryCompilationErrors.cannotCreateJDBCNamespaceUsingProviderError()
+            case _ =>
+              throw QueryCompilationErrors.cannotCreateJDBCNamespaceWithPropertyError(k)
+          }
+        }
+      }
+      withConnection { conn =>
+        classifyException(s"Failed create name space: $db") {
+          JdbcUtils.createNamespace(conn, options, db, comment)
+        }
+      }
+
+    case Array(_) =>
+      throw QueryCompilationErrors.namespaceAlreadyExistsError(namespace)
+
+    case _ =>
+      throw QueryExecutionErrors.invalidNamespaceNameError(namespace)
+  }
+
+  override def alterNamespace(namespace: Array[String], changes: NamespaceChange*): Unit = {
+    namespace match {
+      case Array(db) =>
+        changes.foreach {
+          case set: NamespaceChange.SetProperty =>
+            if (set.property() == SupportsNamespaces.PROP_COMMENT) {
+              withConnection { conn =>
+                JdbcUtils.createNamespaceComment(conn, options, db, set.value)
+              }
+            } else {
+              throw QueryCompilationErrors.cannotSetJDBCNamespaceWithPropertyError(set.property)
+            }
+
+          case unset: NamespaceChange.RemoveProperty =>
+            if (unset.property() == SupportsNamespaces.PROP_COMMENT) {
+              withConnection { conn =>
+                JdbcUtils.removeNamespaceComment(conn, options, db)
+              }
+            } else {
+              throw QueryCompilationErrors.cannotUnsetJDBCNamespaceWithPropertyError(unset.property)
+            }
+
+          case _ =>
+            throw QueryCompilationErrors.unsupportedJDBCNamespaceChangeInCatalogError(changes)
+        }
+
+      case _ =>
+        throw QueryCompilationErrors.noSuchNamespaceError(namespace)
+    }
+  }
+
+  override def dropNamespace(namespace: Array[String]): Boolean = namespace match {
+    case Array(db) if namespaceExists(namespace) =>
+      if (listTables(Array(db)).nonEmpty) {
+        throw QueryExecutionErrors.namespaceNotEmptyError(namespace)
+      }
+      withConnection { conn =>
+        classifyException(s"Failed drop name space: $db") {
+          JdbcUtils.dropNamespace(conn, options, db)
+          true
+        }
+      }
+
+    case _ =>
+      throw QueryCompilationErrors.noSuchNamespaceError(namespace)
+  }
+
+  private def checkNamespace(namespace: Array[String]): Unit = {
+    // In JDBC there is no nested database/schema
+    if (namespace.length > 1) {
+      throw QueryCompilationErrors.noSuchNamespaceError(namespace)
+    }
+  }
+
+  private def withConnection[T](f: Connection => T): T = {
+    val conn = JdbcUtils.createConnectionFactory(options)()
+    try {
+      f(conn)
+    } finally {
+      conn.close()
+    }
+  }
+
+  private def getTableName(ident: Identifier): String = {
+    (ident.namespace() :+ ident.name()).map(dialect.quoteIdentifier).mkString(".")
+  }
+
+  private def classifyException[T](message: String)(f: => T): T = {
+    try {
+      f
+    } catch {
+      case e: Throwable => throw dialect.classifyException(message, e)
+    }
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCWriteBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCWriteBuilder.scala
new file mode 100644
index 0000000000000..0e6c72c2cc331
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCWriteBuilder.scala
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2.jdbc
+
+import org.apache.spark.sql._
+import org.apache.spark.sql.connector.write._
+import org.apache.spark.sql.execution.datasources.jdbc.{JdbcOptionsInWrite, JdbcUtils}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.sources.InsertableRelation
+import org.apache.spark.sql.types.StructType
+
+case class JDBCWriteBuilder(schema: StructType, options: JdbcOptionsInWrite) extends WriteBuilder
+  with SupportsTruncate {
+
+  private var isTruncate = false
+
+  override def truncate(): WriteBuilder = {
+    isTruncate = true
+    this
+  }
+
+  override def build(): V1Write = new V1Write {
+    override def toInsertableRelation: InsertableRelation = (data: DataFrame, _: Boolean) => {
+      // TODO (SPARK-32595): do truncate and append atomically.
+      if (isTruncate) {
+        val conn = JdbcUtils.createConnectionFactory(options)()
+        JdbcUtils.truncateTable(conn, options)
+      }
+      JdbcUtils.saveTable(data, Some(schema), SQLConf.get.caseSensitiveAnalysis, options)
+    }
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonDataSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonDataSourceV2.scala
index 7a0949e586cd8..cd0eba0ca9c51 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonDataSourceV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonDataSourceV2.scala
@@ -31,14 +31,16 @@ class JsonDataSourceV2 extends FileDataSourceV2 {
 
   override def getTable(options: CaseInsensitiveStringMap): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
-    JsonTable(tableName, sparkSession, options, paths, None, fallbackFileFormat)
+    val tableName = getTableName(options, paths)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    JsonTable(tableName, sparkSession, optionsWithoutPaths, paths, None, fallbackFileFormat)
   }
 
   override def getTable(options: CaseInsensitiveStringMap, schema: StructType): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
-    JsonTable(tableName, sparkSession, options, paths, Some(schema), fallbackFileFormat)
+    val tableName = getTableName(options, paths)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    JsonTable(tableName, sparkSession, optionsWithoutPaths, paths, Some(schema), fallbackFileFormat)
   }
 }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonPartitionReaderFactory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonPartitionReaderFactory.scala
index 698423948f916..9737803b597a5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonPartitionReaderFactory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonPartitionReaderFactory.scala
@@ -24,6 +24,7 @@ import org.apache.spark.sql.execution.datasources.PartitionedFile
 import org.apache.spark.sql.execution.datasources.json.JsonDataSource
 import org.apache.spark.sql.execution.datasources.v2._
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.util.SerializableConfiguration
 
@@ -36,6 +37,7 @@ import org.apache.spark.util.SerializableConfiguration
  * @param readDataSchema Required schema of JSON files.
  * @param partitionSchema Schema of partitions.
  * @param parsedOptions Options for parsing JSON files.
+ * @param filters The filters pushed down to JSON datasource.
  */
 case class JsonPartitionReaderFactory(
     sqlConf: SQLConf,
@@ -43,12 +45,17 @@ case class JsonPartitionReaderFactory(
     dataSchema: StructType,
     readDataSchema: StructType,
     partitionSchema: StructType,
-    parsedOptions: JSONOptionsInRead) extends FilePartitionReaderFactory {
+    parsedOptions: JSONOptionsInRead,
+    filters: Seq[Filter]) extends FilePartitionReaderFactory {
 
   override def buildReader(partitionedFile: PartitionedFile): PartitionReader[InternalRow] = {
     val actualSchema =
       StructType(readDataSchema.filterNot(_.name == parsedOptions.columnNameOfCorruptRecord))
-    val parser = new JacksonParser(actualSchema, parsedOptions, allowArrayAsStructs = true)
+    val parser = new JacksonParser(
+      actualSchema,
+      parsedOptions,
+      allowArrayAsStructs = true,
+      filters)
     val iter = JsonDataSource(parsedOptions).readFile(
       broadcastedConf.value.value,
       partitionedFile,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonScan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonScan.scala
index 75231625676ff..7ad106627a083 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonScan.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonScan.scala
@@ -28,6 +28,7 @@ import org.apache.spark.sql.connector.read.PartitionReaderFactory
 import org.apache.spark.sql.execution.datasources.PartitioningAwareFileIndex
 import org.apache.spark.sql.execution.datasources.json.JsonDataSource
 import org.apache.spark.sql.execution.datasources.v2.{FileScan, TextBasedFileScan}
+import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 import org.apache.spark.util.SerializableConfiguration
@@ -39,6 +40,7 @@ case class JsonScan(
     readDataSchema: StructType,
     readPartitionSchema: StructType,
     options: CaseInsensitiveStringMap,
+    pushedFilters: Array[Filter],
     partitionFilters: Seq[Expression] = Seq.empty,
     dataFilters: Seq[Expression] = Seq.empty)
   extends TextBasedFileScan(sparkSession, options) {
@@ -86,7 +88,7 @@ case class JsonScan(
     // The partition values are already truncated in `FileScan.partitions`.
     // We should use `readPartitionSchema` as the partition schema here.
     JsonPartitionReaderFactory(sparkSession.sessionState.conf, broadcastedConf,
-      dataSchema, readDataSchema, readPartitionSchema, parsedOptions)
+      dataSchema, readDataSchema, readPartitionSchema, parsedOptions, pushedFilters)
   }
 
   override def withFilters(
@@ -94,10 +96,14 @@ case class JsonScan(
     this.copy(partitionFilters = partitionFilters, dataFilters = dataFilters)
 
   override def equals(obj: Any): Boolean = obj match {
-    case j: JsonScan => super.equals(j) && dataSchema == j.dataSchema && options == j.options
-
+    case j: JsonScan => super.equals(j) && dataSchema == j.dataSchema && options == j.options &&
+      equivalentFilters(pushedFilters, j.pushedFilters)
     case _ => false
   }
 
   override def hashCode(): Int = super.hashCode()
+
+  override def description(): String = {
+    super.description() + ", PushedFilters: " + pushedFilters.mkString("[", ", ", "]")
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonScanBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonScanBuilder.scala
index be53b1b1676f1..cf1204566ddbd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonScanBuilder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonScanBuilder.scala
@@ -17,9 +17,11 @@
 package org.apache.spark.sql.execution.datasources.v2.json
 
 import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.connector.read.Scan
+import org.apache.spark.sql.catalyst.StructFilters
+import org.apache.spark.sql.connector.read.{Scan, SupportsPushDownFilters}
 import org.apache.spark.sql.execution.datasources.PartitioningAwareFileIndex
 import org.apache.spark.sql.execution.datasources.v2.FileScanBuilder
+import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
@@ -29,8 +31,26 @@ class JsonScanBuilder (
     schema: StructType,
     dataSchema: StructType,
     options: CaseInsensitiveStringMap)
-  extends FileScanBuilder(sparkSession, fileIndex, dataSchema) {
+  extends FileScanBuilder(sparkSession, fileIndex, dataSchema) with SupportsPushDownFilters {
   override def build(): Scan = {
-    JsonScan(sparkSession, fileIndex, dataSchema, readDataSchema(), readPartitionSchema(), options)
+    JsonScan(
+      sparkSession,
+      fileIndex,
+      dataSchema,
+      readDataSchema(),
+      readPartitionSchema(),
+      options,
+      pushedFilters())
   }
+
+  private var _pushedFilters: Array[Filter] = Array.empty
+
+  override def pushFilters(filters: Array[Filter]): Array[Filter] = {
+    if (sparkSession.sessionState.conf.jsonFilterPushDown) {
+      _pushedFilters = StructFilters.pushedFilters(filters, dataSchema)
+    }
+    filters
+  }
+
+  override def pushedFilters(): Array[Filter] = _pushedFilters
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonTable.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonTable.scala
index 4b66aec6acbed..52168007aaa18 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonTable.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonTable.scala
@@ -22,7 +22,7 @@ import org.apache.hadoop.fs.FileStatus
 
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.json.JSONOptionsInRead
-import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder}
+import org.apache.spark.sql.connector.write.{LogicalWriteInfo, Write, WriteBuilder}
 import org.apache.spark.sql.execution.datasources.FileFormat
 import org.apache.spark.sql.execution.datasources.json.JsonDataSource
 import org.apache.spark.sql.execution.datasources.v2.FileTable
@@ -50,7 +50,9 @@ case class JsonTable(
   }
 
   override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder =
-    new JsonWriteBuilder(paths, formatName, supportsDataType, info)
+    new WriteBuilder {
+      override def build(): Write = JsonWrite(paths, formatName, supportsDataType, info)
+    }
 
   override def supportsDataType(dataType: DataType): Boolean = dataType match {
     case _: AtomicType => true
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonWrite.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonWrite.scala
new file mode 100644
index 0000000000000..ea1f6793cb9ca
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonWrite.scala
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2.json
+
+import org.apache.hadoop.mapreduce.{Job, TaskAttemptContext}
+
+import org.apache.spark.sql.catalyst.json.JSONOptions
+import org.apache.spark.sql.catalyst.util.CompressionCodecs
+import org.apache.spark.sql.connector.write.LogicalWriteInfo
+import org.apache.spark.sql.execution.datasources.{CodecStreams, OutputWriter, OutputWriterFactory}
+import org.apache.spark.sql.execution.datasources.json.JsonOutputWriter
+import org.apache.spark.sql.execution.datasources.v2.FileWrite
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types._
+
+case class JsonWrite(
+    paths: Seq[String],
+    formatName: String,
+    supportsDataType: DataType => Boolean,
+    info: LogicalWriteInfo) extends FileWrite {
+  override def prepareWrite(
+      sqlConf: SQLConf,
+      job: Job,
+      options: Map[String, String],
+      dataSchema: StructType): OutputWriterFactory = {
+    val conf = job.getConfiguration
+    val parsedOptions = new JSONOptions(
+      options,
+      sqlConf.sessionLocalTimeZone,
+      sqlConf.columnNameOfCorruptRecord)
+    parsedOptions.compressionCodec.foreach { codec =>
+      CompressionCodecs.setCodecConfiguration(conf, codec)
+    }
+
+    new OutputWriterFactory {
+      override def newInstance(
+          path: String,
+          dataSchema: StructType,
+          context: TaskAttemptContext): OutputWriter = {
+        new JsonOutputWriter(path, parsedOptions, dataSchema, context)
+      }
+
+      override def getFileExtension(context: TaskAttemptContext): String = {
+        ".json" + CodecStreams.getCompressionExtension(context)
+      }
+    }
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonWriteBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonWriteBuilder.scala
deleted file mode 100644
index 19f472057ea7d..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonWriteBuilder.scala
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.execution.datasources.v2.json
-
-import org.apache.hadoop.mapreduce.{Job, TaskAttemptContext}
-
-import org.apache.spark.sql.catalyst.json.JSONOptions
-import org.apache.spark.sql.catalyst.util.CompressionCodecs
-import org.apache.spark.sql.connector.write.LogicalWriteInfo
-import org.apache.spark.sql.execution.datasources.{CodecStreams, OutputWriter, OutputWriterFactory}
-import org.apache.spark.sql.execution.datasources.json.JsonOutputWriter
-import org.apache.spark.sql.execution.datasources.v2.FileWriteBuilder
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types._
-
-class JsonWriteBuilder(
-    paths: Seq[String],
-    formatName: String,
-    supportsDataType: DataType => Boolean,
-    info: LogicalWriteInfo)
-  extends FileWriteBuilder(paths, formatName, supportsDataType, info) {
-  override def prepareWrite(
-      sqlConf: SQLConf,
-      job: Job,
-      options: Map[String, String],
-      dataSchema: StructType): OutputWriterFactory = {
-    val conf = job.getConfiguration
-    val parsedOptions = new JSONOptions(
-      options,
-      sqlConf.sessionLocalTimeZone,
-      sqlConf.columnNameOfCorruptRecord)
-    parsedOptions.compressionCodec.foreach { codec =>
-      CompressionCodecs.setCodecConfiguration(conf, codec)
-    }
-
-    new OutputWriterFactory {
-      override def newInstance(
-          path: String,
-          dataSchema: StructType,
-          context: TaskAttemptContext): OutputWriter = {
-        new JsonOutputWriter(path, parsedOptions, dataSchema, context)
-      }
-
-      override def getFileExtension(context: TaskAttemptContext): String = {
-        ".json" + CodecStreams.getCompressionExtension(context)
-      }
-    }
-  }
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcDataSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcDataSourceV2.scala
index 8665af33b976a..6303723e4b805 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcDataSourceV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcDataSourceV2.scala
@@ -31,14 +31,16 @@ class OrcDataSourceV2 extends FileDataSourceV2 {
 
   override def getTable(options: CaseInsensitiveStringMap): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
-    OrcTable(tableName, sparkSession, options, paths, None, fallbackFileFormat)
+    val tableName = getTableName(options, paths)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    OrcTable(tableName, sparkSession, optionsWithoutPaths, paths, None, fallbackFileFormat)
   }
 
   override def getTable(options: CaseInsensitiveStringMap, schema: StructType): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
-    OrcTable(tableName, sparkSession, options, paths, Some(schema), fallbackFileFormat)
+    val tableName = getTableName(options, paths)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    OrcTable(tableName, sparkSession, optionsWithoutPaths, paths, Some(schema), fallbackFileFormat)
   }
 }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcPartitionReaderFactory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcPartitionReaderFactory.scala
index 03d58fdcb7f67..414252cc12481 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcPartitionReaderFactory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcPartitionReaderFactory.scala
@@ -31,9 +31,10 @@ import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.connector.read.{InputPartition, PartitionReader}
 import org.apache.spark.sql.execution.datasources.PartitionedFile
-import org.apache.spark.sql.execution.datasources.orc.{OrcColumnarBatchReader, OrcDeserializer, OrcUtils}
+import org.apache.spark.sql.execution.datasources.orc.{OrcColumnarBatchReader, OrcDeserializer, OrcFilters, OrcUtils}
 import org.apache.spark.sql.execution.datasources.v2._
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types.{AtomicType, StructType}
 import org.apache.spark.sql.vectorized.ColumnarBatch
 import org.apache.spark.util.{SerializableConfiguration, Utils}
@@ -52,10 +53,13 @@ case class OrcPartitionReaderFactory(
     broadcastedConf: Broadcast[SerializableConfiguration],
     dataSchema: StructType,
     readDataSchema: StructType,
-    partitionSchema: StructType) extends FilePartitionReaderFactory {
+    partitionSchema: StructType,
+    filters: Array[Filter]) extends FilePartitionReaderFactory {
   private val resultSchema = StructType(readDataSchema.fields ++ partitionSchema.fields)
   private val isCaseSensitive = sqlConf.caseSensitiveAnalysis
   private val capacity = sqlConf.orcVectorizedReaderBatchSize
+  private val orcFilterPushDown = sqlConf.orcFilterPushDown
+  private val ignoreCorruptFiles = sqlConf.ignoreCorruptFiles
 
   override def supportColumnarReads(partition: InputPartition): Boolean = {
     sqlConf.orcVectorizedReaderEnabled && sqlConf.wholeStageEnabled &&
@@ -63,27 +67,38 @@ case class OrcPartitionReaderFactory(
       resultSchema.forall(_.dataType.isInstanceOf[AtomicType])
   }
 
+  private def pushDownPredicates(filePath: Path, conf: Configuration): Unit = {
+    if (orcFilterPushDown && filters.nonEmpty) {
+      OrcUtils.readCatalystSchema(filePath, conf, ignoreCorruptFiles).foreach { fileSchema =>
+        OrcFilters.createFilter(fileSchema, filters).foreach { f =>
+          OrcInputFormat.setSearchArgument(conf, f, fileSchema.fieldNames)
+        }
+      }
+    }
+  }
+
   override def buildReader(file: PartitionedFile): PartitionReader[InternalRow] = {
     val conf = broadcastedConf.value.value
 
-    val resultSchemaString = OrcUtils.orcTypeDescriptionString(resultSchema)
-    OrcConf.MAPRED_INPUT_SCHEMA.setString(conf, resultSchemaString)
     OrcConf.IS_SCHEMA_EVOLUTION_CASE_SENSITIVE.setBoolean(conf, isCaseSensitive)
 
     val filePath = new Path(new URI(file.filePath))
 
+    pushDownPredicates(filePath, conf)
+
     val fs = filePath.getFileSystem(conf)
     val readerOptions = OrcFile.readerOptions(conf).filesystem(fs)
-    val requestedColIdsOrEmptyFile =
+    val resultedColPruneInfo =
       Utils.tryWithResource(OrcFile.createReader(filePath, readerOptions)) { reader =>
         OrcUtils.requestedColumnIds(
           isCaseSensitive, dataSchema, readDataSchema, reader, conf)
       }
 
-    if (requestedColIdsOrEmptyFile.isEmpty) {
+    if (resultedColPruneInfo.isEmpty) {
       new EmptyPartitionReader[InternalRow]
     } else {
-      val requestedColIds = requestedColIdsOrEmptyFile.get
+      val (requestedColIds, canPruneCols) = resultedColPruneInfo.get
+      OrcUtils.orcResultSchemaString(canPruneCols, dataSchema, resultSchema, partitionSchema, conf)
       assert(requestedColIds.length == readDataSchema.length,
         "[BUG] requested column IDs do not match required schema")
 
@@ -95,7 +110,7 @@ case class OrcPartitionReaderFactory(
 
       val orcRecordReader = new OrcInputFormat[OrcStruct]
         .createRecordReader(fileSplit, taskAttemptContext)
-      val deserializer = new OrcDeserializer(dataSchema, readDataSchema, requestedColIds)
+      val deserializer = new OrcDeserializer(readDataSchema, requestedColIds)
       val fileReader = new PartitionReader[InternalRow] {
         override def next(): Boolean = orcRecordReader.nextKeyValue()
 
@@ -112,24 +127,27 @@ case class OrcPartitionReaderFactory(
   override def buildColumnarReader(file: PartitionedFile): PartitionReader[ColumnarBatch] = {
     val conf = broadcastedConf.value.value
 
-    val resultSchemaString = OrcUtils.orcTypeDescriptionString(resultSchema)
-    OrcConf.MAPRED_INPUT_SCHEMA.setString(conf, resultSchemaString)
     OrcConf.IS_SCHEMA_EVOLUTION_CASE_SENSITIVE.setBoolean(conf, isCaseSensitive)
 
     val filePath = new Path(new URI(file.filePath))
 
+    pushDownPredicates(filePath, conf)
+
     val fs = filePath.getFileSystem(conf)
     val readerOptions = OrcFile.readerOptions(conf).filesystem(fs)
-    val requestedColIdsOrEmptyFile =
+    val resultedColPruneInfo =
       Utils.tryWithResource(OrcFile.createReader(filePath, readerOptions)) { reader =>
         OrcUtils.requestedColumnIds(
           isCaseSensitive, dataSchema, readDataSchema, reader, conf)
       }
 
-    if (requestedColIdsOrEmptyFile.isEmpty) {
+    if (resultedColPruneInfo.isEmpty) {
       new EmptyPartitionReader
     } else {
-      val requestedColIds = requestedColIdsOrEmptyFile.get ++ Array.fill(partitionSchema.length)(-1)
+      val (requestedDataColIds, canPruneCols) = resultedColPruneInfo.get
+      val resultSchemaString = OrcUtils.orcResultSchemaString(canPruneCols,
+        dataSchema, resultSchema, partitionSchema, conf)
+      val requestedColIds = requestedDataColIds ++ Array.fill(partitionSchema.length)(-1)
       assert(requestedColIds.length == resultSchema.length,
         "[BUG] requested column IDs do not match required schema")
       val taskConf = new Configuration(conf)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScan.scala
index 62894fa7a2538..8fa7f8dc41ead 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScan.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScan.scala
@@ -48,7 +48,7 @@ case class OrcScan(
     // The partition values are already truncated in `FileScan.partitions`.
     // We should use `readPartitionSchema` as the partition schema here.
     OrcPartitionReaderFactory(sparkSession.sessionState.conf, broadcastedConf,
-      dataSchema, readDataSchema, readPartitionSchema)
+      dataSchema, readDataSchema, readPartitionSchema, pushedFilters)
   }
 
   override def equals(obj: Any): Boolean = obj match {
@@ -65,6 +65,10 @@ case class OrcScan(
     super.description() + ", PushedFilters: " + seqToString(pushedFilters)
   }
 
+  override def getMetaData(): Map[String, String] = {
+    super.getMetaData() ++ Map("PushedFilters" -> seqToString(pushedFilters))
+  }
+
   override def withFilters(
       partitionFilters: Seq[Expression], dataFilters: Seq[Expression]): FileScan =
     this.copy(partitionFilters = partitionFilters, dataFilters = dataFilters)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScanBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScanBuilder.scala
index 9f40f5faa2e99..a8c813a03e0ca 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScanBuilder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScanBuilder.scala
@@ -19,14 +19,12 @@ package org.apache.spark.sql.execution.datasources.v2.orc
 
 import scala.collection.JavaConverters._
 
-import org.apache.orc.mapreduce.OrcInputFormat
-
 import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.quoteIfNeeded
 import org.apache.spark.sql.connector.read.{Scan, SupportsPushDownFilters}
 import org.apache.spark.sql.execution.datasources.PartitioningAwareFileIndex
 import org.apache.spark.sql.execution.datasources.orc.OrcFilters
 import org.apache.spark.sql.execution.datasources.v2.FileScanBuilder
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
@@ -55,15 +53,8 @@ case class OrcScanBuilder(
 
   override def pushFilters(filters: Array[Filter]): Array[Filter] = {
     if (sparkSession.sessionState.conf.orcFilterPushDown) {
-      OrcFilters.createFilter(schema, filters).foreach { f =>
-        // The pushed filters will be set in `hadoopConf`. After that, we can simply use the
-        // changed `hadoopConf` in executors.
-        OrcInputFormat.setSearchArgument(hadoopConf, f, schema.fieldNames)
-      }
-      val dataTypeMap = schema.map(f => quoteIfNeeded(f.name) -> f.dataType).toMap
-      // TODO (SPARK-25557): ORC doesn't support nested predicate pushdown, so they are removed.
-      val newFilters = filters.filter(!_.containsNestedColumn)
-      _pushedFilters = OrcFilters.convertibleFilters(schema, dataTypeMap, newFilters).toArray
+      val dataTypeMap = OrcFilters.getSearchableTypeMap(schema, SQLConf.get.caseSensitiveAnalysis)
+      _pushedFilters = OrcFilters.convertibleFilters(dataTypeMap, filters).toArray
     }
     filters
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcTable.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcTable.scala
index 3ef41210de181..9cc4525badd81 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcTable.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcTable.scala
@@ -21,7 +21,7 @@ import scala.collection.JavaConverters._
 import org.apache.hadoop.fs.FileStatus
 
 import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder}
+import org.apache.spark.sql.connector.write.{LogicalWriteInfo, Write, WriteBuilder}
 import org.apache.spark.sql.execution.datasources.FileFormat
 import org.apache.spark.sql.execution.datasources.orc.OrcUtils
 import org.apache.spark.sql.execution.datasources.v2.FileTable
@@ -44,7 +44,9 @@ case class OrcTable(
     OrcUtils.inferSchema(sparkSession, files, options.asScala.toMap)
 
   override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder =
-    new OrcWriteBuilder(paths, formatName, supportsDataType, info)
+    new WriteBuilder {
+      override def build(): Write = OrcWrite(paths, formatName, supportsDataType, info)
+    }
 
   override def supportsDataType(dataType: DataType): Boolean = dataType match {
     case _: AtomicType => true
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcWrite.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcWrite.scala
new file mode 100644
index 0000000000000..286e871080535
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcWrite.scala
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2.orc
+
+import org.apache.hadoop.mapred.JobConf
+import org.apache.hadoop.mapreduce.{Job, TaskAttemptContext}
+import org.apache.orc.OrcConf.{COMPRESS, MAPRED_OUTPUT_SCHEMA}
+import org.apache.orc.mapred.OrcStruct
+
+import org.apache.spark.sql.connector.write.LogicalWriteInfo
+import org.apache.spark.sql.execution.datasources.{OutputWriter, OutputWriterFactory}
+import org.apache.spark.sql.execution.datasources.orc.{OrcFileFormat, OrcOptions, OrcOutputWriter, OrcUtils}
+import org.apache.spark.sql.execution.datasources.v2.FileWrite
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types._
+
+case class OrcWrite(
+    paths: Seq[String],
+    formatName: String,
+    supportsDataType: DataType => Boolean,
+    info: LogicalWriteInfo) extends FileWrite {
+
+  override def prepareWrite(
+      sqlConf: SQLConf,
+      job: Job,
+      options: Map[String, String],
+      dataSchema: StructType): OutputWriterFactory = {
+    val orcOptions = new OrcOptions(options, sqlConf)
+
+    val conf = job.getConfiguration
+
+    conf.set(MAPRED_OUTPUT_SCHEMA.getAttribute, OrcFileFormat.getQuotedSchemaString(dataSchema))
+
+    conf.set(COMPRESS.getAttribute, orcOptions.compressionCodec)
+
+    conf.asInstanceOf[JobConf]
+      .setOutputFormat(classOf[org.apache.orc.mapred.OrcOutputFormat[OrcStruct]])
+
+    new OutputWriterFactory {
+      override def newInstance(
+          path: String,
+          dataSchema: StructType,
+          context: TaskAttemptContext): OutputWriter = {
+        new OrcOutputWriter(path, dataSchema, context)
+      }
+
+      override def getFileExtension(context: TaskAttemptContext): String = {
+        val compressionExtension: String = {
+          val name = context.getConfiguration.get(COMPRESS.getAttribute)
+          OrcUtils.extensionsForCompressionCodecNames.getOrElse(name, "")
+        }
+
+        compressionExtension + ".orc"
+      }
+    }
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcWriteBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcWriteBuilder.scala
deleted file mode 100644
index 48044748708d9..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcWriteBuilder.scala
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.execution.datasources.v2.orc
-
-import org.apache.hadoop.mapred.JobConf
-import org.apache.hadoop.mapreduce.{Job, TaskAttemptContext}
-import org.apache.orc.OrcConf.{COMPRESS, MAPRED_OUTPUT_SCHEMA}
-import org.apache.orc.mapred.OrcStruct
-
-import org.apache.spark.sql.connector.write.LogicalWriteInfo
-import org.apache.spark.sql.execution.datasources.{OutputWriter, OutputWriterFactory}
-import org.apache.spark.sql.execution.datasources.orc.{OrcFileFormat, OrcOptions, OrcOutputWriter, OrcUtils}
-import org.apache.spark.sql.execution.datasources.v2.FileWriteBuilder
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types._
-
-class OrcWriteBuilder(
-    paths: Seq[String],
-    formatName: String,
-    supportsDataType: DataType => Boolean,
-    info: LogicalWriteInfo)
-  extends FileWriteBuilder(paths, formatName, supportsDataType, info) {
-
-  override def prepareWrite(
-      sqlConf: SQLConf,
-      job: Job,
-      options: Map[String, String],
-      dataSchema: StructType): OutputWriterFactory = {
-    val orcOptions = new OrcOptions(options, sqlConf)
-
-    val conf = job.getConfiguration
-
-    conf.set(MAPRED_OUTPUT_SCHEMA.getAttribute, OrcFileFormat.getQuotedSchemaString(dataSchema))
-
-    conf.set(COMPRESS.getAttribute, orcOptions.compressionCodec)
-
-    conf.asInstanceOf[JobConf]
-      .setOutputFormat(classOf[org.apache.orc.mapred.OrcOutputFormat[OrcStruct]])
-
-    new OutputWriterFactory {
-      override def newInstance(
-          path: String,
-          dataSchema: StructType,
-          context: TaskAttemptContext): OutputWriter = {
-        new OrcOutputWriter(path, dataSchema, context)
-      }
-
-      override def getFileExtension(context: TaskAttemptContext): String = {
-        val compressionExtension: String = {
-          val name = context.getConfiguration.get(COMPRESS.getAttribute)
-          OrcUtils.extensionsForCompressionCodecNames.getOrElse(name, "")
-        }
-
-        compressionExtension + ".orc"
-      }
-    }
-  }
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetDataSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetDataSourceV2.scala
index 8cb6186c12ff3..4590660bc7b7e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetDataSourceV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetDataSourceV2.scala
@@ -31,14 +31,17 @@ class ParquetDataSourceV2 extends FileDataSourceV2 {
 
   override def getTable(options: CaseInsensitiveStringMap): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
-    ParquetTable(tableName, sparkSession, options, paths, None, fallbackFileFormat)
+    val tableName = getTableName(options, paths)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    ParquetTable(tableName, sparkSession, optionsWithoutPaths, paths, None, fallbackFileFormat)
   }
 
   override def getTable(options: CaseInsensitiveStringMap, schema: StructType): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
-    ParquetTable(tableName, sparkSession, options, paths, Some(schema), fallbackFileFormat)
+    val tableName = getTableName(options, paths)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    ParquetTable(
+      tableName, sparkSession, optionsWithoutPaths, paths, Some(schema), fallbackFileFormat)
   }
 }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetPartitionReaderFactory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetPartitionReaderFactory.scala
index 3b482b0c8ab62..78076040e7cf5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetPartitionReaderFactory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetPartitionReaderFactory.scala
@@ -20,12 +20,13 @@ import java.net.URI
 import java.time.ZoneId
 
 import org.apache.hadoop.fs.Path
+import org.apache.hadoop.mapred.FileSplit
 import org.apache.hadoop.mapreduce._
 import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
 import org.apache.parquet.filter2.compat.FilterCompat
 import org.apache.parquet.filter2.predicate.{FilterApi, FilterPredicate}
 import org.apache.parquet.format.converter.ParquetMetadataConverter.SKIP_ROW_GROUPS
-import org.apache.parquet.hadoop.{ParquetFileReader, ParquetInputFormat, ParquetInputSplit, ParquetRecordReader}
+import org.apache.parquet.hadoop.{ParquetInputFormat, ParquetRecordReader}
 
 import org.apache.spark.TaskContext
 import org.apache.spark.broadcast.Broadcast
@@ -52,6 +53,7 @@ import org.apache.spark.util.SerializableConfiguration
  * @param readDataSchema Required schema of Parquet files.
  * @param partitionSchema Schema of partitions.
  * @param filters Filters to be pushed down in the batch scan.
+ * @param parquetOptions The options of Parquet datasource that are set for the read.
  */
 case class ParquetPartitionReaderFactory(
     sqlConf: SQLConf,
@@ -59,7 +61,8 @@ case class ParquetPartitionReaderFactory(
     dataSchema: StructType,
     readDataSchema: StructType,
     partitionSchema: StructType,
-    filters: Array[Filter]) extends FilePartitionReaderFactory with Logging {
+    filters: Array[Filter],
+    parquetOptions: ParquetOptions) extends FilePartitionReaderFactory with Logging {
   private val isCaseSensitive = sqlConf.caseSensitiveAnalysis
   private val resultSchema = StructType(partitionSchema.fields ++ readDataSchema.fields)
   private val enableOffHeapColumnVector = sqlConf.offHeapColumnVectorEnabled
@@ -74,6 +77,8 @@ case class ParquetPartitionReaderFactory(
   private val pushDownDecimal = sqlConf.parquetFilterPushDownDecimal
   private val pushDownStringStartWith = sqlConf.parquetFilterPushDownStringStartWith
   private val pushDownInFilterThreshold = sqlConf.parquetFilterPushDownInFilterThreshold
+  private val datetimeRebaseModeInRead = parquetOptions.datetimeRebaseModeInRead
+  private val int96RebaseModeInRead = parquetOptions.int96RebaseModeInRead
 
   override def supportColumnarReads(partition: InputPartition): Boolean = {
     sqlConf.parquetVectorizedReaderEnabled && sqlConf.wholeStageEnabled &&
@@ -117,23 +122,17 @@ case class ParquetPartitionReaderFactory(
   private def buildReaderBase[T](
       file: PartitionedFile,
       buildReaderFunc: (
-        ParquetInputSplit, InternalRow, TaskAttemptContextImpl,
+        FileSplit, InternalRow, TaskAttemptContextImpl,
           Option[FilterPredicate], Option[ZoneId],
+          LegacyBehaviorPolicy.Value,
           LegacyBehaviorPolicy.Value) => RecordReader[Void, T]): RecordReader[Void, T] = {
     val conf = broadcastedConf.value.value
 
     val filePath = new Path(new URI(file.filePath))
-    val split =
-      new org.apache.parquet.hadoop.ParquetInputSplit(
-        filePath,
-        file.start,
-        file.start + file.length,
-        file.length,
-        Array.empty,
-        null)
+    val split = new FileSplit(filePath, file.start, file.length, Array.empty[String])
 
     lazy val footerFileMetaData =
-      ParquetFileReader.readFooter(conf, filePath, SKIP_ROW_GROUPS).getFileMetaData
+      ParquetFooterReader.readFooter(conf, filePath, SKIP_ROW_GROUPS).getFileMetaData
     // Try to push down filters when filter push-down is enabled.
     val pushed = if (enableParquetFilterPushDown) {
       val parquetSchema = footerFileMetaData.getSchema
@@ -173,9 +172,18 @@ case class ParquetPartitionReaderFactory(
     }
     val datetimeRebaseMode = DataSourceUtils.datetimeRebaseMode(
       footerFileMetaData.getKeyValueMetaData.get,
-      SQLConf.get.getConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ))
+      datetimeRebaseModeInRead)
+    val int96RebaseMode = DataSourceUtils.int96RebaseMode(
+      footerFileMetaData.getKeyValueMetaData.get,
+      int96RebaseModeInRead)
     val reader = buildReaderFunc(
-      split, file.partitionValues, hadoopAttemptContext, pushed, convertTz, datetimeRebaseMode)
+      split,
+      file.partitionValues,
+      hadoopAttemptContext,
+      pushed,
+      convertTz,
+      datetimeRebaseMode,
+      int96RebaseMode)
     reader.initialize(split, hadoopAttemptContext)
     reader
   }
@@ -185,17 +193,21 @@ case class ParquetPartitionReaderFactory(
   }
 
   private def createRowBaseParquetReader(
-      split: ParquetInputSplit,
+      split: FileSplit,
       partitionValues: InternalRow,
       hadoopAttemptContext: TaskAttemptContextImpl,
       pushed: Option[FilterPredicate],
       convertTz: Option[ZoneId],
-      datetimeRebaseMode: LegacyBehaviorPolicy.Value): RecordReader[Void, InternalRow] = {
+      datetimeRebaseMode: LegacyBehaviorPolicy.Value,
+      int96RebaseMode: LegacyBehaviorPolicy.Value): RecordReader[Void, InternalRow] = {
     logDebug(s"Falling back to parquet-mr")
     val taskContext = Option(TaskContext.get())
     // ParquetRecordReader returns InternalRow
     val readSupport = new ParquetReadSupport(
-      convertTz, enableVectorizedReader = false, datetimeRebaseMode)
+      convertTz,
+      enableVectorizedReader = false,
+      datetimeRebaseMode,
+      int96RebaseMode)
     val reader = if (pushed.isDefined && enableRecordFilter) {
       val parquetFilter = FilterCompat.get(pushed.get, null)
       new ParquetRecordReader[InternalRow](readSupport, parquetFilter)
@@ -216,16 +228,18 @@ case class ParquetPartitionReaderFactory(
   }
 
   private def createParquetVectorizedReader(
-      split: ParquetInputSplit,
+      split: FileSplit,
       partitionValues: InternalRow,
       hadoopAttemptContext: TaskAttemptContextImpl,
       pushed: Option[FilterPredicate],
       convertTz: Option[ZoneId],
-      datetimeRebaseMode: LegacyBehaviorPolicy.Value): VectorizedParquetRecordReader = {
+      datetimeRebaseMode: LegacyBehaviorPolicy.Value,
+      int96RebaseMode: LegacyBehaviorPolicy.Value): VectorizedParquetRecordReader = {
     val taskContext = Option(TaskContext.get())
     val vectorizedReader = new VectorizedParquetRecordReader(
       convertTz.orNull,
       datetimeRebaseMode.toString,
+      int96RebaseMode.toString,
       enableOffHeapColumnVector && taskContext.isDefined,
       capacity)
     val iter = new RecordReaderIterator(vectorizedReader)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetScan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetScan.scala
index bb315262a8211..60573ba10ccb6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetScan.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetScan.scala
@@ -16,6 +16,8 @@
  */
 package org.apache.spark.sql.execution.datasources.v2.parquet
 
+import scala.collection.JavaConverters._
+
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.Path
 import org.apache.parquet.hadoop.ParquetInputFormat
@@ -24,7 +26,7 @@ import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.expressions.Expression
 import org.apache.spark.sql.connector.read.PartitionReaderFactory
 import org.apache.spark.sql.execution.datasources.PartitioningAwareFileIndex
-import org.apache.spark.sql.execution.datasources.parquet.{ParquetReadSupport, ParquetWriteSupport}
+import org.apache.spark.sql.execution.datasources.parquet.{ParquetOptions, ParquetReadSupport, ParquetWriteSupport}
 import org.apache.spark.sql.execution.datasources.v2.FileScan
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.sources.Filter
@@ -76,8 +78,15 @@ case class ParquetScan(
 
     val broadcastedConf = sparkSession.sparkContext.broadcast(
       new SerializableConfiguration(hadoopConf))
-    ParquetPartitionReaderFactory(sparkSession.sessionState.conf, broadcastedConf,
-      dataSchema, readDataSchema, readPartitionSchema, pushedFilters)
+    val sqlConf = sparkSession.sessionState.conf
+    ParquetPartitionReaderFactory(
+      sqlConf,
+      broadcastedConf,
+      dataSchema,
+      readDataSchema,
+      readPartitionSchema,
+      pushedFilters,
+      new ParquetOptions(options.asCaseSensitiveMap.asScala.toMap, sqlConf))
   }
 
   override def equals(obj: Any): Boolean = obj match {
@@ -93,6 +102,10 @@ case class ParquetScan(
     super.description() + ", PushedFilters: " + seqToString(pushedFilters)
   }
 
+  override def getMetaData(): Map[String, String] = {
+    super.getMetaData() ++ Map("PushedFilters" -> seqToString(pushedFilters))
+  }
+
   override def withFilters(
       partitionFilters: Seq[Expression], dataFilters: Seq[Expression]): FileScan =
     this.copy(partitionFilters = partitionFilters, dataFilters = dataFilters)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetScanBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetScanBuilder.scala
index 2f861356e9499..44053830defe5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetScanBuilder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetScanBuilder.scala
@@ -50,7 +50,7 @@ case class ParquetScanBuilder(
     val pushDownInFilterThreshold = sqlConf.parquetFilterPushDownInFilterThreshold
     val isCaseSensitive = sqlConf.caseSensitiveAnalysis
     val parquetSchema =
-      new SparkToParquetSchemaConverter(sparkSession.sessionState.conf).convert(schema)
+      new SparkToParquetSchemaConverter(sparkSession.sessionState.conf).convert(readDataSchema())
     val parquetFilters = new ParquetFilters(parquetSchema, pushDownDate, pushDownTimestamp,
       pushDownDecimal, pushDownStringStartWith, pushDownInFilterThreshold, isCaseSensitive)
     parquetFilters.convertibleFilters(this.filters).toArray
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetTable.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetTable.scala
index e9f9bf8df35e6..c8bb4b2eb221e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetTable.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetTable.scala
@@ -21,7 +21,7 @@ import scala.collection.JavaConverters._
 import org.apache.hadoop.fs.FileStatus
 
 import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder}
+import org.apache.spark.sql.connector.write.{LogicalWriteInfo, Write, WriteBuilder}
 import org.apache.spark.sql.execution.datasources.FileFormat
 import org.apache.spark.sql.execution.datasources.parquet.ParquetUtils
 import org.apache.spark.sql.execution.datasources.v2.FileTable
@@ -44,7 +44,9 @@ case class ParquetTable(
     ParquetUtils.inferSchema(sparkSession, options.asScala.toMap, files)
 
   override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder =
-    new ParquetWriteBuilder(paths, formatName, supportsDataType, info)
+    new WriteBuilder {
+      override def build(): Write = ParquetWrite(paths, formatName, supportsDataType, info)
+    }
 
   override def supportsDataType(dataType: DataType): Boolean = dataType match {
     case _: AtomicType => true
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetWrite.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetWrite.scala
new file mode 100644
index 0000000000000..0316d91f40732
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetWrite.scala
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2.parquet
+
+import org.apache.hadoop.mapreduce.{Job, OutputCommitter, TaskAttemptContext}
+import org.apache.parquet.hadoop.{ParquetOutputCommitter, ParquetOutputFormat}
+import org.apache.parquet.hadoop.ParquetOutputFormat.JobSummaryLevel
+import org.apache.parquet.hadoop.codec.CodecConfig
+import org.apache.parquet.hadoop.util.ContextUtil
+
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.connector.write.LogicalWriteInfo
+import org.apache.spark.sql.execution.datasources.{OutputWriter, OutputWriterFactory}
+import org.apache.spark.sql.execution.datasources.parquet._
+import org.apache.spark.sql.execution.datasources.v2.FileWrite
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types._
+
+case class ParquetWrite(
+    paths: Seq[String],
+    formatName: String,
+    supportsDataType: DataType => Boolean,
+    info: LogicalWriteInfo) extends FileWrite with Logging {
+
+  override def prepareWrite(
+      sqlConf: SQLConf,
+      job: Job,
+      options: Map[String, String],
+      dataSchema: StructType): OutputWriterFactory = {
+    val parquetOptions = new ParquetOptions(options, sqlConf)
+
+    val conf = ContextUtil.getConfiguration(job)
+
+    val committerClass =
+      conf.getClass(
+        SQLConf.PARQUET_OUTPUT_COMMITTER_CLASS.key,
+        classOf[ParquetOutputCommitter],
+        classOf[OutputCommitter])
+
+    if (conf.get(SQLConf.PARQUET_OUTPUT_COMMITTER_CLASS.key) == null) {
+      logInfo("Using default output committer for Parquet: " +
+        classOf[ParquetOutputCommitter].getCanonicalName)
+    } else {
+      logInfo("Using user defined output committer for Parquet: " + committerClass.getCanonicalName)
+    }
+
+    conf.setClass(
+      SQLConf.OUTPUT_COMMITTER_CLASS.key,
+      committerClass,
+      classOf[OutputCommitter])
+
+    // We're not really using `ParquetOutputFormat[Row]` for writing data here, because we override
+    // it in `ParquetOutputWriter` to support appending and dynamic partitioning.  The reason why
+    // we set it here is to setup the output committer class to `ParquetOutputCommitter`, which is
+    // bundled with `ParquetOutputFormat[Row]`.
+    job.setOutputFormatClass(classOf[ParquetOutputFormat[Row]])
+
+    ParquetOutputFormat.setWriteSupportClass(job, classOf[ParquetWriteSupport])
+
+    // This metadata is useful for keeping UDTs like Vector/Matrix.
+    ParquetWriteSupport.setSchema(dataSchema, conf)
+
+    // Sets flags for `ParquetWriteSupport`, which converts Catalyst schema to Parquet
+    // schema and writes actual rows to Parquet files.
+    conf.set(SQLConf.PARQUET_WRITE_LEGACY_FORMAT.key, sqlConf.writeLegacyParquetFormat.toString)
+
+    conf.set(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key, sqlConf.parquetOutputTimestampType.toString)
+
+    // Sets compression scheme
+    conf.set(ParquetOutputFormat.COMPRESSION, parquetOptions.compressionCodecClassName)
+
+    // SPARK-15719: Disables writing Parquet summary files by default.
+    if (conf.get(ParquetOutputFormat.JOB_SUMMARY_LEVEL) == null
+      && conf.get(ParquetOutputFormat.ENABLE_JOB_SUMMARY) == null) {
+      conf.setEnum(ParquetOutputFormat.JOB_SUMMARY_LEVEL, JobSummaryLevel.NONE)
+    }
+
+    if (ParquetOutputFormat.getJobSummaryLevel(conf) == JobSummaryLevel.NONE
+      && !classOf[ParquetOutputCommitter].isAssignableFrom(committerClass)) {
+      // output summary is requested, but the class is not a Parquet Committer
+      logWarning(s"Committer $committerClass is not a ParquetOutputCommitter and cannot" +
+        s" create job summaries. " +
+        s"Set Parquet option ${ParquetOutputFormat.JOB_SUMMARY_LEVEL} to NONE.")
+    }
+
+    new OutputWriterFactory {
+      override def newInstance(
+          path: String,
+          dataSchema: StructType,
+          context: TaskAttemptContext): OutputWriter = {
+        new ParquetOutputWriter(path, context)
+      }
+
+      override def getFileExtension(context: TaskAttemptContext): String = {
+        CodecConfig.from(context).getCodec.getExtension + ".parquet"
+      }
+    }
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetWriteBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetWriteBuilder.scala
deleted file mode 100644
index a4e22c21a11f3..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetWriteBuilder.scala
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.execution.datasources.v2.parquet
-
-import org.apache.hadoop.mapreduce.{Job, OutputCommitter, TaskAttemptContext}
-import org.apache.parquet.hadoop.{ParquetOutputCommitter, ParquetOutputFormat}
-import org.apache.parquet.hadoop.ParquetOutputFormat.JobSummaryLevel
-import org.apache.parquet.hadoop.codec.CodecConfig
-import org.apache.parquet.hadoop.util.ContextUtil
-
-import org.apache.spark.internal.Logging
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.connector.write.LogicalWriteInfo
-import org.apache.spark.sql.execution.datasources.{OutputWriter, OutputWriterFactory}
-import org.apache.spark.sql.execution.datasources.parquet._
-import org.apache.spark.sql.execution.datasources.v2.FileWriteBuilder
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types._
-
-class ParquetWriteBuilder(
-    paths: Seq[String],
-    formatName: String,
-    supportsDataType: DataType => Boolean,
-    info: LogicalWriteInfo)
-  extends FileWriteBuilder(paths, formatName, supportsDataType, info) with Logging {
-
-  override def prepareWrite(
-      sqlConf: SQLConf,
-      job: Job,
-      options: Map[String, String],
-      dataSchema: StructType): OutputWriterFactory = {
-    val parquetOptions = new ParquetOptions(options, sqlConf)
-
-    val conf = ContextUtil.getConfiguration(job)
-
-    val committerClass =
-      conf.getClass(
-        SQLConf.PARQUET_OUTPUT_COMMITTER_CLASS.key,
-        classOf[ParquetOutputCommitter],
-        classOf[OutputCommitter])
-
-    if (conf.get(SQLConf.PARQUET_OUTPUT_COMMITTER_CLASS.key) == null) {
-      logInfo("Using default output committer for Parquet: " +
-        classOf[ParquetOutputCommitter].getCanonicalName)
-    } else {
-      logInfo("Using user defined output committer for Parquet: " + committerClass.getCanonicalName)
-    }
-
-    conf.setClass(
-      SQLConf.OUTPUT_COMMITTER_CLASS.key,
-      committerClass,
-      classOf[OutputCommitter])
-
-    // We're not really using `ParquetOutputFormat[Row]` for writing data here, because we override
-    // it in `ParquetOutputWriter` to support appending and dynamic partitioning.  The reason why
-    // we set it here is to setup the output committer class to `ParquetOutputCommitter`, which is
-    // bundled with `ParquetOutputFormat[Row]`.
-    job.setOutputFormatClass(classOf[ParquetOutputFormat[Row]])
-
-    ParquetOutputFormat.setWriteSupportClass(job, classOf[ParquetWriteSupport])
-
-    // This metadata is useful for keeping UDTs like Vector/Matrix.
-    ParquetWriteSupport.setSchema(dataSchema, conf)
-
-    // Sets flags for `ParquetWriteSupport`, which converts Catalyst schema to Parquet
-    // schema and writes actual rows to Parquet files.
-    conf.set(SQLConf.PARQUET_WRITE_LEGACY_FORMAT.key, sqlConf.writeLegacyParquetFormat.toString)
-
-    conf.set(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key, sqlConf.parquetOutputTimestampType.toString)
-
-    // Sets compression scheme
-    conf.set(ParquetOutputFormat.COMPRESSION, parquetOptions.compressionCodecClassName)
-
-    // SPARK-15719: Disables writing Parquet summary files by default.
-    if (conf.get(ParquetOutputFormat.JOB_SUMMARY_LEVEL) == null
-      && conf.get(ParquetOutputFormat.ENABLE_JOB_SUMMARY) == null) {
-      conf.setEnum(ParquetOutputFormat.JOB_SUMMARY_LEVEL, JobSummaryLevel.NONE)
-    }
-
-    if (ParquetOutputFormat.getJobSummaryLevel(conf) == JobSummaryLevel.NONE
-      && !classOf[ParquetOutputCommitter].isAssignableFrom(committerClass)) {
-      // output summary is requested, but the class is not a Parquet Committer
-      logWarning(s"Committer $committerClass is not a ParquetOutputCommitter and cannot" +
-        s" create job summaries. " +
-        s"Set Parquet option ${ParquetOutputFormat.JOB_SUMMARY_LEVEL} to NONE.")
-    }
-
-    new OutputWriterFactory {
-      override def newInstance(
-          path: String,
-          dataSchema: StructType,
-          context: TaskAttemptContext): OutputWriter = {
-        new ParquetOutputWriter(path, context)
-      }
-
-      override def getFileExtension(context: TaskAttemptContext): String = {
-        CodecConfig.from(context).getCodec.getExtension + ".parquet"
-      }
-    }
-  }
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextDataSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextDataSourceV2.scala
index 049c717effa26..f375a128d8177 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextDataSourceV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextDataSourceV2.scala
@@ -31,14 +31,16 @@ class TextDataSourceV2 extends FileDataSourceV2 {
 
   override def getTable(options: CaseInsensitiveStringMap): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
-    TextTable(tableName, sparkSession, options, paths, None, fallbackFileFormat)
+    val tableName = getTableName(options, paths)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    TextTable(tableName, sparkSession, optionsWithoutPaths, paths, None, fallbackFileFormat)
   }
 
   override def getTable(options: CaseInsensitiveStringMap, schema: StructType): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
-    TextTable(tableName, sparkSession, options, paths, Some(schema), fallbackFileFormat)
+    val tableName = getTableName(options, paths)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    TextTable(tableName, sparkSession, optionsWithoutPaths, paths, Some(schema), fallbackFileFormat)
   }
 }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextTable.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextTable.scala
index 36304a9b17a1e..c09eca208b037 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextTable.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextTable.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.execution.datasources.v2.text
 import org.apache.hadoop.fs.FileStatus
 
 import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder}
+import org.apache.spark.sql.connector.write.{LogicalWriteInfo, Write, WriteBuilder}
 import org.apache.spark.sql.execution.datasources.FileFormat
 import org.apache.spark.sql.execution.datasources.v2.FileTable
 import org.apache.spark.sql.types.{DataType, StringType, StructField, StructType}
@@ -40,7 +40,9 @@ case class TextTable(
     Some(StructType(Seq(StructField("value", StringType))))
 
   override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder =
-    new TextWriteBuilder(paths, formatName, supportsDataType, info)
+    new WriteBuilder {
+      override def build(): Write = TextWrite(paths, formatName, supportsDataType, info)
+    }
 
   override def supportsDataType(dataType: DataType): Boolean = dataType == StringType
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextWrite.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextWrite.scala
new file mode 100644
index 0000000000000..cd66f62eca956
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextWrite.scala
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2.text
+
+import org.apache.hadoop.mapreduce.{Job, TaskAttemptContext}
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.util.CompressionCodecs
+import org.apache.spark.sql.connector.write.LogicalWriteInfo
+import org.apache.spark.sql.execution.datasources.{CodecStreams, OutputWriter, OutputWriterFactory}
+import org.apache.spark.sql.execution.datasources.text.{TextOptions, TextOutputWriter}
+import org.apache.spark.sql.execution.datasources.v2.FileWrite
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types._
+
+case class TextWrite(
+    paths: Seq[String],
+    formatName: String,
+    supportsDataType: DataType => Boolean,
+    info: LogicalWriteInfo) extends FileWrite {
+  private def verifySchema(schema: StructType): Unit = {
+    if (schema.size != 1) {
+      throw new AnalysisException(
+        s"Text data source supports only a single column, and you have ${schema.size} columns.")
+    }
+  }
+
+  override def prepareWrite(
+      sqlConf: SQLConf,
+      job: Job,
+      options: Map[String, String],
+      dataSchema: StructType): OutputWriterFactory = {
+    verifySchema(dataSchema)
+
+    val textOptions = new TextOptions(options)
+    val conf = job.getConfiguration
+
+    textOptions.compressionCodec.foreach { codec =>
+      CompressionCodecs.setCodecConfiguration(conf, codec)
+    }
+
+    new OutputWriterFactory {
+      override def newInstance(
+          path: String,
+          dataSchema: StructType,
+          context: TaskAttemptContext): OutputWriter = {
+        new TextOutputWriter(path, dataSchema, textOptions.lineSeparatorInWrite, context)
+      }
+
+      override def getFileExtension(context: TaskAttemptContext): String = {
+        ".txt" + CodecStreams.getCompressionExtension(context)
+      }
+    }
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextWriteBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextWriteBuilder.scala
deleted file mode 100644
index a3bf4dcae3f33..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextWriteBuilder.scala
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.execution.datasources.v2.text
-
-import org.apache.hadoop.mapreduce.{Job, TaskAttemptContext}
-
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.util.CompressionCodecs
-import org.apache.spark.sql.connector.write.LogicalWriteInfo
-import org.apache.spark.sql.execution.datasources.{CodecStreams, OutputWriter, OutputWriterFactory}
-import org.apache.spark.sql.execution.datasources.text.{TextOptions, TextOutputWriter}
-import org.apache.spark.sql.execution.datasources.v2.FileWriteBuilder
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types._
-
-class TextWriteBuilder(
-    paths: Seq[String],
-    formatName: String,
-    supportsDataType: DataType => Boolean,
-    info: LogicalWriteInfo)
-  extends FileWriteBuilder(paths, formatName, supportsDataType, info) {
-  private def verifySchema(schema: StructType): Unit = {
-    if (schema.size != 1) {
-      throw new AnalysisException(
-        s"Text data source supports only a single column, and you have ${schema.size} columns.")
-    }
-  }
-
-  override def prepareWrite(
-      sqlConf: SQLConf,
-      job: Job,
-      options: Map[String, String],
-      dataSchema: StructType): OutputWriterFactory = {
-    verifySchema(dataSchema)
-
-    val textOptions = new TextOptions(options)
-    val conf = job.getConfiguration
-
-    textOptions.compressionCodec.foreach { codec =>
-      CompressionCodecs.setCodecConfiguration(conf, codec)
-    }
-
-    new OutputWriterFactory {
-      override def newInstance(
-          path: String,
-          dataSchema: StructType,
-          context: TaskAttemptContext): OutputWriter = {
-        new TextOutputWriter(path, dataSchema, textOptions.lineSeparatorInWrite, context)
-      }
-
-      override def getFileExtension(context: TaskAttemptContext): String = {
-        ".txt" + CodecStreams.getCompressionExtension(context)
-      }
-    }
-  }
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
index 6c40104e52a5f..7d3aa7440c45a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
@@ -32,6 +32,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.{ByteCodeStats, CodeFor
 import org.apache.spark.sql.catalyst.plans.physical.Partitioning
 import org.apache.spark.sql.catalyst.trees.TreeNodeRef
 import org.apache.spark.sql.catalyst.util.StringUtils.StringConcat
+import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, QueryStageExec}
 import org.apache.spark.sql.execution.streaming.{StreamExecution, StreamingQueryWrapper}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.streaming.StreamingQuery
@@ -107,12 +108,22 @@ package object debug {
    */
   def codegenStringSeq(plan: SparkPlan): Seq[(String, String, ByteCodeStats)] = {
     val codegenSubtrees = new collection.mutable.HashSet[WholeStageCodegenExec]()
-    plan transform {
-      case s: WholeStageCodegenExec =>
-        codegenSubtrees += s
-        s
-      case s => s
+
+    def findSubtrees(plan: SparkPlan): Unit = {
+      plan foreach {
+        case s: WholeStageCodegenExec =>
+          codegenSubtrees += s
+        case p: AdaptiveSparkPlanExec =>
+          // Find subtrees from current executed plan of AQE.
+          findSubtrees(p.executedPlan)
+        case s: QueryStageExec =>
+          findSubtrees(s.plan)
+        case s =>
+          s.subqueries.foreach(findSubtrees)
+      }
     }
+
+    findSubtrees(plan)
     codegenSubtrees.toSeq.sortBy(_.codegenStageId).map { subtree =>
       val (_, source) = subtree.doCodeGen()
       val codeStats = try {
@@ -283,5 +294,8 @@ package object debug {
     }
 
     override def supportsColumnar: Boolean = child.supportsColumnar
+
+    override protected def withNewChildInternal(newChild: SparkPlan): DebugExec =
+      copy(child = newChild)
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/CleanupDynamicPruningFilters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/CleanupDynamicPruningFilters.scala
index 1cf55a2f2954d..75ac71e62282b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/CleanupDynamicPruningFilters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/CleanupDynamicPruningFilters.scala
@@ -23,7 +23,6 @@ import org.apache.spark.sql.catalyst.planning.PhysicalOperation
 import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan}
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation}
-import org.apache.spark.sql.internal.SQLConf
 
 /**
  *  Removes the filter nodes with dynamic pruning that were not pushed down to the scan.
@@ -33,7 +32,7 @@ import org.apache.spark.sql.internal.SQLConf
 object CleanupDynamicPruningFilters extends Rule[LogicalPlan] with PredicateHelper {
 
   override def apply(plan: LogicalPlan): LogicalPlan = {
-    if (!SQLConf.get.dynamicPartitionPruningEnabled) {
+    if (!conf.dynamicPartitionPruningEnabled) {
       return plan
     }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PartitionPruning.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PartitionPruning.scala
index 43c6581632687..1d828b139665d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PartitionPruning.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PartitionPruning.scala
@@ -18,12 +18,12 @@
 package org.apache.spark.sql.execution.dynamicpruning
 
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.optimizer.JoinSelectionHelper
 import org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation}
-import org.apache.spark.sql.internal.SQLConf
 
 /**
  * Dynamic partition pruning optimization is performed based on the type and
@@ -34,7 +34,7 @@ import org.apache.spark.sql.internal.SQLConf
  * The basic mechanism for DPP inserts a duplicated subquery with the filter from the other side,
  * when the following conditions are met:
  *    (1) the table to prune is partitioned by the JOIN key
- *    (2) the join operation is one of the following types: INNER, LEFT SEMI (partitioned on left),
+ *    (2) the join operation is one of the following types: INNER, LEFT SEMI,
  *    LEFT OUTER (partitioned on right), or RIGHT OUTER (partitioned on left)
  *
  * In order to enable partition pruning directly in broadcasts, we use a custom DynamicPruning
@@ -46,7 +46,7 @@ import org.apache.spark.sql.internal.SQLConf
  *    subquery query twice, we keep the duplicated subquery
  *    (3) otherwise, we drop the subquery.
  */
-object PartitionPruning extends Rule[LogicalPlan] with PredicateHelper {
+object PartitionPruning extends Rule[LogicalPlan] with PredicateHelper with JoinSelectionHelper {
 
   /**
    * Search the partitioned table scan for a given partition column in a logical plan
@@ -85,10 +85,13 @@ object PartitionPruning extends Rule[LogicalPlan] with PredicateHelper {
       filteringKey: Expression,
       filteringPlan: LogicalPlan,
       joinKeys: Seq[Expression],
-      hasBenefit: Boolean): LogicalPlan = {
-    val reuseEnabled = SQLConf.get.exchangeReuseEnabled
+      partScan: LogicalRelation,
+      canBuildBroadcast: Boolean): LogicalPlan = {
+    val reuseEnabled = conf.exchangeReuseEnabled
     val index = joinKeys.indexOf(filteringKey)
-    if (hasBenefit || reuseEnabled) {
+    lazy val hasBenefit =
+      pruningHasBenefit(pruningKey, partScan, filteringKey, filteringPlan, canBuildBroadcast)
+    if (reuseEnabled || hasBenefit) {
       // insert a DynamicPruning wrapper to identify the subquery during query planning
       Filter(
         DynamicPruningSubquery(
@@ -96,7 +99,7 @@ object PartitionPruning extends Rule[LogicalPlan] with PredicateHelper {
           filteringPlan,
           joinKeys,
           index,
-          !hasBenefit || SQLConf.get.dynamicPartitionPruningReuseBroadcastOnly),
+          conf.dynamicPartitionPruningReuseBroadcastOnly || !hasBenefit),
         pruningPlan)
     } else {
       // abort dynamic partition pruning
@@ -105,17 +108,19 @@ object PartitionPruning extends Rule[LogicalPlan] with PredicateHelper {
   }
 
   /**
-   * Given an estimated filtering ratio we assume the partition pruning has benefit if
+   * Given an estimated filtering ratio(and extra filter ratio if filtering side can't
+   * build broadcast by join type) we assume the partition pruning has benefit if
    * the size in bytes of the partitioned plan after filtering is greater than the size
    * in bytes of the plan on the other side of the join. We estimate the filtering ratio
    * using column statistics if they are available, otherwise we use the config value of
-   * `spark.sql.optimizer.joinFilterRatio`.
+   * `spark.sql.optimizer.dynamicPartitionPruning.fallbackFilterRatio`.
    */
   private def pruningHasBenefit(
       partExpr: Expression,
       partPlan: LogicalPlan,
       otherExpr: Expression,
-      otherPlan: LogicalPlan): Boolean = {
+      otherPlan: LogicalPlan,
+      canBuildBroadcast: Boolean): Boolean = {
 
     // get the distinct counts of an attribute for a given table
     def distinctCounts(attr: Attribute, plan: LogicalPlan): Option[BigInt] = {
@@ -124,12 +129,12 @@ object PartitionPruning extends Rule[LogicalPlan] with PredicateHelper {
 
     // the default filtering ratio when CBO stats are missing, but there is a
     // predicate that is likely to be selective
-    val fallbackRatio = SQLConf.get.dynamicPartitionPruningFallbackFilterRatio
+    val fallbackRatio = conf.dynamicPartitionPruningFallbackFilterRatio
     // the filtering ratio based on the type of the join condition and on the column statistics
     val filterRatio = (partExpr.references.toList, otherExpr.references.toList) match {
       // filter out expressions with more than one attribute on any side of the operator
       case (leftAttr :: Nil, rightAttr :: Nil)
-        if SQLConf.get.dynamicPartitionPruningUseStats =>
+        if conf.dynamicPartitionPruningUseStats =>
           // get the CBO stats for each attribute in the join condition
           val partDistinctCount = distinctCounts(leftAttr, partPlan)
           val otherDistinctCount = distinctCounts(rightAttr, otherPlan)
@@ -146,10 +151,18 @@ object PartitionPruning extends Rule[LogicalPlan] with PredicateHelper {
       case _ => fallbackRatio
     }
 
+    val estimatePruningSideSize = filterRatio * partPlan.stats.sizeInBytes.toFloat
     // the pruning overhead is the total size in bytes of all scan relations
     val overhead = otherPlan.collectLeaves().map(_.stats.sizeInBytes).sum.toFloat
-
-    filterRatio * partPlan.stats.sizeInBytes.toFloat > overhead.toFloat
+    if (canBuildBroadcast) {
+      estimatePruningSideSize > overhead
+    } else {
+      // We can't reuse the broadcast because the join type doesn't support broadcast,
+      // and doing DPP means running an extra query that may have significant overhead.
+      // We need to make sure the pruning side is very big so that DPP is still worthy.
+      canBroadcastBySize(otherPlan, conf) &&
+        estimatePruningSideSize * conf.dynamicPartitionPruningPruningSideExtraFilterRatio > overhead
+    }
   }
 
   /**
@@ -159,10 +172,11 @@ object PartitionPruning extends Rule[LogicalPlan] with PredicateHelper {
     case Not(expr) => isLikelySelective(expr)
     case And(l, r) => isLikelySelective(l) || isLikelySelective(r)
     case Or(l, r) => isLikelySelective(l) && isLikelySelective(r)
-    case Like(_, _, _) => true
+    case _: StringRegexExpression => true
     case _: BinaryComparison => true
     case _: In | _: InSet => true
     case _: StringPredicate => true
+    case _: MultiLikeBase => true
     case _ => false
   }
 
@@ -192,7 +206,7 @@ object PartitionPruning extends Rule[LogicalPlan] with PredicateHelper {
   }
 
   private def canPruneRight(joinType: JoinType): Boolean = joinType match {
-    case Inner | LeftOuter => true
+    case Inner | LeftSemi | LeftOuter => true
     case _ => false
   }
 
@@ -234,14 +248,14 @@ object PartitionPruning extends Rule[LogicalPlan] with PredicateHelper {
             var partScan = getPartitionTableScan(l, left)
             if (partScan.isDefined && canPruneLeft(joinType) &&
                 hasPartitionPruningFilter(right)) {
-              val hasBenefit = pruningHasBenefit(l, partScan.get, r, right)
-              newLeft = insertPredicate(l, newLeft, r, right, rightKeys, hasBenefit)
+              newLeft = insertPredicate(l, newLeft, r, right, rightKeys, partScan.get,
+                canBuildBroadcastRight(joinType))
             } else {
               partScan = getPartitionTableScan(r, right)
               if (partScan.isDefined && canPruneRight(joinType) &&
                   hasPartitionPruningFilter(left) ) {
-                val hasBenefit = pruningHasBenefit(r, partScan.get, l, left)
-                newRight = insertPredicate(r, newRight, l, left, leftKeys, hasBenefit)
+                newRight = insertPredicate(r, newRight, l, left, leftKeys, partScan.get,
+                  canBuildBroadcastLeft(joinType))
               }
             }
           case _ =>
@@ -253,7 +267,7 @@ object PartitionPruning extends Rule[LogicalPlan] with PredicateHelper {
   override def apply(plan: LogicalPlan): LogicalPlan = plan match {
     // Do not rewrite subqueries.
     case s: Subquery if s.correlated => plan
-    case _ if !SQLConf.get.dynamicPartitionPruningEnabled => plan
+    case _ if !conf.dynamicPartitionPruningEnabled => plan
     case _ => prune(plan)
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PlanDynamicPruningFilters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PlanDynamicPruningFilters.scala
index eb091758910cd..9a05e396d4a70 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PlanDynamicPruningFilters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PlanDynamicPruningFilters.scala
@@ -19,14 +19,15 @@ package org.apache.spark.sql.execution.dynamicpruning
 
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.expressions
-import org.apache.spark.sql.catalyst.expressions.{Alias, BindReferences, DynamicPruningExpression, DynamicPruningSubquery, Expression, ListQuery, Literal, PredicateHelper}
-import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LogicalPlan}
+import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeSeq, BindReferences, DynamicPruningExpression, DynamicPruningSubquery, Expression, ListQuery, Literal, PredicateHelper}
+import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight}
+import org.apache.spark.sql.catalyst.plans.logical.Aggregate
 import org.apache.spark.sql.catalyst.plans.physical.BroadcastMode
 import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreePattern.DYNAMIC_PRUNING_SUBQUERY
 import org.apache.spark.sql.execution.{InSubqueryExec, QueryExecution, SparkPlan, SubqueryBroadcastExec}
 import org.apache.spark.sql.execution.exchange.BroadcastExchangeExec
 import org.apache.spark.sql.execution.joins._
-import org.apache.spark.sql.internal.SQLConf
 
 /**
  * This planner rule aims at rewriting dynamic pruning predicates in order to reuse the
@@ -39,35 +40,35 @@ case class PlanDynamicPruningFilters(sparkSession: SparkSession)
   /**
    * Identify the shape in which keys of a given plan are broadcasted.
    */
-  private def broadcastMode(keys: Seq[Expression], plan: LogicalPlan): BroadcastMode = {
-    val packedKeys = BindReferences.bindReferences(HashJoin.rewriteKeyExpr(keys), plan.output)
+  private def broadcastMode(keys: Seq[Expression], output: AttributeSeq): BroadcastMode = {
+    val packedKeys = BindReferences.bindReferences(HashJoin.rewriteKeyExpr(keys), output)
     HashedRelationBroadcastMode(packedKeys)
   }
 
   override def apply(plan: SparkPlan): SparkPlan = {
-    if (!SQLConf.get.dynamicPartitionPruningEnabled) {
+    if (!conf.dynamicPartitionPruningEnabled) {
       return plan
     }
 
-    plan transformAllExpressions {
+    plan.transformAllExpressionsWithPruning(_.containsPattern(DYNAMIC_PRUNING_SUBQUERY)) {
       case DynamicPruningSubquery(
           value, buildPlan, buildKeys, broadcastKeyIndex, onlyInBroadcast, exprId) =>
         val sparkPlan = QueryExecution.createSparkPlan(
           sparkSession, sparkSession.sessionState.planner, buildPlan)
         // Using `sparkPlan` is a little hacky as it is based on the assumption that this rule is
         // the first to be applied (apart from `InsertAdaptiveSparkPlan`).
-        val canReuseExchange = SQLConf.get.exchangeReuseEnabled && buildKeys.nonEmpty &&
+        val canReuseExchange = conf.exchangeReuseEnabled && buildKeys.nonEmpty &&
           plan.find {
-            case BroadcastHashJoinExec(_, _, _, BuildLeft, _, left, _) =>
+            case BroadcastHashJoinExec(_, _, _, BuildLeft, _, left, _, _) =>
               left.sameResult(sparkPlan)
-            case BroadcastHashJoinExec(_, _, _, BuildRight, _, _, right) =>
+            case BroadcastHashJoinExec(_, _, _, BuildRight, _, _, right, _) =>
               right.sameResult(sparkPlan)
             case _ => false
           }.isDefined
 
         if (canReuseExchange) {
-          val mode = broadcastMode(buildKeys, buildPlan)
           val executedPlan = QueryExecution.prepareExecutedPlan(sparkSession, sparkPlan)
+          val mode = broadcastMode(buildKeys, executedPlan.output)
           // plan a broadcast exchange of the build side of the join
           val exchange = BroadcastExchangeExec(mode, executedPlan)
           val name = s"dynamicpruning#${exprId.id}"
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/BroadcastExchangeExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/BroadcastExchangeExec.scala
index d35bbe9b8adc0..94a8a8f0d9e5c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/BroadcastExchangeExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/BroadcastExchangeExec.scala
@@ -29,6 +29,7 @@ import org.apache.spark.launcher.SparkLauncher
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.UnsafeRow
+import org.apache.spark.sql.catalyst.plans.logical.Statistics
 import org.apache.spark.sql.catalyst.plans.physical.{BroadcastMode, BroadcastPartitioning, Partitioning}
 import org.apache.spark.sql.execution.{SparkPlan, SQLExecution}
 import org.apache.spark.sql.execution.joins.HashedRelation
@@ -37,19 +38,47 @@ import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
 import org.apache.spark.unsafe.map.BytesToBytesMap
 import org.apache.spark.util.{SparkFatalException, ThreadUtils}
 
+/**
+ * Common trait for all broadcast exchange implementations to facilitate pattern matching.
+ */
+trait BroadcastExchangeLike extends Exchange {
+
+  /**
+   * The broadcast job group ID
+   */
+  def runId: UUID = UUID.randomUUID
+
+  /**
+   * The asynchronous job that prepares the broadcast relation.
+   */
+  def relationFuture: Future[broadcast.Broadcast[Any]]
+
+  /**
+   * For registering callbacks on `relationFuture`.
+   * Note that calling this method may not start the execution of broadcast job.
+   */
+  def completionFuture: scala.concurrent.Future[broadcast.Broadcast[Any]]
+
+  /**
+   * Returns the runtime statistics after broadcast materialization.
+   */
+  def runtimeStatistics: Statistics
+}
+
 /**
  * A [[BroadcastExchangeExec]] collects, transforms and finally broadcasts the result of
  * a transformed SparkPlan.
  */
 case class BroadcastExchangeExec(
     mode: BroadcastMode,
-    child: SparkPlan) extends Exchange {
+    child: SparkPlan) extends BroadcastExchangeLike {
   import BroadcastExchangeExec._
 
-  private[sql] val runId: UUID = UUID.randomUUID
+  override val runId: UUID = UUID.randomUUID
 
   override lazy val metrics = Map(
     "dataSize" -> SQLMetrics.createSizeMetric(sparkContext, "data size"),
+    "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
     "collectTime" -> SQLMetrics.createTimingMetric(sparkContext, "time to collect"),
     "buildTime" -> SQLMetrics.createTimingMetric(sparkContext, "time to build"),
     "broadcastTime" -> SQLMetrics.createTimingMetric(sparkContext, "time to broadcast"))
@@ -60,21 +89,24 @@ case class BroadcastExchangeExec(
     BroadcastExchangeExec(mode.canonicalized, child.canonicalized)
   }
 
+  override def runtimeStatistics: Statistics = {
+    val dataSize = metrics("dataSize").value
+    val rowCount = metrics("numOutputRows").value
+    Statistics(dataSize, Some(rowCount))
+  }
+
   @transient
   private lazy val promise = Promise[broadcast.Broadcast[Any]]()
 
-  /**
-   * For registering callbacks on `relationFuture`.
-   * Note that calling this field will not start the execution of broadcast job.
-   */
   @transient
-  lazy val completionFuture: scala.concurrent.Future[broadcast.Broadcast[Any]] = promise.future
+  override lazy val completionFuture: scala.concurrent.Future[broadcast.Broadcast[Any]] =
+    promise.future
 
   @transient
-  private val timeout: Long = SQLConf.get.broadcastTimeout
+  private val timeout: Long = conf.broadcastTimeout
 
   @transient
-  private[sql] lazy val relationFuture: Future[broadcast.Broadcast[Any]] = {
+  override lazy val relationFuture: Future[broadcast.Broadcast[Any]] = {
     SQLExecution.withThreadLocalCaptured[broadcast.Broadcast[Any]](
       sqlContext.sparkSession, BroadcastExchangeExec.executionContext) {
           try {
@@ -84,6 +116,7 @@ case class BroadcastExchangeExec(
             val beforeCollect = System.nanoTime()
             // Use executeCollect/executeCollectIterator to avoid conversion to Scala types
             val (numRows, input) = child.executeCollectIterator()
+            longMetric("numOutputRows") += numRows
             if (numRows >= MAX_BROADCAST_TABLE_ROWS) {
               throw new SparkException(
                 s"Cannot broadcast the table over $MAX_BROADCAST_TABLE_ROWS rows: $numRows rows")
@@ -172,6 +205,9 @@ case class BroadcastExchangeExec(
           ex)
     }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): BroadcastExchangeExec =
+    copy(child = newChild)
 }
 
 object BroadcastExchangeExec {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala
index 28ef793ed62db..0d1c4f2c25418 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala
@@ -25,7 +25,6 @@ import org.apache.spark.sql.catalyst.plans.physical._
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.execution.joins.{ShuffledHashJoinExec, SortMergeJoinExec}
-import org.apache.spark.sql.internal.SQLConf
 
 /**
  * Ensures that the [[org.apache.spark.sql.catalyst.plans.physical.Partitioning Partitioning]]
@@ -34,13 +33,7 @@ import org.apache.spark.sql.internal.SQLConf
  * each operator by inserting [[ShuffleExchangeExec]] Operators where required.  Also ensure that
  * the input partition ordering requirements are met.
  */
-case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] {
-  private def defaultNumPreShufflePartitions: Int =
-    if (conf.adaptiveExecutionEnabled && conf.coalesceShufflePartitionsEnabled) {
-      conf.initialShufflePartitionNum
-    } else {
-      conf.numShufflePartitions
-    }
+object EnsureRequirements extends Rule[SparkPlan] {
 
   private def ensureDistributionAndOrdering(operator: SparkPlan): SparkPlan = {
     val requiredChildDistributions: Seq[Distribution] = operator.requiredChildDistribution
@@ -57,7 +50,7 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] {
         BroadcastExchangeExec(mode, child)
       case (child, distribution) =>
         val numPartitions = distribution.requiredNumPartitions
-          .getOrElse(defaultNumPreShufflePartitions)
+          .getOrElse(conf.numShufflePartitions)
         ShuffleExchangeExec(distribution.createPartitioning(numPartitions), child)
     }
 
@@ -91,11 +84,16 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] {
         childrenIndexes.map(children).filterNot(_.isInstanceOf[ShuffleExchangeExec])
           .map(_.outputPartitioning.numPartitions)
       val expectedChildrenNumPartitions = if (nonShuffleChildrenNumPartitions.nonEmpty) {
-        // Here we pick the max number of partitions among these non-shuffle children as the
-        // expected number of shuffle partitions. However, if it's smaller than
-        // `conf.numShufflePartitions`, we pick `conf.numShufflePartitions` as the
-        // expected number of shuffle partitions.
-        math.max(nonShuffleChildrenNumPartitions.max, conf.numShufflePartitions)
+        if (nonShuffleChildrenNumPartitions.length == childrenIndexes.length) {
+          // Here we pick the max number of partitions among these non-shuffle children.
+          nonShuffleChildrenNumPartitions.max
+        } else {
+          // Here we pick the max number of partitions among these non-shuffle children as the
+          // expected number of shuffle partitions. However, if it's smaller than
+          // `conf.numShufflePartitions`, we pick `conf.numShufflePartitions` as the
+          // expected number of shuffle partitions.
+          math.max(nonShuffleChildrenNumPartitions.max, conf.defaultNumShufflePartitions)
+        }
       } else {
         childrenNumPartitions.max
       }
@@ -136,9 +134,14 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] {
       leftKeys: IndexedSeq[Expression],
       rightKeys: IndexedSeq[Expression],
       expectedOrderOfKeys: Seq[Expression],
-      currentOrderOfKeys: Seq[Expression]): (Seq[Expression], Seq[Expression]) = {
+      currentOrderOfKeys: Seq[Expression]): Option[(Seq[Expression], Seq[Expression])] = {
     if (expectedOrderOfKeys.size != currentOrderOfKeys.size) {
-      return (leftKeys, rightKeys)
+      return None
+    }
+
+    // Check if the current order already satisfies the expected order.
+    if (expectedOrderOfKeys.zip(currentOrderOfKeys).forall(p => p._1.semanticEquals(p._2))) {
+      return Some(leftKeys, rightKeys)
     }
 
     // Build a lookup between an expression and the positions its holds in the current key seq.
@@ -165,10 +168,10 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] {
           rightKeysBuffer += rightKeys(index)
         case _ =>
           // The expression cannot be found, or we have exhausted all indices for that expression.
-          return (leftKeys, rightKeys)
+          return None
       }
     }
-    (leftKeysBuffer, rightKeysBuffer)
+    Some(leftKeysBuffer.toSeq, rightKeysBuffer.toSeq)
   }
 
   private def reorderJoinKeys(
@@ -177,19 +180,48 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] {
       leftPartitioning: Partitioning,
       rightPartitioning: Partitioning): (Seq[Expression], Seq[Expression]) = {
     if (leftKeys.forall(_.deterministic) && rightKeys.forall(_.deterministic)) {
-      (leftPartitioning, rightPartitioning) match {
-        case (HashPartitioning(leftExpressions, _), _) =>
-          reorder(leftKeys.toIndexedSeq, rightKeys.toIndexedSeq, leftExpressions, leftKeys)
-        case (_, HashPartitioning(rightExpressions, _)) =>
-          reorder(leftKeys.toIndexedSeq, rightKeys.toIndexedSeq, rightExpressions, rightKeys)
-        case _ =>
-          (leftKeys, rightKeys)
-      }
+      reorderJoinKeysRecursively(
+        leftKeys,
+        rightKeys,
+        Some(leftPartitioning),
+        Some(rightPartitioning))
+        .getOrElse((leftKeys, rightKeys))
     } else {
       (leftKeys, rightKeys)
     }
   }
 
+  /**
+   * Recursively reorders the join keys based on partitioning. It starts reordering the
+   * join keys to match HashPartitioning on either side, followed by PartitioningCollection.
+   */
+  private def reorderJoinKeysRecursively(
+      leftKeys: Seq[Expression],
+      rightKeys: Seq[Expression],
+      leftPartitioning: Option[Partitioning],
+      rightPartitioning: Option[Partitioning]): Option[(Seq[Expression], Seq[Expression])] = {
+    (leftPartitioning, rightPartitioning) match {
+      case (Some(HashPartitioning(leftExpressions, _)), _) =>
+        reorder(leftKeys.toIndexedSeq, rightKeys.toIndexedSeq, leftExpressions, leftKeys)
+          .orElse(reorderJoinKeysRecursively(
+            leftKeys, rightKeys, None, rightPartitioning))
+      case (_, Some(HashPartitioning(rightExpressions, _))) =>
+        reorder(leftKeys.toIndexedSeq, rightKeys.toIndexedSeq, rightExpressions, rightKeys)
+          .orElse(reorderJoinKeysRecursively(
+            leftKeys, rightKeys, leftPartitioning, None))
+      case (Some(PartitioningCollection(partitionings)), _) =>
+        partitionings.foldLeft(Option.empty[(Seq[Expression], Seq[Expression])]) { (res, p) =>
+          res.orElse(reorderJoinKeysRecursively(leftKeys, rightKeys, Some(p), rightPartitioning))
+        }.orElse(reorderJoinKeysRecursively(leftKeys, rightKeys, None, rightPartitioning))
+      case (_, Some(PartitioningCollection(partitionings))) =>
+        partitionings.foldLeft(Option.empty[(Seq[Expression], Seq[Expression])]) { (res, p) =>
+          res.orElse(reorderJoinKeysRecursively(leftKeys, rightKeys, leftPartitioning, Some(p)))
+        }.orElse(None)
+      case _ =>
+        None
+    }
+  }
+
   /**
    * When the physical operators are created for JOIN, the ordering of join keys is based on order
    * in which the join keys appear in the user query. That might not match with the output
@@ -199,17 +231,18 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] {
    */
   private def reorderJoinPredicates(plan: SparkPlan): SparkPlan = {
     plan match {
-      case ShuffledHashJoinExec(leftKeys, rightKeys, joinType, buildSide, condition, left, right) =>
+      case ShuffledHashJoinExec(
+        leftKeys, rightKeys, joinType, buildSide, condition, left, right, isSkew) =>
         val (reorderedLeftKeys, reorderedRightKeys) =
           reorderJoinKeys(leftKeys, rightKeys, left.outputPartitioning, right.outputPartitioning)
         ShuffledHashJoinExec(reorderedLeftKeys, reorderedRightKeys, joinType, buildSide, condition,
-          left, right)
+          left, right, isSkew)
 
-      case SortMergeJoinExec(leftKeys, rightKeys, joinType, condition, left, right, isPartial) =>
+      case SortMergeJoinExec(leftKeys, rightKeys, joinType, condition, left, right, isSkew) =>
         val (reorderedLeftKeys, reorderedRightKeys) =
           reorderJoinKeys(leftKeys, rightKeys, left.outputPartitioning, right.outputPartitioning)
         SortMergeJoinExec(reorderedLeftKeys, reorderedRightKeys, joinType, condition,
-          left, right, isPartial)
+          left, right, isSkew)
 
       case other => other
     }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/Exchange.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/Exchange.scala
index c4062879c2727..e58733b35990a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/Exchange.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/Exchange.scala
@@ -27,7 +27,6 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeMap, Expre
 import org.apache.spark.sql.catalyst.plans.physical.Partitioning
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution._
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.vectorized.ColumnarBatch
 
@@ -100,7 +99,7 @@ case class ReusedExchangeExec(override val output: Seq[Attribute], child: Exchan
  * Find out duplicated exchanges in the spark plan, then use the same exchange for all the
  * references.
  */
-case class ReuseExchange(conf: SQLConf) extends Rule[SparkPlan] {
+object ReuseExchange extends Rule[SparkPlan] {
 
   def apply(plan: SparkPlan): SparkPlan = {
     if (!conf.exchangeReuseEnabled) {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala
index b06742e8470c7..6ec376764a38f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala
@@ -29,9 +29,9 @@ import org.apache.spark.serializer.Serializer
 import org.apache.spark.shuffle.{ShuffleWriteMetricsReporter, ShuffleWriteProcessor}
 import org.apache.spark.shuffle.sort.SortShuffleManager
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.errors._
-import org.apache.spark.sql.catalyst.expressions.{Attribute, BoundReference, Divide, Literal, UnsafeProjection, UnsafeRow}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, BoundReference, UnsafeProjection, UnsafeRow}
 import org.apache.spark.sql.catalyst.expressions.codegen.LazilyGeneratedOrdering
+import org.apache.spark.sql.catalyst.plans.logical.Statistics
 import org.apache.spark.sql.catalyst.plans.physical._
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics, SQLShuffleReadMetricsReporter, SQLShuffleWriteMetricsReporter}
@@ -40,13 +40,66 @@ import org.apache.spark.sql.types.StructType
 import org.apache.spark.util.MutablePair
 import org.apache.spark.util.collection.unsafe.sort.{PrefixComparators, RecordComparator}
 
+/**
+ * Common trait for all shuffle exchange implementations to facilitate pattern matching.
+ */
+trait ShuffleExchangeLike extends Exchange {
+
+  /**
+   * Returns the number of mappers of this shuffle.
+   */
+  def numMappers: Int
+
+  /**
+   * Returns the shuffle partition number.
+   */
+  def numPartitions: Int
+
+  /**
+   * The origin of this shuffle operator.
+   */
+  def shuffleOrigin: ShuffleOrigin
+
+  /**
+   * The asynchronous job that materializes the shuffle.
+   */
+  def mapOutputStatisticsFuture: Future[MapOutputStatistics]
+
+  /**
+   * Returns the shuffle RDD with specified partition specs.
+   */
+  def getShuffleRDD(partitionSpecs: Array[ShufflePartitionSpec]): RDD[_]
+
+  /**
+   * Returns the runtime statistics after shuffle materialization.
+   */
+  def runtimeStatistics: Statistics
+}
+
+// Describes where the shuffle operator comes from.
+sealed trait ShuffleOrigin
+
+// Indicates that the shuffle operator was added by the internal `EnsureRequirements` rule. It
+// means that the shuffle operator is used to ensure internal data partitioning requirements and
+// Spark is free to optimize it as long as the requirements are still ensured.
+case object ENSURE_REQUIREMENTS extends ShuffleOrigin
+
+// Indicates that the shuffle operator was added by the user-specified repartition operator. Spark
+// can still optimize it via changing shuffle partition number, as data partitioning won't change.
+case object REPARTITION extends ShuffleOrigin
+
+// Indicates that the shuffle operator was added by the user-specified repartition operator with
+// a certain partition number. Spark can't optimize it.
+case object REPARTITION_WITH_NUM extends ShuffleOrigin
+
 /**
  * Performs a shuffle that will result in the desired partitioning.
  */
 case class ShuffleExchangeExec(
     override val outputPartitioning: Partitioning,
     child: SparkPlan,
-    canChangeNumPartitions: Boolean = true) extends Exchange {
+    shuffleOrigin: ShuffleOrigin = ENSURE_REQUIREMENTS)
+  extends ShuffleExchangeLike {
 
   private lazy val writeMetrics =
     SQLShuffleWriteMetricsReporter.createShuffleWriteMetrics(sparkContext)
@@ -64,7 +117,7 @@ case class ShuffleExchangeExec(
   @transient lazy val inputRDD: RDD[InternalRow] = child.execute()
 
   // 'mapOutputStatisticsFuture' is only needed when enable AQE.
-  @transient lazy val mapOutputStatisticsFuture: Future[MapOutputStatistics] = {
+  @transient override lazy val mapOutputStatisticsFuture: Future[MapOutputStatistics] = {
     if (inputRDD.getNumPartitions == 0) {
       Future.successful(null)
     } else {
@@ -72,6 +125,20 @@ case class ShuffleExchangeExec(
     }
   }
 
+  override def numMappers: Int = shuffleDependency.rdd.getNumPartitions
+
+  override def numPartitions: Int = shuffleDependency.partitioner.numPartitions
+
+  override def getShuffleRDD(partitionSpecs: Array[ShufflePartitionSpec]): RDD[InternalRow] = {
+    new ShuffledRowRDD(shuffleDependency, readMetrics, partitionSpecs)
+  }
+
+  override def runtimeStatistics: Statistics = {
+    val dataSize = metrics("dataSize").value
+    val rowCount = metrics(SQLShuffleWriteMetricsReporter.SHUFFLE_RECORDS_WRITTEN).value
+    Statistics(dataSize, Some(rowCount))
+  }
+
   /**
    * A [[ShuffleDependency]] that will partition rows of its child based on
    * the partitioning scheme defined in `newPartitioning`. Those partitions of
@@ -92,13 +159,16 @@ case class ShuffleExchangeExec(
    */
   private var cachedShuffleRDD: ShuffledRowRDD = null
 
-  protected override def doExecute(): RDD[InternalRow] = attachTree(this, "execute") {
+  protected override def doExecute(): RDD[InternalRow] = {
     // Returns the same ShuffleRowRDD if this plan is used by multiple plans.
     if (cachedShuffleRDD == null) {
       cachedShuffleRDD = new ShuffledRowRDD(shuffleDependency, readMetrics)
     }
     cachedShuffleRDD
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): ShuffleExchangeExec =
+    copy(child = newChild)
 }
 
 object ShuffleExchangeExec {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala
index 08128d8f69dab..ccbcaa2573f64 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala
@@ -17,18 +17,19 @@
 
 package org.apache.spark.sql.execution.joins
 
+import scala.collection.mutable
+
 import org.apache.spark.TaskContext
 import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen._
-import org.apache.spark.sql.catalyst.expressions.codegen.Block._
+import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide}
 import org.apache.spark.sql.catalyst.plans._
-import org.apache.spark.sql.catalyst.plans.physical.{BroadcastDistribution, Distribution, UnspecifiedDistribution}
+import org.apache.spark.sql.catalyst.plans.physical.{BroadcastDistribution, Distribution, HashPartitioning, Partitioning, PartitioningCollection, UnspecifiedDistribution}
 import org.apache.spark.sql.execution.{CodegenSupport, SparkPlan}
 import org.apache.spark.sql.execution.metric.SQLMetrics
-import org.apache.spark.sql.types.{BooleanType, LongType}
 
 /**
  * Performs an inner hash join of two child relations.  When the output RDD of this operator is
@@ -43,14 +44,23 @@ case class BroadcastHashJoinExec(
     buildSide: BuildSide,
     condition: Option[Expression],
     left: SparkPlan,
-    right: SparkPlan)
-  extends HashJoin with CodegenSupport {
+    right: SparkPlan,
+    isNullAwareAntiJoin: Boolean = false)
+  extends HashJoin {
+
+  if (isNullAwareAntiJoin) {
+    require(leftKeys.length == 1, "leftKeys length should be 1")
+    require(rightKeys.length == 1, "rightKeys length should be 1")
+    require(joinType == LeftAnti, "joinType must be LeftAnti.")
+    require(buildSide == BuildRight, "buildSide must be BuildRight.")
+    require(condition.isEmpty, "null aware anti join optimize condition should be empty.")
+  }
 
   override lazy val metrics = Map(
     "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"))
 
   override def requiredChildDistribution: Seq[Distribution] = {
-    val mode = HashedRelationBroadcastMode(buildKeys)
+    val mode = HashedRelationBroadcastMode(buildBoundKeys, isNullAwareAntiJoin)
     buildSide match {
       case BuildLeft =>
         BroadcastDistribution(mode) :: UnspecifiedDistribution :: Nil
@@ -59,14 +69,108 @@ case class BroadcastHashJoinExec(
     }
   }
 
+  override lazy val outputPartitioning: Partitioning = {
+    joinType match {
+      case _: InnerLike if sqlContext.conf.broadcastHashJoinOutputPartitioningExpandLimit > 0 =>
+        streamedPlan.outputPartitioning match {
+          case h: HashPartitioning => expandOutputPartitioning(h)
+          case c: PartitioningCollection => expandOutputPartitioning(c)
+          case other => other
+        }
+      case _ => streamedPlan.outputPartitioning
+    }
+  }
+
+  // An one-to-many mapping from a streamed key to build keys.
+  private lazy val streamedKeyToBuildKeyMapping = {
+    val mapping = mutable.Map.empty[Expression, Seq[Expression]]
+    streamedKeys.zip(buildKeys).foreach {
+      case (streamedKey, buildKey) =>
+        val key = streamedKey.canonicalized
+        mapping.get(key) match {
+          case Some(v) => mapping.put(key, v :+ buildKey)
+          case None => mapping.put(key, Seq(buildKey))
+        }
+    }
+    mapping.toMap
+  }
+
+  // Expands the given partitioning collection recursively.
+  private def expandOutputPartitioning(
+      partitioning: PartitioningCollection): PartitioningCollection = {
+    PartitioningCollection(partitioning.partitionings.flatMap {
+      case h: HashPartitioning => expandOutputPartitioning(h).partitionings
+      case c: PartitioningCollection => Seq(expandOutputPartitioning(c))
+      case other => Seq(other)
+    })
+  }
+
+  // Expands the given hash partitioning by substituting streamed keys with build keys.
+  // For example, if the expressions for the given partitioning are Seq("a", "b", "c")
+  // where the streamed keys are Seq("b", "c") and the build keys are Seq("x", "y"),
+  // the expanded partitioning will have the following expressions:
+  // Seq("a", "b", "c"), Seq("a", "b", "y"), Seq("a", "x", "c"), Seq("a", "x", "y").
+  // The expanded expressions are returned as PartitioningCollection.
+  private def expandOutputPartitioning(partitioning: HashPartitioning): PartitioningCollection = {
+    val maxNumCombinations = sqlContext.conf.broadcastHashJoinOutputPartitioningExpandLimit
+    var currentNumCombinations = 0
+
+    def generateExprCombinations(
+        current: Seq[Expression],
+        accumulated: Seq[Expression]): Seq[Seq[Expression]] = {
+      if (currentNumCombinations >= maxNumCombinations) {
+        Nil
+      } else if (current.isEmpty) {
+        currentNumCombinations += 1
+        Seq(accumulated)
+      } else {
+        val buildKeysOpt = streamedKeyToBuildKeyMapping.get(current.head.canonicalized)
+        generateExprCombinations(current.tail, accumulated :+ current.head) ++
+          buildKeysOpt.map(_.flatMap(b => generateExprCombinations(current.tail, accumulated :+ b)))
+            .getOrElse(Nil)
+      }
+    }
+
+    PartitioningCollection(
+      generateExprCombinations(partitioning.expressions, Nil)
+        .map(HashPartitioning(_, partitioning.numPartitions)))
+  }
+
   protected override def doExecute(): RDD[InternalRow] = {
     val numOutputRows = longMetric("numOutputRows")
 
     val broadcastRelation = buildPlan.executeBroadcast[HashedRelation]()
-    streamedPlan.execute().mapPartitions { streamedIter =>
-      val hashed = broadcastRelation.value.asReadOnlyCopy()
-      TaskContext.get().taskMetrics().incPeakExecutionMemory(hashed.estimatedSize)
-      join(streamedIter, hashed, numOutputRows)
+    if (isNullAwareAntiJoin) {
+      streamedPlan.execute().mapPartitionsInternal { streamedIter =>
+        val hashed = broadcastRelation.value.asReadOnlyCopy()
+        TaskContext.get().taskMetrics().incPeakExecutionMemory(hashed.estimatedSize)
+        if (hashed == EmptyHashedRelation) {
+          streamedIter
+        } else if (hashed == HashedRelationWithAllNullKeys) {
+          Iterator.empty
+        } else {
+          val keyGenerator = UnsafeProjection.create(
+            BindReferences.bindReferences[Expression](
+              leftKeys,
+              AttributeSeq(left.output))
+          )
+          streamedIter.filter(row => {
+            val lookupKey: UnsafeRow = keyGenerator(row)
+            if (lookupKey.anyNull()) {
+              false
+            } else {
+              // Anti Join: Drop the row on the streamed side if it is a match on the build
+              hashed.get(lookupKey) == null
+            }
+          })
+        }
+      }
+    } else {
+      streamedPlan.execute().mapPartitions { streamedIter =>
+        val hashed = broadcastRelation.value.asReadOnlyCopy()
+        TaskContext.get().taskMetrics().incPeakExecutionMemory(hashed.estimatedSize)
+        join(streamedIter, hashed, numOutputRows)
+      }
     }
   }
 
@@ -91,23 +195,6 @@ case class BroadcastHashJoinExec(
   override def needCopyResult: Boolean =
     streamedPlan.asInstanceOf[CodegenSupport].needCopyResult || multipleOutputForOneInput
 
-  override def doProduce(ctx: CodegenContext): String = {
-    streamedPlan.asInstanceOf[CodegenSupport].produce(ctx, this)
-  }
-
-  override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = {
-    joinType match {
-      case _: InnerLike => codegenInner(ctx, input)
-      case LeftOuter | RightOuter => codegenOuter(ctx, input)
-      case LeftSemi => codegenSemi(ctx, input)
-      case LeftAnti => codegenAnti(ctx, input)
-      case j: ExistenceJoin => codegenExistence(ctx, input)
-      case x =>
-        throw new IllegalArgumentException(
-          s"BroadcastHashJoin should not take $x as the JoinType")
-    }
-  }
-
   /**
    * Returns a tuple of Broadcast of HashedRelation and the variable name for it.
    */
@@ -126,377 +213,49 @@ case class BroadcastHashJoinExec(
     (broadcastRelation, relationTerm)
   }
 
-  /**
-   * Returns the code for generating join key for stream side, and expression of whether the key
-   * has any null in it or not.
-   */
-  private def genStreamSideJoinKey(
-      ctx: CodegenContext,
-      input: Seq[ExprCode]): (ExprCode, String) = {
-    ctx.currentVars = input
-    if (streamedKeys.length == 1 && streamedKeys.head.dataType == LongType) {
-      // generate the join key as Long
-      val ev = streamedKeys.head.genCode(ctx)
-      (ev, ev.isNull)
-    } else {
-      // generate the join key as UnsafeRow
-      val ev = GenerateUnsafeProjection.createCode(ctx, streamedKeys)
-      (ev, s"${ev.value}.anyNull()")
-    }
-  }
-
-  /**
-   * Generates the code for variable of build side.
-   */
-  private def genBuildSideVars(ctx: CodegenContext, matched: String): Seq[ExprCode] = {
-    ctx.currentVars = null
-    ctx.INPUT_ROW = matched
-    buildPlan.output.zipWithIndex.map { case (a, i) =>
-      val ev = BoundReference(i, a.dataType, a.nullable).genCode(ctx)
-      if (joinType.isInstanceOf[InnerLike]) {
-        ev
-      } else {
-        // the variables are needed even there is no matched rows
-        val isNull = ctx.freshName("isNull")
-        val value = ctx.freshName("value")
-        val javaType = CodeGenerator.javaType(a.dataType)
-        val code = code"""
-          |boolean $isNull = true;
-          |$javaType $value = ${CodeGenerator.defaultValue(a.dataType)};
-          |if ($matched != null) {
-          |  ${ev.code}
-          |  $isNull = ${ev.isNull};
-          |  $value = ${ev.value};
-          |}
-         """.stripMargin
-        ExprCode(code, JavaCode.isNullVariable(isNull), JavaCode.variable(value, a.dataType))
-      }
-    }
-  }
-
-  /**
-   * Generate the (non-equi) condition used to filter joined rows. This is used in Inner, Left Semi
-   * and Left Anti joins.
-   */
-  private def getJoinCondition(
-      ctx: CodegenContext,
-      input: Seq[ExprCode]): (String, String, Seq[ExprCode]) = {
-    val matched = ctx.freshName("matched")
-    val buildVars = genBuildSideVars(ctx, matched)
-    val checkCondition = if (condition.isDefined) {
-      val expr = condition.get
-      // evaluate the variables from build side that used by condition
-      val eval = evaluateRequiredVariables(buildPlan.output, buildVars, expr.references)
-      // filter the output via condition
-      ctx.currentVars = input ++ buildVars
-      val ev =
-        BindReferences.bindReference(expr, streamedPlan.output ++ buildPlan.output).genCode(ctx)
-      val skipRow = s"${ev.isNull} || !${ev.value}"
-      s"""
-         |$eval
-         |${ev.code}
-         |if (!($skipRow))
-       """.stripMargin
-    } else {
-      ""
-    }
-    (matched, checkCondition, buildVars)
-  }
-
-  /**
-   * Generates the code for Inner join.
-   */
-  private def codegenInner(ctx: CodegenContext, input: Seq[ExprCode]): String = {
+  protected override def prepareRelation(ctx: CodegenContext): HashedRelationInfo = {
     val (broadcastRelation, relationTerm) = prepareBroadcast(ctx)
-    val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input)
-    val (matched, checkCondition, buildVars) = getJoinCondition(ctx, input)
-    val numOutput = metricTerm(ctx, "numOutputRows")
-
-    val resultVars = buildSide match {
-      case BuildLeft => buildVars ++ input
-      case BuildRight => input ++ buildVars
-    }
-    if (broadcastRelation.value.keyIsUnique) {
-      s"""
-         |// generate join key for stream side
-         |${keyEv.code}
-         |// find matches from HashedRelation
-         |UnsafeRow $matched = $anyNull ? null: (UnsafeRow)$relationTerm.getValue(${keyEv.value});
-         |if ($matched != null) {
-         |  $checkCondition {
-         |    $numOutput.add(1);
-         |    ${consume(ctx, resultVars)}
-         |  }
-         |}
-       """.stripMargin
-
-    } else {
-      val matches = ctx.freshName("matches")
-      val iteratorCls = classOf[Iterator[UnsafeRow]].getName
-      s"""
-         |// generate join key for stream side
-         |${keyEv.code}
-         |// find matches from HashRelation
-         |$iteratorCls $matches = $anyNull ? null : ($iteratorCls)$relationTerm.get(${keyEv.value});
-         |if ($matches != null) {
-         |  while ($matches.hasNext()) {
-         |    UnsafeRow $matched = (UnsafeRow) $matches.next();
-         |    $checkCondition {
-         |      $numOutput.add(1);
-         |      ${consume(ctx, resultVars)}
-         |    }
-         |  }
-         |}
-       """.stripMargin
-    }
-  }
-
-  /**
-   * Generates the code for left or right outer join.
-   */
-  private def codegenOuter(ctx: CodegenContext, input: Seq[ExprCode]): String = {
-    val (broadcastRelation, relationTerm) = prepareBroadcast(ctx)
-    val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input)
-    val matched = ctx.freshName("matched")
-    val buildVars = genBuildSideVars(ctx, matched)
-    val numOutput = metricTerm(ctx, "numOutputRows")
-
-    // filter the output via condition
-    val conditionPassed = ctx.freshName("conditionPassed")
-    val checkCondition = if (condition.isDefined) {
-      val expr = condition.get
-      // evaluate the variables from build side that used by condition
-      val eval = evaluateRequiredVariables(buildPlan.output, buildVars, expr.references)
-      ctx.currentVars = input ++ buildVars
-      val ev =
-        BindReferences.bindReference(expr, streamedPlan.output ++ buildPlan.output).genCode(ctx)
-      s"""
-         |boolean $conditionPassed = true;
-         |${eval.trim}
-         |if ($matched != null) {
-         |  ${ev.code}
-         |  $conditionPassed = !${ev.isNull} && ${ev.value};
-         |}
-       """.stripMargin
-    } else {
-      s"final boolean $conditionPassed = true;"
-    }
-
-    val resultVars = buildSide match {
-      case BuildLeft => buildVars ++ input
-      case BuildRight => input ++ buildVars
-    }
-    if (broadcastRelation.value.keyIsUnique) {
-      s"""
-         |// generate join key for stream side
-         |${keyEv.code}
-         |// find matches from HashedRelation
-         |UnsafeRow $matched = $anyNull ? null: (UnsafeRow)$relationTerm.getValue(${keyEv.value});
-         |${checkCondition.trim}
-         |if (!$conditionPassed) {
-         |  $matched = null;
-         |  // reset the variables those are already evaluated.
-         |  ${buildVars.filter(_.code.isEmpty).map(v => s"${v.isNull} = true;").mkString("\n")}
-         |}
-         |$numOutput.add(1);
-         |${consume(ctx, resultVars)}
-       """.stripMargin
-
-    } else {
-      val matches = ctx.freshName("matches")
-      val iteratorCls = classOf[Iterator[UnsafeRow]].getName
-      val found = ctx.freshName("found")
-      s"""
-         |// generate join key for stream side
-         |${keyEv.code}
-         |// find matches from HashRelation
-         |$iteratorCls $matches = $anyNull ? null : ($iteratorCls)$relationTerm.get(${keyEv.value});
-         |boolean $found = false;
-         |// the last iteration of this loop is to emit an empty row if there is no matched rows.
-         |while ($matches != null && $matches.hasNext() || !$found) {
-         |  UnsafeRow $matched = $matches != null && $matches.hasNext() ?
-         |    (UnsafeRow) $matches.next() : null;
-         |  ${checkCondition.trim}
-         |  if ($conditionPassed) {
-         |    $found = true;
-         |    $numOutput.add(1);
-         |    ${consume(ctx, resultVars)}
-         |  }
-         |}
-       """.stripMargin
-    }
-  }
-
-  /**
-   * Generates the code for left semi join.
-   */
-  private def codegenSemi(ctx: CodegenContext, input: Seq[ExprCode]): String = {
-    val (broadcastRelation, relationTerm) = prepareBroadcast(ctx)
-    val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input)
-    val (matched, checkCondition, _) = getJoinCondition(ctx, input)
-    val numOutput = metricTerm(ctx, "numOutputRows")
-    if (broadcastRelation.value.keyIsUnique) {
-      s"""
-         |// generate join key for stream side
-         |${keyEv.code}
-         |// find matches from HashedRelation
-         |UnsafeRow $matched = $anyNull ? null: (UnsafeRow)$relationTerm.getValue(${keyEv.value});
-         |if ($matched != null) {
-         |  $checkCondition {
-         |    $numOutput.add(1);
-         |    ${consume(ctx, input)}
-         |  }
-         |}
-       """.stripMargin
-    } else {
-      val matches = ctx.freshName("matches")
-      val iteratorCls = classOf[Iterator[UnsafeRow]].getName
-      val found = ctx.freshName("found")
-      s"""
-         |// generate join key for stream side
-         |${keyEv.code}
-         |// find matches from HashRelation
-         |$iteratorCls $matches = $anyNull ? null : ($iteratorCls)$relationTerm.get(${keyEv.value});
-         |if ($matches != null) {
-         |  boolean $found = false;
-         |  while (!$found && $matches.hasNext()) {
-         |    UnsafeRow $matched = (UnsafeRow) $matches.next();
-         |    $checkCondition {
-         |      $found = true;
-         |    }
-         |  }
-         |  if ($found) {
-         |    $numOutput.add(1);
-         |    ${consume(ctx, input)}
-         |  }
-         |}
-       """.stripMargin
-    }
+    HashedRelationInfo(relationTerm,
+      broadcastRelation.value.keyIsUnique,
+      broadcastRelation.value == EmptyHashedRelation)
   }
 
   /**
    * Generates the code for anti join.
+   * Handles NULL-aware anti join (NAAJ) separately here.
    */
-  private def codegenAnti(ctx: CodegenContext, input: Seq[ExprCode]): String = {
-    val (broadcastRelation, relationTerm) = prepareBroadcast(ctx)
-    val uniqueKeyCodePath = broadcastRelation.value.keyIsUnique
-    val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input)
-    val (matched, checkCondition, _) = getJoinCondition(ctx, input)
-    val numOutput = metricTerm(ctx, "numOutputRows")
+  protected override def codegenAnti(ctx: CodegenContext, input: Seq[ExprCode]): String = {
+    if (isNullAwareAntiJoin) {
+      val (broadcastRelation, relationTerm) = prepareBroadcast(ctx)
+      val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input)
+      val numOutput = metricTerm(ctx, "numOutputRows")
 
-    if (uniqueKeyCodePath) {
-      val found = ctx.freshName("found")
-      s"""
-         |boolean $found = false;
-         |// generate join key for stream side
-         |${keyEv.code}
-         |// Check if the key has nulls.
-         |if (!($anyNull)) {
-         |  // Check if the HashedRelation exists.
-         |  UnsafeRow $matched = (UnsafeRow)$relationTerm.getValue(${keyEv.value});
-         |  if ($matched != null) {
-         |    // Evaluate the condition.
-         |    $checkCondition {
-         |      $found = true;
-         |    }
-         |  }
-         |}
-         |if (!$found) {
-         |  $numOutput.add(1);
-         |  ${consume(ctx, input)}
-         |}
-       """.stripMargin
+      if (broadcastRelation.value == EmptyHashedRelation) {
+        s"""
+           |// If the right side is empty, NAAJ simply returns the left side.
+           |$numOutput.add(1);
+           |${consume(ctx, input)}
+         """.stripMargin
+      } else if (broadcastRelation.value == HashedRelationWithAllNullKeys) {
+        s"""
+           |// If the right side contains any all-null key, NAAJ simply returns Nothing.
+         """.stripMargin
+      } else {
+        s"""
+           |// generate join key for stream side
+           |${keyEv.code}
+           |if (!$anyNull && $relationTerm.getValue(${keyEv.value}) == null) {
+           |  $numOutput.add(1);
+           |  ${consume(ctx, input)}
+           |}
+         """.stripMargin
+      }
     } else {
-      val matches = ctx.freshName("matches")
-      val iteratorCls = classOf[Iterator[UnsafeRow]].getName
-      val found = ctx.freshName("found")
-      s"""
-         |boolean $found = false;
-         |// generate join key for stream side
-         |${keyEv.code}
-         |// Check if the key has nulls.
-         |if (!($anyNull)) {
-         |  // Check if the HashedRelation exists.
-         |  $iteratorCls $matches = ($iteratorCls)$relationTerm.get(${keyEv.value});
-         |  if ($matches != null) {
-         |    // Evaluate the condition.
-         |    while (!$found && $matches.hasNext()) {
-         |      UnsafeRow $matched = (UnsafeRow) $matches.next();
-         |      $checkCondition {
-         |        $found = true;
-         |      }
-         |    }
-         |  }
-         |}
-         |if (!$found) {
-         |  $numOutput.add(1);
-         |  ${consume(ctx, input)}
-         |}
-       """.stripMargin
+      super.codegenAnti(ctx, input)
     }
   }
 
-  /**
-   * Generates the code for existence join.
-   */
-  private def codegenExistence(ctx: CodegenContext, input: Seq[ExprCode]): String = {
-    val (broadcastRelation, relationTerm) = prepareBroadcast(ctx)
-    val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input)
-    val numOutput = metricTerm(ctx, "numOutputRows")
-    val existsVar = ctx.freshName("exists")
-
-    val matched = ctx.freshName("matched")
-    val buildVars = genBuildSideVars(ctx, matched)
-    val checkCondition = if (condition.isDefined) {
-      val expr = condition.get
-      // evaluate the variables from build side that used by condition
-      val eval = evaluateRequiredVariables(buildPlan.output, buildVars, expr.references)
-      // filter the output via condition
-      ctx.currentVars = input ++ buildVars
-      val ev =
-        BindReferences.bindReference(expr, streamedPlan.output ++ buildPlan.output).genCode(ctx)
-      s"""
-         |$eval
-         |${ev.code}
-         |$existsVar = !${ev.isNull} && ${ev.value};
-       """.stripMargin
-    } else {
-      s"$existsVar = true;"
-    }
-
-    val resultVar = input ++ Seq(ExprCode.forNonNullValue(
-      JavaCode.variable(existsVar, BooleanType)))
-    if (broadcastRelation.value.keyIsUnique) {
-      s"""
-         |// generate join key for stream side
-         |${keyEv.code}
-         |// find matches from HashedRelation
-         |UnsafeRow $matched = $anyNull ? null: (UnsafeRow)$relationTerm.getValue(${keyEv.value});
-         |boolean $existsVar = false;
-         |if ($matched != null) {
-         |  $checkCondition
-         |}
-         |$numOutput.add(1);
-         |${consume(ctx, resultVar)}
-       """.stripMargin
-    } else {
-      val matches = ctx.freshName("matches")
-      val iteratorCls = classOf[Iterator[UnsafeRow]].getName
-      s"""
-         |// generate join key for stream side
-         |${keyEv.code}
-         |// find matches from HashRelation
-         |$iteratorCls $matches = $anyNull ? null : ($iteratorCls)$relationTerm.get(${keyEv.value});
-         |boolean $existsVar = false;
-         |if ($matches != null) {
-         |  while (!$existsVar && $matches.hasNext()) {
-         |    UnsafeRow $matched = (UnsafeRow) $matches.next();
-         |    $checkCondition
-         |  }
-         |}
-         |$numOutput.add(1);
-         |${consume(ctx, resultVar)}
-       """.stripMargin
-    }
-  }
+  override protected def withNewChildrenInternal(
+      newLeft: SparkPlan, newRight: SparkPlan): BroadcastHashJoinExec =
+    copy(left = newLeft, right = newRight)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNestedLoopJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNestedLoopJoinExec.scala
index 888e7af7c07ed..77a30b7b02b58 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNestedLoopJoinExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNestedLoopJoinExec.scala
@@ -21,9 +21,11 @@ import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
+import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide}
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.physical._
-import org.apache.spark.sql.execution.{ExplainUtils, SparkPlan}
+import org.apache.spark.sql.execution.{CodegenSupport, ExplainUtils, SparkPlan}
 import org.apache.spark.sql.execution.metric.SQLMetrics
 import org.apache.spark.util.collection.{BitSet, CompactBuffer}
 
@@ -32,7 +34,7 @@ case class BroadcastNestedLoopJoinExec(
     right: SparkPlan,
     buildSide: BuildSide,
     joinType: JoinType,
-    condition: Option[Expression]) extends BaseJoinExec {
+    condition: Option[Expression]) extends JoinCodegenSupport {
 
   override def leftKeys: Seq[Expression] = Nil
   override def rightKeys: Seq[Expression] = Nil
@@ -40,7 +42,7 @@ case class BroadcastNestedLoopJoinExec(
   override lazy val metrics = Map(
     "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"))
 
-  /** BuildRight means the right relation <=> the broadcast relation. */
+  /** BuildRight means the right relation is the broadcast relation. */
   private val (streamed, broadcast) = buildSide match {
     case BuildRight => (left, right)
     case BuildLeft => (right, left)
@@ -48,7 +50,7 @@ case class BroadcastNestedLoopJoinExec(
 
   override def simpleStringWithNodeId(): String = {
     val opId = ExplainUtils.getOpId(this)
-    s"$nodeName $joinType ${buildSide} ($opId)".trim
+    s"$nodeName $joinType $buildSide ($opId)".trim
   }
 
   override def requiredChildDistribution: Seq[Distribution] = buildSide match {
@@ -58,10 +60,22 @@ case class BroadcastNestedLoopJoinExec(
       UnspecifiedDistribution :: BroadcastDistribution(IdentityBroadcastMode) :: Nil
   }
 
+  override def outputPartitioning: Partitioning = (joinType, buildSide) match {
+    case (_: InnerLike, _) | (LeftOuter, BuildRight) | (RightOuter, BuildLeft) |
+         (LeftSemi, BuildRight) | (LeftAnti, BuildRight) => streamed.outputPartitioning
+    case _ => super.outputPartitioning
+  }
+
+  override def outputOrdering: Seq[SortOrder] = (joinType, buildSide) match {
+    case (_: InnerLike, _) | (LeftOuter, BuildRight) | (RightOuter, BuildLeft) |
+         (LeftSemi, BuildRight) | (LeftAnti, BuildRight) => streamed.outputOrdering
+    case _ => Nil
+  }
+
   private[this] def genResultProjection: UnsafeProjection = joinType match {
-    case LeftExistence(j) =>
+    case LeftExistence(_) =>
       UnsafeProjection.create(output, output)
-    case other =>
+    case _ =>
       // Always put the stream side on left to simplify implementation
       // both of left and right side could be null
       UnsafeProjection.create(
@@ -179,125 +193,99 @@ case class BroadcastNestedLoopJoinExec(
   }
 
   /**
-   * The implementation for these joins:
-   *
-   *   LeftSemi with BuildRight
-   *   Anti with BuildRight
+   * The implementation for LeftSemi and LeftAnti joins.
    */
   private def leftExistenceJoin(
       relation: Broadcast[Array[InternalRow]],
       exists: Boolean): RDD[InternalRow] = {
-    assert(buildSide == BuildRight)
-    streamed.execute().mapPartitionsInternal { streamedIter =>
-      val buildRows = relation.value
-      val joinedRow = new JoinedRow
-
-      if (condition.isDefined) {
-        streamedIter.filter(l =>
-          buildRows.exists(r => boundCondition(joinedRow(l, r))) == exists
-        )
-      } else if (buildRows.nonEmpty == exists) {
-        streamedIter
-      } else {
-        Iterator.empty
-      }
-    }
-  }
-
-  private def existenceJoin(relation: Broadcast[Array[InternalRow]]): RDD[InternalRow] = {
-    assert(buildSide == BuildRight)
-    streamed.execute().mapPartitionsInternal { streamedIter =>
-      val buildRows = relation.value
-      val joinedRow = new JoinedRow
-
-      if (condition.isDefined) {
-        val resultRow = new GenericInternalRow(Array[Any](null))
-        streamedIter.map { row =>
-          val result = buildRows.exists(r => boundCondition(joinedRow(row, r)))
-          resultRow.setBoolean(0, result)
-          joinedRow(row, resultRow)
+    buildSide match {
+      case BuildRight =>
+        streamed.execute().mapPartitionsInternal { streamedIter =>
+          val buildRows = relation.value
+          val joinedRow = new JoinedRow
+
+          if (condition.isDefined) {
+            streamedIter.filter(l =>
+              buildRows.exists(r => boundCondition(joinedRow(l, r))) == exists
+            )
+          } else if (buildRows.nonEmpty == exists) {
+            streamedIter
+          } else {
+            Iterator.empty
+          }
         }
-      } else {
-        val resultRow = new GenericInternalRow(Array[Any](buildRows.nonEmpty))
-        streamedIter.map { row =>
-          joinedRow(row, resultRow)
+      case BuildLeft if condition.isEmpty =>
+        // If condition is empty, do not need to read rows from streamed side at all.
+        // Only need to know whether streamed side is empty or not.
+        val streamExists = !streamed.executeTake(1).isEmpty
+        if (streamExists == exists) {
+          sparkContext.makeRDD(relation.value)
+        } else {
+          sparkContext.emptyRDD
         }
-      }
-    }
-  }
-
-  /**
-   * The implementation for these joins:
-   *
-   *   LeftOuter with BuildLeft
-   *   RightOuter with BuildRight
-   *   FullOuter
-   *   LeftSemi with BuildLeft
-   *   LeftAnti with BuildLeft
-   *   ExistenceJoin with BuildLeft
-   */
-  private def defaultJoin(relation: Broadcast[Array[InternalRow]]): RDD[InternalRow] = {
-    /** All rows that either match both-way, or rows from streamed joined with nulls. */
-    val streamRdd = streamed.execute()
-
-    val matchedBuildRows = streamRdd.mapPartitionsInternal { streamedIter =>
-      val buildRows = relation.value
-      val matched = new BitSet(buildRows.length)
-      val joinedRow = new JoinedRow
-
-      streamedIter.foreach { streamedRow =>
+      case _ => // BuildLeft
+        val matchedBroadcastRows = getMatchedBroadcastRowsBitSet(streamed.execute(), relation)
+        val buf: CompactBuffer[InternalRow] = new CompactBuffer()
         var i = 0
+        val buildRows = relation.value
         while (i < buildRows.length) {
-          if (boundCondition(joinedRow(streamedRow, buildRows(i)))) {
-            matched.set(i)
+          if (matchedBroadcastRows.get(i) == exists) {
+            buf += buildRows(i).copy()
           }
           i += 1
         }
-      }
-      Seq(matched).toIterator
+        sparkContext.makeRDD(buf)
     }
+  }
 
-    val matchedBroadcastRows = matchedBuildRows.fold(
-      new BitSet(relation.value.length)
-    )(_ | _)
-
-    joinType match {
-      case LeftSemi =>
-        assert(buildSide == BuildLeft)
-        val buf: CompactBuffer[InternalRow] = new CompactBuffer()
-        var i = 0
-        val rel = relation.value
-        while (i < rel.length) {
-          if (matchedBroadcastRows.get(i)) {
-            buf += rel(i).copy()
+  /**
+   * The implementation for ExistenceJoin
+   */
+  private def existenceJoin(relation: Broadcast[Array[InternalRow]]): RDD[InternalRow] = {
+    buildSide match {
+      case BuildRight =>
+        streamed.execute().mapPartitionsInternal { streamedIter =>
+          val buildRows = relation.value
+          val joinedRow = new JoinedRow
+
+          if (condition.isDefined) {
+            val resultRow = new GenericInternalRow(Array[Any](null))
+            streamedIter.map { row =>
+              val result = buildRows.exists(r => boundCondition(joinedRow(row, r)))
+              resultRow.setBoolean(0, result)
+              joinedRow(row, resultRow)
+            }
+          } else {
+            val resultRow = new GenericInternalRow(Array[Any](buildRows.nonEmpty))
+            streamedIter.map { row =>
+              joinedRow(row, resultRow)
+            }
           }
-          i += 1
         }
-        return sparkContext.makeRDD(buf)
-      case j: ExistenceJoin =>
+      case _ => // BuildLeft
+        val matchedBroadcastRows = getMatchedBroadcastRowsBitSet(streamed.execute(), relation)
         val buf: CompactBuffer[InternalRow] = new CompactBuffer()
         var i = 0
-        val rel = relation.value
-        while (i < rel.length) {
+        val buildRows = relation.value
+        while (i < buildRows.length) {
           val result = new GenericInternalRow(Array[Any](matchedBroadcastRows.get(i)))
-          buf += new JoinedRow(rel(i).copy(), result)
+          buf += new JoinedRow(buildRows(i).copy(), result)
           i += 1
         }
-        return sparkContext.makeRDD(buf)
-      case LeftAnti =>
-        val notMatched: CompactBuffer[InternalRow] = new CompactBuffer()
-        var i = 0
-        val rel = relation.value
-        while (i < rel.length) {
-          if (!matchedBroadcastRows.get(i)) {
-            notMatched += rel(i).copy()
-          }
-          i += 1
-        }
-        return sparkContext.makeRDD(notMatched)
-      case o =>
+        sparkContext.makeRDD(buf)
     }
+  }
 
+  /**
+   * The implementation for these joins:
+   *
+   *   LeftOuter with BuildLeft
+   *   RightOuter with BuildRight
+   *   FullOuter
+   */
+  private def defaultJoin(relation: Broadcast[Array[InternalRow]]): RDD[InternalRow] = {
+    val streamRdd = streamed.execute()
+    val matchedBroadcastRows = getMatchedBroadcastRowsBitSet(streamRdd, relation)
     val notMatchedBroadcastRows: Seq[InternalRow] = {
       val nulls = new GenericInternalRow(streamed.output.size)
       val buf: CompactBuffer[InternalRow] = new CompactBuffer()
@@ -345,6 +333,34 @@ case class BroadcastNestedLoopJoinExec(
     )
   }
 
+  /**
+   * Get matched rows from broadcast side as a [[BitSet]].
+   * Create a local [[BitSet]] for broadcast side on each RDD partition,
+   * and merge all [[BitSet]]s together.
+   */
+  private def getMatchedBroadcastRowsBitSet(
+      streamRdd: RDD[InternalRow],
+      relation: Broadcast[Array[InternalRow]]): BitSet = {
+    val matchedBuildRows = streamRdd.mapPartitionsInternal { streamedIter =>
+      val buildRows = relation.value
+      val matched = new BitSet(buildRows.length)
+      val joinedRow = new JoinedRow
+
+      streamedIter.foreach { streamedRow =>
+        var i = 0
+        while (i < buildRows.length) {
+          if (boundCondition(joinedRow(streamedRow, buildRows(i)))) {
+            matched.set(i)
+          }
+          i += 1
+        }
+      }
+      Seq(matched).toIterator
+    }
+
+    matchedBuildRows.fold(new BitSet(relation.value.length))(_ | _)
+  }
+
   protected override def doExecute(): RDD[InternalRow] = {
     val broadcastedRelation = broadcast.executeBroadcast[Array[InternalRow]]()
 
@@ -353,20 +369,17 @@ case class BroadcastNestedLoopJoinExec(
         innerJoin(broadcastedRelation)
       case (LeftOuter, BuildRight) | (RightOuter, BuildLeft) =>
         outerJoin(broadcastedRelation)
-      case (LeftSemi, BuildRight) =>
+      case (LeftSemi, _) =>
         leftExistenceJoin(broadcastedRelation, exists = true)
-      case (LeftAnti, BuildRight) =>
+      case (LeftAnti, _) =>
         leftExistenceJoin(broadcastedRelation, exists = false)
-      case (j: ExistenceJoin, BuildRight) =>
+      case (_: ExistenceJoin, _) =>
         existenceJoin(broadcastedRelation)
       case _ =>
         /**
          * LeftOuter with BuildLeft
          * RightOuter with BuildRight
          * FullOuter
-         * LeftSemi with BuildLeft
-         * LeftAnti with BuildLeft
-         * ExistenceJoin with BuildLeft
          */
         defaultJoin(broadcastedRelation)
     }
@@ -381,4 +394,162 @@ case class BroadcastNestedLoopJoinExec(
       }
     }
   }
+
+  override def supportCodegen: Boolean = (joinType, buildSide) match {
+    case (_: InnerLike, _) | (LeftOuter, BuildRight) | (RightOuter, BuildLeft) |
+         (LeftSemi | LeftAnti, BuildRight) => true
+    case _ => false
+  }
+
+  override def inputRDDs(): Seq[RDD[InternalRow]] = {
+    streamed.asInstanceOf[CodegenSupport].inputRDDs()
+  }
+
+  override def needCopyResult: Boolean = true
+
+  override def doProduce(ctx: CodegenContext): String = {
+    streamed.asInstanceOf[CodegenSupport].produce(ctx, this)
+  }
+
+  override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = {
+    (joinType, buildSide) match {
+      case (_: InnerLike, _) => codegenInner(ctx, input)
+      case (LeftOuter, BuildRight) | (RightOuter, BuildLeft) => codegenOuter(ctx, input)
+      case (LeftSemi, BuildRight) => codegenLeftExistence(ctx, input, exists = true)
+      case (LeftAnti, BuildRight) => codegenLeftExistence(ctx, input, exists = false)
+      case _ =>
+        throw new IllegalArgumentException(
+          s"BroadcastNestedLoopJoin code-gen should not take neither $joinType as the JoinType " +
+          s"nor $buildSide as the BuildSide")
+    }
+  }
+
+  /**
+   * Returns a tuple of [[Broadcast]] side and the variable name for it.
+   */
+  private def prepareBroadcast(ctx: CodegenContext): (Array[InternalRow], String) = {
+    // Create a name for broadcast side
+    val broadcastArray = broadcast.executeBroadcast[Array[InternalRow]]()
+    val broadcastTerm = ctx.addReferenceObj("broadcastTerm", broadcastArray)
+
+    // Inline mutable state since not many join operations in a task
+    val arrayTerm = ctx.addMutableState("InternalRow[]", "buildRowArray",
+      v => s"$v = (InternalRow[]) $broadcastTerm.value();", forceInline = true)
+    (broadcastArray.value, arrayTerm)
+  }
+
+  private def codegenInner(ctx: CodegenContext, input: Seq[ExprCode]): String = {
+    val (_, buildRowArrayTerm) = prepareBroadcast(ctx)
+    val (buildRow, checkCondition, buildVars) = getJoinCondition(ctx, input, streamed, broadcast)
+
+    val resultVars = buildSide match {
+      case BuildLeft => buildVars ++ input
+      case BuildRight => input ++ buildVars
+    }
+    val arrayIndex = ctx.freshName("arrayIndex")
+    val numOutput = metricTerm(ctx, "numOutputRows")
+
+    s"""
+       |for (int $arrayIndex = 0; $arrayIndex < $buildRowArrayTerm.length; $arrayIndex++) {
+       |  UnsafeRow $buildRow = (UnsafeRow) $buildRowArrayTerm[$arrayIndex];
+       |  $checkCondition {
+       |    $numOutput.add(1);
+       |    ${consume(ctx, resultVars)}
+       |  }
+       |}
+     """.stripMargin
+  }
+
+  private def codegenOuter(ctx: CodegenContext, input: Seq[ExprCode]): String = {
+    val (buildRowArray, buildRowArrayTerm) = prepareBroadcast(ctx)
+    val (buildRow, checkCondition, _) = getJoinCondition(ctx, input, streamed, broadcast)
+    val buildVars = genBuildSideVars(ctx, buildRow, broadcast)
+
+    val resultVars = buildSide match {
+      case BuildLeft => buildVars ++ input
+      case BuildRight => input ++ buildVars
+    }
+    val arrayIndex = ctx.freshName("arrayIndex")
+    val shouldOutputRow = ctx.freshName("shouldOutputRow")
+    val foundMatch = ctx.freshName("foundMatch")
+    val numOutput = metricTerm(ctx, "numOutputRows")
+
+    if (buildRowArray.isEmpty) {
+      s"""
+         |UnsafeRow $buildRow = null;
+         |$numOutput.add(1);
+         |${consume(ctx, resultVars)}
+       """.stripMargin
+    } else {
+      s"""
+         |boolean $foundMatch = false;
+         |for (int $arrayIndex = 0; $arrayIndex < $buildRowArrayTerm.length; $arrayIndex++) {
+         |  UnsafeRow $buildRow = (UnsafeRow) $buildRowArrayTerm[$arrayIndex];
+         |  boolean $shouldOutputRow = false;
+         |  $checkCondition {
+         |    $shouldOutputRow = true;
+         |    $foundMatch = true;
+         |  }
+         |  if ($arrayIndex == $buildRowArrayTerm.length - 1 && !$foundMatch) {
+         |    $buildRow = null;
+         |    $shouldOutputRow = true;
+         |  }
+         |  if ($shouldOutputRow) {
+         |    $numOutput.add(1);
+         |    ${consume(ctx, resultVars)}
+         |  }
+         |}
+       """.stripMargin
+    }
+  }
+
+  private def codegenLeftExistence(
+      ctx: CodegenContext,
+      input: Seq[ExprCode],
+      exists: Boolean): String = {
+    val (buildRowArray, buildRowArrayTerm) = prepareBroadcast(ctx)
+    val numOutput = metricTerm(ctx, "numOutputRows")
+
+    if (condition.isEmpty) {
+      if (buildRowArray.nonEmpty == exists) {
+        // Return streamed side if join condition is empty and
+        // 1. build side is non-empty for LeftSemi join
+        // or
+        // 2. build side is empty for LeftAnti join.
+        s"""
+           |$numOutput.add(1);
+           |${consume(ctx, input)}
+         """.stripMargin
+      } else {
+        // Return nothing if join condition is empty and
+        // 1. build side is empty for LeftSemi join
+        // or
+        // 2. build side is non-empty for LeftAnti join.
+        ""
+      }
+    } else {
+      val (buildRow, checkCondition, _) = getJoinCondition(ctx, input, streamed, broadcast)
+      val foundMatch = ctx.freshName("foundMatch")
+      val arrayIndex = ctx.freshName("arrayIndex")
+
+      s"""
+         |boolean $foundMatch = false;
+         |for (int $arrayIndex = 0; $arrayIndex < $buildRowArrayTerm.length; $arrayIndex++) {
+         |  UnsafeRow $buildRow = (UnsafeRow) $buildRowArrayTerm[$arrayIndex];
+         |  $checkCondition {
+         |    $foundMatch = true;
+         |    break;
+         |  }
+         |}
+         |if ($foundMatch == $exists) {
+         |  $numOutput.add(1);
+         |  ${consume(ctx, input)}
+         |}
+     """.stripMargin
+    }
+  }
+
+  override protected def withNewChildrenInternal(
+      newLeft: SparkPlan, newRight: SparkPlan): BroadcastNestedLoopJoinExec =
+    copy(left = newLeft, right = newRight)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/CartesianProductExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/CartesianProductExec.scala
index a71bf94c45034..1b2d3731f7e8d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/CartesianProductExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/CartesianProductExec.scala
@@ -35,7 +35,6 @@ import org.apache.spark.util.CompletionIterator
 class UnsafeCartesianRDD(
     left : RDD[UnsafeRow],
     right : RDD[UnsafeRow],
-    numFieldsOfRight: Int,
     inMemoryBufferThreshold: Int,
     spillThreshold: Int)
   extends CartesianRDD[UnsafeRow, UnsafeRow](left.sparkContext, left, right) {
@@ -81,7 +80,6 @@ case class CartesianProductExec(
     val pair = new UnsafeCartesianRDD(
       leftResults,
       rightResults,
-      right.output.size,
       sqlContext.conf.cartesianProductExecBufferInMemoryThreshold,
       sqlContext.conf.cartesianProductExecBufferSpillThreshold)
     pair.mapPartitionsWithIndexInternal { (index, iter) =>
@@ -103,4 +101,8 @@ case class CartesianProductExec(
       }
     }
   }
+
+  override protected def withNewChildrenInternal(
+      newLeft: SparkPlan, newRight: SparkPlan): CartesianProductExec =
+    copy(left = newLeft, right = newRight)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
index 7f90a51c1f234..f87acb8b5dac3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
@@ -17,16 +17,29 @@
 
 package org.apache.spark.sql.execution.joins
 
-import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.{InternalRow, SQLConfHelper}
+import org.apache.spark.sql.catalyst.analysis.CastSupport
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.BindReferences.bindReferences
+import org.apache.spark.sql.catalyst.expressions.codegen._
+import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide}
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.physical.Partitioning
-import org.apache.spark.sql.execution.{ExplainUtils, RowIterator}
+import org.apache.spark.sql.execution.{CodegenSupport, ExplainUtils, RowIterator}
 import org.apache.spark.sql.execution.metric.SQLMetric
-import org.apache.spark.sql.types.{IntegralType, LongType}
+import org.apache.spark.sql.types.{BooleanType, IntegralType, LongType}
 
-trait HashJoin extends BaseJoinExec {
+/**
+ * @param relationTerm variable name for HashedRelation
+ * @param keyIsUnique  indicate whether keys of HashedRelation known to be unique in code-gen time
+ * @param isEmpty indicate whether it known to be EmptyHashedRelation in code-gen time
+ */
+private[joins] case class HashedRelationInfo(
+    relationTerm: String,
+    keyIsUnique: Boolean,
+    isEmpty: Boolean)
+
+trait HashJoin extends JoinCodegenSupport {
   def buildSide: BuildSide
 
   override def simpleStringWithNodeId(): String = {
@@ -51,7 +64,41 @@ trait HashJoin extends BaseJoinExec {
     }
   }
 
-  override def outputPartitioning: Partitioning = streamedPlan.outputPartitioning
+  override def outputPartitioning: Partitioning = buildSide match {
+    case BuildLeft =>
+      joinType match {
+        case _: InnerLike | RightOuter => right.outputPartitioning
+        case x =>
+          throw new IllegalArgumentException(
+            s"HashJoin should not take $x as the JoinType with building left side")
+      }
+    case BuildRight =>
+      joinType match {
+        case _: InnerLike | LeftOuter | LeftSemi | LeftAnti | _: ExistenceJoin =>
+          left.outputPartitioning
+        case x =>
+          throw new IllegalArgumentException(
+            s"HashJoin should not take $x as the JoinType with building right side")
+      }
+  }
+
+  override def outputOrdering: Seq[SortOrder] = buildSide match {
+    case BuildLeft =>
+      joinType match {
+        case _: InnerLike | RightOuter => right.outputOrdering
+        case x =>
+          throw new IllegalArgumentException(
+            s"HashJoin should not take $x as the JoinType with building left side")
+      }
+    case BuildRight =>
+      joinType match {
+        case _: InnerLike | LeftOuter | LeftSemi | LeftAnti | _: ExistenceJoin =>
+          left.outputOrdering
+        case x =>
+          throw new IllegalArgumentException(
+            s"HashJoin should not take $x as the JoinType with building right side")
+      }
+  }
 
   protected lazy val (buildPlan, streamedPlan) = buildSide match {
     case BuildLeft => (left, right)
@@ -59,26 +106,44 @@ trait HashJoin extends BaseJoinExec {
   }
 
   protected lazy val (buildKeys, streamedKeys) = {
-    require(leftKeys.map(_.dataType) == rightKeys.map(_.dataType),
-      "Join keys from two sides should have same types")
-    val lkeys = bindReferences(HashJoin.rewriteKeyExpr(leftKeys), left.output)
-    val rkeys = bindReferences(HashJoin.rewriteKeyExpr(rightKeys), right.output)
+    require(leftKeys.length == rightKeys.length &&
+      leftKeys.map(_.dataType)
+        .zip(rightKeys.map(_.dataType))
+        .forall(types => types._1.sameType(types._2)),
+      "Join keys from two sides should have same length and types")
     buildSide match {
-      case BuildLeft => (lkeys, rkeys)
-      case BuildRight => (rkeys, lkeys)
+      case BuildLeft => (leftKeys, rightKeys)
+      case BuildRight => (rightKeys, leftKeys)
     }
   }
 
+  @transient protected lazy val (buildOutput, streamedOutput) = {
+    buildSide match {
+      case BuildLeft => (left.output, right.output)
+      case BuildRight => (right.output, left.output)
+    }
+  }
+
+  @transient protected lazy val buildBoundKeys =
+    bindReferences(HashJoin.rewriteKeyExpr(buildKeys), buildOutput)
 
+  @transient protected lazy val streamedBoundKeys =
+    bindReferences(HashJoin.rewriteKeyExpr(streamedKeys), streamedOutput)
 
   protected def buildSideKeyGenerator(): Projection =
-    UnsafeProjection.create(buildKeys)
+    UnsafeProjection.create(buildBoundKeys)
 
   protected def streamSideKeyGenerator(): UnsafeProjection =
-    UnsafeProjection.create(streamedKeys)
+    UnsafeProjection.create(streamedBoundKeys)
 
-  @transient private[this] lazy val boundCondition = if (condition.isDefined) {
-    Predicate.create(condition.get, streamedPlan.output ++ buildPlan.output).eval _
+  @transient protected[this] lazy val boundCondition = if (condition.isDefined) {
+    if (joinType == FullOuter && buildSide == BuildLeft) {
+      // Put join left side before right side. This is to be consistent with
+      // `ShuffledHashJoinExec.fullOuterJoin`.
+      Predicate.create(condition.get, buildPlan.output ++ streamedPlan.output).eval _
+    } else {
+      Predicate.create(condition.get, streamedPlan.output ++ buildPlan.output).eval _
+    }
   } else {
     (r: InternalRow) => true
   }
@@ -98,47 +163,75 @@ trait HashJoin extends BaseJoinExec {
       hashedRelation: HashedRelation): Iterator[InternalRow] = {
     val joinRow = new JoinedRow
     val joinKeys = streamSideKeyGenerator()
-    streamIter.flatMap { srow =>
-      joinRow.withLeft(srow)
-      val matches = hashedRelation.get(joinKeys(srow))
-      if (matches != null) {
-        matches.map(joinRow.withRight(_)).filter(boundCondition)
-      } else {
-        Seq.empty
+
+    if (hashedRelation == EmptyHashedRelation) {
+      Iterator.empty
+    } else if (hashedRelation.keyIsUnique) {
+      streamIter.flatMap { srow =>
+        joinRow.withLeft(srow)
+        val matched = hashedRelation.getValue(joinKeys(srow))
+        if (matched != null) {
+          Some(joinRow.withRight(matched)).filter(boundCondition)
+        } else {
+          None
+        }
+      }
+    } else {
+      streamIter.flatMap { srow =>
+        joinRow.withLeft(srow)
+        val matches = hashedRelation.get(joinKeys(srow))
+        if (matches != null) {
+          matches.map(joinRow.withRight).filter(boundCondition)
+        } else {
+          Seq.empty
+        }
       }
     }
   }
 
   private def outerJoin(
       streamedIter: Iterator[InternalRow],
-    hashedRelation: HashedRelation): Iterator[InternalRow] = {
+      hashedRelation: HashedRelation): Iterator[InternalRow] = {
     val joinedRow = new JoinedRow()
     val keyGenerator = streamSideKeyGenerator()
     val nullRow = new GenericInternalRow(buildPlan.output.length)
 
-    streamedIter.flatMap { currentRow =>
-      val rowKey = keyGenerator(currentRow)
-      joinedRow.withLeft(currentRow)
-      val buildIter = hashedRelation.get(rowKey)
-      new RowIterator {
-        private var found = false
-        override def advanceNext(): Boolean = {
-          while (buildIter != null && buildIter.hasNext) {
-            val nextBuildRow = buildIter.next()
-            if (boundCondition(joinedRow.withRight(nextBuildRow))) {
+    if (hashedRelation.keyIsUnique) {
+      streamedIter.map { currentRow =>
+        val rowKey = keyGenerator(currentRow)
+        joinedRow.withLeft(currentRow)
+        val matched = hashedRelation.getValue(rowKey)
+        if (matched != null && boundCondition(joinedRow.withRight(matched))) {
+          joinedRow
+        } else {
+          joinedRow.withRight(nullRow)
+        }
+      }
+    } else {
+      streamedIter.flatMap { currentRow =>
+        val rowKey = keyGenerator(currentRow)
+        joinedRow.withLeft(currentRow)
+        val buildIter = hashedRelation.get(rowKey)
+        new RowIterator {
+          private var found = false
+          override def advanceNext(): Boolean = {
+            while (buildIter != null && buildIter.hasNext) {
+              val nextBuildRow = buildIter.next()
+              if (boundCondition(joinedRow.withRight(nextBuildRow))) {
+                found = true
+                return true
+              }
+            }
+            if (!found) {
+              joinedRow.withRight(nullRow)
               found = true
               return true
             }
+            false
           }
-          if (!found) {
-            joinedRow.withRight(nullRow)
-            found = true
-            return true
-          }
-          false
-        }
-        override def getRow: InternalRow = joinedRow
-      }.toScala
+          override def getRow: InternalRow = joinedRow
+        }.toScala
+      }
     }
   }
 
@@ -147,12 +240,24 @@ trait HashJoin extends BaseJoinExec {
       hashedRelation: HashedRelation): Iterator[InternalRow] = {
     val joinKeys = streamSideKeyGenerator()
     val joinedRow = new JoinedRow
-    streamIter.filter { current =>
-      val key = joinKeys(current)
-      lazy val buildIter = hashedRelation.get(key)
-      !key.anyNull && buildIter != null && (condition.isEmpty || buildIter.exists {
-        (row: InternalRow) => boundCondition(joinedRow(current, row))
-      })
+
+    if (hashedRelation == EmptyHashedRelation) {
+      Iterator.empty
+    } else if (hashedRelation.keyIsUnique) {
+      streamIter.filter { current =>
+        val key = joinKeys(current)
+        lazy val matched = hashedRelation.getValue(key)
+        !key.anyNull && matched != null &&
+          (condition.isEmpty || boundCondition(joinedRow(current, matched)))
+      }
+    } else {
+      streamIter.filter { current =>
+        val key = joinKeys(current)
+        lazy val buildIter = hashedRelation.get(key)
+        !key.anyNull && buildIter != null && (condition.isEmpty || buildIter.exists {
+          (row: InternalRow) => boundCondition(joinedRow(current, row))
+        })
+      }
     }
   }
 
@@ -162,28 +267,55 @@ trait HashJoin extends BaseJoinExec {
     val joinKeys = streamSideKeyGenerator()
     val result = new GenericInternalRow(Array[Any](null))
     val joinedRow = new JoinedRow
-    streamIter.map { current =>
-      val key = joinKeys(current)
-      lazy val buildIter = hashedRelation.get(key)
-      val exists = !key.anyNull && buildIter != null && (condition.isEmpty || buildIter.exists {
-        (row: InternalRow) => boundCondition(joinedRow(current, row))
-      })
-      result.setBoolean(0, exists)
-      joinedRow(current, result)
+
+    if (hashedRelation.keyIsUnique) {
+      streamIter.map { current =>
+        val key = joinKeys(current)
+        lazy val matched = hashedRelation.getValue(key)
+        val exists = !key.anyNull && matched != null &&
+          (condition.isEmpty || boundCondition(joinedRow(current, matched)))
+        result.setBoolean(0, exists)
+        joinedRow(current, result)
+      }
+    } else {
+      streamIter.map { current =>
+        val key = joinKeys(current)
+        lazy val buildIter = hashedRelation.get(key)
+        val exists = !key.anyNull && buildIter != null && (condition.isEmpty || buildIter.exists {
+          (row: InternalRow) => boundCondition(joinedRow(current, row))
+        })
+        result.setBoolean(0, exists)
+        joinedRow(current, result)
+      }
     }
   }
 
   private def antiJoin(
       streamIter: Iterator[InternalRow],
       hashedRelation: HashedRelation): Iterator[InternalRow] = {
+    // If the right side is empty, AntiJoin simply returns the left side.
+    if (hashedRelation == EmptyHashedRelation) {
+      return streamIter
+    }
+
     val joinKeys = streamSideKeyGenerator()
     val joinedRow = new JoinedRow
-    streamIter.filter { current =>
-      val key = joinKeys(current)
-      lazy val buildIter = hashedRelation.get(key)
-      key.anyNull || buildIter == null || (condition.isDefined && !buildIter.exists {
-        row => boundCondition(joinedRow(current, row))
-      })
+
+    if (hashedRelation.keyIsUnique) {
+      streamIter.filter { current =>
+        val key = joinKeys(current)
+        lazy val matched = hashedRelation.getValue(key)
+        key.anyNull || matched == null ||
+          (condition.isDefined && !boundCondition(joinedRow(current, matched)))
+      }
+    } else {
+      streamIter.filter { current =>
+        val key = joinKeys(current)
+        lazy val buildIter = hashedRelation.get(key)
+        key.anyNull || buildIter == null || (condition.isDefined && !buildIter.exists {
+          row => boundCondition(joinedRow(current, row))
+        })
+      }
     }
   }
 
@@ -201,11 +333,11 @@ trait HashJoin extends BaseJoinExec {
         semiJoin(streamedIter, hashed)
       case LeftAnti =>
         antiJoin(streamedIter, hashed)
-      case j: ExistenceJoin =>
+      case _: ExistenceJoin =>
         existenceJoin(streamedIter, hashed)
       case x =>
         throw new IllegalArgumentException(
-          s"BroadcastHashJoin should not take $x as the JoinType")
+          s"HashJoin should not take $x as the JoinType")
     }
 
     val resultProj = createResultProjection
@@ -214,9 +346,365 @@ trait HashJoin extends BaseJoinExec {
       resultProj(r)
     }
   }
+
+  override def doProduce(ctx: CodegenContext): String = {
+    streamedPlan.asInstanceOf[CodegenSupport].produce(ctx, this)
+  }
+
+  override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = {
+    joinType match {
+      case _: InnerLike => codegenInner(ctx, input)
+      case LeftOuter | RightOuter => codegenOuter(ctx, input)
+      case LeftSemi => codegenSemi(ctx, input)
+      case LeftAnti => codegenAnti(ctx, input)
+      case _: ExistenceJoin => codegenExistence(ctx, input)
+      case x =>
+        throw new IllegalArgumentException(
+          s"HashJoin should not take $x as the JoinType")
+    }
+  }
+
+  /**
+   * Returns the code for generating join key for stream side, and expression of whether the key
+   * has any null in it or not.
+   */
+  protected def genStreamSideJoinKey(
+      ctx: CodegenContext,
+      input: Seq[ExprCode]): (ExprCode, String) = {
+    ctx.currentVars = input
+    if (streamedBoundKeys.length == 1 && streamedBoundKeys.head.dataType == LongType) {
+      // generate the join key as Long
+      val ev = streamedBoundKeys.head.genCode(ctx)
+      (ev, ev.isNull)
+    } else {
+      // generate the join key as UnsafeRow
+      val ev = GenerateUnsafeProjection.createCode(ctx, streamedBoundKeys)
+      (ev, s"${ev.value}.anyNull()")
+    }
+  }
+
+  /**
+   * Generates the code for Inner join.
+   */
+  protected def codegenInner(ctx: CodegenContext, input: Seq[ExprCode]): String = {
+    val HashedRelationInfo(relationTerm, keyIsUnique, isEmptyHashedRelation) = prepareRelation(ctx)
+    val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input)
+    val (matched, checkCondition, buildVars) = getJoinCondition(ctx, input, streamedPlan, buildPlan)
+    val numOutput = metricTerm(ctx, "numOutputRows")
+
+    val resultVars = buildSide match {
+      case BuildLeft => buildVars ++ input
+      case BuildRight => input ++ buildVars
+    }
+
+    if (isEmptyHashedRelation) {
+      """
+        |// If HashedRelation is empty, hash inner join simply returns nothing.
+      """.stripMargin
+    } else if (keyIsUnique) {
+      s"""
+         |// generate join key for stream side
+         |${keyEv.code}
+         |// find matches from HashedRelation
+         |UnsafeRow $matched = $anyNull ? null: (UnsafeRow)$relationTerm.getValue(${keyEv.value});
+         |if ($matched != null) {
+         |  $checkCondition {
+         |    $numOutput.add(1);
+         |    ${consume(ctx, resultVars)}
+         |  }
+         |}
+       """.stripMargin
+    } else {
+      val matches = ctx.freshName("matches")
+      val iteratorCls = classOf[Iterator[UnsafeRow]].getName
+
+      s"""
+         |// generate join key for stream side
+         |${keyEv.code}
+         |// find matches from HashRelation
+         |$iteratorCls $matches = $anyNull ?
+         |  null : ($iteratorCls)$relationTerm.get(${keyEv.value});
+         |if ($matches != null) {
+         |  while ($matches.hasNext()) {
+         |    UnsafeRow $matched = (UnsafeRow) $matches.next();
+         |    $checkCondition {
+         |      $numOutput.add(1);
+         |      ${consume(ctx, resultVars)}
+         |    }
+         |  }
+         |}
+       """.stripMargin
+    }
+  }
+
+  /**
+   * Generates the code for left or right outer join.
+   */
+  protected def codegenOuter(ctx: CodegenContext, input: Seq[ExprCode]): String = {
+    val HashedRelationInfo(relationTerm, keyIsUnique, _) = prepareRelation(ctx)
+    val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input)
+    val matched = ctx.freshName("matched")
+    val buildVars = genBuildSideVars(ctx, matched, buildPlan)
+    val numOutput = metricTerm(ctx, "numOutputRows")
+
+    // filter the output via condition
+    val conditionPassed = ctx.freshName("conditionPassed")
+    val checkCondition = if (condition.isDefined) {
+      val expr = condition.get
+      // evaluate the variables from build side that used by condition
+      val eval = evaluateRequiredVariables(buildPlan.output, buildVars, expr.references)
+      ctx.currentVars = input ++ buildVars
+      val ev =
+        BindReferences.bindReference(expr, streamedPlan.output ++ buildPlan.output).genCode(ctx)
+      s"""
+         |boolean $conditionPassed = true;
+         |${eval.trim}
+         |if ($matched != null) {
+         |  ${ev.code}
+         |  $conditionPassed = !${ev.isNull} && ${ev.value};
+         |}
+       """.stripMargin
+    } else {
+      s"final boolean $conditionPassed = true;"
+    }
+
+    val resultVars = buildSide match {
+      case BuildLeft => buildVars ++ input
+      case BuildRight => input ++ buildVars
+    }
+
+    if (keyIsUnique) {
+      s"""
+         |// generate join key for stream side
+         |${keyEv.code}
+         |// find matches from HashedRelation
+         |UnsafeRow $matched = $anyNull ? null: (UnsafeRow)$relationTerm.getValue(${keyEv.value});
+         |${checkCondition.trim}
+         |if (!$conditionPassed) {
+         |  $matched = null;
+         |  // reset the variables those are already evaluated.
+         |  ${buildVars.filter(_.code.isEmpty).map(v => s"${v.isNull} = true;").mkString("\n")}
+         |}
+         |$numOutput.add(1);
+         |${consume(ctx, resultVars)}
+       """.stripMargin
+    } else {
+      val matches = ctx.freshName("matches")
+      val iteratorCls = classOf[Iterator[UnsafeRow]].getName
+      val found = ctx.freshName("found")
+
+      s"""
+         |// generate join key for stream side
+         |${keyEv.code}
+         |// find matches from HashRelation
+         |$iteratorCls $matches = $anyNull ? null : ($iteratorCls)$relationTerm.get(${keyEv.value});
+         |boolean $found = false;
+         |// the last iteration of this loop is to emit an empty row if there is no matched rows.
+         |while ($matches != null && $matches.hasNext() || !$found) {
+         |  UnsafeRow $matched = $matches != null && $matches.hasNext() ?
+         |    (UnsafeRow) $matches.next() : null;
+         |  ${checkCondition.trim}
+         |  if ($conditionPassed) {
+         |    $found = true;
+         |    $numOutput.add(1);
+         |    ${consume(ctx, resultVars)}
+         |  }
+         |}
+       """.stripMargin
+    }
+  }
+
+  /**
+   * Generates the code for left semi join.
+   */
+  protected def codegenSemi(ctx: CodegenContext, input: Seq[ExprCode]): String = {
+    val HashedRelationInfo(relationTerm, keyIsUnique, isEmptyHashedRelation) = prepareRelation(ctx)
+    val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input)
+    val (matched, checkCondition, _) = getJoinCondition(ctx, input, streamedPlan, buildPlan)
+    val numOutput = metricTerm(ctx, "numOutputRows")
+
+    if (isEmptyHashedRelation) {
+      """
+        |// If HashedRelation is empty, hash semi join simply returns nothing.
+      """.stripMargin
+    } else if (keyIsUnique) {
+      s"""
+         |// generate join key for stream side
+         |${keyEv.code}
+         |// find matches from HashedRelation
+         |UnsafeRow $matched = $anyNull ? null: (UnsafeRow)$relationTerm.getValue(${keyEv.value});
+         |if ($matched != null) {
+         |  $checkCondition {
+         |    $numOutput.add(1);
+         |    ${consume(ctx, input)}
+         |  }
+         |}
+       """.stripMargin
+    } else {
+      val matches = ctx.freshName("matches")
+      val iteratorCls = classOf[Iterator[UnsafeRow]].getName
+      val found = ctx.freshName("found")
+
+      s"""
+         |// generate join key for stream side
+         |${keyEv.code}
+         |// find matches from HashRelation
+         |$iteratorCls $matches = $anyNull ? null : ($iteratorCls)$relationTerm.get(${keyEv.value});
+         |if ($matches != null) {
+         |  boolean $found = false;
+         |  while (!$found && $matches.hasNext()) {
+         |    UnsafeRow $matched = (UnsafeRow) $matches.next();
+         |    $checkCondition {
+         |      $found = true;
+         |    }
+         |  }
+         |  if ($found) {
+         |    $numOutput.add(1);
+         |    ${consume(ctx, input)}
+         |  }
+         |}
+       """.stripMargin
+    }
+  }
+
+  /**
+   * Generates the code for anti join.
+   */
+  protected def codegenAnti(ctx: CodegenContext, input: Seq[ExprCode]): String = {
+    val HashedRelationInfo(relationTerm, keyIsUnique, isEmptyHashedRelation) = prepareRelation(ctx)
+    val numOutput = metricTerm(ctx, "numOutputRows")
+    if (isEmptyHashedRelation) {
+      return s"""
+                |// If HashedRelation is empty, hash anti join simply returns the stream side.
+                |$numOutput.add(1);
+                |${consume(ctx, input)}
+              """.stripMargin
+    }
+
+    val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input)
+    val (matched, checkCondition, _) = getJoinCondition(ctx, input, streamedPlan, buildPlan)
+
+    if (keyIsUnique) {
+      val found = ctx.freshName("found")
+      s"""
+         |boolean $found = false;
+         |// generate join key for stream side
+         |${keyEv.code}
+         |// Check if the key has nulls.
+         |if (!($anyNull)) {
+         |  // Check if the HashedRelation exists.
+         |  UnsafeRow $matched = (UnsafeRow)$relationTerm.getValue(${keyEv.value});
+         |  if ($matched != null) {
+         |    // Evaluate the condition.
+         |    $checkCondition {
+         |      $found = true;
+         |    }
+         |  }
+         |}
+         |if (!$found) {
+         |  $numOutput.add(1);
+         |  ${consume(ctx, input)}
+         |}
+       """.stripMargin
+    } else {
+      val matches = ctx.freshName("matches")
+      val iteratorCls = classOf[Iterator[UnsafeRow]].getName
+      val found = ctx.freshName("found")
+      s"""
+         |boolean $found = false;
+         |// generate join key for stream side
+         |${keyEv.code}
+         |// Check if the key has nulls.
+         |if (!($anyNull)) {
+         |  // Check if the HashedRelation exists.
+         |  $iteratorCls $matches = ($iteratorCls)$relationTerm.get(${keyEv.value});
+         |  if ($matches != null) {
+         |    // Evaluate the condition.
+         |    while (!$found && $matches.hasNext()) {
+         |      UnsafeRow $matched = (UnsafeRow) $matches.next();
+         |      $checkCondition {
+         |        $found = true;
+         |      }
+         |    }
+         |  }
+         |}
+         |if (!$found) {
+         |  $numOutput.add(1);
+         |  ${consume(ctx, input)}
+         |}
+       """.stripMargin
+    }
+  }
+
+  /**
+   * Generates the code for existence join.
+   */
+  protected def codegenExistence(ctx: CodegenContext, input: Seq[ExprCode]): String = {
+    val HashedRelationInfo(relationTerm, keyIsUnique, _) = prepareRelation(ctx)
+    val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input)
+    val numOutput = metricTerm(ctx, "numOutputRows")
+    val existsVar = ctx.freshName("exists")
+
+    val matched = ctx.freshName("matched")
+    val buildVars = genBuildSideVars(ctx, matched, buildPlan)
+    val checkCondition = if (condition.isDefined) {
+      val expr = condition.get
+      // evaluate the variables from build side that used by condition
+      val eval = evaluateRequiredVariables(buildPlan.output, buildVars, expr.references)
+      // filter the output via condition
+      ctx.currentVars = input ++ buildVars
+      val ev =
+        BindReferences.bindReference(expr, streamedPlan.output ++ buildPlan.output).genCode(ctx)
+      s"""
+         |$eval
+         |${ev.code}
+         |$existsVar = !${ev.isNull} && ${ev.value};
+       """.stripMargin
+    } else {
+      s"$existsVar = true;"
+    }
+
+    val resultVar = input ++ Seq(ExprCode.forNonNullValue(
+      JavaCode.variable(existsVar, BooleanType)))
+
+    if (keyIsUnique) {
+      s"""
+         |// generate join key for stream side
+         |${keyEv.code}
+         |// find matches from HashedRelation
+         |UnsafeRow $matched = $anyNull ? null: (UnsafeRow)$relationTerm.getValue(${keyEv.value});
+         |boolean $existsVar = false;
+         |if ($matched != null) {
+         |  $checkCondition
+         |}
+         |$numOutput.add(1);
+         |${consume(ctx, resultVar)}
+       """.stripMargin
+    } else {
+      val matches = ctx.freshName("matches")
+      val iteratorCls = classOf[Iterator[UnsafeRow]].getName
+      s"""
+         |// generate join key for stream side
+         |${keyEv.code}
+         |// find matches from HashRelation
+         |$iteratorCls $matches = $anyNull ? null : ($iteratorCls)$relationTerm.get(${keyEv.value});
+         |boolean $existsVar = false;
+         |if ($matches != null) {
+         |  while (!$existsVar && $matches.hasNext()) {
+         |    UnsafeRow $matched = (UnsafeRow) $matches.next();
+         |    $checkCondition
+         |  }
+         |}
+         |$numOutput.add(1);
+         |${consume(ctx, resultVar)}
+       """.stripMargin
+    }
+  }
+
+  protected def prepareRelation(ctx: CodegenContext): HashedRelationInfo
 }
 
-object HashJoin {
+object HashJoin extends CastSupport with SQLConfHelper {
   /**
    * Try to rewrite the key as LongType so we can use getLong(), if they key can fit with a long.
    *
@@ -231,14 +719,14 @@ object HashJoin {
     }
 
     var keyExpr: Expression = if (keys.head.dataType != LongType) {
-      Cast(keys.head, LongType)
+      cast(keys.head, LongType)
     } else {
       keys.head
     }
     keys.tail.foreach { e =>
       val bits = e.dataType.defaultSize * 8
       keyExpr = BitwiseOr(ShiftLeft(keyExpr, Literal(bits)),
-        BitwiseAnd(Cast(e, LongType), Literal((1L << bits) - 1)))
+        BitwiseAnd(cast(e, LongType), Literal((1L << bits) - 1)))
     }
     keyExpr :: Nil
   }
@@ -251,13 +739,13 @@ object HashJoin {
     // jump over keys that have a higher index value than the required key
     if (keys.size == 1) {
       assert(index == 0)
-      Cast(BoundReference(0, LongType, nullable = false), keys(index).dataType)
+      cast(BoundReference(0, LongType, nullable = false), keys(index).dataType)
     } else {
       val shiftedBits =
         keys.slice(index + 1, keys.size).map(_.dataType.defaultSize * 8).sum
       val mask = (1L << (keys(index).dataType.defaultSize * 8)) - 1
       // build the schema for unpacking the required key
-      Cast(BitwiseAnd(
+      cast(BitwiseAnd(
         ShiftRightUnsigned(BoundReference(0, LongType, nullable = false), Literal(shiftedBits)),
         Literal(mask)), keys(index).dataType)
     }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala
index 13180d6b20902..c5369f0005e43 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala
@@ -28,6 +28,7 @@ import org.apache.spark.memory._
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.physical.BroadcastMode
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.types.LongType
 import org.apache.spark.unsafe.Platform
 import org.apache.spark.unsafe.map.BytesToBytesMap
@@ -66,6 +67,39 @@ private[execution] sealed trait HashedRelation extends KnownSizeEstimation {
     throw new UnsupportedOperationException
   }
 
+  /**
+   * Returns an iterator for key index and matched rows.
+   *
+   * Returns null if there is no matched rows.
+   */
+  def getWithKeyIndex(key: InternalRow): Iterator[ValueRowWithKeyIndex] = {
+    throw new UnsupportedOperationException
+  }
+
+  /**
+   * Returns key index and matched single row.
+   * This is for unique key case.
+   *
+   * Returns null if there is no matched rows.
+   */
+  def getValueWithKeyIndex(key: InternalRow): ValueRowWithKeyIndex = {
+    throw new UnsupportedOperationException
+  }
+
+  /**
+   * Returns an iterator for keys index and rows of InternalRow type.
+   */
+  def valuesWithKeyIndex(): Iterator[ValueRowWithKeyIndex] = {
+    throw new UnsupportedOperationException
+  }
+
+  /**
+   * Returns the maximum number of allowed keys index.
+   */
+  def maxNumKeysIndex: Int = {
+    throw new UnsupportedOperationException
+  }
+
   /**
    * Returns true iff all the keys are unique.
    */
@@ -91,12 +125,17 @@ private[execution] object HashedRelation {
 
   /**
    * Create a HashedRelation from an Iterator of InternalRow.
+   *
+   * @param allowsNullKey Allow NULL keys in HashedRelation.
+   *                      This is used for full outer join in `ShuffledHashJoinExec` only.
    */
   def apply(
       input: Iterator[InternalRow],
       key: Seq[Expression],
       sizeEstimate: Int = 64,
-      taskMemoryManager: TaskMemoryManager = null): HashedRelation = {
+      taskMemoryManager: TaskMemoryManager = null,
+      isNullAware: Boolean = false,
+      allowsNullKey: Boolean = false): HashedRelation = {
     val mm = Option(taskMemoryManager).getOrElse {
       new TaskMemoryManager(
         new UnifiedMemoryManager(
@@ -107,14 +146,53 @@ private[execution] object HashedRelation {
         0)
     }
 
-    if (key.length == 1 && key.head.dataType == LongType) {
-      LongHashedRelation(input, key, sizeEstimate, mm)
+    if (!input.hasNext && !allowsNullKey) {
+      EmptyHashedRelation
+    } else if (key.length == 1 && key.head.dataType == LongType && !allowsNullKey) {
+      // NOTE: LongHashedRelation does not support NULL keys.
+      LongHashedRelation(input, key, sizeEstimate, mm, isNullAware)
     } else {
-      UnsafeHashedRelation(input, key, sizeEstimate, mm)
+      UnsafeHashedRelation(input, key, sizeEstimate, mm, isNullAware, allowsNullKey)
     }
   }
 }
 
+/**
+ * A wrapper for key index and value in InternalRow type.
+ * Designed to be instantiated once per thread and reused.
+ */
+private[execution] class ValueRowWithKeyIndex {
+  private var keyIndex: Int = _
+  private var value: InternalRow = _
+
+  /** Updates this ValueRowWithKeyIndex by updating its key index.  Returns itself. */
+  def withNewKeyIndex(newKeyIndex: Int): ValueRowWithKeyIndex = {
+    keyIndex = newKeyIndex
+    this
+  }
+
+  /** Updates this ValueRowWithKeyIndex by updating its value.  Returns itself. */
+  def withNewValue(newValue: InternalRow): ValueRowWithKeyIndex = {
+    value = newValue
+    this
+  }
+
+  /** Updates this ValueRowWithKeyIndex.  Returns itself. */
+  def update(newKeyIndex: Int, newValue: InternalRow): ValueRowWithKeyIndex = {
+    keyIndex = newKeyIndex
+    value = newValue
+    this
+  }
+
+  def getKeyIndex: Int = {
+    keyIndex
+  }
+
+  def getValue: InternalRow = {
+    value
+  }
+}
+
 /**
  * A HashedRelation for UnsafeRow, which is backed BytesToBytesMap.
  *
@@ -138,9 +216,12 @@ private[joins] class UnsafeHashedRelation(
 
   override def estimatedSize: Long = binaryMap.getTotalMemoryConsumption
 
-  // re-used in get()/getValue()
+  // re-used in get()/getValue()/getWithKeyIndex()/getValueWithKeyIndex()/valuesWithKeyIndex()
   var resultRow = new UnsafeRow(numFields)
 
+  // re-used in getWithKeyIndex()/getValueWithKeyIndex()/valuesWithKeyIndex()
+  val valueRowWithKeyIndex = new ValueRowWithKeyIndex
+
   override def get(key: InternalRow): Iterator[InternalRow] = {
     val unsafeKey = key.asInstanceOf[UnsafeRow]
     val map = binaryMap  // avoid the compiler error
@@ -176,6 +257,63 @@ private[joins] class UnsafeHashedRelation(
     }
   }
 
+  override def getWithKeyIndex(key: InternalRow): Iterator[ValueRowWithKeyIndex] = {
+    val unsafeKey = key.asInstanceOf[UnsafeRow]
+    val map = binaryMap  // avoid the compiler error
+    val loc = new map.Location  // this could be allocated in stack
+    binaryMap.safeLookup(unsafeKey.getBaseObject, unsafeKey.getBaseOffset,
+      unsafeKey.getSizeInBytes, loc, unsafeKey.hashCode())
+    if (loc.isDefined) {
+      valueRowWithKeyIndex.withNewKeyIndex(loc.getKeyIndex)
+      new Iterator[ValueRowWithKeyIndex] {
+        private var _hasNext = true
+        override def hasNext: Boolean = _hasNext
+        override def next(): ValueRowWithKeyIndex = {
+          resultRow.pointTo(loc.getValueBase, loc.getValueOffset, loc.getValueLength)
+          _hasNext = loc.nextValue()
+          valueRowWithKeyIndex.withNewValue(resultRow)
+        }
+      }
+    } else {
+      null
+    }
+  }
+
+  override def getValueWithKeyIndex(key: InternalRow): ValueRowWithKeyIndex = {
+    val unsafeKey = key.asInstanceOf[UnsafeRow]
+    val map = binaryMap  // avoid the compiler error
+    val loc = new map.Location  // this could be allocated in stack
+    binaryMap.safeLookup(unsafeKey.getBaseObject, unsafeKey.getBaseOffset,
+      unsafeKey.getSizeInBytes, loc, unsafeKey.hashCode())
+    if (loc.isDefined) {
+      resultRow.pointTo(loc.getValueBase, loc.getValueOffset, loc.getValueLength)
+      valueRowWithKeyIndex.update(loc.getKeyIndex, resultRow)
+    } else {
+      null
+    }
+  }
+
+  override def valuesWithKeyIndex(): Iterator[ValueRowWithKeyIndex] = {
+    val iter = binaryMap.iteratorWithKeyIndex()
+
+    new Iterator[ValueRowWithKeyIndex] {
+      override def hasNext: Boolean = iter.hasNext
+
+      override def next(): ValueRowWithKeyIndex = {
+        if (!hasNext) {
+          throw QueryExecutionErrors.endOfIteratorError()
+        }
+        val loc = iter.next()
+        resultRow.pointTo(loc.getValueBase, loc.getValueOffset, loc.getValueLength)
+        valueRowWithKeyIndex.update(loc.getKeyIndex, resultRow)
+      }
+    }
+  }
+
+  override def maxNumKeysIndex: Int = {
+    binaryMap.maxNumKeysIndex
+  }
+
   override def keys(): Iterator[InternalRow] = {
     val iter = binaryMap.iterator()
 
@@ -188,7 +326,7 @@ private[joins] class UnsafeHashedRelation(
 
       override def next(): InternalRow = {
         if (!hasNext) {
-          throw new NoSuchElementException("End of the iterator")
+          throw QueryExecutionErrors.endOfIteratorError()
         } else {
           val loc = iter.next()
           unsafeRow.pointTo(loc.getKeyBase, loc.getKeyOffset, loc.getKeyLength)
@@ -289,11 +427,11 @@ private[joins] class UnsafeHashedRelation(
       readBuffer(valuesBuffer, 0, valuesSize)
 
       val loc = binaryMap.lookup(keyBuffer, Platform.BYTE_ARRAY_OFFSET, keySize)
-      val putSuceeded = loc.append(keyBuffer, Platform.BYTE_ARRAY_OFFSET, keySize,
+      val putSucceeded = loc.append(keyBuffer, Platform.BYTE_ARRAY_OFFSET, keySize,
         valuesBuffer, Platform.BYTE_ARRAY_OFFSET, valuesSize)
-      if (!putSuceeded) {
+      if (!putSucceeded) {
         binaryMap.free()
-        throw new IOException("Could not allocate memory to grow BytesToBytesMap")
+        throw QueryExecutionErrors.cannotAllocateMemoryToGrowBytesToBytesMapError()
       }
       i += 1
     }
@@ -310,7 +448,11 @@ private[joins] object UnsafeHashedRelation {
       input: Iterator[InternalRow],
       key: Seq[Expression],
       sizeEstimate: Int,
-      taskMemoryManager: TaskMemoryManager): HashedRelation = {
+      taskMemoryManager: TaskMemoryManager,
+      isNullAware: Boolean = false,
+      allowsNullKey: Boolean = false): HashedRelation = {
+    require(!(isNullAware && allowsNullKey),
+      "isNullAware and allowsNullKey cannot be enabled at same time")
 
     val pageSizeBytes = Option(SparkEnv.get).map(_.memoryManager.pageSizeBytes)
       .getOrElse(new SparkConf().get(BUFFER_PAGESIZE).getOrElse(16L * 1024 * 1024))
@@ -327,17 +469,17 @@ private[joins] object UnsafeHashedRelation {
       val row = input.next().asInstanceOf[UnsafeRow]
       numFields = row.numFields()
       val key = keyGenerator(row)
-      if (!key.anyNull) {
+      if (!key.anyNull || allowsNullKey) {
         val loc = binaryMap.lookup(key.getBaseObject, key.getBaseOffset, key.getSizeInBytes)
         val success = loc.append(
           key.getBaseObject, key.getBaseOffset, key.getSizeInBytes,
           row.getBaseObject, row.getBaseOffset, row.getSizeInBytes)
         if (!success) {
           binaryMap.free()
-          // scalastyle:off throwerror
-          throw new SparkOutOfMemoryError("There is not enough memory to build hash map")
-          // scalastyle:on throwerror
+          throw QueryExecutionErrors.cannotAcquireMemoryToBuildUnsafeHashedRelationError()
         }
+      } else if (isNullAware) {
+        return HashedRelationWithAllNullKeys
       }
     }
 
@@ -380,7 +522,7 @@ private[joins] object UnsafeHashedRelation {
  * see http://java-performance.info/implementing-world-fastest-java-int-to-int-hash-map/
  */
 private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, capacity: Int)
-  extends MemoryConsumer(mm) with Externalizable with KryoSerializable {
+  extends MemoryConsumer(mm, MemoryMode.ON_HEAP) with Externalizable with KryoSerializable {
 
   // Whether the keys are stored in dense mode or not.
   private var isDense = false
@@ -433,8 +575,7 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap
     val got = acquireMemory(size)
     if (got < size) {
       freeMemory(got)
-      throw new SparkException(s"Can't acquire $size bytes memory to build hash relation, " +
-        s"got $got bytes")
+      throw QueryExecutionErrors.cannotAcquireMemoryToBuildLongHashedRelationError(size, got)
     }
   }
 
@@ -585,7 +726,7 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap
 
       override def next(): InternalRow = {
         if (!hasNext) {
-          throw new NoSuchElementException("End of the iterator")
+          throw QueryExecutionErrors.endOfIteratorError()
         } else {
           // the key is retrieved based on the map mode
           val ret = if (isDense) minKey + pos else array(pos)
@@ -604,7 +745,7 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap
   def append(key: Long, row: UnsafeRow): Unit = {
     val sizeInBytes = row.getSizeInBytes
     if (sizeInBytes >= (1 << SIZE_BITS)) {
-      throw new UnsupportedOperationException("Does not support row that is larger than 256M")
+      throw QueryExecutionErrors.rowLargerThan256MUnsupportedError()
     }
 
     if (key < minKey) {
@@ -647,8 +788,7 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap
           growArray()
         } else if (numKeys > array.length / 2 * 0.75) {
           // The fill ratio should be less than 0.75
-          throw new UnsupportedOperationException(
-            "Cannot build HashedRelation with more than 1/3 billions unique keys")
+          throw QueryExecutionErrors.cannotBuildHashedRelationWithUniqueKeysExceededError()
         }
       }
     } else {
@@ -664,8 +804,7 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap
     val neededNumWords = (cursor - Platform.LONG_ARRAY_OFFSET + 8 + inputRowSize + 7) / 8
     if (neededNumWords > page.length) {
       if (neededNumWords > (1 << 30)) {
-        throw new UnsupportedOperationException(
-          "Can not build a HashedRelation that is larger than 8G")
+        throw QueryExecutionErrors.cannotBuildHashedRelationLargerThan8GError()
       }
       val newNumWords = math.max(neededNumWords, math.min(page.length * 2, 1 << 30))
       ensureAcquireMemory(newNumWords * 8L)
@@ -879,6 +1018,22 @@ class LongHashedRelation(
    * Returns an iterator for keys of InternalRow type.
    */
   override def keys(): Iterator[InternalRow] = map.keys()
+
+  override def getWithKeyIndex(key: InternalRow): Iterator[ValueRowWithKeyIndex] = {
+    throw new UnsupportedOperationException
+  }
+
+  override def getValueWithKeyIndex(key: InternalRow): ValueRowWithKeyIndex = {
+    throw new UnsupportedOperationException
+  }
+
+  override def valuesWithKeyIndex(): Iterator[ValueRowWithKeyIndex] = {
+    throw new UnsupportedOperationException
+  }
+
+  override def maxNumKeysIndex: Int = {
+    throw new UnsupportedOperationException
+  }
 }
 
 /**
@@ -889,7 +1044,8 @@ private[joins] object LongHashedRelation {
       input: Iterator[InternalRow],
       key: Seq[Expression],
       sizeEstimate: Int,
-      taskMemoryManager: TaskMemoryManager): LongHashedRelation = {
+      taskMemoryManager: TaskMemoryManager,
+      isNullAware: Boolean = false): HashedRelation = {
 
     val map = new LongToUnsafeRowMap(taskMemoryManager, sizeEstimate)
     val keyGenerator = UnsafeProjection.create(key)
@@ -903,6 +1059,8 @@ private[joins] object LongHashedRelation {
       if (!rowKey.isNullAt(0)) {
         val key = rowKey.getLong(0)
         map.append(key, unsafeRow)
+      } else if (isNullAware) {
+        return HashedRelationWithAllNullKeys
       }
     }
     map.optimize()
@@ -910,8 +1068,61 @@ private[joins] object LongHashedRelation {
   }
 }
 
+/**
+ * A special HashedRelation indicating that it's built from a empty input:Iterator[InternalRow].
+ * get & getValue will return null just like
+ * empty LongHashedRelation or empty UnsafeHashedRelation does.
+ */
+case object EmptyHashedRelation extends HashedRelation {
+  override def get(key: Long): Iterator[InternalRow] = null
+
+  override def get(key: InternalRow): Iterator[InternalRow] = null
+
+  override def getValue(key: Long): InternalRow = null
+
+  override def getValue(key: InternalRow): InternalRow = null
+
+  override def asReadOnlyCopy(): EmptyHashedRelation.type = this
+
+  override def keyIsUnique: Boolean = true
+
+  override def keys(): Iterator[InternalRow] = {
+    Iterator.empty
+  }
+
+  override def close(): Unit = {}
+
+  override def estimatedSize: Long = 0
+}
+
+/**
+ * A special HashedRelation indicating that it's built from a non-empty input:Iterator[InternalRow]
+ * with all the keys to be null.
+ */
+case object HashedRelationWithAllNullKeys extends HashedRelation {
+  override def get(key: InternalRow): Iterator[InternalRow] = {
+    throw new UnsupportedOperationException
+  }
+
+  override def getValue(key: InternalRow): InternalRow = {
+    throw new UnsupportedOperationException
+  }
+
+  override def asReadOnlyCopy(): HashedRelationWithAllNullKeys.type = this
+
+  override def keyIsUnique: Boolean = true
+
+  override def keys(): Iterator[InternalRow] = {
+    throw new UnsupportedOperationException
+  }
+
+  override def close(): Unit = {}
+
+  override def estimatedSize: Long = 0
+}
+
 /** The HashedRelationBroadcastMode requires that rows are broadcasted as a HashedRelation. */
-case class HashedRelationBroadcastMode(key: Seq[Expression])
+case class HashedRelationBroadcastMode(key: Seq[Expression], isNullAware: Boolean = false)
   extends BroadcastMode {
 
   override def transform(rows: Array[InternalRow]): HashedRelation = {
@@ -923,9 +1134,9 @@ case class HashedRelationBroadcastMode(key: Seq[Expression])
       sizeHint: Option[Long]): HashedRelation = {
     sizeHint match {
       case Some(numRows) =>
-        HashedRelation(rows, canonicalized.key, numRows.toInt)
+        HashedRelation(rows, canonicalized.key, numRows.toInt, isNullAware = isNullAware)
       case None =>
-        HashedRelation(rows, canonicalized.key)
+        HashedRelation(rows, canonicalized.key, isNullAware = isNullAware)
     }
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/JoinCodegenSupport.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/JoinCodegenSupport.scala
new file mode 100644
index 0000000000000..96aa0be5f59ec
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/JoinCodegenSupport.scala
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.joins
+
+import org.apache.spark.sql.catalyst.expressions.{BindReferences, BoundReference}
+import org.apache.spark.sql.catalyst.expressions.codegen._
+import org.apache.spark.sql.catalyst.expressions.codegen.Block._
+import org.apache.spark.sql.catalyst.plans.{ExistenceJoin, InnerLike, LeftAnti, LeftOuter, LeftSemi, RightOuter}
+import org.apache.spark.sql.execution.{CodegenSupport, SparkPlan}
+
+/**
+ * An interface for those join physical operators that support codegen.
+ */
+trait JoinCodegenSupport extends CodegenSupport with BaseJoinExec {
+
+  /**
+   * Generate the (non-equi) condition used to filter joined rows.
+   * This is used in Inner, Left Semi and Left Anti joins.
+   *
+   * @return Tuple of variable name for row of build side, generated code for condition,
+   *         and generated code for variables of build side.
+   */
+  protected def getJoinCondition(
+      ctx: CodegenContext,
+      streamVars: Seq[ExprCode],
+      streamPlan: SparkPlan,
+      buildPlan: SparkPlan): (String, String, Seq[ExprCode]) = {
+    val buildRow = ctx.freshName("buildRow")
+    val buildVars = genBuildSideVars(ctx, buildRow, buildPlan)
+    val checkCondition = if (condition.isDefined) {
+      val expr = condition.get
+      // evaluate the variables from build side that used by condition
+      val eval = evaluateRequiredVariables(buildPlan.output, buildVars, expr.references)
+      // filter the output via condition
+      ctx.currentVars = streamVars ++ buildVars
+      val ev =
+        BindReferences.bindReference(expr, streamPlan.output ++ buildPlan.output).genCode(ctx)
+      val skipRow = s"${ev.isNull} || !${ev.value}"
+      s"""
+         |$eval
+         |${ev.code}
+         |if (!($skipRow))
+       """.stripMargin
+    } else {
+      ""
+    }
+    (buildRow, checkCondition, buildVars)
+  }
+
+  /**
+   * Generates the code for variables of build side.
+   */
+  protected def genBuildSideVars(
+      ctx: CodegenContext,
+      buildRow: String,
+      buildPlan: SparkPlan): Seq[ExprCode] = {
+    ctx.currentVars = null
+    ctx.INPUT_ROW = buildRow
+    buildPlan.output.zipWithIndex.map { case (a, i) =>
+      val ev = BoundReference(i, a.dataType, a.nullable).genCode(ctx)
+      joinType match {
+        case _: InnerLike | LeftSemi | LeftAnti | _: ExistenceJoin =>
+          ev
+        case LeftOuter | RightOuter =>
+          // the variables are needed even there is no matched rows
+          val isNull = ctx.freshName("isNull")
+          val value = ctx.freshName("value")
+          val javaType = CodeGenerator.javaType(a.dataType)
+          val code = code"""
+            |boolean $isNull = true;
+            |$javaType $value = ${CodeGenerator.defaultValue(a.dataType)};
+            |if ($buildRow != null) {
+            |  ${ev.code}
+            |  $isNull = ${ev.isNull};
+            |  $value = ${ev.value};
+            |}
+          """.stripMargin
+          ExprCode(code, JavaCode.isNullVariable(isNull), JavaCode.variable(value, a.dataType))
+        case _ =>
+          throw new IllegalArgumentException(
+            s"JoinCodegenSupport.genBuildSideVars should not take $joinType as the JoinType")
+      }
+    }
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/ShuffledHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/ShuffledHashJoinExec.scala
index 755a63e545ef1..5c00790c2ab5a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/ShuffledHashJoinExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/ShuffledHashJoinExec.scala
@@ -22,11 +22,14 @@ import java.util.concurrent.TimeUnit._
 import org.apache.spark.TaskContext
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.expressions.codegen._
+import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide}
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.physical._
-import org.apache.spark.sql.execution.SparkPlan
-import org.apache.spark.sql.execution.metric.SQLMetrics
+import org.apache.spark.sql.execution.{RowIterator, SparkPlan}
+import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics}
+import org.apache.spark.util.collection.{BitSet, OpenHashSet}
 
 /**
  * Performs a hash join of two child relations by first shuffling the data using the join keys.
@@ -38,23 +41,38 @@ case class ShuffledHashJoinExec(
     buildSide: BuildSide,
     condition: Option[Expression],
     left: SparkPlan,
-    right: SparkPlan)
-  extends HashJoin {
+    right: SparkPlan,
+    isSkewJoin: Boolean = false)
+  extends HashJoin with ShuffledJoin {
 
   override lazy val metrics = Map(
     "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
     "buildDataSize" -> SQLMetrics.createSizeMetric(sparkContext, "data size of build side"),
     "buildTime" -> SQLMetrics.createTimingMetric(sparkContext, "time to build hash map"))
 
-  override def requiredChildDistribution: Seq[Distribution] =
-    HashClusteredDistribution(leftKeys) :: HashClusteredDistribution(rightKeys) :: Nil
+  override def output: Seq[Attribute] = super[ShuffledJoin].output
 
-  private def buildHashedRelation(iter: Iterator[InternalRow]): HashedRelation = {
+  override def outputPartitioning: Partitioning = super[ShuffledJoin].outputPartitioning
+
+  override def outputOrdering: Seq[SortOrder] = joinType match {
+    case FullOuter => Nil
+    case _ => super.outputOrdering
+  }
+
+  /**
+   * This is called by generated Java class, should be public.
+   */
+  def buildHashedRelation(iter: Iterator[InternalRow]): HashedRelation = {
     val buildDataSize = longMetric("buildDataSize")
     val buildTime = longMetric("buildTime")
     val start = System.nanoTime()
     val context = TaskContext.get()
-    val relation = HashedRelation(iter, buildKeys, taskMemoryManager = context.taskMemoryManager())
+    val relation = HashedRelation(
+      iter,
+      buildBoundKeys,
+      taskMemoryManager = context.taskMemoryManager(),
+      // Full outer join needs support for NULL key in HashedRelation.
+      allowsNullKey = joinType == FullOuter)
     buildTime += NANOSECONDS.toMillis(System.nanoTime() - start)
     buildDataSize += relation.estimatedSize
     // This relation is usually used until the end of task.
@@ -66,7 +84,243 @@ case class ShuffledHashJoinExec(
     val numOutputRows = longMetric("numOutputRows")
     streamedPlan.execute().zipPartitions(buildPlan.execute()) { (streamIter, buildIter) =>
       val hashed = buildHashedRelation(buildIter)
-      join(streamIter, hashed, numOutputRows)
+      joinType match {
+        case FullOuter => fullOuterJoin(streamIter, hashed, numOutputRows)
+        case _ => join(streamIter, hashed, numOutputRows)
+      }
+    }
+  }
+
+  private def fullOuterJoin(
+      streamIter: Iterator[InternalRow],
+      hashedRelation: HashedRelation,
+      numOutputRows: SQLMetric): Iterator[InternalRow] = {
+    val joinKeys = streamSideKeyGenerator()
+    val joinRow = new JoinedRow
+    val (joinRowWithStream, joinRowWithBuild) = {
+      buildSide match {
+        case BuildLeft => (joinRow.withRight _, joinRow.withLeft _)
+        case BuildRight => (joinRow.withLeft _, joinRow.withRight _)
+      }
+    }
+    val buildNullRow = new GenericInternalRow(buildOutput.length)
+    val streamNullRow = new GenericInternalRow(streamedOutput.length)
+    lazy val streamNullJoinRowWithBuild = {
+      buildSide match {
+        case BuildLeft =>
+          joinRow.withRight(streamNullRow)
+          joinRow.withLeft _
+        case BuildRight =>
+          joinRow.withLeft(streamNullRow)
+          joinRow.withRight _
+      }
+    }
+
+    val iter = if (hashedRelation.keyIsUnique) {
+      fullOuterJoinWithUniqueKey(streamIter, hashedRelation, joinKeys, joinRowWithStream,
+        joinRowWithBuild, streamNullJoinRowWithBuild, buildNullRow)
+    } else {
+      fullOuterJoinWithNonUniqueKey(streamIter, hashedRelation, joinKeys, joinRowWithStream,
+        joinRowWithBuild, streamNullJoinRowWithBuild, buildNullRow)
+    }
+
+    val resultProj = UnsafeProjection.create(output, output)
+    iter.map { r =>
+      numOutputRows += 1
+      resultProj(r)
+    }
+  }
+
+  /**
+   * Full outer shuffled hash join with unique join keys:
+   * 1. Process rows from stream side by looking up hash relation.
+   *    Mark the matched rows from build side be looked up.
+   *    A bit set is used to track matched rows with key index.
+   * 2. Process rows from build side by iterating hash relation.
+   *    Filter out rows from build side being matched already,
+   *    by checking key index from bit set.
+   */
+  private def fullOuterJoinWithUniqueKey(
+      streamIter: Iterator[InternalRow],
+      hashedRelation: HashedRelation,
+      joinKeys: UnsafeProjection,
+      joinRowWithStream: InternalRow => JoinedRow,
+      joinRowWithBuild: InternalRow => JoinedRow,
+      streamNullJoinRowWithBuild: => InternalRow => JoinedRow,
+      buildNullRow: GenericInternalRow): Iterator[InternalRow] = {
+    val matchedKeys = new BitSet(hashedRelation.maxNumKeysIndex)
+    longMetric("buildDataSize") += matchedKeys.capacity / 8
+
+    // Process stream side with looking up hash relation
+    val streamResultIter = streamIter.map { srow =>
+      joinRowWithStream(srow)
+      val keys = joinKeys(srow)
+      if (keys.anyNull) {
+        joinRowWithBuild(buildNullRow)
+      } else {
+        val matched = hashedRelation.getValueWithKeyIndex(keys)
+        if (matched != null) {
+          val keyIndex = matched.getKeyIndex
+          val buildRow = matched.getValue
+          val joinRow = joinRowWithBuild(buildRow)
+          if (boundCondition(joinRow)) {
+            matchedKeys.set(keyIndex)
+            joinRow
+          } else {
+            joinRowWithBuild(buildNullRow)
+          }
+        } else {
+          joinRowWithBuild(buildNullRow)
+        }
+      }
     }
+
+    // Process build side with filtering out the matched rows
+    val buildResultIter = hashedRelation.valuesWithKeyIndex().flatMap {
+      valueRowWithKeyIndex =>
+        val keyIndex = valueRowWithKeyIndex.getKeyIndex
+        val isMatched = matchedKeys.get(keyIndex)
+        if (!isMatched) {
+          val buildRow = valueRowWithKeyIndex.getValue
+          Some(streamNullJoinRowWithBuild(buildRow))
+        } else {
+          None
+        }
+    }
+
+    streamResultIter ++ buildResultIter
   }
+
+  /**
+   * Full outer shuffled hash join with non-unique join keys:
+   * 1. Process rows from stream side by looking up hash relation.
+   *    Mark the matched rows from build side be looked up.
+   *    A [[OpenHashSet]] (Long) is used to track matched rows with
+   *    key index (Int) and value index (Int) together.
+   * 2. Process rows from build side by iterating hash relation.
+   *    Filter out rows from build side being matched already,
+   *    by checking key index and value index from [[OpenHashSet]].
+   *
+   * The "value index" is defined as the index of the tuple in the chain
+   * of tuples having the same key. For example, if certain key is found thrice,
+   * the value indices of its tuples will be 0, 1 and 2.
+   * Note that value indices of tuples with different keys are incomparable.
+   */
+  private def fullOuterJoinWithNonUniqueKey(
+      streamIter: Iterator[InternalRow],
+      hashedRelation: HashedRelation,
+      joinKeys: UnsafeProjection,
+      joinRowWithStream: InternalRow => JoinedRow,
+      joinRowWithBuild: InternalRow => JoinedRow,
+      streamNullJoinRowWithBuild: => InternalRow => JoinedRow,
+      buildNullRow: GenericInternalRow): Iterator[InternalRow] = {
+    val matchedRows = new OpenHashSet[Long]
+    TaskContext.get().addTaskCompletionListener[Unit](_ => {
+      // At the end of the task, update the task's memory usage for this
+      // [[OpenHashSet]] to track matched rows, which has two parts:
+      // [[OpenHashSet._bitset]] and [[OpenHashSet._data]].
+      val bitSetEstimatedSize = matchedRows.getBitSet.capacity / 8
+      val dataEstimatedSize = matchedRows.capacity * 8
+      longMetric("buildDataSize") += bitSetEstimatedSize + dataEstimatedSize
+    })
+
+    def markRowMatched(keyIndex: Int, valueIndex: Int): Unit = {
+      val rowIndex: Long = (keyIndex.toLong << 32) | valueIndex
+      matchedRows.add(rowIndex)
+    }
+
+    def isRowMatched(keyIndex: Int, valueIndex: Int): Boolean = {
+      val rowIndex: Long = (keyIndex.toLong << 32) | valueIndex
+      matchedRows.contains(rowIndex)
+    }
+
+    // Process stream side with looking up hash relation
+    val streamResultIter = streamIter.flatMap { srow =>
+      val joinRow = joinRowWithStream(srow)
+      val keys = joinKeys(srow)
+      if (keys.anyNull) {
+        Iterator.single(joinRowWithBuild(buildNullRow))
+      } else {
+        val buildIter = hashedRelation.getWithKeyIndex(keys)
+        new RowIterator {
+          private var found = false
+          private var valueIndex = -1
+          override def advanceNext(): Boolean = {
+            while (buildIter != null && buildIter.hasNext) {
+              val buildRowWithKeyIndex = buildIter.next()
+              val keyIndex = buildRowWithKeyIndex.getKeyIndex
+              val buildRow = buildRowWithKeyIndex.getValue
+              valueIndex += 1
+              if (boundCondition(joinRowWithBuild(buildRow))) {
+                markRowMatched(keyIndex, valueIndex)
+                found = true
+                return true
+              }
+            }
+            // When we reach here, it means no match is found for this key.
+            // So we need to return one row with build side NULL row,
+            // to satisfy the full outer join semantic.
+            if (!found) {
+              joinRowWithBuild(buildNullRow)
+              // Set `found` to be true as we only need to return one row
+              // but no more.
+              found = true
+              return true
+            }
+            false
+          }
+          override def getRow: InternalRow = joinRow
+        }.toScala
+      }
+    }
+
+    // Process build side with filtering out the matched rows
+    var prevKeyIndex = -1
+    var valueIndex = -1
+    val buildResultIter = hashedRelation.valuesWithKeyIndex().flatMap {
+      valueRowWithKeyIndex =>
+        val keyIndex = valueRowWithKeyIndex.getKeyIndex
+        if (prevKeyIndex == -1 || keyIndex != prevKeyIndex) {
+          valueIndex = 0
+          prevKeyIndex = keyIndex
+        } else {
+          valueIndex += 1
+        }
+
+        val isMatched = isRowMatched(keyIndex, valueIndex)
+        if (!isMatched) {
+          val buildRow = valueRowWithKeyIndex.getValue
+          Some(streamNullJoinRowWithBuild(buildRow))
+        } else {
+          None
+        }
+    }
+
+    streamResultIter ++ buildResultIter
+  }
+
+  // TODO(SPARK-32567): support full outer shuffled hash join code-gen
+  override def supportCodegen: Boolean = {
+    joinType != FullOuter
+  }
+
+  override def inputRDDs(): Seq[RDD[InternalRow]] = {
+    streamedPlan.execute() :: buildPlan.execute() :: Nil
+  }
+
+  override def needCopyResult: Boolean = true
+
+  protected override def prepareRelation(ctx: CodegenContext): HashedRelationInfo = {
+    val thisPlan = ctx.addReferenceObj("plan", this)
+    val clsName = classOf[HashedRelation].getName
+
+    // Inline mutable state since not many join operations in a task
+    val relationTerm = ctx.addMutableState(clsName, "relation",
+      v => s"$v = $thisPlan.buildHashedRelation(inputs[1]);", forceInline = true)
+    HashedRelationInfo(relationTerm, keyIsUnique = false, isEmpty = false)
+  }
+
+  override protected def withNewChildrenInternal(
+      newLeft: SparkPlan, newRight: SparkPlan): ShuffledHashJoinExec =
+    copy(left = newLeft, right = newRight)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/ShuffledJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/ShuffledJoin.scala
new file mode 100644
index 0000000000000..794807fd3b463
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/ShuffledJoin.scala
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.joins
+
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.plans.{ExistenceJoin, FullOuter, InnerLike, LeftExistence, LeftOuter, RightOuter}
+import org.apache.spark.sql.catalyst.plans.physical.{Distribution, HashClusteredDistribution, Partitioning, PartitioningCollection, UnknownPartitioning, UnspecifiedDistribution}
+
+/**
+ * Holds common logic for join operators by shuffling two child relations
+ * using the join keys.
+ */
+trait ShuffledJoin extends JoinCodegenSupport {
+  def isSkewJoin: Boolean
+
+  override def nodeName: String = {
+    if (isSkewJoin) super.nodeName + "(skew=true)" else super.nodeName
+  }
+
+  override def stringArgs: Iterator[Any] = super.stringArgs.toSeq.dropRight(1).iterator
+
+  override def requiredChildDistribution: Seq[Distribution] = {
+    if (isSkewJoin) {
+      // We re-arrange the shuffle partitions to deal with skew join, and the new children
+      // partitioning doesn't satisfy `HashClusteredDistribution`.
+      UnspecifiedDistribution :: UnspecifiedDistribution :: Nil
+    } else {
+      HashClusteredDistribution(leftKeys) :: HashClusteredDistribution(rightKeys) :: Nil
+    }
+  }
+
+  override def outputPartitioning: Partitioning = joinType match {
+    case _: InnerLike =>
+      PartitioningCollection(Seq(left.outputPartitioning, right.outputPartitioning))
+    case LeftOuter => left.outputPartitioning
+    case RightOuter => right.outputPartitioning
+    case FullOuter => UnknownPartitioning(left.outputPartitioning.numPartitions)
+    case LeftExistence(_) => left.outputPartitioning
+    case x =>
+      throw new IllegalArgumentException(
+        s"ShuffledJoin should not take $x as the JoinType")
+  }
+
+  override def output: Seq[Attribute] = {
+    joinType match {
+      case _: InnerLike =>
+        left.output ++ right.output
+      case LeftOuter =>
+        left.output ++ right.output.map(_.withNullability(true))
+      case RightOuter =>
+        left.output.map(_.withNullability(true)) ++ right.output
+      case FullOuter =>
+        (left.output ++ right.output).map(_.withNullability(true))
+      case j: ExistenceJoin =>
+        left.output :+ j.exists
+      case LeftExistence(_) =>
+        left.output
+      case x =>
+        throw new IllegalArgumentException(
+          s"${getClass.getSimpleName} not take $x as the JoinType")
+    }
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala
index 2c57956de5bca..db6626bd18abc 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala
@@ -26,7 +26,6 @@ import org.apache.spark.sql.catalyst.expressions.BindReferences.bindReferences
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
 import org.apache.spark.sql.catalyst.plans._
-import org.apache.spark.sql.catalyst.plans.physical._
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics}
 import org.apache.spark.util.collection.BitSet
@@ -41,69 +40,20 @@ case class SortMergeJoinExec(
     condition: Option[Expression],
     left: SparkPlan,
     right: SparkPlan,
-    isSkewJoin: Boolean = false) extends BaseJoinExec with CodegenSupport {
+    isSkewJoin: Boolean = false) extends ShuffledJoin {
 
   override lazy val metrics = Map(
     "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"))
 
-  override def nodeName: String = {
-    if (isSkewJoin) super.nodeName + "(skew=true)" else super.nodeName
-  }
-
-  override def stringArgs: Iterator[Any] = super.stringArgs.toSeq.dropRight(1).iterator
-
-  override def output: Seq[Attribute] = {
-    joinType match {
-      case _: InnerLike =>
-        left.output ++ right.output
-      case LeftOuter =>
-        left.output ++ right.output.map(_.withNullability(true))
-      case RightOuter =>
-        left.output.map(_.withNullability(true)) ++ right.output
-      case FullOuter =>
-        (left.output ++ right.output).map(_.withNullability(true))
-      case j: ExistenceJoin =>
-        left.output :+ j.exists
-      case LeftExistence(_) =>
-        left.output
-      case x =>
-        throw new IllegalArgumentException(
-          s"${getClass.getSimpleName} should not take $x as the JoinType")
-    }
-  }
-
-  override def outputPartitioning: Partitioning = joinType match {
-    case _: InnerLike =>
-      PartitioningCollection(Seq(left.outputPartitioning, right.outputPartitioning))
-    // For left and right outer joins, the output is partitioned by the streamed input's join keys.
-    case LeftOuter => left.outputPartitioning
-    case RightOuter => right.outputPartitioning
-    case FullOuter => UnknownPartitioning(left.outputPartitioning.numPartitions)
-    case LeftExistence(_) => left.outputPartitioning
-    case x =>
-      throw new IllegalArgumentException(
-        s"${getClass.getSimpleName} should not take $x as the JoinType")
-  }
-
-  override def requiredChildDistribution: Seq[Distribution] = {
-    if (isSkewJoin) {
-      // We re-arrange the shuffle partitions to deal with skew join, and the new children
-      // partitioning doesn't satisfy `HashClusteredDistribution`.
-      UnspecifiedDistribution :: UnspecifiedDistribution :: Nil
-    } else {
-      HashClusteredDistribution(leftKeys) :: HashClusteredDistribution(rightKeys) :: Nil
-    }
-  }
-
   override def outputOrdering: Seq[SortOrder] = joinType match {
     // For inner join, orders of both sides keys should be kept.
     case _: InnerLike =>
       val leftKeyOrdering = getKeyOrdering(leftKeys, left.outputOrdering)
       val rightKeyOrdering = getKeyOrdering(rightKeys, right.outputOrdering)
       leftKeyOrdering.zip(rightKeyOrdering).map { case (lKey, rKey) =>
-        // Also add the right key and its `sameOrderExpressions`
-        SortOrder(lKey.child, Ascending, lKey.sameOrderExpressions + rKey.child ++ rKey
-          .sameOrderExpressions)
+        // Also add expressions from right side sort order
+        val sameOrderExpressions = ExpressionSet(lKey.sameOrderExpressions ++ rKey.children)
+        SortOrder(lKey.child, Ascending, sameOrderExpressions.toSeq)
       }
     // For left and right outer joins, the output is ordered by the streamed input's join keys.
     case LeftOuter => getKeyOrdering(leftKeys, left.outputOrdering)
@@ -129,7 +79,8 @@ case class SortMergeJoinExec(
     val requiredOrdering = requiredOrders(keys)
     if (SortOrder.orderingSatisfies(childOutputOrdering, requiredOrdering)) {
       keys.zip(childOutputOrdering).map { case (key, childOrder) =>
-        SortOrder(key, Ascending, childOrder.sameOrderExpressions + childOrder.child - key)
+        val sameOrderExpressionsSet = ExpressionSet(childOrder.children) - key
+        SortOrder(key, Ascending, sameOrderExpressionsSet.toSeq)
       }
     } else {
       requiredOrdering
@@ -154,8 +105,18 @@ case class SortMergeJoinExec(
     sqlContext.conf.sortMergeJoinExecBufferSpillThreshold
   }
 
+  // Flag to only buffer first matched row, to avoid buffering unnecessary rows.
+  private val onlyBufferFirstMatchedRow = (joinType, condition) match {
+    case (LeftExistence(_), None) => true
+    case _ => false
+  }
+
   private def getInMemoryThreshold: Int = {
-    sqlContext.conf.sortMergeJoinExecBufferInMemoryThreshold
+    if (onlyBufferFirstMatchedRow) {
+      1
+    } else {
+      sqlContext.conf.sortMergeJoinExecBufferInMemoryThreshold
+    }
   }
 
   protected override def doExecute(): RDD[InternalRow] = {
@@ -284,7 +245,8 @@ case class SortMergeJoinExec(
               RowIterator.fromScala(rightIter),
               inMemoryThreshold,
               spillThreshold,
-              cleanupResources
+              cleanupResources,
+              onlyBufferFirstMatchedRow
             )
             private[this] val joinRow = new JoinedRow
 
@@ -320,7 +282,8 @@ case class SortMergeJoinExec(
               RowIterator.fromScala(rightIter),
               inMemoryThreshold,
               spillThreshold,
-              cleanupResources
+              cleanupResources,
+              onlyBufferFirstMatchedRow
             )
             private[this] val joinRow = new JoinedRow
 
@@ -363,7 +326,8 @@ case class SortMergeJoinExec(
               RowIterator.fromScala(rightIter),
               inMemoryThreshold,
               spillThreshold,
-              cleanupResources
+              cleanupResources,
+              onlyBufferFirstMatchedRow
             )
             private[this] val joinRow = new JoinedRow
 
@@ -399,12 +363,24 @@ case class SortMergeJoinExec(
     }
   }
 
-  override def supportCodegen: Boolean = {
-    joinType.isInstanceOf[InnerLike]
+  private lazy val ((streamedPlan, streamedKeys), (bufferedPlan, bufferedKeys)) = joinType match {
+    case _: InnerLike | LeftOuter | LeftSemi => ((left, leftKeys), (right, rightKeys))
+    case RightOuter => ((right, rightKeys), (left, leftKeys))
+    case x =>
+      throw new IllegalArgumentException(
+        s"SortMergeJoin.streamedPlan/bufferedPlan should not take $x as the JoinType")
+  }
+
+  private lazy val streamedOutput = streamedPlan.output
+  private lazy val bufferedOutput = bufferedPlan.output
+
+  override def supportCodegen: Boolean = joinType match {
+    case _: InnerLike | LeftOuter | RightOuter | LeftSemi => true
+    case _ => false
   }
 
   override def inputRDDs(): Seq[RDD[InternalRow]] = {
-    left.execute() :: right.execute() :: Nil
+    streamedPlan.execute() :: bufferedPlan.execute() :: Nil
   }
 
   private def createJoinKey(
@@ -438,24 +414,24 @@ case class SortMergeJoinExec(
   }
 
   /**
-   * Generate a function to scan both left and right to find a match, returns the term for
-   * matched one row from left side and buffered rows from right side.
+   * Generate a function to scan both sides to find a match, returns the term for
+   * matched one row from streamed side and buffered rows from buffered side.
    */
   private def genScanner(ctx: CodegenContext): (String, String) = {
     // Create class member for next row from both sides.
     // Inline mutable state since not many join operations in a task
-    val leftRow = ctx.addMutableState("InternalRow", "leftRow", forceInline = true)
-    val rightRow = ctx.addMutableState("InternalRow", "rightRow", forceInline = true)
+    val streamedRow = ctx.addMutableState("InternalRow", "streamedRow", forceInline = true)
+    val bufferedRow = ctx.addMutableState("InternalRow", "bufferedRow", forceInline = true)
 
     // Create variables for join keys from both sides.
-    val leftKeyVars = createJoinKey(ctx, leftRow, leftKeys, left.output)
-    val leftAnyNull = leftKeyVars.map(_.isNull).mkString(" || ")
-    val rightKeyTmpVars = createJoinKey(ctx, rightRow, rightKeys, right.output)
-    val rightAnyNull = rightKeyTmpVars.map(_.isNull).mkString(" || ")
-    // Copy the right key as class members so they could be used in next function call.
-    val rightKeyVars = copyKeys(ctx, rightKeyTmpVars)
-
-    // A list to hold all matched rows from right side.
+    val streamedKeyVars = createJoinKey(ctx, streamedRow, streamedKeys, streamedOutput)
+    val streamedAnyNull = streamedKeyVars.map(_.isNull).mkString(" || ")
+    val bufferedKeyTmpVars = createJoinKey(ctx, bufferedRow, bufferedKeys, bufferedOutput)
+    val bufferedAnyNull = bufferedKeyTmpVars.map(_.isNull).mkString(" || ")
+    // Copy the buffered key as class members so they could be used in next function call.
+    val bufferedKeyVars = copyKeys(ctx, bufferedKeyTmpVars)
+
+    // A list to hold all matched rows from buffered side.
     val clsName = classOf[ExternalAppendOnlyUnsafeRowArray].getName
 
     val spillThreshold = getSpillThreshold
@@ -464,26 +440,100 @@ case class SortMergeJoinExec(
     // Inline mutable state since not many join operations in a task
     val matches = ctx.addMutableState(clsName, "matches",
       v => s"$v = new $clsName($inMemoryThreshold, $spillThreshold);", forceInline = true)
-    // Copy the left keys as class members so they could be used in next function call.
-    val matchedKeyVars = copyKeys(ctx, leftKeyVars)
+    // Copy the streamed keys as class members so they could be used in next function call.
+    val matchedKeyVars = copyKeys(ctx, streamedKeyVars)
+
+    // Handle the case when streamed rows has any NULL keys.
+    val handleStreamedAnyNull = joinType match {
+      case _: InnerLike | LeftSemi =>
+        // Skip streamed row.
+        s"""
+           |$streamedRow = null;
+           |continue;
+         """.stripMargin
+      case LeftOuter | RightOuter =>
+        // Eagerly return streamed row. Only call `matches.clear()` when `matches.isEmpty()` is
+        // false, to reduce unnecessary computation.
+        s"""
+           |if (!$matches.isEmpty()) {
+           |  $matches.clear();
+           |}
+           |return false;
+         """.stripMargin
+      case x =>
+        throw new IllegalArgumentException(
+          s"SortMergeJoin.genScanner should not take $x as the JoinType")
+    }
+
+    // Handle the case when streamed keys has no match with buffered side.
+    val handleStreamedWithoutMatch = joinType match {
+      case _: InnerLike | LeftSemi =>
+        // Skip streamed row.
+        s"$streamedRow = null;"
+      case LeftOuter | RightOuter =>
+        // Eagerly return with streamed row.
+        "return false;"
+      case x =>
+        throw new IllegalArgumentException(
+          s"SortMergeJoin.genScanner should not take $x as the JoinType")
+    }
 
-    ctx.addNewFunction("findNextInnerJoinRows",
+    val addRowToBuffer =
+      if (onlyBufferFirstMatchedRow) {
+        s"""
+           |if ($matches.isEmpty()) {
+           |  $matches.add((UnsafeRow) $bufferedRow);
+           |}
+         """.stripMargin
+      } else {
+        s"$matches.add((UnsafeRow) $bufferedRow);"
+      }
+
+    // Generate a function to scan both streamed and buffered sides to find a match.
+    // Return whether a match is found.
+    //
+    // `streamedIter`: the iterator for streamed side.
+    // `bufferedIter`: the iterator for buffered side.
+    // `streamedRow`: the current row from streamed side.
+    //                When `streamedIter` is empty, `streamedRow` is null.
+    // `matches`: the rows from buffered side already matched with `streamedRow`.
+    //            `matches` is buffered and reused for all `streamedRow`s having same join keys.
+    //            If there is no match with `streamedRow`, `matches` is empty.
+    // `bufferedRow`: the current matched row from buffered side.
+    //
+    // The function has the following step:
+    //  - Step 1: Find the next `streamedRow` with non-null join keys.
+    //            For `streamedRow` with null join keys (`handleStreamedAnyNull`):
+    //            1. Inner and Left Semi join: skip the row. `matches` will be cleared later when
+    //                                         hitting the next `streamedRow` with non-null join
+    //                                         keys.
+    //            2. Left/Right Outer join: clear the previous `matches` if needed, keep the row,
+    //                                      and return false.
+    //
+    //  - Step 2: Find the `matches` from buffered side having same join keys with `streamedRow`.
+    //            Clear `matches` if we hit a new `streamedRow`, as we need to find new matches.
+    //            Use `bufferedRow` to iterate buffered side to put all matched rows into
+    //            `matches` (`addRowToBuffer`). Return true when getting all matched rows.
+    //            For `streamedRow` without `matches` (`handleStreamedWithoutMatch`):
+    //            1. Inner and Left Semi join: skip the row.
+    //            2. Left/Right Outer join: keep the row and return false (with `matches` being
+    //                                      empty).
+    ctx.addNewFunction("findNextJoinRows",
       s"""
-         |private boolean findNextInnerJoinRows(
-         |    scala.collection.Iterator leftIter,
-         |    scala.collection.Iterator rightIter) {
-         |  $leftRow = null;
+         |private boolean findNextJoinRows(
+         |    scala.collection.Iterator streamedIter,
+         |    scala.collection.Iterator bufferedIter) {
+         |  $streamedRow = null;
          |  int comp = 0;
-         |  while ($leftRow == null) {
-         |    if (!leftIter.hasNext()) return false;
-         |    $leftRow = (InternalRow) leftIter.next();
-         |    ${leftKeyVars.map(_.code).mkString("\n")}
-         |    if ($leftAnyNull) {
-         |      $leftRow = null;
-         |      continue;
+         |  while ($streamedRow == null) {
+         |    if (!streamedIter.hasNext()) return false;
+         |    $streamedRow = (InternalRow) streamedIter.next();
+         |    ${streamedKeyVars.map(_.code).mkString("\n")}
+         |    if ($streamedAnyNull) {
+         |      $handleStreamedAnyNull
          |    }
          |    if (!$matches.isEmpty()) {
-         |      ${genComparison(ctx, leftKeyVars, matchedKeyVars)}
+         |      ${genComparison(ctx, streamedKeyVars, matchedKeyVars)}
          |      if (comp == 0) {
          |        return true;
          |      }
@@ -491,88 +541,80 @@ case class SortMergeJoinExec(
          |    }
          |
          |    do {
-         |      if ($rightRow == null) {
-         |        if (!rightIter.hasNext()) {
+         |      if ($bufferedRow == null) {
+         |        if (!bufferedIter.hasNext()) {
          |          ${matchedKeyVars.map(_.code).mkString("\n")}
          |          return !$matches.isEmpty();
          |        }
-         |        $rightRow = (InternalRow) rightIter.next();
-         |        ${rightKeyTmpVars.map(_.code).mkString("\n")}
-         |        if ($rightAnyNull) {
-         |          $rightRow = null;
+         |        $bufferedRow = (InternalRow) bufferedIter.next();
+         |        ${bufferedKeyTmpVars.map(_.code).mkString("\n")}
+         |        if ($bufferedAnyNull) {
+         |          $bufferedRow = null;
          |          continue;
          |        }
-         |        ${rightKeyVars.map(_.code).mkString("\n")}
+         |        ${bufferedKeyVars.map(_.code).mkString("\n")}
          |      }
-         |      ${genComparison(ctx, leftKeyVars, rightKeyVars)}
+         |      ${genComparison(ctx, streamedKeyVars, bufferedKeyVars)}
          |      if (comp > 0) {
-         |        $rightRow = null;
+         |        $bufferedRow = null;
          |      } else if (comp < 0) {
          |        if (!$matches.isEmpty()) {
          |          ${matchedKeyVars.map(_.code).mkString("\n")}
          |          return true;
+         |        } else {
+         |          $handleStreamedWithoutMatch
          |        }
-         |        $leftRow = null;
          |      } else {
-         |        $matches.add((UnsafeRow) $rightRow);
-         |        $rightRow = null;
+         |        $addRowToBuffer
+         |        $bufferedRow = null;
          |      }
-         |    } while ($leftRow != null);
+         |    } while ($streamedRow != null);
          |  }
          |  return false; // unreachable
          |}
        """.stripMargin, inlineToOuterClass = true)
 
-    (leftRow, matches)
+    (streamedRow, matches)
   }
 
   /**
-   * Creates variables and declarations for left part of result row.
+   * Creates variables and declarations for streamed part of result row.
    *
    * In order to defer the access after condition and also only access once in the loop,
    * the variables should be declared separately from accessing the columns, we can't use the
    * codegen of BoundReference here.
    */
-  private def createLeftVars(ctx: CodegenContext, leftRow: String): (Seq[ExprCode], Seq[String]) = {
-    ctx.INPUT_ROW = leftRow
-    left.output.zipWithIndex.map { case (a, i) =>
+  private def createStreamedVars(
+      ctx: CodegenContext,
+      streamedRow: String): (Seq[ExprCode], Seq[String]) = {
+    ctx.INPUT_ROW = streamedRow
+    streamedPlan.output.zipWithIndex.map { case (a, i) =>
       val value = ctx.freshName("value")
-      val valueCode = CodeGenerator.getValue(leftRow, a.dataType, i.toString)
+      val valueCode = CodeGenerator.getValue(streamedRow, a.dataType, i.toString)
       val javaType = CodeGenerator.javaType(a.dataType)
       val defaultValue = CodeGenerator.defaultValue(a.dataType)
       if (a.nullable) {
         val isNull = ctx.freshName("isNull")
         val code =
           code"""
-             |$isNull = $leftRow.isNullAt($i);
+             |$isNull = $streamedRow.isNullAt($i);
              |$value = $isNull ? $defaultValue : ($valueCode);
            """.stripMargin
-        val leftVarsDecl =
+        val streamedVarsDecl =
           s"""
              |boolean $isNull = false;
              |$javaType $value = $defaultValue;
            """.stripMargin
         (ExprCode(code, JavaCode.isNullVariable(isNull), JavaCode.variable(value, a.dataType)),
-          leftVarsDecl)
+          streamedVarsDecl)
       } else {
         val code = code"$value = $valueCode;"
-        val leftVarsDecl = s"""$javaType $value = $defaultValue;"""
-        (ExprCode(code, FalseLiteral, JavaCode.variable(value, a.dataType)), leftVarsDecl)
+        val streamedVarsDecl = s"""$javaType $value = $defaultValue;"""
+        (ExprCode(code, FalseLiteral, JavaCode.variable(value, a.dataType)), streamedVarsDecl)
       }
     }.unzip
   }
 
-  /**
-   * Creates the variables for right part of result row, using BoundReference, since the right
-   * part are accessed inside the loop.
-   */
-  private def createRightVar(ctx: CodegenContext, rightRow: String): Seq[ExprCode] = {
-    ctx.INPUT_ROW = rightRow
-    right.output.zipWithIndex.map { case (a, i) =>
-      BoundReference(i, a.dataType, a.nullable).genCode(ctx)
-    }
-  }
-
   /**
    * Splits variables based on whether it's used by condition or not, returns the code to create
    * these variables before the condition and after the condition.
@@ -600,68 +642,189 @@ case class SortMergeJoinExec(
 
   override def doProduce(ctx: CodegenContext): String = {
     // Inline mutable state since not many join operations in a task
-    val leftInput = ctx.addMutableState("scala.collection.Iterator", "leftInput",
+    val streamedInput = ctx.addMutableState("scala.collection.Iterator", "streamedInput",
       v => s"$v = inputs[0];", forceInline = true)
-    val rightInput = ctx.addMutableState("scala.collection.Iterator", "rightInput",
+    val bufferedInput = ctx.addMutableState("scala.collection.Iterator", "bufferedInput",
       v => s"$v = inputs[1];", forceInline = true)
 
-    val (leftRow, matches) = genScanner(ctx)
+    val (streamedRow, matches) = genScanner(ctx)
 
     // Create variables for row from both sides.
-    val (leftVars, leftVarDecl) = createLeftVars(ctx, leftRow)
-    val rightRow = ctx.freshName("rightRow")
-    val rightVars = createRightVar(ctx, rightRow)
+    val (streamedVars, streamedVarDecl) = createStreamedVars(ctx, streamedRow)
+    val bufferedRow = ctx.freshName("bufferedRow")
+    val bufferedVars = genBuildSideVars(ctx, bufferedRow, bufferedPlan)
 
     val iterator = ctx.freshName("iterator")
     val numOutput = metricTerm(ctx, "numOutputRows")
-    val (beforeLoop, condCheck) = if (condition.isDefined) {
+    val resultVars = joinType match {
+      case _: InnerLike | LeftOuter =>
+        streamedVars ++ bufferedVars
+      case RightOuter =>
+        bufferedVars ++ streamedVars
+      case LeftSemi =>
+        streamedVars
+      case x =>
+        throw new IllegalArgumentException(
+          s"SortMergeJoin.doProduce should not take $x as the JoinType")
+    }
+
+    val (streamedBeforeLoop, condCheck) = if (condition.isDefined) {
       // Split the code of creating variables based on whether it's used by condition or not.
       val loaded = ctx.freshName("loaded")
-      val (leftBefore, leftAfter) = splitVarsByCondition(left.output, leftVars)
-      val (rightBefore, rightAfter) = splitVarsByCondition(right.output, rightVars)
+      val (streamedBefore, streamedAfter) = splitVarsByCondition(streamedOutput, streamedVars)
+      val (bufferedBefore, bufferedAfter) = splitVarsByCondition(bufferedOutput, bufferedVars)
       // Generate code for condition
-      ctx.currentVars = leftVars ++ rightVars
-      val cond = BindReferences.bindReference(condition.get, output).genCode(ctx)
+      ctx.currentVars = streamedVars ++ bufferedVars
+      val cond = BindReferences.bindReference(
+        condition.get, streamedPlan.output ++ bufferedPlan.output).genCode(ctx)
       // evaluate the columns those used by condition before loop
-      val before = s"""
+      val before =
+        s"""
            |boolean $loaded = false;
-           |$leftBefore
+           |$streamedBefore
          """.stripMargin
 
-      val checking = s"""
-         |$rightBefore
-         |${cond.code}
-         |if (${cond.isNull} || !${cond.value}) continue;
-         |if (!$loaded) {
-         |  $loaded = true;
-         |  $leftAfter
-         |}
-         |$rightAfter
-     """.stripMargin
-      (before, checking)
+      val checking =
+        s"""
+           |$bufferedBefore
+           |if ($bufferedRow != null) {
+           |  ${cond.code}
+           |  if (${cond.isNull} || !${cond.value}) {
+           |    continue;
+           |  }
+           |}
+           |if (!$loaded) {
+           |  $loaded = true;
+           |  $streamedAfter
+           |}
+           |$bufferedAfter
+         """.stripMargin
+      (before, checking.trim)
     } else {
-      (evaluateVariables(leftVars), "")
+      (evaluateVariables(streamedVars), "")
     }
 
+    val beforeLoop =
+      s"""
+         |${streamedVarDecl.mkString("\n")}
+         |${streamedBeforeLoop.trim}
+         |scala.collection.Iterator<UnsafeRow> $iterator = $matches.generateIterator();
+       """.stripMargin
+    val outputRow =
+      s"""
+         |$numOutput.add(1);
+         |${consume(ctx, resultVars)}
+       """.stripMargin
+    val findNextJoinRows = s"findNextJoinRows($streamedInput, $bufferedInput)"
     val thisPlan = ctx.addReferenceObj("plan", this)
     val eagerCleanup = s"$thisPlan.cleanupResources();"
 
+    joinType match {
+      case _: InnerLike =>
+        codegenInner(findNextJoinRows, beforeLoop, iterator, bufferedRow, condCheck, outputRow,
+          eagerCleanup)
+      case LeftOuter | RightOuter =>
+        codegenOuter(streamedInput, findNextJoinRows, beforeLoop, iterator, bufferedRow, condCheck,
+          ctx.freshName("hasOutputRow"), outputRow, eagerCleanup)
+      case LeftSemi =>
+        codegenSemi(findNextJoinRows, beforeLoop, iterator, bufferedRow, condCheck,
+          ctx.freshName("hasOutputRow"), outputRow, eagerCleanup)
+      case x =>
+        throw new IllegalArgumentException(
+          s"SortMergeJoin.doProduce should not take $x as the JoinType")
+    }
+  }
+
+  /**
+   * Generates the code for Inner join.
+   */
+  private def codegenInner(
+      findNextJoinRows: String,
+      beforeLoop: String,
+      matchIterator: String,
+      bufferedRow: String,
+      conditionCheck: String,
+      outputRow: String,
+      eagerCleanup: String): String = {
+    s"""
+       |while ($findNextJoinRows) {
+       |  $beforeLoop
+       |  while ($matchIterator.hasNext()) {
+       |    InternalRow $bufferedRow = (InternalRow) $matchIterator.next();
+       |    $conditionCheck
+       |    $outputRow
+       |  }
+       |  if (shouldStop()) return;
+       |}
+       |$eagerCleanup
+     """.stripMargin
+  }
+
+  /**
+   * Generates the code for Left or Right Outer join.
+   */
+  private def codegenOuter(
+      streamedInput: String,
+      findNextJoinRows: String,
+      beforeLoop: String,
+      matchIterator: String,
+      bufferedRow: String,
+      conditionCheck: String,
+      hasOutputRow: String,
+      outputRow: String,
+      eagerCleanup: String): String = {
     s"""
-       |while (findNextInnerJoinRows($leftInput, $rightInput)) {
-       |  ${leftVarDecl.mkString("\n")}
-       |  ${beforeLoop.trim}
-       |  scala.collection.Iterator<UnsafeRow> $iterator = $matches.generateIterator();
-       |  while ($iterator.hasNext()) {
-       |    InternalRow $rightRow = (InternalRow) $iterator.next();
-       |    ${condCheck.trim}
-       |    $numOutput.add(1);
-       |    ${consume(ctx, leftVars ++ rightVars)}
+       |while ($streamedInput.hasNext()) {
+       |  $findNextJoinRows;
+       |  $beforeLoop
+       |  boolean $hasOutputRow = false;
+       |
+       |  // the last iteration of this loop is to emit an empty row if there is no matched rows.
+       |  while ($matchIterator.hasNext() || !$hasOutputRow) {
+       |    InternalRow $bufferedRow = $matchIterator.hasNext() ?
+       |      (InternalRow) $matchIterator.next() : null;
+       |    $conditionCheck
+       |    $hasOutputRow = true;
+       |    $outputRow
        |  }
        |  if (shouldStop()) return;
        |}
        |$eagerCleanup
      """.stripMargin
   }
+
+  /**
+   * Generates the code for Left Semi join.
+   */
+  private def codegenSemi(
+      findNextJoinRows: String,
+      beforeLoop: String,
+      matchIterator: String,
+      bufferedRow: String,
+      conditionCheck: String,
+      hasOutputRow: String,
+      outputRow: String,
+      eagerCleanup: String): String = {
+    s"""
+       |while ($findNextJoinRows) {
+       |  $beforeLoop
+       |  boolean $hasOutputRow = false;
+       |
+       |  while (!$hasOutputRow && $matchIterator.hasNext()) {
+       |    InternalRow $bufferedRow = (InternalRow) $matchIterator.next();
+       |    $conditionCheck
+       |    $hasOutputRow = true;
+       |    $outputRow
+       |  }
+       |  if (shouldStop()) return;
+       |}
+       |$eagerCleanup
+     """.stripMargin
+  }
+
+  override protected def withNewChildrenInternal(
+      newLeft: SparkPlan, newRight: SparkPlan): SortMergeJoinExec =
+    copy(left = newLeft, right = newRight)
 }
 
 /**
@@ -686,6 +849,7 @@ case class SortMergeJoinExec(
  *                          internal buffer
  * @param spillThreshold Threshold for number of rows to be spilled by internal buffer
  * @param eagerCleanupResources the eager cleanup function to be invoked when no join row found
+ * @param onlyBufferFirstMatch [[bufferMatchingRows]] should buffer only the first matching row
  */
 private[joins] class SortMergeJoinScanner(
     streamedKeyGenerator: Projection,
@@ -695,7 +859,8 @@ private[joins] class SortMergeJoinScanner(
     bufferedIter: RowIterator,
     inMemoryThreshold: Int,
     spillThreshold: Int,
-    eagerCleanupResources: () => Unit) {
+    eagerCleanupResources: () => Unit,
+    onlyBufferFirstMatch: Boolean = false) {
   private[this] var streamedRow: InternalRow = _
   private[this] var streamedRowKey: InternalRow = _
   private[this] var bufferedRow: InternalRow = _
@@ -706,7 +871,7 @@ private[joins] class SortMergeJoinScanner(
    */
   private[this] var matchJoinKey: InternalRow = _
   /** Buffered rows from the buffered side of the join. This is empty if there are no matches. */
-  private[this] val bufferedMatches =
+  private[this] val bufferedMatches: ExternalAppendOnlyUnsafeRowArray =
     new ExternalAppendOnlyUnsafeRowArray(inMemoryThreshold, spillThreshold)
 
   // Initialization (note: do _not_ want to advance streamed here).
@@ -867,7 +1032,9 @@ private[joins] class SortMergeJoinScanner(
     matchJoinKey = streamedRowKey.copy()
     bufferedMatches.clear()
     do {
-      bufferedMatches.add(bufferedRow.asInstanceOf[UnsafeRow])
+      if (!onlyBufferFirstMatch || bufferedMatches.isEmpty) {
+        bufferedMatches.add(bufferedRow.asInstanceOf[UnsafeRow])
+      }
       advancedBufferedToRowWithNullFreeJoinKey()
     } while (bufferedRow != null && keyOrdering.compare(streamedRowKey, bufferedRowKey) == 0)
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/package.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/package.scala
deleted file mode 100644
index 134376628ae7f..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/package.scala
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution
-
-/**
- * Physical execution operators for join operations.
- */
-package object joins {
-
-  sealed abstract class BuildSide
-
-  case object BuildRight extends BuildSide
-
-  case object BuildLeft extends BuildSide
-
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/limit.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/limit.scala
index ddbd0a343ffcf..5114c075a72d0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/limit.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/limit.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.execution
 
-import org.apache.spark.rdd.RDD
+import org.apache.spark.rdd.{ParallelCollectionRDD, RDD}
 import org.apache.spark.serializer.Serializer
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
@@ -26,6 +26,7 @@ import org.apache.spark.sql.catalyst.plans.physical._
 import org.apache.spark.sql.catalyst.util.truncatedString
 import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
 import org.apache.spark.sql.execution.metric.{SQLShuffleReadMetricsReporter, SQLShuffleWriteMetricsReporter}
+import org.apache.spark.util.collection.Utils
 
 /**
  * The operator takes limited number of elements from its child operator.
@@ -52,17 +53,29 @@ case class CollectLimitExec(limit: Int, child: SparkPlan) extends LimitExec {
     SQLShuffleReadMetricsReporter.createShuffleReadMetrics(sparkContext)
   override lazy val metrics = readMetrics ++ writeMetrics
   protected override def doExecute(): RDD[InternalRow] = {
-    val locallyLimited = child.execute().mapPartitionsInternal(_.take(limit))
-    val shuffled = new ShuffledRowRDD(
-      ShuffleExchangeExec.prepareShuffleDependency(
-        locallyLimited,
-        child.output,
-        SinglePartition,
-        serializer,
-        writeMetrics),
-      readMetrics)
-    shuffled.mapPartitionsInternal(_.take(limit))
+    val childRDD = child.execute()
+    if (childRDD.getNumPartitions == 0) {
+      new ParallelCollectionRDD(sparkContext, Seq.empty[InternalRow], 1, Map.empty)
+    } else {
+      val singlePartitionRDD = if (childRDD.getNumPartitions == 1) {
+        childRDD
+      } else {
+        val locallyLimited = childRDD.mapPartitionsInternal(_.take(limit))
+        new ShuffledRowRDD(
+          ShuffleExchangeExec.prepareShuffleDependency(
+            locallyLimited,
+            child.output,
+            SinglePartition,
+            serializer,
+            writeMetrics),
+          readMetrics)
+      }
+      singlePartitionRDD.mapPartitionsInternal(_.take(limit))
+    }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan =
+    copy(child = newChild)
 }
 
 /**
@@ -85,6 +98,9 @@ case class CollectTailExec(limit: Int, child: SparkPlan) extends LimitExec {
     // job launch, we might just have to mimic the implementation of `CollectLimitExec`.
     sparkContext.parallelize(executeCollect(), numSlices = 1)
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan =
+    copy(child = newChild)
 }
 
 object BaseLimitExec {
@@ -103,6 +119,10 @@ object BaseLimitExec {
 trait BaseLimitExec extends LimitExec with CodegenSupport {
   override def output: Seq[Attribute] = child.output
 
+  override def outputPartitioning: Partitioning = child.outputPartitioning
+
+  override def outputOrdering: Seq[SortOrder] = child.outputOrdering
+
   protected override def doExecute(): RDD[InternalRow] = child.execute().mapPartitions { iter =>
     iter.take(limit)
   }
@@ -122,14 +142,18 @@ trait BaseLimitExec extends LimitExec with CodegenSupport {
   }
 
   protected override def doProduce(ctx: CodegenContext): String = {
-    child.asInstanceOf[CodegenSupport].produce(ctx, this)
-  }
-
-  override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = {
     // The counter name is already obtained by the upstream operators via `limitNotReachedChecks`.
     // Here we have to inline it to not change its name. This is fine as we won't have many limit
     // operators in one query.
+    //
+    // Note: create counter variable here instead of `doConsume()` to avoid compilation error,
+    // because upstream operators might not call `doConsume()` here
+    // (e.g. `HashJoin.codegenInner()`).
     ctx.addMutableState(CodeGenerator.JAVA_INT, countTerm, forceInline = true, useFreshName = false)
+    child.asInstanceOf[CodegenSupport].produce(ctx, this)
+  }
+
+  override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = {
     s"""
        | if ($countTerm < $limit) {
        |   $countTerm += 1;
@@ -143,10 +167,8 @@ trait BaseLimitExec extends LimitExec with CodegenSupport {
  * Take the first `limit` elements of each child partition, but do not collect or shuffle them.
  */
 case class LocalLimitExec(limit: Int, child: SparkPlan) extends BaseLimitExec {
-
-  override def outputOrdering: Seq[SortOrder] = child.outputOrdering
-
-  override def outputPartitioning: Partitioning = child.outputPartitioning
+  override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan =
+    copy(child = newChild)
 }
 
 /**
@@ -156,9 +178,8 @@ case class GlobalLimitExec(limit: Int, child: SparkPlan) extends BaseLimitExec {
 
   override def requiredChildDistribution: List[Distribution] = AllTuples :: Nil
 
-  override def outputPartitioning: Partitioning = child.outputPartitioning
-
-  override def outputOrdering: Seq[SortOrder] = child.outputOrdering
+  override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan =
+    copy(child = newChild)
 }
 
 /**
@@ -199,26 +220,33 @@ case class TakeOrderedAndProjectExec(
 
   protected override def doExecute(): RDD[InternalRow] = {
     val ord = new LazilyGeneratedOrdering(sortOrder, child.output)
-    val localTopK: RDD[InternalRow] = {
-      child.execute().map(_.copy()).mapPartitions { iter =>
-        org.apache.spark.util.collection.Utils.takeOrdered(iter, limit)(ord)
-      }
-    }
-    val shuffled = new ShuffledRowRDD(
-      ShuffleExchangeExec.prepareShuffleDependency(
-        localTopK,
-        child.output,
-        SinglePartition,
-        serializer,
-        writeMetrics),
-      readMetrics)
-    shuffled.mapPartitions { iter =>
-      val topK = org.apache.spark.util.collection.Utils.takeOrdered(iter.map(_.copy()), limit)(ord)
-      if (projectList != child.output) {
-        val proj = UnsafeProjection.create(projectList, child.output)
-        topK.map(r => proj(r))
+    val childRDD = child.execute()
+    if (childRDD.getNumPartitions == 0) {
+      new ParallelCollectionRDD(sparkContext, Seq.empty[InternalRow], 1, Map.empty)
+    } else {
+      val singlePartitionRDD = if (childRDD.getNumPartitions == 1) {
+        childRDD
       } else {
-        topK
+        val localTopK = childRDD.mapPartitions { iter =>
+          Utils.takeOrdered(iter.map(_.copy()), limit)(ord)
+        }
+        new ShuffledRowRDD(
+          ShuffleExchangeExec.prepareShuffleDependency(
+            localTopK,
+            child.output,
+            SinglePartition,
+            serializer,
+            writeMetrics),
+          readMetrics)
+      }
+      singlePartitionRDD.mapPartitions { iter =>
+        val topK = Utils.takeOrdered(iter.map(_.copy()), limit)(ord)
+        if (projectList != child.output) {
+          val proj = UnsafeProjection.create(projectList, child.output)
+          topK.map(r => proj(r))
+        } else {
+          topK
+        }
       }
     }
   }
@@ -233,4 +261,7 @@ case class TakeOrderedAndProjectExec(
 
     s"TakeOrderedAndProject(limit=$limit, orderBy=$orderByString, output=$outputString)"
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan =
+    copy(child = newChild)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/CustomMetrics.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/CustomMetrics.scala
new file mode 100644
index 0000000000000..3e6cad2676e15
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/CustomMetrics.scala
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.metric
+
+import org.apache.spark.sql.connector.metric.{CustomMetric, CustomTaskMetric}
+
+object CustomMetrics {
+  private[spark] val V2_CUSTOM = "v2Custom"
+
+  private[spark] val NUM_ROWS_PER_UPDATE = 100
+
+  /**
+   * Given a class name, builds and returns a metric type for a V2 custom metric class
+   * `CustomMetric`.
+   */
+  def buildV2CustomMetricTypeName(customMetric: CustomMetric): String = {
+    s"${V2_CUSTOM}_${customMetric.getClass.getCanonicalName}"
+  }
+
+  /**
+   * Given a V2 custom metric type name, this method parses it and returns the corresponding
+   * `CustomMetric` class name.
+   */
+  def parseV2CustomMetricType(metricType: String): Option[String] = {
+    if (metricType.startsWith(s"${V2_CUSTOM}_")) {
+      Some(metricType.drop(V2_CUSTOM.length + 1))
+    } else {
+      None
+    }
+  }
+
+  /**
+   * Updates given custom metrics.
+   */
+  def updateMetrics(
+      currentMetricsValues: Seq[CustomTaskMetric],
+      customMetrics: Map[String, SQLMetric]): Unit = {
+    currentMetricsValues.foreach { metric =>
+      customMetrics(metric.name()).set(metric.value())
+    }
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala
index a0acbbe5a2181..959144bab33f4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala
@@ -24,6 +24,7 @@ import scala.concurrent.duration._
 
 import org.apache.spark.SparkContext
 import org.apache.spark.scheduler.AccumulableInfo
+import org.apache.spark.sql.connector.metric.CustomMetric
 import org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates
 import org.apache.spark.util.{AccumulatorContext, AccumulatorV2, Utils}
 
@@ -107,6 +108,15 @@ object SQLMetrics {
     acc
   }
 
+  /**
+   * Create a metric to report data source v2 custom metric.
+   */
+  def createV2CustomMetric(sc: SparkContext, customMetric: CustomMetric): SQLMetric = {
+    val acc = new SQLMetric(CustomMetrics.buildV2CustomMetricTypeName(customMetric))
+    acc.register(sc, name = Some(customMetric.description()), countFailedValues = false)
+    acc
+  }
+
   /**
    * Create a metric to report the size information (including total, min, med, max) like data size,
    * spill size, etc.
@@ -200,7 +210,7 @@ object SQLMetrics {
       } else if (metricsType == NS_TIMING_METRIC) {
         duration => Utils.msDurationToString(duration.nanos.toMillis)
       } else {
-        throw new IllegalStateException("unexpected metrics type: " + metricsType)
+        throw new IllegalStateException(s"unexpected metrics type: $metricsType")
       }
 
       val validValues = values.filter(_ >= 0)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/objects.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/objects.scala
index d05113431df41..fa46f75abe8f3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/objects.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/objects.scala
@@ -99,6 +99,9 @@ case class DeserializeToObjectExec(
       iter.map(projection)
     }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): DeserializeToObjectExec =
+    copy(child = newChild)
 }
 
 /**
@@ -135,6 +138,9 @@ case class SerializeFromObjectExec(
       iter.map(projection)
     }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): SerializeFromObjectExec =
+    copy(child = newChild)
 }
 
 /**
@@ -195,6 +201,9 @@ case class MapPartitionsExec(
       func(iter.map(getObject)).map(outputObject)
     }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): MapPartitionsExec =
+    copy(child = newChild)
 }
 
 /**
@@ -252,6 +261,9 @@ case class MapPartitionsInRWithArrowExec(
       }.map(outputProject)
     }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): MapPartitionsInRWithArrowExec =
+    copy(child = newChild)
 }
 
 /**
@@ -276,12 +288,12 @@ case class MapElementsExec(
   }
 
   override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = {
-    val (funcClass, methodName) = func match {
+    val (funcClass, funcName) = func match {
       case m: MapFunction[_, _] => classOf[MapFunction[_, _]] -> "call"
       case _ => FunctionUtils.getFunctionOneName(outputObjectType, child.output(0).dataType)
     }
     val funcObj = Literal.create(func, ObjectType(funcClass))
-    val callFunc = Invoke(funcObj, methodName, outputObjectType, child.output)
+    val callFunc = Invoke(funcObj, funcName, outputObjectType, child.output, propagateNull = false)
 
     val result = BindReferences.bindReference(callFunc, child.output).genCode(ctx)
 
@@ -304,6 +316,9 @@ case class MapElementsExec(
   override def outputOrdering: Seq[SortOrder] = child.outputOrdering
 
   override def outputPartitioning: Partitioning = child.outputPartitioning
+
+  override protected def withNewChildInternal(newChild: SparkPlan): MapElementsExec =
+    copy(child = newChild)
 }
 
 /**
@@ -333,6 +348,9 @@ case class AppendColumnsExec(
       }
     }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): AppendColumnsExec =
+    copy(child = newChild)
 }
 
 /**
@@ -366,6 +384,9 @@ case class AppendColumnsWithObjectExec(
       }
     }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): AppendColumnsWithObjectExec =
+    copy(child = newChild)
 }
 
 /**
@@ -405,6 +426,9 @@ case class MapGroupsExec(
       }
     }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): MapGroupsExec =
+    copy(child = newChild)
 }
 
 object MapGroupsExec {
@@ -495,6 +519,9 @@ case class FlatMapGroupsInRExec(
       }
     }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): FlatMapGroupsInRExec =
+    copy(child = newChild)
 }
 
 /**
@@ -567,9 +594,19 @@ case class FlatMapGroupsInRWithArrowExec(
       // binary in a batch due to the limitation of R API. See also ARROW-4512.
       val columnarBatchIter = runner.compute(groupedByRKey, -1)
       val outputProject = UnsafeProjection.create(output, output)
-      columnarBatchIter.flatMap(_.rowIterator().asScala).map(outputProject)
+      val outputTypes = StructType.fromAttributes(output).map(_.dataType)
+
+      columnarBatchIter.flatMap { batch =>
+        val actualDataTypes = (0 until batch.numCols()).map(i => batch.column(i).dataType())
+        assert(outputTypes == actualDataTypes, "Invalid schema from gapply(): " +
+          s"expected ${outputTypes.mkString(", ")}, got ${actualDataTypes.mkString(", ")}")
+        batch.rowIterator().asScala
+      }.map(outputProject)
     }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): FlatMapGroupsInRWithArrowExec =
+    copy(child = newChild)
 }
 
 /**
@@ -616,4 +653,7 @@ case class CoGroupExec(
       }
     }
   }
+
+  override protected def withNewChildrenInternal(
+    newLeft: SparkPlan, newRight: SparkPlan): CoGroupExec = copy(left = newLeft, right = newRight)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/AggregateInPandasExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/AggregateInPandasExec.scala
index fcbd0b19515b1..5019008ec5e32 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/AggregateInPandasExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/AggregateInPandasExec.scala
@@ -103,11 +103,11 @@ case class AggregateInPandasExec(
     // Schema of input rows to the python runner
     val aggInputSchema = StructType(dataTypes.zipWithIndex.map { case (dt, i) =>
       StructField(s"_$i", dt)
-    })
+    }.toSeq)
 
     // Map grouped rows to ArrowPythonRunner results, Only execute if partition is not empty
     inputRDD.mapPartitionsInternal { iter => if (iter.isEmpty) iter else {
-      val prunedProj = UnsafeProjection.create(allInputs, child.output)
+      val prunedProj = UnsafeProjection.create(allInputs.toSeq, child.output)
 
       val grouped = if (groupingExpressions.isEmpty) {
         // Use an empty unsafe row as a place holder for the grouping key
@@ -154,4 +154,7 @@ case class AggregateInPandasExec(
       }
     }}
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan =
+    copy(child = newChild)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowEvalPythonExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowEvalPythonExec.scala
index 67f075f0785fb..096712cf93529 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowEvalPythonExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowEvalPythonExec.scala
@@ -94,4 +94,7 @@ case class ArrowEvalPythonExec(udfs: Seq[PythonUDF], resultAttrs: Seq[Attribute]
       batch.rowIterator.asScala
     }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan =
+    copy(child = newChild)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonRunner.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonRunner.scala
index b44b13c8de0da..7171c7f7f9746 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonRunner.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonRunner.scala
@@ -46,6 +46,8 @@ class ArrowPythonRunner(
   extends BasePythonRunner[Iterator[InternalRow], ColumnarBatch](funcs, evalType, argOffsets)
   with PythonArrowOutput {
 
+  override val simplifiedTraceback: Boolean = SQLConf.get.pysparkSimplifiedTraceback
+
   override val bufferSize: Int = SQLConf.get.pandasUDFBufferSize
   require(
     bufferSize >= 4,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/BatchEvalPythonExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/BatchEvalPythonExec.scala
index b6d8e59877f17..10f7966b93d1a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/BatchEvalPythonExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/BatchEvalPythonExec.scala
@@ -46,7 +46,18 @@ case class BatchEvalPythonExec(udfs: Seq[PythonUDF], resultAttrs: Seq[Attribute]
     val needConversion = dataTypes.exists(EvaluatePython.needConversionInPython)
 
     // enable memo iff we serialize the row with schema (schema and class should be memorized)
-    val pickle = new Pickler(needConversion)
+    // pyrolite 4.21+ can lookup objects in its cache by value, but `GenericRowWithSchema` objects,
+    // that we pass from JVM to Python, don't define their `equals()` to take the type of the
+    // values or the schema of the row into account. This causes like
+    // `GenericRowWithSchema(Array(1.0, 1.0),
+    //    StructType(Seq(StructField("_1", DoubleType), StructField("_2", DoubleType))))`
+    // and
+    // `GenericRowWithSchema(Array(1, 1),
+    //    StructType(Seq(StructField("_1", IntegerType), StructField("_2", IntegerType))))`
+    // to be `equal()` and so we need to disable this feature explicitly (`valueCompare=false`).
+    // Please note that cache by reference is still enabled depending on `needConversion`.
+    val pickle = new Pickler(/* useMemo = */ needConversion,
+      /* valueCompare = */ false)
     // Input iterator to Python: input rows are grouped so we send them in batches to Python.
     // For each row, add it to the queue.
     val inputIterator = iter.map { row =>
@@ -92,4 +103,7 @@ case class BatchEvalPythonExec(udfs: Seq[PythonUDF], resultAttrs: Seq[Attribute]
       }
     }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): BatchEvalPythonExec =
+    copy(child = newChild)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/CoGroupedArrowPythonRunner.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/CoGroupedArrowPythonRunner.scala
index 25ce16db264ac..e3d8a943d8cf2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/CoGroupedArrowPythonRunner.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/CoGroupedArrowPythonRunner.scala
@@ -27,6 +27,7 @@ import org.apache.spark.{SparkEnv, TaskContext}
 import org.apache.spark.api.python.{BasePythonRunner, ChainedPythonFunctions, PythonRDD}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.execution.arrow.ArrowWriter
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.ArrowUtils
 import org.apache.spark.sql.vectorized.ColumnarBatch
@@ -49,6 +50,8 @@ class CoGroupedArrowPythonRunner(
     (Iterator[InternalRow], Iterator[InternalRow]), ColumnarBatch](funcs, evalType, argOffsets)
   with PythonArrowOutput {
 
+  override val simplifiedTraceback: Boolean = SQLConf.get.pysparkSimplifiedTraceback
+
   protected def newWriterThread(
       env: SparkEnv,
       worker: Socket,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExec.scala
index 96e3bb721a822..fca43e454bff5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExec.scala
@@ -21,12 +21,12 @@ import java.io.File
 
 import scala.collection.mutable.ArrayBuffer
 
-import org.apache.spark.{SparkEnv, TaskContext}
+import org.apache.spark.{ContextAwareIterator, SparkEnv, TaskContext}
 import org.apache.spark.api.python.ChainedPythonFunctions
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.execution.{SparkPlan, UnaryExecNode}
+import org.apache.spark.sql.execution.UnaryExecNode
 import org.apache.spark.sql.types.{DataType, StructField, StructType}
 import org.apache.spark.util.Utils
 
@@ -89,6 +89,7 @@ trait EvalPythonExec extends UnaryExecNode {
 
     inputRDD.mapPartitions { iter =>
       val context = TaskContext.get()
+      val contextAwareIterator = new ContextAwareIterator(context, iter)
 
       // The queue used to buffer input rows so we can drain it to
       // combine input with output from Python.
@@ -114,13 +115,13 @@ trait EvalPythonExec extends UnaryExecNode {
           }
         }.toArray
       }.toArray
-      val projection = MutableProjection.create(allInputs, child.output)
+      val projection = MutableProjection.create(allInputs.toSeq, child.output)
       val schema = StructType(dataTypes.zipWithIndex.map { case (dt, i) =>
         StructField(s"_$i", dt)
-      })
+      }.toSeq)
 
       // Add rows to queue to join later with the result.
-      val projectedRowIter = iter.map { inputRow =>
+      val projectedRowIter = contextAwareIterator.map { inputRow =>
         queue.add(inputRow.asInstanceOf[UnsafeRow])
         projection(inputRow)
       }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvaluatePython.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvaluatePython.scala
index 520afad287648..4885f63113807 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvaluatePython.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvaluatePython.scala
@@ -175,7 +175,7 @@ object EvaluatePython {
       }
 
     case StructType(fields) =>
-      val fieldsFromJava = fields.map(f => makeFromJava(f.dataType)).toArray
+      val fieldsFromJava = fields.map(f => makeFromJava(f.dataType))
 
       (obj: Any) => nullSafeConvert(obj) {
         case c if c.getClass.isArray =>
@@ -198,7 +198,7 @@ object EvaluatePython {
 
     case udt: UserDefinedType[_] => makeFromJava(udt.sqlType)
 
-    case other => (obj: Any) => nullSafeConvert(other)(PartialFunction.empty)
+    case other => (obj: Any) => nullSafeConvert(obj)(PartialFunction.empty)
   }
 
   private def nullSafeConvert(input: Any)(f: PartialFunction[Any, Any]): Any = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFs.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFs.scala
index 7bc8b95cfb03b..c6a70fb204354 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFs.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFs.scala
@@ -21,11 +21,11 @@ import scala.collection.mutable
 import scala.collection.mutable.ArrayBuffer
 
 import org.apache.spark.api.python.PythonEvalType
-import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.errors.QueryCompilationErrors
 
 
 /**
@@ -72,7 +72,7 @@ object ExtractPythonUDFFromAggregate extends Rule[LogicalPlan] {
       }
     }
     // There is no Python UDF over aggregate expression
-    Project(projList, agg.copy(aggregateExpressions = aggExpr))
+    Project(projList.toSeq, agg.copy(aggregateExpressions = aggExpr.toSeq))
   }
 
   def apply(plan: LogicalPlan): LogicalPlan = plan transformUp {
@@ -102,7 +102,7 @@ object ExtractGroupingPythonUDFFromAggregate extends Rule[LogicalPlan] {
           case p: PythonUDF =>
             // This is just a sanity check, the rule PullOutNondeterministic should
             // already pull out those nondeterministic expressions.
-            assert(p.udfDeterministic, "Non-determinstic PythonUDFs should not appear " +
+            assert(p.udfDeterministic, "Non-deterministic PythonUDFs should not appear " +
               "in grouping expression")
             val canonicalized = p.canonicalized.asInstanceOf[PythonUDF]
             if (attributeMap.contains(canonicalized)) {
@@ -134,9 +134,9 @@ object ExtractGroupingPythonUDFFromAggregate extends Rule[LogicalPlan] {
       }.asInstanceOf[NamedExpression]
     }
     agg.copy(
-      groupingExpressions = groupingExpr,
+      groupingExpressions = groupingExpr.toSeq,
       aggregateExpressions = aggExpr,
-      child = Project(projList ++ agg.child.output, agg.child))
+      child = Project((projList ++ agg.child.output).toSeq, agg.child))
   }
 
   def apply(plan: LogicalPlan): LogicalPlan = plan transformUp {
@@ -174,7 +174,7 @@ object ExtractPythonUDFs extends Rule[LogicalPlan] with PredicateHelper {
   }
 
   private def collectEvaluableUDFsFromExpressions(expressions: Seq[Expression]): Seq[PythonUDF] = {
-    // If fisrt UDF is SQL_SCALAR_PANDAS_ITER_UDF, then only return this UDF,
+    // If first UDF is SQL_SCALAR_PANDAS_ITER_UDF, then only return this UDF,
     // otherwise check if subsequent UDFs are of the same type as the first UDF. (since we can only
     // extract UDFs of the same eval type)
 
@@ -218,13 +218,22 @@ object ExtractPythonUDFs extends Rule[LogicalPlan] with PredicateHelper {
     }
   }
 
+  private def canonicalizeDeterministic(u: PythonUDF) = {
+    if (u.deterministic) {
+      u.canonicalized.asInstanceOf[PythonUDF]
+    } else {
+      u
+    }
+  }
+
   /**
    * Extract all the PythonUDFs from the current operator and evaluate them before the operator.
    */
   private def extract(plan: LogicalPlan): LogicalPlan = {
-    val udfs = collectEvaluableUDFsFromExpressions(plan.expressions)
+    val udfs = ExpressionSet(collectEvaluableUDFsFromExpressions(plan.expressions))
       // ignore the PythonUDF that come from second/third aggregate, which is not used
       .filter(udf => udf.references.subsetOf(plan.inputSet))
+      .toSeq.asInstanceOf[Seq[PythonUDF]]
     if (udfs.isEmpty) {
       // If there aren't any, we are done.
       plan
@@ -248,9 +257,7 @@ object ExtractPythonUDFs extends Rule[LogicalPlan] with PredicateHelper {
 
           val evalTypes = validUdfs.map(_.evalType).toSet
           if (evalTypes.size != 1) {
-            throw new AnalysisException(
-              s"Expected udfs have the same evalType but got different evalTypes: " +
-              s"${evalTypes.mkString(",")}")
+            throw QueryCompilationErrors.unexpectedEvalTypesForUDFsError(evalTypes)
           }
           val evalType = evalTypes.head
           val evaluation = evalType match {
@@ -259,10 +266,10 @@ object ExtractPythonUDFs extends Rule[LogicalPlan] with PredicateHelper {
             case PythonEvalType.SQL_SCALAR_PANDAS_UDF | PythonEvalType.SQL_SCALAR_PANDAS_ITER_UDF =>
               ArrowEvalPython(validUdfs, resultAttrs, child, evalType)
             case _ =>
-              throw new AnalysisException("Unexcepted UDF evalType")
+              throw new IllegalStateException("Unexpected UDF evalType")
           }
 
-          attributeMap ++= validUdfs.zip(resultAttrs)
+          attributeMap ++= validUdfs.map(canonicalizeDeterministic).zip(resultAttrs)
           evaluation
         } else {
           child
@@ -270,13 +277,12 @@ object ExtractPythonUDFs extends Rule[LogicalPlan] with PredicateHelper {
       }
       // Other cases are disallowed as they are ambiguous or would require a cartesian
       // product.
-      udfs.filterNot(attributeMap.contains).foreach { udf =>
-        sys.error(s"Invalid PythonUDF $udf, requires attributes from more than one child.")
+      udfs.map(canonicalizeDeterministic).filterNot(attributeMap.contains).foreach {
+        udf => sys.error(s"Invalid PythonUDF $udf, requires attributes from more than one child.")
       }
 
       val rewritten = plan.withNewChildren(newChildren).transformExpressions {
-        case p: PythonUDF if attributeMap.contains(p) =>
-          attributeMap(p)
+        case p: PythonUDF => attributeMap.getOrElse(canonicalizeDeterministic(p), p)
       }
 
       // extract remaining python UDFs recursively
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/FlatMapCoGroupsInPandasExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/FlatMapCoGroupsInPandasExec.scala
index b079405bdc2f8..e830ea6b54662 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/FlatMapCoGroupsInPandasExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/FlatMapCoGroupsInPandasExec.scala
@@ -103,4 +103,8 @@ case class FlatMapCoGroupsInPandasExec(
       }
     }
   }
+
+  override protected def withNewChildrenInternal(
+      newLeft: SparkPlan, newRight: SparkPlan): FlatMapCoGroupsInPandasExec =
+    copy(left = newLeft, right = newRight)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/FlatMapGroupsInPandasExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/FlatMapGroupsInPandasExec.scala
index 5032bc81327b9..3a3a6022f9985 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/FlatMapGroupsInPandasExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/FlatMapGroupsInPandasExec.scala
@@ -94,4 +94,7 @@ case class FlatMapGroupsInPandasExec(
       executePython(data, output, runner)
     }}
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): FlatMapGroupsInPandasExec =
+    copy(child = newChild)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/MapInPandasExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/MapInPandasExec.scala
index 2bb808119c0ae..0434710da43ae 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/MapInPandasExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/MapInPandasExec.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.execution.python
 
 import scala.collection.JavaConverters._
 
-import org.apache.spark.TaskContext
+import org.apache.spark.{ContextAwareIterator, TaskContext}
 import org.apache.spark.api.python.{ChainedPythonFunctions, PythonEvalType}
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
@@ -61,16 +61,17 @@ case class MapInPandasExec(
       val pythonRunnerConf = ArrowUtils.getPythonRunnerConfMap(conf)
       val outputTypes = child.schema
 
+      val context = TaskContext.get()
+      val contextAwareIterator = new ContextAwareIterator(context, inputIter)
+
       // Here we wrap it via another row so that Python sides understand it
       // as a DataFrame.
-      val wrappedIter = inputIter.map(InternalRow(_))
+      val wrappedIter = contextAwareIterator.map(InternalRow(_))
 
       // DO NOT use iter.grouped(). See BatchIterator.
       val batchIter =
         if (batchSize > 0) new BatchIterator(wrappedIter, batchSize) else Iterator(wrappedIter)
 
-      val context = TaskContext.get()
-
       val columnarBatchIter = new ArrowPythonRunner(
         chainedFunc,
         PythonEvalType.SQL_MAP_PANDAS_ITER_UDF,
@@ -92,4 +93,7 @@ case class MapInPandasExec(
       }.map(unsafeProj)
     }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): MapInPandasExec =
+    copy(child = newChild)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PandasGroupUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PandasGroupUtils.scala
index 68ce991a8ae7f..2da0000dad4ef 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PandasGroupUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PandasGroupUtils.scala
@@ -118,6 +118,6 @@ private[python] object PandasGroupUtils {
 
     // Attributes after deduplication
     val dedupAttributes = nonDupGroupingAttributes ++ dataAttributes
-    (dedupAttributes, argOffsets)
+    (dedupAttributes.toSeq, argOffsets)
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonUDFRunner.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonUDFRunner.scala
index 0a250b27ccb94..d9fe07214d061 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonUDFRunner.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonUDFRunner.scala
@@ -23,6 +23,7 @@ import java.util.concurrent.atomic.AtomicBoolean
 
 import org.apache.spark._
 import org.apache.spark.api.python._
+import org.apache.spark.sql.internal.SQLConf
 
 /**
  * A helper class to run Python UDFs in Spark.
@@ -34,6 +35,8 @@ class PythonUDFRunner(
   extends BasePythonRunner[Array[Byte], Array[Byte]](
     funcs, evalType, argOffsets) {
 
+  override val simplifiedTraceback: Boolean = SQLConf.get.pysparkSimplifiedTraceback
+
   protected override def newWriterThread(
       env: SparkEnv,
       worker: Socket,
@@ -104,7 +107,7 @@ object PythonUDFRunner {
       dataOut.writeInt(chained.funcs.length)
       chained.funcs.foreach { f =>
         dataOut.writeInt(f.command.length)
-        dataOut.write(f.command)
+        dataOut.write(f.command.toArray)
       }
     }
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/RowQueue.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/RowQueue.scala
index eb12641f548ad..0e3243eac6230 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/RowQueue.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/RowQueue.scala
@@ -21,11 +21,12 @@ import java.io._
 
 import com.google.common.io.Closeables
 
-import org.apache.spark.{SparkEnv, SparkException}
+import org.apache.spark.SparkEnv
 import org.apache.spark.io.NioBufferedFileInputStream
 import org.apache.spark.memory.{MemoryConsumer, SparkOutOfMemoryError, TaskMemoryManager}
 import org.apache.spark.serializer.SerializerManager
 import org.apache.spark.sql.catalyst.expressions.UnsafeRow
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.unsafe.Platform
 import org.apache.spark.unsafe.memory.MemoryBlock
 
@@ -174,7 +175,7 @@ private[python] case class HybridRowQueue(
     tempDir: File,
     numFields: Int,
     serMgr: SerializerManager)
-  extends MemoryConsumer(memManager) with RowQueue {
+  extends MemoryConsumer(memManager, memManager.getTungstenMemoryMode) with RowQueue {
 
   // Each buffer should have at least one row
   private var queues = new java.util.LinkedList[RowQueue]()
@@ -249,7 +250,7 @@ private[python] case class HybridRowQueue(
     if (writing == null || !writing.add(row)) {
       writing = createNewQueue(4 + row.getSizeInBytes)
       if (!writing.add(row)) {
-        throw new SparkException(s"failed to push a row into $writing")
+        throw QueryExecutionErrors.failedToPushRowIntoRowQueueError(writing.toString)
       }
     }
     true
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/WindowInPandasExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/WindowInPandasExec.scala
index 983fe9db73824..07c0aab1b6b74 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/WindowInPandasExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/WindowInPandasExec.scala
@@ -28,6 +28,7 @@ import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.physical.{AllTuples, ClusteredDistribution, Distribution, Partitioning}
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.execution.{ExternalAppendOnlyUnsafeRowArray, SparkPlan}
 import org.apache.spark.sql.execution.window._
 import org.apache.spark.sql.types._
@@ -153,7 +154,7 @@ case class WindowInPandasExec(
         _: SlidingWindowFunctionFrame |
         _: UnboundedPrecedingWindowFunctionFrame => BoundedWindow
       // It should be impossible to get other types of window function frame here
-      case frame => throw new RuntimeException(s"Unexpected window function frame $frame.")
+      case frame => throw QueryExecutionErrors.unexpectedWindowFunctionFrameError(frame.toString)
     }
 
     val requiredIndices = functionFrames.map {
@@ -401,4 +402,7 @@ case class WindowInPandasExec(
       }
     }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): WindowInPandasExec =
+    copy(child = newChild)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/r/ArrowRRunner.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/r/ArrowRRunner.scala
index 59f5a7078a151..ae7b7ef23512c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/r/ArrowRRunner.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/r/ArrowRRunner.scala
@@ -26,7 +26,7 @@ import org.apache.arrow.vector.VectorSchemaRoot
 import org.apache.arrow.vector.ipc.{ArrowStreamReader, ArrowStreamWriter}
 import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel
 
-import org.apache.spark.{SparkException, TaskContext}
+import org.apache.spark.TaskContext
 import org.apache.spark.api.r._
 import org.apache.spark.api.r.SpecialLengths
 import org.apache.spark.broadcast.Broadcast
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala
index 5094e5eab5955..0a9954e6796bf 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Cast, Expr
 import org.apache.spark.sql.catalyst.expressions.aggregate._
 import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
 import org.apache.spark.sql.catalyst.util.{GenericArrayData, QuantileSummaries}
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
@@ -235,7 +236,7 @@ object StatFunctions extends Logging {
         p.stripSuffix("%").toDouble / 100.0
       } catch {
         case e: NumberFormatException =>
-          throw new IllegalArgumentException(s"Unable to parse $p as a percentile", e)
+          throw QueryExecutionErrors.cannotParseStatisticAsPercentileError(p, e)
       }
     }
     require(percentiles.forall(p => p >= 0 && p <= 1), "Percentiles must be in the range [0, 1]")
@@ -254,11 +255,15 @@ object StatFunctions extends Logging {
       } else {
         stats.toLowerCase(Locale.ROOT) match {
           case "count" => (child: Expression) => Count(child).toAggregateExpression()
+          case "count_distinct" => (child: Expression) =>
+            Count(child).toAggregateExpression(isDistinct = true)
+          case "approx_count_distinct" => (child: Expression) =>
+            HyperLogLogPlusPlus(child).toAggregateExpression()
           case "mean" => (child: Expression) => Average(child).toAggregateExpression()
           case "stddev" => (child: Expression) => StddevSamp(child).toAggregateExpression()
           case "min" => (child: Expression) => Min(child).toAggregateExpression()
           case "max" => (child: Expression) => Max(child).toAggregateExpression()
-          case _ => throw new IllegalArgumentException(s"$stats is not a recognised statistic")
+          case _ => throw QueryExecutionErrors.statisticNotRecognizedError(stats)
         }
       }
     }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CheckpointFileManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CheckpointFileManager.scala
index 26f42b6e3f472..85484d37539a3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CheckpointFileManager.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CheckpointFileManager.scala
@@ -83,6 +83,12 @@ trait CheckpointFileManager {
 
   /** Is the default file system this implementation is operating on the local file system. */
   def isLocal: Boolean
+
+  /**
+   * Creates the checkpoint path if it does not exist, and returns the qualified
+   * checkpoint path.
+   */
+  def createCheckpointDirectory(): Path
 }
 
 object CheckpointFileManager extends Logging {
@@ -160,11 +166,17 @@ object CheckpointFileManager extends Logging {
     override def cancel(): Unit = synchronized {
       try {
         if (terminated) return
-        underlyingStream.close()
+        try {
+          underlyingStream.close()
+        } catch {
+          case NonFatal(e) =>
+            logWarning(s"Error cancelling write to $finalPath, " +
+              s"continuing to delete temp path $tempPath", e)
+        }
         fm.delete(tempPath)
       } catch {
         case NonFatal(e) =>
-          logWarning(s"Error cancelling write to $finalPath", e)
+          logWarning(s"Error deleting temp file $tempPath", e)
       } finally {
         terminated = true
       }
@@ -271,7 +283,6 @@ class FileSystemBasedCheckpointFileManager(path: Path, hadoopConf: Configuration
       fs.delete(path, true)
     } catch {
       case e: FileNotFoundException =>
-        logInfo(s"Failed to delete $path as it does not exist")
         // ignore if file has already been deleted
     }
   }
@@ -280,6 +291,12 @@ class FileSystemBasedCheckpointFileManager(path: Path, hadoopConf: Configuration
     case _: LocalFileSystem | _: RawLocalFileSystem => true
     case _ => false
   }
+
+  override def createCheckpointDirectory(): Path = {
+    val qualifiedPath = fs.makeQualified(path)
+    fs.mkdirs(qualifiedPath, FsPermission.getDirDefault)
+    qualifiedPath
+  }
 }
 
 
@@ -346,6 +363,12 @@ class FileContextBasedCheckpointFileManager(path: Path, hadoopConf: Configuratio
     case _ => false
   }
 
+  override def createCheckpointDirectory(): Path = {
+    val qualifiedPath = fc.makeQualified(path)
+    fc.mkdir(qualifiedPath, FsPermission.getDirDefault, true)
+    qualifiedPath
+  }
+
   private def mayRemoveCrcFile(path: Path): Unit = {
     try {
       val checksumFile = new Path(path.getParent, s".${path.getName}.crc")
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLog.scala
index 10bcfe6649802..835c7c4d5261f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLog.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.execution.streaming
 
-import java.io.{InputStream, IOException, OutputStream}
+import java.io.{FileNotFoundException, InputStream, IOException, OutputStream}
 import java.nio.charset.StandardCharsets.UTF_8
 
 import scala.io.{Source => IOSource}
@@ -28,7 +28,7 @@ import org.json4s.NoTypeHints
 import org.json4s.jackson.Serialization
 
 import org.apache.spark.sql.SparkSession
-import org.apache.spark.util.{SizeEstimator, Utils}
+import org.apache.spark.util.Utils
 
 /**
  * An abstract class for compactible metadata logs. It will write one log file for each batch.
@@ -107,9 +107,12 @@ abstract class CompactibleFileStreamLog[T <: AnyRef : ClassTag](
   }
 
   /**
-   * Filter out the obsolete logs.
+   * Determine whether the log should be retained or not.
+   *
+   * Default implementation retains all log entries. Implementations should override the method
+   * to change the behavior.
    */
-  def compactLogs(logs: Seq[T]): Seq[T]
+  def shouldRetain(log: T, currentTime: Long): Boolean = true
 
   override def batchIdToPath(batchId: Long): Path = {
     if (isCompactionBatch(batchId, compactInterval)) {
@@ -132,12 +135,18 @@ abstract class CompactibleFileStreamLog[T <: AnyRef : ClassTag](
     }
   }
 
+  private def serializeEntry(entry: T, out: OutputStream): Unit = {
+    out.write(Serialization.write(entry).getBytes(UTF_8))
+  }
+
+  private def deserializeEntry(line: String): T = Serialization.read[T](line)
+
   override def serialize(logData: Array[T], out: OutputStream): Unit = {
     // called inside a try-finally where the underlying stream is closed in the caller
     out.write(("v" + metadataLogVersion).getBytes(UTF_8))
     logData.foreach { data =>
       out.write('\n')
-      out.write(Serialization.write(data).getBytes(UTF_8))
+      serializeEntry(data, out)
     }
   }
 
@@ -147,7 +156,7 @@ abstract class CompactibleFileStreamLog[T <: AnyRef : ClassTag](
       throw new IllegalStateException("Incomplete log file")
     }
     validateVersion(lines.next(), metadataLogVersion)
-    lines.map(Serialization.read[T]).toArray
+    lines.map(deserializeEntry).toArray
   }
 
   override def add(batchId: Long, logs: Array[T]): Boolean = {
@@ -173,37 +182,65 @@ abstract class CompactibleFileStreamLog[T <: AnyRef : ClassTag](
   override def purge(thresholdBatchId: Long): Unit = throw new UnsupportedOperationException(
     s"Cannot purge as it might break internal state.")
 
+  /**
+   * Apply function on all entries in the specific batch. The method will throw
+   * FileNotFoundException if the metadata log file doesn't exist.
+   *
+   * NOTE: This doesn't fail early on corruption. The caller should handle the exception
+   * properly and make sure the logic is not affected by failing in the middle.
+   */
+  def foreachInBatch(batchId: Long)(fn: T => Unit): Unit = applyFnInBatch(batchId)(_.foreach(fn))
+
+  /**
+   * Apply filter on all entries in the specific batch.
+   */
+  def filterInBatch(batchId: Long)(predicate: T => Boolean): Option[Array[T]] = {
+    try {
+      Some(applyFnInBatch(batchId)(_.filter(predicate).toArray))
+    } catch {
+      case _: FileNotFoundException => None
+    }
+  }
+
+  private def applyFnInBatch[RET](batchId: Long)(fn: Iterator[T] => RET): RET = {
+    applyFnToBatchByStream(batchId) { input =>
+      val lines = IOSource.fromInputStream(input, UTF_8.name()).getLines()
+      if (!lines.hasNext) {
+        throw new IllegalStateException("Incomplete log file")
+      }
+      validateVersion(lines.next(), metadataLogVersion)
+      fn(lines.map(deserializeEntry))
+    }
+  }
+
   /**
    * Compacts all logs before `batchId` plus the provided `logs`, and writes them into the
    * corresponding `batchId` file. It will delete expired files as well if enabled.
    */
   private def compact(batchId: Long, logs: Array[T]): Boolean = {
-    val (allLogs, loadElapsedMs) = Utils.timeTakenMs {
-      val validBatches = getValidBatchesBeforeCompactionBatch(batchId, compactInterval)
-      validBatches.flatMap { id =>
-        super.get(id).getOrElse {
-          throw new IllegalStateException(
-            s"${batchIdToPath(id)} doesn't exist when compacting batch $batchId " +
-              s"(compactInterval: $compactInterval)")
-        }
-      } ++ logs
+    val curTime = System.currentTimeMillis()
+    def writeEntry(entry: T, output: OutputStream): Unit = {
+      if (shouldRetain(entry, curTime)) {
+        output.write('\n')
+        serializeEntry(entry, output)
+      }
     }
-    val compactedLogs = compactLogs(allLogs)
 
-    // Return false as there is another writer.
-    val (writeSucceed, writeElapsedMs) = Utils.timeTakenMs {
-      super.add(batchId, compactedLogs.toArray)
+    val (writeSucceed, elapsedMs) = Utils.timeTakenMs {
+      addNewBatchByStream(batchId) { output =>
+        output.write(("v" + metadataLogVersion).getBytes(UTF_8))
+        val validBatches = getValidBatchesBeforeCompactionBatch(batchId, compactInterval)
+        validBatches.foreach { id =>
+          foreachInBatch(id) { entry => writeEntry(entry, output) }
+        }
+        logs.foreach { entry => writeEntry(entry, output) }
+      }
     }
 
-    val elapsedMs = loadElapsedMs + writeElapsedMs
     if (elapsedMs >= COMPACT_LATENCY_WARN_THRESHOLD_MS) {
-      logWarning(s"Compacting took $elapsedMs ms (load: $loadElapsedMs ms," +
-        s" write: $writeElapsedMs ms) for compact batch $batchId")
-      logWarning(s"Loaded ${allLogs.size} entries (estimated ${SizeEstimator.estimate(allLogs)} " +
-        s"bytes in memory), and wrote ${compactedLogs.size} entries for compact batch $batchId")
+      logWarning(s"Compacting took $elapsedMs ms for compact batch $batchId")
     } else {
-      logDebug(s"Compacting took $elapsedMs ms (load: $loadElapsedMs ms," +
-        s" write: $writeElapsedMs ms) for compact batch $batchId")
+      logDebug(s"Compacting took $elapsedMs ms for compact batch $batchId")
     }
 
     writeSucceed
@@ -213,7 +250,8 @@ abstract class CompactibleFileStreamLog[T <: AnyRef : ClassTag](
    * Returns all files except the deleted ones.
    */
   def allFiles(): Array[T] = {
-    var latestId = getLatest().map(_._1).getOrElse(-1L)
+    val curTime = System.currentTimeMillis()
+    var latestId = getLatestBatchId().getOrElse(-1L)
     // There is a race condition when `FileStreamSink` is deleting old files and `StreamFileIndex`
     // is calling this method. This loop will retry the reading to deal with the
     // race condition.
@@ -222,21 +260,22 @@ abstract class CompactibleFileStreamLog[T <: AnyRef : ClassTag](
         try {
           val logs =
             getAllValidBatches(latestId, compactInterval).flatMap { id =>
-              super.get(id).getOrElse {
+              filterInBatch(id)(shouldRetain(_, curTime)).getOrElse {
                 throw new IllegalStateException(
                   s"${batchIdToPath(id)} doesn't exist " +
                     s"(latestId: $latestId, compactInterval: $compactInterval)")
               }
             }
-          return compactLogs(logs).toArray
+          return logs.toArray
         } catch {
           case e: IOException =>
             // Another process using `CompactibleFileStreamLog` may delete the batch files when
             // `StreamFileIndex` are reading. However, it only happens when a compaction is
             // deleting old files. If so, let's try the next compaction batch and we should find it.
             // Otherwise, this is a real IO issue and we should throw it.
-            latestId = nextCompactionBatchId(latestId, compactInterval)
-            super.get(latestId).getOrElse {
+            val expectedMinLatestId = nextCompactionBatchId(latestId, compactInterval)
+            latestId = super.getLatestBatchId().getOrElse(-1)
+            if (latestId < expectedMinLatestId) {
               throw e
             }
         }
@@ -249,7 +288,7 @@ abstract class CompactibleFileStreamLog[T <: AnyRef : ClassTag](
 
   /**
    * Delete expired log entries that proceed the currentBatchId and retain
-   * sufficient minimum number of batches (given by minBatchsToRetain). This
+   * sufficient minimum number of batches (given by minBatchesToRetain). This
    * equates to retaining the earliest compaction log that proceeds
    * batch id position currentBatchId + 1 - minBatchesToRetain. All log entries
    * prior to the earliest compaction log proceeding that position will be removed.
@@ -347,7 +386,7 @@ object CompactibleFileStreamLog {
     } else if (defaultInterval < (latestCompactBatchId + 1) / 2) {
       // Find the first divisor >= default compact interval
       def properDivisors(min: Int, n: Int) =
-        (min to n/2).view.filter(i => n % i == 0) :+ n
+        (min to n/2).view.filter(i => n % i == 0).toSeq :+ n
 
       properDivisors(defaultInterval, latestCompactBatchId + 1).head
     } else {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ContinuousRecordEndpoint.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ContinuousRecordEndpoint.scala
index 985a5fa6063ef..11bdfee460e66 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ContinuousRecordEndpoint.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ContinuousRecordEndpoint.scala
@@ -16,6 +16,8 @@
  */
 package org.apache.spark.sql.execution.streaming
 
+import scala.collection.mutable
+
 import org.apache.spark.SparkEnv
 import org.apache.spark.rpc.{RpcCallContext, RpcEnv, ThreadSafeRpcEndpoint}
 import org.apache.spark.sql.catalyst.expressions.UnsafeRow
@@ -33,7 +35,7 @@ case class GetRecord(offset: ContinuousRecordPartitionOffset)
  *                to the number of partitions.
  * @param lock a lock object for locking the buckets for read
  */
-class ContinuousRecordEndpoint(buckets: Seq[Seq[UnsafeRow]], lock: Object)
+class ContinuousRecordEndpoint(buckets: Seq[mutable.Seq[UnsafeRow]], lock: Object)
   extends ThreadSafeRpcEndpoint {
 
   private var startOffsets: Seq[Int] = List.fill(buckets.size)(0)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/EventTimeWatermarkExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/EventTimeWatermarkExec.scala
index 20fb06a851dd7..7e094fee32547 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/EventTimeWatermarkExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/EventTimeWatermarkExec.scala
@@ -125,4 +125,7 @@ case class EventTimeWatermarkExec(
       a
     }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): EventTimeWatermarkExec =
+    copy(child = newChild)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamOptions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamOptions.scala
index 712ed1585bc8a..6f43542fd6595 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamOptions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamOptions.scala
@@ -23,6 +23,7 @@ import scala.util.Try
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
+import org.apache.spark.sql.execution.datasources.{ModifiedAfterFilter, ModifiedBeforeFilter}
 import org.apache.spark.util.Utils
 
 /**
@@ -32,6 +33,16 @@ class FileStreamOptions(parameters: CaseInsensitiveMap[String]) extends Logging
 
   def this(parameters: Map[String, String]) = this(CaseInsensitiveMap(parameters))
 
+  checkDisallowedOptions(parameters)
+
+  private def checkDisallowedOptions(options: Map[String, String]): Unit = {
+    Seq(ModifiedBeforeFilter.PARAM_NAME, ModifiedAfterFilter.PARAM_NAME).foreach { param =>
+      if (parameters.contains(param)) {
+        throw new IllegalArgumentException(s"option '$param' is not allowed in file stream sources")
+      }
+    }
+  }
+
   val maxFilesPerTrigger: Option[Int] = parameters.get("maxFilesPerTrigger").map { str =>
     Try(str.toInt).toOption.filter(_ > 0).getOrElse {
       throw new IllegalArgumentException(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSink.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSink.scala
index b679f163fc561..876671c61d811 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSink.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSink.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.{DataFrame, SparkSession}
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.execution.datasources.{BasicWriteJobStatsTracker, FileFormat, FileFormatWriter}
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.util.SerializableConfiguration
+import org.apache.spark.util.{SerializableConfiguration, Utils}
 
 object FileStreamSink extends Logging {
   // The name of the subdirectory that is used to store metadata about which files are valid.
@@ -43,18 +43,31 @@ object FileStreamSink extends Logging {
     path match {
       case Seq(singlePath) =>
         val hdfsPath = new Path(singlePath)
-        val fs = hdfsPath.getFileSystem(hadoopConf)
-        if (fs.isDirectory(hdfsPath)) {
-          val metadataPath = new Path(hdfsPath, metadataDir)
-          checkEscapedMetadataPath(fs, metadataPath, sqlConf)
-          fs.exists(metadataPath)
-        } else {
-          false
+        try {
+          val fs = hdfsPath.getFileSystem(hadoopConf)
+          if (fs.isDirectory(hdfsPath)) {
+            val metadataPath = getMetadataLogPath(fs, hdfsPath, sqlConf)
+            fs.exists(metadataPath)
+          } else {
+            false
+          }
+        } catch {
+          case e: SparkException => throw e
+          case NonFatal(e) =>
+            logWarning(s"Assume no metadata directory. Error while looking for " +
+              s"metadata directory in the path: $singlePath.", e)
+            false
         }
       case _ => false
     }
   }
 
+  def getMetadataLogPath(fs: FileSystem, path: Path, sqlConf: SQLConf): Path = {
+    val metadataDir = new Path(path, FileStreamSink.metadataDir)
+    FileStreamSink.checkEscapedMetadataPath(fs, metadataDir, sqlConf)
+    metadataDir
+  }
+
   def checkEscapedMetadataPath(fs: FileSystem, metadataPath: Path, sqlConf: SQLConf): Unit = {
     if (sqlConf.getConf(SQLConf.STREAMING_CHECKPOINT_ESCAPED_PATH_CHECK_ENABLED)
         && StreamExecution.containsSpecialCharsInPath(metadataPath)) {
@@ -125,16 +138,15 @@ class FileStreamSink(
     partitionColumnNames: Seq[String],
     options: Map[String, String]) extends Sink with Logging {
 
+  import FileStreamSink._
+
   private val hadoopConf = sparkSession.sessionState.newHadoopConf()
   private val basePath = new Path(path)
-  private val logPath = {
-    val metadataDir = new Path(basePath, FileStreamSink.metadataDir)
-    val fs = metadataDir.getFileSystem(hadoopConf)
-    FileStreamSink.checkEscapedMetadataPath(fs, metadataDir, sparkSession.sessionState.conf)
-    metadataDir
-  }
-  private val fileLog =
-    new FileStreamSinkLog(FileStreamSinkLog.VERSION, sparkSession, logPath.toString)
+  private val logPath = getMetadataLogPath(basePath.getFileSystem(hadoopConf), basePath,
+    sparkSession.sessionState.conf)
+  private val retention = options.get("retention").map(Utils.timeStringAsMs)
+  private val fileLog = new FileStreamSinkLog(FileStreamSinkLog.VERSION, sparkSession,
+    logPath.toString, retention)
 
   private def basicWriteJobStatsTracker: BasicWriteJobStatsTracker = {
     val serializableHadoopConf = new SerializableConfiguration(hadoopConf)
@@ -142,7 +154,7 @@ class FileStreamSink(
   }
 
   override def addBatch(batchId: Long, data: DataFrame): Unit = {
-    if (batchId <= fileLog.getLatest().map(_._1).getOrElse(-1L)) {
+    if (batchId <= fileLog.getLatestBatchId().getOrElse(-1L)) {
       logInfo(s"Skipping already committed batch $batchId")
     } else {
       val committer = FileCommitProtocol.instantiate(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLog.scala
index 17b6874a61648..2d70d95c6850d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLog.scala
@@ -81,7 +81,8 @@ object SinkFileStatus {
 class FileStreamSinkLog(
     metadataLogVersion: Int,
     sparkSession: SparkSession,
-    path: String)
+    path: String,
+    _retentionMs: Option[Long] = None)
   extends CompactibleFileStreamLog[SinkFileStatus](metadataLogVersion, sparkSession, path) {
 
   private implicit val formats = Serialization.formats(NoTypeHints)
@@ -97,18 +98,30 @@ class FileStreamSinkLog(
     s"Please set ${SQLConf.FILE_SINK_LOG_COMPACT_INTERVAL.key} (was $defaultCompactInterval) " +
       "to a positive value.")
 
-  override def compactLogs(logs: Seq[SinkFileStatus]): Seq[SinkFileStatus] = {
-    val deletedFiles = logs.filter(_.action == FileStreamSinkLog.DELETE_ACTION).map(_.path).toSet
-    if (deletedFiles.isEmpty) {
-      logs
+  val retentionMs: Long = _retentionMs match {
+    case Some(retention) =>
+      logInfo(s"Retention is set to $retention ms")
+      retention
+
+    case _ => Long.MaxValue
+  }
+
+  override def shouldRetain(log: SinkFileStatus, currentTime: Long): Boolean = {
+    if (retentionMs < Long.MaxValue) {
+      if (currentTime - log.modificationTime > retentionMs) {
+        logDebug(s"${log.path} excluded by retention - current time: $currentTime / " +
+          s"modification time: ${log.modificationTime} / retention: $retentionMs ms.")
+        false
+      } else {
+        true
+      }
     } else {
-      logs.filter(f => !deletedFiles.contains(f.path))
+      true
     }
   }
 }
 
 object FileStreamSinkLog {
   val VERSION = 1
-  val DELETE_ACTION = "delete"
   val ADD_ACTION = "add"
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSource.scala
index e8ce8e1487093..e53c5a9c4024e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSource.scala
@@ -104,13 +104,15 @@ class FileStreamSource(
   // Visible for testing and debugging in production.
   val seenFiles = new SeenFilesMap(maxFileAgeMs, fileNameOnly)
 
-  metadataLog.allFiles().foreach { entry =>
+  metadataLog.restore().foreach { entry =>
     seenFiles.add(entry.path, entry.timestamp)
   }
   seenFiles.purge()
 
   logInfo(s"maxFilesPerBatch = $maxFilesPerBatch, maxFileAgeMs = $maxFileAgeMs")
 
+  private var unreadFiles: Seq[(String, Long)] = _
+
   /**
    * Returns the maximum offset that can be retrieved from the source.
    *
@@ -118,15 +120,45 @@ class FileStreamSource(
    * there is no race here, so the cost of `synchronized` should be rare.
    */
   private def fetchMaxOffset(limit: ReadLimit): FileStreamSourceOffset = synchronized {
-    // All the new files found - ignore aged files and files that we have seen.
-    val newFiles = fetchAllFiles().filter {
-      case (path, timestamp) => seenFiles.isNewFile(path, timestamp)
+    val newFiles = if (unreadFiles != null) {
+      logDebug(s"Reading from unread files - ${unreadFiles.size} files are available.")
+      unreadFiles
+    } else {
+      // All the new files found - ignore aged files and files that we have seen.
+      fetchAllFiles().filter {
+        case (path, timestamp) => seenFiles.isNewFile(path, timestamp)
+      }
     }
 
     // Obey user's setting to limit the number of files in this batch trigger.
-    val batchFiles = limit match {
-      case files: ReadMaxFiles => newFiles.take(files.maxFiles())
-      case _: ReadAllAvailable => newFiles
+    val (batchFiles, unselectedFiles) = limit match {
+      case files: ReadMaxFiles if !sourceOptions.latestFirst =>
+        // we can cache and reuse remaining fetched list of files in further batches
+        val (bFiles, usFiles) = newFiles.splitAt(files.maxFiles())
+        if (usFiles.size < files.maxFiles() * DISCARD_UNSEEN_FILES_RATIO) {
+          // Discard unselected files if the number of files are smaller than threshold.
+          // This is to avoid the case when the next batch would have too few files to read
+          // whereas there're new files available.
+          logTrace(s"Discarding ${usFiles.length} unread files as it's smaller than threshold.")
+          (bFiles, null)
+        } else {
+          (bFiles, usFiles)
+        }
+
+      case files: ReadMaxFiles =>
+        // implies "sourceOptions.latestFirst = true" which we want to refresh the list per batch
+        (newFiles.take(files.maxFiles()), null)
+
+      case _: ReadAllAvailable => (newFiles, null)
+    }
+
+    if (unselectedFiles != null && unselectedFiles.nonEmpty) {
+      logTrace(s"Taking first $MAX_CACHED_UNSEEN_FILES unread files.")
+      unreadFiles = unselectedFiles.take(MAX_CACHED_UNSEEN_FILES)
+      logTrace(s"${unreadFiles.size} unread files are available for further batches.")
+    } else {
+      unreadFiles = null
+      logTrace(s"No unread file is available for further batches.")
     }
 
     batchFiles.foreach { file =>
@@ -139,16 +171,23 @@ class FileStreamSource(
       s"""
          |Number of new files = ${newFiles.size}
          |Number of files selected for batch = ${batchFiles.size}
+         |Number of unread files = ${Option(unreadFiles).map(_.size).getOrElse(0)}
          |Number of seen files = ${seenFiles.size}
          |Number of files purged from tracking map = $numPurged
        """.stripMargin)
 
     if (batchFiles.nonEmpty) {
       metadataLogCurrentOffset += 1
-      metadataLog.add(metadataLogCurrentOffset, batchFiles.map { case (p, timestamp) =>
+
+      val fileEntries = batchFiles.map { case (p, timestamp) =>
         FileEntry(path = p, timestamp = timestamp, batchId = metadataLogCurrentOffset)
-      }.toArray)
-      logInfo(s"Log offset set to $metadataLogCurrentOffset with ${batchFiles.size} new files")
+      }.toArray
+      if (metadataLog.add(metadataLogCurrentOffset, fileEntries)) {
+        logInfo(s"Log offset set to $metadataLogCurrentOffset with ${batchFiles.size} new files")
+      } else {
+        throw new IllegalStateException("Concurrent update to the log. Multiple streaming jobs " +
+          s"detected for $metadataLogCurrentOffset")
+      }
     }
 
     FileStreamSourceOffset(metadataLogCurrentOffset)
@@ -311,6 +350,9 @@ object FileStreamSource {
   /** Timestamp for file modification time, in ms since January 1, 1970 UTC. */
   type Timestamp = Long
 
+  val DISCARD_UNSEEN_FILES_RATIO = 0.2
+  val MAX_CACHED_UNSEEN_FILES = 10000
+
   case class FileEntry(path: String, timestamp: Timestamp, batchId: Long) extends Serializable
 
   /**
@@ -482,7 +524,7 @@ object FileStreamSource {
     }
 
     private def buildSourceGlobFilters(sourcePath: Path): Seq[GlobFilter] = {
-      val filters = new scala.collection.mutable.MutableList[GlobFilter]()
+      val filters = new scala.collection.mutable.ArrayBuffer[GlobFilter]()
 
       var currentPath = sourcePath
       while (!currentPath.isRoot) {
@@ -490,7 +532,7 @@ object FileStreamSource {
         currentPath = currentPath.getParent
       }
 
-      filters.toList
+      filters.toSeq
     }
 
     override protected def cleanTask(entry: FileEntry): Unit = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSourceLog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSourceLog.scala
index 7b2ea9627a98e..5fe9a39c91e0b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSourceLog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSourceLog.scala
@@ -36,6 +36,7 @@ class FileStreamSourceLog(
   extends CompactibleFileStreamLog[FileEntry](metadataLogVersion, sparkSession, path) {
 
   import CompactibleFileStreamLog._
+  import FileStreamSourceLog._
 
   // Configurations about metadata compaction
   protected override val defaultCompactInterval: Int =
@@ -61,10 +62,6 @@ class FileStreamSourceLog(
     }
   }
 
-  def compactLogs(logs: Seq[FileEntry]): Seq[FileEntry] = {
-    logs
-  }
-
   override def add(batchId: Long, logs: Array[FileEntry]): Boolean = {
     if (super.add(batchId, logs)) {
       if (isCompactionBatch(batchId, compactInterval)) {
@@ -84,7 +81,7 @@ class FileStreamSourceLog(
       if (isCompactionBatch(id, compactInterval) && fileEntryCache.containsKey(id)) {
         (id, Some(fileEntryCache.get(id)))
       } else {
-        val logs = super.get(id).map(_.filter(_.batchId == id))
+        val logs = filterInBatch(id)(_.batchId == id)
         (id, logs)
       }
     }.partition(_._2.isDefined)
@@ -96,7 +93,7 @@ class FileStreamSourceLog(
     val searchKeys = removedBatches.map(_._1)
     val retrievedBatches = if (searchKeys.nonEmpty) {
       logWarning(s"Get batches from removed files, this is unexpected in the current code path!!!")
-      val latestBatchId = getLatest().map(_._1).getOrElse(-1L)
+      val latestBatchId = getLatestBatchId().getOrElse(-1L)
       if (latestBatchId < 0) {
         Map.empty[Long, Option[Array[FileEntry]]]
       } else {
@@ -122,8 +119,34 @@ class FileStreamSourceLog(
     }
     batches
   }
+
+  def restore(): Array[FileEntry] = {
+    val files = allFiles()
+
+    // When restarting the query, there is a case which the query starts from compaction batch,
+    // and the batch has source metadata file to read. One case is that the previous query
+    // succeeded to read from inputs, but not finalized the batch for various reasons.
+    // The below code finds the latest compaction batch, and put entries for the batch into the
+    // file entry cache which would avoid reading compact batch file twice.
+    // It doesn't know about offset / commit metadata in checkpoint so doesn't know which exactly
+    // batch to start from, but in practice, only couple of latest batches are candidates to
+    // be started. We leverage the fact to skip calculation if possible.
+    files.lastOption.foreach { lastEntry =>
+      val latestBatchId = lastEntry.batchId
+      val latestCompactedBatchId = getAllValidBatches(latestBatchId, compactInterval)(0)
+      if ((latestBatchId - latestCompactedBatchId) < PREV_NUM_BATCHES_TO_READ_IN_RESTORE) {
+        val logsForLatestCompactedBatch = files.filter { entry =>
+          entry.batchId == latestCompactedBatchId
+        }
+        fileEntryCache.put(latestCompactedBatchId, logsForLatestCompactedBatch)
+      }
+    }
+
+    files
+  }
 }
 
 object FileStreamSourceLog {
   val VERSION = 1
+  val PREV_NUM_BATCHES_TO_READ_IN_RESTORE = 2
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FlatMapGroupsWithStateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FlatMapGroupsWithStateExec.scala
index d65c4ffbb7a24..fe788dd8b9408 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FlatMapGroupsWithStateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FlatMapGroupsWithStateExec.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.execution.streaming
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-import org.apache.spark.sql.catalyst.expressions.{Ascending, Attribute, AttributeReference, Expression, Literal, SortOrder, UnsafeRow}
+import org.apache.spark.sql.catalyst.expressions.{Ascending, Attribute, Expression, SortOrder, UnsafeRow}
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.plans.physical.{ClusteredDistribution, Distribution}
 import org.apache.spark.sql.execution._
@@ -122,7 +122,7 @@ case class FlatMapGroupsWithStateExec(
           // If timeout is based on event time, then filter late data based on watermark
           val filteredIter = watermarkPredicateForData match {
             case Some(predicate) if timeoutConf == EventTimeTimeout =>
-              iter.filter(row => !predicate.eval(row))
+              applyRemovingRowsOlderThanWatermark(iter, predicate)
             case _ =>
               iter
           }
@@ -246,4 +246,7 @@ case class FlatMapGroupsWithStateExec(
       CompletionIterator[InternalRow, Iterator[InternalRow]](mappedIterator, onIteratorCompletion)
     }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): FlatMapGroupsWithStateExec =
+    copy(child = newChild)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLog.scala
index ed0c44da08c5d..a7756a4a99251 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLog.scala
@@ -19,14 +19,12 @@ package org.apache.spark.sql.execution.streaming
 
 import java.io._
 import java.nio.charset.StandardCharsets
-import java.util.{ConcurrentModificationException, EnumSet, UUID}
+import java.util.ConcurrentModificationException
 
 import scala.reflect.ClassTag
 
 import org.apache.commons.io.IOUtils
-import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs._
-import org.apache.hadoop.fs.permission.FsPermission
 import org.json4s.NoTypeHints
 import org.json4s.jackson.Serialization
 
@@ -115,42 +113,34 @@ class HDFSMetadataLog[T <: AnyRef : ClassTag](sparkSession: SparkSession, path:
    */
   override def add(batchId: Long, metadata: T): Boolean = {
     require(metadata != null, "'null' metadata cannot written to a metadata log")
-    get(batchId).map(_ => false).getOrElse {
-      // Only write metadata when the batch has not yet been written
-      writeBatchToFile(metadata, batchIdToPath(batchId))
-      true
-    }
+    addNewBatchByStream(batchId) { output => serialize(metadata, output) }
   }
 
-  /** Write a batch to a temp file then rename it to the batch file.
-   *
-   * There may be multiple [[HDFSMetadataLog]] using the same metadata path. Although it is not a
-   * valid behavior, we still need to prevent it from destroying the files.
-   */
-  private def writeBatchToFile(metadata: T, path: Path): Unit = {
-    val output = fileManager.createAtomic(path, overwriteIfPossible = false)
+  override def get(batchId: Long): Option[T] = {
     try {
-      serialize(metadata, output)
-      output.close()
+      applyFnToBatchByStream(batchId) { input => Some(deserialize(input)) }
     } catch {
-      case e: FileAlreadyExistsException =>
-        output.cancel()
-        // If next batch file already exists, then another concurrently running query has
-        // written it.
-        throw new ConcurrentModificationException(
-          s"Multiple streaming queries are concurrently using $path", e)
-      case e: Throwable =>
-        output.cancel()
-        throw e
+      case fne: FileNotFoundException =>
+        logDebug(fne.getMessage)
+        None
     }
   }
 
-  override def get(batchId: Long): Option[T] = {
+  /**
+   * Apply provided function to each entry in the specific batch metadata log.
+   *
+   * Unlike get which will materialize all entries into memory, this method streamlines the process
+   * via READ-AND-PROCESS. This helps to avoid the memory issue on huge metadata log file.
+   *
+   * NOTE: This no longer fails early on corruption. The caller should handle the exception
+   * properly and make sure the logic is not affected by failing in the middle.
+   */
+  def applyFnToBatchByStream[RET](batchId: Long)(fn: InputStream => RET): RET = {
     val batchMetadataFile = batchIdToPath(batchId)
     if (fileManager.exists(batchMetadataFile)) {
       val input = fileManager.open(batchMetadataFile)
       try {
-        Some(deserialize(input))
+        fn(input)
       } catch {
         case ise: IllegalStateException =>
           // re-throw the exception with the log file path added
@@ -160,8 +150,42 @@ class HDFSMetadataLog[T <: AnyRef : ClassTag](sparkSession: SparkSession, path:
         IOUtils.closeQuietly(input)
       }
     } else {
-      logDebug(s"Unable to find batch $batchMetadataFile")
-      None
+      throw new FileNotFoundException(s"Unable to find batch $batchMetadataFile")
+    }
+  }
+
+  /**
+   * Store the metadata for the specified batchId and return `true` if successful. This method
+   * fills the content of metadata via executing function. If the function throws an exception,
+   * writing will be automatically cancelled and this method will propagate the exception.
+   *
+   * If the batchId's metadata has already been stored, this method will return `false`.
+   *
+   * Writing the metadata is done by writing a batch to a temp file then rename it to the batch
+   * file.
+   *
+   * There may be multiple [[HDFSMetadataLog]] using the same metadata path. Although it is not a
+   * valid behavior, we still need to prevent it from destroying the files.
+   */
+  def addNewBatchByStream(batchId: Long)(fn: OutputStream => Unit): Boolean = {
+    get(batchId).map(_ => false).getOrElse {
+      // Only write metadata when the batch has not yet been written
+      val output = fileManager.createAtomic(batchIdToPath(batchId), overwriteIfPossible = false)
+      try {
+        fn(output)
+        output.close()
+      } catch {
+        case e: FileAlreadyExistsException =>
+          output.cancel()
+          // If next batch file already exists, then another concurrently running query has
+          // written it.
+          throw new ConcurrentModificationException(
+            s"Multiple streaming queries are concurrently using $path", e)
+        case e: Throwable =>
+          output.cancel()
+          throw e
+      }
+      true
     }
   }
 
@@ -182,17 +206,26 @@ class HDFSMetadataLog[T <: AnyRef : ClassTag](sparkSession: SparkSession, path:
     }
   }
 
-  override def getLatest(): Option[(Long, T)] = {
-    val batchIds = fileManager.list(metadataPath, batchFilesFilter)
+  /**
+   * Return the latest batch Id without reading the file. This method only checks for existence of
+   * file to avoid cost on reading and deserializing log file.
+   */
+  def getLatestBatchId(): Option[Long] = {
+    fileManager.list(metadataPath, batchFilesFilter)
       .map(f => pathToBatchId(f.getPath))
       .sorted(Ordering.Long.reverse)
-    for (batchId <- batchIds) {
-      val batch = get(batchId)
-      if (batch.isDefined) {
-        return Some((batchId, batch.get))
+      .headOption
+  }
+
+  override def getLatest(): Option[(Long, T)] = {
+    getLatestBatchId().map { batchId =>
+      val content = get(batchId).getOrElse {
+        // If we find the last batch file, we must read that file, other than failing back to
+        // old batches.
+        throw new IllegalStateException(s"failed to read log file for batch $batchId")
       }
+      (batchId, content)
     }
-    None
   }
 
   /**
@@ -206,18 +239,33 @@ class HDFSMetadataLog[T <: AnyRef : ClassTag](sparkSession: SparkSession, path:
       .reverse
   }
 
+  private var lastPurgedBatchId: Long = -1L
+
   /**
    * Removes all the log entry earlier than thresholdBatchId (exclusive).
    */
   override def purge(thresholdBatchId: Long): Unit = {
-    val batchIds = fileManager.list(metadataPath, batchFilesFilter)
-      .map(f => pathToBatchId(f.getPath))
-
-    for (batchId <- batchIds if batchId < thresholdBatchId) {
-      val path = batchIdToPath(batchId)
-      fileManager.delete(path)
-      logTrace(s"Removed metadata log file: $path")
+    val possibleTargetBatchIds = (lastPurgedBatchId + 1 until thresholdBatchId)
+    if (possibleTargetBatchIds.length <= 3) {
+      // avoid using list if we only need to purge at most 3 elements
+      possibleTargetBatchIds.foreach { batchId =>
+        val path = batchIdToPath(batchId)
+        fileManager.delete(path)
+        logTrace(s"Removed metadata log file: $path")
+      }
+    } else {
+      // using list to retrieve all elements
+      val batchIds = fileManager.list(metadataPath, batchFilesFilter)
+        .map(f => pathToBatchId(f.getPath))
+
+      for (batchId <- batchIds if batchId < thresholdBatchId) {
+        val path = batchIdToPath(batchId)
+        fileManager.delete(path)
+        logTrace(s"Removed metadata log file: $path")
+      }
     }
+
+    lastPurgedBatchId = thresholdBatchId - 1
   }
 
   /**
@@ -234,36 +282,8 @@ class HDFSMetadataLog[T <: AnyRef : ClassTag](sparkSession: SparkSession, path:
     }
   }
 
-  /**
-   * Parse the log version from the given `text` -- will throw exception when the parsed version
-   * exceeds `maxSupportedVersion`, or when `text` is malformed (such as "xyz", "v", "v-1",
-   * "v123xyz" etc.)
-   */
-  private[sql] def validateVersion(text: String, maxSupportedVersion: Int): Int = {
-    if (text.length > 0 && text(0) == 'v') {
-      val version =
-        try {
-          text.substring(1, text.length).toInt
-        } catch {
-          case _: NumberFormatException =>
-            throw new IllegalStateException(s"Log file was malformed: failed to read correct log " +
-              s"version from $text.")
-        }
-      if (version > 0) {
-        if (version > maxSupportedVersion) {
-          throw new IllegalStateException(s"UnsupportedLogVersion: maximum supported log version " +
-            s"is v${maxSupportedVersion}, but encountered v$version. The log file was produced " +
-            s"by a newer version of Spark and cannot be read by this version. Please upgrade.")
-        } else {
-          return version
-        }
-      }
-    }
-
-    // reaching here means we failed to read the correct log version
-    throw new IllegalStateException(s"Log file was malformed: failed to read correct log " +
-      s"version from $text.")
-  }
+  private[sql] def validateVersion(text: String, maxSupportedVersion: Int): Int =
+    MetadataVersionUtil.validateVersion(text, maxSupportedVersion)
 }
 
 object HDFSMetadataLog {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala
index 09ae7692ec518..b871874f52967 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala
@@ -21,14 +21,13 @@ import java.util.UUID
 import java.util.concurrent.atomic.AtomicInteger
 
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.{AnalysisException, SparkSession, Strategy}
+import org.apache.spark.sql.{SparkSession, Strategy}
 import org.apache.spark.sql.catalyst.QueryPlanningTracker
 import org.apache.spark.sql.catalyst.expressions.{CurrentBatchTimestamp, ExpressionWithRandomSeed}
 import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.catalyst.plans.physical.{AllTuples, ClusteredDistribution, HashPartitioning, SinglePartition}
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.execution.{LeafExecNode, LocalLimitExec, QueryExecution, SparkPlan, SparkPlanner, UnaryExecNode}
-import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
+import org.apache.spark.sql.execution.{LocalLimitExec, QueryExecution, SparkPlan, SparkPlanner, UnaryExecNode}
+import org.apache.spark.sql.execution.exchange.ShuffleExchangeLike
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.streaming.OutputMode
 import org.apache.spark.util.Utils
@@ -51,7 +50,6 @@ class IncrementalExecution(
   // Modified planner with stateful operations.
   override val planner: SparkPlanner = new SparkPlanner(
       sparkSession,
-      sparkSession.sessionState.conf,
       sparkSession.sessionState.experimentalMethods) {
     override def strategies: Seq[Strategy] =
       extraPlanningStrategies ++
@@ -76,7 +74,7 @@ class IncrementalExecution(
    * with the desired literal
    */
   override
-  lazy val optimizedPlan: LogicalPlan = tracker.measurePhase(QueryPlanningTracker.OPTIMIZATION) {
+  lazy val optimizedPlan: LogicalPlan = executePhase(QueryPlanningTracker.OPTIMIZATION) {
     sparkSession.sessionState.optimizer.executeAndTrack(withCachedData,
       tracker) transformAllExpressions {
       case ts @ CurrentBatchTimestamp(timestamp, _, _) =>
@@ -118,7 +116,7 @@ class IncrementalExecution(
           case s: StatefulOperator =>
             statefulOpFound = true
 
-          case e: ShuffleExchangeExec =>
+          case e: ShuffleExchangeLike =>
             // Don't search recursively any further as any child stateful operator as we
             // are only looking for stateful subplans that this plan has narrow dependencies on.
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ManifestFileCommitProtocol.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ManifestFileCommitProtocol.scala
index f6cc8116c6c4c..de8a8cd7d3b58 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ManifestFileCommitProtocol.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ManifestFileCommitProtocol.scala
@@ -139,7 +139,7 @@ class ManifestFileCommitProtocol(jobId: String, path: String)
     if (addedFiles.nonEmpty) {
       val fs = new Path(addedFiles.head).getFileSystem(taskContext.getConfiguration)
       val statuses: Seq[SinkFileStatus] =
-        addedFiles.map(f => SinkFileStatus(fs.getFileStatus(new Path(f))))
+        addedFiles.map(f => SinkFileStatus(fs.getFileStatus(new Path(f)))).toSeq
       new TaskCommitMessage(statuses)
     } else {
       new TaskCommitMessage(Seq.empty[SinkFileStatus])
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MetadataVersionUtil.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MetadataVersionUtil.scala
new file mode 100644
index 0000000000000..548f2aa5d5c5b
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MetadataVersionUtil.scala
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.streaming
+
+object MetadataVersionUtil {
+  /**
+   * Parse the log version from the given `text` -- will throw exception when the parsed version
+   * exceeds `maxSupportedVersion`, or when `text` is malformed (such as "xyz", "v", "v-1",
+   * "v123xyz" etc.)
+   */
+  def validateVersion(text: String, maxSupportedVersion: Int): Int = {
+    if (text.length > 0 && text(0) == 'v') {
+      val version =
+        try {
+          text.substring(1, text.length).toInt
+        } catch {
+          case _: NumberFormatException =>
+            throw new IllegalStateException(s"Log file was malformed: failed to read correct log " +
+              s"version from $text.")
+        }
+      if (version > 0) {
+        if (version > maxSupportedVersion) {
+          throw new IllegalStateException(s"UnsupportedLogVersion: maximum supported log version " +
+            s"is v${maxSupportedVersion}, but encountered v$version. The log file was produced " +
+            s"by a newer version of Spark and cannot be read by this version. Please upgrade.")
+        } else {
+          return version
+        }
+      }
+    }
+
+    // reaching here means we failed to read the correct log version
+    throw new IllegalStateException(s"Log file was malformed: failed to read correct log " +
+      s"version from $text.")
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala
index e022bfb6835d2..8f969db1342d5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala
@@ -23,30 +23,26 @@ import org.apache.spark.sql.{Dataset, SparkSession}
 import org.apache.spark.sql.catalyst.encoders.RowEncoder
 import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, CurrentBatchTimestamp, CurrentDate, CurrentTimestamp}
 import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LocalRelation, LogicalPlan, Project}
+import org.apache.spark.sql.catalyst.streaming.{StreamingRelationV2, WriteToStream}
 import org.apache.spark.sql.catalyst.util.truncatedString
-import org.apache.spark.sql.connector.catalog.{SupportsRead, SupportsWrite, Table, TableCapability}
+import org.apache.spark.sql.connector.catalog.{SupportsRead, SupportsWrite, TableCapability}
 import org.apache.spark.sql.connector.read.streaming.{MicroBatchStream, Offset => OffsetV2, ReadLimit, SparkDataStream, SupportsAdmissionControl}
 import org.apache.spark.sql.execution.SQLExecution
-import org.apache.spark.sql.execution.datasources.v2.{StreamingDataSourceV2Relation, StreamWriterCommitProgress, WriteToDataSourceV2Exec}
+import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Relation, StreamingDataSourceV2Relation, StreamWriterCommitProgress, WriteToDataSourceV2Exec}
 import org.apache.spark.sql.execution.streaming.sources.WriteToMicroBatchDataSource
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.streaming.{OutputMode, Trigger}
-import org.apache.spark.util.Clock
+import org.apache.spark.sql.streaming.Trigger
+import org.apache.spark.util.{Clock, Utils}
 
 class MicroBatchExecution(
     sparkSession: SparkSession,
-    name: String,
-    checkpointRoot: String,
-    analyzedPlan: LogicalPlan,
-    sink: Table,
     trigger: Trigger,
     triggerClock: Clock,
-    outputMode: OutputMode,
     extraOptions: Map[String, String],
-    deleteCheckpointOnStop: Boolean)
+    plan: WriteToStream)
   extends StreamExecution(
-    sparkSession, name, checkpointRoot, analyzedPlan, sink,
-    trigger, triggerClock, outputMode, deleteCheckpointOnStop) {
+    sparkSession, plan.name, plan.resolvedCheckpointLocation, plan.inputQuery, plan.sink, trigger,
+    triggerClock, plan.outputMode, plan.deleteCheckpointOnStop) {
 
   @volatile protected var sources: Seq[SparkDataStream] = Seq.empty
 
@@ -75,7 +71,7 @@ class MicroBatchExecution(
     // transformation is responsible for replacing attributes with their final values.
 
     val disabledSources =
-      sparkSession.sqlContext.conf.disabledV2StreamingMicroBatchReaders.split(",")
+      Utils.stringToSeq(sparkSession.sqlContext.conf.disabledV2StreamingMicroBatchReaders)
 
     import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Implicits._
     val _logicalPlan = analyzedPlan.transform {
@@ -89,14 +85,15 @@ class MicroBatchExecution(
           StreamingExecutionRelation(source, output)(sparkSession)
         })
 
-      case s @ StreamingRelationV2(src, srcName, table: SupportsRead, options, output, v1) =>
-        val v2Disabled = disabledSources.contains(src.getClass.getCanonicalName)
+      case s @ StreamingRelationV2(src, srcName, table: SupportsRead, options, output, _, _, v1) =>
+        val dsStr = if (src.nonEmpty) s"[${src.get}]" else ""
+        val v2Disabled = disabledSources.contains(src.getOrElse(None).getClass.getCanonicalName)
         if (!v2Disabled && table.supports(TableCapability.MICRO_BATCH_READ)) {
           v2ToRelationMap.getOrElseUpdate(s, {
             // Materialize source to avoid creating it in every batch
             val metadataPath = s"$resolvedCheckpointRoot/sources/$nextSourceId"
             nextSourceId += 1
-            logInfo(s"Reading table [$table] from DataSourceV2 named '$srcName' [$src]")
+            logInfo(s"Reading table [$table] from DataSourceV2 named '$srcName' $dsStr")
             // TODO: operator pushdown.
             val scan = table.newScanBuilder(options).build()
             val stream = scan.toMicroBatchStream(metadataPath)
@@ -109,9 +106,10 @@ class MicroBatchExecution(
           v2ToExecutionRelationMap.getOrElseUpdate(s, {
             // Materialize source to avoid creating it in every batch
             val metadataPath = s"$resolvedCheckpointRoot/sources/$nextSourceId"
-            val source = v1.get.dataSource.createSource(metadataPath)
+            val source =
+              v1.get.asInstanceOf[StreamingRelation].dataSource.createSource(metadataPath)
             nextSourceId += 1
-            logInfo(s"Using Source [$source] from DataSourceV2 named '$srcName' [$src]")
+            logInfo(s"Using Source [$source] from DataSourceV2 named '$srcName' $dsStr")
             StreamingExecutionRelation(source, output)(sparkSession)
           })
         }
@@ -139,7 +137,10 @@ class MicroBatchExecution(
     sink match {
       case s: SupportsWrite =>
         val streamingWrite = createStreamingWrite(s, extraOptions, _logicalPlan)
-        WriteToMicroBatchDataSource(streamingWrite, _logicalPlan)
+        val relationOpt = plan.catalogAndIdent.map {
+          case (catalog, ident) => DataSourceV2Relation.create(s, Some(catalog), Some(ident))
+        }
+        WriteToMicroBatchDataSource(relationOpt, streamingWrite, _logicalPlan)
 
       case _ => _logicalPlan
     }
@@ -209,7 +210,10 @@ class MicroBatchExecution(
           }
 
           // Record the trigger offset range for progress reporting *before* processing the batch
-          recordTriggerOffsets(from = committedOffsets, to = availableOffsets)
+          recordTriggerOffsets(
+            from = committedOffsets,
+            to = availableOffsets,
+            latest = latestOffsets)
 
           // Remember whether the current batch has data or not. This will be required later
           // for bookkeeping after running the batch, when `isNewDataAvailable` will have changed
@@ -318,6 +322,17 @@ class MicroBatchExecution(
               committedOffsets ++= availableOffsets
               watermarkTracker.setWatermark(
                 math.max(watermarkTracker.currentWatermark, commitMetadata.nextBatchWatermarkMs))
+            } else if (latestCommittedBatchId == latestBatchId - 1) {
+              availableOffsets.foreach {
+                case (source: Source, end: Offset) =>
+                  val start = committedOffsets.get(source).map(_.asInstanceOf[Offset])
+                  if (start.map(_ == end).getOrElse(true)) {
+                    source.getBatch(start, end)
+                  }
+                case nonV1Tuple =>
+                  // The V2 API does not have the same edge case requiring getBatch to be called
+                  // here, so we do nothing here.
+              }
             } else if (latestCommittedBatchId < latestBatchId - 1) {
               logWarning(s"Batch completion log latest batch id is " +
                 s"${latestCommittedBatchId}, which is not trailing " +
@@ -365,7 +380,7 @@ class MicroBatchExecution(
     if (isCurrentBatchConstructed) return true
 
     // Generate a map from each unique source to the next available offset.
-    val latestOffsets: Map[SparkDataStream, Option[OffsetV2]] = uniqueSources.map {
+    val (nextOffsets, recentOffsets) = uniqueSources.toSeq.map {
       case (s: SupportsAdmissionControl, limit) =>
         updateStatusMessage(s"Getting offsets from $s")
         reportTimeTaken("latestOffset") {
@@ -377,23 +392,31 @@ class MicroBatchExecution(
               startOffsetOpt.map(offset => v2.deserializeOffset(offset.json))
                 .getOrElse(v2.initialOffset())
           }
-          (s, Option(s.latestOffset(startOffset, limit)))
+          val next = s.latestOffset(startOffset, limit)
+          val latest = s.reportLatestOffset()
+          ((s, Option(next)), (s, Option(latest)))
         }
       case (s: Source, _) =>
         updateStatusMessage(s"Getting offsets from $s")
         reportTimeTaken("getOffset") {
-          (s, s.getOffset)
+          val offset = s.getOffset
+          ((s, offset), (s, offset))
         }
       case (s: MicroBatchStream, _) =>
         updateStatusMessage(s"Getting offsets from $s")
         reportTimeTaken("latestOffset") {
-          (s, Option(s.latestOffset()))
+          val latest = s.latestOffset()
+          ((s, Option(latest)), (s, Option(latest)))
         }
       case (s, _) =>
         // for some reason, the compiler is unhappy and thinks the match is not exhaustive
         throw new IllegalStateException(s"Unexpected source: $s")
-    }
-    availableOffsets ++= latestOffsets.filter { case (_, o) => o.nonEmpty }.mapValues(_.get)
+    }.unzip
+
+    availableOffsets ++= nextOffsets.filter { case (_, o) => o.nonEmpty }
+      .map(p => p._1 -> p._2.get).toMap
+    latestOffsets ++= recentOffsets.filter { case (_, o) => o.nonEmpty }
+      .map(p => p._1 -> p._2.get).toMap
 
     // Update the query metadata
     offsetSeqMetadata = offsetSeqMetadata.copy(
@@ -566,8 +589,7 @@ class MicroBatchExecution(
     val nextBatch =
       new Dataset(lastExecution, RowEncoder(lastExecution.analyzed.schema))
 
-    val batchSinkProgress: Option[StreamWriterCommitProgress] =
-      reportTimeTaken("addBatch") {
+    val batchSinkProgress: Option[StreamWriterCommitProgress] = reportTimeTaken("addBatch") {
       SQLExecution.withNewExecutionId(lastExecution) {
         sink match {
           case s: Sink => s.addBatch(currentBatchId, nextBatch)
@@ -585,7 +607,9 @@ class MicroBatchExecution(
     withProgressLocked {
       sinkCommitProgress = batchSinkProgress
       watermarkTracker.updateWatermark(lastExecution.executedPlan)
-      commitLog.add(currentBatchId, CommitMetadata(watermarkTracker.currentWatermark))
+      assert(commitLog.add(currentBatchId, CommitMetadata(watermarkTracker.currentWatermark)),
+        "Concurrent update to the commit log. Multiple streaming jobs detected for " +
+          s"$currentBatchId")
       committedOffsets ++= availableOffsets
     }
     logDebug(s"Completed batch ${currentBatchId}")
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/OffsetSeq.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/OffsetSeq.scala
index 1c59464268444..7d7ec76467836 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/OffsetSeq.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/OffsetSeq.scala
@@ -89,10 +89,15 @@ case class OffsetSeqMetadata(
 
 object OffsetSeqMetadata extends Logging {
   private implicit val format = Serialization.formats(NoTypeHints)
+  /**
+   * These configs are related to streaming query execution and should not be changed across
+   * batches of a streaming query. The values of these configs are persisted into the offset
+   * log in the checkpoint position.
+   */
   private val relevantSQLConfs = Seq(
     SHUFFLE_PARTITIONS, STATE_STORE_PROVIDER_CLASS, STREAMING_MULTIPLE_WATERMARK_POLICY,
     FLATMAPGROUPSWITHSTATE_STATE_FORMAT_VERSION, STREAMING_AGGREGATION_STATE_FORMAT_VERSION,
-    STREAMING_JOIN_STATE_FORMAT_VERSION)
+    STREAMING_JOIN_STATE_FORMAT_VERSION, STATE_STORE_COMPRESSION_CODEC)
 
   /**
    * Default values of relevant configurations that are used for backward compatibility.
@@ -111,7 +116,8 @@ object OffsetSeqMetadata extends Logging {
     STREAMING_AGGREGATION_STATE_FORMAT_VERSION.key ->
       StreamingAggregationStateManager.legacyVersion.toString,
     STREAMING_JOIN_STATE_FORMAT_VERSION.key ->
-      SymmetricHashJoinStateManager.legacyVersion.toString
+      SymmetricHashJoinStateManager.legacyVersion.toString,
+    STATE_STORE_COMPRESSION_CODEC.key -> "lz4"
   )
 
   def apply(json: String): OffsetSeqMetadata = Serialization.read[OffsetSeqMetadata](json)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/OffsetSeqLog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/OffsetSeqLog.scala
index f6543c3e4c4ca..82e50263893db 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/OffsetSeqLog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/OffsetSeqLog.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.execution.streaming
 
 
+import java.{util => ju}
 import java.io.{InputStream, OutputStream}
 import java.nio.charset.StandardCharsets._
 
@@ -46,6 +47,23 @@ import org.apache.spark.sql.connector.read.streaming.{Offset => OffsetV2}
 class OffsetSeqLog(sparkSession: SparkSession, path: String)
   extends HDFSMetadataLog[OffsetSeq](sparkSession, path) {
 
+  private val cachedMetadata = new ju.TreeMap[Long, OffsetSeq]()
+
+  override def add(batchId: Long, metadata: OffsetSeq): Boolean = {
+    val added = super.add(batchId, metadata)
+    if (added) {
+      // cache metadata as it will be read again
+      cachedMetadata.put(batchId, metadata)
+      // we don't access metadata for (batchId - 2) batches; evict them
+      cachedMetadata.headMap(batchId - 2, true).clear()
+    }
+    added
+  }
+
+  override def get(batchId: Long): Option[OffsetSeq] = {
+    Option(cachedMetadata.get(batchId)).orElse(super.get(batchId))
+  }
+
   override protected def deserialize(in: InputStream): OffsetSeq = {
     // called inside a try-finally where the underlying stream is closed in the caller
     def parseOffset(value: String): OffsetV2 = value match {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala
index 0dff1c2fe5768..f9313977fe1f2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.execution.streaming
 
 import java.text.SimpleDateFormat
-import java.util.{Date, UUID}
+import java.util.{Date, Optional, UUID}
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable
@@ -29,7 +29,7 @@ import org.apache.spark.sql.catalyst.plans.logical.{EventTimeWatermark, LogicalP
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.MILLIS_PER_SECOND
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.connector.catalog.Table
-import org.apache.spark.sql.connector.read.streaming.{MicroBatchStream, SparkDataStream}
+import org.apache.spark.sql.connector.read.streaming.{MicroBatchStream, ReportsSourceMetrics, SparkDataStream}
 import org.apache.spark.sql.execution.QueryExecution
 import org.apache.spark.sql.execution.datasources.v2.{MicroBatchScanExec, StreamingDataSourceV2Relation, StreamWriterCommitProgress}
 import org.apache.spark.sql.streaming._
@@ -71,6 +71,8 @@ trait ProgressReporter extends Logging {
   private var currentTriggerEndTimestamp = -1L
   private var currentTriggerStartOffsets: Map[SparkDataStream, String] = _
   private var currentTriggerEndOffsets: Map[SparkDataStream, String] = _
+  private var currentTriggerLatestOffsets: Map[SparkDataStream, String] = _
+
   // TODO: Restore this from the checkpoint when possible.
   private var lastTriggerStartTimestamp = -1L
 
@@ -99,6 +101,8 @@ trait ProgressReporter extends Logging {
       isTriggerActive = false)
   }
 
+  private var latestStreamProgress: StreamProgress = _
+
   /** Returns the current status of the query. */
   def status: StreamingQueryStatus = currentStatus
 
@@ -119,6 +123,7 @@ trait ProgressReporter extends Logging {
     currentTriggerStartTimestamp = triggerClock.getTimeMillis()
     currentTriggerStartOffsets = null
     currentTriggerEndOffsets = null
+    currentTriggerLatestOffsets = null
     currentDurationsMs.clear()
   }
 
@@ -126,9 +131,14 @@ trait ProgressReporter extends Logging {
    * Record the offsets range this trigger will process. Call this before updating
    * `committedOffsets` in `StreamExecution` to make sure that the correct range is recorded.
    */
-  protected def recordTriggerOffsets(from: StreamProgress, to: StreamProgress): Unit = {
-    currentTriggerStartOffsets = from.mapValues(_.json)
-    currentTriggerEndOffsets = to.mapValues(_.json)
+  protected def recordTriggerOffsets(
+      from: StreamProgress,
+      to: StreamProgress,
+      latest: StreamProgress): Unit = {
+    currentTriggerStartOffsets = from.mapValues(_.json).toMap
+    currentTriggerEndOffsets = to.mapValues(_.json).toMap
+    currentTriggerLatestOffsets = latest.mapValues(_.json).toMap
+    latestStreamProgress = to
   }
 
   private def updateProgress(newProgress: StreamingQueryProgress): Unit = {
@@ -151,7 +161,8 @@ trait ProgressReporter extends Logging {
    *                    though the sources don't have any new data.
    */
   protected def finishTrigger(hasNewData: Boolean, hasExecuted: Boolean): Unit = {
-    assert(currentTriggerStartOffsets != null && currentTriggerEndOffsets != null)
+    assert(currentTriggerStartOffsets != null && currentTriggerEndOffsets != null &&
+      currentTriggerLatestOffsets != null)
     currentTriggerEndTimestamp = triggerClock.getTimeMillis()
 
     val executionStats = extractExecutionStats(hasNewData, hasExecuted)
@@ -161,19 +172,26 @@ trait ProgressReporter extends Logging {
     val inputTimeSec = if (lastTriggerStartTimestamp >= 0) {
       (currentTriggerStartTimestamp - lastTriggerStartTimestamp).toDouble / MILLIS_PER_SECOND
     } else {
-      Double.NaN
+      Double.PositiveInfinity
     }
     logDebug(s"Execution stats: $executionStats")
 
     val sourceProgress = sources.distinct.map { source =>
       val numRecords = executionStats.inputRows.getOrElse(source, 0L)
+      val sourceMetrics = source match {
+        case withMetrics: ReportsSourceMetrics =>
+          withMetrics.metrics(Optional.ofNullable(latestStreamProgress.get(source).orNull))
+        case _ => Map[String, String]().asJava
+      }
       new SourceProgress(
         description = source.toString,
         startOffset = currentTriggerStartOffsets.get(source).orNull,
         endOffset = currentTriggerEndOffsets.get(source).orNull,
+        latestOffset = currentTriggerLatestOffsets.get(source).orNull,
         numInputRows = numRecords,
         inputRowsPerSecond = numRecords / inputTimeSec,
-        processedRowsPerSecond = numRecords / processingTimeSec
+        processedRowsPerSecond = numRecords / processingTimeSec,
+        metrics = sourceMetrics
       )
     }
 
@@ -192,7 +210,8 @@ trait ProgressReporter extends Logging {
       timestamp = formatTimestamp(currentTriggerStartTimestamp),
       batchId = currentBatchId,
       batchDuration = processingTimeMills,
-      durationMs = new java.util.HashMap(currentDurationsMs.toMap.mapValues(long2Long).asJava),
+      durationMs =
+        new java.util.HashMap(currentDurationsMs.toMap.mapValues(long2Long).toMap.asJava),
       eventTime = new java.util.HashMap(executionStats.eventTimeStats.asJava),
       stateOperators = executionStats.stateOperators.toArray,
       sources = sourceProgress.toArray,
@@ -201,7 +220,7 @@ trait ProgressReporter extends Logging {
 
     if (hasExecuted) {
       // Reset noDataEventTimestamp if we processed any data
-      lastNoExecutionProgressEventTime = Long.MinValue
+      lastNoExecutionProgressEventTime = triggerClock.getTimeMillis()
       updateProgress(newProgress)
     } else {
       val now = triggerClock.getTimeMillis()
@@ -222,7 +241,11 @@ trait ProgressReporter extends Logging {
     lastExecution.executedPlan.collect {
       case p if p.isInstanceOf[StateStoreWriter] =>
         val progress = p.asInstanceOf[StateStoreWriter].getProgress()
-        if (hasExecuted) progress else progress.copy(newNumRowsUpdated = 0)
+        if (hasExecuted) {
+          progress
+        } else {
+          progress.copy(newNumRowsUpdated = 0, newNumRowsDroppedByWatermark = 0)
+        }
     }
   }
 
@@ -251,14 +274,14 @@ trait ProgressReporter extends Logging {
           "avg" -> stats.avg.toLong).mapValues(formatTimestamp)
     }.headOption.getOrElse(Map.empty) ++ watermarkTimestamp
 
-    ExecutionStats(numInputRows, stateOperators, eventTimeStats)
+    ExecutionStats(numInputRows, stateOperators, eventTimeStats.toMap)
   }
 
   /** Extract number of input sources for each streaming source in plan */
   private def extractSourceToNumInputRows(): Map[SparkDataStream, Long] = {
 
     def sumRows(tuples: Seq[(SparkDataStream, Long)]): Map[SparkDataStream, Long] = {
-      tuples.groupBy(_._1).mapValues(_.map(_._2).sum) // sum up rows for each source
+      tuples.groupBy(_._1).mapValues(_.map(_._2).sum).toMap // sum up rows for each source
     }
 
     val onlyDataSourceV2Sources = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ResolveWriteToStream.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ResolveWriteToStream.scala
new file mode 100644
index 0000000000000..10bc927c2623f
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ResolveWriteToStream.scala
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.streaming
+
+import java.util.UUID
+
+import scala.util.control.NonFatal
+
+import org.apache.hadoop.fs.Path
+
+import org.apache.spark.SparkException
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.SQLConfHelper
+import org.apache.spark.sql.catalyst.analysis.UnsupportedOperationChecker
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.streaming.{WriteToStream, WriteToStreamStatement}
+import org.apache.spark.sql.connector.catalog.SupportsWrite
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.util.Utils
+
+/**
+ * Replaces logical [[WriteToStreamStatement]] operator with an [[WriteToStream]] operator.
+ */
+object ResolveWriteToStream extends Rule[LogicalPlan] with SQLConfHelper {
+  def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperators {
+    case s: WriteToStreamStatement =>
+      val (resolvedCheckpointLocation, deleteCheckpointOnStop) = resolveCheckpointLocation(s)
+
+      if (conf.adaptiveExecutionEnabled) {
+        logWarning(s"${SQLConf.ADAPTIVE_EXECUTION_ENABLED.key} " +
+          "is not supported in streaming DataFrames/Datasets and will be disabled.")
+      }
+
+      if (conf.isUnsupportedOperationCheckEnabled) {
+        if (s.sink.isInstanceOf[SupportsWrite] && s.isContinuousTrigger) {
+          UnsupportedOperationChecker.checkForContinuous(s.inputQuery, s.outputMode)
+        } else {
+          UnsupportedOperationChecker.checkForStreaming(s.inputQuery, s.outputMode)
+        }
+      }
+
+      WriteToStream(
+        s.userSpecifiedName.orNull,
+        resolvedCheckpointLocation,
+        s.sink,
+        s.outputMode,
+        deleteCheckpointOnStop,
+        s.inputQuery,
+        s.catalogAndIdent)
+  }
+
+  def resolveCheckpointLocation(s: WriteToStreamStatement): (String, Boolean) = {
+    var deleteCheckpointOnStop = false
+    val checkpointLocation = s.userSpecifiedCheckpointLocation.map { userSpecified =>
+      new Path(userSpecified).toString
+    }.orElse {
+      conf.checkpointLocation.map { location =>
+        new Path(location, s.userSpecifiedName.getOrElse(UUID.randomUUID().toString)).toString
+      }
+    }.getOrElse {
+      if (s.useTempCheckpointLocation) {
+        deleteCheckpointOnStop = true
+        val tempDir = Utils.createTempDir(namePrefix = s"temporary").getCanonicalPath
+        logWarning("Temporary checkpoint location created which is deleted normally when" +
+          s" the query didn't fail: $tempDir. If it's required to delete it under any" +
+          s" circumstances, please set ${SQLConf.FORCE_DELETE_TEMP_CHECKPOINT_LOCATION.key} to" +
+          s" true. Important to know deleting temp checkpoint folder is best effort.")
+        tempDir
+      } else {
+        throw new AnalysisException(
+          "checkpointLocation must be specified either " +
+            """through option("checkpointLocation", ...) or """ +
+            s"""SparkSession.conf.set("${SQLConf.CHECKPOINT_LOCATION.key}", ...)""")
+      }
+    }
+    val fileManager = CheckpointFileManager.create(new Path(checkpointLocation), s.hadoopConf)
+
+    // If offsets have already been created, we trying to resume a query.
+    if (!s.recoverFromCheckpointLocation) {
+      val checkpointPath = new Path(checkpointLocation, "offsets")
+      if (fileManager.exists(checkpointPath)) {
+        throw new AnalysisException(
+          s"This query does not support recovering from checkpoint location. " +
+            s"Delete $checkpointPath to start over.")
+      }
+    }
+
+    val resolvedCheckpointRoot = {
+      val checkpointPath = new Path(checkpointLocation)
+      if (conf.getConf(SQLConf.STREAMING_CHECKPOINT_ESCAPED_PATH_CHECK_ENABLED)
+        && StreamExecution.containsSpecialCharsInPath(checkpointPath)) {
+        // In Spark 2.4 and earlier, the checkpoint path is escaped 3 times (3 `Path.toUri.toString`
+        // calls). If this legacy checkpoint path exists, we will throw an error to tell the user
+        // how to migrate.
+        val legacyCheckpointDir =
+        new Path(new Path(checkpointPath.toUri.toString).toUri.toString).toUri.toString
+        val legacyCheckpointDirExists =
+          try {
+            fileManager.exists(new Path(legacyCheckpointDir))
+          } catch {
+            case NonFatal(e) =>
+              // We may not have access to this directory. Don't fail the query if that happens.
+              logWarning(e.getMessage, e)
+              false
+          }
+        if (legacyCheckpointDirExists) {
+          throw new SparkException(
+            s"""Error: we detected a possible problem with the location of your checkpoint and you
+               |likely need to move it before restarting this query.
+               |
+               |Earlier version of Spark incorrectly escaped paths when writing out checkpoints for
+               |structured streaming. While this was corrected in Spark 3.0, it appears that your
+               |query was started using an earlier version that incorrectly handled the checkpoint
+               |path.
+               |
+               |Correct Checkpoint Directory: $checkpointPath
+               |Incorrect Checkpoint Directory: $legacyCheckpointDir
+               |
+               |Please move the data from the incorrect directory to the correct one, delete the
+               |incorrect directory, and then restart this query. If you believe you are receiving
+               |this message in error, you can disable it with the SQL conf
+               |${SQLConf.STREAMING_CHECKPOINT_ESCAPED_PATH_CHECK_ENABLED.key}."""
+              .stripMargin)
+        }
+      }
+      val checkpointDir = fileManager.createCheckpointDirectory()
+      checkpointDir.toString
+    }
+    logInfo(s"Checkpoint root $checkpointLocation resolved to $resolvedCheckpointRoot.")
+    (resolvedCheckpointRoot, deleteCheckpointOnStop)
+  }
+}
+
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
index 9b1951a834d9a..400906bb79645 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
@@ -40,10 +40,10 @@ import org.apache.spark.sql.connector.catalog.{SupportsWrite, Table}
 import org.apache.spark.sql.connector.read.streaming.{Offset => OffsetV2, ReadLimit, SparkDataStream}
 import org.apache.spark.sql.connector.write.{LogicalWriteInfoImpl, SupportsTruncate}
 import org.apache.spark.sql.connector.write.streaming.StreamingWrite
-import org.apache.spark.sql.execution.QueryExecution
 import org.apache.spark.sql.execution.command.StreamingExplainCommand
 import org.apache.spark.sql.execution.datasources.v2.StreamWriterCommitProgress
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.connector.SupportsStreamingUpdateAsAppend
 import org.apache.spark.sql.streaming._
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 import org.apache.spark.util.{Clock, UninterruptibleThread, Utils}
@@ -68,8 +68,8 @@ case object RECONFIGURING extends State
 abstract class StreamExecution(
     override val sparkSession: SparkSession,
     override val name: String,
-    private val checkpointRoot: String,
-    analyzedPlan: LogicalPlan,
+    val resolvedCheckpointRoot: String,
+    val analyzedPlan: LogicalPlan,
     val sink: Table,
     val trigger: Trigger,
     val triggerClock: Clock,
@@ -94,51 +94,6 @@ abstract class StreamExecution(
   private val startLatch = new CountDownLatch(1)
   private val terminationLatch = new CountDownLatch(1)
 
-  val resolvedCheckpointRoot = {
-    val checkpointPath = new Path(checkpointRoot)
-    val fs = checkpointPath.getFileSystem(sparkSession.sessionState.newHadoopConf())
-    if (sparkSession.conf.get(SQLConf.STREAMING_CHECKPOINT_ESCAPED_PATH_CHECK_ENABLED)
-        && StreamExecution.containsSpecialCharsInPath(checkpointPath)) {
-      // In Spark 2.4 and earlier, the checkpoint path is escaped 3 times (3 `Path.toUri.toString`
-      // calls). If this legacy checkpoint path exists, we will throw an error to tell the user how
-      // to migrate.
-      val legacyCheckpointDir =
-        new Path(new Path(checkpointPath.toUri.toString).toUri.toString).toUri.toString
-      val legacyCheckpointDirExists =
-        try {
-          fs.exists(new Path(legacyCheckpointDir))
-        } catch {
-          case NonFatal(e) =>
-            // We may not have access to this directory. Don't fail the query if that happens.
-            logWarning(e.getMessage, e)
-            false
-        }
-      if (legacyCheckpointDirExists) {
-        throw new SparkException(
-          s"""Error: we detected a possible problem with the location of your checkpoint and you
-             |likely need to move it before restarting this query.
-             |
-             |Earlier version of Spark incorrectly escaped paths when writing out checkpoints for
-             |structured streaming. While this was corrected in Spark 3.0, it appears that your
-             |query was started using an earlier version that incorrectly handled the checkpoint
-             |path.
-             |
-             |Correct Checkpoint Directory: $checkpointPath
-             |Incorrect Checkpoint Directory: $legacyCheckpointDir
-             |
-             |Please move the data from the incorrect directory to the correct one, delete the
-             |incorrect directory, and then restart this query. If you believe you are receiving
-             |this message in error, you can disable it with the SQL conf
-             |${SQLConf.STREAMING_CHECKPOINT_ESCAPED_PATH_CHECK_ENABLED.key}."""
-            .stripMargin)
-      }
-    }
-    val checkpointDir = checkpointPath.makeQualified(fs.getUri, fs.getWorkingDirectory)
-    fs.mkdirs(checkpointDir)
-    checkpointDir.toString
-  }
-  logInfo(s"Checkpoint root $checkpointRoot resolved to $resolvedCheckpointRoot.")
-
   def logicalPlan: LogicalPlan
 
   /**
@@ -161,6 +116,15 @@ abstract class StreamExecution(
   @volatile
   var availableOffsets = new StreamProgress
 
+  /**
+   * Tracks the latest offsets for each input source.
+   * Only the scheduler thread should modify this field, and only in atomic steps.
+   * Other threads should make a shallow copy if they are going to access this field more than
+   * once, since the field's value may change at any time.
+   */
+  @volatile
+  var latestOffsets = new StreamProgress
+
   @volatile
   var sinkCommitProgress: Option[StreamWriterCommitProgress] = None
 
@@ -273,6 +237,10 @@ abstract class StreamExecution(
   protected def checkpointFile(name: String): String =
     new Path(new Path(resolvedCheckpointRoot), name).toString
 
+  /** All checkpoint file operations should be performed through `CheckpointFileManager`. */
+  private val fileManager = CheckpointFileManager.create(new Path(resolvedCheckpointRoot),
+      sparkSession.sessionState.newHadoopConf)
+
   /**
    * Starts the execution. This returns only after the thread has started and [[QueryStartedEvent]]
    * has been posted to all the listeners.
@@ -314,26 +282,28 @@ abstract class StreamExecution(
       startLatch.countDown()
 
       // While active, repeatedly attempt to run batches.
-      SparkSession.setActiveSession(sparkSession)
-
-      updateStatusMessage("Initializing sources")
-      // force initialization of the logical plan so that the sources can be created
-      logicalPlan
-
-      // Adaptive execution can change num shuffle partitions, disallow
-      sparkSessionForStream.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false")
-      // Disable cost-based join optimization as we do not want stateful operations to be rearranged
-      sparkSessionForStream.conf.set(SQLConf.CBO_ENABLED.key, "false")
-      offsetSeqMetadata = OffsetSeqMetadata(
-        batchWatermarkMs = 0, batchTimestampMs = 0, sparkSessionForStream.conf)
-
-      if (state.compareAndSet(INITIALIZING, ACTIVE)) {
-        // Unblock `awaitInitialization`
-        initializationLatch.countDown()
-        runActivatedStream(sparkSessionForStream)
-        updateStatusMessage("Stopped")
-      } else {
-        // `stop()` is already called. Let `finally` finish the cleanup.
+      sparkSessionForStream.withActive {
+        // Adaptive execution can change num shuffle partitions, disallow
+        sparkSessionForStream.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false")
+        // Disable cost-based join optimization as we do not want stateful operations
+        // to be rearranged
+        sparkSessionForStream.conf.set(SQLConf.CBO_ENABLED.key, "false")
+
+        updateStatusMessage("Initializing sources")
+        // force initialization of the logical plan so that the sources can be created
+        logicalPlan
+
+        offsetSeqMetadata = OffsetSeqMetadata(
+          batchWatermarkMs = 0, batchTimestampMs = 0, sparkSessionForStream.conf)
+
+        if (state.compareAndSet(INITIALIZING, ACTIVE)) {
+          // Unblock `awaitInitialization`
+          initializationLatch.countDown()
+          runActivatedStream(sparkSessionForStream)
+          updateStatusMessage("Stopped")
+        } else {
+          // `stop()` is already called. Let `finally` finish the cleanup.
+        }
       }
     } catch {
       case e if isInterruptedByStop(e, sparkSession.sparkContext) =>
@@ -389,8 +359,7 @@ abstract class StreamExecution(
           val checkpointPath = new Path(resolvedCheckpointRoot)
           try {
             logInfo(s"Deleting checkpoint $checkpointPath.")
-            val fs = checkpointPath.getFileSystem(sparkSession.sessionState.newHadoopConf())
-            fs.delete(checkpointPath, true)
+            fileManager.delete(checkpointPath)
           } catch {
             case NonFatal(e) =>
               // Deleting temp checkpoint folder is best effort, don't throw non fatal exceptions
@@ -618,26 +587,22 @@ abstract class StreamExecution(
       inputPlan.schema,
       new CaseInsensitiveStringMap(options.asJava))
     val writeBuilder = table.newWriteBuilder(info)
-    outputMode match {
+    val write = outputMode match {
       case Append =>
-        writeBuilder.buildForStreaming()
+        writeBuilder.build()
 
       case Complete =>
         // TODO: we should do this check earlier when we have capability API.
         require(writeBuilder.isInstanceOf[SupportsTruncate],
           table.name + " does not support Complete mode.")
-        writeBuilder.asInstanceOf[SupportsTruncate].truncate().buildForStreaming()
+        writeBuilder.asInstanceOf[SupportsTruncate].truncate().build()
 
       case Update =>
-        // Although no v2 sinks really support Update mode now, but during tests we do want them
-        // to pretend to support Update mode, and treat Update mode same as Append mode.
-        if (Utils.isTesting) {
-          writeBuilder.buildForStreaming()
-        } else {
-          throw new IllegalArgumentException(
-            "Data source v2 streaming sinks does not support Update mode.")
-        }
+        require(writeBuilder.isInstanceOf[SupportsStreamingUpdateAsAppend],
+          table.name + " does not support Update mode.")
+        writeBuilder.asInstanceOf[SupportsStreamingUpdateAsAppend].build()
     }
+    write.toStreaming
   }
 
   protected def purge(threshold: Long): Unit = {
@@ -690,6 +655,6 @@ object StreamExecution {
 
 /**
  * A special thread to run the stream query. Some codes require to run in the QueryExecutionThread
- * and will use `classOf[QueryxecutionThread]` to check.
+ * and will use `classOf[QueryExecutionThread]` to check.
  */
 abstract class QueryExecutionThread(name: String) extends UninterruptibleThread(name)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamMetadata.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamMetadata.scala
index 516afbea5d9de..a4bcc5fba1048 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamMetadata.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamMetadata.scala
@@ -31,14 +31,14 @@ import org.json4s.jackson.Serialization
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.execution.streaming.CheckpointFileManager.CancellableFSDataOutputStream
-import org.apache.spark.sql.streaming.StreamingQuery
 
 /**
- * Contains metadata associated with a [[StreamingQuery]]. This information is written
- * in the checkpoint location the first time a query is started and recovered every time the query
- * is restarted.
+ * Contains metadata associated with a [[org.apache.spark.sql.streaming.StreamingQuery]].
+ * This information is written in the checkpoint location the first time a query is started
+ * and recovered every time the query is restarted.
  *
- * @param id  unique id of the [[StreamingQuery]] that needs to be persisted across restarts
+ * @param id  unique id of the [[org.apache.spark.sql.streaming.StreamingQuery]]
+ *            that needs to be persisted across restarts
  */
 case class StreamMetadata(id: String) {
   def json: String = Serialization.write(this)(StreamMetadata.format)
@@ -49,11 +49,12 @@ object StreamMetadata extends Logging {
 
   /** Read the metadata from file if it exists */
   def read(metadataFile: Path, hadoopConf: Configuration): Option[StreamMetadata] = {
-    val fs = metadataFile.getFileSystem(hadoopConf)
-    if (fs.exists(metadataFile)) {
+    val fileManager = CheckpointFileManager.create(metadataFile.getParent, hadoopConf)
+
+    if (fileManager.exists(metadataFile)) {
       var input: FSDataInputStream = null
       try {
-        input = fs.open(metadataFile)
+        input = fileManager.open(metadataFile)
         val reader = new InputStreamReader(input, StandardCharsets.UTF_8)
         val metadata = Serialization.read[StreamMetadata](reader)
         Some(metadata)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingQueryListenerBus.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingQueryListenerBus.scala
index 1b8d69ffb7521..4b98acd16f6fc 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingQueryListenerBus.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingQueryListenerBus.scala
@@ -31,16 +31,21 @@ import org.apache.spark.util.ListenerBus
  * Spark listener bus, so that it can receive [[StreamingQueryListener.Event]]s and dispatch them
  * to StreamingQueryListeners.
  *
- * Note that each bus and its registered listeners are associated with a single SparkSession
+ * Note 1: Each bus and its registered listeners are associated with a single SparkSession
  * and StreamingQueryManager. So this bus will dispatch events to registered listeners for only
  * those queries that were started in the associated SparkSession.
+ *
+ * Note 2: To rebuild Structured Streaming UI in SHS, this bus will be registered into
+ * [[org.apache.spark.scheduler.ReplayListenerBus]]. We check `sparkListenerBus` defined or not to
+ * determine how to process [[StreamingQueryListener.Event]]. If false, it means this bus is used to
+ * replay all streaming query event from eventLog.
  */
-class StreamingQueryListenerBus(sparkListenerBus: LiveListenerBus)
+class StreamingQueryListenerBus(sparkListenerBus: Option[LiveListenerBus])
   extends SparkListener with ListenerBus[StreamingQueryListener, StreamingQueryListener.Event] {
 
   import StreamingQueryListener._
 
-  sparkListenerBus.addToQueue(this, StreamingQueryListenerBus.STREAM_EVENT_QUERY)
+  sparkListenerBus.foreach(_.addToQueue(this, StreamingQueryListenerBus.STREAM_EVENT_QUERY))
 
   /**
    * RunIds of active queries whose events are supposed to be forwarded by this ListenerBus
@@ -67,11 +72,11 @@ class StreamingQueryListenerBus(sparkListenerBus: LiveListenerBus)
     event match {
       case s: QueryStartedEvent =>
         activeQueryRunIds.synchronized { activeQueryRunIds += s.runId }
-        sparkListenerBus.post(s)
+        sparkListenerBus.foreach(bus => bus.post(s))
         // post to local listeners to trigger callbacks
         postToAll(s)
       case _ =>
-        sparkListenerBus.post(event)
+        sparkListenerBus.foreach(bus => bus.post(event))
     }
   }
 
@@ -95,7 +100,11 @@ class StreamingQueryListenerBus(sparkListenerBus: LiveListenerBus)
         // synchronously and the ones attached to LiveListenerBus asynchronously. Therefore,
         // we need to ignore QueryStartedEvent if this method is called within SparkListenerBus
         // thread
-        if (!LiveListenerBus.withinListenerThread.value || !e.isInstanceOf[QueryStartedEvent]) {
+        //
+        // When loaded by Spark History Server, we should process all event coming from replay
+        // listener bus.
+        if (sparkListenerBus.isEmpty || !LiveListenerBus.withinListenerThread.value ||
+            !e.isInstanceOf[QueryStartedEvent])  {
           postToAll(e)
         }
       case _ =>
@@ -110,7 +119,10 @@ class StreamingQueryListenerBus(sparkListenerBus: LiveListenerBus)
       listener: StreamingQueryListener,
       event: StreamingQueryListener.Event): Unit = {
     def shouldReport(runId: UUID): Boolean = {
-      activeQueryRunIds.synchronized { activeQueryRunIds.contains(runId) }
+      // When loaded by Spark History Server, we should process all event coming from replay
+      // listener bus.
+      sparkListenerBus.isEmpty ||
+        activeQueryRunIds.synchronized { activeQueryRunIds.contains(runId) }
     }
 
     event match {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingRelation.scala
index 5858c54ce554a..b841ac8eea803 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingRelation.scala
@@ -23,11 +23,9 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.MultiInstanceRelation
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, Statistics}
-import org.apache.spark.sql.connector.catalog.{Table, TableProvider}
 import org.apache.spark.sql.connector.read.streaming.SparkDataStream
 import org.apache.spark.sql.execution.LeafExecNode
 import org.apache.spark.sql.execution.datasources.DataSource
-import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
 object StreamingRelation {
   def apply(dataSource: DataSource): StreamingRelation = {
@@ -83,35 +81,6 @@ case class StreamingExecutionRelation(
   override def newInstance(): LogicalPlan = this.copy(output = output.map(_.newInstance()))(session)
 }
 
-// We have to pack in the V1 data source as a shim, for the case when a source implements
-// continuous processing (which is always V2) but only has V1 microbatch support. We don't
-// know at read time whether the query is continuous or not, so we need to be able to
-// swap a V1 relation back in.
-/**
- * Used to link a [[TableProvider]] into a streaming
- * [[org.apache.spark.sql.catalyst.plans.logical.LogicalPlan]]. This is only used for creating
- * a streaming [[org.apache.spark.sql.DataFrame]] from [[org.apache.spark.sql.DataFrameReader]],
- * and should be converted before passing to [[StreamExecution]].
- */
-case class StreamingRelationV2(
-    source: TableProvider,
-    sourceName: String,
-    table: Table,
-    extraOptions: CaseInsensitiveStringMap,
-    output: Seq[Attribute],
-    v1Relation: Option[StreamingRelation])(session: SparkSession)
-  extends LeafNode with MultiInstanceRelation {
-  override def otherCopyArgs: Seq[AnyRef] = session :: Nil
-  override def isStreaming: Boolean = true
-  override def toString: String = sourceName
-
-  override def computeStats(): Statistics = Statistics(
-    sizeInBytes = BigInt(session.sessionState.conf.defaultSizeInBytes)
-  )
-
-  override def newInstance(): LogicalPlan = this.copy(output = output.map(_.newInstance()))(session)
-}
-
 /**
  * A dummy physical plan for [[StreamingRelation]] to support
  * [[org.apache.spark.sql.Dataset.explain]]
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingSymmetricHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingSymmetricHashJoinExec.scala
index 198e17db419a7..b2c8141e5db0e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingSymmetricHashJoinExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingSymmetricHashJoinExec.scala
@@ -56,8 +56,8 @@ import org.apache.spark.util.{CompletionIterator, SerializableConfiguration}
  * - Apply the optional condition to filter the joined rows as the final output.
  *
  * If a timestamp column with event time watermark is present in the join keys or in the input
- * data, then the it uses the watermark figure out which rows in the buffer will not join with
- * and the new data, and therefore can be discarded. Depending on the provided query conditions, we
+ * data, then it uses the watermark to figure out which rows in the buffer will not join with
+ * the new data, and therefore can be discarded. Depending on the provided query conditions, we
  * can define thresholds on both state key (i.e. joining keys) and state value (i.e. input rows).
  * There are three kinds of queries possible regarding this as explained below.
  * Assume that watermark has been defined on both `leftTime` and `rightTime` columns used below.
@@ -134,7 +134,7 @@ case class StreamingSymmetricHashJoinExec(
     stateWatermarkPredicates: JoinStateWatermarkPredicates,
     stateFormatVersion: Int,
     left: SparkPlan,
-    right: SparkPlan) extends SparkPlan with BinaryExecNode with StateStoreWriter {
+    right: SparkPlan) extends BinaryExecNode with StateStoreWriter {
 
   def this(
       leftKeys: Seq[Expression],
@@ -152,19 +152,28 @@ case class StreamingSymmetricHashJoinExec(
   }
 
   if (stateFormatVersion < 2 && joinType != Inner) {
-    throw new IllegalArgumentException("The query is using stream-stream outer join with state" +
+    throw new IllegalArgumentException(
+      s"The query is using stream-stream $joinType join with state" +
       s" format version ${stateFormatVersion} - correctness issue is discovered. Please discard" +
       " the checkpoint and rerun the query. See SPARK-26154 for more details.")
   }
 
+  private lazy val errorMessageForJoinType =
+    s"${getClass.getSimpleName} should not take $joinType as the JoinType"
+
   private def throwBadJoinTypeException(): Nothing = {
-    throw new IllegalArgumentException(
-      s"${getClass.getSimpleName} should not take $joinType as the JoinType")
+    throw new IllegalArgumentException(errorMessageForJoinType)
+  }
+
+  private def throwBadStateFormatVersionException(): Nothing = {
+    throw new IllegalStateException("Unexpected state format version! " +
+      s"version $stateFormatVersion")
   }
 
   require(
-    joinType == Inner || joinType == LeftOuter || joinType == RightOuter,
-    s"${getClass.getSimpleName} should not take $joinType as the JoinType")
+    joinType == Inner || joinType == LeftOuter || joinType == RightOuter || joinType == FullOuter ||
+    joinType == LeftSemi,
+    errorMessageForJoinType)
   require(leftKeys.map(_.dataType) == rightKeys.map(_.dataType))
 
   private val storeConf = new StateStoreConf(sqlContext.conf)
@@ -183,17 +192,19 @@ case class StreamingSymmetricHashJoinExec(
     case _: InnerLike => left.output ++ right.output
     case LeftOuter => left.output ++ right.output.map(_.withNullability(true))
     case RightOuter => left.output.map(_.withNullability(true)) ++ right.output
+    case FullOuter => (left.output ++ right.output).map(_.withNullability(true))
+    case LeftSemi => left.output
     case _ => throwBadJoinTypeException()
   }
 
   override def outputPartitioning: Partitioning = joinType match {
     case _: InnerLike =>
       PartitioningCollection(Seq(left.outputPartitioning, right.outputPartitioning))
-    case LeftOuter => PartitioningCollection(Seq(left.outputPartitioning))
-    case RightOuter => PartitioningCollection(Seq(right.outputPartitioning))
-    case x =>
-      throw new IllegalArgumentException(
-        s"${getClass.getSimpleName} should not take $x as the JoinType")
+    case LeftOuter => left.outputPartitioning
+    case RightOuter => right.outputPartitioning
+    case FullOuter => UnknownPartitioning(left.outputPartitioning.numPartitions)
+    case LeftSemi => left.outputPartitioning
+    case _ => throwBadJoinTypeException()
   }
 
   override def shouldRunAnotherBatch(newMetadata: OffsetSeqMetadata): Boolean = {
@@ -246,13 +257,21 @@ case class StreamingSymmetricHashJoinExec(
 
     //  Join one side input using the other side's buffered/state rows. Here is how it is done.
     //
-    //  - `leftJoiner.joinWith(rightJoiner)` generates all rows from matching new left input with
-    //    stored right input, and also stores all the left input
+    //  - `leftSideJoiner.storeAndJoinWithOtherSide(rightSideJoiner)`
+    //    - Inner, Left Outer, Right Outer, Full Outer Join: generates all rows from matching
+    //      new left input with stored right input, and also stores all the left input.
+    //    - Left Semi Join: generates all new left input rows from matching new left input with
+    //      stored right input, and also stores all the non-matched left input.
     //
-    //  - `rightJoiner.joinWith(leftJoiner)` generates all rows from matching new right input with
-    //    stored left input, and also stores all the right input. It also generates all rows from
-    //    matching new left input with new right input, since the new left input has become stored
-    //    by that point. This tiny asymmetry is necessary to avoid duplication.
+    //  - `rightSideJoiner.storeAndJoinWithOtherSide(leftSideJoiner)`
+    //    - Inner, Left Outer, Right Outer, Full Outer Join: generates all rows from matching
+    //      new right input with stored left input, and also stores all the right input.
+    //      It also generates all rows from matching new left input with new right input, since
+    //      the new left input has become stored by that point. This tiny asymmetry is necessary
+    //      to avoid duplication.
+    //    - Left Semi Join: generates all stored left input rows, from matching new right input
+    //      with stored left input, and also stores all the right input. Note only first-time
+    //      matched left input rows will be generated, this is to guarantee left semi semantics.
     val leftOutputIter = leftSideJoiner.storeAndJoinWithOtherSide(rightSideJoiner) {
       (input: InternalRow, matched: InternalRow) => joinedRow.withLeft(input).withRight(matched)
     }
@@ -260,22 +279,21 @@ case class StreamingSymmetricHashJoinExec(
       (input: InternalRow, matched: InternalRow) => joinedRow.withLeft(matched).withRight(input)
     }
 
-    // We need to save the time that the inner join output iterator completes, since outer join
-    // output counts as both update and removal time.
-    var innerOutputCompletionTimeNs: Long = 0
-    def onInnerOutputCompletion = {
-      innerOutputCompletionTimeNs = System.nanoTime
+    // We need to save the time that the one side hash join output iterator completes, since
+    // other join output counts as both update and removal time.
+    var hashJoinOutputCompletionTimeNs: Long = 0
+    def onHashJoinOutputCompletion(): Unit = {
+      hashJoinOutputCompletionTimeNs = System.nanoTime
     }
-    // This is the iterator which produces the inner join rows. For outer joins, this will be
-    // prepended to a second iterator producing outer join rows; for inner joins, this is the full
-    // output.
-    val innerOutputIter = CompletionIterator[InternalRow, Iterator[InternalRow]](
-      (leftOutputIter ++ rightOutputIter), onInnerOutputCompletion)
-
+    // This is the iterator which produces the inner and left semi join rows. For other joins,
+    // this will be prepended to a second iterator producing other rows; for inner and left semi
+    // joins, this is the full output.
+    val hashJoinOutputIter = CompletionIterator[InternalRow, Iterator[InternalRow]](
+      leftOutputIter ++ rightOutputIter, onHashJoinOutputCompletion())
 
     val outputIter: Iterator[InternalRow] = joinType match {
-      case Inner =>
-        innerOutputIter
+      case Inner | LeftSemi =>
+        hashJoinOutputIter
       case LeftOuter =>
         // We generate the outer join input by:
         // * Getting an iterator over the rows that have aged out on the left side. These rows are
@@ -295,18 +313,20 @@ case class StreamingSymmetricHashJoinExec(
             postJoinFilter(joinedRow.withLeft(leftKeyValue.value).withRight(rightValue))
           }
         }
+
+        // NOTE: we need to make sure `outerOutputIter` is evaluated "after" exhausting all of
+        // elements in `innerOutputIter`, because evaluation of `innerOutputIter` may update
+        // the match flag which the logic for outer join is relying on.
         val removedRowIter = leftSideJoiner.removeOldState()
         val outerOutputIter = removedRowIter.filterNot { kv =>
           stateFormatVersion match {
             case 1 => matchesWithRightSideState(new UnsafeRowPair(kv.key, kv.value))
             case 2 => kv.matched
-            case _ =>
-              throw new IllegalStateException("Unexpected state format version! " +
-                s"version $stateFormatVersion")
+            case _ => throwBadStateFormatVersionException()
           }
         }.map(pair => joinedRow.withLeft(pair.value).withRight(nullRight))
 
-        innerOutputIter ++ outerOutputIter
+        hashJoinOutputIter ++ outerOutputIter
       case RightOuter =>
         // See comments for left outer case.
         def matchesWithLeftSideState(rightKeyValue: UnsafeRowPair) = {
@@ -319,17 +339,31 @@ case class StreamingSymmetricHashJoinExec(
           stateFormatVersion match {
             case 1 => matchesWithLeftSideState(new UnsafeRowPair(kv.key, kv.value))
             case 2 => kv.matched
-            case _ =>
-              throw new IllegalStateException("Unexpected state format version! " +
-                s"version $stateFormatVersion")
+            case _ => throwBadStateFormatVersionException()
           }
         }.map(pair => joinedRow.withLeft(nullLeft).withRight(pair.value))
 
-        innerOutputIter ++ outerOutputIter
+        hashJoinOutputIter ++ outerOutputIter
+      case FullOuter =>
+        lazy val isKeyToValuePairMatched = (kv: KeyToValuePair) =>
+          stateFormatVersion match {
+            case 2 => kv.matched
+            case _ => throwBadStateFormatVersionException()
+          }
+        val leftSideOutputIter = leftSideJoiner.removeOldState().filterNot(
+          isKeyToValuePairMatched).map(pair => joinedRow.withLeft(pair.value).withRight(nullRight))
+        val rightSideOutputIter = rightSideJoiner.removeOldState().filterNot(
+          isKeyToValuePairMatched).map(pair => joinedRow.withLeft(nullLeft).withRight(pair.value))
+
+        hashJoinOutputIter ++ leftSideOutputIter ++ rightSideOutputIter
       case _ => throwBadJoinTypeException()
     }
 
-    val outputProjection = UnsafeProjection.create(left.output ++ right.output, output)
+    val outputProjection = if (joinType == LeftSemi) {
+      UnsafeProjection.create(output, output)
+    } else {
+      UnsafeProjection.create(left.output ++ right.output, output)
+    }
     val outputIterWithMetrics = outputIter.map { row =>
       numOutputRows += 1
       outputProjection(row)
@@ -340,26 +374,35 @@ case class StreamingSymmetricHashJoinExec(
       // All processing time counts as update time.
       allUpdatesTimeMs += math.max(NANOSECONDS.toMillis(System.nanoTime - updateStartTimeNs), 0)
 
-      // Processing time between inner output completion and here comes from the outer portion of a
-      // join, and thus counts as removal time as we remove old state from one side while iterating.
-      if (innerOutputCompletionTimeNs != 0) {
+      // Processing time between one side hash join output completion and here comes from the
+      // outer portion of a join, and thus counts as removal time as we remove old state from
+      // one side while iterating.
+      if (hashJoinOutputCompletionTimeNs != 0) {
         allRemovalsTimeMs +=
-          math.max(NANOSECONDS.toMillis(System.nanoTime - innerOutputCompletionTimeNs), 0)
+          math.max(NANOSECONDS.toMillis(System.nanoTime - hashJoinOutputCompletionTimeNs), 0)
       }
 
       allRemovalsTimeMs += timeTakenMs {
         // Remove any remaining state rows which aren't needed because they're below the watermark.
         //
-        // For inner joins, we have to remove unnecessary state rows from both sides if possible.
-        // For outer joins, we have already removed unnecessary state rows from the outer side
-        // (e.g., left side for left outer join) while generating the outer "null" outputs. Now, we
-        // have to remove unnecessary state rows from the other side (e.g., right side for the left
-        // outer join) if possible. In all cases, nothing needs to be outputted, hence the removal
-        // needs to be done greedily by immediately consuming the returned iterator.
+        // For inner and left semi joins, we have to remove unnecessary state rows from both sides
+        // if possible.
+        //
+        // For left outer and right outer joins, we have already removed unnecessary state rows from
+        // the outer side (e.g., left side for left outer join) while generating the outer "null"
+        // outputs. Now, we have to remove unnecessary state rows from the other side (e.g., right
+        // side for the left outer join) if possible. In all cases, nothing needs to be outputted,
+        // hence the removal needs to be done greedily by immediately consuming the returned
+        // iterator.
+        //
+        // For full outer joins, we have already removed unnecessary states from both sides, so
+        // nothing needs to be outputted here.
         val cleanupIter = joinType match {
-          case Inner => leftSideJoiner.removeOldState() ++ rightSideJoiner.removeOldState()
+          case Inner | LeftSemi =>
+            leftSideJoiner.removeOldState() ++ rightSideJoiner.removeOldState()
           case LeftOuter => rightSideJoiner.removeOldState()
           case RightOuter => leftSideJoiner.removeOldState()
+          case FullOuter => Iterator.empty
           case _ => throwBadJoinTypeException()
         }
         while (cleanupIter.hasNext) {
@@ -455,18 +498,47 @@ case class StreamingSymmetricHashJoinExec(
      */
     def storeAndJoinWithOtherSide(
         otherSideJoiner: OneSideHashJoiner)(
-        generateJoinedRow: (InternalRow, InternalRow) => JoinedRow):
-    Iterator[InternalRow] = {
+        generateJoinedRow: (InternalRow, InternalRow) => JoinedRow)
+      : Iterator[InternalRow] = {
+
       val watermarkAttribute = inputAttributes.find(_.metadata.contains(delayKey))
       val nonLateRows =
         WatermarkSupport.watermarkExpression(watermarkAttribute, eventTimeWatermark) match {
           case Some(watermarkExpr) =>
             val predicate = Predicate.create(watermarkExpr, inputAttributes)
-            inputIter.filter { row => !predicate.eval(row) }
+            applyRemovingRowsOlderThanWatermark(inputIter, predicate)
           case None =>
             inputIter
         }
 
+      val generateFilteredJoinedRow: InternalRow => Iterator[InternalRow] = joinSide match {
+        case LeftSide if joinType == LeftOuter || joinType == FullOuter =>
+          (row: InternalRow) => Iterator(generateJoinedRow(row, nullRight))
+        case RightSide if joinType == RightOuter || joinType == FullOuter =>
+          (row: InternalRow) => Iterator(generateJoinedRow(row, nullLeft))
+        case _ => (_: InternalRow) => Iterator.empty
+      }
+
+      val excludeRowsAlreadyMatched = joinType == LeftSemi && joinSide == RightSide
+
+      val generateOutputIter: (InternalRow, Iterator[JoinedRow]) => Iterator[InternalRow] =
+        joinSide match {
+          case LeftSide if joinType == LeftSemi =>
+            (input: InternalRow, joinedRowIter: Iterator[JoinedRow]) =>
+              // For left side of left semi join, generate one left row if there is matched
+              // rows from right side. Otherwise, generate nothing.
+              if (joinedRowIter.nonEmpty) {
+                Iterator.single(input)
+              } else {
+                Iterator.empty
+              }
+          case RightSide if joinType == LeftSemi =>
+            (_: InternalRow, joinedRowIter: Iterator[JoinedRow]) =>
+              // For right side of left semi join, generate matched left rows only.
+              joinedRowIter.map(_.getLeft)
+          case _ => (_: InternalRow, joinedRowIter: Iterator[JoinedRow]) => joinedRowIter
+        }
+
       nonLateRows.flatMap { row =>
         val thisRow = row.asInstanceOf[UnsafeRow]
         // If this row fails the pre join filter, that means it can never satisfy the full join
@@ -475,17 +547,15 @@ case class StreamingSymmetricHashJoinExec(
         // the case of inner join).
         if (preJoinFilter(thisRow)) {
           val key = keyGenerator(thisRow)
-          val outputIter: Iterator[JoinedRow] = otherSideJoiner.joinStateManager
-            .getJoinedRows(key, thatRow => generateJoinedRow(thisRow, thatRow), postJoinFilter)
+          val joinedRowIter: Iterator[JoinedRow] = otherSideJoiner.joinStateManager.getJoinedRows(
+            key,
+            thatRow => generateJoinedRow(thisRow, thatRow),
+            postJoinFilter,
+            excludeRowsAlreadyMatched)
+          val outputIter = generateOutputIter(thisRow, joinedRowIter)
           new AddingProcessedRowToStateCompletionIterator(key, thisRow, outputIter)
         } else {
-          joinSide match {
-            case LeftSide if joinType == LeftOuter =>
-              Iterator(generateJoinedRow(thisRow, nullRight))
-            case RightSide if joinType == RightOuter =>
-              Iterator(generateJoinedRow(thisRow, nullLeft))
-            case _ => Iterator()
-          }
+          generateFilteredJoinedRow(thisRow)
         }
       }
     }
@@ -493,13 +563,19 @@ case class StreamingSymmetricHashJoinExec(
     private class AddingProcessedRowToStateCompletionIterator(
         key: UnsafeRow,
         thisRow: UnsafeRow,
-        subIter: Iterator[JoinedRow])
-      extends CompletionIterator[JoinedRow, Iterator[JoinedRow]](subIter) {
+        subIter: Iterator[InternalRow])
+      extends CompletionIterator[InternalRow, Iterator[InternalRow]](subIter) {
+
       private val iteratorNotEmpty: Boolean = super.hasNext
 
       override def completion(): Unit = {
-        val shouldAddToState = // add only if both removal predicates do not match
-          !stateKeyWatermarkPredicateFunc(key) && !stateValueWatermarkPredicateFunc(thisRow)
+        val isLeftSemiWithMatch =
+          joinType == LeftSemi && joinSide == LeftSide && iteratorNotEmpty
+        // Add to state store only if both removal predicates do not match,
+        // and the row is not matched for left side of left semi join.
+        val shouldAddToState =
+          !stateKeyWatermarkPredicateFunc(key) && !stateValueWatermarkPredicateFunc(thisRow) &&
+          !isLeftSemiWithMatch
         if (shouldAddToState) {
           joinStateManager.append(key, thisRow, matched = iteratorNotEmpty)
           updatedStateRowsCount += 1
@@ -544,4 +620,8 @@ case class StreamingSymmetricHashJoinExec(
 
     def numUpdatedStateRows: Long = updatedStateRowsCount
   }
+
+  override protected def withNewChildrenInternal(
+      newLeft: SparkPlan, newRight: SparkPlan): StreamingSymmetricHashJoinExec =
+    copy(left = newLeft, right = newRight)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingSymmetricHashJoinHelper.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingSymmetricHashJoinHelper.scala
index 71792facf698a..2f62dbd7ec578 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingSymmetricHashJoinHelper.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingSymmetricHashJoinHelper.scala
@@ -21,13 +21,13 @@ import scala.reflect.ClassTag
 
 import org.apache.spark.{Partition, SparkContext, TaskContext}
 import org.apache.spark.internal.Logging
-import org.apache.spark.rdd.{RDD, ZippedPartitionsBaseRDD, ZippedPartitionsPartition, ZippedPartitionsRDD2}
+import org.apache.spark.rdd.{RDD, ZippedPartitionsBaseRDD, ZippedPartitionsPartition}
 import org.apache.spark.sql.catalyst.analysis.StreamingJoinHelper
 import org.apache.spark.sql.catalyst.expressions.{And, Attribute, AttributeSet, BoundReference, Expression, NamedExpression, PredicateHelper}
 import org.apache.spark.sql.catalyst.plans.logical.EventTimeWatermark._
 import org.apache.spark.sql.execution.SparkPlan
 import org.apache.spark.sql.execution.streaming.WatermarkSupport.watermarkExpression
-import org.apache.spark.sql.execution.streaming.state.{StateStoreCoordinatorRef, StateStoreProvider, StateStoreProviderId}
+import org.apache.spark.sql.execution.streaming.state.{StateStoreCoordinatorRef, StateStoreProviderId}
 
 
 /**
@@ -200,8 +200,8 @@ object StreamingSymmetricHashJoinHelper extends Logging {
   /**
    * A custom RDD that allows partitions to be "zipped" together, while ensuring the tasks'
    * preferred location is based on which executors have the required join state stores already
-   * loaded. This class is a variant of [[ZippedPartitionsRDD2]] which only changes signature
-   * of `f`.
+   * loaded. This class is a variant of [[org.apache.spark.rdd.ZippedPartitionsRDD2]] which only
+   * changes signature of `f`.
    */
   class StateStoreAwareZipPartitionsRDD[A: ClassTag, B: ClassTag, V: ClassTag](
       sc: SparkContext,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/Triggers.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/Triggers.scala
index f29970d5de192..ebd237bfc0c19 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/Triggers.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/Triggers.scala
@@ -50,17 +50,17 @@ private object Triggers {
  * A [[Trigger]] that processes only one batch of data in a streaming query then terminates
  * the query.
  */
-private[sql] case object OneTimeTrigger extends Trigger
+case object OneTimeTrigger extends Trigger
 
 /**
  * A [[Trigger]] that runs a query periodically based on the processing time. If `interval` is 0,
  * the query will run as fast as possible.
  */
-private[sql] case class ProcessingTimeTrigger(intervalMs: Long) extends Trigger {
+case class ProcessingTimeTrigger(intervalMs: Long) extends Trigger {
   Triggers.validate(intervalMs)
 }
 
-private[sql] object ProcessingTimeTrigger {
+object ProcessingTimeTrigger {
   import Triggers._
 
   def apply(interval: String): ProcessingTimeTrigger = {
@@ -84,11 +84,11 @@ private[sql] object ProcessingTimeTrigger {
  * A [[Trigger]] that continuously processes streaming data, asynchronously checkpointing at
  * the specified interval.
  */
-private[sql] case class ContinuousTrigger(intervalMs: Long) extends Trigger {
+case class ContinuousTrigger(intervalMs: Long) extends Trigger {
   Triggers.validate(intervalMs)
 }
 
-private[sql] object ContinuousTrigger {
+object ContinuousTrigger {
   import Triggers._
 
   def apply(interval: String): ContinuousTrigger = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/console.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/console.scala
index e471e6c601d16..c1f5bd318382d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/console.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/console.scala
@@ -23,10 +23,10 @@ import scala.collection.JavaConverters._
 
 import org.apache.spark.sql._
 import org.apache.spark.sql.connector.catalog.{SupportsWrite, Table, TableCapability}
-import org.apache.spark.sql.connector.write.{LogicalWriteInfo, SupportsTruncate, WriteBuilder}
+import org.apache.spark.sql.connector.write.{LogicalWriteInfo, SupportsTruncate, Write, WriteBuilder}
 import org.apache.spark.sql.connector.write.streaming.StreamingWrite
 import org.apache.spark.sql.execution.streaming.sources.ConsoleWrite
-import org.apache.spark.sql.internal.connector.SimpleTableProvider
+import org.apache.spark.sql.internal.connector.{SimpleTableProvider, SupportsStreamingUpdateAsAppend}
 import org.apache.spark.sql.sources.{BaseRelation, CreatableRelationProvider, DataSourceRegister}
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
@@ -73,15 +73,19 @@ object ConsoleTable extends Table with SupportsWrite {
   }
 
   override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
-    new WriteBuilder with SupportsTruncate {
+    new WriteBuilder with SupportsTruncate with SupportsStreamingUpdateAsAppend {
       private val inputSchema: StructType = info.schema()
 
-      // Do nothing for truncate. Console sink is special that it just prints all the records.
+      // Do nothing for truncate. Console sink is special and it just prints all the records.
       override def truncate(): WriteBuilder = this
 
-      override def buildForStreaming(): StreamingWrite = {
-        assert(inputSchema != null)
-        new ConsoleWrite(inputSchema, info.options)
+      override def build(): Write = {
+        new Write {
+          override def toStreaming: StreamingWrite = {
+            assert(inputSchema != null)
+            new ConsoleWrite(inputSchema, info.options)
+          }
+        }
       }
     }
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousCoalesceExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousCoalesceExec.scala
deleted file mode 100644
index 4c621890c9793..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousCoalesceExec.scala
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.streaming.continuous
-
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, SinglePartition}
-import org.apache.spark.sql.execution.{SparkPlan, UnaryExecNode}
-
-/**
- * Physical plan for coalescing a continuous processing plan.
- *
- * Currently, only coalesces to a single partition are supported. `numPartitions` must be 1.
- */
-case class ContinuousCoalesceExec(numPartitions: Int, child: SparkPlan) extends UnaryExecNode {
-  override def output: Seq[Attribute] = child.output
-
-  override def outputPartitioning: Partitioning = SinglePartition
-
-  override def doExecute(): RDD[InternalRow] = {
-    assert(numPartitions == 1)
-    new ContinuousCoalesceRDD(
-      sparkContext,
-      numPartitions,
-      conf.continuousStreamingExecutorQueueSize,
-      sparkContext.getLocalProperty(ContinuousExecution.EPOCH_INTERVAL_KEY).toLong,
-      child.execute())
-  }
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousCoalesceRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousCoalesceRDD.scala
deleted file mode 100644
index 14046f6a99c24..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousCoalesceRDD.scala
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.streaming.continuous
-
-import java.util.UUID
-
-import org.apache.spark._
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.UnsafeRow
-import org.apache.spark.sql.execution.streaming.continuous.shuffle._
-import org.apache.spark.util.ThreadUtils
-
-case class ContinuousCoalesceRDDPartition(
-    index: Int,
-    endpointName: String,
-    queueSize: Int,
-    numShuffleWriters: Int,
-    epochIntervalMs: Long)
-  extends Partition {
-  // Initialized only on the executor, and only once even as we call compute() multiple times.
-  lazy val (reader: ContinuousShuffleReader, endpoint) = {
-    val env = SparkEnv.get.rpcEnv
-    val receiver = new RPCContinuousShuffleReader(
-      queueSize, numShuffleWriters, epochIntervalMs, env)
-    val endpoint = env.setupEndpoint(endpointName, receiver)
-
-    TaskContext.get().addTaskCompletionListener[Unit] { ctx =>
-      env.stop(endpoint)
-    }
-    (receiver, endpoint)
-  }
-  // This flag will be flipped on the executors to indicate that the threads processing
-  // partitions of the write-side RDD have been started. These will run indefinitely
-  // asynchronously as epochs of the coalesce RDD complete on the read side.
-  private[continuous] var writersInitialized: Boolean = false
-}
-
-/**
- * RDD for continuous coalescing. Asynchronously writes all partitions of `prev` into a local
- * continuous shuffle, and then reads them in the task thread using `reader`.
- */
-class ContinuousCoalesceRDD(
-    context: SparkContext,
-    numPartitions: Int,
-    readerQueueSize: Int,
-    epochIntervalMs: Long,
-    prev: RDD[InternalRow])
-  extends RDD[InternalRow](context, Nil) {
-
-  // When we support more than 1 target partition, we'll need to figure out how to pass in the
-  // required partitioner.
-  private val outputPartitioner = new HashPartitioner(1)
-
-  private val readerEndpointNames = (0 until numPartitions).map { i =>
-    s"ContinuousCoalesceRDD-part$i-${UUID.randomUUID()}"
-  }
-
-  override def getPartitions: Array[Partition] = {
-    (0 until numPartitions).map { partIndex =>
-      ContinuousCoalesceRDDPartition(
-        partIndex,
-        readerEndpointNames(partIndex),
-        readerQueueSize,
-        prev.getNumPartitions,
-        epochIntervalMs)
-    }.toArray
-  }
-
-  private lazy val threadPool = ThreadUtils.newDaemonFixedThreadPool(
-    prev.getNumPartitions,
-    this.name)
-
-  override def compute(split: Partition, context: TaskContext): Iterator[InternalRow] = {
-    val part = split.asInstanceOf[ContinuousCoalesceRDDPartition]
-
-    if (!part.writersInitialized) {
-      val rpcEnv = SparkEnv.get.rpcEnv
-
-      // trigger lazy initialization
-      part.endpoint
-      val endpointRefs = readerEndpointNames.map { endpointName =>
-        rpcEnv.setupEndpointRef(rpcEnv.address, endpointName)
-      }
-
-      val runnables = prev.partitions.map { prevSplit =>
-        new Runnable() {
-          override def run(): Unit = {
-            TaskContext.setTaskContext(context)
-
-            val writer: ContinuousShuffleWriter = new RPCContinuousShuffleWriter(
-              prevSplit.index, outputPartitioner, endpointRefs.toArray)
-
-            EpochTracker.initializeCurrentEpoch(
-              context.getLocalProperty(ContinuousExecution.START_EPOCH_KEY).toLong)
-            while (!context.isInterrupted() && !context.isCompleted()) {
-              writer.write(prev.compute(prevSplit, context).asInstanceOf[Iterator[UnsafeRow]])
-              // Note that current epoch is a inheritable thread local but makes another instance,
-              // so each writer thread can properly increment its own epoch without affecting
-              // the main task thread.
-              EpochTracker.incrementCurrentEpoch()
-            }
-          }
-        }
-      }
-
-      context.addTaskCompletionListener[Unit] { ctx =>
-        threadPool.shutdownNow()
-      }
-
-      part.writersInitialized = true
-
-      runnables.foreach(threadPool.execute)
-    }
-
-    part.reader.read()
-  }
-
-  override def clearDependencies(): Unit = {
-    throw new IllegalStateException("Continuous RDDs cannot be checkpointed")
-  }
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousDataSourceRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousDataSourceRDD.scala
index 5ee27c71aa731..6d27961fa0bcb 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousDataSourceRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousDataSourceRDD.scala
@@ -22,6 +22,7 @@ import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.connector.read.InputPartition
 import org.apache.spark.sql.connector.read.streaming.ContinuousPartitionReaderFactory
+import org.apache.spark.sql.execution.metric.{CustomMetrics, SQLMetric}
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.util.NextIterator
 
@@ -52,7 +53,8 @@ class ContinuousDataSourceRDD(
     epochPollIntervalMs: Long,
     private val inputPartitions: Seq[InputPartition],
     schema: StructType,
-    partitionReaderFactory: ContinuousPartitionReaderFactory)
+    partitionReaderFactory: ContinuousPartitionReaderFactory,
+    customMetrics: Map[String, SQLMetric])
   extends RDD[InternalRow](sc, Nil) {
 
   override protected def getPartitions: Array[Partition] = {
@@ -88,8 +90,15 @@ class ContinuousDataSourceRDD(
       partition.queueReader
     }
 
+    val partitionReader = readerForPartition.getPartitionReader()
     new NextIterator[InternalRow] {
+      private var numRow = 0L
+
       override def getNext(): InternalRow = {
+        if (numRow % CustomMetrics.NUM_ROWS_PER_UPDATE == 0) {
+          CustomMetrics.updateMetrics(partitionReader.currentMetricsValues, customMetrics)
+        }
+        numRow += 1
         readerForPartition.next() match {
           case null =>
             finished = true
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousExecution.scala
index a109c2171f3d2..7faa45e9d2435 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousExecution.scala
@@ -28,28 +28,24 @@ import org.apache.spark.SparkEnv
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.expressions.{CurrentDate, CurrentTimestamp}
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.streaming.{StreamingRelationV2, WriteToStream}
 import org.apache.spark.sql.connector.catalog.{SupportsRead, SupportsWrite, TableCapability}
 import org.apache.spark.sql.connector.read.streaming.{ContinuousStream, Offset => OffsetV2, PartitionOffset, ReadLimit}
 import org.apache.spark.sql.execution.SQLExecution
 import org.apache.spark.sql.execution.datasources.v2.StreamingDataSourceV2Relation
-import org.apache.spark.sql.execution.streaming.{StreamingRelationV2, _}
-import org.apache.spark.sql.streaming.{OutputMode, Trigger}
+import org.apache.spark.sql.execution.streaming._
+import org.apache.spark.sql.streaming.Trigger
 import org.apache.spark.util.Clock
 
 class ContinuousExecution(
     sparkSession: SparkSession,
-    name: String,
-    checkpointRoot: String,
-    analyzedPlan: LogicalPlan,
-    sink: SupportsWrite,
     trigger: Trigger,
     triggerClock: Clock,
-    outputMode: OutputMode,
     extraOptions: Map[String, String],
-    deleteCheckpointOnStop: Boolean)
+    plan: WriteToStream)
   extends StreamExecution(
-    sparkSession, name, checkpointRoot, analyzedPlan, sink,
-    trigger, triggerClock, outputMode, deleteCheckpointOnStop) {
+    sparkSession, plan.name, plan.resolvedCheckpointLocation, plan.inputQuery, plan.sink,
+    trigger, triggerClock, plan.outputMode, plan.deleteCheckpointOnStop) {
 
   @volatile protected var sources: Seq[ContinuousStream] = Seq()
 
@@ -64,7 +60,8 @@ class ContinuousExecution(
     var nextSourceId = 0
     import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Implicits._
     val _logicalPlan = analyzedPlan.transform {
-      case s @ StreamingRelationV2(ds, sourceName, table: SupportsRead, options, output, _) =>
+      case s @ StreamingRelationV2(ds, sourceName, table: SupportsRead, options, output, _, _, _) =>
+        val dsStr = if (ds.nonEmpty) s"[${ds.get}]" else ""
         if (!table.supports(TableCapability.CONTINUOUS_READ)) {
           throw new UnsupportedOperationException(
             s"Data source $sourceName does not support continuous processing.")
@@ -73,7 +70,7 @@ class ContinuousExecution(
         v2ToRelationMap.getOrElseUpdate(s, {
           val metadataPath = s"$resolvedCheckpointRoot/sources/$nextSourceId"
           nextSourceId += 1
-          logInfo(s"Reading table [$table] from DataSourceV2 named '$sourceName' [$ds]")
+          logInfo(s"Reading table [$table] from DataSourceV2 named '$sourceName' $dsStr")
           // TODO: operator pushdown.
           val scan = table.newScanBuilder(options).build()
           val stream = scan.toContinuousStream(metadataPath)
@@ -88,7 +85,8 @@ class ContinuousExecution(
 
     // TODO (SPARK-27484): we should add the writing node before the plan is analyzed.
     WriteToContinuousDataSource(
-      createStreamingWrite(sink, extraOptions, _logicalPlan), _logicalPlan)
+      createStreamingWrite(
+        plan.sink.asInstanceOf[SupportsWrite], extraOptions, _logicalPlan), _logicalPlan)
   }
 
   private val triggerExecutor = trigger match {
@@ -206,9 +204,6 @@ class ContinuousExecution(
     currentEpochCoordinatorId = epochCoordinatorId
     sparkSessionForQuery.sparkContext.setLocalProperty(
       ContinuousExecution.EPOCH_COORDINATOR_ID_KEY, epochCoordinatorId)
-    sparkSessionForQuery.sparkContext.setLocalProperty(
-      ContinuousExecution.EPOCH_INTERVAL_KEY,
-      trigger.asInstanceOf[ContinuousTrigger].intervalMs.toString)
 
     // Use the parent Spark session for the endpoint since it's where this query ID is registered.
     val epochEndpoint = EpochCoordinatorRef.create(
@@ -333,7 +328,7 @@ class ContinuousExecution(
 
     synchronized {
       // Record offsets before updating `committedOffsets`
-      recordTriggerOffsets(from = committedOffsets, to = availableOffsets)
+      recordTriggerOffsets(from = committedOffsets, to = availableOffsets, latest = latestOffsets)
       if (queryExecutionThread.isAlive) {
         commitLog.add(epoch, CommitMetadata())
         val offset =
@@ -436,5 +431,4 @@ class ContinuousExecution(
 object ContinuousExecution {
   val START_EPOCH_KEY = "__continuous_start_epoch"
   val EPOCH_COORDINATOR_ID_KEY = "__epoch_coordinator_id"
-  val EPOCH_INTERVAL_KEY = "__continuous_epoch_interval"
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousQueuedDataReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousQueuedDataReader.scala
index dff2fa69e42fd..02893f274902e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousQueuedDataReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousQueuedDataReader.scala
@@ -26,6 +26,7 @@ import org.apache.spark.{SparkEnv, SparkException, TaskContext}
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.UnsafeProjection
+import org.apache.spark.sql.connector.read.PartitionReader
 import org.apache.spark.sql.connector.read.streaming.{ContinuousPartitionReader, PartitionOffset}
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.util.ThreadUtils
@@ -47,6 +48,8 @@ class ContinuousQueuedDataReader(
   // Important sequencing - we must get our starting point before the provider threads start running
   private var currentOffset: PartitionOffset = reader.getOffset
 
+  def getPartitionReader(): PartitionReader[InternalRow] = reader
+
   /**
    * The record types in the read buffer.
    */
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/WriteToContinuousDataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/WriteToContinuousDataSource.scala
index cecb2843fc3b0..ceb52f520df66 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/WriteToContinuousDataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/WriteToContinuousDataSource.scala
@@ -18,14 +18,16 @@
 package org.apache.spark.sql.execution.streaming.continuous
 
 import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, UnaryNode}
 import org.apache.spark.sql.connector.write.streaming.StreamingWrite
 
 /**
  * The logical plan for writing data in a continuous stream.
  */
 case class WriteToContinuousDataSource(write: StreamingWrite, query: LogicalPlan)
-  extends LogicalPlan {
-  override def children: Seq[LogicalPlan] = Seq(query)
+  extends UnaryNode {
+  override def child: LogicalPlan = query
   override def output: Seq[Attribute] = Nil
+  override protected def withNewChildInternal(
+    newChild: LogicalPlan): WriteToContinuousDataSource = copy(query = newChild)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/WriteToContinuousDataSourceExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/WriteToContinuousDataSourceExec.scala
index f1898ad3f27ca..1e0caf4785d5e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/WriteToContinuousDataSourceExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/WriteToContinuousDataSourceExec.scala
@@ -70,4 +70,7 @@ case class WriteToContinuousDataSourceExec(write: StreamingWrite, query: SparkPl
 
     sparkContext.emptyRDD
   }
+
+  override protected def withNewChildInternal(
+    newChild: SparkPlan): WriteToContinuousDataSourceExec = copy(query = newChild)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/ContinuousShuffleReadRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/ContinuousShuffleReadRDD.scala
deleted file mode 100644
index 9b13f6398d837..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/ContinuousShuffleReadRDD.scala
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.streaming.continuous.shuffle
-
-import java.util.UUID
-
-import org.apache.spark.{Partition, SparkContext, SparkEnv, TaskContext}
-import org.apache.spark.rdd.RDD
-import org.apache.spark.rpc.RpcAddress
-import org.apache.spark.sql.catalyst.expressions.UnsafeRow
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.util.NextIterator
-
-case class ContinuousShuffleReadPartition(
-      index: Int,
-      endpointName: String,
-      queueSize: Int,
-      numShuffleWriters: Int,
-      epochIntervalMs: Long)
-    extends Partition {
-  // Initialized only on the executor, and only once even as we call compute() multiple times.
-  lazy val (reader: ContinuousShuffleReader, endpoint) = {
-    val env = SparkEnv.get.rpcEnv
-    val receiver = new RPCContinuousShuffleReader(
-      queueSize, numShuffleWriters, epochIntervalMs, env)
-    val endpoint = env.setupEndpoint(endpointName, receiver)
-
-    TaskContext.get().addTaskCompletionListener[Unit] { ctx =>
-      env.stop(endpoint)
-    }
-    (receiver, endpoint)
-  }
-}
-
-/**
- * RDD at the map side of each continuous processing shuffle task. Upstream tasks send their
- * shuffle output to the wrapped receivers in partitions of this RDD; each of the RDD's tasks
- * poll from their receiver until an epoch marker is sent.
- *
- * @param sc the RDD context
- * @param numPartitions the number of read partitions for this RDD
- * @param queueSize the size of the row buffers to use
- * @param numShuffleWriters the number of continuous shuffle writers feeding into this RDD
- * @param epochIntervalMs the checkpoint interval of the streaming query
- */
-class ContinuousShuffleReadRDD(
-    sc: SparkContext,
-    numPartitions: Int,
-    queueSize: Int = 1024,
-    numShuffleWriters: Int = 1,
-    epochIntervalMs: Long = 1000,
-    val endpointNames: Seq[String] = Seq(s"RPCContinuousShuffleReader-${UUID.randomUUID()}"))
-  extends RDD[UnsafeRow](sc, Nil) {
-
-  override protected def getPartitions: Array[Partition] = {
-    (0 until numPartitions).map { partIndex =>
-      ContinuousShuffleReadPartition(
-        partIndex, endpointNames(partIndex), queueSize, numShuffleWriters, epochIntervalMs)
-    }.toArray
-  }
-
-  override def compute(split: Partition, context: TaskContext): Iterator[UnsafeRow] = {
-    split.asInstanceOf[ContinuousShuffleReadPartition].reader.read()
-  }
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/ContinuousShuffleReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/ContinuousShuffleReader.scala
deleted file mode 100644
index 42631c90ebc55..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/ContinuousShuffleReader.scala
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.streaming.continuous.shuffle
-
-import org.apache.spark.sql.catalyst.expressions.UnsafeRow
-
-/**
- * Trait for reading from a continuous processing shuffle.
- */
-trait ContinuousShuffleReader {
-  /**
-   * Returns an iterator over the incoming rows in an epoch. Implementations should block waiting
-   * for new rows to arrive, and end the iterator once they've received epoch markers from all
-   * shuffle writers.
-   */
-  def read(): Iterator[UnsafeRow]
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/ContinuousShuffleWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/ContinuousShuffleWriter.scala
deleted file mode 100644
index 47b1f78b24505..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/ContinuousShuffleWriter.scala
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.streaming.continuous.shuffle
-
-import org.apache.spark.sql.catalyst.expressions.UnsafeRow
-
-/**
- * Trait for writing to a continuous processing shuffle.
- */
-trait ContinuousShuffleWriter {
-  def write(epoch: Iterator[UnsafeRow]): Unit
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/RPCContinuousShuffleReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/RPCContinuousShuffleReader.scala
deleted file mode 100644
index 502ae0d4822e8..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/RPCContinuousShuffleReader.scala
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.streaming.continuous.shuffle
-
-import java.util.concurrent._
-import java.util.concurrent.atomic.AtomicBoolean
-
-import org.apache.spark.internal.Logging
-import org.apache.spark.rpc.{RpcCallContext, RpcEnv, ThreadSafeRpcEndpoint}
-import org.apache.spark.sql.catalyst.expressions.UnsafeRow
-import org.apache.spark.util.NextIterator
-
-/**
- * Messages for the RPCContinuousShuffleReader endpoint. Either an incoming row or an epoch marker.
- *
- * Each message comes tagged with writerId, identifying which writer the message is coming
- * from. The receiver will only begin the next epoch once all writers have sent an epoch
- * marker ending the current epoch.
- */
-private[shuffle] sealed trait RPCContinuousShuffleMessage extends Serializable {
-  def writerId: Int
-}
-private[shuffle] case class ReceiverRow(writerId: Int, row: UnsafeRow)
-  extends RPCContinuousShuffleMessage
-private[shuffle] case class ReceiverEpochMarker(writerId: Int) extends RPCContinuousShuffleMessage
-
-/**
- * RPC endpoint for receiving rows into a continuous processing shuffle task. Continuous shuffle
- * writers will send rows here, with continuous shuffle readers polling for new rows as needed.
- *
- * TODO: Support multiple source tasks. We need to output a single epoch marker once all
- * source tasks have sent one.
- */
-private[continuous] class RPCContinuousShuffleReader(
-      queueSize: Int,
-      numShuffleWriters: Int,
-      epochIntervalMs: Long,
-      override val rpcEnv: RpcEnv)
-    extends ThreadSafeRpcEndpoint with ContinuousShuffleReader with Logging {
-  // Note that this queue will be drained from the main task thread and populated in the RPC
-  // response thread.
-  private val queues = Array.fill(numShuffleWriters) {
-    new ArrayBlockingQueue[RPCContinuousShuffleMessage](queueSize)
-  }
-
-  // Exposed for testing to determine if the endpoint gets stopped on task end.
-  private[shuffle] val stopped = new AtomicBoolean(false)
-
-  override def onStop(): Unit = {
-    stopped.set(true)
-  }
-
-  override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
-    case r: RPCContinuousShuffleMessage =>
-      // Note that this will block a thread the shared RPC handler pool!
-      // The TCP based shuffle handler (SPARK-24541) will avoid this problem.
-      queues(r.writerId).put(r)
-      context.reply(())
-  }
-
-  override def read(): Iterator[UnsafeRow] = {
-    new NextIterator[UnsafeRow] {
-      // An array of flags for whether each writer ID has gotten an epoch marker.
-      private val writerEpochMarkersReceived = Array.fill(numShuffleWriters)(false)
-
-      private val executor = Executors.newFixedThreadPool(numShuffleWriters)
-      private val completion = new ExecutorCompletionService[RPCContinuousShuffleMessage](executor)
-
-      private def completionTask(writerId: Int) = new Callable[RPCContinuousShuffleMessage] {
-        override def call(): RPCContinuousShuffleMessage = queues(writerId).take()
-      }
-
-      // Initialize by submitting tasks to read the first row from each writer.
-      (0 until numShuffleWriters).foreach(writerId => completion.submit(completionTask(writerId)))
-
-      /**
-       * In each call to getNext(), we pull the next row available in the completion queue, and then
-       * submit another task to read the next row from the writer which returned it.
-       *
-       * When a writer sends an epoch marker, we note that it's finished and don't submit another
-       * task for it in this epoch. The iterator is over once all writers have sent an epoch marker.
-       */
-      override def getNext(): UnsafeRow = {
-        var nextRow: UnsafeRow = null
-        while (!finished && nextRow == null) {
-          completion.poll(epochIntervalMs, TimeUnit.MILLISECONDS) match {
-            case null =>
-              // Try again if the poll didn't wait long enough to get a real result.
-              // But we should be getting at least an epoch marker every checkpoint interval.
-              val writerIdsUncommitted = writerEpochMarkersReceived.zipWithIndex.collect {
-                case (flag, idx) if !flag => idx
-              }
-              logWarning(
-                s"Completion service failed to make progress after $epochIntervalMs ms. Waiting " +
-                  s"for writers ${writerIdsUncommitted.mkString(",")} to send epoch markers.")
-
-            // The completion service guarantees this future will be available immediately.
-            case future => future.get() match {
-              case ReceiverRow(writerId, r) =>
-                // Start reading the next element in the queue we just took from.
-                completion.submit(completionTask(writerId))
-                nextRow = r
-              case ReceiverEpochMarker(writerId) =>
-                // Don't read any more from this queue. If all the writers have sent epoch markers,
-                // the epoch is over; otherwise we need to loop again to poll from the remaining
-                // writers.
-                writerEpochMarkersReceived(writerId) = true
-                if (writerEpochMarkersReceived.forall(_ == true)) {
-                  finished = true
-                }
-            }
-          }
-        }
-
-        nextRow
-      }
-
-      override def close(): Unit = {
-        executor.shutdownNow()
-      }
-    }
-  }
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/RPCContinuousShuffleWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/RPCContinuousShuffleWriter.scala
deleted file mode 100644
index 1c6f3ddb395e6..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/RPCContinuousShuffleWriter.scala
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.streaming.continuous.shuffle
-
-import scala.concurrent.Future
-import scala.concurrent.duration.Duration
-
-import org.apache.spark.Partitioner
-import org.apache.spark.rpc.RpcEndpointRef
-import org.apache.spark.sql.catalyst.expressions.UnsafeRow
-import org.apache.spark.util.ThreadUtils
-
-/**
- * A [[ContinuousShuffleWriter]] sending data to [[RPCContinuousShuffleReader]] instances.
- *
- * @param writerId The partition ID of this writer.
- * @param outputPartitioner The partitioner on the reader side of the shuffle.
- * @param endpoints The [[RPCContinuousShuffleReader]] endpoints to write to. Indexed by
- *                  partition ID within outputPartitioner.
- */
-class RPCContinuousShuffleWriter(
-    writerId: Int,
-    outputPartitioner: Partitioner,
-    endpoints: Array[RpcEndpointRef]) extends ContinuousShuffleWriter {
-
-  if (outputPartitioner.numPartitions != 1) {
-    throw new IllegalArgumentException("multiple readers not yet supported")
-  }
-
-  if (outputPartitioner.numPartitions != endpoints.length) {
-    throw new IllegalArgumentException(s"partitioner size ${outputPartitioner.numPartitions} did " +
-      s"not match endpoint count ${endpoints.length}")
-  }
-
-  def write(epoch: Iterator[UnsafeRow]): Unit = {
-    while (epoch.hasNext) {
-      val row = epoch.next()
-      endpoints(outputPartitioner.getPartition(row)).askSync[Unit](ReceiverRow(writerId, row))
-    }
-
-    val futures = endpoints.map(_.ask[Unit](ReceiverEpochMarker(writerId))).toSeq
-    implicit val ec = ThreadUtils.sameThread
-    ThreadUtils.awaitResult(Future.sequence(futures), Duration.Inf)
-  }
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala
index e5b9e68d71026..ee1cb127a3bc5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala
@@ -30,6 +30,7 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.encoders.{encoderFor, ExpressionEncoder}
 import org.apache.spark.sql.catalyst.expressions.UnsafeRow
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.streaming.StreamingRelationV2
 import org.apache.spark.sql.catalyst.util.truncatedString
 import org.apache.spark.sql.connector.catalog.{SupportsRead, Table, TableCapability}
 import org.apache.spark.sql.connector.read.{InputPartition, PartitionReader, PartitionReaderFactory, Scan, ScanBuilder}
@@ -77,12 +78,14 @@ abstract class MemoryStreamBase[A : Encoder](sqlContext: SQLContext) extends Spa
 
   protected val logicalPlan: LogicalPlan = {
     StreamingRelationV2(
-      MemoryStreamTableProvider,
+      Some(MemoryStreamTableProvider),
       "memory",
       new MemoryStreamTable(this),
       CaseInsensitiveStringMap.empty(),
       attributes,
-      None)(sqlContext.sparkSession)
+      None,
+      None,
+      None)
   }
 
   override def initialOffset(): OffsetV2 = {
@@ -215,7 +218,7 @@ case class MemoryStream[A : Encoder](
         batches.slice(sliceStart, sliceEnd)
       }
 
-      logDebug(generateDebugString(newBlocks.flatten, startOrdinal, endOrdinal))
+      logDebug(generateDebugString(newBlocks.flatten.toSeq, startOrdinal, endOrdinal))
 
       numPartitions match {
         case Some(numParts) =>
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ConsoleWrite.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ConsoleStreamingWrite.scala
similarity index 100%
rename from sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ConsoleWrite.scala
rename to sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ConsoleStreamingWrite.scala
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachBatchSink.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachBatchSink.scala
index 6d5e7fd5c5cf3..60c66d863a3c5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachBatchSink.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachBatchSink.scala
@@ -17,7 +17,6 @@
 
 package org.apache.spark.sql.execution.streaming.sources
 
-import org.apache.spark.api.python.PythonException
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.execution.streaming.Sink
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterTable.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterTable.scala
index ba54c85d07303..5944b4b98e180 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterTable.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterTable.scala
@@ -27,9 +27,10 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.catalyst.expressions.UnsafeRow
 import org.apache.spark.sql.connector.catalog.{SupportsWrite, Table, TableCapability}
-import org.apache.spark.sql.connector.write.{DataWriter, LogicalWriteInfo, PhysicalWriteInfo, SupportsTruncate, WriteBuilder, WriterCommitMessage}
+import org.apache.spark.sql.connector.write.{DataWriter, LogicalWriteInfo, PhysicalWriteInfo, SupportsTruncate, Write, WriteBuilder, WriterCommitMessage}
 import org.apache.spark.sql.connector.write.streaming.{StreamingDataWriterFactory, StreamingWrite}
 import org.apache.spark.sql.execution.python.PythonForeachWriter
+import org.apache.spark.sql.internal.connector.SupportsStreamingUpdateAsAppend
 import org.apache.spark.sql.types.StructType
 
 /**
@@ -54,32 +55,41 @@ case class ForeachWriterTable[T](
   }
 
   override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
-    new WriteBuilder with SupportsTruncate {
-      private var inputSchema: StructType = info.schema()
+    new WriteBuilder with SupportsTruncate with SupportsStreamingUpdateAsAppend {
 
-      // Do nothing for truncate. Foreach sink is special that it just forwards all the records to
-      // ForeachWriter.
+      // Do nothing for truncate. Foreach sink is special and it just forwards all the
+      // records to ForeachWriter.
       override def truncate(): WriteBuilder = this
 
-      override def buildForStreaming(): StreamingWrite = {
-        new StreamingWrite {
-          override def commit(epochId: Long, messages: Array[WriterCommitMessage]): Unit = {}
-          override def abort(epochId: Long, messages: Array[WriterCommitMessage]): Unit = {}
-
-          override def createStreamingWriterFactory(
-              info: PhysicalWriteInfo): StreamingDataWriterFactory = {
-            val rowConverter: InternalRow => T = converter match {
-              case Left(enc) =>
-                val boundEnc = enc.resolveAndBind(
-                  inputSchema.toAttributes,
-                  SparkSession.getActiveSession.get.sessionState.analyzer)
-                boundEnc.createDeserializer()
-              case Right(func) =>
-                func
-            }
-            ForeachWriterFactory(writer, rowConverter)
-          }
+      override def build(): Write = {
+        new ForeachWrite(info, writer, converter)
+      }
+    }
+  }
+}
+
+class ForeachWrite[T](
+    info: LogicalWriteInfo,
+    writer: ForeachWriter[T],
+    converter: Either[ExpressionEncoder[T], InternalRow => T]) extends Write {
+  private val inputSchema: StructType = info.schema()
+  override def toStreaming: StreamingWrite = {
+    new StreamingWrite {
+      override def commit(epochId: Long, messages: Array[WriterCommitMessage]): Unit = {}
+      override def abort(epochId: Long, messages: Array[WriterCommitMessage]): Unit = {}
+
+      override def createStreamingWriterFactory(
+        info: PhysicalWriteInfo): StreamingDataWriterFactory = {
+        val rowConverter: InternalRow => T = converter match {
+          case Left(enc) =>
+            val boundEnc = enc.resolveAndBind(
+              inputSchema.toAttributes,
+              SparkSession.getActiveSession.get.sessionState.analyzer)
+            boundEnc.createDeserializer()
+          case Right(func) =>
+            func
         }
+        ForeachWriterFactory(writer, rowConverter)
       }
     }
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/PackedRowWriterFactory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/PackedRowWriterFactory.scala
index 507f860e0452a..fa51dd61a939b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/PackedRowWriterFactory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/PackedRowWriterFactory.scala
@@ -21,12 +21,13 @@ import scala.collection.mutable
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.connector.write.{BatchWrite, DataWriter, DataWriterFactory, WriterCommitMessage}
+import org.apache.spark.sql.connector.write.{DataWriter, WriterCommitMessage}
 import org.apache.spark.sql.connector.write.streaming.StreamingDataWriterFactory
 
 /**
- * A simple [[DataWriterFactory]] whose tasks just pack rows into the commit message for delivery
- * to a [[BatchWrite]] on the driver.
+ * A simple [[org.apache.spark.sql.connector.write.DataWriterFactory]] whose tasks just pack rows
+ * into the commit message for delivery to a
+ * [[org.apache.spark.sql.connector.write.BatchWrite]] on the driver.
  *
  * Note that, because it sends all rows to the driver, this factory will generally be unsuitable
  * for production-quality sinks. It's intended for use in tests.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/WriteToMicroBatchDataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/WriteToMicroBatchDataSource.scala
index ef1115e6d9e01..f0422fb4bd949 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/WriteToMicroBatchDataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/WriteToMicroBatchDataSource.scala
@@ -18,9 +18,9 @@
 package org.apache.spark.sql.execution.streaming.sources
 
 import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, UnaryNode}
 import org.apache.spark.sql.connector.write.streaming.StreamingWrite
-import org.apache.spark.sql.execution.datasources.v2.WriteToDataSourceV2
+import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Relation, WriteToDataSourceV2}
 
 /**
  * The logical plan for writing data to a micro-batch stream.
@@ -28,12 +28,18 @@ import org.apache.spark.sql.execution.datasources.v2.WriteToDataSourceV2
  * Note that this logical plan does not have a corresponding physical plan, as it will be converted
  * to [[WriteToDataSourceV2]] with [[MicroBatchWrite]] before execution.
  */
-case class WriteToMicroBatchDataSource(write: StreamingWrite, query: LogicalPlan)
-  extends LogicalPlan {
-  override def children: Seq[LogicalPlan] = Seq(query)
+case class WriteToMicroBatchDataSource(
+    relation: Option[DataSourceV2Relation],
+    write: StreamingWrite,
+    query: LogicalPlan)
+  extends UnaryNode {
+  override def child: LogicalPlan = query
   override def output: Seq[Attribute] = Nil
 
   def createPlan(batchId: Long): WriteToDataSourceV2 = {
-    WriteToDataSourceV2(new MicroBatchWrite(batchId, write), query)
+    WriteToDataSourceV2(relation, new MicroBatchWrite(batchId, write), query)
   }
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): WriteToMicroBatchDataSource =
+    copy(query = newChild)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/memory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/memory.scala
index deab42bea36ad..4bd2b332fdb54 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/memory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/memory.scala
@@ -33,14 +33,14 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, Statistics}
 import org.apache.spark.sql.catalyst.plans.logical.statsEstimation.EstimationUtils
 import org.apache.spark.sql.connector.catalog.{SupportsWrite, Table, TableCapability}
-import org.apache.spark.sql.connector.write.{DataWriter, DataWriterFactory, LogicalWriteInfo, PhysicalWriteInfo, SupportsTruncate, WriteBuilder, WriterCommitMessage}
+import org.apache.spark.sql.connector.write.{DataWriter, DataWriterFactory, LogicalWriteInfo, PhysicalWriteInfo, SupportsTruncate, Write, WriteBuilder, WriterCommitMessage}
 import org.apache.spark.sql.connector.write.streaming.{StreamingDataWriterFactory, StreamingWrite}
-import org.apache.spark.sql.execution.streaming.Sink
+import org.apache.spark.sql.internal.connector.SupportsStreamingUpdateAsAppend
 import org.apache.spark.sql.types.StructType
 
 /**
- * A sink that stores the results in memory. This [[Sink]] is primarily intended for use in unit
- * tests and does not provide durability.
+ * A sink that stores the results in memory. This [[org.apache.spark.sql.execution.streaming.Sink]]
+ * is primarily intended for use in unit tests and does not provide durability.
  */
 class MemorySink extends Table with SupportsWrite with Logging {
 
@@ -53,7 +53,7 @@ class MemorySink extends Table with SupportsWrite with Logging {
   }
 
   override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
-    new WriteBuilder with SupportsTruncate {
+    new WriteBuilder with SupportsTruncate with SupportsStreamingUpdateAsAppend {
       private var needTruncate: Boolean = false
       private val inputSchema: StructType = info.schema()
 
@@ -62,8 +62,8 @@ class MemorySink extends Table with SupportsWrite with Logging {
         this
       }
 
-      override def buildForStreaming(): StreamingWrite = {
-        new MemoryStreamingWrite(MemorySink.this, inputSchema, needTruncate)
+      override def build(): Write = {
+        new MemoryWrite(MemorySink.this, inputSchema, needTruncate)
       }
     }
   }
@@ -76,7 +76,7 @@ class MemorySink extends Table with SupportsWrite with Logging {
 
   /** Returns all rows that are stored in this [[Sink]]. */
   def allData: Seq[Row] = synchronized {
-    batches.flatMap(_.data)
+    batches.flatMap(_.data).toSeq
   }
 
   def latestBatchId: Option[Long] = synchronized {
@@ -88,7 +88,7 @@ class MemorySink extends Table with SupportsWrite with Logging {
   }
 
   def dataSinceBatch(sinceBatchId: Long): Seq[Row] = synchronized {
-    batches.filter(_.batchId > sinceBatchId).flatMap(_.data)
+    batches.filter(_.batchId > sinceBatchId).flatMap(_.data).toSeq
   }
 
   def toDebugString: String = synchronized {
@@ -130,6 +130,12 @@ class MemorySink extends Table with SupportsWrite with Logging {
 case class MemoryWriterCommitMessage(partition: Int, data: Seq[Row])
   extends WriterCommitMessage {}
 
+class MemoryWrite(sink: MemorySink, schema: StructType, needTruncate: Boolean) extends Write {
+  override def toStreaming: StreamingWrite = {
+    new MemoryStreamingWrite(sink, schema, needTruncate)
+  }
+}
+
 class MemoryStreamingWrite(
     val sink: MemorySink, schema: StructType, needTruncate: Boolean)
   extends StreamingWrite {
@@ -179,7 +185,7 @@ class MemoryDataWriter(partition: Int, schema: StructType)
   }
 
   override def commit(): MemoryWriterCommitMessage = {
-    val msg = MemoryWriterCommitMessage(partition, data.clone())
+    val msg = MemoryWriterCommitMessage(partition, data.clone().toSeq)
     data.clear()
     msg
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/FlatMapGroupsWithStateExecHelper.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/FlatMapGroupsWithStateExecHelper.scala
index 0a16a3819b778..cc785ee4247c4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/FlatMapGroupsWithStateExecHelper.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/FlatMapGroupsWithStateExecHelper.scala
@@ -77,7 +77,7 @@ object FlatMapGroupsWithStateExecHelper {
   // =========================== Private implementations of StateManager ===========================
   // ===============================================================================================
 
-  /** Commmon methods for StateManager implementations */
+  /** Common methods for StateManager implementations */
   private abstract class StateManagerImplBase(shouldStoreTimestamp: Boolean)
     extends StateManager {
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/HDFSBackedStateStoreProvider.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/HDFSBackedStateStoreProvider.scala
index 05c651f9951b9..5c55034e88df5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/HDFSBackedStateStoreProvider.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/HDFSBackedStateStoreProvider.scala
@@ -33,7 +33,7 @@ import org.apache.hadoop.fs._
 
 import org.apache.spark.{SparkConf, SparkEnv}
 import org.apache.spark.internal.Logging
-import org.apache.spark.io.LZ4CompressionCodec
+import org.apache.spark.io.CompressionCodec
 import org.apache.spark.sql.catalyst.expressions.UnsafeRow
 import org.apache.spark.sql.execution.streaming.CheckpointFileManager
 import org.apache.spark.sql.execution.streaming.CheckpointFileManager.CancellableFSDataOutputStream
@@ -58,7 +58,6 @@ import org.apache.spark.util.{SizeEstimator, Utils}
  * - store.remove(...)
  * - store.commit()    // commits all the updates to made; the new version will be returned
  * - store.iterator()  // key-value data after last commit as an iterator
- * - store.updates()   // updates made in the last commit as an iterator
  *
  * Fault-tolerance model:
  * - Every set of updates is written to a delta file before committing.
@@ -79,6 +78,27 @@ private[state] class HDFSBackedStateStoreProvider extends StateStoreProvider wit
   //   java.util.ConcurrentModificationException
   type MapType = java.util.concurrent.ConcurrentHashMap[UnsafeRow, UnsafeRow]
 
+  class HDFSBackedReadStateStore(val version: Long, map: MapType)
+    extends ReadStateStore {
+
+    override def id: StateStoreId = HDFSBackedStateStoreProvider.this.stateStoreId
+
+    override def get(key: UnsafeRow): UnsafeRow = map.get(key)
+
+    override def iterator(): Iterator[UnsafeRowPair] = {
+      val unsafeRowPair = new UnsafeRowPair()
+      map.entrySet.asScala.iterator.map { entry =>
+        unsafeRowPair.withRows(entry.getKey, entry.getValue)
+      }
+    }
+
+    override def abort(): Unit = {}
+
+    override def toString(): String = {
+      s"HDFSReadStateStore[id=(op=${id.operatorId},part=${id.partitionId}),dir=$baseDir]"
+    }
+  }
+
   /** Implementation of [[StateStore]] API which is backed by an HDFS-compatible file system */
   class HDFSBackedStateStore(val version: Long, mapToUpdate: MapType)
     extends StateStore {
@@ -142,9 +162,8 @@ private[state] class HDFSBackedStateStoreProvider extends StateStoreProvider wit
 
     /** Abort all the updates made on this store. This store will not be usable any more. */
     override def abort(): Unit = {
-      // This if statement is to ensure that files are deleted only if there are changes to the
-      // StateStore. We have two StateStores for each task, one which is used only for reading, and
-      // the other used for read+write. We don't want the read-only to delete state files.
+      // This if statement is to ensure that files are deleted only once: if either commit or abort
+      // is called before, it will be no-op.
       if (state == UPDATING) {
         state = ABORTED
         cancelDeltaFile(compressedStream, deltaFileStream)
@@ -197,15 +216,26 @@ private[state] class HDFSBackedStateStoreProvider extends StateStoreProvider wit
   }
 
   /** Get the state store for making updates to create a new `version` of the store. */
-  override def getStore(version: Long): StateStore = synchronized {
+  override def getStore(version: Long): StateStore = {
+    val newMap = getLoadedMapForStore(version)
+    logInfo(s"Retrieved version $version of ${HDFSBackedStateStoreProvider.this} for update")
+    new HDFSBackedStateStore(version, newMap)
+  }
+
+  /** Get the state store for reading to specific `version` of the store. */
+  override def getReadStore(version: Long): ReadStateStore = {
+    val newMap = getLoadedMapForStore(version)
+    logInfo(s"Retrieved version $version of ${HDFSBackedStateStoreProvider.this} for readonly")
+    new HDFSBackedReadStateStore(version, newMap)
+  }
+
+  private def getLoadedMapForStore(version: Long): MapType = synchronized {
     require(version >= 0, "Version cannot be less than 0")
     val newMap = new MapType()
     if (version > 0) {
       newMap.putAll(loadMap(version))
     }
-    val store = new HDFSBackedStateStore(version, newMap)
-    logInfo(s"Retrieved version $version of ${HDFSBackedStateStoreProvider.this} for update")
-    store
+    newMap
   }
 
   override def init(
@@ -259,6 +289,9 @@ private[state] class HDFSBackedStateStoreProvider extends StateStoreProvider wit
   @volatile private var storeConf: StateStoreConf = _
   @volatile private var hadoopConf: Configuration = _
   @volatile private var numberOfVersionsToRetainInMemory: Int = _
+  // TODO: The validation should be moved to a higher level so that it works for all state store
+  // implementations
+  @volatile private var isValidated = false
 
   private lazy val loadedMaps = new util.TreeMap[Long, MapType](Ordering[Long].reverse)
   private lazy val baseDir = stateStoreId.storeCheckpointLocation()
@@ -457,6 +490,11 @@ private[state] class HDFSBackedStateStoreProvider extends StateStoreProvider wit
             // Prior to Spark 2.3 mistakenly append 4 bytes to the value row in
             // `RowBasedKeyValueBatch`, which gets persisted into the checkpoint data
             valueRow.pointTo(valueRowBuffer, (valueSize / 8) * 8)
+            if (!isValidated) {
+              StateStoreProvider.validateStateRowFormat(
+                keyRow, keySchema, valueRow, valueSchema, storeConf)
+              isValidated = true
+            }
             map.put(keyRow, valueRow)
           }
         }
@@ -551,6 +589,11 @@ private[state] class HDFSBackedStateStoreProvider extends StateStoreProvider wit
             // Prior to Spark 2.3 mistakenly append 4 bytes to the value row in
             // `RowBasedKeyValueBatch`, which gets persisted into the checkpoint data
             valueRow.pointTo(valueRowBuffer, (valueSize / 8) * 8)
+            if (!isValidated) {
+              StateStoreProvider.validateStateRowFormat(
+                keyRow, keySchema, valueRow, valueSchema, storeConf)
+              isValidated = true
+            }
             map.put(keyRow, valueRow)
           }
         }
@@ -683,12 +726,14 @@ private[state] class HDFSBackedStateStoreProvider extends StateStoreProvider wit
   }
 
   private def compressStream(outputStream: DataOutputStream): DataOutputStream = {
-    val compressed = new LZ4CompressionCodec(sparkConf).compressedOutputStream(outputStream)
+    val compressed = CompressionCodec.createCodec(sparkConf, storeConf.compressionCodec)
+      .compressedOutputStream(outputStream)
     new DataOutputStream(compressed)
   }
 
   private def decompressStream(inputStream: DataInputStream): DataInputStream = {
-    val compressed = new LZ4CompressionCodec(sparkConf).compressedInputStream(inputStream)
+    val compressed = CompressionCodec.createCodec(sparkConf, storeConf.compressionCodec)
+      .compressedInputStream(inputStream)
     new DataInputStream(compressed)
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateSchemaCompatibilityChecker.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateSchemaCompatibilityChecker.scala
new file mode 100644
index 0000000000000..4ac12c089c0d3
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateSchemaCompatibilityChecker.scala
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.streaming.state
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.Path
+
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.execution.streaming.{CheckpointFileManager, MetadataVersionUtil}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types.{DataType, StructType}
+
+case class StateSchemaNotCompatible(message: String) extends Exception(message)
+
+class StateSchemaCompatibilityChecker(
+    providerId: StateStoreProviderId,
+    hadoopConf: Configuration) extends Logging {
+
+  private val storeCpLocation = providerId.storeId.storeCheckpointLocation()
+  private val fm = CheckpointFileManager.create(storeCpLocation, hadoopConf)
+  private val schemaFileLocation = schemaFile(storeCpLocation)
+
+  fm.mkdirs(schemaFileLocation.getParent)
+
+  def check(keySchema: StructType, valueSchema: StructType): Unit = {
+    if (fm.exists(schemaFileLocation)) {
+      logDebug(s"Schema file for provider $providerId exists. Comparing with provided schema.")
+      val (storedKeySchema, storedValueSchema) = readSchemaFile()
+      if (storedKeySchema.equals(keySchema) && storedValueSchema.equals(valueSchema)) {
+        // schema is exactly same
+      } else if (!schemasCompatible(storedKeySchema, keySchema) ||
+        !schemasCompatible(storedValueSchema, valueSchema)) {
+        val errorMsg = "Provided schema doesn't match to the schema for existing state! " +
+          "Please note that Spark allow difference of field name: check count of fields " +
+          "and data type of each field.\n" +
+          s"- Provided key schema: $keySchema\n" +
+          s"- Provided value schema: $valueSchema\n" +
+          s"- Existing key schema: $storedKeySchema\n" +
+          s"- Existing value schema: $storedValueSchema\n" +
+          s"If you want to force running query without schema validation, please set " +
+          s"${SQLConf.STATE_SCHEMA_CHECK_ENABLED.key} to false.\n" +
+          "Please note running query with incompatible schema could cause indeterministic" +
+          " behavior."
+        logError(errorMsg)
+        throw StateSchemaNotCompatible(errorMsg)
+      } else {
+        logInfo("Detected schema change which is compatible. Allowing to put rows.")
+      }
+    } else {
+      // schema doesn't exist, create one now
+      logDebug(s"Schema file for provider $providerId doesn't exist. Creating one.")
+      createSchemaFile(keySchema, valueSchema)
+    }
+  }
+
+  private def schemasCompatible(storedSchema: StructType, schema: StructType): Boolean =
+    DataType.equalsIgnoreNameAndCompatibleNullability(storedSchema, schema)
+
+  private def readSchemaFile(): (StructType, StructType) = {
+    val inStream = fm.open(schemaFileLocation)
+    try {
+      val versionStr = inStream.readUTF()
+      // Currently we only support version 1, which we can simplify the version validation and
+      // the parse logic.
+      val version = MetadataVersionUtil.validateVersion(versionStr,
+        StateSchemaCompatibilityChecker.VERSION)
+      require(version == 1)
+
+      val keySchemaStr = inStream.readUTF()
+      val valueSchemaStr = inStream.readUTF()
+
+      (StructType.fromString(keySchemaStr), StructType.fromString(valueSchemaStr))
+    } catch {
+      case e: Throwable =>
+        logError(s"Fail to read schema file from $schemaFileLocation", e)
+        throw e
+    } finally {
+      inStream.close()
+    }
+  }
+
+  private def createSchemaFile(keySchema: StructType, valueSchema: StructType): Unit = {
+    val outStream = fm.createAtomic(schemaFileLocation, overwriteIfPossible = false)
+    try {
+      outStream.writeUTF(s"v${StateSchemaCompatibilityChecker.VERSION}")
+      outStream.writeUTF(keySchema.json)
+      outStream.writeUTF(valueSchema.json)
+      outStream.close()
+    } catch {
+      case e: Throwable =>
+        logError(s"Fail to write schema file to $schemaFileLocation", e)
+        outStream.cancel()
+        throw e
+    }
+  }
+
+  private def schemaFile(storeCpLocation: Path): Path =
+    new Path(new Path(storeCpLocation, "_metadata"), "schema")
+}
+
+object StateSchemaCompatibilityChecker {
+  val VERSION = 1
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStore.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStore.scala
index 7d80fd0c591f9..4aaa053a8906a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStore.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStore.scala
@@ -22,6 +22,7 @@ import java.util.concurrent.{ScheduledFuture, TimeUnit}
 import javax.annotation.concurrent.GuardedBy
 
 import scala.collection.mutable
+import scala.util.Try
 import scala.util.control.NonFatal
 
 import org.apache.hadoop.conf.Configuration
@@ -30,15 +31,20 @@ import org.apache.hadoop.fs.Path
 import org.apache.spark.{SparkContext, SparkEnv}
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.expressions.UnsafeRow
+import org.apache.spark.sql.catalyst.util.UnsafeRowUtils
 import org.apache.spark.sql.execution.streaming.StatefulOperatorStateInfo
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.util.{ThreadUtils, Utils}
 
 /**
- * Base trait for a versioned key-value store. Each instance of a `StateStore` represents a specific
- * version of state data, and such instances are created through a [[StateStoreProvider]].
+ * Base trait for a versioned key-value store which provides read operations. Each instance of a
+ * `ReadStateStore` represents a specific version of state data, and such instances are created
+ * through a [[StateStoreProvider]].
+ *
+ * `abort` method will be called when the task is completed - please clean up the resources in
+ * the method.
  */
-trait StateStore {
+trait ReadStateStore {
 
   /** Unique identifier of the store */
   def id: StateStoreId
@@ -52,17 +58,6 @@ trait StateStore {
    */
   def get(key: UnsafeRow): UnsafeRow
 
-  /**
-   * Put a new value for a non-null key. Implementations must be aware that the UnsafeRows in
-   * the params can be reused, and must make copies of the data as needed for persistence.
-   */
-  def put(key: UnsafeRow, value: UnsafeRow): Unit
-
-  /**
-   * Remove a single non-null key.
-   */
-  def remove(key: UnsafeRow): Unit
-
   /**
    * Get key value pairs with optional approximate `start` and `end` extents.
    * If the State Store implementation maintains indices for the data based on the optional
@@ -80,6 +75,40 @@ trait StateStore {
     iterator()
   }
 
+  /** Return an iterator containing all the key-value pairs in the StateStore. */
+  def iterator(): Iterator[UnsafeRowPair]
+
+  /**
+   * Clean up the resource.
+   *
+   * The method name is to respect backward compatibility on [[StateStore]].
+   */
+  def abort(): Unit
+}
+
+/**
+ * Base trait for a versioned key-value store which provides both read and write operations. Each
+ * instance of a `StateStore` represents a specific version of state data, and such instances are
+ * created through a [[StateStoreProvider]].
+ *
+ * Unlike [[ReadStateStore]], `abort` method may not be called if the `commit` method succeeds
+ * to commit the change. (`hasCommitted` returns `true`.) Otherwise, `abort` method will be called.
+ * Implementation should deal with resource cleanup in both methods, and also need to guard with
+ * double resource cleanup.
+ */
+trait StateStore extends ReadStateStore {
+
+  /**
+   * Put a new value for a non-null key. Implementations must be aware that the UnsafeRows in
+   * the params can be reused, and must make copies of the data as needed for persistence.
+   */
+  def put(key: UnsafeRow, value: UnsafeRow): Unit
+
+  /**
+   * Remove a single non-null key.
+   */
+  def remove(key: UnsafeRow): Unit
+
   /**
    * Commit all the updates that have been made to the store, and return the new version.
    * Implementations should ensure that no more updates (puts, removes) can be after a commit in
@@ -91,13 +120,13 @@ trait StateStore {
    * Abort all the updates that have been made to the store. Implementations should ensure that
    * no more updates (puts, removes) can be after an abort in order to avoid incorrect usage.
    */
-  def abort(): Unit
+  override def abort(): Unit
 
   /**
    * Return an iterator containing all the key-value pairs in the StateStore. Implementations must
    * ensure that updates (puts, removes) can be made while iterating over this iterator.
    */
-  def iterator(): Iterator[UnsafeRowPair]
+  override def iterator(): Iterator[UnsafeRowPair]
 
   /** Current metrics of the state store */
   def metrics: StateStoreMetrics
@@ -108,6 +137,19 @@ trait StateStore {
   def hasCommitted: Boolean
 }
 
+/** Wraps the instance of StateStore to make the instance read-only. */
+class WrappedReadStateStore(store: StateStore) extends ReadStateStore {
+  override def id: StateStoreId = store.id
+
+  override def version: Long = store.version
+
+  override def get(key: UnsafeRow): UnsafeRow = store.get(key)
+
+  override def iterator(): Iterator[UnsafeRowPair] = store.iterator()
+
+  override def abort(): Unit = store.abort()
+}
+
 /**
  * Metrics reported by a state store
  * @param numKeys         Number of keys in the state store
@@ -123,10 +165,18 @@ case class StateStoreMetrics(
 
 object StateStoreMetrics {
   def combine(allMetrics: Seq[StateStoreMetrics]): StateStoreMetrics = {
+    val distinctCustomMetrics = allMetrics.flatMap(_.customMetrics.keys).distinct
+    val customMetrics = allMetrics.flatMap(_.customMetrics)
+    val combinedCustomMetrics = distinctCustomMetrics.map { customMetric =>
+      val sameMetrics = customMetrics.filter(_._1 == customMetric)
+      val sumOfMetrics = sameMetrics.map(_._2).sum
+      customMetric -> sumOfMetrics
+    }.toMap
+
     StateStoreMetrics(
       allMetrics.map(_.numKeys).sum,
       allMetrics.map(_.memoryUsedBytes).sum,
-      allMetrics.flatMap(_.customMetrics).toMap)
+      combinedCustomMetrics)
   }
 }
 
@@ -143,6 +193,16 @@ case class StateStoreCustomSumMetric(name: String, desc: String) extends StateSt
 case class StateStoreCustomSizeMetric(name: String, desc: String) extends StateStoreCustomMetric
 case class StateStoreCustomTimingMetric(name: String, desc: String) extends StateStoreCustomMetric
 
+/**
+ * An exception thrown when an invalid UnsafeRow is detected in state store.
+ */
+class InvalidUnsafeRowException
+  extends RuntimeException("The streaming query failed by state format invalidation. " +
+    "The following reasons may cause this: 1. An old Spark version wrote the checkpoint that is " +
+    "incompatible with the current one; 2. Broken checkpoint files; 3. The query is changed " +
+    "among restart. For the first case, you can try to restart the application without " +
+    "checkpoint or use the legacy Spark version to process the streaming state.", null)
+
 /**
  * Trait representing a provider that provide [[StateStore]] instances representing
  * versions of state data.
@@ -195,6 +255,15 @@ trait StateStoreProvider {
   /** Return an instance of [[StateStore]] representing state data of the given version */
   def getStore(version: Long): StateStore
 
+  /**
+   * Return an instance of [[ReadStateStore]] representing state data of the given version.
+   * By default it will return the same instance as getStore(version) but wrapped to prevent
+   * modification. Providers can override and return optimized version of [[ReadStateStore]]
+   * based on the fact the instance will be only used for reading.
+   */
+  def getReadStore(version: Long): ReadStateStore =
+    new WrappedReadStateStore(getStore(version))
+
   /** Optional method for providers to allow for background maintenance (e.g. compactions) */
   def doMaintenance(): Unit = { }
 
@@ -220,16 +289,36 @@ object StateStoreProvider {
    * Return a instance of the required provider, initialized with the given configurations.
    */
   def createAndInit(
-      stateStoreId: StateStoreId,
+      providerId: StateStoreProviderId,
       keySchema: StructType,
       valueSchema: StructType,
       indexOrdinal: Option[Int], // for sorting the data
       storeConf: StateStoreConf,
       hadoopConf: Configuration): StateStoreProvider = {
     val provider = create(storeConf.providerClass)
-    provider.init(stateStoreId, keySchema, valueSchema, indexOrdinal, storeConf, hadoopConf)
+    provider.init(providerId.storeId, keySchema, valueSchema, indexOrdinal, storeConf, hadoopConf)
     provider
   }
+
+  /**
+   * Use the expected schema to check whether the UnsafeRow is valid.
+   */
+  def validateStateRowFormat(
+      keyRow: UnsafeRow,
+      keySchema: StructType,
+      valueRow: UnsafeRow,
+      valueSchema: StructType,
+      conf: StateStoreConf): Unit = {
+    if (conf.formatValidationEnabled) {
+      if (!UnsafeRowUtils.validateStructuralIntegrity(keyRow, keySchema)) {
+        throw new InvalidUnsafeRowException
+      }
+      if (conf.formatValidationCheckValue &&
+          !UnsafeRowUtils.validateStructuralIntegrity(valueRow, valueSchema)) {
+        throw new InvalidUnsafeRowException
+      }
+    }
+  }
 }
 
 /**
@@ -304,12 +393,14 @@ class UnsafeRowPair(var key: UnsafeRow = null, var value: UnsafeRow = null) {
  */
 object StateStore extends Logging {
 
-  val MAINTENANCE_INTERVAL_CONFIG = "spark.sql.streaming.stateStore.maintenanceInterval"
-  val MAINTENANCE_INTERVAL_DEFAULT_SECS = 60
+  val PARTITION_ID_TO_CHECK_SCHEMA = 0
 
   @GuardedBy("loadedProviders")
   private val loadedProviders = new mutable.HashMap[StateStoreProviderId, StateStoreProvider]()
 
+  @GuardedBy("loadedProviders")
+  private val schemaValidated = new mutable.HashMap[StateStoreProviderId, Option[Throwable]]()
+
   /**
    * Runs the `task` periodically and automatically cancels it if there is an exception. `onError`
    * will be called when an exception happens.
@@ -348,6 +439,21 @@ object StateStore extends Logging {
   @GuardedBy("loadedProviders")
   private var _coordRef: StateStoreCoordinatorRef = null
 
+  /** Get or create a read-only store associated with the id. */
+  def getReadOnly(
+      storeProviderId: StateStoreProviderId,
+      keySchema: StructType,
+      valueSchema: StructType,
+      indexOrdinal: Option[Int],
+      version: Long,
+      storeConf: StateStoreConf,
+      hadoopConf: Configuration): ReadStateStore = {
+    require(version >= 0)
+    val storeProvider = getStateStoreProvider(storeProviderId, keySchema, valueSchema,
+      indexOrdinal, storeConf, hadoopConf)
+    storeProvider.getReadStore(version)
+  }
+
   /** Get or create a store associated with the id. */
   def get(
       storeProviderId: StateStoreProviderId,
@@ -358,17 +464,49 @@ object StateStore extends Logging {
       storeConf: StateStoreConf,
       hadoopConf: Configuration): StateStore = {
     require(version >= 0)
-    val storeProvider = loadedProviders.synchronized {
-      startMaintenanceIfNeeded()
+    val storeProvider = getStateStoreProvider(storeProviderId, keySchema, valueSchema,
+      indexOrdinal, storeConf, hadoopConf)
+    storeProvider.getStore(version)
+  }
+
+  private def getStateStoreProvider(
+      storeProviderId: StateStoreProviderId,
+      keySchema: StructType,
+      valueSchema: StructType,
+      indexOrdinal: Option[Int],
+      storeConf: StateStoreConf,
+      hadoopConf: Configuration): StateStoreProvider = {
+    loadedProviders.synchronized {
+      startMaintenanceIfNeeded(storeConf)
+
+      if (storeProviderId.storeId.partitionId == PARTITION_ID_TO_CHECK_SCHEMA) {
+        val result = schemaValidated.getOrElseUpdate(storeProviderId, {
+          val checker = new StateSchemaCompatibilityChecker(storeProviderId, hadoopConf)
+          // regardless of configuration, we check compatibility to at least write schema file
+          // if necessary
+          val ret = Try(checker.check(keySchema, valueSchema)).toEither.fold(Some(_), _ => None)
+          if (storeConf.stateSchemaCheckEnabled) {
+            ret
+          } else {
+            None
+          }
+        })
+
+        if (result.isDefined) {
+          throw result.get
+        }
+      }
+
       val provider = loadedProviders.getOrElseUpdate(
         storeProviderId,
         StateStoreProvider.createAndInit(
-          storeProviderId.storeId, keySchema, valueSchema, indexOrdinal, storeConf, hadoopConf)
+          storeProviderId, keySchema, valueSchema, indexOrdinal, storeConf, hadoopConf)
       )
-      reportActiveStoreInstance(storeProviderId)
+      val otherProviderIds = loadedProviders.keys.filter(_ != storeProviderId).toSeq
+      val providerIdsToUnload = reportActiveStoreInstance(storeProviderId, otherProviderIds)
+      providerIdsToUnload.foreach(unload(_))
       provider
     }
-    storeProvider.getStore(version)
   }
 
   /** Unload a state store provider */
@@ -376,6 +514,12 @@ object StateStore extends Logging {
     loadedProviders.remove(storeProviderId).foreach(_.close())
   }
 
+  /** Unload all state store providers: unit test purpose */
+  private[sql] def unloadAll(): Unit = loadedProviders.synchronized {
+    loadedProviders.keySet.foreach { key => unload(key) }
+    loadedProviders.clear()
+  }
+
   /** Whether a state store provider is loaded or not */
   def isLoaded(storeProviderId: StateStoreProviderId): Boolean = loadedProviders.synchronized {
     loadedProviders.contains(storeProviderId)
@@ -398,19 +542,17 @@ object StateStore extends Logging {
   }
 
   /** Start the periodic maintenance task if not already started and if Spark active */
-  private def startMaintenanceIfNeeded(): Unit = loadedProviders.synchronized {
-    val env = SparkEnv.get
-    if (env != null && !isMaintenanceRunning) {
-      val periodMs = env.conf.getTimeAsMs(
-        MAINTENANCE_INTERVAL_CONFIG, s"${MAINTENANCE_INTERVAL_DEFAULT_SECS}s")
-      maintenanceTask = new MaintenanceTask(
-        periodMs,
-        task = { doMaintenance() },
-        onError = { loadedProviders.synchronized { loadedProviders.clear() } }
-      )
-      logInfo("State Store maintenance task started")
+  private def startMaintenanceIfNeeded(storeConf: StateStoreConf): Unit =
+    loadedProviders.synchronized {
+      if (SparkEnv.get != null && !isMaintenanceRunning) {
+        maintenanceTask = new MaintenanceTask(
+          storeConf.maintenanceInterval,
+          task = { doMaintenance() },
+          onError = { loadedProviders.synchronized { loadedProviders.clear() } }
+        )
+        logInfo("State Store maintenance task started")
+      }
     }
-  }
 
   /**
    * Execute background maintenance task in all the loaded store providers if they are still
@@ -437,12 +579,20 @@ object StateStore extends Logging {
     }
   }
 
-  private def reportActiveStoreInstance(storeProviderId: StateStoreProviderId): Unit = {
+  private def reportActiveStoreInstance(
+      storeProviderId: StateStoreProviderId,
+      otherProviderIds: Seq[StateStoreProviderId]): Seq[StateStoreProviderId] = {
     if (SparkEnv.get != null) {
       val host = SparkEnv.get.blockManager.blockManagerId.host
       val executorId = SparkEnv.get.blockManager.blockManagerId.executorId
-      coordinatorRef.foreach(_.reportActiveInstance(storeProviderId, host, executorId))
+      val providerIdsToUnload = coordinatorRef
+        .map(_.reportActiveInstance(storeProviderId, host, executorId, otherProviderIds))
+        .getOrElse(Seq.empty[StateStoreProviderId])
       logInfo(s"Reported that the loaded instance $storeProviderId is active")
+      logDebug(s"The loaded instances are going to unload: ${providerIdsToUnload.mkString(", ")}")
+      providerIdsToUnload
+    } else {
+      Seq.empty[StateStoreProviderId]
     }
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStoreConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStoreConf.scala
index d145082a39b57..58af8272d1c09 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStoreConf.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStoreConf.scala
@@ -20,7 +20,9 @@ package org.apache.spark.sql.execution.streaming.state
 import org.apache.spark.sql.internal.SQLConf
 
 /** A class that contains configuration parameters for [[StateStore]]s. */
-class StateStoreConf(@transient private val sqlConf: SQLConf)
+class StateStoreConf(
+    @transient private val sqlConf: SQLConf,
+    extraOptions: Map[String, String] = Map.empty)
   extends Serializable {
 
   def this() = this(new SQLConf)
@@ -43,14 +45,34 @@ class StateStoreConf(@transient private val sqlConf: SQLConf)
    */
   val providerClass: String = sqlConf.stateStoreProviderClass
 
+  /** Whether validate the underlying format or not. */
+  val formatValidationEnabled: Boolean = sqlConf.stateStoreFormatValidationEnabled
+
+  /** Whether validate the value format when the format invalidation enabled. */
+  val formatValidationCheckValue: Boolean =
+    extraOptions.getOrElse(StateStoreConf.FORMAT_VALIDATION_CHECK_VALUE_CONFIG, "true") == "true"
+
+  /** The compression codec used to compress delta and snapshot files. */
+  val compressionCodec: String = sqlConf.stateStoreCompressionCodec
+
+  /** whether to validate state schema during query run. */
+  val stateSchemaCheckEnabled = sqlConf.isStateSchemaCheckEnabled
+
+  /** The interval of maintenance tasks. */
+  val maintenanceInterval = sqlConf.streamingMaintenanceInterval
+
   /**
    * Additional configurations related to state store. This will capture all configs in
-   * SQLConf that start with `spark.sql.streaming.stateStore.` */
+   * SQLConf that start with `spark.sql.streaming.stateStore.` and extraOptions for a specific
+   * operator.
+   */
   val confs: Map[String, String] =
-    sqlConf.getAllConfs.filter(_._1.startsWith("spark.sql.streaming.stateStore."))
+    sqlConf.getAllConfs.filter(_._1.startsWith("spark.sql.streaming.stateStore.")) ++ extraOptions
 }
 
 object StateStoreConf {
+  val FORMAT_VALIDATION_CHECK_VALUE_CONFIG = "formatValidationCheckValue"
+
   val empty = new StateStoreConf()
 
   def apply(conf: SQLConf): StateStoreConf = new StateStoreConf(conf)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStoreCoordinator.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStoreCoordinator.scala
index 2b14d37ee21ee..84b77efea3caf 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStoreCoordinator.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStoreCoordinator.scala
@@ -31,10 +31,19 @@ import org.apache.spark.util.RpcUtils
 private sealed trait StateStoreCoordinatorMessage extends Serializable
 
 /** Classes representing messages */
+
+/**
+ * This message is used to report active instance of a state store provider
+ * to [[StateStoreCoordinator]]. This message also carries other loaded state
+ * store providers on the same executor. [[StateStoreCoordinator]] will check
+ * if these providers are inactive now. Inactive providers will be returned
+ * back to the sender of the message for unloading.
+ */
 private case class ReportActiveInstance(
     storeId: StateStoreProviderId,
     host: String,
-    executorId: String)
+    executorId: String,
+    providerIdsToCheck: Seq[StateStoreProviderId])
   extends StateStoreCoordinatorMessage
 
 private case class VerifyIfInstanceActive(storeId: StateStoreProviderId, executorId: String)
@@ -87,8 +96,10 @@ class StateStoreCoordinatorRef private(rpcEndpointRef: RpcEndpointRef) {
   private[sql] def reportActiveInstance(
       stateStoreProviderId: StateStoreProviderId,
       host: String,
-      executorId: String): Unit = {
-    rpcEndpointRef.send(ReportActiveInstance(stateStoreProviderId, host, executorId))
+      executorId: String,
+      otherProviderIds: Seq[StateStoreProviderId]): Seq[StateStoreProviderId] = {
+    rpcEndpointRef.askSync[Seq[StateStoreProviderId]](
+      ReportActiveInstance(stateStoreProviderId, host, executorId, otherProviderIds))
   }
 
   /** Verify whether the given executor has the active instance of a state store */
@@ -122,13 +133,20 @@ private class StateStoreCoordinator(override val rpcEnv: RpcEnv)
     extends ThreadSafeRpcEndpoint with Logging {
   private val instances = new mutable.HashMap[StateStoreProviderId, ExecutorCacheTaskLocation]
 
-  override def receive: PartialFunction[Any, Unit] = {
-    case ReportActiveInstance(id, host, executorId) =>
+  override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
+    case ReportActiveInstance(id, host, executorId, providerIdsToCheck) =>
       logDebug(s"Reported state store $id is active at $executorId")
-      instances.put(id, ExecutorCacheTaskLocation(host, executorId))
-  }
+      val taskLocation = ExecutorCacheTaskLocation(host, executorId)
+      instances.put(id, taskLocation)
+
+      // Check if any loaded provider id is already loaded in other executor.
+      val providerIdsToUnload = providerIdsToCheck.filter { providerId =>
+        val providerLoc = instances.get(providerId)
+        // This provider is is already loaded in other executor. Marked it to unload.
+        providerLoc.map(_ != taskLocation).getOrElse(false)
+      }
+      context.reply(providerIdsToUnload)
 
-  override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
     case VerifyIfInstanceActive(id, execId) =>
       val response = instances.get(id) match {
         case Some(location) => location.executorId == execId
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStoreRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStoreRDD.scala
index 4a69a48fed75f..f21e2ffb80a7b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStoreRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStoreRDD.scala
@@ -23,70 +23,105 @@ import scala.reflect.ClassTag
 
 import org.apache.spark.{Partition, TaskContext}
 import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.execution.streaming.StreamExecution
-import org.apache.spark.sql.execution.streaming.continuous.EpochTracker
 import org.apache.spark.sql.internal.SessionState
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.util.SerializableConfiguration
 
-/**
- * An RDD that allows computations to be executed against [[StateStore]]s. It
- * uses the [[StateStoreCoordinator]] to get the locations of loaded state stores
- * and use that as the preferred locations.
- */
-class StateStoreRDD[T: ClassTag, U: ClassTag](
+abstract class BaseStateStoreRDD[T: ClassTag, U: ClassTag](
     dataRDD: RDD[T],
-    storeUpdateFunction: (StateStore, Iterator[T]) => Iterator[U],
     checkpointLocation: String,
     queryRunId: UUID,
     operatorId: Long,
-    storeVersion: Long,
-    keySchema: StructType,
-    valueSchema: StructType,
-    indexOrdinal: Option[Int],
     sessionState: SessionState,
-    @transient private val storeCoordinator: Option[StateStoreCoordinatorRef])
-  extends RDD[U](dataRDD) {
+    @transient private val storeCoordinator: Option[StateStoreCoordinatorRef],
+    extraOptions: Map[String, String] = Map.empty) extends RDD[U](dataRDD) {
 
-  private val storeConf = new StateStoreConf(sessionState.conf)
+  protected val storeConf = new StateStoreConf(sessionState.conf, extraOptions)
 
   // A Hadoop Configuration can be about 10 KB, which is pretty big, so broadcast it
-  private val hadoopConfBroadcast = dataRDD.context.broadcast(
+  protected val hadoopConfBroadcast = dataRDD.context.broadcast(
     new SerializableConfiguration(sessionState.newHadoopConf()))
 
-  override protected def getPartitions: Array[Partition] = dataRDD.partitions
-
   /**
    * Set the preferred location of each partition using the executor that has the related
    * [[StateStoreProvider]] already loaded.
+   *
+   * Implementations can simply call this method in getPreferredLocations.
    */
   override def getPreferredLocations(partition: Partition): Seq[String] = {
-    val stateStoreProviderId = StateStoreProviderId(
-      StateStoreId(checkpointLocation, operatorId, partition.index),
-      queryRunId)
+    val stateStoreProviderId = getStateProviderId(partition)
     storeCoordinator.flatMap(_.getLocation(stateStoreProviderId)).toSeq
   }
 
-  override def compute(partition: Partition, ctxt: TaskContext): Iterator[U] = {
-    var store: StateStore = null
-    val storeProviderId = StateStoreProviderId(
+  protected def getStateProviderId(partition: Partition): StateStoreProviderId = {
+    StateStoreProviderId(
       StateStoreId(checkpointLocation, operatorId, partition.index),
       queryRunId)
+  }
+}
+
+/**
+ * An RDD that allows computations to be executed against [[ReadStateStore]]s. It
+ * uses the [[StateStoreCoordinator]] to get the locations of loaded state stores
+ * and use that as the preferred locations.
+ */
+class ReadStateStoreRDD[T: ClassTag, U: ClassTag](
+    dataRDD: RDD[T],
+    storeReadFunction: (ReadStateStore, Iterator[T]) => Iterator[U],
+    checkpointLocation: String,
+    queryRunId: UUID,
+    operatorId: Long,
+    storeVersion: Long,
+    keySchema: StructType,
+    valueSchema: StructType,
+    indexOrdinal: Option[Int],
+    sessionState: SessionState,
+    @transient private val storeCoordinator: Option[StateStoreCoordinatorRef],
+    extraOptions: Map[String, String] = Map.empty)
+  extends BaseStateStoreRDD[T, U](dataRDD, checkpointLocation, queryRunId, operatorId,
+    sessionState, storeCoordinator, extraOptions) {
+
+  override protected def getPartitions: Array[Partition] = dataRDD.partitions
+
+  override def compute(partition: Partition, ctxt: TaskContext): Iterator[U] = {
+    val storeProviderId = getStateProviderId(partition)
+
+    val store = StateStore.getReadOnly(
+      storeProviderId, keySchema, valueSchema, indexOrdinal, storeVersion,
+      storeConf, hadoopConfBroadcast.value.value)
+    val inputIter = dataRDD.iterator(partition, ctxt)
+    storeReadFunction(store, inputIter)
+  }
+}
+
+/**
+ * An RDD that allows computations to be executed against [[StateStore]]s. It
+ * uses the [[StateStoreCoordinator]] to get the locations of loaded state stores
+ * and use that as the preferred locations.
+ */
+class StateStoreRDD[T: ClassTag, U: ClassTag](
+    dataRDD: RDD[T],
+    storeUpdateFunction: (StateStore, Iterator[T]) => Iterator[U],
+    checkpointLocation: String,
+    queryRunId: UUID,
+    operatorId: Long,
+    storeVersion: Long,
+    keySchema: StructType,
+    valueSchema: StructType,
+    indexOrdinal: Option[Int],
+    sessionState: SessionState,
+    @transient private val storeCoordinator: Option[StateStoreCoordinatorRef],
+    extraOptions: Map[String, String] = Map.empty)
+  extends BaseStateStoreRDD[T, U](dataRDD, checkpointLocation, queryRunId, operatorId,
+    sessionState, storeCoordinator, extraOptions) {
+
+  override protected def getPartitions: Array[Partition] = dataRDD.partitions
+
+  override def compute(partition: Partition, ctxt: TaskContext): Iterator[U] = {
+    val storeProviderId = getStateProviderId(partition)
 
-    // If we're in continuous processing mode, we should get the store version for the current
-    // epoch rather than the one at planning time.
-    val isContinuous = Option(ctxt.getLocalProperty(StreamExecution.IS_CONTINUOUS_PROCESSING))
-      .map(_.toBoolean).getOrElse(false)
-    val currentVersion = if (isContinuous) {
-      val epoch = EpochTracker.getCurrentEpoch
-      assert(epoch.isDefined, "Current epoch must be defined for continuous processing streams.")
-      epoch.get
-    } else {
-      storeVersion
-    }
-
-    store = StateStore.get(
-      storeProviderId, keySchema, valueSchema, indexOrdinal, currentVersion,
+    val store = StateStore.get(
+      storeProviderId, keySchema, valueSchema, indexOrdinal, storeVersion,
       storeConf, hadoopConfBroadcast.value.value)
     val inputIter = dataRDD.iterator(partition, ctxt)
     storeUpdateFunction(store, inputIter)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StreamingAggregationStateManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StreamingAggregationStateManager.scala
index 9bfb9561b42a1..0496e4768b681 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StreamingAggregationStateManager.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StreamingAggregationStateManager.scala
@@ -34,7 +34,7 @@ sealed trait StreamingAggregationStateManager extends Serializable {
   def getStateValueSchema: StructType
 
   /** Get the current value of a non-null key from the target state store. */
-  def get(store: StateStore, key: UnsafeRow): UnsafeRow
+  def get(store: ReadStateStore, key: UnsafeRow): UnsafeRow
 
   /**
    * Put a new value for a non-null key to the target state store. Note that key will be
@@ -52,13 +52,13 @@ sealed trait StreamingAggregationStateManager extends Serializable {
   def remove(store: StateStore, key: UnsafeRow): Unit
 
   /** Return an iterator containing all the key-value pairs in target state store. */
-  def iterator(store: StateStore): Iterator[UnsafeRowPair]
+  def iterator(store: ReadStateStore): Iterator[UnsafeRowPair]
 
   /** Return an iterator containing all the keys in target state store. */
-  def keys(store: StateStore): Iterator[UnsafeRow]
+  def keys(store: ReadStateStore): Iterator[UnsafeRow]
 
   /** Return an iterator containing all the values in target state store. */
-  def values(store: StateStore): Iterator[UnsafeRow]
+  def values(store: ReadStateStore): Iterator[UnsafeRow]
 }
 
 object StreamingAggregationStateManager extends Logging {
@@ -90,7 +90,7 @@ abstract class StreamingAggregationStateManagerBaseImpl(
 
   override def remove(store: StateStore, key: UnsafeRow): Unit = store.remove(key)
 
-  override def keys(store: StateStore): Iterator[UnsafeRow] = {
+  override def keys(store: ReadStateStore): Iterator[UnsafeRow] = {
     // discard and don't convert values to avoid computation
     store.getRange(None, None).map(_.key)
   }
@@ -113,7 +113,7 @@ class StreamingAggregationStateManagerImplV1(
 
   override def getStateValueSchema: StructType = inputRowAttributes.toStructType
 
-  override def get(store: StateStore, key: UnsafeRow): UnsafeRow = {
+  override def get(store: ReadStateStore, key: UnsafeRow): UnsafeRow = {
     store.get(key)
   }
 
@@ -121,11 +121,11 @@ class StreamingAggregationStateManagerImplV1(
     store.put(getKey(row), row)
   }
 
-  override def iterator(store: StateStore): Iterator[UnsafeRowPair] = {
+  override def iterator(store: ReadStateStore): Iterator[UnsafeRowPair] = {
     store.iterator()
   }
 
-  override def values(store: StateStore): Iterator[UnsafeRow] = {
+  override def values(store: ReadStateStore): Iterator[UnsafeRow] = {
     store.iterator().map(_.value)
   }
 }
@@ -167,7 +167,7 @@ class StreamingAggregationStateManagerImplV2(
 
   override def getStateValueSchema: StructType = valueExpressions.toStructType
 
-  override def get(store: StateStore, key: UnsafeRow): UnsafeRow = {
+  override def get(store: ReadStateStore, key: UnsafeRow): UnsafeRow = {
     val savedState = store.get(key)
     if (savedState == null) {
       return savedState
@@ -182,11 +182,11 @@ class StreamingAggregationStateManagerImplV2(
     store.put(key, value)
   }
 
-  override def iterator(store: StateStore): Iterator[UnsafeRowPair] = {
+  override def iterator(store: ReadStateStore): Iterator[UnsafeRowPair] = {
     store.iterator().map(rowPair => new UnsafeRowPair(rowPair.key, restoreOriginalRow(rowPair)))
   }
 
-  override def values(store: StateStore): Iterator[UnsafeRow] = {
+  override def values(store: ReadStateStore): Iterator[UnsafeRow] = {
     store.iterator().map(rowPair => restoreOriginalRow(rowPair))
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/SymmetricHashJoinStateManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/SymmetricHashJoinStateManager.scala
index 1a0a43c083879..dae771c613131 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/SymmetricHashJoinStateManager.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/SymmetricHashJoinStateManager.scala
@@ -25,14 +25,14 @@ import org.apache.spark.TaskContext
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression, JoinedRow, Literal, SpecificInternalRow, UnsafeProjection, UnsafeRow}
-import org.apache.spark.sql.execution.streaming.{StatefulOperatorStateInfo, StreamingSymmetricHashJoinExec}
+import org.apache.spark.sql.execution.streaming.StatefulOperatorStateInfo
 import org.apache.spark.sql.execution.streaming.StreamingSymmetricHashJoinHelper._
-import org.apache.spark.sql.execution.streaming.state.SymmetricHashJoinStateManager.KeyToValuePair
 import org.apache.spark.sql.types.{BooleanType, LongType, StructField, StructType}
 import org.apache.spark.util.NextIterator
 
 /**
- * Helper class to manage state required by a single side of [[StreamingSymmetricHashJoinExec]].
+ * Helper class to manage state required by a single side of
+ * [[org.apache.spark.sql.execution.streaming.StreamingSymmetricHashJoinExec]].
  * The interface of this class is basically that of a multi-map:
  * - Get: Returns an iterator of multiple values for given key
  * - Append: Append a new value to the given key
@@ -99,13 +99,20 @@ class SymmetricHashJoinStateManager(
   /**
    * Get all the matched values for given join condition, with marking matched.
    * This method is designed to mark joined rows properly without exposing internal index of row.
+   *
+   * @param excludeRowsAlreadyMatched Do not join with rows already matched previously.
+   *                                  This is used for right side of left semi join in
+   *                                  [[StreamingSymmetricHashJoinExec]] only.
    */
   def getJoinedRows(
       key: UnsafeRow,
       generateJoinedRow: InternalRow => JoinedRow,
-      predicate: JoinedRow => Boolean): Iterator[JoinedRow] = {
+      predicate: JoinedRow => Boolean,
+      excludeRowsAlreadyMatched: Boolean = false): Iterator[JoinedRow] = {
     val numValues = keyToNumValues.get(key)
-    keyWithIndexToValue.getAll(key, numValues).map { keyIdxToValue =>
+    keyWithIndexToValue.getAll(key, numValues).filterNot { keyIdxToValue =>
+      excludeRowsAlreadyMatched && keyIdxToValue.matched
+    }.map { keyIdxToValue =>
       val joinedRow = generateJoinedRow(keyIdxToValue.value)
       if (predicate(joinedRow)) {
         if (!keyIdxToValue.matched) {
@@ -171,7 +178,7 @@ class SymmetricHashJoinStateManager(
         return null
       }
 
-      override def close: Unit = {}
+      override def close(): Unit = {}
     }
   }
 
@@ -280,7 +287,7 @@ class SymmetricHashJoinStateManager(
         return reusedRet.withNew(currentKey, currentValue.value, currentValue.matched)
       }
 
-      override def close: Unit = {}
+      override def close(): Unit = {}
     }
   }
 
@@ -451,10 +458,25 @@ class SymmetricHashJoinStateManager(
   }
 
   private trait KeyWithIndexToValueRowConverter {
+    /** Defines the schema of the value row (the value side of K-V in state store). */
     def valueAttributes: Seq[Attribute]
 
+    /**
+     * Convert the value row to (actual value, match) pair.
+     *
+     * NOTE: implementations should ensure the result row is NOT reused during execution, so
+     * that caller can safely read the value in any time.
+     */
     def convertValue(value: UnsafeRow): ValueAndMatchPair
 
+    /**
+     * Build the value row from (actual value, match) pair. This is expected to be called just
+     * before storing to the state store.
+     *
+     * NOTE: depending on the implementation, the result row "may" be reused during execution
+     * (to avoid initialization of object), so the caller should ensure that the logic doesn't
+     * affect by such behavior. Call copy() against the result row if needed.
+     */
     def convertToValueRow(value: UnsafeRow, matched: Boolean): UnsafeRow
   }
 
@@ -493,7 +515,7 @@ class SymmetricHashJoinStateManager(
 
     override def convertValue(value: UnsafeRow): ValueAndMatchPair = {
       if (value != null) {
-        ValueAndMatchPair(valueRowGenerator(value),
+        ValueAndMatchPair(valueRowGenerator(value).copy(),
           value.getBoolean(indexOrdinalInValueWithMatchedRow))
       } else {
         null
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/package.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/package.scala
index b6021438e902b..fa89c506587b1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/package.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/package.scala
@@ -55,7 +55,8 @@ package object state {
         valueSchema: StructType,
         indexOrdinal: Option[Int],
         sessionState: SessionState,
-        storeCoordinator: Option[StateStoreCoordinatorRef])(
+        storeCoordinator: Option[StateStoreCoordinatorRef],
+        extraOptions: Map[String, String] = Map.empty)(
         storeUpdateFunction: (StateStore, Iterator[T]) => Iterator[U]): StateStoreRDD[T, U] = {
 
       val cleanedF = dataRDD.sparkContext.clean(storeUpdateFunction)
@@ -78,7 +79,43 @@ package object state {
         valueSchema,
         indexOrdinal,
         sessionState,
-        storeCoordinator)
+        storeCoordinator,
+        extraOptions)
+    }
+
+    /** Map each partition of an RDD along with data in a [[ReadStateStore]]. */
+    private[streaming] def mapPartitionsWithReadStateStore[U: ClassTag](
+        stateInfo: StatefulOperatorStateInfo,
+        keySchema: StructType,
+        valueSchema: StructType,
+        indexOrdinal: Option[Int],
+        sessionState: SessionState,
+        storeCoordinator: Option[StateStoreCoordinatorRef],
+        extraOptions: Map[String, String] = Map.empty)(
+        storeReadFn: (ReadStateStore, Iterator[T]) => Iterator[U])
+      : ReadStateStoreRDD[T, U] = {
+
+      val cleanedF = dataRDD.sparkContext.clean(storeReadFn)
+      val wrappedF = (store: ReadStateStore, iter: Iterator[T]) => {
+        // Clean up the state store.
+        TaskContext.get().addTaskCompletionListener[Unit](_ => {
+          store.abort()
+        })
+        cleanedF(store, iter)
+      }
+      new ReadStateStoreRDD(
+        dataRDD,
+        wrappedF,
+        stateInfo.checkpointLocation,
+        stateInfo.queryRunId,
+        stateInfo.operatorId,
+        stateInfo.storeVersion,
+        keySchema,
+        valueSchema,
+        indexOrdinal,
+        sessionState,
+        storeCoordinator,
+        extraOptions)
     }
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/statefulOperators.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/statefulOperators.scala
index 1bec924ba219a..b52603ebc0443 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/statefulOperators.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/statefulOperators.scala
@@ -24,7 +24,6 @@ import scala.collection.JavaConverters._
 
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.errors._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
 import org.apache.spark.sql.catalyst.plans.logical.EventTimeWatermark
@@ -59,7 +58,7 @@ case class StatefulOperatorStateInfo(
 trait StatefulOperator extends SparkPlan {
   def stateInfo: Option[StatefulOperatorStateInfo]
 
-  protected def getStateInfo: StatefulOperatorStateInfo = attachTree(this) {
+  protected def getStateInfo: StatefulOperatorStateInfo = {
     stateInfo.getOrElse {
       throw new IllegalStateException("State location not present for execution")
     }
@@ -77,6 +76,8 @@ trait StateStoreWriter extends StatefulOperator { self: SparkPlan =>
 
   override lazy val metrics = Map(
     "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
+    "numRowsDroppedByWatermark" -> SQLMetrics.createMetric(sparkContext,
+      "number of rows which are dropped by watermark"),
     "numTotalStateRows" -> SQLMetrics.createMetric(sparkContext, "number of total state rows"),
     "numUpdatedStateRows" -> SQLMetrics.createMetric(sparkContext, "number of updated state rows"),
     "allUpdatesTimeMs" -> SQLMetrics.createTimingMetric(sparkContext, "time to update"),
@@ -94,12 +95,13 @@ trait StateStoreWriter extends StatefulOperator { self: SparkPlan =>
       .map(entry => entry._1 -> longMetric(entry._1).value)
 
     val javaConvertedCustomMetrics: java.util.HashMap[String, java.lang.Long] =
-      new java.util.HashMap(customMetrics.mapValues(long2Long).asJava)
+      new java.util.HashMap(customMetrics.mapValues(long2Long).toMap.asJava)
 
     new StateOperatorProgress(
       numRowsTotal = longMetric("numTotalStateRows").value,
       numRowsUpdated = longMetric("numUpdatedStateRows").value,
       memoryUsedBytes = longMetric("stateMemory").value,
+      numRowsDroppedByWatermark = longMetric("numRowsDroppedByWatermark").value,
       javaConvertedCustomMetrics
     )
   }
@@ -132,6 +134,16 @@ trait StateStoreWriter extends StatefulOperator { self: SparkPlan =>
     }.toMap
   }
 
+  protected def applyRemovingRowsOlderThanWatermark(
+      iter: Iterator[InternalRow],
+      predicateDropRowByWatermark: BasePredicate): Iterator[InternalRow] = {
+    iter.filterNot { row =>
+      val shouldDrop = predicateDropRowByWatermark.eval(row)
+      if (shouldDrop) longMetric("numRowsDroppedByWatermark") += 1
+      shouldDrop
+    }
+  }
+
   /**
    * Should the MicroBatchExecution run another batch based on this stateful operator and the
    * current updated metadata.
@@ -233,7 +245,7 @@ case class StateStoreRestoreExec(
   override protected def doExecute(): RDD[InternalRow] = {
     val numOutputRows = longMetric("numOutputRows")
 
-    child.execute().mapPartitionsWithStateStore(
+    child.execute().mapPartitionsWithReadStateStore(
       getStateInfo,
       keyExpressions.toStructType,
       stateManager.getStateValueSchema,
@@ -269,6 +281,9 @@ case class StateStoreRestoreExec(
       ClusteredDistribution(keyExpressions, stateInfo.map(_.numPartitions)) :: Nil
     }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): StateStoreRestoreExec =
+    copy(child = newChild)
 }
 
 /**
@@ -328,7 +343,8 @@ case class StateStoreSaveExec(
           // Assumption: watermark predicates must be non-empty if append mode is allowed
           case Some(Append) =>
             allUpdatesTimeMs += timeTakenMs {
-              val filteredIter = iter.filter(row => !watermarkPredicateForData.get.eval(row))
+              val filteredIter = applyRemovingRowsOlderThanWatermark(iter,
+                watermarkPredicateForData.get)
               while (filteredIter.hasNext) {
                 val row = filteredIter.next().asInstanceOf[UnsafeRow]
                 stateManager.put(store, row)
@@ -371,7 +387,7 @@ case class StateStoreSaveExec(
             new NextIterator[InternalRow] {
               // Filter late date using watermark if specified
               private[this] val baseIterator = watermarkPredicateForData match {
-                case Some(predicate) => iter.filter((row: InternalRow) => !predicate.eval(row))
+                case Some(predicate) => applyRemovingRowsOlderThanWatermark(iter, predicate)
                 case None => iter
               }
               private val updatesStartTimeNs = System.nanoTime
@@ -423,6 +439,9 @@ case class StateStoreSaveExec(
       eventTimeWatermark.isDefined &&
       newMetadata.batchWatermarkMs > eventTimeWatermark.get
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): StateStoreSaveExec =
+    copy(child = newChild)
 }
 
 /** Physical operator for executing streaming Deduplicate. */
@@ -446,7 +465,10 @@ case class StreamingDeduplicateExec(
       child.output.toStructType,
       indexOrdinal = None,
       sqlContext.sessionState,
-      Some(sqlContext.streams.stateStoreCoordinator)) { (store, iter) =>
+      Some(sqlContext.streams.stateStoreCoordinator),
+      // We won't check value row in state store since the value StreamingDeduplicateExec.EMPTY_ROW
+      // is unrelated to the output schema.
+      Map(StateStoreConf.FORMAT_VALIDATION_CHECK_VALUE_CONFIG -> "false")) { (store, iter) =>
       val getKey = GenerateUnsafeProjection.generate(keyExpressions, child.output)
       val numOutputRows = longMetric("numOutputRows")
       val numTotalStateRows = longMetric("numTotalStateRows")
@@ -456,7 +478,7 @@ case class StreamingDeduplicateExec(
       val commitTimeMs = longMetric("commitTimeMs")
 
       val baseIterator = watermarkPredicateForData match {
-        case Some(predicate) => iter.filter(row => !predicate.eval(row))
+        case Some(predicate) => applyRemovingRowsOlderThanWatermark(iter, predicate)
         case None => iter
       }
 
@@ -493,6 +515,9 @@ case class StreamingDeduplicateExec(
   override def shouldRunAnotherBatch(newMetadata: OffsetSeqMetadata): Boolean = {
     eventTimeWatermark.isDefined && newMetadata.batchWatermarkMs > eventTimeWatermark.get
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): StreamingDeduplicateExec =
+    copy(child = newChild)
 }
 
 object StreamingDeduplicateExec {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/streamingLimits.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/streamingLimits.scala
index b19540253d7eb..51723a25e04e1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/streamingLimits.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/streamingLimits.scala
@@ -22,7 +22,6 @@ import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{Attribute, GenericInternalRow, SortOrder, UnsafeProjection, UnsafeRow}
 import org.apache.spark.sql.catalyst.plans.physical.{AllTuples, Distribution, Partitioning}
-import org.apache.spark.sql.catalyst.streaming.InternalOutputModes
 import org.apache.spark.sql.execution.{LimitExec, SparkPlan, UnaryExecNode}
 import org.apache.spark.sql.execution.streaming.state.StateStoreOps
 import org.apache.spark.sql.streaming.OutputMode
@@ -96,6 +95,9 @@ case class StreamingGlobalLimitExec(
   private def getValueRow(value: Long): UnsafeRow = {
     UnsafeProjection.create(valueSchema)(new GenericInternalRow(Array[Any](value)))
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): StreamingGlobalLimitExec =
+    copy(child = newChild)
 }
 
 
@@ -134,4 +136,7 @@ case class StreamingLocalLimitExec(limit: Int, child: SparkPlan)
   override def outputPartitioning: Partitioning = child.outputPartitioning
 
   override def output: Seq[Attribute] = child.output
+
+  override protected def withNewChildInternal(newChild: SparkPlan): StreamingLocalLimitExec =
+    copy(child = newChild)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/subquery.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/subquery.scala
index c2270c57eb941..f96e9ee3ce3aa 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/subquery.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/subquery.scala
@@ -23,9 +23,11 @@ import scala.collection.mutable.ArrayBuffer
 import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.{expressions, InternalRow}
-import org.apache.spark.sql.catalyst.expressions.{AttributeSeq, CreateNamedStruct, Expression, ExprId, InSet, ListQuery, Literal, PlanExpression}
+import org.apache.spark.sql.catalyst.expressions.{CreateNamedStruct, Expression, ExprId, InSet, ListQuery, Literal, PlanExpression}
 import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
 import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.{LeafLike, UnaryLike}
+import org.apache.spark.sql.catalyst.trees.TreePattern.{IN_SUBQUERY, SCALAR_SUBQUERY}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{BooleanType, DataType, StructType}
 
@@ -62,10 +64,9 @@ object ExecSubqueryExpression {
 case class ScalarSubquery(
     plan: BaseSubqueryExec,
     exprId: ExprId)
-  extends ExecSubqueryExpression {
+  extends ExecSubqueryExpression with LeafLike[Expression] {
 
   override def dataType: DataType = plan.schema.fields.head.dataType
-  override def children: Seq[Expression] = Nil
   override def nullable: Boolean = true
   override def toString: String = plan.simpleString(SQLConf.get.maxToStringFields)
   override def withNewPlan(query: BaseSubqueryExec): ScalarSubquery = copy(plan = query)
@@ -114,12 +115,13 @@ case class InSubqueryExec(
     child: Expression,
     plan: BaseSubqueryExec,
     exprId: ExprId,
-    private var resultBroadcast: Broadcast[Array[Any]] = null) extends ExecSubqueryExpression {
+    private var resultBroadcast: Broadcast[Array[Any]] = null)
+  extends ExecSubqueryExpression with UnaryLike[Expression] {
 
   @transient private var result: Array[Any] = _
+  @transient private lazy val inSet = InSet(child, result.toSet)
 
   override def dataType: DataType = BooleanType
-  override def children: Seq[Expression] = child :: Nil
   override def nullable: Boolean = child.nullable
   override def toString: String = s"$child IN ${plan.name}"
   override def withNewPlan(plan: BaseSubqueryExec): InSubqueryExec = copy(plan = plan)
@@ -131,9 +133,10 @@ case class InSubqueryExec(
 
   def updateResult(): Unit = {
     val rows = plan.executeCollect()
-    result = child.dataType match {
-      case _: StructType => rows.toArray
-      case _ => rows.map(_.get(0, child.dataType))
+    result = if (plan.output.length > 1) {
+      rows.asInstanceOf[Array[Any]]
+    } else {
+      rows.map(_.get(0, child.dataType))
     }
     resultBroadcast = plan.sqlContext.sparkContext.broadcast(result)
   }
@@ -149,17 +152,12 @@ case class InSubqueryExec(
 
   override def eval(input: InternalRow): Any = {
     prepareResult()
-    val v = child.eval(input)
-    if (v == null) {
-      null
-    } else {
-      result.contains(v)
-    }
+    inSet.eval(input)
   }
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     prepareResult()
-    InSet(child, result.toSet).doGenCode(ctx, ev)
+    inSet.doGenCode(ctx, ev)
   }
 
   override lazy val canonicalized: InSubqueryExec = {
@@ -169,6 +167,9 @@ case class InSubqueryExec(
       exprId = ExprId(0),
       resultBroadcast = null)
   }
+
+  override protected def withNewChildInternal(newChild: Expression): InSubqueryExec =
+    copy(child = newChild)
 }
 
 /**
@@ -176,11 +177,12 @@ case class InSubqueryExec(
  */
 case class PlanSubqueries(sparkSession: SparkSession) extends Rule[SparkPlan] {
   def apply(plan: SparkPlan): SparkPlan = {
-    plan.transformAllExpressions {
+    plan.transformAllExpressionsWithPruning(_.containsAnyPattern(SCALAR_SUBQUERY, IN_SUBQUERY)) {
       case subquery: expressions.ScalarSubquery =>
         val executedPlan = QueryExecution.prepareExecutedPlan(sparkSession, subquery.plan)
         ScalarSubquery(
-          SubqueryExec(s"scalar-subquery#${subquery.exprId.id}", executedPlan),
+          SubqueryExec.createForScalarSubquery(
+            s"scalar-subquery#${subquery.exprId.id}", executedPlan),
           subquery.exprId)
       case expressions.InSubquery(values, ListQuery(query, _, exprId, _)) =>
         val expr = if (values.length == 1) {
@@ -202,7 +204,7 @@ case class PlanSubqueries(sparkSession: SparkSession) extends Rule[SparkPlan] {
  * Find out duplicated subqueries in the spark plan, then use the same subquery result for all the
  * references.
  */
-case class ReuseSubquery(conf: SQLConf) extends Rule[SparkPlan] {
+object ReuseSubquery extends Rule[SparkPlan] {
 
   def apply(plan: SparkPlan): SparkPlan = {
     if (!conf.subqueryReuseEnabled) {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala
index 5ce052a0ae997..71bfc98b9eeb7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala
@@ -21,12 +21,12 @@ import java.net.URLEncoder
 import java.nio.charset.StandardCharsets.UTF_8
 import javax.servlet.http.HttpServletRequest
 
-import scala.collection.JavaConverters._
 import scala.collection.mutable
-import scala.xml.{Node, NodeSeq, Unparsed}
+import scala.xml.{Node, NodeSeq}
 
 import org.apache.spark.JobExecutionStatus
 import org.apache.spark.internal.Logging
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.ui.{PagedDataSource, PagedTable, UIUtils, WebUIPage}
 import org.apache.spark.util.Utils
 
@@ -58,7 +58,7 @@ private[ui] class AllExecutionsPage(parent: SQLTab) extends WebUIPage("") with L
 
       if (running.nonEmpty) {
         val runningPageTable =
-          executionsTable(request, "running", running, currentTime, true, true, true)
+          executionsTable(request, "running", running.toSeq, currentTime, true, true, true)
 
         _content ++=
           <span id="running" class="collapse-aggregated-runningExecutions collapse-table"
@@ -76,7 +76,7 @@ private[ui] class AllExecutionsPage(parent: SQLTab) extends WebUIPage("") with L
 
       if (completed.nonEmpty) {
         val completedPageTable =
-          executionsTable(request, "completed", completed, currentTime, false, true, false)
+          executionsTable(request, "completed", completed.toSeq, currentTime, false, true, false)
 
         _content ++=
           <span id="completed" class="collapse-aggregated-completedExecutions collapse-table"
@@ -94,7 +94,7 @@ private[ui] class AllExecutionsPage(parent: SQLTab) extends WebUIPage("") with L
 
       if (failed.nonEmpty) {
         val failedPageTable =
-          executionsTable(request, "failed", failed, currentTime, false, true, true)
+          executionsTable(request, "failed", failed.toSeq, currentTime, false, true, true)
 
         _content ++=
           <span id="failed" class="collapse-aggregated-failedExecutions collapse-table"
@@ -159,26 +159,8 @@ private[ui] class AllExecutionsPage(parent: SQLTab) extends WebUIPage("") with L
     showSucceededJobs: Boolean,
     showFailedJobs: Boolean): Seq[Node] = {
 
-    val parameterOtherTable = request.getParameterMap().asScala
-      .filterNot(_._1.startsWith(executionTag))
-      .map { case (name, vals) =>
-        name + "=" + vals(0)
-      }
-
-    val parameterExecutionPage = request.getParameter(s"$executionTag.page")
-    val parameterExecutionSortColumn = request.getParameter(s"$executionTag.sort")
-    val parameterExecutionSortDesc = request.getParameter(s"$executionTag.desc")
-    val parameterExecutionPageSize = request.getParameter(s"$executionTag.pageSize")
-
-    val executionPage = Option(parameterExecutionPage).map(_.toInt).getOrElse(1)
-    val executionSortColumn = Option(parameterExecutionSortColumn).map { sortColumn =>
-      UIUtils.decodeURLParameter(sortColumn)
-    }.getOrElse("ID")
-    val executionSortDesc = Option(parameterExecutionSortDesc).map(_.toBoolean).getOrElse(
-      // New executions should be shown above old executions by default.
-      executionSortColumn == "ID"
-    )
-    val executionPageSize = Option(parameterExecutionPageSize).map(_.toInt).getOrElse(100)
+    val executionPage =
+      Option(request.getParameter(s"$executionTag.page")).map(_.toInt).getOrElse(1)
 
     val tableHeaderId = executionTag // "running", "completed" or "failed"
 
@@ -191,11 +173,7 @@ private[ui] class AllExecutionsPage(parent: SQLTab) extends WebUIPage("") with L
         executionTag,
         UIUtils.prependBaseUri(request, parent.basePath),
         "SQL", // subPath
-        parameterOtherTable,
         currentTime,
-        pageSize = executionPageSize,
-        sortColumn = executionSortColumn,
-        desc = executionSortDesc,
         showRunningJobs,
         showSucceededJobs,
         showFailedJobs).table(executionPage)
@@ -219,20 +197,17 @@ private[ui] class ExecutionPagedTable(
     executionTag: String,
     basePath: String,
     subPath: String,
-    parameterOtherTable: Iterable[String],
     currentTime: Long,
-    pageSize: Int,
-    sortColumn: String,
-    desc: Boolean,
     showRunningJobs: Boolean,
     showSucceededJobs: Boolean,
     showFailedJobs: Boolean) extends PagedTable[ExecutionTableRowData] {
 
+  private val (sortColumn, desc, pageSize) = getTableParameters(request, executionTag, "ID")
+
+  private val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
+
   override val dataSource = new ExecutionDataSource(
-    request,
-    parent,
     data,
-    basePath,
     currentTime,
     pageSize,
     sortColumn,
@@ -241,16 +216,15 @@ private[ui] class ExecutionPagedTable(
     showSucceededJobs,
     showFailedJobs)
 
-  private val parameterPath = s"$basePath/$subPath/?${parameterOtherTable.mkString("&")}"
+  private val parameterPath =
+    s"$basePath/$subPath/?${getParameterOtherTable(request, executionTag)}"
 
   override def tableId: String = s"$executionTag-table"
 
   override def tableCssClass: String =
-    "table table-bordered table-sm table-striped " +
-      "table-head-clickable table-cell-width-limited"
+    "table table-bordered table-sm table-striped table-head-clickable table-cell-width-limited"
 
   override def pageLink(page: Int): String = {
-    val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
     parameterPath +
       s"&$pageNumberFormField=$page" +
       s"&$executionTag.sort=$encodedSortColumn" +
@@ -263,89 +237,36 @@ private[ui] class ExecutionPagedTable(
 
   override def pageNumberFormField: String = s"$executionTag.page"
 
-  override def goButtonFormPath: String = {
-    val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
+  override def goButtonFormPath: String =
     s"$parameterPath&$executionTag.sort=$encodedSortColumn&$executionTag.desc=$desc#$tableHeaderId"
-  }
 
   override def headers: Seq[Node] = {
-    // Information for each header: title, sortable
-    val executionHeadersAndCssClasses: Seq[(String, Boolean)] =
+    // Information for each header: title, sortable, tooltip
+    val executionHeadersAndCssClasses: Seq[(String, Boolean, Option[String])] =
       Seq(
-        ("ID", true),
-        ("Description", true),
-        ("Submitted", true),
-        ("Duration", true)) ++ {
+        ("ID", true, None),
+        ("Description", true, None),
+        ("Submitted", true, None),
+        ("Duration", true, Some("Time from query submission to completion (or if still executing," +
+          "time since submission)"))) ++ {
         if (showRunningJobs && showSucceededJobs && showFailedJobs) {
           Seq(
-            ("Running Job IDs", true),
-            ("Succeeded Job IDs", true),
-            ("Failed Job IDs", true))
+            ("Running Job IDs", true, None),
+            ("Succeeded Job IDs", true, None),
+            ("Failed Job IDs", true, None))
         } else if (showSucceededJobs && showFailedJobs) {
           Seq(
-            ("Succeeded Job IDs", true),
-            ("Failed Job IDs", true))
+            ("Succeeded Job IDs", true, None),
+            ("Failed Job IDs", true, None))
         } else {
-          Seq(("Job IDs", true))
+          Seq(("Job IDs", true, None))
         }
       }
 
-    val sortableColumnHeaders = executionHeadersAndCssClasses.filter {
-      case (_, sortable) => sortable
-    }.map { case (title, _) => title }
-
-    require(sortableColumnHeaders.contains(sortColumn), s"Unknown column: $sortColumn")
-
-    val headerRow: Seq[Node] = {
-      executionHeadersAndCssClasses.map { case (header, sortable) =>
-        if (header == sortColumn) {
-          val headerLink = Unparsed(
-            parameterPath +
-              s"&$executionTag.sort=${URLEncoder.encode(header, UTF_8.name())}" +
-              s"&$executionTag.desc=${!desc}" +
-              s"&$executionTag.pageSize=$pageSize" +
-              s"#$tableHeaderId")
-          val arrow = if (desc) "&#x25BE;" else "&#x25B4;" // UP or DOWN
-
-          <th>
-            <a href={headerLink}>
-              {header}<span>
-              &nbsp;{Unparsed(arrow)}
-            </span>
-            </a>
-          </th>
-        } else {
-          if (sortable) {
-            val headerLink = Unparsed(
-              parameterPath +
-                s"&$executionTag.sort=${URLEncoder.encode(header, UTF_8.name())}" +
-                s"&$executionTag.pageSize=$pageSize" +
-                s"#$tableHeaderId")
-
-            <th>
-              <a href={headerLink}>
-                {if (header == "Duration") {
-                  <span data-toggle="tooltip" data-placement="top"
-                    title="Time from query submission to completion
-                    (or if still executing, time since submission)">
-                    {header}
-                  </span>
-                } else {
-                  {header}
-                }}
-              </a>
-            </th>
-          } else {
-            <th>
-              {header}
-            </th>
-          }
-        }
-      }
-    }
-    <thead>
-      {headerRow}
-    </thead>
+    isSortColumnValid(executionHeadersAndCssClasses, sortColumn)
+
+    headerRow(executionHeadersAndCssClasses, desc, pageSize, sortColumn, parameterPath,
+      executionTag, tableHeaderId)
   }
 
   override def row(executionTableRow: ExecutionTableRowData): Seq[Node] = {
@@ -423,7 +344,6 @@ private[ui] class ExecutionPagedTable(
 
 
 private[ui] class ExecutionTableRowData(
-    val submissionTime: Long,
     val duration: Long,
     val executionUIData: SQLExecutionUIData,
     val runningJobData: Seq[Int],
@@ -432,10 +352,7 @@ private[ui] class ExecutionTableRowData(
 
 
 private[ui] class ExecutionDataSource(
-    request: HttpServletRequest,
-    parent: SQLTab,
     executionData: Seq[SQLExecutionUIData],
-    basePath: String,
     currentTime: Long,
     pageSize: Int,
     sortColumn: String,
@@ -448,20 +365,13 @@ private[ui] class ExecutionDataSource(
   // in the table so that we can avoid creating duplicate contents during sorting the data
   private val data = executionData.map(executionRow).sorted(ordering(sortColumn, desc))
 
-  private var _sliceExecutionIds: Set[Int] = _
-
   override def dataSize: Int = data.size
 
-  override def sliceData(from: Int, to: Int): Seq[ExecutionTableRowData] = {
-    val r = data.slice(from, to)
-    _sliceExecutionIds = r.map(_.executionUIData.executionId.toInt).toSet
-    r
-  }
+  override def sliceData(from: Int, to: Int): Seq[ExecutionTableRowData] = data.slice(from, to)
 
   private def executionRow(executionUIData: SQLExecutionUIData): ExecutionTableRowData = {
-    val submissionTime = executionUIData.submissionTime
     val duration = executionUIData.completionTime.map(_.getTime())
-      .getOrElse(currentTime) - submissionTime
+      .getOrElse(currentTime) - executionUIData.submissionTime
 
     val runningJobData = if (showRunningJobs) {
       executionUIData.jobs.filter {
@@ -482,7 +392,6 @@ private[ui] class ExecutionDataSource(
     } else Seq.empty
 
     new ExecutionTableRowData(
-      submissionTime,
       duration,
       executionUIData,
       runningJobData,
@@ -500,7 +409,7 @@ private[ui] class ExecutionDataSource(
       case "Job IDs" | "Succeeded Job IDs" => Ordering by (_.completedJobData.headOption)
       case "Running Job IDs" => Ordering.by(_.runningJobData.headOption)
       case "Failed Job IDs" => Ordering.by(_.failedJobData.headOption)
-      case unknownColumn => throw new IllegalArgumentException(s"Unknown column: $unknownColumn")
+      case unknownColumn => throw QueryExecutionErrors.unknownColumnError(unknownColumn)
     }
     if (desc) {
       ordering.reverse
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala
index 76bc7faf18d01..b15c70a7eba75 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala
@@ -45,7 +45,7 @@ class ExecutionPage(parent: SQLTab) extends WebUIPage("execution") with Logging
           if (jobStatus == status) Some(jobId) else None
         }
         if (jobs.nonEmpty) {
-          <li>
+          <li class="job-url">
             <strong>{label} </strong>
             {jobs.toSeq.sorted.map { jobId =>
               <a href={jobURL(request, jobId.intValue())}>{jobId.toString}</a><span>&nbsp;</span>
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListener.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListener.scala
index a938ae49d7970..e7ab4a184b07b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListener.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListener.scala
@@ -22,15 +22,19 @@ import java.util.concurrent.atomic.AtomicInteger
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable
+import scala.util.control.NonFatal
 
 import org.apache.spark.{JobExecutionStatus, SparkConf}
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.Status._
 import org.apache.spark.scheduler._
+import org.apache.spark.sql.connector.metric.CustomMetric
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.execution.SQLExecution
 import org.apache.spark.sql.execution.metric._
 import org.apache.spark.sql.internal.StaticSQLConf._
 import org.apache.spark.status.{ElementTrackingStore, KVUtils, LiveEntity}
+import org.apache.spark.util.Utils
 import org.apache.spark.util.collection.OpenHashMap
 
 class SQLAppStatusListener(
@@ -183,7 +187,7 @@ class SQLAppStatusListener(
     } else {
       info.accumulables
     }
-    updateStageMetrics(event.stageId, event.stageAttemptId, info.taskId, info.index, accums,
+    updateStageMetrics(event.stageId, event.stageAttemptId, info.taskId, info.index, accums.toSeq,
       info.successful)
   }
 
@@ -198,7 +202,37 @@ class SQLAppStatusListener(
   }
 
   private def aggregateMetrics(exec: LiveExecutionData): Map[Long, String] = {
-    val metricTypes = exec.metrics.map { m => (m.accumulatorId, m.metricType) }.toMap
+    val accumIds = exec.metrics.map(_.accumulatorId).toSet
+
+    val metricAggregationMap = new mutable.HashMap[String, (Array[Long], Array[Long]) => String]()
+    val metricAggregationMethods = exec.metrics.map { m =>
+      val optClassName = CustomMetrics.parseV2CustomMetricType(m.metricType)
+      val metricAggMethod = optClassName.map { className =>
+        if (metricAggregationMap.contains(className)) {
+          metricAggregationMap(className)
+        } else {
+          // Try to initiate custom metric object
+          try {
+            val metric = Utils.loadExtensions(classOf[CustomMetric], Seq(className), conf).head
+            val method =
+              (metrics: Array[Long], _: Array[Long]) => metric.aggregateTaskMetrics(metrics)
+            metricAggregationMap.put(className, method)
+            method
+          } catch {
+            case NonFatal(_) =>
+              // Cannot initialize custom metric object, we might be in history server that does
+              // not have the custom metric class.
+              val defaultMethod = (_: Array[Long], _: Array[Long]) => "N/A"
+              metricAggregationMap.put(className, defaultMethod)
+              defaultMethod
+          }
+        }
+      }.getOrElse(
+        // Built-in SQLMetric
+        SQLMetrics.stringValue(m.metricType, _, _)
+      )
+      (m.accumulatorId, metricAggMethod)
+    }.toMap
 
     val liveStageMetrics = exec.stages.toSeq
       .flatMap { stageId => Option(stageMetrics.get(stageId)) }
@@ -211,7 +245,7 @@ class SQLAppStatusListener(
 
     val maxMetricsFromAllStages = new mutable.HashMap[Long, Array[Long]]()
 
-    taskMetrics.foreach { case (id, values) =>
+    taskMetrics.filter(m => accumIds.contains(m._1)).foreach { case (id, values) =>
       val prev = allMetrics.getOrElse(id, null)
       val updated = if (prev != null) {
         prev ++ values
@@ -222,7 +256,8 @@ class SQLAppStatusListener(
     }
 
     // Find the max for each metric id between all stages.
-    maxMetrics.foreach { case (id, value, taskId, stageId, attemptId) =>
+    val validMaxMetrics = maxMetrics.filter(m => accumIds.contains(m._1))
+    validMaxMetrics.foreach { case (id, value, taskId, stageId, attemptId) =>
       val updated = maxMetricsFromAllStages.getOrElse(id, Array(value, stageId, attemptId, taskId))
       if (value > updated(0)) {
         updated(0) = value
@@ -234,7 +269,7 @@ class SQLAppStatusListener(
     }
 
     exec.driverAccumUpdates.foreach { case (id, value) =>
-      if (metricTypes.contains(id)) {
+      if (accumIds.contains(id)) {
         val prev = allMetrics.getOrElse(id, null)
         val updated = if (prev != null) {
           // If the driver updates same metrics as tasks and has higher value then remove
@@ -254,7 +289,7 @@ class SQLAppStatusListener(
     }
 
     val aggregatedMetrics = allMetrics.map { case (id, values) =>
-      id -> SQLMetrics.stringValue(metricTypes(id), values, maxMetricsFromAllStages.getOrElse(id,
+      id -> metricAggregationMethods(id)(values, maxMetricsFromAllStages.getOrElse(id,
         Array.empty[Long]))
     }.toMap
 
@@ -290,7 +325,7 @@ class SQLAppStatusListener(
           cluster.id,
           cluster.name,
           cluster.desc,
-          toStoredNodes(cluster.nodes),
+          toStoredNodes(cluster.nodes.toSeq),
           cluster.metrics)
         new SparkPlanGraphNodeWrapper(null, storedCluster)
 
@@ -340,7 +375,7 @@ class SQLAppStatusListener(
 
     val exec = getOrCreateExecution(executionId)
     exec.physicalPlanDescription = physicalPlanDescription
-    exec.metrics = sqlPlanMetrics
+    exec.metrics ++= sqlPlanMetrics
     update(exec)
   }
 
@@ -348,7 +383,7 @@ class SQLAppStatusListener(
     val SparkListenerSQLAdaptiveSQLMetricUpdates(executionId, sqlPlanMetrics) = event
 
     val exec = getOrCreateExecution(executionId)
-    exec.metrics = exec.metrics ++ sqlPlanMetrics
+    exec.metrics ++= sqlPlanMetrics
     update(exec)
   }
 
@@ -533,7 +568,7 @@ private class LiveStageMetrics(
         val value = acc.update.get match {
           case s: String => s.toLong
           case l: Long => l
-          case o => throw new IllegalArgumentException(s"Unexpected: $o")
+          case o => throw QueryExecutionErrors.unexpectedAccumulableUpdateValueError(o)
         }
 
         val metricValues = taskMetrics.computeIfAbsent(acc.id, _ => new Array(numTasks))
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
index a798fe02700e3..3b011301421f5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
@@ -66,7 +66,7 @@ object SparkPlanGraph {
     val edges = mutable.ArrayBuffer[SparkPlanGraphEdge]()
     val exchanges = mutable.HashMap[SparkPlanInfo, SparkPlanGraphNode]()
     buildSparkPlanGraphNode(planInfo, nodeIdGenerator, nodes, edges, null, null, exchanges)
-    new SparkPlanGraph(nodes, edges)
+    new SparkPlanGraph(nodes.toSeq, edges.toSeq)
   }
 
   private def buildSparkPlanGraphNode(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/StreamingQueryHistoryServerPlugin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/StreamingQueryHistoryServerPlugin.scala
new file mode 100644
index 0000000000000..a127fa59b7433
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/StreamingQueryHistoryServerPlugin.scala
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.ui
+
+import org.apache.spark.SparkConf
+import org.apache.spark.scheduler.SparkListener
+import org.apache.spark.sql.execution.streaming.StreamingQueryListenerBus
+import org.apache.spark.sql.streaming.ui.{StreamingQueryStatusListener, StreamingQueryTab}
+import org.apache.spark.status.{AppHistoryServerPlugin, ElementTrackingStore}
+import org.apache.spark.ui.SparkUI
+
+class StreamingQueryHistoryServerPlugin extends AppHistoryServerPlugin {
+
+  override def createListeners(conf: SparkConf, store: ElementTrackingStore): Seq[SparkListener] = {
+    val listenerBus = new StreamingQueryListenerBus(None)
+    listenerBus.addListener(new StreamingQueryStatusListener(conf, store))
+    Seq(listenerBus)
+  }
+
+  override def setupUI(ui: SparkUI): Unit = {
+    val streamingQueryStatusStore = new StreamingQueryStatusStore(ui.store.store)
+    if (streamingQueryStatusStore.allQueryUIData.nonEmpty) {
+      new StreamingQueryTab(streamingQueryStatusStore, ui)
+    }
+  }
+
+  override def displayOrder: Int = 1
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/StreamingQueryStatusStore.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/StreamingQueryStatusStore.scala
new file mode 100644
index 0000000000000..9eb14a6a63063
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/StreamingQueryStatusStore.scala
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.ui
+
+import java.util.UUID
+
+import org.apache.spark.sql.streaming.ui.{StreamingQueryData, StreamingQueryProgressWrapper, StreamingQueryUIData}
+import org.apache.spark.status.KVUtils
+import org.apache.spark.util.kvstore.KVStore
+
+/**
+ * Provides a view of a KVStore with methods that make it easy to query Streaming Query state.
+ * There's no state kept in this class, so it's ok to have multiple instances of it in an
+ * application.
+ */
+class StreamingQueryStatusStore(store: KVStore) {
+
+  def allQueryUIData: Seq[StreamingQueryUIData] = {
+    val view = store.view(classOf[StreamingQueryData]).index("startTimestamp").first(0L)
+    KVUtils.viewToSeq(view, Int.MaxValue)(_ => true).map(makeUIData)
+  }
+
+  // visible for test
+  private[sql] def getQueryProgressData(runId: UUID): Seq[StreamingQueryProgressWrapper] = {
+    val view = store.view(classOf[StreamingQueryProgressWrapper])
+      .index("runId").first(runId.toString).last(runId.toString)
+    KVUtils.viewToSeq(view, Int.MaxValue)(_ => true)
+  }
+
+  private def makeUIData(summary: StreamingQueryData): StreamingQueryUIData = {
+    val runId = summary.runId.toString
+    val view = store.view(classOf[StreamingQueryProgressWrapper])
+      .index("runId").first(runId).last(runId)
+    val recentProgress = KVUtils.viewToSeq(view, Int.MaxValue)(_ => true)
+      .map(_.progress).sortBy(_.timestamp).toArray
+    StreamingQueryUIData(summary, recentProgress)
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/AggregateProcessor.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/AggregateProcessor.scala
index bc141b36e63b4..1ebbd5f40646e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/AggregateProcessor.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/AggregateProcessor.scala
@@ -94,9 +94,10 @@ private[window] object AggregateProcessor {
     }
 
     // Create the projections.
-    val initialProj = newMutableProjection(initialValues, partitionSize.toSeq)
-    val updateProj = newMutableProjection(updateExpressions, aggBufferAttributes ++ inputAttributes)
-    val evalProj = newMutableProjection(evaluateExpressions, aggBufferAttributes)
+    val initialProj = newMutableProjection(initialValues.toSeq, partitionSize.toSeq)
+    val updateProj =
+      newMutableProjection(updateExpressions.toSeq, (aggBufferAttributes ++ inputAttributes).toSeq)
+    val evalProj = newMutableProjection(evaluateExpressions.toSeq, aggBufferAttributes.toSeq)
 
     // Create the processor
     new AggregateProcessor(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowExec.scala
index eaca55df08d06..8011c803394d3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowExec.scala
@@ -17,17 +17,11 @@
 
 package org.apache.spark.sql.execution.window
 
-import scala.collection.mutable
-import scala.collection.mutable.ArrayBuffer
-
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.expressions.aggregate._
 import org.apache.spark.sql.catalyst.plans.physical._
-import org.apache.spark.sql.catalyst.util.DateTimeUtils
-import org.apache.spark.sql.execution.{ExternalAppendOnlyUnsafeRowArray, SparkPlan, UnaryExecNode}
-import org.apache.spark.sql.types.{CalendarIntervalType, DateType, IntegerType, TimestampType}
+import org.apache.spark.sql.execution.{ExternalAppendOnlyUnsafeRowArray, SparkPlan}
 
 /**
  * This class calculates and outputs (windowed) aggregates over the rows in a single (sorted)
@@ -58,7 +52,9 @@ import org.apache.spark.sql.types.{CalendarIntervalType, DateType, IntegerType,
  *     4. 1 PRECEDING AND 1 FOLLOWING
  *     5. 1 FOLLOWING AND 2 FOLLOWING
  * - Offset frame: The frame consist of one row, which is an offset number of rows away from the
- *   current row. Only [[OffsetWindowFunction]]s can be processed in an offset frame.
+ *   current row. Only [[OffsetWindowFunction]]s can be processed in an offset frame. There are
+ *   three implements of offset frame: [[FrameLessOffsetWindowFunctionFrame]],
+ *   [[UnboundedOffsetWindowFunctionFrame]] and [[UnboundedPrecedingOffsetWindowFunctionFrame]].
  *
  * Different frame boundaries can be used in Growing, Shrinking and Moving frames. A frame
  * boundary can be either Row or Range based:
@@ -215,4 +211,7 @@ case class WindowExec(
       }
     }
   }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): WindowExec =
+    copy(child = newChild)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowExecBase.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowExecBase.scala
index ed055bb801ae5..b7f3dec224c9c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowExecBase.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowExecBase.scala
@@ -23,8 +23,8 @@ import scala.collection.mutable.ArrayBuffer
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
-import org.apache.spark.sql.execution.{SparkPlan, UnaryExecNode}
-import org.apache.spark.sql.types.{CalendarIntervalType, DateType, IntegerType, TimestampType}
+import org.apache.spark.sql.execution.UnaryExecNode
+import org.apache.spark.sql.types.{CalendarIntervalType, DateType, DayTimeIntervalType, IntegerType, TimestampType, YearMonthIntervalType}
 
 trait WindowExecBase extends UnaryExecNode {
   def windowExpression: Seq[NamedExpression]
@@ -71,6 +71,9 @@ trait WindowExecBase extends UnaryExecNode {
       case (RowFrame, IntegerLiteral(offset)) =>
         RowBoundOrdering(offset)
 
+      case (RowFrame, _) =>
+        sys.error(s"Unhandled bound in windows expressions: $bound")
+
       case (RangeFrame, CurrentRow) =>
         val ordering = RowOrdering.create(orderSpec, child.output)
         RangeBoundOrdering(ordering, IdentityProjection, IdentityProjection)
@@ -92,8 +95,11 @@ trait WindowExecBase extends UnaryExecNode {
         // Create the projection which returns the current 'value' modified by adding the offset.
         val boundExpr = (expr.dataType, boundOffset.dataType) match {
           case (DateType, IntegerType) => DateAdd(expr, boundOffset)
-          case (TimestampType, CalendarIntervalType) =>
-            TimeAdd(expr, boundOffset, Some(timeZone))
+          case (DateType, YearMonthIntervalType) => DateAddYMInterval(expr, boundOffset)
+          case (TimestampType, CalendarIntervalType) => TimeAdd(expr, boundOffset, Some(timeZone))
+          case (TimestampType, YearMonthIntervalType) =>
+            TimestampAddYMInterval(expr, boundOffset, Some(timeZone))
+          case (TimestampType, DayTimeIntervalType) => TimeAdd(expr, boundOffset, Some(timeZone))
           case (a, b) if a == b => Add(expr, boundOffset)
         }
         val bound = MutableProjection.create(boundExpr :: Nil, child.output)
@@ -116,13 +122,22 @@ trait WindowExecBase extends UnaryExecNode {
    * [[WindowExpression]]s and factory function for the [[WindowFrameFunction]].
    */
   protected lazy val windowFrameExpressionFactoryPairs = {
-    type FrameKey = (String, FrameType, Expression, Expression)
+    type FrameKey = (String, FrameType, Expression, Expression, Seq[Expression])
     type ExpressionBuffer = mutable.Buffer[Expression]
     val framedFunctions = mutable.Map.empty[FrameKey, (ExpressionBuffer, ExpressionBuffer)]
 
     // Add a function and its function to the map for a given frame.
     def collect(tpe: String, fr: SpecifiedWindowFrame, e: Expression, fn: Expression): Unit = {
-      val key = (tpe, fr.frameType, fr.lower, fr.upper)
+      val key = fn match {
+        // This branch is used for Lead/Lag to support ignoring null and optimize the performance
+        // for NthValue ignoring null.
+        // All window frames move in rows. If there are multiple Leads, Lags or NthValues acting on
+        // a row and operating on different input expressions, they should not be moved uniformly
+        // by row. Therefore, we put these functions in different window frames.
+        case f: OffsetWindowFunction if f.ignoreNulls =>
+          (tpe, fr.frameType, fr.lower, fr.upper, f.children.map(_.canonicalized))
+        case _ => (tpe, fr.frameType, fr.lower, fr.upper, Nil)
+      }
       val (es, fns) = framedFunctions.getOrElseUpdate(
         key, (ArrayBuffer.empty[Expression], ArrayBuffer.empty[Expression]))
       es += e
@@ -136,8 +151,16 @@ trait WindowExecBase extends UnaryExecNode {
           val frame = spec.frameSpecification.asInstanceOf[SpecifiedWindowFrame]
           function match {
             case AggregateExpression(f, _, _, _, _) => collect("AGGREGATE", frame, e, f)
+            case f: FrameLessOffsetWindowFunction =>
+              collect("FRAME_LESS_OFFSET", f.fakeFrame, e, f)
+            case f: OffsetWindowFunction if frame.frameType == RowFrame &&
+              frame.lower == UnboundedPreceding =>
+              frame.upper match {
+                case UnboundedFollowing => collect("UNBOUNDED_OFFSET", f.fakeFrame, e, f)
+                case CurrentRow => collect("UNBOUNDED_PRECEDING_OFFSET", f.fakeFrame, e, f)
+                case _ => collect("AGGREGATE", frame, e, f)
+              }
             case f: AggregateWindowFunction => collect("AGGREGATE", frame, e, f)
-            case f: OffsetWindowFunction => collect("OFFSET", frame, e, f)
             case f: PythonUDF => collect("AGGREGATE", frame, e, f)
             case f => sys.error(s"Unsupported window function: $f")
           }
@@ -171,27 +194,54 @@ trait WindowExecBase extends UnaryExecNode {
 
         // Create the factory to produce WindowFunctionFrame.
         val factory = key match {
-          // Offset Frame
-          case ("OFFSET", _, IntegerLiteral(offset), _) =>
+          // Frameless offset Frame
+          case ("FRAME_LESS_OFFSET", _, IntegerLiteral(offset), _, expr) =>
             target: InternalRow =>
-              new OffsetWindowFunctionFrame(
+              new FrameLessOffsetWindowFunctionFrame(
+                target,
+                ordinal,
+                // OFFSET frame functions are guaranteed be OffsetWindowFunction.
+                functions.map(_.asInstanceOf[OffsetWindowFunction]),
+                child.output,
+                (expressions, schema) =>
+                  MutableProjection.create(expressions, schema),
+                offset,
+                expr.nonEmpty)
+          case ("UNBOUNDED_OFFSET", _, IntegerLiteral(offset), _, expr) =>
+            target: InternalRow => {
+              new UnboundedOffsetWindowFunctionFrame(
                 target,
                 ordinal,
-                // OFFSET frame functions are guaranteed be OffsetWindowFunctions.
+                // OFFSET frame functions are guaranteed be OffsetWindowFunction.
                 functions.map(_.asInstanceOf[OffsetWindowFunction]),
                 child.output,
                 (expressions, schema) =>
                   MutableProjection.create(expressions, schema),
-                offset)
+                offset,
+                expr.nonEmpty)
+            }
+          case ("UNBOUNDED_PRECEDING_OFFSET", _, IntegerLiteral(offset), _, expr) =>
+            target: InternalRow => {
+              new UnboundedPrecedingOffsetWindowFunctionFrame(
+                target,
+                ordinal,
+                // OFFSET frame functions are guaranteed be OffsetWindowFunction.
+                functions.map(_.asInstanceOf[OffsetWindowFunction]),
+                child.output,
+                (expressions, schema) =>
+                  MutableProjection.create(expressions, schema),
+                offset,
+                expr.nonEmpty)
+            }
 
           // Entire Partition Frame.
-          case ("AGGREGATE", _, UnboundedPreceding, UnboundedFollowing) =>
+          case ("AGGREGATE", _, UnboundedPreceding, UnboundedFollowing, _) =>
             target: InternalRow => {
               new UnboundedWindowFunctionFrame(target, processor)
             }
 
           // Growing Frame.
-          case ("AGGREGATE", frameType, UnboundedPreceding, upper) =>
+          case ("AGGREGATE", frameType, UnboundedPreceding, upper, _) =>
             target: InternalRow => {
               new UnboundedPrecedingWindowFunctionFrame(
                 target,
@@ -200,7 +250,7 @@ trait WindowExecBase extends UnaryExecNode {
             }
 
           // Shrinking Frame.
-          case ("AGGREGATE", frameType, lower, UnboundedFollowing) =>
+          case ("AGGREGATE", frameType, lower, UnboundedFollowing, _) =>
             target: InternalRow => {
               new UnboundedFollowingWindowFunctionFrame(
                 target,
@@ -209,7 +259,7 @@ trait WindowExecBase extends UnaryExecNode {
             }
 
           // Moving Frame.
-          case ("AGGREGATE", frameType, lower, upper) =>
+          case ("AGGREGATE", frameType, lower, upper, _) =>
             target: InternalRow => {
               new SlidingWindowFunctionFrame(
                 target,
@@ -217,6 +267,9 @@ trait WindowExecBase extends UnaryExecNode {
                 createBoundOrdering(frameType, lower, timeZone),
                 createBoundOrdering(frameType, upper, timeZone))
             }
+
+          case _ =>
+            sys.error(s"Unsupported factory: $key")
         }
 
         // Keep track of the number of expressions. This is a side-effect in a map...
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowFunctionFrame.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowFunctionFrame.scala
index dc1b919feefe4..7d0859564d18e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowFunctionFrame.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowFunctionFrame.scala
@@ -30,6 +30,9 @@ import org.apache.spark.sql.execution.ExternalAppendOnlyUnsafeRowArray
  * A window function calculates the results of a number of window functions for a window frame.
  * Before use a frame must be prepared by passing it all the rows in the current partition. After
  * preparation the update method can be called to fill the output rows.
+ *
+ * Note: `WindowFunctionFrame` instances are reused during window execution. The `prepare` method
+ * will be called before processing the next partition, and must reset the states.
  */
 abstract class WindowFunctionFrame {
   /**
@@ -77,7 +80,7 @@ object WindowFunctionFrame {
  * @param newMutableProjection function used to create the projection.
  * @param offset by which rows get moved within a partition.
  */
-final class OffsetWindowFunctionFrame(
+abstract class OffsetWindowFunctionFrameBase(
     target: InternalRow,
     ordinal: Int,
     expressions: Array[OffsetWindowFunction],
@@ -87,23 +90,25 @@ final class OffsetWindowFunctionFrame(
   extends WindowFunctionFrame {
 
   /** Rows of the partition currently being processed. */
-  private[this] var input: ExternalAppendOnlyUnsafeRowArray = null
+  protected var input: ExternalAppendOnlyUnsafeRowArray = null
 
   /**
    * An iterator over the [[input]]
    */
-  private[this] var inputIterator: Iterator[UnsafeRow] = _
+  protected var inputIterator: Iterator[UnsafeRow] = _
 
   /** Index of the input row currently used for output. */
-  private[this] var inputIndex = 0
+  protected var inputIndex = 0
+
+  /** Attributes of the input row currently used for output. */
+  protected val inputAttrs = inputSchema.map(_.withNullability(true))
 
   /**
    * Create the projection used when the offset row exists.
    * Please note that this project always respect null input values (like PostgreSQL).
    */
-  private[this] val projection = {
+  protected val projection = {
     // Collect the expressions and bind them.
-    val inputAttrs = inputSchema.map(_.withNullability(true))
     val boundExpressions = Seq.fill(ordinal)(NoOp) ++ bindReferences(
       expressions.toSeq.map(_.input), inputAttrs)
 
@@ -112,9 +117,8 @@ final class OffsetWindowFunctionFrame(
   }
 
   /** Create the projection used when the offset row DOES NOT exists. */
-  private[this] val fillDefaultValue = {
+  protected val fillDefaultValue = {
     // Collect the expressions and bind them.
-    val inputAttrs: AttributeSeq = inputSchema.map(_.withNullability(true))
     val boundExpressions = Seq.fill(ordinal)(NoOp) ++ expressions.toSeq.map { e =>
       if (e.default == null || e.default.foldable && e.default.eval() == null) {
         // The default value is null.
@@ -129,32 +133,254 @@ final class OffsetWindowFunctionFrame(
     newMutableProjection(boundExpressions, Nil).target(target)
   }
 
-  override def prepare(rows: ExternalAppendOnlyUnsafeRowArray): Unit = {
+  /** Holder the UnsafeRow where the input operator by function is not null. */
+  protected var nextSelectedRow = EmptyRow
+
+  // The number of rows skipped to get the next UnsafeRow where the input operator by function
+  // is not null.
+  protected var skippedNonNullCount = 0
+
+  // Reset the states by the data of the new partition.
+  protected def resetStates(rows: ExternalAppendOnlyUnsafeRowArray): Unit = {
     input = rows
     inputIterator = input.generateIterator()
-    // drain the first few rows if offset is larger than zero
     inputIndex = 0
-    while (inputIndex < offset) {
-      if (inputIterator.hasNext) inputIterator.next()
+    skippedNonNullCount = 0
+    nextSelectedRow = EmptyRow
+  }
+
+  /** Create the projection to determine whether input is null. */
+  protected val project = UnsafeProjection.create(Seq(IsNull(expressions.head.input)), inputSchema)
+
+  /** Check if the output value of the first index is null. */
+  protected def nullCheck(row: InternalRow): Boolean = project(row).getBoolean(0)
+
+  /** find the offset row whose input is not null */
+  protected def findNextRowWithNonNullInput(): Unit = {
+    while (skippedNonNullCount < offset && inputIndex < input.length) {
+      val r = WindowFunctionFrame.getNextOrNull(inputIterator)
+      if (!nullCheck(r)) {
+        nextSelectedRow = r
+        skippedNonNullCount += 1
+      }
       inputIndex += 1
     }
-    inputIndex = offset
+  }
+
+  override def currentLowerBound(): Int = throw new UnsupportedOperationException()
+
+  override def currentUpperBound(): Int = throw new UnsupportedOperationException()
+}
+
+/**
+ * The frameless offset window frame is an internal window frame just used to optimize the
+ * performance for the window function that returns the value of the input column offset
+ * by a number of rows according to the current row. The internal window frame is not a popular
+ * window frame cannot be specified and used directly by the users. This window frame
+ * calculates frames containing LEAD/LAG statements.
+ */
+class FrameLessOffsetWindowFunctionFrame(
+    target: InternalRow,
+    ordinal: Int,
+    expressions: Array[OffsetWindowFunction],
+    inputSchema: Seq[Attribute],
+    newMutableProjection: (Seq[Expression], Seq[Attribute]) => MutableProjection,
+    offset: Int,
+    ignoreNulls: Boolean = false)
+  extends OffsetWindowFunctionFrameBase(
+    target, ordinal, expressions, inputSchema, newMutableProjection, offset) {
+
+  override def prepare(rows: ExternalAppendOnlyUnsafeRowArray): Unit = {
+    resetStates(rows)
+    if (ignoreNulls) {
+      findNextRowWithNonNullInput()
+    } else {
+      // drain the first few rows if offset is larger than zero
+      while (inputIndex < offset) {
+        if (inputIterator.hasNext) inputIterator.next()
+        inputIndex += 1
+      }
+      inputIndex = offset
+    }
+  }
+
+  private val doWrite = if (ignoreNulls && offset > 0) {
+    // For illustration, here is one example: the input data contains nine rows,
+    // and the input values of each row are: null, x, null, null, y, null, z, v, null.
+    // We use lead(input, 2) with IGNORE NULLS and the process is as follows:
+    // 1. current row -> null, next selected row -> y, output: y;
+    // 2. current row -> x, next selected row -> z, output: z;
+    // 3. current row -> null, next selected row -> z, output: z;
+    // 4. current row -> null, next selected row -> z, output: z;
+    // 5. current row -> y, next selected row -> v, output: v;
+    // 6. current row -> null, next selected row -> v, output: v;
+    // 7. current row -> z, next selected row -> empty, output: null;
+    // ... next selected row is empty, all following return null.
+    (current: InternalRow) =>
+      if (nextSelectedRow == EmptyRow) {
+        // Use default values since the offset row whose input value is not null does not exist.
+        fillDefaultValue(current)
+      } else {
+        if (nullCheck(current)) {
+          projection(nextSelectedRow)
+        } else {
+          skippedNonNullCount -= 1
+          findNextRowWithNonNullInput()
+          if (skippedNonNullCount == offset) {
+            projection(nextSelectedRow)
+          } else {
+            // Use default values since the offset row whose input value is not null does not exist.
+            fillDefaultValue(current)
+            nextSelectedRow = EmptyRow
+          }
+        }
+      }
+  } else if (ignoreNulls && offset < 0) {
+    // For illustration, here is one example: the input data contains nine rows,
+    // and the input values of each row are: null, x, null, null, y, null, z, v, null.
+    // We use lag(input, 1) with IGNORE NULLS and the process is as follows:
+    // 1. current row -> null, next selected row -> empty, output: null;
+    // 2. current row -> x, next selected row -> empty, output: null;
+    // 3. current row -> null, next selected row -> x, output: x;
+    // 4. current row -> null, next selected row -> x, output: x;
+    // 5. current row -> y, next selected row -> x, output: x;
+    // 6. current row -> null, next selected row -> y, output: y;
+    // 7. current row -> z, next selected row -> y, output: y;
+    // 8. current row -> v, next selected row -> z, output: z;
+    // 9. current row -> null, next selected row -> v, output: v;
+    val absOffset = Math.abs(offset)
+    (current: InternalRow) =>
+      if (skippedNonNullCount == absOffset) {
+        nextSelectedRow = EmptyRow
+        skippedNonNullCount -= 1
+        while (nextSelectedRow == EmptyRow && inputIndex < input.length) {
+          val r = WindowFunctionFrame.getNextOrNull(inputIterator)
+          if (!nullCheck(r)) {
+            nextSelectedRow = r
+          }
+          inputIndex += 1
+        }
+      }
+      if (nextSelectedRow == EmptyRow) {
+        // Use default values since the offset row whose input value is not null does not exist.
+        fillDefaultValue(current)
+      } else {
+        projection(nextSelectedRow)
+      }
+      if (!nullCheck(current)) {
+        skippedNonNullCount += 1
+      }
+  } else {
+    (current: InternalRow) =>
+      if (inputIndex >= 0 && inputIndex < input.length) {
+        val r = WindowFunctionFrame.getNextOrNull(inputIterator)
+        projection(r)
+      } else {
+        // Use default values since the offset row does not exist.
+        fillDefaultValue(current)
+      }
+      inputIndex += 1
   }
 
   override def write(index: Int, current: InternalRow): Unit = {
-    if (inputIndex >= 0 && inputIndex < input.length) {
-      val r = WindowFunctionFrame.getNextOrNull(inputIterator)
-      projection(r)
+    doWrite(current)
+  }
+}
+
+/**
+ * The unbounded offset window frame is an internal window frame just used to optimize the
+ * performance for the window function that returns the value of the input column offset
+ * by a number of rows within the frame and has specified ROWS BETWEEN UNBOUNDED PRECEDING
+ * AND UNBOUNDED FOLLOWING. The internal window frame is not a popular window frame cannot be
+ * specified and used directly by the users.
+ * The unbounded offset window frame calculates frames containing NTH_VALUE statements.
+ * The unbounded offset window frame return the same value for all rows in the window partition.
+ */
+class UnboundedOffsetWindowFunctionFrame(
+    target: InternalRow,
+    ordinal: Int,
+    expressions: Array[OffsetWindowFunction],
+    inputSchema: Seq[Attribute],
+    newMutableProjection: (Seq[Expression], Seq[Attribute]) => MutableProjection,
+    offset: Int,
+    ignoreNulls: Boolean = false)
+  extends OffsetWindowFunctionFrameBase(
+    target, ordinal, expressions, inputSchema, newMutableProjection, offset) {
+  assert(offset > 0)
+
+  override def prepare(rows: ExternalAppendOnlyUnsafeRowArray): Unit = {
+    if (offset > rows.length) {
+      fillDefaultValue(EmptyRow)
     } else {
-      // Use default values since the offset row does not exist.
-      fillDefaultValue(current)
+      resetStates(rows)
+      if (ignoreNulls) {
+        findNextRowWithNonNullInput()
+        if (nextSelectedRow == EmptyRow) {
+          // Use default values since the offset row whose input value is not null does not exist.
+          fillDefaultValue(EmptyRow)
+        } else {
+          projection(nextSelectedRow)
+        }
+      } else {
+        var selectedRow: UnsafeRow = null
+        // drain the first few rows if offset is larger than one
+        while (inputIndex < offset) {
+          selectedRow = WindowFunctionFrame.getNextOrNull(inputIterator)
+          inputIndex += 1
+        }
+        projection(selectedRow)
+      }
     }
-    inputIndex += 1
   }
 
-  override def currentLowerBound(): Int = throw new UnsupportedOperationException()
+  override def write(index: Int, current: InternalRow): Unit = {
+    // The results are the same for each row in the partition, and have been evaluated in prepare.
+    // Don't need to recalculate here.
+  }
+}
 
-  override def currentUpperBound(): Int = throw new UnsupportedOperationException()
+/**
+ * The unbounded preceding offset window frame is an internal window frame just used to optimize
+ * the performance for the window function that returns the value of the input column offset
+ * by a number of rows within the frame and has specified ROWS BETWEEN UNBOUNDED PRECEDING
+ * AND CURRENT ROW. The internal window frame is not a popular window frame cannot be specified
+ * and used directly by the users.
+ * The unbounded preceding offset window frame calculates frames containing NTH_VALUE statements.
+ * The unbounded preceding offset window frame return the same value for rows which index
+ * (starting from 1) equal to or greater than offset in the window partition.
+ */
+class UnboundedPrecedingOffsetWindowFunctionFrame(
+    target: InternalRow,
+    ordinal: Int,
+    expressions: Array[OffsetWindowFunction],
+    inputSchema: Seq[Attribute],
+    newMutableProjection: (Seq[Expression], Seq[Attribute]) => MutableProjection,
+    offset: Int,
+    ignoreNulls: Boolean = false)
+  extends OffsetWindowFunctionFrameBase(
+    target, ordinal, expressions, inputSchema, newMutableProjection, offset) {
+  assert(offset > 0)
+
+  override def prepare(rows: ExternalAppendOnlyUnsafeRowArray): Unit = {
+    resetStates(rows)
+    if (ignoreNulls) {
+      findNextRowWithNonNullInput()
+    } else {
+      // drain the first few rows if offset is larger than one
+      while (inputIndex < offset) {
+        nextSelectedRow = WindowFunctionFrame.getNextOrNull(inputIterator)
+        inputIndex += 1
+      }
+    }
+  }
+
+  override def write(index: Int, current: InternalRow): Unit = {
+    if (index >= inputIndex - 1 && nextSelectedRow != null) {
+      projection(nextSelectedRow)
+    } else {
+      fillDefaultValue(EmptyRow)
+    }
+  }
 }
 
 /**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/expressions/Aggregator.scala b/sql/core/src/main/scala/org/apache/spark/sql/expressions/Aggregator.scala
index 878dc0d83f45a..62d04cf7f7cac 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/expressions/Aggregator.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/expressions/Aggregator.scala
@@ -35,6 +35,8 @@ import org.apache.spark.sql.execution.aggregate.TypedAggregateExpression
  *     def reduce(b: Int, a: Data): Int = b + a.i
  *     def merge(b1: Int, b2: Int): Int = b1 + b2
  *     def finish(r: Int): Int = r
+ *     def bufferEncoder: Encoder[Int] = Encoders.scalaInt
+ *     def outputEncoder: Encoder[Int] = Encoders.scalaInt
  *   }.toColumn()
  *
  *   val ds: Dataset[Data] = ...
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala b/sql/core/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala
index 2ef6e3d291cef..03dc9abf081fe 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala
@@ -17,16 +17,13 @@
 
 package org.apache.spark.sql.expressions
 
-import scala.reflect.runtime.universe.TypeTag
-
-import org.apache.spark.annotation.{Experimental, Stable}
+import org.apache.spark.annotation.Stable
 import org.apache.spark.sql.{Column, Encoder}
-import org.apache.spark.sql.catalyst.ScalaReflection
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.catalyst.expressions.{Expression, ScalaUDF}
 import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Complete}
 import org.apache.spark.sql.execution.aggregate.ScalaAggregator
-import org.apache.spark.sql.types.{AnyDataType, DataType}
+import org.apache.spark.sql.types.DataType
 
 /**
  * A user-defined function. To create one, use the `udf` functions in `functions`.
@@ -94,6 +91,7 @@ private[spark] case class SparkUserDefinedFunction(
     f: AnyRef,
     dataType: DataType,
     inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Nil,
+    outputEncoder: Option[ExpressionEncoder[_]] = None,
     name: Option[String] = None,
     nullable: Boolean = true,
     deterministic: Boolean = true) extends UserDefinedFunction {
@@ -109,6 +107,7 @@ private[spark] case class SparkUserDefinedFunction(
       dataType,
       exprs,
       inputEncoders,
+      outputEncoder,
       udfName = name,
       nullable = nullable,
       udfDeterministic = deterministic)
@@ -150,7 +149,9 @@ private[sql] case class UserDefinedAggregator[IN, BUF, OUT](
   // This is also used by udf.register(...) when it detects a UserDefinedAggregator
   def scalaAggregator(exprs: Seq[Expression]): ScalaAggregator[IN, BUF, OUT] = {
     val iEncoder = inputEncoder.asInstanceOf[ExpressionEncoder[IN]]
-    ScalaAggregator(exprs, aggregator, iEncoder, nullable, deterministic)
+    val bEncoder = aggregator.bufferEncoder.asInstanceOf[ExpressionEncoder[BUF]]
+    ScalaAggregator(
+      exprs, aggregator, iEncoder, bEncoder, nullable, deterministic, aggregatorName = name)
   }
 
   override def withName(name: String): UserDefinedAggregator[IN, BUF, OUT] = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/expressions/Window.scala b/sql/core/src/main/scala/org/apache/spark/sql/expressions/Window.scala
index d13baaedbaeff..93bf738a53daf 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/expressions/Window.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/expressions/Window.scala
@@ -136,8 +136,8 @@ object Window {
    *   val df = Seq((1, "a"), (1, "a"), (2, "a"), (1, "b"), (2, "b"), (3, "b"))
    *     .toDF("id", "category")
    *   val byCategoryOrderedById =
-   *     Window.partitionBy('category).orderBy('id).rowsBetween(Window.currentRow, 1)
-   *   df.withColumn("sum", sum('id) over byCategoryOrderedById).show()
+   *     Window.partitionBy($"category").orderBy($"id").rowsBetween(Window.currentRow, 1)
+   *   df.withColumn("sum", sum($"id") over byCategoryOrderedById).show()
    *
    *   +---+--------+---+
    *   | id|category|sum|
@@ -188,8 +188,8 @@ object Window {
    *   val df = Seq((1, "a"), (1, "a"), (2, "a"), (1, "b"), (2, "b"), (3, "b"))
    *     .toDF("id", "category")
    *   val byCategoryOrderedById =
-   *     Window.partitionBy('category).orderBy('id).rangeBetween(Window.currentRow, 1)
-   *   df.withColumn("sum", sum('id) over byCategoryOrderedById).show()
+   *     Window.partitionBy($"category").orderBy($"id").rangeBetween(Window.currentRow, 1)
+   *   df.withColumn("sum", sum($"id") over byCategoryOrderedById).show()
    *
    *   +---+--------+---+
    *   | id|category|sum|
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/expressions/WindowSpec.scala b/sql/core/src/main/scala/org/apache/spark/sql/expressions/WindowSpec.scala
index 58227f075f2c7..09a945f162a98 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/expressions/WindowSpec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/expressions/WindowSpec.scala
@@ -100,8 +100,8 @@ class WindowSpec private[sql](
    *   val df = Seq((1, "a"), (1, "a"), (2, "a"), (1, "b"), (2, "b"), (3, "b"))
    *     .toDF("id", "category")
    *   val byCategoryOrderedById =
-   *     Window.partitionBy('category).orderBy('id).rowsBetween(Window.currentRow, 1)
-   *   df.withColumn("sum", sum('id) over byCategoryOrderedById).show()
+   *     Window.partitionBy($"category").orderBy($"id").rowsBetween(Window.currentRow, 1)
+   *   df.withColumn("sum", sum($"id") over byCategoryOrderedById).show()
    *
    *   +---+--------+---+
    *   | id|category|sum|
@@ -168,8 +168,8 @@ class WindowSpec private[sql](
    *   val df = Seq((1, "a"), (1, "a"), (2, "a"), (1, "b"), (2, "b"), (3, "b"))
    *     .toDF("id", "category")
    *   val byCategoryOrderedById =
-   *     Window.partitionBy('category).orderBy('id).rangeBetween(Window.currentRow, 1)
-   *   df.withColumn("sum", sum('id) over byCategoryOrderedById).show()
+   *     Window.partitionBy($"category").orderBy($"id").rangeBetween(Window.currentRow, 1)
+   *   df.withColumn("sum", sum($"id") over byCategoryOrderedById).show()
    *
    *   +---+--------+---+
    *   | id|category|sum|
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/expressions/scalalang/typed.scala b/sql/core/src/main/scala/org/apache/spark/sql/expressions/scalalang/typed.scala
index f7591e4d265e0..4e3c5586209e4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/expressions/scalalang/typed.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/expressions/scalalang/typed.scala
@@ -44,8 +44,6 @@ object typed {
     override protected def _sqlContext: SQLContext = null
   }
 
-  import implicits._
-
   /**
    * Average aggregate function.
    *
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index 5481337bf6cee..bd222d150d46a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -18,10 +18,8 @@
 package org.apache.spark.sql
 
 import scala.collection.JavaConverters._
-import scala.language.implicitConversions
-import scala.reflect.runtime.universe.{typeTag, TypeTag}
+import scala.reflect.runtime.universe.TypeTag
 import scala.util.Try
-import scala.util.control.NonFatal
 
 import org.apache.spark.annotation.Stable
 import org.apache.spark.sql.api.java._
@@ -31,11 +29,12 @@ import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate._
 import org.apache.spark.sql.catalyst.plans.logical.{BROADCAST, HintInfo, ResolvedHint}
-import org.apache.spark.sql.catalyst.util.TimestampFormatter
+import org.apache.spark.sql.catalyst.util.{CharVarcharUtils, TimestampFormatter}
 import org.apache.spark.sql.execution.SparkSqlParser
 import org.apache.spark.sql.expressions.{Aggregator, SparkUserDefinedFunction, UserDefinedAggregator, UserDefinedFunction}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
+import org.apache.spark.sql.types.DataType.parseTypeWithFallback
 import org.apache.spark.util.Utils
 
 /**
@@ -114,6 +113,16 @@ object functions {
    */
   def lit(literal: Any): Column = typedLit(literal)
 
+  /**
+   * Creates a [[Column]] of literal value.
+   *
+   * An alias of `typedlit`, and it is encouraged to use `typedlit` directly.
+   *
+   * @group normal_funcs
+   * @since 2.2.0
+   */
+  def typedLit[T : TypeTag](literal: T): Column = typedlit(literal)
+
   /**
    * Creates a [[Column]] of literal value.
    *
@@ -124,9 +133,9 @@ object functions {
    * can handle parameterized scala types e.g.: List, Seq and Map.
    *
    * @group normal_funcs
-   * @since 2.2.0
+   * @since 3.2.0
    */
-  def typedLit[T : TypeTag](literal: T): Column = literal match {
+  def typedlit[T : TypeTag](literal: T): Column = literal match {
     case c: Column => c
     case s: Symbol => new ColumnName(s.name)
     case _ => Column(Literal.create(literal))
@@ -262,7 +271,7 @@ object functions {
   /**
    * Aggregate function: returns the approximate number of distinct items in a group.
    *
-   * @param rsd maximum estimation error allowed (default = 0.05)
+   * @param rsd maximum relative standard deviation allowed (default = 0.05)
    *
    * @group agg_funcs
    * @since 2.1.0
@@ -274,7 +283,7 @@ object functions {
   /**
    * Aggregate function: returns the approximate number of distinct items in a group.
    *
-   * @param rsd maximum estimation error allowed (default = 0.05)
+   * @param rsd maximum relative standard deviation allowed (default = 0.05)
    *
    * @group agg_funcs
    * @since 2.1.0
@@ -389,24 +398,37 @@ object functions {
   /**
    * Aggregate function: returns the number of distinct items in a group.
    *
+   * An alias of `count_distinct`, and it is encouraged to use `count_distinct` directly.
+   *
    * @group agg_funcs
    * @since 1.3.0
    */
   @scala.annotation.varargs
-  def countDistinct(expr: Column, exprs: Column*): Column =
-    // For usage like countDistinct("*"), we should let analyzer expand star and
-    // resolve function.
-    Column(UnresolvedFunction("count", (expr +: exprs).map(_.expr), isDistinct = true))
+  def countDistinct(expr: Column, exprs: Column*): Column = count_distinct(expr, exprs: _*)
 
   /**
    * Aggregate function: returns the number of distinct items in a group.
    *
+   * An alias of `count_distinct`, and it is encouraged to use `count_distinct` directly.
+   *
    * @group agg_funcs
    * @since 1.3.0
    */
   @scala.annotation.varargs
   def countDistinct(columnName: String, columnNames: String*): Column =
-    countDistinct(Column(columnName), columnNames.map(Column.apply) : _*)
+    count_distinct(Column(columnName), columnNames.map(Column.apply) : _*)
+
+  /**
+   * Aggregate function: returns the number of distinct items in a group.
+   *
+   * @group agg_funcs
+   * @since 3.2.0
+   */
+  @scala.annotation.varargs
+  def count_distinct(expr: Column, exprs: Column*): Column =
+    // For usage like countDistinct("*"), we should let analyzer expand star and
+    // resolve function.
+    Column(UnresolvedFunction("count", (expr +: exprs).map(_.expr), isDistinct = true))
 
   /**
    * Aggregate function: returns the population covariance for two columns.
@@ -461,7 +483,7 @@ object functions {
    * @since 2.0.0
    */
   def first(e: Column, ignoreNulls: Boolean): Column = withAggregateFunction {
-    new First(e.expr, Literal(ignoreNulls))
+    First(e.expr, ignoreNulls)
   }
 
   /**
@@ -586,7 +608,7 @@ object functions {
    * @since 2.0.0
    */
   def last(e: Column, ignoreNulls: Boolean): Column = withAggregateFunction {
-    new Last(e.expr, Literal(ignoreNulls))
+    new Last(e.expr, ignoreNulls)
   }
 
   /**
@@ -684,8 +706,9 @@ object functions {
   def min(columnName: String): Column = min(Column(columnName))
 
   /**
-   * Aggregate function: returns and array of the approximate percentile values
-   * of numeric column col at the given percentages.
+   * Aggregate function: returns the approximate `percentile` of the numeric column `col` which
+   * is the smallest value in the ordered `col` values (sorted from least to greatest) such that
+   * no more than `percentage` of `col` values is less than the value or equal to that value.
    *
    * If percentage is an array, each value must be between 0.0 and 1.0.
    * If it is a single floating point value, it must be between 0.0 and 1.0.
@@ -706,6 +729,15 @@ object functions {
     }
   }
 
+  /**
+   * Aggregate function: returns the product of all numerical elements in a group.
+   *
+   * @group agg_funcs
+   * @since 3.2.0
+   */
+  def product(e: Column): Column =
+    withAggregateFunction { new Product(e.expr) }
+
   /**
    * Aggregate function: returns the skewness of the values in a group.
    *
@@ -796,6 +828,7 @@ object functions {
    * @group agg_funcs
    * @since 1.3.0
    */
+  @deprecated("Use sum_distinct", "3.2.0")
   def sumDistinct(e: Column): Column = withAggregateFunction(Sum(e.expr), isDistinct = true)
 
   /**
@@ -804,7 +837,16 @@ object functions {
    * @group agg_funcs
    * @since 1.3.0
    */
-  def sumDistinct(columnName: String): Column = sumDistinct(Column(columnName))
+  @deprecated("Use sum_distinct", "3.2.0")
+  def sumDistinct(columnName: String): Column = sum_distinct(Column(columnName))
+
+  /**
+   * Aggregate function: returns the sum of distinct values in the expression.
+   *
+   * @group agg_funcs
+   * @since 3.2.0
+   */
+  def sum_distinct(e: Column): Column = withAggregateFunction(Sum(e.expr), isDistinct = true)
 
   /**
    * Aggregate function: alias for `var_samp`.
@@ -937,8 +979,24 @@ object functions {
    * @group window_funcs
    * @since 1.4.0
    */
-  def lag(e: Column, offset: Int, defaultValue: Any): Column = withExpr {
-    Lag(e.expr, Literal(offset), Literal(defaultValue))
+  def lag(e: Column, offset: Int, defaultValue: Any): Column = {
+    lag(e, offset, defaultValue, false)
+  }
+
+  /**
+   * Window function: returns the value that is `offset` rows before the current row, and
+   * `defaultValue` if there is less than `offset` rows before the current row. `ignoreNulls`
+   * determines whether null values of row are included in or eliminated from the calculation.
+   * For example, an `offset` of one will return the previous row at any given point in the
+   * window partition.
+   *
+   * This is equivalent to the LAG function in SQL.
+   *
+   * @group window_funcs
+   * @since 3.2.0
+   */
+  def lag(e: Column, offset: Int, defaultValue: Any, ignoreNulls: Boolean): Column = withExpr {
+    Lag(e.expr, Literal(offset), Literal(defaultValue), ignoreNulls)
   }
 
   /**
@@ -989,8 +1047,53 @@ object functions {
    * @group window_funcs
    * @since 1.4.0
    */
-  def lead(e: Column, offset: Int, defaultValue: Any): Column = withExpr {
-    Lead(e.expr, Literal(offset), Literal(defaultValue))
+  def lead(e: Column, offset: Int, defaultValue: Any): Column = {
+    lead(e, offset, defaultValue, false)
+  }
+
+  /**
+   * Window function: returns the value that is `offset` rows after the current row, and
+   * `defaultValue` if there is less than `offset` rows after the current row. `ignoreNulls`
+   * determines whether null values of row are included in or eliminated from the calculation.
+   * The default value of `ignoreNulls` is false. For example, an `offset` of one will return
+   * the next row at any given point in the window partition.
+   *
+   * This is equivalent to the LEAD function in SQL.
+   *
+   * @group window_funcs
+   * @since 3.2.0
+   */
+  def lead(e: Column, offset: Int, defaultValue: Any, ignoreNulls: Boolean): Column = withExpr {
+    Lead(e.expr, Literal(offset), Literal(defaultValue), ignoreNulls)
+  }
+
+  /**
+   * Window function: returns the value that is the `offset`th row of the window frame
+   * (counting from 1), and `null` if the size of window frame is less than `offset` rows.
+   *
+   * It will return the `offset`th non-null value it sees when ignoreNulls is set to true.
+   * If all values are null, then null is returned.
+   *
+   * This is equivalent to the nth_value function in SQL.
+   *
+   * @group window_funcs
+   * @since 3.1.0
+   */
+  def nth_value(e: Column, offset: Int, ignoreNulls: Boolean): Column = withExpr {
+    NthValue(e.expr, Literal(offset), ignoreNulls)
+  }
+
+  /**
+   * Window function: returns the value that is the `offset`th row of the window frame
+   * (counting from 1), and `null` if the size of window frame is less than `offset` rows.
+   *
+   * This is equivalent to the nth_value function in SQL.
+   *
+   * @group window_funcs
+   * @since 3.1.0
+   */
+  def nth_value(e: Column, offset: Int): Column = withExpr {
+    NthValue(e.expr, Literal(offset), false)
   }
 
   /**
@@ -1306,7 +1409,7 @@ object functions {
    * @since 1.4.0
    */
   @scala.annotation.varargs
-  def struct(cols: Column*): Column = withExpr { CreateStruct(cols.map(_.expr)) }
+  def struct(cols: Column*): Column = withExpr { CreateStruct.create(cols.map(_.expr)) }
 
   /**
    * Creates a new struct column that composes multiple input columns.
@@ -1350,7 +1453,16 @@ object functions {
    * @group normal_funcs
    * @since 1.4.0
    */
-  def bitwiseNOT(e: Column): Column = withExpr { BitwiseNot(e.expr) }
+  @deprecated("Use bitwise_not", "3.2.0")
+  def bitwiseNOT(e: Column): Column = bitwise_not(e)
+
+  /**
+   * Computes bitwise NOT (~) of a number.
+   *
+   * @group normal_funcs
+   * @since 3.2.0
+   */
+  def bitwise_not(e: Column): Column = withExpr { BitwiseNot(e.expr) }
 
   /**
    * Parses the expression string into the column that it represents, similar to
@@ -1364,7 +1476,7 @@ object functions {
    */
   def expr(expr: String): Column = {
     val parser = SparkSession.getActiveSession.map(_.sessionState.sqlParser).getOrElse {
-      new SparkSqlParser(new SQLConf)
+      new SparkSqlParser()
     }
     Column(parser.parseExpression(expr))
   }
@@ -1397,6 +1509,22 @@ object functions {
    */
   def acos(columnName: String): Column = acos(Column(columnName))
 
+  /**
+   * @return inverse hyperbolic cosine of `e`
+   *
+   * @group math_funcs
+   * @since 3.1.0
+   */
+  def acosh(e: Column): Column = withExpr { Acosh(e.expr) }
+
+  /**
+   * @return inverse hyperbolic cosine of `columnName`
+   *
+   * @group math_funcs
+   * @since 3.1.0
+   */
+  def acosh(columnName: String): Column = acosh(Column(columnName))
+
   /**
    * @return inverse sine of `e` in radians, as if computed by `java.lang.Math.asin`
    *
@@ -1414,7 +1542,23 @@ object functions {
   def asin(columnName: String): Column = asin(Column(columnName))
 
   /**
-   * @return inverse tangent of `e`, as if computed by `java.lang.Math.atan`
+   * @return inverse hyperbolic sine of `e`
+   *
+   * @group math_funcs
+   * @since 3.1.0
+   */
+  def asinh(e: Column): Column = withExpr { Asinh(e.expr) }
+
+  /**
+   * @return inverse hyperbolic sine of `columnName`
+   *
+   * @group math_funcs
+   * @since 3.1.0
+   */
+  def asinh(columnName: String): Column = asinh(Column(columnName))
+
+  /**
+   * @return inverse tangent of `e` as if computed by `java.lang.Math.atan`
    *
    * @group math_funcs
    * @since 1.4.0
@@ -1542,6 +1686,22 @@ object functions {
    */
   def atan2(yValue: Double, xName: String): Column = atan2(yValue, Column(xName))
 
+  /**
+   * @return inverse hyperbolic tangent of `e`
+   *
+   * @group math_funcs
+   * @since 3.1.0
+   */
+  def atanh(e: Column): Column = withExpr { Atanh(e.expr) }
+
+  /**
+   * @return inverse hyperbolic tangent of `columnName`
+   *
+   * @group math_funcs
+   * @since 3.1.0
+   */
+  def atanh(columnName: String): Column = atanh(Column(columnName))
+
   /**
    * An expression that returns the string representation of the binary value of the given long
    * column. For example, bin("12") returns "1100".
@@ -2033,7 +2193,17 @@ object functions {
    * @group math_funcs
    * @since 1.5.0
    */
-  def shiftLeft(e: Column, numBits: Int): Column = withExpr { ShiftLeft(e.expr, lit(numBits).expr) }
+  @deprecated("Use shiftleft", "3.2.0")
+  def shiftLeft(e: Column, numBits: Int): Column = shiftleft(e, numBits)
+
+  /**
+   * Shift the given value numBits left. If the given value is a long value, this function
+   * will return a long value else it will return an integer value.
+   *
+   * @group math_funcs
+   * @since 3.2.0
+   */
+  def shiftleft(e: Column, numBits: Int): Column = withExpr { ShiftLeft(e.expr, lit(numBits).expr) }
 
   /**
    * (Signed) shift the given value numBits right. If the given value is a long value, it will
@@ -2042,7 +2212,17 @@ object functions {
    * @group math_funcs
    * @since 1.5.0
    */
-  def shiftRight(e: Column, numBits: Int): Column = withExpr {
+  @deprecated("Use shiftright", "3.2.0")
+  def shiftRight(e: Column, numBits: Int): Column = shiftright(e, numBits)
+
+  /**
+   * (Signed) shift the given value numBits right. If the given value is a long value, it will
+   * return a long value else it will return an integer value.
+   *
+   * @group math_funcs
+   * @since 3.2.0
+   */
+  def shiftright(e: Column, numBits: Int): Column = withExpr {
     ShiftRight(e.expr, lit(numBits).expr)
   }
 
@@ -2053,7 +2233,17 @@ object functions {
    * @group math_funcs
    * @since 1.5.0
    */
-  def shiftRightUnsigned(e: Column, numBits: Int): Column = withExpr {
+  @deprecated("Use shiftrightunsigned", "3.2.0")
+  def shiftRightUnsigned(e: Column, numBits: Int): Column = shiftrightunsigned(e, numBits)
+
+  /**
+   * Unsigned shift the given value numBits right. If the given value is a long value,
+   * it will return a long value else it will return an integer value.
+   *
+   * @group math_funcs
+   * @since 3.2.0
+   */
+  def shiftrightunsigned(e: Column, numBits: Int): Column = withExpr {
     ShiftRightUnsigned(e.expr, lit(numBits).expr)
   }
 
@@ -2288,6 +2478,36 @@ object functions {
     new XxHash64(cols.map(_.expr))
   }
 
+  /**
+   * Returns null if the condition is true, and throws an exception otherwise.
+   *
+   * @group misc_funcs
+   * @since 3.1.0
+   */
+  def assert_true(c: Column): Column = withExpr {
+    new AssertTrue(c.expr)
+  }
+
+  /**
+   * Returns null if the condition is true; throws an exception with the error message otherwise.
+   *
+   * @group misc_funcs
+   * @since 3.1.0
+   */
+  def assert_true(c: Column, e: Column): Column = withExpr {
+    new AssertTrue(c.expr, e.expr)
+  }
+
+  /**
+   * Throws an exception with the provided error message.
+   *
+   * @group misc_funcs
+   * @since 3.1.0
+   */
+  def raise_error(c: Column): Column = withExpr {
+    RaiseError(c.expr)
+  }
+
   //////////////////////////////////////////////////////////////////////////////////////////////
   // String functions
   //////////////////////////////////////////////////////////////////////////////////////////////
@@ -2331,7 +2551,7 @@ object functions {
    * @since 1.5.0
    */
   def decode(value: Column, charset: String): Column = withExpr {
-    Decode(value.expr, lit(charset).expr)
+    StringDecode(value.expr, lit(charset).expr)
   }
 
   /**
@@ -2478,6 +2698,8 @@ object functions {
   /**
    * Extract a specific group matched by a Java regex, from the specified string column.
    * If the regex did not match, or the specified group did not match, an empty string is returned.
+   * if the specified group index exceeds the group count of regex, an IllegalArgumentException
+   * will be thrown.
    *
    * @group string_funcs
    * @since 1.5.0
@@ -2687,6 +2909,31 @@ object functions {
   // DateTime functions
   //////////////////////////////////////////////////////////////////////////////////////////////
 
+  /**
+   * (Scala-specific) Creates a datetime interval
+   *
+   * @param years Number of years
+   * @param months Number of months
+   * @param weeks Number of weeks
+   * @param days Number of days
+   * @param hours Number of hours
+   * @param mins Number of mins
+   * @param secs Number of secs
+   * @return A datetime interval
+   * @group datetime_funcs
+   * @since 3.2.0
+   */
+  def make_interval(
+      years: Column = lit(0),
+      months: Column = lit(0),
+      weeks: Column = lit(0),
+      days: Column = lit(0),
+      hours: Column = lit(0),
+      mins: Column = lit(0),
+      secs: Column = lit(0)): Column = withExpr {
+    MakeInterval(years.expr, months.expr, weeks.expr, days.expr, hours.expr, mins.expr, secs.expr)
+  }
+
   /**
    * Returns the date that is `numMonths` after `startDate`.
    *
@@ -2715,7 +2962,8 @@ object functions {
   }
 
   /**
-   * Returns the current date as a date column.
+   * Returns the current date at the start of query evaluation as a date column.
+   * All calls of current_date within the same query return the same value.
    *
    * @group datetime_funcs
    * @since 1.5.0
@@ -2723,7 +2971,8 @@ object functions {
   def current_date(): Column = withExpr { CurrentDate() }
 
   /**
-   * Returns the current timestamp as a timestamp column.
+   * Returns the current timestamp at the start of query evaluation as a timestamp column.
+   * All calls of current_timestamp within the same query return the same value.
    *
    * @group datetime_funcs
    * @since 1.5.0
@@ -2950,8 +3199,26 @@ object functions {
    * @group datetime_funcs
    * @since 1.5.0
    */
-  def next_day(date: Column, dayOfWeek: String): Column = withExpr {
-    NextDay(date.expr, lit(dayOfWeek).expr)
+  def next_day(date: Column, dayOfWeek: String): Column = next_day(date, lit(dayOfWeek))
+
+  /**
+   * Returns the first date which is later than the value of the `date` column that is on the
+   * specified day of the week.
+   *
+   * For example, `next_day('2015-07-27', "Sunday")` returns 2015-08-02 because that is the first
+   * Sunday after 2015-07-27.
+   *
+   * @param date      A date, timestamp or string. If a string, the data must be in a format that
+   *                  can be cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+   * @param dayOfWeek A column of the day of week. Case insensitive, and accepts: "Mon", "Tue",
+   *                  "Wed", "Thu", "Fri", "Sat", "Sun"
+   * @return A date, or null if `date` was a string that could not be cast to a date or if
+   *         `dayOfWeek` was an invalid value
+   * @group datetime_funcs
+   * @since 3.2.0
+   */
+  def next_day(date: Column, dayOfWeek: Column): Column = withExpr {
+    NextDay(date.expr, dayOfWeek.expr)
   }
 
   /**
@@ -3229,7 +3496,7 @@ object functions {
    *
    * {{{
    *   val df = ... // schema => timestamp: TimestampType, stockId: StringType, price: DoubleType
-   *   df.groupBy(window($"time", "1 minute", "10 seconds", "5 seconds"), $"stockId")
+   *   df.groupBy(window($"timestamp", "1 minute", "10 seconds", "5 seconds"), $"stockId")
    *     .agg(mean("price"))
    * }}}
    *
@@ -3285,7 +3552,7 @@ object functions {
    *
    * {{{
    *   val df = ... // schema => timestamp: TimestampType, stockId: StringType, price: DoubleType
-   *   df.groupBy(window($"time", "1 minute", "10 seconds"), $"stockId")
+   *   df.groupBy(window($"timestamp", "1 minute", "10 seconds"), $"stockId")
    *     .agg(mean("price"))
    * }}}
    *
@@ -3330,7 +3597,7 @@ object functions {
    *
    * {{{
    *   val df = ... // schema => timestamp: TimestampType, stockId: StringType, price: DoubleType
-   *   df.groupBy(window($"time", "1 minute"), $"stockId")
+   *   df.groupBy(window($"timestamp", "1 minute"), $"stockId")
    *     .agg(mean("price"))
    * }}}
    *
@@ -3358,6 +3625,15 @@ object functions {
     window(timeColumn, windowDuration, windowDuration, "0 second")
   }
 
+  /**
+   * Creates timestamp from the number of seconds since UTC epoch.
+   * @group datetime_funcs
+   * @since 3.1.0
+   */
+  def timestamp_seconds(e: Column): Column = withExpr {
+    SecondsToTimestamp(e.expr)
+  }
+
   //////////////////////////////////////////////////////////////////////////////////////////////
   // Collection functions
   //////////////////////////////////////////////////////////////////////////////////////////////
@@ -3393,8 +3669,22 @@ object functions {
    * @group collection_funcs
    * @since 2.4.0
    */
-  def slice(x: Column, start: Int, length: Int): Column = withExpr {
-    Slice(x.expr, Literal(start), Literal(length))
+  def slice(x: Column, start: Int, length: Int): Column =
+    slice(x, lit(start), lit(length))
+
+  /**
+   * Returns an array containing all the elements in `x` from index `start` (or starting from the
+   * end if `start` is negative) with the specified `length`.
+   *
+   * @param x the array column to be sliced
+   * @param start the starting index
+   * @param length the length of the slice
+   *
+   * @group collection_funcs
+   * @since 3.1.0
+   */
+  def slice(x: Column, start: Column, length: Column): Column = withExpr {
+    Slice(x.expr, start.expr, length.expr)
   }
 
   /**
@@ -3510,22 +3800,22 @@ object functions {
   }
 
   private def createLambda(f: Column => Column) = {
-    val x = UnresolvedNamedLambdaVariable(Seq("x"))
+    val x = UnresolvedNamedLambdaVariable(Seq(UnresolvedNamedLambdaVariable.freshVarName("x")))
     val function = f(Column(x)).expr
     LambdaFunction(function, Seq(x))
   }
 
   private def createLambda(f: (Column, Column) => Column) = {
-    val x = UnresolvedNamedLambdaVariable(Seq("x"))
-    val y = UnresolvedNamedLambdaVariable(Seq("y"))
+    val x = UnresolvedNamedLambdaVariable(Seq(UnresolvedNamedLambdaVariable.freshVarName("x")))
+    val y = UnresolvedNamedLambdaVariable(Seq(UnresolvedNamedLambdaVariable.freshVarName("y")))
     val function = f(Column(x), Column(y)).expr
     LambdaFunction(function, Seq(x, y))
   }
 
   private def createLambda(f: (Column, Column, Column) => Column) = {
-    val x = UnresolvedNamedLambdaVariable(Seq("x"))
-    val y = UnresolvedNamedLambdaVariable(Seq("y"))
-    val z = UnresolvedNamedLambdaVariable(Seq("z"))
+    val x = UnresolvedNamedLambdaVariable(Seq(UnresolvedNamedLambdaVariable.freshVarName("x")))
+    val y = UnresolvedNamedLambdaVariable(Seq(UnresolvedNamedLambdaVariable.freshVarName("y")))
+    val z = UnresolvedNamedLambdaVariable(Seq(UnresolvedNamedLambdaVariable.freshVarName("z")))
     val function = f(Column(x), Column(y), Column(z)).expr
     LambdaFunction(function, Seq(x, y, z))
   }
@@ -3862,7 +4152,7 @@ object functions {
    * @since 2.2.0
    */
   def from_json(e: Column, schema: DataType, options: Map[String, String]): Column = withExpr {
-    JsonToStructs(schema, options, e.expr)
+    JsonToStructs(CharVarcharUtils.failIfHasCharVarchar(schema), options, e.expr)
   }
 
   /**
@@ -3893,8 +4183,9 @@ object functions {
    * @group collection_funcs
    * @since 2.2.0
    */
-  def from_json(e: Column, schema: DataType, options: java.util.Map[String, String]): Column =
-    from_json(e, schema, options.asScala.toMap)
+  def from_json(e: Column, schema: DataType, options: java.util.Map[String, String]): Column = {
+    from_json(e, CharVarcharUtils.failIfHasCharVarchar(schema), options.asScala.toMap)
+  }
 
   /**
    * Parses a column containing a JSON string into a `StructType` with the specified schema.
@@ -3929,9 +4220,7 @@ object functions {
    * Returns `null`, in the case of an unparseable string.
    *
    * @param e a string column containing JSON data.
-   * @param schema the schema to use when parsing the json string as a json string. In Spark 2.1,
-   *               the user-provided schema has to be in JSON format. Since Spark 2.2, the DDL
-   *               format is also supported for the schema.
+   * @param schema the schema as a DDL-formatted string.
    *
    * @group collection_funcs
    * @since 2.1.0
@@ -3946,18 +4235,17 @@ object functions {
    * Returns `null`, in the case of an unparseable string.
    *
    * @param e a string column containing JSON data.
-   * @param schema the schema to use when parsing the json string as a json string, it could be a
-   *               JSON format string or a DDL-formatted string.
+   * @param schema the schema as a DDL-formatted string.
    *
    * @group collection_funcs
    * @since 2.3.0
    */
   def from_json(e: Column, schema: String, options: Map[String, String]): Column = {
-    val dataType = try {
-      DataType.fromJson(schema)
-    } catch {
-      case NonFatal(_) => DataType.fromDDL(schema)
-    }
+    val dataType = parseTypeWithFallback(
+      schema,
+      DataType.fromJson,
+      "Cannot parse the schema in JSON format: ",
+      fallbackParser = DataType.fromDDL)
     from_json(e, dataType, options)
   }
 
@@ -4006,7 +4294,7 @@ object functions {
   /**
    * Parses a JSON string and infers its schema in DDL format.
    *
-   * @param json a string literal containing a JSON string.
+   * @param json a foldable string column containing a JSON string.
    *
    * @group collection_funcs
    * @since 2.4.0
@@ -4016,7 +4304,7 @@ object functions {
   /**
    * Parses a JSON string and infers its schema in DDL format using options.
    *
-   * @param json a string column containing JSON data.
+   * @param json a foldable string column containing JSON data.
    * @param options options to control how the json is parsed. accepts the same options and the
    *                json data source. See [[DataFrameReader#json]].
    * @return a column with string literal containing schema in DDL format.
@@ -4249,7 +4537,8 @@ object functions {
    * @since 3.0.0
    */
   def from_csv(e: Column, schema: StructType, options: Map[String, String]): Column = withExpr {
-    CsvToStructs(schema, options, e.expr)
+    val replaced = CharVarcharUtils.failIfHasCharVarchar(schema).asInstanceOf[StructType]
+    CsvToStructs(replaced, options, e.expr)
   }
 
   /**
@@ -4281,7 +4570,7 @@ object functions {
   /**
    * Parses a CSV string and infers its schema in DDL format.
    *
-   * @param csv a string literal containing a CSV string.
+   * @param csv a foldable string column containing a CSV string.
    *
    * @group collection_funcs
    * @since 3.0.0
@@ -4291,7 +4580,7 @@ object functions {
   /**
    * Parses a CSV string and infers its schema in DDL format using options.
    *
-   * @param csv a string literal containing a CSV string.
+   * @param csv a foldable string column containing a CSV string.
    * @param options options to control how the CSV is parsed. accepts the same options and the
    *                json data source. See [[DataFrameReader#csv]].
    * @return a column with string literal containing schema in DDL format.
@@ -4406,9 +4695,10 @@ object functions {
       | * @since 1.3.0
       | */
       |def udf[$typeTags](f: Function$x[$types]): UserDefinedFunction = {
-      |  val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+      |  val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+      |  val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(UDFRegistration.outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
       |  val inputEncoders = $inputEncoders
-      |  val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
+      |  val udf = SparkUserDefinedFunction(f, dataType, inputEncoders, outputEncoder)
       |  if (nullable) udf else udf.asNonNullable()
       |}""".stripMargin)
   }
@@ -4512,9 +4802,10 @@ object functions {
    * @since 1.3.0
    */
   def udf[RT: TypeTag](f: Function0[RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(UDFRegistration.outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders = Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders, outputEncoder)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4528,9 +4819,10 @@ object functions {
    * @since 1.3.0
    */
   def udf[RT: TypeTag, A1: TypeTag](f: Function1[A1, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(UDFRegistration.outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders, outputEncoder)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4544,9 +4836,10 @@ object functions {
    * @since 1.3.0
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag](f: Function2[A1, A2, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(UDFRegistration.outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders, outputEncoder)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4560,9 +4853,10 @@ object functions {
    * @since 1.3.0
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag](f: Function3[A1, A2, A3, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(UDFRegistration.outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders, outputEncoder)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4576,9 +4870,10 @@ object functions {
    * @since 1.3.0
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag](f: Function4[A1, A2, A3, A4, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(UDFRegistration.outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders, outputEncoder)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4592,9 +4887,10 @@ object functions {
    * @since 1.3.0
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag](f: Function5[A1, A2, A3, A4, A5, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(UDFRegistration.outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders, outputEncoder)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4608,9 +4904,10 @@ object functions {
    * @since 1.3.0
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag](f: Function6[A1, A2, A3, A4, A5, A6, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(UDFRegistration.outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders, outputEncoder)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4624,9 +4921,10 @@ object functions {
    * @since 1.3.0
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag](f: Function7[A1, A2, A3, A4, A5, A6, A7, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(UDFRegistration.outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders, outputEncoder)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4640,9 +4938,10 @@ object functions {
    * @since 1.3.0
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag](f: Function8[A1, A2, A3, A4, A5, A6, A7, A8, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(UDFRegistration.outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders, outputEncoder)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4656,9 +4955,10 @@ object functions {
    * @since 1.3.0
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag](f: Function9[A1, A2, A3, A4, A5, A6, A7, A8, A9, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(UDFRegistration.outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders, outputEncoder)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4672,9 +4972,10 @@ object functions {
    * @since 1.3.0
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag](f: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, RT]): UserDefinedFunction = {
-    val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
+    val outputEncoder = Try(ExpressionEncoder[RT]()).toOption
+    val ScalaReflection.Schema(dataType, nullable) = outputEncoder.map(UDFRegistration.outputSchema).getOrElse(ScalaReflection.schemaFor[RT])
     val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders, outputEncoder)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4878,6 +5179,17 @@ object functions {
     SparkUserDefinedFunction(f, dataType, inputEncoders = Nil)
   }
 
+  /**
+   * Call an user-defined function.
+   *
+   * @group udf_funcs
+   * @since 1.5.0
+   */
+  @scala.annotation.varargs
+  @deprecated("Use call_udf")
+  def callUDF(udfName: String, cols: Column*): Column =
+    call_udf(udfName, cols: _*)
+
   /**
    * Call an user-defined function.
    * Example:
@@ -4887,14 +5199,14 @@ object functions {
    *  val df = Seq(("id1", 1), ("id2", 4), ("id3", 5)).toDF("id", "value")
    *  val spark = df.sparkSession
    *  spark.udf.register("simpleUDF", (v: Int) => v * v)
-   *  df.select($"id", callUDF("simpleUDF", $"value"))
+   *  df.select($"id", call_udf("simpleUDF", $"value"))
    * }}}
    *
    * @group udf_funcs
-   * @since 1.5.0
+   * @since 3.2.0
    */
   @scala.annotation.varargs
-  def callUDF(udfName: String, cols: Column*): Column = withExpr {
+  def call_udf(udfName: String, cols: Column*): Column = withExpr {
     UnresolvedFunction(udfName, cols.map(_.expr), isDistinct = false)
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
index 3bbdbb002cca8..28805d611ceb7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
@@ -16,21 +16,22 @@
  */
 package org.apache.spark.sql.internal
 
-import org.apache.spark.SparkConf
 import org.apache.spark.annotation.Unstable
 import org.apache.spark.sql.{ExperimentalMethods, SparkSession, UDFRegistration, _}
-import org.apache.spark.sql.catalyst.analysis.{Analyzer, FunctionRegistry, ResolveSessionCatalog}
+import org.apache.spark.sql.catalyst.analysis.{Analyzer, FunctionRegistry, ResolveSessionCatalog, TableFunctionRegistry}
 import org.apache.spark.sql.catalyst.catalog.SessionCatalog
 import org.apache.spark.sql.catalyst.optimizer.Optimizer
 import org.apache.spark.sql.catalyst.parser.ParserInterface
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.connector.catalog.CatalogManager
-import org.apache.spark.sql.execution.{ColumnarRule, QueryExecution, SparkOptimizer, SparkPlanner, SparkSqlParser}
+import org.apache.spark.sql.execution.{ColumnarRule, QueryExecution, SparkOptimizer, SparkPlan, SparkPlanner, SparkSqlParser}
+import org.apache.spark.sql.execution.aggregate.ResolveEncodersInScalaAgg
 import org.apache.spark.sql.execution.analysis.DetectAmbiguousSelfJoin
 import org.apache.spark.sql.execution.command.CommandCheck
 import org.apache.spark.sql.execution.datasources._
 import org.apache.spark.sql.execution.datasources.v2.{TableCapabilityCheck, V2SessionCatalog}
+import org.apache.spark.sql.execution.streaming.ResolveWriteToStream
 import org.apache.spark.sql.streaming.StreamingQueryManager
 import org.apache.spark.sql.util.ExecutionListenerManager
 
@@ -56,7 +57,7 @@ import org.apache.spark.sql.util.ExecutionListenerManager
 @Unstable
 abstract class BaseSessionStateBuilder(
     val session: SparkSession,
-    val parentState: Option[SessionState] = None) {
+    val parentState: Option[SessionState]) {
   type NewBuilder = (SparkSession, Option[SessionState]) => BaseSessionStateBuilder
 
   /**
@@ -71,15 +72,6 @@ abstract class BaseSessionStateBuilder(
    */
   protected def extensions: SparkSessionExtensions = session.extensions
 
-  /**
-   * Extract entries from `SparkConf` and put them in the `SQLConf`
-   */
-  protected def mergeSparkConf(sqlConf: SQLConf, sparkConf: SparkConf): Unit = {
-    sparkConf.getAll.foreach { case (k, v) =>
-      sqlConf.setConfString(k, v)
-    }
-  }
-
   /**
    * SQL-specific key-value configurations.
    *
@@ -90,12 +82,15 @@ abstract class BaseSessionStateBuilder(
     parentState.map { s =>
       val cloned = s.conf.clone()
       if (session.sparkContext.conf.get(StaticSQLConf.SQL_LEGACY_SESSION_INIT_WITH_DEFAULTS)) {
-        mergeSparkConf(cloned, session.sparkContext.conf)
+        SQLConf.mergeSparkConf(cloned, session.sparkContext.conf)
       }
       cloned
     }.getOrElse {
       val conf = new SQLConf
-      mergeSparkConf(conf, session.sparkContext.conf)
+      SQLConf.mergeSparkConf(conf, session.sharedState.conf)
+      // the later added configs to spark conf shall be respected too
+      SQLConf.mergeNonStaticSQLConfigs(conf, session.sparkContext.conf.getAll.toMap)
+      SQLConf.mergeNonStaticSQLConfigs(conf, session.initialSessionOptions)
       conf
     }
   }
@@ -110,6 +105,16 @@ abstract class BaseSessionStateBuilder(
       .getOrElse(extensions.registerFunctions(FunctionRegistry.builtin.clone()))
   }
 
+  /**
+   * Internal catalog managing functions registered by the user.
+   *
+   * This either gets cloned from a pre-existing version or cloned from the built-in registry.
+   */
+  protected lazy val tableFunctionRegistry: TableFunctionRegistry = {
+    parentState.map(_.tableFunctionRegistry.clone())
+      .getOrElse(extensions.registerTableFunctions(TableFunctionRegistry.builtin.clone()))
+  }
+
   /**
    * Experimental methods that can be used to define custom optimization rules and custom planning
    * strategies.
@@ -126,7 +131,7 @@ abstract class BaseSessionStateBuilder(
    * Note: this depends on the `conf` field.
    */
   protected lazy val sqlParser: ParserInterface = {
-    extensions.buildParser(session, new SparkSqlParser(conf))
+    extensions.buildParser(session, new SparkSqlParser())
   }
 
   /**
@@ -145,7 +150,7 @@ abstract class BaseSessionStateBuilder(
       () => session.sharedState.externalCatalog,
       () => session.sharedState.globalTempViewManager,
       functionRegistry,
-      conf,
+      tableFunctionRegistry,
       SessionState.newHadoopConf(session.sparkContext.hadoopConfiguration, conf),
       sqlParser,
       resourceLoader)
@@ -153,9 +158,9 @@ abstract class BaseSessionStateBuilder(
     catalog
   }
 
-  protected lazy val v2SessionCatalog = new V2SessionCatalog(catalog, conf)
+  protected lazy val v2SessionCatalog = new V2SessionCatalog(catalog)
 
-  protected lazy val catalogManager = new CatalogManager(conf, v2SessionCatalog, catalog)
+  protected lazy val catalogManager = new CatalogManager(v2SessionCatalog, catalog)
 
   /**
    * Interface exposed to the user for registering user-defined functions.
@@ -170,20 +175,21 @@ abstract class BaseSessionStateBuilder(
    *
    * Note: this depends on the `conf` and `catalog` fields.
    */
-  protected def analyzer: Analyzer = new Analyzer(catalogManager, conf) {
+  protected def analyzer: Analyzer = new Analyzer(catalogManager) {
     override val extendedResolutionRules: Seq[Rule[LogicalPlan]] =
       new FindDataSourceTable(session) +:
         new ResolveSQLOnFile(session) +:
         new FallBackFileSourceV2(session) +:
-        new ResolveSessionCatalog(
-          catalogManager, conf, catalog.isTempView, catalog.isTempFunction) +:
+        ResolveEncodersInScalaAgg +:
+        new ResolveSessionCatalog(catalogManager) +:
+        ResolveWriteToStream +:
         customResolutionRules
 
     override val postHocResolutionRules: Seq[Rule[LogicalPlan]] =
-      new DetectAmbiguousSelfJoin(conf) +:
+      DetectAmbiguousSelfJoin +:
         PreprocessTableCreation(session) +:
-        PreprocessTableInsertion(conf) +:
-        DataSourceAnalysis(conf) +:
+        PreprocessTableInsertion +:
+        DataSourceAnalysis +:
         customPostHocResolutionRules
 
     override val extendedCheckRules: Seq[LogicalPlan => Unit] =
@@ -191,7 +197,7 @@ abstract class BaseSessionStateBuilder(
         PreReadCheck +:
         HiveOnlyCheck +:
         TableCapabilityCheck +:
-        CommandCheck(conf) +:
+        CommandCheck +:
         customCheckRules
   }
 
@@ -235,6 +241,9 @@ abstract class BaseSessionStateBuilder(
       override def earlyScanPushDownRules: Seq[Rule[LogicalPlan]] =
         super.earlyScanPushDownRules ++ customEarlyScanPushDownRules
 
+      override def preCBORules: Seq[Rule[LogicalPlan]] =
+        super.preCBORules ++ customPreCBORules
+
       override def extendedOperatorOptimizationRules: Seq[Rule[LogicalPlan]] =
         super.extendedOperatorOptimizationRules ++ customOperatorOptimizationRules
     }
@@ -258,13 +267,23 @@ abstract class BaseSessionStateBuilder(
    */
   protected def customEarlyScanPushDownRules: Seq[Rule[LogicalPlan]] = Nil
 
+  /**
+   * Custom rules for rewriting plans after operator optimization and before CBO.
+   * Prefer overriding this instead of creating your own Optimizer.
+   *
+   * Note that this may NOT depend on the `optimizer` function.
+   */
+  protected def customPreCBORules: Seq[Rule[LogicalPlan]] = {
+    extensions.buildPreCBORules(session)
+  }
+
   /**
    * Planner that converts optimized logical plans to physical plans.
    *
    * Note: this depends on the `conf` and `experimentalMethods` fields.
    */
   protected def planner: SparkPlanner = {
-    new SparkPlanner(session, conf, experimentalMethods) {
+    new SparkPlanner(session, experimentalMethods) {
       override def extraPlanningStrategies: Seq[Strategy] =
         super.extraPlanningStrategies ++ customPlanningStrategies
     }
@@ -284,6 +303,10 @@ abstract class BaseSessionStateBuilder(
     extensions.buildColumnarRules(session)
   }
 
+  protected def queryStagePrepRules: Seq[Rule[SparkPlan]] = {
+    extensions.buildQueryStagePrepRules(session)
+  }
+
   /**
    * Create a query execution object.
    */
@@ -324,6 +347,7 @@ abstract class BaseSessionStateBuilder(
       conf,
       experimentalMethods,
       functionRegistry,
+      tableFunctionRegistry,
       udfRegistration,
       () => catalog,
       sqlParser,
@@ -335,7 +359,8 @@ abstract class BaseSessionStateBuilder(
       () => resourceLoader,
       createQueryExecution,
       createClone,
-      columnarRules)
+      columnarRules,
+      queryStagePrepRules)
   }
 }
 
@@ -350,7 +375,7 @@ private[sql] trait WithTestConf { self: BaseSessionStateBuilder =>
     parentState.map { s =>
       val cloned = s.conf.clone()
       if (session.sparkContext.conf.get(StaticSQLConf.SQL_LEGACY_SESSION_INIT_WITH_DEFAULTS)) {
-        mergeSparkConf(conf, session.sparkContext.conf)
+        SQLConf.mergeSparkConf(conf, session.sparkContext.conf)
       }
       cloned
     }.getOrElse {
@@ -362,7 +387,7 @@ private[sql] trait WithTestConf { self: BaseSessionStateBuilder =>
           overrideConfigurations.foreach { case (key, value) => setConfString(key, value) }
         }
       }
-      mergeSparkConf(conf, session.sparkContext.conf)
+      SQLConf.mergeSparkConf(conf, session.sparkContext.conf)
       conf
     }
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala
index 7ca9fbb40d9f5..30b391bd0e092 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala
@@ -23,10 +23,11 @@ import scala.util.control.NonFatal
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalog.{Catalog, Column, Database, Function, Table}
 import org.apache.spark.sql.catalyst.{DefinedByConstructorParams, FunctionIdentifier, TableIdentifier}
+import org.apache.spark.sql.catalyst.analysis.UnresolvedTable
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
-import org.apache.spark.sql.execution.command.AlterTableRecoverPartitionsCommand
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, RecoverPartitions, SubqueryAlias, View}
+import org.apache.spark.sql.catalyst.util.CharVarcharUtils
 import org.apache.spark.sql.execution.datasources.{CreateTable, DataSource}
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.storage.StorageLevel
@@ -114,7 +115,7 @@ class CatalogImpl(sparkSession: SparkSession) extends Catalog {
     } catch {
       case NonFatal(_) => None
     }
-    val isTemp = sessionCatalog.isTemporaryTable(tableIdent)
+    val isTemp = sessionCatalog.isTempView(tableIdent)
     new Table(
       name = tableIdent.table,
       database = metadata.map(_.identifier.database).getOrElse(tableIdent.database).orNull,
@@ -181,7 +182,7 @@ class CatalogImpl(sparkSession: SparkSession) extends Catalog {
       new Column(
         name = c.name,
         description = c.getComment().orNull,
-        dataType = c.dataType.catalogString,
+        dataType = CharVarcharUtils.getRawType(c.metadata).getOrElse(c.dataType).catalogString,
         nullable = c.nullable,
         isPartition = partitionColumnNames.contains(c.name),
         isBucket = bucketColumnNames.contains(c.name))
@@ -256,7 +257,7 @@ class CatalogImpl(sparkSession: SparkSession) extends Catalog {
    */
   override def tableExists(dbName: String, tableName: String): Boolean = {
     val tableIdent = TableIdentifier(tableName, Option(dbName))
-    sessionCatalog.isTemporaryTable(tableIdent) || sessionCatalog.tableExists(tableIdent)
+    sessionCatalog.isTempView(tableIdent) || sessionCatalog.tableExists(tableIdent)
   }
 
   /**
@@ -313,6 +314,22 @@ class CatalogImpl(sparkSession: SparkSession) extends Catalog {
     createTable(tableName, source, new StructType, options)
   }
 
+  /**
+   * (Scala-specific)
+   * Creates a table based on the dataset in a data source and a set of options.
+   * Then, returns the corresponding DataFrame.
+   *
+   * @group ddl_ops
+   * @since 3.1.0
+   */
+  override def createTable(
+      tableName: String,
+      source: String,
+      description: String,
+      options: Map[String, String]): DataFrame = {
+    createTable(tableName, source, new StructType, description, options)
+  }
+
   /**
    * (Scala-specific)
    * Creates a table based on the dataset in a data source, a schema and a set of options.
@@ -326,6 +343,29 @@ class CatalogImpl(sparkSession: SparkSession) extends Catalog {
       source: String,
       schema: StructType,
       options: Map[String, String]): DataFrame = {
+    createTable(
+      tableName = tableName,
+      source = source,
+      schema = schema,
+      description = "",
+      options = options
+    )
+  }
+
+  /**
+   * (Scala-specific)
+   * Creates a table based on the dataset in a data source, a schema and a set of options.
+   * Then, returns the corresponding DataFrame.
+   *
+   * @group ddl_ops
+   * @since 3.1.0
+   */
+  override def createTable(
+      tableName: String,
+      source: String,
+      schema: StructType,
+      description: String,
+      options: Map[String, String]): DataFrame = {
     val tableIdent = sparkSession.sessionState.sqlParser.parseTableIdentifier(tableName)
     val storage = DataSource.buildStorageFormatFromOptions(options)
     val tableType = if (storage.locationUri.isDefined) {
@@ -338,7 +378,8 @@ class CatalogImpl(sparkSession: SparkSession) extends Catalog {
       tableType = tableType,
       storage = storage,
       schema = schema,
-      provider = Some(source)
+      provider = Some(source),
+      comment = { if (description.isEmpty) None else Some(description) }
     )
     val plan = CreateTable(tableDesc, SaveMode.ErrorIfExists, None)
     sparkSession.sessionState.executePlan(plan).toRdd
@@ -355,8 +396,7 @@ class CatalogImpl(sparkSession: SparkSession) extends Catalog {
    */
   override def dropTempView(viewName: String): Boolean = {
     sparkSession.sessionState.catalog.getTempView(viewName).exists { viewDef =>
-      sparkSession.sharedState.cacheManager.uncacheQuery(
-        sparkSession, viewDef, cascade = false)
+      uncacheView(viewDef)
       sessionCatalog.dropTempView(viewName)
     }
   }
@@ -371,12 +411,27 @@ class CatalogImpl(sparkSession: SparkSession) extends Catalog {
    */
   override def dropGlobalTempView(viewName: String): Boolean = {
     sparkSession.sessionState.catalog.getGlobalTempView(viewName).exists { viewDef =>
-      sparkSession.sharedState.cacheManager.uncacheQuery(
-        sparkSession, viewDef, cascade = false)
+      uncacheView(viewDef)
       sessionCatalog.dropGlobalTempView(viewName)
     }
   }
 
+  private def uncacheView(viewDef: View): Unit = {
+    try {
+      // If view text is defined, it means we are not storing analyzed logical plan for the view
+      // and instead its behavior follows that of a permanent view (see SPARK-33142 for more
+      // details). Therefore, when uncaching the view we should also do in a cascade fashion, the
+      // same way as how a permanent view is handled. This also avoids a potential issue where a
+      // dependent view becomes invalid because of the above while its data is still cached.
+      val viewText = viewDef.desc.viewText
+      val plan = sparkSession.sessionState.executePlan(viewDef)
+      sparkSession.sharedState.cacheManager.uncacheQuery(
+        sparkSession, plan.analyzed, cascade = viewText.isDefined)
+    } catch {
+      case NonFatal(_) => // ignore
+    }
+  }
+
   /**
    * Recovers all the partitions in the directory of a table and update the catalog.
    * Only works with a partitioned table, and not a temporary view.
@@ -388,9 +443,10 @@ class CatalogImpl(sparkSession: SparkSession) extends Catalog {
    * @since 2.1.1
    */
   override def recoverPartitions(tableName: String): Unit = {
-    val tableIdent = sparkSession.sessionState.sqlParser.parseTableIdentifier(tableName)
+    val multiPartIdent = sparkSession.sessionState.sqlParser.parseMultipartIdentifier(tableName)
     sparkSession.sessionState.executePlan(
-      AlterTableRecoverPartitionsCommand(tableIdent)).toRdd
+      RecoverPartitions(
+        UnresolvedTable(multiPartIdent, "recoverPartitions()", None))).toRdd
   }
 
   /**
@@ -432,8 +488,10 @@ class CatalogImpl(sparkSession: SparkSession) extends Catalog {
    */
   override def uncacheTable(tableName: String): Unit = {
     val tableIdent = sparkSession.sessionState.sqlParser.parseTableIdentifier(tableName)
-    val cascade = !sessionCatalog.isTemporaryTable(tableIdent)
-    sparkSession.sharedState.cacheManager.uncacheQuery(sparkSession.table(tableName), cascade)
+    sessionCatalog.lookupTempView(tableIdent).map(uncacheView).getOrElse {
+      sparkSession.sharedState.cacheManager.uncacheQuery(sparkSession.table(tableName),
+        cascade = true)
+    }
   }
 
   /**
@@ -457,43 +515,47 @@ class CatalogImpl(sparkSession: SparkSession) extends Catalog {
   }
 
   /**
-   * Invalidates and refreshes all the cached data and metadata of the given table or view.
-   * For Hive metastore table, the metadata is refreshed. For data source tables, the schema will
-   * not be inferred and refreshed.
+   * The method fully refreshes a table or view with the given name including:
+   *   1. The relation cache in the session catalog. The method removes table entry from the cache.
+   *   2. The file indexes of all relations used by the given view.
+   *   3. Table/View schema in the Hive Metastore if the SQL config
+   *      `spark.sql.hive.caseSensitiveInferenceMode` is set to `INFER_AND_SAVE`.
+   *   4. Cached data of the given table or view, and all its dependents that refer to it.
+   *      Existing cached data will be cleared and the cache will be lazily filled when
+   *      the next time the table/view or the dependents are accessed.
    *
-   * If this table is cached as an InMemoryRelation, drop the original cached version and make the
-   * new version cached lazily.
+   * The method does not do:
+   *   - schema inference for file source tables
+   *   - statistics update
+   *
+   * The method is supposed to be used in all cases when need to refresh table/view data
+   * and meta-data.
    *
    * @group cachemgmt
    * @since 2.0.0
    */
   override def refreshTable(tableName: String): Unit = {
     val tableIdent = sparkSession.sessionState.sqlParser.parseTableIdentifier(tableName)
-    val tableMetadata = sessionCatalog.getTempViewOrPermanentTableMetadata(tableIdent)
-    val table = sparkSession.table(tableIdent)
+    val relation = sparkSession.table(tableIdent).queryExecution.analyzed
 
-    if (tableMetadata.tableType == CatalogTableType.VIEW) {
-      // Temp or persistent views: refresh (or invalidate) any metadata/data cached
-      // in the plan recursively.
-      table.queryExecution.analyzed.refresh()
-    } else {
-      // Non-temp tables: refresh the metadata cache.
-      sessionCatalog.refreshTable(tableIdent)
-    }
+    relation.refresh()
 
-    // If this table is cached as an InMemoryRelation, drop the original
-    // cached version and make the new version cached lazily.
-    val cache = sparkSession.sharedState.cacheManager.lookupCachedData(table)
-    if (cache.nonEmpty) {
-      // save the cache name and cache level for recreation
-      val cacheName = cache.get.cachedRepresentation.cacheBuilder.tableName
-      val cacheLevel = cache.get.cachedRepresentation.cacheBuilder.storageLevel
-
-      // uncache the logical plan.
-      sparkSession.sharedState.cacheManager.uncacheQuery(table, cascade = true)
+    // Temporary and global temporary views are not supposed to be put into the relation cache
+    // since they are tracked separately.
+    if (!sessionCatalog.isTempView(tableIdent)) {
+      sessionCatalog.invalidateCachedTable(tableIdent)
+    }
 
-      // recache with the same name and cache level.
-      sparkSession.sharedState.cacheManager.cacheQuery(table, cacheName, cacheLevel)
+    // Re-caches the logical plan of the relation.
+    // Note this is a no-op for the relation itself if it's not cached, but will clear all
+    // caches referencing this relation. If this relation is cached as an InMemoryRelation,
+    // this will clear the relation cache and caches of all its dependents.
+    relation match {
+      case SubqueryAlias(_, relationPlan) =>
+        sparkSession.sharedState.cacheManager.recacheByPlan(sparkSession, relationPlan)
+      case _ =>
+        throw new AnalysisException(
+          s"Unexpected type ${relation.getClass.getCanonicalName} of the relation $tableName")
     }
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala
index 64b7e7fe7923a..cfcfeabbf1f6e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala
@@ -65,7 +65,7 @@ object HiveSerDe {
         outputFormat = Option("org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat"),
         serde = Option("org.apache.hadoop.hive.serde2.avro.AvroSerDe")))
 
-  // `HiveSerDe` in `serdeMap` should be dintinct.
+  // `HiveSerDe` in `serdeMap` should be distinct.
   val serdeInverseMap: Map[HiveSerDe, String] = serdeMap.flatMap {
     case ("sequencefile", _) => None
     case ("rcfile", _) => None
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SessionState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SessionState.scala
index abd1250628539..79fbca6338905 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SessionState.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SessionState.scala
@@ -18,21 +18,24 @@
 package org.apache.spark.sql.internal
 
 import java.io.File
+import java.net.URI
 
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.Path
 
 import org.apache.spark.annotation.Unstable
 import org.apache.spark.sql._
-import org.apache.spark.sql.catalyst.analysis.{Analyzer, FunctionRegistry}
+import org.apache.spark.sql.catalyst.analysis.{Analyzer, FunctionRegistry, TableFunctionRegistry}
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.optimizer.Optimizer
 import org.apache.spark.sql.catalyst.parser.ParserInterface
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.connector.catalog.CatalogManager
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.streaming.StreamingQueryManager
-import org.apache.spark.sql.util.{ExecutionListenerManager, QueryExecutionListener}
+import org.apache.spark.sql.util.ExecutionListenerManager
+import org.apache.spark.util.{DependencyUtils, Utils}
 
 /**
  * A class that holds all session-specific state in a given [[SparkSession]].
@@ -51,7 +54,8 @@ import org.apache.spark.sql.util.{ExecutionListenerManager, QueryExecutionListen
  * @param planner Planner that converts optimized logical plans to physical plans.
  * @param streamingQueryManagerBuilder A function to create a streaming query manager to
  *                                     start and stop streaming queries.
- * @param listenerManager Interface to register custom [[QueryExecutionListener]]s.
+ * @param listenerManager Interface to register custominternal/SessionState.scala
+ *                        [[org.apache.spark.sql.util.QueryExecutionListener]]s.
  * @param resourceLoaderBuilder a function to create a session shared resource loader to load JARs,
  *                              files, etc.
  * @param createQueryExecution Function used to create QueryExecution objects.
@@ -62,6 +66,7 @@ private[sql] class SessionState(
     val conf: SQLConf,
     val experimentalMethods: ExperimentalMethods,
     val functionRegistry: FunctionRegistry,
+    val tableFunctionRegistry: TableFunctionRegistry,
     val udfRegistration: UDFRegistration,
     catalogBuilder: () => SessionCatalog,
     val sqlParser: ParserInterface,
@@ -73,7 +78,8 @@ private[sql] class SessionState(
     resourceLoaderBuilder: () => SessionResourceLoader,
     createQueryExecution: LogicalPlan => QueryExecution,
     createClone: (SparkSession, SessionState) => SessionState,
-    val columnarRules: Seq[ColumnarRule]) {
+    val columnarRules: Seq[ColumnarRule],
+    val queryStagePrepRules: Seq[Rule[SparkPlan]]) {
 
   // The following fields are lazy to avoid creating the Hive client when creating SessionState.
   lazy val catalog: SessionCatalog = catalogBuilder()
@@ -114,10 +120,6 @@ private[sql] class SessionState(
   // ------------------------------------------------------
 
   def executePlan(plan: LogicalPlan): QueryExecution = createQueryExecution(plan)
-
-  def refreshTable(tableName: String): Unit = {
-    catalog.refreshTable(sqlParser.parseTableIdentifier(tableName))
-  }
 }
 
 private[sql] object SessionState {
@@ -134,7 +136,7 @@ private[sql] object SessionState {
 @Unstable
 class SessionStateBuilder(
     session: SparkSession,
-    parentState: Option[SessionState] = None)
+    parentState: Option[SessionState])
   extends BaseSessionStateBuilder(session, parentState) {
   override protected def newBuilder: NewBuilder = new SessionStateBuilder(_, _)
 }
@@ -148,10 +150,14 @@ class SessionResourceLoader(session: SparkSession) extends FunctionResourceLoade
     resource.resourceType match {
       case JarResource => addJar(resource.uri)
       case FileResource => session.sparkContext.addFile(resource.uri)
-      case ArchiveResource =>
-        throw new AnalysisException(
-          "Archive is not allowed to be loaded. If YARN mode is used, " +
-            "please use --archives options while calling spark-submit.")
+      case ArchiveResource => session.sparkContext.addArchive(resource.uri)
+    }
+  }
+
+  def resolveJars(path: URI): Seq[String] = {
+    path.getScheme match {
+      case "ivy" => DependencyUtils.resolveMavenDependencies(path)
+      case _ => path.toString :: Nil
     }
   }
 
@@ -163,16 +169,19 @@ class SessionResourceLoader(session: SparkSession) extends FunctionResourceLoade
    * [[SessionState]].
    */
   def addJar(path: String): Unit = {
-    session.sparkContext.addJar(path)
-    val uri = new Path(path).toUri
-    val jarURL = if (uri.getScheme == null) {
-      // `path` is a local file path without a URL scheme
-      new File(path).toURI.toURL
-    } else {
-      // `path` is a URL with a scheme
-      uri.toURL
+    val uri = Utils.resolveURI(path)
+    resolveJars(uri).foreach { p =>
+      session.sparkContext.addJar(p)
+      val uri = new Path(p).toUri
+      val jarURL = if (uri.getScheme == null) {
+        // `path` is a local file path without a URL scheme
+        new File(p).toURI.toURL
+      } else {
+        // `path` is a URL with a scheme
+        uri.toURL
+      }
+      session.sharedState.jarClassLoader.addURL(jarURL)
     }
-    session.sharedState.jarClassLoader.addURL(jarURL)
     Thread.currentThread().setContextClassLoader(session.sharedState.jarClassLoader)
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
index ce4385d88f1e9..3f676bec9ff0b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
@@ -22,22 +22,19 @@ import java.util.UUID
 import java.util.concurrent.ConcurrentHashMap
 import javax.annotation.concurrent.GuardedBy
 
-import scala.collection.JavaConverters._
 import scala.reflect.ClassTag
 import scala.util.control.NonFatal
 
 import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.FsUrlStreamHandlerFactory
+import org.apache.hadoop.fs.{FsUrlStreamHandlerFactory, Path}
 
 import org.apache.spark.{SparkConf, SparkContext, SparkException}
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.SQLContext
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.execution.CacheManager
 import org.apache.spark.sql.execution.streaming.StreamExecution
-import org.apache.spark.sql.execution.ui.{SQLAppStatusListener, SQLAppStatusStore, SQLTab}
+import org.apache.spark.sql.execution.ui.{SQLAppStatusListener, SQLAppStatusStore, SQLTab, StreamingQueryStatusStore}
 import org.apache.spark.sql.internal.StaticSQLConf._
-import org.apache.spark.sql.streaming.StreamingQueryListener
 import org.apache.spark.sql.streaming.ui.{StreamingQueryStatusListener, StreamingQueryTab}
 import org.apache.spark.status.ElementTrackingStore
 import org.apache.spark.util.Utils
@@ -55,25 +52,34 @@ private[sql] class SharedState(
 
   SharedState.setFsUrlStreamHandlerFactory(sparkContext.conf, sparkContext.hadoopConfiguration)
 
-  private val (conf, hadoopConf) = {
-    // Load hive-site.xml into hadoopConf and determine the warehouse path which will be set into
-    // both spark conf and hadoop conf avoiding be affected by any SparkSession level options
-    SharedState.loadHiveConfFile(sparkContext.conf, sparkContext.hadoopConfiguration)
+  private[sql] val (conf, hadoopConf) = {
+    val warehousePath = SharedState.resolveWarehousePath(
+      sparkContext.conf, sparkContext.hadoopConfiguration, initialConfigs)
+
     val confClone = sparkContext.conf.clone()
     val hadoopConfClone = new Configuration(sparkContext.hadoopConfiguration)
+    // Extract entries from `SparkConf` and put them in the Hadoop conf.
+    confClone.getAll.foreach { case (k, v) =>
+      if (v ne null) hadoopConfClone.set(k, v)
+    }
     // If `SparkSession` is instantiated using an existing `SparkContext` instance and no existing
     // `SharedState`, all `SparkSession` level configurations have higher priority to generate a
     // `SharedState` instance. This will be done only once then shared across `SparkSession`s
     initialConfigs.foreach {
-      case (k, _)  if k == "hive.metastore.warehouse.dir" || k == WAREHOUSE_PATH.key =>
-        logWarning(s"Not allowing to set ${WAREHOUSE_PATH.key} or hive.metastore.warehouse.dir " +
-          s"in SparkSession's options, it should be set statically for cross-session usages")
-      case (k, v) =>
-        logDebug(s"Applying initial SparkSession options to SparkConf/HadoopConf: $k -> $v")
+      // We have resolved the warehouse path and should not set warehouse conf here.
+      case (k, _) if k == WAREHOUSE_PATH.key || k == SharedState.HIVE_WAREHOUSE_CONF_NAME =>
+      case (k, v) if SQLConf.staticConfKeys.contains(k) =>
+        logDebug(s"Applying static initial session options to SparkConf: $k -> $v")
         confClone.set(k, v)
+      case (k, v) =>
+        logDebug(s"Applying other initial session options to HadoopConf: $k -> $v")
         hadoopConfClone.set(k, v)
-
     }
+    val qualified = SharedState.qualifyWarehousePath(hadoopConfClone, warehousePath)
+    // Set warehouse path in the SparkConf and Hadoop conf, so that it's application wide reachable
+    // from `SparkContext`.
+    SharedState.setWarehousePathConf(sparkContext.conf, sparkContext.hadoopConfiguration, qualified)
+    SharedState.setWarehousePathConf(confClone, hadoopConfClone, qualified)
     (confClone, hadoopConfClone)
   }
 
@@ -112,9 +118,9 @@ private[sql] class SharedState(
   lazy val streamingQueryStatusListener: Option[StreamingQueryStatusListener] = {
     sparkContext.ui.flatMap { ui =>
       if (conf.get(STREAMING_UI_ENABLED)) {
-        val statusListener = new StreamingQueryStatusListener(conf)
-        new StreamingQueryTab(statusListener, ui)
-        Some(statusListener)
+        val kvStore = sparkContext.statusStore.store.asInstanceOf[ElementTrackingStore]
+        new StreamingQueryTab(new StreamingQueryStatusStore(kvStore), ui)
+        Some(new StreamingQueryStatusListener(conf, kvStore))
       } else {
         None
       }
@@ -221,45 +227,64 @@ object SharedState extends Logging {
     }
   }
 
+  private val HIVE_WAREHOUSE_CONF_NAME = "hive.metastore.warehouse.dir"
+
   /**
-   * Load hive-site.xml into hadoopConf and determine the warehouse path we want to use, based on
-   * the config from both hive and Spark SQL. Finally set the warehouse config value to sparkConf.
+   * Determine the warehouse path using the key `spark.sql.warehouse.dir` in the [[SparkConf]]
+   * or the initial options from the very first created SparkSession instance, and
+   * `hive.metastore.warehouse.dir` in hadoop [[Configuration]].
+   * The priority order is:
+   * s.s.w.d in initialConfigs
+   *   > s.s.w.d in spark conf (user specified)
+   *   > h.m.w.d in hadoop conf (user specified)
+   *   > s.s.w.d in spark conf (default)
+   *
+   * @return the resolved warehouse path.
    */
-  def loadHiveConfFile(
+  def resolveWarehousePath(
       sparkConf: SparkConf,
-      hadoopConf: Configuration): Unit = {
-    val hiveWarehouseKey = "hive.metastore.warehouse.dir"
-    val configFile = Utils.getContextOrSparkClassLoader.getResource("hive-site.xml")
-    if (configFile != null) {
-      logInfo(s"loading hive config file: $configFile")
-      val hadoopConfTemp = new Configuration()
-      hadoopConfTemp.addResource(configFile)
-      hadoopConfTemp.asScala.foreach { entry =>
-        hadoopConf.setIfUnset(entry.getKey, entry.getValue)
-      }
+      hadoopConf: Configuration,
+      initialConfigs: scala.collection.Map[String, String] = Map.empty): String = {
+    val sparkWarehouseOption =
+      initialConfigs.get(WAREHOUSE_PATH.key).orElse(sparkConf.getOption(WAREHOUSE_PATH.key))
+    if (initialConfigs.contains(HIVE_WAREHOUSE_CONF_NAME)) {
+      logWarning(s"Not allowing to set $HIVE_WAREHOUSE_CONF_NAME in SparkSession's " +
+        s"options, please use ${WAREHOUSE_PATH.key} to set statically for cross-session usages")
     }
     // hive.metastore.warehouse.dir only stay in hadoopConf
-    sparkConf.remove(hiveWarehouseKey)
+    sparkConf.remove(HIVE_WAREHOUSE_CONF_NAME)
     // Set the Hive metastore warehouse path to the one we use
-    val hiveWarehouseDir = hadoopConf.get(hiveWarehouseKey)
-    val warehousePath = if (hiveWarehouseDir != null && !sparkConf.contains(WAREHOUSE_PATH.key)) {
+    val hiveWarehouseDir = hadoopConf.get(HIVE_WAREHOUSE_CONF_NAME)
+    if (hiveWarehouseDir != null && sparkWarehouseOption.isEmpty) {
       // If hive.metastore.warehouse.dir is set and spark.sql.warehouse.dir is not set,
       // we will respect the value of hive.metastore.warehouse.dir.
-      sparkConf.set(WAREHOUSE_PATH.key, hiveWarehouseDir)
-      logInfo(s"${WAREHOUSE_PATH.key} is not set, but $hiveWarehouseKey is set. Setting" +
-        s" ${WAREHOUSE_PATH.key} to the value of $hiveWarehouseKey ('$hiveWarehouseDir').")
+      logInfo(s"${WAREHOUSE_PATH.key} is not set, but $HIVE_WAREHOUSE_CONF_NAME is set. " +
+        s"Setting ${WAREHOUSE_PATH.key} to the value of $HIVE_WAREHOUSE_CONF_NAME.")
       hiveWarehouseDir
     } else {
       // If spark.sql.warehouse.dir is set, we will override hive.metastore.warehouse.dir using
       // the value of spark.sql.warehouse.dir.
       // When neither spark.sql.warehouse.dir nor hive.metastore.warehouse.dir is set
       // we will set hive.metastore.warehouse.dir to the default value of spark.sql.warehouse.dir.
-      val sparkWarehouseDir = sparkConf.get(WAREHOUSE_PATH)
-      logInfo(s"Setting $hiveWarehouseKey ('$hiveWarehouseDir') to the value of " +
-        s"${WAREHOUSE_PATH.key} ('$sparkWarehouseDir').")
-      hadoopConf.set(hiveWarehouseKey, sparkWarehouseDir)
+      val sparkWarehouseDir = sparkWarehouseOption.getOrElse(WAREHOUSE_PATH.defaultValueString)
+      logInfo(s"Setting $HIVE_WAREHOUSE_CONF_NAME ('$hiveWarehouseDir') to the value of " +
+        s"${WAREHOUSE_PATH.key}.")
       sparkWarehouseDir
     }
-    logInfo(s"Warehouse path is '$warehousePath'.")
+  }
+
+  def qualifyWarehousePath(hadoopConf: Configuration, warehousePath: String): String = {
+    val tempPath = new Path(warehousePath)
+    val qualified = tempPath.getFileSystem(hadoopConf).makeQualified(tempPath).toString
+    logInfo(s"Warehouse path is '$qualified'.")
+    qualified
+  }
+
+  def setWarehousePathConf(
+      sparkConf: SparkConf,
+      hadoopConf: Configuration,
+      warehousePath: String): Unit = {
+    sparkConf.set(WAREHOUSE_PATH.key, warehousePath)
+    hadoopConf.set(HIVE_WAREHOUSE_CONF_NAME, warehousePath)
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/VariableSubstitution.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/VariableSubstitution.scala
index 4e7c813be9922..248dfa107bc4b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/VariableSubstitution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/VariableSubstitution.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.internal
 
 import org.apache.spark.internal.config._
+import org.apache.spark.sql.catalyst.SQLConfHelper
 
 /**
  * A helper class that enables substitution using syntax like
@@ -25,7 +26,7 @@ import org.apache.spark.internal.config._
  *
  * Variable substitution is controlled by `SQLConf.variableSubstituteEnabled`.
  */
-class VariableSubstitution(conf: SQLConf) {
+class VariableSubstitution extends SQLConfHelper {
 
   private val provider = new ConfigProvider {
     override def get(key: String): Option[String] = Option(conf.getConfString(key, ""))
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/DB2Dialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/DB2Dialect.scala
index ab574df4557a5..0b394db5c8932 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/DB2Dialect.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/DB2Dialect.scala
@@ -51,4 +51,32 @@ private object DB2Dialect extends JdbcDialect {
   }
 
   override def isCascadingTruncateTable(): Option[Boolean] = Some(false)
+
+  // scalastyle:off line.size.limit
+  // See https://www.ibm.com/support/knowledgecenter/en/SSEPGG_11.5.0/com.ibm.db2.luw.sql.ref.doc/doc/r0000980.html
+  // scalastyle:on line.size.limit
+  override def renameTable(oldTable: String, newTable: String): String = {
+    s"RENAME TABLE $oldTable TO $newTable"
+  }
+
+  // scalastyle:off line.size.limit
+  // See https://www.ibm.com/support/knowledgecenter/en/SSEPGG_11.5.0/com.ibm.db2.luw.sql.ref.doc/doc/r0000888.html
+  // scalastyle:on line.size.limit
+  override def getUpdateColumnTypeQuery(
+      tableName: String,
+      columnName: String,
+      newDataType: String): String =
+    s"ALTER TABLE $tableName ALTER COLUMN ${quoteIdentifier(columnName)}" +
+      s" SET DATA TYPE $newDataType"
+
+  // scalastyle:off line.size.limit
+  // See https://www.ibm.com/support/knowledgecenter/en/SSEPGG_11.5.0/com.ibm.db2.luw.sql.ref.doc/doc/r0000888.html
+  // scalastyle:on line.size.limit
+  override def getUpdateColumnNullabilityQuery(
+      tableName: String,
+      columnName: String,
+      isNullable: Boolean): String = {
+    val nullable = if (isNullable) "DROP NOT NULL" else "SET NOT NULL"
+    s"ALTER TABLE $tableName ALTER COLUMN ${quoteIdentifier(columnName)} $nullable"
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/DerbyDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/DerbyDialect.scala
index d528d5a9fef5a..3a2c9a5428be0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/DerbyDialect.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/DerbyDialect.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.jdbc
 
-import java.sql.Types
+import java.sql.{SQLFeatureNotSupportedException, Types}
 import java.util.Locale
 
 import org.apache.spark.sql.types._
@@ -45,4 +45,15 @@ private object DerbyDialect extends JdbcDialect {
   }
 
   override def isCascadingTruncateTable(): Option[Boolean] = Some(false)
+
+  // See https://db.apache.org/derby/docs/10.5/ref/rrefsqljrenametablestatement.html
+  override def renameTable(oldTable: String, newTable: String): String = {
+    s"RENAME TABLE $oldTable TO $newTable"
+  }
+
+  // Derby currently doesn't support comment on table. Here is the ticket to add the support
+  // https://issues.apache.org/jira/browse/DERBY-7008
+  override def getTableCommentQuery(table: String, comment: String): String = {
+    throw new SQLFeatureNotSupportedException(s"comment on table is not supported")
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/H2Dialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/H2Dialect.scala
new file mode 100644
index 0000000000000..9c727957ffab8
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/H2Dialect.scala
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc
+
+import java.sql.SQLException
+import java.util.Locale
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.analysis.{NoSuchNamespaceException, NoSuchTableException, TableAlreadyExistsException}
+
+private object H2Dialect extends JdbcDialect {
+  override def canHandle(url: String): Boolean =
+    url.toLowerCase(Locale.ROOT).startsWith("jdbc:h2")
+
+  override def classifyException(message: String, e: Throwable): AnalysisException = {
+    if (e.isInstanceOf[SQLException]) {
+      // Error codes are from https://www.h2database.com/javadoc/org/h2/api/ErrorCode.html
+      e.asInstanceOf[SQLException].getErrorCode match {
+        // TABLE_OR_VIEW_ALREADY_EXISTS_1
+        case 42101 =>
+          throw new TableAlreadyExistsException(message, cause = Some(e))
+        // TABLE_OR_VIEW_NOT_FOUND_1
+        case 42102 =>
+          throw new NoSuchTableException(message, cause = Some(e))
+        // SCHEMA_NOT_FOUND_1
+        case 90079 =>
+          throw new NoSuchNamespaceException(message, cause = Some(e))
+        case _ =>
+      }
+    }
+    super.classifyException(message, e)
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcConnectionProvider.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcConnectionProvider.scala
new file mode 100644
index 0000000000000..1e8abca197025
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcConnectionProvider.scala
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc
+
+import java.sql.{Connection, Driver}
+
+import org.apache.spark.annotation.{DeveloperApi, Unstable}
+
+/**
+ * ::DeveloperApi::
+ * Connection provider which opens connection toward various databases (database specific instance
+ * needed). If any authentication required then it's the provider's responsibility to set all
+ * the parameters.
+ * Important to mention connection providers within a JVM used from multiple threads so adding
+ * internal state is not advised. If any state added then it must be synchronized properly.
+ *
+ * @since 3.1.0
+ */
+@DeveloperApi
+@Unstable
+abstract class JdbcConnectionProvider {
+  /**
+   * Name of the service to provide JDBC connections. This name should be unique. Spark will
+   * internally use this name to differentiate JDBC connection providers.
+   */
+  val name: String
+
+  /**
+   * Checks if this connection provider instance can handle the connection initiated by the driver.
+   * There must be exactly one active connection provider which can handle the connection for a
+   * specific driver. If this requirement doesn't met then `IllegalArgumentException`
+   * will be thrown by the provider framework.
+   *
+   * @param driver  Java driver which initiates the connection
+   * @param options Driver options which initiates the connection
+   * @return True if the connection provider can handle the driver with the given options.
+   */
+  def canHandle(driver: Driver, options: Map[String, String]): Boolean
+
+  /**
+   * Opens connection toward the database. Since global JVM security configuration change may needed
+   * this API is called synchronized by `SecurityConfigurationLock` to avoid race.
+   *
+   * @param driver  Java driver which initiates the connection
+   * @param options Driver options which initiates the connection
+   * @return a `Connection` object that represents a connection to the URL
+   */
+  def getConnection(driver: Driver, options: Map[String, String]): Connection
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala
index a0c6d20f36451..6c7217257a238 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala
@@ -18,10 +18,20 @@
 package org.apache.spark.sql.jdbc
 
 import java.sql.{Connection, Date, Timestamp}
+import java.time.{Instant, LocalDate}
+
+import scala.collection.mutable.ArrayBuilder
 
 import org.apache.commons.lang3.StringUtils
 
 import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.util.{DateFormatter, DateTimeUtils, TimestampFormatter}
+import org.apache.spark.sql.connector.catalog.TableChange
+import org.apache.spark.sql.connector.catalog.TableChange._
+import org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 /**
@@ -55,7 +65,7 @@ case class JdbcType(databaseTypeDefinition : String, jdbcNullType : Int)
  * for the given Catalyst type.
  */
 @DeveloperApi
-abstract class JdbcDialect extends Serializable {
+abstract class JdbcDialect extends Serializable with Logging{
   /**
    * Check if this dialect instance can handle a certain jdbc url.
    * @param url the jdbc url.
@@ -168,7 +178,14 @@ abstract class JdbcDialect extends Serializable {
   def compileValue(value: Any): Any = value match {
     case stringValue: String => s"'${escapeSql(stringValue)}'"
     case timestampValue: Timestamp => "'" + timestampValue + "'"
+    case timestampValue: Instant =>
+      val timestampFormatter = TimestampFormatter.getFractionFormatter(
+        DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone))
+      s"'${timestampFormatter.format(timestampValue)}'"
     case dateValue: Date => "'" + dateValue + "'"
+    case dateValue: LocalDate =>
+      val dateFormatter = DateFormatter(DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone))
+      s"'${dateFormatter.format(dateValue)}'"
     case arrayValue: Array[Any] => arrayValue.map(compileValue).mkString(", ")
     case _ => value
   }
@@ -180,6 +197,106 @@ abstract class JdbcDialect extends Serializable {
    * None: The behavior of TRUNCATE TABLE is unknown (default).
    */
   def isCascadingTruncateTable(): Option[Boolean] = None
+
+  /**
+   * Rename an existing table.
+   *
+   * @param oldTable The existing table.
+   * @param newTable New name of the table.
+   * @return The SQL statement to use for renaming the table.
+   */
+  def renameTable(oldTable: String, newTable: String): String = {
+    s"ALTER TABLE $oldTable RENAME TO $newTable"
+  }
+
+  /**
+   * Alter an existing table.
+   *
+   * @param tableName The name of the table to be altered.
+   * @param changes Changes to apply to the table.
+   * @return The SQL statements to use for altering the table.
+   */
+  def alterTable(
+      tableName: String,
+      changes: Seq[TableChange],
+      dbMajorVersion: Int): Array[String] = {
+    val updateClause = ArrayBuilder.make[String]
+    for (change <- changes) {
+      change match {
+        case add: AddColumn if add.fieldNames.length == 1 =>
+          val dataType = JdbcUtils.getJdbcType(add.dataType(), this).databaseTypeDefinition
+          val name = add.fieldNames
+          updateClause += getAddColumnQuery(tableName, name(0), dataType)
+        case rename: RenameColumn if rename.fieldNames.length == 1 =>
+          val name = rename.fieldNames
+          updateClause += getRenameColumnQuery(tableName, name(0), rename.newName, dbMajorVersion)
+        case delete: DeleteColumn if delete.fieldNames.length == 1 =>
+          val name = delete.fieldNames
+          updateClause += getDeleteColumnQuery(tableName, name(0))
+        case updateColumnType: UpdateColumnType if updateColumnType.fieldNames.length == 1 =>
+          val name = updateColumnType.fieldNames
+          val dataType = JdbcUtils.getJdbcType(updateColumnType.newDataType(), this)
+            .databaseTypeDefinition
+          updateClause += getUpdateColumnTypeQuery(tableName, name(0), dataType)
+        case updateNull: UpdateColumnNullability if updateNull.fieldNames.length == 1 =>
+          val name = updateNull.fieldNames
+          updateClause += getUpdateColumnNullabilityQuery(tableName, name(0), updateNull.nullable())
+        case _ =>
+          throw new AnalysisException(s"Unsupported TableChange $change in JDBC catalog.")
+      }
+    }
+    updateClause.result()
+  }
+
+  def getAddColumnQuery(tableName: String, columnName: String, dataType: String): String =
+    s"ALTER TABLE $tableName ADD COLUMN ${quoteIdentifier(columnName)} $dataType"
+
+  def getRenameColumnQuery(
+      tableName: String,
+      columnName: String,
+      newName: String,
+      dbMajorVersion: Int): String =
+    s"ALTER TABLE $tableName RENAME COLUMN ${quoteIdentifier(columnName)} TO" +
+      s" ${quoteIdentifier(newName)}"
+
+  def getDeleteColumnQuery(tableName: String, columnName: String): String =
+    s"ALTER TABLE $tableName DROP COLUMN ${quoteIdentifier(columnName)}"
+
+  def getUpdateColumnTypeQuery(
+      tableName: String,
+      columnName: String,
+      newDataType: String): String =
+    s"ALTER TABLE $tableName ALTER COLUMN ${quoteIdentifier(columnName)} $newDataType"
+
+  def getUpdateColumnNullabilityQuery(
+      tableName: String,
+      columnName: String,
+      isNullable: Boolean): String = {
+    val nullable = if (isNullable) "NULL" else "NOT NULL"
+    s"ALTER TABLE $tableName ALTER COLUMN ${quoteIdentifier(columnName)} SET $nullable"
+  }
+
+  def getTableCommentQuery(table: String, comment: String): String = {
+    s"COMMENT ON TABLE $table IS '$comment'"
+  }
+
+  def getSchemaCommentQuery(schema: String, comment: String): String = {
+    s"COMMENT ON SCHEMA ${quoteIdentifier(schema)} IS '$comment'"
+  }
+
+  def removeSchemaCommentQuery(schema: String): String = {
+    s"COMMENT ON SCHEMA ${quoteIdentifier(schema)} IS NULL"
+  }
+
+  /**
+   * Gets a dialect exception, classifies it and wraps it by `AnalysisException`.
+   * @param message The error message to be placed to the returned exception.
+   * @param e The dialect specific exception.
+   * @return `AnalysisException` or its sub-class.
+   */
+  def classifyException(message: String, e: Throwable): AnalysisException = {
+    new AnalysisException(message, cause = Some(e))
+  }
 }
 
 /**
@@ -224,6 +341,7 @@ object JdbcDialects {
   registerDialect(DerbyDialect)
   registerDialect(OracleDialect)
   registerDialect(TeradataDialect)
+  registerDialect(H2Dialect)
 
   /**
    * Fetch the JdbcDialect class corresponding to a given database url.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala
index 72284b5996201..f78022a950217 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.jdbc
 
+import java.sql.SQLFeatureNotSupportedException
 import java.util.Locale
 
 import org.apache.spark.sql.internal.SQLConf
@@ -25,6 +26,13 @@ import org.apache.spark.sql.types._
 
 private object MsSqlServerDialect extends JdbcDialect {
 
+  // Special JDBC types in Microsoft SQL Server.
+  // https://github.com/microsoft/mssql-jdbc/blob/v8.2.2/src/main/java/microsoft/sql/Types.java
+  private object SpecificTypes {
+    val GEOMETRY = -157
+    val GEOGRAPHY = -158
+  }
+
   override def canHandle(url: String): Boolean =
     url.toLowerCase(Locale.ROOT).startsWith("jdbc:sqlserver")
 
@@ -40,6 +48,7 @@ private object MsSqlServerDialect extends JdbcDialect {
         sqlType match {
           case java.sql.Types.SMALLINT => Some(ShortType)
           case java.sql.Types.REAL => Some(FloatType)
+          case SpecificTypes.GEOMETRY | SpecificTypes.GEOGRAPHY => Some(BinaryType)
           case _ => None
         }
       }
@@ -57,4 +66,56 @@ private object MsSqlServerDialect extends JdbcDialect {
   }
 
   override def isCascadingTruncateTable(): Option[Boolean] = Some(false)
+
+  // scalastyle:off line.size.limit
+  // See https://docs.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-rename-transact-sql?view=sql-server-ver15
+  // scalastyle:on line.size.limit
+  override def renameTable(oldTable: String, newTable: String): String = {
+    s"EXEC sp_rename $oldTable, $newTable"
+  }
+
+  // scalastyle:off line.size.limit
+  // see https://docs.microsoft.com/en-us/sql/relational-databases/tables/add-columns-to-a-table-database-engine?view=sql-server-ver15
+  // scalastyle:on line.size.limit
+  override def getAddColumnQuery(
+      tableName: String,
+      columnName: String,
+      dataType: String): String = {
+    s"ALTER TABLE $tableName ADD ${quoteIdentifier(columnName)} $dataType"
+  }
+
+  // scalastyle:off line.size.limit
+  // See https://docs.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-rename-transact-sql?view=sql-server-ver15
+  // scalastyle:on line.size.limit
+  override def getRenameColumnQuery(
+      tableName: String,
+      columnName: String,
+      newName: String,
+      dbMajorVersion: Int): String = {
+    s"EXEC sp_rename '$tableName.${quoteIdentifier(columnName)}'," +
+      s" ${quoteIdentifier(newName)}, 'COLUMN'"
+  }
+
+  // scalastyle:off line.size.limit
+  // see https://docs.microsoft.com/en-us/sql/t-sql/statements/alter-table-transact-sql?view=sql-server-ver15
+  // scalastyle:on line.size.limit
+  // require to have column data type to change the column nullability
+  // ALTER TABLE tbl_name ALTER COLUMN col_name datatype [NULL | NOT NULL]
+  // column_definition:
+  //    data_type [NOT NULL | NULL]
+  // We don't have column data type here, so we throw Exception for now
+  override def getUpdateColumnNullabilityQuery(
+      tableName: String,
+      columnName: String,
+      isNullable: Boolean): String = {
+    throw new SQLFeatureNotSupportedException(s"UpdateColumnNullability is not supported")
+  }
+
+  // scalastyle:off line.size.limit
+  // https://docs.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-addextendedproperty-transact-sql?redirectedfrom=MSDN&view=sql-server-ver15
+  // scalastyle:on line.size.limit
+  // need to use the stored procedure called sp_addextendedproperty to add comments to tables
+  override def getTableCommentQuery(table: String, comment: String): String = {
+    throw new SQLFeatureNotSupportedException(s"comment on table is not supported")
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala
index 24b31b14d9427..71bba6f1105ba 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.jdbc
 
-import java.sql.Types
+import java.sql.{SQLFeatureNotSupportedException, Types}
 import java.util.Locale
 
 import org.apache.spark.sql.types.{BooleanType, DataType, LongType, MetadataBuilder}
@@ -48,4 +48,50 @@ private case object MySQLDialect extends JdbcDialect {
   }
 
   override def isCascadingTruncateTable(): Option[Boolean] = Some(false)
+
+  // See https://dev.mysql.com/doc/refman/8.0/en/alter-table.html
+  override def getUpdateColumnTypeQuery(
+      tableName: String,
+      columnName: String,
+      newDataType: String): String = {
+    s"ALTER TABLE $tableName MODIFY COLUMN ${quoteIdentifier(columnName)} $newDataType"
+  }
+
+  // See Old Syntax: https://dev.mysql.com/doc/refman/5.6/en/alter-table.html
+  // According to https://dev.mysql.com/worklog/task/?id=10761 old syntax works for
+  // both versions of MySQL i.e. 5.x and 8.0
+  // The old syntax requires us to have type definition. Since we do not have type
+  // information, we throw the exception for old version.
+  override def getRenameColumnQuery(
+      tableName: String,
+      columnName: String,
+      newName: String,
+      dbMajorVersion: Int): String = {
+    if (dbMajorVersion >= 8) {
+      s"ALTER TABLE $tableName RENAME COLUMN ${quoteIdentifier(columnName)} TO" +
+        s" ${quoteIdentifier(newName)}"
+    } else {
+      throw new SQLFeatureNotSupportedException(
+        s"Rename column is only supported for MySQL version 8.0 and above.")
+    }
+  }
+
+  // See https://dev.mysql.com/doc/refman/8.0/en/alter-table.html
+  // require to have column data type to change the column nullability
+  // ALTER TABLE tbl_name MODIFY [COLUMN] col_name column_definition
+  // column_definition:
+  //    data_type [NOT NULL | NULL]
+  // e.g. ALTER TABLE t1 MODIFY b INT NOT NULL;
+  // We don't have column data type here, so throw Exception for now
+  override def getUpdateColumnNullabilityQuery(
+      tableName: String,
+      columnName: String,
+      isNullable: Boolean): String = {
+    throw new SQLFeatureNotSupportedException(s"UpdateColumnNullability is not supported")
+  }
+
+  // See https://dev.mysql.com/doc/refman/8.0/en/alter-table.html
+  override def getTableCommentQuery(table: String, comment: String): String = {
+    s"ALTER TABLE $table COMMENT = '$comment'"
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/OracleDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/OracleDialect.scala
index 4c0623729e00d..b741ece8dda9b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/OracleDialect.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/OracleDialect.scala
@@ -112,4 +112,26 @@ private case object OracleDialect extends JdbcDialect {
       case _ => s"TRUNCATE TABLE $table"
     }
   }
+
+  // see https://docs.oracle.com/cd/B28359_01/server.111/b28286/statements_3001.htm#SQLRF01001
+  override def getAddColumnQuery(
+      tableName: String,
+      columnName: String,
+      dataType: String): String =
+    s"ALTER TABLE $tableName ADD ${quoteIdentifier(columnName)} $dataType"
+
+  // see https://docs.oracle.com/cd/B28359_01/server.111/b28286/statements_3001.htm#SQLRF01001
+  override def getUpdateColumnTypeQuery(
+    tableName: String,
+    columnName: String,
+    newDataType: String): String =
+    s"ALTER TABLE $tableName MODIFY ${quoteIdentifier(columnName)} $newDataType"
+
+  override def getUpdateColumnNullabilityQuery(
+    tableName: String,
+    columnName: String,
+    isNullable: Boolean): String = {
+    val nullable = if (isNullable) "NULL" else "NOT NULL"
+    s"ALTER TABLE $tableName MODIFY ${quoteIdentifier(columnName)} $nullable"
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala
index c8d8a3392128e..3ce785ed844c5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala
@@ -35,8 +35,12 @@ private object PostgresDialect extends JdbcDialect {
       Some(FloatType)
     } else if (sqlType == Types.SMALLINT) {
       Some(ShortType)
-    } else if (sqlType == Types.BIT && typeName.equals("bit") && size != 1) {
+    } else if (sqlType == Types.BIT && typeName == "bit" && size != 1) {
       Some(BinaryType)
+    } else if (sqlType == Types.DOUBLE && typeName == "money") {
+      // money type seems to be broken but one workaround is to handle it as string.
+      // See SPARK-34333 and https://github.com/pgjdbc/pgjdbc/issues/100
+      Some(StringType)
     } else if (sqlType == Types.OTHER) {
       Some(StringType)
     } else if (sqlType == Types.ARRAY) {
@@ -56,8 +60,11 @@ private object PostgresDialect extends JdbcDialect {
     case "int4" => Some(IntegerType)
     case "int8" | "oid" => Some(LongType)
     case "float4" => Some(FloatType)
-    case "money" | "float8" => Some(DoubleType)
-    case "text" | "varchar" | "char" | "cidr" | "inet" | "json" | "jsonb" | "uuid" =>
+    case "float8" => Some(DoubleType)
+    case "text" | "varchar" | "char" | "bpchar" | "cidr" | "inet" | "json" | "jsonb" | "uuid" |
+         "xml" | "tsvector" | "tsquery" | "macaddr" | "macaddr8" | "txid_snapshot" | "point" |
+         "line" | "lseg" | "box" | "path" | "polygon" | "circle" | "pg_lsn" | "varbit" |
+         "interval" | "pg_snapshot" =>
       Some(StringType)
     case "bytea" => Some(BinaryType)
     case "timestamp" | "timestamptz" | "time" | "timetz" => Some(TimestampType)
@@ -66,6 +73,11 @@ private object PostgresDialect extends JdbcDialect {
     case "numeric" | "decimal" =>
       // SPARK-26538: handle numeric without explicit precision and scale.
       Some(DecimalType. SYSTEM_DEFAULT)
+    case "money" =>
+      // money[] type seems to be broken and difficult to handle.
+      // So this method returns None for now.
+      // See SPARK-34333 and https://github.com/pgjdbc/pgjdbc/issues/1405
+      None
     case _ => None
   }
 
@@ -126,4 +138,20 @@ private object PostgresDialect extends JdbcDialect {
     }
   }
 
+  // See https://www.postgresql.org/docs/12/sql-altertable.html
+  override def getUpdateColumnTypeQuery(
+      tableName: String,
+      columnName: String,
+      newDataType: String): String = {
+    s"ALTER TABLE $tableName ALTER COLUMN ${quoteIdentifier(columnName)} TYPE $newDataType"
+  }
+
+  // See https://www.postgresql.org/docs/12/sql-altertable.html
+  override def getUpdateColumnNullabilityQuery(
+      tableName: String,
+      columnName: String,
+      isNullable: Boolean): String = {
+    val nullable = if (isNullable) "DROP NOT NULL" else "SET NOT NULL"
+    s"ALTER TABLE $tableName ALTER COLUMN ${quoteIdentifier(columnName)} $nullable"
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/README.md b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/README.md
new file mode 100644
index 0000000000000..f8a4ae08f8fc0
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/README.md
@@ -0,0 +1,84 @@
+---
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+# JDBC Connection Provider Handling In Spark
+
+This document aims to explain and demystify JDBC connection providers as they are used by Spark
+and the usage or custom provider implementation is not obvious.
+
+## What are JDBC connection providers and why use them?
+
+Spark initially provided non-authenticated or user/password authenticated connections.
+This is quite insecure and must be avoided when possible.
+
+JDBC connection providers (CPs from now on) are making JDBC connections initiated by JDBC sources
+which can be a more secure alternative.
+
+Spark provides two ways to deal with stronger authentication:
+* Built-in CPs added which support kerberos authentication using `keytab` and `principal` (but only if the JDBC driver supports keytab)
+* `org.apache.spark.sql.jdbc.JdbcConnectionProvider` developer API added which allows developers
+  to implement any kind of database/use-case specific authentication method.
+
+## How JDBC connection providers loaded?
+
+CPs are loaded with service loader independently. So, if one CP is failed to load it has no
+effect on all other CPs.
+
+## How to disable JDBC connection providers?
+
+There are cases where the built-in CP doesn't provide the exact feature which needed
+so they can be turned off and can be replaced with custom implementation. All CPs must provide a `name`
+which must be unique. One can set the following configuration entry in `SparkConf` to turn off CPs:
+`spark.sql.sources.disabledJdbcConnProviderList=name1,name2`.
+
+## How a JDBC connection provider found when new connection initiated?
+
+When a Spark source initiates JDBC connection it looks for a CP which supports the included driver,
+the user just need to provide the `keytab` location and the `principal`. The `keytab` file must exist
+on each node where connection is initiated.
+
+CPs has a mandatory API which must be implemented:
+
+`def canHandle(driver: Driver, options: Map[String, String]): Boolean`
+
+If this function returns `true` then `Spark` considers the CP can handle the connection setup.
+Built-in CPs returning `true` in the following cases:
+* If the connection is not secure (no `keytab` or `principal` provided) then the `basic` named CP responds.
+* If the connection is secure (`keytab` and `principal` provided) then the database specific CP responds.
+  Database specific providers are checking the JDBC driver class name and the decision is made based
+  on that. For example `PostgresConnectionProvider` responds only when the driver class name is `org.postgresql.Driver`.
+
+Important to mention that exactly one CP must return `true` from `canHandle` for a particular connection
+request because otherwise `Spark` can't decide which CP need to be used to make the connection.
+Such cases exception is thrown and the data processing stops.
+
+## How to implement a custom JDBC connection provider?
+
+Spark provides an example CP in the examples project (which does nothing).
+There are basically 2 files:
+* `examples/src/main/scala/org/apache/spark/examples/sql/jdbc/ExampleJdbcConnectionProvider.scala`
+  which contains the main logic that can be further developed.
+* `examples/src/main/resources/META-INF/services/org.apache.spark.sql.jdbc.JdbcConnectionProvider`
+  which manifest file is used by service loader (this tells `Spark` that this CP needs to be loaded).
+
+Implementation considerations:
+* CPs are running in heavy multi-threaded environment and adding a state into a CP is not advised.
+  If any state added then it must be synchronized properly. It could cause quite some headache to
+  hunt down such issues.
+* Some of the CPs are modifying the JVM global security context so `getConnection` method is
+  synchronized by `org.apache.spark.security.SecurityConfigurationLock` to avoid race.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/TeradataDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/TeradataDialect.scala
index 552d7a484f3fd..58fe62cb6e088 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/TeradataDialect.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/TeradataDialect.scala
@@ -50,4 +50,9 @@ private case object TeradataDialect extends JdbcDialect {
       cascade: Option[Boolean] = isCascadingTruncateTable): String = {
     s"DELETE FROM $table ALL"
   }
+
+  // See https://docs.teradata.com/reader/scPHvjfglIlB8F70YliLAw/wysTNUMsP~0aGzksLCl1kg
+  override def renameTable(oldTable: String, newTable: String): String = {
+    s"RENAME TABLE $oldTable TO $newTable"
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/package.scala b/sql/core/src/main/scala/org/apache/spark/sql/package.scala
index c0397010acba3..022fecf1ae412 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/package.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/package.scala
@@ -60,4 +60,10 @@ package object sql {
    * values.
    */
   private[sql] val SPARK_LEGACY_DATETIME = "org.apache.spark.legacyDateTime"
+
+  /**
+   * Parquet file metadata key to indicate that the file with INT96 column type was written
+   * with rebasing.
+   */
+  private[sql] val SPARK_LEGACY_INT96 = "org.apache.spark.legacyINT96"
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
index 1d7e4d3059c68..1798f6e2c88bd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
@@ -23,13 +23,17 @@ import scala.collection.JavaConverters._
 
 import org.apache.spark.annotation.Evolving
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset, SparkSession}
+import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}
+import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
+import org.apache.spark.sql.catalyst.streaming.StreamingRelationV2
+import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, CharVarcharUtils}
 import org.apache.spark.sql.connector.catalog.{SupportsRead, TableProvider}
 import org.apache.spark.sql.connector.catalog.TableCapability._
+import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.execution.command.DDLUtils
 import org.apache.spark.sql.execution.datasources.DataSource
 import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Utils, FileDataSourceV2}
-import org.apache.spark.sql.execution.streaming.{StreamingRelation, StreamingRelationV2}
+import org.apache.spark.sql.execution.streaming.StreamingRelation
 import org.apache.spark.sql.sources.StreamSourceProvider
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
@@ -60,7 +64,10 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
    * @since 2.0.0
    */
   def schema(schema: StructType): DataStreamReader = {
-    this.userSpecifiedSchema = Option(schema)
+    if (schema != null) {
+      val replaced = CharVarcharUtils.failIfHasCharVarchar(schema).asInstanceOf[StructType]
+      this.userSpecifiedSchema = Option(replaced)
+    }
     this
   }
 
@@ -72,8 +79,7 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
    * @since 2.3.0
    */
   def schema(schemaString: String): DataStreamReader = {
-    this.userSpecifiedSchema = Option(StructType.fromDDL(schemaString))
-    this
+    schema(StructType.fromDDL(schemaString))
   }
 
   /**
@@ -185,10 +191,17 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
    *
    * @since 2.0.0
    */
-  def load(): DataFrame = {
+  def load(): DataFrame = loadInternal(None)
+
+  private def loadInternal(path: Option[String]): DataFrame = {
     if (source.toLowerCase(Locale.ROOT) == DDLUtils.HIVE_PROVIDER) {
-      throw new AnalysisException("Hive data source can only be used with tables, you can not " +
-        "read files of Hive data source directly.")
+      throw QueryCompilationErrors.cannotOperateOnHiveDataSourceFilesError("read")
+    }
+
+    val optionsWithPath = if (path.isEmpty) {
+      extraOptions
+    } else {
+      extraOptions + ("path" -> path.get)
     }
 
     val ds = DataSource.lookupDataSource(source, sparkSession.sqlContext.conf).
@@ -200,7 +213,7 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
       sparkSession,
       userSpecifiedSchema = userSpecifiedSchema,
       className = source,
-      options = extraOptions.toMap)
+      options = optionsWithPath.originalMap)
     val v1Relation = ds match {
       case _: StreamSourceProvider => Some(StreamingRelation(v1DataSource))
       case _ => None
@@ -210,8 +223,9 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
       case provider: TableProvider if !provider.isInstanceOf[FileDataSourceV2] =>
         val sessionOptions = DataSourceV2Utils.extractSessionConfigs(
           source = provider, conf = sparkSession.sessionState.conf)
-        val options = sessionOptions ++ extraOptions
-        val dsOptions = new CaseInsensitiveStringMap(options.asJava)
+        val finalOptions = sessionOptions.filterKeys(!optionsWithPath.contains(_)).toMap ++
+            optionsWithPath.originalMap
+        val dsOptions = new CaseInsensitiveStringMap(finalOptions.asJava)
         val table = DataSourceV2Utils.getTableFromProvider(provider, dsOptions, userSpecifiedSchema)
         import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Implicits._
         table match {
@@ -219,8 +233,8 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
             Dataset.ofRows(
               sparkSession,
               StreamingRelationV2(
-                provider, source, table, dsOptions, table.schema.toAttributes, v1Relation)(
-                sparkSession))
+                Some(provider), source, table, dsOptions,
+                table.schema.toAttributes, None, None, v1Relation))
 
           // fallback to v1
           // TODO (SPARK-27483): we should move this fallback logic to an analyzer rule.
@@ -239,7 +253,11 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
    * @since 2.0.0
    */
   def load(path: String): DataFrame = {
-    option("path", path).load()
+    if (!sparkSession.sessionState.conf.legacyPathOptionBehavior &&
+        extraOptions.contains("path")) {
+      throw QueryCompilationErrors.setPathOptionAndCallWithPathParameterError("load")
+    }
+    loadInternal(Some(path))
   }
 
   /**
@@ -308,6 +326,14 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
    * It does not change the behavior of partition discovery.</li>
    * <li>`recursiveFileLookup`: recursively scan a directory for files. Using this option
    * disables partition discovery</li>
+   * <li>`allowNonNumericNumbers` (default `true`): allows JSON parser to recognize set of
+   * "Not-a-Number" (NaN) tokens as legal floating number values:
+   *   <ul>
+   *     <li>`+INF` for positive infinity, as well as alias of `+Infinity` and `Infinity`.
+   *     <li>`-INF` for negative infinity, alias `-Infinity`.
+   *     <li>`NaN` for other not-a-numbers, like result of division by zero.
+   *   </ul>
+   * </li>
    * </ul>
    *
    * @since 2.0.0
@@ -369,6 +395,27 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
    * a record can have.</li>
    * <li>`maxCharsPerColumn` (default `-1`): defines the maximum number of characters allowed
    * for any given value being read. By default, it is -1 meaning unlimited length</li>
+   * <li>`unescapedQuoteHandling` (default `STOP_AT_DELIMITER`): defines how the CsvParser
+   * will handle values with unescaped quotes.
+   *   <ul>
+   *     <li>`STOP_AT_CLOSING_QUOTE`: If unescaped quotes are found in the input, accumulate
+   *     the quote character and proceed parsing the value as a quoted value, until a closing
+   *     quote is found.</li>
+   *     <li>`BACK_TO_DELIMITER`: If unescaped quotes are found in the input, consider the value
+   *     as an unquoted value. This will make the parser accumulate all characters of the current
+   *     parsed value until the delimiter is found. If no delimiter is found in the value, the
+   *     parser will continue accumulating characters from the input until a delimiter or line
+   *     ending is found.</li>
+   *     <li>`STOP_AT_DELIMITER`: If unescaped quotes are found in the input, consider the value
+   *     as an unquoted value. This will make the parser accumulate all characters until the
+   *     delimiter or a line ending is found in the input.</li>
+   *     <li>`SKIP_VALUE`: If unescaped quotes are found in the input, the content parsed
+   *     for the given value will be skipped and the value set in nullValue will be produced
+   *     instead.</li>
+   *     <li>`RAISE_ERROR`: If unescaped quotes are found in the input, a TextParsingException
+   *     will be thrown.</li>
+   *   </ul>
+   * </li>
    * <li>`mode` (default `PERMISSIVE`): allows a mode for dealing with corrupt records
    *    during parsing. It supports the following case-insensitive modes.
    *   <ul>
@@ -442,6 +489,29 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
    * It does not change the behavior of partition discovery.</li>
    * <li>`recursiveFileLookup`: recursively scan a directory for files. Using this option
    * disables partition discovery</li>
+   * <li>`datetimeRebaseMode` (default is the value specified in the SQL config
+   * `spark.sql.parquet.datetimeRebaseModeInRead`): the rebasing mode for the values
+   * of the `DATE`, `TIMESTAMP_MICROS`, `TIMESTAMP_MILLIS` logical types from the Julian to
+   * Proleptic Gregorian calendar:
+   *   <ul>
+   *     <li>`EXCEPTION` : Spark fails in reads of ancient dates/timestamps that are ambiguous
+   *     between the two calendars</li>
+   *     <li>`CORRECTED` : loading of dates/timestamps without rebasing</li>
+   *     <li>`LEGACY` : perform rebasing of ancient dates/timestamps from the Julian to Proleptic
+   *     Gregorian calendar</li>
+   *   </ul>
+   * </li>
+   * <li>`int96RebaseMode` (default is the value specified in the SQL config
+   * `spark.sql.parquet.int96RebaseModeInRead`): the rebasing mode for `INT96` timestamps
+   * from the Julian to Proleptic Gregorian calendar:
+   *   <ul>
+   *     <li>`EXCEPTION` : Spark fails in reads of ancient `INT96` timestamps that are ambiguous
+   *     between the two calendars</li>
+   *     <li>`CORRECTED` : loading of timestamps without rebasing</li>
+   *     <li>`LEGACY` : perform rebasing of ancient `INT96` timestamps from the Julian to Proleptic
+   *     Gregorian calendar</li>
+   *   </ul>
+   * </li>
    * </ul>
    *
    * @since 2.0.0
@@ -450,6 +520,23 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
     format("parquet").load(path)
   }
 
+  /**
+   * Define a Streaming DataFrame on a Table. The DataSource corresponding to the table should
+   * support streaming mode.
+   * @param tableName The name of the table
+   * @since 3.1.0
+   */
+  def table(tableName: String): DataFrame = {
+    require(tableName != null, "The table name can't be null")
+    val identifier = sparkSession.sessionState.sqlParser.parseMultipartIdentifier(tableName)
+    Dataset.ofRows(
+      sparkSession,
+      UnresolvedRelation(
+        identifier,
+        new CaseInsensitiveStringMap(extraOptions.toMap.asJava),
+        isStreaming = true))
+  }
+
   /**
    * Loads text files and returns a `DataFrame` whose schema starts with a string column named
    * "value", and followed by partitioned columns if there are any.
@@ -507,7 +594,7 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
    */
   def textFile(path: String): Dataset[String] = {
     if (userSpecifiedSchema.nonEmpty) {
-      throw new AnalysisException("User specified schema not supported with `textFile`")
+      throw QueryCompilationErrors.userSpecifiedSchemaWithTextFileError()
     }
     text(path).select("value").as[String](sparkSession.implicits.newStringEncoder)
   }
@@ -520,5 +607,5 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
 
   private var userSpecifiedSchema: Option[StructType] = None
 
-  private var extraOptions = new scala.collection.mutable.HashMap[String, String]
+  private var extraOptions = CaseInsensitiveMap[String](Map.empty)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala
index 1d0ca4d9453a5..b25aedbeca79e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala
@@ -22,18 +22,25 @@ import java.util.concurrent.TimeoutException
 
 import scala.collection.JavaConverters._
 
+import org.apache.hadoop.fs.Path
+
 import org.apache.spark.annotation.Evolving
 import org.apache.spark.api.java.function.VoidFunction2
 import org.apache.spark.sql._
+import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType}
+import org.apache.spark.sql.catalyst.plans.logical.CreateTableStatement
 import org.apache.spark.sql.catalyst.streaming.InternalOutputModes
-import org.apache.spark.sql.connector.catalog.{SupportsWrite, TableProvider}
+import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
+import org.apache.spark.sql.connector.catalog.{Identifier, SupportsWrite, Table, TableCatalog, TableProvider, V1Table, V2TableWithV1Fallback}
 import org.apache.spark.sql.connector.catalog.TableCapability._
+import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.execution.command.DDLUtils
 import org.apache.spark.sql.execution.datasources.DataSource
 import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Utils, FileDataSourceV2}
 import org.apache.spark.sql.execution.streaming._
 import org.apache.spark.sql.execution.streaming.sources._
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
+import org.apache.spark.util.Utils
 
 /**
  * Interface used to write a streaming `Dataset` to external storage systems (e.g. file systems,
@@ -43,6 +50,7 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap
  */
 @Evolving
 final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
+  import DataStreamWriter._
 
   private val df = ds.toDF()
 
@@ -266,7 +274,11 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
    * @since 2.0.0
    */
   def start(path: String): StreamingQuery = {
-    option("path", path).start()
+    if (!df.sparkSession.sessionState.conf.legacyPathOptionBehavior &&
+        extraOptions.contains("path")) {
+      throw QueryCompilationErrors.setPathOptionAndCallWithPathParameterError("start")
+    }
+    startInternal(Some(path))
   }
 
   /**
@@ -284,103 +296,189 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
    * @since 2.0.0
    */
   @throws[TimeoutException]
-  def start(): StreamingQuery = {
+  def start(): StreamingQuery = startInternal(None)
+
+  /**
+   * Starts the execution of the streaming query, which will continually output results to the given
+   * table as new data arrives. The returned [[StreamingQuery]] object can be used to interact with
+   * the stream.
+   *
+   * For v1 table, partitioning columns provided by `partitionBy` will be respected no matter the
+   * table exists or not. A new table will be created if the table not exists.
+   *
+   * For v2 table, `partitionBy` will be ignored if the table already exists. `partitionBy` will be
+   * respected only if the v2 table does not exist. Besides, the v2 table created by this API lacks
+   * some functionalities (e.g., customized properties, options, and serde info). If you need them,
+   * please create the v2 table manually before the execution to avoid creating a table with
+   * incomplete information.
+   *
+   * @since 3.1.0
+   */
+  @Evolving
+  @throws[TimeoutException]
+  def toTable(tableName: String): StreamingQuery = {
+    this.tableName = tableName
+
+    import df.sparkSession.sessionState.analyzer.CatalogAndIdentifier
+
+    import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+    val originalMultipartIdentifier = df.sparkSession.sessionState.sqlParser
+      .parseMultipartIdentifier(tableName)
+    val CatalogAndIdentifier(catalog, identifier) = originalMultipartIdentifier
+
+    // Currently we don't create a logical streaming writer node in logical plan, so cannot rely
+    // on analyzer to resolve it. Directly lookup only for temp view to provide clearer message.
+    // TODO (SPARK-27484): we should add the writing node before the plan is analyzed.
+    if (df.sparkSession.sessionState.catalog.isTempView(originalMultipartIdentifier)) {
+      throw QueryCompilationErrors.tempViewNotSupportStreamingWriteError(tableName)
+    }
+
+    if (!catalog.asTableCatalog.tableExists(identifier)) {
+      import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+      /**
+       * Note, currently the new table creation by this API doesn't fully cover the V2 table.
+       * TODO (SPARK-33638): Full support of v2 table creation
+       */
+      val cmd = CreateTableStatement(
+        originalMultipartIdentifier,
+        df.schema.asNullable,
+        partitioningColumns.getOrElse(Nil).asTransforms.toSeq,
+        None,
+        Map.empty[String, String],
+        Some(source),
+        Map.empty[String, String],
+        extraOptions.get("path"),
+        None,
+        None,
+        external = false,
+        ifNotExists = false)
+      Dataset.ofRows(df.sparkSession, cmd)
+    }
+
+    val tableInstance = catalog.asTableCatalog.loadTable(identifier)
+
+    def writeToV1Table(table: CatalogTable): StreamingQuery = {
+      if (table.tableType == CatalogTableType.VIEW) {
+        throw QueryCompilationErrors.streamingIntoViewNotSupportedError(tableName)
+      }
+      require(table.provider.isDefined)
+      if (source != table.provider.get) {
+        throw QueryCompilationErrors.inputSourceDiffersFromDataSourceProviderError(
+          source, tableName, table)
+      }
+      format(table.provider.get)
+        .option("path", new Path(table.location).toString).start()
+    }
+
+    import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Implicits._
+    tableInstance match {
+      case t: SupportsWrite if t.supports(STREAMING_WRITE) =>
+        startQuery(t, extraOptions, catalogAndIdent = Some(catalog.asTableCatalog, identifier))
+      case t: V2TableWithV1Fallback =>
+        writeToV1Table(t.v1Table)
+      case t: V1Table =>
+        writeToV1Table(t.v1Table)
+      case t => throw QueryCompilationErrors.tableNotSupportStreamingWriteError(tableName, t)
+    }
+  }
+
+  private def startInternal(path: Option[String]): StreamingQuery = {
     if (source.toLowerCase(Locale.ROOT) == DDLUtils.HIVE_PROVIDER) {
-      throw new AnalysisException("Hive data source can only be used with tables, you can not " +
-        "write files of Hive data source directly.")
+      throw QueryCompilationErrors.cannotOperateOnHiveDataSourceFilesError("write")
     }
 
-    if (source == "memory") {
-      assertNotPartitioned("memory")
+    if (source == SOURCE_NAME_MEMORY) {
+      assertNotPartitioned(SOURCE_NAME_MEMORY)
       if (extraOptions.get("queryName").isEmpty) {
-        throw new AnalysisException("queryName must be specified for memory sink")
+        throw QueryCompilationErrors.queryNameNotSpecifiedForMemorySinkError()
       }
       val sink = new MemorySink()
       val resultDf = Dataset.ofRows(df.sparkSession, new MemoryPlan(sink, df.schema.toAttributes))
-      val chkpointLoc = extraOptions.get("checkpointLocation")
-      val recoverFromChkpoint = outputMode == OutputMode.Complete()
-      val query = df.sparkSession.sessionState.streamingQueryManager.startQuery(
-        extraOptions.get("queryName"),
-        chkpointLoc,
-        df,
-        extraOptions.toMap,
-        sink,
-        outputMode,
-        useTempCheckpointLocation = true,
-        recoverFromCheckpointLocation = recoverFromChkpoint,
-        trigger = trigger)
+      val recoverFromCheckpoint = outputMode == OutputMode.Complete()
+      val query = startQuery(sink, extraOptions, recoverFromCheckpoint = recoverFromCheckpoint)
       resultDf.createOrReplaceTempView(query.name)
       query
-    } else if (source == "foreach") {
-      assertNotPartitioned("foreach")
+    } else if (source == SOURCE_NAME_FOREACH) {
+      assertNotPartitioned(SOURCE_NAME_FOREACH)
       val sink = ForeachWriterTable[T](foreachWriter, ds.exprEnc)
-      df.sparkSession.sessionState.streamingQueryManager.startQuery(
-        extraOptions.get("queryName"),
-        extraOptions.get("checkpointLocation"),
-        df,
-        extraOptions.toMap,
-        sink,
-        outputMode,
-        useTempCheckpointLocation = true,
-        trigger = trigger)
-    } else if (source == "foreachBatch") {
-      assertNotPartitioned("foreachBatch")
+      startQuery(sink, extraOptions)
+    } else if (source == SOURCE_NAME_FOREACH_BATCH) {
+      assertNotPartitioned(SOURCE_NAME_FOREACH_BATCH)
       if (trigger.isInstanceOf[ContinuousTrigger]) {
-        throw new AnalysisException("'foreachBatch' is not supported with continuous trigger")
+        throw QueryCompilationErrors.sourceNotSupportedWithContinuousTriggerError(source)
       }
       val sink = new ForeachBatchSink[T](foreachBatchWriter, ds.exprEnc)
-      df.sparkSession.sessionState.streamingQueryManager.startQuery(
-        extraOptions.get("queryName"),
-        extraOptions.get("checkpointLocation"),
-        df,
-        extraOptions.toMap,
-        sink,
-        outputMode,
-        useTempCheckpointLocation = true,
-        trigger = trigger)
+      startQuery(sink, extraOptions)
     } else {
       val cls = DataSource.lookupDataSource(source, df.sparkSession.sessionState.conf)
-      val disabledSources = df.sparkSession.sqlContext.conf.disabledV2StreamingWriters.split(",")
+      val disabledSources =
+        Utils.stringToSeq(df.sparkSession.sqlContext.conf.disabledV2StreamingWriters)
       val useV1Source = disabledSources.contains(cls.getCanonicalName) ||
         // file source v2 does not support streaming yet.
         classOf[FileDataSourceV2].isAssignableFrom(cls)
 
+      val optionsWithPath = if (path.isEmpty) {
+        extraOptions
+      } else {
+        extraOptions + ("path" -> path.get)
+      }
+
       val sink = if (classOf[TableProvider].isAssignableFrom(cls) && !useV1Source) {
         val provider = cls.getConstructor().newInstance().asInstanceOf[TableProvider]
         val sessionOptions = DataSourceV2Utils.extractSessionConfigs(
           source = provider, conf = df.sparkSession.sessionState.conf)
-        val options = sessionOptions ++ extraOptions
-        val dsOptions = new CaseInsensitiveStringMap(options.asJava)
+        val finalOptions = sessionOptions.filterKeys(!optionsWithPath.contains(_)).toMap ++
+          optionsWithPath.originalMap
+        val dsOptions = new CaseInsensitiveStringMap(finalOptions.asJava)
+        // If the source accepts external table metadata, here we pass the schema of input query
+        // to `getTable`. This is for avoiding schema inference, which can be very expensive.
+        // If the query schema is not compatible with the existing data, the behavior is undefined.
+        val outputSchema = if (provider.supportsExternalMetadata()) {
+          Some(df.schema)
+        } else {
+          None
+        }
         val table = DataSourceV2Utils.getTableFromProvider(
-          provider, dsOptions, userSpecifiedSchema = None)
+          provider, dsOptions, userSpecifiedSchema = outputSchema)
         import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Implicits._
         table match {
           case table: SupportsWrite if table.supports(STREAMING_WRITE) =>
             table
-          case _ => createV1Sink()
+          case _ => createV1Sink(optionsWithPath)
         }
       } else {
-        createV1Sink()
+        createV1Sink(optionsWithPath)
       }
 
-      df.sparkSession.sessionState.streamingQueryManager.startQuery(
-        extraOptions.get("queryName"),
-        extraOptions.get("checkpointLocation"),
-        df,
-        extraOptions.toMap,
-        sink,
-        outputMode,
-        useTempCheckpointLocation = source == "console" || source == "noop",
-        recoverFromCheckpointLocation = true,
-        trigger = trigger)
+      startQuery(sink, optionsWithPath)
     }
   }
 
-  private def createV1Sink(): Sink = {
+  private def startQuery(
+      sink: Table,
+      newOptions: CaseInsensitiveMap[String],
+      recoverFromCheckpoint: Boolean = true,
+      catalogAndIdent: Option[(TableCatalog, Identifier)] = None): StreamingQuery = {
+    val useTempCheckpointLocation = SOURCES_ALLOW_ONE_TIME_QUERY.contains(source)
+
+    df.sparkSession.sessionState.streamingQueryManager.startQuery(
+      newOptions.get("queryName"),
+      newOptions.get("checkpointLocation"),
+      df,
+      newOptions.originalMap,
+      sink,
+      outputMode,
+      useTempCheckpointLocation = useTempCheckpointLocation,
+      recoverFromCheckpointLocation = recoverFromCheckpoint,
+      trigger = trigger,
+      catalogAndIdent = catalogAndIdent)
+  }
+
+  private def createV1Sink(optionsWithPath: CaseInsensitiveMap[String]): Sink = {
     val ds = DataSource(
       df.sparkSession,
       className = source,
-      options = extraOptions.toMap,
+      options = optionsWithPath.originalMap,
       partitionColumns = normalizedParCols.getOrElse(Nil))
     ds.createSink(outputMode)
   }
@@ -392,7 +490,7 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
    * @since 2.0.0
    */
   def foreach(writer: ForeachWriter[T]): DataStreamWriter[T] = {
-    this.source = "foreach"
+    this.source = SOURCE_NAME_FOREACH
     this.foreachWriter = if (writer != null) {
       ds.sparkSession.sparkContext.clean(writer)
     } else {
@@ -405,18 +503,19 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
    * :: Experimental ::
    *
    * (Scala-specific) Sets the output of the streaming query to be processed using the provided
-   * function. This is supported only the in the micro-batch execution modes (that is, when the
+   * function. This is supported only in the micro-batch execution modes (that is, when the
    * trigger is not continuous). In every micro-batch, the provided function will be called in
    * every micro-batch with (i) the output rows as a Dataset and (ii) the batch identifier.
-   * The batchId can be used deduplicate and transactionally write the output
+   * The batchId can be used to deduplicate and transactionally write the output
    * (that is, the provided Dataset) to external systems. The output Dataset is guaranteed
-   * to exactly same for the same batchId (assuming all operations are deterministic in the query).
+   * to be exactly the same for the same batchId (assuming all operations are deterministic
+   * in the query).
    *
    * @since 2.4.0
    */
   @Evolving
   def foreachBatch(function: (Dataset[T], Long) => Unit): DataStreamWriter[T] = {
-    this.source = "foreachBatch"
+    this.source = SOURCE_NAME_FOREACH_BATCH
     if (function == null) throw new IllegalArgumentException("foreachBatch function cannot be null")
     this.foreachBatchWriter = function
     this
@@ -426,12 +525,13 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
    * :: Experimental ::
    *
    * (Java-specific) Sets the output of the streaming query to be processed using the provided
-   * function. This is supported only the in the micro-batch execution modes (that is, when the
+   * function. This is supported only in the micro-batch execution modes (that is, when the
    * trigger is not continuous). In every micro-batch, the provided function will be called in
    * every micro-batch with (i) the output rows as a Dataset and (ii) the batch identifier.
-   * The batchId can be used deduplicate and transactionally write the output
+   * The batchId can be used to deduplicate and transactionally write the output
    * (that is, the provided Dataset) to external systems. The output Dataset is guaranteed
-   * to exactly same for the same batchId (assuming all operations are deterministic in the query).
+   * to be exactly the same for the same batchId (assuming all operations are deterministic
+   * in the query).
    *
    * @since 2.4.0
    */
@@ -452,13 +552,13 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
   private def normalize(columnName: String, columnType: String): String = {
     val validColumnNames = df.logicalPlan.output.map(_.name)
     validColumnNames.find(df.sparkSession.sessionState.analyzer.resolver(_, columnName))
-      .getOrElse(throw new AnalysisException(s"$columnType column $columnName not found in " +
-        s"existing columns (${validColumnNames.mkString(", ")})"))
+      .getOrElse(throw QueryCompilationErrors.columnNotFoundInExistingColumnsError(
+        columnType, columnName, validColumnNames))
   }
 
   private def assertNotPartitioned(operation: String): Unit = {
     if (partitioningColumns.isDefined) {
-      throw new AnalysisException(s"'$operation' does not support partitioning")
+      throw QueryCompilationErrors.operationNotSupportPartitioningError(operation)
     }
   }
 
@@ -468,11 +568,13 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
 
   private var source: String = df.sparkSession.sessionState.conf.defaultDataSourceName
 
+  private var tableName: String = null
+
   private var outputMode: OutputMode = OutputMode.Append
 
   private var trigger: Trigger = Trigger.ProcessingTime(0L)
 
-  private var extraOptions = new scala.collection.mutable.HashMap[String, String]
+  private var extraOptions = CaseInsensitiveMap[String](Map.empty)
 
   private var foreachWriter: ForeachWriter[T] = null
 
@@ -480,3 +582,16 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
 
   private var partitioningColumns: Option[Seq[String]] = None
 }
+
+object DataStreamWriter {
+  val SOURCE_NAME_MEMORY = "memory"
+  val SOURCE_NAME_FOREACH = "foreach"
+  val SOURCE_NAME_FOREACH_BATCH = "foreachBatch"
+  val SOURCE_NAME_CONSOLE = "console"
+  val SOURCE_NAME_TABLE = "table"
+  val SOURCE_NAME_NOOP = "noop"
+
+  // these writer sources are also used for one-time query, hence allow temp checkpoint location
+  val SOURCES_ALLOW_ONE_TIME_QUERY = Seq(SOURCE_NAME_MEMORY, SOURCE_NAME_FOREACH,
+    SOURCE_NAME_FOREACH_BATCH, SOURCE_NAME_CONSOLE, SOURCE_NAME_NOOP)
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/GroupState.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/GroupState.scala
index af08a53e465b3..f50727307d9f6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/GroupState.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/GroupState.scala
@@ -30,19 +30,19 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalGroupState
  * --------------------------------------------------------------
  * Both, `mapGroupsWithState` and `flatMapGroupsWithState` in `KeyValueGroupedDataset`
  * will invoke the user-given function on each group (defined by the grouping function in
- * `Dataset.groupByKey()`) while maintaining user-defined per-group state between invocations.
+ * `Dataset.groupByKey()`) while maintaining a user-defined per-group state between invocations.
  * For a static batch Dataset, the function will be invoked once per group. For a streaming
  * Dataset, the function will be invoked for each group repeatedly in every trigger.
  * That is, in every batch of the `StreamingQuery`,
  * the function will be invoked once for each group that has data in the trigger. Furthermore,
- * if timeout is set, then the function will invoked on timed out groups (more detail below).
+ * if timeout is set, then the function will be invoked on timed-out groups (more detail below).
  *
- * The function is invoked with following parameters.
+ * The function is invoked with the following parameters.
  *  - The key of the group.
  *  - An iterator containing all the values for this group.
  *  - A user-defined state object set by previous invocations of the given function.
  *
- * In case of a batch Dataset, there is only one invocation and state object will be empty as
+ * In case of a batch Dataset, there is only one invocation and the state object will be empty as
  * there is no prior state. Essentially, for batch Datasets, `[map/flatMap]GroupsWithState`
  * is equivalent to `[map/flatMap]Groups` and any updates to the state and/or timeouts have
  * no effect.
@@ -85,31 +85,31 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalGroupState
  *  - With `ProcessingTimeTimeout`, the timeout duration can be set by calling
  *    `GroupState.setTimeoutDuration`. The timeout will occur when the clock has advanced by the set
  *    duration. Guarantees provided by this timeout with a duration of D ms are as follows:
- *    - Timeout will never be occur before the clock time has advanced by D ms
+ *    - Timeout will never occur before the clock time has advanced by D ms
  *    - Timeout will occur eventually when there is a trigger in the query
- *      (i.e. after D ms). So there is a no strict upper bound on when the timeout would occur.
+ *      (i.e. after D ms). So there is no strict upper bound on when the timeout would occur.
  *      For example, the trigger interval of the query will affect when the timeout actually occurs.
- *      If there is no data in the stream (for any group) for a while, then their will not be
+ *      If there is no data in the stream (for any group) for a while, then there will not be
  *      any trigger and timeout function call will not occur until there is data.
  *    - Since the processing time timeout is based on the clock time, it is affected by the
  *      variations in the system clock (i.e. time zone changes, clock skew, etc.).
  *  - With `EventTimeTimeout`, the user also has to specify the event time watermark in
  *    the query using `Dataset.withWatermark()`. With this setting, data that is older than the
- *    watermark are filtered out. The timeout can be set for a group by setting a timeout timestamp
+ *    watermark is filtered out. The timeout can be set for a group by setting a timeout timestamp
  *    using`GroupState.setTimeoutTimestamp()`, and the timeout would occur when the watermark
  *    advances beyond the set timestamp. You can control the timeout delay by two parameters -
  *    (i) watermark delay and an additional duration beyond the timestamp in the event (which
  *    is guaranteed to be newer than watermark due to the filtering). Guarantees provided by this
  *    timeout are as follows:
- *    - Timeout will never be occur before watermark has exceeded the set timeout.
- *    - Similar to processing time timeouts, there is a no strict upper bound on the delay when
+ *    - Timeout will never occur before the watermark has exceeded the set timeout.
+ *    - Similar to processing time timeouts, there is no strict upper bound on the delay when
  *      the timeout actually occurs. The watermark can advance only when there is data in the
- *      stream, and the event time of the data has actually advanced.
+ *      stream and the event time of the data has actually advanced.
  *  - When the timeout occurs for a group, the function is called for that group with no values, and
  *    `GroupState.hasTimedOut()` set to true.
  *  - The timeout is reset every time the function is called on a group, that is,
  *    when the group has new data, or the group has timed out. So the user has to set the timeout
- *    duration every time the function is called, otherwise there will not be any timeout set.
+ *    duration every time the function is called, otherwise, there will not be any timeout set.
  *
  * Scala example of using GroupState in `mapGroupsWithState`:
  * {{{
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryManager.scala
index 0fe2d0be966d0..dd5194038a979 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryManager.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryManager.scala
@@ -17,22 +17,19 @@
 
 package org.apache.spark.sql.streaming
 
-import java.util.{ConcurrentModificationException, UUID}
+import java.util.UUID
 import java.util.concurrent.{TimeoutException, TimeUnit}
 import javax.annotation.concurrent.GuardedBy
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable
 
-import org.apache.hadoop.fs.Path
-
-import org.apache.spark.SparkException
 import org.apache.spark.annotation.Evolving
 import org.apache.spark.internal.Logging
-import org.apache.spark.internal.config.UI.UI_ENABLED
-import org.apache.spark.sql.{AnalysisException, DataFrame, SparkSession}
-import org.apache.spark.sql.catalyst.analysis.UnsupportedOperationChecker
-import org.apache.spark.sql.connector.catalog.{SupportsWrite, Table}
+import org.apache.spark.sql.{DataFrame, SparkSession}
+import org.apache.spark.sql.catalyst.streaming.{WriteToStream, WriteToStreamStatement}
+import org.apache.spark.sql.connector.catalog.{Identifier, SupportsWrite, Table, TableCatalog}
+import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.execution.streaming._
 import org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution
 import org.apache.spark.sql.execution.streaming.state.StateStoreCoordinatorRef
@@ -50,7 +47,8 @@ class StreamingQueryManager private[sql] (sparkSession: SparkSession) extends Lo
 
   private[sql] val stateStoreCoordinator =
     StateStoreCoordinatorRef.forDriver(sparkSession.sparkContext.env)
-  private val listenerBus = new StreamingQueryListenerBus(sparkSession.sparkContext.listenerBus)
+  private val listenerBus =
+    new StreamingQueryListenerBus(Some(sparkSession.sparkContext.listenerBus))
 
   @GuardedBy("activeQueriesSharedLock")
   private val activeQueries = new mutable.HashMap[UUID, StreamingQuery]
@@ -83,7 +81,7 @@ class StreamingQueryManager private[sql] (sparkSession: SparkSession) extends Lo
     }
   } catch {
     case e: Exception =>
-      throw new SparkException("Exception when registering StreamingQueryListener", e)
+      throw QueryExecutionErrors.registeringStreamingQueryListenerError(e)
   }
 
   /**
@@ -228,6 +226,7 @@ class StreamingQueryManager private[sql] (sparkSession: SparkSession) extends Lo
     listenerBus.post(event)
   }
 
+  // scalastyle:off argcount
   private def createQuery(
       userSpecifiedName: Option[String],
       userSpecifiedCheckpointLocation: Option[String],
@@ -238,86 +237,47 @@ class StreamingQueryManager private[sql] (sparkSession: SparkSession) extends Lo
       useTempCheckpointLocation: Boolean,
       recoverFromCheckpointLocation: Boolean,
       trigger: Trigger,
-      triggerClock: Clock): StreamingQueryWrapper = {
-    var deleteCheckpointOnStop = false
-    val checkpointLocation = userSpecifiedCheckpointLocation.map { userSpecified =>
-      new Path(userSpecified).toString
-    }.orElse {
-      df.sparkSession.sessionState.conf.checkpointLocation.map { location =>
-        new Path(location, userSpecifiedName.getOrElse(UUID.randomUUID().toString)).toString
-      }
-    }.getOrElse {
-      if (useTempCheckpointLocation) {
-        deleteCheckpointOnStop = true
-        val tempDir = Utils.createTempDir(namePrefix = s"temporary").getCanonicalPath
-        logWarning("Temporary checkpoint location created which is deleted normally when" +
-          s" the query didn't fail: $tempDir. If it's required to delete it under any" +
-          s" circumstances, please set ${SQLConf.FORCE_DELETE_TEMP_CHECKPOINT_LOCATION.key} to" +
-          s" true. Important to know deleting temp checkpoint folder is best effort.")
-        tempDir
-      } else {
-        throw new AnalysisException(
-          "checkpointLocation must be specified either " +
-            """through option("checkpointLocation", ...) or """ +
-            s"""SparkSession.conf.set("${SQLConf.CHECKPOINT_LOCATION.key}", ...)""")
-      }
-    }
-
-    // If offsets have already been created, we trying to resume a query.
-    if (!recoverFromCheckpointLocation) {
-      val checkpointPath = new Path(checkpointLocation, "offsets")
-      val fs = checkpointPath.getFileSystem(df.sparkSession.sessionState.newHadoopConf())
-      if (fs.exists(checkpointPath)) {
-        throw new AnalysisException(
-          s"This query does not support recovering from checkpoint location. " +
-            s"Delete $checkpointPath to start over.")
-      }
-    }
-
+      triggerClock: Clock,
+      catalogAndIdent: Option[(TableCatalog, Identifier)] = None): StreamingQueryWrapper = {
     val analyzedPlan = df.queryExecution.analyzed
     df.queryExecution.assertAnalyzed()
 
-    val operationCheckEnabled = sparkSession.sessionState.conf.isUnsupportedOperationCheckEnabled
+    val dataStreamWritePlan = WriteToStreamStatement(
+      userSpecifiedName,
+      userSpecifiedCheckpointLocation,
+      useTempCheckpointLocation,
+      recoverFromCheckpointLocation,
+      sink,
+      outputMode,
+      df.sparkSession.sessionState.newHadoopConf(),
+      trigger.isInstanceOf[ContinuousTrigger],
+      analyzedPlan,
+      catalogAndIdent)
 
-    if (sparkSession.sessionState.conf.adaptiveExecutionEnabled) {
-      logWarning(s"${SQLConf.ADAPTIVE_EXECUTION_ENABLED.key} " +
-          "is not supported in streaming DataFrames/Datasets and will be disabled.")
-    }
+    val analyzedStreamWritePlan =
+      sparkSession.sessionState.executePlan(dataStreamWritePlan).analyzed
+        .asInstanceOf[WriteToStream]
 
     (sink, trigger) match {
-      case (table: SupportsWrite, trigger: ContinuousTrigger) =>
-        if (operationCheckEnabled) {
-          UnsupportedOperationChecker.checkForContinuous(analyzedPlan, outputMode)
-        }
+      case (_: SupportsWrite, trigger: ContinuousTrigger) =>
         new StreamingQueryWrapper(new ContinuousExecution(
           sparkSession,
-          userSpecifiedName.orNull,
-          checkpointLocation,
-          analyzedPlan,
-          table,
           trigger,
           triggerClock,
-          outputMode,
           extraOptions,
-          deleteCheckpointOnStop))
+          analyzedStreamWritePlan))
       case _ =>
-        if (operationCheckEnabled) {
-          UnsupportedOperationChecker.checkForStreaming(analyzedPlan, outputMode)
-        }
         new StreamingQueryWrapper(new MicroBatchExecution(
           sparkSession,
-          userSpecifiedName.orNull,
-          checkpointLocation,
-          analyzedPlan,
-          sink,
           trigger,
           triggerClock,
-          outputMode,
           extraOptions,
-          deleteCheckpointOnStop))
+          analyzedStreamWritePlan))
     }
   }
+  // scalastyle:on argcount
 
+  // scalastyle:off argcount
   /**
    * Start a [[StreamingQuery]].
    *
@@ -333,6 +293,7 @@ class StreamingQueryManager private[sql] (sparkSession: SparkSession) extends Lo
    *                                       will be thrown.
    * @param trigger [[Trigger]] for the query.
    * @param triggerClock [[Clock]] to use for the triggering.
+   * @param catalogAndIdent Catalog and identifier for the sink, set when it is a V2 catalog table
    */
   @throws[TimeoutException]
   private[sql] def startQuery(
@@ -345,7 +306,8 @@ class StreamingQueryManager private[sql] (sparkSession: SparkSession) extends Lo
       useTempCheckpointLocation: Boolean = false,
       recoverFromCheckpointLocation: Boolean = true,
       trigger: Trigger = Trigger.ProcessingTime(0),
-      triggerClock: Clock = new SystemClock()): StreamingQuery = {
+      triggerClock: Clock = new SystemClock(),
+      catalogAndIdent: Option[(TableCatalog, Identifier)] = None): StreamingQuery = {
     val query = createQuery(
       userSpecifiedName,
       userSpecifiedCheckpointLocation,
@@ -356,7 +318,9 @@ class StreamingQueryManager private[sql] (sparkSession: SparkSession) extends Lo
       useTempCheckpointLocation,
       recoverFromCheckpointLocation,
       trigger,
-      triggerClock)
+      triggerClock,
+      catalogAndIdent)
+    // scalastyle:on argcount
 
     // The following code block checks if a stream with the same name or id is running. Then it
     // returns an Option of an already active stream to stop outside of the lock
@@ -407,8 +371,7 @@ class StreamingQueryManager private[sql] (sparkSession: SparkSession) extends Lo
       val oldActiveQuery = sparkSession.sharedState.activeStreamingQueries.put(
         query.id, query.streamingQuery) // we need to put the StreamExecution, not the wrapper
       if (oldActiveQuery != null) {
-        throw new ConcurrentModificationException(
-          "Another instance of this query was just started by a concurrent session.")
+        throw QueryExecutionErrors.concurrentQueryInstanceError()
       }
       activeQueries.put(query.id, query)
     }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/progress.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/progress.scala
index 13b506b60a126..554780a0e78b9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/progress.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/progress.scala
@@ -33,6 +33,7 @@ import org.json4s.jackson.JsonMethods._
 import org.apache.spark.annotation.Evolving
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema
+import org.apache.spark.sql.streaming.SafeJsonSerializer.{safeDoubleToJValue, safeMapToJValue}
 import org.apache.spark.sql.streaming.SinkProgress.DEFAULT_NUM_OUTPUT_ROWS
 
 /**
@@ -43,6 +44,7 @@ class StateOperatorProgress private[sql](
     val numRowsTotal: Long,
     val numRowsUpdated: Long,
     val memoryUsedBytes: Long,
+    val numRowsDroppedByWatermark: Long,
     val customMetrics: ju.Map[String, JLong] = new ju.HashMap()
   ) extends Serializable {
 
@@ -52,13 +54,17 @@ class StateOperatorProgress private[sql](
   /** The pretty (i.e. indented) JSON representation of this progress. */
   def prettyJson: String = pretty(render(jsonValue))
 
-  private[sql] def copy(newNumRowsUpdated: Long): StateOperatorProgress =
-    new StateOperatorProgress(numRowsTotal, newNumRowsUpdated, memoryUsedBytes, customMetrics)
+  private[sql] def copy(
+      newNumRowsUpdated: Long,
+      newNumRowsDroppedByWatermark: Long): StateOperatorProgress =
+    new StateOperatorProgress(numRowsTotal, newNumRowsUpdated, memoryUsedBytes,
+      newNumRowsDroppedByWatermark, customMetrics)
 
   private[sql] def jsonValue: JValue = {
     ("numRowsTotal" -> JInt(numRowsTotal)) ~
     ("numRowsUpdated" -> JInt(numRowsUpdated)) ~
     ("memoryUsedBytes" -> JInt(memoryUsedBytes)) ~
+    ("numRowsDroppedByWatermark" -> JInt(numRowsDroppedByWatermark)) ~
     ("customMetrics" -> {
       if (!customMetrics.isEmpty) {
         val keys = customMetrics.keySet.asScala.toSeq.sorted
@@ -133,17 +139,6 @@ class StreamingQueryProgress private[sql](
   override def toString: String = prettyJson
 
   private[sql] def jsonValue: JValue = {
-    def safeDoubleToJValue(value: Double): JValue = {
-      if (value.isNaN || value.isInfinity) JNothing else JDouble(value)
-    }
-
-    /** Convert map to JValue while handling empty maps. Also, this sorts the keys. */
-    def safeMapToJValue[T](map: ju.Map[String, T], valueToJValue: T => JValue): JValue = {
-      if (map.isEmpty) return JNothing
-      val keys = map.asScala.keySet.toSeq.sorted
-      keys.map { k => k -> valueToJValue(map.get(k)) : JObject }.reduce(_ ~ _)
-    }
-
     ("id" -> JString(id.toString)) ~
     ("runId" -> JString(runId.toString)) ~
     ("name" -> JString(name)) ~
@@ -168,6 +163,7 @@ class StreamingQueryProgress private[sql](
  * @param description            Description of the source.
  * @param startOffset            The starting offset for data being read.
  * @param endOffset              The ending offset for data being read.
+ * @param latestOffset           The latest offset from this source.
  * @param numInputRows           The number of records read from this source.
  * @param inputRowsPerSecond     The rate at which data is arriving from this source.
  * @param processedRowsPerSecond The rate at which data from this source is being processed by
@@ -179,9 +175,11 @@ class SourceProgress protected[sql](
   val description: String,
   val startOffset: String,
   val endOffset: String,
+  val latestOffset: String,
   val numInputRows: Long,
   val inputRowsPerSecond: Double,
-  val processedRowsPerSecond: Double) extends Serializable {
+  val processedRowsPerSecond: Double,
+  val metrics: ju.Map[String, String] = Map[String, String]().asJava) extends Serializable {
 
   /** The compact JSON representation of this progress. */
   def json: String = compact(render(jsonValue))
@@ -192,16 +190,14 @@ class SourceProgress protected[sql](
   override def toString: String = prettyJson
 
   private[sql] def jsonValue: JValue = {
-    def safeDoubleToJValue(value: Double): JValue = {
-      if (value.isNaN || value.isInfinity) JNothing else JDouble(value)
-    }
-
     ("description" -> JString(description)) ~
-      ("startOffset" -> tryParse(startOffset)) ~
-      ("endOffset" -> tryParse(endOffset)) ~
-      ("numInputRows" -> JInt(numInputRows)) ~
-      ("inputRowsPerSecond" -> safeDoubleToJValue(inputRowsPerSecond)) ~
-      ("processedRowsPerSecond" -> safeDoubleToJValue(processedRowsPerSecond))
+    ("startOffset" -> tryParse(startOffset)) ~
+    ("endOffset" -> tryParse(endOffset)) ~
+    ("latestOffset" -> tryParse(latestOffset)) ~
+    ("numInputRows" -> JInt(numInputRows)) ~
+    ("inputRowsPerSecond" -> safeDoubleToJValue(inputRowsPerSecond)) ~
+    ("processedRowsPerSecond" -> safeDoubleToJValue(processedRowsPerSecond)) ~
+    ("metrics" -> safeMapToJValue[String](metrics, s => JString(s)))
   }
 
   private def tryParse(json: String) = try {
@@ -226,7 +222,7 @@ class SinkProgress protected[sql](
     val numOutputRows: Long) extends Serializable {
 
   /** SinkProgress without custom metrics. */
-  protected[sql] def this(description: String) {
+  protected[sql] def this(description: String) = {
     this(description, DEFAULT_NUM_OUTPUT_ROWS)
   }
 
@@ -250,3 +246,16 @@ private[sql] object SinkProgress {
   def apply(description: String, numOutputRows: Option[Long]): SinkProgress =
     new SinkProgress(description, numOutputRows.getOrElse(DEFAULT_NUM_OUTPUT_ROWS))
 }
+
+private object SafeJsonSerializer {
+  def safeDoubleToJValue(value: Double): JValue = {
+    if (value.isNaN || value.isInfinity) JNothing else JDouble(value)
+  }
+
+  /** Convert map to JValue while handling empty maps. Also, this sorts the keys. */
+  def safeMapToJValue[T](map: ju.Map[String, T], valueToJValue: T => JValue): JValue = {
+    if (map.isEmpty) return JNothing
+    val keys = map.asScala.keySet.toSeq.sorted
+    keys.map { k => k -> valueToJValue(map.get(k)) : JObject }.reduce(_ ~ _)
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala
index 733676546eab3..6b0aa48858a72 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala
@@ -17,15 +17,19 @@
 
 package org.apache.spark.sql.streaming.ui
 
+import java.net.URLEncoder
+import java.nio.charset.StandardCharsets.UTF_8
 import javax.servlet.http.HttpServletRequest
 
+import scala.collection.mutable
 import scala.xml.Node
 
 import org.apache.commons.text.StringEscapeUtils
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.streaming.ui.UIUtils._
-import org.apache.spark.ui.{UIUtils => SparkUIUtils, WebUIPage}
+import org.apache.spark.ui.{PagedDataSource, PagedTable, UIUtils => SparkUIUtils, WebUIPage}
+import org.apache.spark.util.Utils
 
 private[ui] class StreamingQueryPage(parent: StreamingQueryTab)
     extends WebUIPage("") with Logging {
@@ -35,11 +39,147 @@ private[ui] class StreamingQueryPage(parent: StreamingQueryTab)
     SparkUIUtils.headerSparkPage(request, "Streaming Query", content, parent)
   }
 
-  def generateDataRow(request: HttpServletRequest, queryActive: Boolean)
-    (query: StreamingQueryUIData): Seq[Node] = {
+  private def generateStreamingQueryTable(request: HttpServletRequest): Seq[Node] = {
+    val (activeQueries, inactiveQueries) =
+      parent.store.allQueryUIData.partition(_.summary.isActive)
+
+    val content = mutable.ListBuffer[Node]()
+    // show active queries table only if there is at least one active query
+    if (activeQueries.nonEmpty) {
+      // scalastyle:off
+      content ++=
+        <span id="active" class="collapse-aggregated-activeQueries collapse-table"
+            onClick="collapseTable('collapse-aggregated-activeQueries','aggregated-activeQueries')">
+          <h5 id="activequeries">
+            <span class="collapse-table-arrow arrow-open"></span>
+            <a>Active Streaming Queries ({activeQueries.length})</a>
+          </h5>
+        </span> ++
+          <div>
+            <ul class="aggregated-activeQueries collapsible-table">
+              {queryTable(activeQueries, request, "active")}
+            </ul>
+          </div>
+      // scalastyle:on
+    }
+    // show active queries table only if there is at least one completed query
+    if (inactiveQueries.nonEmpty) {
+      // scalastyle:off
+      content ++=
+        <span id="completed" class="collapse-aggregated-completedQueries collapse-table"
+            onClick="collapseTable('collapse-aggregated-completedQueries','aggregated-completedQueries')">
+          <h5 id="completedqueries">
+            <span class="collapse-table-arrow arrow-open"></span>
+            <a>Completed Streaming Queries ({inactiveQueries.length})</a>
+          </h5>
+        </span> ++
+          <div>
+            <ul class="aggregated-completedQueries collapsible-table">
+              {queryTable(inactiveQueries, request, "completed")}
+            </ul>
+          </div>
+      // scalastyle:on
+    }
+    content
+  }
+
+  private def queryTable(data: Seq[StreamingQueryUIData], request: HttpServletRequest,
+      tableTag: String): Seq[Node] = {
+
+    val isActive = if (tableTag.contains("active")) true else false
+    val page = Option(request.getParameter(s"$tableTag.page")).map(_.toInt).getOrElse(1)
+
+    try {
+      new StreamingQueryPagedTable(
+        request,
+        parent,
+        data,
+        tableTag,
+        isActive,
+        SparkUIUtils.prependBaseUri(request, parent.basePath),
+        "StreamingQuery"
+      ).table(page)
+    } catch {
+      case e@(_: IllegalArgumentException | _: IndexOutOfBoundsException) =>
+        <div class="alert alert-error">
+          <p>Error while rendering execution table:</p>
+          <pre>
+            {Utils.exceptionString(e)}
+          </pre>
+        </div>
+    }
+  }
+}
+
+private[ui] class StreamingQueryPagedTable(
+    request: HttpServletRequest,
+    parent: StreamingQueryTab,
+    data: Seq[StreamingQueryUIData],
+    tableTag: String,
+    isActive: Boolean,
+    basePath: String,
+    subPath: String) extends PagedTable[StructuredStreamingRow] {
+
+  private val (sortColumn, sortDesc, pageSize) = getTableParameters(request, tableTag, "Start Time")
+  private val parameterPath = s"$basePath/$subPath/?${getParameterOtherTable(request, tableTag)}"
+  private val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
+
+  override def tableId: String = s"$tableTag-table"
+
+  override def tableCssClass: String =
+    "table table-bordered table-sm table-striped table-head-clickable table-cell-width-limited"
+
+  override def pageSizeFormField: String = s"$tableTag.pageSize"
+
+  override def pageNumberFormField: String = s"$tableTag.page"
+
+  override def pageLink(page: Int): String = {
+    parameterPath +
+      s"&$pageNumberFormField=$page" +
+      s"&$tableTag.sort=$encodedSortColumn" +
+      s"&$tableTag.desc=$sortDesc" +
+      s"&$pageSizeFormField=$pageSize" +
+      s"#$tableTag"
+  }
+
+  override def goButtonFormPath: String =
+    s"$parameterPath&$tableTag.sort=$encodedSortColumn&$tableTag.desc=$sortDesc#$tableTag"
+
+  override def dataSource: PagedDataSource[StructuredStreamingRow] =
+    new StreamingQueryDataSource(data, sortColumn, sortDesc, pageSize, isActive)
+
+  override def headers: Seq[Node] = {
+    val headerAndCss: Seq[(String, Boolean, Option[String])] = {
+      Seq(
+        ("Name", true, None),
+        ("Status", true, None),
+        ("ID", true, None),
+        ("Run ID", true, None),
+        ("Start Time", true, None),
+        ("Duration", true, None),
+        ("Avg Input /sec", true, None),
+        ("Avg Process /sec", true, None),
+        ("Latest Batch", true, None)) ++ {
+        if (!isActive) {
+          Seq(("Error", false, None))
+        } else {
+          Nil
+        }
+      }
+    }
+    isSortColumnValid(headerAndCss, sortColumn)
+
+    headerRow(headerAndCss, sortDesc, pageSize, sortColumn, parameterPath, tableTag, tableTag)
+  }
+
+  override def row(query: StructuredStreamingRow): Seq[Node] = {
+    val streamingQuery = query.streamingUIData
+    val statisticsLink = "%s/%s/statistics?id=%s"
+      .format(SparkUIUtils.prependBaseUri(request, parent.basePath), parent.prefix,
+        streamingQuery.summary.runId)
 
     def details(detail: Any): Seq[Node] = {
-      if (queryActive) {
+      if (isActive) {
         return Seq.empty[Node]
       }
       val detailString = detail.asInstanceOf[String]
@@ -51,93 +191,77 @@ private[ui] class StreamingQueryPage(parent: StreamingQueryTab)
       <td>{summary}{details}</td>
     }
 
-    val statisticsLink = "%s/%s/statistics?id=%s"
-      .format(SparkUIUtils.prependBaseUri(request, parent.basePath), parent.prefix, query.runId)
-
-    val name = UIUtils.getQueryName(query)
-    val status = UIUtils.getQueryStatus(query)
-    val duration = if (queryActive) {
-      SparkUIUtils.formatDurationVerbose(System.currentTimeMillis() - query.startTimestamp)
-    } else {
-      withNoProgress(query, {
-        val endTimeMs = query.lastProgress.timestamp
-        SparkUIUtils.formatDurationVerbose(parseProgressTimestamp(endTimeMs) - query.startTimestamp)
-      }, "-")
-    }
-
     <tr>
-      <td> {name} </td>
-      <td> {status} </td>
-      <td> {query.id} </td>
-      <td> <a href={statisticsLink}> {query.runId} </a> </td>
-      <td> {SparkUIUtils.formatDate(query.startTimestamp)} </td>
-      <td> {duration} </td>
-      <td> {withNoProgress(query, {
-        (query.recentProgress.map(p => withNumberInvalid(p.inputRowsPerSecond)).sum /
-          query.recentProgress.length).formatted("%.2f") }, "NaN")}
-      </td>
-      <td> {withNoProgress(query, {
-        (query.recentProgress.map(p => withNumberInvalid(p.processedRowsPerSecond)).sum /
-          query.recentProgress.length).formatted("%.2f") }, "NaN")}
-      </td>
-      <td> {withNoProgress(query, { query.lastProgress.batchId }, "NaN")} </td>
-      {details(query.exception.getOrElse("-"))}
+      <td>{UIUtils.getQueryName(streamingQuery)}</td>
+      <td>{UIUtils.getQueryStatus(streamingQuery)}</td>
+      <td>{streamingQuery.summary.id}</td>
+      <td><a href={statisticsLink}>{streamingQuery.summary.runId}</a></td>
+      <td>{SparkUIUtils.formatDate(streamingQuery.summary.startTimestamp)}</td>
+      <td>{SparkUIUtils.formatDurationVerbose(query.duration)}</td>
+      <td>{withNoProgress(streamingQuery, {query.avgInput.formatted("%.2f")}, "NaN")}</td>
+      <td>{withNoProgress(streamingQuery, {query.avgProcess.formatted("%.2f")}, "NaN")}</td>
+      <td>{withNoProgress(streamingQuery, {streamingQuery.lastProgress.batchId}, "NaN")}</td>
+      {details(streamingQuery.summary.exception.getOrElse("-"))}
     </tr>
   }
+}
 
-  private def generateStreamingQueryTable(request: HttpServletRequest): Seq[Node] = {
-    val (activeQueries, inactiveQueries) = parent.statusListener.allQueryStatus
-      .partition(_.isActive)
-    val activeQueryTables = if (activeQueries.nonEmpty) {
-      val headerRow = Seq(
-        "Name", "Status", "Id", "Run ID", "Start Time", "Duration", "Avg Input /sec",
-        "Avg Process /sec", "Lastest Batch")
-
-      Some(SparkUIUtils.listingTable(headerRow, generateDataRow(request, queryActive = true),
-        activeQueries, true, Some("activeQueries-table"), Seq(null), false))
-    } else {
-      None
-    }
+private[ui] case class StructuredStreamingRow(
+    duration: Long,
+    avgInput: Double,
+    avgProcess: Double,
+    streamingUIData: StreamingQueryUIData)
+
+private[ui] class StreamingQueryDataSource(
+    uiData: Seq[StreamingQueryUIData],
+    sortColumn: String,
+    desc: Boolean,
+    pageSize: Int,
+    isActive: Boolean) extends PagedDataSource[StructuredStreamingRow](pageSize) {
+
+  // convert StreamingQueryUIData to StreamingRow to provide required data for sorting and sort it
+  private val data = uiData.map(streamingRow).sorted(ordering(sortColumn, desc))
+
+  override def dataSize: Int = data.size
 
-    val inactiveQueryTables = if (inactiveQueries.nonEmpty) {
-      val headerRow = Seq(
-        "Name", "Status", "Id", "Run ID", "Start Time", "Duration", "Avg Input /sec",
-        "Avg Process /sec", "Lastest Batch", "Error")
+  override def sliceData(from: Int, to: Int): Seq[StructuredStreamingRow] = data.slice(from, to)
 
-      Some(SparkUIUtils.listingTable(headerRow, generateDataRow(request, queryActive = false),
-        inactiveQueries, true, Some("completedQueries-table"), Seq(null), false))
+  private def streamingRow(uiData: StreamingQueryUIData): StructuredStreamingRow = {
+    val duration = if (isActive) {
+      System.currentTimeMillis() - uiData.summary.startTimestamp
     } else {
-      None
+      withNoProgress(uiData, {
+        val endTimeMs = uiData.lastProgress.timestamp
+        parseProgressTimestamp(endTimeMs) - uiData.summary.startTimestamp
+      }, 0)
     }
 
-    // scalastyle:off
-    val content =
-      <span id="active" class="collapse-aggregated-activeQueries collapse-table"
-            onClick="collapseTable('collapse-aggregated-activeQueries','aggregated-activeQueries')">
-        <h5 id="activequeries">
-          <span class="collapse-table-arrow arrow-open"></span>
-          <a>Active Streaming Queries ({activeQueries.length})</a>
-        </h5>
-      </span> ++
-      <div>
-        <ul class="aggregated-activeQueries collapsible-table">
-          {activeQueryTables.getOrElse(Seq.empty[Node])}
-        </ul>
-      </div> ++
-      <span id="completed" class="collapse-aggregated-completedQueries collapse-table"
-            onClick="collapseTable('collapse-aggregated-completedQueries','aggregated-completedQueries')">
-        <h5 id="completedqueries">
-          <span class="collapse-table-arrow arrow-open"></span>
-          <a>Completed Streaming Queries ({inactiveQueries.length})</a>
-        </h5>
-      </span> ++
-      <div>
-        <ul class="aggregated-completedQueries collapsible-table">
-          {inactiveQueryTables.getOrElse(Seq.empty[Node])}
-        </ul>
-      </div>
-    // scalastyle:on
+    val avgInput = (uiData.recentProgress.map(p => withNumberInvalid(p.inputRowsPerSecond)).sum /
+      uiData.recentProgress.length)
 
-    content
+    val avgProcess = (uiData.recentProgress.map(p =>
+      withNumberInvalid(p.processedRowsPerSecond)).sum / uiData.recentProgress.length)
+
+    StructuredStreamingRow(duration, avgInput, avgProcess, uiData)
+  }
+
+  private def ordering(sortColumn: String, desc: Boolean): Ordering[StructuredStreamingRow] = {
+    val ordering: Ordering[StructuredStreamingRow] = sortColumn match {
+      case "Name" => Ordering.by(row => UIUtils.getQueryName(row.streamingUIData))
+      case "Status" => Ordering.by(row => UIUtils.getQueryStatus(row.streamingUIData))
+      case "ID" => Ordering.by(_.streamingUIData.summary.id)
+      case "Run ID" => Ordering.by(_.streamingUIData.summary.runId)
+      case "Start Time" => Ordering.by(_.streamingUIData.summary.startTimestamp)
+      case "Duration" => Ordering.by(_.duration)
+      case "Avg Input /sec" => Ordering.by(_.avgInput)
+      case "Avg Process /sec" => Ordering.by(_.avgProcess)
+      case "Latest Batch" => Ordering.by(_.streamingUIData.lastProgress.batchId)
+      case unknownColumn => throw new IllegalArgumentException(s"Unknown Column: $unknownColumn")
+    }
+    if (desc) {
+      ordering.reverse
+    } else {
+      ordering
+    }
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatisticsPage.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatisticsPage.scala
index 227e5e5af3983..97691d9d7e827 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatisticsPage.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatisticsPage.scala
@@ -19,18 +19,32 @@ package org.apache.spark.sql.streaming.ui
 
 import java.{util => ju}
 import java.lang.{Long => JLong}
-import java.util.UUID
+import java.util.{Locale, UUID}
 import javax.servlet.http.HttpServletRequest
 
-import scala.xml.{Node, Unparsed}
+import scala.collection.JavaConverters._
+import scala.xml.{Node, NodeBuffer, Unparsed}
 
 import org.apache.spark.internal.Logging
+import org.apache.spark.sql.execution.streaming.state.StateStoreProvider
+import org.apache.spark.sql.internal.SQLConf.STATE_STORE_PROVIDER_CLASS
+import org.apache.spark.sql.internal.StaticSQLConf.ENABLED_STREAMING_UI_CUSTOM_METRIC_LIST
 import org.apache.spark.sql.streaming.ui.UIUtils._
 import org.apache.spark.ui.{GraphUIData, JsCollector, UIUtils => SparkUIUtils, WebUIPage}
 
 private[ui] class StreamingQueryStatisticsPage(parent: StreamingQueryTab)
   extends WebUIPage("statistics") with Logging {
 
+  // State store provider implementation mustn't do any heavyweight initialiation in constructor
+  // but in its init method.
+  private val supportedCustomMetrics = StateStoreProvider.create(
+    parent.parent.conf.get(STATE_STORE_PROVIDER_CLASS)).supportedCustomMetrics
+  logDebug(s"Supported custom metrics: $supportedCustomMetrics")
+
+  private val enabledCustomMetrics =
+    parent.parent.conf.get(ENABLED_STREAMING_UI_CUSTOM_METRIC_LIST).map(_.toLowerCase(Locale.ROOT))
+  logDebug(s"Enabled custom metrics: $enabledCustomMetrics")
+
   def generateLoadResources(request: HttpServletRequest): Seq[Node] = {
     // scalastyle:off
     <script src={SparkUIUtils.prependBaseUri(request, "/static/d3.min.js")}></script>
@@ -44,8 +58,8 @@ private[ui] class StreamingQueryStatisticsPage(parent: StreamingQueryTab)
     val parameterId = request.getParameter("id")
     require(parameterId != null && parameterId.nonEmpty, "Missing id parameter")
 
-    val query = parent.statusListener.allQueryStatus.find { case q =>
-      q.runId.equals(UUID.fromString(parameterId))
+    val query = parent.store.allQueryUIData.find { uiData =>
+      uiData.summary.runId.equals(UUID.fromString(parameterId))
     }.getOrElse(throw new IllegalArgumentException(s"Failed to find streaming query $parameterId"))
 
     val resources = generateLoadResources(request)
@@ -95,37 +109,257 @@ private[ui] class StreamingQueryStatisticsPage(parent: StreamingQueryTab)
     <script>{Unparsed(js)}</script>
   }
 
-  def generateBasicInfo(query: StreamingQueryUIData): Seq[Node] = {
-    val duration = if (query.isActive) {
-      SparkUIUtils.formatDurationVerbose(System.currentTimeMillis() - query.startTimestamp)
+  def generateBasicInfo(uiData: StreamingQueryUIData): Seq[Node] = {
+    val duration = if (uiData.summary.isActive) {
+      val durationMs = System.currentTimeMillis() - uiData.summary.startTimestamp
+      SparkUIUtils.formatDurationVerbose(durationMs)
     } else {
-      withNoProgress(query, {
-        val end = query.lastProgress.timestamp
-        val start = query.recentProgress.head.timestamp
+      withNoProgress(uiData, {
+        val end = uiData.lastProgress.timestamp
+        val start = uiData.recentProgress.head.timestamp
         SparkUIUtils.formatDurationVerbose(
           parseProgressTimestamp(end) - parseProgressTimestamp(start))
       }, "-")
     }
 
-    val name = UIUtils.getQueryName(query)
-    val numBatches = withNoProgress(query, { query.lastProgress.batchId + 1L }, 0)
+    val name = UIUtils.getQueryName(uiData)
+    val numBatches = withNoProgress(uiData, { uiData.lastProgress.batchId + 1L }, 0)
     <div>Running batches for
       <strong>
         {duration}
       </strong>
       since
       <strong>
-        {SparkUIUtils.formatDate(query.startTimestamp)}
+        {SparkUIUtils.formatDate(uiData.summary.startTimestamp)}
       </strong>
       (<strong>{numBatches}</strong> completed batches)
     </div>
     <br />
     <div><strong>Name: </strong>{name}</div>
-    <div><strong>Id: </strong>{query.id}</div>
-    <div><strong>RunId: </strong>{query.runId}</div>
+    <div><strong>Id: </strong>{uiData.summary.id}</div>
+    <div><strong>RunId: </strong>{uiData.summary.runId}</div>
     <br />
   }
 
+  def generateWatermark(
+      query: StreamingQueryUIData,
+      minBatchTime: Long,
+      maxBatchTime: Long,
+      jsCollector: JsCollector): Seq[Node] = {
+    // This is made sure on caller side but put it here to be defensive
+    require(query.lastProgress != null)
+    if (query.lastProgress.eventTime.containsKey("watermark")) {
+      val watermarkData = query.recentProgress.flatMap { p =>
+        val batchTimestamp = parseProgressTimestamp(p.timestamp)
+        val watermarkValue = parseProgressTimestamp(p.eventTime.get("watermark"))
+        if (watermarkValue > 0L) {
+          // seconds
+          Some((batchTimestamp, ((batchTimestamp - watermarkValue) / 1000.0)))
+        } else {
+          None
+        }
+      }
+
+      if (watermarkData.nonEmpty) {
+        val maxWatermark = watermarkData.maxBy(_._2)._2
+        val graphUIDataForWatermark =
+          new GraphUIData(
+            "watermark-gap-timeline",
+            "watermark-gap-histogram",
+            watermarkData,
+            minBatchTime,
+            maxBatchTime,
+            0,
+            maxWatermark,
+            "seconds")
+        graphUIDataForWatermark.generateDataJs(jsCollector)
+
+        // scalastyle:off
+        <tr>
+          <td style="vertical-align: middle;">
+            <div style="width: 160px;">
+              <div><strong>Global Watermark Gap {SparkUIUtils.tooltip("The gap between batch timestamp and global watermark for the batch.", "right")}</strong></div>
+            </div>
+          </td>
+          <td class="watermark-gap-timeline">{graphUIDataForWatermark.generateTimelineHtml(jsCollector)}</td>
+          <td class="watermark-gap-histogram">{graphUIDataForWatermark.generateHistogramHtml(jsCollector)}</td>
+        </tr>
+        // scalastyle:on
+      } else {
+        Seq.empty[Node]
+      }
+    } else {
+      Seq.empty[Node]
+    }
+  }
+
+  def generateAggregatedStateOperators(
+      query: StreamingQueryUIData,
+      minBatchTime: Long,
+      maxBatchTime: Long,
+      jsCollector: JsCollector): NodeBuffer = {
+    // This is made sure on caller side but put it here to be defensive
+    require(query.lastProgress != null)
+    if (query.lastProgress.stateOperators.nonEmpty) {
+      val numRowsTotalData = query.recentProgress.map(p => (parseProgressTimestamp(p.timestamp),
+        p.stateOperators.map(_.numRowsTotal).sum.toDouble))
+      val maxNumRowsTotal = numRowsTotalData.maxBy(_._2)._2
+
+      val numRowsUpdatedData = query.recentProgress.map(p => (parseProgressTimestamp(p.timestamp),
+        p.stateOperators.map(_.numRowsUpdated).sum.toDouble))
+      val maxNumRowsUpdated = numRowsUpdatedData.maxBy(_._2)._2
+
+      val memoryUsedBytesData = query.recentProgress.map(p => (parseProgressTimestamp(p.timestamp),
+        p.stateOperators.map(_.memoryUsedBytes).sum.toDouble))
+      val maxMemoryUsedBytes = memoryUsedBytesData.maxBy(_._2)._2
+
+      val numRowsDroppedByWatermarkData = query.recentProgress
+        .map(p => (parseProgressTimestamp(p.timestamp),
+          p.stateOperators.map(_.numRowsDroppedByWatermark).sum.toDouble))
+      val maxNumRowsDroppedByWatermark = numRowsDroppedByWatermarkData.maxBy(_._2)._2
+
+      val graphUIDataForNumberTotalRows =
+        new GraphUIData(
+          "aggregated-num-total-state-rows-timeline",
+          "aggregated-num-total-state-rows-histogram",
+          numRowsTotalData,
+          minBatchTime,
+          maxBatchTime,
+          0,
+          maxNumRowsTotal,
+          "records")
+      graphUIDataForNumberTotalRows.generateDataJs(jsCollector)
+
+      val graphUIDataForNumberUpdatedRows =
+        new GraphUIData(
+          "aggregated-num-updated-state-rows-timeline",
+          "aggregated-num-updated-state-rows-histogram",
+          numRowsUpdatedData,
+          minBatchTime,
+          maxBatchTime,
+          0,
+          maxNumRowsUpdated,
+          "records")
+      graphUIDataForNumberUpdatedRows.generateDataJs(jsCollector)
+
+      val graphUIDataForMemoryUsedBytes =
+        new GraphUIData(
+          "aggregated-state-memory-used-bytes-timeline",
+          "aggregated-state-memory-used-bytes-histogram",
+          memoryUsedBytesData,
+          minBatchTime,
+          maxBatchTime,
+          0,
+          maxMemoryUsedBytes,
+          "bytes")
+      graphUIDataForMemoryUsedBytes.generateDataJs(jsCollector)
+
+      val graphUIDataForNumRowsDroppedByWatermark =
+        new GraphUIData(
+          "aggregated-num-rows-dropped-by-watermark-timeline",
+          "aggregated-num-rows-dropped-by-watermark-histogram",
+          numRowsDroppedByWatermarkData,
+          minBatchTime,
+          maxBatchTime,
+          0,
+          maxNumRowsDroppedByWatermark,
+          "records")
+      graphUIDataForNumRowsDroppedByWatermark.generateDataJs(jsCollector)
+
+      val result =
+        // scalastyle:off
+        <tr>
+          <td style="vertical-align: middle;">
+            <div style="width: 160px;">
+              <div><strong>Aggregated Number Of Total State Rows {SparkUIUtils.tooltip("Aggregated number of total state rows.", "right")}</strong></div>
+            </div>
+          </td>
+          <td class={"aggregated-num-total-state-rows-timeline"}>{graphUIDataForNumberTotalRows.generateTimelineHtml(jsCollector)}</td>
+          <td class={"aggregated-num-total-state-rows-histogram"}>{graphUIDataForNumberTotalRows.generateHistogramHtml(jsCollector)}</td>
+        </tr>
+        <tr>
+          <td style="vertical-align: middle;">
+            <div style="width: 160px;">
+              <div><strong>Aggregated Number Of Updated State Rows {SparkUIUtils.tooltip("Aggregated number of updated state rows.", "right")}</strong></div>
+            </div>
+          </td>
+          <td class={"aggregated-num-updated-state-rows-timeline"}>{graphUIDataForNumberUpdatedRows.generateTimelineHtml(jsCollector)}</td>
+          <td class={"aggregated-num-updated-state-rows-histogram"}>{graphUIDataForNumberUpdatedRows.generateHistogramHtml(jsCollector)}</td>
+        </tr>
+        <tr>
+          <td style="vertical-align: middle;">
+            <div style="width: 160px;">
+              <div><strong>Aggregated State Memory Used In Bytes {SparkUIUtils.tooltip("Aggregated state memory used in bytes.", "right")}</strong></div>
+            </div>
+          </td>
+          <td class={"aggregated-state-memory-used-bytes-timeline"}>{graphUIDataForMemoryUsedBytes.generateTimelineHtml(jsCollector)}</td>
+          <td class={"aggregated-state-memory-used-bytes-histogram"}>{graphUIDataForMemoryUsedBytes.generateHistogramHtml(jsCollector)}</td>
+        </tr>
+        <tr>
+          <td style="vertical-align: middle;">
+            <div style="width: 160px;">
+              <div><strong>Aggregated Number Of Rows Dropped By Watermark {SparkUIUtils.tooltip("Accumulates all input rows being dropped in stateful operators by watermark. 'Inputs' are relative to operators.", "right")}</strong></div>
+            </div>
+          </td>
+          <td class={"aggregated-num-rows-dropped-by-watermark-timeline"}>{graphUIDataForNumRowsDroppedByWatermark.generateTimelineHtml(jsCollector)}</td>
+          <td class={"aggregated-num-rows-dropped-by-watermark-histogram"}>{graphUIDataForNumRowsDroppedByWatermark.generateHistogramHtml(jsCollector)}</td>
+        </tr>
+        // scalastyle:on
+
+      if (enabledCustomMetrics.nonEmpty) {
+        result ++= generateAggregatedCustomMetrics(query, minBatchTime, maxBatchTime, jsCollector)
+      }
+      result
+    } else {
+      new NodeBuffer()
+    }
+  }
+
+  def generateAggregatedCustomMetrics(
+      query: StreamingQueryUIData,
+      minBatchTime: Long,
+      maxBatchTime: Long,
+      jsCollector: JsCollector): NodeBuffer = {
+    val result: NodeBuffer = new NodeBuffer
+
+    // This is made sure on caller side but put it here to be defensive
+    require(query.lastProgress.stateOperators.nonEmpty)
+    query.lastProgress.stateOperators.head.customMetrics.keySet().asScala
+      .filter(m => enabledCustomMetrics.contains(m.toLowerCase(Locale.ROOT))).map { metricName =>
+        val data = query.recentProgress.map(p => (parseProgressTimestamp(p.timestamp),
+          p.stateOperators.map(_.customMetrics.get(metricName).toDouble).sum))
+        val max = data.maxBy(_._2)._2
+        val metric = supportedCustomMetrics.find(_.name.equalsIgnoreCase(metricName)).get
+
+        val graphUIData =
+          new GraphUIData(
+            s"aggregated-$metricName-timeline",
+            s"aggregated-$metricName-histogram",
+            data,
+            minBatchTime,
+            maxBatchTime,
+            0,
+            max,
+            "")
+        graphUIData.generateDataJs(jsCollector)
+
+        result ++=
+          // scalastyle:off
+          <tr>
+            <td style="vertical-align: middle;">
+              <div style="width: 240px;">
+                <div><strong>Aggregated Custom Metric {s"$metricName"} {SparkUIUtils.tooltip(metric.desc, "right")}</strong></div>
+              </div>
+            </td>
+            <td class={s"aggregated-$metricName-timeline"}>{graphUIData.generateTimelineHtml(jsCollector)}</td>
+            <td class={s"aggregated-$metricName-histogram"}>{graphUIData.generateHistogramHtml(jsCollector)}</td>
+          </tr>
+          // scalastyle:on
+      }
+
+    result
+  }
+
   def generateStatTable(query: StreamingQueryUIData): Seq[Node] = {
     val batchToTimestamps = withNoProgress(query,
       query.recentProgress.map(p => (p.batchId, parseProgressTimestamp(p.timestamp))),
@@ -284,6 +518,8 @@ private[ui] class StreamingQueryStatisticsPage(parent: StreamingQueryTab)
             </td>
             <td class="duration-area-stack" colspan="2">{graphUIDataForDuration.generateAreaStackHtmlWithData(jsCollector, operationDurationData)}</td>
           </tr>
+          {generateWatermark(query, minBatchTime, maxBatchTime, jsCollector)}
+          {generateAggregatedStateOperators(query, minBatchTime, maxBatchTime, jsCollector)}
         </tbody>
       </table>
     } else {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListener.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListener.scala
index e331083b30024..fdd3754344108 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListener.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListener.scala
@@ -20,102 +20,144 @@ package org.apache.spark.sql.streaming.ui
 import java.util.UUID
 import java.util.concurrent.ConcurrentHashMap
 
-import scala.collection.JavaConverters._
 import scala.collection.mutable
 
+import com.fasterxml.jackson.annotation.JsonIgnore
+
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.internal.StaticSQLConf
 import org.apache.spark.sql.streaming.{StreamingQueryListener, StreamingQueryProgress}
+import org.apache.spark.sql.streaming.ui.StreamingQueryProgressWrapper._
 import org.apache.spark.sql.streaming.ui.UIUtils.parseProgressTimestamp
+import org.apache.spark.status.{ElementTrackingStore, KVUtils}
+import org.apache.spark.status.KVUtils.KVIndexParam
+import org.apache.spark.util.kvstore.KVIndex
 
 /**
  * A customized StreamingQueryListener used in structured streaming UI, which contains all
  * UI data for both active and inactive query.
- * TODO: Add support for history server.
  */
-private[sql] class StreamingQueryStatusListener(conf: SparkConf) extends StreamingQueryListener {
-
-  /**
-   * We use runId as the key here instead of id in active query status map,
-   * because the runId is unique for every started query, even it its a restart.
-   */
-  private[ui] val activeQueryStatus = new ConcurrentHashMap[UUID, StreamingQueryUIData]()
-  private[ui] val inactiveQueryStatus = new mutable.Queue[StreamingQueryUIData]()
+private[sql] class StreamingQueryStatusListener(
+    conf: SparkConf,
+    store: ElementTrackingStore) extends StreamingQueryListener {
 
   private val streamingProgressRetention =
     conf.get(StaticSQLConf.STREAMING_UI_RETAINED_PROGRESS_UPDATES)
   private val inactiveQueryStatusRetention = conf.get(StaticSQLConf.STREAMING_UI_RETAINED_QUERIES)
 
+  store.addTrigger(classOf[StreamingQueryData], inactiveQueryStatusRetention) { count =>
+    cleanupInactiveQueries(count)
+  }
+
+  // Events from the same query run will never be processed concurrently, so it's safe to
+  // access `progressIds` without any protection.
+  private val queryToProgress = new ConcurrentHashMap[UUID, mutable.Queue[String]]()
+
+  private def cleanupInactiveQueries(count: Long): Unit = {
+    val view = store.view(classOf[StreamingQueryData]).index("active").first(false).last(false)
+    val inactiveQueries = KVUtils.viewToSeq(view, Int.MaxValue)(_ => true)
+    val numInactiveQueries = inactiveQueries.size
+    if (numInactiveQueries <= inactiveQueryStatusRetention) {
+      return
+    }
+    val toDelete = inactiveQueries.sortBy(_.endTimestamp.get)
+      .take(numInactiveQueries - inactiveQueryStatusRetention)
+    val runIds = toDelete.map { e =>
+      store.delete(e.getClass, e.runId)
+      e.runId.toString
+    }
+    // Delete wrappers in one pass, as deleting them for each summary is slow
+    store.removeAllByIndexValues(classOf[StreamingQueryProgressWrapper], "runId", runIds)
+  }
+
   override def onQueryStarted(event: StreamingQueryListener.QueryStartedEvent): Unit = {
     val startTimestamp = parseProgressTimestamp(event.timestamp)
-    activeQueryStatus.putIfAbsent(event.runId,
-      new StreamingQueryUIData(event.name, event.id, event.runId, startTimestamp))
+    store.write(new StreamingQueryData(
+      event.name,
+      event.id,
+      event.runId,
+      isActive = true,
+      None,
+      startTimestamp
+    ), checkTriggers = true)
   }
 
   override def onQueryProgress(event: StreamingQueryListener.QueryProgressEvent): Unit = {
-    val batchTimestamp = parseProgressTimestamp(event.progress.timestamp)
-    val queryStatus = activeQueryStatus.getOrDefault(
-      event.progress.runId,
-      new StreamingQueryUIData(event.progress.name, event.progress.id, event.progress.runId,
-        batchTimestamp))
-    queryStatus.updateProcess(event.progress, streamingProgressRetention)
-  }
-
-  override def onQueryTerminated(
-      event: StreamingQueryListener.QueryTerminatedEvent): Unit = synchronized {
-    val queryStatus = activeQueryStatus.remove(event.runId)
-    if (queryStatus != null) {
-      queryStatus.queryTerminated(event)
-      inactiveQueryStatus += queryStatus
-      while (inactiveQueryStatus.length >= inactiveQueryStatusRetention) {
-        inactiveQueryStatus.dequeue()
-      }
+    val runId = event.progress.runId
+    val batchId = event.progress.batchId
+    val timestamp = event.progress.timestamp
+    if (!queryToProgress.containsKey(runId)) {
+      queryToProgress.put(runId, mutable.Queue.empty[String])
+    }
+    val progressIds = queryToProgress.get(runId)
+    progressIds.enqueue(getUniqueId(runId, batchId, timestamp))
+    store.write(new StreamingQueryProgressWrapper(event.progress))
+    while (progressIds.length > streamingProgressRetention) {
+      val uniqueId = progressIds.dequeue
+      store.delete(classOf[StreamingQueryProgressWrapper], uniqueId)
     }
   }
 
-  def allQueryStatus: Seq[StreamingQueryUIData] = synchronized {
-    activeQueryStatus.values().asScala.toSeq ++ inactiveQueryStatus
+  override def onQueryTerminated(
+      event: StreamingQueryListener.QueryTerminatedEvent): Unit = {
+    val querySummary = store.read(classOf[StreamingQueryData], event.runId)
+    val curTime = System.currentTimeMillis()
+    store.write(new StreamingQueryData(
+      querySummary.name,
+      querySummary.id,
+      querySummary.runId,
+      isActive = false,
+      querySummary.exception,
+      querySummary.startTimestamp,
+      Some(curTime)
+    ), checkTriggers = true)
+    queryToProgress.remove(event.runId)
   }
 }
 
+private[sql] class StreamingQueryData(
+    val name: String,
+    val id: UUID,
+    @KVIndexParam val runId: UUID,
+    @KVIndexParam("active") val isActive: Boolean,
+    val exception: Option[String],
+    @KVIndexParam("startTimestamp") val startTimestamp: Long,
+    val endTimestamp: Option[Long] = None)
+
 /**
  * This class contains all message related to UI display, each instance corresponds to a single
  * [[org.apache.spark.sql.streaming.StreamingQuery]].
  */
-private[ui] class StreamingQueryUIData(
-    val name: String,
-    val id: UUID,
-    val runId: UUID,
-    val startTimestamp: Long) {
-
-  /** Holds the most recent query progress updates. */
-  private val progressBuffer = new mutable.Queue[StreamingQueryProgress]()
-
-  private var _isActive = true
-  private var _exception: Option[String] = None
-
-  def isActive: Boolean = synchronized { _isActive }
-
-  def exception: Option[String] = synchronized { _exception }
-
-  def queryTerminated(event: StreamingQueryListener.QueryTerminatedEvent): Unit = synchronized {
-    _isActive = false
-    _exception = event.exception
-  }
-
-  def updateProcess(
-      newProgress: StreamingQueryProgress, retentionNum: Int): Unit = progressBuffer.synchronized {
-    progressBuffer += newProgress
-    while (progressBuffer.length >= retentionNum) {
-      progressBuffer.dequeue()
+private[sql] case class StreamingQueryUIData(
+    summary: StreamingQueryData,
+    recentProgress: Array[StreamingQueryProgress]) {
+
+  def lastProgress: StreamingQueryProgress = {
+    if (recentProgress.nonEmpty) {
+      recentProgress.last
+    } else {
+      null
     }
   }
+}
 
-  def recentProgress: Array[StreamingQueryProgress] = progressBuffer.synchronized {
-    progressBuffer.toArray
-  }
+private[sql] class StreamingQueryProgressWrapper(val progress: StreamingQueryProgress) {
+  @JsonIgnore @KVIndex
+  private val uniqueId: String = getUniqueId(progress.runId, progress.batchId, progress.timestamp)
 
-  def lastProgress: StreamingQueryProgress = progressBuffer.synchronized {
-    progressBuffer.lastOption.orNull
+  @JsonIgnore @KVIndex("runId")
+  private def runIdIndex: String = progress.runId.toString
+}
+
+private[sql] object StreamingQueryProgressWrapper {
+  /**
+   * Adding `timestamp` into unique id to support reporting `empty` query progress
+   * in which no data comes but with the same batchId.
+   */
+  def getUniqueId(
+      runId: UUID,
+      batchId: Long,
+      timestamp: String): String = {
+    s"${runId}_${batchId}_$timestamp"
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryTab.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryTab.scala
index bb097ffc06912..65cad8f06cc1c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryTab.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryTab.scala
@@ -17,10 +17,11 @@
 package org.apache.spark.sql.streaming.ui
 
 import org.apache.spark.internal.Logging
+import org.apache.spark.sql.execution.ui.StreamingQueryStatusStore
 import org.apache.spark.ui.{SparkUI, SparkUITab}
 
 private[sql] class StreamingQueryTab(
-    val statusListener: StreamingQueryStatusListener,
+    val store: StreamingQueryStatusStore,
     sparkUI: SparkUI) extends SparkUITab(sparkUI, "StreamingQuery") with Logging {
 
   override val name = "Structured Streaming"
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/UIUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/UIUtils.scala
index cdad5ed9942b5..88a110fa9a329 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/UIUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/UIUtils.scala
@@ -18,7 +18,6 @@
 package org.apache.spark.sql.streaming.ui
 
 import java.text.SimpleDateFormat
-import java.util.Locale
 
 import org.apache.spark.sql.catalyst.util.DateTimeUtils.getTimeZone
 
@@ -47,19 +46,19 @@ private[ui] object UIUtils {
     }
   }
 
-  def getQueryName(query: StreamingQueryUIData): String = {
-    if (query.name == null || query.name.isEmpty) {
+  def getQueryName(uiData: StreamingQueryUIData): String = {
+    if (uiData.summary.name == null || uiData.summary.name.isEmpty) {
       "<no name>"
     } else {
-      query.name
+      uiData.summary.name
     }
   }
 
-  def getQueryStatus(query: StreamingQueryUIData): String = {
-    if (query.isActive) {
+  def getQueryStatus(uiData: StreamingQueryUIData): String = {
+    if (uiData.summary.isActive) {
       "RUNNING"
     } else {
-      query.exception.map(_ => "FAILED").getOrElse("FINISHED")
+      uiData.summary.exception.map(_ => "FAILED").getOrElse("FINISHED")
     }
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/util/QueryExecutionListener.scala b/sql/core/src/main/scala/org/apache/spark/sql/util/QueryExecutionListener.scala
index 0b5951ec2ac97..b1742078cc125 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/util/QueryExecutionListener.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/util/QueryExecutionListener.scala
@@ -76,7 +76,7 @@ trait QueryExecutionListener {
 class ExecutionListenerManager private[sql](session: SparkSession, loadExtensions: Boolean)
   extends Logging {
 
-  private val listenerBus = new ExecutionListenerBus(session)
+  private val listenerBus = new ExecutionListenerBus(this, session)
 
   if (loadExtensions) {
     val conf = session.sparkContext.conf
@@ -124,10 +124,16 @@ class ExecutionListenerManager private[sql](session: SparkSession, loadExtension
   }
 }
 
-private[sql] class ExecutionListenerBus(session: SparkSession)
+private[sql] class ExecutionListenerBus private(sessionUUID: String)
   extends SparkListener with ListenerBus[QueryExecutionListener, SparkListenerSQLExecutionEnd] {
 
-  session.sparkContext.listenerBus.addToSharedQueue(this)
+  def this(manager: ExecutionListenerManager, session: SparkSession) = {
+    this(session.sessionUUID)
+    session.sparkContext.listenerBus.addToSharedQueue(this)
+    session.sparkContext.cleaner.foreach { cleaner =>
+      cleaner.registerSparkListenerForCleanup(manager, this)
+    }
+  }
 
   override def onOtherEvent(event: SparkListenerEvent): Unit = event match {
     case e: SparkListenerSQLExecutionEnd => postToAll(e)
@@ -158,6 +164,6 @@ private[sql] class ExecutionListenerBus(session: SparkSession)
   private def shouldReport(e: SparkListenerSQLExecutionEnd): Boolean = {
     // Only catch SQL execution with a name, and triggered by the same spark session that this
     // listener manager belongs.
-    e.executionName.isDefined && e.qe.sparkSession.eq(this.session)
+    e.executionName.isDefined && e.qe.sparkSession.sessionUUID == sessionUUID
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/ApiSqlRootResource.scala b/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/ApiSqlRootResource.scala
index 5fc7123c9097b..747c05b9b0626 100644
--- a/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/ApiSqlRootResource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/ApiSqlRootResource.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.status.api.v1.sql
 
-import javax.ws.rs.Path
+import javax.ws.rs.{Path, PathParam}
 
 import org.apache.spark.status.api.v1.ApiRequestContext
 
@@ -25,5 +25,10 @@ import org.apache.spark.status.api.v1.ApiRequestContext
 private[v1] class ApiSqlRootResource extends ApiRequestContext {
 
   @Path("applications/{appId}/sql")
-  def sqlList(): Class[SqlResource] = classOf[SqlResource]
+  def sqlList(@PathParam("appId") appId: String): Class[SqlResource] = classOf[SqlResource]
+
+  @Path("applications/{appId}/{attemptId}/sql")
+  def sqlList(
+      @PathParam("appId") appId: String,
+      @PathParam("attemptId") attemptId: String): Class[SqlResource] = classOf[SqlResource]
 }
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/Java8DatasetAggregatorSuite.java b/sql/core/src/test/java/test/org/apache/spark/sql/Java8DatasetAggregatorSuite.java
index dd3755d3f904e..de88f80eb53b8 100644
--- a/sql/core/src/test/java/test/org/apache/spark/sql/Java8DatasetAggregatorSuite.java
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/Java8DatasetAggregatorSuite.java
@@ -34,43 +34,43 @@ public class Java8DatasetAggregatorSuite extends JavaDatasetAggregatorSuiteBase
   @Test
   public void testTypedAggregationAverage() {
     KeyValueGroupedDataset<String, Tuple2<String, Integer>> grouped = generateGroupedDataset();
-    Dataset<Tuple2<String, Double>> agged = grouped.agg(
+    Dataset<Tuple2<String, Double>> aggregated = grouped.agg(
       org.apache.spark.sql.expressions.javalang.typed.avg(v -> (double)(v._2() * 2)));
     Assert.assertEquals(
         Arrays.asList(new Tuple2<>("a", 3.0), new Tuple2<>("b", 6.0)),
-        agged.collectAsList());
+        aggregated.collectAsList());
   }
 
   @SuppressWarnings("deprecation")
   @Test
   public void testTypedAggregationCount() {
     KeyValueGroupedDataset<String, Tuple2<String, Integer>> grouped = generateGroupedDataset();
-    Dataset<Tuple2<String, Long>> agged = grouped.agg(
+    Dataset<Tuple2<String, Long>> aggregated = grouped.agg(
       org.apache.spark.sql.expressions.javalang.typed.count(v -> v));
     Assert.assertEquals(
         Arrays.asList(new Tuple2<>("a", 2L), new Tuple2<>("b", 1L)),
-        agged.collectAsList());
+        aggregated.collectAsList());
   }
 
   @SuppressWarnings("deprecation")
   @Test
   public void testTypedAggregationSumDouble() {
     KeyValueGroupedDataset<String, Tuple2<String, Integer>> grouped = generateGroupedDataset();
-    Dataset<Tuple2<String, Double>> agged = grouped.agg(
+    Dataset<Tuple2<String, Double>> aggregated = grouped.agg(
       org.apache.spark.sql.expressions.javalang.typed.sum(v -> (double)v._2()));
     Assert.assertEquals(
         Arrays.asList(new Tuple2<>("a", 3.0), new Tuple2<>("b", 3.0)),
-        agged.collectAsList());
+        aggregated.collectAsList());
   }
 
   @SuppressWarnings("deprecation")
   @Test
   public void testTypedAggregationSumLong() {
     KeyValueGroupedDataset<String, Tuple2<String, Integer>> grouped = generateGroupedDataset();
-    Dataset<Tuple2<String, Long>> agged = grouped.agg(
+    Dataset<Tuple2<String, Long>> aggregated = grouped.agg(
       org.apache.spark.sql.expressions.javalang.typed.sumLong(v -> (long)v._2()));
     Assert.assertEquals(
         Arrays.asList(new Tuple2<>("a", 3L), new Tuple2<>("b", 3L)),
-        agged.collectAsList());
+        aggregated.collectAsList());
   }
 }
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/JavaBeanDeserializationSuite.java b/sql/core/src/test/java/test/org/apache/spark/sql/JavaBeanDeserializationSuite.java
index 5603cb988b8e7..af0a22b036030 100644
--- a/sql/core/src/test/java/test/org/apache/spark/sql/JavaBeanDeserializationSuite.java
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/JavaBeanDeserializationSuite.java
@@ -18,6 +18,8 @@
 package test.org.apache.spark.sql;
 
 import java.io.Serializable;
+import java.sql.Timestamp;
+import java.text.SimpleDateFormat;
 import java.time.Instant;
 import java.time.LocalDate;
 import java.util.*;
@@ -210,6 +212,17 @@ private static Row createRecordSpark22000Row(Long index) {
     return new GenericRow(values);
   }
 
+  private static String timestampToString(Timestamp ts) {
+    String timestampString = String.valueOf(ts);
+    String formatted = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(ts);
+
+    if (timestampString.length() > 19 && !timestampString.substring(19).equals(".0")) {
+      return formatted + timestampString.substring(19);
+    } else {
+      return formatted;
+    }
+  }
+
   private static RecordSpark22000 createRecordSpark22000(Row recordRow) {
     RecordSpark22000 record = new RecordSpark22000();
     record.setShortField(String.valueOf(recordRow.getShort(0)));
@@ -219,7 +232,7 @@ private static RecordSpark22000 createRecordSpark22000(Row recordRow) {
     record.setDoubleField(String.valueOf(recordRow.getDouble(4)));
     record.setStringField(recordRow.getString(5));
     record.setBooleanField(String.valueOf(recordRow.getBoolean(6)));
-    record.setTimestampField(String.valueOf(recordRow.getTimestamp(7)));
+    record.setTimestampField(timestampToString(recordRow.getTimestamp(7)));
     // This would figure out that null value will not become "null".
     record.setNullIntField(null);
     return record;
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDataFrameSuite.java b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDataFrameSuite.java
index f4bffd9d79828..da7c62251b385 100644
--- a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDataFrameSuite.java
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDataFrameSuite.java
@@ -105,7 +105,7 @@ public void testVarargMethods() {
 
     // Varargs in column expressions
     df.groupBy().agg(countDistinct("key", "value"));
-    df.groupBy().agg(countDistinct(col("key"), col("value")));
+    df.groupBy().agg(count_distinct(col("key"), col("value")));
     df.select(coalesce(col("key")));
 
     // Varargs with mathfunctions
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDataFrameWriterV2Suite.java b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDataFrameWriterV2Suite.java
index e418958bef94d..59c5263563b27 100644
--- a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDataFrameWriterV2Suite.java
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDataFrameWriterV2Suite.java
@@ -23,7 +23,7 @@
 import org.apache.spark.sql.catalyst.analysis.CannotReplaceMissingTableException;
 import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
 import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException;
-import org.apache.spark.sql.connector.InMemoryTableCatalog;
+import org.apache.spark.sql.connector.catalog.InMemoryTableCatalog;
 import org.apache.spark.sql.test.TestSparkSession;
 import org.apache.spark.sql.types.StructType;
 import org.junit.After;
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetAggregatorSuite.java b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetAggregatorSuite.java
index 8a90624f2070b..979b7751fa9a8 100644
--- a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetAggregatorSuite.java
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetAggregatorSuite.java
@@ -38,18 +38,18 @@ public class JavaDatasetAggregatorSuite extends JavaDatasetAggregatorSuiteBase {
   public void testTypedAggregationAnonClass() {
     KeyValueGroupedDataset<String, Tuple2<String, Integer>> grouped = generateGroupedDataset();
 
-    Dataset<Tuple2<String, Integer>> agged = grouped.agg(new IntSumOf().toColumn());
+    Dataset<Tuple2<String, Integer>> aggregated = grouped.agg(new IntSumOf().toColumn());
     Assert.assertEquals(
         Arrays.asList(new Tuple2<>("a", 3), new Tuple2<>("b", 3)),
-        agged.collectAsList());
+        aggregated.collectAsList());
 
-    Dataset<Tuple2<String, Integer>> agged2 = grouped.agg(new IntSumOf().toColumn())
+    Dataset<Tuple2<String, Integer>> aggregated2 = grouped.agg(new IntSumOf().toColumn())
       .as(Encoders.tuple(Encoders.STRING(), Encoders.INT()));
     Assert.assertEquals(
       Arrays.asList(
         new Tuple2<>("a", 3),
         new Tuple2<>("b", 3)),
-      agged2.collectAsList());
+      aggregated2.collectAsList());
   }
 
   static class IntSumOf extends Aggregator<Tuple2<String, Integer>, Integer, Integer> {
@@ -88,43 +88,43 @@ public Encoder<Integer> outputEncoder() {
   @Test
   public void testTypedAggregationAverage() {
     KeyValueGroupedDataset<String, Tuple2<String, Integer>> grouped = generateGroupedDataset();
-    Dataset<Tuple2<String, Double>> agged = grouped.agg(
+    Dataset<Tuple2<String, Double>> aggregated = grouped.agg(
       org.apache.spark.sql.expressions.javalang.typed.avg(value -> value._2() * 2.0));
     Assert.assertEquals(
         Arrays.asList(new Tuple2<>("a", 3.0), new Tuple2<>("b", 6.0)),
-        agged.collectAsList());
+        aggregated.collectAsList());
   }
 
   @SuppressWarnings("deprecation")
   @Test
   public void testTypedAggregationCount() {
     KeyValueGroupedDataset<String, Tuple2<String, Integer>> grouped = generateGroupedDataset();
-    Dataset<Tuple2<String, Long>> agged = grouped.agg(
+    Dataset<Tuple2<String, Long>> aggregated = grouped.agg(
       org.apache.spark.sql.expressions.javalang.typed.count(value -> value));
     Assert.assertEquals(
         Arrays.asList(new Tuple2<>("a", 2L), new Tuple2<>("b", 1L)),
-        agged.collectAsList());
+        aggregated.collectAsList());
   }
 
   @SuppressWarnings("deprecation")
   @Test
   public void testTypedAggregationSumDouble() {
     KeyValueGroupedDataset<String, Tuple2<String, Integer>> grouped = generateGroupedDataset();
-    Dataset<Tuple2<String, Double>> agged = grouped.agg(
+    Dataset<Tuple2<String, Double>> aggregated = grouped.agg(
       org.apache.spark.sql.expressions.javalang.typed.sum(value -> (double) value._2()));
     Assert.assertEquals(
         Arrays.asList(new Tuple2<>("a", 3.0), new Tuple2<>("b", 3.0)),
-        agged.collectAsList());
+        aggregated.collectAsList());
   }
 
   @SuppressWarnings("deprecation")
   @Test
   public void testTypedAggregationSumLong() {
     KeyValueGroupedDataset<String, Tuple2<String, Integer>> grouped = generateGroupedDataset();
-    Dataset<Tuple2<String, Long>> agged = grouped.agg(
+    Dataset<Tuple2<String, Long>> aggregated = grouped.agg(
       org.apache.spark.sql.expressions.javalang.typed.sumLong(value -> (long) value._2()));
     Assert.assertEquals(
         Arrays.asList(new Tuple2<>("a", 3L), new Tuple2<>("b", 3L)),
-        agged.collectAsList());
+        aggregated.collectAsList());
   }
 }
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java
index 05c4a04b20b17..3e988c2a2390e 100644
--- a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java
@@ -21,8 +21,10 @@
 import java.math.BigDecimal;
 import java.sql.Date;
 import java.sql.Timestamp;
+import java.time.Duration;
 import java.time.Instant;
 import java.time.LocalDate;
+import java.time.Period;
 import java.util.*;
 import javax.annotation.Nonnull;
 
@@ -35,7 +37,6 @@
 
 import com.google.common.base.Objects;
 import org.junit.*;
-import org.junit.rules.ExpectedException;
 
 import org.apache.spark.api.java.JavaPairRDD;
 import org.apache.spark.api.java.JavaSparkContext;
@@ -412,6 +413,22 @@ public void testLocalDateAndInstantEncoders() {
     Assert.assertEquals(data, ds.collectAsList());
   }
 
+  @Test
+  public void testDurationEncoder() {
+    Encoder<Duration> encoder = Encoders.DURATION();
+    List<Duration> data = Arrays.asList(Duration.ofDays(0));
+    Dataset<Duration> ds = spark.createDataset(data, encoder);
+    Assert.assertEquals(data, ds.collectAsList());
+  }
+
+  @Test
+  public void testPeriodEncoder() {
+    Encoder<Period> encoder = Encoders.PERIOD();
+    List<Period> data = Arrays.asList(Period.ofYears(10));
+    Dataset<Period> ds = spark.createDataset(data, encoder);
+    Assert.assertEquals(data, ds.collectAsList());
+  }
+
   public static class KryoSerializable {
     String value;
 
@@ -893,9 +910,6 @@ public int hashCode() {
     }
   }
 
-  @Rule
-  public transient ExpectedException nullabilityCheck = ExpectedException.none();
-
   @Test
   public void testRuntimeNullabilityCheck() {
     OuterScopes.addOuterScope(this);
@@ -937,9 +951,6 @@ public void testRuntimeNullabilityCheck() {
       Assert.assertEquals(Collections.singletonList(nestedSmallBean), ds.collectAsList());
     }
 
-    nullabilityCheck.expect(RuntimeException.class);
-    nullabilityCheck.expectMessage("Null value appeared in non-nullable field");
-
     {
       Row row = new GenericRow(new Object[] {
           new GenericRow(new Object[] {
@@ -950,7 +961,8 @@ public void testRuntimeNullabilityCheck() {
       Dataset<Row> df = spark.createDataFrame(Collections.singletonList(row), schema);
       Dataset<NestedSmallBean> ds = df.as(Encoders.bean(NestedSmallBean.class));
 
-      ds.collect();
+      Assert.assertThrows("Null value appeared in non-nullable field", RuntimeException.class,
+        ds::collect);
     }
   }
 
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDateFunctionsSuite.java b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDateFunctionsSuite.java
new file mode 100644
index 0000000000000..2d1de77699508
--- /dev/null
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDateFunctionsSuite.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test.org.apache.spark.sql;
+
+import org.apache.spark.sql.Column;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.RowFactory;
+import org.apache.spark.sql.test.TestSparkSession;
+import org.apache.spark.sql.types.StructType;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.sql.Date;
+import java.util.*;
+
+import static org.apache.spark.sql.types.DataTypes.*;
+import static org.apache.spark.sql.functions.*;
+
+public class JavaDateFunctionsSuite {
+  private transient TestSparkSession spark;
+
+  @Before
+  public void setUp() {
+        spark = new TestSparkSession();
+    }
+
+  @After
+  public void tearDown() {
+    spark.stop();
+    spark = null;
+  }
+
+  @Test
+  public void makeIntervalWorksWithJava() {
+    Column twoYears = make_interval(lit(2), lit(0), lit(0), lit(0), lit(0), lit(0), lit(0));
+    List<Row> rows = Arrays.asList(
+      RowFactory.create(Date.valueOf("2014-06-30"), Date.valueOf("2016-06-30")),
+      RowFactory.create(Date.valueOf("2015-05-01"), Date.valueOf("2017-05-01")),
+      RowFactory.create(Date.valueOf("2018-12-30"), Date.valueOf("2020-12-30")));
+    StructType schema = createStructType(Arrays.asList(
+      createStructField("some_date", DateType, false),
+      createStructField("expected", DateType, false)));
+    Dataset<Row> df = spark.createDataFrame(rows, schema)
+            .withColumn("plus_two_years", col("some_date").plus(twoYears));
+    Assert.assertTrue(Arrays.equals(
+      (Row[]) df.select(df.col("plus_two_years")).collect(),
+      (Row[]) df.select(df.col("expected")).collect()));
+  }
+
+}
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/connector/JavaSimpleBatchTable.java b/sql/core/src/test/java/test/org/apache/spark/sql/connector/JavaSimpleBatchTable.java
index 71cf97b56fe54..4d147ac14a522 100644
--- a/sql/core/src/test/java/test/org/apache/spark/sql/connector/JavaSimpleBatchTable.java
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/connector/JavaSimpleBatchTable.java
@@ -17,7 +17,7 @@
 
 package test.org.apache.spark.sql.connector;
 
-import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
@@ -28,11 +28,8 @@
 import org.apache.spark.sql.types.StructType;
 
 abstract class JavaSimpleBatchTable implements Table, SupportsRead {
-  private static final Set<TableCapability> CAPABILITIES = new HashSet<>(Arrays.asList(
-      TableCapability.BATCH_READ,
-      TableCapability.BATCH_WRITE,
-      TableCapability.TRUNCATE));
-
+  private static final Set<TableCapability> CAPABILITIES =
+      new HashSet<>(Collections.singletonList(TableCapability.BATCH_READ));
   @Override
   public StructType schema() {
     return TestingV2Source.schema();
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/connector/JavaSimpleWritableDataSource.java b/sql/core/src/test/java/test/org/apache/spark/sql/connector/JavaSimpleWritableDataSource.java
new file mode 100644
index 0000000000000..74140d707c25a
--- /dev/null
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/connector/JavaSimpleWritableDataSource.java
@@ -0,0 +1,365 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test.org.apache.spark.sql.connector;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.*;
+
+import org.apache.spark.deploy.SparkHadoopUtil;
+import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.catalyst.expressions.GenericInternalRow;
+import org.apache.spark.sql.connector.SimpleCounter;
+import org.apache.spark.sql.connector.TestingV2Source;
+import org.apache.spark.sql.connector.catalog.SupportsWrite;
+import org.apache.spark.sql.connector.catalog.Table;
+import org.apache.spark.sql.connector.catalog.TableCapability;
+import org.apache.spark.sql.connector.read.InputPartition;
+import org.apache.spark.sql.connector.read.PartitionReader;
+import org.apache.spark.sql.connector.read.PartitionReaderFactory;
+import org.apache.spark.sql.connector.read.ScanBuilder;
+import org.apache.spark.sql.connector.write.*;
+import org.apache.spark.sql.util.CaseInsensitiveStringMap;
+import org.apache.spark.util.SerializableConfiguration;
+
+/**
+ * A HDFS based transactional writable data source which is implemented by java.
+ * Each task writes data to `target/_temporary/uniqueId/$jobId-$partitionId-$attemptNumber`.
+ * Each job moves files from `target/_temporary/uniqueId/` to `target`.
+ */
+public class JavaSimpleWritableDataSource implements TestingV2Source {
+
+  static class MyScanBuilder extends JavaSimpleScanBuilder {
+
+    private final String path;
+    private final Configuration conf;
+
+    MyScanBuilder(String path, Configuration conf) {
+      this.path = path;
+      this.conf = conf;
+    }
+
+    @Override
+    public InputPartition[] planInputPartitions() {
+      Path dataPath = new Path(this.path);
+      try {
+        FileSystem fs = dataPath.getFileSystem(conf);
+        if (fs.exists(dataPath)) {
+          return Arrays.stream(fs.listStatus(dataPath))
+              .filter(
+                  status -> {
+                    String name = status.getPath().getName();
+                    return !name.startsWith("_") && !name.startsWith(".");
+                  })
+              .map(f -> new JavaCSVInputPartitionReader(f.getPath().toUri().toString()))
+              .toArray(InputPartition[]::new);
+        } else {
+          return new InputPartition[0];
+        }
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
+
+    @Override
+    public PartitionReaderFactory createReaderFactory() {
+      SerializableConfiguration serializableConf = new SerializableConfiguration(conf);
+      return new JavaCSVReaderFactory(serializableConf);
+    }
+  }
+
+  static class MyWriteBuilder implements WriteBuilder, SupportsTruncate {
+
+    private final String path;
+    private final String queryId;
+    private boolean needTruncate = false;
+
+    MyWriteBuilder(String path, LogicalWriteInfo info) {
+      this.path = path;
+      this.queryId = info.queryId();
+    }
+
+    @Override
+    public WriteBuilder truncate() {
+      this.needTruncate = true;
+      return this;
+    }
+
+    @Override
+    public Write build() {
+      return new MyWrite(path, queryId, needTruncate);
+    }
+  }
+
+  static class MyWrite implements Write {
+
+    private final String path;
+    private final String queryId;
+    private final boolean needTruncate;
+
+    MyWrite(String path, String queryId, boolean needTruncate) {
+      this.path = path;
+      this.queryId = queryId;
+      this.needTruncate = needTruncate;
+    }
+
+    @Override
+    public BatchWrite toBatch() {
+      Path hadoopPath = new Path(path);
+      Configuration hadoopConf = SparkHadoopUtil.get().conf();
+      try {
+        FileSystem fs = hadoopPath.getFileSystem(hadoopConf);
+        if (needTruncate) {
+          fs.delete(hadoopPath, true);
+        }
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+      String pathStr = hadoopPath.toUri().toString();
+      return new MyBatchWrite(queryId, pathStr, hadoopConf);
+    }
+  }
+
+  static class MyBatchWrite implements BatchWrite {
+
+    private final String queryId;
+    private final String path;
+    private final Configuration conf;
+
+    MyBatchWrite(String queryId, String path, Configuration conf) {
+      this.queryId = queryId;
+      this.path = path;
+      this.conf = conf;
+    }
+
+    @Override
+    public DataWriterFactory createBatchWriterFactory(PhysicalWriteInfo info) {
+      SimpleCounter.resetCounter();
+      return new JavaCSVDataWriterFactory(path, queryId, new SerializableConfiguration(conf));
+    }
+
+    @Override
+    public void onDataWriterCommit(WriterCommitMessage message) {
+      SimpleCounter.increaseCounter();
+    }
+
+    @Override
+    public void commit(WriterCommitMessage[] messages) {
+      Path finalPath = new Path(this.path);
+      Path jobPath = new Path(new Path(finalPath, "_temporary"), queryId);
+      try {
+        FileSystem fs = jobPath.getFileSystem(conf);
+        FileStatus[] fileStatuses = fs.listStatus(jobPath);
+        try {
+          for (FileStatus status : fileStatuses) {
+            Path file = status.getPath();
+            Path dest = new Path(finalPath, file.getName());
+            if (!fs.rename(file, dest)) {
+              throw new IOException(String.format("failed to rename(%s, %s)", file, dest));
+            }
+          }
+        } finally {
+          fs.delete(jobPath, true);
+        }
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
+
+    @Override
+    public void abort(WriterCommitMessage[] messages) {
+      try {
+        Path jobPath = new Path(new Path(this.path, "_temporary"), queryId);
+        FileSystem fs = jobPath.getFileSystem(conf);
+        fs.delete(jobPath, true);
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
+  }
+
+  static class MyTable extends JavaSimpleBatchTable implements SupportsWrite {
+
+    private final String path;
+    private final Configuration conf = SparkHadoopUtil.get().conf();
+
+    MyTable(CaseInsensitiveStringMap options) {
+      this.path = options.get("path");
+    }
+
+    @Override
+    public ScanBuilder newScanBuilder(CaseInsensitiveStringMap options) {
+      return new MyScanBuilder(new Path(path).toUri().toString(), conf);
+    }
+
+    @Override
+    public WriteBuilder newWriteBuilder(LogicalWriteInfo info) {
+      return new MyWriteBuilder(path, info);
+    }
+
+    @Override
+    public Set<TableCapability> capabilities() {
+      return new HashSet<>(Arrays.asList(
+          TableCapability.BATCH_READ,
+          TableCapability.BATCH_WRITE,
+          TableCapability.TRUNCATE));
+    }
+  }
+
+  @Override
+  public Table getTable(CaseInsensitiveStringMap options) {
+    return new MyTable(options);
+  }
+
+  static class JavaCSVInputPartitionReader implements InputPartition {
+
+    private String path;
+
+    JavaCSVInputPartitionReader(String path) {
+      this.path = path;
+    }
+
+    public String getPath() {
+      return path;
+    }
+
+    public void setPath(String path) {
+      this.path = path;
+    }
+  }
+
+  static class JavaCSVReaderFactory implements PartitionReaderFactory {
+
+    private final SerializableConfiguration conf;
+
+    JavaCSVReaderFactory(SerializableConfiguration conf) {
+      this.conf = conf;
+    }
+
+    @Override
+    public PartitionReader<InternalRow> createReader(InputPartition partition) {
+      String path = ((JavaCSVInputPartitionReader) partition).getPath();
+      Path filePath = new Path(path);
+      try {
+        FileSystem fs = filePath.getFileSystem(conf.value());
+        return new PartitionReader<InternalRow>() {
+          private final FSDataInputStream inputStream = fs.open(filePath);
+          private final Iterator<String> lines =
+              new BufferedReader(new InputStreamReader(inputStream)).lines().iterator();
+          private String currentLine = "";
+
+          @Override
+          public boolean next() {
+            if (lines.hasNext()) {
+              currentLine = lines.next();
+              return true;
+            } else {
+              return false;
+            }
+          }
+
+          @Override
+          public InternalRow get() {
+            Object[] objects =
+                Arrays.stream(currentLine.split(","))
+                    .map(String::trim)
+                    .map(Integer::parseInt)
+                    .toArray();
+            return new GenericInternalRow(objects);
+          }
+
+          @Override
+          public void close() throws IOException {
+            inputStream.close();
+          }
+        };
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
+  }
+
+  static class JavaCSVDataWriterFactory implements DataWriterFactory {
+
+    private final String path;
+    private final String jobId;
+    private final SerializableConfiguration conf;
+
+    JavaCSVDataWriterFactory(String path, String jobId, SerializableConfiguration conf) {
+      this.path = path;
+      this.jobId = jobId;
+      this.conf = conf;
+    }
+
+    @Override
+    public DataWriter<InternalRow> createWriter(int partitionId, long taskId) {
+      try {
+        Path jobPath = new Path(new Path(path, "_temporary"), jobId);
+        Path filePath = new Path(jobPath, String.format("%s-%d-%d", jobId, partitionId, taskId));
+        FileSystem fs = filePath.getFileSystem(conf.value());
+        return new JavaCSVDataWriter(fs, filePath);
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
+  }
+
+  static class JavaCSVDataWriter implements DataWriter<InternalRow> {
+
+    private final FileSystem fs;
+    private final Path file;
+    private final FSDataOutputStream out;
+
+    JavaCSVDataWriter(FileSystem fs, Path file) throws IOException {
+      this.fs = fs;
+      this.file = file;
+      out = fs.create(file);
+    }
+
+    @Override
+    public void write(InternalRow record) throws IOException {
+      out.writeBytes(String.format("%d,%d\n", record.getInt(0), record.getInt(1)));
+    }
+
+    @Override
+    public WriterCommitMessage commit() throws IOException {
+      out.close();
+      return null;
+    }
+
+    @Override
+    public void abort() throws IOException {
+      try {
+        out.close();
+      } finally {
+        fs.delete(file, false);
+      }
+    }
+
+    @Override
+    public void close() {
+    }
+  }
+}
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/connector/catalog/functions/JavaAverage.java b/sql/core/src/test/java/test/org/apache/spark/sql/connector/catalog/functions/JavaAverage.java
new file mode 100644
index 0000000000000..4e783fdd439b6
--- /dev/null
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/connector/catalog/functions/JavaAverage.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test.org.apache.spark.sql.connector.catalog.functions;
+
+import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.connector.catalog.functions.AggregateFunction;
+import org.apache.spark.sql.connector.catalog.functions.BoundFunction;
+import org.apache.spark.sql.connector.catalog.functions.UnboundFunction;
+import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.DoubleType;
+import org.apache.spark.sql.types.StructType;
+
+import java.io.Serializable;
+
+public class JavaAverage implements UnboundFunction {
+  @Override
+  public String name() {
+    return "avg";
+  }
+
+  @Override
+  public BoundFunction bind(StructType inputType) {
+    if (inputType.fields().length != 1) {
+      throw new UnsupportedOperationException("Expect exactly one argument");
+    }
+    if (inputType.fields()[0].dataType() instanceof DoubleType) {
+      return new JavaDoubleAverage();
+    }
+    throw new UnsupportedOperationException("Unsupported non-integral type: " +
+        inputType.fields()[0].dataType());
+  }
+
+  @Override
+  public String description() {
+    return null;
+  }
+
+  public static class JavaDoubleAverage implements AggregateFunction<State<Double>, Double> {
+    @Override
+    public State<Double> newAggregationState() {
+      return new State<>(0.0, 0.0);
+    }
+
+    @Override
+    public State<Double> update(State<Double> state, InternalRow input) {
+      if (input.isNullAt(0)) {
+        return state;
+      }
+      return new State<>(state.sum + input.getDouble(0), state.count + 1);
+    }
+
+    @Override
+    public Double produceResult(State<Double> state) {
+      return state.sum / state.count;
+    }
+
+    @Override
+    public State<Double> merge(State<Double> leftState, State<Double> rightState) {
+      return new State<>(leftState.sum + rightState.sum, leftState.count + rightState.count);
+    }
+
+    @Override
+    public DataType[] inputTypes() {
+      return new DataType[] { DataTypes.DoubleType };
+    }
+
+    @Override
+    public DataType resultType() {
+      return DataTypes.DoubleType;
+    }
+
+    @Override
+    public String name() {
+      return "davg";
+    }
+  }
+
+  public static class State<T> implements Serializable {
+    T sum, count;
+
+    State(T sum, T count) {
+      this.sum = sum;
+      this.count = count;
+    }
+  }
+}
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/connector/catalog/functions/JavaLongAdd.java b/sql/core/src/test/java/test/org/apache/spark/sql/connector/catalog/functions/JavaLongAdd.java
new file mode 100644
index 0000000000000..e2e7136d6f44c
--- /dev/null
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/connector/catalog/functions/JavaLongAdd.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test.org.apache.spark.sql.connector.catalog.functions;
+
+import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.connector.catalog.functions.BoundFunction;
+import org.apache.spark.sql.connector.catalog.functions.ScalarFunction;
+import org.apache.spark.sql.connector.catalog.functions.UnboundFunction;
+import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.LongType;
+import org.apache.spark.sql.types.StructField;
+import org.apache.spark.sql.types.StructType;
+
+public class JavaLongAdd implements UnboundFunction {
+  private final ScalarFunction<Long> impl;
+
+  public JavaLongAdd(ScalarFunction<Long> impl) {
+    this.impl = impl;
+  }
+
+  @Override
+  public String name() {
+    return "long_add";
+  }
+
+  @Override
+  public BoundFunction bind(StructType inputType) {
+    if (inputType.fields().length != 2) {
+      throw new UnsupportedOperationException("Expect two arguments");
+    }
+    StructField[] fields = inputType.fields();
+    if (!(fields[0].dataType() instanceof LongType)) {
+      throw new UnsupportedOperationException("Expect first argument to be LongType");
+    }
+    if (!(fields[1].dataType() instanceof LongType)) {
+      throw new UnsupportedOperationException("Expect second argument to be LongType");
+    }
+    return impl;
+  }
+
+  @Override
+  public String description() {
+    return "long_add";
+  }
+
+  private abstract static class JavaLongAddBase implements ScalarFunction<Long> {
+    private final boolean isResultNullable;
+
+    JavaLongAddBase(boolean isResultNullable) {
+      this.isResultNullable = isResultNullable;
+    }
+
+    @Override
+    public DataType[] inputTypes() {
+      return new DataType[] { DataTypes.LongType, DataTypes.LongType };
+    }
+
+    @Override
+    public DataType resultType() {
+      return DataTypes.LongType;
+    }
+
+    @Override
+    public boolean isResultNullable() {
+      return isResultNullable;
+    }
+  }
+
+  public static class JavaLongAddDefault extends JavaLongAddBase {
+    public JavaLongAddDefault(boolean isResultNullable) {
+      super(isResultNullable);
+    }
+
+    @Override
+    public String name() {
+      return "long_add_default";
+    }
+
+    @Override
+    public Long produceResult(InternalRow input) {
+      return input.getLong(0) + input.getLong(1);
+    }
+  }
+
+  public static class JavaLongAddMagic extends JavaLongAddBase {
+    public JavaLongAddMagic(boolean isResultNullable) {
+      super(isResultNullable);
+    }
+
+    @Override
+    public String name() {
+      return "long_add_magic";
+    }
+
+    public long invoke(long left, long right) {
+      return left + right;
+    }
+  }
+
+  public static class JavaLongAddStaticMagic extends JavaLongAddBase {
+    public JavaLongAddStaticMagic(boolean isResultNullable) {
+      super(isResultNullable);
+    }
+
+    @Override
+    public String name() {
+      return "long_add_static_magic";
+    }
+
+    public static long invoke(long left, long right) {
+      return left + right;
+    }
+  }
+}
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/connector/catalog/functions/JavaStrLen.java b/sql/core/src/test/java/test/org/apache/spark/sql/connector/catalog/functions/JavaStrLen.java
new file mode 100644
index 0000000000000..7cd010b9365bb
--- /dev/null
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/connector/catalog/functions/JavaStrLen.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test.org.apache.spark.sql.connector.catalog.functions;
+
+import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.connector.catalog.functions.BoundFunction;
+import org.apache.spark.sql.connector.catalog.functions.ScalarFunction;
+import org.apache.spark.sql.connector.catalog.functions.UnboundFunction;
+import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.StringType;
+import org.apache.spark.sql.types.StructType;
+import org.apache.spark.unsafe.types.UTF8String;
+
+public class JavaStrLen implements UnboundFunction {
+  private final BoundFunction fn;
+
+  public JavaStrLen(BoundFunction fn) {
+    this.fn = fn;
+  }
+
+  @Override
+  public String name() {
+    return "strlen";
+  }
+
+  @Override
+  public BoundFunction bind(StructType inputType) {
+    if (inputType.fields().length != 1) {
+      throw new UnsupportedOperationException("Expect exactly one argument");
+    }
+
+    if (inputType.fields()[0].dataType() instanceof StringType) {
+      return fn;
+    }
+
+    throw new UnsupportedOperationException("Except StringType");
+  }
+
+  @Override
+  public String description() {
+    return "strlen: returns the length of the input string\n" +
+        " strlen(string) -> int";
+  }
+
+  private abstract static class JavaStrLenBase implements ScalarFunction<Integer> {
+    @Override
+    public DataType[] inputTypes() {
+      return new DataType[] { DataTypes.StringType };
+    }
+
+    @Override
+    public DataType resultType() {
+      return DataTypes.IntegerType;
+    }
+
+    @Override
+    public String name() {
+      return "strlen";
+    }
+  }
+
+  public static class JavaStrLenDefault extends JavaStrLenBase {
+    @Override
+    public Integer produceResult(InternalRow input) {
+      String str = input.getString(0);
+      return str.length();
+    }
+  }
+
+  public static class JavaStrLenMagic extends JavaStrLenBase {
+    public int invoke(UTF8String str) {
+      return str.toString().length();
+    }
+  }
+
+  public static class JavaStrLenStaticMagic extends JavaStrLenBase {
+    public static int invoke(UTF8String str) {
+      return str.toString().length();
+    }
+  }
+
+  public static class JavaStrLenBoth extends JavaStrLenBase {
+    @Override
+    public Integer produceResult(InternalRow input) {
+      String str = input.getString(0);
+      return str.length();
+    }
+    public int invoke(UTF8String str) {
+      return str.toString().length() + 100;
+    }
+  }
+
+  // even though the static magic method is present, it has incorrect parameter type and so Spark
+  // should fallback to the non-static magic method
+  public static class JavaStrLenBadStaticMagic extends JavaStrLenBase {
+    public static int invoke(String str) {
+      return str.length();
+    }
+
+    public int invoke(UTF8String str) {
+      return str.toString().length() + 100;
+    }
+  }
+
+  public static class JavaStrLenNoImpl extends JavaStrLenBase {
+  }
+}
+
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/execution/datasources/orc/FakeKeyProvider.java b/sql/core/src/test/java/test/org/apache/spark/sql/execution/datasources/orc/FakeKeyProvider.java
new file mode 100644
index 0000000000000..70c9269962089
--- /dev/null
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/execution/datasources/orc/FakeKeyProvider.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test.org.apache.spark.sql.execution.datasources.orc;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
+import org.apache.hadoop.crypto.key.KeyProviderFactory;
+import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
+
+/**
+ * A Hadoop KeyProvider that lets us test the interaction
+ * with the Hadoop code.
+ *
+ * https://github.com/apache/orc/blob/rel/release-1.6.7/java/tools/src/test/org/apache/orc/impl/FakeKeyProvider.java
+ *
+ * This file intentionally keeps the original file except
+ * (1) package name, (2) import order, (3) a few indentation
+ */
+public class FakeKeyProvider extends KeyProvider {
+  // map from key name to metadata
+  private final Map<String, TestMetadata> keyMetdata = new HashMap<>();
+  // map from key version name to material
+  private final Map<String, KeyVersion> keyVersions = new HashMap<>();
+
+  public FakeKeyProvider(Configuration conf) {
+    super(conf);
+  }
+
+  @Override
+  public KeyVersion getKeyVersion(String name) {
+    return keyVersions.get(name);
+  }
+
+  @Override
+  public List<String> getKeys() {
+    return new ArrayList<>(keyMetdata.keySet());
+  }
+
+  @Override
+  public List<KeyVersion> getKeyVersions(String name) {
+    List<KeyVersion> result = new ArrayList<>();
+    Metadata meta = getMetadata(name);
+    for(int v=0; v < meta.getVersions(); ++v) {
+      String versionName = buildVersionName(name, v);
+      KeyVersion material = keyVersions.get(versionName);
+      if (material != null) {
+        result.add(material);
+      }
+    }
+    return result;
+  }
+
+  @Override
+  public Metadata getMetadata(String name)  {
+    return keyMetdata.get(name);
+  }
+
+  @Override
+  public KeyVersion createKey(String name, byte[] bytes, Options options) {
+    String versionName = buildVersionName(name, 0);
+    keyMetdata.put(name, new TestMetadata(options.getCipher(),
+        options.getBitLength(), 1));
+    KeyVersion result = new KMSClientProvider.KMSKeyVersion(name, versionName, bytes);
+    keyVersions.put(versionName, result);
+    return result;
+  }
+
+  @Override
+  public void deleteKey(String name) {
+    throw new UnsupportedOperationException("Can't delete keys");
+  }
+
+  @Override
+  public KeyVersion rollNewVersion(String name, byte[] bytes) {
+    TestMetadata key = keyMetdata.get(name);
+    String versionName = buildVersionName(name, key.addVersion());
+    KeyVersion result = new KMSClientProvider.KMSKeyVersion(name, versionName,
+        bytes);
+    keyVersions.put(versionName, result);
+    return result;
+  }
+
+  @Override
+  public void flush() {
+    // Nothing
+  }
+
+  static class TestMetadata extends KeyProvider.Metadata {
+
+    TestMetadata(String cipher, int bitLength, int versions) {
+      super(cipher, bitLength, null, null, null, versions);
+    }
+
+    public int addVersion() {
+      return super.addVersion();
+    }
+  }
+
+  public static class Factory extends KeyProviderFactory {
+
+    @Override
+    public KeyProvider createProvider(URI uri, Configuration conf) throws IOException {
+      if ("test".equals(uri.getScheme())) {
+        KeyProvider provider = new FakeKeyProvider(conf);
+        // populate a couple keys into the provider
+        byte[] piiKey = new byte[]{0,1,2,3,4,5,6,7,8,9,0xa,0xb,0xc,0xd,0xe,0xf};
+        org.apache.hadoop.crypto.key.KeyProvider.Options aes128 = new KeyProvider.Options(conf);
+        provider.createKey("pii", piiKey, aes128);
+        byte[] piiKey2 = new byte[]{0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,
+            0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f};
+        provider.rollNewVersion("pii", piiKey2);
+        byte[] secretKey = new byte[]{0x20,0x21,0x22,0x23,0x24,0x25,0x26,0x27,
+            0x28,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f};
+        provider.createKey("secret", secretKey, aes128);
+        return KeyProviderCryptoExtension.createKeyProviderCryptoExtension(provider);
+      }
+      return null;
+    }
+  }
+}
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/execution/sort/RecordBinaryComparatorSuite.java b/sql/core/src/test/java/test/org/apache/spark/sql/execution/sort/RecordBinaryComparatorSuite.java
index 564e76737ecde..6cb7c40e3332b 100644
--- a/sql/core/src/test/java/test/org/apache/spark/sql/execution/sort/RecordBinaryComparatorSuite.java
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/execution/sort/RecordBinaryComparatorSuite.java
@@ -37,6 +37,8 @@
 import org.junit.Before;
 import org.junit.Test;
 
+import java.nio.ByteOrder;
+
 /**
  * Test the RecordBinaryComparator, which compares two UnsafeRows by their binary form.
  */
@@ -261,40 +263,74 @@ public void testBinaryComparatorForNullColumns() throws Exception {
   public void testBinaryComparatorWhenSubtractionIsDivisibleByMaxIntValue() throws Exception {
     int numFields = 1;
 
+    // Place the following bytes (hex) into UnsafeRows for the comparison:
+    //
+    //   index | 00 01 02 03 04 05 06 07
+    //   ------+------------------------
+    //   row1  | 00 00 00 00 00 00 00 0b
+    //   row2  | 00 00 00 00 80 00 00 0a
+    //
+    // The byte layout needs to be identical on all platforms regardless of
+    // of endianness. To achieve this the bytes in each value are reversed
+    // on little-endian platforms.
+    long row1Data = 11L;
+    long row2Data = 11L + Integer.MAX_VALUE;
+    if (ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN)) {
+      row1Data = Long.reverseBytes(row1Data);
+      row2Data = Long.reverseBytes(row2Data);
+    }
+
     UnsafeRow row1 = new UnsafeRow(numFields);
     byte[] data1 = new byte[100];
     row1.pointTo(data1, computeSizeInBytes(numFields * 8));
-    row1.setLong(0, 11);
+    row1.setLong(0, row1Data);
 
     UnsafeRow row2 = new UnsafeRow(numFields);
     byte[] data2 = new byte[100];
     row2.pointTo(data2, computeSizeInBytes(numFields * 8));
-    row2.setLong(0, 11L + Integer.MAX_VALUE);
+    row2.setLong(0, row2Data);
 
     insertRow(row1);
     insertRow(row2);
 
-    Assert.assertTrue(compare(0, 1) > 0);
+    Assert.assertTrue(compare(0, 1) < 0);
   }
 
   @Test
   public void testBinaryComparatorWhenSubtractionCanOverflowLongValue() throws Exception {
     int numFields = 1;
 
+    // Place the following bytes (hex) into UnsafeRows for the comparison:
+    //
+    //   index | 00 01 02 03 04 05 06 07
+    //   ------+------------------------
+    //   row1  | 80 00 00 00 00 00 00 00
+    //   row2  | 00 00 00 00 00 00 00 01
+    //
+    // The byte layout needs to be identical on all platforms regardless of
+    // of endianness. To achieve this the bytes in each value are reversed
+    // on little-endian platforms.
+    long row1Data = Long.MIN_VALUE;
+    long row2Data = 1L;
+    if (ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN)) {
+      row1Data = Long.reverseBytes(row1Data);
+      row2Data = Long.reverseBytes(row2Data);
+    }
+
     UnsafeRow row1 = new UnsafeRow(numFields);
     byte[] data1 = new byte[100];
     row1.pointTo(data1, computeSizeInBytes(numFields * 8));
-    row1.setLong(0, Long.MIN_VALUE);
+    row1.setLong(0, row1Data);
 
     UnsafeRow row2 = new UnsafeRow(numFields);
     byte[] data2 = new byte[100];
     row2.pointTo(data2, computeSizeInBytes(numFields * 8));
-    row2.setLong(0, 1);
+    row2.setLong(0, row2Data);
 
     insertRow(row1);
     insertRow(row2);
 
-    Assert.assertTrue(compare(0, 1) < 0);
+    Assert.assertTrue(compare(0, 1) > 0);
   }
 
   @Test
diff --git a/sql/core/src/test/resources/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory b/sql/core/src/test/resources/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory
new file mode 100644
index 0000000000000..f436622b5fb42
--- /dev/null
+++ b/sql/core/src/test/resources/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+test.org.apache.spark.sql.execution.datasources.orc.FakeKeyProvider$Factory
diff --git a/sql/core/src/test/resources/META-INF/services/org.apache.spark.sql.SparkSessionExtensionsProvider b/sql/core/src/test/resources/META-INF/services/org.apache.spark.sql.SparkSessionExtensionsProvider
new file mode 100644
index 0000000000000..b5b01a09e6995
--- /dev/null
+++ b/sql/core/src/test/resources/META-INF/services/org.apache.spark.sql.SparkSessionExtensionsProvider
@@ -0,0 +1 @@
+org.apache.spark.sql.YourExtensions
diff --git a/sql/core/src/test/resources/META-INF/services/org.apache.spark.sql.jdbc.JdbcConnectionProvider b/sql/core/src/test/resources/META-INF/services/org.apache.spark.sql.jdbc.JdbcConnectionProvider
new file mode 100644
index 0000000000000..afb48e1a3511f
--- /dev/null
+++ b/sql/core/src/test/resources/META-INF/services/org.apache.spark.sql.jdbc.JdbcConnectionProvider
@@ -0,0 +1 @@
+org.apache.spark.sql.execution.datasources.jdbc.connection.IntentionallyFaultyConnectionProvider
\ No newline at end of file
diff --git a/sql/core/src/test/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/sql/core/src/test/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
index 914af589384df..dd22970203b3c 100644
--- a/sql/core/src/test/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
+++ b/sql/core/src/test/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
@@ -11,4 +11,5 @@ org.apache.spark.sql.streaming.sources.FakeReadBothModes
 org.apache.spark.sql.streaming.sources.FakeReadNeitherMode
 org.apache.spark.sql.streaming.sources.FakeWriteOnly
 org.apache.spark.sql.streaming.sources.FakeNoWrite
+org.apache.spark.sql.streaming.sources.FakeWriteSupportingExternalMetadata
 org.apache.spark.sql.streaming.sources.FakeWriteSupportProviderV1Fallback
diff --git a/sql/core/src/test/resources/SPARK-33084.jar b/sql/core/src/test/resources/SPARK-33084.jar
new file mode 100644
index 0000000000000..61e1663ad3a28
Binary files /dev/null and b/sql/core/src/test/resources/SPARK-33084.jar differ
diff --git a/sql/core/src/test/resources/hive-site.xml b/sql/core/src/test/resources/hive-site.xml
index 17297b3e22a7e..0d1072f832765 100644
--- a/sql/core/src/test/resources/hive-site.xml
+++ b/sql/core/src/test/resources/hive-site.xml
@@ -23,4 +23,13 @@
       <value>true</value>
       <description>Internal marker for test.</description>
   </property>
+  <property>
+    <name>hadoop.tmp.dir</name>
+    <value>/tmp/hive_one</value>
+    <description>default is /tmp/hadoop-${user.name} and will be overridden </description>
+  </property>
+  <property>
+    <name>hadoop.this.is.a.test.key</name>
+    <value>2018-11-17 13:33:33.333</value>
+  </property>
 </configuration>
diff --git a/sql/core/src/test/resources/spark-events/local-1596020211915 b/sql/core/src/test/resources/spark-events/local-1596020211915
new file mode 100644
index 0000000000000..ff34bbc16ef3a
--- /dev/null
+++ b/sql/core/src/test/resources/spark-events/local-1596020211915
@@ -0,0 +1,160 @@
+{"Event":"SparkListenerLogStart","Spark Version":"3.1.0-SNAPSHOT"}
+{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"cores":{"Resource Name":"cores","Amount":1,"Discovery Script":"","Vendor":""},"memory":{"Resource Name":"memory","Amount":1024,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}}
+{"Event":"SparkListenerExecutorAdded","Timestamp":1596020212090,"Executor ID":"driver","Executor Info":{"Host":"iZbp19vpr16ix621sdw476Z","Total Cores":4,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0}}
+{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Port":39845},"Maximum Memory":384093388,"Timestamp":1596020212109,"Maximum Onheap Memory":384093388,"Maximum Offheap Memory":0}
+{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.252.b09-2.el7_8.x86_64/jre","Java Version":"1.8.0_252 (Oracle Corporation)","Scala Version":"version 2.12.10"},"Spark Properties":{"spark.driver.host":"iZbp19vpr16ix621sdw476Z","spark.eventLog.enabled":"true","spark.driver.port":"46309","spark.jars":"file:/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar","spark.app.name":"StructuredKafkaWordCount","spark.scheduler.mode":"FIFO","spark.submit.pyFiles":"","spark.executor.id":"driver","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"/tmp/spark-history","spark.app.id":"local-1596020211915","spark.sql.shuffle.partitions":"2"},"Hadoop Properties":{"yarn.resourcemanager.amlauncher.thread-count":"50","yarn.sharedcache.enabled":"false","fs.s3a.connection.maximum":"15","fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","hadoop.security.kms.client.timeout":"60","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.framework.name":"local","yarn.sharedcache.uploader.server.thread-count":"50","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"30","hadoop.security.groups.cache.background.reload.threads":"3","yarn.resourcemanager.webapp.cross-origin.enabled":"false","fs.AbstractFileSystem.ftp.impl":"org.apache.hadoop.fs.ftp.FtpFs","fs.s3.block.size":"67108864","hadoop.registry.secure":"false","hadoop.shell.safely.delete.limit.num.files":"100","dfs.bytes-per-checksum":"512","fs.s3.buffer.dir":"${hadoop.tmp.dir}/s3","mapreduce.job.acl-view-job":" ","mapreduce.jobhistory.loadedjobs.cache.size":"5","mapreduce.input.fileinputformat.split.minsize":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","yarn.sharedcache.checksum.algo.impl":"org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl","yarn.nodemanager.amrmproxy.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor","yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size":"10485760","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","mapreduce.task.profile.maps":"0-2","yarn.scheduler.include-port-in-node-name":"false","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","yarn.resourcemanager.node-removal-untracked.timeout-ms":"60000","mapreduce.am.max-attempts":"2","hadoop.security.kms.client.failover.sleep.base.millis":"100","mapreduce.jobhistory.webapp.https.address":"0.0.0.0:19890","yarn.node-labels.fs-store.impl.class":"org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore","fs.trash.checkpoint.interval":"0","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","yarn.resourcemanager.node-ip-cache.expiry-interval-secs":"-1","hadoop.http.authentication.signature.secret.file":"*********(redacted)","hadoop.jetty.logs.serve.aliases":"true","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"10000","yarn.resourcemanager.system-metrics-publisher.enabled":"false","yarn.sharedcache.webapp.address":"0.0.0.0:8788","yarn.resourcemanager.delegation.token.renew-interval":"*********(redacted)","yarn.sharedcache.nm.uploader.replication.factor":"10","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","mapreduce.reduce.skip.proc-count.auto-incr":"true","hadoop.security.group.mapping.ldap.posix.attr.gid.name":"gidNumber","ipc.client.fallback-to-simple-auth-allowed":"false","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","yarn.acl.reservation-enable":"false","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","yarn.app.mapreduce.am.hard-kill-timeout-ms":"10000","yarn.nodemanager.container-metrics.enable":"true","yarn.timeline-service.client.fd-clean-interval-secs":"60","yarn.nodemanager.docker-container-executor.exec-name":"/usr/bin/docker","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","mapred.child.java.opts":"-Xmx200m","hadoop.common.configuration.version":"0.23.0","yarn.nodemanager.remote-app-log-dir-suffix":"logs","yarn.nodemanager.windows-container.cpu-limit.enabled":"false","yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed":"false","file.blocksize":"67108864","hadoop.registry.zk.retry.ceiling.ms":"60000","yarn.sharedcache.store.in-memory.initial-delay-mins":"10","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","mapreduce.map.skip.proc-count.auto-incr":"true","mapreduce.task.profile.reduces":"0-2","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"${yarn.resourcemanager.hostname}:8030","yarn.node-labels.enabled":"false","yarn.resourcemanager.webapp.ui-actions.enabled":"true","mapreduce.task.timeout":"600000","yarn.sharedcache.client-server.thread-count":"50","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"file:///","yarn.minicluster.use-rpc":"false","fs.har.impl.disable.cache":"true","io.compression.codec.bzip2.library":"system-native","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"${yarn.resourcemanager.hostname}:8090","mapreduce.jobhistory.address":"0.0.0.0:10020","yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.is.minicluster":"false","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A","mapreduce.task.combine.progress.records":"10000","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","ipc.server.log.slow.rpc":"false","yarn.resourcemanager.node-labels.provider.fetch-interval-ms":"1800000","yarn.nodemanager.webapp.cross-origin.enabled":"false","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"-1","fs.s3a.fast.upload.active.blocks":"4","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","yarn.resourcemanager.fs.state-store.num-retries":"0","yarn.resourcemanager.nodemanager-connect-retries":"10","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","mapreduce.task.io.sort.factor":"10","yarn.nodemanager.amrmproxy.client.thread-count":"25","ha.failover-controller.new-active.rpc-timeout.ms":"60000","yarn.nodemanager.container-localizer.java.opts":"-Xmx256m","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","yarn.nodemanager.windows-container.memory-limit.enabled":"false","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.minicluster.fixed.ports":"false","yarn.cluster.max-application-priority":"0","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","hadoop.caller.context.signature.max.size":"40","ha.zookeeper.session-timeout.ms":"10000","tfile.io.chunk.size":"1048576","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.kms.client.failover.sleep.max.millis":"2000","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","fs.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","yarn.nodemanager.local-cache.max-files-per-directory":"8192","hadoop.http.cross-origin.enabled":"false","mapreduce.map.sort.spill.percent":"0.80","yarn.timeline-service.entity-group-fs-store.scan-interval-seconds":"60","yarn.timeline-service.client.best-effort":"false","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","hadoop.security.group.mapping.ldap.posix.attr.uid.name":"uidNumber","fs.AbstractFileSystem.swebhdfs.impl":"org.apache.hadoop.fs.SWebHdfs","mapreduce.ifile.readahead":"true","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler","yarn.app.mapreduce.am.command-opts":"-Xmx1024m","mapreduce.cluster.local.dir":"${hadoop.tmp.dir}/mapred/local","io.mapfile.bloom.error.rate":"0.005","yarn.nodemanager.runtime.linux.allowed-runtimes":"default","yarn.sharedcache.store.class":"org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","io.skip.checksum.errors":"false","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","fs.s3a.connection.timeout":"200000","mapreduce.job.max.split.locations":"10","hadoop.registry.zk.session.timeout.ms":"60000","mapreduce.jvm.system-properties-to-log":"os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name","yarn.timeline-service.entity-group-fs-store.active-dir":"/tmp/entity-file-history/active","mapreduce.shuffle.transfer.buffer.size":"131072","yarn.timeline-service.client.retry-interval-ms":"1000","yarn.http.policy":"HTTP_ONLY","fs.s3a.socket.send.buffer":"8192","yarn.sharedcache.uploader.server.address":"0.0.0.0:8046","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","yarn.minicluster.yarn.nodemanager.resource.memory-mb":"4096","mapreduce.job.emit-timeline-data":"false","yarn.nodemanager.resource.system-reserved-memory-mb":"-1","hadoop.kerberos.min.seconds.before.relogin":"60","mapreduce.jobhistory.move.thread-count":"3","yarn.resourcemanager.admin.client.thread-count":"1","yarn.dispatcher.drain-events.timeout":"300000","fs.s3a.buffer.dir":"${hadoop.tmp.dir}/s3a","hadoop.ssl.enabled.protocols":"TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2","mapreduce.jobhistory.admin.address":"0.0.0.0:10033","yarn.log-aggregation-status.time-out.ms":"600000","mapreduce.shuffle.port":"13562","yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory":"10","yarn.nodemanager.health-checker.interval-ms":"600000","ftp.blocksize":"67108864","yarn.nodemanager.log-container-debug-info.enabled":"false","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms":"20","yarn.nodemanager.delete.debug-delay-sec":"0","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","hadoop.security.groups.cache.secs":"300","yarn.resourcemanager.zk-retry-interval-ms":"1000","ipc.maximum.data.length":"67108864","mapreduce.shuffle.max.threads":"0","hadoop.security.authorization":"false","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","mapreduce.jobhistory.http.policy":"HTTP_ONLY","yarn.sharedcache.store.in-memory.check-period-mins":"720","s3native.replication":"3","hadoop.security.group.mapping.ldap.ssl":"false","yarn.client.application-client-protocol.poll-interval-ms":"200","ha.zookeeper.parent-znode":"/hadoop-ha","yarn.nodemanager.log-aggregation.policy.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","yarn.nodemanager.recovery.supervised":"false","yarn.sharedcache.admin.thread-count":"1","yarn.resourcemanager.ha.automatic-failover.enabled":"true","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.resourcemanager.address":"${yarn.resourcemanager.hostname}:8032","ipc.client.ping":"true","mapreduce.task.local-fs.write-limit.bytes":"-1","fs.adl.oauth2.access.token.provider.type":"*********(redacted)","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","hadoop.ssl.enabled":"false","fs.s3a.multipart.purge":"false","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"10","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","mapreduce.cluster.temp.dir":"${hadoop.tmp.dir}/mapred/temp","s3.replication":"3","yarn.nodemanager.node-labels.resync-interval-ms":"120000","hadoop.tmp.dir":"/tmp/hadoop-${user.name}","mapreduce.job.maps":"2","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","mapreduce.client.submit.file.replication":"10","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"2147483647","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","yarn.resourcemanager.zk-acl":"world:anyone:rwcda","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","yarn.nodemanager.amrmproxy.enable":"false","fs.ftp.host.port":"21","ipc.ping.interval":"60000","yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size":"10","yarn.resourcemanager.admin.address":"${yarn.resourcemanager.hostname}:8033","file.client-write-packet-size":"65536","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"true","mapreduce.local.clientfactory.class.name":"org.apache.hadoop.mapred.LocalClientFactory","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","yarn.nodemanager.log.deletion-threads-count":"4","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"100","yarn.nodemanager.localizer.client.thread-count":"5","yarn.sharedcache.admin.address":"0.0.0.0:8047","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec","fs.s3a.connection.ssl.enabled":"true","yarn.nodemanager.process-kill-wait.ms":"2000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","hadoop.workaround.non.threadsafe.getpwuid":"true","fs.df.interval":"60000","fs.s3.sleepTimeSeconds":"10","fs.s3a.multiobjectdelete.enable":"true","yarn.sharedcache.cleaner.resource-sleep-ms":"0","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","hadoop.shell.missing.defaultFs.warning":"false","io.file.buffer.size":"65536","hadoop.security.group.mapping.ldap.search.attr.member":"member","hadoop.security.random.device.file.path":"/dev/urandom","hadoop.security.sensitive-config-keys":"*********(redacted)","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","yarn.intermediate-data-encryption.enable":"false","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/tmp/hadoop-yarn/staging","mapreduce.reduce.shuffle.read.timeout":"180000","hadoop.http.cross-origin.max-age":"1800","fs.s3a.connection.establish.timeout":"5000","mapreduce.job.running.map.limit":"0","yarn.minicluster.control-resource-monitoring":"false","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","mapreduce.reduce.log.level":"INFO","hadoop.security.dns.log-slow-lookups.threshold.ms":"1000","mapreduce.job.ubertask.enable":"false","hadoop.caller.context.enabled":"false","yarn.nodemanager.vmem-pmem-ratio":"2.1","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","s3native.stream-buffer-size":"4096","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","yarn.nodemanager.resource.pcores-vcores-multiplier":"1.0","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.s3n.multipart.uploads.enabled":"false","hadoop.security.crypto.buffer.size":"8192","yarn.nodemanager.node-labels.provider.fetch-interval-ms":"600000","mapreduce.jobhistory.recovery.store.leveldb.path":"${hadoop.tmp.dir}/mapred/history/recoverystore","yarn.client.failover-retries-on-socket-timeouts":"0","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","mapreduce.job.finish-when-all-reducers-done":"false","hadoop.registry.jaas.context":"Client","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","s3.blocksize":"67108864","io.map.index.interval":"128","mapreduce.job.counters.max":"120","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","yarn.timeline-service.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/timeline","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","s3.stream-buffer-size":"4096","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.reservation-system.planfollower.time-step":"1000","s3native.bytes-per-checksum":"512","mapreduce.jobtracker.address":"local","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","hadoop.security.group.mapping.ldap.read.timeout.ms":"60000","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","yarn.node-labels.fs-store.retry-policy-spec":"2000, 500","hadoop.security.groups.cache.background.reload":"false","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","yarn.resourcemanager.rm.container-allocation.expiry-interval-ms":"600000","mapreduce.fileoutputcommitter.algorithm.version":"1","yarn.resourcemanager.work-preserving-recovery.enabled":"true","mapreduce.map.skip.maxrecords":"0","yarn.sharedcache.root-dir":"/sharedcache","hadoop.http.authentication.type":"simple","mapreduce.task.userlog.limit.kb":"0","yarn.resourcemanager.scheduler.monitor.enable":"false","fs.s3n.block.size":"67108864","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","mapreduce.jobtracker.staging.root.dir":"${hadoop.tmp.dir}/mapred/staging","yarn.nodemanager.resource-monitor.interval-ms":"3000","mapreduce.shuffle.listen.queue.size":"128","mapreduce.map.cpu.vcores":"1","yarn.timeline-service.client.fd-retain-secs":"300","hadoop.user.group.static.mapping.overrides":"dr.who=;","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"false","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","yarn.app.mapreduce.client.job.max-retries":"0","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","hadoop.security.group.mapping.ldap.connection.timeout.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","hadoop.registry.zk.retry.interval.ms":"1000","yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms":"1000","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","hadoop.ssl.server.conf":"ssl-server.xml","yarn.sharedcache.cleaner.initial-delay-mins":"10","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.nodemanager.runtime.linux.docker.capabilities":"CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE","yarn.acl.enable":"false","yarn.timeline-service.entity-group-fs-store.done-dir":"/tmp/entity-file-history/done/","mapreduce.task.profile":"false","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"nobody","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","yarn.resourcemanager.configuration.file-system-based-store":"/yarn/conf","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"${yarn.log.dir}/userlogs","fs.automatic.close":"true","fs.s3n.multipart.copy.block.size":"5368709120","yarn.nodemanager.hostname":"0.0.0.0","yarn.resourcemanager.zk-timeout-ms":"10000","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","hadoop.security.group.mapping.ldap.search.filter.user":"(&(objectClass=user)(sAMAccountName={0}))","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","mapreduce.job.ubertask.maxmaps":"9","fs.s3a.threads.keepalivetime":"60","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","yarn.resourcemanager.delegation.token.max-lifetime":"*********(redacted)","yarn.timeline-service.entity-group-fs-store.summary-store":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.reduce.cpu.vcores":"1","fs.client.resolve.remote.symlinks":"true","yarn.nodemanager.webapp.https.address":"0.0.0.0:8044","hadoop.http.cross-origin.allowed-origins":"*","yarn.timeline-service.entity-group-fs-store.retain-seconds":"604800","yarn.resourcemanager.metrics.runtime.buckets":"60,300,1440","yarn.timeline-service.generic-application-history.max-applications":"10000","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","yarn.node-labels.configuration-type":"centralized","fs.s3a.path.style.access":"false","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","yarn.sharedcache.store.in-memory.staleness-period-mins":"10080","fs.adl.impl":"org.apache.hadoop.fs.adl.AdlFileSystem","yarn.resourcemanager.nodemanager.minimum.version":"NONE","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","yarn.scheduler.maximum-allocation-vcores":"4","hadoop.http.cross-origin.allowed-headers":"X-Requested-With,Content-Type,Accept,Origin","yarn.nodemanager.log-aggregation.compression-type":"none","yarn.timeline-service.version":"1.0f","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","mapreduce.reduce.maxattempts":"4","hadoop.security.dns.log-slow-lookups.enabled":"false","mapreduce.job.committer.setup.cleanup.needed":"true","mapreduce.job.running.reduce.limit":"0","ipc.maximum.response.length":"134217728","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","hadoop.caller.context.max.size":"128","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","fs.s3a.fast.upload.buffer":"disk","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","fs.s3a.max.total.tasks":"5","fs.s3a.readahead.range":"64K","hadoop.http.authentication.simple.anonymous.allowed":"true","fs.s3a.fast.upload":"false","fs.s3a.attempts.maximum":"20","hadoop.registry.zk.connection.timeout.ms":"15000","yarn.resourcemanager.delegation-token-renewer.thread-count":"*********(redacted)","yarn.nodemanager.health-checker.script.timeout-ms":"1200000","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","mapreduce.map.log.level":"INFO","mapreduce.output.fileoutputformat.compress.type":"RECORD","yarn.resourcemanager.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/system/rmstore","hadoop.registry.rm.enabled":"false","mapreduce.ifile.readahead.bytes":"4194304","yarn.resourcemanager.fs.state-store.retry-policy-spec":"2000, 500","yarn.sharedcache.app-checker.class":"org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"true","yarn.nodemanager.resource.detect-hardware-capabilities":"false","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","yarn.resourcemanager.fs.state-store.retry-interval-ms":"1000","file.stream-buffer-size":"4096","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","mapreduce.map.speculative":"true","mapreduce.job.speculative.retry-after-speculate":"15000","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.app.mapreduce.am.container.log.backups":"0","yarn.app.mapreduce.am.log.level":"INFO","mapreduce.job.reduce.slowstart.completedmaps":"0.05","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","yarn.timeline-service.client.internal-timers-ttl-secs":"420","fs.s3a.block.size":"32M","yarn.sharedcache.client-server.address":"0.0.0.0:8045","yarn.resourcemanager.hostname":"0.0.0.0","yarn.resourcemanager.delegation.key.update-interval":"86400000","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"1024","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","yarn.nodemanager.disk-health-checker.enable":"true","ipc.client.tcpnodelay":"true","ipc.client.rpc-timeout.ms":"0","fs.s3.maxRetries":"4","ipc.client.low-latency":"false","mapreduce.input.lineinputformat.linespermap":"1","ipc.client.connect.max.retries.on.timeouts":"45","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.timeline-service.entity-group-fs-store.app-cache-size":"10","fs.s3a.socket.recv.buffer":"8192","fs.s3n.multipart.uploads.block.size":"67108864","yarn.resourcemanager.resource-tracker.address":"${yarn.resourcemanager.hostname}:8031","yarn.nodemanager.node-labels.provider.fetch-timeout-ms":"1200000","yarn.resourcemanager.leveldb-state-store.compaction-interval-secs":"3600","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"org.apache.hadoop.http.lib.StaticUserWebFilter","mapreduce.reduce.memory.mb":"1024","s3native.client-write-packet-size":"65536","yarn.timeline-service.hostname":"0.0.0.0","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","yarn.nodemanager.container-metrics.period-ms":"-1","yarn.nodemanager.log.retain-seconds":"10800","yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds":"3600","yarn.resourcemanager.keytab":"/etc/krb5.keytab","hadoop.security.group.mapping.providers.combined":"true","mapreduce.reduce.merge.inmem.threshold":"1000","yarn.timeline-service.recovery.enabled":"false","yarn.sharedcache.nm.uploader.thread-count":"20","mapreduce.shuffle.ssl.enabled":"false","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","mapreduce.job.speculative.minimum-allowed-tasks":"10","yarn.log-aggregation.retain-seconds":"-1","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.http.cross-origin.allowed-methods":"GET,POST,HEAD","mapreduce.jobhistory.webapp.address":"0.0.0.0:19888","mapreduce.jobtracker.system.dir":"${hadoop.tmp.dir}/mapred/system","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"${yarn.resourcemanager.hostname}:8088","mapreduce.jobhistory.recovery.enable":"false","mapreduce.reduce.shuffle.parallelcopies":"5","fs.AbstractFileSystem.webhdfs.impl":"org.apache.hadoop.fs.WebHdfs","fs.trash.interval":"0","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","yarn.app.mapreduce.am.resource.mb":"1536","mapreduce.input.fileinputformat.list-status.num-threads":"1","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.nodemanager.resource.cpu-vcores":"-1","mapreduce.job.reduces":"1","fs.s3a.multipart.size":"100M","yarn.scheduler.minimum-allocation-vcores":"1","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","ha.health-monitor.sleep-after-disconnect.ms":"1000","s3.bytes-per-checksum":"512","yarn.app.mapreduce.shuffle.log.limit.kb":"0","hadoop.security.group.mapping":"org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback","yarn.client.application-client-protocol.poll-timeout-ms":"-1","mapreduce.jobhistory.jhist.format":"json","yarn.resourcemanager.ha.enabled":"false","hadoop.http.staticuser.user":"dr.who","mapreduce.task.exit.timeout.check-interval-ms":"20000","mapreduce.task.exit.timeout":"60000","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","yarn.resourcemanager.reservation-system.enable":"false","s3.client-write-packet-size":"65536","mapreduce.map.output.compress":"false","ha.zookeeper.acl":"world:anyone:rwcda","ipc.server.max.connections":"0","yarn.scheduler.maximum-allocation-mb":"8192","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.sharedcache.cleaner.period-mins":"1440","yarn.app.mapreduce.am.container.log.limit.kb":"0","s3native.blocksize":"67108864","ipc.client.connect.retry.interval":"1000","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","mapreduce.jobhistory.cleaner.enable":"true","yarn.timeline-service.client.fd-flush-interval-secs":"10","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","mapreduce.job.end-notification.retry.attempts":"0","yarn.nodemanager.resource.count-logical-processors-as-cores":"false","yarn.resourcemanager.zk-num-retries":"1000","hadoop.registry.zk.root":"/registry","adl.feature.ownerandgroup.enableupn":"false","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","yarn.nodemanager.localizer.cache.target-size-mb":"10240","ftp.client-write-packet-size":"65536","fs.AbstractFileSystem.adl.impl":"org.apache.hadoop.fs.adl.Adl","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"86400","io.native.lib.available":"true","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","yarn.nodemanager.amrmproxy.address":"0.0.0.0:8048","ipc.server.listen.queue.size":"128","map.sort.class":"org.apache.hadoop.util.QuickSort","fs.viewfs.rename.strategy":"SAME_MOUNTPOINT","hadoop.security.kms.client.authentication.retry-count":"1","fs.permissions.umask-mode":"022","yarn.nodemanager.vmem-check-enabled":"true","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","mapreduce.job.ubertask.maxreduces":"1","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","mapreduce.jobhistory.jobname.limit":"50","yarn.client.nodemanager-connect.retry-interval-ms":"10000","yarn.timeline-service.state-store-class":"org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME","yarn.sharedcache.nested-level":"3","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","yarn.resourcemanager.recovery.enabled":"false"},"System Properties":{"java.io.tmpdir":"/tmp","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.version":"1.8","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Oracle Corporation","java.vm.specification.version":"1.8","user.home":"/root","file.encoding.pkg":"sun.io","sun.nio.ch.bugLevel":"","sun.arch.data.model":"64","sun.boot.library.path":"/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.252.b09-2.el7_8.x86_64/jre/lib/amd64","user.dir":"/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8","java.library.path":"/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib","sun.cpu.isalist":"","os.arch":"amd64","java.vm.version":"25.252-b09","jetty.git.hash":"ab228fde9e55e9164c738d7fa121f8ac5acd51c9","java.endorsed.dirs":"/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.252.b09-2.el7_8.x86_64/jre/lib/endorsed","java.runtime.version":"1.8.0_252-b09","java.vm.info":"mixed mode","java.ext.dirs":"/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.252.b09-2.el7_8.x86_64/jre/lib/ext:/usr/java/packages/lib/ext","java.runtime.name":"OpenJDK Runtime Environment","file.separator":"/","java.class.version":"52.0","java.specification.name":"Java Platform API Specification","sun.boot.class.path":"/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.252.b09-2.el7_8.x86_64/jre/lib/resources.jar:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.252.b09-2.el7_8.x86_64/jre/lib/rt.jar:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.252.b09-2.el7_8.x86_64/jre/lib/sunrsasign.jar:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.252.b09-2.el7_8.x86_64/jre/lib/jsse.jar:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.252.b09-2.el7_8.x86_64/jre/lib/jce.jar:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.252.b09-2.el7_8.x86_64/jre/lib/charsets.jar:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.252.b09-2.el7_8.x86_64/jre/lib/jfr.jar:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.252.b09-2.el7_8.x86_64/jre/classes","file.encoding":"UTF-8","user.timezone":"Asia/Shanghai","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","os.version":"3.10.0-1127.10.1.el7.x86_64","sun.os.patch.level":"unknown","java.vm.specification.vendor":"Oracle Corporation","user.country":"US","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.url":"http://java.oracle.com/","java.awt.printerjob":"sun.print.PSPrinterJob","java.awt.graphicsenv":"sun.awt.X11GraphicsEnvironment","awt.toolkit":"sun.awt.X11.XToolkit","os.name":"Linux","java.vm.vendor":"Oracle Corporation","java.vendor.url.bug":"http://bugreport.sun.com/bugreport/","user.name":"root","java.vm.name":"OpenJDK 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.SparkSubmit --master local[*] --conf spark.eventLog.dir=/tmp/spark-history --conf spark.eventLog.enabled=true --conf spark.sql.shuffle.partitions=2 --class org.apache.spark.examples.sql.streaming.StructuredKafkaWordCount ./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar 192.168.130.97:9092 subscribe test5","java.home":"/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.252.b09-2.el7_8.x86_64/jre","java.version":"1.8.0_252","sun.io.unicode.encoding":"UnicodeLittle"},"Classpath Entries":{"/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/metrics-graphite-4.1.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/nimbus-jose-jwt-4.41.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hive-vector-code-gen-2.3.7.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jackson-jaxrs-1.9.13.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jersey-server-2.30.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/pyrolite-4.30.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/conf/":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/json-smart-2.3.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/objenesis-2.5.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hadoop-auth-2.8.5.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jsp-api-2.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/spark-unsafe_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/commons-codec-1.10.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/protobuf-java-2.5.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/avro-1.8.2.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/guice-3.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/aopalliance-repackaged-2.6.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/transaction-api-1.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/spire_2.12-0.17.0-M1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/gson-2.2.4.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/datanucleus-rdbms-4.1.19.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jackson-module-paranamer-2.10.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/libfb303-0.9.3.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/commons-cli-1.2.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/spark-tags_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/scala-library-2.12.10.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/xbean-asm7-shaded-4.15.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jersey-container-servlet-2.30.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hk2-api-2.6.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jakarta.xml.bind-api-2.3.2.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/okhttp-2.4.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/derby-10.12.1.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jackson-core-asl-1.9.13.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/commons-collections-3.2.2.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/httpcore-4.4.12.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/commons-beanutils-1.9.4.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/spire-util_2.12-0.17.0-M1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/commons-crypto-1.0.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/spark-launcher_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/stax-api-1.0-2.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/json4s-ast_2.12-3.6.6.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/lz4-java-1.7.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/scala-parser-combinators_2.12-1.1.2.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/parquet-format-2.4.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/parquet-column-1.10.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/commons-logging-1.1.3.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/audience-annotations-0.5.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hive-jdbc-2.3.7.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/spark-hive-thriftserver_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hive-cli-2.3.7.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/javolution-5.5.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/JLargeArrays-1.5.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hadoop-yarn-api-2.8.5.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/algebra_2.12-2.0.0-M2.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/commons-dbcp-1.4.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jakarta.ws.rs-api-2.1.6.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/threeten-extra-1.5.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/commons-io-2.4.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/metrics-json-4.1.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/libthrift-0.12.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/compress-lzf-1.0.3.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/metrics-jmx-4.1.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jakarta.inject-2.6.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/stax-api-1.0.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hive-shims-common-2.3.7.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/curator-recipes-2.7.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/antlr4-runtime-4.7.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/slf4j-api-1.7.30.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/oro-2.0.8.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/arrow-memory-0.15.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jpam-1.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/velocity-1.5.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/orc-core-1.5.10.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/spark-sql_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jackson-databind-2.10.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/commons-text-1.6.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jersey-client-2.30.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/htrace-core4-4.0.1-incubating.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/json-1.8.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/spark-graphx_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/avro-ipc-1.8.2.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/macro-compat_2.12-1.1.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jetty-util-6.1.26.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/core-1.1.2.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hive-shims-2.3.7.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/joda-time-2.10.5.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/parquet-encoding-1.10.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hive-llap-common-2.3.7.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/spark-network-common_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/datanucleus-api-jdo-4.2.4.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/paranamer-2.8.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hive-shims-0.23-2.3.7.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/activation-1.1.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/curator-framework-2.7.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/commons-compress-1.8.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hadoop-mapreduce-client-common-2.8.5.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/RoaringBitmap-0.7.45.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/ivy-2.4.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jackson-core-2.10.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hadoop-yarn-client-2.8.5.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/commons-httpclient-3.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/spark-yarn_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/parquet-common-1.10.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/zstd-jni-1.4.5-2.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jersey-container-servlet-core-2.30.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/snappy-java-1.1.7.5.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/shapeless_2.12-2.3.3.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/commons-pool-1.5.4.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/json4s-core_2.12-3.6.6.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/httpclient-4.5.6.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/api-util-1.0.0-M20.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/aircompressor-0.10.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/spark-repl_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hadoop-yarn-common-2.8.5.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/leveldbjni-all-1.8.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jersey-hk2-2.30.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jta-1.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jetty-sslengine-6.1.26.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/commons-net-3.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/datanucleus-core-4.1.17.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hadoop-yarn-server-web-proxy-2.8.5.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/breeze_2.12-1.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/orc-mapreduce-1.5.10.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jetty-6.1.26.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/spark-core_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/xz-1.5.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/javax.inject-1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/scala-compiler-2.12.10.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/metrics-jvm-4.1.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/orc-shims-1.5.10.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jaxb-api-2.2.2.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jakarta.validation-api-2.0.2.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/spire-macros_2.12-0.17.0-M1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/janino-3.1.2.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/osgi-resource-locator-1.0.3.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jcl-over-slf4j-1.7.30.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hadoop-mapreduce-client-app-2.8.5.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hk2-utils-2.6.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/spark-sketch_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/JTransforms-3.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/kafka-clients-2.4.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/guice-servlet-3.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/okio-1.4.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hadoop-annotations-2.8.5.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/commons-math3-3.4.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/json4s-scalap_2.12-3.6.6.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/bonecp-0.8.0.RELEASE.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/spark-streaming_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/accessors-smart-1.2.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/guava-14.0.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/shims-0.7.45.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/ST4-4.0.4.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jackson-module-scala_2.12-2.10.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/scala-xml_2.12-1.2.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/chill-java-0.9.5.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hadoop-mapreduce-client-shuffle-2.8.5.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/cats-kernel_2.12-2.0.0-M4.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/stream-2.9.6.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/commons-configuration-1.6.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jodd-core-3.5.2.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/scala-collection-compat_2.12-2.1.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/commons-pool2-2.6.2.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jul-to-slf4j-1.7.30.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/xmlenc-0.52.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/flatbuffers-java-1.9.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/spark-token-provider-kafka-0-10_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/json4s-jackson_2.12-3.6.6.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/commons-compiler-3.1.2.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jline-2.14.6.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/breeze-macros_2.12-1.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/kryo-shaded-4.0.2.jar":"System Classpath","spark://iZbp19vpr16ix621sdw476Z:46309/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar":"Added By User","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hadoop-common-2.8.5.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/spark-hive_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jersey-common-2.30.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/istack-commons-runtime-3.0.8.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/curator-client-2.7.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jackson-xc-1.9.13.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/zookeeper-3.4.14.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/parquet-hadoop-1.10.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jakarta.annotation-api-1.3.5.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hive-shims-scheduler-2.3.7.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/univocity-parsers-2.8.3.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/commons-digester-1.8.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/spark-mllib_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/arpack_combined_all-0.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/spark-sql-kafka-0-10_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jackson-annotations-2.10.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hk2-locator-2.6.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hadoop-mapreduce-client-core-2.8.5.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/avro-mapred-1.8.2-hadoop2.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hadoop-yarn-server-common-2.8.5.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/scala-reflect-2.12.10.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/super-csv-2.2.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hadoop-mapreduce-client-jobclient-2.8.5.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hadoop-client-2.8.5.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hive-common-2.3.7.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/apacheds-kerberos-codec-2.0.0-M15.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hive-exec-2.3.7-core.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/opencsv-2.3.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/api-asn1-api-1.0.0-M20.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hive-storage-api-2.7.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/spire-platform_2.12-0.17.0-M1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/aopalliance-1.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/HikariCP-2.5.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hive-metastore-2.3.7.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/minlog-1.3.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/arrow-format-0.15.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jsr305-3.0.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/commons-lang-2.6.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/commons-lang3-3.9.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/javax.jdo-3.2.0-m3.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/apacheds-i18n-2.0.0-M15.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/javassist-3.25.0-GA.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jackson-mapper-asl-1.9.13.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/antlr-runtime-3.5.2.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/log4j-1.2.17.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hive-beeline-2.3.7.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/chill_2.12-0.9.5.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jdo-api-3.0.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/spark-kvstore_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/metrics-core-4.1.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jaxb-runtime-2.3.2.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/spark-mllib-local_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/py4j-0.10.9.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/javax.servlet-api-3.1.0.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hive-serde-2.3.7.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/hadoop-hdfs-client-2.8.5.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/spark-network-shuffle_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jcip-annotations-1.0-1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/jersey-media-jaxb-2.30.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/parquet-jackson-1.10.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/slf4j-log4j12-1.7.30.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/netty-all-4.1.47.Final.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/arrow-vector-0.15.1.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/spark-catalyst_2.12-3.1.0-SNAPSHOT.jar":"System Classpath","/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/jars/machinist_2.12-0.6.8.jar":"System Classpath"}}
+{"Event":"SparkListenerApplicationStart","App Name":"StructuredKafkaWordCount","App ID":"local-1596020211915","Timestamp":1596020210919,"User":"root"}
+{"Event":"org.apache.spark.sql.streaming.StreamingQueryListener$QueryStartedEvent","id":"8d268dc2-bc9c-4be8-97a9-b135d2943028","runId":"e225d92f-2545-48f8-87a2-9c0309580f8a","name":null,"timestamp":"2020-07-29T10:56:55.947Z"}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":0,"description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 0","details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nWriteToDataSourceV2 (14)\n+- * HashAggregate (13)\n   +- StateStoreSave (12)\n      +- * HashAggregate (11)\n         +- StateStoreRestore (10)\n            +- Exchange (9)\n               +- * HashAggregate (8)\n                  +- * HashAggregate (7)\n                     +- * SerializeFromObject (6)\n                        +- MapPartitions (5)\n                           +- DeserializeToObject (4)\n                              +- * Project (3)\n                                 +- * Project (2)\n                                    +- MicroBatchScan (1)\n\n\n(1) MicroBatchScan\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nArguments: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@7e7b182c, KafkaV2[Subscribe[test5]], {\"test5\":{\"0\":48276}}, {\"test5\":{\"0\":48279}}\n\n(2) Project [codegen id : 1]\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(3) Project [codegen id : 1]\nOutput [1]: [cast(value#8 as string) AS value#21]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(4) DeserializeToObject\nInput [1]: [value#21]\nArguments: value#21.toString, obj#27: java.lang.String\n\n(5) MapPartitions\nInput [1]: [obj#27]\nArguments: org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String\n\n(6) SerializeFromObject [codegen id : 2]\nInput [1]: [obj#28]\nArguments: [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]\n\n(7) HashAggregate [codegen id : 2]\nInput [1]: [value#29]\nKeys [1]: [value#29]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(8) HashAggregate [codegen id : 2]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(9) Exchange\nInput [2]: [value#29, count#38L]\nArguments: hashpartitioning(value#29, 2), true, [id=#142]\n\n(10) StateStoreRestore\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = f7faa1e9-69d9-41b4-9d77-919795af2413, opId = 0, ver = 0, numPartitions = 2], 2\n\n(11) HashAggregate [codegen id : 3]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(12) StateStoreSave\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = f7faa1e9-69d9-41b4-9d77-919795af2413, opId = 0, ver = 0, numPartitions = 2], Append, 0, 2\n\n(13) HashAggregate [codegen id : 4]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count(1)#31L AS count#32L]\n\n(14) WriteToDataSourceV2\nInput [2]: [value#29, count#32L]\nArguments: org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@27fafcca\n\n","sparkPlanInfo":{"nodeName":"WriteToDataSourceV2","simpleString":"WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@27fafcca","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreSave","simpleString":"StateStoreSave [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 0, numPartitions = 2], Complete, 0, 2","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreRestore","simpleString":"StateStoreRestore [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 0, numPartitions = 2], 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(value#29, 2), true, [id=#66]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[partial_count(1)])","children":[{"nodeName":"SerializeFromObject","simpleString":"SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MapPartitions","simpleString":"MapPartitions org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject value#21.toString, obj#27: java.lang.String","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [cast(value#8 as string) AS value#21]","children":[{"nodeName":"Project","simpleString":"Project [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MicroBatchScan","simpleString":"MicroBatchScan[key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13] class org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":80,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":79,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":76,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":77,"metricType":"timing"},{"name":"peak memory","accumulatorId":75,"metricType":"size"},{"name":"number of output rows","accumulatorId":74,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":78,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":71,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":72,"metricType":"timing"},{"name":"peak memory","accumulatorId":70,"metricType":"size"},{"name":"number of output rows","accumulatorId":69,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":73,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":68,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":20,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":21,"metricType":"nsTiming"},{"name":"records read","accumulatorId":18,"metricType":"sum"},{"name":"local bytes read","accumulatorId":16,"metricType":"size"},{"name":"fetch wait time","accumulatorId":17,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":14,"metricType":"size"},{"name":"local blocks read","accumulatorId":13,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":12,"metricType":"sum"},{"name":"data size","accumulatorId":11,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":15,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":19,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":67,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":64,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":65,"metricType":"timing"},{"name":"peak memory","accumulatorId":63,"metricType":"size"},{"name":"number of output rows","accumulatorId":62,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":66,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":61,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of inputs which are later than watermark ('inputs' are relative to operators)","accumulatorId":51,"metricType":"sum"},{"name":"number of total state rows","accumulatorId":52,"metricType":"sum"},{"name":"memory used by state","accumulatorId":57,"metricType":"size"},{"name":"count of cache hit on states cache in provider","accumulatorId":59,"metricType":"sum"},{"name":"number of output rows","accumulatorId":50,"metricType":"sum"},{"name":"estimated size of state only on current version","accumulatorId":58,"metricType":"size"},{"name":"count of cache miss on states cache in provider","accumulatorId":60,"metricType":"sum"},{"name":"time to commit changes","accumulatorId":56,"metricType":"timing"},{"name":"time to remove","accumulatorId":55,"metricType":"timing"},{"name":"number of updated state rows","accumulatorId":53,"metricType":"sum"},{"name":"time to update","accumulatorId":54,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":47,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":48,"metricType":"timing"},{"name":"peak memory","accumulatorId":46,"metricType":"size"},{"name":"number of output rows","accumulatorId":45,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":49,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":44,"metricType":"timing"}]}],"metadata":{},"metrics":[]},"time":1596020220179}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":1,"description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 0","details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nWriteToDataSourceV2 (14)\n+- * HashAggregate (13)\n   +- StateStoreSave (12)\n      +- * HashAggregate (11)\n         +- StateStoreRestore (10)\n            +- Exchange (9)\n               +- * HashAggregate (8)\n                  +- * HashAggregate (7)\n                     +- * SerializeFromObject (6)\n                        +- MapPartitions (5)\n                           +- DeserializeToObject (4)\n                              +- * Project (3)\n                                 +- * Project (2)\n                                    +- MicroBatchScan (1)\n\n\n(1) MicroBatchScan\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nArguments: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@7e7b182c, KafkaV2[Subscribe[test5]], {\"test5\":{\"0\":48276}}, {\"test5\":{\"0\":48279}}\n\n(2) Project [codegen id : 1]\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(3) Project [codegen id : 1]\nOutput [1]: [cast(value#8 as string) AS value#21]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(4) DeserializeToObject\nInput [1]: [value#21]\nArguments: value#21.toString, obj#27: java.lang.String\n\n(5) MapPartitions\nInput [1]: [obj#27]\nArguments: org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String\n\n(6) SerializeFromObject [codegen id : 2]\nInput [1]: [obj#28]\nArguments: [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]\n\n(7) HashAggregate [codegen id : 2]\nInput [1]: [value#29]\nKeys [1]: [value#29]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(8) HashAggregate [codegen id : 2]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(9) Exchange\nInput [2]: [value#29, count#38L]\nArguments: hashpartitioning(value#29, 2), true, [id=#218]\n\n(10) StateStoreRestore\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = 64a4779b-846a-4f20-9f5c-899a8dbf68d8, opId = 0, ver = 0, numPartitions = 2], 2\n\n(11) HashAggregate [codegen id : 3]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(12) StateStoreSave\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = 64a4779b-846a-4f20-9f5c-899a8dbf68d8, opId = 0, ver = 0, numPartitions = 2], Append, 0, 2\n\n(13) HashAggregate [codegen id : 4]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count(1)#31L AS count#32L]\n\n(14) WriteToDataSourceV2\nInput [2]: [value#29, count#32L]\nArguments: org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@27fafcca\n\n","sparkPlanInfo":{"nodeName":"WriteToDataSourceV2","simpleString":"WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@27fafcca","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreSave","simpleString":"StateStoreSave [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 0, numPartitions = 2], Complete, 0, 2","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreRestore","simpleString":"StateStoreRestore [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 0, numPartitions = 2], 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(value#29, 2), true, [id=#66]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[partial_count(1)])","children":[{"nodeName":"SerializeFromObject","simpleString":"SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MapPartitions","simpleString":"MapPartitions org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject value#21.toString, obj#27: java.lang.String","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [cast(value#8 as string) AS value#21]","children":[{"nodeName":"Project","simpleString":"Project [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MicroBatchScan","simpleString":"MicroBatchScan[key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13] class org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":80,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":79,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":76,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":77,"metricType":"timing"},{"name":"peak memory","accumulatorId":75,"metricType":"size"},{"name":"number of output rows","accumulatorId":74,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":78,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":71,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":72,"metricType":"timing"},{"name":"peak memory","accumulatorId":70,"metricType":"size"},{"name":"number of output rows","accumulatorId":69,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":73,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":68,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":20,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":21,"metricType":"nsTiming"},{"name":"records read","accumulatorId":18,"metricType":"sum"},{"name":"local bytes read","accumulatorId":16,"metricType":"size"},{"name":"fetch wait time","accumulatorId":17,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":14,"metricType":"size"},{"name":"local blocks read","accumulatorId":13,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":12,"metricType":"sum"},{"name":"data size","accumulatorId":11,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":15,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":19,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":67,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":64,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":65,"metricType":"timing"},{"name":"peak memory","accumulatorId":63,"metricType":"size"},{"name":"number of output rows","accumulatorId":62,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":66,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":61,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of inputs which are later than watermark ('inputs' are relative to operators)","accumulatorId":51,"metricType":"sum"},{"name":"number of total state rows","accumulatorId":52,"metricType":"sum"},{"name":"memory used by state","accumulatorId":57,"metricType":"size"},{"name":"count of cache hit on states cache in provider","accumulatorId":59,"metricType":"sum"},{"name":"number of output rows","accumulatorId":50,"metricType":"sum"},{"name":"estimated size of state only on current version","accumulatorId":58,"metricType":"size"},{"name":"count of cache miss on states cache in provider","accumulatorId":60,"metricType":"sum"},{"name":"time to commit changes","accumulatorId":56,"metricType":"timing"},{"name":"time to remove","accumulatorId":55,"metricType":"timing"},{"name":"number of updated state rows","accumulatorId":53,"metricType":"sum"},{"name":"time to update","accumulatorId":54,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":47,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":48,"metricType":"timing"},{"name":"peak memory","accumulatorId":46,"metricType":"size"},{"name":"number of output rows","accumulatorId":45,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":49,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":44,"metricType":"timing"}]}],"metadata":{},"metrics":[]},"time":1596020220258}
+{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1596020221633,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"15\",\"name\":\"DeserializeToObject\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"20\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"16\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"14\",\"name\":\"MapPartitions\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"DataSourceRDD","Scope":"{\"id\":\"20\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":2,"RDD Info":[{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"StateStoreRDD","Scope":"{\"id\":\"3\",\"name\":\"StateStoreSave\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"StateStoreRDD","Scope":"{\"id\":\"7\",\"name\":\"StateStoreRestore\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[0],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[0,1],"Properties":{"sql.streaming.queryId":"8d268dc2-bc9c-4be8-97a9-b135d2943028","spark.driver.host":"iZbp19vpr16ix621sdw476Z","spark.eventLog.enabled":"true","spark.sql.adaptive.enabled":"false","spark.job.interruptOnCancel":"true","spark.driver.port":"46309","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar","__is_continuous_processing":"false","spark.app.name":"StructuredKafkaWordCount","callSite.long":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","callSite.short":"start at StructuredKafkaWordCount.scala:86","spark.submit.pyFiles":"","spark.job.description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 0","spark.executor.id":"driver","spark.sql.cbo.enabled":"false","streaming.sql.batchId":"0","spark.jobGroup.id":"e225d92f-2545-48f8-87a2-9c0309580f8a","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"/tmp/spark-history","spark.sql.execution.id":"1","spark.app.id":"local-1596020211915","spark.sql.shuffle.partitions":"2"}}
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"15\",\"name\":\"DeserializeToObject\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"20\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"16\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"14\",\"name\":\"MapPartitions\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"DataSourceRDD","Scope":"{\"id\":\"20\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020221656,"Accumulables":[],"Resource Profile Id":0},"Properties":{"sql.streaming.queryId":"8d268dc2-bc9c-4be8-97a9-b135d2943028","spark.driver.host":"iZbp19vpr16ix621sdw476Z","spark.eventLog.enabled":"true","spark.sql.adaptive.enabled":"false","spark.job.interruptOnCancel":"true","spark.driver.port":"46309","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar","__is_continuous_processing":"false","spark.app.name":"StructuredKafkaWordCount","callSite.long":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","callSite.short":"start at StructuredKafkaWordCount.scala:86","spark.submit.pyFiles":"","spark.job.description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 0","spark.executor.id":"driver","spark.sql.cbo.enabled":"false","streaming.sql.batchId":"0","spark.jobGroup.id":"e225d92f-2545-48f8-87a2-9c0309580f8a","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"/tmp/spark-history","spark.sql.execution.id":"1","spark.app.id":"local-1596020211915","spark.sql.shuffle.partitions":"2"}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1596020221738,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1596020221738,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1596020222649,"Failed":false,"Killed":false,"Accumulables":[{"ID":21,"Name":"shuffle write time","Update":"9599308","Value":"9599308","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":20,"Name":"shuffle records written","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":19,"Name":"shuffle bytes written","Update":"168","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":11,"Name":"data size","Update":"128","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":68,"Name":"duration","Update":"296","Value":"296","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":69,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":70,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":72,"Name":"time in aggregation build","Update":"200","Value":"200","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"time in aggregation build","Update":"190","Value":"190","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":79,"Name":"duration","Update":"336","Value":"336","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":80,"Name":"number of output rows","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":3,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":9599308,"Value":9599308,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":168,"Value":168,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":524288,"Internal":true,"Count Failed Values":true},{"ID":109,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":17,"Value":17,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2630,"Value":2630,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":466139164,"Value":466139164,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":503,"Value":503,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":301869581,"Value":301869581,"Internal":true,"Count Failed Values":true},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":361,"Value":361,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":361,"Executor Deserialize CPU Time":301869581,"Executor Run Time":503,"Executor CPU Time":466139164,"Peak Execution Memory":524288,"Result Size":2630,"JVM GC Time":17,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":168,"Shuffle Write Time":9599308,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":3},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"15\",\"name\":\"DeserializeToObject\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"20\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"16\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"14\",\"name\":\"MapPartitions\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"DataSourceRDD","Scope":"{\"id\":\"20\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020221656,"Completion Time":1596020222661,"Accumulables":[{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Value":301869581,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":1,"Internal":true,"Count Failed Values":true},{"ID":77,"Name":"time in aggregation build","Value":"190","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":68,"Name":"duration","Value":"296","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":80,"Name":"number of output rows","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":125,"Name":"internal.metrics.input.recordsRead","Value":3,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Value":2630,"Internal":true,"Count Failed Values":true},{"ID":74,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":11,"Name":"data size","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":20,"Name":"shuffle records written","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":106,"Name":"internal.metrics.executorCpuTime","Value":466139164,"Internal":true,"Count Failed Values":true},{"ID":109,"Name":"internal.metrics.resultSerializationTime","Value":1,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":168,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Value":524288,"Internal":true,"Count Failed Values":true},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Value":361,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"duration","Value":"336","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":70,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":19,"Name":"shuffle bytes written","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Value":9599308,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Value":503,"Internal":true,"Count Failed Values":true},{"ID":69,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":72,"Name":"time in aggregation build","Value":"200","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":108,"Name":"internal.metrics.jvmGCTime","Value":17,"Internal":true,"Count Failed Values":true},{"ID":21,"Name":"shuffle write time","Value":"9599308","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"}],"Resource Profile Id":0}}
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":2,"RDD Info":[{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"StateStoreRDD","Scope":"{\"id\":\"3\",\"name\":\"StateStoreSave\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"StateStoreRDD","Scope":"{\"id\":\"7\",\"name\":\"StateStoreRestore\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[0],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020222688,"Accumulables":[],"Resource Profile Id":0},"Properties":{"sql.streaming.queryId":"8d268dc2-bc9c-4be8-97a9-b135d2943028","spark.driver.host":"iZbp19vpr16ix621sdw476Z","spark.eventLog.enabled":"true","spark.sql.adaptive.enabled":"false","spark.job.interruptOnCancel":"true","spark.driver.port":"46309","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar","__is_continuous_processing":"false","spark.app.name":"StructuredKafkaWordCount","callSite.long":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","callSite.short":"start at StructuredKafkaWordCount.scala:86","spark.submit.pyFiles":"","spark.job.description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 0","spark.executor.id":"driver","spark.sql.cbo.enabled":"false","streaming.sql.batchId":"0","spark.jobGroup.id":"e225d92f-2545-48f8-87a2-9c0309580f8a","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"/tmp/spark-history","spark.sql.execution.id":"1","spark.app.id":"local-1596020211915","spark.sql.shuffle.partitions":"2"}}
+{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Launch Time":1596020222709,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":2,"Index":0,"Attempt":0,"Launch Time":1596020222713,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":2,"Index":0,"Attempt":0,"Launch Time":1596020222713,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1596020222954,"Failed":false,"Killed":false,"Accumulables":[{"ID":44,"Name":"duration","Update":"19","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":48,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"time to update","Update":"14","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"time to remove","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"time to commit changes","Update":"50","Value":"50","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":58,"Name":"estimated size of state only on current version","Update":"64","Value":"64","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"memory used by state","Update":"208","Value":"208","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"14","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":524288,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":5354,"Value":5354,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":93367533,"Value":93367533,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":203,"Value":203,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":10308753,"Value":10308753,"Internal":true,"Count Failed Values":true},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":23,"Value":23,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":23,"Executor Deserialize CPU Time":10308753,"Executor Run Time":203,"Executor CPU Time":93367533,"Peak Execution Memory":524288,"Result Size":5354,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Launch Time":1596020222709,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1596020222965,"Failed":false,"Killed":false,"Accumulables":[{"ID":44,"Name":"duration","Update":"33","Value":"52","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":49,"Name":"avg hash probe bucket list iters","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"peak memory","Update":"4456448","Value":"4718592","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":48,"Name":"time in aggregation build","Update":"19","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"time to update","Update":"28","Value":"42","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"number of updated state rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"time to remove","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"time to commit changes","Update":"31","Value":"81","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":58,"Name":"estimated size of state only on current version","Update":"424","Value":"488","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":50,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"memory used by state","Update":"568","Value":"776","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"number of total state rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"28","Value":"42","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"262144","Value":"524288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":13,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":17,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":16,"Name":"local bytes read","Update":"168","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":18,"Name":"records read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":168,"Value":168,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4718592,"Value":5242880,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":5574,"Value":10928,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":91355172,"Value":184722705,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":205,"Value":408,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":21029530,"Value":31338283,"Internal":true,"Count Failed Values":true},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":34,"Value":57,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":34,"Executor Deserialize CPU Time":21029530,"Executor Run Time":205,"Executor CPU Time":91355172,"Peak Execution Memory":4718592,"Result Size":5574,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":168,"Total Records Read":1},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":2,"RDD Info":[{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"StateStoreRDD","Scope":"{\"id\":\"3\",\"name\":\"StateStoreSave\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"StateStoreRDD","Scope":"{\"id\":\"7\",\"name\":\"StateStoreRestore\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[0],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020222688,"Completion Time":1596020222967,"Accumulables":[{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Value":5242880,"Internal":true,"Count Failed Values":true},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Value":57,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Value":184722705,"Internal":true,"Count Failed Values":true},{"ID":50,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":53,"Name":"number of updated state rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":17,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Value":1,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"duration","Value":"52","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"time to commit changes","Value":"81","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":46,"Name":"peak memory","Value":"4718592","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Value":1,"Internal":true,"Count Failed Values":true},{"ID":55,"Name":"time to remove","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":49,"Name":"avg hash probe bucket list iters","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":58,"Name":"estimated size of state only on current version","Value":"488","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":13,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":130,"Name":"internal.metrics.executorRunTime","Value":408,"Internal":true,"Count Failed Values":true},{"ID":16,"Name":"local bytes read","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"number of total state rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Value":"42","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Value":31338283,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Value":10928,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":45,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Value":"524288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"time to update","Value":"42","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":18,"Name":"records read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"memory used by state","Value":"776","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":48,"Name":"time in aggregation build","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":168,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}}
+{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1596020222973,"Job Result":{"Result":"JobSucceeded"}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":2,"description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 0","details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nLocalTableScan (1)\n\n\n(1) LocalTableScan\nOutput [2]: [value#46, count#47]\nArguments: [value#46, count#47]\n\n","sparkPlanInfo":{"nodeName":"LocalTableScan","simpleString":"LocalTableScan [value#46, count#47]","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":153,"metricType":"sum"}]},"time":1596020223028}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":2,"time":1596020223062}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":1,"time":1596020223069}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":0,"time":1596020223069}
+{"Event":"org.apache.spark.sql.streaming.StreamingQueryListener$QueryProgressEvent","progress":{"id":"8d268dc2-bc9c-4be8-97a9-b135d2943028","runId":"e225d92f-2545-48f8-87a2-9c0309580f8a","name":null,"timestamp":"2020-07-29T10:56:56.015Z","batchId":0,"batchDuration":7110,"durationMs":{"triggerExecution":7109,"queryPlanning":439,"getBatch":21,"latestOffset":3524,"addBatch":3011,"walCommit":35},"eventTime":{},"stateOperators":[{"numRowsTotal":1,"numRowsUpdated":1,"memoryUsedBytes":776,"numLateInputs":0,"customMetrics":{"stateOnCurrentVersionSizeBytes":488,"loadedMapCacheHitCount":0,"loadedMapCacheMissCount":0}}],"sources":[{"description":"KafkaV2[Subscribe[test5]]","startOffset":null,"endOffset":"{\"test5\":{\"0\":48279}}","numInputRows":3,"inputRowsPerSecond":"NaN","processedRowsPerSecond":0.42194092827004215}],"sink":{"description":"org.apache.spark.sql.execution.streaming.ConsoleTable$@514ba885","numOutputRows":1},"observedMetrics":{}}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":3,"description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 1","details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nWriteToDataSourceV2 (14)\n+- * HashAggregate (13)\n   +- StateStoreSave (12)\n      +- * HashAggregate (11)\n         +- StateStoreRestore (10)\n            +- Exchange (9)\n               +- * HashAggregate (8)\n                  +- * HashAggregate (7)\n                     +- * SerializeFromObject (6)\n                        +- MapPartitions (5)\n                           +- DeserializeToObject (4)\n                              +- * Project (3)\n                                 +- * Project (2)\n                                    +- MicroBatchScan (1)\n\n\n(1) MicroBatchScan\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nArguments: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@7e7b182c, KafkaV2[Subscribe[test5]], {\"test5\":{\"0\":48279}}, {\"test5\":{\"0\":48642}}\n\n(2) Project [codegen id : 1]\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(3) Project [codegen id : 1]\nOutput [1]: [cast(value#8 as string) AS value#21]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(4) DeserializeToObject\nInput [1]: [value#21]\nArguments: value#21.toString, obj#27: java.lang.String\n\n(5) MapPartitions\nInput [1]: [obj#27]\nArguments: org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String\n\n(6) SerializeFromObject [codegen id : 2]\nInput [1]: [obj#28]\nArguments: [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]\n\n(7) HashAggregate [codegen id : 2]\nInput [1]: [value#29]\nKeys [1]: [value#29]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(8) HashAggregate [codegen id : 2]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(9) Exchange\nInput [2]: [value#29, count#38L]\nArguments: hashpartitioning(value#29, 2), true, [id=#373]\n\n(10) StateStoreRestore\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = 1fb6b6c6-ced8-4f85-80af-1f3f4c424457, opId = 0, ver = 0, numPartitions = 2], 2\n\n(11) HashAggregate [codegen id : 3]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(12) StateStoreSave\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = 1fb6b6c6-ced8-4f85-80af-1f3f4c424457, opId = 0, ver = 0, numPartitions = 2], Append, 0, 2\n\n(13) HashAggregate [codegen id : 4]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count(1)#31L AS count#32L]\n\n(14) WriteToDataSourceV2\nInput [2]: [value#29, count#32L]\nArguments: org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@3a1eb73c\n\n","sparkPlanInfo":{"nodeName":"WriteToDataSourceV2","simpleString":"WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@3a1eb73c","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreSave","simpleString":"StateStoreSave [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 1, numPartitions = 2], Complete, 0, 2","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreRestore","simpleString":"StateStoreRestore [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 1, numPartitions = 2], 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(value#29, 2), true, [id=#297]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[partial_count(1)])","children":[{"nodeName":"SerializeFromObject","simpleString":"SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MapPartitions","simpleString":"MapPartitions org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject value#21.toString, obj#27: java.lang.String","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [cast(value#8 as string) AS value#21]","children":[{"nodeName":"Project","simpleString":"Project [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MicroBatchScan","simpleString":"MicroBatchScan[key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13] class org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":237,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":236,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":233,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":234,"metricType":"timing"},{"name":"peak memory","accumulatorId":232,"metricType":"size"},{"name":"number of output rows","accumulatorId":231,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":235,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":228,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":229,"metricType":"timing"},{"name":"peak memory","accumulatorId":227,"metricType":"size"},{"name":"number of output rows","accumulatorId":226,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":230,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":225,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":177,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":178,"metricType":"nsTiming"},{"name":"records read","accumulatorId":175,"metricType":"sum"},{"name":"local bytes read","accumulatorId":173,"metricType":"size"},{"name":"fetch wait time","accumulatorId":174,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":171,"metricType":"size"},{"name":"local blocks read","accumulatorId":170,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":169,"metricType":"sum"},{"name":"data size","accumulatorId":168,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":172,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":176,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":224,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":221,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":222,"metricType":"timing"},{"name":"peak memory","accumulatorId":220,"metricType":"size"},{"name":"number of output rows","accumulatorId":219,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":223,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":218,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of inputs which are later than watermark ('inputs' are relative to operators)","accumulatorId":208,"metricType":"sum"},{"name":"number of total state rows","accumulatorId":209,"metricType":"sum"},{"name":"memory used by state","accumulatorId":214,"metricType":"size"},{"name":"count of cache hit on states cache in provider","accumulatorId":216,"metricType":"sum"},{"name":"number of output rows","accumulatorId":207,"metricType":"sum"},{"name":"estimated size of state only on current version","accumulatorId":215,"metricType":"size"},{"name":"count of cache miss on states cache in provider","accumulatorId":217,"metricType":"sum"},{"name":"time to commit changes","accumulatorId":213,"metricType":"timing"},{"name":"time to remove","accumulatorId":212,"metricType":"timing"},{"name":"number of updated state rows","accumulatorId":210,"metricType":"sum"},{"name":"time to update","accumulatorId":211,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":204,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":205,"metricType":"timing"},{"name":"peak memory","accumulatorId":203,"metricType":"size"},{"name":"number of output rows","accumulatorId":202,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":206,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":201,"metricType":"timing"}]}],"metadata":{},"metrics":[]},"time":1596020223333}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":4,"description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 1","details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nWriteToDataSourceV2 (14)\n+- * HashAggregate (13)\n   +- StateStoreSave (12)\n      +- * HashAggregate (11)\n         +- StateStoreRestore (10)\n            +- Exchange (9)\n               +- * HashAggregate (8)\n                  +- * HashAggregate (7)\n                     +- * SerializeFromObject (6)\n                        +- MapPartitions (5)\n                           +- DeserializeToObject (4)\n                              +- * Project (3)\n                                 +- * Project (2)\n                                    +- MicroBatchScan (1)\n\n\n(1) MicroBatchScan\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nArguments: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@7e7b182c, KafkaV2[Subscribe[test5]], {\"test5\":{\"0\":48279}}, {\"test5\":{\"0\":48642}}\n\n(2) Project [codegen id : 1]\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(3) Project [codegen id : 1]\nOutput [1]: [cast(value#8 as string) AS value#21]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(4) DeserializeToObject\nInput [1]: [value#21]\nArguments: value#21.toString, obj#27: java.lang.String\n\n(5) MapPartitions\nInput [1]: [obj#27]\nArguments: org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String\n\n(6) SerializeFromObject [codegen id : 2]\nInput [1]: [obj#28]\nArguments: [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]\n\n(7) HashAggregate [codegen id : 2]\nInput [1]: [value#29]\nKeys [1]: [value#29]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(8) HashAggregate [codegen id : 2]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(9) Exchange\nInput [2]: [value#29, count#38L]\nArguments: hashpartitioning(value#29, 2), true, [id=#449]\n\n(10) StateStoreRestore\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = 7992c0a8-0641-440d-aaf7-ad453fe25c0a, opId = 0, ver = 0, numPartitions = 2], 2\n\n(11) HashAggregate [codegen id : 3]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(12) StateStoreSave\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = 7992c0a8-0641-440d-aaf7-ad453fe25c0a, opId = 0, ver = 0, numPartitions = 2], Append, 0, 2\n\n(13) HashAggregate [codegen id : 4]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count(1)#31L AS count#32L]\n\n(14) WriteToDataSourceV2\nInput [2]: [value#29, count#32L]\nArguments: org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@3a1eb73c\n\n","sparkPlanInfo":{"nodeName":"WriteToDataSourceV2","simpleString":"WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@3a1eb73c","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreSave","simpleString":"StateStoreSave [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 1, numPartitions = 2], Complete, 0, 2","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreRestore","simpleString":"StateStoreRestore [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 1, numPartitions = 2], 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(value#29, 2), true, [id=#297]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[partial_count(1)])","children":[{"nodeName":"SerializeFromObject","simpleString":"SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MapPartitions","simpleString":"MapPartitions org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject value#21.toString, obj#27: java.lang.String","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [cast(value#8 as string) AS value#21]","children":[{"nodeName":"Project","simpleString":"Project [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MicroBatchScan","simpleString":"MicroBatchScan[key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13] class org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":237,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":236,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":233,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":234,"metricType":"timing"},{"name":"peak memory","accumulatorId":232,"metricType":"size"},{"name":"number of output rows","accumulatorId":231,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":235,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":228,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":229,"metricType":"timing"},{"name":"peak memory","accumulatorId":227,"metricType":"size"},{"name":"number of output rows","accumulatorId":226,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":230,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":225,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":177,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":178,"metricType":"nsTiming"},{"name":"records read","accumulatorId":175,"metricType":"sum"},{"name":"local bytes read","accumulatorId":173,"metricType":"size"},{"name":"fetch wait time","accumulatorId":174,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":171,"metricType":"size"},{"name":"local blocks read","accumulatorId":170,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":169,"metricType":"sum"},{"name":"data size","accumulatorId":168,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":172,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":176,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":224,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":221,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":222,"metricType":"timing"},{"name":"peak memory","accumulatorId":220,"metricType":"size"},{"name":"number of output rows","accumulatorId":219,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":223,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":218,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of inputs which are later than watermark ('inputs' are relative to operators)","accumulatorId":208,"metricType":"sum"},{"name":"number of total state rows","accumulatorId":209,"metricType":"sum"},{"name":"memory used by state","accumulatorId":214,"metricType":"size"},{"name":"count of cache hit on states cache in provider","accumulatorId":216,"metricType":"sum"},{"name":"number of output rows","accumulatorId":207,"metricType":"sum"},{"name":"estimated size of state only on current version","accumulatorId":215,"metricType":"size"},{"name":"count of cache miss on states cache in provider","accumulatorId":217,"metricType":"sum"},{"name":"time to commit changes","accumulatorId":213,"metricType":"timing"},{"name":"time to remove","accumulatorId":212,"metricType":"timing"},{"name":"number of updated state rows","accumulatorId":210,"metricType":"sum"},{"name":"time to update","accumulatorId":211,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":204,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":205,"metricType":"timing"},{"name":"peak memory","accumulatorId":203,"metricType":"size"},{"name":"number of output rows","accumulatorId":202,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":206,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":201,"metricType":"timing"}]}],"metadata":{},"metrics":[]},"time":1596020223382}
+{"Event":"SparkListenerJobStart","Job ID":1,"Submission Time":1596020223482,"Stage Infos":[{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":1,"RDD Info":[{"RDD ID":18,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"41\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"DataSourceRDD","Scope":"{\"id\":\"53\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"53\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"48\",\"name\":\"DeserializeToObject\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"49\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"47\",\"name\":\"MapPartitions\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"42\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":2,"RDD Info":[{"RDD ID":23,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"33\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[22],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":22,"Name":"StateStoreRDD","Scope":"{\"id\":\"36\",\"name\":\"StateStoreSave\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[21],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"41\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":21,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"37\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[20],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":20,"Name":"StateStoreRDD","Scope":"{\"id\":\"40\",\"name\":\"StateStoreRestore\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[19],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[2],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[2,3],"Properties":{"sql.streaming.queryId":"8d268dc2-bc9c-4be8-97a9-b135d2943028","spark.driver.host":"iZbp19vpr16ix621sdw476Z","spark.eventLog.enabled":"true","spark.sql.adaptive.enabled":"false","spark.job.interruptOnCancel":"true","spark.driver.port":"46309","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar","__is_continuous_processing":"false","spark.app.name":"StructuredKafkaWordCount","callSite.long":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","callSite.short":"start at StructuredKafkaWordCount.scala:86","spark.submit.pyFiles":"","spark.job.description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 1","spark.executor.id":"driver","spark.sql.cbo.enabled":"false","streaming.sql.batchId":"1","spark.jobGroup.id":"e225d92f-2545-48f8-87a2-9c0309580f8a","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"/tmp/spark-history","spark.sql.execution.id":"4","spark.app.id":"local-1596020211915","spark.sql.shuffle.partitions":"2"}}
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":1,"RDD Info":[{"RDD ID":18,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"41\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"DataSourceRDD","Scope":"{\"id\":\"53\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"53\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"48\",\"name\":\"DeserializeToObject\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"49\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"47\",\"name\":\"MapPartitions\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"42\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020223485,"Accumulables":[],"Resource Profile Id":0},"Properties":{"sql.streaming.queryId":"8d268dc2-bc9c-4be8-97a9-b135d2943028","spark.driver.host":"iZbp19vpr16ix621sdw476Z","spark.eventLog.enabled":"true","spark.sql.adaptive.enabled":"false","spark.job.interruptOnCancel":"true","spark.driver.port":"46309","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar","__is_continuous_processing":"false","spark.app.name":"StructuredKafkaWordCount","callSite.long":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","callSite.short":"start at StructuredKafkaWordCount.scala:86","spark.submit.pyFiles":"","spark.job.description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 1","spark.executor.id":"driver","spark.sql.cbo.enabled":"false","streaming.sql.batchId":"1","spark.jobGroup.id":"e225d92f-2545-48f8-87a2-9c0309580f8a","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"/tmp/spark-history","spark.sql.execution.id":"4","spark.app.id":"local-1596020211915","spark.sql.shuffle.partitions":"2"}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":3,"Index":0,"Attempt":0,"Launch Time":1596020223493,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":3,"Index":0,"Attempt":0,"Launch Time":1596020223493,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1596020223601,"Failed":false,"Killed":false,"Accumulables":[{"ID":178,"Name":"shuffle write time","Update":"837580","Value":"837580","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":177,"Name":"shuffle records written","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":176,"Name":"shuffle bytes written","Update":"169","Value":"169","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":168,"Name":"data size","Update":"128","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":225,"Name":"duration","Update":"84","Value":"84","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":226,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":227,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":229,"Name":"time in aggregation build","Update":"74","Value":"74","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":231,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":232,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":234,"Name":"time in aggregation build","Update":"68","Value":"68","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":236,"Name":"duration","Update":"84","Value":"84","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":237,"Name":"number of output rows","Update":"363","Value":"363","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":282,"Name":"internal.metrics.input.recordsRead","Update":363,"Value":363,"Internal":true,"Count Failed Values":true},{"ID":280,"Name":"internal.metrics.shuffle.write.writeTime","Update":837580,"Value":837580,"Internal":true,"Count Failed Values":true},{"ID":279,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":278,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":169,"Value":169,"Internal":true,"Count Failed Values":true},{"ID":269,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":524288,"Internal":true,"Count Failed Values":true},{"ID":264,"Name":"internal.metrics.resultSize","Update":2544,"Value":2544,"Internal":true,"Count Failed Values":true},{"ID":263,"Name":"internal.metrics.executorCpuTime","Update":95945587,"Value":95945587,"Internal":true,"Count Failed Values":true},{"ID":262,"Name":"internal.metrics.executorRunTime","Update":96,"Value":96,"Internal":true,"Count Failed Values":true},{"ID":261,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7437557,"Value":7437557,"Internal":true,"Count Failed Values":true},{"ID":260,"Name":"internal.metrics.executorDeserializeTime","Update":7,"Value":7,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":7,"Executor Deserialize CPU Time":7437557,"Executor Run Time":96,"Executor CPU Time":95945587,"Peak Execution Memory":524288,"Result Size":2544,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":169,"Shuffle Write Time":837580,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":363},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":1,"RDD Info":[{"RDD ID":18,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"41\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"DataSourceRDD","Scope":"{\"id\":\"53\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"53\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"48\",\"name\":\"DeserializeToObject\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"49\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"47\",\"name\":\"MapPartitions\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"42\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020223485,"Completion Time":1596020223603,"Accumulables":[{"ID":227,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":236,"Name":"duration","Value":"84","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":176,"Name":"shuffle bytes written","Value":"169","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":262,"Name":"internal.metrics.executorRunTime","Value":96,"Internal":true,"Count Failed Values":true},{"ID":226,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":280,"Name":"internal.metrics.shuffle.write.writeTime","Value":837580,"Internal":true,"Count Failed Values":true},{"ID":229,"Name":"time in aggregation build","Value":"74","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":232,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"shuffle write time","Value":"837580","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":225,"Name":"duration","Value":"84","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":261,"Name":"internal.metrics.executorDeserializeCpuTime","Value":7437557,"Internal":true,"Count Failed Values":true},{"ID":279,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":1,"Internal":true,"Count Failed Values":true},{"ID":234,"Name":"time in aggregation build","Value":"68","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":264,"Name":"internal.metrics.resultSize","Value":2544,"Internal":true,"Count Failed Values":true},{"ID":282,"Name":"internal.metrics.input.recordsRead","Value":363,"Internal":true,"Count Failed Values":true},{"ID":237,"Name":"number of output rows","Value":"363","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":177,"Name":"shuffle records written","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":168,"Name":"data size","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":231,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":263,"Name":"internal.metrics.executorCpuTime","Value":95945587,"Internal":true,"Count Failed Values":true},{"ID":260,"Name":"internal.metrics.executorDeserializeTime","Value":7,"Internal":true,"Count Failed Values":true},{"ID":269,"Name":"internal.metrics.peakExecutionMemory","Value":524288,"Internal":true,"Count Failed Values":true},{"ID":278,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":169,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}}
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":2,"RDD Info":[{"RDD ID":23,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"33\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[22],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":22,"Name":"StateStoreRDD","Scope":"{\"id\":\"36\",\"name\":\"StateStoreSave\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[21],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"41\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":21,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"37\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[20],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":20,"Name":"StateStoreRDD","Scope":"{\"id\":\"40\",\"name\":\"StateStoreRestore\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[19],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[2],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020223613,"Accumulables":[],"Resource Profile Id":0},"Properties":{"sql.streaming.queryId":"8d268dc2-bc9c-4be8-97a9-b135d2943028","spark.driver.host":"iZbp19vpr16ix621sdw476Z","spark.eventLog.enabled":"true","spark.sql.adaptive.enabled":"false","spark.job.interruptOnCancel":"true","spark.driver.port":"46309","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar","__is_continuous_processing":"false","spark.app.name":"StructuredKafkaWordCount","callSite.long":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","callSite.short":"start at StructuredKafkaWordCount.scala:86","spark.submit.pyFiles":"","spark.job.description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 1","spark.executor.id":"driver","spark.sql.cbo.enabled":"false","streaming.sql.batchId":"1","spark.jobGroup.id":"e225d92f-2545-48f8-87a2-9c0309580f8a","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"/tmp/spark-history","spark.sql.execution.id":"4","spark.app.id":"local-1596020211915","spark.sql.shuffle.partitions":"2"}}
+{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":4,"Index":0,"Attempt":0,"Launch Time":1596020223625,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":5,"Index":1,"Attempt":0,"Launch Time":1596020223626,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":4,"Index":0,"Attempt":0,"Launch Time":1596020223625,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1596020223717,"Failed":false,"Killed":false,"Accumulables":[{"ID":201,"Name":"duration","Update":"4","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":203,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":205,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":211,"Name":"time to update","Update":"6","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":212,"Name":"time to remove","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":213,"Name":"time to commit changes","Update":"38","Value":"38","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":215,"Name":"estimated size of state only on current version","Update":"88","Value":"88","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":216,"Name":"count of cache hit on states cache in provider","Update":"2","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":214,"Name":"memory used by state","Update":"376","Value":"376","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":218,"Name":"duration","Update":"6","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":220,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":222,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":302,"Name":"internal.metrics.shuffle.read.recordsRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":301,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":300,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":299,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":298,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":297,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":296,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":294,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":524288,"Internal":true,"Count Failed Values":true},{"ID":289,"Name":"internal.metrics.resultSize","Update":5311,"Value":5311,"Internal":true,"Count Failed Values":true},{"ID":288,"Name":"internal.metrics.executorCpuTime","Update":22954307,"Value":22954307,"Internal":true,"Count Failed Values":true},{"ID":287,"Name":"internal.metrics.executorRunTime","Update":77,"Value":77,"Internal":true,"Count Failed Values":true},{"ID":286,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6627382,"Value":6627382,"Internal":true,"Count Failed Values":true},{"ID":285,"Name":"internal.metrics.executorDeserializeTime","Update":6,"Value":6,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":6,"Executor Deserialize CPU Time":6627382,"Executor Run Time":77,"Executor CPU Time":22954307,"Peak Execution Memory":524288,"Result Size":5311,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":5,"Index":1,"Attempt":0,"Launch Time":1596020223626,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1596020223720,"Failed":false,"Killed":false,"Accumulables":[{"ID":201,"Name":"duration","Update":"4","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":206,"Name":"avg hash probe bucket list iters","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":202,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":203,"Name":"peak memory","Update":"4456448","Value":"4718592","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":205,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":211,"Name":"time to update","Update":"18","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":210,"Name":"number of updated state rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":212,"Name":"time to remove","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":213,"Name":"time to commit changes","Update":"30","Value":"68","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":215,"Name":"estimated size of state only on current version","Update":"368","Value":"456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":207,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":216,"Name":"count of cache hit on states cache in provider","Update":"2","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":214,"Name":"memory used by state","Update":"840","Value":"1216","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":209,"Name":"number of total state rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":218,"Name":"duration","Update":"19","Value":"25","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":219,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":220,"Name":"peak memory","Update":"262144","Value":"524288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":222,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":224,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"local bytes read","Update":"169","Value":"169","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"records read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":302,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":301,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":300,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":169,"Value":169,"Internal":true,"Count Failed Values":true},{"ID":299,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":298,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":297,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":296,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":294,"Name":"internal.metrics.peakExecutionMemory","Update":4718592,"Value":5242880,"Internal":true,"Count Failed Values":true},{"ID":289,"Name":"internal.metrics.resultSize","Update":5574,"Value":10885,"Internal":true,"Count Failed Values":true},{"ID":288,"Name":"internal.metrics.executorCpuTime","Update":25907369,"Value":48861676,"Internal":true,"Count Failed Values":true},{"ID":287,"Name":"internal.metrics.executorRunTime","Update":82,"Value":159,"Internal":true,"Count Failed Values":true},{"ID":286,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7573630,"Value":14201012,"Internal":true,"Count Failed Values":true},{"ID":285,"Name":"internal.metrics.executorDeserializeTime","Update":7,"Value":13,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":7,"Executor Deserialize CPU Time":7573630,"Executor Run Time":82,"Executor CPU Time":25907369,"Peak Execution Memory":4718592,"Result Size":5574,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":169,"Total Records Read":1},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":2,"RDD Info":[{"RDD ID":23,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"33\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[22],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":22,"Name":"StateStoreRDD","Scope":"{\"id\":\"36\",\"name\":\"StateStoreSave\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[21],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"41\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":21,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"37\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[20],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":20,"Name":"StateStoreRDD","Scope":"{\"id\":\"40\",\"name\":\"StateStoreRestore\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[19],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[2],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020223613,"Completion Time":1596020223724,"Accumulables":[{"ID":218,"Name":"duration","Value":"25","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":209,"Name":"number of total state rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"local bytes read","Value":"169","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":286,"Name":"internal.metrics.executorDeserializeCpuTime","Value":14201012,"Internal":true,"Count Failed Values":true},{"ID":298,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":289,"Name":"internal.metrics.resultSize","Value":10885,"Internal":true,"Count Failed Values":true},{"ID":301,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":175,"Name":"records read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":211,"Name":"time to update","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":202,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":220,"Name":"peak memory","Value":"524288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":214,"Name":"memory used by state","Value":"1216","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":205,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":300,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":169,"Internal":true,"Count Failed Values":true},{"ID":294,"Name":"internal.metrics.peakExecutionMemory","Value":5242880,"Internal":true,"Count Failed Values":true},{"ID":285,"Name":"internal.metrics.executorDeserializeTime","Value":13,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":297,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":288,"Name":"internal.metrics.executorCpuTime","Value":48861676,"Internal":true,"Count Failed Values":true},{"ID":216,"Name":"count of cache hit on states cache in provider","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":210,"Name":"number of updated state rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":219,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":201,"Name":"duration","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":222,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":213,"Name":"time to commit changes","Value":"68","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":299,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":302,"Name":"internal.metrics.shuffle.read.recordsRead","Value":1,"Internal":true,"Count Failed Values":true},{"ID":212,"Name":"time to remove","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":203,"Name":"peak memory","Value":"4718592","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":215,"Name":"estimated size of state only on current version","Value":"456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":287,"Name":"internal.metrics.executorRunTime","Value":159,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"avg hash probe bucket list iters","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":224,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":296,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}}
+{"Event":"SparkListenerJobEnd","Job ID":1,"Completion Time":1596020223725,"Job Result":{"Result":"JobSucceeded"}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":5,"description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 1","details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nLocalTableScan (1)\n\n\n(1) LocalTableScan\nOutput [2]: [value#60, count#61]\nArguments: [value#60, count#61]\n\n","sparkPlanInfo":{"nodeName":"LocalTableScan","simpleString":"LocalTableScan [value#60, count#61]","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":310,"metricType":"sum"}]},"time":1596020223752}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":5,"time":1596020223761}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":4,"time":1596020223762}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":3,"time":1596020223762}
+{"Event":"org.apache.spark.sql.streaming.StreamingQueryListener$QueryProgressEvent","progress":{"id":"8d268dc2-bc9c-4be8-97a9-b135d2943028","runId":"e225d92f-2545-48f8-87a2-9c0309580f8a","name":null,"timestamp":"2020-07-29T10:57:03.168Z","batchId":1,"batchDuration":622,"durationMs":{"triggerExecution":622,"queryPlanning":47,"getBatch":0,"latestOffset":7,"addBatch":478,"walCommit":59},"eventTime":{},"stateOperators":[{"numRowsTotal":1,"numRowsUpdated":1,"memoryUsedBytes":1216,"numLateInputs":0,"customMetrics":{"stateOnCurrentVersionSizeBytes":456,"loadedMapCacheHitCount":4,"loadedMapCacheMissCount":0}}],"sources":[{"description":"KafkaV2[Subscribe[test5]]","startOffset":"{\"test5\":{\"0\":48279}}","endOffset":"{\"test5\":{\"0\":48642}}","numInputRows":363,"inputRowsPerSecond":50.74793792814204,"processedRowsPerSecond":583.6012861736334}],"sink":{"description":"org.apache.spark.sql.execution.streaming.ConsoleTable$@514ba885","numOutputRows":1},"observedMetrics":{}}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":6,"description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 2","details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nWriteToDataSourceV2 (14)\n+- * HashAggregate (13)\n   +- StateStoreSave (12)\n      +- * HashAggregate (11)\n         +- StateStoreRestore (10)\n            +- Exchange (9)\n               +- * HashAggregate (8)\n                  +- * HashAggregate (7)\n                     +- * SerializeFromObject (6)\n                        +- MapPartitions (5)\n                           +- DeserializeToObject (4)\n                              +- * Project (3)\n                                 +- * Project (2)\n                                    +- MicroBatchScan (1)\n\n\n(1) MicroBatchScan\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nArguments: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@7e7b182c, KafkaV2[Subscribe[test5]], {\"test5\":{\"0\":48642}}, {\"test5\":{\"0\":48705}}\n\n(2) Project [codegen id : 1]\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(3) Project [codegen id : 1]\nOutput [1]: [cast(value#8 as string) AS value#21]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(4) DeserializeToObject\nInput [1]: [value#21]\nArguments: value#21.toString, obj#27: java.lang.String\n\n(5) MapPartitions\nInput [1]: [obj#27]\nArguments: org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String\n\n(6) SerializeFromObject [codegen id : 2]\nInput [1]: [obj#28]\nArguments: [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]\n\n(7) HashAggregate [codegen id : 2]\nInput [1]: [value#29]\nKeys [1]: [value#29]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(8) HashAggregate [codegen id : 2]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(9) Exchange\nInput [2]: [value#29, count#38L]\nArguments: hashpartitioning(value#29, 2), true, [id=#604]\n\n(10) StateStoreRestore\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = 39c861a0-0e30-4ca2-b363-495aff0f3f93, opId = 0, ver = 0, numPartitions = 2], 2\n\n(11) HashAggregate [codegen id : 3]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(12) StateStoreSave\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = 39c861a0-0e30-4ca2-b363-495aff0f3f93, opId = 0, ver = 0, numPartitions = 2], Append, 0, 2\n\n(13) HashAggregate [codegen id : 4]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count(1)#31L AS count#32L]\n\n(14) WriteToDataSourceV2\nInput [2]: [value#29, count#32L]\nArguments: org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@52d6c50a\n\n","sparkPlanInfo":{"nodeName":"WriteToDataSourceV2","simpleString":"WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@52d6c50a","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreSave","simpleString":"StateStoreSave [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 2, numPartitions = 2], Complete, 0, 2","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreRestore","simpleString":"StateStoreRestore [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 2, numPartitions = 2], 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(value#29, 2), true, [id=#528]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[partial_count(1)])","children":[{"nodeName":"SerializeFromObject","simpleString":"SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MapPartitions","simpleString":"MapPartitions org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject value#21.toString, obj#27: java.lang.String","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [cast(value#8 as string) AS value#21]","children":[{"nodeName":"Project","simpleString":"Project [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MicroBatchScan","simpleString":"MicroBatchScan[key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13] class org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":394,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":393,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":390,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":391,"metricType":"timing"},{"name":"peak memory","accumulatorId":389,"metricType":"size"},{"name":"number of output rows","accumulatorId":388,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":392,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":385,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":386,"metricType":"timing"},{"name":"peak memory","accumulatorId":384,"metricType":"size"},{"name":"number of output rows","accumulatorId":383,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":387,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":382,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":334,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":335,"metricType":"nsTiming"},{"name":"records read","accumulatorId":332,"metricType":"sum"},{"name":"local bytes read","accumulatorId":330,"metricType":"size"},{"name":"fetch wait time","accumulatorId":331,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":328,"metricType":"size"},{"name":"local blocks read","accumulatorId":327,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":326,"metricType":"sum"},{"name":"data size","accumulatorId":325,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":329,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":333,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":381,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":378,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":379,"metricType":"timing"},{"name":"peak memory","accumulatorId":377,"metricType":"size"},{"name":"number of output rows","accumulatorId":376,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":380,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":375,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of inputs which are later than watermark ('inputs' are relative to operators)","accumulatorId":365,"metricType":"sum"},{"name":"number of total state rows","accumulatorId":366,"metricType":"sum"},{"name":"memory used by state","accumulatorId":371,"metricType":"size"},{"name":"count of cache hit on states cache in provider","accumulatorId":373,"metricType":"sum"},{"name":"number of output rows","accumulatorId":364,"metricType":"sum"},{"name":"estimated size of state only on current version","accumulatorId":372,"metricType":"size"},{"name":"count of cache miss on states cache in provider","accumulatorId":374,"metricType":"sum"},{"name":"time to commit changes","accumulatorId":370,"metricType":"timing"},{"name":"time to remove","accumulatorId":369,"metricType":"timing"},{"name":"number of updated state rows","accumulatorId":367,"metricType":"sum"},{"name":"time to update","accumulatorId":368,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":361,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":362,"metricType":"timing"},{"name":"peak memory","accumulatorId":360,"metricType":"size"},{"name":"number of output rows","accumulatorId":359,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":363,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":358,"metricType":"timing"}]}],"metadata":{},"metrics":[]},"time":1596020223909}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":7,"description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 2","details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nWriteToDataSourceV2 (14)\n+- * HashAggregate (13)\n   +- StateStoreSave (12)\n      +- * HashAggregate (11)\n         +- StateStoreRestore (10)\n            +- Exchange (9)\n               +- * HashAggregate (8)\n                  +- * HashAggregate (7)\n                     +- * SerializeFromObject (6)\n                        +- MapPartitions (5)\n                           +- DeserializeToObject (4)\n                              +- * Project (3)\n                                 +- * Project (2)\n                                    +- MicroBatchScan (1)\n\n\n(1) MicroBatchScan\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nArguments: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@7e7b182c, KafkaV2[Subscribe[test5]], {\"test5\":{\"0\":48642}}, {\"test5\":{\"0\":48705}}\n\n(2) Project [codegen id : 1]\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(3) Project [codegen id : 1]\nOutput [1]: [cast(value#8 as string) AS value#21]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(4) DeserializeToObject\nInput [1]: [value#21]\nArguments: value#21.toString, obj#27: java.lang.String\n\n(5) MapPartitions\nInput [1]: [obj#27]\nArguments: org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String\n\n(6) SerializeFromObject [codegen id : 2]\nInput [1]: [obj#28]\nArguments: [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]\n\n(7) HashAggregate [codegen id : 2]\nInput [1]: [value#29]\nKeys [1]: [value#29]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(8) HashAggregate [codegen id : 2]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(9) Exchange\nInput [2]: [value#29, count#38L]\nArguments: hashpartitioning(value#29, 2), true, [id=#680]\n\n(10) StateStoreRestore\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = c2fd3b95-1ba6-4d3e-8b9c-0256dfd90973, opId = 0, ver = 0, numPartitions = 2], 2\n\n(11) HashAggregate [codegen id : 3]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(12) StateStoreSave\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = c2fd3b95-1ba6-4d3e-8b9c-0256dfd90973, opId = 0, ver = 0, numPartitions = 2], Append, 0, 2\n\n(13) HashAggregate [codegen id : 4]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count(1)#31L AS count#32L]\n\n(14) WriteToDataSourceV2\nInput [2]: [value#29, count#32L]\nArguments: org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@52d6c50a\n\n","sparkPlanInfo":{"nodeName":"WriteToDataSourceV2","simpleString":"WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@52d6c50a","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreSave","simpleString":"StateStoreSave [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 2, numPartitions = 2], Complete, 0, 2","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreRestore","simpleString":"StateStoreRestore [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 2, numPartitions = 2], 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(value#29, 2), true, [id=#528]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[partial_count(1)])","children":[{"nodeName":"SerializeFromObject","simpleString":"SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MapPartitions","simpleString":"MapPartitions org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject value#21.toString, obj#27: java.lang.String","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [cast(value#8 as string) AS value#21]","children":[{"nodeName":"Project","simpleString":"Project [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MicroBatchScan","simpleString":"MicroBatchScan[key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13] class org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":394,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":393,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":390,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":391,"metricType":"timing"},{"name":"peak memory","accumulatorId":389,"metricType":"size"},{"name":"number of output rows","accumulatorId":388,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":392,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":385,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":386,"metricType":"timing"},{"name":"peak memory","accumulatorId":384,"metricType":"size"},{"name":"number of output rows","accumulatorId":383,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":387,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":382,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":334,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":335,"metricType":"nsTiming"},{"name":"records read","accumulatorId":332,"metricType":"sum"},{"name":"local bytes read","accumulatorId":330,"metricType":"size"},{"name":"fetch wait time","accumulatorId":331,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":328,"metricType":"size"},{"name":"local blocks read","accumulatorId":327,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":326,"metricType":"sum"},{"name":"data size","accumulatorId":325,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":329,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":333,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":381,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":378,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":379,"metricType":"timing"},{"name":"peak memory","accumulatorId":377,"metricType":"size"},{"name":"number of output rows","accumulatorId":376,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":380,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":375,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of inputs which are later than watermark ('inputs' are relative to operators)","accumulatorId":365,"metricType":"sum"},{"name":"number of total state rows","accumulatorId":366,"metricType":"sum"},{"name":"memory used by state","accumulatorId":371,"metricType":"size"},{"name":"count of cache hit on states cache in provider","accumulatorId":373,"metricType":"sum"},{"name":"number of output rows","accumulatorId":364,"metricType":"sum"},{"name":"estimated size of state only on current version","accumulatorId":372,"metricType":"size"},{"name":"count of cache miss on states cache in provider","accumulatorId":374,"metricType":"sum"},{"name":"time to commit changes","accumulatorId":370,"metricType":"timing"},{"name":"time to remove","accumulatorId":369,"metricType":"timing"},{"name":"number of updated state rows","accumulatorId":367,"metricType":"sum"},{"name":"time to update","accumulatorId":368,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":361,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":362,"metricType":"timing"},{"name":"peak memory","accumulatorId":360,"metricType":"size"},{"name":"number of output rows","accumulatorId":359,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":363,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":358,"metricType":"timing"}]}],"metadata":{},"metrics":[]},"time":1596020224006}
+{"Event":"SparkListenerJobStart","Job ID":2,"Submission Time":1596020224100,"Stage Infos":[{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":2,"RDD Info":[{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"66\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":32,"Name":"StateStoreRDD","Scope":"{\"id\":\"73\",\"name\":\"StateStoreRestore\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[31],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"StateStoreRDD","Scope":"{\"id\":\"69\",\"name\":\"StateStoreSave\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":33,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"70\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[32],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":31,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"74\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[30],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[4],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":1,"RDD Info":[{"RDD ID":30,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"74\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":27,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"81\",\"name\":\"DeserializeToObject\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[26],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"75\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"80\",\"name\":\"MapPartitions\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[27],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":26,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"82\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[25],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":24,"Name":"DataSourceRDD","Scope":"{\"id\":\"86\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":25,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"86\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[24],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[5,4],"Properties":{"sql.streaming.queryId":"8d268dc2-bc9c-4be8-97a9-b135d2943028","spark.driver.host":"iZbp19vpr16ix621sdw476Z","spark.eventLog.enabled":"true","spark.sql.adaptive.enabled":"false","spark.job.interruptOnCancel":"true","spark.driver.port":"46309","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar","__is_continuous_processing":"false","spark.app.name":"StructuredKafkaWordCount","callSite.long":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","callSite.short":"start at StructuredKafkaWordCount.scala:86","spark.submit.pyFiles":"","spark.job.description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 2","spark.executor.id":"driver","spark.sql.cbo.enabled":"false","streaming.sql.batchId":"2","spark.jobGroup.id":"e225d92f-2545-48f8-87a2-9c0309580f8a","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"/tmp/spark-history","spark.sql.execution.id":"7","spark.app.id":"local-1596020211915","spark.sql.shuffle.partitions":"2"}}
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":1,"RDD Info":[{"RDD ID":30,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"74\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":27,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"81\",\"name\":\"DeserializeToObject\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[26],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"75\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"80\",\"name\":\"MapPartitions\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[27],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":26,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"82\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[25],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":24,"Name":"DataSourceRDD","Scope":"{\"id\":\"86\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":25,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"86\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[24],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020224103,"Accumulables":[],"Resource Profile Id":0},"Properties":{"sql.streaming.queryId":"8d268dc2-bc9c-4be8-97a9-b135d2943028","spark.driver.host":"iZbp19vpr16ix621sdw476Z","spark.eventLog.enabled":"true","spark.sql.adaptive.enabled":"false","spark.job.interruptOnCancel":"true","spark.driver.port":"46309","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar","__is_continuous_processing":"false","spark.app.name":"StructuredKafkaWordCount","callSite.long":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","callSite.short":"start at StructuredKafkaWordCount.scala:86","spark.submit.pyFiles":"","spark.job.description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 2","spark.executor.id":"driver","spark.sql.cbo.enabled":"false","streaming.sql.batchId":"2","spark.jobGroup.id":"e225d92f-2545-48f8-87a2-9c0309580f8a","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"/tmp/spark-history","spark.sql.execution.id":"7","spark.app.id":"local-1596020211915","spark.sql.shuffle.partitions":"2"}}
+{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":6,"Index":0,"Attempt":0,"Launch Time":1596020224113,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":6,"Index":0,"Attempt":0,"Launch Time":1596020224113,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1596020224174,"Failed":false,"Killed":false,"Accumulables":[{"ID":335,"Name":"shuffle write time","Update":"686296","Value":"686296","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":334,"Name":"shuffle records written","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":333,"Name":"shuffle bytes written","Update":"168","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":325,"Name":"data size","Update":"128","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":382,"Name":"duration","Update":"39","Value":"39","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":383,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":384,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":386,"Name":"time in aggregation build","Update":"32","Value":"32","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":388,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":389,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":391,"Name":"time in aggregation build","Update":"26","Value":"26","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":393,"Name":"duration","Update":"40","Value":"40","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":394,"Name":"number of output rows","Update":"63","Value":"63","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":439,"Name":"internal.metrics.input.recordsRead","Update":63,"Value":63,"Internal":true,"Count Failed Values":true},{"ID":437,"Name":"internal.metrics.shuffle.write.writeTime","Update":686296,"Value":686296,"Internal":true,"Count Failed Values":true},{"ID":436,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":435,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":168,"Value":168,"Internal":true,"Count Failed Values":true},{"ID":426,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":524288,"Internal":true,"Count Failed Values":true},{"ID":421,"Name":"internal.metrics.resultSize","Update":2544,"Value":2544,"Internal":true,"Count Failed Values":true},{"ID":420,"Name":"internal.metrics.executorCpuTime","Update":33390843,"Value":33390843,"Internal":true,"Count Failed Values":true},{"ID":419,"Name":"internal.metrics.executorRunTime","Update":49,"Value":49,"Internal":true,"Count Failed Values":true},{"ID":418,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4867521,"Value":4867521,"Internal":true,"Count Failed Values":true},{"ID":417,"Name":"internal.metrics.executorDeserializeTime","Update":8,"Value":8,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":8,"Executor Deserialize CPU Time":4867521,"Executor Run Time":49,"Executor CPU Time":33390843,"Peak Execution Memory":524288,"Result Size":2544,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":168,"Shuffle Write Time":686296,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":63},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":1,"RDD Info":[{"RDD ID":30,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"74\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":27,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"81\",\"name\":\"DeserializeToObject\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[26],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"75\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"80\",\"name\":\"MapPartitions\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[27],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":26,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"82\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[25],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":24,"Name":"DataSourceRDD","Scope":"{\"id\":\"86\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":25,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"86\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[24],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020224103,"Completion Time":1596020224175,"Accumulables":[{"ID":436,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":1,"Internal":true,"Count Failed Values":true},{"ID":391,"Name":"time in aggregation build","Value":"26","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":382,"Name":"duration","Value":"39","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":418,"Name":"internal.metrics.executorDeserializeCpuTime","Value":4867521,"Internal":true,"Count Failed Values":true},{"ID":421,"Name":"internal.metrics.resultSize","Value":2544,"Internal":true,"Count Failed Values":true},{"ID":394,"Name":"number of output rows","Value":"63","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":439,"Name":"internal.metrics.input.recordsRead","Value":63,"Internal":true,"Count Failed Values":true},{"ID":388,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":334,"Name":"shuffle records written","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":325,"Name":"data size","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":420,"Name":"internal.metrics.executorCpuTime","Value":33390843,"Internal":true,"Count Failed Values":true},{"ID":426,"Name":"internal.metrics.peakExecutionMemory","Value":524288,"Internal":true,"Count Failed Values":true},{"ID":417,"Name":"internal.metrics.executorDeserializeTime","Value":8,"Internal":true,"Count Failed Values":true},{"ID":435,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":168,"Internal":true,"Count Failed Values":true},{"ID":384,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":393,"Name":"duration","Value":"40","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":333,"Name":"shuffle bytes written","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":383,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":437,"Name":"internal.metrics.shuffle.write.writeTime","Value":686296,"Internal":true,"Count Failed Values":true},{"ID":419,"Name":"internal.metrics.executorRunTime","Value":49,"Internal":true,"Count Failed Values":true},{"ID":386,"Name":"time in aggregation build","Value":"32","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":335,"Name":"shuffle write time","Value":"686296","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":389,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"}],"Resource Profile Id":0}}
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":2,"RDD Info":[{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"66\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":32,"Name":"StateStoreRDD","Scope":"{\"id\":\"73\",\"name\":\"StateStoreRestore\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[31],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"StateStoreRDD","Scope":"{\"id\":\"69\",\"name\":\"StateStoreSave\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":33,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"70\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[32],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":31,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"74\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[30],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[4],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020224179,"Accumulables":[],"Resource Profile Id":0},"Properties":{"sql.streaming.queryId":"8d268dc2-bc9c-4be8-97a9-b135d2943028","spark.driver.host":"iZbp19vpr16ix621sdw476Z","spark.eventLog.enabled":"true","spark.sql.adaptive.enabled":"false","spark.job.interruptOnCancel":"true","spark.driver.port":"46309","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar","__is_continuous_processing":"false","spark.app.name":"StructuredKafkaWordCount","callSite.long":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","callSite.short":"start at StructuredKafkaWordCount.scala:86","spark.submit.pyFiles":"","spark.job.description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 2","spark.executor.id":"driver","spark.sql.cbo.enabled":"false","streaming.sql.batchId":"2","spark.jobGroup.id":"e225d92f-2545-48f8-87a2-9c0309580f8a","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"/tmp/spark-history","spark.sql.execution.id":"7","spark.app.id":"local-1596020211915","spark.sql.shuffle.partitions":"2"}}
+{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":7,"Index":0,"Attempt":0,"Launch Time":1596020224187,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":8,"Index":1,"Attempt":0,"Launch Time":1596020224187,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":7,"Index":0,"Attempt":0,"Launch Time":1596020224187,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1596020224256,"Failed":false,"Killed":false,"Accumulables":[{"ID":358,"Name":"duration","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":360,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":362,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":368,"Name":"time to update","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":369,"Name":"time to remove","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":370,"Name":"time to commit changes","Update":"32","Value":"32","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":372,"Name":"estimated size of state only on current version","Update":"88","Value":"88","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":373,"Name":"count of cache hit on states cache in provider","Update":"4","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":371,"Name":"memory used by state","Update":"400","Value":"400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":375,"Name":"duration","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":377,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":379,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":459,"Name":"internal.metrics.shuffle.read.recordsRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":458,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":457,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":456,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":455,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":454,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":453,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":451,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":524288,"Internal":true,"Count Failed Values":true},{"ID":446,"Name":"internal.metrics.resultSize","Update":5311,"Value":5311,"Internal":true,"Count Failed Values":true},{"ID":445,"Name":"internal.metrics.executorCpuTime","Update":17230622,"Value":17230622,"Internal":true,"Count Failed Values":true},{"ID":444,"Name":"internal.metrics.executorRunTime","Update":56,"Value":56,"Internal":true,"Count Failed Values":true},{"ID":443,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5948051,"Value":5948051,"Internal":true,"Count Failed Values":true},{"ID":442,"Name":"internal.metrics.executorDeserializeTime","Update":6,"Value":6,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":6,"Executor Deserialize CPU Time":5948051,"Executor Run Time":56,"Executor CPU Time":17230622,"Peak Execution Memory":524288,"Result Size":5311,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":8,"Index":1,"Attempt":0,"Launch Time":1596020224187,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1596020224257,"Failed":false,"Killed":false,"Accumulables":[{"ID":358,"Name":"duration","Update":"4","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":363,"Name":"avg hash probe bucket list iters","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":359,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":360,"Name":"peak memory","Update":"4456448","Value":"4718592","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":362,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":368,"Name":"time to update","Update":"21","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":367,"Name":"number of updated state rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":369,"Name":"time to remove","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":370,"Name":"time to commit changes","Update":"18","Value":"50","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":372,"Name":"estimated size of state only on current version","Update":"368","Value":"456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":364,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":373,"Name":"count of cache hit on states cache in provider","Update":"4","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":371,"Name":"memory used by state","Update":"784","Value":"1184","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":366,"Name":"number of total state rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":375,"Name":"duration","Update":"22","Value":"25","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":376,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":377,"Name":"peak memory","Update":"262144","Value":"524288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":379,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":381,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":327,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":331,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":330,"Name":"local bytes read","Update":"168","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":332,"Name":"records read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":459,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":458,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":457,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":168,"Value":168,"Internal":true,"Count Failed Values":true},{"ID":456,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":455,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":454,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":453,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":451,"Name":"internal.metrics.peakExecutionMemory","Update":4718592,"Value":5242880,"Internal":true,"Count Failed Values":true},{"ID":446,"Name":"internal.metrics.resultSize","Update":5574,"Value":10885,"Internal":true,"Count Failed Values":true},{"ID":445,"Name":"internal.metrics.executorCpuTime","Update":23808555,"Value":41039177,"Internal":true,"Count Failed Values":true},{"ID":444,"Name":"internal.metrics.executorRunTime","Update":56,"Value":112,"Internal":true,"Count Failed Values":true},{"ID":443,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6247106,"Value":12195157,"Internal":true,"Count Failed Values":true},{"ID":442,"Name":"internal.metrics.executorDeserializeTime","Update":6,"Value":12,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":6,"Executor Deserialize CPU Time":6247106,"Executor Run Time":56,"Executor CPU Time":23808555,"Peak Execution Memory":4718592,"Result Size":5574,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":168,"Total Records Read":1},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":2,"RDD Info":[{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"66\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":32,"Name":"StateStoreRDD","Scope":"{\"id\":\"73\",\"name\":\"StateStoreRestore\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[31],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"StateStoreRDD","Scope":"{\"id\":\"69\",\"name\":\"StateStoreSave\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":33,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"70\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[32],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":31,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"74\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[30],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[4],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020224179,"Completion Time":1596020224259,"Accumulables":[{"ID":442,"Name":"internal.metrics.executorDeserializeTime","Value":12,"Internal":true,"Count Failed Values":true},{"ID":451,"Name":"internal.metrics.peakExecutionMemory","Value":5242880,"Internal":true,"Count Failed Values":true},{"ID":445,"Name":"internal.metrics.executorCpuTime","Value":41039177,"Internal":true,"Count Failed Values":true},{"ID":364,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":454,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":373,"Name":"count of cache hit on states cache in provider","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":367,"Name":"number of updated state rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":376,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":358,"Name":"duration","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":331,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":457,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":168,"Internal":true,"Count Failed Values":true},{"ID":379,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":370,"Name":"time to commit changes","Value":"50","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":456,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":369,"Name":"time to remove","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":459,"Name":"internal.metrics.shuffle.read.recordsRead","Value":1,"Internal":true,"Count Failed Values":true},{"ID":360,"Name":"peak memory","Value":"4718592","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":381,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":453,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":372,"Name":"estimated size of state only on current version","Value":"456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":363,"Name":"avg hash probe bucket list iters","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":327,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":444,"Name":"internal.metrics.executorRunTime","Value":112,"Internal":true,"Count Failed Values":true},{"ID":375,"Name":"duration","Value":"25","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":366,"Name":"number of total state rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":330,"Name":"local bytes read","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":443,"Name":"internal.metrics.executorDeserializeCpuTime","Value":12195157,"Internal":true,"Count Failed Values":true},{"ID":455,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":446,"Name":"internal.metrics.resultSize","Value":10885,"Internal":true,"Count Failed Values":true},{"ID":332,"Name":"records read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":377,"Name":"peak memory","Value":"524288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":359,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":458,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":368,"Name":"time to update","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":362,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":371,"Name":"memory used by state","Value":"1184","Internal":true,"Count Failed Values":true,"Metadata":"sql"}],"Resource Profile Id":0}}
+{"Event":"SparkListenerJobEnd","Job ID":2,"Completion Time":1596020224259,"Job Result":{"Result":"JobSucceeded"}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":8,"description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 2","details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nLocalTableScan (1)\n\n\n(1) LocalTableScan\nOutput [2]: [value#74, count#75]\nArguments: [value#74, count#75]\n\n","sparkPlanInfo":{"nodeName":"LocalTableScan","simpleString":"LocalTableScan [value#74, count#75]","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":467,"metricType":"sum"}]},"time":1596020224278}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":8,"time":1596020224287}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":7,"time":1596020224287}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":6,"time":1596020224288}
+{"Event":"org.apache.spark.sql.streaming.StreamingQueryListener$QueryProgressEvent","progress":{"id":"8d268dc2-bc9c-4be8-97a9-b135d2943028","runId":"e225d92f-2545-48f8-87a2-9c0309580f8a","name":null,"timestamp":"2020-07-29T10:57:03.793Z","batchId":2,"batchDuration":522,"durationMs":{"triggerExecution":522,"queryPlanning":41,"getBatch":1,"latestOffset":3,"addBatch":421,"walCommit":27},"eventTime":{},"stateOperators":[{"numRowsTotal":1,"numRowsUpdated":1,"memoryUsedBytes":1184,"numLateInputs":0,"customMetrics":{"stateOnCurrentVersionSizeBytes":456,"loadedMapCacheHitCount":8,"loadedMapCacheMissCount":0}}],"sources":[{"description":"KafkaV2[Subscribe[test5]]","startOffset":"{\"test5\":{\"0\":48642}}","endOffset":"{\"test5\":{\"0\":48705}}","numInputRows":63,"inputRowsPerSecond":100.8,"processedRowsPerSecond":120.6896551724138}],"sink":{"description":"org.apache.spark.sql.execution.streaming.ConsoleTable$@514ba885","numOutputRows":1},"observedMetrics":{}}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":9,"description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 3","details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nWriteToDataSourceV2 (14)\n+- * HashAggregate (13)\n   +- StateStoreSave (12)\n      +- * HashAggregate (11)\n         +- StateStoreRestore (10)\n            +- Exchange (9)\n               +- * HashAggregate (8)\n                  +- * HashAggregate (7)\n                     +- * SerializeFromObject (6)\n                        +- MapPartitions (5)\n                           +- DeserializeToObject (4)\n                              +- * Project (3)\n                                 +- * Project (2)\n                                    +- MicroBatchScan (1)\n\n\n(1) MicroBatchScan\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nArguments: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@7e7b182c, KafkaV2[Subscribe[test5]], {\"test5\":{\"0\":48705}}, {\"test5\":{\"0\":48757}}\n\n(2) Project [codegen id : 1]\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(3) Project [codegen id : 1]\nOutput [1]: [cast(value#8 as string) AS value#21]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(4) DeserializeToObject\nInput [1]: [value#21]\nArguments: value#21.toString, obj#27: java.lang.String\n\n(5) MapPartitions\nInput [1]: [obj#27]\nArguments: org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String\n\n(6) SerializeFromObject [codegen id : 2]\nInput [1]: [obj#28]\nArguments: [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]\n\n(7) HashAggregate [codegen id : 2]\nInput [1]: [value#29]\nKeys [1]: [value#29]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(8) HashAggregate [codegen id : 2]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(9) Exchange\nInput [2]: [value#29, count#38L]\nArguments: hashpartitioning(value#29, 2), true, [id=#835]\n\n(10) StateStoreRestore\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = 8bb5d8a6-42f8-4141-8f25-e1b98f81aac4, opId = 0, ver = 0, numPartitions = 2], 2\n\n(11) HashAggregate [codegen id : 3]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(12) StateStoreSave\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = 8bb5d8a6-42f8-4141-8f25-e1b98f81aac4, opId = 0, ver = 0, numPartitions = 2], Append, 0, 2\n\n(13) HashAggregate [codegen id : 4]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count(1)#31L AS count#32L]\n\n(14) WriteToDataSourceV2\nInput [2]: [value#29, count#32L]\nArguments: org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@59b7c509\n\n","sparkPlanInfo":{"nodeName":"WriteToDataSourceV2","simpleString":"WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@59b7c509","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreSave","simpleString":"StateStoreSave [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 3, numPartitions = 2], Complete, 0, 2","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreRestore","simpleString":"StateStoreRestore [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 3, numPartitions = 2], 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(value#29, 2), true, [id=#759]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[partial_count(1)])","children":[{"nodeName":"SerializeFromObject","simpleString":"SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MapPartitions","simpleString":"MapPartitions org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject value#21.toString, obj#27: java.lang.String","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [cast(value#8 as string) AS value#21]","children":[{"nodeName":"Project","simpleString":"Project [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MicroBatchScan","simpleString":"MicroBatchScan[key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13] class org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":551,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":550,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":547,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":548,"metricType":"timing"},{"name":"peak memory","accumulatorId":546,"metricType":"size"},{"name":"number of output rows","accumulatorId":545,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":549,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":542,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":543,"metricType":"timing"},{"name":"peak memory","accumulatorId":541,"metricType":"size"},{"name":"number of output rows","accumulatorId":540,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":544,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":539,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":491,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":492,"metricType":"nsTiming"},{"name":"records read","accumulatorId":489,"metricType":"sum"},{"name":"local bytes read","accumulatorId":487,"metricType":"size"},{"name":"fetch wait time","accumulatorId":488,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":485,"metricType":"size"},{"name":"local blocks read","accumulatorId":484,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":483,"metricType":"sum"},{"name":"data size","accumulatorId":482,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":486,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":490,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":538,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":535,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":536,"metricType":"timing"},{"name":"peak memory","accumulatorId":534,"metricType":"size"},{"name":"number of output rows","accumulatorId":533,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":537,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":532,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of inputs which are later than watermark ('inputs' are relative to operators)","accumulatorId":522,"metricType":"sum"},{"name":"number of total state rows","accumulatorId":523,"metricType":"sum"},{"name":"memory used by state","accumulatorId":528,"metricType":"size"},{"name":"count of cache hit on states cache in provider","accumulatorId":530,"metricType":"sum"},{"name":"number of output rows","accumulatorId":521,"metricType":"sum"},{"name":"estimated size of state only on current version","accumulatorId":529,"metricType":"size"},{"name":"count of cache miss on states cache in provider","accumulatorId":531,"metricType":"sum"},{"name":"time to commit changes","accumulatorId":527,"metricType":"timing"},{"name":"time to remove","accumulatorId":526,"metricType":"timing"},{"name":"number of updated state rows","accumulatorId":524,"metricType":"sum"},{"name":"time to update","accumulatorId":525,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":518,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":519,"metricType":"timing"},{"name":"peak memory","accumulatorId":517,"metricType":"size"},{"name":"number of output rows","accumulatorId":516,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":520,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":515,"metricType":"timing"}]}],"metadata":{},"metrics":[]},"time":1596020224419}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":10,"description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 3","details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nWriteToDataSourceV2 (14)\n+- * HashAggregate (13)\n   +- StateStoreSave (12)\n      +- * HashAggregate (11)\n         +- StateStoreRestore (10)\n            +- Exchange (9)\n               +- * HashAggregate (8)\n                  +- * HashAggregate (7)\n                     +- * SerializeFromObject (6)\n                        +- MapPartitions (5)\n                           +- DeserializeToObject (4)\n                              +- * Project (3)\n                                 +- * Project (2)\n                                    +- MicroBatchScan (1)\n\n\n(1) MicroBatchScan\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nArguments: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@7e7b182c, KafkaV2[Subscribe[test5]], {\"test5\":{\"0\":48705}}, {\"test5\":{\"0\":48757}}\n\n(2) Project [codegen id : 1]\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(3) Project [codegen id : 1]\nOutput [1]: [cast(value#8 as string) AS value#21]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(4) DeserializeToObject\nInput [1]: [value#21]\nArguments: value#21.toString, obj#27: java.lang.String\n\n(5) MapPartitions\nInput [1]: [obj#27]\nArguments: org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String\n\n(6) SerializeFromObject [codegen id : 2]\nInput [1]: [obj#28]\nArguments: [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]\n\n(7) HashAggregate [codegen id : 2]\nInput [1]: [value#29]\nKeys [1]: [value#29]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(8) HashAggregate [codegen id : 2]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(9) Exchange\nInput [2]: [value#29, count#38L]\nArguments: hashpartitioning(value#29, 2), true, [id=#911]\n\n(10) StateStoreRestore\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = 29402d2a-a5da-4bb1-8d1a-c6d1c2d998d5, opId = 0, ver = 0, numPartitions = 2], 2\n\n(11) HashAggregate [codegen id : 3]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(12) StateStoreSave\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = 29402d2a-a5da-4bb1-8d1a-c6d1c2d998d5, opId = 0, ver = 0, numPartitions = 2], Append, 0, 2\n\n(13) HashAggregate [codegen id : 4]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count(1)#31L AS count#32L]\n\n(14) WriteToDataSourceV2\nInput [2]: [value#29, count#32L]\nArguments: org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@59b7c509\n\n","sparkPlanInfo":{"nodeName":"WriteToDataSourceV2","simpleString":"WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@59b7c509","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreSave","simpleString":"StateStoreSave [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 3, numPartitions = 2], Complete, 0, 2","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreRestore","simpleString":"StateStoreRestore [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 3, numPartitions = 2], 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(value#29, 2), true, [id=#759]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[partial_count(1)])","children":[{"nodeName":"SerializeFromObject","simpleString":"SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MapPartitions","simpleString":"MapPartitions org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject value#21.toString, obj#27: java.lang.String","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [cast(value#8 as string) AS value#21]","children":[{"nodeName":"Project","simpleString":"Project [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MicroBatchScan","simpleString":"MicroBatchScan[key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13] class org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":551,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":550,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":547,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":548,"metricType":"timing"},{"name":"peak memory","accumulatorId":546,"metricType":"size"},{"name":"number of output rows","accumulatorId":545,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":549,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":542,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":543,"metricType":"timing"},{"name":"peak memory","accumulatorId":541,"metricType":"size"},{"name":"number of output rows","accumulatorId":540,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":544,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":539,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":491,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":492,"metricType":"nsTiming"},{"name":"records read","accumulatorId":489,"metricType":"sum"},{"name":"local bytes read","accumulatorId":487,"metricType":"size"},{"name":"fetch wait time","accumulatorId":488,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":485,"metricType":"size"},{"name":"local blocks read","accumulatorId":484,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":483,"metricType":"sum"},{"name":"data size","accumulatorId":482,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":486,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":490,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":538,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":535,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":536,"metricType":"timing"},{"name":"peak memory","accumulatorId":534,"metricType":"size"},{"name":"number of output rows","accumulatorId":533,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":537,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":532,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of inputs which are later than watermark ('inputs' are relative to operators)","accumulatorId":522,"metricType":"sum"},{"name":"number of total state rows","accumulatorId":523,"metricType":"sum"},{"name":"memory used by state","accumulatorId":528,"metricType":"size"},{"name":"count of cache hit on states cache in provider","accumulatorId":530,"metricType":"sum"},{"name":"number of output rows","accumulatorId":521,"metricType":"sum"},{"name":"estimated size of state only on current version","accumulatorId":529,"metricType":"size"},{"name":"count of cache miss on states cache in provider","accumulatorId":531,"metricType":"sum"},{"name":"time to commit changes","accumulatorId":527,"metricType":"timing"},{"name":"time to remove","accumulatorId":526,"metricType":"timing"},{"name":"number of updated state rows","accumulatorId":524,"metricType":"sum"},{"name":"time to update","accumulatorId":525,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":518,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":519,"metricType":"timing"},{"name":"peak memory","accumulatorId":517,"metricType":"size"},{"name":"number of output rows","accumulatorId":516,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":520,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":515,"metricType":"timing"}]}],"metadata":{},"metrics":[]},"time":1596020224452}
+{"Event":"SparkListenerJobStart","Job ID":3,"Submission Time":1596020224533,"Stage Infos":[{"Stage ID":6,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":1,"RDD Info":[{"RDD ID":42,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"107\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[41],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":38,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"115\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[37],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":36,"Name":"DataSourceRDD","Scope":"{\"id\":\"119\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":41,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"108\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[40],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":37,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"119\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[36],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":40,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"113\",\"name\":\"MapPartitions\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[39],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":39,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"114\",\"name\":\"DeserializeToObject\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[38],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":7,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":2,"RDD Info":[{"RDD ID":47,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"99\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[46],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":46,"Name":"StateStoreRDD","Scope":"{\"id\":\"102\",\"name\":\"StateStoreSave\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[45],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":45,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"103\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[44],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":44,"Name":"StateStoreRDD","Scope":"{\"id\":\"106\",\"name\":\"StateStoreRestore\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[43],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":43,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"107\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[42],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[6],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[6,7],"Properties":{"sql.streaming.queryId":"8d268dc2-bc9c-4be8-97a9-b135d2943028","spark.driver.host":"iZbp19vpr16ix621sdw476Z","spark.eventLog.enabled":"true","spark.sql.adaptive.enabled":"false","spark.job.interruptOnCancel":"true","spark.driver.port":"46309","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar","__is_continuous_processing":"false","spark.app.name":"StructuredKafkaWordCount","callSite.long":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","callSite.short":"start at StructuredKafkaWordCount.scala:86","spark.submit.pyFiles":"","spark.job.description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 3","spark.executor.id":"driver","spark.sql.cbo.enabled":"false","streaming.sql.batchId":"3","spark.jobGroup.id":"e225d92f-2545-48f8-87a2-9c0309580f8a","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"/tmp/spark-history","spark.sql.execution.id":"10","spark.app.id":"local-1596020211915","spark.sql.shuffle.partitions":"2"}}
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":6,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":1,"RDD Info":[{"RDD ID":42,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"107\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[41],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":38,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"115\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[37],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":36,"Name":"DataSourceRDD","Scope":"{\"id\":\"119\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":41,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"108\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[40],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":37,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"119\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[36],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":40,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"113\",\"name\":\"MapPartitions\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[39],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":39,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"114\",\"name\":\"DeserializeToObject\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[38],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020224535,"Accumulables":[],"Resource Profile Id":0},"Properties":{"sql.streaming.queryId":"8d268dc2-bc9c-4be8-97a9-b135d2943028","spark.driver.host":"iZbp19vpr16ix621sdw476Z","spark.eventLog.enabled":"true","spark.sql.adaptive.enabled":"false","spark.job.interruptOnCancel":"true","spark.driver.port":"46309","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar","__is_continuous_processing":"false","spark.app.name":"StructuredKafkaWordCount","callSite.long":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","callSite.short":"start at StructuredKafkaWordCount.scala:86","spark.submit.pyFiles":"","spark.job.description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 3","spark.executor.id":"driver","spark.sql.cbo.enabled":"false","streaming.sql.batchId":"3","spark.jobGroup.id":"e225d92f-2545-48f8-87a2-9c0309580f8a","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"/tmp/spark-history","spark.sql.execution.id":"10","spark.app.id":"local-1596020211915","spark.sql.shuffle.partitions":"2"}}
+{"Event":"SparkListenerTaskStart","Stage ID":6,"Stage Attempt ID":0,"Task Info":{"Task ID":9,"Index":0,"Attempt":0,"Launch Time":1596020224541,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":6,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":9,"Index":0,"Attempt":0,"Launch Time":1596020224541,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1596020224581,"Failed":false,"Killed":false,"Accumulables":[{"ID":492,"Name":"shuffle write time","Update":"643278","Value":"643278","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":491,"Name":"shuffle records written","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":490,"Name":"shuffle bytes written","Update":"168","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":482,"Name":"data size","Update":"128","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":539,"Name":"duration","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":540,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":541,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":543,"Name":"time in aggregation build","Update":"13","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":545,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":546,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":548,"Name":"time in aggregation build","Update":"9","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":550,"Name":"duration","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":551,"Name":"number of output rows","Update":"52","Value":"52","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":596,"Name":"internal.metrics.input.recordsRead","Update":52,"Value":52,"Internal":true,"Count Failed Values":true},{"ID":594,"Name":"internal.metrics.shuffle.write.writeTime","Update":643278,"Value":643278,"Internal":true,"Count Failed Values":true},{"ID":593,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":592,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":168,"Value":168,"Internal":true,"Count Failed Values":true},{"ID":583,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":524288,"Internal":true,"Count Failed Values":true},{"ID":578,"Name":"internal.metrics.resultSize","Update":2544,"Value":2544,"Internal":true,"Count Failed Values":true},{"ID":577,"Name":"internal.metrics.executorCpuTime","Update":29099071,"Value":29099071,"Internal":true,"Count Failed Values":true},{"ID":576,"Name":"internal.metrics.executorRunTime","Update":29,"Value":29,"Internal":true,"Count Failed Values":true},{"ID":575,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3091128,"Value":3091128,"Internal":true,"Count Failed Values":true},{"ID":574,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":3,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3091128,"Executor Run Time":29,"Executor CPU Time":29099071,"Peak Execution Memory":524288,"Result Size":2544,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":168,"Shuffle Write Time":643278,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":52},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":6,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":1,"RDD Info":[{"RDD ID":42,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"107\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[41],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":38,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"115\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[37],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":36,"Name":"DataSourceRDD","Scope":"{\"id\":\"119\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":41,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"108\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[40],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":37,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"119\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[36],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":40,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"113\",\"name\":\"MapPartitions\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[39],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":39,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"114\",\"name\":\"DeserializeToObject\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[38],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020224535,"Completion Time":1596020224582,"Accumulables":[{"ID":550,"Name":"duration","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":541,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":577,"Name":"internal.metrics.executorCpuTime","Value":29099071,"Internal":true,"Count Failed Values":true},{"ID":490,"Name":"shuffle bytes written","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":576,"Name":"internal.metrics.executorRunTime","Value":29,"Internal":true,"Count Failed Values":true},{"ID":540,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":594,"Name":"internal.metrics.shuffle.write.writeTime","Value":643278,"Internal":true,"Count Failed Values":true},{"ID":543,"Name":"time in aggregation build","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":492,"Name":"shuffle write time","Value":"643278","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":546,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":539,"Name":"duration","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":575,"Name":"internal.metrics.executorDeserializeCpuTime","Value":3091128,"Internal":true,"Count Failed Values":true},{"ID":593,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":1,"Internal":true,"Count Failed Values":true},{"ID":548,"Name":"time in aggregation build","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":578,"Name":"internal.metrics.resultSize","Value":2544,"Internal":true,"Count Failed Values":true},{"ID":596,"Name":"internal.metrics.input.recordsRead","Value":52,"Internal":true,"Count Failed Values":true},{"ID":551,"Name":"number of output rows","Value":"52","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":482,"Name":"data size","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":491,"Name":"shuffle records written","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":545,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":592,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":168,"Internal":true,"Count Failed Values":true},{"ID":574,"Name":"internal.metrics.executorDeserializeTime","Value":3,"Internal":true,"Count Failed Values":true},{"ID":583,"Name":"internal.metrics.peakExecutionMemory","Value":524288,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}}
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":7,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":2,"RDD Info":[{"RDD ID":47,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"99\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[46],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":46,"Name":"StateStoreRDD","Scope":"{\"id\":\"102\",\"name\":\"StateStoreSave\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[45],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":45,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"103\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[44],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":44,"Name":"StateStoreRDD","Scope":"{\"id\":\"106\",\"name\":\"StateStoreRestore\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[43],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":43,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"107\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[42],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[6],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020224588,"Accumulables":[],"Resource Profile Id":0},"Properties":{"sql.streaming.queryId":"8d268dc2-bc9c-4be8-97a9-b135d2943028","spark.driver.host":"iZbp19vpr16ix621sdw476Z","spark.eventLog.enabled":"true","spark.sql.adaptive.enabled":"false","spark.job.interruptOnCancel":"true","spark.driver.port":"46309","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar","__is_continuous_processing":"false","spark.app.name":"StructuredKafkaWordCount","callSite.long":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","callSite.short":"start at StructuredKafkaWordCount.scala:86","spark.submit.pyFiles":"","spark.job.description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 3","spark.executor.id":"driver","spark.sql.cbo.enabled":"false","streaming.sql.batchId":"3","spark.jobGroup.id":"e225d92f-2545-48f8-87a2-9c0309580f8a","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"/tmp/spark-history","spark.sql.execution.id":"10","spark.app.id":"local-1596020211915","spark.sql.shuffle.partitions":"2"}}
+{"Event":"SparkListenerTaskStart","Stage ID":7,"Stage Attempt ID":0,"Task Info":{"Task ID":10,"Index":0,"Attempt":0,"Launch Time":1596020224596,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":7,"Stage Attempt ID":0,"Task Info":{"Task ID":11,"Index":1,"Attempt":0,"Launch Time":1596020224597,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":7,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":10,"Index":0,"Attempt":0,"Launch Time":1596020224596,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1596020224670,"Failed":false,"Killed":false,"Accumulables":[{"ID":515,"Name":"duration","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":517,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":525,"Name":"time to update","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":526,"Name":"time to remove","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":527,"Name":"time to commit changes","Update":"27","Value":"27","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":529,"Name":"estimated size of state only on current version","Update":"88","Value":"88","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"count of cache hit on states cache in provider","Update":"6","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":528,"Name":"memory used by state","Update":"400","Value":"400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":532,"Name":"duration","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":534,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":536,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":616,"Name":"internal.metrics.shuffle.read.recordsRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":615,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":614,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":613,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":612,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":611,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":610,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":608,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":524288,"Internal":true,"Count Failed Values":true},{"ID":603,"Name":"internal.metrics.resultSize","Update":5311,"Value":5311,"Internal":true,"Count Failed Values":true},{"ID":602,"Name":"internal.metrics.executorCpuTime","Update":19967906,"Value":19967906,"Internal":true,"Count Failed Values":true},{"ID":601,"Name":"internal.metrics.executorRunTime","Update":62,"Value":62,"Internal":true,"Count Failed Values":true},{"ID":600,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4899567,"Value":4899567,"Internal":true,"Count Failed Values":true},{"ID":599,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":4,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4899567,"Executor Run Time":62,"Executor CPU Time":19967906,"Peak Execution Memory":524288,"Result Size":5311,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":7,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":11,"Index":1,"Attempt":0,"Launch Time":1596020224597,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1596020224687,"Failed":false,"Killed":false,"Accumulables":[{"ID":515,"Name":"duration","Update":"4","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"avg hash probe bucket list iters","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":516,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":517,"Name":"peak memory","Update":"4456448","Value":"4718592","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":525,"Name":"time to update","Update":"17","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":524,"Name":"number of updated state rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":526,"Name":"time to remove","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":527,"Name":"time to commit changes","Update":"26","Value":"53","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":529,"Name":"estimated size of state only on current version","Update":"368","Value":"456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"count of cache hit on states cache in provider","Update":"6","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":528,"Name":"memory used by state","Update":"784","Value":"1184","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":523,"Name":"number of total state rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":532,"Name":"duration","Update":"17","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":533,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":534,"Name":"peak memory","Update":"262144","Value":"524288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":536,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":538,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":484,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":488,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":487,"Name":"local bytes read","Update":"168","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":489,"Name":"records read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":616,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":615,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":614,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":168,"Value":168,"Internal":true,"Count Failed Values":true},{"ID":613,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":612,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":611,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":610,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":608,"Name":"internal.metrics.peakExecutionMemory","Update":4718592,"Value":5242880,"Internal":true,"Count Failed Values":true},{"ID":603,"Name":"internal.metrics.resultSize","Update":5574,"Value":10885,"Internal":true,"Count Failed Values":true},{"ID":602,"Name":"internal.metrics.executorCpuTime","Update":22402538,"Value":42370444,"Internal":true,"Count Failed Values":true},{"ID":601,"Name":"internal.metrics.executorRunTime","Update":79,"Value":141,"Internal":true,"Count Failed Values":true},{"ID":600,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4671511,"Value":9571078,"Internal":true,"Count Failed Values":true},{"ID":599,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":8,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4671511,"Executor Run Time":79,"Executor CPU Time":22402538,"Peak Execution Memory":4718592,"Result Size":5574,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":168,"Total Records Read":1},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":7,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":2,"RDD Info":[{"RDD ID":47,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"99\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[46],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":46,"Name":"StateStoreRDD","Scope":"{\"id\":\"102\",\"name\":\"StateStoreSave\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[45],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":45,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"103\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[44],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":44,"Name":"StateStoreRDD","Scope":"{\"id\":\"106\",\"name\":\"StateStoreRestore\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[43],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":43,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"107\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[42],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[6],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020224588,"Completion Time":1596020224688,"Accumulables":[{"ID":523,"Name":"number of total state rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":532,"Name":"duration","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":487,"Name":"local bytes read","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":517,"Name":"peak memory","Value":"4718592","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":600,"Name":"internal.metrics.executorDeserializeCpuTime","Value":9571078,"Internal":true,"Count Failed Values":true},{"ID":603,"Name":"internal.metrics.resultSize","Value":10885,"Internal":true,"Count Failed Values":true},{"ID":612,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":516,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":615,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":534,"Name":"peak memory","Value":"524288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":525,"Name":"time to update","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":489,"Name":"records read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":528,"Name":"memory used by state","Value":"1184","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":608,"Name":"internal.metrics.peakExecutionMemory","Value":5242880,"Internal":true,"Count Failed Values":true},{"ID":599,"Name":"internal.metrics.executorDeserializeTime","Value":8,"Internal":true,"Count Failed Values":true},{"ID":521,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"count of cache hit on states cache in provider","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":611,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":602,"Name":"internal.metrics.executorCpuTime","Value":42370444,"Internal":true,"Count Failed Values":true},{"ID":488,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":515,"Name":"duration","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":524,"Name":"number of updated state rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":533,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":614,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":168,"Internal":true,"Count Failed Values":true},{"ID":536,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":527,"Name":"time to commit changes","Value":"53","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":613,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":616,"Name":"internal.metrics.shuffle.read.recordsRead","Value":1,"Internal":true,"Count Failed Values":true},{"ID":526,"Name":"time to remove","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"avg hash probe bucket list iters","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":610,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":601,"Name":"internal.metrics.executorRunTime","Value":141,"Internal":true,"Count Failed Values":true},{"ID":484,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":538,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":529,"Name":"estimated size of state only on current version","Value":"456","Internal":true,"Count Failed Values":true,"Metadata":"sql"}],"Resource Profile Id":0}}
+{"Event":"SparkListenerJobEnd","Job ID":3,"Completion Time":1596020224689,"Job Result":{"Result":"JobSucceeded"}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":11,"description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 3","details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nLocalTableScan (1)\n\n\n(1) LocalTableScan\nOutput [2]: [value#88, count#89]\nArguments: [value#88, count#89]\n\n","sparkPlanInfo":{"nodeName":"LocalTableScan","simpleString":"LocalTableScan [value#88, count#89]","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":624,"metricType":"sum"}]},"time":1596020224709}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":11,"time":1596020224713}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":10,"time":1596020224714}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":9,"time":1596020224714}
+{"Event":"org.apache.spark.sql.streaming.StreamingQueryListener$QueryProgressEvent","progress":{"id":"8d268dc2-bc9c-4be8-97a9-b135d2943028","runId":"e225d92f-2545-48f8-87a2-9c0309580f8a","name":null,"timestamp":"2020-07-29T10:57:04.317Z","batchId":3,"batchDuration":415,"durationMs":{"triggerExecution":415,"queryPlanning":38,"getBatch":1,"latestOffset":3,"addBatch":332,"walCommit":21},"eventTime":{},"stateOperators":[{"numRowsTotal":1,"numRowsUpdated":1,"memoryUsedBytes":1184,"numLateInputs":0,"customMetrics":{"stateOnCurrentVersionSizeBytes":456,"loadedMapCacheHitCount":12,"loadedMapCacheMissCount":0}}],"sources":[{"description":"KafkaV2[Subscribe[test5]]","startOffset":"{\"test5\":{\"0\":48705}}","endOffset":"{\"test5\":{\"0\":48757}}","numInputRows":52,"inputRowsPerSecond":99.23664122137404,"processedRowsPerSecond":125.30120481927712}],"sink":{"description":"org.apache.spark.sql.execution.streaming.ConsoleTable$@514ba885","numOutputRows":1},"observedMetrics":{}}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":12,"description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 4","details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nWriteToDataSourceV2 (14)\n+- * HashAggregate (13)\n   +- StateStoreSave (12)\n      +- * HashAggregate (11)\n         +- StateStoreRestore (10)\n            +- Exchange (9)\n               +- * HashAggregate (8)\n                  +- * HashAggregate (7)\n                     +- * SerializeFromObject (6)\n                        +- MapPartitions (5)\n                           +- DeserializeToObject (4)\n                              +- * Project (3)\n                                 +- * Project (2)\n                                    +- MicroBatchScan (1)\n\n\n(1) MicroBatchScan\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nArguments: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@7e7b182c, KafkaV2[Subscribe[test5]], {\"test5\":{\"0\":48757}}, {\"test5\":{\"0\":48799}}\n\n(2) Project [codegen id : 1]\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(3) Project [codegen id : 1]\nOutput [1]: [cast(value#8 as string) AS value#21]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(4) DeserializeToObject\nInput [1]: [value#21]\nArguments: value#21.toString, obj#27: java.lang.String\n\n(5) MapPartitions\nInput [1]: [obj#27]\nArguments: org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String\n\n(6) SerializeFromObject [codegen id : 2]\nInput [1]: [obj#28]\nArguments: [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]\n\n(7) HashAggregate [codegen id : 2]\nInput [1]: [value#29]\nKeys [1]: [value#29]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(8) HashAggregate [codegen id : 2]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(9) Exchange\nInput [2]: [value#29, count#38L]\nArguments: hashpartitioning(value#29, 2), true, [id=#1066]\n\n(10) StateStoreRestore\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = 42efe357-12ef-4061-9b83-20bf4c29a257, opId = 0, ver = 0, numPartitions = 2], 2\n\n(11) HashAggregate [codegen id : 3]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(12) StateStoreSave\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = 42efe357-12ef-4061-9b83-20bf4c29a257, opId = 0, ver = 0, numPartitions = 2], Append, 0, 2\n\n(13) HashAggregate [codegen id : 4]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count(1)#31L AS count#32L]\n\n(14) WriteToDataSourceV2\nInput [2]: [value#29, count#32L]\nArguments: org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@1717338b\n\n","sparkPlanInfo":{"nodeName":"WriteToDataSourceV2","simpleString":"WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@1717338b","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreSave","simpleString":"StateStoreSave [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 4, numPartitions = 2], Complete, 0, 2","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreRestore","simpleString":"StateStoreRestore [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 4, numPartitions = 2], 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(value#29, 2), true, [id=#990]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[partial_count(1)])","children":[{"nodeName":"SerializeFromObject","simpleString":"SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MapPartitions","simpleString":"MapPartitions org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject value#21.toString, obj#27: java.lang.String","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [cast(value#8 as string) AS value#21]","children":[{"nodeName":"Project","simpleString":"Project [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MicroBatchScan","simpleString":"MicroBatchScan[key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13] class org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":708,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":707,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":704,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":705,"metricType":"timing"},{"name":"peak memory","accumulatorId":703,"metricType":"size"},{"name":"number of output rows","accumulatorId":702,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":706,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":699,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":700,"metricType":"timing"},{"name":"peak memory","accumulatorId":698,"metricType":"size"},{"name":"number of output rows","accumulatorId":697,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":701,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":696,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":648,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":649,"metricType":"nsTiming"},{"name":"records read","accumulatorId":646,"metricType":"sum"},{"name":"local bytes read","accumulatorId":644,"metricType":"size"},{"name":"fetch wait time","accumulatorId":645,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":642,"metricType":"size"},{"name":"local blocks read","accumulatorId":641,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":640,"metricType":"sum"},{"name":"data size","accumulatorId":639,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":643,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":647,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":695,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":692,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":693,"metricType":"timing"},{"name":"peak memory","accumulatorId":691,"metricType":"size"},{"name":"number of output rows","accumulatorId":690,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":694,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":689,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of inputs which are later than watermark ('inputs' are relative to operators)","accumulatorId":679,"metricType":"sum"},{"name":"number of total state rows","accumulatorId":680,"metricType":"sum"},{"name":"memory used by state","accumulatorId":685,"metricType":"size"},{"name":"count of cache hit on states cache in provider","accumulatorId":687,"metricType":"sum"},{"name":"number of output rows","accumulatorId":678,"metricType":"sum"},{"name":"estimated size of state only on current version","accumulatorId":686,"metricType":"size"},{"name":"count of cache miss on states cache in provider","accumulatorId":688,"metricType":"sum"},{"name":"time to commit changes","accumulatorId":684,"metricType":"timing"},{"name":"time to remove","accumulatorId":683,"metricType":"timing"},{"name":"number of updated state rows","accumulatorId":681,"metricType":"sum"},{"name":"time to update","accumulatorId":682,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":675,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":676,"metricType":"timing"},{"name":"peak memory","accumulatorId":674,"metricType":"size"},{"name":"number of output rows","accumulatorId":673,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":677,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":672,"metricType":"timing"}]}],"metadata":{},"metrics":[]},"time":1596020224817}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":13,"description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 4","details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nWriteToDataSourceV2 (14)\n+- * HashAggregate (13)\n   +- StateStoreSave (12)\n      +- * HashAggregate (11)\n         +- StateStoreRestore (10)\n            +- Exchange (9)\n               +- * HashAggregate (8)\n                  +- * HashAggregate (7)\n                     +- * SerializeFromObject (6)\n                        +- MapPartitions (5)\n                           +- DeserializeToObject (4)\n                              +- * Project (3)\n                                 +- * Project (2)\n                                    +- MicroBatchScan (1)\n\n\n(1) MicroBatchScan\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nArguments: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@7e7b182c, KafkaV2[Subscribe[test5]], {\"test5\":{\"0\":48757}}, {\"test5\":{\"0\":48799}}\n\n(2) Project [codegen id : 1]\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(3) Project [codegen id : 1]\nOutput [1]: [cast(value#8 as string) AS value#21]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(4) DeserializeToObject\nInput [1]: [value#21]\nArguments: value#21.toString, obj#27: java.lang.String\n\n(5) MapPartitions\nInput [1]: [obj#27]\nArguments: org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String\n\n(6) SerializeFromObject [codegen id : 2]\nInput [1]: [obj#28]\nArguments: [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]\n\n(7) HashAggregate [codegen id : 2]\nInput [1]: [value#29]\nKeys [1]: [value#29]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(8) HashAggregate [codegen id : 2]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(9) Exchange\nInput [2]: [value#29, count#38L]\nArguments: hashpartitioning(value#29, 2), true, [id=#1142]\n\n(10) StateStoreRestore\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = 6fa28bd2-2924-4e01-8bbe-128888d2669b, opId = 0, ver = 0, numPartitions = 2], 2\n\n(11) HashAggregate [codegen id : 3]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(12) StateStoreSave\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = 6fa28bd2-2924-4e01-8bbe-128888d2669b, opId = 0, ver = 0, numPartitions = 2], Append, 0, 2\n\n(13) HashAggregate [codegen id : 4]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count(1)#31L AS count#32L]\n\n(14) WriteToDataSourceV2\nInput [2]: [value#29, count#32L]\nArguments: org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@1717338b\n\n","sparkPlanInfo":{"nodeName":"WriteToDataSourceV2","simpleString":"WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@1717338b","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreSave","simpleString":"StateStoreSave [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 4, numPartitions = 2], Complete, 0, 2","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreRestore","simpleString":"StateStoreRestore [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 4, numPartitions = 2], 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(value#29, 2), true, [id=#990]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[partial_count(1)])","children":[{"nodeName":"SerializeFromObject","simpleString":"SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MapPartitions","simpleString":"MapPartitions org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject value#21.toString, obj#27: java.lang.String","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [cast(value#8 as string) AS value#21]","children":[{"nodeName":"Project","simpleString":"Project [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MicroBatchScan","simpleString":"MicroBatchScan[key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13] class org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":708,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":707,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":704,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":705,"metricType":"timing"},{"name":"peak memory","accumulatorId":703,"metricType":"size"},{"name":"number of output rows","accumulatorId":702,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":706,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":699,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":700,"metricType":"timing"},{"name":"peak memory","accumulatorId":698,"metricType":"size"},{"name":"number of output rows","accumulatorId":697,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":701,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":696,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":648,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":649,"metricType":"nsTiming"},{"name":"records read","accumulatorId":646,"metricType":"sum"},{"name":"local bytes read","accumulatorId":644,"metricType":"size"},{"name":"fetch wait time","accumulatorId":645,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":642,"metricType":"size"},{"name":"local blocks read","accumulatorId":641,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":640,"metricType":"sum"},{"name":"data size","accumulatorId":639,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":643,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":647,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":695,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":692,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":693,"metricType":"timing"},{"name":"peak memory","accumulatorId":691,"metricType":"size"},{"name":"number of output rows","accumulatorId":690,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":694,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":689,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of inputs which are later than watermark ('inputs' are relative to operators)","accumulatorId":679,"metricType":"sum"},{"name":"number of total state rows","accumulatorId":680,"metricType":"sum"},{"name":"memory used by state","accumulatorId":685,"metricType":"size"},{"name":"count of cache hit on states cache in provider","accumulatorId":687,"metricType":"sum"},{"name":"number of output rows","accumulatorId":678,"metricType":"sum"},{"name":"estimated size of state only on current version","accumulatorId":686,"metricType":"size"},{"name":"count of cache miss on states cache in provider","accumulatorId":688,"metricType":"sum"},{"name":"time to commit changes","accumulatorId":684,"metricType":"timing"},{"name":"time to remove","accumulatorId":683,"metricType":"timing"},{"name":"number of updated state rows","accumulatorId":681,"metricType":"sum"},{"name":"time to update","accumulatorId":682,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":675,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":676,"metricType":"timing"},{"name":"peak memory","accumulatorId":674,"metricType":"size"},{"name":"number of output rows","accumulatorId":673,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":677,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":672,"metricType":"timing"}]}],"metadata":{},"metrics":[]},"time":1596020224849}
+{"Event":"SparkListenerJobStart","Job ID":4,"Submission Time":1596020224928,"Stage Infos":[{"Stage ID":9,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":2,"RDD Info":[{"RDD ID":59,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"132\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[58],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":55,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"140\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[54],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":56,"Name":"StateStoreRDD","Scope":"{\"id\":\"139\",\"name\":\"StateStoreRestore\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[55],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":57,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"136\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[56],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":58,"Name":"StateStoreRDD","Scope":"{\"id\":\"135\",\"name\":\"StateStoreSave\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[57],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[8],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":8,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":1,"RDD Info":[{"RDD ID":54,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"140\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[53],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":53,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"141\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[52],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":51,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"147\",\"name\":\"DeserializeToObject\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[50],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":49,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"152\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[48],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":52,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"146\",\"name\":\"MapPartitions\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[51],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":50,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"148\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[49],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":48,"Name":"DataSourceRDD","Scope":"{\"id\":\"152\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[9,8],"Properties":{"sql.streaming.queryId":"8d268dc2-bc9c-4be8-97a9-b135d2943028","spark.driver.host":"iZbp19vpr16ix621sdw476Z","spark.eventLog.enabled":"true","spark.sql.adaptive.enabled":"false","spark.job.interruptOnCancel":"true","spark.driver.port":"46309","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar","__is_continuous_processing":"false","spark.app.name":"StructuredKafkaWordCount","callSite.long":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","callSite.short":"start at StructuredKafkaWordCount.scala:86","spark.submit.pyFiles":"","spark.job.description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 4","spark.executor.id":"driver","spark.sql.cbo.enabled":"false","streaming.sql.batchId":"4","spark.jobGroup.id":"e225d92f-2545-48f8-87a2-9c0309580f8a","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"/tmp/spark-history","spark.sql.execution.id":"13","spark.app.id":"local-1596020211915","spark.sql.shuffle.partitions":"2"}}
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":8,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":1,"RDD Info":[{"RDD ID":54,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"140\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[53],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":53,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"141\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[52],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":51,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"147\",\"name\":\"DeserializeToObject\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[50],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":49,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"152\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[48],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":52,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"146\",\"name\":\"MapPartitions\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[51],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":50,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"148\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[49],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":48,"Name":"DataSourceRDD","Scope":"{\"id\":\"152\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020224929,"Accumulables":[],"Resource Profile Id":0},"Properties":{"sql.streaming.queryId":"8d268dc2-bc9c-4be8-97a9-b135d2943028","spark.driver.host":"iZbp19vpr16ix621sdw476Z","spark.eventLog.enabled":"true","spark.sql.adaptive.enabled":"false","spark.job.interruptOnCancel":"true","spark.driver.port":"46309","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar","__is_continuous_processing":"false","spark.app.name":"StructuredKafkaWordCount","callSite.long":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","callSite.short":"start at StructuredKafkaWordCount.scala:86","spark.submit.pyFiles":"","spark.job.description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 4","spark.executor.id":"driver","spark.sql.cbo.enabled":"false","streaming.sql.batchId":"4","spark.jobGroup.id":"e225d92f-2545-48f8-87a2-9c0309580f8a","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"/tmp/spark-history","spark.sql.execution.id":"13","spark.app.id":"local-1596020211915","spark.sql.shuffle.partitions":"2"}}
+{"Event":"SparkListenerTaskStart","Stage ID":8,"Stage Attempt ID":0,"Task Info":{"Task ID":12,"Index":0,"Attempt":0,"Launch Time":1596020224941,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":8,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":12,"Index":0,"Attempt":0,"Launch Time":1596020224941,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1596020224979,"Failed":false,"Killed":false,"Accumulables":[{"ID":649,"Name":"shuffle write time","Update":"572754","Value":"572754","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":648,"Name":"shuffle records written","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":647,"Name":"shuffle bytes written","Update":"168","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":639,"Name":"data size","Update":"128","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":696,"Name":"duration","Update":"19","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":697,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":698,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":700,"Name":"time in aggregation build","Update":"13","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":702,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":703,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":705,"Name":"time in aggregation build","Update":"9","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":707,"Name":"duration","Update":"19","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":708,"Name":"number of output rows","Update":"42","Value":"42","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":753,"Name":"internal.metrics.input.recordsRead","Update":42,"Value":42,"Internal":true,"Count Failed Values":true},{"ID":751,"Name":"internal.metrics.shuffle.write.writeTime","Update":572754,"Value":572754,"Internal":true,"Count Failed Values":true},{"ID":750,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":749,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":168,"Value":168,"Internal":true,"Count Failed Values":true},{"ID":740,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":524288,"Internal":true,"Count Failed Values":true},{"ID":735,"Name":"internal.metrics.resultSize","Update":2544,"Value":2544,"Internal":true,"Count Failed Values":true},{"ID":734,"Name":"internal.metrics.executorCpuTime","Update":27800373,"Value":27800373,"Internal":true,"Count Failed Values":true},{"ID":733,"Name":"internal.metrics.executorRunTime","Update":28,"Value":28,"Internal":true,"Count Failed Values":true},{"ID":732,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4768103,"Value":4768103,"Internal":true,"Count Failed Values":true},{"ID":731,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":4,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4768103,"Executor Run Time":28,"Executor CPU Time":27800373,"Peak Execution Memory":524288,"Result Size":2544,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":168,"Shuffle Write Time":572754,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":42},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":8,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":1,"RDD Info":[{"RDD ID":54,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"140\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[53],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":53,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"141\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[52],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":51,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"147\",\"name\":\"DeserializeToObject\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[50],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":49,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"152\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[48],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":52,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"146\",\"name\":\"MapPartitions\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[51],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":50,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"148\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[49],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":48,"Name":"DataSourceRDD","Scope":"{\"id\":\"152\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020224929,"Completion Time":1596020224979,"Accumulables":[{"ID":732,"Name":"internal.metrics.executorDeserializeCpuTime","Value":4768103,"Internal":true,"Count Failed Values":true},{"ID":696,"Name":"duration","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":750,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":1,"Internal":true,"Count Failed Values":true},{"ID":705,"Name":"time in aggregation build","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":735,"Name":"internal.metrics.resultSize","Value":2544,"Internal":true,"Count Failed Values":true},{"ID":708,"Name":"number of output rows","Value":"42","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":753,"Name":"internal.metrics.input.recordsRead","Value":42,"Internal":true,"Count Failed Values":true},{"ID":648,"Name":"shuffle records written","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":639,"Name":"data size","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":702,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":740,"Name":"internal.metrics.peakExecutionMemory","Value":524288,"Internal":true,"Count Failed Values":true},{"ID":731,"Name":"internal.metrics.executorDeserializeTime","Value":4,"Internal":true,"Count Failed Values":true},{"ID":749,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":168,"Internal":true,"Count Failed Values":true},{"ID":698,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":734,"Name":"internal.metrics.executorCpuTime","Value":27800373,"Internal":true,"Count Failed Values":true},{"ID":707,"Name":"duration","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":647,"Name":"shuffle bytes written","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":733,"Name":"internal.metrics.executorRunTime","Value":28,"Internal":true,"Count Failed Values":true},{"ID":697,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":751,"Name":"internal.metrics.shuffle.write.writeTime","Value":572754,"Internal":true,"Count Failed Values":true},{"ID":700,"Name":"time in aggregation build","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":649,"Name":"shuffle write time","Value":"572754","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":703,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"}],"Resource Profile Id":0}}
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":9,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":2,"RDD Info":[{"RDD ID":59,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"132\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[58],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":55,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"140\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[54],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":56,"Name":"StateStoreRDD","Scope":"{\"id\":\"139\",\"name\":\"StateStoreRestore\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[55],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":57,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"136\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[56],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":58,"Name":"StateStoreRDD","Scope":"{\"id\":\"135\",\"name\":\"StateStoreSave\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[57],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[8],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020224987,"Accumulables":[],"Resource Profile Id":0},"Properties":{"sql.streaming.queryId":"8d268dc2-bc9c-4be8-97a9-b135d2943028","spark.driver.host":"iZbp19vpr16ix621sdw476Z","spark.eventLog.enabled":"true","spark.sql.adaptive.enabled":"false","spark.job.interruptOnCancel":"true","spark.driver.port":"46309","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar","__is_continuous_processing":"false","spark.app.name":"StructuredKafkaWordCount","callSite.long":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","callSite.short":"start at StructuredKafkaWordCount.scala:86","spark.submit.pyFiles":"","spark.job.description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 4","spark.executor.id":"driver","spark.sql.cbo.enabled":"false","streaming.sql.batchId":"4","spark.jobGroup.id":"e225d92f-2545-48f8-87a2-9c0309580f8a","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"/tmp/spark-history","spark.sql.execution.id":"13","spark.app.id":"local-1596020211915","spark.sql.shuffle.partitions":"2"}}
+{"Event":"SparkListenerTaskStart","Stage ID":9,"Stage Attempt ID":0,"Task Info":{"Task ID":13,"Index":0,"Attempt":0,"Launch Time":1596020224994,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":9,"Stage Attempt ID":0,"Task Info":{"Task ID":14,"Index":1,"Attempt":0,"Launch Time":1596020224994,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":9,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":14,"Index":1,"Attempt":0,"Launch Time":1596020224994,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1596020225056,"Failed":false,"Killed":false,"Accumulables":[{"ID":672,"Name":"duration","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":677,"Name":"avg hash probe bucket list iters","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":673,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":674,"Name":"peak memory","Update":"4456448","Value":"4456448","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":676,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":682,"Name":"time to update","Update":"19","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":681,"Name":"number of updated state rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":683,"Name":"time to remove","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":684,"Name":"time to commit changes","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":686,"Name":"estimated size of state only on current version","Update":"368","Value":"368","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":678,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":687,"Name":"count of cache hit on states cache in provider","Update":"8","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":685,"Name":"memory used by state","Update":"784","Value":"784","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":680,"Name":"number of total state rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":689,"Name":"duration","Update":"19","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":690,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":691,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":693,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":695,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":641,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":645,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":644,"Name":"local bytes read","Update":"168","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":646,"Name":"records read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":773,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":772,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":771,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":168,"Value":168,"Internal":true,"Count Failed Values":true},{"ID":770,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":769,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":768,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":767,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":765,"Name":"internal.metrics.peakExecutionMemory","Update":4718592,"Value":4718592,"Internal":true,"Count Failed Values":true},{"ID":760,"Name":"internal.metrics.resultSize","Update":5574,"Value":5574,"Internal":true,"Count Failed Values":true},{"ID":759,"Name":"internal.metrics.executorCpuTime","Update":19548688,"Value":19548688,"Internal":true,"Count Failed Values":true},{"ID":758,"Name":"internal.metrics.executorRunTime","Update":52,"Value":52,"Internal":true,"Count Failed Values":true},{"ID":757,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5622533,"Value":5622533,"Internal":true,"Count Failed Values":true},{"ID":756,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":5,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":5622533,"Executor Run Time":52,"Executor CPU Time":19548688,"Peak Execution Memory":4718592,"Result Size":5574,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":168,"Total Records Read":1},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":9,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":13,"Index":0,"Attempt":0,"Launch Time":1596020224994,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1596020225058,"Failed":false,"Killed":false,"Accumulables":[{"ID":672,"Name":"duration","Update":"2","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":674,"Name":"peak memory","Update":"262144","Value":"4718592","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":676,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":682,"Name":"time to update","Update":"4","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":683,"Name":"time to remove","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":684,"Name":"time to commit changes","Update":"35","Value":"46","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":686,"Name":"estimated size of state only on current version","Update":"88","Value":"456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":687,"Name":"count of cache hit on states cache in provider","Update":"8","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":685,"Name":"memory used by state","Update":"400","Value":"1184","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":689,"Name":"duration","Update":"4","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":691,"Name":"peak memory","Update":"262144","Value":"524288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":693,"Name":"time in aggregation build","Update":"0","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":773,"Name":"internal.metrics.shuffle.read.recordsRead","Update":0,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":772,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":771,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":168,"Internal":true,"Count Failed Values":true},{"ID":770,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":769,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":768,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":767,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":765,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":5242880,"Internal":true,"Count Failed Values":true},{"ID":760,"Name":"internal.metrics.resultSize","Update":5311,"Value":10885,"Internal":true,"Count Failed Values":true},{"ID":759,"Name":"internal.metrics.executorCpuTime","Update":16813539,"Value":36362227,"Internal":true,"Count Failed Values":true},{"ID":758,"Name":"internal.metrics.executorRunTime","Update":55,"Value":107,"Internal":true,"Count Failed Values":true},{"ID":757,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4322992,"Value":9945525,"Internal":true,"Count Failed Values":true},{"ID":756,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":9,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4322992,"Executor Run Time":55,"Executor CPU Time":16813539,"Peak Execution Memory":524288,"Result Size":5311,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":9,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":2,"RDD Info":[{"RDD ID":59,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"132\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[58],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":55,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"140\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[54],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":56,"Name":"StateStoreRDD","Scope":"{\"id\":\"139\",\"name\":\"StateStoreRestore\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[55],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":57,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"136\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[56],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":58,"Name":"StateStoreRDD","Scope":"{\"id\":\"135\",\"name\":\"StateStoreSave\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[57],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[8],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020224987,"Completion Time":1596020225059,"Accumulables":[{"ID":765,"Name":"internal.metrics.peakExecutionMemory","Value":5242880,"Internal":true,"Count Failed Values":true},{"ID":756,"Name":"internal.metrics.executorDeserializeTime","Value":9,"Internal":true,"Count Failed Values":true},{"ID":678,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":759,"Name":"internal.metrics.executorCpuTime","Value":36362227,"Internal":true,"Count Failed Values":true},{"ID":768,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":687,"Name":"count of cache hit on states cache in provider","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":681,"Name":"number of updated state rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":771,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":168,"Internal":true,"Count Failed Values":true},{"ID":690,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":672,"Name":"duration","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":645,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":684,"Name":"time to commit changes","Value":"46","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":693,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":770,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":683,"Name":"time to remove","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":773,"Name":"internal.metrics.shuffle.read.recordsRead","Value":1,"Internal":true,"Count Failed Values":true},{"ID":686,"Name":"estimated size of state only on current version","Value":"456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":695,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":677,"Name":"avg hash probe bucket list iters","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":767,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":641,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":758,"Name":"internal.metrics.executorRunTime","Value":107,"Internal":true,"Count Failed Values":true},{"ID":644,"Name":"local bytes read","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":680,"Name":"number of total state rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":689,"Name":"duration","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":674,"Name":"peak memory","Value":"4718592","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":757,"Name":"internal.metrics.executorDeserializeCpuTime","Value":9945525,"Internal":true,"Count Failed Values":true},{"ID":769,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":760,"Name":"internal.metrics.resultSize","Value":10885,"Internal":true,"Count Failed Values":true},{"ID":772,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":646,"Name":"records read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":682,"Name":"time to update","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":691,"Name":"peak memory","Value":"524288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":673,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":676,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":685,"Name":"memory used by state","Value":"1184","Internal":true,"Count Failed Values":true,"Metadata":"sql"}],"Resource Profile Id":0}}
+{"Event":"SparkListenerJobEnd","Job ID":4,"Completion Time":1596020225059,"Job Result":{"Result":"JobSucceeded"}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":14,"description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 4","details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nLocalTableScan (1)\n\n\n(1) LocalTableScan\nOutput [2]: [value#102, count#103]\nArguments: [value#102, count#103]\n\n","sparkPlanInfo":{"nodeName":"LocalTableScan","simpleString":"LocalTableScan [value#102, count#103]","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":781,"metricType":"sum"}]},"time":1596020225079}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":14,"time":1596020225087}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":13,"time":1596020225087}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":12,"time":1596020225087}
+{"Event":"org.apache.spark.sql.streaming.StreamingQueryListener$QueryProgressEvent","progress":{"id":"8d268dc2-bc9c-4be8-97a9-b135d2943028","runId":"e225d92f-2545-48f8-87a2-9c0309580f8a","name":null,"timestamp":"2020-07-29T10:57:04.734Z","batchId":4,"batchDuration":387,"durationMs":{"triggerExecution":387,"queryPlanning":30,"getBatch":1,"latestOffset":3,"addBatch":306,"walCommit":12},"eventTime":{},"stateOperators":[{"numRowsTotal":1,"numRowsUpdated":1,"memoryUsedBytes":1184,"numLateInputs":0,"customMetrics":{"stateOnCurrentVersionSizeBytes":456,"loadedMapCacheHitCount":16,"loadedMapCacheMissCount":0}}],"sources":[{"description":"KafkaV2[Subscribe[test5]]","startOffset":"{\"test5\":{\"0\":48757}}","endOffset":"{\"test5\":{\"0\":48799}}","numInputRows":42,"inputRowsPerSecond":100.71942446043165,"processedRowsPerSecond":108.52713178294573}],"sink":{"description":"org.apache.spark.sql.execution.streaming.ConsoleTable$@514ba885","numOutputRows":1},"observedMetrics":{}}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":15,"description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 5","details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nWriteToDataSourceV2 (14)\n+- * HashAggregate (13)\n   +- StateStoreSave (12)\n      +- * HashAggregate (11)\n         +- StateStoreRestore (10)\n            +- Exchange (9)\n               +- * HashAggregate (8)\n                  +- * HashAggregate (7)\n                     +- * SerializeFromObject (6)\n                        +- MapPartitions (5)\n                           +- DeserializeToObject (4)\n                              +- * Project (3)\n                                 +- * Project (2)\n                                    +- MicroBatchScan (1)\n\n\n(1) MicroBatchScan\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nArguments: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@7e7b182c, KafkaV2[Subscribe[test5]], {\"test5\":{\"0\":48799}}, {\"test5\":{\"0\":48837}}\n\n(2) Project [codegen id : 1]\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(3) Project [codegen id : 1]\nOutput [1]: [cast(value#8 as string) AS value#21]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(4) DeserializeToObject\nInput [1]: [value#21]\nArguments: value#21.toString, obj#27: java.lang.String\n\n(5) MapPartitions\nInput [1]: [obj#27]\nArguments: org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String\n\n(6) SerializeFromObject [codegen id : 2]\nInput [1]: [obj#28]\nArguments: [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]\n\n(7) HashAggregate [codegen id : 2]\nInput [1]: [value#29]\nKeys [1]: [value#29]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(8) HashAggregate [codegen id : 2]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(9) Exchange\nInput [2]: [value#29, count#38L]\nArguments: hashpartitioning(value#29, 2), true, [id=#1297]\n\n(10) StateStoreRestore\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = 9579cc6c-8827-43f7-9678-7747602e493e, opId = 0, ver = 0, numPartitions = 2], 2\n\n(11) HashAggregate [codegen id : 3]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(12) StateStoreSave\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = 9579cc6c-8827-43f7-9678-7747602e493e, opId = 0, ver = 0, numPartitions = 2], Append, 0, 2\n\n(13) HashAggregate [codegen id : 4]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count(1)#31L AS count#32L]\n\n(14) WriteToDataSourceV2\nInput [2]: [value#29, count#32L]\nArguments: org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@2c214312\n\n","sparkPlanInfo":{"nodeName":"WriteToDataSourceV2","simpleString":"WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@2c214312","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreSave","simpleString":"StateStoreSave [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 5, numPartitions = 2], Complete, 0, 2","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreRestore","simpleString":"StateStoreRestore [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 5, numPartitions = 2], 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(value#29, 2), true, [id=#1221]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[partial_count(1)])","children":[{"nodeName":"SerializeFromObject","simpleString":"SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MapPartitions","simpleString":"MapPartitions org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject value#21.toString, obj#27: java.lang.String","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [cast(value#8 as string) AS value#21]","children":[{"nodeName":"Project","simpleString":"Project [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MicroBatchScan","simpleString":"MicroBatchScan[key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13] class org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":865,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":864,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":861,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":862,"metricType":"timing"},{"name":"peak memory","accumulatorId":860,"metricType":"size"},{"name":"number of output rows","accumulatorId":859,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":863,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":856,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":857,"metricType":"timing"},{"name":"peak memory","accumulatorId":855,"metricType":"size"},{"name":"number of output rows","accumulatorId":854,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":858,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":853,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":805,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":806,"metricType":"nsTiming"},{"name":"records read","accumulatorId":803,"metricType":"sum"},{"name":"local bytes read","accumulatorId":801,"metricType":"size"},{"name":"fetch wait time","accumulatorId":802,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":799,"metricType":"size"},{"name":"local blocks read","accumulatorId":798,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":797,"metricType":"sum"},{"name":"data size","accumulatorId":796,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":800,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":804,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":852,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":849,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":850,"metricType":"timing"},{"name":"peak memory","accumulatorId":848,"metricType":"size"},{"name":"number of output rows","accumulatorId":847,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":851,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":846,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of inputs which are later than watermark ('inputs' are relative to operators)","accumulatorId":836,"metricType":"sum"},{"name":"number of total state rows","accumulatorId":837,"metricType":"sum"},{"name":"memory used by state","accumulatorId":842,"metricType":"size"},{"name":"count of cache hit on states cache in provider","accumulatorId":844,"metricType":"sum"},{"name":"number of output rows","accumulatorId":835,"metricType":"sum"},{"name":"estimated size of state only on current version","accumulatorId":843,"metricType":"size"},{"name":"count of cache miss on states cache in provider","accumulatorId":845,"metricType":"sum"},{"name":"time to commit changes","accumulatorId":841,"metricType":"timing"},{"name":"time to remove","accumulatorId":840,"metricType":"timing"},{"name":"number of updated state rows","accumulatorId":838,"metricType":"sum"},{"name":"time to update","accumulatorId":839,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":832,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":833,"metricType":"timing"},{"name":"peak memory","accumulatorId":831,"metricType":"size"},{"name":"number of output rows","accumulatorId":830,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":834,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":829,"metricType":"timing"}]}],"metadata":{},"metrics":[]},"time":1596020225211}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":16,"description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 5","details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nWriteToDataSourceV2 (14)\n+- * HashAggregate (13)\n   +- StateStoreSave (12)\n      +- * HashAggregate (11)\n         +- StateStoreRestore (10)\n            +- Exchange (9)\n               +- * HashAggregate (8)\n                  +- * HashAggregate (7)\n                     +- * SerializeFromObject (6)\n                        +- MapPartitions (5)\n                           +- DeserializeToObject (4)\n                              +- * Project (3)\n                                 +- * Project (2)\n                                    +- MicroBatchScan (1)\n\n\n(1) MicroBatchScan\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nArguments: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@7e7b182c, KafkaV2[Subscribe[test5]], {\"test5\":{\"0\":48799}}, {\"test5\":{\"0\":48837}}\n\n(2) Project [codegen id : 1]\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(3) Project [codegen id : 1]\nOutput [1]: [cast(value#8 as string) AS value#21]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(4) DeserializeToObject\nInput [1]: [value#21]\nArguments: value#21.toString, obj#27: java.lang.String\n\n(5) MapPartitions\nInput [1]: [obj#27]\nArguments: org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String\n\n(6) SerializeFromObject [codegen id : 2]\nInput [1]: [obj#28]\nArguments: [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]\n\n(7) HashAggregate [codegen id : 2]\nInput [1]: [value#29]\nKeys [1]: [value#29]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(8) HashAggregate [codegen id : 2]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(9) Exchange\nInput [2]: [value#29, count#38L]\nArguments: hashpartitioning(value#29, 2), true, [id=#1373]\n\n(10) StateStoreRestore\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = b800d96e-7584-4e8d-8df8-c9b901b7f2e2, opId = 0, ver = 0, numPartitions = 2], 2\n\n(11) HashAggregate [codegen id : 3]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(12) StateStoreSave\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = b800d96e-7584-4e8d-8df8-c9b901b7f2e2, opId = 0, ver = 0, numPartitions = 2], Append, 0, 2\n\n(13) HashAggregate [codegen id : 4]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count(1)#31L AS count#32L]\n\n(14) WriteToDataSourceV2\nInput [2]: [value#29, count#32L]\nArguments: org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@2c214312\n\n","sparkPlanInfo":{"nodeName":"WriteToDataSourceV2","simpleString":"WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@2c214312","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreSave","simpleString":"StateStoreSave [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 5, numPartitions = 2], Complete, 0, 2","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreRestore","simpleString":"StateStoreRestore [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 5, numPartitions = 2], 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(value#29, 2), true, [id=#1221]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[partial_count(1)])","children":[{"nodeName":"SerializeFromObject","simpleString":"SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MapPartitions","simpleString":"MapPartitions org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject value#21.toString, obj#27: java.lang.String","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [cast(value#8 as string) AS value#21]","children":[{"nodeName":"Project","simpleString":"Project [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MicroBatchScan","simpleString":"MicroBatchScan[key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13] class org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":865,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":864,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":861,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":862,"metricType":"timing"},{"name":"peak memory","accumulatorId":860,"metricType":"size"},{"name":"number of output rows","accumulatorId":859,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":863,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":856,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":857,"metricType":"timing"},{"name":"peak memory","accumulatorId":855,"metricType":"size"},{"name":"number of output rows","accumulatorId":854,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":858,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":853,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":805,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":806,"metricType":"nsTiming"},{"name":"records read","accumulatorId":803,"metricType":"sum"},{"name":"local bytes read","accumulatorId":801,"metricType":"size"},{"name":"fetch wait time","accumulatorId":802,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":799,"metricType":"size"},{"name":"local blocks read","accumulatorId":798,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":797,"metricType":"sum"},{"name":"data size","accumulatorId":796,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":800,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":804,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":852,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":849,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":850,"metricType":"timing"},{"name":"peak memory","accumulatorId":848,"metricType":"size"},{"name":"number of output rows","accumulatorId":847,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":851,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":846,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of inputs which are later than watermark ('inputs' are relative to operators)","accumulatorId":836,"metricType":"sum"},{"name":"number of total state rows","accumulatorId":837,"metricType":"sum"},{"name":"memory used by state","accumulatorId":842,"metricType":"size"},{"name":"count of cache hit on states cache in provider","accumulatorId":844,"metricType":"sum"},{"name":"number of output rows","accumulatorId":835,"metricType":"sum"},{"name":"estimated size of state only on current version","accumulatorId":843,"metricType":"size"},{"name":"count of cache miss on states cache in provider","accumulatorId":845,"metricType":"sum"},{"name":"time to commit changes","accumulatorId":841,"metricType":"timing"},{"name":"time to remove","accumulatorId":840,"metricType":"timing"},{"name":"number of updated state rows","accumulatorId":838,"metricType":"sum"},{"name":"time to update","accumulatorId":839,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":832,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":833,"metricType":"timing"},{"name":"peak memory","accumulatorId":831,"metricType":"size"},{"name":"number of output rows","accumulatorId":830,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":834,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":829,"metricType":"timing"}]}],"metadata":{},"metrics":[]},"time":1596020225270}
+{"Event":"SparkListenerJobStart","Job ID":5,"Submission Time":1596020225342,"Stage Infos":[{"Stage ID":10,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":1,"RDD Info":[{"RDD ID":66,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"173\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[65],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":62,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"181\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[61],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":64,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"179\",\"name\":\"MapPartitions\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[63],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":61,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"185\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[60],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":65,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"174\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[64],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":60,"Name":"DataSourceRDD","Scope":"{\"id\":\"185\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":63,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"180\",\"name\":\"DeserializeToObject\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[62],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":11,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":2,"RDD Info":[{"RDD ID":71,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"165\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[70],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":70,"Name":"StateStoreRDD","Scope":"{\"id\":\"168\",\"name\":\"StateStoreSave\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[69],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":69,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"169\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[68],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":67,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"173\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[66],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":68,"Name":"StateStoreRDD","Scope":"{\"id\":\"172\",\"name\":\"StateStoreRestore\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[67],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[10],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[10,11],"Properties":{"sql.streaming.queryId":"8d268dc2-bc9c-4be8-97a9-b135d2943028","spark.driver.host":"iZbp19vpr16ix621sdw476Z","spark.eventLog.enabled":"true","spark.sql.adaptive.enabled":"false","spark.job.interruptOnCancel":"true","spark.driver.port":"46309","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar","__is_continuous_processing":"false","spark.app.name":"StructuredKafkaWordCount","callSite.long":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","callSite.short":"start at StructuredKafkaWordCount.scala:86","spark.submit.pyFiles":"","spark.job.description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 5","spark.executor.id":"driver","spark.sql.cbo.enabled":"false","streaming.sql.batchId":"5","spark.jobGroup.id":"e225d92f-2545-48f8-87a2-9c0309580f8a","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"/tmp/spark-history","spark.sql.execution.id":"16","spark.app.id":"local-1596020211915","spark.sql.shuffle.partitions":"2"}}
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":10,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":1,"RDD Info":[{"RDD ID":66,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"173\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[65],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":62,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"181\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[61],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":64,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"179\",\"name\":\"MapPartitions\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[63],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":61,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"185\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[60],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":65,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"174\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[64],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":60,"Name":"DataSourceRDD","Scope":"{\"id\":\"185\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":63,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"180\",\"name\":\"DeserializeToObject\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[62],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020225343,"Accumulables":[],"Resource Profile Id":0},"Properties":{"sql.streaming.queryId":"8d268dc2-bc9c-4be8-97a9-b135d2943028","spark.driver.host":"iZbp19vpr16ix621sdw476Z","spark.eventLog.enabled":"true","spark.sql.adaptive.enabled":"false","spark.job.interruptOnCancel":"true","spark.driver.port":"46309","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar","__is_continuous_processing":"false","spark.app.name":"StructuredKafkaWordCount","callSite.long":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","callSite.short":"start at StructuredKafkaWordCount.scala:86","spark.submit.pyFiles":"","spark.job.description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 5","spark.executor.id":"driver","spark.sql.cbo.enabled":"false","streaming.sql.batchId":"5","spark.jobGroup.id":"e225d92f-2545-48f8-87a2-9c0309580f8a","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"/tmp/spark-history","spark.sql.execution.id":"16","spark.app.id":"local-1596020211915","spark.sql.shuffle.partitions":"2"}}
+{"Event":"SparkListenerTaskStart","Stage ID":10,"Stage Attempt ID":0,"Task Info":{"Task ID":15,"Index":0,"Attempt":0,"Launch Time":1596020225359,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":10,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":15,"Index":0,"Attempt":0,"Launch Time":1596020225359,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1596020225400,"Failed":false,"Killed":false,"Accumulables":[{"ID":806,"Name":"shuffle write time","Update":"530930","Value":"530930","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":805,"Name":"shuffle records written","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":804,"Name":"shuffle bytes written","Update":"168","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":796,"Name":"data size","Update":"128","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":853,"Name":"duration","Update":"21","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":854,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":855,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":857,"Name":"time in aggregation build","Update":"14","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":859,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":860,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":862,"Name":"time in aggregation build","Update":"9","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":864,"Name":"duration","Update":"21","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":865,"Name":"number of output rows","Update":"38","Value":"38","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":910,"Name":"internal.metrics.input.recordsRead","Update":38,"Value":38,"Internal":true,"Count Failed Values":true},{"ID":908,"Name":"internal.metrics.shuffle.write.writeTime","Update":530930,"Value":530930,"Internal":true,"Count Failed Values":true},{"ID":907,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":906,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":168,"Value":168,"Internal":true,"Count Failed Values":true},{"ID":897,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":524288,"Internal":true,"Count Failed Values":true},{"ID":892,"Name":"internal.metrics.resultSize","Update":2544,"Value":2544,"Internal":true,"Count Failed Values":true},{"ID":891,"Name":"internal.metrics.executorCpuTime","Update":22440089,"Value":22440089,"Internal":true,"Count Failed Values":true},{"ID":890,"Name":"internal.metrics.executorRunTime","Update":29,"Value":29,"Internal":true,"Count Failed Values":true},{"ID":889,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6808170,"Value":6808170,"Internal":true,"Count Failed Values":true},{"ID":888,"Name":"internal.metrics.executorDeserializeTime","Update":6,"Value":6,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":6,"Executor Deserialize CPU Time":6808170,"Executor Run Time":29,"Executor CPU Time":22440089,"Peak Execution Memory":524288,"Result Size":2544,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":168,"Shuffle Write Time":530930,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":38},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":10,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":1,"RDD Info":[{"RDD ID":66,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"173\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[65],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":62,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"181\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[61],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":64,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"179\",\"name\":\"MapPartitions\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[63],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":61,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"185\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[60],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":65,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"174\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[64],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":60,"Name":"DataSourceRDD","Scope":"{\"id\":\"185\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":63,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"180\",\"name\":\"DeserializeToObject\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[62],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020225343,"Completion Time":1596020225401,"Accumulables":[{"ID":855,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":891,"Name":"internal.metrics.executorCpuTime","Value":22440089,"Internal":true,"Count Failed Values":true},{"ID":864,"Name":"duration","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":804,"Name":"shuffle bytes written","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":908,"Name":"internal.metrics.shuffle.write.writeTime","Value":530930,"Internal":true,"Count Failed Values":true},{"ID":890,"Name":"internal.metrics.executorRunTime","Value":29,"Internal":true,"Count Failed Values":true},{"ID":857,"Name":"time in aggregation build","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":860,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":806,"Name":"shuffle write time","Value":"530930","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":854,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":853,"Name":"duration","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":862,"Name":"time in aggregation build","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":889,"Name":"internal.metrics.executorDeserializeCpuTime","Value":6808170,"Internal":true,"Count Failed Values":true},{"ID":907,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":1,"Internal":true,"Count Failed Values":true},{"ID":892,"Name":"internal.metrics.resultSize","Value":2544,"Internal":true,"Count Failed Values":true},{"ID":910,"Name":"internal.metrics.input.recordsRead","Value":38,"Internal":true,"Count Failed Values":true},{"ID":865,"Name":"number of output rows","Value":"38","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":805,"Name":"shuffle records written","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":796,"Name":"data size","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":859,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":888,"Name":"internal.metrics.executorDeserializeTime","Value":6,"Internal":true,"Count Failed Values":true},{"ID":897,"Name":"internal.metrics.peakExecutionMemory","Value":524288,"Internal":true,"Count Failed Values":true},{"ID":906,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":168,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}}
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":11,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":2,"RDD Info":[{"RDD ID":71,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"165\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[70],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":70,"Name":"StateStoreRDD","Scope":"{\"id\":\"168\",\"name\":\"StateStoreSave\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[69],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":69,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"169\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[68],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":67,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"173\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[66],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":68,"Name":"StateStoreRDD","Scope":"{\"id\":\"172\",\"name\":\"StateStoreRestore\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[67],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[10],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020225410,"Accumulables":[],"Resource Profile Id":0},"Properties":{"sql.streaming.queryId":"8d268dc2-bc9c-4be8-97a9-b135d2943028","spark.driver.host":"iZbp19vpr16ix621sdw476Z","spark.eventLog.enabled":"true","spark.sql.adaptive.enabled":"false","spark.job.interruptOnCancel":"true","spark.driver.port":"46309","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar","__is_continuous_processing":"false","spark.app.name":"StructuredKafkaWordCount","callSite.long":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","callSite.short":"start at StructuredKafkaWordCount.scala:86","spark.submit.pyFiles":"","spark.job.description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 5","spark.executor.id":"driver","spark.sql.cbo.enabled":"false","streaming.sql.batchId":"5","spark.jobGroup.id":"e225d92f-2545-48f8-87a2-9c0309580f8a","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"/tmp/spark-history","spark.sql.execution.id":"16","spark.app.id":"local-1596020211915","spark.sql.shuffle.partitions":"2"}}
+{"Event":"SparkListenerTaskStart","Stage ID":11,"Stage Attempt ID":0,"Task Info":{"Task ID":16,"Index":0,"Attempt":0,"Launch Time":1596020225417,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":11,"Stage Attempt ID":0,"Task Info":{"Task ID":17,"Index":1,"Attempt":0,"Launch Time":1596020225417,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":11,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":17,"Index":1,"Attempt":0,"Launch Time":1596020225417,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1596020225498,"Failed":false,"Killed":false,"Accumulables":[{"ID":829,"Name":"duration","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":834,"Name":"avg hash probe bucket list iters","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":830,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":831,"Name":"peak memory","Update":"4456448","Value":"4456448","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":833,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":839,"Name":"time to update","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":838,"Name":"number of updated state rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":840,"Name":"time to remove","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":841,"Name":"time to commit changes","Update":"37","Value":"37","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":843,"Name":"estimated size of state only on current version","Update":"368","Value":"368","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":835,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":844,"Name":"count of cache hit on states cache in provider","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":842,"Name":"memory used by state","Update":"784","Value":"784","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":837,"Name":"number of total state rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":846,"Name":"duration","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":847,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":848,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":850,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":852,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":798,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":802,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":801,"Name":"local bytes read","Update":"168","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":803,"Name":"records read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":930,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":929,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":928,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":168,"Value":168,"Internal":true,"Count Failed Values":true},{"ID":927,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":926,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":925,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":924,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":922,"Name":"internal.metrics.peakExecutionMemory","Update":4718592,"Value":4718592,"Internal":true,"Count Failed Values":true},{"ID":917,"Name":"internal.metrics.resultSize","Update":5574,"Value":5574,"Internal":true,"Count Failed Values":true},{"ID":916,"Name":"internal.metrics.executorCpuTime","Update":17945299,"Value":17945299,"Internal":true,"Count Failed Values":true},{"ID":915,"Name":"internal.metrics.executorRunTime","Update":68,"Value":68,"Internal":true,"Count Failed Values":true},{"ID":914,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3451032,"Value":3451032,"Internal":true,"Count Failed Values":true},{"ID":913,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":3,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3451032,"Executor Run Time":68,"Executor CPU Time":17945299,"Peak Execution Memory":4718592,"Result Size":5574,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":168,"Total Records Read":1},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":11,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":16,"Index":0,"Attempt":0,"Launch Time":1596020225417,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1596020225509,"Failed":false,"Killed":false,"Accumulables":[{"ID":829,"Name":"duration","Update":"2","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":831,"Name":"peak memory","Update":"262144","Value":"4718592","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":833,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":839,"Name":"time to update","Update":"4","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":840,"Name":"time to remove","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":841,"Name":"time to commit changes","Update":"50","Value":"87","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":843,"Name":"estimated size of state only on current version","Update":"88","Value":"456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":844,"Name":"count of cache hit on states cache in provider","Update":"10","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":842,"Name":"memory used by state","Update":"400","Value":"1184","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":846,"Name":"duration","Update":"4","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":848,"Name":"peak memory","Update":"262144","Value":"524288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":850,"Name":"time in aggregation build","Update":"0","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":930,"Name":"internal.metrics.shuffle.read.recordsRead","Update":0,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":929,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":928,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":168,"Internal":true,"Count Failed Values":true},{"ID":927,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":926,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":925,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":924,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":922,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":5242880,"Internal":true,"Count Failed Values":true},{"ID":917,"Name":"internal.metrics.resultSize","Update":5311,"Value":10885,"Internal":true,"Count Failed Values":true},{"ID":916,"Name":"internal.metrics.executorCpuTime","Update":15599091,"Value":33544390,"Internal":true,"Count Failed Values":true},{"ID":915,"Name":"internal.metrics.executorRunTime","Update":84,"Value":152,"Internal":true,"Count Failed Values":true},{"ID":914,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4357806,"Value":7808838,"Internal":true,"Count Failed Values":true},{"ID":913,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":7,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4357806,"Executor Run Time":84,"Executor CPU Time":15599091,"Peak Execution Memory":524288,"Result Size":5311,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":11,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":2,"RDD Info":[{"RDD ID":71,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"165\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[70],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":70,"Name":"StateStoreRDD","Scope":"{\"id\":\"168\",\"name\":\"StateStoreSave\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[69],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":69,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"169\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[68],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":67,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"173\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[66],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":68,"Name":"StateStoreRDD","Scope":"{\"id\":\"172\",\"name\":\"StateStoreRestore\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[67],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[10],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020225410,"Completion Time":1596020225514,"Accumulables":[{"ID":846,"Name":"duration","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":837,"Name":"number of total state rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":801,"Name":"local bytes read","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":831,"Name":"peak memory","Value":"4718592","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":926,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":917,"Name":"internal.metrics.resultSize","Value":10885,"Internal":true,"Count Failed Values":true},{"ID":830,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":848,"Name":"peak memory","Value":"524288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":803,"Name":"records read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":839,"Name":"time to update","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":929,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":833,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":842,"Name":"memory used by state","Value":"1184","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":914,"Name":"internal.metrics.executorDeserializeCpuTime","Value":7808838,"Internal":true,"Count Failed Values":true},{"ID":922,"Name":"internal.metrics.peakExecutionMemory","Value":5242880,"Internal":true,"Count Failed Values":true},{"ID":913,"Name":"internal.metrics.executorDeserializeTime","Value":7,"Internal":true,"Count Failed Values":true},{"ID":925,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":844,"Name":"count of cache hit on states cache in provider","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":835,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":916,"Name":"internal.metrics.executorCpuTime","Value":33544390,"Internal":true,"Count Failed Values":true},{"ID":829,"Name":"duration","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":928,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":168,"Internal":true,"Count Failed Values":true},{"ID":802,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":838,"Name":"number of updated state rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":847,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":850,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":841,"Name":"time to commit changes","Value":"87","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":927,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":930,"Name":"internal.metrics.shuffle.read.recordsRead","Value":1,"Internal":true,"Count Failed Values":true},{"ID":840,"Name":"time to remove","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":834,"Name":"avg hash probe bucket list iters","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":852,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":798,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":843,"Name":"estimated size of state only on current version","Value":"456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":915,"Name":"internal.metrics.executorRunTime","Value":152,"Internal":true,"Count Failed Values":true},{"ID":924,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}}
+{"Event":"SparkListenerJobEnd","Job ID":5,"Completion Time":1596020225514,"Job Result":{"Result":"JobSucceeded"}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":17,"description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 5","details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nLocalTableScan (1)\n\n\n(1) LocalTableScan\nOutput [2]: [value#116, count#117]\nArguments: [value#116, count#117]\n\n","sparkPlanInfo":{"nodeName":"LocalTableScan","simpleString":"LocalTableScan [value#116, count#117]","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":938,"metricType":"sum"}]},"time":1596020225536}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":17,"time":1596020225541}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":16,"time":1596020225542}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":15,"time":1596020225542}
+{"Event":"org.apache.spark.sql.streaming.StreamingQueryListener$QueryProgressEvent","progress":{"id":"8d268dc2-bc9c-4be8-97a9-b135d2943028","runId":"e225d92f-2545-48f8-87a2-9c0309580f8a","name":null,"timestamp":"2020-07-29T10:57:05.123Z","batchId":5,"batchDuration":437,"durationMs":{"triggerExecution":437,"queryPlanning":35,"getBatch":1,"latestOffset":3,"addBatch":361,"walCommit":18},"eventTime":{},"stateOperators":[{"numRowsTotal":1,"numRowsUpdated":1,"memoryUsedBytes":1184,"numLateInputs":0,"customMetrics":{"stateOnCurrentVersionSizeBytes":456,"loadedMapCacheHitCount":20,"loadedMapCacheMissCount":0}}],"sources":[{"description":"KafkaV2[Subscribe[test5]]","startOffset":"{\"test5\":{\"0\":48799}}","endOffset":"{\"test5\":{\"0\":48837}}","numInputRows":38,"inputRowsPerSecond":97.68637532133675,"processedRowsPerSecond":86.95652173913044}],"sink":{"description":"org.apache.spark.sql.execution.streaming.ConsoleTable$@514ba885","numOutputRows":1},"observedMetrics":{}}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":18,"description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 6","details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nWriteToDataSourceV2 (14)\n+- * HashAggregate (13)\n   +- StateStoreSave (12)\n      +- * HashAggregate (11)\n         +- StateStoreRestore (10)\n            +- Exchange (9)\n               +- * HashAggregate (8)\n                  +- * HashAggregate (7)\n                     +- * SerializeFromObject (6)\n                        +- MapPartitions (5)\n                           +- DeserializeToObject (4)\n                              +- * Project (3)\n                                 +- * Project (2)\n                                    +- MicroBatchScan (1)\n\n\n(1) MicroBatchScan\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nArguments: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@7e7b182c, KafkaV2[Subscribe[test5]], {\"test5\":{\"0\":48837}}, {\"test5\":{\"0\":48881}}\n\n(2) Project [codegen id : 1]\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(3) Project [codegen id : 1]\nOutput [1]: [cast(value#8 as string) AS value#21]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(4) DeserializeToObject\nInput [1]: [value#21]\nArguments: value#21.toString, obj#27: java.lang.String\n\n(5) MapPartitions\nInput [1]: [obj#27]\nArguments: org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String\n\n(6) SerializeFromObject [codegen id : 2]\nInput [1]: [obj#28]\nArguments: [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]\n\n(7) HashAggregate [codegen id : 2]\nInput [1]: [value#29]\nKeys [1]: [value#29]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(8) HashAggregate [codegen id : 2]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(9) Exchange\nInput [2]: [value#29, count#38L]\nArguments: hashpartitioning(value#29, 2), true, [id=#1528]\n\n(10) StateStoreRestore\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = 6a12c2d9-8d02-4241-93fc-f53da01bb454, opId = 0, ver = 0, numPartitions = 2], 2\n\n(11) HashAggregate [codegen id : 3]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(12) StateStoreSave\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = 6a12c2d9-8d02-4241-93fc-f53da01bb454, opId = 0, ver = 0, numPartitions = 2], Append, 0, 2\n\n(13) HashAggregate [codegen id : 4]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count(1)#31L AS count#32L]\n\n(14) WriteToDataSourceV2\nInput [2]: [value#29, count#32L]\nArguments: org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@27ec018d\n\n","sparkPlanInfo":{"nodeName":"WriteToDataSourceV2","simpleString":"WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@27ec018d","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreSave","simpleString":"StateStoreSave [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 6, numPartitions = 2], Complete, 0, 2","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreRestore","simpleString":"StateStoreRestore [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 6, numPartitions = 2], 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(value#29, 2), true, [id=#1452]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[partial_count(1)])","children":[{"nodeName":"SerializeFromObject","simpleString":"SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MapPartitions","simpleString":"MapPartitions org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject value#21.toString, obj#27: java.lang.String","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [cast(value#8 as string) AS value#21]","children":[{"nodeName":"Project","simpleString":"Project [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MicroBatchScan","simpleString":"MicroBatchScan[key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13] class org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1022,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1021,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1018,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1019,"metricType":"timing"},{"name":"peak memory","accumulatorId":1017,"metricType":"size"},{"name":"number of output rows","accumulatorId":1016,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":1020,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1013,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1014,"metricType":"timing"},{"name":"peak memory","accumulatorId":1012,"metricType":"size"},{"name":"number of output rows","accumulatorId":1011,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":1015,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1010,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":963,"metricType":"nsTiming"},{"name":"records read","accumulatorId":960,"metricType":"sum"},{"name":"local bytes read","accumulatorId":958,"metricType":"size"},{"name":"fetch wait time","accumulatorId":959,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":956,"metricType":"size"},{"name":"local blocks read","accumulatorId":955,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":954,"metricType":"sum"},{"name":"data size","accumulatorId":953,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":957,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":961,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1009,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1006,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1007,"metricType":"timing"},{"name":"peak memory","accumulatorId":1005,"metricType":"size"},{"name":"number of output rows","accumulatorId":1004,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":1008,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1003,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of inputs which are later than watermark ('inputs' are relative to operators)","accumulatorId":993,"metricType":"sum"},{"name":"number of total state rows","accumulatorId":994,"metricType":"sum"},{"name":"memory used by state","accumulatorId":999,"metricType":"size"},{"name":"count of cache hit on states cache in provider","accumulatorId":1001,"metricType":"sum"},{"name":"number of output rows","accumulatorId":992,"metricType":"sum"},{"name":"estimated size of state only on current version","accumulatorId":1000,"metricType":"size"},{"name":"count of cache miss on states cache in provider","accumulatorId":1002,"metricType":"sum"},{"name":"time to commit changes","accumulatorId":998,"metricType":"timing"},{"name":"time to remove","accumulatorId":997,"metricType":"timing"},{"name":"number of updated state rows","accumulatorId":995,"metricType":"sum"},{"name":"time to update","accumulatorId":996,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":989,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":990,"metricType":"timing"},{"name":"peak memory","accumulatorId":988,"metricType":"size"},{"name":"number of output rows","accumulatorId":987,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":991,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":986,"metricType":"timing"}]}],"metadata":{},"metrics":[]},"time":1596020225657}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":19,"description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 6","details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nWriteToDataSourceV2 (14)\n+- * HashAggregate (13)\n   +- StateStoreSave (12)\n      +- * HashAggregate (11)\n         +- StateStoreRestore (10)\n            +- Exchange (9)\n               +- * HashAggregate (8)\n                  +- * HashAggregate (7)\n                     +- * SerializeFromObject (6)\n                        +- MapPartitions (5)\n                           +- DeserializeToObject (4)\n                              +- * Project (3)\n                                 +- * Project (2)\n                                    +- MicroBatchScan (1)\n\n\n(1) MicroBatchScan\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nArguments: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@7e7b182c, KafkaV2[Subscribe[test5]], {\"test5\":{\"0\":48837}}, {\"test5\":{\"0\":48881}}\n\n(2) Project [codegen id : 1]\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(3) Project [codegen id : 1]\nOutput [1]: [cast(value#8 as string) AS value#21]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(4) DeserializeToObject\nInput [1]: [value#21]\nArguments: value#21.toString, obj#27: java.lang.String\n\n(5) MapPartitions\nInput [1]: [obj#27]\nArguments: org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String\n\n(6) SerializeFromObject [codegen id : 2]\nInput [1]: [obj#28]\nArguments: [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]\n\n(7) HashAggregate [codegen id : 2]\nInput [1]: [value#29]\nKeys [1]: [value#29]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(8) HashAggregate [codegen id : 2]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(9) Exchange\nInput [2]: [value#29, count#38L]\nArguments: hashpartitioning(value#29, 2), true, [id=#1604]\n\n(10) StateStoreRestore\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = 96456757-8d0b-46da-a006-9fe2cb6fc936, opId = 0, ver = 0, numPartitions = 2], 2\n\n(11) HashAggregate [codegen id : 3]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(12) StateStoreSave\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = 96456757-8d0b-46da-a006-9fe2cb6fc936, opId = 0, ver = 0, numPartitions = 2], Append, 0, 2\n\n(13) HashAggregate [codegen id : 4]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count(1)#31L AS count#32L]\n\n(14) WriteToDataSourceV2\nInput [2]: [value#29, count#32L]\nArguments: org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@27ec018d\n\n","sparkPlanInfo":{"nodeName":"WriteToDataSourceV2","simpleString":"WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@27ec018d","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreSave","simpleString":"StateStoreSave [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 6, numPartitions = 2], Complete, 0, 2","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreRestore","simpleString":"StateStoreRestore [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 6, numPartitions = 2], 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(value#29, 2), true, [id=#1452]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[partial_count(1)])","children":[{"nodeName":"SerializeFromObject","simpleString":"SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MapPartitions","simpleString":"MapPartitions org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject value#21.toString, obj#27: java.lang.String","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [cast(value#8 as string) AS value#21]","children":[{"nodeName":"Project","simpleString":"Project [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MicroBatchScan","simpleString":"MicroBatchScan[key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13] class org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1022,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1021,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1018,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1019,"metricType":"timing"},{"name":"peak memory","accumulatorId":1017,"metricType":"size"},{"name":"number of output rows","accumulatorId":1016,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":1020,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1013,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1014,"metricType":"timing"},{"name":"peak memory","accumulatorId":1012,"metricType":"size"},{"name":"number of output rows","accumulatorId":1011,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":1015,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1010,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":963,"metricType":"nsTiming"},{"name":"records read","accumulatorId":960,"metricType":"sum"},{"name":"local bytes read","accumulatorId":958,"metricType":"size"},{"name":"fetch wait time","accumulatorId":959,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":956,"metricType":"size"},{"name":"local blocks read","accumulatorId":955,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":954,"metricType":"sum"},{"name":"data size","accumulatorId":953,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":957,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":961,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1009,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1006,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1007,"metricType":"timing"},{"name":"peak memory","accumulatorId":1005,"metricType":"size"},{"name":"number of output rows","accumulatorId":1004,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":1008,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1003,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of inputs which are later than watermark ('inputs' are relative to operators)","accumulatorId":993,"metricType":"sum"},{"name":"number of total state rows","accumulatorId":994,"metricType":"sum"},{"name":"memory used by state","accumulatorId":999,"metricType":"size"},{"name":"count of cache hit on states cache in provider","accumulatorId":1001,"metricType":"sum"},{"name":"number of output rows","accumulatorId":992,"metricType":"sum"},{"name":"estimated size of state only on current version","accumulatorId":1000,"metricType":"size"},{"name":"count of cache miss on states cache in provider","accumulatorId":1002,"metricType":"sum"},{"name":"time to commit changes","accumulatorId":998,"metricType":"timing"},{"name":"time to remove","accumulatorId":997,"metricType":"timing"},{"name":"number of updated state rows","accumulatorId":995,"metricType":"sum"},{"name":"time to update","accumulatorId":996,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":989,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":990,"metricType":"timing"},{"name":"peak memory","accumulatorId":988,"metricType":"size"},{"name":"number of output rows","accumulatorId":987,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":991,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":986,"metricType":"timing"}]}],"metadata":{},"metrics":[]},"time":1596020225687}
+{"Event":"SparkListenerJobStart","Job ID":6,"Submission Time":1596020225759,"Stage Infos":[{"Stage ID":12,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":1,"RDD Info":[{"RDD ID":78,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"206\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[77],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":75,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"213\",\"name\":\"DeserializeToObject\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[74],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":74,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"214\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[73],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":77,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"207\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[76],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":72,"Name":"DataSourceRDD","Scope":"{\"id\":\"218\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":73,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"218\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[72],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":76,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"212\",\"name\":\"MapPartitions\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[75],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":13,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":2,"RDD Info":[{"RDD ID":83,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"198\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[82],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":81,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"202\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[80],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":79,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"206\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[78],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":80,"Name":"StateStoreRDD","Scope":"{\"id\":\"205\",\"name\":\"StateStoreRestore\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[79],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":82,"Name":"StateStoreRDD","Scope":"{\"id\":\"201\",\"name\":\"StateStoreSave\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[81],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[12],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[12,13],"Properties":{"sql.streaming.queryId":"8d268dc2-bc9c-4be8-97a9-b135d2943028","spark.driver.host":"iZbp19vpr16ix621sdw476Z","spark.eventLog.enabled":"true","spark.sql.adaptive.enabled":"false","spark.job.interruptOnCancel":"true","spark.driver.port":"46309","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar","__is_continuous_processing":"false","spark.app.name":"StructuredKafkaWordCount","callSite.long":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","callSite.short":"start at StructuredKafkaWordCount.scala:86","spark.submit.pyFiles":"","spark.job.description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 6","spark.executor.id":"driver","spark.sql.cbo.enabled":"false","streaming.sql.batchId":"6","spark.jobGroup.id":"e225d92f-2545-48f8-87a2-9c0309580f8a","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"/tmp/spark-history","spark.sql.execution.id":"19","spark.app.id":"local-1596020211915","spark.sql.shuffle.partitions":"2"}}
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":12,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":1,"RDD Info":[{"RDD ID":78,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"206\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[77],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":75,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"213\",\"name\":\"DeserializeToObject\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[74],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":74,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"214\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[73],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":77,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"207\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[76],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":72,"Name":"DataSourceRDD","Scope":"{\"id\":\"218\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":73,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"218\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[72],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":76,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"212\",\"name\":\"MapPartitions\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[75],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020225760,"Accumulables":[],"Resource Profile Id":0},"Properties":{"sql.streaming.queryId":"8d268dc2-bc9c-4be8-97a9-b135d2943028","spark.driver.host":"iZbp19vpr16ix621sdw476Z","spark.eventLog.enabled":"true","spark.sql.adaptive.enabled":"false","spark.job.interruptOnCancel":"true","spark.driver.port":"46309","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar","__is_continuous_processing":"false","spark.app.name":"StructuredKafkaWordCount","callSite.long":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","callSite.short":"start at StructuredKafkaWordCount.scala:86","spark.submit.pyFiles":"","spark.job.description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 6","spark.executor.id":"driver","spark.sql.cbo.enabled":"false","streaming.sql.batchId":"6","spark.jobGroup.id":"e225d92f-2545-48f8-87a2-9c0309580f8a","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"/tmp/spark-history","spark.sql.execution.id":"19","spark.app.id":"local-1596020211915","spark.sql.shuffle.partitions":"2"}}
+{"Event":"SparkListenerTaskStart","Stage ID":12,"Stage Attempt ID":0,"Task Info":{"Task ID":18,"Index":0,"Attempt":0,"Launch Time":1596020225766,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":12,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":18,"Index":0,"Attempt":0,"Launch Time":1596020225766,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1596020225796,"Failed":false,"Killed":false,"Accumulables":[{"ID":963,"Name":"shuffle write time","Update":"543836","Value":"543836","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":962,"Name":"shuffle records written","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":961,"Name":"shuffle bytes written","Update":"168","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":953,"Name":"data size","Update":"128","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1010,"Name":"duration","Update":"17","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1011,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1012,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1014,"Name":"time in aggregation build","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1016,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1017,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1019,"Name":"time in aggregation build","Update":"8","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1021,"Name":"duration","Update":"17","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1022,"Name":"number of output rows","Update":"44","Value":"44","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1067,"Name":"internal.metrics.input.recordsRead","Update":44,"Value":44,"Internal":true,"Count Failed Values":true},{"ID":1065,"Name":"internal.metrics.shuffle.write.writeTime","Update":543836,"Value":543836,"Internal":true,"Count Failed Values":true},{"ID":1064,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1063,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":168,"Value":168,"Internal":true,"Count Failed Values":true},{"ID":1054,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":524288,"Internal":true,"Count Failed Values":true},{"ID":1049,"Name":"internal.metrics.resultSize","Update":2544,"Value":2544,"Internal":true,"Count Failed Values":true},{"ID":1048,"Name":"internal.metrics.executorCpuTime","Update":23733439,"Value":23733439,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.executorRunTime","Update":23,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3714406,"Value":3714406,"Internal":true,"Count Failed Values":true},{"ID":1045,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":3,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3714406,"Executor Run Time":23,"Executor CPU Time":23733439,"Peak Execution Memory":524288,"Result Size":2544,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":168,"Shuffle Write Time":543836,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":44},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":12,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":1,"RDD Info":[{"RDD ID":78,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"206\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[77],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":75,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"213\",\"name\":\"DeserializeToObject\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[74],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":74,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"214\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[73],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":77,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"207\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[76],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":72,"Name":"DataSourceRDD","Scope":"{\"id\":\"218\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":73,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"218\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[72],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":76,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"212\",\"name\":\"MapPartitions\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[75],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020225760,"Completion Time":1596020225797,"Accumulables":[{"ID":1064,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1010,"Name":"duration","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1046,"Name":"internal.metrics.executorDeserializeCpuTime","Value":3714406,"Internal":true,"Count Failed Values":true},{"ID":1019,"Name":"time in aggregation build","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1067,"Name":"internal.metrics.input.recordsRead","Value":44,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"number of output rows","Value":"44","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1049,"Name":"internal.metrics.resultSize","Value":2544,"Internal":true,"Count Failed Values":true},{"ID":1016,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":962,"Name":"shuffle records written","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":953,"Name":"data size","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1054,"Name":"internal.metrics.peakExecutionMemory","Value":524288,"Internal":true,"Count Failed Values":true},{"ID":1045,"Name":"internal.metrics.executorDeserializeTime","Value":3,"Internal":true,"Count Failed Values":true},{"ID":1063,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":168,"Internal":true,"Count Failed Values":true},{"ID":1048,"Name":"internal.metrics.executorCpuTime","Value":23733439,"Internal":true,"Count Failed Values":true},{"ID":1012,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1021,"Name":"duration","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":961,"Name":"shuffle bytes written","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1065,"Name":"internal.metrics.shuffle.write.writeTime","Value":543836,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.executorRunTime","Value":23,"Internal":true,"Count Failed Values":true},{"ID":1014,"Name":"time in aggregation build","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":963,"Name":"shuffle write time","Value":"543836","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1017,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1011,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"}],"Resource Profile Id":0}}
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":13,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":2,"RDD Info":[{"RDD ID":83,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"198\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[82],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":81,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"202\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[80],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":79,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"206\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[78],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":80,"Name":"StateStoreRDD","Scope":"{\"id\":\"205\",\"name\":\"StateStoreRestore\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[79],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":82,"Name":"StateStoreRDD","Scope":"{\"id\":\"201\",\"name\":\"StateStoreSave\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[81],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[12],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020225801,"Accumulables":[],"Resource Profile Id":0},"Properties":{"sql.streaming.queryId":"8d268dc2-bc9c-4be8-97a9-b135d2943028","spark.driver.host":"iZbp19vpr16ix621sdw476Z","spark.eventLog.enabled":"true","spark.sql.adaptive.enabled":"false","spark.job.interruptOnCancel":"true","spark.driver.port":"46309","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar","__is_continuous_processing":"false","spark.app.name":"StructuredKafkaWordCount","callSite.long":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","callSite.short":"start at StructuredKafkaWordCount.scala:86","spark.submit.pyFiles":"","spark.job.description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 6","spark.executor.id":"driver","spark.sql.cbo.enabled":"false","streaming.sql.batchId":"6","spark.jobGroup.id":"e225d92f-2545-48f8-87a2-9c0309580f8a","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"/tmp/spark-history","spark.sql.execution.id":"19","spark.app.id":"local-1596020211915","spark.sql.shuffle.partitions":"2"}}
+{"Event":"SparkListenerTaskStart","Stage ID":13,"Stage Attempt ID":0,"Task Info":{"Task ID":19,"Index":0,"Attempt":0,"Launch Time":1596020225808,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":13,"Stage Attempt ID":0,"Task Info":{"Task ID":20,"Index":1,"Attempt":0,"Launch Time":1596020225809,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":13,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":19,"Index":0,"Attempt":0,"Launch Time":1596020225808,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1596020225868,"Failed":false,"Killed":false,"Accumulables":[{"ID":986,"Name":"duration","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":988,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":990,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":996,"Name":"time to update","Update":"4","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":997,"Name":"time to remove","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":998,"Name":"time to commit changes","Update":"26","Value":"26","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1000,"Name":"estimated size of state only on current version","Update":"88","Value":"88","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1001,"Name":"count of cache hit on states cache in provider","Update":"12","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":999,"Name":"memory used by state","Update":"400","Value":"400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1003,"Name":"duration","Update":"4","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1005,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1007,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1087,"Name":"internal.metrics.shuffle.read.recordsRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1086,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1085,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1084,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1083,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1082,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1081,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1079,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":524288,"Internal":true,"Count Failed Values":true},{"ID":1074,"Name":"internal.metrics.resultSize","Update":5311,"Value":5311,"Internal":true,"Count Failed Values":true},{"ID":1073,"Name":"internal.metrics.executorCpuTime","Update":17503528,"Value":17503528,"Internal":true,"Count Failed Values":true},{"ID":1072,"Name":"internal.metrics.executorRunTime","Update":50,"Value":50,"Internal":true,"Count Failed Values":true},{"ID":1071,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4255703,"Value":4255703,"Internal":true,"Count Failed Values":true},{"ID":1070,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":4,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4255703,"Executor Run Time":50,"Executor CPU Time":17503528,"Peak Execution Memory":524288,"Result Size":5311,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":13,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":20,"Index":1,"Attempt":0,"Launch Time":1596020225809,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1596020225874,"Failed":false,"Killed":false,"Accumulables":[{"ID":986,"Name":"duration","Update":"2","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":991,"Name":"avg hash probe bucket list iters","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":987,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":988,"Name":"peak memory","Update":"4456448","Value":"4718592","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":990,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":996,"Name":"time to update","Update":"15","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":995,"Name":"number of updated state rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":997,"Name":"time to remove","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":998,"Name":"time to commit changes","Update":"23","Value":"49","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1000,"Name":"estimated size of state only on current version","Update":"368","Value":"456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":992,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1001,"Name":"count of cache hit on states cache in provider","Update":"12","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":999,"Name":"memory used by state","Update":"784","Value":"1184","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":994,"Name":"number of total state rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1003,"Name":"duration","Update":"15","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1004,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1005,"Name":"peak memory","Update":"262144","Value":"524288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1007,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1009,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":955,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":959,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":958,"Name":"local bytes read","Update":"168","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":960,"Name":"records read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1087,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1086,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1085,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":168,"Value":168,"Internal":true,"Count Failed Values":true},{"ID":1084,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1083,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1082,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1081,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1079,"Name":"internal.metrics.peakExecutionMemory","Update":4718592,"Value":5242880,"Internal":true,"Count Failed Values":true},{"ID":1074,"Name":"internal.metrics.resultSize","Update":5574,"Value":10885,"Internal":true,"Count Failed Values":true},{"ID":1073,"Name":"internal.metrics.executorCpuTime","Update":17516707,"Value":35020235,"Internal":true,"Count Failed Values":true},{"ID":1072,"Name":"internal.metrics.executorRunTime","Update":56,"Value":106,"Internal":true,"Count Failed Values":true},{"ID":1071,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3750230,"Value":8005933,"Internal":true,"Count Failed Values":true},{"ID":1070,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":7,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3750230,"Executor Run Time":56,"Executor CPU Time":17516707,"Peak Execution Memory":4718592,"Result Size":5574,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":168,"Total Records Read":1},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":13,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":2,"RDD Info":[{"RDD ID":83,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"198\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[82],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":81,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"202\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[80],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":79,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"206\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[78],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":80,"Name":"StateStoreRDD","Scope":"{\"id\":\"205\",\"name\":\"StateStoreRestore\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[79],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":82,"Name":"StateStoreRDD","Scope":"{\"id\":\"201\",\"name\":\"StateStoreSave\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[81],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[12],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020225801,"Completion Time":1596020225874,"Accumulables":[{"ID":1070,"Name":"internal.metrics.executorDeserializeTime","Value":7,"Internal":true,"Count Failed Values":true},{"ID":1079,"Name":"internal.metrics.peakExecutionMemory","Value":5242880,"Internal":true,"Count Failed Values":true},{"ID":992,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1073,"Name":"internal.metrics.executorCpuTime","Value":35020235,"Internal":true,"Count Failed Values":true},{"ID":1082,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1001,"Name":"count of cache hit on states cache in provider","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":995,"Name":"number of updated state rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1004,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":986,"Name":"duration","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":959,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1085,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":168,"Internal":true,"Count Failed Values":true},{"ID":1007,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":998,"Name":"time to commit changes","Value":"49","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1084,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":997,"Name":"time to remove","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1087,"Name":"internal.metrics.shuffle.read.recordsRead","Value":1,"Internal":true,"Count Failed Values":true},{"ID":955,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1081,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":991,"Name":"avg hash probe bucket list iters","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1009,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1072,"Name":"internal.metrics.executorRunTime","Value":106,"Internal":true,"Count Failed Values":true},{"ID":1000,"Name":"estimated size of state only on current version","Value":"456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":994,"Name":"number of total state rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1003,"Name":"duration","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":958,"Name":"local bytes read","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":988,"Name":"peak memory","Value":"4718592","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1074,"Name":"internal.metrics.resultSize","Value":10885,"Internal":true,"Count Failed Values":true},{"ID":1083,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":960,"Name":"records read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1086,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":987,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1005,"Name":"peak memory","Value":"524288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":996,"Name":"time to update","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1071,"Name":"internal.metrics.executorDeserializeCpuTime","Value":8005933,"Internal":true,"Count Failed Values":true},{"ID":999,"Name":"memory used by state","Value":"1184","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":990,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"}],"Resource Profile Id":0}}
+{"Event":"SparkListenerJobEnd","Job ID":6,"Completion Time":1596020225875,"Job Result":{"Result":"JobSucceeded"}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":20,"description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 6","details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nLocalTableScan (1)\n\n\n(1) LocalTableScan\nOutput [2]: [value#130, count#131]\nArguments: [value#130, count#131]\n\n","sparkPlanInfo":{"nodeName":"LocalTableScan","simpleString":"LocalTableScan [value#130, count#131]","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1095,"metricType":"sum"}]},"time":1596020225891}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":20,"time":1596020225896}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":19,"time":1596020225897}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":18,"time":1596020225897}
+{"Event":"org.apache.spark.sql.streaming.StreamingQueryListener$QueryProgressEvent","progress":{"id":"8d268dc2-bc9c-4be8-97a9-b135d2943028","runId":"e225d92f-2545-48f8-87a2-9c0309580f8a","name":null,"timestamp":"2020-07-29T10:57:05.562Z","batchId":6,"batchDuration":351,"durationMs":{"triggerExecution":351,"queryPlanning":28,"getBatch":1,"latestOffset":6,"addBatch":273,"walCommit":25},"eventTime":{},"stateOperators":[{"numRowsTotal":1,"numRowsUpdated":1,"memoryUsedBytes":1184,"numLateInputs":0,"customMetrics":{"stateOnCurrentVersionSizeBytes":456,"loadedMapCacheHitCount":24,"loadedMapCacheMissCount":0}}],"sources":[{"description":"KafkaV2[Subscribe[test5]]","startOffset":"{\"test5\":{\"0\":48837}}","endOffset":"{\"test5\":{\"0\":48881}}","numInputRows":44,"inputRowsPerSecond":100.22779043280183,"processedRowsPerSecond":125.35612535612536}],"sink":{"description":"org.apache.spark.sql.execution.streaming.ConsoleTable$@514ba885","numOutputRows":1},"observedMetrics":{}}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":21,"description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 7","details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nWriteToDataSourceV2 (14)\n+- * HashAggregate (13)\n   +- StateStoreSave (12)\n      +- * HashAggregate (11)\n         +- StateStoreRestore (10)\n            +- Exchange (9)\n               +- * HashAggregate (8)\n                  +- * HashAggregate (7)\n                     +- * SerializeFromObject (6)\n                        +- MapPartitions (5)\n                           +- DeserializeToObject (4)\n                              +- * Project (3)\n                                 +- * Project (2)\n                                    +- MicroBatchScan (1)\n\n\n(1) MicroBatchScan\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nArguments: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@7e7b182c, KafkaV2[Subscribe[test5]], {\"test5\":{\"0\":48881}}, {\"test5\":{\"0\":48917}}\n\n(2) Project [codegen id : 1]\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(3) Project [codegen id : 1]\nOutput [1]: [cast(value#8 as string) AS value#21]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(4) DeserializeToObject\nInput [1]: [value#21]\nArguments: value#21.toString, obj#27: java.lang.String\n\n(5) MapPartitions\nInput [1]: [obj#27]\nArguments: org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String\n\n(6) SerializeFromObject [codegen id : 2]\nInput [1]: [obj#28]\nArguments: [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]\n\n(7) HashAggregate [codegen id : 2]\nInput [1]: [value#29]\nKeys [1]: [value#29]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(8) HashAggregate [codegen id : 2]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(9) Exchange\nInput [2]: [value#29, count#38L]\nArguments: hashpartitioning(value#29, 2), true, [id=#1759]\n\n(10) StateStoreRestore\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = c0968891-bf48-4112-a19b-444014085d1d, opId = 0, ver = 0, numPartitions = 2], 2\n\n(11) HashAggregate [codegen id : 3]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(12) StateStoreSave\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = c0968891-bf48-4112-a19b-444014085d1d, opId = 0, ver = 0, numPartitions = 2], Append, 0, 2\n\n(13) HashAggregate [codegen id : 4]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count(1)#31L AS count#32L]\n\n(14) WriteToDataSourceV2\nInput [2]: [value#29, count#32L]\nArguments: org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@6313b68e\n\n","sparkPlanInfo":{"nodeName":"WriteToDataSourceV2","simpleString":"WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@6313b68e","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreSave","simpleString":"StateStoreSave [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 7, numPartitions = 2], Complete, 0, 2","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreRestore","simpleString":"StateStoreRestore [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 7, numPartitions = 2], 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(value#29, 2), true, [id=#1683]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[partial_count(1)])","children":[{"nodeName":"SerializeFromObject","simpleString":"SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MapPartitions","simpleString":"MapPartitions org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject value#21.toString, obj#27: java.lang.String","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [cast(value#8 as string) AS value#21]","children":[{"nodeName":"Project","simpleString":"Project [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MicroBatchScan","simpleString":"MicroBatchScan[key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13] class org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1179,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1178,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1175,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1176,"metricType":"timing"},{"name":"peak memory","accumulatorId":1174,"metricType":"size"},{"name":"number of output rows","accumulatorId":1173,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":1177,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1170,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1171,"metricType":"timing"},{"name":"peak memory","accumulatorId":1169,"metricType":"size"},{"name":"number of output rows","accumulatorId":1168,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":1172,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1167,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1119,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1120,"metricType":"nsTiming"},{"name":"records read","accumulatorId":1117,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1115,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1116,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1113,"metricType":"size"},{"name":"local blocks read","accumulatorId":1112,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1111,"metricType":"sum"},{"name":"data size","accumulatorId":1110,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":1114,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1118,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1166,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1163,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1164,"metricType":"timing"},{"name":"peak memory","accumulatorId":1162,"metricType":"size"},{"name":"number of output rows","accumulatorId":1161,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":1165,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1160,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of inputs which are later than watermark ('inputs' are relative to operators)","accumulatorId":1150,"metricType":"sum"},{"name":"number of total state rows","accumulatorId":1151,"metricType":"sum"},{"name":"memory used by state","accumulatorId":1156,"metricType":"size"},{"name":"count of cache hit on states cache in provider","accumulatorId":1158,"metricType":"sum"},{"name":"number of output rows","accumulatorId":1149,"metricType":"sum"},{"name":"estimated size of state only on current version","accumulatorId":1157,"metricType":"size"},{"name":"count of cache miss on states cache in provider","accumulatorId":1159,"metricType":"sum"},{"name":"time to commit changes","accumulatorId":1155,"metricType":"timing"},{"name":"time to remove","accumulatorId":1154,"metricType":"timing"},{"name":"number of updated state rows","accumulatorId":1152,"metricType":"sum"},{"name":"time to update","accumulatorId":1153,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1146,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1147,"metricType":"timing"},{"name":"peak memory","accumulatorId":1145,"metricType":"size"},{"name":"number of output rows","accumulatorId":1144,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":1148,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1143,"metricType":"timing"}]}],"metadata":{},"metrics":[]},"time":1596020225988}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":22,"description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 7","details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nWriteToDataSourceV2 (14)\n+- * HashAggregate (13)\n   +- StateStoreSave (12)\n      +- * HashAggregate (11)\n         +- StateStoreRestore (10)\n            +- Exchange (9)\n               +- * HashAggregate (8)\n                  +- * HashAggregate (7)\n                     +- * SerializeFromObject (6)\n                        +- MapPartitions (5)\n                           +- DeserializeToObject (4)\n                              +- * Project (3)\n                                 +- * Project (2)\n                                    +- MicroBatchScan (1)\n\n\n(1) MicroBatchScan\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nArguments: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@7e7b182c, KafkaV2[Subscribe[test5]], {\"test5\":{\"0\":48881}}, {\"test5\":{\"0\":48917}}\n\n(2) Project [codegen id : 1]\nOutput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(3) Project [codegen id : 1]\nOutput [1]: [cast(value#8 as string) AS value#21]\nInput [7]: [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]\n\n(4) DeserializeToObject\nInput [1]: [value#21]\nArguments: value#21.toString, obj#27: java.lang.String\n\n(5) MapPartitions\nInput [1]: [obj#27]\nArguments: org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String\n\n(6) SerializeFromObject [codegen id : 2]\nInput [1]: [obj#28]\nArguments: [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]\n\n(7) HashAggregate [codegen id : 2]\nInput [1]: [value#29]\nKeys [1]: [value#29]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(8) HashAggregate [codegen id : 2]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(9) Exchange\nInput [2]: [value#29, count#38L]\nArguments: hashpartitioning(value#29, 2), true, [id=#1835]\n\n(10) StateStoreRestore\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = e165b23b-1a6f-459f-9c51-288922bb2647, opId = 0, ver = 0, numPartitions = 2], 2\n\n(11) HashAggregate [codegen id : 3]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [merge_count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count#38L]\n\n(12) StateStoreSave\nInput [2]: [value#29, count#38L]\nArguments: [value#29], state info [ checkpoint = <unknown>, runId = e165b23b-1a6f-459f-9c51-288922bb2647, opId = 0, ver = 0, numPartitions = 2], Append, 0, 2\n\n(13) HashAggregate [codegen id : 4]\nInput [2]: [value#29, count#38L]\nKeys [1]: [value#29]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#31L]\nResults [2]: [value#29, count(1)#31L AS count#32L]\n\n(14) WriteToDataSourceV2\nInput [2]: [value#29, count#32L]\nArguments: org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@6313b68e\n\n","sparkPlanInfo":{"nodeName":"WriteToDataSourceV2","simpleString":"WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@6313b68e","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreSave","simpleString":"StateStoreSave [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 7, numPartitions = 2], Complete, 0, 2","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"StateStoreRestore","simpleString":"StateStoreRestore [value#29], state info [ checkpoint = file:/tmp/temporary-025d7997-5b66-4def-abbf-bdcca57312b9/state, runId = e225d92f-2545-48f8-87a2-9c0309580f8a, opId = 0, ver = 7, numPartitions = 2], 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(value#29, 2), true, [id=#1683]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[merge_count(1)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#29], functions=[partial_count(1)])","children":[{"nodeName":"SerializeFromObject","simpleString":"SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#29]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MapPartitions","simpleString":"MapPartitions org.apache.spark.sql.Dataset$$Lambda$1321/872917583@67b99068, obj#28: java.lang.String","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject value#21.toString, obj#27: java.lang.String","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [cast(value#8 as string) AS value#21]","children":[{"nodeName":"Project","simpleString":"Project [key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"MicroBatchScan","simpleString":"MicroBatchScan[key#7, value#8, topic#9, partition#10, offset#11L, timestamp#12, timestampType#13] class org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1179,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1178,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1175,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1176,"metricType":"timing"},{"name":"peak memory","accumulatorId":1174,"metricType":"size"},{"name":"number of output rows","accumulatorId":1173,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":1177,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1170,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1171,"metricType":"timing"},{"name":"peak memory","accumulatorId":1169,"metricType":"size"},{"name":"number of output rows","accumulatorId":1168,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":1172,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1167,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1119,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1120,"metricType":"nsTiming"},{"name":"records read","accumulatorId":1117,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1115,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1116,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1113,"metricType":"size"},{"name":"local blocks read","accumulatorId":1112,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1111,"metricType":"sum"},{"name":"data size","accumulatorId":1110,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":1114,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1118,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1166,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1163,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1164,"metricType":"timing"},{"name":"peak memory","accumulatorId":1162,"metricType":"size"},{"name":"number of output rows","accumulatorId":1161,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":1165,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1160,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of inputs which are later than watermark ('inputs' are relative to operators)","accumulatorId":1150,"metricType":"sum"},{"name":"number of total state rows","accumulatorId":1151,"metricType":"sum"},{"name":"memory used by state","accumulatorId":1156,"metricType":"size"},{"name":"count of cache hit on states cache in provider","accumulatorId":1158,"metricType":"sum"},{"name":"number of output rows","accumulatorId":1149,"metricType":"sum"},{"name":"estimated size of state only on current version","accumulatorId":1157,"metricType":"size"},{"name":"count of cache miss on states cache in provider","accumulatorId":1159,"metricType":"sum"},{"name":"time to commit changes","accumulatorId":1155,"metricType":"timing"},{"name":"time to remove","accumulatorId":1154,"metricType":"timing"},{"name":"number of updated state rows","accumulatorId":1152,"metricType":"sum"},{"name":"time to update","accumulatorId":1153,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1146,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1147,"metricType":"timing"},{"name":"peak memory","accumulatorId":1145,"metricType":"size"},{"name":"number of output rows","accumulatorId":1144,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":1148,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1143,"metricType":"timing"}]}],"metadata":{},"metrics":[]},"time":1596020226019}
+{"Event":"SparkListenerJobStart","Job ID":7,"Submission Time":1596020226076,"Stage Infos":[{"Stage ID":15,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":2,"RDD Info":[{"RDD ID":95,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"231\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[94],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":93,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"235\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[92],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":91,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"239\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[90],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":94,"Name":"StateStoreRDD","Scope":"{\"id\":\"234\",\"name\":\"StateStoreSave\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[93],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":92,"Name":"StateStoreRDD","Scope":"{\"id\":\"238\",\"name\":\"StateStoreRestore\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[91],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[14],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":14,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":1,"RDD Info":[{"RDD ID":90,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"239\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[89],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":88,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"245\",\"name\":\"MapPartitions\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[87],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":84,"Name":"DataSourceRDD","Scope":"{\"id\":\"251\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":85,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"251\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[84],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":89,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"240\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[88],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":86,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"247\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[85],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":87,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"246\",\"name\":\"DeserializeToObject\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[86],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[15,14],"Properties":{"sql.streaming.queryId":"8d268dc2-bc9c-4be8-97a9-b135d2943028","spark.driver.host":"iZbp19vpr16ix621sdw476Z","spark.eventLog.enabled":"true","spark.sql.adaptive.enabled":"false","spark.job.interruptOnCancel":"true","spark.driver.port":"46309","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar","__is_continuous_processing":"false","spark.app.name":"StructuredKafkaWordCount","callSite.long":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","callSite.short":"start at StructuredKafkaWordCount.scala:86","spark.submit.pyFiles":"","spark.job.description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 7","spark.executor.id":"driver","spark.sql.cbo.enabled":"false","streaming.sql.batchId":"7","spark.jobGroup.id":"e225d92f-2545-48f8-87a2-9c0309580f8a","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"/tmp/spark-history","spark.sql.execution.id":"22","spark.app.id":"local-1596020211915","spark.sql.shuffle.partitions":"2"}}
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":14,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":1,"RDD Info":[{"RDD ID":90,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"239\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[89],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":88,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"245\",\"name\":\"MapPartitions\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[87],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":84,"Name":"DataSourceRDD","Scope":"{\"id\":\"251\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":85,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"251\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[84],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":89,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"240\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[88],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":86,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"247\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[85],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":87,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"246\",\"name\":\"DeserializeToObject\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[86],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020226077,"Accumulables":[],"Resource Profile Id":0},"Properties":{"sql.streaming.queryId":"8d268dc2-bc9c-4be8-97a9-b135d2943028","spark.driver.host":"iZbp19vpr16ix621sdw476Z","spark.eventLog.enabled":"true","spark.sql.adaptive.enabled":"false","spark.job.interruptOnCancel":"true","spark.driver.port":"46309","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar","__is_continuous_processing":"false","spark.app.name":"StructuredKafkaWordCount","callSite.long":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","callSite.short":"start at StructuredKafkaWordCount.scala:86","spark.submit.pyFiles":"","spark.job.description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 7","spark.executor.id":"driver","spark.sql.cbo.enabled":"false","streaming.sql.batchId":"7","spark.jobGroup.id":"e225d92f-2545-48f8-87a2-9c0309580f8a","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"/tmp/spark-history","spark.sql.execution.id":"22","spark.app.id":"local-1596020211915","spark.sql.shuffle.partitions":"2"}}
+{"Event":"SparkListenerTaskStart","Stage ID":14,"Stage Attempt ID":0,"Task Info":{"Task ID":21,"Index":0,"Attempt":0,"Launch Time":1596020226086,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":14,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":21,"Index":0,"Attempt":0,"Launch Time":1596020226086,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1596020226116,"Failed":false,"Killed":false,"Accumulables":[{"ID":1120,"Name":"shuffle write time","Update":"543034","Value":"543034","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1119,"Name":"shuffle records written","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1118,"Name":"shuffle bytes written","Update":"168","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1110,"Name":"data size","Update":"128","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1167,"Name":"duration","Update":"13","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1168,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1169,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1171,"Name":"time in aggregation build","Update":"8","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1173,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1174,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1176,"Name":"time in aggregation build","Update":"6","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1178,"Name":"duration","Update":"13","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1179,"Name":"number of output rows","Update":"36","Value":"36","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1224,"Name":"internal.metrics.input.recordsRead","Update":36,"Value":36,"Internal":true,"Count Failed Values":true},{"ID":1222,"Name":"internal.metrics.shuffle.write.writeTime","Update":543034,"Value":543034,"Internal":true,"Count Failed Values":true},{"ID":1221,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1220,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":168,"Value":168,"Internal":true,"Count Failed Values":true},{"ID":1211,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":524288,"Internal":true,"Count Failed Values":true},{"ID":1206,"Name":"internal.metrics.resultSize","Update":2544,"Value":2544,"Internal":true,"Count Failed Values":true},{"ID":1205,"Name":"internal.metrics.executorCpuTime","Update":19652237,"Value":19652237,"Internal":true,"Count Failed Values":true},{"ID":1204,"Name":"internal.metrics.executorRunTime","Update":19,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":1203,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2829254,"Value":2829254,"Internal":true,"Count Failed Values":true},{"ID":1202,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2829254,"Executor Run Time":19,"Executor CPU Time":19652237,"Peak Execution Memory":524288,"Result Size":2544,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":168,"Shuffle Write Time":543034,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":36},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":14,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":1,"RDD Info":[{"RDD ID":90,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"239\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[89],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":88,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"245\",\"name\":\"MapPartitions\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[87],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":84,"Name":"DataSourceRDD","Scope":"{\"id\":\"251\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":85,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"251\",\"name\":\"MicroBatchScan\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[84],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":89,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"240\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[88],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":86,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"247\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[85],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":87,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"246\",\"name\":\"DeserializeToObject\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[86],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020226077,"Completion Time":1596020226117,"Accumulables":[{"ID":1205,"Name":"internal.metrics.executorCpuTime","Value":19652237,"Internal":true,"Count Failed Values":true},{"ID":1178,"Name":"duration","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1169,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1118,"Name":"shuffle bytes written","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1204,"Name":"internal.metrics.executorRunTime","Value":19,"Internal":true,"Count Failed Values":true},{"ID":1222,"Name":"internal.metrics.shuffle.write.writeTime","Value":543034,"Internal":true,"Count Failed Values":true},{"ID":1171,"Name":"time in aggregation build","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1120,"Name":"shuffle write time","Value":"543034","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1174,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1168,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1203,"Name":"internal.metrics.executorDeserializeCpuTime","Value":2829254,"Internal":true,"Count Failed Values":true},{"ID":1167,"Name":"duration","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1221,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1176,"Name":"time in aggregation build","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1206,"Name":"internal.metrics.resultSize","Value":2544,"Internal":true,"Count Failed Values":true},{"ID":1224,"Name":"internal.metrics.input.recordsRead","Value":36,"Internal":true,"Count Failed Values":true},{"ID":1179,"Name":"number of output rows","Value":"36","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1119,"Name":"shuffle records written","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1110,"Name":"data size","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1173,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1202,"Name":"internal.metrics.executorDeserializeTime","Value":2,"Internal":true,"Count Failed Values":true},{"ID":1211,"Name":"internal.metrics.peakExecutionMemory","Value":524288,"Internal":true,"Count Failed Values":true},{"ID":1220,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":168,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}}
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":15,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":2,"RDD Info":[{"RDD ID":95,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"231\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[94],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":93,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"235\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[92],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":91,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"239\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[90],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":94,"Name":"StateStoreRDD","Scope":"{\"id\":\"234\",\"name\":\"StateStoreSave\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[93],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":92,"Name":"StateStoreRDD","Scope":"{\"id\":\"238\",\"name\":\"StateStoreRestore\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[91],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[14],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020226120,"Accumulables":[],"Resource Profile Id":0},"Properties":{"sql.streaming.queryId":"8d268dc2-bc9c-4be8-97a9-b135d2943028","spark.driver.host":"iZbp19vpr16ix621sdw476Z","spark.eventLog.enabled":"true","spark.sql.adaptive.enabled":"false","spark.job.interruptOnCancel":"true","spark.driver.port":"46309","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/root/spark-3.1.0-SNAPSHOT-bin-hadoop2.8/./examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar","__is_continuous_processing":"false","spark.app.name":"StructuredKafkaWordCount","callSite.long":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","callSite.short":"start at StructuredKafkaWordCount.scala:86","spark.submit.pyFiles":"","spark.job.description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 7","spark.executor.id":"driver","spark.sql.cbo.enabled":"false","streaming.sql.batchId":"7","spark.jobGroup.id":"e225d92f-2545-48f8-87a2-9c0309580f8a","spark.submit.deployMode":"client","spark.master":"local[*]","spark.eventLog.dir":"/tmp/spark-history","spark.sql.execution.id":"22","spark.app.id":"local-1596020211915","spark.sql.shuffle.partitions":"2"}}
+{"Event":"SparkListenerTaskStart","Stage ID":15,"Stage Attempt ID":0,"Task Info":{"Task ID":22,"Index":0,"Attempt":0,"Launch Time":1596020226128,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":15,"Stage Attempt ID":0,"Task Info":{"Task ID":23,"Index":1,"Attempt":0,"Launch Time":1596020226129,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":15,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":23,"Index":1,"Attempt":0,"Launch Time":1596020226129,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1596020226196,"Failed":false,"Killed":false,"Accumulables":[{"ID":1143,"Name":"duration","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1148,"Name":"avg hash probe bucket list iters","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1144,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1145,"Name":"peak memory","Update":"4456448","Value":"4456448","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1147,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1153,"Name":"time to update","Update":"21","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1152,"Name":"number of updated state rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1154,"Name":"time to remove","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1155,"Name":"time to commit changes","Update":"19","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1157,"Name":"estimated size of state only on current version","Update":"368","Value":"368","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1149,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1158,"Name":"count of cache hit on states cache in provider","Update":"14","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1156,"Name":"memory used by state","Update":"784","Value":"784","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1151,"Name":"number of total state rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1160,"Name":"duration","Update":"21","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1161,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1162,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1164,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1166,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1112,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1116,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1115,"Name":"local bytes read","Update":"168","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1117,"Name":"records read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1244,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1243,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1242,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":168,"Value":168,"Internal":true,"Count Failed Values":true},{"ID":1241,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1240,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1239,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1238,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1236,"Name":"internal.metrics.peakExecutionMemory","Update":4718592,"Value":4718592,"Internal":true,"Count Failed Values":true},{"ID":1231,"Name":"internal.metrics.resultSize","Update":5574,"Value":5574,"Internal":true,"Count Failed Values":true},{"ID":1230,"Name":"internal.metrics.executorCpuTime","Update":19415818,"Value":19415818,"Internal":true,"Count Failed Values":true},{"ID":1229,"Name":"internal.metrics.executorRunTime","Update":60,"Value":60,"Internal":true,"Count Failed Values":true},{"ID":1228,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3845429,"Value":3845429,"Internal":true,"Count Failed Values":true},{"ID":1227,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":3,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3845429,"Executor Run Time":60,"Executor CPU Time":19415818,"Peak Execution Memory":4718592,"Result Size":5574,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":168,"Total Records Read":1},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":15,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":22,"Index":0,"Attempt":0,"Launch Time":1596020226128,"Executor ID":"driver","Host":"iZbp19vpr16ix621sdw476Z","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1596020226204,"Failed":false,"Killed":false,"Accumulables":[{"ID":1143,"Name":"duration","Update":"2","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1145,"Name":"peak memory","Update":"262144","Value":"4718592","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1147,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1153,"Name":"time to update","Update":"3","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1154,"Name":"time to remove","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1155,"Name":"time to commit changes","Update":"48","Value":"67","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1157,"Name":"estimated size of state only on current version","Update":"88","Value":"456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1158,"Name":"count of cache hit on states cache in provider","Update":"14","Value":"28","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1156,"Name":"memory used by state","Update":"400","Value":"1184","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1160,"Name":"duration","Update":"3","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1162,"Name":"peak memory","Update":"262144","Value":"524288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1164,"Name":"time in aggregation build","Update":"0","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1244,"Name":"internal.metrics.shuffle.read.recordsRead","Update":0,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1243,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1242,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":168,"Internal":true,"Count Failed Values":true},{"ID":1241,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1240,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1239,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1238,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1236,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":5242880,"Internal":true,"Count Failed Values":true},{"ID":1231,"Name":"internal.metrics.resultSize","Update":5311,"Value":10885,"Internal":true,"Count Failed Values":true},{"ID":1230,"Name":"internal.metrics.executorCpuTime","Update":14652861,"Value":34068679,"Internal":true,"Count Failed Values":true},{"ID":1229,"Name":"internal.metrics.executorRunTime","Update":65,"Value":125,"Internal":true,"Count Failed Values":true},{"ID":1228,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3933877,"Value":7779306,"Internal":true,"Count Failed Values":true},{"ID":1227,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":6,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3933877,"Executor Run Time":65,"Executor CPU Time":14652861,"Peak Execution Memory":524288,"Result Size":5311,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":15,"Stage Attempt ID":0,"Stage Name":"start at StructuredKafkaWordCount.scala:86","Number of Tasks":2,"RDD Info":[{"RDD ID":95,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"231\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[94],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":93,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"235\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[92],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":91,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"239\",\"name\":\"Exchange\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[90],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":94,"Name":"StateStoreRDD","Scope":"{\"id\":\"234\",\"name\":\"StateStoreSave\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[93],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":92,"Name":"StateStoreRDD","Scope":"{\"id\":\"238\",\"name\":\"StateStoreRestore\"}","Callsite":"start at StructuredKafkaWordCount.scala:86","Parent IDs":[91],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[14],"Details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1596020226120,"Completion Time":1596020226204,"Accumulables":[{"ID":1115,"Name":"local bytes read","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1160,"Name":"duration","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1151,"Name":"number of total state rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1145,"Name":"peak memory","Value":"4718592","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1154,"Name":"time to remove","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1231,"Name":"internal.metrics.resultSize","Value":10885,"Internal":true,"Count Failed Values":true},{"ID":1240,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1153,"Name":"time to update","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1162,"Name":"peak memory","Value":"524288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1144,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1243,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1117,"Name":"records read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1228,"Name":"internal.metrics.executorDeserializeCpuTime","Value":7779306,"Internal":true,"Count Failed Values":true},{"ID":1147,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1156,"Name":"memory used by state","Value":"1184","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1236,"Name":"internal.metrics.peakExecutionMemory","Value":5242880,"Internal":true,"Count Failed Values":true},{"ID":1227,"Name":"internal.metrics.executorDeserializeTime","Value":6,"Internal":true,"Count Failed Values":true},{"ID":1158,"Name":"count of cache hit on states cache in provider","Value":"28","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1149,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1239,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1230,"Name":"internal.metrics.executorCpuTime","Value":34068679,"Internal":true,"Count Failed Values":true},{"ID":1152,"Name":"number of updated state rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1242,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":168,"Internal":true,"Count Failed Values":true},{"ID":1116,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1161,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1143,"Name":"duration","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1155,"Name":"time to commit changes","Value":"67","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1164,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1241,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1244,"Name":"internal.metrics.shuffle.read.recordsRead","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1148,"Name":"avg hash probe bucket list iters","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1157,"Name":"estimated size of state only on current version","Value":"456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1166,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1238,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1229,"Name":"internal.metrics.executorRunTime","Value":125,"Internal":true,"Count Failed Values":true},{"ID":1112,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"}],"Resource Profile Id":0}}
+{"Event":"SparkListenerJobEnd","Job ID":7,"Completion Time":1596020226204,"Job Result":{"Result":"JobSucceeded"}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":23,"description":"\nid = 8d268dc2-bc9c-4be8-97a9-b135d2943028\nrunId = e225d92f-2545-48f8-87a2-9c0309580f8a\nbatch = 7","details":"org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount$.main(StructuredKafkaWordCount.scala:86)\norg.apache.spark.examples.sql.streaming.StructuredKafkaWordCount.main(StructuredKafkaWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nLocalTableScan (1)\n\n\n(1) LocalTableScan\nOutput [2]: [value#144, count#145]\nArguments: [value#144, count#145]\n\n","sparkPlanInfo":{"nodeName":"LocalTableScan","simpleString":"LocalTableScan [value#144, count#145]","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1252,"metricType":"sum"}]},"time":1596020226221}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":23,"time":1596020226230}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":22,"time":1596020226231}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":21,"time":1596020226231}
+{"Event":"org.apache.spark.sql.streaming.StreamingQueryListener$QueryProgressEvent","progress":{"id":"8d268dc2-bc9c-4be8-97a9-b135d2943028","runId":"e225d92f-2545-48f8-87a2-9c0309580f8a","name":null,"timestamp":"2020-07-29T10:57:05.916Z","batchId":7,"batchDuration":341,"durationMs":{"triggerExecution":341,"queryPlanning":24,"getBatch":0,"latestOffset":3,"addBatch":271,"walCommit":14},"eventTime":{},"stateOperators":[{"numRowsTotal":1,"numRowsUpdated":1,"memoryUsedBytes":1184,"numLateInputs":0,"customMetrics":{"stateOnCurrentVersionSizeBytes":456,"loadedMapCacheHitCount":28,"loadedMapCacheMissCount":0}}],"sources":[{"description":"KafkaV2[Subscribe[test5]]","startOffset":"{\"test5\":{\"0\":48881}}","endOffset":"{\"test5\":{\"0\":48917}}","numInputRows":36,"inputRowsPerSecond":101.69491525423729,"processedRowsPerSecond":105.57184750733137}],"sink":{"description":"org.apache.spark.sql.execution.streaming.ConsoleTable$@514ba885","numOutputRows":1},"observedMetrics":{}}}
+{"Event":"SparkListenerApplicationEnd","Timestamp":1596020226301}
diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
index 3570fb61e2880..87cabb29cbbeb 100644
--- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
+++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
@@ -1,30 +1,30 @@
 <!-- Automatically generated by ExpressionsSchemaSuite -->
 ## Summary
-  - Number of queries: 333
-  - Number of expressions that missing example: 34
-  - Expressions missing examples: and,string,tinyint,double,smallint,date,decimal,boolean,float,binary,bigint,int,timestamp,cume_dist,dense_rank,input_file_block_length,input_file_block_start,input_file_name,lag,lead,monotonically_increasing_id,ntile,struct,!,not,or,percent_rank,rank,row_number,spark_partition_id,version,window,positive,count_min_sketch
+  - Number of queries: 352
+  - Number of expressions that missing example: 13
+  - Expressions missing examples: bigint,binary,boolean,date,decimal,double,float,int,smallint,string,timestamp,tinyint,window
 ## Schema of Built-in Functions
 | Class name | Function name or alias | Query example | Output schema |
 | ---------- | ---------------------- | ------------- | ------------- |
 | org.apache.spark.sql.catalyst.expressions.Abs | abs | SELECT abs(-1) | struct<abs(-1):int> |
-| org.apache.spark.sql.catalyst.expressions.Acos | acos | SELECT acos(1) | struct<ACOS(CAST(1 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.Acosh | acosh | SELECT acosh(1) | struct<ACOSH(CAST(1 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Acos | acos | SELECT acos(1) | struct<ACOS(1):double> |
+| org.apache.spark.sql.catalyst.expressions.Acosh | acosh | SELECT acosh(1) | struct<ACOSH(1):double> |
 | org.apache.spark.sql.catalyst.expressions.Add | + | SELECT 1 + 2 | struct<(1 + 2):int> |
-| org.apache.spark.sql.catalyst.expressions.AddMonths | add_months | SELECT add_months('2016-08-31', 1) | struct<add_months(CAST(2016-08-31 AS DATE), 1):date> |
-| org.apache.spark.sql.catalyst.expressions.And | and | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.AddMonths | add_months | SELECT add_months('2016-08-31', 1) | struct<add_months(2016-08-31, 1):date> |
+| org.apache.spark.sql.catalyst.expressions.And | and | SELECT true and true | struct<(true AND true):boolean> |
 | org.apache.spark.sql.catalyst.expressions.ArrayAggregate | aggregate | SELECT aggregate(array(1, 2, 3), 0, (acc, x) -> acc + x) | struct<aggregate(array(1, 2, 3), 0, lambdafunction((namedlambdavariable() + namedlambdavariable()), namedlambdavariable(), namedlambdavariable()), lambdafunction(namedlambdavariable(), namedlambdavariable())):int> |
 | org.apache.spark.sql.catalyst.expressions.ArrayContains | array_contains | SELECT array_contains(array(1, 2, 3), 2) | struct<array_contains(array(1, 2, 3), 2):boolean> |
-| org.apache.spark.sql.catalyst.expressions.ArrayDistinct | array_distinct | SELECT array_distinct(array(1, 2, 3, null, 3)) | struct<array_distinct(array(1, 2, 3, CAST(NULL AS INT), 3)):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.ArrayDistinct | array_distinct | SELECT array_distinct(array(1, 2, 3, null, 3)) | struct<array_distinct(array(1, 2, 3, NULL, 3)):array<int>> |
 | org.apache.spark.sql.catalyst.expressions.ArrayExcept | array_except | SELECT array_except(array(1, 2, 3), array(1, 3, 5)) | struct<array_except(array(1, 2, 3), array(1, 3, 5)):array<int>> |
 | org.apache.spark.sql.catalyst.expressions.ArrayExists | exists | SELECT exists(array(1, 2, 3), x -> x % 2 == 0) | struct<exists(array(1, 2, 3), lambdafunction(((namedlambdavariable() % 2) = 0), namedlambdavariable())):boolean> |
 | org.apache.spark.sql.catalyst.expressions.ArrayFilter | filter | SELECT filter(array(1, 2, 3), x -> x % 2 == 1) | struct<filter(array(1, 2, 3), lambdafunction(((namedlambdavariable() % 2) = 1), namedlambdavariable())):array<int>> |
 | org.apache.spark.sql.catalyst.expressions.ArrayForAll | forall | SELECT forall(array(1, 2, 3), x -> x % 2 == 0) | struct<forall(array(1, 2, 3), lambdafunction(((namedlambdavariable() % 2) = 0), namedlambdavariable())):boolean> |
 | org.apache.spark.sql.catalyst.expressions.ArrayIntersect | array_intersect | SELECT array_intersect(array(1, 2, 3), array(1, 3, 5)) | struct<array_intersect(array(1, 2, 3), array(1, 3, 5)):array<int>> |
 | org.apache.spark.sql.catalyst.expressions.ArrayJoin | array_join | SELECT array_join(array('hello', 'world'), ' ') | struct<array_join(array(hello, world),  ):string> |
-| org.apache.spark.sql.catalyst.expressions.ArrayMax | array_max | SELECT array_max(array(1, 20, null, 3)) | struct<array_max(array(1, 20, CAST(NULL AS INT), 3)):int> |
-| org.apache.spark.sql.catalyst.expressions.ArrayMin | array_min | SELECT array_min(array(1, 20, null, 3)) | struct<array_min(array(1, 20, CAST(NULL AS INT), 3)):int> |
+| org.apache.spark.sql.catalyst.expressions.ArrayMax | array_max | SELECT array_max(array(1, 20, null, 3)) | struct<array_max(array(1, 20, NULL, 3)):int> |
+| org.apache.spark.sql.catalyst.expressions.ArrayMin | array_min | SELECT array_min(array(1, 20, null, 3)) | struct<array_min(array(1, 20, NULL, 3)):int> |
 | org.apache.spark.sql.catalyst.expressions.ArrayPosition | array_position | SELECT array_position(array(3, 2, 1), 1) | struct<array_position(array(3, 2, 1), 1):bigint> |
-| org.apache.spark.sql.catalyst.expressions.ArrayRemove | array_remove | SELECT array_remove(array(1, 2, 3, null, 3), 3) | struct<array_remove(array(1, 2, 3, CAST(NULL AS INT), 3), 3):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.ArrayRemove | array_remove | SELECT array_remove(array(1, 2, 3, null, 3), 3) | struct<array_remove(array(1, 2, 3, NULL, 3), 3):array<int>> |
 | org.apache.spark.sql.catalyst.expressions.ArrayRepeat | array_repeat | SELECT array_repeat('123', 2) | struct<array_repeat(123, 2):array<string>> |
 | org.apache.spark.sql.catalyst.expressions.ArraySort | array_sort | SELECT array_sort(array(5, 6, 1), (left, right) -> case when left < right then -1 when left > right then 1 else 0 end) | struct<array_sort(array(5, 6, 1), lambdafunction(CASE WHEN (namedlambdavariable() < namedlambdavariable()) THEN -1 WHEN (namedlambdavariable() > namedlambdavariable()) THEN 1 ELSE 0 END, namedlambdavariable(), namedlambdavariable())):array<int>> |
 | org.apache.spark.sql.catalyst.expressions.ArrayTransform | transform | SELECT transform(array(1, 2, 3), x -> x + 1) | struct<transform(array(1, 2, 3), lambdafunction((namedlambdavariable() + 1), namedlambdavariable())):array<int>> |
@@ -32,71 +32,76 @@
 | org.apache.spark.sql.catalyst.expressions.ArraysOverlap | arrays_overlap | SELECT arrays_overlap(array(1, 2, 3), array(3, 4, 5)) | struct<arrays_overlap(array(1, 2, 3), array(3, 4, 5)):boolean> |
 | org.apache.spark.sql.catalyst.expressions.ArraysZip | arrays_zip | SELECT arrays_zip(array(1, 2, 3), array(2, 3, 4)) | struct<arrays_zip(array(1, 2, 3), array(2, 3, 4)):array<struct<0:int,1:int>>> |
 | org.apache.spark.sql.catalyst.expressions.Ascii | ascii | SELECT ascii('222') | struct<ascii(222):int> |
-| org.apache.spark.sql.catalyst.expressions.Asin | asin | SELECT asin(0) | struct<ASIN(CAST(0 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.Asinh | asinh | SELECT asinh(0) | struct<ASINH(CAST(0 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.AssertTrue | assert_true | SELECT assert_true(0 < 1) | struct<assert_true((0 < 1)):null> |
-| org.apache.spark.sql.catalyst.expressions.Atan | atan | SELECT atan(0) | struct<ATAN(CAST(0 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.Atan2 | atan2 | SELECT atan2(0, 0) | struct<ATAN2(CAST(0 AS DOUBLE), CAST(0 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.Atanh | atanh | SELECT atanh(0) | struct<ATANH(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Asin | asin | SELECT asin(0) | struct<ASIN(0):double> |
+| org.apache.spark.sql.catalyst.expressions.Asinh | asinh | SELECT asinh(0) | struct<ASINH(0):double> |
+| org.apache.spark.sql.catalyst.expressions.AssertTrue | assert_true | SELECT assert_true(0 < 1) | struct<assert_true((0 < 1), '(0 < 1)' is not true!):null> |
+| org.apache.spark.sql.catalyst.expressions.Atan | atan | SELECT atan(0) | struct<ATAN(0):double> |
+| org.apache.spark.sql.catalyst.expressions.Atan2 | atan2 | SELECT atan2(0, 0) | struct<ATAN2(0, 0):double> |
+| org.apache.spark.sql.catalyst.expressions.Atanh | atanh | SELECT atanh(0) | struct<ATANH(0):double> |
 | org.apache.spark.sql.catalyst.expressions.BRound | bround | SELECT bround(2.5, 0) | struct<bround(2.5, 0):decimal(2,0)> |
-| org.apache.spark.sql.catalyst.expressions.Base64 | base64 | SELECT base64('Spark SQL') | struct<base64(CAST(Spark SQL AS BINARY)):string> |
-| org.apache.spark.sql.catalyst.expressions.Bin | bin | SELECT bin(13) | struct<bin(CAST(13 AS BIGINT)):string> |
+| org.apache.spark.sql.catalyst.expressions.Base64 | base64 | SELECT base64('Spark SQL') | struct<base64(Spark SQL):string> |
+| org.apache.spark.sql.catalyst.expressions.Bin | bin | SELECT bin(13) | struct<bin(13):string> |
 | org.apache.spark.sql.catalyst.expressions.BitLength | bit_length | SELECT bit_length('Spark SQL') | struct<bit_length(Spark SQL):int> |
 | org.apache.spark.sql.catalyst.expressions.BitwiseAnd | & | SELECT 3 & 5 | struct<(3 & 5):int> |
 | org.apache.spark.sql.catalyst.expressions.BitwiseCount | bit_count | SELECT bit_count(0) | struct<bit_count(0):int> |
+| org.apache.spark.sql.catalyst.expressions.BitwiseGet | bit_get | SELECT bit_get(11, 0) | struct<bit_get(11, 0):tinyint> |
+| org.apache.spark.sql.catalyst.expressions.BitwiseGet | getbit | SELECT getbit(11, 0) | struct<getbit(11, 0):tinyint> |
 | org.apache.spark.sql.catalyst.expressions.BitwiseNot | ~ | SELECT ~ 0 | struct<~0:int> |
 | org.apache.spark.sql.catalyst.expressions.BitwiseOr | &#124; | SELECT 3 &#124; 5 | struct<(3 &#124; 5):int> |
 | org.apache.spark.sql.catalyst.expressions.BitwiseXor | ^ | SELECT 3 ^ 5 | struct<(3 ^ 5):int> |
 | org.apache.spark.sql.catalyst.expressions.CallMethodViaReflection | java_method | SELECT java_method('java.util.UUID', 'randomUUID') | struct<java_method(java.util.UUID, randomUUID):string> |
 | org.apache.spark.sql.catalyst.expressions.CallMethodViaReflection | reflect | SELECT reflect('java.util.UUID', 'randomUUID') | struct<reflect(java.util.UUID, randomUUID):string> |
-| org.apache.spark.sql.catalyst.expressions.CaseWhen | when | SELECT CASE WHEN 1 > 0 THEN 1 WHEN 2 > 0 THEN 2.0 ELSE 1.2 END | struct<CASE WHEN (1 > 0) THEN CAST(1 AS DECIMAL(11,1)) WHEN (2 > 0) THEN CAST(2.0 AS DECIMAL(11,1)) ELSE CAST(1.2 AS DECIMAL(11,1)) END:decimal(11,1)> |
-| org.apache.spark.sql.catalyst.expressions.Cast | string | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.CaseWhen | when | SELECT CASE WHEN 1 > 0 THEN 1 WHEN 2 > 0 THEN 2.0 ELSE 1.2 END | struct<CASE WHEN (1 > 0) THEN 1 WHEN (2 > 0) THEN 2.0 ELSE 1.2 END:decimal(11,1)> |
+| org.apache.spark.sql.catalyst.expressions.Cast | bigint | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | binary | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | boolean | N/A | N/A |
 | org.apache.spark.sql.catalyst.expressions.Cast | cast | SELECT cast('10' as int) | struct<CAST(10 AS INT):int> |
-| org.apache.spark.sql.catalyst.expressions.Cast | tinyint | N/A | N/A |
-| org.apache.spark.sql.catalyst.expressions.Cast | double | N/A | N/A |
-| org.apache.spark.sql.catalyst.expressions.Cast | smallint | N/A | N/A |
 | org.apache.spark.sql.catalyst.expressions.Cast | date | N/A | N/A |
 | org.apache.spark.sql.catalyst.expressions.Cast | decimal | N/A | N/A |
-| org.apache.spark.sql.catalyst.expressions.Cast | boolean | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | double | N/A | N/A |
 | org.apache.spark.sql.catalyst.expressions.Cast | float | N/A | N/A |
-| org.apache.spark.sql.catalyst.expressions.Cast | binary | N/A | N/A |
-| org.apache.spark.sql.catalyst.expressions.Cast | bigint | N/A | N/A |
 | org.apache.spark.sql.catalyst.expressions.Cast | int | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | smallint | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | string | N/A | N/A |
 | org.apache.spark.sql.catalyst.expressions.Cast | timestamp | N/A | N/A |
-| org.apache.spark.sql.catalyst.expressions.Cbrt | cbrt | SELECT cbrt(27.0) | struct<CBRT(CAST(27.0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Cast | tinyint | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cbrt | cbrt | SELECT cbrt(27.0) | struct<CBRT(27.0):double> |
 | org.apache.spark.sql.catalyst.expressions.Ceil | ceil | SELECT ceil(-0.1) | struct<CEIL(-0.1):decimal(1,0)> |
 | org.apache.spark.sql.catalyst.expressions.Ceil | ceiling | SELECT ceiling(-0.1) | struct<ceiling(-0.1):decimal(1,0)> |
-| org.apache.spark.sql.catalyst.expressions.Chr | char | SELECT char(65) | struct<char(CAST(65 AS BIGINT)):string> |
-| org.apache.spark.sql.catalyst.expressions.Chr | chr | SELECT chr(65) | struct<chr(CAST(65 AS BIGINT)):string> |
-| org.apache.spark.sql.catalyst.expressions.Coalesce | coalesce | SELECT coalesce(NULL, 1, NULL) | struct<coalesce(CAST(NULL AS INT), 1, CAST(NULL AS INT)):int> |
+| org.apache.spark.sql.catalyst.expressions.Chr | char | SELECT char(65) | struct<char(65):string> |
+| org.apache.spark.sql.catalyst.expressions.Chr | chr | SELECT chr(65) | struct<chr(65):string> |
+| org.apache.spark.sql.catalyst.expressions.Coalesce | coalesce | SELECT coalesce(NULL, 1, NULL) | struct<coalesce(NULL, 1, NULL):int> |
 | org.apache.spark.sql.catalyst.expressions.Concat | concat | SELECT concat('Spark', 'SQL') | struct<concat(Spark, SQL):string> |
 | org.apache.spark.sql.catalyst.expressions.ConcatWs | concat_ws | SELECT concat_ws(' ', 'Spark', 'SQL') | struct<concat_ws( , Spark, SQL):string> |
 | org.apache.spark.sql.catalyst.expressions.Conv | conv | SELECT conv('100', 2, 10) | struct<conv(100, 2, 10):string> |
-| org.apache.spark.sql.catalyst.expressions.Cos | cos | SELECT cos(0) | struct<COS(CAST(0 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.Cosh | cosh | SELECT cosh(0) | struct<COSH(CAST(0 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.Cot | cot | SELECT cot(1) | struct<COT(CAST(1 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.Crc32 | crc32 | SELECT crc32('Spark') | struct<crc32(CAST(Spark AS BINARY)):bigint> |
+| org.apache.spark.sql.catalyst.expressions.Cos | cos | SELECT cos(0) | struct<COS(0):double> |
+| org.apache.spark.sql.catalyst.expressions.Cosh | cosh | SELECT cosh(0) | struct<COSH(0):double> |
+| org.apache.spark.sql.catalyst.expressions.Cot | cot | SELECT cot(1) | struct<COT(1):double> |
+| org.apache.spark.sql.catalyst.expressions.Crc32 | crc32 | SELECT crc32('Spark') | struct<crc32(Spark):bigint> |
 | org.apache.spark.sql.catalyst.expressions.CreateArray | array | SELECT array(1, 2, 3) | struct<array(1, 2, 3):array<int>> |
 | org.apache.spark.sql.catalyst.expressions.CreateMap | map | SELECT map(1.0, '2', 3.0, '4') | struct<map(1.0, 2, 3.0, 4):map<decimal(2,1),string>> |
 | org.apache.spark.sql.catalyst.expressions.CreateNamedStruct | named_struct | SELECT named_struct("a", 1, "b", 2, "c", 3) | struct<named_struct(a, 1, b, 2, c, 3):struct<a:int,b:int,c:int>> |
+| org.apache.spark.sql.catalyst.expressions.CreateNamedStruct | struct | SELECT struct(1, 2, 3) | struct<struct(1, 2, 3):struct<col1:int,col2:int,col3:int>> |
 | org.apache.spark.sql.catalyst.expressions.CsvToStructs | from_csv | SELECT from_csv('1, 0.8', 'a INT, b DOUBLE') | struct<from_csv(1, 0.8):struct<a:int,b:double>> |
-| org.apache.spark.sql.catalyst.expressions.Cube | cube | SELECT name, age, count(*) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY cube(name, age) | struct<name:string,age:int,count(1):bigint> |
-| org.apache.spark.sql.catalyst.expressions.CumeDist | cume_dist | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.CumeDist | cume_dist | SELECT a, b, cume_dist() OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b) | struct<a:string,b:int,cume_dist() OVER (PARTITION BY a ORDER BY b ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):double> |
+| org.apache.spark.sql.catalyst.expressions.CurrentCatalog | current_catalog | SELECT current_catalog() | struct<current_catalog():string> |
 | org.apache.spark.sql.catalyst.expressions.CurrentDatabase | current_database | SELECT current_database() | struct<current_database():string> |
 | org.apache.spark.sql.catalyst.expressions.CurrentDate | current_date | SELECT current_date() | struct<current_date():date> |
+| org.apache.spark.sql.catalyst.expressions.CurrentTimeZone | current_timezone | SELECT current_timezone() | struct<current_timezone():string> |
 | org.apache.spark.sql.catalyst.expressions.CurrentTimestamp | current_timestamp | SELECT current_timestamp() | struct<current_timestamp():timestamp> |
-| org.apache.spark.sql.catalyst.expressions.DateAdd | date_add | SELECT date_add('2016-07-30', 1) | struct<date_add(CAST(2016-07-30 AS DATE), 1):date> |
-| org.apache.spark.sql.catalyst.expressions.DateDiff | datediff | SELECT datediff('2009-07-31', '2009-07-30') | struct<datediff(CAST(2009-07-31 AS DATE), CAST(2009-07-30 AS DATE)):int> |
-| org.apache.spark.sql.catalyst.expressions.DateFormatClass | date_format | SELECT date_format('2016-04-08', 'y') | struct<date_format(CAST(2016-04-08 AS TIMESTAMP), y):string> |
+| org.apache.spark.sql.catalyst.expressions.DateAdd | date_add | SELECT date_add('2016-07-30', 1) | struct<date_add(2016-07-30, 1):date> |
+| org.apache.spark.sql.catalyst.expressions.DateDiff | datediff | SELECT datediff('2009-07-31', '2009-07-30') | struct<datediff(2009-07-31, 2009-07-30):int> |
+| org.apache.spark.sql.catalyst.expressions.DateFormatClass | date_format | SELECT date_format('2016-04-08', 'y') | struct<date_format(2016-04-08, y):string> |
+| org.apache.spark.sql.catalyst.expressions.DateFromUnixDate | date_from_unix_date | SELECT date_from_unix_date(1) | struct<date_from_unix_date(1):date> |
 | org.apache.spark.sql.catalyst.expressions.DatePart | date_part | SELECT date_part('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456') | struct<date_part(YEAR, TIMESTAMP '2019-08-12 01:00:00.123456'):int> |
-| org.apache.spark.sql.catalyst.expressions.DateSub | date_sub | SELECT date_sub('2016-07-30', 1) | struct<date_sub(CAST(2016-07-30 AS DATE), 1):date> |
-| org.apache.spark.sql.catalyst.expressions.DayOfMonth | day | SELECT day('2009-07-30') | struct<day(CAST(2009-07-30 AS DATE)):int> |
-| org.apache.spark.sql.catalyst.expressions.DayOfMonth | dayofmonth | SELECT dayofmonth('2009-07-30') | struct<dayofmonth(CAST(2009-07-30 AS DATE)):int> |
-| org.apache.spark.sql.catalyst.expressions.DayOfWeek | dayofweek | SELECT dayofweek('2009-07-30') | struct<dayofweek(CAST(2009-07-30 AS DATE)):int> |
-| org.apache.spark.sql.catalyst.expressions.DayOfYear | dayofyear | SELECT dayofyear('2016-04-09') | struct<dayofyear(CAST(2016-04-09 AS DATE)):int> |
+| org.apache.spark.sql.catalyst.expressions.DateSub | date_sub | SELECT date_sub('2016-07-30', 1) | struct<date_sub(2016-07-30, 1):date> |
+| org.apache.spark.sql.catalyst.expressions.DayOfMonth | day | SELECT day('2009-07-30') | struct<day(2009-07-30):int> |
+| org.apache.spark.sql.catalyst.expressions.DayOfMonth | dayofmonth | SELECT dayofmonth('2009-07-30') | struct<dayofmonth(2009-07-30):int> |
+| org.apache.spark.sql.catalyst.expressions.DayOfWeek | dayofweek | SELECT dayofweek('2009-07-30') | struct<dayofweek(2009-07-30):int> |
+| org.apache.spark.sql.catalyst.expressions.DayOfYear | dayofyear | SELECT dayofyear('2016-04-09') | struct<dayofyear(2016-04-09):int> |
 | org.apache.spark.sql.catalyst.expressions.Decode | decode | SELECT decode(encode('abc', 'utf-8'), 'utf-8') | struct<decode(encode(abc, utf-8), utf-8):string> |
-| org.apache.spark.sql.catalyst.expressions.DenseRank | dense_rank | N/A | N/A |
-| org.apache.spark.sql.catalyst.expressions.Divide | / | SELECT 3 / 2 | struct<(CAST(3 AS DOUBLE) / CAST(2 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.DenseRank | dense_rank | SELECT a, b, dense_rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b) | struct<a:string,b:int,DENSE_RANK() OVER (PARTITION BY a ORDER BY b ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int> |
+| org.apache.spark.sql.catalyst.expressions.Divide | / | SELECT 3 / 2 | struct<(3 / 2):double> |
 | org.apache.spark.sql.catalyst.expressions.ElementAt | element_at | SELECT element_at(array(1, 2, 3), 2) | struct<element_at(array(1, 2, 3), 2):int> |
 | org.apache.spark.sql.catalyst.expressions.Elt | elt | SELECT elt(1, 'scala', 'java') | struct<elt(1, scala, java):string> |
 | org.apache.spark.sql.catalyst.expressions.Encode | encode | SELECT encode('abc', 'utf-8') | struct<encode(abc, utf-8):binary> |
@@ -104,38 +109,38 @@
 | org.apache.spark.sql.catalyst.expressions.EqualTo | = | SELECT 2 = 2 | struct<(2 = 2):boolean> |
 | org.apache.spark.sql.catalyst.expressions.EqualTo | == | SELECT 2 == 2 | struct<(2 = 2):boolean> |
 | org.apache.spark.sql.catalyst.expressions.EulerNumber | e | SELECT e() | struct<E():double> |
-| org.apache.spark.sql.catalyst.expressions.Exp | exp | SELECT exp(0) | struct<EXP(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Exp | exp | SELECT exp(0) | struct<EXP(0):double> |
 | org.apache.spark.sql.catalyst.expressions.Explode | explode | SELECT explode(array(10, 20)) | struct<col:int> |
 | org.apache.spark.sql.catalyst.expressions.Explode | explode_outer | SELECT explode_outer(array(10, 20)) | struct<col:int> |
-| org.apache.spark.sql.catalyst.expressions.Expm1 | expm1 | SELECT expm1(0) | struct<EXPM1(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Expm1 | expm1 | SELECT expm1(0) | struct<EXPM1(0):double> |
 | org.apache.spark.sql.catalyst.expressions.Extract | extract | SELECT extract(YEAR FROM TIMESTAMP '2019-08-12 01:00:00.123456') | struct<extract(YEAR FROM TIMESTAMP '2019-08-12 01:00:00.123456'):int> |
 | org.apache.spark.sql.catalyst.expressions.Factorial | factorial | SELECT factorial(5) | struct<factorial(5):bigint> |
 | org.apache.spark.sql.catalyst.expressions.FindInSet | find_in_set | SELECT find_in_set('ab','abc,b,ab,c,def') | struct<find_in_set(ab, abc,b,ab,c,def):int> |
 | org.apache.spark.sql.catalyst.expressions.Flatten | flatten | SELECT flatten(array(array(1, 2), array(3, 4))) | struct<flatten(array(array(1, 2), array(3, 4))):array<int>> |
 | org.apache.spark.sql.catalyst.expressions.Floor | floor | SELECT floor(-0.1) | struct<FLOOR(-0.1):decimal(1,0)> |
 | org.apache.spark.sql.catalyst.expressions.FormatNumber | format_number | SELECT format_number(12332.123456, 4) | struct<format_number(12332.123456, 4):string> |
-| org.apache.spark.sql.catalyst.expressions.FormatString | printf | SELECT printf("Hello World %d %s", 100, "days") | struct<printf(Hello World %d %s, 100, days):string> |
 | org.apache.spark.sql.catalyst.expressions.FormatString | format_string | SELECT format_string("Hello World %d %s", 100, "days") | struct<format_string(Hello World %d %s, 100, days):string> |
-| org.apache.spark.sql.catalyst.expressions.FromUTCTimestamp | from_utc_timestamp | SELECT from_utc_timestamp('2016-08-31', 'Asia/Seoul') | struct<from_utc_timestamp(CAST(2016-08-31 AS TIMESTAMP), Asia/Seoul):timestamp> |
-| org.apache.spark.sql.catalyst.expressions.FromUnixTime | from_unixtime | SELECT from_unixtime(0, 'yyyy-MM-dd HH:mm:ss') | struct<from_unixtime(CAST(0 AS BIGINT), yyyy-MM-dd HH:mm:ss):string> |
+| org.apache.spark.sql.catalyst.expressions.FormatString | printf | SELECT printf("Hello World %d %s", 100, "days") | struct<printf(Hello World %d %s, 100, days):string> |
+| org.apache.spark.sql.catalyst.expressions.FromUTCTimestamp | from_utc_timestamp | SELECT from_utc_timestamp('2016-08-31', 'Asia/Seoul') | struct<from_utc_timestamp(2016-08-31, Asia/Seoul):timestamp> |
+| org.apache.spark.sql.catalyst.expressions.FromUnixTime | from_unixtime | SELECT from_unixtime(0, 'yyyy-MM-dd HH:mm:ss') | struct<from_unixtime(0, yyyy-MM-dd HH:mm:ss):string> |
 | org.apache.spark.sql.catalyst.expressions.GetJsonObject | get_json_object | SELECT get_json_object('{"a":"b"}', '$.a') | struct<get_json_object({"a":"b"}, $.a):string> |
 | org.apache.spark.sql.catalyst.expressions.GreaterThan | > | SELECT 2 > 1 | struct<(2 > 1):boolean> |
 | org.apache.spark.sql.catalyst.expressions.GreaterThanOrEqual | >= | SELECT 2 >= 1 | struct<(2 >= 1):boolean> |
 | org.apache.spark.sql.catalyst.expressions.Greatest | greatest | SELECT greatest(10, 9, 2, 4, 3) | struct<greatest(10, 9, 2, 4, 3):int> |
 | org.apache.spark.sql.catalyst.expressions.Grouping | grouping | SELECT name, grouping(name), sum(age) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY cube(name) | struct<name:string,grouping(name):tinyint,sum(age):bigint> |
 | org.apache.spark.sql.catalyst.expressions.GroupingID | grouping_id | SELECT name, grouping_id(), sum(age), avg(height) FROM VALUES (2, 'Alice', 165), (5, 'Bob', 180) people(age, name, height) GROUP BY cube(name, height) | struct<name:string,grouping_id():bigint,sum(age):bigint,avg(height):double> |
-| org.apache.spark.sql.catalyst.expressions.Hex | hex | SELECT hex(17) | struct<hex(CAST(17 AS BIGINT)):string> |
-| org.apache.spark.sql.catalyst.expressions.Hour | hour | SELECT hour('2009-07-30 12:58:59') | struct<hour(CAST(2009-07-30 12:58:59 AS TIMESTAMP)):int> |
-| org.apache.spark.sql.catalyst.expressions.Hypot | hypot | SELECT hypot(3, 4) | struct<HYPOT(CAST(3 AS DOUBLE), CAST(4 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Hex | hex | SELECT hex(17) | struct<hex(17):string> |
+| org.apache.spark.sql.catalyst.expressions.Hour | hour | SELECT hour('2009-07-30 12:58:59') | struct<hour(2009-07-30 12:58:59):int> |
+| org.apache.spark.sql.catalyst.expressions.Hypot | hypot | SELECT hypot(3, 4) | struct<HYPOT(3, 4):double> |
 | org.apache.spark.sql.catalyst.expressions.If | if | SELECT if(1 < 2, 'a', 'b') | struct<(IF((1 < 2), a, b)):string> |
 | org.apache.spark.sql.catalyst.expressions.IfNull | ifnull | SELECT ifnull(NULL, array('2')) | struct<ifnull(NULL, array(2)):array<string>> |
 | org.apache.spark.sql.catalyst.expressions.In | in | SELECT 1 in(1, 2, 3) | struct<(1 IN (1, 2, 3)):boolean> |
 | org.apache.spark.sql.catalyst.expressions.InitCap | initcap | SELECT initcap('sPark sql') | struct<initcap(sPark sql):string> |
 | org.apache.spark.sql.catalyst.expressions.Inline | inline | SELECT inline(array(struct(1, 'a'), struct(2, 'b'))) | struct<col1:int,col2:string> |
 | org.apache.spark.sql.catalyst.expressions.Inline | inline_outer | SELECT inline_outer(array(struct(1, 'a'), struct(2, 'b'))) | struct<col1:int,col2:string> |
-| org.apache.spark.sql.catalyst.expressions.InputFileBlockLength | input_file_block_length | N/A | N/A |
-| org.apache.spark.sql.catalyst.expressions.InputFileBlockStart | input_file_block_start | N/A | N/A |
-| org.apache.spark.sql.catalyst.expressions.InputFileName | input_file_name | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.InputFileBlockLength | input_file_block_length | SELECT input_file_block_length() | struct<input_file_block_length():bigint> |
+| org.apache.spark.sql.catalyst.expressions.InputFileBlockStart | input_file_block_start | SELECT input_file_block_start() | struct<input_file_block_start():bigint> |
+| org.apache.spark.sql.catalyst.expressions.InputFileName | input_file_name | SELECT input_file_name() | struct<input_file_name():string> |
 | org.apache.spark.sql.catalyst.expressions.IntegralDivide | div | SELECT 3 div 2 | struct<(3 div 2):bigint> |
 | org.apache.spark.sql.catalyst.expressions.IsNaN | isnan | SELECT isnan(cast('NaN' as double)) | struct<isnan(CAST(NaN AS DOUBLE)):boolean> |
 | org.apache.spark.sql.catalyst.expressions.IsNotNull | isnotnull | SELECT isnotnull(1) | struct<(1 IS NOT NULL):boolean> |
@@ -143,112 +148,118 @@
 | org.apache.spark.sql.catalyst.expressions.JsonObjectKeys | json_object_keys | SELECT json_object_keys('{}') | struct<json_object_keys({}):array<string>> |
 | org.apache.spark.sql.catalyst.expressions.JsonToStructs | from_json | SELECT from_json('{"a":1, "b":0.8}', 'a INT, b DOUBLE') | struct<from_json({"a":1, "b":0.8}):struct<a:int,b:double>> |
 | org.apache.spark.sql.catalyst.expressions.JsonTuple | json_tuple | SELECT json_tuple('{"a":1, "b":2}', 'a', 'b') | struct<c0:string,c1:string> |
-| org.apache.spark.sql.catalyst.expressions.Lag | lag | N/A | N/A |
-| org.apache.spark.sql.catalyst.expressions.LastDay | last_day | SELECT last_day('2009-01-12') | struct<last_day(CAST(2009-01-12 AS DATE)):date> |
-| org.apache.spark.sql.catalyst.expressions.Lead | lead | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Lag | lag | SELECT a, b, lag(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b) | struct<a:string,b:int,lag(b, 1, NULL) OVER (PARTITION BY a ORDER BY b ASC NULLS FIRST ROWS BETWEEN -1 FOLLOWING AND -1 FOLLOWING):int> |
+| org.apache.spark.sql.catalyst.expressions.LastDay | last_day | SELECT last_day('2009-01-12') | struct<last_day(2009-01-12):date> |
+| org.apache.spark.sql.catalyst.expressions.Lead | lead | SELECT a, b, lead(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b) | struct<a:string,b:int,lead(b, 1, NULL) OVER (PARTITION BY a ORDER BY b ASC NULLS FIRST ROWS BETWEEN 1 FOLLOWING AND 1 FOLLOWING):int> |
 | org.apache.spark.sql.catalyst.expressions.Least | least | SELECT least(10, 9, 2, 4, 3) | struct<least(10, 9, 2, 4, 3):int> |
 | org.apache.spark.sql.catalyst.expressions.Left | left | SELECT left('Spark SQL', 3) | struct<left(Spark SQL, 3):string> |
-| org.apache.spark.sql.catalyst.expressions.Length | character_length | SELECT character_length('Spark SQL ') | struct<character_length(Spark SQL ):int> |
 | org.apache.spark.sql.catalyst.expressions.Length | char_length | SELECT char_length('Spark SQL ') | struct<char_length(Spark SQL ):int> |
+| org.apache.spark.sql.catalyst.expressions.Length | character_length | SELECT character_length('Spark SQL ') | struct<character_length(Spark SQL ):int> |
 | org.apache.spark.sql.catalyst.expressions.Length | length | SELECT length('Spark SQL ') | struct<length(Spark SQL ):int> |
 | org.apache.spark.sql.catalyst.expressions.LengthOfJsonArray | json_array_length | SELECT json_array_length('[1,2,3,4]') | struct<json_array_length([1,2,3,4]):int> |
 | org.apache.spark.sql.catalyst.expressions.LessThan | < | SELECT 1 < 2 | struct<(1 < 2):boolean> |
 | org.apache.spark.sql.catalyst.expressions.LessThanOrEqual | <= | SELECT 2 <= 2 | struct<(2 <= 2):boolean> |
 | org.apache.spark.sql.catalyst.expressions.Levenshtein | levenshtein | SELECT levenshtein('kitten', 'sitting') | struct<levenshtein(kitten, sitting):int> |
 | org.apache.spark.sql.catalyst.expressions.Like | like | SELECT like('Spark', '_park') | struct<Spark LIKE _park:boolean> |
-| org.apache.spark.sql.catalyst.expressions.Log | ln | SELECT ln(1) | struct<LOG(CAST(1 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.Log10 | log10 | SELECT log10(10) | struct<LOG10(CAST(10 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.Log1p | log1p | SELECT log1p(0) | struct<LOG1P(CAST(0 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.Log2 | log2 | SELECT log2(2) | struct<LOG2(CAST(2 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.Logarithm | log | SELECT log(10, 100) | struct<LOG(CAST(10 AS DOUBLE), CAST(100 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.Lower | lcase | SELECT lcase('SparkSql') | struct<lower(SparkSql):string> |
+| org.apache.spark.sql.catalyst.expressions.Log | ln | SELECT ln(1) | struct<ln(1):double> |
+| org.apache.spark.sql.catalyst.expressions.Log10 | log10 | SELECT log10(10) | struct<LOG10(10):double> |
+| org.apache.spark.sql.catalyst.expressions.Log1p | log1p | SELECT log1p(0) | struct<LOG1P(0):double> |
+| org.apache.spark.sql.catalyst.expressions.Log2 | log2 | SELECT log2(2) | struct<LOG2(2):double> |
+| org.apache.spark.sql.catalyst.expressions.Logarithm | log | SELECT log(10, 100) | struct<LOG(10, 100):double> |
+| org.apache.spark.sql.catalyst.expressions.Lower | lcase | SELECT lcase('SparkSql') | struct<lcase(SparkSql):string> |
 | org.apache.spark.sql.catalyst.expressions.Lower | lower | SELECT lower('SparkSql') | struct<lower(SparkSql):string> |
 | org.apache.spark.sql.catalyst.expressions.MakeDate | make_date | SELECT make_date(2013, 7, 15) | struct<make_date(2013, 7, 15):date> |
-| org.apache.spark.sql.catalyst.expressions.MakeInterval | make_interval | SELECT make_interval(100, 11, 1, 1, 12, 30, 01.001001) | struct<make_interval(100, 11, 1, 1, 12, 30, CAST(1.001001 AS DECIMAL(8,6))):interval> |
-| org.apache.spark.sql.catalyst.expressions.MakeTimestamp | make_timestamp | SELECT make_timestamp(2014, 12, 28, 6, 30, 45.887) | struct<make_timestamp(2014, 12, 28, 6, 30, CAST(45.887 AS DECIMAL(8,6))):timestamp> |
+| org.apache.spark.sql.catalyst.expressions.MakeInterval | make_interval | SELECT make_interval(100, 11, 1, 1, 12, 30, 01.001001) | struct<make_interval(100, 11, 1, 1, 12, 30, 1.001001):interval> |
+| org.apache.spark.sql.catalyst.expressions.MakeTimestamp | make_timestamp | SELECT make_timestamp(2014, 12, 28, 6, 30, 45.887) | struct<make_timestamp(2014, 12, 28, 6, 30, 45.887):timestamp> |
 | org.apache.spark.sql.catalyst.expressions.MapConcat | map_concat | SELECT map_concat(map(1, 'a', 2, 'b'), map(3, 'c')) | struct<map_concat(map(1, a, 2, b), map(3, c)):map<int,string>> |
 | org.apache.spark.sql.catalyst.expressions.MapEntries | map_entries | SELECT map_entries(map(1, 'a', 2, 'b')) | struct<map_entries(map(1, a, 2, b)):array<struct<key:int,value:string>>> |
 | org.apache.spark.sql.catalyst.expressions.MapFilter | map_filter | SELECT map_filter(map(1, 0, 2, 2, 3, -1), (k, v) -> k > v) | struct<map_filter(map(1, 0, 2, 2, 3, -1), lambdafunction((namedlambdavariable() > namedlambdavariable()), namedlambdavariable(), namedlambdavariable())):map<int,int>> |
 | org.apache.spark.sql.catalyst.expressions.MapFromArrays | map_from_arrays | SELECT map_from_arrays(array(1.0, 3.0), array('2', '4')) | struct<map_from_arrays(array(1.0, 3.0), array(2, 4)):map<decimal(2,1),string>> |
-| org.apache.spark.sql.catalyst.expressions.MapFromEntries | map_from_entries | SELECT map_from_entries(array(struct(1, 'a'), struct(2, 'b'))) | struct<map_from_entries(array(named_struct(col1, 1, col2, a), named_struct(col1, 2, col2, b))):map<int,string>> |
+| org.apache.spark.sql.catalyst.expressions.MapFromEntries | map_from_entries | SELECT map_from_entries(array(struct(1, 'a'), struct(2, 'b'))) | struct<map_from_entries(array(struct(1, a), struct(2, b))):map<int,string>> |
 | org.apache.spark.sql.catalyst.expressions.MapKeys | map_keys | SELECT map_keys(map(1, 'a', 2, 'b')) | struct<map_keys(map(1, a, 2, b)):array<int>> |
 | org.apache.spark.sql.catalyst.expressions.MapValues | map_values | SELECT map_values(map(1, 'a', 2, 'b')) | struct<map_values(map(1, a, 2, b)):array<string>> |
 | org.apache.spark.sql.catalyst.expressions.MapZipWith | map_zip_with | SELECT map_zip_with(map(1, 'a', 2, 'b'), map(1, 'x', 2, 'y'), (k, v1, v2) -> concat(v1, v2)) | struct<map_zip_with(map(1, a, 2, b), map(1, x, 2, y), lambdafunction(concat(namedlambdavariable(), namedlambdavariable()), namedlambdavariable(), namedlambdavariable(), namedlambdavariable())):map<int,string>> |
-| org.apache.spark.sql.catalyst.expressions.Md5 | md5 | SELECT md5('Spark') | struct<md5(CAST(Spark AS BINARY)):string> |
-| org.apache.spark.sql.catalyst.expressions.Minute | minute | SELECT minute('2009-07-30 12:58:59') | struct<minute(CAST(2009-07-30 12:58:59 AS TIMESTAMP)):int> |
-| org.apache.spark.sql.catalyst.expressions.MonotonicallyIncreasingID | monotonically_increasing_id | N/A | N/A |
-| org.apache.spark.sql.catalyst.expressions.Month | month | SELECT month('2016-07-30') | struct<month(CAST(2016-07-30 AS DATE)):int> |
-| org.apache.spark.sql.catalyst.expressions.MonthsBetween | months_between | SELECT months_between('1997-02-28 10:30:00', '1996-10-30') | struct<months_between(CAST(1997-02-28 10:30:00 AS TIMESTAMP), CAST(1996-10-30 AS TIMESTAMP), true):double> |
+| org.apache.spark.sql.catalyst.expressions.Md5 | md5 | SELECT md5('Spark') | struct<md5(Spark):string> |
+| org.apache.spark.sql.catalyst.expressions.MicrosToTimestamp | timestamp_micros | SELECT timestamp_micros(1230219000123123) | struct<timestamp_micros(1230219000123123):timestamp> |
+| org.apache.spark.sql.catalyst.expressions.MillisToTimestamp | timestamp_millis | SELECT timestamp_millis(1230219000123) | struct<timestamp_millis(1230219000123):timestamp> |
+| org.apache.spark.sql.catalyst.expressions.Minute | minute | SELECT minute('2009-07-30 12:58:59') | struct<minute(2009-07-30 12:58:59):int> |
+| org.apache.spark.sql.catalyst.expressions.MonotonicallyIncreasingID | monotonically_increasing_id | SELECT monotonically_increasing_id() | struct<monotonically_increasing_id():bigint> |
+| org.apache.spark.sql.catalyst.expressions.Month | month | SELECT month('2016-07-30') | struct<month(2016-07-30):int> |
+| org.apache.spark.sql.catalyst.expressions.MonthsBetween | months_between | SELECT months_between('1997-02-28 10:30:00', '1996-10-30') | struct<months_between(1997-02-28 10:30:00, 1996-10-30, true):double> |
 | org.apache.spark.sql.catalyst.expressions.Multiply | * | SELECT 2 * 3 | struct<(2 * 3):int> |
 | org.apache.spark.sql.catalyst.expressions.Murmur3Hash | hash | SELECT hash('Spark', array(123), 2) | struct<hash(Spark, array(123), 2):int> |
-| org.apache.spark.sql.catalyst.expressions.NTile | ntile | N/A | N/A |
-| org.apache.spark.sql.catalyst.expressions.NaNvl | nanvl | SELECT nanvl(cast('NaN' as double), 123) | struct<nanvl(CAST(NaN AS DOUBLE), CAST(123 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.NamedStruct | struct | N/A | N/A |
-| org.apache.spark.sql.catalyst.expressions.NextDay | next_day | SELECT next_day('2015-01-14', 'TU') | struct<next_day(CAST(2015-01-14 AS DATE), TU):date> |
-| org.apache.spark.sql.catalyst.expressions.Not | ! | N/A | N/A |
-| org.apache.spark.sql.catalyst.expressions.Not | not | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.NTile | ntile | SELECT a, b, ntile(2) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b) | struct<a:string,b:int,ntile(2) OVER (PARTITION BY a ORDER BY b ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int> |
+| org.apache.spark.sql.catalyst.expressions.NaNvl | nanvl | SELECT nanvl(cast('NaN' as double), 123) | struct<nanvl(CAST(NaN AS DOUBLE), 123):double> |
+| org.apache.spark.sql.catalyst.expressions.NextDay | next_day | SELECT next_day('2015-01-14', 'TU') | struct<next_day(2015-01-14, TU):date> |
+| org.apache.spark.sql.catalyst.expressions.Not | ! | SELECT ! true | struct<(NOT true):boolean> |
+| org.apache.spark.sql.catalyst.expressions.Not | not | SELECT not true | struct<(NOT true):boolean> |
 | org.apache.spark.sql.catalyst.expressions.Now | now | SELECT now() | struct<now():timestamp> |
+| org.apache.spark.sql.catalyst.expressions.NthValue | nth_value | SELECT a, b, nth_value(b, 2) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b) | struct<a:string,b:int,nth_value(b, 2) OVER (PARTITION BY a ORDER BY b ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int> |
 | org.apache.spark.sql.catalyst.expressions.NullIf | nullif | SELECT nullif(2, 2) | struct<nullif(2, 2):int> |
 | org.apache.spark.sql.catalyst.expressions.Nvl | nvl | SELECT nvl(NULL, array('2')) | struct<nvl(NULL, array(2)):array<string>> |
 | org.apache.spark.sql.catalyst.expressions.Nvl2 | nvl2 | SELECT nvl2(NULL, 2, 1) | struct<nvl2(NULL, 2, 1):int> |
 | org.apache.spark.sql.catalyst.expressions.OctetLength | octet_length | SELECT octet_length('Spark SQL') | struct<octet_length(Spark SQL):int> |
-| org.apache.spark.sql.catalyst.expressions.Or | or | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Or | or | SELECT true or false | struct<(true OR false):boolean> |
 | org.apache.spark.sql.catalyst.expressions.Overlay | overlay | SELECT overlay('Spark SQL' PLACING '_' FROM 6) | struct<overlay(Spark SQL, _, 6, -1):string> |
 | org.apache.spark.sql.catalyst.expressions.ParseToDate | to_date | SELECT to_date('2009-07-30 04:17:52') | struct<to_date(2009-07-30 04:17:52):date> |
 | org.apache.spark.sql.catalyst.expressions.ParseToTimestamp | to_timestamp | SELECT to_timestamp('2016-12-31 00:12:00') | struct<to_timestamp(2016-12-31 00:12:00):timestamp> |
 | org.apache.spark.sql.catalyst.expressions.ParseUrl | parse_url | SELECT parse_url('http://spark.apache.org/path?query=1', 'HOST') | struct<parse_url(http://spark.apache.org/path?query=1, HOST):string> |
-| org.apache.spark.sql.catalyst.expressions.PercentRank | percent_rank | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.PercentRank | percent_rank | SELECT a, b, percent_rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b) | struct<a:string,b:int,PERCENT_RANK() OVER (PARTITION BY a ORDER BY b ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):double> |
 | org.apache.spark.sql.catalyst.expressions.Pi | pi | SELECT pi() | struct<PI():double> |
 | org.apache.spark.sql.catalyst.expressions.Pmod | pmod | SELECT pmod(10, 3) | struct<pmod(10, 3):int> |
-| org.apache.spark.sql.catalyst.expressions.PosExplode | posexplode_outer | SELECT posexplode_outer(array(10,20)) | struct<pos:int,col:int> |
 | org.apache.spark.sql.catalyst.expressions.PosExplode | posexplode | SELECT posexplode(array(10,20)) | struct<pos:int,col:int> |
-| org.apache.spark.sql.catalyst.expressions.Pow | pow | SELECT pow(2, 3) | struct<pow(CAST(2 AS DOUBLE), CAST(3 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.Pow | power | SELECT power(2, 3) | struct<POWER(CAST(2 AS DOUBLE), CAST(3 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.Quarter | quarter | SELECT quarter('2016-08-31') | struct<quarter(CAST(2016-08-31 AS DATE)):int> |
-| org.apache.spark.sql.catalyst.expressions.RLike | rlike | SELECT '%SystemDrive%\Users\John' rlike '%SystemDrive%\\Users.*' | struct<%SystemDrive%UsersJohn RLIKE %SystemDrive%\Users.*:boolean> |
-| org.apache.spark.sql.catalyst.expressions.Rand | random | SELECT random() | struct<rand():double> |
+| org.apache.spark.sql.catalyst.expressions.PosExplode | posexplode_outer | SELECT posexplode_outer(array(10,20)) | struct<pos:int,col:int> |
+| org.apache.spark.sql.catalyst.expressions.Pow | pow | SELECT pow(2, 3) | struct<pow(2, 3):double> |
+| org.apache.spark.sql.catalyst.expressions.Pow | power | SELECT power(2, 3) | struct<POWER(2, 3):double> |
+| org.apache.spark.sql.catalyst.expressions.Quarter | quarter | SELECT quarter('2016-08-31') | struct<quarter(2016-08-31):int> |
+| org.apache.spark.sql.catalyst.expressions.RLike | regexp | SELECT regexp('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*') | struct<REGEXP(%SystemDrive%UsersJohn, %SystemDrive%\Users.*):boolean> |
+| org.apache.spark.sql.catalyst.expressions.RLike | regexp_like | SELECT regexp_like('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*') | struct<REGEXP_LIKE(%SystemDrive%UsersJohn, %SystemDrive%\Users.*):boolean> |
+| org.apache.spark.sql.catalyst.expressions.RLike | rlike | SELECT rlike('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*') | struct<RLIKE(%SystemDrive%UsersJohn, %SystemDrive%\Users.*):boolean> |
+| org.apache.spark.sql.catalyst.expressions.RaiseError | raise_error | SELECT raise_error('custom error message') | struct<raise_error(custom error message):null> |
 | org.apache.spark.sql.catalyst.expressions.Rand | rand | SELECT rand() | struct<rand():double> |
+| org.apache.spark.sql.catalyst.expressions.Rand | random | SELECT random() | struct<rand():double> |
 | org.apache.spark.sql.catalyst.expressions.Randn | randn | SELECT randn() | struct<randn():double> |
-| org.apache.spark.sql.catalyst.expressions.Rank | rank | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Rank | rank | SELECT a, b, rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b) | struct<a:string,b:int,RANK() OVER (PARTITION BY a ORDER BY b ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int> |
 | org.apache.spark.sql.catalyst.expressions.RegExpExtract | regexp_extract | SELECT regexp_extract('100-200', '(\\d+)-(\\d+)', 1) | struct<regexp_extract(100-200, (\d+)-(\d+), 1):string> |
-| org.apache.spark.sql.catalyst.expressions.RegExpReplace | regexp_replace | SELECT regexp_replace('100-200', '(\\d+)', 'num') | struct<regexp_replace(100-200, (\d+), num):string> |
-| org.apache.spark.sql.catalyst.expressions.Remainder | % | SELECT 2 % 1.8 | struct<(CAST(CAST(2 AS DECIMAL(1,0)) AS DECIMAL(2,1)) % CAST(1.8 AS DECIMAL(2,1))):decimal(2,1)> |
-| org.apache.spark.sql.catalyst.expressions.Remainder | mod | SELECT 2 % 1.8 | struct<(CAST(CAST(2 AS DECIMAL(1,0)) AS DECIMAL(2,1)) % CAST(1.8 AS DECIMAL(2,1))):decimal(2,1)> |
+| org.apache.spark.sql.catalyst.expressions.RegExpExtractAll | regexp_extract_all | SELECT regexp_extract_all('100-200, 300-400', '(\\d+)-(\\d+)', 1) | struct<regexp_extract_all(100-200, 300-400, (\d+)-(\d+), 1):array<string>> |
+| org.apache.spark.sql.catalyst.expressions.RegExpReplace | regexp_replace | SELECT regexp_replace('100-200', '(\\d+)', 'num') | struct<regexp_replace(100-200, (\d+), num, 1):string> |
+| org.apache.spark.sql.catalyst.expressions.Remainder | % | SELECT 2 % 1.8 | struct<(2 % 1.8):decimal(2,1)> |
+| org.apache.spark.sql.catalyst.expressions.Remainder | mod | SELECT 2 % 1.8 | struct<(2 % 1.8):decimal(2,1)> |
 | org.apache.spark.sql.catalyst.expressions.Reverse | reverse | SELECT reverse('Spark SQL') | struct<reverse(Spark SQL):string> |
 | org.apache.spark.sql.catalyst.expressions.Right | right | SELECT right('Spark SQL', 3) | struct<right(Spark SQL, 3):string> |
-| org.apache.spark.sql.catalyst.expressions.Rint | rint | SELECT rint(12.3456) | struct<ROUND(CAST(12.3456 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.Rollup | rollup | SELECT name, age, count(*) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY rollup(name, age) | struct<name:string,age:int,count(1):bigint> |
+| org.apache.spark.sql.catalyst.expressions.Rint | rint | SELECT rint(12.3456) | struct<rint(12.3456):double> |
 | org.apache.spark.sql.catalyst.expressions.Round | round | SELECT round(2.5, 0) | struct<round(2.5, 0):decimal(2,0)> |
-| org.apache.spark.sql.catalyst.expressions.RowNumber | row_number | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.RowNumber | row_number | SELECT a, b, row_number() OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b) | struct<a:string,b:int,row_number() OVER (PARTITION BY a ORDER BY b ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int> |
 | org.apache.spark.sql.catalyst.expressions.SchemaOfCsv | schema_of_csv | SELECT schema_of_csv('1,abc') | struct<schema_of_csv(1,abc):string> |
 | org.apache.spark.sql.catalyst.expressions.SchemaOfJson | schema_of_json | SELECT schema_of_json('[{"col":0}]') | struct<schema_of_json([{"col":0}]):string> |
-| org.apache.spark.sql.catalyst.expressions.Second | second | SELECT second('2009-07-30 12:58:59') | struct<second(CAST(2009-07-30 12:58:59 AS TIMESTAMP)):int> |
+| org.apache.spark.sql.catalyst.expressions.Second | second | SELECT second('2009-07-30 12:58:59') | struct<second(2009-07-30 12:58:59):int> |
+| org.apache.spark.sql.catalyst.expressions.SecondsToTimestamp | timestamp_seconds | SELECT timestamp_seconds(1230219000) | struct<timestamp_seconds(1230219000):timestamp> |
 | org.apache.spark.sql.catalyst.expressions.Sentences | sentences | SELECT sentences('Hi there! Good morning.') | struct<sentences(Hi there! Good morning., , ):array<array<string>>> |
 | org.apache.spark.sql.catalyst.expressions.Sequence | sequence | SELECT sequence(1, 5) | struct<sequence(1, 5):array<int>> |
-| org.apache.spark.sql.catalyst.expressions.Sha1 | sha1 | SELECT sha1('Spark') | struct<sha1(CAST(Spark AS BINARY)):string> |
-| org.apache.spark.sql.catalyst.expressions.Sha1 | sha | SELECT sha('Spark') | struct<sha(CAST(Spark AS BINARY)):string> |
-| org.apache.spark.sql.catalyst.expressions.Sha2 | sha2 | SELECT sha2('Spark', 256) | struct<sha2(CAST(Spark AS BINARY), 256):string> |
+| org.apache.spark.sql.catalyst.expressions.Sha1 | sha | SELECT sha('Spark') | struct<sha(Spark):string> |
+| org.apache.spark.sql.catalyst.expressions.Sha1 | sha1 | SELECT sha1('Spark') | struct<sha1(Spark):string> |
+| org.apache.spark.sql.catalyst.expressions.Sha2 | sha2 | SELECT sha2('Spark', 256) | struct<sha2(Spark, 256):string> |
 | org.apache.spark.sql.catalyst.expressions.ShiftLeft | shiftleft | SELECT shiftleft(2, 1) | struct<shiftleft(2, 1):int> |
 | org.apache.spark.sql.catalyst.expressions.ShiftRight | shiftright | SELECT shiftright(4, 1) | struct<shiftright(4, 1):int> |
 | org.apache.spark.sql.catalyst.expressions.ShiftRightUnsigned | shiftrightunsigned | SELECT shiftrightunsigned(4, 1) | struct<shiftrightunsigned(4, 1):int> |
 | org.apache.spark.sql.catalyst.expressions.Shuffle | shuffle | SELECT shuffle(array(1, 20, 3, 5)) | struct<shuffle(array(1, 20, 3, 5)):array<int>> |
-| org.apache.spark.sql.catalyst.expressions.Signum | signum | SELECT signum(40) | struct<SIGNUM(CAST(40 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.Signum | sign | SELECT sign(40) | struct<sign(CAST(40 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.Sin | sin | SELECT sin(0) | struct<SIN(CAST(0 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.Sinh | sinh | SELECT sinh(0) | struct<SINH(CAST(0 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.Size | size | SELECT size(array('b', 'd', 'c', 'a')) | struct<size(array(b, d, c, a)):int> |
+| org.apache.spark.sql.catalyst.expressions.Signum | sign | SELECT sign(40) | struct<sign(40):double> |
+| org.apache.spark.sql.catalyst.expressions.Signum | signum | SELECT signum(40) | struct<SIGNUM(40):double> |
+| org.apache.spark.sql.catalyst.expressions.Sin | sin | SELECT sin(0) | struct<SIN(0):double> |
+| org.apache.spark.sql.catalyst.expressions.Sinh | sinh | SELECT sinh(0) | struct<SINH(0):double> |
 | org.apache.spark.sql.catalyst.expressions.Size | cardinality | SELECT cardinality(array('b', 'd', 'c', 'a')) | struct<cardinality(array(b, d, c, a)):int> |
+| org.apache.spark.sql.catalyst.expressions.Size | size | SELECT size(array('b', 'd', 'c', 'a')) | struct<size(array(b, d, c, a)):int> |
 | org.apache.spark.sql.catalyst.expressions.Slice | slice | SELECT slice(array(1, 2, 3, 4), 2, 2) | struct<slice(array(1, 2, 3, 4), 2, 2):array<int>> |
-| org.apache.spark.sql.catalyst.expressions.SortArray | sort_array | SELECT sort_array(array('b', 'd', null, 'c', 'a'), true) | struct<sort_array(array(b, d, CAST(NULL AS STRING), c, a), true):array<string>> |
+| org.apache.spark.sql.catalyst.expressions.SortArray | sort_array | SELECT sort_array(array('b', 'd', null, 'c', 'a'), true) | struct<sort_array(array(b, d, NULL, c, a), true):array<string>> |
 | org.apache.spark.sql.catalyst.expressions.SoundEx | soundex | SELECT soundex('Miller') | struct<soundex(Miller):string> |
-| org.apache.spark.sql.catalyst.expressions.SparkPartitionID | spark_partition_id | N/A | N/A |
-| org.apache.spark.sql.catalyst.expressions.SparkVersion | version | N/A | N/A |
-| org.apache.spark.sql.catalyst.expressions.Sqrt | sqrt | SELECT sqrt(4) | struct<SQRT(CAST(4 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.SparkPartitionID | spark_partition_id | SELECT spark_partition_id() | struct<SPARK_PARTITION_ID():int> |
+| org.apache.spark.sql.catalyst.expressions.SparkVersion | version | SELECT version() | struct<version():string> |
+| org.apache.spark.sql.catalyst.expressions.Sqrt | sqrt | SELECT sqrt(4) | struct<SQRT(4):double> |
 | org.apache.spark.sql.catalyst.expressions.Stack | stack | SELECT stack(2, 1, 2, 3) | struct<col0:int,col1:int> |
 | org.apache.spark.sql.catalyst.expressions.StringInstr | instr | SELECT instr('SparkSQL', 'SQL') | struct<instr(SparkSQL, SQL):int> |
 | org.apache.spark.sql.catalyst.expressions.StringLPad | lpad | SELECT lpad('hi', 5, '??') | struct<lpad(hi, 5, ??):string> |
-| org.apache.spark.sql.catalyst.expressions.StringLocate | position | SELECT position('bar', 'foobarbar') | struct<locate(bar, foobarbar, 1):int> |
 | org.apache.spark.sql.catalyst.expressions.StringLocate | locate | SELECT locate('bar', 'foobarbar') | struct<locate(bar, foobarbar, 1):int> |
+| org.apache.spark.sql.catalyst.expressions.StringLocate | position | SELECT position('bar', 'foobarbar') | struct<position(bar, foobarbar, 1):int> |
 | org.apache.spark.sql.catalyst.expressions.StringRPad | rpad | SELECT rpad('hi', 5, '??') | struct<rpad(hi, 5, ??):string> |
 | org.apache.spark.sql.catalyst.expressions.StringRepeat | repeat | SELECT repeat('123', 2) | struct<repeat(123, 2):string> |
 | org.apache.spark.sql.catalyst.expressions.StringReplace | replace | SELECT replace('ABCabc', 'abc', 'DEF') | struct<replace(ABCabc, abc, DEF):string> |
@@ -257,6 +268,7 @@
 | org.apache.spark.sql.catalyst.expressions.StringToMap | str_to_map | SELECT str_to_map('a:1,b:2,c:3', ',', ':') | struct<str_to_map(a:1,b:2,c:3, ,, :):map<string,string>> |
 | org.apache.spark.sql.catalyst.expressions.StringTranslate | translate | SELECT translate('AaBbCc', 'abc', '123') | struct<translate(AaBbCc, abc, 123):string> |
 | org.apache.spark.sql.catalyst.expressions.StringTrim | trim | SELECT trim('    SparkSQL   ') | struct<trim(    SparkSQL   ):string> |
+| org.apache.spark.sql.catalyst.expressions.StringTrimBoth | btrim | SELECT btrim('    SparkSQL   ') | struct<btrim(    SparkSQL   ):string> |
 | org.apache.spark.sql.catalyst.expressions.StringTrimLeft | ltrim | SELECT ltrim('    SparkSQL   ') | struct<ltrim(    SparkSQL   ):string> |
 | org.apache.spark.sql.catalyst.expressions.StringTrimRight | rtrim | SELECT rtrim('    SparkSQL   ') | struct<rtrim(    SparkSQL   ):string> |
 | org.apache.spark.sql.catalyst.expressions.StructsToCsv | to_csv | SELECT to_csv(named_struct('a', 1, 'b', 2)) | struct<to_csv(named_struct(a, 1, b, 2)):string> |
@@ -265,74 +277,81 @@
 | org.apache.spark.sql.catalyst.expressions.Substring | substring | SELECT substring('Spark SQL', 5) | struct<substring(Spark SQL, 5, 2147483647):string> |
 | org.apache.spark.sql.catalyst.expressions.SubstringIndex | substring_index | SELECT substring_index('www.apache.org', '.', 2) | struct<substring_index(www.apache.org, ., 2):string> |
 | org.apache.spark.sql.catalyst.expressions.Subtract | - | SELECT 2 - 1 | struct<(2 - 1):int> |
-| org.apache.spark.sql.catalyst.expressions.Tan | tan | SELECT tan(0) | struct<TAN(CAST(0 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.Tanh | tanh | SELECT tanh(0) | struct<TANH(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Tan | tan | SELECT tan(0) | struct<TAN(0):double> |
+| org.apache.spark.sql.catalyst.expressions.Tanh | tanh | SELECT tanh(0) | struct<TANH(0):double> |
 | org.apache.spark.sql.catalyst.expressions.TimeWindow | window | N/A | N/A |
-| org.apache.spark.sql.catalyst.expressions.ToDegrees | degrees | SELECT degrees(3.141592653589793) | struct<DEGREES(CAST(3.141592653589793 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.ToRadians | radians | SELECT radians(180) | struct<RADIANS(CAST(180 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.ToUTCTimestamp | to_utc_timestamp | SELECT to_utc_timestamp('2016-08-31', 'Asia/Seoul') | struct<to_utc_timestamp(CAST(2016-08-31 AS TIMESTAMP), Asia/Seoul):timestamp> |
+| org.apache.spark.sql.catalyst.expressions.ToDegrees | degrees | SELECT degrees(3.141592653589793) | struct<DEGREES(3.141592653589793):double> |
+| org.apache.spark.sql.catalyst.expressions.ToRadians | radians | SELECT radians(180) | struct<RADIANS(180):double> |
+| org.apache.spark.sql.catalyst.expressions.ToUTCTimestamp | to_utc_timestamp | SELECT to_utc_timestamp('2016-08-31', 'Asia/Seoul') | struct<to_utc_timestamp(2016-08-31, Asia/Seoul):timestamp> |
 | org.apache.spark.sql.catalyst.expressions.ToUnixTimestamp | to_unix_timestamp | SELECT to_unix_timestamp('2016-04-08', 'yyyy-MM-dd') | struct<to_unix_timestamp(2016-04-08, yyyy-MM-dd):bigint> |
 | org.apache.spark.sql.catalyst.expressions.TransformKeys | transform_keys | SELECT transform_keys(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + 1) | struct<transform_keys(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), lambdafunction((namedlambdavariable() + 1), namedlambdavariable(), namedlambdavariable())):map<int,int>> |
 | org.apache.spark.sql.catalyst.expressions.TransformValues | transform_values | SELECT transform_values(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> v + 1) | struct<transform_values(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), lambdafunction((namedlambdavariable() + 1), namedlambdavariable(), namedlambdavariable())):map<int,int>> |
-| org.apache.spark.sql.catalyst.expressions.TruncDate | trunc | SELECT trunc('2019-08-04', 'week') | struct<trunc(CAST(2019-08-04 AS DATE), week):date> |
-| org.apache.spark.sql.catalyst.expressions.TruncTimestamp | date_trunc | SELECT date_trunc('YEAR', '2015-03-05T09:32:05.359') | struct<date_trunc(YEAR, CAST(2015-03-05T09:32:05.359 AS TIMESTAMP)):timestamp> |
+| org.apache.spark.sql.catalyst.expressions.TruncDate | trunc | SELECT trunc('2019-08-04', 'week') | struct<trunc(2019-08-04, week):date> |
+| org.apache.spark.sql.catalyst.expressions.TruncTimestamp | date_trunc | SELECT date_trunc('YEAR', '2015-03-05T09:32:05.359') | struct<date_trunc(YEAR, 2015-03-05T09:32:05.359):timestamp> |
+| org.apache.spark.sql.catalyst.expressions.TryAdd | try_add | SELECT try_add(1, 2) | struct<try_add(1, 2):int> |
+| org.apache.spark.sql.catalyst.expressions.TryDivide | try_divide | SELECT try_divide(3, 2) | struct<try_divide(3, 2):double> |
 | org.apache.spark.sql.catalyst.expressions.TypeOf | typeof | SELECT typeof(1) | struct<typeof(1):string> |
 | org.apache.spark.sql.catalyst.expressions.UnBase64 | unbase64 | SELECT unbase64('U3BhcmsgU1FM') | struct<unbase64(U3BhcmsgU1FM):binary> |
 | org.apache.spark.sql.catalyst.expressions.UnaryMinus | negative | SELECT negative(1) | struct<negative(1):int> |
-| org.apache.spark.sql.catalyst.expressions.UnaryPositive | positive | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.UnaryPositive | positive | SELECT positive(1) | struct<(+ 1):int> |
 | org.apache.spark.sql.catalyst.expressions.Unhex | unhex | SELECT decode(unhex('537061726B2053514C'), 'UTF-8') | struct<decode(unhex(537061726B2053514C), UTF-8):string> |
+| org.apache.spark.sql.catalyst.expressions.UnixDate | unix_date | SELECT unix_date(DATE("1970-01-02")) | struct<unix_date(1970-01-02):int> |
+| org.apache.spark.sql.catalyst.expressions.UnixMicros | unix_micros | SELECT unix_micros(TIMESTAMP('1970-01-01 00:00:01Z')) | struct<unix_micros(1970-01-01 00:00:01Z):bigint> |
+| org.apache.spark.sql.catalyst.expressions.UnixMillis | unix_millis | SELECT unix_millis(TIMESTAMP('1970-01-01 00:00:01Z')) | struct<unix_millis(1970-01-01 00:00:01Z):bigint> |
+| org.apache.spark.sql.catalyst.expressions.UnixSeconds | unix_seconds | SELECT unix_seconds(TIMESTAMP('1970-01-01 00:00:01Z')) | struct<unix_seconds(1970-01-01 00:00:01Z):bigint> |
 | org.apache.spark.sql.catalyst.expressions.UnixTimestamp | unix_timestamp | SELECT unix_timestamp() | struct<unix_timestamp(current_timestamp(), yyyy-MM-dd HH:mm:ss):bigint> |
 | org.apache.spark.sql.catalyst.expressions.Upper | ucase | SELECT ucase('SparkSql') | struct<ucase(SparkSql):string> |
 | org.apache.spark.sql.catalyst.expressions.Upper | upper | SELECT upper('SparkSql') | struct<upper(SparkSql):string> |
 | org.apache.spark.sql.catalyst.expressions.Uuid | uuid | SELECT uuid() | struct<uuid():string> |
-| org.apache.spark.sql.catalyst.expressions.WeekDay | weekday | SELECT weekday('2009-07-30') | struct<weekday(CAST(2009-07-30 AS DATE)):int> |
-| org.apache.spark.sql.catalyst.expressions.WeekOfYear | weekofyear | SELECT weekofyear('2008-02-20') | struct<weekofyear(CAST(2008-02-20 AS DATE)):int> |
+| org.apache.spark.sql.catalyst.expressions.WeekDay | weekday | SELECT weekday('2009-07-30') | struct<weekday(2009-07-30):int> |
+| org.apache.spark.sql.catalyst.expressions.WeekOfYear | weekofyear | SELECT weekofyear('2008-02-20') | struct<weekofyear(2008-02-20):int> |
+| org.apache.spark.sql.catalyst.expressions.WidthBucket | width_bucket | SELECT width_bucket(5.3, 0.2, 10.6, 5) | struct<width_bucket(5.3, 0.2, 10.6, 5):bigint> |
 | org.apache.spark.sql.catalyst.expressions.XxHash64 | xxhash64 | SELECT xxhash64('Spark', array(123), 2) | struct<xxhash64(Spark, array(123), 2):bigint> |
-| org.apache.spark.sql.catalyst.expressions.Year | year | SELECT year('2016-07-30') | struct<year(CAST(2016-07-30 AS DATE)):int> |
+| org.apache.spark.sql.catalyst.expressions.Year | year | SELECT year('2016-07-30') | struct<year(2016-07-30):int> |
 | org.apache.spark.sql.catalyst.expressions.ZipWith | zip_with | SELECT zip_with(array(1, 2, 3), array('a', 'b', 'c'), (x, y) -> (y, x)) | struct<zip_with(array(1, 2, 3), array(a, b, c), lambdafunction(named_struct(y, namedlambdavariable(), x, namedlambdavariable()), namedlambdavariable(), namedlambdavariable())):array<struct<y:string,x:int>>> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.ApproximatePercentile | approx_percentile | SELECT approx_percentile(10.0, array(0.5, 0.4, 0.1), 100) | struct<approx_percentile(10.0, array(0.5, 0.4, 0.1), 100):array<decimal(3,1)>> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.ApproximatePercentile | percentile_approx | SELECT percentile_approx(10.0, array(0.5, 0.4, 0.1), 100) | struct<percentile_approx(10.0, array(0.5, 0.4, 0.1), 100):array<decimal(3,1)>> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.ApproximatePercentile | approx_percentile | SELECT approx_percentile(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col) | struct<approx_percentile(col, array(0.5, 0.4, 0.1), 100):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.ApproximatePercentile | percentile_approx | SELECT percentile_approx(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col) | struct<percentile_approx(col, array(0.5, 0.4, 0.1), 100):array<int>> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.Average | avg | SELECT avg(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<avg(col):double> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.Average | mean | SELECT mean(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<mean(col):double> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.BitAndAgg | bit_and | SELECT bit_and(col) FROM VALUES (3), (5) AS tab(col) | struct<bit_and(col):int> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.BitOrAgg | bit_or | SELECT bit_or(col) FROM VALUES (3), (5) AS tab(col) | struct<bit_or(col):int> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.BitXorAgg | bit_xor | SELECT bit_xor(col) FROM VALUES (3), (5) AS tab(col) | struct<bit_xor(col):int> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.BoolAnd | every | SELECT every(col) FROM VALUES (true), (true), (true) AS tab(col) | struct<every(col):boolean> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.BoolAnd | bool_and | SELECT bool_and(col) FROM VALUES (true), (true), (true) AS tab(col) | struct<bool_and(col):boolean> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.BoolAnd | every | SELECT every(col) FROM VALUES (true), (true), (true) AS tab(col) | struct<every(col):boolean> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.BoolOr | any | SELECT any(col) FROM VALUES (true), (false), (false) AS tab(col) | struct<any(col):boolean> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.BoolOr | bool_or | SELECT bool_or(col) FROM VALUES (true), (false), (false) AS tab(col) | struct<bool_or(col):boolean> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.BoolOr | some | SELECT some(col) FROM VALUES (true), (false), (false) AS tab(col) | struct<some(col):boolean> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.BoolOr | any | SELECT any(col) FROM VALUES (true), (false), (false) AS tab(col) | struct<any(col):boolean> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.CollectList | collect_list | SELECT collect_list(col) FROM VALUES (1), (2), (1) AS tab(col) | struct<collect_list(col):array<int>> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.CollectSet | collect_set | SELECT collect_set(col) FROM VALUES (1), (2), (1) AS tab(col) | struct<collect_set(col):array<int>> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.Corr | corr | SELECT corr(c1, c2) FROM VALUES (3, 2), (3, 3), (6, 4) as tab(c1, c2) | struct<corr(CAST(c1 AS DOUBLE), CAST(c2 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Corr | corr | SELECT corr(c1, c2) FROM VALUES (3, 2), (3, 3), (6, 4) as tab(c1, c2) | struct<corr(c1, c2):double> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.Count | count | SELECT count(*) FROM VALUES (NULL), (5), (5), (20) AS tab(col) | struct<count(1):bigint> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.CountIf | count_if | SELECT count_if(col % 2 = 0) FROM VALUES (NULL), (0), (1), (2), (3) AS tab(col) | struct<count_if(((col % 2) = 0)):bigint> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.CountMinSketchAgg | count_min_sketch | N/A | N/A |
-| org.apache.spark.sql.catalyst.expressions.aggregate.CovPopulation | covar_pop | SELECT covar_pop(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2) | struct<covar_pop(CAST(c1 AS DOUBLE), CAST(c2 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.CovSample | covar_samp | SELECT covar_samp(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2) | struct<covar_samp(CAST(c1 AS DOUBLE), CAST(c2 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.First | first_value | SELECT first_value(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<first_value(col, false):int> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.First | first | SELECT first(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<first(col, false):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.CountMinSketchAgg | count_min_sketch | SELECT hex(count_min_sketch(col, 0.5d, 0.5d, 1)) FROM VALUES (1), (2), (1) AS tab(col) | struct<hex(count_min_sketch(col, 0.5, 0.5, 1)):string> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.CovPopulation | covar_pop | SELECT covar_pop(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2) | struct<covar_pop(c1, c2):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.CovSample | covar_samp | SELECT covar_samp(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2) | struct<covar_samp(c1, c2):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.First | first | SELECT first(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<first(col):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.First | first_value | SELECT first_value(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<first_value(col):int> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.HyperLogLogPlusPlus | approx_count_distinct | SELECT approx_count_distinct(col1) FROM VALUES (1), (1), (2), (2), (3) tab(col1) | struct<approx_count_distinct(col1):bigint> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.Kurtosis | kurtosis | SELECT kurtosis(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col) | struct<kurtosis(CAST(col AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.Last | last_value | SELECT last_value(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<last_value(col, false):int> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.Last | last | SELECT last(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<last(col, false):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Kurtosis | kurtosis | SELECT kurtosis(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col) | struct<kurtosis(col):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Last | last | SELECT last(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<last(col):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Last | last_value | SELECT last_value(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<last_value(col):int> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.Max | max | SELECT max(col) FROM VALUES (10), (50), (20) AS tab(col) | struct<max(col):int> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.MaxBy | max_by | SELECT max_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y) | struct<max_by(x, y):string> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.Min | min | SELECT min(col) FROM VALUES (10), (-1), (20) AS tab(col) | struct<min(col):int> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.MinBy | min_by | SELECT min_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y) | struct<min_by(x, y):string> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.Percentile | percentile | SELECT percentile(col, 0.3) FROM VALUES (0), (10) AS tab(col) | struct<percentile(col, CAST(0.3 AS DOUBLE), 1):double> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.Skewness | skewness | SELECT skewness(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col) | struct<skewness(CAST(col AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.StddevPop | stddev_pop | SELECT stddev_pop(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<stddev_pop(CAST(col AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.StddevSamp | stddev_samp | SELECT stddev_samp(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<stddev_samp(CAST(col AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.StddevSamp | stddev | SELECT stddev(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<stddev(CAST(col AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.StddevSamp | std | SELECT std(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<std(CAST(col AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Percentile | percentile | SELECT percentile(col, 0.3) FROM VALUES (0), (10) AS tab(col) | struct<percentile(col, 0.3, 1):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Skewness | skewness | SELECT skewness(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col) | struct<skewness(col):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.StddevPop | stddev_pop | SELECT stddev_pop(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<stddev_pop(col):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.StddevSamp | std | SELECT std(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<std(col):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.StddevSamp | stddev | SELECT stddev(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<stddev(col):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.StddevSamp | stddev_samp | SELECT stddev_samp(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<stddev_samp(col):double> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.Sum | sum | SELECT sum(col) FROM VALUES (5), (10), (15) AS tab(col) | struct<sum(col):bigint> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.VariancePop | var_pop | SELECT var_pop(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<var_pop(CAST(col AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.VarianceSamp | var_samp | SELECT var_samp(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<var_samp(CAST(col AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.VarianceSamp | variance | SELECT variance(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<variance(CAST(col AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.VariancePop | var_pop | SELECT var_pop(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<var_pop(col):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.VarianceSamp | var_samp | SELECT var_samp(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<var_samp(col):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.VarianceSamp | variance | SELECT variance(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<variance(col):double> |
 | org.apache.spark.sql.catalyst.expressions.xml.XPathBoolean | xpath_boolean | SELECT xpath_boolean('<a><b>1</b></a>','a/b') | struct<xpath_boolean(<a><b>1</b></a>, a/b):boolean> |
-| org.apache.spark.sql.catalyst.expressions.xml.XPathDouble | xpath_number | SELECT xpath_number('<a><b>1</b><b>2</b></a>', 'sum(a/b)') | struct<xpath_number(<a><b>1</b><b>2</b></a>, sum(a/b)):double> |
 | org.apache.spark.sql.catalyst.expressions.xml.XPathDouble | xpath_double | SELECT xpath_double('<a><b>1</b><b>2</b></a>', 'sum(a/b)') | struct<xpath_double(<a><b>1</b><b>2</b></a>, sum(a/b)):double> |
+| org.apache.spark.sql.catalyst.expressions.xml.XPathDouble | xpath_number | SELECT xpath_number('<a><b>1</b><b>2</b></a>', 'sum(a/b)') | struct<xpath_number(<a><b>1</b><b>2</b></a>, sum(a/b)):double> |
 | org.apache.spark.sql.catalyst.expressions.xml.XPathFloat | xpath_float | SELECT xpath_float('<a><b>1</b><b>2</b></a>', 'sum(a/b)') | struct<xpath_float(<a><b>1</b><b>2</b></a>, sum(a/b)):float> |
 | org.apache.spark.sql.catalyst.expressions.xml.XPathInt | xpath_int | SELECT xpath_int('<a><b>1</b><b>2</b></a>', 'sum(a/b)') | struct<xpath_int(<a><b>1</b><b>2</b></a>, sum(a/b)):int> |
 | org.apache.spark.sql.catalyst.expressions.xml.XPathList | xpath | SELECT xpath('<a><b>b1</b><b>b2</b><b>b3</b><c>c1</c><c>c2</c></a>','a/b/text()') | struct<xpath(<a><b>b1</b><b>b2</b><b>b3</b><c>c1</c><c>c2</c></a>, a/b/text()):array<string>> |
diff --git a/sql/core/src/test/resources/sql-tests/inputs/ansi/array.sql b/sql/core/src/test/resources/sql-tests/inputs/ansi/array.sql
new file mode 100644
index 0000000000000..662756cbfb0b0
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/ansi/array.sql
@@ -0,0 +1 @@
+--IMPORT array.sql
diff --git a/sql/core/src/test/resources/sql-tests/inputs/ansi/decimalArithmeticOperations.sql b/sql/core/src/test/resources/sql-tests/inputs/ansi/decimalArithmeticOperations.sql
index d190f38345d6b..d843847e6a149 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/ansi/decimalArithmeticOperations.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/ansi/decimalArithmeticOperations.sql
@@ -1,6 +1,6 @@
 -- SPARK-23179: SQL ANSI 2011 states that in case of overflow during arithmetic operations,
 -- an exception should be thrown instead of returning NULL.
--- This is what most of the SQL DBs do (eg. SQLServer, DB2).
+-- This is what most of the SQL DBs do (e.g. SQLServer, DB2).
 
 -- tests for decimals handling in operations
 create table decimals_test(id int, a decimal(38,18), b decimal(38,18)) using parquet;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/ansi/group-analytics.sql b/sql/core/src/test/resources/sql-tests/inputs/ansi/group-analytics.sql
new file mode 100644
index 0000000000000..d78689233ddf9
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/ansi/group-analytics.sql
@@ -0,0 +1 @@
+--IMPORT group-analytics.sql
\ No newline at end of file
diff --git a/sql/core/src/test/resources/sql-tests/inputs/ansi/map.sql b/sql/core/src/test/resources/sql-tests/inputs/ansi/map.sql
new file mode 100644
index 0000000000000..23e5b9562973b
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/ansi/map.sql
@@ -0,0 +1 @@
+--IMPORT map.sql
diff --git a/sql/core/src/test/resources/sql-tests/inputs/ansi/parse-schema-string.sql b/sql/core/src/test/resources/sql-tests/inputs/ansi/parse-schema-string.sql
new file mode 100644
index 0000000000000..42775102e650e
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/ansi/parse-schema-string.sql
@@ -0,0 +1 @@
+--IMPORT parse-schema-string.sql
diff --git a/sql/core/src/test/resources/sql-tests/inputs/ansi/string-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/ansi/string-functions.sql
new file mode 100644
index 0000000000000..dd28e9b97fb20
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/ansi/string-functions.sql
@@ -0,0 +1 @@
+--IMPORT string-functions.sql
diff --git a/sql/core/src/test/resources/sql-tests/inputs/array.sql b/sql/core/src/test/resources/sql-tests/inputs/array.sql
index 984321ab795fc..f73b653659eb4 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/array.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/array.sql
@@ -90,3 +90,15 @@ select
   size(date_array),
   size(timestamp_array)
 from primitive_arrays;
+
+-- index out of range for array elements
+select element_at(array(1, 2, 3), 5);
+select element_at(array(1, 2, 3), -5);
+select element_at(array(1, 2, 3), 0);
+
+select elt(4, '123', '456');
+select elt(0, '123', '456');
+select elt(-1, '123', '456');
+
+select array(1, 2, 3)[5];
+select array(1, 2, 3)[-1];
diff --git a/sql/core/src/test/resources/sql-tests/inputs/bitwise.sql b/sql/core/src/test/resources/sql-tests/inputs/bitwise.sql
index 5e665e4c0c384..f9dfd161d0c07 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/bitwise.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/bitwise.sql
@@ -68,3 +68,10 @@ SELECT b1, bit_xor(b2) FROM bitwise_test GROUP BY b1 HAVING bit_and(b2) < 7;
 
 -- window
 SELECT b1, b2, bit_xor(b2) OVER (PARTITION BY b1 ORDER BY b2) FROM bitwise_test;
+
+-- getbit
+select getbit(11L, 3), getbit(11L, 2), getbit(11L, 1), getbit(11L, 0);
+select getbit(11L, 2 + 1), getbit(11L, 3 - 1), getbit(10L + 1, 1 * 1), getbit(cast(11L / 1 AS long), 1 - 1);
+select getbit(11L, 63);
+select getbit(11L, -1);
+select getbit(11L, 64);
diff --git a/sql/core/src/test/resources/sql-tests/inputs/blacklist.sql b/sql/core/src/test/resources/sql-tests/inputs/blacklist.sql
deleted file mode 100644
index d69f8147a5264..0000000000000
--- a/sql/core/src/test/resources/sql-tests/inputs/blacklist.sql
+++ /dev/null
@@ -1,4 +0,0 @@
--- This is a query file that has been blacklisted.
--- It includes a query that should crash Spark.
--- If the test case is run, the whole suite would fail.
-some random not working query that should crash Spark.
diff --git a/sql/core/src/test/resources/sql-tests/inputs/cast.sql b/sql/core/src/test/resources/sql-tests/inputs/cast.sql
index 972ebdd01f61e..81c741a5ca8e9 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/cast.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/cast.sql
@@ -70,6 +70,11 @@ select cast(' 1' as bigint);
 select cast(' 1' as float);
 select cast(' 1 ' as DOUBLE);
 select cast('1.0 ' as DEC);
+select cast('1中文' as tinyint);
+select cast('1中文' as smallint);
+select cast('1中文' as INT);
+select cast('中文1' as bigint);
+select cast('1中文' as bigint);
 
 -- trim string before cast to boolean
 select cast('\t\t true \n\r ' as boolean);
diff --git a/sql/core/src/test/resources/sql-tests/inputs/charvarchar.sql b/sql/core/src/test/resources/sql-tests/inputs/charvarchar.sql
new file mode 100644
index 0000000000000..098d09d98218d
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/charvarchar.sql
@@ -0,0 +1,115 @@
+create table char_tbl(c char(5), v varchar(6)) using parquet;
+desc formatted char_tbl;
+desc formatted char_tbl c;
+show create table char_tbl;
+
+create table char_tbl2 using parquet as select * from char_tbl;
+show create table char_tbl2;
+desc formatted char_tbl2;
+desc formatted char_tbl2 c;
+
+create table char_tbl3 like char_tbl;
+desc formatted char_tbl3;
+desc formatted char_tbl3 c;
+show create table char_tbl3;
+
+create view char_view as select * from char_tbl;
+desc formatted char_view;
+desc formatted char_view c;
+show create table char_view;
+
+alter table char_tbl rename to char_tbl1;
+desc formatted char_tbl1;
+
+alter table char_tbl1 change column c type char(6);
+alter table char_tbl1 change column c type char(5);
+desc formatted char_tbl1;
+
+alter table char_tbl1 add columns (d char(5));
+desc formatted char_tbl1;
+
+alter view char_view as select * from char_tbl2;
+desc formatted char_view;
+
+alter table char_tbl1 SET TBLPROPERTIES('yes'='no');
+desc formatted char_tbl1;
+
+alter view char_view SET TBLPROPERTIES('yes'='no');
+desc formatted char_view;
+
+alter table char_tbl1 UNSET TBLPROPERTIES('yes');
+desc formatted char_tbl1;
+
+alter view char_view UNSET TBLPROPERTIES('yes');
+desc formatted char_view;
+
+alter table char_tbl1 SET SERDEPROPERTIES('yes'='no');
+desc formatted char_tbl1;
+
+create table char_part(c1 char(5), c2 char(2), v1 varchar(6), v2 varchar(2)) using parquet partitioned by (v2, c2);
+desc formatted char_part;
+
+alter table char_part add partition (v2='ke', c2='nt') location 'loc1';
+desc formatted char_part;
+
+alter table char_part partition (v2='ke') rename to partition (v2='nt');
+desc formatted char_part;
+
+alter table char_part partition (v2='ke', c2='nt') set location 'loc2';
+desc formatted char_part;
+
+MSCK REPAIR TABLE char_part;
+desc formatted char_part;
+
+create temporary view str_view as select c, v from values
+    (null, null),
+    (null, 'S'),
+    ('N', 'N '),
+    ('Ne', 'Sp'),
+    ('Net  ', 'Spa  '),
+    ('NetE', 'Spar'),
+    ('NetEa ', 'Spark '),
+    ('NetEas ', 'Spark'),
+    ('NetEase', 'Spark-') t(c, v);
+
+create table char_tbl4(c7 char(7), c8 char(8), v varchar(6), s string) using parquet;
+insert into char_tbl4 select c, c, v, c from str_view;
+
+select c7, c8, v, s from char_tbl4;
+select c7, c8, v, s from char_tbl4 where c7 = c8;
+select c7, c8, v, s from char_tbl4 where c7 = v;
+select c7, c8, v, s from char_tbl4 where c7 = s;
+select c7, c8, v, s from char_tbl4 where c7 = 'NetEase               ';
+select c7, c8, v, s from char_tbl4 where v = 'Spark ';
+select c7, c8, v, s from char_tbl4 order by c7;
+select c7, c8, v, s from char_tbl4 order by v;
+
+select ascii(c7), ascii(c8), ascii(v), ascii(s) from char_tbl4;
+select base64(c7), base64(c8), base64(v), ascii(s) from char_tbl4;
+select bit_length(c7), bit_length(c8), bit_length(v), bit_length(s) from char_tbl4;
+select char_length(c7), char_length(c8), char_length(v), char_length(s) from char_tbl4;
+select octet_length(c7), octet_length(c8), octet_length(v), octet_length(s) from char_tbl4;
+select concat_ws('|', c7, c8), concat_ws('|', c7, v), concat_ws('|', c7, s), concat_ws('|', v, s) from char_tbl4;
+select concat(c7, c8), concat(c7, v), concat(c7, s), concat(v, s) from char_tbl4;
+select like(c7, 'Ne     _'), like(c8, 'Ne     _') from char_tbl4;
+select like(v, 'Spark_') from char_tbl4;
+select c7 = c8, upper(c7) = upper(c8), lower(c7) = lower(c8) from char_tbl4 where s = 'NetEase';
+select c7 = s, upper(c7) = upper(s), lower(c7) = lower(s) from char_tbl4 where s = 'NetEase';
+select c7 = 'NetEase', upper(c7) = upper('NetEase'), lower(c7) = lower('NetEase') from char_tbl4 where s = 'NetEase';
+select printf('Hey, %s%s%s%s', c7, c8, v, s) from char_tbl4;
+select repeat(c7, 2), repeat(c8, 2), repeat(v, 2), repeat(s, 2) from char_tbl4;
+select replace(c7, 'Net', 'Apache'), replace(c8, 'Net', 'Apache'), replace(v, 'Spark', 'Kyuubi'), replace(s, 'Net', 'Apache') from char_tbl4;
+select rpad(c7, 10), rpad(c8, 5), rpad(v, 5), rpad(s, 5)  from char_tbl4;
+select rtrim(c7), rtrim(c8), rtrim(v), rtrim(s) from char_tbl4;
+select split(c7, 'e'), split(c8, 'e'), split(v, 'a'), split(s, 'e') from char_tbl4;
+select substring(c7, 2), substring(c8, 2), substring(v, 3), substring(s, 2) from char_tbl4;
+select left(c7, 2), left(c8, 2), left(v, 3), left(s, 2) from char_tbl4;
+select right(c7, 2), right(c8, 2), right(v, 3), right(s, 2) from char_tbl4;
+select typeof(c7), typeof(c8), typeof(v), typeof(s) from char_tbl4 limit 1;
+select cast(c7 as char(1)), cast(c8 as char(10)), cast(v as char(1)), cast(v as varchar(1)), cast(s as char(5)) from char_tbl4;
+
+-- char_tbl has renamed to char_tbl1
+drop table char_tbl1;
+drop table char_tbl2;
+drop table char_tbl3;
+drop table char_tbl4;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/count.sql b/sql/core/src/test/resources/sql-tests/inputs/count.sql
index 9f9ee4a873d4f..ae27c1cae33b5 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/count.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/count.sql
@@ -19,9 +19,35 @@ SELECT
 FROM testData;
 
 -- count with multiple expressions
-SELECT count(a, b), count(b, a), count(testData.*) FROM testData;
+SELECT count(a, b), count(b, a), count(testData.*, testData.*) FROM testData;
 
 -- distinct count with multiple expressions
 SELECT
-  count(DISTINCT a, b), count(DISTINCT b, a), count(DISTINCT *), count(DISTINCT testData.*)
+  count(DISTINCT a, b), count(DISTINCT b, a), count(DISTINCT *), count(DISTINCT testData.*, testData.*)
 FROM testData;
+
+-- distinct count with multiple literals
+SELECT count(DISTINCT 3,2);
+SELECT count(DISTINCT 2), count(DISTINCT 2,3);
+SELECT count(DISTINCT 2), count(DISTINCT 3,2);
+SELECT count(DISTINCT a), count(DISTINCT 2,3) FROM testData;
+SELECT count(DISTINCT a), count(DISTINCT 3,2) FROM testData;
+SELECT count(DISTINCT a), count(DISTINCT 2), count(DISTINCT 2,3) FROM testData;
+SELECT count(DISTINCT a), count(DISTINCT 2), count(DISTINCT 3,2) FROM testData;
+SELECT count(distinct 0.8), percentile_approx(distinct a, 0.8) FROM testData;
+
+-- legacy behavior: allow calling function count without parameters
+set spark.sql.legacy.allowParameterlessCount=true;
+SELECT count() FROM testData;
+
+-- count without expressions
+set spark.sql.legacy.allowParameterlessCount=false;
+SELECT count() FROM testData;
+
+-- legacy behavior: allow count(testData.*)
+set spark.sql.legacy.allowStarWithSingleTableIdentifierInCount=true;
+SELECT count(testData.*) FROM testData;
+
+-- count with a single tblName.* as parameter
+set spark.sql.legacy.allowStarWithSingleTableIdentifierInCount=false;
+SELECT count(testData.*) FROM testData;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/current_database_catalog.sql b/sql/core/src/test/resources/sql-tests/inputs/current_database_catalog.sql
new file mode 100644
index 0000000000000..4406f1bc2e6e3
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/current_database_catalog.sql
@@ -0,0 +1,2 @@
+-- get current_datebase and current_catalog
+select current_database(), current_catalog();
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting-invalid.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting-invalid.sql
new file mode 100644
index 0000000000000..11bba00e91abf
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting-invalid.sql
@@ -0,0 +1,53 @@
+--- TESTS FOR DATETIME FORMATTING FUNCTIONS WITH INVALID PATTERNS ---
+
+-- separating this from datetime-formatting.sql, because the text form
+-- for patterns with 5 letters in SimpleDateFormat varies from different JDKs
+select date_format('2018-11-17 13:33:33.333', 'GGGGG');
+-- pattern letter count can not be greater than 6
+select date_format('2018-11-17 13:33:33.333', 'yyyyyyy');
+-- q/L in JDK 8 will fail when the count is more than 2
+select date_format('2018-11-17 13:33:33.333', 'qqqqq');
+select date_format('2018-11-17 13:33:33.333', 'QQQQQ');
+select date_format('2018-11-17 13:33:33.333', 'MMMMM');
+select date_format('2018-11-17 13:33:33.333', 'LLLLL');
+
+select date_format('2018-11-17 13:33:33.333', 'EEEEE');
+select date_format('2018-11-17 13:33:33.333', 'FF');
+select date_format('2018-11-17 13:33:33.333', 'ddd');
+-- DD is invalid if the day-of-year exceeds 100, but it becomes valid in Java 11
+-- select date_format('2018-11-17 13:33:33.333', 'DD');
+select date_format('2018-11-17 13:33:33.333', 'DDDD');
+select date_format('2018-11-17 13:33:33.333', 'HHH');
+select date_format('2018-11-17 13:33:33.333', 'hhh');
+select date_format('2018-11-17 13:33:33.333', 'kkk');
+select date_format('2018-11-17 13:33:33.333', 'KKK');
+select date_format('2018-11-17 13:33:33.333', 'mmm');
+select date_format('2018-11-17 13:33:33.333', 'sss');
+select date_format('2018-11-17 13:33:33.333', 'SSSSSSSSSS');
+select date_format('2018-11-17 13:33:33.333', 'aa');
+select date_format('2018-11-17 13:33:33.333', 'V');
+select date_format('2018-11-17 13:33:33.333', 'zzzzz');
+select date_format('2018-11-17 13:33:33.333', 'XXXXXX');
+select date_format('2018-11-17 13:33:33.333', 'ZZZZZZ');
+select date_format('2018-11-17 13:33:33.333', 'OO');
+select date_format('2018-11-17 13:33:33.333', 'xxxxxx');
+
+select date_format('2018-11-17 13:33:33.333', 'A');
+select date_format('2018-11-17 13:33:33.333', 'n');
+select date_format('2018-11-17 13:33:33.333', 'N');
+select date_format('2018-11-17 13:33:33.333', 'p');
+
+-- disabled week-based patterns
+select date_format('2018-11-17 13:33:33.333', 'Y');
+select date_format('2018-11-17 13:33:33.333', 'w');
+select date_format('2018-11-17 13:33:33.333', 'W');
+select date_format('2018-11-17 13:33:33.333', 'u');
+select date_format('2018-11-17 13:33:33.333', 'e');
+select date_format('2018-11-17 13:33:33.333', 'c');
+
+-- others
+select date_format('2018-11-17 13:33:33.333', 'B');
+select date_format('2018-11-17 13:33:33.333', 'C');
+select date_format('2018-11-17 13:33:33.333', 'I');
+
+
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting-legacy.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting-legacy.sql
new file mode 100644
index 0000000000000..19cab61a7ee56
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting-legacy.sql
@@ -0,0 +1,2 @@
+--SET spark.sql.legacy.timeParserPolicy=LEGACY
+--IMPORT datetime-formatting.sql
\ No newline at end of file
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting.sql
new file mode 100644
index 0000000000000..2d70326f4f3c0
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting.sql
@@ -0,0 +1,68 @@
+--- TESTS FOR DATETIME FORMATTING FUNCTIONS ---
+
+create temporary view v as select col from values
+ (timestamp '1582-06-01 11:33:33.123UTC+080000'),
+ (timestamp '1970-01-01 00:00:00.000Europe/Paris'),
+ (timestamp '1970-12-31 23:59:59.999Asia/Srednekolymsk'),
+ (timestamp '1996-04-01 00:33:33.123Australia/Darwin'),
+ (timestamp '2018-11-17 13:33:33.123Z'),
+ (timestamp '2020-01-01 01:33:33.123Asia/Shanghai'),
+ (timestamp '2100-01-01 01:33:33.123America/Los_Angeles') t(col);
+
+select col, date_format(col, 'G GG GGG GGGG') from v;
+
+select col, date_format(col, 'y yy yyy yyyy yyyyy yyyyyy') from v;
+
+select col, date_format(col, 'q qq') from v;
+
+select col, date_format(col, 'Q QQ QQQ QQQQ') from v;
+
+select col, date_format(col, 'M MM MMM MMMM') from v;
+
+select col, date_format(col, 'L LL') from v;
+
+select col, date_format(col, 'E EE EEE EEEE') from v;
+
+select col, date_format(col, 'F') from v;
+
+select col, date_format(col, 'd dd') from v;
+
+select col, date_format(col, 'DD') from v where col = timestamp '2100-01-01 01:33:33.123America/Los_Angeles';
+select col, date_format(col, 'D DDD') from v;
+
+select col, date_format(col, 'H HH') from v;
+
+select col, date_format(col, 'h hh') from v;
+
+select col, date_format(col, 'k kk') from v;
+
+select col, date_format(col, 'K KK') from v;
+
+select col, date_format(col, 'm mm') from v;
+
+select col, date_format(col, 's ss') from v;
+
+select col, date_format(col, 'S SS SSS SSSS SSSSS SSSSSS SSSSSSS SSSSSSSS SSSSSSSSS') from v;
+
+select col, date_format(col, 'a') from v;
+
+select col, date_format(col, 'VV') from v;
+
+select col, date_format(col, 'z zz zzz zzzz') from v;
+
+select col, date_format(col, 'X XX XXX') from v;
+select col, date_format(col, 'XXXX XXXXX') from v;
+
+select col, date_format(col, 'Z ZZ ZZZ ZZZZ ZZZZZ') from v;
+
+select col, date_format(col, 'O OOOO') from v;
+
+select col, date_format(col, 'x xx xxx xxxx xxxx xxxxx') from v;
+
+-- optional pattern, but the results won't be optional for formatting
+select col, date_format(col, '[yyyy-MM-dd HH:mm:ss]') from v;
+
+-- literals
+select col, date_format(col, "姚123'GyYqQMLwWuEFDdhHmsSaVzZxXOV'") from v;
+select col, date_format(col, "''") from v;
+select col, date_format(col, '') from v;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-legacy.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-legacy.sql
new file mode 100644
index 0000000000000..daec2b40a620b
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-legacy.sql
@@ -0,0 +1,2 @@
+--SET spark.sql.legacy.timeParserPolicy=LEGACY
+--IMPORT datetime.sql
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-invalid.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-invalid.sql
new file mode 100644
index 0000000000000..a1c02eaa3b0a0
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-invalid.sql
@@ -0,0 +1,29 @@
+--- TESTS FOR DATETIME PARSING FUNCTIONS WITH INVALID VALUES ---
+
+-- parsing invalid value with pattern 'y'
+select to_timestamp('294248', 'y'); -- out of year value range [0, 294247]
+select to_timestamp('1', 'yy'); -- the number of digits must be 2 for 'yy'.
+select to_timestamp('-12', 'yy'); -- out of year value range [0, 99] for reduced two digit form
+select to_timestamp('123', 'yy'); -- the number of digits must be 2 for 'yy'.
+select to_timestamp('1', 'yyy'); -- the number of digits must be in [3, 6] for 'yyy'
+
+select to_timestamp('1234567', 'yyyyyyy'); -- the length of 'y' pattern must be less than 7
+
+-- parsing invalid values with pattern 'D'
+select to_timestamp('366', 'D');
+select to_timestamp('9', 'DD');
+-- in java 8 this case is invalid, but valid in java 11, disabled for jenkins
+-- select to_timestamp('100', 'DD');
+select to_timestamp('366', 'DD');
+select to_timestamp('9', 'DDD');
+select to_timestamp('99', 'DDD');
+select to_timestamp('30-365', 'dd-DDD');
+select to_timestamp('11-365', 'MM-DDD');
+select to_timestamp('2019-366', 'yyyy-DDD');
+select to_timestamp('12-30-365', 'MM-dd-DDD');
+select to_timestamp('2020-01-365', 'yyyy-dd-DDD');
+select to_timestamp('2020-10-350', 'yyyy-MM-DDD');
+select to_timestamp('2020-11-31-366', 'yyyy-MM-dd-DDD');
+-- add a special case to test csv, because the legacy formatter it uses is lenient then Spark should
+-- throw SparkUpgradeException
+select from_csv('2018-366', 'date Date', map('dateFormat', 'yyyy-DDD'))
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-legacy.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-legacy.sql
new file mode 100644
index 0000000000000..ee1afe502ab79
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-legacy.sql
@@ -0,0 +1,2 @@
+--SET spark.sql.legacy.timeParserPolicy=LEGACY
+--IMPORT datetime-parsing.sql
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing.sql
new file mode 100644
index 0000000000000..e058bd675c375
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing.sql
@@ -0,0 +1,44 @@
+--- TESTS FOR DATETIME PARSING FUNCTIONS ---
+
+-- parsing with pattern 'y'.
+-- the range of valid year is [-290307, 294247],
+-- but particularly, some thrift client use java.sql.Timestamp to parse timestamp, which allows
+-- only positive year values less or equal than 9999. So the cases bellow only use [1, 9999] to pass
+-- ThriftServerQueryTestSuite
+select to_timestamp('1', 'y');
+select to_timestamp('009999', 'y');
+
+-- reduced two digit form is used, the range of valid year is 20-[01, 99]
+select to_timestamp('00', 'yy');
+select to_timestamp('99', 'yy');
+
+-- the range of valid year is [-290307, 294247], the number of digits must be in [3, 6] for 'yyy'
+select to_timestamp('001', 'yyy');
+select to_timestamp('009999', 'yyy');
+
+-- the range of valid year is [-9999, 9999], the number of digits must be 4 for 'yyyy'.
+select to_timestamp('0001', 'yyyy');
+select to_timestamp('9999', 'yyyy');
+
+-- the range of valid year is [-99999, 99999], the number of digits must be 5 for 'yyyyy'.
+select to_timestamp('00001', 'yyyyy');
+select to_timestamp('09999', 'yyyyy');
+
+-- the range of valid year is [-290307, 294247], the number of digits must be 6 for 'yyyyyy'.
+select to_timestamp('000001', 'yyyyyy');
+select to_timestamp('009999', 'yyyyyy');
+
+-- parsing with pattern 'D'
+select to_timestamp('9', 'D');
+select to_timestamp('300', 'D');
+select to_timestamp('09', 'DD');
+select to_timestamp('99', 'DD');
+select to_timestamp('009', 'DDD');
+select to_timestamp('365', 'DDD');
+select to_timestamp('31-365', 'dd-DDD');
+select to_timestamp('12-365', 'MM-DDD');
+select to_timestamp('2020-365', 'yyyy-DDD');
+select to_timestamp('12-31-365', 'MM-dd-DDD');
+select to_timestamp('2020-30-365', 'yyyy-dd-DDD');
+select to_timestamp('2020-12-350', 'yyyy-MM-DDD');
+select to_timestamp('2020-12-31-366', 'yyyy-MM-dd-DDD');
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
index fd3325085df96..0493d8653c01f 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
@@ -1,5 +1,27 @@
 -- date time functions
 
+-- [SPARK-31710] TIMESTAMP_SECONDS, TIMESTAMP_MILLISECONDS and TIMESTAMP_MICROSECONDS to timestamp transfer
+select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null);
+select TIMESTAMP_SECONDS(1.23), TIMESTAMP_SECONDS(1.23d), TIMESTAMP_SECONDS(FLOAT(1.23));
+select TIMESTAMP_MILLIS(1230219000123),TIMESTAMP_MILLIS(-1230219000123),TIMESTAMP_MILLIS(null);
+select TIMESTAMP_MICROS(1230219000123123),TIMESTAMP_MICROS(-1230219000123123),TIMESTAMP_MICROS(null);
+-- overflow exception
+select TIMESTAMP_SECONDS(1230219000123123);
+select TIMESTAMP_SECONDS(-1230219000123123);
+select TIMESTAMP_MILLIS(92233720368547758);
+select TIMESTAMP_MILLIS(-92233720368547758);
+-- truncate exception
+select TIMESTAMP_SECONDS(0.1234567);
+-- truncation is OK for float/double
+select TIMESTAMP_SECONDS(0.1234567d), TIMESTAMP_SECONDS(FLOAT(0.1234567));
+-- UNIX_SECONDS, UNIX_MILLISECONDS and UNIX_MICROSECONDS
+select UNIX_SECONDS(TIMESTAMP('2020-12-01 14:30:08Z')), UNIX_SECONDS(TIMESTAMP('2020-12-01 14:30:08.999999Z')), UNIX_SECONDS(null);
+select UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08Z')), UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08.999999Z')), UNIX_MILLIS(null);
+select UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08Z')), UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08.999999Z')), UNIX_MICROS(null);
+-- DATE_FROM_UNIX_DATE
+select DATE_FROM_UNIX_DATE(0), DATE_FROM_UNIX_DATE(1000), DATE_FROM_UNIX_DATE(null);
+-- UNIX_DATE
+select UNIX_DATE(DATE('1970-01-01')), UNIX_DATE(DATE('2020-12-04')), UNIX_DATE(null);
 -- [SPARK-16836] current_date and current_timestamp literals
 select current_date = current_date(), current_timestamp = current_timestamp();
 
@@ -33,6 +55,8 @@ select year('1500-01-01'), month('1500-01-01'), dayOfYear('1500-01-01');
 
 select date '2019-01-01\t';
 select timestamp '2019-01-01\t';
+select date '2020-01-01中文';
+select timestamp '2019-01-01中文';
 
 -- time add/sub
 select timestamp'2011-11-11 11:11:11' + interval '2' day;
@@ -86,7 +110,7 @@ select date_sub('2011-11-11', str) from v;
 select null - date '2019-10-06';
 select date '2001-10-01' - date '2001-09-28';
 
--- variable-length tests
+-- variable-length second fraction tests
 select to_timestamp('2019-10-06 10:11:12.', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
 select to_timestamp('2019-10-06 10:11:12.0', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
 select to_timestamp('2019-10-06 10:11:12.1', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
@@ -95,7 +119,7 @@ select to_timestamp('2019-10-06 10:11:12.123UTC', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zz
 select to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
 select to_timestamp('2019-10-06 10:11:12.12345CST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
 select to_timestamp('2019-10-06 10:11:12.123456PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
--- exceeded max variable length
+-- second fraction exceeded max variable length
 select to_timestamp('2019-10-06 10:11:12.1234567PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
 -- special cases
 select to_timestamp('123456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
@@ -114,11 +138,44 @@ select to_timestamp("12.1234019-10-06S10:11", "ss.SSSSy-MM-dd'S'HH:mm");
 select to_timestamp("2019-10-06S", "yyyy-MM-dd'S'");
 select to_timestamp("S2019-10-06", "'S'yyyy-MM-dd");
 
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuee');
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uucc');
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuuu');
-
 select to_timestamp("2019-10-06T10:11:12'12", "yyyy-MM-dd'T'HH:mm:ss''SSSS"); -- middle
 select to_timestamp("2019-10-06T10:11:12'", "yyyy-MM-dd'T'HH:mm:ss''"); -- tail
 select to_timestamp("'2019-10-06T10:11:12", "''yyyy-MM-dd'T'HH:mm:ss"); -- head
 select to_timestamp("P2019-10-06T10:11:12", "'P'yyyy-MM-dd'T'HH:mm:ss"); -- head but as single quote
+
+-- missing fields
+select to_timestamp("16", "dd");
+select to_timestamp("02-29", "MM-dd");
+select to_date("16", "dd");
+select to_date("02-29", "MM-dd");
+select to_timestamp("2019 40", "yyyy mm");
+select to_timestamp("2019 10:10:10", "yyyy hh:mm:ss");
+
+-- Unsupported narrow text style
+select to_timestamp('2019-10-06 A', 'yyyy-MM-dd GGGGG');
+select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEEE');
+select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE');
+select unix_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE');
+select from_json('{"t":"26/October/2015"}', 't Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'));
+select from_json('{"d":"26/October/2015"}', 'd Date', map('dateFormat', 'dd/MMMMM/yyyy'));
+select from_csv('26/October/2015', 't Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'));
+select from_csv('26/October/2015', 'd Date', map('dateFormat', 'dd/MMMMM/yyyy'));
+
+-- Datetime types parse error
+select to_date("2020-01-27T20:06:11.847", "yyyy-MM-dd HH:mm:ss.SSS");
+select to_date("Unparseable", "yyyy-MM-dd HH:mm:ss.SSS");
+select to_timestamp("2020-01-27T20:06:11.847", "yyyy-MM-dd HH:mm:ss.SSS");
+select to_timestamp("Unparseable", "yyyy-MM-dd HH:mm:ss.SSS");
+select unix_timestamp("2020-01-27T20:06:11.847", "yyyy-MM-dd HH:mm:ss.SSS");
+select unix_timestamp("Unparseable", "yyyy-MM-dd HH:mm:ss.SSS");
+select to_unix_timestamp("2020-01-27T20:06:11.847", "yyyy-MM-dd HH:mm:ss.SSS");
+select to_unix_timestamp("Unparseable", "yyyy-MM-dd HH:mm:ss.SSS");
+select cast("Unparseable" as timestamp);
+select cast("Unparseable" as date);
+
+-- next_day
+select next_day("2015-07-23", "Mon");
+select next_day("2015-07-23", "xx");
+select next_day("xx", "Mon");
+select next_day(null, "Mon");
+select next_day(null, "xx");
diff --git a/sql/core/src/test/resources/sql-tests/inputs/describe-table-column.sql b/sql/core/src/test/resources/sql-tests/inputs/describe-table-column.sql
index d55e398329b76..146977c806182 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/describe-table-column.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/describe-table-column.sql
@@ -1,5 +1,5 @@
 -- Test temp table
-CREATE TEMPORARY VIEW desc_col_temp_view (key int COMMENT 'column_comment') USING PARQUET;
+CREATE TEMPORARY VIEW desc_col_temp_view (key int COMMENT 'column_comment', col struct<x:int, y:string>) USING PARQUET;
 
 DESC desc_col_temp_view key;
 
@@ -13,6 +13,9 @@ DESC FORMATTED desc_col_temp_view desc_col_temp_view.key;
 -- Describe a non-existent column
 DESC desc_col_temp_view key1;
 
+-- Describe a nested column
+DESC desc_col_temp_view col.x;
+
 -- Test persistent table
 CREATE TABLE desc_col_table (key int COMMENT 'column_comment') USING PARQUET;
 
@@ -24,6 +27,9 @@ DESC EXTENDED desc_col_table key;
 
 DESC FORMATTED desc_col_table key;
 
+-- Describe a non-existent column
+DESC desc_col_table key1;
+
 -- Test complex columns
 CREATE TABLE desc_complex_col_table (`a.b` int, col struct<x:int, y:string>) USING PARQUET;
 
diff --git a/sql/core/src/test/resources/sql-tests/inputs/except.sql b/sql/core/src/test/resources/sql-tests/inputs/except.sql
index 1d579e65f3473..ffdf1f4f3d24d 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/except.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/except.sql
@@ -55,3 +55,22 @@ FROM   t1
 WHERE  t1.v >= (SELECT   min(t2.v)
                 FROM     t2
                 WHERE    t2.k = t1.k);
+
+-- SPARK-32638: corrects references when adding aliases in WidenSetOperationTypes
+CREATE OR REPLACE TEMPORARY VIEW t3 AS VALUES (decimal(1)) tbl(v);
+SELECT t.v FROM (
+  SELECT v FROM t3
+  EXCEPT
+  SELECT v + v AS v FROM t3
+) t;
+
+SELECT SUM(t.v) FROM (
+  SELECT v FROM t3
+  EXCEPT
+  SELECT v + v AS v FROM t3
+) t;
+
+-- Clean-up
+DROP VIEW IF EXISTS t1;
+DROP VIEW IF EXISTS t2;
+DROP VIEW IF EXISTS t3;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/explain-aqe.sql b/sql/core/src/test/resources/sql-tests/inputs/explain-aqe.sql
index f4afa2b77a9d7..7aef901da4fb5 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/explain-aqe.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/explain-aqe.sql
@@ -1,3 +1,4 @@
 --IMPORT explain.sql
 
 --SET spark.sql.adaptive.enabled=true
+--SET spark.sql.maxMetadataStringLength = 500
diff --git a/sql/core/src/test/resources/sql-tests/inputs/explain-cbo.sql b/sql/core/src/test/resources/sql-tests/inputs/explain-cbo.sql
new file mode 100644
index 0000000000000..eeb2180f7a542
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/explain-cbo.sql
@@ -0,0 +1,27 @@
+--SET spark.sql.cbo.enabled=true
+--SET spark.sql.maxMetadataStringLength = 500
+
+CREATE TABLE explain_temp1(a INT, b INT) USING PARQUET;
+CREATE TABLE explain_temp2(c INT, d INT) USING PARQUET;
+
+ANALYZE TABLE explain_temp1 COMPUTE STATISTICS FOR ALL COLUMNS;
+ANALYZE TABLE explain_temp2 COMPUTE STATISTICS FOR ALL COLUMNS;
+
+EXPLAIN COST WITH max_store_sales AS
+(
+  SELECT max(csales) tpcds_cmax
+  FROM (
+    SELECT sum(b) csales
+    FROM explain_temp1 WHERE a < 100
+  ) x
+),
+best_ss_customer AS
+(
+  SELECT c
+  FROM explain_temp2
+  WHERE d > (SELECT * FROM max_store_sales)
+)
+SELECT c FROM best_ss_customer;
+
+DROP TABLE explain_temp1;
+DROP TABLE explain_temp2;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/explain.sql b/sql/core/src/test/resources/sql-tests/inputs/explain.sql
index 80bf258704c70..736084597eb79 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/explain.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/explain.sql
@@ -1,5 +1,6 @@
 --SET spark.sql.codegen.wholeStage = true
 --SET spark.sql.adaptive.enabled = false
+--SET spark.sql.maxMetadataStringLength = 500
 
 -- Test tables
 CREATE table  explain_temp1 (key int, val int) USING PARQUET;
@@ -9,6 +10,11 @@ CREATE table  explain_temp4 (key int, val string) USING PARQUET;
 
 SET spark.sql.codegen.wholeStage = true;
 
+-- distinct func
+EXPLAIN EXTENDED
+  SELECT sum(distinct val)
+  FROM explain_temp1;
+
 -- single table
 EXPLAIN FORMATTED
   SELECT key, max(val) 
@@ -28,7 +34,7 @@ EXPLAIN FORMATTED
 EXPLAIN FORMATTED
   SELECT key, val FROM explain_temp1 WHERE key > 0
   UNION 
-  SELECT key, val FROM explain_temp1 WHERE key > 0;
+  SELECT key, val FROM explain_temp1 WHERE key > 1;
 
 -- Join
 EXPLAIN FORMATTED
diff --git a/sql/core/src/test/resources/sql-tests/inputs/extract.sql b/sql/core/src/test/resources/sql-tests/inputs/extract.sql
index 0f1fd5bbcca0b..31603fba99f3d 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/extract.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/extract.sql
@@ -128,3 +128,34 @@ select c - i from t;
 select year(c - i) from t;
 select extract(year from c - i) from t;
 select extract(month from to_timestamp(c) - i) from t;
+
+-- extract fields from year-month/day-time intervals
+select extract(YEAR from interval '2-1' YEAR TO MONTH);
+select date_part('YEAR', interval '2-1' YEAR TO MONTH);
+select extract(YEAR from -interval '2-1' YEAR TO MONTH);
+select extract(MONTH from interval '2-1' YEAR TO MONTH);
+select date_part('MONTH', interval '2-1' YEAR TO MONTH);
+select extract(MONTH from -interval '2-1' YEAR TO MONTH);
+select date_part(NULL, interval '2-1' YEAR TO MONTH);
+
+-- invalid
+select extract(DAY from interval '2-1' YEAR TO MONTH);
+select date_part('DAY', interval '2-1' YEAR TO MONTH);
+select date_part('not_supported', interval '2-1' YEAR TO MONTH);
+
+select extract(DAY from interval '123 12:34:56.789123123' DAY TO SECOND);
+select date_part('DAY', interval '123 12:34:56.789123123' DAY TO SECOND);
+select extract(DAY from -interval '123 12:34:56.789123123' DAY TO SECOND);
+select extract(HOUR from interval '123 12:34:56.789123123' DAY TO SECOND);
+select date_part('HOUR', interval '123 12:34:56.789123123' DAY TO SECOND);
+select extract(HOUR from -interval '123 12:34:56.789123123' DAY TO SECOND);
+select extract(MINUTE from interval '123 12:34:56.789123123' DAY TO SECOND);
+select date_part('MINUTE', interval '123 12:34:56.789123123' DAY TO SECOND);
+select extract(MINUTE from -interval '123 12:34:56.789123123' DAY TO SECOND);
+select extract(SECOND from interval '123 12:34:56.789123123' DAY TO SECOND);
+select date_part('SECOND', interval '123 12:34:56.789123123' DAY TO SECOND);
+select extract(SECOND from -interval '123 12:34:56.789123123' DAY TO SECOND);
+select date_part(NULL, interval '123 12:34:56.789123123' DAY TO SECOND);
+
+select extract(MONTH from interval '123 12:34:56.789123123' DAY TO SECOND);
+select date_part('not_supported', interval '123 12:34:56.789123123' DAY TO SECOND);
diff --git a/sql/core/src/test/resources/sql-tests/inputs/group-analytics.sql b/sql/core/src/test/resources/sql-tests/inputs/group-analytics.sql
index 9721f8c60ebce..d6381e59e0d84 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/group-analytics.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/group-analytics.sql
@@ -18,15 +18,25 @@ AS courseSales(course, year, earnings);
 
 -- ROLLUP
 SELECT course, year, SUM(earnings) FROM courseSales GROUP BY ROLLUP(course, year) ORDER BY course, year;
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY ROLLUP(course, year, (course, year)) ORDER BY course, year;
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY ROLLUP(course, year, (course, year), ()) ORDER BY course, year;
 
 -- CUBE
 SELECT course, year, SUM(earnings) FROM courseSales GROUP BY CUBE(course, year) ORDER BY course, year;
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY CUBE(course, year, (course, year)) ORDER BY course, year;
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY CUBE(course, year, (course, year), ()) ORDER BY course, year;
 
 -- GROUPING SETS
 SELECT course, year, SUM(earnings) FROM courseSales GROUP BY course, year GROUPING SETS(course, year);
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY course, year GROUPING SETS(course, year, ());
 SELECT course, year, SUM(earnings) FROM courseSales GROUP BY course, year GROUPING SETS(course);
 SELECT course, year, SUM(earnings) FROM courseSales GROUP BY course, year GROUPING SETS(year);
 
+-- Partial ROLLUP/CUBE/GROUPING SETS
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY course, CUBE(course, year) ORDER BY course, year;
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY CUBE(course, year), ROLLUP(course, year) ORDER BY course, year;
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY CUBE(course, year), ROLLUP(course, year), GROUPING SETS(course, year) ORDER BY course, year;
+
 -- GROUPING SETS with aggregate functions containing groupBy columns
 SELECT course, SUM(earnings) AS sum FROM courseSales
 GROUP BY course, earnings GROUPING SETS((), (course), (course, earnings)) ORDER BY course, sum;
@@ -59,4 +69,25 @@ SELECT course, year FROM courseSales GROUP BY CUBE(course, year) ORDER BY groupi
 -- Aliases in SELECT could be used in ROLLUP/CUBE/GROUPING SETS
 SELECT a + b AS k1, b AS k2, SUM(a - b) FROM testData GROUP BY CUBE(k1, k2);
 SELECT a + b AS k, b, SUM(a - b) FROM testData GROUP BY ROLLUP(k, b);
-SELECT a + b, b AS k, SUM(a - b) FROM testData GROUP BY a + b, k GROUPING SETS(k)
+SELECT a + b, b AS k, SUM(a - b) FROM testData GROUP BY a + b, k GROUPING SETS(k);
+
+-- GROUP BY use mixed Separate columns and CUBE/ROLLUP/Gr
+SELECT a, b, count(1) FROM testData GROUP BY a, b, CUBE(a, b);
+SELECT a, b, count(1) FROM testData GROUP BY a, b, ROLLUP(a, b);
+SELECT a, b, count(1) FROM testData GROUP BY CUBE(a, b), ROLLUP(a, b);
+SELECT a, b, count(1) FROM testData GROUP BY a, CUBE(a, b), ROLLUP(b);
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS((a, b), (a), ());
+SELECT a, b, count(1) FROM testData GROUP BY a, CUBE(a, b), GROUPING SETS((a, b), (a), ());
+SELECT a, b, count(1) FROM testData GROUP BY a, CUBE(a, b), ROLLUP(a, b), GROUPING SETS((a, b), (a), ());
+
+-- Support nested CUBE/ROLLUP/GROUPING SETS in GROUPING SETS
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS(ROLLUP(a, b));
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS(GROUPING SETS((a, b), (a), ()));
+
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS((a, b), GROUPING SETS(ROLLUP(a, b)));
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS((a, b, a, b), (a, b, a), (a, b));
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS(GROUPING SETS((a, b, a, b), (a, b, a), (a, b)));
+
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS(ROLLUP(a, b), CUBE(a, b));
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS(GROUPING SETS((a, b), (a), ()), GROUPING SETS((a, b), (a), (b), ()));
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS((a, b), (a), (), (a, b), (a), (b), ());
diff --git a/sql/core/src/test/resources/sql-tests/inputs/group-by-filter.sql b/sql/core/src/test/resources/sql-tests/inputs/group-by-filter.sql
index beb5b9e5fe516..c1ccb654ee085 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/group-by-filter.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/group-by-filter.sql
@@ -1,4 +1,10 @@
--- Test filter clause for aggregate expression.
+-- Test filter clause for aggregate expression with codegen on and off.
+--CONFIG_DIM1 spark.sql.codegen.wholeStage=true
+--CONFIG_DIM1 spark.sql.codegen.wholeStage=false,spark.sql.codegen.factoryMode=CODEGEN_ONLY
+--CONFIG_DIM1 spark.sql.codegen.wholeStage=false,spark.sql.codegen.factoryMode=NO_CODEGEN
+
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false
 
 -- Test data.
 CREATE OR REPLACE TEMPORARY VIEW testData AS SELECT * FROM VALUES
@@ -33,8 +39,21 @@ SELECT COUNT(id) FILTER (WHERE hiredate = date "2001-01-01") FROM emp;
 SELECT COUNT(id) FILTER (WHERE hiredate = to_date('2001-01-01 00:00:00')) FROM emp;
 SELECT COUNT(id) FILTER (WHERE hiredate = to_timestamp("2001-01-01 00:00:00")) FROM emp;
 SELECT COUNT(id) FILTER (WHERE date_format(hiredate, "yyyy-MM-dd") = "2001-01-01") FROM emp;
--- [SPARK-30276] Support Filter expression allows simultaneous use of DISTINCT
--- SELECT COUNT(DISTINCT id) FILTER (WHERE date_format(hiredate, "yyyy-MM-dd HH:mm:ss") = "2001-01-01 00:00:00") FROM emp;
+SELECT COUNT(DISTINCT id) FILTER (WHERE date_format(hiredate, "yyyy-MM-dd HH:mm:ss") = "2001-01-01 00:00:00") FROM emp;
+SELECT COUNT(DISTINCT id), COUNT(DISTINCT id) FILTER (WHERE date_format(hiredate, "yyyy-MM-dd HH:mm:ss") = "2001-01-01 00:00:00") FROM emp;
+SELECT COUNT(DISTINCT id) FILTER (WHERE hiredate = to_timestamp("2001-01-01 00:00:00")), COUNT(DISTINCT id) FILTER (WHERE hiredate = to_date('2001-01-01 00:00:00')) FROM emp;
+SELECT SUM(salary), COUNT(DISTINCT id), COUNT(DISTINCT id) FILTER (WHERE hiredate = date "2001-01-01") FROM emp;
+SELECT COUNT(DISTINCT 1) FILTER (WHERE a = 1) FROM testData;
+SELECT COUNT(DISTINCT id) FILTER (WHERE true) FROM emp;
+SELECT COUNT(DISTINCT id) FILTER (WHERE false) FROM emp;
+SELECT COUNT(DISTINCT 2), COUNT(DISTINCT 2,3) FILTER (WHERE dept_id = 40) FROM emp;
+SELECT COUNT(DISTINCT 2), COUNT(DISTINCT 3,2) FILTER (WHERE dept_id = 40) FROM emp;
+SELECT COUNT(DISTINCT 2), COUNT(DISTINCT 2,3) FILTER (WHERE dept_id > 0) FROM emp;
+SELECT COUNT(DISTINCT 2), COUNT(DISTINCT 3,2) FILTER (WHERE dept_id > 0) FROM emp;
+SELECT COUNT(DISTINCT id), COUNT(DISTINCT 2,3) FILTER (WHERE dept_id = 40) FROM emp;
+SELECT COUNT(DISTINCT id), COUNT(DISTINCT 3,2) FILTER (WHERE dept_id = 40) FROM emp;
+SELECT COUNT(DISTINCT id), COUNT(DISTINCT 2,3) FILTER (WHERE dept_id > 0) FROM emp;
+SELECT COUNT(DISTINCT id), COUNT(DISTINCT 3,2) FILTER (WHERE dept_id > 0) FROM emp;
 
 -- Aggregate with filter and non-empty GroupBy expressions.
 SELECT a, COUNT(b) FILTER (WHERE a >= 2) FROM testData GROUP BY a;
@@ -44,8 +63,11 @@ SELECT dept_id, SUM(salary) FILTER (WHERE hiredate > date "2003-01-01") FROM emp
 SELECT dept_id, SUM(salary) FILTER (WHERE hiredate > to_date("2003-01-01")) FROM emp GROUP BY dept_id;
 SELECT dept_id, SUM(salary) FILTER (WHERE hiredate > to_timestamp("2003-01-01 00:00:00")) FROM emp GROUP BY dept_id;
 SELECT dept_id, SUM(salary) FILTER (WHERE date_format(hiredate, "yyyy-MM-dd") > "2003-01-01") FROM emp GROUP BY dept_id;
--- [SPARK-30276] Support Filter expression allows simultaneous use of DISTINCT
--- SELECT dept_id, SUM(DISTINCT salary) FILTER (WHERE date_format(hiredate, "yyyy-MM-dd HH:mm:ss") > "2001-01-01 00:00:00") FROM emp GROUP BY dept_id;
+SELECT dept_id, SUM(DISTINCT salary) FILTER (WHERE date_format(hiredate, "yyyy-MM-dd HH:mm:ss") > "2001-01-01 00:00:00") FROM emp GROUP BY dept_id;
+SELECT dept_id, SUM(DISTINCT salary), SUM(DISTINCT salary) FILTER (WHERE date_format(hiredate, "yyyy-MM-dd HH:mm:ss") > "2001-01-01 00:00:00") FROM emp GROUP BY dept_id;
+SELECT dept_id, SUM(DISTINCT salary) FILTER (WHERE hiredate > date "2001-01-01"), SUM(DISTINCT salary) FILTER (WHERE date_format(hiredate, "yyyy-MM-dd HH:mm:ss") > "2001-01-01 00:00:00") FROM emp GROUP BY dept_id;
+SELECT dept_id, COUNT(id), SUM(DISTINCT salary), SUM(DISTINCT salary) FILTER (WHERE date_format(hiredate, "yyyy-MM-dd") > "2001-01-01") FROM emp GROUP BY dept_id;
+SELECT b, COUNT(DISTINCT 1) FILTER (WHERE a = 1) FROM testData GROUP BY b;
 
 -- Aggregate with filter and grouped by literals.
 SELECT 'foo', COUNT(a) FILTER (WHERE b <= 2) FROM testData GROUP BY 1;
@@ -58,13 +80,24 @@ select dept_id, count(distinct emp_name), count(distinct hiredate), sum(salary),
 select dept_id, count(distinct emp_name), count(distinct hiredate), sum(salary), sum(salary) filter (where id + dept_id > 500) from emp group by dept_id;
 select dept_id, count(distinct emp_name), count(distinct hiredate), sum(salary) filter (where salary < 400.00D), sum(salary) filter (where id > 200) from emp group by dept_id;
 select dept_id, count(distinct emp_name), count(distinct hiredate), sum(salary) filter (where salary < 400.00D), sum(salary) filter (where id + dept_id > 500) from emp group by dept_id;
--- [SPARK-30276] Support Filter expression allows simultaneous use of DISTINCT
--- select dept_id, count(distinct emp_name) filter (where id > 200), count(distinct hiredate), sum(salary) from emp group by dept_id;
--- select dept_id, count(distinct emp_name) filter (where id > 200), count(distinct hiredate) filter (where hiredate > date "2003-01-01"), sum(salary) from emp group by dept_id;
--- select dept_id, count(distinct emp_name) filter (where id > 200), count(distinct hiredate) filter (where hiredate > date "2003-01-01"), sum(salary) filter (where salary < 400.00D) from emp group by dept_id;
--- select dept_id, count(distinct emp_name) filter (where id > 200), count(distinct hiredate) filter (where hiredate > date "2003-01-01"), sum(salary) filter (where salary < 400.00D), sum(salary) filter (where id > 200) from emp group by dept_id;
--- select dept_id, count(distinct emp_name) filter (where id > 200), count(distinct emp_name), sum(salary) from emp group by dept_id;
--- select dept_id, count(distinct emp_name) filter (where id > 200), count(distinct emp_name) filter (where hiredate > date "2003-01-01"), sum(salary) from emp group by dept_id;
+select dept_id, count(distinct emp_name) filter (where id > 200), sum(salary) from emp group by dept_id;
+select dept_id, count(distinct emp_name) filter (where id + dept_id > 500), sum(salary) from emp group by dept_id;
+select dept_id, count(distinct emp_name), count(distinct emp_name) filter (where id > 200), sum(salary) from emp group by dept_id;
+select dept_id, count(distinct emp_name), count(distinct emp_name) filter (where id + dept_id > 500), sum(salary) from emp group by dept_id;
+select dept_id, count(distinct emp_name), count(distinct emp_name) filter (where id > 200), sum(salary), sum(salary) filter (where id > 200) from emp group by dept_id;
+select dept_id, count(distinct emp_name), count(distinct emp_name) filter (where id + dept_id > 500), sum(salary), sum(salary) filter (where id > 200) from emp group by dept_id;
+select dept_id, count(distinct emp_name) filter (where id > 200), count(distinct hiredate), sum(salary) from emp group by dept_id;
+select dept_id, count(distinct emp_name) filter (where id > 200), count(distinct hiredate) filter (where hiredate > date "2003-01-01"), sum(salary) from emp group by dept_id;
+select dept_id, count(distinct emp_name) filter (where id > 200), count(distinct hiredate) filter (where hiredate > date "2003-01-01"), sum(salary) filter (where salary < 400.00D) from emp group by dept_id;
+select dept_id, count(distinct emp_name) filter (where id > 200), count(distinct hiredate) filter (where hiredate > date "2003-01-01"), sum(salary) filter (where salary < 400.00D), sum(salary) filter (where id > 200) from emp group by dept_id;
+select dept_id, count(distinct emp_name) filter (where id > 200), count(distinct emp_name), sum(salary) from emp group by dept_id;
+select dept_id, count(distinct emp_name) filter (where id > 200), count(distinct emp_name) filter (where hiredate > date "2003-01-01"), sum(salary) from emp group by dept_id;
+select dept_id, sum(distinct (id + dept_id)) filter (where id > 200), count(distinct hiredate), sum(salary) from emp group by dept_id;
+select dept_id, sum(distinct (id + dept_id)) filter (where id > 200), count(distinct hiredate) filter (where hiredate > date "2003-01-01"), sum(salary) from emp group by dept_id;
+select dept_id, avg(distinct (id + dept_id)) filter (where id > 200), count(distinct hiredate) filter (where hiredate > date "2003-01-01"), sum(salary) filter (where salary < 400.00D) from emp group by dept_id;
+select dept_id, count(distinct emp_name, hiredate) filter (where id > 200), sum(salary) from emp group by dept_id;
+select dept_id, count(distinct emp_name, hiredate) filter (where id > 0), sum(salary) from emp group by dept_id;
+select dept_id, count(distinct 1), count(distinct 1) filter (where id > 200), sum(salary) from emp group by dept_id;
 
 -- Aggregate with filter and grouped by literals (hash aggregate), here the input table is filtered using WHERE.
 SELECT 'foo', APPROX_COUNT_DISTINCT(a) FILTER (WHERE b >= 0) FROM testData WHERE a = 0 GROUP BY 1;
@@ -78,9 +111,8 @@ SELECT a + 2, COUNT(b) FILTER (WHERE b IN (1, 2)) FROM testData GROUP BY a + 1;
 SELECT a + 1 + 1, COUNT(b) FILTER (WHERE b > 0) FROM testData GROUP BY a + 1;
 
 -- Aggregate with filter, foldable input and multiple distinct groups.
--- [SPARK-30276] Support Filter expression allows simultaneous use of DISTINCT
--- SELECT COUNT(DISTINCT b) FILTER (WHERE b > 0), COUNT(DISTINCT b, c) FILTER (WHERE b > 0 AND c > 2)
--- FROM (SELECT 1 AS a, 2 AS b, 3 AS c) GROUP BY a;
+SELECT COUNT(DISTINCT b) FILTER (WHERE b > 0), COUNT(DISTINCT b, c) FILTER (WHERE b > 0 AND c > 2)
+FROM (SELECT 1 AS a, 2 AS b, 3 AS c) GROUP BY a;
 
 -- Check analysis exceptions
 SELECT a AS k, COUNT(b) FILTER (WHERE b > 0) FROM testData GROUP BY k;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/group-by-ordinal.sql b/sql/core/src/test/resources/sql-tests/inputs/group-by-ordinal.sql
index 3144833b608be..b773396c050d2 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/group-by-ordinal.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/group-by-ordinal.sql
@@ -54,6 +54,41 @@ select count(a), a from (select 1 as a) tmp group by 2 having a > 0;
 -- mixed cases: group-by ordinals and aliases
 select a, a AS k, count(b) from data group by k, 1;
 
+-- can use ordinal in CUBE
+select a, b, count(1) from data group by cube(1, 2);
+
+-- mixed cases: can use ordinal in CUBE
+select a, b, count(1) from data group by cube(1, b);
+
+-- can use ordinal with cube
+select a, b, count(1) from data group by 1, 2 with cube;
+
+-- can use ordinal in ROLLUP
+select a, b, count(1) from data group by rollup(1, 2);
+
+-- mixed cases: can use ordinal in ROLLUP
+select a, b, count(1) from data group by rollup(1, b);
+
+-- can use ordinal with rollup
+select a, b, count(1) from data group by 1, 2 with rollup;
+
+-- can use ordinal in GROUPING SETS
+select a, b, count(1) from data group by grouping sets((1), (2), (1, 2));
+
+-- mixed cases: can use ordinal in GROUPING SETS
+select a, b, count(1) from data group by grouping sets((1), (b), (a, 2));
+
+select a, b, count(1) from data group by a, 2 grouping sets((1), (b), (a, 2));
+
+-- range error
+select a, b, count(1) from data group by a, -1;
+
+select a, b, count(1) from data group by a, 3;
+
+select a, b, count(1) from data group by cube(-1, 2);
+
+select a, b, count(1) from data group by cube(1, 3);
+
 -- turn off group by ordinal
 set spark.sql.groupByOrdinal=false;
 
diff --git a/sql/core/src/test/resources/sql-tests/inputs/group-by.sql b/sql/core/src/test/resources/sql-tests/inputs/group-by.sql
index fedf03d774e42..e2c3672a24dcb 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/group-by.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/group-by.sql
@@ -86,6 +86,16 @@ SELECT 1 FROM range(10) HAVING MAX(id) > 0;
 
 SELECT id FROM range(10) HAVING id > 0;
 
+SET spark.sql.legacy.parser.havingWithoutGroupByAsWhere=true;
+
+SELECT 1 FROM range(10) HAVING true;
+
+SELECT 1 FROM range(10) HAVING MAX(id) > 0;
+
+SELECT id FROM range(10) HAVING id > 0;
+
+SET spark.sql.legacy.parser.havingWithoutGroupByAsWhere=false;
+
 -- Test data
 CREATE OR REPLACE TEMPORARY VIEW test_agg AS SELECT * FROM VALUES
   (1, true), (1, false),
@@ -166,3 +176,16 @@ SELECT * FROM (SELECT COUNT(*) AS cnt FROM test_agg) WHERE cnt > 1L;
 SELECT count(*) FROM test_agg WHERE count(*) > 1L;
 SELECT count(*) FROM test_agg WHERE count(*) + 1L > 1L;
 SELECT count(*) FROM test_agg WHERE k = 1 or k = 2 or count(*) + 1L > 1L or max(k) > 1;
+
+-- Aggregate with multiple distinct decimal columns
+SELECT AVG(DISTINCT decimal_col), SUM(DISTINCT decimal_col) FROM VALUES (CAST(1 AS DECIMAL(9, 0))) t(decimal_col);
+
+-- SPARK-34581: Don't optimize out grouping expressions from aggregate expressions without aggregate function
+SELECT not(a IS NULL), count(*) AS c
+FROM testData
+GROUP BY a IS NULL;
+
+SELECT if(not(a IS NULL), rand(0), 1), count(*) AS c
+FROM testData
+GROUP BY a IS NULL;
+
diff --git a/sql/core/src/test/resources/sql-tests/inputs/having.sql b/sql/core/src/test/resources/sql-tests/inputs/having.sql
index 3b75be19b5677..2799b1a94d085 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/having.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/having.sql
@@ -24,3 +24,9 @@ SELECT SUM(a) AS b, CAST('2020-01-01' AS DATE) AS fake FROM VALUES (1, 10), (2,
 SELECT SUM(a) AS b FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY GROUPING SETS ((b), (a, b)) HAVING b > 10;
 SELECT SUM(a) AS b FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY CUBE(a, b) HAVING b > 10;
 SELECT SUM(a) AS b FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY ROLLUP(a, b) HAVING b > 10;
+
+-- SPARK-33131: Grouping sets with having clause can not resolve qualified col name.
+SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY GROUPING SETS(t.c1) HAVING t.c1 = 1;
+SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY CUBE(t.c1) HAVING t.c1 = 1;
+SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY ROLLUP(t.c1) HAVING t.c1 = 1;
+SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY t.c1 HAVING t.c1 = 1;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/higher-order-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/higher-order-functions.sql
index cfa06aea82b04..0b7bf517f916b 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/higher-order-functions.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/higher-order-functions.sql
@@ -92,3 +92,6 @@ select transform_values(ys, (k, v) -> k + v) as v from nested;
 -- use non reversed keywords: all is non reversed only if !ansi
 select transform(ys, all -> all * all) as v from values (array(32, 97)) as t(ys);
 select transform(ys, (all, i) -> all + i) as v from values (array(32, 97)) as t(ys);
+
+-- SPARK-32819: Aggregate on nested string arrays
+select aggregate(split('abcdefgh',''), array(array('')), (acc, x) -> array(array(x)));
\ No newline at end of file
diff --git a/sql/core/src/test/resources/sql-tests/inputs/ignored.sql b/sql/core/src/test/resources/sql-tests/inputs/ignored.sql
new file mode 100644
index 0000000000000..0b5331a1d0229
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/ignored.sql
@@ -0,0 +1,4 @@
+-- This is a query file that has been ignored.
+-- It includes a query that should crash Spark.
+-- If the test case is run, the whole suite would fail.
+some random not working query that should crash Spark.
diff --git a/sql/core/src/test/resources/sql-tests/inputs/intersect-all.sql b/sql/core/src/test/resources/sql-tests/inputs/intersect-all.sql
index b0b2244048caa..077caa5dd44a0 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/intersect-all.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/intersect-all.sql
@@ -155,6 +155,21 @@ SELECT * FROM tab2;
 -- Restore the property
 SET spark.sql.legacy.setopsPrecedence.enabled = false;
 
+-- SPARK-32638: corrects references when adding aliases in WidenSetOperationTypes
+CREATE OR REPLACE TEMPORARY VIEW tab3 AS VALUES (decimal(1)), (decimal(2)) tbl3(v);
+SELECT t.v FROM (
+  SELECT v FROM tab3
+  INTERSECT
+  SELECT v + v AS v FROM tab3
+) t;
+
+SELECT SUM(t.v) FROM (
+  SELECT v FROM tab3
+  INTERSECT
+  SELECT v + v AS v FROM tab3
+) t;
+
 -- Clean-up 
 DROP VIEW IF EXISTS tab1;
 DROP VIEW IF EXISTS tab2;
+DROP VIEW IF EXISTS tab3;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval.sql b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
index 76d4a8c87eef5..063727a76ea6b 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/interval.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
@@ -20,6 +20,12 @@ select -interval '-1 month 1 day -1 second';
 select -interval -1 month 1 day -1 second;
 select +interval '-1 month 1 day -1 second';
 select +interval -1 month 1 day -1 second;
+select interval -'1-1' year to month;
+select interval -'-1-1' year to month;
+select interval +'-1-1' year to month;
+select interval - '1 2:3:4.001' day to second;
+select interval +'1 2:3:4.001' day to second;
+select interval -'-1 2:3:4.001' day to second;
 
 -- make intervals
 select make_interval(1);
@@ -29,6 +35,8 @@ select make_interval(1, 2, 3, 4);
 select make_interval(1, 2, 3, 4, 5);
 select make_interval(1, 2, 3, 4, 5, 6);
 select make_interval(1, 2, 3, 4, 5, 6, 7.008009);
+select make_interval(1, 2, 3, 4, 0, 0, 123456789012.123456);
+select make_interval(0, 0, 0, 0, 0, 0, 1234567890123456789);
 
 -- cast string to intervals
 select cast('1 second' as interval);
@@ -43,6 +51,8 @@ select cast('- +1 second' as interval);
 select interval 13.123456789 seconds, interval -13.123456789 second;
 select interval 1 year 2 month 3 week 4 day 5 hour 6 minute 7 seconds 8 millisecond 9 microsecond;
 select interval '30' year '25' month '-100' day '40' hour '80' minute '299.889987299' second;
+select interval '0-0' year to month;
+select interval '0 0:0:0' day to second;
 select interval '0 0:0:0.1' day to second;
 select interval '10-9' year to month;
 select interval '20 15' day to hour;
@@ -53,6 +63,7 @@ select interval '15:40:32.99899999' hour to second;
 select interval '40:32.99899999' minute to second;
 select interval '40:32' minute to second;
 select interval 30 day day;
+select interval 30 days days;
 
 -- invalid day-time string intervals
 select interval '20 15:40:32.99899999' day to hour;
@@ -88,6 +99,13 @@ select interval '12:11:10' hour to second '1' year;
 select interval (-30) day;
 select interval (a + 1) day;
 select interval 30 day day day;
+select interval (-30) days;
+select interval (a + 1) days;
+select interval 30 days days days;
+SELECT INTERVAL '178956970-7' YEAR TO MONTH;
+SELECT INTERVAL '178956970-8' YEAR TO MONTH;
+SELECT INTERVAL '-178956970-8' YEAR TO MONTH;
+SELECT INTERVAL -'178956970-8' YEAR TO MONTH;
 
 -- Interval year-month arithmetic
 
@@ -169,6 +187,10 @@ select interval '2-2\t' year to month;
 select interval '-\t2-2\t' year to month;
 select interval '\n0 12:34:46.789\t' day to second;
 select interval '\n-\t10\t 12:34:46.789\t' day to second;
+select interval '中文 interval 1 day';
+select interval 'interval中文 1 day';
+select interval 'interval 1中文day';
+
 
 -- interval overflow if (ansi) exception else NULL
 select -(a) from values (interval '-2147483648 months', interval '2147483647 months') t(a, b);
@@ -188,3 +210,29 @@ SELECT
   to_json(from_json('{"a":"1 days"}', 'a interval')),
   to_json(map('a', interval 25 month 100 day 130 minute)),
   from_json(to_json(map('a', interval 25 month 100 day 130 minute)), 'a interval');
+
+select interval '+';
+select interval '+.';
+select interval '1';
+select interval '1.2';
+select interval '- 2';
+select interval '1 day -';
+select interval '1 day 1';
+
+select interval '1 day 2' day;
+select interval 'interval 1' day;
+select interval '-\t 1' day;
+
+SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / 2;
+SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / 5;
+SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1;
+SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1L;
+SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1.0;
+SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1.0D;
+
+SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / 2;
+SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / 5;
+SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1;
+SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1L;
+SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1.0;
+SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1.0D;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql
index 131890fddb0db..f6fa44161a771 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql
@@ -48,6 +48,21 @@ select from_json('[null, {"a":2}]', 'array<struct<a:int>>');
 select from_json('[{"a": 1}, {"b":2}]', 'array<map<string,int>>');
 select from_json('[{"a": 1}, 2]', 'array<map<string,int>>');
 
+-- from_json - datetime type
+select from_json('{"d": "2012-12-15", "t": "2012-12-15 15:15:15"}', 'd date, t timestamp');
+select from_json(
+  '{"d": "12/15 2012", "t": "12/15 2012 15:15:15"}',
+  'd date, t timestamp',
+  map('dateFormat', 'MM/dd yyyy', 'timestampFormat', 'MM/dd yyyy HH:mm:ss'));
+select from_json(
+  '{"d": "02-29"}',
+  'd date',
+  map('dateFormat', 'MM-dd'));
+select from_json(
+  '{"t": "02-29"}',
+  't timestamp',
+  map('timestampFormat', 'MM-dd'));
+
 -- to_json - array type
 select to_json(array('1', '2', '3'));
 select to_json(array(array(1, 2, 3), array(4)));
diff --git a/sql/core/src/test/resources/sql-tests/inputs/like-all.sql b/sql/core/src/test/resources/sql-tests/inputs/like-all.sql
index a084dbef61a0c..51b689607e8e3 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/like-all.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/like-all.sql
@@ -1,3 +1,5 @@
+-- test cases for like all
+
 CREATE OR REPLACE TEMPORARY VIEW like_all_table AS SELECT * FROM (VALUES
   ('google', '%oo%'),
   ('facebook', '%oo%'),
diff --git a/sql/core/src/test/resources/sql-tests/inputs/like-any.sql b/sql/core/src/test/resources/sql-tests/inputs/like-any.sql
index 5758a2a494944..a6e9827d58d94 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/like-any.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/like-any.sql
@@ -1,3 +1,5 @@
+-- test cases for like any
+
 CREATE OR REPLACE TEMPORARY VIEW like_any_table AS SELECT * FROM (VALUES
   ('google', '%oo%'),
   ('facebook', '%oo%'),
diff --git a/sql/core/src/test/resources/sql-tests/inputs/literals.sql b/sql/core/src/test/resources/sql-tests/inputs/literals.sql
index 108cfd766af2c..9f0eefc16a8cd 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/literals.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/literals.sql
@@ -45,6 +45,11 @@ select 9223372036854775808, -9223372036854775809;
 select 1234567890123456789012345678901234567890;
 select 1234567890123456789012345678901234567890.0;
 
+-- float
+select 1F, 1.2F, .10f, 0.10f;
+select -1F, -1.2F, -.10F, -0.10F;
+select -3.4028235E39f;
+
 -- double
 select 1D, 1.2D, 1e10, 1.5e5, .10D, 0.10D, .1e5, .9e+2, 0.9e+2, 900e-1, 9.e+1;
 select -1D, -1.2D, -1e10, -1.5e5, -.10D, -0.10D, -.1e5;
@@ -55,6 +60,7 @@ select 1E309, -1E309;
 
 -- decimal parsing
 select 0.3, -0.8, .5, -.18, 0.1111, .1111;
+select 0.3 F, 0.4 D, 0.5 BD;
 
 -- super large scientific notation double literals should still be valid doubles
 select 123456789012345678901234567890123456789e10d, 123456789012345678901234567890123456789.1e10d;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/map.sql b/sql/core/src/test/resources/sql-tests/inputs/map.sql
new file mode 100644
index 0000000000000..e2d855fba154e
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/map.sql
@@ -0,0 +1,5 @@
+-- test cases for map functions
+
+-- key does not exist
+select element_at(map(1, 'a', 2, 'b'), 5);
+select map(1, 'a', 2, 'b')[5];
diff --git a/sql/core/src/test/resources/sql-tests/inputs/misc-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/misc-functions.sql
index 95f71925e9294..907ff33000d8e 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/misc-functions.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/misc-functions.sql
@@ -8,3 +8,15 @@ select typeof(cast(1.0 as float)), typeof(1.0D), typeof(1.2);
 select typeof(date '1986-05-23'),  typeof(timestamp '1986-05-23'), typeof(interval '23 days');
 select typeof(x'ABCD'), typeof('SPARK');
 select typeof(array(1, 2)), typeof(map(1, 2)), typeof(named_struct('a', 1, 'b', 'spark'));
+
+-- Spark-32793: Rewrite AssertTrue with RaiseError
+SELECT assert_true(true), assert_true(boolean(1));
+SELECT assert_true(false);
+SELECT assert_true(boolean(0));
+SELECT assert_true(null);
+SELECT assert_true(boolean(null));
+SELECT assert_true(false, 'custom error message');
+
+CREATE TEMPORARY VIEW tbl_misc AS SELECT * FROM (VALUES (1), (8), (2)) AS T(v);
+SELECT raise_error('error message');
+SELECT if(v > 5, raise_error('too big: ' || v), v + 1) FROM tbl_misc;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/natural-join.sql b/sql/core/src/test/resources/sql-tests/inputs/natural-join.sql
index 71a50157b766c..060f15e3d2e87 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/natural-join.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/natural-join.sql
@@ -10,6 +10,19 @@ create temporary view nt2 as select * from values
   ("one", 5)
   as nt2(k, v2);
 
+create temporary view nt3 as select * from values
+  ("one", 4),
+  ("two", 5),
+  ("one", 6)
+  as nt3(k, v3);
+
+create temporary view nt4 as select * from values
+  ("one", 7),
+  ("two", 8),
+  ("one", 9)
+  as nt4(k, v4);
+
+SELECT * FROM nt1 natural join nt2;
 
 SELECT * FROM nt1 natural join nt2 where k = "one";
 
@@ -18,3 +31,43 @@ SELECT * FROM nt1 natural left join nt2 order by v1, v2;
 SELECT * FROM nt1 natural right join nt2 order by v1, v2;
 
 SELECT count(*) FROM nt1 natural full outer join nt2;
+
+SELECT k FROM nt1 natural join nt2;
+
+SELECT k FROM nt1 natural join nt2 where k = "one";
+
+SELECT nt1.* FROM nt1 natural join nt2;
+
+SELECT nt2.* FROM nt1 natural join nt2;
+
+SELECT sbq.* from (SELECT * FROM nt1 natural join nt2) sbq;
+
+SELECT sbq.k from (SELECT * FROM nt1 natural join nt2) sbq;
+
+SELECT nt1.*, nt2.* FROM nt1 natural join nt2;
+
+SELECT *, nt2.k FROM nt1 natural join nt2;
+
+SELECT nt1.k, nt2.k FROM nt1 natural join nt2;
+
+SELECT nt1.k, nt2.k FROM nt1 natural join nt2 where k = "one";
+
+SELECT * FROM (SELECT * FROM nt1 natural join nt2);
+
+SELECT * FROM (SELECT nt1.*, nt2.* FROM nt1 natural join nt2);
+
+SELECT * FROM (SELECT nt1.v1, nt2.k FROM nt1 natural join nt2);
+
+SELECT nt2.k FROM (SELECT * FROM nt1 natural join nt2);
+
+SELECT * FROM nt1 natural join nt2 natural join nt3;
+
+SELECT nt1.*, nt2.*, nt3.* FROM nt1 natural join nt2 natural join nt3;
+
+SELECT nt1.*, nt2.*, nt3.* FROM nt1 natural join nt2 join nt3 on nt2.k = nt3.k;
+
+SELECT * FROM nt1 natural join nt2 join nt3 on nt1.k = nt3.k;
+
+SELECT * FROM nt1 natural join nt2 join nt3 on nt2.k = nt3.k;
+
+SELECT nt1.*, nt2.*, nt3.*, nt4.* FROM nt1 natural join nt2 natural join nt3 natural join nt4;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/operators.sql b/sql/core/src/test/resources/sql-tests/inputs/operators.sql
index 20bf0eb15c5b2..c296fa5f7e87e 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/operators.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/operators.sql
@@ -81,3 +81,17 @@ select positive('-1.11'), positive(-1.11), negative('-1.11'), negative(-1.11);
 -- pmod
 select pmod(-7, 2), pmod(0, 2), pmod(7, 0), pmod(7, null), pmod(null, 2), pmod(null, null);
 select pmod(cast(3.13 as decimal), cast(0 as decimal)), pmod(cast(2 as smallint), cast(0 as smallint));
+
+-- width_bucket
+select width_bucket(5.35, 0.024, 10.06, 5);
+select width_bucket(5.35, 0.024, 10.06, 3 + 2);
+select width_bucket('5.35', '0.024', '10.06', '5');
+select width_bucket(5.35, 0.024, 10.06, 2.5);
+select width_bucket(5.35, 0.024, 10.06, 0.5);
+select width_bucket(null, 0.024, 10.06, 5);
+select width_bucket(5.35, null, 10.06, 5);
+select width_bucket(5.35, 0.024, null, -5);
+select width_bucket(5.35, 0.024, 10.06, null);
+select width_bucket(5.35, 0.024, 10.06, -5);
+select width_bucket(5.35, 0.024, 10.06, 9223372036854775807L); -- long max value
+select width_bucket(5.35, 0.024, 10.06, 9223372036854775807L - 1);
diff --git a/sql/core/src/test/resources/sql-tests/inputs/parse-schema-string.sql b/sql/core/src/test/resources/sql-tests/inputs/parse-schema-string.sql
new file mode 100644
index 0000000000000..c67d45139fd6c
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/parse-schema-string.sql
@@ -0,0 +1,5 @@
+-- Use keywords as attribute names
+select from_csv('1', 'create INT');
+select from_csv('1', 'cube INT');
+select from_json('{"create":1}', 'create INT');
+select from_json('{"cube":1}', 'cube INT');
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part3.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part3.sql
index 746b677234832..657ea59ec8f11 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part3.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part3.sql
@@ -241,10 +241,9 @@ select sum(1/ten) filter (where ten > 0) from tenk1;
 -- select ten, sum(distinct four) filter (where four::text ~ '123') from onek a
 -- group by ten;
 
--- [SPARK-30276] Support Filter expression allows simultaneous use of DISTINCT
--- select ten, sum(distinct four) filter (where four > 10) from onek a
--- group by ten
--- having exists (select 1 from onek b where sum(distinct a.four) = b.four);
+select ten, sum(distinct four) filter (where four > 10) from onek a
+group by ten
+having exists (select 1 from onek b where sum(distinct a.four) = b.four);
 
 -- [SPARK-28682] ANSI SQL: Collation Support
 -- select max(foo COLLATE "C") filter (where (bar collate "POSIX") > '0')
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/case.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/case.sql
index 6d9c44c67a96b..b39ccb85fb366 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/case.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/case.sql
@@ -65,11 +65,11 @@ SELECT '7' AS `None`,
   CASE WHEN rand() < 0 THEN 1
   END AS `NULL on no matches`;
 
+-- [SPARK-33008] Spark SQL throws an exception
 -- Constant-expression folding shouldn't evaluate unreachable subexpressions
 SELECT CASE WHEN 1=0 THEN 1/0 WHEN 1=1 THEN 1 ELSE 2/0 END;
 SELECT CASE 1 WHEN 0 THEN 1/0 WHEN 1 THEN 1 ELSE 2/0 END;
 
--- [SPARK-27923] PostgreSQL throws an exception but Spark SQL is NULL
 -- However we do not currently suppress folding of potentially
 -- reachable subexpressions
 SELECT CASE WHEN i > 100 THEN 1/0 ELSE 0 END FROM case_tbl;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/create_view.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/create_view.sql
index 21ffd85f7d01f..2889941c1fcc1 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/create_view.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/create_view.sql
@@ -636,7 +636,7 @@ DESC TABLE vv6;
 -- Check cases involving dropped/altered columns in a function's rowtype result
 --
 
--- Skip the tests below because Spark does't support PostgreSQL-specific UDFs/transactions
+-- Skip the tests below because Spark doesn't support PostgreSQL-specific UDFs/transactions
 -- create table tt14t (f1 text, f2 text, f3 text, f4 text);
 -- insert into tt14t values('foo', 'bar', 'baz', '42');
 --
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/groupingsets.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/groupingsets.sql
index fc54d179f742c..45617c53166aa 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/groupingsets.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/groupingsets.sql
@@ -336,9 +336,8 @@ order by 2,1;
 -- order by 2,1;
 
 -- FILTER queries
--- [SPARK-30276] Support Filter expression allows simultaneous use of DISTINCT
--- select ten, sum(distinct four) filter (where string(four) like '123') from onek a
--- group by rollup(ten);
+select ten, sum(distinct four) filter (where string(four) like '123') from onek a
+group by rollup(ten);
 
 -- More rescan tests
 -- [SPARK-27877] ANSI SQL: LATERAL derived table(T491)
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/join.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/join.sql
index cc07b00cc3670..9d6d296ce4a5a 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/join.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/join.sql
@@ -743,20 +743,25 @@ select * from a left join b on i = x and i = y and x = i;
 --
 -- test NULL behavior of whole-row Vars, per bug #5025
 --
-select t1.q2, count(t2.*)
+--- [SPARK-34199] changed the `count(t2.*)` to `count(t2.q1, t2.q2)` since we have
+--- blocked `count(tblName.*)`. Besides this, in pgsql, `count(t2.*)` of outter join
+--- means how many matching rows produced by t2 while Spark SQL doesn't have this semantic.
+--- So here we use `count(t2.q1, t2.q2)` instead of `count(1)` to keep the query output
+--- unchanged.
+select t1.q2, count(t2.q1, t2.q2)
 from int8_tbl t1 left join int8_tbl t2 on (t1.q2 = t2.q1)
 group by t1.q2 order by 1;
 
-select t1.q2, count(t2.*)
+select t1.q2, count(t2.q1, t2.q2)
 from int8_tbl t1 left join (select * from int8_tbl) t2 on (t1.q2 = t2.q1)
 group by t1.q2 order by 1;
 
 -- [SPARK-28330] Enhance query limit
--- select t1.q2, count(t2.*)
+-- select t1.q2, count(t2.q1, t2.q2)
 -- from int8_tbl t1 left join (select * from int8_tbl offset 0) t2 on (t1.q2 = t2.q1)
 -- group by t1.q2 order by 1;
 
-select t1.q2, count(t2.*)
+select t1.q2, count(t2.q1, t2.q2)
 from int8_tbl t1 left join
   (select q1, case when q2=1 then 1 else q2 end as q2 from int8_tbl) t2
   on (t1.q2 = t2.q1)
@@ -884,7 +889,7 @@ from nt3 as nt3
     on ss2.id = nt3.nt2_id
 where nt3.id = 1 and ss2.b3;
 
--- [SPARK-28379] Correlated scalar subqueries must be aggregated
+-- Accessing outer query column is not allowed in LocalLimit
 --
 -- test case where a PlaceHolderVar is propagated into a subquery
 --
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql
index dbdb2cace0e0c..53f2aa41ae3fa 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql
@@ -754,22 +754,22 @@ DROP TABLE ceil_floor_round;
 -- 	round((2.5 * 10 ^ i)::numeric, -i)
 -- FROM generate_series(-5,5) AS t(i);
 
--- [SPARK-21117] Built-in SQL Function Support - WIDTH_BUCKET
 -- Testing for width_bucket(). For convenience, we test both the
 -- numeric and float8 versions of the function in this file.
 
 -- errors
--- SELECT width_bucket(5.0, 3.0, 4.0, 0);
--- SELECT width_bucket(5.0, 3.0, 4.0, -5);
--- SELECT width_bucket(3.5, 3.0, 3.0, 888);
--- SELECT width_bucket(5.0::float8, 3.0::float8, 4.0::float8, 0);
--- SELECT width_bucket(5.0::float8, 3.0::float8, 4.0::float8, -5);
--- SELECT width_bucket(3.5::float8, 3.0::float8, 3.0::float8, 888);
--- SELECT width_bucket('NaN', 3.0, 4.0, 888);
--- SELECT width_bucket(0::float8, 'NaN', 4.0::float8, 888);
+SELECT width_bucket(5.0, 3.0, 4.0, 0);
+SELECT width_bucket(5.0, 3.0, 4.0, -5);
+SELECT width_bucket(3.5, 3.0, 3.0, 888);
+SELECT width_bucket(double(5.0), double(3.0), double(4.0), 0);
+SELECT width_bucket(double(5.0), double(3.0), double(4.0), -5);
+SELECT width_bucket(double(3.5), double(3.0), double(3.0), 888);
+SELECT width_bucket('NaN', 3.0, 4.0, 888);
+SELECT width_bucket(double(0), 'NaN', double(4.0), 888);
 
 -- normal operation
 -- CREATE TABLE width_bucket_test (operand_num numeric, operand_f8 float8);
+CREATE TABLE width_bucket_test (operand_num decimal(30,15), operand_f8 double) USING parquet;
 
 -- COPY width_bucket_test (operand_num) FROM stdin;
 -- -5.2
@@ -795,28 +795,50 @@ DROP TABLE ceil_floor_round;
 
 -- UPDATE width_bucket_test SET operand_f8 = operand_num::float8;
 
--- SELECT
---     operand_num,
---     width_bucket(operand_num, 0, 10, 5) AS wb_1,
---     width_bucket(operand_f8, 0, 10, 5) AS wb_1f,
---     width_bucket(operand_num, 10, 0, 5) AS wb_2,
---     width_bucket(operand_f8, 10, 0, 5) AS wb_2f,
---     width_bucket(operand_num, 2, 8, 4) AS wb_3,
---     width_bucket(operand_f8, 2, 8, 4) AS wb_3f,
---     width_bucket(operand_num, 5.0, 5.5, 20) AS wb_4,
---     width_bucket(operand_f8, 5.0, 5.5, 20) AS wb_4f,
---     width_bucket(operand_num, -25, 25, 10) AS wb_5,
---     width_bucket(operand_f8, -25, 25, 10) AS wb_5f
---     FROM width_bucket_test;
+INSERT INTO width_bucket_test VALUES
+    (-5.2, -5.2),
+    (-0.0000000001, -0.0000000001),
+    (0.000000000001, 0.000000000001),
+    (1, 1),
+    (1.99999999999999, 1.99999999999999),
+    (2, 2),
+    (2.00000000000001, 2.00000000000001),
+    (3, 3),
+    (4, 4),
+    (4.5, 4.5),
+    (5, 5),
+    (5.5, 5.5),
+    (6, 6),
+    (7, 7),
+    (8, 8),
+    (9, 9),
+    (9.99999999999999, 9.99999999999999),
+    (10, 10),
+    (10.0000000000001, 10.0000000000001);
+
+SELECT
+    operand_num,
+    width_bucket(operand_num, 0, 10, 5) AS wb_1,
+    width_bucket(operand_f8, 0, 10, 5) AS wb_1f,
+    width_bucket(operand_num, 10, 0, 5) AS wb_2,
+    width_bucket(operand_f8, 10, 0, 5) AS wb_2f,
+    width_bucket(operand_num, 2, 8, 4) AS wb_3,
+    width_bucket(operand_f8, 2, 8, 4) AS wb_3f,
+    width_bucket(operand_num, 5.0, 5.5, 20) AS wb_4,
+    width_bucket(operand_f8, 5.0, 5.5, 20) AS wb_4f,
+    width_bucket(operand_num, -25, 25, 10) AS wb_5,
+    width_bucket(operand_f8, -25, 25, 10) AS wb_5f
+    FROM width_bucket_test
+    ORDER BY operand_num ASC;
 
 -- for float8 only, check positive and negative infinity: we require
 -- finite bucket bounds, but allow an infinite operand
--- SELECT width_bucket(0.0::float8, 'Infinity'::float8, 5, 10); -- error
--- SELECT width_bucket(0.0::float8, 5, '-Infinity'::float8, 20); -- error
--- SELECT width_bucket('Infinity'::float8, 1, 10, 10),
---        width_bucket('-Infinity'::float8, 1, 10, 10);
+SELECT width_bucket(double(0.0), double('Infinity'), 5, 10); -- error
+SELECT width_bucket(double(0.0), 5, double('-Infinity'), 20); -- error
+SELECT width_bucket(double('Infinity'), 1, 10, 10),
+       width_bucket(double('-Infinity'), 1, 10, 10);
 
--- DROP TABLE width_bucket_test;
+DROP TABLE width_bucket_test;
 
 -- [SPARK-28137] Missing Data Type Formatting Functions: TO_CHAR
 -- TO_CHAR()
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql
index 2edde8df08047..0efe0877e9b3e 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql
@@ -49,6 +49,7 @@ SELECT 1 AS one FROM test_having HAVING a > 1;
 SELECT 1 AS one FROM test_having HAVING 1 > 2;
 SELECT 1 AS one FROM test_having HAVING 1 < 2;
 
+-- [SPARK-33008] Spark SQL throws an exception
 -- and just to prove that we aren't scanning the table:
 SELECT 1 AS one FROM test_having WHERE 1/a = 1 HAVING 1 < 2;
 
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/strings.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/strings.sql
index e2a94404395bc..c252db7d3a8ae 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/strings.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/strings.sql
@@ -647,10 +647,9 @@ SELECT repeat('Pg', 4);
 SELECT repeat('Pg', -4);
 
 SELECT trim(binary('\\000') from binary('\\000Tom\\000'));
--- [SPARK-28123] Add support btrim
--- SELECT btrim(E'\\000trim\\000'::bytea, E'\\000'::bytea);
--- SELECT btrim(''::bytea, E'\\000'::bytea);
--- SELECT btrim(E'\\000trim\\000'::bytea, ''::bytea);
+SELECT btrim(binary('\\000trim\\000'), binary('\\000'));
+SELECT btrim(binary(''), binary('\\000'));
+SELECT btrim(binary('\\000trim\\000'), binary(''));
 -- [SPARK-28121] decode can not accept 'escape' as charset
 -- [SPARK-28412][SQL] ANSI SQL: OVERLAY function support byte array
 -- SELECT encode(overlay(E'Th\\000omas'::bytea placing E'Th\\001omas'::bytea from 2),'escape');
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part1.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part1.sql
index 087d7a5befd19..3ebe9f91b2a08 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part1.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part1.sql
@@ -95,7 +95,7 @@ SELECT last(ten) OVER (PARTITION BY four), ten, four FROM
 (SELECT * FROM tenk1 WHERE unique2 < 10 ORDER BY four, ten)s
 ORDER BY four, ten;
 
--- [SPARK-27951] ANSI SQL: NTH_VALUE function
+-- [SPARK-30707] Lead/Lag window function throws AnalysisException without ORDER BY clause
 -- SELECT nth_value(ten, four + 1) OVER (PARTITION BY four), ten, four
 -- FROM (SELECT * FROM tenk1 WHERE unique2 < 10 ORDER BY four, ten)s;
 
@@ -135,7 +135,7 @@ FROM tenk1 GROUP BY ten, two WINDOW win AS (PARTITION BY two ORDER BY ten);
 -- FROM empsalary WINDOW w1 AS (ORDER BY salary), w2 AS (ORDER BY salary);
 
 -- subplan
--- [SPARK-28379] Correlated scalar subqueries must be aggregated
+-- Cannot specify window frame for lead function
 -- SELECT lead(ten, (SELECT two FROM tenk1 WHERE s.unique2 = unique2)) OVER (PARTITION BY four ORDER BY ten)
 -- FROM tenk1 s WHERE unique2 < 10;
 
@@ -146,7 +146,7 @@ SELECT count(*) OVER (PARTITION BY four) FROM (SELECT * FROM tenk1 WHERE FALSE)s
 -- mixture of agg/wfunc in the same window
 -- SELECT sum(salary) OVER w, rank() OVER w FROM empsalary WINDOW w AS (PARTITION BY depname ORDER BY salary DESC);
 
--- Cannot safely cast 'enroll_date': StringType to DateType;
+-- Cannot safely cast 'enroll_date': string to date;
 -- SELECT empno, depname, salary, bonus, depadj, MIN(bonus) OVER (ORDER BY empno), MAX(depadj) OVER () FROM(
 -- SELECT *,
 --   CASE WHEN enroll_date < '2008-01-01' THEN 2008 - extract(year FROM enroll_date) END * 500 AS bonus,
@@ -301,7 +301,7 @@ FROM tenk1 WHERE unique1 < 10;
 -- unique1, four
 -- FROM tenk1 WHERE unique1 < 10 WINDOW w AS (order by four);
 
--- [SPARK-27951] ANSI SQL: NTH_VALUE function
+-- [SPARK-30707] Lead/Lag window function throws AnalysisException without ORDER BY clause
 -- SELECT first_value(unique1) over w,
 -- nth_value(unique1, 2) over w AS nth_2,
 -- last_value(unique1) over w, unique1, four
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part2.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part2.sql
index ba1acc9f56b4a..50c0bc3410312 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part2.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part2.sql
@@ -105,7 +105,7 @@ FROM tenk1 WHERE unique1 < 10;
 -- select sum(salary) over (order by enroll_date range between '1 year' preceding and '1 year' following
 --   exclude ties), salary, enroll_date from empsalary;
 
--- [SPARK-27951] ANSI SQL: NTH_VALUE function
+-- [SPARK-28310] ANSI SQL grammar support: first_value/last_value(expression, [RESPECT NULLS | IGNORE NULLS])
 -- select first_value(salary) over(order by salary range between 1000 preceding and 1000 following),
 -- lead(salary) over(order by salary range between 1000 preceding and 1000 following),
 -- nth_value(salary, 1) over(order by salary range between 1000 preceding and 1000 following),
@@ -116,7 +116,7 @@ FROM tenk1 WHERE unique1 < 10;
 -- lag(salary) over(order by salary range between 1000 preceding and 1000 following),
 -- salary from empsalary;
 
--- [SPARK-27951] ANSI SQL: NTH_VALUE function
+-- [SPARK-28310] ANSI SQL grammar support: first_value/last_value(expression, [RESPECT NULLS | IGNORE NULLS])
 -- select first_value(salary) over(order by salary range between 1000 following and 3000 following
 --   exclude current row),
 -- lead(salary) over(order by salary range between 1000 following and 3000 following exclude ties),
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part3.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part3.sql
index cd3b74b3aa03f..6f33a07631f7a 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part3.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part3.sql
@@ -42,7 +42,7 @@ create table datetimes (
     f_timestamp timestamp
 ) using parquet;
 
--- Spark cannot safely cast StringType to TimestampType
+-- Spark cannot safely cast string to timestamp
 -- [SPARK-29636] Spark can't parse '11:00 BST' or '2000-10-19 10:23:54+01' signatures to timestamp
 insert into datetimes values
 (1, timestamp '11:00', cast ('11:00 BST' as timestamp), cast ('1 year' as timestamp), cast ('2000-10-19 10:23:54+01' as timestamp), timestamp '2000-10-19 10:23:54'),
@@ -399,8 +399,7 @@ SELECT range(1, 100) OVER () FROM empsalary;
 
 SELECT ntile(0) OVER (ORDER BY ten), ten, four FROM tenk1;
 
--- [SPARK-27951] ANSI SQL: NTH_VALUE function
--- SELECT nth_value(four, 0) OVER (ORDER BY ten), ten, four FROM tenk1;
+SELECT nth_value(four, 0) OVER (ORDER BY ten), ten, four FROM tenk1;
 
 -- filter
 
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/with.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/with.sql
index 83c6fd8cbac91..a3e0b15b582f5 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/with.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/with.sql
@@ -931,7 +931,7 @@ SELECT * FROM outermost ORDER BY 1;
 -- data-modifying WITH containing INSERT...ON CONFLICT DO UPDATE
 -- [ORIGINAL SQL]
 --CREATE TABLE withz AS SELECT i AS k, (i || ' v')::text v FROM generate_series(1, 16, 3) i;
-CREATE TABLE withz USING parquet AS SELECT i AS k, CAST(i || ' v' AS string) v FROM (SELECT EXPLODE(SEQUENCE(1, 16, 3)) i);
+CREATE TABLE withz USING parquet AS SELECT i AS k, CAST(i AS string) || ' v' AS v FROM (SELECT EXPLODE(SEQUENCE(1, 16, 3)) i);
 -- [NOTE] Spark SQL doesn't support UNIQUE constraints
 --ALTER TABLE withz ADD UNIQUE (k);
 
diff --git a/sql/core/src/test/resources/sql-tests/inputs/predicate-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/predicate-functions.sql
index fadb4bb27fa13..d19120cfbdc56 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/predicate-functions.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/predicate-functions.sql
@@ -46,3 +46,23 @@ select to_timestamp('2017-03-01 00:00:01') > to_date('2017-03-01');
 select to_timestamp('2017-03-01 00:00:01') >= to_date('2017-03-01');
 select to_date('2017-03-01') < to_timestamp('2017-03-01 00:00:01');
 select to_date('2017-03-01') <= to_timestamp('2017-03-01 00:00:01');
+
+-- In
+select 1 in (1, 2, 3);
+select 1 in (1, 2, 3, null);
+select 1 in (1.0, 2.0, 3.0);
+select 1 in (1.0, 2.0, 3.0, null);
+select 1 in ('2', '3', '4');
+select 1 in ('2', '3', '4', null);
+select null in (1, 2, 3);
+select null in (1, 2, null);
+
+-- Not(In)
+select 1 not in (1, 2, 3);
+select 1 not in (1, 2, 3, null);
+select 1 not in (1.0, 2.0, 3.0);
+select 1 not in (1.0, 2.0, 3.0, null);
+select 1 not in ('2', '3', '4');
+select 1 not in ('2', '3', '4', null);
+select null not in (1, 2, 3);
+select null not in (1, 2, null);
diff --git a/sql/core/src/test/resources/sql-tests/inputs/regexp-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/regexp-functions.sql
index c0827a3cba39b..efe5c278730a6 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/regexp-functions.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/regexp-functions.sql
@@ -3,7 +3,51 @@ SELECT regexp_extract('1a 2b 14m', '\\d+');
 SELECT regexp_extract('1a 2b 14m', '\\d+', 0);
 SELECT regexp_extract('1a 2b 14m', '\\d+', 1);
 SELECT regexp_extract('1a 2b 14m', '\\d+', 2);
+SELECT regexp_extract('1a 2b 14m', '\\d+', -1);
+SELECT regexp_extract('1a 2b 14m', '(\\d+)?', 1);
+SELECT regexp_extract('a b m', '(\\d+)?', 1);
 SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)');
 SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', 0);
 SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', 1);
 SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', 2);
+SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', 3);
+SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', -1);
+SELECT regexp_extract('1a 2b 14m', '(\\d+)?([a-z]+)', 1);
+SELECT regexp_extract('a b m', '(\\d+)?([a-z]+)', 1);
+
+-- regexp_extract_all
+SELECT regexp_extract_all('1a 2b 14m', '\\d+');
+SELECT regexp_extract_all('1a 2b 14m', '\\d+', 0);
+SELECT regexp_extract_all('1a 2b 14m', '\\d+', 1);
+SELECT regexp_extract_all('1a 2b 14m', '\\d+', 2);
+SELECT regexp_extract_all('1a 2b 14m', '\\d+', -1);
+SELECT regexp_extract_all('1a 2b 14m', '(\\d+)?', 1);
+SELECT regexp_extract_all('a 2b 14m', '(\\d+)?', 1);
+SELECT regexp_extract_all('1a 2b 14m', '(\\d+)([a-z]+)');
+SELECT regexp_extract_all('1a 2b 14m', '(\\d+)([a-z]+)', 0);
+SELECT regexp_extract_all('1a 2b 14m', '(\\d+)([a-z]+)', 1);
+SELECT regexp_extract_all('1a 2b 14m', '(\\d+)([a-z]+)', 2);
+SELECT regexp_extract_all('1a 2b 14m', '(\\d+)([a-z]+)', 3);
+SELECT regexp_extract_all('1a 2b 14m', '(\\d+)([a-z]+)', -1);
+SELECT regexp_extract_all('1a 2b 14m', '(\\d+)?([a-z]+)', 1);
+SELECT regexp_extract_all('a 2b 14m', '(\\d+)?([a-z]+)', 1);
+
+-- regexp_replace
+SELECT regexp_replace('healthy, wealthy, and wise', '\\w+thy', 'something');
+SELECT regexp_replace('healthy, wealthy, and wise', '\\w+thy', 'something', -2);
+SELECT regexp_replace('healthy, wealthy, and wise', '\\w+thy', 'something', 0);
+SELECT regexp_replace('healthy, wealthy, and wise', '\\w+thy', 'something', 1);
+SELECT regexp_replace('healthy, wealthy, and wise', '\\w+thy', 'something', 2);
+SELECT regexp_replace('healthy, wealthy, and wise', '\\w+thy', 'something', 8);
+SELECT regexp_replace('healthy, wealthy, and wise', '\\w', 'something', 26);
+SELECT regexp_replace('healthy, wealthy, and wise', '\\w', 'something', 27);
+SELECT regexp_replace('healthy, wealthy, and wise', '\\w', 'something', 30);
+SELECT regexp_replace('healthy, wealthy, and wise', '\\w', 'something', null);
+
+-- regexp_like
+SELECT regexp_like('1a 2b 14m', '\\d+b');
+SELECT regexp_like('1a 2b 14m', '[a-z]+b');
+SELECT regexp('1a 2b 14m', '\\d+b');
+SELECT regexp('1a 2b 14m', '[a-z]+b');
+SELECT rlike('1a 2b 14m', '\\d+b');
+SELECT rlike('1a 2b 14m', '[a-z]+b');
diff --git a/sql/core/src/test/resources/sql-tests/inputs/show-tblproperties.sql b/sql/core/src/test/resources/sql-tests/inputs/show-tblproperties.sql
index 2861b2b43a113..0ba936dcad741 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/show-tblproperties.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/show-tblproperties.sql
@@ -17,8 +17,8 @@ SHOW TBLPROPERTIES view("p3");
 
 DROP VIEW view;
 
--- create a temporary view with properties
-CREATE TEMPORARY VIEW tv TBLPROPERTIES('p1'='v1') AS SELECT 1 AS c1;
+-- create a temporary view
+CREATE TEMPORARY VIEW tv AS SELECT 1 AS c1;
 
 -- Properties for a temporary view should be empty
 SHOW TBLPROPERTIES tv;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/string-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/string-functions.sql
index 8e33471e8b129..d44055d72e3bc 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/string-functions.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/string-functions.sql
@@ -17,9 +17,11 @@ select position('bar' in 'foobarbar'), position(null, 'foobarbar'), position('aa
 
 -- left && right
 select left("abcd", 2), left("abcd", 5), left("abcd", '2'), left("abcd", null);
-select left(null, -2), left("abcd", -2), left("abcd", 0), left("abcd", 'a');
+select left(null, -2);
+select left("abcd", -2), left("abcd", 0), left("abcd", 'a');
 select right("abcd", 2), right("abcd", 5), right("abcd", '2'), right("abcd", null);
-select right(null, -2), right("abcd", -2), right("abcd", 0), right("abcd", 'a');
+select right(null, -2);
+select right("abcd", -2), right("abcd", 0), right("abcd", 'a');
 
 -- split function
 SELECT split('aa1cc2ee3', '[1-9]+');
@@ -48,4 +50,24 @@ SELECT trim(LEADING 'xyz' FROM 'zzzytestxyz');
 SELECT trim(LEADING 'xy' FROM 'xyxXxyLAST WORD');
 SELECT trim(TRAILING 'xyz' FROM 'testxxzx');
 SELECT trim(TRAILING 'xyz' FROM 'xyztestxxzx');
-SELECT trim(TRAILING 'xy' FROM 'TURNERyxXxy');
\ No newline at end of file
+SELECT trim(TRAILING 'xy' FROM 'TURNERyxXxy');
+
+-- btrim
+SELECT btrim('xyxtrimyyx', 'xy');
+SELECT btrim(encode(" xyz ", 'utf-8'));
+SELECT btrim(encode('yxTomxx', 'utf-8'), encode('xyz', 'utf-8'));
+SELECT btrim(encode('xxxbarxxx', 'utf-8'), encode('x', 'utf-8'));
+
+-- Check lpad/rpad with invalid length parameter
+SELECT lpad('hi', 'invalid_length');
+SELECT rpad('hi', 'invalid_length');
+
+-- decode
+select decode();
+select decode(encode('abc', 'utf-8'));
+select decode(encode('abc', 'utf-8'), 'utf-8');
+select decode(1, 1, 'Southlake');
+select decode(2, 1, 'Southlake');
+select decode(2, 1, 'Southlake', 2, 'San Francisco', 3, 'New Jersey', 4, 'Seattle', 'Non domestic');
+select decode(6, 1, 'Southlake', 2, 'San Francisco', 3, 'New Jersey', 4, 'Seattle', 'Non domestic');
+select decode(6, 1, 'Southlake', 2, 'San Francisco', 3, 'New Jersey', 4, 'Seattle');
\ No newline at end of file
diff --git a/sql/core/src/test/resources/sql-tests/inputs/subexp-elimination.sql b/sql/core/src/test/resources/sql-tests/inputs/subexp-elimination.sql
new file mode 100644
index 0000000000000..9a594e0928ddc
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/subexp-elimination.sql
@@ -0,0 +1,37 @@
+-- Test for subexpression elimination.
+
+--SET spark.sql.optimizer.enableJsonExpressionOptimization=false
+
+--CONFIG_DIM1 spark.sql.codegen.wholeStage=true
+--CONFIG_DIM1 spark.sql.codegen.wholeStage=false
+
+--CONFIG_DIM2 spark.sql.codegen.factoryMode=CODEGEN_ONLY
+--CONFIG_DIM2 spark.sql.codegen.factoryMode=NO_CODEGEN
+
+--CONFIG_DIM3 spark.sql.subexpressionElimination.enabled=true
+--CONFIG_DIM3 spark.sql.subexpressionElimination.enabled=false
+
+-- Test data.
+CREATE OR REPLACE TEMPORARY VIEW testData AS SELECT * FROM VALUES
+('{"a":1, "b":"2"}', '[{"a": 1, "b":2}, {"a":2, "b":2}]'), ('{"a":1, "b":"2"}', null), ('{"a":2, "b":"3"}', '[{"a": 3, "b":4}, {"a":4, "b":5}]'), ('{"a":5, "b":"6"}', '[{"a": 6, "b":7}, {"a":8, "b":9}]'), (null, '[{"a": 1, "b":2}, {"a":2, "b":2}]')
+AS testData(a, b);
+
+SELECT from_json(a, 'struct<a:int,b:string>').a, from_json(a, 'struct<a:int,b:string>').b, from_json(b, 'array<struct<a:int,b:int>>')[0].a, from_json(b, 'array<struct<a:int,b:int>>')[0].b FROM testData;
+
+SELECT if(from_json(a, 'struct<a:int,b:string>').a > 1, from_json(b, 'array<struct<a:int,b:int>>')[0].a, from_json(b, 'array<struct<a:int,b:int>>')[0].a + 1) FROM testData;
+
+SELECT if(isnull(from_json(a, 'struct<a:int,b:string>').a), from_json(b, 'array<struct<a:int,b:int>>')[0].b + 1, from_json(b, 'array<struct<a:int,b:int>>')[0].b) FROM testData;
+
+SELECT case when from_json(a, 'struct<a:int,b:string>').a > 5 then from_json(a, 'struct<a:int,b:string>').b when from_json(a, 'struct<a:int,b:string>').a > 4 then from_json(a, 'struct<a:int,b:string>').b + 1 else from_json(a, 'struct<a:int,b:string>').b + 2 end FROM testData;
+
+SELECT case when from_json(a, 'struct<a:int,b:string>').a > 5 then from_json(b, 'array<struct<a:int,b:int>>')[0].b when from_json(a, 'struct<a:int,b:string>').a > 4 then from_json(b, 'array<struct<a:int,b:int>>')[0].b + 1 else from_json(b, 'array<struct<a:int,b:int>>')[0].b + 2 end FROM testData;
+
+-- With non-deterministic expressions.
+SELECT from_json(a, 'struct<a:int,b:string>').a + random() > 2, from_json(a, 'struct<a:int,b:string>').b, from_json(b, 'array<struct<a:int,b:int>>')[0].a, from_json(b, 'array<struct<a:int,b:int>>')[0].b + + random() > 2 FROM testData;
+
+SELECT if(from_json(a, 'struct<a:int,b:string>').a + random() > 5, from_json(b, 'array<struct<a:int,b:int>>')[0].a, from_json(b, 'array<struct<a:int,b:int>>')[0].a + 1) FROM testData;
+
+SELECT case when from_json(a, 'struct<a:int,b:string>').a > 5 then from_json(a, 'struct<a:int,b:string>').b + random() > 5 when from_json(a, 'struct<a:int,b:string>').a > 4 then from_json(a, 'struct<a:int,b:string>').b + 1 + random() > 2 else from_json(a, 'struct<a:int,b:string>').b + 2 + random() > 5 end FROM testData;
+
+-- Clean up
+DROP VIEW IF EXISTS testData;
\ No newline at end of file
diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-basic.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-basic.sql
index f4ffc20086386..5669423148f80 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-basic.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-basic.sql
@@ -1,3 +1,6 @@
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false
+
 create temporary view tab_a as select * from values (1, 1) as tab_a(a1, b1);
 create temporary view tab_b as select * from values (1, 1) as tab_b(a2, b2);
 create temporary view struct_tab as select struct(col1 as a, col2 as b) as record from
diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-having.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-having.sql
index 8f98ae1155062..750cc42b8641c 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-having.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-having.sql
@@ -1,6 +1,9 @@
 -- A test suite for IN HAVING in parent side, subquery, and both predicate subquery
 -- It includes correlated cases.
 
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false
+
 create temporary view t1 as select * from values
   ("val1a", 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'),
   ("val1b", 8S, 16, 19L, float(17.0), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'),
diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql
index 200a71ebbb622..2353560137d21 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql
@@ -13,6 +13,9 @@
 --CONFIG_DIM2 spark.sql.codegen.wholeStage=false,spark.sql.codegen.factoryMode=CODEGEN_ONLY
 --CONFIG_DIM2 spark.sql.codegen.wholeStage=false,spark.sql.codegen.factoryMode=NO_CODEGEN
 
+--CONFIG_DIM3 spark.sql.optimizeNullAwareAntiJoin=true
+--CONFIG_DIM3 spark.sql.optimizeNullAwareAntiJoin=false
+
 create temporary view t1 as select * from values
   ("val1a", 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'),
   ("val1b", 8S, 16, 19L, float(17.0), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'),
diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-limit.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-limit.sql
index 481b5e8cc7700..53fc2b8be7501 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-limit.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-limit.sql
@@ -1,6 +1,9 @@
 -- A test suite for IN LIMIT in parent side, subquery, and both predicate subquery
 -- It includes correlated cases.
 
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false
+
 create temporary view t1 as select * from values
   ("val1a", 6S, 8, 10L, float(15.0), 20D, 20E2BD, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'),
   ("val1b", 8S, 16, 19L, float(17.0), 25D, 26E2BD, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'),
@@ -72,7 +75,7 @@ SELECT Count(DISTINCT( t1a )),
 FROM   t1
 WHERE  t1d IN (SELECT t2d
                FROM   t2
-               ORDER  BY t2c
+               ORDER  BY t2c, t2d
                LIMIT 2)
 GROUP  BY t1b
 ORDER  BY t1b DESC NULLS FIRST
@@ -93,7 +96,7 @@ SELECT Count(DISTINCT( t1a )),
 FROM   t1
 WHERE  t1d NOT IN (SELECT t2d
                    FROM   t2
-                   ORDER  BY t2b DESC nulls first
+                   ORDER  BY t2b DESC nulls first, t2d
                    LIMIT 1)
 GROUP  BY t1b
 ORDER BY t1b NULLS last
diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-multiple-columns.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-multiple-columns.sql
index 4643605148a0c..1a6c06f9dad49 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-multiple-columns.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-multiple-columns.sql
@@ -1,6 +1,9 @@
 -- A test suite for multiple columns in predicate in parent side, subquery, and both predicate subquery
 -- It includes correlated cases.
 
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false
+
 create temporary view t1 as select * from values
   ("val1a", 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'),
   ("val1b", 8S, 16, 19L, float(17.0), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'),
diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-order-by.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-order-by.sql
index 001c49c460b06..568854ebe2d9b 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-order-by.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-order-by.sql
@@ -6,6 +6,9 @@
 --CONFIG_DIM1 spark.sql.codegen.wholeStage=false,spark.sql.codegen.factoryMode=CODEGEN_ONLY
 --CONFIG_DIM1 spark.sql.codegen.wholeStage=false,spark.sql.codegen.factoryMode=NO_CODEGEN
 
+--CONFIG_DIM2 spark.sql.optimizeNullAwareAntiJoin=true
+--CONFIG_DIM2 spark.sql.optimizeNullAwareAntiJoin=false
+
 create temporary view t1 as select * from values
   ("val1a", 6S, 8, 10L, float(15.0), 20D, 20E2BD, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'),
   ("val1b", 8S, 16, 19L, float(17.0), 25D, 26E2BD, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'),
diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-with-cte.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-with-cte.sql
index e65cb9106c1d4..fa4ae87f041cf 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-with-cte.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-with-cte.sql
@@ -1,6 +1,9 @@
 -- A test suite for in with cte in parent side, subquery, and both predicate subquery
 -- It includes correlated cases.
 
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false
+
 create temporary view t1 as select * from values
   ("val1a", 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'),
   ("val1b", 8S, 16, 19L, float(17.0), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'),
diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/nested-not-in.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/nested-not-in.sql
index 2f6835b59fdd5..e2d4ad522d446 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/nested-not-in.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/nested-not-in.sql
@@ -1,5 +1,8 @@
 -- Tests NOT-IN subqueries nested inside OR expression(s).
 
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false
+
 CREATE TEMPORARY VIEW EMP AS SELECT * FROM VALUES
   (100, "emp 1", 10),
   (200, "emp 2", NULL),
diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql
index fcdb667ad4523..2d11c5da20633 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql
@@ -1,6 +1,9 @@
 -- A test suite for not-in-joins in parent side, subquery, and both predicate subquery
 -- It includes correlated cases.
 
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false
+
 create temporary view t1 as select * from values
   ("val1a", 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'),
   ("val1b", 8S, 16, 19L, float(17.0), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'),
diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-multi-column-literal.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-multi-column-literal.sql
index 8eea84f4f5272..a061e495f51b8 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-multi-column-literal.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-multi-column-literal.sql
@@ -4,6 +4,9 @@
 -- This file has the same test cases as not-in-unit-tests-multi-column.sql with literals instead of
 -- subqueries. Small changes have been made to the literals to make them typecheck.
 
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false
+
 CREATE TEMPORARY VIEW m AS SELECT * FROM VALUES
   (null, null),
   (null, 1.0),
diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-multi-column.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-multi-column.sql
index 9f8dc7fca3b94..28ab75121573a 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-multi-column.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-multi-column.sql
@@ -15,6 +15,9 @@
 -- This can be generalized to include more tests for more columns, but it covers the main cases
 -- when there is more than one column.
 
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false
+
 CREATE TEMPORARY VIEW m AS SELECT * FROM VALUES
   (null, null),
   (null, 1.0),
diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-single-column-literal.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-single-column-literal.sql
index b261363d1dde7..79747022eb1e8 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-single-column-literal.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-single-column-literal.sql
@@ -4,6 +4,9 @@
 -- This file has the same test cases as not-in-unit-tests-single-column.sql with literals instead of
 -- subqueries.
 
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false
+
 CREATE TEMPORARY VIEW m AS SELECT * FROM VALUES
   (null, 1.0),
   (2, 3.0),
diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-single-column.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-single-column.sql
index 2cc08e10acf67..8060246bf3a3f 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-single-column.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-single-column.sql
@@ -31,6 +31,9 @@
 -- cause cases 2, 3, or 4 to be reduced to case 1 by limiting the number of rows returned by the
 -- subquery, so the row from the parent table should always be included in the output.
 
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false
+
 CREATE TEMPORARY VIEW m AS SELECT * FROM VALUES
   (null, 1.0),
   (2, 3.0),
diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/simple-in.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/simple-in.sql
index 2748a959cbef8..d8a58afa344db 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/simple-in.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/simple-in.sql
@@ -1,6 +1,9 @@
 -- A test suite for simple IN predicate subquery
 -- It includes correlated cases.
 
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true
+--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false
+
 create temporary view t1 as select * from values
   ("t1a", 6S, 8, 10L, float(15.0), 20D, 20E2BD, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'),
   ("t1b", 8S, 16, 19L, float(17.0), 25D, 26E2BD, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'),
diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/scalar-subquery/scalar-subquery-select.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/scalar-subquery/scalar-subquery-select.sql
index eabbd0a932253..81712bfac239a 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/subquery/scalar-subquery/scalar-subquery-select.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/scalar-subquery/scalar-subquery-select.sql
@@ -128,3 +128,13 @@ WHERE  NOT EXISTS (SELECT (SELECT max(t2b)
                                  ON     t2a = t1a
                                  WHERE  t2c = t3c)
                    AND    t3a = t1a);
+
+-- SPARK-34876: Non-nullable aggregates should not return NULL in a correlated subquery
+SELECT t1a,
+    (SELECT count(t2d) FROM t2 WHERE t2a = t1a) count_t2,
+    (SELECT count_if(t2d > 0) FROM t2 WHERE t2a = t1a) count_if_t2,
+    (SELECT approx_count_distinct(t2d) FROM t2 WHERE t2a = t1a) approx_count_distinct_t2,
+    (SELECT collect_list(t2d) FROM t2 WHERE t2a = t1a) collect_list_t2,
+    (SELECT collect_set(t2d) FROM t2 WHERE t2a = t1a) collect_set_t2,
+    (SELECT hex(count_min_sketch(t2d, 0.5d, 0.5d, 1)) FROM t2 WHERE t2a = t1a) collect_set_t2
+FROM t1;
\ No newline at end of file
diff --git a/sql/core/src/test/resources/sql-tests/inputs/table-aliases.sql b/sql/core/src/test/resources/sql-tests/inputs/table-aliases.sql
index 4cfd5f28afdaa..5b98f056ebc5a 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/table-aliases.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/table-aliases.sql
@@ -19,9 +19,17 @@ SELECT a AS col1, b AS col2 FROM testData AS t(c, d);
 -- Subquery aliases in FROM clause
 SELECT * FROM (SELECT 1 AS a, 1 AS b) t(col1, col2);
 
+SELECT t.* FROM (SELECT 1 AS a, 1 AS b) t(col1, col2);
+
+SELECT col1, col2 FROM (SELECT 1 AS a, 1 AS b) t(col1, col2);
+
+SELECT t.col1, t.col2 FROM (SELECT 1 AS a, 1 AS b) t(col1, col2);
+
 -- Aliases for join relations in FROM clause
 CREATE OR REPLACE TEMPORARY VIEW src1 AS SELECT * FROM VALUES (1, "a"), (2, "b"), (3, "c") AS src1(id, v1);
 
 CREATE OR REPLACE TEMPORARY VIEW src2 AS SELECT * FROM VALUES (2, 1.0), (3, 3.2), (1, 8.5) AS src2(id, v2);
 
 SELECT * FROM (src1 s1 INNER JOIN src2 s2 ON s1.id = s2.id) dst(a, b, c, d);
+
+SELECT dst.* FROM (src1 s1 INNER JOIN src2 s2 ON s1.id = s2.id) dst(a, b, c, d);
diff --git a/sql/core/src/test/resources/sql-tests/inputs/table-valued-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/table-valued-functions.sql
index 6f14c8ca87821..7ea6af37c1470 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/table-valued-functions.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/table-valued-functions.sql
@@ -13,11 +13,17 @@ select * from range(0, 10, 2);
 -- range call with numPartitions
 select * from range(0, 10, 1, 200);
 
--- range call error
+-- range call with invalid number of arguments
 select * from range(1, 1, 1, 1, 1);
 
 -- range call with null
 select * from range(1, null);
 
+-- range call with incompatible type
+select * from range(array(1, 2, 3));
+
+-- range call with illegal step
+select * from range(0, 5, 0);
+
 -- range call with a mixed-case function name
 select * from RaNgE(2);
diff --git a/sql/core/src/test/resources/sql-tests/inputs/timezone.sql b/sql/core/src/test/resources/sql-tests/inputs/timezone.sql
new file mode 100644
index 0000000000000..105f2816fb549
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/timezone.sql
@@ -0,0 +1,15 @@
+-- valid time zones
+SET TIME ZONE 'Asia/Hong_Kong';
+SET TIME ZONE 'GMT+1';
+SET TIME ZONE INTERVAL 10 HOURS;
+SET TIME ZONE INTERVAL '15:40:32' HOUR TO SECOND;
+SET TIME ZONE LOCAL;
+
+-- invalid time zone
+SET TIME ZONE;
+SET TIME ZONE 'invalid/zone';
+SET TIME ZONE INTERVAL 3 DAYS;
+SET TIME ZONE INTERVAL 24 HOURS;
+SET TIME ZONE INTERVAL '19:40:32' HOUR TO SECOND;
+SET TIME ZONE INTERVAL 10 HOURS 'GMT+1';
+SET TIME ZONE INTERVAL 10 HOURS 1 MILLISECOND;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/transform.sql b/sql/core/src/test/resources/sql-tests/inputs/transform.sql
new file mode 100644
index 0000000000000..d84659c4ccb60
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/transform.sql
@@ -0,0 +1,375 @@
+-- Test data.
+CREATE OR REPLACE TEMPORARY VIEW t AS SELECT * FROM VALUES
+('1', true, unhex('537061726B2053514C'), tinyint(1), 1, smallint(100), bigint(1), float(1.0), 1.0, Decimal(1.0), timestamp('1997-01-02'), date('2000-04-01')),
+('2', false, unhex('537061726B2053514C'), tinyint(2), 2,  smallint(200), bigint(2), float(2.0), 2.0, Decimal(2.0), timestamp('1997-01-02 03:04:05'), date('2000-04-02')),
+('3', true, unhex('537061726B2053514C'), tinyint(3), 3, smallint(300), bigint(3), float(3.0), 3.0, Decimal(3.0), timestamp('1997-02-10 17:32:01-08'), date('2000-04-03'))
+AS t(a, b, c, d, e, f, g, h, i, j, k, l);
+
+CREATE OR REPLACE TEMPORARY VIEW script_trans AS SELECT * FROM VALUES
+(1, 2, 3),
+(4, 5, 6),
+(7, 8, 9)
+AS script_trans(a, b, c);
+
+CREATE OR REPLACE TEMPORARY VIEW complex_trans AS SELECT * FROM VALUES
+(1, 1),
+(1, 1),
+(2, 2),
+(2, 2),
+(3, 3),
+(2, 2),
+(3, 3),
+(1, 1),
+(3, 3)
+as complex_trans(a, b);
+
+SELECT TRANSFORM(a)
+USING 'cat' AS (a)
+FROM t;
+
+-- common supported data types between no serde and serde transform
+SELECT a, b, decode(c, 'UTF-8'), d, e, f, g, h, i, j, k, l FROM (
+  SELECT TRANSFORM(a, b, c, d, e, f, g, h, i, j, k, l)
+  USING 'cat' AS (
+    a string,
+    b boolean,
+    c binary,
+    d tinyint,
+    e int,
+    f smallint,
+    g long,
+    h float,
+    i double,
+    j decimal(38, 18),
+    k timestamp,
+    l date)
+  FROM t
+) tmp;
+
+-- common supported data types between no serde and serde transform
+SELECT a, b, decode(c, 'UTF-8'), d, e, f, g, h, i, j, k, l FROM (
+  SELECT TRANSFORM(a, b, c, d, e, f, g, h, i, j, k, l)
+  USING 'cat' AS (
+    a string,
+    b string,
+    c string,
+    d string,
+    e string,
+    f string,
+    g string,
+    h string,
+    i string,
+    j string,
+    k string,
+    l string)
+  FROM t
+) tmp;
+
+-- SPARK-32388 handle schema less
+SELECT TRANSFORM(a)
+USING 'cat'
+FROM t;
+
+SELECT TRANSFORM(a, b)
+USING 'cat'
+FROM t;
+
+SELECT TRANSFORM(a, b, c)
+USING 'cat'
+FROM t;
+
+-- return null when return string incompatible (no serde)
+SELECT TRANSFORM(a, b, c, d, e, f, g, h, i)
+USING 'cat' AS (a int, b short, c long, d byte, e float, f double, g decimal(38, 18), h date, i timestamp)
+FROM VALUES
+('a','','1231a','a','213.21a','213.21a','0a.21d','2000-04-01123','1997-0102 00:00:') tmp(a, b, c, d, e, f, g, h, i);
+
+-- SPARK-28227: transform can't run with aggregation
+SELECT TRANSFORM(b, max(a), sum(f))
+USING 'cat' AS (a, b)
+FROM t
+GROUP BY b;
+
+-- transform use MAP
+MAP a, b USING 'cat' AS (a, b) FROM t;
+
+-- transform use REDUCE
+REDUCE a, b USING 'cat' AS (a, b) FROM t;
+
+-- transform with defined row format delimit
+SELECT TRANSFORM(a, b, c, null)
+  ROW FORMAT DELIMITED
+  FIELDS TERMINATED BY '@'
+  LINES TERMINATED BY '\n'
+  NULL DEFINED AS 'NULL'
+USING 'cat' AS (a, b, c, d)
+  ROW FORMAT DELIMITED
+  FIELDS TERMINATED BY '@'
+  LINES TERMINATED BY '\n'
+  NULL DEFINED AS 'NULL'
+FROM t;
+
+SELECT TRANSFORM(a, b, c, null)
+  ROW FORMAT DELIMITED
+  FIELDS TERMINATED BY '@'
+  LINES TERMINATED BY '\n'
+  NULL DEFINED AS 'NULL'
+USING 'cat' AS (d)
+  ROW FORMAT DELIMITED
+  FIELDS TERMINATED BY '@'
+  LINES TERMINATED BY '\n'
+  NULL DEFINED AS 'NULL'
+FROM t;
+
+-- transform with defined row format delimit handle schema with correct type
+SELECT a, b, decode(c, 'UTF-8'), d, e, f, g, h, i, j, k, l FROM (
+  SELECT TRANSFORM(a, b, c, d, e, f, g, h, i, j, k, l)
+    ROW FORMAT DELIMITED
+    FIELDS TERMINATED BY ','
+    LINES TERMINATED BY '\n'
+    NULL DEFINED AS 'NULL'
+    USING 'cat' AS (
+      a string,
+      b boolean,
+      c binary,
+      d tinyint,
+      e int,
+      f smallint,
+      g long,
+      h float,
+      i double,
+      j decimal(38, 18),
+      k timestamp,
+      l date)
+    ROW FORMAT DELIMITED
+    FIELDS TERMINATED BY ','
+    LINES TERMINATED BY '\n'
+    NULL DEFINED AS 'NULL'
+  FROM t
+) tmp;
+
+-- transform with defined row format delimit handle schema with wrong type
+SELECT a, b, decode(c, 'UTF-8'), d, e, f, g, h, i, j, k, l FROM (
+  SELECT TRANSFORM(a, b, c, d, e, f, g, h, i, j, k, l)
+    ROW FORMAT DELIMITED
+    FIELDS TERMINATED BY ','
+    LINES TERMINATED BY '\n'
+    NULL DEFINED AS 'NULL'
+    USING 'cat' AS (
+      a string,
+      b long,
+      c binary,
+      d tinyint,
+      e int,
+      f smallint,
+      g long,
+      h float,
+      i double,
+      j decimal(38, 18),
+      k int,
+      l long)
+    ROW FORMAT DELIMITED
+    FIELDS TERMINATED BY ','
+    LINES TERMINATED BY '\n'
+    NULL DEFINED AS 'NULL'
+  FROM t
+) tmp;
+
+-- transform with defined row format delimit LINE TERMINATED BY only support '\n'
+SELECT a, b, decode(c, 'UTF-8'), d, e, f, g, h, i, j, k, l FROM (
+  SELECT TRANSFORM(a, b, c, d, e, f, g, h, i, j, k, l)
+    ROW FORMAT DELIMITED
+    FIELDS TERMINATED BY ','
+    LINES TERMINATED BY '@'
+    NULL DEFINED AS 'NULL'
+    USING 'cat' AS (
+      a string,
+      b string,
+      c string,
+      d string,
+      e string,
+      f string,
+      g string,
+      h string,
+      i string,
+      j string,
+      k string,
+      l string)
+    ROW FORMAT DELIMITED
+    FIELDS TERMINATED BY ','
+    LINES TERMINATED BY '@'
+    NULL DEFINED AS 'NULL'
+  FROM t
+) tmp;
+
+SELECT TRANSFORM(b, a, CAST(c AS STRING))
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 4;
+
+SELECT TRANSFORM(1, 2, 3)
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 4;
+
+SELECT TRANSFORM(1, 2)
+  USING 'cat' AS (a INT, b INT)
+FROM script_trans
+LIMIT 1;
+
+SELECT TRANSFORM(
+  b, a,
+  CASE
+    WHEN c > 100 THEN 1
+    WHEN c < 100 THEN 2
+  ELSE 3 END)
+  USING 'cat' AS (a, b,  c)
+FROM script_trans
+WHERE a <= 4;
+
+SELECT TRANSFORM(b, a, c + 1)
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 4;
+
+SELECT TRANSFORM(*)
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 4;
+
+SELECT TRANSFORM(b, MAX(a), CAST(SUM(c) AS STRING))
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 4
+GROUP BY b;
+
+SELECT TRANSFORM(b, MAX(a) FILTER (WHERE a > 3), CAST(SUM(c) AS STRING))
+  USING 'cat' AS (a,b,c)
+FROM script_trans
+WHERE a <= 4
+GROUP BY b;
+
+SELECT TRANSFORM(b, MAX(a), CAST(sum(c) AS STRING))
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 2
+GROUP BY b;
+
+SELECT TRANSFORM(b, MAX(a), CAST(SUM(c) AS STRING))
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 4
+GROUP BY b
+HAVING MAX(a) > 0;
+
+SELECT TRANSFORM(b, MAX(a), CAST(SUM(c) AS STRING))
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 4
+GROUP BY b
+HAVING MAX(a) > 1;
+
+SELECT TRANSFORM(b, MAX(a) OVER w, CAST(SUM(c) OVER w AS STRING))
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 4
+WINDOW w AS (PARTITION BY b ORDER BY a);
+
+SELECT TRANSFORM(b, MAX(a), CAST(SUM(c) AS STRING), myCol, myCol2)
+  USING 'cat' AS (a STRING, b STRING, c STRING, d ARRAY<INT>, e STRING)
+FROM script_trans
+LATERAL VIEW explode(array(array(1,2,3))) myTable AS myCol
+LATERAL VIEW explode(myTable.myCol) myTable2 AS myCol2
+WHERE a <= 4
+GROUP BY b, myCol, myCol2
+HAVING max(a) > 1;
+
+FROM(
+  FROM script_trans
+  SELECT TRANSFORM(a, b)
+    USING 'cat' AS (`a` INT, b STRING)
+) t
+SELECT a + 1;
+
+FROM(
+  SELECT TRANSFORM(a, SUM(b))
+    USING 'cat' AS (`a` INT, b STRING)
+  FROM script_trans
+  GROUP BY a
+) t
+SELECT (b + 1) AS result
+ORDER BY result;
+
+MAP k / 10 USING 'cat' AS (one) FROM (SELECT 10 AS k);
+
+FROM (SELECT 1 AS key, 100 AS value) src
+MAP src.*, src.key, CAST(src.key / 10 AS INT), CAST(src.key % 10 AS INT), src.value
+  USING 'cat' AS (k, v, tkey, ten, one, tvalue);
+
+SELECT TRANSFORM(1)
+  USING 'cat' AS (a)
+FROM script_trans
+HAVING true;
+
+SET spark.sql.legacy.parser.havingWithoutGroupByAsWhere=true;
+
+SELECT TRANSFORM(1)
+  USING 'cat' AS (a)
+FROM script_trans
+HAVING true;
+
+SET spark.sql.legacy.parser.havingWithoutGroupByAsWhere=false;
+
+-- SPARK-34634: self join using CTE contains transform
+WITH temp AS (
+  SELECT TRANSFORM(a) USING 'cat' AS (b string) FROM t
+)
+SELECT t1.b FROM temp t1 JOIN temp t2 ON t1.b = t2.b;
+
+SELECT TRANSFORM(DISTINCT b, a, c)
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 4;
+
+SELECT TRANSFORM(ALL b, a, c)
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 4;
+
+-- SPARK-35070: TRANSFORM not support alias in inputs
+SELECT TRANSFORM(b AS b_1, MAX(a), CAST(sum(c) AS STRING))
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 2
+GROUP BY b;
+
+SELECT TRANSFORM(b b_1, MAX(a), CAST(sum(c) AS STRING))
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 2
+GROUP BY b;
+
+SELECT TRANSFORM(b, MAX(a) AS max_a, CAST(sum(c) AS STRING))
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 2
+GROUP BY b;
+
+-- SPARK-33985: TRANSFORM with CLUSTER BY/ORDER BY/SORT BY
+FROM (
+  SELECT TRANSFORM(a, b)
+    USING 'cat' AS (a, b)
+  FROM complex_trans
+  CLUSTER BY a
+) map_output
+SELECT TRANSFORM(a, b)
+  USING 'cat' AS (a, b);
+
+FROM (
+  SELECT TRANSFORM(a, b)
+    USING 'cat' AS (a, b)
+  FROM complex_trans
+  ORDER BY a
+) map_output
+SELECT TRANSFORM(a, b)
+  USING 'cat' AS (a, b);
\ No newline at end of file
diff --git a/sql/core/src/test/resources/sql-tests/inputs/try_arithmetic.sql b/sql/core/src/test/resources/sql-tests/inputs/try_arithmetic.sql
new file mode 100644
index 0000000000000..cda83e80a53f9
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/try_arithmetic.sql
@@ -0,0 +1,11 @@
+-- TRY_ADD
+SELECT try_add(1, 1);
+SELECT try_add(2147483647, 1);
+SELECT try_add(-2147483648, -1);
+SELECT try_add(9223372036854775807L, 1);
+SELECT try_add(-9223372036854775808L, -1);
+
+-- TRY_DIVIDE
+SELECT try_divide(1, 0.5);
+SELECT try_divide(1, 0);
+SELECT try_divide(0, 0);
\ No newline at end of file
diff --git a/sql/core/src/test/resources/sql-tests/inputs/try_cast.sql b/sql/core/src/test/resources/sql-tests/inputs/try_cast.sql
new file mode 100644
index 0000000000000..2d584843ada3d
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/try_cast.sql
@@ -0,0 +1,54 @@
+-- TRY_CAST string representing a valid fractional number to integral should truncate the number
+SELECT TRY_CAST('1.23' AS int);
+SELECT TRY_CAST('1.23' AS long);
+SELECT TRY_CAST('-4.56' AS int);
+SELECT TRY_CAST('-4.56' AS long);
+
+-- TRY_CAST string which are not numbers to integral should return null
+SELECT TRY_CAST('abc' AS int);
+SELECT TRY_CAST('abc' AS long);
+
+-- TRY_CAST empty string to integral should return null
+SELECT TRY_CAST('' AS int);
+SELECT TRY_CAST('' AS long);
+
+-- TRY_CAST null to integral should return null
+SELECT TRY_CAST(NULL AS int);
+SELECT TRY_CAST(NULL AS long);
+
+-- TRY_CAST invalid decimal string to integral should return null
+SELECT TRY_CAST('123.a' AS int);
+SELECT TRY_CAST('123.a' AS long);
+
+-- '-2147483648' is the smallest int value
+SELECT TRY_CAST('-2147483648' AS int);
+SELECT TRY_CAST('-2147483649' AS int);
+
+-- '2147483647' is the largest int value
+SELECT TRY_CAST('2147483647' AS int);
+SELECT TRY_CAST('2147483648' AS int);
+
+-- '-9223372036854775808' is the smallest long value
+SELECT TRY_CAST('-9223372036854775808' AS long);
+SELECT TRY_CAST('-9223372036854775809' AS long);
+
+-- '9223372036854775807' is the largest long value
+SELECT TRY_CAST('9223372036854775807' AS long);
+SELECT TRY_CAST('9223372036854775808' AS long);
+
+-- TRY_CAST string to interval and interval to string
+SELECT TRY_CAST('interval 3 month 1 hour' AS interval);
+SELECT TRY_CAST('abc' AS interval);
+
+-- TRY_CAST string to boolean
+select TRY_CAST('true' as boolean);
+select TRY_CAST('false' as boolean);
+select TRY_CAST('abc' as boolean);
+
+-- TRY_CAST string to date
+SELECT TRY_CAST("2021-01-01" AS date);
+SELECT TRY_CAST("2021-101-01" AS date);
+
+-- TRY_CAST string to timestamp
+SELECT TRY_CAST("2021-01-01 00:00:00" AS timestamp);
+SELECT TRY_CAST("2021-101-01 00:00:00" AS timestamp);
\ No newline at end of file
diff --git a/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-case.sql b/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-case.sql
index 8fa3c0a6dfec9..5322c1b502439 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-case.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-case.sql
@@ -67,11 +67,11 @@ SELECT '7' AS `None`,
   CASE WHEN rand() < udf(0) THEN 1
   END AS `NULL on no matches`;
 
+-- [SPARK-33008] Spark SQL throws an exception
 -- Constant-expression folding shouldn't evaluate unreachable subexpressions
 SELECT CASE WHEN udf(1=0) THEN 1/0 WHEN 1=1 THEN 1 ELSE 2/0 END;
 SELECT CASE 1 WHEN 0 THEN 1/udf(0) WHEN 1 THEN 1 ELSE 2/0 END;
 
--- [SPARK-27923] PostgreSQL throws an exception but Spark SQL is NULL
 -- However we do not currently suppress folding of potentially
 -- reachable subexpressions
 SELECT CASE WHEN i > 100 THEN udf(1/0) ELSE udf(0) END FROM case_tbl;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-join.sql b/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-join.sql
index 77bcfe7e10cbc..16202a6765dc9 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-join.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-join.sql
@@ -735,20 +735,20 @@ select * from a left join b on udf(i) = x and i = udf(y) and udf(x) = udf(i);
 --
 -- test NULL behavior of whole-row Vars, per bug #5025
 --
-select udf(t1.q2), udf(count(t2.*))
+select udf(t1.q2), udf(count(t2.q1, t2.q2))
 from int8_tbl t1 left join int8_tbl t2 on (udf(udf(t1.q2)) = t2.q1)
 group by udf(t1.q2) order by 1;
 
-select udf(udf(t1.q2)), udf(count(t2.*))
+select udf(udf(t1.q2)), udf(count(t2.q1, t2.q2))
 from int8_tbl t1 left join (select * from int8_tbl) t2 on (udf(udf(t1.q2)) = udf(t2.q1))
 group by udf(udf(t1.q2)) order by 1;
 
 -- [SPARK-28330] Enhance query limit
--- select t1.q2, count(t2.*)
+-- select t1.q2, count(t2.q1, t2.q2)
 -- from int8_tbl t1 left join (select * from int8_tbl offset 0) t2 on (t1.q2 = t2.q1)
 -- group by t1.q2 order by 1;
 
-select udf(t1.q2) as q2, udf(udf(count(t2.*)))
+select udf(t1.q2) as q2, udf(udf(count(t2.q1, t2.q2)))
 from int8_tbl t1 left join
   (select udf(q1) as q1, case when q2=1 then 1 else q2 end as q2 from int8_tbl) t2
   on (udf(t1.q2) = udf(t2.q1))
@@ -876,7 +876,7 @@ from nt3 as nt3
     on udf(ss2.id) = nt3.nt2_id
 where udf(nt3.id) = 1 and udf(ss2.b3);
 
--- [SPARK-28379] Correlated scalar subqueries must be aggregated
+-- Accessing outer query column is not allowed in LocalLimit
 --
 -- test case where a PlaceHolderVar is propagated into a subquery
 --
diff --git a/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-select_having.sql b/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-select_having.sql
index 412d45b49a184..76c0b198aa439 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-select_having.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-select_having.sql
@@ -51,6 +51,7 @@ SELECT 1 AS one FROM test_having HAVING udf(a) > 1;
 SELECT 1 AS one FROM test_having HAVING udf(udf(1) > udf(2));
 SELECT 1 AS one FROM test_having HAVING udf(udf(1) < udf(2));
 
+-- [SPARK-33008] Spark SQL throws an exception
 -- and just to prove that we aren't scanning the table:
 SELECT 1 AS one FROM test_having WHERE 1/udf(a) = 1 HAVING 1 < 2;
 
diff --git a/sql/core/src/test/resources/sql-tests/inputs/udf/udf-count.sql b/sql/core/src/test/resources/sql-tests/inputs/udf/udf-count.sql
index 6ab207b93f561..82058d43f88a2 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/udf/udf-count.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/udf/udf-count.sql
@@ -20,9 +20,9 @@ SELECT
 FROM testData;
 
 -- count with multiple expressions
-SELECT udf(count(a, b)), udf(count(b, a)), udf(count(testData.*)) FROM testData;
+SELECT udf(count(a, b)), udf(count(b, a)), udf(count(testData.*, testData.*)) FROM testData;
 
 -- distinct count with multiple expressions
 SELECT
-  udf(count(DISTINCT a, b)), udf(count(DISTINCT b, a)), udf(count(DISTINCT *)), udf(count(DISTINCT testData.*))
+  udf(count(DISTINCT a, b)), udf(count(DISTINCT b, a)), udf(count(DISTINCT *)), udf(count(DISTINCT testData.*, testData.*))
 FROM testData;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/udf/udf-window.sql b/sql/core/src/test/resources/sql-tests/inputs/udf/udf-window.sql
index bcbf87f8a04c5..1659f1c819592 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/udf/udf-window.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/udf/udf-window.sql
@@ -1,15 +1,15 @@
 --This test file was converted from window.sql.
 -- Test data.
 CREATE OR REPLACE TEMPORARY VIEW testData AS SELECT * FROM VALUES
-(null, 1L, 1.0D, date("2017-08-01"), timestamp(1501545600), "a"),
-(1, 1L, 1.0D, date("2017-08-01"), timestamp(1501545600), "a"),
-(1, 2L, 2.5D, date("2017-08-02"), timestamp(1502000000), "a"),
-(2, 2147483650L, 100.001D, date("2020-12-31"), timestamp(1609372800), "a"),
-(1, null, 1.0D, date("2017-08-01"), timestamp(1501545600), "b"),
-(2, 3L, 3.3D, date("2017-08-03"), timestamp(1503000000), "b"),
-(3, 2147483650L, 100.001D, date("2020-12-31"), timestamp(1609372800), "b"),
+(null, 1L, 1.0D, date("2017-08-01"), timestamp_seconds(1501545600), "a"),
+(1, 1L, 1.0D, date("2017-08-01"), timestamp_seconds(1501545600), "a"),
+(1, 2L, 2.5D, date("2017-08-02"), timestamp_seconds(1502000000), "a"),
+(2, 2147483650L, 100.001D, date("2020-12-31"), timestamp_seconds(1609372800), "a"),
+(1, null, 1.0D, date("2017-08-01"), timestamp_seconds(1501545600), "b"),
+(2, 3L, 3.3D, date("2017-08-03"), timestamp_seconds(1503000000), "b"),
+(3, 2147483650L, 100.001D, date("2020-12-31"), timestamp_seconds(1609372800), "b"),
 (null, null, null, null, null, null),
-(3, 1L, 1.0D, date("2017-08-01"), timestamp(1501545600), null)
+(3, 1L, 1.0D, date("2017-08-01"), timestamp_seconds(1501545600), null)
 AS testData(val, val_long, val_double, val_date, val_timestamp, cate);
 
 -- RowsBetween
diff --git a/sql/core/src/test/resources/sql-tests/inputs/union.sql b/sql/core/src/test/resources/sql-tests/inputs/union.sql
index 6da1b9b49b226..8a5b6c50fc1e3 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/union.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/union.sql
@@ -45,10 +45,24 @@ SELECT array(1, 2), 'str'
 UNION ALL
 SELECT array(1, 2, 3, NULL), 1;
 
+-- SPARK-32638: corrects references when adding aliases in WidenSetOperationTypes
+CREATE OR REPLACE TEMPORARY VIEW t3 AS VALUES (decimal(1)) tbl(v);
+SELECT t.v FROM (
+  SELECT v FROM t3
+  UNION ALL
+  SELECT v + v AS v FROM t3
+) t;
+
+SELECT SUM(t.v) FROM (
+  SELECT v FROM t3
+  UNION
+  SELECT v + v AS v FROM t3
+) t;
 
 -- Clean-up
 DROP VIEW IF EXISTS t1;
 DROP VIEW IF EXISTS t2;
+DROP VIEW IF EXISTS t3;
 DROP VIEW IF EXISTS p1;
 DROP VIEW IF EXISTS p2;
 DROP VIEW IF EXISTS p3;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/using-join.sql b/sql/core/src/test/resources/sql-tests/inputs/using-join.sql
new file mode 100644
index 0000000000000..336d19f0f2a3d
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/using-join.sql
@@ -0,0 +1,70 @@
+create temporary view nt1 as select * from values
+  ("one", 1),
+  ("two", 2),
+  ("three", 3)
+  as nt1(k, v1);
+
+create temporary view nt2 as select * from values
+  ("one", 1),
+  ("two", 22),
+  ("one", 5),
+  ("four", 4)
+  as nt2(k, v2);
+
+SELECT * FROM nt1 left outer join nt2 using (k);
+
+SELECT k FROM nt1 left outer join nt2 using (k);
+
+SELECT nt1.*, nt2.* FROM nt1 left outer join nt2 using (k);
+
+SELECT nt1.k, nt2.k FROM nt1 left outer join nt2 using (k);
+
+SELECT k, nt1.k FROM nt1 left outer join nt2 using (k);
+
+SELECT k, nt2.k FROM nt1 left outer join nt2 using (k);
+
+SELECT * FROM nt1 left semi join nt2 using (k);
+
+SELECT k FROM nt1 left semi join nt2 using (k);
+
+SELECT nt1.* FROM nt1 left semi join nt2 using (k);
+
+SELECT nt1.k FROM nt1 left semi join nt2 using (k);
+
+SELECT k, nt1.k FROM nt1 left semi join nt2 using (k);
+
+SELECT * FROM nt1 right outer join nt2 using (k);
+
+SELECT k FROM nt1 right outer join nt2 using (k);
+
+SELECT nt1.*, nt2.* FROM nt1 right outer join nt2 using (k);
+
+SELECT nt1.k, nt2.k FROM nt1 right outer join nt2 using (k);
+
+SELECT k, nt1.k FROM nt1 right outer join nt2 using (k);
+
+SELECT k, nt2.k FROM nt1 right outer join nt2 using (k);
+
+SELECT * FROM nt1 full outer join nt2 using (k);
+
+SELECT k FROM nt1 full outer join nt2 using (k);
+
+SELECT nt1.*, nt2.* FROM nt1 full outer join nt2 using (k);
+
+SELECT nt1.k, nt2.k FROM nt1 full outer join nt2 using (k);
+
+SELECT k, nt1.k FROM nt1 full outer join nt2 using (k);
+
+SELECT k, nt2.k FROM nt1 full outer join nt2 using (k);
+
+SELECT * FROM nt1 full outer join nt2 using (k);
+
+SELECT k FROM nt1 inner join nt2 using (k);
+
+SELECT nt1.*, nt2.* FROM nt1 inner join nt2 using (k);
+
+SELECT nt1.k, nt2.k FROM nt1 inner join nt2 using (k);
+
+SELECT k, nt1.k FROM nt1 inner join nt2 using (k);
+
+SELECT k, nt2.k FROM nt1 inner join nt2 using (k);
diff --git a/sql/core/src/test/resources/sql-tests/inputs/window.sql b/sql/core/src/test/resources/sql-tests/inputs/window.sql
index 3d05dfda6c3fa..46d3629a5dc50 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/window.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/window.sql
@@ -5,17 +5,49 @@
 
 -- Test data.
 CREATE OR REPLACE TEMPORARY VIEW testData AS SELECT * FROM VALUES
-(null, 1L, 1.0D, date("2017-08-01"), timestamp(1501545600), "a"),
-(1, 1L, 1.0D, date("2017-08-01"), timestamp(1501545600), "a"),
-(1, 2L, 2.5D, date("2017-08-02"), timestamp(1502000000), "a"),
-(2, 2147483650L, 100.001D, date("2020-12-31"), timestamp(1609372800), "a"),
-(1, null, 1.0D, date("2017-08-01"), timestamp(1501545600), "b"),
-(2, 3L, 3.3D, date("2017-08-03"), timestamp(1503000000), "b"),
-(3, 2147483650L, 100.001D, date("2020-12-31"), timestamp(1609372800), "b"),
+(null, 1L, 1.0D, date("2017-08-01"), timestamp_seconds(1501545600), "a"),
+(1, 1L, 1.0D, date("2017-08-01"), timestamp_seconds(1501545600), "a"),
+(1, 2L, 2.5D, date("2017-08-02"), timestamp_seconds(1502000000), "a"),
+(2, 2147483650L, 100.001D, date("2020-12-31"), timestamp_seconds(1609372800), "a"),
+(1, null, 1.0D, date("2017-08-01"), timestamp_seconds(1501545600), "b"),
+(2, 3L, 3.3D, date("2017-08-03"), timestamp_seconds(1503000000), "b"),
+(3, 2147483650L, 100.001D, date("2020-12-31"), timestamp_seconds(1609372800), "b"),
 (null, null, null, null, null, null),
-(3, 1L, 1.0D, date("2017-08-01"), timestamp(1501545600), null)
+(3, 1L, 1.0D, date("2017-08-01"), timestamp_seconds(1501545600), null)
 AS testData(val, val_long, val_double, val_date, val_timestamp, cate);
 
+CREATE OR REPLACE TEMPORARY VIEW basic_pays AS SELECT * FROM VALUES
+('Diane Murphy','Accounting',8435),
+('Mary Patterson','Accounting',9998),
+('Jeff Firrelli','Accounting',8992),
+('William Patterson','Accounting',8870),
+('Gerard Bondur','Accounting',11472),
+('Anthony Bow','Accounting',6627),
+('Leslie Jennings','IT',8113),
+('Leslie Thompson','IT',5186),
+('Julie Firrelli','Sales',9181),
+('Steve Patterson','Sales',9441),
+('Foon Yue Tseng','Sales',6660),
+('George Vanauf','Sales',10563),
+('Loui Bondur','SCM',10449),
+('Gerard Hernandez','SCM',6949),
+('Pamela Castillo','SCM',11303),
+('Larry Bott','SCM',11798),
+('Barry Jones','SCM',10586)
+AS basic_pays(employee_name, department, salary);
+
+CREATE OR REPLACE TEMPORARY VIEW test_ignore_null AS SELECT * FROM VALUES
+('a', 0, null),
+('a', 1, 'x'),
+('b', 2, null),
+('c', 3, null),
+('a', 4, 'y'),
+('b', 5, null),
+('a', 6, 'z'),
+('a', 7, 'v'),
+('a', 8, null)
+AS test_ignore_null(content, id, v);
+
 -- RowsBetween
 SELECT val, cate, count(val) OVER(PARTITION BY cate ORDER BY val ROWS CURRENT ROW) FROM testData
 ORDER BY cate, val;
@@ -38,6 +70,18 @@ RANGE BETWEEN CURRENT ROW AND 2 FOLLOWING) FROM testData ORDER BY cate, val_date
 SELECT val_timestamp, cate, avg(val_timestamp) OVER(PARTITION BY cate ORDER BY val_timestamp
 RANGE BETWEEN CURRENT ROW AND interval 23 days 4 hours FOLLOWING) FROM testData
 ORDER BY cate, val_timestamp;
+SELECT val_timestamp, cate, avg(val_timestamp) OVER(PARTITION BY cate ORDER BY val_timestamp
+RANGE BETWEEN CURRENT ROW AND interval '1-1' year to month FOLLOWING) FROM testData
+ORDER BY cate, val_timestamp;
+SELECT val_timestamp, cate, avg(val_timestamp) OVER(PARTITION BY cate ORDER BY val_timestamp
+RANGE BETWEEN CURRENT ROW AND interval '1 2:3:4.001' day to second FOLLOWING) FROM testData
+ORDER BY cate, val_timestamp;
+SELECT val_date, cate, avg(val_timestamp) OVER(PARTITION BY cate ORDER BY val_date
+RANGE BETWEEN CURRENT ROW AND interval '1-1' year to month FOLLOWING) FROM testData
+ORDER BY cate, val_date;
+SELECT val_date, cate, avg(val_timestamp) OVER(PARTITION BY cate ORDER BY val_date
+RANGE BETWEEN CURRENT ROW AND interval '1 2:3:4.001' day to second FOLLOWING) FROM testData
+ORDER BY cate, val_date;
 
 -- RangeBetween with reverse OrderBy
 SELECT val, cate, sum(val) OVER(PARTITION BY cate ORDER BY val DESC
@@ -124,4 +168,256 @@ WINDOW w AS (PARTITION BY cate ORDER BY val);
 -- with filter predicate
 SELECT val, cate,
 count(val) FILTER (WHERE val > 1) OVER(PARTITION BY cate)
-FROM testData ORDER BY cate, val;
\ No newline at end of file
+FROM testData ORDER BY cate, val;
+
+-- nth_value()/first_value() over ()
+SELECT
+    employee_name,
+    salary,
+    first_value(employee_name) OVER w highest_salary,
+    nth_value(employee_name, 2) OVER w second_highest_salary
+FROM
+    basic_pays
+WINDOW w AS (ORDER BY salary DESC)
+ORDER BY salary DESC;
+
+SELECT
+    employee_name,
+    salary,
+    first_value(employee_name) OVER w highest_salary,
+    nth_value(employee_name, 2) OVER w second_highest_salary
+FROM
+    basic_pays
+WINDOW w AS (ORDER BY salary DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)
+ORDER BY salary DESC;
+
+SELECT
+    employee_name,
+    salary,
+    first_value(employee_name) OVER w highest_salary,
+    nth_value(employee_name, 2) OVER w second_highest_salary
+FROM
+    basic_pays
+WINDOW w AS (ORDER BY salary DESC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)
+ORDER BY salary DESC;
+
+SELECT
+    employee_name,
+    salary,
+    first_value(employee_name) OVER w highest_salary,
+    nth_value(employee_name, 2) OVER w second_highest_salary
+FROM
+    basic_pays
+WINDOW w AS (ORDER BY salary RANGE BETWEEN 2000 PRECEDING AND 1000 FOLLOWING)
+ORDER BY salary;
+
+SELECT
+    employee_name,
+    salary,
+    first_value(employee_name) OVER w highest_salary,
+    nth_value(employee_name, 2) OVER w second_highest_salary
+FROM
+    basic_pays
+WINDOW w AS (ORDER BY salary DESC ROWS BETWEEN 2 PRECEDING AND 2 FOLLOWING)
+ORDER BY salary DESC;
+
+SELECT
+    employee_name,
+    salary,
+    first_value(employee_name) OVER w highest_salary,
+    nth_value(employee_name, 2) OVER w second_highest_salary
+FROM
+    basic_pays
+WINDOW w AS (ORDER BY salary DESC RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
+ORDER BY salary DESC;
+
+SELECT
+    employee_name,
+    salary,
+    first_value(employee_name) OVER w highest_salary,
+    nth_value(employee_name, 2) OVER w second_highest_salary
+FROM
+    basic_pays
+WINDOW w AS (ORDER BY salary DESC RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)
+ORDER BY salary DESC;
+
+SELECT
+    employee_name,
+    salary,
+    first_value(employee_name) OVER w highest_salary,
+    nth_value(employee_name, 2) OVER w second_highest_salary
+FROM
+    basic_pays
+WINDOW w AS (ORDER BY salary DESC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)
+ORDER BY salary DESC;
+
+SELECT
+    employee_name,
+    salary,
+    first_value(employee_name) OVER w highest_salary,
+    nth_value(employee_name, 2) OVER w second_highest_salary
+FROM
+    basic_pays
+WINDOW w AS (ORDER BY salary DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING)
+ORDER BY salary DESC;
+
+SELECT
+	employee_name,
+	department,
+	salary,
+	FIRST_VALUE(employee_name) OVER w highest_salary,
+	NTH_VALUE(employee_name, 2) OVER w second_highest_salary
+FROM
+	basic_pays
+WINDOW w AS (
+  PARTITION BY department
+  ORDER BY salary DESC
+  RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
+)
+ORDER BY department;
+
+SELECT
+    employee_name,
+    salary,
+    first_value(employee_name) OVER w highest_salary,
+    nth_value(employee_name, 2) OVER w second_highest_salary
+FROM
+    basic_pays
+WINDOW
+    w AS (ORDER BY salary DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING),
+    w AS (ORDER BY salary DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 2 FOLLOWING)
+ORDER BY salary DESC;
+
+SELECT
+    content,
+    id,
+    v,
+    lead(v, 0) IGNORE NULLS OVER w lead_0,
+    lead(v, 1) IGNORE NULLS OVER w lead_1,
+    lead(v, 2) IGNORE NULLS OVER w lead_2,
+    lead(v, 3) IGNORE NULLS OVER w lead_3,
+    lag(v, 0) IGNORE NULLS OVER w lag_0,
+    lag(v, 1) IGNORE NULLS OVER w lag_1,
+    lag(v, 2) IGNORE NULLS OVER w lag_2,
+    lag(v, 3) IGNORE NULLS OVER w lag_3,
+    nth_value(v, 1) IGNORE NULLS OVER w nth_value_1,
+    nth_value(v, 2) IGNORE NULLS OVER w nth_value_2,
+    nth_value(v, 3) IGNORE NULLS OVER w nth_value_3,
+    first_value(v) IGNORE NULLS OVER w first_value,
+    last_value(v) IGNORE NULLS OVER w last_value
+FROM
+    test_ignore_null
+WINDOW w AS (ORDER BY id)
+ORDER BY id;
+
+SELECT
+    content,
+    id,
+    v,
+    nth_value(v, 1) IGNORE NULLS OVER w nth_value_1,
+    nth_value(v, 2) IGNORE NULLS OVER w nth_value_2,
+    nth_value(v, 3) IGNORE NULLS OVER w nth_value_3,
+    first_value(v) IGNORE NULLS OVER w first_value,
+    last_value(v) IGNORE NULLS OVER w last_value
+FROM
+    test_ignore_null
+WINDOW w AS (ORDER BY id RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)
+ORDER BY id;
+
+SELECT
+    content,
+    id,
+    v,
+    nth_value(v, 1) IGNORE NULLS OVER w nth_value_1,
+    nth_value(v, 2) IGNORE NULLS OVER w nth_value_2,
+    nth_value(v, 3) IGNORE NULLS OVER w nth_value_3,
+    first_value(v) IGNORE NULLS OVER w first_value,
+    last_value(v) IGNORE NULLS OVER w last_value
+FROM
+    test_ignore_null
+WINDOW w AS (ORDER BY id ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)
+ORDER BY id;
+
+SELECT
+    content,
+    id,
+    v,
+    nth_value(v, 1) IGNORE NULLS OVER w nth_value_1,
+    nth_value(v, 2) IGNORE NULLS OVER w nth_value_2,
+    nth_value(v, 3) IGNORE NULLS OVER w nth_value_3,
+    first_value(v) IGNORE NULLS OVER w first_value,
+    last_value(v) IGNORE NULLS OVER w last_value
+FROM
+    test_ignore_null
+WINDOW w AS (ORDER BY id RANGE BETWEEN 2 PRECEDING AND 2 FOLLOWING)
+ORDER BY id;
+
+SELECT
+    content,
+    id,
+    v,
+    nth_value(v, 1) IGNORE NULLS OVER w nth_value_1,
+    nth_value(v, 2) IGNORE NULLS OVER w nth_value_2,
+    nth_value(v, 3) IGNORE NULLS OVER w nth_value_3,
+    first_value(v) IGNORE NULLS OVER w first_value,
+    last_value(v) IGNORE NULLS OVER w last_value
+FROM
+    test_ignore_null
+WINDOW w AS (ORDER BY id ROWS BETWEEN 2 PRECEDING AND 2 FOLLOWING)
+ORDER BY id;
+
+SELECT
+    content,
+    id,
+    v,
+    nth_value(v, 1) IGNORE NULLS OVER w nth_value_1,
+    nth_value(v, 2) IGNORE NULLS OVER w nth_value_2,
+    nth_value(v, 3) IGNORE NULLS OVER w nth_value_3,
+    first_value(v) IGNORE NULLS OVER w first_value,
+    last_value(v) IGNORE NULLS OVER w last_value
+FROM
+    test_ignore_null
+WINDOW w AS (ORDER BY id RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
+ORDER BY id;
+
+SELECT
+    content,
+    id,
+    v,
+    nth_value(v, 1) IGNORE NULLS OVER w nth_value_1,
+    nth_value(v, 2) IGNORE NULLS OVER w nth_value_2,
+    nth_value(v, 3) IGNORE NULLS OVER w nth_value_3,
+    first_value(v) IGNORE NULLS OVER w first_value,
+    last_value(v) IGNORE NULLS OVER w last_value
+FROM
+    test_ignore_null
+WINDOW w AS (ORDER BY id RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)
+ORDER BY id;
+
+SELECT
+    content,
+    id,
+    v,
+    nth_value(v, 1) IGNORE NULLS OVER w nth_value_1,
+    nth_value(v, 2) IGNORE NULLS OVER w nth_value_2,
+    nth_value(v, 3) IGNORE NULLS OVER w nth_value_3,
+    first_value(v) IGNORE NULLS OVER w first_value,
+    last_value(v) IGNORE NULLS OVER w last_value
+FROM
+    test_ignore_null
+WINDOW w AS (ORDER BY id ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)
+ORDER BY id;
+
+SELECT
+    content,
+    id,
+    v,
+    nth_value(v, 1) IGNORE NULLS OVER w nth_value_1,
+    nth_value(v, 2) IGNORE NULLS OVER w nth_value_2,
+    nth_value(v, 3) IGNORE NULLS OVER w nth_value_3,
+    first_value(v) IGNORE NULLS OVER w first_value,
+    last_value(v) IGNORE NULLS OVER w last_value
+FROM
+    test_ignore_null
+WINDOW w AS (ORDER BY id ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING)
+ORDER BY id;
\ No newline at end of file
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/array.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/array.sql.out
new file mode 100644
index 0000000000000..a029a28200073
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/array.sql.out
@@ -0,0 +1,234 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 20
+
+
+-- !query
+create temporary view data as select * from values
+  ("one", array(11, 12, 13), array(array(111, 112, 113), array(121, 122, 123))),
+  ("two", array(21, 22, 23), array(array(211, 212, 213), array(221, 222, 223)))
+  as data(a, b, c)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+select * from data
+-- !query schema
+struct<a:string,b:array<int>,c:array<array<int>>>
+-- !query output
+one	[11,12,13]	[[111,112,113],[121,122,123]]
+two	[21,22,23]	[[211,212,213],[221,222,223]]
+
+
+-- !query
+select a, b[0], b[0] + b[1] from data
+-- !query schema
+struct<a:string,b[0]:int,(b[0] + b[1]):int>
+-- !query output
+one	11	23
+two	21	43
+
+
+-- !query
+select a, c[0][0] + c[0][0 + 1] from data
+-- !query schema
+struct<a:string,(c[0][0] + c[0][(0 + 1)]):int>
+-- !query output
+one	223
+two	423
+
+
+-- !query
+create temporary view primitive_arrays as select * from values (
+  array(true),
+  array(2Y, 1Y),
+  array(2S, 1S),
+  array(2, 1),
+  array(2L, 1L),
+  array(9223372036854775809, 9223372036854775808),
+  array(2.0D, 1.0D),
+  array(float(2.0), float(1.0)),
+  array(date '2016-03-14', date '2016-03-13'),
+  array(timestamp '2016-11-15 20:54:00.000',  timestamp '2016-11-12 20:54:00.000')
+) as primitive_arrays(
+  boolean_array,
+  tinyint_array,
+  smallint_array,
+  int_array,
+  bigint_array,
+  decimal_array,
+  double_array,
+  float_array,
+  date_array,
+  timestamp_array
+)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+select * from primitive_arrays
+-- !query schema
+struct<boolean_array:array<boolean>,tinyint_array:array<tinyint>,smallint_array:array<smallint>,int_array:array<int>,bigint_array:array<bigint>,decimal_array:array<decimal(19,0)>,double_array:array<double>,float_array:array<float>,date_array:array<date>,timestamp_array:array<timestamp>>
+-- !query output
+[true]	[2,1]	[2,1]	[2,1]	[2,1]	[9223372036854775809,9223372036854775808]	[2.0,1.0]	[2.0,1.0]	[2016-03-14,2016-03-13]	[2016-11-15 20:54:00,2016-11-12 20:54:00]
+
+
+-- !query
+select
+  array_contains(boolean_array, true), array_contains(boolean_array, false),
+  array_contains(tinyint_array, 2Y), array_contains(tinyint_array, 0Y),
+  array_contains(smallint_array, 2S), array_contains(smallint_array, 0S),
+  array_contains(int_array, 2), array_contains(int_array, 0),
+  array_contains(bigint_array, 2L), array_contains(bigint_array, 0L),
+  array_contains(decimal_array, 9223372036854775809), array_contains(decimal_array, 1),
+  array_contains(double_array, 2.0D), array_contains(double_array, 0.0D),
+  array_contains(float_array, float(2.0)), array_contains(float_array, float(0.0)),
+  array_contains(date_array, date '2016-03-14'), array_contains(date_array, date '2016-01-01'),
+  array_contains(timestamp_array, timestamp '2016-11-15 20:54:00.000'), array_contains(timestamp_array, timestamp '2016-01-01 20:54:00.000')
+from primitive_arrays
+-- !query schema
+struct<array_contains(boolean_array, true):boolean,array_contains(boolean_array, false):boolean,array_contains(tinyint_array, 2):boolean,array_contains(tinyint_array, 0):boolean,array_contains(smallint_array, 2):boolean,array_contains(smallint_array, 0):boolean,array_contains(int_array, 2):boolean,array_contains(int_array, 0):boolean,array_contains(bigint_array, 2):boolean,array_contains(bigint_array, 0):boolean,array_contains(decimal_array, 9223372036854775809):boolean,array_contains(decimal_array, 1):boolean,array_contains(double_array, 2.0):boolean,array_contains(double_array, 0.0):boolean,array_contains(float_array, 2.0):boolean,array_contains(float_array, 0.0):boolean,array_contains(date_array, DATE '2016-03-14'):boolean,array_contains(date_array, DATE '2016-01-01'):boolean,array_contains(timestamp_array, TIMESTAMP '2016-11-15 20:54:00'):boolean,array_contains(timestamp_array, TIMESTAMP '2016-01-01 20:54:00'):boolean>
+-- !query output
+true	false	true	false	true	false	true	false	true	false	true	false	true	false	true	false	true	false	true	false
+
+
+-- !query
+select array_contains(b, 11), array_contains(c, array(111, 112, 113)) from data
+-- !query schema
+struct<array_contains(b, 11):boolean,array_contains(c, array(111, 112, 113)):boolean>
+-- !query output
+false	false
+true	true
+
+
+-- !query
+select
+  sort_array(boolean_array),
+  sort_array(tinyint_array),
+  sort_array(smallint_array),
+  sort_array(int_array),
+  sort_array(bigint_array),
+  sort_array(decimal_array),
+  sort_array(double_array),
+  sort_array(float_array),
+  sort_array(date_array),
+  sort_array(timestamp_array)
+from primitive_arrays
+-- !query schema
+struct<sort_array(boolean_array, true):array<boolean>,sort_array(tinyint_array, true):array<tinyint>,sort_array(smallint_array, true):array<smallint>,sort_array(int_array, true):array<int>,sort_array(bigint_array, true):array<bigint>,sort_array(decimal_array, true):array<decimal(19,0)>,sort_array(double_array, true):array<double>,sort_array(float_array, true):array<float>,sort_array(date_array, true):array<date>,sort_array(timestamp_array, true):array<timestamp>>
+-- !query output
+[true]	[1,2]	[1,2]	[1,2]	[1,2]	[9223372036854775808,9223372036854775809]	[1.0,2.0]	[1.0,2.0]	[2016-03-13,2016-03-14]	[2016-11-12 20:54:00,2016-11-15 20:54:00]
+
+
+-- !query
+select sort_array(array('b', 'd'), '1')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'sort_array(array('b', 'd'), '1')' due to data type mismatch: Sort order in second argument requires a boolean literal.; line 1 pos 7
+
+
+-- !query
+select sort_array(array('b', 'd'), cast(NULL as boolean))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'sort_array(array('b', 'd'), CAST(NULL AS BOOLEAN))' due to data type mismatch: Sort order in second argument requires a boolean literal.; line 1 pos 7
+
+
+-- !query
+select
+  size(boolean_array),
+  size(tinyint_array),
+  size(smallint_array),
+  size(int_array),
+  size(bigint_array),
+  size(decimal_array),
+  size(double_array),
+  size(float_array),
+  size(date_array),
+  size(timestamp_array)
+from primitive_arrays
+-- !query schema
+struct<size(boolean_array):int,size(tinyint_array):int,size(smallint_array):int,size(int_array):int,size(bigint_array):int,size(decimal_array):int,size(double_array):int,size(float_array):int,size(date_array):int,size(timestamp_array):int>
+-- !query output
+1	2	2	2	2	2	2	2	2	2
+
+
+-- !query
+select element_at(array(1, 2, 3), 5)
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArrayIndexOutOfBoundsException
+Invalid index: 5, numElements: 3
+
+
+-- !query
+select element_at(array(1, 2, 3), -5)
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArrayIndexOutOfBoundsException
+Invalid index: -5, numElements: 3
+
+
+-- !query
+select element_at(array(1, 2, 3), 0)
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArrayIndexOutOfBoundsException
+SQL array indices start at 1
+
+
+-- !query
+select elt(4, '123', '456')
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArrayIndexOutOfBoundsException
+Invalid index: 4, numElements: 2
+
+
+-- !query
+select elt(0, '123', '456')
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArrayIndexOutOfBoundsException
+Invalid index: 0, numElements: 2
+
+
+-- !query
+select elt(-1, '123', '456')
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArrayIndexOutOfBoundsException
+Invalid index: -1, numElements: 2
+
+
+-- !query
+select array(1, 2, 3)[5]
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArrayIndexOutOfBoundsException
+Invalid index: 5, numElements: 3
+
+
+-- !query
+select array(1, 2, 3)[-1]
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArrayIndexOutOfBoundsException
+Invalid index: -1, numElements: 3
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
index aad1e5f343879..5bcc08f2bd618 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
@@ -1,5 +1,130 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 85
+-- Number of queries: 128
+
+
+-- !query
+select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null)
+-- !query schema
+struct<timestamp_seconds(1230219000):timestamp,timestamp_seconds(-1230219000):timestamp,timestamp_seconds(NULL):timestamp>
+-- !query output
+2008-12-25 07:30:00	1931-01-07 00:30:00	NULL
+
+
+-- !query
+select TIMESTAMP_SECONDS(1.23), TIMESTAMP_SECONDS(1.23d), TIMESTAMP_SECONDS(FLOAT(1.23))
+-- !query schema
+struct<timestamp_seconds(1.23):timestamp,timestamp_seconds(1.23):timestamp,timestamp_seconds(1.23):timestamp>
+-- !query output
+1969-12-31 16:00:01.23	1969-12-31 16:00:01.23	1969-12-31 16:00:01.23
+
+
+-- !query
+select TIMESTAMP_MILLIS(1230219000123),TIMESTAMP_MILLIS(-1230219000123),TIMESTAMP_MILLIS(null)
+-- !query schema
+struct<timestamp_millis(1230219000123):timestamp,timestamp_millis(-1230219000123):timestamp,timestamp_millis(NULL):timestamp>
+-- !query output
+2008-12-25 07:30:00.123	1931-01-07 00:29:59.877	NULL
+
+
+-- !query
+select TIMESTAMP_MICROS(1230219000123123),TIMESTAMP_MICROS(-1230219000123123),TIMESTAMP_MICROS(null)
+-- !query schema
+struct<timestamp_micros(1230219000123123):timestamp,timestamp_micros(-1230219000123123):timestamp,timestamp_micros(NULL):timestamp>
+-- !query output
+2008-12-25 07:30:00.123123	1931-01-07 00:29:59.876877	NULL
+
+
+-- !query
+select TIMESTAMP_SECONDS(1230219000123123)
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+long overflow
+
+
+-- !query
+select TIMESTAMP_SECONDS(-1230219000123123)
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+long overflow
+
+
+-- !query
+select TIMESTAMP_MILLIS(92233720368547758)
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+long overflow
+
+
+-- !query
+select TIMESTAMP_MILLIS(-92233720368547758)
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+long overflow
+
+
+-- !query
+select TIMESTAMP_SECONDS(0.1234567)
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+Rounding necessary
+
+
+-- !query
+select TIMESTAMP_SECONDS(0.1234567d), TIMESTAMP_SECONDS(FLOAT(0.1234567))
+-- !query schema
+struct<timestamp_seconds(0.1234567):timestamp,timestamp_seconds(0.1234567):timestamp>
+-- !query output
+1969-12-31 16:00:00.123456	1969-12-31 16:00:00.123456
+
+
+-- !query
+select UNIX_SECONDS(TIMESTAMP('2020-12-01 14:30:08Z')), UNIX_SECONDS(TIMESTAMP('2020-12-01 14:30:08.999999Z')), UNIX_SECONDS(null)
+-- !query schema
+struct<unix_seconds(2020-12-01 14:30:08Z):bigint,unix_seconds(2020-12-01 14:30:08.999999Z):bigint,unix_seconds(NULL):bigint>
+-- !query output
+1606833008	1606833008	NULL
+
+
+-- !query
+select UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08Z')), UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08.999999Z')), UNIX_MILLIS(null)
+-- !query schema
+struct<unix_millis(2020-12-01 14:30:08Z):bigint,unix_millis(2020-12-01 14:30:08.999999Z):bigint,unix_millis(NULL):bigint>
+-- !query output
+1606833008000	1606833008999	NULL
+
+
+-- !query
+select UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08Z')), UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08.999999Z')), UNIX_MICROS(null)
+-- !query schema
+struct<unix_micros(2020-12-01 14:30:08Z):bigint,unix_micros(2020-12-01 14:30:08.999999Z):bigint,unix_micros(NULL):bigint>
+-- !query output
+1606833008000000	1606833008999999	NULL
+
+
+-- !query
+select DATE_FROM_UNIX_DATE(0), DATE_FROM_UNIX_DATE(1000), DATE_FROM_UNIX_DATE(null)
+-- !query schema
+struct<date_from_unix_date(0):date,date_from_unix_date(1000):date,date_from_unix_date(NULL):date>
+-- !query output
+1970-01-01	1972-09-27	NULL
+
+
+-- !query
+select UNIX_DATE(DATE('1970-01-01')), UNIX_DATE(DATE('2020-12-04')), UNIX_DATE(null)
+-- !query schema
+struct<unix_date(1970-01-01):int,unix_date(2020-12-04):int,unix_date(NULL):int>
+-- !query output
+0	18600	NULL
 
 
 -- !query
@@ -35,7 +160,7 @@ NULL	2016-12-31 00:12:00	2016-12-31 00:00:00
 -- !query
 select dayofweek('2007-02-03'), dayofweek('2009-07-30'), dayofweek('2017-05-27'), dayofweek(null), dayofweek('1582-10-15 13:10:15')
 -- !query schema
-struct<dayofweek(CAST(2007-02-03 AS DATE)):int,dayofweek(CAST(2009-07-30 AS DATE)):int,dayofweek(CAST(2017-05-27 AS DATE)):int,dayofweek(CAST(NULL AS DATE)):int,dayofweek(CAST(1582-10-15 13:10:15 AS DATE)):int>
+struct<dayofweek(2007-02-03):int,dayofweek(2009-07-30):int,dayofweek(2017-05-27):int,dayofweek(NULL):int,dayofweek(1582-10-15 13:10:15):int>
 -- !query output
 7	5	7	NULL	6
 
@@ -106,7 +231,7 @@ struct<a:int,b:int>
 -- !query
 select weekday('2007-02-03'), weekday('2009-07-30'), weekday('2017-05-27'), weekday(null), weekday('1582-10-15 13:10:15')
 -- !query schema
-struct<weekday(CAST(2007-02-03 AS DATE)):int,weekday(CAST(2009-07-30 AS DATE)):int,weekday(CAST(2017-05-27 AS DATE)):int,weekday(CAST(NULL AS DATE)):int,weekday(CAST(1582-10-15 13:10:15 AS DATE)):int>
+struct<weekday(2007-02-03):int,weekday(2009-07-30):int,weekday(2017-05-27):int,weekday(NULL):int,weekday(1582-10-15 13:10:15):int>
 -- !query output
 5	3	5	NULL	4
 
@@ -114,15 +239,9 @@ struct<weekday(CAST(2007-02-03 AS DATE)):int,weekday(CAST(2009-07-30 AS DATE)):i
 -- !query
 select year('1500-01-01'), month('1500-01-01'), dayOfYear('1500-01-01')
 -- !query schema
-struct<>
+struct<year(1500-01-01):int,month(1500-01-01):int,dayofyear(1500-01-01):int>
 -- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
-
-no viable alternative at input 'year'(line 1, pos 7)
-
-== SQL ==
-select year('1500-01-01'), month('1500-01-01'), dayOfYear('1500-01-01')
--------^^^
+1500	1	1
 
 
 -- !query
@@ -141,10 +260,38 @@ struct<TIMESTAMP '2019-01-01 00:00:00':timestamp>
 2019-01-01 00:00:00
 
 
+-- !query
+select date '2020-01-01中文'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the DATE value: 2020-01-01中文(line 1, pos 7)
+
+== SQL ==
+select date '2020-01-01中文'
+-------^^^
+
+
+-- !query
+select timestamp '2019-01-01中文'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the TIMESTAMP value: 2019-01-01中文(line 1, pos 7)
+
+== SQL ==
+select timestamp '2019-01-01中文'
+-------^^^
+
+
 -- !query
 select timestamp'2011-11-11 11:11:11' + interval '2' day
 -- !query schema
-struct<CAST(TIMESTAMP '2011-11-11 11:11:11' + INTERVAL '2 days' AS TIMESTAMP):timestamp>
+struct<TIMESTAMP '2011-11-11 11:11:11' + INTERVAL '2 days':timestamp>
 -- !query output
 2011-11-13 11:11:11
 
@@ -152,7 +299,7 @@ struct<CAST(TIMESTAMP '2011-11-11 11:11:11' + INTERVAL '2 days' AS TIMESTAMP):ti
 -- !query
 select timestamp'2011-11-11 11:11:11' - interval '2' day
 -- !query schema
-struct<CAST(TIMESTAMP '2011-11-11 11:11:11' - INTERVAL '2 days' AS TIMESTAMP):timestamp>
+struct<TIMESTAMP '2011-11-11 11:11:11' - INTERVAL '2 days':timestamp>
 -- !query output
 2011-11-09 11:11:11
 
@@ -178,7 +325,7 @@ requirement failed: Cannot add hours, minutes or seconds, milliseconds, microsec
 -- !query
 select '2011-11-11' - interval '2' day
 -- !query schema
-struct<CAST(2011-11-11 - INTERVAL '2 days' AS STRING):string>
+struct<2011-11-11 - INTERVAL '2 days':string>
 -- !query output
 2011-11-09 00:00:00
 
@@ -186,7 +333,7 @@ struct<CAST(2011-11-11 - INTERVAL '2 days' AS STRING):string>
 -- !query
 select '2011-11-11 11:11:11' - interval '2' second
 -- !query schema
-struct<CAST(2011-11-11 11:11:11 - INTERVAL '2 seconds' AS STRING):string>
+struct<2011-11-11 11:11:11 - INTERVAL '2 seconds':string>
 -- !query output
 2011-11-11 11:11:09
 
@@ -194,9 +341,10 @@ struct<CAST(2011-11-11 11:11:11 - INTERVAL '2 seconds' AS STRING):string>
 -- !query
 select '1' - interval '2' second
 -- !query schema
-struct<CAST(1 - INTERVAL '2 seconds' AS STRING):string>
+struct<>
 -- !query output
-NULL
+java.time.DateTimeException
+Cannot cast 1 to TimestampType.
 
 
 -- !query
@@ -211,23 +359,23 @@ cannot resolve '1 + (- INTERVAL '2 seconds')' due to data type mismatch: argumen
 -- !query
 select date'2020-01-01' - timestamp'2019-10-06 10:11:12.345678'
 -- !query schema
-struct<subtracttimestamps(CAST(DATE '2020-01-01' AS TIMESTAMP), TIMESTAMP '2019-10-06 10:11:12.345678'):interval>
+struct<(DATE '2020-01-01' - TIMESTAMP '2019-10-06 10:11:12.345678'):interval day to second>
 -- !query output
-2078 hours 48 minutes 47.654322 seconds
+86 13:48:47.654322000
 
 
 -- !query
 select timestamp'2019-10-06 10:11:12.345678' - date'2020-01-01'
 -- !query schema
-struct<subtracttimestamps(TIMESTAMP '2019-10-06 10:11:12.345678', CAST(DATE '2020-01-01' AS TIMESTAMP)):interval>
+struct<(TIMESTAMP '2019-10-06 10:11:12.345678' - DATE '2020-01-01'):interval day to second>
 -- !query output
--2078 hours -48 minutes -47.654322 seconds
+-86 13:48:47.654322000
 
 
 -- !query
 select timestamp'2019-10-06 10:11:12.345678' - null
 -- !query schema
-struct<subtracttimestamps(TIMESTAMP '2019-10-06 10:11:12.345678', CAST(NULL AS TIMESTAMP)):interval>
+struct<(TIMESTAMP '2019-10-06 10:11:12.345678' - NULL):interval day to second>
 -- !query output
 NULL
 
@@ -235,7 +383,7 @@ NULL
 -- !query
 select null - timestamp'2019-10-06 10:11:12.345678'
 -- !query schema
-struct<subtracttimestamps(CAST(NULL AS TIMESTAMP), TIMESTAMP '2019-10-06 10:11:12.345678'):interval>
+struct<(NULL - TIMESTAMP '2019-10-06 10:11:12.345678'):interval day to second>
 -- !query output
 NULL
 
@@ -243,7 +391,7 @@ NULL
 -- !query
 select date_add('2011-11-11', 1Y)
 -- !query schema
-struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+struct<date_add(2011-11-11, 1):date>
 -- !query output
 2011-11-12
 
@@ -251,7 +399,7 @@ struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
 -- !query
 select date_add('2011-11-11', 1S)
 -- !query schema
-struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+struct<date_add(2011-11-11, 1):date>
 -- !query output
 2011-11-12
 
@@ -259,7 +407,7 @@ struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
 -- !query
 select date_add('2011-11-11', 1)
 -- !query schema
-struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+struct<date_add(2011-11-11, 1):date>
 -- !query output
 2011-11-12
 
@@ -294,7 +442,7 @@ cannot resolve 'date_add(CAST('2011-11-11' AS DATE), 10.0D)' due to data type mi
 -- !query
 select date_add('2011-11-11', '1')
 -- !query schema
-struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+struct<date_add(2011-11-11, 1):date>
 -- !query output
 2011-11-12
 
@@ -305,7 +453,7 @@ select date_add('2011-11-11', '1.2')
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-The second argument of 'date_add' function needs to be an integer.;
+The second argument of 'date_add' function needs to be an integer.
 
 
 -- !query
@@ -319,7 +467,7 @@ struct<date_add(DATE '2011-11-11', 1):date>
 -- !query
 select date_add(timestamp'2011-11-11', 1)
 -- !query schema
-struct<date_add(CAST(TIMESTAMP '2011-11-11 00:00:00' AS DATE), 1):date>
+struct<date_add(TIMESTAMP '2011-11-11 00:00:00', 1):date>
 -- !query output
 2011-11-12
 
@@ -346,13 +494,13 @@ select date_sub(date'2011-11-11', '1.2')
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-The second argument of 'date_sub' function needs to be an integer.;
+The second argument of 'date_sub' function needs to be an integer.
 
 
 -- !query
 select date_sub(timestamp'2011-11-11', 1)
 -- !query schema
-struct<date_sub(CAST(TIMESTAMP '2011-11-11 00:00:00' AS DATE), 1):date>
+struct<date_sub(TIMESTAMP '2011-11-11 00:00:00', 1):date>
 -- !query output
 2011-11-10
 
@@ -360,7 +508,7 @@ struct<date_sub(CAST(TIMESTAMP '2011-11-11 00:00:00' AS DATE), 1):date>
 -- !query
 select date_sub(null, 1)
 -- !query schema
-struct<date_sub(CAST(NULL AS DATE), 1):date>
+struct<date_sub(NULL, 1):date>
 -- !query output
 NULL
 
@@ -368,7 +516,7 @@ NULL
 -- !query
 select date_sub(date'2011-11-11', null)
 -- !query schema
-struct<date_sub(DATE '2011-11-11', CAST(NULL AS INT)):date>
+struct<date_sub(DATE '2011-11-11', NULL):date>
 -- !query output
 NULL
 
@@ -388,13 +536,13 @@ select date'2011-11-11' + '1'
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'date_add(DATE '2011-11-11', CAST('1' AS DOUBLE))' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'CAST('1' AS DOUBLE)' is of double type.; line 1 pos 7
+cannot resolve 'date_add(DATE '2011-11-11', CAST('1' AS DATE))' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'CAST('1' AS DATE)' is of date type.; line 1 pos 7
 
 
 -- !query
 select null + date '2001-09-28'
 -- !query schema
-struct<date_add(DATE '2001-09-28', CAST(NULL AS INT)):date>
+struct<date_add(DATE '2001-09-28', NULL):date>
 -- !query output
 NULL
 
@@ -428,14 +576,14 @@ select date '2001-10-01' - '7'
 -- !query schema
 struct<>
 -- !query output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'date_sub(DATE '2001-10-01', CAST('7' AS DOUBLE))' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'CAST('7' AS DOUBLE)' is of double type.; line 1 pos 7
+java.time.DateTimeException
+Cannot cast 7 to DateType.
 
 
 -- !query
 select date '2001-09-28' + null
 -- !query schema
-struct<date_add(DATE '2001-09-28', CAST(NULL AS INT)):date>
+struct<date_add(DATE '2001-09-28', NULL):date>
 -- !query output
 NULL
 
@@ -443,7 +591,7 @@ NULL
 -- !query
 select date '2001-09-28' - null
 -- !query schema
-struct<date_sub(DATE '2001-09-28', CAST(NULL AS INT)):date>
+struct<date_sub(DATE '2001-09-28', NULL):date>
 -- !query output
 NULL
 
@@ -462,7 +610,7 @@ select date_add('2011-11-11', str) from v
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'date_add(CAST('2011-11-11' AS DATE), v.`str`)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'v.`str`' is of string type.; line 1 pos 7
+cannot resolve 'date_add(CAST('2011-11-11' AS DATE), v.str)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'v.str' is of string type.; line 1 pos 7
 
 
 -- !query
@@ -471,13 +619,13 @@ select date_sub('2011-11-11', str) from v
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'date_sub(CAST('2011-11-11' AS DATE), v.`str`)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'v.`str`' is of string type.; line 1 pos 7
+cannot resolve 'date_sub(CAST('2011-11-11' AS DATE), v.str)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'v.str' is of string type.; line 1 pos 7
 
 
 -- !query
 select null - date '2019-10-06'
 -- !query schema
-struct<subtractdates(CAST(NULL AS DATE), DATE '2019-10-06'):interval>
+struct<(NULL - DATE '2019-10-06'):interval day to second>
 -- !query output
 NULL
 
@@ -485,17 +633,18 @@ NULL
 -- !query
 select date '2001-10-01' - date '2001-09-28'
 -- !query schema
-struct<subtractdates(DATE '2001-10-01', DATE '2001-09-28'):interval>
+struct<(DATE '2001-10-01' - DATE '2001-09-28'):interval day to second>
 -- !query output
-3 days
+3 00:00:00.000000000
 
 
 -- !query
 select to_timestamp('2019-10-06 10:11:12.', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
 -- !query schema
-struct<to_timestamp(2019-10-06 10:11:12., yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]):timestamp>
+struct<>
 -- !query output
-NULL
+java.time.format.DateTimeParseException
+Text '2019-10-06 10:11:12.' could not be parsed at index 20
 
 
 -- !query
@@ -557,9 +706,10 @@ struct<to_timestamp(2019-10-06 10:11:12.123456PST, yyyy-MM-dd HH:mm:ss.SSSSSS[zz
 -- !query
 select to_timestamp('2019-10-06 10:11:12.1234567PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
 -- !query schema
-struct<to_timestamp(2019-10-06 10:11:12.1234567PST, yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]):timestamp>
+struct<>
 -- !query output
-NULL
+java.time.format.DateTimeParseException
+Text '2019-10-06 10:11:12.1234567PST' could not be parsed, unparsed text found at index 26
 
 
 -- !query
@@ -573,9 +723,10 @@ struct<to_timestamp(123456 2019-10-06 10:11:12.123456PST, SSSSSS yyyy-MM-dd HH:m
 -- !query
 select to_timestamp('223456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
 -- !query schema
-struct<to_timestamp(223456 2019-10-06 10:11:12.123456PST, SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]):timestamp>
+struct<>
 -- !query output
-NULL
+java.time.format.DateTimeParseException
+Text '223456 2019-10-06 10:11:12.123456PST' could not be parsed at index 27
 
 
 -- !query
@@ -637,17 +788,19 @@ struct<to_timestamp(12.12342019-10-06S10:11, ss.SSSSyyyy-MM-dd'S'HH:mm):timestam
 -- !query
 select to_timestamp("12.1232019-10-06S10:11", "ss.SSSSyyyy-MM-dd'S'HH:mm")
 -- !query schema
-struct<to_timestamp(12.1232019-10-06S10:11, ss.SSSSyyyy-MM-dd'S'HH:mm):timestamp>
+struct<>
 -- !query output
-NULL
+java.time.format.DateTimeParseException
+Text '12.1232019-10-06S10:11' could not be parsed at index 7
 
 
 -- !query
 select to_timestamp("12.1232019-10-06S10:11", "ss.SSSSyy-MM-dd'S'HH:mm")
 -- !query schema
-struct<to_timestamp(12.1232019-10-06S10:11, ss.SSSSyy-MM-dd'S'HH:mm):timestamp>
+struct<>
 -- !query output
-NULL
+java.time.format.DateTimeParseException
+Text '12.1232019-10-06S10:11' could not be parsed at index 9
 
 
 -- !query
@@ -675,58 +828,286 @@ struct<to_timestamp(S2019-10-06, 'S'yyyy-MM-dd):timestamp>
 
 
 -- !query
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuee')
+select to_timestamp("2019-10-06T10:11:12'12", "yyyy-MM-dd'T'HH:mm:ss''SSSS")
+-- !query schema
+struct<to_timestamp(2019-10-06T10:11:12'12, yyyy-MM-dd'T'HH:mm:ss''SSSS):timestamp>
+-- !query output
+2019-10-06 10:11:12.12
+
+
+-- !query
+select to_timestamp("2019-10-06T10:11:12'", "yyyy-MM-dd'T'HH:mm:ss''")
+-- !query schema
+struct<to_timestamp(2019-10-06T10:11:12', yyyy-MM-dd'T'HH:mm:ss''):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("'2019-10-06T10:11:12", "''yyyy-MM-dd'T'HH:mm:ss")
+-- !query schema
+struct<to_timestamp('2019-10-06T10:11:12, ''yyyy-MM-dd'T'HH:mm:ss):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("P2019-10-06T10:11:12", "'P'yyyy-MM-dd'T'HH:mm:ss")
+-- !query schema
+struct<to_timestamp(P2019-10-06T10:11:12, 'P'yyyy-MM-dd'T'HH:mm:ss):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("16", "dd")
+-- !query schema
+struct<to_timestamp(16, dd):timestamp>
+-- !query output
+1970-01-16 00:00:00
+
+
+-- !query
+select to_timestamp("02-29", "MM-dd")
 -- !query schema
 struct<>
 -- !query output
-java.lang.IllegalArgumentException
-Illegal pattern character: e
+java.time.DateTimeException
+Invalid date 'February 29' as '1970' is not a leap year
 
 
 -- !query
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uucc')
+select to_date("16", "dd")
+-- !query schema
+struct<to_date(16, dd):date>
+-- !query output
+1970-01-16
+
+
+-- !query
+select to_date("02-29", "MM-dd")
 -- !query schema
 struct<>
 -- !query output
-java.lang.IllegalArgumentException
-Illegal pattern character: c
+java.time.DateTimeException
+Invalid date 'February 29' as '1970' is not a leap year
 
 
 -- !query
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuuu')
+select to_timestamp("2019 40", "yyyy mm")
 -- !query schema
-struct<date_format(TIMESTAMP '2019-10-06 00:00:00', yyyy-MM-dd uuuu):string>
+struct<to_timestamp(2019 40, yyyy mm):timestamp>
 -- !query output
-2019-10-06 Sunday
+2019-01-01 00:40:00
 
 
 -- !query
-select to_timestamp("2019-10-06T10:11:12'12", "yyyy-MM-dd'T'HH:mm:ss''SSSS")
+select to_timestamp("2019 10:10:10", "yyyy hh:mm:ss")
 -- !query schema
-struct<to_timestamp(2019-10-06T10:11:12'12, yyyy-MM-dd'T'HH:mm:ss''SSSS):timestamp>
+struct<to_timestamp(2019 10:10:10, yyyy hh:mm:ss):timestamp>
 -- !query output
-2019-10-06 10:11:12.12
+2019-01-01 10:10:10
 
 
 -- !query
-select to_timestamp("2019-10-06T10:11:12'", "yyyy-MM-dd'T'HH:mm:ss''")
+select to_timestamp('2019-10-06 A', 'yyyy-MM-dd GGGGG')
 -- !query schema
-struct<to_timestamp(2019-10-06T10:11:12', yyyy-MM-dd'T'HH:mm:ss''):timestamp>
+struct<>
 -- !query output
-2019-10-06 10:11:12
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyy-MM-dd GGGGG' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
 
 
 -- !query
-select to_timestamp("'2019-10-06T10:11:12", "''yyyy-MM-dd'T'HH:mm:ss")
+select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEEE')
 -- !query schema
-struct<to_timestamp('2019-10-06T10:11:12, ''yyyy-MM-dd'T'HH:mm:ss):timestamp>
+struct<>
 -- !query output
-2019-10-06 10:11:12
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd MM yyyy EEEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
 
 
 -- !query
-select to_timestamp("P2019-10-06T10:11:12", "'P'yyyy-MM-dd'T'HH:mm:ss")
+select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE')
 -- !query schema
-struct<to_timestamp(P2019-10-06T10:11:12, 'P'yyyy-MM-dd'T'HH:mm:ss):timestamp>
+struct<>
 -- !query output
-2019-10-06 10:11:12
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd MM yyyy EEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select unix_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd MM yyyy EEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_json('{"t":"26/October/2015"}', 't Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_json('{"d":"26/October/2015"}', 'd Date', map('dateFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_csv('26/October/2015', 't Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_csv('26/October/2015', 'd Date', map('dateFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select to_date("2020-01-27T20:06:11.847", "yyyy-MM-dd HH:mm:ss.SSS")
+-- !query schema
+struct<>
+-- !query output
+java.time.format.DateTimeParseException
+Text '2020-01-27T20:06:11.847' could not be parsed at index 10
+
+
+-- !query
+select to_date("Unparseable", "yyyy-MM-dd HH:mm:ss.SSS")
+-- !query schema
+struct<>
+-- !query output
+java.time.format.DateTimeParseException
+Text 'Unparseable' could not be parsed at index 0
+
+
+-- !query
+select to_timestamp("2020-01-27T20:06:11.847", "yyyy-MM-dd HH:mm:ss.SSS")
+-- !query schema
+struct<>
+-- !query output
+java.time.format.DateTimeParseException
+Text '2020-01-27T20:06:11.847' could not be parsed at index 10
+
+
+-- !query
+select to_timestamp("Unparseable", "yyyy-MM-dd HH:mm:ss.SSS")
+-- !query schema
+struct<>
+-- !query output
+java.time.format.DateTimeParseException
+Text 'Unparseable' could not be parsed at index 0
+
+
+-- !query
+select unix_timestamp("2020-01-27T20:06:11.847", "yyyy-MM-dd HH:mm:ss.SSS")
+-- !query schema
+struct<>
+-- !query output
+java.time.format.DateTimeParseException
+Text '2020-01-27T20:06:11.847' could not be parsed at index 10
+
+
+-- !query
+select unix_timestamp("Unparseable", "yyyy-MM-dd HH:mm:ss.SSS")
+-- !query schema
+struct<>
+-- !query output
+java.time.format.DateTimeParseException
+Text 'Unparseable' could not be parsed at index 0
+
+
+-- !query
+select to_unix_timestamp("2020-01-27T20:06:11.847", "yyyy-MM-dd HH:mm:ss.SSS")
+-- !query schema
+struct<>
+-- !query output
+java.time.format.DateTimeParseException
+Text '2020-01-27T20:06:11.847' could not be parsed at index 10
+
+
+-- !query
+select to_unix_timestamp("Unparseable", "yyyy-MM-dd HH:mm:ss.SSS")
+-- !query schema
+struct<>
+-- !query output
+java.time.format.DateTimeParseException
+Text 'Unparseable' could not be parsed at index 0
+
+
+-- !query
+select cast("Unparseable" as timestamp)
+-- !query schema
+struct<>
+-- !query output
+java.time.DateTimeException
+Cannot cast Unparseable to TimestampType.
+
+
+-- !query
+select cast("Unparseable" as date)
+-- !query schema
+struct<>
+-- !query output
+java.time.DateTimeException
+Cannot cast Unparseable to DateType.
+
+
+-- !query
+select next_day("2015-07-23", "Mon")
+-- !query schema
+struct<next_day(2015-07-23, Mon):date>
+-- !query output
+2015-07-27
+
+
+-- !query
+select next_day("2015-07-23", "xx")
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal input for day of week: xx
+
+
+-- !query
+select next_day("xx", "Mon")
+-- !query schema
+struct<>
+-- !query output
+java.time.DateTimeException
+Cannot cast xx to DateType.
+
+
+-- !query
+select next_day(null, "Mon")
+-- !query schema
+struct<next_day(NULL, Mon):date>
+-- !query output
+NULL
+
+
+-- !query
+select next_day(null, "xx")
+-- !query schema
+struct<next_day(NULL, xx):date>
+-- !query output
+NULL
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/decimalArithmeticOperations.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/decimalArithmeticOperations.sql.out
index ce53e1c2863e0..2f3513e734f56 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/decimalArithmeticOperations.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/decimalArithmeticOperations.sql.out
@@ -22,7 +22,7 @@ struct<>
 -- !query
 select id, a*10, b/10 from decimals_test order by id
 -- !query schema
-struct<id:int,(CAST(a AS DECIMAL(38,18)) * CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,15),(CAST(b AS DECIMAL(38,18)) / CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,18)>
+struct<id:int,(a * 10):decimal(38,15),(b / 10):decimal(38,18)>
 -- !query output
 1	1000.000000000000000	99.900000000000000000
 2	123451.230000000000000	1234.512300000000000000
@@ -33,7 +33,7 @@ struct<id:int,(CAST(a AS DECIMAL(38,18)) * CAST(CAST(10 AS DECIMAL(2,0)) AS DECI
 -- !query
 select 10.3 * 3.0
 -- !query schema
-struct<(CAST(10.3 AS DECIMAL(3,1)) * CAST(3.0 AS DECIMAL(3,1))):decimal(6,2)>
+struct<(10.3 * 3.0):decimal(6,2)>
 -- !query output
 30.90
 
@@ -41,7 +41,7 @@ struct<(CAST(10.3 AS DECIMAL(3,1)) * CAST(3.0 AS DECIMAL(3,1))):decimal(6,2)>
 -- !query
 select 10.3000 * 3.0
 -- !query schema
-struct<(CAST(10.3000 AS DECIMAL(6,4)) * CAST(3.0 AS DECIMAL(6,4))):decimal(9,5)>
+struct<(10.3000 * 3.0):decimal(9,5)>
 -- !query output
 30.90000
 
@@ -49,7 +49,7 @@ struct<(CAST(10.3000 AS DECIMAL(6,4)) * CAST(3.0 AS DECIMAL(6,4))):decimal(9,5)>
 -- !query
 select 10.30000 * 30.0
 -- !query schema
-struct<(CAST(10.30000 AS DECIMAL(7,5)) * CAST(30.0 AS DECIMAL(7,5))):decimal(11,6)>
+struct<(10.30000 * 30.0):decimal(11,6)>
 -- !query output
 309.000000
 
@@ -57,7 +57,7 @@ struct<(CAST(10.30000 AS DECIMAL(7,5)) * CAST(30.0 AS DECIMAL(7,5))):decimal(11,
 -- !query
 select 10.300000000000000000 * 3.000000000000000000
 -- !query schema
-struct<(CAST(10.300000000000000000 AS DECIMAL(20,18)) * CAST(3.000000000000000000 AS DECIMAL(20,18))):decimal(38,34)>
+struct<(10.300000000000000000 * 3.000000000000000000):decimal(38,34)>
 -- !query output
 30.9000000000000000000000000000000000
 
@@ -65,7 +65,7 @@ struct<(CAST(10.300000000000000000 AS DECIMAL(20,18)) * CAST(3.00000000000000000
 -- !query
 select 10.300000000000000000 * 3.0000000000000000000
 -- !query schema
-struct<(CAST(10.300000000000000000 AS DECIMAL(21,19)) * CAST(3.0000000000000000000 AS DECIMAL(21,19))):decimal(38,34)>
+struct<(10.300000000000000000 * 3.0000000000000000000):decimal(38,34)>
 -- !query output
 30.9000000000000000000000000000000000
 
@@ -109,7 +109,7 @@ Decimal(expanded,1000000000000000000000000000000000000,37,0}) cannot be represen
 -- !query
 select 123456789123456789.1234567890 * 1.123456789123456789
 -- !query schema
-struct<(CAST(123456789123456789.1234567890 AS DECIMAL(36,18)) * CAST(1.123456789123456789 AS DECIMAL(36,18))):decimal(38,18)>
+struct<(123456789123456789.1234567890 * 1.123456789123456789):decimal(38,18)>
 -- !query output
 138698367904130467.654320988515622621
 
@@ -117,7 +117,7 @@ struct<(CAST(123456789123456789.1234567890 AS DECIMAL(36,18)) * CAST(1.123456789
 -- !query
 select 123456789123456789.1234567890 * 1.123456789123456789
 -- !query schema
-struct<(CAST(123456789123456789.1234567890 AS DECIMAL(36,18)) * CAST(1.123456789123456789 AS DECIMAL(36,18))):decimal(38,18)>
+struct<(123456789123456789.1234567890 * 1.123456789123456789):decimal(38,18)>
 -- !query output
 138698367904130467.654320988515622621
 
@@ -125,7 +125,7 @@ struct<(CAST(123456789123456789.1234567890 AS DECIMAL(36,18)) * CAST(1.123456789
 -- !query
 select 12345678912345.123456789123 / 0.000000012345678
 -- !query schema
-struct<(CAST(12345678912345.123456789123 AS DECIMAL(29,15)) / CAST(1.2345678E-8 AS DECIMAL(29,15))):decimal(38,9)>
+struct<(12345678912345.123456789123 / 1.2345678E-8):decimal(38,9)>
 -- !query output
 1000000073899961059796.725866332
 
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/group-analytics.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/group-analytics.sql.out
new file mode 100644
index 0000000000000..9dbfc4cf4fa18
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/group-analytics.sql.out
@@ -0,0 +1,1293 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 52
+
+
+-- !query
+CREATE OR REPLACE TEMPORARY VIEW testData AS SELECT * FROM VALUES
+(1, 1), (1, 2), (2, 1), (2, 2), (3, 1), (3, 2)
+AS testData(a, b)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SELECT a + b, b, SUM(a - b) FROM testData GROUP BY a + b, b WITH CUBE
+-- !query schema
+struct<(a + b):int,b:int,sum((a - b)):bigint>
+-- !query output
+2	1	0
+2	NULL	0
+3	1	1
+3	2	-1
+3	NULL	0
+4	1	2
+4	2	0
+4	NULL	2
+5	2	1
+5	NULL	1
+NULL	1	3
+NULL	2	0
+NULL	NULL	3
+
+
+-- !query
+SELECT a, b, SUM(b) FROM testData GROUP BY a, b WITH CUBE
+-- !query schema
+struct<a:int,b:int,sum(b):bigint>
+-- !query output
+1	1	1
+1	2	2
+1	NULL	3
+2	1	1
+2	2	2
+2	NULL	3
+3	1	1
+3	2	2
+3	NULL	3
+NULL	1	3
+NULL	2	6
+NULL	NULL	9
+
+
+-- !query
+SELECT a + b, b, SUM(a - b) FROM testData GROUP BY a + b, b WITH ROLLUP
+-- !query schema
+struct<(a + b):int,b:int,sum((a - b)):bigint>
+-- !query output
+2	1	0
+2	NULL	0
+3	1	1
+3	2	-1
+3	NULL	0
+4	1	2
+4	2	0
+4	NULL	2
+5	2	1
+5	NULL	1
+NULL	NULL	3
+
+
+-- !query
+SELECT a, b, SUM(b) FROM testData GROUP BY a, b WITH ROLLUP
+-- !query schema
+struct<a:int,b:int,sum(b):bigint>
+-- !query output
+1	1	1
+1	2	2
+1	NULL	3
+2	1	1
+2	2	2
+2	NULL	3
+3	1	1
+3	2	2
+3	NULL	3
+NULL	NULL	9
+
+
+-- !query
+CREATE OR REPLACE TEMPORARY VIEW courseSales AS SELECT * FROM VALUES
+("dotNET", 2012, 10000), ("Java", 2012, 20000), ("dotNET", 2012, 5000), ("dotNET", 2013, 48000), ("Java", 2013, 30000)
+AS courseSales(course, year, earnings)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY ROLLUP(course, year) ORDER BY course, year
+-- !query schema
+struct<course:string,year:int,sum(earnings):bigint>
+-- !query output
+NULL	NULL	113000
+Java	NULL	50000
+Java	2012	20000
+Java	2013	30000
+dotNET	NULL	63000
+dotNET	2012	15000
+dotNET	2013	48000
+
+
+-- !query
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY ROLLUP(course, year, (course, year)) ORDER BY course, year
+-- !query schema
+struct<course:string,year:int,sum(earnings):bigint>
+-- !query output
+NULL	NULL	113000
+Java	NULL	50000
+Java	2012	20000
+Java	2012	20000
+Java	2013	30000
+Java	2013	30000
+dotNET	NULL	63000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2013	48000
+dotNET	2013	48000
+
+
+-- !query
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY ROLLUP(course, year, (course, year), ()) ORDER BY course, year
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Empty set in ROLLUP grouping sets is not supported.(line 1, pos 61)
+
+== SQL ==
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY ROLLUP(course, year, (course, year), ()) ORDER BY course, year
+-------------------------------------------------------------^^^
+
+
+-- !query
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY CUBE(course, year) ORDER BY course, year
+-- !query schema
+struct<course:string,year:int,sum(earnings):bigint>
+-- !query output
+NULL	NULL	113000
+NULL	2012	35000
+NULL	2013	78000
+Java	NULL	50000
+Java	2012	20000
+Java	2013	30000
+dotNET	NULL	63000
+dotNET	2012	15000
+dotNET	2013	48000
+
+
+-- !query
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY CUBE(course, year, (course, year)) ORDER BY course, year
+-- !query schema
+struct<course:string,year:int,sum(earnings):bigint>
+-- !query output
+NULL	NULL	113000
+NULL	2012	35000
+NULL	2013	78000
+Java	NULL	50000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+dotNET	NULL	63000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+
+
+-- !query
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY CUBE(course, year, (course, year), ()) ORDER BY course, year
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Empty set in CUBE grouping sets is not supported.(line 1, pos 61)
+
+== SQL ==
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY CUBE(course, year, (course, year), ()) ORDER BY course, year
+-------------------------------------------------------------^^^
+
+
+-- !query
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY course, year GROUPING SETS(course, year)
+-- !query schema
+struct<course:string,year:int,sum(earnings):bigint>
+-- !query output
+Java	NULL	50000
+NULL	2012	35000
+NULL	2013	78000
+dotNET	NULL	63000
+
+
+-- !query
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY course, year GROUPING SETS(course, year, ())
+-- !query schema
+struct<course:string,year:int,sum(earnings):bigint>
+-- !query output
+Java	NULL	50000
+NULL	2012	35000
+NULL	2013	78000
+NULL	NULL	113000
+dotNET	NULL	63000
+
+
+-- !query
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY course, year GROUPING SETS(course)
+-- !query schema
+struct<course:string,year:int,sum(earnings):bigint>
+-- !query output
+Java	NULL	50000
+dotNET	NULL	63000
+
+
+-- !query
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY course, year GROUPING SETS(year)
+-- !query schema
+struct<course:string,year:int,sum(earnings):bigint>
+-- !query output
+NULL	2012	35000
+NULL	2013	78000
+
+
+-- !query
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY course, CUBE(course, year) ORDER BY course, year
+-- !query schema
+struct<course:string,year:int,sum(earnings):bigint>
+-- !query output
+Java	NULL	50000
+Java	NULL	50000
+Java	2012	20000
+Java	2012	20000
+Java	2013	30000
+Java	2013	30000
+dotNET	NULL	63000
+dotNET	NULL	63000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2013	48000
+dotNET	2013	48000
+
+
+-- !query
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY CUBE(course, year), ROLLUP(course, year) ORDER BY course, year
+-- !query schema
+struct<course:string,year:int,sum(earnings):bigint>
+-- !query output
+NULL	NULL	113000
+NULL	2012	35000
+NULL	2013	78000
+Java	NULL	50000
+Java	NULL	50000
+Java	NULL	50000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+dotNET	NULL	63000
+dotNET	NULL	63000
+dotNET	NULL	63000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+
+
+-- !query
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY CUBE(course, year), ROLLUP(course, year), GROUPING SETS(course, year) ORDER BY course, year
+-- !query schema
+struct<course:string,year:int,sum(earnings):bigint>
+-- !query output
+NULL	2012	35000
+NULL	2012	35000
+NULL	2013	78000
+NULL	2013	78000
+Java	NULL	50000
+Java	NULL	50000
+Java	NULL	50000
+Java	NULL	50000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+dotNET	NULL	63000
+dotNET	NULL	63000
+dotNET	NULL	63000
+dotNET	NULL	63000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+
+
+-- !query
+SELECT course, SUM(earnings) AS sum FROM courseSales
+GROUP BY course, earnings GROUPING SETS((), (course), (course, earnings)) ORDER BY course, sum
+-- !query schema
+struct<course:string,sum:bigint>
+-- !query output
+NULL	113000
+Java	20000
+Java	30000
+Java	50000
+dotNET	5000
+dotNET	10000
+dotNET	48000
+dotNET	63000
+
+
+-- !query
+SELECT course, SUM(earnings) AS sum, GROUPING_ID(course, earnings) FROM courseSales
+GROUP BY course, earnings GROUPING SETS((), (course), (course, earnings)) ORDER BY course, sum
+-- !query schema
+struct<course:string,sum:bigint,grouping_id(course, earnings):bigint>
+-- !query output
+NULL	113000	3
+Java	20000	0
+Java	30000	0
+Java	50000	1
+dotNET	5000	0
+dotNET	10000	0
+dotNET	48000	0
+dotNET	63000	1
+
+
+-- !query
+SELECT course, year, GROUPING(course), GROUPING(year), GROUPING_ID(course, year) FROM courseSales
+GROUP BY CUBE(course, year)
+-- !query schema
+struct<course:string,year:int,grouping(course):tinyint,grouping(year):tinyint,grouping_id(course, year):bigint>
+-- !query output
+Java	2012	0	0	0
+Java	2013	0	0	0
+Java	NULL	0	1	1
+NULL	2012	1	0	2
+NULL	2013	1	0	2
+NULL	NULL	1	1	3
+dotNET	2012	0	0	0
+dotNET	2013	0	0	0
+dotNET	NULL	0	1	1
+
+
+-- !query
+SELECT course, year, GROUPING(course) FROM courseSales GROUP BY course, year
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+grouping() can only be used with GroupingSets/Cube/Rollup
+
+
+-- !query
+SELECT course, year, GROUPING_ID(course, year) FROM courseSales GROUP BY course, year
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+grouping_id() can only be used with GroupingSets/Cube/Rollup
+
+
+-- !query
+SELECT course, year, grouping__id FROM courseSales GROUP BY CUBE(course, year) ORDER BY grouping__id, course, year
+-- !query schema
+struct<course:string,year:int,grouping__id:bigint>
+-- !query output
+Java	2012	0
+Java	2013	0
+dotNET	2012	0
+dotNET	2013	0
+Java	NULL	1
+dotNET	NULL	1
+NULL	2012	2
+NULL	2013	2
+NULL	NULL	3
+
+
+-- !query
+SELECT course, year FROM courseSales GROUP BY CUBE(course, year)
+HAVING GROUPING(year) = 1 AND GROUPING_ID(course, year) > 0 ORDER BY course, year
+-- !query schema
+struct<course:string,year:int>
+-- !query output
+NULL	NULL
+Java	NULL
+dotNET	NULL
+
+
+-- !query
+SELECT course, year FROM courseSales GROUP BY course, year HAVING GROUPING(course) > 0
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+grouping()/grouping_id() can only be used with GroupingSets/Cube/Rollup
+
+
+-- !query
+SELECT course, year FROM courseSales GROUP BY course, year HAVING GROUPING_ID(course) > 0
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+grouping()/grouping_id() can only be used with GroupingSets/Cube/Rollup
+
+
+-- !query
+SELECT course, year FROM courseSales GROUP BY CUBE(course, year) HAVING grouping__id > 0
+-- !query schema
+struct<course:string,year:int>
+-- !query output
+Java	NULL
+NULL	2012
+NULL	2013
+NULL	NULL
+dotNET	NULL
+
+
+-- !query
+SELECT course, year, GROUPING(course), GROUPING(year) FROM courseSales GROUP BY CUBE(course, year)
+ORDER BY GROUPING(course), GROUPING(year), course, year
+-- !query schema
+struct<course:string,year:int,grouping(course):tinyint,grouping(year):tinyint>
+-- !query output
+Java	2012	0	0
+Java	2013	0	0
+dotNET	2012	0	0
+dotNET	2013	0	0
+Java	NULL	0	1
+dotNET	NULL	0	1
+NULL	2012	1	0
+NULL	2013	1	0
+NULL	NULL	1	1
+
+
+-- !query
+SELECT course, year, GROUPING_ID(course, year) FROM courseSales GROUP BY CUBE(course, year)
+ORDER BY GROUPING(course), GROUPING(year), course, year
+-- !query schema
+struct<course:string,year:int,grouping_id(course, year):bigint>
+-- !query output
+Java	2012	0
+Java	2013	0
+dotNET	2012	0
+dotNET	2013	0
+Java	NULL	1
+dotNET	NULL	1
+NULL	2012	2
+NULL	2013	2
+NULL	NULL	3
+
+
+-- !query
+SELECT course, year FROM courseSales GROUP BY course, year ORDER BY GROUPING(course)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+grouping()/grouping_id() can only be used with GroupingSets/Cube/Rollup
+
+
+-- !query
+SELECT course, year FROM courseSales GROUP BY course, year ORDER BY GROUPING_ID(course)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+grouping()/grouping_id() can only be used with GroupingSets/Cube/Rollup
+
+
+-- !query
+SELECT course, year FROM courseSales GROUP BY CUBE(course, year) ORDER BY grouping__id, course, year
+-- !query schema
+struct<course:string,year:int>
+-- !query output
+Java	2012
+Java	2013
+dotNET	2012
+dotNET	2013
+Java	NULL
+dotNET	NULL
+NULL	2012
+NULL	2013
+NULL	NULL
+
+
+-- !query
+SELECT a + b AS k1, b AS k2, SUM(a - b) FROM testData GROUP BY CUBE(k1, k2)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'k1' given input columns: [testdata.a, testdata.b]; line 1 pos 68
+
+
+-- !query
+SELECT a + b AS k, b, SUM(a - b) FROM testData GROUP BY ROLLUP(k, b)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'k' given input columns: [testdata.a, testdata.b]; line 1 pos 63
+
+
+-- !query
+SELECT a + b, b AS k, SUM(a - b) FROM testData GROUP BY a + b, k GROUPING SETS(k)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'k' given input columns: [testdata.a, testdata.b]; line 1 pos 79
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, b, CUBE(a, b)
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, b, ROLLUP(a, b)
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	1	1
+1	1	1
+1	2	1
+1	2	1
+1	2	1
+2	1	1
+2	1	1
+2	1	1
+2	2	1
+2	2	1
+2	2	1
+3	1	1
+3	1	1
+3	1	1
+3	2	1
+3	2	1
+3	2	1
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY CUBE(a, b), ROLLUP(a, b)
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	NULL	2
+1	NULL	2
+1	NULL	2
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	NULL	2
+2	NULL	2
+2	NULL	2
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	NULL	2
+3	NULL	2
+3	NULL	2
+NULL	1	3
+NULL	2	3
+NULL	NULL	6
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, CUBE(a, b), ROLLUP(b)
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	NULL	2
+1	NULL	2
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	NULL	2
+2	NULL	2
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	NULL	2
+3	NULL	2
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS((a, b), (a), ())
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	2	1
+1	NULL	2
+1	NULL	2
+2	1	1
+2	2	1
+2	NULL	2
+2	NULL	2
+3	1	1
+3	2	1
+3	NULL	2
+3	NULL	2
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, CUBE(a, b), GROUPING SETS((a, b), (a), ())
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	NULL	2
+1	NULL	2
+1	NULL	2
+1	NULL	2
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	NULL	2
+2	NULL	2
+2	NULL	2
+2	NULL	2
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	NULL	2
+3	NULL	2
+3	NULL	2
+3	NULL	2
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, CUBE(a, b), ROLLUP(a, b), GROUPING SETS((a, b), (a), ())
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	NULL	2
+1	NULL	2
+1	NULL	2
+1	NULL	2
+1	NULL	2
+1	NULL	2
+1	NULL	2
+1	NULL	2
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	NULL	2
+2	NULL	2
+2	NULL	2
+2	NULL	2
+2	NULL	2
+2	NULL	2
+2	NULL	2
+2	NULL	2
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	NULL	2
+3	NULL	2
+3	NULL	2
+3	NULL	2
+3	NULL	2
+3	NULL	2
+3	NULL	2
+3	NULL	2
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS(ROLLUP(a, b))
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	2	1
+1	NULL	2
+1	NULL	2
+2	1	1
+2	2	1
+2	NULL	2
+2	NULL	2
+3	1	1
+3	2	1
+3	NULL	2
+3	NULL	2
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS(GROUPING SETS((a, b), (a), ()))
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	2	1
+1	NULL	2
+1	NULL	2
+2	1	1
+2	2	1
+2	NULL	2
+2	NULL	2
+3	1	1
+3	2	1
+3	NULL	2
+3	NULL	2
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS((a, b), GROUPING SETS(ROLLUP(a, b)))
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	1	1
+1	2	1
+1	2	1
+1	NULL	2
+1	NULL	2
+2	1	1
+2	1	1
+2	2	1
+2	2	1
+2	NULL	2
+2	NULL	2
+3	1	1
+3	1	1
+3	2	1
+3	2	1
+3	NULL	2
+3	NULL	2
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS((a, b, a, b), (a, b, a), (a, b))
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	1	1
+1	1	1
+1	2	1
+1	2	1
+1	2	1
+2	1	1
+2	1	1
+2	1	1
+2	2	1
+2	2	1
+2	2	1
+3	1	1
+3	1	1
+3	1	1
+3	2	1
+3	2	1
+3	2	1
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS(GROUPING SETS((a, b, a, b), (a, b, a), (a, b)))
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	1	1
+1	1	1
+1	2	1
+1	2	1
+1	2	1
+2	1	1
+2	1	1
+2	1	1
+2	2	1
+2	2	1
+2	2	1
+3	1	1
+3	1	1
+3	1	1
+3	2	1
+3	2	1
+3	2	1
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS(ROLLUP(a, b), CUBE(a, b))
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	1	1
+1	1	1
+1	2	1
+1	2	1
+1	2	1
+1	NULL	2
+1	NULL	2
+1	NULL	2
+1	NULL	2
+2	1	1
+2	1	1
+2	1	1
+2	2	1
+2	2	1
+2	2	1
+2	NULL	2
+2	NULL	2
+2	NULL	2
+2	NULL	2
+3	1	1
+3	1	1
+3	1	1
+3	2	1
+3	2	1
+3	2	1
+3	NULL	2
+3	NULL	2
+3	NULL	2
+3	NULL	2
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS(GROUPING SETS((a, b), (a), ()), GROUPING SETS((a, b), (a), (b), ()))
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	1	1
+1	1	1
+1	2	1
+1	2	1
+1	2	1
+1	NULL	2
+1	NULL	2
+1	NULL	2
+1	NULL	2
+2	1	1
+2	1	1
+2	1	1
+2	2	1
+2	2	1
+2	2	1
+2	NULL	2
+2	NULL	2
+2	NULL	2
+2	NULL	2
+3	1	1
+3	1	1
+3	1	1
+3	2	1
+3	2	1
+3	2	1
+3	NULL	2
+3	NULL	2
+3	NULL	2
+3	NULL	2
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS((a, b), (a), (), (a, b), (a), (b), ())
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	1	1
+1	1	1
+1	2	1
+1	2	1
+1	2	1
+1	NULL	2
+1	NULL	2
+1	NULL	2
+1	NULL	2
+2	1	1
+2	1	1
+2	1	1
+2	2	1
+2	2	1
+2	2	1
+2	NULL	2
+2	NULL	2
+2	NULL	2
+2	NULL	2
+3	1	1
+3	1	1
+3	1	1
+3	2	1
+3	2	1
+3	2	1
+3	NULL	2
+3	NULL	2
+3	NULL	2
+3	NULL	2
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/higher-order-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/higher-order-functions.sql.out
index 7bef1bad4507e..6d26fae73b11c 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/higher-order-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/higher-order-functions.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 29
+-- Number of queries: 30
 
 
 -- !query
@@ -282,3 +282,11 @@ no viable alternative at input 'all'(line 1, pos 22)
 == SQL ==
 select transform(ys, (all, i) -> all + i) as v from values (array(32, 97)) as t(ys)
 ----------------------^^^
+
+
+-- !query
+select aggregate(split('abcdefgh',''), array(array('')), (acc, x) -> array(array(x)))
+-- !query schema
+struct<aggregate(split(abcdefgh, , -1), array(array()), lambdafunction(array(array(namedlambdavariable())), namedlambdavariable(), namedlambdavariable()), lambdafunction(namedlambdavariable(), namedlambdavariable())):array<array<string>>>
+-- !query output
+[[""]]
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
index 39b230fd19f3d..320525920cbb7 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
@@ -1,19 +1,19 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 91
+-- Number of queries: 134
 
 
 -- !query
 select 3 * (timestamp'2019-10-15 10:11:12.001002' - date'2019-10-15')
 -- !query schema
-struct<multiply_interval(subtracttimestamps(TIMESTAMP '2019-10-15 10:11:12.001002', CAST(DATE '2019-10-15' AS TIMESTAMP)), CAST(3 AS DOUBLE)):interval>
+struct<((TIMESTAMP '2019-10-15 10:11:12.001002' - DATE '2019-10-15') * 3):interval day to second>
 -- !query output
-30 hours 33 minutes 36.003006 seconds
+1 06:33:36.003006000
 
 
 -- !query
 select interval 4 month 2 weeks 3 microseconds * 1.5
 -- !query schema
-struct<multiply_interval(INTERVAL '4 months 14 days 0.000003 seconds', CAST(1.5 AS DOUBLE)):interval>
+struct<multiply_interval(INTERVAL '4 months 14 days 0.000003 seconds', 1.5):interval>
 -- !query output
 6 months 21 days 0.000005 seconds
 
@@ -21,9 +21,9 @@ struct<multiply_interval(INTERVAL '4 months 14 days 0.000003 seconds', CAST(1.5
 -- !query
 select (timestamp'2019-10-15' - timestamp'2019-10-14') / 1.5
 -- !query schema
-struct<divide_interval(subtracttimestamps(TIMESTAMP '2019-10-15 00:00:00', TIMESTAMP '2019-10-14 00:00:00'), CAST(1.5 AS DOUBLE)):interval>
+struct<((TIMESTAMP '2019-10-15 00:00:00' - TIMESTAMP '2019-10-14 00:00:00') / 1.5):interval day to second>
 -- !query output
-16 hours
+0 16:00:00.000000000
 
 
 -- !query
@@ -74,7 +74,7 @@ divide by zero
 -- !query
 select interval '2 seconds' / null
 -- !query schema
-struct<divide_interval(INTERVAL '2 seconds', CAST(NULL AS DOUBLE)):interval>
+struct<divide_interval(INTERVAL '2 seconds', NULL):interval>
 -- !query output
 NULL
 
@@ -82,7 +82,7 @@ NULL
 -- !query
 select interval '2 seconds' * null
 -- !query schema
-struct<multiply_interval(INTERVAL '2 seconds', CAST(NULL AS DOUBLE)):interval>
+struct<multiply_interval(INTERVAL '2 seconds', NULL):interval>
 -- !query output
 NULL
 
@@ -90,7 +90,7 @@ NULL
 -- !query
 select null * interval '2 seconds'
 -- !query schema
-struct<multiply_interval(INTERVAL '2 seconds', CAST(NULL AS DOUBLE)):interval>
+struct<multiply_interval(INTERVAL '2 seconds', NULL):interval>
 -- !query output
 NULL
 
@@ -127,6 +127,54 @@ struct<(+ INTERVAL '-1 months 1 days -1 seconds'):interval>
 -1 months 1 days -1 seconds
 
 
+-- !query
+select interval -'1-1' year to month
+-- !query schema
+struct<INTERVAL '-1-1' YEAR TO MONTH:interval year to month>
+-- !query output
+-1-1
+
+
+-- !query
+select interval -'-1-1' year to month
+-- !query schema
+struct<INTERVAL '1-1' YEAR TO MONTH:interval year to month>
+-- !query output
+1-1
+
+
+-- !query
+select interval +'-1-1' year to month
+-- !query schema
+struct<INTERVAL '-1-1' YEAR TO MONTH:interval year to month>
+-- !query output
+-1-1
+
+
+-- !query
+select interval - '1 2:3:4.001' day to second
+-- !query schema
+struct<INTERVAL '-1 02:03:04.001' DAY TO SECOND:interval day to second>
+-- !query output
+-1 02:03:04.001000000
+
+
+-- !query
+select interval +'1 2:3:4.001' day to second
+-- !query schema
+struct<INTERVAL '1 02:03:04.001' DAY TO SECOND:interval day to second>
+-- !query output
+1 02:03:04.001000000
+
+
+-- !query
+select interval -'-1 2:3:4.001' day to second
+-- !query schema
+struct<INTERVAL '1 02:03:04.001' DAY TO SECOND:interval day to second>
+-- !query output
+1 02:03:04.001000000
+
+
 -- !query
 select make_interval(1)
 -- !query schema
@@ -178,11 +226,28 @@ struct<make_interval(1, 2, 3, 4, 5, 6, 0.000000):interval>
 -- !query
 select make_interval(1, 2, 3, 4, 5, 6, 7.008009)
 -- !query schema
-struct<make_interval(1, 2, 3, 4, 5, 6, CAST(7.008009 AS DECIMAL(8,6))):interval>
+struct<make_interval(1, 2, 3, 4, 5, 6, 7.008009):interval>
 -- !query output
 1 years 2 months 25 days 5 hours 6 minutes 7.008009 seconds
 
 
+-- !query
+select make_interval(1, 2, 3, 4, 0, 0, 123456789012.123456)
+-- !query schema
+struct<make_interval(1, 2, 3, 4, 0, 0, 123456789012.123456):interval>
+-- !query output
+1 years 2 months 25 days 34293552 hours 30 minutes 12.123456 seconds
+
+
+-- !query
+select make_interval(0, 0, 0, 0, 0, 0, 1234567890123456789)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'make_interval(0, 0, 0, 0, 0, 0, 1234567890123456789L)' due to data type mismatch: argument 7 requires decimal(18,6) type, however, '1234567890123456789L' is of bigint type.; line 1 pos 7
+
+
 -- !query
 select cast('1 second' as interval)
 -- !query schema
@@ -263,90 +328,108 @@ struct<INTERVAL '32 years 1 months -100 days 41 hours 24 minutes 59.889987 secon
 32 years 1 months -100 days 41 hours 24 minutes 59.889987 seconds
 
 
+-- !query
+select interval '0-0' year to month
+-- !query schema
+struct<INTERVAL '0-0' YEAR TO MONTH:interval year to month>
+-- !query output
+0-0
+
+
+-- !query
+select interval '0 0:0:0' day to second
+-- !query schema
+struct<INTERVAL '0 00:00:00' DAY TO SECOND:interval day to second>
+-- !query output
+0 00:00:00.000000000
+
+
 -- !query
 select interval '0 0:0:0.1' day to second
 -- !query schema
-struct<INTERVAL '0.1 seconds':interval>
+struct<INTERVAL '0 00:00:00.1' DAY TO SECOND:interval day to second>
 -- !query output
-0.1 seconds
+0 00:00:00.100000000
 
 
 -- !query
 select interval '10-9' year to month
 -- !query schema
-struct<INTERVAL '10 years 9 months':interval>
+struct<INTERVAL '10-9' YEAR TO MONTH:interval year to month>
 -- !query output
-10 years 9 months
+10-9
 
 
 -- !query
 select interval '20 15' day to hour
 -- !query schema
-struct<INTERVAL '20 days 15 hours':interval>
+struct<INTERVAL '20 15:00:00' DAY TO SECOND:interval day to second>
 -- !query output
-20 days 15 hours
+20 15:00:00.000000000
 
 
 -- !query
 select interval '20 15:40' day to minute
 -- !query schema
-struct<INTERVAL '20 days 15 hours 40 minutes':interval>
+struct<INTERVAL '20 15:40:00' DAY TO SECOND:interval day to second>
 -- !query output
-20 days 15 hours 40 minutes
+20 15:40:00.000000000
 
 
 -- !query
 select interval '20 15:40:32.99899999' day to second
 -- !query schema
-struct<INTERVAL '20 days 15 hours 40 minutes 32.998999 seconds':interval>
+struct<INTERVAL '20 15:40:32.998999' DAY TO SECOND:interval day to second>
 -- !query output
-20 days 15 hours 40 minutes 32.998999 seconds
+20 15:40:32.998999000
 
 
 -- !query
 select interval '15:40' hour to minute
 -- !query schema
-struct<INTERVAL '15 hours 40 minutes':interval>
+struct<INTERVAL '0 15:40:00' DAY TO SECOND:interval day to second>
 -- !query output
-15 hours 40 minutes
+0 15:40:00.000000000
 
 
 -- !query
 select interval '15:40:32.99899999' hour to second
 -- !query schema
-struct<INTERVAL '15 hours 40 minutes 32.998999 seconds':interval>
+struct<INTERVAL '0 15:40:32.998999' DAY TO SECOND:interval day to second>
 -- !query output
-15 hours 40 minutes 32.998999 seconds
+0 15:40:32.998999000
 
 
 -- !query
 select interval '40:32.99899999' minute to second
 -- !query schema
-struct<INTERVAL '40 minutes 32.998999 seconds':interval>
+struct<INTERVAL '0 00:40:32.998999' DAY TO SECOND:interval day to second>
 -- !query output
-40 minutes 32.998999 seconds
+0 00:40:32.998999000
 
 
 -- !query
 select interval '40:32' minute to second
 -- !query schema
-struct<INTERVAL '40 minutes 32 seconds':interval>
+struct<INTERVAL '0 00:40:32' DAY TO SECOND:interval day to second>
 -- !query output
-40 minutes 32 seconds
+0 00:40:32.000000000
 
 
 -- !query
 select interval 30 day day
 -- !query schema
-struct<>
+struct<day:interval>
 -- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
+30 days
 
-no viable alternative at input 'day'(line 1, pos 23)
 
-== SQL ==
-select interval 30 day day
------------------------^^^
+-- !query
+select interval 30 days days
+-- !query schema
+struct<days:interval>
+-- !query output
+30 days
 
 
 -- !query
@@ -638,41 +721,101 @@ select interval (-30) day
 -- !query schema
 struct<>
 -- !query output
+org.apache.spark.sql.AnalysisException
+Undefined function: 'interval'. This function is neither a registered temporary function nor a permanent function registered in the database 'default'.; line 1 pos 7
+
+
+-- !query
+select interval (a + 1) day
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Undefined function: 'interval'. This function is neither a registered temporary function nor a permanent function registered in the database 'default'.; line 1 pos 7
+
+
+-- !query
+select interval 30 day day day
+-- !query schema
+struct<>
+-- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-no viable alternative at input 'day'(line 1, pos 22)
+extraneous input 'day' expecting {<EOF>, ';'}(line 1, pos 27)
 
 == SQL ==
-select interval (-30) day
-----------------------^^^
+select interval 30 day day day
+---------------------------^^^
 
 
 -- !query
-select interval (a + 1) day
+select interval (-30) days
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Undefined function: 'interval'. This function is neither a registered temporary function nor a permanent function registered in the database 'default'.; line 1 pos 7
+
+
+-- !query
+select interval (a + 1) days
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Undefined function: 'interval'. This function is neither a registered temporary function nor a permanent function registered in the database 'default'.; line 1 pos 7
+
+
+-- !query
+select interval 30 days days days
 -- !query schema
 struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-no viable alternative at input 'day'(line 1, pos 24)
+extraneous input 'days' expecting {<EOF>, ';'}(line 1, pos 29)
 
 == SQL ==
-select interval (a + 1) day
-------------------------^^^
+select interval 30 days days days
+-----------------------------^^^
 
 
 -- !query
-select interval 30 day day day
+SELECT INTERVAL '178956970-7' YEAR TO MONTH
+-- !query schema
+struct<INTERVAL '178956970-7' YEAR TO MONTH:interval year to month>
+-- !query output
+178956970-7
+
+
+-- !query
+SELECT INTERVAL '178956970-8' YEAR TO MONTH
 -- !query schema
 struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-no viable alternative at input 'day'(line 1, pos 23)
+Error parsing interval year-month string: integer overflow(line 1, pos 16)
 
 == SQL ==
-select interval 30 day day day
------------------------^^^
+SELECT INTERVAL '178956970-8' YEAR TO MONTH
+----------------^^^
+
+
+-- !query
+SELECT INTERVAL '-178956970-8' YEAR TO MONTH
+-- !query schema
+struct<INTERVAL '-178956970-8' YEAR TO MONTH:interval year to month>
+-- !query output
+-178956970-8
+
+
+-- !query
+SELECT INTERVAL -'178956970-8' YEAR TO MONTH
+-- !query schema
+struct<INTERVAL '-178956970-8' YEAR TO MONTH:interval year to month>
+-- !query output
+-178956970-8
 
 
 -- !query
@@ -697,7 +840,7 @@ select
   interval '2-2' year to month + dateval
 from interval_arithmetic
 -- !query schema
-struct<dateval:date,dateval - INTERVAL '2 years 2 months':date,dateval - INTERVAL '-2 years -2 months':date,dateval + INTERVAL '2 years 2 months':date,dateval + INTERVAL '-2 years -2 months':date,dateval + (- INTERVAL '2 years 2 months'):date,dateval + INTERVAL '2 years 2 months':date>
+struct<dateval:date,dateval - INTERVAL '2-2' YEAR TO MONTH:date,dateval - INTERVAL '-2-2' YEAR TO MONTH:date,dateval + INTERVAL '2-2' YEAR TO MONTH:date,dateval + INTERVAL '-2-2' YEAR TO MONTH:date,dateval + (- INTERVAL '2-2' YEAR TO MONTH):date,dateval + INTERVAL '2-2' YEAR TO MONTH:date>
 -- !query output
 2012-01-01	2009-11-01	2014-03-01	2014-03-01	2009-11-01	2009-11-01	2014-03-01
 
@@ -713,7 +856,7 @@ select
   interval '2-2' year to month + tsval
 from interval_arithmetic
 -- !query schema
-struct<tsval:timestamp,CAST(tsval - INTERVAL '2 years 2 months' AS TIMESTAMP):timestamp,CAST(tsval - INTERVAL '-2 years -2 months' AS TIMESTAMP):timestamp,CAST(tsval + INTERVAL '2 years 2 months' AS TIMESTAMP):timestamp,CAST(tsval + INTERVAL '-2 years -2 months' AS TIMESTAMP):timestamp,CAST(tsval + (- INTERVAL '2 years 2 months') AS TIMESTAMP):timestamp,CAST(tsval + INTERVAL '2 years 2 months' AS TIMESTAMP):timestamp>
+struct<tsval:timestamp,tsval - INTERVAL '2-2' YEAR TO MONTH:timestamp,tsval - INTERVAL '-2-2' YEAR TO MONTH:timestamp,tsval + INTERVAL '2-2' YEAR TO MONTH:timestamp,tsval + INTERVAL '-2-2' YEAR TO MONTH:timestamp,tsval + (- INTERVAL '2-2' YEAR TO MONTH):timestamp,tsval + INTERVAL '2-2' YEAR TO MONTH:timestamp>
 -- !query output
 2012-01-01 00:00:00	2009-11-01 00:00:00	2014-03-01 00:00:00	2014-03-01 00:00:00	2009-11-01 00:00:00	2009-11-01 00:00:00	2014-03-01 00:00:00
 
@@ -724,9 +867,9 @@ select
   interval '2-2' year to month - interval '3-3' year to month
 from interval_arithmetic
 -- !query schema
-struct<(INTERVAL '2 years 2 months' + INTERVAL '3 years 3 months'):interval,(INTERVAL '2 years 2 months' - INTERVAL '3 years 3 months'):interval>
+struct<(INTERVAL '2-2' YEAR TO MONTH + INTERVAL '3-3' YEAR TO MONTH):interval year to month,(INTERVAL '2-2' YEAR TO MONTH - INTERVAL '3-3' YEAR TO MONTH):interval year to month>
 -- !query output
-5 years 5 months	-1 years -1 months
+5-5	-1-1
 
 
 -- !query
@@ -740,10 +883,9 @@ select
   interval '99 11:22:33.123456789' day to second + dateval
 from interval_arithmetic
 -- !query schema
-struct<>
+struct<dateval:date,dateval - INTERVAL '99 11:22:33.123456' DAY TO SECOND:timestamp,dateval - INTERVAL '-99 11:22:33.123456' DAY TO SECOND:timestamp,dateval + INTERVAL '99 11:22:33.123456' DAY TO SECOND:timestamp,dateval + INTERVAL '-99 11:22:33.123456' DAY TO SECOND:timestamp,dateval + (- INTERVAL '99 11:22:33.123456' DAY TO SECOND):timestamp,dateval + INTERVAL '99 11:22:33.123456' DAY TO SECOND:timestamp>
 -- !query output
-java.lang.IllegalArgumentException
-requirement failed: Cannot add hours, minutes or seconds, milliseconds, microseconds to a date
+2012-01-01	2011-09-23 12:37:26.876544	2012-04-09 11:22:33.123456	2012-04-09 11:22:33.123456	2011-09-23 12:37:26.876544	2011-09-23 12:37:26.876544	2012-04-09 11:22:33.123456
 
 
 -- !query
@@ -757,7 +899,7 @@ select
   interval '99 11:22:33.123456789' day to second + tsval
 from interval_arithmetic
 -- !query schema
-struct<tsval:timestamp,CAST(tsval - INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' AS TIMESTAMP):timestamp,CAST(tsval - INTERVAL '-99 days -11 hours -22 minutes -33.123456 seconds' AS TIMESTAMP):timestamp,CAST(tsval + INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' AS TIMESTAMP):timestamp,CAST(tsval + INTERVAL '-99 days -11 hours -22 minutes -33.123456 seconds' AS TIMESTAMP):timestamp,CAST(tsval + (- INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds') AS TIMESTAMP):timestamp,CAST(tsval + INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' AS TIMESTAMP):timestamp>
+struct<tsval:timestamp,tsval - INTERVAL '99 11:22:33.123456' DAY TO SECOND:timestamp,tsval - INTERVAL '-99 11:22:33.123456' DAY TO SECOND:timestamp,tsval + INTERVAL '99 11:22:33.123456' DAY TO SECOND:timestamp,tsval + INTERVAL '-99 11:22:33.123456' DAY TO SECOND:timestamp,tsval + (- INTERVAL '99 11:22:33.123456' DAY TO SECOND):timestamp,tsval + INTERVAL '99 11:22:33.123456' DAY TO SECOND:timestamp>
 -- !query output
 2012-01-01 00:00:00	2011-09-23 12:37:26.876544	2012-04-09 11:22:33.123456	2012-04-09 11:22:33.123456	2011-09-23 12:37:26.876544	2011-09-23 12:37:26.876544	2012-04-09 11:22:33.123456
 
@@ -773,7 +915,7 @@ select
   interval '99 11:22:33.123456789' day to second + strval
 from interval_arithmetic
 -- !query schema
-struct<strval:string,CAST(strval - INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' AS STRING):string,CAST(strval - INTERVAL '-99 days -11 hours -22 minutes -33.123456 seconds' AS STRING):string,CAST(CAST(strval AS TIMESTAMP) + INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' AS STRING):string,CAST(CAST(strval AS TIMESTAMP) + INTERVAL '-99 days -11 hours -22 minutes -33.123456 seconds' AS STRING):string,CAST(CAST(strval AS TIMESTAMP) + (- INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds') AS STRING):string,CAST(CAST(strval AS TIMESTAMP) + INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' AS STRING):string>
+struct<strval:string,strval - INTERVAL '99 11:22:33.123456' DAY TO SECOND:string,strval - INTERVAL '-99 11:22:33.123456' DAY TO SECOND:string,strval + INTERVAL '99 11:22:33.123456' DAY TO SECOND:string,strval + INTERVAL '-99 11:22:33.123456' DAY TO SECOND:string,strval + (- INTERVAL '99 11:22:33.123456' DAY TO SECOND):string,strval + INTERVAL '99 11:22:33.123456' DAY TO SECOND:string>
 -- !query output
 2012-01-01	2011-09-23 12:37:26.876544	2012-04-09 11:22:33.123456	2012-04-09 11:22:33.123456	2011-09-23 12:37:26.876544	2011-09-23 12:37:26.876544	2012-04-09 11:22:33.123456
 
@@ -784,9 +926,9 @@ select
   interval '99 11:22:33.123456789' day to second - interval '10 9:8:7.123456789' day to second
 from interval_arithmetic
 -- !query schema
-struct<(INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' + INTERVAL '10 days 9 hours 8 minutes 7.123456 seconds'):interval,(INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' - INTERVAL '10 days 9 hours 8 minutes 7.123456 seconds'):interval>
+struct<(INTERVAL '99 11:22:33.123456' DAY TO SECOND + INTERVAL '10 09:08:07.123456' DAY TO SECOND):interval day to second,(INTERVAL '99 11:22:33.123456' DAY TO SECOND - INTERVAL '10 09:08:07.123456' DAY TO SECOND):interval day to second>
 -- !query output
-109 days 20 hours 30 minutes 40.246912 seconds	89 days 2 hours 14 minutes 26 seconds
+109 20:30:40.246912000	89 02:14:26.000000000
 
 
 -- !query
@@ -832,9 +974,9 @@ struct<INTERVAL '1 days':interval>
 -- !query
 select interval '2-2\t' year to month
 -- !query schema
-struct<INTERVAL '2 years 2 months':interval>
+struct<INTERVAL '2-2' YEAR TO MONTH:interval year to month>
 -- !query output
-2 years 2 months
+2-2
 
 
 -- !query
@@ -854,9 +996,9 @@ select interval '-\t2-2\t' year to month
 -- !query
 select interval '\n0 12:34:46.789\t' day to second
 -- !query schema
-struct<INTERVAL '12 hours 34 minutes 46.789 seconds':interval>
+struct<INTERVAL '0 12:34:46.789' DAY TO SECOND:interval day to second>
 -- !query output
-12 hours 34 minutes 46.789 seconds
+0 12:34:46.789000000
 
 
 -- !query
@@ -874,6 +1016,48 @@ select interval '\n-\t10\t 12:34:46.789\t' day to second
 ----------------^^^
 
 
+-- !query
+select interval '中文 interval 1 day'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 中文 interval 1 day(line 1, pos 7)
+
+== SQL ==
+select interval '中文 interval 1 day'
+-------^^^
+
+
+-- !query
+select interval 'interval中文 1 day'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: interval中文 1 day(line 1, pos 7)
+
+== SQL ==
+select interval 'interval中文 1 day'
+-------^^^
+
+
+-- !query
+select interval 'interval 1中文day'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: interval 1中文day(line 1, pos 7)
+
+== SQL ==
+select interval 'interval 1中文day'
+-------^^^
+
+
 -- !query
 select -(a) from values (interval '-2147483648 months', interval '2147483647 months') t(a, b)
 -- !query schema
@@ -941,3 +1125,241 @@ SELECT
 struct<from_json({"a":"1 days"}):struct<a:interval>,to_json(from_json({"a":"1 days"})):string,to_json(map(a, INTERVAL '2 years 1 months 100 days 2 hours 10 minutes')):string,from_json(to_json(map(a, INTERVAL '2 years 1 months 100 days 2 hours 10 minutes'))):struct<a:interval>>
 -- !query output
 {"a":1 days}	{"a":"1 days"}	{"a":"2 years 1 months 100 days 2 hours 10 minutes"}	{"a":2 years 1 months 100 days 2 hours 10 minutes}
+
+
+-- !query
+select interval '+'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: +(line 1, pos 7)
+
+== SQL ==
+select interval '+'
+-------^^^
+
+
+-- !query
+select interval '+.'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: +.(line 1, pos 7)
+
+== SQL ==
+select interval '+.'
+-------^^^
+
+
+-- !query
+select interval '1'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 1(line 1, pos 7)
+
+== SQL ==
+select interval '1'
+-------^^^
+
+
+-- !query
+select interval '1.2'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 1.2(line 1, pos 7)
+
+== SQL ==
+select interval '1.2'
+-------^^^
+
+
+-- !query
+select interval '- 2'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: - 2(line 1, pos 7)
+
+== SQL ==
+select interval '- 2'
+-------^^^
+
+
+-- !query
+select interval '1 day -'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 1 day -(line 1, pos 7)
+
+== SQL ==
+select interval '1 day -'
+-------^^^
+
+
+-- !query
+select interval '1 day 1'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 1 day 1(line 1, pos 7)
+
+== SQL ==
+select interval '1 day 1'
+-------^^^
+
+
+-- !query
+select interval '1 day 2' day
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Can only use numbers in the interval value part for multiple unit value pairs interval form, but got invalid value: 1 day 2(line 1, pos 16)
+
+== SQL ==
+select interval '1 day 2' day
+----------------^^^
+
+
+-- !query
+select interval 'interval 1' day
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Can only use numbers in the interval value part for multiple unit value pairs interval form, but got invalid value: interval 1(line 1, pos 16)
+
+== SQL ==
+select interval 'interval 1' day
+----------------^^^
+
+
+-- !query
+select interval '-\t 1' day
+-- !query schema
+struct<INTERVAL '-1 days':interval>
+-- !query output
+-1 days
+
+
+-- !query
+SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / 2
+-- !query schema
+struct<(INTERVAL '-178956970-8' YEAR TO MONTH / 2):interval year to month>
+-- !query output
+-89478485-4
+
+
+-- !query
+SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / 5
+-- !query schema
+struct<(INTERVAL '-178956970-8' YEAR TO MONTH / 5):interval year to month>
+-- !query output
+-35791394-2
+
+
+-- !query
+SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+Overflow in integral divide.
+
+
+-- !query
+SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1L
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+Overflow in integral divide.
+
+
+-- !query
+SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1.0
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+Overflow
+
+
+-- !query
+SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1.0D
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+not in range
+
+
+-- !query
+SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / 2
+-- !query schema
+struct<(INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND / 2):interval day to second>
+-- !query output
+-53375995 14:00:27.387904000
+
+
+-- !query
+SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / 5
+-- !query schema
+struct<(INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND / 5):interval day to second>
+-- !query output
+-21350398 05:36:10.955162000
+
+
+-- !query
+SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+Overflow in integral divide.
+
+
+-- !query
+SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1L
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+Overflow in integral divide.
+
+
+-- !query
+SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1.0
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+Overflow
+
+
+-- !query
+SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1.0D
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+not in range
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/literals.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/literals.sql.out
index f6720f6c5faa4..9748d8bc56d72 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/literals.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/literals.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 50
+-- Number of queries: 54
 
 
 -- !query
@@ -164,6 +164,36 @@ decimal can only support precision up to 38
 select 1234567890123456789012345678901234567890.0
 
 
+-- !query
+select 1F, 1.2F, .10f, 0.10f
+-- !query schema
+struct<1.0:float,1.2:float,0.1:float,0.1:float>
+-- !query output
+1.0	1.2	0.1	0.1
+
+
+-- !query
+select -1F, -1.2F, -.10F, -0.10F
+-- !query schema
+struct<-1.0:float,-1.2:float,-0.1:float,-0.1:float>
+-- !query output
+-1.0	-1.2	-0.1	-0.1
+
+
+-- !query
+select -3.4028235E39f
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Numeric literal -3.4028235E39 does not fit in range [-3.4028234663852886E+38, 3.4028234663852886E+38] for type float(line 1, pos 7)
+
+== SQL ==
+select -3.4028235E39f
+-------^^^
+
+
 -- !query
 select 1D, 1.2D, 1e10, 1.5e5, .10D, 0.10D, .1e5, .9e+2, 0.9e+2, 900e-1, 9.e+1
 -- !query schema
@@ -216,6 +246,14 @@ struct<0.3:decimal(1,1),-0.8:decimal(1,1),0.5:decimal(1,1),-0.18:decimal(2,2),0.
 0.3	-0.8	0.5	-0.18	0.1111	0.1111
 
 
+-- !query
+select 0.3 F, 0.4 D, 0.5 BD
+-- !query schema
+struct<F:decimal(1,1),D:decimal(1,1),BD:decimal(1,1)>
+-- !query output
+0.3	0.4	0.5
+
+
 -- !query
 select 123456789012345678901234567890123456789e10d, 123456789012345678901234567890123456789.1e10d
 -- !query schema
@@ -398,7 +436,7 @@ select +date '1999-01-01'
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(+ DATE '1999-01-01')' due to data type mismatch: argument 1 requires (numeric or interval) type, however, 'DATE '1999-01-01'' is of date type.; line 1 pos 7
+cannot resolve '(+ DATE '1999-01-01')' due to data type mismatch: argument 1 requires (numeric or interval or interval day to second or interval year to month) type, however, 'DATE '1999-01-01'' is of date type.; line 1 pos 7
 
 
 -- !query
@@ -407,7 +445,7 @@ select +timestamp '1999-01-01'
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(+ TIMESTAMP '1999-01-01 00:00:00')' due to data type mismatch: argument 1 requires (numeric or interval) type, however, 'TIMESTAMP '1999-01-01 00:00:00'' is of timestamp type.; line 1 pos 7
+cannot resolve '(+ TIMESTAMP '1999-01-01 00:00:00')' due to data type mismatch: argument 1 requires (numeric or interval or interval day to second or interval year to month) type, however, 'TIMESTAMP '1999-01-01 00:00:00'' is of timestamp type.; line 1 pos 7
 
 
 -- !query
@@ -424,7 +462,7 @@ select +map(1, 2)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(+ map(1, 2))' due to data type mismatch: argument 1 requires (numeric or interval) type, however, 'map(1, 2)' is of map<int,int> type.; line 1 pos 7
+cannot resolve '(+ map(1, 2))' due to data type mismatch: argument 1 requires (numeric or interval or interval day to second or interval year to month) type, however, 'map(1, 2)' is of map<int,int> type.; line 1 pos 7
 
 
 -- !query
@@ -433,7 +471,7 @@ select +array(1,2)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(+ array(1, 2))' due to data type mismatch: argument 1 requires (numeric or interval) type, however, 'array(1, 2)' is of array<int> type.; line 1 pos 7
+cannot resolve '(+ array(1, 2))' due to data type mismatch: argument 1 requires (numeric or interval or interval day to second or interval year to month) type, however, 'array(1, 2)' is of array<int> type.; line 1 pos 7
 
 
 -- !query
@@ -442,7 +480,7 @@ select +named_struct('a', 1, 'b', 'spark')
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(+ named_struct('a', 1, 'b', 'spark'))' due to data type mismatch: argument 1 requires (numeric or interval) type, however, 'named_struct('a', 1, 'b', 'spark')' is of struct<a:int,b:string> type.; line 1 pos 7
+cannot resolve '(+ named_struct('a', 1, 'b', 'spark'))' due to data type mismatch: argument 1 requires (numeric or interval or interval day to second or interval year to month) type, however, 'named_struct('a', 1, 'b', 'spark')' is of struct<a:int,b:string> type.; line 1 pos 7
 
 
 -- !query
@@ -451,7 +489,7 @@ select +X'1'
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(+ X'01')' due to data type mismatch: argument 1 requires (numeric or interval) type, however, 'X'01'' is of binary type.; line 1 pos 7
+cannot resolve '(+ X'01')' due to data type mismatch: argument 1 requires (numeric or interval or interval day to second or interval year to month) type, however, 'X'01'' is of binary type.; line 1 pos 7
 
 
 -- !query
@@ -460,7 +498,7 @@ select -date '1999-01-01'
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(- DATE '1999-01-01')' due to data type mismatch: argument 1 requires (numeric or interval) type, however, 'DATE '1999-01-01'' is of date type.; line 1 pos 7
+cannot resolve '(- DATE '1999-01-01')' due to data type mismatch: argument 1 requires (numeric or interval or interval day to second or interval year to month) type, however, 'DATE '1999-01-01'' is of date type.; line 1 pos 7
 
 
 -- !query
@@ -469,7 +507,7 @@ select -timestamp '1999-01-01'
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(- TIMESTAMP '1999-01-01 00:00:00')' due to data type mismatch: argument 1 requires (numeric or interval) type, however, 'TIMESTAMP '1999-01-01 00:00:00'' is of timestamp type.; line 1 pos 7
+cannot resolve '(- TIMESTAMP '1999-01-01 00:00:00')' due to data type mismatch: argument 1 requires (numeric or interval or interval day to second or interval year to month) type, however, 'TIMESTAMP '1999-01-01 00:00:00'' is of timestamp type.; line 1 pos 7
 
 
 -- !query
@@ -478,4 +516,4 @@ select -x'2379ACFe'
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(- X'2379ACFE')' due to data type mismatch: argument 1 requires (numeric or interval) type, however, 'X'2379ACFE'' is of binary type.; line 1 pos 7
+cannot resolve '(- X'2379ACFE')' due to data type mismatch: argument 1 requires (numeric or interval or interval day to second or interval year to month) type, however, 'X'2379ACFE'' is of binary type.; line 1 pos 7
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/map.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/map.sql.out
new file mode 100644
index 0000000000000..12c599812cdee
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/map.sql.out
@@ -0,0 +1,20 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 2
+
+
+-- !query
+select element_at(map(1, 'a', 2, 'b'), 5)
+-- !query schema
+struct<>
+-- !query output
+java.util.NoSuchElementException
+Key 5 does not exist.
+
+
+-- !query
+select map(1, 'a', 2, 'b')[5]
+-- !query schema
+struct<>
+-- !query output
+java.util.NoSuchElementException
+Key 5 does not exist.
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/parse-schema-string.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/parse-schema-string.sql.out
new file mode 100644
index 0000000000000..bfbf11d54489c
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/parse-schema-string.sql.out
@@ -0,0 +1,62 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 4
+
+
+-- !query
+select from_csv('1', 'create INT')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Cannot parse the data type: 
+no viable alternative at input 'create'(line 1, pos 0)
+
+== SQL ==
+create INT
+^^^
+
+Failed fallback parsing: 
+no viable alternative at input 'create'(line 1, pos 0)
+
+== SQL ==
+create INT
+^^^
+; line 1 pos 7
+
+
+-- !query
+select from_csv('1', 'cube INT')
+-- !query schema
+struct<from_csv(1):struct<cube:int>>
+-- !query output
+{"cube":1}
+
+
+-- !query
+select from_json('{"create":1}', 'create INT')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Cannot parse the data type: 
+no viable alternative at input 'create'(line 1, pos 0)
+
+== SQL ==
+create INT
+^^^
+
+Failed fallback parsing: 
+no viable alternative at input 'create'(line 1, pos 0)
+
+== SQL ==
+create INT
+^^^
+; line 1 pos 7
+
+
+-- !query
+select from_json('{"cube":1}', 'cube INT')
+-- !query schema
+struct<from_json({"cube":1}):struct<cube:int>>
+-- !query output
+{"cube":1}
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out
new file mode 100644
index 0000000000000..3f4399fe088b3
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out
@@ -0,0 +1,416 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 50
+
+
+-- !query
+select concat_ws()
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+requirement failed: concat_ws requires at least one argument.; line 1 pos 7
+
+
+-- !query
+select format_string()
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+requirement failed: format_string() should take at least 1 argument; line 1 pos 7
+
+
+-- !query
+select 'a' || 'b' || 'c'
+-- !query schema
+struct<concat(concat(a, b), c):string>
+-- !query output
+abc
+
+
+-- !query
+select replace('abc', 'b', '123')
+-- !query schema
+struct<replace(abc, b, 123):string>
+-- !query output
+a123c
+
+
+-- !query
+select replace('abc', 'b')
+-- !query schema
+struct<replace(abc, b, ):string>
+-- !query output
+ac
+
+
+-- !query
+select length(uuid()), (uuid() <> uuid())
+-- !query schema
+struct<length(uuid()):int,(NOT (uuid() = uuid())):boolean>
+-- !query output
+36	true
+
+
+-- !query
+select position('bar' in 'foobarbar'), position(null, 'foobarbar'), position('aaads', null)
+-- !query schema
+struct<locate(bar, foobarbar, 1):int,position(NULL, foobarbar, 1):int,position(aaads, NULL, 1):int>
+-- !query output
+4	NULL	NULL
+
+
+-- !query
+select left("abcd", 2), left("abcd", 5), left("abcd", '2'), left("abcd", null)
+-- !query schema
+struct<left(abcd, 2):string,left(abcd, 5):string,left(abcd, 2):string,left(abcd, NULL):string>
+-- !query output
+ab	abcd	ab	NULL
+
+
+-- !query
+select left(null, -2)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'substring(NULL, 1, -2)' due to data type mismatch: argument 1 requires (string or binary) type, however, 'NULL' is of null type.; line 1 pos 7
+
+
+-- !query
+select left("abcd", -2), left("abcd", 0), left("abcd", 'a')
+-- !query schema
+struct<>
+-- !query output
+java.lang.NumberFormatException
+invalid input syntax for type numeric: a
+
+
+-- !query
+select right("abcd", 2), right("abcd", 5), right("abcd", '2'), right("abcd", null)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'substring('abcd', (- CAST('2' AS DOUBLE)), 2147483647)' due to data type mismatch: argument 2 requires int type, however, '(- CAST('2' AS DOUBLE))' is of double type.; line 1 pos 43
+
+
+-- !query
+select right(null, -2)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'substring(NULL, (- -2), 2147483647)' due to data type mismatch: argument 1 requires (string or binary) type, however, 'NULL' is of null type.; line 1 pos 7
+
+
+-- !query
+select right("abcd", -2), right("abcd", 0), right("abcd", 'a')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'substring('abcd', (- CAST('a' AS DOUBLE)), 2147483647)' due to data type mismatch: argument 2 requires int type, however, '(- CAST('a' AS DOUBLE))' is of double type.; line 1 pos 44
+
+
+-- !query
+SELECT split('aa1cc2ee3', '[1-9]+')
+-- !query schema
+struct<split(aa1cc2ee3, [1-9]+, -1):array<string>>
+-- !query output
+["aa","cc","ee",""]
+
+
+-- !query
+SELECT split('aa1cc2ee3', '[1-9]+', 2)
+-- !query schema
+struct<split(aa1cc2ee3, [1-9]+, 2):array<string>>
+-- !query output
+["aa","cc2ee3"]
+
+
+-- !query
+SELECT substr('Spark SQL', 5)
+-- !query schema
+struct<substr(Spark SQL, 5, 2147483647):string>
+-- !query output
+k SQL
+
+
+-- !query
+SELECT substr('Spark SQL', -3)
+-- !query schema
+struct<substr(Spark SQL, -3, 2147483647):string>
+-- !query output
+SQL
+
+
+-- !query
+SELECT substr('Spark SQL', 5, 1)
+-- !query schema
+struct<substr(Spark SQL, 5, 1):string>
+-- !query output
+k
+
+
+-- !query
+SELECT substr('Spark SQL' from 5)
+-- !query schema
+struct<substring(Spark SQL, 5, 2147483647):string>
+-- !query output
+k SQL
+
+
+-- !query
+SELECT substr('Spark SQL' from -3)
+-- !query schema
+struct<substring(Spark SQL, -3, 2147483647):string>
+-- !query output
+SQL
+
+
+-- !query
+SELECT substr('Spark SQL' from 5 for 1)
+-- !query schema
+struct<substring(Spark SQL, 5, 1):string>
+-- !query output
+k
+
+
+-- !query
+SELECT substring('Spark SQL', 5)
+-- !query schema
+struct<substring(Spark SQL, 5, 2147483647):string>
+-- !query output
+k SQL
+
+
+-- !query
+SELECT substring('Spark SQL', -3)
+-- !query schema
+struct<substring(Spark SQL, -3, 2147483647):string>
+-- !query output
+SQL
+
+
+-- !query
+SELECT substring('Spark SQL', 5, 1)
+-- !query schema
+struct<substring(Spark SQL, 5, 1):string>
+-- !query output
+k
+
+
+-- !query
+SELECT substring('Spark SQL' from 5)
+-- !query schema
+struct<substring(Spark SQL, 5, 2147483647):string>
+-- !query output
+k SQL
+
+
+-- !query
+SELECT substring('Spark SQL' from -3)
+-- !query schema
+struct<substring(Spark SQL, -3, 2147483647):string>
+-- !query output
+SQL
+
+
+-- !query
+SELECT substring('Spark SQL' from 5 for 1)
+-- !query schema
+struct<substring(Spark SQL, 5, 1):string>
+-- !query output
+k
+
+
+-- !query
+SELECT trim(" xyz "), ltrim(" xyz "), rtrim(" xyz ")
+-- !query schema
+struct<trim( xyz ):string,ltrim( xyz ):string,rtrim( xyz ):string>
+-- !query output
+xyz	xyz 	 xyz
+
+
+-- !query
+SELECT trim(BOTH 'xyz' FROM 'yxTomxx'), trim('xyz' FROM 'yxTomxx')
+-- !query schema
+struct<TRIM(BOTH xyz FROM yxTomxx):string,TRIM(BOTH xyz FROM yxTomxx):string>
+-- !query output
+Tom	Tom
+
+
+-- !query
+SELECT trim(BOTH 'x' FROM 'xxxbarxxx'), trim('x' FROM 'xxxbarxxx')
+-- !query schema
+struct<TRIM(BOTH x FROM xxxbarxxx):string,TRIM(BOTH x FROM xxxbarxxx):string>
+-- !query output
+bar	bar
+
+
+-- !query
+SELECT trim(LEADING 'xyz' FROM 'zzzytest')
+-- !query schema
+struct<TRIM(LEADING xyz FROM zzzytest):string>
+-- !query output
+test
+
+
+-- !query
+SELECT trim(LEADING 'xyz' FROM 'zzzytestxyz')
+-- !query schema
+struct<TRIM(LEADING xyz FROM zzzytestxyz):string>
+-- !query output
+testxyz
+
+
+-- !query
+SELECT trim(LEADING 'xy' FROM 'xyxXxyLAST WORD')
+-- !query schema
+struct<TRIM(LEADING xy FROM xyxXxyLAST WORD):string>
+-- !query output
+XxyLAST WORD
+
+
+-- !query
+SELECT trim(TRAILING 'xyz' FROM 'testxxzx')
+-- !query schema
+struct<TRIM(TRAILING xyz FROM testxxzx):string>
+-- !query output
+test
+
+
+-- !query
+SELECT trim(TRAILING 'xyz' FROM 'xyztestxxzx')
+-- !query schema
+struct<TRIM(TRAILING xyz FROM xyztestxxzx):string>
+-- !query output
+xyztest
+
+
+-- !query
+SELECT trim(TRAILING 'xy' FROM 'TURNERyxXxy')
+-- !query schema
+struct<TRIM(TRAILING xy FROM TURNERyxXxy):string>
+-- !query output
+TURNERyxX
+
+
+-- !query
+SELECT btrim('xyxtrimyyx', 'xy')
+-- !query schema
+struct<btrim(xyxtrimyyx, xy):string>
+-- !query output
+trim
+
+
+-- !query
+SELECT btrim(encode(" xyz ", 'utf-8'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'trim(encode(' xyz ', 'utf-8'))' due to data type mismatch: argument 1 requires string type, however, 'encode(' xyz ', 'utf-8')' is of binary type.; line 1 pos 7
+
+
+-- !query
+SELECT btrim(encode('yxTomxx', 'utf-8'), encode('xyz', 'utf-8'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'TRIM(BOTH encode('xyz', 'utf-8') FROM encode('yxTomxx', 'utf-8'))' due to data type mismatch: argument 1 requires string type, however, 'encode('yxTomxx', 'utf-8')' is of binary type. argument 2 requires string type, however, 'encode('xyz', 'utf-8')' is of binary type.; line 1 pos 7
+
+
+-- !query
+SELECT btrim(encode('xxxbarxxx', 'utf-8'), encode('x', 'utf-8'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'TRIM(BOTH encode('x', 'utf-8') FROM encode('xxxbarxxx', 'utf-8'))' due to data type mismatch: argument 1 requires string type, however, 'encode('xxxbarxxx', 'utf-8')' is of binary type. argument 2 requires string type, however, 'encode('x', 'utf-8')' is of binary type.; line 1 pos 7
+
+
+-- !query
+SELECT lpad('hi', 'invalid_length')
+-- !query schema
+struct<>
+-- !query output
+java.lang.NumberFormatException
+invalid input syntax for type numeric: invalid_length
+
+
+-- !query
+SELECT rpad('hi', 'invalid_length')
+-- !query schema
+struct<>
+-- !query output
+java.lang.NumberFormatException
+invalid input syntax for type numeric: invalid_length
+
+
+-- !query
+select decode()
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Invalid number of arguments for function decode. Expected: 2; Found: 0; line 1 pos 7
+
+
+-- !query
+select decode(encode('abc', 'utf-8'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Invalid number of arguments for function decode. Expected: 2; Found: 1; line 1 pos 7
+
+
+-- !query
+select decode(encode('abc', 'utf-8'), 'utf-8')
+-- !query schema
+struct<decode(encode(abc, utf-8), utf-8):string>
+-- !query output
+abc
+
+
+-- !query
+select decode(1, 1, 'Southlake')
+-- !query schema
+struct<decode(1, 1, Southlake):string>
+-- !query output
+Southlake
+
+
+-- !query
+select decode(2, 1, 'Southlake')
+-- !query schema
+struct<decode(2, 1, Southlake):string>
+-- !query output
+NULL
+
+
+-- !query
+select decode(2, 1, 'Southlake', 2, 'San Francisco', 3, 'New Jersey', 4, 'Seattle', 'Non domestic')
+-- !query schema
+struct<decode(2, 1, Southlake, 2, San Francisco, 3, New Jersey, 4, Seattle, Non domestic):string>
+-- !query output
+San Francisco
+
+
+-- !query
+select decode(6, 1, 'Southlake', 2, 'San Francisco', 3, 'New Jersey', 4, 'Seattle', 'Non domestic')
+-- !query schema
+struct<decode(6, 1, Southlake, 2, San Francisco, 3, New Jersey, 4, Seattle, Non domestic):string>
+-- !query output
+Non domestic
+
+
+-- !query
+select decode(6, 1, 'Southlake', 2, 'San Francisco', 3, 'New Jersey', 4, 'Seattle')
+-- !query schema
+struct<decode(6, 1, Southlake, 2, San Francisco, 3, New Jersey, 4, Seattle):string>
+-- !query output
+NULL
diff --git a/sql/core/src/test/resources/sql-tests/results/array.sql.out b/sql/core/src/test/resources/sql-tests/results/array.sql.out
index 2c2b1a7856304..76fdf035ad4ec 100644
--- a/sql/core/src/test/resources/sql-tests/results/array.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/array.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 12
+-- Number of queries: 20
 
 
 -- !query
@@ -92,7 +92,7 @@ select
   array_contains(timestamp_array, timestamp '2016-11-15 20:54:00.000'), array_contains(timestamp_array, timestamp '2016-01-01 20:54:00.000')
 from primitive_arrays
 -- !query schema
-struct<array_contains(boolean_array, true):boolean,array_contains(boolean_array, false):boolean,array_contains(tinyint_array, 2):boolean,array_contains(tinyint_array, 0):boolean,array_contains(smallint_array, 2):boolean,array_contains(smallint_array, 0):boolean,array_contains(int_array, 2):boolean,array_contains(int_array, 0):boolean,array_contains(bigint_array, 2):boolean,array_contains(bigint_array, 0):boolean,array_contains(decimal_array, 9223372036854775809):boolean,array_contains(decimal_array, CAST(1 AS DECIMAL(19,0))):boolean,array_contains(double_array, 2.0):boolean,array_contains(double_array, 0.0):boolean,array_contains(float_array, CAST(2.0 AS FLOAT)):boolean,array_contains(float_array, CAST(0.0 AS FLOAT)):boolean,array_contains(date_array, DATE '2016-03-14'):boolean,array_contains(date_array, DATE '2016-01-01'):boolean,array_contains(timestamp_array, TIMESTAMP '2016-11-15 20:54:00'):boolean,array_contains(timestamp_array, TIMESTAMP '2016-01-01 20:54:00'):boolean>
+struct<array_contains(boolean_array, true):boolean,array_contains(boolean_array, false):boolean,array_contains(tinyint_array, 2):boolean,array_contains(tinyint_array, 0):boolean,array_contains(smallint_array, 2):boolean,array_contains(smallint_array, 0):boolean,array_contains(int_array, 2):boolean,array_contains(int_array, 0):boolean,array_contains(bigint_array, 2):boolean,array_contains(bigint_array, 0):boolean,array_contains(decimal_array, 9223372036854775809):boolean,array_contains(decimal_array, 1):boolean,array_contains(double_array, 2.0):boolean,array_contains(double_array, 0.0):boolean,array_contains(float_array, 2.0):boolean,array_contains(float_array, 0.0):boolean,array_contains(date_array, DATE '2016-03-14'):boolean,array_contains(date_array, DATE '2016-01-01'):boolean,array_contains(timestamp_array, TIMESTAMP '2016-11-15 20:54:00'):boolean,array_contains(timestamp_array, TIMESTAMP '2016-01-01 20:54:00'):boolean>
 -- !query output
 true	false	true	false	true	false	true	false	true	false	true	false	true	false	true	false	true	false	true	false
 
@@ -160,3 +160,68 @@ from primitive_arrays
 struct<size(boolean_array):int,size(tinyint_array):int,size(smallint_array):int,size(int_array):int,size(bigint_array):int,size(decimal_array):int,size(double_array):int,size(float_array):int,size(date_array):int,size(timestamp_array):int>
 -- !query output
 1	2	2	2	2	2	2	2	2	2
+
+
+-- !query
+select element_at(array(1, 2, 3), 5)
+-- !query schema
+struct<element_at(array(1, 2, 3), 5):int>
+-- !query output
+NULL
+
+
+-- !query
+select element_at(array(1, 2, 3), -5)
+-- !query schema
+struct<element_at(array(1, 2, 3), -5):int>
+-- !query output
+NULL
+
+
+-- !query
+select element_at(array(1, 2, 3), 0)
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArrayIndexOutOfBoundsException
+SQL array indices start at 1
+
+
+-- !query
+select elt(4, '123', '456')
+-- !query schema
+struct<elt(4, 123, 456):string>
+-- !query output
+NULL
+
+
+-- !query
+select elt(0, '123', '456')
+-- !query schema
+struct<elt(0, 123, 456):string>
+-- !query output
+NULL
+
+
+-- !query
+select elt(-1, '123', '456')
+-- !query schema
+struct<elt(-1, 123, 456):string>
+-- !query output
+NULL
+
+
+-- !query
+select array(1, 2, 3)[5]
+-- !query schema
+struct<array(1, 2, 3)[5]:int>
+-- !query output
+NULL
+
+
+-- !query
+select array(1, 2, 3)[-1]
+-- !query schema
+struct<array(1, 2, 3)[-1]:int>
+-- !query output
+NULL
diff --git a/sql/core/src/test/resources/sql-tests/results/bitwise.sql.out b/sql/core/src/test/resources/sql-tests/results/bitwise.sql.out
index 552b027df1bc0..649cf21b51eeb 100644
--- a/sql/core/src/test/resources/sql-tests/results/bitwise.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/bitwise.sql.out
@@ -1,11 +1,11 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 27
+-- Number of queries: 32
 
 
 -- !query
 select bit_count(null)
 -- !query schema
-struct<bit_count(CAST(NULL AS INT)):int>
+struct<bit_count(NULL):int>
 -- !query output
 NULL
 
@@ -231,3 +231,45 @@ struct<b1:int,b2:int,bit_xor(b2) OVER (PARTITION BY b1 ORDER BY b2 ASC NULLS FIR
 1	1	1
 2	3	3
 7	7	7
+
+
+-- !query
+select getbit(11L, 3), getbit(11L, 2), getbit(11L, 1), getbit(11L, 0)
+-- !query schema
+struct<getbit(11, 3):tinyint,getbit(11, 2):tinyint,getbit(11, 1):tinyint,getbit(11, 0):tinyint>
+-- !query output
+1	0	1	1
+
+
+-- !query
+select getbit(11L, 2 + 1), getbit(11L, 3 - 1), getbit(10L + 1, 1 * 1), getbit(cast(11L / 1 AS long), 1 - 1)
+-- !query schema
+struct<getbit(11, (2 + 1)):tinyint,getbit(11, (3 - 1)):tinyint,getbit((10 + 1), (1 * 1)):tinyint,getbit(CAST((11 / 1) AS BIGINT), (1 - 1)):tinyint>
+-- !query output
+1	0	1	1
+
+
+-- !query
+select getbit(11L, 63)
+-- !query schema
+struct<getbit(11, 63):tinyint>
+-- !query output
+0
+
+
+-- !query
+select getbit(11L, -1)
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Invalid bit position: -1 is less than zero
+
+
+-- !query
+select getbit(11L, 64)
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Invalid bit position: 64 exceeds the bit upper limit
diff --git a/sql/core/src/test/resources/sql-tests/results/cast.sql.out b/sql/core/src/test/resources/sql-tests/results/cast.sql.out
index 35b4c0e79720b..42d12b80be989 100644
--- a/sql/core/src/test/resources/sql-tests/results/cast.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/cast.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 46
+-- Number of queries: 51
 
 
 -- !query
@@ -269,6 +269,8 @@ Class: org.apache.spark.sql.catalyst.expressions.Cast
 Extended Usage:
     No example/argument for boolean.
 
+    Since: 2.0.1
+
 Function: boolean
 Usage: boolean(expr) - Casts the value `expr` to the target data type `boolean`.
 
@@ -353,6 +355,46 @@ struct<CAST(1.0  AS DECIMAL(10,0)):decimal(10,0)>
 1
 
 
+-- !query
+select cast('1中文' as tinyint)
+-- !query schema
+struct<CAST(1中文 AS TINYINT):tinyint>
+-- !query output
+NULL
+
+
+-- !query
+select cast('1中文' as smallint)
+-- !query schema
+struct<CAST(1中文 AS SMALLINT):smallint>
+-- !query output
+NULL
+
+
+-- !query
+select cast('1中文' as INT)
+-- !query schema
+struct<CAST(1中文 AS INT):int>
+-- !query output
+NULL
+
+
+-- !query
+select cast('中文1' as bigint)
+-- !query schema
+struct<CAST(中文1 AS BIGINT):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select cast('1中文' as bigint)
+-- !query schema
+struct<CAST(1中文 AS BIGINT):bigint>
+-- !query output
+NULL
+
+
 -- !query
 select cast('\t\t true \n\r ' as boolean)
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/change-column.sql.out b/sql/core/src/test/resources/sql-tests/results/change-column.sql.out
index b1a32ad1f63e9..96b28d734f5a7 100644
--- a/sql/core/src/test/resources/sql-tests/results/change-column.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/change-column.sql.out
@@ -50,7 +50,7 @@ ALTER TABLE test_change RENAME COLUMN a TO a1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-RENAME COLUMN is only supported with v2 tables.;
+RENAME COLUMN is only supported with v2 tables.
 
 
 -- !query
@@ -69,7 +69,7 @@ ALTER TABLE test_change CHANGE a TYPE STRING
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-ALTER TABLE CHANGE COLUMN is not supported for changing column 'a' with type 'IntegerType' to 'a' with type 'StringType';
+ALTER TABLE CHANGE COLUMN is not supported for changing column 'a' with type 'IntegerType' to 'a' with type 'StringType'
 
 
 -- !query
@@ -88,7 +88,7 @@ ALTER TABLE test_change CHANGE a AFTER b
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-ALTER COLUMN ... FIRST | ALTER is only supported with v2 tables.;
+ALTER COLUMN ... FIRST | ALTER is only supported with v2 tables.
 
 
 -- !query
@@ -97,7 +97,7 @@ ALTER TABLE test_change CHANGE b FIRST
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-ALTER COLUMN ... FIRST | ALTER is only supported with v2 tables.;
+ALTER COLUMN ... FIRST | ALTER is only supported with v2 tables.
 
 
 -- !query
@@ -176,7 +176,7 @@ ALTER TABLE test_change CHANGE invalid_col TYPE INT
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Can't find column `invalid_col` given table data columns [`a`, `b`, `c`];
+Can't find column `invalid_col` given table data columns [`a`, `b`, `c`]
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/charvarchar.sql.out b/sql/core/src/test/resources/sql-tests/results/charvarchar.sql.out
new file mode 100644
index 0000000000000..00180eec5653f
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/charvarchar.sql.out
@@ -0,0 +1,1159 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 83
+
+
+-- !query
+create table char_tbl(c char(5), v varchar(6)) using parquet
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+desc formatted char_tbl
+-- !query schema
+struct<col_name:string,data_type:string,comment:string>
+-- !query output
+c                   	char(5)             	                    
+v                   	varchar(6)          	                    
+                    	                    	                    
+# Detailed Table Information	                    	                    
+Database            	default             	                    
+Table               	char_tbl            	                    
+Created Time [not included in comparison]
+Last Access [not included in comparison]
+Created By [not included in comparison]
+Type                	MANAGED             	                    
+Provider            	parquet             	                    
+Location [not included in comparison]/{warehouse_dir}/char_tbl
+
+
+-- !query
+desc formatted char_tbl c
+-- !query schema
+struct<info_name:string,info_value:string>
+-- !query output
+col_name	c
+data_type	char(5)
+comment	NULL
+min	NULL
+max	NULL
+num_nulls	NULL
+distinct_count	NULL
+avg_col_len	NULL
+max_col_len	NULL
+histogram	NULL
+
+
+-- !query
+show create table char_tbl
+-- !query schema
+struct<createtab_stmt:string>
+-- !query output
+CREATE TABLE `default`.`char_tbl` (
+  `c` CHAR(5),
+  `v` VARCHAR(6))
+USING parquet
+
+
+-- !query
+create table char_tbl2 using parquet as select * from char_tbl
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+show create table char_tbl2
+-- !query schema
+struct<createtab_stmt:string>
+-- !query output
+CREATE TABLE `default`.`char_tbl2` (
+  `c` CHAR(5),
+  `v` VARCHAR(6))
+USING parquet
+
+
+-- !query
+desc formatted char_tbl2
+-- !query schema
+struct<col_name:string,data_type:string,comment:string>
+-- !query output
+c                   	char(5)             	                    
+v                   	varchar(6)          	                    
+                    	                    	                    
+# Detailed Table Information	                    	                    
+Database            	default             	                    
+Table               	char_tbl2           	                    
+Created Time [not included in comparison]
+Last Access [not included in comparison]
+Created By [not included in comparison]
+Type                	MANAGED             	                    
+Provider            	parquet             	                    
+Location [not included in comparison]/{warehouse_dir}/char_tbl2
+
+
+-- !query
+desc formatted char_tbl2 c
+-- !query schema
+struct<info_name:string,info_value:string>
+-- !query output
+col_name	c
+data_type	char(5)
+comment	NULL
+min	NULL
+max	NULL
+num_nulls	NULL
+distinct_count	NULL
+avg_col_len	NULL
+max_col_len	NULL
+histogram	NULL
+
+
+-- !query
+create table char_tbl3 like char_tbl
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+desc formatted char_tbl3
+-- !query schema
+struct<col_name:string,data_type:string,comment:string>
+-- !query output
+c                   	char(5)             	                    
+v                   	varchar(6)          	                    
+                    	                    	                    
+# Detailed Table Information	                    	                    
+Database            	default             	                    
+Table               	char_tbl3           	                    
+Created Time [not included in comparison]
+Last Access [not included in comparison]
+Created By [not included in comparison]
+Type                	MANAGED             	                    
+Provider            	parquet             	                    
+Location [not included in comparison]/{warehouse_dir}/char_tbl3
+
+
+-- !query
+desc formatted char_tbl3 c
+-- !query schema
+struct<info_name:string,info_value:string>
+-- !query output
+col_name	c
+data_type	char(5)
+comment	NULL
+min	NULL
+max	NULL
+num_nulls	NULL
+distinct_count	NULL
+avg_col_len	NULL
+max_col_len	NULL
+histogram	NULL
+
+
+-- !query
+show create table char_tbl3
+-- !query schema
+struct<createtab_stmt:string>
+-- !query output
+CREATE TABLE `default`.`char_tbl3` (
+  `c` CHAR(5),
+  `v` VARCHAR(6))
+USING parquet
+
+
+-- !query
+create view char_view as select * from char_tbl
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+desc formatted char_view
+-- !query schema
+struct<col_name:string,data_type:string,comment:string>
+-- !query output
+c                   	char(5)             	                    
+v                   	varchar(6)          	                    
+                    	                    	                    
+# Detailed Table Information	                    	                    
+Database            	default             	                    
+Table               	char_view           	                    
+Created Time [not included in comparison]
+Last Access [not included in comparison]
+Created By [not included in comparison]
+Type                	VIEW                	                    
+View Text           	select * from char_tbl	                    
+View Original Text  	select * from char_tbl	                    
+View Catalog and Namespace	spark_catalog.default	                    
+View Query Output Columns	[c, v]              	                    
+Table Properties    	[]
+
+
+-- !query
+desc formatted char_view c
+-- !query schema
+struct<info_name:string,info_value:string>
+-- !query output
+col_name	c
+data_type	char(5)
+comment	NULL
+min	NULL
+max	NULL
+num_nulls	NULL
+distinct_count	NULL
+avg_col_len	NULL
+max_col_len	NULL
+histogram	NULL
+
+
+-- !query
+show create table char_view
+-- !query schema
+struct<createtab_stmt:string>
+-- !query output
+CREATE VIEW `default`.`char_view` (
+  `c`,
+  `v`)
+AS select * from char_tbl
+
+
+-- !query
+alter table char_tbl rename to char_tbl1
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+desc formatted char_tbl1
+-- !query schema
+struct<col_name:string,data_type:string,comment:string>
+-- !query output
+c                   	char(5)             	                    
+v                   	varchar(6)          	                    
+                    	                    	                    
+# Detailed Table Information	                    	                    
+Database            	default             	                    
+Table               	char_tbl1           	                    
+Created Time [not included in comparison]
+Last Access [not included in comparison]
+Created By [not included in comparison]
+Type                	MANAGED             	                    
+Provider            	parquet             	                    
+Location [not included in comparison]/{warehouse_dir}/char_tbl1
+
+
+-- !query
+alter table char_tbl1 change column c type char(6)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+ALTER TABLE CHANGE COLUMN is not supported for changing column 'c' with type 'CharType(5)' to 'c' with type 'CharType(6)'
+
+
+-- !query
+alter table char_tbl1 change column c type char(5)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+desc formatted char_tbl1
+-- !query schema
+struct<col_name:string,data_type:string,comment:string>
+-- !query output
+c                   	char(5)             	                    
+v                   	varchar(6)          	                    
+                    	                    	                    
+# Detailed Table Information	                    	                    
+Database            	default             	                    
+Table               	char_tbl1           	                    
+Created Time [not included in comparison]
+Last Access [not included in comparison]
+Created By [not included in comparison]
+Type                	MANAGED             	                    
+Provider            	parquet             	                    
+Location [not included in comparison]/{warehouse_dir}/char_tbl1
+
+
+-- !query
+alter table char_tbl1 add columns (d char(5))
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+desc formatted char_tbl1
+-- !query schema
+struct<col_name:string,data_type:string,comment:string>
+-- !query output
+c                   	char(5)             	                    
+v                   	varchar(6)          	                    
+d                   	char(5)             	                    
+                    	                    	                    
+# Detailed Table Information	                    	                    
+Database            	default             	                    
+Table               	char_tbl1           	                    
+Created Time [not included in comparison]
+Last Access [not included in comparison]
+Created By [not included in comparison]
+Type                	MANAGED             	                    
+Provider            	parquet             	                    
+Location [not included in comparison]/{warehouse_dir}/char_tbl1
+
+
+-- !query
+alter view char_view as select * from char_tbl2
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+desc formatted char_view
+-- !query schema
+struct<col_name:string,data_type:string,comment:string>
+-- !query output
+c                   	char(5)             	                    
+v                   	varchar(6)          	                    
+                    	                    	                    
+# Detailed Table Information	                    	                    
+Database            	default             	                    
+Table               	char_view           	                    
+Created Time [not included in comparison]
+Last Access [not included in comparison]
+Created By [not included in comparison]
+Type                	VIEW                	                    
+View Text           	select * from char_tbl2	                    
+View Original Text  	select * from char_tbl2	                    
+View Catalog and Namespace	spark_catalog.default	                    
+View Query Output Columns	[c, v]              	                    
+Table Properties    	[]
+
+
+-- !query
+alter table char_tbl1 SET TBLPROPERTIES('yes'='no')
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+desc formatted char_tbl1
+-- !query schema
+struct<col_name:string,data_type:string,comment:string>
+-- !query output
+c                   	char(5)             	                    
+v                   	varchar(6)          	                    
+d                   	char(5)             	                    
+                    	                    	                    
+# Detailed Table Information	                    	                    
+Database            	default             	                    
+Table               	char_tbl1           	                    
+Created Time [not included in comparison]
+Last Access [not included in comparison]
+Created By [not included in comparison]
+Type                	MANAGED             	                    
+Provider            	parquet             	                    
+Table Properties    	[yes=no]            	                    
+Location [not included in comparison]/{warehouse_dir}/char_tbl1
+
+
+-- !query
+alter view char_view SET TBLPROPERTIES('yes'='no')
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+desc formatted char_view
+-- !query schema
+struct<col_name:string,data_type:string,comment:string>
+-- !query output
+c                   	char(5)             	                    
+v                   	varchar(6)          	                    
+                    	                    	                    
+# Detailed Table Information	                    	                    
+Database            	default             	                    
+Table               	char_view           	                    
+Created Time [not included in comparison]
+Last Access [not included in comparison]
+Created By [not included in comparison]
+Type                	VIEW                	                    
+View Text           	select * from char_tbl2	                    
+View Original Text  	select * from char_tbl2	                    
+View Catalog and Namespace	spark_catalog.default	                    
+View Query Output Columns	[c, v]              	                    
+Table Properties    	[yes=no]
+
+
+-- !query
+alter table char_tbl1 UNSET TBLPROPERTIES('yes')
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+desc formatted char_tbl1
+-- !query schema
+struct<col_name:string,data_type:string,comment:string>
+-- !query output
+c                   	char(5)             	                    
+v                   	varchar(6)          	                    
+d                   	char(5)             	                    
+                    	                    	                    
+# Detailed Table Information	                    	                    
+Database            	default             	                    
+Table               	char_tbl1           	                    
+Created Time [not included in comparison]
+Last Access [not included in comparison]
+Created By [not included in comparison]
+Type                	MANAGED             	                    
+Provider            	parquet             	                    
+Location [not included in comparison]/{warehouse_dir}/char_tbl1
+
+
+-- !query
+alter view char_view UNSET TBLPROPERTIES('yes')
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+desc formatted char_view
+-- !query schema
+struct<col_name:string,data_type:string,comment:string>
+-- !query output
+c                   	char(5)             	                    
+v                   	varchar(6)          	                    
+                    	                    	                    
+# Detailed Table Information	                    	                    
+Database            	default             	                    
+Table               	char_view           	                    
+Created Time [not included in comparison]
+Last Access [not included in comparison]
+Created By [not included in comparison]
+Type                	VIEW                	                    
+View Text           	select * from char_tbl2	                    
+View Original Text  	select * from char_tbl2	                    
+View Catalog and Namespace	spark_catalog.default	                    
+View Query Output Columns	[c, v]              	                    
+Table Properties    	[]
+
+
+-- !query
+alter table char_tbl1 SET SERDEPROPERTIES('yes'='no')
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+desc formatted char_tbl1
+-- !query schema
+struct<col_name:string,data_type:string,comment:string>
+-- !query output
+c                   	char(5)             	                    
+v                   	varchar(6)          	                    
+d                   	char(5)             	                    
+                    	                    	                    
+# Detailed Table Information	                    	                    
+Database            	default             	                    
+Table               	char_tbl1           	                    
+Created Time [not included in comparison]
+Last Access [not included in comparison]
+Created By [not included in comparison]
+Type                	MANAGED             	                    
+Provider            	parquet             	                    
+Location [not included in comparison]/{warehouse_dir}/char_tbl1	                    
+Storage Properties  	[yes=no]
+
+
+-- !query
+create table char_part(c1 char(5), c2 char(2), v1 varchar(6), v2 varchar(2)) using parquet partitioned by (v2, c2)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+desc formatted char_part
+-- !query schema
+struct<col_name:string,data_type:string,comment:string>
+-- !query output
+c1                  	char(5)             	                    
+v1                  	varchar(6)          	                    
+v2                  	varchar(2)          	                    
+c2                  	char(2)             	                    
+# Partition Information	                    	                    
+# col_name          	data_type           	comment             
+v2                  	varchar(2)          	                    
+c2                  	char(2)             	                    
+                    	                    	                    
+# Detailed Table Information	                    	                    
+Database            	default             	                    
+Table               	char_part           	                    
+Created Time [not included in comparison]
+Last Access [not included in comparison]
+Created By [not included in comparison]
+Type                	MANAGED             	                    
+Provider            	parquet             	                    
+Location [not included in comparison]/{warehouse_dir}/char_part	                    
+Partition Provider  	Catalog
+
+
+-- !query
+alter table char_part add partition (v2='ke', c2='nt') location 'loc1'
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+desc formatted char_part
+-- !query schema
+struct<col_name:string,data_type:string,comment:string>
+-- !query output
+c1                  	char(5)             	                    
+v1                  	varchar(6)          	                    
+v2                  	varchar(2)          	                    
+c2                  	char(2)             	                    
+# Partition Information	                    	                    
+# col_name          	data_type           	comment             
+v2                  	varchar(2)          	                    
+c2                  	char(2)             	                    
+                    	                    	                    
+# Detailed Table Information	                    	                    
+Database            	default             	                    
+Table               	char_part           	                    
+Created Time [not included in comparison]
+Last Access [not included in comparison]
+Created By [not included in comparison]
+Type                	MANAGED             	                    
+Provider            	parquet             	                    
+Location [not included in comparison]/{warehouse_dir}/char_part	                    
+Partition Provider  	Catalog
+
+
+-- !query
+alter table char_part partition (v2='ke') rename to partition (v2='nt')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Partition spec is invalid. The spec (v2) must match the partition spec (v2, c2) defined in table '`default`.`char_part`'
+
+
+-- !query
+desc formatted char_part
+-- !query schema
+struct<col_name:string,data_type:string,comment:string>
+-- !query output
+c1                  	char(5)             	                    
+v1                  	varchar(6)          	                    
+v2                  	varchar(2)          	                    
+c2                  	char(2)             	                    
+# Partition Information	                    	                    
+# col_name          	data_type           	comment             
+v2                  	varchar(2)          	                    
+c2                  	char(2)             	                    
+                    	                    	                    
+# Detailed Table Information	                    	                    
+Database            	default             	                    
+Table               	char_part           	                    
+Created Time [not included in comparison]
+Last Access [not included in comparison]
+Created By [not included in comparison]
+Type                	MANAGED             	                    
+Provider            	parquet             	                    
+Location [not included in comparison]/{warehouse_dir}/char_part	                    
+Partition Provider  	Catalog
+
+
+-- !query
+alter table char_part partition (v2='ke', c2='nt') set location 'loc2'
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+desc formatted char_part
+-- !query schema
+struct<col_name:string,data_type:string,comment:string>
+-- !query output
+c1                  	char(5)             	                    
+v1                  	varchar(6)          	                    
+v2                  	varchar(2)          	                    
+c2                  	char(2)             	                    
+# Partition Information	                    	                    
+# col_name          	data_type           	comment             
+v2                  	varchar(2)          	                    
+c2                  	char(2)             	                    
+                    	                    	                    
+# Detailed Table Information	                    	                    
+Database            	default             	                    
+Table               	char_part           	                    
+Created Time [not included in comparison]
+Last Access [not included in comparison]
+Created By [not included in comparison]
+Type                	MANAGED             	                    
+Provider            	parquet             	                    
+Location [not included in comparison]/{warehouse_dir}/char_part	                    
+Partition Provider  	Catalog
+
+
+-- !query
+MSCK REPAIR TABLE char_part
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+desc formatted char_part
+-- !query schema
+struct<col_name:string,data_type:string,comment:string>
+-- !query output
+c1                  	char(5)             	                    
+v1                  	varchar(6)          	                    
+v2                  	varchar(2)          	                    
+c2                  	char(2)             	                    
+# Partition Information	                    	                    
+# col_name          	data_type           	comment             
+v2                  	varchar(2)          	                    
+c2                  	char(2)             	                    
+                    	                    	                    
+# Detailed Table Information	                    	                    
+Database            	default             	                    
+Table               	char_part           	                    
+Created Time [not included in comparison]
+Last Access [not included in comparison]
+Created By [not included in comparison]
+Type                	MANAGED             	                    
+Provider            	parquet             	                    
+Location [not included in comparison]/{warehouse_dir}/char_part	                    
+Partition Provider  	Catalog
+
+
+-- !query
+create temporary view str_view as select c, v from values
+    (null, null),
+    (null, 'S'),
+    ('N', 'N '),
+    ('Ne', 'Sp'),
+    ('Net  ', 'Spa  '),
+    ('NetE', 'Spar'),
+    ('NetEa ', 'Spark '),
+    ('NetEas ', 'Spark'),
+    ('NetEase', 'Spark-') t(c, v)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+create table char_tbl4(c7 char(7), c8 char(8), v varchar(6), s string) using parquet
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+insert into char_tbl4 select c, c, v, c from str_view
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+select c7, c8, v, s from char_tbl4
+-- !query schema
+struct<c7:string,c8:string,v:string,s:string>
+-- !query output
+N      	N       	N 	N
+NULL	NULL	NULL	NULL
+NULL	NULL	S	NULL
+Ne     	Ne      	Sp	Ne
+Net    	Net     	Spa  	Net  
+NetE   	NetE    	Spar	NetE
+NetEa  	NetEa   	Spark 	NetEa 
+NetEas 	NetEas  	Spark	NetEas 
+NetEase	NetEase 	Spark-	NetEase
+
+
+-- !query
+select c7, c8, v, s from char_tbl4 where c7 = c8
+-- !query schema
+struct<c7:string,c8:string,v:string,s:string>
+-- !query output
+N      	N       	N 	N
+Ne     	Ne      	Sp	Ne
+Net    	Net     	Spa  	Net  
+NetE   	NetE    	Spar	NetE
+NetEa  	NetEa   	Spark 	NetEa 
+NetEas 	NetEas  	Spark	NetEas 
+NetEase	NetEase 	Spark-	NetEase
+
+
+-- !query
+select c7, c8, v, s from char_tbl4 where c7 = v
+-- !query schema
+struct<c7:string,c8:string,v:string,s:string>
+-- !query output
+
+
+
+-- !query
+select c7, c8, v, s from char_tbl4 where c7 = s
+-- !query schema
+struct<c7:string,c8:string,v:string,s:string>
+-- !query output
+NetEas 	NetEas  	Spark	NetEas 
+NetEase	NetEase 	Spark-	NetEase
+
+
+-- !query
+select c7, c8, v, s from char_tbl4 where c7 = 'NetEase               '
+-- !query schema
+struct<c7:string,c8:string,v:string,s:string>
+-- !query output
+NetEase	NetEase 	Spark-	NetEase
+
+
+-- !query
+select c7, c8, v, s from char_tbl4 where v = 'Spark '
+-- !query schema
+struct<c7:string,c8:string,v:string,s:string>
+-- !query output
+NetEa  	NetEa   	Spark 	NetEa
+
+
+-- !query
+select c7, c8, v, s from char_tbl4 order by c7
+-- !query schema
+struct<c7:string,c8:string,v:string,s:string>
+-- !query output
+NULL	NULL	NULL	NULL
+NULL	NULL	S	NULL
+N      	N       	N 	N
+Ne     	Ne      	Sp	Ne
+Net    	Net     	Spa  	Net  
+NetE   	NetE    	Spar	NetE
+NetEa  	NetEa   	Spark 	NetEa 
+NetEas 	NetEas  	Spark	NetEas 
+NetEase	NetEase 	Spark-	NetEase
+
+
+-- !query
+select c7, c8, v, s from char_tbl4 order by v
+-- !query schema
+struct<c7:string,c8:string,v:string,s:string>
+-- !query output
+NULL	NULL	NULL	NULL
+N      	N       	N 	N
+NULL	NULL	S	NULL
+Ne     	Ne      	Sp	Ne
+Net    	Net     	Spa  	Net  
+NetE   	NetE    	Spar	NetE
+NetEas 	NetEas  	Spark	NetEas 
+NetEa  	NetEa   	Spark 	NetEa 
+NetEase	NetEase 	Spark-	NetEase
+
+
+-- !query
+select ascii(c7), ascii(c8), ascii(v), ascii(s) from char_tbl4
+-- !query schema
+struct<ascii(c7):int,ascii(c8):int,ascii(v):int,ascii(s):int>
+-- !query output
+78	78	78	78
+78	78	83	78
+78	78	83	78
+78	78	83	78
+78	78	83	78
+78	78	83	78
+78	78	83	78
+NULL	NULL	83	NULL
+NULL	NULL	NULL	NULL
+
+
+-- !query
+select base64(c7), base64(c8), base64(v), ascii(s) from char_tbl4
+-- !query schema
+struct<base64(c7):string,base64(c8):string,base64(v):string,ascii(s):int>
+-- !query output
+NULL	NULL	NULL	NULL
+NULL	NULL	Uw==	NULL
+TiAgICAgIA==	TiAgICAgICA=	TiA=	78
+TmUgICAgIA==	TmUgICAgICA=	U3A=	78
+TmV0ICAgIA==	TmV0ICAgICA=	U3BhICA=	78
+TmV0RSAgIA==	TmV0RSAgICA=	U3Bhcg==	78
+TmV0RWEgIA==	TmV0RWEgICA=	U3Bhcmsg	78
+TmV0RWFzIA==	TmV0RWFzICA=	U3Bhcms=	78
+TmV0RWFzZQ==	TmV0RWFzZSA=	U3Bhcmst	78
+
+
+-- !query
+select bit_length(c7), bit_length(c8), bit_length(v), bit_length(s) from char_tbl4
+-- !query schema
+struct<bit_length(c7):int,bit_length(c8):int,bit_length(v):int,bit_length(s):int>
+-- !query output
+56	64	16	16
+56	64	16	8
+56	64	32	32
+56	64	40	40
+56	64	40	56
+56	64	48	48
+56	64	48	56
+NULL	NULL	8	NULL
+NULL	NULL	NULL	NULL
+
+
+-- !query
+select char_length(c7), char_length(c8), char_length(v), char_length(s) from char_tbl4
+-- !query schema
+struct<char_length(c7):int,char_length(c8):int,char_length(v):int,char_length(s):int>
+-- !query output
+7	8	2	1
+7	8	2	2
+7	8	4	4
+7	8	5	5
+7	8	5	7
+7	8	6	6
+7	8	6	7
+NULL	NULL	1	NULL
+NULL	NULL	NULL	NULL
+
+
+-- !query
+select octet_length(c7), octet_length(c8), octet_length(v), octet_length(s) from char_tbl4
+-- !query schema
+struct<octet_length(c7):int,octet_length(c8):int,octet_length(v):int,octet_length(s):int>
+-- !query output
+7	8	2	1
+7	8	2	2
+7	8	4	4
+7	8	5	5
+7	8	5	7
+7	8	6	6
+7	8	6	7
+NULL	NULL	1	NULL
+NULL	NULL	NULL	NULL
+
+
+-- !query
+select concat_ws('|', c7, c8), concat_ws('|', c7, v), concat_ws('|', c7, s), concat_ws('|', v, s) from char_tbl4
+-- !query schema
+struct<concat_ws(|, c7, c8):string,concat_ws(|, c7, v):string,concat_ws(|, c7, s):string,concat_ws(|, v, s):string>
+-- !query output
+			
+	S		S
+N      |N       	N      |N 	N      |N	N |N
+Ne     |Ne      	Ne     |Sp	Ne     |Ne	Sp|Ne
+Net    |Net     	Net    |Spa  	Net    |Net  	Spa  |Net  
+NetE   |NetE    	NetE   |Spar	NetE   |NetE	Spar|NetE
+NetEa  |NetEa   	NetEa  |Spark 	NetEa  |NetEa 	Spark |NetEa 
+NetEas |NetEas  	NetEas |Spark	NetEas |NetEas 	Spark|NetEas 
+NetEase|NetEase 	NetEase|Spark-	NetEase|NetEase	Spark-|NetEase
+
+
+-- !query
+select concat(c7, c8), concat(c7, v), concat(c7, s), concat(v, s) from char_tbl4
+-- !query schema
+struct<concat(c7, c8):string,concat(c7, v):string,concat(c7, s):string,concat(v, s):string>
+-- !query output
+N      N       	N      N 	N      N	N N
+NULL	NULL	NULL	NULL
+NULL	NULL	NULL	NULL
+Ne     Ne      	Ne     Sp	Ne     Ne	SpNe
+Net    Net     	Net    Spa  	Net    Net  	Spa  Net  
+NetE   NetE    	NetE   Spar	NetE   NetE	SparNetE
+NetEa  NetEa   	NetEa  Spark 	NetEa  NetEa 	Spark NetEa 
+NetEas NetEas  	NetEas Spark	NetEas NetEas 	SparkNetEas 
+NetEaseNetEase 	NetEaseSpark-	NetEaseNetEase	Spark-NetEase
+
+
+-- !query
+select like(c7, 'Ne     _'), like(c8, 'Ne     _') from char_tbl4
+-- !query schema
+struct<c7 LIKE Ne     _:boolean,c8 LIKE Ne     _:boolean>
+-- !query output
+NULL	NULL
+NULL	NULL
+false	false
+false	false
+false	false
+false	false
+false	false
+false	false
+false	true
+
+
+-- !query
+select like(v, 'Spark_') from char_tbl4
+-- !query schema
+struct<v LIKE Spark_:boolean>
+-- !query output
+NULL
+false
+false
+false
+false
+false
+false
+true
+true
+
+
+-- !query
+select c7 = c8, upper(c7) = upper(c8), lower(c7) = lower(c8) from char_tbl4 where s = 'NetEase'
+-- !query schema
+struct<(c7 = c8):boolean,(upper(c7) = upper(c8)):boolean,(lower(c7) = lower(c8)):boolean>
+-- !query output
+true	false	false
+
+
+-- !query
+select c7 = s, upper(c7) = upper(s), lower(c7) = lower(s) from char_tbl4 where s = 'NetEase'
+-- !query schema
+struct<(c7 = s):boolean,(upper(c7) = upper(s)):boolean,(lower(c7) = lower(s)):boolean>
+-- !query output
+true	true	true
+
+
+-- !query
+select c7 = 'NetEase', upper(c7) = upper('NetEase'), lower(c7) = lower('NetEase') from char_tbl4 where s = 'NetEase'
+-- !query schema
+struct<(c7 = NetEase):boolean,(upper(c7) = upper(NetEase)):boolean,(lower(c7) = lower(NetEase)):boolean>
+-- !query output
+true	true	true
+
+
+-- !query
+select printf('Hey, %s%s%s%s', c7, c8, v, s) from char_tbl4
+-- !query schema
+struct<printf(Hey, %s%s%s%s, c7, c8, v, s):string>
+-- !query output
+Hey, N      N       N N
+Hey, Ne     Ne      SpNe
+Hey, Net    Net     Spa  Net  
+Hey, NetE   NetE    SparNetE
+Hey, NetEa  NetEa   Spark NetEa 
+Hey, NetEas NetEas  SparkNetEas 
+Hey, NetEaseNetEase Spark-NetEase
+Hey, nullnullSnull
+Hey, nullnullnullnull
+
+
+-- !query
+select repeat(c7, 2), repeat(c8, 2), repeat(v, 2), repeat(s, 2) from char_tbl4
+-- !query schema
+struct<repeat(c7, 2):string,repeat(c8, 2):string,repeat(v, 2):string,repeat(s, 2):string>
+-- !query output
+N      N      	N       N       	N N 	NN
+NULL	NULL	NULL	NULL
+NULL	NULL	SS	NULL
+Ne     Ne     	Ne      Ne      	SpSp	NeNe
+Net    Net    	Net     Net     	Spa  Spa  	Net  Net  
+NetE   NetE   	NetE    NetE    	SparSpar	NetENetE
+NetEa  NetEa  	NetEa   NetEa   	Spark Spark 	NetEa NetEa 
+NetEas NetEas 	NetEas  NetEas  	SparkSpark	NetEas NetEas 
+NetEaseNetEase	NetEase NetEase 	Spark-Spark-	NetEaseNetEase
+
+
+-- !query
+select replace(c7, 'Net', 'Apache'), replace(c8, 'Net', 'Apache'), replace(v, 'Spark', 'Kyuubi'), replace(s, 'Net', 'Apache') from char_tbl4
+-- !query schema
+struct<replace(c7, Net, Apache):string,replace(c8, Net, Apache):string,replace(v, Spark, Kyuubi):string,replace(s, Net, Apache):string>
+-- !query output
+Apache    	Apache     	Spa  	Apache  
+ApacheE   	ApacheE    	Spar	ApacheE
+ApacheEa  	ApacheEa   	Kyuubi 	ApacheEa 
+ApacheEas 	ApacheEas  	Kyuubi	ApacheEas 
+ApacheEase	ApacheEase 	Kyuubi-	ApacheEase
+N      	N       	N 	N
+NULL	NULL	NULL	NULL
+NULL	NULL	S	NULL
+Ne     	Ne      	Sp	Ne
+
+
+-- !query
+select rpad(c7, 10), rpad(c8, 5), rpad(v, 5), rpad(s, 5)  from char_tbl4
+-- !query schema
+struct<rpad(c7, 10,  ):string,rpad(c8, 5,  ):string,rpad(v, 5,  ):string,rpad(s, 5,  ):string>
+-- !query output
+N         	N    	N    	N    
+NULL	NULL	NULL	NULL
+NULL	NULL	S    	NULL
+Ne        	Ne   	Sp   	Ne   
+Net       	Net  	Spa  	Net  
+NetE      	NetE 	Spar 	NetE 
+NetEa     	NetEa	Spark	NetEa
+NetEas    	NetEa	Spark	NetEa
+NetEase   	NetEa	Spark	NetEa
+
+
+-- !query
+select rtrim(c7), rtrim(c8), rtrim(v), rtrim(s) from char_tbl4
+-- !query schema
+struct<rtrim(c7):string,rtrim(c8):string,rtrim(v):string,rtrim(s):string>
+-- !query output
+N	N	N	N
+NULL	NULL	NULL	NULL
+NULL	NULL	S	NULL
+Ne	Ne	Sp	Ne
+Net	Net	Spa	Net
+NetE	NetE	Spar	NetE
+NetEa	NetEa	Spark	NetEa
+NetEas	NetEas	Spark	NetEas
+NetEase	NetEase	Spark-	NetEase
+
+
+-- !query
+select split(c7, 'e'), split(c8, 'e'), split(v, 'a'), split(s, 'e') from char_tbl4
+-- !query schema
+struct<split(c7, e, -1):array<string>,split(c8, e, -1):array<string>,split(v, a, -1):array<string>,split(s, e, -1):array<string>>
+-- !query output
+NULL	NULL	NULL	NULL
+NULL	NULL	["S"]	NULL
+["N      "]	["N       "]	["N "]	["N"]
+["N","     "]	["N","      "]	["Sp"]	["N",""]
+["N","t    "]	["N","t     "]	["Sp","  "]	["N","t  "]
+["N","tE   "]	["N","tE    "]	["Sp","r"]	["N","tE"]
+["N","tEa  "]	["N","tEa   "]	["Sp","rk "]	["N","tEa "]
+["N","tEas "]	["N","tEas  "]	["Sp","rk"]	["N","tEas "]
+["N","tEas",""]	["N","tEas"," "]	["Sp","rk-"]	["N","tEas",""]
+
+
+-- !query
+select substring(c7, 2), substring(c8, 2), substring(v, 3), substring(s, 2) from char_tbl4
+-- !query schema
+struct<substring(c7, 2, 2147483647):string,substring(c8, 2, 2147483647):string,substring(v, 3, 2147483647):string,substring(s, 2, 2147483647):string>
+-- !query output
+      	       		
+NULL	NULL		NULL
+NULL	NULL	NULL	NULL
+e     	e      		e
+et    	et     	a  	et  
+etE   	etE    	ar	etE
+etEa  	etEa   	ark 	etEa 
+etEas 	etEas  	ark	etEas 
+etEase	etEase 	ark-	etEase
+
+
+-- !query
+select left(c7, 2), left(c8, 2), left(v, 3), left(s, 2) from char_tbl4
+-- !query schema
+struct<left(c7, 2):string,left(c8, 2):string,left(v, 3):string,left(s, 2):string>
+-- !query output
+N 	N 	N 	N
+NULL	NULL	NULL	NULL
+NULL	NULL	S	NULL
+Ne	Ne	Sp	Ne
+Ne	Ne	Spa	Ne
+Ne	Ne	Spa	Ne
+Ne	Ne	Spa	Ne
+Ne	Ne	Spa	Ne
+Ne	Ne	Spa	Ne
+
+
+-- !query
+select right(c7, 2), right(c8, 2), right(v, 3), right(s, 2) from char_tbl4
+-- !query schema
+struct<right(c7, 2):string,right(c8, 2):string,right(v, 3):string,right(s, 2):string>
+-- !query output
+  	  	N 	N
+  	  	Sp	Ne
+  	  	a  	  
+  	  	par	tE
+  	  	rk 	a 
+NULL	NULL	NULL	NULL
+NULL	NULL	S	NULL
+s 	  	ark	s 
+se	e 	rk-	se
+
+
+-- !query
+select typeof(c7), typeof(c8), typeof(v), typeof(s) from char_tbl4 limit 1
+-- !query schema
+struct<typeof(c7):string,typeof(c8):string,typeof(v):string,typeof(s):string>
+-- !query output
+string	string	string	string
+
+
+-- !query
+select cast(c7 as char(1)), cast(c8 as char(10)), cast(v as char(1)), cast(v as varchar(1)), cast(s as char(5)) from char_tbl4
+-- !query schema
+struct<c7:string,c8:string,v:string,v:string,s:string>
+-- !query output
+N      	N       	N 	N 	N
+NULL	NULL	NULL	NULL	NULL
+NULL	NULL	S	S	NULL
+Ne     	Ne      	Sp	Sp	Ne
+Net    	Net     	Spa  	Spa  	Net  
+NetE   	NetE    	Spar	Spar	NetE
+NetEa  	NetEa   	Spark 	Spark 	NetEa 
+NetEas 	NetEas  	Spark	Spark	NetEas 
+NetEase	NetEase 	Spark-	Spark-	NetEase
+
+
+-- !query
+drop table char_tbl1
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+drop table char_tbl2
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+drop table char_tbl3
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+drop table char_tbl4
+-- !query schema
+struct<>
+-- !query output
+
diff --git a/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out b/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out
index 04ddfe0ac128c..994f90c7dfc4d 100644
--- a/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out
@@ -161,7 +161,7 @@ SELECT db1.t1.i1 FROM t1, mydb2.t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`db1.t1.i1`' given input columns: [spark_catalog.mydb2.t1.i1, spark_catalog.mydb2.t1.i1]; line 1 pos 7
+cannot resolve 'db1.t1.i1' given input columns: [spark_catalog.mydb2.t1.i1, spark_catalog.mydb2.t1.i1]; line 1 pos 7
 
 
 -- !query
@@ -186,7 +186,7 @@ SELECT mydb1.t1 FROM t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`mydb1.t1`' given input columns: [spark_catalog.mydb1.t1.i1]; line 1 pos 7
+cannot resolve 'mydb1.t1' given input columns: [spark_catalog.mydb1.t1.i1]; line 1 pos 7
 
 
 -- !query
@@ -195,7 +195,7 @@ SELECT t1.x.y.* FROM t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 't1.x.y.*' given input columns 'i1';
+cannot resolve 't1.x.y.*' given input columns 'i1'
 
 
 -- !query
@@ -204,7 +204,7 @@ SELECT t1 FROM mydb1.t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`t1`' given input columns: [spark_catalog.mydb1.t1.i1]; line 1 pos 7
+cannot resolve 't1' given input columns: [spark_catalog.mydb1.t1.i1]; line 1 pos 7
 
 
 -- !query
@@ -221,7 +221,7 @@ SELECT mydb1.t1.i1 FROM t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`mydb1.t1.i1`' given input columns: [spark_catalog.mydb2.t1.i1]; line 1 pos 7
+cannot resolve 'mydb1.t1.i1' given input columns: [spark_catalog.mydb2.t1.i1]; line 1 pos 7
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/comparator.sql.out b/sql/core/src/test/resources/sql-tests/results/comparator.sql.out
index 721b56cc1da2f..a19b3228a7f99 100644
--- a/sql/core/src/test/resources/sql-tests/results/comparator.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/comparator.sql.out
@@ -21,7 +21,7 @@ true
 -- !query
 select '1 ' = 1Y
 -- !query schema
-struct<(CAST(1  AS TINYINT) = 1):boolean>
+struct<(1  = 1):boolean>
 -- !query output
 true
 
@@ -29,7 +29,7 @@ true
 -- !query
 select '\t1 ' = 1Y
 -- !query schema
-struct<(CAST(	1  AS TINYINT) = 1):boolean>
+struct<(	1  = 1):boolean>
 -- !query output
 true
 
@@ -37,7 +37,7 @@ true
 -- !query
 select '1 ' = 1S
 -- !query schema
-struct<(CAST(1  AS SMALLINT) = 1):boolean>
+struct<(1  = 1):boolean>
 -- !query output
 true
 
@@ -45,7 +45,7 @@ true
 -- !query
 select '1 ' = 1
 -- !query schema
-struct<(CAST(1  AS INT) = 1):boolean>
+struct<(1  = 1):boolean>
 -- !query output
 true
 
@@ -53,7 +53,7 @@ true
 -- !query
 select ' 1' = 1L
 -- !query schema
-struct<(CAST( 1 AS BIGINT) = 1):boolean>
+struct<( 1 = 1):boolean>
 -- !query output
 true
 
@@ -61,7 +61,7 @@ true
 -- !query
 select ' 1' = cast(1.0 as float)
 -- !query schema
-struct<(CAST( 1 AS FLOAT) = CAST(1.0 AS FLOAT)):boolean>
+struct<( 1 = CAST(1.0 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -69,7 +69,7 @@ true
 -- !query
 select ' 1.0 ' = 1.0D
 -- !query schema
-struct<(CAST( 1.0  AS DOUBLE) = 1.0):boolean>
+struct<( 1.0  = 1.0):boolean>
 -- !query output
 true
 
@@ -77,6 +77,6 @@ true
 -- !query
 select ' 1.0 ' = 1.0BD
 -- !query schema
-struct<(CAST( 1.0  AS DOUBLE) = CAST(1.0 AS DOUBLE)):boolean>
+struct<( 1.0  = 1.0):boolean>
 -- !query output
 true
diff --git a/sql/core/src/test/resources/sql-tests/results/count.sql.out b/sql/core/src/test/resources/sql-tests/results/count.sql.out
index 68a5114bb5859..c0a04a1ddffd8 100644
--- a/sql/core/src/test/resources/sql-tests/results/count.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/count.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 5
+-- Number of queries: 21
 
 
 -- !query
@@ -38,18 +38,148 @@ struct<count(DISTINCT 1):bigint,count(DISTINCT NULL):bigint,count(DISTINCT a):bi
 
 
 -- !query
-SELECT count(a, b), count(b, a), count(testData.*) FROM testData
+SELECT count(a, b), count(b, a), count(testData.*, testData.*) FROM testData
 -- !query schema
-struct<count(a, b):bigint,count(b, a):bigint,count(a, b):bigint>
+struct<count(a, b):bigint,count(b, a):bigint,count(a, b, a, b):bigint>
 -- !query output
 4	4	4
 
 
 -- !query
 SELECT
-  count(DISTINCT a, b), count(DISTINCT b, a), count(DISTINCT *), count(DISTINCT testData.*)
+  count(DISTINCT a, b), count(DISTINCT b, a), count(DISTINCT *), count(DISTINCT testData.*, testData.*)
 FROM testData
 -- !query schema
-struct<count(DISTINCT a, b):bigint,count(DISTINCT b, a):bigint,count(DISTINCT a, b):bigint,count(DISTINCT a, b):bigint>
+struct<count(DISTINCT a, b):bigint,count(DISTINCT b, a):bigint,count(DISTINCT a, b):bigint,count(DISTINCT a, b, a, b):bigint>
 -- !query output
 3	3	3	3
+
+
+-- !query
+SELECT count(DISTINCT 3,2)
+-- !query schema
+struct<count(DISTINCT 3, 2):bigint>
+-- !query output
+1
+
+
+-- !query
+SELECT count(DISTINCT 2), count(DISTINCT 2,3)
+-- !query schema
+struct<count(DISTINCT 2):bigint,count(DISTINCT 2, 3):bigint>
+-- !query output
+1	1
+
+
+-- !query
+SELECT count(DISTINCT 2), count(DISTINCT 3,2)
+-- !query schema
+struct<count(DISTINCT 2):bigint,count(DISTINCT 3, 2):bigint>
+-- !query output
+1	1
+
+
+-- !query
+SELECT count(DISTINCT a), count(DISTINCT 2,3) FROM testData
+-- !query schema
+struct<count(DISTINCT a):bigint,count(DISTINCT 2, 3):bigint>
+-- !query output
+2	1
+
+
+-- !query
+SELECT count(DISTINCT a), count(DISTINCT 3,2) FROM testData
+-- !query schema
+struct<count(DISTINCT a):bigint,count(DISTINCT 3, 2):bigint>
+-- !query output
+2	1
+
+
+-- !query
+SELECT count(DISTINCT a), count(DISTINCT 2), count(DISTINCT 2,3) FROM testData
+-- !query schema
+struct<count(DISTINCT a):bigint,count(DISTINCT 2):bigint,count(DISTINCT 2, 3):bigint>
+-- !query output
+2	1	1
+
+
+-- !query
+SELECT count(DISTINCT a), count(DISTINCT 2), count(DISTINCT 3,2) FROM testData
+-- !query schema
+struct<count(DISTINCT a):bigint,count(DISTINCT 2):bigint,count(DISTINCT 3, 2):bigint>
+-- !query output
+2	1	1
+
+
+-- !query
+SELECT count(distinct 0.8), percentile_approx(distinct a, 0.8) FROM testData
+-- !query schema
+struct<count(DISTINCT 0.8):bigint,percentile_approx(DISTINCT a, 0.8, 10000):int>
+-- !query output
+1	2
+
+
+-- !query
+set spark.sql.legacy.allowParameterlessCount=true
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+spark.sql.legacy.allowParameterlessCount	true
+
+
+-- !query
+SELECT count() FROM testData
+-- !query schema
+struct<count():bigint>
+-- !query output
+0
+
+
+-- !query
+set spark.sql.legacy.allowParameterlessCount=false
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+spark.sql.legacy.allowParameterlessCount	false
+
+
+-- !query
+SELECT count() FROM testData
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'count()' due to data type mismatch: count requires at least one argument. If you have to call the function count without arguments, set the legacy configuration `spark.sql.legacy.allowParameterlessCount` as true; line 1 pos 7
+
+
+-- !query
+set spark.sql.legacy.allowStarWithSingleTableIdentifierInCount=true
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+spark.sql.legacy.allowStarWithSingleTableIdentifierInCount	true
+
+
+-- !query
+SELECT count(testData.*) FROM testData
+-- !query schema
+struct<count(a, b):bigint>
+-- !query output
+4
+
+
+-- !query
+set spark.sql.legacy.allowStarWithSingleTableIdentifierInCount=false
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+spark.sql.legacy.allowStarWithSingleTableIdentifierInCount	false
+
+
+-- !query
+SELECT count(testData.*) FROM testData
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+count(testData.*) is not allowed. Please use count(*) or expand the columns manually, e.g. count(col1, col2)
diff --git a/sql/core/src/test/resources/sql-tests/results/csv-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/csv-functions.sql.out
index 1e3173172a528..2ca44d51244a5 100644
--- a/sql/core/src/test/resources/sql-tests/results/csv-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/csv-functions.sql.out
@@ -24,7 +24,7 @@ select from_csv('1', 1)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-The expression '1' is not a valid schema string.;; line 1 pos 7
+The expression '1' is not a valid schema string.; line 1 pos 7
 
 
 -- !query
@@ -33,7 +33,14 @@ select from_csv('1', 'a InvalidType')
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
+Cannot parse the data type: 
+extraneous input 'InvalidType' expecting <EOF>(line 1, pos 2)
 
+== SQL ==
+a InvalidType
+--^^^
+
+Failed fallback parsing: 
 DataType invalidtype is not supported.(line 1, pos 2)
 
 == SQL ==
@@ -48,7 +55,7 @@ select from_csv('1', 'a INT', named_struct('mode', 'PERMISSIVE'))
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Must use a map() function for options;; line 1 pos 7
+Must use a map() function for options; line 1 pos 7
 
 
 -- !query
@@ -57,7 +64,7 @@ select from_csv('1', 'a INT', map('mode', 1))
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-A type of keys and values in map() must be string, but got map<string,int>;; line 1 pos 7
+A type of keys and values in map() must be string, but got map<string,int>; line 1 pos 7
 
 
 -- !query
@@ -82,7 +89,7 @@ select schema_of_csv('1|abc', map('delimiter', '|'))
 -- !query schema
 struct<schema_of_csv(1|abc):string>
 -- !query output
-struct<_c0:int,_c1:string>
+STRUCT<`_c0`: INT, `_c1`: STRING>
 
 
 -- !query
@@ -108,7 +115,7 @@ SELECT schema_of_csv(csvField) FROM csvTable
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'schema_of_csv(csvtable.`csvField`)' due to data type mismatch: The input csv should be a foldable string expression and not null; however, got csvtable.`csvField`.; line 1 pos 7
+cannot resolve 'schema_of_csv(csvtable.csvField)' due to data type mismatch: The input csv should be a foldable string expression and not null; however, got csvtable.csvField.; line 1 pos 7
 
 
 -- !query
@@ -141,7 +148,7 @@ select to_csv(named_struct('a', 1, 'b', 2), named_struct('mode', 'PERMISSIVE'))
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Must use a map() function for options;; line 1 pos 7
+Must use a map() function for options; line 1 pos 7
 
 
 -- !query
@@ -150,4 +157,4 @@ select to_csv(named_struct('a', 1, 'b', 2), map('mode', 1))
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-A type of keys and values in map() must be string, but got map<string,int>;; line 1 pos 7
+A type of keys and values in map() must be string, but got map<string,int>; line 1 pos 7
diff --git a/sql/core/src/test/resources/sql-tests/results/cte-nested.sql.out b/sql/core/src/test/resources/sql-tests/results/cte-nested.sql.out
index 2f736c7b4978f..a8db4599dafcc 100644
--- a/sql/core/src/test/resources/sql-tests/results/cte-nested.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/cte-nested.sql.out
@@ -48,7 +48,7 @@ SELECT * FROM t2
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.
 
 
 -- !query
@@ -85,7 +85,7 @@ SELECT * FROM t2
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.
 
 
 -- !query
@@ -139,7 +139,7 @@ SELECT (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.
 
 
 -- !query
@@ -154,7 +154,7 @@ SELECT (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.
 
 
 -- !query
@@ -170,7 +170,7 @@ SELECT (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.
 
 
 -- !query
@@ -184,7 +184,7 @@ WHERE c IN (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.
 
 
 -- !query
@@ -213,7 +213,7 @@ SELECT * FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Name aBc is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
+Name aBc is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.
 
 
 -- !query
@@ -226,4 +226,4 @@ SELECT (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Name aBc is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
+Name aBc is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.
diff --git a/sql/core/src/test/resources/sql-tests/results/current_database_catalog.sql.out b/sql/core/src/test/resources/sql-tests/results/current_database_catalog.sql.out
new file mode 100644
index 0000000000000..b714463a0aa0c
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/current_database_catalog.sql.out
@@ -0,0 +1,10 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 1
+
+
+-- !query
+select current_database(), current_catalog()
+-- !query schema
+struct<current_database():string,current_catalog():string>
+-- !query output
+default	spark_catalog
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-formatting-invalid.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-formatting-invalid.sql.out
new file mode 100644
index 0000000000000..18d1a10068794
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-formatting-invalid.sql.out
@@ -0,0 +1,335 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 37
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'GGGGG')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'GGGGG' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'yyyyyyy')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'qqqqq')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Too many pattern letters: q
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'QQQQQ')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Too many pattern letters: Q
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'MMMMM')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'MMMMM' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'LLLLL')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'LLLLL' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'EEEEE')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'EEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'FF')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'FF' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'ddd')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'ddd' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'DDDD')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'DDDD' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'HHH')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'HHH' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'hhh')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'hhh' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'kkk')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'kkk' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'KKK')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'KKK' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'mmm')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'mmm' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'sss')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'sss' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'SSSSSSSSSS')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'SSSSSSSSSS' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'aa')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'aa' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'V')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Pattern letter count must be 2: V
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'zzzzz')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'zzzzz' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'XXXXXX')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Too many pattern letters: X
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'ZZZZZZ')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'ZZZZZZ' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'OO')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Pattern letter count must be 1 or 4: O
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'xxxxxx')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Too many pattern letters: x
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'A')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character: A
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'n')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character: n
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'N')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character: N
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'p')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character: p
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'Y')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'Y' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'w')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'w' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'W')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'W' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'u')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'u' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'e')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+All week-based patterns are unsupported since Spark 3.0, detected: e, Please use the SQL function EXTRACT instead
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'c')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+All week-based patterns are unsupported since Spark 3.0, detected: c, Please use the SQL function EXTRACT instead
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'B')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Unknown pattern letter: B
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'C')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Unknown pattern letter: C
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'I')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Unknown pattern letter: I
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-formatting-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-formatting-legacy.sql.out
new file mode 100644
index 0000000000000..b37922b20807d
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-formatting-legacy.sql.out
@@ -0,0 +1,401 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 31
+
+
+-- !query
+create temporary view v as select col from values
+ (timestamp '1582-06-01 11:33:33.123UTC+080000'),
+ (timestamp '1970-01-01 00:00:00.000Europe/Paris'),
+ (timestamp '1970-12-31 23:59:59.999Asia/Srednekolymsk'),
+ (timestamp '1996-04-01 00:33:33.123Australia/Darwin'),
+ (timestamp '2018-11-17 13:33:33.123Z'),
+ (timestamp '2020-01-01 01:33:33.123Asia/Shanghai'),
+ (timestamp '2100-01-01 01:33:33.123America/Los_Angeles') t(col)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+select col, date_format(col, 'G GG GGG GGGG') from v
+-- !query schema
+struct<col:timestamp,date_format(col, G GG GGG GGGG):string>
+-- !query output
+1582-05-31 19:40:35.123	AD AD AD AD
+1969-12-31 15:00:00	AD AD AD AD
+1970-12-31 04:59:59.999	AD AD AD AD
+1996-03-31 07:03:33.123	AD AD AD AD
+2018-11-17 05:33:33.123	AD AD AD AD
+2019-12-31 09:33:33.123	AD AD AD AD
+2100-01-01 01:33:33.123	AD AD AD AD
+
+
+-- !query
+select col, date_format(col, 'y yy yyy yyyy yyyyy yyyyyy') from v
+-- !query schema
+struct<col:timestamp,date_format(col, y yy yyy yyyy yyyyy yyyyyy):string>
+-- !query output
+1582-05-31 19:40:35.123	1582 82 1582 1582 01582 001582
+1969-12-31 15:00:00	1969 69 1969 1969 01969 001969
+1970-12-31 04:59:59.999	1970 70 1970 1970 01970 001970
+1996-03-31 07:03:33.123	1996 96 1996 1996 01996 001996
+2018-11-17 05:33:33.123	2018 18 2018 2018 02018 002018
+2019-12-31 09:33:33.123	2019 19 2019 2019 02019 002019
+2100-01-01 01:33:33.123	2100 00 2100 2100 02100 002100
+
+
+-- !query
+select col, date_format(col, 'q qq') from v
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character 'q'
+
+
+-- !query
+select col, date_format(col, 'Q QQ QQQ QQQQ') from v
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character 'Q'
+
+
+-- !query
+select col, date_format(col, 'M MM MMM MMMM') from v
+-- !query schema
+struct<col:timestamp,date_format(col, M MM MMM MMMM):string>
+-- !query output
+1582-05-31 19:40:35.123	5 05 May May
+1969-12-31 15:00:00	12 12 Dec December
+1970-12-31 04:59:59.999	12 12 Dec December
+1996-03-31 07:03:33.123	3 03 Mar March
+2018-11-17 05:33:33.123	11 11 Nov November
+2019-12-31 09:33:33.123	12 12 Dec December
+2100-01-01 01:33:33.123	1 01 Jan January
+
+
+-- !query
+select col, date_format(col, 'L LL') from v
+-- !query schema
+struct<col:timestamp,date_format(col, L LL):string>
+-- !query output
+1582-05-31 19:40:35.123	5 05
+1969-12-31 15:00:00	12 12
+1970-12-31 04:59:59.999	12 12
+1996-03-31 07:03:33.123	3 03
+2018-11-17 05:33:33.123	11 11
+2019-12-31 09:33:33.123	12 12
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'E EE EEE EEEE') from v
+-- !query schema
+struct<col:timestamp,date_format(col, E EE EEE EEEE):string>
+-- !query output
+1582-05-31 19:40:35.123	Thu Thu Thu Thursday
+1969-12-31 15:00:00	Wed Wed Wed Wednesday
+1970-12-31 04:59:59.999	Thu Thu Thu Thursday
+1996-03-31 07:03:33.123	Sun Sun Sun Sunday
+2018-11-17 05:33:33.123	Sat Sat Sat Saturday
+2019-12-31 09:33:33.123	Tue Tue Tue Tuesday
+2100-01-01 01:33:33.123	Fri Fri Fri Friday
+
+
+-- !query
+select col, date_format(col, 'F') from v
+-- !query schema
+struct<col:timestamp,date_format(col, F):string>
+-- !query output
+1582-05-31 19:40:35.123	5
+1969-12-31 15:00:00	5
+1970-12-31 04:59:59.999	5
+1996-03-31 07:03:33.123	5
+2018-11-17 05:33:33.123	3
+2019-12-31 09:33:33.123	5
+2100-01-01 01:33:33.123	1
+
+
+-- !query
+select col, date_format(col, 'd dd') from v
+-- !query schema
+struct<col:timestamp,date_format(col, d dd):string>
+-- !query output
+1582-05-31 19:40:35.123	31 31
+1969-12-31 15:00:00	31 31
+1970-12-31 04:59:59.999	31 31
+1996-03-31 07:03:33.123	31 31
+2018-11-17 05:33:33.123	17 17
+2019-12-31 09:33:33.123	31 31
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'DD') from v where col = timestamp '2100-01-01 01:33:33.123America/Los_Angeles'
+-- !query schema
+struct<col:timestamp,date_format(col, DD):string>
+-- !query output
+2100-01-01 01:33:33.123	01
+
+
+-- !query
+select col, date_format(col, 'D DDD') from v
+-- !query schema
+struct<col:timestamp,date_format(col, D DDD):string>
+-- !query output
+1582-05-31 19:40:35.123	151 151
+1969-12-31 15:00:00	365 365
+1970-12-31 04:59:59.999	365 365
+1996-03-31 07:03:33.123	91 091
+2018-11-17 05:33:33.123	321 321
+2019-12-31 09:33:33.123	365 365
+2100-01-01 01:33:33.123	1 001
+
+
+-- !query
+select col, date_format(col, 'H HH') from v
+-- !query schema
+struct<col:timestamp,date_format(col, H HH):string>
+-- !query output
+1582-05-31 19:40:35.123	19 19
+1969-12-31 15:00:00	15 15
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	7 07
+2018-11-17 05:33:33.123	5 05
+2019-12-31 09:33:33.123	9 09
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'h hh') from v
+-- !query schema
+struct<col:timestamp,date_format(col, h hh):string>
+-- !query output
+1582-05-31 19:40:35.123	7 07
+1969-12-31 15:00:00	3 03
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	7 07
+2018-11-17 05:33:33.123	5 05
+2019-12-31 09:33:33.123	9 09
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'k kk') from v
+-- !query schema
+struct<col:timestamp,date_format(col, k kk):string>
+-- !query output
+1582-05-31 19:40:35.123	19 19
+1969-12-31 15:00:00	15 15
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	7 07
+2018-11-17 05:33:33.123	5 05
+2019-12-31 09:33:33.123	9 09
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'K KK') from v
+-- !query schema
+struct<col:timestamp,date_format(col, K KK):string>
+-- !query output
+1582-05-31 19:40:35.123	7 07
+1969-12-31 15:00:00	3 03
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	7 07
+2018-11-17 05:33:33.123	5 05
+2019-12-31 09:33:33.123	9 09
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'm mm') from v
+-- !query schema
+struct<col:timestamp,date_format(col, m mm):string>
+-- !query output
+1582-05-31 19:40:35.123	40 40
+1969-12-31 15:00:00	0 00
+1970-12-31 04:59:59.999	59 59
+1996-03-31 07:03:33.123	3 03
+2018-11-17 05:33:33.123	33 33
+2019-12-31 09:33:33.123	33 33
+2100-01-01 01:33:33.123	33 33
+
+
+-- !query
+select col, date_format(col, 's ss') from v
+-- !query schema
+struct<col:timestamp,date_format(col, s ss):string>
+-- !query output
+1582-05-31 19:40:35.123	35 35
+1969-12-31 15:00:00	0 00
+1970-12-31 04:59:59.999	59 59
+1996-03-31 07:03:33.123	33 33
+2018-11-17 05:33:33.123	33 33
+2019-12-31 09:33:33.123	33 33
+2100-01-01 01:33:33.123	33 33
+
+
+-- !query
+select col, date_format(col, 'S SS SSS SSSS SSSSS SSSSSS SSSSSSS SSSSSSSS SSSSSSSSS') from v
+-- !query schema
+struct<col:timestamp,date_format(col, S SS SSS SSSS SSSSS SSSSSS SSSSSSS SSSSSSSS SSSSSSSSS):string>
+-- !query output
+1582-05-31 19:40:35.123	123 123 123 0123 00123 000123 0000123 00000123 000000123
+1969-12-31 15:00:00	0 00 000 0000 00000 000000 0000000 00000000 000000000
+1970-12-31 04:59:59.999	999 999 999 0999 00999 000999 0000999 00000999 000000999
+1996-03-31 07:03:33.123	123 123 123 0123 00123 000123 0000123 00000123 000000123
+2018-11-17 05:33:33.123	123 123 123 0123 00123 000123 0000123 00000123 000000123
+2019-12-31 09:33:33.123	123 123 123 0123 00123 000123 0000123 00000123 000000123
+2100-01-01 01:33:33.123	123 123 123 0123 00123 000123 0000123 00000123 000000123
+
+
+-- !query
+select col, date_format(col, 'a') from v
+-- !query schema
+struct<col:timestamp,date_format(col, a):string>
+-- !query output
+1582-05-31 19:40:35.123	PM
+1969-12-31 15:00:00	PM
+1970-12-31 04:59:59.999	AM
+1996-03-31 07:03:33.123	AM
+2018-11-17 05:33:33.123	AM
+2019-12-31 09:33:33.123	AM
+2100-01-01 01:33:33.123	AM
+
+
+-- !query
+select col, date_format(col, 'VV') from v
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character 'V'
+
+
+-- !query
+select col, date_format(col, 'z zz zzz zzzz') from v
+-- !query schema
+struct<col:timestamp,date_format(col, z zz zzz zzzz):string>
+-- !query output
+1582-05-31 19:40:35.123	PST PST PST Pacific Standard Time
+1969-12-31 15:00:00	PST PST PST Pacific Standard Time
+1970-12-31 04:59:59.999	PST PST PST Pacific Standard Time
+1996-03-31 07:03:33.123	PST PST PST Pacific Standard Time
+2018-11-17 05:33:33.123	PST PST PST Pacific Standard Time
+2019-12-31 09:33:33.123	PST PST PST Pacific Standard Time
+2100-01-01 01:33:33.123	PST PST PST Pacific Standard Time
+
+
+-- !query
+select col, date_format(col, 'X XX XXX') from v
+-- !query schema
+struct<col:timestamp,date_format(col, X XX XXX):string>
+-- !query output
+1582-05-31 19:40:35.123	-08 -0800 -08:00
+1969-12-31 15:00:00	-08 -0800 -08:00
+1970-12-31 04:59:59.999	-08 -0800 -08:00
+1996-03-31 07:03:33.123	-08 -0800 -08:00
+2018-11-17 05:33:33.123	-08 -0800 -08:00
+2019-12-31 09:33:33.123	-08 -0800 -08:00
+2100-01-01 01:33:33.123	-08 -0800 -08:00
+
+
+-- !query
+select col, date_format(col, 'XXXX XXXXX') from v
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+invalid ISO 8601 format: length=4
+
+
+-- !query
+select col, date_format(col, 'Z ZZ ZZZ ZZZZ ZZZZZ') from v
+-- !query schema
+struct<col:timestamp,date_format(col, Z ZZ ZZZ ZZZZ ZZZZZ):string>
+-- !query output
+1582-05-31 19:40:35.123	-0800 -0800 -0800 -0800 -0800
+1969-12-31 15:00:00	-0800 -0800 -0800 -0800 -0800
+1970-12-31 04:59:59.999	-0800 -0800 -0800 -0800 -0800
+1996-03-31 07:03:33.123	-0800 -0800 -0800 -0800 -0800
+2018-11-17 05:33:33.123	-0800 -0800 -0800 -0800 -0800
+2019-12-31 09:33:33.123	-0800 -0800 -0800 -0800 -0800
+2100-01-01 01:33:33.123	-0800 -0800 -0800 -0800 -0800
+
+
+-- !query
+select col, date_format(col, 'O OOOO') from v
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character 'O'
+
+
+-- !query
+select col, date_format(col, 'x xx xxx xxxx xxxx xxxxx') from v
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character 'x'
+
+
+-- !query
+select col, date_format(col, '[yyyy-MM-dd HH:mm:ss]') from v
+-- !query schema
+struct<col:timestamp,date_format(col, [yyyy-MM-dd HH:mm:ss]):string>
+-- !query output
+1582-05-31 19:40:35.123	[1582-05-31 19:40:35]
+1969-12-31 15:00:00	[1969-12-31 15:00:00]
+1970-12-31 04:59:59.999	[1970-12-31 04:59:59]
+1996-03-31 07:03:33.123	[1996-03-31 07:03:33]
+2018-11-17 05:33:33.123	[2018-11-17 05:33:33]
+2019-12-31 09:33:33.123	[2019-12-31 09:33:33]
+2100-01-01 01:33:33.123	[2100-01-01 01:33:33]
+
+
+-- !query
+select col, date_format(col, "姚123'GyYqQMLwWuEFDdhHmsSaVzZxXOV'") from v
+-- !query schema
+struct<col:timestamp,date_format(col, 姚123'GyYqQMLwWuEFDdhHmsSaVzZxXOV'):string>
+-- !query output
+1582-05-31 19:40:35.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+1969-12-31 15:00:00	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+1970-12-31 04:59:59.999	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+1996-03-31 07:03:33.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+2018-11-17 05:33:33.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+2019-12-31 09:33:33.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+2100-01-01 01:33:33.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+
+
+-- !query
+select col, date_format(col, "''") from v
+-- !query schema
+struct<col:timestamp,date_format(col, ''):string>
+-- !query output
+1582-05-31 19:40:35.123	'
+1969-12-31 15:00:00	'
+1970-12-31 04:59:59.999	'
+1996-03-31 07:03:33.123	'
+2018-11-17 05:33:33.123	'
+2019-12-31 09:33:33.123	'
+2100-01-01 01:33:33.123	'
+
+
+-- !query
+select col, date_format(col, '') from v
+-- !query schema
+struct<col:timestamp,date_format(col, ):string>
+-- !query output
+1582-05-31 19:40:35.123	
+1969-12-31 15:00:00	
+1970-12-31 04:59:59.999	
+1996-03-31 07:03:33.123	
+2018-11-17 05:33:33.123	
+2019-12-31 09:33:33.123	
+2100-01-01 01:33:33.123
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-formatting.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-formatting.sql.out
new file mode 100644
index 0000000000000..5bed88e168f1e
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-formatting.sql.out
@@ -0,0 +1,431 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 31
+
+
+-- !query
+create temporary view v as select col from values
+ (timestamp '1582-06-01 11:33:33.123UTC+080000'),
+ (timestamp '1970-01-01 00:00:00.000Europe/Paris'),
+ (timestamp '1970-12-31 23:59:59.999Asia/Srednekolymsk'),
+ (timestamp '1996-04-01 00:33:33.123Australia/Darwin'),
+ (timestamp '2018-11-17 13:33:33.123Z'),
+ (timestamp '2020-01-01 01:33:33.123Asia/Shanghai'),
+ (timestamp '2100-01-01 01:33:33.123America/Los_Angeles') t(col)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+select col, date_format(col, 'G GG GGG GGGG') from v
+-- !query schema
+struct<col:timestamp,date_format(col, G GG GGG GGGG):string>
+-- !query output
+1582-05-31 19:40:35.123	AD AD AD Anno Domini
+1969-12-31 15:00:00	AD AD AD Anno Domini
+1970-12-31 04:59:59.999	AD AD AD Anno Domini
+1996-03-31 07:03:33.123	AD AD AD Anno Domini
+2018-11-17 05:33:33.123	AD AD AD Anno Domini
+2019-12-31 09:33:33.123	AD AD AD Anno Domini
+2100-01-01 01:33:33.123	AD AD AD Anno Domini
+
+
+-- !query
+select col, date_format(col, 'y yy yyy yyyy yyyyy yyyyyy') from v
+-- !query schema
+struct<col:timestamp,date_format(col, y yy yyy yyyy yyyyy yyyyyy):string>
+-- !query output
+1582-05-31 19:40:35.123	1582 82 1582 1582 01582 001582
+1969-12-31 15:00:00	1969 69 1969 1969 01969 001969
+1970-12-31 04:59:59.999	1970 70 1970 1970 01970 001970
+1996-03-31 07:03:33.123	1996 96 1996 1996 01996 001996
+2018-11-17 05:33:33.123	2018 18 2018 2018 02018 002018
+2019-12-31 09:33:33.123	2019 19 2019 2019 02019 002019
+2100-01-01 01:33:33.123	2100 00 2100 2100 02100 002100
+
+
+-- !query
+select col, date_format(col, 'q qq') from v
+-- !query schema
+struct<col:timestamp,date_format(col, q qq):string>
+-- !query output
+1582-05-31 19:40:35.123	2 02
+1969-12-31 15:00:00	4 04
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	1 01
+2018-11-17 05:33:33.123	4 04
+2019-12-31 09:33:33.123	4 04
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'Q QQ QQQ QQQQ') from v
+-- !query schema
+struct<col:timestamp,date_format(col, Q QQ QQQ QQQQ):string>
+-- !query output
+1582-05-31 19:40:35.123	2 02 Q2 2nd quarter
+1969-12-31 15:00:00	4 04 Q4 4th quarter
+1970-12-31 04:59:59.999	4 04 Q4 4th quarter
+1996-03-31 07:03:33.123	1 01 Q1 1st quarter
+2018-11-17 05:33:33.123	4 04 Q4 4th quarter
+2019-12-31 09:33:33.123	4 04 Q4 4th quarter
+2100-01-01 01:33:33.123	1 01 Q1 1st quarter
+
+
+-- !query
+select col, date_format(col, 'M MM MMM MMMM') from v
+-- !query schema
+struct<col:timestamp,date_format(col, M MM MMM MMMM):string>
+-- !query output
+1582-05-31 19:40:35.123	5 05 May May
+1969-12-31 15:00:00	12 12 Dec December
+1970-12-31 04:59:59.999	12 12 Dec December
+1996-03-31 07:03:33.123	3 03 Mar March
+2018-11-17 05:33:33.123	11 11 Nov November
+2019-12-31 09:33:33.123	12 12 Dec December
+2100-01-01 01:33:33.123	1 01 Jan January
+
+
+-- !query
+select col, date_format(col, 'L LL') from v
+-- !query schema
+struct<col:timestamp,date_format(col, L LL):string>
+-- !query output
+1582-05-31 19:40:35.123	5 05
+1969-12-31 15:00:00	12 12
+1970-12-31 04:59:59.999	12 12
+1996-03-31 07:03:33.123	3 03
+2018-11-17 05:33:33.123	11 11
+2019-12-31 09:33:33.123	12 12
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'E EE EEE EEEE') from v
+-- !query schema
+struct<col:timestamp,date_format(col, E EE EEE EEEE):string>
+-- !query output
+1582-05-31 19:40:35.123	Mon Mon Mon Monday
+1969-12-31 15:00:00	Wed Wed Wed Wednesday
+1970-12-31 04:59:59.999	Thu Thu Thu Thursday
+1996-03-31 07:03:33.123	Sun Sun Sun Sunday
+2018-11-17 05:33:33.123	Sat Sat Sat Saturday
+2019-12-31 09:33:33.123	Tue Tue Tue Tuesday
+2100-01-01 01:33:33.123	Fri Fri Fri Friday
+
+
+-- !query
+select col, date_format(col, 'F') from v
+-- !query schema
+struct<col:timestamp,date_format(col, F):string>
+-- !query output
+1582-05-31 19:40:35.123	3
+1969-12-31 15:00:00	3
+1970-12-31 04:59:59.999	3
+1996-03-31 07:03:33.123	3
+2018-11-17 05:33:33.123	3
+2019-12-31 09:33:33.123	3
+2100-01-01 01:33:33.123	1
+
+
+-- !query
+select col, date_format(col, 'd dd') from v
+-- !query schema
+struct<col:timestamp,date_format(col, d dd):string>
+-- !query output
+1582-05-31 19:40:35.123	31 31
+1969-12-31 15:00:00	31 31
+1970-12-31 04:59:59.999	31 31
+1996-03-31 07:03:33.123	31 31
+2018-11-17 05:33:33.123	17 17
+2019-12-31 09:33:33.123	31 31
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'DD') from v where col = timestamp '2100-01-01 01:33:33.123America/Los_Angeles'
+-- !query schema
+struct<col:timestamp,date_format(col, DD):string>
+-- !query output
+2100-01-01 01:33:33.123	01
+
+
+-- !query
+select col, date_format(col, 'D DDD') from v
+-- !query schema
+struct<col:timestamp,date_format(col, D DDD):string>
+-- !query output
+1582-05-31 19:40:35.123	151 151
+1969-12-31 15:00:00	365 365
+1970-12-31 04:59:59.999	365 365
+1996-03-31 07:03:33.123	91 091
+2018-11-17 05:33:33.123	321 321
+2019-12-31 09:33:33.123	365 365
+2100-01-01 01:33:33.123	1 001
+
+
+-- !query
+select col, date_format(col, 'H HH') from v
+-- !query schema
+struct<col:timestamp,date_format(col, H HH):string>
+-- !query output
+1582-05-31 19:40:35.123	19 19
+1969-12-31 15:00:00	15 15
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	7 07
+2018-11-17 05:33:33.123	5 05
+2019-12-31 09:33:33.123	9 09
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'h hh') from v
+-- !query schema
+struct<col:timestamp,date_format(col, h hh):string>
+-- !query output
+1582-05-31 19:40:35.123	7 07
+1969-12-31 15:00:00	3 03
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	7 07
+2018-11-17 05:33:33.123	5 05
+2019-12-31 09:33:33.123	9 09
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'k kk') from v
+-- !query schema
+struct<col:timestamp,date_format(col, k kk):string>
+-- !query output
+1582-05-31 19:40:35.123	19 19
+1969-12-31 15:00:00	15 15
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	7 07
+2018-11-17 05:33:33.123	5 05
+2019-12-31 09:33:33.123	9 09
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'K KK') from v
+-- !query schema
+struct<col:timestamp,date_format(col, K KK):string>
+-- !query output
+1582-05-31 19:40:35.123	7 07
+1969-12-31 15:00:00	3 03
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	7 07
+2018-11-17 05:33:33.123	5 05
+2019-12-31 09:33:33.123	9 09
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'm mm') from v
+-- !query schema
+struct<col:timestamp,date_format(col, m mm):string>
+-- !query output
+1582-05-31 19:40:35.123	40 40
+1969-12-31 15:00:00	0 00
+1970-12-31 04:59:59.999	59 59
+1996-03-31 07:03:33.123	3 03
+2018-11-17 05:33:33.123	33 33
+2019-12-31 09:33:33.123	33 33
+2100-01-01 01:33:33.123	33 33
+
+
+-- !query
+select col, date_format(col, 's ss') from v
+-- !query schema
+struct<col:timestamp,date_format(col, s ss):string>
+-- !query output
+1582-05-31 19:40:35.123	35 35
+1969-12-31 15:00:00	0 00
+1970-12-31 04:59:59.999	59 59
+1996-03-31 07:03:33.123	33 33
+2018-11-17 05:33:33.123	33 33
+2019-12-31 09:33:33.123	33 33
+2100-01-01 01:33:33.123	33 33
+
+
+-- !query
+select col, date_format(col, 'S SS SSS SSSS SSSSS SSSSSS SSSSSSS SSSSSSSS SSSSSSSSS') from v
+-- !query schema
+struct<col:timestamp,date_format(col, S SS SSS SSSS SSSSS SSSSSS SSSSSSS SSSSSSSS SSSSSSSSS):string>
+-- !query output
+1582-05-31 19:40:35.123	1 12 123 1230 12300 123000 1230000 12300000 123000000
+1969-12-31 15:00:00	0 00 000 0000 00000 000000 0000000 00000000 000000000
+1970-12-31 04:59:59.999	9 99 999 9990 99900 999000 9990000 99900000 999000000
+1996-03-31 07:03:33.123	1 12 123 1230 12300 123000 1230000 12300000 123000000
+2018-11-17 05:33:33.123	1 12 123 1230 12300 123000 1230000 12300000 123000000
+2019-12-31 09:33:33.123	1 12 123 1230 12300 123000 1230000 12300000 123000000
+2100-01-01 01:33:33.123	1 12 123 1230 12300 123000 1230000 12300000 123000000
+
+
+-- !query
+select col, date_format(col, 'a') from v
+-- !query schema
+struct<col:timestamp,date_format(col, a):string>
+-- !query output
+1582-05-31 19:40:35.123	PM
+1969-12-31 15:00:00	PM
+1970-12-31 04:59:59.999	AM
+1996-03-31 07:03:33.123	AM
+2018-11-17 05:33:33.123	AM
+2019-12-31 09:33:33.123	AM
+2100-01-01 01:33:33.123	AM
+
+
+-- !query
+select col, date_format(col, 'VV') from v
+-- !query schema
+struct<col:timestamp,date_format(col, VV):string>
+-- !query output
+1582-05-31 19:40:35.123	America/Los_Angeles
+1969-12-31 15:00:00	America/Los_Angeles
+1970-12-31 04:59:59.999	America/Los_Angeles
+1996-03-31 07:03:33.123	America/Los_Angeles
+2018-11-17 05:33:33.123	America/Los_Angeles
+2019-12-31 09:33:33.123	America/Los_Angeles
+2100-01-01 01:33:33.123	America/Los_Angeles
+
+
+-- !query
+select col, date_format(col, 'z zz zzz zzzz') from v
+-- !query schema
+struct<col:timestamp,date_format(col, z zz zzz zzzz):string>
+-- !query output
+1582-05-31 19:40:35.123	PST PST PST Pacific Standard Time
+1969-12-31 15:00:00	PST PST PST Pacific Standard Time
+1970-12-31 04:59:59.999	PST PST PST Pacific Standard Time
+1996-03-31 07:03:33.123	PST PST PST Pacific Standard Time
+2018-11-17 05:33:33.123	PST PST PST Pacific Standard Time
+2019-12-31 09:33:33.123	PST PST PST Pacific Standard Time
+2100-01-01 01:33:33.123	PST PST PST Pacific Standard Time
+
+
+-- !query
+select col, date_format(col, 'X XX XXX') from v
+-- !query schema
+struct<col:timestamp,date_format(col, X XX XXX):string>
+-- !query output
+1582-05-31 19:40:35.123	-0752 -0752 -07:52
+1969-12-31 15:00:00	-08 -0800 -08:00
+1970-12-31 04:59:59.999	-08 -0800 -08:00
+1996-03-31 07:03:33.123	-08 -0800 -08:00
+2018-11-17 05:33:33.123	-08 -0800 -08:00
+2019-12-31 09:33:33.123	-08 -0800 -08:00
+2100-01-01 01:33:33.123	-08 -0800 -08:00
+
+
+-- !query
+select col, date_format(col, 'XXXX XXXXX') from v
+-- !query schema
+struct<col:timestamp,date_format(col, XXXX XXXXX):string>
+-- !query output
+1582-05-31 19:40:35.123	-075258 -07:52:58
+1969-12-31 15:00:00	-0800 -08:00
+1970-12-31 04:59:59.999	-0800 -08:00
+1996-03-31 07:03:33.123	-0800 -08:00
+2018-11-17 05:33:33.123	-0800 -08:00
+2019-12-31 09:33:33.123	-0800 -08:00
+2100-01-01 01:33:33.123	-0800 -08:00
+
+
+-- !query
+select col, date_format(col, 'Z ZZ ZZZ ZZZZ ZZZZZ') from v
+-- !query schema
+struct<col:timestamp,date_format(col, Z ZZ ZZZ ZZZZ ZZZZZ):string>
+-- !query output
+1582-05-31 19:40:35.123	-0752 -0752 -0752 GMT-07:52:58 -07:52:58
+1969-12-31 15:00:00	-0800 -0800 -0800 GMT-08:00 -08:00
+1970-12-31 04:59:59.999	-0800 -0800 -0800 GMT-08:00 -08:00
+1996-03-31 07:03:33.123	-0800 -0800 -0800 GMT-08:00 -08:00
+2018-11-17 05:33:33.123	-0800 -0800 -0800 GMT-08:00 -08:00
+2019-12-31 09:33:33.123	-0800 -0800 -0800 GMT-08:00 -08:00
+2100-01-01 01:33:33.123	-0800 -0800 -0800 GMT-08:00 -08:00
+
+
+-- !query
+select col, date_format(col, 'O OOOO') from v
+-- !query schema
+struct<col:timestamp,date_format(col, O OOOO):string>
+-- !query output
+1582-05-31 19:40:35.123	GMT-7:52:58 GMT-07:52:58
+1969-12-31 15:00:00	GMT-8 GMT-08:00
+1970-12-31 04:59:59.999	GMT-8 GMT-08:00
+1996-03-31 07:03:33.123	GMT-8 GMT-08:00
+2018-11-17 05:33:33.123	GMT-8 GMT-08:00
+2019-12-31 09:33:33.123	GMT-8 GMT-08:00
+2100-01-01 01:33:33.123	GMT-8 GMT-08:00
+
+
+-- !query
+select col, date_format(col, 'x xx xxx xxxx xxxx xxxxx') from v
+-- !query schema
+struct<col:timestamp,date_format(col, x xx xxx xxxx xxxx xxxxx):string>
+-- !query output
+1582-05-31 19:40:35.123	-0752 -0752 -07:52 -075258 -075258 -07:52:58
+1969-12-31 15:00:00	-08 -0800 -08:00 -0800 -0800 -08:00
+1970-12-31 04:59:59.999	-08 -0800 -08:00 -0800 -0800 -08:00
+1996-03-31 07:03:33.123	-08 -0800 -08:00 -0800 -0800 -08:00
+2018-11-17 05:33:33.123	-08 -0800 -08:00 -0800 -0800 -08:00
+2019-12-31 09:33:33.123	-08 -0800 -08:00 -0800 -0800 -08:00
+2100-01-01 01:33:33.123	-08 -0800 -08:00 -0800 -0800 -08:00
+
+
+-- !query
+select col, date_format(col, '[yyyy-MM-dd HH:mm:ss]') from v
+-- !query schema
+struct<col:timestamp,date_format(col, [yyyy-MM-dd HH:mm:ss]):string>
+-- !query output
+1582-05-31 19:40:35.123	1582-05-31 19:40:35
+1969-12-31 15:00:00	1969-12-31 15:00:00
+1970-12-31 04:59:59.999	1970-12-31 04:59:59
+1996-03-31 07:03:33.123	1996-03-31 07:03:33
+2018-11-17 05:33:33.123	2018-11-17 05:33:33
+2019-12-31 09:33:33.123	2019-12-31 09:33:33
+2100-01-01 01:33:33.123	2100-01-01 01:33:33
+
+
+-- !query
+select col, date_format(col, "姚123'GyYqQMLwWuEFDdhHmsSaVzZxXOV'") from v
+-- !query schema
+struct<col:timestamp,date_format(col, 姚123'GyYqQMLwWuEFDdhHmsSaVzZxXOV'):string>
+-- !query output
+1582-05-31 19:40:35.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+1969-12-31 15:00:00	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+1970-12-31 04:59:59.999	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+1996-03-31 07:03:33.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+2018-11-17 05:33:33.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+2019-12-31 09:33:33.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+2100-01-01 01:33:33.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+
+
+-- !query
+select col, date_format(col, "''") from v
+-- !query schema
+struct<col:timestamp,date_format(col, ''):string>
+-- !query output
+1582-05-31 19:40:35.123	'
+1969-12-31 15:00:00	'
+1970-12-31 04:59:59.999	'
+1996-03-31 07:03:33.123	'
+2018-11-17 05:33:33.123	'
+2019-12-31 09:33:33.123	'
+2100-01-01 01:33:33.123	'
+
+
+-- !query
+select col, date_format(col, '') from v
+-- !query schema
+struct<col:timestamp,date_format(col, ):string>
+-- !query output
+1582-05-31 19:40:35.123	
+1969-12-31 15:00:00	
+1970-12-31 04:59:59.999	
+1996-03-31 07:03:33.123	
+2018-11-17 05:33:33.123	
+2019-12-31 09:33:33.123	
+2100-01-01 01:33:33.123
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
new file mode 100644
index 0000000000000..aaa1c469147b4
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
@@ -0,0 +1,1063 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 128
+
+
+-- !query
+select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null)
+-- !query schema
+struct<timestamp_seconds(1230219000):timestamp,timestamp_seconds(-1230219000):timestamp,timestamp_seconds(NULL):timestamp>
+-- !query output
+2008-12-25 07:30:00	1931-01-07 00:30:00	NULL
+
+
+-- !query
+select TIMESTAMP_SECONDS(1.23), TIMESTAMP_SECONDS(1.23d), TIMESTAMP_SECONDS(FLOAT(1.23))
+-- !query schema
+struct<timestamp_seconds(1.23):timestamp,timestamp_seconds(1.23):timestamp,timestamp_seconds(1.23):timestamp>
+-- !query output
+1969-12-31 16:00:01.23	1969-12-31 16:00:01.23	1969-12-31 16:00:01.23
+
+
+-- !query
+select TIMESTAMP_MILLIS(1230219000123),TIMESTAMP_MILLIS(-1230219000123),TIMESTAMP_MILLIS(null)
+-- !query schema
+struct<timestamp_millis(1230219000123):timestamp,timestamp_millis(-1230219000123):timestamp,timestamp_millis(NULL):timestamp>
+-- !query output
+2008-12-25 07:30:00.123	1931-01-07 00:29:59.877	NULL
+
+
+-- !query
+select TIMESTAMP_MICROS(1230219000123123),TIMESTAMP_MICROS(-1230219000123123),TIMESTAMP_MICROS(null)
+-- !query schema
+struct<timestamp_micros(1230219000123123):timestamp,timestamp_micros(-1230219000123123):timestamp,timestamp_micros(NULL):timestamp>
+-- !query output
+2008-12-25 07:30:00.123123	1931-01-07 00:29:59.876877	NULL
+
+
+-- !query
+select TIMESTAMP_SECONDS(1230219000123123)
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+long overflow
+
+
+-- !query
+select TIMESTAMP_SECONDS(-1230219000123123)
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+long overflow
+
+
+-- !query
+select TIMESTAMP_MILLIS(92233720368547758)
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+long overflow
+
+
+-- !query
+select TIMESTAMP_MILLIS(-92233720368547758)
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+long overflow
+
+
+-- !query
+select TIMESTAMP_SECONDS(0.1234567)
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+Rounding necessary
+
+
+-- !query
+select TIMESTAMP_SECONDS(0.1234567d), TIMESTAMP_SECONDS(FLOAT(0.1234567))
+-- !query schema
+struct<timestamp_seconds(0.1234567):timestamp,timestamp_seconds(0.1234567):timestamp>
+-- !query output
+1969-12-31 16:00:00.123456	1969-12-31 16:00:00.123456
+
+
+-- !query
+select UNIX_SECONDS(TIMESTAMP('2020-12-01 14:30:08Z')), UNIX_SECONDS(TIMESTAMP('2020-12-01 14:30:08.999999Z')), UNIX_SECONDS(null)
+-- !query schema
+struct<unix_seconds(2020-12-01 14:30:08Z):bigint,unix_seconds(2020-12-01 14:30:08.999999Z):bigint,unix_seconds(NULL):bigint>
+-- !query output
+1606833008	1606833008	NULL
+
+
+-- !query
+select UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08Z')), UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08.999999Z')), UNIX_MILLIS(null)
+-- !query schema
+struct<unix_millis(2020-12-01 14:30:08Z):bigint,unix_millis(2020-12-01 14:30:08.999999Z):bigint,unix_millis(NULL):bigint>
+-- !query output
+1606833008000	1606833008999	NULL
+
+
+-- !query
+select UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08Z')), UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08.999999Z')), UNIX_MICROS(null)
+-- !query schema
+struct<unix_micros(2020-12-01 14:30:08Z):bigint,unix_micros(2020-12-01 14:30:08.999999Z):bigint,unix_micros(NULL):bigint>
+-- !query output
+1606833008000000	1606833008999999	NULL
+
+
+-- !query
+select DATE_FROM_UNIX_DATE(0), DATE_FROM_UNIX_DATE(1000), DATE_FROM_UNIX_DATE(null)
+-- !query schema
+struct<date_from_unix_date(0):date,date_from_unix_date(1000):date,date_from_unix_date(NULL):date>
+-- !query output
+1970-01-01	1972-09-27	NULL
+
+
+-- !query
+select UNIX_DATE(DATE('1970-01-01')), UNIX_DATE(DATE('2020-12-04')), UNIX_DATE(null)
+-- !query schema
+struct<unix_date(1970-01-01):int,unix_date(2020-12-04):int,unix_date(NULL):int>
+-- !query output
+0	18600	NULL
+
+
+-- !query
+select current_date = current_date(), current_timestamp = current_timestamp()
+-- !query schema
+struct<(current_date() = current_date()):boolean,(current_timestamp() = current_timestamp()):boolean>
+-- !query output
+true	true
+
+
+-- !query
+select to_date(null), to_date('2016-12-31'), to_date('2016-12-31', 'yyyy-MM-dd')
+-- !query schema
+struct<to_date(NULL):date,to_date(2016-12-31):date,to_date(2016-12-31, yyyy-MM-dd):date>
+-- !query output
+NULL	2016-12-31	2016-12-31
+
+
+-- !query
+select to_timestamp(null), to_timestamp('2016-12-31 00:12:00'), to_timestamp('2016-12-31', 'yyyy-MM-dd')
+-- !query schema
+struct<to_timestamp(NULL):timestamp,to_timestamp(2016-12-31 00:12:00):timestamp,to_timestamp(2016-12-31, yyyy-MM-dd):timestamp>
+-- !query output
+NULL	2016-12-31 00:12:00	2016-12-31 00:00:00
+
+
+-- !query
+select dayofweek('2007-02-03'), dayofweek('2009-07-30'), dayofweek('2017-05-27'), dayofweek(null), dayofweek('1582-10-15 13:10:15')
+-- !query schema
+struct<dayofweek(2007-02-03):int,dayofweek(2009-07-30):int,dayofweek(2017-05-27):int,dayofweek(NULL):int,dayofweek(1582-10-15 13:10:15):int>
+-- !query output
+7	5	7	NULL	6
+
+
+-- !query
+create temporary view ttf1 as select * from values
+  (1, 2),
+  (2, 3)
+  as ttf1(current_date, current_timestamp)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+select current_date, current_timestamp from ttf1
+-- !query schema
+struct<current_date:int,current_timestamp:int>
+-- !query output
+1	2
+2	3
+
+
+-- !query
+create temporary view ttf2 as select * from values
+  (1, 2),
+  (2, 3)
+  as ttf2(a, b)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+select current_date = current_date(), current_timestamp = current_timestamp(), a, b from ttf2
+-- !query schema
+struct<(current_date() = current_date()):boolean,(current_timestamp() = current_timestamp()):boolean,a:int,b:int>
+-- !query output
+true	true	1	2
+true	true	2	3
+
+
+-- !query
+select a, b from ttf2 order by a, current_date
+-- !query schema
+struct<a:int,b:int>
+-- !query output
+1	2
+2	3
+
+
+-- !query
+select weekday('2007-02-03'), weekday('2009-07-30'), weekday('2017-05-27'), weekday(null), weekday('1582-10-15 13:10:15')
+-- !query schema
+struct<weekday(2007-02-03):int,weekday(2009-07-30):int,weekday(2017-05-27):int,weekday(NULL):int,weekday(1582-10-15 13:10:15):int>
+-- !query output
+5	3	5	NULL	4
+
+
+-- !query
+select year('1500-01-01'), month('1500-01-01'), dayOfYear('1500-01-01')
+-- !query schema
+struct<year(1500-01-01):int,month(1500-01-01):int,dayofyear(1500-01-01):int>
+-- !query output
+1500	1	1
+
+
+-- !query
+select date '2019-01-01\t'
+-- !query schema
+struct<DATE '2019-01-01':date>
+-- !query output
+2019-01-01
+
+
+-- !query
+select timestamp '2019-01-01\t'
+-- !query schema
+struct<TIMESTAMP '2019-01-01 00:00:00':timestamp>
+-- !query output
+2019-01-01 00:00:00
+
+
+-- !query
+select date '2020-01-01中文'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the DATE value: 2020-01-01中文(line 1, pos 7)
+
+== SQL ==
+select date '2020-01-01中文'
+-------^^^
+
+
+-- !query
+select timestamp '2019-01-01中文'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the TIMESTAMP value: 2019-01-01中文(line 1, pos 7)
+
+== SQL ==
+select timestamp '2019-01-01中文'
+-------^^^
+
+
+-- !query
+select timestamp'2011-11-11 11:11:11' + interval '2' day
+-- !query schema
+struct<TIMESTAMP '2011-11-11 11:11:11' + INTERVAL '2 days':timestamp>
+-- !query output
+2011-11-13 11:11:11
+
+
+-- !query
+select timestamp'2011-11-11 11:11:11' - interval '2' day
+-- !query schema
+struct<TIMESTAMP '2011-11-11 11:11:11' - INTERVAL '2 days':timestamp>
+-- !query output
+2011-11-09 11:11:11
+
+
+-- !query
+select date'2011-11-11 11:11:11' + interval '2' second
+-- !query schema
+struct<DATE '2011-11-11' + INTERVAL '2 seconds':date>
+-- !query output
+2011-11-11
+
+
+-- !query
+select date'2011-11-11 11:11:11' - interval '2' second
+-- !query schema
+struct<DATE '2011-11-11' - INTERVAL '2 seconds':date>
+-- !query output
+2011-11-10
+
+
+-- !query
+select '2011-11-11' - interval '2' day
+-- !query schema
+struct<2011-11-11 - INTERVAL '2 days':string>
+-- !query output
+2011-11-09 00:00:00
+
+
+-- !query
+select '2011-11-11 11:11:11' - interval '2' second
+-- !query schema
+struct<2011-11-11 11:11:11 - INTERVAL '2 seconds':string>
+-- !query output
+2011-11-11 11:11:09
+
+
+-- !query
+select '1' - interval '2' second
+-- !query schema
+struct<1 - INTERVAL '2 seconds':string>
+-- !query output
+NULL
+
+
+-- !query
+select 1 - interval '2' second
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve '1 + (- INTERVAL '2 seconds')' due to data type mismatch: argument 1 requires timestamp type, however, '1' is of int type.; line 1 pos 7
+
+
+-- !query
+select date'2020-01-01' - timestamp'2019-10-06 10:11:12.345678'
+-- !query schema
+struct<(DATE '2020-01-01' - TIMESTAMP '2019-10-06 10:11:12.345678'):interval day to second>
+-- !query output
+86 13:48:47.654322000
+
+
+-- !query
+select timestamp'2019-10-06 10:11:12.345678' - date'2020-01-01'
+-- !query schema
+struct<(TIMESTAMP '2019-10-06 10:11:12.345678' - DATE '2020-01-01'):interval day to second>
+-- !query output
+-86 13:48:47.654322000
+
+
+-- !query
+select timestamp'2019-10-06 10:11:12.345678' - null
+-- !query schema
+struct<(TIMESTAMP '2019-10-06 10:11:12.345678' - NULL):interval day to second>
+-- !query output
+NULL
+
+
+-- !query
+select null - timestamp'2019-10-06 10:11:12.345678'
+-- !query schema
+struct<(NULL - TIMESTAMP '2019-10-06 10:11:12.345678'):interval day to second>
+-- !query output
+NULL
+
+
+-- !query
+select date_add('2011-11-11', 1Y)
+-- !query schema
+struct<date_add(2011-11-11, 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add('2011-11-11', 1S)
+-- !query schema
+struct<date_add(2011-11-11, 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add('2011-11-11', 1)
+-- !query schema
+struct<date_add(2011-11-11, 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add('2011-11-11', 1L)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(CAST('2011-11-11' AS DATE), 1L)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, '1L' is of bigint type.; line 1 pos 7
+
+
+-- !query
+select date_add('2011-11-11', 1.0)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(CAST('2011-11-11' AS DATE), 1.0BD)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, '1.0BD' is of decimal(2,1) type.; line 1 pos 7
+
+
+-- !query
+select date_add('2011-11-11', 1E1)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(CAST('2011-11-11' AS DATE), 10.0D)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, '10.0D' is of double type.; line 1 pos 7
+
+
+-- !query
+select date_add('2011-11-11', '1')
+-- !query schema
+struct<date_add(2011-11-11, 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add('2011-11-11', '1.2')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+The second argument of 'date_add' function needs to be an integer.
+
+
+-- !query
+select date_add(date'2011-11-11', 1)
+-- !query schema
+struct<date_add(DATE '2011-11-11', 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add(timestamp'2011-11-11', 1)
+-- !query schema
+struct<date_add(TIMESTAMP '2011-11-11 00:00:00', 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_sub(date'2011-11-11', 1)
+-- !query schema
+struct<date_sub(DATE '2011-11-11', 1):date>
+-- !query output
+2011-11-10
+
+
+-- !query
+select date_sub(date'2011-11-11', '1')
+-- !query schema
+struct<date_sub(DATE '2011-11-11', 1):date>
+-- !query output
+2011-11-10
+
+
+-- !query
+select date_sub(date'2011-11-11', '1.2')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+The second argument of 'date_sub' function needs to be an integer.
+
+
+-- !query
+select date_sub(timestamp'2011-11-11', 1)
+-- !query schema
+struct<date_sub(TIMESTAMP '2011-11-11 00:00:00', 1):date>
+-- !query output
+2011-11-10
+
+
+-- !query
+select date_sub(null, 1)
+-- !query schema
+struct<date_sub(NULL, 1):date>
+-- !query output
+NULL
+
+
+-- !query
+select date_sub(date'2011-11-11', null)
+-- !query schema
+struct<date_sub(DATE '2011-11-11', NULL):date>
+-- !query output
+NULL
+
+
+-- !query
+select date'2011-11-11' + 1E1
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(DATE '2011-11-11', 10.0D)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, '10.0D' is of double type.; line 1 pos 7
+
+
+-- !query
+select date'2011-11-11' + '1'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(DATE '2011-11-11', CAST('1' AS DOUBLE))' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'CAST('1' AS DOUBLE)' is of double type.; line 1 pos 7
+
+
+-- !query
+select null + date '2001-09-28'
+-- !query schema
+struct<date_add(DATE '2001-09-28', NULL):date>
+-- !query output
+NULL
+
+
+-- !query
+select date '2001-09-28' + 7Y
+-- !query schema
+struct<date_add(DATE '2001-09-28', 7):date>
+-- !query output
+2001-10-05
+
+
+-- !query
+select 7S + date '2001-09-28'
+-- !query schema
+struct<date_add(DATE '2001-09-28', 7):date>
+-- !query output
+2001-10-05
+
+
+-- !query
+select date '2001-10-01' - 7
+-- !query schema
+struct<date_sub(DATE '2001-10-01', 7):date>
+-- !query output
+2001-09-24
+
+
+-- !query
+select date '2001-10-01' - '7'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_sub(DATE '2001-10-01', CAST('7' AS DOUBLE))' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'CAST('7' AS DOUBLE)' is of double type.; line 1 pos 7
+
+
+-- !query
+select date '2001-09-28' + null
+-- !query schema
+struct<date_add(DATE '2001-09-28', NULL):date>
+-- !query output
+NULL
+
+
+-- !query
+select date '2001-09-28' - null
+-- !query schema
+struct<date_sub(DATE '2001-09-28', NULL):date>
+-- !query output
+NULL
+
+
+-- !query
+create temp view v as select '1' str
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+select date_add('2011-11-11', str) from v
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(CAST('2011-11-11' AS DATE), v.str)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'v.str' is of string type.; line 1 pos 7
+
+
+-- !query
+select date_sub('2011-11-11', str) from v
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_sub(CAST('2011-11-11' AS DATE), v.str)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'v.str' is of string type.; line 1 pos 7
+
+
+-- !query
+select null - date '2019-10-06'
+-- !query schema
+struct<(NULL - DATE '2019-10-06'):interval day to second>
+-- !query output
+NULL
+
+
+-- !query
+select date '2001-10-01' - date '2001-09-28'
+-- !query schema
+struct<(DATE '2001-10-01' - DATE '2001-09-28'):interval day to second>
+-- !query output
+3 00:00:00.000000000
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp(2019-10-06 10:11:12., yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.0', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp(2019-10-06 10:11:12.0, yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp(2019-10-06 10:11:12.1, yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp(2019-10-06 10:11:12.12, yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.123UTC', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp(2019-10-06 10:11:12.123UTC, yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp(2019-10-06 10:11:12.1234, yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.12345CST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp(2019-10-06 10:11:12.12345CST, yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.123456PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp(2019-10-06 10:11:12.123456PST, yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1234567PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp(2019-10-06 10:11:12.1234567PST, yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('123456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp(123456 2019-10-06 10:11:12.123456PST, SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('223456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp(223456 2019-10-06 10:11:12.123456PST, SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.[SSSSSS]')
+-- !query schema
+struct<to_timestamp(2019-10-06 10:11:12.1234, yyyy-MM-dd HH:mm:ss.[SSSSSS]):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.123', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]')
+-- !query schema
+struct<to_timestamp(2019-10-06 10:11:12.123, yyyy-MM-dd HH:mm:ss[.SSSSSS]):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]')
+-- !query schema
+struct<to_timestamp(2019-10-06 10:11:12, yyyy-MM-dd HH:mm:ss[.SSSSSS]):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]')
+-- !query schema
+struct<to_timestamp(2019-10-06 10:11:12.12, yyyy-MM-dd HH:mm[:ss.SSSSSS]):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]')
+-- !query schema
+struct<to_timestamp(2019-10-06 10:11, yyyy-MM-dd HH:mm[:ss.SSSSSS]):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("2019-10-06S10:11:12.12345", "yyyy-MM-dd'S'HH:mm:ss.SSSSSS")
+-- !query schema
+struct<to_timestamp(2019-10-06S10:11:12.12345, yyyy-MM-dd'S'HH:mm:ss.SSSSSS):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("12.12342019-10-06S10:11", "ss.SSSSyyyy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp(12.12342019-10-06S10:11, ss.SSSSyyyy-MM-dd'S'HH:mm):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("12.1232019-10-06S10:11", "ss.SSSSyyyy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp(12.1232019-10-06S10:11, ss.SSSSyyyy-MM-dd'S'HH:mm):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("12.1232019-10-06S10:11", "ss.SSSSyy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp(12.1232019-10-06S10:11, ss.SSSSyy-MM-dd'S'HH:mm):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("12.1234019-10-06S10:11", "ss.SSSSy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp(12.1234019-10-06S10:11, ss.SSSSy-MM-dd'S'HH:mm):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("2019-10-06S", "yyyy-MM-dd'S'")
+-- !query schema
+struct<to_timestamp(2019-10-06S, yyyy-MM-dd'S'):timestamp>
+-- !query output
+2019-10-06 00:00:00
+
+
+-- !query
+select to_timestamp("S2019-10-06", "'S'yyyy-MM-dd")
+-- !query schema
+struct<to_timestamp(S2019-10-06, 'S'yyyy-MM-dd):timestamp>
+-- !query output
+2019-10-06 00:00:00
+
+
+-- !query
+select to_timestamp("2019-10-06T10:11:12'12", "yyyy-MM-dd'T'HH:mm:ss''SSSS")
+-- !query schema
+struct<to_timestamp(2019-10-06T10:11:12'12, yyyy-MM-dd'T'HH:mm:ss''SSSS):timestamp>
+-- !query output
+2019-10-06 10:11:12.012
+
+
+-- !query
+select to_timestamp("2019-10-06T10:11:12'", "yyyy-MM-dd'T'HH:mm:ss''")
+-- !query schema
+struct<to_timestamp(2019-10-06T10:11:12', yyyy-MM-dd'T'HH:mm:ss''):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("'2019-10-06T10:11:12", "''yyyy-MM-dd'T'HH:mm:ss")
+-- !query schema
+struct<to_timestamp('2019-10-06T10:11:12, ''yyyy-MM-dd'T'HH:mm:ss):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("P2019-10-06T10:11:12", "'P'yyyy-MM-dd'T'HH:mm:ss")
+-- !query schema
+struct<to_timestamp(P2019-10-06T10:11:12, 'P'yyyy-MM-dd'T'HH:mm:ss):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("16", "dd")
+-- !query schema
+struct<to_timestamp(16, dd):timestamp>
+-- !query output
+1970-01-16 00:00:00
+
+
+-- !query
+select to_timestamp("02-29", "MM-dd")
+-- !query schema
+struct<to_timestamp(02-29, MM-dd):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_date("16", "dd")
+-- !query schema
+struct<to_date(16, dd):date>
+-- !query output
+1970-01-16
+
+
+-- !query
+select to_date("02-29", "MM-dd")
+-- !query schema
+struct<to_date(02-29, MM-dd):date>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("2019 40", "yyyy mm")
+-- !query schema
+struct<to_timestamp(2019 40, yyyy mm):timestamp>
+-- !query output
+2019-01-01 00:40:00
+
+
+-- !query
+select to_timestamp("2019 10:10:10", "yyyy hh:mm:ss")
+-- !query schema
+struct<to_timestamp(2019 10:10:10, yyyy hh:mm:ss):timestamp>
+-- !query output
+2019-01-01 10:10:10
+
+
+-- !query
+select to_timestamp('2019-10-06 A', 'yyyy-MM-dd GGGGG')
+-- !query schema
+struct<to_timestamp(2019-10-06 A, yyyy-MM-dd GGGGG):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEEE')
+-- !query schema
+struct<to_timestamp(22 05 2020 Friday, dd MM yyyy EEEEEE):timestamp>
+-- !query output
+2020-05-22 00:00:00
+
+
+-- !query
+select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE')
+-- !query schema
+struct<to_timestamp(22 05 2020 Friday, dd MM yyyy EEEEE):timestamp>
+-- !query output
+2020-05-22 00:00:00
+
+
+-- !query
+select unix_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE')
+-- !query schema
+struct<unix_timestamp(22 05 2020 Friday, dd MM yyyy EEEEE):bigint>
+-- !query output
+1590130800
+
+
+-- !query
+select from_json('{"t":"26/October/2015"}', 't Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<from_json({"t":"26/October/2015"}):struct<t:timestamp>>
+-- !query output
+{"t":2015-10-26 00:00:00}
+
+
+-- !query
+select from_json('{"d":"26/October/2015"}', 'd Date', map('dateFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<from_json({"d":"26/October/2015"}):struct<d:date>>
+-- !query output
+{"d":2015-10-26}
+
+
+-- !query
+select from_csv('26/October/2015', 't Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<from_csv(26/October/2015):struct<t:timestamp>>
+-- !query output
+{"t":2015-10-26 00:00:00}
+
+
+-- !query
+select from_csv('26/October/2015', 'd Date', map('dateFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<from_csv(26/October/2015):struct<d:date>>
+-- !query output
+{"d":2015-10-26}
+
+
+-- !query
+select to_date("2020-01-27T20:06:11.847", "yyyy-MM-dd HH:mm:ss.SSS")
+-- !query schema
+struct<to_date(2020-01-27T20:06:11.847, yyyy-MM-dd HH:mm:ss.SSS):date>
+-- !query output
+NULL
+
+
+-- !query
+select to_date("Unparseable", "yyyy-MM-dd HH:mm:ss.SSS")
+-- !query schema
+struct<to_date(Unparseable, yyyy-MM-dd HH:mm:ss.SSS):date>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("2020-01-27T20:06:11.847", "yyyy-MM-dd HH:mm:ss.SSS")
+-- !query schema
+struct<to_timestamp(2020-01-27T20:06:11.847, yyyy-MM-dd HH:mm:ss.SSS):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("Unparseable", "yyyy-MM-dd HH:mm:ss.SSS")
+-- !query schema
+struct<to_timestamp(Unparseable, yyyy-MM-dd HH:mm:ss.SSS):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select unix_timestamp("2020-01-27T20:06:11.847", "yyyy-MM-dd HH:mm:ss.SSS")
+-- !query schema
+struct<unix_timestamp(2020-01-27T20:06:11.847, yyyy-MM-dd HH:mm:ss.SSS):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select unix_timestamp("Unparseable", "yyyy-MM-dd HH:mm:ss.SSS")
+-- !query schema
+struct<unix_timestamp(Unparseable, yyyy-MM-dd HH:mm:ss.SSS):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select to_unix_timestamp("2020-01-27T20:06:11.847", "yyyy-MM-dd HH:mm:ss.SSS")
+-- !query schema
+struct<to_unix_timestamp(2020-01-27T20:06:11.847, yyyy-MM-dd HH:mm:ss.SSS):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select to_unix_timestamp("Unparseable", "yyyy-MM-dd HH:mm:ss.SSS")
+-- !query schema
+struct<to_unix_timestamp(Unparseable, yyyy-MM-dd HH:mm:ss.SSS):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select cast("Unparseable" as timestamp)
+-- !query schema
+struct<CAST(Unparseable AS TIMESTAMP):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select cast("Unparseable" as date)
+-- !query schema
+struct<CAST(Unparseable AS DATE):date>
+-- !query output
+NULL
+
+
+-- !query
+select next_day("2015-07-23", "Mon")
+-- !query schema
+struct<next_day(2015-07-23, Mon):date>
+-- !query output
+2015-07-27
+
+
+-- !query
+select next_day("2015-07-23", "xx")
+-- !query schema
+struct<next_day(2015-07-23, xx):date>
+-- !query output
+NULL
+
+
+-- !query
+select next_day("xx", "Mon")
+-- !query schema
+struct<next_day(xx, Mon):date>
+-- !query output
+NULL
+
+
+-- !query
+select next_day(null, "Mon")
+-- !query schema
+struct<next_day(NULL, Mon):date>
+-- !query output
+NULL
+
+
+-- !query
+select next_day(null, "xx")
+-- !query schema
+struct<next_day(NULL, xx):date>
+-- !query output
+NULL
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-parsing-invalid.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-parsing-invalid.sql.out
new file mode 100644
index 0000000000000..c11f5f39bebd0
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-parsing-invalid.sql.out
@@ -0,0 +1,163 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 19
+
+
+-- !query
+select to_timestamp('294248', 'y')
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+long overflow
+
+
+-- !query
+select to_timestamp('1', 'yy')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '1' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
+-- !query
+select to_timestamp('-12', 'yy')
+-- !query schema
+struct<to_timestamp(-12, yy):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('123', 'yy')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '123' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
+-- !query
+select to_timestamp('1', 'yyy')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '1' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
+-- !query
+select to_timestamp('1234567', 'yyyyyyy')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select to_timestamp('366', 'D')
+-- !query schema
+struct<to_timestamp(366, D):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('9', 'DD')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '9' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
+-- !query
+select to_timestamp('366', 'DD')
+-- !query schema
+struct<to_timestamp(366, DD):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('9', 'DDD')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '9' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
+-- !query
+select to_timestamp('99', 'DDD')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '99' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
+-- !query
+select to_timestamp('30-365', 'dd-DDD')
+-- !query schema
+struct<to_timestamp(30-365, dd-DDD):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('11-365', 'MM-DDD')
+-- !query schema
+struct<to_timestamp(11-365, MM-DDD):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-366', 'yyyy-DDD')
+-- !query schema
+struct<to_timestamp(2019-366, yyyy-DDD):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('12-30-365', 'MM-dd-DDD')
+-- !query schema
+struct<to_timestamp(12-30-365, MM-dd-DDD):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2020-01-365', 'yyyy-dd-DDD')
+-- !query schema
+struct<to_timestamp(2020-01-365, yyyy-dd-DDD):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2020-10-350', 'yyyy-MM-DDD')
+-- !query schema
+struct<to_timestamp(2020-10-350, yyyy-MM-DDD):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2020-11-31-366', 'yyyy-MM-dd-DDD')
+-- !query schema
+struct<to_timestamp(2020-11-31-366, yyyy-MM-dd-DDD):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select from_csv('2018-366', 'date Date', map('dateFormat', 'yyyy-DDD'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '2018-366' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-parsing-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-parsing-legacy.sql.out
new file mode 100644
index 0000000000000..bb7ce74a29ef5
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-parsing-legacy.sql.out
@@ -0,0 +1,202 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 25
+
+
+-- !query
+select to_timestamp('1', 'y')
+-- !query schema
+struct<to_timestamp(1, y):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('009999', 'y')
+-- !query schema
+struct<to_timestamp(009999, y):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('00', 'yy')
+-- !query schema
+struct<to_timestamp(00, yy):timestamp>
+-- !query output
+2000-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('99', 'yy')
+-- !query schema
+struct<to_timestamp(99, yy):timestamp>
+-- !query output
+1999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('001', 'yyy')
+-- !query schema
+struct<to_timestamp(001, yyy):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('009999', 'yyy')
+-- !query schema
+struct<to_timestamp(009999, yyy):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('0001', 'yyyy')
+-- !query schema
+struct<to_timestamp(0001, yyyy):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('9999', 'yyyy')
+-- !query schema
+struct<to_timestamp(9999, yyyy):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('00001', 'yyyyy')
+-- !query schema
+struct<to_timestamp(00001, yyyyy):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('09999', 'yyyyy')
+-- !query schema
+struct<to_timestamp(09999, yyyyy):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('000001', 'yyyyyy')
+-- !query schema
+struct<to_timestamp(000001, yyyyyy):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('009999', 'yyyyyy')
+-- !query schema
+struct<to_timestamp(009999, yyyyyy):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('9', 'D')
+-- !query schema
+struct<to_timestamp(9, D):timestamp>
+-- !query output
+1970-01-09 00:00:00
+
+
+-- !query
+select to_timestamp('300', 'D')
+-- !query schema
+struct<to_timestamp(300, D):timestamp>
+-- !query output
+1970-10-27 00:00:00
+
+
+-- !query
+select to_timestamp('09', 'DD')
+-- !query schema
+struct<to_timestamp(09, DD):timestamp>
+-- !query output
+1970-01-09 00:00:00
+
+
+-- !query
+select to_timestamp('99', 'DD')
+-- !query schema
+struct<to_timestamp(99, DD):timestamp>
+-- !query output
+1970-04-09 00:00:00
+
+
+-- !query
+select to_timestamp('009', 'DDD')
+-- !query schema
+struct<to_timestamp(009, DDD):timestamp>
+-- !query output
+1970-01-09 00:00:00
+
+
+-- !query
+select to_timestamp('365', 'DDD')
+-- !query schema
+struct<to_timestamp(365, DDD):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('31-365', 'dd-DDD')
+-- !query schema
+struct<to_timestamp(31-365, dd-DDD):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('12-365', 'MM-DDD')
+-- !query schema
+struct<to_timestamp(12-365, MM-DDD):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('2020-365', 'yyyy-DDD')
+-- !query schema
+struct<to_timestamp(2020-365, yyyy-DDD):timestamp>
+-- !query output
+2020-12-30 00:00:00
+
+
+-- !query
+select to_timestamp('12-31-365', 'MM-dd-DDD')
+-- !query schema
+struct<to_timestamp(12-31-365, MM-dd-DDD):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('2020-30-365', 'yyyy-dd-DDD')
+-- !query schema
+struct<to_timestamp(2020-30-365, yyyy-dd-DDD):timestamp>
+-- !query output
+2020-12-30 00:00:00
+
+
+-- !query
+select to_timestamp('2020-12-350', 'yyyy-MM-DDD')
+-- !query schema
+struct<to_timestamp(2020-12-350, yyyy-MM-DDD):timestamp>
+-- !query output
+2020-12-15 00:00:00
+
+
+-- !query
+select to_timestamp('2020-12-31-366', 'yyyy-MM-dd-DDD')
+-- !query schema
+struct<to_timestamp(2020-12-31-366, yyyy-MM-dd-DDD):timestamp>
+-- !query output
+2020-12-31 00:00:00
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-parsing.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-parsing.sql.out
new file mode 100644
index 0000000000000..98146a189a005
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-parsing.sql.out
@@ -0,0 +1,202 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 25
+
+
+-- !query
+select to_timestamp('1', 'y')
+-- !query schema
+struct<to_timestamp(1, y):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('009999', 'y')
+-- !query schema
+struct<to_timestamp(009999, y):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('00', 'yy')
+-- !query schema
+struct<to_timestamp(00, yy):timestamp>
+-- !query output
+2000-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('99', 'yy')
+-- !query schema
+struct<to_timestamp(99, yy):timestamp>
+-- !query output
+2099-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('001', 'yyy')
+-- !query schema
+struct<to_timestamp(001, yyy):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('009999', 'yyy')
+-- !query schema
+struct<to_timestamp(009999, yyy):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('0001', 'yyyy')
+-- !query schema
+struct<to_timestamp(0001, yyyy):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('9999', 'yyyy')
+-- !query schema
+struct<to_timestamp(9999, yyyy):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('00001', 'yyyyy')
+-- !query schema
+struct<to_timestamp(00001, yyyyy):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('09999', 'yyyyy')
+-- !query schema
+struct<to_timestamp(09999, yyyyy):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('000001', 'yyyyyy')
+-- !query schema
+struct<to_timestamp(000001, yyyyyy):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('009999', 'yyyyyy')
+-- !query schema
+struct<to_timestamp(009999, yyyyyy):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('9', 'D')
+-- !query schema
+struct<to_timestamp(9, D):timestamp>
+-- !query output
+1970-01-09 00:00:00
+
+
+-- !query
+select to_timestamp('300', 'D')
+-- !query schema
+struct<to_timestamp(300, D):timestamp>
+-- !query output
+1970-10-27 00:00:00
+
+
+-- !query
+select to_timestamp('09', 'DD')
+-- !query schema
+struct<to_timestamp(09, DD):timestamp>
+-- !query output
+1970-01-09 00:00:00
+
+
+-- !query
+select to_timestamp('99', 'DD')
+-- !query schema
+struct<to_timestamp(99, DD):timestamp>
+-- !query output
+1970-04-09 00:00:00
+
+
+-- !query
+select to_timestamp('009', 'DDD')
+-- !query schema
+struct<to_timestamp(009, DDD):timestamp>
+-- !query output
+1970-01-09 00:00:00
+
+
+-- !query
+select to_timestamp('365', 'DDD')
+-- !query schema
+struct<to_timestamp(365, DDD):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('31-365', 'dd-DDD')
+-- !query schema
+struct<to_timestamp(31-365, dd-DDD):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('12-365', 'MM-DDD')
+-- !query schema
+struct<to_timestamp(12-365, MM-DDD):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('2020-365', 'yyyy-DDD')
+-- !query schema
+struct<to_timestamp(2020-365, yyyy-DDD):timestamp>
+-- !query output
+2020-12-30 00:00:00
+
+
+-- !query
+select to_timestamp('12-31-365', 'MM-dd-DDD')
+-- !query schema
+struct<to_timestamp(12-31-365, MM-dd-DDD):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('2020-30-365', 'yyyy-dd-DDD')
+-- !query schema
+struct<to_timestamp(2020-30-365, yyyy-dd-DDD):timestamp>
+-- !query output
+2020-12-30 00:00:00
+
+
+-- !query
+select to_timestamp('2020-12-350', 'yyyy-MM-DDD')
+-- !query schema
+struct<to_timestamp(2020-12-350, yyyy-MM-DDD):timestamp>
+-- !query output
+2020-12-15 00:00:00
+
+
+-- !query
+select to_timestamp('2020-12-31-366', 'yyyy-MM-dd-DDD')
+-- !query schema
+struct<to_timestamp(2020-12-31-366, yyyy-MM-dd-DDD):timestamp>
+-- !query output
+2020-12-31 00:00:00
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
index a4f5b3772d2d2..b0820bb70011a 100755
--- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
@@ -1,5 +1,130 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 85
+-- Number of queries: 128
+
+
+-- !query
+select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null)
+-- !query schema
+struct<timestamp_seconds(1230219000):timestamp,timestamp_seconds(-1230219000):timestamp,timestamp_seconds(NULL):timestamp>
+-- !query output
+2008-12-25 07:30:00	1931-01-07 00:30:00	NULL
+
+
+-- !query
+select TIMESTAMP_SECONDS(1.23), TIMESTAMP_SECONDS(1.23d), TIMESTAMP_SECONDS(FLOAT(1.23))
+-- !query schema
+struct<timestamp_seconds(1.23):timestamp,timestamp_seconds(1.23):timestamp,timestamp_seconds(1.23):timestamp>
+-- !query output
+1969-12-31 16:00:01.23	1969-12-31 16:00:01.23	1969-12-31 16:00:01.23
+
+
+-- !query
+select TIMESTAMP_MILLIS(1230219000123),TIMESTAMP_MILLIS(-1230219000123),TIMESTAMP_MILLIS(null)
+-- !query schema
+struct<timestamp_millis(1230219000123):timestamp,timestamp_millis(-1230219000123):timestamp,timestamp_millis(NULL):timestamp>
+-- !query output
+2008-12-25 07:30:00.123	1931-01-07 00:29:59.877	NULL
+
+
+-- !query
+select TIMESTAMP_MICROS(1230219000123123),TIMESTAMP_MICROS(-1230219000123123),TIMESTAMP_MICROS(null)
+-- !query schema
+struct<timestamp_micros(1230219000123123):timestamp,timestamp_micros(-1230219000123123):timestamp,timestamp_micros(NULL):timestamp>
+-- !query output
+2008-12-25 07:30:00.123123	1931-01-07 00:29:59.876877	NULL
+
+
+-- !query
+select TIMESTAMP_SECONDS(1230219000123123)
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+long overflow
+
+
+-- !query
+select TIMESTAMP_SECONDS(-1230219000123123)
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+long overflow
+
+
+-- !query
+select TIMESTAMP_MILLIS(92233720368547758)
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+long overflow
+
+
+-- !query
+select TIMESTAMP_MILLIS(-92233720368547758)
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+long overflow
+
+
+-- !query
+select TIMESTAMP_SECONDS(0.1234567)
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+Rounding necessary
+
+
+-- !query
+select TIMESTAMP_SECONDS(0.1234567d), TIMESTAMP_SECONDS(FLOAT(0.1234567))
+-- !query schema
+struct<timestamp_seconds(0.1234567):timestamp,timestamp_seconds(0.1234567):timestamp>
+-- !query output
+1969-12-31 16:00:00.123456	1969-12-31 16:00:00.123456
+
+
+-- !query
+select UNIX_SECONDS(TIMESTAMP('2020-12-01 14:30:08Z')), UNIX_SECONDS(TIMESTAMP('2020-12-01 14:30:08.999999Z')), UNIX_SECONDS(null)
+-- !query schema
+struct<unix_seconds(2020-12-01 14:30:08Z):bigint,unix_seconds(2020-12-01 14:30:08.999999Z):bigint,unix_seconds(NULL):bigint>
+-- !query output
+1606833008	1606833008	NULL
+
+
+-- !query
+select UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08Z')), UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08.999999Z')), UNIX_MILLIS(null)
+-- !query schema
+struct<unix_millis(2020-12-01 14:30:08Z):bigint,unix_millis(2020-12-01 14:30:08.999999Z):bigint,unix_millis(NULL):bigint>
+-- !query output
+1606833008000	1606833008999	NULL
+
+
+-- !query
+select UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08Z')), UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08.999999Z')), UNIX_MICROS(null)
+-- !query schema
+struct<unix_micros(2020-12-01 14:30:08Z):bigint,unix_micros(2020-12-01 14:30:08.999999Z):bigint,unix_micros(NULL):bigint>
+-- !query output
+1606833008000000	1606833008999999	NULL
+
+
+-- !query
+select DATE_FROM_UNIX_DATE(0), DATE_FROM_UNIX_DATE(1000), DATE_FROM_UNIX_DATE(null)
+-- !query schema
+struct<date_from_unix_date(0):date,date_from_unix_date(1000):date,date_from_unix_date(NULL):date>
+-- !query output
+1970-01-01	1972-09-27	NULL
+
+
+-- !query
+select UNIX_DATE(DATE('1970-01-01')), UNIX_DATE(DATE('2020-12-04')), UNIX_DATE(null)
+-- !query schema
+struct<unix_date(1970-01-01):int,unix_date(2020-12-04):int,unix_date(NULL):int>
+-- !query output
+0	18600	NULL
 
 
 -- !query
@@ -29,7 +154,7 @@ NULL	2016-12-31 00:12:00	2016-12-31 00:00:00
 -- !query
 select dayofweek('2007-02-03'), dayofweek('2009-07-30'), dayofweek('2017-05-27'), dayofweek(null), dayofweek('1582-10-15 13:10:15')
 -- !query schema
-struct<dayofweek(CAST(2007-02-03 AS DATE)):int,dayofweek(CAST(2009-07-30 AS DATE)):int,dayofweek(CAST(2017-05-27 AS DATE)):int,dayofweek(CAST(NULL AS DATE)):int,dayofweek(CAST(1582-10-15 13:10:15 AS DATE)):int>
+struct<dayofweek(2007-02-03):int,dayofweek(2009-07-30):int,dayofweek(2017-05-27):int,dayofweek(NULL):int,dayofweek(1582-10-15 13:10:15):int>
 -- !query output
 7	5	7	NULL	6
 
@@ -86,7 +211,7 @@ struct<a:int,b:int>
 -- !query
 select weekday('2007-02-03'), weekday('2009-07-30'), weekday('2017-05-27'), weekday(null), weekday('1582-10-15 13:10:15')
 -- !query schema
-struct<weekday(CAST(2007-02-03 AS DATE)):int,weekday(CAST(2009-07-30 AS DATE)):int,weekday(CAST(2017-05-27 AS DATE)):int,weekday(CAST(NULL AS DATE)):int,weekday(CAST(1582-10-15 13:10:15 AS DATE)):int>
+struct<weekday(2007-02-03):int,weekday(2009-07-30):int,weekday(2017-05-27):int,weekday(NULL):int,weekday(1582-10-15 13:10:15):int>
 -- !query output
 5	3	5	NULL	4
 
@@ -94,7 +219,7 @@ struct<weekday(CAST(2007-02-03 AS DATE)):int,weekday(CAST(2009-07-30 AS DATE)):i
 -- !query
 select year('1500-01-01'), month('1500-01-01'), dayOfYear('1500-01-01')
 -- !query schema
-struct<year(CAST(1500-01-01 AS DATE)):int,month(CAST(1500-01-01 AS DATE)):int,dayofyear(CAST(1500-01-01 AS DATE)):int>
+struct<year(1500-01-01):int,month(1500-01-01):int,dayofyear(1500-01-01):int>
 -- !query output
 1500	1	1
 
@@ -115,10 +240,38 @@ struct<TIMESTAMP '2019-01-01 00:00:00':timestamp>
 2019-01-01 00:00:00
 
 
+-- !query
+select date '2020-01-01中文'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the DATE value: 2020-01-01中文(line 1, pos 7)
+
+== SQL ==
+select date '2020-01-01中文'
+-------^^^
+
+
+-- !query
+select timestamp '2019-01-01中文'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the TIMESTAMP value: 2019-01-01中文(line 1, pos 7)
+
+== SQL ==
+select timestamp '2019-01-01中文'
+-------^^^
+
+
 -- !query
 select timestamp'2011-11-11 11:11:11' + interval '2' day
 -- !query schema
-struct<CAST(TIMESTAMP '2011-11-11 11:11:11' + INTERVAL '2 days' AS TIMESTAMP):timestamp>
+struct<TIMESTAMP '2011-11-11 11:11:11' + INTERVAL '2 days':timestamp>
 -- !query output
 2011-11-13 11:11:11
 
@@ -126,7 +279,7 @@ struct<CAST(TIMESTAMP '2011-11-11 11:11:11' + INTERVAL '2 days' AS TIMESTAMP):ti
 -- !query
 select timestamp'2011-11-11 11:11:11' - interval '2' day
 -- !query schema
-struct<CAST(TIMESTAMP '2011-11-11 11:11:11' - INTERVAL '2 days' AS TIMESTAMP):timestamp>
+struct<TIMESTAMP '2011-11-11 11:11:11' - INTERVAL '2 days':timestamp>
 -- !query output
 2011-11-09 11:11:11
 
@@ -150,7 +303,7 @@ struct<DATE '2011-11-11' - INTERVAL '2 seconds':date>
 -- !query
 select '2011-11-11' - interval '2' day
 -- !query schema
-struct<CAST(2011-11-11 - INTERVAL '2 days' AS STRING):string>
+struct<2011-11-11 - INTERVAL '2 days':string>
 -- !query output
 2011-11-09 00:00:00
 
@@ -158,7 +311,7 @@ struct<CAST(2011-11-11 - INTERVAL '2 days' AS STRING):string>
 -- !query
 select '2011-11-11 11:11:11' - interval '2' second
 -- !query schema
-struct<CAST(2011-11-11 11:11:11 - INTERVAL '2 seconds' AS STRING):string>
+struct<2011-11-11 11:11:11 - INTERVAL '2 seconds':string>
 -- !query output
 2011-11-11 11:11:09
 
@@ -166,7 +319,7 @@ struct<CAST(2011-11-11 11:11:11 - INTERVAL '2 seconds' AS STRING):string>
 -- !query
 select '1' - interval '2' second
 -- !query schema
-struct<CAST(1 - INTERVAL '2 seconds' AS STRING):string>
+struct<1 - INTERVAL '2 seconds':string>
 -- !query output
 NULL
 
@@ -183,23 +336,23 @@ cannot resolve '1 + (- INTERVAL '2 seconds')' due to data type mismatch: argumen
 -- !query
 select date'2020-01-01' - timestamp'2019-10-06 10:11:12.345678'
 -- !query schema
-struct<subtracttimestamps(CAST(DATE '2020-01-01' AS TIMESTAMP), TIMESTAMP '2019-10-06 10:11:12.345678'):interval>
+struct<(DATE '2020-01-01' - TIMESTAMP '2019-10-06 10:11:12.345678'):interval day to second>
 -- !query output
-2078 hours 48 minutes 47.654322 seconds
+86 13:48:47.654322000
 
 
 -- !query
 select timestamp'2019-10-06 10:11:12.345678' - date'2020-01-01'
 -- !query schema
-struct<subtracttimestamps(TIMESTAMP '2019-10-06 10:11:12.345678', CAST(DATE '2020-01-01' AS TIMESTAMP)):interval>
+struct<(TIMESTAMP '2019-10-06 10:11:12.345678' - DATE '2020-01-01'):interval day to second>
 -- !query output
--2078 hours -48 minutes -47.654322 seconds
+-86 13:48:47.654322000
 
 
 -- !query
 select timestamp'2019-10-06 10:11:12.345678' - null
 -- !query schema
-struct<subtracttimestamps(TIMESTAMP '2019-10-06 10:11:12.345678', CAST(NULL AS TIMESTAMP)):interval>
+struct<(TIMESTAMP '2019-10-06 10:11:12.345678' - NULL):interval day to second>
 -- !query output
 NULL
 
@@ -207,7 +360,7 @@ NULL
 -- !query
 select null - timestamp'2019-10-06 10:11:12.345678'
 -- !query schema
-struct<subtracttimestamps(CAST(NULL AS TIMESTAMP), TIMESTAMP '2019-10-06 10:11:12.345678'):interval>
+struct<(NULL - TIMESTAMP '2019-10-06 10:11:12.345678'):interval day to second>
 -- !query output
 NULL
 
@@ -215,7 +368,7 @@ NULL
 -- !query
 select date_add('2011-11-11', 1Y)
 -- !query schema
-struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+struct<date_add(2011-11-11, 1):date>
 -- !query output
 2011-11-12
 
@@ -223,7 +376,7 @@ struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
 -- !query
 select date_add('2011-11-11', 1S)
 -- !query schema
-struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+struct<date_add(2011-11-11, 1):date>
 -- !query output
 2011-11-12
 
@@ -231,7 +384,7 @@ struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
 -- !query
 select date_add('2011-11-11', 1)
 -- !query schema
-struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+struct<date_add(2011-11-11, 1):date>
 -- !query output
 2011-11-12
 
@@ -266,7 +419,7 @@ cannot resolve 'date_add(CAST('2011-11-11' AS DATE), 10.0D)' due to data type mi
 -- !query
 select date_add('2011-11-11', '1')
 -- !query schema
-struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+struct<date_add(2011-11-11, 1):date>
 -- !query output
 2011-11-12
 
@@ -277,7 +430,7 @@ select date_add('2011-11-11', '1.2')
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-The second argument of 'date_add' function needs to be an integer.;
+The second argument of 'date_add' function needs to be an integer.
 
 
 -- !query
@@ -291,7 +444,7 @@ struct<date_add(DATE '2011-11-11', 1):date>
 -- !query
 select date_add(timestamp'2011-11-11', 1)
 -- !query schema
-struct<date_add(CAST(TIMESTAMP '2011-11-11 00:00:00' AS DATE), 1):date>
+struct<date_add(TIMESTAMP '2011-11-11 00:00:00', 1):date>
 -- !query output
 2011-11-12
 
@@ -318,13 +471,13 @@ select date_sub(date'2011-11-11', '1.2')
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-The second argument of 'date_sub' function needs to be an integer.;
+The second argument of 'date_sub' function needs to be an integer.
 
 
 -- !query
 select date_sub(timestamp'2011-11-11', 1)
 -- !query schema
-struct<date_sub(CAST(TIMESTAMP '2011-11-11 00:00:00' AS DATE), 1):date>
+struct<date_sub(TIMESTAMP '2011-11-11 00:00:00', 1):date>
 -- !query output
 2011-11-10
 
@@ -332,7 +485,7 @@ struct<date_sub(CAST(TIMESTAMP '2011-11-11 00:00:00' AS DATE), 1):date>
 -- !query
 select date_sub(null, 1)
 -- !query schema
-struct<date_sub(CAST(NULL AS DATE), 1):date>
+struct<date_sub(NULL, 1):date>
 -- !query output
 NULL
 
@@ -340,7 +493,7 @@ NULL
 -- !query
 select date_sub(date'2011-11-11', null)
 -- !query schema
-struct<date_sub(DATE '2011-11-11', CAST(NULL AS INT)):date>
+struct<date_sub(DATE '2011-11-11', NULL):date>
 -- !query output
 NULL
 
@@ -366,7 +519,7 @@ cannot resolve 'date_add(DATE '2011-11-11', CAST('1' AS DOUBLE))' due to data ty
 -- !query
 select null + date '2001-09-28'
 -- !query schema
-struct<date_add(DATE '2001-09-28', CAST(NULL AS INT)):date>
+struct<date_add(DATE '2001-09-28', NULL):date>
 -- !query output
 NULL
 
@@ -407,7 +560,7 @@ cannot resolve 'date_sub(DATE '2001-10-01', CAST('7' AS DOUBLE))' due to data ty
 -- !query
 select date '2001-09-28' + null
 -- !query schema
-struct<date_add(DATE '2001-09-28', CAST(NULL AS INT)):date>
+struct<date_add(DATE '2001-09-28', NULL):date>
 -- !query output
 NULL
 
@@ -415,7 +568,7 @@ NULL
 -- !query
 select date '2001-09-28' - null
 -- !query schema
-struct<date_sub(DATE '2001-09-28', CAST(NULL AS INT)):date>
+struct<date_sub(DATE '2001-09-28', NULL):date>
 -- !query output
 NULL
 
@@ -434,7 +587,7 @@ select date_add('2011-11-11', str) from v
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'date_add(CAST('2011-11-11' AS DATE), v.`str`)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'v.`str`' is of string type.; line 1 pos 7
+cannot resolve 'date_add(CAST('2011-11-11' AS DATE), v.str)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'v.str' is of string type.; line 1 pos 7
 
 
 -- !query
@@ -443,13 +596,13 @@ select date_sub('2011-11-11', str) from v
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'date_sub(CAST('2011-11-11' AS DATE), v.`str`)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'v.`str`' is of string type.; line 1 pos 7
+cannot resolve 'date_sub(CAST('2011-11-11' AS DATE), v.str)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'v.str' is of string type.; line 1 pos 7
 
 
 -- !query
 select null - date '2019-10-06'
 -- !query schema
-struct<subtractdates(CAST(NULL AS DATE), DATE '2019-10-06'):interval>
+struct<(NULL - DATE '2019-10-06'):interval day to second>
 -- !query output
 NULL
 
@@ -457,9 +610,9 @@ NULL
 -- !query
 select date '2001-10-01' - date '2001-09-28'
 -- !query schema
-struct<subtractdates(DATE '2001-10-01', DATE '2001-09-28'):interval>
+struct<(DATE '2001-10-01' - DATE '2001-09-28'):interval day to second>
 -- !query output
-3 days
+3 00:00:00.000000000
 
 
 -- !query
@@ -647,58 +800,272 @@ struct<to_timestamp(S2019-10-06, 'S'yyyy-MM-dd):timestamp>
 
 
 -- !query
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuee')
+select to_timestamp("2019-10-06T10:11:12'12", "yyyy-MM-dd'T'HH:mm:ss''SSSS")
+-- !query schema
+struct<to_timestamp(2019-10-06T10:11:12'12, yyyy-MM-dd'T'HH:mm:ss''SSSS):timestamp>
+-- !query output
+2019-10-06 10:11:12.12
+
+
+-- !query
+select to_timestamp("2019-10-06T10:11:12'", "yyyy-MM-dd'T'HH:mm:ss''")
+-- !query schema
+struct<to_timestamp(2019-10-06T10:11:12', yyyy-MM-dd'T'HH:mm:ss''):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("'2019-10-06T10:11:12", "''yyyy-MM-dd'T'HH:mm:ss")
+-- !query schema
+struct<to_timestamp('2019-10-06T10:11:12, ''yyyy-MM-dd'T'HH:mm:ss):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("P2019-10-06T10:11:12", "'P'yyyy-MM-dd'T'HH:mm:ss")
+-- !query schema
+struct<to_timestamp(P2019-10-06T10:11:12, 'P'yyyy-MM-dd'T'HH:mm:ss):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("16", "dd")
+-- !query schema
+struct<to_timestamp(16, dd):timestamp>
+-- !query output
+1970-01-16 00:00:00
+
+
+-- !query
+select to_timestamp("02-29", "MM-dd")
+-- !query schema
+struct<to_timestamp(02-29, MM-dd):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_date("16", "dd")
+-- !query schema
+struct<to_date(16, dd):date>
+-- !query output
+1970-01-16
+
+
+-- !query
+select to_date("02-29", "MM-dd")
+-- !query schema
+struct<to_date(02-29, MM-dd):date>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("2019 40", "yyyy mm")
+-- !query schema
+struct<to_timestamp(2019 40, yyyy mm):timestamp>
+-- !query output
+2019-01-01 00:40:00
+
+
+-- !query
+select to_timestamp("2019 10:10:10", "yyyy hh:mm:ss")
+-- !query schema
+struct<to_timestamp(2019 10:10:10, yyyy hh:mm:ss):timestamp>
+-- !query output
+2019-01-01 10:10:10
+
+
+-- !query
+select to_timestamp('2019-10-06 A', 'yyyy-MM-dd GGGGG')
 -- !query schema
 struct<>
 -- !query output
-java.lang.IllegalArgumentException
-Illegal pattern character: e
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyy-MM-dd GGGGG' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
 
 
 -- !query
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uucc')
+select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEEE')
 -- !query schema
 struct<>
 -- !query output
-java.lang.IllegalArgumentException
-Illegal pattern character: c
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd MM yyyy EEEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
 
 
 -- !query
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuuu')
+select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE')
 -- !query schema
-struct<date_format(TIMESTAMP '2019-10-06 00:00:00', yyyy-MM-dd uuuu):string>
+struct<>
 -- !query output
-2019-10-06 Sunday
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd MM yyyy EEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
 
 
 -- !query
-select to_timestamp("2019-10-06T10:11:12'12", "yyyy-MM-dd'T'HH:mm:ss''SSSS")
+select unix_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE')
 -- !query schema
-struct<to_timestamp(2019-10-06T10:11:12'12, yyyy-MM-dd'T'HH:mm:ss''SSSS):timestamp>
+struct<>
 -- !query output
-2019-10-06 10:11:12.12
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd MM yyyy EEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
 
 
 -- !query
-select to_timestamp("2019-10-06T10:11:12'", "yyyy-MM-dd'T'HH:mm:ss''")
+select from_json('{"t":"26/October/2015"}', 't Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'))
 -- !query schema
-struct<to_timestamp(2019-10-06T10:11:12', yyyy-MM-dd'T'HH:mm:ss''):timestamp>
+struct<>
 -- !query output
-2019-10-06 10:11:12
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
 
 
 -- !query
-select to_timestamp("'2019-10-06T10:11:12", "''yyyy-MM-dd'T'HH:mm:ss")
+select from_json('{"d":"26/October/2015"}', 'd Date', map('dateFormat', 'dd/MMMMM/yyyy'))
 -- !query schema
-struct<to_timestamp('2019-10-06T10:11:12, ''yyyy-MM-dd'T'HH:mm:ss):timestamp>
+struct<>
 -- !query output
-2019-10-06 10:11:12
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
 
 
 -- !query
-select to_timestamp("P2019-10-06T10:11:12", "'P'yyyy-MM-dd'T'HH:mm:ss")
+select from_csv('26/October/2015', 't Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'))
 -- !query schema
-struct<to_timestamp(P2019-10-06T10:11:12, 'P'yyyy-MM-dd'T'HH:mm:ss):timestamp>
+struct<>
 -- !query output
-2019-10-06 10:11:12
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_csv('26/October/2015', 'd Date', map('dateFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select to_date("2020-01-27T20:06:11.847", "yyyy-MM-dd HH:mm:ss.SSS")
+-- !query schema
+struct<to_date(2020-01-27T20:06:11.847, yyyy-MM-dd HH:mm:ss.SSS):date>
+-- !query output
+NULL
+
+
+-- !query
+select to_date("Unparseable", "yyyy-MM-dd HH:mm:ss.SSS")
+-- !query schema
+struct<to_date(Unparseable, yyyy-MM-dd HH:mm:ss.SSS):date>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("2020-01-27T20:06:11.847", "yyyy-MM-dd HH:mm:ss.SSS")
+-- !query schema
+struct<to_timestamp(2020-01-27T20:06:11.847, yyyy-MM-dd HH:mm:ss.SSS):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("Unparseable", "yyyy-MM-dd HH:mm:ss.SSS")
+-- !query schema
+struct<to_timestamp(Unparseable, yyyy-MM-dd HH:mm:ss.SSS):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select unix_timestamp("2020-01-27T20:06:11.847", "yyyy-MM-dd HH:mm:ss.SSS")
+-- !query schema
+struct<unix_timestamp(2020-01-27T20:06:11.847, yyyy-MM-dd HH:mm:ss.SSS):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select unix_timestamp("Unparseable", "yyyy-MM-dd HH:mm:ss.SSS")
+-- !query schema
+struct<unix_timestamp(Unparseable, yyyy-MM-dd HH:mm:ss.SSS):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select to_unix_timestamp("2020-01-27T20:06:11.847", "yyyy-MM-dd HH:mm:ss.SSS")
+-- !query schema
+struct<to_unix_timestamp(2020-01-27T20:06:11.847, yyyy-MM-dd HH:mm:ss.SSS):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select to_unix_timestamp("Unparseable", "yyyy-MM-dd HH:mm:ss.SSS")
+-- !query schema
+struct<to_unix_timestamp(Unparseable, yyyy-MM-dd HH:mm:ss.SSS):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select cast("Unparseable" as timestamp)
+-- !query schema
+struct<CAST(Unparseable AS TIMESTAMP):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select cast("Unparseable" as date)
+-- !query schema
+struct<CAST(Unparseable AS DATE):date>
+-- !query output
+NULL
+
+
+-- !query
+select next_day("2015-07-23", "Mon")
+-- !query schema
+struct<next_day(2015-07-23, Mon):date>
+-- !query output
+2015-07-27
+
+
+-- !query
+select next_day("2015-07-23", "xx")
+-- !query schema
+struct<next_day(2015-07-23, xx):date>
+-- !query output
+NULL
+
+
+-- !query
+select next_day("xx", "Mon")
+-- !query schema
+struct<next_day(xx, Mon):date>
+-- !query output
+NULL
+
+
+-- !query
+select next_day(null, "Mon")
+-- !query schema
+struct<next_day(NULL, Mon):date>
+-- !query output
+NULL
+
+
+-- !query
+select next_day(null, "xx")
+-- !query schema
+struct<next_day(NULL, xx):date>
+-- !query output
+NULL
diff --git a/sql/core/src/test/resources/sql-tests/results/decimalArithmeticOperations.sql.out b/sql/core/src/test/resources/sql-tests/results/decimalArithmeticOperations.sql.out
index 72e46ef493a5d..1d92dc3501020 100644
--- a/sql/core/src/test/resources/sql-tests/results/decimalArithmeticOperations.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/decimalArithmeticOperations.sql.out
@@ -13,7 +13,7 @@ struct<>
 -- !query
 select a / b from t
 -- !query schema
-struct<(CAST(a AS DECIMAL(2,1)) / CAST(b AS DECIMAL(2,1))):decimal(8,6)>
+struct<(a / b):decimal(8,6)>
 -- !query output
 NULL
 
@@ -21,7 +21,7 @@ NULL
 -- !query
 select a % b from t
 -- !query schema
-struct<(CAST(a AS DECIMAL(2,1)) % CAST(b AS DECIMAL(2,1))):decimal(1,1)>
+struct<(a % b):decimal(1,1)>
 -- !query output
 NULL
 
@@ -29,7 +29,7 @@ NULL
 -- !query
 select pmod(a, b) from t
 -- !query schema
-struct<pmod(CAST(a AS DECIMAL(2,1)), CAST(b AS DECIMAL(2,1))):decimal(1,1)>
+struct<pmod(a, b):decimal(1,1)>
 -- !query output
 NULL
 
@@ -65,7 +65,7 @@ struct<id:int,(a + b):decimal(38,17),(a - b):decimal(38,17),(a * b):decimal(38,6
 -- !query
 select id, a*10, b/10 from decimals_test order by id
 -- !query schema
-struct<id:int,(CAST(a AS DECIMAL(38,18)) * CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,15),(CAST(b AS DECIMAL(38,18)) / CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,18)>
+struct<id:int,(a * 10):decimal(38,15),(b / 10):decimal(38,18)>
 -- !query output
 1	1000.000000000000000	99.900000000000000000
 2	123451.230000000000000	1234.512300000000000000
@@ -76,7 +76,7 @@ struct<id:int,(CAST(a AS DECIMAL(38,18)) * CAST(CAST(10 AS DECIMAL(2,0)) AS DECI
 -- !query
 select 10.3 * 3.0
 -- !query schema
-struct<(CAST(10.3 AS DECIMAL(3,1)) * CAST(3.0 AS DECIMAL(3,1))):decimal(6,2)>
+struct<(10.3 * 3.0):decimal(6,2)>
 -- !query output
 30.90
 
@@ -84,7 +84,7 @@ struct<(CAST(10.3 AS DECIMAL(3,1)) * CAST(3.0 AS DECIMAL(3,1))):decimal(6,2)>
 -- !query
 select 10.3000 * 3.0
 -- !query schema
-struct<(CAST(10.3000 AS DECIMAL(6,4)) * CAST(3.0 AS DECIMAL(6,4))):decimal(9,5)>
+struct<(10.3000 * 3.0):decimal(9,5)>
 -- !query output
 30.90000
 
@@ -92,7 +92,7 @@ struct<(CAST(10.3000 AS DECIMAL(6,4)) * CAST(3.0 AS DECIMAL(6,4))):decimal(9,5)>
 -- !query
 select 10.30000 * 30.0
 -- !query schema
-struct<(CAST(10.30000 AS DECIMAL(7,5)) * CAST(30.0 AS DECIMAL(7,5))):decimal(11,6)>
+struct<(10.30000 * 30.0):decimal(11,6)>
 -- !query output
 309.000000
 
@@ -100,7 +100,7 @@ struct<(CAST(10.30000 AS DECIMAL(7,5)) * CAST(30.0 AS DECIMAL(7,5))):decimal(11,
 -- !query
 select 10.300000000000000000 * 3.000000000000000000
 -- !query schema
-struct<(CAST(10.300000000000000000 AS DECIMAL(20,18)) * CAST(3.000000000000000000 AS DECIMAL(20,18))):decimal(38,34)>
+struct<(10.300000000000000000 * 3.000000000000000000):decimal(38,34)>
 -- !query output
 30.9000000000000000000000000000000000
 
@@ -108,7 +108,7 @@ struct<(CAST(10.300000000000000000 AS DECIMAL(20,18)) * CAST(3.00000000000000000
 -- !query
 select 10.300000000000000000 * 3.0000000000000000000
 -- !query schema
-struct<(CAST(10.300000000000000000 AS DECIMAL(21,19)) * CAST(3.0000000000000000000 AS DECIMAL(21,19))):decimal(38,34)>
+struct<(10.300000000000000000 * 3.0000000000000000000):decimal(38,34)>
 -- !query output
 30.9000000000000000000000000000000000
 
@@ -116,7 +116,7 @@ struct<(CAST(10.300000000000000000 AS DECIMAL(21,19)) * CAST(3.00000000000000000
 -- !query
 select 2.35E10 * 1.0
 -- !query schema
-struct<(2.35E10 * CAST(1.0 AS DOUBLE)):double>
+struct<(2.35E10 * 1.0):double>
 -- !query output
 2.35E10
 
@@ -124,7 +124,7 @@ struct<(2.35E10 * CAST(1.0 AS DOUBLE)):double>
 -- !query
 select (5e36BD + 0.1) + 5e36BD
 -- !query schema
-struct<(CAST((CAST(5000000000000000000000000000000000000 AS DECIMAL(38,1)) + CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) + CAST(5000000000000000000000000000000000000 AS DECIMAL(38,1))):decimal(38,1)>
+struct<((5000000000000000000000000000000000000 + 0.1) + 5000000000000000000000000000000000000):decimal(38,1)>
 -- !query output
 NULL
 
@@ -132,7 +132,7 @@ NULL
 -- !query
 select (-4e36BD - 0.1) - 7e36BD
 -- !query schema
-struct<(CAST((CAST(-4000000000000000000000000000000000000 AS DECIMAL(38,1)) - CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) - CAST(7000000000000000000000000000000000000 AS DECIMAL(38,1))):decimal(38,1)>
+struct<((-4000000000000000000000000000000000000 - 0.1) - 7000000000000000000000000000000000000):decimal(38,1)>
 -- !query output
 NULL
 
@@ -148,7 +148,7 @@ NULL
 -- !query
 select 1e35BD / 0.1
 -- !query schema
-struct<(CAST(100000000000000000000000000000000000 AS DECIMAL(37,1)) / CAST(0.1 AS DECIMAL(37,1))):decimal(38,6)>
+struct<(100000000000000000000000000000000000 / 0.1):decimal(38,6)>
 -- !query output
 NULL
 
@@ -156,7 +156,7 @@ NULL
 -- !query
 select 1.2345678901234567890E30BD * 1.2345678901234567890E25BD
 -- !query schema
-struct<(CAST(1234567890123456789000000000000 AS DECIMAL(31,0)) * CAST(12345678901234567890000000 AS DECIMAL(31,0))):decimal(38,0)>
+struct<(1234567890123456789000000000000 * 12345678901234567890000000):decimal(38,0)>
 -- !query output
 NULL
 
@@ -164,7 +164,7 @@ NULL
 -- !query
 select 12345678912345678912345678912.1234567 + 9999999999999999999999999999999.12345
 -- !query schema
-struct<(CAST(12345678912345678912345678912.1234567 AS DECIMAL(38,6)) + CAST(9999999999999999999999999999999.12345 AS DECIMAL(38,6))):decimal(38,6)>
+struct<(12345678912345678912345678912.1234567 + 9999999999999999999999999999999.12345):decimal(38,6)>
 -- !query output
 10012345678912345678912345678911.246907
 
@@ -172,7 +172,7 @@ struct<(CAST(12345678912345678912345678912.1234567 AS DECIMAL(38,6)) + CAST(9999
 -- !query
 select 123456789123456789.1234567890 * 1.123456789123456789
 -- !query schema
-struct<(CAST(123456789123456789.1234567890 AS DECIMAL(36,18)) * CAST(1.123456789123456789 AS DECIMAL(36,18))):decimal(38,18)>
+struct<(123456789123456789.1234567890 * 1.123456789123456789):decimal(38,18)>
 -- !query output
 138698367904130467.654320988515622621
 
@@ -180,7 +180,7 @@ struct<(CAST(123456789123456789.1234567890 AS DECIMAL(36,18)) * CAST(1.123456789
 -- !query
 select 12345678912345.123456789123 / 0.000000012345678
 -- !query schema
-struct<(CAST(12345678912345.123456789123 AS DECIMAL(29,15)) / CAST(1.2345678E-8 AS DECIMAL(29,15))):decimal(38,9)>
+struct<(12345678912345.123456789123 / 1.2345678E-8):decimal(38,9)>
 -- !query output
 1000000073899961059796.725866332
 
@@ -207,7 +207,7 @@ struct<id:int,(a + b):decimal(38,18),(a - b):decimal(38,18),(a * b):decimal(38,3
 -- !query
 select id, a*10, b/10 from decimals_test order by id
 -- !query schema
-struct<id:int,(CAST(a AS DECIMAL(38,18)) * CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,18),(CAST(b AS DECIMAL(38,18)) / CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,19)>
+struct<id:int,(a * 10):decimal(38,18),(b / 10):decimal(38,19)>
 -- !query output
 1	1000.000000000000000000	99.9000000000000000000
 2	123451.230000000000000000	1234.5123000000000000000
@@ -218,7 +218,7 @@ struct<id:int,(CAST(a AS DECIMAL(38,18)) * CAST(CAST(10 AS DECIMAL(2,0)) AS DECI
 -- !query
 select 10.3 * 3.0
 -- !query schema
-struct<(CAST(10.3 AS DECIMAL(3,1)) * CAST(3.0 AS DECIMAL(3,1))):decimal(6,2)>
+struct<(10.3 * 3.0):decimal(6,2)>
 -- !query output
 30.90
 
@@ -226,7 +226,7 @@ struct<(CAST(10.3 AS DECIMAL(3,1)) * CAST(3.0 AS DECIMAL(3,1))):decimal(6,2)>
 -- !query
 select 10.3000 * 3.0
 -- !query schema
-struct<(CAST(10.3000 AS DECIMAL(6,4)) * CAST(3.0 AS DECIMAL(6,4))):decimal(9,5)>
+struct<(10.3000 * 3.0):decimal(9,5)>
 -- !query output
 30.90000
 
@@ -234,7 +234,7 @@ struct<(CAST(10.3000 AS DECIMAL(6,4)) * CAST(3.0 AS DECIMAL(6,4))):decimal(9,5)>
 -- !query
 select 10.30000 * 30.0
 -- !query schema
-struct<(CAST(10.30000 AS DECIMAL(7,5)) * CAST(30.0 AS DECIMAL(7,5))):decimal(11,6)>
+struct<(10.30000 * 30.0):decimal(11,6)>
 -- !query output
 309.000000
 
@@ -242,7 +242,7 @@ struct<(CAST(10.30000 AS DECIMAL(7,5)) * CAST(30.0 AS DECIMAL(7,5))):decimal(11,
 -- !query
 select 10.300000000000000000 * 3.000000000000000000
 -- !query schema
-struct<(CAST(10.300000000000000000 AS DECIMAL(20,18)) * CAST(3.000000000000000000 AS DECIMAL(20,18))):decimal(38,36)>
+struct<(10.300000000000000000 * 3.000000000000000000):decimal(38,36)>
 -- !query output
 30.900000000000000000000000000000000000
 
@@ -250,7 +250,7 @@ struct<(CAST(10.300000000000000000 AS DECIMAL(20,18)) * CAST(3.00000000000000000
 -- !query
 select 10.300000000000000000 * 3.0000000000000000000
 -- !query schema
-struct<(CAST(10.300000000000000000 AS DECIMAL(21,19)) * CAST(3.0000000000000000000 AS DECIMAL(21,19))):decimal(38,37)>
+struct<(10.300000000000000000 * 3.0000000000000000000):decimal(38,37)>
 -- !query output
 NULL
 
@@ -258,7 +258,7 @@ NULL
 -- !query
 select 2.35E10 * 1.0
 -- !query schema
-struct<(2.35E10 * CAST(1.0 AS DOUBLE)):double>
+struct<(2.35E10 * 1.0):double>
 -- !query output
 2.35E10
 
@@ -266,7 +266,7 @@ struct<(2.35E10 * CAST(1.0 AS DOUBLE)):double>
 -- !query
 select (5e36BD + 0.1) + 5e36BD
 -- !query schema
-struct<(CAST((CAST(5000000000000000000000000000000000000 AS DECIMAL(38,1)) + CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) + CAST(5000000000000000000000000000000000000 AS DECIMAL(38,1))):decimal(38,1)>
+struct<((5000000000000000000000000000000000000 + 0.1) + 5000000000000000000000000000000000000):decimal(38,1)>
 -- !query output
 NULL
 
@@ -274,7 +274,7 @@ NULL
 -- !query
 select (-4e36BD - 0.1) - 7e36BD
 -- !query schema
-struct<(CAST((CAST(-4000000000000000000000000000000000000 AS DECIMAL(38,1)) - CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) - CAST(7000000000000000000000000000000000000 AS DECIMAL(38,1))):decimal(38,1)>
+struct<((-4000000000000000000000000000000000000 - 0.1) - 7000000000000000000000000000000000000):decimal(38,1)>
 -- !query output
 NULL
 
@@ -290,7 +290,7 @@ NULL
 -- !query
 select 1e35BD / 0.1
 -- !query schema
-struct<(CAST(100000000000000000000000000000000000 AS DECIMAL(37,1)) / CAST(0.1 AS DECIMAL(37,1))):decimal(38,3)>
+struct<(100000000000000000000000000000000000 / 0.1):decimal(38,3)>
 -- !query output
 NULL
 
@@ -298,7 +298,7 @@ NULL
 -- !query
 select 1.2345678901234567890E30BD * 1.2345678901234567890E25BD
 -- !query schema
-struct<(CAST(1234567890123456789000000000000 AS DECIMAL(31,0)) * CAST(12345678901234567890000000 AS DECIMAL(31,0))):decimal(38,0)>
+struct<(1234567890123456789000000000000 * 12345678901234567890000000):decimal(38,0)>
 -- !query output
 NULL
 
@@ -306,7 +306,7 @@ NULL
 -- !query
 select 12345678912345678912345678912.1234567 + 9999999999999999999999999999999.12345
 -- !query schema
-struct<(CAST(12345678912345678912345678912.1234567 AS DECIMAL(38,7)) + CAST(9999999999999999999999999999999.12345 AS DECIMAL(38,7))):decimal(38,7)>
+struct<(12345678912345678912345678912.1234567 + 9999999999999999999999999999999.12345):decimal(38,7)>
 -- !query output
 NULL
 
@@ -314,7 +314,7 @@ NULL
 -- !query
 select 123456789123456789.1234567890 * 1.123456789123456789
 -- !query schema
-struct<(CAST(123456789123456789.1234567890 AS DECIMAL(36,18)) * CAST(1.123456789123456789 AS DECIMAL(36,18))):decimal(38,28)>
+struct<(123456789123456789.1234567890 * 1.123456789123456789):decimal(38,28)>
 -- !query output
 NULL
 
@@ -322,7 +322,7 @@ NULL
 -- !query
 select 12345678912345.123456789123 / 0.000000012345678
 -- !query schema
-struct<(CAST(12345678912345.123456789123 AS DECIMAL(29,15)) / CAST(1.2345678E-8 AS DECIMAL(29,15))):decimal(38,18)>
+struct<(12345678912345.123456789123 / 1.2345678E-8):decimal(38,18)>
 -- !query output
 NULL
 
diff --git a/sql/core/src/test/resources/sql-tests/results/describe-table-column.sql.out b/sql/core/src/test/resources/sql-tests/results/describe-table-column.sql.out
index c6d3d45879eb1..cc5b836b74109 100644
--- a/sql/core/src/test/resources/sql-tests/results/describe-table-column.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/describe-table-column.sql.out
@@ -1,9 +1,9 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 28
+-- Number of queries: 30
 
 
 -- !query
-CREATE TEMPORARY VIEW desc_col_temp_view (key int COMMENT 'column_comment') USING PARQUET
+CREATE TEMPORARY VIEW desc_col_temp_view (key int COMMENT 'column_comment', col struct<x:int, y:string>) USING PARQUET
 -- !query schema
 struct<>
 -- !query output
@@ -77,7 +77,16 @@ DESC desc_col_temp_view key1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Column key1 does not exist;
+Column key1 does not exist
+
+
+-- !query
+DESC desc_col_temp_view col.x
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+DESC TABLE COLUMN does not support nested column: col.x
 
 
 -- !query
@@ -140,6 +149,15 @@ max_col_len	4
 histogram	NULL
 
 
+-- !query
+DESC desc_col_table key1
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Column key1 does not exist
+
+
 -- !query
 CREATE TABLE desc_complex_col_table (`a.b` int, col struct<x:int, y:string>) USING PARQUET
 -- !query schema
@@ -188,7 +206,7 @@ DESC FORMATTED desc_complex_col_table col.x
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-DESC TABLE COLUMN command does not support nested data types: col.x;
+DESC TABLE COLUMN does not support nested column: col.x
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out
index a7de033e3a1ac..7634c112ab81a 100644
--- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out
@@ -130,7 +130,7 @@ Num Buckets         	2
 Bucket Columns      	[`a`]               	                    
 Sort Columns        	[`b`]               	                    
 Comment             	table_comment       	                    
-Table Properties    	[t=test, e=3]       	                    
+Table Properties    	[e=3, t=test]       	                    
 Location [not included in comparison]/{warehouse_dir}/t	                    
 Storage Properties  	[a=1, b=2]          	                    
 Partition Provider  	Catalog
@@ -162,7 +162,7 @@ Num Buckets         	2
 Bucket Columns      	[`a`]               	                    
 Sort Columns        	[`b`]               	                    
 Comment             	table_comment       	                    
-Table Properties    	[t=test, e=3]       	                    
+Table Properties    	[e=3, t=test]       	                    
 Location [not included in comparison]/{warehouse_dir}/t	                    
 Storage Properties  	[a=1, b=2]          	                    
 Partition Provider  	Catalog
@@ -332,7 +332,7 @@ struct<>
 org.apache.spark.sql.catalyst.analysis.NoSuchPartitionException
 Partition not found in table 't' database 'default':
 c -> Us
-d -> 2;
+d -> 2
 
 
 -- !query
@@ -341,7 +341,7 @@ DESC t PARTITION (c='Us')
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Partition spec is invalid. The spec (c) must match the partition spec (c, d) defined in table '`default`.`t`';
+Partition spec is invalid. The spec (c) must match the partition spec (c, d) defined in table '`default`.`t`'
 
 
 -- !query
@@ -431,7 +431,7 @@ DESC temp_v PARTITION (c='Us', d=1)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-DESC PARTITION is not allowed on a temporary view: temp_v;
+DESC PARTITION is not allowed on a temporary view: temp_v
 
 
 -- !query
@@ -477,7 +477,7 @@ View Text           	SELECT * FROM t
 View Original Text  	SELECT * FROM t     	                    
 View Catalog and Namespace	spark_catalog.default	                    
 View Query Output Columns	[a, b, c, d]        	                    
-Table Properties    	[view.query.out.col.3=d, view.catalogAndNamespace.numParts=2, view.query.out.col.0=a, view.query.out.numCols=4, view.query.out.col.1=b, view.catalogAndNamespace.part.0=spark_catalog, view.query.out.col.2=c, view.catalogAndNamespace.part.1=default]
+Table Properties    	[]
 
 
 -- !query
@@ -501,7 +501,7 @@ View Text           	SELECT * FROM t
 View Original Text  	SELECT * FROM t     	                    
 View Catalog and Namespace	spark_catalog.default	                    
 View Query Output Columns	[a, b, c, d]        	                    
-Table Properties    	[view.query.out.col.3=d, view.catalogAndNamespace.numParts=2, view.query.out.col.0=a, view.query.out.numCols=4, view.query.out.col.1=b, view.catalogAndNamespace.part.0=spark_catalog, view.query.out.col.2=c, view.catalogAndNamespace.part.1=default]
+Table Properties    	[]
 
 
 -- !query
@@ -510,7 +510,7 @@ DESC v PARTITION (c='Us', d=1)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-DESC PARTITION is not allowed on a view: v;
+DESC PARTITION is not allowed on a view: v
 
 
 -- !query
@@ -520,7 +520,7 @@ struct<plan:string>
 -- !query output
 == Physical Plan ==
 Execute DescribeTableCommand
-   +- DescribeTableCommand `default`.`t`, false
+   +- DescribeTableCommand `default`.`t`, false, [col_name#x, data_type#x, comment#x]
 
 
 -- !query
@@ -530,7 +530,7 @@ struct<plan:string>
 -- !query output
 == Physical Plan ==
 Execute DescribeTableCommand
-   +- DescribeTableCommand `default`.`t`, true
+   +- DescribeTableCommand `default`.`t`, true, [col_name#x, data_type#x, comment#x]
 
 
 -- !query
@@ -539,19 +539,19 @@ EXPLAIN EXTENDED DESC t
 struct<plan:string>
 -- !query output
 == Parsed Logical Plan ==
-'DescribeRelation false
-+- 'UnresolvedTableOrView [t]
+'DescribeRelation false, [col_name#x, data_type#x, comment#x]
++- 'UnresolvedTableOrView [t], DESCRIBE TABLE, true
 
 == Analyzed Logical Plan ==
 col_name: string, data_type: string, comment: string
-DescribeTableCommand `default`.`t`, false
+DescribeTableCommand `default`.`t`, false, [col_name#x, data_type#x, comment#x]
 
 == Optimized Logical Plan ==
-DescribeTableCommand `default`.`t`, false
+DescribeTableCommand `default`.`t`, false, [col_name#x, data_type#x, comment#x]
 
 == Physical Plan ==
 Execute DescribeTableCommand
-   +- DescribeTableCommand `default`.`t`, false
+   +- DescribeTableCommand `default`.`t`, false, [col_name#x, data_type#x, comment#x]
 
 
 -- !query
@@ -561,7 +561,7 @@ struct<plan:string>
 -- !query output
 == Physical Plan ==
 Execute DescribeColumnCommand
-   +- DescribeColumnCommand `default`.`t`, [b], false
+   +- DescribeColumnCommand `default`.`t`, [spark_catalog, default, t, b], false, [info_name#x, info_value#x]
 
 
 -- !query
@@ -571,7 +571,7 @@ struct<plan:string>
 -- !query output
 == Physical Plan ==
 Execute DescribeTableCommand
-   +- DescribeTableCommand `default`.`t`, Map(c -> Us, d -> 2), false
+   +- DescribeTableCommand `default`.`t`, [c=Us, d=2], false, [col_name#x, data_type#x, comment#x]
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/except-all.sql.out b/sql/core/src/test/resources/sql-tests/results/except-all.sql.out
index 601ff8f024214..a1fe952e2c032 100644
--- a/sql/core/src/test/resources/sql-tests/results/except-all.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/except-all.sql.out
@@ -141,7 +141,7 @@ SELECT array(1)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-ExceptAll can only be performed on tables with the compatible column types. array<int> <> int at the first column of the second table;
+ExceptAll can only be performed on tables with the compatible column types. array<int> <> int at the first column of the second table
 
 
 -- !query
@@ -213,7 +213,7 @@ SELECT k, v FROM tab4
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-ExceptAll can only be performed on tables with the same number of columns, but the first table has 1 columns and the second table has 2 columns;
+ExceptAll can only be performed on tables with the same number of columns, but the first table has 1 columns and the second table has 2 columns
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/except.sql.out b/sql/core/src/test/resources/sql-tests/results/except.sql.out
index 62d695219d01d..061b122eac7cf 100644
--- a/sql/core/src/test/resources/sql-tests/results/except.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/except.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 9
+-- Number of queries: 15
 
 
 -- !query
@@ -103,3 +103,59 @@ WHERE  t1.v >= (SELECT   min(t2.v)
 struct<k:string>
 -- !query output
 two
+
+
+-- !query
+CREATE OR REPLACE TEMPORARY VIEW t3 AS VALUES (decimal(1)) tbl(v)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SELECT t.v FROM (
+  SELECT v FROM t3
+  EXCEPT
+  SELECT v + v AS v FROM t3
+) t
+-- !query schema
+struct<v:decimal(11,0)>
+-- !query output
+1
+
+
+-- !query
+SELECT SUM(t.v) FROM (
+  SELECT v FROM t3
+  EXCEPT
+  SELECT v + v AS v FROM t3
+) t
+-- !query schema
+struct<sum(v):decimal(21,0)>
+-- !query output
+1
+
+
+-- !query
+DROP VIEW IF EXISTS t1
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+DROP VIEW IF EXISTS t2
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+DROP VIEW IF EXISTS t3
+-- !query schema
+struct<>
+-- !query output
+
diff --git a/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out b/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out
index 36757863ffcb5..357445a806da4 100644
--- a/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 23
+-- Number of queries: 24
 
 
 -- !query
@@ -42,6 +42,39 @@ struct<key:string,value:string>
 spark.sql.codegen.wholeStage	true
 
 
+-- !query
+EXPLAIN EXTENDED
+  SELECT sum(distinct val)
+  FROM explain_temp1
+-- !query schema
+struct<plan:string>
+-- !query output
+== Parsed Logical Plan ==
+'Project [unresolvedalias('sum(distinct 'val), None)]
++- 'UnresolvedRelation [explain_temp1], [], false
+
+== Analyzed Logical Plan ==
+sum(DISTINCT val): bigint
+Aggregate [sum(distinct val#x) AS sum(DISTINCT val)#xL]
++- SubqueryAlias spark_catalog.default.explain_temp1
+   +- Relation default.explain_temp1[key#x,val#x] parquet
+
+== Optimized Logical Plan ==
+Aggregate [sum(distinct val#x) AS sum(DISTINCT val)#xL]
++- Project [val#x]
+   +- Relation default.explain_temp1[key#x,val#x] parquet
+
+== Physical Plan ==
+AdaptiveSparkPlan isFinalPlan=false
++- HashAggregate(keys=[], functions=[sum(distinct val#x)], output=[sum(DISTINCT val)#xL])
+   +- Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#x]
+      +- HashAggregate(keys=[], functions=[partial_sum(distinct val#x)], output=[sum#xL])
+         +- HashAggregate(keys=[val#x], functions=[], output=[val#x])
+            +- Exchange hashpartitioning(val#x, 4), ENSURE_REQUIREMENTS, [id=#x]
+               +- HashAggregate(keys=[val#x], functions=[], output=[val#x])
+                  +- FileScan parquet default.explain_temp1[val#x] Batched: true, DataFilters: [], Format: Parquet, Location [not included in comparison]/{warehouse_dir}/explain_temp1], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<val:int>
+
+
 -- !query
 EXPLAIN FORMATTED
   SELECT key, max(val) 
@@ -53,15 +86,14 @@ EXPLAIN FORMATTED
 struct<plan:string>
 -- !query output
 == Physical Plan ==
-AdaptiveSparkPlan (9)
-+- Sort (8)
-   +- Exchange (7)
-      +- HashAggregate (6)
-         +- Exchange (5)
-            +- HashAggregate (4)
-               +- Project (3)
-                  +- Filter (2)
-                     +- Scan parquet default.explain_temp1 (1)
+AdaptiveSparkPlan (8)
++- Sort (7)
+   +- Exchange (6)
+      +- HashAggregate (5)
+         +- Exchange (4)
+            +- HashAggregate (3)
+               +- Filter (2)
+                  +- Scan parquet default.explain_temp1 (1)
 
 
 (1) Scan parquet default.explain_temp1
@@ -75,37 +107,33 @@ ReadSchema: struct<key:int,val:int>
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 0))
 
-(3) Project
-Output [2]: [key#x, val#x]
-Input [2]: [key#x, val#x]
-
-(4) HashAggregate
+(3) HashAggregate
 Input [2]: [key#x, val#x]
 Keys [1]: [key#x]
 Functions [1]: [partial_max(val#x)]
 Aggregate Attributes [1]: [max#x]
 Results [2]: [key#x, max#x]
 
-(5) Exchange
+(4) Exchange
 Input [2]: [key#x, max#x]
-Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+Arguments: hashpartitioning(key#x, 4), ENSURE_REQUIREMENTS, [id=#x]
 
-(6) HashAggregate
+(5) HashAggregate
 Input [2]: [key#x, max#x]
 Keys [1]: [key#x]
 Functions [1]: [max(val#x)]
 Aggregate Attributes [1]: [max(val#x)#x]
 Results [2]: [key#x, max(val#x)#x AS max(val)#x]
 
-(7) Exchange
+(6) Exchange
 Input [2]: [key#x, max(val)#x]
-Arguments: rangepartitioning(key#x ASC NULLS FIRST, 4), true, [id=#x]
+Arguments: rangepartitioning(key#x ASC NULLS FIRST, 4), ENSURE_REQUIREMENTS, [id=#x]
 
-(8) Sort
+(7) Sort
 Input [2]: [key#x, max(val)#x]
 Arguments: [key#x ASC NULLS FIRST], true, 0
 
-(9) AdaptiveSparkPlan
+(8) AdaptiveSparkPlan
 Output [2]: [key#x, max(val)#x]
 Arguments: isFinalPlan=false
 
@@ -121,15 +149,14 @@ EXPLAIN FORMATTED
 struct<plan:string>
 -- !query output
 == Physical Plan ==
-AdaptiveSparkPlan (9)
-+- Project (8)
-   +- Filter (7)
-      +- HashAggregate (6)
-         +- Exchange (5)
-            +- HashAggregate (4)
-               +- Project (3)
-                  +- Filter (2)
-                     +- Scan parquet default.explain_temp1 (1)
+AdaptiveSparkPlan (8)
++- Project (7)
+   +- Filter (6)
+      +- HashAggregate (5)
+         +- Exchange (4)
+            +- HashAggregate (3)
+               +- Filter (2)
+                  +- Scan parquet default.explain_temp1 (1)
 
 
 (1) Scan parquet default.explain_temp1
@@ -143,37 +170,33 @@ ReadSchema: struct<key:int,val:int>
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 0))
 
-(3) Project
-Output [2]: [key#x, val#x]
-Input [2]: [key#x, val#x]
-
-(4) HashAggregate
+(3) HashAggregate
 Input [2]: [key#x, val#x]
 Keys [1]: [key#x]
 Functions [1]: [partial_max(val#x)]
 Aggregate Attributes [1]: [max#x]
 Results [2]: [key#x, max#x]
 
-(5) Exchange
+(4) Exchange
 Input [2]: [key#x, max#x]
-Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+Arguments: hashpartitioning(key#x, 4), ENSURE_REQUIREMENTS, [id=#x]
 
-(6) HashAggregate
+(5) HashAggregate
 Input [2]: [key#x, max#x]
 Keys [1]: [key#x]
 Functions [1]: [max(val#x)]
 Aggregate Attributes [1]: [max(val#x)#x]
 Results [3]: [key#x, max(val#x)#x AS max(val)#x, max(val#x)#x AS max(val#x)#x]
 
-(7) Filter
+(6) Filter
 Input [3]: [key#x, max(val)#x, max(val#x)#x]
 Condition : (isnotnull(max(val#x)#x) AND (max(val#x)#x > 0))
 
-(8) Project
+(7) Project
 Output [2]: [key#x, max(val)#x]
 Input [3]: [key#x, max(val)#x, max(val#x)#x]
 
-(9) AdaptiveSparkPlan
+(8) AdaptiveSparkPlan
 Output [2]: [key#x, max(val)#x]
 Arguments: isFinalPlan=false
 
@@ -182,22 +205,20 @@ Arguments: isFinalPlan=false
 EXPLAIN FORMATTED
   SELECT key, val FROM explain_temp1 WHERE key > 0
   UNION 
-  SELECT key, val FROM explain_temp1 WHERE key > 0
+  SELECT key, val FROM explain_temp1 WHERE key > 1
 -- !query schema
 struct<plan:string>
 -- !query output
 == Physical Plan ==
-AdaptiveSparkPlan (11)
-+- HashAggregate (10)
-   +- Exchange (9)
-      +- HashAggregate (8)
-         +- Union (7)
-            :- Project (3)
-            :  +- Filter (2)
-            :     +- Scan parquet default.explain_temp1 (1)
-            +- Project (6)
-               +- Filter (5)
-                  +- Scan parquet default.explain_temp1 (4)
+AdaptiveSparkPlan (9)
++- HashAggregate (8)
+   +- Exchange (7)
+      +- HashAggregate (6)
+         +- Union (5)
+            :- Filter (2)
+            :  +- Scan parquet default.explain_temp1 (1)
+            +- Filter (4)
+               +- Scan parquet default.explain_temp1 (3)
 
 
 (1) Scan parquet default.explain_temp1
@@ -211,46 +232,38 @@ ReadSchema: struct<key:int,val:int>
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 0))
 
-(3) Project
-Output [2]: [key#x, val#x]
-Input [2]: [key#x, val#x]
-
-(4) Scan parquet default.explain_temp1
+(3) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
-PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
+PushedFilters: [IsNotNull(key), GreaterThan(key,1)]
 ReadSchema: struct<key:int,val:int>
 
-(5) Filter
-Input [2]: [key#x, val#x]
-Condition : (isnotnull(key#x) AND (key#x > 0))
-
-(6) Project
-Output [2]: [key#x, val#x]
+(4) Filter
 Input [2]: [key#x, val#x]
+Condition : (isnotnull(key#x) AND (key#x > 1))
 
-(7) Union
+(5) Union
 
-(8) HashAggregate
+(6) HashAggregate
 Input [2]: [key#x, val#x]
 Keys [2]: [key#x, val#x]
 Functions: []
 Aggregate Attributes: []
 Results [2]: [key#x, val#x]
 
-(9) Exchange
+(7) Exchange
 Input [2]: [key#x, val#x]
-Arguments: hashpartitioning(key#x, val#x, 4), true, [id=#x]
+Arguments: hashpartitioning(key#x, val#x, 4), ENSURE_REQUIREMENTS, [id=#x]
 
-(10) HashAggregate
+(8) HashAggregate
 Input [2]: [key#x, val#x]
 Keys [2]: [key#x, val#x]
 Functions: []
 Aggregate Attributes: []
 Results [2]: [key#x, val#x]
 
-(11) AdaptiveSparkPlan
+(9) AdaptiveSparkPlan
 Output [2]: [key#x, val#x]
 Arguments: isFinalPlan=false
 
@@ -265,15 +278,13 @@ EXPLAIN FORMATTED
 struct<plan:string>
 -- !query output
 == Physical Plan ==
-AdaptiveSparkPlan (9)
-+- BroadcastHashJoin Inner BuildRight (8)
-   :- Project (3)
-   :  +- Filter (2)
-   :     +- Scan parquet default.explain_temp1 (1)
-   +- BroadcastExchange (7)
-      +- Project (6)
-         +- Filter (5)
-            +- Scan parquet default.explain_temp2 (4)
+AdaptiveSparkPlan (7)
++- BroadcastHashJoin Inner BuildRight (6)
+   :- Filter (2)
+   :  +- Scan parquet default.explain_temp1 (1)
+   +- BroadcastExchange (5)
+      +- Filter (4)
+         +- Scan parquet default.explain_temp2 (3)
 
 
 (1) Scan parquet default.explain_temp1
@@ -287,35 +298,27 @@ ReadSchema: struct<key:int,val:int>
 Input [2]: [key#x, val#x]
 Condition : isnotnull(key#x)
 
-(3) Project
-Output [2]: [key#x, val#x]
-Input [2]: [key#x, val#x]
-
-(4) Scan parquet default.explain_temp2
+(3) Scan parquet default.explain_temp2
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp2]
 PushedFilters: [IsNotNull(key)]
 ReadSchema: struct<key:int,val:int>
 
-(5) Filter
+(4) Filter
 Input [2]: [key#x, val#x]
 Condition : isnotnull(key#x)
 
-(6) Project
-Output [2]: [key#x, val#x]
-Input [2]: [key#x, val#x]
-
-(7) BroadcastExchange
+(5) BroadcastExchange
 Input [2]: [key#x, val#x]
-Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#x]
 
-(8) BroadcastHashJoin
+(6) BroadcastHashJoin
 Left keys [1]: [key#x]
 Right keys [1]: [key#x]
 Join condition: None
 
-(9) AdaptiveSparkPlan
+(7) AdaptiveSparkPlan
 Output [4]: [key#x, val#x, key#x, val#x]
 Arguments: isFinalPlan=false
 
@@ -330,13 +333,12 @@ EXPLAIN FORMATTED
 struct<plan:string>
 -- !query output
 == Physical Plan ==
-AdaptiveSparkPlan (7)
-+- BroadcastHashJoin LeftOuter BuildRight (6)
+AdaptiveSparkPlan (6)
++- BroadcastHashJoin LeftOuter BuildRight (5)
    :- Scan parquet default.explain_temp1 (1)
-   +- BroadcastExchange (5)
-      +- Project (4)
-         +- Filter (3)
-            +- Scan parquet default.explain_temp2 (2)
+   +- BroadcastExchange (4)
+      +- Filter (3)
+         +- Scan parquet default.explain_temp2 (2)
 
 
 (1) Scan parquet default.explain_temp1
@@ -356,20 +358,16 @@ ReadSchema: struct<key:int,val:int>
 Input [2]: [key#x, val#x]
 Condition : isnotnull(key#x)
 
-(4) Project
-Output [2]: [key#x, val#x]
+(4) BroadcastExchange
 Input [2]: [key#x, val#x]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#x]
 
-(5) BroadcastExchange
-Input [2]: [key#x, val#x]
-Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
-
-(6) BroadcastHashJoin
+(5) BroadcastHashJoin
 Left keys [1]: [key#x]
 Right keys [1]: [key#x]
 Join condition: None
 
-(7) AdaptiveSparkPlan
+(6) AdaptiveSparkPlan
 Output [4]: [key#x, val#x, key#x, val#x]
 Arguments: isFinalPlan=false
 
@@ -389,10 +387,9 @@ EXPLAIN FORMATTED
 struct<plan:string>
 -- !query output
 == Physical Plan ==
-AdaptiveSparkPlan (4)
-+- Project (3)
-   +- Filter (2)
-      +- Scan parquet default.explain_temp1 (1)
+AdaptiveSparkPlan (3)
++- Filter (2)
+   +- Scan parquet default.explain_temp1 (1)
 
 
 (1) Scan parquet default.explain_temp1
@@ -406,12 +403,104 @@ ReadSchema: struct<key:int,val:int>
 Input [2]: [key#x, val#x]
 Condition : (((isnotnull(key#x) AND isnotnull(val#x)) AND (key#x = Subquery subquery#x, [id=#x])) AND (val#x > 3))
 
-(3) Project
+(3) AdaptiveSparkPlan
+Output [2]: [key#x, val#x]
+Arguments: isFinalPlan=false
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#x, [id=#x]
+AdaptiveSparkPlan (10)
++- HashAggregate (9)
+   +- Exchange (8)
+      +- HashAggregate (7)
+         +- Project (6)
+            +- Filter (5)
+               +- Scan parquet default.explain_temp2 (4)
+
+
+(4) Scan parquet default.explain_temp2
 Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp2]
+PushedFilters: [IsNotNull(key), IsNotNull(val), EqualTo(val,2)]
+ReadSchema: struct<key:int,val:int>
+
+(5) Filter
 Input [2]: [key#x, val#x]
+Condition : (((isnotnull(key#x) AND isnotnull(val#x)) AND (key#x = Subquery subquery#x, [id=#x])) AND (val#x = 2))
+
+(6) Project
+Output [1]: [key#x]
+Input [2]: [key#x, val#x]
+
+(7) HashAggregate
+Input [1]: [key#x]
+Keys: []
+Functions [1]: [partial_max(key#x)]
+Aggregate Attributes [1]: [max#x]
+Results [1]: [max#x]
+
+(8) Exchange
+Input [1]: [max#x]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#x]
+
+(9) HashAggregate
+Input [1]: [max#x]
+Keys: []
+Functions [1]: [max(key#x)]
+Aggregate Attributes [1]: [max(key#x)#x]
+Results [1]: [max(key#x)#x AS max(key)#x]
+
+(10) AdaptiveSparkPlan
+Output [1]: [max(key)#x]
+Arguments: isFinalPlan=false
 
-(4) AdaptiveSparkPlan
+Subquery:2 Hosting operator id = 5 Hosting Expression = Subquery subquery#x, [id=#x]
+AdaptiveSparkPlan (17)
++- HashAggregate (16)
+   +- Exchange (15)
+      +- HashAggregate (14)
+         +- Project (13)
+            +- Filter (12)
+               +- Scan parquet default.explain_temp3 (11)
+
+
+(11) Scan parquet default.explain_temp3
 Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp3]
+PushedFilters: [IsNotNull(val), GreaterThan(val,0)]
+ReadSchema: struct<key:int,val:int>
+
+(12) Filter
+Input [2]: [key#x, val#x]
+Condition : (isnotnull(val#x) AND (val#x > 0))
+
+(13) Project
+Output [1]: [key#x]
+Input [2]: [key#x, val#x]
+
+(14) HashAggregate
+Input [1]: [key#x]
+Keys: []
+Functions [1]: [partial_max(key#x)]
+Aggregate Attributes [1]: [max#x]
+Results [1]: [max#x]
+
+(15) Exchange
+Input [1]: [max#x]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#x]
+
+(16) HashAggregate
+Input [1]: [max#x]
+Keys: []
+Functions [1]: [max(key#x)]
+Aggregate Attributes [1]: [max(key#x)#x]
+Results [1]: [max(key#x)#x AS max(key)#x]
+
+(17) AdaptiveSparkPlan
+Output [1]: [max(key)#x]
 Arguments: isFinalPlan=false
 
 
@@ -449,6 +538,102 @@ Condition : ((key#x = Subquery subquery#x, [id=#x]) OR (cast(key#x as double) =
 Output [2]: [key#x, val#x]
 Arguments: isFinalPlan=false
 
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#x, [id=#x]
+AdaptiveSparkPlan (10)
++- HashAggregate (9)
+   +- Exchange (8)
+      +- HashAggregate (7)
+         +- Project (6)
+            +- Filter (5)
+               +- Scan parquet default.explain_temp2 (4)
+
+
+(4) Scan parquet default.explain_temp2
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp2]
+PushedFilters: [IsNotNull(val), GreaterThan(val,0)]
+ReadSchema: struct<key:int,val:int>
+
+(5) Filter
+Input [2]: [key#x, val#x]
+Condition : (isnotnull(val#x) AND (val#x > 0))
+
+(6) Project
+Output [1]: [key#x]
+Input [2]: [key#x, val#x]
+
+(7) HashAggregate
+Input [1]: [key#x]
+Keys: []
+Functions [1]: [partial_max(key#x)]
+Aggregate Attributes [1]: [max#x]
+Results [1]: [max#x]
+
+(8) Exchange
+Input [1]: [max#x]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#x]
+
+(9) HashAggregate
+Input [1]: [max#x]
+Keys: []
+Functions [1]: [max(key#x)]
+Aggregate Attributes [1]: [max(key#x)#x]
+Results [1]: [max(key#x)#x AS max(key)#x]
+
+(10) AdaptiveSparkPlan
+Output [1]: [max(key)#x]
+Arguments: isFinalPlan=false
+
+Subquery:2 Hosting operator id = 2 Hosting Expression = Subquery subquery#x, [id=#x]
+AdaptiveSparkPlan (17)
++- HashAggregate (16)
+   +- Exchange (15)
+      +- HashAggregate (14)
+         +- Project (13)
+            +- Filter (12)
+               +- Scan parquet default.explain_temp3 (11)
+
+
+(11) Scan parquet default.explain_temp3
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp3]
+PushedFilters: [IsNotNull(val), GreaterThan(val,0)]
+ReadSchema: struct<key:int,val:int>
+
+(12) Filter
+Input [2]: [key#x, val#x]
+Condition : (isnotnull(val#x) AND (val#x > 0))
+
+(13) Project
+Output [1]: [key#x]
+Input [2]: [key#x, val#x]
+
+(14) HashAggregate
+Input [1]: [key#x]
+Keys: []
+Functions [1]: [partial_avg(key#x)]
+Aggregate Attributes [2]: [sum#x, count#xL]
+Results [2]: [sum#x, count#xL]
+
+(15) Exchange
+Input [2]: [sum#x, count#xL]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#x]
+
+(16) HashAggregate
+Input [2]: [sum#x, count#xL]
+Keys: []
+Functions [1]: [avg(key#x)]
+Aggregate Attributes [1]: [avg(key#x)#x]
+Results [1]: [avg(key#x)#x AS avg(key)#x]
+
+(17) AdaptiveSparkPlan
+Output [1]: [avg(key)#x]
+Arguments: isFinalPlan=false
+
 
 -- !query
 EXPLAIN FORMATTED
@@ -477,6 +662,80 @@ Input: []
 Output [1]: [(scalarsubquery() + scalarsubquery())#x]
 Arguments: isFinalPlan=false
 
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#x, [id=#x]
+AdaptiveSparkPlan (8)
++- HashAggregate (7)
+   +- Exchange (6)
+      +- HashAggregate (5)
+         +- Scan parquet default.explain_temp1 (4)
+
+
+(4) Scan parquet default.explain_temp1
+Output [1]: [key#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+ReadSchema: struct<key:int>
+
+(5) HashAggregate
+Input [1]: [key#x]
+Keys: []
+Functions [1]: [partial_avg(key#x)]
+Aggregate Attributes [2]: [sum#x, count#xL]
+Results [2]: [sum#x, count#xL]
+
+(6) Exchange
+Input [2]: [sum#x, count#xL]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#x]
+
+(7) HashAggregate
+Input [2]: [sum#x, count#xL]
+Keys: []
+Functions [1]: [avg(key#x)]
+Aggregate Attributes [1]: [avg(key#x)#x]
+Results [1]: [avg(key#x)#x AS avg(key)#x]
+
+(8) AdaptiveSparkPlan
+Output [1]: [avg(key)#x]
+Arguments: isFinalPlan=false
+
+Subquery:2 Hosting operator id = 2 Hosting Expression = Subquery subquery#x, [id=#x]
+AdaptiveSparkPlan (13)
++- HashAggregate (12)
+   +- Exchange (11)
+      +- HashAggregate (10)
+         +- Scan parquet default.explain_temp1 (9)
+
+
+(9) Scan parquet default.explain_temp1
+Output [1]: [key#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+ReadSchema: struct<key:int>
+
+(10) HashAggregate
+Input [1]: [key#x]
+Keys: []
+Functions [1]: [partial_avg(key#x)]
+Aggregate Attributes [2]: [sum#x, count#xL]
+Results [2]: [sum#x, count#xL]
+
+(11) Exchange
+Input [2]: [sum#x, count#xL]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#x]
+
+(12) HashAggregate
+Input [2]: [sum#x, count#xL]
+Keys: []
+Functions [1]: [avg(key#x)]
+Aggregate Attributes [1]: [avg(key#x)#x]
+Results [1]: [avg(key#x)#x AS avg(key)#x]
+
+(13) AdaptiveSparkPlan
+Output [1]: [avg(key)#x]
+Arguments: isFinalPlan=false
+
 
 -- !query
 EXPLAIN FORMATTED
@@ -490,15 +749,13 @@ EXPLAIN FORMATTED
 struct<plan:string>
 -- !query output
 == Physical Plan ==
-AdaptiveSparkPlan (9)
-+- BroadcastHashJoin Inner BuildRight (8)
-   :- Project (3)
-   :  +- Filter (2)
-   :     +- Scan parquet default.explain_temp1 (1)
-   +- BroadcastExchange (7)
-      +- Project (6)
-         +- Filter (5)
-            +- Scan parquet default.explain_temp1 (4)
+AdaptiveSparkPlan (7)
++- BroadcastHashJoin Inner BuildRight (6)
+   :- Filter (2)
+   :  +- Scan parquet default.explain_temp1 (1)
+   +- BroadcastExchange (5)
+      +- Filter (4)
+         +- Scan parquet default.explain_temp1 (3)
 
 
 (1) Scan parquet default.explain_temp1
@@ -512,35 +769,27 @@ ReadSchema: struct<key:int,val:int>
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 10))
 
-(3) Project
-Output [2]: [key#x, val#x]
-Input [2]: [key#x, val#x]
-
-(4) Scan parquet default.explain_temp1
+(3) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,10)]
 ReadSchema: struct<key:int,val:int>
 
-(5) Filter
+(4) Filter
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 10))
 
-(6) Project
-Output [2]: [key#x, val#x]
-Input [2]: [key#x, val#x]
-
-(7) BroadcastExchange
+(5) BroadcastExchange
 Input [2]: [key#x, val#x]
-Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#x]
 
-(8) BroadcastHashJoin
+(6) BroadcastHashJoin
 Left keys [1]: [key#x]
 Right keys [1]: [key#x]
 Join condition: None
 
-(9) AdaptiveSparkPlan
+(7) AdaptiveSparkPlan
 Output [4]: [key#x, val#x, key#x, val#x]
 Arguments: isFinalPlan=false
 
@@ -558,21 +807,19 @@ EXPLAIN FORMATTED
 struct<plan:string>
 -- !query output
 == Physical Plan ==
-AdaptiveSparkPlan (15)
-+- BroadcastHashJoin Inner BuildRight (14)
-   :- HashAggregate (6)
-   :  +- Exchange (5)
-   :     +- HashAggregate (4)
-   :        +- Project (3)
-   :           +- Filter (2)
-   :              +- Scan parquet default.explain_temp1 (1)
-   +- BroadcastExchange (13)
-      +- HashAggregate (12)
-         +- Exchange (11)
-            +- HashAggregate (10)
-               +- Project (9)
-                  +- Filter (8)
-                     +- Scan parquet default.explain_temp1 (7)
+AdaptiveSparkPlan (13)
++- BroadcastHashJoin Inner BuildRight (12)
+   :- HashAggregate (5)
+   :  +- Exchange (4)
+   :     +- HashAggregate (3)
+   :        +- Filter (2)
+   :           +- Scan parquet default.explain_temp1 (1)
+   +- BroadcastExchange (11)
+      +- HashAggregate (10)
+         +- Exchange (9)
+            +- HashAggregate (8)
+               +- Filter (7)
+                  +- Scan parquet default.explain_temp1 (6)
 
 
 (1) Scan parquet default.explain_temp1
@@ -586,71 +833,63 @@ ReadSchema: struct<key:int,val:int>
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 10))
 
-(3) Project
-Output [2]: [key#x, val#x]
-Input [2]: [key#x, val#x]
-
-(4) HashAggregate
+(3) HashAggregate
 Input [2]: [key#x, val#x]
 Keys [1]: [key#x]
 Functions [1]: [partial_max(val#x)]
 Aggregate Attributes [1]: [max#x]
 Results [2]: [key#x, max#x]
 
-(5) Exchange
+(4) Exchange
 Input [2]: [key#x, max#x]
-Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+Arguments: hashpartitioning(key#x, 4), ENSURE_REQUIREMENTS, [id=#x]
 
-(6) HashAggregate
+(5) HashAggregate
 Input [2]: [key#x, max#x]
 Keys [1]: [key#x]
 Functions [1]: [max(val#x)]
 Aggregate Attributes [1]: [max(val#x)#x]
 Results [2]: [key#x, max(val#x)#x AS max(val)#x]
 
-(7) Scan parquet default.explain_temp1
+(6) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,10)]
 ReadSchema: struct<key:int,val:int>
 
-(8) Filter
+(7) Filter
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 10))
 
-(9) Project
-Output [2]: [key#x, val#x]
-Input [2]: [key#x, val#x]
-
-(10) HashAggregate
+(8) HashAggregate
 Input [2]: [key#x, val#x]
 Keys [1]: [key#x]
 Functions [1]: [partial_max(val#x)]
 Aggregate Attributes [1]: [max#x]
 Results [2]: [key#x, max#x]
 
-(11) Exchange
+(9) Exchange
 Input [2]: [key#x, max#x]
-Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+Arguments: hashpartitioning(key#x, 4), ENSURE_REQUIREMENTS, [id=#x]
 
-(12) HashAggregate
+(10) HashAggregate
 Input [2]: [key#x, max#x]
 Keys [1]: [key#x]
 Functions [1]: [max(val#x)]
 Aggregate Attributes [1]: [max(val#x)#x]
 Results [2]: [key#x, max(val#x)#x AS max(val)#x]
 
-(13) BroadcastExchange
+(11) BroadcastExchange
 Input [2]: [key#x, max(val)#x]
-Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#x]
 
-(14) BroadcastHashJoin
+(12) BroadcastHashJoin
 Left keys [1]: [key#x]
 Right keys [1]: [key#x]
 Join condition: None
 
-(15) AdaptiveSparkPlan
+(13) AdaptiveSparkPlan
 Output [4]: [key#x, max(val)#x, key#x, max(val)#x]
 Arguments: isFinalPlan=false
 
@@ -665,21 +904,37 @@ struct<plan:string>
 == Physical Plan ==
 Execute CreateViewCommand (1)
    +- CreateViewCommand (2)
-         +- Project (4)
-            +- UnresolvedRelation (3)
+         +- Project (5)
+            +- SubqueryAlias (4)
+               +- LogicalRelation (3)
 
 
 (1) Execute CreateViewCommand
 Output: []
 
 (2) CreateViewCommand
-Arguments: `default`.`explain_view`, SELECT key, val FROM explain_temp1, false, false, PersistedView
-
-(3) UnresolvedRelation
-Arguments: [explain_temp1]
-
-(4) Project
-Arguments: ['key, 'val]
+Arguments: `default`.`explain_view`, SELECT key, val FROM explain_temp1, false, false, PersistedView, true
+
+(3) LogicalRelation
+Arguments: parquet, [key#x, val#x], CatalogTable(
+Database: default
+Table: explain_temp1
+Created Time [not included in comparison]
+Last Access [not included in comparison]
+Created By [not included in comparison]
+Type: MANAGED
+Provider: PARQUET
+Location [not included in comparison]/{warehouse_dir}/explain_temp1
+Schema: root
+-- key: integer (nullable = true)
+-- val: integer (nullable = true)
+), false
+
+(4) SubqueryAlias
+Arguments: spark_catalog.default.explain_temp1
+
+(5) Project
+Arguments: [key#x, val#x]
 
 
 -- !query
@@ -708,20 +963,20 @@ ReadSchema: struct<key:int,val:int>
 (2) HashAggregate
 Input [2]: [key#x, val#x]
 Keys: []
-Functions [3]: [partial_count(val#x), partial_sum(cast(key#x as bigint)), partial_count(key#x) FILTER (WHERE (val#x > 1))]
+Functions [3]: [partial_count(val#x), partial_sum(key#x), partial_count(key#x) FILTER (WHERE (val#x > 1))]
 Aggregate Attributes [3]: [count#xL, sum#xL, count#xL]
 Results [3]: [count#xL, sum#xL, count#xL]
 
 (3) Exchange
 Input [3]: [count#xL, sum#xL, count#xL]
-Arguments: SinglePartition, true, [id=#x]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#x]
 
 (4) HashAggregate
 Input [3]: [count#xL, sum#xL, count#xL]
 Keys: []
-Functions [3]: [count(val#x), sum(cast(key#x as bigint)), count(key#x)]
-Aggregate Attributes [3]: [count(val#x)#xL, sum(cast(key#x as bigint))#xL, count(key#x)#xL]
-Results [2]: [(count(val#x)#xL + sum(cast(key#x as bigint))#xL) AS TOTAL#xL, count(key#x)#xL AS count(key) FILTER (WHERE (val > 1))#xL]
+Functions [3]: [count(val#x), sum(key#x), count(key#x)]
+Aggregate Attributes [3]: [count(val#x)#xL, sum(key#x)#xL, count(key#x)#xL]
+Results [2]: [(count(val#x)#xL + sum(key#x)#xL) AS TOTAL#xL, count(key#x)#xL AS count(key) FILTER (WHERE (val > 1))#xL]
 
 (5) AdaptiveSparkPlan
 Output [2]: [TOTAL#xL, count(key) FILTER (WHERE (val > 1))#xL]
@@ -759,7 +1014,7 @@ Results [2]: [key#x, buf#x]
 
 (3) Exchange
 Input [2]: [key#x, buf#x]
-Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+Arguments: hashpartitioning(key#x, 4), ENSURE_REQUIREMENTS, [id=#x]
 
 (4) ObjectHashAggregate
 Input [2]: [key#x, buf#x]
@@ -810,7 +1065,7 @@ Results [2]: [key#x, min#x]
 
 (4) Exchange
 Input [2]: [key#x, min#x]
-Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+Arguments: hashpartitioning(key#x, 4), ENSURE_REQUIREMENTS, [id=#x]
 
 (5) Sort
 Input [2]: [key#x, min#x]
diff --git a/sql/core/src/test/resources/sql-tests/results/explain-cbo.sql.out b/sql/core/src/test/resources/sql-tests/results/explain-cbo.sql.out
new file mode 100644
index 0000000000000..086b4d2e0b194
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/explain-cbo.sql.out
@@ -0,0 +1,96 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 7
+
+
+-- !query
+CREATE TABLE explain_temp1(a INT, b INT) USING PARQUET
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+CREATE TABLE explain_temp2(c INT, d INT) USING PARQUET
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+ANALYZE TABLE explain_temp1 COMPUTE STATISTICS FOR ALL COLUMNS
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+ANALYZE TABLE explain_temp2 COMPUTE STATISTICS FOR ALL COLUMNS
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+EXPLAIN COST WITH max_store_sales AS
+(
+  SELECT max(csales) tpcds_cmax
+  FROM (
+    SELECT sum(b) csales
+    FROM explain_temp1 WHERE a < 100
+  ) x
+),
+best_ss_customer AS
+(
+  SELECT c
+  FROM explain_temp2
+  WHERE d > (SELECT * FROM max_store_sales)
+)
+SELECT c FROM best_ss_customer
+-- !query schema
+struct<plan:string>
+-- !query output
+== Optimized Logical Plan ==
+Project [c#x], Statistics(sizeInBytes=1.0 B, rowCount=0)
++- Filter (isnotnull(d#x) AND (cast(d#x as bigint) > scalar-subquery#x [])), Statistics(sizeInBytes=1.0 B, rowCount=0)
+   :  +- Aggregate [max(csales#xL) AS tpcds_cmax#xL], Statistics(sizeInBytes=16.0 B, rowCount=1)
+   :     +- Aggregate [sum(b#x) AS csales#xL], Statistics(sizeInBytes=16.0 B, rowCount=1)
+   :        +- Project [b#x], Statistics(sizeInBytes=1.0 B, rowCount=0)
+   :           +- Filter (isnotnull(a#x) AND (a#x < 100)), Statistics(sizeInBytes=1.0 B, rowCount=0)
+   :              +- Relation default.explain_temp1[a#x,b#x] parquet, Statistics(sizeInBytes=1.0 B, rowCount=0)
+   +- Relation default.explain_temp2[c#x,d#x] parquet, Statistics(sizeInBytes=1.0 B, rowCount=0)
+
+== Physical Plan ==
+AdaptiveSparkPlan isFinalPlan=false
++- Project [c#x]
+   +- Filter (isnotnull(d#x) AND (cast(d#x as bigint) > Subquery subquery#x, [id=#x]))
+      :  +- Subquery subquery#x, [id=#x]
+      :     +- AdaptiveSparkPlan isFinalPlan=false
+      :        +- HashAggregate(keys=[], functions=[max(csales#xL)], output=[tpcds_cmax#xL])
+      :           +- HashAggregate(keys=[], functions=[partial_max(csales#xL)], output=[max#xL])
+      :              +- HashAggregate(keys=[], functions=[sum(b#x)], output=[csales#xL])
+      :                 +- Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#x]
+      :                    +- HashAggregate(keys=[], functions=[partial_sum(b#x)], output=[sum#xL])
+      :                       +- Project [b#x]
+      :                          +- Filter (isnotnull(a#x) AND (a#x < 100))
+      :                             +- FileScan parquet default.explain_temp1[a#x,b#x] Batched: true, DataFilters: [isnotnull(a#x), (a#x < 100)], Format: Parquet, Location [not included in comparison]/{warehouse_dir}/explain_temp1], PartitionFilters: [], PushedFilters: [IsNotNull(a), LessThan(a,100)], ReadSchema: struct<a:int,b:int>
+      +- FileScan parquet default.explain_temp2[c#x,d#x] Batched: true, DataFilters: [isnotnull(d#x)], Format: Parquet, Location [not included in comparison]/{warehouse_dir}/explain_temp2], PartitionFilters: [], PushedFilters: [IsNotNull(d)], ReadSchema: struct<c:int,d:int>
+
+
+-- !query
+DROP TABLE explain_temp1
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+DROP TABLE explain_temp2
+-- !query schema
+struct<>
+-- !query output
+
diff --git a/sql/core/src/test/resources/sql-tests/results/explain.sql.out b/sql/core/src/test/resources/sql-tests/results/explain.sql.out
index 2b07dac0e5d0a..3d00872028fcb 100644
--- a/sql/core/src/test/resources/sql-tests/results/explain.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/explain.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 23
+-- Number of queries: 24
 
 
 -- !query
@@ -42,6 +42,39 @@ struct<key:string,value:string>
 spark.sql.codegen.wholeStage	true
 
 
+-- !query
+EXPLAIN EXTENDED
+  SELECT sum(distinct val)
+  FROM explain_temp1
+-- !query schema
+struct<plan:string>
+-- !query output
+== Parsed Logical Plan ==
+'Project [unresolvedalias('sum(distinct 'val), None)]
++- 'UnresolvedRelation [explain_temp1], [], false
+
+== Analyzed Logical Plan ==
+sum(DISTINCT val): bigint
+Aggregate [sum(distinct val#x) AS sum(DISTINCT val)#xL]
++- SubqueryAlias spark_catalog.default.explain_temp1
+   +- Relation default.explain_temp1[key#x,val#x] parquet
+
+== Optimized Logical Plan ==
+Aggregate [sum(distinct val#x) AS sum(DISTINCT val)#xL]
++- Project [val#x]
+   +- Relation default.explain_temp1[key#x,val#x] parquet
+
+== Physical Plan ==
+*HashAggregate(keys=[], functions=[sum(distinct val#x)], output=[sum(DISTINCT val)#xL])
++- Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#x]
+   +- *HashAggregate(keys=[], functions=[partial_sum(distinct val#x)], output=[sum#xL])
+      +- *HashAggregate(keys=[val#x], functions=[], output=[val#x])
+         +- Exchange hashpartitioning(val#x, 4), ENSURE_REQUIREMENTS, [id=#x]
+            +- *HashAggregate(keys=[val#x], functions=[], output=[val#x])
+               +- *ColumnarToRow
+                  +- FileScan parquet default.explain_temp1[val#x] Batched: true, DataFilters: [], Format: Parquet, Location [not included in comparison]/{warehouse_dir}/explain_temp1], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<val:int>
+
+
 -- !query
 EXPLAIN FORMATTED
   SELECT key, max(val) 
@@ -53,15 +86,14 @@ EXPLAIN FORMATTED
 struct<plan:string>
 -- !query output
 == Physical Plan ==
-* Sort (9)
-+- Exchange (8)
-   +- * HashAggregate (7)
-      +- Exchange (6)
-         +- * HashAggregate (5)
-            +- * Project (4)
-               +- * Filter (3)
-                  +- * ColumnarToRow (2)
-                     +- Scan parquet default.explain_temp1 (1)
+* Sort (8)
++- Exchange (7)
+   +- * HashAggregate (6)
+      +- Exchange (5)
+         +- * HashAggregate (4)
+            +- * Filter (3)
+               +- * ColumnarToRow (2)
+                  +- Scan parquet default.explain_temp1 (1)
 
 
 (1) Scan parquet default.explain_temp1
@@ -78,33 +110,29 @@ Input [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 0))
 
-(4) Project [codegen id : 1]
-Output [2]: [key#x, val#x]
-Input [2]: [key#x, val#x]
-
-(5) HashAggregate [codegen id : 1]
+(4) HashAggregate [codegen id : 1]
 Input [2]: [key#x, val#x]
 Keys [1]: [key#x]
 Functions [1]: [partial_max(val#x)]
 Aggregate Attributes [1]: [max#x]
 Results [2]: [key#x, max#x]
 
-(6) Exchange
+(5) Exchange
 Input [2]: [key#x, max#x]
-Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+Arguments: hashpartitioning(key#x, 4), ENSURE_REQUIREMENTS, [id=#x]
 
-(7) HashAggregate [codegen id : 2]
+(6) HashAggregate [codegen id : 2]
 Input [2]: [key#x, max#x]
 Keys [1]: [key#x]
 Functions [1]: [max(val#x)]
 Aggregate Attributes [1]: [max(val#x)#x]
 Results [2]: [key#x, max(val#x)#x AS max(val)#x]
 
-(8) Exchange
+(7) Exchange
 Input [2]: [key#x, max(val)#x]
-Arguments: rangepartitioning(key#x ASC NULLS FIRST, 4), true, [id=#x]
+Arguments: rangepartitioning(key#x ASC NULLS FIRST, 4), ENSURE_REQUIREMENTS, [id=#x]
 
-(9) Sort [codegen id : 3]
+(8) Sort [codegen id : 3]
 Input [2]: [key#x, max(val)#x]
 Arguments: [key#x ASC NULLS FIRST], true, 0
 
@@ -120,15 +148,14 @@ EXPLAIN FORMATTED
 struct<plan:string>
 -- !query output
 == Physical Plan ==
-* Project (9)
-+- * Filter (8)
-   +- * HashAggregate (7)
-      +- Exchange (6)
-         +- * HashAggregate (5)
-            +- * Project (4)
-               +- * Filter (3)
-                  +- * ColumnarToRow (2)
-                     +- Scan parquet default.explain_temp1 (1)
+* Project (8)
++- * Filter (7)
+   +- * HashAggregate (6)
+      +- Exchange (5)
+         +- * HashAggregate (4)
+            +- * Filter (3)
+               +- * ColumnarToRow (2)
+                  +- Scan parquet default.explain_temp1 (1)
 
 
 (1) Scan parquet default.explain_temp1
@@ -145,33 +172,29 @@ Input [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 0))
 
-(4) Project [codegen id : 1]
-Output [2]: [key#x, val#x]
-Input [2]: [key#x, val#x]
-
-(5) HashAggregate [codegen id : 1]
+(4) HashAggregate [codegen id : 1]
 Input [2]: [key#x, val#x]
 Keys [1]: [key#x]
 Functions [1]: [partial_max(val#x)]
 Aggregate Attributes [1]: [max#x]
 Results [2]: [key#x, max#x]
 
-(6) Exchange
+(5) Exchange
 Input [2]: [key#x, max#x]
-Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+Arguments: hashpartitioning(key#x, 4), ENSURE_REQUIREMENTS, [id=#x]
 
-(7) HashAggregate [codegen id : 2]
+(6) HashAggregate [codegen id : 2]
 Input [2]: [key#x, max#x]
 Keys [1]: [key#x]
 Functions [1]: [max(val#x)]
 Aggregate Attributes [1]: [max(val#x)#x]
 Results [3]: [key#x, max(val#x)#x AS max(val)#x, max(val#x)#x AS max(val#x)#x]
 
-(8) Filter [codegen id : 2]
+(7) Filter [codegen id : 2]
 Input [3]: [key#x, max(val)#x, max(val#x)#x]
 Condition : (isnotnull(max(val#x)#x) AND (max(val#x)#x > 0))
 
-(9) Project [codegen id : 2]
+(8) Project [codegen id : 2]
 Output [2]: [key#x, max(val)#x]
 Input [3]: [key#x, max(val)#x, max(val#x)#x]
 
@@ -180,23 +203,21 @@ Input [3]: [key#x, max(val)#x, max(val#x)#x]
 EXPLAIN FORMATTED
   SELECT key, val FROM explain_temp1 WHERE key > 0
   UNION 
-  SELECT key, val FROM explain_temp1 WHERE key > 0
+  SELECT key, val FROM explain_temp1 WHERE key > 1
 -- !query schema
 struct<plan:string>
 -- !query output
 == Physical Plan ==
-* HashAggregate (12)
-+- Exchange (11)
-   +- * HashAggregate (10)
-      +- Union (9)
-         :- * Project (4)
-         :  +- * Filter (3)
-         :     +- * ColumnarToRow (2)
-         :        +- Scan parquet default.explain_temp1 (1)
-         +- * Project (8)
-            +- * Filter (7)
-               +- * ColumnarToRow (6)
-                  +- Scan parquet default.explain_temp1 (5)
+* HashAggregate (10)
++- Exchange (9)
+   +- * HashAggregate (8)
+      +- Union (7)
+         :- * Filter (3)
+         :  +- * ColumnarToRow (2)
+         :     +- Scan parquet default.explain_temp1 (1)
+         +- * Filter (6)
+            +- * ColumnarToRow (5)
+               +- Scan parquet default.explain_temp1 (4)
 
 
 (1) Scan parquet default.explain_temp1
@@ -213,42 +234,34 @@ Input [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 0))
 
-(4) Project [codegen id : 1]
-Output [2]: [key#x, val#x]
-Input [2]: [key#x, val#x]
-
-(5) Scan parquet default.explain_temp1
+(4) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
-PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
+PushedFilters: [IsNotNull(key), GreaterThan(key,1)]
 ReadSchema: struct<key:int,val:int>
 
-(6) ColumnarToRow [codegen id : 2]
+(5) ColumnarToRow [codegen id : 2]
 Input [2]: [key#x, val#x]
 
-(7) Filter [codegen id : 2]
-Input [2]: [key#x, val#x]
-Condition : (isnotnull(key#x) AND (key#x > 0))
-
-(8) Project [codegen id : 2]
-Output [2]: [key#x, val#x]
+(6) Filter [codegen id : 2]
 Input [2]: [key#x, val#x]
+Condition : (isnotnull(key#x) AND (key#x > 1))
 
-(9) Union
+(7) Union
 
-(10) HashAggregate [codegen id : 3]
+(8) HashAggregate [codegen id : 3]
 Input [2]: [key#x, val#x]
 Keys [2]: [key#x, val#x]
 Functions: []
 Aggregate Attributes: []
 Results [2]: [key#x, val#x]
 
-(11) Exchange
+(9) Exchange
 Input [2]: [key#x, val#x]
-Arguments: hashpartitioning(key#x, val#x, 4), true, [id=#x]
+Arguments: hashpartitioning(key#x, val#x, 4), ENSURE_REQUIREMENTS, [id=#x]
 
-(12) HashAggregate [codegen id : 4]
+(10) HashAggregate [codegen id : 4]
 Input [2]: [key#x, val#x]
 Keys [2]: [key#x, val#x]
 Functions: []
@@ -266,16 +279,14 @@ EXPLAIN FORMATTED
 struct<plan:string>
 -- !query output
 == Physical Plan ==
-* BroadcastHashJoin Inner BuildRight (10)
-:- * Project (4)
-:  +- * Filter (3)
-:     +- * ColumnarToRow (2)
-:        +- Scan parquet default.explain_temp1 (1)
-+- BroadcastExchange (9)
-   +- * Project (8)
-      +- * Filter (7)
-         +- * ColumnarToRow (6)
-            +- Scan parquet default.explain_temp2 (5)
+* BroadcastHashJoin Inner BuildRight (8)
+:- * Filter (3)
+:  +- * ColumnarToRow (2)
+:     +- Scan parquet default.explain_temp1 (1)
++- BroadcastExchange (7)
+   +- * Filter (6)
+      +- * ColumnarToRow (5)
+         +- Scan parquet default.explain_temp2 (4)
 
 
 (1) Scan parquet default.explain_temp1
@@ -292,33 +303,25 @@ Input [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
 Condition : isnotnull(key#x)
 
-(4) Project [codegen id : 2]
-Output [2]: [key#x, val#x]
-Input [2]: [key#x, val#x]
-
-(5) Scan parquet default.explain_temp2
+(4) Scan parquet default.explain_temp2
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp2]
 PushedFilters: [IsNotNull(key)]
 ReadSchema: struct<key:int,val:int>
 
-(6) ColumnarToRow [codegen id : 1]
+(5) ColumnarToRow [codegen id : 1]
 Input [2]: [key#x, val#x]
 
-(7) Filter [codegen id : 1]
+(6) Filter [codegen id : 1]
 Input [2]: [key#x, val#x]
 Condition : isnotnull(key#x)
 
-(8) Project [codegen id : 1]
-Output [2]: [key#x, val#x]
-Input [2]: [key#x, val#x]
-
-(9) BroadcastExchange
+(7) BroadcastExchange
 Input [2]: [key#x, val#x]
-Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#x]
 
-(10) BroadcastHashJoin [codegen id : 2]
+(8) BroadcastHashJoin [codegen id : 2]
 Left keys [1]: [key#x]
 Right keys [1]: [key#x]
 Join condition: None
@@ -334,14 +337,13 @@ EXPLAIN FORMATTED
 struct<plan:string>
 -- !query output
 == Physical Plan ==
-* BroadcastHashJoin LeftOuter BuildRight (8)
+* BroadcastHashJoin LeftOuter BuildRight (7)
 :- * ColumnarToRow (2)
 :  +- Scan parquet default.explain_temp1 (1)
-+- BroadcastExchange (7)
-   +- * Project (6)
-      +- * Filter (5)
-         +- * ColumnarToRow (4)
-            +- Scan parquet default.explain_temp2 (3)
++- BroadcastExchange (6)
+   +- * Filter (5)
+      +- * ColumnarToRow (4)
+         +- Scan parquet default.explain_temp2 (3)
 
 
 (1) Scan parquet default.explain_temp1
@@ -367,15 +369,11 @@ Input [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
 Condition : isnotnull(key#x)
 
-(6) Project [codegen id : 1]
-Output [2]: [key#x, val#x]
-Input [2]: [key#x, val#x]
-
-(7) BroadcastExchange
+(6) BroadcastExchange
 Input [2]: [key#x, val#x]
-Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#x]
 
-(8) BroadcastHashJoin [codegen id : 2]
+(7) BroadcastHashJoin [codegen id : 2]
 Left keys [1]: [key#x]
 Right keys [1]: [key#x]
 Join condition: None
@@ -396,10 +394,9 @@ EXPLAIN FORMATTED
 struct<plan:string>
 -- !query output
 == Physical Plan ==
-* Project (4)
-+- * Filter (3)
-   +- * ColumnarToRow (2)
-      +- Scan parquet default.explain_temp1 (1)
+* Filter (3)
++- * ColumnarToRow (2)
+   +- Scan parquet default.explain_temp1 (1)
 
 
 (1) Scan parquet default.explain_temp1
@@ -416,98 +413,94 @@ Input [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
 Condition : (((isnotnull(key#x) AND isnotnull(val#x)) AND (key#x = Subquery scalar-subquery#x, [id=#x])) AND (val#x > 3))
 
-(4) Project [codegen id : 1]
-Output [2]: [key#x, val#x]
-Input [2]: [key#x, val#x]
-
 ===== Subqueries =====
 
 Subquery:1 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#x, [id=#x]
-* HashAggregate (11)
-+- Exchange (10)
-   +- * HashAggregate (9)
-      +- * Project (8)
-         +- * Filter (7)
-            +- * ColumnarToRow (6)
-               +- Scan parquet default.explain_temp2 (5)
+* HashAggregate (10)
++- Exchange (9)
+   +- * HashAggregate (8)
+      +- * Project (7)
+         +- * Filter (6)
+            +- * ColumnarToRow (5)
+               +- Scan parquet default.explain_temp2 (4)
 
 
-(5) Scan parquet default.explain_temp2
+(4) Scan parquet default.explain_temp2
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp2]
 PushedFilters: [IsNotNull(key), IsNotNull(val), EqualTo(val,2)]
 ReadSchema: struct<key:int,val:int>
 
-(6) ColumnarToRow [codegen id : 1]
+(5) ColumnarToRow [codegen id : 1]
 Input [2]: [key#x, val#x]
 
-(7) Filter [codegen id : 1]
+(6) Filter [codegen id : 1]
 Input [2]: [key#x, val#x]
 Condition : (((isnotnull(key#x) AND isnotnull(val#x)) AND (key#x = Subquery scalar-subquery#x, [id=#x])) AND (val#x = 2))
 
-(8) Project [codegen id : 1]
+(7) Project [codegen id : 1]
 Output [1]: [key#x]
 Input [2]: [key#x, val#x]
 
-(9) HashAggregate [codegen id : 1]
+(8) HashAggregate [codegen id : 1]
 Input [1]: [key#x]
 Keys: []
 Functions [1]: [partial_max(key#x)]
 Aggregate Attributes [1]: [max#x]
 Results [1]: [max#x]
 
-(10) Exchange
+(9) Exchange
 Input [1]: [max#x]
-Arguments: SinglePartition, true, [id=#x]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#x]
 
-(11) HashAggregate [codegen id : 2]
+(10) HashAggregate [codegen id : 2]
 Input [1]: [max#x]
 Keys: []
 Functions [1]: [max(key#x)]
 Aggregate Attributes [1]: [max(key#x)#x]
 Results [1]: [max(key#x)#x AS max(key)#x]
 
-Subquery:2 Hosting operator id = 7 Hosting Expression = Subquery scalar-subquery#x, [id=#x]
-* HashAggregate (18)
-+- Exchange (17)
-   +- * HashAggregate (16)
-      +- * Project (15)
-         +- * Filter (14)
-            +- * ColumnarToRow (13)
-               +- Scan parquet default.explain_temp3 (12)
+Subquery:2 Hosting operator id = 6 Hosting Expression = Subquery scalar-subquery#x, [id=#x]
+* HashAggregate (17)
++- Exchange (16)
+   +- * HashAggregate (15)
+      +- * Project (14)
+         +- * Filter (13)
+            +- * ColumnarToRow (12)
+               +- Scan parquet default.explain_temp3 (11)
 
 
-(12) Scan parquet default.explain_temp3
+(11) Scan parquet default.explain_temp3
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp3]
 PushedFilters: [IsNotNull(val), GreaterThan(val,0)]
 ReadSchema: struct<key:int,val:int>
 
-(13) ColumnarToRow [codegen id : 1]
+(12) ColumnarToRow [codegen id : 1]
 Input [2]: [key#x, val#x]
 
-(14) Filter [codegen id : 1]
+(13) Filter [codegen id : 1]
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(val#x) AND (val#x > 0))
 
-(15) Project [codegen id : 1]
+(14) Project [codegen id : 1]
 Output [1]: [key#x]
 Input [2]: [key#x, val#x]
 
-(16) HashAggregate [codegen id : 1]
+(15) HashAggregate [codegen id : 1]
 Input [1]: [key#x]
 Keys: []
 Functions [1]: [partial_max(key#x)]
 Aggregate Attributes [1]: [max#x]
 Results [1]: [max#x]
 
-(17) Exchange
+(16) Exchange
 Input [1]: [max#x]
-Arguments: SinglePartition, true, [id=#x]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#x]
 
-(18) HashAggregate [codegen id : 2]
+(17) HashAggregate [codegen id : 2]
 Input [1]: [max#x]
 Keys: []
 Functions [1]: [max(key#x)]
@@ -587,7 +580,7 @@ Results [1]: [max#x]
 
 (9) Exchange
 Input [1]: [max#x]
-Arguments: SinglePartition, true, [id=#x]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#x]
 
 (10) HashAggregate [codegen id : 2]
 Input [1]: [max#x]
@@ -627,20 +620,20 @@ Input [2]: [key#x, val#x]
 (15) HashAggregate [codegen id : 1]
 Input [1]: [key#x]
 Keys: []
-Functions [1]: [partial_avg(cast(key#x as bigint))]
+Functions [1]: [partial_avg(key#x)]
 Aggregate Attributes [2]: [sum#x, count#xL]
 Results [2]: [sum#x, count#xL]
 
 (16) Exchange
 Input [2]: [sum#x, count#xL]
-Arguments: SinglePartition, true, [id=#x]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#x]
 
 (17) HashAggregate [codegen id : 2]
 Input [2]: [sum#x, count#xL]
 Keys: []
-Functions [1]: [avg(cast(key#x as bigint))]
-Aggregate Attributes [1]: [avg(cast(key#x as bigint))#x]
-Results [1]: [avg(cast(key#x as bigint))#x AS avg(key)#x]
+Functions [1]: [avg(key#x)]
+Aggregate Attributes [1]: [avg(key#x)#x]
+Results [1]: [avg(key#x)#x AS avg(key)#x]
 
 
 -- !query
@@ -691,20 +684,20 @@ Input [1]: [key#x]
 (6) HashAggregate [codegen id : 1]
 Input [1]: [key#x]
 Keys: []
-Functions [1]: [partial_avg(cast(key#x as bigint))]
+Functions [1]: [partial_avg(key#x)]
 Aggregate Attributes [2]: [sum#x, count#xL]
 Results [2]: [sum#x, count#xL]
 
 (7) Exchange
 Input [2]: [sum#x, count#xL]
-Arguments: SinglePartition, true, [id=#x]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#x]
 
 (8) HashAggregate [codegen id : 2]
 Input [2]: [sum#x, count#xL]
 Keys: []
-Functions [1]: [avg(cast(key#x as bigint))]
-Aggregate Attributes [1]: [avg(cast(key#x as bigint))#x]
-Results [1]: [avg(cast(key#x as bigint))#x AS avg(key)#x]
+Functions [1]: [avg(key#x)]
+Aggregate Attributes [1]: [avg(key#x)#x]
+Results [1]: [avg(key#x)#x AS avg(key)#x]
 
 Subquery:2 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#x, [id=#x]
 
@@ -721,16 +714,14 @@ EXPLAIN FORMATTED
 struct<plan:string>
 -- !query output
 == Physical Plan ==
-* BroadcastHashJoin Inner BuildRight (10)
-:- * Project (4)
-:  +- * Filter (3)
-:     +- * ColumnarToRow (2)
-:        +- Scan parquet default.explain_temp1 (1)
-+- BroadcastExchange (9)
-   +- * Project (8)
-      +- * Filter (7)
-         +- * ColumnarToRow (6)
-            +- Scan parquet default.explain_temp1 (5)
+* BroadcastHashJoin Inner BuildRight (8)
+:- * Filter (3)
+:  +- * ColumnarToRow (2)
+:     +- Scan parquet default.explain_temp1 (1)
++- BroadcastExchange (7)
+   +- * Filter (6)
+      +- * ColumnarToRow (5)
+         +- Scan parquet default.explain_temp1 (4)
 
 
 (1) Scan parquet default.explain_temp1
@@ -747,33 +738,25 @@ Input [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 10))
 
-(4) Project [codegen id : 2]
-Output [2]: [key#x, val#x]
-Input [2]: [key#x, val#x]
-
-(5) Scan parquet default.explain_temp1
+(4) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,10)]
 ReadSchema: struct<key:int,val:int>
 
-(6) ColumnarToRow [codegen id : 1]
+(5) ColumnarToRow [codegen id : 1]
 Input [2]: [key#x, val#x]
 
-(7) Filter [codegen id : 1]
+(6) Filter [codegen id : 1]
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 10))
 
-(8) Project [codegen id : 1]
-Output [2]: [key#x, val#x]
-Input [2]: [key#x, val#x]
-
-(9) BroadcastExchange
+(7) BroadcastExchange
 Input [2]: [key#x, val#x]
-Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#x]
 
-(10) BroadcastHashJoin [codegen id : 2]
+(8) BroadcastHashJoin [codegen id : 2]
 Left keys [1]: [key#x]
 Right keys [1]: [key#x]
 Join condition: None
@@ -792,17 +775,16 @@ EXPLAIN FORMATTED
 struct<plan:string>
 -- !query output
 == Physical Plan ==
-* BroadcastHashJoin Inner BuildRight (11)
-:- * HashAggregate (7)
-:  +- Exchange (6)
-:     +- * HashAggregate (5)
-:        +- * Project (4)
-:           +- * Filter (3)
-:              +- * ColumnarToRow (2)
-:                 +- Scan parquet default.explain_temp1 (1)
-+- BroadcastExchange (10)
-   +- * HashAggregate (9)
-      +- ReusedExchange (8)
+* BroadcastHashJoin Inner BuildRight (10)
+:- * HashAggregate (6)
+:  +- Exchange (5)
+:     +- * HashAggregate (4)
+:        +- * Filter (3)
+:           +- * ColumnarToRow (2)
+:              +- Scan parquet default.explain_temp1 (1)
++- BroadcastExchange (9)
+   +- * HashAggregate (8)
+      +- ReusedExchange (7)
 
 
 (1) Scan parquet default.explain_temp1
@@ -819,43 +801,39 @@ Input [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 10))
 
-(4) Project [codegen id : 1]
-Output [2]: [key#x, val#x]
-Input [2]: [key#x, val#x]
-
-(5) HashAggregate [codegen id : 1]
+(4) HashAggregate [codegen id : 1]
 Input [2]: [key#x, val#x]
 Keys [1]: [key#x]
 Functions [1]: [partial_max(val#x)]
 Aggregate Attributes [1]: [max#x]
 Results [2]: [key#x, max#x]
 
-(6) Exchange
+(5) Exchange
 Input [2]: [key#x, max#x]
-Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+Arguments: hashpartitioning(key#x, 4), ENSURE_REQUIREMENTS, [id=#x]
 
-(7) HashAggregate [codegen id : 4]
+(6) HashAggregate [codegen id : 4]
 Input [2]: [key#x, max#x]
 Keys [1]: [key#x]
 Functions [1]: [max(val#x)]
 Aggregate Attributes [1]: [max(val#x)#x]
 Results [2]: [key#x, max(val#x)#x AS max(val)#x]
 
-(8) ReusedExchange [Reuses operator id: 6]
+(7) ReusedExchange [Reuses operator id: 5]
 Output [2]: [key#x, max#x]
 
-(9) HashAggregate [codegen id : 3]
+(8) HashAggregate [codegen id : 3]
 Input [2]: [key#x, max#x]
 Keys [1]: [key#x]
 Functions [1]: [max(val#x)]
 Aggregate Attributes [1]: [max(val#x)#x]
 Results [2]: [key#x, max(val#x)#x AS max(val)#x]
 
-(10) BroadcastExchange
+(9) BroadcastExchange
 Input [2]: [key#x, max(val)#x]
-Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#x]
 
-(11) BroadcastHashJoin [codegen id : 4]
+(10) BroadcastHashJoin [codegen id : 4]
 Left keys [1]: [key#x]
 Right keys [1]: [key#x]
 Join condition: None
@@ -871,21 +849,37 @@ struct<plan:string>
 == Physical Plan ==
 Execute CreateViewCommand (1)
    +- CreateViewCommand (2)
-         +- Project (4)
-            +- UnresolvedRelation (3)
+         +- Project (5)
+            +- SubqueryAlias (4)
+               +- LogicalRelation (3)
 
 
 (1) Execute CreateViewCommand
 Output: []
 
 (2) CreateViewCommand
-Arguments: `default`.`explain_view`, SELECT key, val FROM explain_temp1, false, false, PersistedView
-
-(3) UnresolvedRelation
-Arguments: [explain_temp1]
-
-(4) Project
-Arguments: ['key, 'val]
+Arguments: `default`.`explain_view`, SELECT key, val FROM explain_temp1, false, false, PersistedView, true
+
+(3) LogicalRelation
+Arguments: parquet, [key#x, val#x], CatalogTable(
+Database: default
+Table: explain_temp1
+Created Time [not included in comparison]
+Last Access [not included in comparison]
+Created By [not included in comparison]
+Type: MANAGED
+Provider: PARQUET
+Location [not included in comparison]/{warehouse_dir}/explain_temp1
+Schema: root
+-- key: integer (nullable = true)
+-- val: integer (nullable = true)
+), false
+
+(4) SubqueryAlias
+Arguments: spark_catalog.default.explain_temp1
+
+(5) Project
+Arguments: [key#x, val#x]
 
 
 -- !query
@@ -900,7 +894,7 @@ struct<plan:string>
 == Physical Plan ==
 * HashAggregate (5)
 +- Exchange (4)
-   +- HashAggregate (3)
+   +- * HashAggregate (3)
       +- * ColumnarToRow (2)
          +- Scan parquet default.explain_temp1 (1)
 
@@ -914,23 +908,23 @@ ReadSchema: struct<key:int,val:int>
 (2) ColumnarToRow [codegen id : 1]
 Input [2]: [key#x, val#x]
 
-(3) HashAggregate
+(3) HashAggregate [codegen id : 1]
 Input [2]: [key#x, val#x]
 Keys: []
-Functions [3]: [partial_count(val#x), partial_sum(cast(key#x as bigint)), partial_count(key#x) FILTER (WHERE (val#x > 1))]
+Functions [3]: [partial_count(val#x), partial_sum(key#x), partial_count(key#x) FILTER (WHERE (val#x > 1))]
 Aggregate Attributes [3]: [count#xL, sum#xL, count#xL]
 Results [3]: [count#xL, sum#xL, count#xL]
 
 (4) Exchange
 Input [3]: [count#xL, sum#xL, count#xL]
-Arguments: SinglePartition, true, [id=#x]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#x]
 
 (5) HashAggregate [codegen id : 2]
 Input [3]: [count#xL, sum#xL, count#xL]
 Keys: []
-Functions [3]: [count(val#x), sum(cast(key#x as bigint)), count(key#x)]
-Aggregate Attributes [3]: [count(val#x)#xL, sum(cast(key#x as bigint))#xL, count(key#x)#xL]
-Results [2]: [(count(val#x)#xL + sum(cast(key#x as bigint))#xL) AS TOTAL#xL, count(key#x)#xL AS count(key) FILTER (WHERE (val > 1))#xL]
+Functions [3]: [count(val#x), sum(key#x), count(key#x)]
+Aggregate Attributes [3]: [count(val#x)#xL, sum(key#x)#xL, count(key#x)#xL]
+Results [2]: [(count(val#x)#xL + sum(key#x)#xL) AS TOTAL#xL, count(key#x)#xL AS count(key) FILTER (WHERE (val > 1))#xL]
 
 
 -- !query
@@ -967,7 +961,7 @@ Results [2]: [key#x, buf#x]
 
 (4) Exchange
 Input [2]: [key#x, buf#x]
-Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+Arguments: hashpartitioning(key#x, 4), ENSURE_REQUIREMENTS, [id=#x]
 
 (5) ObjectHashAggregate
 Input [2]: [key#x, buf#x]
@@ -1017,7 +1011,7 @@ Results [2]: [key#x, min#x]
 
 (5) Exchange
 Input [2]: [key#x, min#x]
-Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+Arguments: hashpartitioning(key#x, 4), ENSURE_REQUIREMENTS, [id=#x]
 
 (6) Sort [codegen id : 2]
 Input [2]: [key#x, min#x]
diff --git a/sql/core/src/test/resources/sql-tests/results/extract.sql.out b/sql/core/src/test/resources/sql-tests/results/extract.sql.out
index 9d3fe5d17fafa..5a7cd665255c1 100644
--- a/sql/core/src/test/resources/sql-tests/results/extract.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/extract.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 100
+-- Number of queries: 125
 
 
 -- !query
@@ -197,7 +197,7 @@ struct<extract(doy FROM c):int>
 -- !query
 select extract(hour from c), extract(hour from i) from t
 -- !query schema
-struct<extract(hour FROM c):int,extract(hour FROM i):bigint>
+struct<extract(hour FROM c):int,extract(hour FROM i):tinyint>
 -- !query output
 7	16
 
@@ -205,7 +205,7 @@ struct<extract(hour FROM c):int,extract(hour FROM i):bigint>
 -- !query
 select extract(h from c), extract(h from i) from t
 -- !query schema
-struct<extract(h FROM c):int,extract(h FROM i):bigint>
+struct<extract(h FROM c):int,extract(h FROM i):tinyint>
 -- !query output
 7	16
 
@@ -213,7 +213,7 @@ struct<extract(h FROM c):int,extract(h FROM i):bigint>
 -- !query
 select extract(hours from c), extract(hours from i) from t
 -- !query schema
-struct<extract(hours FROM c):int,extract(hours FROM i):bigint>
+struct<extract(hours FROM c):int,extract(hours FROM i):tinyint>
 -- !query output
 7	16
 
@@ -221,7 +221,7 @@ struct<extract(hours FROM c):int,extract(hours FROM i):bigint>
 -- !query
 select extract(hr from c), extract(hr from i) from t
 -- !query schema
-struct<extract(hr FROM c):int,extract(hr FROM i):bigint>
+struct<extract(hr FROM c):int,extract(hr FROM i):tinyint>
 -- !query output
 7	16
 
@@ -229,7 +229,7 @@ struct<extract(hr FROM c):int,extract(hr FROM i):bigint>
 -- !query
 select extract(hrs from c), extract(hrs from i) from t
 -- !query schema
-struct<extract(hrs FROM c):int,extract(hrs FROM i):bigint>
+struct<extract(hrs FROM c):int,extract(hrs FROM i):tinyint>
 -- !query output
 7	16
 
@@ -320,7 +320,7 @@ select extract(not_supported from c) from t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Literals of type 'not_supported' are currently not supported for the string type.;; line 1 pos 7
+Literals of type 'not_supported' are currently not supported for the string type.; line 1 pos 7
 
 
 -- !query
@@ -329,7 +329,7 @@ select extract(not_supported from i) from t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Literals of type 'not_supported' are currently not supported for the interval type.;; line 1 pos 7
+Literals of type 'not_supported' are currently not supported for the interval type.; line 1 pos 7
 
 
 -- !query
@@ -519,7 +519,7 @@ struct<date_part(doy, c):int>
 -- !query
 select date_part('hour', c), date_part('hour', i) from t
 -- !query schema
-struct<date_part(hour, c):int,date_part(hour, i):bigint>
+struct<date_part(hour, c):int,date_part(hour, i):tinyint>
 -- !query output
 7	16
 
@@ -527,7 +527,7 @@ struct<date_part(hour, c):int,date_part(hour, i):bigint>
 -- !query
 select date_part('h', c), date_part('h', i) from t
 -- !query schema
-struct<date_part(h, c):int,date_part(h, i):bigint>
+struct<date_part(h, c):int,date_part(h, i):tinyint>
 -- !query output
 7	16
 
@@ -535,7 +535,7 @@ struct<date_part(h, c):int,date_part(h, i):bigint>
 -- !query
 select date_part('hours', c), date_part('hours', i) from t
 -- !query schema
-struct<date_part(hours, c):int,date_part(hours, i):bigint>
+struct<date_part(hours, c):int,date_part(hours, i):tinyint>
 -- !query output
 7	16
 
@@ -543,7 +543,7 @@ struct<date_part(hours, c):int,date_part(hours, i):bigint>
 -- !query
 select date_part('hr', c), date_part('hr', i) from t
 -- !query schema
-struct<date_part(hr, c):int,date_part(hr, i):bigint>
+struct<date_part(hr, c):int,date_part(hr, i):tinyint>
 -- !query output
 7	16
 
@@ -551,7 +551,7 @@ struct<date_part(hr, c):int,date_part(hr, i):bigint>
 -- !query
 select date_part('hrs', c), date_part('hrs', i) from t
 -- !query schema
-struct<date_part(hrs, c):int,date_part(hrs, i):bigint>
+struct<date_part(hrs, c):int,date_part(hrs, i):tinyint>
 -- !query output
 7	16
 
@@ -642,7 +642,7 @@ select date_part('not_supported', c) from t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Literals of type 'not_supported' are currently not supported for the string type.;; line 1 pos 7
+Literals of type 'not_supported' are currently not supported for the string type.; line 1 pos 7
 
 
 -- !query
@@ -651,7 +651,7 @@ select date_part(c, c) from t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-The field parameter needs to be a foldable string value.;; line 1 pos 7
+The field parameter needs to be a foldable string value.; line 1 pos 7
 
 
 -- !query
@@ -668,7 +668,7 @@ select date_part(i, i) from t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-The field parameter needs to be a foldable string value.;; line 1 pos 7
+The field parameter needs to be a foldable string value.; line 1 pos 7
 
 
 -- !query
@@ -778,7 +778,7 @@ struct<extract(second FROM c):decimal(8,6)>
 -- !query
 select c - i from t
 -- !query schema
-struct<CAST(c - i AS STRING):string>
+struct<c - i:string>
 -- !query output
 1999-08-05 14:18:02.334456
 
@@ -786,7 +786,7 @@ struct<CAST(c - i AS STRING):string>
 -- !query
 select year(c - i) from t
 -- !query schema
-struct<year(CAST(CAST(c - i AS STRING) AS DATE)):int>
+struct<year(c - i):int>
 -- !query output
 1999
 
@@ -794,7 +794,7 @@ struct<year(CAST(CAST(c - i AS STRING) AS DATE)):int>
 -- !query
 select extract(year from c - i) from t
 -- !query schema
-struct<extract(year FROM CAST(c - i AS STRING)):int>
+struct<extract(year FROM c - i):int>
 -- !query output
 1999
 
@@ -802,6 +802,211 @@ struct<extract(year FROM CAST(c - i AS STRING)):int>
 -- !query
 select extract(month from to_timestamp(c) - i) from t
 -- !query schema
-struct<extract(month FROM CAST(to_timestamp(c) - i AS TIMESTAMP)):int>
+struct<extract(month FROM to_timestamp(c) - i):int>
 -- !query output
 8
+
+
+-- !query
+select extract(YEAR from interval '2-1' YEAR TO MONTH)
+-- !query schema
+struct<extract(YEAR FROM INTERVAL '2-1' YEAR TO MONTH):int>
+-- !query output
+2
+
+
+-- !query
+select date_part('YEAR', interval '2-1' YEAR TO MONTH)
+-- !query schema
+struct<date_part(YEAR, INTERVAL '2-1' YEAR TO MONTH):int>
+-- !query output
+2
+
+
+-- !query
+select extract(YEAR from -interval '2-1' YEAR TO MONTH)
+-- !query schema
+struct<extract(YEAR FROM (- INTERVAL '2-1' YEAR TO MONTH)):int>
+-- !query output
+-2
+
+
+-- !query
+select extract(MONTH from interval '2-1' YEAR TO MONTH)
+-- !query schema
+struct<extract(MONTH FROM INTERVAL '2-1' YEAR TO MONTH):tinyint>
+-- !query output
+1
+
+
+-- !query
+select date_part('MONTH', interval '2-1' YEAR TO MONTH)
+-- !query schema
+struct<date_part(MONTH, INTERVAL '2-1' YEAR TO MONTH):tinyint>
+-- !query output
+1
+
+
+-- !query
+select extract(MONTH from -interval '2-1' YEAR TO MONTH)
+-- !query schema
+struct<extract(MONTH FROM (- INTERVAL '2-1' YEAR TO MONTH)):tinyint>
+-- !query output
+-1
+
+
+-- !query
+select date_part(NULL, interval '2-1' YEAR TO MONTH)
+-- !query schema
+struct<date_part(NULL, INTERVAL '2-1' YEAR TO MONTH):double>
+-- !query output
+NULL
+
+
+-- !query
+select extract(DAY from interval '2-1' YEAR TO MONTH)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Literals of type 'DAY' are currently not supported for the interval year to month type.; line 1 pos 7
+
+
+-- !query
+select date_part('DAY', interval '2-1' YEAR TO MONTH)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Literals of type 'DAY' are currently not supported for the interval year to month type.; line 1 pos 7
+
+
+-- !query
+select date_part('not_supported', interval '2-1' YEAR TO MONTH)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Literals of type 'not_supported' are currently not supported for the interval year to month type.; line 1 pos 7
+
+
+-- !query
+select extract(DAY from interval '123 12:34:56.789123123' DAY TO SECOND)
+-- !query schema
+struct<extract(DAY FROM INTERVAL '123 12:34:56.789123' DAY TO SECOND):int>
+-- !query output
+123
+
+
+-- !query
+select date_part('DAY', interval '123 12:34:56.789123123' DAY TO SECOND)
+-- !query schema
+struct<date_part(DAY, INTERVAL '123 12:34:56.789123' DAY TO SECOND):int>
+-- !query output
+123
+
+
+-- !query
+select extract(DAY from -interval '123 12:34:56.789123123' DAY TO SECOND)
+-- !query schema
+struct<extract(DAY FROM (- INTERVAL '123 12:34:56.789123' DAY TO SECOND)):int>
+-- !query output
+-123
+
+
+-- !query
+select extract(HOUR from interval '123 12:34:56.789123123' DAY TO SECOND)
+-- !query schema
+struct<extract(HOUR FROM INTERVAL '123 12:34:56.789123' DAY TO SECOND):tinyint>
+-- !query output
+12
+
+
+-- !query
+select date_part('HOUR', interval '123 12:34:56.789123123' DAY TO SECOND)
+-- !query schema
+struct<date_part(HOUR, INTERVAL '123 12:34:56.789123' DAY TO SECOND):tinyint>
+-- !query output
+12
+
+
+-- !query
+select extract(HOUR from -interval '123 12:34:56.789123123' DAY TO SECOND)
+-- !query schema
+struct<extract(HOUR FROM (- INTERVAL '123 12:34:56.789123' DAY TO SECOND)):tinyint>
+-- !query output
+-12
+
+
+-- !query
+select extract(MINUTE from interval '123 12:34:56.789123123' DAY TO SECOND)
+-- !query schema
+struct<extract(MINUTE FROM INTERVAL '123 12:34:56.789123' DAY TO SECOND):tinyint>
+-- !query output
+34
+
+
+-- !query
+select date_part('MINUTE', interval '123 12:34:56.789123123' DAY TO SECOND)
+-- !query schema
+struct<date_part(MINUTE, INTERVAL '123 12:34:56.789123' DAY TO SECOND):tinyint>
+-- !query output
+34
+
+
+-- !query
+select extract(MINUTE from -interval '123 12:34:56.789123123' DAY TO SECOND)
+-- !query schema
+struct<extract(MINUTE FROM (- INTERVAL '123 12:34:56.789123' DAY TO SECOND)):tinyint>
+-- !query output
+-34
+
+
+-- !query
+select extract(SECOND from interval '123 12:34:56.789123123' DAY TO SECOND)
+-- !query schema
+struct<extract(SECOND FROM INTERVAL '123 12:34:56.789123' DAY TO SECOND):decimal(8,6)>
+-- !query output
+56.789123
+
+
+-- !query
+select date_part('SECOND', interval '123 12:34:56.789123123' DAY TO SECOND)
+-- !query schema
+struct<date_part(SECOND, INTERVAL '123 12:34:56.789123' DAY TO SECOND):decimal(8,6)>
+-- !query output
+56.789123
+
+
+-- !query
+select extract(SECOND from -interval '123 12:34:56.789123123' DAY TO SECOND)
+-- !query schema
+struct<extract(SECOND FROM (- INTERVAL '123 12:34:56.789123' DAY TO SECOND)):decimal(8,6)>
+-- !query output
+-56.789123
+
+
+-- !query
+select date_part(NULL, interval '123 12:34:56.789123123' DAY TO SECOND)
+-- !query schema
+struct<date_part(NULL, INTERVAL '123 12:34:56.789123' DAY TO SECOND):double>
+-- !query output
+NULL
+
+
+-- !query
+select extract(MONTH from interval '123 12:34:56.789123123' DAY TO SECOND)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Literals of type 'MONTH' are currently not supported for the interval day to second type.; line 1 pos 7
+
+
+-- !query
+select date_part('not_supported', interval '123 12:34:56.789123123' DAY TO SECOND)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Literals of type 'not_supported' are currently not supported for the interval day to second type.; line 1 pos 7
diff --git a/sql/core/src/test/resources/sql-tests/results/group-analytics.sql.out b/sql/core/src/test/resources/sql-tests/results/group-analytics.sql.out
index c4f9ea1fe026a..f249908163d01 100644
--- a/sql/core/src/test/resources/sql-tests/results/group-analytics.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/group-analytics.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 29
+-- Number of queries: 52
 
 
 -- !query
@@ -110,6 +110,38 @@ dotNET	2012	15000
 dotNET	2013	48000
 
 
+-- !query
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY ROLLUP(course, year, (course, year)) ORDER BY course, year
+-- !query schema
+struct<course:string,year:int,sum(earnings):bigint>
+-- !query output
+NULL	NULL	113000
+Java	NULL	50000
+Java	2012	20000
+Java	2012	20000
+Java	2013	30000
+Java	2013	30000
+dotNET	NULL	63000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2013	48000
+dotNET	2013	48000
+
+
+-- !query
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY ROLLUP(course, year, (course, year), ()) ORDER BY course, year
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Empty set in ROLLUP grouping sets is not supported.(line 1, pos 61)
+
+== SQL ==
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY ROLLUP(course, year, (course, year), ()) ORDER BY course, year
+-------------------------------------------------------------^^^
+
+
 -- !query
 SELECT course, year, SUM(earnings) FROM courseSales GROUP BY CUBE(course, year) ORDER BY course, year
 -- !query schema
@@ -126,6 +158,52 @@ dotNET	2012	15000
 dotNET	2013	48000
 
 
+-- !query
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY CUBE(course, year, (course, year)) ORDER BY course, year
+-- !query schema
+struct<course:string,year:int,sum(earnings):bigint>
+-- !query output
+NULL	NULL	113000
+NULL	2012	35000
+NULL	2013	78000
+Java	NULL	50000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+dotNET	NULL	63000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+
+
+-- !query
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY CUBE(course, year, (course, year), ()) ORDER BY course, year
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Empty set in CUBE grouping sets is not supported.(line 1, pos 61)
+
+== SQL ==
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY CUBE(course, year, (course, year), ()) ORDER BY course, year
+-------------------------------------------------------------^^^
+
+
 -- !query
 SELECT course, year, SUM(earnings) FROM courseSales GROUP BY course, year GROUPING SETS(course, year)
 -- !query schema
@@ -137,6 +215,18 @@ NULL	2013	78000
 dotNET	NULL	63000
 
 
+-- !query
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY course, year GROUPING SETS(course, year, ())
+-- !query schema
+struct<course:string,year:int,sum(earnings):bigint>
+-- !query output
+Java	NULL	50000
+NULL	2012	35000
+NULL	2013	78000
+NULL	NULL	113000
+dotNET	NULL	63000
+
+
 -- !query
 SELECT course, year, SUM(earnings) FROM courseSales GROUP BY course, year GROUPING SETS(course)
 -- !query schema
@@ -155,6 +245,160 @@ NULL	2012	35000
 NULL	2013	78000
 
 
+-- !query
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY course, CUBE(course, year) ORDER BY course, year
+-- !query schema
+struct<course:string,year:int,sum(earnings):bigint>
+-- !query output
+Java	NULL	50000
+Java	NULL	50000
+Java	2012	20000
+Java	2012	20000
+Java	2013	30000
+Java	2013	30000
+dotNET	NULL	63000
+dotNET	NULL	63000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2013	48000
+dotNET	2013	48000
+
+
+-- !query
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY CUBE(course, year), ROLLUP(course, year) ORDER BY course, year
+-- !query schema
+struct<course:string,year:int,sum(earnings):bigint>
+-- !query output
+NULL	NULL	113000
+NULL	2012	35000
+NULL	2013	78000
+Java	NULL	50000
+Java	NULL	50000
+Java	NULL	50000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+dotNET	NULL	63000
+dotNET	NULL	63000
+dotNET	NULL	63000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+
+
+-- !query
+SELECT course, year, SUM(earnings) FROM courseSales GROUP BY CUBE(course, year), ROLLUP(course, year), GROUPING SETS(course, year) ORDER BY course, year
+-- !query schema
+struct<course:string,year:int,sum(earnings):bigint>
+-- !query output
+NULL	2012	35000
+NULL	2012	35000
+NULL	2013	78000
+NULL	2013	78000
+Java	NULL	50000
+Java	NULL	50000
+Java	NULL	50000
+Java	NULL	50000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2012	20000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+Java	2013	30000
+dotNET	NULL	63000
+dotNET	NULL	63000
+dotNET	NULL	63000
+dotNET	NULL	63000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2012	15000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+dotNET	2013	48000
+
+
 -- !query
 SELECT course, SUM(earnings) AS sum FROM courseSales
 GROUP BY course, earnings GROUPING SETS((), (course), (course, earnings)) ORDER BY course, sum
@@ -210,7 +454,7 @@ SELECT course, year, GROUPING(course) FROM courseSales GROUP BY course, year
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-grouping() can only be used with GroupingSets/Cube/Rollup;
+grouping() can only be used with GroupingSets/Cube/Rollup
 
 
 -- !query
@@ -219,7 +463,7 @@ SELECT course, year, GROUPING_ID(course, year) FROM courseSales GROUP BY course,
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-grouping_id() can only be used with GroupingSets/Cube/Rollup;
+grouping_id() can only be used with GroupingSets/Cube/Rollup
 
 
 -- !query
@@ -255,7 +499,7 @@ SELECT course, year FROM courseSales GROUP BY course, year HAVING GROUPING(cours
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-grouping()/grouping_id() can only be used with GroupingSets/Cube/Rollup;
+grouping()/grouping_id() can only be used with GroupingSets/Cube/Rollup
 
 
 -- !query
@@ -264,7 +508,7 @@ SELECT course, year FROM courseSales GROUP BY course, year HAVING GROUPING_ID(co
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-grouping()/grouping_id() can only be used with GroupingSets/Cube/Rollup;
+grouping()/grouping_id() can only be used with GroupingSets/Cube/Rollup
 
 
 -- !query
@@ -319,7 +563,7 @@ SELECT course, year FROM courseSales GROUP BY course, year ORDER BY GROUPING(cou
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-grouping()/grouping_id() can only be used with GroupingSets/Cube/Rollup;
+grouping()/grouping_id() can only be used with GroupingSets/Cube/Rollup
 
 
 -- !query
@@ -328,7 +572,7 @@ SELECT course, year FROM courseSales GROUP BY course, year ORDER BY GROUPING_ID(
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-grouping()/grouping_id() can only be used with GroupingSets/Cube/Rollup;
+grouping()/grouping_id() can only be used with GroupingSets/Cube/Rollup
 
 
 -- !query
@@ -392,3 +636,678 @@ struct<(a + b):int,k:int,sum((a - b)):bigint>
 -- !query output
 NULL	1	3
 NULL	2	0
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, b, CUBE(a, b)
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, b, ROLLUP(a, b)
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	1	1
+1	1	1
+1	2	1
+1	2	1
+1	2	1
+2	1	1
+2	1	1
+2	1	1
+2	2	1
+2	2	1
+2	2	1
+3	1	1
+3	1	1
+3	1	1
+3	2	1
+3	2	1
+3	2	1
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY CUBE(a, b), ROLLUP(a, b)
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	NULL	2
+1	NULL	2
+1	NULL	2
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	NULL	2
+2	NULL	2
+2	NULL	2
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	NULL	2
+3	NULL	2
+3	NULL	2
+NULL	1	3
+NULL	2	3
+NULL	NULL	6
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, CUBE(a, b), ROLLUP(b)
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	NULL	2
+1	NULL	2
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	NULL	2
+2	NULL	2
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	NULL	2
+3	NULL	2
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS((a, b), (a), ())
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	2	1
+1	NULL	2
+1	NULL	2
+2	1	1
+2	2	1
+2	NULL	2
+2	NULL	2
+3	1	1
+3	2	1
+3	NULL	2
+3	NULL	2
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, CUBE(a, b), GROUPING SETS((a, b), (a), ())
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	NULL	2
+1	NULL	2
+1	NULL	2
+1	NULL	2
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	NULL	2
+2	NULL	2
+2	NULL	2
+2	NULL	2
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	NULL	2
+3	NULL	2
+3	NULL	2
+3	NULL	2
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, CUBE(a, b), ROLLUP(a, b), GROUPING SETS((a, b), (a), ())
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	1	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	2	1
+1	NULL	2
+1	NULL	2
+1	NULL	2
+1	NULL	2
+1	NULL	2
+1	NULL	2
+1	NULL	2
+1	NULL	2
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	1	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	2	1
+2	NULL	2
+2	NULL	2
+2	NULL	2
+2	NULL	2
+2	NULL	2
+2	NULL	2
+2	NULL	2
+2	NULL	2
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	1	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	2	1
+3	NULL	2
+3	NULL	2
+3	NULL	2
+3	NULL	2
+3	NULL	2
+3	NULL	2
+3	NULL	2
+3	NULL	2
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS(ROLLUP(a, b))
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	2	1
+1	NULL	2
+1	NULL	2
+2	1	1
+2	2	1
+2	NULL	2
+2	NULL	2
+3	1	1
+3	2	1
+3	NULL	2
+3	NULL	2
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS(GROUPING SETS((a, b), (a), ()))
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	2	1
+1	NULL	2
+1	NULL	2
+2	1	1
+2	2	1
+2	NULL	2
+2	NULL	2
+3	1	1
+3	2	1
+3	NULL	2
+3	NULL	2
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS((a, b), GROUPING SETS(ROLLUP(a, b)))
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	1	1
+1	2	1
+1	2	1
+1	NULL	2
+1	NULL	2
+2	1	1
+2	1	1
+2	2	1
+2	2	1
+2	NULL	2
+2	NULL	2
+3	1	1
+3	1	1
+3	2	1
+3	2	1
+3	NULL	2
+3	NULL	2
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS((a, b, a, b), (a, b, a), (a, b))
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	1	1
+1	1	1
+1	2	1
+1	2	1
+1	2	1
+2	1	1
+2	1	1
+2	1	1
+2	2	1
+2	2	1
+2	2	1
+3	1	1
+3	1	1
+3	1	1
+3	2	1
+3	2	1
+3	2	1
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS(GROUPING SETS((a, b, a, b), (a, b, a), (a, b)))
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	1	1
+1	1	1
+1	2	1
+1	2	1
+1	2	1
+2	1	1
+2	1	1
+2	1	1
+2	2	1
+2	2	1
+2	2	1
+3	1	1
+3	1	1
+3	1	1
+3	2	1
+3	2	1
+3	2	1
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS(ROLLUP(a, b), CUBE(a, b))
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	1	1
+1	1	1
+1	2	1
+1	2	1
+1	2	1
+1	NULL	2
+1	NULL	2
+1	NULL	2
+1	NULL	2
+2	1	1
+2	1	1
+2	1	1
+2	2	1
+2	2	1
+2	2	1
+2	NULL	2
+2	NULL	2
+2	NULL	2
+2	NULL	2
+3	1	1
+3	1	1
+3	1	1
+3	2	1
+3	2	1
+3	2	1
+3	NULL	2
+3	NULL	2
+3	NULL	2
+3	NULL	2
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS(GROUPING SETS((a, b), (a), ()), GROUPING SETS((a, b), (a), (b), ()))
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	1	1
+1	1	1
+1	2	1
+1	2	1
+1	2	1
+1	NULL	2
+1	NULL	2
+1	NULL	2
+1	NULL	2
+2	1	1
+2	1	1
+2	1	1
+2	2	1
+2	2	1
+2	2	1
+2	NULL	2
+2	NULL	2
+2	NULL	2
+2	NULL	2
+3	1	1
+3	1	1
+3	1	1
+3	2	1
+3	2	1
+3	2	1
+3	NULL	2
+3	NULL	2
+3	NULL	2
+3	NULL	2
+
+
+-- !query
+SELECT a, b, count(1) FROM testData GROUP BY a, GROUPING SETS((a, b), (a), (), (a, b), (a), (b), ())
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	1	1
+1	1	1
+1	2	1
+1	2	1
+1	2	1
+1	NULL	2
+1	NULL	2
+1	NULL	2
+1	NULL	2
+2	1	1
+2	1	1
+2	1	1
+2	2	1
+2	2	1
+2	2	1
+2	NULL	2
+2	NULL	2
+2	NULL	2
+2	NULL	2
+3	1	1
+3	1	1
+3	1	1
+3	2	1
+3	2	1
+3	2	1
+3	NULL	2
+3	NULL	2
+3	NULL	2
+3	NULL	2
diff --git a/sql/core/src/test/resources/sql-tests/results/group-by-filter.sql.out b/sql/core/src/test/resources/sql-tests/results/group-by-filter.sql.out
index 3fcd132701a3f..e1633d5017e73 100644
--- a/sql/core/src/test/resources/sql-tests/results/group-by-filter.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/group-by-filter.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 37
+-- Number of queries: 76
 
 
 -- !query
@@ -51,7 +51,7 @@ SELECT a, COUNT(b) FILTER (WHERE a >= 2) FROM testData
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-grouping expressions sequence is empty, and 'testdata.`a`' is not an aggregate function. Wrap '(count(testdata.`b`) FILTER (WHERE (testdata.`a` >= 2)) AS `count(b) FILTER (WHERE (a >= 2))`)' in windowing function(s) or wrap 'testdata.`a`' in first() (or first_value) if you don't care which value you get.;
+grouping expressions sequence is empty, and 'testdata.a' is not an aggregate function. Wrap '(count(testdata.b) FILTER (WHERE (testdata.a >= 2)) AS `count(b) FILTER (WHERE (a >= 2))`)' in windowing function(s) or wrap 'testdata.a' in first() (or first_value) if you don't care which value you get.
 
 
 -- !query
@@ -81,7 +81,7 @@ struct<count(id) FILTER (WHERE (hiredate = to_date(2001-01-01 00:00:00))):bigint
 -- !query
 SELECT COUNT(id) FILTER (WHERE hiredate = to_timestamp("2001-01-01 00:00:00")) FROM emp
 -- !query schema
-struct<count(id) FILTER (WHERE (CAST(hiredate AS TIMESTAMP) = to_timestamp(2001-01-01 00:00:00))):bigint>
+struct<count(id) FILTER (WHERE (hiredate = to_timestamp(2001-01-01 00:00:00))):bigint>
 -- !query output
 2
 
@@ -89,11 +89,131 @@ struct<count(id) FILTER (WHERE (CAST(hiredate AS TIMESTAMP) = to_timestamp(2001-
 -- !query
 SELECT COUNT(id) FILTER (WHERE date_format(hiredate, "yyyy-MM-dd") = "2001-01-01") FROM emp
 -- !query schema
-struct<count(id) FILTER (WHERE (date_format(CAST(hiredate AS TIMESTAMP), yyyy-MM-dd) = 2001-01-01)):bigint>
+struct<count(id) FILTER (WHERE (date_format(hiredate, yyyy-MM-dd) = 2001-01-01)):bigint>
 -- !query output
 2
 
 
+-- !query
+SELECT COUNT(DISTINCT id) FILTER (WHERE date_format(hiredate, "yyyy-MM-dd HH:mm:ss") = "2001-01-01 00:00:00") FROM emp
+-- !query schema
+struct<count(DISTINCT id) FILTER (WHERE (date_format(hiredate, yyyy-MM-dd HH:mm:ss) = 2001-01-01 00:00:00)):bigint>
+-- !query output
+2
+
+
+-- !query
+SELECT COUNT(DISTINCT id), COUNT(DISTINCT id) FILTER (WHERE date_format(hiredate, "yyyy-MM-dd HH:mm:ss") = "2001-01-01 00:00:00") FROM emp
+-- !query schema
+struct<count(DISTINCT id):bigint,count(DISTINCT id) FILTER (WHERE (date_format(hiredate, yyyy-MM-dd HH:mm:ss) = 2001-01-01 00:00:00)):bigint>
+-- !query output
+8	2
+
+
+-- !query
+SELECT COUNT(DISTINCT id) FILTER (WHERE hiredate = to_timestamp("2001-01-01 00:00:00")), COUNT(DISTINCT id) FILTER (WHERE hiredate = to_date('2001-01-01 00:00:00')) FROM emp
+-- !query schema
+struct<count(DISTINCT id) FILTER (WHERE (hiredate = to_timestamp(2001-01-01 00:00:00))):bigint,count(DISTINCT id) FILTER (WHERE (hiredate = to_date(2001-01-01 00:00:00))):bigint>
+-- !query output
+2	2
+
+
+-- !query
+SELECT SUM(salary), COUNT(DISTINCT id), COUNT(DISTINCT id) FILTER (WHERE hiredate = date "2001-01-01") FROM emp
+-- !query schema
+struct<sum(salary):double,count(DISTINCT id):bigint,count(DISTINCT id) FILTER (WHERE (hiredate = DATE '2001-01-01')):bigint>
+-- !query output
+2450.0	8	2
+
+
+-- !query
+SELECT COUNT(DISTINCT 1) FILTER (WHERE a = 1) FROM testData
+-- !query schema
+struct<count(DISTINCT 1) FILTER (WHERE (a = 1)):bigint>
+-- !query output
+1
+
+
+-- !query
+SELECT COUNT(DISTINCT id) FILTER (WHERE true) FROM emp
+-- !query schema
+struct<count(DISTINCT id) FILTER (WHERE true):bigint>
+-- !query output
+8
+
+
+-- !query
+SELECT COUNT(DISTINCT id) FILTER (WHERE false) FROM emp
+-- !query schema
+struct<count(DISTINCT id) FILTER (WHERE false):bigint>
+-- !query output
+0
+
+
+-- !query
+SELECT COUNT(DISTINCT 2), COUNT(DISTINCT 2,3) FILTER (WHERE dept_id = 40) FROM emp
+-- !query schema
+struct<count(DISTINCT 2):bigint,count(DISTINCT 2, 3) FILTER (WHERE (dept_id = 40)):bigint>
+-- !query output
+1	0
+
+
+-- !query
+SELECT COUNT(DISTINCT 2), COUNT(DISTINCT 3,2) FILTER (WHERE dept_id = 40) FROM emp
+-- !query schema
+struct<count(DISTINCT 2):bigint,count(DISTINCT 3, 2) FILTER (WHERE (dept_id = 40)):bigint>
+-- !query output
+1	0
+
+
+-- !query
+SELECT COUNT(DISTINCT 2), COUNT(DISTINCT 2,3) FILTER (WHERE dept_id > 0) FROM emp
+-- !query schema
+struct<count(DISTINCT 2):bigint,count(DISTINCT 2, 3) FILTER (WHERE (dept_id > 0)):bigint>
+-- !query output
+1	1
+
+
+-- !query
+SELECT COUNT(DISTINCT 2), COUNT(DISTINCT 3,2) FILTER (WHERE dept_id > 0) FROM emp
+-- !query schema
+struct<count(DISTINCT 2):bigint,count(DISTINCT 3, 2) FILTER (WHERE (dept_id > 0)):bigint>
+-- !query output
+1	1
+
+
+-- !query
+SELECT COUNT(DISTINCT id), COUNT(DISTINCT 2,3) FILTER (WHERE dept_id = 40) FROM emp
+-- !query schema
+struct<count(DISTINCT id):bigint,count(DISTINCT 2, 3) FILTER (WHERE (dept_id = 40)):bigint>
+-- !query output
+8	0
+
+
+-- !query
+SELECT COUNT(DISTINCT id), COUNT(DISTINCT 3,2) FILTER (WHERE dept_id = 40) FROM emp
+-- !query schema
+struct<count(DISTINCT id):bigint,count(DISTINCT 3, 2) FILTER (WHERE (dept_id = 40)):bigint>
+-- !query output
+8	0
+
+
+-- !query
+SELECT COUNT(DISTINCT id), COUNT(DISTINCT 2,3) FILTER (WHERE dept_id > 0) FROM emp
+-- !query schema
+struct<count(DISTINCT id):bigint,count(DISTINCT 2, 3) FILTER (WHERE (dept_id > 0)):bigint>
+-- !query output
+8	1
+
+
+-- !query
+SELECT COUNT(DISTINCT id), COUNT(DISTINCT 3,2) FILTER (WHERE dept_id > 0) FROM emp
+-- !query schema
+struct<count(DISTINCT id):bigint,count(DISTINCT 3, 2) FILTER (WHERE (dept_id > 0)):bigint>
+-- !query output
+8	1
+
+
 -- !query
 SELECT a, COUNT(b) FILTER (WHERE a >= 2) FROM testData GROUP BY a
 -- !query schema
@@ -111,7 +231,7 @@ SELECT a, COUNT(b) FILTER (WHERE a != 2) FROM testData GROUP BY b
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-expression 'testdata.`a`' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.;
+expression 'testdata.a' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.
 
 
 -- !query
@@ -154,7 +274,7 @@ NULL	NULL
 -- !query
 SELECT dept_id, SUM(salary) FILTER (WHERE hiredate > to_timestamp("2003-01-01 00:00:00")) FROM emp GROUP BY dept_id
 -- !query schema
-struct<dept_id:int,sum(salary) FILTER (WHERE (CAST(hiredate AS TIMESTAMP) > to_timestamp(2003-01-01 00:00:00))):double>
+struct<dept_id:int,sum(salary) FILTER (WHERE (hiredate > to_timestamp(2003-01-01 00:00:00))):double>
 -- !query output
 10	200.0
 100	400.0
@@ -167,7 +287,7 @@ NULL	NULL
 -- !query
 SELECT dept_id, SUM(salary) FILTER (WHERE date_format(hiredate, "yyyy-MM-dd") > "2003-01-01") FROM emp GROUP BY dept_id
 -- !query schema
-struct<dept_id:int,sum(salary) FILTER (WHERE (date_format(CAST(hiredate AS TIMESTAMP), yyyy-MM-dd) > 2003-01-01)):double>
+struct<dept_id:int,sum(salary) FILTER (WHERE (date_format(hiredate, yyyy-MM-dd) > 2003-01-01)):double>
 -- !query output
 10	200.0
 100	400.0
@@ -177,6 +297,68 @@ struct<dept_id:int,sum(salary) FILTER (WHERE (date_format(CAST(hiredate AS TIMES
 NULL	NULL
 
 
+-- !query
+SELECT dept_id, SUM(DISTINCT salary) FILTER (WHERE date_format(hiredate, "yyyy-MM-dd HH:mm:ss") > "2001-01-01 00:00:00") FROM emp GROUP BY dept_id
+-- !query schema
+struct<dept_id:int,sum(DISTINCT salary) FILTER (WHERE (date_format(hiredate, yyyy-MM-dd HH:mm:ss) > 2001-01-01 00:00:00)):double>
+-- !query output
+10	300.0
+100	400.0
+20	300.0
+30	400.0
+70	150.0
+NULL	NULL
+
+
+-- !query
+SELECT dept_id, SUM(DISTINCT salary), SUM(DISTINCT salary) FILTER (WHERE date_format(hiredate, "yyyy-MM-dd HH:mm:ss") > "2001-01-01 00:00:00") FROM emp GROUP BY dept_id
+-- !query schema
+struct<dept_id:int,sum(DISTINCT salary):double,sum(DISTINCT salary) FILTER (WHERE (date_format(hiredate, yyyy-MM-dd HH:mm:ss) > 2001-01-01 00:00:00)):double>
+-- !query output
+10	300.0	300.0
+100	400.0	400.0
+20	300.0	300.0
+30	400.0	400.0
+70	150.0	150.0
+NULL	400.0	NULL
+
+
+-- !query
+SELECT dept_id, SUM(DISTINCT salary) FILTER (WHERE hiredate > date "2001-01-01"), SUM(DISTINCT salary) FILTER (WHERE date_format(hiredate, "yyyy-MM-dd HH:mm:ss") > "2001-01-01 00:00:00") FROM emp GROUP BY dept_id
+-- !query schema
+struct<dept_id:int,sum(DISTINCT salary) FILTER (WHERE (hiredate > DATE '2001-01-01')):double,sum(DISTINCT salary) FILTER (WHERE (date_format(hiredate, yyyy-MM-dd HH:mm:ss) > 2001-01-01 00:00:00)):double>
+-- !query output
+10	300.0	300.0
+100	400.0	400.0
+20	300.0	300.0
+30	400.0	400.0
+70	150.0	150.0
+NULL	NULL	NULL
+
+
+-- !query
+SELECT dept_id, COUNT(id), SUM(DISTINCT salary), SUM(DISTINCT salary) FILTER (WHERE date_format(hiredate, "yyyy-MM-dd") > "2001-01-01") FROM emp GROUP BY dept_id
+-- !query schema
+struct<dept_id:int,count(id):bigint,sum(DISTINCT salary):double,sum(DISTINCT salary) FILTER (WHERE (date_format(hiredate, yyyy-MM-dd) > 2001-01-01)):double>
+-- !query output
+10	3	300.0	300.0
+100	2	400.0	400.0
+20	1	300.0	300.0
+30	1	400.0	400.0
+70	1	150.0	150.0
+NULL	1	400.0	NULL
+
+
+-- !query
+SELECT b, COUNT(DISTINCT 1) FILTER (WHERE a = 1) FROM testData GROUP BY b
+-- !query schema
+struct<b:int,count(DISTINCT 1) FILTER (WHERE (a = 1)):bigint>
+-- !query output
+1	1
+2	1
+NULL	0
+
+
 -- !query
 SELECT 'foo', COUNT(a) FILTER (WHERE b <= 2) FROM testData GROUP BY 1
 -- !query schema
@@ -204,7 +386,7 @@ foo	1350.0
 -- !query
 SELECT 'foo', SUM(salary) FILTER (WHERE hiredate >= to_timestamp("2003-01-01")) FROM emp GROUP BY 1
 -- !query schema
-struct<foo:string,sum(salary) FILTER (WHERE (CAST(hiredate AS TIMESTAMP) >= to_timestamp(2003-01-01))):double>
+struct<foo:string,sum(salary) FILTER (WHERE (hiredate >= to_timestamp(2003-01-01))):double>
 -- !query output
 foo	1350.0
 
@@ -261,6 +443,240 @@ struct<dept_id:int,count(DISTINCT emp_name):bigint,count(DISTINCT hiredate):bigi
 NULL	1	1	NULL	NULL
 
 
+-- !query
+select dept_id, count(distinct emp_name) filter (where id > 200), sum(salary) from emp group by dept_id
+-- !query schema
+struct<dept_id:int,count(DISTINCT emp_name) FILTER (WHERE (id > 200)):bigint,sum(salary):double>
+-- !query output
+10	0	400.0
+100	2	800.0
+20	1	300.0
+30	1	400.0
+70	1	150.0
+NULL	1	400.0
+
+
+-- !query
+select dept_id, count(distinct emp_name) filter (where id + dept_id > 500), sum(salary) from emp group by dept_id
+-- !query schema
+struct<dept_id:int,count(DISTINCT emp_name) FILTER (WHERE ((id + dept_id) > 500)):bigint,sum(salary):double>
+-- !query output
+10	0	400.0
+100	2	800.0
+20	0	300.0
+30	0	400.0
+70	1	150.0
+NULL	0	400.0
+
+
+-- !query
+select dept_id, count(distinct emp_name), count(distinct emp_name) filter (where id > 200), sum(salary) from emp group by dept_id
+-- !query schema
+struct<dept_id:int,count(DISTINCT emp_name):bigint,count(DISTINCT emp_name) FILTER (WHERE (id > 200)):bigint,sum(salary):double>
+-- !query output
+10	2	0	400.0
+100	2	2	800.0
+20	1	1	300.0
+30	1	1	400.0
+70	1	1	150.0
+NULL	1	1	400.0
+
+
+-- !query
+select dept_id, count(distinct emp_name), count(distinct emp_name) filter (where id + dept_id > 500), sum(salary) from emp group by dept_id
+-- !query schema
+struct<dept_id:int,count(DISTINCT emp_name):bigint,count(DISTINCT emp_name) FILTER (WHERE ((id + dept_id) > 500)):bigint,sum(salary):double>
+-- !query output
+10	2	0	400.0
+100	2	2	800.0
+20	1	0	300.0
+30	1	0	400.0
+70	1	1	150.0
+NULL	1	0	400.0
+
+
+-- !query
+select dept_id, count(distinct emp_name), count(distinct emp_name) filter (where id > 200), sum(salary), sum(salary) filter (where id > 200) from emp group by dept_id
+-- !query schema
+struct<dept_id:int,count(DISTINCT emp_name):bigint,count(DISTINCT emp_name) FILTER (WHERE (id > 200)):bigint,sum(salary):double,sum(salary) FILTER (WHERE (id > 200)):double>
+-- !query output
+10	2	0	400.0	NULL
+100	2	2	800.0	800.0
+20	1	1	300.0	300.0
+30	1	1	400.0	400.0
+70	1	1	150.0	150.0
+NULL	1	1	400.0	400.0
+
+
+-- !query
+select dept_id, count(distinct emp_name), count(distinct emp_name) filter (where id + dept_id > 500), sum(salary), sum(salary) filter (where id > 200) from emp group by dept_id
+-- !query schema
+struct<dept_id:int,count(DISTINCT emp_name):bigint,count(DISTINCT emp_name) FILTER (WHERE ((id + dept_id) > 500)):bigint,sum(salary):double,sum(salary) FILTER (WHERE (id > 200)):double>
+-- !query output
+10	2	0	400.0	NULL
+100	2	2	800.0	800.0
+20	1	0	300.0	300.0
+30	1	0	400.0	400.0
+70	1	1	150.0	150.0
+NULL	1	0	400.0	400.0
+
+
+-- !query
+select dept_id, count(distinct emp_name) filter (where id > 200), count(distinct hiredate), sum(salary) from emp group by dept_id
+-- !query schema
+struct<dept_id:int,count(DISTINCT emp_name) FILTER (WHERE (id > 200)):bigint,count(DISTINCT hiredate):bigint,sum(salary):double>
+-- !query output
+10	0	2	400.0
+100	2	2	800.0
+20	1	1	300.0
+30	1	1	400.0
+70	1	1	150.0
+NULL	1	1	400.0
+
+
+-- !query
+select dept_id, count(distinct emp_name) filter (where id > 200), count(distinct hiredate) filter (where hiredate > date "2003-01-01"), sum(salary) from emp group by dept_id
+-- !query schema
+struct<dept_id:int,count(DISTINCT emp_name) FILTER (WHERE (id > 200)):bigint,count(DISTINCT hiredate) FILTER (WHERE (hiredate > DATE '2003-01-01')):bigint,sum(salary):double>
+-- !query output
+10	0	1	400.0
+100	2	1	800.0
+20	1	0	300.0
+30	1	1	400.0
+70	1	1	150.0
+NULL	1	0	400.0
+
+
+-- !query
+select dept_id, count(distinct emp_name) filter (where id > 200), count(distinct hiredate) filter (where hiredate > date "2003-01-01"), sum(salary) filter (where salary < 400.00D) from emp group by dept_id
+-- !query schema
+struct<dept_id:int,count(DISTINCT emp_name) FILTER (WHERE (id > 200)):bigint,count(DISTINCT hiredate) FILTER (WHERE (hiredate > DATE '2003-01-01')):bigint,sum(salary) FILTER (WHERE (salary < 400.0)):double>
+-- !query output
+10	0	1	400.0
+100	2	1	NULL
+20	1	0	300.0
+30	1	1	NULL
+70	1	1	150.0
+NULL	1	0	NULL
+
+
+-- !query
+select dept_id, count(distinct emp_name) filter (where id > 200), count(distinct hiredate) filter (where hiredate > date "2003-01-01"), sum(salary) filter (where salary < 400.00D), sum(salary) filter (where id > 200) from emp group by dept_id
+-- !query schema
+struct<dept_id:int,count(DISTINCT emp_name) FILTER (WHERE (id > 200)):bigint,count(DISTINCT hiredate) FILTER (WHERE (hiredate > DATE '2003-01-01')):bigint,sum(salary) FILTER (WHERE (salary < 400.0)):double,sum(salary) FILTER (WHERE (id > 200)):double>
+-- !query output
+10	0	1	400.0	NULL
+100	2	1	NULL	800.0
+20	1	0	300.0	300.0
+30	1	1	NULL	400.0
+70	1	1	150.0	150.0
+NULL	1	0	NULL	400.0
+
+
+-- !query
+select dept_id, count(distinct emp_name) filter (where id > 200), count(distinct emp_name), sum(salary) from emp group by dept_id
+-- !query schema
+struct<dept_id:int,count(DISTINCT emp_name) FILTER (WHERE (id > 200)):bigint,count(DISTINCT emp_name):bigint,sum(salary):double>
+-- !query output
+10	0	2	400.0
+100	2	2	800.0
+20	1	1	300.0
+30	1	1	400.0
+70	1	1	150.0
+NULL	1	1	400.0
+
+
+-- !query
+select dept_id, count(distinct emp_name) filter (where id > 200), count(distinct emp_name) filter (where hiredate > date "2003-01-01"), sum(salary) from emp group by dept_id
+-- !query schema
+struct<dept_id:int,count(DISTINCT emp_name) FILTER (WHERE (id > 200)):bigint,count(DISTINCT emp_name) FILTER (WHERE (hiredate > DATE '2003-01-01')):bigint,sum(salary):double>
+-- !query output
+10	0	1	400.0
+100	2	1	800.0
+20	1	0	300.0
+30	1	1	400.0
+70	1	1	150.0
+NULL	1	0	400.0
+
+
+-- !query
+select dept_id, sum(distinct (id + dept_id)) filter (where id > 200), count(distinct hiredate), sum(salary) from emp group by dept_id
+-- !query schema
+struct<dept_id:int,sum(DISTINCT (id + dept_id)) FILTER (WHERE (id > 200)):bigint,count(DISTINCT hiredate):bigint,sum(salary):double>
+-- !query output
+10	NULL	2	400.0
+100	1500	2	800.0
+20	320	1	300.0
+30	430	1	400.0
+70	870	1	150.0
+NULL	NULL	1	400.0
+
+
+-- !query
+select dept_id, sum(distinct (id + dept_id)) filter (where id > 200), count(distinct hiredate) filter (where hiredate > date "2003-01-01"), sum(salary) from emp group by dept_id
+-- !query schema
+struct<dept_id:int,sum(DISTINCT (id + dept_id)) FILTER (WHERE (id > 200)):bigint,count(DISTINCT hiredate) FILTER (WHERE (hiredate > DATE '2003-01-01')):bigint,sum(salary):double>
+-- !query output
+10	NULL	1	400.0
+100	1500	1	800.0
+20	320	0	300.0
+30	430	1	400.0
+70	870	1	150.0
+NULL	NULL	0	400.0
+
+
+-- !query
+select dept_id, avg(distinct (id + dept_id)) filter (where id > 200), count(distinct hiredate) filter (where hiredate > date "2003-01-01"), sum(salary) filter (where salary < 400.00D) from emp group by dept_id
+-- !query schema
+struct<dept_id:int,avg(DISTINCT (id + dept_id)) FILTER (WHERE (id > 200)):double,count(DISTINCT hiredate) FILTER (WHERE (hiredate > DATE '2003-01-01')):bigint,sum(salary) FILTER (WHERE (salary < 400.0)):double>
+-- !query output
+10	NULL	1	400.0
+100	750.0	1	NULL
+20	320.0	0	300.0
+30	430.0	1	NULL
+70	870.0	1	150.0
+NULL	NULL	0	NULL
+
+
+-- !query
+select dept_id, count(distinct emp_name, hiredate) filter (where id > 200), sum(salary) from emp group by dept_id
+-- !query schema
+struct<dept_id:int,count(DISTINCT emp_name, hiredate) FILTER (WHERE (id > 200)):bigint,sum(salary):double>
+-- !query output
+10	0	400.0
+100	2	800.0
+20	1	300.0
+30	1	400.0
+70	1	150.0
+NULL	1	400.0
+
+
+-- !query
+select dept_id, count(distinct emp_name, hiredate) filter (where id > 0), sum(salary) from emp group by dept_id
+-- !query schema
+struct<dept_id:int,count(DISTINCT emp_name, hiredate) FILTER (WHERE (id > 0)):bigint,sum(salary):double>
+-- !query output
+10	2	400.0
+100	2	800.0
+20	1	300.0
+30	1	400.0
+70	1	150.0
+NULL	1	400.0
+
+
+-- !query
+select dept_id, count(distinct 1), count(distinct 1) filter (where id > 200), sum(salary) from emp group by dept_id
+-- !query schema
+struct<dept_id:int,count(DISTINCT 1):bigint,count(DISTINCT 1) FILTER (WHERE (id > 200)):bigint,sum(salary):double>
+-- !query output
+10	1	0	400.0
+100	1	1	800.0
+20	1	1	300.0
+30	1	1	400.0
+70	1	1	150.0
+NULL	1	1	400.0
+
+
 -- !query
 SELECT 'foo', APPROX_COUNT_DISTINCT(a) FILTER (WHERE b >= 0) FROM testData WHERE a = 0 GROUP BY 1
 -- !query schema
@@ -272,7 +688,7 @@ struct<foo:string,approx_count_distinct(a) FILTER (WHERE (b >= 0)):bigint>
 -- !query
 SELECT 'foo', MAX(STRUCT(a)) FILTER (WHERE b >= 1) FROM testData WHERE a = 0 GROUP BY 1
 -- !query schema
-struct<foo:string,max(named_struct(a, a)) FILTER (WHERE (b >= 1)):struct<a:int>>
+struct<foo:string,max(struct(a)) FILTER (WHERE (b >= 1)):struct<a:int>>
 -- !query output
 
 
@@ -295,7 +711,7 @@ SELECT a + 2, COUNT(b) FILTER (WHERE b IN (1, 2)) FROM testData GROUP BY a + 1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-expression 'testdata.`a`' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.;
+expression 'testdata.a' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.
 
 
 -- !query
@@ -309,6 +725,15 @@ struct<((a + 1) + 1):int,count(b) FILTER (WHERE (b > 0)):bigint>
 NULL	1
 
 
+-- !query
+SELECT COUNT(DISTINCT b) FILTER (WHERE b > 0), COUNT(DISTINCT b, c) FILTER (WHERE b > 0 AND c > 2)
+FROM (SELECT 1 AS a, 2 AS b, 3 AS c) GROUP BY a
+-- !query schema
+struct<count(DISTINCT b) FILTER (WHERE (b > 0)):bigint,count(DISTINCT b, c) FILTER (WHERE ((b > 0) AND (c > 2))):bigint>
+-- !query output
+1	1
+
+
 -- !query
 SELECT a AS k, COUNT(b) FILTER (WHERE b > 0) FROM testData GROUP BY k
 -- !query schema
@@ -370,14 +795,17 @@ IN/EXISTS predicate sub-queries can only be used in Filter/Join and a few comman
 :  +- Project [state#x]
 :     +- Filter (dept_id#x = outer(dept_id#x))
 :        +- SubqueryAlias dept
-:           +- Project [dept_id#x, dept_name#x, state#x]
-:              +- SubqueryAlias DEPT
-:                 +- LocalRelation [dept_id#x, dept_name#x, state#x]
+:           +- View (`DEPT`, [dept_id#x,dept_name#x,state#x])
+:              +- Project [cast(dept_id#x as int) AS dept_id#x, cast(dept_name#x as string) AS dept_name#x, cast(state#x as string) AS state#x]
+:                 +- Project [dept_id#x, dept_name#x, state#x]
+:                    +- SubqueryAlias DEPT
+:                       +- LocalRelation [dept_id#x, dept_name#x, state#x]
 +- SubqueryAlias emp
-   +- Project [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
-      +- SubqueryAlias EMP
-         +- LocalRelation [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
-;
+   +- View (`EMP`, [id#x,emp_name#x,hiredate#x,salary#x,dept_id#x])
+      +- Project [cast(id#x as int) AS id#x, cast(emp_name#x as string) AS emp_name#x, cast(hiredate#x as date) AS hiredate#x, cast(salary#x as double) AS salary#x, cast(dept_id#x as int) AS dept_id#x]
+         +- Project [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
+            +- SubqueryAlias EMP
+               +- LocalRelation [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
 
 
 -- !query
@@ -396,14 +824,17 @@ IN/EXISTS predicate sub-queries can only be used in Filter/Join and a few comman
 :  +- Project [state#x]
 :     +- Filter (dept_id#x = outer(dept_id#x))
 :        +- SubqueryAlias dept
-:           +- Project [dept_id#x, dept_name#x, state#x]
-:              +- SubqueryAlias DEPT
-:                 +- LocalRelation [dept_id#x, dept_name#x, state#x]
+:           +- View (`DEPT`, [dept_id#x,dept_name#x,state#x])
+:              +- Project [cast(dept_id#x as int) AS dept_id#x, cast(dept_name#x as string) AS dept_name#x, cast(state#x as string) AS state#x]
+:                 +- Project [dept_id#x, dept_name#x, state#x]
+:                    +- SubqueryAlias DEPT
+:                       +- LocalRelation [dept_id#x, dept_name#x, state#x]
 +- SubqueryAlias emp
-   +- Project [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
-      +- SubqueryAlias EMP
-         +- LocalRelation [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
-;
+   +- View (`EMP`, [id#x,emp_name#x,hiredate#x,salary#x,dept_id#x])
+      +- Project [cast(id#x as int) AS id#x, cast(emp_name#x as string) AS emp_name#x, cast(hiredate#x as date) AS hiredate#x, cast(salary#x as double) AS salary#x, cast(dept_id#x as int) AS dept_id#x]
+         +- Project [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
+            +- SubqueryAlias EMP
+               +- LocalRelation [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
 
 
 -- !query
@@ -421,14 +852,17 @@ IN/EXISTS predicate sub-queries can only be used in Filter/Join and a few comman
 :  +- Distinct
 :     +- Project [dept_id#x]
 :        +- SubqueryAlias dept
-:           +- Project [dept_id#x, dept_name#x, state#x]
-:              +- SubqueryAlias DEPT
-:                 +- LocalRelation [dept_id#x, dept_name#x, state#x]
+:           +- View (`DEPT`, [dept_id#x,dept_name#x,state#x])
+:              +- Project [cast(dept_id#x as int) AS dept_id#x, cast(dept_name#x as string) AS dept_name#x, cast(state#x as string) AS state#x]
+:                 +- Project [dept_id#x, dept_name#x, state#x]
+:                    +- SubqueryAlias DEPT
+:                       +- LocalRelation [dept_id#x, dept_name#x, state#x]
 +- SubqueryAlias emp
-   +- Project [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
-      +- SubqueryAlias EMP
-         +- LocalRelation [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
-;
+   +- View (`EMP`, [id#x,emp_name#x,hiredate#x,salary#x,dept_id#x])
+      +- Project [cast(id#x as int) AS id#x, cast(emp_name#x as string) AS emp_name#x, cast(hiredate#x as date) AS hiredate#x, cast(salary#x as double) AS salary#x, cast(dept_id#x as int) AS dept_id#x]
+         +- Project [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
+            +- SubqueryAlias EMP
+               +- LocalRelation [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
 
 
 -- !query
@@ -446,14 +880,17 @@ IN/EXISTS predicate sub-queries can only be used in Filter/Join and a few comman
 :  +- Distinct
 :     +- Project [dept_id#x]
 :        +- SubqueryAlias dept
-:           +- Project [dept_id#x, dept_name#x, state#x]
-:              +- SubqueryAlias DEPT
-:                 +- LocalRelation [dept_id#x, dept_name#x, state#x]
+:           +- View (`DEPT`, [dept_id#x,dept_name#x,state#x])
+:              +- Project [cast(dept_id#x as int) AS dept_id#x, cast(dept_name#x as string) AS dept_name#x, cast(state#x as string) AS state#x]
+:                 +- Project [dept_id#x, dept_name#x, state#x]
+:                    +- SubqueryAlias DEPT
+:                       +- LocalRelation [dept_id#x, dept_name#x, state#x]
 +- SubqueryAlias emp
-   +- Project [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
-      +- SubqueryAlias EMP
-         +- LocalRelation [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
-;
+   +- View (`EMP`, [id#x,emp_name#x,hiredate#x,salary#x,dept_id#x])
+      +- Project [cast(id#x as int) AS id#x, cast(emp_name#x as string) AS emp_name#x, cast(hiredate#x as date) AS hiredate#x, cast(salary#x as double) AS salary#x, cast(dept_id#x as int) AS dept_id#x]
+         +- Project [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
+            +- SubqueryAlias EMP
+               +- LocalRelation [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/group-by-ordinal.sql.out b/sql/core/src/test/resources/sql-tests/results/group-by-ordinal.sql.out
index bf9f606a2224e..92e4a861fa1aa 100644
--- a/sql/core/src/test/resources/sql-tests/results/group-by-ordinal.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/group-by-ordinal.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 20
+-- Number of queries: 33
 
 
 -- !query
@@ -122,7 +122,7 @@ select a, b, sum(b) from data group by 3
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-aggregate functions are not allowed in GROUP BY, but found sum(CAST(data.`b` AS BIGINT));
+GROUP BY 3 refers to an expression that is or contains an aggregate function. Aggregate functions are not allowed in GROUP BY, but got sum(data.b) AS `sum(b)`; line 1 pos 39
 
 
 -- !query
@@ -131,7 +131,7 @@ select a, b, sum(b) + 2 from data group by 3
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-aggregate functions are not allowed in GROUP BY, but found (sum(CAST(data.`b` AS BIGINT)) + CAST(2 AS BIGINT));
+GROUP BY 3 refers to an expression that is or contains an aggregate function. Aggregate functions are not allowed in GROUP BY, but got (sum(data.b) + CAST(2 AS BIGINT)) AS `(sum(b) + 2)`; line 1 pos 43
 
 
 -- !query
@@ -155,7 +155,7 @@ select * from data group by a, b, 1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Star (*) is not allowed in select list when GROUP BY ordinal position is used;
+Star (*) is not allowed in select list when GROUP BY ordinal position is used
 
 
 -- !query
@@ -184,6 +184,204 @@ struct<a:int,k:int,count(b):bigint>
 3	3	2
 
 
+-- !query
+select a, b, count(1) from data group by cube(1, 2)
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	2	1
+1	NULL	2
+2	1	1
+2	2	1
+2	NULL	2
+3	1	1
+3	2	1
+3	NULL	2
+NULL	1	3
+NULL	2	3
+NULL	NULL	6
+
+
+-- !query
+select a, b, count(1) from data group by cube(1, b)
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	2	1
+1	NULL	2
+2	1	1
+2	2	1
+2	NULL	2
+3	1	1
+3	2	1
+3	NULL	2
+NULL	1	3
+NULL	2	3
+NULL	NULL	6
+
+
+-- !query
+select a, b, count(1) from data group by 1, 2 with cube
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	2	1
+1	NULL	2
+2	1	1
+2	2	1
+2	NULL	2
+3	1	1
+3	2	1
+3	NULL	2
+NULL	1	3
+NULL	2	3
+NULL	NULL	6
+
+
+-- !query
+select a, b, count(1) from data group by rollup(1, 2)
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	2	1
+1	NULL	2
+2	1	1
+2	2	1
+2	NULL	2
+3	1	1
+3	2	1
+3	NULL	2
+NULL	NULL	6
+
+
+-- !query
+select a, b, count(1) from data group by rollup(1, b)
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	2	1
+1	NULL	2
+2	1	1
+2	2	1
+2	NULL	2
+3	1	1
+3	2	1
+3	NULL	2
+NULL	NULL	6
+
+
+-- !query
+select a, b, count(1) from data group by 1, 2 with rollup
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	2	1
+1	NULL	2
+2	1	1
+2	2	1
+2	NULL	2
+3	1	1
+3	2	1
+3	NULL	2
+NULL	NULL	6
+
+
+-- !query
+select a, b, count(1) from data group by grouping sets((1), (2), (1, 2))
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	2	1
+1	NULL	2
+2	1	1
+2	2	1
+2	NULL	2
+3	1	1
+3	2	1
+3	NULL	2
+NULL	1	3
+NULL	2	3
+
+
+-- !query
+select a, b, count(1) from data group by grouping sets((1), (b), (a, 2))
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	2	1
+1	NULL	2
+2	1	1
+2	2	1
+2	NULL	2
+3	1	1
+3	2	1
+3	NULL	2
+NULL	1	3
+NULL	2	3
+
+
+-- !query
+select a, b, count(1) from data group by a, 2 grouping sets((1), (b), (a, 2))
+-- !query schema
+struct<a:int,b:int,count(1):bigint>
+-- !query output
+1	1	1
+1	2	1
+1	NULL	2
+2	1	1
+2	2	1
+2	NULL	2
+3	1	1
+3	2	1
+3	NULL	2
+NULL	1	3
+NULL	2	3
+
+
+-- !query
+select a, b, count(1) from data group by a, -1
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+GROUP BY position -1 is not in select list (valid range is [1, 3]); line 1 pos 44
+
+
+-- !query
+select a, b, count(1) from data group by a, 3
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+GROUP BY 3 refers to an expression that is or contains an aggregate function. Aggregate functions are not allowed in GROUP BY, but got count(1) AS `count(1)`; line 1 pos 44
+
+
+-- !query
+select a, b, count(1) from data group by cube(-1, 2)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+GROUP BY position -1 is not in select list (valid range is [1, 3]); line 1 pos 46
+
+
+-- !query
+select a, b, count(1) from data group by cube(1, 3)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+GROUP BY 3 refers to an expression that is or contains an aggregate function. Aggregate functions are not allowed in GROUP BY, but got count(1) AS `count(1)`; line 1 pos 49
+
+
 -- !query
 set spark.sql.groupByOrdinal=false
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/group-by.sql.out b/sql/core/src/test/resources/sql-tests/results/group-by.sql.out
index 7bfdd0ad53a95..b5471a785a224 100644
--- a/sql/core/src/test/resources/sql-tests/results/group-by.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/group-by.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 56
+-- Number of queries: 64
 
 
 -- !query
@@ -18,7 +18,7 @@ SELECT a, COUNT(b) FROM testData
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-grouping expressions sequence is empty, and 'testdata.`a`' is not an aggregate function. Wrap '(count(testdata.`b`) AS `count(b)`)' in windowing function(s) or wrap 'testdata.`a`' in first() (or first_value) if you don't care which value you get.;
+grouping expressions sequence is empty, and 'testdata.a' is not an aggregate function. Wrap '(count(testdata.b) AS `count(b)`)' in windowing function(s) or wrap 'testdata.a' in first() (or first_value) if you don't care which value you get.
 
 
 -- !query
@@ -46,7 +46,7 @@ SELECT a, COUNT(b) FROM testData GROUP BY b
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-expression 'testdata.`a`' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.;
+expression 'testdata.a' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.
 
 
 -- !query
@@ -87,7 +87,7 @@ struct<foo:string,approx_count_distinct(a):bigint>
 -- !query
 SELECT 'foo', MAX(STRUCT(a)) FROM testData WHERE a = 0 GROUP BY 1
 -- !query schema
-struct<foo:string,max(named_struct(a, a)):struct<a:int>>
+struct<foo:string,max(struct(a)):struct<a:int>>
 -- !query output
 
 
@@ -110,7 +110,7 @@ SELECT a + 2, COUNT(b) FROM testData GROUP BY a + 1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-expression 'testdata.`a`' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.;
+expression 'testdata.a' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.
 
 
 -- !query
@@ -128,7 +128,7 @@ NULL	1
 SELECT SKEWNESS(a), KURTOSIS(a), MIN(a), MAX(a), AVG(a), VARIANCE(a), STDDEV(a), SUM(a), COUNT(a)
 FROM testData
 -- !query schema
-struct<skewness(CAST(a AS DOUBLE)):double,kurtosis(CAST(a AS DOUBLE)):double,min(a):int,max(a):int,avg(a):double,variance(CAST(a AS DOUBLE)):double,stddev(CAST(a AS DOUBLE)):double,sum(a):bigint,count(a):bigint>
+struct<skewness(a):double,kurtosis(a):double,min(a):int,max(a):int,avg(a):double,variance(a):double,stddev(a):double,sum(a):bigint,count(a):bigint>
 -- !query output
 -0.2723801058145729	-1.5069204152249134	1	3	2.142857142857143	0.8095238095238094	0.8997354108424372	15	7
 
@@ -167,7 +167,7 @@ SELECT COUNT(b) AS k FROM testData GROUP BY k
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-aggregate functions are not allowed in GROUP BY, but found count(testdata.`b`);
+aggregate functions are not allowed in GROUP BY, but found count(testdata.b)
 
 
 -- !query
@@ -185,7 +185,7 @@ SELECT k AS a, COUNT(v) FROM testDataHasSameNameWithAlias GROUP BY a
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-expression 'testdatahassamenamewithalias.`k`' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.;
+expression 'testdatahassamenamewithalias.k' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.
 
 
 -- !query
@@ -202,7 +202,7 @@ SELECT a AS k, COUNT(b) FROM testData GROUP BY k
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`k`' given input columns: [testdata.a, testdata.b]; line 1 pos 47
+cannot resolve 'k' given input columns: [testdata.a, testdata.b]; line 1 pos 47
 
 
 -- !query
@@ -247,7 +247,7 @@ struct<1:int>
 SELECT corr(DISTINCT x, y), corr(DISTINCT y, x), count(*)
   FROM (VALUES (1, 1), (2, 2), (2, 2)) t(x, y)
 -- !query schema
-struct<corr(DISTINCT CAST(x AS DOUBLE), CAST(y AS DOUBLE)):double,corr(DISTINCT CAST(y AS DOUBLE), CAST(x AS DOUBLE)):double,count(1):bigint>
+struct<corr(DISTINCT x, y):double,corr(DISTINCT y, x):double,count(1):bigint>
 -- !query output
 1.0	1.0	3
 
@@ -274,7 +274,68 @@ SELECT id FROM range(10) HAVING id > 0
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-grouping expressions sequence is empty, and '`id`' is not an aggregate function. Wrap '()' in windowing function(s) or wrap '`id`' in first() (or first_value) if you don't care which value you get.;
+grouping expressions sequence is empty, and 'id' is not an aggregate function. Wrap '()' in windowing function(s) or wrap 'id' in first() (or first_value) if you don't care which value you get.
+
+
+-- !query
+SET spark.sql.legacy.parser.havingWithoutGroupByAsWhere=true
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+spark.sql.legacy.parser.havingWithoutGroupByAsWhere	true
+
+
+-- !query
+SELECT 1 FROM range(10) HAVING true
+-- !query schema
+struct<1:int>
+-- !query output
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+
+
+-- !query
+SELECT 1 FROM range(10) HAVING MAX(id) > 0
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+
+Aggregate/Window/Generate expressions are not valid in where clause of the query.
+Expression in where clause: [(max(id) > CAST(0 AS BIGINT))]
+Invalid expressions: [max(id)]
+
+
+-- !query
+SELECT id FROM range(10) HAVING id > 0
+-- !query schema
+struct<id:bigint>
+-- !query output
+1
+2
+3
+4
+5
+6
+7
+8
+9
+
+
+-- !query
+SET spark.sql.legacy.parser.havingWithoutGroupByAsWhere=false
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+spark.sql.legacy.parser.havingWithoutGroupByAsWhere	false
 
 
 -- !query
@@ -548,7 +609,7 @@ org.apache.spark.sql.AnalysisException
 
 Aggregate/Window/Generate expressions are not valid in where clause of the query.
 Expression in where clause: [(count(1) > 1L)]
-Invalid expressions: [count(1)];
+Invalid expressions: [count(1)]
 
 
 -- !query
@@ -560,7 +621,7 @@ org.apache.spark.sql.AnalysisException
 
 Aggregate/Window/Generate expressions are not valid in where clause of the query.
 Expression in where clause: [((count(1) + 1L) > 1L)]
-Invalid expressions: [count(1)];
+Invalid expressions: [count(1)]
 
 
 -- !query
@@ -571,5 +632,35 @@ struct<>
 org.apache.spark.sql.AnalysisException
 
 Aggregate/Window/Generate expressions are not valid in where clause of the query.
-Expression in where clause: [(((test_agg.`k` = 1) OR (test_agg.`k` = 2)) OR (((count(1) + 1L) > 1L) OR (max(test_agg.`k`) > 1)))]
-Invalid expressions: [count(1), max(test_agg.`k`)];
+Expression in where clause: [(((test_agg.k = 1) OR (test_agg.k = 2)) OR (((count(1) + 1L) > 1L) OR (max(test_agg.k) > 1)))]
+Invalid expressions: [count(1), max(test_agg.k)]
+
+
+-- !query
+SELECT AVG(DISTINCT decimal_col), SUM(DISTINCT decimal_col) FROM VALUES (CAST(1 AS DECIMAL(9, 0))) t(decimal_col)
+-- !query schema
+struct<avg(DISTINCT decimal_col):decimal(13,4),sum(DISTINCT decimal_col):decimal(19,0)>
+-- !query output
+1.0000	1
+
+
+-- !query
+SELECT not(a IS NULL), count(*) AS c
+FROM testData
+GROUP BY a IS NULL
+-- !query schema
+struct<(NOT (a IS NULL)):boolean,c:bigint>
+-- !query output
+false	2
+true	7
+
+
+-- !query
+SELECT if(not(a IS NULL), rand(0), 1), count(*) AS c
+FROM testData
+GROUP BY a IS NULL
+-- !query schema
+struct<(IF((NOT (a IS NULL)), rand(0), 1)):double,c:bigint>
+-- !query output
+0.7604953758285915	7
+1.0	2
diff --git a/sql/core/src/test/resources/sql-tests/results/grouping_set.sql.out b/sql/core/src/test/resources/sql-tests/results/grouping_set.sql.out
index 7089e10cdef27..6af8e7048c815 100644
--- a/sql/core/src/test/resources/sql-tests/results/grouping_set.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/grouping_set.sql.out
@@ -165,7 +165,7 @@ SELECT c1 FROM (values (1,2), (3,2)) t(c1, c2) GROUP BY GROUPING SETS (())
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-expression '`c1`' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.;
+expression 't.c1' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/having.sql.out b/sql/core/src/test/resources/sql-tests/results/having.sql.out
index 1b3ac7865159f..22e71f4f8a687 100644
--- a/sql/core/src/test/resources/sql-tests/results/having.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/having.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 9
+-- Number of queries: 13
 
 
 -- !query
@@ -43,7 +43,7 @@ struct<min(v):int>
 -- !query
 SELECT a + b FROM VALUES (1L, 2), (3L, 4) AS T(a, b) GROUP BY a + b HAVING a + b > 1
 -- !query schema
-struct<(a + CAST(b AS BIGINT)):bigint>
+struct<(a + b):bigint>
 -- !query output
 3
 7
@@ -81,3 +81,35 @@ SELECT SUM(a) AS b FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY ROLLUP(a, b)
 struct<b:bigint>
 -- !query output
 2
+
+
+-- !query
+SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY GROUPING SETS(t.c1) HAVING t.c1 = 1
+-- !query schema
+struct<c1:int>
+-- !query output
+1
+
+
+-- !query
+SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY CUBE(t.c1) HAVING t.c1 = 1
+-- !query schema
+struct<c1:int>
+-- !query output
+1
+
+
+-- !query
+SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY ROLLUP(t.c1) HAVING t.c1 = 1
+-- !query schema
+struct<c1:int>
+-- !query output
+1
+
+
+-- !query
+SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY t.c1 HAVING t.c1 = 1
+-- !query schema
+struct<c1:int>
+-- !query output
+1
diff --git a/sql/core/src/test/resources/sql-tests/results/higher-order-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/higher-order-functions.sql.out
index d35d0d5d944bb..7b31b5690998c 100644
--- a/sql/core/src/test/resources/sql-tests/results/higher-order-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/higher-order-functions.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 29
+-- Number of queries: 30
 
 
 -- !query
@@ -270,3 +270,11 @@ select transform(ys, (all, i) -> all + i) as v from values (array(32, 97)) as t(
 struct<v:array<int>>
 -- !query output
 [32,98]
+
+
+-- !query
+select aggregate(split('abcdefgh',''), array(array('')), (acc, x) -> array(array(x)))
+-- !query schema
+struct<aggregate(split(abcdefgh, , -1), array(array()), lambdafunction(array(array(namedlambdavariable())), namedlambdavariable(), namedlambdavariable()), lambdafunction(namedlambdavariable(), namedlambdavariable())):array<array<string>>>
+-- !query output
+[[""]]
diff --git a/sql/core/src/test/resources/sql-tests/results/intersect-all.sql.out b/sql/core/src/test/resources/sql-tests/results/intersect-all.sql.out
index 4762082dc3be2..caba8c6942c55 100644
--- a/sql/core/src/test/resources/sql-tests/results/intersect-all.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/intersect-all.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 22
+-- Number of queries: 26
 
 
 -- !query
@@ -98,7 +98,7 @@ SELECT array(1), 2
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-IntersectAll can only be performed on tables with the compatible column types. array<int> <> int at the first column of the second table;
+IntersectAll can only be performed on tables with the compatible column types. array<int> <> int at the first column of the second table
 
 
 -- !query
@@ -109,7 +109,7 @@ SELECT k, v FROM tab2
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-IntersectAll can only be performed on tables with the same number of columns, but the first table has 1 columns and the second table has 2 columns;
+IntersectAll can only be performed on tables with the same number of columns, but the first table has 1 columns and the second table has 2 columns
 
 
 -- !query
@@ -291,6 +291,38 @@ struct<key:string,value:string>
 spark.sql.legacy.setopsPrecedence.enabled	false
 
 
+-- !query
+CREATE OR REPLACE TEMPORARY VIEW tab3 AS VALUES (decimal(1)), (decimal(2)) tbl3(v)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SELECT t.v FROM (
+  SELECT v FROM tab3
+  INTERSECT
+  SELECT v + v AS v FROM tab3
+) t
+-- !query schema
+struct<v:decimal(11,0)>
+-- !query output
+2
+
+
+-- !query
+SELECT SUM(t.v) FROM (
+  SELECT v FROM tab3
+  INTERSECT
+  SELECT v + v AS v FROM tab3
+) t
+-- !query schema
+struct<sum(v):decimal(21,0)>
+-- !query output
+2
+
+
 -- !query
 DROP VIEW IF EXISTS tab1
 -- !query schema
@@ -305,3 +337,11 @@ DROP VIEW IF EXISTS tab2
 struct<>
 -- !query output
 
+
+
+-- !query
+DROP VIEW IF EXISTS tab3
+-- !query schema
+struct<>
+-- !query output
+
diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
index 01db43ce9e8bc..ef9ef8f67bdd7 100644
--- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
@@ -1,19 +1,19 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 91
+-- Number of queries: 134
 
 
 -- !query
 select 3 * (timestamp'2019-10-15 10:11:12.001002' - date'2019-10-15')
 -- !query schema
-struct<multiply_interval(subtracttimestamps(TIMESTAMP '2019-10-15 10:11:12.001002', CAST(DATE '2019-10-15' AS TIMESTAMP)), CAST(3 AS DOUBLE)):interval>
+struct<((TIMESTAMP '2019-10-15 10:11:12.001002' - DATE '2019-10-15') * 3):interval day to second>
 -- !query output
-30 hours 33 minutes 36.003006 seconds
+1 06:33:36.003006000
 
 
 -- !query
 select interval 4 month 2 weeks 3 microseconds * 1.5
 -- !query schema
-struct<multiply_interval(INTERVAL '4 months 14 days 0.000003 seconds', CAST(1.5 AS DOUBLE)):interval>
+struct<multiply_interval(INTERVAL '4 months 14 days 0.000003 seconds', 1.5):interval>
 -- !query output
 6 months 21 days 0.000005 seconds
 
@@ -21,15 +21,15 @@ struct<multiply_interval(INTERVAL '4 months 14 days 0.000003 seconds', CAST(1.5
 -- !query
 select (timestamp'2019-10-15' - timestamp'2019-10-14') / 1.5
 -- !query schema
-struct<divide_interval(subtracttimestamps(TIMESTAMP '2019-10-15 00:00:00', TIMESTAMP '2019-10-14 00:00:00'), CAST(1.5 AS DOUBLE)):interval>
+struct<((TIMESTAMP '2019-10-15 00:00:00' - TIMESTAMP '2019-10-14 00:00:00') / 1.5):interval day to second>
 -- !query output
-16 hours
+0 16:00:00.000000000
 
 
 -- !query
 select interval 2147483647 month * 2
 -- !query schema
-struct<multiply_interval(INTERVAL '178956970 years 7 months', CAST(2 AS DOUBLE)):interval>
+struct<multiply_interval(INTERVAL '178956970 years 7 months', 2):interval>
 -- !query output
 178956970 years 7 months
 
@@ -37,7 +37,7 @@ struct<multiply_interval(INTERVAL '178956970 years 7 months', CAST(2 AS DOUBLE))
 -- !query
 select interval 2147483647 month / 0.5
 -- !query schema
-struct<divide_interval(INTERVAL '178956970 years 7 months', CAST(0.5 AS DOUBLE)):interval>
+struct<divide_interval(INTERVAL '178956970 years 7 months', 0.5):interval>
 -- !query output
 178956970 years 7 months
 
@@ -45,7 +45,7 @@ struct<divide_interval(INTERVAL '178956970 years 7 months', CAST(0.5 AS DOUBLE))
 -- !query
 select interval 2147483647 day * 2
 -- !query schema
-struct<multiply_interval(INTERVAL '2147483647 days', CAST(2 AS DOUBLE)):interval>
+struct<multiply_interval(INTERVAL '2147483647 days', 2):interval>
 -- !query output
 2147483647 days 2562047788 hours 54.775807 seconds
 
@@ -53,7 +53,7 @@ struct<multiply_interval(INTERVAL '2147483647 days', CAST(2 AS DOUBLE)):interval
 -- !query
 select interval 2147483647 day / 0.5
 -- !query schema
-struct<divide_interval(INTERVAL '2147483647 days', CAST(0.5 AS DOUBLE)):interval>
+struct<divide_interval(INTERVAL '2147483647 days', 0.5):interval>
 -- !query output
 2147483647 days 2562047788 hours 54.775807 seconds
 
@@ -61,7 +61,7 @@ struct<divide_interval(INTERVAL '2147483647 days', CAST(0.5 AS DOUBLE)):interval
 -- !query
 select interval '2 seconds' / 0
 -- !query schema
-struct<divide_interval(INTERVAL '2 seconds', CAST(0 AS DOUBLE)):interval>
+struct<divide_interval(INTERVAL '2 seconds', 0):interval>
 -- !query output
 NULL
 
@@ -69,7 +69,7 @@ NULL
 -- !query
 select interval '2 seconds' / null
 -- !query schema
-struct<divide_interval(INTERVAL '2 seconds', CAST(NULL AS DOUBLE)):interval>
+struct<divide_interval(INTERVAL '2 seconds', NULL):interval>
 -- !query output
 NULL
 
@@ -77,7 +77,7 @@ NULL
 -- !query
 select interval '2 seconds' * null
 -- !query schema
-struct<multiply_interval(INTERVAL '2 seconds', CAST(NULL AS DOUBLE)):interval>
+struct<multiply_interval(INTERVAL '2 seconds', NULL):interval>
 -- !query output
 NULL
 
@@ -85,7 +85,7 @@ NULL
 -- !query
 select null * interval '2 seconds'
 -- !query schema
-struct<multiply_interval(INTERVAL '2 seconds', CAST(NULL AS DOUBLE)):interval>
+struct<multiply_interval(INTERVAL '2 seconds', NULL):interval>
 -- !query output
 NULL
 
@@ -122,6 +122,54 @@ struct<(+ INTERVAL '-1 months 1 days -1 seconds'):interval>
 -1 months 1 days -1 seconds
 
 
+-- !query
+select interval -'1-1' year to month
+-- !query schema
+struct<INTERVAL '-1-1' YEAR TO MONTH:interval year to month>
+-- !query output
+-1-1
+
+
+-- !query
+select interval -'-1-1' year to month
+-- !query schema
+struct<INTERVAL '1-1' YEAR TO MONTH:interval year to month>
+-- !query output
+1-1
+
+
+-- !query
+select interval +'-1-1' year to month
+-- !query schema
+struct<INTERVAL '-1-1' YEAR TO MONTH:interval year to month>
+-- !query output
+-1-1
+
+
+-- !query
+select interval - '1 2:3:4.001' day to second
+-- !query schema
+struct<INTERVAL '-1 02:03:04.001' DAY TO SECOND:interval day to second>
+-- !query output
+-1 02:03:04.001000000
+
+
+-- !query
+select interval +'1 2:3:4.001' day to second
+-- !query schema
+struct<INTERVAL '1 02:03:04.001' DAY TO SECOND:interval day to second>
+-- !query output
+1 02:03:04.001000000
+
+
+-- !query
+select interval -'-1 2:3:4.001' day to second
+-- !query schema
+struct<INTERVAL '1 02:03:04.001' DAY TO SECOND:interval day to second>
+-- !query output
+1 02:03:04.001000000
+
+
 -- !query
 select make_interval(1)
 -- !query schema
@@ -173,11 +221,27 @@ struct<make_interval(1, 2, 3, 4, 5, 6, 0.000000):interval>
 -- !query
 select make_interval(1, 2, 3, 4, 5, 6, 7.008009)
 -- !query schema
-struct<make_interval(1, 2, 3, 4, 5, 6, CAST(7.008009 AS DECIMAL(8,6))):interval>
+struct<make_interval(1, 2, 3, 4, 5, 6, 7.008009):interval>
 -- !query output
 1 years 2 months 25 days 5 hours 6 minutes 7.008009 seconds
 
 
+-- !query
+select make_interval(1, 2, 3, 4, 0, 0, 123456789012.123456)
+-- !query schema
+struct<make_interval(1, 2, 3, 4, 0, 0, 123456789012.123456):interval>
+-- !query output
+1 years 2 months 25 days 34293552 hours 30 minutes 12.123456 seconds
+
+
+-- !query
+select make_interval(0, 0, 0, 0, 0, 0, 1234567890123456789)
+-- !query schema
+struct<make_interval(0, 0, 0, 0, 0, 0, 1234567890123456789):interval>
+-- !query output
+NULL
+
+
 -- !query
 select cast('1 second' as interval)
 -- !query schema
@@ -258,76 +322,92 @@ struct<INTERVAL '32 years 1 months -100 days 41 hours 24 minutes 59.889987 secon
 32 years 1 months -100 days 41 hours 24 minutes 59.889987 seconds
 
 
+-- !query
+select interval '0-0' year to month
+-- !query schema
+struct<INTERVAL '0-0' YEAR TO MONTH:interval year to month>
+-- !query output
+0-0
+
+
+-- !query
+select interval '0 0:0:0' day to second
+-- !query schema
+struct<INTERVAL '0 00:00:00' DAY TO SECOND:interval day to second>
+-- !query output
+0 00:00:00.000000000
+
+
 -- !query
 select interval '0 0:0:0.1' day to second
 -- !query schema
-struct<INTERVAL '0.1 seconds':interval>
+struct<INTERVAL '0 00:00:00.1' DAY TO SECOND:interval day to second>
 -- !query output
-0.1 seconds
+0 00:00:00.100000000
 
 
 -- !query
 select interval '10-9' year to month
 -- !query schema
-struct<INTERVAL '10 years 9 months':interval>
+struct<INTERVAL '10-9' YEAR TO MONTH:interval year to month>
 -- !query output
-10 years 9 months
+10-9
 
 
 -- !query
 select interval '20 15' day to hour
 -- !query schema
-struct<INTERVAL '20 days 15 hours':interval>
+struct<INTERVAL '20 15:00:00' DAY TO SECOND:interval day to second>
 -- !query output
-20 days 15 hours
+20 15:00:00.000000000
 
 
 -- !query
 select interval '20 15:40' day to minute
 -- !query schema
-struct<INTERVAL '20 days 15 hours 40 minutes':interval>
+struct<INTERVAL '20 15:40:00' DAY TO SECOND:interval day to second>
 -- !query output
-20 days 15 hours 40 minutes
+20 15:40:00.000000000
 
 
 -- !query
 select interval '20 15:40:32.99899999' day to second
 -- !query schema
-struct<INTERVAL '20 days 15 hours 40 minutes 32.998999 seconds':interval>
+struct<INTERVAL '20 15:40:32.998999' DAY TO SECOND:interval day to second>
 -- !query output
-20 days 15 hours 40 minutes 32.998999 seconds
+20 15:40:32.998999000
 
 
 -- !query
 select interval '15:40' hour to minute
 -- !query schema
-struct<INTERVAL '15 hours 40 minutes':interval>
+struct<INTERVAL '0 15:40:00' DAY TO SECOND:interval day to second>
 -- !query output
-15 hours 40 minutes
+0 15:40:00.000000000
 
 
 -- !query
 select interval '15:40:32.99899999' hour to second
 -- !query schema
-struct<INTERVAL '15 hours 40 minutes 32.998999 seconds':interval>
+struct<INTERVAL '0 15:40:32.998999' DAY TO SECOND:interval day to second>
 -- !query output
-15 hours 40 minutes 32.998999 seconds
+0 15:40:32.998999000
 
 
 -- !query
 select interval '40:32.99899999' minute to second
 -- !query schema
-struct<INTERVAL '40 minutes 32.998999 seconds':interval>
+struct<INTERVAL '0 00:40:32.998999' DAY TO SECOND:interval day to second>
 -- !query output
-40 minutes 32.998999 seconds
+0 00:40:32.998999000
 
 
 -- !query
 select interval '40:32' minute to second
 -- !query schema
-struct<INTERVAL '40 minutes 32 seconds':interval>
+struct<INTERVAL '0 00:40:32' DAY TO SECOND:interval day to second>
 -- !query output
-40 minutes 32 seconds
+0 00:40:32.000000000
 
 
 -- !query
@@ -338,6 +418,14 @@ struct<day:interval>
 30 days
 
 
+-- !query
+select interval 30 days days
+-- !query schema
+struct<days:interval>
+-- !query output
+30 days
+
+
 -- !query
 select interval '20 15:40:32.99899999' day to hour
 -- !query schema
@@ -654,6 +742,76 @@ select interval 30 day day day
 ---------------------------^^^
 
 
+-- !query
+select interval (-30) days
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Undefined function: 'interval'. This function is neither a registered temporary function nor a permanent function registered in the database 'default'.; line 1 pos 7
+
+
+-- !query
+select interval (a + 1) days
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Undefined function: 'interval'. This function is neither a registered temporary function nor a permanent function registered in the database 'default'.; line 1 pos 7
+
+
+-- !query
+select interval 30 days days days
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+extraneous input 'days' expecting {<EOF>, ';'}(line 1, pos 29)
+
+== SQL ==
+select interval 30 days days days
+-----------------------------^^^
+
+
+-- !query
+SELECT INTERVAL '178956970-7' YEAR TO MONTH
+-- !query schema
+struct<INTERVAL '178956970-7' YEAR TO MONTH:interval year to month>
+-- !query output
+178956970-7
+
+
+-- !query
+SELECT INTERVAL '178956970-8' YEAR TO MONTH
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Error parsing interval year-month string: integer overflow(line 1, pos 16)
+
+== SQL ==
+SELECT INTERVAL '178956970-8' YEAR TO MONTH
+----------------^^^
+
+
+-- !query
+SELECT INTERVAL '-178956970-8' YEAR TO MONTH
+-- !query schema
+struct<INTERVAL '-178956970-8' YEAR TO MONTH:interval year to month>
+-- !query output
+-178956970-8
+
+
+-- !query
+SELECT INTERVAL -'178956970-8' YEAR TO MONTH
+-- !query schema
+struct<INTERVAL '-178956970-8' YEAR TO MONTH:interval year to month>
+-- !query output
+-178956970-8
+
+
 -- !query
 create temporary view interval_arithmetic as
   select CAST(dateval AS date), CAST(tsval AS timestamp), dateval as strval from values
@@ -676,7 +834,7 @@ select
   interval '2-2' year to month + dateval
 from interval_arithmetic
 -- !query schema
-struct<dateval:date,dateval - INTERVAL '2 years 2 months':date,dateval - INTERVAL '-2 years -2 months':date,dateval + INTERVAL '2 years 2 months':date,dateval + INTERVAL '-2 years -2 months':date,dateval + (- INTERVAL '2 years 2 months'):date,dateval + INTERVAL '2 years 2 months':date>
+struct<dateval:date,dateval - INTERVAL '2-2' YEAR TO MONTH:date,dateval - INTERVAL '-2-2' YEAR TO MONTH:date,dateval + INTERVAL '2-2' YEAR TO MONTH:date,dateval + INTERVAL '-2-2' YEAR TO MONTH:date,dateval + (- INTERVAL '2-2' YEAR TO MONTH):date,dateval + INTERVAL '2-2' YEAR TO MONTH:date>
 -- !query output
 2012-01-01	2009-11-01	2014-03-01	2014-03-01	2009-11-01	2009-11-01	2014-03-01
 
@@ -692,7 +850,7 @@ select
   interval '2-2' year to month + tsval
 from interval_arithmetic
 -- !query schema
-struct<tsval:timestamp,CAST(tsval - INTERVAL '2 years 2 months' AS TIMESTAMP):timestamp,CAST(tsval - INTERVAL '-2 years -2 months' AS TIMESTAMP):timestamp,CAST(tsval + INTERVAL '2 years 2 months' AS TIMESTAMP):timestamp,CAST(tsval + INTERVAL '-2 years -2 months' AS TIMESTAMP):timestamp,CAST(tsval + (- INTERVAL '2 years 2 months') AS TIMESTAMP):timestamp,CAST(tsval + INTERVAL '2 years 2 months' AS TIMESTAMP):timestamp>
+struct<tsval:timestamp,tsval - INTERVAL '2-2' YEAR TO MONTH:timestamp,tsval - INTERVAL '-2-2' YEAR TO MONTH:timestamp,tsval + INTERVAL '2-2' YEAR TO MONTH:timestamp,tsval + INTERVAL '-2-2' YEAR TO MONTH:timestamp,tsval + (- INTERVAL '2-2' YEAR TO MONTH):timestamp,tsval + INTERVAL '2-2' YEAR TO MONTH:timestamp>
 -- !query output
 2012-01-01 00:00:00	2009-11-01 00:00:00	2014-03-01 00:00:00	2014-03-01 00:00:00	2009-11-01 00:00:00	2009-11-01 00:00:00	2014-03-01 00:00:00
 
@@ -703,9 +861,9 @@ select
   interval '2-2' year to month - interval '3-3' year to month
 from interval_arithmetic
 -- !query schema
-struct<(INTERVAL '2 years 2 months' + INTERVAL '3 years 3 months'):interval,(INTERVAL '2 years 2 months' - INTERVAL '3 years 3 months'):interval>
+struct<(INTERVAL '2-2' YEAR TO MONTH + INTERVAL '3-3' YEAR TO MONTH):interval year to month,(INTERVAL '2-2' YEAR TO MONTH - INTERVAL '3-3' YEAR TO MONTH):interval year to month>
 -- !query output
-5 years 5 months	-1 years -1 months
+5-5	-1-1
 
 
 -- !query
@@ -719,9 +877,9 @@ select
   interval '99 11:22:33.123456789' day to second + dateval
 from interval_arithmetic
 -- !query schema
-struct<dateval:date,dateval - INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds':date,dateval - INTERVAL '-99 days -11 hours -22 minutes -33.123456 seconds':date,dateval + INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds':date,dateval + INTERVAL '-99 days -11 hours -22 minutes -33.123456 seconds':date,dateval + (- INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds'):date,dateval + INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds':date>
+struct<dateval:date,dateval - INTERVAL '99 11:22:33.123456' DAY TO SECOND:timestamp,dateval - INTERVAL '-99 11:22:33.123456' DAY TO SECOND:timestamp,dateval + INTERVAL '99 11:22:33.123456' DAY TO SECOND:timestamp,dateval + INTERVAL '-99 11:22:33.123456' DAY TO SECOND:timestamp,dateval + (- INTERVAL '99 11:22:33.123456' DAY TO SECOND):timestamp,dateval + INTERVAL '99 11:22:33.123456' DAY TO SECOND:timestamp>
 -- !query output
-2012-01-01	2011-09-23	2012-04-09	2012-04-09	2011-09-23	2011-09-23	2012-04-09
+2012-01-01	2011-09-23 12:37:26.876544	2012-04-09 11:22:33.123456	2012-04-09 11:22:33.123456	2011-09-23 12:37:26.876544	2011-09-23 12:37:26.876544	2012-04-09 11:22:33.123456
 
 
 -- !query
@@ -735,7 +893,7 @@ select
   interval '99 11:22:33.123456789' day to second + tsval
 from interval_arithmetic
 -- !query schema
-struct<tsval:timestamp,CAST(tsval - INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' AS TIMESTAMP):timestamp,CAST(tsval - INTERVAL '-99 days -11 hours -22 minutes -33.123456 seconds' AS TIMESTAMP):timestamp,CAST(tsval + INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' AS TIMESTAMP):timestamp,CAST(tsval + INTERVAL '-99 days -11 hours -22 minutes -33.123456 seconds' AS TIMESTAMP):timestamp,CAST(tsval + (- INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds') AS TIMESTAMP):timestamp,CAST(tsval + INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' AS TIMESTAMP):timestamp>
+struct<tsval:timestamp,tsval - INTERVAL '99 11:22:33.123456' DAY TO SECOND:timestamp,tsval - INTERVAL '-99 11:22:33.123456' DAY TO SECOND:timestamp,tsval + INTERVAL '99 11:22:33.123456' DAY TO SECOND:timestamp,tsval + INTERVAL '-99 11:22:33.123456' DAY TO SECOND:timestamp,tsval + (- INTERVAL '99 11:22:33.123456' DAY TO SECOND):timestamp,tsval + INTERVAL '99 11:22:33.123456' DAY TO SECOND:timestamp>
 -- !query output
 2012-01-01 00:00:00	2011-09-23 12:37:26.876544	2012-04-09 11:22:33.123456	2012-04-09 11:22:33.123456	2011-09-23 12:37:26.876544	2011-09-23 12:37:26.876544	2012-04-09 11:22:33.123456
 
@@ -751,7 +909,7 @@ select
   interval '99 11:22:33.123456789' day to second + strval
 from interval_arithmetic
 -- !query schema
-struct<strval:string,CAST(strval - INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' AS STRING):string,CAST(strval - INTERVAL '-99 days -11 hours -22 minutes -33.123456 seconds' AS STRING):string,CAST(CAST(strval AS TIMESTAMP) + INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' AS STRING):string,CAST(CAST(strval AS TIMESTAMP) + INTERVAL '-99 days -11 hours -22 minutes -33.123456 seconds' AS STRING):string,CAST(CAST(strval AS TIMESTAMP) + (- INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds') AS STRING):string,CAST(CAST(strval AS TIMESTAMP) + INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' AS STRING):string>
+struct<strval:string,strval - INTERVAL '99 11:22:33.123456' DAY TO SECOND:string,strval - INTERVAL '-99 11:22:33.123456' DAY TO SECOND:string,strval + INTERVAL '99 11:22:33.123456' DAY TO SECOND:string,strval + INTERVAL '-99 11:22:33.123456' DAY TO SECOND:string,strval + (- INTERVAL '99 11:22:33.123456' DAY TO SECOND):string,strval + INTERVAL '99 11:22:33.123456' DAY TO SECOND:string>
 -- !query output
 2012-01-01	2011-09-23 12:37:26.876544	2012-04-09 11:22:33.123456	2012-04-09 11:22:33.123456	2011-09-23 12:37:26.876544	2011-09-23 12:37:26.876544	2012-04-09 11:22:33.123456
 
@@ -762,9 +920,9 @@ select
   interval '99 11:22:33.123456789' day to second - interval '10 9:8:7.123456789' day to second
 from interval_arithmetic
 -- !query schema
-struct<(INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' + INTERVAL '10 days 9 hours 8 minutes 7.123456 seconds'):interval,(INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' - INTERVAL '10 days 9 hours 8 minutes 7.123456 seconds'):interval>
+struct<(INTERVAL '99 11:22:33.123456' DAY TO SECOND + INTERVAL '10 09:08:07.123456' DAY TO SECOND):interval day to second,(INTERVAL '99 11:22:33.123456' DAY TO SECOND - INTERVAL '10 09:08:07.123456' DAY TO SECOND):interval day to second>
 -- !query output
-109 days 20 hours 30 minutes 40.246912 seconds	89 days 2 hours 14 minutes 26 seconds
+109 20:30:40.246912000	89 02:14:26.000000000
 
 
 -- !query
@@ -810,9 +968,9 @@ struct<INTERVAL '1 days':interval>
 -- !query
 select interval '2-2\t' year to month
 -- !query schema
-struct<INTERVAL '2 years 2 months':interval>
+struct<INTERVAL '2-2' YEAR TO MONTH:interval year to month>
 -- !query output
-2 years 2 months
+2-2
 
 
 -- !query
@@ -832,9 +990,9 @@ select interval '-\t2-2\t' year to month
 -- !query
 select interval '\n0 12:34:46.789\t' day to second
 -- !query schema
-struct<INTERVAL '12 hours 34 minutes 46.789 seconds':interval>
+struct<INTERVAL '0 12:34:46.789' DAY TO SECOND:interval day to second>
 -- !query output
-12 hours 34 minutes 46.789 seconds
+0 12:34:46.789000000
 
 
 -- !query
@@ -852,6 +1010,48 @@ select interval '\n-\t10\t 12:34:46.789\t' day to second
 ----------------^^^
 
 
+-- !query
+select interval '中文 interval 1 day'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 中文 interval 1 day(line 1, pos 7)
+
+== SQL ==
+select interval '中文 interval 1 day'
+-------^^^
+
+
+-- !query
+select interval 'interval中文 1 day'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: interval中文 1 day(line 1, pos 7)
+
+== SQL ==
+select interval 'interval中文 1 day'
+-------^^^
+
+
+-- !query
+select interval 'interval 1中文day'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: interval 1中文day(line 1, pos 7)
+
+== SQL ==
+select interval 'interval 1中文day'
+-------^^^
+
+
 -- !query
 select -(a) from values (interval '-2147483648 months', interval '2147483647 months') t(a, b)
 -- !query schema
@@ -879,7 +1079,7 @@ struct<(b + INTERVAL '1 months'):interval>
 -- !query
 select a * 1.1 from values (interval '-2147483648 months', interval '2147483647 months') t(a, b)
 -- !query schema
-struct<multiply_interval(a, CAST(1.1 AS DOUBLE)):interval>
+struct<multiply_interval(a, 1.1):interval>
 -- !query output
 -178956970 years -8 months
 
@@ -887,7 +1087,7 @@ struct<multiply_interval(a, CAST(1.1 AS DOUBLE)):interval>
 -- !query
 select a / 0.5 from values (interval '-2147483648 months', interval '2147483647 months') t(a, b)
 -- !query schema
-struct<divide_interval(a, CAST(0.5 AS DOUBLE)):interval>
+struct<divide_interval(a, 0.5):interval>
 -- !query output
 -178956970 years -8 months
 
@@ -914,3 +1114,241 @@ SELECT
 struct<from_json({"a":"1 days"}):struct<a:interval>,to_json(from_json({"a":"1 days"})):string,to_json(map(a, INTERVAL '2 years 1 months 100 days 2 hours 10 minutes')):string,from_json(to_json(map(a, INTERVAL '2 years 1 months 100 days 2 hours 10 minutes'))):struct<a:interval>>
 -- !query output
 {"a":1 days}	{"a":"1 days"}	{"a":"2 years 1 months 100 days 2 hours 10 minutes"}	{"a":2 years 1 months 100 days 2 hours 10 minutes}
+
+
+-- !query
+select interval '+'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: +(line 1, pos 7)
+
+== SQL ==
+select interval '+'
+-------^^^
+
+
+-- !query
+select interval '+.'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: +.(line 1, pos 7)
+
+== SQL ==
+select interval '+.'
+-------^^^
+
+
+-- !query
+select interval '1'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 1(line 1, pos 7)
+
+== SQL ==
+select interval '1'
+-------^^^
+
+
+-- !query
+select interval '1.2'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 1.2(line 1, pos 7)
+
+== SQL ==
+select interval '1.2'
+-------^^^
+
+
+-- !query
+select interval '- 2'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: - 2(line 1, pos 7)
+
+== SQL ==
+select interval '- 2'
+-------^^^
+
+
+-- !query
+select interval '1 day -'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 1 day -(line 1, pos 7)
+
+== SQL ==
+select interval '1 day -'
+-------^^^
+
+
+-- !query
+select interval '1 day 1'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 1 day 1(line 1, pos 7)
+
+== SQL ==
+select interval '1 day 1'
+-------^^^
+
+
+-- !query
+select interval '1 day 2' day
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Can only use numbers in the interval value part for multiple unit value pairs interval form, but got invalid value: 1 day 2(line 1, pos 16)
+
+== SQL ==
+select interval '1 day 2' day
+----------------^^^
+
+
+-- !query
+select interval 'interval 1' day
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Can only use numbers in the interval value part for multiple unit value pairs interval form, but got invalid value: interval 1(line 1, pos 16)
+
+== SQL ==
+select interval 'interval 1' day
+----------------^^^
+
+
+-- !query
+select interval '-\t 1' day
+-- !query schema
+struct<INTERVAL '-1 days':interval>
+-- !query output
+-1 days
+
+
+-- !query
+SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / 2
+-- !query schema
+struct<(INTERVAL '-178956970-8' YEAR TO MONTH / 2):interval year to month>
+-- !query output
+-89478485-4
+
+
+-- !query
+SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / 5
+-- !query schema
+struct<(INTERVAL '-178956970-8' YEAR TO MONTH / 5):interval year to month>
+-- !query output
+-35791394-2
+
+
+-- !query
+SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+Overflow in integral divide.
+
+
+-- !query
+SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1L
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+Overflow in integral divide.
+
+
+-- !query
+SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1.0
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+Overflow
+
+
+-- !query
+SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1.0D
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+not in range
+
+
+-- !query
+SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / 2
+-- !query schema
+struct<(INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND / 2):interval day to second>
+-- !query output
+-53375995 14:00:27.387904000
+
+
+-- !query
+SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / 5
+-- !query schema
+struct<(INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND / 5):interval day to second>
+-- !query output
+-21350398 05:36:10.955162000
+
+
+-- !query
+SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+Overflow in integral divide.
+
+
+-- !query
+SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1L
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+Overflow in integral divide.
+
+
+-- !query
+SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1.0
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+Overflow
+
+
+-- !query
+SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1.0D
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+not in range
diff --git a/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out
index e85237f1c9f6d..a344f618226e4 100644
--- a/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 67
+-- Number of queries: 71
 
 
 -- !query
@@ -72,7 +72,7 @@ select to_json(named_struct('a', 1, 'b', 2), named_struct('mode', 'PERMISSIVE'))
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Must use a map() function for options;; line 1 pos 7
+Must use a map() function for options; line 1 pos 7
 
 
 -- !query
@@ -81,7 +81,7 @@ select to_json(named_struct('a', 1, 'b', 2), map('mode', 1))
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-A type of keys and values in map() must be string, but got map<string,int>;; line 1 pos 7
+A type of keys and values in map() must be string, but got map<string,int>; line 1 pos 7
 
 
 -- !query
@@ -115,7 +115,7 @@ select from_json('{"a":1}', 1)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-The expression '1' is not a valid schema string.;; line 1 pos 7
+The expression '1' is not a valid schema string.; line 1 pos 7
 
 
 -- !query
@@ -124,7 +124,14 @@ select from_json('{"a":1}', 'a InvalidType')
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
+Cannot parse the data type: 
+extraneous input 'InvalidType' expecting <EOF>(line 1, pos 2)
 
+== SQL ==
+a InvalidType
+--^^^
+
+Failed fallback parsing: 
 DataType invalidtype is not supported.(line 1, pos 2)
 
 == SQL ==
@@ -139,7 +146,7 @@ select from_json('{"a":1}', 'a INT', named_struct('mode', 'PERMISSIVE'))
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Must use a map() function for options;; line 1 pos 7
+Must use a map() function for options; line 1 pos 7
 
 
 -- !query
@@ -148,7 +155,7 @@ select from_json('{"a":1}', 'a INT', map('mode', 1))
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-A type of keys and values in map() must be string, but got map<string,int>;; line 1 pos 7
+A type of keys and values in map() must be string, but got map<string,int>; line 1 pos 7
 
 
 -- !query
@@ -213,7 +220,7 @@ select schema_of_json('{"c1":0, "c2":[1]}')
 -- !query schema
 struct<schema_of_json({"c1":0, "c2":[1]}):string>
 -- !query output
-struct<c1:bigint,c2:array<bigint>>
+STRUCT<`c1`: BIGINT, `c2`: ARRAY<BIGINT>>
 
 
 -- !query
@@ -288,6 +295,49 @@ struct<from_json([{"a": 1}, 2]):array<map<string,int>>>
 NULL
 
 
+-- !query
+select from_json('{"d": "2012-12-15", "t": "2012-12-15 15:15:15"}', 'd date, t timestamp')
+-- !query schema
+struct<from_json({"d": "2012-12-15", "t": "2012-12-15 15:15:15"}):struct<d:date,t:timestamp>>
+-- !query output
+{"d":2012-12-15,"t":2012-12-15 15:15:15}
+
+
+-- !query
+select from_json(
+  '{"d": "12/15 2012", "t": "12/15 2012 15:15:15"}',
+  'd date, t timestamp',
+  map('dateFormat', 'MM/dd yyyy', 'timestampFormat', 'MM/dd yyyy HH:mm:ss'))
+-- !query schema
+struct<from_json({"d": "12/15 2012", "t": "12/15 2012 15:15:15"}):struct<d:date,t:timestamp>>
+-- !query output
+{"d":2012-12-15,"t":2012-12-15 15:15:15}
+
+
+-- !query
+select from_json(
+  '{"d": "02-29"}',
+  'd date',
+  map('dateFormat', 'MM-dd'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '02-29' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
+-- !query
+select from_json(
+  '{"t": "02-29"}',
+  't timestamp',
+  map('timestampFormat', 'MM-dd'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '02-29' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
 -- !query
 select to_json(array('1', '2', '3'))
 -- !query schema
@@ -309,7 +359,7 @@ select schema_of_json('{"c1":1}', map('primitivesAsString', 'true'))
 -- !query schema
 struct<schema_of_json({"c1":1}):string>
 -- !query output
-struct<c1:string>
+STRUCT<`c1`: STRING>
 
 
 -- !query
@@ -317,7 +367,7 @@ select schema_of_json('{"c1":01, "c2":0.1}', map('allowNumericLeadingZeros', 'tr
 -- !query schema
 struct<schema_of_json({"c1":01, "c2":0.1}):string>
 -- !query output
-struct<c1:bigint,c2:decimal(1,1)>
+STRUCT<`c1`: BIGINT, `c2`: DECIMAL(1,1)>
 
 
 -- !query
@@ -343,13 +393,13 @@ SELECT schema_of_json(jsonField) FROM jsonTable
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'schema_of_json(jsontable.`jsonField`)' due to data type mismatch: The input json should be a foldable string expression and not null; however, got jsontable.`jsonField`.; line 1 pos 7
+cannot resolve 'schema_of_json(jsontable.jsonField)' due to data type mismatch: The input json should be a foldable string expression and not null; however, got jsontable.jsonField.; line 1 pos 7
 
 
 -- !query
 select json_array_length(null)
 -- !query schema
-struct<json_array_length(CAST(NULL AS STRING)):int>
+struct<json_array_length(NULL):int>
 -- !query output
 NULL
 
@@ -448,7 +498,7 @@ Invalid number of arguments for function json_object_keys. Expected: 1; Found: 0
 -- !query
 select json_object_keys(null)
 -- !query schema
-struct<json_object_keys(CAST(NULL AS STRING)):array<string>>
+struct<json_object_keys(NULL):array<string>>
 -- !query output
 NULL
 
diff --git a/sql/core/src/test/resources/sql-tests/results/limit.sql.out b/sql/core/src/test/resources/sql-tests/results/limit.sql.out
index 074e7a6d28c47..3b10bbd57401d 100644
--- a/sql/core/src/test/resources/sql-tests/results/limit.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/limit.sql.out
@@ -53,7 +53,7 @@ SELECT * FROM testdata LIMIT -1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-The limit expression must be equal to or greater than 0, but got -1;
+The limit expression must be equal to or greater than 0, but got -1
 
 
 -- !query
@@ -62,7 +62,7 @@ SELECT * FROM testData TABLESAMPLE (-1 ROWS)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-The limit expression must be equal to or greater than 0, but got -1;
+The limit expression must be equal to or greater than 0, but got -1
 
 
 -- !query
@@ -79,7 +79,7 @@ SELECT * FROM testdata LIMIT CAST(NULL AS INT)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-The evaluated limit expression must not be null, but got CAST(NULL AS INT);
+The evaluated limit expression must not be null, but got CAST(NULL AS INT)
 
 
 -- !query
@@ -88,7 +88,7 @@ SELECT * FROM testdata LIMIT key > 3
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-The limit expression must evaluate to a constant value, but got (spark_catalog.default.testdata.`key` > 3);
+The limit expression must evaluate to a constant value, but got (spark_catalog.default.testdata.key > 3)
 
 
 -- !query
@@ -97,7 +97,7 @@ SELECT * FROM testdata LIMIT true
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-The limit expression must be integer type, but got boolean;
+The limit expression must be integer type, but got boolean
 
 
 -- !query
@@ -106,7 +106,7 @@ SELECT * FROM testdata LIMIT 'a'
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-The limit expression must be integer type, but got string;
+The limit expression must be integer type, but got string
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/literals.sql.out b/sql/core/src/test/resources/sql-tests/results/literals.sql.out
index f6720f6c5faa4..9748d8bc56d72 100644
--- a/sql/core/src/test/resources/sql-tests/results/literals.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/literals.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 50
+-- Number of queries: 54
 
 
 -- !query
@@ -164,6 +164,36 @@ decimal can only support precision up to 38
 select 1234567890123456789012345678901234567890.0
 
 
+-- !query
+select 1F, 1.2F, .10f, 0.10f
+-- !query schema
+struct<1.0:float,1.2:float,0.1:float,0.1:float>
+-- !query output
+1.0	1.2	0.1	0.1
+
+
+-- !query
+select -1F, -1.2F, -.10F, -0.10F
+-- !query schema
+struct<-1.0:float,-1.2:float,-0.1:float,-0.1:float>
+-- !query output
+-1.0	-1.2	-0.1	-0.1
+
+
+-- !query
+select -3.4028235E39f
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Numeric literal -3.4028235E39 does not fit in range [-3.4028234663852886E+38, 3.4028234663852886E+38] for type float(line 1, pos 7)
+
+== SQL ==
+select -3.4028235E39f
+-------^^^
+
+
 -- !query
 select 1D, 1.2D, 1e10, 1.5e5, .10D, 0.10D, .1e5, .9e+2, 0.9e+2, 900e-1, 9.e+1
 -- !query schema
@@ -216,6 +246,14 @@ struct<0.3:decimal(1,1),-0.8:decimal(1,1),0.5:decimal(1,1),-0.18:decimal(2,2),0.
 0.3	-0.8	0.5	-0.18	0.1111	0.1111
 
 
+-- !query
+select 0.3 F, 0.4 D, 0.5 BD
+-- !query schema
+struct<F:decimal(1,1),D:decimal(1,1),BD:decimal(1,1)>
+-- !query output
+0.3	0.4	0.5
+
+
 -- !query
 select 123456789012345678901234567890123456789e10d, 123456789012345678901234567890123456789.1e10d
 -- !query schema
@@ -398,7 +436,7 @@ select +date '1999-01-01'
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(+ DATE '1999-01-01')' due to data type mismatch: argument 1 requires (numeric or interval) type, however, 'DATE '1999-01-01'' is of date type.; line 1 pos 7
+cannot resolve '(+ DATE '1999-01-01')' due to data type mismatch: argument 1 requires (numeric or interval or interval day to second or interval year to month) type, however, 'DATE '1999-01-01'' is of date type.; line 1 pos 7
 
 
 -- !query
@@ -407,7 +445,7 @@ select +timestamp '1999-01-01'
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(+ TIMESTAMP '1999-01-01 00:00:00')' due to data type mismatch: argument 1 requires (numeric or interval) type, however, 'TIMESTAMP '1999-01-01 00:00:00'' is of timestamp type.; line 1 pos 7
+cannot resolve '(+ TIMESTAMP '1999-01-01 00:00:00')' due to data type mismatch: argument 1 requires (numeric or interval or interval day to second or interval year to month) type, however, 'TIMESTAMP '1999-01-01 00:00:00'' is of timestamp type.; line 1 pos 7
 
 
 -- !query
@@ -424,7 +462,7 @@ select +map(1, 2)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(+ map(1, 2))' due to data type mismatch: argument 1 requires (numeric or interval) type, however, 'map(1, 2)' is of map<int,int> type.; line 1 pos 7
+cannot resolve '(+ map(1, 2))' due to data type mismatch: argument 1 requires (numeric or interval or interval day to second or interval year to month) type, however, 'map(1, 2)' is of map<int,int> type.; line 1 pos 7
 
 
 -- !query
@@ -433,7 +471,7 @@ select +array(1,2)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(+ array(1, 2))' due to data type mismatch: argument 1 requires (numeric or interval) type, however, 'array(1, 2)' is of array<int> type.; line 1 pos 7
+cannot resolve '(+ array(1, 2))' due to data type mismatch: argument 1 requires (numeric or interval or interval day to second or interval year to month) type, however, 'array(1, 2)' is of array<int> type.; line 1 pos 7
 
 
 -- !query
@@ -442,7 +480,7 @@ select +named_struct('a', 1, 'b', 'spark')
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(+ named_struct('a', 1, 'b', 'spark'))' due to data type mismatch: argument 1 requires (numeric or interval) type, however, 'named_struct('a', 1, 'b', 'spark')' is of struct<a:int,b:string> type.; line 1 pos 7
+cannot resolve '(+ named_struct('a', 1, 'b', 'spark'))' due to data type mismatch: argument 1 requires (numeric or interval or interval day to second or interval year to month) type, however, 'named_struct('a', 1, 'b', 'spark')' is of struct<a:int,b:string> type.; line 1 pos 7
 
 
 -- !query
@@ -451,7 +489,7 @@ select +X'1'
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(+ X'01')' due to data type mismatch: argument 1 requires (numeric or interval) type, however, 'X'01'' is of binary type.; line 1 pos 7
+cannot resolve '(+ X'01')' due to data type mismatch: argument 1 requires (numeric or interval or interval day to second or interval year to month) type, however, 'X'01'' is of binary type.; line 1 pos 7
 
 
 -- !query
@@ -460,7 +498,7 @@ select -date '1999-01-01'
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(- DATE '1999-01-01')' due to data type mismatch: argument 1 requires (numeric or interval) type, however, 'DATE '1999-01-01'' is of date type.; line 1 pos 7
+cannot resolve '(- DATE '1999-01-01')' due to data type mismatch: argument 1 requires (numeric or interval or interval day to second or interval year to month) type, however, 'DATE '1999-01-01'' is of date type.; line 1 pos 7
 
 
 -- !query
@@ -469,7 +507,7 @@ select -timestamp '1999-01-01'
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(- TIMESTAMP '1999-01-01 00:00:00')' due to data type mismatch: argument 1 requires (numeric or interval) type, however, 'TIMESTAMP '1999-01-01 00:00:00'' is of timestamp type.; line 1 pos 7
+cannot resolve '(- TIMESTAMP '1999-01-01 00:00:00')' due to data type mismatch: argument 1 requires (numeric or interval or interval day to second or interval year to month) type, however, 'TIMESTAMP '1999-01-01 00:00:00'' is of timestamp type.; line 1 pos 7
 
 
 -- !query
@@ -478,4 +516,4 @@ select -x'2379ACFe'
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(- X'2379ACFE')' due to data type mismatch: argument 1 requires (numeric or interval) type, however, 'X'2379ACFE'' is of binary type.; line 1 pos 7
+cannot resolve '(- X'2379ACFE')' due to data type mismatch: argument 1 requires (numeric or interval or interval day to second or interval year to month) type, however, 'X'2379ACFE'' is of binary type.; line 1 pos 7
diff --git a/sql/core/src/test/resources/sql-tests/results/map.sql.out b/sql/core/src/test/resources/sql-tests/results/map.sql.out
new file mode 100644
index 0000000000000..7a0c0d776ca2b
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/map.sql.out
@@ -0,0 +1,18 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 2
+
+
+-- !query
+select element_at(map(1, 'a', 2, 'b'), 5)
+-- !query schema
+struct<element_at(map(1, a, 2, b), 5):string>
+-- !query output
+NULL
+
+
+-- !query
+select map(1, 'a', 2, 'b')[5]
+-- !query schema
+struct<map(1, a, 2, b)[5]:string>
+-- !query output
+NULL
diff --git a/sql/core/src/test/resources/sql-tests/results/misc-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/misc-functions.sql.out
index bd8ffb82ee129..d927e890b0336 100644
--- a/sql/core/src/test/resources/sql-tests/results/misc-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/misc-functions.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 7
+-- Number of queries: 16
 
 
 -- !query
@@ -56,3 +56,82 @@ select typeof(array(1, 2)), typeof(map(1, 2)), typeof(named_struct('a', 1, 'b',
 struct<typeof(array(1, 2)):string,typeof(map(1, 2)):string,typeof(named_struct(a, 1, b, spark)):string>
 -- !query output
 array<int>	map<int,int>	struct<a:int,b:string>
+
+
+-- !query
+SELECT assert_true(true), assert_true(boolean(1))
+-- !query schema
+struct<assert_true(true, 'true' is not true!):null,assert_true(1, 'cast(1 as boolean)' is not true!):null>
+-- !query output
+NULL	NULL
+
+
+-- !query
+SELECT assert_true(false)
+-- !query schema
+struct<>
+-- !query output
+java.lang.RuntimeException
+'false' is not true!
+
+
+-- !query
+SELECT assert_true(boolean(0))
+-- !query schema
+struct<>
+-- !query output
+java.lang.RuntimeException
+'cast(0 as boolean)' is not true!
+
+
+-- !query
+SELECT assert_true(null)
+-- !query schema
+struct<>
+-- !query output
+java.lang.RuntimeException
+'null' is not true!
+
+
+-- !query
+SELECT assert_true(boolean(null))
+-- !query schema
+struct<>
+-- !query output
+java.lang.RuntimeException
+'cast(null as boolean)' is not true!
+
+
+-- !query
+SELECT assert_true(false, 'custom error message')
+-- !query schema
+struct<>
+-- !query output
+java.lang.RuntimeException
+custom error message
+
+
+-- !query
+CREATE TEMPORARY VIEW tbl_misc AS SELECT * FROM (VALUES (1), (8), (2)) AS T(v)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SELECT raise_error('error message')
+-- !query schema
+struct<>
+-- !query output
+java.lang.RuntimeException
+error message
+
+
+-- !query
+SELECT if(v > 5, raise_error('too big: ' || v), v + 1) FROM tbl_misc
+-- !query schema
+struct<>
+-- !query output
+java.lang.RuntimeException
+too big: 8
diff --git a/sql/core/src/test/resources/sql-tests/results/natural-join.sql.out b/sql/core/src/test/resources/sql-tests/results/natural-join.sql.out
index 13f319700df3f..794e4725d9ea4 100644
--- a/sql/core/src/test/resources/sql-tests/results/natural-join.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/natural-join.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 6
+-- Number of queries: 29
 
 
 -- !query
@@ -26,6 +26,40 @@ struct<>
 
 
 
+-- !query
+create temporary view nt3 as select * from values
+  ("one", 4),
+  ("two", 5),
+  ("one", 6)
+  as nt3(k, v3)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+create temporary view nt4 as select * from values
+  ("one", 7),
+  ("two", 8),
+  ("one", 9)
+  as nt4(k, v4)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SELECT * FROM nt1 natural join nt2
+-- !query schema
+struct<k:string,v1:int,v2:int>
+-- !query output
+one	1	1
+one	1	5
+two	2	22
+
+
 -- !query
 SELECT * FROM nt1 natural join nt2 where k = "one"
 -- !query schema
@@ -62,3 +96,216 @@ SELECT count(*) FROM nt1 natural full outer join nt2
 struct<count(1):bigint>
 -- !query output
 4
+
+
+-- !query
+SELECT k FROM nt1 natural join nt2
+-- !query schema
+struct<k:string>
+-- !query output
+one
+one
+two
+
+
+-- !query
+SELECT k FROM nt1 natural join nt2 where k = "one"
+-- !query schema
+struct<k:string>
+-- !query output
+one
+one
+
+
+-- !query
+SELECT nt1.* FROM nt1 natural join nt2
+-- !query schema
+struct<k:string,v1:int>
+-- !query output
+one	1
+one	1
+two	2
+
+
+-- !query
+SELECT nt2.* FROM nt1 natural join nt2
+-- !query schema
+struct<k:string,v2:int>
+-- !query output
+one	1
+one	5
+two	22
+
+
+-- !query
+SELECT sbq.* from (SELECT * FROM nt1 natural join nt2) sbq
+-- !query schema
+struct<k:string,v1:int,v2:int>
+-- !query output
+one	1	1
+one	1	5
+two	2	22
+
+
+-- !query
+SELECT sbq.k from (SELECT * FROM nt1 natural join nt2) sbq
+-- !query schema
+struct<k:string>
+-- !query output
+one
+one
+two
+
+
+-- !query
+SELECT nt1.*, nt2.* FROM nt1 natural join nt2
+-- !query schema
+struct<k:string,v1:int,k:string,v2:int>
+-- !query output
+one	1	one	1
+one	1	one	5
+two	2	two	22
+
+
+-- !query
+SELECT *, nt2.k FROM nt1 natural join nt2
+-- !query schema
+struct<k:string,v1:int,v2:int,k:string>
+-- !query output
+one	1	1	one
+one	1	5	one
+two	2	22	two
+
+
+-- !query
+SELECT nt1.k, nt2.k FROM nt1 natural join nt2
+-- !query schema
+struct<k:string,k:string>
+-- !query output
+one	one
+one	one
+two	two
+
+
+-- !query
+SELECT nt1.k, nt2.k FROM nt1 natural join nt2 where k = "one"
+-- !query schema
+struct<k:string,k:string>
+-- !query output
+one	one
+one	one
+
+
+-- !query
+SELECT * FROM (SELECT * FROM nt1 natural join nt2)
+-- !query schema
+struct<k:string,v1:int,v2:int>
+-- !query output
+one	1	1
+one	1	5
+two	2	22
+
+
+-- !query
+SELECT * FROM (SELECT nt1.*, nt2.* FROM nt1 natural join nt2)
+-- !query schema
+struct<k:string,v1:int,k:string,v2:int>
+-- !query output
+one	1	one	1
+one	1	one	5
+two	2	two	22
+
+
+-- !query
+SELECT * FROM (SELECT nt1.v1, nt2.k FROM nt1 natural join nt2)
+-- !query schema
+struct<v1:int,k:string>
+-- !query output
+1	one
+1	one
+2	two
+
+
+-- !query
+SELECT nt2.k FROM (SELECT * FROM nt1 natural join nt2)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'nt2.k' given input columns: [__auto_generated_subquery_name.k, __auto_generated_subquery_name.v1, __auto_generated_subquery_name.v2]; line 1 pos 7
+
+
+-- !query
+SELECT * FROM nt1 natural join nt2 natural join nt3
+-- !query schema
+struct<k:string,v1:int,v2:int,v3:int>
+-- !query output
+one	1	1	4
+one	1	1	6
+one	1	5	4
+one	1	5	6
+two	2	22	5
+
+
+-- !query
+SELECT nt1.*, nt2.*, nt3.* FROM nt1 natural join nt2 natural join nt3
+-- !query schema
+struct<k:string,v1:int,k:string,v2:int,k:string,v3:int>
+-- !query output
+one	1	one	1	one	4
+one	1	one	1	one	6
+one	1	one	5	one	4
+one	1	one	5	one	6
+two	2	two	22	two	5
+
+
+-- !query
+SELECT nt1.*, nt2.*, nt3.* FROM nt1 natural join nt2 join nt3 on nt2.k = nt3.k
+-- !query schema
+struct<k:string,v1:int,k:string,v2:int,k:string,v3:int>
+-- !query output
+one	1	one	1	one	4
+one	1	one	1	one	6
+one	1	one	5	one	4
+one	1	one	5	one	6
+two	2	two	22	two	5
+
+
+-- !query
+SELECT * FROM nt1 natural join nt2 join nt3 on nt1.k = nt3.k
+-- !query schema
+struct<k:string,v1:int,v2:int,k:string,v3:int>
+-- !query output
+one	1	1	one	4
+one	1	1	one	6
+one	1	5	one	4
+one	1	5	one	6
+two	2	22	two	5
+
+
+-- !query
+SELECT * FROM nt1 natural join nt2 join nt3 on nt2.k = nt3.k
+-- !query schema
+struct<k:string,v1:int,v2:int,k:string,v3:int>
+-- !query output
+one	1	1	one	4
+one	1	1	one	6
+one	1	5	one	4
+one	1	5	one	6
+two	2	22	two	5
+
+
+-- !query
+SELECT nt1.*, nt2.*, nt3.*, nt4.* FROM nt1 natural join nt2 natural join nt3 natural join nt4
+-- !query schema
+struct<k:string,v1:int,k:string,v2:int,k:string,v3:int,k:string,v4:int>
+-- !query output
+one	1	one	1	one	4	one	7
+one	1	one	1	one	4	one	9
+one	1	one	1	one	6	one	7
+one	1	one	1	one	6	one	9
+one	1	one	5	one	4	one	7
+one	1	one	5	one	4	one	9
+one	1	one	5	one	6	one	7
+one	1	one	5	one	6	one	9
+two	2	two	22	two	5	two	8
diff --git a/sql/core/src/test/resources/sql-tests/results/null-propagation.sql.out b/sql/core/src/test/resources/sql-tests/results/null-propagation.sql.out
index 76a41f9170388..b972d963c8dbc 100644
--- a/sql/core/src/test/resources/sql-tests/results/null-propagation.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/null-propagation.sql.out
@@ -13,7 +13,7 @@ struct<count(NULL):bigint>
 -- !query
 SELECT COUNT(1 + NULL) FROM VALUES 1, 2, 3
 -- !query schema
-struct<count((1 + CAST(NULL AS INT))):bigint>
+struct<count((1 + NULL)):bigint>
 -- !query output
 0
 
@@ -31,7 +31,7 @@ struct<count(NULL) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWIN
 -- !query
 SELECT COUNT(1 + NULL) OVER () FROM VALUES 1, 2, 3
 -- !query schema
-struct<count((1 + CAST(NULL AS INT))) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):bigint>
+struct<count((1 + NULL)) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):bigint>
 -- !query output
 0
 0
diff --git a/sql/core/src/test/resources/sql-tests/results/operators.sql.out b/sql/core/src/test/resources/sql-tests/results/operators.sql.out
index a94a123b1b8ab..3af92bf935713 100644
--- a/sql/core/src/test/resources/sql-tests/results/operators.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/operators.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 57
+-- Number of queries: 69
 
 
 -- !query
@@ -125,7 +125,7 @@ struct<(+ (- key)):int>
 -- !query
 select 5 / 2
 -- !query schema
-struct<(CAST(5 AS DOUBLE) / CAST(2 AS DOUBLE)):double>
+struct<(5 / 2):double>
 -- !query output
 2.5
 
@@ -133,7 +133,7 @@ struct<(CAST(5 AS DOUBLE) / CAST(2 AS DOUBLE)):double>
 -- !query
 select 5 / 0
 -- !query schema
-struct<(CAST(5 AS DOUBLE) / CAST(0 AS DOUBLE)):double>
+struct<(5 / 0):double>
 -- !query output
 NULL
 
@@ -141,7 +141,7 @@ NULL
 -- !query
 select 5 / null
 -- !query schema
-struct<(CAST(5 AS DOUBLE) / CAST(NULL AS DOUBLE)):double>
+struct<(5 / NULL):double>
 -- !query output
 NULL
 
@@ -149,7 +149,7 @@ NULL
 -- !query
 select null / 5
 -- !query schema
-struct<(CAST(NULL AS DOUBLE) / CAST(5 AS DOUBLE)):double>
+struct<(NULL / 5):double>
 -- !query output
 NULL
 
@@ -173,7 +173,7 @@ NULL
 -- !query
 select 5 div null
 -- !query schema
-struct<(5 div CAST(NULL AS INT)):bigint>
+struct<(5 div NULL):bigint>
 -- !query output
 NULL
 
@@ -181,7 +181,7 @@ NULL
 -- !query
 select null div 5
 -- !query schema
-struct<(CAST(NULL AS INT) div 5):bigint>
+struct<(NULL div 5):bigint>
 -- !query output
 NULL
 
@@ -189,7 +189,7 @@ NULL
 -- !query
 select cast(51 as decimal(10, 0)) div cast(2 as decimal(2, 0))
 -- !query schema
-struct<(CAST(CAST(51 AS DECIMAL(10,0)) AS DECIMAL(10,0)) div CAST(CAST(2 AS DECIMAL(2,0)) AS DECIMAL(10,0))):bigint>
+struct<(CAST(51 AS DECIMAL(10,0)) div CAST(2 AS DECIMAL(2,0))):bigint>
 -- !query output
 25
 
@@ -197,7 +197,7 @@ struct<(CAST(CAST(51 AS DECIMAL(10,0)) AS DECIMAL(10,0)) div CAST(CAST(2 AS DECI
 -- !query
 select cast(5 as decimal(1, 0)) div cast(0 as decimal(2, 0))
 -- !query schema
-struct<(CAST(CAST(5 AS DECIMAL(1,0)) AS DECIMAL(2,0)) div CAST(CAST(0 AS DECIMAL(2,0)) AS DECIMAL(2,0))):bigint>
+struct<(CAST(5 AS DECIMAL(1,0)) div CAST(0 AS DECIMAL(2,0))):bigint>
 -- !query output
 NULL
 
@@ -205,7 +205,7 @@ NULL
 -- !query
 select cast(5 as decimal(1, 0)) div cast(null as decimal(2, 0))
 -- !query schema
-struct<(CAST(CAST(5 AS DECIMAL(1,0)) AS DECIMAL(2,0)) div CAST(CAST(NULL AS DECIMAL(2,0)) AS DECIMAL(2,0))):bigint>
+struct<(CAST(5 AS DECIMAL(1,0)) div CAST(NULL AS DECIMAL(2,0))):bigint>
 -- !query output
 NULL
 
@@ -213,7 +213,7 @@ NULL
 -- !query
 select cast(null as decimal(1, 0)) div cast(5 as decimal(2, 0))
 -- !query schema
-struct<(CAST(CAST(NULL AS DECIMAL(1,0)) AS DECIMAL(2,0)) div CAST(CAST(5 AS DECIMAL(2,0)) AS DECIMAL(2,0))):bigint>
+struct<(CAST(NULL AS DECIMAL(1,0)) div CAST(5 AS DECIMAL(2,0))):bigint>
 -- !query output
 NULL
 
@@ -261,7 +261,7 @@ struct<pmod(-7, 3):int>
 -- !query
 select cot(1)
 -- !query schema
-struct<COT(CAST(1 AS DOUBLE)):double>
+struct<COT(1):double>
 -- !query output
 0.6420926159343306
 
@@ -269,7 +269,7 @@ struct<COT(CAST(1 AS DOUBLE)):double>
 -- !query
 select cot(null)
 -- !query schema
-struct<COT(CAST(NULL AS DOUBLE)):double>
+struct<COT(NULL):double>
 -- !query output
 NULL
 
@@ -277,7 +277,7 @@ NULL
 -- !query
 select cot(0)
 -- !query schema
-struct<COT(CAST(0 AS DOUBLE)):double>
+struct<COT(0):double>
 -- !query output
 Infinity
 
@@ -285,7 +285,7 @@ Infinity
 -- !query
 select cot(-1)
 -- !query schema
-struct<COT(CAST(-1 AS DOUBLE)):double>
+struct<COT(-1):double>
 -- !query output
 -0.6420926159343306
 
@@ -293,7 +293,7 @@ struct<COT(CAST(-1 AS DOUBLE)):double>
 -- !query
 select ceiling(0)
 -- !query schema
-struct<ceiling(CAST(0 AS DOUBLE)):bigint>
+struct<ceiling(0):bigint>
 -- !query output
 0
 
@@ -301,7 +301,7 @@ struct<ceiling(CAST(0 AS DOUBLE)):bigint>
 -- !query
 select ceiling(1)
 -- !query schema
-struct<ceiling(CAST(1 AS DOUBLE)):bigint>
+struct<ceiling(1):bigint>
 -- !query output
 1
 
@@ -341,7 +341,7 @@ struct<ceiling(-0.10):decimal(1,0)>
 -- !query
 select floor(0)
 -- !query schema
-struct<FLOOR(CAST(0 AS DOUBLE)):bigint>
+struct<FLOOR(0):bigint>
 -- !query output
 0
 
@@ -349,7 +349,7 @@ struct<FLOOR(CAST(0 AS DOUBLE)):bigint>
 -- !query
 select floor(1)
 -- !query schema
-struct<FLOOR(CAST(1 AS DOUBLE)):bigint>
+struct<FLOOR(1):bigint>
 -- !query output
 1
 
@@ -381,7 +381,7 @@ struct<FLOOR(-0.10):decimal(1,0)>
 -- !query
 select 1 > 0.00001
 -- !query schema
-struct<(CAST(1 AS BIGINT) > 0):boolean>
+struct<(1 > 0):boolean>
 -- !query output
 true
 
@@ -389,7 +389,7 @@ true
 -- !query
 select mod(7, 2), mod(7, 0), mod(0, 2), mod(7, null), mod(null, 2), mod(null, null)
 -- !query schema
-struct<mod(7, 2):int,mod(7, 0):int,mod(0, 2):int,mod(7, CAST(NULL AS INT)):int,mod(CAST(NULL AS INT), 2):int,mod(CAST(NULL AS DOUBLE), CAST(NULL AS DOUBLE)):double>
+struct<mod(7, 2):int,mod(7, 0):int,mod(0, 2):int,mod(7, NULL):int,mod(NULL, 2):int,mod(NULL, NULL):double>
 -- !query output
 1	NULL	0	NULL	NULL	NULL
 
@@ -429,7 +429,7 @@ struct<octet_length(abc):int>
 -- !query
 select abs(-3.13), abs('-2.19')
 -- !query schema
-struct<abs(-3.13):decimal(3,2),abs(CAST(-2.19 AS DOUBLE)):double>
+struct<abs(-3.13):decimal(3,2),abs(-2.19):double>
 -- !query output
 3.13	2.19
 
@@ -437,7 +437,7 @@ struct<abs(-3.13):decimal(3,2),abs(CAST(-2.19 AS DOUBLE)):double>
 -- !query
 select positive('-1.11'), positive(-1.11), negative('-1.11'), negative(-1.11)
 -- !query schema
-struct<(+ CAST(-1.11 AS DOUBLE)):double,(+ -1.11):decimal(3,2),negative(CAST(-1.11 AS DOUBLE)):double,negative(-1.11):decimal(3,2)>
+struct<(+ -1.11):double,(+ -1.11):decimal(3,2),negative(-1.11):double,negative(-1.11):decimal(3,2)>
 -- !query output
 -1.11	-1.11	1.11	1.11
 
@@ -445,7 +445,7 @@ struct<(+ CAST(-1.11 AS DOUBLE)):double,(+ -1.11):decimal(3,2),negative(CAST(-1.
 -- !query
 select pmod(-7, 2), pmod(0, 2), pmod(7, 0), pmod(7, null), pmod(null, 2), pmod(null, null)
 -- !query schema
-struct<pmod(-7, 2):int,pmod(0, 2):int,pmod(7, 0):int,pmod(7, CAST(NULL AS INT)):int,pmod(CAST(NULL AS INT), 2):int,pmod(CAST(NULL AS DOUBLE), CAST(NULL AS DOUBLE)):double>
+struct<pmod(-7, 2):int,pmod(0, 2):int,pmod(7, 0):int,pmod(7, NULL):int,pmod(NULL, 2):int,pmod(NULL, NULL):double>
 -- !query output
 1	0	NULL	NULL	NULL	NULL
 
@@ -456,3 +456,99 @@ select pmod(cast(3.13 as decimal), cast(0 as decimal)), pmod(cast(2 as smallint)
 struct<pmod(CAST(3.13 AS DECIMAL(10,0)), CAST(0 AS DECIMAL(10,0))):decimal(10,0),pmod(CAST(2 AS SMALLINT), CAST(0 AS SMALLINT)):smallint>
 -- !query output
 NULL	NULL
+
+
+-- !query
+select width_bucket(5.35, 0.024, 10.06, 5)
+-- !query schema
+struct<width_bucket(5.35, 0.024, 10.06, 5):bigint>
+-- !query output
+3
+
+
+-- !query
+select width_bucket(5.35, 0.024, 10.06, 3 + 2)
+-- !query schema
+struct<width_bucket(5.35, 0.024, 10.06, (3 + 2)):bigint>
+-- !query output
+3
+
+
+-- !query
+select width_bucket('5.35', '0.024', '10.06', '5')
+-- !query schema
+struct<width_bucket(5.35, 0.024, 10.06, 5):bigint>
+-- !query output
+3
+
+
+-- !query
+select width_bucket(5.35, 0.024, 10.06, 2.5)
+-- !query schema
+struct<width_bucket(5.35, 0.024, 10.06, 2.5):bigint>
+-- !query output
+2
+
+
+-- !query
+select width_bucket(5.35, 0.024, 10.06, 0.5)
+-- !query schema
+struct<width_bucket(5.35, 0.024, 10.06, 0.5):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select width_bucket(null, 0.024, 10.06, 5)
+-- !query schema
+struct<width_bucket(NULL, 0.024, 10.06, 5):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select width_bucket(5.35, null, 10.06, 5)
+-- !query schema
+struct<width_bucket(5.35, NULL, 10.06, 5):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select width_bucket(5.35, 0.024, null, -5)
+-- !query schema
+struct<width_bucket(5.35, 0.024, NULL, -5):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select width_bucket(5.35, 0.024, 10.06, null)
+-- !query schema
+struct<width_bucket(5.35, 0.024, 10.06, NULL):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select width_bucket(5.35, 0.024, 10.06, -5)
+-- !query schema
+struct<width_bucket(5.35, 0.024, 10.06, -5):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select width_bucket(5.35, 0.024, 10.06, 9223372036854775807L)
+-- !query schema
+struct<width_bucket(5.35, 0.024, 10.06, 9223372036854775807):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select width_bucket(5.35, 0.024, 10.06, 9223372036854775807L - 1)
+-- !query schema
+struct<width_bucket(5.35, 0.024, 10.06, (9223372036854775807 - 1)):bigint>
+-- !query output
+4894746858139549697
diff --git a/sql/core/src/test/resources/sql-tests/results/parse-schema-string.sql.out b/sql/core/src/test/resources/sql-tests/results/parse-schema-string.sql.out
new file mode 100644
index 0000000000000..4440dd763bd2b
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/parse-schema-string.sql.out
@@ -0,0 +1,34 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 4
+
+
+-- !query
+select from_csv('1', 'create INT')
+-- !query schema
+struct<from_csv(1):struct<create:int>>
+-- !query output
+{"create":1}
+
+
+-- !query
+select from_csv('1', 'cube INT')
+-- !query schema
+struct<from_csv(1):struct<cube:int>>
+-- !query output
+{"cube":1}
+
+
+-- !query
+select from_json('{"create":1}', 'create INT')
+-- !query schema
+struct<from_json({"create":1}):struct<create:int>>
+-- !query output
+{"create":1}
+
+
+-- !query
+select from_json('{"cube":1}', 'cube INT')
+-- !query schema
+struct<from_json({"cube":1}):struct<cube:int>>
+-- !query output
+{"cube":1}
diff --git a/sql/core/src/test/resources/sql-tests/results/pivot.sql.out b/sql/core/src/test/resources/sql-tests/results/pivot.sql.out
index ac4e71e244bc0..69679f8be5fe4 100644
--- a/sql/core/src/test/resources/sql-tests/results/pivot.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/pivot.sql.out
@@ -75,7 +75,7 @@ PIVOT (
   FOR course IN ('dotNET', 'Java')
 )
 -- !query schema
-struct<year:int,dotNET_sum(CAST(earnings AS BIGINT)):bigint,dotNET_avg(CAST(earnings AS BIGINT)):double,Java_sum(CAST(earnings AS BIGINT)):bigint,Java_avg(CAST(earnings AS BIGINT)):double>
+struct<year:int,dotNET_sum(earnings):bigint,dotNET_avg(earnings):double,Java_sum(earnings):bigint,Java_avg(earnings):double>
 -- !query output
 2012	15000	7500.0	20000	20000.0
 2013	48000	48000.0	30000	30000.0
@@ -104,7 +104,7 @@ PIVOT (
   FOR course IN ('dotNET', 'Java')
 )
 -- !query schema
-struct<dotNET_sum(CAST(earnings AS BIGINT)):bigint,dotNET_min(year):int,Java_sum(CAST(earnings AS BIGINT)):bigint,Java_min(year):int>
+struct<dotNET_sum(earnings):bigint,dotNET_min(year):int,Java_sum(earnings):bigint,Java_min(year):int>
 -- !query output
 63000	2012	50000	2012
 
@@ -139,7 +139,7 @@ PIVOT (
   FOR course IN ('dotNET', 'Java')
 )
 -- !query schema
-struct<year:int,dotNET_sum(CAST(earnings AS BIGINT)):bigint,dotNET_min(s):int,Java_sum(CAST(earnings AS BIGINT)):bigint,Java_min(s):int>
+struct<year:int,dotNET_sum(earnings):bigint,dotNET_min(s):int,Java_sum(earnings):bigint,Java_min(s):int>
 -- !query output
 2012	15000	1	20000	1
 2013	48000	2	30000	2
@@ -202,7 +202,7 @@ PIVOT (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Aggregate expression required for pivot, but 'coursesales.`earnings`' did not appear in any aggregate function.;
+Aggregate expression required for pivot, but 'coursesales.earnings' did not appear in any aggregate function.
 
 
 -- !query
@@ -217,7 +217,7 @@ PIVOT (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Aggregate expression required for pivot, but '__auto_generated_subquery_name.`year`' did not appear in any aggregate function.;
+Aggregate expression required for pivot, but '__auto_generated_subquery_name.year' did not appear in any aggregate function.
 
 
 -- !query
@@ -232,7 +232,7 @@ PIVOT (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`year`' given input columns: [__auto_generated_subquery_name.course, __auto_generated_subquery_name.earnings]; line 4 pos 0
+cannot resolve 'year' given input columns: [__auto_generated_subquery_name.course, __auto_generated_subquery_name.earnings]; line 4 pos 0
 
 
 -- !query
@@ -244,7 +244,7 @@ PIVOT (
   FOR course IN ('dotNET', 'Java')
 )
 -- !query schema
-struct<year:int,dotNET_CEIL(sum(CAST(earnings AS BIGINT))):bigint,dotNET_a1:double,Java_CEIL(sum(CAST(earnings AS BIGINT))):bigint,Java_a1:double>
+struct<year:int,dotNET_CEIL(sum(earnings)):bigint,dotNET_a1:double,Java_CEIL(sum(earnings)):bigint,Java_a1:double>
 -- !query output
 2012	15000	7501.0	20000	20001.0
 2013	48000	48001.0	30000	30001.0
@@ -262,7 +262,7 @@ PIVOT (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-It is not allowed to use an aggregate function in the argument of another aggregate function. Please use the inner aggregate function in a sub-query.;
+It is not allowed to use an aggregate function in the argument of another aggregate function. Please use the inner aggregate function in a sub-query.
 
 
 -- !query
@@ -276,7 +276,7 @@ PIVOT (
   FOR (course, year) IN (('dotNET', 2012), ('Java', 2013))
 )
 -- !query schema
-struct<s:int,[dotNET, 2012]:bigint,[Java, 2013]:bigint>
+struct<s:int,{dotNET, 2012}:bigint,{Java, 2013}:bigint>
 -- !query output
 1	15000	NULL
 2	NULL	30000
@@ -313,7 +313,7 @@ PIVOT (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Invalid pivot value 'dotNET': value data type string does not match pivot column data type struct<course:string,year:int>;
+Invalid pivot value 'dotNET': value data type string does not match pivot column data type struct<course:string,year:int>
 
 
 -- !query
@@ -326,7 +326,7 @@ PIVOT (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`s`' given input columns: [coursesales.course, coursesales.earnings, coursesales.year]; line 4 pos 15
+cannot resolve 's' given input columns: [coursesales.course, coursesales.earnings, coursesales.year]; line 4 pos 15
 
 
 -- !query
@@ -339,7 +339,7 @@ PIVOT (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Literal expressions required for pivot values, found 'course#x';
+Literal expressions required for pivot values, found 'course#x'
 
 
 -- !query
@@ -370,7 +370,7 @@ PIVOT (
   FOR (y, course) IN ((2012, 'dotNET'), (2013, 'Java'))
 )
 -- !query schema
-struct<year:int,[2012, dotNET]:array<int>,[2013, Java]:array<int>>
+struct<year:int,{2012, dotNET}:array<int>,{2013, Java}:array<int>>
 -- !query output
 2012	[1,1]	NULL
 2013	NULL	[2,2]
@@ -404,7 +404,7 @@ PIVOT (
   FOR (course, a) IN (('dotNET', array(1, 1)), ('Java', array(2, 2)))
 )
 -- !query schema
-struct<year:int,[dotNET, [1, 1]]:bigint,[Java, [2, 2]]:bigint>
+struct<year:int,{dotNET, [1, 1]}:bigint,{Java, [2, 2]}:bigint>
 -- !query output
 2012	15000	NULL
 2013	NULL	30000
@@ -421,7 +421,7 @@ PIVOT (
   FOR s IN ((1, 'a'), (2, 'b'))
 )
 -- !query schema
-struct<year:int,[1, a]:bigint,[2, b]:bigint>
+struct<year:int,{1, a}:bigint,{2, b}:bigint>
 -- !query output
 2012	35000	NULL
 2013	NULL	78000
@@ -438,7 +438,7 @@ PIVOT (
   FOR (course, s) IN (('dotNET', (1, 'a')), ('Java', (2, 'b')))
 )
 -- !query schema
-struct<year:int,[dotNET, [1, a]]:bigint,[Java, [2, b]]:bigint>
+struct<year:int,{dotNET, {1, a}}:bigint,{Java, {2, b}}:bigint>
 -- !query output
 2012	15000	NULL
 2013	NULL	30000
@@ -458,7 +458,7 @@ PIVOT (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Invalid pivot column 'm#x'. Pivot columns must be comparable.;
+Invalid pivot column 'm#x'. Pivot columns must be comparable.
 
 
 -- !query
@@ -475,7 +475,7 @@ PIVOT (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Invalid pivot column 'named_struct(course, course#x, m, m#x)'. Pivot columns must be comparable.;
+Invalid pivot column 'named_struct(course, course#x, m, m#x)'. Pivot columns must be comparable.
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part1.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part1.sql.out
index f7bba96738eab..0d836258f9179 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part1.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part1.sql.out
@@ -77,7 +77,7 @@ struct<max_324_78:float>
 -- !query
 SELECT stddev_pop(b) FROM aggtest
 -- !query schema
-struct<stddev_pop(CAST(b AS DOUBLE)):double>
+struct<stddev_pop(b):double>
 -- !query output
 131.10703231895047
 
@@ -85,7 +85,7 @@ struct<stddev_pop(CAST(b AS DOUBLE)):double>
 -- !query
 SELECT stddev_samp(b) FROM aggtest
 -- !query schema
-struct<stddev_samp(CAST(b AS DOUBLE)):double>
+struct<stddev_samp(b):double>
 -- !query output
 151.38936080399804
 
@@ -93,7 +93,7 @@ struct<stddev_samp(CAST(b AS DOUBLE)):double>
 -- !query
 SELECT var_pop(b) FROM aggtest
 -- !query schema
-struct<var_pop(CAST(b AS DOUBLE)):double>
+struct<var_pop(b):double>
 -- !query output
 17189.053923482323
 
@@ -101,7 +101,7 @@ struct<var_pop(CAST(b AS DOUBLE)):double>
 -- !query
 SELECT var_samp(b) FROM aggtest
 -- !query schema
-struct<var_samp(CAST(b AS DOUBLE)):double>
+struct<var_samp(b):double>
 -- !query output
 22918.738564643096
 
@@ -109,7 +109,7 @@ struct<var_samp(CAST(b AS DOUBLE)):double>
 -- !query
 SELECT stddev_pop(CAST(b AS Decimal(38,0))) FROM aggtest
 -- !query schema
-struct<stddev_pop(CAST(CAST(b AS DECIMAL(38,0)) AS DOUBLE)):double>
+struct<stddev_pop(CAST(b AS DECIMAL(38,0))):double>
 -- !query output
 131.18117242958306
 
@@ -117,7 +117,7 @@ struct<stddev_pop(CAST(CAST(b AS DECIMAL(38,0)) AS DOUBLE)):double>
 -- !query
 SELECT stddev_samp(CAST(b AS Decimal(38,0))) FROM aggtest
 -- !query schema
-struct<stddev_samp(CAST(CAST(b AS DECIMAL(38,0)) AS DOUBLE)):double>
+struct<stddev_samp(CAST(b AS DECIMAL(38,0))):double>
 -- !query output
 151.47497042966097
 
@@ -125,7 +125,7 @@ struct<stddev_samp(CAST(CAST(b AS DECIMAL(38,0)) AS DOUBLE)):double>
 -- !query
 SELECT var_pop(CAST(b AS Decimal(38,0))) FROM aggtest
 -- !query schema
-struct<var_pop(CAST(CAST(b AS DECIMAL(38,0)) AS DOUBLE)):double>
+struct<var_pop(CAST(b AS DECIMAL(38,0))):double>
 -- !query output
 17208.5
 
@@ -133,7 +133,7 @@ struct<var_pop(CAST(CAST(b AS DECIMAL(38,0)) AS DOUBLE)):double>
 -- !query
 SELECT var_samp(CAST(b AS Decimal(38,0))) FROM aggtest
 -- !query schema
-struct<var_samp(CAST(CAST(b AS DECIMAL(38,0)) AS DOUBLE)):double>
+struct<var_samp(CAST(b AS DECIMAL(38,0))):double>
 -- !query output
 22944.666666666668
 
@@ -141,17 +141,17 @@ struct<var_samp(CAST(CAST(b AS DECIMAL(38,0)) AS DOUBLE)):double>
 -- !query
 SELECT var_pop(1.0), var_samp(2.0)
 -- !query schema
-struct<var_pop(CAST(1.0 AS DOUBLE)):double,var_samp(CAST(2.0 AS DOUBLE)):double>
+struct<var_pop(1.0):double,var_samp(2.0):double>
 -- !query output
-0.0	NaN
+0.0	NULL
 
 
 -- !query
 SELECT stddev_pop(CAST(3.0 AS Decimal(38,0))), stddev_samp(CAST(4.0 AS Decimal(38,0)))
 -- !query schema
-struct<stddev_pop(CAST(CAST(3.0 AS DECIMAL(38,0)) AS DOUBLE)):double,stddev_samp(CAST(CAST(4.0 AS DECIMAL(38,0)) AS DOUBLE)):double>
+struct<stddev_pop(CAST(3.0 AS DECIMAL(38,0))):double,stddev_samp(CAST(4.0 AS DECIMAL(38,0))):double>
 -- !query output
-0.0	NaN
+0.0	NULL
 
 
 -- !query
@@ -291,7 +291,7 @@ struct<avg(CAST(x AS DOUBLE)):double,var_pop(CAST(x AS DOUBLE)):double>
 -- !query
 SELECT covar_pop(b, a), covar_samp(b, a) FROM aggtest
 -- !query schema
-struct<covar_pop(CAST(b AS DOUBLE), CAST(a AS DOUBLE)):double,covar_samp(CAST(b AS DOUBLE), CAST(a AS DOUBLE)):double>
+struct<covar_pop(b, a):double,covar_samp(b, a):double>
 -- !query output
 653.6289553875104	871.5052738500139
 
@@ -299,7 +299,7 @@ struct<covar_pop(CAST(b AS DOUBLE), CAST(a AS DOUBLE)):double,covar_samp(CAST(b
 -- !query
 SELECT corr(b, a) FROM aggtest
 -- !query schema
-struct<corr(CAST(b AS DOUBLE), CAST(a AS DOUBLE)):double>
+struct<corr(b, a):double>
 -- !query output
 0.1396345165178734
 
@@ -381,8 +381,8 @@ struct<>
 org.apache.spark.sql.AnalysisException
 
 Aggregate/Window/Generate expressions are not valid in where clause of the query.
-Expression in where clause: [(sum(DISTINCT CAST((outer(a.`four`) + b.`four`) AS BIGINT)) = CAST(b.`four` AS BIGINT))]
-Invalid expressions: [sum(DISTINCT CAST((outer(a.`four`) + b.`four`) AS BIGINT))];
+Expression in where clause: [(sum(DISTINCT (outer(a.four) + b.four)) = CAST(b.four AS BIGINT))]
+Invalid expressions: [sum(DISTINCT (outer(a.four) + b.four))]
 
 
 -- !query
@@ -393,4 +393,4 @@ from tenk1 o
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`o.unique1`' given input columns: [i.even, i.fivethous, i.four, i.hundred, i.odd, i.string4, i.stringu1, i.stringu2, i.ten, i.tenthous, i.thousand, i.twenty, i.two, i.twothousand, i.unique1, i.unique2]; line 2 pos 63
+cannot resolve 'o.unique1' given input columns: [i.even, i.fivethous, i.four, i.hundred, i.odd, i.string4, i.stringu1, i.stringu2, i.ten, i.tenthous, i.thousand, i.twenty, i.two, i.twothousand, i.unique1, i.unique2]; line 2 pos 63
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part3.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part3.sql.out
index 69f96b02782e3..f3ab092baf696 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part3.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part3.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 4
+-- Number of queries: 5
 
 
 -- !query
@@ -8,7 +8,7 @@ select max(min(unique1)) from tenk1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-It is not allowed to use an aggregate function in the argument of another aggregate function. Please use the inner aggregate function in a sub-query.;
+It is not allowed to use an aggregate function in the argument of another aggregate function. Please use the inner aggregate function in a sub-query.
 
 
 -- !query
@@ -22,11 +22,25 @@ struct<min(unique1) FILTER (WHERE (unique1 > 100)):int>
 -- !query
 select sum(1/ten) filter (where ten > 0) from tenk1
 -- !query schema
-struct<sum((CAST(1 AS DOUBLE) / CAST(ten AS DOUBLE))) FILTER (WHERE (ten > 0)):double>
+struct<sum((1 / ten)) FILTER (WHERE (ten > 0)):double>
 -- !query output
 2828.9682539682954
 
 
+-- !query
+select ten, sum(distinct four) filter (where four > 10) from onek a
+group by ten
+having exists (select 1 from onek b where sum(distinct a.four) = b.four)
+-- !query schema
+struct<ten:int,sum(DISTINCT four) FILTER (WHERE (four > 10)):bigint>
+-- !query output
+0	NULL
+2	NULL
+4	NULL
+6	NULL
+8	NULL
+
+
 -- !query
 select (select count(*)
         from (values (1)) t0(inner_c))
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/boolean.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/boolean.sql.out
index 0347e0dc7853b..ef9a67a50b1db 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/boolean.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/boolean.sql.out
@@ -53,9 +53,10 @@ true
 -- !query
 SELECT boolean('test') AS error
 -- !query schema
-struct<error:boolean>
+struct<>
 -- !query output
-NULL
+java.lang.UnsupportedOperationException
+invalid input syntax for type boolean: test
 
 
 -- !query
@@ -69,9 +70,10 @@ false
 -- !query
 SELECT boolean('foo') AS error
 -- !query schema
-struct<error:boolean>
+struct<>
 -- !query output
-NULL
+java.lang.UnsupportedOperationException
+invalid input syntax for type boolean: foo
 
 
 -- !query
@@ -93,9 +95,10 @@ true
 -- !query
 SELECT boolean('yeah') AS error
 -- !query schema
-struct<error:boolean>
+struct<>
 -- !query output
-NULL
+java.lang.UnsupportedOperationException
+invalid input syntax for type boolean: yeah
 
 
 -- !query
@@ -117,57 +120,64 @@ false
 -- !query
 SELECT boolean('nay') AS error
 -- !query schema
-struct<error:boolean>
+struct<>
 -- !query output
-NULL
+java.lang.UnsupportedOperationException
+invalid input syntax for type boolean: nay
 
 
 -- !query
 SELECT boolean('on') AS true
 -- !query schema
-struct<true:boolean>
+struct<>
 -- !query output
-NULL
+java.lang.UnsupportedOperationException
+invalid input syntax for type boolean: on
 
 
 -- !query
 SELECT boolean('off') AS `false`
 -- !query schema
-struct<false:boolean>
+struct<>
 -- !query output
-NULL
+java.lang.UnsupportedOperationException
+invalid input syntax for type boolean: off
 
 
 -- !query
 SELECT boolean('of') AS `false`
 -- !query schema
-struct<false:boolean>
+struct<>
 -- !query output
-NULL
+java.lang.UnsupportedOperationException
+invalid input syntax for type boolean: of
 
 
 -- !query
 SELECT boolean('o') AS error
 -- !query schema
-struct<error:boolean>
+struct<>
 -- !query output
-NULL
+java.lang.UnsupportedOperationException
+invalid input syntax for type boolean: o
 
 
 -- !query
 SELECT boolean('on_') AS error
 -- !query schema
-struct<error:boolean>
+struct<>
 -- !query output
-NULL
+java.lang.UnsupportedOperationException
+invalid input syntax for type boolean: on_
 
 
 -- !query
 SELECT boolean('off_') AS error
 -- !query schema
-struct<error:boolean>
+struct<>
 -- !query output
-NULL
+java.lang.UnsupportedOperationException
+invalid input syntax for type boolean: off_
 
 
 -- !query
@@ -181,9 +191,10 @@ true
 -- !query
 SELECT boolean('11') AS error
 -- !query schema
-struct<error:boolean>
+struct<>
 -- !query output
-NULL
+java.lang.UnsupportedOperationException
+invalid input syntax for type boolean: 11
 
 
 -- !query
@@ -197,17 +208,19 @@ false
 -- !query
 SELECT boolean('000') AS error
 -- !query schema
-struct<error:boolean>
+struct<>
 -- !query output
-NULL
+java.lang.UnsupportedOperationException
+invalid input syntax for type boolean: 000
 
 
 -- !query
 SELECT boolean('') AS error
 -- !query schema
-struct<error:boolean>
+struct<>
 -- !query output
-NULL
+java.lang.UnsupportedOperationException
+invalid input syntax for type boolean:
 
 
 -- !query
@@ -310,17 +323,19 @@ true	false
 -- !query
 SELECT boolean(string('  tru e ')) AS invalid
 -- !query schema
-struct<invalid:boolean>
+struct<>
 -- !query output
-NULL
+java.lang.UnsupportedOperationException
+invalid input syntax for type boolean:   tru e
 
 
 -- !query
 SELECT boolean(string('')) AS invalid
 -- !query schema
-struct<invalid:boolean>
+struct<>
 -- !query output
-NULL
+java.lang.UnsupportedOperationException
+invalid input syntax for type boolean:
 
 
 -- !query
@@ -463,7 +478,8 @@ INSERT INTO BOOLTBL2
 -- !query schema
 struct<>
 -- !query output
-
+org.apache.spark.sql.AnalysisException
+failed to evaluate expression CAST('XXX' AS BOOLEAN): invalid input syntax for type boolean: XXX; line 2 pos 3
 
 
 -- !query
@@ -471,7 +487,6 @@ SELECT '' AS f_4, BOOLTBL2.* FROM BOOLTBL2
 -- !query schema
 struct<f_4:string,f1:boolean>
 -- !query output
-	NULL
 	false
 	false
 	false
@@ -545,9 +560,6 @@ struct<tf_12_ff_4:string,f1:boolean,f1:boolean>
 	false	false
 	false	false
 	false	false
-	true	NULL
-	true	NULL
-	true	NULL
 	true	false
 	true	false
 	true	false
@@ -623,7 +635,7 @@ SELECT '' AS `Not False`, f1
 -- !query schema
 struct<Not False:string,f1:boolean>
 -- !query output
-	NULL
+
 
 
 -- !query
@@ -646,7 +658,6 @@ SELECT '' AS `Not True`, f1
 -- !query schema
 struct<Not True:string,f1:boolean>
 -- !query output
-	NULL
 	false
 	false
 	false
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/case.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/case.sql.out
index 1b002c3f48ae2..090b97e97bea6 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/case.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/case.sql.out
@@ -176,28 +176,28 @@ struct<None:string,NULL on no matches:int>
 -- !query
 SELECT CASE WHEN 1=0 THEN 1/0 WHEN 1=1 THEN 1 ELSE 2/0 END
 -- !query schema
-struct<CASE WHEN (1 = 0) THEN (CAST(1 AS DOUBLE) / CAST(0 AS DOUBLE)) WHEN (1 = 1) THEN CAST(1 AS DOUBLE) ELSE (CAST(2 AS DOUBLE) / CAST(0 AS DOUBLE)) END:double>
+struct<>
 -- !query output
-1.0
+java.lang.ArithmeticException
+divide by zero
 
 
 -- !query
 SELECT CASE 1 WHEN 0 THEN 1/0 WHEN 1 THEN 1 ELSE 2/0 END
 -- !query schema
-struct<CASE WHEN (1 = 0) THEN (CAST(1 AS DOUBLE) / CAST(0 AS DOUBLE)) WHEN (1 = 1) THEN CAST(1 AS DOUBLE) ELSE (CAST(2 AS DOUBLE) / CAST(0 AS DOUBLE)) END:double>
+struct<>
 -- !query output
-1.0
+java.lang.ArithmeticException
+divide by zero
 
 
 -- !query
 SELECT CASE WHEN i > 100 THEN 1/0 ELSE 0 END FROM case_tbl
 -- !query schema
-struct<CASE WHEN (i > 100) THEN (CAST(1 AS DOUBLE) / CAST(0 AS DOUBLE)) ELSE CAST(0 AS DOUBLE) END:double>
+struct<>
 -- !query output
-0.0
-0.0
-0.0
-0.0
+java.lang.ArithmeticException
+divide by zero
 
 
 -- !query
@@ -294,7 +294,7 @@ struct<i:int,f:double>
 SELECT COALESCE(a.f, b.i, b.j)
   FROM CASE_TBL a, CASE2_TBL b
 -- !query schema
-struct<coalesce(f, CAST(i AS DOUBLE), CAST(j AS DOUBLE)):double>
+struct<coalesce(f, i, j):double>
 -- !query output
 -30.3
 -30.3
@@ -388,7 +388,7 @@ SELECT CASE
   WHEN 'it was bar!' THEN 'bar recognized'
   ELSE 'unrecognized' END
 -- !query schema
-struct<CASE WHEN (CASE WHEN (vol(bar) = foo) THEN it was foo! WHEN (vol(bar) = vol(null)) THEN null input WHEN (vol(bar) = bar) THEN it was bar! END = it was foo!) THEN foo recognized WHEN (CASE WHEN (vol(bar) = foo) THEN it was foo! WHEN (vol(bar) = vol(null)) THEN null input WHEN (vol(bar) = bar) THEN it was bar! END = it was bar!) THEN bar recognized ELSE unrecognized END:string>
+struct<CASE WHEN (CASE WHEN (vol(bar) = foo) THEN it was foo! WHEN (vol(bar) = vol(NULL)) THEN null input WHEN (vol(bar) = bar) THEN it was bar! END = it was foo!) THEN foo recognized WHEN (CASE WHEN (vol(bar) = foo) THEN it was foo! WHEN (vol(bar) = vol(NULL)) THEN null input WHEN (vol(bar) = bar) THEN it was bar! END = it was bar!) THEN bar recognized ELSE unrecognized END:string>
 -- !query output
 bar recognized
 
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out
index ae1cb2f171704..9d6a8e656b2a4 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out
@@ -56,7 +56,7 @@ CREATE VIEW key_dependent_view AS
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-expression 'spark_catalog.default.view_base_table.`data`' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.;
+expression 'spark_catalog.default.view_base_table.data' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.
 
 
 -- !query
@@ -257,7 +257,7 @@ View Text           	SELECT * FROM base_table
 View Original Text  	SELECT * FROM base_table	                    
 View Catalog and Namespace	spark_catalog.temp_view_test	                    
 View Query Output Columns	[a, id]             	                    
-Table Properties    	[view.catalogAndNamespace.numParts=2, view.query.out.col.0=a, view.query.out.numCols=2, view.query.out.col.1=id, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=temp_view_test]
+Table Properties    	[]
 
 
 -- !query
@@ -266,7 +266,7 @@ CREATE VIEW v1_temp AS SELECT * FROM temp_table
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `temp_view_test`.`v1_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `temp_view_test`.`v1_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW
 
 
 -- !query
@@ -313,7 +313,7 @@ View Text           	SELECT * FROM base_table
 View Original Text  	SELECT * FROM base_table	                    
 View Catalog and Namespace	spark_catalog.temp_view_test	                    
 View Query Output Columns	[a, id]             	                    
-Table Properties    	[view.catalogAndNamespace.numParts=2, view.query.out.col.0=a, view.query.out.numCols=2, view.query.out.col.1=id, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=temp_view_test]
+Table Properties    	[]
 
 
 -- !query
@@ -322,7 +322,7 @@ CREATE VIEW temp_view_test.v3_temp AS SELECT * FROM temp_table
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `temp_view_test`.`v3_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `temp_view_test`.`v3_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW
 
 
 -- !query
@@ -359,7 +359,7 @@ View Original Text  	SELECT t1.a AS t1_a, t2.a AS t2_a
     WHERE t1.id = t2.id	                    
 View Catalog and Namespace	spark_catalog.temp_view_test	                    
 View Query Output Columns	[t1_a, t2_a]        	                    
-Table Properties    	[view.catalogAndNamespace.numParts=2, view.query.out.col.0=t1_a, view.query.out.numCols=2, view.query.out.col.1=t2_a, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=temp_view_test]
+Table Properties    	[]
 
 
 -- !query
@@ -371,7 +371,7 @@ CREATE VIEW v4_temp AS
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `temp_view_test`.`v4_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `temp_view_test`.`v4_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW
 
 
 -- !query
@@ -383,7 +383,7 @@ CREATE VIEW v5_temp AS
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `temp_view_test`.`v5_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `temp_view_test`.`v5_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW
 
 
 -- !query
@@ -413,7 +413,7 @@ View Text           	SELECT * FROM base_table WHERE id IN (SELECT id FROM base_t
 View Original Text  	SELECT * FROM base_table WHERE id IN (SELECT id FROM base_table2)	                    
 View Catalog and Namespace	spark_catalog.temp_view_test	                    
 View Query Output Columns	[a, id]             	                    
-Table Properties    	[view.catalogAndNamespace.numParts=2, view.query.out.col.0=a, view.query.out.numCols=2, view.query.out.col.1=id, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=temp_view_test]
+Table Properties    	[]
 
 
 -- !query
@@ -443,7 +443,7 @@ View Text           	SELECT t1.id, t2.a FROM base_table t1, (SELECT * FROM base_
 View Original Text  	SELECT t1.id, t2.a FROM base_table t1, (SELECT * FROM base_table2) t2	                    
 View Catalog and Namespace	spark_catalog.temp_view_test	                    
 View Query Output Columns	[id, a]             	                    
-Table Properties    	[view.catalogAndNamespace.numParts=2, view.query.out.col.0=id, view.query.out.numCols=2, view.query.out.col.1=a, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=temp_view_test]
+Table Properties    	[]
 
 
 -- !query
@@ -473,7 +473,7 @@ View Text           	SELECT * FROM base_table WHERE EXISTS (SELECT 1 FROM base_t
 View Original Text  	SELECT * FROM base_table WHERE EXISTS (SELECT 1 FROM base_table2)	                    
 View Catalog and Namespace	spark_catalog.temp_view_test	                    
 View Query Output Columns	[a, id]             	                    
-Table Properties    	[view.catalogAndNamespace.numParts=2, view.query.out.col.0=a, view.query.out.numCols=2, view.query.out.col.1=id, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=temp_view_test]
+Table Properties    	[]
 
 
 -- !query
@@ -503,7 +503,7 @@ View Text           	SELECT * FROM base_table WHERE NOT EXISTS (SELECT 1 FROM ba
 View Original Text  	SELECT * FROM base_table WHERE NOT EXISTS (SELECT 1 FROM base_table2)	                    
 View Catalog and Namespace	spark_catalog.temp_view_test	                    
 View Query Output Columns	[a, id]             	                    
-Table Properties    	[view.catalogAndNamespace.numParts=2, view.query.out.col.0=a, view.query.out.numCols=2, view.query.out.col.1=id, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=temp_view_test]
+Table Properties    	[]
 
 
 -- !query
@@ -533,7 +533,7 @@ View Text           	SELECT * FROM base_table WHERE EXISTS (SELECT 1)
 View Original Text  	SELECT * FROM base_table WHERE EXISTS (SELECT 1)	                    
 View Catalog and Namespace	spark_catalog.temp_view_test	                    
 View Query Output Columns	[a, id]             	                    
-Table Properties    	[view.catalogAndNamespace.numParts=2, view.query.out.col.0=a, view.query.out.numCols=2, view.query.out.col.1=id, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=temp_view_test]
+Table Properties    	[]
 
 
 -- !query
@@ -542,7 +542,7 @@ CREATE VIEW v6_temp AS SELECT * FROM base_table WHERE id IN (SELECT id FROM temp
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `temp_view_test`.`v6_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `temp_view_test`.`v6_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW
 
 
 -- !query
@@ -551,7 +551,7 @@ CREATE VIEW v7_temp AS SELECT t1.id, t2.a FROM base_table t1, (SELECT * FROM tem
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `temp_view_test`.`v7_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `temp_view_test`.`v7_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW
 
 
 -- !query
@@ -560,7 +560,7 @@ CREATE VIEW v8_temp AS SELECT * FROM base_table WHERE EXISTS (SELECT 1 FROM temp
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `temp_view_test`.`v8_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `temp_view_test`.`v8_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW
 
 
 -- !query
@@ -569,7 +569,7 @@ CREATE VIEW v9_temp AS SELECT * FROM base_table WHERE NOT EXISTS (SELECT 1 FROM
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `temp_view_test`.`v9_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `temp_view_test`.`v9_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW
 
 
 -- !query
@@ -669,7 +669,7 @@ View Text           	SELECT * FROM t1 CROSS JOIN t2
 View Original Text  	SELECT * FROM t1 CROSS JOIN t2	                    
 View Catalog and Namespace	spark_catalog.testviewschm2	                    
 View Query Output Columns	[num, name, num2, value]	                    
-Table Properties    	[view.query.out.col.3=value, view.catalogAndNamespace.numParts=2, view.query.out.col.0=num, view.query.out.numCols=4, view.query.out.col.1=name, view.catalogAndNamespace.part.0=spark_catalog, view.query.out.col.2=num2, view.catalogAndNamespace.part.1=testviewschm2]
+Table Properties    	[]
 
 
 -- !query
@@ -678,7 +678,7 @@ CREATE VIEW temporal1 AS SELECT * FROM t1 CROSS JOIN tt
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `testviewschm2`.`temporal1` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `testviewschm2`.`temporal1` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW
 
 
 -- !query
@@ -710,7 +710,7 @@ View Text           	SELECT * FROM t1 INNER JOIN t2 ON t1.num = t2.num2
 View Original Text  	SELECT * FROM t1 INNER JOIN t2 ON t1.num = t2.num2	                    
 View Catalog and Namespace	spark_catalog.testviewschm2	                    
 View Query Output Columns	[num, name, num2, value]	                    
-Table Properties    	[view.query.out.col.3=value, view.catalogAndNamespace.numParts=2, view.query.out.col.0=num, view.query.out.numCols=4, view.query.out.col.1=name, view.catalogAndNamespace.part.0=spark_catalog, view.query.out.col.2=num2, view.catalogAndNamespace.part.1=testviewschm2]
+Table Properties    	[]
 
 
 -- !query
@@ -719,7 +719,7 @@ CREATE VIEW temporal2 AS SELECT * FROM t1 INNER JOIN tt ON t1.num = tt.num2
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `testviewschm2`.`temporal2` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `testviewschm2`.`temporal2` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW
 
 
 -- !query
@@ -751,7 +751,7 @@ View Text           	SELECT * FROM t1 LEFT JOIN t2 ON t1.num = t2.num2
 View Original Text  	SELECT * FROM t1 LEFT JOIN t2 ON t1.num = t2.num2	                    
 View Catalog and Namespace	spark_catalog.testviewschm2	                    
 View Query Output Columns	[num, name, num2, value]	                    
-Table Properties    	[view.query.out.col.3=value, view.catalogAndNamespace.numParts=2, view.query.out.col.0=num, view.query.out.numCols=4, view.query.out.col.1=name, view.catalogAndNamespace.part.0=spark_catalog, view.query.out.col.2=num2, view.catalogAndNamespace.part.1=testviewschm2]
+Table Properties    	[]
 
 
 -- !query
@@ -760,7 +760,7 @@ CREATE VIEW temporal3 AS SELECT * FROM t1 LEFT JOIN tt ON t1.num = tt.num2
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `testviewschm2`.`temporal3` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `testviewschm2`.`temporal3` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW
 
 
 -- !query
@@ -792,7 +792,7 @@ View Text           	SELECT * FROM t1 LEFT JOIN t2 ON t1.num = t2.num2 AND t2.va
 View Original Text  	SELECT * FROM t1 LEFT JOIN t2 ON t1.num = t2.num2 AND t2.value = 'xxx'	                    
 View Catalog and Namespace	spark_catalog.testviewschm2	                    
 View Query Output Columns	[num, name, num2, value]	                    
-Table Properties    	[view.query.out.col.3=value, view.catalogAndNamespace.numParts=2, view.query.out.col.0=num, view.query.out.numCols=4, view.query.out.col.1=name, view.catalogAndNamespace.part.0=spark_catalog, view.query.out.col.2=num2, view.catalogAndNamespace.part.1=testviewschm2]
+Table Properties    	[]
 
 
 -- !query
@@ -801,7 +801,7 @@ CREATE VIEW temporal4 AS SELECT * FROM t1 LEFT JOIN tt ON t1.num = tt.num2 AND t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `testviewschm2`.`temporal4` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `testviewschm2`.`temporal4` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW
 
 
 -- !query
@@ -810,7 +810,7 @@ CREATE VIEW temporal5 AS SELECT * FROM t1 WHERE num IN (SELECT num FROM t1 WHERE
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `testviewschm2`.`temporal5` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `testviewschm2`.`temporal5` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW
 
 
 -- !query
@@ -894,7 +894,7 @@ BETWEEN (SELECT d FROM tbl2 WHERE c = 1) AND (SELECT e FROM tbl3 WHERE f = 2)
 AND EXISTS (SELECT g FROM tbl4 LEFT JOIN tbl3 ON tbl4.h = tbl3.f)	                    
 View Catalog and Namespace	spark_catalog.testviewschm2	                    
 View Query Output Columns	[a, b]              	                    
-Table Properties    	[view.catalogAndNamespace.numParts=2, view.query.out.col.0=a, view.query.out.numCols=2, view.query.out.col.1=b, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=testviewschm2]
+Table Properties    	[]
 
 
 -- !query
@@ -933,7 +933,7 @@ AND EXISTS (SELECT g FROM tbl4 LEFT JOIN tbl3 ON tbl4.h = tbl3.f)
 AND NOT EXISTS (SELECT g FROM tbl4 LEFT JOIN tmptbl ON tbl4.h = tmptbl.j)	                    
 View Catalog and Namespace	spark_catalog.testviewschm2	                    
 View Query Output Columns	[a, b]              	                    
-Table Properties    	[view.catalogAndNamespace.numParts=2, view.query.out.col.0=a, view.query.out.numCols=2, view.query.out.col.1=b, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=testviewschm2]
+Table Properties    	[]
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out
index 1d862ba8a41a8..a959284750483 100755
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out
@@ -584,31 +584,34 @@ select make_date(-44, 3, 15)
 -- !query schema
 struct<make_date(-44, 3, 15):date>
 -- !query output
--0044-03-15
+0045-03-15
 
 
 -- !query
 select make_date(2013, 2, 30)
 -- !query schema
-struct<make_date(2013, 2, 30):date>
+struct<>
 -- !query output
-NULL
+java.time.DateTimeException
+Invalid date 'FEBRUARY 30'
 
 
 -- !query
 select make_date(2013, 13, 1)
 -- !query schema
-struct<make_date(2013, 13, 1):date>
+struct<>
 -- !query output
-NULL
+java.time.DateTimeException
+Invalid value for MonthOfYear (valid values 1 - 12): 13
 
 
 -- !query
 select make_date(2013, 11, -1)
 -- !query schema
-struct<make_date(2013, 11, -1):date>
+struct<>
 -- !query output
-NULL
+java.time.DateTimeException
+Invalid value for DayOfMonth (valid values 1 - 28/31): -1
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out
index fe8375c5eab8f..c9ff3c8debc21 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out
@@ -53,7 +53,7 @@ struct<>
 -- !query
 SELECT float('NaN')
 -- !query schema
-struct<CAST(NaN AS FLOAT):float>
+struct<NaN:float>
 -- !query output
 NaN
 
@@ -61,7 +61,7 @@ NaN
 -- !query
 SELECT float('nan')
 -- !query schema
-struct<CAST(nan AS FLOAT):float>
+struct<nan:float>
 -- !query output
 NaN
 
@@ -69,7 +69,7 @@ NaN
 -- !query
 SELECT float('   NAN  ')
 -- !query schema
-struct<CAST(   NAN   AS FLOAT):float>
+struct<   NAN  :float>
 -- !query output
 NaN
 
@@ -77,7 +77,7 @@ NaN
 -- !query
 SELECT float('infinity')
 -- !query schema
-struct<CAST(infinity AS FLOAT):float>
+struct<infinity:float>
 -- !query output
 Infinity
 
@@ -85,7 +85,7 @@ Infinity
 -- !query
 SELECT float('          -INFINiTY   ')
 -- !query schema
-struct<CAST(          -INFINiTY    AS FLOAT):float>
+struct<          -INFINiTY   :float>
 -- !query output
 -Infinity
 
@@ -120,7 +120,7 @@ invalid input syntax for type numeric:  INFINITY    x
 -- !query
 SELECT float('Infinity') + 100.0
 -- !query schema
-struct<(CAST(CAST(Infinity AS FLOAT) AS DOUBLE) + CAST(100.0 AS DOUBLE)):double>
+struct<(Infinity + 100.0):double>
 -- !query output
 Infinity
 
@@ -128,7 +128,7 @@ Infinity
 -- !query
 SELECT float('Infinity') / float('Infinity')
 -- !query schema
-struct<(CAST(CAST(Infinity AS FLOAT) AS DOUBLE) / CAST(CAST(Infinity AS FLOAT) AS DOUBLE)):double>
+struct<(Infinity / Infinity):double>
 -- !query output
 NaN
 
@@ -136,7 +136,7 @@ NaN
 -- !query
 SELECT float('nan') / float('nan')
 -- !query schema
-struct<(CAST(CAST(nan AS FLOAT) AS DOUBLE) / CAST(CAST(nan AS FLOAT) AS DOUBLE)):double>
+struct<(nan / nan):double>
 -- !query output
 NaN
 
@@ -227,22 +227,22 @@ struct<four:string,f1:float>
 SELECT '' AS three, f.f1, f.f1 * '-10' AS x FROM FLOAT4_TBL f
    WHERE f.f1 > '0.0'
 -- !query schema
-struct<three:string,f1:float,x:double>
+struct<three:string,f1:float,x:float>
 -- !query output
-	1.2345679E-20	-1.2345678720289608E-19
-	1.2345679E20	-1.2345678955701443E21
-	1004.3	-10042.999877929688
+	1.2345679E-20	-1.2345678E-19
+	1.2345679E20	-1.2345678E21
+	1004.3	-10043.0
 
 
 -- !query
 SELECT '' AS three, f.f1, f.f1 + '-10' AS x FROM FLOAT4_TBL f
    WHERE f.f1 > '0.0'
 -- !query schema
-struct<three:string,f1:float,x:double>
+struct<three:string,f1:float,x:float>
 -- !query output
 	1.2345679E-20	-10.0
-	1.2345679E20	1.2345678955701443E20
-	1004.3	994.2999877929688
+	1.2345679E20	1.2345679E20
+	1004.3	994.3
 
 
 -- !query
@@ -260,11 +260,11 @@ struct<three:string,f1:float,x:double>
 SELECT '' AS three, f.f1, f.f1 - '-10' AS x FROM FLOAT4_TBL f
    WHERE f.f1 > '0.0'
 -- !query schema
-struct<three:string,f1:float,x:double>
+struct<three:string,f1:float,x:float>
 -- !query output
 	1.2345679E-20	10.0
-	1.2345679E20	1.2345678955701443E20
-	1004.3	1014.2999877929688
+	1.2345679E20	1.2345679E20
+	1004.3	1014.3
 
 
 -- !query
@@ -282,7 +282,7 @@ struct<five:string,f1:float>
 -- !query
 SELECT smallint(float('32767.4'))
 -- !query schema
-struct<CAST(CAST(32767.4 AS FLOAT) AS SMALLINT):smallint>
+struct<32767.4:smallint>
 -- !query output
 32767
 
@@ -290,7 +290,7 @@ struct<CAST(CAST(32767.4 AS FLOAT) AS SMALLINT):smallint>
 -- !query
 SELECT smallint(float('32767.6'))
 -- !query schema
-struct<CAST(CAST(32767.6 AS FLOAT) AS SMALLINT):smallint>
+struct<32767.6:smallint>
 -- !query output
 32767
 
@@ -298,7 +298,7 @@ struct<CAST(CAST(32767.6 AS FLOAT) AS SMALLINT):smallint>
 -- !query
 SELECT smallint(float('-32768.4'))
 -- !query schema
-struct<CAST(CAST(-32768.4 AS FLOAT) AS SMALLINT):smallint>
+struct<-32768.4:smallint>
 -- !query output
 -32768
 
@@ -306,7 +306,7 @@ struct<CAST(CAST(-32768.4 AS FLOAT) AS SMALLINT):smallint>
 -- !query
 SELECT smallint(float('-32768.6'))
 -- !query schema
-struct<CAST(CAST(-32768.6 AS FLOAT) AS SMALLINT):smallint>
+struct<-32768.6:smallint>
 -- !query output
 -32768
 
@@ -314,7 +314,7 @@ struct<CAST(CAST(-32768.6 AS FLOAT) AS SMALLINT):smallint>
 -- !query
 SELECT int(float('2147483520'))
 -- !query schema
-struct<CAST(CAST(2147483520 AS FLOAT) AS INT):int>
+struct<2147483520:int>
 -- !query output
 2147483520
 
@@ -331,7 +331,7 @@ Casting 2.14748365E9 to int causes overflow
 -- !query
 SELECT int(float('-2147483648.5'))
 -- !query schema
-struct<CAST(CAST(-2147483648.5 AS FLOAT) AS INT):int>
+struct<-2147483648.5:int>
 -- !query output
 -2147483648
 
@@ -348,7 +348,7 @@ Casting -2.1474839E9 to int causes overflow
 -- !query
 SELECT bigint(float('9223369837831520256'))
 -- !query schema
-struct<CAST(CAST(9223369837831520256 AS FLOAT) AS BIGINT):bigint>
+struct<9223369837831520256:bigint>
 -- !query output
 9223369837831520256
 
@@ -356,7 +356,7 @@ struct<CAST(CAST(9223369837831520256 AS FLOAT) AS BIGINT):bigint>
 -- !query
 SELECT bigint(float('9223372036854775807'))
 -- !query schema
-struct<CAST(CAST(9223372036854775807 AS FLOAT) AS BIGINT):bigint>
+struct<9223372036854775807:bigint>
 -- !query output
 9223372036854775807
 
@@ -364,7 +364,7 @@ struct<CAST(CAST(9223372036854775807 AS FLOAT) AS BIGINT):bigint>
 -- !query
 SELECT bigint(float('-9223372036854775808.5'))
 -- !query schema
-struct<CAST(CAST(-9223372036854775808.5 AS FLOAT) AS BIGINT):bigint>
+struct<-9223372036854775808.5:bigint>
 -- !query output
 -9223372036854775808
 
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/float8.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/float8.sql.out
index 4cdb6958a230a..45e103e9482a3 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/float8.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/float8.sql.out
@@ -53,7 +53,7 @@ struct<>
 -- !query
 SELECT double('10e400')
 -- !query schema
-struct<CAST(10e400 AS DOUBLE):double>
+struct<10e400:double>
 -- !query output
 Infinity
 
@@ -61,7 +61,7 @@ Infinity
 -- !query
 SELECT double('-10e400')
 -- !query schema
-struct<CAST(-10e400 AS DOUBLE):double>
+struct<-10e400:double>
 -- !query output
 -Infinity
 
@@ -69,7 +69,7 @@ struct<CAST(-10e400 AS DOUBLE):double>
 -- !query
 SELECT double('10e-400')
 -- !query schema
-struct<CAST(10e-400 AS DOUBLE):double>
+struct<10e-400:double>
 -- !query output
 0.0
 
@@ -77,7 +77,7 @@ struct<CAST(10e-400 AS DOUBLE):double>
 -- !query
 SELECT double('-10e-400')
 -- !query schema
-struct<CAST(-10e-400 AS DOUBLE):double>
+struct<-10e-400:double>
 -- !query output
 -0.0
 
@@ -85,7 +85,7 @@ struct<CAST(-10e-400 AS DOUBLE):double>
 -- !query
 SELECT double('NaN')
 -- !query schema
-struct<CAST(NaN AS DOUBLE):double>
+struct<NaN:double>
 -- !query output
 NaN
 
@@ -93,7 +93,7 @@ NaN
 -- !query
 SELECT double('nan')
 -- !query schema
-struct<CAST(nan AS DOUBLE):double>
+struct<nan:double>
 -- !query output
 NaN
 
@@ -101,7 +101,7 @@ NaN
 -- !query
 SELECT double('   NAN  ')
 -- !query schema
-struct<CAST(   NAN   AS DOUBLE):double>
+struct<   NAN  :double>
 -- !query output
 NaN
 
@@ -109,7 +109,7 @@ NaN
 -- !query
 SELECT double('infinity')
 -- !query schema
-struct<CAST(infinity AS DOUBLE):double>
+struct<infinity:double>
 -- !query output
 Infinity
 
@@ -117,7 +117,7 @@ Infinity
 -- !query
 SELECT double('          -INFINiTY   ')
 -- !query schema
-struct<CAST(          -INFINiTY    AS DOUBLE):double>
+struct<          -INFINiTY   :double>
 -- !query output
 -Infinity
 
@@ -152,7 +152,7 @@ invalid input syntax for type numeric:  INFINITY    x
 -- !query
 SELECT double('Infinity') + 100.0
 -- !query schema
-struct<(CAST(Infinity AS DOUBLE) + CAST(100.0 AS DOUBLE)):double>
+struct<(Infinity + 100.0):double>
 -- !query output
 Infinity
 
@@ -160,7 +160,7 @@ Infinity
 -- !query
 SELECT double('Infinity') / double('Infinity')
 -- !query schema
-struct<(CAST(Infinity AS DOUBLE) / CAST(Infinity AS DOUBLE)):double>
+struct<(Infinity / Infinity):double>
 -- !query output
 NaN
 
@@ -168,7 +168,7 @@ NaN
 -- !query
 SELECT double('NaN') / double('NaN')
 -- !query schema
-struct<(CAST(NaN AS DOUBLE) / CAST(NaN AS DOUBLE)):double>
+struct<(NaN / NaN):double>
 -- !query output
 NaN
 
@@ -375,7 +375,7 @@ struct<eight:double>
 -- !query
 SELECT power(double('144'), double('0.5'))
 -- !query schema
-struct<POWER(CAST(144 AS DOUBLE), CAST(0.5 AS DOUBLE)):double>
+struct<POWER(144, 0.5):double>
 -- !query output
 12.0
 
@@ -383,7 +383,7 @@ struct<POWER(CAST(144 AS DOUBLE), CAST(0.5 AS DOUBLE)):double>
 -- !query
 SELECT power(double('NaN'), double('0.5'))
 -- !query schema
-struct<POWER(CAST(NaN AS DOUBLE), CAST(0.5 AS DOUBLE)):double>
+struct<POWER(NaN, 0.5):double>
 -- !query output
 NaN
 
@@ -391,7 +391,7 @@ NaN
 -- !query
 SELECT power(double('144'), double('NaN'))
 -- !query schema
-struct<POWER(CAST(144 AS DOUBLE), CAST(NaN AS DOUBLE)):double>
+struct<POWER(144, NaN):double>
 -- !query output
 NaN
 
@@ -399,7 +399,7 @@ NaN
 -- !query
 SELECT power(double('NaN'), double('NaN'))
 -- !query schema
-struct<POWER(CAST(NaN AS DOUBLE), CAST(NaN AS DOUBLE)):double>
+struct<POWER(NaN, NaN):double>
 -- !query output
 NaN
 
@@ -407,7 +407,7 @@ NaN
 -- !query
 SELECT power(double('-1'), double('NaN'))
 -- !query schema
-struct<POWER(CAST(-1 AS DOUBLE), CAST(NaN AS DOUBLE)):double>
+struct<POWER(-1, NaN):double>
 -- !query output
 NaN
 
@@ -415,7 +415,7 @@ NaN
 -- !query
 SELECT power(double('1'), double('NaN'))
 -- !query schema
-struct<POWER(CAST(1 AS DOUBLE), CAST(NaN AS DOUBLE)):double>
+struct<POWER(1, NaN):double>
 -- !query output
 NaN
 
@@ -423,7 +423,7 @@ NaN
 -- !query
 SELECT power(double('NaN'), double('0'))
 -- !query schema
-struct<POWER(CAST(NaN AS DOUBLE), CAST(0 AS DOUBLE)):double>
+struct<POWER(NaN, 0):double>
 -- !query output
 1.0
 
@@ -466,7 +466,7 @@ struct<>
 -- !query
 SELECT '' AS bad, f.f1 * '1e200' from UPDATED_FLOAT8_TBL f
 -- !query schema
-struct<bad:string,(f1 * CAST(1e200 AS DOUBLE)):double>
+struct<bad:string,(f1 * 1e200):double>
 -- !query output
 	-1.0042999999999999E203
 	-1.2345678901234
@@ -490,7 +490,7 @@ struct<five:string,f1:double>
 -- !query
 SELECT sinh(double('1'))
 -- !query schema
-struct<SINH(CAST(1 AS DOUBLE)):double>
+struct<SINH(1):double>
 -- !query output
 1.1752011936438014
 
@@ -498,7 +498,7 @@ struct<SINH(CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cosh(double('1'))
 -- !query schema
-struct<COSH(CAST(1 AS DOUBLE)):double>
+struct<COSH(1):double>
 -- !query output
 1.543080634815244
 
@@ -506,7 +506,7 @@ struct<COSH(CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT tanh(double('1'))
 -- !query schema
-struct<TANH(CAST(1 AS DOUBLE)):double>
+struct<TANH(1):double>
 -- !query output
 0.7615941559557649
 
@@ -514,7 +514,7 @@ struct<TANH(CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT asinh(double('1'))
 -- !query schema
-struct<ASINH(CAST(1 AS DOUBLE)):double>
+struct<ASINH(1):double>
 -- !query output
 0.8813735870195429
 
@@ -522,7 +522,7 @@ struct<ASINH(CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT acosh(double('2'))
 -- !query schema
-struct<ACOSH(CAST(2 AS DOUBLE)):double>
+struct<ACOSH(2):double>
 -- !query output
 1.3169578969248166
 
@@ -530,7 +530,7 @@ struct<ACOSH(CAST(2 AS DOUBLE)):double>
 -- !query
 SELECT atanh(double('0.5'))
 -- !query schema
-struct<ATANH(CAST(0.5 AS DOUBLE)):double>
+struct<ATANH(0.5):double>
 -- !query output
 0.5493061443340548
 
@@ -538,7 +538,7 @@ struct<ATANH(CAST(0.5 AS DOUBLE)):double>
 -- !query
 SELECT sinh(double('Infinity'))
 -- !query schema
-struct<SINH(CAST(Infinity AS DOUBLE)):double>
+struct<SINH(Infinity):double>
 -- !query output
 Infinity
 
@@ -546,7 +546,7 @@ Infinity
 -- !query
 SELECT sinh(double('-Infinity'))
 -- !query schema
-struct<SINH(CAST(-Infinity AS DOUBLE)):double>
+struct<SINH(-Infinity):double>
 -- !query output
 -Infinity
 
@@ -554,7 +554,7 @@ struct<SINH(CAST(-Infinity AS DOUBLE)):double>
 -- !query
 SELECT sinh(double('NaN'))
 -- !query schema
-struct<SINH(CAST(NaN AS DOUBLE)):double>
+struct<SINH(NaN):double>
 -- !query output
 NaN
 
@@ -562,7 +562,7 @@ NaN
 -- !query
 SELECT cosh(double('Infinity'))
 -- !query schema
-struct<COSH(CAST(Infinity AS DOUBLE)):double>
+struct<COSH(Infinity):double>
 -- !query output
 Infinity
 
@@ -570,7 +570,7 @@ Infinity
 -- !query
 SELECT cosh(double('-Infinity'))
 -- !query schema
-struct<COSH(CAST(-Infinity AS DOUBLE)):double>
+struct<COSH(-Infinity):double>
 -- !query output
 Infinity
 
@@ -578,7 +578,7 @@ Infinity
 -- !query
 SELECT cosh(double('NaN'))
 -- !query schema
-struct<COSH(CAST(NaN AS DOUBLE)):double>
+struct<COSH(NaN):double>
 -- !query output
 NaN
 
@@ -586,7 +586,7 @@ NaN
 -- !query
 SELECT tanh(double('Infinity'))
 -- !query schema
-struct<TANH(CAST(Infinity AS DOUBLE)):double>
+struct<TANH(Infinity):double>
 -- !query output
 1.0
 
@@ -594,7 +594,7 @@ struct<TANH(CAST(Infinity AS DOUBLE)):double>
 -- !query
 SELECT tanh(double('-Infinity'))
 -- !query schema
-struct<TANH(CAST(-Infinity AS DOUBLE)):double>
+struct<TANH(-Infinity):double>
 -- !query output
 -1.0
 
@@ -602,7 +602,7 @@ struct<TANH(CAST(-Infinity AS DOUBLE)):double>
 -- !query
 SELECT tanh(double('NaN'))
 -- !query schema
-struct<TANH(CAST(NaN AS DOUBLE)):double>
+struct<TANH(NaN):double>
 -- !query output
 NaN
 
@@ -610,7 +610,7 @@ NaN
 -- !query
 SELECT asinh(double('Infinity'))
 -- !query schema
-struct<ASINH(CAST(Infinity AS DOUBLE)):double>
+struct<ASINH(Infinity):double>
 -- !query output
 Infinity
 
@@ -618,7 +618,7 @@ Infinity
 -- !query
 SELECT asinh(double('-Infinity'))
 -- !query schema
-struct<ASINH(CAST(-Infinity AS DOUBLE)):double>
+struct<ASINH(-Infinity):double>
 -- !query output
 -Infinity
 
@@ -626,7 +626,7 @@ struct<ASINH(CAST(-Infinity AS DOUBLE)):double>
 -- !query
 SELECT asinh(double('NaN'))
 -- !query schema
-struct<ASINH(CAST(NaN AS DOUBLE)):double>
+struct<ASINH(NaN):double>
 -- !query output
 NaN
 
@@ -634,7 +634,7 @@ NaN
 -- !query
 SELECT acosh(double('Infinity'))
 -- !query schema
-struct<ACOSH(CAST(Infinity AS DOUBLE)):double>
+struct<ACOSH(Infinity):double>
 -- !query output
 Infinity
 
@@ -642,7 +642,7 @@ Infinity
 -- !query
 SELECT acosh(double('-Infinity'))
 -- !query schema
-struct<ACOSH(CAST(-Infinity AS DOUBLE)):double>
+struct<ACOSH(-Infinity):double>
 -- !query output
 NaN
 
@@ -650,7 +650,7 @@ NaN
 -- !query
 SELECT acosh(double('NaN'))
 -- !query schema
-struct<ACOSH(CAST(NaN AS DOUBLE)):double>
+struct<ACOSH(NaN):double>
 -- !query output
 NaN
 
@@ -658,7 +658,7 @@ NaN
 -- !query
 SELECT atanh(double('Infinity'))
 -- !query schema
-struct<ATANH(CAST(Infinity AS DOUBLE)):double>
+struct<ATANH(Infinity):double>
 -- !query output
 NaN
 
@@ -666,7 +666,7 @@ NaN
 -- !query
 SELECT atanh(double('-Infinity'))
 -- !query schema
-struct<ATANH(CAST(-Infinity AS DOUBLE)):double>
+struct<ATANH(-Infinity):double>
 -- !query output
 NaN
 
@@ -674,7 +674,7 @@ NaN
 -- !query
 SELECT atanh(double('NaN'))
 -- !query schema
-struct<ATANH(CAST(NaN AS DOUBLE)):double>
+struct<ATANH(NaN):double>
 -- !query output
 NaN
 
@@ -742,7 +742,7 @@ struct<five:string,f1:double>
 -- !query
 SELECT smallint(double('32767.4'))
 -- !query schema
-struct<CAST(CAST(32767.4 AS DOUBLE) AS SMALLINT):smallint>
+struct<32767.4:smallint>
 -- !query output
 32767
 
@@ -750,7 +750,7 @@ struct<CAST(CAST(32767.4 AS DOUBLE) AS SMALLINT):smallint>
 -- !query
 SELECT smallint(double('32767.6'))
 -- !query schema
-struct<CAST(CAST(32767.6 AS DOUBLE) AS SMALLINT):smallint>
+struct<32767.6:smallint>
 -- !query output
 32767
 
@@ -758,7 +758,7 @@ struct<CAST(CAST(32767.6 AS DOUBLE) AS SMALLINT):smallint>
 -- !query
 SELECT smallint(double('-32768.4'))
 -- !query schema
-struct<CAST(CAST(-32768.4 AS DOUBLE) AS SMALLINT):smallint>
+struct<-32768.4:smallint>
 -- !query output
 -32768
 
@@ -766,7 +766,7 @@ struct<CAST(CAST(-32768.4 AS DOUBLE) AS SMALLINT):smallint>
 -- !query
 SELECT smallint(double('-32768.6'))
 -- !query schema
-struct<CAST(CAST(-32768.6 AS DOUBLE) AS SMALLINT):smallint>
+struct<-32768.6:smallint>
 -- !query output
 -32768
 
@@ -774,7 +774,7 @@ struct<CAST(CAST(-32768.6 AS DOUBLE) AS SMALLINT):smallint>
 -- !query
 SELECT int(double('2147483647.4'))
 -- !query schema
-struct<CAST(CAST(2147483647.4 AS DOUBLE) AS INT):int>
+struct<2147483647.4:int>
 -- !query output
 2147483647
 
@@ -782,7 +782,7 @@ struct<CAST(CAST(2147483647.4 AS DOUBLE) AS INT):int>
 -- !query
 SELECT int(double('2147483647.6'))
 -- !query schema
-struct<CAST(CAST(2147483647.6 AS DOUBLE) AS INT):int>
+struct<2147483647.6:int>
 -- !query output
 2147483647
 
@@ -790,7 +790,7 @@ struct<CAST(CAST(2147483647.6 AS DOUBLE) AS INT):int>
 -- !query
 SELECT int(double('-2147483648.4'))
 -- !query schema
-struct<CAST(CAST(-2147483648.4 AS DOUBLE) AS INT):int>
+struct<-2147483648.4:int>
 -- !query output
 -2147483648
 
@@ -798,7 +798,7 @@ struct<CAST(CAST(-2147483648.4 AS DOUBLE) AS INT):int>
 -- !query
 SELECT int(double('-2147483648.6'))
 -- !query schema
-struct<CAST(CAST(-2147483648.6 AS DOUBLE) AS INT):int>
+struct<-2147483648.6:int>
 -- !query output
 -2147483648
 
@@ -806,7 +806,7 @@ struct<CAST(CAST(-2147483648.6 AS DOUBLE) AS INT):int>
 -- !query
 SELECT bigint(double('9223372036854773760'))
 -- !query schema
-struct<CAST(CAST(9223372036854773760 AS DOUBLE) AS BIGINT):bigint>
+struct<9223372036854773760:bigint>
 -- !query output
 9223372036854773760
 
@@ -814,7 +814,7 @@ struct<CAST(CAST(9223372036854773760 AS DOUBLE) AS BIGINT):bigint>
 -- !query
 SELECT bigint(double('9223372036854775807'))
 -- !query schema
-struct<CAST(CAST(9223372036854775807 AS DOUBLE) AS BIGINT):bigint>
+struct<9223372036854775807:bigint>
 -- !query output
 9223372036854775807
 
@@ -822,7 +822,7 @@ struct<CAST(CAST(9223372036854775807 AS DOUBLE) AS BIGINT):bigint>
 -- !query
 SELECT bigint(double('-9223372036854775808.5'))
 -- !query schema
-struct<CAST(CAST(-9223372036854775808.5 AS DOUBLE) AS BIGINT):bigint>
+struct<-9223372036854775808.5:bigint>
 -- !query output
 -9223372036854775808
 
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/groupingsets.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/groupingsets.sql.out
index 7312c20876296..1aa34b29575c2 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/groupingsets.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/groupingsets.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 54
+-- Number of queries: 55
 
 
 -- !query
@@ -443,6 +443,25 @@ struct<ten:int,grouping(ten):tinyint>
 NULL	1
 
 
+-- !query
+select ten, sum(distinct four) filter (where string(four) like '123') from onek a
+group by rollup(ten)
+-- !query schema
+struct<ten:int,sum(DISTINCT four) FILTER (WHERE four LIKE 123):bigint>
+-- !query output
+0	NULL
+1	NULL
+2	NULL
+3	NULL
+4	NULL
+5	NULL
+6	NULL
+7	NULL
+8	NULL
+9	NULL
+NULL	NULL
+
+
 -- !query
 select count(*) from gstest4 group by rollup(unhashable_col,unsortable_col)
 -- !query schema
@@ -654,7 +673,7 @@ select v||'a', case grouping(v||'a') when 1 then 1 else 0 end, count(*)
   from values (1, 'a'), (1, 'b') u(i,v)
  group by rollup(i, v||'a') order by 1,3
 -- !query schema
-struct<concat(v, a):string,CASE WHEN (CAST(grouping(concat(v, a)) AS INT) = 1) THEN 1 ELSE 0 END:int,count(1):bigint>
+struct<concat(v, a):string,CASE WHEN (grouping(concat(v, a)) = 1) THEN 1 ELSE 0 END:int,count(1):bigint>
 -- !query output
 NULL	1	2
 NULL	1	2
@@ -667,7 +686,7 @@ select v||'a', case when grouping(v||'a') = 1 then 1 else 0 end, count(*)
   from values (1, 'a'), (1, 'b') u(i,v)
  group by rollup(i, v||'a') order by 1,3
 -- !query schema
-struct<concat(v, a):string,CASE WHEN (CAST(grouping(concat(v, a)) AS INT) = 1) THEN 1 ELSE 0 END:int,count(1):bigint>
+struct<concat(v, a):string,CASE WHEN (grouping(concat(v, a)) = 1) THEN 1 ELSE 0 END:int,count(1):bigint>
 -- !query output
 NULL	1	2
 NULL	1	2
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/int2.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/int2.sql.out
index 02e373f2d2b2b..dee21ced28d90 100755
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/int2.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/int2.sql.out
@@ -292,7 +292,7 @@ struct<five:string,f1:smallint,x:double>
 -- !query
 SELECT string(shiftleft(smallint(-1), 15))
 -- !query schema
-struct<CAST(shiftleft(CAST(CAST(-1 AS SMALLINT) AS INT), 15) AS STRING):string>
+struct<shiftleft(-1, 15):string>
 -- !query output
 -32768
 
@@ -300,7 +300,7 @@ struct<CAST(shiftleft(CAST(CAST(-1 AS SMALLINT) AS INT), 15) AS STRING):string>
 -- !query
 SELECT string(smallint(shiftleft(smallint(-1), 15))+1)
 -- !query schema
-struct<CAST((CAST(CAST(shiftleft(CAST(CAST(-1 AS SMALLINT) AS INT), 15) AS SMALLINT) AS INT) + 1) AS STRING):string>
+struct<(shiftleft(-1, 15) + 1):string>
 -- !query output
 -32767
 
@@ -308,7 +308,7 @@ struct<CAST((CAST(CAST(shiftleft(CAST(CAST(-1 AS SMALLINT) AS INT), 15) AS SMALL
 -- !query
 SELECT smallint(-32768) % smallint(-1)
 -- !query schema
-struct<(CAST(-32768 AS SMALLINT) % CAST(-1 AS SMALLINT)):smallint>
+struct<(-32768 % -1):smallint>
 -- !query output
 0
 
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/int4.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/int4.sql.out
index 3d80c5d595d53..2bb71645af0e5 100755
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/int4.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/int4.sql.out
@@ -433,7 +433,7 @@ struct<two:double>
 -- !query
 SELECT string(shiftleft(int(-1), 31))
 -- !query schema
-struct<CAST(shiftleft(CAST(-1 AS INT), 31) AS STRING):string>
+struct<shiftleft(-1, 31):string>
 -- !query output
 -2147483648
 
@@ -441,7 +441,7 @@ struct<CAST(shiftleft(CAST(-1 AS INT), 31) AS STRING):string>
 -- !query
 SELECT string(int(shiftleft(int(-1), 31))+1)
 -- !query schema
-struct<CAST((CAST(shiftleft(CAST(-1 AS INT), 31) AS INT) + 1) AS STRING):string>
+struct<(shiftleft(-1, 31) + 1):string>
 -- !query output
 -2147483647
 
@@ -449,7 +449,7 @@ struct<CAST((CAST(shiftleft(CAST(-1 AS INT), 31) AS INT) + 1) AS STRING):string>
 -- !query
 SELECT int(-2147483648) % int(-1)
 -- !query schema
-struct<(CAST(-2147483648 AS INT) % CAST(-1 AS INT)):int>
+struct<(-2147483648 % -1):int>
 -- !query output
 0
 
@@ -457,7 +457,7 @@ struct<(CAST(-2147483648 AS INT) % CAST(-1 AS INT)):int>
 -- !query
 SELECT int(-2147483648) % smallint(-1)
 -- !query schema
-struct<(CAST(-2147483648 AS INT) % CAST(CAST(-1 AS SMALLINT) AS INT)):int>
+struct<(-2147483648 % -1):int>
 -- !query output
 0
 
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/int8.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/int8.sql.out
index 18b0c821ae70f..116e0c090d148 100755
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/int8.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/int8.sql.out
@@ -569,25 +569,28 @@ struct<max(q1):bigint,max(q2):bigint>
 -- !query
 select bigint('9223372036854775800') / bigint('0')
 -- !query schema
-struct<(CAST(CAST(9223372036854775800 AS BIGINT) AS DOUBLE) / CAST(CAST(0 AS BIGINT) AS DOUBLE)):double>
+struct<>
 -- !query output
-NULL
+java.lang.ArithmeticException
+divide by zero
 
 
 -- !query
 select bigint('-9223372036854775808') / smallint('0')
 -- !query schema
-struct<(CAST(CAST(-9223372036854775808 AS BIGINT) AS DOUBLE) / CAST(CAST(0 AS SMALLINT) AS DOUBLE)):double>
+struct<>
 -- !query output
-NULL
+java.lang.ArithmeticException
+divide by zero
 
 
 -- !query
 select smallint('100') / bigint('0')
 -- !query schema
-struct<(CAST(CAST(100 AS SMALLINT) AS DOUBLE) / CAST(CAST(0 AS BIGINT) AS DOUBLE)):double>
+struct<>
 -- !query output
-NULL
+java.lang.ArithmeticException
+divide by zero
 
 
 -- !query
@@ -621,13 +624,13 @@ SELECT CAST(q1 AS smallint) FROM int8_tbl WHERE q2 <> 456
 struct<>
 -- !query output
 java.lang.ArithmeticException
-Casting 4567890123456789 to short causes overflow
+Casting 4567890123456789 to smallint causes overflow
 
 
 -- !query
 SELECT CAST(smallint('42') AS bigint), CAST(smallint('-37') AS bigint)
 -- !query schema
-struct<CAST(CAST(42 AS SMALLINT) AS BIGINT):bigint,CAST(CAST(-37 AS SMALLINT) AS BIGINT):bigint>
+struct<CAST(42 AS BIGINT):bigint,CAST(-37 AS BIGINT):bigint>
 -- !query output
 42	-37
 
@@ -647,7 +650,7 @@ struct<q1:float,q2:double>
 -- !query
 SELECT CAST(float('36854775807.0') AS bigint)
 -- !query schema
-struct<CAST(CAST(36854775807.0 AS FLOAT) AS BIGINT):bigint>
+struct<CAST(36854775807.0 AS BIGINT):bigint>
 -- !query output
 36854775808
 
@@ -695,8 +698,13 @@ SELECT * FROM range(bigint('+4567890123456789'), bigint('+4567890123456799'), 0)
 -- !query schema
 struct<>
 -- !query output
-java.lang.IllegalArgumentException
-requirement failed: step (0) cannot be 0
+org.apache.spark.sql.AnalysisException
+Table-valued function range with alternatives: 
+    range(start: long, end: long, step: long, numSlices: integer)
+    range(start: long, end: long, step: long)
+    range(start: long, end: long)
+    range(end: long)
+cannot be applied to (long, long, integer): requirement failed: step (0) cannot be 0; line 1 pos 14
 
 
 -- !query
@@ -714,7 +722,7 @@ struct<id:bigint>
 -- !query
 SELECT string(shiftleft(bigint(-1), 63))
 -- !query schema
-struct<CAST(shiftleft(CAST(-1 AS BIGINT), 63) AS STRING):string>
+struct<shiftleft(-1, 63):string>
 -- !query output
 -9223372036854775808
 
@@ -740,7 +748,7 @@ long overflow
 -- !query
 SELECT bigint((-9223372036854775808)) / bigint((-1))
 -- !query schema
-struct<(CAST(CAST(-9223372036854775808 AS BIGINT) AS DOUBLE) / CAST(CAST(-1 AS BIGINT) AS DOUBLE)):double>
+struct<(-9223372036854775808 / -1):double>
 -- !query output
 9.223372036854776E18
 
@@ -748,7 +756,7 @@ struct<(CAST(CAST(-9223372036854775808 AS BIGINT) AS DOUBLE) / CAST(CAST(-1 AS B
 -- !query
 SELECT bigint((-9223372036854775808)) % bigint((-1))
 -- !query schema
-struct<(CAST(-9223372036854775808 AS BIGINT) % CAST(-1 AS BIGINT)):bigint>
+struct<(-9223372036854775808 % -1):bigint>
 -- !query output
 0
 
@@ -765,7 +773,7 @@ long overflow
 -- !query
 SELECT bigint((-9223372036854775808)) / int((-1))
 -- !query schema
-struct<(CAST(CAST(-9223372036854775808 AS BIGINT) AS DOUBLE) / CAST(CAST(-1 AS INT) AS DOUBLE)):double>
+struct<(-9223372036854775808 / -1):double>
 -- !query output
 9.223372036854776E18
 
@@ -773,7 +781,7 @@ struct<(CAST(CAST(-9223372036854775808 AS BIGINT) AS DOUBLE) / CAST(CAST(-1 AS I
 -- !query
 SELECT bigint((-9223372036854775808)) % int((-1))
 -- !query schema
-struct<(CAST(-9223372036854775808 AS BIGINT) % CAST(CAST(-1 AS INT) AS BIGINT)):bigint>
+struct<(-9223372036854775808 % -1):bigint>
 -- !query output
 0
 
@@ -790,7 +798,7 @@ long overflow
 -- !query
 SELECT bigint((-9223372036854775808)) / smallint((-1))
 -- !query schema
-struct<(CAST(CAST(-9223372036854775808 AS BIGINT) AS DOUBLE) / CAST(CAST(-1 AS SMALLINT) AS DOUBLE)):double>
+struct<(-9223372036854775808 / -1):double>
 -- !query output
 9.223372036854776E18
 
@@ -798,7 +806,7 @@ struct<(CAST(CAST(-9223372036854775808 AS BIGINT) AS DOUBLE) / CAST(CAST(-1 AS S
 -- !query
 SELECT bigint((-9223372036854775808)) % smallint((-1))
 -- !query schema
-struct<(CAST(-9223372036854775808 AS BIGINT) % CAST(CAST(-1 AS SMALLINT) AS BIGINT)):bigint>
+struct<(-9223372036854775808 % -1):bigint>
 -- !query output
 0
 
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/join.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/join.sql.out
index 20f4f6b1f213f..4bc5b5e8df9fa 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/join.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/join.sql.out
@@ -2729,7 +2729,7 @@ struct<i:int,x:int,y:int>
 
 
 -- !query
-select t1.q2, count(t2.*)
+select t1.q2, count(t2.q1, t2.q2)
 from int8_tbl t1 left join int8_tbl t2 on (t1.q2 = t2.q1)
 group by t1.q2 order by 1
 -- !query schema
@@ -2742,7 +2742,7 @@ struct<q2:bigint,count(q1, q2):bigint>
 
 
 -- !query
-select t1.q2, count(t2.*)
+select t1.q2, count(t2.q1, t2.q2)
 from int8_tbl t1 left join (select * from int8_tbl) t2 on (t1.q2 = t2.q1)
 group by t1.q2 order by 1
 -- !query schema
@@ -2755,7 +2755,7 @@ struct<q2:bigint,count(q1, q2):bigint>
 
 
 -- !query
-select t1.q2, count(t2.*)
+select t1.q2, count(t2.q1, t2.q2)
 from int8_tbl t1 left join
   (select q1, case when q2=1 then 1 else q2 end as q2 from int8_tbl) t2
   on (t1.q2 = t2.q1)
@@ -3235,7 +3235,7 @@ select * from
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`y.f1`' given input columns: [j.f1, j.f1, x.q1, x.q2]; line 2 pos 63
+cannot resolve 'y.f1' given input columns: [j.f1, j.f1, x.q1, x.q2]; line 2 pos 63
 
 
 -- !query
@@ -3254,7 +3254,7 @@ select t1.uunique1 from
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`t1.uunique1`' given input columns: [t1.even, t2.even, t1.fivethous, t2.fivethous, t1.four, t2.four, t1.hundred, t2.hundred, t1.odd, t2.odd, t1.string4, t2.string4, t1.stringu1, t2.stringu1, t1.stringu2, t2.stringu2, t1.ten, t2.ten, t1.tenthous, t2.tenthous, t1.thousand, t2.thousand, t1.twenty, t2.twenty, t1.two, t2.two, t1.twothousand, t2.twothousand, t1.unique1, t2.unique1, t1.unique2, t2.unique2]; line 1 pos 7
+cannot resolve 't1.uunique1' given input columns: [t1.even, t2.even, t1.fivethous, t2.fivethous, t1.four, t2.four, t1.hundred, t2.hundred, t1.odd, t2.odd, t1.string4, t2.string4, t1.stringu1, t2.stringu1, t1.stringu2, t2.stringu2, t1.ten, t2.ten, t1.tenthous, t2.tenthous, t1.thousand, t2.thousand, t1.twenty, t2.twenty, t1.two, t2.two, t1.twothousand, t2.twothousand, t1.unique1, t2.unique1, t1.unique2, t2.unique2]; line 1 pos 7
 
 
 -- !query
@@ -3264,7 +3264,7 @@ select t2.uunique1 from
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`t2.uunique1`' given input columns: [t1.even, t2.even, t1.fivethous, t2.fivethous, t1.four, t2.four, t1.hundred, t2.hundred, t1.odd, t2.odd, t1.string4, t2.string4, t1.stringu1, t2.stringu1, t1.stringu2, t2.stringu2, t1.ten, t2.ten, t1.tenthous, t2.tenthous, t1.thousand, t2.thousand, t1.twenty, t2.twenty, t1.two, t2.two, t1.twothousand, t2.twothousand, t1.unique1, t2.unique1, t1.unique2, t2.unique2]; line 1 pos 7
+cannot resolve 't2.uunique1' given input columns: [t1.even, t2.even, t1.fivethous, t2.fivethous, t1.four, t2.four, t1.hundred, t2.hundred, t1.odd, t2.odd, t1.string4, t2.string4, t1.stringu1, t2.stringu1, t1.stringu2, t2.stringu2, t1.ten, t2.ten, t1.tenthous, t2.tenthous, t1.thousand, t2.thousand, t1.twenty, t2.twenty, t1.two, t2.two, t1.twothousand, t2.twothousand, t1.unique1, t2.unique1, t1.unique2, t2.unique2]; line 1 pos 7
 
 
 -- !query
@@ -3274,7 +3274,7 @@ select uunique1 from
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`uunique1`' given input columns: [t1.even, t2.even, t1.fivethous, t2.fivethous, t1.four, t2.four, t1.hundred, t2.hundred, t1.odd, t2.odd, t1.string4, t2.string4, t1.stringu1, t2.stringu1, t1.stringu2, t2.stringu2, t1.ten, t2.ten, t1.tenthous, t2.tenthous, t1.thousand, t2.thousand, t1.twenty, t2.twenty, t1.two, t2.two, t1.twothousand, t2.twothousand, t1.unique1, t2.unique1, t1.unique2, t2.unique2]; line 1 pos 7
+cannot resolve 'uunique1' given input columns: [t1.even, t2.even, t1.fivethous, t2.fivethous, t1.four, t2.four, t1.hundred, t2.hundred, t1.odd, t2.odd, t1.string4, t2.string4, t1.stringu1, t2.stringu1, t1.stringu2, t2.stringu2, t1.ten, t2.ten, t1.tenthous, t2.tenthous, t1.thousand, t2.thousand, t1.twenty, t2.twenty, t1.two, t2.two, t1.twothousand, t2.twothousand, t1.unique1, t2.unique1, t1.unique2, t2.unique2]; line 1 pos 7
 
 
 -- !query
@@ -3283,7 +3283,7 @@ select f1,g from int4_tbl a, (select f1 as g) ss
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`f1`' given input columns: []; line 1 pos 37
+cannot resolve 'f1' given input columns: []; line 1 pos 37
 
 
 -- !query
@@ -3292,7 +3292,7 @@ select f1,g from int4_tbl a, (select a.f1 as g) ss
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`a.f1`' given input columns: []; line 1 pos 37
+cannot resolve 'a.f1' given input columns: []; line 1 pos 37
 
 
 -- !query
@@ -3301,7 +3301,7 @@ select f1,g from int4_tbl a cross join (select f1 as g) ss
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`f1`' given input columns: []; line 1 pos 47
+cannot resolve 'f1' given input columns: []; line 1 pos 47
 
 
 -- !query
@@ -3310,7 +3310,7 @@ select f1,g from int4_tbl a cross join (select a.f1 as g) ss
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`a.f1`' given input columns: []; line 1 pos 47
+cannot resolve 'a.f1' given input columns: []; line 1 pos 47
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/limit.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/limit.sql.out
index 2c8bc31dbc6ca..2384010c67b4d 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/limit.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/limit.sql.out
@@ -59,7 +59,7 @@ select * from int8_tbl limit (case when random() < 0.5 then bigint(null) end)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-The limit expression must evaluate to a constant value, but got CASE WHEN (`_nondeterministic` < CAST(0.5BD AS DOUBLE)) THEN CAST(NULL AS BIGINT) END;
+The limit expression must evaluate to a constant value, but got CASE WHEN (_nondeterministic < CAST(0.5BD AS DOUBLE)) THEN CAST(NULL AS BIGINT) END
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/numeric.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/numeric.sql.out
index e59b9d5b63a40..8a4ee142011ce 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/numeric.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/numeric.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 577
+-- Number of queries: 592
 
 
 -- !query
@@ -3830,7 +3830,7 @@ INSERT INTO num_result SELECT t1.id, t2.id, t1.val, t2.val, t1.val * t2.val
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-`default`.`num_result` requires that the data to be inserted have the same number of columns as the target table: target table has 3 column(s) but the inserted data has 5 column(s), including 0 partition column(s) having constant value(s).;
+`default`.`num_result` requires that the data to be inserted have the same number of columns as the target table: target table has 3 column(s) but the inserted data has 5 column(s), including 0 partition column(s) having constant value(s).
 
 
 -- !query
@@ -4292,7 +4292,7 @@ struct<>
 -- !query
 SELECT decimal(double('NaN'))
 -- !query schema
-struct<CAST(CAST(NaN AS DOUBLE) AS DECIMAL(10,0)):decimal(10,0)>
+struct<NaN:decimal(10,0)>
 -- !query output
 NULL
 
@@ -4300,7 +4300,7 @@ NULL
 -- !query
 SELECT decimal(double('Infinity'))
 -- !query schema
-struct<CAST(CAST(Infinity AS DOUBLE) AS DECIMAL(10,0)):decimal(10,0)>
+struct<Infinity:decimal(10,0)>
 -- !query output
 NULL
 
@@ -4308,7 +4308,7 @@ NULL
 -- !query
 SELECT decimal(double('-Infinity'))
 -- !query schema
-struct<CAST(CAST(-Infinity AS DOUBLE) AS DECIMAL(10,0)):decimal(10,0)>
+struct<-Infinity:decimal(10,0)>
 -- !query output
 NULL
 
@@ -4316,7 +4316,7 @@ NULL
 -- !query
 SELECT decimal(float('NaN'))
 -- !query schema
-struct<CAST(CAST(NaN AS FLOAT) AS DECIMAL(10,0)):decimal(10,0)>
+struct<NaN:decimal(10,0)>
 -- !query output
 NULL
 
@@ -4324,7 +4324,7 @@ NULL
 -- !query
 SELECT decimal(float('Infinity'))
 -- !query schema
-struct<CAST(CAST(Infinity AS FLOAT) AS DECIMAL(10,0)):decimal(10,0)>
+struct<Infinity:decimal(10,0)>
 -- !query output
 NULL
 
@@ -4332,7 +4332,7 @@ NULL
 -- !query
 SELECT decimal(float('-Infinity'))
 -- !query schema
-struct<CAST(CAST(-Infinity AS FLOAT) AS DECIMAL(10,0)):decimal(10,0)>
+struct<-Infinity:decimal(10,0)>
 -- !query output
 NULL
 
@@ -4423,6 +4423,177 @@ struct<>
 
 
 
+-- !query
+SELECT width_bucket(5.0, 3.0, 4.0, 0)
+-- !query schema
+struct<width_bucket(5.0, 3.0, 4.0, 0):bigint>
+-- !query output
+NULL
+
+
+-- !query
+SELECT width_bucket(5.0, 3.0, 4.0, -5)
+-- !query schema
+struct<width_bucket(5.0, 3.0, 4.0, -5):bigint>
+-- !query output
+NULL
+
+
+-- !query
+SELECT width_bucket(3.5, 3.0, 3.0, 888)
+-- !query schema
+struct<width_bucket(3.5, 3.0, 3.0, 888):bigint>
+-- !query output
+NULL
+
+
+-- !query
+SELECT width_bucket(double(5.0), double(3.0), double(4.0), 0)
+-- !query schema
+struct<width_bucket(5.0, 3.0, 4.0, 0):bigint>
+-- !query output
+NULL
+
+
+-- !query
+SELECT width_bucket(double(5.0), double(3.0), double(4.0), -5)
+-- !query schema
+struct<width_bucket(5.0, 3.0, 4.0, -5):bigint>
+-- !query output
+NULL
+
+
+-- !query
+SELECT width_bucket(double(3.5), double(3.0), double(3.0), 888)
+-- !query schema
+struct<width_bucket(3.5, 3.0, 3.0, 888):bigint>
+-- !query output
+NULL
+
+
+-- !query
+SELECT width_bucket('NaN', 3.0, 4.0, 888)
+-- !query schema
+struct<width_bucket(NaN, 3.0, 4.0, 888):bigint>
+-- !query output
+NULL
+
+
+-- !query
+SELECT width_bucket(double(0), 'NaN', double(4.0), 888)
+-- !query schema
+struct<width_bucket(0, NaN, 4.0, 888):bigint>
+-- !query output
+NULL
+
+
+-- !query
+CREATE TABLE width_bucket_test (operand_num decimal(30,15), operand_f8 double) USING parquet
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+INSERT INTO width_bucket_test VALUES
+    (-5.2, -5.2),
+    (-0.0000000001, -0.0000000001),
+    (0.000000000001, 0.000000000001),
+    (1, 1),
+    (1.99999999999999, 1.99999999999999),
+    (2, 2),
+    (2.00000000000001, 2.00000000000001),
+    (3, 3),
+    (4, 4),
+    (4.5, 4.5),
+    (5, 5),
+    (5.5, 5.5),
+    (6, 6),
+    (7, 7),
+    (8, 8),
+    (9, 9),
+    (9.99999999999999, 9.99999999999999),
+    (10, 10),
+    (10.0000000000001, 10.0000000000001)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SELECT
+    operand_num,
+    width_bucket(operand_num, 0, 10, 5) AS wb_1,
+    width_bucket(operand_f8, 0, 10, 5) AS wb_1f,
+    width_bucket(operand_num, 10, 0, 5) AS wb_2,
+    width_bucket(operand_f8, 10, 0, 5) AS wb_2f,
+    width_bucket(operand_num, 2, 8, 4) AS wb_3,
+    width_bucket(operand_f8, 2, 8, 4) AS wb_3f,
+    width_bucket(operand_num, 5.0, 5.5, 20) AS wb_4,
+    width_bucket(operand_f8, 5.0, 5.5, 20) AS wb_4f,
+    width_bucket(operand_num, -25, 25, 10) AS wb_5,
+    width_bucket(operand_f8, -25, 25, 10) AS wb_5f
+    FROM width_bucket_test
+    ORDER BY operand_num ASC
+-- !query schema
+struct<operand_num:decimal(30,15),wb_1:bigint,wb_1f:bigint,wb_2:bigint,wb_2f:bigint,wb_3:bigint,wb_3f:bigint,wb_4:bigint,wb_4f:bigint,wb_5:bigint,wb_5f:bigint>
+-- !query output
+-5.200000000000000	0	0	6	6	0	0	0	0	4	4
+-0.000000000100000	0	0	6	6	0	0	0	0	5	5
+0.000000000001000	1	1	5	5	0	0	0	0	6	6
+1.000000000000000	1	1	5	5	0	0	0	0	6	6
+1.999999999999990	1	1	5	5	0	0	0	0	6	6
+2.000000000000000	2	2	5	5	1	1	0	0	6	6
+2.000000000000010	2	2	4	4	1	1	0	0	6	6
+3.000000000000000	2	2	4	4	1	1	0	0	6	6
+4.000000000000000	3	3	4	4	2	2	0	0	6	6
+4.500000000000000	3	3	3	3	2	2	0	0	6	6
+5.000000000000000	3	3	3	3	3	3	1	1	7	7
+5.500000000000000	3	3	3	3	3	3	21	21	7	7
+6.000000000000000	4	4	3	3	3	3	21	21	7	7
+7.000000000000000	4	4	2	2	4	4	21	21	7	7
+8.000000000000000	5	5	2	2	5	5	21	21	7	7
+9.000000000000000	5	5	1	1	5	5	21	21	7	7
+9.999999999999990	5	5	1	1	5	5	21	21	7	7
+10.000000000000000	6	6	1	1	5	5	21	21	8	8
+10.000000000000100	6	6	0	0	5	5	21	21	8	8
+
+
+-- !query
+SELECT width_bucket(double(0.0), double('Infinity'), 5, 10)
+-- !query schema
+struct<width_bucket(0.0, Infinity, 5, 10):bigint>
+-- !query output
+NULL
+
+
+-- !query
+SELECT width_bucket(double(0.0), 5, double('-Infinity'), 20)
+-- !query schema
+struct<width_bucket(0.0, 5, -Infinity, 20):bigint>
+-- !query output
+NULL
+
+
+-- !query
+SELECT width_bucket(double('Infinity'), 1, 10, 10),
+       width_bucket(double('-Infinity'), 1, 10, 10)
+-- !query schema
+struct<width_bucket(Infinity, 1, 10, 10):bigint,width_bucket(-Infinity, 1, 10, 10):bigint>
+-- !query output
+11	0
+
+
+-- !query
+DROP TABLE width_bucket_test
+-- !query schema
+struct<>
+-- !query output
+
+
+
 -- !query
 CREATE TABLE num_input_test (n1 decimal(38, 18)) USING parquet
 -- !query schema
@@ -4486,7 +4657,7 @@ struct<n1:decimal(38,18)>
 -- !query
 select cast(999999999999999999999 as decimal(38, 0))/1000000000000000000000
 -- !query schema
-struct<(CAST(CAST(999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) / CAST(1000000000000000000000 AS DECIMAL(38,0))):decimal(38,6)>
+struct<(CAST(999999999999999999999 AS DECIMAL(38,0)) / 1000000000000000000000):decimal(38,6)>
 -- !query output
 1.000000
 
@@ -4494,7 +4665,7 @@ struct<(CAST(CAST(999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) / CA
 -- !query
 select div(cast(999999999999999999999 as decimal(38, 0)),1000000000000000000000)
 -- !query schema
-struct<(CAST(CAST(999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) div CAST(1000000000000000000000 AS DECIMAL(38,0))):bigint>
+struct<(CAST(999999999999999999999 AS DECIMAL(38,0)) div 1000000000000000000000):bigint>
 -- !query output
 0
 
@@ -4502,7 +4673,7 @@ struct<(CAST(CAST(999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) div
 -- !query
 select mod(cast(999999999999999999999 as decimal(38, 0)),1000000000000000000000)
 -- !query schema
-struct<(CAST(CAST(999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) % CAST(1000000000000000000000 AS DECIMAL(38,0))):decimal(22,0)>
+struct<(CAST(999999999999999999999 AS DECIMAL(38,0)) % 1000000000000000000000):decimal(22,0)>
 -- !query output
 999999999999999999999
 
@@ -4510,7 +4681,7 @@ struct<(CAST(CAST(999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) % CA
 -- !query
 select div(cast(-9999999999999999999999 as decimal(38, 0)),1000000000000000000000)
 -- !query schema
-struct<(CAST(CAST(-9999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) div CAST(1000000000000000000000 AS DECIMAL(38,0))):bigint>
+struct<(CAST(-9999999999999999999999 AS DECIMAL(38,0)) div 1000000000000000000000):bigint>
 -- !query output
 -9
 
@@ -4518,7 +4689,7 @@ struct<(CAST(CAST(-9999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) di
 -- !query
 select mod(cast(-9999999999999999999999 as decimal(38, 0)),1000000000000000000000)
 -- !query schema
-struct<(CAST(CAST(-9999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) % CAST(1000000000000000000000 AS DECIMAL(38,0))):decimal(22,0)>
+struct<(CAST(-9999999999999999999999 AS DECIMAL(38,0)) % 1000000000000000000000):decimal(22,0)>
 -- !query output
 -999999999999999999999
 
@@ -4526,7 +4697,7 @@ struct<(CAST(CAST(-9999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) %
 -- !query
 select div(cast(-9999999999999999999999 as decimal(38, 0)),1000000000000000000000)*1000000000000000000000 + mod(cast(-9999999999999999999999 as decimal(38, 0)),1000000000000000000000)
 -- !query schema
-struct<(CAST((CAST(CAST((CAST(CAST(-9999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) div CAST(1000000000000000000000 AS DECIMAL(38,0))) AS DECIMAL(20,0)) AS DECIMAL(22,0)) * CAST(1000000000000000000000 AS DECIMAL(22,0))) AS DECIMAL(38,0)) + CAST((CAST(CAST(-9999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) % CAST(1000000000000000000000 AS DECIMAL(38,0))) AS DECIMAL(38,0))):decimal(38,0)>
+struct<(((CAST(-9999999999999999999999 AS DECIMAL(38,0)) div 1000000000000000000000) * 1000000000000000000000) + (CAST(-9999999999999999999999 AS DECIMAL(38,0)) % 1000000000000000000000)):decimal(38,0)>
 -- !query output
 -9999999999999999999999
 
@@ -4534,7 +4705,7 @@ struct<(CAST((CAST(CAST((CAST(CAST(-9999999999999999999999 AS DECIMAL(38,0)) AS
 -- !query
 select mod (70.0,70)
 -- !query schema
-struct<(CAST(70.0 AS DECIMAL(3,1)) % CAST(CAST(70 AS DECIMAL(2,0)) AS DECIMAL(3,1))):decimal(3,1)>
+struct<(70.0 % 70):decimal(3,1)>
 -- !query output
 0.0
 
@@ -4542,7 +4713,7 @@ struct<(CAST(70.0 AS DECIMAL(3,1)) % CAST(CAST(70 AS DECIMAL(2,0)) AS DECIMAL(3,
 -- !query
 select div (70.0,70)
 -- !query schema
-struct<(CAST(70.0 AS DECIMAL(3,1)) div CAST(CAST(70 AS DECIMAL(2,0)) AS DECIMAL(3,1))):bigint>
+struct<(70.0 div 70):bigint>
 -- !query output
 1
 
@@ -4550,7 +4721,7 @@ struct<(CAST(70.0 AS DECIMAL(3,1)) div CAST(CAST(70 AS DECIMAL(2,0)) AS DECIMAL(
 -- !query
 select 70.0 / 70
 -- !query schema
-struct<(CAST(70.0 AS DECIMAL(3,1)) / CAST(CAST(70 AS DECIMAL(2,0)) AS DECIMAL(3,1))):decimal(8,6)>
+struct<(70.0 / 70):decimal(8,6)>
 -- !query output
 1.000000
 
@@ -4558,7 +4729,7 @@ struct<(CAST(70.0 AS DECIMAL(3,1)) / CAST(CAST(70 AS DECIMAL(2,0)) AS DECIMAL(3,
 -- !query
 select 12345678901234567890 % 123
 -- !query schema
-struct<(CAST(12345678901234567890 AS DECIMAL(20,0)) % CAST(CAST(123 AS DECIMAL(3,0)) AS DECIMAL(20,0))):decimal(3,0)>
+struct<(12345678901234567890 % 123):decimal(3,0)>
 -- !query output
 78
 
@@ -4566,7 +4737,7 @@ struct<(CAST(12345678901234567890 AS DECIMAL(20,0)) % CAST(CAST(123 AS DECIMAL(3
 -- !query
 select exp(0.0)
 -- !query schema
-struct<EXP(CAST(0.0 AS DOUBLE)):double>
+struct<EXP(0.0):double>
 -- !query output
 1.0
 
@@ -4574,7 +4745,7 @@ struct<EXP(CAST(0.0 AS DOUBLE)):double>
 -- !query
 select exp(1.0)
 -- !query schema
-struct<EXP(CAST(1.0 AS DOUBLE)):double>
+struct<EXP(1.0):double>
 -- !query output
 2.7182818284590455
 
@@ -4582,7 +4753,7 @@ struct<EXP(CAST(1.0 AS DOUBLE)):double>
 -- !query
 select exp(32.999)
 -- !query schema
-struct<EXP(CAST(32.999 AS DOUBLE)):double>
+struct<EXP(32.999):double>
 -- !query output
 2.1442904349215556E14
 
@@ -4590,7 +4761,7 @@ struct<EXP(CAST(32.999 AS DOUBLE)):double>
 -- !query
 select exp(-32.999)
 -- !query schema
-struct<EXP(CAST(-32.999 AS DOUBLE)):double>
+struct<EXP(-32.999):double>
 -- !query output
 4.663547361468238E-15
 
@@ -4598,7 +4769,7 @@ struct<EXP(CAST(-32.999 AS DOUBLE)):double>
 -- !query
 select exp(123.456)
 -- !query schema
-struct<EXP(CAST(123.456 AS DOUBLE)):double>
+struct<EXP(123.456):double>
 -- !query output
 4.132944352778106E53
 
@@ -4606,7 +4777,7 @@ struct<EXP(CAST(123.456 AS DOUBLE)):double>
 -- !query
 select exp(-123.456)
 -- !query schema
-struct<EXP(CAST(-123.456 AS DOUBLE)):double>
+struct<EXP(-123.456):double>
 -- !query output
 2.4195825412645934E-54
 
@@ -4614,7 +4785,7 @@ struct<EXP(CAST(-123.456 AS DOUBLE)):double>
 -- !query
 select exp(1234.5678)
 -- !query schema
-struct<EXP(CAST(1234.5678 AS DOUBLE)):double>
+struct<EXP(1234.5678):double>
 -- !query output
 Infinity
 
@@ -4622,39 +4793,49 @@ Infinity
 -- !query
 select * from range(cast(0.0 as decimal(38, 18)), cast(4.0 as decimal(38, 18)))
 -- !query schema
-struct<id:bigint>
+struct<>
 -- !query output
-0
-1
-2
-3
+org.apache.spark.sql.AnalysisException
+Table-valued function range with alternatives: 
+    range(start: long, end: long, step: long, numSlices: integer)
+    range(start: long, end: long, step: long)
+    range(start: long, end: long)
+    range(end: long)
+cannot be applied to (decimal(38,18), decimal(38,18)): Incompatible input data type. Expected: long; Found: decimal(38,18); line 1 pos 14
 
 
 -- !query
 select * from range(cast(0.1 as decimal(38, 18)), cast(4.0 as decimal(38, 18)), cast(1.3 as decimal(38, 18)))
 -- !query schema
-struct<id:bigint>
+struct<>
 -- !query output
-0
-1
-2
-3
+org.apache.spark.sql.AnalysisException
+Table-valued function range with alternatives: 
+    range(start: long, end: long, step: long, numSlices: integer)
+    range(start: long, end: long, step: long)
+    range(start: long, end: long)
+    range(end: long)
+cannot be applied to (decimal(38,18), decimal(38,18), decimal(38,18)): Incompatible input data type. Expected: long; Found: decimal(38,18); line 1 pos 14
 
 
 -- !query
 select * from range(cast(4.0 as decimal(38, 18)), cast(-1.5 as decimal(38, 18)), cast(-2.2 as decimal(38, 18)))
 -- !query schema
-struct<id:bigint>
+struct<>
 -- !query output
-0
-2
-4
+org.apache.spark.sql.AnalysisException
+Table-valued function range with alternatives: 
+    range(start: long, end: long, step: long, numSlices: integer)
+    range(start: long, end: long, step: long)
+    range(start: long, end: long)
+    range(end: long)
+cannot be applied to (decimal(38,18), decimal(38,18), decimal(38,18)): Incompatible input data type. Expected: long; Found: decimal(38,18); line 1 pos 14
 
 
 -- !query
 select ln(1.2345678e-28)
 -- !query schema
-struct<LOG(1.2345678E-28):double>
+struct<ln(1.2345678E-28):double>
 -- !query output
 -64.26166165451762
 
@@ -4662,7 +4843,7 @@ struct<LOG(1.2345678E-28):double>
 -- !query
 select ln(0.0456789)
 -- !query schema
-struct<LOG(CAST(0.0456789 AS DOUBLE)):double>
+struct<ln(0.0456789):double>
 -- !query output
 -3.0861187944847437
 
@@ -4670,7 +4851,7 @@ struct<LOG(CAST(0.0456789 AS DOUBLE)):double>
 -- !query
 select ln(0.99949452)
 -- !query schema
-struct<LOG(CAST(0.99949452 AS DOUBLE)):double>
+struct<ln(0.99949452):double>
 -- !query output
 -5.056077980832118E-4
 
@@ -4678,7 +4859,7 @@ struct<LOG(CAST(0.99949452 AS DOUBLE)):double>
 -- !query
 select ln(1.00049687395)
 -- !query schema
-struct<LOG(CAST(1.00049687395 AS DOUBLE)):double>
+struct<ln(1.00049687395):double>
 -- !query output
 4.967505490136803E-4
 
@@ -4686,7 +4867,7 @@ struct<LOG(CAST(1.00049687395 AS DOUBLE)):double>
 -- !query
 select ln(1234.567890123456789)
 -- !query schema
-struct<LOG(CAST(1234.567890123456789 AS DOUBLE)):double>
+struct<ln(1234.567890123456789):double>
 -- !query output
 7.11847630129779
 
@@ -4694,7 +4875,7 @@ struct<LOG(CAST(1234.567890123456789 AS DOUBLE)):double>
 -- !query
 select ln(5.80397490724e5)
 -- !query schema
-struct<LOG(580397.490724):double>
+struct<ln(580397.490724):double>
 -- !query output
 13.271468476626518
 
@@ -4702,7 +4883,7 @@ struct<LOG(580397.490724):double>
 -- !query
 select ln(9.342536355e34)
 -- !query schema
-struct<LOG(9.342536355E34):double>
+struct<ln(9.342536355E34):double>
 -- !query output
 80.52247093552418
 
@@ -4710,7 +4891,7 @@ struct<LOG(9.342536355E34):double>
 -- !query
 select log(3.4634998359873254962349856073435545)
 -- !query schema
-struct<LOG(E(), CAST(3.4634998359873254962349856073435545 AS DOUBLE)):double>
+struct<LOG(E(), 3.4634998359873254962349856073435545):double>
 -- !query output
 1.2422795911259166
 
@@ -4718,7 +4899,7 @@ struct<LOG(E(), CAST(3.4634998359873254962349856073435545 AS DOUBLE)):double>
 -- !query
 select log(9.999999999999999999)
 -- !query schema
-struct<LOG(E(), CAST(9.999999999999999999 AS DOUBLE)):double>
+struct<LOG(E(), 9.999999999999999999):double>
 -- !query output
 2.302585092994046
 
@@ -4726,7 +4907,7 @@ struct<LOG(E(), CAST(9.999999999999999999 AS DOUBLE)):double>
 -- !query
 select log(10.00000000000000000)
 -- !query schema
-struct<LOG(E(), CAST(10.00000000000000000 AS DOUBLE)):double>
+struct<LOG(E(), 10.00000000000000000):double>
 -- !query output
 2.302585092994046
 
@@ -4734,7 +4915,7 @@ struct<LOG(E(), CAST(10.00000000000000000 AS DOUBLE)):double>
 -- !query
 select log(10.00000000000000001)
 -- !query schema
-struct<LOG(E(), CAST(10.00000000000000001 AS DOUBLE)):double>
+struct<LOG(E(), 10.00000000000000001):double>
 -- !query output
 2.302585092994046
 
@@ -4742,7 +4923,7 @@ struct<LOG(E(), CAST(10.00000000000000001 AS DOUBLE)):double>
 -- !query
 select log(590489.45235237)
 -- !query schema
-struct<LOG(E(), CAST(590489.45235237 AS DOUBLE)):double>
+struct<LOG(E(), 590489.45235237):double>
 -- !query output
 13.288707052228641
 
@@ -4750,7 +4931,7 @@ struct<LOG(E(), CAST(590489.45235237 AS DOUBLE)):double>
 -- !query
 select log(0.99923, 4.58934e34)
 -- !query schema
-struct<LOG(CAST(0.99923 AS DOUBLE), 4.58934E34):double>
+struct<LOG(0.99923, 4.58934E34):double>
 -- !query output
 -103611.55579543479
 
@@ -4758,7 +4939,7 @@ struct<LOG(CAST(0.99923 AS DOUBLE), 4.58934E34):double>
 -- !query
 select log(1.000016, 8.452010e18)
 -- !query schema
-struct<LOG(CAST(1.000016 AS DOUBLE), 8.45201E18):double>
+struct<LOG(1.000016, 8.45201E18):double>
 -- !query output
 2723830.287707013
 
@@ -4766,7 +4947,7 @@ struct<LOG(CAST(1.000016 AS DOUBLE), 8.45201E18):double>
 -- !query
 SELECT SUM(decimal(9999)) FROM range(1, 100001)
 -- !query schema
-struct<sum(CAST(9999 AS DECIMAL(10,0))):decimal(20,0)>
+struct<sum(9999):decimal(20,0)>
 -- !query output
 999900000
 
@@ -4774,7 +4955,7 @@ struct<sum(CAST(9999 AS DECIMAL(10,0))):decimal(20,0)>
 -- !query
 SELECT SUM(decimal(-9999)) FROM range(1, 100001)
 -- !query schema
-struct<sum(CAST(-9999 AS DECIMAL(10,0))):decimal(20,0)>
+struct<sum(-9999):decimal(20,0)>
 -- !query output
 -999900000
 
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/select_having.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/select_having.sql.out
index d8d33d92a7cc4..d001e22e9b766 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/select_having.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/select_having.sql.out
@@ -143,7 +143,7 @@ SELECT a FROM test_having HAVING min(a) < max(a)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-grouping expressions sequence is empty, and 'spark_catalog.default.test_having.`a`' is not an aggregate function. Wrap '(min(spark_catalog.default.test_having.`a`) AS `min(a#x)`, max(spark_catalog.default.test_having.`a`) AS `max(a#x)`)' in windowing function(s) or wrap 'spark_catalog.default.test_having.`a`' in first() (or first_value) if you don't care which value you get.;
+grouping expressions sequence is empty, and 'spark_catalog.default.test_having.a' is not an aggregate function. Wrap '(min(spark_catalog.default.test_having.a) AS `min(a#x)`, max(spark_catalog.default.test_having.a) AS `max(a#x)`)' in windowing function(s) or wrap 'spark_catalog.default.test_having.a' in first() (or first_value) if you don't care which value you get.
 
 
 -- !query
@@ -152,7 +152,7 @@ SELECT 1 AS one FROM test_having HAVING a > 1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`a`' given input columns: [one]; line 1 pos 40
+cannot resolve 'a' given input columns: [one]; line 1 pos 40
 
 
 -- !query
@@ -174,9 +174,10 @@ struct<one:int>
 -- !query
 SELECT 1 AS one FROM test_having WHERE 1/a = 1 HAVING 1 < 2
 -- !query schema
-struct<one:int>
+struct<>
 -- !query output
-1
+java.lang.ArithmeticException
+divide by zero
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/select_implicit.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/select_implicit.sql.out
index 4ecfabccdf414..4a304aeb606bb 100755
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/select_implicit.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/select_implicit.sql.out
@@ -122,7 +122,7 @@ SELECT count(*) FROM test_missing_target GROUP BY a ORDER BY b
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`b`' given input columns: [count(1)]; line 1 pos 61
+cannot resolve 'b' given input columns: [count(1)]; line 1 pos 61
 
 
 -- !query
@@ -239,7 +239,7 @@ struct<a:int,a:int>
 SELECT a/2, a/2 FROM test_missing_target
 	ORDER BY a/2
 -- !query schema
-struct<(CAST(a AS DOUBLE) / CAST(2 AS DOUBLE)):double,(CAST(a AS DOUBLE) / CAST(2 AS DOUBLE)):double>
+struct<(a / 2):double,(a / 2):double>
 -- !query output
 0.0	0.0
 0.5	0.5
@@ -257,7 +257,7 @@ struct<(CAST(a AS DOUBLE) / CAST(2 AS DOUBLE)):double,(CAST(a AS DOUBLE) / CAST(
 SELECT a/2, a/2 FROM test_missing_target
 	GROUP BY a/2 ORDER BY a/2
 -- !query schema
-struct<(CAST(a AS DOUBLE) / CAST(2 AS DOUBLE)):double,(CAST(a AS DOUBLE) / CAST(2 AS DOUBLE)):double>
+struct<(a / 2):double,(a / 2):double>
 -- !query output
 0.0	0.0
 0.5	0.5
@@ -327,7 +327,7 @@ SELECT count(a) FROM test_missing_target GROUP BY a ORDER BY b
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`b`' given input columns: [count(a)]; line 1 pos 61
+cannot resolve 'b' given input columns: [count(a)]; line 1 pos 61
 
 
 -- !query
@@ -398,7 +398,7 @@ SELECT x.b/2, count(x.b) FROM test_missing_target x, test_missing_target y
 	WHERE x.a = y.a
 	GROUP BY x.b/2 ORDER BY x.b/2
 -- !query schema
-struct<(CAST(b AS DOUBLE) / CAST(2 AS DOUBLE)):double,count(b):bigint>
+struct<(b / 2):double,count(b):bigint>
 -- !query output
 0.5	1
 1.0	2
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/strings.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/strings.sql.out
index e8a3a9b9731a6..253a5e49b81fa 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/strings.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/strings.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 121
+-- Number of queries: 124
 
 
 -- !query
@@ -27,7 +27,7 @@ first line - next line - third line
 -- !query
 SELECT binary('\\xDeAdBeEf')
 -- !query schema
-struct<CAST(\xDeAdBeEf AS BINARY):binary>
+struct<\xDeAdBeEf:binary>
 -- !query output
 \xDeAdBeEf
 
@@ -35,7 +35,7 @@ struct<CAST(\xDeAdBeEf AS BINARY):binary>
 -- !query
 SELECT binary('\\x De Ad Be Ef ')
 -- !query schema
-struct<CAST(\x De Ad Be Ef  AS BINARY):binary>
+struct<\x De Ad Be Ef :binary>
 -- !query output
 \x De Ad Be Ef
 
@@ -43,7 +43,7 @@ struct<CAST(\x De Ad Be Ef  AS BINARY):binary>
 -- !query
 SELECT binary('\\xDe00BeEf')
 -- !query schema
-struct<CAST(\xDe00BeEf AS BINARY):binary>
+struct<\xDe00BeEf:binary>
 -- !query output
 \xDe00BeEf
 
@@ -51,7 +51,7 @@ struct<CAST(\xDe00BeEf AS BINARY):binary>
 -- !query
 SELECT binary('DeAdBeEf')
 -- !query schema
-struct<CAST(DeAdBeEf AS BINARY):binary>
+struct<DeAdBeEf:binary>
 -- !query output
 DeAdBeEf
 
@@ -59,7 +59,7 @@ DeAdBeEf
 -- !query
 SELECT binary('De\\000dBeEf')
 -- !query schema
-struct<CAST(De\000dBeEf AS BINARY):binary>
+struct<De\000dBeEf:binary>
 -- !query output
 De\000dBeEf
 
@@ -67,7 +67,7 @@ De\000dBeEf
 -- !query
 SELECT binary('De\\123dBeEf')
 -- !query schema
-struct<CAST(De\123dBeEf AS BINARY):binary>
+struct<De\123dBeEf:binary>
 -- !query output
 De\123dBeEf
 
@@ -446,7 +446,7 @@ SELECT 'maca' LIKE 'm%aca' ESCAPE '%' AS `true`
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-the pattern 'm%aca' is invalid, the escape character is not allowed to precede 'a';
+the pattern 'm%aca' is invalid, the escape character is not allowed to precede 'a'
 
 
 -- !query
@@ -455,7 +455,7 @@ SELECT 'maca' NOT LIKE 'm%aca' ESCAPE '%' AS `false`
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-the pattern 'm%aca' is invalid, the escape character is not allowed to precede 'a';
+the pattern 'm%aca' is invalid, the escape character is not allowed to precede 'a'
 
 
 -- !query
@@ -464,7 +464,7 @@ SELECT 'ma%a' LIKE 'm%a%%a' ESCAPE '%' AS `true`
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-the pattern 'm%a%%a' is invalid, the escape character is not allowed to precede 'a';
+the pattern 'm%a%%a' is invalid, the escape character is not allowed to precede 'a'
 
 
 -- !query
@@ -473,7 +473,7 @@ SELECT 'ma%a' NOT LIKE 'm%a%%a' ESCAPE '%' AS `false`
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-the pattern 'm%a%%a' is invalid, the escape character is not allowed to precede 'a';
+the pattern 'm%a%%a' is invalid, the escape character is not allowed to precede 'a'
 
 
 -- !query
@@ -482,7 +482,7 @@ SELECT 'bear' LIKE 'b_ear' ESCAPE '_' AS `true`
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-the pattern 'b_ear' is invalid, the escape character is not allowed to precede 'e';
+the pattern 'b_ear' is invalid, the escape character is not allowed to precede 'e'
 
 
 -- !query
@@ -491,7 +491,7 @@ SELECT 'bear' NOT LIKE 'b_ear' ESCAPE '_' AS `false`
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-the pattern 'b_ear' is invalid, the escape character is not allowed to precede 'e';
+the pattern 'b_ear' is invalid, the escape character is not allowed to precede 'e'
 
 
 -- !query
@@ -500,7 +500,7 @@ SELECT 'be_r' LIKE 'b_e__r' ESCAPE '_' AS `true`
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-the pattern 'b_e__r' is invalid, the escape character is not allowed to precede 'e';
+the pattern 'b_e__r' is invalid, the escape character is not allowed to precede 'e'
 
 
 -- !query
@@ -509,7 +509,7 @@ SELECT 'be_r' NOT LIKE 'b_e__r' ESCAPE '_' AS `false`
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-the pattern 'b_e__r' is invalid, the escape character is not allowed to precede 'e';
+the pattern 'b_e__r' is invalid, the escape character is not allowed to precede 'e'
 
 
 -- !query
@@ -945,7 +945,7 @@ struct<ascii():int>
 -- !query
 SELECT chr(65)
 -- !query schema
-struct<chr(CAST(65 AS BIGINT)):string>
+struct<chr(65):string>
 -- !query output
 A
 
@@ -953,7 +953,7 @@ A
 -- !query
 SELECT chr(0)
 -- !query schema
-struct<chr(CAST(0 AS BIGINT)):string>
+struct<chr(0):string>
 -- !query output
 
 
@@ -977,9 +977,37 @@ struct<repeat(Pg, -4):string>
 -- !query
 SELECT trim(binary('\\000') from binary('\\000Tom\\000'))
 -- !query schema
-struct<TRIM(BOTH CAST(CAST(\000 AS BINARY) AS STRING) FROM CAST(CAST(\000Tom\000 AS BINARY) AS STRING)):string>
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'TRIM(BOTH CAST('\\000' AS BINARY) FROM CAST('\\000Tom\\000' AS BINARY))' due to data type mismatch: argument 1 requires string type, however, 'CAST('\\000Tom\\000' AS BINARY)' is of binary type. argument 2 requires string type, however, 'CAST('\\000' AS BINARY)' is of binary type.; line 1 pos 7
+
+
+-- !query
+SELECT btrim(binary('\\000trim\\000'), binary('\\000'))
+-- !query schema
+struct<>
 -- !query output
-Tom
+org.apache.spark.sql.AnalysisException
+cannot resolve 'TRIM(BOTH CAST('\\000' AS BINARY) FROM CAST('\\000trim\\000' AS BINARY))' due to data type mismatch: argument 1 requires string type, however, 'CAST('\\000trim\\000' AS BINARY)' is of binary type. argument 2 requires string type, however, 'CAST('\\000' AS BINARY)' is of binary type.; line 1 pos 7
+
+
+-- !query
+SELECT btrim(binary(''), binary('\\000'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'TRIM(BOTH CAST('\\000' AS BINARY) FROM CAST('' AS BINARY))' due to data type mismatch: argument 1 requires string type, however, 'CAST('' AS BINARY)' is of binary type. argument 2 requires string type, however, 'CAST('\\000' AS BINARY)' is of binary type.; line 1 pos 7
+
+
+-- !query
+SELECT btrim(binary('\\000trim\\000'), binary(''))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'TRIM(BOTH CAST('' AS BINARY) FROM CAST('\\000trim\\000' AS BINARY))' due to data type mismatch: argument 1 requires string type, however, 'CAST('\\000trim\\000' AS BINARY)' is of binary type. argument 2 requires string type, however, 'CAST('' AS BINARY)' is of binary type.; line 1 pos 7
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/text.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/text.sql.out
index 811e7d6e4ca65..2387dd244181b 100755
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/text.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/text.sql.out
@@ -54,9 +54,10 @@ struct<two:string,f1:string>
 -- !query
 select length(42)
 -- !query schema
-struct<length(CAST(42 AS STRING)):int>
+struct<>
 -- !query output
-2
+org.apache.spark.sql.AnalysisException
+cannot resolve 'length(42)' due to data type mismatch: argument 1 requires (string or binary) type, however, '42' is of int type.; line 1 pos 7
 
 
 -- !query
@@ -64,8 +65,8 @@ select string('four: ') || 2+2
 -- !query schema
 struct<>
 -- !query output
-java.lang.NumberFormatException
-invalid input syntax for type numeric: four: 2
+org.apache.spark.sql.AnalysisException
+cannot resolve 'concat(CAST('four: ' AS STRING), 2)' due to data type mismatch: input to function concat should have been string, binary or array, but it's [string, int]; line 1 pos 7
 
 
 -- !query
@@ -73,16 +74,17 @@ select 'four: ' || 2+2
 -- !query schema
 struct<>
 -- !query output
-java.lang.NumberFormatException
-invalid input syntax for type numeric: four: 2
+org.apache.spark.sql.AnalysisException
+cannot resolve 'concat('four: ', 2)' due to data type mismatch: input to function concat should have been string, binary or array, but it's [string, int]; line 1 pos 7
 
 
 -- !query
 select 3 || 4.0
 -- !query schema
-struct<concat(CAST(3 AS STRING), CAST(4.0 AS STRING)):string>
+struct<>
 -- !query output
-34.0
+org.apache.spark.sql.AnalysisException
+cannot resolve 'concat(3, 4.0BD)' due to data type mismatch: input to function concat should have been string, binary or array, but it's [int, decimal(2,1)]; line 1 pos 7
 
 
 -- !query
@@ -99,9 +101,10 @@ one
 -- !query
 select concat(1,2,3,'hello',true, false, to_date('20100309','yyyyMMdd'))
 -- !query schema
-struct<concat(CAST(1 AS STRING), CAST(2 AS STRING), CAST(3 AS STRING), hello, CAST(true AS STRING), CAST(false AS STRING), CAST(to_date(20100309, yyyyMMdd) AS STRING)):string>
+struct<>
 -- !query output
-123hellotruefalse2010-03-09
+org.apache.spark.sql.AnalysisException
+cannot resolve 'concat(1, 2, 3, 'hello', true, false, to_date('20100309', 'yyyyMMdd'))' due to data type mismatch: input to function concat should have been string, binary or array, but it's [int, int, int, string, boolean, boolean, date]; line 1 pos 7
 
 
 -- !query
@@ -115,33 +118,37 @@ one
 -- !query
 select concat_ws('#',1,2,3,'hello',true, false, to_date('20100309','yyyyMMdd'))
 -- !query schema
-struct<concat_ws(#, CAST(1 AS STRING), CAST(2 AS STRING), CAST(3 AS STRING), hello, CAST(true AS STRING), CAST(false AS STRING), CAST(to_date(20100309, yyyyMMdd) AS STRING)):string>
+struct<>
 -- !query output
-1#x#x#hello#true#false#x-03-09
+org.apache.spark.sql.AnalysisException
+cannot resolve 'concat_ws('#', 1, 2, 3, 'hello', true, false, to_date('20100309', 'yyyyMMdd'))' due to data type mismatch: argument 2 requires (array<string> or string) type, however, '1' is of int type. argument 3 requires (array<string> or string) type, however, '2' is of int type. argument 4 requires (array<string> or string) type, however, '3' is of int type. argument 6 requires (array<string> or string) type, however, 'true' is of boolean type. argument 7 requires (array<string> or string) type, however, 'false' is of boolean type. argument 8 requires (array<string> or string) type, however, 'to_date('20100309', 'yyyyMMdd')' is of date type.; line 1 pos 7
 
 
 -- !query
 select concat_ws(',',10,20,null,30)
 -- !query schema
-struct<concat_ws(,, CAST(10 AS STRING), CAST(20 AS STRING), NULL, CAST(30 AS STRING)):string>
+struct<>
 -- !query output
-10,20,30
+org.apache.spark.sql.AnalysisException
+cannot resolve 'concat_ws(',', 10, 20, NULL, 30)' due to data type mismatch: argument 2 requires (array<string> or string) type, however, '10' is of int type. argument 3 requires (array<string> or string) type, however, '20' is of int type. argument 4 requires (array<string> or string) type, however, 'NULL' is of null type. argument 5 requires (array<string> or string) type, however, '30' is of int type.; line 1 pos 7
 
 
 -- !query
 select concat_ws('',10,20,null,30)
 -- !query schema
-struct<concat_ws(, CAST(10 AS STRING), CAST(20 AS STRING), NULL, CAST(30 AS STRING)):string>
+struct<>
 -- !query output
-102030
+org.apache.spark.sql.AnalysisException
+cannot resolve 'concat_ws('', 10, 20, NULL, 30)' due to data type mismatch: argument 2 requires (array<string> or string) type, however, '10' is of int type. argument 3 requires (array<string> or string) type, however, '20' is of int type. argument 4 requires (array<string> or string) type, however, 'NULL' is of null type. argument 5 requires (array<string> or string) type, however, '30' is of int type.; line 1 pos 7
 
 
 -- !query
 select concat_ws(NULL,10,20,null,30) is null
 -- !query schema
-struct<(concat_ws(CAST(NULL AS STRING), CAST(10 AS STRING), CAST(20 AS STRING), NULL, CAST(30 AS STRING)) IS NULL):boolean>
+struct<>
 -- !query output
-true
+org.apache.spark.sql.AnalysisException
+cannot resolve 'concat_ws(CAST(NULL AS STRING), 10, 20, NULL, 30)' due to data type mismatch: argument 2 requires (array<string> or string) type, however, '10' is of int type. argument 3 requires (array<string> or string) type, however, '20' is of int type. argument 4 requires (array<string> or string) type, however, 'NULL' is of null type. argument 5 requires (array<string> or string) type, however, '30' is of int type.; line 1 pos 7
 
 
 -- !query
@@ -155,19 +162,10 @@ edcba
 -- !query
 select i, left('ahoj', i), right('ahoj', i) from range(-5, 6) t(i) order by i
 -- !query schema
-struct<i:bigint,left(ahoj, i):string,right(ahoj, i):string>
+struct<>
 -- !query output
--5		
--4		
--3		
--2		
--1		
-0		
-1	a	j
-2	ah	oj
-3	aho	hoj
-4	ahoj	ahoj
-5	ahoj	ahoj
+org.apache.spark.sql.AnalysisException
+cannot resolve 'substring('ahoj', 1, t.i)' due to data type mismatch: argument 3 requires int type, however, 't.i' is of bigint type.; line 1 pos 10
 
 
 -- !query
@@ -176,7 +174,7 @@ struct<i:bigint,left(ahoj, i):string,right(ahoj, i):string>
  */
 select format_string(NULL)
 -- !query schema
-struct<format_string(CAST(NULL AS STRING)):string>
+struct<format_string(NULL):string>
 -- !query output
 NULL
 
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/timestamp.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/timestamp.sql.out
index 5b0b636ff0c29..68d2b5c9ce1bf 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/timestamp.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/timestamp.sql.out
@@ -256,19 +256,16 @@ SELECT '' AS `54`, d1 as `timestamp`,
     date_part( 'minute', d1) AS `minute`, date_part( 'second', d1) AS `second`
     FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01'
 -- !query schema
-struct<54:string,timestamp:timestamp,year:int,month:int,day:int,hour:int,minute:int,second:decimal(8,6)>
+struct<>
 -- !query output
-	1969-12-31 16:00:00	1969	12	31	16	0	0.000000
-	1997-01-02 00:00:00	1997	1	2	0	0	0.000000
-	1997-01-02 03:04:05	1997	1	2	3	4	5.000000
-	1997-02-10 17:32:01	1997	2	10	17	32	1.000000
-	2001-09-22 18:19:20	2001	9	22	18	19	20.000000
+org.apache.spark.sql.AnalysisException
+cannot resolve 'year(spark_catalog.default.timestamp_tbl.d1)' due to data type mismatch: argument 1 requires date type, however, 'spark_catalog.default.timestamp_tbl.d1' is of timestamp type.; line 2 pos 4
 
 
 -- !query
 SELECT make_timestamp(2014,12,28,6,30,45.887)
 -- !query schema
-struct<make_timestamp(2014, 12, 28, 6, 30, CAST(45.887 AS DECIMAL(8,6))):timestamp>
+struct<make_timestamp(2014, 12, 28, 6, 30, 45.887):timestamp>
 -- !query output
 2014-12-28 06:30:45.887
 
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/union.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/union.sql.out
index 2fe53055cf656..5f7a82b5c80ce 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/union.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/union.sql.out
@@ -526,7 +526,7 @@ SELECT q1 FROM int8_tbl EXCEPT SELECT q2 FROM int8_tbl ORDER BY q2 LIMIT 1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`q2`' given input columns: [int8_tbl.q1]; line 1 pos 64
+cannot resolve 'q2' given input columns: [int8_tbl.q1]; line 1 pos 64
 
 
 -- !query
@@ -682,10 +682,10 @@ struct<id:bigint>
 -- !query
 SELECT cast('3.4' as decimal(38, 18)) UNION SELECT 'foo'
 -- !query schema
-struct<CAST(3.4 AS DECIMAL(38,18)):string>
+struct<>
 -- !query output
-3.400000000000000000
-foo
+org.apache.spark.SparkException
+Failed to merge incompatible data types decimal(38,18) and string
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part1.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part1.sql.out
index 2b1de87a6be5e..ba449ccf393e4 100755
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part1.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part1.sql.out
@@ -14,7 +14,7 @@ struct<>
 SELECT four, ten, SUM(SUM(four)) OVER (PARTITION BY four), AVG(ten) FROM tenk1
 GROUP BY four, ten ORDER BY four, ten
 -- !query schema
-struct<four:int,ten:int,sum(sum(CAST(four AS BIGINT))) OVER (PARTITION BY four ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):bigint,avg(ten):double>
+struct<four:int,ten:int,sum(sum(four)) OVER (PARTITION BY four ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):bigint,avg(ten):double>
 -- !query output
 0	0	0	0.0
 0	2	0	2.0
@@ -270,7 +270,7 @@ struct<lead((ten * 2), 1, -1) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIR
 -- !query
 SELECT first(ten) OVER (PARTITION BY four ORDER BY ten), ten, four FROM tenk1 WHERE unique2 < 10
 -- !query schema
-struct<first(ten, false) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int,ten:int,four:int>
+struct<first(ten) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int,ten:int,four:int>
 -- !query output
 0	0	0
 0	0	0
@@ -287,7 +287,7 @@ struct<first(ten, false) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RA
 -- !query
 SELECT last(four) OVER (ORDER BY ten), ten, four FROM tenk1 WHERE unique2 < 10
 -- !query schema
-struct<last(four, false) OVER (ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int,ten:int,four:int>
+struct<last(four) OVER (ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int,ten:int,four:int>
 -- !query output
 0	4	0
 1	1	1
@@ -306,7 +306,7 @@ SELECT last(ten) OVER (PARTITION BY four), ten, four FROM
 (SELECT * FROM tenk1 WHERE unique2 < 10 ORDER BY four, ten)s
 ORDER BY four, ten
 -- !query schema
-struct<last(ten, false) OVER (PARTITION BY four ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):int,ten:int,four:int>
+struct<last(ten) OVER (PARTITION BY four ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):int,ten:int,four:int>
 -- !query output
 4	0	0
 4	0	0
@@ -387,7 +387,7 @@ struct<total:bigint,fourcount:bigint,twosum:bigint>
 -- !query
 SELECT avg(four) OVER (PARTITION BY four ORDER BY thousand / 100) FROM tenk1 WHERE unique2 < 10
 -- !query schema
-struct<avg(CAST(four AS BIGINT)) OVER (PARTITION BY four ORDER BY (CAST(thousand AS DOUBLE) / CAST(100 AS DOUBLE)) ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):double>
+struct<avg(four) OVER (PARTITION BY four ORDER BY (thousand / 100) ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):double>
 -- !query output
 0.0
 0.0
@@ -476,7 +476,7 @@ sum(ten) over (partition by four order by ten),
 last(ten) over (partition by four order by ten)
 FROM (select distinct ten, four from tenk1) ss
 -- !query schema
-struct<four:int,ten:int,sum(CAST(ten AS BIGINT)) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):bigint,last(ten, false) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int>
+struct<four:int,ten:int,sum(ten) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):bigint,last(ten) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int>
 -- !query output
 0	0	0	0
 0	2	2	2
@@ -506,7 +506,7 @@ sum(ten) over (partition by four order by ten range between unbounded preceding
 last(ten) over (partition by four order by ten range between unbounded preceding and current row)
 FROM (select distinct ten, four from tenk1) ss
 -- !query schema
-struct<four:int,ten:int,sum(ten) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):bigint,last(ten, false) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int>
+struct<four:int,ten:int,sum(ten) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):bigint,last(ten) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int>
 -- !query output
 0	0	0	0
 0	2	2	2
@@ -536,7 +536,7 @@ sum(ten) over (partition by four order by ten range between unbounded preceding
 last(ten) over (partition by four order by ten range between unbounded preceding and unbounded following)
 FROM (select distinct ten, four from tenk1) ss
 -- !query schema
-struct<four:int,ten:int,sum(ten) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):bigint,last(ten, false) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):int>
+struct<four:int,ten:int,sum(ten) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):bigint,last(ten) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):int>
 -- !query output
 0	0	20	8
 0	2	20	8
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part2.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part2.sql.out
index f41659a196ae1..3d07d9f155bed 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part2.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part2.sql.out
@@ -101,7 +101,7 @@ from
 window w as
   (order by ss.id asc nulls first range between 2 preceding and 2 following)
 -- !query schema
-struct<id:bigint,y:bigint,first(y, false) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint,last(y, false) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint>
+struct<id:bigint,y:bigint,first(y) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN (- 2) FOLLOWING AND 2 FOLLOWING):bigint,last(y) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN (- 2) FOLLOWING AND 2 FOLLOWING):bigint>
 -- !query output
 1	1	1	3
 2	2	1	4
@@ -123,7 +123,7 @@ from
 window w as
   (order by ss.id asc nulls last range between 2 preceding and 2 following)
 -- !query schema
-struct<id:bigint,y:bigint,first(y, false) OVER (ORDER BY id ASC NULLS LAST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint,last(y, false) OVER (ORDER BY id ASC NULLS LAST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint>
+struct<id:bigint,y:bigint,first(y) OVER (ORDER BY id ASC NULLS LAST RANGE BETWEEN (- 2) FOLLOWING AND 2 FOLLOWING):bigint,last(y) OVER (ORDER BY id ASC NULLS LAST RANGE BETWEEN (- 2) FOLLOWING AND 2 FOLLOWING):bigint>
 -- !query output
 1	1	1	3
 2	2	1	4
@@ -145,7 +145,7 @@ from
 window w as
   (order by ss.id desc nulls first range between 2 preceding and 2 following)
 -- !query schema
-struct<id:bigint,y:bigint,first(y, false) OVER (ORDER BY id DESC NULLS FIRST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint,last(y, false) OVER (ORDER BY id DESC NULLS FIRST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint>
+struct<id:bigint,y:bigint,first(y) OVER (ORDER BY id DESC NULLS FIRST RANGE BETWEEN (- 2) FOLLOWING AND 2 FOLLOWING):bigint,last(y) OVER (ORDER BY id DESC NULLS FIRST RANGE BETWEEN (- 2) FOLLOWING AND 2 FOLLOWING):bigint>
 -- !query output
 1	1	3	1
 2	2	4	1
@@ -167,7 +167,7 @@ from
 window w as
   (order by ss.id desc nulls last range between 2 preceding and 2 following)
 -- !query schema
-struct<id:bigint,y:bigint,first(y, false) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint,last(y, false) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint>
+struct<id:bigint,y:bigint,first(y) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN (- 2) FOLLOWING AND 2 FOLLOWING):bigint,last(y) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN (- 2) FOLLOWING AND 2 FOLLOWING):bigint>
 -- !query output
 1	1	3	1
 2	2	4	1
@@ -182,7 +182,7 @@ NULL	43	42	43
 select x.id, last(x.id) over (order by x.id range between current row and 2147450884 following)
 from range(32764, 32767) x
 -- !query schema
-struct<id:bigint,last(id, false) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND CAST(2147450884 AS BIGINT) FOLLOWING):bigint>
+struct<id:bigint,last(id) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND 2147450884 FOLLOWING):bigint>
 -- !query output
 32764	32766
 32765	32766
@@ -193,7 +193,7 @@ struct<id:bigint,last(id, false) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN
 select x.id, last(x.id) over (order by x.id desc range between current row and 2147450885 following)
 from range(-32766, -32765) x
 -- !query schema
-struct<id:bigint,last(id, false) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND CAST(2147450885 AS BIGINT) FOLLOWING):bigint>
+struct<id:bigint,last(id) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND 2147450885 FOLLOWING):bigint>
 -- !query output
 -32766	-32766
 
@@ -202,7 +202,7 @@ struct<id:bigint,last(id, false) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN
 select x.id, last(x.id) over (order by x.id range between current row and 4 following)
 from range(2147483644, 2147483647) x
 -- !query schema
-struct<id:bigint,last(id, false) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND CAST(4 AS BIGINT) FOLLOWING):bigint>
+struct<id:bigint,last(id) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND 4 FOLLOWING):bigint>
 -- !query output
 2147483644	2147483646
 2147483645	2147483646
@@ -213,7 +213,7 @@ struct<id:bigint,last(id, false) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN
 select x.id, last(x.id) over (order by x.id desc range between current row and 5 following)
 from range(-2147483646, -2147483645) x
 -- !query schema
-struct<id:bigint,last(id, false) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND CAST(5 AS BIGINT) FOLLOWING):bigint>
+struct<id:bigint,last(id) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND 5 FOLLOWING):bigint>
 -- !query output
 -2147483646	-2147483646
 
@@ -272,7 +272,7 @@ from numerics
 window w as (order by f_float4 range between
              1 preceding and 1 following)
 -- !query schema
-struct<id:int,f_float4:float,first(id, false) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1 AS FLOAT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1 AS FLOAT) FOLLOWING):int>
+struct<id:int,f_float4:float,first(id) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN (- 1) FOLLOWING AND 1 FOLLOWING):int,last(id) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN (- 1) FOLLOWING AND 1 FOLLOWING):int>
 -- !query output
 1	-3.0	1	1
 2	-1.0	2	3
@@ -289,7 +289,7 @@ from numerics
 window w as (order by f_float4 range between
              1 preceding and 1.1 following)
 -- !query schema
-struct<id:int,f_float4:float,first(id, false) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1.1 AS FLOAT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1.1 AS FLOAT) FOLLOWING):int>
+struct<id:int,f_float4:float,first(id) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN (- 1) FOLLOWING AND 1.1 FOLLOWING):int,last(id) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN (- 1) FOLLOWING AND 1.1 FOLLOWING):int>
 -- !query output
 1	-3.0	1	1
 2	-1.0	2	3
@@ -306,7 +306,7 @@ from numerics
 window w as (order by f_float4 range between
              'inf' preceding and 'inf' following)
 -- !query schema
-struct<id:int,f_float4:float,first(id, false) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- CAST(inf AS DOUBLE)) AS FLOAT) FOLLOWING AND CAST(inf AS FLOAT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- CAST(inf AS DOUBLE)) AS FLOAT) FOLLOWING AND CAST(inf AS FLOAT) FOLLOWING):int>
+struct<id:int,f_float4:float,first(id) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN (- inf) FOLLOWING AND inf FOLLOWING):int,last(id) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN (- inf) FOLLOWING AND inf FOLLOWING):int>
 -- !query output
 1	-3.0	1	7
 2	-1.0	1	7
@@ -323,7 +323,7 @@ from numerics
 window w as (order by f_float4 range between
              1.1 preceding and 'NaN' following)
 -- !query schema
-struct<id:int,f_float4:float,first(id, false) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1.1) AS FLOAT) FOLLOWING AND CAST(NaN AS FLOAT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1.1) AS FLOAT) FOLLOWING AND CAST(NaN AS FLOAT) FOLLOWING):int>
+struct<id:int,f_float4:float,first(id) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN (- 1.1) FOLLOWING AND NaN FOLLOWING):int,last(id) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN (- 1.1) FOLLOWING AND NaN FOLLOWING):int>
 -- !query output
 1	-3.0	1	7
 2	-1.0	2	7
@@ -340,7 +340,7 @@ from numerics
 window w as (order by f_float8 range between
              1 preceding and 1 following)
 -- !query schema
-struct<id:int,f_float8:float,first(id, false) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1 AS FLOAT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1 AS FLOAT) FOLLOWING):int>
+struct<id:int,f_float8:float,first(id) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN (- 1) FOLLOWING AND 1 FOLLOWING):int,last(id) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN (- 1) FOLLOWING AND 1 FOLLOWING):int>
 -- !query output
 1	-3.0	1	1
 2	-1.0	2	3
@@ -357,7 +357,7 @@ from numerics
 window w as (order by f_float8 range between
              1 preceding and 1.1 following)
 -- !query schema
-struct<id:int,f_float8:float,first(id, false) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1.1 AS FLOAT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1.1 AS FLOAT) FOLLOWING):int>
+struct<id:int,f_float8:float,first(id) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN (- 1) FOLLOWING AND 1.1 FOLLOWING):int,last(id) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN (- 1) FOLLOWING AND 1.1 FOLLOWING):int>
 -- !query output
 1	-3.0	1	1
 2	-1.0	2	3
@@ -374,7 +374,7 @@ from numerics
 window w as (order by f_float8 range between
              'inf' preceding and 'inf' following)
 -- !query schema
-struct<id:int,f_float8:float,first(id, false) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- CAST(inf AS DOUBLE)) AS FLOAT) FOLLOWING AND CAST(inf AS FLOAT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- CAST(inf AS DOUBLE)) AS FLOAT) FOLLOWING AND CAST(inf AS FLOAT) FOLLOWING):int>
+struct<id:int,f_float8:float,first(id) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN (- inf) FOLLOWING AND inf FOLLOWING):int,last(id) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN (- inf) FOLLOWING AND inf FOLLOWING):int>
 -- !query output
 1	-3.0	1	7
 2	-1.0	1	7
@@ -391,7 +391,7 @@ from numerics
 window w as (order by f_float8 range between
              1.1 preceding and 'NaN' following)
 -- !query schema
-struct<id:int,f_float8:float,first(id, false) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1.1) AS FLOAT) FOLLOWING AND CAST(NaN AS FLOAT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1.1) AS FLOAT) FOLLOWING AND CAST(NaN AS FLOAT) FOLLOWING):int>
+struct<id:int,f_float8:float,first(id) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN (- 1.1) FOLLOWING AND NaN FOLLOWING):int,last(id) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN (- 1.1) FOLLOWING AND NaN FOLLOWING):int>
 -- !query output
 1	-3.0	1	7
 2	-1.0	2	7
@@ -408,7 +408,7 @@ from numerics
 window w as (order by f_numeric range between
              1 preceding and 1 following)
 -- !query schema
-struct<id:int,f_numeric:int,first(id, false) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING):int,last(id, false) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING):int>
+struct<id:int,f_numeric:int,first(id) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING):int,last(id) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING):int>
 -- !query output
 1	-3	1	1
 2	-1	2	3
@@ -425,7 +425,7 @@ from numerics
 window w as (order by f_numeric range between
              1 preceding and 1.1 following)
 -- !query schema
-struct<id:int,f_numeric:int,first(id, false) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND CAST(1.1 AS INT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND CAST(1.1 AS INT) FOLLOWING):int>
+struct<id:int,f_numeric:int,first(id) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND 1.1 FOLLOWING):int,last(id) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND 1.1 FOLLOWING):int>
 -- !query output
 1	-3	1	1
 2	-1	2	3
@@ -442,7 +442,7 @@ from numerics
 window w as (order by f_numeric range between
              1 preceding and 1.1 following)
 -- !query schema
-struct<id:int,f_numeric:int,first(id, false) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND CAST(1.1 AS INT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND CAST(1.1 AS INT) FOLLOWING):int>
+struct<id:int,f_numeric:int,first(id) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND 1.1 FOLLOWING):int,last(id) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND 1.1 FOLLOWING):int>
 -- !query output
 1	-3	1	1
 2	-1	2	3
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out
index 08eba6797b01d..14fd1bb281903 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 29
+-- Number of queries: 30
 
 
 -- !query
@@ -71,7 +71,8 @@ insert into datetimes values
 -- !query schema
 struct<>
 -- !query output
-
+org.apache.spark.sql.AnalysisException
+failed to evaluate expression CAST('11:00 BST' AS TIMESTAMP): Cannot cast 11:00 BST to TimestampType.; line 1 pos 22
 
 
 -- !query
@@ -112,7 +113,7 @@ SELECT x, (sum(x) over w)
 FROM cte
 WINDOW w AS (ORDER BY x range between 1 preceding and 1 following)
 -- !query schema
-struct<x:bigint,sum(x) OVER (ORDER BY x ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS BIGINT) FOLLOWING AND CAST(1 AS BIGINT) FOLLOWING):bigint>
+struct<x:bigint,sum(x) OVER (ORDER BY x ASC NULLS FIRST RANGE BETWEEN (- 1) FOLLOWING AND 1 FOLLOWING):bigint>
 -- !query output
 1	1
 11	11
@@ -182,7 +183,7 @@ SELECT x, (sum(x) over w)
 FROM cte
 WINDOW w AS (ORDER BY x range between 1 preceding and 1 following)
 -- !query schema
-struct<x:bigint,sum(x) OVER (ORDER BY x ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS BIGINT) FOLLOWING AND CAST(1 AS BIGINT) FOLLOWING):bigint>
+struct<x:bigint,sum(x) OVER (ORDER BY x ASC NULLS FIRST RANGE BETWEEN (- 1) FOLLOWING AND 1 FOLLOWING):bigint>
 -- !query output
 1	3
 1	3
@@ -244,7 +245,7 @@ from t1 where f1 = f2
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(PARTITION BY spark_catalog.default.t1.`f1` RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING)' due to data type mismatch: A range window frame cannot be used in an unordered window specification.; line 1 pos 24
+cannot resolve '(PARTITION BY spark_catalog.default.t1.f1 RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING)' due to data type mismatch: A range window frame cannot be used in an unordered window specification.; line 1 pos 24
 
 
 -- !query
@@ -294,7 +295,7 @@ SELECT * FROM empsalary WHERE row_number() OVER (ORDER BY salary) < 10
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-It is not allowed to use window functions inside WHERE clause;
+It is not allowed to use window functions inside WHERE clause
 
 
 -- !query
@@ -306,7 +307,7 @@ org.apache.spark.sql.AnalysisException
 
 The query operator `Join` contains one or more unsupported
 expression types Aggregate, Window or Generate.
-Invalid expressions: [row_number() OVER (ORDER BY spark_catalog.default.empsalary.`salary` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)];
+Invalid expressions: [row_number() OVER (ORDER BY spark_catalog.default.empsalary.salary ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)]
 
 
 -- !query
@@ -318,7 +319,7 @@ org.apache.spark.sql.AnalysisException
 
 The query operator `Aggregate` contains one or more unsupported
 expression types Aggregate, Window or Generate.
-Invalid expressions: [RANK() OVER (ORDER BY 1 ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)];
+Invalid expressions: [RANK() OVER (ORDER BY 1 ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)]
 
 
 -- !query
@@ -341,7 +342,7 @@ SELECT * FROM empsalary WHERE (rank() OVER (ORDER BY random())) > 10
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-It is not allowed to use window functions inside WHERE clause;
+It is not allowed to use window functions inside WHERE clause
 
 
 -- !query
@@ -350,7 +351,7 @@ SELECT * FROM empsalary WHERE rank() OVER (ORDER BY random())
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-It is not allowed to use window functions inside WHERE clause;
+It is not allowed to use window functions inside WHERE clause
 
 
 -- !query
@@ -385,6 +386,15 @@ org.apache.spark.sql.AnalysisException
 cannot resolve 'ntile(0)' due to data type mismatch: Buckets expression must be positive, but got: 0; line 1 pos 7
 
 
+-- !query
+SELECT nth_value(four, 0) OVER (ORDER BY ten), ten, four FROM tenk1
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'nth_value(spark_catalog.default.tenk1.four, 0)' due to data type mismatch: The 'offset' argument of nth_value must be greater than zero but it is 0.; line 1 pos 7
+
+
 -- !query
 DROP TABLE empsalary
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part4.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part4.sql.out
index 4dd4712345a89..738a1381f99cc 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part4.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part4.sql.out
@@ -137,7 +137,7 @@ struct<i:int,count(1) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW
 SELECT VAR_POP(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
   FROM (VALUES(1,600),(2,470),(3,170),(4,430),(5,300)) r(i,n)
 -- !query schema
-struct<var_pop(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
+struct<var_pop(n) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
 -- !query output
 0.0
 11266.666666666666
@@ -150,7 +150,7 @@ struct<var_pop(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN
 SELECT VAR_POP(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
   FROM (VALUES(1,600),(2,470),(3,170),(4,430),(5,300)) r(i,n)
 -- !query schema
-struct<var_pop(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
+struct<var_pop(n) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
 -- !query output
 0.0
 11266.666666666666
@@ -163,7 +163,7 @@ struct<var_pop(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN
 SELECT VAR_POP(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
   FROM (VALUES(1,600),(2,470),(3,170),(4,430),(5,300)) r(i,n)
 -- !query schema
-struct<var_pop(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
+struct<var_pop(n) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
 -- !query output
 0.0
 11266.666666666666
@@ -176,7 +176,7 @@ struct<var_pop(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN
 SELECT VAR_POP(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
   FROM (VALUES(1,600),(2,470),(3,170),(4,430),(5,300)) r(i,n)
 -- !query schema
-struct<var_pop(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
+struct<var_pop(n) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
 -- !query output
 0.0
 11266.666666666666
@@ -189,111 +189,111 @@ struct<var_pop(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN
 SELECT VAR_SAMP(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
   FROM (VALUES(1,600),(2,470),(3,170),(4,430),(5,300)) r(i,n)
 -- !query schema
-struct<var_samp(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
+struct<var_samp(n) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
 -- !query output
 16900.0
 18491.666666666668
 27129.999999999996
 8450.0
-NaN
+NULL
 
 
 -- !query
 SELECT VAR_SAMP(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
   FROM (VALUES(1,600),(2,470),(3,170),(4,430),(5,300)) r(i,n)
 -- !query schema
-struct<var_samp(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
+struct<var_samp(n) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
 -- !query output
 16900.0
 18491.666666666668
 27129.999999999996
 8450.0
-NaN
+NULL
 
 
 -- !query
 SELECT VAR_SAMP(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
   FROM (VALUES(1,600),(2,470),(3,170),(4,430),(5,300)) r(i,n)
 -- !query schema
-struct<var_samp(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
+struct<var_samp(n) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
 -- !query output
 16900.0
 18491.666666666668
 27129.999999999996
 8450.0
-NaN
+NULL
 
 
 -- !query
 SELECT VAR_SAMP(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
   FROM (VALUES(1,600),(2,470),(3,170),(4,430),(5,300)) r(i,n)
 -- !query schema
-struct<var_samp(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
+struct<var_samp(n) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
 -- !query output
 16900.0
 18491.666666666668
 27129.999999999996
 8450.0
-NaN
+NULL
 
 
 -- !query
 SELECT VARIANCE(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
   FROM (VALUES(1,600),(2,470),(3,170),(4,430),(5,300)) r(i,n)
 -- !query schema
-struct<variance(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
+struct<variance(n) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
 -- !query output
 16900.0
 18491.666666666668
 27129.999999999996
 8450.0
-NaN
+NULL
 
 
 -- !query
 SELECT VARIANCE(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
   FROM (VALUES(1,600),(2,470),(3,170),(4,430),(5,300)) r(i,n)
 -- !query schema
-struct<variance(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
+struct<variance(n) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
 -- !query output
 16900.0
 18491.666666666668
 27129.999999999996
 8450.0
-NaN
+NULL
 
 
 -- !query
 SELECT VARIANCE(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
   FROM (VALUES(1,600),(2,470),(3,170),(4,430),(5,300)) r(i,n)
 -- !query schema
-struct<variance(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
+struct<variance(n) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
 -- !query output
 16900.0
 18491.666666666668
 27129.999999999996
 8450.0
-NaN
+NULL
 
 
 -- !query
 SELECT VARIANCE(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
   FROM (VALUES(1,600),(2,470),(3,170),(4,430),(5,300)) r(i,n)
 -- !query schema
-struct<variance(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
+struct<variance(n) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
 -- !query output
 16900.0
 18491.666666666668
 27129.999999999996
 8450.0
-NaN
+NULL
 
 
 -- !query
 SELECT STDDEV_POP(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
   FROM (VALUES(1,NULL),(2,600),(3,470),(4,170),(5,430),(6,300)) r(i,n)
 -- !query schema
-struct<stddev_pop(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
+struct<stddev_pop(n) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
 -- !query output
 0.0
 106.14455552060438
@@ -307,7 +307,7 @@ struct<stddev_pop(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWE
 SELECT STDDEV_POP(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
   FROM (VALUES(1,NULL),(2,600),(3,470),(4,170),(5,430),(6,300)) r(i,n)
 -- !query schema
-struct<stddev_pop(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
+struct<stddev_pop(n) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
 -- !query output
 0.0
 106.14455552060438
@@ -321,7 +321,7 @@ struct<stddev_pop(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWE
 SELECT STDDEV_POP(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
   FROM (VALUES(1,NULL),(2,600),(3,470),(4,170),(5,430),(6,300)) r(i,n)
 -- !query schema
-struct<stddev_pop(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
+struct<stddev_pop(n) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
 -- !query output
 0.0
 106.14455552060438
@@ -335,7 +335,7 @@ struct<stddev_pop(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWE
 SELECT STDDEV_POP(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
   FROM (VALUES(1,NULL),(2,600),(3,470),(4,170),(5,430),(6,300)) r(i,n)
 -- !query schema
-struct<stddev_pop(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
+struct<stddev_pop(n) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
 -- !query output
 0.0
 106.14455552060438
@@ -349,112 +349,112 @@ struct<stddev_pop(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWE
 SELECT STDDEV_SAMP(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
   FROM (VALUES(1,NULL),(2,600),(3,470),(4,170),(5,430),(6,300)) r(i,n)
 -- !query schema
-struct<stddev_samp(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
+struct<stddev_samp(n) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
 -- !query output
 130.0
 135.9840676942217
 164.7118696390761
 164.7118696390761
 91.92388155425118
-NaN
+NULL
 
 
 -- !query
 SELECT STDDEV_SAMP(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
   FROM (VALUES(1,NULL),(2,600),(3,470),(4,170),(5,430),(6,300)) r(i,n)
 -- !query schema
-struct<stddev_samp(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
+struct<stddev_samp(n) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
 -- !query output
 130.0
 135.9840676942217
 164.7118696390761
 164.7118696390761
 91.92388155425118
-NaN
+NULL
 
 
 -- !query
 SELECT STDDEV_SAMP(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
   FROM (VALUES(1,NULL),(2,600),(3,470),(4,170),(5,430),(6,300)) r(i,n)
 -- !query schema
-struct<stddev_samp(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
+struct<stddev_samp(n) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
 -- !query output
 130.0
 135.9840676942217
 164.7118696390761
 164.7118696390761
 91.92388155425118
-NaN
+NULL
 
 
 -- !query
 SELECT STDDEV_SAMP(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
   FROM (VALUES(1,NULL),(2,600),(3,470),(4,170),(5,430),(6,300)) r(i,n)
 -- !query schema
-struct<stddev_samp(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
+struct<stddev_samp(n) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
 -- !query output
 130.0
 135.9840676942217
 164.7118696390761
 164.7118696390761
 91.92388155425118
-NaN
+NULL
 
 
 -- !query
 SELECT STDDEV(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
   FROM (VALUES(0,NULL),(1,600),(2,470),(3,170),(4,430),(5,300)) r(i,n)
 -- !query schema
-struct<stddev(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
+struct<stddev(n) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
 -- !query output
 130.0
 135.9840676942217
 164.7118696390761
 164.7118696390761
 91.92388155425118
-NaN
+NULL
 
 
 -- !query
 SELECT STDDEV(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
   FROM (VALUES(0,NULL),(1,600),(2,470),(3,170),(4,430),(5,300)) r(i,n)
 -- !query schema
-struct<stddev(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
+struct<stddev(n) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
 -- !query output
 130.0
 135.9840676942217
 164.7118696390761
 164.7118696390761
 91.92388155425118
-NaN
+NULL
 
 
 -- !query
 SELECT STDDEV(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
   FROM (VALUES(0,NULL),(1,600),(2,470),(3,170),(4,430),(5,300)) r(i,n)
 -- !query schema
-struct<stddev(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
+struct<stddev(n) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
 -- !query output
 130.0
 135.9840676942217
 164.7118696390761
 164.7118696390761
 91.92388155425118
-NaN
+NULL
 
 
 -- !query
 SELECT STDDEV(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
   FROM (VALUES(0,NULL),(1,600),(2,470),(3,170),(4,430),(5,300)) r(i,n)
 -- !query schema
-struct<stddev(CAST(n AS DOUBLE)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
+struct<stddev(n) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):double>
 -- !query output
 130.0
 135.9840676942217
 164.7118696390761
 164.7118696390761
 91.92388155425118
-NaN
+NULL
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/with.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/with.sql.out
index badafc9e659e2..21bad134706bb 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/with.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/with.sql.out
@@ -223,7 +223,7 @@ Table or view not found: outermost; line 4 pos 23
 
 
 -- !query
-CREATE TABLE withz USING parquet AS SELECT i AS k, CAST(i || ' v' AS string) v FROM (SELECT EXPLODE(SEQUENCE(1, 16, 3)) i)
+CREATE TABLE withz USING parquet AS SELECT i AS k, CAST(i AS string) || ' v' AS v FROM (SELECT EXPLODE(SEQUENCE(1, 16, 3)) i)
 -- !query schema
 struct<>
 -- !query output
@@ -385,7 +385,7 @@ WITH test AS (SELECT 42) INSERT INTO test VALUES (1)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Table not found: test;
+Table not found: test; line 1 pos 37
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/predicate-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/predicate-functions.sql.out
index 819be95603b0c..32f9efde688e0 100644
--- a/sql/core/src/test/resources/sql-tests/results/predicate-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/predicate-functions.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 37
+-- Number of queries: 53
 
 
 -- !query
@@ -13,7 +13,7 @@ true
 -- !query
 select 1 = '1'
 -- !query schema
-struct<(1 = CAST(1 AS INT)):boolean>
+struct<(1 = 1):boolean>
 -- !query output
 true
 
@@ -21,7 +21,7 @@ true
 -- !query
 select 1.0 = '1'
 -- !query schema
-struct<(CAST(1.0 AS DOUBLE) = CAST(1 AS DOUBLE)):boolean>
+struct<(1.0 = 1):boolean>
 -- !query output
 true
 
@@ -29,7 +29,7 @@ true
 -- !query
 select 1.5 = '1.51'
 -- !query schema
-struct<(CAST(1.5 AS DOUBLE) = CAST(1.51 AS DOUBLE)):boolean>
+struct<(1.5 = 1.51):boolean>
 -- !query output
 false
 
@@ -37,7 +37,7 @@ false
 -- !query
 select 1 > '1'
 -- !query schema
-struct<(1 > CAST(1 AS INT)):boolean>
+struct<(1 > 1):boolean>
 -- !query output
 false
 
@@ -45,7 +45,7 @@ false
 -- !query
 select 2 > '1.0'
 -- !query schema
-struct<(2 > CAST(1.0 AS INT)):boolean>
+struct<(2 > 1.0):boolean>
 -- !query output
 true
 
@@ -53,7 +53,7 @@ true
 -- !query
 select 2 > '2.0'
 -- !query schema
-struct<(2 > CAST(2.0 AS INT)):boolean>
+struct<(2 > 2.0):boolean>
 -- !query output
 false
 
@@ -61,7 +61,7 @@ false
 -- !query
 select 2 > '2.2'
 -- !query schema
-struct<(2 > CAST(2.2 AS INT)):boolean>
+struct<(2 > 2.2):boolean>
 -- !query output
 false
 
@@ -69,7 +69,7 @@ false
 -- !query
 select '1.5' > 0.5
 -- !query schema
-struct<(CAST(1.5 AS DOUBLE) > CAST(0.5 AS DOUBLE)):boolean>
+struct<(1.5 > 0.5):boolean>
 -- !query output
 true
 
@@ -85,7 +85,7 @@ false
 -- !query
 select to_date('2009-07-30 04:17:52') > '2009-07-30 04:17:52'
 -- !query schema
-struct<(to_date(2009-07-30 04:17:52) > CAST(2009-07-30 04:17:52 AS DATE)):boolean>
+struct<(to_date(2009-07-30 04:17:52) > 2009-07-30 04:17:52):boolean>
 -- !query output
 false
 
@@ -93,7 +93,7 @@ false
 -- !query
 select 1 >= '1'
 -- !query schema
-struct<(1 >= CAST(1 AS INT)):boolean>
+struct<(1 >= 1):boolean>
 -- !query output
 true
 
@@ -101,7 +101,7 @@ true
 -- !query
 select 2 >= '1.0'
 -- !query schema
-struct<(2 >= CAST(1.0 AS INT)):boolean>
+struct<(2 >= 1.0):boolean>
 -- !query output
 true
 
@@ -109,7 +109,7 @@ true
 -- !query
 select 2 >= '2.0'
 -- !query schema
-struct<(2 >= CAST(2.0 AS INT)):boolean>
+struct<(2 >= 2.0):boolean>
 -- !query output
 true
 
@@ -117,7 +117,7 @@ true
 -- !query
 select 2.0 >= '2.2'
 -- !query schema
-struct<(CAST(2.0 AS DOUBLE) >= CAST(2.2 AS DOUBLE)):boolean>
+struct<(2.0 >= 2.2):boolean>
 -- !query output
 false
 
@@ -125,7 +125,7 @@ false
 -- !query
 select '1.5' >= 0.5
 -- !query schema
-struct<(CAST(1.5 AS DOUBLE) >= CAST(0.5 AS DOUBLE)):boolean>
+struct<(1.5 >= 0.5):boolean>
 -- !query output
 true
 
@@ -141,7 +141,7 @@ true
 -- !query
 select to_date('2009-07-30 04:17:52') >= '2009-07-30 04:17:52'
 -- !query schema
-struct<(to_date(2009-07-30 04:17:52) >= CAST(2009-07-30 04:17:52 AS DATE)):boolean>
+struct<(to_date(2009-07-30 04:17:52) >= 2009-07-30 04:17:52):boolean>
 -- !query output
 true
 
@@ -149,7 +149,7 @@ true
 -- !query
 select 1 < '1'
 -- !query schema
-struct<(1 < CAST(1 AS INT)):boolean>
+struct<(1 < 1):boolean>
 -- !query output
 false
 
@@ -157,7 +157,7 @@ false
 -- !query
 select 2 < '1.0'
 -- !query schema
-struct<(2 < CAST(1.0 AS INT)):boolean>
+struct<(2 < 1.0):boolean>
 -- !query output
 false
 
@@ -165,7 +165,7 @@ false
 -- !query
 select 2 < '2.0'
 -- !query schema
-struct<(2 < CAST(2.0 AS INT)):boolean>
+struct<(2 < 2.0):boolean>
 -- !query output
 false
 
@@ -173,7 +173,7 @@ false
 -- !query
 select 2.0 < '2.2'
 -- !query schema
-struct<(CAST(2.0 AS DOUBLE) < CAST(2.2 AS DOUBLE)):boolean>
+struct<(2.0 < 2.2):boolean>
 -- !query output
 true
 
@@ -181,7 +181,7 @@ true
 -- !query
 select 0.5 < '1.5'
 -- !query schema
-struct<(CAST(0.5 AS DOUBLE) < CAST(1.5 AS DOUBLE)):boolean>
+struct<(0.5 < 1.5):boolean>
 -- !query output
 true
 
@@ -197,7 +197,7 @@ false
 -- !query
 select to_date('2009-07-30 04:17:52') < '2009-07-30 04:17:52'
 -- !query schema
-struct<(to_date(2009-07-30 04:17:52) < CAST(2009-07-30 04:17:52 AS DATE)):boolean>
+struct<(to_date(2009-07-30 04:17:52) < 2009-07-30 04:17:52):boolean>
 -- !query output
 false
 
@@ -205,7 +205,7 @@ false
 -- !query
 select 1 <= '1'
 -- !query schema
-struct<(1 <= CAST(1 AS INT)):boolean>
+struct<(1 <= 1):boolean>
 -- !query output
 true
 
@@ -213,7 +213,7 @@ true
 -- !query
 select 2 <= '1.0'
 -- !query schema
-struct<(2 <= CAST(1.0 AS INT)):boolean>
+struct<(2 <= 1.0):boolean>
 -- !query output
 false
 
@@ -221,7 +221,7 @@ false
 -- !query
 select 2 <= '2.0'
 -- !query schema
-struct<(2 <= CAST(2.0 AS INT)):boolean>
+struct<(2 <= 2.0):boolean>
 -- !query output
 true
 
@@ -229,7 +229,7 @@ true
 -- !query
 select 2.0 <= '2.2'
 -- !query schema
-struct<(CAST(2.0 AS DOUBLE) <= CAST(2.2 AS DOUBLE)):boolean>
+struct<(2.0 <= 2.2):boolean>
 -- !query output
 true
 
@@ -237,7 +237,7 @@ true
 -- !query
 select 0.5 <= '1.5'
 -- !query schema
-struct<(CAST(0.5 AS DOUBLE) <= CAST(1.5 AS DOUBLE)):boolean>
+struct<(0.5 <= 1.5):boolean>
 -- !query output
 true
 
@@ -253,7 +253,7 @@ true
 -- !query
 select to_date('2009-07-30 04:17:52') <= '2009-07-30 04:17:52'
 -- !query schema
-struct<(to_date(2009-07-30 04:17:52) <= CAST(2009-07-30 04:17:52 AS DATE)):boolean>
+struct<(to_date(2009-07-30 04:17:52) <= 2009-07-30 04:17:52):boolean>
 -- !query output
 true
 
@@ -261,7 +261,7 @@ true
 -- !query
 select to_date('2017-03-01') = to_timestamp('2017-03-01 00:00:00')
 -- !query schema
-struct<(CAST(to_date(2017-03-01) AS TIMESTAMP) = to_timestamp(2017-03-01 00:00:00)):boolean>
+struct<(to_date(2017-03-01) = to_timestamp(2017-03-01 00:00:00)):boolean>
 -- !query output
 true
 
@@ -269,7 +269,7 @@ true
 -- !query
 select to_timestamp('2017-03-01 00:00:01') > to_date('2017-03-01')
 -- !query schema
-struct<(to_timestamp(2017-03-01 00:00:01) > CAST(to_date(2017-03-01) AS TIMESTAMP)):boolean>
+struct<(to_timestamp(2017-03-01 00:00:01) > to_date(2017-03-01)):boolean>
 -- !query output
 true
 
@@ -277,7 +277,7 @@ true
 -- !query
 select to_timestamp('2017-03-01 00:00:01') >= to_date('2017-03-01')
 -- !query schema
-struct<(to_timestamp(2017-03-01 00:00:01) >= CAST(to_date(2017-03-01) AS TIMESTAMP)):boolean>
+struct<(to_timestamp(2017-03-01 00:00:01) >= to_date(2017-03-01)):boolean>
 -- !query output
 true
 
@@ -285,7 +285,7 @@ true
 -- !query
 select to_date('2017-03-01') < to_timestamp('2017-03-01 00:00:01')
 -- !query schema
-struct<(CAST(to_date(2017-03-01) AS TIMESTAMP) < to_timestamp(2017-03-01 00:00:01)):boolean>
+struct<(to_date(2017-03-01) < to_timestamp(2017-03-01 00:00:01)):boolean>
 -- !query output
 true
 
@@ -293,6 +293,134 @@ true
 -- !query
 select to_date('2017-03-01') <= to_timestamp('2017-03-01 00:00:01')
 -- !query schema
-struct<(CAST(to_date(2017-03-01) AS TIMESTAMP) <= to_timestamp(2017-03-01 00:00:01)):boolean>
+struct<(to_date(2017-03-01) <= to_timestamp(2017-03-01 00:00:01)):boolean>
+-- !query output
+true
+
+
+-- !query
+select 1 in (1, 2, 3)
+-- !query schema
+struct<(1 IN (1, 2, 3)):boolean>
+-- !query output
+true
+
+
+-- !query
+select 1 in (1, 2, 3, null)
+-- !query schema
+struct<(1 IN (1, 2, 3, NULL)):boolean>
+-- !query output
+true
+
+
+-- !query
+select 1 in (1.0, 2.0, 3.0)
+-- !query schema
+struct<(1 IN (1.0, 2.0, 3.0)):boolean>
+-- !query output
+true
+
+
+-- !query
+select 1 in (1.0, 2.0, 3.0, null)
+-- !query schema
+struct<(1 IN (1.0, 2.0, 3.0, NULL)):boolean>
 -- !query output
 true
+
+
+-- !query
+select 1 in ('2', '3', '4')
+-- !query schema
+struct<(1 IN (2, 3, 4)):boolean>
+-- !query output
+false
+
+
+-- !query
+select 1 in ('2', '3', '4', null)
+-- !query schema
+struct<(1 IN (2, 3, 4, NULL)):boolean>
+-- !query output
+NULL
+
+
+-- !query
+select null in (1, 2, 3)
+-- !query schema
+struct<(NULL IN (1, 2, 3)):boolean>
+-- !query output
+NULL
+
+
+-- !query
+select null in (1, 2, null)
+-- !query schema
+struct<(NULL IN (1, 2, NULL)):boolean>
+-- !query output
+NULL
+
+
+-- !query
+select 1 not in (1, 2, 3)
+-- !query schema
+struct<(NOT (1 IN (1, 2, 3))):boolean>
+-- !query output
+false
+
+
+-- !query
+select 1 not in (1, 2, 3, null)
+-- !query schema
+struct<(NOT (1 IN (1, 2, 3, NULL))):boolean>
+-- !query output
+false
+
+
+-- !query
+select 1 not in (1.0, 2.0, 3.0)
+-- !query schema
+struct<(NOT (1 IN (1.0, 2.0, 3.0))):boolean>
+-- !query output
+false
+
+
+-- !query
+select 1 not in (1.0, 2.0, 3.0, null)
+-- !query schema
+struct<(NOT (1 IN (1.0, 2.0, 3.0, NULL))):boolean>
+-- !query output
+false
+
+
+-- !query
+select 1 not in ('2', '3', '4')
+-- !query schema
+struct<(NOT (1 IN (2, 3, 4))):boolean>
+-- !query output
+true
+
+
+-- !query
+select 1 not in ('2', '3', '4', null)
+-- !query schema
+struct<(NOT (1 IN (2, 3, 4, NULL))):boolean>
+-- !query output
+NULL
+
+
+-- !query
+select null not in (1, 2, 3)
+-- !query schema
+struct<(NOT (NULL IN (1, 2, 3))):boolean>
+-- !query output
+NULL
+
+
+-- !query
+select null not in (1, 2, null)
+-- !query schema
+struct<(NOT (NULL IN (1, 2, NULL))):boolean>
+-- !query output
+NULL
diff --git a/sql/core/src/test/resources/sql-tests/results/random.sql.out b/sql/core/src/test/resources/sql-tests/results/random.sql.out
index 9d00a82b76780..b269d40c3566c 100644
--- a/sql/core/src/test/resources/sql-tests/results/random.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/random.sql.out
@@ -13,7 +13,7 @@ struct<rand(0):double>
 -- !query
 SELECT rand(cast(3 / 7 AS int))
 -- !query schema
-struct<rand(CAST((CAST(3 AS DOUBLE) / CAST(7 AS DOUBLE)) AS INT)):double>
+struct<rand(CAST((3 / 7) AS INT)):double>
 -- !query output
 0.7604953758285915
 
@@ -21,7 +21,7 @@ struct<rand(CAST((CAST(3 AS DOUBLE) / CAST(7 AS DOUBLE)) AS INT)):double>
 -- !query
 SELECT rand(NULL)
 -- !query schema
-struct<rand(CAST(NULL AS INT)):double>
+struct<rand(NULL):double>
 -- !query output
 0.7604953758285915
 
@@ -54,7 +54,7 @@ struct<randn(0):double>
 -- !query
 SELECT randn(cast(3 / 7 AS long))
 -- !query schema
-struct<randn(CAST((CAST(3 AS DOUBLE) / CAST(7 AS DOUBLE)) AS BIGINT)):double>
+struct<randn(CAST((3 / 7) AS BIGINT)):double>
 -- !query output
 1.6034991609278433
 
@@ -62,7 +62,7 @@ struct<randn(CAST((CAST(3 AS DOUBLE) / CAST(7 AS DOUBLE)) AS BIGINT)):double>
 -- !query
 SELECT randn(NULL)
 -- !query schema
-struct<randn(CAST(NULL AS INT)):double>
+struct<randn(NULL):double>
 -- !query output
 1.6034991609278433
 
diff --git a/sql/core/src/test/resources/sql-tests/results/regexp-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/regexp-functions.sql.out
index c92c1ddca774f..f0a6fa064d05b 100644
--- a/sql/core/src/test/resources/sql-tests/results/regexp-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/regexp-functions.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 8
+-- Number of queries: 46
 
 
 -- !query
@@ -37,6 +37,31 @@ java.lang.IllegalArgumentException
 Regex group count is 0, but the specified group index is 2
 
 
+-- !query
+SELECT regexp_extract('1a 2b 14m', '\\d+', -1)
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+The specified group index cannot be less than zero
+
+
+-- !query
+SELECT regexp_extract('1a 2b 14m', '(\\d+)?', 1)
+-- !query schema
+struct<regexp_extract(1a 2b 14m, (\d+)?, 1):string>
+-- !query output
+1
+
+
+-- !query
+SELECT regexp_extract('a b m', '(\\d+)?', 1)
+-- !query schema
+struct<regexp_extract(a b m, (\d+)?, 1):string>
+-- !query output
+
+
+
 -- !query
 SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)')
 -- !query schema
@@ -67,3 +92,293 @@ SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', 2)
 struct<regexp_extract(1a 2b 14m, (\d+)([a-z]+), 2):string>
 -- !query output
 a
+
+
+-- !query
+SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', 3)
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Regex group count is 2, but the specified group index is 3
+
+
+-- !query
+SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', -1)
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+The specified group index cannot be less than zero
+
+
+-- !query
+SELECT regexp_extract('1a 2b 14m', '(\\d+)?([a-z]+)', 1)
+-- !query schema
+struct<regexp_extract(1a 2b 14m, (\d+)?([a-z]+), 1):string>
+-- !query output
+1
+
+
+-- !query
+SELECT regexp_extract('a b m', '(\\d+)?([a-z]+)', 1)
+-- !query schema
+struct<regexp_extract(a b m, (\d+)?([a-z]+), 1):string>
+-- !query output
+
+
+
+-- !query
+SELECT regexp_extract_all('1a 2b 14m', '\\d+')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Regex group count is 0, but the specified group index is 1
+
+
+-- !query
+SELECT regexp_extract_all('1a 2b 14m', '\\d+', 0)
+-- !query schema
+struct<regexp_extract_all(1a 2b 14m, \d+, 0):array<string>>
+-- !query output
+["1","2","14"]
+
+
+-- !query
+SELECT regexp_extract_all('1a 2b 14m', '\\d+', 1)
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Regex group count is 0, but the specified group index is 1
+
+
+-- !query
+SELECT regexp_extract_all('1a 2b 14m', '\\d+', 2)
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Regex group count is 0, but the specified group index is 2
+
+
+-- !query
+SELECT regexp_extract_all('1a 2b 14m', '\\d+', -1)
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+The specified group index cannot be less than zero
+
+
+-- !query
+SELECT regexp_extract_all('1a 2b 14m', '(\\d+)?', 1)
+-- !query schema
+struct<regexp_extract_all(1a 2b 14m, (\d+)?, 1):array<string>>
+-- !query output
+["1","","","2","","","14","",""]
+
+
+-- !query
+SELECT regexp_extract_all('a 2b 14m', '(\\d+)?', 1)
+-- !query schema
+struct<regexp_extract_all(a 2b 14m, (\d+)?, 1):array<string>>
+-- !query output
+["","","2","","","14","",""]
+
+
+-- !query
+SELECT regexp_extract_all('1a 2b 14m', '(\\d+)([a-z]+)')
+-- !query schema
+struct<regexp_extract_all(1a 2b 14m, (\d+)([a-z]+), 1):array<string>>
+-- !query output
+["1","2","14"]
+
+
+-- !query
+SELECT regexp_extract_all('1a 2b 14m', '(\\d+)([a-z]+)', 0)
+-- !query schema
+struct<regexp_extract_all(1a 2b 14m, (\d+)([a-z]+), 0):array<string>>
+-- !query output
+["1a","2b","14m"]
+
+
+-- !query
+SELECT regexp_extract_all('1a 2b 14m', '(\\d+)([a-z]+)', 1)
+-- !query schema
+struct<regexp_extract_all(1a 2b 14m, (\d+)([a-z]+), 1):array<string>>
+-- !query output
+["1","2","14"]
+
+
+-- !query
+SELECT regexp_extract_all('1a 2b 14m', '(\\d+)([a-z]+)', 2)
+-- !query schema
+struct<regexp_extract_all(1a 2b 14m, (\d+)([a-z]+), 2):array<string>>
+-- !query output
+["a","b","m"]
+
+
+-- !query
+SELECT regexp_extract_all('1a 2b 14m', '(\\d+)([a-z]+)', 3)
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Regex group count is 2, but the specified group index is 3
+
+
+-- !query
+SELECT regexp_extract_all('1a 2b 14m', '(\\d+)([a-z]+)', -1)
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+The specified group index cannot be less than zero
+
+
+-- !query
+SELECT regexp_extract_all('1a 2b 14m', '(\\d+)?([a-z]+)', 1)
+-- !query schema
+struct<regexp_extract_all(1a 2b 14m, (\d+)?([a-z]+), 1):array<string>>
+-- !query output
+["1","2","14"]
+
+
+-- !query
+SELECT regexp_extract_all('a 2b 14m', '(\\d+)?([a-z]+)', 1)
+-- !query schema
+struct<regexp_extract_all(a 2b 14m, (\d+)?([a-z]+), 1):array<string>>
+-- !query output
+["","2","14"]
+
+
+-- !query
+SELECT regexp_replace('healthy, wealthy, and wise', '\\w+thy', 'something')
+-- !query schema
+struct<regexp_replace(healthy, wealthy, and wise, \w+thy, something, 1):string>
+-- !query output
+something, something, and wise
+
+
+-- !query
+SELECT regexp_replace('healthy, wealthy, and wise', '\\w+thy', 'something', -2)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'regexp_replace('healthy, wealthy, and wise', '\\w+thy', 'something', -2)' due to data type mismatch: Position expression must be positive, but got: -2; line 1 pos 7
+
+
+-- !query
+SELECT regexp_replace('healthy, wealthy, and wise', '\\w+thy', 'something', 0)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'regexp_replace('healthy, wealthy, and wise', '\\w+thy', 'something', 0)' due to data type mismatch: Position expression must be positive, but got: 0; line 1 pos 7
+
+
+-- !query
+SELECT regexp_replace('healthy, wealthy, and wise', '\\w+thy', 'something', 1)
+-- !query schema
+struct<regexp_replace(healthy, wealthy, and wise, \w+thy, something, 1):string>
+-- !query output
+something, something, and wise
+
+
+-- !query
+SELECT regexp_replace('healthy, wealthy, and wise', '\\w+thy', 'something', 2)
+-- !query schema
+struct<regexp_replace(healthy, wealthy, and wise, \w+thy, something, 2):string>
+-- !query output
+hsomething, something, and wise
+
+
+-- !query
+SELECT regexp_replace('healthy, wealthy, and wise', '\\w+thy', 'something', 8)
+-- !query schema
+struct<regexp_replace(healthy, wealthy, and wise, \w+thy, something, 8):string>
+-- !query output
+healthy, something, and wise
+
+
+-- !query
+SELECT regexp_replace('healthy, wealthy, and wise', '\\w', 'something', 26)
+-- !query schema
+struct<regexp_replace(healthy, wealthy, and wise, \w, something, 26):string>
+-- !query output
+healthy, wealthy, and wissomething
+
+
+-- !query
+SELECT regexp_replace('healthy, wealthy, and wise', '\\w', 'something', 27)
+-- !query schema
+struct<regexp_replace(healthy, wealthy, and wise, \w, something, 27):string>
+-- !query output
+healthy, wealthy, and wise
+
+
+-- !query
+SELECT regexp_replace('healthy, wealthy, and wise', '\\w', 'something', 30)
+-- !query schema
+struct<regexp_replace(healthy, wealthy, and wise, \w, something, 30):string>
+-- !query output
+healthy, wealthy, and wise
+
+
+-- !query
+SELECT regexp_replace('healthy, wealthy, and wise', '\\w', 'something', null)
+-- !query schema
+struct<regexp_replace(healthy, wealthy, and wise, \w, something, NULL):string>
+-- !query output
+NULL
+
+
+-- !query
+SELECT regexp_like('1a 2b 14m', '\\d+b')
+-- !query schema
+struct<REGEXP_LIKE(1a 2b 14m, \d+b):boolean>
+-- !query output
+true
+
+
+-- !query
+SELECT regexp_like('1a 2b 14m', '[a-z]+b')
+-- !query schema
+struct<REGEXP_LIKE(1a 2b 14m, [a-z]+b):boolean>
+-- !query output
+false
+
+
+-- !query
+SELECT regexp('1a 2b 14m', '\\d+b')
+-- !query schema
+struct<REGEXP(1a 2b 14m, \d+b):boolean>
+-- !query output
+true
+
+
+-- !query
+SELECT regexp('1a 2b 14m', '[a-z]+b')
+-- !query schema
+struct<REGEXP(1a 2b 14m, [a-z]+b):boolean>
+-- !query output
+false
+
+
+-- !query
+SELECT rlike('1a 2b 14m', '\\d+b')
+-- !query schema
+struct<RLIKE(1a 2b 14m, \d+b):boolean>
+-- !query output
+true
+
+
+-- !query
+SELECT rlike('1a 2b 14m', '[a-z]+b')
+-- !query schema
+struct<RLIKE(1a 2b 14m, [a-z]+b):boolean>
+-- !query output
+false
diff --git a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out
index a95b02c7f7743..276989db73e9b 100644
--- a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out
@@ -61,7 +61,7 @@ struct<>
 -- !query
 SHOW TABLES
 -- !query schema
-struct<database:string,tableName:string,isTemporary:boolean>
+struct<namespace:string,tableName:string,isTemporary:boolean>
 -- !query output
 show_t1
 show_t2
@@ -71,7 +71,7 @@ show_t3
 -- !query
 SHOW TABLES IN showdb
 -- !query schema
-struct<database:string,tableName:string,isTemporary:boolean>
+struct<namespace:string,tableName:string,isTemporary:boolean>
 -- !query output
 show_t1
 show_t2
@@ -81,7 +81,7 @@ show_t3
 -- !query
 SHOW TABLES 'show_t*'
 -- !query schema
-struct<database:string,tableName:string,isTemporary:boolean>
+struct<namespace:string,tableName:string,isTemporary:boolean>
 -- !query output
 show_t1
 show_t2
@@ -91,7 +91,7 @@ show_t3
 -- !query
 SHOW TABLES LIKE 'show_t1*|show_t2*'
 -- !query schema
-struct<database:string,tableName:string,isTemporary:boolean>
+struct<namespace:string,tableName:string,isTemporary:boolean>
 -- !query output
 show_t1
 show_t2
@@ -100,7 +100,7 @@ show_t2
 -- !query
 SHOW TABLES IN showdb 'show_t*'
 -- !query schema
-struct<database:string,tableName:string,isTemporary:boolean>
+struct<namespace:string,tableName:string,isTemporary:boolean>
 -- !query output
 show_t1
 show_t2
@@ -110,7 +110,7 @@ show_t3
 -- !query
 SHOW TABLES IN showdb LIKE 'show_t*'
 -- !query schema
-struct<database:string,tableName:string,isTemporary:boolean>
+struct<namespace:string,tableName:string,isTemporary:boolean>
 -- !query output
 show_t1
 show_t2
@@ -120,13 +120,14 @@ show_t3
 -- !query
 SHOW TABLE EXTENDED LIKE 'show_t*'
 -- !query schema
-struct<database:string,tableName:string,isTemporary:boolean,information:string>
+struct<namespace:string,tableName:string,isTemporary:boolean,information:string>
 -- !query output
 	show_t3	true	Table: show_t3
 Created Time [not included in comparison]
 Last Access [not included in comparison]
 Created By [not included in comparison]
 Type: VIEW
+Table Properties: []
 Schema: root
  |-- e: integer (nullable = true)
 
@@ -178,7 +179,7 @@ SHOW TABLE EXTENDED
 -- !query
 SHOW TABLE EXTENDED LIKE 'show_t1' PARTITION(c='Us', d=1)
 -- !query schema
-struct<database:string,tableName:string,isTemporary:boolean,information:string>
+struct<namespace:string,tableName:string,isTemporary:boolean,information:string>
 -- !query output
 showdb	show_t1	false	Partition Values: [c=Us, d=1]
 Location [not included in comparison]/{warehouse_dir}/showdb.db/show_t1/c=Us/d=1
@@ -206,7 +207,7 @@ SHOW TABLE EXTENDED LIKE 'show_t*' PARTITION(c='Us', d=1)
 struct<>
 -- !query output
 org.apache.spark.sql.catalyst.analysis.NoSuchTableException
-Table or view 'show_t*' not found in database 'showdb';
+Table or view 'show_t*' not found in database 'showdb'
 
 
 -- !query
@@ -215,7 +216,7 @@ SHOW TABLE EXTENDED LIKE 'show_t1' PARTITION(c='Us')
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Partition spec is invalid. The spec (c) must match the partition spec (c, d) defined in table '`showdb`.`show_t1`';
+Partition spec is invalid. The spec (c) must match the partition spec (c, d) defined in table '`showdb`.`show_t1`'
 
 
 -- !query
@@ -224,7 +225,7 @@ SHOW TABLE EXTENDED LIKE 'show_t1' PARTITION(a='Us', d=1)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Partition spec is invalid. The spec (a, d) must match the partition spec (c, d) defined in table '`showdb`.`show_t1`';
+a is not a valid partition column in table `showdb`.`show_t1`.
 
 
 -- !query
@@ -235,7 +236,7 @@ struct<>
 org.apache.spark.sql.catalyst.analysis.NoSuchPartitionException
 Partition not found in table 'show_t1' database 'showdb':
 c -> Ch
-d -> 1;
+d -> 1
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/show-tblproperties.sql.out b/sql/core/src/test/resources/sql-tests/results/show-tblproperties.sql.out
index 6984b34c365ec..1008f9af0deb6 100644
--- a/sql/core/src/test/resources/sql-tests/results/show-tblproperties.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/show-tblproperties.sql.out
@@ -23,17 +23,17 @@ p2	v2
 -- !query
 SHOW TBLPROPERTIES tbl("p1")
 -- !query schema
-struct<value:string>
+struct<key:string,value:string>
 -- !query output
-v1
+p1	v1
 
 
 -- !query
 SHOW TBLPROPERTIES tbl("p3")
 -- !query schema
-struct<value:string>
+struct<key:string,value:string>
 -- !query output
-Table default.tbl does not have property: p3
+p3	Table default.tbl does not have property: p3
 
 
 -- !query
@@ -64,22 +64,24 @@ view.catalogAndNamespace.part.0	spark_catalog
 view.catalogAndNamespace.part.1	default
 view.query.out.col.0	c1
 view.query.out.numCols	1
+view.referredTempFunctionsNames	[]
+view.referredTempViewNames	[]
 
 
 -- !query
 SHOW TBLPROPERTIES view("p1")
 -- !query schema
-struct<value:string>
+struct<key:string,value:string>
 -- !query output
-v1
+p1	v1
 
 
 -- !query
 SHOW TBLPROPERTIES view("p3")
 -- !query schema
-struct<value:string>
+struct<key:string,value:string>
 -- !query output
-Table default.view does not have property: p3
+p3	Table default.view does not have property: p3
 
 
 -- !query
@@ -91,7 +93,7 @@ struct<>
 
 
 -- !query
-CREATE TEMPORARY VIEW tv TBLPROPERTIES('p1'='v1') AS SELECT 1 AS c1
+CREATE TEMPORARY VIEW tv AS SELECT 1 AS c1
 -- !query schema
 struct<>
 -- !query output
diff --git a/sql/core/src/test/resources/sql-tests/results/show-views.sql.out b/sql/core/src/test/resources/sql-tests/results/show-views.sql.out
index d88790d8b5ec8..c80f8fab433fb 100644
--- a/sql/core/src/test/resources/sql-tests/results/show-views.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/show-views.sql.out
@@ -142,7 +142,7 @@ SHOW VIEWS IN wrongdb LIKE 'view_*'
 struct<>
 -- !query output
 org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException
-Database 'wrongdb' not found;
+Database 'wrongdb' not found
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/show_columns.sql.out b/sql/core/src/test/resources/sql-tests/results/show_columns.sql.out
index 4f5db7f6c6b2f..d8d0926d242cf 100644
--- a/sql/core/src/test/resources/sql-tests/results/show_columns.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/show_columns.sql.out
@@ -93,8 +93,8 @@ SHOW COLUMNS IN badtable FROM showdb
 -- !query schema
 struct<>
 -- !query output
-org.apache.spark.sql.catalyst.analysis.NoSuchTableException
-Table or view 'badtable' not found in database 'showdb';
+org.apache.spark.sql.AnalysisException
+Table or view not found: showdb.badtable; line 1 pos 16
 
 
 -- !query
@@ -112,7 +112,7 @@ SHOW COLUMNS IN showdb.showcolumn1 FROM baddb
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-SHOW COLUMNS with conflicting databases: 'baddb' != 'showdb';
+SHOW COLUMNS with conflicting databases: 'baddb' != 'showdb'
 
 
 -- !query
@@ -129,8 +129,8 @@ SHOW COLUMNS IN showdb.showcolumn3
 -- !query schema
 struct<>
 -- !query output
-org.apache.spark.sql.catalyst.analysis.NoSuchTableException
-Table or view 'showcolumn3' not found in database 'showdb';
+org.apache.spark.sql.AnalysisException
+Table or view not found: showdb.showcolumn3; line 1 pos 16
 
 
 -- !query
@@ -138,8 +138,8 @@ SHOW COLUMNS IN showcolumn3 FROM showdb
 -- !query schema
 struct<>
 -- !query output
-org.apache.spark.sql.catalyst.analysis.NoSuchTableException
-Table or view 'showcolumn3' not found in database 'showdb';
+org.apache.spark.sql.AnalysisException
+Table or view not found: showdb.showcolumn3; line 1 pos 16
 
 
 -- !query
@@ -147,8 +147,8 @@ SHOW COLUMNS IN showcolumn4
 -- !query schema
 struct<>
 -- !query output
-org.apache.spark.sql.catalyst.analysis.NoSuchTableException
-Table or view 'showcolumn4' not found in database 'showdb';
+org.apache.spark.sql.AnalysisException
+Table or view not found: showcolumn4; line 1 pos 16
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/sql-compatibility-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/sql-compatibility-functions.sql.out
index 26a44a85841e0..3549f2ccf9ae1 100644
--- a/sql/core/src/test/resources/sql-tests/results/sql-compatibility-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/sql-compatibility-functions.sql.out
@@ -69,7 +69,7 @@ struct<nvl2(NULL, 1, 2.1):double,nvl2(n, 1, 2.1):double>
 -- !query
 SELECT boolean(1), tinyint(1), smallint(1), int(1), bigint(1)
 -- !query schema
-struct<CAST(1 AS BOOLEAN):boolean,CAST(1 AS TINYINT):tinyint,CAST(1 AS SMALLINT):smallint,CAST(1 AS INT):int,CAST(1 AS BIGINT):bigint>
+struct<1:boolean,1:tinyint,1:smallint,1:int,1:bigint>
 -- !query output
 true	1	1	1	1
 
@@ -77,7 +77,7 @@ true	1	1	1	1
 -- !query
 SELECT float(1), double(1), decimal(1)
 -- !query schema
-struct<CAST(1 AS FLOAT):float,CAST(1 AS DOUBLE):double,CAST(1 AS DECIMAL(10,0)):decimal(10,0)>
+struct<1:float,1:double,1:decimal(10,0)>
 -- !query output
 1.0	1.0	1
 
@@ -85,7 +85,7 @@ struct<CAST(1 AS FLOAT):float,CAST(1 AS DOUBLE):double,CAST(1 AS DECIMAL(10,0)):
 -- !query
 SELECT date("2014-04-04"), timestamp(date("2014-04-04"))
 -- !query schema
-struct<CAST(2014-04-04 AS DATE):date,CAST(CAST(2014-04-04 AS DATE) AS TIMESTAMP):timestamp>
+struct<2014-04-04:date,2014-04-04:timestamp>
 -- !query output
 2014-04-04	2014-04-04 00:00:00
 
@@ -110,6 +110,6 @@ struct<>
 -- !query
 SELECT nvl(st.col1, "value"), count(*) FROM from tempView1 GROUP BY nvl(st.col1, "value")
 -- !query schema
-struct<nvl(st.col1 AS `col1`, value):string,FROM:bigint>
+struct<nvl(st.col1, value):string,FROM:bigint>
 -- !query output
 gamma	1
diff --git a/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
index 068b8bc35c265..80e88d0566411 100644
--- a/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 34
+-- Number of queries: 50
 
 
 -- !query
@@ -55,7 +55,7 @@ struct<length(uuid()):int,(NOT (uuid() = uuid())):boolean>
 -- !query
 select position('bar' in 'foobarbar'), position(null, 'foobarbar'), position('aaads', null)
 -- !query schema
-struct<locate(bar, foobarbar, 1):int,locate(CAST(NULL AS STRING), foobarbar, 1):int,locate(aaads, CAST(NULL AS STRING), 1):int>
+struct<locate(bar, foobarbar, 1):int,position(NULL, foobarbar, 1):int,position(aaads, NULL, 1):int>
 -- !query output
 4	NULL	NULL
 
@@ -69,11 +69,19 @@ ab	abcd	ab	NULL
 
 
 -- !query
-select left(null, -2), left("abcd", -2), left("abcd", 0), left("abcd", 'a')
+select left(null, -2)
 -- !query schema
-struct<left(NULL, -2):string,left(abcd, -2):string,left(abcd, 0):string,left(abcd, a):string>
+struct<left(NULL, -2):string>
 -- !query output
-NULL			NULL
+NULL
+
+
+-- !query
+select left("abcd", -2), left("abcd", 0), left("abcd", 'a')
+-- !query schema
+struct<left(abcd, -2):string,left(abcd, 0):string,left(abcd, a):string>
+-- !query output
+		NULL
 
 
 -- !query
@@ -85,11 +93,19 @@ cd	abcd	cd	NULL
 
 
 -- !query
-select right(null, -2), right("abcd", -2), right("abcd", 0), right("abcd", 'a')
+select right(null, -2)
 -- !query schema
-struct<right(NULL, -2):string,right(abcd, -2):string,right(abcd, 0):string,right(abcd, a):string>
+struct<right(NULL, -2):string>
 -- !query output
-NULL			NULL
+NULL
+
+
+-- !query
+select right("abcd", -2), right("abcd", 0), right("abcd", 'a')
+-- !query schema
+struct<right(abcd, -2):string,right(abcd, 0):string,right(abcd, a):string>
+-- !query output
+		NULL
 
 
 -- !query
@@ -274,3 +290,117 @@ SELECT trim(TRAILING 'xy' FROM 'TURNERyxXxy')
 struct<TRIM(TRAILING xy FROM TURNERyxXxy):string>
 -- !query output
 TURNERyxX
+
+
+-- !query
+SELECT btrim('xyxtrimyyx', 'xy')
+-- !query schema
+struct<btrim(xyxtrimyyx, xy):string>
+-- !query output
+trim
+
+
+-- !query
+SELECT btrim(encode(" xyz ", 'utf-8'))
+-- !query schema
+struct<btrim(encode( xyz , utf-8)):string>
+-- !query output
+xyz
+
+
+-- !query
+SELECT btrim(encode('yxTomxx', 'utf-8'), encode('xyz', 'utf-8'))
+-- !query schema
+struct<btrim(encode(yxTomxx, utf-8), encode(xyz, utf-8)):string>
+-- !query output
+Tom
+
+
+-- !query
+SELECT btrim(encode('xxxbarxxx', 'utf-8'), encode('x', 'utf-8'))
+-- !query schema
+struct<btrim(encode(xxxbarxxx, utf-8), encode(x, utf-8)):string>
+-- !query output
+bar
+
+
+-- !query
+SELECT lpad('hi', 'invalid_length')
+-- !query schema
+struct<lpad(hi, invalid_length,  ):string>
+-- !query output
+NULL
+
+
+-- !query
+SELECT rpad('hi', 'invalid_length')
+-- !query schema
+struct<rpad(hi, invalid_length,  ):string>
+-- !query output
+NULL
+
+
+-- !query
+select decode()
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Invalid number of arguments for function decode. Expected: 2; Found: 0; line 1 pos 7
+
+
+-- !query
+select decode(encode('abc', 'utf-8'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Invalid number of arguments for function decode. Expected: 2; Found: 1; line 1 pos 7
+
+
+-- !query
+select decode(encode('abc', 'utf-8'), 'utf-8')
+-- !query schema
+struct<decode(encode(abc, utf-8), utf-8):string>
+-- !query output
+abc
+
+
+-- !query
+select decode(1, 1, 'Southlake')
+-- !query schema
+struct<decode(1, 1, Southlake):string>
+-- !query output
+Southlake
+
+
+-- !query
+select decode(2, 1, 'Southlake')
+-- !query schema
+struct<decode(2, 1, Southlake):string>
+-- !query output
+NULL
+
+
+-- !query
+select decode(2, 1, 'Southlake', 2, 'San Francisco', 3, 'New Jersey', 4, 'Seattle', 'Non domestic')
+-- !query schema
+struct<decode(2, 1, Southlake, 2, San Francisco, 3, New Jersey, 4, Seattle, Non domestic):string>
+-- !query output
+San Francisco
+
+
+-- !query
+select decode(6, 1, 'Southlake', 2, 'San Francisco', 3, 'New Jersey', 4, 'Seattle', 'Non domestic')
+-- !query schema
+struct<decode(6, 1, Southlake, 2, San Francisco, 3, New Jersey, 4, Seattle, Non domestic):string>
+-- !query output
+Non domestic
+
+
+-- !query
+select decode(6, 1, 'Southlake', 2, 'San Francisco', 3, 'New Jersey', 4, 'Seattle')
+-- !query schema
+struct<decode(6, 1, Southlake, 2, San Francisco, 3, New Jersey, 4, Seattle):string>
+-- !query output
+NULL
diff --git a/sql/core/src/test/resources/sql-tests/results/struct.sql.out b/sql/core/src/test/resources/sql-tests/results/struct.sql.out
index f294c5213d319..3b2da6c85882b 100644
--- a/sql/core/src/test/resources/sql-tests/results/struct.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/struct.sql.out
@@ -83,7 +83,7 @@ struct<ID:int,NST:string>
 -- !query
 SELECT ID, STRUCT(ST.C as STC, ST.D as STD).STD FROM tbl_x
 -- !query schema
-struct<ID:int,named_struct(STC, ST.C AS `C` AS `STC`, STD, ST.D AS `D` AS `STD`).STD:string>
+struct<ID:int,struct(ST.C AS STC, ST.D AS STD).STD:string>
 -- !query output
 1	delta
 2	eta
diff --git a/sql/core/src/test/resources/sql-tests/results/subexp-elimination.sql.out b/sql/core/src/test/resources/sql-tests/results/subexp-elimination.sql.out
new file mode 100644
index 0000000000000..aca2c4c2f5422
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/subexp-elimination.sql.out
@@ -0,0 +1,116 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 10
+
+
+-- !query
+CREATE OR REPLACE TEMPORARY VIEW testData AS SELECT * FROM VALUES
+('{"a":1, "b":"2"}', '[{"a": 1, "b":2}, {"a":2, "b":2}]'), ('{"a":1, "b":"2"}', null), ('{"a":2, "b":"3"}', '[{"a": 3, "b":4}, {"a":4, "b":5}]'), ('{"a":5, "b":"6"}', '[{"a": 6, "b":7}, {"a":8, "b":9}]'), (null, '[{"a": 1, "b":2}, {"a":2, "b":2}]')
+AS testData(a, b)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SELECT from_json(a, 'struct<a:int,b:string>').a, from_json(a, 'struct<a:int,b:string>').b, from_json(b, 'array<struct<a:int,b:int>>')[0].a, from_json(b, 'array<struct<a:int,b:int>>')[0].b FROM testData
+-- !query schema
+struct<from_json(a).a:int,from_json(a).b:string,from_json(b)[0].a:int,from_json(b)[0].b:int>
+-- !query output
+1	2	1	2
+1	2	NULL	NULL
+2	3	3	4
+5	6	6	7
+NULL	NULL	1	2
+
+
+-- !query
+SELECT if(from_json(a, 'struct<a:int,b:string>').a > 1, from_json(b, 'array<struct<a:int,b:int>>')[0].a, from_json(b, 'array<struct<a:int,b:int>>')[0].a + 1) FROM testData
+-- !query schema
+struct<(IF((from_json(a).a > 1), from_json(b)[0].a, (from_json(b)[0].a + 1))):int>
+-- !query output
+2
+2
+3
+6
+NULL
+
+
+-- !query
+SELECT if(isnull(from_json(a, 'struct<a:int,b:string>').a), from_json(b, 'array<struct<a:int,b:int>>')[0].b + 1, from_json(b, 'array<struct<a:int,b:int>>')[0].b) FROM testData
+-- !query schema
+struct<(IF((from_json(a).a IS NULL), (from_json(b)[0].b + 1), from_json(b)[0].b)):int>
+-- !query output
+2
+3
+4
+7
+NULL
+
+
+-- !query
+SELECT case when from_json(a, 'struct<a:int,b:string>').a > 5 then from_json(a, 'struct<a:int,b:string>').b when from_json(a, 'struct<a:int,b:string>').a > 4 then from_json(a, 'struct<a:int,b:string>').b + 1 else from_json(a, 'struct<a:int,b:string>').b + 2 end FROM testData
+-- !query schema
+struct<CASE WHEN (from_json(a).a > 5) THEN from_json(a).b WHEN (from_json(a).a > 4) THEN (from_json(a).b + 1) ELSE (from_json(a).b + 2) END:string>
+-- !query output
+4.0
+4.0
+5.0
+7.0
+NULL
+
+
+-- !query
+SELECT case when from_json(a, 'struct<a:int,b:string>').a > 5 then from_json(b, 'array<struct<a:int,b:int>>')[0].b when from_json(a, 'struct<a:int,b:string>').a > 4 then from_json(b, 'array<struct<a:int,b:int>>')[0].b + 1 else from_json(b, 'array<struct<a:int,b:int>>')[0].b + 2 end FROM testData
+-- !query schema
+struct<CASE WHEN (from_json(a).a > 5) THEN from_json(b)[0].b WHEN (from_json(a).a > 4) THEN (from_json(b)[0].b + 1) ELSE (from_json(b)[0].b + 2) END:int>
+-- !query output
+4
+4
+6
+8
+NULL
+
+
+-- !query
+SELECT from_json(a, 'struct<a:int,b:string>').a + random() > 2, from_json(a, 'struct<a:int,b:string>').b, from_json(b, 'array<struct<a:int,b:int>>')[0].a, from_json(b, 'array<struct<a:int,b:int>>')[0].b + + random() > 2 FROM testData
+-- !query schema
+struct<((from_json(a).a + rand()) > 2):boolean,from_json(a).b:string,from_json(b)[0].a:int,((from_json(b)[0].b + (+ rand())) > 2):boolean>
+-- !query output
+NULL	NULL	1	true
+false	2	1	true
+false	2	NULL	NULL
+true	3	3	true
+true	6	6	true
+
+
+-- !query
+SELECT if(from_json(a, 'struct<a:int,b:string>').a + random() > 5, from_json(b, 'array<struct<a:int,b:int>>')[0].a, from_json(b, 'array<struct<a:int,b:int>>')[0].a + 1) FROM testData
+-- !query schema
+struct<(IF(((from_json(a).a + rand()) > 5), from_json(b)[0].a, (from_json(b)[0].a + 1))):int>
+-- !query output
+2
+2
+4
+6
+NULL
+
+
+-- !query
+SELECT case when from_json(a, 'struct<a:int,b:string>').a > 5 then from_json(a, 'struct<a:int,b:string>').b + random() > 5 when from_json(a, 'struct<a:int,b:string>').a > 4 then from_json(a, 'struct<a:int,b:string>').b + 1 + random() > 2 else from_json(a, 'struct<a:int,b:string>').b + 2 + random() > 5 end FROM testData
+-- !query schema
+struct<CASE WHEN (from_json(a).a > 5) THEN ((from_json(a).b + rand()) > 5) WHEN (from_json(a).a > 4) THEN (((from_json(a).b + 1) + rand()) > 2) ELSE (((from_json(a).b + 2) + rand()) > 5) END:boolean>
+-- !query output
+NULL
+false
+false
+true
+true
+
+
+-- !query
+DROP VIEW IF EXISTS testData
+-- !query schema
+struct<>
+-- !query output
+
diff --git a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-basic.sql.out b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-basic.sql.out
index a33f78abf27f9..2acea22e91b66 100644
--- a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-basic.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-basic.sql.out
@@ -41,15 +41,15 @@ select 1 from tab_a where (a1, b1) not in (select (a2, b2) from tab_b)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(named_struct('a1', tab_a.`a1`, 'b1', tab_a.`b1`) IN (listquery()))' due to data type mismatch: 
+cannot resolve '(named_struct('a1', tab_a.a1, 'b1', tab_a.b1) IN (listquery()))' due to data type mismatch: 
 The number of columns in the left hand side of an IN subquery does not match the
 number of columns in the output of subquery.
 #columns in left hand side: 2.
 #columns in right hand side: 1.
 Left side columns:
-[tab_a.`a1`, tab_a.`b1`].
+[tab_a.a1, tab_a.b1].
 Right side columns:
-[`named_struct(a2, a2, b2, b2)`].;
+[`named_struct(a2, a2, b2, b2)`].; line 1 pos 35
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-limit.sql.out b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-limit.sql.out
index 1c335445114c7..e24538b9138ba 100644
--- a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-limit.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-limit.sql.out
@@ -103,7 +103,7 @@ SELECT Count(DISTINCT( t1a )),
 FROM   t1
 WHERE  t1d IN (SELECT t2d
                FROM   t2
-               ORDER  BY t2c
+               ORDER  BY t2c, t2d
                LIMIT 2)
 GROUP  BY t1b
 ORDER  BY t1b DESC NULLS FIRST
@@ -136,7 +136,7 @@ SELECT Count(DISTINCT( t1a )),
 FROM   t1
 WHERE  t1d NOT IN (SELECT t2d
                    FROM   t2
-                   ORDER  BY t2b DESC nulls first
+                   ORDER  BY t2b DESC nulls first, t2d
                    LIMIT 1)
 GROUP  BY t1b
 ORDER BY t1b NULLS last
diff --git a/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/invalid-correlation.sql.out b/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/invalid-correlation.sql.out
index d703d4e9112e9..5b54a52e69724 100644
--- a/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/invalid-correlation.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/invalid-correlation.sql.out
@@ -46,7 +46,7 @@ AND    t2b = (SELECT max(avg)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-grouping expressions sequence is empty, and 't2.`t2b`' is not an aggregate function. Wrap '(avg(CAST(t2.`t2b` AS BIGINT)) AS `avg`)' in windowing function(s) or wrap 't2.`t2b`' in first() (or first_value) if you don't care which value you get.;
+grouping expressions sequence is empty, and 't2.t2b' is not an aggregate function. Wrap '(avg(t2.t2b) AS avg)' in windowing function(s) or wrap 't2.t2b' in first() (or first_value) if you don't care which value you get.
 
 
 -- !query
@@ -63,7 +63,7 @@ WHERE  t1a IN (SELECT   min(t2a)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Resolved attribute(s) t2b#x missing from min(t2a)#x,t2c#x in operator !Filter t2c#x IN (list#x [t2b#x]).;
+Resolved attribute(s) t2b#x missing from min(t2a)#x,t2c#x in operator !Filter t2c#x IN (list#x [t2b#x]).
 
 
 -- !query
@@ -78,7 +78,7 @@ HAVING EXISTS (SELECT t2a
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Found an aggregate expression in a correlated predicate that has both outer and local references, which is not supported yet. Aggregate expression: min((t1.`t1a` + t2.`t2a`)), Outer references: t1.`t1a`, Local references: t2.`t2a`.;
+Found an aggregate expression in a correlated predicate that has both outer and local references, which is not supported yet. Aggregate expression: min((t1.t1a + t2.t2a)), Outer references: t1.t1a, Local references: t2.t2a.
 
 
 -- !query
@@ -94,7 +94,7 @@ WHERE  t1a IN (SELECT t2a
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Found an aggregate expression in a correlated predicate that has both outer and local references, which is not supported yet. Aggregate expression: min((t2.`t2a` + t3.`t3a`)), Outer references: t2.`t2a`, Local references: t3.`t3a`.;
+Found an aggregate expression in a correlated predicate that has both outer and local references, which is not supported yet. Aggregate expression: min((t2.t2a + t3.t3a)), Outer references: t2.t2a, Local references: t3.t3a.
 
 
 -- !query
@@ -109,9 +109,10 @@ struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
 Expressions referencing the outer query are not supported outside of WHERE/HAVING clauses:
-Aggregate [min(outer(t2a#x)) AS min(outer(t2.`t2a`))#x]
+Aggregate [min(outer(t2a#x)) AS min(outer(t2.t2a))#x]
 +- SubqueryAlias t3
-   +- Project [t3a#x, t3b#x, t3c#x]
-      +- SubqueryAlias t3
-         +- LocalRelation [t3a#x, t3b#x, t3c#x]
-;
+   +- View (`t3`, [t3a#x,t3b#x,t3c#x])
+      +- Project [cast(t3a#x as int) AS t3a#x, cast(t3b#x as int) AS t3b#x, cast(t3c#x as int) AS t3c#x]
+         +- Project [t3a#x, t3b#x, t3c#x]
+            +- SubqueryAlias t3
+               +- LocalRelation [t3a#x, t3b#x, t3c#x]
diff --git a/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/subq-input-typecheck.sql.out b/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/subq-input-typecheck.sql.out
index 776598127075b..0b1bfdf50c5f8 100644
--- a/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/subq-input-typecheck.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/subq-input-typecheck.sql.out
@@ -64,7 +64,7 @@ FROM t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Scalar subquery must return only one column, but got 2;
+Scalar subquery must return only one column, but got 2
 
 
 -- !query
@@ -79,7 +79,7 @@ FROM t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Scalar subquery must return only one column, but got 2;
+Scalar subquery must return only one column, but got 2
 
 
 -- !query
@@ -92,15 +92,15 @@ t1a IN (SELECT t2a, t2b
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(t1.`t1a` IN (listquery(t1.`t1a`)))' due to data type mismatch: 
+cannot resolve '(t1.t1a IN (listquery(t1.t1a)))' due to data type mismatch: 
 The number of columns in the left hand side of an IN subquery does not match the
 number of columns in the output of subquery.
 #columns in left hand side: 1.
 #columns in right hand side: 2.
 Left side columns:
-[t1.`t1a`].
+[t1.t1a].
 Right side columns:
-[t2.`t2a`, t2.`t2b`].;
+[t2.t2a, t2.t2b].; line 3 pos 4
 
 
 -- !query
@@ -113,15 +113,15 @@ WHERE
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(named_struct('t1a', t1.`t1a`, 't1b', t1.`t1b`) IN (listquery(t1.`t1a`)))' due to data type mismatch: 
+cannot resolve '(named_struct('t1a', t1.t1a, 't1b', t1.t1b) IN (listquery(t1.t1a)))' due to data type mismatch: 
 The number of columns in the left hand side of an IN subquery does not match the
 number of columns in the output of subquery.
 #columns in left hand side: 2.
 #columns in right hand side: 1.
 Left side columns:
-[t1.`t1a`, t1.`t1b`].
+[t1.t1a, t1.t1b].
 Right side columns:
-[t2.`t2a`].;
+[t2.t2a].; line 3 pos 11
 
 
 -- !query
@@ -135,12 +135,12 @@ WHERE
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(named_struct('t4a', t4.`t4a`, 't4b', t4.`t4b`, 't4c', t4.`t4c`) IN (listquery()))' due to data type mismatch: 
+cannot resolve '(named_struct('t4a', t4.t4a, 't4b', t4.t4b, 't4c', t4.t4c) IN (listquery()))' due to data type mismatch: 
 The data type of one or more elements in the left hand side of an IN subquery
 is not compatible with the data type of the output of the subquery
 Mismatched columns:
-[(t4.`t4a`:double, t5.`t5a`:timestamp), (t4.`t4c`:string, t5.`t5c`:bigint)]
+[(t4.t4a:double, t5.t5a:timestamp), (t4.t4c:string, t5.t5c:bigint)]
 Left side:
 [double, string, string].
 Right side:
-[timestamp, string, bigint].;
+[timestamp, string, bigint].; line 3 pos 16
diff --git a/sql/core/src/test/resources/sql-tests/results/subquery/scalar-subquery/scalar-subquery-select.sql.out b/sql/core/src/test/resources/sql-tests/results/subquery/scalar-subquery/scalar-subquery-select.sql.out
index 184b8daf9d28e..16570c659dc38 100644
--- a/sql/core/src/test/resources/sql-tests/results/subquery/scalar-subquery/scalar-subquery-select.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/subquery/scalar-subquery/scalar-subquery-select.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 11
+-- Number of queries: 12
 
 
 -- !query
@@ -196,3 +196,29 @@ val1d	NULL
 val1e	10
 val1e	10
 val1e	10
+
+
+-- !query
+SELECT t1a,
+    (SELECT count(t2d) FROM t2 WHERE t2a = t1a) count_t2,
+    (SELECT count_if(t2d > 0) FROM t2 WHERE t2a = t1a) count_if_t2,
+    (SELECT approx_count_distinct(t2d) FROM t2 WHERE t2a = t1a) approx_count_distinct_t2,
+    (SELECT collect_list(t2d) FROM t2 WHERE t2a = t1a) collect_list_t2,
+    (SELECT collect_set(t2d) FROM t2 WHERE t2a = t1a) collect_set_t2,
+    (SELECT hex(count_min_sketch(t2d, 0.5d, 0.5d, 1)) FROM t2 WHERE t2a = t1a) collect_set_t2
+FROM t1
+-- !query schema
+struct<t1a:string,count_t2:bigint,count_if_t2:bigint,approx_count_distinct_t2:bigint,collect_list_t2:array<bigint>,collect_set_t2:array<bigint>,collect_set_t2:string>
+-- !query output
+val1a	0	0	0	[]	[]	0000000100000000000000000000000100000004000000005D8D6AB90000000000000000000000000000000000000000000000000000000000000000
+val1a	0	0	0	[]	[]	0000000100000000000000000000000100000004000000005D8D6AB90000000000000000000000000000000000000000000000000000000000000000
+val1a	0	0	0	[]	[]	0000000100000000000000000000000100000004000000005D8D6AB90000000000000000000000000000000000000000000000000000000000000000
+val1a	0	0	0	[]	[]	0000000100000000000000000000000100000004000000005D8D6AB90000000000000000000000000000000000000000000000000000000000000000
+val1b	6	6	3	[19,119,319,19,19,19]	[19,119,319]	0000000100000000000000060000000100000004000000005D8D6AB90000000000000000000000000000000400000000000000010000000000000001
+val1c	2	2	2	[219,19]	[219,19]	0000000100000000000000020000000100000004000000005D8D6AB90000000000000000000000000000000100000000000000000000000000000001
+val1d	0	0	0	[]	[]	0000000100000000000000000000000100000004000000005D8D6AB90000000000000000000000000000000000000000000000000000000000000000
+val1d	0	0	0	[]	[]	0000000100000000000000000000000100000004000000005D8D6AB90000000000000000000000000000000000000000000000000000000000000000
+val1d	0	0	0	[]	[]	0000000100000000000000000000000100000004000000005D8D6AB90000000000000000000000000000000000000000000000000000000000000000
+val1e	1	1	1	[19]	[19]	0000000100000000000000010000000100000004000000005D8D6AB90000000000000000000000000000000100000000000000000000000000000000
+val1e	1	1	1	[19]	[19]	0000000100000000000000010000000100000004000000005D8D6AB90000000000000000000000000000000100000000000000000000000000000000
+val1e	1	1	1	[19]	[19]	0000000100000000000000010000000100000004000000005D8D6AB90000000000000000000000000000000100000000000000000000000000000000
diff --git a/sql/core/src/test/resources/sql-tests/results/table-aliases.sql.out b/sql/core/src/test/resources/sql-tests/results/table-aliases.sql.out
index 25967a3968f23..c80f421c7061e 100644
--- a/sql/core/src/test/resources/sql-tests/results/table-aliases.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/table-aliases.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 11
+-- Number of queries: 15
 
 
 -- !query
@@ -60,7 +60,7 @@ SELECT a AS col1, b AS col2 FROM testData AS t(c, d)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`a`' given input columns: [c, d]; line 1 pos 7
+cannot resolve 'a' given input columns: [t.c, t.d]; line 1 pos 7
 
 
 -- !query
@@ -71,6 +71,30 @@ struct<col1:int,col2:int>
 1	1
 
 
+-- !query
+SELECT t.* FROM (SELECT 1 AS a, 1 AS b) t(col1, col2)
+-- !query schema
+struct<col1:int,col2:int>
+-- !query output
+1	1
+
+
+-- !query
+SELECT col1, col2 FROM (SELECT 1 AS a, 1 AS b) t(col1, col2)
+-- !query schema
+struct<col1:int,col2:int>
+-- !query output
+1	1
+
+
+-- !query
+SELECT t.col1, t.col2 FROM (SELECT 1 AS a, 1 AS b) t(col1, col2)
+-- !query schema
+struct<col1:int,col2:int>
+-- !query output
+1	1
+
+
 -- !query
 CREATE OR REPLACE TEMPORARY VIEW src1 AS SELECT * FROM VALUES (1, "a"), (2, "b"), (3, "c") AS src1(id, v1)
 -- !query schema
@@ -95,3 +119,13 @@ struct<a:int,b:string,c:int,d:decimal(2,1)>
 1	a	1	8.5
 2	b	2	1.0
 3	c	3	3.2
+
+
+-- !query
+SELECT dst.* FROM (src1 s1 INNER JOIN src2 s2 ON s1.id = s2.id) dst(a, b, c, d)
+-- !query schema
+struct<a:int,b:string,c:int,d:decimal(2,1)>
+-- !query output
+1	a	1	8.5
+2	b	2	1.0
+3	c	3	3.2
diff --git a/sql/core/src/test/resources/sql-tests/results/table-valued-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/table-valued-functions.sql.out
index 16d483df62fd5..157b2cebf3792 100644
--- a/sql/core/src/test/resources/sql-tests/results/table-valued-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/table-valued-functions.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 8
+-- Number of queries: 10
 
 
 -- !query
@@ -70,12 +70,12 @@ select * from range(1, 1, 1, 1, 1)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-error: table-valued function range with alternatives:
- (end: long)
- (start: long, end: long)
- (start: long, end: long, step: long)
- (start: long, end: long, step: long, numPartitions: integer)
-cannot be applied to: (integer, integer, integer, integer, integer); line 1 pos 14
+Table-valued function range with alternatives: 
+    range(start: long, end: long, step: long, numSlices: integer)
+    range(start: long, end: long, step: long)
+    range(start: long, end: long)
+    range(end: long)
+cannot be applied to (integer, integer, integer, integer, integer): Invalid number of arguments for function range. Expected: one of 1, 2, 3 and 4; Found: 5; line 1 pos 14
 
 
 -- !query
@@ -84,12 +84,40 @@ select * from range(1, null)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-error: table-valued function range with alternatives:
- (end: long)
- (start: long, end: long)
- (start: long, end: long, step: long)
- (start: long, end: long, step: long, numPartitions: integer)
-cannot be applied to: (integer, null); line 1 pos 14
+Table-valued function range with alternatives: 
+    range(start: long, end: long, step: long, numSlices: integer)
+    range(start: long, end: long, step: long)
+    range(start: long, end: long)
+    range(end: long)
+cannot be applied to (integer, null): Incompatible input data type. Expected: long; Found: null; line 1 pos 14
+
+
+-- !query
+select * from range(array(1, 2, 3))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Table-valued function range with alternatives: 
+    range(start: long, end: long, step: long, numSlices: integer)
+    range(start: long, end: long, step: long)
+    range(start: long, end: long)
+    range(end: long)
+cannot be applied to (array): Incompatible input data type. Expected: long; Found: array; line 1 pos 14
+
+
+-- !query
+select * from range(0, 5, 0)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Table-valued function range with alternatives: 
+    range(start: long, end: long, step: long, numSlices: integer)
+    range(start: long, end: long, step: long)
+    range(start: long, end: long)
+    range(end: long)
+cannot be applied to (integer, integer, integer): requirement failed: step (0) cannot be 0; line 1 pos 14
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/timezone.sql.out b/sql/core/src/test/resources/sql-tests/results/timezone.sql.out
new file mode 100644
index 0000000000000..d816043130737
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/timezone.sql.out
@@ -0,0 +1,135 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 12
+
+
+-- !query
+SET TIME ZONE 'Asia/Hong_Kong'
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+spark.sql.session.timeZone	Asia/Hong_Kong
+
+
+-- !query
+SET TIME ZONE 'GMT+1'
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+spark.sql.session.timeZone	GMT+1
+
+
+-- !query
+SET TIME ZONE INTERVAL 10 HOURS
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+spark.sql.session.timeZone	+10:00
+
+
+-- !query
+SET TIME ZONE INTERVAL '15:40:32' HOUR TO SECOND
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+spark.sql.session.timeZone	+15:40:32
+
+
+-- !query
+SET TIME ZONE LOCAL
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+spark.sql.session.timeZone	America/Los_Angeles
+
+
+-- !query
+SET TIME ZONE
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Invalid time zone displacement value(line 1, pos 0)
+
+== SQL ==
+SET TIME ZONE
+^^^
+
+
+-- !query
+SET TIME ZONE 'invalid/zone'
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Cannot resolve the given timezone with ZoneId.of(_, ZoneId.SHORT_IDS)
+
+
+-- !query
+SET TIME ZONE INTERVAL 3 DAYS
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+The interval value must be in the range of [-18, +18] hours with second precision(line 1, pos 14)
+
+== SQL ==
+SET TIME ZONE INTERVAL 3 DAYS
+--------------^^^
+
+
+-- !query
+SET TIME ZONE INTERVAL 24 HOURS
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+The interval value must be in the range of [-18, +18] hours with second precision(line 1, pos 14)
+
+== SQL ==
+SET TIME ZONE INTERVAL 24 HOURS
+--------------^^^
+
+
+-- !query
+SET TIME ZONE INTERVAL '19:40:32' HOUR TO SECOND
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+The interval value must be in the range of [-18, +18] hours with second precision(line 1, pos 14)
+
+== SQL ==
+SET TIME ZONE INTERVAL '19:40:32' HOUR TO SECOND
+--------------^^^
+
+
+-- !query
+SET TIME ZONE INTERVAL 10 HOURS 'GMT+1'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Invalid time zone displacement value(line 1, pos 0)
+
+== SQL ==
+SET TIME ZONE INTERVAL 10 HOURS 'GMT+1'
+^^^
+
+
+-- !query
+SET TIME ZONE INTERVAL 10 HOURS 1 MILLISECOND
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+The interval value must be in the range of [-18, +18] hours with second precision(line 1, pos 14)
+
+== SQL ==
+SET TIME ZONE INTERVAL 10 HOURS 1 MILLISECOND
+--------------^^^
diff --git a/sql/core/src/test/resources/sql-tests/results/transform.sql.out b/sql/core/src/test/resources/sql-tests/results/transform.sql.out
new file mode 100644
index 0000000000000..6f94e742b8776
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/transform.sql.out
@@ -0,0 +1,783 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 47
+
+
+-- !query
+CREATE OR REPLACE TEMPORARY VIEW t AS SELECT * FROM VALUES
+('1', true, unhex('537061726B2053514C'), tinyint(1), 1, smallint(100), bigint(1), float(1.0), 1.0, Decimal(1.0), timestamp('1997-01-02'), date('2000-04-01')),
+('2', false, unhex('537061726B2053514C'), tinyint(2), 2,  smallint(200), bigint(2), float(2.0), 2.0, Decimal(2.0), timestamp('1997-01-02 03:04:05'), date('2000-04-02')),
+('3', true, unhex('537061726B2053514C'), tinyint(3), 3, smallint(300), bigint(3), float(3.0), 3.0, Decimal(3.0), timestamp('1997-02-10 17:32:01-08'), date('2000-04-03'))
+AS t(a, b, c, d, e, f, g, h, i, j, k, l)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+CREATE OR REPLACE TEMPORARY VIEW script_trans AS SELECT * FROM VALUES
+(1, 2, 3),
+(4, 5, 6),
+(7, 8, 9)
+AS script_trans(a, b, c)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+CREATE OR REPLACE TEMPORARY VIEW complex_trans AS SELECT * FROM VALUES
+(1, 1),
+(1, 1),
+(2, 2),
+(2, 2),
+(3, 3),
+(2, 2),
+(3, 3),
+(1, 1),
+(3, 3)
+as complex_trans(a, b)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SELECT TRANSFORM(a)
+USING 'cat' AS (a)
+FROM t
+-- !query schema
+struct<a:string>
+-- !query output
+1
+2
+3
+
+
+-- !query
+SELECT a, b, decode(c, 'UTF-8'), d, e, f, g, h, i, j, k, l FROM (
+  SELECT TRANSFORM(a, b, c, d, e, f, g, h, i, j, k, l)
+  USING 'cat' AS (
+    a string,
+    b boolean,
+    c binary,
+    d tinyint,
+    e int,
+    f smallint,
+    g long,
+    h float,
+    i double,
+    j decimal(38, 18),
+    k timestamp,
+    l date)
+  FROM t
+) tmp
+-- !query schema
+struct<a:string,b:boolean,decode(c, UTF-8):string,d:tinyint,e:int,f:smallint,g:bigint,h:float,i:double,j:decimal(38,18),k:timestamp,l:date>
+-- !query output
+1	true	Spark SQL	1	1	100	1	1.0	1.0	1.000000000000000000	1997-01-02 00:00:00	2000-04-01
+2	false	Spark SQL	2	2	200	2	2.0	2.0	2.000000000000000000	1997-01-02 03:04:05	2000-04-02
+3	true	Spark SQL	3	3	300	3	3.0	3.0	3.000000000000000000	1997-02-10 17:32:01	2000-04-03
+
+
+-- !query
+SELECT a, b, decode(c, 'UTF-8'), d, e, f, g, h, i, j, k, l FROM (
+  SELECT TRANSFORM(a, b, c, d, e, f, g, h, i, j, k, l)
+  USING 'cat' AS (
+    a string,
+    b string,
+    c string,
+    d string,
+    e string,
+    f string,
+    g string,
+    h string,
+    i string,
+    j string,
+    k string,
+    l string)
+  FROM t
+) tmp
+-- !query schema
+struct<a:string,b:string,decode(c, UTF-8):string,d:string,e:string,f:string,g:string,h:string,i:string,j:string,k:string,l:string>
+-- !query output
+1	true	Spark SQL	1	1	100	1	1.0	1.0	1	1997-01-02 00:00:00	2000-04-01
+2	false	Spark SQL	2	2	200	2	2.0	2.0	2	1997-01-02 03:04:05	2000-04-02
+3	true	Spark SQL	3	3	300	3	3.0	3.0	3	1997-02-10 17:32:01	2000-04-03
+
+
+-- !query
+SELECT TRANSFORM(a)
+USING 'cat'
+FROM t
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+1	NULL
+2	NULL
+3	NULL
+
+
+-- !query
+SELECT TRANSFORM(a, b)
+USING 'cat'
+FROM t
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+1	true
+2	false
+3	true
+
+
+-- !query
+SELECT TRANSFORM(a, b, c)
+USING 'cat'
+FROM t
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+1	true
+2	false
+3	true
+
+
+-- !query
+SELECT TRANSFORM(a, b, c, d, e, f, g, h, i)
+USING 'cat' AS (a int, b short, c long, d byte, e float, f double, g decimal(38, 18), h date, i timestamp)
+FROM VALUES
+('a','','1231a','a','213.21a','213.21a','0a.21d','2000-04-01123','1997-0102 00:00:') tmp(a, b, c, d, e, f, g, h, i)
+-- !query schema
+struct<a:int,b:smallint,c:bigint,d:tinyint,e:float,f:double,g:decimal(38,18),h:date,i:timestamp>
+-- !query output
+NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL
+
+
+-- !query
+SELECT TRANSFORM(b, max(a), sum(f))
+USING 'cat' AS (a, b)
+FROM t
+GROUP BY b
+-- !query schema
+struct<a:string,b:string>
+-- !query output
+false	2
+true	3
+
+
+-- !query
+MAP a, b USING 'cat' AS (a, b) FROM t
+-- !query schema
+struct<a:string,b:string>
+-- !query output
+1	true
+2	false
+3	true
+
+
+-- !query
+REDUCE a, b USING 'cat' AS (a, b) FROM t
+-- !query schema
+struct<a:string,b:string>
+-- !query output
+1	true
+2	false
+3	true
+
+
+-- !query
+SELECT TRANSFORM(a, b, c, null)
+  ROW FORMAT DELIMITED
+  FIELDS TERMINATED BY '@'
+  LINES TERMINATED BY '\n'
+  NULL DEFINED AS 'NULL'
+USING 'cat' AS (a, b, c, d)
+  ROW FORMAT DELIMITED
+  FIELDS TERMINATED BY '@'
+  LINES TERMINATED BY '\n'
+  NULL DEFINED AS 'NULL'
+FROM t
+-- !query schema
+struct<a:string,b:string,c:string,d:string>
+-- !query output
+1	true	Spark SQL	null
+2	false	Spark SQL	null
+3	true	Spark SQL	null
+
+
+-- !query
+SELECT TRANSFORM(a, b, c, null)
+  ROW FORMAT DELIMITED
+  FIELDS TERMINATED BY '@'
+  LINES TERMINATED BY '\n'
+  NULL DEFINED AS 'NULL'
+USING 'cat' AS (d)
+  ROW FORMAT DELIMITED
+  FIELDS TERMINATED BY '@'
+  LINES TERMINATED BY '\n'
+  NULL DEFINED AS 'NULL'
+FROM t
+-- !query schema
+struct<d:string>
+-- !query output
+1
+2
+3
+
+
+-- !query
+SELECT a, b, decode(c, 'UTF-8'), d, e, f, g, h, i, j, k, l FROM (
+  SELECT TRANSFORM(a, b, c, d, e, f, g, h, i, j, k, l)
+    ROW FORMAT DELIMITED
+    FIELDS TERMINATED BY ','
+    LINES TERMINATED BY '\n'
+    NULL DEFINED AS 'NULL'
+    USING 'cat' AS (
+      a string,
+      b boolean,
+      c binary,
+      d tinyint,
+      e int,
+      f smallint,
+      g long,
+      h float,
+      i double,
+      j decimal(38, 18),
+      k timestamp,
+      l date)
+    ROW FORMAT DELIMITED
+    FIELDS TERMINATED BY ','
+    LINES TERMINATED BY '\n'
+    NULL DEFINED AS 'NULL'
+  FROM t
+) tmp
+-- !query schema
+struct<a:string,b:boolean,decode(c, UTF-8):string,d:tinyint,e:int,f:smallint,g:bigint,h:float,i:double,j:decimal(38,18),k:timestamp,l:date>
+-- !query output
+1	true	Spark SQL	1	1	100	1	1.0	1.0	1.000000000000000000	1997-01-02 00:00:00	2000-04-01
+2	false	Spark SQL	2	2	200	2	2.0	2.0	2.000000000000000000	1997-01-02 03:04:05	2000-04-02
+3	true	Spark SQL	3	3	300	3	3.0	3.0	3.000000000000000000	1997-02-10 17:32:01	2000-04-03
+
+
+-- !query
+SELECT a, b, decode(c, 'UTF-8'), d, e, f, g, h, i, j, k, l FROM (
+  SELECT TRANSFORM(a, b, c, d, e, f, g, h, i, j, k, l)
+    ROW FORMAT DELIMITED
+    FIELDS TERMINATED BY ','
+    LINES TERMINATED BY '\n'
+    NULL DEFINED AS 'NULL'
+    USING 'cat' AS (
+      a string,
+      b long,
+      c binary,
+      d tinyint,
+      e int,
+      f smallint,
+      g long,
+      h float,
+      i double,
+      j decimal(38, 18),
+      k int,
+      l long)
+    ROW FORMAT DELIMITED
+    FIELDS TERMINATED BY ','
+    LINES TERMINATED BY '\n'
+    NULL DEFINED AS 'NULL'
+  FROM t
+) tmp
+-- !query schema
+struct<a:string,b:bigint,decode(c, UTF-8):string,d:tinyint,e:int,f:smallint,g:bigint,h:float,i:double,j:decimal(38,18),k:int,l:bigint>
+-- !query output
+1	NULL	Spark SQL	1	1	100	1	1.0	1.0	1.000000000000000000	NULL	NULL
+2	NULL	Spark SQL	2	2	200	2	2.0	2.0	2.000000000000000000	NULL	NULL
+3	NULL	Spark SQL	3	3	300	3	3.0	3.0	3.000000000000000000	NULL	NULL
+
+
+-- !query
+SELECT a, b, decode(c, 'UTF-8'), d, e, f, g, h, i, j, k, l FROM (
+  SELECT TRANSFORM(a, b, c, d, e, f, g, h, i, j, k, l)
+    ROW FORMAT DELIMITED
+    FIELDS TERMINATED BY ','
+    LINES TERMINATED BY '@'
+    NULL DEFINED AS 'NULL'
+    USING 'cat' AS (
+      a string,
+      b string,
+      c string,
+      d string,
+      e string,
+      f string,
+      g string,
+      h string,
+      i string,
+      j string,
+      k string,
+      l string)
+    ROW FORMAT DELIMITED
+    FIELDS TERMINATED BY ','
+    LINES TERMINATED BY '@'
+    NULL DEFINED AS 'NULL'
+  FROM t
+) tmp
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+LINES TERMINATED BY only supports newline '\n' right now: @(line 3, pos 4)
+
+== SQL ==
+SELECT a, b, decode(c, 'UTF-8'), d, e, f, g, h, i, j, k, l FROM (
+  SELECT TRANSFORM(a, b, c, d, e, f, g, h, i, j, k, l)
+    ROW FORMAT DELIMITED
+----^^^
+    FIELDS TERMINATED BY ','
+    LINES TERMINATED BY '@'
+    NULL DEFINED AS 'NULL'
+    USING 'cat' AS (
+      a string,
+      b string,
+      c string,
+      d string,
+      e string,
+      f string,
+      g string,
+      h string,
+      i string,
+      j string,
+      k string,
+      l string)
+    ROW FORMAT DELIMITED
+    FIELDS TERMINATED BY ','
+    LINES TERMINATED BY '@'
+    NULL DEFINED AS 'NULL'
+  FROM t
+) tmp
+
+
+-- !query
+SELECT TRANSFORM(b, a, CAST(c AS STRING))
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 4
+-- !query schema
+struct<a:string,b:string,c:string>
+-- !query output
+2	1	3
+5	4	6
+
+
+-- !query
+SELECT TRANSFORM(1, 2, 3)
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 4
+-- !query schema
+struct<a:string,b:string,c:string>
+-- !query output
+1	2	3
+1	2	3
+
+
+-- !query
+SELECT TRANSFORM(1, 2)
+  USING 'cat' AS (a INT, b INT)
+FROM script_trans
+LIMIT 1
+-- !query schema
+struct<a:int,b:int>
+-- !query output
+1	2
+
+
+-- !query
+SELECT TRANSFORM(
+  b, a,
+  CASE
+    WHEN c > 100 THEN 1
+    WHEN c < 100 THEN 2
+  ELSE 3 END)
+  USING 'cat' AS (a, b,  c)
+FROM script_trans
+WHERE a <= 4
+-- !query schema
+struct<a:string,b:string,c:string>
+-- !query output
+2	1	2
+5	4	2
+
+
+-- !query
+SELECT TRANSFORM(b, a, c + 1)
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 4
+-- !query schema
+struct<a:string,b:string,c:string>
+-- !query output
+2	1	4
+5	4	7
+
+
+-- !query
+SELECT TRANSFORM(*)
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 4
+-- !query schema
+struct<a:string,b:string,c:string>
+-- !query output
+1	2	3
+4	5	6
+
+
+-- !query
+SELECT TRANSFORM(b, MAX(a), CAST(SUM(c) AS STRING))
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 4
+GROUP BY b
+-- !query schema
+struct<a:string,b:string,c:string>
+-- !query output
+2	1	3
+5	4	6
+
+
+-- !query
+SELECT TRANSFORM(b, MAX(a) FILTER (WHERE a > 3), CAST(SUM(c) AS STRING))
+  USING 'cat' AS (a,b,c)
+FROM script_trans
+WHERE a <= 4
+GROUP BY b
+-- !query schema
+struct<a:string,b:string,c:string>
+-- !query output
+2	null	3
+5	4	6
+
+
+-- !query
+SELECT TRANSFORM(b, MAX(a), CAST(sum(c) AS STRING))
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 2
+GROUP BY b
+-- !query schema
+struct<a:string,b:string,c:string>
+-- !query output
+2	1	3
+
+
+-- !query
+SELECT TRANSFORM(b, MAX(a), CAST(SUM(c) AS STRING))
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 4
+GROUP BY b
+HAVING MAX(a) > 0
+-- !query schema
+struct<a:string,b:string,c:string>
+-- !query output
+2	1	3
+5	4	6
+
+
+-- !query
+SELECT TRANSFORM(b, MAX(a), CAST(SUM(c) AS STRING))
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 4
+GROUP BY b
+HAVING MAX(a) > 1
+-- !query schema
+struct<a:string,b:string,c:string>
+-- !query output
+5	4	6
+
+
+-- !query
+SELECT TRANSFORM(b, MAX(a) OVER w, CAST(SUM(c) OVER w AS STRING))
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 4
+WINDOW w AS (PARTITION BY b ORDER BY a)
+-- !query schema
+struct<a:string,b:string,c:string>
+-- !query output
+2	1	3
+5	4	6
+
+
+-- !query
+SELECT TRANSFORM(b, MAX(a), CAST(SUM(c) AS STRING), myCol, myCol2)
+  USING 'cat' AS (a STRING, b STRING, c STRING, d ARRAY<INT>, e STRING)
+FROM script_trans
+LATERAL VIEW explode(array(array(1,2,3))) myTable AS myCol
+LATERAL VIEW explode(myTable.myCol) myTable2 AS myCol2
+WHERE a <= 4
+GROUP BY b, myCol, myCol2
+HAVING max(a) > 1
+-- !query schema
+struct<a:string,b:string,c:string,d:array<int>,e:string>
+-- !query output
+5	4	6	[1,2,3]	1
+5	4	6	[1,2,3]	2
+5	4	6	[1,2,3]	3
+
+
+-- !query
+FROM(
+  FROM script_trans
+  SELECT TRANSFORM(a, b)
+    USING 'cat' AS (`a` INT, b STRING)
+) t
+SELECT a + 1
+-- !query schema
+struct<(a + 1):int>
+-- !query output
+2
+5
+8
+
+
+-- !query
+FROM(
+  SELECT TRANSFORM(a, SUM(b))
+    USING 'cat' AS (`a` INT, b STRING)
+  FROM script_trans
+  GROUP BY a
+) t
+SELECT (b + 1) AS result
+ORDER BY result
+-- !query schema
+struct<result:double>
+-- !query output
+3.0
+6.0
+9.0
+
+
+-- !query
+MAP k / 10 USING 'cat' AS (one) FROM (SELECT 10 AS k)
+-- !query schema
+struct<one:string>
+-- !query output
+1.0
+
+
+-- !query
+FROM (SELECT 1 AS key, 100 AS value) src
+MAP src.*, src.key, CAST(src.key / 10 AS INT), CAST(src.key % 10 AS INT), src.value
+  USING 'cat' AS (k, v, tkey, ten, one, tvalue)
+-- !query schema
+struct<k:string,v:string,tkey:string,ten:string,one:string,tvalue:string>
+-- !query output
+1	100	1	0	1	100
+
+
+-- !query
+SELECT TRANSFORM(1)
+  USING 'cat' AS (a)
+FROM script_trans
+HAVING true
+-- !query schema
+struct<a:string>
+-- !query output
+1
+
+
+-- !query
+SET spark.sql.legacy.parser.havingWithoutGroupByAsWhere=true
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+spark.sql.legacy.parser.havingWithoutGroupByAsWhere	true
+
+
+-- !query
+SELECT TRANSFORM(1)
+  USING 'cat' AS (a)
+FROM script_trans
+HAVING true
+-- !query schema
+struct<a:string>
+-- !query output
+1
+1
+1
+
+
+-- !query
+SET spark.sql.legacy.parser.havingWithoutGroupByAsWhere=false
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+spark.sql.legacy.parser.havingWithoutGroupByAsWhere	false
+
+
+-- !query
+WITH temp AS (
+  SELECT TRANSFORM(a) USING 'cat' AS (b string) FROM t
+)
+SELECT t1.b FROM temp t1 JOIN temp t2 ON t1.b = t2.b
+-- !query schema
+struct<b:string>
+-- !query output
+1
+2
+3
+
+
+-- !query
+SELECT TRANSFORM(DISTINCT b, a, c)
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 4
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+TRANSFORM does not support DISTINCT/ALL in inputs(line 1, pos 17)
+
+== SQL ==
+SELECT TRANSFORM(DISTINCT b, a, c)
+-----------------^^^
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 4
+
+
+-- !query
+SELECT TRANSFORM(ALL b, a, c)
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 4
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+TRANSFORM does not support DISTINCT/ALL in inputs(line 1, pos 17)
+
+== SQL ==
+SELECT TRANSFORM(ALL b, a, c)
+-----------------^^^
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 4
+
+
+-- !query
+SELECT TRANSFORM(b AS b_1, MAX(a), CAST(sum(c) AS STRING))
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 2
+GROUP BY b
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+no viable alternative at input 'SELECT TRANSFORM(b AS'(line 1, pos 19)
+
+== SQL ==
+SELECT TRANSFORM(b AS b_1, MAX(a), CAST(sum(c) AS STRING))
+-------------------^^^
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 2
+GROUP BY b
+
+
+-- !query
+SELECT TRANSFORM(b b_1, MAX(a), CAST(sum(c) AS STRING))
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 2
+GROUP BY b
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+no viable alternative at input 'SELECT TRANSFORM(b b_1'(line 1, pos 19)
+
+== SQL ==
+SELECT TRANSFORM(b b_1, MAX(a), CAST(sum(c) AS STRING))
+-------------------^^^
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 2
+GROUP BY b
+
+
+-- !query
+SELECT TRANSFORM(b, MAX(a) AS max_a, CAST(sum(c) AS STRING))
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 2
+GROUP BY b
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+no viable alternative at input 'SELECT TRANSFORM(b, MAX(a) AS'(line 1, pos 27)
+
+== SQL ==
+SELECT TRANSFORM(b, MAX(a) AS max_a, CAST(sum(c) AS STRING))
+---------------------------^^^
+  USING 'cat' AS (a, b, c)
+FROM script_trans
+WHERE a <= 2
+GROUP BY b
+
+
+-- !query
+FROM (
+  SELECT TRANSFORM(a, b)
+    USING 'cat' AS (a, b)
+  FROM complex_trans
+  CLUSTER BY a
+) map_output
+SELECT TRANSFORM(a, b)
+  USING 'cat' AS (a, b)
+-- !query schema
+struct<a:string,b:string>
+-- !query output
+1	1
+1	1
+1	1
+2	2
+2	2
+2	2
+3	3
+3	3
+3	3
+
+
+-- !query
+FROM (
+  SELECT TRANSFORM(a, b)
+    USING 'cat' AS (a, b)
+  FROM complex_trans
+  ORDER BY a
+) map_output
+SELECT TRANSFORM(a, b)
+  USING 'cat' AS (a, b)
+-- !query schema
+struct<a:string,b:string>
+-- !query output
+1	1
+1	1
+1	1
+2	2
+2	2
+2	2
+3	3
+3	3
+3	3
diff --git a/sql/core/src/test/resources/sql-tests/results/try_arithmetic.sql.out b/sql/core/src/test/resources/sql-tests/results/try_arithmetic.sql.out
new file mode 100644
index 0000000000000..490ff50cc8782
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/try_arithmetic.sql.out
@@ -0,0 +1,66 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 8
+
+
+-- !query
+SELECT try_add(1, 1)
+-- !query schema
+struct<try_add(1, 1):int>
+-- !query output
+2
+
+
+-- !query
+SELECT try_add(2147483647, 1)
+-- !query schema
+struct<try_add(2147483647, 1):int>
+-- !query output
+NULL
+
+
+-- !query
+SELECT try_add(-2147483648, -1)
+-- !query schema
+struct<try_add(-2147483648, -1):int>
+-- !query output
+NULL
+
+
+-- !query
+SELECT try_add(9223372036854775807L, 1)
+-- !query schema
+struct<try_add(9223372036854775807, 1):bigint>
+-- !query output
+NULL
+
+
+-- !query
+SELECT try_add(-9223372036854775808L, -1)
+-- !query schema
+struct<try_add(-9223372036854775808, -1):bigint>
+-- !query output
+NULL
+
+
+-- !query
+SELECT try_divide(1, 0.5)
+-- !query schema
+struct<try_divide(1, 0.5):decimal(8,6)>
+-- !query output
+2.000000
+
+
+-- !query
+SELECT try_divide(1, 0)
+-- !query schema
+struct<try_divide(1, 0):double>
+-- !query output
+NULL
+
+
+-- !query
+SELECT try_divide(0, 0)
+-- !query schema
+struct<try_divide(0, 0):double>
+-- !query output
+NULL
diff --git a/sql/core/src/test/resources/sql-tests/results/try_cast.sql.out b/sql/core/src/test/resources/sql-tests/results/try_cast.sql.out
new file mode 100644
index 0000000000000..8be8d6be3437e
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/try_cast.sql.out
@@ -0,0 +1,234 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 29
+
+
+-- !query
+SELECT TRY_CAST('1.23' AS int)
+-- !query schema
+struct<TRY_CAST(1.23 AS INT):int>
+-- !query output
+NULL
+
+
+-- !query
+SELECT TRY_CAST('1.23' AS long)
+-- !query schema
+struct<TRY_CAST(1.23 AS BIGINT):bigint>
+-- !query output
+NULL
+
+
+-- !query
+SELECT TRY_CAST('-4.56' AS int)
+-- !query schema
+struct<TRY_CAST(-4.56 AS INT):int>
+-- !query output
+NULL
+
+
+-- !query
+SELECT TRY_CAST('-4.56' AS long)
+-- !query schema
+struct<TRY_CAST(-4.56 AS BIGINT):bigint>
+-- !query output
+NULL
+
+
+-- !query
+SELECT TRY_CAST('abc' AS int)
+-- !query schema
+struct<TRY_CAST(abc AS INT):int>
+-- !query output
+NULL
+
+
+-- !query
+SELECT TRY_CAST('abc' AS long)
+-- !query schema
+struct<TRY_CAST(abc AS BIGINT):bigint>
+-- !query output
+NULL
+
+
+-- !query
+SELECT TRY_CAST('' AS int)
+-- !query schema
+struct<TRY_CAST( AS INT):int>
+-- !query output
+NULL
+
+
+-- !query
+SELECT TRY_CAST('' AS long)
+-- !query schema
+struct<TRY_CAST( AS BIGINT):bigint>
+-- !query output
+NULL
+
+
+-- !query
+SELECT TRY_CAST(NULL AS int)
+-- !query schema
+struct<TRY_CAST(NULL AS INT):int>
+-- !query output
+NULL
+
+
+-- !query
+SELECT TRY_CAST(NULL AS long)
+-- !query schema
+struct<TRY_CAST(NULL AS BIGINT):bigint>
+-- !query output
+NULL
+
+
+-- !query
+SELECT TRY_CAST('123.a' AS int)
+-- !query schema
+struct<TRY_CAST(123.a AS INT):int>
+-- !query output
+NULL
+
+
+-- !query
+SELECT TRY_CAST('123.a' AS long)
+-- !query schema
+struct<TRY_CAST(123.a AS BIGINT):bigint>
+-- !query output
+NULL
+
+
+-- !query
+SELECT TRY_CAST('-2147483648' AS int)
+-- !query schema
+struct<TRY_CAST(-2147483648 AS INT):int>
+-- !query output
+-2147483648
+
+
+-- !query
+SELECT TRY_CAST('-2147483649' AS int)
+-- !query schema
+struct<TRY_CAST(-2147483649 AS INT):int>
+-- !query output
+NULL
+
+
+-- !query
+SELECT TRY_CAST('2147483647' AS int)
+-- !query schema
+struct<TRY_CAST(2147483647 AS INT):int>
+-- !query output
+2147483647
+
+
+-- !query
+SELECT TRY_CAST('2147483648' AS int)
+-- !query schema
+struct<TRY_CAST(2147483648 AS INT):int>
+-- !query output
+NULL
+
+
+-- !query
+SELECT TRY_CAST('-9223372036854775808' AS long)
+-- !query schema
+struct<TRY_CAST(-9223372036854775808 AS BIGINT):bigint>
+-- !query output
+-9223372036854775808
+
+
+-- !query
+SELECT TRY_CAST('-9223372036854775809' AS long)
+-- !query schema
+struct<TRY_CAST(-9223372036854775809 AS BIGINT):bigint>
+-- !query output
+NULL
+
+
+-- !query
+SELECT TRY_CAST('9223372036854775807' AS long)
+-- !query schema
+struct<TRY_CAST(9223372036854775807 AS BIGINT):bigint>
+-- !query output
+9223372036854775807
+
+
+-- !query
+SELECT TRY_CAST('9223372036854775808' AS long)
+-- !query schema
+struct<TRY_CAST(9223372036854775808 AS BIGINT):bigint>
+-- !query output
+NULL
+
+
+-- !query
+SELECT TRY_CAST('interval 3 month 1 hour' AS interval)
+-- !query schema
+struct<TRY_CAST(interval 3 month 1 hour AS INTERVAL):interval>
+-- !query output
+3 months 1 hours
+
+
+-- !query
+SELECT TRY_CAST('abc' AS interval)
+-- !query schema
+struct<TRY_CAST(abc AS INTERVAL):interval>
+-- !query output
+NULL
+
+
+-- !query
+select TRY_CAST('true' as boolean)
+-- !query schema
+struct<TRY_CAST(true AS BOOLEAN):boolean>
+-- !query output
+true
+
+
+-- !query
+select TRY_CAST('false' as boolean)
+-- !query schema
+struct<TRY_CAST(false AS BOOLEAN):boolean>
+-- !query output
+false
+
+
+-- !query
+select TRY_CAST('abc' as boolean)
+-- !query schema
+struct<TRY_CAST(abc AS BOOLEAN):boolean>
+-- !query output
+NULL
+
+
+-- !query
+SELECT TRY_CAST("2021-01-01" AS date)
+-- !query schema
+struct<TRY_CAST(2021-01-01 AS DATE):date>
+-- !query output
+2021-01-01
+
+
+-- !query
+SELECT TRY_CAST("2021-101-01" AS date)
+-- !query schema
+struct<TRY_CAST(2021-101-01 AS DATE):date>
+-- !query output
+NULL
+
+
+-- !query
+SELECT TRY_CAST("2021-01-01 00:00:00" AS timestamp)
+-- !query schema
+struct<TRY_CAST(2021-01-01 00:00:00 AS TIMESTAMP):timestamp>
+-- !query output
+2021-01-01 00:00:00
+
+
+-- !query
+SELECT TRY_CAST("2021-101-01 00:00:00" AS timestamp)
+-- !query schema
+struct<TRY_CAST(2021-101-01 00:00:00 AS TIMESTAMP):timestamp>
+-- !query output
+NULL
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/arrayJoin.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/arrayJoin.sql.out
index b49e6b5f21b16..d23b86651457f 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/arrayJoin.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/arrayJoin.sql.out
@@ -61,7 +61,7 @@ struct<array_join(array(2.0, 1.0), , ):string>
 -- !query
 SELECT array_join(array(float(2.0), float(1.0)), ', ')
 -- !query schema
-struct<array_join(array(CAST(2.0 AS FLOAT), CAST(1.0 AS FLOAT)), , ):string>
+struct<array_join(array(2.0, 1.0), , ):string>
 -- !query output
 2.0, 1.0
 
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/binaryComparison.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/binaryComparison.sql.out
index 0e1a3d0bc4d9e..4cc005d215517 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/binaryComparison.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/binaryComparison.sql.out
@@ -13,7 +13,7 @@ struct<>
 -- !query
 SELECT cast(1 as binary) = '1' FROM t
 -- !query schema
-struct<(CAST(1 AS BINARY) = CAST(1 AS BINARY)):boolean>
+struct<(CAST(1 AS BINARY) = 1):boolean>
 -- !query output
 false
 
@@ -21,7 +21,7 @@ false
 -- !query
 SELECT cast(1 as binary) > '2' FROM t
 -- !query schema
-struct<(CAST(1 AS BINARY) > CAST(2 AS BINARY)):boolean>
+struct<(CAST(1 AS BINARY) > 2):boolean>
 -- !query output
 false
 
@@ -29,7 +29,7 @@ false
 -- !query
 SELECT cast(1 as binary) >= '2' FROM t
 -- !query schema
-struct<(CAST(1 AS BINARY) >= CAST(2 AS BINARY)):boolean>
+struct<(CAST(1 AS BINARY) >= 2):boolean>
 -- !query output
 false
 
@@ -37,7 +37,7 @@ false
 -- !query
 SELECT cast(1 as binary) < '2' FROM t
 -- !query schema
-struct<(CAST(1 AS BINARY) < CAST(2 AS BINARY)):boolean>
+struct<(CAST(1 AS BINARY) < 2):boolean>
 -- !query output
 true
 
@@ -45,7 +45,7 @@ true
 -- !query
 SELECT cast(1 as binary) <= '2' FROM t
 -- !query schema
-struct<(CAST(1 AS BINARY) <= CAST(2 AS BINARY)):boolean>
+struct<(CAST(1 AS BINARY) <= 2):boolean>
 -- !query output
 true
 
@@ -53,7 +53,7 @@ true
 -- !query
 SELECT cast(1 as binary) <> '2' FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS BINARY) = CAST(2 AS BINARY))):boolean>
+struct<(NOT (CAST(1 AS BINARY) = 2)):boolean>
 -- !query output
 true
 
@@ -61,7 +61,7 @@ true
 -- !query
 SELECT cast(1 as binary) = cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS BINARY) = CAST(CAST(NULL AS STRING) AS BINARY)):boolean>
+struct<(CAST(1 AS BINARY) = CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -69,7 +69,7 @@ NULL
 -- !query
 SELECT cast(1 as binary) > cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS BINARY) > CAST(CAST(NULL AS STRING) AS BINARY)):boolean>
+struct<(CAST(1 AS BINARY) > CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -77,7 +77,7 @@ NULL
 -- !query
 SELECT cast(1 as binary) >= cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS BINARY) >= CAST(CAST(NULL AS STRING) AS BINARY)):boolean>
+struct<(CAST(1 AS BINARY) >= CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -85,7 +85,7 @@ NULL
 -- !query
 SELECT cast(1 as binary) < cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS BINARY) < CAST(CAST(NULL AS STRING) AS BINARY)):boolean>
+struct<(CAST(1 AS BINARY) < CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -93,7 +93,7 @@ NULL
 -- !query
 SELECT cast(1 as binary) <= cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS BINARY) <= CAST(CAST(NULL AS STRING) AS BINARY)):boolean>
+struct<(CAST(1 AS BINARY) <= CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -101,7 +101,7 @@ NULL
 -- !query
 SELECT cast(1 as binary) <> cast(null as string) FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS BINARY) = CAST(CAST(NULL AS STRING) AS BINARY))):boolean>
+struct<(NOT (CAST(1 AS BINARY) = CAST(NULL AS STRING))):boolean>
 -- !query output
 NULL
 
@@ -109,7 +109,7 @@ NULL
 -- !query
 SELECT '1' = cast(1 as binary) FROM t
 -- !query schema
-struct<(CAST(1 AS BINARY) = CAST(1 AS BINARY)):boolean>
+struct<(1 = CAST(1 AS BINARY)):boolean>
 -- !query output
 false
 
@@ -117,7 +117,7 @@ false
 -- !query
 SELECT '2' > cast(1 as binary) FROM t
 -- !query schema
-struct<(CAST(2 AS BINARY) > CAST(1 AS BINARY)):boolean>
+struct<(2 > CAST(1 AS BINARY)):boolean>
 -- !query output
 true
 
@@ -125,7 +125,7 @@ true
 -- !query
 SELECT '2' >= cast(1 as binary) FROM t
 -- !query schema
-struct<(CAST(2 AS BINARY) >= CAST(1 AS BINARY)):boolean>
+struct<(2 >= CAST(1 AS BINARY)):boolean>
 -- !query output
 true
 
@@ -133,7 +133,7 @@ true
 -- !query
 SELECT '2' < cast(1 as binary) FROM t
 -- !query schema
-struct<(CAST(2 AS BINARY) < CAST(1 AS BINARY)):boolean>
+struct<(2 < CAST(1 AS BINARY)):boolean>
 -- !query output
 false
 
@@ -141,7 +141,7 @@ false
 -- !query
 SELECT '2' <= cast(1 as binary) FROM t
 -- !query schema
-struct<(CAST(2 AS BINARY) <= CAST(1 AS BINARY)):boolean>
+struct<(2 <= CAST(1 AS BINARY)):boolean>
 -- !query output
 false
 
@@ -149,7 +149,7 @@ false
 -- !query
 SELECT '2' <> cast(1 as binary) FROM t
 -- !query schema
-struct<(NOT (CAST(2 AS BINARY) = CAST(1 AS BINARY))):boolean>
+struct<(NOT (2 = CAST(1 AS BINARY))):boolean>
 -- !query output
 true
 
@@ -157,7 +157,7 @@ true
 -- !query
 SELECT cast(null as string) = cast(1 as binary) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS BINARY) = CAST(1 AS BINARY)):boolean>
+struct<(CAST(NULL AS STRING) = CAST(1 AS BINARY)):boolean>
 -- !query output
 NULL
 
@@ -165,7 +165,7 @@ NULL
 -- !query
 SELECT cast(null as string) > cast(1 as binary) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS BINARY) > CAST(1 AS BINARY)):boolean>
+struct<(CAST(NULL AS STRING) > CAST(1 AS BINARY)):boolean>
 -- !query output
 NULL
 
@@ -173,7 +173,7 @@ NULL
 -- !query
 SELECT cast(null as string) >= cast(1 as binary) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS BINARY) >= CAST(1 AS BINARY)):boolean>
+struct<(CAST(NULL AS STRING) >= CAST(1 AS BINARY)):boolean>
 -- !query output
 NULL
 
@@ -181,7 +181,7 @@ NULL
 -- !query
 SELECT cast(null as string) < cast(1 as binary) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS BINARY) < CAST(1 AS BINARY)):boolean>
+struct<(CAST(NULL AS STRING) < CAST(1 AS BINARY)):boolean>
 -- !query output
 NULL
 
@@ -189,7 +189,7 @@ NULL
 -- !query
 SELECT cast(null as string) <= cast(1 as binary) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS BINARY) <= CAST(1 AS BINARY)):boolean>
+struct<(CAST(NULL AS STRING) <= CAST(1 AS BINARY)):boolean>
 -- !query output
 NULL
 
@@ -197,7 +197,7 @@ NULL
 -- !query
 SELECT cast(null as string) <> cast(1 as binary) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(NULL AS STRING) AS BINARY) = CAST(1 AS BINARY))):boolean>
+struct<(NOT (CAST(NULL AS STRING) = CAST(1 AS BINARY))):boolean>
 -- !query output
 NULL
 
@@ -205,7 +205,7 @@ NULL
 -- !query
 SELECT cast(1 as tinyint) = '1' FROM t
 -- !query schema
-struct<(CAST(1 AS TINYINT) = CAST(1 AS TINYINT)):boolean>
+struct<(CAST(1 AS TINYINT) = 1):boolean>
 -- !query output
 true
 
@@ -213,7 +213,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) > '2' FROM t
 -- !query schema
-struct<(CAST(1 AS TINYINT) > CAST(2 AS TINYINT)):boolean>
+struct<(CAST(1 AS TINYINT) > 2):boolean>
 -- !query output
 false
 
@@ -221,7 +221,7 @@ false
 -- !query
 SELECT cast(1 as tinyint) >= '2' FROM t
 -- !query schema
-struct<(CAST(1 AS TINYINT) >= CAST(2 AS TINYINT)):boolean>
+struct<(CAST(1 AS TINYINT) >= 2):boolean>
 -- !query output
 false
 
@@ -229,7 +229,7 @@ false
 -- !query
 SELECT cast(1 as tinyint) < '2' FROM t
 -- !query schema
-struct<(CAST(1 AS TINYINT) < CAST(2 AS TINYINT)):boolean>
+struct<(CAST(1 AS TINYINT) < 2):boolean>
 -- !query output
 true
 
@@ -237,7 +237,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) <= '2' FROM t
 -- !query schema
-struct<(CAST(1 AS TINYINT) <= CAST(2 AS TINYINT)):boolean>
+struct<(CAST(1 AS TINYINT) <= 2):boolean>
 -- !query output
 true
 
@@ -245,7 +245,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) <> '2' FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS TINYINT) = CAST(2 AS TINYINT))):boolean>
+struct<(NOT (CAST(1 AS TINYINT) = 2)):boolean>
 -- !query output
 true
 
@@ -253,7 +253,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) = cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS TINYINT) = CAST(CAST(NULL AS STRING) AS TINYINT)):boolean>
+struct<(CAST(1 AS TINYINT) = CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -261,7 +261,7 @@ NULL
 -- !query
 SELECT cast(1 as tinyint) > cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS TINYINT) > CAST(CAST(NULL AS STRING) AS TINYINT)):boolean>
+struct<(CAST(1 AS TINYINT) > CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -269,7 +269,7 @@ NULL
 -- !query
 SELECT cast(1 as tinyint) >= cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS TINYINT) >= CAST(CAST(NULL AS STRING) AS TINYINT)):boolean>
+struct<(CAST(1 AS TINYINT) >= CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -277,7 +277,7 @@ NULL
 -- !query
 SELECT cast(1 as tinyint) < cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS TINYINT) < CAST(CAST(NULL AS STRING) AS TINYINT)):boolean>
+struct<(CAST(1 AS TINYINT) < CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -285,7 +285,7 @@ NULL
 -- !query
 SELECT cast(1 as tinyint) <= cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS TINYINT) <= CAST(CAST(NULL AS STRING) AS TINYINT)):boolean>
+struct<(CAST(1 AS TINYINT) <= CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -293,7 +293,7 @@ NULL
 -- !query
 SELECT cast(1 as tinyint) <> cast(null as string) FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS TINYINT) = CAST(CAST(NULL AS STRING) AS TINYINT))):boolean>
+struct<(NOT (CAST(1 AS TINYINT) = CAST(NULL AS STRING))):boolean>
 -- !query output
 NULL
 
@@ -301,7 +301,7 @@ NULL
 -- !query
 SELECT '1' = cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(1 AS TINYINT) = CAST(1 AS TINYINT)):boolean>
+struct<(1 = CAST(1 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -309,7 +309,7 @@ true
 -- !query
 SELECT '2' > cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(2 AS TINYINT) > CAST(1 AS TINYINT)):boolean>
+struct<(2 > CAST(1 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -317,7 +317,7 @@ true
 -- !query
 SELECT '2' >= cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(2 AS TINYINT) >= CAST(1 AS TINYINT)):boolean>
+struct<(2 >= CAST(1 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -325,7 +325,7 @@ true
 -- !query
 SELECT '2' < cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(2 AS TINYINT) < CAST(1 AS TINYINT)):boolean>
+struct<(2 < CAST(1 AS TINYINT)):boolean>
 -- !query output
 false
 
@@ -333,7 +333,7 @@ false
 -- !query
 SELECT '2' <= cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(2 AS TINYINT) <= CAST(1 AS TINYINT)):boolean>
+struct<(2 <= CAST(1 AS TINYINT)):boolean>
 -- !query output
 false
 
@@ -341,7 +341,7 @@ false
 -- !query
 SELECT '2' <> cast(1 as tinyint) FROM t
 -- !query schema
-struct<(NOT (CAST(2 AS TINYINT) = CAST(1 AS TINYINT))):boolean>
+struct<(NOT (2 = CAST(1 AS TINYINT))):boolean>
 -- !query output
 true
 
@@ -349,7 +349,7 @@ true
 -- !query
 SELECT cast(null as string) = cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS TINYINT) = CAST(1 AS TINYINT)):boolean>
+struct<(CAST(NULL AS STRING) = CAST(1 AS TINYINT)):boolean>
 -- !query output
 NULL
 
@@ -357,7 +357,7 @@ NULL
 -- !query
 SELECT cast(null as string) > cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS TINYINT) > CAST(1 AS TINYINT)):boolean>
+struct<(CAST(NULL AS STRING) > CAST(1 AS TINYINT)):boolean>
 -- !query output
 NULL
 
@@ -365,7 +365,7 @@ NULL
 -- !query
 SELECT cast(null as string) >= cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS TINYINT) >= CAST(1 AS TINYINT)):boolean>
+struct<(CAST(NULL AS STRING) >= CAST(1 AS TINYINT)):boolean>
 -- !query output
 NULL
 
@@ -373,7 +373,7 @@ NULL
 -- !query
 SELECT cast(null as string) < cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS TINYINT) < CAST(1 AS TINYINT)):boolean>
+struct<(CAST(NULL AS STRING) < CAST(1 AS TINYINT)):boolean>
 -- !query output
 NULL
 
@@ -381,7 +381,7 @@ NULL
 -- !query
 SELECT cast(null as string) <= cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS TINYINT) <= CAST(1 AS TINYINT)):boolean>
+struct<(CAST(NULL AS STRING) <= CAST(1 AS TINYINT)):boolean>
 -- !query output
 NULL
 
@@ -389,7 +389,7 @@ NULL
 -- !query
 SELECT cast(null as string) <> cast(1 as tinyint) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(NULL AS STRING) AS TINYINT) = CAST(1 AS TINYINT))):boolean>
+struct<(NOT (CAST(NULL AS STRING) = CAST(1 AS TINYINT))):boolean>
 -- !query output
 NULL
 
@@ -397,7 +397,7 @@ NULL
 -- !query
 SELECT cast(1 as smallint) = '1' FROM t
 -- !query schema
-struct<(CAST(1 AS SMALLINT) = CAST(1 AS SMALLINT)):boolean>
+struct<(CAST(1 AS SMALLINT) = 1):boolean>
 -- !query output
 true
 
@@ -405,7 +405,7 @@ true
 -- !query
 SELECT cast(1 as smallint) > '2' FROM t
 -- !query schema
-struct<(CAST(1 AS SMALLINT) > CAST(2 AS SMALLINT)):boolean>
+struct<(CAST(1 AS SMALLINT) > 2):boolean>
 -- !query output
 false
 
@@ -413,7 +413,7 @@ false
 -- !query
 SELECT cast(1 as smallint) >= '2' FROM t
 -- !query schema
-struct<(CAST(1 AS SMALLINT) >= CAST(2 AS SMALLINT)):boolean>
+struct<(CAST(1 AS SMALLINT) >= 2):boolean>
 -- !query output
 false
 
@@ -421,7 +421,7 @@ false
 -- !query
 SELECT cast(1 as smallint) < '2' FROM t
 -- !query schema
-struct<(CAST(1 AS SMALLINT) < CAST(2 AS SMALLINT)):boolean>
+struct<(CAST(1 AS SMALLINT) < 2):boolean>
 -- !query output
 true
 
@@ -429,7 +429,7 @@ true
 -- !query
 SELECT cast(1 as smallint) <= '2' FROM t
 -- !query schema
-struct<(CAST(1 AS SMALLINT) <= CAST(2 AS SMALLINT)):boolean>
+struct<(CAST(1 AS SMALLINT) <= 2):boolean>
 -- !query output
 true
 
@@ -437,7 +437,7 @@ true
 -- !query
 SELECT cast(1 as smallint) <> '2' FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS SMALLINT) = CAST(2 AS SMALLINT))):boolean>
+struct<(NOT (CAST(1 AS SMALLINT) = 2)):boolean>
 -- !query output
 true
 
@@ -445,7 +445,7 @@ true
 -- !query
 SELECT cast(1 as smallint) = cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS SMALLINT) = CAST(CAST(NULL AS STRING) AS SMALLINT)):boolean>
+struct<(CAST(1 AS SMALLINT) = CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -453,7 +453,7 @@ NULL
 -- !query
 SELECT cast(1 as smallint) > cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS SMALLINT) > CAST(CAST(NULL AS STRING) AS SMALLINT)):boolean>
+struct<(CAST(1 AS SMALLINT) > CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -461,7 +461,7 @@ NULL
 -- !query
 SELECT cast(1 as smallint) >= cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS SMALLINT) >= CAST(CAST(NULL AS STRING) AS SMALLINT)):boolean>
+struct<(CAST(1 AS SMALLINT) >= CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -469,7 +469,7 @@ NULL
 -- !query
 SELECT cast(1 as smallint) < cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS SMALLINT) < CAST(CAST(NULL AS STRING) AS SMALLINT)):boolean>
+struct<(CAST(1 AS SMALLINT) < CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -477,7 +477,7 @@ NULL
 -- !query
 SELECT cast(1 as smallint) <= cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS SMALLINT) <= CAST(CAST(NULL AS STRING) AS SMALLINT)):boolean>
+struct<(CAST(1 AS SMALLINT) <= CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -485,7 +485,7 @@ NULL
 -- !query
 SELECT cast(1 as smallint) <> cast(null as string) FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS SMALLINT) = CAST(CAST(NULL AS STRING) AS SMALLINT))):boolean>
+struct<(NOT (CAST(1 AS SMALLINT) = CAST(NULL AS STRING))):boolean>
 -- !query output
 NULL
 
@@ -493,7 +493,7 @@ NULL
 -- !query
 SELECT '1' = cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(1 AS SMALLINT) = CAST(1 AS SMALLINT)):boolean>
+struct<(1 = CAST(1 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -501,7 +501,7 @@ true
 -- !query
 SELECT '2' > cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(2 AS SMALLINT) > CAST(1 AS SMALLINT)):boolean>
+struct<(2 > CAST(1 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -509,7 +509,7 @@ true
 -- !query
 SELECT '2' >= cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(2 AS SMALLINT) >= CAST(1 AS SMALLINT)):boolean>
+struct<(2 >= CAST(1 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -517,7 +517,7 @@ true
 -- !query
 SELECT '2' < cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(2 AS SMALLINT) < CAST(1 AS SMALLINT)):boolean>
+struct<(2 < CAST(1 AS SMALLINT)):boolean>
 -- !query output
 false
 
@@ -525,7 +525,7 @@ false
 -- !query
 SELECT '2' <= cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(2 AS SMALLINT) <= CAST(1 AS SMALLINT)):boolean>
+struct<(2 <= CAST(1 AS SMALLINT)):boolean>
 -- !query output
 false
 
@@ -533,7 +533,7 @@ false
 -- !query
 SELECT '2' <> cast(1 as smallint) FROM t
 -- !query schema
-struct<(NOT (CAST(2 AS SMALLINT) = CAST(1 AS SMALLINT))):boolean>
+struct<(NOT (2 = CAST(1 AS SMALLINT))):boolean>
 -- !query output
 true
 
@@ -541,7 +541,7 @@ true
 -- !query
 SELECT cast(null as string) = cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS SMALLINT) = CAST(1 AS SMALLINT)):boolean>
+struct<(CAST(NULL AS STRING) = CAST(1 AS SMALLINT)):boolean>
 -- !query output
 NULL
 
@@ -549,7 +549,7 @@ NULL
 -- !query
 SELECT cast(null as string) > cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS SMALLINT) > CAST(1 AS SMALLINT)):boolean>
+struct<(CAST(NULL AS STRING) > CAST(1 AS SMALLINT)):boolean>
 -- !query output
 NULL
 
@@ -557,7 +557,7 @@ NULL
 -- !query
 SELECT cast(null as string) >= cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS SMALLINT) >= CAST(1 AS SMALLINT)):boolean>
+struct<(CAST(NULL AS STRING) >= CAST(1 AS SMALLINT)):boolean>
 -- !query output
 NULL
 
@@ -565,7 +565,7 @@ NULL
 -- !query
 SELECT cast(null as string) < cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS SMALLINT) < CAST(1 AS SMALLINT)):boolean>
+struct<(CAST(NULL AS STRING) < CAST(1 AS SMALLINT)):boolean>
 -- !query output
 NULL
 
@@ -573,7 +573,7 @@ NULL
 -- !query
 SELECT cast(null as string) <= cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS SMALLINT) <= CAST(1 AS SMALLINT)):boolean>
+struct<(CAST(NULL AS STRING) <= CAST(1 AS SMALLINT)):boolean>
 -- !query output
 NULL
 
@@ -581,7 +581,7 @@ NULL
 -- !query
 SELECT cast(null as string) <> cast(1 as smallint) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(NULL AS STRING) AS SMALLINT) = CAST(1 AS SMALLINT))):boolean>
+struct<(NOT (CAST(NULL AS STRING) = CAST(1 AS SMALLINT))):boolean>
 -- !query output
 NULL
 
@@ -589,7 +589,7 @@ NULL
 -- !query
 SELECT cast(1 as int) = '1' FROM t
 -- !query schema
-struct<(CAST(1 AS INT) = CAST(1 AS INT)):boolean>
+struct<(CAST(1 AS INT) = 1):boolean>
 -- !query output
 true
 
@@ -597,7 +597,7 @@ true
 -- !query
 SELECT cast(1 as int) > '2' FROM t
 -- !query schema
-struct<(CAST(1 AS INT) > CAST(2 AS INT)):boolean>
+struct<(CAST(1 AS INT) > 2):boolean>
 -- !query output
 false
 
@@ -605,7 +605,7 @@ false
 -- !query
 SELECT cast(1 as int) >= '2' FROM t
 -- !query schema
-struct<(CAST(1 AS INT) >= CAST(2 AS INT)):boolean>
+struct<(CAST(1 AS INT) >= 2):boolean>
 -- !query output
 false
 
@@ -613,7 +613,7 @@ false
 -- !query
 SELECT cast(1 as int) < '2' FROM t
 -- !query schema
-struct<(CAST(1 AS INT) < CAST(2 AS INT)):boolean>
+struct<(CAST(1 AS INT) < 2):boolean>
 -- !query output
 true
 
@@ -621,7 +621,7 @@ true
 -- !query
 SELECT cast(1 as int) <= '2' FROM t
 -- !query schema
-struct<(CAST(1 AS INT) <= CAST(2 AS INT)):boolean>
+struct<(CAST(1 AS INT) <= 2):boolean>
 -- !query output
 true
 
@@ -629,7 +629,7 @@ true
 -- !query
 SELECT cast(1 as int) <> '2' FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS INT) = CAST(2 AS INT))):boolean>
+struct<(NOT (CAST(1 AS INT) = 2)):boolean>
 -- !query output
 true
 
@@ -637,7 +637,7 @@ true
 -- !query
 SELECT cast(1 as int) = cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS INT) = CAST(CAST(NULL AS STRING) AS INT)):boolean>
+struct<(CAST(1 AS INT) = CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -645,7 +645,7 @@ NULL
 -- !query
 SELECT cast(1 as int) > cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS INT) > CAST(CAST(NULL AS STRING) AS INT)):boolean>
+struct<(CAST(1 AS INT) > CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -653,7 +653,7 @@ NULL
 -- !query
 SELECT cast(1 as int) >= cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS INT) >= CAST(CAST(NULL AS STRING) AS INT)):boolean>
+struct<(CAST(1 AS INT) >= CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -661,7 +661,7 @@ NULL
 -- !query
 SELECT cast(1 as int) < cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS INT) < CAST(CAST(NULL AS STRING) AS INT)):boolean>
+struct<(CAST(1 AS INT) < CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -669,7 +669,7 @@ NULL
 -- !query
 SELECT cast(1 as int) <= cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS INT) <= CAST(CAST(NULL AS STRING) AS INT)):boolean>
+struct<(CAST(1 AS INT) <= CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -677,7 +677,7 @@ NULL
 -- !query
 SELECT cast(1 as int) <> cast(null as string) FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS INT) = CAST(CAST(NULL AS STRING) AS INT))):boolean>
+struct<(NOT (CAST(1 AS INT) = CAST(NULL AS STRING))):boolean>
 -- !query output
 NULL
 
@@ -685,7 +685,7 @@ NULL
 -- !query
 SELECT '1' = cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(1 AS INT) = CAST(1 AS INT)):boolean>
+struct<(1 = CAST(1 AS INT)):boolean>
 -- !query output
 true
 
@@ -693,7 +693,7 @@ true
 -- !query
 SELECT '2' > cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(2 AS INT) > CAST(1 AS INT)):boolean>
+struct<(2 > CAST(1 AS INT)):boolean>
 -- !query output
 true
 
@@ -701,7 +701,7 @@ true
 -- !query
 SELECT '2' >= cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(2 AS INT) >= CAST(1 AS INT)):boolean>
+struct<(2 >= CAST(1 AS INT)):boolean>
 -- !query output
 true
 
@@ -709,7 +709,7 @@ true
 -- !query
 SELECT '2' < cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(2 AS INT) < CAST(1 AS INT)):boolean>
+struct<(2 < CAST(1 AS INT)):boolean>
 -- !query output
 false
 
@@ -717,7 +717,7 @@ false
 -- !query
 SELECT '2' <> cast(1 as int) FROM t
 -- !query schema
-struct<(NOT (CAST(2 AS INT) = CAST(1 AS INT))):boolean>
+struct<(NOT (2 = CAST(1 AS INT))):boolean>
 -- !query output
 true
 
@@ -725,7 +725,7 @@ true
 -- !query
 SELECT '2' <= cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(2 AS INT) <= CAST(1 AS INT)):boolean>
+struct<(2 <= CAST(1 AS INT)):boolean>
 -- !query output
 false
 
@@ -733,7 +733,7 @@ false
 -- !query
 SELECT cast(null as string) = cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS INT) = CAST(1 AS INT)):boolean>
+struct<(CAST(NULL AS STRING) = CAST(1 AS INT)):boolean>
 -- !query output
 NULL
 
@@ -741,7 +741,7 @@ NULL
 -- !query
 SELECT cast(null as string) > cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS INT) > CAST(1 AS INT)):boolean>
+struct<(CAST(NULL AS STRING) > CAST(1 AS INT)):boolean>
 -- !query output
 NULL
 
@@ -749,7 +749,7 @@ NULL
 -- !query
 SELECT cast(null as string) >= cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS INT) >= CAST(1 AS INT)):boolean>
+struct<(CAST(NULL AS STRING) >= CAST(1 AS INT)):boolean>
 -- !query output
 NULL
 
@@ -757,7 +757,7 @@ NULL
 -- !query
 SELECT cast(null as string) < cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS INT) < CAST(1 AS INT)):boolean>
+struct<(CAST(NULL AS STRING) < CAST(1 AS INT)):boolean>
 -- !query output
 NULL
 
@@ -765,7 +765,7 @@ NULL
 -- !query
 SELECT cast(null as string) <> cast(1 as int) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(NULL AS STRING) AS INT) = CAST(1 AS INT))):boolean>
+struct<(NOT (CAST(NULL AS STRING) = CAST(1 AS INT))):boolean>
 -- !query output
 NULL
 
@@ -773,7 +773,7 @@ NULL
 -- !query
 SELECT cast(null as string) <= cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS INT) <= CAST(1 AS INT)):boolean>
+struct<(CAST(NULL AS STRING) <= CAST(1 AS INT)):boolean>
 -- !query output
 NULL
 
@@ -781,7 +781,7 @@ NULL
 -- !query
 SELECT cast(1 as bigint) = '1' FROM t
 -- !query schema
-struct<(CAST(1 AS BIGINT) = CAST(1 AS BIGINT)):boolean>
+struct<(CAST(1 AS BIGINT) = 1):boolean>
 -- !query output
 true
 
@@ -789,7 +789,7 @@ true
 -- !query
 SELECT cast(1 as bigint) > '2' FROM t
 -- !query schema
-struct<(CAST(1 AS BIGINT) > CAST(2 AS BIGINT)):boolean>
+struct<(CAST(1 AS BIGINT) > 2):boolean>
 -- !query output
 false
 
@@ -797,7 +797,7 @@ false
 -- !query
 SELECT cast(1 as bigint) >= '2' FROM t
 -- !query schema
-struct<(CAST(1 AS BIGINT) >= CAST(2 AS BIGINT)):boolean>
+struct<(CAST(1 AS BIGINT) >= 2):boolean>
 -- !query output
 false
 
@@ -805,7 +805,7 @@ false
 -- !query
 SELECT cast(1 as bigint) < '2' FROM t
 -- !query schema
-struct<(CAST(1 AS BIGINT) < CAST(2 AS BIGINT)):boolean>
+struct<(CAST(1 AS BIGINT) < 2):boolean>
 -- !query output
 true
 
@@ -813,7 +813,7 @@ true
 -- !query
 SELECT cast(1 as bigint) <= '2' FROM t
 -- !query schema
-struct<(CAST(1 AS BIGINT) <= CAST(2 AS BIGINT)):boolean>
+struct<(CAST(1 AS BIGINT) <= 2):boolean>
 -- !query output
 true
 
@@ -821,7 +821,7 @@ true
 -- !query
 SELECT cast(1 as bigint) <> '2' FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS BIGINT) = CAST(2 AS BIGINT))):boolean>
+struct<(NOT (CAST(1 AS BIGINT) = 2)):boolean>
 -- !query output
 true
 
@@ -829,7 +829,7 @@ true
 -- !query
 SELECT cast(1 as bigint) = cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS BIGINT) = CAST(CAST(NULL AS STRING) AS BIGINT)):boolean>
+struct<(CAST(1 AS BIGINT) = CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -837,7 +837,7 @@ NULL
 -- !query
 SELECT cast(1 as bigint) > cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS BIGINT) > CAST(CAST(NULL AS STRING) AS BIGINT)):boolean>
+struct<(CAST(1 AS BIGINT) > CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -845,7 +845,7 @@ NULL
 -- !query
 SELECT cast(1 as bigint) >= cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS BIGINT) >= CAST(CAST(NULL AS STRING) AS BIGINT)):boolean>
+struct<(CAST(1 AS BIGINT) >= CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -853,7 +853,7 @@ NULL
 -- !query
 SELECT cast(1 as bigint) < cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS BIGINT) < CAST(CAST(NULL AS STRING) AS BIGINT)):boolean>
+struct<(CAST(1 AS BIGINT) < CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -861,7 +861,7 @@ NULL
 -- !query
 SELECT cast(1 as bigint) <= cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS BIGINT) <= CAST(CAST(NULL AS STRING) AS BIGINT)):boolean>
+struct<(CAST(1 AS BIGINT) <= CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -869,7 +869,7 @@ NULL
 -- !query
 SELECT cast(1 as bigint) <> cast(null as string) FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS BIGINT) = CAST(CAST(NULL AS STRING) AS BIGINT))):boolean>
+struct<(NOT (CAST(1 AS BIGINT) = CAST(NULL AS STRING))):boolean>
 -- !query output
 NULL
 
@@ -877,7 +877,7 @@ NULL
 -- !query
 SELECT '1' = cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(1 AS BIGINT) = CAST(1 AS BIGINT)):boolean>
+struct<(1 = CAST(1 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -885,7 +885,7 @@ true
 -- !query
 SELECT '2' > cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(2 AS BIGINT) > CAST(1 AS BIGINT)):boolean>
+struct<(2 > CAST(1 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -893,7 +893,7 @@ true
 -- !query
 SELECT '2' >= cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(2 AS BIGINT) >= CAST(1 AS BIGINT)):boolean>
+struct<(2 >= CAST(1 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -901,7 +901,7 @@ true
 -- !query
 SELECT '2' < cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(2 AS BIGINT) < CAST(1 AS BIGINT)):boolean>
+struct<(2 < CAST(1 AS BIGINT)):boolean>
 -- !query output
 false
 
@@ -909,7 +909,7 @@ false
 -- !query
 SELECT '2' <= cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(2 AS BIGINT) <= CAST(1 AS BIGINT)):boolean>
+struct<(2 <= CAST(1 AS BIGINT)):boolean>
 -- !query output
 false
 
@@ -917,7 +917,7 @@ false
 -- !query
 SELECT '2' <> cast(1 as bigint) FROM t
 -- !query schema
-struct<(NOT (CAST(2 AS BIGINT) = CAST(1 AS BIGINT))):boolean>
+struct<(NOT (2 = CAST(1 AS BIGINT))):boolean>
 -- !query output
 true
 
@@ -925,7 +925,7 @@ true
 -- !query
 SELECT cast(null as string) = cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS BIGINT) = CAST(1 AS BIGINT)):boolean>
+struct<(CAST(NULL AS STRING) = CAST(1 AS BIGINT)):boolean>
 -- !query output
 NULL
 
@@ -933,7 +933,7 @@ NULL
 -- !query
 SELECT cast(null as string) > cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS BIGINT) > CAST(1 AS BIGINT)):boolean>
+struct<(CAST(NULL AS STRING) > CAST(1 AS BIGINT)):boolean>
 -- !query output
 NULL
 
@@ -941,7 +941,7 @@ NULL
 -- !query
 SELECT cast(null as string) >= cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS BIGINT) >= CAST(1 AS BIGINT)):boolean>
+struct<(CAST(NULL AS STRING) >= CAST(1 AS BIGINT)):boolean>
 -- !query output
 NULL
 
@@ -949,7 +949,7 @@ NULL
 -- !query
 SELECT cast(null as string) < cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS BIGINT) < CAST(1 AS BIGINT)):boolean>
+struct<(CAST(NULL AS STRING) < CAST(1 AS BIGINT)):boolean>
 -- !query output
 NULL
 
@@ -957,7 +957,7 @@ NULL
 -- !query
 SELECT cast(null as string) <= cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS BIGINT) <= CAST(1 AS BIGINT)):boolean>
+struct<(CAST(NULL AS STRING) <= CAST(1 AS BIGINT)):boolean>
 -- !query output
 NULL
 
@@ -965,7 +965,7 @@ NULL
 -- !query
 SELECT cast(null as string) <> cast(1 as bigint) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(NULL AS STRING) AS BIGINT) = CAST(1 AS BIGINT))):boolean>
+struct<(NOT (CAST(NULL AS STRING) = CAST(1 AS BIGINT))):boolean>
 -- !query output
 NULL
 
@@ -973,7 +973,7 @@ NULL
 -- !query
 SELECT cast(1 as decimal(10, 0)) = '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) = CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) = 1):boolean>
 -- !query output
 true
 
@@ -981,7 +981,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) > '2' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) > CAST(2 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) > 2):boolean>
 -- !query output
 false
 
@@ -989,7 +989,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) >= '2' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) >= CAST(2 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) >= 2):boolean>
 -- !query output
 false
 
@@ -997,7 +997,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) < '2' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) < CAST(2 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) < 2):boolean>
 -- !query output
 true
 
@@ -1005,7 +1005,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) <> '2' FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) = CAST(2 AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(10,0)) = 2)):boolean>
 -- !query output
 true
 
@@ -1013,7 +1013,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) <= '2' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) <= CAST(2 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <= 2):boolean>
 -- !query output
 true
 
@@ -1021,7 +1021,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) = cast(null as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) = CAST(CAST(NULL AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) = CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -1029,7 +1029,7 @@ NULL
 -- !query
 SELECT cast(1 as decimal(10, 0)) > cast(null as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) > CAST(CAST(NULL AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) > CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -1037,7 +1037,7 @@ NULL
 -- !query
 SELECT cast(1 as decimal(10, 0)) >= cast(null as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) >= CAST(CAST(NULL AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) >= CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -1045,7 +1045,7 @@ NULL
 -- !query
 SELECT cast(1 as decimal(10, 0)) < cast(null as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) < CAST(CAST(NULL AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) < CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -1053,7 +1053,7 @@ NULL
 -- !query
 SELECT cast(1 as decimal(10, 0)) <> cast(null as string) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) = CAST(CAST(NULL AS STRING) AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(10,0)) = CAST(NULL AS STRING))):boolean>
 -- !query output
 NULL
 
@@ -1061,7 +1061,7 @@ NULL
 -- !query
 SELECT cast(1 as decimal(10, 0)) <= cast(null as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) <= CAST(CAST(NULL AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <= CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -1069,7 +1069,7 @@ NULL
 -- !query
 SELECT '1' = cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) = CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(1 = CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -1077,7 +1077,7 @@ true
 -- !query
 SELECT '2' > cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(2 AS DOUBLE) > CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(2 > CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -1085,7 +1085,7 @@ true
 -- !query
 SELECT '2' >= cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(2 AS DOUBLE) >= CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(2 >= CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -1093,7 +1093,7 @@ true
 -- !query
 SELECT '2' < cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(2 AS DOUBLE) < CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(2 < CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 false
 
@@ -1101,7 +1101,7 @@ false
 -- !query
 SELECT '2' <= cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(2 AS DOUBLE) <= CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(2 <= CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 false
 
@@ -1109,7 +1109,7 @@ false
 -- !query
 SELECT '2' <> cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(2 AS DOUBLE) = CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE))):boolean>
+struct<(NOT (2 = CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 true
 
@@ -1117,7 +1117,7 @@ true
 -- !query
 SELECT cast(null as string) = cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS DOUBLE) = CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(CAST(NULL AS STRING) = CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 NULL
 
@@ -1125,7 +1125,7 @@ NULL
 -- !query
 SELECT cast(null as string) > cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS DOUBLE) > CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(CAST(NULL AS STRING) > CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 NULL
 
@@ -1133,7 +1133,7 @@ NULL
 -- !query
 SELECT cast(null as string) >= cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS DOUBLE) >= CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(CAST(NULL AS STRING) >= CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 NULL
 
@@ -1141,7 +1141,7 @@ NULL
 -- !query
 SELECT cast(null as string) < cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS DOUBLE) < CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(CAST(NULL AS STRING) < CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 NULL
 
@@ -1149,7 +1149,7 @@ NULL
 -- !query
 SELECT cast(null as string) <= cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS DOUBLE) <= CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(CAST(NULL AS STRING) <= CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 NULL
 
@@ -1157,7 +1157,7 @@ NULL
 -- !query
 SELECT cast(null as string) <> cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(NULL AS STRING) AS DOUBLE) = CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE))):boolean>
+struct<(NOT (CAST(NULL AS STRING) = CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 NULL
 
@@ -1165,7 +1165,7 @@ NULL
 -- !query
 SELECT cast(1 as double) = '1' FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) = CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) = 1):boolean>
 -- !query output
 true
 
@@ -1173,7 +1173,7 @@ true
 -- !query
 SELECT cast(1 as double) > '2' FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) > CAST(2 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) > 2):boolean>
 -- !query output
 false
 
@@ -1181,7 +1181,7 @@ false
 -- !query
 SELECT cast(1 as double) >= '2' FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) >= CAST(2 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) >= 2):boolean>
 -- !query output
 false
 
@@ -1189,7 +1189,7 @@ false
 -- !query
 SELECT cast(1 as double) < '2' FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) < CAST(2 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) < 2):boolean>
 -- !query output
 true
 
@@ -1197,7 +1197,7 @@ true
 -- !query
 SELECT cast(1 as double) <= '2' FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) <= CAST(2 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) <= 2):boolean>
 -- !query output
 true
 
@@ -1205,7 +1205,7 @@ true
 -- !query
 SELECT cast(1 as double) <> '2' FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS DOUBLE) = CAST(2 AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS DOUBLE) = 2)):boolean>
 -- !query output
 true
 
@@ -1213,7 +1213,7 @@ true
 -- !query
 SELECT cast(1 as double) = cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) = CAST(CAST(NULL AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) = CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -1221,7 +1221,7 @@ NULL
 -- !query
 SELECT cast(1 as double) > cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) > CAST(CAST(NULL AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) > CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -1229,7 +1229,7 @@ NULL
 -- !query
 SELECT cast(1 as double) >= cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) >= CAST(CAST(NULL AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) >= CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -1237,7 +1237,7 @@ NULL
 -- !query
 SELECT cast(1 as double) < cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) < CAST(CAST(NULL AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) < CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -1245,7 +1245,7 @@ NULL
 -- !query
 SELECT cast(1 as double) <= cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) <= CAST(CAST(NULL AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) <= CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -1253,7 +1253,7 @@ NULL
 -- !query
 SELECT cast(1 as double) <> cast(null as string) FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS DOUBLE) = CAST(CAST(NULL AS STRING) AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS DOUBLE) = CAST(NULL AS STRING))):boolean>
 -- !query output
 NULL
 
@@ -1261,7 +1261,7 @@ NULL
 -- !query
 SELECT '1' = cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) = CAST(1 AS DOUBLE)):boolean>
+struct<(1 = CAST(1 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -1269,7 +1269,7 @@ true
 -- !query
 SELECT '2' > cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(2 AS DOUBLE) > CAST(1 AS DOUBLE)):boolean>
+struct<(2 > CAST(1 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -1277,7 +1277,7 @@ true
 -- !query
 SELECT '2' >= cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(2 AS DOUBLE) >= CAST(1 AS DOUBLE)):boolean>
+struct<(2 >= CAST(1 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -1285,7 +1285,7 @@ true
 -- !query
 SELECT '2' < cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(2 AS DOUBLE) < CAST(1 AS DOUBLE)):boolean>
+struct<(2 < CAST(1 AS DOUBLE)):boolean>
 -- !query output
 false
 
@@ -1293,7 +1293,7 @@ false
 -- !query
 SELECT '2' <= cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(2 AS DOUBLE) <= CAST(1 AS DOUBLE)):boolean>
+struct<(2 <= CAST(1 AS DOUBLE)):boolean>
 -- !query output
 false
 
@@ -1301,7 +1301,7 @@ false
 -- !query
 SELECT '2' <> cast(1 as double) FROM t
 -- !query schema
-struct<(NOT (CAST(2 AS DOUBLE) = CAST(1 AS DOUBLE))):boolean>
+struct<(NOT (2 = CAST(1 AS DOUBLE))):boolean>
 -- !query output
 true
 
@@ -1309,7 +1309,7 @@ true
 -- !query
 SELECT cast(null as string) = cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS DOUBLE) = CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(NULL AS STRING) = CAST(1 AS DOUBLE)):boolean>
 -- !query output
 NULL
 
@@ -1317,7 +1317,7 @@ NULL
 -- !query
 SELECT cast(null as string) > cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS DOUBLE) > CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(NULL AS STRING) > CAST(1 AS DOUBLE)):boolean>
 -- !query output
 NULL
 
@@ -1325,7 +1325,7 @@ NULL
 -- !query
 SELECT cast(null as string) >= cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS DOUBLE) >= CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(NULL AS STRING) >= CAST(1 AS DOUBLE)):boolean>
 -- !query output
 NULL
 
@@ -1333,7 +1333,7 @@ NULL
 -- !query
 SELECT cast(null as string) < cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS DOUBLE) < CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(NULL AS STRING) < CAST(1 AS DOUBLE)):boolean>
 -- !query output
 NULL
 
@@ -1341,7 +1341,7 @@ NULL
 -- !query
 SELECT cast(null as string) <= cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS DOUBLE) <= CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(NULL AS STRING) <= CAST(1 AS DOUBLE)):boolean>
 -- !query output
 NULL
 
@@ -1349,7 +1349,7 @@ NULL
 -- !query
 SELECT cast(null as string) <> cast(1 as double) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(NULL AS STRING) AS DOUBLE) = CAST(1 AS DOUBLE))):boolean>
+struct<(NOT (CAST(NULL AS STRING) = CAST(1 AS DOUBLE))):boolean>
 -- !query output
 NULL
 
@@ -1357,7 +1357,7 @@ NULL
 -- !query
 SELECT cast(1 as float) = '1' FROM t
 -- !query schema
-struct<(CAST(1 AS FLOAT) = CAST(1 AS FLOAT)):boolean>
+struct<(CAST(1 AS FLOAT) = 1):boolean>
 -- !query output
 true
 
@@ -1365,7 +1365,7 @@ true
 -- !query
 SELECT cast(1 as float) > '2' FROM t
 -- !query schema
-struct<(CAST(1 AS FLOAT) > CAST(2 AS FLOAT)):boolean>
+struct<(CAST(1 AS FLOAT) > 2):boolean>
 -- !query output
 false
 
@@ -1373,7 +1373,7 @@ false
 -- !query
 SELECT cast(1 as float) >= '2' FROM t
 -- !query schema
-struct<(CAST(1 AS FLOAT) >= CAST(2 AS FLOAT)):boolean>
+struct<(CAST(1 AS FLOAT) >= 2):boolean>
 -- !query output
 false
 
@@ -1381,7 +1381,7 @@ false
 -- !query
 SELECT cast(1 as float) < '2' FROM t
 -- !query schema
-struct<(CAST(1 AS FLOAT) < CAST(2 AS FLOAT)):boolean>
+struct<(CAST(1 AS FLOAT) < 2):boolean>
 -- !query output
 true
 
@@ -1389,7 +1389,7 @@ true
 -- !query
 SELECT cast(1 as float) <= '2' FROM t
 -- !query schema
-struct<(CAST(1 AS FLOAT) <= CAST(2 AS FLOAT)):boolean>
+struct<(CAST(1 AS FLOAT) <= 2):boolean>
 -- !query output
 true
 
@@ -1397,7 +1397,7 @@ true
 -- !query
 SELECT cast(1 as float) <> '2' FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS FLOAT) = CAST(2 AS FLOAT))):boolean>
+struct<(NOT (CAST(1 AS FLOAT) = 2)):boolean>
 -- !query output
 true
 
@@ -1405,7 +1405,7 @@ true
 -- !query
 SELECT cast(1 as float) = cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS FLOAT) = CAST(CAST(NULL AS STRING) AS FLOAT)):boolean>
+struct<(CAST(1 AS FLOAT) = CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -1413,7 +1413,7 @@ NULL
 -- !query
 SELECT cast(1 as float) > cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS FLOAT) > CAST(CAST(NULL AS STRING) AS FLOAT)):boolean>
+struct<(CAST(1 AS FLOAT) > CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -1421,7 +1421,7 @@ NULL
 -- !query
 SELECT cast(1 as float) >= cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS FLOAT) >= CAST(CAST(NULL AS STRING) AS FLOAT)):boolean>
+struct<(CAST(1 AS FLOAT) >= CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -1429,7 +1429,7 @@ NULL
 -- !query
 SELECT cast(1 as float) < cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS FLOAT) < CAST(CAST(NULL AS STRING) AS FLOAT)):boolean>
+struct<(CAST(1 AS FLOAT) < CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -1437,7 +1437,7 @@ NULL
 -- !query
 SELECT cast(1 as float) <= cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1 AS FLOAT) <= CAST(CAST(NULL AS STRING) AS FLOAT)):boolean>
+struct<(CAST(1 AS FLOAT) <= CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -1445,7 +1445,7 @@ NULL
 -- !query
 SELECT cast(1 as float) <> cast(null as string) FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS FLOAT) = CAST(CAST(NULL AS STRING) AS FLOAT))):boolean>
+struct<(NOT (CAST(1 AS FLOAT) = CAST(NULL AS STRING))):boolean>
 -- !query output
 NULL
 
@@ -1453,7 +1453,7 @@ NULL
 -- !query
 SELECT '1' = cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(1 AS FLOAT) = CAST(1 AS FLOAT)):boolean>
+struct<(1 = CAST(1 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -1461,7 +1461,7 @@ true
 -- !query
 SELECT '2' > cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(2 AS FLOAT) > CAST(1 AS FLOAT)):boolean>
+struct<(2 > CAST(1 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -1469,7 +1469,7 @@ true
 -- !query
 SELECT '2' >= cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(2 AS FLOAT) >= CAST(1 AS FLOAT)):boolean>
+struct<(2 >= CAST(1 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -1477,7 +1477,7 @@ true
 -- !query
 SELECT '2' < cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(2 AS FLOAT) < CAST(1 AS FLOAT)):boolean>
+struct<(2 < CAST(1 AS FLOAT)):boolean>
 -- !query output
 false
 
@@ -1485,7 +1485,7 @@ false
 -- !query
 SELECT '2' <= cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(2 AS FLOAT) <= CAST(1 AS FLOAT)):boolean>
+struct<(2 <= CAST(1 AS FLOAT)):boolean>
 -- !query output
 false
 
@@ -1493,7 +1493,7 @@ false
 -- !query
 SELECT '2' <> cast(1 as float) FROM t
 -- !query schema
-struct<(NOT (CAST(2 AS FLOAT) = CAST(1 AS FLOAT))):boolean>
+struct<(NOT (2 = CAST(1 AS FLOAT))):boolean>
 -- !query output
 true
 
@@ -1501,7 +1501,7 @@ true
 -- !query
 SELECT cast(null as string) = cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS FLOAT) = CAST(1 AS FLOAT)):boolean>
+struct<(CAST(NULL AS STRING) = CAST(1 AS FLOAT)):boolean>
 -- !query output
 NULL
 
@@ -1509,7 +1509,7 @@ NULL
 -- !query
 SELECT cast(null as string) > cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS FLOAT) > CAST(1 AS FLOAT)):boolean>
+struct<(CAST(NULL AS STRING) > CAST(1 AS FLOAT)):boolean>
 -- !query output
 NULL
 
@@ -1517,7 +1517,7 @@ NULL
 -- !query
 SELECT cast(null as string) >= cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS FLOAT) >= CAST(1 AS FLOAT)):boolean>
+struct<(CAST(NULL AS STRING) >= CAST(1 AS FLOAT)):boolean>
 -- !query output
 NULL
 
@@ -1525,7 +1525,7 @@ NULL
 -- !query
 SELECT cast(null as string) < cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS FLOAT) < CAST(1 AS FLOAT)):boolean>
+struct<(CAST(NULL AS STRING) < CAST(1 AS FLOAT)):boolean>
 -- !query output
 NULL
 
@@ -1533,7 +1533,7 @@ NULL
 -- !query
 SELECT cast(null as string) <= cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS FLOAT) <= CAST(1 AS FLOAT)):boolean>
+struct<(CAST(NULL AS STRING) <= CAST(1 AS FLOAT)):boolean>
 -- !query output
 NULL
 
@@ -1541,7 +1541,7 @@ NULL
 -- !query
 SELECT cast(null as string) <> cast(1 as float) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(NULL AS STRING) AS FLOAT) = CAST(1 AS FLOAT))):boolean>
+struct<(NOT (CAST(NULL AS STRING) = CAST(1 AS FLOAT))):boolean>
 -- !query output
 NULL
 
@@ -1549,7 +1549,7 @@ NULL
 -- !query
 SELECT '1996-09-09' = date('1996-09-09') FROM t
 -- !query schema
-struct<(CAST(1996-09-09 AS DATE) = CAST(1996-09-09 AS DATE)):boolean>
+struct<(1996-09-09 = 1996-09-09):boolean>
 -- !query output
 true
 
@@ -1557,7 +1557,7 @@ true
 -- !query
 SELECT '1996-9-10' > date('1996-09-09') FROM t
 -- !query schema
-struct<(CAST(1996-9-10 AS DATE) > CAST(1996-09-09 AS DATE)):boolean>
+struct<(1996-9-10 > 1996-09-09):boolean>
 -- !query output
 true
 
@@ -1565,7 +1565,7 @@ true
 -- !query
 SELECT '1996-9-10' >= date('1996-09-09') FROM t
 -- !query schema
-struct<(CAST(1996-9-10 AS DATE) >= CAST(1996-09-09 AS DATE)):boolean>
+struct<(1996-9-10 >= 1996-09-09):boolean>
 -- !query output
 true
 
@@ -1573,7 +1573,7 @@ true
 -- !query
 SELECT '1996-9-10' < date('1996-09-09') FROM t
 -- !query schema
-struct<(CAST(1996-9-10 AS DATE) < CAST(1996-09-09 AS DATE)):boolean>
+struct<(1996-9-10 < 1996-09-09):boolean>
 -- !query output
 false
 
@@ -1581,7 +1581,7 @@ false
 -- !query
 SELECT '1996-9-10' <= date('1996-09-09') FROM t
 -- !query schema
-struct<(CAST(1996-9-10 AS DATE) <= CAST(1996-09-09 AS DATE)):boolean>
+struct<(1996-9-10 <= 1996-09-09):boolean>
 -- !query output
 false
 
@@ -1589,7 +1589,7 @@ false
 -- !query
 SELECT '1996-9-10' <> date('1996-09-09') FROM t
 -- !query schema
-struct<(NOT (CAST(1996-9-10 AS DATE) = CAST(1996-09-09 AS DATE))):boolean>
+struct<(NOT (1996-9-10 = 1996-09-09)):boolean>
 -- !query output
 true
 
@@ -1597,7 +1597,7 @@ true
 -- !query
 SELECT cast(null as string) = date('1996-09-09') FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS DATE) = CAST(1996-09-09 AS DATE)):boolean>
+struct<(CAST(NULL AS STRING) = 1996-09-09):boolean>
 -- !query output
 NULL
 
@@ -1605,7 +1605,7 @@ NULL
 -- !query
 SELECT cast(null as string)> date('1996-09-09') FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS DATE) > CAST(1996-09-09 AS DATE)):boolean>
+struct<(CAST(NULL AS STRING) > 1996-09-09):boolean>
 -- !query output
 NULL
 
@@ -1613,7 +1613,7 @@ NULL
 -- !query
 SELECT cast(null as string)>= date('1996-09-09') FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS DATE) >= CAST(1996-09-09 AS DATE)):boolean>
+struct<(CAST(NULL AS STRING) >= 1996-09-09):boolean>
 -- !query output
 NULL
 
@@ -1621,7 +1621,7 @@ NULL
 -- !query
 SELECT cast(null as string)< date('1996-09-09') FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS DATE) < CAST(1996-09-09 AS DATE)):boolean>
+struct<(CAST(NULL AS STRING) < 1996-09-09):boolean>
 -- !query output
 NULL
 
@@ -1629,7 +1629,7 @@ NULL
 -- !query
 SELECT cast(null as string)<= date('1996-09-09') FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS DATE) <= CAST(1996-09-09 AS DATE)):boolean>
+struct<(CAST(NULL AS STRING) <= 1996-09-09):boolean>
 -- !query output
 NULL
 
@@ -1637,7 +1637,7 @@ NULL
 -- !query
 SELECT cast(null as string)<> date('1996-09-09') FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(NULL AS STRING) AS DATE) = CAST(1996-09-09 AS DATE))):boolean>
+struct<(NOT (CAST(NULL AS STRING) = 1996-09-09)):boolean>
 -- !query output
 NULL
 
@@ -1645,7 +1645,7 @@ NULL
 -- !query
 SELECT date('1996-09-09') = '1996-09-09' FROM t
 -- !query schema
-struct<(CAST(1996-09-09 AS DATE) = CAST(1996-09-09 AS DATE)):boolean>
+struct<(1996-09-09 = 1996-09-09):boolean>
 -- !query output
 true
 
@@ -1653,7 +1653,7 @@ true
 -- !query
 SELECT date('1996-9-10') > '1996-09-09' FROM t
 -- !query schema
-struct<(CAST(1996-9-10 AS DATE) > CAST(1996-09-09 AS DATE)):boolean>
+struct<(1996-9-10 > 1996-09-09):boolean>
 -- !query output
 true
 
@@ -1661,7 +1661,7 @@ true
 -- !query
 SELECT date('1996-9-10') >= '1996-09-09' FROM t
 -- !query schema
-struct<(CAST(1996-9-10 AS DATE) >= CAST(1996-09-09 AS DATE)):boolean>
+struct<(1996-9-10 >= 1996-09-09):boolean>
 -- !query output
 true
 
@@ -1669,7 +1669,7 @@ true
 -- !query
 SELECT date('1996-9-10') < '1996-09-09' FROM t
 -- !query schema
-struct<(CAST(1996-9-10 AS DATE) < CAST(1996-09-09 AS DATE)):boolean>
+struct<(1996-9-10 < 1996-09-09):boolean>
 -- !query output
 false
 
@@ -1677,7 +1677,7 @@ false
 -- !query
 SELECT date('1996-9-10') <= '1996-09-09' FROM t
 -- !query schema
-struct<(CAST(1996-9-10 AS DATE) <= CAST(1996-09-09 AS DATE)):boolean>
+struct<(1996-9-10 <= 1996-09-09):boolean>
 -- !query output
 false
 
@@ -1685,7 +1685,7 @@ false
 -- !query
 SELECT date('1996-9-10') <> '1996-09-09' FROM t
 -- !query schema
-struct<(NOT (CAST(1996-9-10 AS DATE) = CAST(1996-09-09 AS DATE))):boolean>
+struct<(NOT (1996-9-10 = 1996-09-09)):boolean>
 -- !query output
 true
 
@@ -1693,7 +1693,7 @@ true
 -- !query
 SELECT date('1996-09-09') = cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1996-09-09 AS DATE) = CAST(CAST(NULL AS STRING) AS DATE)):boolean>
+struct<(1996-09-09 = CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -1701,7 +1701,7 @@ NULL
 -- !query
 SELECT date('1996-9-10') > cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1996-9-10 AS DATE) > CAST(CAST(NULL AS STRING) AS DATE)):boolean>
+struct<(1996-9-10 > CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -1709,7 +1709,7 @@ NULL
 -- !query
 SELECT date('1996-9-10') >= cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1996-9-10 AS DATE) >= CAST(CAST(NULL AS STRING) AS DATE)):boolean>
+struct<(1996-9-10 >= CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -1717,7 +1717,7 @@ NULL
 -- !query
 SELECT date('1996-9-10') < cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1996-9-10 AS DATE) < CAST(CAST(NULL AS STRING) AS DATE)):boolean>
+struct<(1996-9-10 < CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -1725,7 +1725,7 @@ NULL
 -- !query
 SELECT date('1996-9-10') <= cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1996-9-10 AS DATE) <= CAST(CAST(NULL AS STRING) AS DATE)):boolean>
+struct<(1996-9-10 <= CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -1733,7 +1733,7 @@ NULL
 -- !query
 SELECT date('1996-9-10') <> cast(null as string) FROM t
 -- !query schema
-struct<(NOT (CAST(1996-9-10 AS DATE) = CAST(CAST(NULL AS STRING) AS DATE))):boolean>
+struct<(NOT (1996-9-10 = CAST(NULL AS STRING))):boolean>
 -- !query output
 NULL
 
@@ -1741,7 +1741,7 @@ NULL
 -- !query
 SELECT '1996-09-09 12:12:12.4' = timestamp('1996-09-09 12:12:12.4') FROM t
 -- !query schema
-struct<(CAST(1996-09-09 12:12:12.4 AS TIMESTAMP) = CAST(1996-09-09 12:12:12.4 AS TIMESTAMP)):boolean>
+struct<(1996-09-09 12:12:12.4 = 1996-09-09 12:12:12.4):boolean>
 -- !query output
 true
 
@@ -1749,7 +1749,7 @@ true
 -- !query
 SELECT '1996-09-09 12:12:12.5' > timestamp('1996-09-09 12:12:12.4') FROM t
 -- !query schema
-struct<(CAST(1996-09-09 12:12:12.5 AS TIMESTAMP) > CAST(1996-09-09 12:12:12.4 AS TIMESTAMP)):boolean>
+struct<(1996-09-09 12:12:12.5 > 1996-09-09 12:12:12.4):boolean>
 -- !query output
 true
 
@@ -1757,7 +1757,7 @@ true
 -- !query
 SELECT '1996-09-09 12:12:12.5' >= timestamp('1996-09-09 12:12:12.4') FROM t
 -- !query schema
-struct<(CAST(1996-09-09 12:12:12.5 AS TIMESTAMP) >= CAST(1996-09-09 12:12:12.4 AS TIMESTAMP)):boolean>
+struct<(1996-09-09 12:12:12.5 >= 1996-09-09 12:12:12.4):boolean>
 -- !query output
 true
 
@@ -1765,7 +1765,7 @@ true
 -- !query
 SELECT '1996-09-09 12:12:12.5' < timestamp('1996-09-09 12:12:12.4') FROM t
 -- !query schema
-struct<(CAST(1996-09-09 12:12:12.5 AS TIMESTAMP) < CAST(1996-09-09 12:12:12.4 AS TIMESTAMP)):boolean>
+struct<(1996-09-09 12:12:12.5 < 1996-09-09 12:12:12.4):boolean>
 -- !query output
 false
 
@@ -1773,7 +1773,7 @@ false
 -- !query
 SELECT '1996-09-09 12:12:12.5' <= timestamp('1996-09-09 12:12:12.4') FROM t
 -- !query schema
-struct<(CAST(1996-09-09 12:12:12.5 AS TIMESTAMP) <= CAST(1996-09-09 12:12:12.4 AS TIMESTAMP)):boolean>
+struct<(1996-09-09 12:12:12.5 <= 1996-09-09 12:12:12.4):boolean>
 -- !query output
 false
 
@@ -1781,7 +1781,7 @@ false
 -- !query
 SELECT '1996-09-09 12:12:12.5' <> timestamp('1996-09-09 12:12:12.4') FROM t
 -- !query schema
-struct<(NOT (CAST(1996-09-09 12:12:12.5 AS TIMESTAMP) = CAST(1996-09-09 12:12:12.4 AS TIMESTAMP))):boolean>
+struct<(NOT (1996-09-09 12:12:12.5 = 1996-09-09 12:12:12.4)):boolean>
 -- !query output
 true
 
@@ -1789,7 +1789,7 @@ true
 -- !query
 SELECT cast(null as string) = timestamp('1996-09-09 12:12:12.4') FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS TIMESTAMP) = CAST(1996-09-09 12:12:12.4 AS TIMESTAMP)):boolean>
+struct<(CAST(NULL AS STRING) = 1996-09-09 12:12:12.4):boolean>
 -- !query output
 NULL
 
@@ -1797,7 +1797,7 @@ NULL
 -- !query
 SELECT cast(null as string) > timestamp('1996-09-09 12:12:12.4') FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS TIMESTAMP) > CAST(1996-09-09 12:12:12.4 AS TIMESTAMP)):boolean>
+struct<(CAST(NULL AS STRING) > 1996-09-09 12:12:12.4):boolean>
 -- !query output
 NULL
 
@@ -1805,7 +1805,7 @@ NULL
 -- !query
 SELECT cast(null as string) >= timestamp('1996-09-09 12:12:12.4') FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS TIMESTAMP) >= CAST(1996-09-09 12:12:12.4 AS TIMESTAMP)):boolean>
+struct<(CAST(NULL AS STRING) >= 1996-09-09 12:12:12.4):boolean>
 -- !query output
 NULL
 
@@ -1813,7 +1813,7 @@ NULL
 -- !query
 SELECT cast(null as string) < timestamp('1996-09-09 12:12:12.4') FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS TIMESTAMP) < CAST(1996-09-09 12:12:12.4 AS TIMESTAMP)):boolean>
+struct<(CAST(NULL AS STRING) < 1996-09-09 12:12:12.4):boolean>
 -- !query output
 NULL
 
@@ -1821,7 +1821,7 @@ NULL
 -- !query
 SELECT cast(null as string) <= timestamp('1996-09-09 12:12:12.4') FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS TIMESTAMP) <= CAST(1996-09-09 12:12:12.4 AS TIMESTAMP)):boolean>
+struct<(CAST(NULL AS STRING) <= 1996-09-09 12:12:12.4):boolean>
 -- !query output
 NULL
 
@@ -1829,7 +1829,7 @@ NULL
 -- !query
 SELECT cast(null as string) <> timestamp('1996-09-09 12:12:12.4') FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(NULL AS STRING) AS TIMESTAMP) = CAST(1996-09-09 12:12:12.4 AS TIMESTAMP))):boolean>
+struct<(NOT (CAST(NULL AS STRING) = 1996-09-09 12:12:12.4)):boolean>
 -- !query output
 NULL
 
@@ -1837,7 +1837,7 @@ NULL
 -- !query
 SELECT timestamp('1996-09-09 12:12:12.4' )= '1996-09-09 12:12:12.4' FROM t
 -- !query schema
-struct<(CAST(1996-09-09 12:12:12.4 AS TIMESTAMP) = CAST(1996-09-09 12:12:12.4 AS TIMESTAMP)):boolean>
+struct<(1996-09-09 12:12:12.4 = 1996-09-09 12:12:12.4):boolean>
 -- !query output
 true
 
@@ -1845,7 +1845,7 @@ true
 -- !query
 SELECT timestamp('1996-09-09 12:12:12.5' )> '1996-09-09 12:12:12.4' FROM t
 -- !query schema
-struct<(CAST(1996-09-09 12:12:12.5 AS TIMESTAMP) > CAST(1996-09-09 12:12:12.4 AS TIMESTAMP)):boolean>
+struct<(1996-09-09 12:12:12.5 > 1996-09-09 12:12:12.4):boolean>
 -- !query output
 true
 
@@ -1853,7 +1853,7 @@ true
 -- !query
 SELECT timestamp('1996-09-09 12:12:12.5' )>= '1996-09-09 12:12:12.4' FROM t
 -- !query schema
-struct<(CAST(1996-09-09 12:12:12.5 AS TIMESTAMP) >= CAST(1996-09-09 12:12:12.4 AS TIMESTAMP)):boolean>
+struct<(1996-09-09 12:12:12.5 >= 1996-09-09 12:12:12.4):boolean>
 -- !query output
 true
 
@@ -1861,7 +1861,7 @@ true
 -- !query
 SELECT timestamp('1996-09-09 12:12:12.5' )< '1996-09-09 12:12:12.4' FROM t
 -- !query schema
-struct<(CAST(1996-09-09 12:12:12.5 AS TIMESTAMP) < CAST(1996-09-09 12:12:12.4 AS TIMESTAMP)):boolean>
+struct<(1996-09-09 12:12:12.5 < 1996-09-09 12:12:12.4):boolean>
 -- !query output
 false
 
@@ -1869,7 +1869,7 @@ false
 -- !query
 SELECT timestamp('1996-09-09 12:12:12.5' )<= '1996-09-09 12:12:12.4' FROM t
 -- !query schema
-struct<(CAST(1996-09-09 12:12:12.5 AS TIMESTAMP) <= CAST(1996-09-09 12:12:12.4 AS TIMESTAMP)):boolean>
+struct<(1996-09-09 12:12:12.5 <= 1996-09-09 12:12:12.4):boolean>
 -- !query output
 false
 
@@ -1877,7 +1877,7 @@ false
 -- !query
 SELECT timestamp('1996-09-09 12:12:12.5' )<> '1996-09-09 12:12:12.4' FROM t
 -- !query schema
-struct<(NOT (CAST(1996-09-09 12:12:12.5 AS TIMESTAMP) = CAST(1996-09-09 12:12:12.4 AS TIMESTAMP))):boolean>
+struct<(NOT (1996-09-09 12:12:12.5 = 1996-09-09 12:12:12.4)):boolean>
 -- !query output
 true
 
@@ -1885,7 +1885,7 @@ true
 -- !query
 SELECT timestamp('1996-09-09 12:12:12.4' )= cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1996-09-09 12:12:12.4 AS TIMESTAMP) = CAST(CAST(NULL AS STRING) AS TIMESTAMP)):boolean>
+struct<(1996-09-09 12:12:12.4 = CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -1893,7 +1893,7 @@ NULL
 -- !query
 SELECT timestamp('1996-09-09 12:12:12.5' )> cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1996-09-09 12:12:12.5 AS TIMESTAMP) > CAST(CAST(NULL AS STRING) AS TIMESTAMP)):boolean>
+struct<(1996-09-09 12:12:12.5 > CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -1901,7 +1901,7 @@ NULL
 -- !query
 SELECT timestamp('1996-09-09 12:12:12.5' )>= cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1996-09-09 12:12:12.5 AS TIMESTAMP) >= CAST(CAST(NULL AS STRING) AS TIMESTAMP)):boolean>
+struct<(1996-09-09 12:12:12.5 >= CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -1909,7 +1909,7 @@ NULL
 -- !query
 SELECT timestamp('1996-09-09 12:12:12.5' )< cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1996-09-09 12:12:12.5 AS TIMESTAMP) < CAST(CAST(NULL AS STRING) AS TIMESTAMP)):boolean>
+struct<(1996-09-09 12:12:12.5 < CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -1917,7 +1917,7 @@ NULL
 -- !query
 SELECT timestamp('1996-09-09 12:12:12.5' )<= cast(null as string) FROM t
 -- !query schema
-struct<(CAST(1996-09-09 12:12:12.5 AS TIMESTAMP) <= CAST(CAST(NULL AS STRING) AS TIMESTAMP)):boolean>
+struct<(1996-09-09 12:12:12.5 <= CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -1925,7 +1925,7 @@ NULL
 -- !query
 SELECT timestamp('1996-09-09 12:12:12.5' )<> cast(null as string) FROM t
 -- !query schema
-struct<(NOT (CAST(1996-09-09 12:12:12.5 AS TIMESTAMP) = CAST(CAST(NULL AS STRING) AS TIMESTAMP))):boolean>
+struct<(NOT (1996-09-09 12:12:12.5 = CAST(NULL AS STRING))):boolean>
 -- !query output
 NULL
 
@@ -1933,7 +1933,7 @@ NULL
 -- !query
 SELECT ' ' = X'0020' FROM t
 -- !query schema
-struct<(CAST(  AS BINARY) = X'0020'):boolean>
+struct<(  = X'0020'):boolean>
 -- !query output
 false
 
@@ -1941,7 +1941,7 @@ false
 -- !query
 SELECT ' ' > X'001F' FROM t
 -- !query schema
-struct<(CAST(  AS BINARY) > X'001F'):boolean>
+struct<(  > X'001F'):boolean>
 -- !query output
 true
 
@@ -1949,7 +1949,7 @@ true
 -- !query
 SELECT ' ' >= X'001F' FROM t
 -- !query schema
-struct<(CAST(  AS BINARY) >= X'001F'):boolean>
+struct<(  >= X'001F'):boolean>
 -- !query output
 true
 
@@ -1957,7 +1957,7 @@ true
 -- !query
 SELECT ' ' < X'001F' FROM t
 -- !query schema
-struct<(CAST(  AS BINARY) < X'001F'):boolean>
+struct<(  < X'001F'):boolean>
 -- !query output
 false
 
@@ -1965,7 +1965,7 @@ false
 -- !query
 SELECT ' ' <= X'001F' FROM t
 -- !query schema
-struct<(CAST(  AS BINARY) <= X'001F'):boolean>
+struct<(  <= X'001F'):boolean>
 -- !query output
 false
 
@@ -1973,7 +1973,7 @@ false
 -- !query
 SELECT ' ' <> X'001F' FROM t
 -- !query schema
-struct<(NOT (CAST(  AS BINARY) = X'001F')):boolean>
+struct<(NOT (  = X'001F')):boolean>
 -- !query output
 true
 
@@ -1981,7 +1981,7 @@ true
 -- !query
 SELECT cast(null as string) = X'0020' FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS BINARY) = X'0020'):boolean>
+struct<(CAST(NULL AS STRING) = X'0020'):boolean>
 -- !query output
 NULL
 
@@ -1989,7 +1989,7 @@ NULL
 -- !query
 SELECT cast(null as string) > X'001F' FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS BINARY) > X'001F'):boolean>
+struct<(CAST(NULL AS STRING) > X'001F'):boolean>
 -- !query output
 NULL
 
@@ -1997,7 +1997,7 @@ NULL
 -- !query
 SELECT cast(null as string) >= X'001F' FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS BINARY) >= X'001F'):boolean>
+struct<(CAST(NULL AS STRING) >= X'001F'):boolean>
 -- !query output
 NULL
 
@@ -2005,7 +2005,7 @@ NULL
 -- !query
 SELECT cast(null as string) < X'001F' FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS BINARY) < X'001F'):boolean>
+struct<(CAST(NULL AS STRING) < X'001F'):boolean>
 -- !query output
 NULL
 
@@ -2013,7 +2013,7 @@ NULL
 -- !query
 SELECT cast(null as string) <= X'001F' FROM t
 -- !query schema
-struct<(CAST(CAST(NULL AS STRING) AS BINARY) <= X'001F'):boolean>
+struct<(CAST(NULL AS STRING) <= X'001F'):boolean>
 -- !query output
 NULL
 
@@ -2021,7 +2021,7 @@ NULL
 -- !query
 SELECT cast(null as string) <> X'001F' FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(NULL AS STRING) AS BINARY) = X'001F')):boolean>
+struct<(NOT (CAST(NULL AS STRING) = X'001F')):boolean>
 -- !query output
 NULL
 
@@ -2029,7 +2029,7 @@ NULL
 -- !query
 SELECT X'0020' = ' ' FROM t
 -- !query schema
-struct<(X'0020' = CAST(  AS BINARY)):boolean>
+struct<(X'0020' =  ):boolean>
 -- !query output
 false
 
@@ -2037,7 +2037,7 @@ false
 -- !query
 SELECT X'001F' > ' ' FROM t
 -- !query schema
-struct<(X'001F' > CAST(  AS BINARY)):boolean>
+struct<(X'001F' >  ):boolean>
 -- !query output
 false
 
@@ -2045,7 +2045,7 @@ false
 -- !query
 SELECT X'001F' >= ' ' FROM t
 -- !query schema
-struct<(X'001F' >= CAST(  AS BINARY)):boolean>
+struct<(X'001F' >=  ):boolean>
 -- !query output
 false
 
@@ -2053,7 +2053,7 @@ false
 -- !query
 SELECT X'001F' < ' ' FROM t
 -- !query schema
-struct<(X'001F' < CAST(  AS BINARY)):boolean>
+struct<(X'001F' <  ):boolean>
 -- !query output
 true
 
@@ -2061,7 +2061,7 @@ true
 -- !query
 SELECT X'001F' <= ' ' FROM t
 -- !query schema
-struct<(X'001F' <= CAST(  AS BINARY)):boolean>
+struct<(X'001F' <=  ):boolean>
 -- !query output
 true
 
@@ -2069,7 +2069,7 @@ true
 -- !query
 SELECT X'001F' <> ' ' FROM t
 -- !query schema
-struct<(NOT (X'001F' = CAST(  AS BINARY))):boolean>
+struct<(NOT (X'001F' =  )):boolean>
 -- !query output
 true
 
@@ -2077,7 +2077,7 @@ true
 -- !query
 SELECT X'0020' = cast(null as string) FROM t
 -- !query schema
-struct<(X'0020' = CAST(CAST(NULL AS STRING) AS BINARY)):boolean>
+struct<(X'0020' = CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -2085,7 +2085,7 @@ NULL
 -- !query
 SELECT X'001F' > cast(null as string) FROM t
 -- !query schema
-struct<(X'001F' > CAST(CAST(NULL AS STRING) AS BINARY)):boolean>
+struct<(X'001F' > CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -2093,7 +2093,7 @@ NULL
 -- !query
 SELECT X'001F' >= cast(null as string) FROM t
 -- !query schema
-struct<(X'001F' >= CAST(CAST(NULL AS STRING) AS BINARY)):boolean>
+struct<(X'001F' >= CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -2101,7 +2101,7 @@ NULL
 -- !query
 SELECT X'001F' < cast(null as string) FROM t
 -- !query schema
-struct<(X'001F' < CAST(CAST(NULL AS STRING) AS BINARY)):boolean>
+struct<(X'001F' < CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -2109,7 +2109,7 @@ NULL
 -- !query
 SELECT X'001F' <= cast(null as string) FROM t
 -- !query schema
-struct<(X'001F' <= CAST(CAST(NULL AS STRING) AS BINARY)):boolean>
+struct<(X'001F' <= CAST(NULL AS STRING)):boolean>
 -- !query output
 NULL
 
@@ -2117,6 +2117,6 @@ NULL
 -- !query
 SELECT X'001F' <> cast(null as string) FROM t
 -- !query schema
-struct<(NOT (X'001F' = CAST(CAST(NULL AS STRING) AS BINARY))):boolean>
+struct<(NOT (X'001F' = CAST(NULL AS STRING))):boolean>
 -- !query output
 NULL
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/booleanEquality.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/booleanEquality.sql.out
index dc068e70d66db..1b63931d9229a 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/booleanEquality.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/booleanEquality.sql.out
@@ -13,7 +13,7 @@ struct<>
 -- !query
 SELECT true = cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(true AS TINYINT) = CAST(1 AS TINYINT)):boolean>
+struct<(true = CAST(1 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -21,7 +21,7 @@ true
 -- !query
 SELECT true = cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(true AS SMALLINT) = CAST(1 AS SMALLINT)):boolean>
+struct<(true = CAST(1 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -29,7 +29,7 @@ true
 -- !query
 SELECT true = cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(true AS INT) = CAST(1 AS INT)):boolean>
+struct<(true = CAST(1 AS INT)):boolean>
 -- !query output
 true
 
@@ -37,7 +37,7 @@ true
 -- !query
 SELECT true = cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(true AS BIGINT) = CAST(1 AS BIGINT)):boolean>
+struct<(true = CAST(1 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -45,7 +45,7 @@ true
 -- !query
 SELECT true = cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(true AS FLOAT) = CAST(1 AS FLOAT)):boolean>
+struct<(true = CAST(1 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -53,7 +53,7 @@ true
 -- !query
 SELECT true = cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(true AS DOUBLE) = CAST(1 AS DOUBLE)):boolean>
+struct<(true = CAST(1 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -61,7 +61,7 @@ true
 -- !query
 SELECT true = cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(true AS DECIMAL(10,0)) = CAST(1 AS DECIMAL(10,0))):boolean>
+struct<(true = CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -69,7 +69,7 @@ true
 -- !query
 SELECT true = cast(1 as string) FROM t
 -- !query schema
-struct<(true = CAST(CAST(1 AS STRING) AS BOOLEAN)):boolean>
+struct<(true = CAST(1 AS STRING)):boolean>
 -- !query output
 true
 
@@ -112,7 +112,7 @@ cannot resolve '(true = CAST('2017-12-11 09:30:00' AS DATE))' due to data type m
 -- !query
 SELECT true <=> cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(true AS TINYINT) <=> CAST(1 AS TINYINT)):boolean>
+struct<(true <=> CAST(1 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -120,7 +120,7 @@ true
 -- !query
 SELECT true <=> cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(true AS SMALLINT) <=> CAST(1 AS SMALLINT)):boolean>
+struct<(true <=> CAST(1 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -128,7 +128,7 @@ true
 -- !query
 SELECT true <=> cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(true AS INT) <=> CAST(1 AS INT)):boolean>
+struct<(true <=> CAST(1 AS INT)):boolean>
 -- !query output
 true
 
@@ -136,7 +136,7 @@ true
 -- !query
 SELECT true <=> cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(true AS BIGINT) <=> CAST(1 AS BIGINT)):boolean>
+struct<(true <=> CAST(1 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -144,7 +144,7 @@ true
 -- !query
 SELECT true <=> cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(true AS FLOAT) <=> CAST(1 AS FLOAT)):boolean>
+struct<(true <=> CAST(1 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -152,7 +152,7 @@ true
 -- !query
 SELECT true <=> cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(true AS DOUBLE) <=> CAST(1 AS DOUBLE)):boolean>
+struct<(true <=> CAST(1 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -160,7 +160,7 @@ true
 -- !query
 SELECT true <=> cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(true AS DECIMAL(10,0)) <=> CAST(1 AS DECIMAL(10,0))):boolean>
+struct<(true <=> CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -168,7 +168,7 @@ true
 -- !query
 SELECT true <=> cast(1 as string) FROM t
 -- !query schema
-struct<(true <=> CAST(CAST(1 AS STRING) AS BOOLEAN)):boolean>
+struct<(true <=> CAST(1 AS STRING)):boolean>
 -- !query output
 true
 
@@ -211,7 +211,7 @@ cannot resolve '(true <=> CAST('2017-12-11 09:30:00' AS DATE))' due to data type
 -- !query
 SELECT cast(1 as tinyint) = true FROM t
 -- !query schema
-struct<(CAST(1 AS TINYINT) = CAST(true AS TINYINT)):boolean>
+struct<(CAST(1 AS TINYINT) = true):boolean>
 -- !query output
 true
 
@@ -219,7 +219,7 @@ true
 -- !query
 SELECT cast(1 as smallint) = true FROM t
 -- !query schema
-struct<(CAST(1 AS SMALLINT) = CAST(true AS SMALLINT)):boolean>
+struct<(CAST(1 AS SMALLINT) = true):boolean>
 -- !query output
 true
 
@@ -227,7 +227,7 @@ true
 -- !query
 SELECT cast(1 as int) = true FROM t
 -- !query schema
-struct<(CAST(1 AS INT) = CAST(true AS INT)):boolean>
+struct<(CAST(1 AS INT) = true):boolean>
 -- !query output
 true
 
@@ -235,7 +235,7 @@ true
 -- !query
 SELECT cast(1 as bigint) = true FROM t
 -- !query schema
-struct<(CAST(1 AS BIGINT) = CAST(true AS BIGINT)):boolean>
+struct<(CAST(1 AS BIGINT) = true):boolean>
 -- !query output
 true
 
@@ -243,7 +243,7 @@ true
 -- !query
 SELECT cast(1 as float) = true FROM t
 -- !query schema
-struct<(CAST(1 AS FLOAT) = CAST(true AS FLOAT)):boolean>
+struct<(CAST(1 AS FLOAT) = true):boolean>
 -- !query output
 true
 
@@ -251,7 +251,7 @@ true
 -- !query
 SELECT cast(1 as double) = true FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) = CAST(true AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) = true):boolean>
 -- !query output
 true
 
@@ -259,7 +259,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) = true FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(10,0)) = CAST(true AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) = true):boolean>
 -- !query output
 true
 
@@ -267,7 +267,7 @@ true
 -- !query
 SELECT cast(1 as string) = true FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS BOOLEAN) = true):boolean>
+struct<(CAST(1 AS STRING) = true):boolean>
 -- !query output
 true
 
@@ -310,7 +310,7 @@ cannot resolve '(CAST('2017-12-11 09:30:00' AS DATE) = true)' due to data type m
 -- !query
 SELECT cast(1 as tinyint) <=> true FROM t
 -- !query schema
-struct<(CAST(1 AS TINYINT) <=> CAST(true AS TINYINT)):boolean>
+struct<(CAST(1 AS TINYINT) <=> true):boolean>
 -- !query output
 true
 
@@ -318,7 +318,7 @@ true
 -- !query
 SELECT cast(1 as smallint) <=> true FROM t
 -- !query schema
-struct<(CAST(1 AS SMALLINT) <=> CAST(true AS SMALLINT)):boolean>
+struct<(CAST(1 AS SMALLINT) <=> true):boolean>
 -- !query output
 true
 
@@ -326,7 +326,7 @@ true
 -- !query
 SELECT cast(1 as int) <=> true FROM t
 -- !query schema
-struct<(CAST(1 AS INT) <=> CAST(true AS INT)):boolean>
+struct<(CAST(1 AS INT) <=> true):boolean>
 -- !query output
 true
 
@@ -334,7 +334,7 @@ true
 -- !query
 SELECT cast(1 as bigint) <=> true FROM t
 -- !query schema
-struct<(CAST(1 AS BIGINT) <=> CAST(true AS BIGINT)):boolean>
+struct<(CAST(1 AS BIGINT) <=> true):boolean>
 -- !query output
 true
 
@@ -342,7 +342,7 @@ true
 -- !query
 SELECT cast(1 as float) <=> true FROM t
 -- !query schema
-struct<(CAST(1 AS FLOAT) <=> CAST(true AS FLOAT)):boolean>
+struct<(CAST(1 AS FLOAT) <=> true):boolean>
 -- !query output
 true
 
@@ -350,7 +350,7 @@ true
 -- !query
 SELECT cast(1 as double) <=> true FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) <=> CAST(true AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) <=> true):boolean>
 -- !query output
 true
 
@@ -358,7 +358,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) <=> true FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(10,0)) <=> CAST(true AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <=> true):boolean>
 -- !query output
 true
 
@@ -366,7 +366,7 @@ true
 -- !query
 SELECT cast(1 as string) <=> true FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS BOOLEAN) <=> true):boolean>
+struct<(CAST(1 AS STRING) <=> true):boolean>
 -- !query output
 true
 
@@ -409,7 +409,7 @@ cannot resolve '(CAST('2017-12-11 09:30:00' AS DATE) <=> true)' due to data type
 -- !query
 SELECT false = cast(0 as tinyint) FROM t
 -- !query schema
-struct<(CAST(false AS TINYINT) = CAST(0 AS TINYINT)):boolean>
+struct<(false = CAST(0 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -417,7 +417,7 @@ true
 -- !query
 SELECT false = cast(0 as smallint) FROM t
 -- !query schema
-struct<(CAST(false AS SMALLINT) = CAST(0 AS SMALLINT)):boolean>
+struct<(false = CAST(0 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -425,7 +425,7 @@ true
 -- !query
 SELECT false = cast(0 as int) FROM t
 -- !query schema
-struct<(CAST(false AS INT) = CAST(0 AS INT)):boolean>
+struct<(false = CAST(0 AS INT)):boolean>
 -- !query output
 true
 
@@ -433,7 +433,7 @@ true
 -- !query
 SELECT false = cast(0 as bigint) FROM t
 -- !query schema
-struct<(CAST(false AS BIGINT) = CAST(0 AS BIGINT)):boolean>
+struct<(false = CAST(0 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -441,7 +441,7 @@ true
 -- !query
 SELECT false = cast(0 as float) FROM t
 -- !query schema
-struct<(CAST(false AS FLOAT) = CAST(0 AS FLOAT)):boolean>
+struct<(false = CAST(0 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -449,7 +449,7 @@ true
 -- !query
 SELECT false = cast(0 as double) FROM t
 -- !query schema
-struct<(CAST(false AS DOUBLE) = CAST(0 AS DOUBLE)):boolean>
+struct<(false = CAST(0 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -457,7 +457,7 @@ true
 -- !query
 SELECT false = cast(0 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(false AS DECIMAL(10,0)) = CAST(0 AS DECIMAL(10,0))):boolean>
+struct<(false = CAST(0 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -465,7 +465,7 @@ true
 -- !query
 SELECT false = cast(0 as string) FROM t
 -- !query schema
-struct<(false = CAST(CAST(0 AS STRING) AS BOOLEAN)):boolean>
+struct<(false = CAST(0 AS STRING)):boolean>
 -- !query output
 true
 
@@ -508,7 +508,7 @@ cannot resolve '(false = CAST('2017-12-11 09:30:00' AS DATE))' due to data type
 -- !query
 SELECT false <=> cast(0 as tinyint) FROM t
 -- !query schema
-struct<(CAST(false AS TINYINT) <=> CAST(0 AS TINYINT)):boolean>
+struct<(false <=> CAST(0 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -516,7 +516,7 @@ true
 -- !query
 SELECT false <=> cast(0 as smallint) FROM t
 -- !query schema
-struct<(CAST(false AS SMALLINT) <=> CAST(0 AS SMALLINT)):boolean>
+struct<(false <=> CAST(0 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -524,7 +524,7 @@ true
 -- !query
 SELECT false <=> cast(0 as int) FROM t
 -- !query schema
-struct<(CAST(false AS INT) <=> CAST(0 AS INT)):boolean>
+struct<(false <=> CAST(0 AS INT)):boolean>
 -- !query output
 true
 
@@ -532,7 +532,7 @@ true
 -- !query
 SELECT false <=> cast(0 as bigint) FROM t
 -- !query schema
-struct<(CAST(false AS BIGINT) <=> CAST(0 AS BIGINT)):boolean>
+struct<(false <=> CAST(0 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -540,7 +540,7 @@ true
 -- !query
 SELECT false <=> cast(0 as float) FROM t
 -- !query schema
-struct<(CAST(false AS FLOAT) <=> CAST(0 AS FLOAT)):boolean>
+struct<(false <=> CAST(0 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -548,7 +548,7 @@ true
 -- !query
 SELECT false <=> cast(0 as double) FROM t
 -- !query schema
-struct<(CAST(false AS DOUBLE) <=> CAST(0 AS DOUBLE)):boolean>
+struct<(false <=> CAST(0 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -556,7 +556,7 @@ true
 -- !query
 SELECT false <=> cast(0 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(false AS DECIMAL(10,0)) <=> CAST(0 AS DECIMAL(10,0))):boolean>
+struct<(false <=> CAST(0 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -564,7 +564,7 @@ true
 -- !query
 SELECT false <=> cast(0 as string) FROM t
 -- !query schema
-struct<(false <=> CAST(CAST(0 AS STRING) AS BOOLEAN)):boolean>
+struct<(false <=> CAST(0 AS STRING)):boolean>
 -- !query output
 true
 
@@ -607,7 +607,7 @@ cannot resolve '(false <=> CAST('2017-12-11 09:30:00' AS DATE))' due to data typ
 -- !query
 SELECT cast(0 as tinyint) = false FROM t
 -- !query schema
-struct<(CAST(0 AS TINYINT) = CAST(false AS TINYINT)):boolean>
+struct<(CAST(0 AS TINYINT) = false):boolean>
 -- !query output
 true
 
@@ -615,7 +615,7 @@ true
 -- !query
 SELECT cast(0 as smallint) = false FROM t
 -- !query schema
-struct<(CAST(0 AS SMALLINT) = CAST(false AS SMALLINT)):boolean>
+struct<(CAST(0 AS SMALLINT) = false):boolean>
 -- !query output
 true
 
@@ -623,7 +623,7 @@ true
 -- !query
 SELECT cast(0 as int) = false FROM t
 -- !query schema
-struct<(CAST(0 AS INT) = CAST(false AS INT)):boolean>
+struct<(CAST(0 AS INT) = false):boolean>
 -- !query output
 true
 
@@ -631,7 +631,7 @@ true
 -- !query
 SELECT cast(0 as bigint) = false FROM t
 -- !query schema
-struct<(CAST(0 AS BIGINT) = CAST(false AS BIGINT)):boolean>
+struct<(CAST(0 AS BIGINT) = false):boolean>
 -- !query output
 true
 
@@ -639,7 +639,7 @@ true
 -- !query
 SELECT cast(0 as float) = false FROM t
 -- !query schema
-struct<(CAST(0 AS FLOAT) = CAST(false AS FLOAT)):boolean>
+struct<(CAST(0 AS FLOAT) = false):boolean>
 -- !query output
 true
 
@@ -647,7 +647,7 @@ true
 -- !query
 SELECT cast(0 as double) = false FROM t
 -- !query schema
-struct<(CAST(0 AS DOUBLE) = CAST(false AS DOUBLE)):boolean>
+struct<(CAST(0 AS DOUBLE) = false):boolean>
 -- !query output
 true
 
@@ -655,7 +655,7 @@ true
 -- !query
 SELECT cast(0 as decimal(10, 0)) = false FROM t
 -- !query schema
-struct<(CAST(0 AS DECIMAL(10,0)) = CAST(false AS DECIMAL(10,0))):boolean>
+struct<(CAST(0 AS DECIMAL(10,0)) = false):boolean>
 -- !query output
 true
 
@@ -663,7 +663,7 @@ true
 -- !query
 SELECT cast(0 as string) = false FROM t
 -- !query schema
-struct<(CAST(CAST(0 AS STRING) AS BOOLEAN) = false):boolean>
+struct<(CAST(0 AS STRING) = false):boolean>
 -- !query output
 true
 
@@ -706,7 +706,7 @@ cannot resolve '(CAST('2017-12-11 09:30:00' AS DATE) = false)' due to data type
 -- !query
 SELECT cast(0 as tinyint) <=> false FROM t
 -- !query schema
-struct<(CAST(0 AS TINYINT) <=> CAST(false AS TINYINT)):boolean>
+struct<(CAST(0 AS TINYINT) <=> false):boolean>
 -- !query output
 true
 
@@ -714,7 +714,7 @@ true
 -- !query
 SELECT cast(0 as smallint) <=> false FROM t
 -- !query schema
-struct<(CAST(0 AS SMALLINT) <=> CAST(false AS SMALLINT)):boolean>
+struct<(CAST(0 AS SMALLINT) <=> false):boolean>
 -- !query output
 true
 
@@ -722,7 +722,7 @@ true
 -- !query
 SELECT cast(0 as int) <=> false FROM t
 -- !query schema
-struct<(CAST(0 AS INT) <=> CAST(false AS INT)):boolean>
+struct<(CAST(0 AS INT) <=> false):boolean>
 -- !query output
 true
 
@@ -730,7 +730,7 @@ true
 -- !query
 SELECT cast(0 as bigint) <=> false FROM t
 -- !query schema
-struct<(CAST(0 AS BIGINT) <=> CAST(false AS BIGINT)):boolean>
+struct<(CAST(0 AS BIGINT) <=> false):boolean>
 -- !query output
 true
 
@@ -738,7 +738,7 @@ true
 -- !query
 SELECT cast(0 as float) <=> false FROM t
 -- !query schema
-struct<(CAST(0 AS FLOAT) <=> CAST(false AS FLOAT)):boolean>
+struct<(CAST(0 AS FLOAT) <=> false):boolean>
 -- !query output
 true
 
@@ -746,7 +746,7 @@ true
 -- !query
 SELECT cast(0 as double) <=> false FROM t
 -- !query schema
-struct<(CAST(0 AS DOUBLE) <=> CAST(false AS DOUBLE)):boolean>
+struct<(CAST(0 AS DOUBLE) <=> false):boolean>
 -- !query output
 true
 
@@ -754,7 +754,7 @@ true
 -- !query
 SELECT cast(0 as decimal(10, 0)) <=> false FROM t
 -- !query schema
-struct<(CAST(0 AS DECIMAL(10,0)) <=> CAST(false AS DECIMAL(10,0))):boolean>
+struct<(CAST(0 AS DECIMAL(10,0)) <=> false):boolean>
 -- !query output
 true
 
@@ -762,7 +762,7 @@ true
 -- !query
 SELECT cast(0 as string) <=> false FROM t
 -- !query schema
-struct<(CAST(CAST(0 AS STRING) AS BOOLEAN) <=> false):boolean>
+struct<(CAST(0 AS STRING) <=> false):boolean>
 -- !query output
 true
 
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/caseWhenCoercion.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/caseWhenCoercion.sql.out
index 18d97c2f1b42a..da1d1993b90d8 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/caseWhenCoercion.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/caseWhenCoercion.sql.out
@@ -21,7 +21,7 @@ struct<CASE WHEN true THEN CAST(1 AS TINYINT) ELSE CAST(2 AS TINYINT) END:tinyin
 -- !query
 SELECT CASE WHEN true THEN cast(1 as tinyint) ELSE cast(2 as smallint) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS TINYINT) AS SMALLINT) ELSE CAST(2 AS SMALLINT) END:smallint>
+struct<CASE WHEN true THEN CAST(1 AS TINYINT) ELSE CAST(2 AS SMALLINT) END:smallint>
 -- !query output
 1
 
@@ -29,7 +29,7 @@ struct<CASE WHEN true THEN CAST(CAST(1 AS TINYINT) AS SMALLINT) ELSE CAST(2 AS S
 -- !query
 SELECT CASE WHEN true THEN cast(1 as tinyint) ELSE cast(2 as int) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS TINYINT) AS INT) ELSE CAST(2 AS INT) END:int>
+struct<CASE WHEN true THEN CAST(1 AS TINYINT) ELSE CAST(2 AS INT) END:int>
 -- !query output
 1
 
@@ -37,7 +37,7 @@ struct<CASE WHEN true THEN CAST(CAST(1 AS TINYINT) AS INT) ELSE CAST(2 AS INT) E
 -- !query
 SELECT CASE WHEN true THEN cast(1 as tinyint) ELSE cast(2 as bigint) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS TINYINT) AS BIGINT) ELSE CAST(2 AS BIGINT) END:bigint>
+struct<CASE WHEN true THEN CAST(1 AS TINYINT) ELSE CAST(2 AS BIGINT) END:bigint>
 -- !query output
 1
 
@@ -45,7 +45,7 @@ struct<CASE WHEN true THEN CAST(CAST(1 AS TINYINT) AS BIGINT) ELSE CAST(2 AS BIG
 -- !query
 SELECT CASE WHEN true THEN cast(1 as tinyint) ELSE cast(2 as float) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS TINYINT) AS FLOAT) ELSE CAST(2 AS FLOAT) END:float>
+struct<CASE WHEN true THEN CAST(1 AS TINYINT) ELSE CAST(2 AS FLOAT) END:float>
 -- !query output
 1.0
 
@@ -53,7 +53,7 @@ struct<CASE WHEN true THEN CAST(CAST(1 AS TINYINT) AS FLOAT) ELSE CAST(2 AS FLOA
 -- !query
 SELECT CASE WHEN true THEN cast(1 as tinyint) ELSE cast(2 as double) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS TINYINT) AS DOUBLE) ELSE CAST(2 AS DOUBLE) END:double>
+struct<CASE WHEN true THEN CAST(1 AS TINYINT) ELSE CAST(2 AS DOUBLE) END:double>
 -- !query output
 1.0
 
@@ -61,7 +61,7 @@ struct<CASE WHEN true THEN CAST(CAST(1 AS TINYINT) AS DOUBLE) ELSE CAST(2 AS DOU
 -- !query
 SELECT CASE WHEN true THEN cast(1 as tinyint) ELSE cast(2 as decimal(10, 0)) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS TINYINT) AS DECIMAL(10,0)) ELSE CAST(2 AS DECIMAL(10,0)) END:decimal(10,0)>
+struct<CASE WHEN true THEN CAST(1 AS TINYINT) ELSE CAST(2 AS DECIMAL(10,0)) END:decimal(10,0)>
 -- !query output
 1
 
@@ -69,7 +69,7 @@ struct<CASE WHEN true THEN CAST(CAST(1 AS TINYINT) AS DECIMAL(10,0)) ELSE CAST(2
 -- !query
 SELECT CASE WHEN true THEN cast(1 as tinyint) ELSE cast(2 as string) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS TINYINT) AS STRING) ELSE CAST(2 AS STRING) END:string>
+struct<CASE WHEN true THEN CAST(1 AS TINYINT) ELSE CAST(2 AS STRING) END:string>
 -- !query output
 1
 
@@ -113,7 +113,7 @@ cannot resolve 'CASE WHEN true THEN CAST(1 AS TINYINT) ELSE CAST('2017-12-11 09:
 -- !query
 SELECT CASE WHEN true THEN cast(1 as smallint) ELSE cast(2 as tinyint) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS SMALLINT) ELSE CAST(CAST(2 AS TINYINT) AS SMALLINT) END:smallint>
+struct<CASE WHEN true THEN CAST(1 AS SMALLINT) ELSE CAST(2 AS TINYINT) END:smallint>
 -- !query output
 1
 
@@ -129,7 +129,7 @@ struct<CASE WHEN true THEN CAST(1 AS SMALLINT) ELSE CAST(2 AS SMALLINT) END:smal
 -- !query
 SELECT CASE WHEN true THEN cast(1 as smallint) ELSE cast(2 as int) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS SMALLINT) AS INT) ELSE CAST(2 AS INT) END:int>
+struct<CASE WHEN true THEN CAST(1 AS SMALLINT) ELSE CAST(2 AS INT) END:int>
 -- !query output
 1
 
@@ -137,7 +137,7 @@ struct<CASE WHEN true THEN CAST(CAST(1 AS SMALLINT) AS INT) ELSE CAST(2 AS INT)
 -- !query
 SELECT CASE WHEN true THEN cast(1 as smallint) ELSE cast(2 as bigint) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS SMALLINT) AS BIGINT) ELSE CAST(2 AS BIGINT) END:bigint>
+struct<CASE WHEN true THEN CAST(1 AS SMALLINT) ELSE CAST(2 AS BIGINT) END:bigint>
 -- !query output
 1
 
@@ -145,7 +145,7 @@ struct<CASE WHEN true THEN CAST(CAST(1 AS SMALLINT) AS BIGINT) ELSE CAST(2 AS BI
 -- !query
 SELECT CASE WHEN true THEN cast(1 as smallint) ELSE cast(2 as float) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS SMALLINT) AS FLOAT) ELSE CAST(2 AS FLOAT) END:float>
+struct<CASE WHEN true THEN CAST(1 AS SMALLINT) ELSE CAST(2 AS FLOAT) END:float>
 -- !query output
 1.0
 
@@ -153,7 +153,7 @@ struct<CASE WHEN true THEN CAST(CAST(1 AS SMALLINT) AS FLOAT) ELSE CAST(2 AS FLO
 -- !query
 SELECT CASE WHEN true THEN cast(1 as smallint) ELSE cast(2 as double) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS SMALLINT) AS DOUBLE) ELSE CAST(2 AS DOUBLE) END:double>
+struct<CASE WHEN true THEN CAST(1 AS SMALLINT) ELSE CAST(2 AS DOUBLE) END:double>
 -- !query output
 1.0
 
@@ -161,7 +161,7 @@ struct<CASE WHEN true THEN CAST(CAST(1 AS SMALLINT) AS DOUBLE) ELSE CAST(2 AS DO
 -- !query
 SELECT CASE WHEN true THEN cast(1 as smallint) ELSE cast(2 as decimal(10, 0)) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS SMALLINT) AS DECIMAL(10,0)) ELSE CAST(2 AS DECIMAL(10,0)) END:decimal(10,0)>
+struct<CASE WHEN true THEN CAST(1 AS SMALLINT) ELSE CAST(2 AS DECIMAL(10,0)) END:decimal(10,0)>
 -- !query output
 1
 
@@ -169,7 +169,7 @@ struct<CASE WHEN true THEN CAST(CAST(1 AS SMALLINT) AS DECIMAL(10,0)) ELSE CAST(
 -- !query
 SELECT CASE WHEN true THEN cast(1 as smallint) ELSE cast(2 as string) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS SMALLINT) AS STRING) ELSE CAST(2 AS STRING) END:string>
+struct<CASE WHEN true THEN CAST(1 AS SMALLINT) ELSE CAST(2 AS STRING) END:string>
 -- !query output
 1
 
@@ -213,7 +213,7 @@ cannot resolve 'CASE WHEN true THEN CAST(1 AS SMALLINT) ELSE CAST('2017-12-11 09
 -- !query
 SELECT CASE WHEN true THEN cast(1 as int) ELSE cast(2 as tinyint) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS INT) ELSE CAST(CAST(2 AS TINYINT) AS INT) END:int>
+struct<CASE WHEN true THEN CAST(1 AS INT) ELSE CAST(2 AS TINYINT) END:int>
 -- !query output
 1
 
@@ -221,7 +221,7 @@ struct<CASE WHEN true THEN CAST(1 AS INT) ELSE CAST(CAST(2 AS TINYINT) AS INT) E
 -- !query
 SELECT CASE WHEN true THEN cast(1 as int) ELSE cast(2 as smallint) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS INT) ELSE CAST(CAST(2 AS SMALLINT) AS INT) END:int>
+struct<CASE WHEN true THEN CAST(1 AS INT) ELSE CAST(2 AS SMALLINT) END:int>
 -- !query output
 1
 
@@ -237,7 +237,7 @@ struct<CASE WHEN true THEN CAST(1 AS INT) ELSE CAST(2 AS INT) END:int>
 -- !query
 SELECT CASE WHEN true THEN cast(1 as int) ELSE cast(2 as bigint) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS INT) AS BIGINT) ELSE CAST(2 AS BIGINT) END:bigint>
+struct<CASE WHEN true THEN CAST(1 AS INT) ELSE CAST(2 AS BIGINT) END:bigint>
 -- !query output
 1
 
@@ -245,7 +245,7 @@ struct<CASE WHEN true THEN CAST(CAST(1 AS INT) AS BIGINT) ELSE CAST(2 AS BIGINT)
 -- !query
 SELECT CASE WHEN true THEN cast(1 as int) ELSE cast(2 as float) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS INT) AS FLOAT) ELSE CAST(2 AS FLOAT) END:float>
+struct<CASE WHEN true THEN CAST(1 AS INT) ELSE CAST(2 AS FLOAT) END:float>
 -- !query output
 1.0
 
@@ -253,7 +253,7 @@ struct<CASE WHEN true THEN CAST(CAST(1 AS INT) AS FLOAT) ELSE CAST(2 AS FLOAT) E
 -- !query
 SELECT CASE WHEN true THEN cast(1 as int) ELSE cast(2 as double) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS INT) AS DOUBLE) ELSE CAST(2 AS DOUBLE) END:double>
+struct<CASE WHEN true THEN CAST(1 AS INT) ELSE CAST(2 AS DOUBLE) END:double>
 -- !query output
 1.0
 
@@ -261,7 +261,7 @@ struct<CASE WHEN true THEN CAST(CAST(1 AS INT) AS DOUBLE) ELSE CAST(2 AS DOUBLE)
 -- !query
 SELECT CASE WHEN true THEN cast(1 as int) ELSE cast(2 as decimal(10, 0)) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS INT) AS DECIMAL(10,0)) ELSE CAST(2 AS DECIMAL(10,0)) END:decimal(10,0)>
+struct<CASE WHEN true THEN CAST(1 AS INT) ELSE CAST(2 AS DECIMAL(10,0)) END:decimal(10,0)>
 -- !query output
 1
 
@@ -269,7 +269,7 @@ struct<CASE WHEN true THEN CAST(CAST(1 AS INT) AS DECIMAL(10,0)) ELSE CAST(2 AS
 -- !query
 SELECT CASE WHEN true THEN cast(1 as int) ELSE cast(2 as string) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS INT) AS STRING) ELSE CAST(2 AS STRING) END:string>
+struct<CASE WHEN true THEN CAST(1 AS INT) ELSE CAST(2 AS STRING) END:string>
 -- !query output
 1
 
@@ -313,7 +313,7 @@ cannot resolve 'CASE WHEN true THEN CAST(1 AS INT) ELSE CAST('2017-12-11 09:30:0
 -- !query
 SELECT CASE WHEN true THEN cast(1 as bigint) ELSE cast(2 as tinyint) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS BIGINT) ELSE CAST(CAST(2 AS TINYINT) AS BIGINT) END:bigint>
+struct<CASE WHEN true THEN CAST(1 AS BIGINT) ELSE CAST(2 AS TINYINT) END:bigint>
 -- !query output
 1
 
@@ -321,7 +321,7 @@ struct<CASE WHEN true THEN CAST(1 AS BIGINT) ELSE CAST(CAST(2 AS TINYINT) AS BIG
 -- !query
 SELECT CASE WHEN true THEN cast(1 as bigint) ELSE cast(2 as smallint) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS BIGINT) ELSE CAST(CAST(2 AS SMALLINT) AS BIGINT) END:bigint>
+struct<CASE WHEN true THEN CAST(1 AS BIGINT) ELSE CAST(2 AS SMALLINT) END:bigint>
 -- !query output
 1
 
@@ -329,7 +329,7 @@ struct<CASE WHEN true THEN CAST(1 AS BIGINT) ELSE CAST(CAST(2 AS SMALLINT) AS BI
 -- !query
 SELECT CASE WHEN true THEN cast(1 as bigint) ELSE cast(2 as int) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS BIGINT) ELSE CAST(CAST(2 AS INT) AS BIGINT) END:bigint>
+struct<CASE WHEN true THEN CAST(1 AS BIGINT) ELSE CAST(2 AS INT) END:bigint>
 -- !query output
 1
 
@@ -345,7 +345,7 @@ struct<CASE WHEN true THEN CAST(1 AS BIGINT) ELSE CAST(2 AS BIGINT) END:bigint>
 -- !query
 SELECT CASE WHEN true THEN cast(1 as bigint) ELSE cast(2 as float) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS BIGINT) AS FLOAT) ELSE CAST(2 AS FLOAT) END:float>
+struct<CASE WHEN true THEN CAST(1 AS BIGINT) ELSE CAST(2 AS FLOAT) END:float>
 -- !query output
 1.0
 
@@ -353,7 +353,7 @@ struct<CASE WHEN true THEN CAST(CAST(1 AS BIGINT) AS FLOAT) ELSE CAST(2 AS FLOAT
 -- !query
 SELECT CASE WHEN true THEN cast(1 as bigint) ELSE cast(2 as double) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS BIGINT) AS DOUBLE) ELSE CAST(2 AS DOUBLE) END:double>
+struct<CASE WHEN true THEN CAST(1 AS BIGINT) ELSE CAST(2 AS DOUBLE) END:double>
 -- !query output
 1.0
 
@@ -361,7 +361,7 @@ struct<CASE WHEN true THEN CAST(CAST(1 AS BIGINT) AS DOUBLE) ELSE CAST(2 AS DOUB
 -- !query
 SELECT CASE WHEN true THEN cast(1 as bigint) ELSE cast(2 as decimal(10, 0)) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) ELSE CAST(CAST(2 AS DECIMAL(10,0)) AS DECIMAL(20,0)) END:decimal(20,0)>
+struct<CASE WHEN true THEN CAST(1 AS BIGINT) ELSE CAST(2 AS DECIMAL(10,0)) END:decimal(20,0)>
 -- !query output
 1
 
@@ -369,7 +369,7 @@ struct<CASE WHEN true THEN CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) ELSE CAST(CA
 -- !query
 SELECT CASE WHEN true THEN cast(1 as bigint) ELSE cast(2 as string) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS BIGINT) AS STRING) ELSE CAST(2 AS STRING) END:string>
+struct<CASE WHEN true THEN CAST(1 AS BIGINT) ELSE CAST(2 AS STRING) END:string>
 -- !query output
 1
 
@@ -413,7 +413,7 @@ cannot resolve 'CASE WHEN true THEN CAST(1 AS BIGINT) ELSE CAST('2017-12-11 09:3
 -- !query
 SELECT CASE WHEN true THEN cast(1 as float) ELSE cast(2 as tinyint) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS FLOAT) ELSE CAST(CAST(2 AS TINYINT) AS FLOAT) END:float>
+struct<CASE WHEN true THEN CAST(1 AS FLOAT) ELSE CAST(2 AS TINYINT) END:float>
 -- !query output
 1.0
 
@@ -421,7 +421,7 @@ struct<CASE WHEN true THEN CAST(1 AS FLOAT) ELSE CAST(CAST(2 AS TINYINT) AS FLOA
 -- !query
 SELECT CASE WHEN true THEN cast(1 as float) ELSE cast(2 as smallint) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS FLOAT) ELSE CAST(CAST(2 AS SMALLINT) AS FLOAT) END:float>
+struct<CASE WHEN true THEN CAST(1 AS FLOAT) ELSE CAST(2 AS SMALLINT) END:float>
 -- !query output
 1.0
 
@@ -429,7 +429,7 @@ struct<CASE WHEN true THEN CAST(1 AS FLOAT) ELSE CAST(CAST(2 AS SMALLINT) AS FLO
 -- !query
 SELECT CASE WHEN true THEN cast(1 as float) ELSE cast(2 as int) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS FLOAT) ELSE CAST(CAST(2 AS INT) AS FLOAT) END:float>
+struct<CASE WHEN true THEN CAST(1 AS FLOAT) ELSE CAST(2 AS INT) END:float>
 -- !query output
 1.0
 
@@ -437,7 +437,7 @@ struct<CASE WHEN true THEN CAST(1 AS FLOAT) ELSE CAST(CAST(2 AS INT) AS FLOAT) E
 -- !query
 SELECT CASE WHEN true THEN cast(1 as float) ELSE cast(2 as bigint) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS FLOAT) ELSE CAST(CAST(2 AS BIGINT) AS FLOAT) END:float>
+struct<CASE WHEN true THEN CAST(1 AS FLOAT) ELSE CAST(2 AS BIGINT) END:float>
 -- !query output
 1.0
 
@@ -453,7 +453,7 @@ struct<CASE WHEN true THEN CAST(1 AS FLOAT) ELSE CAST(2 AS FLOAT) END:float>
 -- !query
 SELECT CASE WHEN true THEN cast(1 as float) ELSE cast(2 as double) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS FLOAT) AS DOUBLE) ELSE CAST(2 AS DOUBLE) END:double>
+struct<CASE WHEN true THEN CAST(1 AS FLOAT) ELSE CAST(2 AS DOUBLE) END:double>
 -- !query output
 1.0
 
@@ -461,7 +461,7 @@ struct<CASE WHEN true THEN CAST(CAST(1 AS FLOAT) AS DOUBLE) ELSE CAST(2 AS DOUBL
 -- !query
 SELECT CASE WHEN true THEN cast(1 as float) ELSE cast(2 as decimal(10, 0)) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS FLOAT) AS DOUBLE) ELSE CAST(CAST(2 AS DECIMAL(10,0)) AS DOUBLE) END:double>
+struct<CASE WHEN true THEN CAST(1 AS FLOAT) ELSE CAST(2 AS DECIMAL(10,0)) END:double>
 -- !query output
 1.0
 
@@ -469,7 +469,7 @@ struct<CASE WHEN true THEN CAST(CAST(1 AS FLOAT) AS DOUBLE) ELSE CAST(CAST(2 AS
 -- !query
 SELECT CASE WHEN true THEN cast(1 as float) ELSE cast(2 as string) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS FLOAT) AS STRING) ELSE CAST(2 AS STRING) END:string>
+struct<CASE WHEN true THEN CAST(1 AS FLOAT) ELSE CAST(2 AS STRING) END:string>
 -- !query output
 1.0
 
@@ -513,7 +513,7 @@ cannot resolve 'CASE WHEN true THEN CAST(1 AS FLOAT) ELSE CAST('2017-12-11 09:30
 -- !query
 SELECT CASE WHEN true THEN cast(1 as double) ELSE cast(2 as tinyint) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS DOUBLE) ELSE CAST(CAST(2 AS TINYINT) AS DOUBLE) END:double>
+struct<CASE WHEN true THEN CAST(1 AS DOUBLE) ELSE CAST(2 AS TINYINT) END:double>
 -- !query output
 1.0
 
@@ -521,7 +521,7 @@ struct<CASE WHEN true THEN CAST(1 AS DOUBLE) ELSE CAST(CAST(2 AS TINYINT) AS DOU
 -- !query
 SELECT CASE WHEN true THEN cast(1 as double) ELSE cast(2 as smallint) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS DOUBLE) ELSE CAST(CAST(2 AS SMALLINT) AS DOUBLE) END:double>
+struct<CASE WHEN true THEN CAST(1 AS DOUBLE) ELSE CAST(2 AS SMALLINT) END:double>
 -- !query output
 1.0
 
@@ -529,7 +529,7 @@ struct<CASE WHEN true THEN CAST(1 AS DOUBLE) ELSE CAST(CAST(2 AS SMALLINT) AS DO
 -- !query
 SELECT CASE WHEN true THEN cast(1 as double) ELSE cast(2 as int) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS DOUBLE) ELSE CAST(CAST(2 AS INT) AS DOUBLE) END:double>
+struct<CASE WHEN true THEN CAST(1 AS DOUBLE) ELSE CAST(2 AS INT) END:double>
 -- !query output
 1.0
 
@@ -537,7 +537,7 @@ struct<CASE WHEN true THEN CAST(1 AS DOUBLE) ELSE CAST(CAST(2 AS INT) AS DOUBLE)
 -- !query
 SELECT CASE WHEN true THEN cast(1 as double) ELSE cast(2 as bigint) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS DOUBLE) ELSE CAST(CAST(2 AS BIGINT) AS DOUBLE) END:double>
+struct<CASE WHEN true THEN CAST(1 AS DOUBLE) ELSE CAST(2 AS BIGINT) END:double>
 -- !query output
 1.0
 
@@ -545,7 +545,7 @@ struct<CASE WHEN true THEN CAST(1 AS DOUBLE) ELSE CAST(CAST(2 AS BIGINT) AS DOUB
 -- !query
 SELECT CASE WHEN true THEN cast(1 as double) ELSE cast(2 as float) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS DOUBLE) ELSE CAST(CAST(2 AS FLOAT) AS DOUBLE) END:double>
+struct<CASE WHEN true THEN CAST(1 AS DOUBLE) ELSE CAST(2 AS FLOAT) END:double>
 -- !query output
 1.0
 
@@ -561,7 +561,7 @@ struct<CASE WHEN true THEN CAST(1 AS DOUBLE) ELSE CAST(2 AS DOUBLE) END:double>
 -- !query
 SELECT CASE WHEN true THEN cast(1 as double) ELSE cast(2 as decimal(10, 0)) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS DOUBLE) ELSE CAST(CAST(2 AS DECIMAL(10,0)) AS DOUBLE) END:double>
+struct<CASE WHEN true THEN CAST(1 AS DOUBLE) ELSE CAST(2 AS DECIMAL(10,0)) END:double>
 -- !query output
 1.0
 
@@ -569,7 +569,7 @@ struct<CASE WHEN true THEN CAST(1 AS DOUBLE) ELSE CAST(CAST(2 AS DECIMAL(10,0))
 -- !query
 SELECT CASE WHEN true THEN cast(1 as double) ELSE cast(2 as string) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS DOUBLE) AS STRING) ELSE CAST(2 AS STRING) END:string>
+struct<CASE WHEN true THEN CAST(1 AS DOUBLE) ELSE CAST(2 AS STRING) END:string>
 -- !query output
 1.0
 
@@ -613,7 +613,7 @@ cannot resolve 'CASE WHEN true THEN CAST(1 AS DOUBLE) ELSE CAST('2017-12-11 09:3
 -- !query
 SELECT CASE WHEN true THEN cast(1 as decimal(10, 0)) ELSE cast(2 as tinyint) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS DECIMAL(10,0)) ELSE CAST(CAST(2 AS TINYINT) AS DECIMAL(10,0)) END:decimal(10,0)>
+struct<CASE WHEN true THEN CAST(1 AS DECIMAL(10,0)) ELSE CAST(2 AS TINYINT) END:decimal(10,0)>
 -- !query output
 1
 
@@ -621,7 +621,7 @@ struct<CASE WHEN true THEN CAST(1 AS DECIMAL(10,0)) ELSE CAST(CAST(2 AS TINYINT)
 -- !query
 SELECT CASE WHEN true THEN cast(1 as decimal(10, 0)) ELSE cast(2 as smallint) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS DECIMAL(10,0)) ELSE CAST(CAST(2 AS SMALLINT) AS DECIMAL(10,0)) END:decimal(10,0)>
+struct<CASE WHEN true THEN CAST(1 AS DECIMAL(10,0)) ELSE CAST(2 AS SMALLINT) END:decimal(10,0)>
 -- !query output
 1
 
@@ -629,7 +629,7 @@ struct<CASE WHEN true THEN CAST(1 AS DECIMAL(10,0)) ELSE CAST(CAST(2 AS SMALLINT
 -- !query
 SELECT CASE WHEN true THEN cast(1 as decimal(10, 0)) ELSE cast(2 as int) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS DECIMAL(10,0)) ELSE CAST(CAST(2 AS INT) AS DECIMAL(10,0)) END:decimal(10,0)>
+struct<CASE WHEN true THEN CAST(1 AS DECIMAL(10,0)) ELSE CAST(2 AS INT) END:decimal(10,0)>
 -- !query output
 1
 
@@ -637,7 +637,7 @@ struct<CASE WHEN true THEN CAST(1 AS DECIMAL(10,0)) ELSE CAST(CAST(2 AS INT) AS
 -- !query
 SELECT CASE WHEN true THEN cast(1 as decimal(10, 0)) ELSE cast(2 as bigint) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) ELSE CAST(CAST(2 AS BIGINT) AS DECIMAL(20,0)) END:decimal(20,0)>
+struct<CASE WHEN true THEN CAST(1 AS DECIMAL(10,0)) ELSE CAST(2 AS BIGINT) END:decimal(20,0)>
 -- !query output
 1
 
@@ -645,7 +645,7 @@ struct<CASE WHEN true THEN CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) ELSE
 -- !query
 SELECT CASE WHEN true THEN cast(1 as decimal(10, 0)) ELSE cast(2 as float) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) ELSE CAST(CAST(2 AS FLOAT) AS DOUBLE) END:double>
+struct<CASE WHEN true THEN CAST(1 AS DECIMAL(10,0)) ELSE CAST(2 AS FLOAT) END:double>
 -- !query output
 1.0
 
@@ -653,7 +653,7 @@ struct<CASE WHEN true THEN CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) ELSE CAST(CA
 -- !query
 SELECT CASE WHEN true THEN cast(1 as decimal(10, 0)) ELSE cast(2 as double) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) ELSE CAST(2 AS DOUBLE) END:double>
+struct<CASE WHEN true THEN CAST(1 AS DECIMAL(10,0)) ELSE CAST(2 AS DOUBLE) END:double>
 -- !query output
 1.0
 
@@ -669,7 +669,7 @@ struct<CASE WHEN true THEN CAST(1 AS DECIMAL(10,0)) ELSE CAST(2 AS DECIMAL(10,0)
 -- !query
 SELECT CASE WHEN true THEN cast(1 as decimal(10, 0)) ELSE cast(2 as string) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(1 AS DECIMAL(10,0)) AS STRING) ELSE CAST(2 AS STRING) END:string>
+struct<CASE WHEN true THEN CAST(1 AS DECIMAL(10,0)) ELSE CAST(2 AS STRING) END:string>
 -- !query output
 1
 
@@ -713,7 +713,7 @@ cannot resolve 'CASE WHEN true THEN CAST(1 AS DECIMAL(10,0)) ELSE CAST('2017-12-
 -- !query
 SELECT CASE WHEN true THEN cast(1 as string) ELSE cast(2 as tinyint) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(CAST(2 AS TINYINT) AS STRING) END:string>
+struct<CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(2 AS TINYINT) END:string>
 -- !query output
 1
 
@@ -721,7 +721,7 @@ struct<CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(CAST(2 AS TINYINT) AS STR
 -- !query
 SELECT CASE WHEN true THEN cast(1 as string) ELSE cast(2 as smallint) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(CAST(2 AS SMALLINT) AS STRING) END:string>
+struct<CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(2 AS SMALLINT) END:string>
 -- !query output
 1
 
@@ -729,7 +729,7 @@ struct<CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(CAST(2 AS SMALLINT) AS ST
 -- !query
 SELECT CASE WHEN true THEN cast(1 as string) ELSE cast(2 as int) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(CAST(2 AS INT) AS STRING) END:string>
+struct<CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(2 AS INT) END:string>
 -- !query output
 1
 
@@ -737,7 +737,7 @@ struct<CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(CAST(2 AS INT) AS STRING)
 -- !query
 SELECT CASE WHEN true THEN cast(1 as string) ELSE cast(2 as bigint) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(CAST(2 AS BIGINT) AS STRING) END:string>
+struct<CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(2 AS BIGINT) END:string>
 -- !query output
 1
 
@@ -745,7 +745,7 @@ struct<CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(CAST(2 AS BIGINT) AS STRI
 -- !query
 SELECT CASE WHEN true THEN cast(1 as string) ELSE cast(2 as float) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(CAST(2 AS FLOAT) AS STRING) END:string>
+struct<CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(2 AS FLOAT) END:string>
 -- !query output
 1
 
@@ -753,7 +753,7 @@ struct<CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(CAST(2 AS FLOAT) AS STRIN
 -- !query
 SELECT CASE WHEN true THEN cast(1 as string) ELSE cast(2 as double) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(CAST(2 AS DOUBLE) AS STRING) END:string>
+struct<CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(2 AS DOUBLE) END:string>
 -- !query output
 1
 
@@ -761,7 +761,7 @@ struct<CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(CAST(2 AS DOUBLE) AS STRI
 -- !query
 SELECT CASE WHEN true THEN cast(1 as string) ELSE cast(2 as decimal(10, 0)) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(CAST(2 AS DECIMAL(10,0)) AS STRING) END:string>
+struct<CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(2 AS DECIMAL(10,0)) END:string>
 -- !query output
 1
 
@@ -795,7 +795,7 @@ cannot resolve 'CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(2 AS BOOLEAN) EN
 -- !query
 SELECT CASE WHEN true THEN cast(1 as string) ELSE cast('2017-12-11 09:30:00.0' as timestamp) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) AS STRING) END:string>
+struct<CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) END:string>
 -- !query output
 1
 
@@ -803,7 +803,7 @@ struct<CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(CAST(2017-12-11 09:30:00.
 -- !query
 SELECT CASE WHEN true THEN cast(1 as string) ELSE cast('2017-12-11 09:30:00' as date) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(CAST(2017-12-11 09:30:00 AS DATE) AS STRING) END:string>
+struct<CASE WHEN true THEN CAST(1 AS STRING) ELSE CAST(2017-12-11 09:30:00 AS DATE) END:string>
 -- !query output
 1
 
@@ -1088,7 +1088,7 @@ cannot resolve 'CASE WHEN true THEN CAST('2017-12-12 09:30:00.0' AS TIMESTAMP) E
 -- !query
 SELECT CASE WHEN true THEN cast('2017-12-12 09:30:00.0' as timestamp) ELSE cast(2 as string) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(2017-12-12 09:30:00.0 AS TIMESTAMP) AS STRING) ELSE CAST(2 AS STRING) END:string>
+struct<CASE WHEN true THEN CAST(2017-12-12 09:30:00.0 AS TIMESTAMP) ELSE CAST(2 AS STRING) END:string>
 -- !query output
 2017-12-12 09:30:00
 
@@ -1122,7 +1122,7 @@ struct<CASE WHEN true THEN CAST(2017-12-12 09:30:00.0 AS TIMESTAMP) ELSE CAST(20
 -- !query
 SELECT CASE WHEN true THEN cast('2017-12-12 09:30:00.0' as timestamp) ELSE cast('2017-12-11 09:30:00' as date) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(2017-12-12 09:30:00.0 AS TIMESTAMP) ELSE CAST(CAST(2017-12-11 09:30:00 AS DATE) AS TIMESTAMP) END:timestamp>
+struct<CASE WHEN true THEN CAST(2017-12-12 09:30:00.0 AS TIMESTAMP) ELSE CAST(2017-12-11 09:30:00 AS DATE) END:timestamp>
 -- !query output
 2017-12-12 09:30:00
 
@@ -1193,7 +1193,7 @@ cannot resolve 'CASE WHEN true THEN CAST('2017-12-12 09:30:00' AS DATE) ELSE CAS
 -- !query
 SELECT CASE WHEN true THEN cast('2017-12-12 09:30:00' as date) ELSE cast(2 as string) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(2017-12-12 09:30:00 AS DATE) AS STRING) ELSE CAST(2 AS STRING) END:string>
+struct<CASE WHEN true THEN CAST(2017-12-12 09:30:00 AS DATE) ELSE CAST(2 AS STRING) END:string>
 -- !query output
 2017-12-12
 
@@ -1219,7 +1219,7 @@ cannot resolve 'CASE WHEN true THEN CAST('2017-12-12 09:30:00' AS DATE) ELSE CAS
 -- !query
 SELECT CASE WHEN true THEN cast('2017-12-12 09:30:00' as date) ELSE cast('2017-12-11 09:30:00.0' as timestamp) END FROM t
 -- !query schema
-struct<CASE WHEN true THEN CAST(CAST(2017-12-12 09:30:00 AS DATE) AS TIMESTAMP) ELSE CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) END:timestamp>
+struct<CASE WHEN true THEN CAST(2017-12-12 09:30:00 AS DATE) ELSE CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) END:timestamp>
 -- !query output
 2017-12-12 00:00:00
 
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/dateTimeOperations.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/dateTimeOperations.sql.out
index 41f888ee28923..9cad1e6887a09 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/dateTimeOperations.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/dateTimeOperations.sql.out
@@ -76,7 +76,7 @@ cannot resolve 'CAST(1 AS DECIMAL(10,0)) + INTERVAL '2 days'' due to data type m
 -- !query
 select cast('2017-12-11' as string) + interval 2 day
 -- !query schema
-struct<CAST(CAST(CAST(2017-12-11 AS STRING) AS TIMESTAMP) + INTERVAL '2 days' AS STRING):string>
+struct<CAST(2017-12-11 AS STRING) + INTERVAL '2 days':string>
 -- !query output
 2017-12-13 00:00:00
 
@@ -84,7 +84,7 @@ struct<CAST(CAST(CAST(2017-12-11 AS STRING) AS TIMESTAMP) + INTERVAL '2 days' AS
 -- !query
 select cast('2017-12-11 09:30:00' as string) + interval 2 day
 -- !query schema
-struct<CAST(CAST(CAST(2017-12-11 09:30:00 AS STRING) AS TIMESTAMP) + INTERVAL '2 days' AS STRING):string>
+struct<CAST(2017-12-11 09:30:00 AS STRING) + INTERVAL '2 days':string>
 -- !query output
 2017-12-13 09:30:00
 
@@ -110,7 +110,7 @@ cannot resolve 'CAST(1 AS BOOLEAN) + INTERVAL '2 days'' due to data type mismatc
 -- !query
 select cast('2017-12-11 09:30:00.0' as timestamp) + interval 2 day
 -- !query schema
-struct<CAST(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) + INTERVAL '2 days' AS TIMESTAMP):timestamp>
+struct<CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) + INTERVAL '2 days':timestamp>
 -- !query output
 2017-12-13 09:30:00
 
@@ -189,7 +189,7 @@ cannot resolve 'CAST(1 AS DECIMAL(10,0)) + INTERVAL '2 days'' due to data type m
 -- !query
 select interval 2 day + cast('2017-12-11' as string)
 -- !query schema
-struct<CAST(CAST(CAST(2017-12-11 AS STRING) AS TIMESTAMP) + INTERVAL '2 days' AS STRING):string>
+struct<CAST(2017-12-11 AS STRING) + INTERVAL '2 days':string>
 -- !query output
 2017-12-13 00:00:00
 
@@ -197,7 +197,7 @@ struct<CAST(CAST(CAST(2017-12-11 AS STRING) AS TIMESTAMP) + INTERVAL '2 days' AS
 -- !query
 select interval 2 day + cast('2017-12-11 09:30:00' as string)
 -- !query schema
-struct<CAST(CAST(CAST(2017-12-11 09:30:00 AS STRING) AS TIMESTAMP) + INTERVAL '2 days' AS STRING):string>
+struct<CAST(2017-12-11 09:30:00 AS STRING) + INTERVAL '2 days':string>
 -- !query output
 2017-12-13 09:30:00
 
@@ -223,7 +223,7 @@ cannot resolve 'CAST(1 AS BOOLEAN) + INTERVAL '2 days'' due to data type mismatc
 -- !query
 select interval 2 day + cast('2017-12-11 09:30:00.0' as timestamp)
 -- !query schema
-struct<CAST(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) + INTERVAL '2 days' AS TIMESTAMP):timestamp>
+struct<CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) + INTERVAL '2 days':timestamp>
 -- !query output
 2017-12-13 09:30:00
 
@@ -302,7 +302,7 @@ cannot resolve 'CAST(1 AS DECIMAL(10,0)) + (- INTERVAL '2 days')' due to data ty
 -- !query
 select cast('2017-12-11' as string) - interval 2 day
 -- !query schema
-struct<CAST(CAST(2017-12-11 AS STRING) - INTERVAL '2 days' AS STRING):string>
+struct<CAST(2017-12-11 AS STRING) - INTERVAL '2 days':string>
 -- !query output
 2017-12-09 00:00:00
 
@@ -310,7 +310,7 @@ struct<CAST(CAST(2017-12-11 AS STRING) - INTERVAL '2 days' AS STRING):string>
 -- !query
 select cast('2017-12-11 09:30:00' as string) - interval 2 day
 -- !query schema
-struct<CAST(CAST(2017-12-11 09:30:00 AS STRING) - INTERVAL '2 days' AS STRING):string>
+struct<CAST(2017-12-11 09:30:00 AS STRING) - INTERVAL '2 days':string>
 -- !query output
 2017-12-09 09:30:00
 
@@ -336,7 +336,7 @@ cannot resolve 'CAST(1 AS BOOLEAN) + (- INTERVAL '2 days')' due to data type mis
 -- !query
 select cast('2017-12-11 09:30:00.0' as timestamp) - interval 2 day
 -- !query schema
-struct<CAST(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) - INTERVAL '2 days' AS TIMESTAMP):timestamp>
+struct<CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) - INTERVAL '2 days':timestamp>
 -- !query output
 2017-12-09 09:30:00
 
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/decimalPrecision.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/decimalPrecision.sql.out
index 33bd3850732f0..58c34aafa3207 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/decimalPrecision.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/decimalPrecision.sql.out
@@ -13,7 +13,7 @@ struct<>
 -- !query
 SELECT cast(1 as tinyint) + cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) + CAST(1 AS DECIMAL(3,0))):decimal(4,0)>
+struct<(CAST(1 AS TINYINT) + CAST(1 AS DECIMAL(3,0))):decimal(4,0)>
 -- !query output
 2
 
@@ -21,7 +21,7 @@ struct<(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) + CAST(1 AS DECIMAL(3,0))):deci
 -- !query
 SELECT cast(1 as tinyint) + cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(6,0)) + CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(6,0))):decimal(6,0)>
+struct<(CAST(1 AS TINYINT) + CAST(1 AS DECIMAL(5,0))):decimal(6,0)>
 -- !query output
 2
 
@@ -29,7 +29,7 @@ struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(6,0)) + CAST(CA
 -- !query
 SELECT cast(1 as tinyint) + cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(11,0)) + CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(11,0))):decimal(11,0)>
+struct<(CAST(1 AS TINYINT) + CAST(1 AS DECIMAL(10,0))):decimal(11,0)>
 -- !query output
 2
 
@@ -37,7 +37,7 @@ struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(11,0)) + CAST(C
 -- !query
 SELECT cast(1 as tinyint) + cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(21,0)) + CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS TINYINT) + CAST(1 AS DECIMAL(20,0))):decimal(21,0)>
 -- !query output
 2
 
@@ -45,7 +45,7 @@ struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(21,0)) + CAST(C
 -- !query
 SELECT cast(1 as smallint) + cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(6,0)) + CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(6,0))):decimal(6,0)>
+struct<(CAST(1 AS SMALLINT) + CAST(1 AS DECIMAL(3,0))):decimal(6,0)>
 -- !query output
 2
 
@@ -53,7 +53,7 @@ struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(6,0)) + CAST(C
 -- !query
 SELECT cast(1 as smallint) + cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) + CAST(1 AS DECIMAL(5,0))):decimal(6,0)>
+struct<(CAST(1 AS SMALLINT) + CAST(1 AS DECIMAL(5,0))):decimal(6,0)>
 -- !query output
 2
 
@@ -61,7 +61,7 @@ struct<(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) + CAST(1 AS DECIMAL(5,0))):dec
 -- !query
 SELECT cast(1 as smallint) + cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(11,0)) + CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(11,0))):decimal(11,0)>
+struct<(CAST(1 AS SMALLINT) + CAST(1 AS DECIMAL(10,0))):decimal(11,0)>
 -- !query output
 2
 
@@ -69,7 +69,7 @@ struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(11,0)) + CAST(
 -- !query
 SELECT cast(1 as smallint) + cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(21,0)) + CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS SMALLINT) + CAST(1 AS DECIMAL(20,0))):decimal(21,0)>
 -- !query output
 2
 
@@ -77,7 +77,7 @@ struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(21,0)) + CAST(
 -- !query
 SELECT cast(1 as int) + cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(11,0)) + CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(11,0))):decimal(11,0)>
+struct<(CAST(1 AS INT) + CAST(1 AS DECIMAL(3,0))):decimal(11,0)>
 -- !query output
 2
 
@@ -85,7 +85,7 @@ struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(11,0)) + CAST(CAST
 -- !query
 SELECT cast(1 as int) + cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(11,0)) + CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(11,0))):decimal(11,0)>
+struct<(CAST(1 AS INT) + CAST(1 AS DECIMAL(5,0))):decimal(11,0)>
 -- !query output
 2
 
@@ -93,7 +93,7 @@ struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(11,0)) + CAST(CAST
 -- !query
 SELECT cast(1 as int) + cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) + CAST(1 AS DECIMAL(10,0))):decimal(11,0)>
+struct<(CAST(1 AS INT) + CAST(1 AS DECIMAL(10,0))):decimal(11,0)>
 -- !query output
 2
 
@@ -101,7 +101,7 @@ struct<(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) + CAST(1 AS DECIMAL(10,0))):decima
 -- !query
 SELECT cast(1 as int) + cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(21,0)) + CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS INT) + CAST(1 AS DECIMAL(20,0))):decimal(21,0)>
 -- !query output
 2
 
@@ -109,7 +109,7 @@ struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(21,0)) + CAST(CAST
 -- !query
 SELECT cast(1 as bigint) + cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(21,0)) + CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS BIGINT) + CAST(1 AS DECIMAL(3,0))):decimal(21,0)>
 -- !query output
 2
 
@@ -117,7 +117,7 @@ struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(21,0)) + CAST(C
 -- !query
 SELECT cast(1 as bigint) + cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(21,0)) + CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS BIGINT) + CAST(1 AS DECIMAL(5,0))):decimal(21,0)>
 -- !query output
 2
 
@@ -125,7 +125,7 @@ struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(21,0)) + CAST(C
 -- !query
 SELECT cast(1 as bigint) + cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(21,0)) + CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS BIGINT) + CAST(1 AS DECIMAL(10,0))):decimal(21,0)>
 -- !query output
 2
 
@@ -133,7 +133,7 @@ struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(21,0)) + CAST(C
 -- !query
 SELECT cast(1 as bigint) + cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) + CAST(1 AS DECIMAL(20,0))):decimal(21,0)>
+struct<(CAST(1 AS BIGINT) + CAST(1 AS DECIMAL(20,0))):decimal(21,0)>
 -- !query output
 2
 
@@ -141,7 +141,7 @@ struct<(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) + CAST(1 AS DECIMAL(20,0))):dec
 -- !query
 SELECT cast(1 as float) + cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) + CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) + CAST(1 AS DECIMAL(3,0))):double>
 -- !query output
 2.0
 
@@ -149,7 +149,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) + CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBL
 -- !query
 SELECT cast(1 as float) + cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) + CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) + CAST(1 AS DECIMAL(5,0))):double>
 -- !query output
 2.0
 
@@ -157,7 +157,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) + CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBL
 -- !query
 SELECT cast(1 as float) + cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) + CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) + CAST(1 AS DECIMAL(10,0))):double>
 -- !query output
 2.0
 
@@ -165,7 +165,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) + CAST(CAST(1 AS DECIMAL(10,0)) AS DOUB
 -- !query
 SELECT cast(1 as float) + cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) + CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) + CAST(1 AS DECIMAL(20,0))):double>
 -- !query output
 2.0
 
@@ -173,7 +173,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) + CAST(CAST(1 AS DECIMAL(20,0)) AS DOUB
 -- !query
 SELECT cast(1 as double) + cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) + CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) + CAST(1 AS DECIMAL(3,0))):double>
 -- !query output
 2.0
 
@@ -181,7 +181,7 @@ struct<(CAST(1 AS DOUBLE) + CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as double) + cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) + CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) + CAST(1 AS DECIMAL(5,0))):double>
 -- !query output
 2.0
 
@@ -189,7 +189,7 @@ struct<(CAST(1 AS DOUBLE) + CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as double) + cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) + CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) + CAST(1 AS DECIMAL(10,0))):double>
 -- !query output
 2.0
 
@@ -197,7 +197,7 @@ struct<(CAST(1 AS DOUBLE) + CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as double) + cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) + CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) + CAST(1 AS DECIMAL(20,0))):double>
 -- !query output
 2.0
 
@@ -205,7 +205,7 @@ struct<(CAST(1 AS DOUBLE) + CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(10, 0)) + cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(11,0)) + CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(11,0))):decimal(11,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) + CAST(1 AS DECIMAL(3,0))):decimal(11,0)>
 -- !query output
 2
 
@@ -213,7 +213,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(11,0)) + CAST(CAST(1 AS DECIMAL
 -- !query
 SELECT cast(1 as decimal(10, 0)) + cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(11,0)) + CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(11,0))):decimal(11,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) + CAST(1 AS DECIMAL(5,0))):decimal(11,0)>
 -- !query output
 2
 
@@ -229,7 +229,7 @@ struct<(CAST(1 AS DECIMAL(10,0)) + CAST(1 AS DECIMAL(10,0))):decimal(11,0)>
 -- !query
 SELECT cast(1 as decimal(10, 0)) + cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(21,0)) + CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) + CAST(1 AS DECIMAL(20,0))):decimal(21,0)>
 -- !query output
 2
 
@@ -345,7 +345,7 @@ cannot resolve 'date_add(CAST('2017-12-11 09:30:00' AS DATE), CAST(1 AS DECIMAL(
 -- !query
 SELECT cast(1 as decimal(3, 0))  + cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(3,0)) + CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0))):decimal(4,0)>
+struct<(CAST(1 AS DECIMAL(3,0)) + CAST(1 AS TINYINT)):decimal(4,0)>
 -- !query output
 2
 
@@ -353,7 +353,7 @@ struct<(CAST(1 AS DECIMAL(3,0)) + CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0))):deci
 -- !query
 SELECT cast(1 as decimal(5, 0))  + cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(6,0)) + CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(6,0))):decimal(6,0)>
+struct<(CAST(1 AS DECIMAL(5,0)) + CAST(1 AS TINYINT)):decimal(6,0)>
 -- !query output
 2
 
@@ -361,7 +361,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(6,0)) + CAST(CAST(CAST(1 AS TINY
 -- !query
 SELECT cast(1 as decimal(10, 0)) + cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(11,0)) + CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(11,0))):decimal(11,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) + CAST(1 AS TINYINT)):decimal(11,0)>
 -- !query output
 2
 
@@ -369,7 +369,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(11,0)) + CAST(CAST(CAST(1 AS TI
 -- !query
 SELECT cast(1 as decimal(20, 0)) + cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(21,0)) + CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS DECIMAL(20,0)) + CAST(1 AS TINYINT)):decimal(21,0)>
 -- !query output
 2
 
@@ -377,7 +377,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(21,0)) + CAST(CAST(CAST(1 AS TI
 -- !query
 SELECT cast(1 as decimal(3, 0))  + cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(6,0)) + CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(6,0))):decimal(6,0)>
+struct<(CAST(1 AS DECIMAL(3,0)) + CAST(1 AS SMALLINT)):decimal(6,0)>
 -- !query output
 2
 
@@ -385,7 +385,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(6,0)) + CAST(CAST(CAST(1 AS SMAL
 -- !query
 SELECT cast(1 as decimal(5, 0))  + cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(5,0)) + CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0))):decimal(6,0)>
+struct<(CAST(1 AS DECIMAL(5,0)) + CAST(1 AS SMALLINT)):decimal(6,0)>
 -- !query output
 2
 
@@ -393,7 +393,7 @@ struct<(CAST(1 AS DECIMAL(5,0)) + CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0))):dec
 -- !query
 SELECT cast(1 as decimal(10, 0)) + cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(11,0)) + CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(11,0))):decimal(11,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) + CAST(1 AS SMALLINT)):decimal(11,0)>
 -- !query output
 2
 
@@ -401,7 +401,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(11,0)) + CAST(CAST(CAST(1 AS SM
 -- !query
 SELECT cast(1 as decimal(20, 0)) + cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(21,0)) + CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS DECIMAL(20,0)) + CAST(1 AS SMALLINT)):decimal(21,0)>
 -- !query output
 2
 
@@ -409,7 +409,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(21,0)) + CAST(CAST(CAST(1 AS SM
 -- !query
 SELECT cast(1 as decimal(3, 0))  + cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(11,0)) + CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(11,0))):decimal(11,0)>
+struct<(CAST(1 AS DECIMAL(3,0)) + CAST(1 AS INT)):decimal(11,0)>
 -- !query output
 2
 
@@ -417,7 +417,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(11,0)) + CAST(CAST(CAST(1 AS INT
 -- !query
 SELECT cast(1 as decimal(5, 0))  + cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(11,0)) + CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(11,0))):decimal(11,0)>
+struct<(CAST(1 AS DECIMAL(5,0)) + CAST(1 AS INT)):decimal(11,0)>
 -- !query output
 2
 
@@ -425,7 +425,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(11,0)) + CAST(CAST(CAST(1 AS INT
 -- !query
 SELECT cast(1 as decimal(10, 0)) + cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(10,0)) + CAST(CAST(1 AS INT) AS DECIMAL(10,0))):decimal(11,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) + CAST(1 AS INT)):decimal(11,0)>
 -- !query output
 2
 
@@ -433,7 +433,7 @@ struct<(CAST(1 AS DECIMAL(10,0)) + CAST(CAST(1 AS INT) AS DECIMAL(10,0))):decima
 -- !query
 SELECT cast(1 as decimal(20, 0)) + cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(21,0)) + CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS DECIMAL(20,0)) + CAST(1 AS INT)):decimal(21,0)>
 -- !query output
 2
 
@@ -441,7 +441,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(21,0)) + CAST(CAST(CAST(1 AS IN
 -- !query
 SELECT cast(1 as decimal(3, 0))  + cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(21,0)) + CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS DECIMAL(3,0)) + CAST(1 AS BIGINT)):decimal(21,0)>
 -- !query output
 2
 
@@ -449,7 +449,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(21,0)) + CAST(CAST(CAST(1 AS BIG
 -- !query
 SELECT cast(1 as decimal(5, 0))  + cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(21,0)) + CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS DECIMAL(5,0)) + CAST(1 AS BIGINT)):decimal(21,0)>
 -- !query output
 2
 
@@ -457,7 +457,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(21,0)) + CAST(CAST(CAST(1 AS BIG
 -- !query
 SELECT cast(1 as decimal(10, 0)) + cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(21,0)) + CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) + CAST(1 AS BIGINT)):decimal(21,0)>
 -- !query output
 2
 
@@ -465,7 +465,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(21,0)) + CAST(CAST(CAST(1 AS BI
 -- !query
 SELECT cast(1 as decimal(20, 0)) + cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(20,0)) + CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0))):decimal(21,0)>
+struct<(CAST(1 AS DECIMAL(20,0)) + CAST(1 AS BIGINT)):decimal(21,0)>
 -- !query output
 2
 
@@ -473,7 +473,7 @@ struct<(CAST(1 AS DECIMAL(20,0)) + CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0))):dec
 -- !query
 SELECT cast(1 as decimal(3, 0))  + cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) + CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(3,0)) + CAST(1 AS FLOAT)):double>
 -- !query output
 2.0
 
@@ -481,7 +481,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) + CAST(CAST(1 AS FLOAT) AS DOUBL
 -- !query
 SELECT cast(1 as decimal(5, 0))  + cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) + CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(5,0)) + CAST(1 AS FLOAT)):double>
 -- !query output
 2.0
 
@@ -489,7 +489,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) + CAST(CAST(1 AS FLOAT) AS DOUBL
 -- !query
 SELECT cast(1 as decimal(10, 0)) + cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) + CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(10,0)) + CAST(1 AS FLOAT)):double>
 -- !query output
 2.0
 
@@ -497,7 +497,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) + CAST(CAST(1 AS FLOAT) AS DOUB
 -- !query
 SELECT cast(1 as decimal(20, 0)) + cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) + CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(20,0)) + CAST(1 AS FLOAT)):double>
 -- !query output
 2.0
 
@@ -505,7 +505,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) + CAST(CAST(1 AS FLOAT) AS DOUB
 -- !query
 SELECT cast(1 as decimal(3, 0))  + cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) + CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(3,0)) + CAST(1 AS DOUBLE)):double>
 -- !query output
 2.0
 
@@ -513,7 +513,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) + CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(5, 0))  + cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) + CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(5,0)) + CAST(1 AS DOUBLE)):double>
 -- !query output
 2.0
 
@@ -521,7 +521,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) + CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(10, 0)) + cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) + CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(10,0)) + CAST(1 AS DOUBLE)):double>
 -- !query output
 2.0
 
@@ -529,7 +529,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) + CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(20, 0)) + cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) + CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(20,0)) + CAST(1 AS DOUBLE)):double>
 -- !query output
 2.0
 
@@ -537,7 +537,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) + CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(3, 0))  + cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(11,0)) + CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(11,0))):decimal(11,0)>
+struct<(CAST(1 AS DECIMAL(3,0)) + CAST(1 AS DECIMAL(10,0))):decimal(11,0)>
 -- !query output
 2
 
@@ -545,7 +545,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(11,0)) + CAST(CAST(1 AS DECIMAL(
 -- !query
 SELECT cast(1 as decimal(5, 0))  + cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(11,0)) + CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(11,0))):decimal(11,0)>
+struct<(CAST(1 AS DECIMAL(5,0)) + CAST(1 AS DECIMAL(10,0))):decimal(11,0)>
 -- !query output
 2
 
@@ -561,7 +561,7 @@ struct<(CAST(1 AS DECIMAL(10,0)) + CAST(1 AS DECIMAL(10,0))):decimal(11,0)>
 -- !query
 SELECT cast(1 as decimal(20, 0)) + cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(21,0)) + CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS DECIMAL(20,0)) + CAST(1 AS DECIMAL(10,0))):decimal(21,0)>
 -- !query output
 2
 
@@ -569,7 +569,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(21,0)) + CAST(CAST(1 AS DECIMAL
 -- !query
 SELECT cast(1 as decimal(3, 0))  + cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) + CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(3,0)) + CAST(1 AS STRING)):double>
 -- !query output
 2.0
 
@@ -577,7 +577,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) + CAST(CAST(1 AS STRING) AS DOUB
 -- !query
 SELECT cast(1 as decimal(5, 0))  + cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) + CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(5,0)) + CAST(1 AS STRING)):double>
 -- !query output
 2.0
 
@@ -585,7 +585,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) + CAST(CAST(1 AS STRING) AS DOUB
 -- !query
 SELECT cast(1 as decimal(10, 0)) + cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) + CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(10,0)) + CAST(1 AS STRING)):double>
 -- !query output
 2.0
 
@@ -593,7 +593,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) + CAST(CAST(1 AS STRING) AS DOU
 -- !query
 SELECT cast(1 as decimal(20, 0)) + cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) + CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(20,0)) + CAST(1 AS STRING)):double>
 -- !query output
 2.0
 
@@ -745,7 +745,7 @@ cannot resolve 'date_add(CAST('2017-12-11 09:30:00' AS DATE), CAST(1 AS DECIMAL(
 -- !query
 SELECT cast(1 as tinyint) - cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) - CAST(1 AS DECIMAL(3,0))):decimal(4,0)>
+struct<(CAST(1 AS TINYINT) - CAST(1 AS DECIMAL(3,0))):decimal(4,0)>
 -- !query output
 0
 
@@ -753,7 +753,7 @@ struct<(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) - CAST(1 AS DECIMAL(3,0))):deci
 -- !query
 SELECT cast(1 as tinyint) - cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(6,0)) - CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(6,0))):decimal(6,0)>
+struct<(CAST(1 AS TINYINT) - CAST(1 AS DECIMAL(5,0))):decimal(6,0)>
 -- !query output
 0
 
@@ -761,7 +761,7 @@ struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(6,0)) - CAST(CA
 -- !query
 SELECT cast(1 as tinyint) - cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(11,0)) - CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(11,0))):decimal(11,0)>
+struct<(CAST(1 AS TINYINT) - CAST(1 AS DECIMAL(10,0))):decimal(11,0)>
 -- !query output
 0
 
@@ -769,7 +769,7 @@ struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(11,0)) - CAST(C
 -- !query
 SELECT cast(1 as tinyint) - cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(21,0)) - CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS TINYINT) - CAST(1 AS DECIMAL(20,0))):decimal(21,0)>
 -- !query output
 0
 
@@ -777,7 +777,7 @@ struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(21,0)) - CAST(C
 -- !query
 SELECT cast(1 as smallint) - cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(6,0)) - CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(6,0))):decimal(6,0)>
+struct<(CAST(1 AS SMALLINT) - CAST(1 AS DECIMAL(3,0))):decimal(6,0)>
 -- !query output
 0
 
@@ -785,7 +785,7 @@ struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(6,0)) - CAST(C
 -- !query
 SELECT cast(1 as smallint) - cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) - CAST(1 AS DECIMAL(5,0))):decimal(6,0)>
+struct<(CAST(1 AS SMALLINT) - CAST(1 AS DECIMAL(5,0))):decimal(6,0)>
 -- !query output
 0
 
@@ -793,7 +793,7 @@ struct<(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) - CAST(1 AS DECIMAL(5,0))):dec
 -- !query
 SELECT cast(1 as smallint) - cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(11,0)) - CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(11,0))):decimal(11,0)>
+struct<(CAST(1 AS SMALLINT) - CAST(1 AS DECIMAL(10,0))):decimal(11,0)>
 -- !query output
 0
 
@@ -801,7 +801,7 @@ struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(11,0)) - CAST(
 -- !query
 SELECT cast(1 as smallint) - cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(21,0)) - CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS SMALLINT) - CAST(1 AS DECIMAL(20,0))):decimal(21,0)>
 -- !query output
 0
 
@@ -809,7 +809,7 @@ struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(21,0)) - CAST(
 -- !query
 SELECT cast(1 as int) - cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(11,0)) - CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(11,0))):decimal(11,0)>
+struct<(CAST(1 AS INT) - CAST(1 AS DECIMAL(3,0))):decimal(11,0)>
 -- !query output
 0
 
@@ -817,7 +817,7 @@ struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(11,0)) - CAST(CAST
 -- !query
 SELECT cast(1 as int) - cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(11,0)) - CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(11,0))):decimal(11,0)>
+struct<(CAST(1 AS INT) - CAST(1 AS DECIMAL(5,0))):decimal(11,0)>
 -- !query output
 0
 
@@ -825,7 +825,7 @@ struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(11,0)) - CAST(CAST
 -- !query
 SELECT cast(1 as int) - cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) - CAST(1 AS DECIMAL(10,0))):decimal(11,0)>
+struct<(CAST(1 AS INT) - CAST(1 AS DECIMAL(10,0))):decimal(11,0)>
 -- !query output
 0
 
@@ -833,7 +833,7 @@ struct<(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) - CAST(1 AS DECIMAL(10,0))):decima
 -- !query
 SELECT cast(1 as int) - cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(21,0)) - CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS INT) - CAST(1 AS DECIMAL(20,0))):decimal(21,0)>
 -- !query output
 0
 
@@ -841,7 +841,7 @@ struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(21,0)) - CAST(CAST
 -- !query
 SELECT cast(1 as bigint) - cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(21,0)) - CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS BIGINT) - CAST(1 AS DECIMAL(3,0))):decimal(21,0)>
 -- !query output
 0
 
@@ -849,7 +849,7 @@ struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(21,0)) - CAST(C
 -- !query
 SELECT cast(1 as bigint) - cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(21,0)) - CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS BIGINT) - CAST(1 AS DECIMAL(5,0))):decimal(21,0)>
 -- !query output
 0
 
@@ -857,7 +857,7 @@ struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(21,0)) - CAST(C
 -- !query
 SELECT cast(1 as bigint) - cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(21,0)) - CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS BIGINT) - CAST(1 AS DECIMAL(10,0))):decimal(21,0)>
 -- !query output
 0
 
@@ -865,7 +865,7 @@ struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(21,0)) - CAST(C
 -- !query
 SELECT cast(1 as bigint) - cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) - CAST(1 AS DECIMAL(20,0))):decimal(21,0)>
+struct<(CAST(1 AS BIGINT) - CAST(1 AS DECIMAL(20,0))):decimal(21,0)>
 -- !query output
 0
 
@@ -873,7 +873,7 @@ struct<(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) - CAST(1 AS DECIMAL(20,0))):dec
 -- !query
 SELECT cast(1 as float) - cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) - CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) - CAST(1 AS DECIMAL(3,0))):double>
 -- !query output
 0.0
 
@@ -881,7 +881,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) - CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBL
 -- !query
 SELECT cast(1 as float) - cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) - CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) - CAST(1 AS DECIMAL(5,0))):double>
 -- !query output
 0.0
 
@@ -889,7 +889,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) - CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBL
 -- !query
 SELECT cast(1 as float) - cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) - CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) - CAST(1 AS DECIMAL(10,0))):double>
 -- !query output
 0.0
 
@@ -897,7 +897,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) - CAST(CAST(1 AS DECIMAL(10,0)) AS DOUB
 -- !query
 SELECT cast(1 as float) - cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) - CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) - CAST(1 AS DECIMAL(20,0))):double>
 -- !query output
 0.0
 
@@ -905,7 +905,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) - CAST(CAST(1 AS DECIMAL(20,0)) AS DOUB
 -- !query
 SELECT cast(1 as double) - cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) - CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) - CAST(1 AS DECIMAL(3,0))):double>
 -- !query output
 0.0
 
@@ -913,7 +913,7 @@ struct<(CAST(1 AS DOUBLE) - CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as double) - cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) - CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) - CAST(1 AS DECIMAL(5,0))):double>
 -- !query output
 0.0
 
@@ -921,7 +921,7 @@ struct<(CAST(1 AS DOUBLE) - CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as double) - cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) - CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) - CAST(1 AS DECIMAL(10,0))):double>
 -- !query output
 0.0
 
@@ -929,7 +929,7 @@ struct<(CAST(1 AS DOUBLE) - CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as double) - cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) - CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) - CAST(1 AS DECIMAL(20,0))):double>
 -- !query output
 0.0
 
@@ -937,7 +937,7 @@ struct<(CAST(1 AS DOUBLE) - CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(10, 0)) - cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(11,0)) - CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(11,0))):decimal(11,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) - CAST(1 AS DECIMAL(3,0))):decimal(11,0)>
 -- !query output
 0
 
@@ -945,7 +945,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(11,0)) - CAST(CAST(1 AS DECIMAL
 -- !query
 SELECT cast(1 as decimal(10, 0)) - cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(11,0)) - CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(11,0))):decimal(11,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) - CAST(1 AS DECIMAL(5,0))):decimal(11,0)>
 -- !query output
 0
 
@@ -961,7 +961,7 @@ struct<(CAST(1 AS DECIMAL(10,0)) - CAST(1 AS DECIMAL(10,0))):decimal(11,0)>
 -- !query
 SELECT cast(1 as decimal(10, 0)) - cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(21,0)) - CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) - CAST(1 AS DECIMAL(20,0))):decimal(21,0)>
 -- !query output
 0
 
@@ -1008,7 +1008,7 @@ SELECT cast('2017-12-11 09:30:00.0' as timestamp) - cast(1 as decimal(3, 0)) FRO
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'subtracttimestamps(CAST('2017-12-11 09:30:00.0' AS TIMESTAMP), CAST(1 AS DECIMAL(3,0)))' due to data type mismatch: argument 2 requires timestamp type, however, 'CAST(1 AS DECIMAL(3,0))' is of decimal(3,0) type.; line 1 pos 7
+cannot resolve '(CAST('2017-12-11 09:30:00.0' AS TIMESTAMP) - CAST(1 AS DECIMAL(3,0)))' due to data type mismatch: argument 2 requires timestamp type, however, 'CAST(1 AS DECIMAL(3,0))' is of decimal(3,0) type.; line 1 pos 7
 
 
 -- !query
@@ -1017,7 +1017,7 @@ SELECT cast('2017-12-11 09:30:00.0' as timestamp) - cast(1 as decimal(5, 0)) FRO
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'subtracttimestamps(CAST('2017-12-11 09:30:00.0' AS TIMESTAMP), CAST(1 AS DECIMAL(5,0)))' due to data type mismatch: argument 2 requires timestamp type, however, 'CAST(1 AS DECIMAL(5,0))' is of decimal(5,0) type.; line 1 pos 7
+cannot resolve '(CAST('2017-12-11 09:30:00.0' AS TIMESTAMP) - CAST(1 AS DECIMAL(5,0)))' due to data type mismatch: argument 2 requires timestamp type, however, 'CAST(1 AS DECIMAL(5,0))' is of decimal(5,0) type.; line 1 pos 7
 
 
 -- !query
@@ -1026,7 +1026,7 @@ SELECT cast('2017-12-11 09:30:00.0' as timestamp) - cast(1 as decimal(10, 0)) FR
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'subtracttimestamps(CAST('2017-12-11 09:30:00.0' AS TIMESTAMP), CAST(1 AS DECIMAL(10,0)))' due to data type mismatch: argument 2 requires timestamp type, however, 'CAST(1 AS DECIMAL(10,0))' is of decimal(10,0) type.; line 1 pos 7
+cannot resolve '(CAST('2017-12-11 09:30:00.0' AS TIMESTAMP) - CAST(1 AS DECIMAL(10,0)))' due to data type mismatch: argument 2 requires timestamp type, however, 'CAST(1 AS DECIMAL(10,0))' is of decimal(10,0) type.; line 1 pos 7
 
 
 -- !query
@@ -1035,7 +1035,7 @@ SELECT cast('2017-12-11 09:30:00.0' as timestamp) - cast(1 as decimal(20, 0)) FR
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'subtracttimestamps(CAST('2017-12-11 09:30:00.0' AS TIMESTAMP), CAST(1 AS DECIMAL(20,0)))' due to data type mismatch: argument 2 requires timestamp type, however, 'CAST(1 AS DECIMAL(20,0))' is of decimal(20,0) type.; line 1 pos 7
+cannot resolve '(CAST('2017-12-11 09:30:00.0' AS TIMESTAMP) - CAST(1 AS DECIMAL(20,0)))' due to data type mismatch: argument 2 requires timestamp type, however, 'CAST(1 AS DECIMAL(20,0))' is of decimal(20,0) type.; line 1 pos 7
 
 
 -- !query
@@ -1077,7 +1077,7 @@ cannot resolve 'date_sub(CAST('2017-12-11 09:30:00' AS DATE), CAST(1 AS DECIMAL(
 -- !query
 SELECT cast(1 as decimal(3, 0))  - cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(3,0)) - CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0))):decimal(4,0)>
+struct<(CAST(1 AS DECIMAL(3,0)) - CAST(1 AS TINYINT)):decimal(4,0)>
 -- !query output
 0
 
@@ -1085,7 +1085,7 @@ struct<(CAST(1 AS DECIMAL(3,0)) - CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0))):deci
 -- !query
 SELECT cast(1 as decimal(5, 0))  - cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(6,0)) - CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(6,0))):decimal(6,0)>
+struct<(CAST(1 AS DECIMAL(5,0)) - CAST(1 AS TINYINT)):decimal(6,0)>
 -- !query output
 0
 
@@ -1093,7 +1093,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(6,0)) - CAST(CAST(CAST(1 AS TINY
 -- !query
 SELECT cast(1 as decimal(10, 0)) - cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(11,0)) - CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(11,0))):decimal(11,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) - CAST(1 AS TINYINT)):decimal(11,0)>
 -- !query output
 0
 
@@ -1101,7 +1101,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(11,0)) - CAST(CAST(CAST(1 AS TI
 -- !query
 SELECT cast(1 as decimal(20, 0)) - cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(21,0)) - CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS DECIMAL(20,0)) - CAST(1 AS TINYINT)):decimal(21,0)>
 -- !query output
 0
 
@@ -1109,7 +1109,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(21,0)) - CAST(CAST(CAST(1 AS TI
 -- !query
 SELECT cast(1 as decimal(3, 0))  - cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(6,0)) - CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(6,0))):decimal(6,0)>
+struct<(CAST(1 AS DECIMAL(3,0)) - CAST(1 AS SMALLINT)):decimal(6,0)>
 -- !query output
 0
 
@@ -1117,7 +1117,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(6,0)) - CAST(CAST(CAST(1 AS SMAL
 -- !query
 SELECT cast(1 as decimal(5, 0))  - cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(5,0)) - CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0))):decimal(6,0)>
+struct<(CAST(1 AS DECIMAL(5,0)) - CAST(1 AS SMALLINT)):decimal(6,0)>
 -- !query output
 0
 
@@ -1125,7 +1125,7 @@ struct<(CAST(1 AS DECIMAL(5,0)) - CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0))):dec
 -- !query
 SELECT cast(1 as decimal(10, 0)) - cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(11,0)) - CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(11,0))):decimal(11,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) - CAST(1 AS SMALLINT)):decimal(11,0)>
 -- !query output
 0
 
@@ -1133,7 +1133,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(11,0)) - CAST(CAST(CAST(1 AS SM
 -- !query
 SELECT cast(1 as decimal(20, 0)) - cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(21,0)) - CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS DECIMAL(20,0)) - CAST(1 AS SMALLINT)):decimal(21,0)>
 -- !query output
 0
 
@@ -1141,7 +1141,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(21,0)) - CAST(CAST(CAST(1 AS SM
 -- !query
 SELECT cast(1 as decimal(3, 0))  - cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(11,0)) - CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(11,0))):decimal(11,0)>
+struct<(CAST(1 AS DECIMAL(3,0)) - CAST(1 AS INT)):decimal(11,0)>
 -- !query output
 0
 
@@ -1149,7 +1149,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(11,0)) - CAST(CAST(CAST(1 AS INT
 -- !query
 SELECT cast(1 as decimal(5, 0))  - cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(11,0)) - CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(11,0))):decimal(11,0)>
+struct<(CAST(1 AS DECIMAL(5,0)) - CAST(1 AS INT)):decimal(11,0)>
 -- !query output
 0
 
@@ -1157,7 +1157,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(11,0)) - CAST(CAST(CAST(1 AS INT
 -- !query
 SELECT cast(1 as decimal(10, 0)) - cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(10,0)) - CAST(CAST(1 AS INT) AS DECIMAL(10,0))):decimal(11,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) - CAST(1 AS INT)):decimal(11,0)>
 -- !query output
 0
 
@@ -1165,7 +1165,7 @@ struct<(CAST(1 AS DECIMAL(10,0)) - CAST(CAST(1 AS INT) AS DECIMAL(10,0))):decima
 -- !query
 SELECT cast(1 as decimal(20, 0)) - cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(21,0)) - CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS DECIMAL(20,0)) - CAST(1 AS INT)):decimal(21,0)>
 -- !query output
 0
 
@@ -1173,7 +1173,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(21,0)) - CAST(CAST(CAST(1 AS IN
 -- !query
 SELECT cast(1 as decimal(3, 0))  - cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(21,0)) - CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS DECIMAL(3,0)) - CAST(1 AS BIGINT)):decimal(21,0)>
 -- !query output
 0
 
@@ -1181,7 +1181,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(21,0)) - CAST(CAST(CAST(1 AS BIG
 -- !query
 SELECT cast(1 as decimal(5, 0))  - cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(21,0)) - CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS DECIMAL(5,0)) - CAST(1 AS BIGINT)):decimal(21,0)>
 -- !query output
 0
 
@@ -1189,7 +1189,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(21,0)) - CAST(CAST(CAST(1 AS BIG
 -- !query
 SELECT cast(1 as decimal(10, 0)) - cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(21,0)) - CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) - CAST(1 AS BIGINT)):decimal(21,0)>
 -- !query output
 0
 
@@ -1197,7 +1197,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(21,0)) - CAST(CAST(CAST(1 AS BI
 -- !query
 SELECT cast(1 as decimal(20, 0)) - cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(20,0)) - CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0))):decimal(21,0)>
+struct<(CAST(1 AS DECIMAL(20,0)) - CAST(1 AS BIGINT)):decimal(21,0)>
 -- !query output
 0
 
@@ -1205,7 +1205,7 @@ struct<(CAST(1 AS DECIMAL(20,0)) - CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0))):dec
 -- !query
 SELECT cast(1 as decimal(3, 0))  - cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) - CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(3,0)) - CAST(1 AS FLOAT)):double>
 -- !query output
 0.0
 
@@ -1213,7 +1213,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) - CAST(CAST(1 AS FLOAT) AS DOUBL
 -- !query
 SELECT cast(1 as decimal(5, 0))  - cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) - CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(5,0)) - CAST(1 AS FLOAT)):double>
 -- !query output
 0.0
 
@@ -1221,7 +1221,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) - CAST(CAST(1 AS FLOAT) AS DOUBL
 -- !query
 SELECT cast(1 as decimal(10, 0)) - cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) - CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(10,0)) - CAST(1 AS FLOAT)):double>
 -- !query output
 0.0
 
@@ -1229,7 +1229,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) - CAST(CAST(1 AS FLOAT) AS DOUB
 -- !query
 SELECT cast(1 as decimal(20, 0)) - cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) - CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(20,0)) - CAST(1 AS FLOAT)):double>
 -- !query output
 0.0
 
@@ -1237,7 +1237,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) - CAST(CAST(1 AS FLOAT) AS DOUB
 -- !query
 SELECT cast(1 as decimal(3, 0))  - cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) - CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(3,0)) - CAST(1 AS DOUBLE)):double>
 -- !query output
 0.0
 
@@ -1245,7 +1245,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) - CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(5, 0))  - cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) - CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(5,0)) - CAST(1 AS DOUBLE)):double>
 -- !query output
 0.0
 
@@ -1253,7 +1253,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) - CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(10, 0)) - cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) - CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(10,0)) - CAST(1 AS DOUBLE)):double>
 -- !query output
 0.0
 
@@ -1261,7 +1261,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) - CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(20, 0)) - cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) - CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(20,0)) - CAST(1 AS DOUBLE)):double>
 -- !query output
 0.0
 
@@ -1269,7 +1269,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) - CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(3, 0))  - cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(11,0)) - CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(11,0))):decimal(11,0)>
+struct<(CAST(1 AS DECIMAL(3,0)) - CAST(1 AS DECIMAL(10,0))):decimal(11,0)>
 -- !query output
 0
 
@@ -1277,7 +1277,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(11,0)) - CAST(CAST(1 AS DECIMAL(
 -- !query
 SELECT cast(1 as decimal(5, 0))  - cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(11,0)) - CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(11,0))):decimal(11,0)>
+struct<(CAST(1 AS DECIMAL(5,0)) - CAST(1 AS DECIMAL(10,0))):decimal(11,0)>
 -- !query output
 0
 
@@ -1293,7 +1293,7 @@ struct<(CAST(1 AS DECIMAL(10,0)) - CAST(1 AS DECIMAL(10,0))):decimal(11,0)>
 -- !query
 SELECT cast(1 as decimal(20, 0)) - cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(21,0)) - CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(21,0))):decimal(21,0)>
+struct<(CAST(1 AS DECIMAL(20,0)) - CAST(1 AS DECIMAL(10,0))):decimal(21,0)>
 -- !query output
 0
 
@@ -1301,7 +1301,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(21,0)) - CAST(CAST(1 AS DECIMAL
 -- !query
 SELECT cast(1 as decimal(3, 0))  - cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) - CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(3,0)) - CAST(1 AS STRING)):double>
 -- !query output
 0.0
 
@@ -1309,7 +1309,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) - CAST(CAST(1 AS STRING) AS DOUB
 -- !query
 SELECT cast(1 as decimal(5, 0))  - cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) - CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(5,0)) - CAST(1 AS STRING)):double>
 -- !query output
 0.0
 
@@ -1317,7 +1317,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) - CAST(CAST(1 AS STRING) AS DOUB
 -- !query
 SELECT cast(1 as decimal(10, 0)) - cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) - CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(10,0)) - CAST(1 AS STRING)):double>
 -- !query output
 0.0
 
@@ -1325,7 +1325,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) - CAST(CAST(1 AS STRING) AS DOU
 -- !query
 SELECT cast(1 as decimal(20, 0)) - cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) - CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(20,0)) - CAST(1 AS STRING)):double>
 -- !query output
 0.0
 
@@ -1408,7 +1408,7 @@ SELECT cast(1 as decimal(3, 0))  - cast('2017-12-11 09:30:00.0' as timestamp) FR
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'subtracttimestamps(CAST(1 AS DECIMAL(3,0)), CAST('2017-12-11 09:30:00.0' AS TIMESTAMP))' due to data type mismatch: argument 1 requires timestamp type, however, 'CAST(1 AS DECIMAL(3,0))' is of decimal(3,0) type.; line 1 pos 7
+cannot resolve '(CAST(1 AS DECIMAL(3,0)) - CAST('2017-12-11 09:30:00.0' AS TIMESTAMP))' due to data type mismatch: argument 1 requires timestamp type, however, 'CAST(1 AS DECIMAL(3,0))' is of decimal(3,0) type.; line 1 pos 7
 
 
 -- !query
@@ -1417,7 +1417,7 @@ SELECT cast(1 as decimal(5, 0))  - cast('2017-12-11 09:30:00.0' as timestamp) FR
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'subtracttimestamps(CAST(1 AS DECIMAL(5,0)), CAST('2017-12-11 09:30:00.0' AS TIMESTAMP))' due to data type mismatch: argument 1 requires timestamp type, however, 'CAST(1 AS DECIMAL(5,0))' is of decimal(5,0) type.; line 1 pos 7
+cannot resolve '(CAST(1 AS DECIMAL(5,0)) - CAST('2017-12-11 09:30:00.0' AS TIMESTAMP))' due to data type mismatch: argument 1 requires timestamp type, however, 'CAST(1 AS DECIMAL(5,0))' is of decimal(5,0) type.; line 1 pos 7
 
 
 -- !query
@@ -1426,7 +1426,7 @@ SELECT cast(1 as decimal(10, 0)) - cast('2017-12-11 09:30:00.0' as timestamp) FR
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'subtracttimestamps(CAST(1 AS DECIMAL(10,0)), CAST('2017-12-11 09:30:00.0' AS TIMESTAMP))' due to data type mismatch: argument 1 requires timestamp type, however, 'CAST(1 AS DECIMAL(10,0))' is of decimal(10,0) type.; line 1 pos 7
+cannot resolve '(CAST(1 AS DECIMAL(10,0)) - CAST('2017-12-11 09:30:00.0' AS TIMESTAMP))' due to data type mismatch: argument 1 requires timestamp type, however, 'CAST(1 AS DECIMAL(10,0))' is of decimal(10,0) type.; line 1 pos 7
 
 
 -- !query
@@ -1435,7 +1435,7 @@ SELECT cast(1 as decimal(20, 0)) - cast('2017-12-11 09:30:00.0' as timestamp) FR
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'subtracttimestamps(CAST(1 AS DECIMAL(20,0)), CAST('2017-12-11 09:30:00.0' AS TIMESTAMP))' due to data type mismatch: argument 1 requires timestamp type, however, 'CAST(1 AS DECIMAL(20,0))' is of decimal(20,0) type.; line 1 pos 7
+cannot resolve '(CAST(1 AS DECIMAL(20,0)) - CAST('2017-12-11 09:30:00.0' AS TIMESTAMP))' due to data type mismatch: argument 1 requires timestamp type, however, 'CAST(1 AS DECIMAL(20,0))' is of decimal(20,0) type.; line 1 pos 7
 
 
 -- !query
@@ -1444,7 +1444,7 @@ SELECT cast(1 as decimal(3, 0))  - cast('2017-12-11 09:30:00' as date) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'subtractdates(CAST(1 AS DECIMAL(3,0)), CAST('2017-12-11 09:30:00' AS DATE))' due to data type mismatch: argument 1 requires date type, however, 'CAST(1 AS DECIMAL(3,0))' is of decimal(3,0) type.; line 1 pos 7
+cannot resolve '(CAST(1 AS DECIMAL(3,0)) - CAST('2017-12-11 09:30:00' AS DATE))' due to data type mismatch: argument 1 requires date type, however, 'CAST(1 AS DECIMAL(3,0))' is of decimal(3,0) type.; line 1 pos 7
 
 
 -- !query
@@ -1453,7 +1453,7 @@ SELECT cast(1 as decimal(5, 0))  - cast('2017-12-11 09:30:00' as date) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'subtractdates(CAST(1 AS DECIMAL(5,0)), CAST('2017-12-11 09:30:00' AS DATE))' due to data type mismatch: argument 1 requires date type, however, 'CAST(1 AS DECIMAL(5,0))' is of decimal(5,0) type.; line 1 pos 7
+cannot resolve '(CAST(1 AS DECIMAL(5,0)) - CAST('2017-12-11 09:30:00' AS DATE))' due to data type mismatch: argument 1 requires date type, however, 'CAST(1 AS DECIMAL(5,0))' is of decimal(5,0) type.; line 1 pos 7
 
 
 -- !query
@@ -1462,7 +1462,7 @@ SELECT cast(1 as decimal(10, 0)) - cast('2017-12-11 09:30:00' as date) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'subtractdates(CAST(1 AS DECIMAL(10,0)), CAST('2017-12-11 09:30:00' AS DATE))' due to data type mismatch: argument 1 requires date type, however, 'CAST(1 AS DECIMAL(10,0))' is of decimal(10,0) type.; line 1 pos 7
+cannot resolve '(CAST(1 AS DECIMAL(10,0)) - CAST('2017-12-11 09:30:00' AS DATE))' due to data type mismatch: argument 1 requires date type, however, 'CAST(1 AS DECIMAL(10,0))' is of decimal(10,0) type.; line 1 pos 7
 
 
 -- !query
@@ -1471,13 +1471,13 @@ SELECT cast(1 as decimal(20, 0)) - cast('2017-12-11 09:30:00' as date) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'subtractdates(CAST(1 AS DECIMAL(20,0)), CAST('2017-12-11 09:30:00' AS DATE))' due to data type mismatch: argument 1 requires date type, however, 'CAST(1 AS DECIMAL(20,0))' is of decimal(20,0) type.; line 1 pos 7
+cannot resolve '(CAST(1 AS DECIMAL(20,0)) - CAST('2017-12-11 09:30:00' AS DATE))' due to data type mismatch: argument 1 requires date type, however, 'CAST(1 AS DECIMAL(20,0))' is of decimal(20,0) type.; line 1 pos 7
 
 
 -- !query
 SELECT cast(1 as tinyint) * cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) * CAST(1 AS DECIMAL(3,0))):decimal(7,0)>
+struct<(CAST(1 AS TINYINT) * CAST(1 AS DECIMAL(3,0))):decimal(7,0)>
 -- !query output
 1
 
@@ -1485,7 +1485,7 @@ struct<(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) * CAST(1 AS DECIMAL(3,0))):deci
 -- !query
 SELECT cast(1 as tinyint) * cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0)) * CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0))):decimal(9,0)>
+struct<(CAST(1 AS TINYINT) * CAST(1 AS DECIMAL(5,0))):decimal(9,0)>
 -- !query output
 1
 
@@ -1493,7 +1493,7 @@ struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0)) * CAST(CA
 -- !query
 SELECT cast(1 as tinyint) * cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0)) * CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(14,0)>
+struct<(CAST(1 AS TINYINT) * CAST(1 AS DECIMAL(10,0))):decimal(14,0)>
 -- !query output
 1
 
@@ -1501,7 +1501,7 @@ struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0)) * CAST(C
 -- !query
 SELECT cast(1 as tinyint) * cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0)) * CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(24,0)>
+struct<(CAST(1 AS TINYINT) * CAST(1 AS DECIMAL(20,0))):decimal(24,0)>
 -- !query output
 1
 
@@ -1509,7 +1509,7 @@ struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0)) * CAST(C
 -- !query
 SELECT cast(1 as smallint) * cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0)) * CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0))):decimal(9,0)>
+struct<(CAST(1 AS SMALLINT) * CAST(1 AS DECIMAL(3,0))):decimal(9,0)>
 -- !query output
 1
 
@@ -1517,7 +1517,7 @@ struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0)) * CAST(C
 -- !query
 SELECT cast(1 as smallint) * cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) * CAST(1 AS DECIMAL(5,0))):decimal(11,0)>
+struct<(CAST(1 AS SMALLINT) * CAST(1 AS DECIMAL(5,0))):decimal(11,0)>
 -- !query output
 1
 
@@ -1525,7 +1525,7 @@ struct<(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) * CAST(1 AS DECIMAL(5,0))):dec
 -- !query
 SELECT cast(1 as smallint) * cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0)) * CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(16,0)>
+struct<(CAST(1 AS SMALLINT) * CAST(1 AS DECIMAL(10,0))):decimal(16,0)>
 -- !query output
 1
 
@@ -1533,7 +1533,7 @@ struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0)) * CAST(
 -- !query
 SELECT cast(1 as smallint) * cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0)) * CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(26,0)>
+struct<(CAST(1 AS SMALLINT) * CAST(1 AS DECIMAL(20,0))):decimal(26,0)>
 -- !query output
 1
 
@@ -1541,7 +1541,7 @@ struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0)) * CAST(
 -- !query
 SELECT cast(1 as int) * cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) * CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0))):decimal(14,0)>
+struct<(CAST(1 AS INT) * CAST(1 AS DECIMAL(3,0))):decimal(14,0)>
 -- !query output
 1
 
@@ -1549,7 +1549,7 @@ struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) * CAST(CAST
 -- !query
 SELECT cast(1 as int) * cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) * CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0))):decimal(16,0)>
+struct<(CAST(1 AS INT) * CAST(1 AS DECIMAL(5,0))):decimal(16,0)>
 -- !query output
 1
 
@@ -1557,7 +1557,7 @@ struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) * CAST(CAST
 -- !query
 SELECT cast(1 as int) * cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) * CAST(1 AS DECIMAL(10,0))):decimal(21,0)>
+struct<(CAST(1 AS INT) * CAST(1 AS DECIMAL(10,0))):decimal(21,0)>
 -- !query output
 1
 
@@ -1565,7 +1565,7 @@ struct<(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) * CAST(1 AS DECIMAL(10,0))):decima
 -- !query
 SELECT cast(1 as int) * cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0)) * CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(31,0)>
+struct<(CAST(1 AS INT) * CAST(1 AS DECIMAL(20,0))):decimal(31,0)>
 -- !query output
 1
 
@@ -1573,7 +1573,7 @@ struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0)) * CAST(CAST
 -- !query
 SELECT cast(1 as bigint) * cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) * CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0))):decimal(24,0)>
+struct<(CAST(1 AS BIGINT) * CAST(1 AS DECIMAL(3,0))):decimal(24,0)>
 -- !query output
 1
 
@@ -1581,7 +1581,7 @@ struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) * CAST(C
 -- !query
 SELECT cast(1 as bigint) * cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) * CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0))):decimal(26,0)>
+struct<(CAST(1 AS BIGINT) * CAST(1 AS DECIMAL(5,0))):decimal(26,0)>
 -- !query output
 1
 
@@ -1589,7 +1589,7 @@ struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) * CAST(C
 -- !query
 SELECT cast(1 as bigint) * cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) * CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0))):decimal(31,0)>
+struct<(CAST(1 AS BIGINT) * CAST(1 AS DECIMAL(10,0))):decimal(31,0)>
 -- !query output
 1
 
@@ -1597,7 +1597,7 @@ struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) * CAST(C
 -- !query
 SELECT cast(1 as bigint) * cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) * CAST(1 AS DECIMAL(20,0))):decimal(38,0)>
+struct<(CAST(1 AS BIGINT) * CAST(1 AS DECIMAL(20,0))):decimal(38,0)>
 -- !query output
 1
 
@@ -1605,7 +1605,7 @@ struct<(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) * CAST(1 AS DECIMAL(20,0))):dec
 -- !query
 SELECT cast(1 as float) * cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) * CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) * CAST(1 AS DECIMAL(3,0))):double>
 -- !query output
 1.0
 
@@ -1613,7 +1613,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) * CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBL
 -- !query
 SELECT cast(1 as float) * cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) * CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) * CAST(1 AS DECIMAL(5,0))):double>
 -- !query output
 1.0
 
@@ -1621,7 +1621,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) * CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBL
 -- !query
 SELECT cast(1 as float) * cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) * CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) * CAST(1 AS DECIMAL(10,0))):double>
 -- !query output
 1.0
 
@@ -1629,7 +1629,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) * CAST(CAST(1 AS DECIMAL(10,0)) AS DOUB
 -- !query
 SELECT cast(1 as float) * cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) * CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) * CAST(1 AS DECIMAL(20,0))):double>
 -- !query output
 1.0
 
@@ -1637,7 +1637,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) * CAST(CAST(1 AS DECIMAL(20,0)) AS DOUB
 -- !query
 SELECT cast(1 as double) * cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) * CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) * CAST(1 AS DECIMAL(3,0))):double>
 -- !query output
 1.0
 
@@ -1645,7 +1645,7 @@ struct<(CAST(1 AS DOUBLE) * CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as double) * cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) * CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) * CAST(1 AS DECIMAL(5,0))):double>
 -- !query output
 1.0
 
@@ -1653,7 +1653,7 @@ struct<(CAST(1 AS DOUBLE) * CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as double) * cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) * CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) * CAST(1 AS DECIMAL(10,0))):double>
 -- !query output
 1.0
 
@@ -1661,7 +1661,7 @@ struct<(CAST(1 AS DOUBLE) * CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as double) * cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) * CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) * CAST(1 AS DECIMAL(20,0))):double>
 -- !query output
 1.0
 
@@ -1669,7 +1669,7 @@ struct<(CAST(1 AS DOUBLE) * CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(10, 0)) * cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) * CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0))):decimal(14,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) * CAST(1 AS DECIMAL(3,0))):decimal(14,0)>
 -- !query output
 1
 
@@ -1677,7 +1677,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) * CAST(CAST(1 AS DECIMAL
 -- !query
 SELECT cast(1 as decimal(10, 0)) * cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) * CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0))):decimal(16,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) * CAST(1 AS DECIMAL(5,0))):decimal(16,0)>
 -- !query output
 1
 
@@ -1693,7 +1693,7 @@ struct<(CAST(1 AS DECIMAL(10,0)) * CAST(1 AS DECIMAL(10,0))):decimal(21,0)>
 -- !query
 SELECT cast(1 as decimal(10, 0)) * cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) * CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(31,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) * CAST(1 AS DECIMAL(20,0))):decimal(31,0)>
 -- !query output
 1
 
@@ -1809,7 +1809,7 @@ cannot resolve '(CAST('2017*12*11 09:30:00' AS DATE) * CAST(1 AS DECIMAL(20,0)))
 -- !query
 SELECT cast(1 as decimal(3, 0))  * cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(3,0)) * CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0))):decimal(7,0)>
+struct<(CAST(1 AS DECIMAL(3,0)) * CAST(1 AS TINYINT)):decimal(7,0)>
 -- !query output
 1
 
@@ -1817,7 +1817,7 @@ struct<(CAST(1 AS DECIMAL(3,0)) * CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0))):deci
 -- !query
 SELECT cast(1 as decimal(5, 0))  * cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0)) * CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0))):decimal(9,0)>
+struct<(CAST(1 AS DECIMAL(5,0)) * CAST(1 AS TINYINT)):decimal(9,0)>
 -- !query output
 1
 
@@ -1825,7 +1825,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0)) * CAST(CAST(CAST(1 AS TINY
 -- !query
 SELECT cast(1 as decimal(10, 0)) * cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) * CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0))):decimal(14,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) * CAST(1 AS TINYINT)):decimal(14,0)>
 -- !query output
 1
 
@@ -1833,7 +1833,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) * CAST(CAST(CAST(1 AS TI
 -- !query
 SELECT cast(1 as decimal(20, 0)) * cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) * CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0))):decimal(24,0)>
+struct<(CAST(1 AS DECIMAL(20,0)) * CAST(1 AS TINYINT)):decimal(24,0)>
 -- !query output
 1
 
@@ -1841,7 +1841,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) * CAST(CAST(CAST(1 AS TI
 -- !query
 SELECT cast(1 as decimal(3, 0))  * cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0)) * CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0))):decimal(9,0)>
+struct<(CAST(1 AS DECIMAL(3,0)) * CAST(1 AS SMALLINT)):decimal(9,0)>
 -- !query output
 1
 
@@ -1849,7 +1849,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0)) * CAST(CAST(CAST(1 AS SMAL
 -- !query
 SELECT cast(1 as decimal(5, 0))  * cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(5,0)) * CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0))):decimal(11,0)>
+struct<(CAST(1 AS DECIMAL(5,0)) * CAST(1 AS SMALLINT)):decimal(11,0)>
 -- !query output
 1
 
@@ -1857,7 +1857,7 @@ struct<(CAST(1 AS DECIMAL(5,0)) * CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0))):dec
 -- !query
 SELECT cast(1 as decimal(10, 0)) * cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) * CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0))):decimal(16,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) * CAST(1 AS SMALLINT)):decimal(16,0)>
 -- !query output
 1
 
@@ -1865,7 +1865,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) * CAST(CAST(CAST(1 AS SM
 -- !query
 SELECT cast(1 as decimal(20, 0)) * cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) * CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0))):decimal(26,0)>
+struct<(CAST(1 AS DECIMAL(20,0)) * CAST(1 AS SMALLINT)):decimal(26,0)>
 -- !query output
 1
 
@@ -1873,7 +1873,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) * CAST(CAST(CAST(1 AS SM
 -- !query
 SELECT cast(1 as decimal(3, 0))  * cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) * CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(14,0)>
+struct<(CAST(1 AS DECIMAL(3,0)) * CAST(1 AS INT)):decimal(14,0)>
 -- !query output
 1
 
@@ -1881,7 +1881,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) * CAST(CAST(CAST(1 AS INT
 -- !query
 SELECT cast(1 as decimal(5, 0))  * cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)) * CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(16,0)>
+struct<(CAST(1 AS DECIMAL(5,0)) * CAST(1 AS INT)):decimal(16,0)>
 -- !query output
 1
 
@@ -1889,7 +1889,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)) * CAST(CAST(CAST(1 AS INT
 -- !query
 SELECT cast(1 as decimal(10, 0)) * cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(10,0)) * CAST(CAST(1 AS INT) AS DECIMAL(10,0))):decimal(21,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) * CAST(1 AS INT)):decimal(21,0)>
 -- !query output
 1
 
@@ -1897,7 +1897,7 @@ struct<(CAST(1 AS DECIMAL(10,0)) * CAST(CAST(1 AS INT) AS DECIMAL(10,0))):decima
 -- !query
 SELECT cast(1 as decimal(20, 0)) * cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) * CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0))):decimal(31,0)>
+struct<(CAST(1 AS DECIMAL(20,0)) * CAST(1 AS INT)):decimal(31,0)>
 -- !query output
 1
 
@@ -1905,7 +1905,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) * CAST(CAST(CAST(1 AS IN
 -- !query
 SELECT cast(1 as decimal(3, 0))  * cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0)) * CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(24,0)>
+struct<(CAST(1 AS DECIMAL(3,0)) * CAST(1 AS BIGINT)):decimal(24,0)>
 -- !query output
 1
 
@@ -1913,7 +1913,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0)) * CAST(CAST(CAST(1 AS BIG
 -- !query
 SELECT cast(1 as decimal(5, 0))  * cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0)) * CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(26,0)>
+struct<(CAST(1 AS DECIMAL(5,0)) * CAST(1 AS BIGINT)):decimal(26,0)>
 -- !query output
 1
 
@@ -1921,7 +1921,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0)) * CAST(CAST(CAST(1 AS BIG
 -- !query
 SELECT cast(1 as decimal(10, 0)) * cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) * CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(31,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) * CAST(1 AS BIGINT)):decimal(31,0)>
 -- !query output
 1
 
@@ -1929,7 +1929,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) * CAST(CAST(CAST(1 AS BI
 -- !query
 SELECT cast(1 as decimal(20, 0)) * cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(20,0)) * CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0))):decimal(38,0)>
+struct<(CAST(1 AS DECIMAL(20,0)) * CAST(1 AS BIGINT)):decimal(38,0)>
 -- !query output
 1
 
@@ -1937,7 +1937,7 @@ struct<(CAST(1 AS DECIMAL(20,0)) * CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0))):dec
 -- !query
 SELECT cast(1 as decimal(3, 0))  * cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) * CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(3,0)) * CAST(1 AS FLOAT)):double>
 -- !query output
 1.0
 
@@ -1945,7 +1945,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) * CAST(CAST(1 AS FLOAT) AS DOUBL
 -- !query
 SELECT cast(1 as decimal(5, 0))  * cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) * CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(5,0)) * CAST(1 AS FLOAT)):double>
 -- !query output
 1.0
 
@@ -1953,7 +1953,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) * CAST(CAST(1 AS FLOAT) AS DOUBL
 -- !query
 SELECT cast(1 as decimal(10, 0)) * cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) * CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(10,0)) * CAST(1 AS FLOAT)):double>
 -- !query output
 1.0
 
@@ -1961,7 +1961,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) * CAST(CAST(1 AS FLOAT) AS DOUB
 -- !query
 SELECT cast(1 as decimal(20, 0)) * cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) * CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(20,0)) * CAST(1 AS FLOAT)):double>
 -- !query output
 1.0
 
@@ -1969,7 +1969,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) * CAST(CAST(1 AS FLOAT) AS DOUB
 -- !query
 SELECT cast(1 as decimal(3, 0))  * cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) * CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(3,0)) * CAST(1 AS DOUBLE)):double>
 -- !query output
 1.0
 
@@ -1977,7 +1977,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) * CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(5, 0))  * cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) * CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(5,0)) * CAST(1 AS DOUBLE)):double>
 -- !query output
 1.0
 
@@ -1985,7 +1985,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) * CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(10, 0)) * cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) * CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(10,0)) * CAST(1 AS DOUBLE)):double>
 -- !query output
 1.0
 
@@ -1993,7 +1993,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) * CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(20, 0)) * cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) * CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(20,0)) * CAST(1 AS DOUBLE)):double>
 -- !query output
 1.0
 
@@ -2001,7 +2001,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) * CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(3, 0))  * cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) * CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(14,0)>
+struct<(CAST(1 AS DECIMAL(3,0)) * CAST(1 AS DECIMAL(10,0))):decimal(14,0)>
 -- !query output
 1
 
@@ -2009,7 +2009,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) * CAST(CAST(1 AS DECIMAL(
 -- !query
 SELECT cast(1 as decimal(5, 0))  * cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)) * CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(16,0)>
+struct<(CAST(1 AS DECIMAL(5,0)) * CAST(1 AS DECIMAL(10,0))):decimal(16,0)>
 -- !query output
 1
 
@@ -2025,7 +2025,7 @@ struct<(CAST(1 AS DECIMAL(10,0)) * CAST(1 AS DECIMAL(10,0))):decimal(21,0)>
 -- !query
 SELECT cast(1 as decimal(20, 0)) * cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) * CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0))):decimal(31,0)>
+struct<(CAST(1 AS DECIMAL(20,0)) * CAST(1 AS DECIMAL(10,0))):decimal(31,0)>
 -- !query output
 1
 
@@ -2033,7 +2033,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) * CAST(CAST(1 AS DECIMAL
 -- !query
 SELECT cast(1 as decimal(3, 0))  * cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) * CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(3,0)) * CAST(1 AS STRING)):double>
 -- !query output
 1.0
 
@@ -2041,7 +2041,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) * CAST(CAST(1 AS STRING) AS DOUB
 -- !query
 SELECT cast(1 as decimal(5, 0))  * cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) * CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(5,0)) * CAST(1 AS STRING)):double>
 -- !query output
 1.0
 
@@ -2049,7 +2049,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) * CAST(CAST(1 AS STRING) AS DOUB
 -- !query
 SELECT cast(1 as decimal(10, 0)) * cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) * CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(10,0)) * CAST(1 AS STRING)):double>
 -- !query output
 1.0
 
@@ -2057,7 +2057,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) * CAST(CAST(1 AS STRING) AS DOU
 -- !query
 SELECT cast(1 as decimal(20, 0)) * cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) * CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(20,0)) * CAST(1 AS STRING)):double>
 -- !query output
 1.0
 
@@ -2209,7 +2209,7 @@ cannot resolve '(CAST(1 AS DECIMAL(20,0)) * CAST('2017*12*11 09:30:00' AS DATE))
 -- !query
 SELECT cast(1 as tinyint) / cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) / CAST(1 AS DECIMAL(3,0))):decimal(9,6)>
+struct<(CAST(1 AS TINYINT) / CAST(1 AS DECIMAL(3,0))):decimal(9,6)>
 -- !query output
 1.000000
 
@@ -2217,7 +2217,7 @@ struct<(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) / CAST(1 AS DECIMAL(3,0))):deci
 -- !query
 SELECT cast(1 as tinyint) / cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0)) / CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0))):decimal(9,6)>
+struct<(CAST(1 AS TINYINT) / CAST(1 AS DECIMAL(5,0))):decimal(9,6)>
 -- !query output
 1.000000
 
@@ -2225,7 +2225,7 @@ struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0)) / CAST(CA
 -- !query
 SELECT cast(1 as tinyint) / cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0)) / CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(14,11)>
+struct<(CAST(1 AS TINYINT) / CAST(1 AS DECIMAL(10,0))):decimal(14,11)>
 -- !query output
 1.00000000000
 
@@ -2233,7 +2233,7 @@ struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0)) / CAST(C
 -- !query
 SELECT cast(1 as tinyint) / cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0)) / CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(24,21)>
+struct<(CAST(1 AS TINYINT) / CAST(1 AS DECIMAL(20,0))):decimal(24,21)>
 -- !query output
 1.000000000000000000000
 
@@ -2241,7 +2241,7 @@ struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0)) / CAST(C
 -- !query
 SELECT cast(1 as smallint) / cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0)) / CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0))):decimal(11,6)>
+struct<(CAST(1 AS SMALLINT) / CAST(1 AS DECIMAL(3,0))):decimal(11,6)>
 -- !query output
 1.000000
 
@@ -2249,7 +2249,7 @@ struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0)) / CAST(C
 -- !query
 SELECT cast(1 as smallint) / cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) / CAST(1 AS DECIMAL(5,0))):decimal(11,6)>
+struct<(CAST(1 AS SMALLINT) / CAST(1 AS DECIMAL(5,0))):decimal(11,6)>
 -- !query output
 1.000000
 
@@ -2257,7 +2257,7 @@ struct<(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) / CAST(1 AS DECIMAL(5,0))):dec
 -- !query
 SELECT cast(1 as smallint) / cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0)) / CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(16,11)>
+struct<(CAST(1 AS SMALLINT) / CAST(1 AS DECIMAL(10,0))):decimal(16,11)>
 -- !query output
 1.00000000000
 
@@ -2265,7 +2265,7 @@ struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0)) / CAST(
 -- !query
 SELECT cast(1 as smallint) / cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0)) / CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(26,21)>
+struct<(CAST(1 AS SMALLINT) / CAST(1 AS DECIMAL(20,0))):decimal(26,21)>
 -- !query output
 1.000000000000000000000
 
@@ -2273,7 +2273,7 @@ struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0)) / CAST(
 -- !query
 SELECT cast(1 as int) / cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) / CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0))):decimal(16,6)>
+struct<(CAST(1 AS INT) / CAST(1 AS DECIMAL(3,0))):decimal(16,6)>
 -- !query output
 1.000000
 
@@ -2281,7 +2281,7 @@ struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) / CAST(CAST
 -- !query
 SELECT cast(1 as int) / cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) / CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0))):decimal(16,6)>
+struct<(CAST(1 AS INT) / CAST(1 AS DECIMAL(5,0))):decimal(16,6)>
 -- !query output
 1.000000
 
@@ -2289,7 +2289,7 @@ struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) / CAST(CAST
 -- !query
 SELECT cast(1 as int) / cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) / CAST(1 AS DECIMAL(10,0))):decimal(21,11)>
+struct<(CAST(1 AS INT) / CAST(1 AS DECIMAL(10,0))):decimal(21,11)>
 -- !query output
 1.00000000000
 
@@ -2297,7 +2297,7 @@ struct<(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) / CAST(1 AS DECIMAL(10,0))):decima
 -- !query
 SELECT cast(1 as int) / cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0)) / CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(31,21)>
+struct<(CAST(1 AS INT) / CAST(1 AS DECIMAL(20,0))):decimal(31,21)>
 -- !query output
 1.000000000000000000000
 
@@ -2305,7 +2305,7 @@ struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0)) / CAST(CAST
 -- !query
 SELECT cast(1 as bigint) / cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) / CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0))):decimal(26,6)>
+struct<(CAST(1 AS BIGINT) / CAST(1 AS DECIMAL(3,0))):decimal(26,6)>
 -- !query output
 1.000000
 
@@ -2313,7 +2313,7 @@ struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) / CAST(C
 -- !query
 SELECT cast(1 as bigint) / cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) / CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0))):decimal(26,6)>
+struct<(CAST(1 AS BIGINT) / CAST(1 AS DECIMAL(5,0))):decimal(26,6)>
 -- !query output
 1.000000
 
@@ -2321,7 +2321,7 @@ struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) / CAST(C
 -- !query
 SELECT cast(1 as bigint) / cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) / CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0))):decimal(31,11)>
+struct<(CAST(1 AS BIGINT) / CAST(1 AS DECIMAL(10,0))):decimal(31,11)>
 -- !query output
 1.00000000000
 
@@ -2329,7 +2329,7 @@ struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) / CAST(C
 -- !query
 SELECT cast(1 as bigint) / cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) / CAST(1 AS DECIMAL(20,0))):decimal(38,18)>
+struct<(CAST(1 AS BIGINT) / CAST(1 AS DECIMAL(20,0))):decimal(38,18)>
 -- !query output
 1.000000000000000000
 
@@ -2337,7 +2337,7 @@ struct<(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) / CAST(1 AS DECIMAL(20,0))):dec
 -- !query
 SELECT cast(1 as float) / cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) / CAST(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) / CAST(1 AS DECIMAL(3,0))):double>
 -- !query output
 1.0
 
@@ -2345,7 +2345,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) / CAST(CAST(CAST(1 AS DECIMAL(3,0)) AS
 -- !query
 SELECT cast(1 as float) / cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) / CAST(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) / CAST(1 AS DECIMAL(5,0))):double>
 -- !query output
 1.0
 
@@ -2353,7 +2353,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) / CAST(CAST(CAST(1 AS DECIMAL(5,0)) AS
 -- !query
 SELECT cast(1 as float) / cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) / CAST(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) / CAST(1 AS DECIMAL(10,0))):double>
 -- !query output
 1.0
 
@@ -2361,7 +2361,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) / CAST(CAST(CAST(1 AS DECIMAL(10,0)) AS
 -- !query
 SELECT cast(1 as float) / cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) / CAST(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) / CAST(1 AS DECIMAL(20,0))):double>
 -- !query output
 1.0
 
@@ -2369,7 +2369,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) / CAST(CAST(CAST(1 AS DECIMAL(20,0)) AS
 -- !query
 SELECT cast(1 as double) / cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) / CAST(1 AS DECIMAL(3,0))):double>
 -- !query output
 1.0
 
@@ -2377,7 +2377,7 @@ struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as double) / cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) / CAST(1 AS DECIMAL(5,0))):double>
 -- !query output
 1.0
 
@@ -2385,7 +2385,7 @@ struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as double) / cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) / CAST(1 AS DECIMAL(10,0))):double>
 -- !query output
 1.0
 
@@ -2393,7 +2393,7 @@ struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as double) / cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) / CAST(1 AS DECIMAL(20,0))):double>
 -- !query output
 1.0
 
@@ -2401,7 +2401,7 @@ struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(10, 0)) / cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) / CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0))):decimal(16,6)>
+struct<(CAST(1 AS DECIMAL(10,0)) / CAST(1 AS DECIMAL(3,0))):decimal(16,6)>
 -- !query output
 1.000000
 
@@ -2409,7 +2409,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) / CAST(CAST(1 AS DECIMAL
 -- !query
 SELECT cast(1 as decimal(10, 0)) / cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) / CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0))):decimal(16,6)>
+struct<(CAST(1 AS DECIMAL(10,0)) / CAST(1 AS DECIMAL(5,0))):decimal(16,6)>
 -- !query output
 1.000000
 
@@ -2425,7 +2425,7 @@ struct<(CAST(1 AS DECIMAL(10,0)) / CAST(1 AS DECIMAL(10,0))):decimal(21,11)>
 -- !query
 SELECT cast(1 as decimal(10, 0)) / cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) / CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(31,21)>
+struct<(CAST(1 AS DECIMAL(10,0)) / CAST(1 AS DECIMAL(20,0))):decimal(31,21)>
 -- !query output
 1.000000000000000000000
 
@@ -2541,7 +2541,7 @@ cannot resolve '(CAST('2017/12/11 09:30:00' AS DATE) / CAST(1 AS DECIMAL(20,0)))
 -- !query
 SELECT cast(1 as decimal(3, 0))  / cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(3,0)) / CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0))):decimal(9,6)>
+struct<(CAST(1 AS DECIMAL(3,0)) / CAST(1 AS TINYINT)):decimal(9,6)>
 -- !query output
 1.000000
 
@@ -2549,7 +2549,7 @@ struct<(CAST(1 AS DECIMAL(3,0)) / CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0))):deci
 -- !query
 SELECT cast(1 as decimal(5, 0))  / cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0)) / CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0))):decimal(11,6)>
+struct<(CAST(1 AS DECIMAL(5,0)) / CAST(1 AS TINYINT)):decimal(11,6)>
 -- !query output
 1.000000
 
@@ -2557,7 +2557,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0)) / CAST(CAST(CAST(1 AS TINY
 -- !query
 SELECT cast(1 as decimal(10, 0)) / cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) / CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0))):decimal(16,6)>
+struct<(CAST(1 AS DECIMAL(10,0)) / CAST(1 AS TINYINT)):decimal(16,6)>
 -- !query output
 1.000000
 
@@ -2565,7 +2565,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) / CAST(CAST(CAST(1 AS TI
 -- !query
 SELECT cast(1 as decimal(20, 0)) / cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) / CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0))):decimal(26,6)>
+struct<(CAST(1 AS DECIMAL(20,0)) / CAST(1 AS TINYINT)):decimal(26,6)>
 -- !query output
 1.000000
 
@@ -2573,7 +2573,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) / CAST(CAST(CAST(1 AS TI
 -- !query
 SELECT cast(1 as decimal(3, 0))  / cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0)) / CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0))):decimal(9,6)>
+struct<(CAST(1 AS DECIMAL(3,0)) / CAST(1 AS SMALLINT)):decimal(9,6)>
 -- !query output
 1.000000
 
@@ -2581,7 +2581,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0)) / CAST(CAST(CAST(1 AS SMAL
 -- !query
 SELECT cast(1 as decimal(5, 0))  / cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(5,0)) / CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0))):decimal(11,6)>
+struct<(CAST(1 AS DECIMAL(5,0)) / CAST(1 AS SMALLINT)):decimal(11,6)>
 -- !query output
 1.000000
 
@@ -2589,7 +2589,7 @@ struct<(CAST(1 AS DECIMAL(5,0)) / CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0))):dec
 -- !query
 SELECT cast(1 as decimal(10, 0)) / cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) / CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0))):decimal(16,6)>
+struct<(CAST(1 AS DECIMAL(10,0)) / CAST(1 AS SMALLINT)):decimal(16,6)>
 -- !query output
 1.000000
 
@@ -2597,7 +2597,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) / CAST(CAST(CAST(1 AS SM
 -- !query
 SELECT cast(1 as decimal(20, 0)) / cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) / CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0))):decimal(26,6)>
+struct<(CAST(1 AS DECIMAL(20,0)) / CAST(1 AS SMALLINT)):decimal(26,6)>
 -- !query output
 1.000000
 
@@ -2605,7 +2605,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) / CAST(CAST(CAST(1 AS SM
 -- !query
 SELECT cast(1 as decimal(3, 0))  / cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) / CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(14,11)>
+struct<(CAST(1 AS DECIMAL(3,0)) / CAST(1 AS INT)):decimal(14,11)>
 -- !query output
 1.00000000000
 
@@ -2613,7 +2613,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) / CAST(CAST(CAST(1 AS INT
 -- !query
 SELECT cast(1 as decimal(5, 0))  / cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)) / CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(16,11)>
+struct<(CAST(1 AS DECIMAL(5,0)) / CAST(1 AS INT)):decimal(16,11)>
 -- !query output
 1.00000000000
 
@@ -2621,7 +2621,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)) / CAST(CAST(CAST(1 AS INT
 -- !query
 SELECT cast(1 as decimal(10, 0)) / cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(10,0)) / CAST(CAST(1 AS INT) AS DECIMAL(10,0))):decimal(21,11)>
+struct<(CAST(1 AS DECIMAL(10,0)) / CAST(1 AS INT)):decimal(21,11)>
 -- !query output
 1.00000000000
 
@@ -2629,7 +2629,7 @@ struct<(CAST(1 AS DECIMAL(10,0)) / CAST(CAST(1 AS INT) AS DECIMAL(10,0))):decima
 -- !query
 SELECT cast(1 as decimal(20, 0)) / cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) / CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0))):decimal(31,11)>
+struct<(CAST(1 AS DECIMAL(20,0)) / CAST(1 AS INT)):decimal(31,11)>
 -- !query output
 1.00000000000
 
@@ -2637,7 +2637,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) / CAST(CAST(CAST(1 AS IN
 -- !query
 SELECT cast(1 as decimal(3, 0))  / cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0)) / CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(24,21)>
+struct<(CAST(1 AS DECIMAL(3,0)) / CAST(1 AS BIGINT)):decimal(24,21)>
 -- !query output
 1.000000000000000000000
 
@@ -2645,7 +2645,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0)) / CAST(CAST(CAST(1 AS BIG
 -- !query
 SELECT cast(1 as decimal(5, 0))  / cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0)) / CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(26,21)>
+struct<(CAST(1 AS DECIMAL(5,0)) / CAST(1 AS BIGINT)):decimal(26,21)>
 -- !query output
 1.000000000000000000000
 
@@ -2653,7 +2653,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0)) / CAST(CAST(CAST(1 AS BIG
 -- !query
 SELECT cast(1 as decimal(10, 0)) / cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) / CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(31,21)>
+struct<(CAST(1 AS DECIMAL(10,0)) / CAST(1 AS BIGINT)):decimal(31,21)>
 -- !query output
 1.000000000000000000000
 
@@ -2661,7 +2661,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) / CAST(CAST(CAST(1 AS BI
 -- !query
 SELECT cast(1 as decimal(20, 0)) / cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(20,0)) / CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0))):decimal(38,18)>
+struct<(CAST(1 AS DECIMAL(20,0)) / CAST(1 AS BIGINT)):decimal(38,18)>
 -- !query output
 1.000000000000000000
 
@@ -2669,7 +2669,7 @@ struct<(CAST(1 AS DECIMAL(20,0)) / CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0))):dec
 -- !query
 SELECT cast(1 as decimal(3, 0))  / cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) / CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(3,0)) / CAST(1 AS FLOAT)):double>
 -- !query output
 1.0
 
@@ -2677,7 +2677,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) / CAST(CAST(1 AS FLOAT) AS DOUBL
 -- !query
 SELECT cast(1 as decimal(5, 0))  / cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) / CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(5,0)) / CAST(1 AS FLOAT)):double>
 -- !query output
 1.0
 
@@ -2685,7 +2685,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) / CAST(CAST(1 AS FLOAT) AS DOUBL
 -- !query
 SELECT cast(1 as decimal(10, 0)) / cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) / CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(10,0)) / CAST(1 AS FLOAT)):double>
 -- !query output
 1.0
 
@@ -2693,7 +2693,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) / CAST(CAST(1 AS FLOAT) AS DOUB
 -- !query
 SELECT cast(1 as decimal(20, 0)) / cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) / CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(20,0)) / CAST(1 AS FLOAT)):double>
 -- !query output
 1.0
 
@@ -2701,7 +2701,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) / CAST(CAST(1 AS FLOAT) AS DOUB
 -- !query
 SELECT cast(1 as decimal(3, 0))  / cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) / CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(3,0)) / CAST(1 AS DOUBLE)):double>
 -- !query output
 1.0
 
@@ -2709,7 +2709,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) / CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(5, 0))  / cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) / CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(5,0)) / CAST(1 AS DOUBLE)):double>
 -- !query output
 1.0
 
@@ -2717,7 +2717,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) / CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(10, 0)) / cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) / CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(10,0)) / CAST(1 AS DOUBLE)):double>
 -- !query output
 1.0
 
@@ -2725,7 +2725,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) / CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(20, 0)) / cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) / CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(20,0)) / CAST(1 AS DOUBLE)):double>
 -- !query output
 1.0
 
@@ -2733,7 +2733,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) / CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(3, 0))  / cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) / CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(14,11)>
+struct<(CAST(1 AS DECIMAL(3,0)) / CAST(1 AS DECIMAL(10,0))):decimal(14,11)>
 -- !query output
 1.00000000000
 
@@ -2741,7 +2741,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) / CAST(CAST(1 AS DECIMAL(
 -- !query
 SELECT cast(1 as decimal(5, 0))  / cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)) / CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(16,11)>
+struct<(CAST(1 AS DECIMAL(5,0)) / CAST(1 AS DECIMAL(10,0))):decimal(16,11)>
 -- !query output
 1.00000000000
 
@@ -2757,7 +2757,7 @@ struct<(CAST(1 AS DECIMAL(10,0)) / CAST(1 AS DECIMAL(10,0))):decimal(21,11)>
 -- !query
 SELECT cast(1 as decimal(20, 0)) / cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) / CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0))):decimal(31,11)>
+struct<(CAST(1 AS DECIMAL(20,0)) / CAST(1 AS DECIMAL(10,0))):decimal(31,11)>
 -- !query output
 1.00000000000
 
@@ -2765,7 +2765,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) / CAST(CAST(1 AS DECIMAL
 -- !query
 SELECT cast(1 as decimal(3, 0))  / cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) / CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(3,0)) / CAST(1 AS STRING)):double>
 -- !query output
 1.0
 
@@ -2773,7 +2773,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) / CAST(CAST(1 AS STRING) AS DOUB
 -- !query
 SELECT cast(1 as decimal(5, 0))  / cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) / CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(5,0)) / CAST(1 AS STRING)):double>
 -- !query output
 1.0
 
@@ -2781,7 +2781,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) / CAST(CAST(1 AS STRING) AS DOUB
 -- !query
 SELECT cast(1 as decimal(10, 0)) / cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) / CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(10,0)) / CAST(1 AS STRING)):double>
 -- !query output
 1.0
 
@@ -2789,7 +2789,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) / CAST(CAST(1 AS STRING) AS DOU
 -- !query
 SELECT cast(1 as decimal(20, 0)) / cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) / CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(20,0)) / CAST(1 AS STRING)):double>
 -- !query output
 1.0
 
@@ -2941,7 +2941,7 @@ cannot resolve '(CAST(1 AS DECIMAL(20,0)) / CAST('2017/12/11 09:30:00' AS DATE))
 -- !query
 SELECT cast(1 as tinyint) % cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) % CAST(1 AS DECIMAL(3,0))):decimal(3,0)>
+struct<(CAST(1 AS TINYINT) % CAST(1 AS DECIMAL(3,0))):decimal(3,0)>
 -- !query output
 0
 
@@ -2949,7 +2949,7 @@ struct<(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) % CAST(1 AS DECIMAL(3,0))):deci
 -- !query
 SELECT cast(1 as tinyint) % cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0)) % CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0))):decimal(3,0)>
+struct<(CAST(1 AS TINYINT) % CAST(1 AS DECIMAL(5,0))):decimal(3,0)>
 -- !query output
 0
 
@@ -2957,7 +2957,7 @@ struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0)) % CAST(CA
 -- !query
 SELECT cast(1 as tinyint) % cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0)) % CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(3,0)>
+struct<(CAST(1 AS TINYINT) % CAST(1 AS DECIMAL(10,0))):decimal(3,0)>
 -- !query output
 0
 
@@ -2965,7 +2965,7 @@ struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0)) % CAST(C
 -- !query
 SELECT cast(1 as tinyint) % cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0)) % CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(3,0)>
+struct<(CAST(1 AS TINYINT) % CAST(1 AS DECIMAL(20,0))):decimal(3,0)>
 -- !query output
 0
 
@@ -2973,7 +2973,7 @@ struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0)) % CAST(C
 -- !query
 SELECT cast(1 as smallint) % cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0)) % CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0))):decimal(3,0)>
+struct<(CAST(1 AS SMALLINT) % CAST(1 AS DECIMAL(3,0))):decimal(3,0)>
 -- !query output
 0
 
@@ -2981,7 +2981,7 @@ struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0)) % CAST(C
 -- !query
 SELECT cast(1 as smallint) % cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) % CAST(1 AS DECIMAL(5,0))):decimal(5,0)>
+struct<(CAST(1 AS SMALLINT) % CAST(1 AS DECIMAL(5,0))):decimal(5,0)>
 -- !query output
 0
 
@@ -2989,7 +2989,7 @@ struct<(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) % CAST(1 AS DECIMAL(5,0))):dec
 -- !query
 SELECT cast(1 as smallint) % cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0)) % CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(5,0)>
+struct<(CAST(1 AS SMALLINT) % CAST(1 AS DECIMAL(10,0))):decimal(5,0)>
 -- !query output
 0
 
@@ -2997,7 +2997,7 @@ struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0)) % CAST(
 -- !query
 SELECT cast(1 as smallint) % cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0)) % CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(5,0)>
+struct<(CAST(1 AS SMALLINT) % CAST(1 AS DECIMAL(20,0))):decimal(5,0)>
 -- !query output
 0
 
@@ -3005,7 +3005,7 @@ struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0)) % CAST(
 -- !query
 SELECT cast(1 as int) % cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) % CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0))):decimal(3,0)>
+struct<(CAST(1 AS INT) % CAST(1 AS DECIMAL(3,0))):decimal(3,0)>
 -- !query output
 0
 
@@ -3013,7 +3013,7 @@ struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) % CAST(CAST
 -- !query
 SELECT cast(1 as int) % cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) % CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0))):decimal(5,0)>
+struct<(CAST(1 AS INT) % CAST(1 AS DECIMAL(5,0))):decimal(5,0)>
 -- !query output
 0
 
@@ -3021,7 +3021,7 @@ struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) % CAST(CAST
 -- !query
 SELECT cast(1 as int) % cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) % CAST(1 AS DECIMAL(10,0))):decimal(10,0)>
+struct<(CAST(1 AS INT) % CAST(1 AS DECIMAL(10,0))):decimal(10,0)>
 -- !query output
 0
 
@@ -3029,7 +3029,7 @@ struct<(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) % CAST(1 AS DECIMAL(10,0))):decima
 -- !query
 SELECT cast(1 as int) % cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0)) % CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(10,0)>
+struct<(CAST(1 AS INT) % CAST(1 AS DECIMAL(20,0))):decimal(10,0)>
 -- !query output
 0
 
@@ -3037,7 +3037,7 @@ struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0)) % CAST(CAST
 -- !query
 SELECT cast(1 as bigint) % cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) % CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0))):decimal(3,0)>
+struct<(CAST(1 AS BIGINT) % CAST(1 AS DECIMAL(3,0))):decimal(3,0)>
 -- !query output
 0
 
@@ -3045,7 +3045,7 @@ struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) % CAST(C
 -- !query
 SELECT cast(1 as bigint) % cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) % CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0))):decimal(5,0)>
+struct<(CAST(1 AS BIGINT) % CAST(1 AS DECIMAL(5,0))):decimal(5,0)>
 -- !query output
 0
 
@@ -3053,7 +3053,7 @@ struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) % CAST(C
 -- !query
 SELECT cast(1 as bigint) % cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) % CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0))):decimal(10,0)>
+struct<(CAST(1 AS BIGINT) % CAST(1 AS DECIMAL(10,0))):decimal(10,0)>
 -- !query output
 0
 
@@ -3061,7 +3061,7 @@ struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) % CAST(C
 -- !query
 SELECT cast(1 as bigint) % cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) % CAST(1 AS DECIMAL(20,0))):decimal(20,0)>
+struct<(CAST(1 AS BIGINT) % CAST(1 AS DECIMAL(20,0))):decimal(20,0)>
 -- !query output
 0
 
@@ -3069,7 +3069,7 @@ struct<(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) % CAST(1 AS DECIMAL(20,0))):dec
 -- !query
 SELECT cast(1 as float) % cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) % CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) % CAST(1 AS DECIMAL(3,0))):double>
 -- !query output
 0.0
 
@@ -3077,7 +3077,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) % CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBL
 -- !query
 SELECT cast(1 as float) % cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) % CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) % CAST(1 AS DECIMAL(5,0))):double>
 -- !query output
 0.0
 
@@ -3085,7 +3085,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) % CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBL
 -- !query
 SELECT cast(1 as float) % cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) % CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) % CAST(1 AS DECIMAL(10,0))):double>
 -- !query output
 0.0
 
@@ -3093,7 +3093,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) % CAST(CAST(1 AS DECIMAL(10,0)) AS DOUB
 -- !query
 SELECT cast(1 as float) % cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) % CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) % CAST(1 AS DECIMAL(20,0))):double>
 -- !query output
 0.0
 
@@ -3101,7 +3101,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) % CAST(CAST(1 AS DECIMAL(20,0)) AS DOUB
 -- !query
 SELECT cast(1 as double) % cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) % CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) % CAST(1 AS DECIMAL(3,0))):double>
 -- !query output
 0.0
 
@@ -3109,7 +3109,7 @@ struct<(CAST(1 AS DOUBLE) % CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as double) % cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) % CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) % CAST(1 AS DECIMAL(5,0))):double>
 -- !query output
 0.0
 
@@ -3117,7 +3117,7 @@ struct<(CAST(1 AS DOUBLE) % CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as double) % cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) % CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) % CAST(1 AS DECIMAL(10,0))):double>
 -- !query output
 0.0
 
@@ -3125,7 +3125,7 @@ struct<(CAST(1 AS DOUBLE) % CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as double) % cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) % CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) % CAST(1 AS DECIMAL(20,0))):double>
 -- !query output
 0.0
 
@@ -3133,7 +3133,7 @@ struct<(CAST(1 AS DOUBLE) % CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(10, 0)) % cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) % CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0))):decimal(3,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) % CAST(1 AS DECIMAL(3,0))):decimal(3,0)>
 -- !query output
 0
 
@@ -3141,7 +3141,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) % CAST(CAST(1 AS DECIMAL
 -- !query
 SELECT cast(1 as decimal(10, 0)) % cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) % CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0))):decimal(5,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) % CAST(1 AS DECIMAL(5,0))):decimal(5,0)>
 -- !query output
 0
 
@@ -3157,7 +3157,7 @@ struct<(CAST(1 AS DECIMAL(10,0)) % CAST(1 AS DECIMAL(10,0))):decimal(10,0)>
 -- !query
 SELECT cast(1 as decimal(10, 0)) % cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) % CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(10,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) % CAST(1 AS DECIMAL(20,0))):decimal(10,0)>
 -- !query output
 0
 
@@ -3273,7 +3273,7 @@ cannot resolve '(CAST('2017-12-11 09:30:00' AS DATE) % CAST(1 AS DECIMAL(20,0)))
 -- !query
 SELECT cast(1 as decimal(3, 0))  % cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(3,0)) % CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0))):decimal(3,0)>
+struct<(CAST(1 AS DECIMAL(3,0)) % CAST(1 AS TINYINT)):decimal(3,0)>
 -- !query output
 0
 
@@ -3281,7 +3281,7 @@ struct<(CAST(1 AS DECIMAL(3,0)) % CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0))):deci
 -- !query
 SELECT cast(1 as decimal(5, 0))  % cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0)) % CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0))):decimal(3,0)>
+struct<(CAST(1 AS DECIMAL(5,0)) % CAST(1 AS TINYINT)):decimal(3,0)>
 -- !query output
 0
 
@@ -3289,7 +3289,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0)) % CAST(CAST(CAST(1 AS TINY
 -- !query
 SELECT cast(1 as decimal(10, 0)) % cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) % CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0))):decimal(3,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) % CAST(1 AS TINYINT)):decimal(3,0)>
 -- !query output
 0
 
@@ -3297,7 +3297,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) % CAST(CAST(CAST(1 AS TI
 -- !query
 SELECT cast(1 as decimal(20, 0)) % cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) % CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0))):decimal(3,0)>
+struct<(CAST(1 AS DECIMAL(20,0)) % CAST(1 AS TINYINT)):decimal(3,0)>
 -- !query output
 0
 
@@ -3305,7 +3305,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) % CAST(CAST(CAST(1 AS TI
 -- !query
 SELECT cast(1 as decimal(3, 0))  % cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0)) % CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0))):decimal(3,0)>
+struct<(CAST(1 AS DECIMAL(3,0)) % CAST(1 AS SMALLINT)):decimal(3,0)>
 -- !query output
 0
 
@@ -3313,7 +3313,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0)) % CAST(CAST(CAST(1 AS SMAL
 -- !query
 SELECT cast(1 as decimal(5, 0))  % cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(5,0)) % CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0))):decimal(5,0)>
+struct<(CAST(1 AS DECIMAL(5,0)) % CAST(1 AS SMALLINT)):decimal(5,0)>
 -- !query output
 0
 
@@ -3321,7 +3321,7 @@ struct<(CAST(1 AS DECIMAL(5,0)) % CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0))):dec
 -- !query
 SELECT cast(1 as decimal(10, 0)) % cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) % CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0))):decimal(5,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) % CAST(1 AS SMALLINT)):decimal(5,0)>
 -- !query output
 0
 
@@ -3329,7 +3329,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) % CAST(CAST(CAST(1 AS SM
 -- !query
 SELECT cast(1 as decimal(20, 0)) % cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) % CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0))):decimal(5,0)>
+struct<(CAST(1 AS DECIMAL(20,0)) % CAST(1 AS SMALLINT)):decimal(5,0)>
 -- !query output
 0
 
@@ -3337,7 +3337,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) % CAST(CAST(CAST(1 AS SM
 -- !query
 SELECT cast(1 as decimal(3, 0))  % cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) % CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(3,0)>
+struct<(CAST(1 AS DECIMAL(3,0)) % CAST(1 AS INT)):decimal(3,0)>
 -- !query output
 0
 
@@ -3345,7 +3345,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) % CAST(CAST(CAST(1 AS INT
 -- !query
 SELECT cast(1 as decimal(5, 0))  % cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)) % CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(5,0)>
+struct<(CAST(1 AS DECIMAL(5,0)) % CAST(1 AS INT)):decimal(5,0)>
 -- !query output
 0
 
@@ -3353,7 +3353,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)) % CAST(CAST(CAST(1 AS INT
 -- !query
 SELECT cast(1 as decimal(10, 0)) % cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(10,0)) % CAST(CAST(1 AS INT) AS DECIMAL(10,0))):decimal(10,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) % CAST(1 AS INT)):decimal(10,0)>
 -- !query output
 0
 
@@ -3361,7 +3361,7 @@ struct<(CAST(1 AS DECIMAL(10,0)) % CAST(CAST(1 AS INT) AS DECIMAL(10,0))):decima
 -- !query
 SELECT cast(1 as decimal(20, 0)) % cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) % CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0))):decimal(10,0)>
+struct<(CAST(1 AS DECIMAL(20,0)) % CAST(1 AS INT)):decimal(10,0)>
 -- !query output
 0
 
@@ -3369,7 +3369,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) % CAST(CAST(CAST(1 AS IN
 -- !query
 SELECT cast(1 as decimal(3, 0))  % cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0)) % CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(3,0)>
+struct<(CAST(1 AS DECIMAL(3,0)) % CAST(1 AS BIGINT)):decimal(3,0)>
 -- !query output
 0
 
@@ -3377,7 +3377,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0)) % CAST(CAST(CAST(1 AS BIG
 -- !query
 SELECT cast(1 as decimal(5, 0))  % cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0)) % CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(5,0)>
+struct<(CAST(1 AS DECIMAL(5,0)) % CAST(1 AS BIGINT)):decimal(5,0)>
 -- !query output
 0
 
@@ -3385,7 +3385,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0)) % CAST(CAST(CAST(1 AS BIG
 -- !query
 SELECT cast(1 as decimal(10, 0)) % cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) % CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(10,0)>
+struct<(CAST(1 AS DECIMAL(10,0)) % CAST(1 AS BIGINT)):decimal(10,0)>
 -- !query output
 0
 
@@ -3393,7 +3393,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) % CAST(CAST(CAST(1 AS BI
 -- !query
 SELECT cast(1 as decimal(20, 0)) % cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(20,0)) % CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0))):decimal(20,0)>
+struct<(CAST(1 AS DECIMAL(20,0)) % CAST(1 AS BIGINT)):decimal(20,0)>
 -- !query output
 0
 
@@ -3401,7 +3401,7 @@ struct<(CAST(1 AS DECIMAL(20,0)) % CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0))):dec
 -- !query
 SELECT cast(1 as decimal(3, 0))  % cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) % CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(3,0)) % CAST(1 AS FLOAT)):double>
 -- !query output
 0.0
 
@@ -3409,7 +3409,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) % CAST(CAST(1 AS FLOAT) AS DOUBL
 -- !query
 SELECT cast(1 as decimal(5, 0))  % cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) % CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(5,0)) % CAST(1 AS FLOAT)):double>
 -- !query output
 0.0
 
@@ -3417,7 +3417,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) % CAST(CAST(1 AS FLOAT) AS DOUBL
 -- !query
 SELECT cast(1 as decimal(10, 0)) % cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) % CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(10,0)) % CAST(1 AS FLOAT)):double>
 -- !query output
 0.0
 
@@ -3425,7 +3425,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) % CAST(CAST(1 AS FLOAT) AS DOUB
 -- !query
 SELECT cast(1 as decimal(20, 0)) % cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) % CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(20,0)) % CAST(1 AS FLOAT)):double>
 -- !query output
 0.0
 
@@ -3433,7 +3433,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) % CAST(CAST(1 AS FLOAT) AS DOUB
 -- !query
 SELECT cast(1 as decimal(3, 0))  % cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) % CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(3,0)) % CAST(1 AS DOUBLE)):double>
 -- !query output
 0.0
 
@@ -3441,7 +3441,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) % CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(5, 0))  % cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) % CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(5,0)) % CAST(1 AS DOUBLE)):double>
 -- !query output
 0.0
 
@@ -3449,7 +3449,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) % CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(10, 0)) % cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) % CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(10,0)) % CAST(1 AS DOUBLE)):double>
 -- !query output
 0.0
 
@@ -3457,7 +3457,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) % CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(20, 0)) % cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) % CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(20,0)) % CAST(1 AS DOUBLE)):double>
 -- !query output
 0.0
 
@@ -3465,7 +3465,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) % CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(3, 0))  % cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) % CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(3,0)>
+struct<(CAST(1 AS DECIMAL(3,0)) % CAST(1 AS DECIMAL(10,0))):decimal(3,0)>
 -- !query output
 0
 
@@ -3473,7 +3473,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) % CAST(CAST(1 AS DECIMAL(
 -- !query
 SELECT cast(1 as decimal(5, 0))  % cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)) % CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(5,0)>
+struct<(CAST(1 AS DECIMAL(5,0)) % CAST(1 AS DECIMAL(10,0))):decimal(5,0)>
 -- !query output
 0
 
@@ -3489,7 +3489,7 @@ struct<(CAST(1 AS DECIMAL(10,0)) % CAST(1 AS DECIMAL(10,0))):decimal(10,0)>
 -- !query
 SELECT cast(1 as decimal(20, 0)) % cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) % CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0))):decimal(10,0)>
+struct<(CAST(1 AS DECIMAL(20,0)) % CAST(1 AS DECIMAL(10,0))):decimal(10,0)>
 -- !query output
 0
 
@@ -3497,7 +3497,7 @@ struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) % CAST(CAST(1 AS DECIMAL
 -- !query
 SELECT cast(1 as decimal(3, 0))  % cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) % CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(3,0)) % CAST(1 AS STRING)):double>
 -- !query output
 0.0
 
@@ -3505,7 +3505,7 @@ struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) % CAST(CAST(1 AS STRING) AS DOUB
 -- !query
 SELECT cast(1 as decimal(5, 0))  % cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) % CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(5,0)) % CAST(1 AS STRING)):double>
 -- !query output
 0.0
 
@@ -3513,7 +3513,7 @@ struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) % CAST(CAST(1 AS STRING) AS DOUB
 -- !query
 SELECT cast(1 as decimal(10, 0)) % cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) % CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(10,0)) % CAST(1 AS STRING)):double>
 -- !query output
 0.0
 
@@ -3521,7 +3521,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) % CAST(CAST(1 AS STRING) AS DOU
 -- !query
 SELECT cast(1 as decimal(20, 0)) % cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) % CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(20,0)) % CAST(1 AS STRING)):double>
 -- !query output
 0.0
 
@@ -3673,7 +3673,7 @@ cannot resolve '(CAST(1 AS DECIMAL(20,0)) % CAST('2017-12-11 09:30:00' AS DATE))
 -- !query
 SELECT pmod(cast(1 as tinyint), cast(1 as decimal(3, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)), CAST(1 AS DECIMAL(3,0))):decimal(3,0)>
+struct<pmod(CAST(1 AS TINYINT), CAST(1 AS DECIMAL(3,0))):decimal(3,0)>
 -- !query output
 0
 
@@ -3681,7 +3681,7 @@ struct<pmod(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)), CAST(1 AS DECIMAL(3,0))):d
 -- !query
 SELECT pmod(cast(1 as tinyint), cast(1 as decimal(5, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0)), CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0))):decimal(3,0)>
+struct<pmod(CAST(1 AS TINYINT), CAST(1 AS DECIMAL(5,0))):decimal(3,0)>
 -- !query output
 0
 
@@ -3689,7 +3689,7 @@ struct<pmod(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0)), CAST
 -- !query
 SELECT pmod(cast(1 as tinyint), cast(1 as decimal(10, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0)), CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(3,0)>
+struct<pmod(CAST(1 AS TINYINT), CAST(1 AS DECIMAL(10,0))):decimal(3,0)>
 -- !query output
 0
 
@@ -3697,7 +3697,7 @@ struct<pmod(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0)), CAS
 -- !query
 SELECT pmod(cast(1 as tinyint), cast(1 as decimal(20, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0)), CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(3,0)>
+struct<pmod(CAST(1 AS TINYINT), CAST(1 AS DECIMAL(20,0))):decimal(3,0)>
 -- !query output
 0
 
@@ -3705,7 +3705,7 @@ struct<pmod(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0)), CAS
 -- !query
 SELECT pmod(cast(1 as smallint), cast(1 as decimal(3, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0)), CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0))):decimal(3,0)>
+struct<pmod(CAST(1 AS SMALLINT), CAST(1 AS DECIMAL(3,0))):decimal(3,0)>
 -- !query output
 0
 
@@ -3713,7 +3713,7 @@ struct<pmod(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0)), CAS
 -- !query
 SELECT pmod(cast(1 as smallint), cast(1 as decimal(5, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)), CAST(1 AS DECIMAL(5,0))):decimal(5,0)>
+struct<pmod(CAST(1 AS SMALLINT), CAST(1 AS DECIMAL(5,0))):decimal(5,0)>
 -- !query output
 0
 
@@ -3721,7 +3721,7 @@ struct<pmod(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)), CAST(1 AS DECIMAL(5,0))):
 -- !query
 SELECT pmod(cast(1 as smallint), cast(1 as decimal(10, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0)), CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(5,0)>
+struct<pmod(CAST(1 AS SMALLINT), CAST(1 AS DECIMAL(10,0))):decimal(5,0)>
 -- !query output
 0
 
@@ -3729,7 +3729,7 @@ struct<pmod(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0)), CA
 -- !query
 SELECT pmod(cast(1 as smallint), cast(1 as decimal(20, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0)), CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(5,0)>
+struct<pmod(CAST(1 AS SMALLINT), CAST(1 AS DECIMAL(20,0))):decimal(5,0)>
 -- !query output
 0
 
@@ -3737,7 +3737,7 @@ struct<pmod(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0)), CA
 -- !query
 SELECT pmod(cast(1 as int), cast(1 as decimal(3, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)), CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0))):decimal(3,0)>
+struct<pmod(CAST(1 AS INT), CAST(1 AS DECIMAL(3,0))):decimal(3,0)>
 -- !query output
 0
 
@@ -3745,7 +3745,7 @@ struct<pmod(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)), CAST(C
 -- !query
 SELECT pmod(cast(1 as int), cast(1 as decimal(5, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)), CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0))):decimal(5,0)>
+struct<pmod(CAST(1 AS INT), CAST(1 AS DECIMAL(5,0))):decimal(5,0)>
 -- !query output
 0
 
@@ -3753,7 +3753,7 @@ struct<pmod(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)), CAST(C
 -- !query
 SELECT pmod(cast(1 as int), cast(1 as decimal(10, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS INT) AS DECIMAL(10,0)), CAST(1 AS DECIMAL(10,0))):decimal(10,0)>
+struct<pmod(CAST(1 AS INT), CAST(1 AS DECIMAL(10,0))):decimal(10,0)>
 -- !query output
 0
 
@@ -3761,7 +3761,7 @@ struct<pmod(CAST(CAST(1 AS INT) AS DECIMAL(10,0)), CAST(1 AS DECIMAL(10,0))):dec
 -- !query
 SELECT pmod(cast(1 as int), cast(1 as decimal(20, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0)), CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(10,0)>
+struct<pmod(CAST(1 AS INT), CAST(1 AS DECIMAL(20,0))):decimal(10,0)>
 -- !query output
 0
 
@@ -3769,7 +3769,7 @@ struct<pmod(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0)), CAST(C
 -- !query
 SELECT pmod(cast(1 as bigint), cast(1 as decimal(3, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)), CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0))):decimal(3,0)>
+struct<pmod(CAST(1 AS BIGINT), CAST(1 AS DECIMAL(3,0))):decimal(3,0)>
 -- !query output
 0
 
@@ -3777,7 +3777,7 @@ struct<pmod(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)), CAS
 -- !query
 SELECT pmod(cast(1 as bigint), cast(1 as decimal(5, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)), CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0))):decimal(5,0)>
+struct<pmod(CAST(1 AS BIGINT), CAST(1 AS DECIMAL(5,0))):decimal(5,0)>
 -- !query output
 0
 
@@ -3785,7 +3785,7 @@ struct<pmod(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)), CAS
 -- !query
 SELECT pmod(cast(1 as bigint), cast(1 as decimal(10, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)), CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0))):decimal(10,0)>
+struct<pmod(CAST(1 AS BIGINT), CAST(1 AS DECIMAL(10,0))):decimal(10,0)>
 -- !query output
 0
 
@@ -3793,7 +3793,7 @@ struct<pmod(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)), CAS
 -- !query
 SELECT pmod(cast(1 as bigint), cast(1 as decimal(20, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)), CAST(1 AS DECIMAL(20,0))):decimal(20,0)>
+struct<pmod(CAST(1 AS BIGINT), CAST(1 AS DECIMAL(20,0))):decimal(20,0)>
 -- !query output
 0
 
@@ -3801,7 +3801,7 @@ struct<pmod(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)), CAST(1 AS DECIMAL(20,0))):
 -- !query
 SELECT pmod(cast(1 as float), cast(1 as decimal(3, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS FLOAT) AS DOUBLE), CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):double>
+struct<pmod(CAST(1 AS FLOAT), CAST(1 AS DECIMAL(3,0))):double>
 -- !query output
 0.0
 
@@ -3809,7 +3809,7 @@ struct<pmod(CAST(CAST(1 AS FLOAT) AS DOUBLE), CAST(CAST(1 AS DECIMAL(3,0)) AS DO
 -- !query
 SELECT pmod(cast(1 as float), cast(1 as decimal(5, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS FLOAT) AS DOUBLE), CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):double>
+struct<pmod(CAST(1 AS FLOAT), CAST(1 AS DECIMAL(5,0))):double>
 -- !query output
 0.0
 
@@ -3817,7 +3817,7 @@ struct<pmod(CAST(CAST(1 AS FLOAT) AS DOUBLE), CAST(CAST(1 AS DECIMAL(5,0)) AS DO
 -- !query
 SELECT pmod(cast(1 as float), cast(1 as decimal(10, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS FLOAT) AS DOUBLE), CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
+struct<pmod(CAST(1 AS FLOAT), CAST(1 AS DECIMAL(10,0))):double>
 -- !query output
 0.0
 
@@ -3825,7 +3825,7 @@ struct<pmod(CAST(CAST(1 AS FLOAT) AS DOUBLE), CAST(CAST(1 AS DECIMAL(10,0)) AS D
 -- !query
 SELECT pmod(cast(1 as float), cast(1 as decimal(20, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS FLOAT) AS DOUBLE), CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):double>
+struct<pmod(CAST(1 AS FLOAT), CAST(1 AS DECIMAL(20,0))):double>
 -- !query output
 0.0
 
@@ -3833,7 +3833,7 @@ struct<pmod(CAST(CAST(1 AS FLOAT) AS DOUBLE), CAST(CAST(1 AS DECIMAL(20,0)) AS D
 -- !query
 SELECT pmod(cast(1 as double), cast(1 as decimal(3, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(1 AS DOUBLE), CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):double>
+struct<pmod(CAST(1 AS DOUBLE), CAST(1 AS DECIMAL(3,0))):double>
 -- !query output
 0.0
 
@@ -3841,7 +3841,7 @@ struct<pmod(CAST(1 AS DOUBLE), CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):double>
 -- !query
 SELECT pmod(cast(1 as double), cast(1 as decimal(5, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(1 AS DOUBLE), CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):double>
+struct<pmod(CAST(1 AS DOUBLE), CAST(1 AS DECIMAL(5,0))):double>
 -- !query output
 0.0
 
@@ -3849,7 +3849,7 @@ struct<pmod(CAST(1 AS DOUBLE), CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):double>
 -- !query
 SELECT pmod(cast(1 as double), cast(1 as decimal(10, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(1 AS DOUBLE), CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
+struct<pmod(CAST(1 AS DOUBLE), CAST(1 AS DECIMAL(10,0))):double>
 -- !query output
 0.0
 
@@ -3857,7 +3857,7 @@ struct<pmod(CAST(1 AS DOUBLE), CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
 -- !query
 SELECT pmod(cast(1 as double), cast(1 as decimal(20, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(1 AS DOUBLE), CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):double>
+struct<pmod(CAST(1 AS DOUBLE), CAST(1 AS DECIMAL(20,0))):double>
 -- !query output
 0.0
 
@@ -3865,7 +3865,7 @@ struct<pmod(CAST(1 AS DOUBLE), CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):double>
 -- !query
 SELECT pmod(cast(1 as decimal(10, 0)), cast(1 as decimal(3, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)), CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0))):decimal(3,0)>
+struct<pmod(CAST(1 AS DECIMAL(10,0)), CAST(1 AS DECIMAL(3,0))):decimal(3,0)>
 -- !query output
 0
 
@@ -3873,7 +3873,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)), CAST(CAST(1 AS DECI
 -- !query
 SELECT pmod(cast(1 as decimal(10, 0)), cast(1 as decimal(5, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)), CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0))):decimal(5,0)>
+struct<pmod(CAST(1 AS DECIMAL(10,0)), CAST(1 AS DECIMAL(5,0))):decimal(5,0)>
 -- !query output
 0
 
@@ -3889,7 +3889,7 @@ struct<pmod(CAST(1 AS DECIMAL(10,0)), CAST(1 AS DECIMAL(10,0))):decimal(10,0)>
 -- !query
 SELECT pmod(cast(1 as decimal(10, 0)), cast(1 as decimal(20, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)), CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(10,0)>
+struct<pmod(CAST(1 AS DECIMAL(10,0)), CAST(1 AS DECIMAL(20,0))):decimal(10,0)>
 -- !query output
 0
 
@@ -4005,7 +4005,7 @@ cannot resolve 'pmod(CAST('2017-12-11 09:30:00' AS DATE), CAST(1 AS DECIMAL(20,0
 -- !query
 SELECT pmod(cast(1 as decimal(3, 0)) , cast(1 as tinyint)) FROM t
 -- !query schema
-struct<pmod(CAST(1 AS DECIMAL(3,0)), CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0))):decimal(3,0)>
+struct<pmod(CAST(1 AS DECIMAL(3,0)), CAST(1 AS TINYINT)):decimal(3,0)>
 -- !query output
 0
 
@@ -4013,7 +4013,7 @@ struct<pmod(CAST(1 AS DECIMAL(3,0)), CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0))):d
 -- !query
 SELECT pmod(cast(1 as decimal(5, 0)) , cast(1 as tinyint)) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0)), CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0))):decimal(3,0)>
+struct<pmod(CAST(1 AS DECIMAL(5,0)), CAST(1 AS TINYINT)):decimal(3,0)>
 -- !query output
 0
 
@@ -4021,7 +4021,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0)), CAST(CAST(CAST(1 AS T
 -- !query
 SELECT pmod(cast(1 as decimal(10, 0)), cast(1 as tinyint)) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)), CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0))):decimal(3,0)>
+struct<pmod(CAST(1 AS DECIMAL(10,0)), CAST(1 AS TINYINT)):decimal(3,0)>
 -- !query output
 0
 
@@ -4029,7 +4029,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)), CAST(CAST(CAST(1 AS
 -- !query
 SELECT pmod(cast(1 as decimal(20, 0)), cast(1 as tinyint)) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)), CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0))):decimal(3,0)>
+struct<pmod(CAST(1 AS DECIMAL(20,0)), CAST(1 AS TINYINT)):decimal(3,0)>
 -- !query output
 0
 
@@ -4037,7 +4037,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)), CAST(CAST(CAST(1 AS
 -- !query
 SELECT pmod(cast(1 as decimal(3, 0)) , cast(1 as smallint)) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0)), CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0))):decimal(3,0)>
+struct<pmod(CAST(1 AS DECIMAL(3,0)), CAST(1 AS SMALLINT)):decimal(3,0)>
 -- !query output
 0
 
@@ -4045,7 +4045,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0)), CAST(CAST(CAST(1 AS S
 -- !query
 SELECT pmod(cast(1 as decimal(5, 0)) , cast(1 as smallint)) FROM t
 -- !query schema
-struct<pmod(CAST(1 AS DECIMAL(5,0)), CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0))):decimal(5,0)>
+struct<pmod(CAST(1 AS DECIMAL(5,0)), CAST(1 AS SMALLINT)):decimal(5,0)>
 -- !query output
 0
 
@@ -4053,7 +4053,7 @@ struct<pmod(CAST(1 AS DECIMAL(5,0)), CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0))):
 -- !query
 SELECT pmod(cast(1 as decimal(10, 0)), cast(1 as smallint)) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)), CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0))):decimal(5,0)>
+struct<pmod(CAST(1 AS DECIMAL(10,0)), CAST(1 AS SMALLINT)):decimal(5,0)>
 -- !query output
 0
 
@@ -4061,7 +4061,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)), CAST(CAST(CAST(1 AS
 -- !query
 SELECT pmod(cast(1 as decimal(20, 0)), cast(1 as smallint)) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)), CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0))):decimal(5,0)>
+struct<pmod(CAST(1 AS DECIMAL(20,0)), CAST(1 AS SMALLINT)):decimal(5,0)>
 -- !query output
 0
 
@@ -4069,7 +4069,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)), CAST(CAST(CAST(1 AS
 -- !query
 SELECT pmod(cast(1 as decimal(3, 0)) , cast(1 as int)) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)), CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(3,0)>
+struct<pmod(CAST(1 AS DECIMAL(3,0)), CAST(1 AS INT)):decimal(3,0)>
 -- !query output
 0
 
@@ -4077,7 +4077,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)), CAST(CAST(CAST(1 AS
 -- !query
 SELECT pmod(cast(1 as decimal(5, 0)) , cast(1 as int)) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)), CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(5,0)>
+struct<pmod(CAST(1 AS DECIMAL(5,0)), CAST(1 AS INT)):decimal(5,0)>
 -- !query output
 0
 
@@ -4085,7 +4085,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)), CAST(CAST(CAST(1 AS
 -- !query
 SELECT pmod(cast(1 as decimal(10, 0)), cast(1 as int)) FROM t
 -- !query schema
-struct<pmod(CAST(1 AS DECIMAL(10,0)), CAST(CAST(1 AS INT) AS DECIMAL(10,0))):decimal(10,0)>
+struct<pmod(CAST(1 AS DECIMAL(10,0)), CAST(1 AS INT)):decimal(10,0)>
 -- !query output
 0
 
@@ -4093,7 +4093,7 @@ struct<pmod(CAST(1 AS DECIMAL(10,0)), CAST(CAST(1 AS INT) AS DECIMAL(10,0))):dec
 -- !query
 SELECT pmod(cast(1 as decimal(20, 0)), cast(1 as int)) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)), CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0))):decimal(10,0)>
+struct<pmod(CAST(1 AS DECIMAL(20,0)), CAST(1 AS INT)):decimal(10,0)>
 -- !query output
 0
 
@@ -4101,7 +4101,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)), CAST(CAST(CAST(1 AS
 -- !query
 SELECT pmod(cast(1 as decimal(3, 0)) , cast(1 as bigint)) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0)), CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(3,0)>
+struct<pmod(CAST(1 AS DECIMAL(3,0)), CAST(1 AS BIGINT)):decimal(3,0)>
 -- !query output
 0
 
@@ -4109,7 +4109,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0)), CAST(CAST(CAST(1 AS
 -- !query
 SELECT pmod(cast(1 as decimal(5, 0)) , cast(1 as bigint)) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0)), CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(5,0)>
+struct<pmod(CAST(1 AS DECIMAL(5,0)), CAST(1 AS BIGINT)):decimal(5,0)>
 -- !query output
 0
 
@@ -4117,7 +4117,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0)), CAST(CAST(CAST(1 AS
 -- !query
 SELECT pmod(cast(1 as decimal(10, 0)), cast(1 as bigint)) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)), CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(10,0)>
+struct<pmod(CAST(1 AS DECIMAL(10,0)), CAST(1 AS BIGINT)):decimal(10,0)>
 -- !query output
 0
 
@@ -4125,7 +4125,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)), CAST(CAST(CAST(1 AS
 -- !query
 SELECT pmod(cast(1 as decimal(20, 0)), cast(1 as bigint)) FROM t
 -- !query schema
-struct<pmod(CAST(1 AS DECIMAL(20,0)), CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0))):decimal(20,0)>
+struct<pmod(CAST(1 AS DECIMAL(20,0)), CAST(1 AS BIGINT)):decimal(20,0)>
 -- !query output
 0
 
@@ -4133,7 +4133,7 @@ struct<pmod(CAST(1 AS DECIMAL(20,0)), CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0))):
 -- !query
 SELECT pmod(cast(1 as decimal(3, 0)) , cast(1 as float)) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE), CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<pmod(CAST(1 AS DECIMAL(3,0)), CAST(1 AS FLOAT)):double>
 -- !query output
 0.0
 
@@ -4141,7 +4141,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE), CAST(CAST(1 AS FLOAT) AS DO
 -- !query
 SELECT pmod(cast(1 as decimal(5, 0)) , cast(1 as float)) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE), CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<pmod(CAST(1 AS DECIMAL(5,0)), CAST(1 AS FLOAT)):double>
 -- !query output
 0.0
 
@@ -4149,7 +4149,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE), CAST(CAST(1 AS FLOAT) AS DO
 -- !query
 SELECT pmod(cast(1 as decimal(10, 0)), cast(1 as float)) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE), CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<pmod(CAST(1 AS DECIMAL(10,0)), CAST(1 AS FLOAT)):double>
 -- !query output
 0.0
 
@@ -4157,7 +4157,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE), CAST(CAST(1 AS FLOAT) AS D
 -- !query
 SELECT pmod(cast(1 as decimal(20, 0)), cast(1 as float)) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE), CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<pmod(CAST(1 AS DECIMAL(20,0)), CAST(1 AS FLOAT)):double>
 -- !query output
 0.0
 
@@ -4165,7 +4165,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE), CAST(CAST(1 AS FLOAT) AS D
 -- !query
 SELECT pmod(cast(1 as decimal(3, 0)) , cast(1 as double)) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE), CAST(1 AS DOUBLE)):double>
+struct<pmod(CAST(1 AS DECIMAL(3,0)), CAST(1 AS DOUBLE)):double>
 -- !query output
 0.0
 
@@ -4173,7 +4173,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE), CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT pmod(cast(1 as decimal(5, 0)) , cast(1 as double)) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE), CAST(1 AS DOUBLE)):double>
+struct<pmod(CAST(1 AS DECIMAL(5,0)), CAST(1 AS DOUBLE)):double>
 -- !query output
 0.0
 
@@ -4181,7 +4181,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE), CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT pmod(cast(1 as decimal(10, 0)), cast(1 as double)) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE), CAST(1 AS DOUBLE)):double>
+struct<pmod(CAST(1 AS DECIMAL(10,0)), CAST(1 AS DOUBLE)):double>
 -- !query output
 0.0
 
@@ -4189,7 +4189,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE), CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT pmod(cast(1 as decimal(20, 0)), cast(1 as double)) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE), CAST(1 AS DOUBLE)):double>
+struct<pmod(CAST(1 AS DECIMAL(20,0)), CAST(1 AS DOUBLE)):double>
 -- !query output
 0.0
 
@@ -4197,7 +4197,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE), CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT pmod(cast(1 as decimal(3, 0)) , cast(1 as decimal(10, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)), CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(3,0)>
+struct<pmod(CAST(1 AS DECIMAL(3,0)), CAST(1 AS DECIMAL(10,0))):decimal(3,0)>
 -- !query output
 0
 
@@ -4205,7 +4205,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)), CAST(CAST(1 AS DECIM
 -- !query
 SELECT pmod(cast(1 as decimal(5, 0)) , cast(1 as decimal(10, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)), CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(5,0)>
+struct<pmod(CAST(1 AS DECIMAL(5,0)), CAST(1 AS DECIMAL(10,0))):decimal(5,0)>
 -- !query output
 0
 
@@ -4221,7 +4221,7 @@ struct<pmod(CAST(1 AS DECIMAL(10,0)), CAST(1 AS DECIMAL(10,0))):decimal(10,0)>
 -- !query
 SELECT pmod(cast(1 as decimal(20, 0)), cast(1 as decimal(10, 0))) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)), CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0))):decimal(10,0)>
+struct<pmod(CAST(1 AS DECIMAL(20,0)), CAST(1 AS DECIMAL(10,0))):decimal(10,0)>
 -- !query output
 0
 
@@ -4229,7 +4229,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)), CAST(CAST(1 AS DECI
 -- !query
 SELECT pmod(cast(1 as decimal(3, 0)) , cast(1 as string)) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE), CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<pmod(CAST(1 AS DECIMAL(3,0)), CAST(1 AS STRING)):double>
 -- !query output
 0.0
 
@@ -4237,7 +4237,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE), CAST(CAST(1 AS STRING) AS D
 -- !query
 SELECT pmod(cast(1 as decimal(5, 0)) , cast(1 as string)) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE), CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<pmod(CAST(1 AS DECIMAL(5,0)), CAST(1 AS STRING)):double>
 -- !query output
 0.0
 
@@ -4245,7 +4245,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE), CAST(CAST(1 AS STRING) AS D
 -- !query
 SELECT pmod(cast(1 as decimal(10, 0)), cast(1 as string)) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE), CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<pmod(CAST(1 AS DECIMAL(10,0)), CAST(1 AS STRING)):double>
 -- !query output
 0.0
 
@@ -4253,7 +4253,7 @@ struct<pmod(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE), CAST(CAST(1 AS STRING) AS
 -- !query
 SELECT pmod(cast(1 as decimal(20, 0)), cast(1 as string)) FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE), CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<pmod(CAST(1 AS DECIMAL(20,0)), CAST(1 AS STRING)):double>
 -- !query output
 0.0
 
@@ -4405,7 +4405,7 @@ cannot resolve 'pmod(CAST(1 AS DECIMAL(20,0)), CAST('2017-12-11 09:30:00' AS DAT
 -- !query
 SELECT cast(1 as tinyint) = cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) = CAST(1 AS DECIMAL(3,0))):boolean>
+struct<(CAST(1 AS TINYINT) = CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -4413,7 +4413,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) = cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0)) = CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS TINYINT) = CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -4421,7 +4421,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) = cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0)) = CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS TINYINT) = CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -4429,7 +4429,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) = cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0)) = CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS TINYINT) = CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -4437,7 +4437,7 @@ true
 -- !query
 SELECT cast(1 as smallint) = cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0)) = CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS SMALLINT) = CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -4445,7 +4445,7 @@ true
 -- !query
 SELECT cast(1 as smallint) = cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) = CAST(1 AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS SMALLINT) = CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -4453,7 +4453,7 @@ true
 -- !query
 SELECT cast(1 as smallint) = cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0)) = CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS SMALLINT) = CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -4461,7 +4461,7 @@ true
 -- !query
 SELECT cast(1 as smallint) = cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0)) = CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS SMALLINT) = CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -4469,7 +4469,7 @@ true
 -- !query
 SELECT cast(1 as int) = cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) = CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS INT) = CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -4477,7 +4477,7 @@ true
 -- !query
 SELECT cast(1 as int) = cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) = CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS INT) = CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -4485,7 +4485,7 @@ true
 -- !query
 SELECT cast(1 as int) = cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) = CAST(1 AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS INT) = CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -4493,7 +4493,7 @@ true
 -- !query
 SELECT cast(1 as int) = cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0)) = CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS INT) = CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -4501,7 +4501,7 @@ true
 -- !query
 SELECT cast(1 as bigint) = cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) = CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS BIGINT) = CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -4509,7 +4509,7 @@ true
 -- !query
 SELECT cast(1 as bigint) = cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) = CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS BIGINT) = CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -4517,7 +4517,7 @@ true
 -- !query
 SELECT cast(1 as bigint) = cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) = CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS BIGINT) = CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -4525,7 +4525,7 @@ true
 -- !query
 SELECT cast(1 as bigint) = cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) = CAST(1 AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS BIGINT) = CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -4533,7 +4533,7 @@ true
 -- !query
 SELECT cast(1 as float) = cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) = CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS FLOAT) = CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -4541,7 +4541,7 @@ true
 -- !query
 SELECT cast(1 as float) = cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) = CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS FLOAT) = CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -4549,7 +4549,7 @@ true
 -- !query
 SELECT cast(1 as float) = cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) = CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS FLOAT) = CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -4557,7 +4557,7 @@ true
 -- !query
 SELECT cast(1 as float) = cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) = CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS FLOAT) = CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -4565,7 +4565,7 @@ true
 -- !query
 SELECT cast(1 as double) = cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) = CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) = CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -4573,7 +4573,7 @@ true
 -- !query
 SELECT cast(1 as double) = cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) = CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) = CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -4581,7 +4581,7 @@ true
 -- !query
 SELECT cast(1 as double) = cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) = CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) = CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -4589,7 +4589,7 @@ true
 -- !query
 SELECT cast(1 as double) = cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) = CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) = CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -4597,7 +4597,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) = cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) = CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) = CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -4605,7 +4605,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) = cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) = CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) = CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -4621,7 +4621,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) = cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) = CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) = CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -4737,7 +4737,7 @@ cannot resolve '(CAST('2017-12-11 09:30:00' AS DATE) = CAST(1 AS DECIMAL(20,0)))
 -- !query
 SELECT cast(1 as decimal(3, 0))  = cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(3,0)) = CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) = CAST(1 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -4745,7 +4745,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  = cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0)) = CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) = CAST(1 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -4753,7 +4753,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) = cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) = CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) = CAST(1 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -4761,7 +4761,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) = cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) = CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) = CAST(1 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -4769,7 +4769,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  = cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0)) = CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) = CAST(1 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -4777,7 +4777,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  = cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(5,0)) = CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) = CAST(1 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -4785,7 +4785,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) = cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) = CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) = CAST(1 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -4793,7 +4793,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) = cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) = CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) = CAST(1 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -4801,7 +4801,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  = cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) = CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) = CAST(1 AS INT)):boolean>
 -- !query output
 true
 
@@ -4809,7 +4809,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  = cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)) = CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) = CAST(1 AS INT)):boolean>
 -- !query output
 true
 
@@ -4817,7 +4817,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) = cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(10,0)) = CAST(CAST(1 AS INT) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) = CAST(1 AS INT)):boolean>
 -- !query output
 true
 
@@ -4825,7 +4825,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) = cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) = CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) = CAST(1 AS INT)):boolean>
 -- !query output
 true
 
@@ -4833,7 +4833,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  = cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0)) = CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) = CAST(1 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -4841,7 +4841,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  = cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0)) = CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) = CAST(1 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -4849,7 +4849,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) = cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) = CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) = CAST(1 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -4857,7 +4857,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) = cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(20,0)) = CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) = CAST(1 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -4865,7 +4865,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  = cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) = CAST(CAST(1 AS FLOAT) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) = CAST(1 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -4873,7 +4873,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  = cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) = CAST(CAST(1 AS FLOAT) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) = CAST(1 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -4881,7 +4881,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) = cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) = CAST(CAST(1 AS FLOAT) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) = CAST(1 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -4889,7 +4889,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) = cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) = CAST(CAST(1 AS FLOAT) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) = CAST(1 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -4897,7 +4897,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  = cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) = CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) = CAST(1 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -4905,7 +4905,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  = cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) = CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) = CAST(1 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -4913,7 +4913,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) = cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) = CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) = CAST(1 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -4921,7 +4921,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) = cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) = CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) = CAST(1 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -4929,7 +4929,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  = cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) = CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) = CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -4937,7 +4937,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  = cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)) = CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) = CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -4953,7 +4953,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) = cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) = CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) = CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -4961,7 +4961,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  = cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) = CAST(CAST(1 AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) = CAST(1 AS STRING)):boolean>
 -- !query output
 true
 
@@ -4969,7 +4969,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  = cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) = CAST(CAST(1 AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) = CAST(1 AS STRING)):boolean>
 -- !query output
 true
 
@@ -4977,7 +4977,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) = cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) = CAST(CAST(1 AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) = CAST(1 AS STRING)):boolean>
 -- !query output
 true
 
@@ -4985,7 +4985,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) = cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) = CAST(CAST(1 AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) = CAST(1 AS STRING)):boolean>
 -- !query output
 true
 
@@ -5029,7 +5029,7 @@ cannot resolve '(CAST(1 AS DECIMAL(20,0)) = CAST('1' AS BINARY))' due to data ty
 -- !query
 SELECT cast(1 as decimal(3, 0))  = cast(1 as boolean) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(3,0)) = CAST(CAST(1 AS BOOLEAN) AS DECIMAL(3,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) = CAST(1 AS BOOLEAN)):boolean>
 -- !query output
 true
 
@@ -5037,7 +5037,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  = cast(1 as boolean) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(5,0)) = CAST(CAST(1 AS BOOLEAN) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) = CAST(1 AS BOOLEAN)):boolean>
 -- !query output
 true
 
@@ -5045,7 +5045,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) = cast(1 as boolean) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(10,0)) = CAST(CAST(1 AS BOOLEAN) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) = CAST(1 AS BOOLEAN)):boolean>
 -- !query output
 true
 
@@ -5053,7 +5053,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) = cast(1 as boolean) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(20,0)) = CAST(CAST(1 AS BOOLEAN) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) = CAST(1 AS BOOLEAN)):boolean>
 -- !query output
 true
 
@@ -5133,7 +5133,7 @@ cannot resolve '(CAST(1 AS DECIMAL(20,0)) = CAST('2017-12-11 09:30:00' AS DATE))
 -- !query
 SELECT cast(1 as tinyint) <=> cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) <=> CAST(1 AS DECIMAL(3,0))):boolean>
+struct<(CAST(1 AS TINYINT) <=> CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -5141,7 +5141,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) <=> cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0)) <=> CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS TINYINT) <=> CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -5149,7 +5149,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) <=> cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0)) <=> CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS TINYINT) <=> CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -5157,7 +5157,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) <=> cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0)) <=> CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS TINYINT) <=> CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -5165,7 +5165,7 @@ true
 -- !query
 SELECT cast(1 as smallint) <=> cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0)) <=> CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS SMALLINT) <=> CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -5173,7 +5173,7 @@ true
 -- !query
 SELECT cast(1 as smallint) <=> cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) <=> CAST(1 AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS SMALLINT) <=> CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -5181,7 +5181,7 @@ true
 -- !query
 SELECT cast(1 as smallint) <=> cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0)) <=> CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS SMALLINT) <=> CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -5189,7 +5189,7 @@ true
 -- !query
 SELECT cast(1 as smallint) <=> cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0)) <=> CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS SMALLINT) <=> CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -5197,7 +5197,7 @@ true
 -- !query
 SELECT cast(1 as int) <=> cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) <=> CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS INT) <=> CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -5205,7 +5205,7 @@ true
 -- !query
 SELECT cast(1 as int) <=> cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) <=> CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS INT) <=> CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -5213,7 +5213,7 @@ true
 -- !query
 SELECT cast(1 as int) <=> cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) <=> CAST(1 AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS INT) <=> CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -5221,7 +5221,7 @@ true
 -- !query
 SELECT cast(1 as int) <=> cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0)) <=> CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS INT) <=> CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -5229,7 +5229,7 @@ true
 -- !query
 SELECT cast(1 as bigint) <=> cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) <=> CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS BIGINT) <=> CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -5237,7 +5237,7 @@ true
 -- !query
 SELECT cast(1 as bigint) <=> cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) <=> CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS BIGINT) <=> CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -5245,7 +5245,7 @@ true
 -- !query
 SELECT cast(1 as bigint) <=> cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) <=> CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS BIGINT) <=> CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -5253,7 +5253,7 @@ true
 -- !query
 SELECT cast(1 as bigint) <=> cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) <=> CAST(1 AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS BIGINT) <=> CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -5261,7 +5261,7 @@ true
 -- !query
 SELECT cast(1 as float) <=> cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) <=> CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS FLOAT) <=> CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -5269,7 +5269,7 @@ true
 -- !query
 SELECT cast(1 as float) <=> cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) <=> CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS FLOAT) <=> CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -5277,7 +5277,7 @@ true
 -- !query
 SELECT cast(1 as float) <=> cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) <=> CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS FLOAT) <=> CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -5285,7 +5285,7 @@ true
 -- !query
 SELECT cast(1 as float) <=> cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) <=> CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS FLOAT) <=> CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -5293,7 +5293,7 @@ true
 -- !query
 SELECT cast(1 as double) <=> cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) <=> CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) <=> CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -5301,7 +5301,7 @@ true
 -- !query
 SELECT cast(1 as double) <=> cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) <=> CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) <=> CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -5309,7 +5309,7 @@ true
 -- !query
 SELECT cast(1 as double) <=> cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) <=> CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) <=> CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -5317,7 +5317,7 @@ true
 -- !query
 SELECT cast(1 as double) <=> cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) <=> CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) <=> CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -5325,7 +5325,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) <=> cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) <=> CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <=> CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -5333,7 +5333,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) <=> cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) <=> CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <=> CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -5349,7 +5349,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) <=> cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) <=> CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <=> CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -5465,7 +5465,7 @@ cannot resolve '(CAST('2017-12-11 09:30:00' AS DATE) <=> CAST(1 AS DECIMAL(20,0)
 -- !query
 SELECT cast(1 as decimal(3, 0))  <=> cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(3,0)) <=> CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) <=> CAST(1 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -5473,7 +5473,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  <=> cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0)) <=> CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) <=> CAST(1 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -5481,7 +5481,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) <=> cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) <=> CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <=> CAST(1 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -5489,7 +5489,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) <=> cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) <=> CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) <=> CAST(1 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -5497,7 +5497,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  <=> cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0)) <=> CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) <=> CAST(1 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -5505,7 +5505,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  <=> cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(5,0)) <=> CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) <=> CAST(1 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -5513,7 +5513,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) <=> cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) <=> CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <=> CAST(1 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -5521,7 +5521,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) <=> cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) <=> CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) <=> CAST(1 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -5529,7 +5529,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  <=> cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) <=> CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) <=> CAST(1 AS INT)):boolean>
 -- !query output
 true
 
@@ -5537,7 +5537,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  <=> cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)) <=> CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) <=> CAST(1 AS INT)):boolean>
 -- !query output
 true
 
@@ -5545,7 +5545,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) <=> cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(10,0)) <=> CAST(CAST(1 AS INT) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <=> CAST(1 AS INT)):boolean>
 -- !query output
 true
 
@@ -5553,7 +5553,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) <=> cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) <=> CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) <=> CAST(1 AS INT)):boolean>
 -- !query output
 true
 
@@ -5561,7 +5561,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  <=> cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0)) <=> CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) <=> CAST(1 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -5569,7 +5569,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  <=> cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0)) <=> CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) <=> CAST(1 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -5577,7 +5577,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) <=> cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) <=> CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <=> CAST(1 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -5585,7 +5585,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) <=> cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(20,0)) <=> CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) <=> CAST(1 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -5593,7 +5593,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  <=> cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) <=> CAST(CAST(1 AS FLOAT) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) <=> CAST(1 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -5601,7 +5601,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  <=> cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) <=> CAST(CAST(1 AS FLOAT) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) <=> CAST(1 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -5609,7 +5609,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) <=> cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) <=> CAST(CAST(1 AS FLOAT) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <=> CAST(1 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -5617,7 +5617,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) <=> cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) <=> CAST(CAST(1 AS FLOAT) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) <=> CAST(1 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -5625,7 +5625,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  <=> cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) <=> CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) <=> CAST(1 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -5633,7 +5633,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  <=> cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) <=> CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) <=> CAST(1 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -5641,7 +5641,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) <=> cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) <=> CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <=> CAST(1 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -5649,7 +5649,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) <=> cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) <=> CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) <=> CAST(1 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -5657,7 +5657,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  <=> cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) <=> CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) <=> CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -5665,7 +5665,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  <=> cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)) <=> CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) <=> CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -5681,7 +5681,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) <=> cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) <=> CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) <=> CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -5689,7 +5689,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  <=> cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) <=> CAST(CAST(1 AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) <=> CAST(1 AS STRING)):boolean>
 -- !query output
 true
 
@@ -5697,7 +5697,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  <=> cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) <=> CAST(CAST(1 AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) <=> CAST(1 AS STRING)):boolean>
 -- !query output
 true
 
@@ -5705,7 +5705,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) <=> cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) <=> CAST(CAST(1 AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <=> CAST(1 AS STRING)):boolean>
 -- !query output
 true
 
@@ -5713,7 +5713,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) <=> cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) <=> CAST(CAST(1 AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) <=> CAST(1 AS STRING)):boolean>
 -- !query output
 true
 
@@ -5757,7 +5757,7 @@ cannot resolve '(CAST(1 AS DECIMAL(20,0)) <=> CAST('1' AS BINARY))' due to data
 -- !query
 SELECT cast(1 as decimal(3, 0))  <=> cast(1 as boolean) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(3,0)) <=> CAST(CAST(1 AS BOOLEAN) AS DECIMAL(3,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) <=> CAST(1 AS BOOLEAN)):boolean>
 -- !query output
 true
 
@@ -5765,7 +5765,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  <=> cast(1 as boolean) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(5,0)) <=> CAST(CAST(1 AS BOOLEAN) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) <=> CAST(1 AS BOOLEAN)):boolean>
 -- !query output
 true
 
@@ -5773,7 +5773,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) <=> cast(1 as boolean) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(10,0)) <=> CAST(CAST(1 AS BOOLEAN) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <=> CAST(1 AS BOOLEAN)):boolean>
 -- !query output
 true
 
@@ -5781,7 +5781,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) <=> cast(1 as boolean) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(20,0)) <=> CAST(CAST(1 AS BOOLEAN) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) <=> CAST(1 AS BOOLEAN)):boolean>
 -- !query output
 true
 
@@ -5861,7 +5861,7 @@ cannot resolve '(CAST(1 AS DECIMAL(20,0)) <=> CAST('2017-12-11 09:30:00' AS DATE
 -- !query
 SELECT cast(1 as tinyint) < cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) < CAST(1 AS DECIMAL(3,0))):boolean>
+struct<(CAST(1 AS TINYINT) < CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 false
 
@@ -5869,7 +5869,7 @@ false
 -- !query
 SELECT cast(1 as tinyint) < cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0)) < CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS TINYINT) < CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 false
 
@@ -5877,7 +5877,7 @@ false
 -- !query
 SELECT cast(1 as tinyint) < cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0)) < CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS TINYINT) < CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 false
 
@@ -5885,7 +5885,7 @@ false
 -- !query
 SELECT cast(1 as tinyint) < cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0)) < CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS TINYINT) < CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 false
 
@@ -5893,7 +5893,7 @@ false
 -- !query
 SELECT cast(1 as smallint) < cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0)) < CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS SMALLINT) < CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 false
 
@@ -5901,7 +5901,7 @@ false
 -- !query
 SELECT cast(1 as smallint) < cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) < CAST(1 AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS SMALLINT) < CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 false
 
@@ -5909,7 +5909,7 @@ false
 -- !query
 SELECT cast(1 as smallint) < cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0)) < CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS SMALLINT) < CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 false
 
@@ -5917,7 +5917,7 @@ false
 -- !query
 SELECT cast(1 as smallint) < cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0)) < CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS SMALLINT) < CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 false
 
@@ -5925,7 +5925,7 @@ false
 -- !query
 SELECT cast(1 as int) < cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) < CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS INT) < CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 false
 
@@ -5933,7 +5933,7 @@ false
 -- !query
 SELECT cast(1 as int) < cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) < CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS INT) < CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 false
 
@@ -5941,7 +5941,7 @@ false
 -- !query
 SELECT cast(1 as int) < cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) < CAST(1 AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS INT) < CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 false
 
@@ -5949,7 +5949,7 @@ false
 -- !query
 SELECT cast(1 as int) < cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0)) < CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS INT) < CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 false
 
@@ -5957,7 +5957,7 @@ false
 -- !query
 SELECT cast(1 as bigint) < cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) < CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS BIGINT) < CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 false
 
@@ -5965,7 +5965,7 @@ false
 -- !query
 SELECT cast(1 as bigint) < cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) < CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS BIGINT) < CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 false
 
@@ -5973,7 +5973,7 @@ false
 -- !query
 SELECT cast(1 as bigint) < cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) < CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS BIGINT) < CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 false
 
@@ -5981,7 +5981,7 @@ false
 -- !query
 SELECT cast(1 as bigint) < cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) < CAST(1 AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS BIGINT) < CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 false
 
@@ -5989,7 +5989,7 @@ false
 -- !query
 SELECT cast(1 as float) < cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) < CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS FLOAT) < CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 false
 
@@ -5997,7 +5997,7 @@ false
 -- !query
 SELECT cast(1 as float) < cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) < CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS FLOAT) < CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 false
 
@@ -6005,7 +6005,7 @@ false
 -- !query
 SELECT cast(1 as float) < cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) < CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS FLOAT) < CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 false
 
@@ -6013,7 +6013,7 @@ false
 -- !query
 SELECT cast(1 as float) < cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) < CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS FLOAT) < CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 false
 
@@ -6021,7 +6021,7 @@ false
 -- !query
 SELECT cast(1 as double) < cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) < CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) < CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 false
 
@@ -6029,7 +6029,7 @@ false
 -- !query
 SELECT cast(1 as double) < cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) < CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) < CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 false
 
@@ -6037,7 +6037,7 @@ false
 -- !query
 SELECT cast(1 as double) < cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) < CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) < CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 false
 
@@ -6045,7 +6045,7 @@ false
 -- !query
 SELECT cast(1 as double) < cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) < CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) < CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 false
 
@@ -6053,7 +6053,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) < cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) < CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) < CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 false
 
@@ -6061,7 +6061,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) < cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) < CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) < CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 false
 
@@ -6077,7 +6077,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) < cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) < CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) < CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 false
 
@@ -6193,7 +6193,7 @@ cannot resolve '(CAST('2017-12-11 09:30:00' AS DATE) < CAST(1 AS DECIMAL(20,0)))
 -- !query
 SELECT cast(1 as decimal(3, 0))  < cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(3,0)) < CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) < CAST(1 AS TINYINT)):boolean>
 -- !query output
 false
 
@@ -6201,7 +6201,7 @@ false
 -- !query
 SELECT cast(1 as decimal(5, 0))  < cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0)) < CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) < CAST(1 AS TINYINT)):boolean>
 -- !query output
 false
 
@@ -6209,7 +6209,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) < cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) < CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) < CAST(1 AS TINYINT)):boolean>
 -- !query output
 false
 
@@ -6217,7 +6217,7 @@ false
 -- !query
 SELECT cast(1 as decimal(20, 0)) < cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) < CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) < CAST(1 AS TINYINT)):boolean>
 -- !query output
 false
 
@@ -6225,7 +6225,7 @@ false
 -- !query
 SELECT cast(1 as decimal(3, 0))  < cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0)) < CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) < CAST(1 AS SMALLINT)):boolean>
 -- !query output
 false
 
@@ -6233,7 +6233,7 @@ false
 -- !query
 SELECT cast(1 as decimal(5, 0))  < cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(5,0)) < CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) < CAST(1 AS SMALLINT)):boolean>
 -- !query output
 false
 
@@ -6241,7 +6241,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) < cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) < CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) < CAST(1 AS SMALLINT)):boolean>
 -- !query output
 false
 
@@ -6249,7 +6249,7 @@ false
 -- !query
 SELECT cast(1 as decimal(20, 0)) < cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) < CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) < CAST(1 AS SMALLINT)):boolean>
 -- !query output
 false
 
@@ -6257,7 +6257,7 @@ false
 -- !query
 SELECT cast(1 as decimal(3, 0))  < cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) < CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) < CAST(1 AS INT)):boolean>
 -- !query output
 false
 
@@ -6265,7 +6265,7 @@ false
 -- !query
 SELECT cast(1 as decimal(5, 0))  < cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)) < CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) < CAST(1 AS INT)):boolean>
 -- !query output
 false
 
@@ -6273,7 +6273,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) < cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(10,0)) < CAST(CAST(1 AS INT) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) < CAST(1 AS INT)):boolean>
 -- !query output
 false
 
@@ -6281,7 +6281,7 @@ false
 -- !query
 SELECT cast(1 as decimal(20, 0)) < cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) < CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) < CAST(1 AS INT)):boolean>
 -- !query output
 false
 
@@ -6289,7 +6289,7 @@ false
 -- !query
 SELECT cast(1 as decimal(3, 0))  < cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0)) < CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) < CAST(1 AS BIGINT)):boolean>
 -- !query output
 false
 
@@ -6297,7 +6297,7 @@ false
 -- !query
 SELECT cast(1 as decimal(5, 0))  < cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0)) < CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) < CAST(1 AS BIGINT)):boolean>
 -- !query output
 false
 
@@ -6305,7 +6305,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) < cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) < CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) < CAST(1 AS BIGINT)):boolean>
 -- !query output
 false
 
@@ -6313,7 +6313,7 @@ false
 -- !query
 SELECT cast(1 as decimal(20, 0)) < cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(20,0)) < CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) < CAST(1 AS BIGINT)):boolean>
 -- !query output
 false
 
@@ -6321,7 +6321,7 @@ false
 -- !query
 SELECT cast(1 as decimal(3, 0))  < cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) < CAST(CAST(1 AS FLOAT) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) < CAST(1 AS FLOAT)):boolean>
 -- !query output
 false
 
@@ -6329,7 +6329,7 @@ false
 -- !query
 SELECT cast(1 as decimal(5, 0))  < cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) < CAST(CAST(1 AS FLOAT) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) < CAST(1 AS FLOAT)):boolean>
 -- !query output
 false
 
@@ -6337,7 +6337,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) < cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) < CAST(CAST(1 AS FLOAT) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) < CAST(1 AS FLOAT)):boolean>
 -- !query output
 false
 
@@ -6345,7 +6345,7 @@ false
 -- !query
 SELECT cast(1 as decimal(20, 0)) < cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) < CAST(CAST(1 AS FLOAT) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) < CAST(1 AS FLOAT)):boolean>
 -- !query output
 false
 
@@ -6353,7 +6353,7 @@ false
 -- !query
 SELECT cast(1 as decimal(3, 0))  < cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) < CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) < CAST(1 AS DOUBLE)):boolean>
 -- !query output
 false
 
@@ -6361,7 +6361,7 @@ false
 -- !query
 SELECT cast(1 as decimal(5, 0))  < cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) < CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) < CAST(1 AS DOUBLE)):boolean>
 -- !query output
 false
 
@@ -6369,7 +6369,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) < cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) < CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) < CAST(1 AS DOUBLE)):boolean>
 -- !query output
 false
 
@@ -6377,7 +6377,7 @@ false
 -- !query
 SELECT cast(1 as decimal(20, 0)) < cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) < CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) < CAST(1 AS DOUBLE)):boolean>
 -- !query output
 false
 
@@ -6385,7 +6385,7 @@ false
 -- !query
 SELECT cast(1 as decimal(3, 0))  < cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) < CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) < CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 false
 
@@ -6393,7 +6393,7 @@ false
 -- !query
 SELECT cast(1 as decimal(5, 0))  < cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)) < CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) < CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 false
 
@@ -6409,7 +6409,7 @@ false
 -- !query
 SELECT cast(1 as decimal(20, 0)) < cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) < CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) < CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 false
 
@@ -6417,7 +6417,7 @@ false
 -- !query
 SELECT cast(1 as decimal(3, 0))  < cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) < CAST(CAST(1 AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) < CAST(1 AS STRING)):boolean>
 -- !query output
 false
 
@@ -6425,7 +6425,7 @@ false
 -- !query
 SELECT cast(1 as decimal(5, 0))  < cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) < CAST(CAST(1 AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) < CAST(1 AS STRING)):boolean>
 -- !query output
 false
 
@@ -6433,7 +6433,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) < cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) < CAST(CAST(1 AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) < CAST(1 AS STRING)):boolean>
 -- !query output
 false
 
@@ -6441,7 +6441,7 @@ false
 -- !query
 SELECT cast(1 as decimal(20, 0)) < cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) < CAST(CAST(1 AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) < CAST(1 AS STRING)):boolean>
 -- !query output
 false
 
@@ -6593,7 +6593,7 @@ cannot resolve '(CAST(1 AS DECIMAL(20,0)) < CAST('2017-12-11 09:30:00' AS DATE))
 -- !query
 SELECT cast(1 as tinyint) <= cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) <= CAST(1 AS DECIMAL(3,0))):boolean>
+struct<(CAST(1 AS TINYINT) <= CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -6601,7 +6601,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) <= cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0)) <= CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS TINYINT) <= CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -6609,7 +6609,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) <= cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0)) <= CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS TINYINT) <= CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -6617,7 +6617,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) <= cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0)) <= CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS TINYINT) <= CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -6625,7 +6625,7 @@ true
 -- !query
 SELECT cast(1 as smallint) <= cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0)) <= CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS SMALLINT) <= CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -6633,7 +6633,7 @@ true
 -- !query
 SELECT cast(1 as smallint) <= cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) <= CAST(1 AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS SMALLINT) <= CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -6641,7 +6641,7 @@ true
 -- !query
 SELECT cast(1 as smallint) <= cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0)) <= CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS SMALLINT) <= CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -6649,7 +6649,7 @@ true
 -- !query
 SELECT cast(1 as smallint) <= cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0)) <= CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS SMALLINT) <= CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -6657,7 +6657,7 @@ true
 -- !query
 SELECT cast(1 as int) <= cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) <= CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS INT) <= CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -6665,7 +6665,7 @@ true
 -- !query
 SELECT cast(1 as int) <= cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) <= CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS INT) <= CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -6673,7 +6673,7 @@ true
 -- !query
 SELECT cast(1 as int) <= cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) <= CAST(1 AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS INT) <= CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -6681,7 +6681,7 @@ true
 -- !query
 SELECT cast(1 as int) <= cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0)) <= CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS INT) <= CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -6689,7 +6689,7 @@ true
 -- !query
 SELECT cast(1 as bigint) <= cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) <= CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS BIGINT) <= CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -6697,7 +6697,7 @@ true
 -- !query
 SELECT cast(1 as bigint) <= cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) <= CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS BIGINT) <= CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -6705,7 +6705,7 @@ true
 -- !query
 SELECT cast(1 as bigint) <= cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) <= CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS BIGINT) <= CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -6713,7 +6713,7 @@ true
 -- !query
 SELECT cast(1 as bigint) <= cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) <= CAST(1 AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS BIGINT) <= CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -6721,7 +6721,7 @@ true
 -- !query
 SELECT cast(1 as float) <= cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) <= CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS FLOAT) <= CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -6729,7 +6729,7 @@ true
 -- !query
 SELECT cast(1 as float) <= cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) <= CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS FLOAT) <= CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -6737,7 +6737,7 @@ true
 -- !query
 SELECT cast(1 as float) <= cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) <= CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS FLOAT) <= CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -6745,7 +6745,7 @@ true
 -- !query
 SELECT cast(1 as float) <= cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) <= CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS FLOAT) <= CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -6753,7 +6753,7 @@ true
 -- !query
 SELECT cast(1 as double) <= cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) <= CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) <= CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -6761,7 +6761,7 @@ true
 -- !query
 SELECT cast(1 as double) <= cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) <= CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) <= CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -6769,7 +6769,7 @@ true
 -- !query
 SELECT cast(1 as double) <= cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) <= CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) <= CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -6777,7 +6777,7 @@ true
 -- !query
 SELECT cast(1 as double) <= cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) <= CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) <= CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -6785,7 +6785,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) <= cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) <= CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <= CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -6793,7 +6793,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) <= cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) <= CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <= CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -6809,7 +6809,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) <= cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) <= CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <= CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -6925,7 +6925,7 @@ cannot resolve '(CAST('2017-12-11 09:30:00' AS DATE) <= CAST(1 AS DECIMAL(20,0))
 -- !query
 SELECT cast(1 as decimal(3, 0))  <= cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(3,0)) <= CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) <= CAST(1 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -6933,7 +6933,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  <= cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0)) <= CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) <= CAST(1 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -6941,7 +6941,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) <= cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) <= CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <= CAST(1 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -6949,7 +6949,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) <= cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) <= CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) <= CAST(1 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -6957,7 +6957,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  <= cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0)) <= CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) <= CAST(1 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -6965,7 +6965,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  <= cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(5,0)) <= CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) <= CAST(1 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -6973,7 +6973,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) <= cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) <= CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <= CAST(1 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -6981,7 +6981,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) <= cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) <= CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) <= CAST(1 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -6989,7 +6989,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  <= cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) <= CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) <= CAST(1 AS INT)):boolean>
 -- !query output
 true
 
@@ -6997,7 +6997,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  <= cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)) <= CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) <= CAST(1 AS INT)):boolean>
 -- !query output
 true
 
@@ -7005,7 +7005,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) <= cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(10,0)) <= CAST(CAST(1 AS INT) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <= CAST(1 AS INT)):boolean>
 -- !query output
 true
 
@@ -7013,7 +7013,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) <= cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) <= CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) <= CAST(1 AS INT)):boolean>
 -- !query output
 true
 
@@ -7021,7 +7021,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  <= cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0)) <= CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) <= CAST(1 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -7029,7 +7029,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  <= cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0)) <= CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) <= CAST(1 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -7037,7 +7037,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) <= cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) <= CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <= CAST(1 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -7045,7 +7045,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) <= cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(20,0)) <= CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) <= CAST(1 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -7053,7 +7053,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  <= cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) <= CAST(CAST(1 AS FLOAT) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) <= CAST(1 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -7061,7 +7061,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  <= cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) <= CAST(CAST(1 AS FLOAT) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) <= CAST(1 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -7069,7 +7069,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) <= cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) <= CAST(CAST(1 AS FLOAT) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <= CAST(1 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -7077,7 +7077,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) <= cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) <= CAST(CAST(1 AS FLOAT) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) <= CAST(1 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -7085,7 +7085,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  <= cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) <= CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) <= CAST(1 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -7093,7 +7093,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  <= cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) <= CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) <= CAST(1 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -7101,7 +7101,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) <= cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) <= CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <= CAST(1 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -7109,7 +7109,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) <= cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) <= CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) <= CAST(1 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -7117,7 +7117,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  <= cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) <= CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) <= CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -7125,7 +7125,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  <= cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)) <= CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) <= CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -7141,7 +7141,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) <= cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) <= CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) <= CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -7149,7 +7149,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  <= cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) <= CAST(CAST(1 AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) <= CAST(1 AS STRING)):boolean>
 -- !query output
 true
 
@@ -7157,7 +7157,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  <= cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) <= CAST(CAST(1 AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) <= CAST(1 AS STRING)):boolean>
 -- !query output
 true
 
@@ -7165,7 +7165,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) <= cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) <= CAST(CAST(1 AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <= CAST(1 AS STRING)):boolean>
 -- !query output
 true
 
@@ -7173,7 +7173,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) <= cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) <= CAST(CAST(1 AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) <= CAST(1 AS STRING)):boolean>
 -- !query output
 true
 
@@ -7325,7 +7325,7 @@ cannot resolve '(CAST(1 AS DECIMAL(20,0)) <= CAST('2017-12-11 09:30:00' AS DATE)
 -- !query
 SELECT cast(1 as tinyint) > cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) > CAST(1 AS DECIMAL(3,0))):boolean>
+struct<(CAST(1 AS TINYINT) > CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 false
 
@@ -7333,7 +7333,7 @@ false
 -- !query
 SELECT cast(1 as tinyint) > cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0)) > CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS TINYINT) > CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 false
 
@@ -7341,7 +7341,7 @@ false
 -- !query
 SELECT cast(1 as tinyint) > cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0)) > CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS TINYINT) > CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 false
 
@@ -7349,7 +7349,7 @@ false
 -- !query
 SELECT cast(1 as tinyint) > cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0)) > CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS TINYINT) > CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 false
 
@@ -7357,7 +7357,7 @@ false
 -- !query
 SELECT cast(1 as smallint) > cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0)) > CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS SMALLINT) > CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 false
 
@@ -7365,7 +7365,7 @@ false
 -- !query
 SELECT cast(1 as smallint) > cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) > CAST(1 AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS SMALLINT) > CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 false
 
@@ -7373,7 +7373,7 @@ false
 -- !query
 SELECT cast(1 as smallint) > cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0)) > CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS SMALLINT) > CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 false
 
@@ -7381,7 +7381,7 @@ false
 -- !query
 SELECT cast(1 as smallint) > cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0)) > CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS SMALLINT) > CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 false
 
@@ -7389,7 +7389,7 @@ false
 -- !query
 SELECT cast(1 as int) > cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) > CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS INT) > CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 false
 
@@ -7397,7 +7397,7 @@ false
 -- !query
 SELECT cast(1 as int) > cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) > CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS INT) > CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 false
 
@@ -7405,7 +7405,7 @@ false
 -- !query
 SELECT cast(1 as int) > cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) > CAST(1 AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS INT) > CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 false
 
@@ -7413,7 +7413,7 @@ false
 -- !query
 SELECT cast(1 as int) > cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0)) > CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS INT) > CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 false
 
@@ -7421,7 +7421,7 @@ false
 -- !query
 SELECT cast(1 as bigint) > cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) > CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS BIGINT) > CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 false
 
@@ -7429,7 +7429,7 @@ false
 -- !query
 SELECT cast(1 as bigint) > cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) > CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS BIGINT) > CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 false
 
@@ -7437,7 +7437,7 @@ false
 -- !query
 SELECT cast(1 as bigint) > cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) > CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS BIGINT) > CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 false
 
@@ -7445,7 +7445,7 @@ false
 -- !query
 SELECT cast(1 as bigint) > cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) > CAST(1 AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS BIGINT) > CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 false
 
@@ -7453,7 +7453,7 @@ false
 -- !query
 SELECT cast(1 as float) > cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) > CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS FLOAT) > CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 false
 
@@ -7461,7 +7461,7 @@ false
 -- !query
 SELECT cast(1 as float) > cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) > CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS FLOAT) > CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 false
 
@@ -7469,7 +7469,7 @@ false
 -- !query
 SELECT cast(1 as float) > cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) > CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS FLOAT) > CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 false
 
@@ -7477,7 +7477,7 @@ false
 -- !query
 SELECT cast(1 as float) > cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) > CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS FLOAT) > CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 false
 
@@ -7485,7 +7485,7 @@ false
 -- !query
 SELECT cast(1 as double) > cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) > CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) > CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 false
 
@@ -7493,7 +7493,7 @@ false
 -- !query
 SELECT cast(1 as double) > cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) > CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) > CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 false
 
@@ -7501,7 +7501,7 @@ false
 -- !query
 SELECT cast(1 as double) > cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) > CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) > CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 false
 
@@ -7509,7 +7509,7 @@ false
 -- !query
 SELECT cast(1 as double) > cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) > CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) > CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 false
 
@@ -7517,7 +7517,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) > cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) > CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) > CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 false
 
@@ -7525,7 +7525,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) > cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) > CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) > CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 false
 
@@ -7541,7 +7541,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) > cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) > CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) > CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 false
 
@@ -7657,7 +7657,7 @@ cannot resolve '(CAST('2017-12-11 09:30:00' AS DATE) > CAST(1 AS DECIMAL(20,0)))
 -- !query
 SELECT cast(1 as decimal(3, 0))  > cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(3,0)) > CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) > CAST(1 AS TINYINT)):boolean>
 -- !query output
 false
 
@@ -7665,7 +7665,7 @@ false
 -- !query
 SELECT cast(1 as decimal(5, 0))  > cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0)) > CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) > CAST(1 AS TINYINT)):boolean>
 -- !query output
 false
 
@@ -7673,7 +7673,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) > cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) > CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) > CAST(1 AS TINYINT)):boolean>
 -- !query output
 false
 
@@ -7681,7 +7681,7 @@ false
 -- !query
 SELECT cast(1 as decimal(20, 0)) > cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) > CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) > CAST(1 AS TINYINT)):boolean>
 -- !query output
 false
 
@@ -7689,7 +7689,7 @@ false
 -- !query
 SELECT cast(1 as decimal(3, 0))  > cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0)) > CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) > CAST(1 AS SMALLINT)):boolean>
 -- !query output
 false
 
@@ -7697,7 +7697,7 @@ false
 -- !query
 SELECT cast(1 as decimal(5, 0))  > cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(5,0)) > CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) > CAST(1 AS SMALLINT)):boolean>
 -- !query output
 false
 
@@ -7705,7 +7705,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) > cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) > CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) > CAST(1 AS SMALLINT)):boolean>
 -- !query output
 false
 
@@ -7713,7 +7713,7 @@ false
 -- !query
 SELECT cast(1 as decimal(20, 0)) > cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) > CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) > CAST(1 AS SMALLINT)):boolean>
 -- !query output
 false
 
@@ -7721,7 +7721,7 @@ false
 -- !query
 SELECT cast(1 as decimal(3, 0))  > cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) > CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) > CAST(1 AS INT)):boolean>
 -- !query output
 false
 
@@ -7729,7 +7729,7 @@ false
 -- !query
 SELECT cast(1 as decimal(5, 0))  > cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)) > CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) > CAST(1 AS INT)):boolean>
 -- !query output
 false
 
@@ -7737,7 +7737,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) > cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(10,0)) > CAST(CAST(1 AS INT) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) > CAST(1 AS INT)):boolean>
 -- !query output
 false
 
@@ -7745,7 +7745,7 @@ false
 -- !query
 SELECT cast(1 as decimal(20, 0)) > cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) > CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) > CAST(1 AS INT)):boolean>
 -- !query output
 false
 
@@ -7753,7 +7753,7 @@ false
 -- !query
 SELECT cast(1 as decimal(3, 0))  > cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0)) > CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) > CAST(1 AS BIGINT)):boolean>
 -- !query output
 false
 
@@ -7761,7 +7761,7 @@ false
 -- !query
 SELECT cast(1 as decimal(5, 0))  > cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0)) > CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) > CAST(1 AS BIGINT)):boolean>
 -- !query output
 false
 
@@ -7769,7 +7769,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) > cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) > CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) > CAST(1 AS BIGINT)):boolean>
 -- !query output
 false
 
@@ -7777,7 +7777,7 @@ false
 -- !query
 SELECT cast(1 as decimal(20, 0)) > cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(20,0)) > CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) > CAST(1 AS BIGINT)):boolean>
 -- !query output
 false
 
@@ -7785,7 +7785,7 @@ false
 -- !query
 SELECT cast(1 as decimal(3, 0))  > cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) > CAST(CAST(1 AS FLOAT) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) > CAST(1 AS FLOAT)):boolean>
 -- !query output
 false
 
@@ -7793,7 +7793,7 @@ false
 -- !query
 SELECT cast(1 as decimal(5, 0))  > cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) > CAST(CAST(1 AS FLOAT) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) > CAST(1 AS FLOAT)):boolean>
 -- !query output
 false
 
@@ -7801,7 +7801,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) > cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) > CAST(CAST(1 AS FLOAT) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) > CAST(1 AS FLOAT)):boolean>
 -- !query output
 false
 
@@ -7809,7 +7809,7 @@ false
 -- !query
 SELECT cast(1 as decimal(20, 0)) > cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) > CAST(CAST(1 AS FLOAT) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) > CAST(1 AS FLOAT)):boolean>
 -- !query output
 false
 
@@ -7817,7 +7817,7 @@ false
 -- !query
 SELECT cast(1 as decimal(3, 0))  > cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) > CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) > CAST(1 AS DOUBLE)):boolean>
 -- !query output
 false
 
@@ -7825,7 +7825,7 @@ false
 -- !query
 SELECT cast(1 as decimal(5, 0))  > cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) > CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) > CAST(1 AS DOUBLE)):boolean>
 -- !query output
 false
 
@@ -7833,7 +7833,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) > cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) > CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) > CAST(1 AS DOUBLE)):boolean>
 -- !query output
 false
 
@@ -7841,7 +7841,7 @@ false
 -- !query
 SELECT cast(1 as decimal(20, 0)) > cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) > CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) > CAST(1 AS DOUBLE)):boolean>
 -- !query output
 false
 
@@ -7849,7 +7849,7 @@ false
 -- !query
 SELECT cast(1 as decimal(3, 0))  > cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) > CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) > CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 false
 
@@ -7857,7 +7857,7 @@ false
 -- !query
 SELECT cast(1 as decimal(5, 0))  > cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)) > CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) > CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 false
 
@@ -7873,7 +7873,7 @@ false
 -- !query
 SELECT cast(1 as decimal(20, 0)) > cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) > CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) > CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 false
 
@@ -7881,7 +7881,7 @@ false
 -- !query
 SELECT cast(1 as decimal(3, 0))  > cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) > CAST(CAST(1 AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) > CAST(1 AS STRING)):boolean>
 -- !query output
 false
 
@@ -7889,7 +7889,7 @@ false
 -- !query
 SELECT cast(1 as decimal(5, 0))  > cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) > CAST(CAST(1 AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) > CAST(1 AS STRING)):boolean>
 -- !query output
 false
 
@@ -7897,7 +7897,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) > cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) > CAST(CAST(1 AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) > CAST(1 AS STRING)):boolean>
 -- !query output
 false
 
@@ -7905,7 +7905,7 @@ false
 -- !query
 SELECT cast(1 as decimal(20, 0)) > cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) > CAST(CAST(1 AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) > CAST(1 AS STRING)):boolean>
 -- !query output
 false
 
@@ -8057,7 +8057,7 @@ cannot resolve '(CAST(1 AS DECIMAL(20,0)) > CAST('2017-12-11 09:30:00' AS DATE))
 -- !query
 SELECT cast(1 as tinyint) >= cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) >= CAST(1 AS DECIMAL(3,0))):boolean>
+struct<(CAST(1 AS TINYINT) >= CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -8065,7 +8065,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) >= cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0)) >= CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS TINYINT) >= CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -8073,7 +8073,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) >= cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0)) >= CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS TINYINT) >= CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -8081,7 +8081,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) >= cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0)) >= CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS TINYINT) >= CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -8089,7 +8089,7 @@ true
 -- !query
 SELECT cast(1 as smallint) >= cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0)) >= CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS SMALLINT) >= CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -8097,7 +8097,7 @@ true
 -- !query
 SELECT cast(1 as smallint) >= cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) >= CAST(1 AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS SMALLINT) >= CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -8105,7 +8105,7 @@ true
 -- !query
 SELECT cast(1 as smallint) >= cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0)) >= CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS SMALLINT) >= CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -8113,7 +8113,7 @@ true
 -- !query
 SELECT cast(1 as smallint) >= cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0)) >= CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS SMALLINT) >= CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -8121,7 +8121,7 @@ true
 -- !query
 SELECT cast(1 as int) >= cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) >= CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS INT) >= CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -8129,7 +8129,7 @@ true
 -- !query
 SELECT cast(1 as int) >= cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) >= CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS INT) >= CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -8137,7 +8137,7 @@ true
 -- !query
 SELECT cast(1 as int) >= cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) >= CAST(1 AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS INT) >= CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -8145,7 +8145,7 @@ true
 -- !query
 SELECT cast(1 as int) >= cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0)) >= CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS INT) >= CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -8153,7 +8153,7 @@ true
 -- !query
 SELECT cast(1 as bigint) >= cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) >= CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS BIGINT) >= CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -8161,7 +8161,7 @@ true
 -- !query
 SELECT cast(1 as bigint) >= cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) >= CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS BIGINT) >= CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -8169,7 +8169,7 @@ true
 -- !query
 SELECT cast(1 as bigint) >= cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) >= CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS BIGINT) >= CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -8177,7 +8177,7 @@ true
 -- !query
 SELECT cast(1 as bigint) >= cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) >= CAST(1 AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS BIGINT) >= CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -8185,7 +8185,7 @@ true
 -- !query
 SELECT cast(1 as float) >= cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) >= CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS FLOAT) >= CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -8193,7 +8193,7 @@ true
 -- !query
 SELECT cast(1 as float) >= cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) >= CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS FLOAT) >= CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -8201,7 +8201,7 @@ true
 -- !query
 SELECT cast(1 as float) >= cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) >= CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS FLOAT) >= CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -8209,7 +8209,7 @@ true
 -- !query
 SELECT cast(1 as float) >= cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) >= CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS FLOAT) >= CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -8217,7 +8217,7 @@ true
 -- !query
 SELECT cast(1 as double) >= cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) >= CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) >= CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -8225,7 +8225,7 @@ true
 -- !query
 SELECT cast(1 as double) >= cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) >= CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) >= CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -8233,7 +8233,7 @@ true
 -- !query
 SELECT cast(1 as double) >= cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) >= CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) >= CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -8241,7 +8241,7 @@ true
 -- !query
 SELECT cast(1 as double) >= cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) >= CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) >= CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -8249,7 +8249,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) >= cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) >= CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) >= CAST(1 AS DECIMAL(3,0))):boolean>
 -- !query output
 true
 
@@ -8257,7 +8257,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) >= cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) >= CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) >= CAST(1 AS DECIMAL(5,0))):boolean>
 -- !query output
 true
 
@@ -8273,7 +8273,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) >= cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) >= CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) >= CAST(1 AS DECIMAL(20,0))):boolean>
 -- !query output
 true
 
@@ -8389,7 +8389,7 @@ cannot resolve '(CAST('2017-12-11 09:30:00' AS DATE) >= CAST(1 AS DECIMAL(20,0))
 -- !query
 SELECT cast(1 as decimal(3, 0))  >= cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(3,0)) >= CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) >= CAST(1 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -8397,7 +8397,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  >= cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0)) >= CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) >= CAST(1 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -8405,7 +8405,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) >= cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) >= CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) >= CAST(1 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -8413,7 +8413,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) >= cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) >= CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) >= CAST(1 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -8421,7 +8421,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  >= cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0)) >= CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) >= CAST(1 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -8429,7 +8429,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  >= cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(5,0)) >= CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) >= CAST(1 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -8437,7 +8437,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) >= cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) >= CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) >= CAST(1 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -8445,7 +8445,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) >= cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) >= CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) >= CAST(1 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -8453,7 +8453,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  >= cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) >= CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) >= CAST(1 AS INT)):boolean>
 -- !query output
 true
 
@@ -8461,7 +8461,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  >= cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)) >= CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) >= CAST(1 AS INT)):boolean>
 -- !query output
 true
 
@@ -8469,7 +8469,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) >= cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(10,0)) >= CAST(CAST(1 AS INT) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) >= CAST(1 AS INT)):boolean>
 -- !query output
 true
 
@@ -8477,7 +8477,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) >= cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) >= CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) >= CAST(1 AS INT)):boolean>
 -- !query output
 true
 
@@ -8485,7 +8485,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  >= cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0)) >= CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) >= CAST(1 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -8493,7 +8493,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  >= cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0)) >= CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) >= CAST(1 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -8501,7 +8501,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) >= cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) >= CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) >= CAST(1 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -8509,7 +8509,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) >= cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(20,0)) >= CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) >= CAST(1 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -8517,7 +8517,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  >= cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) >= CAST(CAST(1 AS FLOAT) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) >= CAST(1 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -8525,7 +8525,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  >= cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) >= CAST(CAST(1 AS FLOAT) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) >= CAST(1 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -8533,7 +8533,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) >= cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) >= CAST(CAST(1 AS FLOAT) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) >= CAST(1 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -8541,7 +8541,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) >= cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) >= CAST(CAST(1 AS FLOAT) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) >= CAST(1 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -8549,7 +8549,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  >= cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) >= CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) >= CAST(1 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -8557,7 +8557,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  >= cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) >= CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) >= CAST(1 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -8565,7 +8565,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) >= cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) >= CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) >= CAST(1 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -8573,7 +8573,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) >= cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) >= CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) >= CAST(1 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -8581,7 +8581,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  >= cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) >= CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) >= CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -8589,7 +8589,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  >= cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)) >= CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) >= CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -8605,7 +8605,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) >= cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) >= CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0))):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) >= CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -8613,7 +8613,7 @@ true
 -- !query
 SELECT cast(1 as decimal(3, 0))  >= cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) >= CAST(CAST(1 AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(3,0)) >= CAST(1 AS STRING)):boolean>
 -- !query output
 true
 
@@ -8621,7 +8621,7 @@ true
 -- !query
 SELECT cast(1 as decimal(5, 0))  >= cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) >= CAST(CAST(1 AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(5,0)) >= CAST(1 AS STRING)):boolean>
 -- !query output
 true
 
@@ -8629,7 +8629,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) >= cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) >= CAST(CAST(1 AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) >= CAST(1 AS STRING)):boolean>
 -- !query output
 true
 
@@ -8637,7 +8637,7 @@ true
 -- !query
 SELECT cast(1 as decimal(20, 0)) >= cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) >= CAST(CAST(1 AS STRING) AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(20,0)) >= CAST(1 AS STRING)):boolean>
 -- !query output
 true
 
@@ -8789,7 +8789,7 @@ cannot resolve '(CAST(1 AS DECIMAL(20,0)) >= CAST('2017-12-11 09:30:00' AS DATE)
 -- !query
 SELECT cast(1 as tinyint) <> cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) = CAST(1 AS DECIMAL(3,0)))):boolean>
+struct<(NOT (CAST(1 AS TINYINT) = CAST(1 AS DECIMAL(3,0)))):boolean>
 -- !query output
 false
 
@@ -8797,7 +8797,7 @@ false
 -- !query
 SELECT cast(1 as tinyint) <> cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0)) = CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0)))):boolean>
+struct<(NOT (CAST(1 AS TINYINT) = CAST(1 AS DECIMAL(5,0)))):boolean>
 -- !query output
 false
 
@@ -8805,7 +8805,7 @@ false
 -- !query
 SELECT cast(1 as tinyint) <> cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0)) = CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)))):boolean>
+struct<(NOT (CAST(1 AS TINYINT) = CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 false
 
@@ -8813,7 +8813,7 @@ false
 -- !query
 SELECT cast(1 as tinyint) <> cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0)) = CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)))):boolean>
+struct<(NOT (CAST(1 AS TINYINT) = CAST(1 AS DECIMAL(20,0)))):boolean>
 -- !query output
 false
 
@@ -8821,7 +8821,7 @@ false
 -- !query
 SELECT cast(1 as smallint) <> cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0)) = CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0)))):boolean>
+struct<(NOT (CAST(1 AS SMALLINT) = CAST(1 AS DECIMAL(3,0)))):boolean>
 -- !query output
 false
 
@@ -8829,7 +8829,7 @@ false
 -- !query
 SELECT cast(1 as smallint) <> cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) = CAST(1 AS DECIMAL(5,0)))):boolean>
+struct<(NOT (CAST(1 AS SMALLINT) = CAST(1 AS DECIMAL(5,0)))):boolean>
 -- !query output
 false
 
@@ -8837,7 +8837,7 @@ false
 -- !query
 SELECT cast(1 as smallint) <> cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0)) = CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)))):boolean>
+struct<(NOT (CAST(1 AS SMALLINT) = CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 false
 
@@ -8845,7 +8845,7 @@ false
 -- !query
 SELECT cast(1 as smallint) <> cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0)) = CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)))):boolean>
+struct<(NOT (CAST(1 AS SMALLINT) = CAST(1 AS DECIMAL(20,0)))):boolean>
 -- !query output
 false
 
@@ -8853,7 +8853,7 @@ false
 -- !query
 SELECT cast(1 as int) <> cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) = CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)))):boolean>
+struct<(NOT (CAST(1 AS INT) = CAST(1 AS DECIMAL(3,0)))):boolean>
 -- !query output
 false
 
@@ -8861,7 +8861,7 @@ false
 -- !query
 SELECT cast(1 as int) <> cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)) = CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)))):boolean>
+struct<(NOT (CAST(1 AS INT) = CAST(1 AS DECIMAL(5,0)))):boolean>
 -- !query output
 false
 
@@ -8869,7 +8869,7 @@ false
 -- !query
 SELECT cast(1 as int) <> cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS INT) AS DECIMAL(10,0)) = CAST(1 AS DECIMAL(10,0)))):boolean>
+struct<(NOT (CAST(1 AS INT) = CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 false
 
@@ -8877,7 +8877,7 @@ false
 -- !query
 SELECT cast(1 as int) <> cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0)) = CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)))):boolean>
+struct<(NOT (CAST(1 AS INT) = CAST(1 AS DECIMAL(20,0)))):boolean>
 -- !query output
 false
 
@@ -8885,7 +8885,7 @@ false
 -- !query
 SELECT cast(1 as bigint) <> cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) = CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0)))):boolean>
+struct<(NOT (CAST(1 AS BIGINT) = CAST(1 AS DECIMAL(3,0)))):boolean>
 -- !query output
 false
 
@@ -8893,7 +8893,7 @@ false
 -- !query
 SELECT cast(1 as bigint) <> cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) = CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0)))):boolean>
+struct<(NOT (CAST(1 AS BIGINT) = CAST(1 AS DECIMAL(5,0)))):boolean>
 -- !query output
 false
 
@@ -8901,7 +8901,7 @@ false
 -- !query
 SELECT cast(1 as bigint) <> cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) = CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)))):boolean>
+struct<(NOT (CAST(1 AS BIGINT) = CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 false
 
@@ -8909,7 +8909,7 @@ false
 -- !query
 SELECT cast(1 as bigint) <> cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) = CAST(1 AS DECIMAL(20,0)))):boolean>
+struct<(NOT (CAST(1 AS BIGINT) = CAST(1 AS DECIMAL(20,0)))):boolean>
 -- !query output
 false
 
@@ -8917,7 +8917,7 @@ false
 -- !query
 SELECT cast(1 as float) <> cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS FLOAT) AS DOUBLE) = CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS FLOAT) = CAST(1 AS DECIMAL(3,0)))):boolean>
 -- !query output
 false
 
@@ -8925,7 +8925,7 @@ false
 -- !query
 SELECT cast(1 as float) <> cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS FLOAT) AS DOUBLE) = CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS FLOAT) = CAST(1 AS DECIMAL(5,0)))):boolean>
 -- !query output
 false
 
@@ -8933,7 +8933,7 @@ false
 -- !query
 SELECT cast(1 as float) <> cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS FLOAT) AS DOUBLE) = CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS FLOAT) = CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 false
 
@@ -8941,7 +8941,7 @@ false
 -- !query
 SELECT cast(1 as float) <> cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS FLOAT) AS DOUBLE) = CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS FLOAT) = CAST(1 AS DECIMAL(20,0)))):boolean>
 -- !query output
 false
 
@@ -8949,7 +8949,7 @@ false
 -- !query
 SELECT cast(1 as double) <> cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS DOUBLE) = CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS DOUBLE) = CAST(1 AS DECIMAL(3,0)))):boolean>
 -- !query output
 false
 
@@ -8957,7 +8957,7 @@ false
 -- !query
 SELECT cast(1 as double) <> cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS DOUBLE) = CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS DOUBLE) = CAST(1 AS DECIMAL(5,0)))):boolean>
 -- !query output
 false
 
@@ -8965,7 +8965,7 @@ false
 -- !query
 SELECT cast(1 as double) <> cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS DOUBLE) = CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS DOUBLE) = CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 false
 
@@ -8973,7 +8973,7 @@ false
 -- !query
 SELECT cast(1 as double) <> cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS DOUBLE) = CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS DOUBLE) = CAST(1 AS DECIMAL(20,0)))):boolean>
 -- !query output
 false
 
@@ -8981,7 +8981,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) <> cast(1 as decimal(3, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) = CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(10,0)) = CAST(1 AS DECIMAL(3,0)))):boolean>
 -- !query output
 false
 
@@ -8989,7 +8989,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) <> cast(1 as decimal(5, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) = CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(10,0)) = CAST(1 AS DECIMAL(5,0)))):boolean>
 -- !query output
 false
 
@@ -9005,7 +9005,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) <> cast(1 as decimal(20, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) = CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(10,0)) = CAST(1 AS DECIMAL(20,0)))):boolean>
 -- !query output
 false
 
@@ -9121,7 +9121,7 @@ cannot resolve '(CAST('2017-12-11 09:30:00' AS DATE) = CAST(1 AS DECIMAL(20,0)))
 -- !query
 SELECT cast(1 as decimal(3, 0))  <> cast(1 as tinyint) FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS DECIMAL(3,0)) = CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(3,0)) = CAST(1 AS TINYINT))):boolean>
 -- !query output
 false
 
@@ -9129,7 +9129,7 @@ false
 -- !query
 SELECT cast(1 as decimal(5, 0))  <> cast(1 as tinyint) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(5,0)) = CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(5,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(5,0)) = CAST(1 AS TINYINT))):boolean>
 -- !query output
 false
 
@@ -9137,7 +9137,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) <> cast(1 as tinyint) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) = CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(10,0)) = CAST(1 AS TINYINT))):boolean>
 -- !query output
 false
 
@@ -9145,7 +9145,7 @@ false
 -- !query
 SELECT cast(1 as decimal(20, 0)) <> cast(1 as tinyint) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) = CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(20,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(20,0)) = CAST(1 AS TINYINT))):boolean>
 -- !query output
 false
 
@@ -9153,7 +9153,7 @@ false
 -- !query
 SELECT cast(1 as decimal(3, 0))  <> cast(1 as smallint) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(5,0)) = CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(5,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(3,0)) = CAST(1 AS SMALLINT))):boolean>
 -- !query output
 false
 
@@ -9161,7 +9161,7 @@ false
 -- !query
 SELECT cast(1 as decimal(5, 0))  <> cast(1 as smallint) FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS DECIMAL(5,0)) = CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(5,0)) = CAST(1 AS SMALLINT))):boolean>
 -- !query output
 false
 
@@ -9169,7 +9169,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) <> cast(1 as smallint) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) = CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(10,0)) = CAST(1 AS SMALLINT))):boolean>
 -- !query output
 false
 
@@ -9177,7 +9177,7 @@ false
 -- !query
 SELECT cast(1 as decimal(20, 0)) <> cast(1 as smallint) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) = CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(20,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(20,0)) = CAST(1 AS SMALLINT))):boolean>
 -- !query output
 false
 
@@ -9185,7 +9185,7 @@ false
 -- !query
 SELECT cast(1 as decimal(3, 0))  <> cast(1 as int) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) = CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(3,0)) = CAST(1 AS INT))):boolean>
 -- !query output
 false
 
@@ -9193,7 +9193,7 @@ false
 -- !query
 SELECT cast(1 as decimal(5, 0))  <> cast(1 as int) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)) = CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(10,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(5,0)) = CAST(1 AS INT))):boolean>
 -- !query output
 false
 
@@ -9201,7 +9201,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) <> cast(1 as int) FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS DECIMAL(10,0)) = CAST(CAST(1 AS INT) AS DECIMAL(10,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(10,0)) = CAST(1 AS INT))):boolean>
 -- !query output
 false
 
@@ -9209,7 +9209,7 @@ false
 -- !query
 SELECT cast(1 as decimal(20, 0)) <> cast(1 as int) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) = CAST(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) AS DECIMAL(20,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(20,0)) = CAST(1 AS INT))):boolean>
 -- !query output
 false
 
@@ -9217,7 +9217,7 @@ false
 -- !query
 SELECT cast(1 as decimal(3, 0))  <> cast(1 as bigint) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(20,0)) = CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(3,0)) = CAST(1 AS BIGINT))):boolean>
 -- !query output
 false
 
@@ -9225,7 +9225,7 @@ false
 -- !query
 SELECT cast(1 as decimal(5, 0))  <> cast(1 as bigint) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(20,0)) = CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(5,0)) = CAST(1 AS BIGINT))):boolean>
 -- !query output
 false
 
@@ -9233,7 +9233,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) <> cast(1 as bigint) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) = CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(10,0)) = CAST(1 AS BIGINT))):boolean>
 -- !query output
 false
 
@@ -9241,7 +9241,7 @@ false
 -- !query
 SELECT cast(1 as decimal(20, 0)) <> cast(1 as bigint) FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS DECIMAL(20,0)) = CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(20,0)) = CAST(1 AS BIGINT))):boolean>
 -- !query output
 false
 
@@ -9249,7 +9249,7 @@ false
 -- !query
 SELECT cast(1 as decimal(3, 0))  <> cast(1 as float) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) = CAST(CAST(1 AS FLOAT) AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(3,0)) = CAST(1 AS FLOAT))):boolean>
 -- !query output
 false
 
@@ -9257,7 +9257,7 @@ false
 -- !query
 SELECT cast(1 as decimal(5, 0))  <> cast(1 as float) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) = CAST(CAST(1 AS FLOAT) AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(5,0)) = CAST(1 AS FLOAT))):boolean>
 -- !query output
 false
 
@@ -9265,7 +9265,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) <> cast(1 as float) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) = CAST(CAST(1 AS FLOAT) AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(10,0)) = CAST(1 AS FLOAT))):boolean>
 -- !query output
 false
 
@@ -9273,7 +9273,7 @@ false
 -- !query
 SELECT cast(1 as decimal(20, 0)) <> cast(1 as float) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) = CAST(CAST(1 AS FLOAT) AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(20,0)) = CAST(1 AS FLOAT))):boolean>
 -- !query output
 false
 
@@ -9281,7 +9281,7 @@ false
 -- !query
 SELECT cast(1 as decimal(3, 0))  <> cast(1 as double) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) = CAST(1 AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(3,0)) = CAST(1 AS DOUBLE))):boolean>
 -- !query output
 false
 
@@ -9289,7 +9289,7 @@ false
 -- !query
 SELECT cast(1 as decimal(5, 0))  <> cast(1 as double) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) = CAST(1 AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(5,0)) = CAST(1 AS DOUBLE))):boolean>
 -- !query output
 false
 
@@ -9297,7 +9297,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) <> cast(1 as double) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) = CAST(1 AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(10,0)) = CAST(1 AS DOUBLE))):boolean>
 -- !query output
 false
 
@@ -9305,7 +9305,7 @@ false
 -- !query
 SELECT cast(1 as decimal(20, 0)) <> cast(1 as double) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) = CAST(1 AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(20,0)) = CAST(1 AS DOUBLE))):boolean>
 -- !query output
 false
 
@@ -9313,7 +9313,7 @@ false
 -- !query
 SELECT cast(1 as decimal(3, 0))  <> cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(3,0)) AS DECIMAL(10,0)) = CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(3,0)) = CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 false
 
@@ -9321,7 +9321,7 @@ false
 -- !query
 SELECT cast(1 as decimal(5, 0))  <> cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(5,0)) AS DECIMAL(10,0)) = CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(5,0)) = CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 false
 
@@ -9337,7 +9337,7 @@ false
 -- !query
 SELECT cast(1 as decimal(20, 0)) <> cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(20,0)) AS DECIMAL(20,0)) = CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(20,0)) = CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 false
 
@@ -9345,7 +9345,7 @@ false
 -- !query
 SELECT cast(1 as decimal(3, 0))  <> cast(1 as string) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(3,0)) AS DOUBLE) = CAST(CAST(1 AS STRING) AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(3,0)) = CAST(1 AS STRING))):boolean>
 -- !query output
 false
 
@@ -9353,7 +9353,7 @@ false
 -- !query
 SELECT cast(1 as decimal(5, 0))  <> cast(1 as string) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(5,0)) AS DOUBLE) = CAST(CAST(1 AS STRING) AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(5,0)) = CAST(1 AS STRING))):boolean>
 -- !query output
 false
 
@@ -9361,7 +9361,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) <> cast(1 as string) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) = CAST(CAST(1 AS STRING) AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(10,0)) = CAST(1 AS STRING))):boolean>
 -- !query output
 false
 
@@ -9369,7 +9369,7 @@ false
 -- !query
 SELECT cast(1 as decimal(20, 0)) <> cast(1 as string) FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(20,0)) AS DOUBLE) = CAST(CAST(1 AS STRING) AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(20,0)) = CAST(1 AS STRING))):boolean>
 -- !query output
 false
 
@@ -9413,7 +9413,7 @@ cannot resolve '(CAST(1 AS DECIMAL(20,0)) = CAST('1' AS BINARY))' due to data ty
 -- !query
 SELECT cast(1 as decimal(3, 0))  <> cast(1 as boolean) FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS DECIMAL(3,0)) = CAST(CAST(1 AS BOOLEAN) AS DECIMAL(3,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(3,0)) = CAST(1 AS BOOLEAN))):boolean>
 -- !query output
 false
 
@@ -9421,7 +9421,7 @@ false
 -- !query
 SELECT cast(1 as decimal(5, 0))  <> cast(1 as boolean) FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS DECIMAL(5,0)) = CAST(CAST(1 AS BOOLEAN) AS DECIMAL(5,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(5,0)) = CAST(1 AS BOOLEAN))):boolean>
 -- !query output
 false
 
@@ -9429,7 +9429,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0)) <> cast(1 as boolean) FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS DECIMAL(10,0)) = CAST(CAST(1 AS BOOLEAN) AS DECIMAL(10,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(10,0)) = CAST(1 AS BOOLEAN))):boolean>
 -- !query output
 false
 
@@ -9437,7 +9437,7 @@ false
 -- !query
 SELECT cast(1 as decimal(20, 0)) <> cast(1 as boolean) FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS DECIMAL(20,0)) = CAST(CAST(1 AS BOOLEAN) AS DECIMAL(20,0)))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(20,0)) = CAST(1 AS BOOLEAN))):boolean>
 -- !query output
 false
 
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/division.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/division.sql.out
index ae933da59f63f..987b4dda729f9 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/division.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/division.sql.out
@@ -13,7 +13,7 @@ struct<>
 -- !query
 SELECT cast(1 as tinyint) / cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DOUBLE) / CAST(CAST(1 AS TINYINT) AS DOUBLE)):double>
+struct<(CAST(1 AS TINYINT) / CAST(1 AS TINYINT)):double>
 -- !query output
 1.0
 
@@ -21,7 +21,7 @@ struct<(CAST(CAST(1 AS TINYINT) AS DOUBLE) / CAST(CAST(1 AS TINYINT) AS DOUBLE))
 -- !query
 SELECT cast(1 as tinyint) / cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DOUBLE) / CAST(CAST(1 AS SMALLINT) AS DOUBLE)):double>
+struct<(CAST(1 AS TINYINT) / CAST(1 AS SMALLINT)):double>
 -- !query output
 1.0
 
@@ -29,7 +29,7 @@ struct<(CAST(CAST(1 AS TINYINT) AS DOUBLE) / CAST(CAST(1 AS SMALLINT) AS DOUBLE)
 -- !query
 SELECT cast(1 as tinyint) / cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DOUBLE) / CAST(CAST(1 AS INT) AS DOUBLE)):double>
+struct<(CAST(1 AS TINYINT) / CAST(1 AS INT)):double>
 -- !query output
 1.0
 
@@ -37,7 +37,7 @@ struct<(CAST(CAST(1 AS TINYINT) AS DOUBLE) / CAST(CAST(1 AS INT) AS DOUBLE)):dou
 -- !query
 SELECT cast(1 as tinyint) / cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DOUBLE) / CAST(CAST(1 AS BIGINT) AS DOUBLE)):double>
+struct<(CAST(1 AS TINYINT) / CAST(1 AS BIGINT)):double>
 -- !query output
 1.0
 
@@ -45,7 +45,7 @@ struct<(CAST(CAST(1 AS TINYINT) AS DOUBLE) / CAST(CAST(1 AS BIGINT) AS DOUBLE)):
 -- !query
 SELECT cast(1 as tinyint) / cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DOUBLE) / CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS TINYINT) / CAST(1 AS FLOAT)):double>
 -- !query output
 1.0
 
@@ -53,7 +53,7 @@ struct<(CAST(CAST(1 AS TINYINT) AS DOUBLE) / CAST(CAST(1 AS FLOAT) AS DOUBLE)):d
 -- !query
 SELECT cast(1 as tinyint) / cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DOUBLE) / CAST(CAST(1 AS DOUBLE) AS DOUBLE)):double>
+struct<(CAST(1 AS TINYINT) / CAST(1 AS DOUBLE)):double>
 -- !query output
 1.0
 
@@ -61,7 +61,7 @@ struct<(CAST(CAST(1 AS TINYINT) AS DOUBLE) / CAST(CAST(1 AS DOUBLE) AS DOUBLE)):
 -- !query
 SELECT cast(1 as tinyint) / cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0)) / CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(14,11)>
+struct<(CAST(1 AS TINYINT) / CAST(1 AS DECIMAL(10,0))):decimal(14,11)>
 -- !query output
 1.00000000000
 
@@ -69,7 +69,7 @@ struct<(CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0)) / CAST(C
 -- !query
 SELECT cast(1 as tinyint) / cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DOUBLE) / CAST(CAST(CAST(1 AS STRING) AS DOUBLE) AS DOUBLE)):double>
+struct<(CAST(1 AS TINYINT) / CAST(1 AS STRING)):double>
 -- !query output
 1.0
 
@@ -113,7 +113,7 @@ cannot resolve '(CAST(1 AS TINYINT) / CAST('2017-12-11 09:30:00' AS DATE))' due
 -- !query
 SELECT cast(1 as smallint) / cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DOUBLE) / CAST(CAST(1 AS TINYINT) AS DOUBLE)):double>
+struct<(CAST(1 AS SMALLINT) / CAST(1 AS TINYINT)):double>
 -- !query output
 1.0
 
@@ -121,7 +121,7 @@ struct<(CAST(CAST(1 AS SMALLINT) AS DOUBLE) / CAST(CAST(1 AS TINYINT) AS DOUBLE)
 -- !query
 SELECT cast(1 as smallint) / cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DOUBLE) / CAST(CAST(1 AS SMALLINT) AS DOUBLE)):double>
+struct<(CAST(1 AS SMALLINT) / CAST(1 AS SMALLINT)):double>
 -- !query output
 1.0
 
@@ -129,7 +129,7 @@ struct<(CAST(CAST(1 AS SMALLINT) AS DOUBLE) / CAST(CAST(1 AS SMALLINT) AS DOUBLE
 -- !query
 SELECT cast(1 as smallint) / cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DOUBLE) / CAST(CAST(1 AS INT) AS DOUBLE)):double>
+struct<(CAST(1 AS SMALLINT) / CAST(1 AS INT)):double>
 -- !query output
 1.0
 
@@ -137,7 +137,7 @@ struct<(CAST(CAST(1 AS SMALLINT) AS DOUBLE) / CAST(CAST(1 AS INT) AS DOUBLE)):do
 -- !query
 SELECT cast(1 as smallint) / cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DOUBLE) / CAST(CAST(1 AS BIGINT) AS DOUBLE)):double>
+struct<(CAST(1 AS SMALLINT) / CAST(1 AS BIGINT)):double>
 -- !query output
 1.0
 
@@ -145,7 +145,7 @@ struct<(CAST(CAST(1 AS SMALLINT) AS DOUBLE) / CAST(CAST(1 AS BIGINT) AS DOUBLE))
 -- !query
 SELECT cast(1 as smallint) / cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DOUBLE) / CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS SMALLINT) / CAST(1 AS FLOAT)):double>
 -- !query output
 1.0
 
@@ -153,7 +153,7 @@ struct<(CAST(CAST(1 AS SMALLINT) AS DOUBLE) / CAST(CAST(1 AS FLOAT) AS DOUBLE)):
 -- !query
 SELECT cast(1 as smallint) / cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DOUBLE) / CAST(CAST(1 AS DOUBLE) AS DOUBLE)):double>
+struct<(CAST(1 AS SMALLINT) / CAST(1 AS DOUBLE)):double>
 -- !query output
 1.0
 
@@ -161,7 +161,7 @@ struct<(CAST(CAST(1 AS SMALLINT) AS DOUBLE) / CAST(CAST(1 AS DOUBLE) AS DOUBLE))
 -- !query
 SELECT cast(1 as smallint) / cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0)) / CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0))):decimal(16,11)>
+struct<(CAST(1 AS SMALLINT) / CAST(1 AS DECIMAL(10,0))):decimal(16,11)>
 -- !query output
 1.00000000000
 
@@ -169,7 +169,7 @@ struct<(CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0)) / CAST(
 -- !query
 SELECT cast(1 as smallint) / cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DOUBLE) / CAST(CAST(CAST(1 AS STRING) AS DOUBLE) AS DOUBLE)):double>
+struct<(CAST(1 AS SMALLINT) / CAST(1 AS STRING)):double>
 -- !query output
 1.0
 
@@ -213,7 +213,7 @@ cannot resolve '(CAST(1 AS SMALLINT) / CAST('2017-12-11 09:30:00' AS DATE))' due
 -- !query
 SELECT cast(1 as int) / cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DOUBLE) / CAST(CAST(1 AS TINYINT) AS DOUBLE)):double>
+struct<(CAST(1 AS INT) / CAST(1 AS TINYINT)):double>
 -- !query output
 1.0
 
@@ -221,7 +221,7 @@ struct<(CAST(CAST(1 AS INT) AS DOUBLE) / CAST(CAST(1 AS TINYINT) AS DOUBLE)):dou
 -- !query
 SELECT cast(1 as int) / cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DOUBLE) / CAST(CAST(1 AS SMALLINT) AS DOUBLE)):double>
+struct<(CAST(1 AS INT) / CAST(1 AS SMALLINT)):double>
 -- !query output
 1.0
 
@@ -229,7 +229,7 @@ struct<(CAST(CAST(1 AS INT) AS DOUBLE) / CAST(CAST(1 AS SMALLINT) AS DOUBLE)):do
 -- !query
 SELECT cast(1 as int) / cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DOUBLE) / CAST(CAST(1 AS INT) AS DOUBLE)):double>
+struct<(CAST(1 AS INT) / CAST(1 AS INT)):double>
 -- !query output
 1.0
 
@@ -237,7 +237,7 @@ struct<(CAST(CAST(1 AS INT) AS DOUBLE) / CAST(CAST(1 AS INT) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as int) / cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DOUBLE) / CAST(CAST(1 AS BIGINT) AS DOUBLE)):double>
+struct<(CAST(1 AS INT) / CAST(1 AS BIGINT)):double>
 -- !query output
 1.0
 
@@ -245,7 +245,7 @@ struct<(CAST(CAST(1 AS INT) AS DOUBLE) / CAST(CAST(1 AS BIGINT) AS DOUBLE)):doub
 -- !query
 SELECT cast(1 as int) / cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DOUBLE) / CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS INT) / CAST(1 AS FLOAT)):double>
 -- !query output
 1.0
 
@@ -253,7 +253,7 @@ struct<(CAST(CAST(1 AS INT) AS DOUBLE) / CAST(CAST(1 AS FLOAT) AS DOUBLE)):doubl
 -- !query
 SELECT cast(1 as int) / cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DOUBLE) / CAST(CAST(1 AS DOUBLE) AS DOUBLE)):double>
+struct<(CAST(1 AS INT) / CAST(1 AS DOUBLE)):double>
 -- !query output
 1.0
 
@@ -261,7 +261,7 @@ struct<(CAST(CAST(1 AS INT) AS DOUBLE) / CAST(CAST(1 AS DOUBLE) AS DOUBLE)):doub
 -- !query
 SELECT cast(1 as int) / cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) / CAST(1 AS DECIMAL(10,0))):decimal(21,11)>
+struct<(CAST(1 AS INT) / CAST(1 AS DECIMAL(10,0))):decimal(21,11)>
 -- !query output
 1.00000000000
 
@@ -269,7 +269,7 @@ struct<(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) / CAST(1 AS DECIMAL(10,0))):decima
 -- !query
 SELECT cast(1 as int) / cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DOUBLE) / CAST(CAST(CAST(1 AS STRING) AS DOUBLE) AS DOUBLE)):double>
+struct<(CAST(1 AS INT) / CAST(1 AS STRING)):double>
 -- !query output
 1.0
 
@@ -313,7 +313,7 @@ cannot resolve '(CAST(1 AS INT) / CAST('2017-12-11 09:30:00' AS DATE))' due to d
 -- !query
 SELECT cast(1 as bigint) / cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DOUBLE) / CAST(CAST(1 AS TINYINT) AS DOUBLE)):double>
+struct<(CAST(1 AS BIGINT) / CAST(1 AS TINYINT)):double>
 -- !query output
 1.0
 
@@ -321,7 +321,7 @@ struct<(CAST(CAST(1 AS BIGINT) AS DOUBLE) / CAST(CAST(1 AS TINYINT) AS DOUBLE)):
 -- !query
 SELECT cast(1 as bigint) / cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DOUBLE) / CAST(CAST(1 AS SMALLINT) AS DOUBLE)):double>
+struct<(CAST(1 AS BIGINT) / CAST(1 AS SMALLINT)):double>
 -- !query output
 1.0
 
@@ -329,7 +329,7 @@ struct<(CAST(CAST(1 AS BIGINT) AS DOUBLE) / CAST(CAST(1 AS SMALLINT) AS DOUBLE))
 -- !query
 SELECT cast(1 as bigint) / cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DOUBLE) / CAST(CAST(1 AS INT) AS DOUBLE)):double>
+struct<(CAST(1 AS BIGINT) / CAST(1 AS INT)):double>
 -- !query output
 1.0
 
@@ -337,7 +337,7 @@ struct<(CAST(CAST(1 AS BIGINT) AS DOUBLE) / CAST(CAST(1 AS INT) AS DOUBLE)):doub
 -- !query
 SELECT cast(1 as bigint) / cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DOUBLE) / CAST(CAST(1 AS BIGINT) AS DOUBLE)):double>
+struct<(CAST(1 AS BIGINT) / CAST(1 AS BIGINT)):double>
 -- !query output
 1.0
 
@@ -345,7 +345,7 @@ struct<(CAST(CAST(1 AS BIGINT) AS DOUBLE) / CAST(CAST(1 AS BIGINT) AS DOUBLE)):d
 -- !query
 SELECT cast(1 as bigint) / cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DOUBLE) / CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS BIGINT) / CAST(1 AS FLOAT)):double>
 -- !query output
 1.0
 
@@ -353,7 +353,7 @@ struct<(CAST(CAST(1 AS BIGINT) AS DOUBLE) / CAST(CAST(1 AS FLOAT) AS DOUBLE)):do
 -- !query
 SELECT cast(1 as bigint) / cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DOUBLE) / CAST(CAST(1 AS DOUBLE) AS DOUBLE)):double>
+struct<(CAST(1 AS BIGINT) / CAST(1 AS DOUBLE)):double>
 -- !query output
 1.0
 
@@ -361,7 +361,7 @@ struct<(CAST(CAST(1 AS BIGINT) AS DOUBLE) / CAST(CAST(1 AS DOUBLE) AS DOUBLE)):d
 -- !query
 SELECT cast(1 as bigint) / cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) / CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0))):decimal(31,11)>
+struct<(CAST(1 AS BIGINT) / CAST(1 AS DECIMAL(10,0))):decimal(31,11)>
 -- !query output
 1.00000000000
 
@@ -369,7 +369,7 @@ struct<(CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0)) / CAST(C
 -- !query
 SELECT cast(1 as bigint) / cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DOUBLE) / CAST(CAST(CAST(1 AS STRING) AS DOUBLE) AS DOUBLE)):double>
+struct<(CAST(1 AS BIGINT) / CAST(1 AS STRING)):double>
 -- !query output
 1.0
 
@@ -413,7 +413,7 @@ cannot resolve '(CAST(1 AS BIGINT) / CAST('2017-12-11 09:30:00' AS DATE))' due t
 -- !query
 SELECT cast(1 as float) / cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) / CAST(CAST(1 AS TINYINT) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) / CAST(1 AS TINYINT)):double>
 -- !query output
 1.0
 
@@ -421,7 +421,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) / CAST(CAST(1 AS TINYINT) AS DOUBLE)):d
 -- !query
 SELECT cast(1 as float) / cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) / CAST(CAST(1 AS SMALLINT) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) / CAST(1 AS SMALLINT)):double>
 -- !query output
 1.0
 
@@ -429,7 +429,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) / CAST(CAST(1 AS SMALLINT) AS DOUBLE)):
 -- !query
 SELECT cast(1 as float) / cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) / CAST(CAST(1 AS INT) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) / CAST(1 AS INT)):double>
 -- !query output
 1.0
 
@@ -437,7 +437,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) / CAST(CAST(1 AS INT) AS DOUBLE)):doubl
 -- !query
 SELECT cast(1 as float) / cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) / CAST(CAST(1 AS BIGINT) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) / CAST(1 AS BIGINT)):double>
 -- !query output
 1.0
 
@@ -445,7 +445,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) / CAST(CAST(1 AS BIGINT) AS DOUBLE)):do
 -- !query
 SELECT cast(1 as float) / cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) / CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) / CAST(1 AS FLOAT)):double>
 -- !query output
 1.0
 
@@ -453,7 +453,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) / CAST(CAST(1 AS FLOAT) AS DOUBLE)):dou
 -- !query
 SELECT cast(1 as float) / cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) / CAST(CAST(1 AS DOUBLE) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) / CAST(1 AS DOUBLE)):double>
 -- !query output
 1.0
 
@@ -461,7 +461,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) / CAST(CAST(1 AS DOUBLE) AS DOUBLE)):do
 -- !query
 SELECT cast(1 as float) / cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) / CAST(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) / CAST(1 AS DECIMAL(10,0))):double>
 -- !query output
 1.0
 
@@ -469,7 +469,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) / CAST(CAST(CAST(1 AS DECIMAL(10,0)) AS
 -- !query
 SELECT cast(1 as float) / cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) / CAST(CAST(CAST(1 AS STRING) AS DOUBLE) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) / CAST(1 AS STRING)):double>
 -- !query output
 1.0
 
@@ -513,7 +513,7 @@ cannot resolve '(CAST(1 AS FLOAT) / CAST('2017-12-11 09:30:00' AS DATE))' due to
 -- !query
 SELECT cast(1 as double) / cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS TINYINT) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) / CAST(1 AS TINYINT)):double>
 -- !query output
 1.0
 
@@ -521,7 +521,7 @@ struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS TINYINT) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as double) / cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS SMALLINT) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) / CAST(1 AS SMALLINT)):double>
 -- !query output
 1.0
 
@@ -529,7 +529,7 @@ struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS SMALLINT) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as double) / cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS INT) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) / CAST(1 AS INT)):double>
 -- !query output
 1.0
 
@@ -537,7 +537,7 @@ struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS INT) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as double) / cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS BIGINT) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) / CAST(1 AS BIGINT)):double>
 -- !query output
 1.0
 
@@ -545,7 +545,7 @@ struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS BIGINT) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as double) / cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) / CAST(1 AS FLOAT)):double>
 -- !query output
 1.0
 
@@ -561,7 +561,7 @@ struct<(CAST(1 AS DOUBLE) / CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as double) / cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) / CAST(1 AS DECIMAL(10,0))):double>
 -- !query output
 1.0
 
@@ -569,7 +569,7 @@ struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as double) / cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) / CAST(1 AS STRING)):double>
 -- !query output
 1.0
 
@@ -613,7 +613,7 @@ cannot resolve '(CAST(1 AS DOUBLE) / CAST('2017-12-11 09:30:00' AS DATE))' due t
 -- !query
 SELECT cast(1 as decimal(10, 0)) / cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) / CAST(CAST(CAST(1 AS TINYINT) AS DECIMAL(3,0)) AS DECIMAL(10,0))):decimal(16,6)>
+struct<(CAST(1 AS DECIMAL(10,0)) / CAST(1 AS TINYINT)):decimal(16,6)>
 -- !query output
 1.000000
 
@@ -621,7 +621,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) / CAST(CAST(CAST(1 AS TI
 -- !query
 SELECT cast(1 as decimal(10, 0)) / cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) / CAST(CAST(CAST(1 AS SMALLINT) AS DECIMAL(5,0)) AS DECIMAL(10,0))):decimal(16,6)>
+struct<(CAST(1 AS DECIMAL(10,0)) / CAST(1 AS SMALLINT)):decimal(16,6)>
 -- !query output
 1.000000
 
@@ -629,7 +629,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) / CAST(CAST(CAST(1 AS SM
 -- !query
 SELECT cast(1 as decimal(10, 0)) / cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(1 AS DECIMAL(10,0)) / CAST(CAST(1 AS INT) AS DECIMAL(10,0))):decimal(21,11)>
+struct<(CAST(1 AS DECIMAL(10,0)) / CAST(1 AS INT)):decimal(21,11)>
 -- !query output
 1.00000000000
 
@@ -637,7 +637,7 @@ struct<(CAST(1 AS DECIMAL(10,0)) / CAST(CAST(1 AS INT) AS DECIMAL(10,0))):decima
 -- !query
 SELECT cast(1 as decimal(10, 0)) / cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) / CAST(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) AS DECIMAL(20,0))):decimal(31,21)>
+struct<(CAST(1 AS DECIMAL(10,0)) / CAST(1 AS BIGINT)):decimal(31,21)>
 -- !query output
 1.000000000000000000000
 
@@ -645,7 +645,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) / CAST(CAST(CAST(1 AS BI
 -- !query
 SELECT cast(1 as decimal(10, 0)) / cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) / CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(10,0)) / CAST(1 AS FLOAT)):double>
 -- !query output
 1.0
 
@@ -653,7 +653,7 @@ struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) / CAST(CAST(1 AS FLOAT) AS DOUB
 -- !query
 SELECT cast(1 as decimal(10, 0)) / cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) / CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(10,0)) / CAST(1 AS DOUBLE)):double>
 -- !query output
 1.0
 
@@ -669,7 +669,7 @@ struct<(CAST(1 AS DECIMAL(10,0)) / CAST(1 AS DECIMAL(10,0))):decimal(21,11)>
 -- !query
 SELECT cast(1 as decimal(10, 0)) / cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) / CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(10,0)) / CAST(1 AS STRING)):double>
 -- !query output
 1.0
 
@@ -713,7 +713,7 @@ cannot resolve '(CAST(1 AS DECIMAL(10,0)) / CAST('2017-12-11 09:30:00' AS DATE))
 -- !query
 SELECT cast(1 as string) / cast(1 as tinyint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS DOUBLE) / CAST(CAST(1 AS TINYINT) AS DOUBLE)):double>
+struct<(CAST(1 AS STRING) / CAST(1 AS TINYINT)):double>
 -- !query output
 1.0
 
@@ -721,7 +721,7 @@ struct<(CAST(CAST(1 AS STRING) AS DOUBLE) / CAST(CAST(1 AS TINYINT) AS DOUBLE)):
 -- !query
 SELECT cast(1 as string) / cast(1 as smallint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS DOUBLE) / CAST(CAST(1 AS SMALLINT) AS DOUBLE)):double>
+struct<(CAST(1 AS STRING) / CAST(1 AS SMALLINT)):double>
 -- !query output
 1.0
 
@@ -729,7 +729,7 @@ struct<(CAST(CAST(1 AS STRING) AS DOUBLE) / CAST(CAST(1 AS SMALLINT) AS DOUBLE))
 -- !query
 SELECT cast(1 as string) / cast(1 as int) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS DOUBLE) / CAST(CAST(1 AS INT) AS DOUBLE)):double>
+struct<(CAST(1 AS STRING) / CAST(1 AS INT)):double>
 -- !query output
 1.0
 
@@ -737,7 +737,7 @@ struct<(CAST(CAST(1 AS STRING) AS DOUBLE) / CAST(CAST(1 AS INT) AS DOUBLE)):doub
 -- !query
 SELECT cast(1 as string) / cast(1 as bigint) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS DOUBLE) / CAST(CAST(1 AS BIGINT) AS DOUBLE)):double>
+struct<(CAST(1 AS STRING) / CAST(1 AS BIGINT)):double>
 -- !query output
 1.0
 
@@ -745,7 +745,7 @@ struct<(CAST(CAST(1 AS STRING) AS DOUBLE) / CAST(CAST(1 AS BIGINT) AS DOUBLE)):d
 -- !query
 SELECT cast(1 as string) / cast(1 as float) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS DOUBLE) / CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(CAST(1 AS STRING) / CAST(1 AS FLOAT)):double>
 -- !query output
 1.0
 
@@ -753,7 +753,7 @@ struct<(CAST(CAST(1 AS STRING) AS DOUBLE) / CAST(CAST(1 AS FLOAT) AS DOUBLE)):do
 -- !query
 SELECT cast(1 as string) / cast(1 as double) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS DOUBLE) / CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS STRING) / CAST(1 AS DOUBLE)):double>
 -- !query output
 1.0
 
@@ -761,7 +761,7 @@ struct<(CAST(CAST(1 AS STRING) AS DOUBLE) / CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as string) / cast(1 as decimal(10, 0)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS DOUBLE) / CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
+struct<(CAST(1 AS STRING) / CAST(1 AS DECIMAL(10,0))):double>
 -- !query output
 1.0
 
@@ -769,7 +769,7 @@ struct<(CAST(CAST(1 AS STRING) AS DOUBLE) / CAST(CAST(1 AS DECIMAL(10,0)) AS DOU
 -- !query
 SELECT cast(1 as string) / cast(1 as string) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS DOUBLE) / CAST(CAST(1 AS STRING) AS DOUBLE)):double>
+struct<(CAST(1 AS STRING) / CAST(1 AS STRING)):double>
 -- !query output
 1.0
 
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/ifCoercion.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/ifCoercion.sql.out
index bb49d296eaada..a9424db391b42 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/ifCoercion.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/ifCoercion.sql.out
@@ -21,7 +21,7 @@ struct<(IF(true, CAST(1 AS TINYINT), CAST(2 AS TINYINT))):tinyint>
 -- !query
 SELECT IF(true, cast(1 as tinyint), cast(2 as smallint)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS TINYINT) AS SMALLINT), CAST(2 AS SMALLINT))):smallint>
+struct<(IF(true, CAST(1 AS TINYINT), CAST(2 AS SMALLINT))):smallint>
 -- !query output
 1
 
@@ -29,7 +29,7 @@ struct<(IF(true, CAST(CAST(1 AS TINYINT) AS SMALLINT), CAST(2 AS SMALLINT))):sma
 -- !query
 SELECT IF(true, cast(1 as tinyint), cast(2 as int)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS TINYINT) AS INT), CAST(2 AS INT))):int>
+struct<(IF(true, CAST(1 AS TINYINT), CAST(2 AS INT))):int>
 -- !query output
 1
 
@@ -37,7 +37,7 @@ struct<(IF(true, CAST(CAST(1 AS TINYINT) AS INT), CAST(2 AS INT))):int>
 -- !query
 SELECT IF(true, cast(1 as tinyint), cast(2 as bigint)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS TINYINT) AS BIGINT), CAST(2 AS BIGINT))):bigint>
+struct<(IF(true, CAST(1 AS TINYINT), CAST(2 AS BIGINT))):bigint>
 -- !query output
 1
 
@@ -45,7 +45,7 @@ struct<(IF(true, CAST(CAST(1 AS TINYINT) AS BIGINT), CAST(2 AS BIGINT))):bigint>
 -- !query
 SELECT IF(true, cast(1 as tinyint), cast(2 as float)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS TINYINT) AS FLOAT), CAST(2 AS FLOAT))):float>
+struct<(IF(true, CAST(1 AS TINYINT), CAST(2 AS FLOAT))):float>
 -- !query output
 1.0
 
@@ -53,7 +53,7 @@ struct<(IF(true, CAST(CAST(1 AS TINYINT) AS FLOAT), CAST(2 AS FLOAT))):float>
 -- !query
 SELECT IF(true, cast(1 as tinyint), cast(2 as double)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS TINYINT) AS DOUBLE), CAST(2 AS DOUBLE))):double>
+struct<(IF(true, CAST(1 AS TINYINT), CAST(2 AS DOUBLE))):double>
 -- !query output
 1.0
 
@@ -61,7 +61,7 @@ struct<(IF(true, CAST(CAST(1 AS TINYINT) AS DOUBLE), CAST(2 AS DOUBLE))):double>
 -- !query
 SELECT IF(true, cast(1 as tinyint), cast(2 as decimal(10, 0))) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS TINYINT) AS DECIMAL(10,0)), CAST(2 AS DECIMAL(10,0)))):decimal(10,0)>
+struct<(IF(true, CAST(1 AS TINYINT), CAST(2 AS DECIMAL(10,0)))):decimal(10,0)>
 -- !query output
 1
 
@@ -69,7 +69,7 @@ struct<(IF(true, CAST(CAST(1 AS TINYINT) AS DECIMAL(10,0)), CAST(2 AS DECIMAL(10
 -- !query
 SELECT IF(true, cast(1 as tinyint), cast(2 as string)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS TINYINT) AS STRING), CAST(2 AS STRING))):string>
+struct<(IF(true, CAST(1 AS TINYINT), CAST(2 AS STRING))):string>
 -- !query output
 1
 
@@ -113,7 +113,7 @@ cannot resolve '(IF(true, CAST(1 AS TINYINT), CAST('2017-12-11 09:30:00' AS DATE
 -- !query
 SELECT IF(true, cast(1 as smallint), cast(2 as tinyint)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS SMALLINT), CAST(CAST(2 AS TINYINT) AS SMALLINT))):smallint>
+struct<(IF(true, CAST(1 AS SMALLINT), CAST(2 AS TINYINT))):smallint>
 -- !query output
 1
 
@@ -129,7 +129,7 @@ struct<(IF(true, CAST(1 AS SMALLINT), CAST(2 AS SMALLINT))):smallint>
 -- !query
 SELECT IF(true, cast(1 as smallint), cast(2 as int)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS SMALLINT) AS INT), CAST(2 AS INT))):int>
+struct<(IF(true, CAST(1 AS SMALLINT), CAST(2 AS INT))):int>
 -- !query output
 1
 
@@ -137,7 +137,7 @@ struct<(IF(true, CAST(CAST(1 AS SMALLINT) AS INT), CAST(2 AS INT))):int>
 -- !query
 SELECT IF(true, cast(1 as smallint), cast(2 as bigint)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS SMALLINT) AS BIGINT), CAST(2 AS BIGINT))):bigint>
+struct<(IF(true, CAST(1 AS SMALLINT), CAST(2 AS BIGINT))):bigint>
 -- !query output
 1
 
@@ -145,7 +145,7 @@ struct<(IF(true, CAST(CAST(1 AS SMALLINT) AS BIGINT), CAST(2 AS BIGINT))):bigint
 -- !query
 SELECT IF(true, cast(1 as smallint), cast(2 as float)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS SMALLINT) AS FLOAT), CAST(2 AS FLOAT))):float>
+struct<(IF(true, CAST(1 AS SMALLINT), CAST(2 AS FLOAT))):float>
 -- !query output
 1.0
 
@@ -153,7 +153,7 @@ struct<(IF(true, CAST(CAST(1 AS SMALLINT) AS FLOAT), CAST(2 AS FLOAT))):float>
 -- !query
 SELECT IF(true, cast(1 as smallint), cast(2 as double)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS SMALLINT) AS DOUBLE), CAST(2 AS DOUBLE))):double>
+struct<(IF(true, CAST(1 AS SMALLINT), CAST(2 AS DOUBLE))):double>
 -- !query output
 1.0
 
@@ -161,7 +161,7 @@ struct<(IF(true, CAST(CAST(1 AS SMALLINT) AS DOUBLE), CAST(2 AS DOUBLE))):double
 -- !query
 SELECT IF(true, cast(1 as smallint), cast(2 as decimal(10, 0))) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS SMALLINT) AS DECIMAL(10,0)), CAST(2 AS DECIMAL(10,0)))):decimal(10,0)>
+struct<(IF(true, CAST(1 AS SMALLINT), CAST(2 AS DECIMAL(10,0)))):decimal(10,0)>
 -- !query output
 1
 
@@ -169,7 +169,7 @@ struct<(IF(true, CAST(CAST(1 AS SMALLINT) AS DECIMAL(10,0)), CAST(2 AS DECIMAL(1
 -- !query
 SELECT IF(true, cast(1 as smallint), cast(2 as string)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS SMALLINT) AS STRING), CAST(2 AS STRING))):string>
+struct<(IF(true, CAST(1 AS SMALLINT), CAST(2 AS STRING))):string>
 -- !query output
 1
 
@@ -213,7 +213,7 @@ cannot resolve '(IF(true, CAST(1 AS SMALLINT), CAST('2017-12-11 09:30:00' AS DAT
 -- !query
 SELECT IF(true, cast(1 as int), cast(2 as tinyint)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS INT), CAST(CAST(2 AS TINYINT) AS INT))):int>
+struct<(IF(true, CAST(1 AS INT), CAST(2 AS TINYINT))):int>
 -- !query output
 1
 
@@ -221,7 +221,7 @@ struct<(IF(true, CAST(1 AS INT), CAST(CAST(2 AS TINYINT) AS INT))):int>
 -- !query
 SELECT IF(true, cast(1 as int), cast(2 as smallint)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS INT), CAST(CAST(2 AS SMALLINT) AS INT))):int>
+struct<(IF(true, CAST(1 AS INT), CAST(2 AS SMALLINT))):int>
 -- !query output
 1
 
@@ -237,7 +237,7 @@ struct<(IF(true, CAST(1 AS INT), CAST(2 AS INT))):int>
 -- !query
 SELECT IF(true, cast(1 as int), cast(2 as bigint)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS INT) AS BIGINT), CAST(2 AS BIGINT))):bigint>
+struct<(IF(true, CAST(1 AS INT), CAST(2 AS BIGINT))):bigint>
 -- !query output
 1
 
@@ -245,7 +245,7 @@ struct<(IF(true, CAST(CAST(1 AS INT) AS BIGINT), CAST(2 AS BIGINT))):bigint>
 -- !query
 SELECT IF(true, cast(1 as int), cast(2 as float)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS INT) AS FLOAT), CAST(2 AS FLOAT))):float>
+struct<(IF(true, CAST(1 AS INT), CAST(2 AS FLOAT))):float>
 -- !query output
 1.0
 
@@ -253,7 +253,7 @@ struct<(IF(true, CAST(CAST(1 AS INT) AS FLOAT), CAST(2 AS FLOAT))):float>
 -- !query
 SELECT IF(true, cast(1 as int), cast(2 as double)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS INT) AS DOUBLE), CAST(2 AS DOUBLE))):double>
+struct<(IF(true, CAST(1 AS INT), CAST(2 AS DOUBLE))):double>
 -- !query output
 1.0
 
@@ -261,7 +261,7 @@ struct<(IF(true, CAST(CAST(1 AS INT) AS DOUBLE), CAST(2 AS DOUBLE))):double>
 -- !query
 SELECT IF(true, cast(1 as int), cast(2 as decimal(10, 0))) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS INT) AS DECIMAL(10,0)), CAST(2 AS DECIMAL(10,0)))):decimal(10,0)>
+struct<(IF(true, CAST(1 AS INT), CAST(2 AS DECIMAL(10,0)))):decimal(10,0)>
 -- !query output
 1
 
@@ -269,7 +269,7 @@ struct<(IF(true, CAST(CAST(1 AS INT) AS DECIMAL(10,0)), CAST(2 AS DECIMAL(10,0))
 -- !query
 SELECT IF(true, cast(1 as int), cast(2 as string)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS INT) AS STRING), CAST(2 AS STRING))):string>
+struct<(IF(true, CAST(1 AS INT), CAST(2 AS STRING))):string>
 -- !query output
 1
 
@@ -313,7 +313,7 @@ cannot resolve '(IF(true, CAST(1 AS INT), CAST('2017-12-11 09:30:00' AS DATE)))'
 -- !query
 SELECT IF(true, cast(1 as bigint), cast(2 as tinyint)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS BIGINT), CAST(CAST(2 AS TINYINT) AS BIGINT))):bigint>
+struct<(IF(true, CAST(1 AS BIGINT), CAST(2 AS TINYINT))):bigint>
 -- !query output
 1
 
@@ -321,7 +321,7 @@ struct<(IF(true, CAST(1 AS BIGINT), CAST(CAST(2 AS TINYINT) AS BIGINT))):bigint>
 -- !query
 SELECT IF(true, cast(1 as bigint), cast(2 as smallint)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS BIGINT), CAST(CAST(2 AS SMALLINT) AS BIGINT))):bigint>
+struct<(IF(true, CAST(1 AS BIGINT), CAST(2 AS SMALLINT))):bigint>
 -- !query output
 1
 
@@ -329,7 +329,7 @@ struct<(IF(true, CAST(1 AS BIGINT), CAST(CAST(2 AS SMALLINT) AS BIGINT))):bigint
 -- !query
 SELECT IF(true, cast(1 as bigint), cast(2 as int)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS BIGINT), CAST(CAST(2 AS INT) AS BIGINT))):bigint>
+struct<(IF(true, CAST(1 AS BIGINT), CAST(2 AS INT))):bigint>
 -- !query output
 1
 
@@ -345,7 +345,7 @@ struct<(IF(true, CAST(1 AS BIGINT), CAST(2 AS BIGINT))):bigint>
 -- !query
 SELECT IF(true, cast(1 as bigint), cast(2 as float)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS BIGINT) AS FLOAT), CAST(2 AS FLOAT))):float>
+struct<(IF(true, CAST(1 AS BIGINT), CAST(2 AS FLOAT))):float>
 -- !query output
 1.0
 
@@ -353,7 +353,7 @@ struct<(IF(true, CAST(CAST(1 AS BIGINT) AS FLOAT), CAST(2 AS FLOAT))):float>
 -- !query
 SELECT IF(true, cast(1 as bigint), cast(2 as double)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS BIGINT) AS DOUBLE), CAST(2 AS DOUBLE))):double>
+struct<(IF(true, CAST(1 AS BIGINT), CAST(2 AS DOUBLE))):double>
 -- !query output
 1.0
 
@@ -361,7 +361,7 @@ struct<(IF(true, CAST(CAST(1 AS BIGINT) AS DOUBLE), CAST(2 AS DOUBLE))):double>
 -- !query
 SELECT IF(true, cast(1 as bigint), cast(2 as decimal(10, 0))) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)), CAST(CAST(2 AS DECIMAL(10,0)) AS DECIMAL(20,0)))):decimal(20,0)>
+struct<(IF(true, CAST(1 AS BIGINT), CAST(2 AS DECIMAL(10,0)))):decimal(20,0)>
 -- !query output
 1
 
@@ -369,7 +369,7 @@ struct<(IF(true, CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)), CAST(CAST(2 AS DECIMA
 -- !query
 SELECT IF(true, cast(1 as bigint), cast(2 as string)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS BIGINT) AS STRING), CAST(2 AS STRING))):string>
+struct<(IF(true, CAST(1 AS BIGINT), CAST(2 AS STRING))):string>
 -- !query output
 1
 
@@ -413,7 +413,7 @@ cannot resolve '(IF(true, CAST(1 AS BIGINT), CAST('2017-12-11 09:30:00' AS DATE)
 -- !query
 SELECT IF(true, cast(1 as float), cast(2 as tinyint)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS FLOAT), CAST(CAST(2 AS TINYINT) AS FLOAT))):float>
+struct<(IF(true, CAST(1 AS FLOAT), CAST(2 AS TINYINT))):float>
 -- !query output
 1.0
 
@@ -421,7 +421,7 @@ struct<(IF(true, CAST(1 AS FLOAT), CAST(CAST(2 AS TINYINT) AS FLOAT))):float>
 -- !query
 SELECT IF(true, cast(1 as float), cast(2 as smallint)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS FLOAT), CAST(CAST(2 AS SMALLINT) AS FLOAT))):float>
+struct<(IF(true, CAST(1 AS FLOAT), CAST(2 AS SMALLINT))):float>
 -- !query output
 1.0
 
@@ -429,7 +429,7 @@ struct<(IF(true, CAST(1 AS FLOAT), CAST(CAST(2 AS SMALLINT) AS FLOAT))):float>
 -- !query
 SELECT IF(true, cast(1 as float), cast(2 as int)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS FLOAT), CAST(CAST(2 AS INT) AS FLOAT))):float>
+struct<(IF(true, CAST(1 AS FLOAT), CAST(2 AS INT))):float>
 -- !query output
 1.0
 
@@ -437,7 +437,7 @@ struct<(IF(true, CAST(1 AS FLOAT), CAST(CAST(2 AS INT) AS FLOAT))):float>
 -- !query
 SELECT IF(true, cast(1 as float), cast(2 as bigint)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS FLOAT), CAST(CAST(2 AS BIGINT) AS FLOAT))):float>
+struct<(IF(true, CAST(1 AS FLOAT), CAST(2 AS BIGINT))):float>
 -- !query output
 1.0
 
@@ -453,7 +453,7 @@ struct<(IF(true, CAST(1 AS FLOAT), CAST(2 AS FLOAT))):float>
 -- !query
 SELECT IF(true, cast(1 as float), cast(2 as double)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS FLOAT) AS DOUBLE), CAST(2 AS DOUBLE))):double>
+struct<(IF(true, CAST(1 AS FLOAT), CAST(2 AS DOUBLE))):double>
 -- !query output
 1.0
 
@@ -461,7 +461,7 @@ struct<(IF(true, CAST(CAST(1 AS FLOAT) AS DOUBLE), CAST(2 AS DOUBLE))):double>
 -- !query
 SELECT IF(true, cast(1 as float), cast(2 as decimal(10, 0))) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS FLOAT) AS DOUBLE), CAST(CAST(2 AS DECIMAL(10,0)) AS DOUBLE))):double>
+struct<(IF(true, CAST(1 AS FLOAT), CAST(2 AS DECIMAL(10,0)))):double>
 -- !query output
 1.0
 
@@ -469,7 +469,7 @@ struct<(IF(true, CAST(CAST(1 AS FLOAT) AS DOUBLE), CAST(CAST(2 AS DECIMAL(10,0))
 -- !query
 SELECT IF(true, cast(1 as float), cast(2 as string)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS FLOAT) AS STRING), CAST(2 AS STRING))):string>
+struct<(IF(true, CAST(1 AS FLOAT), CAST(2 AS STRING))):string>
 -- !query output
 1.0
 
@@ -513,7 +513,7 @@ cannot resolve '(IF(true, CAST(1 AS FLOAT), CAST('2017-12-11 09:30:00' AS DATE))
 -- !query
 SELECT IF(true, cast(1 as double), cast(2 as tinyint)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS DOUBLE), CAST(CAST(2 AS TINYINT) AS DOUBLE))):double>
+struct<(IF(true, CAST(1 AS DOUBLE), CAST(2 AS TINYINT))):double>
 -- !query output
 1.0
 
@@ -521,7 +521,7 @@ struct<(IF(true, CAST(1 AS DOUBLE), CAST(CAST(2 AS TINYINT) AS DOUBLE))):double>
 -- !query
 SELECT IF(true, cast(1 as double), cast(2 as smallint)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS DOUBLE), CAST(CAST(2 AS SMALLINT) AS DOUBLE))):double>
+struct<(IF(true, CAST(1 AS DOUBLE), CAST(2 AS SMALLINT))):double>
 -- !query output
 1.0
 
@@ -529,7 +529,7 @@ struct<(IF(true, CAST(1 AS DOUBLE), CAST(CAST(2 AS SMALLINT) AS DOUBLE))):double
 -- !query
 SELECT IF(true, cast(1 as double), cast(2 as int)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS DOUBLE), CAST(CAST(2 AS INT) AS DOUBLE))):double>
+struct<(IF(true, CAST(1 AS DOUBLE), CAST(2 AS INT))):double>
 -- !query output
 1.0
 
@@ -537,7 +537,7 @@ struct<(IF(true, CAST(1 AS DOUBLE), CAST(CAST(2 AS INT) AS DOUBLE))):double>
 -- !query
 SELECT IF(true, cast(1 as double), cast(2 as bigint)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS DOUBLE), CAST(CAST(2 AS BIGINT) AS DOUBLE))):double>
+struct<(IF(true, CAST(1 AS DOUBLE), CAST(2 AS BIGINT))):double>
 -- !query output
 1.0
 
@@ -545,7 +545,7 @@ struct<(IF(true, CAST(1 AS DOUBLE), CAST(CAST(2 AS BIGINT) AS DOUBLE))):double>
 -- !query
 SELECT IF(true, cast(1 as double), cast(2 as float)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS DOUBLE), CAST(CAST(2 AS FLOAT) AS DOUBLE))):double>
+struct<(IF(true, CAST(1 AS DOUBLE), CAST(2 AS FLOAT))):double>
 -- !query output
 1.0
 
@@ -561,7 +561,7 @@ struct<(IF(true, CAST(1 AS DOUBLE), CAST(2 AS DOUBLE))):double>
 -- !query
 SELECT IF(true, cast(1 as double), cast(2 as decimal(10, 0))) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS DOUBLE), CAST(CAST(2 AS DECIMAL(10,0)) AS DOUBLE))):double>
+struct<(IF(true, CAST(1 AS DOUBLE), CAST(2 AS DECIMAL(10,0)))):double>
 -- !query output
 1.0
 
@@ -569,7 +569,7 @@ struct<(IF(true, CAST(1 AS DOUBLE), CAST(CAST(2 AS DECIMAL(10,0)) AS DOUBLE))):d
 -- !query
 SELECT IF(true, cast(1 as double), cast(2 as string)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS DOUBLE) AS STRING), CAST(2 AS STRING))):string>
+struct<(IF(true, CAST(1 AS DOUBLE), CAST(2 AS STRING))):string>
 -- !query output
 1.0
 
@@ -613,7 +613,7 @@ cannot resolve '(IF(true, CAST(1 AS DOUBLE), CAST('2017-12-11 09:30:00' AS DATE)
 -- !query
 SELECT IF(true, cast(1 as decimal(10, 0)), cast(2 as tinyint)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS DECIMAL(10,0)), CAST(CAST(2 AS TINYINT) AS DECIMAL(10,0)))):decimal(10,0)>
+struct<(IF(true, CAST(1 AS DECIMAL(10,0)), CAST(2 AS TINYINT))):decimal(10,0)>
 -- !query output
 1
 
@@ -621,7 +621,7 @@ struct<(IF(true, CAST(1 AS DECIMAL(10,0)), CAST(CAST(2 AS TINYINT) AS DECIMAL(10
 -- !query
 SELECT IF(true, cast(1 as decimal(10, 0)), cast(2 as smallint)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS DECIMAL(10,0)), CAST(CAST(2 AS SMALLINT) AS DECIMAL(10,0)))):decimal(10,0)>
+struct<(IF(true, CAST(1 AS DECIMAL(10,0)), CAST(2 AS SMALLINT))):decimal(10,0)>
 -- !query output
 1
 
@@ -629,7 +629,7 @@ struct<(IF(true, CAST(1 AS DECIMAL(10,0)), CAST(CAST(2 AS SMALLINT) AS DECIMAL(1
 -- !query
 SELECT IF(true, cast(1 as decimal(10, 0)), cast(2 as int)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS DECIMAL(10,0)), CAST(CAST(2 AS INT) AS DECIMAL(10,0)))):decimal(10,0)>
+struct<(IF(true, CAST(1 AS DECIMAL(10,0)), CAST(2 AS INT))):decimal(10,0)>
 -- !query output
 1
 
@@ -637,7 +637,7 @@ struct<(IF(true, CAST(1 AS DECIMAL(10,0)), CAST(CAST(2 AS INT) AS DECIMAL(10,0))
 -- !query
 SELECT IF(true, cast(1 as decimal(10, 0)), cast(2 as bigint)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)), CAST(CAST(2 AS BIGINT) AS DECIMAL(20,0)))):decimal(20,0)>
+struct<(IF(true, CAST(1 AS DECIMAL(10,0)), CAST(2 AS BIGINT))):decimal(20,0)>
 -- !query output
 1
 
@@ -645,7 +645,7 @@ struct<(IF(true, CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)), CAST(CAST(2 AS
 -- !query
 SELECT IF(true, cast(1 as decimal(10, 0)), cast(2 as float)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE), CAST(CAST(2 AS FLOAT) AS DOUBLE))):double>
+struct<(IF(true, CAST(1 AS DECIMAL(10,0)), CAST(2 AS FLOAT))):double>
 -- !query output
 1.0
 
@@ -653,7 +653,7 @@ struct<(IF(true, CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE), CAST(CAST(2 AS FLOAT)
 -- !query
 SELECT IF(true, cast(1 as decimal(10, 0)), cast(2 as double)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE), CAST(2 AS DOUBLE))):double>
+struct<(IF(true, CAST(1 AS DECIMAL(10,0)), CAST(2 AS DOUBLE))):double>
 -- !query output
 1.0
 
@@ -669,7 +669,7 @@ struct<(IF(true, CAST(1 AS DECIMAL(10,0)), CAST(2 AS DECIMAL(10,0)))):decimal(10
 -- !query
 SELECT IF(true, cast(1 as decimal(10, 0)), cast(2 as string)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(1 AS DECIMAL(10,0)) AS STRING), CAST(2 AS STRING))):string>
+struct<(IF(true, CAST(1 AS DECIMAL(10,0)), CAST(2 AS STRING))):string>
 -- !query output
 1
 
@@ -713,7 +713,7 @@ cannot resolve '(IF(true, CAST(1 AS DECIMAL(10,0)), CAST('2017-12-11 09:30:00' A
 -- !query
 SELECT IF(true, cast(1 as string), cast(2 as tinyint)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS STRING), CAST(CAST(2 AS TINYINT) AS STRING))):string>
+struct<(IF(true, CAST(1 AS STRING), CAST(2 AS TINYINT))):string>
 -- !query output
 1
 
@@ -721,7 +721,7 @@ struct<(IF(true, CAST(1 AS STRING), CAST(CAST(2 AS TINYINT) AS STRING))):string>
 -- !query
 SELECT IF(true, cast(1 as string), cast(2 as smallint)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS STRING), CAST(CAST(2 AS SMALLINT) AS STRING))):string>
+struct<(IF(true, CAST(1 AS STRING), CAST(2 AS SMALLINT))):string>
 -- !query output
 1
 
@@ -729,7 +729,7 @@ struct<(IF(true, CAST(1 AS STRING), CAST(CAST(2 AS SMALLINT) AS STRING))):string
 -- !query
 SELECT IF(true, cast(1 as string), cast(2 as int)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS STRING), CAST(CAST(2 AS INT) AS STRING))):string>
+struct<(IF(true, CAST(1 AS STRING), CAST(2 AS INT))):string>
 -- !query output
 1
 
@@ -737,7 +737,7 @@ struct<(IF(true, CAST(1 AS STRING), CAST(CAST(2 AS INT) AS STRING))):string>
 -- !query
 SELECT IF(true, cast(1 as string), cast(2 as bigint)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS STRING), CAST(CAST(2 AS BIGINT) AS STRING))):string>
+struct<(IF(true, CAST(1 AS STRING), CAST(2 AS BIGINT))):string>
 -- !query output
 1
 
@@ -745,7 +745,7 @@ struct<(IF(true, CAST(1 AS STRING), CAST(CAST(2 AS BIGINT) AS STRING))):string>
 -- !query
 SELECT IF(true, cast(1 as string), cast(2 as float)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS STRING), CAST(CAST(2 AS FLOAT) AS STRING))):string>
+struct<(IF(true, CAST(1 AS STRING), CAST(2 AS FLOAT))):string>
 -- !query output
 1
 
@@ -753,7 +753,7 @@ struct<(IF(true, CAST(1 AS STRING), CAST(CAST(2 AS FLOAT) AS STRING))):string>
 -- !query
 SELECT IF(true, cast(1 as string), cast(2 as double)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS STRING), CAST(CAST(2 AS DOUBLE) AS STRING))):string>
+struct<(IF(true, CAST(1 AS STRING), CAST(2 AS DOUBLE))):string>
 -- !query output
 1
 
@@ -761,7 +761,7 @@ struct<(IF(true, CAST(1 AS STRING), CAST(CAST(2 AS DOUBLE) AS STRING))):string>
 -- !query
 SELECT IF(true, cast(1 as string), cast(2 as decimal(10, 0))) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS STRING), CAST(CAST(2 AS DECIMAL(10,0)) AS STRING))):string>
+struct<(IF(true, CAST(1 AS STRING), CAST(2 AS DECIMAL(10,0)))):string>
 -- !query output
 1
 
@@ -795,7 +795,7 @@ cannot resolve '(IF(true, CAST(1 AS STRING), CAST(2 AS BOOLEAN)))' due to data t
 -- !query
 SELECT IF(true, cast(1 as string), cast('2017-12-11 09:30:00.0' as timestamp)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS STRING), CAST(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) AS STRING))):string>
+struct<(IF(true, CAST(1 AS STRING), CAST(2017-12-11 09:30:00.0 AS TIMESTAMP))):string>
 -- !query output
 1
 
@@ -803,7 +803,7 @@ struct<(IF(true, CAST(1 AS STRING), CAST(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP
 -- !query
 SELECT IF(true, cast(1 as string), cast('2017-12-11 09:30:00' as date)) FROM t
 -- !query schema
-struct<(IF(true, CAST(1 AS STRING), CAST(CAST(2017-12-11 09:30:00 AS DATE) AS STRING))):string>
+struct<(IF(true, CAST(1 AS STRING), CAST(2017-12-11 09:30:00 AS DATE))):string>
 -- !query output
 1
 
@@ -1088,7 +1088,7 @@ cannot resolve '(IF(true, CAST('2017-12-12 09:30:00.0' AS TIMESTAMP), CAST(2 AS
 -- !query
 SELECT IF(true, cast('2017-12-12 09:30:00.0' as timestamp), cast(2 as string)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(2017-12-12 09:30:00.0 AS TIMESTAMP) AS STRING), CAST(2 AS STRING))):string>
+struct<(IF(true, CAST(2017-12-12 09:30:00.0 AS TIMESTAMP), CAST(2 AS STRING))):string>
 -- !query output
 2017-12-12 09:30:00
 
@@ -1122,7 +1122,7 @@ struct<(IF(true, CAST(2017-12-12 09:30:00.0 AS TIMESTAMP), CAST(2017-12-11 09:30
 -- !query
 SELECT IF(true, cast('2017-12-12 09:30:00.0' as timestamp), cast('2017-12-11 09:30:00' as date)) FROM t
 -- !query schema
-struct<(IF(true, CAST(2017-12-12 09:30:00.0 AS TIMESTAMP), CAST(CAST(2017-12-11 09:30:00 AS DATE) AS TIMESTAMP))):timestamp>
+struct<(IF(true, CAST(2017-12-12 09:30:00.0 AS TIMESTAMP), CAST(2017-12-11 09:30:00 AS DATE))):timestamp>
 -- !query output
 2017-12-12 09:30:00
 
@@ -1193,7 +1193,7 @@ cannot resolve '(IF(true, CAST('2017-12-12 09:30:00' AS DATE), CAST(2 AS DECIMAL
 -- !query
 SELECT IF(true, cast('2017-12-12 09:30:00' as date), cast(2 as string)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(2017-12-12 09:30:00 AS DATE) AS STRING), CAST(2 AS STRING))):string>
+struct<(IF(true, CAST(2017-12-12 09:30:00 AS DATE), CAST(2 AS STRING))):string>
 -- !query output
 2017-12-12
 
@@ -1219,7 +1219,7 @@ cannot resolve '(IF(true, CAST('2017-12-12 09:30:00' AS DATE), CAST(2 AS BOOLEAN
 -- !query
 SELECT IF(true, cast('2017-12-12 09:30:00' as date), cast('2017-12-11 09:30:00.0' as timestamp)) FROM t
 -- !query schema
-struct<(IF(true, CAST(CAST(2017-12-12 09:30:00 AS DATE) AS TIMESTAMP), CAST(2017-12-11 09:30:00.0 AS TIMESTAMP))):timestamp>
+struct<(IF(true, CAST(2017-12-12 09:30:00 AS DATE), CAST(2017-12-11 09:30:00.0 AS TIMESTAMP))):timestamp>
 -- !query output
 2017-12-12 00:00:00
 
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/implicitTypeCasts.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/implicitTypeCasts.sql.out
index e47decbd33920..4e70f8dd5a6a0 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/implicitTypeCasts.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/implicitTypeCasts.sql.out
@@ -13,7 +13,7 @@ struct<>
 -- !query
 SELECT 1 + '2' FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) + CAST(2 AS DOUBLE)):double>
+struct<(1 + 2):double>
 -- !query output
 3.0
 
@@ -21,7 +21,7 @@ struct<(CAST(1 AS DOUBLE) + CAST(2 AS DOUBLE)):double>
 -- !query
 SELECT 1 - '2' FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) - CAST(2 AS DOUBLE)):double>
+struct<(1 - 2):double>
 -- !query output
 -1.0
 
@@ -29,7 +29,7 @@ struct<(CAST(1 AS DOUBLE) - CAST(2 AS DOUBLE)):double>
 -- !query
 SELECT 1 * '2' FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) * CAST(2 AS DOUBLE)):double>
+struct<(1 * 2):double>
 -- !query output
 2.0
 
@@ -37,7 +37,7 @@ struct<(CAST(1 AS DOUBLE) * CAST(2 AS DOUBLE)):double>
 -- !query
 SELECT 4 / '2' FROM t
 -- !query schema
-struct<(CAST(4 AS DOUBLE) / CAST(CAST(2 AS DOUBLE) AS DOUBLE)):double>
+struct<(4 / 2):double>
 -- !query output
 2.0
 
@@ -45,7 +45,7 @@ struct<(CAST(4 AS DOUBLE) / CAST(CAST(2 AS DOUBLE) AS DOUBLE)):double>
 -- !query
 SELECT 1.1 + '2' FROM t
 -- !query schema
-struct<(CAST(1.1 AS DOUBLE) + CAST(2 AS DOUBLE)):double>
+struct<(1.1 + 2):double>
 -- !query output
 3.1
 
@@ -53,7 +53,7 @@ struct<(CAST(1.1 AS DOUBLE) + CAST(2 AS DOUBLE)):double>
 -- !query
 SELECT 1.1 - '2' FROM t
 -- !query schema
-struct<(CAST(1.1 AS DOUBLE) - CAST(2 AS DOUBLE)):double>
+struct<(1.1 - 2):double>
 -- !query output
 -0.8999999999999999
 
@@ -61,7 +61,7 @@ struct<(CAST(1.1 AS DOUBLE) - CAST(2 AS DOUBLE)):double>
 -- !query
 SELECT 1.1 * '2' FROM t
 -- !query schema
-struct<(CAST(1.1 AS DOUBLE) * CAST(2 AS DOUBLE)):double>
+struct<(1.1 * 2):double>
 -- !query output
 2.2
 
@@ -69,7 +69,7 @@ struct<(CAST(1.1 AS DOUBLE) * CAST(2 AS DOUBLE)):double>
 -- !query
 SELECT 4.4 / '2' FROM t
 -- !query schema
-struct<(CAST(4.4 AS DOUBLE) / CAST(2 AS DOUBLE)):double>
+struct<(4.4 / 2):double>
 -- !query output
 2.2
 
@@ -77,7 +77,7 @@ struct<(CAST(4.4 AS DOUBLE) / CAST(2 AS DOUBLE)):double>
 -- !query
 SELECT 1.1 + '2.2' FROM t
 -- !query schema
-struct<(CAST(1.1 AS DOUBLE) + CAST(2.2 AS DOUBLE)):double>
+struct<(1.1 + 2.2):double>
 -- !query output
 3.3000000000000003
 
@@ -85,7 +85,7 @@ struct<(CAST(1.1 AS DOUBLE) + CAST(2.2 AS DOUBLE)):double>
 -- !query
 SELECT 1.1 - '2.2' FROM t
 -- !query schema
-struct<(CAST(1.1 AS DOUBLE) - CAST(2.2 AS DOUBLE)):double>
+struct<(1.1 - 2.2):double>
 -- !query output
 -1.1
 
@@ -93,7 +93,7 @@ struct<(CAST(1.1 AS DOUBLE) - CAST(2.2 AS DOUBLE)):double>
 -- !query
 SELECT 1.1 * '2.2' FROM t
 -- !query schema
-struct<(CAST(1.1 AS DOUBLE) * CAST(2.2 AS DOUBLE)):double>
+struct<(1.1 * 2.2):double>
 -- !query output
 2.4200000000000004
 
@@ -101,7 +101,7 @@ struct<(CAST(1.1 AS DOUBLE) * CAST(2.2 AS DOUBLE)):double>
 -- !query
 SELECT 4.4 / '2.2' FROM t
 -- !query schema
-struct<(CAST(4.4 AS DOUBLE) / CAST(2.2 AS DOUBLE)):double>
+struct<(4.4 / 2.2):double>
 -- !query output
 2.0
 
@@ -109,7 +109,7 @@ struct<(CAST(4.4 AS DOUBLE) / CAST(2.2 AS DOUBLE)):double>
 -- !query
 SELECT '$' || cast(1 as smallint) || '$' FROM t
 -- !query schema
-struct<concat(concat($, CAST(CAST(1 AS SMALLINT) AS STRING)), $):string>
+struct<concat(concat($, CAST(1 AS SMALLINT)), $):string>
 -- !query output
 $1$
 
@@ -117,7 +117,7 @@ $1$
 -- !query
 SELECT '$' || 1 || '$' FROM t
 -- !query schema
-struct<concat(concat($, CAST(1 AS STRING)), $):string>
+struct<concat(concat($, 1), $):string>
 -- !query output
 $1$
 
@@ -125,7 +125,7 @@ $1$
 -- !query
 SELECT '$' || cast(1 as bigint) || '$' FROM t
 -- !query schema
-struct<concat(concat($, CAST(CAST(1 AS BIGINT) AS STRING)), $):string>
+struct<concat(concat($, CAST(1 AS BIGINT)), $):string>
 -- !query output
 $1$
 
@@ -133,7 +133,7 @@ $1$
 -- !query
 SELECT '$' || cast(1.1 as float) || '$' FROM t
 -- !query schema
-struct<concat(concat($, CAST(CAST(1.1 AS FLOAT) AS STRING)), $):string>
+struct<concat(concat($, CAST(1.1 AS FLOAT)), $):string>
 -- !query output
 $1.1$
 
@@ -141,7 +141,7 @@ $1.1$
 -- !query
 SELECT '$' || cast(1.1 as double) || '$' FROM t
 -- !query schema
-struct<concat(concat($, CAST(CAST(1.1 AS DOUBLE) AS STRING)), $):string>
+struct<concat(concat($, CAST(1.1 AS DOUBLE)), $):string>
 -- !query output
 $1.1$
 
@@ -149,7 +149,7 @@ $1.1$
 -- !query
 SELECT '$' || 1.1 || '$' FROM t
 -- !query schema
-struct<concat(concat($, CAST(1.1 AS STRING)), $):string>
+struct<concat(concat($, 1.1), $):string>
 -- !query output
 $1.1$
 
@@ -157,7 +157,7 @@ $1.1$
 -- !query
 SELECT '$' || cast(1.1 as decimal(8,3)) || '$' FROM t
 -- !query schema
-struct<concat(concat($, CAST(CAST(1.1 AS DECIMAL(8,3)) AS STRING)), $):string>
+struct<concat(concat($, CAST(1.1 AS DECIMAL(8,3))), $):string>
 -- !query output
 $1.100$
 
@@ -173,7 +173,7 @@ $abcd$
 -- !query
 SELECT '$' || date('1996-09-09') || '$' FROM t
 -- !query schema
-struct<concat(concat($, CAST(CAST(1996-09-09 AS DATE) AS STRING)), $):string>
+struct<concat(concat($, 1996-09-09), $):string>
 -- !query output
 $1996-09-09$
 
@@ -181,7 +181,7 @@ $1996-09-09$
 -- !query
 SELECT '$' || timestamp('1996-09-09 10:11:12.4' )|| '$' FROM t
 -- !query schema
-struct<concat(concat($, CAST(CAST(1996-09-09 10:11:12.4 AS TIMESTAMP) AS STRING)), $):string>
+struct<concat(concat($, 1996-09-09 10:11:12.4), $):string>
 -- !query output
 $1996-09-09 10:11:12.4$
 
@@ -189,7 +189,7 @@ $1996-09-09 10:11:12.4$
 -- !query
 SELECT length(cast(1 as smallint)) FROM t
 -- !query schema
-struct<length(CAST(CAST(1 AS SMALLINT) AS STRING)):int>
+struct<length(CAST(1 AS SMALLINT)):int>
 -- !query output
 1
 
@@ -197,7 +197,7 @@ struct<length(CAST(CAST(1 AS SMALLINT) AS STRING)):int>
 -- !query
 SELECT length(cast(1 as int)) FROM t
 -- !query schema
-struct<length(CAST(CAST(1 AS INT) AS STRING)):int>
+struct<length(CAST(1 AS INT)):int>
 -- !query output
 1
 
@@ -205,7 +205,7 @@ struct<length(CAST(CAST(1 AS INT) AS STRING)):int>
 -- !query
 SELECT length(cast(1 as bigint)) FROM t
 -- !query schema
-struct<length(CAST(CAST(1 AS BIGINT) AS STRING)):int>
+struct<length(CAST(1 AS BIGINT)):int>
 -- !query output
 1
 
@@ -213,7 +213,7 @@ struct<length(CAST(CAST(1 AS BIGINT) AS STRING)):int>
 -- !query
 SELECT length(cast(1.1 as float)) FROM t
 -- !query schema
-struct<length(CAST(CAST(1.1 AS FLOAT) AS STRING)):int>
+struct<length(CAST(1.1 AS FLOAT)):int>
 -- !query output
 3
 
@@ -221,7 +221,7 @@ struct<length(CAST(CAST(1.1 AS FLOAT) AS STRING)):int>
 -- !query
 SELECT length(cast(1.1 as double)) FROM t
 -- !query schema
-struct<length(CAST(CAST(1.1 AS DOUBLE) AS STRING)):int>
+struct<length(CAST(1.1 AS DOUBLE)):int>
 -- !query output
 3
 
@@ -229,7 +229,7 @@ struct<length(CAST(CAST(1.1 AS DOUBLE) AS STRING)):int>
 -- !query
 SELECT length(1.1) FROM t
 -- !query schema
-struct<length(CAST(1.1 AS STRING)):int>
+struct<length(1.1):int>
 -- !query output
 3
 
@@ -237,7 +237,7 @@ struct<length(CAST(1.1 AS STRING)):int>
 -- !query
 SELECT length(cast(1.1 as decimal(8,3))) FROM t
 -- !query schema
-struct<length(CAST(CAST(1.1 AS DECIMAL(8,3)) AS STRING)):int>
+struct<length(CAST(1.1 AS DECIMAL(8,3))):int>
 -- !query output
 5
 
@@ -253,7 +253,7 @@ struct<length(four):int>
 -- !query
 SELECT length(date('1996-09-10')) FROM t
 -- !query schema
-struct<length(CAST(CAST(1996-09-10 AS DATE) AS STRING)):int>
+struct<length(1996-09-10):int>
 -- !query output
 10
 
@@ -261,7 +261,7 @@ struct<length(CAST(CAST(1996-09-10 AS DATE) AS STRING)):int>
 -- !query
 SELECT length(timestamp('1996-09-10 10:11:12.4')) FROM t
 -- !query schema
-struct<length(CAST(CAST(1996-09-10 10:11:12.4 AS TIMESTAMP) AS STRING)):int>
+struct<length(1996-09-10 10:11:12.4):int>
 -- !query output
 21
 
@@ -269,7 +269,7 @@ struct<length(CAST(CAST(1996-09-10 10:11:12.4 AS TIMESTAMP) AS STRING)):int>
 -- !query
 SELECT year( '1996-01-10') FROM t
 -- !query schema
-struct<year(CAST(1996-01-10 AS DATE)):int>
+struct<year(1996-01-10):int>
 -- !query output
 1996
 
@@ -277,7 +277,7 @@ struct<year(CAST(1996-01-10 AS DATE)):int>
 -- !query
 SELECT month( '1996-01-10') FROM t
 -- !query schema
-struct<month(CAST(1996-01-10 AS DATE)):int>
+struct<month(1996-01-10):int>
 -- !query output
 1
 
@@ -285,7 +285,7 @@ struct<month(CAST(1996-01-10 AS DATE)):int>
 -- !query
 SELECT day( '1996-01-10') FROM t
 -- !query schema
-struct<day(CAST(1996-01-10 AS DATE)):int>
+struct<day(1996-01-10):int>
 -- !query output
 10
 
@@ -293,7 +293,7 @@ struct<day(CAST(1996-01-10 AS DATE)):int>
 -- !query
 SELECT hour( '10:11:12') FROM t
 -- !query schema
-struct<hour(CAST(10:11:12 AS TIMESTAMP)):int>
+struct<hour(10:11:12):int>
 -- !query output
 10
 
@@ -301,7 +301,7 @@ struct<hour(CAST(10:11:12 AS TIMESTAMP)):int>
 -- !query
 SELECT minute( '10:11:12') FROM t
 -- !query schema
-struct<minute(CAST(10:11:12 AS TIMESTAMP)):int>
+struct<minute(10:11:12):int>
 -- !query output
 11
 
@@ -309,7 +309,7 @@ struct<minute(CAST(10:11:12 AS TIMESTAMP)):int>
 -- !query
 SELECT second( '10:11:12') FROM t
 -- !query schema
-struct<second(CAST(10:11:12 AS TIMESTAMP)):int>
+struct<second(10:11:12):int>
 -- !query output
 12
 
@@ -317,7 +317,7 @@ struct<second(CAST(10:11:12 AS TIMESTAMP)):int>
 -- !query
 select 1 like '%' FROM t
 -- !query schema
-struct<CAST(1 AS STRING) LIKE %:boolean>
+struct<1 LIKE %:boolean>
 -- !query output
 true
 
@@ -325,7 +325,7 @@ true
 -- !query
 select date('1996-09-10') like '19%' FROM t
 -- !query schema
-struct<CAST(CAST(1996-09-10 AS DATE) AS STRING) LIKE 19%:boolean>
+struct<1996-09-10 LIKE 19%:boolean>
 -- !query output
 true
 
@@ -333,7 +333,7 @@ true
 -- !query
 select '1' like 1 FROM t
 -- !query schema
-struct<1 LIKE CAST(1 AS STRING):boolean>
+struct<1 LIKE 1:boolean>
 -- !query output
 true
 
@@ -341,7 +341,7 @@ true
 -- !query
 select '1 ' like 1 FROM t
 -- !query schema
-struct<1  LIKE CAST(1 AS STRING):boolean>
+struct<1  LIKE 1:boolean>
 -- !query output
 false
 
@@ -349,6 +349,6 @@ false
 -- !query
 select '1996-09-10' like date('1996-09-10') FROM t
 -- !query schema
-struct<1996-09-10 LIKE CAST(CAST(1996-09-10 AS DATE) AS STRING):boolean>
+struct<1996-09-10 LIKE 1996-09-10:boolean>
 -- !query output
 true
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/inConversion.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/inConversion.sql.out
index 21d0a0e0fef4e..7f87029a2ea75 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/inConversion.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/inConversion.sql.out
@@ -21,7 +21,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) in (cast(1 as smallint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS SMALLINT) IN (CAST(CAST(1 AS SMALLINT) AS SMALLINT))):boolean>
+struct<(CAST(1 AS TINYINT) IN (CAST(1 AS SMALLINT))):boolean>
 -- !query output
 true
 
@@ -29,7 +29,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) in (cast(1 as int)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS INT) IN (CAST(CAST(1 AS INT) AS INT))):boolean>
+struct<(CAST(1 AS TINYINT) IN (CAST(1 AS INT))):boolean>
 -- !query output
 true
 
@@ -37,7 +37,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) in (cast(1 as bigint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS BIGINT) IN (CAST(CAST(1 AS BIGINT) AS BIGINT))):boolean>
+struct<(CAST(1 AS TINYINT) IN (CAST(1 AS BIGINT))):boolean>
 -- !query output
 true
 
@@ -45,7 +45,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) in (cast(1 as float)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS FLOAT) IN (CAST(CAST(1 AS FLOAT) AS FLOAT))):boolean>
+struct<(CAST(1 AS TINYINT) IN (CAST(1 AS FLOAT))):boolean>
 -- !query output
 true
 
@@ -53,7 +53,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) in (cast(1 as double)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DOUBLE) IN (CAST(CAST(1 AS DOUBLE) AS DOUBLE))):boolean>
+struct<(CAST(1 AS TINYINT) IN (CAST(1 AS DOUBLE))):boolean>
 -- !query output
 true
 
@@ -61,7 +61,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) in (cast(1 as decimal(10, 0))) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DECIMAL(10,0)) IN (CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)))):boolean>
+struct<(CAST(1 AS TINYINT) IN (CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 true
 
@@ -69,7 +69,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) in (cast(1 as string)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS STRING) IN (CAST(CAST(1 AS STRING) AS STRING))):boolean>
+struct<(CAST(1 AS TINYINT) IN (CAST(1 AS STRING))):boolean>
 -- !query output
 true
 
@@ -113,7 +113,7 @@ cannot resolve '(CAST(1 AS TINYINT) IN (CAST('2017-12-11 09:30:00' AS DATE)))' d
 -- !query
 SELECT cast(1 as smallint) in (cast(1 as tinyint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS SMALLINT) IN (CAST(CAST(1 AS TINYINT) AS SMALLINT))):boolean>
+struct<(CAST(1 AS SMALLINT) IN (CAST(1 AS TINYINT))):boolean>
 -- !query output
 true
 
@@ -129,7 +129,7 @@ true
 -- !query
 SELECT cast(1 as smallint) in (cast(1 as int)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS INT) IN (CAST(CAST(1 AS INT) AS INT))):boolean>
+struct<(CAST(1 AS SMALLINT) IN (CAST(1 AS INT))):boolean>
 -- !query output
 true
 
@@ -137,7 +137,7 @@ true
 -- !query
 SELECT cast(1 as smallint) in (cast(1 as bigint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS BIGINT) IN (CAST(CAST(1 AS BIGINT) AS BIGINT))):boolean>
+struct<(CAST(1 AS SMALLINT) IN (CAST(1 AS BIGINT))):boolean>
 -- !query output
 true
 
@@ -145,7 +145,7 @@ true
 -- !query
 SELECT cast(1 as smallint) in (cast(1 as float)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS FLOAT) IN (CAST(CAST(1 AS FLOAT) AS FLOAT))):boolean>
+struct<(CAST(1 AS SMALLINT) IN (CAST(1 AS FLOAT))):boolean>
 -- !query output
 true
 
@@ -153,7 +153,7 @@ true
 -- !query
 SELECT cast(1 as smallint) in (cast(1 as double)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DOUBLE) IN (CAST(CAST(1 AS DOUBLE) AS DOUBLE))):boolean>
+struct<(CAST(1 AS SMALLINT) IN (CAST(1 AS DOUBLE))):boolean>
 -- !query output
 true
 
@@ -161,7 +161,7 @@ true
 -- !query
 SELECT cast(1 as smallint) in (cast(1 as decimal(10, 0))) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DECIMAL(10,0)) IN (CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)))):boolean>
+struct<(CAST(1 AS SMALLINT) IN (CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 true
 
@@ -169,7 +169,7 @@ true
 -- !query
 SELECT cast(1 as smallint) in (cast(1 as string)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS STRING) IN (CAST(CAST(1 AS STRING) AS STRING))):boolean>
+struct<(CAST(1 AS SMALLINT) IN (CAST(1 AS STRING))):boolean>
 -- !query output
 true
 
@@ -213,7 +213,7 @@ cannot resolve '(CAST(1 AS SMALLINT) IN (CAST('2017-12-11 09:30:00' AS DATE)))'
 -- !query
 SELECT cast(1 as int) in (cast(1 as tinyint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS INT) IN (CAST(CAST(1 AS TINYINT) AS INT))):boolean>
+struct<(CAST(1 AS INT) IN (CAST(1 AS TINYINT))):boolean>
 -- !query output
 true
 
@@ -221,7 +221,7 @@ true
 -- !query
 SELECT cast(1 as int) in (cast(1 as smallint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS INT) IN (CAST(CAST(1 AS SMALLINT) AS INT))):boolean>
+struct<(CAST(1 AS INT) IN (CAST(1 AS SMALLINT))):boolean>
 -- !query output
 true
 
@@ -237,7 +237,7 @@ true
 -- !query
 SELECT cast(1 as int) in (cast(1 as bigint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS BIGINT) IN (CAST(CAST(1 AS BIGINT) AS BIGINT))):boolean>
+struct<(CAST(1 AS INT) IN (CAST(1 AS BIGINT))):boolean>
 -- !query output
 true
 
@@ -245,7 +245,7 @@ true
 -- !query
 SELECT cast(1 as int) in (cast(1 as float)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS FLOAT) IN (CAST(CAST(1 AS FLOAT) AS FLOAT))):boolean>
+struct<(CAST(1 AS INT) IN (CAST(1 AS FLOAT))):boolean>
 -- !query output
 true
 
@@ -253,7 +253,7 @@ true
 -- !query
 SELECT cast(1 as int) in (cast(1 as double)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DOUBLE) IN (CAST(CAST(1 AS DOUBLE) AS DOUBLE))):boolean>
+struct<(CAST(1 AS INT) IN (CAST(1 AS DOUBLE))):boolean>
 -- !query output
 true
 
@@ -261,7 +261,7 @@ true
 -- !query
 SELECT cast(1 as int) in (cast(1 as decimal(10, 0))) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) IN (CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)))):boolean>
+struct<(CAST(1 AS INT) IN (CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 true
 
@@ -269,7 +269,7 @@ true
 -- !query
 SELECT cast(1 as int) in (cast(1 as string)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS STRING) IN (CAST(CAST(1 AS STRING) AS STRING))):boolean>
+struct<(CAST(1 AS INT) IN (CAST(1 AS STRING))):boolean>
 -- !query output
 true
 
@@ -313,7 +313,7 @@ cannot resolve '(CAST(1 AS INT) IN (CAST('2017-12-11 09:30:00' AS DATE)))' due t
 -- !query
 SELECT cast(1 as bigint) in (cast(1 as tinyint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS BIGINT) IN (CAST(CAST(1 AS TINYINT) AS BIGINT))):boolean>
+struct<(CAST(1 AS BIGINT) IN (CAST(1 AS TINYINT))):boolean>
 -- !query output
 true
 
@@ -321,7 +321,7 @@ true
 -- !query
 SELECT cast(1 as bigint) in (cast(1 as smallint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS BIGINT) IN (CAST(CAST(1 AS SMALLINT) AS BIGINT))):boolean>
+struct<(CAST(1 AS BIGINT) IN (CAST(1 AS SMALLINT))):boolean>
 -- !query output
 true
 
@@ -329,7 +329,7 @@ true
 -- !query
 SELECT cast(1 as bigint) in (cast(1 as int)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS BIGINT) IN (CAST(CAST(1 AS INT) AS BIGINT))):boolean>
+struct<(CAST(1 AS BIGINT) IN (CAST(1 AS INT))):boolean>
 -- !query output
 true
 
@@ -345,7 +345,7 @@ true
 -- !query
 SELECT cast(1 as bigint) in (cast(1 as float)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS FLOAT) IN (CAST(CAST(1 AS FLOAT) AS FLOAT))):boolean>
+struct<(CAST(1 AS BIGINT) IN (CAST(1 AS FLOAT))):boolean>
 -- !query output
 true
 
@@ -353,7 +353,7 @@ true
 -- !query
 SELECT cast(1 as bigint) in (cast(1 as double)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DOUBLE) IN (CAST(CAST(1 AS DOUBLE) AS DOUBLE))):boolean>
+struct<(CAST(1 AS BIGINT) IN (CAST(1 AS DOUBLE))):boolean>
 -- !query output
 true
 
@@ -361,7 +361,7 @@ true
 -- !query
 SELECT cast(1 as bigint) in (cast(1 as decimal(10, 0))) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) IN (CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)))):boolean>
+struct<(CAST(1 AS BIGINT) IN (CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 true
 
@@ -369,7 +369,7 @@ true
 -- !query
 SELECT cast(1 as bigint) in (cast(1 as string)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS STRING) IN (CAST(CAST(1 AS STRING) AS STRING))):boolean>
+struct<(CAST(1 AS BIGINT) IN (CAST(1 AS STRING))):boolean>
 -- !query output
 true
 
@@ -413,7 +413,7 @@ cannot resolve '(CAST(1 AS BIGINT) IN (CAST('2017-12-11 09:30:00' AS DATE)))' du
 -- !query
 SELECT cast(1 as float) in (cast(1 as tinyint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS FLOAT) IN (CAST(CAST(1 AS TINYINT) AS FLOAT))):boolean>
+struct<(CAST(1 AS FLOAT) IN (CAST(1 AS TINYINT))):boolean>
 -- !query output
 true
 
@@ -421,7 +421,7 @@ true
 -- !query
 SELECT cast(1 as float) in (cast(1 as smallint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS FLOAT) IN (CAST(CAST(1 AS SMALLINT) AS FLOAT))):boolean>
+struct<(CAST(1 AS FLOAT) IN (CAST(1 AS SMALLINT))):boolean>
 -- !query output
 true
 
@@ -429,7 +429,7 @@ true
 -- !query
 SELECT cast(1 as float) in (cast(1 as int)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS FLOAT) IN (CAST(CAST(1 AS INT) AS FLOAT))):boolean>
+struct<(CAST(1 AS FLOAT) IN (CAST(1 AS INT))):boolean>
 -- !query output
 true
 
@@ -437,7 +437,7 @@ true
 -- !query
 SELECT cast(1 as float) in (cast(1 as bigint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS FLOAT) IN (CAST(CAST(1 AS BIGINT) AS FLOAT))):boolean>
+struct<(CAST(1 AS FLOAT) IN (CAST(1 AS BIGINT))):boolean>
 -- !query output
 true
 
@@ -453,7 +453,7 @@ true
 -- !query
 SELECT cast(1 as float) in (cast(1 as double)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) IN (CAST(CAST(1 AS DOUBLE) AS DOUBLE))):boolean>
+struct<(CAST(1 AS FLOAT) IN (CAST(1 AS DOUBLE))):boolean>
 -- !query output
 true
 
@@ -461,7 +461,7 @@ true
 -- !query
 SELECT cast(1 as float) in (cast(1 as decimal(10, 0))) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) IN (CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE))):boolean>
+struct<(CAST(1 AS FLOAT) IN (CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 true
 
@@ -469,7 +469,7 @@ true
 -- !query
 SELECT cast(1 as float) in (cast(1 as string)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS STRING) IN (CAST(CAST(1 AS STRING) AS STRING))):boolean>
+struct<(CAST(1 AS FLOAT) IN (CAST(1 AS STRING))):boolean>
 -- !query output
 false
 
@@ -513,7 +513,7 @@ cannot resolve '(CAST(1 AS FLOAT) IN (CAST('2017-12-11 09:30:00' AS DATE)))' due
 -- !query
 SELECT cast(1 as double) in (cast(1 as tinyint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DOUBLE) AS DOUBLE) IN (CAST(CAST(1 AS TINYINT) AS DOUBLE))):boolean>
+struct<(CAST(1 AS DOUBLE) IN (CAST(1 AS TINYINT))):boolean>
 -- !query output
 true
 
@@ -521,7 +521,7 @@ true
 -- !query
 SELECT cast(1 as double) in (cast(1 as smallint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DOUBLE) AS DOUBLE) IN (CAST(CAST(1 AS SMALLINT) AS DOUBLE))):boolean>
+struct<(CAST(1 AS DOUBLE) IN (CAST(1 AS SMALLINT))):boolean>
 -- !query output
 true
 
@@ -529,7 +529,7 @@ true
 -- !query
 SELECT cast(1 as double) in (cast(1 as int)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DOUBLE) AS DOUBLE) IN (CAST(CAST(1 AS INT) AS DOUBLE))):boolean>
+struct<(CAST(1 AS DOUBLE) IN (CAST(1 AS INT))):boolean>
 -- !query output
 true
 
@@ -537,7 +537,7 @@ true
 -- !query
 SELECT cast(1 as double) in (cast(1 as bigint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DOUBLE) AS DOUBLE) IN (CAST(CAST(1 AS BIGINT) AS DOUBLE))):boolean>
+struct<(CAST(1 AS DOUBLE) IN (CAST(1 AS BIGINT))):boolean>
 -- !query output
 true
 
@@ -545,7 +545,7 @@ true
 -- !query
 SELECT cast(1 as double) in (cast(1 as float)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DOUBLE) AS DOUBLE) IN (CAST(CAST(1 AS FLOAT) AS DOUBLE))):boolean>
+struct<(CAST(1 AS DOUBLE) IN (CAST(1 AS FLOAT))):boolean>
 -- !query output
 true
 
@@ -561,7 +561,7 @@ true
 -- !query
 SELECT cast(1 as double) in (cast(1 as decimal(10, 0))) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DOUBLE) AS DOUBLE) IN (CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE))):boolean>
+struct<(CAST(1 AS DOUBLE) IN (CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 true
 
@@ -569,7 +569,7 @@ true
 -- !query
 SELECT cast(1 as double) in (cast(1 as string)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DOUBLE) AS STRING) IN (CAST(CAST(1 AS STRING) AS STRING))):boolean>
+struct<(CAST(1 AS DOUBLE) IN (CAST(1 AS STRING))):boolean>
 -- !query output
 false
 
@@ -613,7 +613,7 @@ cannot resolve '(CAST(1 AS DOUBLE) IN (CAST('2017-12-11 09:30:00' AS DATE)))' du
 -- !query
 SELECT cast(1 as decimal(10, 0)) in (cast(1 as tinyint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) IN (CAST(CAST(1 AS TINYINT) AS DECIMAL(10,0)))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) IN (CAST(1 AS TINYINT))):boolean>
 -- !query output
 true
 
@@ -621,7 +621,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) in (cast(1 as smallint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) IN (CAST(CAST(1 AS SMALLINT) AS DECIMAL(10,0)))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) IN (CAST(1 AS SMALLINT))):boolean>
 -- !query output
 true
 
@@ -629,7 +629,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) in (cast(1 as int)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) IN (CAST(CAST(1 AS INT) AS DECIMAL(10,0)))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) IN (CAST(1 AS INT))):boolean>
 -- !query output
 true
 
@@ -637,7 +637,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) in (cast(1 as bigint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) IN (CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) IN (CAST(1 AS BIGINT))):boolean>
 -- !query output
 true
 
@@ -645,7 +645,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) in (cast(1 as float)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) IN (CAST(CAST(1 AS FLOAT) AS DOUBLE))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) IN (CAST(1 AS FLOAT))):boolean>
 -- !query output
 true
 
@@ -653,7 +653,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) in (cast(1 as double)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) IN (CAST(CAST(1 AS DOUBLE) AS DOUBLE))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) IN (CAST(1 AS DOUBLE))):boolean>
 -- !query output
 true
 
@@ -669,7 +669,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) in (cast(1 as string)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS STRING) IN (CAST(CAST(1 AS STRING) AS STRING))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) IN (CAST(1 AS STRING))):boolean>
 -- !query output
 true
 
@@ -713,7 +713,7 @@ cannot resolve '(CAST(1 AS DECIMAL(10,0)) IN (CAST('2017-12-11 09:30:00' AS DATE
 -- !query
 SELECT cast(1 as string) in (cast(1 as tinyint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS STRING) IN (CAST(CAST(1 AS TINYINT) AS STRING))):boolean>
+struct<(CAST(1 AS STRING) IN (CAST(1 AS TINYINT))):boolean>
 -- !query output
 true
 
@@ -721,7 +721,7 @@ true
 -- !query
 SELECT cast(1 as string) in (cast(1 as smallint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS STRING) IN (CAST(CAST(1 AS SMALLINT) AS STRING))):boolean>
+struct<(CAST(1 AS STRING) IN (CAST(1 AS SMALLINT))):boolean>
 -- !query output
 true
 
@@ -729,7 +729,7 @@ true
 -- !query
 SELECT cast(1 as string) in (cast(1 as int)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS STRING) IN (CAST(CAST(1 AS INT) AS STRING))):boolean>
+struct<(CAST(1 AS STRING) IN (CAST(1 AS INT))):boolean>
 -- !query output
 true
 
@@ -737,7 +737,7 @@ true
 -- !query
 SELECT cast(1 as string) in (cast(1 as bigint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS STRING) IN (CAST(CAST(1 AS BIGINT) AS STRING))):boolean>
+struct<(CAST(1 AS STRING) IN (CAST(1 AS BIGINT))):boolean>
 -- !query output
 true
 
@@ -745,7 +745,7 @@ true
 -- !query
 SELECT cast(1 as string) in (cast(1 as float)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS STRING) IN (CAST(CAST(1 AS FLOAT) AS STRING))):boolean>
+struct<(CAST(1 AS STRING) IN (CAST(1 AS FLOAT))):boolean>
 -- !query output
 false
 
@@ -753,7 +753,7 @@ false
 -- !query
 SELECT cast(1 as string) in (cast(1 as double)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS STRING) IN (CAST(CAST(1 AS DOUBLE) AS STRING))):boolean>
+struct<(CAST(1 AS STRING) IN (CAST(1 AS DOUBLE))):boolean>
 -- !query output
 false
 
@@ -761,7 +761,7 @@ false
 -- !query
 SELECT cast(1 as string) in (cast(1 as decimal(10, 0))) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS STRING) IN (CAST(CAST(1 AS DECIMAL(10,0)) AS STRING))):boolean>
+struct<(CAST(1 AS STRING) IN (CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 true
 
@@ -795,7 +795,7 @@ cannot resolve '(CAST(1 AS STRING) IN (CAST(1 AS BOOLEAN)))' due to data type mi
 -- !query
 SELECT cast(1 as string) in (cast('2017-12-11 09:30:00.0' as timestamp)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS STRING) IN (CAST(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) AS STRING))):boolean>
+struct<(CAST(1 AS STRING) IN (CAST(2017-12-11 09:30:00.0 AS TIMESTAMP))):boolean>
 -- !query output
 false
 
@@ -803,7 +803,7 @@ false
 -- !query
 SELECT cast(1 as string) in (cast('2017-12-11 09:30:00' as date)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS STRING) IN (CAST(CAST(2017-12-11 09:30:00 AS DATE) AS STRING))):boolean>
+struct<(CAST(1 AS STRING) IN (CAST(2017-12-11 09:30:00 AS DATE))):boolean>
 -- !query output
 false
 
@@ -1088,7 +1088,7 @@ cannot resolve '(CAST('2017-12-12 09:30:00.0' AS TIMESTAMP) IN (CAST(2 AS DECIMA
 -- !query
 SELECT cast('2017-12-12 09:30:00.0' as timestamp) in (cast(2 as string)) FROM t
 -- !query schema
-struct<(CAST(CAST(2017-12-12 09:30:00.0 AS TIMESTAMP) AS STRING) IN (CAST(CAST(2 AS STRING) AS STRING))):boolean>
+struct<(CAST(2017-12-12 09:30:00.0 AS TIMESTAMP) IN (CAST(2 AS STRING))):boolean>
 -- !query output
 false
 
@@ -1122,7 +1122,7 @@ false
 -- !query
 SELECT cast('2017-12-12 09:30:00.0' as timestamp) in (cast('2017-12-11 09:30:00' as date)) FROM t
 -- !query schema
-struct<(CAST(CAST(2017-12-12 09:30:00.0 AS TIMESTAMP) AS TIMESTAMP) IN (CAST(CAST(2017-12-11 09:30:00 AS DATE) AS TIMESTAMP))):boolean>
+struct<(CAST(2017-12-12 09:30:00.0 AS TIMESTAMP) IN (CAST(2017-12-11 09:30:00 AS DATE))):boolean>
 -- !query output
 false
 
@@ -1193,7 +1193,7 @@ cannot resolve '(CAST('2017-12-12 09:30:00' AS DATE) IN (CAST(2 AS DECIMAL(10,0)
 -- !query
 SELECT cast('2017-12-12 09:30:00' as date) in (cast(2 as string)) FROM t
 -- !query schema
-struct<(CAST(CAST(2017-12-12 09:30:00 AS DATE) AS STRING) IN (CAST(CAST(2 AS STRING) AS STRING))):boolean>
+struct<(CAST(2017-12-12 09:30:00 AS DATE) IN (CAST(2 AS STRING))):boolean>
 -- !query output
 false
 
@@ -1219,7 +1219,7 @@ cannot resolve '(CAST('2017-12-12 09:30:00' AS DATE) IN (CAST(2 AS BOOLEAN)))' d
 -- !query
 SELECT cast('2017-12-12 09:30:00' as date) in (cast('2017-12-11 09:30:00.0' as timestamp)) FROM t
 -- !query schema
-struct<(CAST(CAST(2017-12-12 09:30:00 AS DATE) AS TIMESTAMP) IN (CAST(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) AS TIMESTAMP))):boolean>
+struct<(CAST(2017-12-12 09:30:00 AS DATE) IN (CAST(2017-12-11 09:30:00.0 AS TIMESTAMP))):boolean>
 -- !query output
 false
 
@@ -1243,7 +1243,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) in (cast(1 as tinyint), cast(1 as smallint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS SMALLINT) IN (CAST(CAST(1 AS TINYINT) AS SMALLINT), CAST(CAST(1 AS SMALLINT) AS SMALLINT))):boolean>
+struct<(CAST(1 AS TINYINT) IN (CAST(1 AS TINYINT), CAST(1 AS SMALLINT))):boolean>
 -- !query output
 true
 
@@ -1251,7 +1251,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) in (cast(1 as tinyint), cast(1 as int)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS INT) IN (CAST(CAST(1 AS TINYINT) AS INT), CAST(CAST(1 AS INT) AS INT))):boolean>
+struct<(CAST(1 AS TINYINT) IN (CAST(1 AS TINYINT), CAST(1 AS INT))):boolean>
 -- !query output
 true
 
@@ -1259,7 +1259,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) in (cast(1 as tinyint), cast(1 as bigint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS BIGINT) IN (CAST(CAST(1 AS TINYINT) AS BIGINT), CAST(CAST(1 AS BIGINT) AS BIGINT))):boolean>
+struct<(CAST(1 AS TINYINT) IN (CAST(1 AS TINYINT), CAST(1 AS BIGINT))):boolean>
 -- !query output
 true
 
@@ -1267,7 +1267,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) in (cast(1 as tinyint), cast(1 as float)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS FLOAT) IN (CAST(CAST(1 AS TINYINT) AS FLOAT), CAST(CAST(1 AS FLOAT) AS FLOAT))):boolean>
+struct<(CAST(1 AS TINYINT) IN (CAST(1 AS TINYINT), CAST(1 AS FLOAT))):boolean>
 -- !query output
 true
 
@@ -1275,7 +1275,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) in (cast(1 as tinyint), cast(1 as double)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DOUBLE) IN (CAST(CAST(1 AS TINYINT) AS DOUBLE), CAST(CAST(1 AS DOUBLE) AS DOUBLE))):boolean>
+struct<(CAST(1 AS TINYINT) IN (CAST(1 AS TINYINT), CAST(1 AS DOUBLE))):boolean>
 -- !query output
 true
 
@@ -1283,7 +1283,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) in (cast(1 as tinyint), cast(1 as decimal(10, 0))) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DECIMAL(10,0)) IN (CAST(CAST(1 AS TINYINT) AS DECIMAL(10,0)), CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)))):boolean>
+struct<(CAST(1 AS TINYINT) IN (CAST(1 AS TINYINT), CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 true
 
@@ -1291,7 +1291,7 @@ true
 -- !query
 SELECT cast(1 as tinyint) in (cast(1 as tinyint), cast(1 as string)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS STRING) IN (CAST(CAST(1 AS TINYINT) AS STRING), CAST(CAST(1 AS STRING) AS STRING))):boolean>
+struct<(CAST(1 AS TINYINT) IN (CAST(1 AS TINYINT), CAST(1 AS STRING))):boolean>
 -- !query output
 true
 
@@ -1335,7 +1335,7 @@ cannot resolve '(CAST(1 AS TINYINT) IN (CAST(1 AS TINYINT), CAST('2017-12-11 09:
 -- !query
 SELECT cast(1 as smallint) in (cast(1 as smallint), cast(1 as tinyint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS SMALLINT) IN (CAST(CAST(1 AS SMALLINT) AS SMALLINT), CAST(CAST(1 AS TINYINT) AS SMALLINT))):boolean>
+struct<(CAST(1 AS SMALLINT) IN (CAST(1 AS SMALLINT), CAST(1 AS TINYINT))):boolean>
 -- !query output
 true
 
@@ -1351,7 +1351,7 @@ true
 -- !query
 SELECT cast(1 as smallint) in (cast(1 as smallint), cast(1 as int)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS INT) IN (CAST(CAST(1 AS SMALLINT) AS INT), CAST(CAST(1 AS INT) AS INT))):boolean>
+struct<(CAST(1 AS SMALLINT) IN (CAST(1 AS SMALLINT), CAST(1 AS INT))):boolean>
 -- !query output
 true
 
@@ -1359,7 +1359,7 @@ true
 -- !query
 SELECT cast(1 as smallint) in (cast(1 as smallint), cast(1 as bigint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS BIGINT) IN (CAST(CAST(1 AS SMALLINT) AS BIGINT), CAST(CAST(1 AS BIGINT) AS BIGINT))):boolean>
+struct<(CAST(1 AS SMALLINT) IN (CAST(1 AS SMALLINT), CAST(1 AS BIGINT))):boolean>
 -- !query output
 true
 
@@ -1367,7 +1367,7 @@ true
 -- !query
 SELECT cast(1 as smallint) in (cast(1 as smallint), cast(1 as float)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS FLOAT) IN (CAST(CAST(1 AS SMALLINT) AS FLOAT), CAST(CAST(1 AS FLOAT) AS FLOAT))):boolean>
+struct<(CAST(1 AS SMALLINT) IN (CAST(1 AS SMALLINT), CAST(1 AS FLOAT))):boolean>
 -- !query output
 true
 
@@ -1375,7 +1375,7 @@ true
 -- !query
 SELECT cast(1 as smallint) in (cast(1 as smallint), cast(1 as double)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DOUBLE) IN (CAST(CAST(1 AS SMALLINT) AS DOUBLE), CAST(CAST(1 AS DOUBLE) AS DOUBLE))):boolean>
+struct<(CAST(1 AS SMALLINT) IN (CAST(1 AS SMALLINT), CAST(1 AS DOUBLE))):boolean>
 -- !query output
 true
 
@@ -1383,7 +1383,7 @@ true
 -- !query
 SELECT cast(1 as smallint) in (cast(1 as smallint), cast(1 as decimal(10, 0))) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DECIMAL(10,0)) IN (CAST(CAST(1 AS SMALLINT) AS DECIMAL(10,0)), CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)))):boolean>
+struct<(CAST(1 AS SMALLINT) IN (CAST(1 AS SMALLINT), CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 true
 
@@ -1391,7 +1391,7 @@ true
 -- !query
 SELECT cast(1 as smallint) in (cast(1 as smallint), cast(1 as string)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS STRING) IN (CAST(CAST(1 AS SMALLINT) AS STRING), CAST(CAST(1 AS STRING) AS STRING))):boolean>
+struct<(CAST(1 AS SMALLINT) IN (CAST(1 AS SMALLINT), CAST(1 AS STRING))):boolean>
 -- !query output
 true
 
@@ -1435,7 +1435,7 @@ cannot resolve '(CAST(1 AS SMALLINT) IN (CAST(1 AS SMALLINT), CAST('2017-12-11 0
 -- !query
 SELECT cast(1 as int) in (cast(1 as int), cast(1 as tinyint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS INT) IN (CAST(CAST(1 AS INT) AS INT), CAST(CAST(1 AS TINYINT) AS INT))):boolean>
+struct<(CAST(1 AS INT) IN (CAST(1 AS INT), CAST(1 AS TINYINT))):boolean>
 -- !query output
 true
 
@@ -1443,7 +1443,7 @@ true
 -- !query
 SELECT cast(1 as int) in (cast(1 as int), cast(1 as smallint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS INT) IN (CAST(CAST(1 AS INT) AS INT), CAST(CAST(1 AS SMALLINT) AS INT))):boolean>
+struct<(CAST(1 AS INT) IN (CAST(1 AS INT), CAST(1 AS SMALLINT))):boolean>
 -- !query output
 true
 
@@ -1459,7 +1459,7 @@ true
 -- !query
 SELECT cast(1 as int) in (cast(1 as int), cast(1 as bigint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS BIGINT) IN (CAST(CAST(1 AS INT) AS BIGINT), CAST(CAST(1 AS BIGINT) AS BIGINT))):boolean>
+struct<(CAST(1 AS INT) IN (CAST(1 AS INT), CAST(1 AS BIGINT))):boolean>
 -- !query output
 true
 
@@ -1467,7 +1467,7 @@ true
 -- !query
 SELECT cast(1 as int) in (cast(1 as int), cast(1 as float)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS FLOAT) IN (CAST(CAST(1 AS INT) AS FLOAT), CAST(CAST(1 AS FLOAT) AS FLOAT))):boolean>
+struct<(CAST(1 AS INT) IN (CAST(1 AS INT), CAST(1 AS FLOAT))):boolean>
 -- !query output
 true
 
@@ -1475,7 +1475,7 @@ true
 -- !query
 SELECT cast(1 as int) in (cast(1 as int), cast(1 as double)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DOUBLE) IN (CAST(CAST(1 AS INT) AS DOUBLE), CAST(CAST(1 AS DOUBLE) AS DOUBLE))):boolean>
+struct<(CAST(1 AS INT) IN (CAST(1 AS INT), CAST(1 AS DOUBLE))):boolean>
 -- !query output
 true
 
@@ -1483,7 +1483,7 @@ true
 -- !query
 SELECT cast(1 as int) in (cast(1 as int), cast(1 as decimal(10, 0))) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DECIMAL(10,0)) IN (CAST(CAST(1 AS INT) AS DECIMAL(10,0)), CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)))):boolean>
+struct<(CAST(1 AS INT) IN (CAST(1 AS INT), CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 true
 
@@ -1491,7 +1491,7 @@ true
 -- !query
 SELECT cast(1 as int) in (cast(1 as int), cast(1 as string)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS STRING) IN (CAST(CAST(1 AS INT) AS STRING), CAST(CAST(1 AS STRING) AS STRING))):boolean>
+struct<(CAST(1 AS INT) IN (CAST(1 AS INT), CAST(1 AS STRING))):boolean>
 -- !query output
 true
 
@@ -1535,7 +1535,7 @@ cannot resolve '(CAST(1 AS INT) IN (CAST(1 AS INT), CAST('2017-12-11 09:30:00' A
 -- !query
 SELECT cast(1 as bigint) in (cast(1 as bigint), cast(1 as tinyint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS BIGINT) IN (CAST(CAST(1 AS BIGINT) AS BIGINT), CAST(CAST(1 AS TINYINT) AS BIGINT))):boolean>
+struct<(CAST(1 AS BIGINT) IN (CAST(1 AS BIGINT), CAST(1 AS TINYINT))):boolean>
 -- !query output
 true
 
@@ -1543,7 +1543,7 @@ true
 -- !query
 SELECT cast(1 as bigint) in (cast(1 as bigint), cast(1 as smallint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS BIGINT) IN (CAST(CAST(1 AS BIGINT) AS BIGINT), CAST(CAST(1 AS SMALLINT) AS BIGINT))):boolean>
+struct<(CAST(1 AS BIGINT) IN (CAST(1 AS BIGINT), CAST(1 AS SMALLINT))):boolean>
 -- !query output
 true
 
@@ -1551,7 +1551,7 @@ true
 -- !query
 SELECT cast(1 as bigint) in (cast(1 as bigint), cast(1 as int)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS BIGINT) IN (CAST(CAST(1 AS BIGINT) AS BIGINT), CAST(CAST(1 AS INT) AS BIGINT))):boolean>
+struct<(CAST(1 AS BIGINT) IN (CAST(1 AS BIGINT), CAST(1 AS INT))):boolean>
 -- !query output
 true
 
@@ -1567,7 +1567,7 @@ true
 -- !query
 SELECT cast(1 as bigint) in (cast(1 as bigint), cast(1 as float)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS FLOAT) IN (CAST(CAST(1 AS BIGINT) AS FLOAT), CAST(CAST(1 AS FLOAT) AS FLOAT))):boolean>
+struct<(CAST(1 AS BIGINT) IN (CAST(1 AS BIGINT), CAST(1 AS FLOAT))):boolean>
 -- !query output
 true
 
@@ -1575,7 +1575,7 @@ true
 -- !query
 SELECT cast(1 as bigint) in (cast(1 as bigint), cast(1 as double)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DOUBLE) IN (CAST(CAST(1 AS BIGINT) AS DOUBLE), CAST(CAST(1 AS DOUBLE) AS DOUBLE))):boolean>
+struct<(CAST(1 AS BIGINT) IN (CAST(1 AS BIGINT), CAST(1 AS DOUBLE))):boolean>
 -- !query output
 true
 
@@ -1583,7 +1583,7 @@ true
 -- !query
 SELECT cast(1 as bigint) in (cast(1 as bigint), cast(1 as decimal(10, 0))) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)) IN (CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)), CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)))):boolean>
+struct<(CAST(1 AS BIGINT) IN (CAST(1 AS BIGINT), CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 true
 
@@ -1591,7 +1591,7 @@ true
 -- !query
 SELECT cast(1 as bigint) in (cast(1 as bigint), cast(1 as string)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS STRING) IN (CAST(CAST(1 AS BIGINT) AS STRING), CAST(CAST(1 AS STRING) AS STRING))):boolean>
+struct<(CAST(1 AS BIGINT) IN (CAST(1 AS BIGINT), CAST(1 AS STRING))):boolean>
 -- !query output
 true
 
@@ -1635,7 +1635,7 @@ cannot resolve '(CAST(1 AS BIGINT) IN (CAST(1 AS BIGINT), CAST('2017-12-11 09:30
 -- !query
 SELECT cast(1 as float) in (cast(1 as float), cast(1 as tinyint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS FLOAT) IN (CAST(CAST(1 AS FLOAT) AS FLOAT), CAST(CAST(1 AS TINYINT) AS FLOAT))):boolean>
+struct<(CAST(1 AS FLOAT) IN (CAST(1 AS FLOAT), CAST(1 AS TINYINT))):boolean>
 -- !query output
 true
 
@@ -1643,7 +1643,7 @@ true
 -- !query
 SELECT cast(1 as float) in (cast(1 as float), cast(1 as smallint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS FLOAT) IN (CAST(CAST(1 AS FLOAT) AS FLOAT), CAST(CAST(1 AS SMALLINT) AS FLOAT))):boolean>
+struct<(CAST(1 AS FLOAT) IN (CAST(1 AS FLOAT), CAST(1 AS SMALLINT))):boolean>
 -- !query output
 true
 
@@ -1651,7 +1651,7 @@ true
 -- !query
 SELECT cast(1 as float) in (cast(1 as float), cast(1 as int)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS FLOAT) IN (CAST(CAST(1 AS FLOAT) AS FLOAT), CAST(CAST(1 AS INT) AS FLOAT))):boolean>
+struct<(CAST(1 AS FLOAT) IN (CAST(1 AS FLOAT), CAST(1 AS INT))):boolean>
 -- !query output
 true
 
@@ -1659,7 +1659,7 @@ true
 -- !query
 SELECT cast(1 as float) in (cast(1 as float), cast(1 as bigint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS FLOAT) IN (CAST(CAST(1 AS FLOAT) AS FLOAT), CAST(CAST(1 AS BIGINT) AS FLOAT))):boolean>
+struct<(CAST(1 AS FLOAT) IN (CAST(1 AS FLOAT), CAST(1 AS BIGINT))):boolean>
 -- !query output
 true
 
@@ -1675,7 +1675,7 @@ true
 -- !query
 SELECT cast(1 as float) in (cast(1 as float), cast(1 as double)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) IN (CAST(CAST(1 AS FLOAT) AS DOUBLE), CAST(CAST(1 AS DOUBLE) AS DOUBLE))):boolean>
+struct<(CAST(1 AS FLOAT) IN (CAST(1 AS FLOAT), CAST(1 AS DOUBLE))):boolean>
 -- !query output
 true
 
@@ -1683,7 +1683,7 @@ true
 -- !query
 SELECT cast(1 as float) in (cast(1 as float), cast(1 as decimal(10, 0))) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) IN (CAST(CAST(1 AS FLOAT) AS DOUBLE), CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE))):boolean>
+struct<(CAST(1 AS FLOAT) IN (CAST(1 AS FLOAT), CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 true
 
@@ -1691,7 +1691,7 @@ true
 -- !query
 SELECT cast(1 as float) in (cast(1 as float), cast(1 as string)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS STRING) IN (CAST(CAST(1 AS FLOAT) AS STRING), CAST(CAST(1 AS STRING) AS STRING))):boolean>
+struct<(CAST(1 AS FLOAT) IN (CAST(1 AS FLOAT), CAST(1 AS STRING))):boolean>
 -- !query output
 true
 
@@ -1735,7 +1735,7 @@ cannot resolve '(CAST(1 AS FLOAT) IN (CAST(1 AS FLOAT), CAST('2017-12-11 09:30:0
 -- !query
 SELECT cast(1 as double) in (cast(1 as double), cast(1 as tinyint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DOUBLE) AS DOUBLE) IN (CAST(CAST(1 AS DOUBLE) AS DOUBLE), CAST(CAST(1 AS TINYINT) AS DOUBLE))):boolean>
+struct<(CAST(1 AS DOUBLE) IN (CAST(1 AS DOUBLE), CAST(1 AS TINYINT))):boolean>
 -- !query output
 true
 
@@ -1743,7 +1743,7 @@ true
 -- !query
 SELECT cast(1 as double) in (cast(1 as double), cast(1 as smallint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DOUBLE) AS DOUBLE) IN (CAST(CAST(1 AS DOUBLE) AS DOUBLE), CAST(CAST(1 AS SMALLINT) AS DOUBLE))):boolean>
+struct<(CAST(1 AS DOUBLE) IN (CAST(1 AS DOUBLE), CAST(1 AS SMALLINT))):boolean>
 -- !query output
 true
 
@@ -1751,7 +1751,7 @@ true
 -- !query
 SELECT cast(1 as double) in (cast(1 as double), cast(1 as int)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DOUBLE) AS DOUBLE) IN (CAST(CAST(1 AS DOUBLE) AS DOUBLE), CAST(CAST(1 AS INT) AS DOUBLE))):boolean>
+struct<(CAST(1 AS DOUBLE) IN (CAST(1 AS DOUBLE), CAST(1 AS INT))):boolean>
 -- !query output
 true
 
@@ -1759,7 +1759,7 @@ true
 -- !query
 SELECT cast(1 as double) in (cast(1 as double), cast(1 as bigint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DOUBLE) AS DOUBLE) IN (CAST(CAST(1 AS DOUBLE) AS DOUBLE), CAST(CAST(1 AS BIGINT) AS DOUBLE))):boolean>
+struct<(CAST(1 AS DOUBLE) IN (CAST(1 AS DOUBLE), CAST(1 AS BIGINT))):boolean>
 -- !query output
 true
 
@@ -1767,7 +1767,7 @@ true
 -- !query
 SELECT cast(1 as double) in (cast(1 as double), cast(1 as float)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DOUBLE) AS DOUBLE) IN (CAST(CAST(1 AS DOUBLE) AS DOUBLE), CAST(CAST(1 AS FLOAT) AS DOUBLE))):boolean>
+struct<(CAST(1 AS DOUBLE) IN (CAST(1 AS DOUBLE), CAST(1 AS FLOAT))):boolean>
 -- !query output
 true
 
@@ -1783,7 +1783,7 @@ true
 -- !query
 SELECT cast(1 as double) in (cast(1 as double), cast(1 as decimal(10, 0))) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DOUBLE) AS DOUBLE) IN (CAST(CAST(1 AS DOUBLE) AS DOUBLE), CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE))):boolean>
+struct<(CAST(1 AS DOUBLE) IN (CAST(1 AS DOUBLE), CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 true
 
@@ -1791,7 +1791,7 @@ true
 -- !query
 SELECT cast(1 as double) in (cast(1 as double), cast(1 as string)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DOUBLE) AS STRING) IN (CAST(CAST(1 AS DOUBLE) AS STRING), CAST(CAST(1 AS STRING) AS STRING))):boolean>
+struct<(CAST(1 AS DOUBLE) IN (CAST(1 AS DOUBLE), CAST(1 AS STRING))):boolean>
 -- !query output
 true
 
@@ -1835,7 +1835,7 @@ cannot resolve '(CAST(1 AS DOUBLE) IN (CAST(1 AS DOUBLE), CAST('2017-12-11 09:30
 -- !query
 SELECT cast(1 as decimal(10, 0)) in (cast(1 as decimal(10, 0)), cast(1 as tinyint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) IN (CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)), CAST(CAST(1 AS TINYINT) AS DECIMAL(10,0)))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) IN (CAST(1 AS DECIMAL(10,0)), CAST(1 AS TINYINT))):boolean>
 -- !query output
 true
 
@@ -1843,7 +1843,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) in (cast(1 as decimal(10, 0)), cast(1 as smallint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) IN (CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)), CAST(CAST(1 AS SMALLINT) AS DECIMAL(10,0)))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) IN (CAST(1 AS DECIMAL(10,0)), CAST(1 AS SMALLINT))):boolean>
 -- !query output
 true
 
@@ -1851,7 +1851,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) in (cast(1 as decimal(10, 0)), cast(1 as int)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)) IN (CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(10,0)), CAST(CAST(1 AS INT) AS DECIMAL(10,0)))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) IN (CAST(1 AS DECIMAL(10,0)), CAST(1 AS INT))):boolean>
 -- !query output
 true
 
@@ -1859,7 +1859,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) in (cast(1 as decimal(10, 0)), cast(1 as bigint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)) IN (CAST(CAST(1 AS DECIMAL(10,0)) AS DECIMAL(20,0)), CAST(CAST(1 AS BIGINT) AS DECIMAL(20,0)))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) IN (CAST(1 AS DECIMAL(10,0)), CAST(1 AS BIGINT))):boolean>
 -- !query output
 true
 
@@ -1867,7 +1867,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) in (cast(1 as decimal(10, 0)), cast(1 as float)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) IN (CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE), CAST(CAST(1 AS FLOAT) AS DOUBLE))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) IN (CAST(1 AS DECIMAL(10,0)), CAST(1 AS FLOAT))):boolean>
 -- !query output
 true
 
@@ -1875,7 +1875,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) in (cast(1 as decimal(10, 0)), cast(1 as double)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) IN (CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE), CAST(CAST(1 AS DOUBLE) AS DOUBLE))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) IN (CAST(1 AS DECIMAL(10,0)), CAST(1 AS DOUBLE))):boolean>
 -- !query output
 true
 
@@ -1891,7 +1891,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0)) in (cast(1 as decimal(10, 0)), cast(1 as string)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS STRING) IN (CAST(CAST(1 AS DECIMAL(10,0)) AS STRING), CAST(CAST(1 AS STRING) AS STRING))):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) IN (CAST(1 AS DECIMAL(10,0)), CAST(1 AS STRING))):boolean>
 -- !query output
 true
 
@@ -1935,7 +1935,7 @@ cannot resolve '(CAST(1 AS DECIMAL(10,0)) IN (CAST(1 AS DECIMAL(10,0)), CAST('20
 -- !query
 SELECT cast(1 as string) in (cast(1 as string), cast(1 as tinyint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS STRING) IN (CAST(CAST(1 AS STRING) AS STRING), CAST(CAST(1 AS TINYINT) AS STRING))):boolean>
+struct<(CAST(1 AS STRING) IN (CAST(1 AS STRING), CAST(1 AS TINYINT))):boolean>
 -- !query output
 true
 
@@ -1943,7 +1943,7 @@ true
 -- !query
 SELECT cast(1 as string) in (cast(1 as string), cast(1 as smallint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS STRING) IN (CAST(CAST(1 AS STRING) AS STRING), CAST(CAST(1 AS SMALLINT) AS STRING))):boolean>
+struct<(CAST(1 AS STRING) IN (CAST(1 AS STRING), CAST(1 AS SMALLINT))):boolean>
 -- !query output
 true
 
@@ -1951,7 +1951,7 @@ true
 -- !query
 SELECT cast(1 as string) in (cast(1 as string), cast(1 as int)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS STRING) IN (CAST(CAST(1 AS STRING) AS STRING), CAST(CAST(1 AS INT) AS STRING))):boolean>
+struct<(CAST(1 AS STRING) IN (CAST(1 AS STRING), CAST(1 AS INT))):boolean>
 -- !query output
 true
 
@@ -1959,7 +1959,7 @@ true
 -- !query
 SELECT cast(1 as string) in (cast(1 as string), cast(1 as bigint)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS STRING) IN (CAST(CAST(1 AS STRING) AS STRING), CAST(CAST(1 AS BIGINT) AS STRING))):boolean>
+struct<(CAST(1 AS STRING) IN (CAST(1 AS STRING), CAST(1 AS BIGINT))):boolean>
 -- !query output
 true
 
@@ -1967,7 +1967,7 @@ true
 -- !query
 SELECT cast(1 as string) in (cast(1 as string), cast(1 as float)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS STRING) IN (CAST(CAST(1 AS STRING) AS STRING), CAST(CAST(1 AS FLOAT) AS STRING))):boolean>
+struct<(CAST(1 AS STRING) IN (CAST(1 AS STRING), CAST(1 AS FLOAT))):boolean>
 -- !query output
 true
 
@@ -1975,7 +1975,7 @@ true
 -- !query
 SELECT cast(1 as string) in (cast(1 as string), cast(1 as double)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS STRING) IN (CAST(CAST(1 AS STRING) AS STRING), CAST(CAST(1 AS DOUBLE) AS STRING))):boolean>
+struct<(CAST(1 AS STRING) IN (CAST(1 AS STRING), CAST(1 AS DOUBLE))):boolean>
 -- !query output
 true
 
@@ -1983,7 +1983,7 @@ true
 -- !query
 SELECT cast(1 as string) in (cast(1 as string), cast(1 as decimal(10, 0))) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS STRING) IN (CAST(CAST(1 AS STRING) AS STRING), CAST(CAST(1 AS DECIMAL(10,0)) AS STRING))):boolean>
+struct<(CAST(1 AS STRING) IN (CAST(1 AS STRING), CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 true
 
@@ -2017,7 +2017,7 @@ cannot resolve '(CAST(1 AS STRING) IN (CAST(1 AS STRING), CAST(1 AS BOOLEAN)))'
 -- !query
 SELECT cast(1 as string) in (cast(1 as string), cast('2017-12-11 09:30:00.0' as timestamp)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS STRING) IN (CAST(CAST(1 AS STRING) AS STRING), CAST(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) AS STRING))):boolean>
+struct<(CAST(1 AS STRING) IN (CAST(1 AS STRING), CAST(2017-12-11 09:30:00.0 AS TIMESTAMP))):boolean>
 -- !query output
 true
 
@@ -2025,7 +2025,7 @@ true
 -- !query
 SELECT cast(1 as string) in (cast(1 as string), cast('2017-12-11 09:30:00' as date)) FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS STRING) AS STRING) IN (CAST(CAST(1 AS STRING) AS STRING), CAST(CAST(2017-12-11 09:30:00 AS DATE) AS STRING))):boolean>
+struct<(CAST(1 AS STRING) IN (CAST(1 AS STRING), CAST(2017-12-11 09:30:00 AS DATE))):boolean>
 -- !query output
 true
 
@@ -2310,7 +2310,7 @@ cannot resolve '(CAST('2017-12-12 09:30:00.0' AS TIMESTAMP) IN (CAST('2017-12-12
 -- !query
 SELECT cast('2017-12-12 09:30:00.0' as timestamp) in (cast('2017-12-12 09:30:00.0' as timestamp), cast(1 as string)) FROM t
 -- !query schema
-struct<(CAST(CAST(2017-12-12 09:30:00.0 AS TIMESTAMP) AS STRING) IN (CAST(CAST(2017-12-12 09:30:00.0 AS TIMESTAMP) AS STRING), CAST(CAST(1 AS STRING) AS STRING))):boolean>
+struct<(CAST(2017-12-12 09:30:00.0 AS TIMESTAMP) IN (CAST(2017-12-12 09:30:00.0 AS TIMESTAMP), CAST(1 AS STRING))):boolean>
 -- !query output
 true
 
@@ -2344,7 +2344,7 @@ true
 -- !query
 SELECT cast('2017-12-12 09:30:00.0' as timestamp) in (cast('2017-12-12 09:30:00.0' as timestamp), cast('2017-12-11 09:30:00' as date)) FROM t
 -- !query schema
-struct<(CAST(CAST(2017-12-12 09:30:00.0 AS TIMESTAMP) AS TIMESTAMP) IN (CAST(CAST(2017-12-12 09:30:00.0 AS TIMESTAMP) AS TIMESTAMP), CAST(CAST(2017-12-11 09:30:00 AS DATE) AS TIMESTAMP))):boolean>
+struct<(CAST(2017-12-12 09:30:00.0 AS TIMESTAMP) IN (CAST(2017-12-12 09:30:00.0 AS TIMESTAMP), CAST(2017-12-11 09:30:00 AS DATE))):boolean>
 -- !query output
 true
 
@@ -2415,7 +2415,7 @@ cannot resolve '(CAST('2017-12-12 09:30:00' AS DATE) IN (CAST('2017-12-12 09:30:
 -- !query
 SELECT cast('2017-12-12 09:30:00' as date) in (cast('2017-12-12 09:30:00' as date), cast(1 as string)) FROM t
 -- !query schema
-struct<(CAST(CAST(2017-12-12 09:30:00 AS DATE) AS STRING) IN (CAST(CAST(2017-12-12 09:30:00 AS DATE) AS STRING), CAST(CAST(1 AS STRING) AS STRING))):boolean>
+struct<(CAST(2017-12-12 09:30:00 AS DATE) IN (CAST(2017-12-12 09:30:00 AS DATE), CAST(1 AS STRING))):boolean>
 -- !query output
 true
 
@@ -2441,7 +2441,7 @@ cannot resolve '(CAST('2017-12-12 09:30:00' AS DATE) IN (CAST('2017-12-12 09:30:
 -- !query
 SELECT cast('2017-12-12 09:30:00' as date) in (cast('2017-12-12 09:30:00' as date), cast('2017-12-11 09:30:00.0' as timestamp)) FROM t
 -- !query schema
-struct<(CAST(CAST(2017-12-12 09:30:00 AS DATE) AS TIMESTAMP) IN (CAST(CAST(2017-12-12 09:30:00 AS DATE) AS TIMESTAMP), CAST(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) AS TIMESTAMP))):boolean>
+struct<(CAST(2017-12-12 09:30:00 AS DATE) IN (CAST(2017-12-12 09:30:00 AS DATE), CAST(2017-12-11 09:30:00.0 AS TIMESTAMP))):boolean>
 -- !query output
 true
 
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapZipWith.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapZipWith.sql.out
index ed7ab5a342c12..c1ae1bea3e905 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapZipWith.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapZipWith.sql.out
@@ -85,7 +85,7 @@ FROM various_maps
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'map_zip_with(various_maps.`decimal_map1`, various_maps.`decimal_map2`, lambdafunction(named_struct(NamePlaceholder(), k, NamePlaceholder(), v1, NamePlaceholder(), v2), k, v1, v2))' due to argument data type mismatch: The input to function map_zip_with should have been two maps with compatible key types, but the key types are [decimal(36,0), decimal(36,35)].; line 1 pos 7
+cannot resolve 'map_zip_with(various_maps.decimal_map1, various_maps.decimal_map2, lambdafunction(struct(k, v1, v2), k, v1, v2))' due to argument data type mismatch: The input to function map_zip_with should have been two maps with compatible key types, but the key types are [decimal(36,0), decimal(36,35)].; line 1 pos 7
 
 
 -- !query
@@ -113,7 +113,7 @@ FROM various_maps
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'map_zip_with(various_maps.`decimal_map2`, various_maps.`int_map`, lambdafunction(named_struct(NamePlaceholder(), k, NamePlaceholder(), v1, NamePlaceholder(), v2), k, v1, v2))' due to argument data type mismatch: The input to function map_zip_with should have been two maps with compatible key types, but the key types are [decimal(36,35), int].; line 1 pos 7
+cannot resolve 'map_zip_with(various_maps.decimal_map2, various_maps.int_map, lambdafunction(struct(k, v1, v2), k, v1, v2))' due to argument data type mismatch: The input to function map_zip_with should have been two maps with compatible key types, but the key types are [decimal(36,35), int].; line 1 pos 7
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapconcat.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapconcat.sql.out
index fcf1afc72efe9..c8831f3b08813 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapconcat.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapconcat.sql.out
@@ -94,7 +94,7 @@ FROM various_maps
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'map_concat(various_maps.`tinyint_map1`, various_maps.`array_map1`)' due to data type mismatch: input to function map_concat should all be the same type, but it's [map<tinyint,tinyint>, map<array<string>,array<string>>]; line 2 pos 4
+cannot resolve 'map_concat(various_maps.tinyint_map1, various_maps.array_map1)' due to data type mismatch: input to function map_concat should all be the same type, but it's [map<tinyint,tinyint>, map<array<string>,array<string>>]; line 2 pos 4
 
 
 -- !query
@@ -105,7 +105,7 @@ FROM various_maps
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'map_concat(various_maps.`boolean_map1`, various_maps.`int_map2`)' due to data type mismatch: input to function map_concat should all be the same type, but it's [map<boolean,boolean>, map<int,int>]; line 2 pos 4
+cannot resolve 'map_concat(various_maps.boolean_map1, various_maps.int_map2)' due to data type mismatch: input to function map_concat should all be the same type, but it's [map<boolean,boolean>, map<int,int>]; line 2 pos 4
 
 
 -- !query
@@ -116,7 +116,7 @@ FROM various_maps
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'map_concat(various_maps.`int_map1`, various_maps.`struct_map2`)' due to data type mismatch: input to function map_concat should all be the same type, but it's [map<int,int>, map<struct<col1:string,col2:int>,struct<col1:string,col2:int>>]; line 2 pos 4
+cannot resolve 'map_concat(various_maps.int_map1, various_maps.struct_map2)' due to data type mismatch: input to function map_concat should all be the same type, but it's [map<int,int>, map<struct<col1:string,col2:int>,struct<col1:string,col2:int>>]; line 2 pos 4
 
 
 -- !query
@@ -127,7 +127,7 @@ FROM various_maps
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'map_concat(various_maps.`struct_map1`, various_maps.`array_map2`)' due to data type mismatch: input to function map_concat should all be the same type, but it's [map<struct<col1:string,col2:int>,struct<col1:string,col2:int>>, map<array<string>,array<string>>]; line 2 pos 4
+cannot resolve 'map_concat(various_maps.struct_map1, various_maps.array_map2)' due to data type mismatch: input to function map_concat should all be the same type, but it's [map<struct<col1:string,col2:int>,struct<col1:string,col2:int>>, map<array<string>,array<string>>]; line 2 pos 4
 
 
 -- !query
@@ -138,4 +138,4 @@ FROM various_maps
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'map_concat(various_maps.`int_map1`, various_maps.`array_map2`)' due to data type mismatch: input to function map_concat should all be the same type, but it's [map<int,int>, map<array<string>,array<string>>]; line 2 pos 4
+cannot resolve 'map_concat(various_maps.int_map1, various_maps.array_map2)' due to data type mismatch: input to function map_concat should all be the same type, but it's [map<int,int>, map<array<string>,array<string>>]; line 2 pos 4
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/promoteStrings.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/promoteStrings.sql.out
index b8c190beeae19..61eeea989d170 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/promoteStrings.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/promoteStrings.sql.out
@@ -13,7 +13,7 @@ struct<>
 -- !query
 SELECT '1' + cast(1 as tinyint)                         FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) + CAST(CAST(1 AS TINYINT) AS DOUBLE)):double>
+struct<(1 + CAST(1 AS TINYINT)):double>
 -- !query output
 2.0
 
@@ -21,7 +21,7 @@ struct<(CAST(1 AS DOUBLE) + CAST(CAST(1 AS TINYINT) AS DOUBLE)):double>
 -- !query
 SELECT '1' + cast(1 as smallint)                        FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) + CAST(CAST(1 AS SMALLINT) AS DOUBLE)):double>
+struct<(1 + CAST(1 AS SMALLINT)):double>
 -- !query output
 2.0
 
@@ -29,7 +29,7 @@ struct<(CAST(1 AS DOUBLE) + CAST(CAST(1 AS SMALLINT) AS DOUBLE)):double>
 -- !query
 SELECT '1' + cast(1 as int)                             FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) + CAST(CAST(1 AS INT) AS DOUBLE)):double>
+struct<(1 + CAST(1 AS INT)):double>
 -- !query output
 2.0
 
@@ -37,7 +37,7 @@ struct<(CAST(1 AS DOUBLE) + CAST(CAST(1 AS INT) AS DOUBLE)):double>
 -- !query
 SELECT '1' + cast(1 as bigint)                          FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) + CAST(CAST(1 AS BIGINT) AS DOUBLE)):double>
+struct<(1 + CAST(1 AS BIGINT)):double>
 -- !query output
 2.0
 
@@ -45,7 +45,7 @@ struct<(CAST(1 AS DOUBLE) + CAST(CAST(1 AS BIGINT) AS DOUBLE)):double>
 -- !query
 SELECT '1' + cast(1 as float)                           FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) + CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(1 + CAST(1 AS FLOAT)):double>
 -- !query output
 2.0
 
@@ -53,7 +53,7 @@ struct<(CAST(1 AS DOUBLE) + CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
 -- !query
 SELECT '1' + cast(1 as double)                          FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) + CAST(1 AS DOUBLE)):double>
+struct<(1 + CAST(1 AS DOUBLE)):double>
 -- !query output
 2.0
 
@@ -61,7 +61,7 @@ struct<(CAST(1 AS DOUBLE) + CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT '1' + cast(1 as decimal(10, 0))                  FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) + CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
+struct<(1 + CAST(1 AS DECIMAL(10,0))):double>
 -- !query output
 2.0
 
@@ -69,7 +69,7 @@ struct<(CAST(1 AS DOUBLE) + CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
 -- !query
 SELECT '1' + '1'                                        FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) + CAST(1 AS DOUBLE)):double>
+struct<(1 + 1):double>
 -- !query output
 2.0
 
@@ -113,7 +113,7 @@ cannot resolve 'date_add(CAST('2017-12-11 09:30:00' AS DATE), CAST('1' AS DOUBLE
 -- !query
 SELECT '1' - cast(1 as tinyint)                         FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) - CAST(CAST(1 AS TINYINT) AS DOUBLE)):double>
+struct<(1 - CAST(1 AS TINYINT)):double>
 -- !query output
 0.0
 
@@ -121,7 +121,7 @@ struct<(CAST(1 AS DOUBLE) - CAST(CAST(1 AS TINYINT) AS DOUBLE)):double>
 -- !query
 SELECT '1' - cast(1 as smallint)                        FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) - CAST(CAST(1 AS SMALLINT) AS DOUBLE)):double>
+struct<(1 - CAST(1 AS SMALLINT)):double>
 -- !query output
 0.0
 
@@ -129,7 +129,7 @@ struct<(CAST(1 AS DOUBLE) - CAST(CAST(1 AS SMALLINT) AS DOUBLE)):double>
 -- !query
 SELECT '1' - cast(1 as int)                             FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) - CAST(CAST(1 AS INT) AS DOUBLE)):double>
+struct<(1 - CAST(1 AS INT)):double>
 -- !query output
 0.0
 
@@ -137,7 +137,7 @@ struct<(CAST(1 AS DOUBLE) - CAST(CAST(1 AS INT) AS DOUBLE)):double>
 -- !query
 SELECT '1' - cast(1 as bigint)                          FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) - CAST(CAST(1 AS BIGINT) AS DOUBLE)):double>
+struct<(1 - CAST(1 AS BIGINT)):double>
 -- !query output
 0.0
 
@@ -145,7 +145,7 @@ struct<(CAST(1 AS DOUBLE) - CAST(CAST(1 AS BIGINT) AS DOUBLE)):double>
 -- !query
 SELECT '1' - cast(1 as float)                           FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) - CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(1 - CAST(1 AS FLOAT)):double>
 -- !query output
 0.0
 
@@ -153,7 +153,7 @@ struct<(CAST(1 AS DOUBLE) - CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
 -- !query
 SELECT '1' - cast(1 as double)                          FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) - CAST(1 AS DOUBLE)):double>
+struct<(1 - CAST(1 AS DOUBLE)):double>
 -- !query output
 0.0
 
@@ -161,7 +161,7 @@ struct<(CAST(1 AS DOUBLE) - CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT '1' - cast(1 as decimal(10, 0))                  FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) - CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
+struct<(1 - CAST(1 AS DECIMAL(10,0))):double>
 -- !query output
 0.0
 
@@ -169,7 +169,7 @@ struct<(CAST(1 AS DOUBLE) - CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
 -- !query
 SELECT '1' - '1'                                        FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) - CAST(1 AS DOUBLE)):double>
+struct<(1 - 1):double>
 -- !query output
 0.0
 
@@ -198,13 +198,13 @@ SELECT '1' - cast('2017-12-11 09:30:00.0' as timestamp) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'subtracttimestamps('1', CAST('2017-12-11 09:30:00.0' AS TIMESTAMP))' due to data type mismatch: argument 1 requires timestamp type, however, ''1'' is of string type.; line 1 pos 7
+cannot resolve '('1' - CAST('2017-12-11 09:30:00.0' AS TIMESTAMP))' due to data type mismatch: argument 1 requires timestamp type, however, ''1'' is of string type.; line 1 pos 7
 
 
 -- !query
 SELECT '1' - cast('2017-12-11 09:30:00' as date)        FROM t
 -- !query schema
-struct<subtractdates(CAST(1 AS DATE), CAST(2017-12-11 09:30:00 AS DATE)):interval>
+struct<(1 - CAST(2017-12-11 09:30:00 AS DATE)):interval day to second>
 -- !query output
 NULL
 
@@ -212,7 +212,7 @@ NULL
 -- !query
 SELECT '1' * cast(1 as tinyint)                         FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) * CAST(CAST(1 AS TINYINT) AS DOUBLE)):double>
+struct<(1 * CAST(1 AS TINYINT)):double>
 -- !query output
 1.0
 
@@ -220,7 +220,7 @@ struct<(CAST(1 AS DOUBLE) * CAST(CAST(1 AS TINYINT) AS DOUBLE)):double>
 -- !query
 SELECT '1' * cast(1 as smallint)                        FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) * CAST(CAST(1 AS SMALLINT) AS DOUBLE)):double>
+struct<(1 * CAST(1 AS SMALLINT)):double>
 -- !query output
 1.0
 
@@ -228,7 +228,7 @@ struct<(CAST(1 AS DOUBLE) * CAST(CAST(1 AS SMALLINT) AS DOUBLE)):double>
 -- !query
 SELECT '1' * cast(1 as int)                             FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) * CAST(CAST(1 AS INT) AS DOUBLE)):double>
+struct<(1 * CAST(1 AS INT)):double>
 -- !query output
 1.0
 
@@ -236,7 +236,7 @@ struct<(CAST(1 AS DOUBLE) * CAST(CAST(1 AS INT) AS DOUBLE)):double>
 -- !query
 SELECT '1' * cast(1 as bigint)                          FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) * CAST(CAST(1 AS BIGINT) AS DOUBLE)):double>
+struct<(1 * CAST(1 AS BIGINT)):double>
 -- !query output
 1.0
 
@@ -244,7 +244,7 @@ struct<(CAST(1 AS DOUBLE) * CAST(CAST(1 AS BIGINT) AS DOUBLE)):double>
 -- !query
 SELECT '1' * cast(1 as float)                           FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) * CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(1 * CAST(1 AS FLOAT)):double>
 -- !query output
 1.0
 
@@ -252,7 +252,7 @@ struct<(CAST(1 AS DOUBLE) * CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
 -- !query
 SELECT '1' * cast(1 as double)                          FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) * CAST(1 AS DOUBLE)):double>
+struct<(1 * CAST(1 AS DOUBLE)):double>
 -- !query output
 1.0
 
@@ -260,7 +260,7 @@ struct<(CAST(1 AS DOUBLE) * CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT '1' * cast(1 as decimal(10, 0))                  FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) * CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
+struct<(1 * CAST(1 AS DECIMAL(10,0))):double>
 -- !query output
 1.0
 
@@ -268,7 +268,7 @@ struct<(CAST(1 AS DOUBLE) * CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
 -- !query
 SELECT '1' * '1'                                        FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) * CAST(1 AS DOUBLE)):double>
+struct<(1 * 1):double>
 -- !query output
 1.0
 
@@ -312,7 +312,7 @@ cannot resolve '(CAST('1' AS DOUBLE) * CAST('2017-12-11 09:30:00' AS DATE))' due
 -- !query
 SELECT '1' / cast(1 as tinyint)                         FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS TINYINT) AS DOUBLE)):double>
+struct<(1 / CAST(1 AS TINYINT)):double>
 -- !query output
 1.0
 
@@ -320,7 +320,7 @@ struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS TINYINT) AS DOUBLE)):double>
 -- !query
 SELECT '1' / cast(1 as smallint)                        FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS SMALLINT) AS DOUBLE)):double>
+struct<(1 / CAST(1 AS SMALLINT)):double>
 -- !query output
 1.0
 
@@ -328,7 +328,7 @@ struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS SMALLINT) AS DOUBLE)):double>
 -- !query
 SELECT '1' / cast(1 as int)                             FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS INT) AS DOUBLE)):double>
+struct<(1 / CAST(1 AS INT)):double>
 -- !query output
 1.0
 
@@ -336,7 +336,7 @@ struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS INT) AS DOUBLE)):double>
 -- !query
 SELECT '1' / cast(1 as bigint)                          FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS BIGINT) AS DOUBLE)):double>
+struct<(1 / CAST(1 AS BIGINT)):double>
 -- !query output
 1.0
 
@@ -344,7 +344,7 @@ struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS BIGINT) AS DOUBLE)):double>
 -- !query
 SELECT '1' / cast(1 as float)                           FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(1 / CAST(1 AS FLOAT)):double>
 -- !query output
 1.0
 
@@ -352,7 +352,7 @@ struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
 -- !query
 SELECT '1' / cast(1 as double)                          FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) / CAST(1 AS DOUBLE)):double>
+struct<(1 / CAST(1 AS DOUBLE)):double>
 -- !query output
 1.0
 
@@ -360,7 +360,7 @@ struct<(CAST(1 AS DOUBLE) / CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT '1' / cast(1 as decimal(10, 0))                  FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
+struct<(1 / CAST(1 AS DECIMAL(10,0))):double>
 -- !query output
 1.0
 
@@ -368,7 +368,7 @@ struct<(CAST(1 AS DOUBLE) / CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
 -- !query
 SELECT '1' / '1'                                        FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) / CAST(1 AS DOUBLE)):double>
+struct<(1 / 1):double>
 -- !query output
 1.0
 
@@ -412,7 +412,7 @@ cannot resolve '(CAST('1' AS DOUBLE) / CAST('2017-12-11 09:30:00' AS DATE))' due
 -- !query
 SELECT '1' % cast(1 as tinyint)                         FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) % CAST(CAST(1 AS TINYINT) AS DOUBLE)):double>
+struct<(1 % CAST(1 AS TINYINT)):double>
 -- !query output
 0.0
 
@@ -420,7 +420,7 @@ struct<(CAST(1 AS DOUBLE) % CAST(CAST(1 AS TINYINT) AS DOUBLE)):double>
 -- !query
 SELECT '1' % cast(1 as smallint)                        FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) % CAST(CAST(1 AS SMALLINT) AS DOUBLE)):double>
+struct<(1 % CAST(1 AS SMALLINT)):double>
 -- !query output
 0.0
 
@@ -428,7 +428,7 @@ struct<(CAST(1 AS DOUBLE) % CAST(CAST(1 AS SMALLINT) AS DOUBLE)):double>
 -- !query
 SELECT '1' % cast(1 as int)                             FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) % CAST(CAST(1 AS INT) AS DOUBLE)):double>
+struct<(1 % CAST(1 AS INT)):double>
 -- !query output
 0.0
 
@@ -436,7 +436,7 @@ struct<(CAST(1 AS DOUBLE) % CAST(CAST(1 AS INT) AS DOUBLE)):double>
 -- !query
 SELECT '1' % cast(1 as bigint)                          FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) % CAST(CAST(1 AS BIGINT) AS DOUBLE)):double>
+struct<(1 % CAST(1 AS BIGINT)):double>
 -- !query output
 0.0
 
@@ -444,7 +444,7 @@ struct<(CAST(1 AS DOUBLE) % CAST(CAST(1 AS BIGINT) AS DOUBLE)):double>
 -- !query
 SELECT '1' % cast(1 as float)                           FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) % CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<(1 % CAST(1 AS FLOAT)):double>
 -- !query output
 0.0
 
@@ -452,7 +452,7 @@ struct<(CAST(1 AS DOUBLE) % CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
 -- !query
 SELECT '1' % cast(1 as double)                          FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) % CAST(1 AS DOUBLE)):double>
+struct<(1 % CAST(1 AS DOUBLE)):double>
 -- !query output
 0.0
 
@@ -460,7 +460,7 @@ struct<(CAST(1 AS DOUBLE) % CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT '1' % cast(1 as decimal(10, 0))                  FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) % CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
+struct<(1 % CAST(1 AS DECIMAL(10,0))):double>
 -- !query output
 0.0
 
@@ -468,7 +468,7 @@ struct<(CAST(1 AS DOUBLE) % CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
 -- !query
 SELECT '1' % '1'                                        FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) % CAST(1 AS DOUBLE)):double>
+struct<(1 % 1):double>
 -- !query output
 0.0
 
@@ -512,7 +512,7 @@ cannot resolve '(CAST('1' AS DOUBLE) % CAST('2017-12-11 09:30:00' AS DATE))' due
 -- !query
 SELECT pmod('1', cast(1 as tinyint))                         FROM t
 -- !query schema
-struct<pmod(CAST(1 AS DOUBLE), CAST(CAST(1 AS TINYINT) AS DOUBLE)):double>
+struct<pmod(1, CAST(1 AS TINYINT)):double>
 -- !query output
 0.0
 
@@ -520,7 +520,7 @@ struct<pmod(CAST(1 AS DOUBLE), CAST(CAST(1 AS TINYINT) AS DOUBLE)):double>
 -- !query
 SELECT pmod('1', cast(1 as smallint))                        FROM t
 -- !query schema
-struct<pmod(CAST(1 AS DOUBLE), CAST(CAST(1 AS SMALLINT) AS DOUBLE)):double>
+struct<pmod(1, CAST(1 AS SMALLINT)):double>
 -- !query output
 0.0
 
@@ -528,7 +528,7 @@ struct<pmod(CAST(1 AS DOUBLE), CAST(CAST(1 AS SMALLINT) AS DOUBLE)):double>
 -- !query
 SELECT pmod('1', cast(1 as int))                             FROM t
 -- !query schema
-struct<pmod(CAST(1 AS DOUBLE), CAST(CAST(1 AS INT) AS DOUBLE)):double>
+struct<pmod(1, CAST(1 AS INT)):double>
 -- !query output
 0.0
 
@@ -536,7 +536,7 @@ struct<pmod(CAST(1 AS DOUBLE), CAST(CAST(1 AS INT) AS DOUBLE)):double>
 -- !query
 SELECT pmod('1', cast(1 as bigint))                          FROM t
 -- !query schema
-struct<pmod(CAST(1 AS DOUBLE), CAST(CAST(1 AS BIGINT) AS DOUBLE)):double>
+struct<pmod(1, CAST(1 AS BIGINT)):double>
 -- !query output
 0.0
 
@@ -544,7 +544,7 @@ struct<pmod(CAST(1 AS DOUBLE), CAST(CAST(1 AS BIGINT) AS DOUBLE)):double>
 -- !query
 SELECT pmod('1', cast(1 as float))                           FROM t
 -- !query schema
-struct<pmod(CAST(1 AS DOUBLE), CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
+struct<pmod(1, CAST(1 AS FLOAT)):double>
 -- !query output
 0.0
 
@@ -552,7 +552,7 @@ struct<pmod(CAST(1 AS DOUBLE), CAST(CAST(1 AS FLOAT) AS DOUBLE)):double>
 -- !query
 SELECT pmod('1', cast(1 as double))                          FROM t
 -- !query schema
-struct<pmod(CAST(1 AS DOUBLE), CAST(1 AS DOUBLE)):double>
+struct<pmod(1, CAST(1 AS DOUBLE)):double>
 -- !query output
 0.0
 
@@ -560,7 +560,7 @@ struct<pmod(CAST(1 AS DOUBLE), CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT pmod('1', cast(1 as decimal(10, 0)))                  FROM t
 -- !query schema
-struct<pmod(CAST(1 AS DOUBLE), CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
+struct<pmod(1, CAST(1 AS DECIMAL(10,0))):double>
 -- !query output
 0.0
 
@@ -568,7 +568,7 @@ struct<pmod(CAST(1 AS DOUBLE), CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):double>
 -- !query
 SELECT pmod('1', '1')                                        FROM t
 -- !query schema
-struct<pmod(CAST(1 AS DOUBLE), CAST(1 AS DOUBLE)):double>
+struct<pmod(1, 1):double>
 -- !query output
 0.0
 
@@ -612,7 +612,7 @@ cannot resolve 'pmod(CAST('1' AS DOUBLE), CAST('2017-12-11 09:30:00' AS DATE))'
 -- !query
 SELECT cast(1 as tinyint)                         + '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DOUBLE) + CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS TINYINT) + 1):double>
 -- !query output
 2.0
 
@@ -620,7 +620,7 @@ struct<(CAST(CAST(1 AS TINYINT) AS DOUBLE) + CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as smallint)                        + '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DOUBLE) + CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS SMALLINT) + 1):double>
 -- !query output
 2.0
 
@@ -628,7 +628,7 @@ struct<(CAST(CAST(1 AS SMALLINT) AS DOUBLE) + CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as int)                             + '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DOUBLE) + CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS INT) + 1):double>
 -- !query output
 2.0
 
@@ -636,7 +636,7 @@ struct<(CAST(CAST(1 AS INT) AS DOUBLE) + CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as bigint)                          + '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DOUBLE) + CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS BIGINT) + 1):double>
 -- !query output
 2.0
 
@@ -644,7 +644,7 @@ struct<(CAST(CAST(1 AS BIGINT) AS DOUBLE) + CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as float)                           + '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) + CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) + 1):double>
 -- !query output
 2.0
 
@@ -652,7 +652,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) + CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as double)                          + '1' FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) + CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) + 1):double>
 -- !query output
 2.0
 
@@ -660,7 +660,7 @@ struct<(CAST(1 AS DOUBLE) + CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(10, 0))                  + '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) + CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(10,0)) + 1):double>
 -- !query output
 2.0
 
@@ -704,7 +704,7 @@ cannot resolve 'date_add(CAST('2017-12-11 09:30:00' AS DATE), CAST('1' AS DOUBLE
 -- !query
 SELECT cast(1 as tinyint)                         - '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DOUBLE) - CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS TINYINT) - 1):double>
 -- !query output
 0.0
 
@@ -712,7 +712,7 @@ struct<(CAST(CAST(1 AS TINYINT) AS DOUBLE) - CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as smallint)                        - '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DOUBLE) - CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS SMALLINT) - 1):double>
 -- !query output
 0.0
 
@@ -720,7 +720,7 @@ struct<(CAST(CAST(1 AS SMALLINT) AS DOUBLE) - CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as int)                             - '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DOUBLE) - CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS INT) - 1):double>
 -- !query output
 0.0
 
@@ -728,7 +728,7 @@ struct<(CAST(CAST(1 AS INT) AS DOUBLE) - CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as bigint)                          - '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DOUBLE) - CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS BIGINT) - 1):double>
 -- !query output
 0.0
 
@@ -736,7 +736,7 @@ struct<(CAST(CAST(1 AS BIGINT) AS DOUBLE) - CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as float)                           - '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) - CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) - 1):double>
 -- !query output
 0.0
 
@@ -744,7 +744,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) - CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as double)                          - '1' FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) - CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) - 1):double>
 -- !query output
 0.0
 
@@ -752,7 +752,7 @@ struct<(CAST(1 AS DOUBLE) - CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(10, 0))                  - '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) - CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(10,0)) - 1):double>
 -- !query output
 0.0
 
@@ -781,7 +781,7 @@ SELECT cast('2017-12-11 09:30:00.0' as timestamp) - '1' FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'subtracttimestamps(CAST('2017-12-11 09:30:00.0' AS TIMESTAMP), '1')' due to data type mismatch: argument 2 requires timestamp type, however, ''1'' is of string type.; line 1 pos 7
+cannot resolve '(CAST('2017-12-11 09:30:00.0' AS TIMESTAMP) - '1')' due to data type mismatch: argument 2 requires timestamp type, however, ''1'' is of string type.; line 1 pos 7
 
 
 -- !query
@@ -796,7 +796,7 @@ cannot resolve 'date_sub(CAST('2017-12-11 09:30:00' AS DATE), CAST('1' AS DOUBLE
 -- !query
 SELECT cast(1 as tinyint)                         * '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DOUBLE) * CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS TINYINT) * 1):double>
 -- !query output
 1.0
 
@@ -804,7 +804,7 @@ struct<(CAST(CAST(1 AS TINYINT) AS DOUBLE) * CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as smallint)                        * '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DOUBLE) * CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS SMALLINT) * 1):double>
 -- !query output
 1.0
 
@@ -812,7 +812,7 @@ struct<(CAST(CAST(1 AS SMALLINT) AS DOUBLE) * CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as int)                             * '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DOUBLE) * CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS INT) * 1):double>
 -- !query output
 1.0
 
@@ -820,7 +820,7 @@ struct<(CAST(CAST(1 AS INT) AS DOUBLE) * CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as bigint)                          * '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DOUBLE) * CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS BIGINT) * 1):double>
 -- !query output
 1.0
 
@@ -828,7 +828,7 @@ struct<(CAST(CAST(1 AS BIGINT) AS DOUBLE) * CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as float)                           * '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) * CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) * 1):double>
 -- !query output
 1.0
 
@@ -836,7 +836,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) * CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as double)                          * '1' FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) * CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) * 1):double>
 -- !query output
 1.0
 
@@ -844,7 +844,7 @@ struct<(CAST(1 AS DOUBLE) * CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(10, 0))                  * '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) * CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(10,0)) * 1):double>
 -- !query output
 1.0
 
@@ -888,7 +888,7 @@ cannot resolve '(CAST('2017-12-11 09:30:00' AS DATE) * CAST('1' AS DOUBLE))' due
 -- !query
 SELECT cast(1 as tinyint)                         / '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DOUBLE) / CAST(CAST(1 AS DOUBLE) AS DOUBLE)):double>
+struct<(CAST(1 AS TINYINT) / 1):double>
 -- !query output
 1.0
 
@@ -896,7 +896,7 @@ struct<(CAST(CAST(1 AS TINYINT) AS DOUBLE) / CAST(CAST(1 AS DOUBLE) AS DOUBLE)):
 -- !query
 SELECT cast(1 as smallint)                        / '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DOUBLE) / CAST(CAST(1 AS DOUBLE) AS DOUBLE)):double>
+struct<(CAST(1 AS SMALLINT) / 1):double>
 -- !query output
 1.0
 
@@ -904,7 +904,7 @@ struct<(CAST(CAST(1 AS SMALLINT) AS DOUBLE) / CAST(CAST(1 AS DOUBLE) AS DOUBLE))
 -- !query
 SELECT cast(1 as int)                             / '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DOUBLE) / CAST(CAST(1 AS DOUBLE) AS DOUBLE)):double>
+struct<(CAST(1 AS INT) / 1):double>
 -- !query output
 1.0
 
@@ -912,7 +912,7 @@ struct<(CAST(CAST(1 AS INT) AS DOUBLE) / CAST(CAST(1 AS DOUBLE) AS DOUBLE)):doub
 -- !query
 SELECT cast(1 as bigint)                          / '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DOUBLE) / CAST(CAST(1 AS DOUBLE) AS DOUBLE)):double>
+struct<(CAST(1 AS BIGINT) / 1):double>
 -- !query output
 1.0
 
@@ -920,7 +920,7 @@ struct<(CAST(CAST(1 AS BIGINT) AS DOUBLE) / CAST(CAST(1 AS DOUBLE) AS DOUBLE)):d
 -- !query
 SELECT cast(1 as float)                           / '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) / CAST(CAST(1 AS DOUBLE) AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) / 1):double>
 -- !query output
 1.0
 
@@ -928,7 +928,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) / CAST(CAST(1 AS DOUBLE) AS DOUBLE)):do
 -- !query
 SELECT cast(1 as double)                          / '1' FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) / CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) / 1):double>
 -- !query output
 1.0
 
@@ -936,7 +936,7 @@ struct<(CAST(1 AS DOUBLE) / CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(10, 0))                  / '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) / CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(10,0)) / 1):double>
 -- !query output
 1.0
 
@@ -980,7 +980,7 @@ cannot resolve '(CAST('2017-12-11 09:30:00' AS DATE) / CAST('1' AS DOUBLE))' due
 -- !query
 SELECT cast(1 as tinyint)                         % '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS TINYINT) AS DOUBLE) % CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS TINYINT) % 1):double>
 -- !query output
 0.0
 
@@ -988,7 +988,7 @@ struct<(CAST(CAST(1 AS TINYINT) AS DOUBLE) % CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as smallint)                        % '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS SMALLINT) AS DOUBLE) % CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS SMALLINT) % 1):double>
 -- !query output
 0.0
 
@@ -996,7 +996,7 @@ struct<(CAST(CAST(1 AS SMALLINT) AS DOUBLE) % CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as int)                             % '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS INT) AS DOUBLE) % CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS INT) % 1):double>
 -- !query output
 0.0
 
@@ -1004,7 +1004,7 @@ struct<(CAST(CAST(1 AS INT) AS DOUBLE) % CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as bigint)                          % '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS BIGINT) AS DOUBLE) % CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS BIGINT) % 1):double>
 -- !query output
 0.0
 
@@ -1012,7 +1012,7 @@ struct<(CAST(CAST(1 AS BIGINT) AS DOUBLE) % CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as float)                           % '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) % CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS FLOAT) % 1):double>
 -- !query output
 0.0
 
@@ -1020,7 +1020,7 @@ struct<(CAST(CAST(1 AS FLOAT) AS DOUBLE) % CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as double)                          % '1' FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) % CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DOUBLE) % 1):double>
 -- !query output
 0.0
 
@@ -1028,7 +1028,7 @@ struct<(CAST(1 AS DOUBLE) % CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT cast(1 as decimal(10, 0))                  % '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) % CAST(1 AS DOUBLE)):double>
+struct<(CAST(1 AS DECIMAL(10,0)) % 1):double>
 -- !query output
 0.0
 
@@ -1072,7 +1072,7 @@ cannot resolve '(CAST('2017-12-11 09:30:00' AS DATE) % CAST('1' AS DOUBLE))' due
 -- !query
 SELECT pmod(cast(1 as tinyint), '1')                         FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS TINYINT) AS DOUBLE), CAST(1 AS DOUBLE)):double>
+struct<pmod(CAST(1 AS TINYINT), 1):double>
 -- !query output
 0.0
 
@@ -1080,7 +1080,7 @@ struct<pmod(CAST(CAST(1 AS TINYINT) AS DOUBLE), CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT pmod(cast(1 as smallint), '1')                        FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS SMALLINT) AS DOUBLE), CAST(1 AS DOUBLE)):double>
+struct<pmod(CAST(1 AS SMALLINT), 1):double>
 -- !query output
 0.0
 
@@ -1088,7 +1088,7 @@ struct<pmod(CAST(CAST(1 AS SMALLINT) AS DOUBLE), CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT pmod(cast(1 as int), '1')                             FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS INT) AS DOUBLE), CAST(1 AS DOUBLE)):double>
+struct<pmod(CAST(1 AS INT), 1):double>
 -- !query output
 0.0
 
@@ -1096,7 +1096,7 @@ struct<pmod(CAST(CAST(1 AS INT) AS DOUBLE), CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT pmod(cast(1 as bigint), '1')                          FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS BIGINT) AS DOUBLE), CAST(1 AS DOUBLE)):double>
+struct<pmod(CAST(1 AS BIGINT), 1):double>
 -- !query output
 0.0
 
@@ -1104,7 +1104,7 @@ struct<pmod(CAST(CAST(1 AS BIGINT) AS DOUBLE), CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT pmod(cast(1 as float), '1')                           FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS FLOAT) AS DOUBLE), CAST(1 AS DOUBLE)):double>
+struct<pmod(CAST(1 AS FLOAT), 1):double>
 -- !query output
 0.0
 
@@ -1112,7 +1112,7 @@ struct<pmod(CAST(CAST(1 AS FLOAT) AS DOUBLE), CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT pmod(cast(1 as double), '1')                          FROM t
 -- !query schema
-struct<pmod(CAST(1 AS DOUBLE), CAST(1 AS DOUBLE)):double>
+struct<pmod(CAST(1 AS DOUBLE), 1):double>
 -- !query output
 0.0
 
@@ -1120,7 +1120,7 @@ struct<pmod(CAST(1 AS DOUBLE), CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT pmod(cast(1 as decimal(10, 0)), '1')                  FROM t
 -- !query schema
-struct<pmod(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE), CAST(1 AS DOUBLE)):double>
+struct<pmod(CAST(1 AS DECIMAL(10,0)), 1):double>
 -- !query output
 0.0
 
@@ -1164,7 +1164,7 @@ cannot resolve 'pmod(CAST('2017-12-11 09:30:00' AS DATE), CAST('1' AS DOUBLE))'
 -- !query
 SELECT '1' = cast(1 as tinyint)                         FROM t
 -- !query schema
-struct<(CAST(1 AS TINYINT) = CAST(1 AS TINYINT)):boolean>
+struct<(1 = CAST(1 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -1172,7 +1172,7 @@ true
 -- !query
 SELECT '1' = cast(1 as smallint)                        FROM t
 -- !query schema
-struct<(CAST(1 AS SMALLINT) = CAST(1 AS SMALLINT)):boolean>
+struct<(1 = CAST(1 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -1180,7 +1180,7 @@ true
 -- !query
 SELECT '1' = cast(1 as int)                             FROM t
 -- !query schema
-struct<(CAST(1 AS INT) = CAST(1 AS INT)):boolean>
+struct<(1 = CAST(1 AS INT)):boolean>
 -- !query output
 true
 
@@ -1188,7 +1188,7 @@ true
 -- !query
 SELECT '1' = cast(1 as bigint)                          FROM t
 -- !query schema
-struct<(CAST(1 AS BIGINT) = CAST(1 AS BIGINT)):boolean>
+struct<(1 = CAST(1 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -1196,7 +1196,7 @@ true
 -- !query
 SELECT '1' = cast(1 as float)                           FROM t
 -- !query schema
-struct<(CAST(1 AS FLOAT) = CAST(1 AS FLOAT)):boolean>
+struct<(1 = CAST(1 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -1204,7 +1204,7 @@ true
 -- !query
 SELECT '1' = cast(1 as double)                          FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) = CAST(1 AS DOUBLE)):boolean>
+struct<(1 = CAST(1 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -1212,7 +1212,7 @@ true
 -- !query
 SELECT '1' = cast(1 as decimal(10, 0))                  FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) = CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(1 = CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -1228,7 +1228,7 @@ true
 -- !query
 SELECT '1' = cast('1' as binary)                        FROM t
 -- !query schema
-struct<(CAST(1 AS BINARY) = CAST(1 AS BINARY)):boolean>
+struct<(1 = CAST(1 AS BINARY)):boolean>
 -- !query output
 true
 
@@ -1236,7 +1236,7 @@ true
 -- !query
 SELECT '1' = cast(1 as boolean)                         FROM t
 -- !query schema
-struct<(CAST(1 AS BOOLEAN) = CAST(1 AS BOOLEAN)):boolean>
+struct<(1 = CAST(1 AS BOOLEAN)):boolean>
 -- !query output
 true
 
@@ -1244,7 +1244,7 @@ true
 -- !query
 SELECT '1' = cast('2017-12-11 09:30:00.0' as timestamp) FROM t
 -- !query schema
-struct<(CAST(1 AS TIMESTAMP) = CAST(2017-12-11 09:30:00.0 AS TIMESTAMP)):boolean>
+struct<(1 = CAST(2017-12-11 09:30:00.0 AS TIMESTAMP)):boolean>
 -- !query output
 NULL
 
@@ -1252,7 +1252,7 @@ NULL
 -- !query
 SELECT '1' = cast('2017-12-11 09:30:00' as date)        FROM t
 -- !query schema
-struct<(CAST(1 AS DATE) = CAST(2017-12-11 09:30:00 AS DATE)):boolean>
+struct<(1 = CAST(2017-12-11 09:30:00 AS DATE)):boolean>
 -- !query output
 NULL
 
@@ -1260,7 +1260,7 @@ NULL
 -- !query
 SELECT cast(1 as tinyint)                         = '1' FROM t
 -- !query schema
-struct<(CAST(1 AS TINYINT) = CAST(1 AS TINYINT)):boolean>
+struct<(CAST(1 AS TINYINT) = 1):boolean>
 -- !query output
 true
 
@@ -1268,7 +1268,7 @@ true
 -- !query
 SELECT cast(1 as smallint)                        = '1' FROM t
 -- !query schema
-struct<(CAST(1 AS SMALLINT) = CAST(1 AS SMALLINT)):boolean>
+struct<(CAST(1 AS SMALLINT) = 1):boolean>
 -- !query output
 true
 
@@ -1276,7 +1276,7 @@ true
 -- !query
 SELECT cast(1 as int)                             = '1' FROM t
 -- !query schema
-struct<(CAST(1 AS INT) = CAST(1 AS INT)):boolean>
+struct<(CAST(1 AS INT) = 1):boolean>
 -- !query output
 true
 
@@ -1284,7 +1284,7 @@ true
 -- !query
 SELECT cast(1 as bigint)                          = '1' FROM t
 -- !query schema
-struct<(CAST(1 AS BIGINT) = CAST(1 AS BIGINT)):boolean>
+struct<(CAST(1 AS BIGINT) = 1):boolean>
 -- !query output
 true
 
@@ -1292,7 +1292,7 @@ true
 -- !query
 SELECT cast(1 as float)                           = '1' FROM t
 -- !query schema
-struct<(CAST(1 AS FLOAT) = CAST(1 AS FLOAT)):boolean>
+struct<(CAST(1 AS FLOAT) = 1):boolean>
 -- !query output
 true
 
@@ -1300,7 +1300,7 @@ true
 -- !query
 SELECT cast(1 as double)                          = '1' FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) = CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) = 1):boolean>
 -- !query output
 true
 
@@ -1308,7 +1308,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0))                  = '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) = CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) = 1):boolean>
 -- !query output
 true
 
@@ -1316,7 +1316,7 @@ true
 -- !query
 SELECT cast('1' as binary)                        = '1' FROM t
 -- !query schema
-struct<(CAST(1 AS BINARY) = CAST(1 AS BINARY)):boolean>
+struct<(CAST(1 AS BINARY) = 1):boolean>
 -- !query output
 true
 
@@ -1324,7 +1324,7 @@ true
 -- !query
 SELECT cast(1 as boolean)                         = '1' FROM t
 -- !query schema
-struct<(CAST(1 AS BOOLEAN) = CAST(1 AS BOOLEAN)):boolean>
+struct<(CAST(1 AS BOOLEAN) = 1):boolean>
 -- !query output
 true
 
@@ -1332,7 +1332,7 @@ true
 -- !query
 SELECT cast('2017-12-11 09:30:00.0' as timestamp) = '1' FROM t
 -- !query schema
-struct<(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) = CAST(1 AS TIMESTAMP)):boolean>
+struct<(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) = 1):boolean>
 -- !query output
 NULL
 
@@ -1340,7 +1340,7 @@ NULL
 -- !query
 SELECT cast('2017-12-11 09:30:00' as date)        = '1' FROM t
 -- !query schema
-struct<(CAST(2017-12-11 09:30:00 AS DATE) = CAST(1 AS DATE)):boolean>
+struct<(CAST(2017-12-11 09:30:00 AS DATE) = 1):boolean>
 -- !query output
 NULL
 
@@ -1348,7 +1348,7 @@ NULL
 -- !query
 SELECT '1' <=> cast(1 as tinyint)                         FROM t
 -- !query schema
-struct<(CAST(1 AS TINYINT) <=> CAST(1 AS TINYINT)):boolean>
+struct<(1 <=> CAST(1 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -1356,7 +1356,7 @@ true
 -- !query
 SELECT '1' <=> cast(1 as smallint)                        FROM t
 -- !query schema
-struct<(CAST(1 AS SMALLINT) <=> CAST(1 AS SMALLINT)):boolean>
+struct<(1 <=> CAST(1 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -1364,7 +1364,7 @@ true
 -- !query
 SELECT '1' <=> cast(1 as int)                             FROM t
 -- !query schema
-struct<(CAST(1 AS INT) <=> CAST(1 AS INT)):boolean>
+struct<(1 <=> CAST(1 AS INT)):boolean>
 -- !query output
 true
 
@@ -1372,7 +1372,7 @@ true
 -- !query
 SELECT '1' <=> cast(1 as bigint)                          FROM t
 -- !query schema
-struct<(CAST(1 AS BIGINT) <=> CAST(1 AS BIGINT)):boolean>
+struct<(1 <=> CAST(1 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -1380,7 +1380,7 @@ true
 -- !query
 SELECT '1' <=> cast(1 as float)                           FROM t
 -- !query schema
-struct<(CAST(1 AS FLOAT) <=> CAST(1 AS FLOAT)):boolean>
+struct<(1 <=> CAST(1 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -1388,7 +1388,7 @@ true
 -- !query
 SELECT '1' <=> cast(1 as double)                          FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) <=> CAST(1 AS DOUBLE)):boolean>
+struct<(1 <=> CAST(1 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -1396,7 +1396,7 @@ true
 -- !query
 SELECT '1' <=> cast(1 as decimal(10, 0))                  FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) <=> CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(1 <=> CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -1412,7 +1412,7 @@ true
 -- !query
 SELECT '1' <=> cast('1' as binary)                        FROM t
 -- !query schema
-struct<(CAST(1 AS BINARY) <=> CAST(1 AS BINARY)):boolean>
+struct<(1 <=> CAST(1 AS BINARY)):boolean>
 -- !query output
 true
 
@@ -1420,7 +1420,7 @@ true
 -- !query
 SELECT '1' <=> cast(1 as boolean)                         FROM t
 -- !query schema
-struct<(CAST(1 AS BOOLEAN) <=> CAST(1 AS BOOLEAN)):boolean>
+struct<(1 <=> CAST(1 AS BOOLEAN)):boolean>
 -- !query output
 true
 
@@ -1428,7 +1428,7 @@ true
 -- !query
 SELECT '1' <=> cast('2017-12-11 09:30:00.0' as timestamp) FROM t
 -- !query schema
-struct<(CAST(1 AS TIMESTAMP) <=> CAST(2017-12-11 09:30:00.0 AS TIMESTAMP)):boolean>
+struct<(1 <=> CAST(2017-12-11 09:30:00.0 AS TIMESTAMP)):boolean>
 -- !query output
 false
 
@@ -1436,7 +1436,7 @@ false
 -- !query
 SELECT '1' <=> cast('2017-12-11 09:30:00' as date)        FROM t
 -- !query schema
-struct<(CAST(1 AS DATE) <=> CAST(2017-12-11 09:30:00 AS DATE)):boolean>
+struct<(1 <=> CAST(2017-12-11 09:30:00 AS DATE)):boolean>
 -- !query output
 false
 
@@ -1444,7 +1444,7 @@ false
 -- !query
 SELECT cast(1 as tinyint)                         <=> '1' FROM t
 -- !query schema
-struct<(CAST(1 AS TINYINT) <=> CAST(1 AS TINYINT)):boolean>
+struct<(CAST(1 AS TINYINT) <=> 1):boolean>
 -- !query output
 true
 
@@ -1452,7 +1452,7 @@ true
 -- !query
 SELECT cast(1 as smallint)                        <=> '1' FROM t
 -- !query schema
-struct<(CAST(1 AS SMALLINT) <=> CAST(1 AS SMALLINT)):boolean>
+struct<(CAST(1 AS SMALLINT) <=> 1):boolean>
 -- !query output
 true
 
@@ -1460,7 +1460,7 @@ true
 -- !query
 SELECT cast(1 as int)                             <=> '1' FROM t
 -- !query schema
-struct<(CAST(1 AS INT) <=> CAST(1 AS INT)):boolean>
+struct<(CAST(1 AS INT) <=> 1):boolean>
 -- !query output
 true
 
@@ -1468,7 +1468,7 @@ true
 -- !query
 SELECT cast(1 as bigint)                          <=> '1' FROM t
 -- !query schema
-struct<(CAST(1 AS BIGINT) <=> CAST(1 AS BIGINT)):boolean>
+struct<(CAST(1 AS BIGINT) <=> 1):boolean>
 -- !query output
 true
 
@@ -1476,7 +1476,7 @@ true
 -- !query
 SELECT cast(1 as float)                           <=> '1' FROM t
 -- !query schema
-struct<(CAST(1 AS FLOAT) <=> CAST(1 AS FLOAT)):boolean>
+struct<(CAST(1 AS FLOAT) <=> 1):boolean>
 -- !query output
 true
 
@@ -1484,7 +1484,7 @@ true
 -- !query
 SELECT cast(1 as double)                          <=> '1' FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) <=> CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) <=> 1):boolean>
 -- !query output
 true
 
@@ -1492,7 +1492,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0))                  <=> '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) <=> CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <=> 1):boolean>
 -- !query output
 true
 
@@ -1500,7 +1500,7 @@ true
 -- !query
 SELECT cast('1' as binary)                        <=> '1' FROM t
 -- !query schema
-struct<(CAST(1 AS BINARY) <=> CAST(1 AS BINARY)):boolean>
+struct<(CAST(1 AS BINARY) <=> 1):boolean>
 -- !query output
 true
 
@@ -1508,7 +1508,7 @@ true
 -- !query
 SELECT cast(1 as boolean)                         <=> '1' FROM t
 -- !query schema
-struct<(CAST(1 AS BOOLEAN) <=> CAST(1 AS BOOLEAN)):boolean>
+struct<(CAST(1 AS BOOLEAN) <=> 1):boolean>
 -- !query output
 true
 
@@ -1516,7 +1516,7 @@ true
 -- !query
 SELECT cast('2017-12-11 09:30:00.0' as timestamp) <=> '1' FROM t
 -- !query schema
-struct<(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) <=> CAST(1 AS TIMESTAMP)):boolean>
+struct<(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) <=> 1):boolean>
 -- !query output
 false
 
@@ -1524,7 +1524,7 @@ false
 -- !query
 SELECT cast('2017-12-11 09:30:00' as date)        <=> '1' FROM t
 -- !query schema
-struct<(CAST(2017-12-11 09:30:00 AS DATE) <=> CAST(1 AS DATE)):boolean>
+struct<(CAST(2017-12-11 09:30:00 AS DATE) <=> 1):boolean>
 -- !query output
 false
 
@@ -1532,7 +1532,7 @@ false
 -- !query
 SELECT '1' < cast(1 as tinyint)                         FROM t
 -- !query schema
-struct<(CAST(1 AS TINYINT) < CAST(1 AS TINYINT)):boolean>
+struct<(1 < CAST(1 AS TINYINT)):boolean>
 -- !query output
 false
 
@@ -1540,7 +1540,7 @@ false
 -- !query
 SELECT '1' < cast(1 as smallint)                        FROM t
 -- !query schema
-struct<(CAST(1 AS SMALLINT) < CAST(1 AS SMALLINT)):boolean>
+struct<(1 < CAST(1 AS SMALLINT)):boolean>
 -- !query output
 false
 
@@ -1548,7 +1548,7 @@ false
 -- !query
 SELECT '1' < cast(1 as int)                             FROM t
 -- !query schema
-struct<(CAST(1 AS INT) < CAST(1 AS INT)):boolean>
+struct<(1 < CAST(1 AS INT)):boolean>
 -- !query output
 false
 
@@ -1556,7 +1556,7 @@ false
 -- !query
 SELECT '1' < cast(1 as bigint)                          FROM t
 -- !query schema
-struct<(CAST(1 AS BIGINT) < CAST(1 AS BIGINT)):boolean>
+struct<(1 < CAST(1 AS BIGINT)):boolean>
 -- !query output
 false
 
@@ -1564,7 +1564,7 @@ false
 -- !query
 SELECT '1' < cast(1 as float)                           FROM t
 -- !query schema
-struct<(CAST(1 AS FLOAT) < CAST(1 AS FLOAT)):boolean>
+struct<(1 < CAST(1 AS FLOAT)):boolean>
 -- !query output
 false
 
@@ -1572,7 +1572,7 @@ false
 -- !query
 SELECT '1' < cast(1 as double)                          FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) < CAST(1 AS DOUBLE)):boolean>
+struct<(1 < CAST(1 AS DOUBLE)):boolean>
 -- !query output
 false
 
@@ -1580,7 +1580,7 @@ false
 -- !query
 SELECT '1' < cast(1 as decimal(10, 0))                  FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) < CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(1 < CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 false
 
@@ -1596,7 +1596,7 @@ false
 -- !query
 SELECT '1' < cast('1' as binary)                        FROM t
 -- !query schema
-struct<(CAST(1 AS BINARY) < CAST(1 AS BINARY)):boolean>
+struct<(1 < CAST(1 AS BINARY)):boolean>
 -- !query output
 false
 
@@ -1604,7 +1604,7 @@ false
 -- !query
 SELECT '1' < cast(1 as boolean)                         FROM t
 -- !query schema
-struct<(CAST(1 AS BOOLEAN) < CAST(1 AS BOOLEAN)):boolean>
+struct<(1 < CAST(1 AS BOOLEAN)):boolean>
 -- !query output
 false
 
@@ -1612,7 +1612,7 @@ false
 -- !query
 SELECT '1' < cast('2017-12-11 09:30:00.0' as timestamp) FROM t
 -- !query schema
-struct<(CAST(1 AS TIMESTAMP) < CAST(2017-12-11 09:30:00.0 AS TIMESTAMP)):boolean>
+struct<(1 < CAST(2017-12-11 09:30:00.0 AS TIMESTAMP)):boolean>
 -- !query output
 NULL
 
@@ -1620,7 +1620,7 @@ NULL
 -- !query
 SELECT '1' < cast('2017-12-11 09:30:00' as date)        FROM t
 -- !query schema
-struct<(CAST(1 AS DATE) < CAST(2017-12-11 09:30:00 AS DATE)):boolean>
+struct<(1 < CAST(2017-12-11 09:30:00 AS DATE)):boolean>
 -- !query output
 NULL
 
@@ -1628,7 +1628,7 @@ NULL
 -- !query
 SELECT '1' <= cast(1 as tinyint)                         FROM t
 -- !query schema
-struct<(CAST(1 AS TINYINT) <= CAST(1 AS TINYINT)):boolean>
+struct<(1 <= CAST(1 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -1636,7 +1636,7 @@ true
 -- !query
 SELECT '1' <= cast(1 as smallint)                        FROM t
 -- !query schema
-struct<(CAST(1 AS SMALLINT) <= CAST(1 AS SMALLINT)):boolean>
+struct<(1 <= CAST(1 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -1644,7 +1644,7 @@ true
 -- !query
 SELECT '1' <= cast(1 as int)                             FROM t
 -- !query schema
-struct<(CAST(1 AS INT) <= CAST(1 AS INT)):boolean>
+struct<(1 <= CAST(1 AS INT)):boolean>
 -- !query output
 true
 
@@ -1652,7 +1652,7 @@ true
 -- !query
 SELECT '1' <= cast(1 as bigint)                          FROM t
 -- !query schema
-struct<(CAST(1 AS BIGINT) <= CAST(1 AS BIGINT)):boolean>
+struct<(1 <= CAST(1 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -1660,7 +1660,7 @@ true
 -- !query
 SELECT '1' <= cast(1 as float)                           FROM t
 -- !query schema
-struct<(CAST(1 AS FLOAT) <= CAST(1 AS FLOAT)):boolean>
+struct<(1 <= CAST(1 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -1668,7 +1668,7 @@ true
 -- !query
 SELECT '1' <= cast(1 as double)                          FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) <= CAST(1 AS DOUBLE)):boolean>
+struct<(1 <= CAST(1 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -1676,7 +1676,7 @@ true
 -- !query
 SELECT '1' <= cast(1 as decimal(10, 0))                  FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) <= CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(1 <= CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -1692,7 +1692,7 @@ true
 -- !query
 SELECT '1' <= cast('1' as binary)                        FROM t
 -- !query schema
-struct<(CAST(1 AS BINARY) <= CAST(1 AS BINARY)):boolean>
+struct<(1 <= CAST(1 AS BINARY)):boolean>
 -- !query output
 true
 
@@ -1700,7 +1700,7 @@ true
 -- !query
 SELECT '1' <= cast(1 as boolean)                         FROM t
 -- !query schema
-struct<(CAST(1 AS BOOLEAN) <= CAST(1 AS BOOLEAN)):boolean>
+struct<(1 <= CAST(1 AS BOOLEAN)):boolean>
 -- !query output
 true
 
@@ -1708,7 +1708,7 @@ true
 -- !query
 SELECT '1' <= cast('2017-12-11 09:30:00.0' as timestamp) FROM t
 -- !query schema
-struct<(CAST(1 AS TIMESTAMP) <= CAST(2017-12-11 09:30:00.0 AS TIMESTAMP)):boolean>
+struct<(1 <= CAST(2017-12-11 09:30:00.0 AS TIMESTAMP)):boolean>
 -- !query output
 NULL
 
@@ -1716,7 +1716,7 @@ NULL
 -- !query
 SELECT '1' <= cast('2017-12-11 09:30:00' as date)        FROM t
 -- !query schema
-struct<(CAST(1 AS DATE) <= CAST(2017-12-11 09:30:00 AS DATE)):boolean>
+struct<(1 <= CAST(2017-12-11 09:30:00 AS DATE)):boolean>
 -- !query output
 NULL
 
@@ -1724,7 +1724,7 @@ NULL
 -- !query
 SELECT '1' > cast(1 as tinyint)                         FROM t
 -- !query schema
-struct<(CAST(1 AS TINYINT) > CAST(1 AS TINYINT)):boolean>
+struct<(1 > CAST(1 AS TINYINT)):boolean>
 -- !query output
 false
 
@@ -1732,7 +1732,7 @@ false
 -- !query
 SELECT '1' > cast(1 as smallint)                        FROM t
 -- !query schema
-struct<(CAST(1 AS SMALLINT) > CAST(1 AS SMALLINT)):boolean>
+struct<(1 > CAST(1 AS SMALLINT)):boolean>
 -- !query output
 false
 
@@ -1740,7 +1740,7 @@ false
 -- !query
 SELECT '1' > cast(1 as int)                             FROM t
 -- !query schema
-struct<(CAST(1 AS INT) > CAST(1 AS INT)):boolean>
+struct<(1 > CAST(1 AS INT)):boolean>
 -- !query output
 false
 
@@ -1748,7 +1748,7 @@ false
 -- !query
 SELECT '1' > cast(1 as bigint)                          FROM t
 -- !query schema
-struct<(CAST(1 AS BIGINT) > CAST(1 AS BIGINT)):boolean>
+struct<(1 > CAST(1 AS BIGINT)):boolean>
 -- !query output
 false
 
@@ -1756,7 +1756,7 @@ false
 -- !query
 SELECT '1' > cast(1 as float)                           FROM t
 -- !query schema
-struct<(CAST(1 AS FLOAT) > CAST(1 AS FLOAT)):boolean>
+struct<(1 > CAST(1 AS FLOAT)):boolean>
 -- !query output
 false
 
@@ -1764,7 +1764,7 @@ false
 -- !query
 SELECT '1' > cast(1 as double)                          FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) > CAST(1 AS DOUBLE)):boolean>
+struct<(1 > CAST(1 AS DOUBLE)):boolean>
 -- !query output
 false
 
@@ -1772,7 +1772,7 @@ false
 -- !query
 SELECT '1' > cast(1 as decimal(10, 0))                  FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) > CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(1 > CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 false
 
@@ -1788,7 +1788,7 @@ false
 -- !query
 SELECT '1' > cast('1' as binary)                        FROM t
 -- !query schema
-struct<(CAST(1 AS BINARY) > CAST(1 AS BINARY)):boolean>
+struct<(1 > CAST(1 AS BINARY)):boolean>
 -- !query output
 false
 
@@ -1796,7 +1796,7 @@ false
 -- !query
 SELECT '1' > cast(1 as boolean)                         FROM t
 -- !query schema
-struct<(CAST(1 AS BOOLEAN) > CAST(1 AS BOOLEAN)):boolean>
+struct<(1 > CAST(1 AS BOOLEAN)):boolean>
 -- !query output
 false
 
@@ -1804,7 +1804,7 @@ false
 -- !query
 SELECT '1' > cast('2017-12-11 09:30:00.0' as timestamp) FROM t
 -- !query schema
-struct<(CAST(1 AS TIMESTAMP) > CAST(2017-12-11 09:30:00.0 AS TIMESTAMP)):boolean>
+struct<(1 > CAST(2017-12-11 09:30:00.0 AS TIMESTAMP)):boolean>
 -- !query output
 NULL
 
@@ -1812,7 +1812,7 @@ NULL
 -- !query
 SELECT '1' > cast('2017-12-11 09:30:00' as date)        FROM t
 -- !query schema
-struct<(CAST(1 AS DATE) > CAST(2017-12-11 09:30:00 AS DATE)):boolean>
+struct<(1 > CAST(2017-12-11 09:30:00 AS DATE)):boolean>
 -- !query output
 NULL
 
@@ -1820,7 +1820,7 @@ NULL
 -- !query
 SELECT '1' >= cast(1 as tinyint)                         FROM t
 -- !query schema
-struct<(CAST(1 AS TINYINT) >= CAST(1 AS TINYINT)):boolean>
+struct<(1 >= CAST(1 AS TINYINT)):boolean>
 -- !query output
 true
 
@@ -1828,7 +1828,7 @@ true
 -- !query
 SELECT '1' >= cast(1 as smallint)                        FROM t
 -- !query schema
-struct<(CAST(1 AS SMALLINT) >= CAST(1 AS SMALLINT)):boolean>
+struct<(1 >= CAST(1 AS SMALLINT)):boolean>
 -- !query output
 true
 
@@ -1836,7 +1836,7 @@ true
 -- !query
 SELECT '1' >= cast(1 as int)                             FROM t
 -- !query schema
-struct<(CAST(1 AS INT) >= CAST(1 AS INT)):boolean>
+struct<(1 >= CAST(1 AS INT)):boolean>
 -- !query output
 true
 
@@ -1844,7 +1844,7 @@ true
 -- !query
 SELECT '1' >= cast(1 as bigint)                          FROM t
 -- !query schema
-struct<(CAST(1 AS BIGINT) >= CAST(1 AS BIGINT)):boolean>
+struct<(1 >= CAST(1 AS BIGINT)):boolean>
 -- !query output
 true
 
@@ -1852,7 +1852,7 @@ true
 -- !query
 SELECT '1' >= cast(1 as float)                           FROM t
 -- !query schema
-struct<(CAST(1 AS FLOAT) >= CAST(1 AS FLOAT)):boolean>
+struct<(1 >= CAST(1 AS FLOAT)):boolean>
 -- !query output
 true
 
@@ -1860,7 +1860,7 @@ true
 -- !query
 SELECT '1' >= cast(1 as double)                          FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) >= CAST(1 AS DOUBLE)):boolean>
+struct<(1 >= CAST(1 AS DOUBLE)):boolean>
 -- !query output
 true
 
@@ -1868,7 +1868,7 @@ true
 -- !query
 SELECT '1' >= cast(1 as decimal(10, 0))                  FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) >= CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE)):boolean>
+struct<(1 >= CAST(1 AS DECIMAL(10,0))):boolean>
 -- !query output
 true
 
@@ -1884,7 +1884,7 @@ true
 -- !query
 SELECT '1' >= cast('1' as binary)                        FROM t
 -- !query schema
-struct<(CAST(1 AS BINARY) >= CAST(1 AS BINARY)):boolean>
+struct<(1 >= CAST(1 AS BINARY)):boolean>
 -- !query output
 true
 
@@ -1892,7 +1892,7 @@ true
 -- !query
 SELECT '1' >= cast(1 as boolean)                         FROM t
 -- !query schema
-struct<(CAST(1 AS BOOLEAN) >= CAST(1 AS BOOLEAN)):boolean>
+struct<(1 >= CAST(1 AS BOOLEAN)):boolean>
 -- !query output
 true
 
@@ -1900,7 +1900,7 @@ true
 -- !query
 SELECT '1' >= cast('2017-12-11 09:30:00.0' as timestamp) FROM t
 -- !query schema
-struct<(CAST(1 AS TIMESTAMP) >= CAST(2017-12-11 09:30:00.0 AS TIMESTAMP)):boolean>
+struct<(1 >= CAST(2017-12-11 09:30:00.0 AS TIMESTAMP)):boolean>
 -- !query output
 NULL
 
@@ -1908,7 +1908,7 @@ NULL
 -- !query
 SELECT '1' >= cast('2017-12-11 09:30:00' as date)        FROM t
 -- !query schema
-struct<(CAST(1 AS DATE) >= CAST(2017-12-11 09:30:00 AS DATE)):boolean>
+struct<(1 >= CAST(2017-12-11 09:30:00 AS DATE)):boolean>
 -- !query output
 NULL
 
@@ -1916,7 +1916,7 @@ NULL
 -- !query
 SELECT '1' <> cast(1 as tinyint)                         FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS TINYINT) = CAST(1 AS TINYINT))):boolean>
+struct<(NOT (1 = CAST(1 AS TINYINT))):boolean>
 -- !query output
 false
 
@@ -1924,7 +1924,7 @@ false
 -- !query
 SELECT '1' <> cast(1 as smallint)                        FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS SMALLINT) = CAST(1 AS SMALLINT))):boolean>
+struct<(NOT (1 = CAST(1 AS SMALLINT))):boolean>
 -- !query output
 false
 
@@ -1932,7 +1932,7 @@ false
 -- !query
 SELECT '1' <> cast(1 as int)                             FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS INT) = CAST(1 AS INT))):boolean>
+struct<(NOT (1 = CAST(1 AS INT))):boolean>
 -- !query output
 false
 
@@ -1940,7 +1940,7 @@ false
 -- !query
 SELECT '1' <> cast(1 as bigint)                          FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS BIGINT) = CAST(1 AS BIGINT))):boolean>
+struct<(NOT (1 = CAST(1 AS BIGINT))):boolean>
 -- !query output
 false
 
@@ -1948,7 +1948,7 @@ false
 -- !query
 SELECT '1' <> cast(1 as float)                           FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS FLOAT) = CAST(1 AS FLOAT))):boolean>
+struct<(NOT (1 = CAST(1 AS FLOAT))):boolean>
 -- !query output
 false
 
@@ -1956,7 +1956,7 @@ false
 -- !query
 SELECT '1' <> cast(1 as double)                          FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS DOUBLE) = CAST(1 AS DOUBLE))):boolean>
+struct<(NOT (1 = CAST(1 AS DOUBLE))):boolean>
 -- !query output
 false
 
@@ -1964,7 +1964,7 @@ false
 -- !query
 SELECT '1' <> cast(1 as decimal(10, 0))                  FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS DOUBLE) = CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE))):boolean>
+struct<(NOT (1 = CAST(1 AS DECIMAL(10,0)))):boolean>
 -- !query output
 false
 
@@ -1980,7 +1980,7 @@ false
 -- !query
 SELECT '1' <> cast('1' as binary)                        FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS BINARY) = CAST(1 AS BINARY))):boolean>
+struct<(NOT (1 = CAST(1 AS BINARY))):boolean>
 -- !query output
 false
 
@@ -1988,7 +1988,7 @@ false
 -- !query
 SELECT '1' <> cast(1 as boolean)                         FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS BOOLEAN) = CAST(1 AS BOOLEAN))):boolean>
+struct<(NOT (1 = CAST(1 AS BOOLEAN))):boolean>
 -- !query output
 false
 
@@ -1996,7 +1996,7 @@ false
 -- !query
 SELECT '1' <> cast('2017-12-11 09:30:00.0' as timestamp) FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS TIMESTAMP) = CAST(2017-12-11 09:30:00.0 AS TIMESTAMP))):boolean>
+struct<(NOT (1 = CAST(2017-12-11 09:30:00.0 AS TIMESTAMP))):boolean>
 -- !query output
 NULL
 
@@ -2004,7 +2004,7 @@ NULL
 -- !query
 SELECT '1' <> cast('2017-12-11 09:30:00' as date)        FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS DATE) = CAST(2017-12-11 09:30:00 AS DATE))):boolean>
+struct<(NOT (1 = CAST(2017-12-11 09:30:00 AS DATE))):boolean>
 -- !query output
 NULL
 
@@ -2012,7 +2012,7 @@ NULL
 -- !query
 SELECT cast(1 as tinyint)                         < '1' FROM t
 -- !query schema
-struct<(CAST(1 AS TINYINT) < CAST(1 AS TINYINT)):boolean>
+struct<(CAST(1 AS TINYINT) < 1):boolean>
 -- !query output
 false
 
@@ -2020,7 +2020,7 @@ false
 -- !query
 SELECT cast(1 as smallint)                        < '1' FROM t
 -- !query schema
-struct<(CAST(1 AS SMALLINT) < CAST(1 AS SMALLINT)):boolean>
+struct<(CAST(1 AS SMALLINT) < 1):boolean>
 -- !query output
 false
 
@@ -2028,7 +2028,7 @@ false
 -- !query
 SELECT cast(1 as int)                             < '1' FROM t
 -- !query schema
-struct<(CAST(1 AS INT) < CAST(1 AS INT)):boolean>
+struct<(CAST(1 AS INT) < 1):boolean>
 -- !query output
 false
 
@@ -2036,7 +2036,7 @@ false
 -- !query
 SELECT cast(1 as bigint)                          < '1' FROM t
 -- !query schema
-struct<(CAST(1 AS BIGINT) < CAST(1 AS BIGINT)):boolean>
+struct<(CAST(1 AS BIGINT) < 1):boolean>
 -- !query output
 false
 
@@ -2044,7 +2044,7 @@ false
 -- !query
 SELECT cast(1 as float)                           < '1' FROM t
 -- !query schema
-struct<(CAST(1 AS FLOAT) < CAST(1 AS FLOAT)):boolean>
+struct<(CAST(1 AS FLOAT) < 1):boolean>
 -- !query output
 false
 
@@ -2052,7 +2052,7 @@ false
 -- !query
 SELECT cast(1 as double)                          < '1' FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) < CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) < 1):boolean>
 -- !query output
 false
 
@@ -2060,7 +2060,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0))                  < '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) < CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) < 1):boolean>
 -- !query output
 false
 
@@ -2076,7 +2076,7 @@ false
 -- !query
 SELECT cast('1' as binary)                        < '1' FROM t
 -- !query schema
-struct<(CAST(1 AS BINARY) < CAST(1 AS BINARY)):boolean>
+struct<(CAST(1 AS BINARY) < 1):boolean>
 -- !query output
 false
 
@@ -2084,7 +2084,7 @@ false
 -- !query
 SELECT cast(1 as boolean)                         < '1' FROM t
 -- !query schema
-struct<(CAST(1 AS BOOLEAN) < CAST(1 AS BOOLEAN)):boolean>
+struct<(CAST(1 AS BOOLEAN) < 1):boolean>
 -- !query output
 false
 
@@ -2092,7 +2092,7 @@ false
 -- !query
 SELECT cast('2017-12-11 09:30:00.0' as timestamp) < '1' FROM t
 -- !query schema
-struct<(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) < CAST(1 AS TIMESTAMP)):boolean>
+struct<(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) < 1):boolean>
 -- !query output
 NULL
 
@@ -2100,7 +2100,7 @@ NULL
 -- !query
 SELECT cast('2017-12-11 09:30:00' as date)        < '1' FROM t
 -- !query schema
-struct<(CAST(2017-12-11 09:30:00 AS DATE) < CAST(1 AS DATE)):boolean>
+struct<(CAST(2017-12-11 09:30:00 AS DATE) < 1):boolean>
 -- !query output
 NULL
 
@@ -2108,7 +2108,7 @@ NULL
 -- !query
 SELECT cast(1 as tinyint)                         <= '1' FROM t
 -- !query schema
-struct<(CAST(1 AS TINYINT) <= CAST(1 AS TINYINT)):boolean>
+struct<(CAST(1 AS TINYINT) <= 1):boolean>
 -- !query output
 true
 
@@ -2116,7 +2116,7 @@ true
 -- !query
 SELECT cast(1 as smallint)                        <= '1' FROM t
 -- !query schema
-struct<(CAST(1 AS SMALLINT) <= CAST(1 AS SMALLINT)):boolean>
+struct<(CAST(1 AS SMALLINT) <= 1):boolean>
 -- !query output
 true
 
@@ -2124,7 +2124,7 @@ true
 -- !query
 SELECT cast(1 as int)                             <= '1' FROM t
 -- !query schema
-struct<(CAST(1 AS INT) <= CAST(1 AS INT)):boolean>
+struct<(CAST(1 AS INT) <= 1):boolean>
 -- !query output
 true
 
@@ -2132,7 +2132,7 @@ true
 -- !query
 SELECT cast(1 as bigint)                          <= '1' FROM t
 -- !query schema
-struct<(CAST(1 AS BIGINT) <= CAST(1 AS BIGINT)):boolean>
+struct<(CAST(1 AS BIGINT) <= 1):boolean>
 -- !query output
 true
 
@@ -2140,7 +2140,7 @@ true
 -- !query
 SELECT cast(1 as float)                           <= '1' FROM t
 -- !query schema
-struct<(CAST(1 AS FLOAT) <= CAST(1 AS FLOAT)):boolean>
+struct<(CAST(1 AS FLOAT) <= 1):boolean>
 -- !query output
 true
 
@@ -2148,7 +2148,7 @@ true
 -- !query
 SELECT cast(1 as double)                          <= '1' FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) <= CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) <= 1):boolean>
 -- !query output
 true
 
@@ -2156,7 +2156,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0))                  <= '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) <= CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) <= 1):boolean>
 -- !query output
 true
 
@@ -2172,7 +2172,7 @@ true
 -- !query
 SELECT cast('1' as binary)                        <= '1' FROM t
 -- !query schema
-struct<(CAST(1 AS BINARY) <= CAST(1 AS BINARY)):boolean>
+struct<(CAST(1 AS BINARY) <= 1):boolean>
 -- !query output
 true
 
@@ -2180,7 +2180,7 @@ true
 -- !query
 SELECT cast(1 as boolean)                         <= '1' FROM t
 -- !query schema
-struct<(CAST(1 AS BOOLEAN) <= CAST(1 AS BOOLEAN)):boolean>
+struct<(CAST(1 AS BOOLEAN) <= 1):boolean>
 -- !query output
 true
 
@@ -2188,7 +2188,7 @@ true
 -- !query
 SELECT cast('2017-12-11 09:30:00.0' as timestamp) <= '1' FROM t
 -- !query schema
-struct<(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) <= CAST(1 AS TIMESTAMP)):boolean>
+struct<(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) <= 1):boolean>
 -- !query output
 NULL
 
@@ -2196,7 +2196,7 @@ NULL
 -- !query
 SELECT cast('2017-12-11 09:30:00' as date)        <= '1' FROM t
 -- !query schema
-struct<(CAST(2017-12-11 09:30:00 AS DATE) <= CAST(1 AS DATE)):boolean>
+struct<(CAST(2017-12-11 09:30:00 AS DATE) <= 1):boolean>
 -- !query output
 NULL
 
@@ -2204,7 +2204,7 @@ NULL
 -- !query
 SELECT cast(1 as tinyint)                         > '1' FROM t
 -- !query schema
-struct<(CAST(1 AS TINYINT) > CAST(1 AS TINYINT)):boolean>
+struct<(CAST(1 AS TINYINT) > 1):boolean>
 -- !query output
 false
 
@@ -2212,7 +2212,7 @@ false
 -- !query
 SELECT cast(1 as smallint)                        > '1' FROM t
 -- !query schema
-struct<(CAST(1 AS SMALLINT) > CAST(1 AS SMALLINT)):boolean>
+struct<(CAST(1 AS SMALLINT) > 1):boolean>
 -- !query output
 false
 
@@ -2220,7 +2220,7 @@ false
 -- !query
 SELECT cast(1 as int)                             > '1' FROM t
 -- !query schema
-struct<(CAST(1 AS INT) > CAST(1 AS INT)):boolean>
+struct<(CAST(1 AS INT) > 1):boolean>
 -- !query output
 false
 
@@ -2228,7 +2228,7 @@ false
 -- !query
 SELECT cast(1 as bigint)                          > '1' FROM t
 -- !query schema
-struct<(CAST(1 AS BIGINT) > CAST(1 AS BIGINT)):boolean>
+struct<(CAST(1 AS BIGINT) > 1):boolean>
 -- !query output
 false
 
@@ -2236,7 +2236,7 @@ false
 -- !query
 SELECT cast(1 as float)                           > '1' FROM t
 -- !query schema
-struct<(CAST(1 AS FLOAT) > CAST(1 AS FLOAT)):boolean>
+struct<(CAST(1 AS FLOAT) > 1):boolean>
 -- !query output
 false
 
@@ -2244,7 +2244,7 @@ false
 -- !query
 SELECT cast(1 as double)                          > '1' FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) > CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) > 1):boolean>
 -- !query output
 false
 
@@ -2252,7 +2252,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0))                  > '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) > CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) > 1):boolean>
 -- !query output
 false
 
@@ -2268,7 +2268,7 @@ false
 -- !query
 SELECT cast('1' as binary)                        > '1' FROM t
 -- !query schema
-struct<(CAST(1 AS BINARY) > CAST(1 AS BINARY)):boolean>
+struct<(CAST(1 AS BINARY) > 1):boolean>
 -- !query output
 false
 
@@ -2276,7 +2276,7 @@ false
 -- !query
 SELECT cast(1 as boolean)                         > '1' FROM t
 -- !query schema
-struct<(CAST(1 AS BOOLEAN) > CAST(1 AS BOOLEAN)):boolean>
+struct<(CAST(1 AS BOOLEAN) > 1):boolean>
 -- !query output
 false
 
@@ -2284,7 +2284,7 @@ false
 -- !query
 SELECT cast('2017-12-11 09:30:00.0' as timestamp) > '1' FROM t
 -- !query schema
-struct<(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) > CAST(1 AS TIMESTAMP)):boolean>
+struct<(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) > 1):boolean>
 -- !query output
 NULL
 
@@ -2292,7 +2292,7 @@ NULL
 -- !query
 SELECT cast('2017-12-11 09:30:00' as date)        > '1' FROM t
 -- !query schema
-struct<(CAST(2017-12-11 09:30:00 AS DATE) > CAST(1 AS DATE)):boolean>
+struct<(CAST(2017-12-11 09:30:00 AS DATE) > 1):boolean>
 -- !query output
 NULL
 
@@ -2300,7 +2300,7 @@ NULL
 -- !query
 SELECT cast(1 as tinyint)                         >= '1' FROM t
 -- !query schema
-struct<(CAST(1 AS TINYINT) >= CAST(1 AS TINYINT)):boolean>
+struct<(CAST(1 AS TINYINT) >= 1):boolean>
 -- !query output
 true
 
@@ -2308,7 +2308,7 @@ true
 -- !query
 SELECT cast(1 as smallint)                        >= '1' FROM t
 -- !query schema
-struct<(CAST(1 AS SMALLINT) >= CAST(1 AS SMALLINT)):boolean>
+struct<(CAST(1 AS SMALLINT) >= 1):boolean>
 -- !query output
 true
 
@@ -2316,7 +2316,7 @@ true
 -- !query
 SELECT cast(1 as int)                             >= '1' FROM t
 -- !query schema
-struct<(CAST(1 AS INT) >= CAST(1 AS INT)):boolean>
+struct<(CAST(1 AS INT) >= 1):boolean>
 -- !query output
 true
 
@@ -2324,7 +2324,7 @@ true
 -- !query
 SELECT cast(1 as bigint)                          >= '1' FROM t
 -- !query schema
-struct<(CAST(1 AS BIGINT) >= CAST(1 AS BIGINT)):boolean>
+struct<(CAST(1 AS BIGINT) >= 1):boolean>
 -- !query output
 true
 
@@ -2332,7 +2332,7 @@ true
 -- !query
 SELECT cast(1 as float)                           >= '1' FROM t
 -- !query schema
-struct<(CAST(1 AS FLOAT) >= CAST(1 AS FLOAT)):boolean>
+struct<(CAST(1 AS FLOAT) >= 1):boolean>
 -- !query output
 true
 
@@ -2340,7 +2340,7 @@ true
 -- !query
 SELECT cast(1 as double)                          >= '1' FROM t
 -- !query schema
-struct<(CAST(1 AS DOUBLE) >= CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DOUBLE) >= 1):boolean>
 -- !query output
 true
 
@@ -2348,7 +2348,7 @@ true
 -- !query
 SELECT cast(1 as decimal(10, 0))                  >= '1' FROM t
 -- !query schema
-struct<(CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) >= CAST(1 AS DOUBLE)):boolean>
+struct<(CAST(1 AS DECIMAL(10,0)) >= 1):boolean>
 -- !query output
 true
 
@@ -2364,7 +2364,7 @@ true
 -- !query
 SELECT cast('1' as binary)                        >= '1' FROM t
 -- !query schema
-struct<(CAST(1 AS BINARY) >= CAST(1 AS BINARY)):boolean>
+struct<(CAST(1 AS BINARY) >= 1):boolean>
 -- !query output
 true
 
@@ -2372,7 +2372,7 @@ true
 -- !query
 SELECT cast(1 as boolean)                         >= '1' FROM t
 -- !query schema
-struct<(CAST(1 AS BOOLEAN) >= CAST(1 AS BOOLEAN)):boolean>
+struct<(CAST(1 AS BOOLEAN) >= 1):boolean>
 -- !query output
 true
 
@@ -2380,7 +2380,7 @@ true
 -- !query
 SELECT cast('2017-12-11 09:30:00.0' as timestamp) >= '1' FROM t
 -- !query schema
-struct<(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) >= CAST(1 AS TIMESTAMP)):boolean>
+struct<(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) >= 1):boolean>
 -- !query output
 NULL
 
@@ -2388,7 +2388,7 @@ NULL
 -- !query
 SELECT cast('2017-12-11 09:30:00' as date)        >= '1' FROM t
 -- !query schema
-struct<(CAST(2017-12-11 09:30:00 AS DATE) >= CAST(1 AS DATE)):boolean>
+struct<(CAST(2017-12-11 09:30:00 AS DATE) >= 1):boolean>
 -- !query output
 NULL
 
@@ -2396,7 +2396,7 @@ NULL
 -- !query
 SELECT cast(1 as tinyint)                         <> '1' FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS TINYINT) = CAST(1 AS TINYINT))):boolean>
+struct<(NOT (CAST(1 AS TINYINT) = 1)):boolean>
 -- !query output
 false
 
@@ -2404,7 +2404,7 @@ false
 -- !query
 SELECT cast(1 as smallint)                        <> '1' FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS SMALLINT) = CAST(1 AS SMALLINT))):boolean>
+struct<(NOT (CAST(1 AS SMALLINT) = 1)):boolean>
 -- !query output
 false
 
@@ -2412,7 +2412,7 @@ false
 -- !query
 SELECT cast(1 as int)                             <> '1' FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS INT) = CAST(1 AS INT))):boolean>
+struct<(NOT (CAST(1 AS INT) = 1)):boolean>
 -- !query output
 false
 
@@ -2420,7 +2420,7 @@ false
 -- !query
 SELECT cast(1 as bigint)                          <> '1' FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS BIGINT) = CAST(1 AS BIGINT))):boolean>
+struct<(NOT (CAST(1 AS BIGINT) = 1)):boolean>
 -- !query output
 false
 
@@ -2428,7 +2428,7 @@ false
 -- !query
 SELECT cast(1 as float)                           <> '1' FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS FLOAT) = CAST(1 AS FLOAT))):boolean>
+struct<(NOT (CAST(1 AS FLOAT) = 1)):boolean>
 -- !query output
 false
 
@@ -2436,7 +2436,7 @@ false
 -- !query
 SELECT cast(1 as double)                          <> '1' FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS DOUBLE) = CAST(1 AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS DOUBLE) = 1)):boolean>
 -- !query output
 false
 
@@ -2444,7 +2444,7 @@ false
 -- !query
 SELECT cast(1 as decimal(10, 0))                  <> '1' FROM t
 -- !query schema
-struct<(NOT (CAST(CAST(1 AS DECIMAL(10,0)) AS DOUBLE) = CAST(1 AS DOUBLE))):boolean>
+struct<(NOT (CAST(1 AS DECIMAL(10,0)) = 1)):boolean>
 -- !query output
 false
 
@@ -2460,7 +2460,7 @@ false
 -- !query
 SELECT cast('1' as binary)                        <> '1' FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS BINARY) = CAST(1 AS BINARY))):boolean>
+struct<(NOT (CAST(1 AS BINARY) = 1)):boolean>
 -- !query output
 false
 
@@ -2468,7 +2468,7 @@ false
 -- !query
 SELECT cast(1 as boolean)                         <> '1' FROM t
 -- !query schema
-struct<(NOT (CAST(1 AS BOOLEAN) = CAST(1 AS BOOLEAN))):boolean>
+struct<(NOT (CAST(1 AS BOOLEAN) = 1)):boolean>
 -- !query output
 false
 
@@ -2476,7 +2476,7 @@ false
 -- !query
 SELECT cast('2017-12-11 09:30:00.0' as timestamp) <> '1' FROM t
 -- !query schema
-struct<(NOT (CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) = CAST(1 AS TIMESTAMP))):boolean>
+struct<(NOT (CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) = 1)):boolean>
 -- !query output
 NULL
 
@@ -2484,7 +2484,7 @@ NULL
 -- !query
 SELECT cast('2017-12-11 09:30:00' as date)        <> '1' FROM t
 -- !query schema
-struct<(NOT (CAST(2017-12-11 09:30:00 AS DATE) = CAST(1 AS DATE))):boolean>
+struct<(NOT (CAST(2017-12-11 09:30:00 AS DATE) = 1)):boolean>
 -- !query output
 NULL
 
@@ -2492,7 +2492,7 @@ NULL
 -- !query
 SELECT abs('1') FROM t
 -- !query schema
-struct<abs(CAST(1 AS DOUBLE)):double>
+struct<abs(1):double>
 -- !query output
 1.0
 
@@ -2500,7 +2500,7 @@ struct<abs(CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT sum('1') FROM t
 -- !query schema
-struct<sum(CAST(1 AS DOUBLE)):double>
+struct<sum(1):double>
 -- !query output
 1.0
 
@@ -2508,7 +2508,7 @@ struct<sum(CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT avg('1') FROM t
 -- !query schema
-struct<avg(CAST(1 AS DOUBLE)):double>
+struct<avg(1):double>
 -- !query output
 1.0
 
@@ -2516,7 +2516,7 @@ struct<avg(CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT stddev_pop('1') FROM t
 -- !query schema
-struct<stddev_pop(CAST(1 AS DOUBLE)):double>
+struct<stddev_pop(1):double>
 -- !query output
 0.0
 
@@ -2524,15 +2524,15 @@ struct<stddev_pop(CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT stddev_samp('1') FROM t
 -- !query schema
-struct<stddev_samp(CAST(1 AS DOUBLE)):double>
+struct<stddev_samp(1):double>
 -- !query output
-NaN
+NULL
 
 
 -- !query
 SELECT - '1' FROM t
 -- !query schema
-struct<(- CAST(1 AS DOUBLE)):double>
+struct<(- 1):double>
 -- !query output
 -1.0
 
@@ -2540,7 +2540,7 @@ struct<(- CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT + '1' FROM t
 -- !query schema
-struct<(+ CAST(1 AS DOUBLE)):double>
+struct<(+ 1):double>
 -- !query output
 1.0
 
@@ -2548,7 +2548,7 @@ struct<(+ CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT var_pop('1') FROM t
 -- !query schema
-struct<var_pop(CAST(1 AS DOUBLE)):double>
+struct<var_pop(1):double>
 -- !query output
 0.0
 
@@ -2556,22 +2556,22 @@ struct<var_pop(CAST(1 AS DOUBLE)):double>
 -- !query
 SELECT var_samp('1') FROM t
 -- !query schema
-struct<var_samp(CAST(1 AS DOUBLE)):double>
+struct<var_samp(1):double>
 -- !query output
-NaN
+NULL
 
 
 -- !query
 SELECT skewness('1') FROM t
 -- !query schema
-struct<skewness(CAST(1 AS DOUBLE)):double>
+struct<skewness(1):double>
 -- !query output
-NaN
+NULL
 
 
 -- !query
 SELECT kurtosis('1') FROM t
 -- !query schema
-struct<kurtosis(CAST(1 AS DOUBLE)):double>
+struct<kurtosis(1):double>
 -- !query output
-NaN
+NULL
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/stringCastAndExpressions.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/stringCastAndExpressions.sql.out
index 8353c7e73d0bb..14e941c074041 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/stringCastAndExpressions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/stringCastAndExpressions.sql.out
@@ -104,7 +104,7 @@ select cast(a as array<string>) from t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 't.`a`' due to data type mismatch: cannot cast string to array<string>; line 1 pos 7
+cannot resolve 't.a' due to data type mismatch: cannot cast string to array<string>; line 1 pos 7
 
 
 -- !query
@@ -113,7 +113,7 @@ select cast(a as struct<s:string>) from t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 't.`a`' due to data type mismatch: cannot cast string to struct<s:string>; line 1 pos 7
+cannot resolve 't.a' due to data type mismatch: cannot cast string to struct<s:string>; line 1 pos 7
 
 
 -- !query
@@ -122,7 +122,7 @@ select cast(a as map<string, string>) from t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 't.`a`' due to data type mismatch: cannot cast string to map<string,string>; line 1 pos 7
+cannot resolve 't.a' due to data type mismatch: cannot cast string to map<string,string>; line 1 pos 7
 
 
 -- !query
@@ -136,9 +136,10 @@ NULL
 -- !query
 select to_timestamp('2018-01-01', a) from t
 -- !query schema
-struct<to_timestamp(2018-01-01, a):timestamp>
+struct<>
 -- !query output
-NULL
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'aa' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
 
 
 -- !query
@@ -152,9 +153,10 @@ NULL
 -- !query
 select to_unix_timestamp('2018-01-01', a) from t
 -- !query schema
-struct<to_unix_timestamp(2018-01-01, a):bigint>
+struct<>
 -- !query output
-NULL
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'aa' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
 
 
 -- !query
@@ -168,15 +170,16 @@ NULL
 -- !query
 select unix_timestamp('2018-01-01', a) from t
 -- !query schema
-struct<unix_timestamp(2018-01-01, a):bigint>
+struct<>
 -- !query output
-NULL
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'aa' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
 
 
 -- !query
 select from_unixtime(a) from t
 -- !query schema
-struct<from_unixtime(CAST(a AS BIGINT), yyyy-MM-dd HH:mm:ss):string>
+struct<from_unixtime(a, yyyy-MM-dd HH:mm:ss):string>
 -- !query output
 NULL
 
@@ -184,7 +187,7 @@ NULL
 -- !query
 select from_unixtime('2018-01-01', a) from t
 -- !query schema
-struct<from_unixtime(CAST(2018-01-01 AS BIGINT), a):string>
+struct<from_unixtime(2018-01-01, a):string>
 -- !query output
 NULL
 
@@ -192,7 +195,7 @@ NULL
 -- !query
 select next_day(a, 'MO') from t
 -- !query schema
-struct<next_day(CAST(a AS DATE), MO):date>
+struct<next_day(a, MO):date>
 -- !query output
 NULL
 
@@ -200,7 +203,7 @@ NULL
 -- !query
 select next_day('2018-01-01', a) from t
 -- !query schema
-struct<next_day(CAST(2018-01-01 AS DATE), a):date>
+struct<next_day(2018-01-01, a):date>
 -- !query output
 NULL
 
@@ -208,7 +211,7 @@ NULL
 -- !query
 select trunc(a, 'MM') from t
 -- !query schema
-struct<trunc(CAST(a AS DATE), MM):date>
+struct<trunc(a, MM):date>
 -- !query output
 NULL
 
@@ -216,7 +219,7 @@ NULL
 -- !query
 select trunc('2018-01-01', a) from t
 -- !query schema
-struct<trunc(CAST(2018-01-01 AS DATE), a):date>
+struct<trunc(2018-01-01, a):date>
 -- !query output
 NULL
 
@@ -232,7 +235,7 @@ NULL
 -- !query
 select sha2(a, a) from t
 -- !query schema
-struct<sha2(CAST(a AS BINARY), CAST(a AS INT)):string>
+struct<sha2(a, a):string>
 -- !query output
 NULL
 
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/widenSetOperationTypes.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/widenSetOperationTypes.sql.out
index 89b1cdb3e353d..a527b20dc04ff 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/widenSetOperationTypes.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/widenSetOperationTypes.sql.out
@@ -88,7 +88,7 @@ SELECT cast(1 as tinyint) FROM t UNION SELECT cast('2' as binary) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. binary <> tinyint at the first column of the second table;
+Union can only be performed on tables with the compatible column types. binary <> tinyint at the first column of the second table
 
 
 -- !query
@@ -97,7 +97,7 @@ SELECT cast(1 as tinyint) FROM t UNION SELECT cast(2 as boolean) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. boolean <> tinyint at the first column of the second table;
+Union can only be performed on tables with the compatible column types. boolean <> tinyint at the first column of the second table
 
 
 -- !query
@@ -106,7 +106,7 @@ SELECT cast(1 as tinyint) FROM t UNION SELECT cast('2017-12-11 09:30:00.0' as ti
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. timestamp <> tinyint at the first column of the second table;
+Union can only be performed on tables with the compatible column types. timestamp <> tinyint at the first column of the second table
 
 
 -- !query
@@ -115,7 +115,7 @@ SELECT cast(1 as tinyint) FROM t UNION SELECT cast('2017-12-11 09:30:00' as date
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. date <> tinyint at the first column of the second table;
+Union can only be performed on tables with the compatible column types. date <> tinyint at the first column of the second table
 
 
 -- !query
@@ -196,7 +196,7 @@ SELECT cast(1 as smallint) FROM t UNION SELECT cast('2' as binary) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. binary <> smallint at the first column of the second table;
+Union can only be performed on tables with the compatible column types. binary <> smallint at the first column of the second table
 
 
 -- !query
@@ -205,7 +205,7 @@ SELECT cast(1 as smallint) FROM t UNION SELECT cast(2 as boolean) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. boolean <> smallint at the first column of the second table;
+Union can only be performed on tables with the compatible column types. boolean <> smallint at the first column of the second table
 
 
 -- !query
@@ -214,7 +214,7 @@ SELECT cast(1 as smallint) FROM t UNION SELECT cast('2017-12-11 09:30:00.0' as t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. timestamp <> smallint at the first column of the second table;
+Union can only be performed on tables with the compatible column types. timestamp <> smallint at the first column of the second table
 
 
 -- !query
@@ -223,7 +223,7 @@ SELECT cast(1 as smallint) FROM t UNION SELECT cast('2017-12-11 09:30:00' as dat
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. date <> smallint at the first column of the second table;
+Union can only be performed on tables with the compatible column types. date <> smallint at the first column of the second table
 
 
 -- !query
@@ -304,7 +304,7 @@ SELECT cast(1 as int) FROM t UNION SELECT cast('2' as binary) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. binary <> int at the first column of the second table;
+Union can only be performed on tables with the compatible column types. binary <> int at the first column of the second table
 
 
 -- !query
@@ -313,7 +313,7 @@ SELECT cast(1 as int) FROM t UNION SELECT cast(2 as boolean) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. boolean <> int at the first column of the second table;
+Union can only be performed on tables with the compatible column types. boolean <> int at the first column of the second table
 
 
 -- !query
@@ -322,7 +322,7 @@ SELECT cast(1 as int) FROM t UNION SELECT cast('2017-12-11 09:30:00.0' as timest
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. timestamp <> int at the first column of the second table;
+Union can only be performed on tables with the compatible column types. timestamp <> int at the first column of the second table
 
 
 -- !query
@@ -331,7 +331,7 @@ SELECT cast(1 as int) FROM t UNION SELECT cast('2017-12-11 09:30:00' as date) FR
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. date <> int at the first column of the second table;
+Union can only be performed on tables with the compatible column types. date <> int at the first column of the second table
 
 
 -- !query
@@ -412,7 +412,7 @@ SELECT cast(1 as bigint) FROM t UNION SELECT cast('2' as binary) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. binary <> bigint at the first column of the second table;
+Union can only be performed on tables with the compatible column types. binary <> bigint at the first column of the second table
 
 
 -- !query
@@ -421,7 +421,7 @@ SELECT cast(1 as bigint) FROM t UNION SELECT cast(2 as boolean) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. boolean <> bigint at the first column of the second table;
+Union can only be performed on tables with the compatible column types. boolean <> bigint at the first column of the second table
 
 
 -- !query
@@ -430,7 +430,7 @@ SELECT cast(1 as bigint) FROM t UNION SELECT cast('2017-12-11 09:30:00.0' as tim
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. timestamp <> bigint at the first column of the second table;
+Union can only be performed on tables with the compatible column types. timestamp <> bigint at the first column of the second table
 
 
 -- !query
@@ -439,7 +439,7 @@ SELECT cast(1 as bigint) FROM t UNION SELECT cast('2017-12-11 09:30:00' as date)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. date <> bigint at the first column of the second table;
+Union can only be performed on tables with the compatible column types. date <> bigint at the first column of the second table
 
 
 -- !query
@@ -520,7 +520,7 @@ SELECT cast(1 as float) FROM t UNION SELECT cast('2' as binary) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. binary <> float at the first column of the second table;
+Union can only be performed on tables with the compatible column types. binary <> float at the first column of the second table
 
 
 -- !query
@@ -529,7 +529,7 @@ SELECT cast(1 as float) FROM t UNION SELECT cast(2 as boolean) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. boolean <> float at the first column of the second table;
+Union can only be performed on tables with the compatible column types. boolean <> float at the first column of the second table
 
 
 -- !query
@@ -538,7 +538,7 @@ SELECT cast(1 as float) FROM t UNION SELECT cast('2017-12-11 09:30:00.0' as time
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. timestamp <> float at the first column of the second table;
+Union can only be performed on tables with the compatible column types. timestamp <> float at the first column of the second table
 
 
 -- !query
@@ -547,7 +547,7 @@ SELECT cast(1 as float) FROM t UNION SELECT cast('2017-12-11 09:30:00' as date)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. date <> float at the first column of the second table;
+Union can only be performed on tables with the compatible column types. date <> float at the first column of the second table
 
 
 -- !query
@@ -628,7 +628,7 @@ SELECT cast(1 as double) FROM t UNION SELECT cast('2' as binary) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. binary <> double at the first column of the second table;
+Union can only be performed on tables with the compatible column types. binary <> double at the first column of the second table
 
 
 -- !query
@@ -637,7 +637,7 @@ SELECT cast(1 as double) FROM t UNION SELECT cast(2 as boolean) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. boolean <> double at the first column of the second table;
+Union can only be performed on tables with the compatible column types. boolean <> double at the first column of the second table
 
 
 -- !query
@@ -646,7 +646,7 @@ SELECT cast(1 as double) FROM t UNION SELECT cast('2017-12-11 09:30:00.0' as tim
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. timestamp <> double at the first column of the second table;
+Union can only be performed on tables with the compatible column types. timestamp <> double at the first column of the second table
 
 
 -- !query
@@ -655,7 +655,7 @@ SELECT cast(1 as double) FROM t UNION SELECT cast('2017-12-11 09:30:00' as date)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. date <> double at the first column of the second table;
+Union can only be performed on tables with the compatible column types. date <> double at the first column of the second table
 
 
 -- !query
@@ -736,7 +736,7 @@ SELECT cast(1 as decimal(10, 0)) FROM t UNION SELECT cast('2' as binary) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. binary <> decimal(10,0) at the first column of the second table;
+Union can only be performed on tables with the compatible column types. binary <> decimal(10,0) at the first column of the second table
 
 
 -- !query
@@ -745,7 +745,7 @@ SELECT cast(1 as decimal(10, 0)) FROM t UNION SELECT cast(2 as boolean) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. boolean <> decimal(10,0) at the first column of the second table;
+Union can only be performed on tables with the compatible column types. boolean <> decimal(10,0) at the first column of the second table
 
 
 -- !query
@@ -754,7 +754,7 @@ SELECT cast(1 as decimal(10, 0)) FROM t UNION SELECT cast('2017-12-11 09:30:00.0
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. timestamp <> decimal(10,0) at the first column of the second table;
+Union can only be performed on tables with the compatible column types. timestamp <> decimal(10,0) at the first column of the second table
 
 
 -- !query
@@ -763,7 +763,7 @@ SELECT cast(1 as decimal(10, 0)) FROM t UNION SELECT cast('2017-12-11 09:30:00'
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. date <> decimal(10,0) at the first column of the second table;
+Union can only be performed on tables with the compatible column types. date <> decimal(10,0) at the first column of the second table
 
 
 -- !query
@@ -844,7 +844,7 @@ SELECT cast(1 as string) FROM t UNION SELECT cast('2' as binary) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. binary <> string at the first column of the second table;
+Union can only be performed on tables with the compatible column types. binary <> string at the first column of the second table
 
 
 -- !query
@@ -853,7 +853,7 @@ SELECT cast(1 as string) FROM t UNION SELECT cast(2 as boolean) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. boolean <> string at the first column of the second table;
+Union can only be performed on tables with the compatible column types. boolean <> string at the first column of the second table
 
 
 -- !query
@@ -880,7 +880,7 @@ SELECT cast('1' as binary) FROM t UNION SELECT cast(2 as tinyint) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. tinyint <> binary at the first column of the second table;
+Union can only be performed on tables with the compatible column types. tinyint <> binary at the first column of the second table
 
 
 -- !query
@@ -889,7 +889,7 @@ SELECT cast('1' as binary) FROM t UNION SELECT cast(2 as smallint) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. smallint <> binary at the first column of the second table;
+Union can only be performed on tables with the compatible column types. smallint <> binary at the first column of the second table
 
 
 -- !query
@@ -898,7 +898,7 @@ SELECT cast('1' as binary) FROM t UNION SELECT cast(2 as int) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. int <> binary at the first column of the second table;
+Union can only be performed on tables with the compatible column types. int <> binary at the first column of the second table
 
 
 -- !query
@@ -907,7 +907,7 @@ SELECT cast('1' as binary) FROM t UNION SELECT cast(2 as bigint) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. bigint <> binary at the first column of the second table;
+Union can only be performed on tables with the compatible column types. bigint <> binary at the first column of the second table
 
 
 -- !query
@@ -916,7 +916,7 @@ SELECT cast('1' as binary) FROM t UNION SELECT cast(2 as float) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. float <> binary at the first column of the second table;
+Union can only be performed on tables with the compatible column types. float <> binary at the first column of the second table
 
 
 -- !query
@@ -925,7 +925,7 @@ SELECT cast('1' as binary) FROM t UNION SELECT cast(2 as double) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. double <> binary at the first column of the second table;
+Union can only be performed on tables with the compatible column types. double <> binary at the first column of the second table
 
 
 -- !query
@@ -934,7 +934,7 @@ SELECT cast('1' as binary) FROM t UNION SELECT cast(2 as decimal(10, 0)) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. decimal(10,0) <> binary at the first column of the second table;
+Union can only be performed on tables with the compatible column types. decimal(10,0) <> binary at the first column of the second table
 
 
 -- !query
@@ -943,7 +943,7 @@ SELECT cast('1' as binary) FROM t UNION SELECT cast(2 as string) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. string <> binary at the first column of the second table;
+Union can only be performed on tables with the compatible column types. string <> binary at the first column of the second table
 
 
 -- !query
@@ -961,7 +961,7 @@ SELECT cast('1' as binary) FROM t UNION SELECT cast(2 as boolean) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. boolean <> binary at the first column of the second table;
+Union can only be performed on tables with the compatible column types. boolean <> binary at the first column of the second table
 
 
 -- !query
@@ -970,7 +970,7 @@ SELECT cast('1' as binary) FROM t UNION SELECT cast('2017-12-11 09:30:00.0' as t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. timestamp <> binary at the first column of the second table;
+Union can only be performed on tables with the compatible column types. timestamp <> binary at the first column of the second table
 
 
 -- !query
@@ -979,7 +979,7 @@ SELECT cast('1' as binary) FROM t UNION SELECT cast('2017-12-11 09:30:00' as dat
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. date <> binary at the first column of the second table;
+Union can only be performed on tables with the compatible column types. date <> binary at the first column of the second table
 
 
 -- !query
@@ -988,7 +988,7 @@ SELECT cast(1 as boolean) FROM t UNION SELECT cast(2 as tinyint) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. tinyint <> boolean at the first column of the second table;
+Union can only be performed on tables with the compatible column types. tinyint <> boolean at the first column of the second table
 
 
 -- !query
@@ -997,7 +997,7 @@ SELECT cast(1 as boolean) FROM t UNION SELECT cast(2 as smallint) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. smallint <> boolean at the first column of the second table;
+Union can only be performed on tables with the compatible column types. smallint <> boolean at the first column of the second table
 
 
 -- !query
@@ -1006,7 +1006,7 @@ SELECT cast(1 as boolean) FROM t UNION SELECT cast(2 as int) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. int <> boolean at the first column of the second table;
+Union can only be performed on tables with the compatible column types. int <> boolean at the first column of the second table
 
 
 -- !query
@@ -1015,7 +1015,7 @@ SELECT cast(1 as boolean) FROM t UNION SELECT cast(2 as bigint) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. bigint <> boolean at the first column of the second table;
+Union can only be performed on tables with the compatible column types. bigint <> boolean at the first column of the second table
 
 
 -- !query
@@ -1024,7 +1024,7 @@ SELECT cast(1 as boolean) FROM t UNION SELECT cast(2 as float) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. float <> boolean at the first column of the second table;
+Union can only be performed on tables with the compatible column types. float <> boolean at the first column of the second table
 
 
 -- !query
@@ -1033,7 +1033,7 @@ SELECT cast(1 as boolean) FROM t UNION SELECT cast(2 as double) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. double <> boolean at the first column of the second table;
+Union can only be performed on tables with the compatible column types. double <> boolean at the first column of the second table
 
 
 -- !query
@@ -1042,7 +1042,7 @@ SELECT cast(1 as boolean) FROM t UNION SELECT cast(2 as decimal(10, 0)) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. decimal(10,0) <> boolean at the first column of the second table;
+Union can only be performed on tables with the compatible column types. decimal(10,0) <> boolean at the first column of the second table
 
 
 -- !query
@@ -1051,7 +1051,7 @@ SELECT cast(1 as boolean) FROM t UNION SELECT cast(2 as string) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. string <> boolean at the first column of the second table;
+Union can only be performed on tables with the compatible column types. string <> boolean at the first column of the second table
 
 
 -- !query
@@ -1060,7 +1060,7 @@ SELECT cast(1 as boolean) FROM t UNION SELECT cast('2' as binary) FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. binary <> boolean at the first column of the second table;
+Union can only be performed on tables with the compatible column types. binary <> boolean at the first column of the second table
 
 
 -- !query
@@ -1077,7 +1077,7 @@ SELECT cast(1 as boolean) FROM t UNION SELECT cast('2017-12-11 09:30:00.0' as ti
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. timestamp <> boolean at the first column of the second table;
+Union can only be performed on tables with the compatible column types. timestamp <> boolean at the first column of the second table
 
 
 -- !query
@@ -1086,7 +1086,7 @@ SELECT cast(1 as boolean) FROM t UNION SELECT cast('2017-12-11 09:30:00' as date
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. date <> boolean at the first column of the second table;
+Union can only be performed on tables with the compatible column types. date <> boolean at the first column of the second table
 
 
 -- !query
@@ -1095,7 +1095,7 @@ SELECT cast('2017-12-12 09:30:00.0' as timestamp) FROM t UNION SELECT cast(2 as
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. tinyint <> timestamp at the first column of the second table;
+Union can only be performed on tables with the compatible column types. tinyint <> timestamp at the first column of the second table
 
 
 -- !query
@@ -1104,7 +1104,7 @@ SELECT cast('2017-12-12 09:30:00.0' as timestamp) FROM t UNION SELECT cast(2 as
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. smallint <> timestamp at the first column of the second table;
+Union can only be performed on tables with the compatible column types. smallint <> timestamp at the first column of the second table
 
 
 -- !query
@@ -1113,7 +1113,7 @@ SELECT cast('2017-12-12 09:30:00.0' as timestamp) FROM t UNION SELECT cast(2 as
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. int <> timestamp at the first column of the second table;
+Union can only be performed on tables with the compatible column types. int <> timestamp at the first column of the second table
 
 
 -- !query
@@ -1122,7 +1122,7 @@ SELECT cast('2017-12-12 09:30:00.0' as timestamp) FROM t UNION SELECT cast(2 as
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. bigint <> timestamp at the first column of the second table;
+Union can only be performed on tables with the compatible column types. bigint <> timestamp at the first column of the second table
 
 
 -- !query
@@ -1131,7 +1131,7 @@ SELECT cast('2017-12-12 09:30:00.0' as timestamp) FROM t UNION SELECT cast(2 as
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. float <> timestamp at the first column of the second table;
+Union can only be performed on tables with the compatible column types. float <> timestamp at the first column of the second table
 
 
 -- !query
@@ -1140,7 +1140,7 @@ SELECT cast('2017-12-12 09:30:00.0' as timestamp) FROM t UNION SELECT cast(2 as
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. double <> timestamp at the first column of the second table;
+Union can only be performed on tables with the compatible column types. double <> timestamp at the first column of the second table
 
 
 -- !query
@@ -1149,7 +1149,7 @@ SELECT cast('2017-12-12 09:30:00.0' as timestamp) FROM t UNION SELECT cast(2 as
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. decimal(10,0) <> timestamp at the first column of the second table;
+Union can only be performed on tables with the compatible column types. decimal(10,0) <> timestamp at the first column of the second table
 
 
 -- !query
@@ -1167,7 +1167,7 @@ SELECT cast('2017-12-12 09:30:00.0' as timestamp) FROM t UNION SELECT cast('2' a
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. binary <> timestamp at the first column of the second table;
+Union can only be performed on tables with the compatible column types. binary <> timestamp at the first column of the second table
 
 
 -- !query
@@ -1176,7 +1176,7 @@ SELECT cast('2017-12-12 09:30:00.0' as timestamp) FROM t UNION SELECT cast(2 as
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. boolean <> timestamp at the first column of the second table;
+Union can only be performed on tables with the compatible column types. boolean <> timestamp at the first column of the second table
 
 
 -- !query
@@ -1203,7 +1203,7 @@ SELECT cast('2017-12-12 09:30:00' as date) FROM t UNION SELECT cast(2 as tinyint
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. tinyint <> date at the first column of the second table;
+Union can only be performed on tables with the compatible column types. tinyint <> date at the first column of the second table
 
 
 -- !query
@@ -1212,7 +1212,7 @@ SELECT cast('2017-12-12 09:30:00' as date) FROM t UNION SELECT cast(2 as smallin
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. smallint <> date at the first column of the second table;
+Union can only be performed on tables with the compatible column types. smallint <> date at the first column of the second table
 
 
 -- !query
@@ -1221,7 +1221,7 @@ SELECT cast('2017-12-12 09:30:00' as date) FROM t UNION SELECT cast(2 as int) FR
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. int <> date at the first column of the second table;
+Union can only be performed on tables with the compatible column types. int <> date at the first column of the second table
 
 
 -- !query
@@ -1230,7 +1230,7 @@ SELECT cast('2017-12-12 09:30:00' as date) FROM t UNION SELECT cast(2 as bigint)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. bigint <> date at the first column of the second table;
+Union can only be performed on tables with the compatible column types. bigint <> date at the first column of the second table
 
 
 -- !query
@@ -1239,7 +1239,7 @@ SELECT cast('2017-12-12 09:30:00' as date) FROM t UNION SELECT cast(2 as float)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. float <> date at the first column of the second table;
+Union can only be performed on tables with the compatible column types. float <> date at the first column of the second table
 
 
 -- !query
@@ -1248,7 +1248,7 @@ SELECT cast('2017-12-12 09:30:00' as date) FROM t UNION SELECT cast(2 as double)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. double <> date at the first column of the second table;
+Union can only be performed on tables with the compatible column types. double <> date at the first column of the second table
 
 
 -- !query
@@ -1257,7 +1257,7 @@ SELECT cast('2017-12-12 09:30:00' as date) FROM t UNION SELECT cast(2 as decimal
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. decimal(10,0) <> date at the first column of the second table;
+Union can only be performed on tables with the compatible column types. decimal(10,0) <> date at the first column of the second table
 
 
 -- !query
@@ -1275,7 +1275,7 @@ SELECT cast('2017-12-12 09:30:00' as date) FROM t UNION SELECT cast('2' as binar
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. binary <> date at the first column of the second table;
+Union can only be performed on tables with the compatible column types. binary <> date at the first column of the second table
 
 
 -- !query
@@ -1284,7 +1284,7 @@ SELECT cast('2017-12-12 09:30:00' as date) FROM t UNION SELECT cast(2 as boolean
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Union can only be performed on tables with the compatible column types. boolean <> date at the first column of the second table;
+Union can only be performed on tables with the compatible column types. boolean <> date at the first column of the second table
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/windowFrameCoercion.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/windowFrameCoercion.sql.out
index 12af1b7d034da..fa5ad19c31b11 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/windowFrameCoercion.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/windowFrameCoercion.sql.out
@@ -109,7 +109,7 @@ struct<count(1) OVER (PARTITION BY 1 ORDER BY CAST(2017-12-11 09:30:00 AS DATE)
 -- !query
 SELECT COUNT(*) OVER (PARTITION BY 1 ORDER BY cast(1 as tinyint) DESC RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING) FROM t
 -- !query schema
-struct<count(1) OVER (PARTITION BY 1 ORDER BY CAST(1 AS TINYINT) DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND CAST(1 AS TINYINT) FOLLOWING):bigint>
+struct<count(1) OVER (PARTITION BY 1 ORDER BY CAST(1 AS TINYINT) DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING):bigint>
 -- !query output
 1
 
@@ -117,7 +117,7 @@ struct<count(1) OVER (PARTITION BY 1 ORDER BY CAST(1 AS TINYINT) DESC NULLS LAST
 -- !query
 SELECT COUNT(*) OVER (PARTITION BY 1 ORDER BY cast(1 as smallint) DESC RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING) FROM t
 -- !query schema
-struct<count(1) OVER (PARTITION BY 1 ORDER BY CAST(1 AS SMALLINT) DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND CAST(1 AS SMALLINT) FOLLOWING):bigint>
+struct<count(1) OVER (PARTITION BY 1 ORDER BY CAST(1 AS SMALLINT) DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING):bigint>
 -- !query output
 1
 
@@ -133,7 +133,7 @@ struct<count(1) OVER (PARTITION BY 1 ORDER BY CAST(1 AS INT) DESC NULLS LAST RAN
 -- !query
 SELECT COUNT(*) OVER (PARTITION BY 1 ORDER BY cast(1 as bigint) DESC RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING) FROM t
 -- !query schema
-struct<count(1) OVER (PARTITION BY 1 ORDER BY CAST(1 AS BIGINT) DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND CAST(1 AS BIGINT) FOLLOWING):bigint>
+struct<count(1) OVER (PARTITION BY 1 ORDER BY CAST(1 AS BIGINT) DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING):bigint>
 -- !query output
 1
 
@@ -141,7 +141,7 @@ struct<count(1) OVER (PARTITION BY 1 ORDER BY CAST(1 AS BIGINT) DESC NULLS LAST
 -- !query
 SELECT COUNT(*) OVER (PARTITION BY 1 ORDER BY cast(1 as float) DESC RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING) FROM t
 -- !query schema
-struct<count(1) OVER (PARTITION BY 1 ORDER BY CAST(1 AS FLOAT) DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND CAST(1 AS FLOAT) FOLLOWING):bigint>
+struct<count(1) OVER (PARTITION BY 1 ORDER BY CAST(1 AS FLOAT) DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING):bigint>
 -- !query output
 1
 
@@ -149,7 +149,7 @@ struct<count(1) OVER (PARTITION BY 1 ORDER BY CAST(1 AS FLOAT) DESC NULLS LAST R
 -- !query
 SELECT COUNT(*) OVER (PARTITION BY 1 ORDER BY cast(1 as double) DESC RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING) FROM t
 -- !query schema
-struct<count(1) OVER (PARTITION BY 1 ORDER BY CAST(1 AS DOUBLE) DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND CAST(1 AS DOUBLE) FOLLOWING):bigint>
+struct<count(1) OVER (PARTITION BY 1 ORDER BY CAST(1 AS DOUBLE) DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING):bigint>
 -- !query output
 1
 
@@ -157,7 +157,7 @@ struct<count(1) OVER (PARTITION BY 1 ORDER BY CAST(1 AS DOUBLE) DESC NULLS LAST
 -- !query
 SELECT COUNT(*) OVER (PARTITION BY 1 ORDER BY cast(1 as decimal(10, 0)) DESC RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING) FROM t
 -- !query schema
-struct<count(1) OVER (PARTITION BY 1 ORDER BY CAST(1 AS DECIMAL(10,0)) DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND CAST(1 AS DECIMAL(10,0)) FOLLOWING):bigint>
+struct<count(1) OVER (PARTITION BY 1 ORDER BY CAST(1 AS DECIMAL(10,0)) DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING):bigint>
 -- !query output
 1
 
@@ -168,7 +168,7 @@ SELECT COUNT(*) OVER (PARTITION BY 1 ORDER BY cast(1 as string) DESC RANGE BETWE
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'RANGE BETWEEN CURRENT ROW AND CAST(1 AS STRING) FOLLOWING' due to data type mismatch: The data type of the upper bound 'string' does not match the expected data type '(numeric or interval)'.; line 1 pos 21
+cannot resolve 'RANGE BETWEEN CURRENT ROW AND CAST(1 AS STRING) FOLLOWING' due to data type mismatch: The data type of the upper bound 'string' does not match the expected data type '(numeric or interval or interval day to second or interval year to month)'.; line 1 pos 21
 
 
 -- !query
@@ -177,7 +177,7 @@ SELECT COUNT(*) OVER (PARTITION BY 1 ORDER BY cast('1' as binary) DESC RANGE BET
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'RANGE BETWEEN CURRENT ROW AND CAST(1 AS BINARY) FOLLOWING' due to data type mismatch: The data type of the upper bound 'binary' does not match the expected data type '(numeric or interval)'.; line 1 pos 21
+cannot resolve 'RANGE BETWEEN CURRENT ROW AND CAST(1 AS BINARY) FOLLOWING' due to data type mismatch: The data type of the upper bound 'binary' does not match the expected data type '(numeric or interval or interval day to second or interval year to month)'.; line 1 pos 21
 
 
 -- !query
@@ -186,7 +186,7 @@ SELECT COUNT(*) OVER (PARTITION BY 1 ORDER BY cast(1 as boolean) DESC RANGE BETW
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'RANGE BETWEEN CURRENT ROW AND CAST(1 AS BOOLEAN) FOLLOWING' due to data type mismatch: The data type of the upper bound 'boolean' does not match the expected data type '(numeric or interval)'.; line 1 pos 21
+cannot resolve 'RANGE BETWEEN CURRENT ROW AND CAST(1 AS BOOLEAN) FOLLOWING' due to data type mismatch: The data type of the upper bound 'boolean' does not match the expected data type '(numeric or interval or interval day to second or interval year to month)'.; line 1 pos 21
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part1.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part1.sql.out
index 76637bf578e6f..4fffd450e35c2 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part1.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part1.sql.out
@@ -77,7 +77,7 @@ struct<max_324_78:float>
 -- !query
 SELECT stddev_pop(udf(b)) FROM aggtest
 -- !query schema
-struct<stddev_pop(CAST(CAST(udf(ansi_cast(b as string)) AS FLOAT) AS DOUBLE)):double>
+struct<stddev_pop(udf(b)):double>
 -- !query output
 131.10703231895047
 
@@ -85,7 +85,7 @@ struct<stddev_pop(CAST(CAST(udf(ansi_cast(b as string)) AS FLOAT) AS DOUBLE)):do
 -- !query
 SELECT udf(stddev_samp(b)) FROM aggtest
 -- !query schema
-struct<CAST(udf(ansi_cast(stddev_samp(ansi_cast(b as double)) as string)) AS DOUBLE):double>
+struct<udf(stddev_samp(b)):double>
 -- !query output
 151.38936080399804
 
@@ -93,7 +93,7 @@ struct<CAST(udf(ansi_cast(stddev_samp(ansi_cast(b as double)) as string)) AS DOU
 -- !query
 SELECT var_pop(udf(b)) FROM aggtest
 -- !query schema
-struct<var_pop(CAST(CAST(udf(ansi_cast(b as string)) AS FLOAT) AS DOUBLE)):double>
+struct<var_pop(udf(b)):double>
 -- !query output
 17189.053923482323
 
@@ -101,7 +101,7 @@ struct<var_pop(CAST(CAST(udf(ansi_cast(b as string)) AS FLOAT) AS DOUBLE)):doubl
 -- !query
 SELECT udf(var_samp(b)) FROM aggtest
 -- !query schema
-struct<CAST(udf(ansi_cast(var_samp(ansi_cast(b as double)) as string)) AS DOUBLE):double>
+struct<udf(var_samp(b)):double>
 -- !query output
 22918.738564643096
 
@@ -109,7 +109,7 @@ struct<CAST(udf(ansi_cast(var_samp(ansi_cast(b as double)) as string)) AS DOUBLE
 -- !query
 SELECT udf(stddev_pop(CAST(b AS Decimal(38,0)))) FROM aggtest
 -- !query schema
-struct<CAST(udf(ansi_cast(stddev_pop(ansi_cast(ansi_cast(b as decimal(38,0)) as double)) as string)) AS DOUBLE):double>
+struct<udf(stddev_pop(CAST(b AS DECIMAL(38,0)))):double>
 -- !query output
 131.18117242958306
 
@@ -117,7 +117,7 @@ struct<CAST(udf(ansi_cast(stddev_pop(ansi_cast(ansi_cast(b as decimal(38,0)) as
 -- !query
 SELECT stddev_samp(CAST(udf(b) AS Decimal(38,0))) FROM aggtest
 -- !query schema
-struct<stddev_samp(CAST(CAST(CAST(udf(ansi_cast(b as string)) AS FLOAT) AS DECIMAL(38,0)) AS DOUBLE)):double>
+struct<stddev_samp(CAST(udf(b) AS DECIMAL(38,0))):double>
 -- !query output
 151.47497042966097
 
@@ -125,7 +125,7 @@ struct<stddev_samp(CAST(CAST(CAST(udf(ansi_cast(b as string)) AS FLOAT) AS DECIM
 -- !query
 SELECT udf(var_pop(CAST(b AS Decimal(38,0)))) FROM aggtest
 -- !query schema
-struct<CAST(udf(ansi_cast(var_pop(ansi_cast(ansi_cast(b as decimal(38,0)) as double)) as string)) AS DOUBLE):double>
+struct<udf(var_pop(CAST(b AS DECIMAL(38,0)))):double>
 -- !query output
 17208.5
 
@@ -133,7 +133,7 @@ struct<CAST(udf(ansi_cast(var_pop(ansi_cast(ansi_cast(b as decimal(38,0)) as dou
 -- !query
 SELECT var_samp(udf(CAST(b AS Decimal(38,0)))) FROM aggtest
 -- !query schema
-struct<var_samp(CAST(CAST(udf(ansi_cast(ansi_cast(b as decimal(38,0)) as string)) AS DECIMAL(38,0)) AS DOUBLE)):double>
+struct<var_samp(udf(CAST(b AS DECIMAL(38,0)))):double>
 -- !query output
 22944.666666666668
 
@@ -141,23 +141,23 @@ struct<var_samp(CAST(CAST(udf(ansi_cast(ansi_cast(b as decimal(38,0)) as string)
 -- !query
 SELECT udf(var_pop(1.0)), var_samp(udf(2.0))
 -- !query schema
-struct<CAST(udf(ansi_cast(var_pop(ansi_cast(1.0 as double)) as string)) AS DOUBLE):double,var_samp(CAST(CAST(udf(ansi_cast(2.0 as string)) AS DECIMAL(2,1)) AS DOUBLE)):double>
+struct<udf(var_pop(1.0)):double,var_samp(udf(2.0)):double>
 -- !query output
-0.0	NaN
+0.0	NULL
 
 
 -- !query
 SELECT stddev_pop(udf(CAST(3.0 AS Decimal(38,0)))), stddev_samp(CAST(udf(4.0) AS Decimal(38,0)))
 -- !query schema
-struct<stddev_pop(CAST(CAST(udf(ansi_cast(ansi_cast(3.0 as decimal(38,0)) as string)) AS DECIMAL(38,0)) AS DOUBLE)):double,stddev_samp(CAST(CAST(CAST(udf(ansi_cast(4.0 as string)) AS DECIMAL(2,1)) AS DECIMAL(38,0)) AS DOUBLE)):double>
+struct<stddev_pop(udf(CAST(3.0 AS DECIMAL(38,0)))):double,stddev_samp(CAST(udf(4.0) AS DECIMAL(38,0))):double>
 -- !query output
-0.0	NaN
+0.0	NULL
 
 
 -- !query
 select sum(udf(CAST(null AS int))) from range(1,4)
 -- !query schema
-struct<sum(CAST(udf(ansi_cast(ansi_cast(null as int) as string)) AS INT)):bigint>
+struct<sum(udf(CAST(NULL AS INT))):bigint>
 -- !query output
 NULL
 
@@ -165,7 +165,7 @@ NULL
 -- !query
 select sum(udf(CAST(null AS long))) from range(1,4)
 -- !query schema
-struct<sum(CAST(udf(ansi_cast(ansi_cast(null as bigint) as string)) AS BIGINT)):bigint>
+struct<sum(udf(CAST(NULL AS BIGINT))):bigint>
 -- !query output
 NULL
 
@@ -173,7 +173,7 @@ NULL
 -- !query
 select sum(udf(CAST(null AS Decimal(38,0)))) from range(1,4)
 -- !query schema
-struct<sum(CAST(udf(ansi_cast(ansi_cast(null as decimal(38,0)) as string)) AS DECIMAL(38,0))):decimal(38,0)>
+struct<sum(udf(CAST(NULL AS DECIMAL(38,0)))):decimal(38,0)>
 -- !query output
 NULL
 
@@ -181,7 +181,7 @@ NULL
 -- !query
 select sum(udf(CAST(null AS DOUBLE))) from range(1,4)
 -- !query schema
-struct<sum(CAST(udf(ansi_cast(ansi_cast(null as double) as string)) AS DOUBLE)):double>
+struct<sum(udf(CAST(NULL AS DOUBLE))):double>
 -- !query output
 NULL
 
@@ -189,7 +189,7 @@ NULL
 -- !query
 select avg(udf(CAST(null AS int))) from range(1,4)
 -- !query schema
-struct<avg(CAST(udf(ansi_cast(ansi_cast(null as int) as string)) AS INT)):double>
+struct<avg(udf(CAST(NULL AS INT))):double>
 -- !query output
 NULL
 
@@ -197,7 +197,7 @@ NULL
 -- !query
 select avg(udf(CAST(null AS long))) from range(1,4)
 -- !query schema
-struct<avg(CAST(udf(ansi_cast(ansi_cast(null as bigint) as string)) AS BIGINT)):double>
+struct<avg(udf(CAST(NULL AS BIGINT))):double>
 -- !query output
 NULL
 
@@ -205,7 +205,7 @@ NULL
 -- !query
 select avg(udf(CAST(null AS Decimal(38,0)))) from range(1,4)
 -- !query schema
-struct<avg(CAST(udf(ansi_cast(ansi_cast(null as decimal(38,0)) as string)) AS DECIMAL(38,0))):decimal(38,4)>
+struct<avg(udf(CAST(NULL AS DECIMAL(38,0)))):decimal(38,4)>
 -- !query output
 NULL
 
@@ -213,7 +213,7 @@ NULL
 -- !query
 select avg(udf(CAST(null AS DOUBLE))) from range(1,4)
 -- !query schema
-struct<avg(CAST(udf(ansi_cast(ansi_cast(null as double) as string)) AS DOUBLE)):double>
+struct<avg(udf(CAST(NULL AS DOUBLE))):double>
 -- !query output
 NULL
 
@@ -221,7 +221,7 @@ NULL
 -- !query
 select sum(CAST(udf('NaN') AS DOUBLE)) from range(1,4)
 -- !query schema
-struct<sum(CAST(CAST(udf(ansi_cast(NaN as string)) AS STRING) AS DOUBLE)):double>
+struct<sum(CAST(udf(NaN) AS DOUBLE)):double>
 -- !query output
 NaN
 
@@ -229,7 +229,7 @@ NaN
 -- !query
 select avg(CAST(udf('NaN') AS DOUBLE)) from range(1,4)
 -- !query schema
-struct<avg(CAST(CAST(udf(ansi_cast(NaN as string)) AS STRING) AS DOUBLE)):double>
+struct<avg(CAST(udf(NaN) AS DOUBLE)):double>
 -- !query output
 NaN
 
@@ -238,7 +238,7 @@ NaN
 SELECT avg(CAST(udf(x) AS DOUBLE)), var_pop(CAST(udf(x) AS DOUBLE))
 FROM (VALUES ('Infinity'), ('1')) v(x)
 -- !query schema
-struct<avg(CAST(CAST(udf(ansi_cast(x as string)) AS STRING) AS DOUBLE)):double,var_pop(CAST(CAST(udf(ansi_cast(x as string)) AS STRING) AS DOUBLE)):double>
+struct<avg(CAST(udf(x) AS DOUBLE)):double,var_pop(CAST(udf(x) AS DOUBLE)):double>
 -- !query output
 Infinity	NaN
 
@@ -247,7 +247,7 @@ Infinity	NaN
 SELECT avg(CAST(udf(x) AS DOUBLE)), var_pop(CAST(udf(x) AS DOUBLE))
 FROM (VALUES ('Infinity'), ('Infinity')) v(x)
 -- !query schema
-struct<avg(CAST(CAST(udf(ansi_cast(x as string)) AS STRING) AS DOUBLE)):double,var_pop(CAST(CAST(udf(ansi_cast(x as string)) AS STRING) AS DOUBLE)):double>
+struct<avg(CAST(udf(x) AS DOUBLE)):double,var_pop(CAST(udf(x) AS DOUBLE)):double>
 -- !query output
 Infinity	NaN
 
@@ -256,7 +256,7 @@ Infinity	NaN
 SELECT avg(CAST(udf(x) AS DOUBLE)), var_pop(CAST(udf(x) AS DOUBLE))
 FROM (VALUES ('-Infinity'), ('Infinity')) v(x)
 -- !query schema
-struct<avg(CAST(CAST(udf(ansi_cast(x as string)) AS STRING) AS DOUBLE)):double,var_pop(CAST(CAST(udf(ansi_cast(x as string)) AS STRING) AS DOUBLE)):double>
+struct<avg(CAST(udf(x) AS DOUBLE)):double,var_pop(CAST(udf(x) AS DOUBLE)):double>
 -- !query output
 NaN	NaN
 
@@ -265,7 +265,7 @@ NaN	NaN
 SELECT avg(udf(CAST(x AS DOUBLE))), udf(var_pop(CAST(x AS DOUBLE)))
 FROM (VALUES (100000003), (100000004), (100000006), (100000007)) v(x)
 -- !query schema
-struct<avg(CAST(udf(ansi_cast(ansi_cast(x as double) as string)) AS DOUBLE)):double,CAST(udf(ansi_cast(var_pop(ansi_cast(x as double)) as string)) AS DOUBLE):double>
+struct<avg(udf(CAST(x AS DOUBLE))):double,udf(var_pop(CAST(x AS DOUBLE))):double>
 -- !query output
 1.00000005E8	2.5
 
@@ -274,7 +274,7 @@ struct<avg(CAST(udf(ansi_cast(ansi_cast(x as double) as string)) AS DOUBLE)):dou
 SELECT avg(udf(CAST(x AS DOUBLE))), udf(var_pop(CAST(x AS DOUBLE)))
 FROM (VALUES (7000000000005), (7000000000007)) v(x)
 -- !query schema
-struct<avg(CAST(udf(ansi_cast(ansi_cast(x as double) as string)) AS DOUBLE)):double,CAST(udf(ansi_cast(var_pop(ansi_cast(x as double)) as string)) AS DOUBLE):double>
+struct<avg(udf(CAST(x AS DOUBLE))):double,udf(var_pop(CAST(x AS DOUBLE))):double>
 -- !query output
 7.000000000006E12	1.0
 
@@ -282,7 +282,7 @@ struct<avg(CAST(udf(ansi_cast(ansi_cast(x as double) as string)) AS DOUBLE)):dou
 -- !query
 SELECT udf(covar_pop(b, udf(a))), covar_samp(udf(b), a) FROM aggtest
 -- !query schema
-struct<CAST(udf(ansi_cast(covar_pop(ansi_cast(b as double), ansi_cast(ansi_cast(udf(ansi_cast(a as string)) as int) as double)) as string)) AS DOUBLE):double,covar_samp(CAST(CAST(udf(ansi_cast(b as string)) AS FLOAT) AS DOUBLE), CAST(a AS DOUBLE)):double>
+struct<udf(covar_pop(b, udf(a))):double,covar_samp(udf(b), a):double>
 -- !query output
 653.6289553875104	871.5052738500139
 
@@ -290,7 +290,7 @@ struct<CAST(udf(ansi_cast(covar_pop(ansi_cast(b as double), ansi_cast(ansi_cast(
 -- !query
 SELECT corr(b, udf(a)) FROM aggtest
 -- !query schema
-struct<corr(CAST(b AS DOUBLE), CAST(CAST(udf(ansi_cast(a as string)) AS INT) AS DOUBLE)):double>
+struct<corr(b, udf(a)):double>
 -- !query output
 0.1396345165178734
 
@@ -315,7 +315,7 @@ struct<cnt_4:bigint>
 select ten, udf(count(*)), sum(udf(four)) from onek
 group by ten order by ten
 -- !query schema
-struct<ten:int,CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint,sum(CAST(udf(ansi_cast(four as string)) AS INT)):bigint>
+struct<ten:int,udf(count(1)):bigint,sum(udf(four)):bigint>
 -- !query output
 0	100	100
 1	100	200
@@ -333,7 +333,7 @@ struct<ten:int,CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint,sum(CAS
 select ten, count(udf(four)), udf(sum(DISTINCT four)) from onek
 group by ten order by ten
 -- !query schema
-struct<ten:int,count(CAST(udf(ansi_cast(four as string)) AS INT)):bigint,CAST(udf(ansi_cast(sum(distinct ansi_cast(four as bigint)) as string)) AS BIGINT):bigint>
+struct<ten:int,count(udf(four)):bigint,udf(sum(DISTINCT four)):bigint>
 -- !query output
 0	100	2
 1	100	4
@@ -352,7 +352,7 @@ select ten, udf(sum(distinct four)) from onek a
 group by ten
 having exists (select 1 from onek b where udf(sum(distinct a.four)) = b.four)
 -- !query schema
-struct<ten:int,CAST(udf(ansi_cast(sum(distinct ansi_cast(four as bigint)) as string)) AS BIGINT):bigint>
+struct<ten:int,udf(sum(DISTINCT four)):bigint>
 -- !query output
 0	2
 2	2
@@ -372,8 +372,8 @@ struct<>
 org.apache.spark.sql.AnalysisException
 
 Aggregate/Window/Generate expressions are not valid in where clause of the query.
-Expression in where clause: [(sum(DISTINCT CAST((outer(a.`four`) + b.`four`) AS BIGINT)) = CAST(CAST(udf(ansi_cast(four as string)) AS INT) AS BIGINT))]
-Invalid expressions: [sum(DISTINCT CAST((outer(a.`four`) + b.`four`) AS BIGINT))];
+Expression in where clause: [(sum(DISTINCT (outer(a.four) + b.four)) = CAST(CAST(udf(ansi_cast(four as string)) AS INT) AS BIGINT))]
+Invalid expressions: [sum(DISTINCT (outer(a.four) + b.four))]
 
 
 -- !query
@@ -384,4 +384,4 @@ from tenk1 o
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`o.unique1`' given input columns: [i.even, i.fivethous, i.four, i.hundred, i.odd, i.string4, i.stringu1, i.stringu2, i.ten, i.tenthous, i.thousand, i.twenty, i.two, i.twothousand, i.unique1, i.unique2]; line 2 pos 67
+cannot resolve 'o.unique1' given input columns: [i.even, i.fivethous, i.four, i.hundred, i.odd, i.string4, i.stringu1, i.stringu2, i.ten, i.tenthous, i.thousand, i.twenty, i.two, i.twothousand, i.unique1, i.unique2]; line 2 pos 67
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part2.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part2.sql.out
index d4941d0a0b768..a0008c3dd5522 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part2.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part2.sql.out
@@ -161,7 +161,7 @@ true	true	false	NULL	false	true
 -- !query
 select min(udf(unique1)) from tenk1
 -- !query schema
-struct<min(CAST(udf(ansi_cast(unique1 as string)) AS INT)):int>
+struct<min(udf(unique1)):int>
 -- !query output
 0
 
@@ -169,7 +169,7 @@ struct<min(CAST(udf(ansi_cast(unique1 as string)) AS INT)):int>
 -- !query
 select udf(max(unique1)) from tenk1
 -- !query schema
-struct<CAST(udf(ansi_cast(max(unique1) as string)) AS INT):int>
+struct<udf(max(unique1)):int>
 -- !query output
 9999
 
@@ -217,7 +217,7 @@ struct<min(tenthous):int>
 -- !query
 select distinct max(udf(unique2)) from tenk1
 -- !query schema
-struct<max(CAST(udf(ansi_cast(unique2 as string)) AS INT)):int>
+struct<max(udf(unique2)):int>
 -- !query output
 9999
 
@@ -241,7 +241,7 @@ struct<max(unique2):int>
 -- !query
 select udf(max(udf(unique2))) from tenk1 order by udf(max(unique2))+1
 -- !query schema
-struct<CAST(udf(ansi_cast(max(ansi_cast(udf(ansi_cast(unique2 as string)) as int)) as string)) AS INT):int>
+struct<udf(max(udf(unique2))):int>
 -- !query output
 9999
 
@@ -249,7 +249,7 @@ struct<CAST(udf(ansi_cast(max(ansi_cast(udf(ansi_cast(unique2 as string)) as int
 -- !query
 select t1.max_unique2, udf(g) from (select max(udf(unique2)) as max_unique2 FROM tenk1) t1 LATERAL VIEW explode(array(1,2,3)) t2 AS g order by g desc
 -- !query schema
-struct<max_unique2:int,CAST(udf(ansi_cast(g as string)) AS INT):int>
+struct<max_unique2:int,udf(g):int>
 -- !query output
 9999	3
 9999	2
@@ -259,6 +259,6 @@ struct<max_unique2:int,CAST(udf(ansi_cast(g as string)) AS INT):int>
 -- !query
 select udf(max(100)) from tenk1
 -- !query schema
-struct<CAST(udf(ansi_cast(max(100) as string)) AS INT):int>
+struct<udf(max(100)):int>
 -- !query output
 100
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part3.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part3.sql.out
index f491d9b9ba3a8..17b77a8a7aea9 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part3.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part3.sql.out
@@ -8,7 +8,7 @@ select udf(max(min(unique1))) from tenk1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-It is not allowed to use an aggregate function in the argument of another aggregate function. Please use the inner aggregate function in a sub-query.;
+It is not allowed to use an aggregate function in the argument of another aggregate function. Please use the inner aggregate function in a sub-query.
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-case.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-case.sql.out
index 6c733e916d734..1969717423914 100755
--- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-case.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-case.sql.out
@@ -176,34 +176,34 @@ struct<None:string,NULL on no matches:int>
 -- !query
 SELECT CASE WHEN udf(1=0) THEN 1/0 WHEN 1=1 THEN 1 ELSE 2/0 END
 -- !query schema
-struct<CASE WHEN CAST(udf(ansi_cast((1 = 0) as string)) AS BOOLEAN) THEN (CAST(1 AS DOUBLE) / CAST(0 AS DOUBLE)) WHEN (1 = 1) THEN CAST(1 AS DOUBLE) ELSE (CAST(2 AS DOUBLE) / CAST(0 AS DOUBLE)) END:double>
+struct<>
 -- !query output
-1.0
+java.lang.ArithmeticException
+divide by zero
 
 
 -- !query
 SELECT CASE 1 WHEN 0 THEN 1/udf(0) WHEN 1 THEN 1 ELSE 2/0 END
 -- !query schema
-struct<CASE WHEN (1 = 0) THEN (CAST(1 AS DOUBLE) / CAST(CAST(udf(ansi_cast(0 as string)) AS INT) AS DOUBLE)) WHEN (1 = 1) THEN CAST(1 AS DOUBLE) ELSE (CAST(2 AS DOUBLE) / CAST(0 AS DOUBLE)) END:double>
+struct<>
 -- !query output
-1.0
+java.lang.ArithmeticException
+divide by zero
 
 
 -- !query
 SELECT CASE WHEN i > 100 THEN udf(1/0) ELSE udf(0) END FROM case_tbl
 -- !query schema
-struct<CASE WHEN (i > 100) THEN CAST(udf(ansi_cast((ansi_cast(1 as double) / ansi_cast(0 as double)) as string)) AS DOUBLE) ELSE CAST(CAST(udf(ansi_cast(0 as string)) AS INT) AS DOUBLE) END:double>
+struct<>
 -- !query output
-0.0
-0.0
-0.0
-0.0
+java.lang.ArithmeticException
+divide by zero
 
 
 -- !query
 SELECT CASE 'a' WHEN 'a' THEN udf(1) ELSE udf(2) END
 -- !query schema
-struct<CASE WHEN (a = a) THEN CAST(udf(ansi_cast(1 as string)) AS INT) ELSE CAST(udf(ansi_cast(2 as string)) AS INT) END:int>
+struct<CASE WHEN (a = a) THEN udf(1) ELSE udf(2) END:int>
 -- !query output
 1
 
@@ -294,7 +294,7 @@ struct<i:int,f:double>
 SELECT udf(COALESCE(a.f, b.i, b.j))
   FROM CASE_TBL a, CASE2_TBL b
 -- !query schema
-struct<CAST(udf(ansi_cast(coalesce(f, ansi_cast(i as double), ansi_cast(j as double)) as string)) AS DOUBLE):double>
+struct<udf(coalesce(f, i, j)):double>
 -- !query output
 -30.3
 -30.3
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-join.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-join.sql.out
index 188b57ffd58d5..36529eabc758e 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-join.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-join.sql.out
@@ -243,7 +243,7 @@ struct<>
 SELECT udf('') AS `xxx`, udf(i), udf(j), udf(t)
   FROM J1_TBL AS tx
 -- !query schema
-struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string>
+struct<xxx:string,udf(i):int,udf(j):int,udf(t):string>
 -- !query output
 	0	NULL	zero
 	1	4	one
@@ -262,7 +262,7 @@ struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cas
 SELECT udf(udf('')) AS `xxx`, udf(udf(i)), udf(j), udf(t)
   FROM J1_TBL tx
 -- !query schema
-struct<xxx:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(i as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string>
+struct<xxx:string,udf(udf(i)):int,udf(j):int,udf(t):string>
 -- !query output
 	0	NULL	zero
 	1	4	one
@@ -281,7 +281,7 @@ struct<xxx:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(i as string)) as in
 SELECT udf('') AS `xxx`, a, udf(udf(b)), c
   FROM J1_TBL AS t1 (a, b, c)
 -- !query schema
-struct<xxx:string,a:int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(b as string)) as int) as string)) AS INT):int,c:string>
+struct<xxx:string,a:int,udf(udf(b)):int,c:string>
 -- !query output
 	0	NULL	zero
 	1	4	one
@@ -300,7 +300,7 @@ struct<xxx:string,a:int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(b as string))
 SELECT udf('') AS `xxx`, udf(a), udf(b), udf(udf(c))
   FROM J1_TBL t1 (a, b, c)
 -- !query schema
-struct<xxx:string,CAST(udf(ansi_cast(a as string)) AS INT):int,CAST(udf(ansi_cast(b as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(c as string)) as string) as string)) AS STRING):string>
+struct<xxx:string,udf(a):int,udf(b):int,udf(udf(c)):string>
 -- !query output
 	0	NULL	zero
 	1	4	one
@@ -319,7 +319,7 @@ struct<xxx:string,CAST(udf(ansi_cast(a as string)) AS INT):int,CAST(udf(ansi_cas
 SELECT udf('') AS `xxx`, udf(a), b, udf(c), udf(d), e
   FROM J1_TBL t1 (a, b, c), J2_TBL t2 (d, e)
 -- !query schema
-struct<xxx:string,CAST(udf(ansi_cast(a as string)) AS INT):int,b:int,CAST(udf(ansi_cast(c as string)) AS STRING):string,CAST(udf(ansi_cast(d as string)) AS INT):int,e:int>
+struct<xxx:string,udf(a):int,b:int,udf(c):string,udf(d):int,e:int>
 -- !query output
 	0	NULL	zero	0	NULL
 	0	NULL	zero	1	-1
@@ -543,7 +543,7 @@ Reference 'i' is ambiguous, could be: spark_catalog.default.j1_tbl.i, spark_cata
 SELECT udf('') AS `xxx`, udf(t1.i) AS i, udf(k), udf(t)
   FROM J1_TBL t1 CROSS JOIN J2_TBL t2
 -- !query schema
-struct<xxx:string,i:int,CAST(udf(ansi_cast(k as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string>
+struct<xxx:string,i:int,udf(k):int,udf(t):string>
 -- !query output
 	0	-1	zero
 	0	-3	zero
@@ -651,7 +651,7 @@ SELECT udf(udf('')) AS `xxx`, udf(udf(ii)) AS ii, udf(udf(tt)) AS tt, udf(udf(kk
   FROM (J1_TBL CROSS JOIN J2_TBL)
     AS tx (ii, jj, tt, ii2, kk)
 -- !query schema
-struct<xxx:string,ii:int,tt:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(kk as string)) as int) as string)) AS INT):int>
+struct<xxx:string,ii:int,tt:string,udf(udf(kk)):int>
 -- !query output
 	0	zero	-1
 	0	zero	-3
@@ -758,7 +758,7 @@ struct<xxx:string,ii:int,tt:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(kk
 SELECT udf('') AS `xxx`, udf(udf(j1_tbl.i)), udf(j), udf(t), udf(a.i), udf(a.k), udf(b.i),  udf(b.k)
   FROM J1_TBL CROSS JOIN J2_TBL a CROSS JOIN J2_TBL b
 -- !query schema
-struct<xxx:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(i as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(k as string)) AS INT):int,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(k as string)) AS INT):int>
+struct<xxx:string,udf(udf(i)):int,udf(j):int,udf(t):string,udf(i):int,udf(k):int,udf(i):int,udf(k):int>
 -- !query output
 	0	NULL	zero	0	NULL	0	NULL
 	0	NULL	zero	0	NULL	1	-1
@@ -1657,7 +1657,7 @@ struct<xxx:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(i as string)) as in
 SELECT udf('') AS `xxx`, udf(i) AS i, udf(j), udf(t) AS t, udf(k)
   FROM J1_TBL INNER JOIN J2_TBL USING (i)
 -- !query schema
-struct<xxx:string,i:int,CAST(udf(ansi_cast(j as string)) AS INT):int,t:string,CAST(udf(ansi_cast(k as string)) AS INT):int>
+struct<xxx:string,i:int,udf(j):int,t:string,udf(k):int>
 -- !query output
 	0	NULL	zero	NULL
 	1	4	one	-1
@@ -1672,7 +1672,7 @@ struct<xxx:string,i:int,CAST(udf(ansi_cast(j as string)) AS INT):int,t:string,CA
 SELECT udf(udf('')) AS `xxx`, udf(i), udf(j) AS j, udf(t), udf(k) AS k
   FROM J1_TBL JOIN J2_TBL USING (i)
 -- !query schema
-struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,j:int,CAST(udf(ansi_cast(t as string)) AS STRING):string,k:int>
+struct<xxx:string,udf(i):int,j:int,udf(t):string,k:int>
 -- !query output
 	0	NULL	zero	NULL
 	1	4	one	-1
@@ -1703,7 +1703,7 @@ struct<xxx:string,a:int,b:int,c:string,d:int>
 SELECT udf(udf('')) AS `xxx`, udf(i), udf(j), udf(t), udf(k)
   FROM J1_TBL NATURAL JOIN J2_TBL
 -- !query schema
-struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(k as string)) AS INT):int>
+struct<xxx:string,udf(i):int,udf(j):int,udf(t):string,udf(k):int>
 -- !query output
 	0	NULL	zero	NULL
 	1	4	one	-1
@@ -1718,7 +1718,7 @@ struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cas
 SELECT udf('') AS `xxx`, udf(udf(udf(a))) AS a, udf(b), udf(c), udf(d)
   FROM J1_TBL t1 (a, b, c) NATURAL JOIN J2_TBL t2 (a, d)
 -- !query schema
-struct<xxx:string,a:int,CAST(udf(ansi_cast(b as string)) AS INT):int,CAST(udf(ansi_cast(c as string)) AS STRING):string,CAST(udf(ansi_cast(d as string)) AS INT):int>
+struct<xxx:string,a:int,udf(b):int,udf(c):string,udf(d):int>
 -- !query output
 	0	NULL	zero	NULL
 	1	4	one	-1
@@ -1733,7 +1733,7 @@ struct<xxx:string,a:int,CAST(udf(ansi_cast(b as string)) AS INT):int,CAST(udf(an
 SELECT udf('') AS `xxx`, udf(udf(a)), udf(udf(b)), udf(udf(c)) AS c, udf(udf(udf(d))) AS d
   FROM J1_TBL t1 (a, b, c) NATURAL JOIN J2_TBL t2 (d, a)
 -- !query schema
-struct<xxx:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(a as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(b as string)) as int) as string)) AS INT):int,c:string,d:int>
+struct<xxx:string,udf(udf(a)):int,udf(udf(b)):int,c:string,d:int>
 -- !query output
 	0	NULL	zero	NULL
 	2	3	two	2
@@ -1744,7 +1744,7 @@ struct<xxx:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(a as string)) as in
 SELECT udf('') AS `xxx`, udf(J1_TBL.i), udf(udf(J1_TBL.j)), udf(J1_TBL.t), udf(J2_TBL.i), udf(J2_TBL.k)
   FROM J1_TBL JOIN J2_TBL ON (udf(J1_TBL.i) = J2_TBL.i)
 -- !query schema
-struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(j as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(k as string)) AS INT):int>
+struct<xxx:string,udf(i):int,udf(udf(j)):int,udf(t):string,udf(i):int,udf(k):int>
 -- !query output
 	0	NULL	zero	0	NULL
 	1	4	one	1	-1
@@ -1759,7 +1759,7 @@ struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cas
 SELECT udf('') AS `xxx`, udf(udf(J1_TBL.i)), udf(udf(J1_TBL.j)), udf(udf(J1_TBL.t)), J2_TBL.i, J2_TBL.k
   FROM J1_TBL JOIN J2_TBL ON (J1_TBL.i = udf(J2_TBL.k))
 -- !query schema
-struct<xxx:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(i as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(j as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(t as string)) as string) as string)) AS STRING):string,i:int,k:int>
+struct<xxx:string,udf(udf(i)):int,udf(udf(j)):int,udf(udf(t)):string,i:int,k:int>
 -- !query output
 	0	NULL	zero	NULL	0
 	2	3	two	2	2
@@ -1770,7 +1770,7 @@ struct<xxx:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(i as string)) as in
 SELECT udf('') AS `xxx`, udf(J1_TBL.i), udf(J1_TBL.j), udf(J1_TBL.t), udf(J2_TBL.i), udf(J2_TBL.k)
   FROM J1_TBL JOIN J2_TBL ON (udf(J1_TBL.i) <= udf(udf(J2_TBL.k)))
 -- !query schema
-struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(k as string)) AS INT):int>
+struct<xxx:string,udf(i):int,udf(j):int,udf(t):string,udf(i):int,udf(k):int>
 -- !query output
 	0	NULL	zero	2	2
 	0	NULL	zero	2	4
@@ -1788,7 +1788,7 @@ SELECT udf(udf('')) AS `xxx`, udf(i), udf(j), udf(t), udf(k)
   FROM J1_TBL LEFT OUTER JOIN J2_TBL USING (i)
   ORDER BY udf(udf(i)), udf(k), udf(t)
 -- !query schema
-struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(k as string)) AS INT):int>
+struct<xxx:string,udf(i):int,udf(j):int,udf(t):string,udf(k):int>
 -- !query output
 	NULL	NULL	null	NULL
 	NULL	0	zero	NULL
@@ -1810,7 +1810,7 @@ SELECT udf('') AS `xxx`, udf(i), udf(j), udf(t), udf(k)
   FROM J1_TBL LEFT JOIN J2_TBL USING (i)
   ORDER BY udf(i), udf(udf(k)), udf(t)
 -- !query schema
-struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(k as string)) AS INT):int>
+struct<xxx:string,udf(i):int,udf(j):int,udf(t):string,udf(k):int>
 -- !query output
 	NULL	NULL	null	NULL
 	NULL	0	zero	NULL
@@ -1831,7 +1831,7 @@ struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cas
 SELECT udf('') AS `xxx`, udf(udf(i)), udf(j), udf(t), udf(k)
   FROM J1_TBL RIGHT OUTER JOIN J2_TBL USING (i)
 -- !query schema
-struct<xxx:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(i as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(k as string)) AS INT):int>
+struct<xxx:string,udf(udf(i)):int,udf(j):int,udf(t):string,udf(k):int>
 -- !query output
 	0	NULL	zero	NULL
 	1	4	one	-1
@@ -1848,7 +1848,7 @@ struct<xxx:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(i as string)) as in
 SELECT udf('') AS `xxx`, udf(i), udf(udf(j)), udf(t), udf(k)
   FROM J1_TBL RIGHT JOIN J2_TBL USING (i)
 -- !query schema
-struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(j as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(k as string)) AS INT):int>
+struct<xxx:string,udf(i):int,udf(udf(j)):int,udf(t):string,udf(k):int>
 -- !query output
 	0	NULL	zero	NULL
 	1	4	one	-1
@@ -1866,7 +1866,7 @@ SELECT udf('') AS `xxx`, udf(i), udf(j), udf(udf(t)), udf(k)
   FROM J1_TBL FULL OUTER JOIN J2_TBL USING (i)
   ORDER BY udf(udf(i)), udf(k), udf(t)
 -- !query schema
-struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(t as string)) as string) as string)) AS STRING):string,CAST(udf(ansi_cast(k as string)) AS INT):int>
+struct<xxx:string,udf(i):int,udf(j):int,udf(udf(t)):string,udf(k):int>
 -- !query output
 	NULL	NULL	NULL	NULL
 	NULL	NULL	null	NULL
@@ -1890,7 +1890,7 @@ SELECT udf('') AS `xxx`, udf(i), udf(j), t, udf(udf(k))
   FROM J1_TBL FULL JOIN J2_TBL USING (i)
   ORDER BY udf(udf(i)), udf(k), udf(udf(t))
 -- !query schema
-struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,t:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(k as string)) as int) as string)) AS INT):int>
+struct<xxx:string,udf(i):int,udf(j):int,t:string,udf(udf(k)):int>
 -- !query output
 	NULL	NULL	NULL	NULL
 	NULL	NULL	null	NULL
@@ -1913,7 +1913,7 @@ struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cas
 SELECT udf('') AS `xxx`, udf(i), udf(j), udf(t), udf(udf(k))
   FROM J1_TBL LEFT JOIN J2_TBL USING (i) WHERE (udf(k) = 1)
 -- !query schema
-struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(k as string)) as int) as string)) AS INT):int>
+struct<xxx:string,udf(i):int,udf(j):int,udf(t):string,udf(udf(k)):int>
 -- !query output
 
 
@@ -1922,7 +1922,7 @@ struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cas
 SELECT udf('') AS `xxx`, udf(i), udf(j), udf(t), udf(k)
   FROM J1_TBL LEFT JOIN J2_TBL USING (i) WHERE (udf(udf(i)) = udf(1))
 -- !query schema
-struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(k as string)) AS INT):int>
+struct<xxx:string,udf(i):int,udf(j):int,udf(t):string,udf(k):int>
 -- !query output
 	1	4	one	-1
 
@@ -2052,7 +2052,7 @@ FULL JOIN
 (SELECT * FROM t3) s3
 USING (name)
 -- !query schema
-struct<CAST(udf(ansi_cast(name as string)) AS STRING):string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(n as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(n as string)) AS INT):int>
+struct<udf(name):string,udf(udf(n)):int,udf(n):int>
 -- !query output
 bb	12	13
 cc	22	23
@@ -2138,7 +2138,7 @@ NATURAL FULL JOIN
     (SELECT name, udf(udf(n)) as s3_n FROM t3) as s3
   ) ss2
 -- !query schema
-struct<name:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(s1_n as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(s2_n as string)) AS INT):int,CAST(udf(ansi_cast(s3_n as string)) AS INT):int>
+struct<name:string,udf(udf(s1_n)):int,udf(s2_n):int,udf(s3_n):int>
 -- !query output
 bb	11	12	13
 cc	NULL	22	23
@@ -2171,7 +2171,7 @@ FULL JOIN
   (SELECT name, 2 as s2_n FROM t2) as s2
 ON (udf(udf(s1_n)) = udf(s2_n))
 -- !query schema
-struct<name:string,CAST(udf(ansi_cast(s1_n as string)) AS INT):int,name:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(s2_n as string)) as int) as string)) AS INT):int>
+struct<name:string,udf(s1_n):int,name:string,udf(udf(s2_n)):int>
 -- !query output
 NULL	NULL	bb	2
 NULL	NULL	cc	2
@@ -2202,7 +2202,7 @@ struct<>
 -- !query
 select udf(udf(x1)), udf(x2) from x
 -- !query schema
-struct<CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(x1 as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(x2 as string)) AS INT):int>
+struct<udf(udf(x1)):int,udf(x2):int>
 -- !query output
 1	11
 2	22
@@ -2214,7 +2214,7 @@ struct<CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(x1 as string)) as int) as stri
 -- !query
 select udf(y1), udf(udf(y2)) from y
 -- !query schema
-struct<CAST(udf(ansi_cast(y1 as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(y2 as string)) as int) as string)) AS INT):int>
+struct<udf(y1):int,udf(udf(y2)):int>
 -- !query output
 1	111
 2	222
@@ -2336,7 +2336,7 @@ select udf(udf(count(*))) from tenk1 a where udf(udf(unique1)) in
   (select udf(unique1) from tenk1 b join tenk1 c using (unique1)
    where udf(udf(b.unique2)) = udf(42))
 -- !query schema
-struct<CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(count(1) as string)) as bigint) as string)) AS BIGINT):bigint>
+struct<udf(udf(count(1))):bigint>
 -- !query output
 1
 
@@ -2347,7 +2347,7 @@ select udf(count(*)) from tenk1 x where
   udf(x.unique1) = 0 and
   udf(x.unique1) in (select aa.f1 from int4_tbl aa,float8_tbl bb where aa.f1=udf(udf(bb.f1)))
 -- !query schema
-struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
+struct<udf(count(1)):bigint>
 -- !query output
 1
 
@@ -2358,7 +2358,7 @@ select udf(udf(count(*))) from tenk1 x where
   udf(x.unique1) = 0 and
   udf(udf(x.unique1)) in (select udf(aa.f1) from int4_tbl aa,float8_tbl bb where udf(aa.f1)=udf(udf(bb.f1)))
 -- !query schema
-struct<CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(count(1) as string)) as bigint) as string)) AS BIGINT):bigint>
+struct<udf(udf(count(1))):bigint>
 -- !query output
 1
 
@@ -2388,7 +2388,7 @@ from
   tenk1 t5
 where udf(t4.thousand) = udf(t5.unique1) and udf(udf(ss.x1)) = t4.tenthous and udf(ss.x2) = udf(udf(t5.stringu1))
 -- !query schema
-struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
+struct<udf(count(1)):bigint>
 -- !query output
 1000
 
@@ -2400,7 +2400,7 @@ select udf(a.f1), udf(b.f1), udf(t.thousand), udf(t.tenthous) from
   (select udf(sum(udf(f1))) as f1 from int4_tbl i4b) b
 where b.f1 = udf(t.thousand) and udf(a.f1) = udf(b.f1) and udf((udf(a.f1)+udf(b.f1)+999)) = udf(udf(t.tenthous))
 -- !query schema
-struct<CAST(udf(ansi_cast(f1 as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(f1 as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(thousand as string)) AS INT):int,CAST(udf(ansi_cast(tenthous as string)) AS INT):int>
+struct<udf(f1):bigint,udf(f1):bigint,udf(thousand):int,udf(tenthous):int>
 -- !query output
 
 
@@ -2441,7 +2441,7 @@ select udf(count(*)) from
   (select * from tenk1 y order by udf(y.unique2)) y
   on udf(x.thousand) = y.unique2 and x.twothousand = udf(y.hundred) and x.fivethous = y.unique2
 -- !query schema
-struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
+struct<udf(count(1)):bigint>
 -- !query output
 10000
 
@@ -2530,7 +2530,7 @@ struct<tt1_id:int,joincol:int,tt2_id:int,joincol:int>
 select udf(count(*)) from tenk1 a, tenk1 b
   where udf(a.hundred) = b.thousand and udf(udf((b.fivethous % 10)) < 10)
 -- !query schema
-struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
+struct<udf(count(1)):bigint>
 -- !query output
 100000
 
@@ -2727,7 +2727,7 @@ from tenk1 a left join tenk1 b on a.unique2 = udf(b.tenthous)
 where udf(a.unique1) = 42 and
       ((udf(b.unique2) is null and udf(a.ten) = 2) or udf(udf(b.hundred)) = udf(udf(3)))
 -- !query schema
-struct<CAST(udf(ansi_cast(unique2 as string)) AS INT):int,CAST(udf(ansi_cast(ten as string)) AS INT):int,CAST(udf(ansi_cast(tenthous as string)) AS INT):int,CAST(udf(ansi_cast(unique2 as string)) AS INT):int,CAST(udf(ansi_cast(hundred as string)) AS INT):int>
+struct<udf(unique2):int,udf(ten):int,udf(tenthous):int,udf(unique2):int,udf(hundred):int>
 -- !query output
 
 
@@ -2757,11 +2757,11 @@ struct<i:int,x:int,y:int>
 
 
 -- !query
-select udf(t1.q2), udf(count(t2.*))
+select udf(t1.q2), udf(count(t2.q1, t2.q2))
 from int8_tbl t1 left join int8_tbl t2 on (udf(udf(t1.q2)) = t2.q1)
 group by udf(t1.q2) order by 1
 -- !query schema
-struct<CAST(udf(ansi_cast(q2 as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(count(q1, q2) as string)) AS BIGINT):bigint>
+struct<udf(q2):bigint,udf(count(q1, q2)):bigint>
 -- !query output
 -4567890123456789	0
 123	2
@@ -2770,11 +2770,11 @@ struct<CAST(udf(ansi_cast(q2 as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(co
 
 
 -- !query
-select udf(udf(t1.q2)), udf(count(t2.*))
+select udf(udf(t1.q2)), udf(count(t2.q1, t2.q2))
 from int8_tbl t1 left join (select * from int8_tbl) t2 on (udf(udf(t1.q2)) = udf(t2.q1))
 group by udf(udf(t1.q2)) order by 1
 -- !query schema
-struct<CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(q2 as string)) as bigint) as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(count(q1, q2) as string)) AS BIGINT):bigint>
+struct<udf(udf(q2)):bigint,udf(count(q1, q2)):bigint>
 -- !query output
 -4567890123456789	0
 123	2
@@ -2783,13 +2783,13 @@ struct<CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(q2 as string)) as bigint) as s
 
 
 -- !query
-select udf(t1.q2) as q2, udf(udf(count(t2.*)))
+select udf(t1.q2) as q2, udf(udf(count(t2.q1, t2.q2)))
 from int8_tbl t1 left join
   (select udf(q1) as q1, case when q2=1 then 1 else q2 end as q2 from int8_tbl) t2
   on (udf(t1.q2) = udf(t2.q1))
 group by t1.q2 order by 1
 -- !query schema
-struct<q2:bigint,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(count(q1, q2) as string)) as bigint) as string)) AS BIGINT):bigint>
+struct<q2:bigint,udf(udf(count(q1, q2))):bigint>
 -- !query output
 -4567890123456789	0
 123	2
@@ -2838,7 +2838,7 @@ from c left join
   on (udf(udf(c.a)) = udf(ss.code))
 order by c.name
 -- !query schema
-struct<CAST(udf(ansi_cast(name as string)) AS STRING):string,CAST(udf(ansi_cast(code as string)) AS STRING):string,CAST(udf(ansi_cast(b_cnt as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(const as string)) AS INT):int>
+struct<udf(name):string,udf(code):string,udf(b_cnt):bigint,udf(const):int>
 -- !query output
 A	p	2	-1
 B	q	0	-1
@@ -2884,7 +2884,7 @@ LEFT JOIN
 ) sub2
 ON sub1.key1 = udf(udf(sub2.key3))
 -- !query schema
-struct<key1:int,key3:int,CAST(udf(ansi_cast(value2 as string)) AS INT):int,value3:int>
+struct<key1:int,key3:int,udf(value2):int,value3:int>
 -- !query output
 1	1	1	1
 
@@ -2898,7 +2898,7 @@ SELECT udf(qq), udf(udf(unique1))
   USING (qq)
   INNER JOIN tenk1 c ON udf(qq) = udf(unique2)
 -- !query schema
-struct<CAST(udf(ansi_cast(qq as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(unique1 as string)) as int) as string)) AS INT):int>
+struct<udf(qq):bigint,udf(udf(unique1)):int>
 -- !query output
 123	4596
 123	4596
@@ -2948,7 +2948,7 @@ from nt3 as nt3
     on udf(ss2.id) = nt3.nt2_id
 where udf(nt3.id) = 1 and udf(ss2.b3)
 -- !query schema
-struct<CAST(udf(ansi_cast(id as string)) AS INT):int>
+struct<udf(id):int>
 -- !query output
 1
 
@@ -3008,7 +3008,7 @@ select udf(count(*)) from
   left join tenk1 c on udf(a.unique2) = udf(b.unique1) and udf(c.thousand) = udf(udf(a.thousand))
   join int4_tbl on udf(b.thousand) = f1
 -- !query schema
-struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
+struct<udf(count(1)):bigint>
 -- !query output
 10
 
@@ -3021,7 +3021,7 @@ select udf(b.unique1) from
   right join int4_tbl i2 on udf(udf(i2.f1)) = udf(b.tenthous)
   order by udf(1)
 -- !query schema
-struct<CAST(udf(ansi_cast(unique1 as string)) AS INT):int>
+struct<udf(unique1):int>
 -- !query output
 NULL
 NULL
@@ -3039,7 +3039,7 @@ select * from
 where udf(fault) = udf(122)
 order by udf(fault)
 -- !query schema
-struct<CAST(udf(ansi_cast(unique1 as string)) AS INT):int,CAST(udf(ansi_cast(q1 as string)) AS BIGINT):bigint,fault:bigint>
+struct<udf(unique1):int,udf(q1):bigint,fault:bigint>
 -- !query output
 NULL	123	122
 
@@ -3049,7 +3049,7 @@ select udf(q1), udf(unique2), udf(thousand), udf(hundred)
   from int8_tbl a left join tenk1 b on udf(q1) = udf(unique2)
   where udf(coalesce(thousand,123)) = udf(q1) and udf(q1) = udf(udf(coalesce(hundred,123)))
 -- !query schema
-struct<CAST(udf(ansi_cast(q1 as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(unique2 as string)) AS INT):int,CAST(udf(ansi_cast(thousand as string)) AS INT):int,CAST(udf(ansi_cast(hundred as string)) AS INT):int>
+struct<udf(q1):bigint,udf(unique2):int,udf(thousand):int,udf(hundred):int>
 -- !query output
 
 
@@ -3059,7 +3059,7 @@ select udf(f1), udf(unique2), case when udf(udf(unique2)) is null then udf(f1) e
   from int4_tbl a left join tenk1 b on udf(f1) = udf(udf(unique2))
   where (case when udf(unique2) is null then udf(f1) else 0 end) = 0
 -- !query schema
-struct<CAST(udf(ansi_cast(f1 as string)) AS INT):int,CAST(udf(ansi_cast(unique2 as string)) AS INT):int,CASE WHEN (CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(unique2 as string)) as int) as string)) AS INT) IS NULL) THEN CAST(udf(ansi_cast(f1 as string)) AS INT) ELSE 0 END:int>
+struct<udf(f1):int,udf(unique2):int,CASE WHEN (udf(udf(unique2)) IS NULL) THEN udf(f1) ELSE 0 END:int>
 -- !query output
 0	0	0
 
@@ -3069,7 +3069,7 @@ select udf(a.unique1), udf(b.unique1), udf(c.unique1), udf(coalesce(b.twothousan
   from tenk1 a left join tenk1 b on udf(b.thousand) = a.unique1                       left join tenk1 c on udf(c.unique2) = udf(coalesce(b.twothousand, a.twothousand))
   where a.unique2 < udf(10) and udf(udf(coalesce(b.twothousand, a.twothousand))) = udf(44)
 -- !query schema
-struct<CAST(udf(ansi_cast(unique1 as string)) AS INT):int,CAST(udf(ansi_cast(unique1 as string)) AS INT):int,CAST(udf(ansi_cast(unique1 as string)) AS INT):int,CAST(udf(ansi_cast(coalesce(twothousand, twothousand) as string)) AS INT):int>
+struct<udf(unique1):int,udf(unique1):int,udf(unique1):int,udf(coalesce(twothousand, twothousand)):int>
 -- !query output
 
 
@@ -3107,7 +3107,7 @@ select udf(a.q2), udf(b.q1)
   from int8_tbl a left join int8_tbl b on udf(a.q2) = coalesce(b.q1, 1)
   where udf(udf(coalesce(b.q1, 1)) > 0)
 -- !query schema
-struct<CAST(udf(ansi_cast(q2 as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(q1 as string)) AS BIGINT):bigint>
+struct<udf(q2):bigint,udf(q1):bigint>
 -- !query output
 -4567890123456789	NULL
 123	123
@@ -3237,7 +3237,7 @@ SELECT * FROM
      FROM int8_tbl LEFT JOIN innertab ON udf(udf(q2)) = id) ss2
   ON true
 -- !query schema
-struct<x:int,CAST(udf(ansi_cast(q1 as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(q2 as string)) AS BIGINT):bigint,y:bigint>
+struct<x:int,udf(q1):bigint,udf(q2):bigint,y:bigint>
 -- !query output
 1	123	456	123
 1	123	4567890123456789	123
@@ -3263,7 +3263,7 @@ select * from
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`y.f1`' given input columns: [j.f1, j.f1, x.q1, x.q2]; line 2 pos 72
+cannot resolve 'y.f1' given input columns: [j.f1, j.f1, x.q1, x.q2]; line 2 pos 72
 
 
 -- !query
@@ -3282,7 +3282,7 @@ select udf(t1.uunique1) from
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`t1.uunique1`' given input columns: [t1.even, t2.even, t1.fivethous, t2.fivethous, t1.four, t2.four, t1.hundred, t2.hundred, t1.odd, t2.odd, t1.string4, t2.string4, t1.stringu1, t2.stringu1, t1.stringu2, t2.stringu2, t1.ten, t2.ten, t1.tenthous, t2.tenthous, t1.thousand, t2.thousand, t1.twenty, t2.twenty, t1.two, t2.two, t1.twothousand, t2.twothousand, t1.unique1, t2.unique1, t1.unique2, t2.unique2]; line 1 pos 11
+cannot resolve 't1.uunique1' given input columns: [t1.even, t2.even, t1.fivethous, t2.fivethous, t1.four, t2.four, t1.hundred, t2.hundred, t1.odd, t2.odd, t1.string4, t2.string4, t1.stringu1, t2.stringu1, t1.stringu2, t2.stringu2, t1.ten, t2.ten, t1.tenthous, t2.tenthous, t1.thousand, t2.thousand, t1.twenty, t2.twenty, t1.two, t2.two, t1.twothousand, t2.twothousand, t1.unique1, t2.unique1, t1.unique2, t2.unique2]; line 1 pos 11
 
 
 -- !query
@@ -3292,7 +3292,7 @@ select udf(udf(t2.uunique1)) from
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`t2.uunique1`' given input columns: [t1.even, t2.even, t1.fivethous, t2.fivethous, t1.four, t2.four, t1.hundred, t2.hundred, t1.odd, t2.odd, t1.string4, t2.string4, t1.stringu1, t2.stringu1, t1.stringu2, t2.stringu2, t1.ten, t2.ten, t1.tenthous, t2.tenthous, t1.thousand, t2.thousand, t1.twenty, t2.twenty, t1.two, t2.two, t1.twothousand, t2.twothousand, t1.unique1, t2.unique1, t1.unique2, t2.unique2]; line 1 pos 15
+cannot resolve 't2.uunique1' given input columns: [t1.even, t2.even, t1.fivethous, t2.fivethous, t1.four, t2.four, t1.hundred, t2.hundred, t1.odd, t2.odd, t1.string4, t2.string4, t1.stringu1, t2.stringu1, t1.stringu2, t2.stringu2, t1.ten, t2.ten, t1.tenthous, t2.tenthous, t1.thousand, t2.thousand, t1.twenty, t2.twenty, t1.two, t2.two, t1.twothousand, t2.twothousand, t1.unique1, t2.unique1, t1.unique2, t2.unique2]; line 1 pos 15
 
 
 -- !query
@@ -3302,7 +3302,7 @@ select udf(uunique1) from
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`uunique1`' given input columns: [t1.even, t2.even, t1.fivethous, t2.fivethous, t1.four, t2.four, t1.hundred, t2.hundred, t1.odd, t2.odd, t1.string4, t2.string4, t1.stringu1, t2.stringu1, t1.stringu2, t2.stringu2, t1.ten, t2.ten, t1.tenthous, t2.tenthous, t1.thousand, t2.thousand, t1.twenty, t2.twenty, t1.two, t2.two, t1.twothousand, t2.twothousand, t1.unique1, t2.unique1, t1.unique2, t2.unique2]; line 1 pos 11
+cannot resolve 'uunique1' given input columns: [t1.even, t2.even, t1.fivethous, t2.fivethous, t1.four, t2.four, t1.hundred, t2.hundred, t1.odd, t2.odd, t1.string4, t2.string4, t1.stringu1, t2.stringu1, t1.stringu2, t2.stringu2, t1.ten, t2.ten, t1.tenthous, t2.tenthous, t1.thousand, t2.thousand, t1.twenty, t2.twenty, t1.two, t2.two, t1.twothousand, t2.twothousand, t1.unique1, t2.unique1, t1.unique2, t2.unique2]; line 1 pos 11
 
 
 -- !query
@@ -3311,7 +3311,7 @@ select udf(udf(f1,g)) from int4_tbl a, (select udf(udf(f1)) as g) ss
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`f1`' given input columns: []; line 1 pos 55
+cannot resolve 'f1' given input columns: []; line 1 pos 55
 
 
 -- !query
@@ -3320,7 +3320,7 @@ select udf(f1,g) from int4_tbl a, (select a.f1 as g) ss
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`a.f1`' given input columns: []; line 1 pos 42
+cannot resolve 'a.f1' given input columns: []; line 1 pos 42
 
 
 -- !query
@@ -3329,7 +3329,7 @@ select udf(udf(f1,g)) from int4_tbl a cross join (select udf(f1) as g) ss
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`f1`' given input columns: []; line 1 pos 61
+cannot resolve 'f1' given input columns: []; line 1 pos 61
 
 
 -- !query
@@ -3338,7 +3338,7 @@ select udf(f1,g) from int4_tbl a cross join (select udf(udf(a.f1)) as g) ss
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`a.f1`' given input columns: []; line 1 pos 60
+cannot resolve 'a.f1' given input columns: []; line 1 pos 60
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_having.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_having.sql.out
index 50b6e60086747..3e69b2029e514 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_having.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_having.sql.out
@@ -94,7 +94,7 @@ struct<>
 SELECT udf(b), udf(c) FROM test_having
 	GROUP BY b, c HAVING udf(count(*)) = 1 ORDER BY udf(b), udf(c)
 -- !query schema
-struct<CAST(udf(ansi_cast(b as string)) AS INT):int,CAST(udf(ansi_cast(c as string)) AS STRING):string>
+struct<udf(b):int,udf(c):string>
 -- !query output
 1	XXXX
 3	bbbb
@@ -104,7 +104,7 @@ struct<CAST(udf(ansi_cast(b as string)) AS INT):int,CAST(udf(ansi_cast(c as stri
 SELECT udf(b), udf(c) FROM test_having
 	GROUP BY b, c HAVING udf(b) = 3 ORDER BY udf(b), udf(c)
 -- !query schema
-struct<CAST(udf(ansi_cast(b as string)) AS INT):int,CAST(udf(ansi_cast(c as string)) AS STRING):string>
+struct<udf(b):int,udf(c):string>
 -- !query output
 3	BBBB
 3	bbbb
@@ -115,7 +115,7 @@ SELECT udf(c), max(udf(a)) FROM test_having
 	GROUP BY c HAVING udf(count(*)) > 2 OR udf(min(a)) = udf(max(a))
 	ORDER BY c
 -- !query schema
-struct<CAST(udf(ansi_cast(c as string)) AS STRING):string,max(CAST(udf(ansi_cast(a as string)) AS INT)):int>
+struct<udf(c):string,max(udf(a)):int>
 -- !query output
 XXXX	0
 bbbb	5
@@ -124,7 +124,7 @@ bbbb	5
 -- !query
 SELECT udf(udf(min(udf(a)))), udf(udf(max(udf(a)))) FROM test_having HAVING udf(udf(min(udf(a)))) = udf(udf(max(udf(a))))
 -- !query schema
-struct<CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(min(ansi_cast(udf(ansi_cast(a as string)) as int)) as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(max(ansi_cast(udf(ansi_cast(a as string)) as int)) as string)) as int) as string)) AS INT):int>
+struct<udf(udf(min(udf(a)))):int,udf(udf(max(udf(a)))):int>
 -- !query output
 
 
@@ -132,7 +132,7 @@ struct<CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(min(ansi_cast(udf(ansi_cast(a
 -- !query
 SELECT udf(min(udf(a))), udf(udf(max(a))) FROM test_having HAVING udf(min(a)) < udf(max(udf(a)))
 -- !query schema
-struct<CAST(udf(ansi_cast(min(ansi_cast(udf(ansi_cast(a as string)) as int)) as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(max(a) as string)) as int) as string)) AS INT):int>
+struct<udf(min(udf(a))):int,udf(udf(max(a))):int>
 -- !query output
 0	9
 
@@ -143,7 +143,7 @@ SELECT udf(a) FROM test_having HAVING udf(min(a)) < udf(max(a))
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-grouping expressions sequence is empty, and 'spark_catalog.default.test_having.`a`' is not an aggregate function. Wrap '(min(spark_catalog.default.test_having.`a`) AS `min(a#x)`, max(spark_catalog.default.test_having.`a`) AS `max(a#x)`)' in windowing function(s) or wrap 'spark_catalog.default.test_having.`a`' in first() (or first_value) if you don't care which value you get.;
+grouping expressions sequence is empty, and 'spark_catalog.default.test_having.a' is not an aggregate function. Wrap '(min(spark_catalog.default.test_having.a) AS `min(a#x)`, max(spark_catalog.default.test_having.a) AS `max(a#x)`)' in windowing function(s) or wrap 'spark_catalog.default.test_having.a' in first() (or first_value) if you don't care which value you get.
 
 
 -- !query
@@ -152,7 +152,7 @@ SELECT 1 AS one FROM test_having HAVING udf(a) > 1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`a`' given input columns: [one]; line 1 pos 44
+cannot resolve 'a' given input columns: [one]; line 1 pos 44
 
 
 -- !query
@@ -174,9 +174,10 @@ struct<one:int>
 -- !query
 SELECT 1 AS one FROM test_having WHERE 1/udf(a) = 1 HAVING 1 < 2
 -- !query schema
-struct<one:int>
+struct<>
 -- !query output
-1
+java.lang.ArithmeticException
+divide by zero
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_implicit.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_implicit.sql.out
index 66e6c20a2f6f2..ee1f673cc9d9a 100755
--- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_implicit.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_implicit.sql.out
@@ -95,7 +95,7 @@ SELECT udf(c), udf(count(*)) FROM test_missing_target GROUP BY
 udf(test_missing_target.c)
 ORDER BY udf(c)
 -- !query schema
-struct<CAST(udf(ansi_cast(c as string)) AS STRING):string,CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
+struct<udf(c):string,udf(count(1)):bigint>
 -- !query output
 ABAB	2
 BBBB	2
@@ -109,7 +109,7 @@ cccc	2
 SELECT udf(count(*)) FROM test_missing_target GROUP BY udf(test_missing_target.c)
 ORDER BY udf(c)
 -- !query schema
-struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
+struct<udf(count(1)):bigint>
 -- !query output
 2
 2
@@ -125,13 +125,13 @@ SELECT udf(count(*)) FROM test_missing_target GROUP BY udf(a) ORDER BY udf(b)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`b`' given input columns: [CAST(udf(ansi_cast(count(1) as string)) AS BIGINT)]; line 1 pos 75
+cannot resolve 'b' given input columns: [udf(count(1))]; line 1 pos 75
 
 
 -- !query
 SELECT udf(count(*)) FROM test_missing_target GROUP BY udf(b) ORDER BY udf(b)
 -- !query schema
-struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
+struct<udf(count(1)):bigint>
 -- !query output
 1
 2
@@ -143,7 +143,7 @@ struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 SELECT udf(test_missing_target.b), udf(count(*))
   FROM test_missing_target GROUP BY udf(b) ORDER BY udf(b)
 -- !query schema
-struct<CAST(udf(ansi_cast(b as string)) AS INT):int,CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
+struct<udf(b):int,udf(count(1)):bigint>
 -- !query output
 1	1
 2	2
@@ -154,7 +154,7 @@ struct<CAST(udf(ansi_cast(b as string)) AS INT):int,CAST(udf(ansi_cast(count(1)
 -- !query
 SELECT udf(c) FROM test_missing_target ORDER BY udf(a)
 -- !query schema
-struct<CAST(udf(ansi_cast(c as string)) AS STRING):string>
+struct<udf(c):string>
 -- !query output
 XXXX
 ABAB
@@ -171,7 +171,7 @@ CCCC
 -- !query
 SELECT udf(count(*)) FROM test_missing_target GROUP BY udf(b) ORDER BY udf(b) desc
 -- !query schema
-struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
+struct<udf(count(1)):bigint>
 -- !query output
 4
 3
@@ -182,7 +182,7 @@ struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query
 SELECT udf(count(*)) FROM test_missing_target ORDER BY udf(1) desc
 -- !query schema
-struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
+struct<udf(count(1)):bigint>
 -- !query output
 10
 
@@ -190,7 +190,7 @@ struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query
 SELECT udf(c), udf(count(*)) FROM test_missing_target GROUP BY 1 ORDER BY 1
 -- !query schema
-struct<CAST(udf(ansi_cast(c as string)) AS STRING):string,CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
+struct<udf(c):string,udf(count(1)):bigint>
 -- !query output
 ABAB	2
 BBBB	2
@@ -224,7 +224,7 @@ Reference 'b' is ambiguous, could be: x.b, y.b.; line 3 pos 14
 SELECT udf(a), udf(a) FROM test_missing_target
 	ORDER BY udf(a)
 -- !query schema
-struct<CAST(udf(ansi_cast(a as string)) AS INT):int,CAST(udf(ansi_cast(a as string)) AS INT):int>
+struct<udf(a):int,udf(a):int>
 -- !query output
 0	0
 1	1
@@ -242,7 +242,7 @@ struct<CAST(udf(ansi_cast(a as string)) AS INT):int,CAST(udf(ansi_cast(a as stri
 SELECT udf(udf(a)/2), udf(udf(a)/2) FROM test_missing_target
 	ORDER BY udf(udf(a)/2)
 -- !query schema
-struct<CAST(udf(ansi_cast((ansi_cast(ansi_cast(udf(ansi_cast(a as string)) as int) as double) / ansi_cast(2 as double)) as string)) AS DOUBLE):double,CAST(udf(ansi_cast((ansi_cast(ansi_cast(udf(ansi_cast(a as string)) as int) as double) / ansi_cast(2 as double)) as string)) AS DOUBLE):double>
+struct<udf((udf(a) / 2)):double,udf((udf(a) / 2)):double>
 -- !query output
 0.0	0.0
 0.5	0.5
@@ -260,7 +260,7 @@ struct<CAST(udf(ansi_cast((ansi_cast(ansi_cast(udf(ansi_cast(a as string)) as in
 SELECT udf(a/2), udf(a/2) FROM test_missing_target
 	GROUP BY udf(a/2) ORDER BY udf(a/2)
 -- !query schema
-struct<CAST(udf(ansi_cast((ansi_cast(a as double) / ansi_cast(2 as double)) as string)) AS DOUBLE):double,CAST(udf(ansi_cast((ansi_cast(a as double) / ansi_cast(2 as double)) as string)) AS DOUBLE):double>
+struct<udf((a / 2)):double,udf((a / 2)):double>
 -- !query output
 0.0	0.0
 0.5	0.5
@@ -279,7 +279,7 @@ SELECT udf(x.b), udf(count(*)) FROM test_missing_target x, test_missing_target y
 	WHERE udf(x.a) = udf(y.a)
 	GROUP BY udf(x.b) ORDER BY udf(x.b)
 -- !query schema
-struct<CAST(udf(ansi_cast(b as string)) AS INT):int,CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
+struct<udf(b):int,udf(count(1)):bigint>
 -- !query output
 1	1
 2	2
@@ -292,7 +292,7 @@ SELECT udf(count(*)) FROM test_missing_target x, test_missing_target y
 	WHERE udf(x.a) = udf(y.a)
 	GROUP BY udf(x.b) ORDER BY udf(x.b)
 -- !query schema
-struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
+struct<udf(count(1)):bigint>
 -- !query output
 1
 2
@@ -305,7 +305,7 @@ SELECT udf(a%2), udf(count(udf(b))) FROM test_missing_target
 GROUP BY udf(test_missing_target.a%2)
 ORDER BY udf(test_missing_target.a%2)
 -- !query schema
-struct<CAST(udf(ansi_cast((a % 2) as string)) AS INT):int,CAST(udf(ansi_cast(count(ansi_cast(udf(ansi_cast(b as string)) as int)) as string)) AS BIGINT):bigint>
+struct<udf((a % 2)):int,udf(count(udf(b))):bigint>
 -- !query output
 0	5
 1	5
@@ -316,7 +316,7 @@ SELECT udf(count(c)) FROM test_missing_target
 GROUP BY udf(lower(test_missing_target.c))
 ORDER BY udf(lower(test_missing_target.c))
 -- !query schema
-struct<CAST(udf(ansi_cast(count(c) as string)) AS BIGINT):bigint>
+struct<udf(count(c)):bigint>
 -- !query output
 2
 3
@@ -330,13 +330,13 @@ SELECT udf(count(udf(a))) FROM test_missing_target GROUP BY udf(a) ORDER BY udf(
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`b`' given input columns: [CAST(udf(ansi_cast(count(ansi_cast(udf(ansi_cast(a as string)) as int)) as string)) AS BIGINT)]; line 1 pos 80
+cannot resolve 'b' given input columns: [udf(count(udf(a)))]; line 1 pos 80
 
 
 -- !query
 SELECT udf(count(b)) FROM test_missing_target GROUP BY udf(b/2) ORDER BY udf(b/2)
 -- !query schema
-struct<CAST(udf(ansi_cast(count(b) as string)) AS BIGINT):bigint>
+struct<udf(count(b)):bigint>
 -- !query output
 1
 2
@@ -348,7 +348,7 @@ struct<CAST(udf(ansi_cast(count(b) as string)) AS BIGINT):bigint>
 SELECT udf(lower(test_missing_target.c)), udf(count(udf(c)))
   FROM test_missing_target GROUP BY udf(lower(c)) ORDER BY udf(lower(c))
 -- !query schema
-struct<CAST(udf(ansi_cast(lower(c) as string)) AS STRING):string,CAST(udf(ansi_cast(count(ansi_cast(udf(ansi_cast(c as string)) as string)) as string)) AS BIGINT):bigint>
+struct<udf(lower(c)):string,udf(count(udf(c))):bigint>
 -- !query output
 abab	2
 bbbb	3
@@ -359,7 +359,7 @@ xxxx	1
 -- !query
 SELECT udf(a) FROM test_missing_target ORDER BY udf(upper(udf(d)))
 -- !query schema
-struct<CAST(udf(ansi_cast(a as string)) AS INT):int>
+struct<udf(a):int>
 -- !query output
 0
 1
@@ -377,7 +377,7 @@ struct<CAST(udf(ansi_cast(a as string)) AS INT):int>
 SELECT udf(count(b)) FROM test_missing_target
 	GROUP BY udf((b + 1) / 2) ORDER BY udf((b + 1) / 2) desc
 -- !query schema
-struct<CAST(udf(ansi_cast(count(b) as string)) AS BIGINT):bigint>
+struct<udf(count(b)):bigint>
 -- !query output
 4
 3
@@ -402,7 +402,7 @@ test_missing_target y
 	WHERE udf(x.a) = udf(y.a)
 	GROUP BY udf(x.b/2) ORDER BY udf(x.b/2)
 -- !query schema
-struct<CAST(udf(ansi_cast((ansi_cast(b as double) / ansi_cast(2 as double)) as string)) AS DOUBLE):double,CAST(udf(ansi_cast(count(ansi_cast(udf(ansi_cast(b as string)) as int)) as string)) AS BIGINT):bigint>
+struct<udf((b / 2)):double,udf(count(udf(b))):bigint>
 -- !query output
 0.5	1
 1.0	2
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-count.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/udf-count.sql.out
index e66948dcdea34..4bd8da618607a 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/udf-count.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-count.sql.out
@@ -17,7 +17,7 @@ SELECT
   udf(count(*)), udf(count(1)), udf(count(null)), udf(count(a)), udf(count(b)), udf(count(a + b)), udf(count((a, b)))
 FROM testData
 -- !query schema
-struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint,CAST(udf(cast(count(1) as string)) AS BIGINT):bigint,CAST(udf(cast(count(null) as string)) AS BIGINT):bigint,CAST(udf(cast(count(a) as string)) AS BIGINT):bigint,CAST(udf(cast(count(b) as string)) AS BIGINT):bigint,CAST(udf(cast(count((a + b)) as string)) AS BIGINT):bigint,CAST(udf(cast(count(named_struct(a, a, b, b)) as string)) AS BIGINT):bigint>
+struct<udf(count(1)):bigint,udf(count(1)):bigint,udf(count(NULL)):bigint,udf(count(a)):bigint,udf(count(b)):bigint,udf(count((a + b))):bigint,udf(count(named_struct(a, a, b, b))):bigint>
 -- !query output
 7	7	0	5	5	4	7
 
@@ -32,24 +32,24 @@ SELECT
   udf(count(DISTINCT (a, b)))
 FROM testData
 -- !query schema
-struct<CAST(udf(cast(count(distinct 1) as string)) AS BIGINT):bigint,CAST(udf(cast(count(distinct null) as string)) AS BIGINT):bigint,CAST(udf(cast(count(distinct a) as string)) AS BIGINT):bigint,CAST(udf(cast(count(distinct b) as string)) AS BIGINT):bigint,CAST(udf(cast(count(distinct (a + b)) as string)) AS BIGINT):bigint,CAST(udf(cast(count(distinct named_struct(a, a, b, b)) as string)) AS BIGINT):bigint>
+struct<udf(count(DISTINCT 1)):bigint,udf(count(DISTINCT NULL)):bigint,udf(count(DISTINCT a)):bigint,udf(count(DISTINCT b)):bigint,udf(count(DISTINCT (a + b))):bigint,udf(count(DISTINCT named_struct(a, a, b, b))):bigint>
 -- !query output
 1	0	2	2	2	6
 
 
 -- !query
-SELECT udf(count(a, b)), udf(count(b, a)), udf(count(testData.*)) FROM testData
+SELECT udf(count(a, b)), udf(count(b, a)), udf(count(testData.*, testData.*)) FROM testData
 -- !query schema
-struct<CAST(udf(cast(count(a, b) as string)) AS BIGINT):bigint,CAST(udf(cast(count(b, a) as string)) AS BIGINT):bigint,CAST(udf(cast(count(a, b) as string)) AS BIGINT):bigint>
+struct<udf(count(a, b)):bigint,udf(count(b, a)):bigint,udf(count(a, b, a, b)):bigint>
 -- !query output
 4	4	4
 
 
 -- !query
 SELECT
-  udf(count(DISTINCT a, b)), udf(count(DISTINCT b, a)), udf(count(DISTINCT *)), udf(count(DISTINCT testData.*))
+  udf(count(DISTINCT a, b)), udf(count(DISTINCT b, a)), udf(count(DISTINCT *)), udf(count(DISTINCT testData.*, testData.*))
 FROM testData
 -- !query schema
-struct<CAST(udf(cast(count(distinct a, b) as string)) AS BIGINT):bigint,CAST(udf(cast(count(distinct b, a) as string)) AS BIGINT):bigint,CAST(udf(cast(count(distinct a, b) as string)) AS BIGINT):bigint,CAST(udf(cast(count(distinct a, b) as string)) AS BIGINT):bigint>
+struct<udf(count(DISTINCT a, b)):bigint,udf(count(DISTINCT b, a)):bigint,udf(count(DISTINCT a, b)):bigint,udf(count(DISTINCT a, b, a, b)):bigint>
 -- !query output
 3	3	3	3
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-cross-join.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/udf-cross-join.sql.out
index fdddfc55978b4..3843363f5a75c 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/udf-cross-join.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-cross-join.sql.out
@@ -76,7 +76,7 @@ SELECT udf(a.key), udf(b.key) FROM
 CROSS JOIN
 (SELECT udf(k) key FROM nt2 WHERE v2 = 22) b
 -- !query schema
-struct<CAST(udf(cast(key as string)) AS STRING):string,CAST(udf(cast(key as string)) AS STRING):string>
+struct<udf(key):string,udf(key):string>
 -- !query output
 one	two
 
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-except-all.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/udf-except-all.sql.out
index 2613120e004df..ace56befb219f 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/udf-except-all.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-except-all.sql.out
@@ -53,7 +53,7 @@ SELECT udf(c1) FROM tab1
 EXCEPT ALL
 SELECT udf(c1) FROM tab2
 -- !query schema
-struct<CAST(udf(cast(c1 as string)) AS INT):int>
+struct<udf(c1):int>
 -- !query output
 0
 2
@@ -66,7 +66,7 @@ SELECT udf(c1) FROM tab1
 MINUS ALL
 SELECT udf(c1) FROM tab2
 -- !query schema
-struct<CAST(udf(cast(c1 as string)) AS INT):int>
+struct<udf(c1):int>
 -- !query output
 0
 2
@@ -79,7 +79,7 @@ SELECT udf(c1) FROM tab1
 EXCEPT ALL
 SELECT udf(c1) FROM tab2 WHERE udf(c1) IS NOT NULL
 -- !query schema
-struct<CAST(udf(cast(c1 as string)) AS INT):int>
+struct<udf(c1):int>
 -- !query output
 0
 2
@@ -93,7 +93,7 @@ SELECT udf(c1) FROM tab1 WHERE udf(c1) > 5
 EXCEPT ALL
 SELECT udf(c1) FROM tab2
 -- !query schema
-struct<CAST(udf(cast(c1 as string)) AS INT):int>
+struct<udf(c1):int>
 -- !query output
 
 
@@ -103,7 +103,7 @@ SELECT udf(c1) FROM tab1
 EXCEPT ALL
 SELECT udf(c1) FROM tab2 WHERE udf(c1 > udf(6))
 -- !query schema
-struct<CAST(udf(cast(c1 as string)) AS INT):int>
+struct<udf(c1):int>
 -- !query output
 0
 1
@@ -121,7 +121,7 @@ SELECT udf(c1) FROM tab1
 EXCEPT ALL
 SELECT CAST(udf(1) AS BIGINT)
 -- !query schema
-struct<CAST(udf(cast(c1 as string)) AS INT):bigint>
+struct<udf(c1):bigint>
 -- !query output
 0
 2
@@ -141,7 +141,7 @@ SELECT array(1)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-ExceptAll can only be performed on tables with the compatible column types. array<int> <> int at the first column of the second table;
+ExceptAll can only be performed on tables with the compatible column types. array<int> <> int at the first column of the second table
 
 
 -- !query
@@ -149,7 +149,7 @@ SELECT udf(k), v FROM tab3
 EXCEPT ALL
 SELECT k, udf(v) FROM tab4
 -- !query schema
-struct<CAST(udf(cast(k as string)) AS INT):int,v:int>
+struct<udf(k):int,v:int>
 -- !query output
 1	2
 1	3
@@ -160,7 +160,7 @@ SELECT k, udf(v) FROM tab4
 EXCEPT ALL
 SELECT udf(k), v FROM tab3
 -- !query schema
-struct<k:int,CAST(udf(cast(v as string)) AS INT):int>
+struct<k:int,udf(v):int>
 -- !query output
 2	2
 2	20
@@ -173,7 +173,7 @@ SELECT udf(k), udf(v) FROM tab3
 INTERSECT DISTINCT
 SELECT udf(k), udf(v) FROM tab4
 -- !query schema
-struct<CAST(udf(cast(k as string)) AS INT):int,CAST(udf(cast(v as string)) AS INT):int>
+struct<udf(k):int,udf(v):int>
 -- !query output
 2	2
 2	20
@@ -186,7 +186,7 @@ SELECT k, udf(v) FROM tab3
 EXCEPT DISTINCT
 SELECT udf(k), udf(v) FROM tab4
 -- !query schema
-struct<CAST(udf(cast(k as string)) AS INT):int,v:int>
+struct<udf(k):int,v:int>
 -- !query output
 
 
@@ -200,7 +200,7 @@ SELECT udf(k), v FROM tab3
 EXCEPT DISTINCT
 SELECT k, udf(v) FROM tab4
 -- !query schema
-struct<k:int,CAST(udf(cast(v as string)) AS INT):int>
+struct<k:int,udf(v):int>
 -- !query output
 1	3
 
@@ -213,7 +213,7 @@ SELECT k, v FROM tab4
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-ExceptAll can only be performed on tables with the same number of columns, but the first table has 1 columns and the second table has 2 columns;
+ExceptAll can only be performed on tables with the same number of columns, but the first table has 1 columns and the second table has 2 columns
 
 
 -- !query
@@ -225,7 +225,7 @@ SELECT udf(k), udf(v) FROM tab3
 EXCEPT DISTINCT
 SELECT udf(k), udf(v) FROM tab4
 -- !query schema
-struct<CAST(udf(cast(k as string)) AS INT):int,CAST(udf(cast(v as string)) AS INT):int>
+struct<udf(k):int,udf(v):int>
 -- !query output
 1	3
 
@@ -239,7 +239,7 @@ SELECT udf(k), udf(v) FROM tab3
 MINUS DISTINCT
 SELECT k, udf(v) FROM tab4
 -- !query schema
-struct<CAST(udf(cast(k as string)) AS INT):int,CAST(udf(cast(v as string)) AS INT):int>
+struct<udf(k):int,udf(v):int>
 -- !query output
 1	3
 
@@ -253,7 +253,7 @@ SELECT k, udf(v) FROM tab3
 EXCEPT DISTINCT
 SELECT udf(k), v FROM tab4
 -- !query schema
-struct<k:int,CAST(udf(cast(v as string)) AS INT):int>
+struct<k:int,udf(v):int>
 -- !query output
 
 
@@ -273,7 +273,7 @@ FROM   (SELECT udf(tab3.k),
                JOIN tab4 
                  ON tab3.k = udf(tab4.k))
 -- !query schema
-struct<k:int,CAST(udf(cast(v as string)) AS INT):int>
+struct<k:int,udf(v):int>
 -- !query output
 
 
@@ -293,7 +293,7 @@ FROM   (SELECT udf(tab4.v) AS k,
                JOIN tab4 
                  ON udf(tab3.k) = udf(tab4.k))
 -- !query schema
-struct<CAST(udf(cast(cast(udf(cast(k as string)) as int) as string)) AS INT):int,CAST(udf(cast(v as string)) AS INT):int>
+struct<udf(udf(k)):int,udf(v):int>
 -- !query output
 1	2
 1	2
@@ -309,7 +309,7 @@ SELECT udf(v) FROM tab3 GROUP BY v
 EXCEPT ALL
 SELECT udf(k) FROM tab4 GROUP BY k
 -- !query schema
-struct<CAST(udf(cast(v as string)) AS INT):int>
+struct<udf(v):int>
 -- !query output
 3
 
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-except.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/udf-except.sql.out
index 054ee00ecc2ae..7d21715fbaa8a 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/udf-except.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-except.sql.out
@@ -32,7 +32,7 @@ struct<>
 -- !query
 SELECT udf(k), udf(v) FROM t1 EXCEPT SELECT udf(k), udf(v) FROM t2
 -- !query schema
-struct<CAST(udf(cast(k as string)) AS STRING):string,CAST(udf(cast(v as string)) AS INT):int>
+struct<udf(k):string,udf(v):int>
 -- !query output
 three	3
 two	2
@@ -100,6 +100,15 @@ WHERE  udf(t1.v) >= (SELECT   min(udf(t2.v))
                 FROM     t2
                 WHERE    t2.k = t1.k)
 -- !query schema
-struct<k:string>
+struct<>
 -- !query output
-two
+org.apache.spark.sql.AnalysisException
+Correlated column is not allowed in predicate (CAST(udf(cast(k as string)) AS STRING) = CAST(udf(cast(outer(k#x) as string)) AS STRING)):
+Aggregate [cast(udf(cast(max(cast(udf(cast(v#x as string)) as int)) as string)) as int) AS udf(max(udf(v)))#x]
++- Filter (cast(udf(cast(k#x as string)) as string) = cast(udf(cast(outer(k#x) as string)) as string))
+   +- SubqueryAlias t2
+      +- View (`t2`, [k#x,v#x])
+         +- Project [cast(k#x as string) AS k#x, cast(v#x as int) AS v#x]
+            +- Project [k#x, v#x]
+               +- SubqueryAlias t2
+                  +- LocalRelation [k#x, v#x]
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-group-analytics.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/udf-group-analytics.sql.out
index f4cf4196298c1..7d2fad8cc2087 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/udf-group-analytics.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-group-analytics.sql.out
@@ -15,7 +15,7 @@ struct<>
 -- !query
 SELECT udf(a + b), b, udf(SUM(a - b)) FROM testData GROUP BY udf(a + b), b WITH CUBE
 -- !query schema
-struct<CAST(udf(cast((a + b) as string)) AS INT):int,b:int,CAST(udf(cast(sum(cast((a - b) as bigint)) as string)) AS BIGINT):bigint>
+struct<udf((a + b)):int,b:int,udf(sum((a - b))):bigint>
 -- !query output
 2	1	0
 2	NULL	0
@@ -35,7 +35,7 @@ NULL	NULL	3
 -- !query
 SELECT udf(a), udf(b), SUM(b) FROM testData GROUP BY udf(a), b WITH CUBE
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS INT):int,CAST(udf(cast(b as string)) AS INT):int,sum(b):bigint>
+struct<udf(a):int,udf(b):int,sum(b):bigint>
 -- !query output
 1	1	1
 1	2	2
@@ -54,7 +54,7 @@ NULL	NULL	9
 -- !query
 SELECT udf(a + b), b, SUM(a - b) FROM testData GROUP BY a + b, b WITH ROLLUP
 -- !query schema
-struct<CAST(udf(cast((a + b) as string)) AS INT):int,b:int,sum((a - b)):bigint>
+struct<udf((a + b)):int,b:int,sum((a - b)):bigint>
 -- !query output
 2	1	0
 2	NULL	0
@@ -72,7 +72,7 @@ NULL	NULL	3
 -- !query
 SELECT udf(a), b, udf(SUM(b)) FROM testData GROUP BY udf(a), b WITH ROLLUP
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS INT):int,b:int,CAST(udf(cast(sum(cast(b as bigint)) as string)) AS BIGINT):bigint>
+struct<udf(a):int,b:int,udf(sum(b)):bigint>
 -- !query output
 1	1	1
 1	2	2
@@ -129,7 +129,7 @@ dotNET	2013	48000
 -- !query
 SELECT course, udf(year), SUM(earnings) FROM courseSales GROUP BY course, year GROUPING SETS(course, year)
 -- !query schema
-struct<course:string,CAST(udf(cast(year as string)) AS INT):int,sum(earnings):bigint>
+struct<course:string,udf(year):int,sum(earnings):bigint>
 -- !query output
 Java	NULL	50000
 NULL	2012	35000
@@ -140,7 +140,7 @@ dotNET	NULL	63000
 -- !query
 SELECT course, year, udf(SUM(earnings)) FROM courseSales GROUP BY course, year GROUPING SETS(course)
 -- !query schema
-struct<course:string,year:int,CAST(udf(cast(sum(cast(earnings as bigint)) as string)) AS BIGINT):bigint>
+struct<course:string,year:int,udf(sum(earnings)):bigint>
 -- !query output
 Java	NULL	50000
 dotNET	NULL	63000
@@ -149,7 +149,7 @@ dotNET	NULL	63000
 -- !query
 SELECT udf(course), year, SUM(earnings) FROM courseSales GROUP BY course, year GROUPING SETS(year)
 -- !query schema
-struct<CAST(udf(cast(course as string)) AS STRING):string,year:int,sum(earnings):bigint>
+struct<udf(course):string,year:int,sum(earnings):bigint>
 -- !query output
 NULL	2012	35000
 NULL	2013	78000
@@ -191,7 +191,7 @@ dotNET	63000	1
 SELECT udf(course), udf(year), GROUPING(course), GROUPING(year), GROUPING_ID(course, year) FROM courseSales
 GROUP BY CUBE(course, year)
 -- !query schema
-struct<CAST(udf(cast(course as string)) AS STRING):string,CAST(udf(cast(year as string)) AS INT):int,grouping(course):tinyint,grouping(year):tinyint,grouping_id(course, year):bigint>
+struct<udf(course):string,udf(year):int,grouping(course):tinyint,grouping(year):tinyint,grouping_id(course, year):bigint>
 -- !query output
 Java	2012	0	0	0
 Java	2013	0	0	0
@@ -210,7 +210,7 @@ SELECT course, udf(year), GROUPING(course) FROM courseSales GROUP BY course, udf
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-grouping() can only be used with GroupingSets/Cube/Rollup;
+grouping() can only be used with GroupingSets/Cube/Rollup
 
 
 -- !query
@@ -219,7 +219,7 @@ SELECT course, udf(year), GROUPING_ID(course, year) FROM courseSales GROUP BY ud
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-grouping_id() can only be used with GroupingSets/Cube/Rollup;
+grouping_id() can only be used with GroupingSets/Cube/Rollup
 
 
 -- !query
@@ -255,7 +255,7 @@ SELECT course, udf(year) FROM courseSales GROUP BY udf(course), year HAVING GROU
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-grouping()/grouping_id() can only be used with GroupingSets/Cube/Rollup;
+grouping()/grouping_id() can only be used with GroupingSets/Cube/Rollup
 
 
 -- !query
@@ -264,13 +264,13 @@ SELECT course, udf(udf(year)) FROM courseSales GROUP BY course, year HAVING GROU
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-grouping()/grouping_id() can only be used with GroupingSets/Cube/Rollup;
+grouping()/grouping_id() can only be used with GroupingSets/Cube/Rollup
 
 
 -- !query
 SELECT udf(course), year FROM courseSales GROUP BY CUBE(course, year) HAVING grouping__id > 0
 -- !query schema
-struct<CAST(udf(cast(course as string)) AS STRING):string,year:int>
+struct<udf(course):string,year:int>
 -- !query output
 Java	NULL
 NULL	2012
@@ -319,7 +319,7 @@ SELECT course, udf(year) FROM courseSales GROUP BY course, udf(year) ORDER BY GR
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-grouping()/grouping_id() can only be used with GroupingSets/Cube/Rollup;
+grouping()/grouping_id() can only be used with GroupingSets/Cube/Rollup
 
 
 -- !query
@@ -328,7 +328,7 @@ SELECT course, udf(year) FROM courseSales GROUP BY course, udf(year) ORDER BY GR
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-grouping()/grouping_id() can only be used with GroupingSets/Cube/Rollup;
+grouping()/grouping_id() can only be used with GroupingSets/Cube/Rollup
 
 
 -- !query
@@ -388,7 +388,7 @@ NULL	NULL	3
 -- !query
 SELECT udf(a + b), udf(udf(b)) AS k, SUM(a - b) FROM testData GROUP BY a + b, k GROUPING SETS(k)
 -- !query schema
-struct<CAST(udf(cast((a + b) as string)) AS INT):int,k:int,sum((a - b)):bigint>
+struct<udf((a + b)):int,k:int,sum((a - b)):bigint>
 -- !query output
 NULL	1	3
 NULL	2	0
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-group-by.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/udf-group-by.sql.out
index 6403406413db9..07489dcad9a08 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/udf-group-by.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-group-by.sql.out
@@ -18,13 +18,13 @@ SELECT udf(a), udf(COUNT(b)) FROM testData
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-grouping expressions sequence is empty, and 'testdata.`a`' is not an aggregate function. Wrap '(CAST(udf(cast(count(b) as string)) AS BIGINT) AS `CAST(udf(cast(count(b) as string)) AS BIGINT)`)' in windowing function(s) or wrap 'testdata.`a`' in first() (or first_value) if you don't care which value you get.;
+grouping expressions sequence is empty, and 'testdata.a' is not an aggregate function. Wrap '(CAST(udf(cast(count(b) as string)) AS BIGINT) AS `udf(count(b))`)' in windowing function(s) or wrap 'testdata.a' in first() (or first_value) if you don't care which value you get.
 
 
 -- !query
 SELECT COUNT(udf(a)), udf(COUNT(b)) FROM testData
 -- !query schema
-struct<count(CAST(udf(cast(a as string)) AS INT)):bigint,CAST(udf(cast(count(b) as string)) AS BIGINT):bigint>
+struct<count(udf(a)):bigint,udf(count(b)):bigint>
 -- !query output
 7	7
 
@@ -32,7 +32,7 @@ struct<count(CAST(udf(cast(a as string)) AS INT)):bigint,CAST(udf(cast(count(b)
 -- !query
 SELECT udf(a), COUNT(udf(b)) FROM testData GROUP BY a
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS INT):int,count(CAST(udf(cast(b as string)) AS INT)):bigint>
+struct<udf(a):int,count(udf(b)):bigint>
 -- !query output
 1	2
 2	2
@@ -46,13 +46,13 @@ SELECT udf(a), udf(COUNT(udf(b))) FROM testData GROUP BY b
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-expression 'testdata.`a`' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.;
+expression 'testdata.a' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.
 
 
 -- !query
 SELECT COUNT(udf(a)), COUNT(udf(b)) FROM testData GROUP BY udf(a)
 -- !query schema
-struct<count(CAST(udf(cast(a as string)) AS INT)):bigint,count(CAST(udf(cast(b as string)) AS INT)):bigint>
+struct<count(udf(a)):bigint,count(udf(b)):bigint>
 -- !query output
 0	1
 2	2
@@ -63,7 +63,7 @@ struct<count(CAST(udf(cast(a as string)) AS INT)):bigint,count(CAST(udf(cast(b a
 -- !query
 SELECT 'foo', COUNT(udf(a)) FROM testData GROUP BY 1
 -- !query schema
-struct<foo:string,count(CAST(udf(cast(a as string)) AS INT)):bigint>
+struct<foo:string,count(udf(a)):bigint>
 -- !query output
 foo	7
 
@@ -79,7 +79,7 @@ struct<foo:string>
 -- !query
 SELECT 'foo', udf(APPROX_COUNT_DISTINCT(udf(a))) FROM testData WHERE a = 0 GROUP BY udf(1)
 -- !query schema
-struct<foo:string,CAST(udf(cast(approx_count_distinct(cast(udf(cast(a as string)) as int), 0.05, 0, 0) as string)) AS BIGINT):bigint>
+struct<foo:string,udf(approx_count_distinct(udf(a))):bigint>
 -- !query output
 
 
@@ -87,7 +87,7 @@ struct<foo:string,CAST(udf(cast(approx_count_distinct(cast(udf(cast(a as string)
 -- !query
 SELECT 'foo', MAX(STRUCT(udf(a))) FROM testData WHERE a = 0 GROUP BY udf(1)
 -- !query schema
-struct<foo:string,max(named_struct(col1, CAST(udf(cast(a as string)) AS INT))):struct<col1:int>>
+struct<foo:string,max(struct(udf(a))):struct<col1:int>>
 -- !query output
 
 
@@ -95,7 +95,7 @@ struct<foo:string,max(named_struct(col1, CAST(udf(cast(a as string)) AS INT))):s
 -- !query
 SELECT udf(a + b), udf(COUNT(b)) FROM testData GROUP BY a + b
 -- !query schema
-struct<CAST(udf(cast((a + b) as string)) AS INT):int,CAST(udf(cast(count(b) as string)) AS BIGINT):bigint>
+struct<udf((a + b)):int,udf(count(b)):bigint>
 -- !query output
 2	1
 3	2
@@ -110,13 +110,13 @@ SELECT udf(a + 2), udf(COUNT(b)) FROM testData GROUP BY a + 1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-expression 'testdata.`a`' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.;
+expression 'testdata.a' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.
 
 
 -- !query
 SELECT udf(a + 1) + 1, udf(COUNT(b)) FROM testData GROUP BY udf(a + 1)
 -- !query schema
-struct<(CAST(udf(cast((a + 1) as string)) AS INT) + 1):int,CAST(udf(cast(count(b) as string)) AS BIGINT):bigint>
+struct<(udf((a + 1)) + 1):int,udf(count(b)):bigint>
 -- !query output
 3	2
 4	2
@@ -128,7 +128,7 @@ NULL	1
 SELECT SKEWNESS(udf(a)), udf(KURTOSIS(a)), udf(MIN(a)), MAX(udf(a)), udf(AVG(udf(a))), udf(VARIANCE(a)), STDDEV(udf(a)), udf(SUM(a)), udf(COUNT(a))
 FROM testData
 -- !query schema
-struct<skewness(CAST(CAST(udf(cast(a as string)) AS INT) AS DOUBLE)):double,CAST(udf(cast(kurtosis(cast(a as double)) as string)) AS DOUBLE):double,CAST(udf(cast(min(a) as string)) AS INT):int,max(CAST(udf(cast(a as string)) AS INT)):int,CAST(udf(cast(avg(cast(cast(udf(cast(a as string)) as int) as bigint)) as string)) AS DOUBLE):double,CAST(udf(cast(variance(cast(a as double)) as string)) AS DOUBLE):double,stddev(CAST(CAST(udf(cast(a as string)) AS INT) AS DOUBLE)):double,CAST(udf(cast(sum(cast(a as bigint)) as string)) AS BIGINT):bigint,CAST(udf(cast(count(a) as string)) AS BIGINT):bigint>
+struct<skewness(udf(a)):double,udf(kurtosis(a)):double,udf(min(a)):int,max(udf(a)):int,udf(avg(udf(a))):double,udf(variance(a)):double,stddev(udf(a)):double,udf(sum(a)):bigint,udf(count(a)):bigint>
 -- !query output
 -0.2723801058145729	-1.5069204152249134	1	3	2.142857142857143	0.8095238095238094	0.8997354108424372	15	7
 
@@ -136,7 +136,7 @@ struct<skewness(CAST(CAST(udf(cast(a as string)) AS INT) AS DOUBLE)):double,CAST
 -- !query
 SELECT COUNT(DISTINCT udf(b)), udf(COUNT(DISTINCT b, c)) FROM (SELECT 1 AS a, 2 AS b, 3 AS c) GROUP BY udf(a)
 -- !query schema
-struct<count(DISTINCT CAST(udf(cast(b as string)) AS INT)):bigint,CAST(udf(cast(count(distinct b, c) as string)) AS BIGINT):bigint>
+struct<count(DISTINCT udf(b)):bigint,udf(count(DISTINCT b, c)):bigint>
 -- !query output
 1	1
 
@@ -144,7 +144,7 @@ struct<count(DISTINCT CAST(udf(cast(b as string)) AS INT)):bigint,CAST(udf(cast(
 -- !query
 SELECT udf(a) AS k, COUNT(udf(b)) FROM testData GROUP BY k
 -- !query schema
-struct<k:int,count(CAST(udf(cast(b as string)) AS INT)):bigint>
+struct<k:int,count(udf(b)):bigint>
 -- !query output
 1	2
 2	2
@@ -155,7 +155,7 @@ NULL	1
 -- !query
 SELECT a AS k, udf(COUNT(b)) FROM testData GROUP BY k HAVING k > 1
 -- !query schema
-struct<k:int,CAST(udf(cast(count(b) as string)) AS BIGINT):bigint>
+struct<k:int,udf(count(b)):bigint>
 -- !query output
 2	2
 3	2
@@ -167,7 +167,7 @@ SELECT udf(COUNT(b)) AS k FROM testData GROUP BY k
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-aggregate functions are not allowed in GROUP BY, but found CAST(udf(cast(count(b) as string)) AS BIGINT);
+aggregate functions are not allowed in GROUP BY, but found CAST(udf(cast(count(b) as string)) AS BIGINT)
 
 
 -- !query
@@ -185,7 +185,7 @@ SELECT k AS a, udf(COUNT(udf(v))) FROM testDataHasSameNameWithAlias GROUP BY udf
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-expression 'testdatahassamenamewithalias.`k`' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.;
+expression 'testdatahassamenamewithalias.k' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.
 
 
 -- !query
@@ -202,13 +202,13 @@ SELECT a AS k, udf(COUNT(udf(b))) FROM testData GROUP BY k
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`k`' given input columns: [testdata.a, testdata.b]; line 1 pos 57
+cannot resolve 'k' given input columns: [testdata.a, testdata.b]; line 1 pos 57
 
 
 -- !query
 SELECT udf(a), COUNT(udf(1)) FROM testData WHERE false GROUP BY udf(a)
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS INT):int,count(CAST(udf(cast(1 as string)) AS INT)):bigint>
+struct<udf(a):int,count(udf(1)):bigint>
 -- !query output
 
 
@@ -216,7 +216,7 @@ struct<CAST(udf(cast(a as string)) AS INT):int,count(CAST(udf(cast(1 as string))
 -- !query
 SELECT udf(COUNT(1)) FROM testData WHERE false
 -- !query schema
-struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<udf(count(1)):bigint>
 -- !query output
 0
 
@@ -247,7 +247,7 @@ struct<1:int>
 SELECT corr(DISTINCT x, y), udf(corr(DISTINCT y, x)), count(*)
   FROM (VALUES (1, 1), (2, 2), (2, 2)) t(x, y)
 -- !query schema
-struct<corr(DISTINCT CAST(x AS DOUBLE), CAST(y AS DOUBLE)):double,CAST(udf(cast(corr(distinct cast(y as double), cast(x as double)) as string)) AS DOUBLE):double,count(1):bigint>
+struct<corr(DISTINCT x, y):double,udf(corr(DISTINCT y, x)):double,count(1):bigint>
 -- !query output
 1.0	1.0	3
 
@@ -255,7 +255,7 @@ struct<corr(DISTINCT CAST(x AS DOUBLE), CAST(y AS DOUBLE)):double,CAST(udf(cast(
 -- !query
 SELECT udf(1) FROM range(10) HAVING true
 -- !query schema
-struct<CAST(udf(cast(1 as string)) AS INT):int>
+struct<udf(1):int>
 -- !query output
 1
 
@@ -263,7 +263,7 @@ struct<CAST(udf(cast(1 as string)) AS INT):int>
 -- !query
 SELECT udf(udf(1)) FROM range(10) HAVING MAX(id) > 0
 -- !query schema
-struct<CAST(udf(cast(cast(udf(cast(1 as string)) as int) as string)) AS INT):int>
+struct<udf(udf(1)):int>
 -- !query output
 1
 
@@ -274,7 +274,7 @@ SELECT udf(id) FROM range(10) HAVING id > 0
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-grouping expressions sequence is empty, and '`id`' is not an aggregate function. Wrap '()' in windowing function(s) or wrap '`id`' in first() (or first_value) if you don't care which value you get.;
+grouping expressions sequence is empty, and 'id' is not an aggregate function. Wrap '()' in windowing function(s) or wrap 'id' in first() (or first_value) if you don't care which value you get.
 
 
 -- !query
@@ -293,7 +293,7 @@ struct<>
 -- !query
 SELECT udf(every(v)), udf(some(v)), any(v) FROM test_agg WHERE 1 = 0
 -- !query schema
-struct<CAST(udf(cast(every(v) as string)) AS BOOLEAN):boolean,CAST(udf(cast(some(v) as string)) AS BOOLEAN):boolean,any(v):boolean>
+struct<udf(every(v)):boolean,udf(some(v)):boolean,any(v):boolean>
 -- !query output
 NULL	NULL	NULL
 
@@ -301,7 +301,7 @@ NULL	NULL	NULL
 -- !query
 SELECT udf(every(udf(v))), some(v), any(v) FROM test_agg WHERE k = 4
 -- !query schema
-struct<CAST(udf(cast(every(cast(udf(cast(v as string)) as boolean)) as string)) AS BOOLEAN):boolean,some(v):boolean,any(v):boolean>
+struct<udf(every(udf(v))):boolean,some(v):boolean,any(v):boolean>
 -- !query output
 NULL	NULL	NULL
 
@@ -309,7 +309,7 @@ NULL	NULL	NULL
 -- !query
 SELECT every(v), udf(some(v)), any(v) FROM test_agg WHERE k = 5
 -- !query schema
-struct<every(v):boolean,CAST(udf(cast(some(v) as string)) AS BOOLEAN):boolean,any(v):boolean>
+struct<every(v):boolean,udf(some(v)):boolean,any(v):boolean>
 -- !query output
 false	true	true
 
@@ -317,7 +317,7 @@ false	true	true
 -- !query
 SELECT udf(k), every(v), udf(some(v)), any(v) FROM test_agg GROUP BY udf(k)
 -- !query schema
-struct<CAST(udf(cast(k as string)) AS INT):int,every(v):boolean,CAST(udf(cast(some(v) as string)) AS BOOLEAN):boolean,any(v):boolean>
+struct<udf(k):int,every(v):boolean,udf(some(v)):boolean,any(v):boolean>
 -- !query output
 1	false	true	true
 2	true	true	true
@@ -329,7 +329,7 @@ struct<CAST(udf(cast(k as string)) AS INT):int,every(v):boolean,CAST(udf(cast(so
 -- !query
 SELECT udf(k), every(v) FROM test_agg GROUP BY k HAVING every(v) = false
 -- !query schema
-struct<CAST(udf(cast(k as string)) AS INT):int,every(v):boolean>
+struct<udf(k):int,every(v):boolean>
 -- !query output
 1	false
 3	false
@@ -339,7 +339,7 @@ struct<CAST(udf(cast(k as string)) AS INT):int,every(v):boolean>
 -- !query
 SELECT udf(k), udf(every(v)) FROM test_agg GROUP BY udf(k) HAVING every(v) IS NULL
 -- !query schema
-struct<CAST(udf(cast(k as string)) AS INT):int,CAST(udf(cast(every(v) as string)) AS BOOLEAN):boolean>
+struct<udf(k):int,udf(every(v)):boolean>
 -- !query output
 4	NULL
 
@@ -354,7 +354,7 @@ WHERE  k = 2
                  WHERE  k = 1)
 GROUP  BY udf(k)
 -- !query schema
-struct<CAST(udf(cast(k as string)) AS INT):int,every:boolean>
+struct<udf(k):int,every:boolean>
 -- !query output
 2	true
 
@@ -369,7 +369,7 @@ WHERE  k = 2
                  WHERE  k = 1)
 GROUP  BY udf(udf(k))
 -- !query schema
-struct<CAST(udf(cast(cast(udf(cast(k as string)) as int) as string)) AS INT):int,every:boolean>
+struct<udf(udf(k)):int,every:boolean>
 -- !query output
 
 
@@ -430,7 +430,7 @@ struct<k:int,v:boolean,every(v) OVER (PARTITION BY k ORDER BY v ASC NULLS FIRST
 -- !query
 SELECT k, udf(udf(v)), some(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg
 -- !query schema
-struct<k:int,CAST(udf(cast(cast(udf(cast(v as string)) as boolean) as string)) AS BOOLEAN):boolean,some(v) OVER (PARTITION BY k ORDER BY v ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):boolean>
+struct<k:int,udf(udf(v)):boolean,some(v) OVER (PARTITION BY k ORDER BY v ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):boolean>
 -- !query output
 1	false	false
 1	true	true
@@ -447,7 +447,7 @@ struct<k:int,CAST(udf(cast(cast(udf(cast(v as string)) as boolean) as string)) A
 -- !query
 SELECT udf(udf(k)), v, any(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg
 -- !query schema
-struct<CAST(udf(cast(cast(udf(cast(k as string)) as int) as string)) AS INT):int,v:boolean,any(v) OVER (PARTITION BY k ORDER BY v ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):boolean>
+struct<udf(udf(k)):int,v:boolean,any(v) OVER (PARTITION BY k ORDER BY v ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):boolean>
 -- !query output
 1	false	false
 1	true	true
@@ -464,7 +464,7 @@ struct<CAST(udf(cast(cast(udf(cast(k as string)) as int) as string)) AS INT):int
 -- !query
 SELECT udf(count(*)) FROM test_agg HAVING count(*) > 1L
 -- !query schema
-struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<udf(count(1)):bigint>
 -- !query output
 10
 
@@ -472,7 +472,7 @@ struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
 -- !query
 SELECT k, udf(max(v)) FROM test_agg GROUP BY k HAVING max(v) = true
 -- !query schema
-struct<k:int,CAST(udf(cast(max(v) as string)) AS BOOLEAN):boolean>
+struct<k:int,udf(max(v)):boolean>
 -- !query output
 1	true
 2	true
@@ -496,7 +496,7 @@ org.apache.spark.sql.AnalysisException
 
 Aggregate/Window/Generate expressions are not valid in where clause of the query.
 Expression in where clause: [(count(1) > 1L)]
-Invalid expressions: [count(1)];
+Invalid expressions: [count(1)]
 
 
 -- !query
@@ -508,7 +508,7 @@ org.apache.spark.sql.AnalysisException
 
 Aggregate/Window/Generate expressions are not valid in where clause of the query.
 Expression in where clause: [((count(1) + 1L) > 1L)]
-Invalid expressions: [count(1)];
+Invalid expressions: [count(1)]
 
 
 -- !query
@@ -519,5 +519,5 @@ struct<>
 org.apache.spark.sql.AnalysisException
 
 Aggregate/Window/Generate expressions are not valid in where clause of the query.
-Expression in where clause: [(((test_agg.`k` = 1) OR (test_agg.`k` = 2)) OR (((count(1) + 1L) > 1L) OR (max(test_agg.`k`) > 1)))]
-Invalid expressions: [count(1), max(test_agg.`k`)];
+Expression in where clause: [(((test_agg.k = 1) OR (test_agg.k = 2)) OR (((count(1) + 1L) > 1L) OR (max(test_agg.k) > 1)))]
+Invalid expressions: [count(1), max(test_agg.k)]
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-having.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/udf-having.sql.out
index 9be27bb77f81a..fb0a5b04dc1d7 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/udf-having.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-having.sql.out
@@ -18,7 +18,7 @@ struct<>
 -- !query
 SELECT udf(k) AS k, udf(sum(v)) FROM hav GROUP BY k HAVING udf(sum(v)) > 2
 -- !query schema
-struct<k:string,CAST(udf(cast(sum(cast(v as bigint)) as string)) AS BIGINT):bigint>
+struct<k:string,udf(sum(v)):bigint>
 -- !query output
 one	6
 three	3
@@ -27,7 +27,7 @@ three	3
 -- !query
 SELECT udf(count(udf(k))) FROM hav GROUP BY v + 1 HAVING v + 1 = udf(2)
 -- !query schema
-struct<CAST(udf(cast(count(cast(udf(cast(k as string)) as string)) as string)) AS BIGINT):bigint>
+struct<udf(count(udf(k))):bigint>
 -- !query output
 1
 
@@ -35,7 +35,7 @@ struct<CAST(udf(cast(count(cast(udf(cast(k as string)) as string)) as string)) A
 -- !query
 SELECT udf(MIN(t.v)) FROM (SELECT * FROM hav WHERE v > 0) t HAVING(udf(COUNT(udf(1))) > 0)
 -- !query schema
-struct<CAST(udf(cast(min(v) as string)) AS INT):int>
+struct<udf(min(v)):int>
 -- !query output
 1
 
@@ -43,7 +43,7 @@ struct<CAST(udf(cast(min(v) as string)) AS INT):int>
 -- !query
 SELECT udf(a + b) FROM VALUES (1L, 2), (3L, 4) AS T(a, b) GROUP BY a + b HAVING a + b > udf(1)
 -- !query schema
-struct<CAST(udf(cast((a + cast(b as bigint)) as string)) AS BIGINT):bigint>
+struct<udf((a + b)):bigint>
 -- !query output
 3
 7
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-inline-table.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/udf-inline-table.sql.out
index d78d347bc9802..78e9190820297 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/udf-inline-table.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-inline-table.sql.out
@@ -5,7 +5,7 @@
 -- !query
 select udf(col1), udf(col2) from values ("one", 1)
 -- !query schema
-struct<CAST(udf(cast(col1 as string)) AS STRING):string,CAST(udf(cast(col2 as string)) AS INT):int>
+struct<udf(col1):string,udf(col2):int>
 -- !query output
 one	1
 
@@ -13,7 +13,7 @@ one	1
 -- !query
 select udf(col1), udf(udf(col2)) from values ("one", 1) as data
 -- !query schema
-struct<CAST(udf(cast(col1 as string)) AS STRING):string,CAST(udf(cast(cast(udf(cast(col2 as string)) as int) as string)) AS INT):int>
+struct<udf(col1):string,udf(udf(col2)):int>
 -- !query output
 one	1
 
@@ -21,7 +21,7 @@ one	1
 -- !query
 select udf(a), b from values ("one", 1) as data(a, b)
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS STRING):string,b:int>
+struct<udf(a):string,b:int>
 -- !query output
 one	1
 
@@ -29,7 +29,7 @@ one	1
 -- !query
 select udf(a) from values 1, 2, 3 as data(a)
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS INT):int>
+struct<udf(a):int>
 -- !query output
 1
 2
@@ -39,7 +39,7 @@ struct<CAST(udf(cast(a as string)) AS INT):int>
 -- !query
 select udf(a), b from values ("one", 1), ("two", 2), ("three", null) as data(a, b)
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS STRING):string,b:int>
+struct<udf(a):string,b:int>
 -- !query output
 one	1
 three	NULL
@@ -49,7 +49,7 @@ two	2
 -- !query
 select udf(a), b from values ("one", null), ("two", null) as data(a, b)
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS STRING):string,b:null>
+struct<udf(a):string,b:null>
 -- !query output
 one	NULL
 two	NULL
@@ -58,7 +58,7 @@ two	NULL
 -- !query
 select udf(a), b from values ("one", 1), ("two", 2L) as data(a, b)
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS STRING):string,b:bigint>
+struct<udf(a):string,b:bigint>
 -- !query output
 one	1
 two	2
@@ -67,7 +67,7 @@ two	2
 -- !query
 select udf(udf(a)), udf(b) from values ("one", 1 + 0), ("two", 1 + 3L) as data(a, b)
 -- !query schema
-struct<CAST(udf(cast(cast(udf(cast(a as string)) as string) as string)) AS STRING):string,CAST(udf(cast(b as string)) AS BIGINT):bigint>
+struct<udf(udf(a)):string,udf(b):bigint>
 -- !query output
 one	1
 two	4
@@ -76,7 +76,7 @@ two	4
 -- !query
 select udf(a), b from values ("one", array(0, 1)), ("two", array(2, 3)) as data(a, b)
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS STRING):string,b:array<int>>
+struct<udf(a):string,b:array<int>>
 -- !query output
 one	[0,1]
 two	[2,3]
@@ -85,7 +85,7 @@ two	[2,3]
 -- !query
 select udf(a), b from values ("one", 2.0), ("two", 3.0D) as data(a, b)
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS STRING):string,b:double>
+struct<udf(a):string,b:double>
 -- !query output
 one	2.0
 two	3.0
@@ -148,6 +148,6 @@ cannot evaluate expression count(1) in inline table definition; line 1 pos 42
 -- !query
 select udf(a), b from values (timestamp('1991-12-06 00:00:00.0'), array(timestamp('1991-12-06 01:00:00.0'), timestamp('1991-12-06 12:00:00.0'))) as data(a, b)
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS TIMESTAMP):timestamp,b:array<timestamp>>
+struct<udf(a):timestamp,b:array<timestamp>>
 -- !query output
 1991-12-06 00:00:00	[1991-12-06 01:00:00,1991-12-06 12:00:00]
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-intersect-all.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/udf-intersect-all.sql.out
index b3735ae153267..f03e6c061a0e1 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/udf-intersect-all.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-intersect-all.sql.out
@@ -38,7 +38,7 @@ SELECT udf(k), v FROM tab1
 INTERSECT ALL
 SELECT k, udf(v) FROM tab2
 -- !query schema
-struct<CAST(udf(cast(k as string)) AS INT):int,v:int>
+struct<udf(k):int,v:int>
 -- !query output
 1	2
 1	2
@@ -52,7 +52,7 @@ SELECT k, udf(v) FROM tab1
 INTERSECT ALL
 SELECT udf(k), v FROM tab1 WHERE udf(k) = 1
 -- !query schema
-struct<k:int,CAST(udf(cast(v as string)) AS INT):int>
+struct<k:int,udf(v):int>
 -- !query output
 1	2
 1	2
@@ -65,7 +65,7 @@ SELECT udf(k), udf(v) FROM tab1 WHERE k > udf(2)
 INTERSECT ALL
 SELECT udf(k), udf(v) FROM tab2
 -- !query schema
-struct<CAST(udf(cast(k as string)) AS INT):int,CAST(udf(cast(v as string)) AS INT):int>
+struct<udf(k):int,udf(v):int>
 -- !query output
 
 
@@ -75,7 +75,7 @@ SELECT udf(k), v FROM tab1
 INTERSECT ALL
 SELECT udf(k), v FROM tab2 WHERE udf(udf(k)) > 3
 -- !query schema
-struct<CAST(udf(cast(k as string)) AS INT):int,v:int>
+struct<udf(k):int,v:int>
 -- !query output
 
 
@@ -85,7 +85,7 @@ SELECT udf(k), v FROM tab1
 INTERSECT ALL
 SELECT CAST(udf(1) AS BIGINT), CAST(udf(2) AS BIGINT)
 -- !query schema
-struct<CAST(udf(cast(k as string)) AS INT):bigint,v:bigint>
+struct<udf(k):bigint,v:bigint>
 -- !query output
 1	2
 
@@ -98,7 +98,7 @@ SELECT array(1), udf(2)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-IntersectAll can only be performed on tables with the compatible column types. array<int> <> int at the first column of the second table;
+IntersectAll can only be performed on tables with the compatible column types. array<int> <> int at the first column of the second table
 
 
 -- !query
@@ -109,7 +109,7 @@ SELECT udf(k), udf(v) FROM tab2
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-IntersectAll can only be performed on tables with the same number of columns, but the first table has 1 columns and the second table has 2 columns;
+IntersectAll can only be performed on tables with the same number of columns, but the first table has 1 columns and the second table has 2 columns
 
 
 -- !query
@@ -119,7 +119,7 @@ SELECT k, udf(v) FROM tab1
 INTERSECT ALL
 SELECT udf(k), udf(v) FROM tab2
 -- !query schema
-struct<CAST(udf(cast(k as string)) AS INT):int,v:int>
+struct<udf(k):int,v:int>
 -- !query output
 1	2
 1	2
@@ -137,7 +137,7 @@ SELECT k, udf(udf(v)) FROM tab1
 INTERSECT ALL
 SELECT udf(k), v FROM tab2
 -- !query schema
-struct<CAST(udf(cast(k as string)) AS INT):int,v:int>
+struct<udf(k):int,v:int>
 -- !query output
 1	2
 1	2
@@ -156,7 +156,7 @@ SELECT k, udf(v) FROM tab1
 INTERSECT ALL
 SELECT udf(k), udf(udf(v)) FROM tab2
 -- !query schema
-struct<CAST(udf(cast(k as string)) AS INT):int,CAST(udf(cast(v as string)) AS INT):int>
+struct<udf(k):int,udf(v):int>
 -- !query output
 1	3
 
@@ -176,7 +176,7 @@ struct<CAST(udf(cast(k as string)) AS INT):int,CAST(udf(cast(v as string)) AS IN
   SELECT udf(k), udf(v) FROM tab2
 )
 -- !query schema
-struct<CAST(udf(cast(k as string)) AS INT):int,v:int>
+struct<udf(k):int,v:int>
 -- !query output
 
 
@@ -196,7 +196,7 @@ FROM   (SELECT udf(tab1.k),
                JOIN tab2 
                  ON udf(tab1.k) = udf(udf(tab2.k)))
 -- !query schema
-struct<CAST(udf(cast(k as string)) AS INT):int,CAST(udf(cast(v as string)) AS INT):int>
+struct<udf(k):int,udf(v):int>
 -- !query output
 1	2
 1	2
@@ -224,7 +224,7 @@ FROM   (SELECT udf(tab2.v) AS k,
                JOIN tab2 
                  ON tab1.k = udf(tab2.k))
 -- !query schema
-struct<CAST(udf(cast(k as string)) AS INT):int,CAST(udf(cast(v as string)) AS INT):int>
+struct<udf(k):int,udf(v):int>
 -- !query output
 
 
@@ -234,7 +234,7 @@ SELECT udf(v) FROM tab1 GROUP BY v
 INTERSECT ALL
 SELECT udf(udf(k)) FROM tab2 GROUP BY k
 -- !query schema
-struct<CAST(udf(cast(v as string)) AS INT):int>
+struct<udf(v):int>
 -- !query output
 2
 3
@@ -258,7 +258,7 @@ SELECT udf(k), udf(v) FROM tab1
 INTERSECT ALL
 SELECT udf(udf(k)), udf(v) FROM tab2
 -- !query schema
-struct<CAST(udf(cast(k as string)) AS INT):int,v:int>
+struct<udf(k):int,v:int>
 -- !query output
 1	2
 1	2
@@ -276,7 +276,7 @@ SELECT udf(k), udf(v) FROM tab1
 INTERSECT
 SELECT udf(k), udf(udf(v)) FROM tab2
 -- !query schema
-struct<k:int,CAST(udf(cast(v as string)) AS INT):int>
+struct<k:int,udf(v):int>
 -- !query output
 1	2
 2	3
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-join-empty-relation.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/udf-join-empty-relation.sql.out
index 0802eb9a9f62b..94ac49588e3de 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/udf-join-empty-relation.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-join-empty-relation.sql.out
@@ -29,7 +29,7 @@ struct<>
 -- !query
 SELECT udf(t1.a), udf(empty_table.a) FROM t1 INNER JOIN empty_table ON (udf(t1.a) = udf(udf(empty_table.a)))
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS INT):int,CAST(udf(cast(a as string)) AS INT):int>
+struct<udf(a):int,udf(a):int>
 -- !query output
 
 
@@ -37,7 +37,7 @@ struct<CAST(udf(cast(a as string)) AS INT):int,CAST(udf(cast(a as string)) AS IN
 -- !query
 SELECT udf(t1.a), udf(udf(empty_table.a)) FROM t1 CROSS JOIN empty_table ON (udf(udf(t1.a)) = udf(empty_table.a))
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS INT):int,CAST(udf(cast(cast(udf(cast(a as string)) as int) as string)) AS INT):int>
+struct<udf(a):int,udf(udf(a)):int>
 -- !query output
 
 
@@ -45,7 +45,7 @@ struct<CAST(udf(cast(a as string)) AS INT):int,CAST(udf(cast(cast(udf(cast(a as
 -- !query
 SELECT udf(udf(t1.a)), empty_table.a FROM t1 LEFT OUTER JOIN empty_table ON (udf(t1.a) = udf(empty_table.a))
 -- !query schema
-struct<CAST(udf(cast(cast(udf(cast(a as string)) as int) as string)) AS INT):int,a:int>
+struct<udf(udf(a)):int,a:int>
 -- !query output
 1	NULL
 
@@ -53,7 +53,7 @@ struct<CAST(udf(cast(cast(udf(cast(a as string)) as int) as string)) AS INT):int
 -- !query
 SELECT udf(t1.a), udf(empty_table.a) FROM t1 RIGHT OUTER JOIN empty_table ON (udf(t1.a) = udf(empty_table.a))
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS INT):int,CAST(udf(cast(a as string)) AS INT):int>
+struct<udf(a):int,udf(a):int>
 -- !query output
 
 
@@ -61,7 +61,7 @@ struct<CAST(udf(cast(a as string)) AS INT):int,CAST(udf(cast(a as string)) AS IN
 -- !query
 SELECT udf(t1.a), empty_table.a FROM t1 FULL OUTER JOIN empty_table ON (udf(t1.a) = udf(empty_table.a))
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS INT):int,a:int>
+struct<udf(a):int,a:int>
 -- !query output
 1	NULL
 
@@ -69,7 +69,7 @@ struct<CAST(udf(cast(a as string)) AS INT):int,a:int>
 -- !query
 SELECT udf(udf(t1.a)) FROM t1 LEFT SEMI JOIN empty_table ON (udf(t1.a) = udf(udf(empty_table.a)))
 -- !query schema
-struct<CAST(udf(cast(cast(udf(cast(a as string)) as int) as string)) AS INT):int>
+struct<udf(udf(a)):int>
 -- !query output
 
 
@@ -77,7 +77,7 @@ struct<CAST(udf(cast(cast(udf(cast(a as string)) as int) as string)) AS INT):int
 -- !query
 SELECT udf(t1.a) FROM t1 LEFT ANTI JOIN empty_table ON (udf(t1.a) = udf(empty_table.a))
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS INT):int>
+struct<udf(a):int>
 -- !query output
 1
 
@@ -85,7 +85,7 @@ struct<CAST(udf(cast(a as string)) AS INT):int>
 -- !query
 SELECT udf(empty_table.a), udf(t1.a) FROM empty_table INNER JOIN t1 ON (udf(udf(empty_table.a)) = udf(t1.a))
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS INT):int,CAST(udf(cast(a as string)) AS INT):int>
+struct<udf(a):int,udf(a):int>
 -- !query output
 
 
@@ -93,7 +93,7 @@ struct<CAST(udf(cast(a as string)) AS INT):int,CAST(udf(cast(a as string)) AS IN
 -- !query
 SELECT udf(empty_table.a), udf(udf(t1.a)) FROM empty_table CROSS JOIN t1 ON (udf(empty_table.a) = udf(udf(t1.a)))
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS INT):int,CAST(udf(cast(cast(udf(cast(a as string)) as int) as string)) AS INT):int>
+struct<udf(a):int,udf(udf(a)):int>
 -- !query output
 
 
@@ -101,7 +101,7 @@ struct<CAST(udf(cast(a as string)) AS INT):int,CAST(udf(cast(cast(udf(cast(a as
 -- !query
 SELECT udf(udf(empty_table.a)), udf(t1.a) FROM empty_table LEFT OUTER JOIN t1 ON (udf(empty_table.a) = udf(t1.a))
 -- !query schema
-struct<CAST(udf(cast(cast(udf(cast(a as string)) as int) as string)) AS INT):int,CAST(udf(cast(a as string)) AS INT):int>
+struct<udf(udf(a)):int,udf(a):int>
 -- !query output
 
 
@@ -109,7 +109,7 @@ struct<CAST(udf(cast(cast(udf(cast(a as string)) as int) as string)) AS INT):int
 -- !query
 SELECT empty_table.a, udf(t1.a) FROM empty_table RIGHT OUTER JOIN t1 ON (udf(empty_table.a) = udf(t1.a))
 -- !query schema
-struct<a:int,CAST(udf(cast(a as string)) AS INT):int>
+struct<a:int,udf(a):int>
 -- !query output
 NULL	1
 
@@ -117,7 +117,7 @@ NULL	1
 -- !query
 SELECT empty_table.a, udf(udf(t1.a)) FROM empty_table FULL OUTER JOIN t1 ON (udf(empty_table.a) = udf(t1.a))
 -- !query schema
-struct<a:int,CAST(udf(cast(cast(udf(cast(a as string)) as int) as string)) AS INT):int>
+struct<a:int,udf(udf(a)):int>
 -- !query output
 NULL	1
 
@@ -125,7 +125,7 @@ NULL	1
 -- !query
 SELECT udf(udf(empty_table.a)) FROM empty_table LEFT SEMI JOIN t1 ON (udf(empty_table.a) = udf(udf(t1.a)))
 -- !query schema
-struct<CAST(udf(cast(cast(udf(cast(a as string)) as int) as string)) AS INT):int>
+struct<udf(udf(a)):int>
 -- !query output
 
 
@@ -141,7 +141,7 @@ struct<a:int>
 -- !query
 SELECT udf(empty_table.a) FROM empty_table INNER JOIN empty_table AS empty_table2 ON (udf(empty_table.a) = udf(udf(empty_table2.a)))
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS INT):int>
+struct<udf(a):int>
 -- !query output
 
 
@@ -149,7 +149,7 @@ struct<CAST(udf(cast(a as string)) AS INT):int>
 -- !query
 SELECT udf(udf(empty_table.a)) FROM empty_table CROSS JOIN empty_table AS empty_table2 ON (udf(udf(empty_table.a)) = udf(empty_table2.a))
 -- !query schema
-struct<CAST(udf(cast(cast(udf(cast(a as string)) as int) as string)) AS INT):int>
+struct<udf(udf(a)):int>
 -- !query output
 
 
@@ -157,7 +157,7 @@ struct<CAST(udf(cast(cast(udf(cast(a as string)) as int) as string)) AS INT):int
 -- !query
 SELECT udf(empty_table.a) FROM empty_table LEFT OUTER JOIN empty_table AS empty_table2 ON (udf(empty_table.a) = udf(empty_table2.a))
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS INT):int>
+struct<udf(a):int>
 -- !query output
 
 
@@ -165,7 +165,7 @@ struct<CAST(udf(cast(a as string)) AS INT):int>
 -- !query
 SELECT udf(udf(empty_table.a)) FROM empty_table RIGHT OUTER JOIN empty_table AS empty_table2 ON (udf(empty_table.a) = udf(udf(empty_table2.a)))
 -- !query schema
-struct<CAST(udf(cast(cast(udf(cast(a as string)) as int) as string)) AS INT):int>
+struct<udf(udf(a)):int>
 -- !query output
 
 
@@ -173,7 +173,7 @@ struct<CAST(udf(cast(cast(udf(cast(a as string)) as int) as string)) AS INT):int
 -- !query
 SELECT udf(empty_table.a) FROM empty_table FULL OUTER JOIN empty_table AS empty_table2 ON (udf(empty_table.a) = udf(empty_table2.a))
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS INT):int>
+struct<udf(a):int>
 -- !query output
 
 
@@ -181,7 +181,7 @@ struct<CAST(udf(cast(a as string)) AS INT):int>
 -- !query
 SELECT udf(udf(empty_table.a)) FROM empty_table LEFT SEMI JOIN empty_table AS empty_table2 ON (udf(empty_table.a) = udf(empty_table2.a))
 -- !query schema
-struct<CAST(udf(cast(cast(udf(cast(a as string)) as int) as string)) AS INT):int>
+struct<udf(udf(a)):int>
 -- !query output
 
 
@@ -189,6 +189,6 @@ struct<CAST(udf(cast(cast(udf(cast(a as string)) as int) as string)) AS INT):int
 -- !query
 SELECT udf(empty_table.a) FROM empty_table LEFT ANTI JOIN empty_table AS empty_table2 ON (udf(empty_table.a) = udf(empty_table2.a))
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS INT):int>
+struct<udf(a):int>
 -- !query output
 
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-natural-join.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/udf-natural-join.sql.out
index a8233a0e398b0..dd6ba9d6ddd86 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/udf-natural-join.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-natural-join.sql.out
@@ -59,6 +59,6 @@ two	2	22
 -- !query
 SELECT udf(count(*)) FROM nt1 natural full outer join nt2
 -- !query schema
-struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<udf(count(1)):bigint>
 -- !query output
 4
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-outer-join.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/udf-outer-join.sql.out
index afebbb0c1da92..c9a3a519c5e0c 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/udf-outer-join.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-outer-join.sql.out
@@ -34,7 +34,7 @@ GROUP BY udf(GREATEST(COALESCE(udf(t2.int_col1), 109), COALESCE(t1.int_col1, udf
 HAVING (udf(SUM(COALESCE(udf(t1.int_col1), udf(t2.int_col0)))))
             > (udf(COALESCE(t1.int_col1, t2.int_col0)) * 2)
 -- !query schema
-struct<CAST(udf(cast(sum(cast(cast(udf(cast(coalesce(int_col1, int_col0) as string)) as int) as bigint)) as string)) AS BIGINT):bigint,(CAST(udf(cast(coalesce(int_col1, int_col0) as string)) AS INT) * 2):int>
+struct<udf(sum(udf(coalesce(int_col1, int_col0)))):bigint,(udf(coalesce(int_col1, int_col0)) * 2):int>
 -- !query output
 -367	-734
 -507	-1014
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-pivot.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/udf-pivot.sql.out
index 087b4ed9302d8..dc5cc29762657 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/udf-pivot.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-pivot.sql.out
@@ -47,7 +47,7 @@ PIVOT (
   FOR course IN ('dotNET', 'Java')
 )
 -- !query schema
-struct<CAST(udf(cast(year as string)) AS INT):int,dotNET:bigint,Java:bigint>
+struct<udf(year):int,dotNET:bigint,Java:bigint>
 -- !query output
 2012	15000	20000
 2013	48000	30000
@@ -75,7 +75,7 @@ PIVOT (
   FOR course IN ('dotNET', 'Java')
 )
 -- !query schema
-struct<year:int,dotNET_CAST(udf(cast(sum(cast(earnings as bigint)) as string)) AS BIGINT):bigint,dotNET_CAST(udf(cast(avg(cast(earnings as bigint)) as string)) AS DOUBLE):double,Java_CAST(udf(cast(sum(cast(earnings as bigint)) as string)) AS BIGINT):bigint,Java_CAST(udf(cast(avg(cast(earnings as bigint)) as string)) AS DOUBLE):double>
+struct<year:int,dotNET_udf(sum(earnings)):bigint,dotNET_udf(avg(earnings)):double,Java_udf(sum(earnings)):bigint,Java_udf(avg(earnings)):double>
 -- !query output
 2012	15000	7500.0	20000	20000.0
 2013	48000	48000.0	30000	30000.0
@@ -104,7 +104,7 @@ PIVOT (
   FOR course IN ('dotNET', 'Java')
 )
 -- !query schema
-struct<dotNET_CAST(udf(cast(sum(cast(cast(udf(cast(earnings as string)) as int) as bigint)) as string)) AS BIGINT):bigint,dotNET_CAST(udf(cast(min(year) as string)) AS INT):int,Java_CAST(udf(cast(sum(cast(cast(udf(cast(earnings as string)) as int) as bigint)) as string)) AS BIGINT):bigint,Java_CAST(udf(cast(min(year) as string)) AS INT):int>
+struct<dotNET_udf(sum(udf(earnings))):bigint,dotNET_udf(min(year)):int,Java_udf(sum(udf(earnings))):bigint,Java_udf(min(year)):int>
 -- !query output
 63000	2012	50000	2012
 
@@ -139,7 +139,7 @@ PIVOT (
   FOR course IN ('dotNET', 'Java')
 )
 -- !query schema
-struct<year:int,dotNET_CAST(udf(cast(sum(cast(earnings as bigint)) as string)) AS BIGINT):bigint,dotNET_CAST(udf(cast(min(s) as string)) AS INT):int,Java_CAST(udf(cast(sum(cast(earnings as bigint)) as string)) AS BIGINT):bigint,Java_CAST(udf(cast(min(s) as string)) AS INT):int>
+struct<year:int,dotNET_udf(sum(earnings)):bigint,dotNET_udf(min(s)):int,Java_udf(sum(earnings)):bigint,Java_udf(min(s)):int>
 -- !query output
 2012	15000	1	20000	1
 2013	48000	2	30000	2
@@ -202,7 +202,7 @@ PIVOT (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Aggregate expression required for pivot, but 'coursesales.`earnings`' did not appear in any aggregate function.;
+Aggregate expression required for pivot, but 'coursesales.earnings' did not appear in any aggregate function.
 
 
 -- !query
@@ -217,7 +217,7 @@ PIVOT (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Aggregate expression required for pivot, but '__auto_generated_subquery_name.`year`' did not appear in any aggregate function.;
+Aggregate expression required for pivot, but '__auto_generated_subquery_name.year' did not appear in any aggregate function.
 
 
 -- !query
@@ -232,7 +232,7 @@ PIVOT (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`year`' given input columns: [__auto_generated_subquery_name.course, __auto_generated_subquery_name.earnings]; line 4 pos 0
+cannot resolve 'year' given input columns: [__auto_generated_subquery_name.course, __auto_generated_subquery_name.earnings]; line 4 pos 0
 
 
 -- !query
@@ -244,7 +244,7 @@ PIVOT (
   FOR course IN ('dotNET', 'Java')
 )
 -- !query schema
-struct<year:int,dotNET_CAST(udf(cast(CEIL(cast(udf(cast(sum(cast(earnings as bigint)) as string)) as bigint)) as string)) AS BIGINT):bigint,dotNET_a1:double,Java_CAST(udf(cast(CEIL(cast(udf(cast(sum(cast(earnings as bigint)) as string)) as bigint)) as string)) AS BIGINT):bigint,Java_a1:double>
+struct<year:int,dotNET_udf(CEIL(udf(sum(earnings)))):bigint,dotNET_a1:double,Java_udf(CEIL(udf(sum(earnings)))):bigint,Java_a1:double>
 -- !query output
 2012	15000	7501.0	20000	20001.0
 2013	48000	48001.0	30000	30001.0
@@ -262,7 +262,7 @@ PIVOT (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-It is not allowed to use an aggregate function in the argument of another aggregate function. Please use the inner aggregate function in a sub-query.;
+It is not allowed to use an aggregate function in the argument of another aggregate function. Please use the inner aggregate function in a sub-query.
 
 
 -- !query
@@ -276,7 +276,7 @@ PIVOT (
   FOR (course, year) IN (('dotNET', 2012), ('Java', 2013))
 )
 -- !query schema
-struct<s:int,[dotNET, 2012]:bigint,[Java, 2013]:bigint>
+struct<s:int,{dotNET, 2012}:bigint,{Java, 2013}:bigint>
 -- !query output
 1	15000	NULL
 2	NULL	30000
@@ -313,7 +313,7 @@ PIVOT (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Invalid pivot value 'dotNET': value data type string does not match pivot column data type struct<course:string,year:int>;
+Invalid pivot value 'dotNET': value data type string does not match pivot column data type struct<course:string,year:int>
 
 
 -- !query
@@ -326,7 +326,7 @@ PIVOT (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`s`' given input columns: [coursesales.course, coursesales.earnings, coursesales.year]; line 4 pos 15
+cannot resolve 's' given input columns: [coursesales.course, coursesales.earnings, coursesales.year]; line 4 pos 15
 
 
 -- !query
@@ -339,7 +339,7 @@ PIVOT (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Literal expressions required for pivot values, found 'course#x';
+Literal expressions required for pivot values, found 'course#x'
 
 
 -- !query
@@ -370,7 +370,7 @@ PIVOT (
   FOR (course, a) IN (('dotNET', array(1, 1)), ('Java', array(2, 2)))
 )
 -- !query schema
-struct<year:int,[dotNET, [1, 1]]:bigint,[Java, [2, 2]]:bigint>
+struct<year:int,{dotNET, [1, 1]}:bigint,{Java, [2, 2]}:bigint>
 -- !query output
 2012	15000	NULL
 2013	NULL	30000
@@ -387,7 +387,7 @@ PIVOT (
   FOR s IN ((1, 'a'), (2, 'b'))
 )
 -- !query schema
-struct<year:int,[1, a]:bigint,[2, b]:bigint>
+struct<year:int,{1, a}:bigint,{2, b}:bigint>
 -- !query output
 2012	35000	NULL
 2013	NULL	78000
@@ -404,7 +404,7 @@ PIVOT (
   FOR (course, s) IN (('dotNET', (1, 'a')), ('Java', (2, 'b')))
 )
 -- !query schema
-struct<year:int,[dotNET, [1, a]]:bigint,[Java, [2, b]]:bigint>
+struct<year:int,{dotNET, {1, a}}:bigint,{Java, {2, b}}:bigint>
 -- !query output
 2012	15000	NULL
 2013	NULL	30000
@@ -424,7 +424,7 @@ PIVOT (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Invalid pivot column 'm#x'. Pivot columns must be comparable.;
+Invalid pivot column 'm#x'. Pivot columns must be comparable.
 
 
 -- !query
@@ -441,7 +441,7 @@ PIVOT (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Invalid pivot column 'named_struct(course, course#x, m, m#x)'. Pivot columns must be comparable.;
+Invalid pivot column 'named_struct(course, course#x, m, m#x)'. Pivot columns must be comparable.
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-special-values.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/udf-special-values.sql.out
index 5e5c79172bb7a..f7e1874477eb2 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/udf-special-values.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-special-values.sql.out
@@ -5,7 +5,7 @@
 -- !query
 SELECT udf(x) FROM (VALUES (1), (2), (NULL)) v(x)
 -- !query schema
-struct<CAST(udf(cast(x as string)) AS INT):int>
+struct<udf(x):int>
 -- !query output
 1
 2
@@ -15,7 +15,7 @@ NULL
 -- !query
 SELECT udf(x) FROM (VALUES ('A'), ('B'), (NULL)) v(x)
 -- !query schema
-struct<CAST(udf(cast(x as string)) AS STRING):string>
+struct<udf(x):string>
 -- !query output
 A
 B
@@ -25,7 +25,7 @@ NULL
 -- !query
 SELECT udf(x) FROM (VALUES ('NaN'), ('1'), ('2')) v(x)
 -- !query schema
-struct<CAST(udf(cast(x as string)) AS STRING):string>
+struct<udf(x):string>
 -- !query output
 1
 2
@@ -35,7 +35,7 @@ NaN
 -- !query
 SELECT udf(x) FROM (VALUES ('Infinity'), ('1'), ('2')) v(x)
 -- !query schema
-struct<CAST(udf(cast(x as string)) AS STRING):string>
+struct<udf(x):string>
 -- !query output
 1
 2
@@ -45,7 +45,7 @@ Infinity
 -- !query
 SELECT udf(x) FROM (VALUES ('-Infinity'), ('1'), ('2')) v(x)
 -- !query schema
-struct<CAST(udf(cast(x as string)) AS STRING):string>
+struct<udf(x):string>
 -- !query output
 -Infinity
 1
@@ -55,7 +55,7 @@ struct<CAST(udf(cast(x as string)) AS STRING):string>
 -- !query
 SELECT udf(x) FROM (VALUES 0.00000001, 0.00000002, 0.00000003) v(x)
 -- !query schema
-struct<CAST(udf(cast(x as string)) AS DECIMAL(8,8)):decimal(8,8)>
+struct<udf(x):decimal(8,8)>
 -- !query output
 0.00000001
 0.00000002
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-window.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/udf-window.sql.out
index a915c1bd6c717..d4c857d969163 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/udf-window.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-window.sql.out
@@ -4,15 +4,15 @@
 
 -- !query
 CREATE OR REPLACE TEMPORARY VIEW testData AS SELECT * FROM VALUES
-(null, 1L, 1.0D, date("2017-08-01"), timestamp(1501545600), "a"),
-(1, 1L, 1.0D, date("2017-08-01"), timestamp(1501545600), "a"),
-(1, 2L, 2.5D, date("2017-08-02"), timestamp(1502000000), "a"),
-(2, 2147483650L, 100.001D, date("2020-12-31"), timestamp(1609372800), "a"),
-(1, null, 1.0D, date("2017-08-01"), timestamp(1501545600), "b"),
-(2, 3L, 3.3D, date("2017-08-03"), timestamp(1503000000), "b"),
-(3, 2147483650L, 100.001D, date("2020-12-31"), timestamp(1609372800), "b"),
+(null, 1L, 1.0D, date("2017-08-01"), timestamp_seconds(1501545600), "a"),
+(1, 1L, 1.0D, date("2017-08-01"), timestamp_seconds(1501545600), "a"),
+(1, 2L, 2.5D, date("2017-08-02"), timestamp_seconds(1502000000), "a"),
+(2, 2147483650L, 100.001D, date("2020-12-31"), timestamp_seconds(1609372800), "a"),
+(1, null, 1.0D, date("2017-08-01"), timestamp_seconds(1501545600), "b"),
+(2, 3L, 3.3D, date("2017-08-03"), timestamp_seconds(1503000000), "b"),
+(3, 2147483650L, 100.001D, date("2020-12-31"), timestamp_seconds(1609372800), "b"),
 (null, null, null, null, null, null),
-(3, 1L, 1.0D, date("2017-08-01"), timestamp(1501545600), null)
+(3, 1L, 1.0D, date("2017-08-01"), timestamp_seconds(1501545600), null)
 AS testData(val, val_long, val_double, val_date, val_timestamp, cate)
 -- !query schema
 struct<>
@@ -24,7 +24,7 @@ struct<>
 SELECT udf(val), cate, count(val) OVER(PARTITION BY cate ORDER BY udf(val) ROWS CURRENT ROW) FROM testData
 ORDER BY cate, udf(val)
 -- !query schema
-struct<CAST(udf(cast(val as string)) AS INT):int,cate:string,count(val) OVER (PARTITION BY cate ORDER BY CAST(udf(cast(val as string)) AS INT) ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND CURRENT ROW):bigint>
+struct<udf(val):int,cate:string,count(val) OVER (PARTITION BY cate ORDER BY udf(val) ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND CURRENT ROW):bigint>
 -- !query output
 NULL	NULL	0
 3	NULL	1
@@ -41,7 +41,7 @@ NULL	a	0
 SELECT udf(val), cate, sum(val) OVER(PARTITION BY cate ORDER BY udf(val)
 ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING) FROM testData ORDER BY cate, udf(val)
 -- !query schema
-struct<CAST(udf(cast(val as string)) AS INT):int,cate:string,sum(val) OVER (PARTITION BY cate ORDER BY CAST(udf(cast(val as string)) AS INT) ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING):bigint>
+struct<udf(val):int,cate:string,sum(val) OVER (PARTITION BY cate ORDER BY udf(val) ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING):bigint>
 -- !query output
 NULL	NULL	3
 3	NULL	3
@@ -68,7 +68,7 @@ cannot resolve 'ROWS BETWEEN CURRENT ROW AND 2147483648L FOLLOWING' due to data
 SELECT udf(val), cate, count(val) OVER(PARTITION BY udf(cate) ORDER BY val RANGE 1 PRECEDING) FROM testData
 ORDER BY cate, udf(val)
 -- !query schema
-struct<CAST(udf(cast(val as string)) AS INT):int,cate:string,count(val) OVER (PARTITION BY CAST(udf(cast(cate as string)) AS STRING) ORDER BY val ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND CURRENT ROW):bigint>
+struct<udf(val):int,cate:string,count(val) OVER (PARTITION BY udf(cate) ORDER BY val ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND CURRENT ROW):bigint>
 -- !query output
 NULL	NULL	0
 3	NULL	1
@@ -85,7 +85,7 @@ NULL	a	0
 SELECT val, udf(cate), sum(val) OVER(PARTITION BY udf(cate) ORDER BY val
 RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING) FROM testData ORDER BY udf(cate), val
 -- !query schema
-struct<val:int,CAST(udf(cast(cate as string)) AS STRING):string,sum(val) OVER (PARTITION BY CAST(udf(cast(cate as string)) AS STRING) ORDER BY val ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING):bigint>
+struct<val:int,udf(cate):string,sum(val) OVER (PARTITION BY udf(cate) ORDER BY val ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING):bigint>
 -- !query output
 NULL	NULL	NULL
 3	NULL	3
@@ -102,7 +102,7 @@ NULL	a	NULL
 SELECT val_long, udf(cate), sum(val_long) OVER(PARTITION BY udf(cate) ORDER BY val_long
 RANGE BETWEEN CURRENT ROW AND 2147483648 FOLLOWING) FROM testData ORDER BY udf(cate), val_long
 -- !query schema
-struct<val_long:bigint,CAST(udf(cast(cate as string)) AS STRING):string,sum(val_long) OVER (PARTITION BY CAST(udf(cast(cate as string)) AS STRING) ORDER BY val_long ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND 2147483648 FOLLOWING):bigint>
+struct<val_long:bigint,udf(cate):string,sum(val_long) OVER (PARTITION BY udf(cate) ORDER BY val_long ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND 2147483648 FOLLOWING):bigint>
 -- !query output
 NULL	NULL	NULL
 1	NULL	1
@@ -119,7 +119,7 @@ NULL	b	NULL
 SELECT val_double, udf(cate), sum(val_double) OVER(PARTITION BY udf(cate) ORDER BY val_double
 RANGE BETWEEN CURRENT ROW AND 2.5 FOLLOWING) FROM testData ORDER BY udf(cate), val_double
 -- !query schema
-struct<val_double:double,CAST(udf(cast(cate as string)) AS STRING):string,sum(val_double) OVER (PARTITION BY CAST(udf(cast(cate as string)) AS STRING) ORDER BY val_double ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND CAST(2.5 AS DOUBLE) FOLLOWING):double>
+struct<val_double:double,udf(cate):string,sum(val_double) OVER (PARTITION BY udf(cate) ORDER BY val_double ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND 2.5 FOLLOWING):double>
 -- !query output
 NULL	NULL	NULL
 1.0	NULL	1.0
@@ -136,7 +136,7 @@ NULL	NULL	NULL
 SELECT val_date, udf(cate), max(val_date) OVER(PARTITION BY udf(cate) ORDER BY val_date
 RANGE BETWEEN CURRENT ROW AND 2 FOLLOWING) FROM testData ORDER BY udf(cate), val_date
 -- !query schema
-struct<val_date:date,CAST(udf(cast(cate as string)) AS STRING):string,max(val_date) OVER (PARTITION BY CAST(udf(cast(cate as string)) AS STRING) ORDER BY val_date ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND 2 FOLLOWING):date>
+struct<val_date:date,udf(cate):string,max(val_date) OVER (PARTITION BY udf(cate) ORDER BY val_date ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND 2 FOLLOWING):date>
 -- !query output
 NULL	NULL	NULL
 2017-08-01	NULL	2017-08-01
@@ -154,7 +154,7 @@ SELECT val_timestamp, udf(cate), avg(val_timestamp) OVER(PARTITION BY udf(cate)
 RANGE BETWEEN CURRENT ROW AND interval 23 days 4 hours FOLLOWING) FROM testData
 ORDER BY udf(cate), val_timestamp
 -- !query schema
-struct<val_timestamp:timestamp,CAST(udf(cast(cate as string)) AS STRING):string,avg(CAST(val_timestamp AS DOUBLE)) OVER (PARTITION BY CAST(udf(cast(cate as string)) AS STRING) ORDER BY val_timestamp ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND INTERVAL '23 days 4 hours' FOLLOWING):double>
+struct<val_timestamp:timestamp,udf(cate):string,avg(val_timestamp) OVER (PARTITION BY udf(cate) ORDER BY val_timestamp ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND INTERVAL '23 days 4 hours' FOLLOWING):double>
 -- !query output
 NULL	NULL	NULL
 2017-07-31 17:00:00	NULL	1.5015456E9
@@ -171,7 +171,7 @@ NULL	NULL	NULL
 SELECT val, udf(cate), sum(val) OVER(PARTITION BY cate ORDER BY val DESC
 RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING) FROM testData ORDER BY cate, val
 -- !query schema
-struct<val:int,CAST(udf(cast(cate as string)) AS STRING):string,sum(val) OVER (PARTITION BY cate ORDER BY val DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING):bigint>
+struct<val:int,udf(cate):string,sum(val) OVER (PARTITION BY cate ORDER BY val DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING):bigint>
 -- !query output
 NULL	NULL	NULL
 3	NULL	3
@@ -211,7 +211,7 @@ RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING) FROM testData ORDER BY cate, udf(val)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(PARTITION BY CAST(udf(cast(cate as string)) AS STRING) ORDER BY CAST(udf(cast(val as string)) AS INT) ASC NULLS FIRST, testdata.`cate` ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING)' due to data type mismatch: A range window frame with value boundaries cannot be used in a window specification with multiple order by expressions: cast(udf(cast(val#x as string)) as int) ASC NULLS FIRST,cate#x ASC NULLS FIRST; line 1 pos 38
+cannot resolve '(PARTITION BY CAST(udf(cast(cate as string)) AS STRING) ORDER BY CAST(udf(cast(val as string)) AS INT) ASC NULLS FIRST, testdata.cate ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING)' due to data type mismatch: A range window frame with value boundaries cannot be used in a window specification with multiple order by expressions: cast(udf(cast(val#x as string)) as int) ASC NULLS FIRST,cate#x ASC NULLS FIRST; line 1 pos 38
 
 
 -- !query
@@ -286,23 +286,23 @@ FROM testData
 WINDOW w AS (PARTITION BY udf(cate) ORDER BY udf(val))
 ORDER BY cate, udf(val)
 -- !query schema
-struct<CAST(udf(cast(val as string)) AS INT):int,cate:string,max:int,min:int,min:int,count:bigint,sum:bigint,avg:double,stddev:double,first_value:int,first_value_ignore_null:int,first_value_contain_null:int,last_value:int,last_value_ignore_null:int,last_value_contain_null:int,rank:int,dense_rank:int,cume_dist:double,percent_rank:double,ntile:int,row_number:int,var_pop:double,var_samp:double,approx_count_distinct:bigint,covar_pop:double,corr:double,stddev_samp:double,stddev_pop:double,collect_list:array<int>,collect_set:array<int>,skewness:double,kurtosis:double>
+struct<udf(val):int,cate:string,max:int,min:int,min:int,count:bigint,sum:bigint,avg:double,stddev:double,first_value:int,first_value_ignore_null:int,first_value_contain_null:int,last_value:int,last_value_ignore_null:int,last_value_contain_null:int,rank:int,dense_rank:int,cume_dist:double,percent_rank:double,ntile:int,row_number:int,var_pop:double,var_samp:double,approx_count_distinct:bigint,covar_pop:double,corr:double,stddev_samp:double,stddev_pop:double,collect_list:array<int>,collect_set:array<int>,skewness:double,kurtosis:double>
 -- !query output
 NULL	NULL	NULL	NULL	NULL	0	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	1	1	0.5	0.0	1	1	NULL	NULL	0	NULL	NULL	NULL	NULL	[]	[]	NULL	NULL
-3	NULL	3	3	3	1	3	3.0	NaN	NULL	3	NULL	3	3	3	2	2	1.0	1.0	2	2	0.0	NaN	1	0.0	NaN	NaN	0.0	[3]	[3]	NaN	NaN
-NULL	a	NULL	NULL	NULL	0	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	1	1	0.25	0.0	1	1	NULL	NULL	0	NULL	NULL	NULL	NULL	[]	[]	NaN	NaN
+3	NULL	3	3	3	1	3	3.0	NULL	NULL	3	NULL	3	3	3	2	2	1.0	1.0	2	2	0.0	NULL	1	0.0	NULL	NULL	0.0	[3]	[3]	NULL	NULL
+NULL	a	NULL	NULL	NULL	0	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	1	1	0.25	0.0	1	1	NULL	NULL	0	NULL	NULL	NULL	NULL	[]	[]	NULL	NULL
 1	a	1	1	1	2	2	1.0	0.0	NULL	1	NULL	1	1	1	2	2	0.75	0.3333333333333333	1	2	0.0	0.0	1	0.0	NULL	0.0	0.0	[1,1]	[1]	0.7071067811865476	-1.5
 1	a	1	1	1	2	2	1.0	0.0	NULL	1	NULL	1	1	1	2	2	0.75	0.3333333333333333	2	3	0.0	0.0	1	0.0	NULL	0.0	0.0	[1,1]	[1]	0.7071067811865476	-1.5
 2	a	2	1	1	3	4	1.3333333333333333	0.5773502691896258	NULL	1	NULL	2	2	2	4	3	1.0	1.0	2	4	0.22222222222222224	0.33333333333333337	2	4.772185885555555E8	1.0	0.5773502691896258	0.4714045207910317	[1,1,2]	[1,2]	1.1539890888012805	-0.6672217220327235
-1	b	1	1	1	1	1	1.0	NaN	1	1	1	1	1	1	1	1	0.3333333333333333	0.0	1	1	0.0	NaN	1	NULL	NULL	NaN	0.0	[1]	[1]	NaN	NaN
-2	b	2	1	1	2	3	1.5	0.7071067811865476	1	1	1	2	2	2	2	2	0.6666666666666666	0.5	1	2	0.25	0.5	2	0.0	NaN	0.7071067811865476	0.5	[1,2]	[1,2]	0.0	-2.0000000000000013
+1	b	1	1	1	1	1	1.0	NULL	1	1	1	1	1	1	1	1	0.3333333333333333	0.0	1	1	0.0	NULL	1	NULL	NULL	NULL	0.0	[1]	[1]	NULL	NULL
+2	b	2	1	1	2	3	1.5	0.7071067811865476	1	1	1	2	2	2	2	2	0.6666666666666666	0.5	1	2	0.25	0.5	2	0.0	NULL	0.7071067811865476	0.5	[1,2]	[1,2]	0.0	-2.0000000000000013
 3	b	3	1	1	3	6	2.0	1.0	1	1	1	3	3	3	3	3	1.0	1.0	2	3	0.6666666666666666	1.0	3	5.3687091175E8	1.0	1.0	0.816496580927726	[1,2,3]	[1,2,3]	0.7057890433107311	-1.4999999999999984
 
 
 -- !query
 SELECT udf(val), cate, avg(null) OVER(PARTITION BY cate ORDER BY val) FROM testData ORDER BY cate, val
 -- !query schema
-struct<CAST(udf(cast(val as string)) AS INT):int,cate:string,avg(CAST(NULL AS DOUBLE)) OVER (PARTITION BY cate ORDER BY val ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):double>
+struct<udf(val):int,cate:string,avg(NULL) OVER (PARTITION BY cate ORDER BY val ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):double>
 -- !query output
 NULL	NULL	NULL
 3	NULL	NULL
@@ -321,13 +321,13 @@ SELECT udf(val), cate, row_number() OVER(PARTITION BY cate) FROM testData ORDER
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Window function row_number() requires window to be ordered, please add ORDER BY clause. For example SELECT row_number()(value_expr) OVER (PARTITION BY window_partition ORDER BY window_ordering) from table;
+Window function row_number() requires window to be ordered, please add ORDER BY clause. For example SELECT row_number()(value_expr) OVER (PARTITION BY window_partition ORDER BY window_ordering) from table
 
 
 -- !query
 SELECT udf(val), cate, sum(val) OVER(), avg(val) OVER() FROM testData ORDER BY cate, val
 -- !query schema
-struct<CAST(udf(cast(val as string)) AS INT):int,cate:string,sum(CAST(val AS BIGINT)) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):bigint,avg(CAST(val AS BIGINT)) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):double>
+struct<udf(val):int,cate:string,sum(val) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):bigint,avg(val) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):double>
 -- !query output
 NULL	NULL	13	1.8571428571428572
 3	NULL	13	1.8571428571428572
@@ -352,7 +352,7 @@ FROM testData
 WINDOW w AS ()
 ORDER BY cate, val
 -- !query schema
-struct<CAST(udf(cast(val as string)) AS INT):int,cate:string,first_value:boolean,first_value_ignore_null:boolean,first_value_contain_null:boolean,last_value:boolean,last_value_ignore_null:boolean,last_value_contain_null:boolean>
+struct<udf(val):int,cate:string,first_value:boolean,first_value_ignore_null:boolean,first_value_contain_null:boolean,last_value:boolean,last_value_ignore_null:boolean,last_value_contain_null:boolean>
 -- !query output
 NULL	NULL	false	true	false	false	true	false
 3	NULL	false	true	false	false	true	false
@@ -371,7 +371,7 @@ FROM testData
 WHERE val is not null
 WINDOW w AS (PARTITION BY cate ORDER BY val)
 -- !query schema
-struct<CAST(udf(cast(cate as string)) AS STRING):string,sum(CAST(val AS BIGINT)) OVER (PARTITION BY cate ORDER BY val ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):bigint>
+struct<udf(cate):string,sum(val) OVER (PARTITION BY cate ORDER BY val ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):bigint>
 -- !query output
 NULL	3
 a	2
diff --git a/sql/core/src/test/resources/sql-tests/results/union.sql.out b/sql/core/src/test/resources/sql-tests/results/union.sql.out
index 44002406836a4..ce3c761bc5d2d 100644
--- a/sql/core/src/test/resources/sql-tests/results/union.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/union.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 16
+-- Number of queries: 20
 
 
 -- !query
@@ -126,6 +126,39 @@ struct<array(1, 2):array<int>,str:string>
 [1,2]	str
 
 
+-- !query
+CREATE OR REPLACE TEMPORARY VIEW t3 AS VALUES (decimal(1)) tbl(v)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SELECT t.v FROM (
+  SELECT v FROM t3
+  UNION ALL
+  SELECT v + v AS v FROM t3
+) t
+-- !query schema
+struct<v:decimal(11,0)>
+-- !query output
+1
+2
+
+
+-- !query
+SELECT SUM(t.v) FROM (
+  SELECT v FROM t3
+  UNION
+  SELECT v + v AS v FROM t3
+) t
+-- !query schema
+struct<sum(v):decimal(21,0)>
+-- !query output
+3
+
+
 -- !query
 DROP VIEW IF EXISTS t1
 -- !query schema
@@ -142,6 +175,14 @@ struct<>
 
 
 
+-- !query
+DROP VIEW IF EXISTS t3
+-- !query schema
+struct<>
+-- !query output
+
+
+
 -- !query
 DROP VIEW IF EXISTS p1
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/using-join.sql.out b/sql/core/src/test/resources/sql-tests/results/using-join.sql.out
new file mode 100644
index 0000000000000..1d2ae9d96ecad
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/using-join.sql.out
@@ -0,0 +1,338 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 31
+
+
+-- !query
+create temporary view nt1 as select * from values
+  ("one", 1),
+  ("two", 2),
+  ("three", 3)
+  as nt1(k, v1)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+create temporary view nt2 as select * from values
+  ("one", 1),
+  ("two", 22),
+  ("one", 5),
+  ("four", 4)
+  as nt2(k, v2)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SELECT * FROM nt1 left outer join nt2 using (k)
+-- !query schema
+struct<k:string,v1:int,v2:int>
+-- !query output
+one	1	1
+one	1	5
+three	3	NULL
+two	2	22
+
+
+-- !query
+SELECT k FROM nt1 left outer join nt2 using (k)
+-- !query schema
+struct<k:string>
+-- !query output
+one
+one
+three
+two
+
+
+-- !query
+SELECT nt1.*, nt2.* FROM nt1 left outer join nt2 using (k)
+-- !query schema
+struct<k:string,v1:int,k:string,v2:int>
+-- !query output
+one	1	one	1
+one	1	one	5
+three	3	NULL	NULL
+two	2	two	22
+
+
+-- !query
+SELECT nt1.k, nt2.k FROM nt1 left outer join nt2 using (k)
+-- !query schema
+struct<k:string,k:string>
+-- !query output
+one	one
+one	one
+three	NULL
+two	two
+
+
+-- !query
+SELECT k, nt1.k FROM nt1 left outer join nt2 using (k)
+-- !query schema
+struct<k:string,k:string>
+-- !query output
+one	one
+one	one
+three	three
+two	two
+
+
+-- !query
+SELECT k, nt2.k FROM nt1 left outer join nt2 using (k)
+-- !query schema
+struct<k:string,k:string>
+-- !query output
+one	one
+one	one
+three	NULL
+two	two
+
+
+-- !query
+SELECT * FROM nt1 left semi join nt2 using (k)
+-- !query schema
+struct<k:string,v1:int>
+-- !query output
+one	1
+two	2
+
+
+-- !query
+SELECT k FROM nt1 left semi join nt2 using (k)
+-- !query schema
+struct<k:string>
+-- !query output
+one
+two
+
+
+-- !query
+SELECT nt1.* FROM nt1 left semi join nt2 using (k)
+-- !query schema
+struct<k:string,v1:int>
+-- !query output
+one	1
+two	2
+
+
+-- !query
+SELECT nt1.k FROM nt1 left semi join nt2 using (k)
+-- !query schema
+struct<k:string>
+-- !query output
+one
+two
+
+
+-- !query
+SELECT k, nt1.k FROM nt1 left semi join nt2 using (k)
+-- !query schema
+struct<k:string,k:string>
+-- !query output
+one	one
+two	two
+
+
+-- !query
+SELECT * FROM nt1 right outer join nt2 using (k)
+-- !query schema
+struct<k:string,v1:int,v2:int>
+-- !query output
+four	NULL	4
+one	1	1
+one	1	5
+two	2	22
+
+
+-- !query
+SELECT k FROM nt1 right outer join nt2 using (k)
+-- !query schema
+struct<k:string>
+-- !query output
+four
+one
+one
+two
+
+
+-- !query
+SELECT nt1.*, nt2.* FROM nt1 right outer join nt2 using (k)
+-- !query schema
+struct<k:string,v1:int,k:string,v2:int>
+-- !query output
+NULL	NULL	four	4
+one	1	one	1
+one	1	one	5
+two	2	two	22
+
+
+-- !query
+SELECT nt1.k, nt2.k FROM nt1 right outer join nt2 using (k)
+-- !query schema
+struct<k:string,k:string>
+-- !query output
+NULL	four
+one	one
+one	one
+two	two
+
+
+-- !query
+SELECT k, nt1.k FROM nt1 right outer join nt2 using (k)
+-- !query schema
+struct<k:string,k:string>
+-- !query output
+four	NULL
+one	one
+one	one
+two	two
+
+
+-- !query
+SELECT k, nt2.k FROM nt1 right outer join nt2 using (k)
+-- !query schema
+struct<k:string,k:string>
+-- !query output
+four	four
+one	one
+one	one
+two	two
+
+
+-- !query
+SELECT * FROM nt1 full outer join nt2 using (k)
+-- !query schema
+struct<k:string,v1:int,v2:int>
+-- !query output
+four	NULL	4
+one	1	1
+one	1	5
+three	3	NULL
+two	2	22
+
+
+-- !query
+SELECT k FROM nt1 full outer join nt2 using (k)
+-- !query schema
+struct<k:string>
+-- !query output
+four
+one
+one
+three
+two
+
+
+-- !query
+SELECT nt1.*, nt2.* FROM nt1 full outer join nt2 using (k)
+-- !query schema
+struct<k:string,v1:int,k:string,v2:int>
+-- !query output
+NULL	NULL	four	4
+one	1	one	1
+one	1	one	5
+three	3	NULL	NULL
+two	2	two	22
+
+
+-- !query
+SELECT nt1.k, nt2.k FROM nt1 full outer join nt2 using (k)
+-- !query schema
+struct<k:string,k:string>
+-- !query output
+NULL	four
+one	one
+one	one
+three	NULL
+two	two
+
+
+-- !query
+SELECT k, nt1.k FROM nt1 full outer join nt2 using (k)
+-- !query schema
+struct<k:string,k:string>
+-- !query output
+four	NULL
+one	one
+one	one
+three	three
+two	two
+
+
+-- !query
+SELECT k, nt2.k FROM nt1 full outer join nt2 using (k)
+-- !query schema
+struct<k:string,k:string>
+-- !query output
+four	four
+one	one
+one	one
+three	NULL
+two	two
+
+
+-- !query
+SELECT * FROM nt1 full outer join nt2 using (k)
+-- !query schema
+struct<k:string,v1:int,v2:int>
+-- !query output
+four	NULL	4
+one	1	1
+one	1	5
+three	3	NULL
+two	2	22
+
+
+-- !query
+SELECT k FROM nt1 inner join nt2 using (k)
+-- !query schema
+struct<k:string>
+-- !query output
+one
+one
+two
+
+
+-- !query
+SELECT nt1.*, nt2.* FROM nt1 inner join nt2 using (k)
+-- !query schema
+struct<k:string,v1:int,k:string,v2:int>
+-- !query output
+one	1	one	1
+one	1	one	5
+two	2	two	22
+
+
+-- !query
+SELECT nt1.k, nt2.k FROM nt1 inner join nt2 using (k)
+-- !query schema
+struct<k:string,k:string>
+-- !query output
+one	one
+one	one
+two	two
+
+
+-- !query
+SELECT k, nt1.k FROM nt1 inner join nt2 using (k)
+-- !query schema
+struct<k:string,k:string>
+-- !query output
+one	one
+one	one
+two	two
+
+
+-- !query
+SELECT k, nt2.k FROM nt1 inner join nt2 using (k)
+-- !query schema
+struct<k:string,k:string>
+-- !query output
+one	one
+one	one
+two	two
diff --git a/sql/core/src/test/resources/sql-tests/results/window.sql.out b/sql/core/src/test/resources/sql-tests/results/window.sql.out
index 625088f90ced9..b3f9e6cb6132c 100644
--- a/sql/core/src/test/resources/sql-tests/results/window.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/window.sql.out
@@ -1,18 +1,18 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 24
+-- Number of queries: 50
 
 
 -- !query
 CREATE OR REPLACE TEMPORARY VIEW testData AS SELECT * FROM VALUES
-(null, 1L, 1.0D, date("2017-08-01"), timestamp(1501545600), "a"),
-(1, 1L, 1.0D, date("2017-08-01"), timestamp(1501545600), "a"),
-(1, 2L, 2.5D, date("2017-08-02"), timestamp(1502000000), "a"),
-(2, 2147483650L, 100.001D, date("2020-12-31"), timestamp(1609372800), "a"),
-(1, null, 1.0D, date("2017-08-01"), timestamp(1501545600), "b"),
-(2, 3L, 3.3D, date("2017-08-03"), timestamp(1503000000), "b"),
-(3, 2147483650L, 100.001D, date("2020-12-31"), timestamp(1609372800), "b"),
+(null, 1L, 1.0D, date("2017-08-01"), timestamp_seconds(1501545600), "a"),
+(1, 1L, 1.0D, date("2017-08-01"), timestamp_seconds(1501545600), "a"),
+(1, 2L, 2.5D, date("2017-08-02"), timestamp_seconds(1502000000), "a"),
+(2, 2147483650L, 100.001D, date("2020-12-31"), timestamp_seconds(1609372800), "a"),
+(1, null, 1.0D, date("2017-08-01"), timestamp_seconds(1501545600), "b"),
+(2, 3L, 3.3D, date("2017-08-03"), timestamp_seconds(1503000000), "b"),
+(3, 2147483650L, 100.001D, date("2020-12-31"), timestamp_seconds(1609372800), "b"),
 (null, null, null, null, null, null),
-(3, 1L, 1.0D, date("2017-08-01"), timestamp(1501545600), null)
+(3, 1L, 1.0D, date("2017-08-01"), timestamp_seconds(1501545600), null)
 AS testData(val, val_long, val_double, val_date, val_timestamp, cate)
 -- !query schema
 struct<>
@@ -20,6 +20,50 @@ struct<>
 
 
 
+-- !query
+CREATE OR REPLACE TEMPORARY VIEW basic_pays AS SELECT * FROM VALUES
+('Diane Murphy','Accounting',8435),
+('Mary Patterson','Accounting',9998),
+('Jeff Firrelli','Accounting',8992),
+('William Patterson','Accounting',8870),
+('Gerard Bondur','Accounting',11472),
+('Anthony Bow','Accounting',6627),
+('Leslie Jennings','IT',8113),
+('Leslie Thompson','IT',5186),
+('Julie Firrelli','Sales',9181),
+('Steve Patterson','Sales',9441),
+('Foon Yue Tseng','Sales',6660),
+('George Vanauf','Sales',10563),
+('Loui Bondur','SCM',10449),
+('Gerard Hernandez','SCM',6949),
+('Pamela Castillo','SCM',11303),
+('Larry Bott','SCM',11798),
+('Barry Jones','SCM',10586)
+AS basic_pays(employee_name, department, salary)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+CREATE OR REPLACE TEMPORARY VIEW test_ignore_null AS SELECT * FROM VALUES
+('a', 0, null),
+('a', 1, 'x'),
+('b', 2, null),
+('c', 3, null),
+('a', 4, 'y'),
+('b', 5, null),
+('a', 6, 'z'),
+('a', 7, 'v'),
+('a', 8, null)
+AS test_ignore_null(content, id, v)
+-- !query schema
+struct<>
+-- !query output
+
+
+
 -- !query
 SELECT val, cate, count(val) OVER(PARTITION BY cate ORDER BY val ROWS CURRENT ROW) FROM testData
 ORDER BY cate, val
@@ -119,7 +163,7 @@ NULL	b	NULL
 SELECT val_double, cate, sum(val_double) OVER(PARTITION BY cate ORDER BY val_double
 RANGE BETWEEN CURRENT ROW AND 2.5 FOLLOWING) FROM testData ORDER BY cate, val_double
 -- !query schema
-struct<val_double:double,cate:string,sum(val_double) OVER (PARTITION BY cate ORDER BY val_double ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND CAST(2.5 AS DOUBLE) FOLLOWING):double>
+struct<val_double:double,cate:string,sum(val_double) OVER (PARTITION BY cate ORDER BY val_double ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND 2.5 FOLLOWING):double>
 -- !query output
 NULL	NULL	NULL
 1.0	NULL	1.0
@@ -154,7 +198,7 @@ SELECT val_timestamp, cate, avg(val_timestamp) OVER(PARTITION BY cate ORDER BY v
 RANGE BETWEEN CURRENT ROW AND interval 23 days 4 hours FOLLOWING) FROM testData
 ORDER BY cate, val_timestamp
 -- !query schema
-struct<val_timestamp:timestamp,cate:string,avg(CAST(val_timestamp AS DOUBLE)) OVER (PARTITION BY cate ORDER BY val_timestamp ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND INTERVAL '23 days 4 hours' FOLLOWING):double>
+struct<val_timestamp:timestamp,cate:string,avg(val_timestamp) OVER (PARTITION BY cate ORDER BY val_timestamp ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND INTERVAL '23 days 4 hours' FOLLOWING):double>
 -- !query output
 NULL	NULL	NULL
 2017-07-31 17:00:00	NULL	1.5015456E9
@@ -167,6 +211,71 @@ NULL	NULL	NULL
 2020-12-30 16:00:00	b	1.6093728E9
 
 
+-- !query
+SELECT val_timestamp, cate, avg(val_timestamp) OVER(PARTITION BY cate ORDER BY val_timestamp
+RANGE BETWEEN CURRENT ROW AND interval '1-1' year to month FOLLOWING) FROM testData
+ORDER BY cate, val_timestamp
+-- !query schema
+struct<val_timestamp:timestamp,cate:string,avg(val_timestamp) OVER (PARTITION BY cate ORDER BY val_timestamp ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND INTERVAL '1-1' YEAR TO MONTH FOLLOWING):double>
+-- !query output
+NULL	NULL	NULL
+2017-07-31 17:00:00	NULL	1.5015456E9
+2017-07-31 17:00:00	a	1.5016970666666667E9
+2017-07-31 17:00:00	a	1.5016970666666667E9
+2017-08-05 23:13:20	a	1.502E9
+2020-12-30 16:00:00	a	1.6093728E9
+2017-07-31 17:00:00	b	1.5022728E9
+2017-08-17 13:00:00	b	1.503E9
+2020-12-30 16:00:00	b	1.6093728E9
+
+
+-- !query
+SELECT val_timestamp, cate, avg(val_timestamp) OVER(PARTITION BY cate ORDER BY val_timestamp
+RANGE BETWEEN CURRENT ROW AND interval '1 2:3:4.001' day to second FOLLOWING) FROM testData
+ORDER BY cate, val_timestamp
+-- !query schema
+struct<val_timestamp:timestamp,cate:string,avg(val_timestamp) OVER (PARTITION BY cate ORDER BY val_timestamp ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND INTERVAL '1 02:03:04.001' DAY TO SECOND FOLLOWING):double>
+-- !query output
+NULL	NULL	NULL
+2017-07-31 17:00:00	NULL	1.5015456E9
+2017-07-31 17:00:00	a	1.5015456E9
+2017-07-31 17:00:00	a	1.5015456E9
+2017-08-05 23:13:20	a	1.502E9
+2020-12-30 16:00:00	a	1.6093728E9
+2017-07-31 17:00:00	b	1.5015456E9
+2017-08-17 13:00:00	b	1.503E9
+2020-12-30 16:00:00	b	1.6093728E9
+
+
+-- !query
+SELECT val_date, cate, avg(val_timestamp) OVER(PARTITION BY cate ORDER BY val_date
+RANGE BETWEEN CURRENT ROW AND interval '1-1' year to month FOLLOWING) FROM testData
+ORDER BY cate, val_date
+-- !query schema
+struct<val_date:date,cate:string,avg(val_timestamp) OVER (PARTITION BY cate ORDER BY val_date ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND INTERVAL '1-1' YEAR TO MONTH FOLLOWING):double>
+-- !query output
+NULL	NULL	NULL
+2017-08-01	NULL	1.5015456E9
+2017-08-01	a	1.5016970666666667E9
+2017-08-01	a	1.5016970666666667E9
+2017-08-02	a	1.502E9
+2020-12-31	a	1.6093728E9
+2017-08-01	b	1.5022728E9
+2017-08-03	b	1.503E9
+2020-12-31	b	1.6093728E9
+
+
+-- !query
+SELECT val_date, cate, avg(val_timestamp) OVER(PARTITION BY cate ORDER BY val_date
+RANGE BETWEEN CURRENT ROW AND interval '1 2:3:4.001' day to second FOLLOWING) FROM testData
+ORDER BY cate, val_date
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve '(PARTITION BY testdata.cate ORDER BY testdata.val_date ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND INTERVAL '1 02:03:04.001' DAY TO SECOND FOLLOWING)' due to data type mismatch: The data type 'date' used in the order specification does not match the data type 'interval day to second' which is used in the range frame.; line 1 pos 46
+
+
 -- !query
 SELECT val, cate, sum(val) OVER(PARTITION BY cate ORDER BY val DESC
 RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING) FROM testData ORDER BY cate, val
@@ -201,7 +310,7 @@ RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING) FROM testData ORDER BY cate, val
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(PARTITION BY testdata.`cate` RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING)' due to data type mismatch: A range window frame cannot be used in an unordered window specification.; line 1 pos 33
+cannot resolve '(PARTITION BY testdata.cate RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING)' due to data type mismatch: A range window frame cannot be used in an unordered window specification.; line 1 pos 33
 
 
 -- !query
@@ -211,7 +320,7 @@ RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING) FROM testData ORDER BY cate, val
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(PARTITION BY testdata.`cate` ORDER BY testdata.`val` ASC NULLS FIRST, testdata.`cate` ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING)' due to data type mismatch: A range window frame with value boundaries cannot be used in a window specification with multiple order by expressions: val#x ASC NULLS FIRST,cate#x ASC NULLS FIRST; line 1 pos 33
+cannot resolve '(PARTITION BY testdata.cate ORDER BY testdata.val ASC NULLS FIRST, testdata.cate ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING)' due to data type mismatch: A range window frame with value boundaries cannot be used in a window specification with multiple order by expressions: val#x ASC NULLS FIRST,cate#x ASC NULLS FIRST; line 1 pos 33
 
 
 -- !query
@@ -221,7 +330,7 @@ RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING) FROM testData ORDER BY cate, val
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(PARTITION BY testdata.`cate` ORDER BY current_timestamp() ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING)' due to data type mismatch: The data type 'timestamp' used in the order specification does not match the data type 'int' which is used in the range frame.; line 1 pos 33
+cannot resolve '(PARTITION BY testdata.cate ORDER BY current_timestamp() ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING)' due to data type mismatch: The data type 'timestamp' used in the order specification does not match the data type 'int' which is used in the range frame.; line 1 pos 33
 
 
 -- !query
@@ -289,20 +398,20 @@ ORDER BY cate, val
 struct<val:int,cate:string,max:int,min:int,min:int,count:bigint,sum:bigint,avg:double,stddev:double,first_value:int,first_value_ignore_null:int,first_value_contain_null:int,last_value:int,last_value_ignore_null:int,last_value_contain_null:int,rank:int,dense_rank:int,cume_dist:double,percent_rank:double,ntile:int,row_number:int,var_pop:double,var_samp:double,approx_count_distinct:bigint,covar_pop:double,corr:double,stddev_samp:double,stddev_pop:double,collect_list:array<int>,collect_set:array<int>,skewness:double,kurtosis:double>
 -- !query output
 NULL	NULL	NULL	NULL	NULL	0	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	1	1	0.5	0.0	1	1	NULL	NULL	0	NULL	NULL	NULL	NULL	[]	[]	NULL	NULL
-3	NULL	3	3	3	1	3	3.0	NaN	NULL	3	NULL	3	3	3	2	2	1.0	1.0	2	2	0.0	NaN	1	0.0	NaN	NaN	0.0	[3]	[3]	NaN	NaN
-NULL	a	NULL	NULL	NULL	0	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	1	1	0.25	0.0	1	1	NULL	NULL	0	NULL	NULL	NULL	NULL	[]	[]	NaN	NaN
+3	NULL	3	3	3	1	3	3.0	NULL	NULL	3	NULL	3	3	3	2	2	1.0	1.0	2	2	0.0	NULL	1	0.0	NULL	NULL	0.0	[3]	[3]	NULL	NULL
+NULL	a	NULL	NULL	NULL	0	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	1	1	0.25	0.0	1	1	NULL	NULL	0	NULL	NULL	NULL	NULL	[]	[]	NULL	NULL
 1	a	1	1	1	2	2	1.0	0.0	NULL	1	NULL	1	1	1	2	2	0.75	0.3333333333333333	1	2	0.0	0.0	1	0.0	NULL	0.0	0.0	[1,1]	[1]	0.7071067811865476	-1.5
 1	a	1	1	1	2	2	1.0	0.0	NULL	1	NULL	1	1	1	2	2	0.75	0.3333333333333333	2	3	0.0	0.0	1	0.0	NULL	0.0	0.0	[1,1]	[1]	0.7071067811865476	-1.5
 2	a	2	1	1	3	4	1.3333333333333333	0.5773502691896258	NULL	1	NULL	2	2	2	4	3	1.0	1.0	2	4	0.22222222222222224	0.33333333333333337	2	4.772185885555555E8	1.0	0.5773502691896258	0.4714045207910317	[1,1,2]	[1,2]	1.1539890888012805	-0.6672217220327235
-1	b	1	1	1	1	1	1.0	NaN	1	1	1	1	1	1	1	1	0.3333333333333333	0.0	1	1	0.0	NaN	1	NULL	NULL	NaN	0.0	[1]	[1]	NaN	NaN
-2	b	2	1	1	2	3	1.5	0.7071067811865476	1	1	1	2	2	2	2	2	0.6666666666666666	0.5	1	2	0.25	0.5	2	0.0	NaN	0.7071067811865476	0.5	[1,2]	[1,2]	0.0	-2.0000000000000013
+1	b	1	1	1	1	1	1.0	NULL	1	1	1	1	1	1	1	1	0.3333333333333333	0.0	1	1	0.0	NULL	1	NULL	NULL	NULL	0.0	[1]	[1]	NULL	NULL
+2	b	2	1	1	2	3	1.5	0.7071067811865476	1	1	1	2	2	2	2	2	0.6666666666666666	0.5	1	2	0.25	0.5	2	0.0	NULL	0.7071067811865476	0.5	[1,2]	[1,2]	0.0	-2.0000000000000013
 3	b	3	1	1	3	6	2.0	1.0	1	1	1	3	3	3	3	3	1.0	1.0	2	3	0.6666666666666666	1.0	3	5.3687091175E8	1.0	1.0	0.816496580927726	[1,2,3]	[1,2,3]	0.7057890433107311	-1.4999999999999984
 
 
 -- !query
 SELECT val, cate, avg(null) OVER(PARTITION BY cate ORDER BY val) FROM testData ORDER BY cate, val
 -- !query schema
-struct<val:int,cate:string,avg(CAST(NULL AS DOUBLE)) OVER (PARTITION BY cate ORDER BY val ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):double>
+struct<val:int,cate:string,avg(NULL) OVER (PARTITION BY cate ORDER BY val ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):double>
 -- !query output
 NULL	NULL	NULL
 3	NULL	NULL
@@ -321,13 +430,13 @@ SELECT val, cate, row_number() OVER(PARTITION BY cate) FROM testData ORDER BY ca
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Window function row_number() requires window to be ordered, please add ORDER BY clause. For example SELECT row_number()(value_expr) OVER (PARTITION BY window_partition ORDER BY window_ordering) from table;
+Window function row_number() requires window to be ordered, please add ORDER BY clause. For example SELECT row_number()(value_expr) OVER (PARTITION BY window_partition ORDER BY window_ordering) from table
 
 
 -- !query
 SELECT val, cate, sum(val) OVER(), avg(val) OVER() FROM testData ORDER BY cate, val
 -- !query schema
-struct<val:int,cate:string,sum(CAST(val AS BIGINT)) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):bigint,avg(CAST(val AS BIGINT)) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):double>
+struct<val:int,cate:string,sum(val) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):bigint,avg(val) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):double>
 -- !query output
 NULL	NULL	13	1.8571428571428572
 3	NULL	13	1.8571428571428572
@@ -371,7 +480,7 @@ FROM testData
 WHERE val is not null
 WINDOW w AS (PARTITION BY cate ORDER BY val)
 -- !query schema
-struct<cate:string,sum(CAST(val AS BIGINT)) OVER (PARTITION BY cate ORDER BY val ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):bigint>
+struct<cate:string,sum(val) OVER (PARTITION BY cate ORDER BY val ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):bigint>
 -- !query output
 NULL	3
 a	2
@@ -390,4 +499,623 @@ FROM testData ORDER BY cate, val
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-window aggregate function with filter predicate is not supported yet.;
+window aggregate function with filter predicate is not supported yet.
+
+
+-- !query
+SELECT
+    employee_name,
+    salary,
+    first_value(employee_name) OVER w highest_salary,
+    nth_value(employee_name, 2) OVER w second_highest_salary
+FROM
+    basic_pays
+WINDOW w AS (ORDER BY salary DESC)
+ORDER BY salary DESC
+-- !query schema
+struct<employee_name:string,salary:int,highest_salary:string,second_highest_salary:string>
+-- !query output
+Larry Bott	11798	Larry Bott	NULL
+Gerard Bondur	11472	Larry Bott	Gerard Bondur
+Pamela Castillo	11303	Larry Bott	Gerard Bondur
+Barry Jones	10586	Larry Bott	Gerard Bondur
+George Vanauf	10563	Larry Bott	Gerard Bondur
+Loui Bondur	10449	Larry Bott	Gerard Bondur
+Mary Patterson	9998	Larry Bott	Gerard Bondur
+Steve Patterson	9441	Larry Bott	Gerard Bondur
+Julie Firrelli	9181	Larry Bott	Gerard Bondur
+Jeff Firrelli	8992	Larry Bott	Gerard Bondur
+William Patterson	8870	Larry Bott	Gerard Bondur
+Diane Murphy	8435	Larry Bott	Gerard Bondur
+Leslie Jennings	8113	Larry Bott	Gerard Bondur
+Gerard Hernandez	6949	Larry Bott	Gerard Bondur
+Foon Yue Tseng	6660	Larry Bott	Gerard Bondur
+Anthony Bow	6627	Larry Bott	Gerard Bondur
+Leslie Thompson	5186	Larry Bott	Gerard Bondur
+
+
+-- !query
+SELECT
+    employee_name,
+    salary,
+    first_value(employee_name) OVER w highest_salary,
+    nth_value(employee_name, 2) OVER w second_highest_salary
+FROM
+    basic_pays
+WINDOW w AS (ORDER BY salary DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)
+ORDER BY salary DESC
+-- !query schema
+struct<employee_name:string,salary:int,highest_salary:string,second_highest_salary:string>
+-- !query output
+Larry Bott	11798	Larry Bott	NULL
+Gerard Bondur	11472	Larry Bott	Gerard Bondur
+Pamela Castillo	11303	Larry Bott	Gerard Bondur
+Barry Jones	10586	Larry Bott	Gerard Bondur
+George Vanauf	10563	Larry Bott	Gerard Bondur
+Loui Bondur	10449	Larry Bott	Gerard Bondur
+Mary Patterson	9998	Larry Bott	Gerard Bondur
+Steve Patterson	9441	Larry Bott	Gerard Bondur
+Julie Firrelli	9181	Larry Bott	Gerard Bondur
+Jeff Firrelli	8992	Larry Bott	Gerard Bondur
+William Patterson	8870	Larry Bott	Gerard Bondur
+Diane Murphy	8435	Larry Bott	Gerard Bondur
+Leslie Jennings	8113	Larry Bott	Gerard Bondur
+Gerard Hernandez	6949	Larry Bott	Gerard Bondur
+Foon Yue Tseng	6660	Larry Bott	Gerard Bondur
+Anthony Bow	6627	Larry Bott	Gerard Bondur
+Leslie Thompson	5186	Larry Bott	Gerard Bondur
+
+
+-- !query
+SELECT
+    employee_name,
+    salary,
+    first_value(employee_name) OVER w highest_salary,
+    nth_value(employee_name, 2) OVER w second_highest_salary
+FROM
+    basic_pays
+WINDOW w AS (ORDER BY salary DESC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)
+ORDER BY salary DESC
+-- !query schema
+struct<employee_name:string,salary:int,highest_salary:string,second_highest_salary:string>
+-- !query output
+Larry Bott	11798	Larry Bott	NULL
+Gerard Bondur	11472	Larry Bott	Gerard Bondur
+Pamela Castillo	11303	Larry Bott	Gerard Bondur
+Barry Jones	10586	Larry Bott	Gerard Bondur
+George Vanauf	10563	Larry Bott	Gerard Bondur
+Loui Bondur	10449	Larry Bott	Gerard Bondur
+Mary Patterson	9998	Larry Bott	Gerard Bondur
+Steve Patterson	9441	Larry Bott	Gerard Bondur
+Julie Firrelli	9181	Larry Bott	Gerard Bondur
+Jeff Firrelli	8992	Larry Bott	Gerard Bondur
+William Patterson	8870	Larry Bott	Gerard Bondur
+Diane Murphy	8435	Larry Bott	Gerard Bondur
+Leslie Jennings	8113	Larry Bott	Gerard Bondur
+Gerard Hernandez	6949	Larry Bott	Gerard Bondur
+Foon Yue Tseng	6660	Larry Bott	Gerard Bondur
+Anthony Bow	6627	Larry Bott	Gerard Bondur
+Leslie Thompson	5186	Larry Bott	Gerard Bondur
+
+
+-- !query
+SELECT
+    employee_name,
+    salary,
+    first_value(employee_name) OVER w highest_salary,
+    nth_value(employee_name, 2) OVER w second_highest_salary
+FROM
+    basic_pays
+WINDOW w AS (ORDER BY salary RANGE BETWEEN 2000 PRECEDING AND 1000 FOLLOWING)
+ORDER BY salary
+-- !query schema
+struct<employee_name:string,salary:int,highest_salary:string,second_highest_salary:string>
+-- !query output
+Leslie Thompson	5186	Leslie Thompson	NULL
+Anthony Bow	6627	Leslie Thompson	Anthony Bow
+Foon Yue Tseng	6660	Leslie Thompson	Anthony Bow
+Gerard Hernandez	6949	Leslie Thompson	Anthony Bow
+Leslie Jennings	8113	Anthony Bow	Foon Yue Tseng
+Diane Murphy	8435	Anthony Bow	Foon Yue Tseng
+William Patterson	8870	Gerard Hernandez	Leslie Jennings
+Jeff Firrelli	8992	Leslie Jennings	Diane Murphy
+Julie Firrelli	9181	Leslie Jennings	Diane Murphy
+Steve Patterson	9441	Leslie Jennings	Diane Murphy
+Mary Patterson	9998	Leslie Jennings	Diane Murphy
+Loui Bondur	10449	William Patterson	Jeff Firrelli
+George Vanauf	10563	William Patterson	Jeff Firrelli
+Barry Jones	10586	William Patterson	Jeff Firrelli
+Pamela Castillo	11303	Steve Patterson	Mary Patterson
+Gerard Bondur	11472	Mary Patterson	Loui Bondur
+Larry Bott	11798	Mary Patterson	Loui Bondur
+
+
+-- !query
+SELECT
+    employee_name,
+    salary,
+    first_value(employee_name) OVER w highest_salary,
+    nth_value(employee_name, 2) OVER w second_highest_salary
+FROM
+    basic_pays
+WINDOW w AS (ORDER BY salary DESC ROWS BETWEEN 2 PRECEDING AND 2 FOLLOWING)
+ORDER BY salary DESC
+-- !query schema
+struct<employee_name:string,salary:int,highest_salary:string,second_highest_salary:string>
+-- !query output
+Larry Bott	11798	Larry Bott	Gerard Bondur
+Gerard Bondur	11472	Larry Bott	Gerard Bondur
+Pamela Castillo	11303	Larry Bott	Gerard Bondur
+Barry Jones	10586	Gerard Bondur	Pamela Castillo
+George Vanauf	10563	Pamela Castillo	Barry Jones
+Loui Bondur	10449	Barry Jones	George Vanauf
+Mary Patterson	9998	George Vanauf	Loui Bondur
+Steve Patterson	9441	Loui Bondur	Mary Patterson
+Julie Firrelli	9181	Mary Patterson	Steve Patterson
+Jeff Firrelli	8992	Steve Patterson	Julie Firrelli
+William Patterson	8870	Julie Firrelli	Jeff Firrelli
+Diane Murphy	8435	Jeff Firrelli	William Patterson
+Leslie Jennings	8113	William Patterson	Diane Murphy
+Gerard Hernandez	6949	Diane Murphy	Leslie Jennings
+Foon Yue Tseng	6660	Leslie Jennings	Gerard Hernandez
+Anthony Bow	6627	Gerard Hernandez	Foon Yue Tseng
+Leslie Thompson	5186	Foon Yue Tseng	Anthony Bow
+
+
+-- !query
+SELECT
+    employee_name,
+    salary,
+    first_value(employee_name) OVER w highest_salary,
+    nth_value(employee_name, 2) OVER w second_highest_salary
+FROM
+    basic_pays
+WINDOW w AS (ORDER BY salary DESC RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
+ORDER BY salary DESC
+-- !query schema
+struct<employee_name:string,salary:int,highest_salary:string,second_highest_salary:string>
+-- !query output
+Larry Bott	11798	Larry Bott	Gerard Bondur
+Gerard Bondur	11472	Gerard Bondur	Pamela Castillo
+Pamela Castillo	11303	Pamela Castillo	Barry Jones
+Barry Jones	10586	Barry Jones	George Vanauf
+George Vanauf	10563	George Vanauf	Loui Bondur
+Loui Bondur	10449	Loui Bondur	Mary Patterson
+Mary Patterson	9998	Mary Patterson	Steve Patterson
+Steve Patterson	9441	Steve Patterson	Julie Firrelli
+Julie Firrelli	9181	Julie Firrelli	Jeff Firrelli
+Jeff Firrelli	8992	Jeff Firrelli	William Patterson
+William Patterson	8870	William Patterson	Diane Murphy
+Diane Murphy	8435	Diane Murphy	Leslie Jennings
+Leslie Jennings	8113	Leslie Jennings	Gerard Hernandez
+Gerard Hernandez	6949	Gerard Hernandez	Foon Yue Tseng
+Foon Yue Tseng	6660	Foon Yue Tseng	Anthony Bow
+Anthony Bow	6627	Anthony Bow	Leslie Thompson
+Leslie Thompson	5186	Leslie Thompson	NULL
+
+
+-- !query
+SELECT
+    employee_name,
+    salary,
+    first_value(employee_name) OVER w highest_salary,
+    nth_value(employee_name, 2) OVER w second_highest_salary
+FROM
+    basic_pays
+WINDOW w AS (ORDER BY salary DESC RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)
+ORDER BY salary DESC
+-- !query schema
+struct<employee_name:string,salary:int,highest_salary:string,second_highest_salary:string>
+-- !query output
+Larry Bott	11798	Larry Bott	Gerard Bondur
+Gerard Bondur	11472	Larry Bott	Gerard Bondur
+Pamela Castillo	11303	Larry Bott	Gerard Bondur
+Barry Jones	10586	Larry Bott	Gerard Bondur
+George Vanauf	10563	Larry Bott	Gerard Bondur
+Loui Bondur	10449	Larry Bott	Gerard Bondur
+Mary Patterson	9998	Larry Bott	Gerard Bondur
+Steve Patterson	9441	Larry Bott	Gerard Bondur
+Julie Firrelli	9181	Larry Bott	Gerard Bondur
+Jeff Firrelli	8992	Larry Bott	Gerard Bondur
+William Patterson	8870	Larry Bott	Gerard Bondur
+Diane Murphy	8435	Larry Bott	Gerard Bondur
+Leslie Jennings	8113	Larry Bott	Gerard Bondur
+Gerard Hernandez	6949	Larry Bott	Gerard Bondur
+Foon Yue Tseng	6660	Larry Bott	Gerard Bondur
+Anthony Bow	6627	Larry Bott	Gerard Bondur
+Leslie Thompson	5186	Larry Bott	Gerard Bondur
+
+
+-- !query
+SELECT
+    employee_name,
+    salary,
+    first_value(employee_name) OVER w highest_salary,
+    nth_value(employee_name, 2) OVER w second_highest_salary
+FROM
+    basic_pays
+WINDOW w AS (ORDER BY salary DESC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)
+ORDER BY salary DESC
+-- !query schema
+struct<employee_name:string,salary:int,highest_salary:string,second_highest_salary:string>
+-- !query output
+Larry Bott	11798	Larry Bott	Gerard Bondur
+Gerard Bondur	11472	Larry Bott	Gerard Bondur
+Pamela Castillo	11303	Larry Bott	Gerard Bondur
+Barry Jones	10586	Larry Bott	Gerard Bondur
+George Vanauf	10563	Larry Bott	Gerard Bondur
+Loui Bondur	10449	Larry Bott	Gerard Bondur
+Mary Patterson	9998	Larry Bott	Gerard Bondur
+Steve Patterson	9441	Larry Bott	Gerard Bondur
+Julie Firrelli	9181	Larry Bott	Gerard Bondur
+Jeff Firrelli	8992	Larry Bott	Gerard Bondur
+William Patterson	8870	Larry Bott	Gerard Bondur
+Diane Murphy	8435	Larry Bott	Gerard Bondur
+Leslie Jennings	8113	Larry Bott	Gerard Bondur
+Gerard Hernandez	6949	Larry Bott	Gerard Bondur
+Foon Yue Tseng	6660	Larry Bott	Gerard Bondur
+Anthony Bow	6627	Larry Bott	Gerard Bondur
+Leslie Thompson	5186	Larry Bott	Gerard Bondur
+
+
+-- !query
+SELECT
+    employee_name,
+    salary,
+    first_value(employee_name) OVER w highest_salary,
+    nth_value(employee_name, 2) OVER w second_highest_salary
+FROM
+    basic_pays
+WINDOW w AS (ORDER BY salary DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING)
+ORDER BY salary DESC
+-- !query schema
+struct<employee_name:string,salary:int,highest_salary:string,second_highest_salary:string>
+-- !query output
+Larry Bott	11798	Larry Bott	Gerard Bondur
+Gerard Bondur	11472	Larry Bott	Gerard Bondur
+Pamela Castillo	11303	Larry Bott	Gerard Bondur
+Barry Jones	10586	Larry Bott	Gerard Bondur
+George Vanauf	10563	Larry Bott	Gerard Bondur
+Loui Bondur	10449	Larry Bott	Gerard Bondur
+Mary Patterson	9998	Larry Bott	Gerard Bondur
+Steve Patterson	9441	Larry Bott	Gerard Bondur
+Julie Firrelli	9181	Larry Bott	Gerard Bondur
+Jeff Firrelli	8992	Larry Bott	Gerard Bondur
+William Patterson	8870	Larry Bott	Gerard Bondur
+Diane Murphy	8435	Larry Bott	Gerard Bondur
+Leslie Jennings	8113	Larry Bott	Gerard Bondur
+Gerard Hernandez	6949	Larry Bott	Gerard Bondur
+Foon Yue Tseng	6660	Larry Bott	Gerard Bondur
+Anthony Bow	6627	Larry Bott	Gerard Bondur
+Leslie Thompson	5186	Larry Bott	Gerard Bondur
+
+
+-- !query
+SELECT
+	employee_name,
+	department,
+	salary,
+	FIRST_VALUE(employee_name) OVER w highest_salary,
+	NTH_VALUE(employee_name, 2) OVER w second_highest_salary
+FROM
+	basic_pays
+WINDOW w AS (
+  PARTITION BY department
+  ORDER BY salary DESC
+  RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
+)
+ORDER BY department
+-- !query schema
+struct<employee_name:string,department:string,salary:int,highest_salary:string,second_highest_salary:string>
+-- !query output
+Gerard Bondur	Accounting	11472	Gerard Bondur	Mary Patterson
+Mary Patterson	Accounting	9998	Gerard Bondur	Mary Patterson
+Jeff Firrelli	Accounting	8992	Gerard Bondur	Mary Patterson
+William Patterson	Accounting	8870	Gerard Bondur	Mary Patterson
+Diane Murphy	Accounting	8435	Gerard Bondur	Mary Patterson
+Anthony Bow	Accounting	6627	Gerard Bondur	Mary Patterson
+Leslie Jennings	IT	8113	Leslie Jennings	Leslie Thompson
+Leslie Thompson	IT	5186	Leslie Jennings	Leslie Thompson
+Larry Bott	SCM	11798	Larry Bott	Pamela Castillo
+Pamela Castillo	SCM	11303	Larry Bott	Pamela Castillo
+Barry Jones	SCM	10586	Larry Bott	Pamela Castillo
+Loui Bondur	SCM	10449	Larry Bott	Pamela Castillo
+Gerard Hernandez	SCM	6949	Larry Bott	Pamela Castillo
+George Vanauf	Sales	10563	George Vanauf	Steve Patterson
+Steve Patterson	Sales	9441	George Vanauf	Steve Patterson
+Julie Firrelli	Sales	9181	George Vanauf	Steve Patterson
+Foon Yue Tseng	Sales	6660	George Vanauf	Steve Patterson
+
+
+-- !query
+SELECT
+    employee_name,
+    salary,
+    first_value(employee_name) OVER w highest_salary,
+    nth_value(employee_name, 2) OVER w second_highest_salary
+FROM
+    basic_pays
+WINDOW
+    w AS (ORDER BY salary DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING),
+    w AS (ORDER BY salary DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 2 FOLLOWING)
+ORDER BY salary DESC
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+The definition of window 'w' is repetitive(line 8, pos 0)
+
+== SQL ==
+SELECT
+    employee_name,
+    salary,
+    first_value(employee_name) OVER w highest_salary,
+    nth_value(employee_name, 2) OVER w second_highest_salary
+FROM
+    basic_pays
+WINDOW
+^^^
+    w AS (ORDER BY salary DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING),
+    w AS (ORDER BY salary DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 2 FOLLOWING)
+ORDER BY salary DESC
+
+
+-- !query
+SELECT
+    content,
+    id,
+    v,
+    lead(v, 0) IGNORE NULLS OVER w lead_0,
+    lead(v, 1) IGNORE NULLS OVER w lead_1,
+    lead(v, 2) IGNORE NULLS OVER w lead_2,
+    lead(v, 3) IGNORE NULLS OVER w lead_3,
+    lag(v, 0) IGNORE NULLS OVER w lag_0,
+    lag(v, 1) IGNORE NULLS OVER w lag_1,
+    lag(v, 2) IGNORE NULLS OVER w lag_2,
+    lag(v, 3) IGNORE NULLS OVER w lag_3,
+    nth_value(v, 1) IGNORE NULLS OVER w nth_value_1,
+    nth_value(v, 2) IGNORE NULLS OVER w nth_value_2,
+    nth_value(v, 3) IGNORE NULLS OVER w nth_value_3,
+    first_value(v) IGNORE NULLS OVER w first_value,
+    last_value(v) IGNORE NULLS OVER w last_value
+FROM
+    test_ignore_null
+WINDOW w AS (ORDER BY id)
+ORDER BY id
+-- !query schema
+struct<content:string,id:int,v:string,lead_0:string,lead_1:string,lead_2:string,lead_3:string,lag_0:string,lag_1:string,lag_2:string,lag_3:string,nth_value_1:string,nth_value_2:string,nth_value_3:string,first_value:string,last_value:string>
+-- !query output
+a	0	NULL	NULL	x	y	z	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL
+a	1	x	x	y	z	v	x	NULL	NULL	NULL	x	NULL	NULL	x	x
+b	2	NULL	NULL	y	z	v	NULL	x	NULL	NULL	x	NULL	NULL	x	x
+c	3	NULL	NULL	y	z	v	NULL	x	NULL	NULL	x	NULL	NULL	x	x
+a	4	y	y	z	v	NULL	y	x	NULL	NULL	x	y	NULL	x	y
+b	5	NULL	NULL	z	v	NULL	NULL	y	x	NULL	x	y	NULL	x	y
+a	6	z	z	v	NULL	NULL	z	y	x	NULL	x	y	z	x	z
+a	7	v	v	NULL	NULL	NULL	v	z	y	x	x	y	z	x	v
+a	8	NULL	NULL	NULL	NULL	NULL	NULL	v	z	y	x	y	z	x	v
+
+
+-- !query
+SELECT
+    content,
+    id,
+    v,
+    nth_value(v, 1) IGNORE NULLS OVER w nth_value_1,
+    nth_value(v, 2) IGNORE NULLS OVER w nth_value_2,
+    nth_value(v, 3) IGNORE NULLS OVER w nth_value_3,
+    first_value(v) IGNORE NULLS OVER w first_value,
+    last_value(v) IGNORE NULLS OVER w last_value
+FROM
+    test_ignore_null
+WINDOW w AS (ORDER BY id RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)
+ORDER BY id
+-- !query schema
+struct<content:string,id:int,v:string,nth_value_1:string,nth_value_2:string,nth_value_3:string,first_value:string,last_value:string>
+-- !query output
+a	0	NULL	NULL	NULL	NULL	NULL	NULL
+a	1	x	x	NULL	NULL	x	x
+b	2	NULL	x	NULL	NULL	x	x
+c	3	NULL	x	NULL	NULL	x	x
+a	4	y	x	y	NULL	x	y
+b	5	NULL	x	y	NULL	x	y
+a	6	z	x	y	z	x	z
+a	7	v	x	y	z	x	v
+a	8	NULL	x	y	z	x	v
+
+
+-- !query
+SELECT
+    content,
+    id,
+    v,
+    nth_value(v, 1) IGNORE NULLS OVER w nth_value_1,
+    nth_value(v, 2) IGNORE NULLS OVER w nth_value_2,
+    nth_value(v, 3) IGNORE NULLS OVER w nth_value_3,
+    first_value(v) IGNORE NULLS OVER w first_value,
+    last_value(v) IGNORE NULLS OVER w last_value
+FROM
+    test_ignore_null
+WINDOW w AS (ORDER BY id ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)
+ORDER BY id
+-- !query schema
+struct<content:string,id:int,v:string,nth_value_1:string,nth_value_2:string,nth_value_3:string,first_value:string,last_value:string>
+-- !query output
+a	0	NULL	NULL	NULL	NULL	NULL	NULL
+a	1	x	x	NULL	NULL	x	x
+b	2	NULL	x	NULL	NULL	x	x
+c	3	NULL	x	NULL	NULL	x	x
+a	4	y	x	y	NULL	x	y
+b	5	NULL	x	y	NULL	x	y
+a	6	z	x	y	z	x	z
+a	7	v	x	y	z	x	v
+a	8	NULL	x	y	z	x	v
+
+
+-- !query
+SELECT
+    content,
+    id,
+    v,
+    nth_value(v, 1) IGNORE NULLS OVER w nth_value_1,
+    nth_value(v, 2) IGNORE NULLS OVER w nth_value_2,
+    nth_value(v, 3) IGNORE NULLS OVER w nth_value_3,
+    first_value(v) IGNORE NULLS OVER w first_value,
+    last_value(v) IGNORE NULLS OVER w last_value
+FROM
+    test_ignore_null
+WINDOW w AS (ORDER BY id RANGE BETWEEN 2 PRECEDING AND 2 FOLLOWING)
+ORDER BY id
+-- !query schema
+struct<content:string,id:int,v:string,nth_value_1:string,nth_value_2:string,nth_value_3:string,first_value:string,last_value:string>
+-- !query output
+a	0	NULL	x	NULL	NULL	x	x
+a	1	x	x	NULL	NULL	x	x
+b	2	NULL	x	y	NULL	x	y
+c	3	NULL	x	y	NULL	x	y
+a	4	y	y	z	NULL	y	z
+b	5	NULL	y	z	v	y	v
+a	6	z	y	z	v	y	v
+a	7	v	z	v	NULL	z	v
+a	8	NULL	z	v	NULL	z	v
+
+
+-- !query
+SELECT
+    content,
+    id,
+    v,
+    nth_value(v, 1) IGNORE NULLS OVER w nth_value_1,
+    nth_value(v, 2) IGNORE NULLS OVER w nth_value_2,
+    nth_value(v, 3) IGNORE NULLS OVER w nth_value_3,
+    first_value(v) IGNORE NULLS OVER w first_value,
+    last_value(v) IGNORE NULLS OVER w last_value
+FROM
+    test_ignore_null
+WINDOW w AS (ORDER BY id ROWS BETWEEN 2 PRECEDING AND 2 FOLLOWING)
+ORDER BY id
+-- !query schema
+struct<content:string,id:int,v:string,nth_value_1:string,nth_value_2:string,nth_value_3:string,first_value:string,last_value:string>
+-- !query output
+a	0	NULL	x	NULL	NULL	x	x
+a	1	x	x	NULL	NULL	x	x
+b	2	NULL	x	y	NULL	x	y
+c	3	NULL	x	y	NULL	x	y
+a	4	y	y	z	NULL	y	z
+b	5	NULL	y	z	v	y	v
+a	6	z	y	z	v	y	v
+a	7	v	z	v	NULL	z	v
+a	8	NULL	z	v	NULL	z	v
+
+
+-- !query
+SELECT
+    content,
+    id,
+    v,
+    nth_value(v, 1) IGNORE NULLS OVER w nth_value_1,
+    nth_value(v, 2) IGNORE NULLS OVER w nth_value_2,
+    nth_value(v, 3) IGNORE NULLS OVER w nth_value_3,
+    first_value(v) IGNORE NULLS OVER w first_value,
+    last_value(v) IGNORE NULLS OVER w last_value
+FROM
+    test_ignore_null
+WINDOW w AS (ORDER BY id RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
+ORDER BY id
+-- !query schema
+struct<content:string,id:int,v:string,nth_value_1:string,nth_value_2:string,nth_value_3:string,first_value:string,last_value:string>
+-- !query output
+a	0	NULL	x	y	z	x	v
+a	1	x	x	y	z	x	v
+b	2	NULL	y	z	v	y	v
+c	3	NULL	y	z	v	y	v
+a	4	y	y	z	v	y	v
+b	5	NULL	z	v	NULL	z	v
+a	6	z	z	v	NULL	z	v
+a	7	v	v	NULL	NULL	v	v
+a	8	NULL	NULL	NULL	NULL	NULL	NULL
+
+
+-- !query
+SELECT
+    content,
+    id,
+    v,
+    nth_value(v, 1) IGNORE NULLS OVER w nth_value_1,
+    nth_value(v, 2) IGNORE NULLS OVER w nth_value_2,
+    nth_value(v, 3) IGNORE NULLS OVER w nth_value_3,
+    first_value(v) IGNORE NULLS OVER w first_value,
+    last_value(v) IGNORE NULLS OVER w last_value
+FROM
+    test_ignore_null
+WINDOW w AS (ORDER BY id RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)
+ORDER BY id
+-- !query schema
+struct<content:string,id:int,v:string,nth_value_1:string,nth_value_2:string,nth_value_3:string,first_value:string,last_value:string>
+-- !query output
+a	0	NULL	x	y	z	x	v
+a	1	x	x	y	z	x	v
+b	2	NULL	x	y	z	x	v
+c	3	NULL	x	y	z	x	v
+a	4	y	x	y	z	x	v
+b	5	NULL	x	y	z	x	v
+a	6	z	x	y	z	x	v
+a	7	v	x	y	z	x	v
+a	8	NULL	x	y	z	x	v
+
+
+-- !query
+SELECT
+    content,
+    id,
+    v,
+    nth_value(v, 1) IGNORE NULLS OVER w nth_value_1,
+    nth_value(v, 2) IGNORE NULLS OVER w nth_value_2,
+    nth_value(v, 3) IGNORE NULLS OVER w nth_value_3,
+    first_value(v) IGNORE NULLS OVER w first_value,
+    last_value(v) IGNORE NULLS OVER w last_value
+FROM
+    test_ignore_null
+WINDOW w AS (ORDER BY id ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)
+ORDER BY id
+-- !query schema
+struct<content:string,id:int,v:string,nth_value_1:string,nth_value_2:string,nth_value_3:string,first_value:string,last_value:string>
+-- !query output
+a	0	NULL	x	y	z	x	v
+a	1	x	x	y	z	x	v
+b	2	NULL	x	y	z	x	v
+c	3	NULL	x	y	z	x	v
+a	4	y	x	y	z	x	v
+b	5	NULL	x	y	z	x	v
+a	6	z	x	y	z	x	v
+a	7	v	x	y	z	x	v
+a	8	NULL	x	y	z	x	v
+
+
+-- !query
+SELECT
+    content,
+    id,
+    v,
+    nth_value(v, 1) IGNORE NULLS OVER w nth_value_1,
+    nth_value(v, 2) IGNORE NULLS OVER w nth_value_2,
+    nth_value(v, 3) IGNORE NULLS OVER w nth_value_3,
+    first_value(v) IGNORE NULLS OVER w first_value,
+    last_value(v) IGNORE NULLS OVER w last_value
+FROM
+    test_ignore_null
+WINDOW w AS (ORDER BY id ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING)
+ORDER BY id
+-- !query schema
+struct<content:string,id:int,v:string,nth_value_1:string,nth_value_2:string,nth_value_3:string,first_value:string,last_value:string>
+-- !query output
+a	0	NULL	x	NULL	NULL	x	x
+a	1	x	x	NULL	NULL	x	x
+b	2	NULL	x	NULL	NULL	x	x
+c	3	NULL	x	y	NULL	x	y
+a	4	y	x	y	NULL	x	y
+b	5	NULL	x	y	z	x	z
+a	6	z	x	y	z	x	v
+a	7	v	x	y	z	x	v
+a	8	NULL	x	y	z	x	v
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-common-functions/commits/0 b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-common-functions/commits/0
new file mode 100644
index 0000000000000..9c1e3021c3ead
--- /dev/null
+++ b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-common-functions/commits/0
@@ -0,0 +1,2 @@
+v1
+{"nextBatchWatermarkMs":0}
\ No newline at end of file
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-common-functions/metadata b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-common-functions/metadata
new file mode 100644
index 0000000000000..26a0d5d707d44
--- /dev/null
+++ b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-common-functions/metadata
@@ -0,0 +1 @@
+{"id":"5f1362da-99ce-4ba2-97e1-d992c295e563"}
\ No newline at end of file
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-common-functions/offsets/0 b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-common-functions/offsets/0
new file mode 100644
index 0000000000000..43cde63b8f68d
--- /dev/null
+++ b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-common-functions/offsets/0
@@ -0,0 +1,3 @@
+v1
+{"batchWatermarkMs":0,"batchTimestampMs":1591241624051,"conf":{"spark.sql.streaming.stateStore.providerClass":"org.apache.spark.sql.execution.streaming.state.HDFSBackedStateStoreProvider","spark.sql.streaming.flatMapGroupsWithState.stateFormatVersion":"2","spark.sql.streaming.multipleWatermarkPolicy":"min","spark.sql.streaming.aggregation.stateFormatVersion":"2","spark.sql.shuffle.partitions":"5"}}
+0
\ No newline at end of file
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-common-functions/state/0/0/1.delta b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-common-functions/state/0/0/1.delta
new file mode 100644
index 0000000000000..6352978051846
Binary files /dev/null and b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-common-functions/state/0/0/1.delta differ
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-common-functions/state/0/1/1.delta b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-common-functions/state/0/1/1.delta
new file mode 100644
index 0000000000000..e7de33ff90ca7
Binary files /dev/null and b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-common-functions/state/0/1/1.delta differ
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-common-functions/state/0/2/1.delta b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-common-functions/state/0/2/1.delta
new file mode 100644
index 0000000000000..3769f794e6469
Binary files /dev/null and b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-common-functions/state/0/2/1.delta differ
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-common-functions/state/0/3/1.delta b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-common-functions/state/0/3/1.delta
new file mode 100644
index 0000000000000..82386b020a045
Binary files /dev/null and b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-common-functions/state/0/3/1.delta differ
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-common-functions/state/0/4/1.delta b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-common-functions/state/0/4/1.delta
new file mode 100644
index 0000000000000..ac70b46fc576d
Binary files /dev/null and b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-common-functions/state/0/4/1.delta differ
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-deduplicate/commits/0 b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-deduplicate/commits/0
new file mode 100644
index 0000000000000..9c1e3021c3ead
--- /dev/null
+++ b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-deduplicate/commits/0
@@ -0,0 +1,2 @@
+v1
+{"nextBatchWatermarkMs":0}
\ No newline at end of file
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-deduplicate/metadata b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-deduplicate/metadata
new file mode 100644
index 0000000000000..0b78699f07bad
--- /dev/null
+++ b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-deduplicate/metadata
@@ -0,0 +1 @@
+{"id":"26fe8d3d-d101-44b0-b9c1-a2f9f09cea69"}
\ No newline at end of file
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-deduplicate/offsets/0 b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-deduplicate/offsets/0
new file mode 100644
index 0000000000000..3c0c901ee0c38
--- /dev/null
+++ b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-deduplicate/offsets/0
@@ -0,0 +1,3 @@
+v1
+{"batchWatermarkMs":0,"batchTimestampMs":1592306585407,"conf":{"spark.sql.streaming.stateStore.providerClass":"org.apache.spark.sql.execution.streaming.state.HDFSBackedStateStoreProvider","spark.sql.streaming.flatMapGroupsWithState.stateFormatVersion":"2","spark.sql.streaming.multipleWatermarkPolicy":"min","spark.sql.streaming.aggregation.stateFormatVersion":"2","spark.sql.shuffle.partitions":"5"}}
+0
\ No newline at end of file
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-deduplicate/state/0/0/1.delta b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-deduplicate/state/0/0/1.delta
new file mode 100644
index 0000000000000..6352978051846
Binary files /dev/null and b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-deduplicate/state/0/0/1.delta differ
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-deduplicate/state/0/1/1.delta b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-deduplicate/state/0/1/1.delta
new file mode 100644
index 0000000000000..71ab60304105a
Binary files /dev/null and b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-deduplicate/state/0/1/1.delta differ
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-deduplicate/state/0/2/1.delta b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-deduplicate/state/0/2/1.delta
new file mode 100644
index 0000000000000..6352978051846
Binary files /dev/null and b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-deduplicate/state/0/2/1.delta differ
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-deduplicate/state/0/3/1.delta b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-deduplicate/state/0/3/1.delta
new file mode 100644
index 0000000000000..eaaee05d674dc
Binary files /dev/null and b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-deduplicate/state/0/3/1.delta differ
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-deduplicate/state/0/4/1.delta b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-deduplicate/state/0/4/1.delta
new file mode 100644
index 0000000000000..9a014b2029cec
Binary files /dev/null and b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-deduplicate/state/0/4/1.delta differ
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-statistical-functions/commits/0 b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-statistical-functions/commits/0
new file mode 100644
index 0000000000000..9c1e3021c3ead
--- /dev/null
+++ b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-statistical-functions/commits/0
@@ -0,0 +1,2 @@
+v1
+{"nextBatchWatermarkMs":0}
\ No newline at end of file
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-statistical-functions/metadata b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-statistical-functions/metadata
new file mode 100644
index 0000000000000..196f2e0e5c9f3
--- /dev/null
+++ b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-statistical-functions/metadata
@@ -0,0 +1 @@
+{"id":"8fc17276-e48b-4e8a-a9c9-31f0045ed860"}
\ No newline at end of file
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-statistical-functions/offsets/0 b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-statistical-functions/offsets/0
new file mode 100644
index 0000000000000..622db1a5f4f5b
--- /dev/null
+++ b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-statistical-functions/offsets/0
@@ -0,0 +1,3 @@
+v1
+{"batchWatermarkMs":0,"batchTimestampMs":1591253127196,"conf":{"spark.sql.streaming.stateStore.providerClass":"org.apache.spark.sql.execution.streaming.state.HDFSBackedStateStoreProvider","spark.sql.streaming.flatMapGroupsWithState.stateFormatVersion":"2","spark.sql.streaming.multipleWatermarkPolicy":"min","spark.sql.streaming.aggregation.stateFormatVersion":"2","spark.sql.shuffle.partitions":"5"}}
+0
\ No newline at end of file
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-statistical-functions/state/0/0/1.delta b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-statistical-functions/state/0/0/1.delta
new file mode 100644
index 0000000000000..31a67ed57d7d1
Binary files /dev/null and b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-statistical-functions/state/0/0/1.delta differ
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-statistical-functions/state/0/1/1.delta b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-statistical-functions/state/0/1/1.delta
new file mode 100644
index 0000000000000..6352978051846
Binary files /dev/null and b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-statistical-functions/state/0/1/1.delta differ
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-statistical-functions/state/0/2/1.delta b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-statistical-functions/state/0/2/1.delta
new file mode 100644
index 0000000000000..2198a7736401a
Binary files /dev/null and b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-statistical-functions/state/0/2/1.delta differ
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-statistical-functions/state/0/3/1.delta b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-statistical-functions/state/0/3/1.delta
new file mode 100644
index 0000000000000..e0a21939d2645
Binary files /dev/null and b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-statistical-functions/state/0/3/1.delta differ
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-statistical-functions/state/0/4/1.delta b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-statistical-functions/state/0/4/1.delta
new file mode 100644
index 0000000000000..91e1e2dbf560b
Binary files /dev/null and b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-statistical-functions/state/0/4/1.delta differ
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/commits/0 b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/commits/0
new file mode 100644
index 0000000000000..9c1e3021c3ead
--- /dev/null
+++ b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/commits/0
@@ -0,0 +1,2 @@
+v1
+{"nextBatchWatermarkMs":0}
\ No newline at end of file
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/metadata b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/metadata
new file mode 100644
index 0000000000000..1f8077da27c29
--- /dev/null
+++ b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/metadata
@@ -0,0 +1 @@
+{"id":"ac9f2680-3a39-4afd-824b-7beefdf7d7a7"}
\ No newline at end of file
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/offsets/0 b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/offsets/0
new file mode 100644
index 0000000000000..b0b5ea1df545a
--- /dev/null
+++ b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/offsets/0
@@ -0,0 +1,3 @@
+v1
+{"batchWatermarkMs":0,"batchTimestampMs":1591234028611,"conf":{"spark.sql.streaming.stateStore.providerClass":"org.apache.spark.sql.execution.streaming.state.HDFSBackedStateStoreProvider","spark.sql.streaming.flatMapGroupsWithState.stateFormatVersion":"2","spark.sql.streaming.multipleWatermarkPolicy":"min","spark.sql.streaming.aggregation.stateFormatVersion":"2","spark.sql.shuffle.partitions":"5"}}
+0
\ No newline at end of file
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/0/1.delta b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/0/1.delta
new file mode 100644
index 0000000000000..6352978051846
Binary files /dev/null and b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/0/1.delta differ
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/1/1.delta b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/1/1.delta
new file mode 100644
index 0000000000000..99110e438ff2f
Binary files /dev/null and b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/1/1.delta differ
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/2/1.delta b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/2/1.delta
new file mode 100644
index 0000000000000..6352978051846
Binary files /dev/null and b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/2/1.delta differ
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/3/1.delta b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/3/1.delta
new file mode 100644
index 0000000000000..cec2e6be7c9f7
Binary files /dev/null and b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/3/1.delta differ
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4/1.delta b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4/1.delta
new file mode 100644
index 0000000000000..6352978051846
Binary files /dev/null and b/sql/core/src/test/resources/structured-streaming/checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4/1.delta differ
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-3.0.0-streaming-statestore-codec/commits/0 b/sql/core/src/test/resources/structured-streaming/checkpoint-version-3.0.0-streaming-statestore-codec/commits/0
new file mode 100644
index 0000000000000..9c1e3021c3ead
--- /dev/null
+++ b/sql/core/src/test/resources/structured-streaming/checkpoint-version-3.0.0-streaming-statestore-codec/commits/0
@@ -0,0 +1,2 @@
+v1
+{"nextBatchWatermarkMs":0}
\ No newline at end of file
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-3.0.0-streaming-statestore-codec/metadata b/sql/core/src/test/resources/structured-streaming/checkpoint-version-3.0.0-streaming-statestore-codec/metadata
new file mode 100644
index 0000000000000..df5937f800382
--- /dev/null
+++ b/sql/core/src/test/resources/structured-streaming/checkpoint-version-3.0.0-streaming-statestore-codec/metadata
@@ -0,0 +1 @@
+{"id":"6bcf6671-d23e-4ad8-824f-98aa5924ce6d"}
\ No newline at end of file
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-3.0.0-streaming-statestore-codec/offsets/0 b/sql/core/src/test/resources/structured-streaming/checkpoint-version-3.0.0-streaming-statestore-codec/offsets/0
new file mode 100644
index 0000000000000..d12f52147dd6a
--- /dev/null
+++ b/sql/core/src/test/resources/structured-streaming/checkpoint-version-3.0.0-streaming-statestore-codec/offsets/0
@@ -0,0 +1,3 @@
+v1
+{"batchWatermarkMs":0,"batchTimestampMs":1603918440918,"conf":{"spark.sql.streaming.stateStore.providerClass":"org.apache.spark.sql.execution.streaming.state.HDFSBackedStateStoreProvider","spark.sql.streaming.join.stateFormatVersion":"2","spark.sql.streaming.flatMapGroupsWithState.stateFormatVersion":"2","spark.sql.streaming.multipleWatermarkPolicy":"min","spark.sql.streaming.aggregation.stateFormatVersion":"2","spark.sql.shuffle.partitions":"1"}}
+0
\ No newline at end of file
diff --git a/sql/core/src/test/resources/structured-streaming/checkpoint-version-3.0.0-streaming-statestore-codec/state/0/0/1.delta b/sql/core/src/test/resources/structured-streaming/checkpoint-version-3.0.0-streaming-statestore-codec/state/0/0/1.delta
new file mode 100644
index 0000000000000..8de7bc89a5de8
Binary files /dev/null and b/sql/core/src/test/resources/structured-streaming/checkpoint-version-3.0.0-streaming-statestore-codec/state/0/0/1.delta differ
diff --git a/sql/core/src/test/resources/structured-streaming/file-sink-log-version-2.1.0/8 b/sql/core/src/test/resources/structured-streaming/file-sink-log-version-2.1.0/8
index e7989804e8886..dd0513f16a1ad 100644
--- a/sql/core/src/test/resources/structured-streaming/file-sink-log-version-2.1.0/8
+++ b/sql/core/src/test/resources/structured-streaming/file-sink-log-version-2.1.0/8
@@ -1,3 +1,2 @@
 v1
 {"path":"/a/b/8","size":800,"isDir":false,"modificationTime":800,"blockReplication":1,"blockSize":100,"action":"add"}
-{"path":"/a/b/0","size":100,"isDir":false,"modificationTime":100,"blockReplication":1,"blockSize":100,"action":"delete"}
diff --git a/sql/core/src/test/resources/test-data/before_1582_date_v2_4.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_date_v2_4.snappy.parquet
deleted file mode 100644
index 7d5cc12eefe04..0000000000000
Binary files a/sql/core/src/test/resources/test-data/before_1582_date_v2_4.snappy.parquet and /dev/null differ
diff --git a/sql/core/src/test/resources/test-data/before_1582_date_v2_4_5.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_date_v2_4_5.snappy.parquet
new file mode 100644
index 0000000000000..edd61c9b9fec8
Binary files /dev/null and b/sql/core/src/test/resources/test-data/before_1582_date_v2_4_5.snappy.parquet differ
diff --git a/sql/core/src/test/resources/test-data/before_1582_date_v2_4_6.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_date_v2_4_6.snappy.parquet
new file mode 100644
index 0000000000000..01f4887f5e994
Binary files /dev/null and b/sql/core/src/test/resources/test-data/before_1582_date_v2_4_6.snappy.parquet differ
diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_dict_v2_4_5.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_dict_v2_4_5.snappy.parquet
new file mode 100644
index 0000000000000..c7e8d3926f63a
Binary files /dev/null and b/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_dict_v2_4_5.snappy.parquet differ
diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_dict_v2_4_6.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_dict_v2_4_6.snappy.parquet
new file mode 100644
index 0000000000000..939e2b8088eb0
Binary files /dev/null and b/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_dict_v2_4_6.snappy.parquet differ
diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_plain_v2_4_5.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_plain_v2_4_5.snappy.parquet
new file mode 100644
index 0000000000000..88a94ac482052
Binary files /dev/null and b/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_plain_v2_4_5.snappy.parquet differ
diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_plain_v2_4_6.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_plain_v2_4_6.snappy.parquet
new file mode 100644
index 0000000000000..68bfa33aac13f
Binary files /dev/null and b/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_plain_v2_4_6.snappy.parquet differ
diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_v2_4.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_v2_4.snappy.parquet
deleted file mode 100644
index 13254bd93a5e6..0000000000000
Binary files a/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_v2_4.snappy.parquet and /dev/null differ
diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4.snappy.parquet
deleted file mode 100644
index 7d2b46e9bea41..0000000000000
Binary files a/sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4.snappy.parquet and /dev/null differ
diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4_5.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4_5.snappy.parquet
new file mode 100644
index 0000000000000..62e6048354dc1
Binary files /dev/null and b/sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4_5.snappy.parquet differ
diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4_6.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4_6.snappy.parquet
new file mode 100644
index 0000000000000..d7fdaa3e67212
Binary files /dev/null and b/sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4_6.snappy.parquet differ
diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4.snappy.parquet
deleted file mode 100644
index e9825455c2015..0000000000000
Binary files a/sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4.snappy.parquet and /dev/null differ
diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4_5.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4_5.snappy.parquet
new file mode 100644
index 0000000000000..a7cef9e60f134
Binary files /dev/null and b/sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4_5.snappy.parquet differ
diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4_6.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4_6.snappy.parquet
new file mode 100644
index 0000000000000..4c213f4540a73
Binary files /dev/null and b/sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4_6.snappy.parquet differ
diff --git a/sql/core/src/test/resources/test-data/decimal32-written-as-64-bit-dict.snappy.parquet b/sql/core/src/test/resources/test-data/decimal32-written-as-64-bit-dict.snappy.parquet
new file mode 100644
index 0000000000000..e19bd7ae01b85
Binary files /dev/null and b/sql/core/src/test/resources/test-data/decimal32-written-as-64-bit-dict.snappy.parquet differ
diff --git a/sql/core/src/test/resources/test-data/decimal32-written-as-64-bit.snappy.parquet b/sql/core/src/test/resources/test-data/decimal32-written-as-64-bit.snappy.parquet
new file mode 100644
index 0000000000000..1068a4e413235
Binary files /dev/null and b/sql/core/src/test/resources/test-data/decimal32-written-as-64-bit.snappy.parquet differ
diff --git a/sql/core/src/test/resources/test-data/percentile_approx-input.csv.bz2 b/sql/core/src/test/resources/test-data/percentile_approx-input.csv.bz2
new file mode 100644
index 0000000000000..f85e2896b3a89
Binary files /dev/null and b/sql/core/src/test/resources/test-data/percentile_approx-input.csv.bz2 differ
diff --git a/sql/hive/src/test/resources/test_script.py b/sql/core/src/test/resources/test_script.py
similarity index 97%
rename from sql/hive/src/test/resources/test_script.py
rename to sql/core/src/test/resources/test_script.py
index 82ef7b38f0c1b..4fcd483f44d43 100644
--- a/sql/hive/src/test/resources/test_script.py
+++ b/sql/core/src/test/resources/test_script.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q10.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q10.sf100/explain.txt
new file mode 100644
index 0000000000000..214c9260016bf
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q10.sf100/explain.txt
@@ -0,0 +1,303 @@
+== Physical Plan ==
+TakeOrderedAndProject (52)
++- * HashAggregate (51)
+   +- Exchange (50)
+      +- * HashAggregate (49)
+         +- * Project (48)
+            +- * BroadcastHashJoin Inner BuildLeft (47)
+               :- BroadcastExchange (43)
+               :  +- * Project (42)
+               :     +- * BroadcastHashJoin Inner BuildRight (41)
+               :        :- * Project (35)
+               :        :  +- * SortMergeJoin LeftSemi (34)
+               :        :     :- * SortMergeJoin LeftSemi (25)
+               :        :     :  :- * Sort (5)
+               :        :     :  :  +- Exchange (4)
+               :        :     :  :     +- * Filter (3)
+               :        :     :  :        +- * ColumnarToRow (2)
+               :        :     :  :           +- Scan parquet default.customer (1)
+               :        :     :  +- * Sort (24)
+               :        :     :     +- Exchange (23)
+               :        :     :        +- Union (22)
+               :        :     :           :- * Project (15)
+               :        :     :           :  +- * BroadcastHashJoin Inner BuildRight (14)
+               :        :     :           :     :- * Filter (8)
+               :        :     :           :     :  +- * ColumnarToRow (7)
+               :        :     :           :     :     +- Scan parquet default.web_sales (6)
+               :        :     :           :     +- BroadcastExchange (13)
+               :        :     :           :        +- * Project (12)
+               :        :     :           :           +- * Filter (11)
+               :        :     :           :              +- * ColumnarToRow (10)
+               :        :     :           :                 +- Scan parquet default.date_dim (9)
+               :        :     :           +- * Project (21)
+               :        :     :              +- * BroadcastHashJoin Inner BuildRight (20)
+               :        :     :                 :- * Filter (18)
+               :        :     :                 :  +- * ColumnarToRow (17)
+               :        :     :                 :     +- Scan parquet default.catalog_sales (16)
+               :        :     :                 +- ReusedExchange (19)
+               :        :     +- * Sort (33)
+               :        :        +- Exchange (32)
+               :        :           +- * Project (31)
+               :        :              +- * BroadcastHashJoin Inner BuildRight (30)
+               :        :                 :- * Filter (28)
+               :        :                 :  +- * ColumnarToRow (27)
+               :        :                 :     +- Scan parquet default.store_sales (26)
+               :        :                 +- ReusedExchange (29)
+               :        +- BroadcastExchange (40)
+               :           +- * Project (39)
+               :              +- * Filter (38)
+               :                 +- * ColumnarToRow (37)
+               :                    +- Scan parquet default.customer_address (36)
+               +- * Filter (46)
+                  +- * ColumnarToRow (45)
+                     +- Scan parquet default.customer_demographics (44)
+
+
+(1) Scan parquet default.customer
+Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_addr_sk:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+
+(3) Filter [codegen id : 1]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+Condition : ((isnotnull(c_customer_sk#1) AND isnotnull(c_current_addr_sk#3)) AND isnotnull(c_current_cdemo_sk#2))
+
+(4) Exchange
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+Arguments: hashpartitioning(c_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#4]
+
+(5) Sort [codegen id : 2]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+Arguments: [c_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.web_sales
+Output [2]: [ws_bill_customer_sk#5, ws_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#6), dynamicpruningexpression(ws_sold_date_sk#6 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int>
+
+(7) ColumnarToRow [codegen id : 4]
+Input [2]: [ws_bill_customer_sk#5, ws_sold_date_sk#6]
+
+(8) Filter [codegen id : 4]
+Input [2]: [ws_bill_customer_sk#5, ws_sold_date_sk#6]
+Condition : isnotnull(ws_bill_customer_sk#5)
+
+(9) Scan parquet default.date_dim
+Output [3]: [d_date_sk#8, d_year#9, d_moy#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,7), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(10) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#8, d_year#9, d_moy#10]
+
+(11) Filter [codegen id : 3]
+Input [3]: [d_date_sk#8, d_year#9, d_moy#10]
+Condition : (((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2002)) AND (d_moy#10 >= 4)) AND (d_moy#10 <= 7)) AND isnotnull(d_date_sk#8))
+
+(12) Project [codegen id : 3]
+Output [1]: [d_date_sk#8]
+Input [3]: [d_date_sk#8, d_year#9, d_moy#10]
+
+(13) BroadcastExchange
+Input [1]: [d_date_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#11]
+
+(14) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ws_sold_date_sk#6]
+Right keys [1]: [d_date_sk#8]
+Join condition: None
+
+(15) Project [codegen id : 4]
+Output [1]: [ws_bill_customer_sk#5 AS customer_sk#12]
+Input [3]: [ws_bill_customer_sk#5, ws_sold_date_sk#6, d_date_sk#8]
+
+(16) Scan parquet default.catalog_sales
+Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#14), dynamicpruningexpression(cs_sold_date_sk#14 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(cs_ship_customer_sk)]
+ReadSchema: struct<cs_ship_customer_sk:int>
+
+(17) ColumnarToRow [codegen id : 6]
+Input [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14]
+
+(18) Filter [codegen id : 6]
+Input [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14]
+Condition : isnotnull(cs_ship_customer_sk#13)
+
+(19) ReusedExchange [Reuses operator id: 13]
+Output [1]: [d_date_sk#15]
+
+(20) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [cs_sold_date_sk#14]
+Right keys [1]: [d_date_sk#15]
+Join condition: None
+
+(21) Project [codegen id : 6]
+Output [1]: [cs_ship_customer_sk#13 AS customer_sk#16]
+Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#15]
+
+(22) Union
+
+(23) Exchange
+Input [1]: [customer_sk#12]
+Arguments: hashpartitioning(customer_sk#12, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(24) Sort [codegen id : 7]
+Input [1]: [customer_sk#12]
+Arguments: [customer_sk#12 ASC NULLS FIRST], false, 0
+
+(25) SortMergeJoin [codegen id : 8]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [customer_sk#12]
+Join condition: None
+
+(26) Scan parquet default.store_sales
+Output [2]: [ss_customer_sk#18, ss_sold_date_sk#19]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#19), dynamicpruningexpression(ss_sold_date_sk#19 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int>
+
+(27) ColumnarToRow [codegen id : 10]
+Input [2]: [ss_customer_sk#18, ss_sold_date_sk#19]
+
+(28) Filter [codegen id : 10]
+Input [2]: [ss_customer_sk#18, ss_sold_date_sk#19]
+Condition : isnotnull(ss_customer_sk#18)
+
+(29) ReusedExchange [Reuses operator id: 13]
+Output [1]: [d_date_sk#20]
+
+(30) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_sold_date_sk#19]
+Right keys [1]: [d_date_sk#20]
+Join condition: None
+
+(31) Project [codegen id : 10]
+Output [1]: [ss_customer_sk#18 AS customer_sk#21]
+Input [3]: [ss_customer_sk#18, ss_sold_date_sk#19, d_date_sk#20]
+
+(32) Exchange
+Input [1]: [customer_sk#21]
+Arguments: hashpartitioning(customer_sk#21, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(33) Sort [codegen id : 11]
+Input [1]: [customer_sk#21]
+Arguments: [customer_sk#21 ASC NULLS FIRST], false, 0
+
+(34) SortMergeJoin [codegen id : 13]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [customer_sk#21]
+Join condition: None
+
+(35) Project [codegen id : 13]
+Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+
+(36) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#23, ca_county#24]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [In(ca_county, [Walker County,Richland County,Gaines County,Douglas County,Dona Ana County]), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_county:string>
+
+(37) ColumnarToRow [codegen id : 12]
+Input [2]: [ca_address_sk#23, ca_county#24]
+
+(38) Filter [codegen id : 12]
+Input [2]: [ca_address_sk#23, ca_county#24]
+Condition : (ca_county#24 IN (Walker County,Richland County,Gaines County,Douglas County,Dona Ana County) AND isnotnull(ca_address_sk#23))
+
+(39) Project [codegen id : 12]
+Output [1]: [ca_address_sk#23]
+Input [2]: [ca_address_sk#23, ca_county#24]
+
+(40) BroadcastExchange
+Input [1]: [ca_address_sk#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#25]
+
+(41) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [c_current_addr_sk#3]
+Right keys [1]: [ca_address_sk#23]
+Join condition: None
+
+(42) Project [codegen id : 13]
+Output [1]: [c_current_cdemo_sk#2]
+Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#23]
+
+(43) BroadcastExchange
+Input [1]: [c_current_cdemo_sk#2]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#26]
+
+(44) Scan parquet default.customer_demographics
+Output [9]: [cd_demo_sk#27, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_education_status:string,cd_purchase_estimate:int,cd_credit_rating:string,cd_dep_count:int,cd_dep_employed_count:int,cd_dep_college_count:int>
+
+(45) ColumnarToRow
+Input [9]: [cd_demo_sk#27, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35]
+
+(46) Filter
+Input [9]: [cd_demo_sk#27, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35]
+Condition : isnotnull(cd_demo_sk#27)
+
+(47) BroadcastHashJoin [codegen id : 14]
+Left keys [1]: [c_current_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#27]
+Join condition: None
+
+(48) Project [codegen id : 14]
+Output [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35]
+Input [10]: [c_current_cdemo_sk#2, cd_demo_sk#27, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35]
+
+(49) HashAggregate [codegen id : 14]
+Input [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35]
+Keys [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#36]
+Results [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35, count#37]
+
+(50) Exchange
+Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35, count#37]
+Arguments: hashpartitioning(cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35, 5), ENSURE_REQUIREMENTS, [id=#38]
+
+(51) HashAggregate [codegen id : 15]
+Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35, count#37]
+Keys [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#39]
+Results [14]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, count(1)#39 AS cnt1#40, cd_purchase_estimate#31, count(1)#39 AS cnt2#41, cd_credit_rating#32, count(1)#39 AS cnt3#42, cd_dep_count#33, count(1)#39 AS cnt4#43, cd_dep_employed_count#34, count(1)#39 AS cnt5#44, cd_dep_college_count#35, count(1)#39 AS cnt6#45]
+
+(52) TakeOrderedAndProject
+Input [14]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#40, cd_purchase_estimate#31, cnt2#41, cd_credit_rating#32, cnt3#42, cd_dep_count#33, cnt4#43, cd_dep_employed_count#34, cnt5#44, cd_dep_college_count#35, cnt6#45]
+Arguments: 100, [cd_gender#28 ASC NULLS FIRST, cd_marital_status#29 ASC NULLS FIRST, cd_education_status#30 ASC NULLS FIRST, cd_purchase_estimate#31 ASC NULLS FIRST, cd_credit_rating#32 ASC NULLS FIRST, cd_dep_count#33 ASC NULLS FIRST, cd_dep_employed_count#34 ASC NULLS FIRST, cd_dep_college_count#35 ASC NULLS FIRST], [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#40, cd_purchase_estimate#31, cnt2#41, cd_credit_rating#32, cnt3#42, cd_dep_count#33, cnt4#43, cd_dep_employed_count#34, cnt5#44, cd_dep_college_count#35, cnt6#45]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 6 Hosting Expression = ws_sold_date_sk#6 IN dynamicpruning#7
+ReusedExchange (53)
+
+
+(53) ReusedExchange [Reuses operator id: 13]
+Output [1]: [d_date_sk#8]
+
+Subquery:2 Hosting operator id = 16 Hosting Expression = cs_sold_date_sk#14 IN dynamicpruning#7
+
+Subquery:3 Hosting operator id = 26 Hosting Expression = ss_sold_date_sk#19 IN dynamicpruning#7
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q10.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q10.sf100/simplified.txt
new file mode 100644
index 0000000000000..af1e7c10e8e4d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q10.sf100/simplified.txt
@@ -0,0 +1,89 @@
+TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,cnt2,cnt3,cnt4,cnt5,cnt6]
+  WholeStageCodegen (15)
+    HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] [count(1),cnt1,cnt2,cnt3,cnt4,cnt5,cnt6,count]
+      InputAdapter
+        Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1
+          WholeStageCodegen (14)
+            HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count]
+              Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count]
+                BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk]
+                  InputAdapter
+                    BroadcastExchange #2
+                      WholeStageCodegen (13)
+                        Project [c_current_cdemo_sk]
+                          BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                            Project [c_current_cdemo_sk,c_current_addr_sk]
+                              SortMergeJoin [c_customer_sk,customer_sk]
+                                InputAdapter
+                                  WholeStageCodegen (8)
+                                    SortMergeJoin [c_customer_sk,customer_sk]
+                                      InputAdapter
+                                        WholeStageCodegen (2)
+                                          Sort [c_customer_sk]
+                                            InputAdapter
+                                              Exchange [c_customer_sk] #3
+                                                WholeStageCodegen (1)
+                                                  Filter [c_customer_sk,c_current_addr_sk,c_current_cdemo_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk]
+                                      InputAdapter
+                                        WholeStageCodegen (7)
+                                          Sort [customer_sk]
+                                            InputAdapter
+                                              Exchange [customer_sk] #4
+                                                Union
+                                                  WholeStageCodegen (4)
+                                                    Project [ws_bill_customer_sk]
+                                                      BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                        Filter [ws_bill_customer_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.web_sales [ws_bill_customer_sk,ws_sold_date_sk]
+                                                                SubqueryBroadcast [d_date_sk] #1
+                                                                  ReusedExchange [d_date_sk] #5
+                                                        InputAdapter
+                                                          BroadcastExchange #5
+                                                            WholeStageCodegen (3)
+                                                              Project [d_date_sk]
+                                                                Filter [d_year,d_moy,d_date_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                                  WholeStageCodegen (6)
+                                                    Project [cs_ship_customer_sk]
+                                                      BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                        Filter [cs_ship_customer_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk]
+                                                                ReusedSubquery [d_date_sk] #1
+                                                        InputAdapter
+                                                          ReusedExchange [d_date_sk] #5
+                                InputAdapter
+                                  WholeStageCodegen (11)
+                                    Sort [customer_sk]
+                                      InputAdapter
+                                        Exchange [customer_sk] #6
+                                          WholeStageCodegen (10)
+                                            Project [ss_customer_sk]
+                                              BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                Filter [ss_customer_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.store_sales [ss_customer_sk,ss_sold_date_sk]
+                                                        ReusedSubquery [d_date_sk] #1
+                                                InputAdapter
+                                                  ReusedExchange [d_date_sk] #5
+                            InputAdapter
+                              BroadcastExchange #7
+                                WholeStageCodegen (12)
+                                  Project [ca_address_sk]
+                                    Filter [ca_county,ca_address_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.customer_address [ca_address_sk,ca_county]
+                  Filter [cd_demo_sk]
+                    ColumnarToRow
+                      InputAdapter
+                        Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q10/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q10/explain.txt
new file mode 100644
index 0000000000000..ff55b1c511456
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q10/explain.txt
@@ -0,0 +1,283 @@
+== Physical Plan ==
+TakeOrderedAndProject (48)
++- * HashAggregate (47)
+   +- Exchange (46)
+      +- * HashAggregate (45)
+         +- * Project (44)
+            +- * BroadcastHashJoin Inner BuildRight (43)
+               :- * Project (38)
+               :  +- * BroadcastHashJoin Inner BuildRight (37)
+               :     :- * Project (31)
+               :     :  +- * BroadcastHashJoin LeftSemi BuildRight (30)
+               :     :     :- * BroadcastHashJoin LeftSemi BuildRight (22)
+               :     :     :  :- * Filter (3)
+               :     :     :  :  +- * ColumnarToRow (2)
+               :     :     :  :     +- Scan parquet default.customer (1)
+               :     :     :  +- BroadcastExchange (21)
+               :     :     :     +- Union (20)
+               :     :     :        :- * Project (13)
+               :     :     :        :  +- * BroadcastHashJoin Inner BuildRight (12)
+               :     :     :        :     :- * Filter (6)
+               :     :     :        :     :  +- * ColumnarToRow (5)
+               :     :     :        :     :     +- Scan parquet default.web_sales (4)
+               :     :     :        :     +- BroadcastExchange (11)
+               :     :     :        :        +- * Project (10)
+               :     :     :        :           +- * Filter (9)
+               :     :     :        :              +- * ColumnarToRow (8)
+               :     :     :        :                 +- Scan parquet default.date_dim (7)
+               :     :     :        +- * Project (19)
+               :     :     :           +- * BroadcastHashJoin Inner BuildRight (18)
+               :     :     :              :- * Filter (16)
+               :     :     :              :  +- * ColumnarToRow (15)
+               :     :     :              :     +- Scan parquet default.catalog_sales (14)
+               :     :     :              +- ReusedExchange (17)
+               :     :     +- BroadcastExchange (29)
+               :     :        +- * Project (28)
+               :     :           +- * BroadcastHashJoin Inner BuildRight (27)
+               :     :              :- * Filter (25)
+               :     :              :  +- * ColumnarToRow (24)
+               :     :              :     +- Scan parquet default.store_sales (23)
+               :     :              +- ReusedExchange (26)
+               :     +- BroadcastExchange (36)
+               :        +- * Project (35)
+               :           +- * Filter (34)
+               :              +- * ColumnarToRow (33)
+               :                 +- Scan parquet default.customer_address (32)
+               +- BroadcastExchange (42)
+                  +- * Filter (41)
+                     +- * ColumnarToRow (40)
+                        +- Scan parquet default.customer_demographics (39)
+
+
+(1) Scan parquet default.customer
+Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_addr_sk:int>
+
+(2) ColumnarToRow [codegen id : 9]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+
+(3) Filter [codegen id : 9]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+Condition : ((isnotnull(c_customer_sk#1) AND isnotnull(c_current_addr_sk#3)) AND isnotnull(c_current_cdemo_sk#2))
+
+(4) Scan parquet default.web_sales
+Output [2]: [ws_bill_customer_sk#4, ws_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#5), dynamicpruningexpression(ws_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int>
+
+(5) ColumnarToRow [codegen id : 2]
+Input [2]: [ws_bill_customer_sk#4, ws_sold_date_sk#5]
+
+(6) Filter [codegen id : 2]
+Input [2]: [ws_bill_customer_sk#4, ws_sold_date_sk#5]
+Condition : isnotnull(ws_bill_customer_sk#4)
+
+(7) Scan parquet default.date_dim
+Output [3]: [d_date_sk#7, d_year#8, d_moy#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,7), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(8) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_moy#9]
+
+(9) Filter [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_moy#9]
+Condition : (((((isnotnull(d_year#8) AND isnotnull(d_moy#9)) AND (d_year#8 = 2002)) AND (d_moy#9 >= 4)) AND (d_moy#9 <= 7)) AND isnotnull(d_date_sk#7))
+
+(10) Project [codegen id : 1]
+Output [1]: [d_date_sk#7]
+Input [3]: [d_date_sk#7, d_year#8, d_moy#9]
+
+(11) BroadcastExchange
+Input [1]: [d_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(12) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ws_sold_date_sk#5]
+Right keys [1]: [d_date_sk#7]
+Join condition: None
+
+(13) Project [codegen id : 2]
+Output [1]: [ws_bill_customer_sk#4 AS customer_sk#11]
+Input [3]: [ws_bill_customer_sk#4, ws_sold_date_sk#5, d_date_sk#7]
+
+(14) Scan parquet default.catalog_sales
+Output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#13), dynamicpruningexpression(cs_sold_date_sk#13 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(cs_ship_customer_sk)]
+ReadSchema: struct<cs_ship_customer_sk:int>
+
+(15) ColumnarToRow [codegen id : 4]
+Input [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13]
+
+(16) Filter [codegen id : 4]
+Input [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13]
+Condition : isnotnull(cs_ship_customer_sk#12)
+
+(17) ReusedExchange [Reuses operator id: 11]
+Output [1]: [d_date_sk#14]
+
+(18) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_sold_date_sk#13]
+Right keys [1]: [d_date_sk#14]
+Join condition: None
+
+(19) Project [codegen id : 4]
+Output [1]: [cs_ship_customer_sk#12 AS customer_sk#15]
+Input [3]: [cs_ship_customer_sk#12, cs_sold_date_sk#13, d_date_sk#14]
+
+(20) Union
+
+(21) BroadcastExchange
+Input [1]: [customer_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#16]
+
+(22) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [customer_sk#11]
+Join condition: None
+
+(23) Scan parquet default.store_sales
+Output [2]: [ss_customer_sk#17, ss_sold_date_sk#18]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#18), dynamicpruningexpression(ss_sold_date_sk#18 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int>
+
+(24) ColumnarToRow [codegen id : 6]
+Input [2]: [ss_customer_sk#17, ss_sold_date_sk#18]
+
+(25) Filter [codegen id : 6]
+Input [2]: [ss_customer_sk#17, ss_sold_date_sk#18]
+Condition : isnotnull(ss_customer_sk#17)
+
+(26) ReusedExchange [Reuses operator id: 11]
+Output [1]: [d_date_sk#19]
+
+(27) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_sold_date_sk#18]
+Right keys [1]: [d_date_sk#19]
+Join condition: None
+
+(28) Project [codegen id : 6]
+Output [1]: [ss_customer_sk#17 AS customer_sk#20]
+Input [3]: [ss_customer_sk#17, ss_sold_date_sk#18, d_date_sk#19]
+
+(29) BroadcastExchange
+Input [1]: [customer_sk#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(30) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [customer_sk#20]
+Join condition: None
+
+(31) Project [codegen id : 9]
+Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+
+(32) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#22, ca_county#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [In(ca_county, [Walker County,Richland County,Gaines County,Douglas County,Dona Ana County]), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_county:string>
+
+(33) ColumnarToRow [codegen id : 7]
+Input [2]: [ca_address_sk#22, ca_county#23]
+
+(34) Filter [codegen id : 7]
+Input [2]: [ca_address_sk#22, ca_county#23]
+Condition : (ca_county#23 IN (Walker County,Richland County,Gaines County,Douglas County,Dona Ana County) AND isnotnull(ca_address_sk#22))
+
+(35) Project [codegen id : 7]
+Output [1]: [ca_address_sk#22]
+Input [2]: [ca_address_sk#22, ca_county#23]
+
+(36) BroadcastExchange
+Input [1]: [ca_address_sk#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#24]
+
+(37) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_current_addr_sk#3]
+Right keys [1]: [ca_address_sk#22]
+Join condition: None
+
+(38) Project [codegen id : 9]
+Output [1]: [c_current_cdemo_sk#2]
+Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#22]
+
+(39) Scan parquet default.customer_demographics
+Output [9]: [cd_demo_sk#25, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_education_status:string,cd_purchase_estimate:int,cd_credit_rating:string,cd_dep_count:int,cd_dep_employed_count:int,cd_dep_college_count:int>
+
+(40) ColumnarToRow [codegen id : 8]
+Input [9]: [cd_demo_sk#25, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+
+(41) Filter [codegen id : 8]
+Input [9]: [cd_demo_sk#25, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Condition : isnotnull(cd_demo_sk#25)
+
+(42) BroadcastExchange
+Input [9]: [cd_demo_sk#25, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#34]
+
+(43) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_current_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#25]
+Join condition: None
+
+(44) Project [codegen id : 9]
+Output [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Input [10]: [c_current_cdemo_sk#2, cd_demo_sk#25, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+
+(45) HashAggregate [codegen id : 9]
+Input [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Keys [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#35]
+Results [9]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33, count#36]
+
+(46) Exchange
+Input [9]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33, count#36]
+Arguments: hashpartitioning(cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33, 5), ENSURE_REQUIREMENTS, [id=#37]
+
+(47) HashAggregate [codegen id : 10]
+Input [9]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33, count#36]
+Keys [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#38]
+Results [14]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, count(1)#38 AS cnt1#39, cd_purchase_estimate#29, count(1)#38 AS cnt2#40, cd_credit_rating#30, count(1)#38 AS cnt3#41, cd_dep_count#31, count(1)#38 AS cnt4#42, cd_dep_employed_count#32, count(1)#38 AS cnt5#43, cd_dep_college_count#33, count(1)#38 AS cnt6#44]
+
+(48) TakeOrderedAndProject
+Input [14]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cnt1#39, cd_purchase_estimate#29, cnt2#40, cd_credit_rating#30, cnt3#41, cd_dep_count#31, cnt4#42, cd_dep_employed_count#32, cnt5#43, cd_dep_college_count#33, cnt6#44]
+Arguments: 100, [cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_education_status#28 ASC NULLS FIRST, cd_purchase_estimate#29 ASC NULLS FIRST, cd_credit_rating#30 ASC NULLS FIRST, cd_dep_count#31 ASC NULLS FIRST, cd_dep_employed_count#32 ASC NULLS FIRST, cd_dep_college_count#33 ASC NULLS FIRST], [cd_gender#26, cd_marital_status#27, cd_education_status#28, cnt1#39, cd_purchase_estimate#29, cnt2#40, cd_credit_rating#30, cnt3#41, cd_dep_count#31, cnt4#42, cd_dep_employed_count#32, cnt5#43, cd_dep_college_count#33, cnt6#44]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 4 Hosting Expression = ws_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (49)
+
+
+(49) ReusedExchange [Reuses operator id: 11]
+Output [1]: [d_date_sk#7]
+
+Subquery:2 Hosting operator id = 14 Hosting Expression = cs_sold_date_sk#13 IN dynamicpruning#6
+
+Subquery:3 Hosting operator id = 23 Hosting Expression = ss_sold_date_sk#18 IN dynamicpruning#6
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q10/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q10/simplified.txt
new file mode 100644
index 0000000000000..58eb3e3102c81
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q10/simplified.txt
@@ -0,0 +1,75 @@
+TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,cnt2,cnt3,cnt4,cnt5,cnt6]
+  WholeStageCodegen (10)
+    HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] [count(1),cnt1,cnt2,cnt3,cnt4,cnt5,cnt6,count]
+      InputAdapter
+        Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1
+          WholeStageCodegen (9)
+            HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count]
+              Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count]
+                BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk]
+                  Project [c_current_cdemo_sk]
+                    BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                      Project [c_current_cdemo_sk,c_current_addr_sk]
+                        BroadcastHashJoin [c_customer_sk,customer_sk]
+                          BroadcastHashJoin [c_customer_sk,customer_sk]
+                            Filter [c_customer_sk,c_current_addr_sk,c_current_cdemo_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk]
+                            InputAdapter
+                              BroadcastExchange #2
+                                Union
+                                  WholeStageCodegen (2)
+                                    Project [ws_bill_customer_sk]
+                                      BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                        Filter [ws_bill_customer_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.web_sales [ws_bill_customer_sk,ws_sold_date_sk]
+                                                SubqueryBroadcast [d_date_sk] #1
+                                                  ReusedExchange [d_date_sk] #3
+                                        InputAdapter
+                                          BroadcastExchange #3
+                                            WholeStageCodegen (1)
+                                              Project [d_date_sk]
+                                                Filter [d_year,d_moy,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                  WholeStageCodegen (4)
+                                    Project [cs_ship_customer_sk]
+                                      BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                        Filter [cs_ship_customer_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk]
+                                                ReusedSubquery [d_date_sk] #1
+                                        InputAdapter
+                                          ReusedExchange [d_date_sk] #3
+                          InputAdapter
+                            BroadcastExchange #4
+                              WholeStageCodegen (6)
+                                Project [ss_customer_sk]
+                                  BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                    Filter [ss_customer_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.store_sales [ss_customer_sk,ss_sold_date_sk]
+                                            ReusedSubquery [d_date_sk] #1
+                                    InputAdapter
+                                      ReusedExchange [d_date_sk] #3
+                      InputAdapter
+                        BroadcastExchange #5
+                          WholeStageCodegen (7)
+                            Project [ca_address_sk]
+                              Filter [ca_county,ca_address_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.customer_address [ca_address_sk,ca_county]
+                  InputAdapter
+                    BroadcastExchange #6
+                      WholeStageCodegen (8)
+                        Filter [cd_demo_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q19.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q19.sf100/explain.txt
new file mode 100644
index 0000000000000..858cbaae6b019
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q19.sf100/explain.txt
@@ -0,0 +1,262 @@
+== Physical Plan ==
+TakeOrderedAndProject (45)
++- * HashAggregate (44)
+   +- Exchange (43)
+      +- * HashAggregate (42)
+         +- * Project (41)
+            +- * BroadcastHashJoin Inner BuildRight (40)
+               :- * Project (34)
+               :  +- * SortMergeJoin Inner (33)
+               :     :- * Sort (27)
+               :     :  +- Exchange (26)
+               :     :     +- * Project (25)
+               :     :        +- * BroadcastHashJoin Inner BuildRight (24)
+               :     :           :- * Project (19)
+               :     :           :  +- * SortMergeJoin Inner (18)
+               :     :           :     :- * Sort (12)
+               :     :           :     :  +- Exchange (11)
+               :     :           :     :     +- * Project (10)
+               :     :           :     :        +- * BroadcastHashJoin Inner BuildLeft (9)
+               :     :           :     :           :- BroadcastExchange (5)
+               :     :           :     :           :  +- * Project (4)
+               :     :           :     :           :     +- * Filter (3)
+               :     :           :     :           :        +- * ColumnarToRow (2)
+               :     :           :     :           :           +- Scan parquet default.date_dim (1)
+               :     :           :     :           +- * Filter (8)
+               :     :           :     :              +- * ColumnarToRow (7)
+               :     :           :     :                 +- Scan parquet default.store_sales (6)
+               :     :           :     +- * Sort (17)
+               :     :           :        +- Exchange (16)
+               :     :           :           +- * Filter (15)
+               :     :           :              +- * ColumnarToRow (14)
+               :     :           :                 +- Scan parquet default.customer (13)
+               :     :           +- BroadcastExchange (23)
+               :     :              +- * Filter (22)
+               :     :                 +- * ColumnarToRow (21)
+               :     :                    +- Scan parquet default.store (20)
+               :     +- * Sort (32)
+               :        +- Exchange (31)
+               :           +- * Filter (30)
+               :              +- * ColumnarToRow (29)
+               :                 +- Scan parquet default.customer_address (28)
+               +- BroadcastExchange (39)
+                  +- * Project (38)
+                     +- * Filter (37)
+                        +- * ColumnarToRow (36)
+                           +- Scan parquet default.item (35)
+
+
+(1) Scan parquet default.date_dim
+Output [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), GreaterThanOrEqual(d_date_sk,2451484), LessThanOrEqual(d_date_sk,2451513), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(3) Filter [codegen id : 1]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Condition : ((((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 1999)) AND (d_date_sk#1 >= 2451484)) AND (d_date_sk#1 <= 2451513)) AND isnotnull(d_date_sk#1))
+
+(4) Project [codegen id : 1]
+Output [1]: [d_date_sk#1]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(5) BroadcastExchange
+Input [1]: [d_date_sk#1]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#4]
+
+(6) Scan parquet default.store_sales
+Output [5]: [ss_item_sk#5, ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8, ss_sold_date_sk#9]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#9), (ss_sold_date_sk#9 >= 2451484), (ss_sold_date_sk#9 <= 2451513), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(7) ColumnarToRow
+Input [5]: [ss_item_sk#5, ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8, ss_sold_date_sk#9]
+
+(8) Filter
+Input [5]: [ss_item_sk#5, ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8, ss_sold_date_sk#9]
+Condition : ((isnotnull(ss_item_sk#5) AND isnotnull(ss_customer_sk#6)) AND isnotnull(ss_store_sk#7))
+
+(9) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [d_date_sk#1]
+Right keys [1]: [ss_sold_date_sk#9]
+Join condition: None
+
+(10) Project [codegen id : 2]
+Output [4]: [ss_item_sk#5, ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8]
+Input [6]: [d_date_sk#1, ss_item_sk#5, ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8, ss_sold_date_sk#9]
+
+(11) Exchange
+Input [4]: [ss_item_sk#5, ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8]
+Arguments: hashpartitioning(ss_customer_sk#6, 5), ENSURE_REQUIREMENTS, [id=#11]
+
+(12) Sort [codegen id : 3]
+Input [4]: [ss_item_sk#5, ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8]
+Arguments: [ss_customer_sk#6 ASC NULLS FIRST], false, 0
+
+(13) Scan parquet default.customer
+Output [2]: [c_customer_sk#12, c_current_addr_sk#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int>
+
+(14) ColumnarToRow [codegen id : 4]
+Input [2]: [c_customer_sk#12, c_current_addr_sk#13]
+
+(15) Filter [codegen id : 4]
+Input [2]: [c_customer_sk#12, c_current_addr_sk#13]
+Condition : (isnotnull(c_customer_sk#12) AND isnotnull(c_current_addr_sk#13))
+
+(16) Exchange
+Input [2]: [c_customer_sk#12, c_current_addr_sk#13]
+Arguments: hashpartitioning(c_customer_sk#12, 5), ENSURE_REQUIREMENTS, [id=#14]
+
+(17) Sort [codegen id : 5]
+Input [2]: [c_customer_sk#12, c_current_addr_sk#13]
+Arguments: [c_customer_sk#12 ASC NULLS FIRST], false, 0
+
+(18) SortMergeJoin [codegen id : 7]
+Left keys [1]: [ss_customer_sk#6]
+Right keys [1]: [c_customer_sk#12]
+Join condition: None
+
+(19) Project [codegen id : 7]
+Output [4]: [ss_item_sk#5, ss_store_sk#7, ss_ext_sales_price#8, c_current_addr_sk#13]
+Input [6]: [ss_item_sk#5, ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8, c_customer_sk#12, c_current_addr_sk#13]
+
+(20) Scan parquet default.store
+Output [2]: [s_store_sk#15, s_zip#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_zip), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_zip:string>
+
+(21) ColumnarToRow [codegen id : 6]
+Input [2]: [s_store_sk#15, s_zip#16]
+
+(22) Filter [codegen id : 6]
+Input [2]: [s_store_sk#15, s_zip#16]
+Condition : (isnotnull(s_zip#16) AND isnotnull(s_store_sk#15))
+
+(23) BroadcastExchange
+Input [2]: [s_store_sk#15, s_zip#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#17]
+
+(24) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [ss_store_sk#7]
+Right keys [1]: [s_store_sk#15]
+Join condition: None
+
+(25) Project [codegen id : 7]
+Output [4]: [ss_item_sk#5, ss_ext_sales_price#8, c_current_addr_sk#13, s_zip#16]
+Input [6]: [ss_item_sk#5, ss_store_sk#7, ss_ext_sales_price#8, c_current_addr_sk#13, s_store_sk#15, s_zip#16]
+
+(26) Exchange
+Input [4]: [ss_item_sk#5, ss_ext_sales_price#8, c_current_addr_sk#13, s_zip#16]
+Arguments: hashpartitioning(c_current_addr_sk#13, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(27) Sort [codegen id : 8]
+Input [4]: [ss_item_sk#5, ss_ext_sales_price#8, c_current_addr_sk#13, s_zip#16]
+Arguments: [c_current_addr_sk#13 ASC NULLS FIRST], false, 0
+
+(28) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#19, ca_zip#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_zip)]
+ReadSchema: struct<ca_address_sk:int,ca_zip:string>
+
+(29) ColumnarToRow [codegen id : 9]
+Input [2]: [ca_address_sk#19, ca_zip#20]
+
+(30) Filter [codegen id : 9]
+Input [2]: [ca_address_sk#19, ca_zip#20]
+Condition : (isnotnull(ca_address_sk#19) AND isnotnull(ca_zip#20))
+
+(31) Exchange
+Input [2]: [ca_address_sk#19, ca_zip#20]
+Arguments: hashpartitioning(ca_address_sk#19, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(32) Sort [codegen id : 10]
+Input [2]: [ca_address_sk#19, ca_zip#20]
+Arguments: [ca_address_sk#19 ASC NULLS FIRST], false, 0
+
+(33) SortMergeJoin [codegen id : 12]
+Left keys [1]: [c_current_addr_sk#13]
+Right keys [1]: [ca_address_sk#19]
+Join condition: NOT (substr(ca_zip#20, 1, 5) = substr(s_zip#16, 1, 5))
+
+(34) Project [codegen id : 12]
+Output [2]: [ss_item_sk#5, ss_ext_sales_price#8]
+Input [6]: [ss_item_sk#5, ss_ext_sales_price#8, c_current_addr_sk#13, s_zip#16, ca_address_sk#19, ca_zip#20]
+
+(35) Scan parquet default.item
+Output [6]: [i_item_sk#22, i_brand_id#23, i_brand#24, i_manufact_id#25, i_manufact#26, i_manager_id#27]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,7), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_brand:string,i_manufact_id:int,i_manufact:string,i_manager_id:int>
+
+(36) ColumnarToRow [codegen id : 11]
+Input [6]: [i_item_sk#22, i_brand_id#23, i_brand#24, i_manufact_id#25, i_manufact#26, i_manager_id#27]
+
+(37) Filter [codegen id : 11]
+Input [6]: [i_item_sk#22, i_brand_id#23, i_brand#24, i_manufact_id#25, i_manufact#26, i_manager_id#27]
+Condition : ((isnotnull(i_manager_id#27) AND (i_manager_id#27 = 7)) AND isnotnull(i_item_sk#22))
+
+(38) Project [codegen id : 11]
+Output [5]: [i_item_sk#22, i_brand_id#23, i_brand#24, i_manufact_id#25, i_manufact#26]
+Input [6]: [i_item_sk#22, i_brand_id#23, i_brand#24, i_manufact_id#25, i_manufact#26, i_manager_id#27]
+
+(39) BroadcastExchange
+Input [5]: [i_item_sk#22, i_brand_id#23, i_brand#24, i_manufact_id#25, i_manufact#26]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#28]
+
+(40) BroadcastHashJoin [codegen id : 12]
+Left keys [1]: [ss_item_sk#5]
+Right keys [1]: [i_item_sk#22]
+Join condition: None
+
+(41) Project [codegen id : 12]
+Output [5]: [ss_ext_sales_price#8, i_brand_id#23, i_brand#24, i_manufact_id#25, i_manufact#26]
+Input [7]: [ss_item_sk#5, ss_ext_sales_price#8, i_item_sk#22, i_brand_id#23, i_brand#24, i_manufact_id#25, i_manufact#26]
+
+(42) HashAggregate [codegen id : 12]
+Input [5]: [ss_ext_sales_price#8, i_brand_id#23, i_brand#24, i_manufact_id#25, i_manufact#26]
+Keys [4]: [i_brand#24, i_brand_id#23, i_manufact_id#25, i_manufact#26]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#8))]
+Aggregate Attributes [1]: [sum#29]
+Results [5]: [i_brand#24, i_brand_id#23, i_manufact_id#25, i_manufact#26, sum#30]
+
+(43) Exchange
+Input [5]: [i_brand#24, i_brand_id#23, i_manufact_id#25, i_manufact#26, sum#30]
+Arguments: hashpartitioning(i_brand#24, i_brand_id#23, i_manufact_id#25, i_manufact#26, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(44) HashAggregate [codegen id : 13]
+Input [5]: [i_brand#24, i_brand_id#23, i_manufact_id#25, i_manufact#26, sum#30]
+Keys [4]: [i_brand#24, i_brand_id#23, i_manufact_id#25, i_manufact#26]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#8))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#8))#32]
+Results [5]: [i_brand_id#23 AS brand_id#33, i_brand#24 AS brand#34, i_manufact_id#25, i_manufact#26, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#8))#32,17,2) AS ext_price#35]
+
+(45) TakeOrderedAndProject
+Input [5]: [brand_id#33, brand#34, i_manufact_id#25, i_manufact#26, ext_price#35]
+Arguments: 100, [ext_price#35 DESC NULLS LAST, brand#34 ASC NULLS FIRST, brand_id#33 ASC NULLS FIRST, i_manufact_id#25 ASC NULLS FIRST, i_manufact#26 ASC NULLS FIRST], [brand_id#33, brand#34, i_manufact_id#25, i_manufact#26, ext_price#35]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 6 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10
+ReusedExchange (46)
+
+
+(46) ReusedExchange [Reuses operator id: 5]
+Output [1]: [d_date_sk#1]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q19.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q19.sf100/simplified.txt
new file mode 100644
index 0000000000000..36933c4aac86b
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q19.sf100/simplified.txt
@@ -0,0 +1,78 @@
+TakeOrderedAndProject [ext_price,brand,brand_id,i_manufact_id,i_manufact]
+  WholeStageCodegen (13)
+    HashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum]
+      InputAdapter
+        Exchange [i_brand,i_brand_id,i_manufact_id,i_manufact] #1
+          WholeStageCodegen (12)
+            HashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,ss_ext_sales_price] [sum,sum]
+              Project [ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact]
+                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                  Project [ss_item_sk,ss_ext_sales_price]
+                    SortMergeJoin [c_current_addr_sk,ca_address_sk,ca_zip,s_zip]
+                      InputAdapter
+                        WholeStageCodegen (8)
+                          Sort [c_current_addr_sk]
+                            InputAdapter
+                              Exchange [c_current_addr_sk] #2
+                                WholeStageCodegen (7)
+                                  Project [ss_item_sk,ss_ext_sales_price,c_current_addr_sk,s_zip]
+                                    BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                      Project [ss_item_sk,ss_store_sk,ss_ext_sales_price,c_current_addr_sk]
+                                        SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                          InputAdapter
+                                            WholeStageCodegen (3)
+                                              Sort [ss_customer_sk]
+                                                InputAdapter
+                                                  Exchange [ss_customer_sk] #3
+                                                    WholeStageCodegen (2)
+                                                      Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price]
+                                                        BroadcastHashJoin [d_date_sk,ss_sold_date_sk]
+                                                          InputAdapter
+                                                            BroadcastExchange #4
+                                                              WholeStageCodegen (1)
+                                                                Project [d_date_sk]
+                                                                  Filter [d_moy,d_year,d_date_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                                          Filter [ss_item_sk,ss_customer_sk,ss_store_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                                                  SubqueryBroadcast [d_date_sk] #1
+                                                                    ReusedExchange [d_date_sk] #4
+                                          InputAdapter
+                                            WholeStageCodegen (5)
+                                              Sort [c_customer_sk]
+                                                InputAdapter
+                                                  Exchange [c_customer_sk] #5
+                                                    WholeStageCodegen (4)
+                                                      Filter [c_customer_sk,c_current_addr_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.customer [c_customer_sk,c_current_addr_sk]
+                                      InputAdapter
+                                        BroadcastExchange #6
+                                          WholeStageCodegen (6)
+                                            Filter [s_zip,s_store_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store [s_store_sk,s_zip]
+                      InputAdapter
+                        WholeStageCodegen (10)
+                          Sort [ca_address_sk]
+                            InputAdapter
+                              Exchange [ca_address_sk] #7
+                                WholeStageCodegen (9)
+                                  Filter [ca_address_sk,ca_zip]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.customer_address [ca_address_sk,ca_zip]
+                  InputAdapter
+                    BroadcastExchange #8
+                      WholeStageCodegen (11)
+                        Project [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact]
+                          Filter [i_manager_id,i_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q19/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q19/explain.txt
new file mode 100644
index 0000000000000..e58fb8a1e1364
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q19/explain.txt
@@ -0,0 +1,222 @@
+== Physical Plan ==
+TakeOrderedAndProject (39)
++- * HashAggregate (38)
+   +- Exchange (37)
+      +- * HashAggregate (36)
+         +- * Project (35)
+            +- * BroadcastHashJoin Inner BuildRight (34)
+               :- * Project (29)
+               :  +- * BroadcastHashJoin Inner BuildRight (28)
+               :     :- * Project (23)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (22)
+               :     :     :- * Project (17)
+               :     :     :  +- * BroadcastHashJoin Inner BuildRight (16)
+               :     :     :     :- * Project (10)
+               :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :     :     :     :- * Project (4)
+               :     :     :     :     :  +- * Filter (3)
+               :     :     :     :     :     +- * ColumnarToRow (2)
+               :     :     :     :     :        +- Scan parquet default.date_dim (1)
+               :     :     :     :     +- BroadcastExchange (8)
+               :     :     :     :        +- * Filter (7)
+               :     :     :     :           +- * ColumnarToRow (6)
+               :     :     :     :              +- Scan parquet default.store_sales (5)
+               :     :     :     +- BroadcastExchange (15)
+               :     :     :        +- * Project (14)
+               :     :     :           +- * Filter (13)
+               :     :     :              +- * ColumnarToRow (12)
+               :     :     :                 +- Scan parquet default.item (11)
+               :     :     +- BroadcastExchange (21)
+               :     :        +- * Filter (20)
+               :     :           +- * ColumnarToRow (19)
+               :     :              +- Scan parquet default.customer (18)
+               :     +- BroadcastExchange (27)
+               :        +- * Filter (26)
+               :           +- * ColumnarToRow (25)
+               :              +- Scan parquet default.customer_address (24)
+               +- BroadcastExchange (33)
+                  +- * Filter (32)
+                     +- * ColumnarToRow (31)
+                        +- Scan parquet default.store (30)
+
+
+(1) Scan parquet default.date_dim
+Output [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), GreaterThanOrEqual(d_date_sk,2451484), LessThanOrEqual(d_date_sk,2451513), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(2) ColumnarToRow [codegen id : 6]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(3) Filter [codegen id : 6]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Condition : ((((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 1999)) AND (d_date_sk#1 >= 2451484)) AND (d_date_sk#1 <= 2451513)) AND isnotnull(d_date_sk#1))
+
+(4) Project [codegen id : 6]
+Output [1]: [d_date_sk#1]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(5) Scan parquet default.store_sales
+Output [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), (ss_sold_date_sk#8 >= 2451484), (ss_sold_date_sk#8 <= 2451513), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8]
+
+(7) Filter [codegen id : 1]
+Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8]
+Condition : ((isnotnull(ss_item_sk#4) AND isnotnull(ss_customer_sk#5)) AND isnotnull(ss_store_sk#6))
+
+(8) BroadcastExchange
+Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[4, int, true] as bigint)),false), [id=#9]
+
+(9) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [d_date_sk#1]
+Right keys [1]: [ss_sold_date_sk#8]
+Join condition: None
+
+(10) Project [codegen id : 6]
+Output [4]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7]
+Input [6]: [d_date_sk#1, ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8]
+
+(11) Scan parquet default.item
+Output [6]: [i_item_sk#10, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14, i_manager_id#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,7), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_brand:string,i_manufact_id:int,i_manufact:string,i_manager_id:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [6]: [i_item_sk#10, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14, i_manager_id#15]
+
+(13) Filter [codegen id : 2]
+Input [6]: [i_item_sk#10, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14, i_manager_id#15]
+Condition : ((isnotnull(i_manager_id#15) AND (i_manager_id#15 = 7)) AND isnotnull(i_item_sk#10))
+
+(14) Project [codegen id : 2]
+Output [5]: [i_item_sk#10, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14]
+Input [6]: [i_item_sk#10, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14, i_manager_id#15]
+
+(15) BroadcastExchange
+Input [5]: [i_item_sk#10, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#16]
+
+(16) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_item_sk#4]
+Right keys [1]: [i_item_sk#10]
+Join condition: None
+
+(17) Project [codegen id : 6]
+Output [7]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14]
+Input [9]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_item_sk#10, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14]
+
+(18) Scan parquet default.customer
+Output [2]: [c_customer_sk#17, c_current_addr_sk#18]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [2]: [c_customer_sk#17, c_current_addr_sk#18]
+
+(20) Filter [codegen id : 3]
+Input [2]: [c_customer_sk#17, c_current_addr_sk#18]
+Condition : (isnotnull(c_customer_sk#17) AND isnotnull(c_current_addr_sk#18))
+
+(21) BroadcastExchange
+Input [2]: [c_customer_sk#17, c_current_addr_sk#18]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#19]
+
+(22) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_customer_sk#5]
+Right keys [1]: [c_customer_sk#17]
+Join condition: None
+
+(23) Project [codegen id : 6]
+Output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14, c_current_addr_sk#18]
+Input [9]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14, c_customer_sk#17, c_current_addr_sk#18]
+
+(24) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#20, ca_zip#21]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_zip)]
+ReadSchema: struct<ca_address_sk:int,ca_zip:string>
+
+(25) ColumnarToRow [codegen id : 4]
+Input [2]: [ca_address_sk#20, ca_zip#21]
+
+(26) Filter [codegen id : 4]
+Input [2]: [ca_address_sk#20, ca_zip#21]
+Condition : (isnotnull(ca_address_sk#20) AND isnotnull(ca_zip#21))
+
+(27) BroadcastExchange
+Input [2]: [ca_address_sk#20, ca_zip#21]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#22]
+
+(28) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [c_current_addr_sk#18]
+Right keys [1]: [ca_address_sk#20]
+Join condition: None
+
+(29) Project [codegen id : 6]
+Output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14, ca_zip#21]
+Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14, c_current_addr_sk#18, ca_address_sk#20, ca_zip#21]
+
+(30) Scan parquet default.store
+Output [2]: [s_store_sk#23, s_zip#24]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_zip), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_zip:string>
+
+(31) ColumnarToRow [codegen id : 5]
+Input [2]: [s_store_sk#23, s_zip#24]
+
+(32) Filter [codegen id : 5]
+Input [2]: [s_store_sk#23, s_zip#24]
+Condition : (isnotnull(s_zip#24) AND isnotnull(s_store_sk#23))
+
+(33) BroadcastExchange
+Input [2]: [s_store_sk#23, s_zip#24]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#25]
+
+(34) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_store_sk#6]
+Right keys [1]: [s_store_sk#23]
+Join condition: NOT (substr(ca_zip#21, 1, 5) = substr(s_zip#24, 1, 5))
+
+(35) Project [codegen id : 6]
+Output [5]: [ss_ext_sales_price#7, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14]
+Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14, ca_zip#21, s_store_sk#23, s_zip#24]
+
+(36) HashAggregate [codegen id : 6]
+Input [5]: [ss_ext_sales_price#7, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14]
+Keys [4]: [i_brand#12, i_brand_id#11, i_manufact_id#13, i_manufact#14]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#7))]
+Aggregate Attributes [1]: [sum#26]
+Results [5]: [i_brand#12, i_brand_id#11, i_manufact_id#13, i_manufact#14, sum#27]
+
+(37) Exchange
+Input [5]: [i_brand#12, i_brand_id#11, i_manufact_id#13, i_manufact#14, sum#27]
+Arguments: hashpartitioning(i_brand#12, i_brand_id#11, i_manufact_id#13, i_manufact#14, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(38) HashAggregate [codegen id : 7]
+Input [5]: [i_brand#12, i_brand_id#11, i_manufact_id#13, i_manufact#14, sum#27]
+Keys [4]: [i_brand#12, i_brand_id#11, i_manufact_id#13, i_manufact#14]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#7))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#7))#29]
+Results [5]: [i_brand_id#11 AS brand_id#30, i_brand#12 AS brand#31, i_manufact_id#13, i_manufact#14, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#7))#29,17,2) AS ext_price#32]
+
+(39) TakeOrderedAndProject
+Input [5]: [brand_id#30, brand#31, i_manufact_id#13, i_manufact#14, ext_price#32]
+Arguments: 100, [ext_price#32 DESC NULLS LAST, brand#31 ASC NULLS FIRST, brand_id#30 ASC NULLS FIRST, i_manufact_id#13 ASC NULLS FIRST, i_manufact#14 ASC NULLS FIRST], [brand_id#30, brand#31, i_manufact_id#13, i_manufact#14, ext_price#32]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q19/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q19/simplified.txt
new file mode 100644
index 0000000000000..4e00ccb014571
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q19/simplified.txt
@@ -0,0 +1,58 @@
+TakeOrderedAndProject [ext_price,brand,brand_id,i_manufact_id,i_manufact]
+  WholeStageCodegen (7)
+    HashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum]
+      InputAdapter
+        Exchange [i_brand,i_brand_id,i_manufact_id,i_manufact] #1
+          WholeStageCodegen (6)
+            HashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,ss_ext_sales_price] [sum,sum]
+              Project [ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact]
+                BroadcastHashJoin [ss_store_sk,s_store_sk,ca_zip,s_zip]
+                  Project [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip]
+                    BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                      Project [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_current_addr_sk]
+                        BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+                          Project [ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact]
+                            BroadcastHashJoin [ss_item_sk,i_item_sk]
+                              Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price]
+                                BroadcastHashJoin [d_date_sk,ss_sold_date_sk]
+                                  Project [d_date_sk]
+                                    Filter [d_moy,d_year,d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                  InputAdapter
+                                    BroadcastExchange #2
+                                      WholeStageCodegen (1)
+                                        Filter [ss_item_sk,ss_customer_sk,ss_store_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk]
+                              InputAdapter
+                                BroadcastExchange #3
+                                  WholeStageCodegen (2)
+                                    Project [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact]
+                                      Filter [i_manager_id,i_item_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id]
+                          InputAdapter
+                            BroadcastExchange #4
+                              WholeStageCodegen (3)
+                                Filter [c_customer_sk,c_current_addr_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.customer [c_customer_sk,c_current_addr_sk]
+                      InputAdapter
+                        BroadcastExchange #5
+                          WholeStageCodegen (4)
+                            Filter [ca_address_sk,ca_zip]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.customer_address [ca_address_sk,ca_zip]
+                  InputAdapter
+                    BroadcastExchange #6
+                      WholeStageCodegen (5)
+                        Filter [s_zip,s_store_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.store [s_store_sk,s_zip]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q27.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q27.sf100/explain.txt
new file mode 100644
index 0000000000000..8f7cbc7d0ffbc
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q27.sf100/explain.txt
@@ -0,0 +1,445 @@
+== Physical Plan ==
+TakeOrderedAndProject (77)
++- Union (76)
+   :- * HashAggregate (32)
+   :  +- Exchange (31)
+   :     +- * HashAggregate (30)
+   :        +- * Project (29)
+   :           +- * BroadcastHashJoin Inner BuildRight (28)
+   :              :- * Project (23)
+   :              :  +- * BroadcastHashJoin Inner BuildRight (22)
+   :              :     :- * Project (17)
+   :              :     :  +- * BroadcastHashJoin Inner BuildRight (16)
+   :              :     :     :- * Project (10)
+   :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+   :              :     :     :     :- * Filter (3)
+   :              :     :     :     :  +- * ColumnarToRow (2)
+   :              :     :     :     :     +- Scan parquet default.store_sales (1)
+   :              :     :     :     +- BroadcastExchange (8)
+   :              :     :     :        +- * Project (7)
+   :              :     :     :           +- * Filter (6)
+   :              :     :     :              +- * ColumnarToRow (5)
+   :              :     :     :                 +- Scan parquet default.date_dim (4)
+   :              :     :     +- BroadcastExchange (15)
+   :              :     :        +- * Project (14)
+   :              :     :           +- * Filter (13)
+   :              :     :              +- * ColumnarToRow (12)
+   :              :     :                 +- Scan parquet default.customer_demographics (11)
+   :              :     +- BroadcastExchange (21)
+   :              :        +- * Filter (20)
+   :              :           +- * ColumnarToRow (19)
+   :              :              +- Scan parquet default.store (18)
+   :              +- BroadcastExchange (27)
+   :                 +- * Filter (26)
+   :                    +- * ColumnarToRow (25)
+   :                       +- Scan parquet default.item (24)
+   :- * HashAggregate (54)
+   :  +- Exchange (53)
+   :     +- * HashAggregate (52)
+   :        +- * Project (51)
+   :           +- * BroadcastHashJoin Inner BuildRight (50)
+   :              :- * Project (48)
+   :              :  +- * BroadcastHashJoin Inner BuildRight (47)
+   :              :     :- * Project (45)
+   :              :     :  +- * BroadcastHashJoin Inner BuildRight (44)
+   :              :     :     :- * Project (38)
+   :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (37)
+   :              :     :     :     :- * Filter (35)
+   :              :     :     :     :  +- * ColumnarToRow (34)
+   :              :     :     :     :     +- Scan parquet default.store_sales (33)
+   :              :     :     :     +- ReusedExchange (36)
+   :              :     :     +- BroadcastExchange (43)
+   :              :     :        +- * Project (42)
+   :              :     :           +- * Filter (41)
+   :              :     :              +- * ColumnarToRow (40)
+   :              :     :                 +- Scan parquet default.store (39)
+   :              :     +- ReusedExchange (46)
+   :              +- ReusedExchange (49)
+   +- * HashAggregate (75)
+      +- Exchange (74)
+         +- * HashAggregate (73)
+            +- * Project (72)
+               +- * BroadcastHashJoin Inner BuildRight (71)
+                  :- * Project (66)
+                  :  +- * BroadcastHashJoin Inner BuildRight (65)
+                  :     :- * Project (63)
+                  :     :  +- * BroadcastHashJoin Inner BuildRight (62)
+                  :     :     :- * Project (60)
+                  :     :     :  +- * BroadcastHashJoin Inner BuildRight (59)
+                  :     :     :     :- * Filter (57)
+                  :     :     :     :  +- * ColumnarToRow (56)
+                  :     :     :     :     +- Scan parquet default.store_sales (55)
+                  :     :     :     +- ReusedExchange (58)
+                  :     :     +- ReusedExchange (61)
+                  :     +- ReusedExchange (64)
+                  +- BroadcastExchange (70)
+                     +- * Filter (69)
+                        +- * ColumnarToRow (68)
+                           +- Scan parquet default.item (67)
+
+
+(1) Scan parquet default.store_sales
+Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), (ss_sold_date_sk#8 >= 2451545), (ss_sold_date_sk#8 <= 2451910), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_cdemo_sk:int,ss_store_sk:int,ss_quantity:int,ss_list_price:decimal(7,2),ss_sales_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+
+(3) Filter [codegen id : 5]
+Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#10, d_year#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), GreaterThanOrEqual(d_date_sk,2451545), LessThanOrEqual(d_date_sk,2451910), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#10, d_year#11]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#10, d_year#11]
+Condition : ((((isnotnull(d_year#11) AND (d_year#11 = 2000)) AND (d_date_sk#10 >= 2451545)) AND (d_date_sk#10 <= 2451910)) AND isnotnull(d_date_sk#10))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#10]
+Input [2]: [d_date_sk#10, d_year#11]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [7]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7]
+Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#10]
+
+(11) Scan parquet default.customer_demographics
+Output [4]: [cd_demo_sk#13, cd_gender#14, cd_marital_status#15, cd_education_status#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,F), EqualTo(cd_marital_status,D), EqualTo(cd_education_status,Primary             ), IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_education_status:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [4]: [cd_demo_sk#13, cd_gender#14, cd_marital_status#15, cd_education_status#16]
+
+(13) Filter [codegen id : 2]
+Input [4]: [cd_demo_sk#13, cd_gender#14, cd_marital_status#15, cd_education_status#16]
+Condition : ((((((isnotnull(cd_gender#14) AND isnotnull(cd_marital_status#15)) AND isnotnull(cd_education_status#16)) AND (cd_gender#14 = F)) AND (cd_marital_status#15 = D)) AND (cd_education_status#16 = Primary             )) AND isnotnull(cd_demo_sk#13))
+
+(14) Project [codegen id : 2]
+Output [1]: [cd_demo_sk#13]
+Input [4]: [cd_demo_sk#13, cd_gender#14, cd_marital_status#15, cd_education_status#16]
+
+(15) BroadcastExchange
+Input [1]: [cd_demo_sk#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(16) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#13]
+Join condition: None
+
+(17) Project [codegen id : 5]
+Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7]
+Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, cd_demo_sk#13]
+
+(18) Scan parquet default.store
+Output [2]: [s_store_sk#18, s_state#19]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [In(s_state, [TN,AL,SD]), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_state:string>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [2]: [s_store_sk#18, s_state#19]
+
+(20) Filter [codegen id : 3]
+Input [2]: [s_store_sk#18, s_state#19]
+Condition : (s_state#19 IN (TN,AL,SD) AND isnotnull(s_store_sk#18))
+
+(21) BroadcastExchange
+Input [2]: [s_store_sk#18, s_state#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#20]
+
+(22) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#18]
+Join condition: None
+
+(23) Project [codegen id : 5]
+Output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#19]
+Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#18, s_state#19]
+
+(24) Scan parquet default.item
+Output [2]: [i_item_sk#21, i_item_id#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(25) ColumnarToRow [codegen id : 4]
+Input [2]: [i_item_sk#21, i_item_id#22]
+
+(26) Filter [codegen id : 4]
+Input [2]: [i_item_sk#21, i_item_id#22]
+Condition : isnotnull(i_item_sk#21)
+
+(27) BroadcastExchange
+Input [2]: [i_item_sk#21, i_item_id#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#23]
+
+(28) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#21]
+Join condition: None
+
+(29) Project [codegen id : 5]
+Output [6]: [i_item_id#22, s_state#19, ss_quantity#4 AS agg1#24, ss_list_price#5 AS agg2#25, ss_coupon_amt#7 AS agg3#26, ss_sales_price#6 AS agg4#27]
+Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#19, i_item_sk#21, i_item_id#22]
+
+(30) HashAggregate [codegen id : 5]
+Input [6]: [i_item_id#22, s_state#19, agg1#24, agg2#25, agg3#26, agg4#27]
+Keys [2]: [i_item_id#22, s_state#19]
+Functions [4]: [partial_avg(agg1#24), partial_avg(UnscaledValue(agg2#25)), partial_avg(UnscaledValue(agg3#26)), partial_avg(UnscaledValue(agg4#27))]
+Aggregate Attributes [8]: [sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35]
+Results [10]: [i_item_id#22, s_state#19, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43]
+
+(31) Exchange
+Input [10]: [i_item_id#22, s_state#19, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43]
+Arguments: hashpartitioning(i_item_id#22, s_state#19, 5), ENSURE_REQUIREMENTS, [id=#44]
+
+(32) HashAggregate [codegen id : 6]
+Input [10]: [i_item_id#22, s_state#19, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43]
+Keys [2]: [i_item_id#22, s_state#19]
+Functions [4]: [avg(agg1#24), avg(UnscaledValue(agg2#25)), avg(UnscaledValue(agg3#26)), avg(UnscaledValue(agg4#27))]
+Aggregate Attributes [4]: [avg(agg1#24)#45, avg(UnscaledValue(agg2#25))#46, avg(UnscaledValue(agg3#26))#47, avg(UnscaledValue(agg4#27))#48]
+Results [7]: [i_item_id#22, s_state#19, 0 AS g_state#49, avg(agg1#24)#45 AS agg1#50, cast((avg(UnscaledValue(agg2#25))#46 / 100.0) as decimal(11,6)) AS agg2#51, cast((avg(UnscaledValue(agg3#26))#47 / 100.0) as decimal(11,6)) AS agg3#52, cast((avg(UnscaledValue(agg4#27))#48 / 100.0) as decimal(11,6)) AS agg4#53]
+
+(33) Scan parquet default.store_sales
+Output [8]: [ss_item_sk#54, ss_cdemo_sk#55, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, ss_sold_date_sk#61]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#61), (ss_sold_date_sk#61 >= 2451545), (ss_sold_date_sk#61 <= 2451910), dynamicpruningexpression(ss_sold_date_sk#61 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_cdemo_sk:int,ss_store_sk:int,ss_quantity:int,ss_list_price:decimal(7,2),ss_sales_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(34) ColumnarToRow [codegen id : 11]
+Input [8]: [ss_item_sk#54, ss_cdemo_sk#55, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, ss_sold_date_sk#61]
+
+(35) Filter [codegen id : 11]
+Input [8]: [ss_item_sk#54, ss_cdemo_sk#55, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, ss_sold_date_sk#61]
+Condition : ((isnotnull(ss_cdemo_sk#55) AND isnotnull(ss_store_sk#56)) AND isnotnull(ss_item_sk#54))
+
+(36) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#62]
+
+(37) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ss_sold_date_sk#61]
+Right keys [1]: [d_date_sk#62]
+Join condition: None
+
+(38) Project [codegen id : 11]
+Output [7]: [ss_item_sk#54, ss_cdemo_sk#55, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60]
+Input [9]: [ss_item_sk#54, ss_cdemo_sk#55, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, ss_sold_date_sk#61, d_date_sk#62]
+
+(39) Scan parquet default.store
+Output [2]: [s_store_sk#63, s_state#64]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [In(s_state, [TN,AL,SD]), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_state:string>
+
+(40) ColumnarToRow [codegen id : 8]
+Input [2]: [s_store_sk#63, s_state#64]
+
+(41) Filter [codegen id : 8]
+Input [2]: [s_store_sk#63, s_state#64]
+Condition : (s_state#64 IN (TN,AL,SD) AND isnotnull(s_store_sk#63))
+
+(42) Project [codegen id : 8]
+Output [1]: [s_store_sk#63]
+Input [2]: [s_store_sk#63, s_state#64]
+
+(43) BroadcastExchange
+Input [1]: [s_store_sk#63]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#65]
+
+(44) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ss_store_sk#56]
+Right keys [1]: [s_store_sk#63]
+Join condition: None
+
+(45) Project [codegen id : 11]
+Output [6]: [ss_item_sk#54, ss_cdemo_sk#55, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60]
+Input [8]: [ss_item_sk#54, ss_cdemo_sk#55, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, s_store_sk#63]
+
+(46) ReusedExchange [Reuses operator id: 15]
+Output [1]: [cd_demo_sk#66]
+
+(47) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ss_cdemo_sk#55]
+Right keys [1]: [cd_demo_sk#66]
+Join condition: None
+
+(48) Project [codegen id : 11]
+Output [5]: [ss_item_sk#54, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60]
+Input [7]: [ss_item_sk#54, ss_cdemo_sk#55, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, cd_demo_sk#66]
+
+(49) ReusedExchange [Reuses operator id: 27]
+Output [2]: [i_item_sk#67, i_item_id#68]
+
+(50) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ss_item_sk#54]
+Right keys [1]: [i_item_sk#67]
+Join condition: None
+
+(51) Project [codegen id : 11]
+Output [5]: [i_item_id#68, ss_quantity#57 AS agg1#24, ss_list_price#58 AS agg2#25, ss_coupon_amt#60 AS agg3#26, ss_sales_price#59 AS agg4#27]
+Input [7]: [ss_item_sk#54, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, i_item_sk#67, i_item_id#68]
+
+(52) HashAggregate [codegen id : 11]
+Input [5]: [i_item_id#68, agg1#24, agg2#25, agg3#26, agg4#27]
+Keys [1]: [i_item_id#68]
+Functions [4]: [partial_avg(agg1#24), partial_avg(UnscaledValue(agg2#25)), partial_avg(UnscaledValue(agg3#26)), partial_avg(UnscaledValue(agg4#27))]
+Aggregate Attributes [8]: [sum#69, count#70, sum#71, count#72, sum#73, count#74, sum#75, count#76]
+Results [9]: [i_item_id#68, sum#77, count#78, sum#79, count#80, sum#81, count#82, sum#83, count#84]
+
+(53) Exchange
+Input [9]: [i_item_id#68, sum#77, count#78, sum#79, count#80, sum#81, count#82, sum#83, count#84]
+Arguments: hashpartitioning(i_item_id#68, 5), ENSURE_REQUIREMENTS, [id=#85]
+
+(54) HashAggregate [codegen id : 12]
+Input [9]: [i_item_id#68, sum#77, count#78, sum#79, count#80, sum#81, count#82, sum#83, count#84]
+Keys [1]: [i_item_id#68]
+Functions [4]: [avg(agg1#24), avg(UnscaledValue(agg2#25)), avg(UnscaledValue(agg3#26)), avg(UnscaledValue(agg4#27))]
+Aggregate Attributes [4]: [avg(agg1#24)#86, avg(UnscaledValue(agg2#25))#87, avg(UnscaledValue(agg3#26))#88, avg(UnscaledValue(agg4#27))#89]
+Results [7]: [i_item_id#68, null AS s_state#90, 1 AS g_state#91, avg(agg1#24)#86 AS agg1#92, cast((avg(UnscaledValue(agg2#25))#87 / 100.0) as decimal(11,6)) AS agg2#93, cast((avg(UnscaledValue(agg3#26))#88 / 100.0) as decimal(11,6)) AS agg3#94, cast((avg(UnscaledValue(agg4#27))#89 / 100.0) as decimal(11,6)) AS agg4#95]
+
+(55) Scan parquet default.store_sales
+Output [8]: [ss_item_sk#96, ss_cdemo_sk#97, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, ss_sold_date_sk#103]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#103), (ss_sold_date_sk#103 >= 2451545), (ss_sold_date_sk#103 <= 2451910), dynamicpruningexpression(ss_sold_date_sk#103 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_cdemo_sk:int,ss_store_sk:int,ss_quantity:int,ss_list_price:decimal(7,2),ss_sales_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(56) ColumnarToRow [codegen id : 17]
+Input [8]: [ss_item_sk#96, ss_cdemo_sk#97, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, ss_sold_date_sk#103]
+
+(57) Filter [codegen id : 17]
+Input [8]: [ss_item_sk#96, ss_cdemo_sk#97, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, ss_sold_date_sk#103]
+Condition : ((isnotnull(ss_cdemo_sk#97) AND isnotnull(ss_store_sk#98)) AND isnotnull(ss_item_sk#96))
+
+(58) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#104]
+
+(59) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ss_sold_date_sk#103]
+Right keys [1]: [d_date_sk#104]
+Join condition: None
+
+(60) Project [codegen id : 17]
+Output [7]: [ss_item_sk#96, ss_cdemo_sk#97, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102]
+Input [9]: [ss_item_sk#96, ss_cdemo_sk#97, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, ss_sold_date_sk#103, d_date_sk#104]
+
+(61) ReusedExchange [Reuses operator id: 43]
+Output [1]: [s_store_sk#105]
+
+(62) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ss_store_sk#98]
+Right keys [1]: [s_store_sk#105]
+Join condition: None
+
+(63) Project [codegen id : 17]
+Output [6]: [ss_item_sk#96, ss_cdemo_sk#97, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102]
+Input [8]: [ss_item_sk#96, ss_cdemo_sk#97, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, s_store_sk#105]
+
+(64) ReusedExchange [Reuses operator id: 15]
+Output [1]: [cd_demo_sk#106]
+
+(65) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ss_cdemo_sk#97]
+Right keys [1]: [cd_demo_sk#106]
+Join condition: None
+
+(66) Project [codegen id : 17]
+Output [5]: [ss_item_sk#96, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102]
+Input [7]: [ss_item_sk#96, ss_cdemo_sk#97, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, cd_demo_sk#106]
+
+(67) Scan parquet default.item
+Output [1]: [i_item_sk#107]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int>
+
+(68) ColumnarToRow [codegen id : 16]
+Input [1]: [i_item_sk#107]
+
+(69) Filter [codegen id : 16]
+Input [1]: [i_item_sk#107]
+Condition : isnotnull(i_item_sk#107)
+
+(70) BroadcastExchange
+Input [1]: [i_item_sk#107]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#108]
+
+(71) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ss_item_sk#96]
+Right keys [1]: [i_item_sk#107]
+Join condition: None
+
+(72) Project [codegen id : 17]
+Output [4]: [ss_quantity#99 AS agg1#24, ss_list_price#100 AS agg2#25, ss_coupon_amt#102 AS agg3#26, ss_sales_price#101 AS agg4#27]
+Input [6]: [ss_item_sk#96, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, i_item_sk#107]
+
+(73) HashAggregate [codegen id : 17]
+Input [4]: [agg1#24, agg2#25, agg3#26, agg4#27]
+Keys: []
+Functions [4]: [partial_avg(agg1#24), partial_avg(UnscaledValue(agg2#25)), partial_avg(UnscaledValue(agg3#26)), partial_avg(UnscaledValue(agg4#27))]
+Aggregate Attributes [8]: [sum#109, count#110, sum#111, count#112, sum#113, count#114, sum#115, count#116]
+Results [8]: [sum#117, count#118, sum#119, count#120, sum#121, count#122, sum#123, count#124]
+
+(74) Exchange
+Input [8]: [sum#117, count#118, sum#119, count#120, sum#121, count#122, sum#123, count#124]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#125]
+
+(75) HashAggregate [codegen id : 18]
+Input [8]: [sum#117, count#118, sum#119, count#120, sum#121, count#122, sum#123, count#124]
+Keys: []
+Functions [4]: [avg(agg1#24), avg(UnscaledValue(agg2#25)), avg(UnscaledValue(agg3#26)), avg(UnscaledValue(agg4#27))]
+Aggregate Attributes [4]: [avg(agg1#24)#126, avg(UnscaledValue(agg2#25))#127, avg(UnscaledValue(agg3#26))#128, avg(UnscaledValue(agg4#27))#129]
+Results [7]: [null AS i_item_id#130, null AS s_state#131, 1 AS g_state#132, avg(agg1#24)#126 AS agg1#133, cast((avg(UnscaledValue(agg2#25))#127 / 100.0) as decimal(11,6)) AS agg2#134, cast((avg(UnscaledValue(agg3#26))#128 / 100.0) as decimal(11,6)) AS agg3#135, cast((avg(UnscaledValue(agg4#27))#129 / 100.0) as decimal(11,6)) AS agg4#136]
+
+(76) Union
+
+(77) TakeOrderedAndProject
+Input [7]: [i_item_id#22, s_state#19, g_state#49, agg1#50, agg2#51, agg3#52, agg4#53]
+Arguments: 100, [i_item_id#22 ASC NULLS FIRST, s_state#19 ASC NULLS FIRST], [i_item_id#22, s_state#19, g_state#49, agg1#50, agg2#51, agg3#52, agg4#53]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (78)
+
+
+(78) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#10]
+
+Subquery:2 Hosting operator id = 33 Hosting Expression = ss_sold_date_sk#61 IN dynamicpruning#9
+
+Subquery:3 Hosting operator id = 55 Hosting Expression = ss_sold_date_sk#103 IN dynamicpruning#9
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q27.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q27.sf100/simplified.txt
new file mode 100644
index 0000000000000..102307ec83594
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q27.sf100/simplified.txt
@@ -0,0 +1,117 @@
+TakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4]
+  Union
+    WholeStageCodegen (6)
+      HashAggregate [i_item_id,s_state,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4)),g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count]
+        InputAdapter
+          Exchange [i_item_id,s_state] #1
+            WholeStageCodegen (5)
+              HashAggregate [i_item_id,s_state,agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+                Project [i_item_id,s_state,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price]
+                  BroadcastHashJoin [ss_item_sk,i_item_sk]
+                    Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state]
+                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                        Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                          BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk]
+                            Project [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                              BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                                        SubqueryBroadcast [d_date_sk] #1
+                                          ReusedExchange [d_date_sk] #2
+                                InputAdapter
+                                  BroadcastExchange #2
+                                    WholeStageCodegen (1)
+                                      Project [d_date_sk]
+                                        Filter [d_year,d_date_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.date_dim [d_date_sk,d_year]
+                            InputAdapter
+                              BroadcastExchange #3
+                                WholeStageCodegen (2)
+                                  Project [cd_demo_sk]
+                                    Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status]
+                        InputAdapter
+                          BroadcastExchange #4
+                            WholeStageCodegen (3)
+                              Filter [s_state,s_store_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.store [s_store_sk,s_state]
+                    InputAdapter
+                      BroadcastExchange #5
+                        WholeStageCodegen (4)
+                          Filter [i_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.item [i_item_sk,i_item_id]
+    WholeStageCodegen (12)
+      HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4)),s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count]
+        InputAdapter
+          Exchange [i_item_id] #6
+            WholeStageCodegen (11)
+              HashAggregate [i_item_id,agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+                Project [i_item_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price]
+                  BroadcastHashJoin [ss_item_sk,i_item_sk]
+                    Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                      BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk]
+                        Project [ss_item_sk,ss_cdemo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                          BroadcastHashJoin [ss_store_sk,s_store_sk]
+                            Project [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                              BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                                        ReusedSubquery [d_date_sk] #1
+                                InputAdapter
+                                  ReusedExchange [d_date_sk] #2
+                            InputAdapter
+                              BroadcastExchange #7
+                                WholeStageCodegen (8)
+                                  Project [s_store_sk]
+                                    Filter [s_state,s_store_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.store [s_store_sk,s_state]
+                        InputAdapter
+                          ReusedExchange [cd_demo_sk] #3
+                    InputAdapter
+                      ReusedExchange [i_item_sk,i_item_id] #5
+    WholeStageCodegen (18)
+      HashAggregate [sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4)),i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count]
+        InputAdapter
+          Exchange #8
+            WholeStageCodegen (17)
+              HashAggregate [agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+                Project [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price]
+                  BroadcastHashJoin [ss_item_sk,i_item_sk]
+                    Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                      BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk]
+                        Project [ss_item_sk,ss_cdemo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                          BroadcastHashJoin [ss_store_sk,s_store_sk]
+                            Project [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                              BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                                        ReusedSubquery [d_date_sk] #1
+                                InputAdapter
+                                  ReusedExchange [d_date_sk] #2
+                            InputAdapter
+                              ReusedExchange [s_store_sk] #7
+                        InputAdapter
+                          ReusedExchange [cd_demo_sk] #3
+                    InputAdapter
+                      BroadcastExchange #9
+                        WholeStageCodegen (16)
+                          Filter [i_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.item [i_item_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q27/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q27/explain.txt
new file mode 100644
index 0000000000000..bfb9e8384817d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q27/explain.txt
@@ -0,0 +1,445 @@
+== Physical Plan ==
+TakeOrderedAndProject (77)
++- Union (76)
+   :- * HashAggregate (32)
+   :  +- Exchange (31)
+   :     +- * HashAggregate (30)
+   :        +- * Project (29)
+   :           +- * BroadcastHashJoin Inner BuildRight (28)
+   :              :- * Project (23)
+   :              :  +- * BroadcastHashJoin Inner BuildRight (22)
+   :              :     :- * Project (17)
+   :              :     :  +- * BroadcastHashJoin Inner BuildRight (16)
+   :              :     :     :- * Project (10)
+   :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+   :              :     :     :     :- * Filter (3)
+   :              :     :     :     :  +- * ColumnarToRow (2)
+   :              :     :     :     :     +- Scan parquet default.store_sales (1)
+   :              :     :     :     +- BroadcastExchange (8)
+   :              :     :     :        +- * Project (7)
+   :              :     :     :           +- * Filter (6)
+   :              :     :     :              +- * ColumnarToRow (5)
+   :              :     :     :                 +- Scan parquet default.customer_demographics (4)
+   :              :     :     +- BroadcastExchange (15)
+   :              :     :        +- * Project (14)
+   :              :     :           +- * Filter (13)
+   :              :     :              +- * ColumnarToRow (12)
+   :              :     :                 +- Scan parquet default.date_dim (11)
+   :              :     +- BroadcastExchange (21)
+   :              :        +- * Filter (20)
+   :              :           +- * ColumnarToRow (19)
+   :              :              +- Scan parquet default.store (18)
+   :              +- BroadcastExchange (27)
+   :                 +- * Filter (26)
+   :                    +- * ColumnarToRow (25)
+   :                       +- Scan parquet default.item (24)
+   :- * HashAggregate (54)
+   :  +- Exchange (53)
+   :     +- * HashAggregate (52)
+   :        +- * Project (51)
+   :           +- * BroadcastHashJoin Inner BuildRight (50)
+   :              :- * Project (48)
+   :              :  +- * BroadcastHashJoin Inner BuildRight (47)
+   :              :     :- * Project (41)
+   :              :     :  +- * BroadcastHashJoin Inner BuildRight (40)
+   :              :     :     :- * Project (38)
+   :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (37)
+   :              :     :     :     :- * Filter (35)
+   :              :     :     :     :  +- * ColumnarToRow (34)
+   :              :     :     :     :     +- Scan parquet default.store_sales (33)
+   :              :     :     :     +- ReusedExchange (36)
+   :              :     :     +- ReusedExchange (39)
+   :              :     +- BroadcastExchange (46)
+   :              :        +- * Project (45)
+   :              :           +- * Filter (44)
+   :              :              +- * ColumnarToRow (43)
+   :              :                 +- Scan parquet default.store (42)
+   :              +- ReusedExchange (49)
+   +- * HashAggregate (75)
+      +- Exchange (74)
+         +- * HashAggregate (73)
+            +- * Project (72)
+               +- * BroadcastHashJoin Inner BuildRight (71)
+                  :- * Project (66)
+                  :  +- * BroadcastHashJoin Inner BuildRight (65)
+                  :     :- * Project (63)
+                  :     :  +- * BroadcastHashJoin Inner BuildRight (62)
+                  :     :     :- * Project (60)
+                  :     :     :  +- * BroadcastHashJoin Inner BuildRight (59)
+                  :     :     :     :- * Filter (57)
+                  :     :     :     :  +- * ColumnarToRow (56)
+                  :     :     :     :     +- Scan parquet default.store_sales (55)
+                  :     :     :     +- ReusedExchange (58)
+                  :     :     +- ReusedExchange (61)
+                  :     +- ReusedExchange (64)
+                  +- BroadcastExchange (70)
+                     +- * Filter (69)
+                        +- * ColumnarToRow (68)
+                           +- Scan parquet default.item (67)
+
+
+(1) Scan parquet default.store_sales
+Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), (ss_sold_date_sk#8 >= 2451545), (ss_sold_date_sk#8 <= 2451910), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_cdemo_sk:int,ss_store_sk:int,ss_quantity:int,ss_list_price:decimal(7,2),ss_sales_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+
+(3) Filter [codegen id : 5]
+Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1))
+
+(4) Scan parquet default.customer_demographics
+Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,F), EqualTo(cd_marital_status,D), EqualTo(cd_education_status,Primary             ), IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_education_status:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+
+(6) Filter [codegen id : 1]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (cd_gender#11 = F)) AND (cd_marital_status#12 = D)) AND (cd_education_status#13 = Primary             )) AND isnotnull(cd_demo_sk#10))
+
+(7) Project [codegen id : 1]
+Output [1]: [cd_demo_sk#10]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+
+(8) BroadcastExchange
+Input [1]: [cd_demo_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#10]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#10]
+
+(11) Scan parquet default.date_dim
+Output [2]: [d_date_sk#15, d_year#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), GreaterThanOrEqual(d_date_sk,2451545), LessThanOrEqual(d_date_sk,2451910), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#15, d_year#16]
+
+(13) Filter [codegen id : 2]
+Input [2]: [d_date_sk#15, d_year#16]
+Condition : ((((isnotnull(d_year#16) AND (d_year#16 = 2000)) AND (d_date_sk#15 >= 2451545)) AND (d_date_sk#15 <= 2451910)) AND isnotnull(d_date_sk#15))
+
+(14) Project [codegen id : 2]
+Output [1]: [d_date_sk#15]
+Input [2]: [d_date_sk#15, d_year#16]
+
+(15) BroadcastExchange
+Input [1]: [d_date_sk#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(16) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#15]
+Join condition: None
+
+(17) Project [codegen id : 5]
+Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7]
+Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#15]
+
+(18) Scan parquet default.store
+Output [2]: [s_store_sk#18, s_state#19]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [In(s_state, [TN,AL,SD]), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_state:string>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [2]: [s_store_sk#18, s_state#19]
+
+(20) Filter [codegen id : 3]
+Input [2]: [s_store_sk#18, s_state#19]
+Condition : (s_state#19 IN (TN,AL,SD) AND isnotnull(s_store_sk#18))
+
+(21) BroadcastExchange
+Input [2]: [s_store_sk#18, s_state#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#20]
+
+(22) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#18]
+Join condition: None
+
+(23) Project [codegen id : 5]
+Output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#19]
+Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#18, s_state#19]
+
+(24) Scan parquet default.item
+Output [2]: [i_item_sk#21, i_item_id#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(25) ColumnarToRow [codegen id : 4]
+Input [2]: [i_item_sk#21, i_item_id#22]
+
+(26) Filter [codegen id : 4]
+Input [2]: [i_item_sk#21, i_item_id#22]
+Condition : isnotnull(i_item_sk#21)
+
+(27) BroadcastExchange
+Input [2]: [i_item_sk#21, i_item_id#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#23]
+
+(28) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#21]
+Join condition: None
+
+(29) Project [codegen id : 5]
+Output [6]: [i_item_id#22, s_state#19, ss_quantity#4 AS agg1#24, ss_list_price#5 AS agg2#25, ss_coupon_amt#7 AS agg3#26, ss_sales_price#6 AS agg4#27]
+Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#19, i_item_sk#21, i_item_id#22]
+
+(30) HashAggregate [codegen id : 5]
+Input [6]: [i_item_id#22, s_state#19, agg1#24, agg2#25, agg3#26, agg4#27]
+Keys [2]: [i_item_id#22, s_state#19]
+Functions [4]: [partial_avg(agg1#24), partial_avg(UnscaledValue(agg2#25)), partial_avg(UnscaledValue(agg3#26)), partial_avg(UnscaledValue(agg4#27))]
+Aggregate Attributes [8]: [sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35]
+Results [10]: [i_item_id#22, s_state#19, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43]
+
+(31) Exchange
+Input [10]: [i_item_id#22, s_state#19, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43]
+Arguments: hashpartitioning(i_item_id#22, s_state#19, 5), ENSURE_REQUIREMENTS, [id=#44]
+
+(32) HashAggregate [codegen id : 6]
+Input [10]: [i_item_id#22, s_state#19, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43]
+Keys [2]: [i_item_id#22, s_state#19]
+Functions [4]: [avg(agg1#24), avg(UnscaledValue(agg2#25)), avg(UnscaledValue(agg3#26)), avg(UnscaledValue(agg4#27))]
+Aggregate Attributes [4]: [avg(agg1#24)#45, avg(UnscaledValue(agg2#25))#46, avg(UnscaledValue(agg3#26))#47, avg(UnscaledValue(agg4#27))#48]
+Results [7]: [i_item_id#22, s_state#19, 0 AS g_state#49, avg(agg1#24)#45 AS agg1#50, cast((avg(UnscaledValue(agg2#25))#46 / 100.0) as decimal(11,6)) AS agg2#51, cast((avg(UnscaledValue(agg3#26))#47 / 100.0) as decimal(11,6)) AS agg3#52, cast((avg(UnscaledValue(agg4#27))#48 / 100.0) as decimal(11,6)) AS agg4#53]
+
+(33) Scan parquet default.store_sales
+Output [8]: [ss_item_sk#54, ss_cdemo_sk#55, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, ss_sold_date_sk#61]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#61), (ss_sold_date_sk#61 >= 2451545), (ss_sold_date_sk#61 <= 2451910), dynamicpruningexpression(ss_sold_date_sk#61 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_cdemo_sk:int,ss_store_sk:int,ss_quantity:int,ss_list_price:decimal(7,2),ss_sales_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(34) ColumnarToRow [codegen id : 11]
+Input [8]: [ss_item_sk#54, ss_cdemo_sk#55, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, ss_sold_date_sk#61]
+
+(35) Filter [codegen id : 11]
+Input [8]: [ss_item_sk#54, ss_cdemo_sk#55, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, ss_sold_date_sk#61]
+Condition : ((isnotnull(ss_cdemo_sk#55) AND isnotnull(ss_store_sk#56)) AND isnotnull(ss_item_sk#54))
+
+(36) ReusedExchange [Reuses operator id: 8]
+Output [1]: [cd_demo_sk#62]
+
+(37) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ss_cdemo_sk#55]
+Right keys [1]: [cd_demo_sk#62]
+Join condition: None
+
+(38) Project [codegen id : 11]
+Output [7]: [ss_item_sk#54, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, ss_sold_date_sk#61]
+Input [9]: [ss_item_sk#54, ss_cdemo_sk#55, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, ss_sold_date_sk#61, cd_demo_sk#62]
+
+(39) ReusedExchange [Reuses operator id: 15]
+Output [1]: [d_date_sk#63]
+
+(40) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ss_sold_date_sk#61]
+Right keys [1]: [d_date_sk#63]
+Join condition: None
+
+(41) Project [codegen id : 11]
+Output [6]: [ss_item_sk#54, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60]
+Input [8]: [ss_item_sk#54, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, ss_sold_date_sk#61, d_date_sk#63]
+
+(42) Scan parquet default.store
+Output [2]: [s_store_sk#64, s_state#65]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [In(s_state, [TN,AL,SD]), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_state:string>
+
+(43) ColumnarToRow [codegen id : 9]
+Input [2]: [s_store_sk#64, s_state#65]
+
+(44) Filter [codegen id : 9]
+Input [2]: [s_store_sk#64, s_state#65]
+Condition : (s_state#65 IN (TN,AL,SD) AND isnotnull(s_store_sk#64))
+
+(45) Project [codegen id : 9]
+Output [1]: [s_store_sk#64]
+Input [2]: [s_store_sk#64, s_state#65]
+
+(46) BroadcastExchange
+Input [1]: [s_store_sk#64]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#66]
+
+(47) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ss_store_sk#56]
+Right keys [1]: [s_store_sk#64]
+Join condition: None
+
+(48) Project [codegen id : 11]
+Output [5]: [ss_item_sk#54, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60]
+Input [7]: [ss_item_sk#54, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, s_store_sk#64]
+
+(49) ReusedExchange [Reuses operator id: 27]
+Output [2]: [i_item_sk#67, i_item_id#68]
+
+(50) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ss_item_sk#54]
+Right keys [1]: [i_item_sk#67]
+Join condition: None
+
+(51) Project [codegen id : 11]
+Output [5]: [i_item_id#68, ss_quantity#57 AS agg1#24, ss_list_price#58 AS agg2#25, ss_coupon_amt#60 AS agg3#26, ss_sales_price#59 AS agg4#27]
+Input [7]: [ss_item_sk#54, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, i_item_sk#67, i_item_id#68]
+
+(52) HashAggregate [codegen id : 11]
+Input [5]: [i_item_id#68, agg1#24, agg2#25, agg3#26, agg4#27]
+Keys [1]: [i_item_id#68]
+Functions [4]: [partial_avg(agg1#24), partial_avg(UnscaledValue(agg2#25)), partial_avg(UnscaledValue(agg3#26)), partial_avg(UnscaledValue(agg4#27))]
+Aggregate Attributes [8]: [sum#69, count#70, sum#71, count#72, sum#73, count#74, sum#75, count#76]
+Results [9]: [i_item_id#68, sum#77, count#78, sum#79, count#80, sum#81, count#82, sum#83, count#84]
+
+(53) Exchange
+Input [9]: [i_item_id#68, sum#77, count#78, sum#79, count#80, sum#81, count#82, sum#83, count#84]
+Arguments: hashpartitioning(i_item_id#68, 5), ENSURE_REQUIREMENTS, [id=#85]
+
+(54) HashAggregate [codegen id : 12]
+Input [9]: [i_item_id#68, sum#77, count#78, sum#79, count#80, sum#81, count#82, sum#83, count#84]
+Keys [1]: [i_item_id#68]
+Functions [4]: [avg(agg1#24), avg(UnscaledValue(agg2#25)), avg(UnscaledValue(agg3#26)), avg(UnscaledValue(agg4#27))]
+Aggregate Attributes [4]: [avg(agg1#24)#86, avg(UnscaledValue(agg2#25))#87, avg(UnscaledValue(agg3#26))#88, avg(UnscaledValue(agg4#27))#89]
+Results [7]: [i_item_id#68, null AS s_state#90, 1 AS g_state#91, avg(agg1#24)#86 AS agg1#92, cast((avg(UnscaledValue(agg2#25))#87 / 100.0) as decimal(11,6)) AS agg2#93, cast((avg(UnscaledValue(agg3#26))#88 / 100.0) as decimal(11,6)) AS agg3#94, cast((avg(UnscaledValue(agg4#27))#89 / 100.0) as decimal(11,6)) AS agg4#95]
+
+(55) Scan parquet default.store_sales
+Output [8]: [ss_item_sk#96, ss_cdemo_sk#97, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, ss_sold_date_sk#103]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#103), (ss_sold_date_sk#103 >= 2451545), (ss_sold_date_sk#103 <= 2451910), dynamicpruningexpression(ss_sold_date_sk#103 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_cdemo_sk:int,ss_store_sk:int,ss_quantity:int,ss_list_price:decimal(7,2),ss_sales_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(56) ColumnarToRow [codegen id : 17]
+Input [8]: [ss_item_sk#96, ss_cdemo_sk#97, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, ss_sold_date_sk#103]
+
+(57) Filter [codegen id : 17]
+Input [8]: [ss_item_sk#96, ss_cdemo_sk#97, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, ss_sold_date_sk#103]
+Condition : ((isnotnull(ss_cdemo_sk#97) AND isnotnull(ss_store_sk#98)) AND isnotnull(ss_item_sk#96))
+
+(58) ReusedExchange [Reuses operator id: 8]
+Output [1]: [cd_demo_sk#104]
+
+(59) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ss_cdemo_sk#97]
+Right keys [1]: [cd_demo_sk#104]
+Join condition: None
+
+(60) Project [codegen id : 17]
+Output [7]: [ss_item_sk#96, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, ss_sold_date_sk#103]
+Input [9]: [ss_item_sk#96, ss_cdemo_sk#97, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, ss_sold_date_sk#103, cd_demo_sk#104]
+
+(61) ReusedExchange [Reuses operator id: 15]
+Output [1]: [d_date_sk#105]
+
+(62) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ss_sold_date_sk#103]
+Right keys [1]: [d_date_sk#105]
+Join condition: None
+
+(63) Project [codegen id : 17]
+Output [6]: [ss_item_sk#96, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102]
+Input [8]: [ss_item_sk#96, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, ss_sold_date_sk#103, d_date_sk#105]
+
+(64) ReusedExchange [Reuses operator id: 46]
+Output [1]: [s_store_sk#106]
+
+(65) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ss_store_sk#98]
+Right keys [1]: [s_store_sk#106]
+Join condition: None
+
+(66) Project [codegen id : 17]
+Output [5]: [ss_item_sk#96, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102]
+Input [7]: [ss_item_sk#96, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, s_store_sk#106]
+
+(67) Scan parquet default.item
+Output [1]: [i_item_sk#107]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int>
+
+(68) ColumnarToRow [codegen id : 16]
+Input [1]: [i_item_sk#107]
+
+(69) Filter [codegen id : 16]
+Input [1]: [i_item_sk#107]
+Condition : isnotnull(i_item_sk#107)
+
+(70) BroadcastExchange
+Input [1]: [i_item_sk#107]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#108]
+
+(71) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ss_item_sk#96]
+Right keys [1]: [i_item_sk#107]
+Join condition: None
+
+(72) Project [codegen id : 17]
+Output [4]: [ss_quantity#99 AS agg1#24, ss_list_price#100 AS agg2#25, ss_coupon_amt#102 AS agg3#26, ss_sales_price#101 AS agg4#27]
+Input [6]: [ss_item_sk#96, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, i_item_sk#107]
+
+(73) HashAggregate [codegen id : 17]
+Input [4]: [agg1#24, agg2#25, agg3#26, agg4#27]
+Keys: []
+Functions [4]: [partial_avg(agg1#24), partial_avg(UnscaledValue(agg2#25)), partial_avg(UnscaledValue(agg3#26)), partial_avg(UnscaledValue(agg4#27))]
+Aggregate Attributes [8]: [sum#109, count#110, sum#111, count#112, sum#113, count#114, sum#115, count#116]
+Results [8]: [sum#117, count#118, sum#119, count#120, sum#121, count#122, sum#123, count#124]
+
+(74) Exchange
+Input [8]: [sum#117, count#118, sum#119, count#120, sum#121, count#122, sum#123, count#124]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#125]
+
+(75) HashAggregate [codegen id : 18]
+Input [8]: [sum#117, count#118, sum#119, count#120, sum#121, count#122, sum#123, count#124]
+Keys: []
+Functions [4]: [avg(agg1#24), avg(UnscaledValue(agg2#25)), avg(UnscaledValue(agg3#26)), avg(UnscaledValue(agg4#27))]
+Aggregate Attributes [4]: [avg(agg1#24)#126, avg(UnscaledValue(agg2#25))#127, avg(UnscaledValue(agg3#26))#128, avg(UnscaledValue(agg4#27))#129]
+Results [7]: [null AS i_item_id#130, null AS s_state#131, 1 AS g_state#132, avg(agg1#24)#126 AS agg1#133, cast((avg(UnscaledValue(agg2#25))#127 / 100.0) as decimal(11,6)) AS agg2#134, cast((avg(UnscaledValue(agg3#26))#128 / 100.0) as decimal(11,6)) AS agg3#135, cast((avg(UnscaledValue(agg4#27))#129 / 100.0) as decimal(11,6)) AS agg4#136]
+
+(76) Union
+
+(77) TakeOrderedAndProject
+Input [7]: [i_item_id#22, s_state#19, g_state#49, agg1#50, agg2#51, agg3#52, agg4#53]
+Arguments: 100, [i_item_id#22 ASC NULLS FIRST, s_state#19 ASC NULLS FIRST], [i_item_id#22, s_state#19, g_state#49, agg1#50, agg2#51, agg3#52, agg4#53]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (78)
+
+
+(78) ReusedExchange [Reuses operator id: 15]
+Output [1]: [d_date_sk#15]
+
+Subquery:2 Hosting operator id = 33 Hosting Expression = ss_sold_date_sk#61 IN dynamicpruning#9
+
+Subquery:3 Hosting operator id = 55 Hosting Expression = ss_sold_date_sk#103 IN dynamicpruning#9
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q27/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q27/simplified.txt
new file mode 100644
index 0000000000000..ac6635c260051
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q27/simplified.txt
@@ -0,0 +1,117 @@
+TakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4]
+  Union
+    WholeStageCodegen (6)
+      HashAggregate [i_item_id,s_state,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4)),g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count]
+        InputAdapter
+          Exchange [i_item_id,s_state] #1
+            WholeStageCodegen (5)
+              HashAggregate [i_item_id,s_state,agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+                Project [i_item_id,s_state,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price]
+                  BroadcastHashJoin [ss_item_sk,i_item_sk]
+                    Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state]
+                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                        Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                            Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                              BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk]
+                                Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                                        SubqueryBroadcast [d_date_sk] #1
+                                          ReusedExchange [d_date_sk] #2
+                                InputAdapter
+                                  BroadcastExchange #3
+                                    WholeStageCodegen (1)
+                                      Project [cd_demo_sk]
+                                        Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status]
+                            InputAdapter
+                              BroadcastExchange #2
+                                WholeStageCodegen (2)
+                                  Project [d_date_sk]
+                                    Filter [d_year,d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.date_dim [d_date_sk,d_year]
+                        InputAdapter
+                          BroadcastExchange #4
+                            WholeStageCodegen (3)
+                              Filter [s_state,s_store_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.store [s_store_sk,s_state]
+                    InputAdapter
+                      BroadcastExchange #5
+                        WholeStageCodegen (4)
+                          Filter [i_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.item [i_item_sk,i_item_id]
+    WholeStageCodegen (12)
+      HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4)),s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count]
+        InputAdapter
+          Exchange [i_item_id] #6
+            WholeStageCodegen (11)
+              HashAggregate [i_item_id,agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+                Project [i_item_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price]
+                  BroadcastHashJoin [ss_item_sk,i_item_sk]
+                    Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                        Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                            Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                              BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk]
+                                Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                                        ReusedSubquery [d_date_sk] #1
+                                InputAdapter
+                                  ReusedExchange [cd_demo_sk] #3
+                            InputAdapter
+                              ReusedExchange [d_date_sk] #2
+                        InputAdapter
+                          BroadcastExchange #7
+                            WholeStageCodegen (9)
+                              Project [s_store_sk]
+                                Filter [s_state,s_store_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store [s_store_sk,s_state]
+                    InputAdapter
+                      ReusedExchange [i_item_sk,i_item_id] #5
+    WholeStageCodegen (18)
+      HashAggregate [sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4)),i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count]
+        InputAdapter
+          Exchange #8
+            WholeStageCodegen (17)
+              HashAggregate [agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+                Project [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price]
+                  BroadcastHashJoin [ss_item_sk,i_item_sk]
+                    Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                        Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                            Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                              BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk]
+                                Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                                        ReusedSubquery [d_date_sk] #1
+                                InputAdapter
+                                  ReusedExchange [cd_demo_sk] #3
+                            InputAdapter
+                              ReusedExchange [d_date_sk] #2
+                        InputAdapter
+                          ReusedExchange [s_store_sk] #7
+                    InputAdapter
+                      BroadcastExchange #9
+                        WholeStageCodegen (16)
+                          Filter [i_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.item [i_item_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q3.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q3.sf100/explain.txt
new file mode 100644
index 0000000000000..ae3e54365ced4
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q3.sf100/explain.txt
@@ -0,0 +1,133 @@
+== Physical Plan ==
+TakeOrderedAndProject (21)
++- * HashAggregate (20)
+   +- Exchange (19)
+      +- * HashAggregate (18)
+         +- * Project (17)
+            +- * BroadcastHashJoin Inner BuildRight (16)
+               :- * Project (10)
+               :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :- * Filter (3)
+               :     :  +- * ColumnarToRow (2)
+               :     :     +- Scan parquet default.store_sales (1)
+               :     +- BroadcastExchange (8)
+               :        +- * Project (7)
+               :           +- * Filter (6)
+               :              +- * ColumnarToRow (5)
+               :                 +- Scan parquet default.item (4)
+               +- BroadcastExchange (15)
+                  +- * Project (14)
+                     +- * Filter (13)
+                        +- * ColumnarToRow (12)
+                           +- Scan parquet default.date_dim (11)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#1, ss_net_profit#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [((((((((((ss_sold_date_sk#3 >= 2415355) AND (ss_sold_date_sk#3 <= 2415385)) OR ((ss_sold_date_sk#3 >= 2415720) AND (ss_sold_date_sk#3 <= 2415750))) OR (((ss_sold_date_sk#3 >= 2416085) AND (ss_sold_date_sk#3 <= 2416115)) OR ((ss_sold_date_sk#3 >= 2416450) AND (ss_sold_date_sk#3 <= 2416480)))) OR ((((ss_sold_date_sk#3 >= 2416816) AND (ss_sold_date_sk#3 <= 2416846)) OR ((ss_sold_date_sk#3 >= 2417181) AND (ss_sold_date_sk#3 <= 2417211))) OR ((ss_sold_date_sk#3 >= 2417546) AND (ss_sold_date_sk#3 <= 2417576)))) OR (((((ss_sold_date_sk#3 >= 2417911) AND (ss_sold_date_sk#3 <= 2417941)) OR ((ss_sold_date_sk#3 >= 2418277) AND (ss_sold_date_sk#3 <= 2418307))) OR ((ss_sold_date_sk#3 >= 2418642) AND (ss_sold_date_sk#3 <= 2418672))) OR ((((ss_sold_date_sk#3 >= 2419007) AND (ss_sold_date_sk#3 <= 2419037)) OR ((ss_sold_date_sk#3 >= 2419372) AND (ss_sold_date_sk#3 <= 2419402))) OR ((ss_sold_date_sk#3 >= 2419738) AND (ss_sold_date_sk#3 <= 2419768))))) OR ((((((ss_sold_date_sk#3 >= 2420103) AND (ss_sold_date_sk#3 <= 2420133)) OR ((ss_sold_date_sk#3 >= 2420468) AND (ss_sold_date_sk#3 <= 2420498))) OR ((ss_sold_date_sk#3 >= 2420833) AND (ss_sold_date_sk#3 <= 2420863))) OR ((((ss_sold_date_sk#3 >= 2421199) AND (ss_sold_date_sk#3 <= 2421229)) OR ((ss_sold_date_sk#3 >= 2421564) AND (ss_sold_date_sk#3 <= 2421594))) OR ((ss_sold_date_sk#3 >= 2421929) AND (ss_sold_date_sk#3 <= 2421959)))) OR (((((ss_sold_date_sk#3 >= 2422294) AND (ss_sold_date_sk#3 <= 2422324)) OR ((ss_sold_date_sk#3 >= 2422660) AND (ss_sold_date_sk#3 <= 2422690))) OR ((ss_sold_date_sk#3 >= 2423025) AND (ss_sold_date_sk#3 <= 2423055))) OR ((((ss_sold_date_sk#3 >= 2423390) AND (ss_sold_date_sk#3 <= 2423420)) OR ((ss_sold_date_sk#3 >= 2423755) AND (ss_sold_date_sk#3 <= 2423785))) OR ((ss_sold_date_sk#3 >= 2424121) AND (ss_sold_date_sk#3 <= 2424151)))))) OR (((((((ss_sold_date_sk#3 >= 2424486) AND (ss_sold_date_sk#3 <= 2424516)) OR ((ss_sold_date_sk#3 >= 2424851) AND (ss_sold_date_sk#3 <= 2424881))) OR (((ss_sold_date_sk#3 >= 2425216) AND (ss_sold_date_sk#3 <= 2425246)) OR ((ss_sold_date_sk#3 >= 2425582) AND (ss_sold_date_sk#3 <= 2425612)))) OR ((((ss_sold_date_sk#3 >= 2425947) AND (ss_sold_date_sk#3 <= 2425977)) OR ((ss_sold_date_sk#3 >= 2426312) AND (ss_sold_date_sk#3 <= 2426342))) OR ((ss_sold_date_sk#3 >= 2426677) AND (ss_sold_date_sk#3 <= 2426707)))) OR (((((ss_sold_date_sk#3 >= 2427043) AND (ss_sold_date_sk#3 <= 2427073)) OR ((ss_sold_date_sk#3 >= 2427408) AND (ss_sold_date_sk#3 <= 2427438))) OR ((ss_sold_date_sk#3 >= 2427773) AND (ss_sold_date_sk#3 <= 2427803))) OR ((((ss_sold_date_sk#3 >= 2428138) AND (ss_sold_date_sk#3 <= 2428168)) OR ((ss_sold_date_sk#3 >= 2428504) AND (ss_sold_date_sk#3 <= 2428534))) OR ((ss_sold_date_sk#3 >= 2428869) AND (ss_sold_date_sk#3 <= 2428899))))) OR ((((((ss_sold_date_sk#3 >= 2429234) AND (ss_sold_date_sk#3 <= 2429264)) OR ((ss_sold_date_sk#3 >= 2429599) AND (ss_sold_date_sk#3 <= 2429629))) OR ((ss_sold_date_sk#3 >= 2429965) AND (ss_sold_date_sk#3 <= 2429995))) OR ((((ss_sold_date_sk#3 >= 2430330) AND (ss_sold_date_sk#3 <= 2430360)) OR ((ss_sold_date_sk#3 >= 2430695) AND (ss_sold_date_sk#3 <= 2430725))) OR ((ss_sold_date_sk#3 >= 2431060) AND (ss_sold_date_sk#3 <= 2431090)))) OR (((((ss_sold_date_sk#3 >= 2431426) AND (ss_sold_date_sk#3 <= 2431456)) OR ((ss_sold_date_sk#3 >= 2431791) AND (ss_sold_date_sk#3 <= 2431821))) OR ((ss_sold_date_sk#3 >= 2432156) AND (ss_sold_date_sk#3 <= 2432186))) OR ((((ss_sold_date_sk#3 >= 2432521) AND (ss_sold_date_sk#3 <= 2432551)) OR ((ss_sold_date_sk#3 >= 2432887) AND (ss_sold_date_sk#3 <= 2432917))) OR ((ss_sold_date_sk#3 >= 2433252) AND (ss_sold_date_sk#3 <= 2433282))))))) OR ((((((((ss_sold_date_sk#3 >= 2433617) AND (ss_sold_date_sk#3 <= 2433647)) OR ((ss_sold_date_sk#3 >= 2433982) AND (ss_sold_date_sk#3 <= 2434012))) OR (((ss_sold_date_sk#3 >= 2434348) AND (ss_sold_date_sk#3 <= 2434378)) OR ((ss_sold_date_sk#3 >= 2434713) AND (ss_sold_date_sk#3 <= 2434743)))) OR ((((ss_sold_date_sk#3 >= 2435078) AND (ss_sold_date_sk#3 <= 2435108)) OR ((ss_sold_date_sk#3 >= 2435443) AND (ss_sold_date_sk#3 <= 2435473))) OR ((ss_sold_date_sk#3 >= 2435809) AND (ss_sold_date_sk#3 <= 2435839)))) OR (((((ss_sold_date_sk#3 >= 2436174) AND (ss_sold_date_sk#3 <= 2436204)) OR ((ss_sold_date_sk#3 >= 2436539) AND (ss_sold_date_sk#3 <= 2436569))) OR ((ss_sold_date_sk#3 >= 2436904) AND (ss_sold_date_sk#3 <= 2436934))) OR ((((ss_sold_date_sk#3 >= 2437270) AND (ss_sold_date_sk#3 <= 2437300)) OR ((ss_sold_date_sk#3 >= 2437635) AND (ss_sold_date_sk#3 <= 2437665))) OR ((ss_sold_date_sk#3 >= 2438000) AND (ss_sold_date_sk#3 <= 2438030))))) OR ((((((ss_sold_date_sk#3 >= 2438365) AND (ss_sold_date_sk#3 <= 2438395)) OR ((ss_sold_date_sk#3 >= 2438731) AND (ss_sold_date_sk#3 <= 2438761))) OR ((ss_sold_date_sk#3 >= 2439096) AND (ss_sold_date_sk#3 <= 2439126))) OR ((((ss_sold_date_sk#3 >= 2439461) AND (ss_sold_date_sk#3 <= 2439491)) OR ((ss_sold_date_sk#3 >= 2439826) AND (ss_sold_date_sk#3 <= 2439856))) OR ((ss_sold_date_sk#3 >= 2440192) AND (ss_sold_date_sk#3 <= 2440222)))) OR (((((ss_sold_date_sk#3 >= 2440557) AND (ss_sold_date_sk#3 <= 2440587)) OR ((ss_sold_date_sk#3 >= 2440922) AND (ss_sold_date_sk#3 <= 2440952))) OR ((ss_sold_date_sk#3 >= 2441287) AND (ss_sold_date_sk#3 <= 2441317))) OR ((((ss_sold_date_sk#3 >= 2441653) AND (ss_sold_date_sk#3 <= 2441683)) OR ((ss_sold_date_sk#3 >= 2442018) AND (ss_sold_date_sk#3 <= 2442048))) OR ((ss_sold_date_sk#3 >= 2442383) AND (ss_sold_date_sk#3 <= 2442413)))))) OR (((((((ss_sold_date_sk#3 >= 2442748) AND (ss_sold_date_sk#3 <= 2442778)) OR ((ss_sold_date_sk#3 >= 2443114) AND (ss_sold_date_sk#3 <= 2443144))) OR (((ss_sold_date_sk#3 >= 2443479) AND (ss_sold_date_sk#3 <= 2443509)) OR ((ss_sold_date_sk#3 >= 2443844) AND (ss_sold_date_sk#3 <= 2443874)))) OR ((((ss_sold_date_sk#3 >= 2444209) AND (ss_sold_date_sk#3 <= 2444239)) OR ((ss_sold_date_sk#3 >= 2444575) AND (ss_sold_date_sk#3 <= 2444605))) OR ((ss_sold_date_sk#3 >= 2444940) AND (ss_sold_date_sk#3 <= 2444970)))) OR (((((ss_sold_date_sk#3 >= 2445305) AND (ss_sold_date_sk#3 <= 2445335)) OR ((ss_sold_date_sk#3 >= 2445670) AND (ss_sold_date_sk#3 <= 2445700))) OR ((ss_sold_date_sk#3 >= 2446036) AND (ss_sold_date_sk#3 <= 2446066))) OR ((((ss_sold_date_sk#3 >= 2446401) AND (ss_sold_date_sk#3 <= 2446431)) OR ((ss_sold_date_sk#3 >= 2446766) AND (ss_sold_date_sk#3 <= 2446796))) OR ((ss_sold_date_sk#3 >= 2447131) AND (ss_sold_date_sk#3 <= 2447161))))) OR ((((((ss_sold_date_sk#3 >= 2447497) AND (ss_sold_date_sk#3 <= 2447527)) OR ((ss_sold_date_sk#3 >= 2447862) AND (ss_sold_date_sk#3 <= 2447892))) OR ((ss_sold_date_sk#3 >= 2448227) AND (ss_sold_date_sk#3 <= 2448257))) OR ((((ss_sold_date_sk#3 >= 2448592) AND (ss_sold_date_sk#3 <= 2448622)) OR ((ss_sold_date_sk#3 >= 2448958) AND (ss_sold_date_sk#3 <= 2448988))) OR ((ss_sold_date_sk#3 >= 2449323) AND (ss_sold_date_sk#3 <= 2449353)))) OR (((((ss_sold_date_sk#3 >= 2449688) AND (ss_sold_date_sk#3 <= 2449718)) OR ((ss_sold_date_sk#3 >= 2450053) AND (ss_sold_date_sk#3 <= 2450083))) OR ((ss_sold_date_sk#3 >= 2450419) AND (ss_sold_date_sk#3 <= 2450449))) OR ((((ss_sold_date_sk#3 >= 2450784) AND (ss_sold_date_sk#3 <= 2450814)) OR ((ss_sold_date_sk#3 >= 2451149) AND (ss_sold_date_sk#3 <= 2451179))) OR ((ss_sold_date_sk#3 >= 2451514) AND (ss_sold_date_sk#3 <= 2451544)))))))) OR (((((((((ss_sold_date_sk#3 >= 2451880) AND (ss_sold_date_sk#3 <= 2451910)) OR ((ss_sold_date_sk#3 >= 2452245) AND (ss_sold_date_sk#3 <= 2452275))) OR (((ss_sold_date_sk#3 >= 2452610) AND (ss_sold_date_sk#3 <= 2452640)) OR ((ss_sold_date_sk#3 >= 2452975) AND (ss_sold_date_sk#3 <= 2453005)))) OR ((((ss_sold_date_sk#3 >= 2453341) AND (ss_sold_date_sk#3 <= 2453371)) OR ((ss_sold_date_sk#3 >= 2453706) AND (ss_sold_date_sk#3 <= 2453736))) OR ((ss_sold_date_sk#3 >= 2454071) AND (ss_sold_date_sk#3 <= 2454101)))) OR (((((ss_sold_date_sk#3 >= 2454436) AND (ss_sold_date_sk#3 <= 2454466)) OR ((ss_sold_date_sk#3 >= 2454802) AND (ss_sold_date_sk#3 <= 2454832))) OR ((ss_sold_date_sk#3 >= 2455167) AND (ss_sold_date_sk#3 <= 2455197))) OR ((((ss_sold_date_sk#3 >= 2455532) AND (ss_sold_date_sk#3 <= 2455562)) OR ((ss_sold_date_sk#3 >= 2455897) AND (ss_sold_date_sk#3 <= 2455927))) OR ((ss_sold_date_sk#3 >= 2456263) AND (ss_sold_date_sk#3 <= 2456293))))) OR ((((((ss_sold_date_sk#3 >= 2456628) AND (ss_sold_date_sk#3 <= 2456658)) OR ((ss_sold_date_sk#3 >= 2456993) AND (ss_sold_date_sk#3 <= 2457023))) OR ((ss_sold_date_sk#3 >= 2457358) AND (ss_sold_date_sk#3 <= 2457388))) OR ((((ss_sold_date_sk#3 >= 2457724) AND (ss_sold_date_sk#3 <= 2457754)) OR ((ss_sold_date_sk#3 >= 2458089) AND (ss_sold_date_sk#3 <= 2458119))) OR ((ss_sold_date_sk#3 >= 2458454) AND (ss_sold_date_sk#3 <= 2458484)))) OR (((((ss_sold_date_sk#3 >= 2458819) AND (ss_sold_date_sk#3 <= 2458849)) OR ((ss_sold_date_sk#3 >= 2459185) AND (ss_sold_date_sk#3 <= 2459215))) OR ((ss_sold_date_sk#3 >= 2459550) AND (ss_sold_date_sk#3 <= 2459580))) OR ((((ss_sold_date_sk#3 >= 2459915) AND (ss_sold_date_sk#3 <= 2459945)) OR ((ss_sold_date_sk#3 >= 2460280) AND (ss_sold_date_sk#3 <= 2460310))) OR ((ss_sold_date_sk#3 >= 2460646) AND (ss_sold_date_sk#3 <= 2460676)))))) OR (((((((ss_sold_date_sk#3 >= 2461011) AND (ss_sold_date_sk#3 <= 2461041)) OR ((ss_sold_date_sk#3 >= 2461376) AND (ss_sold_date_sk#3 <= 2461406))) OR (((ss_sold_date_sk#3 >= 2461741) AND (ss_sold_date_sk#3 <= 2461771)) OR ((ss_sold_date_sk#3 >= 2462107) AND (ss_sold_date_sk#3 <= 2462137)))) OR ((((ss_sold_date_sk#3 >= 2462472) AND (ss_sold_date_sk#3 <= 2462502)) OR ((ss_sold_date_sk#3 >= 2462837) AND (ss_sold_date_sk#3 <= 2462867))) OR ((ss_sold_date_sk#3 >= 2463202) AND (ss_sold_date_sk#3 <= 2463232)))) OR (((((ss_sold_date_sk#3 >= 2463568) AND (ss_sold_date_sk#3 <= 2463598)) OR ((ss_sold_date_sk#3 >= 2463933) AND (ss_sold_date_sk#3 <= 2463963))) OR ((ss_sold_date_sk#3 >= 2464298) AND (ss_sold_date_sk#3 <= 2464328))) OR ((((ss_sold_date_sk#3 >= 2464663) AND (ss_sold_date_sk#3 <= 2464693)) OR ((ss_sold_date_sk#3 >= 2465029) AND (ss_sold_date_sk#3 <= 2465059))) OR ((ss_sold_date_sk#3 >= 2465394) AND (ss_sold_date_sk#3 <= 2465424))))) OR ((((((ss_sold_date_sk#3 >= 2465759) AND (ss_sold_date_sk#3 <= 2465789)) OR ((ss_sold_date_sk#3 >= 2466124) AND (ss_sold_date_sk#3 <= 2466154))) OR ((ss_sold_date_sk#3 >= 2466490) AND (ss_sold_date_sk#3 <= 2466520))) OR ((((ss_sold_date_sk#3 >= 2466855) AND (ss_sold_date_sk#3 <= 2466885)) OR ((ss_sold_date_sk#3 >= 2467220) AND (ss_sold_date_sk#3 <= 2467250))) OR ((ss_sold_date_sk#3 >= 2467585) AND (ss_sold_date_sk#3 <= 2467615)))) OR (((((ss_sold_date_sk#3 >= 2467951) AND (ss_sold_date_sk#3 <= 2467981)) OR ((ss_sold_date_sk#3 >= 2468316) AND (ss_sold_date_sk#3 <= 2468346))) OR ((ss_sold_date_sk#3 >= 2468681) AND (ss_sold_date_sk#3 <= 2468711))) OR ((((ss_sold_date_sk#3 >= 2469046) AND (ss_sold_date_sk#3 <= 2469076)) OR ((ss_sold_date_sk#3 >= 2469412) AND (ss_sold_date_sk#3 <= 2469442))) OR ((ss_sold_date_sk#3 >= 2469777) AND (ss_sold_date_sk#3 <= 2469807))))))) OR ((((((((ss_sold_date_sk#3 >= 2470142) AND (ss_sold_date_sk#3 <= 2470172)) OR ((ss_sold_date_sk#3 >= 2470507) AND (ss_sold_date_sk#3 <= 2470537))) OR (((ss_sold_date_sk#3 >= 2470873) AND (ss_sold_date_sk#3 <= 2470903)) OR ((ss_sold_date_sk#3 >= 2471238) AND (ss_sold_date_sk#3 <= 2471268)))) OR ((((ss_sold_date_sk#3 >= 2471603) AND (ss_sold_date_sk#3 <= 2471633)) OR ((ss_sold_date_sk#3 >= 2471968) AND (ss_sold_date_sk#3 <= 2471998))) OR ((ss_sold_date_sk#3 >= 2472334) AND (ss_sold_date_sk#3 <= 2472364)))) OR (((((ss_sold_date_sk#3 >= 2472699) AND (ss_sold_date_sk#3 <= 2472729)) OR ((ss_sold_date_sk#3 >= 2473064) AND (ss_sold_date_sk#3 <= 2473094))) OR ((ss_sold_date_sk#3 >= 2473429) AND (ss_sold_date_sk#3 <= 2473459))) OR ((((ss_sold_date_sk#3 >= 2473795) AND (ss_sold_date_sk#3 <= 2473825)) OR ((ss_sold_date_sk#3 >= 2474160) AND (ss_sold_date_sk#3 <= 2474190))) OR ((ss_sold_date_sk#3 >= 2474525) AND (ss_sold_date_sk#3 <= 2474555))))) OR ((((((ss_sold_date_sk#3 >= 2474890) AND (ss_sold_date_sk#3 <= 2474920)) OR ((ss_sold_date_sk#3 >= 2475256) AND (ss_sold_date_sk#3 <= 2475286))) OR ((ss_sold_date_sk#3 >= 2475621) AND (ss_sold_date_sk#3 <= 2475651))) OR ((((ss_sold_date_sk#3 >= 2475986) AND (ss_sold_date_sk#3 <= 2476016)) OR ((ss_sold_date_sk#3 >= 2476351) AND (ss_sold_date_sk#3 <= 2476381))) OR ((ss_sold_date_sk#3 >= 2476717) AND (ss_sold_date_sk#3 <= 2476747)))) OR (((((ss_sold_date_sk#3 >= 2477082) AND (ss_sold_date_sk#3 <= 2477112)) OR ((ss_sold_date_sk#3 >= 2477447) AND (ss_sold_date_sk#3 <= 2477477))) OR ((ss_sold_date_sk#3 >= 2477812) AND (ss_sold_date_sk#3 <= 2477842))) OR ((((ss_sold_date_sk#3 >= 2478178) AND (ss_sold_date_sk#3 <= 2478208)) OR ((ss_sold_date_sk#3 >= 2478543) AND (ss_sold_date_sk#3 <= 2478573))) OR ((ss_sold_date_sk#3 >= 2478908) AND (ss_sold_date_sk#3 <= 2478938)))))) OR (((((((ss_sold_date_sk#3 >= 2479273) AND (ss_sold_date_sk#3 <= 2479303)) OR ((ss_sold_date_sk#3 >= 2479639) AND (ss_sold_date_sk#3 <= 2479669))) OR (((ss_sold_date_sk#3 >= 2480004) AND (ss_sold_date_sk#3 <= 2480034)) OR ((ss_sold_date_sk#3 >= 2480369) AND (ss_sold_date_sk#3 <= 2480399)))) OR ((((ss_sold_date_sk#3 >= 2480734) AND (ss_sold_date_sk#3 <= 2480764)) OR ((ss_sold_date_sk#3 >= 2481100) AND (ss_sold_date_sk#3 <= 2481130))) OR ((ss_sold_date_sk#3 >= 2481465) AND (ss_sold_date_sk#3 <= 2481495)))) OR (((((ss_sold_date_sk#3 >= 2481830) AND (ss_sold_date_sk#3 <= 2481860)) OR ((ss_sold_date_sk#3 >= 2482195) AND (ss_sold_date_sk#3 <= 2482225))) OR ((ss_sold_date_sk#3 >= 2482561) AND (ss_sold_date_sk#3 <= 2482591))) OR ((((ss_sold_date_sk#3 >= 2482926) AND (ss_sold_date_sk#3 <= 2482956)) OR ((ss_sold_date_sk#3 >= 2483291) AND (ss_sold_date_sk#3 <= 2483321))) OR ((ss_sold_date_sk#3 >= 2483656) AND (ss_sold_date_sk#3 <= 2483686))))) OR ((((((ss_sold_date_sk#3 >= 2484022) AND (ss_sold_date_sk#3 <= 2484052)) OR ((ss_sold_date_sk#3 >= 2484387) AND (ss_sold_date_sk#3 <= 2484417))) OR ((ss_sold_date_sk#3 >= 2484752) AND (ss_sold_date_sk#3 <= 2484782))) OR ((((ss_sold_date_sk#3 >= 2485117) AND (ss_sold_date_sk#3 <= 2485147)) OR ((ss_sold_date_sk#3 >= 2485483) AND (ss_sold_date_sk#3 <= 2485513))) OR ((ss_sold_date_sk#3 >= 2485848) AND (ss_sold_date_sk#3 <= 2485878)))) OR (((((ss_sold_date_sk#3 >= 2486213) AND (ss_sold_date_sk#3 <= 2486243)) OR ((ss_sold_date_sk#3 >= 2486578) AND (ss_sold_date_sk#3 <= 2486608))) OR ((ss_sold_date_sk#3 >= 2486944) AND (ss_sold_date_sk#3 <= 2486974))) OR ((((ss_sold_date_sk#3 >= 2487309) AND (ss_sold_date_sk#3 <= 2487339)) OR ((ss_sold_date_sk#3 >= 2487674) AND (ss_sold_date_sk#3 <= 2487704))) OR ((ss_sold_date_sk#3 >= 2488039) AND (ss_sold_date_sk#3 <= 2488069))))))))), isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [ss_item_sk#1, ss_net_profit#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [ss_item_sk#1, ss_net_profit#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_item_sk#1)
+
+(4) Scan parquet default.item
+Output [4]: [i_item_sk#5, i_brand_id#6, i_brand#7, i_manufact_id#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,436), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_brand:string,i_manufact_id:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [i_item_sk#5, i_brand_id#6, i_brand#7, i_manufact_id#8]
+
+(6) Filter [codegen id : 1]
+Input [4]: [i_item_sk#5, i_brand_id#6, i_brand#7, i_manufact_id#8]
+Condition : ((isnotnull(i_manufact_id#8) AND (i_manufact_id#8 = 436)) AND isnotnull(i_item_sk#5))
+
+(7) Project [codegen id : 1]
+Output [3]: [i_item_sk#5, i_brand_id#6, i_brand#7]
+Input [4]: [i_item_sk#5, i_brand_id#6, i_brand#7, i_manufact_id#8]
+
+(8) BroadcastExchange
+Input [3]: [i_item_sk#5, i_brand_id#6, i_brand#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#9]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#5]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [4]: [ss_net_profit#2, ss_sold_date_sk#3, i_brand_id#6, i_brand#7]
+Input [6]: [ss_item_sk#1, ss_net_profit#2, ss_sold_date_sk#3, i_item_sk#5, i_brand_id#6, i_brand#7]
+
+(11) Scan parquet default.date_dim
+Output [3]: [d_date_sk#10, d_year#11, d_moy#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), EqualTo(d_moy,12), Or(Or(Or(Or(Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2415355),LessThanOrEqual(d_date_sk,2415385)),And(GreaterThanOrEqual(d_date_sk,2415720),LessThanOrEqual(d_date_sk,2415750))),Or(And(GreaterThanOrEqual(d_date_sk,2416085),LessThanOrEqual(d_date_sk,2416115)),And(GreaterThanOrEqual(d_date_sk,2416450),LessThanOrEqual(d_date_sk,2416480)))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2416816),LessThanOrEqual(d_date_sk,2416846)),And(GreaterThanOrEqual(d_date_sk,2417181),LessThanOrEqual(d_date_sk,2417211))),And(GreaterThanOrEqual(d_date_sk,2417546),LessThanOrEqual(d_date_sk,2417576)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2417911),LessThanOrEqual(d_date_sk,2417941)),And(GreaterThanOrEqual(d_date_sk,2418277),LessThanOrEqual(d_date_sk,2418307))),And(GreaterThanOrEqual(d_date_sk,2418642),LessThanOrEqual(d_date_sk,2418672))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2419007),LessThanOrEqual(d_date_sk,2419037)),And(GreaterThanOrEqual(d_date_sk,2419372),LessThanOrEqual(d_date_sk,2419402))),And(GreaterThanOrEqual(d_date_sk,2419738),LessThanOrEqual(d_date_sk,2419768))))),Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2420103),LessThanOrEqual(d_date_sk,2420133)),And(GreaterThanOrEqual(d_date_sk,2420468),LessThanOrEqual(d_date_sk,2420498))),And(GreaterThanOrEqual(d_date_sk,2420833),LessThanOrEqual(d_date_sk,2420863))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2421199),LessThanOrEqual(d_date_sk,2421229)),And(GreaterThanOrEqual(d_date_sk,2421564),LessThanOrEqual(d_date_sk,2421594))),And(GreaterThanOrEqual(d_date_sk,2421929),LessThanOrEqual(d_date_sk,2421959)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2422294),LessThanOrEqual(d_date_sk,2422324)),And(GreaterThanOrEqual(d_date_sk,2422660),LessThanOrEqual(d_date_sk,2422690))),And(GreaterThanOrEqual(d_date_sk,2423025),LessThanOrEqual(d_date_sk,2423055))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2423390),LessThanOrEqual(d_date_sk,2423420)),And(GreaterThanOrEqual(d_date_sk,2423755),LessThanOrEqual(d_date_sk,2423785))),And(GreaterThanOrEqual(d_date_sk,2424121),LessThanOrEqual(d_date_sk,2424151)))))),Or(Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2424486),LessThanOrEqual(d_date_sk,2424516)),And(GreaterThanOrEqual(d_date_sk,2424851),LessThanOrEqual(d_date_sk,2424881))),Or(And(GreaterThanOrEqual(d_date_sk,2425216),LessThanOrEqual(d_date_sk,2425246)),And(GreaterThanOrEqual(d_date_sk,2425582),LessThanOrEqual(d_date_sk,2425612)))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2425947),LessThanOrEqual(d_date_sk,2425977)),And(GreaterThanOrEqual(d_date_sk,2426312),LessThanOrEqual(d_date_sk,2426342))),And(GreaterThanOrEqual(d_date_sk,2426677),LessThanOrEqual(d_date_sk,2426707)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2427043),LessThanOrEqual(d_date_sk,2427073)),And(GreaterThanOrEqual(d_date_sk,2427408),LessThanOrEqual(d_date_sk,2427438))),And(GreaterThanOrEqual(d_date_sk,2427773),LessThanOrEqual(d_date_sk,2427803))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2428138),LessThanOrEqual(d_date_sk,2428168)),And(GreaterThanOrEqual(d_date_sk,2428504),LessThanOrEqual(d_date_sk,2428534))),And(GreaterThanOrEqual(d_date_sk,2428869),LessThanOrEqual(d_date_sk,2428899))))),Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2429234),LessThanOrEqual(d_date_sk,2429264)),And(GreaterThanOrEqual(d_date_sk,2429599),LessThanOrEqual(d_date_sk,2429629))),And(GreaterThanOrEqual(d_date_sk,2429965),LessThanOrEqual(d_date_sk,2429995))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2430330),LessThanOrEqual(d_date_sk,2430360)),And(GreaterThanOrEqual(d_date_sk,2430695),LessThanOrEqual(d_date_sk,2430725))),And(GreaterThanOrEqual(d_date_sk,2431060),LessThanOrEqual(d_date_sk,2431090)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2431426),LessThanOrEqual(d_date_sk,2431456)),And(GreaterThanOrEqual(d_date_sk,2431791),LessThanOrEqual(d_date_sk,2431821))),And(GreaterThanOrEqual(d_date_sk,2432156),LessThanOrEqual(d_date_sk,2432186))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2432521),LessThanOrEqual(d_date_sk,2432551)),And(GreaterThanOrEqual(d_date_sk,2432887),LessThanOrEqual(d_date_sk,2432917))),And(GreaterThanOrEqual(d_date_sk,2433252),LessThanOrEqual(d_date_sk,2433282))))))),Or(Or(Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2433617),LessThanOrEqual(d_date_sk,2433647)),And(GreaterThanOrEqual(d_date_sk,2433982),LessThanOrEqual(d_date_sk,2434012))),Or(And(GreaterThanOrEqual(d_date_sk,2434348),LessThanOrEqual(d_date_sk,2434378)),And(GreaterThanOrEqual(d_date_sk,2434713),LessThanOrEqual(d_date_sk,2434743)))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2435078),LessThanOrEqual(d_date_sk,2435108)),And(GreaterThanOrEqual(d_date_sk,2435443),LessThanOrEqual(d_date_sk,2435473))),And(GreaterThanOrEqual(d_date_sk,2435809),LessThanOrEqual(d_date_sk,2435839)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2436174),LessThanOrEqual(d_date_sk,2436204)),And(GreaterThanOrEqual(d_date_sk,2436539),LessThanOrEqual(d_date_sk,2436569))),And(GreaterThanOrEqual(d_date_sk,2436904),LessThanOrEqual(d_date_sk,2436934))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2437270),LessThanOrEqual(d_date_sk,2437300)),And(GreaterThanOrEqual(d_date_sk,2437635),LessThanOrEqual(d_date_sk,2437665))),And(GreaterThanOrEqual(d_date_sk,2438000),LessThanOrEqual(d_date_sk,2438030))))),Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2438365),LessThanOrEqual(d_date_sk,2438395)),And(GreaterThanOrEqual(d_date_sk,2438731),LessThanOrEqual(d_date_sk,2438761))),And(GreaterThanOrEqual(d_date_sk,2439096),LessThanOrEqual(d_date_sk,2439126))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2439461),LessThanOrEqual(d_date_sk,2439491)),And(GreaterThanOrEqual(d_date_sk,2439826),LessThanOrEqual(d_date_sk,2439856))),And(GreaterThanOrEqual(d_date_sk,2440192),LessThanOrEqual(d_date_sk,2440222)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2440557),LessThanOrEqual(d_date_sk,2440587)),And(GreaterThanOrEqual(d_date_sk,2440922),LessThanOrEqual(d_date_sk,2440952))),And(GreaterThanOrEqual(d_date_sk,2441287),LessThanOrEqual(d_date_sk,2441317))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2441653),LessThanOrEqual(d_date_sk,2441683)),And(GreaterThanOrEqual(d_date_sk,2442018),LessThanOrEqual(d_date_sk,2442048))),And(GreaterThanOrEqual(d_date_sk,2442383),LessThanOrEqual(d_date_sk,2442413)))))),Or(Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2442748),LessThanOrEqual(d_date_sk,2442778)),And(GreaterThanOrEqual(d_date_sk,2443114),LessThanOrEqual(d_date_sk,2443144))),Or(And(GreaterThanOrEqual(d_date_sk,2443479),LessThanOrEqual(d_date_sk,2443509)),And(GreaterThanOrEqual(d_date_sk,2443844),LessThanOrEqual(d_date_sk,2443874)))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2444209),LessThanOrEqual(d_date_sk,2444239)),And(GreaterThanOrEqual(d_date_sk,2444575),LessThanOrEqual(d_date_sk,2444605))),And(GreaterThanOrEqual(d_date_sk,2444940),LessThanOrEqual(d_date_sk,2444970)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2445305),LessThanOrEqual(d_date_sk,2445335)),And(GreaterThanOrEqual(d_date_sk,2445670),LessThanOrEqual(d_date_sk,2445700))),And(GreaterThanOrEqual(d_date_sk,2446036),LessThanOrEqual(d_date_sk,2446066))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2446401),LessThanOrEqual(d_date_sk,2446431)),And(GreaterThanOrEqual(d_date_sk,2446766),LessThanOrEqual(d_date_sk,2446796))),And(GreaterThanOrEqual(d_date_sk,2447131),LessThanOrEqual(d_date_sk,2447161))))),Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2447497),LessThanOrEqual(d_date_sk,2447527)),And(GreaterThanOrEqual(d_date_sk,2447862),LessThanOrEqual(d_date_sk,2447892))),And(GreaterThanOrEqual(d_date_sk,2448227),LessThanOrEqual(d_date_sk,2448257))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2448592),LessThanOrEqual(d_date_sk,2448622)),And(GreaterThanOrEqual(d_date_sk,2448958),LessThanOrEqual(d_date_sk,2448988))),And(GreaterThanOrEqual(d_date_sk,2449323),LessThanOrEqual(d_date_sk,2449353)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2449688),LessThanOrEqual(d_date_sk,2449718)),And(GreaterThanOrEqual(d_date_sk,2450053),LessThanOrEqual(d_date_sk,2450083))),And(GreaterThanOrEqual(d_date_sk,2450419),LessThanOrEqual(d_date_sk,2450449))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2450784),LessThanOrEqual(d_date_sk,2450814)),And(GreaterThanOrEqual(d_date_sk,2451149),LessThanOrEqual(d_date_sk,2451179))),And(GreaterThanOrEqual(d_date_sk,2451514),LessThanOrEqual(d_date_sk,2451544)))))))),Or(Or(Or(Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2451880),LessThanOrEqual(d_date_sk,2451910)),And(GreaterThanOrEqual(d_date_sk,2452245),LessThanOrEqual(d_date_sk,2452275))),Or(And(GreaterThanOrEqual(d_date_sk,2452610),LessThanOrEqual(d_date_sk,2452640)),And(GreaterThanOrEqual(d_date_sk,2452975),LessThanOrEqual(d_date_sk,2453005)))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2453341),LessThanOrEqual(d_date_sk,2453371)),And(GreaterThanOrEqual(d_date_sk,2453706),LessThanOrEqual(d_date_sk,2453736))),And(GreaterThanOrEqual(d_date_sk,2454071),LessThanOrEqual(d_date_sk,2454101)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2454436),LessThanOrEqual(d_date_sk,2454466)),And(GreaterThanOrEqual(d_date_sk,2454802),LessThanOrEqual(d_date_sk,2454832))),And(GreaterThanOrEqual(d_date_sk,2455167),LessThanOrEqual(d_date_sk,2455197))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2455532),LessThanOrEqual(d_date_sk,2455562)),And(GreaterThanOrEqual(d_date_sk,2455897),LessThanOrEqual(d_date_sk,2455927))),And(GreaterThanOrEqual(d_date_sk,2456263),LessThanOrEqual(d_date_sk,2456293))))),Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2456628),LessThanOrEqual(d_date_sk,2456658)),And(GreaterThanOrEqual(d_date_sk,2456993),LessThanOrEqual(d_date_sk,2457023))),And(GreaterThanOrEqual(d_date_sk,2457358),LessThanOrEqual(d_date_sk,2457388))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2457724),LessThanOrEqual(d_date_sk,2457754)),And(GreaterThanOrEqual(d_date_sk,2458089),LessThanOrEqual(d_date_sk,2458119))),And(GreaterThanOrEqual(d_date_sk,2458454),LessThanOrEqual(d_date_sk,2458484)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2458819),LessThanOrEqual(d_date_sk,2458849)),And(GreaterThanOrEqual(d_date_sk,2459185),LessThanOrEqual(d_date_sk,2459215))),And(GreaterThanOrEqual(d_date_sk,2459550),LessThanOrEqual(d_date_sk,2459580))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2459915),LessThanOrEqual(d_date_sk,2459945)),And(GreaterThanOrEqual(d_date_sk,2460280),LessThanOrEqual(d_date_sk,2460310))),And(GreaterThanOrEqual(d_date_sk,2460646),LessThanOrEqual(d_date_sk,2460676)))))),Or(Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2461011),LessThanOrEqual(d_date_sk,2461041)),And(GreaterThanOrEqual(d_date_sk,2461376),LessThanOrEqual(d_date_sk,2461406))),Or(And(GreaterThanOrEqual(d_date_sk,2461741),LessThanOrEqual(d_date_sk,2461771)),And(GreaterThanOrEqual(d_date_sk,2462107),LessThanOrEqual(d_date_sk,2462137)))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2462472),LessThanOrEqual(d_date_sk,2462502)),And(GreaterThanOrEqual(d_date_sk,2462837),LessThanOrEqual(d_date_sk,2462867))),And(GreaterThanOrEqual(d_date_sk,2463202),LessThanOrEqual(d_date_sk,2463232)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2463568),LessThanOrEqual(d_date_sk,2463598)),And(GreaterThanOrEqual(d_date_sk,2463933),LessThanOrEqual(d_date_sk,2463963))),And(GreaterThanOrEqual(d_date_sk,2464298),LessThanOrEqual(d_date_sk,2464328))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2464663),LessThanOrEqual(d_date_sk,2464693)),And(GreaterThanOrEqual(d_date_sk,2465029),LessThanOrEqual(d_date_sk,2465059))),And(GreaterThanOrEqual(d_date_sk,2465394),LessThanOrEqual(d_date_sk,2465424))))),Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2465759),LessThanOrEqual(d_date_sk,2465789)),And(GreaterThanOrEqual(d_date_sk,2466124),LessThanOrEqual(d_date_sk,2466154))),And(GreaterThanOrEqual(d_date_sk,2466490),LessThanOrEqual(d_date_sk,2466520))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2466855),LessThanOrEqual(d_date_sk,2466885)),And(GreaterThanOrEqual(d_date_sk,2467220),LessThanOrEqual(d_date_sk,2467250))),And(GreaterThanOrEqual(d_date_sk,2467585),LessThanOrEqual(d_date_sk,2467615)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2467951),LessThanOrEqual(d_date_sk,2467981)),And(GreaterThanOrEqual(d_date_sk,2468316),LessThanOrEqual(d_date_sk,2468346))),And(GreaterThanOrEqual(d_date_sk,2468681),LessThanOrEqual(d_date_sk,2468711))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2469046),LessThanOrEqual(d_date_sk,2469076)),And(GreaterThanOrEqual(d_date_sk,2469412),LessThanOrEqual(d_date_sk,2469442))),And(GreaterThanOrEqual(d_date_sk,2469777),LessThanOrEqual(d_date_sk,2469807))))))),Or(Or(Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2470142),LessThanOrEqual(d_date_sk,2470172)),And(GreaterThanOrEqual(d_date_sk,2470507),LessThanOrEqual(d_date_sk,2470537))),Or(And(GreaterThanOrEqual(d_date_sk,2470873),LessThanOrEqual(d_date_sk,2470903)),And(GreaterThanOrEqual(d_date_sk,2471238),LessThanOrEqual(d_date_sk,2471268)))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2471603),LessThanOrEqual(d_date_sk,2471633)),And(GreaterThanOrEqual(d_date_sk,2471968),LessThanOrEqual(d_date_sk,2471998))),And(GreaterThanOrEqual(d_date_sk,2472334),LessThanOrEqual(d_date_sk,2472364)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2472699),LessThanOrEqual(d_date_sk,2472729)),And(GreaterThanOrEqual(d_date_sk,2473064),LessThanOrEqual(d_date_sk,2473094))),And(GreaterThanOrEqual(d_date_sk,2473429),LessThanOrEqual(d_date_sk,2473459))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2473795),LessThanOrEqual(d_date_sk,2473825)),And(GreaterThanOrEqual(d_date_sk,2474160),LessThanOrEqual(d_date_sk,2474190))),And(GreaterThanOrEqual(d_date_sk,2474525),LessThanOrEqual(d_date_sk,2474555))))),Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2474890),LessThanOrEqual(d_date_sk,2474920)),And(GreaterThanOrEqual(d_date_sk,2475256),LessThanOrEqual(d_date_sk,2475286))),And(GreaterThanOrEqual(d_date_sk,2475621),LessThanOrEqual(d_date_sk,2475651))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2475986),LessThanOrEqual(d_date_sk,2476016)),And(GreaterThanOrEqual(d_date_sk,2476351),LessThanOrEqual(d_date_sk,2476381))),And(GreaterThanOrEqual(d_date_sk,2476717),LessThanOrEqual(d_date_sk,2476747)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2477082),LessThanOrEqual(d_date_sk,2477112)),And(GreaterThanOrEqual(d_date_sk,2477447),LessThanOrEqual(d_date_sk,2477477))),And(GreaterThanOrEqual(d_date_sk,2477812),LessThanOrEqual(d_date_sk,2477842))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2478178),LessThanOrEqual(d_date_sk,2478208)),And(GreaterThanOrEqual(d_date_sk,2478543),LessThanOrEqual(d_date_sk,2478573))),And(GreaterThanOrEqual(d_date_sk,2478908),LessThanOrEqual(d_date_sk,2478938)))))),Or(Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2479273),LessThanOrEqual(d_date_sk,2479303)),And(GreaterThanOrEqual(d_date_sk,2479639),LessThanOrEqual(d_date_sk,2479669))),Or(And(GreaterThanOrEqual(d_date_sk,2480004),LessThanOrEqual(d_date_sk,2480034)),And(GreaterThanOrEqual(d_date_sk,2480369),LessThanOrEqual(d_date_sk,2480399)))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2480734),LessThanOrEqual(d_date_sk,2480764)),And(GreaterThanOrEqual(d_date_sk,2481100),LessThanOrEqual(d_date_sk,2481130))),And(GreaterThanOrEqual(d_date_sk,2481465),LessThanOrEqual(d_date_sk,2481495)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2481830),LessThanOrEqual(d_date_sk,2481860)),And(GreaterThanOrEqual(d_date_sk,2482195),LessThanOrEqual(d_date_sk,2482225))),And(GreaterThanOrEqual(d_date_sk,2482561),LessThanOrEqual(d_date_sk,2482591))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2482926),LessThanOrEqual(d_date_sk,2482956)),And(GreaterThanOrEqual(d_date_sk,2483291),LessThanOrEqual(d_date_sk,2483321))),And(GreaterThanOrEqual(d_date_sk,2483656),LessThanOrEqual(d_date_sk,2483686))))),Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2484022),LessThanOrEqual(d_date_sk,2484052)),And(GreaterThanOrEqual(d_date_sk,2484387),LessThanOrEqual(d_date_sk,2484417))),And(GreaterThanOrEqual(d_date_sk,2484752),LessThanOrEqual(d_date_sk,2484782))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2485117),LessThanOrEqual(d_date_sk,2485147)),And(GreaterThanOrEqual(d_date_sk,2485483),LessThanOrEqual(d_date_sk,2485513))),And(GreaterThanOrEqual(d_date_sk,2485848),LessThanOrEqual(d_date_sk,2485878)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2486213),LessThanOrEqual(d_date_sk,2486243)),And(GreaterThanOrEqual(d_date_sk,2486578),LessThanOrEqual(d_date_sk,2486608))),And(GreaterThanOrEqual(d_date_sk,2486944),LessThanOrEqual(d_date_sk,2486974))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2487309),LessThanOrEqual(d_date_sk,2487339)),And(GreaterThanOrEqual(d_date_sk,2487674),LessThanOrEqual(d_date_sk,2487704))),And(GreaterThanOrEqual(d_date_sk,2488039),LessThanOrEqual(d_date_sk,2488069))))))))), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+
+(13) Filter [codegen id : 2]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+Condition : (((isnotnull(d_moy#12) AND (d_moy#12 = 12)) AND ((((((((((d_date_sk#10 >= 2415355) AND (d_date_sk#10 <= 2415385)) OR ((d_date_sk#10 >= 2415720) AND (d_date_sk#10 <= 2415750))) OR (((d_date_sk#10 >= 2416085) AND (d_date_sk#10 <= 2416115)) OR ((d_date_sk#10 >= 2416450) AND (d_date_sk#10 <= 2416480)))) OR ((((d_date_sk#10 >= 2416816) AND (d_date_sk#10 <= 2416846)) OR ((d_date_sk#10 >= 2417181) AND (d_date_sk#10 <= 2417211))) OR ((d_date_sk#10 >= 2417546) AND (d_date_sk#10 <= 2417576)))) OR (((((d_date_sk#10 >= 2417911) AND (d_date_sk#10 <= 2417941)) OR ((d_date_sk#10 >= 2418277) AND (d_date_sk#10 <= 2418307))) OR ((d_date_sk#10 >= 2418642) AND (d_date_sk#10 <= 2418672))) OR ((((d_date_sk#10 >= 2419007) AND (d_date_sk#10 <= 2419037)) OR ((d_date_sk#10 >= 2419372) AND (d_date_sk#10 <= 2419402))) OR ((d_date_sk#10 >= 2419738) AND (d_date_sk#10 <= 2419768))))) OR ((((((d_date_sk#10 >= 2420103) AND (d_date_sk#10 <= 2420133)) OR ((d_date_sk#10 >= 2420468) AND (d_date_sk#10 <= 2420498))) OR ((d_date_sk#10 >= 2420833) AND (d_date_sk#10 <= 2420863))) OR ((((d_date_sk#10 >= 2421199) AND (d_date_sk#10 <= 2421229)) OR ((d_date_sk#10 >= 2421564) AND (d_date_sk#10 <= 2421594))) OR ((d_date_sk#10 >= 2421929) AND (d_date_sk#10 <= 2421959)))) OR (((((d_date_sk#10 >= 2422294) AND (d_date_sk#10 <= 2422324)) OR ((d_date_sk#10 >= 2422660) AND (d_date_sk#10 <= 2422690))) OR ((d_date_sk#10 >= 2423025) AND (d_date_sk#10 <= 2423055))) OR ((((d_date_sk#10 >= 2423390) AND (d_date_sk#10 <= 2423420)) OR ((d_date_sk#10 >= 2423755) AND (d_date_sk#10 <= 2423785))) OR ((d_date_sk#10 >= 2424121) AND (d_date_sk#10 <= 2424151)))))) OR (((((((d_date_sk#10 >= 2424486) AND (d_date_sk#10 <= 2424516)) OR ((d_date_sk#10 >= 2424851) AND (d_date_sk#10 <= 2424881))) OR (((d_date_sk#10 >= 2425216) AND (d_date_sk#10 <= 2425246)) OR ((d_date_sk#10 >= 2425582) AND (d_date_sk#10 <= 2425612)))) OR ((((d_date_sk#10 >= 2425947) AND (d_date_sk#10 <= 2425977)) OR ((d_date_sk#10 >= 2426312) AND (d_date_sk#10 <= 2426342))) OR ((d_date_sk#10 >= 2426677) AND (d_date_sk#10 <= 2426707)))) OR (((((d_date_sk#10 >= 2427043) AND (d_date_sk#10 <= 2427073)) OR ((d_date_sk#10 >= 2427408) AND (d_date_sk#10 <= 2427438))) OR ((d_date_sk#10 >= 2427773) AND (d_date_sk#10 <= 2427803))) OR ((((d_date_sk#10 >= 2428138) AND (d_date_sk#10 <= 2428168)) OR ((d_date_sk#10 >= 2428504) AND (d_date_sk#10 <= 2428534))) OR ((d_date_sk#10 >= 2428869) AND (d_date_sk#10 <= 2428899))))) OR ((((((d_date_sk#10 >= 2429234) AND (d_date_sk#10 <= 2429264)) OR ((d_date_sk#10 >= 2429599) AND (d_date_sk#10 <= 2429629))) OR ((d_date_sk#10 >= 2429965) AND (d_date_sk#10 <= 2429995))) OR ((((d_date_sk#10 >= 2430330) AND (d_date_sk#10 <= 2430360)) OR ((d_date_sk#10 >= 2430695) AND (d_date_sk#10 <= 2430725))) OR ((d_date_sk#10 >= 2431060) AND (d_date_sk#10 <= 2431090)))) OR (((((d_date_sk#10 >= 2431426) AND (d_date_sk#10 <= 2431456)) OR ((d_date_sk#10 >= 2431791) AND (d_date_sk#10 <= 2431821))) OR ((d_date_sk#10 >= 2432156) AND (d_date_sk#10 <= 2432186))) OR ((((d_date_sk#10 >= 2432521) AND (d_date_sk#10 <= 2432551)) OR ((d_date_sk#10 >= 2432887) AND (d_date_sk#10 <= 2432917))) OR ((d_date_sk#10 >= 2433252) AND (d_date_sk#10 <= 2433282))))))) OR ((((((((d_date_sk#10 >= 2433617) AND (d_date_sk#10 <= 2433647)) OR ((d_date_sk#10 >= 2433982) AND (d_date_sk#10 <= 2434012))) OR (((d_date_sk#10 >= 2434348) AND (d_date_sk#10 <= 2434378)) OR ((d_date_sk#10 >= 2434713) AND (d_date_sk#10 <= 2434743)))) OR ((((d_date_sk#10 >= 2435078) AND (d_date_sk#10 <= 2435108)) OR ((d_date_sk#10 >= 2435443) AND (d_date_sk#10 <= 2435473))) OR ((d_date_sk#10 >= 2435809) AND (d_date_sk#10 <= 2435839)))) OR (((((d_date_sk#10 >= 2436174) AND (d_date_sk#10 <= 2436204)) OR ((d_date_sk#10 >= 2436539) AND (d_date_sk#10 <= 2436569))) OR ((d_date_sk#10 >= 2436904) AND (d_date_sk#10 <= 2436934))) OR ((((d_date_sk#10 >= 2437270) AND (d_date_sk#10 <= 2437300)) OR ((d_date_sk#10 >= 2437635) AND (d_date_sk#10 <= 2437665))) OR ((d_date_sk#10 >= 2438000) AND (d_date_sk#10 <= 2438030))))) OR ((((((d_date_sk#10 >= 2438365) AND (d_date_sk#10 <= 2438395)) OR ((d_date_sk#10 >= 2438731) AND (d_date_sk#10 <= 2438761))) OR ((d_date_sk#10 >= 2439096) AND (d_date_sk#10 <= 2439126))) OR ((((d_date_sk#10 >= 2439461) AND (d_date_sk#10 <= 2439491)) OR ((d_date_sk#10 >= 2439826) AND (d_date_sk#10 <= 2439856))) OR ((d_date_sk#10 >= 2440192) AND (d_date_sk#10 <= 2440222)))) OR (((((d_date_sk#10 >= 2440557) AND (d_date_sk#10 <= 2440587)) OR ((d_date_sk#10 >= 2440922) AND (d_date_sk#10 <= 2440952))) OR ((d_date_sk#10 >= 2441287) AND (d_date_sk#10 <= 2441317))) OR ((((d_date_sk#10 >= 2441653) AND (d_date_sk#10 <= 2441683)) OR ((d_date_sk#10 >= 2442018) AND (d_date_sk#10 <= 2442048))) OR ((d_date_sk#10 >= 2442383) AND (d_date_sk#10 <= 2442413)))))) OR (((((((d_date_sk#10 >= 2442748) AND (d_date_sk#10 <= 2442778)) OR ((d_date_sk#10 >= 2443114) AND (d_date_sk#10 <= 2443144))) OR (((d_date_sk#10 >= 2443479) AND (d_date_sk#10 <= 2443509)) OR ((d_date_sk#10 >= 2443844) AND (d_date_sk#10 <= 2443874)))) OR ((((d_date_sk#10 >= 2444209) AND (d_date_sk#10 <= 2444239)) OR ((d_date_sk#10 >= 2444575) AND (d_date_sk#10 <= 2444605))) OR ((d_date_sk#10 >= 2444940) AND (d_date_sk#10 <= 2444970)))) OR (((((d_date_sk#10 >= 2445305) AND (d_date_sk#10 <= 2445335)) OR ((d_date_sk#10 >= 2445670) AND (d_date_sk#10 <= 2445700))) OR ((d_date_sk#10 >= 2446036) AND (d_date_sk#10 <= 2446066))) OR ((((d_date_sk#10 >= 2446401) AND (d_date_sk#10 <= 2446431)) OR ((d_date_sk#10 >= 2446766) AND (d_date_sk#10 <= 2446796))) OR ((d_date_sk#10 >= 2447131) AND (d_date_sk#10 <= 2447161))))) OR ((((((d_date_sk#10 >= 2447497) AND (d_date_sk#10 <= 2447527)) OR ((d_date_sk#10 >= 2447862) AND (d_date_sk#10 <= 2447892))) OR ((d_date_sk#10 >= 2448227) AND (d_date_sk#10 <= 2448257))) OR ((((d_date_sk#10 >= 2448592) AND (d_date_sk#10 <= 2448622)) OR ((d_date_sk#10 >= 2448958) AND (d_date_sk#10 <= 2448988))) OR ((d_date_sk#10 >= 2449323) AND (d_date_sk#10 <= 2449353)))) OR (((((d_date_sk#10 >= 2449688) AND (d_date_sk#10 <= 2449718)) OR ((d_date_sk#10 >= 2450053) AND (d_date_sk#10 <= 2450083))) OR ((d_date_sk#10 >= 2450419) AND (d_date_sk#10 <= 2450449))) OR ((((d_date_sk#10 >= 2450784) AND (d_date_sk#10 <= 2450814)) OR ((d_date_sk#10 >= 2451149) AND (d_date_sk#10 <= 2451179))) OR ((d_date_sk#10 >= 2451514) AND (d_date_sk#10 <= 2451544)))))))) OR (((((((((d_date_sk#10 >= 2451880) AND (d_date_sk#10 <= 2451910)) OR ((d_date_sk#10 >= 2452245) AND (d_date_sk#10 <= 2452275))) OR (((d_date_sk#10 >= 2452610) AND (d_date_sk#10 <= 2452640)) OR ((d_date_sk#10 >= 2452975) AND (d_date_sk#10 <= 2453005)))) OR ((((d_date_sk#10 >= 2453341) AND (d_date_sk#10 <= 2453371)) OR ((d_date_sk#10 >= 2453706) AND (d_date_sk#10 <= 2453736))) OR ((d_date_sk#10 >= 2454071) AND (d_date_sk#10 <= 2454101)))) OR (((((d_date_sk#10 >= 2454436) AND (d_date_sk#10 <= 2454466)) OR ((d_date_sk#10 >= 2454802) AND (d_date_sk#10 <= 2454832))) OR ((d_date_sk#10 >= 2455167) AND (d_date_sk#10 <= 2455197))) OR ((((d_date_sk#10 >= 2455532) AND (d_date_sk#10 <= 2455562)) OR ((d_date_sk#10 >= 2455897) AND (d_date_sk#10 <= 2455927))) OR ((d_date_sk#10 >= 2456263) AND (d_date_sk#10 <= 2456293))))) OR ((((((d_date_sk#10 >= 2456628) AND (d_date_sk#10 <= 2456658)) OR ((d_date_sk#10 >= 2456993) AND (d_date_sk#10 <= 2457023))) OR ((d_date_sk#10 >= 2457358) AND (d_date_sk#10 <= 2457388))) OR ((((d_date_sk#10 >= 2457724) AND (d_date_sk#10 <= 2457754)) OR ((d_date_sk#10 >= 2458089) AND (d_date_sk#10 <= 2458119))) OR ((d_date_sk#10 >= 2458454) AND (d_date_sk#10 <= 2458484)))) OR (((((d_date_sk#10 >= 2458819) AND (d_date_sk#10 <= 2458849)) OR ((d_date_sk#10 >= 2459185) AND (d_date_sk#10 <= 2459215))) OR ((d_date_sk#10 >= 2459550) AND (d_date_sk#10 <= 2459580))) OR ((((d_date_sk#10 >= 2459915) AND (d_date_sk#10 <= 2459945)) OR ((d_date_sk#10 >= 2460280) AND (d_date_sk#10 <= 2460310))) OR ((d_date_sk#10 >= 2460646) AND (d_date_sk#10 <= 2460676)))))) OR (((((((d_date_sk#10 >= 2461011) AND (d_date_sk#10 <= 2461041)) OR ((d_date_sk#10 >= 2461376) AND (d_date_sk#10 <= 2461406))) OR (((d_date_sk#10 >= 2461741) AND (d_date_sk#10 <= 2461771)) OR ((d_date_sk#10 >= 2462107) AND (d_date_sk#10 <= 2462137)))) OR ((((d_date_sk#10 >= 2462472) AND (d_date_sk#10 <= 2462502)) OR ((d_date_sk#10 >= 2462837) AND (d_date_sk#10 <= 2462867))) OR ((d_date_sk#10 >= 2463202) AND (d_date_sk#10 <= 2463232)))) OR (((((d_date_sk#10 >= 2463568) AND (d_date_sk#10 <= 2463598)) OR ((d_date_sk#10 >= 2463933) AND (d_date_sk#10 <= 2463963))) OR ((d_date_sk#10 >= 2464298) AND (d_date_sk#10 <= 2464328))) OR ((((d_date_sk#10 >= 2464663) AND (d_date_sk#10 <= 2464693)) OR ((d_date_sk#10 >= 2465029) AND (d_date_sk#10 <= 2465059))) OR ((d_date_sk#10 >= 2465394) AND (d_date_sk#10 <= 2465424))))) OR ((((((d_date_sk#10 >= 2465759) AND (d_date_sk#10 <= 2465789)) OR ((d_date_sk#10 >= 2466124) AND (d_date_sk#10 <= 2466154))) OR ((d_date_sk#10 >= 2466490) AND (d_date_sk#10 <= 2466520))) OR ((((d_date_sk#10 >= 2466855) AND (d_date_sk#10 <= 2466885)) OR ((d_date_sk#10 >= 2467220) AND (d_date_sk#10 <= 2467250))) OR ((d_date_sk#10 >= 2467585) AND (d_date_sk#10 <= 2467615)))) OR (((((d_date_sk#10 >= 2467951) AND (d_date_sk#10 <= 2467981)) OR ((d_date_sk#10 >= 2468316) AND (d_date_sk#10 <= 2468346))) OR ((d_date_sk#10 >= 2468681) AND (d_date_sk#10 <= 2468711))) OR ((((d_date_sk#10 >= 2469046) AND (d_date_sk#10 <= 2469076)) OR ((d_date_sk#10 >= 2469412) AND (d_date_sk#10 <= 2469442))) OR ((d_date_sk#10 >= 2469777) AND (d_date_sk#10 <= 2469807))))))) OR ((((((((d_date_sk#10 >= 2470142) AND (d_date_sk#10 <= 2470172)) OR ((d_date_sk#10 >= 2470507) AND (d_date_sk#10 <= 2470537))) OR (((d_date_sk#10 >= 2470873) AND (d_date_sk#10 <= 2470903)) OR ((d_date_sk#10 >= 2471238) AND (d_date_sk#10 <= 2471268)))) OR ((((d_date_sk#10 >= 2471603) AND (d_date_sk#10 <= 2471633)) OR ((d_date_sk#10 >= 2471968) AND (d_date_sk#10 <= 2471998))) OR ((d_date_sk#10 >= 2472334) AND (d_date_sk#10 <= 2472364)))) OR (((((d_date_sk#10 >= 2472699) AND (d_date_sk#10 <= 2472729)) OR ((d_date_sk#10 >= 2473064) AND (d_date_sk#10 <= 2473094))) OR ((d_date_sk#10 >= 2473429) AND (d_date_sk#10 <= 2473459))) OR ((((d_date_sk#10 >= 2473795) AND (d_date_sk#10 <= 2473825)) OR ((d_date_sk#10 >= 2474160) AND (d_date_sk#10 <= 2474190))) OR ((d_date_sk#10 >= 2474525) AND (d_date_sk#10 <= 2474555))))) OR ((((((d_date_sk#10 >= 2474890) AND (d_date_sk#10 <= 2474920)) OR ((d_date_sk#10 >= 2475256) AND (d_date_sk#10 <= 2475286))) OR ((d_date_sk#10 >= 2475621) AND (d_date_sk#10 <= 2475651))) OR ((((d_date_sk#10 >= 2475986) AND (d_date_sk#10 <= 2476016)) OR ((d_date_sk#10 >= 2476351) AND (d_date_sk#10 <= 2476381))) OR ((d_date_sk#10 >= 2476717) AND (d_date_sk#10 <= 2476747)))) OR (((((d_date_sk#10 >= 2477082) AND (d_date_sk#10 <= 2477112)) OR ((d_date_sk#10 >= 2477447) AND (d_date_sk#10 <= 2477477))) OR ((d_date_sk#10 >= 2477812) AND (d_date_sk#10 <= 2477842))) OR ((((d_date_sk#10 >= 2478178) AND (d_date_sk#10 <= 2478208)) OR ((d_date_sk#10 >= 2478543) AND (d_date_sk#10 <= 2478573))) OR ((d_date_sk#10 >= 2478908) AND (d_date_sk#10 <= 2478938)))))) OR (((((((d_date_sk#10 >= 2479273) AND (d_date_sk#10 <= 2479303)) OR ((d_date_sk#10 >= 2479639) AND (d_date_sk#10 <= 2479669))) OR (((d_date_sk#10 >= 2480004) AND (d_date_sk#10 <= 2480034)) OR ((d_date_sk#10 >= 2480369) AND (d_date_sk#10 <= 2480399)))) OR ((((d_date_sk#10 >= 2480734) AND (d_date_sk#10 <= 2480764)) OR ((d_date_sk#10 >= 2481100) AND (d_date_sk#10 <= 2481130))) OR ((d_date_sk#10 >= 2481465) AND (d_date_sk#10 <= 2481495)))) OR (((((d_date_sk#10 >= 2481830) AND (d_date_sk#10 <= 2481860)) OR ((d_date_sk#10 >= 2482195) AND (d_date_sk#10 <= 2482225))) OR ((d_date_sk#10 >= 2482561) AND (d_date_sk#10 <= 2482591))) OR ((((d_date_sk#10 >= 2482926) AND (d_date_sk#10 <= 2482956)) OR ((d_date_sk#10 >= 2483291) AND (d_date_sk#10 <= 2483321))) OR ((d_date_sk#10 >= 2483656) AND (d_date_sk#10 <= 2483686))))) OR ((((((d_date_sk#10 >= 2484022) AND (d_date_sk#10 <= 2484052)) OR ((d_date_sk#10 >= 2484387) AND (d_date_sk#10 <= 2484417))) OR ((d_date_sk#10 >= 2484752) AND (d_date_sk#10 <= 2484782))) OR ((((d_date_sk#10 >= 2485117) AND (d_date_sk#10 <= 2485147)) OR ((d_date_sk#10 >= 2485483) AND (d_date_sk#10 <= 2485513))) OR ((d_date_sk#10 >= 2485848) AND (d_date_sk#10 <= 2485878)))) OR (((((d_date_sk#10 >= 2486213) AND (d_date_sk#10 <= 2486243)) OR ((d_date_sk#10 >= 2486578) AND (d_date_sk#10 <= 2486608))) OR ((d_date_sk#10 >= 2486944) AND (d_date_sk#10 <= 2486974))) OR ((((d_date_sk#10 >= 2487309) AND (d_date_sk#10 <= 2487339)) OR ((d_date_sk#10 >= 2487674) AND (d_date_sk#10 <= 2487704))) OR ((d_date_sk#10 >= 2488039) AND (d_date_sk#10 <= 2488069)))))))))) AND isnotnull(d_date_sk#10))
+
+(14) Project [codegen id : 2]
+Output [2]: [d_date_sk#10, d_year#11]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+
+(15) BroadcastExchange
+Input [2]: [d_date_sk#10, d_year#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(16) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(17) Project [codegen id : 3]
+Output [4]: [d_year#11, ss_net_profit#2, i_brand_id#6, i_brand#7]
+Input [6]: [ss_net_profit#2, ss_sold_date_sk#3, i_brand_id#6, i_brand#7, d_date_sk#10, d_year#11]
+
+(18) HashAggregate [codegen id : 3]
+Input [4]: [d_year#11, ss_net_profit#2, i_brand_id#6, i_brand#7]
+Keys [3]: [d_year#11, i_brand#7, i_brand_id#6]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))]
+Aggregate Attributes [1]: [sum#14]
+Results [4]: [d_year#11, i_brand#7, i_brand_id#6, sum#15]
+
+(19) Exchange
+Input [4]: [d_year#11, i_brand#7, i_brand_id#6, sum#15]
+Arguments: hashpartitioning(d_year#11, i_brand#7, i_brand_id#6, 5), ENSURE_REQUIREMENTS, [id=#16]
+
+(20) HashAggregate [codegen id : 4]
+Input [4]: [d_year#11, i_brand#7, i_brand_id#6, sum#15]
+Keys [3]: [d_year#11, i_brand#7, i_brand_id#6]
+Functions [1]: [sum(UnscaledValue(ss_net_profit#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#17]
+Results [4]: [d_year#11, i_brand_id#6 AS brand_id#18, i_brand#7 AS brand#19, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#17,17,2) AS sum_agg#20]
+
+(21) TakeOrderedAndProject
+Input [4]: [d_year#11, brand_id#18, brand#19, sum_agg#20]
+Arguments: 100, [d_year#11 ASC NULLS FIRST, sum_agg#20 DESC NULLS LAST, brand_id#18 ASC NULLS FIRST], [d_year#11, brand_id#18, brand#19, sum_agg#20]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (22)
+
+
+(22) ReusedExchange [Reuses operator id: 15]
+Output [2]: [d_date_sk#10, d_year#11]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q3.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q3.sf100/simplified.txt
new file mode 100644
index 0000000000000..edf36b708a750
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q3.sf100/simplified.txt
@@ -0,0 +1,33 @@
+TakeOrderedAndProject [d_year,sum_agg,brand_id,brand]
+  WholeStageCodegen (4)
+    HashAggregate [d_year,i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_net_profit)),brand_id,brand,sum_agg,sum]
+      InputAdapter
+        Exchange [d_year,i_brand,i_brand_id] #1
+          WholeStageCodegen (3)
+            HashAggregate [d_year,i_brand,i_brand_id,ss_net_profit] [sum,sum]
+              Project [d_year,ss_net_profit,i_brand_id,i_brand]
+                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                  Project [ss_net_profit,ss_sold_date_sk,i_brand_id,i_brand]
+                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                      Filter [ss_item_sk]
+                        ColumnarToRow
+                          InputAdapter
+                            Scan parquet default.store_sales [ss_item_sk,ss_net_profit,ss_sold_date_sk]
+                              SubqueryBroadcast [d_date_sk] #1
+                                ReusedExchange [d_date_sk,d_year] #2
+                      InputAdapter
+                        BroadcastExchange #3
+                          WholeStageCodegen (1)
+                            Project [i_item_sk,i_brand_id,i_brand]
+                              Filter [i_manufact_id,i_item_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id]
+                  InputAdapter
+                    BroadcastExchange #2
+                      WholeStageCodegen (2)
+                        Project [d_date_sk,d_year]
+                          Filter [d_moy,d_date_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q3/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q3/explain.txt
new file mode 100644
index 0000000000000..a60caf19f23df
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q3/explain.txt
@@ -0,0 +1,123 @@
+== Physical Plan ==
+TakeOrderedAndProject (21)
++- * HashAggregate (20)
+   +- Exchange (19)
+      +- * HashAggregate (18)
+         +- * Project (17)
+            +- * BroadcastHashJoin Inner BuildRight (16)
+               :- * Project (10)
+               :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :- * Project (4)
+               :     :  +- * Filter (3)
+               :     :     +- * ColumnarToRow (2)
+               :     :        +- Scan parquet default.date_dim (1)
+               :     +- BroadcastExchange (8)
+               :        +- * Filter (7)
+               :           +- * ColumnarToRow (6)
+               :              +- Scan parquet default.store_sales (5)
+               +- BroadcastExchange (15)
+                  +- * Project (14)
+                     +- * Filter (13)
+                        +- * ColumnarToRow (12)
+                           +- Scan parquet default.item (11)
+
+
+(1) Scan parquet default.date_dim
+Output [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), EqualTo(d_moy,12), Or(Or(Or(Or(Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2415355),LessThanOrEqual(d_date_sk,2415385)),And(GreaterThanOrEqual(d_date_sk,2415720),LessThanOrEqual(d_date_sk,2415750))),Or(And(GreaterThanOrEqual(d_date_sk,2416085),LessThanOrEqual(d_date_sk,2416115)),And(GreaterThanOrEqual(d_date_sk,2416450),LessThanOrEqual(d_date_sk,2416480)))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2416816),LessThanOrEqual(d_date_sk,2416846)),And(GreaterThanOrEqual(d_date_sk,2417181),LessThanOrEqual(d_date_sk,2417211))),And(GreaterThanOrEqual(d_date_sk,2417546),LessThanOrEqual(d_date_sk,2417576)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2417911),LessThanOrEqual(d_date_sk,2417941)),And(GreaterThanOrEqual(d_date_sk,2418277),LessThanOrEqual(d_date_sk,2418307))),And(GreaterThanOrEqual(d_date_sk,2418642),LessThanOrEqual(d_date_sk,2418672))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2419007),LessThanOrEqual(d_date_sk,2419037)),And(GreaterThanOrEqual(d_date_sk,2419372),LessThanOrEqual(d_date_sk,2419402))),And(GreaterThanOrEqual(d_date_sk,2419738),LessThanOrEqual(d_date_sk,2419768))))),Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2420103),LessThanOrEqual(d_date_sk,2420133)),And(GreaterThanOrEqual(d_date_sk,2420468),LessThanOrEqual(d_date_sk,2420498))),And(GreaterThanOrEqual(d_date_sk,2420833),LessThanOrEqual(d_date_sk,2420863))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2421199),LessThanOrEqual(d_date_sk,2421229)),And(GreaterThanOrEqual(d_date_sk,2421564),LessThanOrEqual(d_date_sk,2421594))),And(GreaterThanOrEqual(d_date_sk,2421929),LessThanOrEqual(d_date_sk,2421959)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2422294),LessThanOrEqual(d_date_sk,2422324)),And(GreaterThanOrEqual(d_date_sk,2422660),LessThanOrEqual(d_date_sk,2422690))),And(GreaterThanOrEqual(d_date_sk,2423025),LessThanOrEqual(d_date_sk,2423055))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2423390),LessThanOrEqual(d_date_sk,2423420)),And(GreaterThanOrEqual(d_date_sk,2423755),LessThanOrEqual(d_date_sk,2423785))),And(GreaterThanOrEqual(d_date_sk,2424121),LessThanOrEqual(d_date_sk,2424151)))))),Or(Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2424486),LessThanOrEqual(d_date_sk,2424516)),And(GreaterThanOrEqual(d_date_sk,2424851),LessThanOrEqual(d_date_sk,2424881))),Or(And(GreaterThanOrEqual(d_date_sk,2425216),LessThanOrEqual(d_date_sk,2425246)),And(GreaterThanOrEqual(d_date_sk,2425582),LessThanOrEqual(d_date_sk,2425612)))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2425947),LessThanOrEqual(d_date_sk,2425977)),And(GreaterThanOrEqual(d_date_sk,2426312),LessThanOrEqual(d_date_sk,2426342))),And(GreaterThanOrEqual(d_date_sk,2426677),LessThanOrEqual(d_date_sk,2426707)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2427043),LessThanOrEqual(d_date_sk,2427073)),And(GreaterThanOrEqual(d_date_sk,2427408),LessThanOrEqual(d_date_sk,2427438))),And(GreaterThanOrEqual(d_date_sk,2427773),LessThanOrEqual(d_date_sk,2427803))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2428138),LessThanOrEqual(d_date_sk,2428168)),And(GreaterThanOrEqual(d_date_sk,2428504),LessThanOrEqual(d_date_sk,2428534))),And(GreaterThanOrEqual(d_date_sk,2428869),LessThanOrEqual(d_date_sk,2428899))))),Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2429234),LessThanOrEqual(d_date_sk,2429264)),And(GreaterThanOrEqual(d_date_sk,2429599),LessThanOrEqual(d_date_sk,2429629))),And(GreaterThanOrEqual(d_date_sk,2429965),LessThanOrEqual(d_date_sk,2429995))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2430330),LessThanOrEqual(d_date_sk,2430360)),And(GreaterThanOrEqual(d_date_sk,2430695),LessThanOrEqual(d_date_sk,2430725))),And(GreaterThanOrEqual(d_date_sk,2431060),LessThanOrEqual(d_date_sk,2431090)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2431426),LessThanOrEqual(d_date_sk,2431456)),And(GreaterThanOrEqual(d_date_sk,2431791),LessThanOrEqual(d_date_sk,2431821))),And(GreaterThanOrEqual(d_date_sk,2432156),LessThanOrEqual(d_date_sk,2432186))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2432521),LessThanOrEqual(d_date_sk,2432551)),And(GreaterThanOrEqual(d_date_sk,2432887),LessThanOrEqual(d_date_sk,2432917))),And(GreaterThanOrEqual(d_date_sk,2433252),LessThanOrEqual(d_date_sk,2433282))))))),Or(Or(Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2433617),LessThanOrEqual(d_date_sk,2433647)),And(GreaterThanOrEqual(d_date_sk,2433982),LessThanOrEqual(d_date_sk,2434012))),Or(And(GreaterThanOrEqual(d_date_sk,2434348),LessThanOrEqual(d_date_sk,2434378)),And(GreaterThanOrEqual(d_date_sk,2434713),LessThanOrEqual(d_date_sk,2434743)))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2435078),LessThanOrEqual(d_date_sk,2435108)),And(GreaterThanOrEqual(d_date_sk,2435443),LessThanOrEqual(d_date_sk,2435473))),And(GreaterThanOrEqual(d_date_sk,2435809),LessThanOrEqual(d_date_sk,2435839)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2436174),LessThanOrEqual(d_date_sk,2436204)),And(GreaterThanOrEqual(d_date_sk,2436539),LessThanOrEqual(d_date_sk,2436569))),And(GreaterThanOrEqual(d_date_sk,2436904),LessThanOrEqual(d_date_sk,2436934))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2437270),LessThanOrEqual(d_date_sk,2437300)),And(GreaterThanOrEqual(d_date_sk,2437635),LessThanOrEqual(d_date_sk,2437665))),And(GreaterThanOrEqual(d_date_sk,2438000),LessThanOrEqual(d_date_sk,2438030))))),Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2438365),LessThanOrEqual(d_date_sk,2438395)),And(GreaterThanOrEqual(d_date_sk,2438731),LessThanOrEqual(d_date_sk,2438761))),And(GreaterThanOrEqual(d_date_sk,2439096),LessThanOrEqual(d_date_sk,2439126))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2439461),LessThanOrEqual(d_date_sk,2439491)),And(GreaterThanOrEqual(d_date_sk,2439826),LessThanOrEqual(d_date_sk,2439856))),And(GreaterThanOrEqual(d_date_sk,2440192),LessThanOrEqual(d_date_sk,2440222)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2440557),LessThanOrEqual(d_date_sk,2440587)),And(GreaterThanOrEqual(d_date_sk,2440922),LessThanOrEqual(d_date_sk,2440952))),And(GreaterThanOrEqual(d_date_sk,2441287),LessThanOrEqual(d_date_sk,2441317))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2441653),LessThanOrEqual(d_date_sk,2441683)),And(GreaterThanOrEqual(d_date_sk,2442018),LessThanOrEqual(d_date_sk,2442048))),And(GreaterThanOrEqual(d_date_sk,2442383),LessThanOrEqual(d_date_sk,2442413)))))),Or(Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2442748),LessThanOrEqual(d_date_sk,2442778)),And(GreaterThanOrEqual(d_date_sk,2443114),LessThanOrEqual(d_date_sk,2443144))),Or(And(GreaterThanOrEqual(d_date_sk,2443479),LessThanOrEqual(d_date_sk,2443509)),And(GreaterThanOrEqual(d_date_sk,2443844),LessThanOrEqual(d_date_sk,2443874)))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2444209),LessThanOrEqual(d_date_sk,2444239)),And(GreaterThanOrEqual(d_date_sk,2444575),LessThanOrEqual(d_date_sk,2444605))),And(GreaterThanOrEqual(d_date_sk,2444940),LessThanOrEqual(d_date_sk,2444970)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2445305),LessThanOrEqual(d_date_sk,2445335)),And(GreaterThanOrEqual(d_date_sk,2445670),LessThanOrEqual(d_date_sk,2445700))),And(GreaterThanOrEqual(d_date_sk,2446036),LessThanOrEqual(d_date_sk,2446066))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2446401),LessThanOrEqual(d_date_sk,2446431)),And(GreaterThanOrEqual(d_date_sk,2446766),LessThanOrEqual(d_date_sk,2446796))),And(GreaterThanOrEqual(d_date_sk,2447131),LessThanOrEqual(d_date_sk,2447161))))),Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2447497),LessThanOrEqual(d_date_sk,2447527)),And(GreaterThanOrEqual(d_date_sk,2447862),LessThanOrEqual(d_date_sk,2447892))),And(GreaterThanOrEqual(d_date_sk,2448227),LessThanOrEqual(d_date_sk,2448257))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2448592),LessThanOrEqual(d_date_sk,2448622)),And(GreaterThanOrEqual(d_date_sk,2448958),LessThanOrEqual(d_date_sk,2448988))),And(GreaterThanOrEqual(d_date_sk,2449323),LessThanOrEqual(d_date_sk,2449353)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2449688),LessThanOrEqual(d_date_sk,2449718)),And(GreaterThanOrEqual(d_date_sk,2450053),LessThanOrEqual(d_date_sk,2450083))),And(GreaterThanOrEqual(d_date_sk,2450419),LessThanOrEqual(d_date_sk,2450449))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2450784),LessThanOrEqual(d_date_sk,2450814)),And(GreaterThanOrEqual(d_date_sk,2451149),LessThanOrEqual(d_date_sk,2451179))),And(GreaterThanOrEqual(d_date_sk,2451514),LessThanOrEqual(d_date_sk,2451544)))))))),Or(Or(Or(Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2451880),LessThanOrEqual(d_date_sk,2451910)),And(GreaterThanOrEqual(d_date_sk,2452245),LessThanOrEqual(d_date_sk,2452275))),Or(And(GreaterThanOrEqual(d_date_sk,2452610),LessThanOrEqual(d_date_sk,2452640)),And(GreaterThanOrEqual(d_date_sk,2452975),LessThanOrEqual(d_date_sk,2453005)))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2453341),LessThanOrEqual(d_date_sk,2453371)),And(GreaterThanOrEqual(d_date_sk,2453706),LessThanOrEqual(d_date_sk,2453736))),And(GreaterThanOrEqual(d_date_sk,2454071),LessThanOrEqual(d_date_sk,2454101)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2454436),LessThanOrEqual(d_date_sk,2454466)),And(GreaterThanOrEqual(d_date_sk,2454802),LessThanOrEqual(d_date_sk,2454832))),And(GreaterThanOrEqual(d_date_sk,2455167),LessThanOrEqual(d_date_sk,2455197))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2455532),LessThanOrEqual(d_date_sk,2455562)),And(GreaterThanOrEqual(d_date_sk,2455897),LessThanOrEqual(d_date_sk,2455927))),And(GreaterThanOrEqual(d_date_sk,2456263),LessThanOrEqual(d_date_sk,2456293))))),Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2456628),LessThanOrEqual(d_date_sk,2456658)),And(GreaterThanOrEqual(d_date_sk,2456993),LessThanOrEqual(d_date_sk,2457023))),And(GreaterThanOrEqual(d_date_sk,2457358),LessThanOrEqual(d_date_sk,2457388))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2457724),LessThanOrEqual(d_date_sk,2457754)),And(GreaterThanOrEqual(d_date_sk,2458089),LessThanOrEqual(d_date_sk,2458119))),And(GreaterThanOrEqual(d_date_sk,2458454),LessThanOrEqual(d_date_sk,2458484)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2458819),LessThanOrEqual(d_date_sk,2458849)),And(GreaterThanOrEqual(d_date_sk,2459185),LessThanOrEqual(d_date_sk,2459215))),And(GreaterThanOrEqual(d_date_sk,2459550),LessThanOrEqual(d_date_sk,2459580))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2459915),LessThanOrEqual(d_date_sk,2459945)),And(GreaterThanOrEqual(d_date_sk,2460280),LessThanOrEqual(d_date_sk,2460310))),And(GreaterThanOrEqual(d_date_sk,2460646),LessThanOrEqual(d_date_sk,2460676)))))),Or(Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2461011),LessThanOrEqual(d_date_sk,2461041)),And(GreaterThanOrEqual(d_date_sk,2461376),LessThanOrEqual(d_date_sk,2461406))),Or(And(GreaterThanOrEqual(d_date_sk,2461741),LessThanOrEqual(d_date_sk,2461771)),And(GreaterThanOrEqual(d_date_sk,2462107),LessThanOrEqual(d_date_sk,2462137)))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2462472),LessThanOrEqual(d_date_sk,2462502)),And(GreaterThanOrEqual(d_date_sk,2462837),LessThanOrEqual(d_date_sk,2462867))),And(GreaterThanOrEqual(d_date_sk,2463202),LessThanOrEqual(d_date_sk,2463232)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2463568),LessThanOrEqual(d_date_sk,2463598)),And(GreaterThanOrEqual(d_date_sk,2463933),LessThanOrEqual(d_date_sk,2463963))),And(GreaterThanOrEqual(d_date_sk,2464298),LessThanOrEqual(d_date_sk,2464328))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2464663),LessThanOrEqual(d_date_sk,2464693)),And(GreaterThanOrEqual(d_date_sk,2465029),LessThanOrEqual(d_date_sk,2465059))),And(GreaterThanOrEqual(d_date_sk,2465394),LessThanOrEqual(d_date_sk,2465424))))),Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2465759),LessThanOrEqual(d_date_sk,2465789)),And(GreaterThanOrEqual(d_date_sk,2466124),LessThanOrEqual(d_date_sk,2466154))),And(GreaterThanOrEqual(d_date_sk,2466490),LessThanOrEqual(d_date_sk,2466520))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2466855),LessThanOrEqual(d_date_sk,2466885)),And(GreaterThanOrEqual(d_date_sk,2467220),LessThanOrEqual(d_date_sk,2467250))),And(GreaterThanOrEqual(d_date_sk,2467585),LessThanOrEqual(d_date_sk,2467615)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2467951),LessThanOrEqual(d_date_sk,2467981)),And(GreaterThanOrEqual(d_date_sk,2468316),LessThanOrEqual(d_date_sk,2468346))),And(GreaterThanOrEqual(d_date_sk,2468681),LessThanOrEqual(d_date_sk,2468711))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2469046),LessThanOrEqual(d_date_sk,2469076)),And(GreaterThanOrEqual(d_date_sk,2469412),LessThanOrEqual(d_date_sk,2469442))),And(GreaterThanOrEqual(d_date_sk,2469777),LessThanOrEqual(d_date_sk,2469807))))))),Or(Or(Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2470142),LessThanOrEqual(d_date_sk,2470172)),And(GreaterThanOrEqual(d_date_sk,2470507),LessThanOrEqual(d_date_sk,2470537))),Or(And(GreaterThanOrEqual(d_date_sk,2470873),LessThanOrEqual(d_date_sk,2470903)),And(GreaterThanOrEqual(d_date_sk,2471238),LessThanOrEqual(d_date_sk,2471268)))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2471603),LessThanOrEqual(d_date_sk,2471633)),And(GreaterThanOrEqual(d_date_sk,2471968),LessThanOrEqual(d_date_sk,2471998))),And(GreaterThanOrEqual(d_date_sk,2472334),LessThanOrEqual(d_date_sk,2472364)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2472699),LessThanOrEqual(d_date_sk,2472729)),And(GreaterThanOrEqual(d_date_sk,2473064),LessThanOrEqual(d_date_sk,2473094))),And(GreaterThanOrEqual(d_date_sk,2473429),LessThanOrEqual(d_date_sk,2473459))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2473795),LessThanOrEqual(d_date_sk,2473825)),And(GreaterThanOrEqual(d_date_sk,2474160),LessThanOrEqual(d_date_sk,2474190))),And(GreaterThanOrEqual(d_date_sk,2474525),LessThanOrEqual(d_date_sk,2474555))))),Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2474890),LessThanOrEqual(d_date_sk,2474920)),And(GreaterThanOrEqual(d_date_sk,2475256),LessThanOrEqual(d_date_sk,2475286))),And(GreaterThanOrEqual(d_date_sk,2475621),LessThanOrEqual(d_date_sk,2475651))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2475986),LessThanOrEqual(d_date_sk,2476016)),And(GreaterThanOrEqual(d_date_sk,2476351),LessThanOrEqual(d_date_sk,2476381))),And(GreaterThanOrEqual(d_date_sk,2476717),LessThanOrEqual(d_date_sk,2476747)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2477082),LessThanOrEqual(d_date_sk,2477112)),And(GreaterThanOrEqual(d_date_sk,2477447),LessThanOrEqual(d_date_sk,2477477))),And(GreaterThanOrEqual(d_date_sk,2477812),LessThanOrEqual(d_date_sk,2477842))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2478178),LessThanOrEqual(d_date_sk,2478208)),And(GreaterThanOrEqual(d_date_sk,2478543),LessThanOrEqual(d_date_sk,2478573))),And(GreaterThanOrEqual(d_date_sk,2478908),LessThanOrEqual(d_date_sk,2478938)))))),Or(Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2479273),LessThanOrEqual(d_date_sk,2479303)),And(GreaterThanOrEqual(d_date_sk,2479639),LessThanOrEqual(d_date_sk,2479669))),Or(And(GreaterThanOrEqual(d_date_sk,2480004),LessThanOrEqual(d_date_sk,2480034)),And(GreaterThanOrEqual(d_date_sk,2480369),LessThanOrEqual(d_date_sk,2480399)))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2480734),LessThanOrEqual(d_date_sk,2480764)),And(GreaterThanOrEqual(d_date_sk,2481100),LessThanOrEqual(d_date_sk,2481130))),And(GreaterThanOrEqual(d_date_sk,2481465),LessThanOrEqual(d_date_sk,2481495)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2481830),LessThanOrEqual(d_date_sk,2481860)),And(GreaterThanOrEqual(d_date_sk,2482195),LessThanOrEqual(d_date_sk,2482225))),And(GreaterThanOrEqual(d_date_sk,2482561),LessThanOrEqual(d_date_sk,2482591))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2482926),LessThanOrEqual(d_date_sk,2482956)),And(GreaterThanOrEqual(d_date_sk,2483291),LessThanOrEqual(d_date_sk,2483321))),And(GreaterThanOrEqual(d_date_sk,2483656),LessThanOrEqual(d_date_sk,2483686))))),Or(Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2484022),LessThanOrEqual(d_date_sk,2484052)),And(GreaterThanOrEqual(d_date_sk,2484387),LessThanOrEqual(d_date_sk,2484417))),And(GreaterThanOrEqual(d_date_sk,2484752),LessThanOrEqual(d_date_sk,2484782))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2485117),LessThanOrEqual(d_date_sk,2485147)),And(GreaterThanOrEqual(d_date_sk,2485483),LessThanOrEqual(d_date_sk,2485513))),And(GreaterThanOrEqual(d_date_sk,2485848),LessThanOrEqual(d_date_sk,2485878)))),Or(Or(Or(And(GreaterThanOrEqual(d_date_sk,2486213),LessThanOrEqual(d_date_sk,2486243)),And(GreaterThanOrEqual(d_date_sk,2486578),LessThanOrEqual(d_date_sk,2486608))),And(GreaterThanOrEqual(d_date_sk,2486944),LessThanOrEqual(d_date_sk,2486974))),Or(Or(And(GreaterThanOrEqual(d_date_sk,2487309),LessThanOrEqual(d_date_sk,2487339)),And(GreaterThanOrEqual(d_date_sk,2487674),LessThanOrEqual(d_date_sk,2487704))),And(GreaterThanOrEqual(d_date_sk,2488039),LessThanOrEqual(d_date_sk,2488069))))))))), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Condition : (((isnotnull(d_moy#3) AND (d_moy#3 = 12)) AND ((((((((((d_date_sk#1 >= 2415355) AND (d_date_sk#1 <= 2415385)) OR ((d_date_sk#1 >= 2415720) AND (d_date_sk#1 <= 2415750))) OR (((d_date_sk#1 >= 2416085) AND (d_date_sk#1 <= 2416115)) OR ((d_date_sk#1 >= 2416450) AND (d_date_sk#1 <= 2416480)))) OR ((((d_date_sk#1 >= 2416816) AND (d_date_sk#1 <= 2416846)) OR ((d_date_sk#1 >= 2417181) AND (d_date_sk#1 <= 2417211))) OR ((d_date_sk#1 >= 2417546) AND (d_date_sk#1 <= 2417576)))) OR (((((d_date_sk#1 >= 2417911) AND (d_date_sk#1 <= 2417941)) OR ((d_date_sk#1 >= 2418277) AND (d_date_sk#1 <= 2418307))) OR ((d_date_sk#1 >= 2418642) AND (d_date_sk#1 <= 2418672))) OR ((((d_date_sk#1 >= 2419007) AND (d_date_sk#1 <= 2419037)) OR ((d_date_sk#1 >= 2419372) AND (d_date_sk#1 <= 2419402))) OR ((d_date_sk#1 >= 2419738) AND (d_date_sk#1 <= 2419768))))) OR ((((((d_date_sk#1 >= 2420103) AND (d_date_sk#1 <= 2420133)) OR ((d_date_sk#1 >= 2420468) AND (d_date_sk#1 <= 2420498))) OR ((d_date_sk#1 >= 2420833) AND (d_date_sk#1 <= 2420863))) OR ((((d_date_sk#1 >= 2421199) AND (d_date_sk#1 <= 2421229)) OR ((d_date_sk#1 >= 2421564) AND (d_date_sk#1 <= 2421594))) OR ((d_date_sk#1 >= 2421929) AND (d_date_sk#1 <= 2421959)))) OR (((((d_date_sk#1 >= 2422294) AND (d_date_sk#1 <= 2422324)) OR ((d_date_sk#1 >= 2422660) AND (d_date_sk#1 <= 2422690))) OR ((d_date_sk#1 >= 2423025) AND (d_date_sk#1 <= 2423055))) OR ((((d_date_sk#1 >= 2423390) AND (d_date_sk#1 <= 2423420)) OR ((d_date_sk#1 >= 2423755) AND (d_date_sk#1 <= 2423785))) OR ((d_date_sk#1 >= 2424121) AND (d_date_sk#1 <= 2424151)))))) OR (((((((d_date_sk#1 >= 2424486) AND (d_date_sk#1 <= 2424516)) OR ((d_date_sk#1 >= 2424851) AND (d_date_sk#1 <= 2424881))) OR (((d_date_sk#1 >= 2425216) AND (d_date_sk#1 <= 2425246)) OR ((d_date_sk#1 >= 2425582) AND (d_date_sk#1 <= 2425612)))) OR ((((d_date_sk#1 >= 2425947) AND (d_date_sk#1 <= 2425977)) OR ((d_date_sk#1 >= 2426312) AND (d_date_sk#1 <= 2426342))) OR ((d_date_sk#1 >= 2426677) AND (d_date_sk#1 <= 2426707)))) OR (((((d_date_sk#1 >= 2427043) AND (d_date_sk#1 <= 2427073)) OR ((d_date_sk#1 >= 2427408) AND (d_date_sk#1 <= 2427438))) OR ((d_date_sk#1 >= 2427773) AND (d_date_sk#1 <= 2427803))) OR ((((d_date_sk#1 >= 2428138) AND (d_date_sk#1 <= 2428168)) OR ((d_date_sk#1 >= 2428504) AND (d_date_sk#1 <= 2428534))) OR ((d_date_sk#1 >= 2428869) AND (d_date_sk#1 <= 2428899))))) OR ((((((d_date_sk#1 >= 2429234) AND (d_date_sk#1 <= 2429264)) OR ((d_date_sk#1 >= 2429599) AND (d_date_sk#1 <= 2429629))) OR ((d_date_sk#1 >= 2429965) AND (d_date_sk#1 <= 2429995))) OR ((((d_date_sk#1 >= 2430330) AND (d_date_sk#1 <= 2430360)) OR ((d_date_sk#1 >= 2430695) AND (d_date_sk#1 <= 2430725))) OR ((d_date_sk#1 >= 2431060) AND (d_date_sk#1 <= 2431090)))) OR (((((d_date_sk#1 >= 2431426) AND (d_date_sk#1 <= 2431456)) OR ((d_date_sk#1 >= 2431791) AND (d_date_sk#1 <= 2431821))) OR ((d_date_sk#1 >= 2432156) AND (d_date_sk#1 <= 2432186))) OR ((((d_date_sk#1 >= 2432521) AND (d_date_sk#1 <= 2432551)) OR ((d_date_sk#1 >= 2432887) AND (d_date_sk#1 <= 2432917))) OR ((d_date_sk#1 >= 2433252) AND (d_date_sk#1 <= 2433282))))))) OR ((((((((d_date_sk#1 >= 2433617) AND (d_date_sk#1 <= 2433647)) OR ((d_date_sk#1 >= 2433982) AND (d_date_sk#1 <= 2434012))) OR (((d_date_sk#1 >= 2434348) AND (d_date_sk#1 <= 2434378)) OR ((d_date_sk#1 >= 2434713) AND (d_date_sk#1 <= 2434743)))) OR ((((d_date_sk#1 >= 2435078) AND (d_date_sk#1 <= 2435108)) OR ((d_date_sk#1 >= 2435443) AND (d_date_sk#1 <= 2435473))) OR ((d_date_sk#1 >= 2435809) AND (d_date_sk#1 <= 2435839)))) OR (((((d_date_sk#1 >= 2436174) AND (d_date_sk#1 <= 2436204)) OR ((d_date_sk#1 >= 2436539) AND (d_date_sk#1 <= 2436569))) OR ((d_date_sk#1 >= 2436904) AND (d_date_sk#1 <= 2436934))) OR ((((d_date_sk#1 >= 2437270) AND (d_date_sk#1 <= 2437300)) OR ((d_date_sk#1 >= 2437635) AND (d_date_sk#1 <= 2437665))) OR ((d_date_sk#1 >= 2438000) AND (d_date_sk#1 <= 2438030))))) OR ((((((d_date_sk#1 >= 2438365) AND (d_date_sk#1 <= 2438395)) OR ((d_date_sk#1 >= 2438731) AND (d_date_sk#1 <= 2438761))) OR ((d_date_sk#1 >= 2439096) AND (d_date_sk#1 <= 2439126))) OR ((((d_date_sk#1 >= 2439461) AND (d_date_sk#1 <= 2439491)) OR ((d_date_sk#1 >= 2439826) AND (d_date_sk#1 <= 2439856))) OR ((d_date_sk#1 >= 2440192) AND (d_date_sk#1 <= 2440222)))) OR (((((d_date_sk#1 >= 2440557) AND (d_date_sk#1 <= 2440587)) OR ((d_date_sk#1 >= 2440922) AND (d_date_sk#1 <= 2440952))) OR ((d_date_sk#1 >= 2441287) AND (d_date_sk#1 <= 2441317))) OR ((((d_date_sk#1 >= 2441653) AND (d_date_sk#1 <= 2441683)) OR ((d_date_sk#1 >= 2442018) AND (d_date_sk#1 <= 2442048))) OR ((d_date_sk#1 >= 2442383) AND (d_date_sk#1 <= 2442413)))))) OR (((((((d_date_sk#1 >= 2442748) AND (d_date_sk#1 <= 2442778)) OR ((d_date_sk#1 >= 2443114) AND (d_date_sk#1 <= 2443144))) OR (((d_date_sk#1 >= 2443479) AND (d_date_sk#1 <= 2443509)) OR ((d_date_sk#1 >= 2443844) AND (d_date_sk#1 <= 2443874)))) OR ((((d_date_sk#1 >= 2444209) AND (d_date_sk#1 <= 2444239)) OR ((d_date_sk#1 >= 2444575) AND (d_date_sk#1 <= 2444605))) OR ((d_date_sk#1 >= 2444940) AND (d_date_sk#1 <= 2444970)))) OR (((((d_date_sk#1 >= 2445305) AND (d_date_sk#1 <= 2445335)) OR ((d_date_sk#1 >= 2445670) AND (d_date_sk#1 <= 2445700))) OR ((d_date_sk#1 >= 2446036) AND (d_date_sk#1 <= 2446066))) OR ((((d_date_sk#1 >= 2446401) AND (d_date_sk#1 <= 2446431)) OR ((d_date_sk#1 >= 2446766) AND (d_date_sk#1 <= 2446796))) OR ((d_date_sk#1 >= 2447131) AND (d_date_sk#1 <= 2447161))))) OR ((((((d_date_sk#1 >= 2447497) AND (d_date_sk#1 <= 2447527)) OR ((d_date_sk#1 >= 2447862) AND (d_date_sk#1 <= 2447892))) OR ((d_date_sk#1 >= 2448227) AND (d_date_sk#1 <= 2448257))) OR ((((d_date_sk#1 >= 2448592) AND (d_date_sk#1 <= 2448622)) OR ((d_date_sk#1 >= 2448958) AND (d_date_sk#1 <= 2448988))) OR ((d_date_sk#1 >= 2449323) AND (d_date_sk#1 <= 2449353)))) OR (((((d_date_sk#1 >= 2449688) AND (d_date_sk#1 <= 2449718)) OR ((d_date_sk#1 >= 2450053) AND (d_date_sk#1 <= 2450083))) OR ((d_date_sk#1 >= 2450419) AND (d_date_sk#1 <= 2450449))) OR ((((d_date_sk#1 >= 2450784) AND (d_date_sk#1 <= 2450814)) OR ((d_date_sk#1 >= 2451149) AND (d_date_sk#1 <= 2451179))) OR ((d_date_sk#1 >= 2451514) AND (d_date_sk#1 <= 2451544)))))))) OR (((((((((d_date_sk#1 >= 2451880) AND (d_date_sk#1 <= 2451910)) OR ((d_date_sk#1 >= 2452245) AND (d_date_sk#1 <= 2452275))) OR (((d_date_sk#1 >= 2452610) AND (d_date_sk#1 <= 2452640)) OR ((d_date_sk#1 >= 2452975) AND (d_date_sk#1 <= 2453005)))) OR ((((d_date_sk#1 >= 2453341) AND (d_date_sk#1 <= 2453371)) OR ((d_date_sk#1 >= 2453706) AND (d_date_sk#1 <= 2453736))) OR ((d_date_sk#1 >= 2454071) AND (d_date_sk#1 <= 2454101)))) OR (((((d_date_sk#1 >= 2454436) AND (d_date_sk#1 <= 2454466)) OR ((d_date_sk#1 >= 2454802) AND (d_date_sk#1 <= 2454832))) OR ((d_date_sk#1 >= 2455167) AND (d_date_sk#1 <= 2455197))) OR ((((d_date_sk#1 >= 2455532) AND (d_date_sk#1 <= 2455562)) OR ((d_date_sk#1 >= 2455897) AND (d_date_sk#1 <= 2455927))) OR ((d_date_sk#1 >= 2456263) AND (d_date_sk#1 <= 2456293))))) OR ((((((d_date_sk#1 >= 2456628) AND (d_date_sk#1 <= 2456658)) OR ((d_date_sk#1 >= 2456993) AND (d_date_sk#1 <= 2457023))) OR ((d_date_sk#1 >= 2457358) AND (d_date_sk#1 <= 2457388))) OR ((((d_date_sk#1 >= 2457724) AND (d_date_sk#1 <= 2457754)) OR ((d_date_sk#1 >= 2458089) AND (d_date_sk#1 <= 2458119))) OR ((d_date_sk#1 >= 2458454) AND (d_date_sk#1 <= 2458484)))) OR (((((d_date_sk#1 >= 2458819) AND (d_date_sk#1 <= 2458849)) OR ((d_date_sk#1 >= 2459185) AND (d_date_sk#1 <= 2459215))) OR ((d_date_sk#1 >= 2459550) AND (d_date_sk#1 <= 2459580))) OR ((((d_date_sk#1 >= 2459915) AND (d_date_sk#1 <= 2459945)) OR ((d_date_sk#1 >= 2460280) AND (d_date_sk#1 <= 2460310))) OR ((d_date_sk#1 >= 2460646) AND (d_date_sk#1 <= 2460676)))))) OR (((((((d_date_sk#1 >= 2461011) AND (d_date_sk#1 <= 2461041)) OR ((d_date_sk#1 >= 2461376) AND (d_date_sk#1 <= 2461406))) OR (((d_date_sk#1 >= 2461741) AND (d_date_sk#1 <= 2461771)) OR ((d_date_sk#1 >= 2462107) AND (d_date_sk#1 <= 2462137)))) OR ((((d_date_sk#1 >= 2462472) AND (d_date_sk#1 <= 2462502)) OR ((d_date_sk#1 >= 2462837) AND (d_date_sk#1 <= 2462867))) OR ((d_date_sk#1 >= 2463202) AND (d_date_sk#1 <= 2463232)))) OR (((((d_date_sk#1 >= 2463568) AND (d_date_sk#1 <= 2463598)) OR ((d_date_sk#1 >= 2463933) AND (d_date_sk#1 <= 2463963))) OR ((d_date_sk#1 >= 2464298) AND (d_date_sk#1 <= 2464328))) OR ((((d_date_sk#1 >= 2464663) AND (d_date_sk#1 <= 2464693)) OR ((d_date_sk#1 >= 2465029) AND (d_date_sk#1 <= 2465059))) OR ((d_date_sk#1 >= 2465394) AND (d_date_sk#1 <= 2465424))))) OR ((((((d_date_sk#1 >= 2465759) AND (d_date_sk#1 <= 2465789)) OR ((d_date_sk#1 >= 2466124) AND (d_date_sk#1 <= 2466154))) OR ((d_date_sk#1 >= 2466490) AND (d_date_sk#1 <= 2466520))) OR ((((d_date_sk#1 >= 2466855) AND (d_date_sk#1 <= 2466885)) OR ((d_date_sk#1 >= 2467220) AND (d_date_sk#1 <= 2467250))) OR ((d_date_sk#1 >= 2467585) AND (d_date_sk#1 <= 2467615)))) OR (((((d_date_sk#1 >= 2467951) AND (d_date_sk#1 <= 2467981)) OR ((d_date_sk#1 >= 2468316) AND (d_date_sk#1 <= 2468346))) OR ((d_date_sk#1 >= 2468681) AND (d_date_sk#1 <= 2468711))) OR ((((d_date_sk#1 >= 2469046) AND (d_date_sk#1 <= 2469076)) OR ((d_date_sk#1 >= 2469412) AND (d_date_sk#1 <= 2469442))) OR ((d_date_sk#1 >= 2469777) AND (d_date_sk#1 <= 2469807))))))) OR ((((((((d_date_sk#1 >= 2470142) AND (d_date_sk#1 <= 2470172)) OR ((d_date_sk#1 >= 2470507) AND (d_date_sk#1 <= 2470537))) OR (((d_date_sk#1 >= 2470873) AND (d_date_sk#1 <= 2470903)) OR ((d_date_sk#1 >= 2471238) AND (d_date_sk#1 <= 2471268)))) OR ((((d_date_sk#1 >= 2471603) AND (d_date_sk#1 <= 2471633)) OR ((d_date_sk#1 >= 2471968) AND (d_date_sk#1 <= 2471998))) OR ((d_date_sk#1 >= 2472334) AND (d_date_sk#1 <= 2472364)))) OR (((((d_date_sk#1 >= 2472699) AND (d_date_sk#1 <= 2472729)) OR ((d_date_sk#1 >= 2473064) AND (d_date_sk#1 <= 2473094))) OR ((d_date_sk#1 >= 2473429) AND (d_date_sk#1 <= 2473459))) OR ((((d_date_sk#1 >= 2473795) AND (d_date_sk#1 <= 2473825)) OR ((d_date_sk#1 >= 2474160) AND (d_date_sk#1 <= 2474190))) OR ((d_date_sk#1 >= 2474525) AND (d_date_sk#1 <= 2474555))))) OR ((((((d_date_sk#1 >= 2474890) AND (d_date_sk#1 <= 2474920)) OR ((d_date_sk#1 >= 2475256) AND (d_date_sk#1 <= 2475286))) OR ((d_date_sk#1 >= 2475621) AND (d_date_sk#1 <= 2475651))) OR ((((d_date_sk#1 >= 2475986) AND (d_date_sk#1 <= 2476016)) OR ((d_date_sk#1 >= 2476351) AND (d_date_sk#1 <= 2476381))) OR ((d_date_sk#1 >= 2476717) AND (d_date_sk#1 <= 2476747)))) OR (((((d_date_sk#1 >= 2477082) AND (d_date_sk#1 <= 2477112)) OR ((d_date_sk#1 >= 2477447) AND (d_date_sk#1 <= 2477477))) OR ((d_date_sk#1 >= 2477812) AND (d_date_sk#1 <= 2477842))) OR ((((d_date_sk#1 >= 2478178) AND (d_date_sk#1 <= 2478208)) OR ((d_date_sk#1 >= 2478543) AND (d_date_sk#1 <= 2478573))) OR ((d_date_sk#1 >= 2478908) AND (d_date_sk#1 <= 2478938)))))) OR (((((((d_date_sk#1 >= 2479273) AND (d_date_sk#1 <= 2479303)) OR ((d_date_sk#1 >= 2479639) AND (d_date_sk#1 <= 2479669))) OR (((d_date_sk#1 >= 2480004) AND (d_date_sk#1 <= 2480034)) OR ((d_date_sk#1 >= 2480369) AND (d_date_sk#1 <= 2480399)))) OR ((((d_date_sk#1 >= 2480734) AND (d_date_sk#1 <= 2480764)) OR ((d_date_sk#1 >= 2481100) AND (d_date_sk#1 <= 2481130))) OR ((d_date_sk#1 >= 2481465) AND (d_date_sk#1 <= 2481495)))) OR (((((d_date_sk#1 >= 2481830) AND (d_date_sk#1 <= 2481860)) OR ((d_date_sk#1 >= 2482195) AND (d_date_sk#1 <= 2482225))) OR ((d_date_sk#1 >= 2482561) AND (d_date_sk#1 <= 2482591))) OR ((((d_date_sk#1 >= 2482926) AND (d_date_sk#1 <= 2482956)) OR ((d_date_sk#1 >= 2483291) AND (d_date_sk#1 <= 2483321))) OR ((d_date_sk#1 >= 2483656) AND (d_date_sk#1 <= 2483686))))) OR ((((((d_date_sk#1 >= 2484022) AND (d_date_sk#1 <= 2484052)) OR ((d_date_sk#1 >= 2484387) AND (d_date_sk#1 <= 2484417))) OR ((d_date_sk#1 >= 2484752) AND (d_date_sk#1 <= 2484782))) OR ((((d_date_sk#1 >= 2485117) AND (d_date_sk#1 <= 2485147)) OR ((d_date_sk#1 >= 2485483) AND (d_date_sk#1 <= 2485513))) OR ((d_date_sk#1 >= 2485848) AND (d_date_sk#1 <= 2485878)))) OR (((((d_date_sk#1 >= 2486213) AND (d_date_sk#1 <= 2486243)) OR ((d_date_sk#1 >= 2486578) AND (d_date_sk#1 <= 2486608))) OR ((d_date_sk#1 >= 2486944) AND (d_date_sk#1 <= 2486974))) OR ((((d_date_sk#1 >= 2487309) AND (d_date_sk#1 <= 2487339)) OR ((d_date_sk#1 >= 2487674) AND (d_date_sk#1 <= 2487704))) OR ((d_date_sk#1 >= 2488039) AND (d_date_sk#1 <= 2488069)))))))))) AND isnotnull(d_date_sk#1))
+
+(4) Project [codegen id : 3]
+Output [2]: [d_date_sk#1, d_year#2]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(5) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#4, ss_net_profit#5, ss_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [((((((((((ss_sold_date_sk#6 >= 2415355) AND (ss_sold_date_sk#6 <= 2415385)) OR ((ss_sold_date_sk#6 >= 2415720) AND (ss_sold_date_sk#6 <= 2415750))) OR (((ss_sold_date_sk#6 >= 2416085) AND (ss_sold_date_sk#6 <= 2416115)) OR ((ss_sold_date_sk#6 >= 2416450) AND (ss_sold_date_sk#6 <= 2416480)))) OR ((((ss_sold_date_sk#6 >= 2416816) AND (ss_sold_date_sk#6 <= 2416846)) OR ((ss_sold_date_sk#6 >= 2417181) AND (ss_sold_date_sk#6 <= 2417211))) OR ((ss_sold_date_sk#6 >= 2417546) AND (ss_sold_date_sk#6 <= 2417576)))) OR (((((ss_sold_date_sk#6 >= 2417911) AND (ss_sold_date_sk#6 <= 2417941)) OR ((ss_sold_date_sk#6 >= 2418277) AND (ss_sold_date_sk#6 <= 2418307))) OR ((ss_sold_date_sk#6 >= 2418642) AND (ss_sold_date_sk#6 <= 2418672))) OR ((((ss_sold_date_sk#6 >= 2419007) AND (ss_sold_date_sk#6 <= 2419037)) OR ((ss_sold_date_sk#6 >= 2419372) AND (ss_sold_date_sk#6 <= 2419402))) OR ((ss_sold_date_sk#6 >= 2419738) AND (ss_sold_date_sk#6 <= 2419768))))) OR ((((((ss_sold_date_sk#6 >= 2420103) AND (ss_sold_date_sk#6 <= 2420133)) OR ((ss_sold_date_sk#6 >= 2420468) AND (ss_sold_date_sk#6 <= 2420498))) OR ((ss_sold_date_sk#6 >= 2420833) AND (ss_sold_date_sk#6 <= 2420863))) OR ((((ss_sold_date_sk#6 >= 2421199) AND (ss_sold_date_sk#6 <= 2421229)) OR ((ss_sold_date_sk#6 >= 2421564) AND (ss_sold_date_sk#6 <= 2421594))) OR ((ss_sold_date_sk#6 >= 2421929) AND (ss_sold_date_sk#6 <= 2421959)))) OR (((((ss_sold_date_sk#6 >= 2422294) AND (ss_sold_date_sk#6 <= 2422324)) OR ((ss_sold_date_sk#6 >= 2422660) AND (ss_sold_date_sk#6 <= 2422690))) OR ((ss_sold_date_sk#6 >= 2423025) AND (ss_sold_date_sk#6 <= 2423055))) OR ((((ss_sold_date_sk#6 >= 2423390) AND (ss_sold_date_sk#6 <= 2423420)) OR ((ss_sold_date_sk#6 >= 2423755) AND (ss_sold_date_sk#6 <= 2423785))) OR ((ss_sold_date_sk#6 >= 2424121) AND (ss_sold_date_sk#6 <= 2424151)))))) OR (((((((ss_sold_date_sk#6 >= 2424486) AND (ss_sold_date_sk#6 <= 2424516)) OR ((ss_sold_date_sk#6 >= 2424851) AND (ss_sold_date_sk#6 <= 2424881))) OR (((ss_sold_date_sk#6 >= 2425216) AND (ss_sold_date_sk#6 <= 2425246)) OR ((ss_sold_date_sk#6 >= 2425582) AND (ss_sold_date_sk#6 <= 2425612)))) OR ((((ss_sold_date_sk#6 >= 2425947) AND (ss_sold_date_sk#6 <= 2425977)) OR ((ss_sold_date_sk#6 >= 2426312) AND (ss_sold_date_sk#6 <= 2426342))) OR ((ss_sold_date_sk#6 >= 2426677) AND (ss_sold_date_sk#6 <= 2426707)))) OR (((((ss_sold_date_sk#6 >= 2427043) AND (ss_sold_date_sk#6 <= 2427073)) OR ((ss_sold_date_sk#6 >= 2427408) AND (ss_sold_date_sk#6 <= 2427438))) OR ((ss_sold_date_sk#6 >= 2427773) AND (ss_sold_date_sk#6 <= 2427803))) OR ((((ss_sold_date_sk#6 >= 2428138) AND (ss_sold_date_sk#6 <= 2428168)) OR ((ss_sold_date_sk#6 >= 2428504) AND (ss_sold_date_sk#6 <= 2428534))) OR ((ss_sold_date_sk#6 >= 2428869) AND (ss_sold_date_sk#6 <= 2428899))))) OR ((((((ss_sold_date_sk#6 >= 2429234) AND (ss_sold_date_sk#6 <= 2429264)) OR ((ss_sold_date_sk#6 >= 2429599) AND (ss_sold_date_sk#6 <= 2429629))) OR ((ss_sold_date_sk#6 >= 2429965) AND (ss_sold_date_sk#6 <= 2429995))) OR ((((ss_sold_date_sk#6 >= 2430330) AND (ss_sold_date_sk#6 <= 2430360)) OR ((ss_sold_date_sk#6 >= 2430695) AND (ss_sold_date_sk#6 <= 2430725))) OR ((ss_sold_date_sk#6 >= 2431060) AND (ss_sold_date_sk#6 <= 2431090)))) OR (((((ss_sold_date_sk#6 >= 2431426) AND (ss_sold_date_sk#6 <= 2431456)) OR ((ss_sold_date_sk#6 >= 2431791) AND (ss_sold_date_sk#6 <= 2431821))) OR ((ss_sold_date_sk#6 >= 2432156) AND (ss_sold_date_sk#6 <= 2432186))) OR ((((ss_sold_date_sk#6 >= 2432521) AND (ss_sold_date_sk#6 <= 2432551)) OR ((ss_sold_date_sk#6 >= 2432887) AND (ss_sold_date_sk#6 <= 2432917))) OR ((ss_sold_date_sk#6 >= 2433252) AND (ss_sold_date_sk#6 <= 2433282))))))) OR ((((((((ss_sold_date_sk#6 >= 2433617) AND (ss_sold_date_sk#6 <= 2433647)) OR ((ss_sold_date_sk#6 >= 2433982) AND (ss_sold_date_sk#6 <= 2434012))) OR (((ss_sold_date_sk#6 >= 2434348) AND (ss_sold_date_sk#6 <= 2434378)) OR ((ss_sold_date_sk#6 >= 2434713) AND (ss_sold_date_sk#6 <= 2434743)))) OR ((((ss_sold_date_sk#6 >= 2435078) AND (ss_sold_date_sk#6 <= 2435108)) OR ((ss_sold_date_sk#6 >= 2435443) AND (ss_sold_date_sk#6 <= 2435473))) OR ((ss_sold_date_sk#6 >= 2435809) AND (ss_sold_date_sk#6 <= 2435839)))) OR (((((ss_sold_date_sk#6 >= 2436174) AND (ss_sold_date_sk#6 <= 2436204)) OR ((ss_sold_date_sk#6 >= 2436539) AND (ss_sold_date_sk#6 <= 2436569))) OR ((ss_sold_date_sk#6 >= 2436904) AND (ss_sold_date_sk#6 <= 2436934))) OR ((((ss_sold_date_sk#6 >= 2437270) AND (ss_sold_date_sk#6 <= 2437300)) OR ((ss_sold_date_sk#6 >= 2437635) AND (ss_sold_date_sk#6 <= 2437665))) OR ((ss_sold_date_sk#6 >= 2438000) AND (ss_sold_date_sk#6 <= 2438030))))) OR ((((((ss_sold_date_sk#6 >= 2438365) AND (ss_sold_date_sk#6 <= 2438395)) OR ((ss_sold_date_sk#6 >= 2438731) AND (ss_sold_date_sk#6 <= 2438761))) OR ((ss_sold_date_sk#6 >= 2439096) AND (ss_sold_date_sk#6 <= 2439126))) OR ((((ss_sold_date_sk#6 >= 2439461) AND (ss_sold_date_sk#6 <= 2439491)) OR ((ss_sold_date_sk#6 >= 2439826) AND (ss_sold_date_sk#6 <= 2439856))) OR ((ss_sold_date_sk#6 >= 2440192) AND (ss_sold_date_sk#6 <= 2440222)))) OR (((((ss_sold_date_sk#6 >= 2440557) AND (ss_sold_date_sk#6 <= 2440587)) OR ((ss_sold_date_sk#6 >= 2440922) AND (ss_sold_date_sk#6 <= 2440952))) OR ((ss_sold_date_sk#6 >= 2441287) AND (ss_sold_date_sk#6 <= 2441317))) OR ((((ss_sold_date_sk#6 >= 2441653) AND (ss_sold_date_sk#6 <= 2441683)) OR ((ss_sold_date_sk#6 >= 2442018) AND (ss_sold_date_sk#6 <= 2442048))) OR ((ss_sold_date_sk#6 >= 2442383) AND (ss_sold_date_sk#6 <= 2442413)))))) OR (((((((ss_sold_date_sk#6 >= 2442748) AND (ss_sold_date_sk#6 <= 2442778)) OR ((ss_sold_date_sk#6 >= 2443114) AND (ss_sold_date_sk#6 <= 2443144))) OR (((ss_sold_date_sk#6 >= 2443479) AND (ss_sold_date_sk#6 <= 2443509)) OR ((ss_sold_date_sk#6 >= 2443844) AND (ss_sold_date_sk#6 <= 2443874)))) OR ((((ss_sold_date_sk#6 >= 2444209) AND (ss_sold_date_sk#6 <= 2444239)) OR ((ss_sold_date_sk#6 >= 2444575) AND (ss_sold_date_sk#6 <= 2444605))) OR ((ss_sold_date_sk#6 >= 2444940) AND (ss_sold_date_sk#6 <= 2444970)))) OR (((((ss_sold_date_sk#6 >= 2445305) AND (ss_sold_date_sk#6 <= 2445335)) OR ((ss_sold_date_sk#6 >= 2445670) AND (ss_sold_date_sk#6 <= 2445700))) OR ((ss_sold_date_sk#6 >= 2446036) AND (ss_sold_date_sk#6 <= 2446066))) OR ((((ss_sold_date_sk#6 >= 2446401) AND (ss_sold_date_sk#6 <= 2446431)) OR ((ss_sold_date_sk#6 >= 2446766) AND (ss_sold_date_sk#6 <= 2446796))) OR ((ss_sold_date_sk#6 >= 2447131) AND (ss_sold_date_sk#6 <= 2447161))))) OR ((((((ss_sold_date_sk#6 >= 2447497) AND (ss_sold_date_sk#6 <= 2447527)) OR ((ss_sold_date_sk#6 >= 2447862) AND (ss_sold_date_sk#6 <= 2447892))) OR ((ss_sold_date_sk#6 >= 2448227) AND (ss_sold_date_sk#6 <= 2448257))) OR ((((ss_sold_date_sk#6 >= 2448592) AND (ss_sold_date_sk#6 <= 2448622)) OR ((ss_sold_date_sk#6 >= 2448958) AND (ss_sold_date_sk#6 <= 2448988))) OR ((ss_sold_date_sk#6 >= 2449323) AND (ss_sold_date_sk#6 <= 2449353)))) OR (((((ss_sold_date_sk#6 >= 2449688) AND (ss_sold_date_sk#6 <= 2449718)) OR ((ss_sold_date_sk#6 >= 2450053) AND (ss_sold_date_sk#6 <= 2450083))) OR ((ss_sold_date_sk#6 >= 2450419) AND (ss_sold_date_sk#6 <= 2450449))) OR ((((ss_sold_date_sk#6 >= 2450784) AND (ss_sold_date_sk#6 <= 2450814)) OR ((ss_sold_date_sk#6 >= 2451149) AND (ss_sold_date_sk#6 <= 2451179))) OR ((ss_sold_date_sk#6 >= 2451514) AND (ss_sold_date_sk#6 <= 2451544)))))))) OR (((((((((ss_sold_date_sk#6 >= 2451880) AND (ss_sold_date_sk#6 <= 2451910)) OR ((ss_sold_date_sk#6 >= 2452245) AND (ss_sold_date_sk#6 <= 2452275))) OR (((ss_sold_date_sk#6 >= 2452610) AND (ss_sold_date_sk#6 <= 2452640)) OR ((ss_sold_date_sk#6 >= 2452975) AND (ss_sold_date_sk#6 <= 2453005)))) OR ((((ss_sold_date_sk#6 >= 2453341) AND (ss_sold_date_sk#6 <= 2453371)) OR ((ss_sold_date_sk#6 >= 2453706) AND (ss_sold_date_sk#6 <= 2453736))) OR ((ss_sold_date_sk#6 >= 2454071) AND (ss_sold_date_sk#6 <= 2454101)))) OR (((((ss_sold_date_sk#6 >= 2454436) AND (ss_sold_date_sk#6 <= 2454466)) OR ((ss_sold_date_sk#6 >= 2454802) AND (ss_sold_date_sk#6 <= 2454832))) OR ((ss_sold_date_sk#6 >= 2455167) AND (ss_sold_date_sk#6 <= 2455197))) OR ((((ss_sold_date_sk#6 >= 2455532) AND (ss_sold_date_sk#6 <= 2455562)) OR ((ss_sold_date_sk#6 >= 2455897) AND (ss_sold_date_sk#6 <= 2455927))) OR ((ss_sold_date_sk#6 >= 2456263) AND (ss_sold_date_sk#6 <= 2456293))))) OR ((((((ss_sold_date_sk#6 >= 2456628) AND (ss_sold_date_sk#6 <= 2456658)) OR ((ss_sold_date_sk#6 >= 2456993) AND (ss_sold_date_sk#6 <= 2457023))) OR ((ss_sold_date_sk#6 >= 2457358) AND (ss_sold_date_sk#6 <= 2457388))) OR ((((ss_sold_date_sk#6 >= 2457724) AND (ss_sold_date_sk#6 <= 2457754)) OR ((ss_sold_date_sk#6 >= 2458089) AND (ss_sold_date_sk#6 <= 2458119))) OR ((ss_sold_date_sk#6 >= 2458454) AND (ss_sold_date_sk#6 <= 2458484)))) OR (((((ss_sold_date_sk#6 >= 2458819) AND (ss_sold_date_sk#6 <= 2458849)) OR ((ss_sold_date_sk#6 >= 2459185) AND (ss_sold_date_sk#6 <= 2459215))) OR ((ss_sold_date_sk#6 >= 2459550) AND (ss_sold_date_sk#6 <= 2459580))) OR ((((ss_sold_date_sk#6 >= 2459915) AND (ss_sold_date_sk#6 <= 2459945)) OR ((ss_sold_date_sk#6 >= 2460280) AND (ss_sold_date_sk#6 <= 2460310))) OR ((ss_sold_date_sk#6 >= 2460646) AND (ss_sold_date_sk#6 <= 2460676)))))) OR (((((((ss_sold_date_sk#6 >= 2461011) AND (ss_sold_date_sk#6 <= 2461041)) OR ((ss_sold_date_sk#6 >= 2461376) AND (ss_sold_date_sk#6 <= 2461406))) OR (((ss_sold_date_sk#6 >= 2461741) AND (ss_sold_date_sk#6 <= 2461771)) OR ((ss_sold_date_sk#6 >= 2462107) AND (ss_sold_date_sk#6 <= 2462137)))) OR ((((ss_sold_date_sk#6 >= 2462472) AND (ss_sold_date_sk#6 <= 2462502)) OR ((ss_sold_date_sk#6 >= 2462837) AND (ss_sold_date_sk#6 <= 2462867))) OR ((ss_sold_date_sk#6 >= 2463202) AND (ss_sold_date_sk#6 <= 2463232)))) OR (((((ss_sold_date_sk#6 >= 2463568) AND (ss_sold_date_sk#6 <= 2463598)) OR ((ss_sold_date_sk#6 >= 2463933) AND (ss_sold_date_sk#6 <= 2463963))) OR ((ss_sold_date_sk#6 >= 2464298) AND (ss_sold_date_sk#6 <= 2464328))) OR ((((ss_sold_date_sk#6 >= 2464663) AND (ss_sold_date_sk#6 <= 2464693)) OR ((ss_sold_date_sk#6 >= 2465029) AND (ss_sold_date_sk#6 <= 2465059))) OR ((ss_sold_date_sk#6 >= 2465394) AND (ss_sold_date_sk#6 <= 2465424))))) OR ((((((ss_sold_date_sk#6 >= 2465759) AND (ss_sold_date_sk#6 <= 2465789)) OR ((ss_sold_date_sk#6 >= 2466124) AND (ss_sold_date_sk#6 <= 2466154))) OR ((ss_sold_date_sk#6 >= 2466490) AND (ss_sold_date_sk#6 <= 2466520))) OR ((((ss_sold_date_sk#6 >= 2466855) AND (ss_sold_date_sk#6 <= 2466885)) OR ((ss_sold_date_sk#6 >= 2467220) AND (ss_sold_date_sk#6 <= 2467250))) OR ((ss_sold_date_sk#6 >= 2467585) AND (ss_sold_date_sk#6 <= 2467615)))) OR (((((ss_sold_date_sk#6 >= 2467951) AND (ss_sold_date_sk#6 <= 2467981)) OR ((ss_sold_date_sk#6 >= 2468316) AND (ss_sold_date_sk#6 <= 2468346))) OR ((ss_sold_date_sk#6 >= 2468681) AND (ss_sold_date_sk#6 <= 2468711))) OR ((((ss_sold_date_sk#6 >= 2469046) AND (ss_sold_date_sk#6 <= 2469076)) OR ((ss_sold_date_sk#6 >= 2469412) AND (ss_sold_date_sk#6 <= 2469442))) OR ((ss_sold_date_sk#6 >= 2469777) AND (ss_sold_date_sk#6 <= 2469807))))))) OR ((((((((ss_sold_date_sk#6 >= 2470142) AND (ss_sold_date_sk#6 <= 2470172)) OR ((ss_sold_date_sk#6 >= 2470507) AND (ss_sold_date_sk#6 <= 2470537))) OR (((ss_sold_date_sk#6 >= 2470873) AND (ss_sold_date_sk#6 <= 2470903)) OR ((ss_sold_date_sk#6 >= 2471238) AND (ss_sold_date_sk#6 <= 2471268)))) OR ((((ss_sold_date_sk#6 >= 2471603) AND (ss_sold_date_sk#6 <= 2471633)) OR ((ss_sold_date_sk#6 >= 2471968) AND (ss_sold_date_sk#6 <= 2471998))) OR ((ss_sold_date_sk#6 >= 2472334) AND (ss_sold_date_sk#6 <= 2472364)))) OR (((((ss_sold_date_sk#6 >= 2472699) AND (ss_sold_date_sk#6 <= 2472729)) OR ((ss_sold_date_sk#6 >= 2473064) AND (ss_sold_date_sk#6 <= 2473094))) OR ((ss_sold_date_sk#6 >= 2473429) AND (ss_sold_date_sk#6 <= 2473459))) OR ((((ss_sold_date_sk#6 >= 2473795) AND (ss_sold_date_sk#6 <= 2473825)) OR ((ss_sold_date_sk#6 >= 2474160) AND (ss_sold_date_sk#6 <= 2474190))) OR ((ss_sold_date_sk#6 >= 2474525) AND (ss_sold_date_sk#6 <= 2474555))))) OR ((((((ss_sold_date_sk#6 >= 2474890) AND (ss_sold_date_sk#6 <= 2474920)) OR ((ss_sold_date_sk#6 >= 2475256) AND (ss_sold_date_sk#6 <= 2475286))) OR ((ss_sold_date_sk#6 >= 2475621) AND (ss_sold_date_sk#6 <= 2475651))) OR ((((ss_sold_date_sk#6 >= 2475986) AND (ss_sold_date_sk#6 <= 2476016)) OR ((ss_sold_date_sk#6 >= 2476351) AND (ss_sold_date_sk#6 <= 2476381))) OR ((ss_sold_date_sk#6 >= 2476717) AND (ss_sold_date_sk#6 <= 2476747)))) OR (((((ss_sold_date_sk#6 >= 2477082) AND (ss_sold_date_sk#6 <= 2477112)) OR ((ss_sold_date_sk#6 >= 2477447) AND (ss_sold_date_sk#6 <= 2477477))) OR ((ss_sold_date_sk#6 >= 2477812) AND (ss_sold_date_sk#6 <= 2477842))) OR ((((ss_sold_date_sk#6 >= 2478178) AND (ss_sold_date_sk#6 <= 2478208)) OR ((ss_sold_date_sk#6 >= 2478543) AND (ss_sold_date_sk#6 <= 2478573))) OR ((ss_sold_date_sk#6 >= 2478908) AND (ss_sold_date_sk#6 <= 2478938)))))) OR (((((((ss_sold_date_sk#6 >= 2479273) AND (ss_sold_date_sk#6 <= 2479303)) OR ((ss_sold_date_sk#6 >= 2479639) AND (ss_sold_date_sk#6 <= 2479669))) OR (((ss_sold_date_sk#6 >= 2480004) AND (ss_sold_date_sk#6 <= 2480034)) OR ((ss_sold_date_sk#6 >= 2480369) AND (ss_sold_date_sk#6 <= 2480399)))) OR ((((ss_sold_date_sk#6 >= 2480734) AND (ss_sold_date_sk#6 <= 2480764)) OR ((ss_sold_date_sk#6 >= 2481100) AND (ss_sold_date_sk#6 <= 2481130))) OR ((ss_sold_date_sk#6 >= 2481465) AND (ss_sold_date_sk#6 <= 2481495)))) OR (((((ss_sold_date_sk#6 >= 2481830) AND (ss_sold_date_sk#6 <= 2481860)) OR ((ss_sold_date_sk#6 >= 2482195) AND (ss_sold_date_sk#6 <= 2482225))) OR ((ss_sold_date_sk#6 >= 2482561) AND (ss_sold_date_sk#6 <= 2482591))) OR ((((ss_sold_date_sk#6 >= 2482926) AND (ss_sold_date_sk#6 <= 2482956)) OR ((ss_sold_date_sk#6 >= 2483291) AND (ss_sold_date_sk#6 <= 2483321))) OR ((ss_sold_date_sk#6 >= 2483656) AND (ss_sold_date_sk#6 <= 2483686))))) OR ((((((ss_sold_date_sk#6 >= 2484022) AND (ss_sold_date_sk#6 <= 2484052)) OR ((ss_sold_date_sk#6 >= 2484387) AND (ss_sold_date_sk#6 <= 2484417))) OR ((ss_sold_date_sk#6 >= 2484752) AND (ss_sold_date_sk#6 <= 2484782))) OR ((((ss_sold_date_sk#6 >= 2485117) AND (ss_sold_date_sk#6 <= 2485147)) OR ((ss_sold_date_sk#6 >= 2485483) AND (ss_sold_date_sk#6 <= 2485513))) OR ((ss_sold_date_sk#6 >= 2485848) AND (ss_sold_date_sk#6 <= 2485878)))) OR (((((ss_sold_date_sk#6 >= 2486213) AND (ss_sold_date_sk#6 <= 2486243)) OR ((ss_sold_date_sk#6 >= 2486578) AND (ss_sold_date_sk#6 <= 2486608))) OR ((ss_sold_date_sk#6 >= 2486944) AND (ss_sold_date_sk#6 <= 2486974))) OR ((((ss_sold_date_sk#6 >= 2487309) AND (ss_sold_date_sk#6 <= 2487339)) OR ((ss_sold_date_sk#6 >= 2487674) AND (ss_sold_date_sk#6 <= 2487704))) OR ((ss_sold_date_sk#6 >= 2488039) AND (ss_sold_date_sk#6 <= 2488069))))))))), isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_net_profit:decimal(7,2)>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_item_sk#4, ss_net_profit#5, ss_sold_date_sk#6]
+
+(7) Filter [codegen id : 1]
+Input [3]: [ss_item_sk#4, ss_net_profit#5, ss_sold_date_sk#6]
+Condition : isnotnull(ss_item_sk#4)
+
+(8) BroadcastExchange
+Input [3]: [ss_item_sk#4, ss_net_profit#5, ss_sold_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [d_date_sk#1]
+Right keys [1]: [ss_sold_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [3]: [d_year#2, ss_item_sk#4, ss_net_profit#5]
+Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_net_profit#5, ss_sold_date_sk#6]
+
+(11) Scan parquet default.item
+Output [4]: [i_item_sk#8, i_brand_id#9, i_brand#10, i_manufact_id#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,436), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_brand:string,i_manufact_id:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [4]: [i_item_sk#8, i_brand_id#9, i_brand#10, i_manufact_id#11]
+
+(13) Filter [codegen id : 2]
+Input [4]: [i_item_sk#8, i_brand_id#9, i_brand#10, i_manufact_id#11]
+Condition : ((isnotnull(i_manufact_id#11) AND (i_manufact_id#11 = 436)) AND isnotnull(i_item_sk#8))
+
+(14) Project [codegen id : 2]
+Output [3]: [i_item_sk#8, i_brand_id#9, i_brand#10]
+Input [4]: [i_item_sk#8, i_brand_id#9, i_brand#10, i_manufact_id#11]
+
+(15) BroadcastExchange
+Input [3]: [i_item_sk#8, i_brand_id#9, i_brand#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(16) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_item_sk#4]
+Right keys [1]: [i_item_sk#8]
+Join condition: None
+
+(17) Project [codegen id : 3]
+Output [4]: [d_year#2, ss_net_profit#5, i_brand_id#9, i_brand#10]
+Input [6]: [d_year#2, ss_item_sk#4, ss_net_profit#5, i_item_sk#8, i_brand_id#9, i_brand#10]
+
+(18) HashAggregate [codegen id : 3]
+Input [4]: [d_year#2, ss_net_profit#5, i_brand_id#9, i_brand#10]
+Keys [3]: [d_year#2, i_brand#10, i_brand_id#9]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#5))]
+Aggregate Attributes [1]: [sum#13]
+Results [4]: [d_year#2, i_brand#10, i_brand_id#9, sum#14]
+
+(19) Exchange
+Input [4]: [d_year#2, i_brand#10, i_brand_id#9, sum#14]
+Arguments: hashpartitioning(d_year#2, i_brand#10, i_brand_id#9, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(20) HashAggregate [codegen id : 4]
+Input [4]: [d_year#2, i_brand#10, i_brand_id#9, sum#14]
+Keys [3]: [d_year#2, i_brand#10, i_brand_id#9]
+Functions [1]: [sum(UnscaledValue(ss_net_profit#5))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#5))#16]
+Results [4]: [d_year#2, i_brand_id#9 AS brand_id#17, i_brand#10 AS brand#18, MakeDecimal(sum(UnscaledValue(ss_net_profit#5))#16,17,2) AS sum_agg#19]
+
+(21) TakeOrderedAndProject
+Input [4]: [d_year#2, brand_id#17, brand#18, sum_agg#19]
+Arguments: 100, [d_year#2 ASC NULLS FIRST, sum_agg#19 DESC NULLS LAST, brand_id#17 ASC NULLS FIRST], [d_year#2, brand_id#17, brand#18, sum_agg#19]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q3/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q3/simplified.txt
new file mode 100644
index 0000000000000..ef31be37f934e
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q3/simplified.txt
@@ -0,0 +1,31 @@
+TakeOrderedAndProject [d_year,sum_agg,brand_id,brand]
+  WholeStageCodegen (4)
+    HashAggregate [d_year,i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_net_profit)),brand_id,brand,sum_agg,sum]
+      InputAdapter
+        Exchange [d_year,i_brand,i_brand_id] #1
+          WholeStageCodegen (3)
+            HashAggregate [d_year,i_brand,i_brand_id,ss_net_profit] [sum,sum]
+              Project [d_year,ss_net_profit,i_brand_id,i_brand]
+                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                  Project [d_year,ss_item_sk,ss_net_profit]
+                    BroadcastHashJoin [d_date_sk,ss_sold_date_sk]
+                      Project [d_date_sk,d_year]
+                        Filter [d_moy,d_date_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                      InputAdapter
+                        BroadcastExchange #2
+                          WholeStageCodegen (1)
+                            Filter [ss_item_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.store_sales [ss_item_sk,ss_net_profit,ss_sold_date_sk]
+                  InputAdapter
+                    BroadcastExchange #3
+                      WholeStageCodegen (2)
+                        Project [i_item_sk,i_brand_id,i_brand]
+                          Filter [i_manufact_id,i_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q34.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q34.sf100/explain.txt
new file mode 100644
index 0000000000000..ebc6009080bfa
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q34.sf100/explain.txt
@@ -0,0 +1,229 @@
+== Physical Plan ==
+* Sort (39)
++- Exchange (38)
+   +- * Project (37)
+      +- * SortMergeJoin Inner (36)
+         :- * Sort (30)
+         :  +- Exchange (29)
+         :     +- * Filter (28)
+         :        +- * HashAggregate (27)
+         :           +- Exchange (26)
+         :              +- * HashAggregate (25)
+         :                 +- * Project (24)
+         :                    +- * BroadcastHashJoin Inner BuildRight (23)
+         :                       :- * Project (17)
+         :                       :  +- * BroadcastHashJoin Inner BuildRight (16)
+         :                       :     :- * Project (10)
+         :                       :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+         :                       :     :     :- * Filter (3)
+         :                       :     :     :  +- * ColumnarToRow (2)
+         :                       :     :     :     +- Scan parquet default.store_sales (1)
+         :                       :     :     +- BroadcastExchange (8)
+         :                       :     :        +- * Project (7)
+         :                       :     :           +- * Filter (6)
+         :                       :     :              +- * ColumnarToRow (5)
+         :                       :     :                 +- Scan parquet default.date_dim (4)
+         :                       :     +- BroadcastExchange (15)
+         :                       :        +- * Project (14)
+         :                       :           +- * Filter (13)
+         :                       :              +- * ColumnarToRow (12)
+         :                       :                 +- Scan parquet default.store (11)
+         :                       +- BroadcastExchange (22)
+         :                          +- * Project (21)
+         :                             +- * Filter (20)
+         :                                +- * ColumnarToRow (19)
+         :                                   +- Scan parquet default.household_demographics (18)
+         +- * Sort (35)
+            +- Exchange (34)
+               +- * Filter (33)
+                  +- * ColumnarToRow (32)
+                     +- Scan parquet default.customer (31)
+
+
+(1) Scan parquet default.store_sales
+Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#5), (ss_sold_date_sk#5 >= 2450816), (ss_sold_date_sk#5 <= 2451910), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_hdemo_sk:int,ss_store_sk:int,ss_ticket_number:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+
+(3) Filter [codegen id : 4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#7, d_year#8, d_dom#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1998,1999,2000]), GreaterThanOrEqual(d_date_sk,2450816), LessThanOrEqual(d_date_sk,2451910), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_dom:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+Condition : (((((((d_dom#9 >= 1) AND (d_dom#9 <= 3)) OR ((d_dom#9 >= 25) AND (d_dom#9 <= 28))) AND d_year#8 IN (1998,1999,2000)) AND (d_date_sk#7 >= 2450816)) AND (d_date_sk#7 <= 2451910)) AND isnotnull(d_date_sk#7))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#7]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#5]
+Right keys [1]: [d_date_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4]
+Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#7]
+
+(11) Scan parquet default.store
+Output [2]: [s_store_sk#11, s_county#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [In(s_county, [Saginaw County,Sumner County,Appanoose County,Daviess County,Fairfield County,Raleigh County,Ziebach County,Williamson County]), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_county:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [s_store_sk#11, s_county#12]
+
+(13) Filter [codegen id : 2]
+Input [2]: [s_store_sk#11, s_county#12]
+Condition : (s_county#12 IN (Saginaw County,Sumner County,Appanoose County,Daviess County,Fairfield County,Raleigh County,Ziebach County,Williamson County) AND isnotnull(s_store_sk#11))
+
+(14) Project [codegen id : 2]
+Output [1]: [s_store_sk#11]
+Input [2]: [s_store_sk#11, s_county#12]
+
+(15) BroadcastExchange
+Input [1]: [s_store_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#11]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#11]
+
+(18) Scan parquet default.household_demographics
+Output [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_vehicle_count), IsNotNull(hd_dep_count), Or(EqualTo(hd_buy_potential,>10000         ),EqualTo(hd_buy_potential,Unknown        )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_buy_potential:string,hd_dep_count:int,hd_vehicle_count:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+
+(20) Filter [codegen id : 3]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+Condition : (((((isnotnull(hd_vehicle_count#17) AND isnotnull(hd_dep_count#16)) AND ((hd_buy_potential#15 = >10000         ) OR (hd_buy_potential#15 = Unknown        ))) AND (hd_vehicle_count#17 > 0)) AND ((cast(hd_dep_count#16 as double) / cast(hd_vehicle_count#17 as double)) > 1.2)) AND isnotnull(hd_demo_sk#14))
+
+(21) Project [codegen id : 3]
+Output [1]: [hd_demo_sk#14]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+
+(22) BroadcastExchange
+Input [1]: [hd_demo_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(23) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#14]
+Join condition: None
+
+(24) Project [codegen id : 4]
+Output [2]: [ss_customer_sk#1, ss_ticket_number#4]
+Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#14]
+
+(25) HashAggregate [codegen id : 4]
+Input [2]: [ss_customer_sk#1, ss_ticket_number#4]
+Keys [2]: [ss_ticket_number#4, ss_customer_sk#1]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#19]
+Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+
+(26) Exchange
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(27) HashAggregate [codegen id : 5]
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+Keys [2]: [ss_ticket_number#4, ss_customer_sk#1]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#22]
+Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count(1)#22 AS cnt#23]
+
+(28) Filter [codegen id : 5]
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23]
+Condition : ((cnt#23 >= 15) AND (cnt#23 <= 20))
+
+(29) Exchange
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23]
+Arguments: hashpartitioning(ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(30) Sort [codegen id : 6]
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23]
+Arguments: [ss_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(31) Scan parquet default.customer
+Output [5]: [c_customer_sk#25, c_salutation#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_salutation:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string>
+
+(32) ColumnarToRow [codegen id : 7]
+Input [5]: [c_customer_sk#25, c_salutation#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29]
+
+(33) Filter [codegen id : 7]
+Input [5]: [c_customer_sk#25, c_salutation#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29]
+Condition : isnotnull(c_customer_sk#25)
+
+(34) Exchange
+Input [5]: [c_customer_sk#25, c_salutation#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29]
+Arguments: hashpartitioning(c_customer_sk#25, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(35) Sort [codegen id : 8]
+Input [5]: [c_customer_sk#25, c_salutation#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29]
+Arguments: [c_customer_sk#25 ASC NULLS FIRST], false, 0
+
+(36) SortMergeJoin [codegen id : 9]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#25]
+Join condition: None
+
+(37) Project [codegen id : 9]
+Output [6]: [c_last_name#28, c_first_name#27, c_salutation#26, c_preferred_cust_flag#29, ss_ticket_number#4, cnt#23]
+Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23, c_customer_sk#25, c_salutation#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29]
+
+(38) Exchange
+Input [6]: [c_last_name#28, c_first_name#27, c_salutation#26, c_preferred_cust_flag#29, ss_ticket_number#4, cnt#23]
+Arguments: rangepartitioning(c_last_name#28 ASC NULLS FIRST, c_first_name#27 ASC NULLS FIRST, c_salutation#26 ASC NULLS FIRST, c_preferred_cust_flag#29 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(39) Sort [codegen id : 10]
+Input [6]: [c_last_name#28, c_first_name#27, c_salutation#26, c_preferred_cust_flag#29, ss_ticket_number#4, cnt#23]
+Arguments: [c_last_name#28 ASC NULLS FIRST, c_first_name#27 ASC NULLS FIRST, c_salutation#26 ASC NULLS FIRST, c_preferred_cust_flag#29 DESC NULLS LAST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (40)
+
+
+(40) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#7]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q34.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q34.sf100/simplified.txt
new file mode 100644
index 0000000000000..24c5c1c256c26
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q34.sf100/simplified.txt
@@ -0,0 +1,65 @@
+WholeStageCodegen (10)
+  Sort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag]
+    InputAdapter
+      Exchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag] #1
+        WholeStageCodegen (9)
+          Project [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt]
+            SortMergeJoin [ss_customer_sk,c_customer_sk]
+              InputAdapter
+                WholeStageCodegen (6)
+                  Sort [ss_customer_sk]
+                    InputAdapter
+                      Exchange [ss_customer_sk] #2
+                        WholeStageCodegen (5)
+                          Filter [cnt]
+                            HashAggregate [ss_ticket_number,ss_customer_sk,count] [count(1),cnt,count]
+                              InputAdapter
+                                Exchange [ss_ticket_number,ss_customer_sk] #3
+                                  WholeStageCodegen (4)
+                                    HashAggregate [ss_ticket_number,ss_customer_sk] [count,count]
+                                      Project [ss_customer_sk,ss_ticket_number]
+                                        BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                          Project [ss_customer_sk,ss_hdemo_sk,ss_ticket_number]
+                                            BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                              Project [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number]
+                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                  Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk]
+                                                          SubqueryBroadcast [d_date_sk] #1
+                                                            ReusedExchange [d_date_sk] #4
+                                                  InputAdapter
+                                                    BroadcastExchange #4
+                                                      WholeStageCodegen (1)
+                                                        Project [d_date_sk]
+                                                          Filter [d_dom,d_year,d_date_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.date_dim [d_date_sk,d_year,d_dom]
+                                              InputAdapter
+                                                BroadcastExchange #5
+                                                  WholeStageCodegen (2)
+                                                    Project [s_store_sk]
+                                                      Filter [s_county,s_store_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.store [s_store_sk,s_county]
+                                          InputAdapter
+                                            BroadcastExchange #6
+                                              WholeStageCodegen (3)
+                                                Project [hd_demo_sk]
+                                                  Filter [hd_vehicle_count,hd_dep_count,hd_buy_potential,hd_demo_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count]
+              InputAdapter
+                WholeStageCodegen (8)
+                  Sort [c_customer_sk]
+                    InputAdapter
+                      Exchange [c_customer_sk] #7
+                        WholeStageCodegen (7)
+                          Filter [c_customer_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q34/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q34/explain.txt
new file mode 100644
index 0000000000000..cd556c0bccb46
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q34/explain.txt
@@ -0,0 +1,214 @@
+== Physical Plan ==
+* Sort (36)
++- Exchange (35)
+   +- * Project (34)
+      +- * BroadcastHashJoin Inner BuildRight (33)
+         :- * Filter (28)
+         :  +- * HashAggregate (27)
+         :     +- Exchange (26)
+         :        +- * HashAggregate (25)
+         :           +- * Project (24)
+         :              +- * BroadcastHashJoin Inner BuildRight (23)
+         :                 :- * Project (17)
+         :                 :  +- * BroadcastHashJoin Inner BuildRight (16)
+         :                 :     :- * Project (10)
+         :                 :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+         :                 :     :     :- * Filter (3)
+         :                 :     :     :  +- * ColumnarToRow (2)
+         :                 :     :     :     +- Scan parquet default.store_sales (1)
+         :                 :     :     +- BroadcastExchange (8)
+         :                 :     :        +- * Project (7)
+         :                 :     :           +- * Filter (6)
+         :                 :     :              +- * ColumnarToRow (5)
+         :                 :     :                 +- Scan parquet default.date_dim (4)
+         :                 :     +- BroadcastExchange (15)
+         :                 :        +- * Project (14)
+         :                 :           +- * Filter (13)
+         :                 :              +- * ColumnarToRow (12)
+         :                 :                 +- Scan parquet default.store (11)
+         :                 +- BroadcastExchange (22)
+         :                    +- * Project (21)
+         :                       +- * Filter (20)
+         :                          +- * ColumnarToRow (19)
+         :                             +- Scan parquet default.household_demographics (18)
+         +- BroadcastExchange (32)
+            +- * Filter (31)
+               +- * ColumnarToRow (30)
+                  +- Scan parquet default.customer (29)
+
+
+(1) Scan parquet default.store_sales
+Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#5), (ss_sold_date_sk#5 >= 2450816), (ss_sold_date_sk#5 <= 2451910), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_hdemo_sk:int,ss_store_sk:int,ss_ticket_number:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+
+(3) Filter [codegen id : 4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#7, d_year#8, d_dom#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1998,1999,2000]), GreaterThanOrEqual(d_date_sk,2450816), LessThanOrEqual(d_date_sk,2451910), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_dom:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+Condition : (((((((d_dom#9 >= 1) AND (d_dom#9 <= 3)) OR ((d_dom#9 >= 25) AND (d_dom#9 <= 28))) AND d_year#8 IN (1998,1999,2000)) AND (d_date_sk#7 >= 2450816)) AND (d_date_sk#7 <= 2451910)) AND isnotnull(d_date_sk#7))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#7]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#5]
+Right keys [1]: [d_date_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4]
+Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#7]
+
+(11) Scan parquet default.store
+Output [2]: [s_store_sk#11, s_county#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [In(s_county, [Saginaw County,Sumner County,Appanoose County,Daviess County,Fairfield County,Raleigh County,Ziebach County,Williamson County]), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_county:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [s_store_sk#11, s_county#12]
+
+(13) Filter [codegen id : 2]
+Input [2]: [s_store_sk#11, s_county#12]
+Condition : (s_county#12 IN (Saginaw County,Sumner County,Appanoose County,Daviess County,Fairfield County,Raleigh County,Ziebach County,Williamson County) AND isnotnull(s_store_sk#11))
+
+(14) Project [codegen id : 2]
+Output [1]: [s_store_sk#11]
+Input [2]: [s_store_sk#11, s_county#12]
+
+(15) BroadcastExchange
+Input [1]: [s_store_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#11]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#11]
+
+(18) Scan parquet default.household_demographics
+Output [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_vehicle_count), IsNotNull(hd_dep_count), Or(EqualTo(hd_buy_potential,>10000         ),EqualTo(hd_buy_potential,Unknown        )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_buy_potential:string,hd_dep_count:int,hd_vehicle_count:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+
+(20) Filter [codegen id : 3]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+Condition : (((((isnotnull(hd_vehicle_count#17) AND isnotnull(hd_dep_count#16)) AND ((hd_buy_potential#15 = >10000         ) OR (hd_buy_potential#15 = Unknown        ))) AND (hd_vehicle_count#17 > 0)) AND ((cast(hd_dep_count#16 as double) / cast(hd_vehicle_count#17 as double)) > 1.2)) AND isnotnull(hd_demo_sk#14))
+
+(21) Project [codegen id : 3]
+Output [1]: [hd_demo_sk#14]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+
+(22) BroadcastExchange
+Input [1]: [hd_demo_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(23) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#14]
+Join condition: None
+
+(24) Project [codegen id : 4]
+Output [2]: [ss_customer_sk#1, ss_ticket_number#4]
+Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#14]
+
+(25) HashAggregate [codegen id : 4]
+Input [2]: [ss_customer_sk#1, ss_ticket_number#4]
+Keys [2]: [ss_ticket_number#4, ss_customer_sk#1]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#19]
+Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+
+(26) Exchange
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(27) HashAggregate [codegen id : 6]
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+Keys [2]: [ss_ticket_number#4, ss_customer_sk#1]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#22]
+Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count(1)#22 AS cnt#23]
+
+(28) Filter [codegen id : 6]
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23]
+Condition : ((cnt#23 >= 15) AND (cnt#23 <= 20))
+
+(29) Scan parquet default.customer
+Output [5]: [c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_salutation:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string>
+
+(30) ColumnarToRow [codegen id : 5]
+Input [5]: [c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+
+(31) Filter [codegen id : 5]
+Input [5]: [c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+Condition : isnotnull(c_customer_sk#24)
+
+(32) BroadcastExchange
+Input [5]: [c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#29]
+
+(33) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#24]
+Join condition: None
+
+(34) Project [codegen id : 6]
+Output [6]: [c_last_name#27, c_first_name#26, c_salutation#25, c_preferred_cust_flag#28, ss_ticket_number#4, cnt#23]
+Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23, c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+
+(35) Exchange
+Input [6]: [c_last_name#27, c_first_name#26, c_salutation#25, c_preferred_cust_flag#28, ss_ticket_number#4, cnt#23]
+Arguments: rangepartitioning(c_last_name#27 ASC NULLS FIRST, c_first_name#26 ASC NULLS FIRST, c_salutation#25 ASC NULLS FIRST, c_preferred_cust_flag#28 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(36) Sort [codegen id : 7]
+Input [6]: [c_last_name#27, c_first_name#26, c_salutation#25, c_preferred_cust_flag#28, ss_ticket_number#4, cnt#23]
+Arguments: [c_last_name#27 ASC NULLS FIRST, c_first_name#26 ASC NULLS FIRST, c_salutation#25 ASC NULLS FIRST, c_preferred_cust_flag#28 DESC NULLS LAST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (37)
+
+
+(37) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#7]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q34/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q34/simplified.txt
new file mode 100644
index 0000000000000..e4f89ab27658b
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q34/simplified.txt
@@ -0,0 +1,56 @@
+WholeStageCodegen (7)
+  Sort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag]
+    InputAdapter
+      Exchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag] #1
+        WholeStageCodegen (6)
+          Project [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt]
+            BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+              Filter [cnt]
+                HashAggregate [ss_ticket_number,ss_customer_sk,count] [count(1),cnt,count]
+                  InputAdapter
+                    Exchange [ss_ticket_number,ss_customer_sk] #2
+                      WholeStageCodegen (4)
+                        HashAggregate [ss_ticket_number,ss_customer_sk] [count,count]
+                          Project [ss_customer_sk,ss_ticket_number]
+                            BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                              Project [ss_customer_sk,ss_hdemo_sk,ss_ticket_number]
+                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                  Project [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number]
+                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                      Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #1
+                                                ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        BroadcastExchange #3
+                                          WholeStageCodegen (1)
+                                            Project [d_date_sk]
+                                              Filter [d_dom,d_year,d_date_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_date_sk,d_year,d_dom]
+                                  InputAdapter
+                                    BroadcastExchange #4
+                                      WholeStageCodegen (2)
+                                        Project [s_store_sk]
+                                          Filter [s_county,s_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store [s_store_sk,s_county]
+                              InputAdapter
+                                BroadcastExchange #5
+                                  WholeStageCodegen (3)
+                                    Project [hd_demo_sk]
+                                      Filter [hd_vehicle_count,hd_dep_count,hd_buy_potential,hd_demo_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count]
+              InputAdapter
+                BroadcastExchange #6
+                  WholeStageCodegen (5)
+                    Filter [c_customer_sk]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q42.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q42.sf100/explain.txt
new file mode 100644
index 0000000000000..5e83e995a3766
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q42.sf100/explain.txt
@@ -0,0 +1,133 @@
+== Physical Plan ==
+TakeOrderedAndProject (21)
++- * HashAggregate (20)
+   +- Exchange (19)
+      +- * HashAggregate (18)
+         +- * Project (17)
+            +- * BroadcastHashJoin Inner BuildRight (16)
+               :- * Project (10)
+               :  +- * BroadcastHashJoin Inner BuildLeft (9)
+               :     :- BroadcastExchange (5)
+               :     :  +- * Project (4)
+               :     :     +- * Filter (3)
+               :     :        +- * ColumnarToRow (2)
+               :     :           +- Scan parquet default.date_dim (1)
+               :     +- * Filter (8)
+               :        +- * ColumnarToRow (7)
+               :           +- Scan parquet default.store_sales (6)
+               +- BroadcastExchange (15)
+                  +- * Project (14)
+                     +- * Filter (13)
+                        +- * ColumnarToRow (12)
+                           +- Scan parquet default.item (11)
+
+
+(1) Scan parquet default.date_dim
+Output [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,12), EqualTo(d_year,1998), GreaterThanOrEqual(d_date_sk,2451149), LessThanOrEqual(d_date_sk,2451179), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(3) Filter [codegen id : 1]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Condition : ((((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 12)) AND (d_year#2 = 1998)) AND (d_date_sk#1 >= 2451149)) AND (d_date_sk#1 <= 2451179)) AND isnotnull(d_date_sk#1))
+
+(4) Project [codegen id : 1]
+Output [2]: [d_date_sk#1, d_year#2]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(5) BroadcastExchange
+Input [2]: [d_date_sk#1, d_year#2]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#4]
+
+(6) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#5, ss_ext_sales_price#6, ss_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#7), (ss_sold_date_sk#7 >= 2451149), (ss_sold_date_sk#7 <= 2451179), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(7) ColumnarToRow
+Input [3]: [ss_item_sk#5, ss_ext_sales_price#6, ss_sold_date_sk#7]
+
+(8) Filter
+Input [3]: [ss_item_sk#5, ss_ext_sales_price#6, ss_sold_date_sk#7]
+Condition : isnotnull(ss_item_sk#5)
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [d_date_sk#1]
+Right keys [1]: [ss_sold_date_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [3]: [d_year#2, ss_item_sk#5, ss_ext_sales_price#6]
+Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#5, ss_ext_sales_price#6, ss_sold_date_sk#7]
+
+(11) Scan parquet default.item
+Output [4]: [i_item_sk#9, i_category_id#10, i_category#11, i_manager_id#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_category_id:int,i_category:string,i_manager_id:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [4]: [i_item_sk#9, i_category_id#10, i_category#11, i_manager_id#12]
+
+(13) Filter [codegen id : 2]
+Input [4]: [i_item_sk#9, i_category_id#10, i_category#11, i_manager_id#12]
+Condition : ((isnotnull(i_manager_id#12) AND (i_manager_id#12 = 1)) AND isnotnull(i_item_sk#9))
+
+(14) Project [codegen id : 2]
+Output [3]: [i_item_sk#9, i_category_id#10, i_category#11]
+Input [4]: [i_item_sk#9, i_category_id#10, i_category#11, i_manager_id#12]
+
+(15) BroadcastExchange
+Input [3]: [i_item_sk#9, i_category_id#10, i_category#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(16) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_item_sk#5]
+Right keys [1]: [i_item_sk#9]
+Join condition: None
+
+(17) Project [codegen id : 3]
+Output [4]: [d_year#2, ss_ext_sales_price#6, i_category_id#10, i_category#11]
+Input [6]: [d_year#2, ss_item_sk#5, ss_ext_sales_price#6, i_item_sk#9, i_category_id#10, i_category#11]
+
+(18) HashAggregate [codegen id : 3]
+Input [4]: [d_year#2, ss_ext_sales_price#6, i_category_id#10, i_category#11]
+Keys [3]: [d_year#2, i_category_id#10, i_category#11]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#6))]
+Aggregate Attributes [1]: [sum#14]
+Results [4]: [d_year#2, i_category_id#10, i_category#11, sum#15]
+
+(19) Exchange
+Input [4]: [d_year#2, i_category_id#10, i_category#11, sum#15]
+Arguments: hashpartitioning(d_year#2, i_category_id#10, i_category#11, 5), ENSURE_REQUIREMENTS, [id=#16]
+
+(20) HashAggregate [codegen id : 4]
+Input [4]: [d_year#2, i_category_id#10, i_category#11, sum#15]
+Keys [3]: [d_year#2, i_category_id#10, i_category#11]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#6))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#6))#17]
+Results [4]: [d_year#2, i_category_id#10, i_category#11, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#6))#17,17,2) AS sum(ss_ext_sales_price)#18]
+
+(21) TakeOrderedAndProject
+Input [4]: [d_year#2, i_category_id#10, i_category#11, sum(ss_ext_sales_price)#18]
+Arguments: 100, [sum(ss_ext_sales_price)#18 DESC NULLS LAST, d_year#2 ASC NULLS FIRST, i_category_id#10 ASC NULLS FIRST, i_category#11 ASC NULLS FIRST], [d_year#2, i_category_id#10, i_category#11, sum(ss_ext_sales_price)#18]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 6 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (22)
+
+
+(22) ReusedExchange [Reuses operator id: 5]
+Output [2]: [d_date_sk#1, d_year#2]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q42.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q42.sf100/simplified.txt
new file mode 100644
index 0000000000000..7e5c7eb6f59c9
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q42.sf100/simplified.txt
@@ -0,0 +1,33 @@
+TakeOrderedAndProject [sum(ss_ext_sales_price),d_year,i_category_id,i_category]
+  WholeStageCodegen (4)
+    HashAggregate [d_year,i_category_id,i_category,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(ss_ext_sales_price),sum]
+      InputAdapter
+        Exchange [d_year,i_category_id,i_category] #1
+          WholeStageCodegen (3)
+            HashAggregate [d_year,i_category_id,i_category,ss_ext_sales_price] [sum,sum]
+              Project [d_year,ss_ext_sales_price,i_category_id,i_category]
+                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                  Project [d_year,ss_item_sk,ss_ext_sales_price]
+                    BroadcastHashJoin [d_date_sk,ss_sold_date_sk]
+                      InputAdapter
+                        BroadcastExchange #2
+                          WholeStageCodegen (1)
+                            Project [d_date_sk,d_year]
+                              Filter [d_moy,d_year,d_date_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                      Filter [ss_item_sk]
+                        ColumnarToRow
+                          InputAdapter
+                            Scan parquet default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk]
+                              SubqueryBroadcast [d_date_sk] #1
+                                ReusedExchange [d_date_sk,d_year] #2
+                  InputAdapter
+                    BroadcastExchange #3
+                      WholeStageCodegen (2)
+                        Project [i_item_sk,i_category_id,i_category]
+                          Filter [i_manager_id,i_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.item [i_item_sk,i_category_id,i_category,i_manager_id]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q42/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q42/explain.txt
new file mode 100644
index 0000000000000..255f39414f908
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q42/explain.txt
@@ -0,0 +1,123 @@
+== Physical Plan ==
+TakeOrderedAndProject (21)
++- * HashAggregate (20)
+   +- Exchange (19)
+      +- * HashAggregate (18)
+         +- * Project (17)
+            +- * BroadcastHashJoin Inner BuildRight (16)
+               :- * Project (10)
+               :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :- * Project (4)
+               :     :  +- * Filter (3)
+               :     :     +- * ColumnarToRow (2)
+               :     :        +- Scan parquet default.date_dim (1)
+               :     +- BroadcastExchange (8)
+               :        +- * Filter (7)
+               :           +- * ColumnarToRow (6)
+               :              +- Scan parquet default.store_sales (5)
+               +- BroadcastExchange (15)
+                  +- * Project (14)
+                     +- * Filter (13)
+                        +- * ColumnarToRow (12)
+                           +- Scan parquet default.item (11)
+
+
+(1) Scan parquet default.date_dim
+Output [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,12), EqualTo(d_year,1998), GreaterThanOrEqual(d_date_sk,2451149), LessThanOrEqual(d_date_sk,2451179), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Condition : ((((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 12)) AND (d_year#2 = 1998)) AND (d_date_sk#1 >= 2451149)) AND (d_date_sk#1 <= 2451179)) AND isnotnull(d_date_sk#1))
+
+(4) Project [codegen id : 3]
+Output [2]: [d_date_sk#1, d_year#2]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(5) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#6), (ss_sold_date_sk#6 >= 2451149), (ss_sold_date_sk#6 <= 2451179), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+
+(7) Filter [codegen id : 1]
+Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Condition : isnotnull(ss_item_sk#4)
+
+(8) BroadcastExchange
+Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [d_date_sk#1]
+Right keys [1]: [ss_sold_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5]
+Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+
+(11) Scan parquet default.item
+Output [4]: [i_item_sk#8, i_category_id#9, i_category#10, i_manager_id#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_category_id:int,i_category:string,i_manager_id:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [4]: [i_item_sk#8, i_category_id#9, i_category#10, i_manager_id#11]
+
+(13) Filter [codegen id : 2]
+Input [4]: [i_item_sk#8, i_category_id#9, i_category#10, i_manager_id#11]
+Condition : ((isnotnull(i_manager_id#11) AND (i_manager_id#11 = 1)) AND isnotnull(i_item_sk#8))
+
+(14) Project [codegen id : 2]
+Output [3]: [i_item_sk#8, i_category_id#9, i_category#10]
+Input [4]: [i_item_sk#8, i_category_id#9, i_category#10, i_manager_id#11]
+
+(15) BroadcastExchange
+Input [3]: [i_item_sk#8, i_category_id#9, i_category#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(16) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_item_sk#4]
+Right keys [1]: [i_item_sk#8]
+Join condition: None
+
+(17) Project [codegen id : 3]
+Output [4]: [d_year#2, ss_ext_sales_price#5, i_category_id#9, i_category#10]
+Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#8, i_category_id#9, i_category#10]
+
+(18) HashAggregate [codegen id : 3]
+Input [4]: [d_year#2, ss_ext_sales_price#5, i_category_id#9, i_category#10]
+Keys [3]: [d_year#2, i_category_id#9, i_category#10]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))]
+Aggregate Attributes [1]: [sum#13]
+Results [4]: [d_year#2, i_category_id#9, i_category#10, sum#14]
+
+(19) Exchange
+Input [4]: [d_year#2, i_category_id#9, i_category#10, sum#14]
+Arguments: hashpartitioning(d_year#2, i_category_id#9, i_category#10, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(20) HashAggregate [codegen id : 4]
+Input [4]: [d_year#2, i_category_id#9, i_category#10, sum#14]
+Keys [3]: [d_year#2, i_category_id#9, i_category#10]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#16]
+Results [4]: [d_year#2, i_category_id#9, i_category#10, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#16,17,2) AS sum(ss_ext_sales_price)#17]
+
+(21) TakeOrderedAndProject
+Input [4]: [d_year#2, i_category_id#9, i_category#10, sum(ss_ext_sales_price)#17]
+Arguments: 100, [sum(ss_ext_sales_price)#17 DESC NULLS LAST, d_year#2 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_category#10 ASC NULLS FIRST], [d_year#2, i_category_id#9, i_category#10, sum(ss_ext_sales_price)#17]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q42/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q42/simplified.txt
new file mode 100644
index 0000000000000..4806a9309bd90
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q42/simplified.txt
@@ -0,0 +1,31 @@
+TakeOrderedAndProject [sum(ss_ext_sales_price),d_year,i_category_id,i_category]
+  WholeStageCodegen (4)
+    HashAggregate [d_year,i_category_id,i_category,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(ss_ext_sales_price),sum]
+      InputAdapter
+        Exchange [d_year,i_category_id,i_category] #1
+          WholeStageCodegen (3)
+            HashAggregate [d_year,i_category_id,i_category,ss_ext_sales_price] [sum,sum]
+              Project [d_year,ss_ext_sales_price,i_category_id,i_category]
+                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                  Project [d_year,ss_item_sk,ss_ext_sales_price]
+                    BroadcastHashJoin [d_date_sk,ss_sold_date_sk]
+                      Project [d_date_sk,d_year]
+                        Filter [d_moy,d_year,d_date_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                      InputAdapter
+                        BroadcastExchange #2
+                          WholeStageCodegen (1)
+                            Filter [ss_item_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk]
+                  InputAdapter
+                    BroadcastExchange #3
+                      WholeStageCodegen (2)
+                        Project [i_item_sk,i_category_id,i_category]
+                          Filter [i_manager_id,i_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.item [i_item_sk,i_category_id,i_category,i_manager_id]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q43.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q43.sf100/explain.txt
new file mode 100644
index 0000000000000..ec49a329ecded
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q43.sf100/explain.txt
@@ -0,0 +1,133 @@
+== Physical Plan ==
+TakeOrderedAndProject (21)
++- * HashAggregate (20)
+   +- Exchange (19)
+      +- * HashAggregate (18)
+         +- * Project (17)
+            +- * BroadcastHashJoin Inner BuildRight (16)
+               :- * Project (10)
+               :  +- * BroadcastHashJoin Inner BuildLeft (9)
+               :     :- BroadcastExchange (5)
+               :     :  +- * Project (4)
+               :     :     +- * Filter (3)
+               :     :        +- * ColumnarToRow (2)
+               :     :           +- Scan parquet default.date_dim (1)
+               :     +- * Filter (8)
+               :        +- * ColumnarToRow (7)
+               :           +- Scan parquet default.store_sales (6)
+               +- BroadcastExchange (15)
+                  +- * Project (14)
+                     +- * Filter (13)
+                        +- * ColumnarToRow (12)
+                           +- Scan parquet default.store (11)
+
+
+(1) Scan parquet default.date_dim
+Output [3]: [d_date_sk#1, d_year#2, d_day_name#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), GreaterThanOrEqual(d_date_sk,2450816), LessThanOrEqual(d_date_sk,2451179), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_day_name:string>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#1, d_year#2, d_day_name#3]
+
+(3) Filter [codegen id : 1]
+Input [3]: [d_date_sk#1, d_year#2, d_day_name#3]
+Condition : ((((isnotnull(d_year#2) AND (d_year#2 = 1998)) AND (d_date_sk#1 >= 2450816)) AND (d_date_sk#1 <= 2451179)) AND isnotnull(d_date_sk#1))
+
+(4) Project [codegen id : 1]
+Output [2]: [d_date_sk#1, d_day_name#3]
+Input [3]: [d_date_sk#1, d_year#2, d_day_name#3]
+
+(5) BroadcastExchange
+Input [2]: [d_date_sk#1, d_day_name#3]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#4]
+
+(6) Scan parquet default.store_sales
+Output [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#7), (ss_sold_date_sk#7 >= 2450816), (ss_sold_date_sk#7 <= 2451179), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(7) ColumnarToRow
+Input [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7]
+
+(8) Filter
+Input [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7]
+Condition : isnotnull(ss_store_sk#5)
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [d_date_sk#1]
+Right keys [1]: [ss_sold_date_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [3]: [d_day_name#3, ss_store_sk#5, ss_sales_price#6]
+Input [5]: [d_date_sk#1, d_day_name#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7]
+
+(11) Scan parquet default.store
+Output [4]: [s_store_sk#9, s_store_id#10, s_store_name#11, s_gmt_offset#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_gmt_offset), EqualTo(s_gmt_offset,-5.00), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string,s_store_name:string,s_gmt_offset:decimal(5,2)>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [4]: [s_store_sk#9, s_store_id#10, s_store_name#11, s_gmt_offset#12]
+
+(13) Filter [codegen id : 2]
+Input [4]: [s_store_sk#9, s_store_id#10, s_store_name#11, s_gmt_offset#12]
+Condition : ((isnotnull(s_gmt_offset#12) AND (s_gmt_offset#12 = -5.00)) AND isnotnull(s_store_sk#9))
+
+(14) Project [codegen id : 2]
+Output [3]: [s_store_sk#9, s_store_id#10, s_store_name#11]
+Input [4]: [s_store_sk#9, s_store_id#10, s_store_name#11, s_gmt_offset#12]
+
+(15) BroadcastExchange
+Input [3]: [s_store_sk#9, s_store_id#10, s_store_name#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(16) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_store_sk#5]
+Right keys [1]: [s_store_sk#9]
+Join condition: None
+
+(17) Project [codegen id : 3]
+Output [4]: [d_day_name#3, ss_sales_price#6, s_store_id#10, s_store_name#11]
+Input [6]: [d_day_name#3, ss_store_sk#5, ss_sales_price#6, s_store_sk#9, s_store_id#10, s_store_name#11]
+
+(18) HashAggregate [codegen id : 3]
+Input [4]: [d_day_name#3, ss_sales_price#6, s_store_id#10, s_store_name#11]
+Keys [2]: [s_store_name#11, s_store_id#10]
+Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday   ) THEN ss_sales_price#6 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday   ) THEN ss_sales_price#6 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday  ) THEN ss_sales_price#6 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#6 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#6 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday   ) THEN ss_sales_price#6 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#6 ELSE null END))]
+Aggregate Attributes [7]: [sum#14, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20]
+Results [9]: [s_store_name#11, s_store_id#10, sum#21, sum#22, sum#23, sum#24, sum#25, sum#26, sum#27]
+
+(19) Exchange
+Input [9]: [s_store_name#11, s_store_id#10, sum#21, sum#22, sum#23, sum#24, sum#25, sum#26, sum#27]
+Arguments: hashpartitioning(s_store_name#11, s_store_id#10, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(20) HashAggregate [codegen id : 4]
+Input [9]: [s_store_name#11, s_store_id#10, sum#21, sum#22, sum#23, sum#24, sum#25, sum#26, sum#27]
+Keys [2]: [s_store_name#11, s_store_id#10]
+Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday   ) THEN ss_sales_price#6 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday   ) THEN ss_sales_price#6 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday  ) THEN ss_sales_price#6 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#6 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#6 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday   ) THEN ss_sales_price#6 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#6 ELSE null END))]
+Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday   ) THEN ss_sales_price#6 ELSE null END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday   ) THEN ss_sales_price#6 ELSE null END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday  ) THEN ss_sales_price#6 ELSE null END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#6 ELSE null END))#32, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#6 ELSE null END))#33, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday   ) THEN ss_sales_price#6 ELSE null END))#34, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#6 ELSE null END))#35]
+Results [9]: [s_store_name#11, s_store_id#10, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday   ) THEN ss_sales_price#6 ELSE null END))#29,17,2) AS sun_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday   ) THEN ss_sales_price#6 ELSE null END))#30,17,2) AS mon_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday  ) THEN ss_sales_price#6 ELSE null END))#31,17,2) AS tue_sales#38, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#6 ELSE null END))#32,17,2) AS wed_sales#39, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#6 ELSE null END))#33,17,2) AS thu_sales#40, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday   ) THEN ss_sales_price#6 ELSE null END))#34,17,2) AS fri_sales#41, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#6 ELSE null END))#35,17,2) AS sat_sales#42]
+
+(21) TakeOrderedAndProject
+Input [9]: [s_store_name#11, s_store_id#10, sun_sales#36, mon_sales#37, tue_sales#38, wed_sales#39, thu_sales#40, fri_sales#41, sat_sales#42]
+Arguments: 100, [s_store_name#11 ASC NULLS FIRST, s_store_id#10 ASC NULLS FIRST, sun_sales#36 ASC NULLS FIRST, mon_sales#37 ASC NULLS FIRST, tue_sales#38 ASC NULLS FIRST, wed_sales#39 ASC NULLS FIRST, thu_sales#40 ASC NULLS FIRST, fri_sales#41 ASC NULLS FIRST, sat_sales#42 ASC NULLS FIRST], [s_store_name#11, s_store_id#10, sun_sales#36, mon_sales#37, tue_sales#38, wed_sales#39, thu_sales#40, fri_sales#41, sat_sales#42]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 6 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (22)
+
+
+(22) ReusedExchange [Reuses operator id: 5]
+Output [2]: [d_date_sk#1, d_day_name#3]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q43.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q43.sf100/simplified.txt
new file mode 100644
index 0000000000000..1fbb5aa612fa9
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q43.sf100/simplified.txt
@@ -0,0 +1,33 @@
+TakeOrderedAndProject [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales]
+  WholeStageCodegen (4)
+    HashAggregate [s_store_name,s_store_id,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday  ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price ELSE null END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum]
+      InputAdapter
+        Exchange [s_store_name,s_store_id] #1
+          WholeStageCodegen (3)
+            HashAggregate [s_store_name,s_store_id,d_day_name,ss_sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum]
+              Project [d_day_name,ss_sales_price,s_store_id,s_store_name]
+                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                  Project [d_day_name,ss_store_sk,ss_sales_price]
+                    BroadcastHashJoin [d_date_sk,ss_sold_date_sk]
+                      InputAdapter
+                        BroadcastExchange #2
+                          WholeStageCodegen (1)
+                            Project [d_date_sk,d_day_name]
+                              Filter [d_year,d_date_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.date_dim [d_date_sk,d_year,d_day_name]
+                      Filter [ss_store_sk]
+                        ColumnarToRow
+                          InputAdapter
+                            Scan parquet default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                              SubqueryBroadcast [d_date_sk] #1
+                                ReusedExchange [d_date_sk,d_day_name] #2
+                  InputAdapter
+                    BroadcastExchange #3
+                      WholeStageCodegen (2)
+                        Project [s_store_sk,s_store_id,s_store_name]
+                          Filter [s_gmt_offset,s_store_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q43/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q43/explain.txt
new file mode 100644
index 0000000000000..2c15f0ebcbd54
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q43/explain.txt
@@ -0,0 +1,123 @@
+== Physical Plan ==
+TakeOrderedAndProject (21)
++- * HashAggregate (20)
+   +- Exchange (19)
+      +- * HashAggregate (18)
+         +- * Project (17)
+            +- * BroadcastHashJoin Inner BuildRight (16)
+               :- * Project (10)
+               :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :- * Project (4)
+               :     :  +- * Filter (3)
+               :     :     +- * ColumnarToRow (2)
+               :     :        +- Scan parquet default.date_dim (1)
+               :     +- BroadcastExchange (8)
+               :        +- * Filter (7)
+               :           +- * ColumnarToRow (6)
+               :              +- Scan parquet default.store_sales (5)
+               +- BroadcastExchange (15)
+                  +- * Project (14)
+                     +- * Filter (13)
+                        +- * ColumnarToRow (12)
+                           +- Scan parquet default.store (11)
+
+
+(1) Scan parquet default.date_dim
+Output [3]: [d_date_sk#1, d_year#2, d_day_name#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), GreaterThanOrEqual(d_date_sk,2450816), LessThanOrEqual(d_date_sk,2451179), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_day_name:string>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#1, d_year#2, d_day_name#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [d_date_sk#1, d_year#2, d_day_name#3]
+Condition : ((((isnotnull(d_year#2) AND (d_year#2 = 1998)) AND (d_date_sk#1 >= 2450816)) AND (d_date_sk#1 <= 2451179)) AND isnotnull(d_date_sk#1))
+
+(4) Project [codegen id : 3]
+Output [2]: [d_date_sk#1, d_day_name#3]
+Input [3]: [d_date_sk#1, d_year#2, d_day_name#3]
+
+(5) Scan parquet default.store_sales
+Output [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#6), (ss_sold_date_sk#6 >= 2450816), (ss_sold_date_sk#6 <= 2451179), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6]
+
+(7) Filter [codegen id : 1]
+Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6]
+Condition : isnotnull(ss_store_sk#4)
+
+(8) BroadcastExchange
+Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [d_date_sk#1]
+Right keys [1]: [ss_sold_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [3]: [d_day_name#3, ss_store_sk#4, ss_sales_price#5]
+Input [5]: [d_date_sk#1, d_day_name#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6]
+
+(11) Scan parquet default.store
+Output [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_gmt_offset), EqualTo(s_gmt_offset,-5.00), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string,s_store_name:string,s_gmt_offset:decimal(5,2)>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11]
+
+(13) Filter [codegen id : 2]
+Input [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11]
+Condition : ((isnotnull(s_gmt_offset#11) AND (s_gmt_offset#11 = -5.00)) AND isnotnull(s_store_sk#8))
+
+(14) Project [codegen id : 2]
+Output [3]: [s_store_sk#8, s_store_id#9, s_store_name#10]
+Input [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11]
+
+(15) BroadcastExchange
+Input [3]: [s_store_sk#8, s_store_id#9, s_store_name#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(16) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_store_sk#4]
+Right keys [1]: [s_store_sk#8]
+Join condition: None
+
+(17) Project [codegen id : 3]
+Output [4]: [d_day_name#3, ss_sales_price#5, s_store_id#9, s_store_name#10]
+Input [6]: [d_day_name#3, ss_store_sk#4, ss_sales_price#5, s_store_sk#8, s_store_id#9, s_store_name#10]
+
+(18) HashAggregate [codegen id : 3]
+Input [4]: [d_day_name#3, ss_sales_price#5, s_store_id#9, s_store_name#10]
+Keys [2]: [s_store_name#10, s_store_id#9]
+Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday   ) THEN ss_sales_price#5 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday   ) THEN ss_sales_price#5 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday  ) THEN ss_sales_price#5 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday   ) THEN ss_sales_price#5 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 ELSE null END))]
+Aggregate Attributes [7]: [sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19]
+Results [9]: [s_store_name#10, s_store_id#9, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25, sum#26]
+
+(19) Exchange
+Input [9]: [s_store_name#10, s_store_id#9, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25, sum#26]
+Arguments: hashpartitioning(s_store_name#10, s_store_id#9, 5), ENSURE_REQUIREMENTS, [id=#27]
+
+(20) HashAggregate [codegen id : 4]
+Input [9]: [s_store_name#10, s_store_id#9, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25, sum#26]
+Keys [2]: [s_store_name#10, s_store_id#9]
+Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday   ) THEN ss_sales_price#5 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday   ) THEN ss_sales_price#5 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday  ) THEN ss_sales_price#5 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday   ) THEN ss_sales_price#5 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 ELSE null END))]
+Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday   ) THEN ss_sales_price#5 ELSE null END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday   ) THEN ss_sales_price#5 ELSE null END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday  ) THEN ss_sales_price#5 ELSE null END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 ELSE null END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 ELSE null END))#32, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday   ) THEN ss_sales_price#5 ELSE null END))#33, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 ELSE null END))#34]
+Results [9]: [s_store_name#10, s_store_id#9, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday   ) THEN ss_sales_price#5 ELSE null END))#28,17,2) AS sun_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday   ) THEN ss_sales_price#5 ELSE null END))#29,17,2) AS mon_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday  ) THEN ss_sales_price#5 ELSE null END))#30,17,2) AS tue_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 ELSE null END))#31,17,2) AS wed_sales#38, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 ELSE null END))#32,17,2) AS thu_sales#39, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday   ) THEN ss_sales_price#5 ELSE null END))#33,17,2) AS fri_sales#40, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 ELSE null END))#34,17,2) AS sat_sales#41]
+
+(21) TakeOrderedAndProject
+Input [9]: [s_store_name#10, s_store_id#9, sun_sales#35, mon_sales#36, tue_sales#37, wed_sales#38, thu_sales#39, fri_sales#40, sat_sales#41]
+Arguments: 100, [s_store_name#10 ASC NULLS FIRST, s_store_id#9 ASC NULLS FIRST, sun_sales#35 ASC NULLS FIRST, mon_sales#36 ASC NULLS FIRST, tue_sales#37 ASC NULLS FIRST, wed_sales#38 ASC NULLS FIRST, thu_sales#39 ASC NULLS FIRST, fri_sales#40 ASC NULLS FIRST, sat_sales#41 ASC NULLS FIRST], [s_store_name#10, s_store_id#9, sun_sales#35, mon_sales#36, tue_sales#37, wed_sales#38, thu_sales#39, fri_sales#40, sat_sales#41]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q43/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q43/simplified.txt
new file mode 100644
index 0000000000000..2d292e81891af
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q43/simplified.txt
@@ -0,0 +1,31 @@
+TakeOrderedAndProject [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales]
+  WholeStageCodegen (4)
+    HashAggregate [s_store_name,s_store_id,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday  ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price ELSE null END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum]
+      InputAdapter
+        Exchange [s_store_name,s_store_id] #1
+          WholeStageCodegen (3)
+            HashAggregate [s_store_name,s_store_id,d_day_name,ss_sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum]
+              Project [d_day_name,ss_sales_price,s_store_id,s_store_name]
+                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                  Project [d_day_name,ss_store_sk,ss_sales_price]
+                    BroadcastHashJoin [d_date_sk,ss_sold_date_sk]
+                      Project [d_date_sk,d_day_name]
+                        Filter [d_year,d_date_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.date_dim [d_date_sk,d_year,d_day_name]
+                      InputAdapter
+                        BroadcastExchange #2
+                          WholeStageCodegen (1)
+                            Filter [ss_store_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                  InputAdapter
+                    BroadcastExchange #3
+                      WholeStageCodegen (2)
+                        Project [s_store_sk,s_store_id,s_store_name]
+                          Filter [s_gmt_offset,s_store_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q46.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q46.sf100/explain.txt
new file mode 100644
index 0000000000000..e62003f82faec
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q46.sf100/explain.txt
@@ -0,0 +1,292 @@
+== Physical Plan ==
+TakeOrderedAndProject (51)
++- * Project (50)
+   +- * SortMergeJoin Inner (49)
+      :- * Sort (46)
+      :  +- Exchange (45)
+      :     +- * Project (44)
+      :        +- * SortMergeJoin Inner (43)
+      :           :- * Sort (37)
+      :           :  +- Exchange (36)
+      :           :     +- * HashAggregate (35)
+      :           :        +- * HashAggregate (34)
+      :           :           +- * Project (33)
+      :           :              +- * SortMergeJoin Inner (32)
+      :           :                 :- * Sort (26)
+      :           :                 :  +- Exchange (25)
+      :           :                 :     +- * Project (24)
+      :           :                 :        +- * BroadcastHashJoin Inner BuildRight (23)
+      :           :                 :           :- * Project (17)
+      :           :                 :           :  +- * BroadcastHashJoin Inner BuildRight (16)
+      :           :                 :           :     :- * Project (10)
+      :           :                 :           :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+      :           :                 :           :     :     :- * Filter (3)
+      :           :                 :           :     :     :  +- * ColumnarToRow (2)
+      :           :                 :           :     :     :     +- Scan parquet default.store_sales (1)
+      :           :                 :           :     :     +- BroadcastExchange (8)
+      :           :                 :           :     :        +- * Project (7)
+      :           :                 :           :     :           +- * Filter (6)
+      :           :                 :           :     :              +- * ColumnarToRow (5)
+      :           :                 :           :     :                 +- Scan parquet default.date_dim (4)
+      :           :                 :           :     +- BroadcastExchange (15)
+      :           :                 :           :        +- * Project (14)
+      :           :                 :           :           +- * Filter (13)
+      :           :                 :           :              +- * ColumnarToRow (12)
+      :           :                 :           :                 +- Scan parquet default.store (11)
+      :           :                 :           +- BroadcastExchange (22)
+      :           :                 :              +- * Project (21)
+      :           :                 :                 +- * Filter (20)
+      :           :                 :                    +- * ColumnarToRow (19)
+      :           :                 :                       +- Scan parquet default.household_demographics (18)
+      :           :                 +- * Sort (31)
+      :           :                    +- Exchange (30)
+      :           :                       +- * Filter (29)
+      :           :                          +- * ColumnarToRow (28)
+      :           :                             +- Scan parquet default.customer_address (27)
+      :           +- * Sort (42)
+      :              +- Exchange (41)
+      :                 +- * Filter (40)
+      :                    +- * ColumnarToRow (39)
+      :                       +- Scan parquet default.customer (38)
+      +- * Sort (48)
+         +- ReusedExchange (47)
+
+
+(1) Scan parquet default.store_sales
+Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [ss_sold_date_sk#8 INSET (2451790,2451609,2451294,2451658,2452099,2451482,2451700,2452035,2452274,2451258,2451847,2451714,2451937,2451860,2451601,2451573,2451686,2452008,2451454,2451882,2451832,2452259,2451671,2451903,2451497,2452162,2451322,2451517,2451434,2451273,2451405,2452105,2451924,2452050,2452126,2452203,2451818,2451559,2451853,2451238,2451209,2451357,2451959,2452239,2451608,2452141,2452252,2451623,2451867,2451504,2451910,2452232,2451874,2451581,2451329,2451223,2451783,2452267,2452042,2451895,2451986,2452091,2451693,2451265,2451678,2451825,2451244,2451490,2451287,2451419,2451546,2451245,2451713,2452070,2451189,2451804,2451468,2451525,2451902,2452077,2452161,2451378,2451567,2451931,2451699,2451251,2451840,2452253,2451938,2451510,2452231,2452036,2451616,2451230,2452112,2451846,2451966,2451538,2451819,2452140,2452183,2451496,2451791,2451595,2451574,2451363,2451994,2451917,2451602,2452273,2451237,2451350,2451685,2451259,2451286,2451972,2452224,2451370,2452245,2451643,2451993,2451315,2451301,2451560,2451433,2452225,2451532,2451755,2451854,2451545,2451210,2451587,2451987,2451447,2452197,2451552,2451896,2451679,2452147,2451735,2452022,2451707,2451868,2451398,2451777,2451181,2451503,2451839,2452175,2451441,2452154,2452029,2452196,2451952,2451805,2451965,2451539,2452001,2451833,2451392,2451524,2451461,2452133,2451448,2451307,2451615,2451769,2451412,2451349,2451651,2451763,2451203,2452064,2451980,2451748,2451637,2452182,2451279,2451231,2451734,2451692,2452071,2451336,2451300,2451727,2451630,2452189,2451875,2451973,2451328,2452084,2451399,2451944,2452204,2451385,2451776,2451384,2451272,2451812,2451749,2451566,2451182,2451945,2451420,2451930,2452057,2451756,2451644,2451314,2451364,2452007,2451798,2451475,2452015,2451440,2452000,2451588,2452148,2451195,2452217,2451371,2452176,2451531,2452134,2452211,2451462,2451188,2451741,2452119,2451342,2451580,2451672,2451889,2451280,2451406,2451293,2451217,2452049,2452106,2451321,2451335,2451483,2452260,2451657,2451979,2451518,2451629,2451728,2451923,2451861,2451951,2452246,2451455,2451356,2451224,2452210,2452021,2451427,2451202,2452098,2452168,2451553,2451391,2451706,2452155,2451196,2451770,2452127,2451762,2452078,2451958,2451721,2451665,2452120,2451252,2452085,2452092,2451476,2452218,2452169,2451797,2451650,2451881,2451511,2451469,2451888,2452043,2452266,2451664,2452014,2451343,2452056,2452190,2452063,2451636,2451742,2451811,2451720,2451308,2451489,2451413,2451216,2451594,2452238,2451784,2451426,2451622,2451916,2452113,2451909,2451266,2451826,2451377,2452028), isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_hdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_coupon_amt:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8]
+
+(3) Filter [codegen id : 4]
+Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8]
+Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#10, d_year#11, d_dow#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [In(d_dow, [6,0]), In(d_year, [1999,2000,2001]), In(d_date_sk, [2451790,2451609,2451294,2451658,2452099,2451482,2451700,2452035,2452274,2451258,2451847,2451714,2451937,2451860,2451601,2451573,2451686,2452008,2451454,2451882,2451832,2452259,2451671,2451903,2451497,2452162,2451322,2451517,2451434,2451273,2451405,2452105,2451924,2452050,2452126,2452203,2451818,2451559,2451853,2451238,2451209,2451357,2451959,2452239,2451608,2452141,2452252,2451623,2451867,2451504,2451910,2452232,2451874,2451581,2451329,2451223,2451783,2452267,2452042,2451895,2451986,2452091,2451693,2451265,2451678,2451825,2451244,2451490,2451287,2451419,2451546,2451245,2451713,2452070,2451189,2451804,2451468,2451525,2451902,2452077,2452161,2451378,2451567,2451931,2451699,2451251,2451840,2452253,2451938,2451510,2452231,2452036,2451616,2451230,2452112,2451846,2451966,2451538,2451819,2452140,2452183,2451496,2451791,2451595,2451574,2451363,2451994,2451917,2451602,2452273,2451237,2451350,2451685,2451259,2451286,2451972,2452224,2451370,2452245,2451643,2451993,2451315,2451301,2451560,2451433,2452225,2451532,2451755,2451854,2451545,2451210,2451587,2451987,2451447,2452197,2451552,2451896,2451679,2452147,2451735,2452022,2451707,2451868,2451398,2451777,2451181,2451503,2451839,2452175,2451441,2452154,2452029,2452196,2451952,2451805,2451965,2451539,2452001,2451833,2451392,2451524,2451461,2452133,2451448,2451307,2451615,2451769,2451412,2451349,2451651,2451763,2451203,2452064,2451980,2451748,2451637,2452182,2451279,2451231,2451734,2451692,2452071,2451336,2451300,2451727,2451630,2452189,2451875,2451973,2451328,2452084,2451399,2451944,2452204,2451385,2451776,2451384,2451272,2451812,2451749,2451566,2451182,2451945,2451420,2451930,2452057,2451756,2451644,2451314,2451364,2452007,2451798,2451475,2452015,2451440,2452000,2451588,2452148,2451195,2452217,2451371,2452176,2451531,2452134,2452211,2451462,2451188,2451741,2452119,2451342,2451580,2451672,2451889,2451280,2451406,2451293,2451217,2452049,2452106,2451321,2451335,2451483,2452260,2451657,2451979,2451518,2451629,2451728,2451923,2451861,2451951,2452246,2451455,2451356,2451224,2452210,2452021,2451427,2451202,2452098,2452168,2451553,2451391,2451706,2452155,2451196,2451770,2452127,2451762,2452078,2451958,2451721,2451665,2452120,2451252,2452085,2452092,2451476,2452218,2452169,2451797,2451650,2451881,2451511,2451469,2451888,2452043,2452266,2451664,2452014,2451343,2452056,2452190,2452063,2451636,2451742,2451811,2451720,2451308,2451489,2451413,2451216,2451594,2452238,2451784,2451426,2451622,2451916,2452113,2451909,2451266,2451826,2451377,2452028]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_dow:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#10, d_year#11, d_dow#12]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#10, d_year#11, d_dow#12]
+Condition : (((d_dow#12 IN (6,0) AND d_year#11 IN (1999,2000,2001)) AND d_date_sk#10 INSET (2451790,2451609,2451294,2451658,2452099,2451482,2451700,2452035,2452274,2451258,2451847,2451714,2451937,2451860,2451601,2451573,2451686,2452008,2451454,2451882,2451832,2452259,2451671,2451903,2451497,2452162,2451322,2451517,2451434,2451273,2451405,2452105,2451924,2452050,2452126,2452203,2451818,2451559,2451853,2451238,2451209,2451357,2451959,2452239,2451608,2452141,2452252,2451623,2451867,2451504,2451910,2452232,2451874,2451581,2451329,2451223,2451783,2452267,2452042,2451895,2451986,2452091,2451693,2451265,2451678,2451825,2451244,2451490,2451287,2451419,2451546,2451245,2451713,2452070,2451189,2451804,2451468,2451525,2451902,2452077,2452161,2451378,2451567,2451931,2451699,2451251,2451840,2452253,2451938,2451510,2452231,2452036,2451616,2451230,2452112,2451846,2451966,2451538,2451819,2452140,2452183,2451496,2451791,2451595,2451574,2451363,2451994,2451917,2451602,2452273,2451237,2451350,2451685,2451259,2451286,2451972,2452224,2451370,2452245,2451643,2451993,2451315,2451301,2451560,2451433,2452225,2451532,2451755,2451854,2451545,2451210,2451587,2451987,2451447,2452197,2451552,2451896,2451679,2452147,2451735,2452022,2451707,2451868,2451398,2451777,2451181,2451503,2451839,2452175,2451441,2452154,2452029,2452196,2451952,2451805,2451965,2451539,2452001,2451833,2451392,2451524,2451461,2452133,2451448,2451307,2451615,2451769,2451412,2451349,2451651,2451763,2451203,2452064,2451980,2451748,2451637,2452182,2451279,2451231,2451734,2451692,2452071,2451336,2451300,2451727,2451630,2452189,2451875,2451973,2451328,2452084,2451399,2451944,2452204,2451385,2451776,2451384,2451272,2451812,2451749,2451566,2451182,2451945,2451420,2451930,2452057,2451756,2451644,2451314,2451364,2452007,2451798,2451475,2452015,2451440,2452000,2451588,2452148,2451195,2452217,2451371,2452176,2451531,2452134,2452211,2451462,2451188,2451741,2452119,2451342,2451580,2451672,2451889,2451280,2451406,2451293,2451217,2452049,2452106,2451321,2451335,2451483,2452260,2451657,2451979,2451518,2451629,2451728,2451923,2451861,2451951,2452246,2451455,2451356,2451224,2452210,2452021,2451427,2451202,2452098,2452168,2451553,2451391,2451706,2452155,2451196,2451770,2452127,2451762,2452078,2451958,2451721,2451665,2452120,2451252,2452085,2452092,2451476,2452218,2452169,2451797,2451650,2451881,2451511,2451469,2451888,2452043,2452266,2451664,2452014,2451343,2452056,2452190,2452063,2451636,2451742,2451811,2451720,2451308,2451489,2451413,2451216,2451594,2452238,2451784,2451426,2451622,2451916,2452113,2451909,2451266,2451826,2451377,2452028)) AND isnotnull(d_date_sk#10))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#10]
+Input [3]: [d_date_sk#10, d_year#11, d_dow#12]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7]
+Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#10]
+
+(11) Scan parquet default.store
+Output [2]: [s_store_sk#14, s_city#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [In(s_city, [Midway,Concord,Spring Hill,Brownsville,Greenville]), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_city:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [s_store_sk#14, s_city#15]
+
+(13) Filter [codegen id : 2]
+Input [2]: [s_store_sk#14, s_city#15]
+Condition : (s_city#15 IN (Midway,Concord,Spring Hill,Brownsville,Greenville) AND isnotnull(s_store_sk#14))
+
+(14) Project [codegen id : 2]
+Output [1]: [s_store_sk#14]
+Input [2]: [s_store_sk#14, s_city#15]
+
+(15) BroadcastExchange
+Input [1]: [s_store_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#16]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#4]
+Right keys [1]: [s_store_sk#14]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7]
+Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#14]
+
+(18) Scan parquet default.household_demographics
+Output [3]: [hd_demo_sk#17, hd_dep_count#18, hd_vehicle_count#19]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [Or(EqualTo(hd_dep_count,5),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_dep_count:int,hd_vehicle_count:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [3]: [hd_demo_sk#17, hd_dep_count#18, hd_vehicle_count#19]
+
+(20) Filter [codegen id : 3]
+Input [3]: [hd_demo_sk#17, hd_dep_count#18, hd_vehicle_count#19]
+Condition : (((hd_dep_count#18 = 5) OR (hd_vehicle_count#19 = 3)) AND isnotnull(hd_demo_sk#17))
+
+(21) Project [codegen id : 3]
+Output [1]: [hd_demo_sk#17]
+Input [3]: [hd_demo_sk#17, hd_dep_count#18, hd_vehicle_count#19]
+
+(22) BroadcastExchange
+Input [1]: [hd_demo_sk#17]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#20]
+
+(23) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#17]
+Join condition: None
+
+(24) Project [codegen id : 4]
+Output [5]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7]
+Input [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, hd_demo_sk#17]
+
+(25) Exchange
+Input [5]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7]
+Arguments: hashpartitioning(ss_addr_sk#3, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(26) Sort [codegen id : 5]
+Input [5]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7]
+Arguments: [ss_addr_sk#3 ASC NULLS FIRST], false, 0
+
+(27) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#22, ca_city#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)]
+ReadSchema: struct<ca_address_sk:int,ca_city:string>
+
+(28) ColumnarToRow [codegen id : 6]
+Input [2]: [ca_address_sk#22, ca_city#23]
+
+(29) Filter [codegen id : 6]
+Input [2]: [ca_address_sk#22, ca_city#23]
+Condition : (isnotnull(ca_address_sk#22) AND isnotnull(ca_city#23))
+
+(30) Exchange
+Input [2]: [ca_address_sk#22, ca_city#23]
+Arguments: hashpartitioning(ca_address_sk#22, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(31) Sort [codegen id : 7]
+Input [2]: [ca_address_sk#22, ca_city#23]
+Arguments: [ca_address_sk#22 ASC NULLS FIRST], false, 0
+
+(32) SortMergeJoin [codegen id : 8]
+Left keys [1]: [ss_addr_sk#3]
+Right keys [1]: [ca_address_sk#22]
+Join condition: None
+
+(33) Project [codegen id : 8]
+Output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#23]
+Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_address_sk#22, ca_city#23]
+
+(34) HashAggregate [codegen id : 8]
+Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#23]
+Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#23]
+Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))]
+Aggregate Attributes [2]: [sum#25, sum#26]
+Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#23, sum#27, sum#28]
+
+(35) HashAggregate [codegen id : 8]
+Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#23, sum#27, sum#28]
+Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#23]
+Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#29, sum(UnscaledValue(ss_net_profit#7))#30]
+Results [5]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#23 AS bought_city#31, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#29,17,2) AS amt#32, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#30,17,2) AS profit#33]
+
+(36) Exchange
+Input [5]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#31, amt#32, profit#33]
+Arguments: hashpartitioning(ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#34]
+
+(37) Sort [codegen id : 9]
+Input [5]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#31, amt#32, profit#33]
+Arguments: [ss_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(38) Scan parquet default.customer
+Output [4]: [c_customer_sk#35, c_current_addr_sk#36, c_first_name#37, c_last_name#38]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int,c_first_name:string,c_last_name:string>
+
+(39) ColumnarToRow [codegen id : 10]
+Input [4]: [c_customer_sk#35, c_current_addr_sk#36, c_first_name#37, c_last_name#38]
+
+(40) Filter [codegen id : 10]
+Input [4]: [c_customer_sk#35, c_current_addr_sk#36, c_first_name#37, c_last_name#38]
+Condition : (isnotnull(c_customer_sk#35) AND isnotnull(c_current_addr_sk#36))
+
+(41) Exchange
+Input [4]: [c_customer_sk#35, c_current_addr_sk#36, c_first_name#37, c_last_name#38]
+Arguments: hashpartitioning(c_customer_sk#35, 5), ENSURE_REQUIREMENTS, [id=#39]
+
+(42) Sort [codegen id : 11]
+Input [4]: [c_customer_sk#35, c_current_addr_sk#36, c_first_name#37, c_last_name#38]
+Arguments: [c_customer_sk#35 ASC NULLS FIRST], false, 0
+
+(43) SortMergeJoin [codegen id : 12]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#35]
+Join condition: None
+
+(44) Project [codegen id : 12]
+Output [7]: [ss_ticket_number#5, bought_city#31, amt#32, profit#33, c_current_addr_sk#36, c_first_name#37, c_last_name#38]
+Input [9]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#31, amt#32, profit#33, c_customer_sk#35, c_current_addr_sk#36, c_first_name#37, c_last_name#38]
+
+(45) Exchange
+Input [7]: [ss_ticket_number#5, bought_city#31, amt#32, profit#33, c_current_addr_sk#36, c_first_name#37, c_last_name#38]
+Arguments: hashpartitioning(c_current_addr_sk#36, 5), ENSURE_REQUIREMENTS, [id=#40]
+
+(46) Sort [codegen id : 13]
+Input [7]: [ss_ticket_number#5, bought_city#31, amt#32, profit#33, c_current_addr_sk#36, c_first_name#37, c_last_name#38]
+Arguments: [c_current_addr_sk#36 ASC NULLS FIRST], false, 0
+
+(47) ReusedExchange [Reuses operator id: 30]
+Output [2]: [ca_address_sk#41, ca_city#42]
+
+(48) Sort [codegen id : 15]
+Input [2]: [ca_address_sk#41, ca_city#42]
+Arguments: [ca_address_sk#41 ASC NULLS FIRST], false, 0
+
+(49) SortMergeJoin [codegen id : 16]
+Left keys [1]: [c_current_addr_sk#36]
+Right keys [1]: [ca_address_sk#41]
+Join condition: NOT (ca_city#42 = bought_city#31)
+
+(50) Project [codegen id : 16]
+Output [7]: [c_last_name#38, c_first_name#37, ca_city#42, bought_city#31, ss_ticket_number#5, amt#32, profit#33]
+Input [9]: [ss_ticket_number#5, bought_city#31, amt#32, profit#33, c_current_addr_sk#36, c_first_name#37, c_last_name#38, ca_address_sk#41, ca_city#42]
+
+(51) TakeOrderedAndProject
+Input [7]: [c_last_name#38, c_first_name#37, ca_city#42, bought_city#31, ss_ticket_number#5, amt#32, profit#33]
+Arguments: 100, [c_last_name#38 ASC NULLS FIRST, c_first_name#37 ASC NULLS FIRST, ca_city#42 ASC NULLS FIRST, bought_city#31 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#38, c_first_name#37, ca_city#42, bought_city#31, ss_ticket_number#5, amt#32, profit#33]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (52)
+
+
+(52) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#10]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q46.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q46.sf100/simplified.txt
new file mode 100644
index 0000000000000..e3a4549474e37
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q46.sf100/simplified.txt
@@ -0,0 +1,89 @@
+TakeOrderedAndProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit]
+  WholeStageCodegen (16)
+    Project [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit]
+      SortMergeJoin [c_current_addr_sk,ca_address_sk,ca_city,bought_city]
+        InputAdapter
+          WholeStageCodegen (13)
+            Sort [c_current_addr_sk]
+              InputAdapter
+                Exchange [c_current_addr_sk] #1
+                  WholeStageCodegen (12)
+                    Project [ss_ticket_number,bought_city,amt,profit,c_current_addr_sk,c_first_name,c_last_name]
+                      SortMergeJoin [ss_customer_sk,c_customer_sk]
+                        InputAdapter
+                          WholeStageCodegen (9)
+                            Sort [ss_customer_sk]
+                              InputAdapter
+                                Exchange [ss_customer_sk] #2
+                                  WholeStageCodegen (8)
+                                    HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum] [sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit)),bought_city,amt,profit,sum,sum]
+                                      HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,ss_coupon_amt,ss_net_profit] [sum,sum,sum,sum]
+                                        Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_city]
+                                          SortMergeJoin [ss_addr_sk,ca_address_sk]
+                                            InputAdapter
+                                              WholeStageCodegen (5)
+                                                Sort [ss_addr_sk]
+                                                  InputAdapter
+                                                    Exchange [ss_addr_sk] #3
+                                                      WholeStageCodegen (4)
+                                                        Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit]
+                                                          BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                                            Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit]
+                                                              BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                                Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit]
+                                                                  BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                    Filter [ss_store_sk,ss_hdemo_sk,ss_addr_sk,ss_customer_sk]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk]
+                                                                            SubqueryBroadcast [d_date_sk] #1
+                                                                              ReusedExchange [d_date_sk] #4
+                                                                    InputAdapter
+                                                                      BroadcastExchange #4
+                                                                        WholeStageCodegen (1)
+                                                                          Project [d_date_sk]
+                                                                            Filter [d_dow,d_year,d_date_sk]
+                                                                              ColumnarToRow
+                                                                                InputAdapter
+                                                                                  Scan parquet default.date_dim [d_date_sk,d_year,d_dow]
+                                                                InputAdapter
+                                                                  BroadcastExchange #5
+                                                                    WholeStageCodegen (2)
+                                                                      Project [s_store_sk]
+                                                                        Filter [s_city,s_store_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.store [s_store_sk,s_city]
+                                                            InputAdapter
+                                                              BroadcastExchange #6
+                                                                WholeStageCodegen (3)
+                                                                  Project [hd_demo_sk]
+                                                                    Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count]
+                                            InputAdapter
+                                              WholeStageCodegen (7)
+                                                Sort [ca_address_sk]
+                                                  InputAdapter
+                                                    Exchange [ca_address_sk] #7
+                                                      WholeStageCodegen (6)
+                                                        Filter [ca_address_sk,ca_city]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.customer_address [ca_address_sk,ca_city]
+                        InputAdapter
+                          WholeStageCodegen (11)
+                            Sort [c_customer_sk]
+                              InputAdapter
+                                Exchange [c_customer_sk] #8
+                                  WholeStageCodegen (10)
+                                    Filter [c_customer_sk,c_current_addr_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name]
+        InputAdapter
+          WholeStageCodegen (15)
+            Sort [ca_address_sk]
+              InputAdapter
+                ReusedExchange [ca_address_sk,ca_city] #7
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q46/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q46/explain.txt
new file mode 100644
index 0000000000000..a00e2eeac49b7
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q46/explain.txt
@@ -0,0 +1,252 @@
+== Physical Plan ==
+TakeOrderedAndProject (43)
++- * Project (42)
+   +- * BroadcastHashJoin Inner BuildRight (41)
+      :- * Project (39)
+      :  +- * BroadcastHashJoin Inner BuildRight (38)
+      :     :- * HashAggregate (33)
+      :     :  +- Exchange (32)
+      :     :     +- * HashAggregate (31)
+      :     :        +- * Project (30)
+      :     :           +- * BroadcastHashJoin Inner BuildRight (29)
+      :     :              :- * Project (24)
+      :     :              :  +- * BroadcastHashJoin Inner BuildRight (23)
+      :     :              :     :- * Project (17)
+      :     :              :     :  +- * BroadcastHashJoin Inner BuildRight (16)
+      :     :              :     :     :- * Project (10)
+      :     :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+      :     :              :     :     :     :- * Filter (3)
+      :     :              :     :     :     :  +- * ColumnarToRow (2)
+      :     :              :     :     :     :     +- Scan parquet default.store_sales (1)
+      :     :              :     :     :     +- BroadcastExchange (8)
+      :     :              :     :     :        +- * Project (7)
+      :     :              :     :     :           +- * Filter (6)
+      :     :              :     :     :              +- * ColumnarToRow (5)
+      :     :              :     :     :                 +- Scan parquet default.date_dim (4)
+      :     :              :     :     +- BroadcastExchange (15)
+      :     :              :     :        +- * Project (14)
+      :     :              :     :           +- * Filter (13)
+      :     :              :     :              +- * ColumnarToRow (12)
+      :     :              :     :                 +- Scan parquet default.store (11)
+      :     :              :     +- BroadcastExchange (22)
+      :     :              :        +- * Project (21)
+      :     :              :           +- * Filter (20)
+      :     :              :              +- * ColumnarToRow (19)
+      :     :              :                 +- Scan parquet default.household_demographics (18)
+      :     :              +- BroadcastExchange (28)
+      :     :                 +- * Filter (27)
+      :     :                    +- * ColumnarToRow (26)
+      :     :                       +- Scan parquet default.customer_address (25)
+      :     +- BroadcastExchange (37)
+      :        +- * Filter (36)
+      :           +- * ColumnarToRow (35)
+      :              +- Scan parquet default.customer (34)
+      +- ReusedExchange (40)
+
+
+(1) Scan parquet default.store_sales
+Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [ss_sold_date_sk#8 INSET (2451790,2451609,2451294,2451658,2452099,2451482,2451700,2452035,2452274,2451258,2451847,2451714,2451937,2451860,2451601,2451573,2451686,2452008,2451454,2451882,2451832,2452259,2451671,2451903,2451497,2452162,2451322,2451517,2451434,2451273,2451405,2452105,2451924,2452050,2452126,2452203,2451818,2451559,2451853,2451238,2451209,2451357,2451959,2452239,2451608,2452141,2452252,2451623,2451867,2451504,2451910,2452232,2451874,2451581,2451329,2451223,2451783,2452267,2452042,2451895,2451986,2452091,2451693,2451265,2451678,2451825,2451244,2451490,2451287,2451419,2451546,2451245,2451713,2452070,2451189,2451804,2451468,2451525,2451902,2452077,2452161,2451378,2451567,2451931,2451699,2451251,2451840,2452253,2451938,2451510,2452231,2452036,2451616,2451230,2452112,2451846,2451966,2451538,2451819,2452140,2452183,2451496,2451791,2451595,2451574,2451363,2451994,2451917,2451602,2452273,2451237,2451350,2451685,2451259,2451286,2451972,2452224,2451370,2452245,2451643,2451993,2451315,2451301,2451560,2451433,2452225,2451532,2451755,2451854,2451545,2451210,2451587,2451987,2451447,2452197,2451552,2451896,2451679,2452147,2451735,2452022,2451707,2451868,2451398,2451777,2451181,2451503,2451839,2452175,2451441,2452154,2452029,2452196,2451952,2451805,2451965,2451539,2452001,2451833,2451392,2451524,2451461,2452133,2451448,2451307,2451615,2451769,2451412,2451349,2451651,2451763,2451203,2452064,2451980,2451748,2451637,2452182,2451279,2451231,2451734,2451692,2452071,2451336,2451300,2451727,2451630,2452189,2451875,2451973,2451328,2452084,2451399,2451944,2452204,2451385,2451776,2451384,2451272,2451812,2451749,2451566,2451182,2451945,2451420,2451930,2452057,2451756,2451644,2451314,2451364,2452007,2451798,2451475,2452015,2451440,2452000,2451588,2452148,2451195,2452217,2451371,2452176,2451531,2452134,2452211,2451462,2451188,2451741,2452119,2451342,2451580,2451672,2451889,2451280,2451406,2451293,2451217,2452049,2452106,2451321,2451335,2451483,2452260,2451657,2451979,2451518,2451629,2451728,2451923,2451861,2451951,2452246,2451455,2451356,2451224,2452210,2452021,2451427,2451202,2452098,2452168,2451553,2451391,2451706,2452155,2451196,2451770,2452127,2451762,2452078,2451958,2451721,2451665,2452120,2451252,2452085,2452092,2451476,2452218,2452169,2451797,2451650,2451881,2451511,2451469,2451888,2452043,2452266,2451664,2452014,2451343,2452056,2452190,2452063,2451636,2451742,2451811,2451720,2451308,2451489,2451413,2451216,2451594,2452238,2451784,2451426,2451622,2451916,2452113,2451909,2451266,2451826,2451377,2452028), isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_hdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_coupon_amt:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8]
+
+(3) Filter [codegen id : 5]
+Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8]
+Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#10, d_year#11, d_dow#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [In(d_dow, [6,0]), In(d_year, [1999,2000,2001]), In(d_date_sk, [2451790,2451609,2451294,2451658,2452099,2451482,2451700,2452035,2452274,2451258,2451847,2451714,2451937,2451860,2451601,2451573,2451686,2452008,2451454,2451882,2451832,2452259,2451671,2451903,2451497,2452162,2451322,2451517,2451434,2451273,2451405,2452105,2451924,2452050,2452126,2452203,2451818,2451559,2451853,2451238,2451209,2451357,2451959,2452239,2451608,2452141,2452252,2451623,2451867,2451504,2451910,2452232,2451874,2451581,2451329,2451223,2451783,2452267,2452042,2451895,2451986,2452091,2451693,2451265,2451678,2451825,2451244,2451490,2451287,2451419,2451546,2451245,2451713,2452070,2451189,2451804,2451468,2451525,2451902,2452077,2452161,2451378,2451567,2451931,2451699,2451251,2451840,2452253,2451938,2451510,2452231,2452036,2451616,2451230,2452112,2451846,2451966,2451538,2451819,2452140,2452183,2451496,2451791,2451595,2451574,2451363,2451994,2451917,2451602,2452273,2451237,2451350,2451685,2451259,2451286,2451972,2452224,2451370,2452245,2451643,2451993,2451315,2451301,2451560,2451433,2452225,2451532,2451755,2451854,2451545,2451210,2451587,2451987,2451447,2452197,2451552,2451896,2451679,2452147,2451735,2452022,2451707,2451868,2451398,2451777,2451181,2451503,2451839,2452175,2451441,2452154,2452029,2452196,2451952,2451805,2451965,2451539,2452001,2451833,2451392,2451524,2451461,2452133,2451448,2451307,2451615,2451769,2451412,2451349,2451651,2451763,2451203,2452064,2451980,2451748,2451637,2452182,2451279,2451231,2451734,2451692,2452071,2451336,2451300,2451727,2451630,2452189,2451875,2451973,2451328,2452084,2451399,2451944,2452204,2451385,2451776,2451384,2451272,2451812,2451749,2451566,2451182,2451945,2451420,2451930,2452057,2451756,2451644,2451314,2451364,2452007,2451798,2451475,2452015,2451440,2452000,2451588,2452148,2451195,2452217,2451371,2452176,2451531,2452134,2452211,2451462,2451188,2451741,2452119,2451342,2451580,2451672,2451889,2451280,2451406,2451293,2451217,2452049,2452106,2451321,2451335,2451483,2452260,2451657,2451979,2451518,2451629,2451728,2451923,2451861,2451951,2452246,2451455,2451356,2451224,2452210,2452021,2451427,2451202,2452098,2452168,2451553,2451391,2451706,2452155,2451196,2451770,2452127,2451762,2452078,2451958,2451721,2451665,2452120,2451252,2452085,2452092,2451476,2452218,2452169,2451797,2451650,2451881,2451511,2451469,2451888,2452043,2452266,2451664,2452014,2451343,2452056,2452190,2452063,2451636,2451742,2451811,2451720,2451308,2451489,2451413,2451216,2451594,2452238,2451784,2451426,2451622,2451916,2452113,2451909,2451266,2451826,2451377,2452028]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_dow:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#10, d_year#11, d_dow#12]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#10, d_year#11, d_dow#12]
+Condition : (((d_dow#12 IN (6,0) AND d_year#11 IN (1999,2000,2001)) AND d_date_sk#10 INSET (2451790,2451609,2451294,2451658,2452099,2451482,2451700,2452035,2452274,2451258,2451847,2451714,2451937,2451860,2451601,2451573,2451686,2452008,2451454,2451882,2451832,2452259,2451671,2451903,2451497,2452162,2451322,2451517,2451434,2451273,2451405,2452105,2451924,2452050,2452126,2452203,2451818,2451559,2451853,2451238,2451209,2451357,2451959,2452239,2451608,2452141,2452252,2451623,2451867,2451504,2451910,2452232,2451874,2451581,2451329,2451223,2451783,2452267,2452042,2451895,2451986,2452091,2451693,2451265,2451678,2451825,2451244,2451490,2451287,2451419,2451546,2451245,2451713,2452070,2451189,2451804,2451468,2451525,2451902,2452077,2452161,2451378,2451567,2451931,2451699,2451251,2451840,2452253,2451938,2451510,2452231,2452036,2451616,2451230,2452112,2451846,2451966,2451538,2451819,2452140,2452183,2451496,2451791,2451595,2451574,2451363,2451994,2451917,2451602,2452273,2451237,2451350,2451685,2451259,2451286,2451972,2452224,2451370,2452245,2451643,2451993,2451315,2451301,2451560,2451433,2452225,2451532,2451755,2451854,2451545,2451210,2451587,2451987,2451447,2452197,2451552,2451896,2451679,2452147,2451735,2452022,2451707,2451868,2451398,2451777,2451181,2451503,2451839,2452175,2451441,2452154,2452029,2452196,2451952,2451805,2451965,2451539,2452001,2451833,2451392,2451524,2451461,2452133,2451448,2451307,2451615,2451769,2451412,2451349,2451651,2451763,2451203,2452064,2451980,2451748,2451637,2452182,2451279,2451231,2451734,2451692,2452071,2451336,2451300,2451727,2451630,2452189,2451875,2451973,2451328,2452084,2451399,2451944,2452204,2451385,2451776,2451384,2451272,2451812,2451749,2451566,2451182,2451945,2451420,2451930,2452057,2451756,2451644,2451314,2451364,2452007,2451798,2451475,2452015,2451440,2452000,2451588,2452148,2451195,2452217,2451371,2452176,2451531,2452134,2452211,2451462,2451188,2451741,2452119,2451342,2451580,2451672,2451889,2451280,2451406,2451293,2451217,2452049,2452106,2451321,2451335,2451483,2452260,2451657,2451979,2451518,2451629,2451728,2451923,2451861,2451951,2452246,2451455,2451356,2451224,2452210,2452021,2451427,2451202,2452098,2452168,2451553,2451391,2451706,2452155,2451196,2451770,2452127,2451762,2452078,2451958,2451721,2451665,2452120,2451252,2452085,2452092,2451476,2452218,2452169,2451797,2451650,2451881,2451511,2451469,2451888,2452043,2452266,2451664,2452014,2451343,2452056,2452190,2452063,2451636,2451742,2451811,2451720,2451308,2451489,2451413,2451216,2451594,2452238,2451784,2451426,2451622,2451916,2452113,2451909,2451266,2451826,2451377,2452028)) AND isnotnull(d_date_sk#10))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#10]
+Input [3]: [d_date_sk#10, d_year#11, d_dow#12]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7]
+Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#10]
+
+(11) Scan parquet default.store
+Output [2]: [s_store_sk#14, s_city#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [In(s_city, [Midway,Concord,Spring Hill,Brownsville,Greenville]), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_city:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [s_store_sk#14, s_city#15]
+
+(13) Filter [codegen id : 2]
+Input [2]: [s_store_sk#14, s_city#15]
+Condition : (s_city#15 IN (Midway,Concord,Spring Hill,Brownsville,Greenville) AND isnotnull(s_store_sk#14))
+
+(14) Project [codegen id : 2]
+Output [1]: [s_store_sk#14]
+Input [2]: [s_store_sk#14, s_city#15]
+
+(15) BroadcastExchange
+Input [1]: [s_store_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#16]
+
+(16) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_store_sk#4]
+Right keys [1]: [s_store_sk#14]
+Join condition: None
+
+(17) Project [codegen id : 5]
+Output [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7]
+Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#14]
+
+(18) Scan parquet default.household_demographics
+Output [3]: [hd_demo_sk#17, hd_dep_count#18, hd_vehicle_count#19]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [Or(EqualTo(hd_dep_count,5),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_dep_count:int,hd_vehicle_count:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [3]: [hd_demo_sk#17, hd_dep_count#18, hd_vehicle_count#19]
+
+(20) Filter [codegen id : 3]
+Input [3]: [hd_demo_sk#17, hd_dep_count#18, hd_vehicle_count#19]
+Condition : (((hd_dep_count#18 = 5) OR (hd_vehicle_count#19 = 3)) AND isnotnull(hd_demo_sk#17))
+
+(21) Project [codegen id : 3]
+Output [1]: [hd_demo_sk#17]
+Input [3]: [hd_demo_sk#17, hd_dep_count#18, hd_vehicle_count#19]
+
+(22) BroadcastExchange
+Input [1]: [hd_demo_sk#17]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#20]
+
+(23) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#17]
+Join condition: None
+
+(24) Project [codegen id : 5]
+Output [5]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7]
+Input [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, hd_demo_sk#17]
+
+(25) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#21, ca_city#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)]
+ReadSchema: struct<ca_address_sk:int,ca_city:string>
+
+(26) ColumnarToRow [codegen id : 4]
+Input [2]: [ca_address_sk#21, ca_city#22]
+
+(27) Filter [codegen id : 4]
+Input [2]: [ca_address_sk#21, ca_city#22]
+Condition : (isnotnull(ca_address_sk#21) AND isnotnull(ca_city#22))
+
+(28) BroadcastExchange
+Input [2]: [ca_address_sk#21, ca_city#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#23]
+
+(29) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_addr_sk#3]
+Right keys [1]: [ca_address_sk#21]
+Join condition: None
+
+(30) Project [codegen id : 5]
+Output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#22]
+Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_address_sk#21, ca_city#22]
+
+(31) HashAggregate [codegen id : 5]
+Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#22]
+Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#22]
+Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))]
+Aggregate Attributes [2]: [sum#24, sum#25]
+Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#22, sum#26, sum#27]
+
+(32) Exchange
+Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#22, sum#26, sum#27]
+Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#22, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(33) HashAggregate [codegen id : 8]
+Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#22, sum#26, sum#27]
+Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#22]
+Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#29, sum(UnscaledValue(ss_net_profit#7))#30]
+Results [5]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#22 AS bought_city#31, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#29,17,2) AS amt#32, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#30,17,2) AS profit#33]
+
+(34) Scan parquet default.customer
+Output [4]: [c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int,c_first_name:string,c_last_name:string>
+
+(35) ColumnarToRow [codegen id : 6]
+Input [4]: [c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37]
+
+(36) Filter [codegen id : 6]
+Input [4]: [c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37]
+Condition : (isnotnull(c_customer_sk#34) AND isnotnull(c_current_addr_sk#35))
+
+(37) BroadcastExchange
+Input [4]: [c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#38]
+
+(38) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#34]
+Join condition: None
+
+(39) Project [codegen id : 8]
+Output [7]: [ss_ticket_number#5, bought_city#31, amt#32, profit#33, c_current_addr_sk#35, c_first_name#36, c_last_name#37]
+Input [9]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#31, amt#32, profit#33, c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37]
+
+(40) ReusedExchange [Reuses operator id: 28]
+Output [2]: [ca_address_sk#39, ca_city#40]
+
+(41) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [c_current_addr_sk#35]
+Right keys [1]: [ca_address_sk#39]
+Join condition: NOT (ca_city#40 = bought_city#31)
+
+(42) Project [codegen id : 8]
+Output [7]: [c_last_name#37, c_first_name#36, ca_city#40, bought_city#31, ss_ticket_number#5, amt#32, profit#33]
+Input [9]: [ss_ticket_number#5, bought_city#31, amt#32, profit#33, c_current_addr_sk#35, c_first_name#36, c_last_name#37, ca_address_sk#39, ca_city#40]
+
+(43) TakeOrderedAndProject
+Input [7]: [c_last_name#37, c_first_name#36, ca_city#40, bought_city#31, ss_ticket_number#5, amt#32, profit#33]
+Arguments: 100, [c_last_name#37 ASC NULLS FIRST, c_first_name#36 ASC NULLS FIRST, ca_city#40 ASC NULLS FIRST, bought_city#31 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#37, c_first_name#36, ca_city#40, bought_city#31, ss_ticket_number#5, amt#32, profit#33]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (44)
+
+
+(44) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#10]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q46/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q46/simplified.txt
new file mode 100644
index 0000000000000..7f729f9874b53
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q46/simplified.txt
@@ -0,0 +1,65 @@
+TakeOrderedAndProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit]
+  WholeStageCodegen (8)
+    Project [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit]
+      BroadcastHashJoin [c_current_addr_sk,ca_address_sk,ca_city,bought_city]
+        Project [ss_ticket_number,bought_city,amt,profit,c_current_addr_sk,c_first_name,c_last_name]
+          BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+            HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum] [sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit)),bought_city,amt,profit,sum,sum]
+              InputAdapter
+                Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1
+                  WholeStageCodegen (5)
+                    HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,ss_coupon_amt,ss_net_profit] [sum,sum,sum,sum]
+                      Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_city]
+                        BroadcastHashJoin [ss_addr_sk,ca_address_sk]
+                          Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit]
+                            BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                              Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit]
+                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                  Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit]
+                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                      Filter [ss_store_sk,ss_hdemo_sk,ss_addr_sk,ss_customer_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #1
+                                                ReusedExchange [d_date_sk] #2
+                                      InputAdapter
+                                        BroadcastExchange #2
+                                          WholeStageCodegen (1)
+                                            Project [d_date_sk]
+                                              Filter [d_dow,d_year,d_date_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_date_sk,d_year,d_dow]
+                                  InputAdapter
+                                    BroadcastExchange #3
+                                      WholeStageCodegen (2)
+                                        Project [s_store_sk]
+                                          Filter [s_city,s_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store [s_store_sk,s_city]
+                              InputAdapter
+                                BroadcastExchange #4
+                                  WholeStageCodegen (3)
+                                    Project [hd_demo_sk]
+                                      Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count]
+                          InputAdapter
+                            BroadcastExchange #5
+                              WholeStageCodegen (4)
+                                Filter [ca_address_sk,ca_city]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.customer_address [ca_address_sk,ca_city]
+            InputAdapter
+              BroadcastExchange #6
+                WholeStageCodegen (6)
+                  Filter [c_customer_sk,c_current_addr_sk]
+                    ColumnarToRow
+                      InputAdapter
+                        Scan parquet default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name]
+        InputAdapter
+          ReusedExchange [ca_address_sk,ca_city] #5
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q52.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q52.sf100/explain.txt
new file mode 100644
index 0000000000000..02f680fc0dd1a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q52.sf100/explain.txt
@@ -0,0 +1,133 @@
+== Physical Plan ==
+TakeOrderedAndProject (21)
++- * HashAggregate (20)
+   +- Exchange (19)
+      +- * HashAggregate (18)
+         +- * Project (17)
+            +- * BroadcastHashJoin Inner BuildRight (16)
+               :- * Project (10)
+               :  +- * BroadcastHashJoin Inner BuildLeft (9)
+               :     :- BroadcastExchange (5)
+               :     :  +- * Project (4)
+               :     :     +- * Filter (3)
+               :     :        +- * ColumnarToRow (2)
+               :     :           +- Scan parquet default.date_dim (1)
+               :     +- * Filter (8)
+               :        +- * ColumnarToRow (7)
+               :           +- Scan parquet default.store_sales (6)
+               +- BroadcastExchange (15)
+                  +- * Project (14)
+                     +- * Filter (13)
+                        +- * ColumnarToRow (12)
+                           +- Scan parquet default.item (11)
+
+
+(1) Scan parquet default.date_dim
+Output [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,12), EqualTo(d_year,1998), GreaterThanOrEqual(d_date_sk,2451149), LessThanOrEqual(d_date_sk,2451179), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(3) Filter [codegen id : 1]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Condition : ((((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 12)) AND (d_year#2 = 1998)) AND (d_date_sk#1 >= 2451149)) AND (d_date_sk#1 <= 2451179)) AND isnotnull(d_date_sk#1))
+
+(4) Project [codegen id : 1]
+Output [2]: [d_date_sk#1, d_year#2]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(5) BroadcastExchange
+Input [2]: [d_date_sk#1, d_year#2]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#4]
+
+(6) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#5, ss_ext_sales_price#6, ss_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#7), (ss_sold_date_sk#7 >= 2451149), (ss_sold_date_sk#7 <= 2451179), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(7) ColumnarToRow
+Input [3]: [ss_item_sk#5, ss_ext_sales_price#6, ss_sold_date_sk#7]
+
+(8) Filter
+Input [3]: [ss_item_sk#5, ss_ext_sales_price#6, ss_sold_date_sk#7]
+Condition : isnotnull(ss_item_sk#5)
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [d_date_sk#1]
+Right keys [1]: [ss_sold_date_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [3]: [d_year#2, ss_item_sk#5, ss_ext_sales_price#6]
+Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#5, ss_ext_sales_price#6, ss_sold_date_sk#7]
+
+(11) Scan parquet default.item
+Output [4]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manager_id#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_brand:string,i_manager_id:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [4]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manager_id#12]
+
+(13) Filter [codegen id : 2]
+Input [4]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manager_id#12]
+Condition : ((isnotnull(i_manager_id#12) AND (i_manager_id#12 = 1)) AND isnotnull(i_item_sk#9))
+
+(14) Project [codegen id : 2]
+Output [3]: [i_item_sk#9, i_brand_id#10, i_brand#11]
+Input [4]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manager_id#12]
+
+(15) BroadcastExchange
+Input [3]: [i_item_sk#9, i_brand_id#10, i_brand#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(16) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_item_sk#5]
+Right keys [1]: [i_item_sk#9]
+Join condition: None
+
+(17) Project [codegen id : 3]
+Output [4]: [d_year#2, ss_ext_sales_price#6, i_brand_id#10, i_brand#11]
+Input [6]: [d_year#2, ss_item_sk#5, ss_ext_sales_price#6, i_item_sk#9, i_brand_id#10, i_brand#11]
+
+(18) HashAggregate [codegen id : 3]
+Input [4]: [d_year#2, ss_ext_sales_price#6, i_brand_id#10, i_brand#11]
+Keys [3]: [d_year#2, i_brand#11, i_brand_id#10]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#6))]
+Aggregate Attributes [1]: [sum#14]
+Results [4]: [d_year#2, i_brand#11, i_brand_id#10, sum#15]
+
+(19) Exchange
+Input [4]: [d_year#2, i_brand#11, i_brand_id#10, sum#15]
+Arguments: hashpartitioning(d_year#2, i_brand#11, i_brand_id#10, 5), ENSURE_REQUIREMENTS, [id=#16]
+
+(20) HashAggregate [codegen id : 4]
+Input [4]: [d_year#2, i_brand#11, i_brand_id#10, sum#15]
+Keys [3]: [d_year#2, i_brand#11, i_brand_id#10]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#6))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#6))#17]
+Results [4]: [d_year#2, i_brand_id#10 AS brand_id#18, i_brand#11 AS brand#19, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#6))#17,17,2) AS ext_price#20]
+
+(21) TakeOrderedAndProject
+Input [4]: [d_year#2, brand_id#18, brand#19, ext_price#20]
+Arguments: 100, [d_year#2 ASC NULLS FIRST, ext_price#20 DESC NULLS LAST, brand_id#18 ASC NULLS FIRST], [d_year#2, brand_id#18, brand#19, ext_price#20]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 6 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (22)
+
+
+(22) ReusedExchange [Reuses operator id: 5]
+Output [2]: [d_date_sk#1, d_year#2]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q52.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q52.sf100/simplified.txt
new file mode 100644
index 0000000000000..6caf3262b7b8f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q52.sf100/simplified.txt
@@ -0,0 +1,33 @@
+TakeOrderedAndProject [d_year,ext_price,brand_id,brand]
+  WholeStageCodegen (4)
+    HashAggregate [d_year,i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum]
+      InputAdapter
+        Exchange [d_year,i_brand,i_brand_id] #1
+          WholeStageCodegen (3)
+            HashAggregate [d_year,i_brand,i_brand_id,ss_ext_sales_price] [sum,sum]
+              Project [d_year,ss_ext_sales_price,i_brand_id,i_brand]
+                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                  Project [d_year,ss_item_sk,ss_ext_sales_price]
+                    BroadcastHashJoin [d_date_sk,ss_sold_date_sk]
+                      InputAdapter
+                        BroadcastExchange #2
+                          WholeStageCodegen (1)
+                            Project [d_date_sk,d_year]
+                              Filter [d_moy,d_year,d_date_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                      Filter [ss_item_sk]
+                        ColumnarToRow
+                          InputAdapter
+                            Scan parquet default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk]
+                              SubqueryBroadcast [d_date_sk] #1
+                                ReusedExchange [d_date_sk,d_year] #2
+                  InputAdapter
+                    BroadcastExchange #3
+                      WholeStageCodegen (2)
+                        Project [i_item_sk,i_brand_id,i_brand]
+                          Filter [i_manager_id,i_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.item [i_item_sk,i_brand_id,i_brand,i_manager_id]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q52/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q52/explain.txt
new file mode 100644
index 0000000000000..8d081d60f9e85
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q52/explain.txt
@@ -0,0 +1,123 @@
+== Physical Plan ==
+TakeOrderedAndProject (21)
++- * HashAggregate (20)
+   +- Exchange (19)
+      +- * HashAggregate (18)
+         +- * Project (17)
+            +- * BroadcastHashJoin Inner BuildRight (16)
+               :- * Project (10)
+               :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :- * Project (4)
+               :     :  +- * Filter (3)
+               :     :     +- * ColumnarToRow (2)
+               :     :        +- Scan parquet default.date_dim (1)
+               :     +- BroadcastExchange (8)
+               :        +- * Filter (7)
+               :           +- * ColumnarToRow (6)
+               :              +- Scan parquet default.store_sales (5)
+               +- BroadcastExchange (15)
+                  +- * Project (14)
+                     +- * Filter (13)
+                        +- * ColumnarToRow (12)
+                           +- Scan parquet default.item (11)
+
+
+(1) Scan parquet default.date_dim
+Output [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,12), EqualTo(d_year,1998), GreaterThanOrEqual(d_date_sk,2451149), LessThanOrEqual(d_date_sk,2451179), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Condition : ((((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 12)) AND (d_year#2 = 1998)) AND (d_date_sk#1 >= 2451149)) AND (d_date_sk#1 <= 2451179)) AND isnotnull(d_date_sk#1))
+
+(4) Project [codegen id : 3]
+Output [2]: [d_date_sk#1, d_year#2]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(5) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#6), (ss_sold_date_sk#6 >= 2451149), (ss_sold_date_sk#6 <= 2451179), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+
+(7) Filter [codegen id : 1]
+Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Condition : isnotnull(ss_item_sk#4)
+
+(8) BroadcastExchange
+Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [d_date_sk#1]
+Right keys [1]: [ss_sold_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5]
+Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+
+(11) Scan parquet default.item
+Output [4]: [i_item_sk#8, i_brand_id#9, i_brand#10, i_manager_id#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_brand:string,i_manager_id:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [4]: [i_item_sk#8, i_brand_id#9, i_brand#10, i_manager_id#11]
+
+(13) Filter [codegen id : 2]
+Input [4]: [i_item_sk#8, i_brand_id#9, i_brand#10, i_manager_id#11]
+Condition : ((isnotnull(i_manager_id#11) AND (i_manager_id#11 = 1)) AND isnotnull(i_item_sk#8))
+
+(14) Project [codegen id : 2]
+Output [3]: [i_item_sk#8, i_brand_id#9, i_brand#10]
+Input [4]: [i_item_sk#8, i_brand_id#9, i_brand#10, i_manager_id#11]
+
+(15) BroadcastExchange
+Input [3]: [i_item_sk#8, i_brand_id#9, i_brand#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(16) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_item_sk#4]
+Right keys [1]: [i_item_sk#8]
+Join condition: None
+
+(17) Project [codegen id : 3]
+Output [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#9, i_brand#10]
+Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#8, i_brand_id#9, i_brand#10]
+
+(18) HashAggregate [codegen id : 3]
+Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#9, i_brand#10]
+Keys [3]: [d_year#2, i_brand#10, i_brand_id#9]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))]
+Aggregate Attributes [1]: [sum#13]
+Results [4]: [d_year#2, i_brand#10, i_brand_id#9, sum#14]
+
+(19) Exchange
+Input [4]: [d_year#2, i_brand#10, i_brand_id#9, sum#14]
+Arguments: hashpartitioning(d_year#2, i_brand#10, i_brand_id#9, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(20) HashAggregate [codegen id : 4]
+Input [4]: [d_year#2, i_brand#10, i_brand_id#9, sum#14]
+Keys [3]: [d_year#2, i_brand#10, i_brand_id#9]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#16]
+Results [4]: [d_year#2, i_brand_id#9 AS brand_id#17, i_brand#10 AS brand#18, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#16,17,2) AS ext_price#19]
+
+(21) TakeOrderedAndProject
+Input [4]: [d_year#2, brand_id#17, brand#18, ext_price#19]
+Arguments: 100, [d_year#2 ASC NULLS FIRST, ext_price#19 DESC NULLS LAST, brand_id#17 ASC NULLS FIRST], [d_year#2, brand_id#17, brand#18, ext_price#19]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q52/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q52/simplified.txt
new file mode 100644
index 0000000000000..d16dd603ec66a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q52/simplified.txt
@@ -0,0 +1,31 @@
+TakeOrderedAndProject [d_year,ext_price,brand_id,brand]
+  WholeStageCodegen (4)
+    HashAggregate [d_year,i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum]
+      InputAdapter
+        Exchange [d_year,i_brand,i_brand_id] #1
+          WholeStageCodegen (3)
+            HashAggregate [d_year,i_brand,i_brand_id,ss_ext_sales_price] [sum,sum]
+              Project [d_year,ss_ext_sales_price,i_brand_id,i_brand]
+                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                  Project [d_year,ss_item_sk,ss_ext_sales_price]
+                    BroadcastHashJoin [d_date_sk,ss_sold_date_sk]
+                      Project [d_date_sk,d_year]
+                        Filter [d_moy,d_year,d_date_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                      InputAdapter
+                        BroadcastExchange #2
+                          WholeStageCodegen (1)
+                            Filter [ss_item_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk]
+                  InputAdapter
+                    BroadcastExchange #3
+                      WholeStageCodegen (2)
+                        Project [i_item_sk,i_brand_id,i_brand]
+                          Filter [i_manager_id,i_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.item [i_item_sk,i_brand_id,i_brand,i_manager_id]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q53.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q53.sf100/explain.txt
new file mode 100644
index 0000000000000..a3223d60c96be
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q53.sf100/explain.txt
@@ -0,0 +1,191 @@
+== Physical Plan ==
+TakeOrderedAndProject (32)
++- * Project (31)
+   +- * Filter (30)
+      +- Window (29)
+         +- * Sort (28)
+            +- Exchange (27)
+               +- * HashAggregate (26)
+                  +- Exchange (25)
+                     +- * HashAggregate (24)
+                        +- * Project (23)
+                           +- * BroadcastHashJoin Inner BuildRight (22)
+                              :- * Project (16)
+                              :  +- * BroadcastHashJoin Inner BuildRight (15)
+                              :     :- * Project (10)
+                              :     :  +- * BroadcastHashJoin Inner BuildLeft (9)
+                              :     :     :- BroadcastExchange (5)
+                              :     :     :  +- * Project (4)
+                              :     :     :     +- * Filter (3)
+                              :     :     :        +- * ColumnarToRow (2)
+                              :     :     :           +- Scan parquet default.item (1)
+                              :     :     +- * Filter (8)
+                              :     :        +- * ColumnarToRow (7)
+                              :     :           +- Scan parquet default.store_sales (6)
+                              :     +- BroadcastExchange (14)
+                              :        +- * Filter (13)
+                              :           +- * ColumnarToRow (12)
+                              :              +- Scan parquet default.store (11)
+                              +- BroadcastExchange (21)
+                                 +- * Project (20)
+                                    +- * Filter (19)
+                                       +- * ColumnarToRow (18)
+                                          +- Scan parquet default.date_dim (17)
+
+
+(1) Scan parquet default.item
+Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [Or(And(And(In(i_category, [Books                                             ,Children                                          ,Electronics                                       ]),In(i_class, [personal                                          ,portable                                          ,reference                                         ,self-help                                         ])),In(i_brand, [scholaramalgamalg #6                             ,scholaramalgamalg #7                              ,exportiunivamalg #8                               ,scholaramalgamalg #8                              ])),And(And(In(i_category, [Women                                             ,Music                                             ,Men                                               ]),In(i_class, [accessories                                       ,classical                                         ,fragrances                                        ,pants                                             ])),In(i_brand, [amalgimporto #9                                   ,edu packscholar #9                                ,exportiimporto #9                                 ,importoamalg #9                                   ]))), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_class:string,i_category:string,i_manufact_id:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5]
+
+(3) Filter [codegen id : 1]
+Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5]
+Condition : ((((i_category#4 IN (Books                                             ,Children                                          ,Electronics                                       ) AND i_class#3 IN (personal                                          ,portable                                          ,reference                                         ,self-help                                         )) AND i_brand#2 IN (scholaramalgamalg #6                             ,scholaramalgamalg #7                              ,exportiunivamalg #8                               ,scholaramalgamalg #8                              )) OR ((i_category#4 IN (Women                                             ,Music                                             ,Men                                               ) AND i_class#3 IN (accessories                                       ,classical                                         ,fragrances                                        ,pants                                             )) AND i_brand#2 IN (amalgimporto #9                                   ,edu packscholar #9                                ,exportiimporto #9                                 ,importoamalg #9                                   ))) AND isnotnull(i_item_sk#1))
+
+(4) Project [codegen id : 1]
+Output [2]: [i_item_sk#1, i_manufact_id#5]
+Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5]
+
+(5) BroadcastExchange
+Input [2]: [i_item_sk#1, i_manufact_id#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(6) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#11, ss_store_sk#12, ss_sales_price#13, ss_sold_date_sk#14]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#14), (ss_sold_date_sk#14 >= 2451911), (ss_sold_date_sk#14 <= 2452275), dynamicpruningexpression(ss_sold_date_sk#14 IN dynamicpruning#15)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(7) ColumnarToRow
+Input [4]: [ss_item_sk#11, ss_store_sk#12, ss_sales_price#13, ss_sold_date_sk#14]
+
+(8) Filter
+Input [4]: [ss_item_sk#11, ss_store_sk#12, ss_sales_price#13, ss_sold_date_sk#14]
+Condition : (isnotnull(ss_item_sk#11) AND isnotnull(ss_store_sk#12))
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [ss_item_sk#11]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [4]: [i_manufact_id#5, ss_store_sk#12, ss_sales_price#13, ss_sold_date_sk#14]
+Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#11, ss_store_sk#12, ss_sales_price#13, ss_sold_date_sk#14]
+
+(11) Scan parquet default.store
+Output [1]: [s_store_sk#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [1]: [s_store_sk#16]
+
+(13) Filter [codegen id : 2]
+Input [1]: [s_store_sk#16]
+Condition : isnotnull(s_store_sk#16)
+
+(14) BroadcastExchange
+Input [1]: [s_store_sk#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#17]
+
+(15) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#12]
+Right keys [1]: [s_store_sk#16]
+Join condition: None
+
+(16) Project [codegen id : 4]
+Output [3]: [i_manufact_id#5, ss_sales_price#13, ss_sold_date_sk#14]
+Input [5]: [i_manufact_id#5, ss_store_sk#12, ss_sales_price#13, ss_sold_date_sk#14, s_store_sk#16]
+
+(17) Scan parquet default.date_dim
+Output [3]: [d_date_sk#18, d_month_seq#19, d_qoy#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [In(d_month_seq, [1222,1215,1223,1217,1214,1219,1213,1218,1220,1221,1216,1212]), GreaterThanOrEqual(d_date_sk,2451911), LessThanOrEqual(d_date_sk,2452275), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int,d_qoy:int>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#18, d_month_seq#19, d_qoy#20]
+
+(19) Filter [codegen id : 3]
+Input [3]: [d_date_sk#18, d_month_seq#19, d_qoy#20]
+Condition : (((d_month_seq#19 INSET (1222,1215,1223,1217,1214,1219,1213,1218,1220,1221,1216,1212) AND (d_date_sk#18 >= 2451911)) AND (d_date_sk#18 <= 2452275)) AND isnotnull(d_date_sk#18))
+
+(20) Project [codegen id : 3]
+Output [2]: [d_date_sk#18, d_qoy#20]
+Input [3]: [d_date_sk#18, d_month_seq#19, d_qoy#20]
+
+(21) BroadcastExchange
+Input [2]: [d_date_sk#18, d_qoy#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(22) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#14]
+Right keys [1]: [d_date_sk#18]
+Join condition: None
+
+(23) Project [codegen id : 4]
+Output [3]: [i_manufact_id#5, ss_sales_price#13, d_qoy#20]
+Input [5]: [i_manufact_id#5, ss_sales_price#13, ss_sold_date_sk#14, d_date_sk#18, d_qoy#20]
+
+(24) HashAggregate [codegen id : 4]
+Input [3]: [i_manufact_id#5, ss_sales_price#13, d_qoy#20]
+Keys [2]: [i_manufact_id#5, d_qoy#20]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#13))]
+Aggregate Attributes [1]: [sum#22]
+Results [3]: [i_manufact_id#5, d_qoy#20, sum#23]
+
+(25) Exchange
+Input [3]: [i_manufact_id#5, d_qoy#20, sum#23]
+Arguments: hashpartitioning(i_manufact_id#5, d_qoy#20, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(26) HashAggregate [codegen id : 5]
+Input [3]: [i_manufact_id#5, d_qoy#20, sum#23]
+Keys [2]: [i_manufact_id#5, d_qoy#20]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#13))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#13))#25]
+Results [3]: [i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#13))#25,17,2) AS sum_sales#26, MakeDecimal(sum(UnscaledValue(ss_sales_price#13))#25,17,2) AS _w0#27]
+
+(27) Exchange
+Input [3]: [i_manufact_id#5, sum_sales#26, _w0#27]
+Arguments: hashpartitioning(i_manufact_id#5, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(28) Sort [codegen id : 6]
+Input [3]: [i_manufact_id#5, sum_sales#26, _w0#27]
+Arguments: [i_manufact_id#5 ASC NULLS FIRST], false, 0
+
+(29) Window
+Input [3]: [i_manufact_id#5, sum_sales#26, _w0#27]
+Arguments: [avg(_w0#27) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#29], [i_manufact_id#5]
+
+(30) Filter [codegen id : 7]
+Input [4]: [i_manufact_id#5, sum_sales#26, _w0#27, avg_quarterly_sales#29]
+Condition : (isnotnull(avg_quarterly_sales#29) AND ((avg_quarterly_sales#29 > 0.000000) AND (CheckOverflow((promote_precision(abs(CheckOverflow((promote_precision(cast(sum_sales#26 as decimal(22,6))) - promote_precision(cast(avg_quarterly_sales#29 as decimal(22,6)))), DecimalType(22,6), true), false)) / promote_precision(cast(avg_quarterly_sales#29 as decimal(22,6)))), DecimalType(38,16), true) > 0.1000000000000000)))
+
+(31) Project [codegen id : 7]
+Output [3]: [i_manufact_id#5, sum_sales#26, avg_quarterly_sales#29]
+Input [4]: [i_manufact_id#5, sum_sales#26, _w0#27, avg_quarterly_sales#29]
+
+(32) TakeOrderedAndProject
+Input [3]: [i_manufact_id#5, sum_sales#26, avg_quarterly_sales#29]
+Arguments: 100, [avg_quarterly_sales#29 ASC NULLS FIRST, sum_sales#26 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#26, avg_quarterly_sales#29]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 6 Hosting Expression = ss_sold_date_sk#14 IN dynamicpruning#15
+ReusedExchange (33)
+
+
+(33) ReusedExchange [Reuses operator id: 21]
+Output [2]: [d_date_sk#18, d_qoy#20]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q53.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q53.sf100/simplified.txt
new file mode 100644
index 0000000000000..2e57ebbc0e801
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q53.sf100/simplified.txt
@@ -0,0 +1,51 @@
+TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id]
+  WholeStageCodegen (7)
+    Project [i_manufact_id,sum_sales,avg_quarterly_sales]
+      Filter [avg_quarterly_sales,sum_sales]
+        InputAdapter
+          Window [_w0,i_manufact_id]
+            WholeStageCodegen (6)
+              Sort [i_manufact_id]
+                InputAdapter
+                  Exchange [i_manufact_id] #1
+                    WholeStageCodegen (5)
+                      HashAggregate [i_manufact_id,d_qoy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum]
+                        InputAdapter
+                          Exchange [i_manufact_id,d_qoy] #2
+                            WholeStageCodegen (4)
+                              HashAggregate [i_manufact_id,d_qoy,ss_sales_price] [sum,sum]
+                                Project [i_manufact_id,ss_sales_price,d_qoy]
+                                  BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                    Project [i_manufact_id,ss_sales_price,ss_sold_date_sk]
+                                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                        Project [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                          BroadcastHashJoin [i_item_sk,ss_item_sk]
+                                            InputAdapter
+                                              BroadcastExchange #3
+                                                WholeStageCodegen (1)
+                                                  Project [i_item_sk,i_manufact_id]
+                                                    Filter [i_category,i_class,i_brand,i_item_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id]
+                                            Filter [ss_item_sk,ss_store_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #1
+                                                      ReusedExchange [d_date_sk,d_qoy] #4
+                                        InputAdapter
+                                          BroadcastExchange #5
+                                            WholeStageCodegen (2)
+                                              Filter [s_store_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.store [s_store_sk]
+                                    InputAdapter
+                                      BroadcastExchange #4
+                                        WholeStageCodegen (3)
+                                          Project [d_date_sk,d_qoy]
+                                            Filter [d_month_seq,d_date_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.date_dim [d_date_sk,d_month_seq,d_qoy]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q53/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q53/explain.txt
new file mode 100644
index 0000000000000..172e66022046d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q53/explain.txt
@@ -0,0 +1,191 @@
+== Physical Plan ==
+TakeOrderedAndProject (32)
++- * Project (31)
+   +- * Filter (30)
+      +- Window (29)
+         +- * Sort (28)
+            +- Exchange (27)
+               +- * HashAggregate (26)
+                  +- Exchange (25)
+                     +- * HashAggregate (24)
+                        +- * Project (23)
+                           +- * BroadcastHashJoin Inner BuildRight (22)
+                              :- * Project (17)
+                              :  +- * BroadcastHashJoin Inner BuildRight (16)
+                              :     :- * Project (10)
+                              :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+                              :     :     :- * Project (4)
+                              :     :     :  +- * Filter (3)
+                              :     :     :     +- * ColumnarToRow (2)
+                              :     :     :        +- Scan parquet default.item (1)
+                              :     :     +- BroadcastExchange (8)
+                              :     :        +- * Filter (7)
+                              :     :           +- * ColumnarToRow (6)
+                              :     :              +- Scan parquet default.store_sales (5)
+                              :     +- BroadcastExchange (15)
+                              :        +- * Project (14)
+                              :           +- * Filter (13)
+                              :              +- * ColumnarToRow (12)
+                              :                 +- Scan parquet default.date_dim (11)
+                              +- BroadcastExchange (21)
+                                 +- * Filter (20)
+                                    +- * ColumnarToRow (19)
+                                       +- Scan parquet default.store (18)
+
+
+(1) Scan parquet default.item
+Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [Or(And(And(In(i_category, [Books                                             ,Children                                          ,Electronics                                       ]),In(i_class, [personal                                          ,portable                                          ,reference                                         ,self-help                                         ])),In(i_brand, [scholaramalgamalg #6                             ,scholaramalgamalg #7                              ,exportiunivamalg #8                               ,scholaramalgamalg #8                              ])),And(And(In(i_category, [Women                                             ,Music                                             ,Men                                               ]),In(i_class, [accessories                                       ,classical                                         ,fragrances                                        ,pants                                             ])),In(i_brand, [amalgimporto #9                                   ,edu packscholar #9                                ,exportiimporto #9                                 ,importoamalg #9                                   ]))), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_class:string,i_category:string,i_manufact_id:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5]
+
+(3) Filter [codegen id : 4]
+Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5]
+Condition : ((((i_category#4 IN (Books                                             ,Children                                          ,Electronics                                       ) AND i_class#3 IN (personal                                          ,portable                                          ,reference                                         ,self-help                                         )) AND i_brand#2 IN (scholaramalgamalg #6                             ,scholaramalgamalg #7                              ,exportiunivamalg #8                               ,scholaramalgamalg #8                              )) OR ((i_category#4 IN (Women                                             ,Music                                             ,Men                                               ) AND i_class#3 IN (accessories                                       ,classical                                         ,fragrances                                        ,pants                                             )) AND i_brand#2 IN (amalgimporto #9                                   ,edu packscholar #9                                ,exportiimporto #9                                 ,importoamalg #9                                   ))) AND isnotnull(i_item_sk#1))
+
+(4) Project [codegen id : 4]
+Output [2]: [i_item_sk#1, i_manufact_id#5]
+Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5]
+
+(5) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#13), (ss_sold_date_sk#13 >= 2451911), (ss_sold_date_sk#13 <= 2452275), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13]
+
+(7) Filter [codegen id : 1]
+Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13]
+Condition : (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11))
+
+(8) BroadcastExchange
+Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#15]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [ss_item_sk#10]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13]
+Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13]
+
+(11) Scan parquet default.date_dim
+Output [3]: [d_date_sk#16, d_month_seq#17, d_qoy#18]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [In(d_month_seq, [1222,1215,1223,1217,1214,1219,1213,1218,1220,1221,1216,1212]), GreaterThanOrEqual(d_date_sk,2451911), LessThanOrEqual(d_date_sk,2452275), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int,d_qoy:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [d_date_sk#16, d_month_seq#17, d_qoy#18]
+
+(13) Filter [codegen id : 2]
+Input [3]: [d_date_sk#16, d_month_seq#17, d_qoy#18]
+Condition : (((d_month_seq#17 INSET (1222,1215,1223,1217,1214,1219,1213,1218,1220,1221,1216,1212) AND (d_date_sk#16 >= 2451911)) AND (d_date_sk#16 <= 2452275)) AND isnotnull(d_date_sk#16))
+
+(14) Project [codegen id : 2]
+Output [2]: [d_date_sk#16, d_qoy#18]
+Input [3]: [d_date_sk#16, d_month_seq#17, d_qoy#18]
+
+(15) BroadcastExchange
+Input [2]: [d_date_sk#16, d_qoy#18]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#19]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#13]
+Right keys [1]: [d_date_sk#16]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#18]
+Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#16, d_qoy#18]
+
+(18) Scan parquet default.store
+Output [1]: [s_store_sk#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [1]: [s_store_sk#20]
+
+(20) Filter [codegen id : 3]
+Input [1]: [s_store_sk#20]
+Condition : isnotnull(s_store_sk#20)
+
+(21) BroadcastExchange
+Input [1]: [s_store_sk#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#21]
+
+(22) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#11]
+Right keys [1]: [s_store_sk#20]
+Join condition: None
+
+(23) Project [codegen id : 4]
+Output [3]: [i_manufact_id#5, ss_sales_price#12, d_qoy#18]
+Input [5]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#18, s_store_sk#20]
+
+(24) HashAggregate [codegen id : 4]
+Input [3]: [i_manufact_id#5, ss_sales_price#12, d_qoy#18]
+Keys [2]: [i_manufact_id#5, d_qoy#18]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))]
+Aggregate Attributes [1]: [sum#22]
+Results [3]: [i_manufact_id#5, d_qoy#18, sum#23]
+
+(25) Exchange
+Input [3]: [i_manufact_id#5, d_qoy#18, sum#23]
+Arguments: hashpartitioning(i_manufact_id#5, d_qoy#18, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(26) HashAggregate [codegen id : 5]
+Input [3]: [i_manufact_id#5, d_qoy#18, sum#23]
+Keys [2]: [i_manufact_id#5, d_qoy#18]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#12))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#25]
+Results [3]: [i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#25,17,2) AS sum_sales#26, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#25,17,2) AS _w0#27]
+
+(27) Exchange
+Input [3]: [i_manufact_id#5, sum_sales#26, _w0#27]
+Arguments: hashpartitioning(i_manufact_id#5, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(28) Sort [codegen id : 6]
+Input [3]: [i_manufact_id#5, sum_sales#26, _w0#27]
+Arguments: [i_manufact_id#5 ASC NULLS FIRST], false, 0
+
+(29) Window
+Input [3]: [i_manufact_id#5, sum_sales#26, _w0#27]
+Arguments: [avg(_w0#27) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#29], [i_manufact_id#5]
+
+(30) Filter [codegen id : 7]
+Input [4]: [i_manufact_id#5, sum_sales#26, _w0#27, avg_quarterly_sales#29]
+Condition : (isnotnull(avg_quarterly_sales#29) AND ((avg_quarterly_sales#29 > 0.000000) AND (CheckOverflow((promote_precision(abs(CheckOverflow((promote_precision(cast(sum_sales#26 as decimal(22,6))) - promote_precision(cast(avg_quarterly_sales#29 as decimal(22,6)))), DecimalType(22,6), true), false)) / promote_precision(cast(avg_quarterly_sales#29 as decimal(22,6)))), DecimalType(38,16), true) > 0.1000000000000000)))
+
+(31) Project [codegen id : 7]
+Output [3]: [i_manufact_id#5, sum_sales#26, avg_quarterly_sales#29]
+Input [4]: [i_manufact_id#5, sum_sales#26, _w0#27, avg_quarterly_sales#29]
+
+(32) TakeOrderedAndProject
+Input [3]: [i_manufact_id#5, sum_sales#26, avg_quarterly_sales#29]
+Arguments: 100, [avg_quarterly_sales#29 ASC NULLS FIRST, sum_sales#26 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#26, avg_quarterly_sales#29]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 5 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14
+ReusedExchange (33)
+
+
+(33) ReusedExchange [Reuses operator id: 15]
+Output [2]: [d_date_sk#16, d_qoy#18]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q53/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q53/simplified.txt
new file mode 100644
index 0000000000000..d67def8b728c1
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q53/simplified.txt
@@ -0,0 +1,51 @@
+TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id]
+  WholeStageCodegen (7)
+    Project [i_manufact_id,sum_sales,avg_quarterly_sales]
+      Filter [avg_quarterly_sales,sum_sales]
+        InputAdapter
+          Window [_w0,i_manufact_id]
+            WholeStageCodegen (6)
+              Sort [i_manufact_id]
+                InputAdapter
+                  Exchange [i_manufact_id] #1
+                    WholeStageCodegen (5)
+                      HashAggregate [i_manufact_id,d_qoy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum]
+                        InputAdapter
+                          Exchange [i_manufact_id,d_qoy] #2
+                            WholeStageCodegen (4)
+                              HashAggregate [i_manufact_id,d_qoy,ss_sales_price] [sum,sum]
+                                Project [i_manufact_id,ss_sales_price,d_qoy]
+                                  BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                    Project [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy]
+                                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                        Project [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                          BroadcastHashJoin [i_item_sk,ss_item_sk]
+                                            Project [i_item_sk,i_manufact_id]
+                                              Filter [i_category,i_class,i_brand,i_item_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id]
+                                            InputAdapter
+                                              BroadcastExchange #3
+                                                WholeStageCodegen (1)
+                                                  Filter [ss_item_sk,ss_store_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                                          SubqueryBroadcast [d_date_sk] #1
+                                                            ReusedExchange [d_date_sk,d_qoy] #4
+                                        InputAdapter
+                                          BroadcastExchange #4
+                                            WholeStageCodegen (2)
+                                              Project [d_date_sk,d_qoy]
+                                                Filter [d_month_seq,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_month_seq,d_qoy]
+                                    InputAdapter
+                                      BroadcastExchange #5
+                                        WholeStageCodegen (3)
+                                          Filter [s_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store [s_store_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q55.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q55.sf100/explain.txt
new file mode 100644
index 0000000000000..df4d94f40bc16
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q55.sf100/explain.txt
@@ -0,0 +1,133 @@
+== Physical Plan ==
+TakeOrderedAndProject (21)
++- * HashAggregate (20)
+   +- Exchange (19)
+      +- * HashAggregate (18)
+         +- * Project (17)
+            +- * BroadcastHashJoin Inner BuildRight (16)
+               :- * Project (10)
+               :  +- * BroadcastHashJoin Inner BuildLeft (9)
+               :     :- BroadcastExchange (5)
+               :     :  +- * Project (4)
+               :     :     +- * Filter (3)
+               :     :        +- * ColumnarToRow (2)
+               :     :           +- Scan parquet default.date_dim (1)
+               :     +- * Filter (8)
+               :        +- * ColumnarToRow (7)
+               :           +- Scan parquet default.store_sales (6)
+               +- BroadcastExchange (15)
+                  +- * Project (14)
+                     +- * Filter (13)
+                        +- * ColumnarToRow (12)
+                           +- Scan parquet default.item (11)
+
+
+(1) Scan parquet default.date_dim
+Output [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,2001), GreaterThanOrEqual(d_date_sk,2452215), LessThanOrEqual(d_date_sk,2452244), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(3) Filter [codegen id : 1]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Condition : ((((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 2001)) AND (d_date_sk#1 >= 2452215)) AND (d_date_sk#1 <= 2452244)) AND isnotnull(d_date_sk#1))
+
+(4) Project [codegen id : 1]
+Output [1]: [d_date_sk#1]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(5) BroadcastExchange
+Input [1]: [d_date_sk#1]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#4]
+
+(6) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#5, ss_ext_sales_price#6, ss_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#7), (ss_sold_date_sk#7 >= 2452215), (ss_sold_date_sk#7 <= 2452244), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(7) ColumnarToRow
+Input [3]: [ss_item_sk#5, ss_ext_sales_price#6, ss_sold_date_sk#7]
+
+(8) Filter
+Input [3]: [ss_item_sk#5, ss_ext_sales_price#6, ss_sold_date_sk#7]
+Condition : isnotnull(ss_item_sk#5)
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [d_date_sk#1]
+Right keys [1]: [ss_sold_date_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [2]: [ss_item_sk#5, ss_ext_sales_price#6]
+Input [4]: [d_date_sk#1, ss_item_sk#5, ss_ext_sales_price#6, ss_sold_date_sk#7]
+
+(11) Scan parquet default.item
+Output [4]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manager_id#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,48), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_brand:string,i_manager_id:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [4]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manager_id#12]
+
+(13) Filter [codegen id : 2]
+Input [4]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manager_id#12]
+Condition : ((isnotnull(i_manager_id#12) AND (i_manager_id#12 = 48)) AND isnotnull(i_item_sk#9))
+
+(14) Project [codegen id : 2]
+Output [3]: [i_item_sk#9, i_brand_id#10, i_brand#11]
+Input [4]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manager_id#12]
+
+(15) BroadcastExchange
+Input [3]: [i_item_sk#9, i_brand_id#10, i_brand#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(16) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_item_sk#5]
+Right keys [1]: [i_item_sk#9]
+Join condition: None
+
+(17) Project [codegen id : 3]
+Output [3]: [ss_ext_sales_price#6, i_brand_id#10, i_brand#11]
+Input [5]: [ss_item_sk#5, ss_ext_sales_price#6, i_item_sk#9, i_brand_id#10, i_brand#11]
+
+(18) HashAggregate [codegen id : 3]
+Input [3]: [ss_ext_sales_price#6, i_brand_id#10, i_brand#11]
+Keys [2]: [i_brand#11, i_brand_id#10]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#6))]
+Aggregate Attributes [1]: [sum#14]
+Results [3]: [i_brand#11, i_brand_id#10, sum#15]
+
+(19) Exchange
+Input [3]: [i_brand#11, i_brand_id#10, sum#15]
+Arguments: hashpartitioning(i_brand#11, i_brand_id#10, 5), ENSURE_REQUIREMENTS, [id=#16]
+
+(20) HashAggregate [codegen id : 4]
+Input [3]: [i_brand#11, i_brand_id#10, sum#15]
+Keys [2]: [i_brand#11, i_brand_id#10]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#6))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#6))#17]
+Results [3]: [i_brand_id#10 AS brand_id#18, i_brand#11 AS brand#19, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#6))#17,17,2) AS ext_price#20]
+
+(21) TakeOrderedAndProject
+Input [3]: [brand_id#18, brand#19, ext_price#20]
+Arguments: 100, [ext_price#20 DESC NULLS LAST, brand_id#18 ASC NULLS FIRST], [brand_id#18, brand#19, ext_price#20]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 6 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (22)
+
+
+(22) ReusedExchange [Reuses operator id: 5]
+Output [1]: [d_date_sk#1]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q55.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q55.sf100/simplified.txt
new file mode 100644
index 0000000000000..48a1308dfc427
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q55.sf100/simplified.txt
@@ -0,0 +1,33 @@
+TakeOrderedAndProject [ext_price,brand_id,brand]
+  WholeStageCodegen (4)
+    HashAggregate [i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum]
+      InputAdapter
+        Exchange [i_brand,i_brand_id] #1
+          WholeStageCodegen (3)
+            HashAggregate [i_brand,i_brand_id,ss_ext_sales_price] [sum,sum]
+              Project [ss_ext_sales_price,i_brand_id,i_brand]
+                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                  Project [ss_item_sk,ss_ext_sales_price]
+                    BroadcastHashJoin [d_date_sk,ss_sold_date_sk]
+                      InputAdapter
+                        BroadcastExchange #2
+                          WholeStageCodegen (1)
+                            Project [d_date_sk]
+                              Filter [d_moy,d_year,d_date_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                      Filter [ss_item_sk]
+                        ColumnarToRow
+                          InputAdapter
+                            Scan parquet default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk]
+                              SubqueryBroadcast [d_date_sk] #1
+                                ReusedExchange [d_date_sk] #2
+                  InputAdapter
+                    BroadcastExchange #3
+                      WholeStageCodegen (2)
+                        Project [i_item_sk,i_brand_id,i_brand]
+                          Filter [i_manager_id,i_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.item [i_item_sk,i_brand_id,i_brand,i_manager_id]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q55/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q55/explain.txt
new file mode 100644
index 0000000000000..a10fc3256db33
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q55/explain.txt
@@ -0,0 +1,123 @@
+== Physical Plan ==
+TakeOrderedAndProject (21)
++- * HashAggregate (20)
+   +- Exchange (19)
+      +- * HashAggregate (18)
+         +- * Project (17)
+            +- * BroadcastHashJoin Inner BuildRight (16)
+               :- * Project (10)
+               :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :- * Project (4)
+               :     :  +- * Filter (3)
+               :     :     +- * ColumnarToRow (2)
+               :     :        +- Scan parquet default.date_dim (1)
+               :     +- BroadcastExchange (8)
+               :        +- * Filter (7)
+               :           +- * ColumnarToRow (6)
+               :              +- Scan parquet default.store_sales (5)
+               +- BroadcastExchange (15)
+                  +- * Project (14)
+                     +- * Filter (13)
+                        +- * ColumnarToRow (12)
+                           +- Scan parquet default.item (11)
+
+
+(1) Scan parquet default.date_dim
+Output [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,2001), GreaterThanOrEqual(d_date_sk,2452215), LessThanOrEqual(d_date_sk,2452244), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Condition : ((((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 2001)) AND (d_date_sk#1 >= 2452215)) AND (d_date_sk#1 <= 2452244)) AND isnotnull(d_date_sk#1))
+
+(4) Project [codegen id : 3]
+Output [1]: [d_date_sk#1]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(5) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#6), (ss_sold_date_sk#6 >= 2452215), (ss_sold_date_sk#6 <= 2452244), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+
+(7) Filter [codegen id : 1]
+Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Condition : isnotnull(ss_item_sk#4)
+
+(8) BroadcastExchange
+Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [d_date_sk#1]
+Right keys [1]: [ss_sold_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [2]: [ss_item_sk#4, ss_ext_sales_price#5]
+Input [4]: [d_date_sk#1, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+
+(11) Scan parquet default.item
+Output [4]: [i_item_sk#8, i_brand_id#9, i_brand#10, i_manager_id#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,48), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_brand:string,i_manager_id:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [4]: [i_item_sk#8, i_brand_id#9, i_brand#10, i_manager_id#11]
+
+(13) Filter [codegen id : 2]
+Input [4]: [i_item_sk#8, i_brand_id#9, i_brand#10, i_manager_id#11]
+Condition : ((isnotnull(i_manager_id#11) AND (i_manager_id#11 = 48)) AND isnotnull(i_item_sk#8))
+
+(14) Project [codegen id : 2]
+Output [3]: [i_item_sk#8, i_brand_id#9, i_brand#10]
+Input [4]: [i_item_sk#8, i_brand_id#9, i_brand#10, i_manager_id#11]
+
+(15) BroadcastExchange
+Input [3]: [i_item_sk#8, i_brand_id#9, i_brand#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(16) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_item_sk#4]
+Right keys [1]: [i_item_sk#8]
+Join condition: None
+
+(17) Project [codegen id : 3]
+Output [3]: [ss_ext_sales_price#5, i_brand_id#9, i_brand#10]
+Input [5]: [ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#8, i_brand_id#9, i_brand#10]
+
+(18) HashAggregate [codegen id : 3]
+Input [3]: [ss_ext_sales_price#5, i_brand_id#9, i_brand#10]
+Keys [2]: [i_brand#10, i_brand_id#9]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))]
+Aggregate Attributes [1]: [sum#13]
+Results [3]: [i_brand#10, i_brand_id#9, sum#14]
+
+(19) Exchange
+Input [3]: [i_brand#10, i_brand_id#9, sum#14]
+Arguments: hashpartitioning(i_brand#10, i_brand_id#9, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(20) HashAggregate [codegen id : 4]
+Input [3]: [i_brand#10, i_brand_id#9, sum#14]
+Keys [2]: [i_brand#10, i_brand_id#9]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#16]
+Results [3]: [i_brand_id#9 AS brand_id#17, i_brand#10 AS brand#18, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#16,17,2) AS ext_price#19]
+
+(21) TakeOrderedAndProject
+Input [3]: [brand_id#17, brand#18, ext_price#19]
+Arguments: 100, [ext_price#19 DESC NULLS LAST, brand_id#17 ASC NULLS FIRST], [brand_id#17, brand#18, ext_price#19]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q55/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q55/simplified.txt
new file mode 100644
index 0000000000000..9157bbec2b06a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q55/simplified.txt
@@ -0,0 +1,31 @@
+TakeOrderedAndProject [ext_price,brand_id,brand]
+  WholeStageCodegen (4)
+    HashAggregate [i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum]
+      InputAdapter
+        Exchange [i_brand,i_brand_id] #1
+          WholeStageCodegen (3)
+            HashAggregate [i_brand,i_brand_id,ss_ext_sales_price] [sum,sum]
+              Project [ss_ext_sales_price,i_brand_id,i_brand]
+                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                  Project [ss_item_sk,ss_ext_sales_price]
+                    BroadcastHashJoin [d_date_sk,ss_sold_date_sk]
+                      Project [d_date_sk]
+                        Filter [d_moy,d_year,d_date_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                      InputAdapter
+                        BroadcastExchange #2
+                          WholeStageCodegen (1)
+                            Filter [ss_item_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk]
+                  InputAdapter
+                    BroadcastExchange #3
+                      WholeStageCodegen (2)
+                        Project [i_item_sk,i_brand_id,i_brand]
+                          Filter [i_manager_id,i_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.item [i_item_sk,i_brand_id,i_brand,i_manager_id]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q59.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q59.sf100/explain.txt
new file mode 100644
index 0000000000000..d20f7cf4a2857
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q59.sf100/explain.txt
@@ -0,0 +1,292 @@
+== Physical Plan ==
+TakeOrderedAndProject (51)
++- * Project (50)
+   +- * BroadcastHashJoin Inner BuildRight (49)
+      :- * Project (25)
+      :  +- * BroadcastHashJoin Inner BuildRight (24)
+      :     :- * Project (18)
+      :     :  +- * BroadcastHashJoin Inner BuildRight (17)
+      :     :     :- * HashAggregate (12)
+      :     :     :  +- Exchange (11)
+      :     :     :     +- * HashAggregate (10)
+      :     :     :        +- * Project (9)
+      :     :     :           +- * BroadcastHashJoin Inner BuildRight (8)
+      :     :     :              :- * Filter (3)
+      :     :     :              :  +- * ColumnarToRow (2)
+      :     :     :              :     +- Scan parquet default.store_sales (1)
+      :     :     :              +- BroadcastExchange (7)
+      :     :     :                 +- * Filter (6)
+      :     :     :                    +- * ColumnarToRow (5)
+      :     :     :                       +- Scan parquet default.date_dim (4)
+      :     :     +- BroadcastExchange (16)
+      :     :        +- * Filter (15)
+      :     :           +- * ColumnarToRow (14)
+      :     :              +- Scan parquet default.store (13)
+      :     +- BroadcastExchange (23)
+      :        +- * Project (22)
+      :           +- * Filter (21)
+      :              +- * ColumnarToRow (20)
+      :                 +- Scan parquet default.date_dim (19)
+      +- BroadcastExchange (48)
+         +- * Project (47)
+            +- * BroadcastHashJoin Inner BuildRight (46)
+               :- * Project (40)
+               :  +- * BroadcastHashJoin Inner BuildRight (39)
+               :     :- * HashAggregate (34)
+               :     :  +- Exchange (33)
+               :     :     +- * HashAggregate (32)
+               :     :        +- * Project (31)
+               :     :           +- * BroadcastHashJoin Inner BuildRight (30)
+               :     :              :- * Filter (28)
+               :     :              :  +- * ColumnarToRow (27)
+               :     :              :     +- Scan parquet default.store_sales (26)
+               :     :              +- ReusedExchange (29)
+               :     +- BroadcastExchange (38)
+               :        +- * Filter (37)
+               :           +- * ColumnarToRow (36)
+               :              +- Scan parquet default.store (35)
+               +- BroadcastExchange (45)
+                  +- * Project (44)
+                     +- * Filter (43)
+                        +- * ColumnarToRow (42)
+                           +- Scan parquet default.date_dim (41)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 2]
+Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_store_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date_sk), IsNotNull(d_week_seq)]
+ReadSchema: struct<d_date_sk:int,d_week_seq:int,d_day_name:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6]
+Condition : (isnotnull(d_date_sk#4) AND isnotnull(d_week_seq#5))
+
+(7) BroadcastExchange
+Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#7]
+
+(8) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#4]
+Join condition: None
+
+(9) Project [codegen id : 2]
+Output [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6]
+Input [6]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_week_seq#5, d_day_name#6]
+
+(10) HashAggregate [codegen id : 2]
+Input [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6]
+Keys [2]: [d_week_seq#5, ss_store_sk#1]
+Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday   ) THEN ss_sales_price#2 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday   ) THEN ss_sales_price#2 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday  ) THEN ss_sales_price#2 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday   ) THEN ss_sales_price#2 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 ELSE null END))]
+Aggregate Attributes [7]: [sum#8, sum#9, sum#10, sum#11, sum#12, sum#13, sum#14]
+Results [9]: [d_week_seq#5, ss_store_sk#1, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20, sum#21]
+
+(11) Exchange
+Input [9]: [d_week_seq#5, ss_store_sk#1, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20, sum#21]
+Arguments: hashpartitioning(d_week_seq#5, ss_store_sk#1, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(12) HashAggregate [codegen id : 10]
+Input [9]: [d_week_seq#5, ss_store_sk#1, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20, sum#21]
+Keys [2]: [d_week_seq#5, ss_store_sk#1]
+Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday   ) THEN ss_sales_price#2 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday   ) THEN ss_sales_price#2 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday  ) THEN ss_sales_price#2 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday   ) THEN ss_sales_price#2 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 ELSE null END))]
+Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday   ) THEN ss_sales_price#2 ELSE null END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday   ) THEN ss_sales_price#2 ELSE null END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday  ) THEN ss_sales_price#2 ELSE null END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 ELSE null END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 ELSE null END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday   ) THEN ss_sales_price#2 ELSE null END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 ELSE null END))#29]
+Results [9]: [d_week_seq#5, ss_store_sk#1, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday   ) THEN ss_sales_price#2 ELSE null END))#23,17,2) AS sun_sales#30, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday   ) THEN ss_sales_price#2 ELSE null END))#24,17,2) AS mon_sales#31, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday  ) THEN ss_sales_price#2 ELSE null END))#25,17,2) AS tue_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 ELSE null END))#26,17,2) AS wed_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 ELSE null END))#27,17,2) AS thu_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday   ) THEN ss_sales_price#2 ELSE null END))#28,17,2) AS fri_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 ELSE null END))#29,17,2) AS sat_sales#36]
+
+(13) Scan parquet default.store
+Output [3]: [s_store_sk#37, s_store_id#38, s_store_name#39]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_id)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string,s_store_name:string>
+
+(14) ColumnarToRow [codegen id : 3]
+Input [3]: [s_store_sk#37, s_store_id#38, s_store_name#39]
+
+(15) Filter [codegen id : 3]
+Input [3]: [s_store_sk#37, s_store_id#38, s_store_name#39]
+Condition : (isnotnull(s_store_sk#37) AND isnotnull(s_store_id#38))
+
+(16) BroadcastExchange
+Input [3]: [s_store_sk#37, s_store_id#38, s_store_name#39]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#40]
+
+(17) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_store_sk#1]
+Right keys [1]: [s_store_sk#37]
+Join condition: None
+
+(18) Project [codegen id : 10]
+Output [10]: [d_week_seq#5, sun_sales#30, mon_sales#31, tue_sales#32, wed_sales#33, thu_sales#34, fri_sales#35, sat_sales#36, s_store_id#38, s_store_name#39]
+Input [12]: [d_week_seq#5, ss_store_sk#1, sun_sales#30, mon_sales#31, tue_sales#32, wed_sales#33, thu_sales#34, fri_sales#35, sat_sales#36, s_store_sk#37, s_store_id#38, s_store_name#39]
+
+(19) Scan parquet default.date_dim
+Output [2]: [d_month_seq#41, d_week_seq#42]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1185), LessThanOrEqual(d_month_seq,1196), IsNotNull(d_week_seq)]
+ReadSchema: struct<d_month_seq:int,d_week_seq:int>
+
+(20) ColumnarToRow [codegen id : 4]
+Input [2]: [d_month_seq#41, d_week_seq#42]
+
+(21) Filter [codegen id : 4]
+Input [2]: [d_month_seq#41, d_week_seq#42]
+Condition : (((isnotnull(d_month_seq#41) AND (d_month_seq#41 >= 1185)) AND (d_month_seq#41 <= 1196)) AND isnotnull(d_week_seq#42))
+
+(22) Project [codegen id : 4]
+Output [1]: [d_week_seq#42]
+Input [2]: [d_month_seq#41, d_week_seq#42]
+
+(23) BroadcastExchange
+Input [1]: [d_week_seq#42]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#43]
+
+(24) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [d_week_seq#5]
+Right keys [1]: [d_week_seq#42]
+Join condition: None
+
+(25) Project [codegen id : 10]
+Output [10]: [s_store_name#39 AS s_store_name1#44, d_week_seq#5 AS d_week_seq1#45, s_store_id#38 AS s_store_id1#46, sun_sales#30 AS sun_sales1#47, mon_sales#31 AS mon_sales1#48, tue_sales#32 AS tue_sales1#49, wed_sales#33 AS wed_sales1#50, thu_sales#34 AS thu_sales1#51, fri_sales#35 AS fri_sales1#52, sat_sales#36 AS sat_sales1#53]
+Input [11]: [d_week_seq#5, sun_sales#30, mon_sales#31, tue_sales#32, wed_sales#33, thu_sales#34, fri_sales#35, sat_sales#36, s_store_id#38, s_store_name#39, d_week_seq#42]
+
+(26) Scan parquet default.store_sales
+Output [3]: [ss_store_sk#54, ss_sales_price#55, ss_sold_date_sk#56]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#56)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(27) ColumnarToRow [codegen id : 6]
+Input [3]: [ss_store_sk#54, ss_sales_price#55, ss_sold_date_sk#56]
+
+(28) Filter [codegen id : 6]
+Input [3]: [ss_store_sk#54, ss_sales_price#55, ss_sold_date_sk#56]
+Condition : isnotnull(ss_store_sk#54)
+
+(29) ReusedExchange [Reuses operator id: 7]
+Output [3]: [d_date_sk#57, d_week_seq#58, d_day_name#59]
+
+(30) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_sold_date_sk#56]
+Right keys [1]: [d_date_sk#57]
+Join condition: None
+
+(31) Project [codegen id : 6]
+Output [4]: [ss_store_sk#54, ss_sales_price#55, d_week_seq#58, d_day_name#59]
+Input [6]: [ss_store_sk#54, ss_sales_price#55, ss_sold_date_sk#56, d_date_sk#57, d_week_seq#58, d_day_name#59]
+
+(32) HashAggregate [codegen id : 6]
+Input [4]: [ss_store_sk#54, ss_sales_price#55, d_week_seq#58, d_day_name#59]
+Keys [2]: [d_week_seq#58, ss_store_sk#54]
+Functions [6]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#59 = Sunday   ) THEN ss_sales_price#55 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#59 = Monday   ) THEN ss_sales_price#55 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#59 = Wednesday) THEN ss_sales_price#55 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#59 = Thursday ) THEN ss_sales_price#55 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#59 = Friday   ) THEN ss_sales_price#55 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#59 = Saturday ) THEN ss_sales_price#55 ELSE null END))]
+Aggregate Attributes [6]: [sum#60, sum#61, sum#62, sum#63, sum#64, sum#65]
+Results [8]: [d_week_seq#58, ss_store_sk#54, sum#66, sum#67, sum#68, sum#69, sum#70, sum#71]
+
+(33) Exchange
+Input [8]: [d_week_seq#58, ss_store_sk#54, sum#66, sum#67, sum#68, sum#69, sum#70, sum#71]
+Arguments: hashpartitioning(d_week_seq#58, ss_store_sk#54, 5), ENSURE_REQUIREMENTS, [id=#72]
+
+(34) HashAggregate [codegen id : 9]
+Input [8]: [d_week_seq#58, ss_store_sk#54, sum#66, sum#67, sum#68, sum#69, sum#70, sum#71]
+Keys [2]: [d_week_seq#58, ss_store_sk#54]
+Functions [6]: [sum(UnscaledValue(CASE WHEN (d_day_name#59 = Sunday   ) THEN ss_sales_price#55 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Monday   ) THEN ss_sales_price#55 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Wednesday) THEN ss_sales_price#55 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Thursday ) THEN ss_sales_price#55 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Friday   ) THEN ss_sales_price#55 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Saturday ) THEN ss_sales_price#55 ELSE null END))]
+Aggregate Attributes [6]: [sum(UnscaledValue(CASE WHEN (d_day_name#59 = Sunday   ) THEN ss_sales_price#55 ELSE null END))#73, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Monday   ) THEN ss_sales_price#55 ELSE null END))#74, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Wednesday) THEN ss_sales_price#55 ELSE null END))#75, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Thursday ) THEN ss_sales_price#55 ELSE null END))#76, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Friday   ) THEN ss_sales_price#55 ELSE null END))#77, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Saturday ) THEN ss_sales_price#55 ELSE null END))#78]
+Results [8]: [d_week_seq#58, ss_store_sk#54, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Sunday   ) THEN ss_sales_price#55 ELSE null END))#73,17,2) AS sun_sales#30, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Monday   ) THEN ss_sales_price#55 ELSE null END))#74,17,2) AS mon_sales#31, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Wednesday) THEN ss_sales_price#55 ELSE null END))#75,17,2) AS wed_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Thursday ) THEN ss_sales_price#55 ELSE null END))#76,17,2) AS thu_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Friday   ) THEN ss_sales_price#55 ELSE null END))#77,17,2) AS fri_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Saturday ) THEN ss_sales_price#55 ELSE null END))#78,17,2) AS sat_sales#36]
+
+(35) Scan parquet default.store
+Output [2]: [s_store_sk#79, s_store_id#80]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_id)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string>
+
+(36) ColumnarToRow [codegen id : 7]
+Input [2]: [s_store_sk#79, s_store_id#80]
+
+(37) Filter [codegen id : 7]
+Input [2]: [s_store_sk#79, s_store_id#80]
+Condition : (isnotnull(s_store_sk#79) AND isnotnull(s_store_id#80))
+
+(38) BroadcastExchange
+Input [2]: [s_store_sk#79, s_store_id#80]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#81]
+
+(39) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_store_sk#54]
+Right keys [1]: [s_store_sk#79]
+Join condition: None
+
+(40) Project [codegen id : 9]
+Output [8]: [d_week_seq#58, sun_sales#30, mon_sales#31, wed_sales#33, thu_sales#34, fri_sales#35, sat_sales#36, s_store_id#80]
+Input [10]: [d_week_seq#58, ss_store_sk#54, sun_sales#30, mon_sales#31, wed_sales#33, thu_sales#34, fri_sales#35, sat_sales#36, s_store_sk#79, s_store_id#80]
+
+(41) Scan parquet default.date_dim
+Output [2]: [d_month_seq#82, d_week_seq#83]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1197), LessThanOrEqual(d_month_seq,1208), IsNotNull(d_week_seq)]
+ReadSchema: struct<d_month_seq:int,d_week_seq:int>
+
+(42) ColumnarToRow [codegen id : 8]
+Input [2]: [d_month_seq#82, d_week_seq#83]
+
+(43) Filter [codegen id : 8]
+Input [2]: [d_month_seq#82, d_week_seq#83]
+Condition : (((isnotnull(d_month_seq#82) AND (d_month_seq#82 >= 1197)) AND (d_month_seq#82 <= 1208)) AND isnotnull(d_week_seq#83))
+
+(44) Project [codegen id : 8]
+Output [1]: [d_week_seq#83]
+Input [2]: [d_month_seq#82, d_week_seq#83]
+
+(45) BroadcastExchange
+Input [1]: [d_week_seq#83]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#84]
+
+(46) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [d_week_seq#58]
+Right keys [1]: [d_week_seq#83]
+Join condition: None
+
+(47) Project [codegen id : 9]
+Output [8]: [d_week_seq#58 AS d_week_seq2#85, s_store_id#80 AS s_store_id2#86, sun_sales#30 AS sun_sales2#87, mon_sales#31 AS mon_sales2#88, wed_sales#33 AS wed_sales2#89, thu_sales#34 AS thu_sales2#90, fri_sales#35 AS fri_sales2#91, sat_sales#36 AS sat_sales2#92]
+Input [9]: [d_week_seq#58, sun_sales#30, mon_sales#31, wed_sales#33, thu_sales#34, fri_sales#35, sat_sales#36, s_store_id#80, d_week_seq#83]
+
+(48) BroadcastExchange
+Input [8]: [d_week_seq2#85, s_store_id2#86, sun_sales2#87, mon_sales2#88, wed_sales2#89, thu_sales2#90, fri_sales2#91, sat_sales2#92]
+Arguments: HashedRelationBroadcastMode(List(input[1, string, true], (input[0, int, true] - 52)),false), [id=#93]
+
+(49) BroadcastHashJoin [codegen id : 10]
+Left keys [2]: [s_store_id1#46, d_week_seq1#45]
+Right keys [2]: [s_store_id2#86, (d_week_seq2#85 - 52)]
+Join condition: None
+
+(50) Project [codegen id : 10]
+Output [10]: [s_store_name1#44, s_store_id1#46, d_week_seq1#45, CheckOverflow((promote_precision(sun_sales1#47) / promote_precision(sun_sales2#87)), DecimalType(37,20), true) AS (sun_sales1 / sun_sales2)#94, CheckOverflow((promote_precision(mon_sales1#48) / promote_precision(mon_sales2#88)), DecimalType(37,20), true) AS (mon_sales1 / mon_sales2)#95, CheckOverflow((promote_precision(tue_sales1#49) / promote_precision(tue_sales1#49)), DecimalType(37,20), true) AS (tue_sales1 / tue_sales1)#96, CheckOverflow((promote_precision(wed_sales1#50) / promote_precision(wed_sales2#89)), DecimalType(37,20), true) AS (wed_sales1 / wed_sales2)#97, CheckOverflow((promote_precision(thu_sales1#51) / promote_precision(thu_sales2#90)), DecimalType(37,20), true) AS (thu_sales1 / thu_sales2)#98, CheckOverflow((promote_precision(fri_sales1#52) / promote_precision(fri_sales2#91)), DecimalType(37,20), true) AS (fri_sales1 / fri_sales2)#99, CheckOverflow((promote_precision(sat_sales1#53) / promote_precision(sat_sales2#92)), DecimalType(37,20), true) AS (sat_sales1 / sat_sales2)#100]
+Input [18]: [s_store_name1#44, d_week_seq1#45, s_store_id1#46, sun_sales1#47, mon_sales1#48, tue_sales1#49, wed_sales1#50, thu_sales1#51, fri_sales1#52, sat_sales1#53, d_week_seq2#85, s_store_id2#86, sun_sales2#87, mon_sales2#88, wed_sales2#89, thu_sales2#90, fri_sales2#91, sat_sales2#92]
+
+(51) TakeOrderedAndProject
+Input [10]: [s_store_name1#44, s_store_id1#46, d_week_seq1#45, (sun_sales1 / sun_sales2)#94, (mon_sales1 / mon_sales2)#95, (tue_sales1 / tue_sales1)#96, (wed_sales1 / wed_sales2)#97, (thu_sales1 / thu_sales2)#98, (fri_sales1 / fri_sales2)#99, (sat_sales1 / sat_sales2)#100]
+Arguments: 100, [s_store_name1#44 ASC NULLS FIRST, s_store_id1#46 ASC NULLS FIRST, d_week_seq1#45 ASC NULLS FIRST], [s_store_name1#44, s_store_id1#46, d_week_seq1#45, (sun_sales1 / sun_sales2)#94, (mon_sales1 / mon_sales2)#95, (tue_sales1 / tue_sales1)#96, (wed_sales1 / wed_sales2)#97, (thu_sales1 / thu_sales2)#98, (fri_sales1 / fri_sales2)#99, (sat_sales1 / sat_sales2)#100]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q59.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q59.sf100/simplified.txt
new file mode 100644
index 0000000000000..eae7970dd89a0
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q59.sf100/simplified.txt
@@ -0,0 +1,76 @@
+TakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_sales2),(mon_sales1 / mon_sales2),(tue_sales1 / tue_sales1),(wed_sales1 / wed_sales2),(thu_sales1 / thu_sales2),(fri_sales1 / fri_sales2),(sat_sales1 / sat_sales2)]
+  WholeStageCodegen (10)
+    Project [s_store_name1,s_store_id1,d_week_seq1,sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2]
+      BroadcastHashJoin [s_store_id1,d_week_seq1,s_store_id2,d_week_seq2]
+        Project [s_store_name,d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales]
+          BroadcastHashJoin [d_week_seq,d_week_seq]
+            Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,s_store_name]
+              BroadcastHashJoin [ss_store_sk,s_store_sk]
+                HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday  ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price ELSE null END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum]
+                  InputAdapter
+                    Exchange [d_week_seq,ss_store_sk] #1
+                      WholeStageCodegen (2)
+                        HashAggregate [d_week_seq,ss_store_sk,d_day_name,ss_sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum]
+                          Project [ss_store_sk,ss_sales_price,d_week_seq,d_day_name]
+                            BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                              Filter [ss_store_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                              InputAdapter
+                                BroadcastExchange #2
+                                  WholeStageCodegen (1)
+                                    Filter [d_date_sk,d_week_seq]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.date_dim [d_date_sk,d_week_seq,d_day_name]
+                InputAdapter
+                  BroadcastExchange #3
+                    WholeStageCodegen (3)
+                      Filter [s_store_sk,s_store_id]
+                        ColumnarToRow
+                          InputAdapter
+                            Scan parquet default.store [s_store_sk,s_store_id,s_store_name]
+            InputAdapter
+              BroadcastExchange #4
+                WholeStageCodegen (4)
+                  Project [d_week_seq]
+                    Filter [d_month_seq,d_week_seq]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.date_dim [d_month_seq,d_week_seq]
+        InputAdapter
+          BroadcastExchange #5
+            WholeStageCodegen (9)
+              Project [d_week_seq,s_store_id,sun_sales,mon_sales,wed_sales,thu_sales,fri_sales,sat_sales]
+                BroadcastHashJoin [d_week_seq,d_week_seq]
+                  Project [d_week_seq,sun_sales,mon_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id]
+                    BroadcastHashJoin [ss_store_sk,s_store_sk]
+                      HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price ELSE null END)),sun_sales,mon_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum]
+                        InputAdapter
+                          Exchange [d_week_seq,ss_store_sk] #6
+                            WholeStageCodegen (6)
+                              HashAggregate [d_week_seq,ss_store_sk,d_day_name,ss_sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum]
+                                Project [ss_store_sk,ss_sales_price,d_week_seq,d_day_name]
+                                  BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                    Filter [ss_store_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                    InputAdapter
+                                      ReusedExchange [d_date_sk,d_week_seq,d_day_name] #2
+                      InputAdapter
+                        BroadcastExchange #7
+                          WholeStageCodegen (7)
+                            Filter [s_store_sk,s_store_id]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.store [s_store_sk,s_store_id]
+                  InputAdapter
+                    BroadcastExchange #8
+                      WholeStageCodegen (8)
+                        Project [d_week_seq]
+                          Filter [d_month_seq,d_week_seq]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.date_dim [d_month_seq,d_week_seq]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q59/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q59/explain.txt
new file mode 100644
index 0000000000000..d20f7cf4a2857
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q59/explain.txt
@@ -0,0 +1,292 @@
+== Physical Plan ==
+TakeOrderedAndProject (51)
++- * Project (50)
+   +- * BroadcastHashJoin Inner BuildRight (49)
+      :- * Project (25)
+      :  +- * BroadcastHashJoin Inner BuildRight (24)
+      :     :- * Project (18)
+      :     :  +- * BroadcastHashJoin Inner BuildRight (17)
+      :     :     :- * HashAggregate (12)
+      :     :     :  +- Exchange (11)
+      :     :     :     +- * HashAggregate (10)
+      :     :     :        +- * Project (9)
+      :     :     :           +- * BroadcastHashJoin Inner BuildRight (8)
+      :     :     :              :- * Filter (3)
+      :     :     :              :  +- * ColumnarToRow (2)
+      :     :     :              :     +- Scan parquet default.store_sales (1)
+      :     :     :              +- BroadcastExchange (7)
+      :     :     :                 +- * Filter (6)
+      :     :     :                    +- * ColumnarToRow (5)
+      :     :     :                       +- Scan parquet default.date_dim (4)
+      :     :     +- BroadcastExchange (16)
+      :     :        +- * Filter (15)
+      :     :           +- * ColumnarToRow (14)
+      :     :              +- Scan parquet default.store (13)
+      :     +- BroadcastExchange (23)
+      :        +- * Project (22)
+      :           +- * Filter (21)
+      :              +- * ColumnarToRow (20)
+      :                 +- Scan parquet default.date_dim (19)
+      +- BroadcastExchange (48)
+         +- * Project (47)
+            +- * BroadcastHashJoin Inner BuildRight (46)
+               :- * Project (40)
+               :  +- * BroadcastHashJoin Inner BuildRight (39)
+               :     :- * HashAggregate (34)
+               :     :  +- Exchange (33)
+               :     :     +- * HashAggregate (32)
+               :     :        +- * Project (31)
+               :     :           +- * BroadcastHashJoin Inner BuildRight (30)
+               :     :              :- * Filter (28)
+               :     :              :  +- * ColumnarToRow (27)
+               :     :              :     +- Scan parquet default.store_sales (26)
+               :     :              +- ReusedExchange (29)
+               :     +- BroadcastExchange (38)
+               :        +- * Filter (37)
+               :           +- * ColumnarToRow (36)
+               :              +- Scan parquet default.store (35)
+               +- BroadcastExchange (45)
+                  +- * Project (44)
+                     +- * Filter (43)
+                        +- * ColumnarToRow (42)
+                           +- Scan parquet default.date_dim (41)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 2]
+Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_store_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date_sk), IsNotNull(d_week_seq)]
+ReadSchema: struct<d_date_sk:int,d_week_seq:int,d_day_name:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6]
+Condition : (isnotnull(d_date_sk#4) AND isnotnull(d_week_seq#5))
+
+(7) BroadcastExchange
+Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#7]
+
+(8) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#4]
+Join condition: None
+
+(9) Project [codegen id : 2]
+Output [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6]
+Input [6]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_week_seq#5, d_day_name#6]
+
+(10) HashAggregate [codegen id : 2]
+Input [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6]
+Keys [2]: [d_week_seq#5, ss_store_sk#1]
+Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday   ) THEN ss_sales_price#2 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday   ) THEN ss_sales_price#2 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday  ) THEN ss_sales_price#2 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday   ) THEN ss_sales_price#2 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 ELSE null END))]
+Aggregate Attributes [7]: [sum#8, sum#9, sum#10, sum#11, sum#12, sum#13, sum#14]
+Results [9]: [d_week_seq#5, ss_store_sk#1, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20, sum#21]
+
+(11) Exchange
+Input [9]: [d_week_seq#5, ss_store_sk#1, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20, sum#21]
+Arguments: hashpartitioning(d_week_seq#5, ss_store_sk#1, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(12) HashAggregate [codegen id : 10]
+Input [9]: [d_week_seq#5, ss_store_sk#1, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20, sum#21]
+Keys [2]: [d_week_seq#5, ss_store_sk#1]
+Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday   ) THEN ss_sales_price#2 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday   ) THEN ss_sales_price#2 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday  ) THEN ss_sales_price#2 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday   ) THEN ss_sales_price#2 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 ELSE null END))]
+Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday   ) THEN ss_sales_price#2 ELSE null END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday   ) THEN ss_sales_price#2 ELSE null END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday  ) THEN ss_sales_price#2 ELSE null END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 ELSE null END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 ELSE null END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday   ) THEN ss_sales_price#2 ELSE null END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 ELSE null END))#29]
+Results [9]: [d_week_seq#5, ss_store_sk#1, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday   ) THEN ss_sales_price#2 ELSE null END))#23,17,2) AS sun_sales#30, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday   ) THEN ss_sales_price#2 ELSE null END))#24,17,2) AS mon_sales#31, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday  ) THEN ss_sales_price#2 ELSE null END))#25,17,2) AS tue_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 ELSE null END))#26,17,2) AS wed_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 ELSE null END))#27,17,2) AS thu_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday   ) THEN ss_sales_price#2 ELSE null END))#28,17,2) AS fri_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 ELSE null END))#29,17,2) AS sat_sales#36]
+
+(13) Scan parquet default.store
+Output [3]: [s_store_sk#37, s_store_id#38, s_store_name#39]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_id)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string,s_store_name:string>
+
+(14) ColumnarToRow [codegen id : 3]
+Input [3]: [s_store_sk#37, s_store_id#38, s_store_name#39]
+
+(15) Filter [codegen id : 3]
+Input [3]: [s_store_sk#37, s_store_id#38, s_store_name#39]
+Condition : (isnotnull(s_store_sk#37) AND isnotnull(s_store_id#38))
+
+(16) BroadcastExchange
+Input [3]: [s_store_sk#37, s_store_id#38, s_store_name#39]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#40]
+
+(17) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_store_sk#1]
+Right keys [1]: [s_store_sk#37]
+Join condition: None
+
+(18) Project [codegen id : 10]
+Output [10]: [d_week_seq#5, sun_sales#30, mon_sales#31, tue_sales#32, wed_sales#33, thu_sales#34, fri_sales#35, sat_sales#36, s_store_id#38, s_store_name#39]
+Input [12]: [d_week_seq#5, ss_store_sk#1, sun_sales#30, mon_sales#31, tue_sales#32, wed_sales#33, thu_sales#34, fri_sales#35, sat_sales#36, s_store_sk#37, s_store_id#38, s_store_name#39]
+
+(19) Scan parquet default.date_dim
+Output [2]: [d_month_seq#41, d_week_seq#42]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1185), LessThanOrEqual(d_month_seq,1196), IsNotNull(d_week_seq)]
+ReadSchema: struct<d_month_seq:int,d_week_seq:int>
+
+(20) ColumnarToRow [codegen id : 4]
+Input [2]: [d_month_seq#41, d_week_seq#42]
+
+(21) Filter [codegen id : 4]
+Input [2]: [d_month_seq#41, d_week_seq#42]
+Condition : (((isnotnull(d_month_seq#41) AND (d_month_seq#41 >= 1185)) AND (d_month_seq#41 <= 1196)) AND isnotnull(d_week_seq#42))
+
+(22) Project [codegen id : 4]
+Output [1]: [d_week_seq#42]
+Input [2]: [d_month_seq#41, d_week_seq#42]
+
+(23) BroadcastExchange
+Input [1]: [d_week_seq#42]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#43]
+
+(24) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [d_week_seq#5]
+Right keys [1]: [d_week_seq#42]
+Join condition: None
+
+(25) Project [codegen id : 10]
+Output [10]: [s_store_name#39 AS s_store_name1#44, d_week_seq#5 AS d_week_seq1#45, s_store_id#38 AS s_store_id1#46, sun_sales#30 AS sun_sales1#47, mon_sales#31 AS mon_sales1#48, tue_sales#32 AS tue_sales1#49, wed_sales#33 AS wed_sales1#50, thu_sales#34 AS thu_sales1#51, fri_sales#35 AS fri_sales1#52, sat_sales#36 AS sat_sales1#53]
+Input [11]: [d_week_seq#5, sun_sales#30, mon_sales#31, tue_sales#32, wed_sales#33, thu_sales#34, fri_sales#35, sat_sales#36, s_store_id#38, s_store_name#39, d_week_seq#42]
+
+(26) Scan parquet default.store_sales
+Output [3]: [ss_store_sk#54, ss_sales_price#55, ss_sold_date_sk#56]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#56)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(27) ColumnarToRow [codegen id : 6]
+Input [3]: [ss_store_sk#54, ss_sales_price#55, ss_sold_date_sk#56]
+
+(28) Filter [codegen id : 6]
+Input [3]: [ss_store_sk#54, ss_sales_price#55, ss_sold_date_sk#56]
+Condition : isnotnull(ss_store_sk#54)
+
+(29) ReusedExchange [Reuses operator id: 7]
+Output [3]: [d_date_sk#57, d_week_seq#58, d_day_name#59]
+
+(30) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_sold_date_sk#56]
+Right keys [1]: [d_date_sk#57]
+Join condition: None
+
+(31) Project [codegen id : 6]
+Output [4]: [ss_store_sk#54, ss_sales_price#55, d_week_seq#58, d_day_name#59]
+Input [6]: [ss_store_sk#54, ss_sales_price#55, ss_sold_date_sk#56, d_date_sk#57, d_week_seq#58, d_day_name#59]
+
+(32) HashAggregate [codegen id : 6]
+Input [4]: [ss_store_sk#54, ss_sales_price#55, d_week_seq#58, d_day_name#59]
+Keys [2]: [d_week_seq#58, ss_store_sk#54]
+Functions [6]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#59 = Sunday   ) THEN ss_sales_price#55 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#59 = Monday   ) THEN ss_sales_price#55 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#59 = Wednesday) THEN ss_sales_price#55 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#59 = Thursday ) THEN ss_sales_price#55 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#59 = Friday   ) THEN ss_sales_price#55 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#59 = Saturday ) THEN ss_sales_price#55 ELSE null END))]
+Aggregate Attributes [6]: [sum#60, sum#61, sum#62, sum#63, sum#64, sum#65]
+Results [8]: [d_week_seq#58, ss_store_sk#54, sum#66, sum#67, sum#68, sum#69, sum#70, sum#71]
+
+(33) Exchange
+Input [8]: [d_week_seq#58, ss_store_sk#54, sum#66, sum#67, sum#68, sum#69, sum#70, sum#71]
+Arguments: hashpartitioning(d_week_seq#58, ss_store_sk#54, 5), ENSURE_REQUIREMENTS, [id=#72]
+
+(34) HashAggregate [codegen id : 9]
+Input [8]: [d_week_seq#58, ss_store_sk#54, sum#66, sum#67, sum#68, sum#69, sum#70, sum#71]
+Keys [2]: [d_week_seq#58, ss_store_sk#54]
+Functions [6]: [sum(UnscaledValue(CASE WHEN (d_day_name#59 = Sunday   ) THEN ss_sales_price#55 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Monday   ) THEN ss_sales_price#55 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Wednesday) THEN ss_sales_price#55 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Thursday ) THEN ss_sales_price#55 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Friday   ) THEN ss_sales_price#55 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Saturday ) THEN ss_sales_price#55 ELSE null END))]
+Aggregate Attributes [6]: [sum(UnscaledValue(CASE WHEN (d_day_name#59 = Sunday   ) THEN ss_sales_price#55 ELSE null END))#73, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Monday   ) THEN ss_sales_price#55 ELSE null END))#74, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Wednesday) THEN ss_sales_price#55 ELSE null END))#75, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Thursday ) THEN ss_sales_price#55 ELSE null END))#76, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Friday   ) THEN ss_sales_price#55 ELSE null END))#77, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Saturday ) THEN ss_sales_price#55 ELSE null END))#78]
+Results [8]: [d_week_seq#58, ss_store_sk#54, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Sunday   ) THEN ss_sales_price#55 ELSE null END))#73,17,2) AS sun_sales#30, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Monday   ) THEN ss_sales_price#55 ELSE null END))#74,17,2) AS mon_sales#31, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Wednesday) THEN ss_sales_price#55 ELSE null END))#75,17,2) AS wed_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Thursday ) THEN ss_sales_price#55 ELSE null END))#76,17,2) AS thu_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Friday   ) THEN ss_sales_price#55 ELSE null END))#77,17,2) AS fri_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Saturday ) THEN ss_sales_price#55 ELSE null END))#78,17,2) AS sat_sales#36]
+
+(35) Scan parquet default.store
+Output [2]: [s_store_sk#79, s_store_id#80]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_id)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string>
+
+(36) ColumnarToRow [codegen id : 7]
+Input [2]: [s_store_sk#79, s_store_id#80]
+
+(37) Filter [codegen id : 7]
+Input [2]: [s_store_sk#79, s_store_id#80]
+Condition : (isnotnull(s_store_sk#79) AND isnotnull(s_store_id#80))
+
+(38) BroadcastExchange
+Input [2]: [s_store_sk#79, s_store_id#80]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#81]
+
+(39) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_store_sk#54]
+Right keys [1]: [s_store_sk#79]
+Join condition: None
+
+(40) Project [codegen id : 9]
+Output [8]: [d_week_seq#58, sun_sales#30, mon_sales#31, wed_sales#33, thu_sales#34, fri_sales#35, sat_sales#36, s_store_id#80]
+Input [10]: [d_week_seq#58, ss_store_sk#54, sun_sales#30, mon_sales#31, wed_sales#33, thu_sales#34, fri_sales#35, sat_sales#36, s_store_sk#79, s_store_id#80]
+
+(41) Scan parquet default.date_dim
+Output [2]: [d_month_seq#82, d_week_seq#83]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1197), LessThanOrEqual(d_month_seq,1208), IsNotNull(d_week_seq)]
+ReadSchema: struct<d_month_seq:int,d_week_seq:int>
+
+(42) ColumnarToRow [codegen id : 8]
+Input [2]: [d_month_seq#82, d_week_seq#83]
+
+(43) Filter [codegen id : 8]
+Input [2]: [d_month_seq#82, d_week_seq#83]
+Condition : (((isnotnull(d_month_seq#82) AND (d_month_seq#82 >= 1197)) AND (d_month_seq#82 <= 1208)) AND isnotnull(d_week_seq#83))
+
+(44) Project [codegen id : 8]
+Output [1]: [d_week_seq#83]
+Input [2]: [d_month_seq#82, d_week_seq#83]
+
+(45) BroadcastExchange
+Input [1]: [d_week_seq#83]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#84]
+
+(46) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [d_week_seq#58]
+Right keys [1]: [d_week_seq#83]
+Join condition: None
+
+(47) Project [codegen id : 9]
+Output [8]: [d_week_seq#58 AS d_week_seq2#85, s_store_id#80 AS s_store_id2#86, sun_sales#30 AS sun_sales2#87, mon_sales#31 AS mon_sales2#88, wed_sales#33 AS wed_sales2#89, thu_sales#34 AS thu_sales2#90, fri_sales#35 AS fri_sales2#91, sat_sales#36 AS sat_sales2#92]
+Input [9]: [d_week_seq#58, sun_sales#30, mon_sales#31, wed_sales#33, thu_sales#34, fri_sales#35, sat_sales#36, s_store_id#80, d_week_seq#83]
+
+(48) BroadcastExchange
+Input [8]: [d_week_seq2#85, s_store_id2#86, sun_sales2#87, mon_sales2#88, wed_sales2#89, thu_sales2#90, fri_sales2#91, sat_sales2#92]
+Arguments: HashedRelationBroadcastMode(List(input[1, string, true], (input[0, int, true] - 52)),false), [id=#93]
+
+(49) BroadcastHashJoin [codegen id : 10]
+Left keys [2]: [s_store_id1#46, d_week_seq1#45]
+Right keys [2]: [s_store_id2#86, (d_week_seq2#85 - 52)]
+Join condition: None
+
+(50) Project [codegen id : 10]
+Output [10]: [s_store_name1#44, s_store_id1#46, d_week_seq1#45, CheckOverflow((promote_precision(sun_sales1#47) / promote_precision(sun_sales2#87)), DecimalType(37,20), true) AS (sun_sales1 / sun_sales2)#94, CheckOverflow((promote_precision(mon_sales1#48) / promote_precision(mon_sales2#88)), DecimalType(37,20), true) AS (mon_sales1 / mon_sales2)#95, CheckOverflow((promote_precision(tue_sales1#49) / promote_precision(tue_sales1#49)), DecimalType(37,20), true) AS (tue_sales1 / tue_sales1)#96, CheckOverflow((promote_precision(wed_sales1#50) / promote_precision(wed_sales2#89)), DecimalType(37,20), true) AS (wed_sales1 / wed_sales2)#97, CheckOverflow((promote_precision(thu_sales1#51) / promote_precision(thu_sales2#90)), DecimalType(37,20), true) AS (thu_sales1 / thu_sales2)#98, CheckOverflow((promote_precision(fri_sales1#52) / promote_precision(fri_sales2#91)), DecimalType(37,20), true) AS (fri_sales1 / fri_sales2)#99, CheckOverflow((promote_precision(sat_sales1#53) / promote_precision(sat_sales2#92)), DecimalType(37,20), true) AS (sat_sales1 / sat_sales2)#100]
+Input [18]: [s_store_name1#44, d_week_seq1#45, s_store_id1#46, sun_sales1#47, mon_sales1#48, tue_sales1#49, wed_sales1#50, thu_sales1#51, fri_sales1#52, sat_sales1#53, d_week_seq2#85, s_store_id2#86, sun_sales2#87, mon_sales2#88, wed_sales2#89, thu_sales2#90, fri_sales2#91, sat_sales2#92]
+
+(51) TakeOrderedAndProject
+Input [10]: [s_store_name1#44, s_store_id1#46, d_week_seq1#45, (sun_sales1 / sun_sales2)#94, (mon_sales1 / mon_sales2)#95, (tue_sales1 / tue_sales1)#96, (wed_sales1 / wed_sales2)#97, (thu_sales1 / thu_sales2)#98, (fri_sales1 / fri_sales2)#99, (sat_sales1 / sat_sales2)#100]
+Arguments: 100, [s_store_name1#44 ASC NULLS FIRST, s_store_id1#46 ASC NULLS FIRST, d_week_seq1#45 ASC NULLS FIRST], [s_store_name1#44, s_store_id1#46, d_week_seq1#45, (sun_sales1 / sun_sales2)#94, (mon_sales1 / mon_sales2)#95, (tue_sales1 / tue_sales1)#96, (wed_sales1 / wed_sales2)#97, (thu_sales1 / thu_sales2)#98, (fri_sales1 / fri_sales2)#99, (sat_sales1 / sat_sales2)#100]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q59/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q59/simplified.txt
new file mode 100644
index 0000000000000..eae7970dd89a0
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q59/simplified.txt
@@ -0,0 +1,76 @@
+TakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_sales2),(mon_sales1 / mon_sales2),(tue_sales1 / tue_sales1),(wed_sales1 / wed_sales2),(thu_sales1 / thu_sales2),(fri_sales1 / fri_sales2),(sat_sales1 / sat_sales2)]
+  WholeStageCodegen (10)
+    Project [s_store_name1,s_store_id1,d_week_seq1,sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2]
+      BroadcastHashJoin [s_store_id1,d_week_seq1,s_store_id2,d_week_seq2]
+        Project [s_store_name,d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales]
+          BroadcastHashJoin [d_week_seq,d_week_seq]
+            Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,s_store_name]
+              BroadcastHashJoin [ss_store_sk,s_store_sk]
+                HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday  ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price ELSE null END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum]
+                  InputAdapter
+                    Exchange [d_week_seq,ss_store_sk] #1
+                      WholeStageCodegen (2)
+                        HashAggregate [d_week_seq,ss_store_sk,d_day_name,ss_sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum]
+                          Project [ss_store_sk,ss_sales_price,d_week_seq,d_day_name]
+                            BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                              Filter [ss_store_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                              InputAdapter
+                                BroadcastExchange #2
+                                  WholeStageCodegen (1)
+                                    Filter [d_date_sk,d_week_seq]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.date_dim [d_date_sk,d_week_seq,d_day_name]
+                InputAdapter
+                  BroadcastExchange #3
+                    WholeStageCodegen (3)
+                      Filter [s_store_sk,s_store_id]
+                        ColumnarToRow
+                          InputAdapter
+                            Scan parquet default.store [s_store_sk,s_store_id,s_store_name]
+            InputAdapter
+              BroadcastExchange #4
+                WholeStageCodegen (4)
+                  Project [d_week_seq]
+                    Filter [d_month_seq,d_week_seq]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.date_dim [d_month_seq,d_week_seq]
+        InputAdapter
+          BroadcastExchange #5
+            WholeStageCodegen (9)
+              Project [d_week_seq,s_store_id,sun_sales,mon_sales,wed_sales,thu_sales,fri_sales,sat_sales]
+                BroadcastHashJoin [d_week_seq,d_week_seq]
+                  Project [d_week_seq,sun_sales,mon_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id]
+                    BroadcastHashJoin [ss_store_sk,s_store_sk]
+                      HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price ELSE null END)),sun_sales,mon_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum]
+                        InputAdapter
+                          Exchange [d_week_seq,ss_store_sk] #6
+                            WholeStageCodegen (6)
+                              HashAggregate [d_week_seq,ss_store_sk,d_day_name,ss_sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum]
+                                Project [ss_store_sk,ss_sales_price,d_week_seq,d_day_name]
+                                  BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                    Filter [ss_store_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                    InputAdapter
+                                      ReusedExchange [d_date_sk,d_week_seq,d_day_name] #2
+                      InputAdapter
+                        BroadcastExchange #7
+                          WholeStageCodegen (7)
+                            Filter [s_store_sk,s_store_id]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.store [s_store_sk,s_store_id]
+                  InputAdapter
+                    BroadcastExchange #8
+                      WholeStageCodegen (8)
+                        Project [d_week_seq]
+                          Filter [d_month_seq,d_week_seq]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.date_dim [d_month_seq,d_week_seq]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q63.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q63.sf100/explain.txt
new file mode 100644
index 0000000000000..75cdaecea7595
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q63.sf100/explain.txt
@@ -0,0 +1,191 @@
+== Physical Plan ==
+TakeOrderedAndProject (32)
++- * Project (31)
+   +- * Filter (30)
+      +- Window (29)
+         +- * Sort (28)
+            +- Exchange (27)
+               +- * HashAggregate (26)
+                  +- Exchange (25)
+                     +- * HashAggregate (24)
+                        +- * Project (23)
+                           +- * BroadcastHashJoin Inner BuildRight (22)
+                              :- * Project (16)
+                              :  +- * BroadcastHashJoin Inner BuildRight (15)
+                              :     :- * Project (10)
+                              :     :  +- * BroadcastHashJoin Inner BuildLeft (9)
+                              :     :     :- BroadcastExchange (5)
+                              :     :     :  +- * Project (4)
+                              :     :     :     +- * Filter (3)
+                              :     :     :        +- * ColumnarToRow (2)
+                              :     :     :           +- Scan parquet default.item (1)
+                              :     :     +- * Filter (8)
+                              :     :        +- * ColumnarToRow (7)
+                              :     :           +- Scan parquet default.store_sales (6)
+                              :     +- BroadcastExchange (14)
+                              :        +- * Filter (13)
+                              :           +- * ColumnarToRow (12)
+                              :              +- Scan parquet default.store (11)
+                              +- BroadcastExchange (21)
+                                 +- * Project (20)
+                                    +- * Filter (19)
+                                       +- * ColumnarToRow (18)
+                                          +- Scan parquet default.date_dim (17)
+
+
+(1) Scan parquet default.item
+Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [Or(And(And(In(i_category, [Books                                             ,Children                                          ,Electronics                                       ]),In(i_class, [personal                                          ,portable                                          ,reference                                         ,self-help                                         ])),In(i_brand, [scholaramalgamalg #6                             ,scholaramalgamalg #7                              ,exportiunivamalg #8                               ,scholaramalgamalg #8                              ])),And(And(In(i_category, [Women                                             ,Music                                             ,Men                                               ]),In(i_class, [accessories                                       ,classical                                         ,fragrances                                        ,pants                                             ])),In(i_brand, [amalgimporto #9                                   ,edu packscholar #9                                ,exportiimporto #9                                 ,importoamalg #9                                   ]))), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_class:string,i_category:string,i_manager_id:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5]
+
+(3) Filter [codegen id : 1]
+Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5]
+Condition : ((((i_category#4 IN (Books                                             ,Children                                          ,Electronics                                       ) AND i_class#3 IN (personal                                          ,portable                                          ,reference                                         ,self-help                                         )) AND i_brand#2 IN (scholaramalgamalg #6                             ,scholaramalgamalg #7                              ,exportiunivamalg #8                               ,scholaramalgamalg #8                              )) OR ((i_category#4 IN (Women                                             ,Music                                             ,Men                                               ) AND i_class#3 IN (accessories                                       ,classical                                         ,fragrances                                        ,pants                                             )) AND i_brand#2 IN (amalgimporto #9                                   ,edu packscholar #9                                ,exportiimporto #9                                 ,importoamalg #9                                   ))) AND isnotnull(i_item_sk#1))
+
+(4) Project [codegen id : 1]
+Output [2]: [i_item_sk#1, i_manager_id#5]
+Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5]
+
+(5) BroadcastExchange
+Input [2]: [i_item_sk#1, i_manager_id#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(6) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#11, ss_store_sk#12, ss_sales_price#13, ss_sold_date_sk#14]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#14), (ss_sold_date_sk#14 >= 2452123), (ss_sold_date_sk#14 <= 2452487), dynamicpruningexpression(ss_sold_date_sk#14 IN dynamicpruning#15)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(7) ColumnarToRow
+Input [4]: [ss_item_sk#11, ss_store_sk#12, ss_sales_price#13, ss_sold_date_sk#14]
+
+(8) Filter
+Input [4]: [ss_item_sk#11, ss_store_sk#12, ss_sales_price#13, ss_sold_date_sk#14]
+Condition : (isnotnull(ss_item_sk#11) AND isnotnull(ss_store_sk#12))
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [ss_item_sk#11]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [4]: [i_manager_id#5, ss_store_sk#12, ss_sales_price#13, ss_sold_date_sk#14]
+Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#11, ss_store_sk#12, ss_sales_price#13, ss_sold_date_sk#14]
+
+(11) Scan parquet default.store
+Output [1]: [s_store_sk#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [1]: [s_store_sk#16]
+
+(13) Filter [codegen id : 2]
+Input [1]: [s_store_sk#16]
+Condition : isnotnull(s_store_sk#16)
+
+(14) BroadcastExchange
+Input [1]: [s_store_sk#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#17]
+
+(15) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#12]
+Right keys [1]: [s_store_sk#16]
+Join condition: None
+
+(16) Project [codegen id : 4]
+Output [3]: [i_manager_id#5, ss_sales_price#13, ss_sold_date_sk#14]
+Input [5]: [i_manager_id#5, ss_store_sk#12, ss_sales_price#13, ss_sold_date_sk#14, s_store_sk#16]
+
+(17) Scan parquet default.date_dim
+Output [3]: [d_date_sk#18, d_month_seq#19, d_moy#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [In(d_month_seq, [1222,1228,1223,1227,1219,1226,1224,1225,1230,1220,1221,1229]), GreaterThanOrEqual(d_date_sk,2452123), LessThanOrEqual(d_date_sk,2452487), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int,d_moy:int>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#18, d_month_seq#19, d_moy#20]
+
+(19) Filter [codegen id : 3]
+Input [3]: [d_date_sk#18, d_month_seq#19, d_moy#20]
+Condition : (((d_month_seq#19 INSET (1222,1228,1223,1227,1219,1226,1224,1225,1230,1220,1221,1229) AND (d_date_sk#18 >= 2452123)) AND (d_date_sk#18 <= 2452487)) AND isnotnull(d_date_sk#18))
+
+(20) Project [codegen id : 3]
+Output [2]: [d_date_sk#18, d_moy#20]
+Input [3]: [d_date_sk#18, d_month_seq#19, d_moy#20]
+
+(21) BroadcastExchange
+Input [2]: [d_date_sk#18, d_moy#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(22) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#14]
+Right keys [1]: [d_date_sk#18]
+Join condition: None
+
+(23) Project [codegen id : 4]
+Output [3]: [i_manager_id#5, ss_sales_price#13, d_moy#20]
+Input [5]: [i_manager_id#5, ss_sales_price#13, ss_sold_date_sk#14, d_date_sk#18, d_moy#20]
+
+(24) HashAggregate [codegen id : 4]
+Input [3]: [i_manager_id#5, ss_sales_price#13, d_moy#20]
+Keys [2]: [i_manager_id#5, d_moy#20]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#13))]
+Aggregate Attributes [1]: [sum#22]
+Results [3]: [i_manager_id#5, d_moy#20, sum#23]
+
+(25) Exchange
+Input [3]: [i_manager_id#5, d_moy#20, sum#23]
+Arguments: hashpartitioning(i_manager_id#5, d_moy#20, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(26) HashAggregate [codegen id : 5]
+Input [3]: [i_manager_id#5, d_moy#20, sum#23]
+Keys [2]: [i_manager_id#5, d_moy#20]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#13))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#13))#25]
+Results [3]: [i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#13))#25,17,2) AS sum_sales#26, MakeDecimal(sum(UnscaledValue(ss_sales_price#13))#25,17,2) AS _w0#27]
+
+(27) Exchange
+Input [3]: [i_manager_id#5, sum_sales#26, _w0#27]
+Arguments: hashpartitioning(i_manager_id#5, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(28) Sort [codegen id : 6]
+Input [3]: [i_manager_id#5, sum_sales#26, _w0#27]
+Arguments: [i_manager_id#5 ASC NULLS FIRST], false, 0
+
+(29) Window
+Input [3]: [i_manager_id#5, sum_sales#26, _w0#27]
+Arguments: [avg(_w0#27) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#29], [i_manager_id#5]
+
+(30) Filter [codegen id : 7]
+Input [4]: [i_manager_id#5, sum_sales#26, _w0#27, avg_monthly_sales#29]
+Condition : (isnotnull(avg_monthly_sales#29) AND ((avg_monthly_sales#29 > 0.000000) AND (CheckOverflow((promote_precision(abs(CheckOverflow((promote_precision(cast(sum_sales#26 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#29 as decimal(22,6)))), DecimalType(22,6), true), false)) / promote_precision(cast(avg_monthly_sales#29 as decimal(22,6)))), DecimalType(38,16), true) > 0.1000000000000000)))
+
+(31) Project [codegen id : 7]
+Output [3]: [i_manager_id#5, sum_sales#26, avg_monthly_sales#29]
+Input [4]: [i_manager_id#5, sum_sales#26, _w0#27, avg_monthly_sales#29]
+
+(32) TakeOrderedAndProject
+Input [3]: [i_manager_id#5, sum_sales#26, avg_monthly_sales#29]
+Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#29 ASC NULLS FIRST, sum_sales#26 ASC NULLS FIRST], [i_manager_id#5, sum_sales#26, avg_monthly_sales#29]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 6 Hosting Expression = ss_sold_date_sk#14 IN dynamicpruning#15
+ReusedExchange (33)
+
+
+(33) ReusedExchange [Reuses operator id: 21]
+Output [2]: [d_date_sk#18, d_moy#20]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q63.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q63.sf100/simplified.txt
new file mode 100644
index 0000000000000..d8e6d0dcef768
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q63.sf100/simplified.txt
@@ -0,0 +1,51 @@
+TakeOrderedAndProject [i_manager_id,avg_monthly_sales,sum_sales]
+  WholeStageCodegen (7)
+    Project [i_manager_id,sum_sales,avg_monthly_sales]
+      Filter [avg_monthly_sales,sum_sales]
+        InputAdapter
+          Window [_w0,i_manager_id]
+            WholeStageCodegen (6)
+              Sort [i_manager_id]
+                InputAdapter
+                  Exchange [i_manager_id] #1
+                    WholeStageCodegen (5)
+                      HashAggregate [i_manager_id,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum]
+                        InputAdapter
+                          Exchange [i_manager_id,d_moy] #2
+                            WholeStageCodegen (4)
+                              HashAggregate [i_manager_id,d_moy,ss_sales_price] [sum,sum]
+                                Project [i_manager_id,ss_sales_price,d_moy]
+                                  BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                    Project [i_manager_id,ss_sales_price,ss_sold_date_sk]
+                                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                        Project [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                          BroadcastHashJoin [i_item_sk,ss_item_sk]
+                                            InputAdapter
+                                              BroadcastExchange #3
+                                                WholeStageCodegen (1)
+                                                  Project [i_item_sk,i_manager_id]
+                                                    Filter [i_category,i_class,i_brand,i_item_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.item [i_item_sk,i_brand,i_class,i_category,i_manager_id]
+                                            Filter [ss_item_sk,ss_store_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #1
+                                                      ReusedExchange [d_date_sk,d_moy] #4
+                                        InputAdapter
+                                          BroadcastExchange #5
+                                            WholeStageCodegen (2)
+                                              Filter [s_store_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.store [s_store_sk]
+                                    InputAdapter
+                                      BroadcastExchange #4
+                                        WholeStageCodegen (3)
+                                          Project [d_date_sk,d_moy]
+                                            Filter [d_month_seq,d_date_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.date_dim [d_date_sk,d_month_seq,d_moy]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q63/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q63/explain.txt
new file mode 100644
index 0000000000000..c8e0821dfacda
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q63/explain.txt
@@ -0,0 +1,191 @@
+== Physical Plan ==
+TakeOrderedAndProject (32)
++- * Project (31)
+   +- * Filter (30)
+      +- Window (29)
+         +- * Sort (28)
+            +- Exchange (27)
+               +- * HashAggregate (26)
+                  +- Exchange (25)
+                     +- * HashAggregate (24)
+                        +- * Project (23)
+                           +- * BroadcastHashJoin Inner BuildRight (22)
+                              :- * Project (17)
+                              :  +- * BroadcastHashJoin Inner BuildRight (16)
+                              :     :- * Project (10)
+                              :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+                              :     :     :- * Project (4)
+                              :     :     :  +- * Filter (3)
+                              :     :     :     +- * ColumnarToRow (2)
+                              :     :     :        +- Scan parquet default.item (1)
+                              :     :     +- BroadcastExchange (8)
+                              :     :        +- * Filter (7)
+                              :     :           +- * ColumnarToRow (6)
+                              :     :              +- Scan parquet default.store_sales (5)
+                              :     +- BroadcastExchange (15)
+                              :        +- * Project (14)
+                              :           +- * Filter (13)
+                              :              +- * ColumnarToRow (12)
+                              :                 +- Scan parquet default.date_dim (11)
+                              +- BroadcastExchange (21)
+                                 +- * Filter (20)
+                                    +- * ColumnarToRow (19)
+                                       +- Scan parquet default.store (18)
+
+
+(1) Scan parquet default.item
+Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [Or(And(And(In(i_category, [Books                                             ,Children                                          ,Electronics                                       ]),In(i_class, [personal                                          ,portable                                          ,reference                                         ,self-help                                         ])),In(i_brand, [scholaramalgamalg #6                             ,scholaramalgamalg #7                              ,exportiunivamalg #8                               ,scholaramalgamalg #8                              ])),And(And(In(i_category, [Women                                             ,Music                                             ,Men                                               ]),In(i_class, [accessories                                       ,classical                                         ,fragrances                                        ,pants                                             ])),In(i_brand, [amalgimporto #9                                   ,edu packscholar #9                                ,exportiimporto #9                                 ,importoamalg #9                                   ]))), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_class:string,i_category:string,i_manager_id:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5]
+
+(3) Filter [codegen id : 4]
+Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5]
+Condition : ((((i_category#4 IN (Books                                             ,Children                                          ,Electronics                                       ) AND i_class#3 IN (personal                                          ,portable                                          ,reference                                         ,self-help                                         )) AND i_brand#2 IN (scholaramalgamalg #6                             ,scholaramalgamalg #7                              ,exportiunivamalg #8                               ,scholaramalgamalg #8                              )) OR ((i_category#4 IN (Women                                             ,Music                                             ,Men                                               ) AND i_class#3 IN (accessories                                       ,classical                                         ,fragrances                                        ,pants                                             )) AND i_brand#2 IN (amalgimporto #9                                   ,edu packscholar #9                                ,exportiimporto #9                                 ,importoamalg #9                                   ))) AND isnotnull(i_item_sk#1))
+
+(4) Project [codegen id : 4]
+Output [2]: [i_item_sk#1, i_manager_id#5]
+Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5]
+
+(5) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#13), (ss_sold_date_sk#13 >= 2452123), (ss_sold_date_sk#13 <= 2452487), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13]
+
+(7) Filter [codegen id : 1]
+Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13]
+Condition : (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11))
+
+(8) BroadcastExchange
+Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#15]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [ss_item_sk#10]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13]
+Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13]
+
+(11) Scan parquet default.date_dim
+Output [3]: [d_date_sk#16, d_month_seq#17, d_moy#18]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [In(d_month_seq, [1222,1228,1223,1227,1219,1226,1224,1225,1230,1220,1221,1229]), GreaterThanOrEqual(d_date_sk,2452123), LessThanOrEqual(d_date_sk,2452487), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int,d_moy:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [d_date_sk#16, d_month_seq#17, d_moy#18]
+
+(13) Filter [codegen id : 2]
+Input [3]: [d_date_sk#16, d_month_seq#17, d_moy#18]
+Condition : (((d_month_seq#17 INSET (1222,1228,1223,1227,1219,1226,1224,1225,1230,1220,1221,1229) AND (d_date_sk#16 >= 2452123)) AND (d_date_sk#16 <= 2452487)) AND isnotnull(d_date_sk#16))
+
+(14) Project [codegen id : 2]
+Output [2]: [d_date_sk#16, d_moy#18]
+Input [3]: [d_date_sk#16, d_month_seq#17, d_moy#18]
+
+(15) BroadcastExchange
+Input [2]: [d_date_sk#16, d_moy#18]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#19]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#13]
+Right keys [1]: [d_date_sk#16]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#18]
+Input [6]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#16, d_moy#18]
+
+(18) Scan parquet default.store
+Output [1]: [s_store_sk#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [1]: [s_store_sk#20]
+
+(20) Filter [codegen id : 3]
+Input [1]: [s_store_sk#20]
+Condition : isnotnull(s_store_sk#20)
+
+(21) BroadcastExchange
+Input [1]: [s_store_sk#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#21]
+
+(22) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#11]
+Right keys [1]: [s_store_sk#20]
+Join condition: None
+
+(23) Project [codegen id : 4]
+Output [3]: [i_manager_id#5, ss_sales_price#12, d_moy#18]
+Input [5]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#18, s_store_sk#20]
+
+(24) HashAggregate [codegen id : 4]
+Input [3]: [i_manager_id#5, ss_sales_price#12, d_moy#18]
+Keys [2]: [i_manager_id#5, d_moy#18]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))]
+Aggregate Attributes [1]: [sum#22]
+Results [3]: [i_manager_id#5, d_moy#18, sum#23]
+
+(25) Exchange
+Input [3]: [i_manager_id#5, d_moy#18, sum#23]
+Arguments: hashpartitioning(i_manager_id#5, d_moy#18, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(26) HashAggregate [codegen id : 5]
+Input [3]: [i_manager_id#5, d_moy#18, sum#23]
+Keys [2]: [i_manager_id#5, d_moy#18]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#12))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#25]
+Results [3]: [i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#25,17,2) AS sum_sales#26, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#25,17,2) AS _w0#27]
+
+(27) Exchange
+Input [3]: [i_manager_id#5, sum_sales#26, _w0#27]
+Arguments: hashpartitioning(i_manager_id#5, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(28) Sort [codegen id : 6]
+Input [3]: [i_manager_id#5, sum_sales#26, _w0#27]
+Arguments: [i_manager_id#5 ASC NULLS FIRST], false, 0
+
+(29) Window
+Input [3]: [i_manager_id#5, sum_sales#26, _w0#27]
+Arguments: [avg(_w0#27) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#29], [i_manager_id#5]
+
+(30) Filter [codegen id : 7]
+Input [4]: [i_manager_id#5, sum_sales#26, _w0#27, avg_monthly_sales#29]
+Condition : (isnotnull(avg_monthly_sales#29) AND ((avg_monthly_sales#29 > 0.000000) AND (CheckOverflow((promote_precision(abs(CheckOverflow((promote_precision(cast(sum_sales#26 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#29 as decimal(22,6)))), DecimalType(22,6), true), false)) / promote_precision(cast(avg_monthly_sales#29 as decimal(22,6)))), DecimalType(38,16), true) > 0.1000000000000000)))
+
+(31) Project [codegen id : 7]
+Output [3]: [i_manager_id#5, sum_sales#26, avg_monthly_sales#29]
+Input [4]: [i_manager_id#5, sum_sales#26, _w0#27, avg_monthly_sales#29]
+
+(32) TakeOrderedAndProject
+Input [3]: [i_manager_id#5, sum_sales#26, avg_monthly_sales#29]
+Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#29 ASC NULLS FIRST, sum_sales#26 ASC NULLS FIRST], [i_manager_id#5, sum_sales#26, avg_monthly_sales#29]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 5 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14
+ReusedExchange (33)
+
+
+(33) ReusedExchange [Reuses operator id: 15]
+Output [2]: [d_date_sk#16, d_moy#18]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q63/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q63/simplified.txt
new file mode 100644
index 0000000000000..4d85c6d1f57a3
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q63/simplified.txt
@@ -0,0 +1,51 @@
+TakeOrderedAndProject [i_manager_id,avg_monthly_sales,sum_sales]
+  WholeStageCodegen (7)
+    Project [i_manager_id,sum_sales,avg_monthly_sales]
+      Filter [avg_monthly_sales,sum_sales]
+        InputAdapter
+          Window [_w0,i_manager_id]
+            WholeStageCodegen (6)
+              Sort [i_manager_id]
+                InputAdapter
+                  Exchange [i_manager_id] #1
+                    WholeStageCodegen (5)
+                      HashAggregate [i_manager_id,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum]
+                        InputAdapter
+                          Exchange [i_manager_id,d_moy] #2
+                            WholeStageCodegen (4)
+                              HashAggregate [i_manager_id,d_moy,ss_sales_price] [sum,sum]
+                                Project [i_manager_id,ss_sales_price,d_moy]
+                                  BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                    Project [i_manager_id,ss_store_sk,ss_sales_price,d_moy]
+                                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                        Project [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                          BroadcastHashJoin [i_item_sk,ss_item_sk]
+                                            Project [i_item_sk,i_manager_id]
+                                              Filter [i_category,i_class,i_brand,i_item_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.item [i_item_sk,i_brand,i_class,i_category,i_manager_id]
+                                            InputAdapter
+                                              BroadcastExchange #3
+                                                WholeStageCodegen (1)
+                                                  Filter [ss_item_sk,ss_store_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                                          SubqueryBroadcast [d_date_sk] #1
+                                                            ReusedExchange [d_date_sk,d_moy] #4
+                                        InputAdapter
+                                          BroadcastExchange #4
+                                            WholeStageCodegen (2)
+                                              Project [d_date_sk,d_moy]
+                                                Filter [d_month_seq,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_month_seq,d_moy]
+                                    InputAdapter
+                                      BroadcastExchange #5
+                                        WholeStageCodegen (3)
+                                          Filter [s_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store [s_store_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q65.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q65.sf100/explain.txt
new file mode 100644
index 0000000000000..240f579025038
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q65.sf100/explain.txt
@@ -0,0 +1,259 @@
+== Physical Plan ==
+TakeOrderedAndProject (42)
++- * Project (41)
+   +- * BroadcastHashJoin Inner BuildLeft (40)
+      :- BroadcastExchange (36)
+      :  +- * Project (35)
+      :     +- * BroadcastHashJoin Inner BuildLeft (34)
+      :        :- BroadcastExchange (30)
+      :        :  +- * Project (29)
+      :        :     +- * BroadcastHashJoin Inner BuildRight (28)
+      :        :        :- * Filter (14)
+      :        :        :  +- * HashAggregate (13)
+      :        :        :     +- Exchange (12)
+      :        :        :        +- * HashAggregate (11)
+      :        :        :           +- * Project (10)
+      :        :        :              +- * BroadcastHashJoin Inner BuildRight (9)
+      :        :        :                 :- * Filter (3)
+      :        :        :                 :  +- * ColumnarToRow (2)
+      :        :        :                 :     +- Scan parquet default.store_sales (1)
+      :        :        :                 +- BroadcastExchange (8)
+      :        :        :                    +- * Project (7)
+      :        :        :                       +- * Filter (6)
+      :        :        :                          +- * ColumnarToRow (5)
+      :        :        :                             +- Scan parquet default.date_dim (4)
+      :        :        +- BroadcastExchange (27)
+      :        :           +- * HashAggregate (26)
+      :        :              +- Exchange (25)
+      :        :                 +- * HashAggregate (24)
+      :        :                    +- * HashAggregate (23)
+      :        :                       +- Exchange (22)
+      :        :                          +- * HashAggregate (21)
+      :        :                             +- * Project (20)
+      :        :                                +- * BroadcastHashJoin Inner BuildRight (19)
+      :        :                                   :- * Filter (17)
+      :        :                                   :  +- * ColumnarToRow (16)
+      :        :                                   :     +- Scan parquet default.store_sales (15)
+      :        :                                   +- ReusedExchange (18)
+      :        +- * Filter (33)
+      :           +- * ColumnarToRow (32)
+      :              +- Scan parquet default.store (31)
+      +- * Filter (39)
+         +- * ColumnarToRow (38)
+            +- Scan parquet default.item (37)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), (ss_sold_date_sk#4 >= 2451911), (ss_sold_date_sk#4 <= 2452275), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 2]
+Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, ss_sold_date_sk#4]
+Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#6, d_month_seq#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), GreaterThanOrEqual(d_date_sk,2451911), LessThanOrEqual(d_date_sk,2452275), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#6, d_month_seq#7]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#6, d_month_seq#7]
+Condition : (((((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND (d_date_sk#6 >= 2451911)) AND (d_date_sk#6 <= 2452275)) AND isnotnull(d_date_sk#6))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#6]
+Input [2]: [d_date_sk#6, d_month_seq#7]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#8]
+
+(9) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 2]
+Output [3]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3]
+Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, ss_sold_date_sk#4, d_date_sk#6]
+
+(11) HashAggregate [codegen id : 2]
+Input [3]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3]
+Keys [2]: [ss_store_sk#2, ss_item_sk#1]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#3))]
+Aggregate Attributes [1]: [sum#9]
+Results [3]: [ss_store_sk#2, ss_item_sk#1, sum#10]
+
+(12) Exchange
+Input [3]: [ss_store_sk#2, ss_item_sk#1, sum#10]
+Arguments: hashpartitioning(ss_store_sk#2, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#11]
+
+(13) HashAggregate [codegen id : 7]
+Input [3]: [ss_store_sk#2, ss_item_sk#1, sum#10]
+Keys [2]: [ss_store_sk#2, ss_item_sk#1]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#3))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#3))#12]
+Results [3]: [ss_store_sk#2, ss_item_sk#1, MakeDecimal(sum(UnscaledValue(ss_sales_price#3))#12,17,2) AS revenue#13]
+
+(14) Filter [codegen id : 7]
+Input [3]: [ss_store_sk#2, ss_item_sk#1, revenue#13]
+Condition : isnotnull(revenue#13)
+
+(15) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#14, ss_store_sk#15, ss_sales_price#16, ss_sold_date_sk#17]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#17), (ss_sold_date_sk#17 >= 2451911), (ss_sold_date_sk#17 <= 2452275), dynamicpruningexpression(ss_sold_date_sk#17 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(16) ColumnarToRow [codegen id : 4]
+Input [4]: [ss_item_sk#14, ss_store_sk#15, ss_sales_price#16, ss_sold_date_sk#17]
+
+(17) Filter [codegen id : 4]
+Input [4]: [ss_item_sk#14, ss_store_sk#15, ss_sales_price#16, ss_sold_date_sk#17]
+Condition : isnotnull(ss_store_sk#15)
+
+(18) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#18]
+
+(19) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#17]
+Right keys [1]: [d_date_sk#18]
+Join condition: None
+
+(20) Project [codegen id : 4]
+Output [3]: [ss_item_sk#14, ss_store_sk#15, ss_sales_price#16]
+Input [5]: [ss_item_sk#14, ss_store_sk#15, ss_sales_price#16, ss_sold_date_sk#17, d_date_sk#18]
+
+(21) HashAggregate [codegen id : 4]
+Input [3]: [ss_item_sk#14, ss_store_sk#15, ss_sales_price#16]
+Keys [2]: [ss_store_sk#15, ss_item_sk#14]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#16))]
+Aggregate Attributes [1]: [sum#19]
+Results [3]: [ss_store_sk#15, ss_item_sk#14, sum#20]
+
+(22) Exchange
+Input [3]: [ss_store_sk#15, ss_item_sk#14, sum#20]
+Arguments: hashpartitioning(ss_store_sk#15, ss_item_sk#14, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(23) HashAggregate [codegen id : 5]
+Input [3]: [ss_store_sk#15, ss_item_sk#14, sum#20]
+Keys [2]: [ss_store_sk#15, ss_item_sk#14]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#16))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#16))#22]
+Results [2]: [ss_store_sk#15, MakeDecimal(sum(UnscaledValue(ss_sales_price#16))#22,17,2) AS revenue#23]
+
+(24) HashAggregate [codegen id : 5]
+Input [2]: [ss_store_sk#15, revenue#23]
+Keys [1]: [ss_store_sk#15]
+Functions [1]: [partial_avg(revenue#23)]
+Aggregate Attributes [2]: [sum#24, count#25]
+Results [3]: [ss_store_sk#15, sum#26, count#27]
+
+(25) Exchange
+Input [3]: [ss_store_sk#15, sum#26, count#27]
+Arguments: hashpartitioning(ss_store_sk#15, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(26) HashAggregate [codegen id : 6]
+Input [3]: [ss_store_sk#15, sum#26, count#27]
+Keys [1]: [ss_store_sk#15]
+Functions [1]: [avg(revenue#23)]
+Aggregate Attributes [1]: [avg(revenue#23)#29]
+Results [2]: [ss_store_sk#15, avg(revenue#23)#29 AS ave#30]
+
+(27) BroadcastExchange
+Input [2]: [ss_store_sk#15, ave#30]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#31]
+
+(28) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [ss_store_sk#2]
+Right keys [1]: [ss_store_sk#15]
+Join condition: (cast(revenue#13 as decimal(23,7)) <= CheckOverflow((0.100000 * promote_precision(ave#30)), DecimalType(23,7), true))
+
+(29) Project [codegen id : 7]
+Output [3]: [ss_store_sk#2, ss_item_sk#1, revenue#13]
+Input [5]: [ss_store_sk#2, ss_item_sk#1, revenue#13, ss_store_sk#15, ave#30]
+
+(30) BroadcastExchange
+Input [3]: [ss_store_sk#2, ss_item_sk#1, revenue#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#32]
+
+(31) Scan parquet default.store
+Output [2]: [s_store_sk#33, s_store_name#34]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string>
+
+(32) ColumnarToRow
+Input [2]: [s_store_sk#33, s_store_name#34]
+
+(33) Filter
+Input [2]: [s_store_sk#33, s_store_name#34]
+Condition : isnotnull(s_store_sk#33)
+
+(34) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_store_sk#2]
+Right keys [1]: [s_store_sk#33]
+Join condition: None
+
+(35) Project [codegen id : 8]
+Output [3]: [ss_item_sk#1, revenue#13, s_store_name#34]
+Input [5]: [ss_store_sk#2, ss_item_sk#1, revenue#13, s_store_sk#33, s_store_name#34]
+
+(36) BroadcastExchange
+Input [3]: [ss_item_sk#1, revenue#13, s_store_name#34]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#35]
+
+(37) Scan parquet default.item
+Output [5]: [i_item_sk#36, i_item_desc#37, i_current_price#38, i_wholesale_cost#39, i_brand#40]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_desc:string,i_current_price:decimal(7,2),i_wholesale_cost:decimal(7,2),i_brand:string>
+
+(38) ColumnarToRow
+Input [5]: [i_item_sk#36, i_item_desc#37, i_current_price#38, i_wholesale_cost#39, i_brand#40]
+
+(39) Filter
+Input [5]: [i_item_sk#36, i_item_desc#37, i_current_price#38, i_wholesale_cost#39, i_brand#40]
+Condition : isnotnull(i_item_sk#36)
+
+(40) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#36]
+Join condition: None
+
+(41) Project [codegen id : 9]
+Output [6]: [s_store_name#34, i_item_desc#37, revenue#13, i_current_price#38, i_wholesale_cost#39, i_brand#40]
+Input [8]: [ss_item_sk#1, revenue#13, s_store_name#34, i_item_sk#36, i_item_desc#37, i_current_price#38, i_wholesale_cost#39, i_brand#40]
+
+(42) TakeOrderedAndProject
+Input [6]: [s_store_name#34, i_item_desc#37, revenue#13, i_current_price#38, i_wholesale_cost#39, i_brand#40]
+Arguments: 100, [s_store_name#34 ASC NULLS FIRST, i_item_desc#37 ASC NULLS FIRST], [s_store_name#34, i_item_desc#37, revenue#13, i_current_price#38, i_wholesale_cost#39, i_brand#40]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (43)
+
+
+(43) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#6]
+
+Subquery:2 Hosting operator id = 15 Hosting Expression = ss_sold_date_sk#17 IN dynamicpruning#5
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q65.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q65.sf100/simplified.txt
new file mode 100644
index 0000000000000..fec6d9ab3aefe
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q65.sf100/simplified.txt
@@ -0,0 +1,66 @@
+TakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand]
+  WholeStageCodegen (9)
+    Project [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand]
+      BroadcastHashJoin [ss_item_sk,i_item_sk]
+        InputAdapter
+          BroadcastExchange #1
+            WholeStageCodegen (8)
+              Project [ss_item_sk,revenue,s_store_name]
+                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                  InputAdapter
+                    BroadcastExchange #2
+                      WholeStageCodegen (7)
+                        Project [ss_store_sk,ss_item_sk,revenue]
+                          BroadcastHashJoin [ss_store_sk,ss_store_sk,revenue,ave]
+                            Filter [revenue]
+                              HashAggregate [ss_store_sk,ss_item_sk,sum] [sum(UnscaledValue(ss_sales_price)),revenue,sum]
+                                InputAdapter
+                                  Exchange [ss_store_sk,ss_item_sk] #3
+                                    WholeStageCodegen (2)
+                                      HashAggregate [ss_store_sk,ss_item_sk,ss_sales_price] [sum,sum]
+                                        Project [ss_item_sk,ss_store_sk,ss_sales_price]
+                                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                            Filter [ss_store_sk,ss_item_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #1
+                                                      ReusedExchange [d_date_sk] #4
+                                            InputAdapter
+                                              BroadcastExchange #4
+                                                WholeStageCodegen (1)
+                                                  Project [d_date_sk]
+                                                    Filter [d_month_seq,d_date_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.date_dim [d_date_sk,d_month_seq]
+                            InputAdapter
+                              BroadcastExchange #5
+                                WholeStageCodegen (6)
+                                  HashAggregate [ss_store_sk,sum,count] [avg(revenue),ave,sum,count]
+                                    InputAdapter
+                                      Exchange [ss_store_sk] #6
+                                        WholeStageCodegen (5)
+                                          HashAggregate [ss_store_sk,revenue] [sum,count,sum,count]
+                                            HashAggregate [ss_store_sk,ss_item_sk,sum] [sum(UnscaledValue(ss_sales_price)),revenue,sum]
+                                              InputAdapter
+                                                Exchange [ss_store_sk,ss_item_sk] #7
+                                                  WholeStageCodegen (4)
+                                                    HashAggregate [ss_store_sk,ss_item_sk,ss_sales_price] [sum,sum]
+                                                      Project [ss_item_sk,ss_store_sk,ss_sales_price]
+                                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                          Filter [ss_store_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                                                  ReusedSubquery [d_date_sk] #1
+                                                          InputAdapter
+                                                            ReusedExchange [d_date_sk] #4
+                  Filter [s_store_sk]
+                    ColumnarToRow
+                      InputAdapter
+                        Scan parquet default.store [s_store_sk,s_store_name]
+        Filter [i_item_sk]
+          ColumnarToRow
+            InputAdapter
+              Scan parquet default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q65/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q65/explain.txt
new file mode 100644
index 0000000000000..43ef320ed235f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q65/explain.txt
@@ -0,0 +1,259 @@
+== Physical Plan ==
+TakeOrderedAndProject (42)
++- * Project (41)
+   +- * BroadcastHashJoin Inner BuildRight (40)
+      :- * Project (26)
+      :  +- * BroadcastHashJoin Inner BuildRight (25)
+      :     :- * Project (20)
+      :     :  +- * BroadcastHashJoin Inner BuildRight (19)
+      :     :     :- * Filter (3)
+      :     :     :  +- * ColumnarToRow (2)
+      :     :     :     +- Scan parquet default.store (1)
+      :     :     +- BroadcastExchange (18)
+      :     :        +- * Filter (17)
+      :     :           +- * HashAggregate (16)
+      :     :              +- Exchange (15)
+      :     :                 +- * HashAggregate (14)
+      :     :                    +- * Project (13)
+      :     :                       +- * BroadcastHashJoin Inner BuildRight (12)
+      :     :                          :- * Filter (6)
+      :     :                          :  +- * ColumnarToRow (5)
+      :     :                          :     +- Scan parquet default.store_sales (4)
+      :     :                          +- BroadcastExchange (11)
+      :     :                             +- * Project (10)
+      :     :                                +- * Filter (9)
+      :     :                                   +- * ColumnarToRow (8)
+      :     :                                      +- Scan parquet default.date_dim (7)
+      :     +- BroadcastExchange (24)
+      :        +- * Filter (23)
+      :           +- * ColumnarToRow (22)
+      :              +- Scan parquet default.item (21)
+      +- BroadcastExchange (39)
+         +- * HashAggregate (38)
+            +- Exchange (37)
+               +- * HashAggregate (36)
+                  +- * HashAggregate (35)
+                     +- Exchange (34)
+                        +- * HashAggregate (33)
+                           +- * Project (32)
+                              +- * BroadcastHashJoin Inner BuildRight (31)
+                                 :- * Filter (29)
+                                 :  +- * ColumnarToRow (28)
+                                 :     +- Scan parquet default.store_sales (27)
+                                 +- ReusedExchange (30)
+
+
+(1) Scan parquet default.store
+Output [2]: [s_store_sk#1, s_store_name#2]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string>
+
+(2) ColumnarToRow [codegen id : 9]
+Input [2]: [s_store_sk#1, s_store_name#2]
+
+(3) Filter [codegen id : 9]
+Input [2]: [s_store_sk#1, s_store_name#2]
+Condition : isnotnull(s_store_sk#1)
+
+(4) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#6), (ss_sold_date_sk#6 >= 2451911), (ss_sold_date_sk#6 <= 2452275), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(5) ColumnarToRow [codegen id : 2]
+Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6]
+
+(6) Filter [codegen id : 2]
+Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6]
+Condition : (isnotnull(ss_store_sk#4) AND isnotnull(ss_item_sk#3))
+
+(7) Scan parquet default.date_dim
+Output [2]: [d_date_sk#8, d_month_seq#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), GreaterThanOrEqual(d_date_sk,2451911), LessThanOrEqual(d_date_sk,2452275), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(8) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#8, d_month_seq#9]
+
+(9) Filter [codegen id : 1]
+Input [2]: [d_date_sk#8, d_month_seq#9]
+Condition : (((((isnotnull(d_month_seq#9) AND (d_month_seq#9 >= 1212)) AND (d_month_seq#9 <= 1223)) AND (d_date_sk#8 >= 2451911)) AND (d_date_sk#8 <= 2452275)) AND isnotnull(d_date_sk#8))
+
+(10) Project [codegen id : 1]
+Output [1]: [d_date_sk#8]
+Input [2]: [d_date_sk#8, d_month_seq#9]
+
+(11) BroadcastExchange
+Input [1]: [d_date_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(12) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#6]
+Right keys [1]: [d_date_sk#8]
+Join condition: None
+
+(13) Project [codegen id : 2]
+Output [3]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5]
+Input [5]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6, d_date_sk#8]
+
+(14) HashAggregate [codegen id : 2]
+Input [3]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5]
+Keys [2]: [ss_store_sk#4, ss_item_sk#3]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#5))]
+Aggregate Attributes [1]: [sum#11]
+Results [3]: [ss_store_sk#4, ss_item_sk#3, sum#12]
+
+(15) Exchange
+Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#12]
+Arguments: hashpartitioning(ss_store_sk#4, ss_item_sk#3, 5), ENSURE_REQUIREMENTS, [id=#13]
+
+(16) HashAggregate [codegen id : 3]
+Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#12]
+Keys [2]: [ss_store_sk#4, ss_item_sk#3]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#5))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#5))#14]
+Results [3]: [ss_store_sk#4, ss_item_sk#3, MakeDecimal(sum(UnscaledValue(ss_sales_price#5))#14,17,2) AS revenue#15]
+
+(17) Filter [codegen id : 3]
+Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#15]
+Condition : isnotnull(revenue#15)
+
+(18) BroadcastExchange
+Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#16]
+
+(19) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [s_store_sk#1]
+Right keys [1]: [ss_store_sk#4]
+Join condition: None
+
+(20) Project [codegen id : 9]
+Output [4]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#15]
+Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#15]
+
+(21) Scan parquet default.item
+Output [5]: [i_item_sk#17, i_item_desc#18, i_current_price#19, i_wholesale_cost#20, i_brand#21]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_desc:string,i_current_price:decimal(7,2),i_wholesale_cost:decimal(7,2),i_brand:string>
+
+(22) ColumnarToRow [codegen id : 4]
+Input [5]: [i_item_sk#17, i_item_desc#18, i_current_price#19, i_wholesale_cost#20, i_brand#21]
+
+(23) Filter [codegen id : 4]
+Input [5]: [i_item_sk#17, i_item_desc#18, i_current_price#19, i_wholesale_cost#20, i_brand#21]
+Condition : isnotnull(i_item_sk#17)
+
+(24) BroadcastExchange
+Input [5]: [i_item_sk#17, i_item_desc#18, i_current_price#19, i_wholesale_cost#20, i_brand#21]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#22]
+
+(25) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_item_sk#3]
+Right keys [1]: [i_item_sk#17]
+Join condition: None
+
+(26) Project [codegen id : 9]
+Output [7]: [s_store_name#2, ss_store_sk#4, revenue#15, i_item_desc#18, i_current_price#19, i_wholesale_cost#20, i_brand#21]
+Input [9]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#15, i_item_sk#17, i_item_desc#18, i_current_price#19, i_wholesale_cost#20, i_brand#21]
+
+(27) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#23, ss_store_sk#24, ss_sales_price#25, ss_sold_date_sk#26]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#26), (ss_sold_date_sk#26 >= 2451911), (ss_sold_date_sk#26 <= 2452275), dynamicpruningexpression(ss_sold_date_sk#26 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(28) ColumnarToRow [codegen id : 6]
+Input [4]: [ss_item_sk#23, ss_store_sk#24, ss_sales_price#25, ss_sold_date_sk#26]
+
+(29) Filter [codegen id : 6]
+Input [4]: [ss_item_sk#23, ss_store_sk#24, ss_sales_price#25, ss_sold_date_sk#26]
+Condition : isnotnull(ss_store_sk#24)
+
+(30) ReusedExchange [Reuses operator id: 11]
+Output [1]: [d_date_sk#27]
+
+(31) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_sold_date_sk#26]
+Right keys [1]: [d_date_sk#27]
+Join condition: None
+
+(32) Project [codegen id : 6]
+Output [3]: [ss_item_sk#23, ss_store_sk#24, ss_sales_price#25]
+Input [5]: [ss_item_sk#23, ss_store_sk#24, ss_sales_price#25, ss_sold_date_sk#26, d_date_sk#27]
+
+(33) HashAggregate [codegen id : 6]
+Input [3]: [ss_item_sk#23, ss_store_sk#24, ss_sales_price#25]
+Keys [2]: [ss_store_sk#24, ss_item_sk#23]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#25))]
+Aggregate Attributes [1]: [sum#28]
+Results [3]: [ss_store_sk#24, ss_item_sk#23, sum#29]
+
+(34) Exchange
+Input [3]: [ss_store_sk#24, ss_item_sk#23, sum#29]
+Arguments: hashpartitioning(ss_store_sk#24, ss_item_sk#23, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(35) HashAggregate [codegen id : 7]
+Input [3]: [ss_store_sk#24, ss_item_sk#23, sum#29]
+Keys [2]: [ss_store_sk#24, ss_item_sk#23]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#25))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#25))#31]
+Results [2]: [ss_store_sk#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#25))#31,17,2) AS revenue#32]
+
+(36) HashAggregate [codegen id : 7]
+Input [2]: [ss_store_sk#24, revenue#32]
+Keys [1]: [ss_store_sk#24]
+Functions [1]: [partial_avg(revenue#32)]
+Aggregate Attributes [2]: [sum#33, count#34]
+Results [3]: [ss_store_sk#24, sum#35, count#36]
+
+(37) Exchange
+Input [3]: [ss_store_sk#24, sum#35, count#36]
+Arguments: hashpartitioning(ss_store_sk#24, 5), ENSURE_REQUIREMENTS, [id=#37]
+
+(38) HashAggregate [codegen id : 8]
+Input [3]: [ss_store_sk#24, sum#35, count#36]
+Keys [1]: [ss_store_sk#24]
+Functions [1]: [avg(revenue#32)]
+Aggregate Attributes [1]: [avg(revenue#32)#38]
+Results [2]: [ss_store_sk#24, avg(revenue#32)#38 AS ave#39]
+
+(39) BroadcastExchange
+Input [2]: [ss_store_sk#24, ave#39]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#40]
+
+(40) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_store_sk#4]
+Right keys [1]: [ss_store_sk#24]
+Join condition: (cast(revenue#15 as decimal(23,7)) <= CheckOverflow((0.100000 * promote_precision(ave#39)), DecimalType(23,7), true))
+
+(41) Project [codegen id : 9]
+Output [6]: [s_store_name#2, i_item_desc#18, revenue#15, i_current_price#19, i_wholesale_cost#20, i_brand#21]
+Input [9]: [s_store_name#2, ss_store_sk#4, revenue#15, i_item_desc#18, i_current_price#19, i_wholesale_cost#20, i_brand#21, ss_store_sk#24, ave#39]
+
+(42) TakeOrderedAndProject
+Input [6]: [s_store_name#2, i_item_desc#18, revenue#15, i_current_price#19, i_wholesale_cost#20, i_brand#21]
+Arguments: 100, [s_store_name#2 ASC NULLS FIRST, i_item_desc#18 ASC NULLS FIRST], [s_store_name#2, i_item_desc#18, revenue#15, i_current_price#19, i_wholesale_cost#20, i_brand#21]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7
+ReusedExchange (43)
+
+
+(43) ReusedExchange [Reuses operator id: 11]
+Output [1]: [d_date_sk#8]
+
+Subquery:2 Hosting operator id = 27 Hosting Expression = ss_sold_date_sk#26 IN dynamicpruning#7
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q65/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q65/simplified.txt
new file mode 100644
index 0000000000000..d7d461677be96
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q65/simplified.txt
@@ -0,0 +1,66 @@
+TakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand]
+  WholeStageCodegen (9)
+    Project [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand]
+      BroadcastHashJoin [ss_store_sk,ss_store_sk,revenue,ave]
+        Project [s_store_name,ss_store_sk,revenue,i_item_desc,i_current_price,i_wholesale_cost,i_brand]
+          BroadcastHashJoin [ss_item_sk,i_item_sk]
+            Project [s_store_name,ss_store_sk,ss_item_sk,revenue]
+              BroadcastHashJoin [s_store_sk,ss_store_sk]
+                Filter [s_store_sk]
+                  ColumnarToRow
+                    InputAdapter
+                      Scan parquet default.store [s_store_sk,s_store_name]
+                InputAdapter
+                  BroadcastExchange #1
+                    WholeStageCodegen (3)
+                      Filter [revenue]
+                        HashAggregate [ss_store_sk,ss_item_sk,sum] [sum(UnscaledValue(ss_sales_price)),revenue,sum]
+                          InputAdapter
+                            Exchange [ss_store_sk,ss_item_sk] #2
+                              WholeStageCodegen (2)
+                                HashAggregate [ss_store_sk,ss_item_sk,ss_sales_price] [sum,sum]
+                                  Project [ss_item_sk,ss_store_sk,ss_sales_price]
+                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                      Filter [ss_store_sk,ss_item_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #1
+                                                ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        BroadcastExchange #3
+                                          WholeStageCodegen (1)
+                                            Project [d_date_sk]
+                                              Filter [d_month_seq,d_date_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_date_sk,d_month_seq]
+            InputAdapter
+              BroadcastExchange #4
+                WholeStageCodegen (4)
+                  Filter [i_item_sk]
+                    ColumnarToRow
+                      InputAdapter
+                        Scan parquet default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand]
+        InputAdapter
+          BroadcastExchange #5
+            WholeStageCodegen (8)
+              HashAggregate [ss_store_sk,sum,count] [avg(revenue),ave,sum,count]
+                InputAdapter
+                  Exchange [ss_store_sk] #6
+                    WholeStageCodegen (7)
+                      HashAggregate [ss_store_sk,revenue] [sum,count,sum,count]
+                        HashAggregate [ss_store_sk,ss_item_sk,sum] [sum(UnscaledValue(ss_sales_price)),revenue,sum]
+                          InputAdapter
+                            Exchange [ss_store_sk,ss_item_sk] #7
+                              WholeStageCodegen (6)
+                                HashAggregate [ss_store_sk,ss_item_sk,ss_sales_price] [sum,sum]
+                                  Project [ss_item_sk,ss_store_sk,ss_sales_price]
+                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                      Filter [ss_store_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                              ReusedSubquery [d_date_sk] #1
+                                      InputAdapter
+                                        ReusedExchange [d_date_sk] #3
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q68.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q68.sf100/explain.txt
new file mode 100644
index 0000000000000..0b96b76a0ffc4
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q68.sf100/explain.txt
@@ -0,0 +1,270 @@
+== Physical Plan ==
+TakeOrderedAndProject (46)
++- * Project (45)
+   +- * BroadcastHashJoin Inner BuildLeft (44)
+      :- BroadcastExchange (40)
+      :  +- * Project (39)
+      :     +- * BroadcastHashJoin Inner BuildLeft (38)
+      :        :- BroadcastExchange (34)
+      :        :  +- * HashAggregate (33)
+      :        :     +- Exchange (32)
+      :        :        +- * HashAggregate (31)
+      :        :           +- * Project (30)
+      :        :              +- * BroadcastHashJoin Inner BuildLeft (29)
+      :        :                 :- BroadcastExchange (25)
+      :        :                 :  +- * Project (24)
+      :        :                 :     +- * BroadcastHashJoin Inner BuildRight (23)
+      :        :                 :        :- * Project (17)
+      :        :                 :        :  +- * BroadcastHashJoin Inner BuildRight (16)
+      :        :                 :        :     :- * Project (10)
+      :        :                 :        :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+      :        :                 :        :     :     :- * Filter (3)
+      :        :                 :        :     :     :  +- * ColumnarToRow (2)
+      :        :                 :        :     :     :     +- Scan parquet default.store_sales (1)
+      :        :                 :        :     :     +- BroadcastExchange (8)
+      :        :                 :        :     :        +- * Project (7)
+      :        :                 :        :     :           +- * Filter (6)
+      :        :                 :        :     :              +- * ColumnarToRow (5)
+      :        :                 :        :     :                 +- Scan parquet default.store (4)
+      :        :                 :        :     +- BroadcastExchange (15)
+      :        :                 :        :        +- * Project (14)
+      :        :                 :        :           +- * Filter (13)
+      :        :                 :        :              +- * ColumnarToRow (12)
+      :        :                 :        :                 +- Scan parquet default.date_dim (11)
+      :        :                 :        +- BroadcastExchange (22)
+      :        :                 :           +- * Project (21)
+      :        :                 :              +- * Filter (20)
+      :        :                 :                 +- * ColumnarToRow (19)
+      :        :                 :                    +- Scan parquet default.household_demographics (18)
+      :        :                 +- * Filter (28)
+      :        :                    +- * ColumnarToRow (27)
+      :        :                       +- Scan parquet default.customer_address (26)
+      :        +- * Filter (37)
+      :           +- * ColumnarToRow (36)
+      :              +- Scan parquet default.customer (35)
+      +- * Filter (43)
+         +- * ColumnarToRow (42)
+            +- Scan parquet default.customer_address (41)
+
+
+(1) Scan parquet default.store_sales
+Output [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [ss_sold_date_sk#9 INSET (2451790,2451180,2452216,2451454,2452184,2451485,2451850,2451514,2452062,2451270,2452123,2451758,2451971,2451546,2451942,2451393,2451667,2451453,2452215,2451819,2451331,2451577,2451911,2452245,2451301,2451545,2451605,2451943,2451851,2451181,2452154,2451820,2452001,2451362,2451392,2451240,2452032,2451637,2451484,2452124,2451300,2451727,2452093,2451759,2451698,2451332,2451606,2451666,2451912,2452185,2451211,2451361,2452031,2451212,2451880,2451789,2451423,2451576,2451728,2452246,2452155,2452092,2451881,2451970,2451697,2452063,2451271,2451636,2451515,2451424,2451239,2452002), isnotnull(ss_sold_date_sk#9), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_hdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_ext_sales_price:decimal(7,2),ss_ext_list_price:decimal(7,2),ss_ext_tax:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9]
+
+(3) Filter [codegen id : 4]
+Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9]
+Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1))
+
+(4) Scan parquet default.store
+Output [2]: [s_store_sk#11, s_city#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [In(s_city, [Midway,Fairview]), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_city:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [s_store_sk#11, s_city#12]
+
+(6) Filter [codegen id : 1]
+Input [2]: [s_store_sk#11, s_city#12]
+Condition : (s_city#12 IN (Midway,Fairview) AND isnotnull(s_store_sk#11))
+
+(7) Project [codegen id : 1]
+Output [1]: [s_store_sk#11]
+Input [2]: [s_store_sk#11, s_city#12]
+
+(8) BroadcastExchange
+Input [1]: [s_store_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#4]
+Right keys [1]: [s_store_sk#11]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9]
+Input [10]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9, s_store_sk#11]
+
+(11) Scan parquet default.date_dim
+Output [3]: [d_date_sk#14, d_year#15, d_dom#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), In(d_date_sk, [2451790,2451180,2452216,2451454,2452184,2451485,2451850,2451514,2452062,2451270,2452123,2451758,2451971,2451546,2451942,2451393,2451667,2451453,2452215,2451819,2451331,2451577,2451911,2452245,2451301,2451545,2451605,2451943,2451851,2451181,2452154,2451820,2452001,2451362,2451392,2451240,2452032,2451637,2451484,2452124,2451300,2451727,2452093,2451759,2451698,2451332,2451606,2451666,2451912,2452185,2451211,2451361,2452031,2451212,2451880,2451789,2451423,2451576,2451728,2452246,2452155,2452092,2451881,2451970,2451697,2452063,2451271,2451636,2451515,2451424,2451239,2452002]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_dom:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [d_date_sk#14, d_year#15, d_dom#16]
+
+(13) Filter [codegen id : 2]
+Input [3]: [d_date_sk#14, d_year#15, d_dom#16]
+Condition : (((((isnotnull(d_dom#16) AND (d_dom#16 >= 1)) AND (d_dom#16 <= 2)) AND d_year#15 IN (1999,2000,2001)) AND d_date_sk#14 INSET (2451790,2451180,2452216,2451454,2452184,2451485,2451850,2451514,2452062,2451270,2452123,2451758,2451971,2451546,2451942,2451393,2451667,2451453,2452215,2451819,2451331,2451577,2451911,2452245,2451301,2451545,2451605,2451943,2451851,2451181,2452154,2451820,2452001,2451362,2451392,2451240,2452032,2451637,2451484,2452124,2451300,2451727,2452093,2451759,2451698,2451332,2451606,2451666,2451912,2452185,2451211,2451361,2452031,2451212,2451880,2451789,2451423,2451576,2451728,2452246,2452155,2452092,2451881,2451970,2451697,2452063,2451271,2451636,2451515,2451424,2451239,2452002)) AND isnotnull(d_date_sk#14))
+
+(14) Project [codegen id : 2]
+Output [1]: [d_date_sk#14]
+Input [3]: [d_date_sk#14, d_year#15, d_dom#16]
+
+(15) BroadcastExchange
+Input [1]: [d_date_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#9]
+Right keys [1]: [d_date_sk#14]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8]
+Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9, d_date_sk#14]
+
+(18) Scan parquet default.household_demographics
+Output [3]: [hd_demo_sk#18, hd_dep_count#19, hd_vehicle_count#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [Or(EqualTo(hd_dep_count,5),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_dep_count:int,hd_vehicle_count:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [3]: [hd_demo_sk#18, hd_dep_count#19, hd_vehicle_count#20]
+
+(20) Filter [codegen id : 3]
+Input [3]: [hd_demo_sk#18, hd_dep_count#19, hd_vehicle_count#20]
+Condition : (((hd_dep_count#19 = 5) OR (hd_vehicle_count#20 = 3)) AND isnotnull(hd_demo_sk#18))
+
+(21) Project [codegen id : 3]
+Output [1]: [hd_demo_sk#18]
+Input [3]: [hd_demo_sk#18, hd_dep_count#19, hd_vehicle_count#20]
+
+(22) BroadcastExchange
+Input [1]: [hd_demo_sk#18]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(23) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#18]
+Join condition: None
+
+(24) Project [codegen id : 4]
+Output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8]
+Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, hd_demo_sk#18]
+
+(25) BroadcastExchange
+Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [id=#22]
+
+(26) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#23, ca_city#24]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)]
+ReadSchema: struct<ca_address_sk:int,ca_city:string>
+
+(27) ColumnarToRow
+Input [2]: [ca_address_sk#23, ca_city#24]
+
+(28) Filter
+Input [2]: [ca_address_sk#23, ca_city#24]
+Condition : (isnotnull(ca_address_sk#23) AND isnotnull(ca_city#24))
+
+(29) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_addr_sk#3]
+Right keys [1]: [ca_address_sk#23]
+Join condition: None
+
+(30) Project [codegen id : 5]
+Output [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#24]
+Input [8]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_address_sk#23, ca_city#24]
+
+(31) HashAggregate [codegen id : 5]
+Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#24]
+Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#24]
+Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#6)), partial_sum(UnscaledValue(ss_ext_list_price#7)), partial_sum(UnscaledValue(ss_ext_tax#8))]
+Aggregate Attributes [3]: [sum#25, sum#26, sum#27]
+Results [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#24, sum#28, sum#29, sum#30]
+
+(32) Exchange
+Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#24, sum#28, sum#29, sum#30]
+Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#24, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(33) HashAggregate [codegen id : 6]
+Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#24, sum#28, sum#29, sum#30]
+Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#24]
+Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#6)), sum(UnscaledValue(ss_ext_list_price#7)), sum(UnscaledValue(ss_ext_tax#8))]
+Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#6))#32, sum(UnscaledValue(ss_ext_list_price#7))#33, sum(UnscaledValue(ss_ext_tax#8))#34]
+Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#24 AS bought_city#35, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#6))#32,17,2) AS extended_price#36, MakeDecimal(sum(UnscaledValue(ss_ext_list_price#7))#33,17,2) AS list_price#37, MakeDecimal(sum(UnscaledValue(ss_ext_tax#8))#34,17,2) AS extended_tax#38]
+
+(34) BroadcastExchange
+Input [6]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#35, extended_price#36, list_price#37, extended_tax#38]
+Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [id=#39]
+
+(35) Scan parquet default.customer
+Output [4]: [c_customer_sk#40, c_current_addr_sk#41, c_first_name#42, c_last_name#43]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int,c_first_name:string,c_last_name:string>
+
+(36) ColumnarToRow
+Input [4]: [c_customer_sk#40, c_current_addr_sk#41, c_first_name#42, c_last_name#43]
+
+(37) Filter
+Input [4]: [c_customer_sk#40, c_current_addr_sk#41, c_first_name#42, c_last_name#43]
+Condition : (isnotnull(c_customer_sk#40) AND isnotnull(c_current_addr_sk#41))
+
+(38) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#40]
+Join condition: None
+
+(39) Project [codegen id : 7]
+Output [8]: [ss_ticket_number#5, bought_city#35, extended_price#36, list_price#37, extended_tax#38, c_current_addr_sk#41, c_first_name#42, c_last_name#43]
+Input [10]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#35, extended_price#36, list_price#37, extended_tax#38, c_customer_sk#40, c_current_addr_sk#41, c_first_name#42, c_last_name#43]
+
+(40) BroadcastExchange
+Input [8]: [ss_ticket_number#5, bought_city#35, extended_price#36, list_price#37, extended_tax#38, c_current_addr_sk#41, c_first_name#42, c_last_name#43]
+Arguments: HashedRelationBroadcastMode(List(cast(input[5, int, true] as bigint)),false), [id=#44]
+
+(41) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#45, ca_city#46]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)]
+ReadSchema: struct<ca_address_sk:int,ca_city:string>
+
+(42) ColumnarToRow
+Input [2]: [ca_address_sk#45, ca_city#46]
+
+(43) Filter
+Input [2]: [ca_address_sk#45, ca_city#46]
+Condition : (isnotnull(ca_address_sk#45) AND isnotnull(ca_city#46))
+
+(44) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [c_current_addr_sk#41]
+Right keys [1]: [ca_address_sk#45]
+Join condition: NOT (ca_city#46 = bought_city#35)
+
+(45) Project [codegen id : 8]
+Output [8]: [c_last_name#43, c_first_name#42, ca_city#46, bought_city#35, ss_ticket_number#5, extended_price#36, extended_tax#38, list_price#37]
+Input [10]: [ss_ticket_number#5, bought_city#35, extended_price#36, list_price#37, extended_tax#38, c_current_addr_sk#41, c_first_name#42, c_last_name#43, ca_address_sk#45, ca_city#46]
+
+(46) TakeOrderedAndProject
+Input [8]: [c_last_name#43, c_first_name#42, ca_city#46, bought_city#35, ss_ticket_number#5, extended_price#36, extended_tax#38, list_price#37]
+Arguments: 100, [c_last_name#43 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#43, c_first_name#42, ca_city#46, bought_city#35, ss_ticket_number#5, extended_price#36, extended_tax#38, list_price#37]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10
+ReusedExchange (47)
+
+
+(47) ReusedExchange [Reuses operator id: 15]
+Output [1]: [d_date_sk#14]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q68.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q68.sf100/simplified.txt
new file mode 100644
index 0000000000000..11ab7865edd5a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q68.sf100/simplified.txt
@@ -0,0 +1,70 @@
+TakeOrderedAndProject [c_last_name,ss_ticket_number,c_first_name,ca_city,bought_city,extended_price,extended_tax,list_price]
+  WholeStageCodegen (8)
+    Project [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,extended_price,extended_tax,list_price]
+      BroadcastHashJoin [c_current_addr_sk,ca_address_sk,ca_city,bought_city]
+        InputAdapter
+          BroadcastExchange #1
+            WholeStageCodegen (7)
+              Project [ss_ticket_number,bought_city,extended_price,list_price,extended_tax,c_current_addr_sk,c_first_name,c_last_name]
+                BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+                  InputAdapter
+                    BroadcastExchange #2
+                      WholeStageCodegen (6)
+                        HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_ext_list_price)),sum(UnscaledValue(ss_ext_tax)),bought_city,extended_price,list_price,extended_tax,sum,sum,sum]
+                          InputAdapter
+                            Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #3
+                              WholeStageCodegen (5)
+                                HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] [sum,sum,sum,sum,sum,sum]
+                                  Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_city]
+                                    BroadcastHashJoin [ss_addr_sk,ca_address_sk]
+                                      InputAdapter
+                                        BroadcastExchange #4
+                                          WholeStageCodegen (4)
+                                            Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax]
+                                              BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                                Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax]
+                                                  BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                    Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk]
+                                                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                        Filter [ss_store_sk,ss_hdemo_sk,ss_addr_sk,ss_customer_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk]
+                                                                SubqueryBroadcast [d_date_sk] #1
+                                                                  ReusedExchange [d_date_sk] #5
+                                                        InputAdapter
+                                                          BroadcastExchange #6
+                                                            WholeStageCodegen (1)
+                                                              Project [s_store_sk]
+                                                                Filter [s_city,s_store_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.store [s_store_sk,s_city]
+                                                    InputAdapter
+                                                      BroadcastExchange #5
+                                                        WholeStageCodegen (2)
+                                                          Project [d_date_sk]
+                                                            Filter [d_dom,d_year,d_date_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.date_dim [d_date_sk,d_year,d_dom]
+                                                InputAdapter
+                                                  BroadcastExchange #7
+                                                    WholeStageCodegen (3)
+                                                      Project [hd_demo_sk]
+                                                        Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count]
+                                      Filter [ca_address_sk,ca_city]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.customer_address [ca_address_sk,ca_city]
+                  Filter [c_customer_sk,c_current_addr_sk]
+                    ColumnarToRow
+                      InputAdapter
+                        Scan parquet default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name]
+        Filter [ca_address_sk,ca_city]
+          ColumnarToRow
+            InputAdapter
+              Scan parquet default.customer_address [ca_address_sk,ca_city]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q68/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q68/explain.txt
new file mode 100644
index 0000000000000..f50329db7e05e
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q68/explain.txt
@@ -0,0 +1,252 @@
+== Physical Plan ==
+TakeOrderedAndProject (43)
++- * Project (42)
+   +- * BroadcastHashJoin Inner BuildRight (41)
+      :- * Project (39)
+      :  +- * BroadcastHashJoin Inner BuildRight (38)
+      :     :- * HashAggregate (33)
+      :     :  +- Exchange (32)
+      :     :     +- * HashAggregate (31)
+      :     :        +- * Project (30)
+      :     :           +- * BroadcastHashJoin Inner BuildRight (29)
+      :     :              :- * Project (24)
+      :     :              :  +- * BroadcastHashJoin Inner BuildRight (23)
+      :     :              :     :- * Project (17)
+      :     :              :     :  +- * BroadcastHashJoin Inner BuildRight (16)
+      :     :              :     :     :- * Project (10)
+      :     :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+      :     :              :     :     :     :- * Filter (3)
+      :     :              :     :     :     :  +- * ColumnarToRow (2)
+      :     :              :     :     :     :     +- Scan parquet default.store_sales (1)
+      :     :              :     :     :     +- BroadcastExchange (8)
+      :     :              :     :     :        +- * Project (7)
+      :     :              :     :     :           +- * Filter (6)
+      :     :              :     :     :              +- * ColumnarToRow (5)
+      :     :              :     :     :                 +- Scan parquet default.date_dim (4)
+      :     :              :     :     +- BroadcastExchange (15)
+      :     :              :     :        +- * Project (14)
+      :     :              :     :           +- * Filter (13)
+      :     :              :     :              +- * ColumnarToRow (12)
+      :     :              :     :                 +- Scan parquet default.store (11)
+      :     :              :     +- BroadcastExchange (22)
+      :     :              :        +- * Project (21)
+      :     :              :           +- * Filter (20)
+      :     :              :              +- * ColumnarToRow (19)
+      :     :              :                 +- Scan parquet default.household_demographics (18)
+      :     :              +- BroadcastExchange (28)
+      :     :                 +- * Filter (27)
+      :     :                    +- * ColumnarToRow (26)
+      :     :                       +- Scan parquet default.customer_address (25)
+      :     +- BroadcastExchange (37)
+      :        +- * Filter (36)
+      :           +- * ColumnarToRow (35)
+      :              +- Scan parquet default.customer (34)
+      +- ReusedExchange (40)
+
+
+(1) Scan parquet default.store_sales
+Output [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [ss_sold_date_sk#9 INSET (2451790,2451180,2452216,2451454,2452184,2451485,2451850,2451514,2452062,2451270,2452123,2451758,2451971,2451546,2451942,2451393,2451667,2451453,2452215,2451819,2451331,2451577,2451911,2452245,2451301,2451545,2451605,2451943,2451851,2451181,2452154,2451820,2452001,2451362,2451392,2451240,2452032,2451637,2451484,2452124,2451300,2451727,2452093,2451759,2451698,2451332,2451606,2451666,2451912,2452185,2451211,2451361,2452031,2451212,2451880,2451789,2451423,2451576,2451728,2452246,2452155,2452092,2451881,2451970,2451697,2452063,2451271,2451636,2451515,2451424,2451239,2452002), isnotnull(ss_sold_date_sk#9), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_hdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_ext_sales_price:decimal(7,2),ss_ext_list_price:decimal(7,2),ss_ext_tax:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9]
+
+(3) Filter [codegen id : 5]
+Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9]
+Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#11, d_year#12, d_dom#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), In(d_date_sk, [2451790,2451180,2452216,2451454,2452184,2451485,2451850,2451514,2452062,2451270,2452123,2451758,2451971,2451546,2451942,2451393,2451667,2451453,2452215,2451819,2451331,2451577,2451911,2452245,2451301,2451545,2451605,2451943,2451851,2451181,2452154,2451820,2452001,2451362,2451392,2451240,2452032,2451637,2451484,2452124,2451300,2451727,2452093,2451759,2451698,2451332,2451606,2451666,2451912,2452185,2451211,2451361,2452031,2451212,2451880,2451789,2451423,2451576,2451728,2452246,2452155,2452092,2451881,2451970,2451697,2452063,2451271,2451636,2451515,2451424,2451239,2452002]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_dom:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#11, d_year#12, d_dom#13]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#11, d_year#12, d_dom#13]
+Condition : (((((isnotnull(d_dom#13) AND (d_dom#13 >= 1)) AND (d_dom#13 <= 2)) AND d_year#12 IN (1999,2000,2001)) AND d_date_sk#11 INSET (2451790,2451180,2452216,2451454,2452184,2451485,2451850,2451514,2452062,2451270,2452123,2451758,2451971,2451546,2451942,2451393,2451667,2451453,2452215,2451819,2451331,2451577,2451911,2452245,2451301,2451545,2451605,2451943,2451851,2451181,2452154,2451820,2452001,2451362,2451392,2451240,2452032,2451637,2451484,2452124,2451300,2451727,2452093,2451759,2451698,2451332,2451606,2451666,2451912,2452185,2451211,2451361,2452031,2451212,2451880,2451789,2451423,2451576,2451728,2452246,2452155,2452092,2451881,2451970,2451697,2452063,2451271,2451636,2451515,2451424,2451239,2452002)) AND isnotnull(d_date_sk#11))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#11]
+Input [3]: [d_date_sk#11, d_year#12, d_dom#13]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_sold_date_sk#9]
+Right keys [1]: [d_date_sk#11]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8]
+Input [10]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9, d_date_sk#11]
+
+(11) Scan parquet default.store
+Output [2]: [s_store_sk#15, s_city#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [In(s_city, [Midway,Fairview]), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_city:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [s_store_sk#15, s_city#16]
+
+(13) Filter [codegen id : 2]
+Input [2]: [s_store_sk#15, s_city#16]
+Condition : (s_city#16 IN (Midway,Fairview) AND isnotnull(s_store_sk#15))
+
+(14) Project [codegen id : 2]
+Output [1]: [s_store_sk#15]
+Input [2]: [s_store_sk#15, s_city#16]
+
+(15) BroadcastExchange
+Input [1]: [s_store_sk#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(16) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_store_sk#4]
+Right keys [1]: [s_store_sk#15]
+Join condition: None
+
+(17) Project [codegen id : 5]
+Output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8]
+Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, s_store_sk#15]
+
+(18) Scan parquet default.household_demographics
+Output [3]: [hd_demo_sk#18, hd_dep_count#19, hd_vehicle_count#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [Or(EqualTo(hd_dep_count,5),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_dep_count:int,hd_vehicle_count:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [3]: [hd_demo_sk#18, hd_dep_count#19, hd_vehicle_count#20]
+
+(20) Filter [codegen id : 3]
+Input [3]: [hd_demo_sk#18, hd_dep_count#19, hd_vehicle_count#20]
+Condition : (((hd_dep_count#19 = 5) OR (hd_vehicle_count#20 = 3)) AND isnotnull(hd_demo_sk#18))
+
+(21) Project [codegen id : 3]
+Output [1]: [hd_demo_sk#18]
+Input [3]: [hd_demo_sk#18, hd_dep_count#19, hd_vehicle_count#20]
+
+(22) BroadcastExchange
+Input [1]: [hd_demo_sk#18]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(23) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#18]
+Join condition: None
+
+(24) Project [codegen id : 5]
+Output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8]
+Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, hd_demo_sk#18]
+
+(25) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#22, ca_city#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)]
+ReadSchema: struct<ca_address_sk:int,ca_city:string>
+
+(26) ColumnarToRow [codegen id : 4]
+Input [2]: [ca_address_sk#22, ca_city#23]
+
+(27) Filter [codegen id : 4]
+Input [2]: [ca_address_sk#22, ca_city#23]
+Condition : (isnotnull(ca_address_sk#22) AND isnotnull(ca_city#23))
+
+(28) BroadcastExchange
+Input [2]: [ca_address_sk#22, ca_city#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#24]
+
+(29) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_addr_sk#3]
+Right keys [1]: [ca_address_sk#22]
+Join condition: None
+
+(30) Project [codegen id : 5]
+Output [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#23]
+Input [8]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_address_sk#22, ca_city#23]
+
+(31) HashAggregate [codegen id : 5]
+Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#23]
+Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#23]
+Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#6)), partial_sum(UnscaledValue(ss_ext_list_price#7)), partial_sum(UnscaledValue(ss_ext_tax#8))]
+Aggregate Attributes [3]: [sum#25, sum#26, sum#27]
+Results [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#23, sum#28, sum#29, sum#30]
+
+(32) Exchange
+Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#23, sum#28, sum#29, sum#30]
+Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#23, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(33) HashAggregate [codegen id : 8]
+Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#23, sum#28, sum#29, sum#30]
+Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#23]
+Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#6)), sum(UnscaledValue(ss_ext_list_price#7)), sum(UnscaledValue(ss_ext_tax#8))]
+Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#6))#32, sum(UnscaledValue(ss_ext_list_price#7))#33, sum(UnscaledValue(ss_ext_tax#8))#34]
+Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#23 AS bought_city#35, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#6))#32,17,2) AS extended_price#36, MakeDecimal(sum(UnscaledValue(ss_ext_list_price#7))#33,17,2) AS list_price#37, MakeDecimal(sum(UnscaledValue(ss_ext_tax#8))#34,17,2) AS extended_tax#38]
+
+(34) Scan parquet default.customer
+Output [4]: [c_customer_sk#39, c_current_addr_sk#40, c_first_name#41, c_last_name#42]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int,c_first_name:string,c_last_name:string>
+
+(35) ColumnarToRow [codegen id : 6]
+Input [4]: [c_customer_sk#39, c_current_addr_sk#40, c_first_name#41, c_last_name#42]
+
+(36) Filter [codegen id : 6]
+Input [4]: [c_customer_sk#39, c_current_addr_sk#40, c_first_name#41, c_last_name#42]
+Condition : (isnotnull(c_customer_sk#39) AND isnotnull(c_current_addr_sk#40))
+
+(37) BroadcastExchange
+Input [4]: [c_customer_sk#39, c_current_addr_sk#40, c_first_name#41, c_last_name#42]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#43]
+
+(38) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#39]
+Join condition: None
+
+(39) Project [codegen id : 8]
+Output [8]: [ss_ticket_number#5, bought_city#35, extended_price#36, list_price#37, extended_tax#38, c_current_addr_sk#40, c_first_name#41, c_last_name#42]
+Input [10]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#35, extended_price#36, list_price#37, extended_tax#38, c_customer_sk#39, c_current_addr_sk#40, c_first_name#41, c_last_name#42]
+
+(40) ReusedExchange [Reuses operator id: 28]
+Output [2]: [ca_address_sk#44, ca_city#45]
+
+(41) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [c_current_addr_sk#40]
+Right keys [1]: [ca_address_sk#44]
+Join condition: NOT (ca_city#45 = bought_city#35)
+
+(42) Project [codegen id : 8]
+Output [8]: [c_last_name#42, c_first_name#41, ca_city#45, bought_city#35, ss_ticket_number#5, extended_price#36, extended_tax#38, list_price#37]
+Input [10]: [ss_ticket_number#5, bought_city#35, extended_price#36, list_price#37, extended_tax#38, c_current_addr_sk#40, c_first_name#41, c_last_name#42, ca_address_sk#44, ca_city#45]
+
+(43) TakeOrderedAndProject
+Input [8]: [c_last_name#42, c_first_name#41, ca_city#45, bought_city#35, ss_ticket_number#5, extended_price#36, extended_tax#38, list_price#37]
+Arguments: 100, [c_last_name#42 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#42, c_first_name#41, ca_city#45, bought_city#35, ss_ticket_number#5, extended_price#36, extended_tax#38, list_price#37]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10
+ReusedExchange (44)
+
+
+(44) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#11]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q68/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q68/simplified.txt
new file mode 100644
index 0000000000000..b4d72f1da879d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q68/simplified.txt
@@ -0,0 +1,65 @@
+TakeOrderedAndProject [c_last_name,ss_ticket_number,c_first_name,ca_city,bought_city,extended_price,extended_tax,list_price]
+  WholeStageCodegen (8)
+    Project [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,extended_price,extended_tax,list_price]
+      BroadcastHashJoin [c_current_addr_sk,ca_address_sk,ca_city,bought_city]
+        Project [ss_ticket_number,bought_city,extended_price,list_price,extended_tax,c_current_addr_sk,c_first_name,c_last_name]
+          BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+            HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_ext_list_price)),sum(UnscaledValue(ss_ext_tax)),bought_city,extended_price,list_price,extended_tax,sum,sum,sum]
+              InputAdapter
+                Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1
+                  WholeStageCodegen (5)
+                    HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] [sum,sum,sum,sum,sum,sum]
+                      Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_city]
+                        BroadcastHashJoin [ss_addr_sk,ca_address_sk]
+                          Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax]
+                            BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                              Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax]
+                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                  Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax]
+                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                      Filter [ss_store_sk,ss_hdemo_sk,ss_addr_sk,ss_customer_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #1
+                                                ReusedExchange [d_date_sk] #2
+                                      InputAdapter
+                                        BroadcastExchange #2
+                                          WholeStageCodegen (1)
+                                            Project [d_date_sk]
+                                              Filter [d_dom,d_year,d_date_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_date_sk,d_year,d_dom]
+                                  InputAdapter
+                                    BroadcastExchange #3
+                                      WholeStageCodegen (2)
+                                        Project [s_store_sk]
+                                          Filter [s_city,s_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store [s_store_sk,s_city]
+                              InputAdapter
+                                BroadcastExchange #4
+                                  WholeStageCodegen (3)
+                                    Project [hd_demo_sk]
+                                      Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count]
+                          InputAdapter
+                            BroadcastExchange #5
+                              WholeStageCodegen (4)
+                                Filter [ca_address_sk,ca_city]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.customer_address [ca_address_sk,ca_city]
+            InputAdapter
+              BroadcastExchange #6
+                WholeStageCodegen (6)
+                  Filter [c_customer_sk,c_current_addr_sk]
+                    ColumnarToRow
+                      InputAdapter
+                        Scan parquet default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name]
+        InputAdapter
+          ReusedExchange [ca_address_sk,ca_city] #5
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q7.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q7.sf100/explain.txt
new file mode 100644
index 0000000000000..daa0ca7b16562
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q7.sf100/explain.txt
@@ -0,0 +1,204 @@
+== Physical Plan ==
+TakeOrderedAndProject (34)
++- * HashAggregate (33)
+   +- Exchange (32)
+      +- * HashAggregate (31)
+         +- * Project (30)
+            +- * BroadcastHashJoin Inner BuildRight (29)
+               :- * Project (24)
+               :  +- * BroadcastHashJoin Inner BuildRight (23)
+               :     :- * Project (17)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (16)
+               :     :     :- * Project (10)
+               :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :     :     :- * Filter (3)
+               :     :     :     :  +- * ColumnarToRow (2)
+               :     :     :     :     +- Scan parquet default.store_sales (1)
+               :     :     :     +- BroadcastExchange (8)
+               :     :     :        +- * Project (7)
+               :     :     :           +- * Filter (6)
+               :     :     :              +- * ColumnarToRow (5)
+               :     :     :                 +- Scan parquet default.date_dim (4)
+               :     :     +- BroadcastExchange (15)
+               :     :        +- * Project (14)
+               :     :           +- * Filter (13)
+               :     :              +- * ColumnarToRow (12)
+               :     :                 +- Scan parquet default.promotion (11)
+               :     +- BroadcastExchange (22)
+               :        +- * Project (21)
+               :           +- * Filter (20)
+               :              +- * ColumnarToRow (19)
+               :                 +- Scan parquet default.customer_demographics (18)
+               +- BroadcastExchange (28)
+                  +- * Filter (27)
+                     +- * ColumnarToRow (26)
+                        +- Scan parquet default.item (25)
+
+
+(1) Scan parquet default.store_sales
+Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), (ss_sold_date_sk#8 >= 2450815), (ss_sold_date_sk#8 <= 2451179), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_cdemo_sk:int,ss_promo_sk:int,ss_quantity:int,ss_list_price:decimal(7,2),ss_sales_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+
+(3) Filter [codegen id : 5]
+Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3))
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#10, d_year#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), GreaterThanOrEqual(d_date_sk,2450815), LessThanOrEqual(d_date_sk,2451179), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#10, d_year#11]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#10, d_year#11]
+Condition : ((((isnotnull(d_year#11) AND (d_year#11 = 1998)) AND (d_date_sk#10 >= 2450815)) AND (d_date_sk#10 <= 2451179)) AND isnotnull(d_date_sk#10))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#10]
+Input [2]: [d_date_sk#10, d_year#11]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [7]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7]
+Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#10]
+
+(11) Scan parquet default.promotion
+Output [3]: [p_promo_sk#13, p_channel_email#14, p_channel_event#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/promotion]
+PushedFilters: [Or(EqualTo(p_channel_email,N),EqualTo(p_channel_event,N)), IsNotNull(p_promo_sk)]
+ReadSchema: struct<p_promo_sk:int,p_channel_email:string,p_channel_event:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [p_promo_sk#13, p_channel_email#14, p_channel_event#15]
+
+(13) Filter [codegen id : 2]
+Input [3]: [p_promo_sk#13, p_channel_email#14, p_channel_event#15]
+Condition : (((p_channel_email#14 = N) OR (p_channel_event#15 = N)) AND isnotnull(p_promo_sk#13))
+
+(14) Project [codegen id : 2]
+Output [1]: [p_promo_sk#13]
+Input [3]: [p_promo_sk#13, p_channel_email#14, p_channel_event#15]
+
+(15) BroadcastExchange
+Input [1]: [p_promo_sk#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#16]
+
+(16) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_promo_sk#3]
+Right keys [1]: [p_promo_sk#13]
+Join condition: None
+
+(17) Project [codegen id : 5]
+Output [6]: [ss_item_sk#1, ss_cdemo_sk#2, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7]
+Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, p_promo_sk#13]
+
+(18) Scan parquet default.customer_demographics
+Output [4]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,F), EqualTo(cd_marital_status,W), EqualTo(cd_education_status,Primary             ), IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_education_status:string>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [4]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20]
+
+(20) Filter [codegen id : 3]
+Input [4]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20]
+Condition : ((((((isnotnull(cd_gender#18) AND isnotnull(cd_marital_status#19)) AND isnotnull(cd_education_status#20)) AND (cd_gender#18 = F)) AND (cd_marital_status#19 = W)) AND (cd_education_status#20 = Primary             )) AND isnotnull(cd_demo_sk#17))
+
+(21) Project [codegen id : 3]
+Output [1]: [cd_demo_sk#17]
+Input [4]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20]
+
+(22) BroadcastExchange
+Input [1]: [cd_demo_sk#17]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(23) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#17]
+Join condition: None
+
+(24) Project [codegen id : 5]
+Output [5]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7]
+Input [7]: [ss_item_sk#1, ss_cdemo_sk#2, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, cd_demo_sk#17]
+
+(25) Scan parquet default.item
+Output [2]: [i_item_sk#22, i_item_id#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(26) ColumnarToRow [codegen id : 4]
+Input [2]: [i_item_sk#22, i_item_id#23]
+
+(27) Filter [codegen id : 4]
+Input [2]: [i_item_sk#22, i_item_id#23]
+Condition : isnotnull(i_item_sk#22)
+
+(28) BroadcastExchange
+Input [2]: [i_item_sk#22, i_item_id#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#24]
+
+(29) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#22]
+Join condition: None
+
+(30) Project [codegen id : 5]
+Output [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#23]
+Input [7]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#22, i_item_id#23]
+
+(31) HashAggregate [codegen id : 5]
+Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#23]
+Keys [1]: [i_item_id#23]
+Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))]
+Aggregate Attributes [8]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31, count#32]
+Results [9]: [i_item_id#23, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40]
+
+(32) Exchange
+Input [9]: [i_item_id#23, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40]
+Arguments: hashpartitioning(i_item_id#23, 5), ENSURE_REQUIREMENTS, [id=#41]
+
+(33) HashAggregate [codegen id : 6]
+Input [9]: [i_item_id#23, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40]
+Keys [1]: [i_item_id#23]
+Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))]
+Aggregate Attributes [4]: [avg(ss_quantity#4)#42, avg(UnscaledValue(ss_list_price#5))#43, avg(UnscaledValue(ss_coupon_amt#7))#44, avg(UnscaledValue(ss_sales_price#6))#45]
+Results [5]: [i_item_id#23, avg(ss_quantity#4)#42 AS agg1#46, cast((avg(UnscaledValue(ss_list_price#5))#43 / 100.0) as decimal(11,6)) AS agg2#47, cast((avg(UnscaledValue(ss_coupon_amt#7))#44 / 100.0) as decimal(11,6)) AS agg3#48, cast((avg(UnscaledValue(ss_sales_price#6))#45 / 100.0) as decimal(11,6)) AS agg4#49]
+
+(34) TakeOrderedAndProject
+Input [5]: [i_item_id#23, agg1#46, agg2#47, agg3#48, agg4#49]
+Arguments: 100, [i_item_id#23 ASC NULLS FIRST], [i_item_id#23, agg1#46, agg2#47, agg3#48, agg4#49]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (35)
+
+
+(35) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#10]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q7.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q7.sf100/simplified.txt
new file mode 100644
index 0000000000000..c3e9e4655d3e7
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q7.sf100/simplified.txt
@@ -0,0 +1,52 @@
+TakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4]
+  WholeStageCodegen (6)
+    HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price)),agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count]
+      InputAdapter
+        Exchange [i_item_id] #1
+          WholeStageCodegen (5)
+            HashAggregate [i_item_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+              Project [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id]
+                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                  Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                    BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk]
+                      Project [ss_item_sk,ss_cdemo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                        BroadcastHashJoin [ss_promo_sk,p_promo_sk]
+                          Project [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                            BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                              Filter [ss_cdemo_sk,ss_item_sk,ss_promo_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.store_sales [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                                      SubqueryBroadcast [d_date_sk] #1
+                                        ReusedExchange [d_date_sk] #2
+                              InputAdapter
+                                BroadcastExchange #2
+                                  WholeStageCodegen (1)
+                                    Project [d_date_sk]
+                                      Filter [d_year,d_date_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.date_dim [d_date_sk,d_year]
+                          InputAdapter
+                            BroadcastExchange #3
+                              WholeStageCodegen (2)
+                                Project [p_promo_sk]
+                                  Filter [p_channel_email,p_channel_event,p_promo_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.promotion [p_promo_sk,p_channel_email,p_channel_event]
+                      InputAdapter
+                        BroadcastExchange #4
+                          WholeStageCodegen (3)
+                            Project [cd_demo_sk]
+                              Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status]
+                  InputAdapter
+                    BroadcastExchange #5
+                      WholeStageCodegen (4)
+                        Filter [i_item_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.item [i_item_sk,i_item_id]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q7/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q7/explain.txt
new file mode 100644
index 0000000000000..a9da9118323b6
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q7/explain.txt
@@ -0,0 +1,204 @@
+== Physical Plan ==
+TakeOrderedAndProject (34)
++- * HashAggregate (33)
+   +- Exchange (32)
+      +- * HashAggregate (31)
+         +- * Project (30)
+            +- * BroadcastHashJoin Inner BuildRight (29)
+               :- * Project (23)
+               :  +- * BroadcastHashJoin Inner BuildRight (22)
+               :     :- * Project (17)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (16)
+               :     :     :- * Project (10)
+               :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :     :     :- * Filter (3)
+               :     :     :     :  +- * ColumnarToRow (2)
+               :     :     :     :     +- Scan parquet default.store_sales (1)
+               :     :     :     +- BroadcastExchange (8)
+               :     :     :        +- * Project (7)
+               :     :     :           +- * Filter (6)
+               :     :     :              +- * ColumnarToRow (5)
+               :     :     :                 +- Scan parquet default.customer_demographics (4)
+               :     :     +- BroadcastExchange (15)
+               :     :        +- * Project (14)
+               :     :           +- * Filter (13)
+               :     :              +- * ColumnarToRow (12)
+               :     :                 +- Scan parquet default.date_dim (11)
+               :     +- BroadcastExchange (21)
+               :        +- * Filter (20)
+               :           +- * ColumnarToRow (19)
+               :              +- Scan parquet default.item (18)
+               +- BroadcastExchange (28)
+                  +- * Project (27)
+                     +- * Filter (26)
+                        +- * ColumnarToRow (25)
+                           +- Scan parquet default.promotion (24)
+
+
+(1) Scan parquet default.store_sales
+Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), (ss_sold_date_sk#8 >= 2450815), (ss_sold_date_sk#8 <= 2451179), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_cdemo_sk:int,ss_promo_sk:int,ss_quantity:int,ss_list_price:decimal(7,2),ss_sales_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+
+(3) Filter [codegen id : 5]
+Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3))
+
+(4) Scan parquet default.customer_demographics
+Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,F), EqualTo(cd_marital_status,W), EqualTo(cd_education_status,Primary             ), IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_education_status:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+
+(6) Filter [codegen id : 1]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (cd_gender#11 = F)) AND (cd_marital_status#12 = W)) AND (cd_education_status#13 = Primary             )) AND isnotnull(cd_demo_sk#10))
+
+(7) Project [codegen id : 1]
+Output [1]: [cd_demo_sk#10]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+
+(8) BroadcastExchange
+Input [1]: [cd_demo_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#10]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#10]
+
+(11) Scan parquet default.date_dim
+Output [2]: [d_date_sk#15, d_year#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), GreaterThanOrEqual(d_date_sk,2450815), LessThanOrEqual(d_date_sk,2451179), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#15, d_year#16]
+
+(13) Filter [codegen id : 2]
+Input [2]: [d_date_sk#15, d_year#16]
+Condition : ((((isnotnull(d_year#16) AND (d_year#16 = 1998)) AND (d_date_sk#15 >= 2450815)) AND (d_date_sk#15 <= 2451179)) AND isnotnull(d_date_sk#15))
+
+(14) Project [codegen id : 2]
+Output [1]: [d_date_sk#15]
+Input [2]: [d_date_sk#15, d_year#16]
+
+(15) BroadcastExchange
+Input [1]: [d_date_sk#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(16) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#15]
+Join condition: None
+
+(17) Project [codegen id : 5]
+Output [6]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7]
+Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#15]
+
+(18) Scan parquet default.item
+Output [2]: [i_item_sk#18, i_item_id#19]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [2]: [i_item_sk#18, i_item_id#19]
+
+(20) Filter [codegen id : 3]
+Input [2]: [i_item_sk#18, i_item_id#19]
+Condition : isnotnull(i_item_sk#18)
+
+(21) BroadcastExchange
+Input [2]: [i_item_sk#18, i_item_id#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#20]
+
+(22) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#18]
+Join condition: None
+
+(23) Project [codegen id : 5]
+Output [6]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19]
+Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#18, i_item_id#19]
+
+(24) Scan parquet default.promotion
+Output [3]: [p_promo_sk#21, p_channel_email#22, p_channel_event#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/promotion]
+PushedFilters: [Or(EqualTo(p_channel_email,N),EqualTo(p_channel_event,N)), IsNotNull(p_promo_sk)]
+ReadSchema: struct<p_promo_sk:int,p_channel_email:string,p_channel_event:string>
+
+(25) ColumnarToRow [codegen id : 4]
+Input [3]: [p_promo_sk#21, p_channel_email#22, p_channel_event#23]
+
+(26) Filter [codegen id : 4]
+Input [3]: [p_promo_sk#21, p_channel_email#22, p_channel_event#23]
+Condition : (((p_channel_email#22 = N) OR (p_channel_event#23 = N)) AND isnotnull(p_promo_sk#21))
+
+(27) Project [codegen id : 4]
+Output [1]: [p_promo_sk#21]
+Input [3]: [p_promo_sk#21, p_channel_email#22, p_channel_event#23]
+
+(28) BroadcastExchange
+Input [1]: [p_promo_sk#21]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#24]
+
+(29) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_promo_sk#3]
+Right keys [1]: [p_promo_sk#21]
+Join condition: None
+
+(30) Project [codegen id : 5]
+Output [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19]
+Input [7]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19, p_promo_sk#21]
+
+(31) HashAggregate [codegen id : 5]
+Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19]
+Keys [1]: [i_item_id#19]
+Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))]
+Aggregate Attributes [8]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31, count#32]
+Results [9]: [i_item_id#19, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40]
+
+(32) Exchange
+Input [9]: [i_item_id#19, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40]
+Arguments: hashpartitioning(i_item_id#19, 5), ENSURE_REQUIREMENTS, [id=#41]
+
+(33) HashAggregate [codegen id : 6]
+Input [9]: [i_item_id#19, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40]
+Keys [1]: [i_item_id#19]
+Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))]
+Aggregate Attributes [4]: [avg(ss_quantity#4)#42, avg(UnscaledValue(ss_list_price#5))#43, avg(UnscaledValue(ss_coupon_amt#7))#44, avg(UnscaledValue(ss_sales_price#6))#45]
+Results [5]: [i_item_id#19, avg(ss_quantity#4)#42 AS agg1#46, cast((avg(UnscaledValue(ss_list_price#5))#43 / 100.0) as decimal(11,6)) AS agg2#47, cast((avg(UnscaledValue(ss_coupon_amt#7))#44 / 100.0) as decimal(11,6)) AS agg3#48, cast((avg(UnscaledValue(ss_sales_price#6))#45 / 100.0) as decimal(11,6)) AS agg4#49]
+
+(34) TakeOrderedAndProject
+Input [5]: [i_item_id#19, agg1#46, agg2#47, agg3#48, agg4#49]
+Arguments: 100, [i_item_id#19 ASC NULLS FIRST], [i_item_id#19, agg1#46, agg2#47, agg3#48, agg4#49]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (35)
+
+
+(35) ReusedExchange [Reuses operator id: 15]
+Output [1]: [d_date_sk#15]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q7/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q7/simplified.txt
new file mode 100644
index 0000000000000..4de61c8e70519
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q7/simplified.txt
@@ -0,0 +1,52 @@
+TakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4]
+  WholeStageCodegen (6)
+    HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price)),agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count]
+      InputAdapter
+        Exchange [i_item_id] #1
+          WholeStageCodegen (5)
+            HashAggregate [i_item_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+              Project [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id]
+                BroadcastHashJoin [ss_promo_sk,p_promo_sk]
+                  Project [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id]
+                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                      Project [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                          Project [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                            BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk]
+                              Filter [ss_cdemo_sk,ss_item_sk,ss_promo_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.store_sales [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                                      SubqueryBroadcast [d_date_sk] #1
+                                        ReusedExchange [d_date_sk] #2
+                              InputAdapter
+                                BroadcastExchange #3
+                                  WholeStageCodegen (1)
+                                    Project [cd_demo_sk]
+                                      Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status]
+                          InputAdapter
+                            BroadcastExchange #2
+                              WholeStageCodegen (2)
+                                Project [d_date_sk]
+                                  Filter [d_year,d_date_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.date_dim [d_date_sk,d_year]
+                      InputAdapter
+                        BroadcastExchange #4
+                          WholeStageCodegen (3)
+                            Filter [i_item_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.item [i_item_sk,i_item_id]
+                  InputAdapter
+                    BroadcastExchange #5
+                      WholeStageCodegen (4)
+                        Project [p_promo_sk]
+                          Filter [p_channel_email,p_channel_event,p_promo_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.promotion [p_promo_sk,p_channel_email,p_channel_event]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q73.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q73.sf100/explain.txt
new file mode 100644
index 0000000000000..d7a40fad9ce61
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q73.sf100/explain.txt
@@ -0,0 +1,204 @@
+== Physical Plan ==
+* Sort (36)
++- Exchange (35)
+   +- * Project (34)
+      +- * BroadcastHashJoin Inner BuildLeft (33)
+         :- BroadcastExchange (29)
+         :  +- * Filter (28)
+         :     +- * HashAggregate (27)
+         :        +- Exchange (26)
+         :           +- * HashAggregate (25)
+         :              +- * Project (24)
+         :                 +- * BroadcastHashJoin Inner BuildLeft (23)
+         :                    :- BroadcastExchange (18)
+         :                    :  +- * Project (17)
+         :                    :     +- * BroadcastHashJoin Inner BuildLeft (16)
+         :                    :        :- BroadcastExchange (11)
+         :                    :        :  +- * Project (10)
+         :                    :        :     +- * BroadcastHashJoin Inner BuildLeft (9)
+         :                    :        :        :- BroadcastExchange (4)
+         :                    :        :        :  +- * Filter (3)
+         :                    :        :        :     +- * ColumnarToRow (2)
+         :                    :        :        :        +- Scan parquet default.store_sales (1)
+         :                    :        :        +- * Project (8)
+         :                    :        :           +- * Filter (7)
+         :                    :        :              +- * ColumnarToRow (6)
+         :                    :        :                 +- Scan parquet default.date_dim (5)
+         :                    :        +- * Project (15)
+         :                    :           +- * Filter (14)
+         :                    :              +- * ColumnarToRow (13)
+         :                    :                 +- Scan parquet default.store (12)
+         :                    +- * Project (22)
+         :                       +- * Filter (21)
+         :                          +- * ColumnarToRow (20)
+         :                             +- Scan parquet default.household_demographics (19)
+         +- * Filter (32)
+            +- * ColumnarToRow (31)
+               +- Scan parquet default.customer (30)
+
+
+(1) Scan parquet default.store_sales
+Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [ss_sold_date_sk#5 INSET (2451790,2451119,2451180,2451454,2450874,2450906,2450967,2451485,2451850,2451514,2451270,2451758,2451028,2451546,2450997,2450996,2451393,2451667,2451453,2451819,2450905,2451331,2451577,2451089,2451301,2451545,2451605,2451851,2451181,2451149,2451820,2451362,2451392,2451240,2450935,2451637,2451484,2451058,2451300,2451727,2451759,2450815,2451698,2451150,2451332,2451606,2451666,2451211,2450846,2450875,2450966,2450936,2451361,2451212,2451880,2451059,2451789,2451423,2451576,2450816,2451088,2451728,2451027,2451120,2451881,2451697,2450847,2451271,2451636,2451515,2451424,2451239), isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_hdemo_sk:int,ss_store_sk:int,ss_ticket_number:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+
+(3) Filter [codegen id : 1]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1))
+
+(4) BroadcastExchange
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[4, int, true] as bigint)),false), [id=#6]
+
+(5) Scan parquet default.date_dim
+Output [3]: [d_date_sk#7, d_year#8, d_dom#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1998,1999,2000]), In(d_date_sk, [2451790,2451119,2451180,2451454,2450874,2450906,2450967,2451485,2451850,2451514,2451270,2451758,2451028,2451546,2450997,2450996,2451393,2451667,2451453,2451819,2450905,2451331,2451577,2451089,2451301,2451545,2451605,2451851,2451181,2451149,2451820,2451362,2451392,2451240,2450935,2451637,2451484,2451058,2451300,2451727,2451759,2450815,2451698,2451150,2451332,2451606,2451666,2451211,2450846,2450875,2450966,2450936,2451361,2451212,2451880,2451059,2451789,2451423,2451576,2450816,2451088,2451728,2451027,2451120,2451881,2451697,2450847,2451271,2451636,2451515,2451424,2451239]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_dom:int>
+
+(6) ColumnarToRow
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+
+(7) Filter
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+Condition : (((((isnotnull(d_dom#9) AND (d_dom#9 >= 1)) AND (d_dom#9 <= 2)) AND d_year#8 IN (1998,1999,2000)) AND d_date_sk#7 INSET (2451790,2451119,2451180,2451454,2450874,2450906,2450967,2451485,2451850,2451514,2451270,2451758,2451028,2451546,2450997,2450996,2451393,2451667,2451453,2451819,2450905,2451331,2451577,2451089,2451301,2451545,2451605,2451851,2451181,2451149,2451820,2451362,2451392,2451240,2450935,2451637,2451484,2451058,2451300,2451727,2451759,2450815,2451698,2451150,2451332,2451606,2451666,2451211,2450846,2450875,2450966,2450936,2451361,2451212,2451880,2451059,2451789,2451423,2451576,2450816,2451088,2451728,2451027,2451120,2451881,2451697,2450847,2451271,2451636,2451515,2451424,2451239)) AND isnotnull(d_date_sk#7))
+
+(8) Project
+Output [1]: [d_date_sk#7]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+
+(9) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#5]
+Right keys [1]: [d_date_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 2]
+Output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4]
+Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#7]
+
+(11) BroadcastExchange
+Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4]
+Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [id=#10]
+
+(12) Scan parquet default.store
+Output [2]: [s_store_sk#11, s_county#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [In(s_county, [Fairfield County,Ziebach County,Bronx County,Barrow County]), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_county:string>
+
+(13) ColumnarToRow
+Input [2]: [s_store_sk#11, s_county#12]
+
+(14) Filter
+Input [2]: [s_store_sk#11, s_county#12]
+Condition : (s_county#12 IN (Fairfield County,Ziebach County,Bronx County,Barrow County) AND isnotnull(s_store_sk#11))
+
+(15) Project
+Output [1]: [s_store_sk#11]
+Input [2]: [s_store_sk#11, s_county#12]
+
+(16) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#11]
+Join condition: None
+
+(17) Project [codegen id : 3]
+Output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#11]
+
+(18) BroadcastExchange
+Input [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4]
+Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [id=#13]
+
+(19) Scan parquet default.household_demographics
+Output [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_vehicle_count), IsNotNull(hd_dep_count), Or(EqualTo(hd_buy_potential,>10000         ),EqualTo(hd_buy_potential,Unknown        )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_buy_potential:string,hd_dep_count:int,hd_vehicle_count:int>
+
+(20) ColumnarToRow
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+
+(21) Filter
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+Condition : (((((isnotnull(hd_vehicle_count#17) AND isnotnull(hd_dep_count#16)) AND ((hd_buy_potential#15 = >10000         ) OR (hd_buy_potential#15 = Unknown        ))) AND (hd_vehicle_count#17 > 0)) AND ((cast(hd_dep_count#16 as double) / cast(hd_vehicle_count#17 as double)) > 1.0)) AND isnotnull(hd_demo_sk#14))
+
+(22) Project
+Output [1]: [hd_demo_sk#14]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+
+(23) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#14]
+Join condition: None
+
+(24) Project [codegen id : 4]
+Output [2]: [ss_customer_sk#1, ss_ticket_number#4]
+Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#14]
+
+(25) HashAggregate [codegen id : 4]
+Input [2]: [ss_customer_sk#1, ss_ticket_number#4]
+Keys [2]: [ss_ticket_number#4, ss_customer_sk#1]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#18]
+Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count#19]
+
+(26) Exchange
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#19]
+Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(27) HashAggregate [codegen id : 5]
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#19]
+Keys [2]: [ss_ticket_number#4, ss_customer_sk#1]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#21]
+Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count(1)#21 AS cnt#22]
+
+(28) Filter [codegen id : 5]
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#22]
+Condition : ((cnt#22 >= 1) AND (cnt#22 <= 5))
+
+(29) BroadcastExchange
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [id=#23]
+
+(30) Scan parquet default.customer
+Output [5]: [c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_salutation:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string>
+
+(31) ColumnarToRow
+Input [5]: [c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+
+(32) Filter
+Input [5]: [c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+Condition : isnotnull(c_customer_sk#24)
+
+(33) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#24]
+Join condition: None
+
+(34) Project [codegen id : 6]
+Output [6]: [c_last_name#27, c_first_name#26, c_salutation#25, c_preferred_cust_flag#28, ss_ticket_number#4, cnt#22]
+Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#22, c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+
+(35) Exchange
+Input [6]: [c_last_name#27, c_first_name#26, c_salutation#25, c_preferred_cust_flag#28, ss_ticket_number#4, cnt#22]
+Arguments: rangepartitioning(cnt#22 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [id=#29]
+
+(36) Sort [codegen id : 7]
+Input [6]: [c_last_name#27, c_first_name#26, c_salutation#25, c_preferred_cust_flag#28, ss_ticket_number#4, cnt#22]
+Arguments: [cnt#22 DESC NULLS LAST], true, 0
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q73.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q73.sf100/simplified.txt
new file mode 100644
index 0000000000000..91deecbd03317
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q73.sf100/simplified.txt
@@ -0,0 +1,54 @@
+WholeStageCodegen (7)
+  Sort [cnt]
+    InputAdapter
+      Exchange [cnt] #1
+        WholeStageCodegen (6)
+          Project [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt]
+            BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+              InputAdapter
+                BroadcastExchange #2
+                  WholeStageCodegen (5)
+                    Filter [cnt]
+                      HashAggregate [ss_ticket_number,ss_customer_sk,count] [count(1),cnt,count]
+                        InputAdapter
+                          Exchange [ss_ticket_number,ss_customer_sk] #3
+                            WholeStageCodegen (4)
+                              HashAggregate [ss_ticket_number,ss_customer_sk] [count,count]
+                                Project [ss_customer_sk,ss_ticket_number]
+                                  BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                    InputAdapter
+                                      BroadcastExchange #4
+                                        WholeStageCodegen (3)
+                                          Project [ss_customer_sk,ss_hdemo_sk,ss_ticket_number]
+                                            BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                              InputAdapter
+                                                BroadcastExchange #5
+                                                  WholeStageCodegen (2)
+                                                    Project [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number]
+                                                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                        InputAdapter
+                                                          BroadcastExchange #6
+                                                            WholeStageCodegen (1)
+                                                              Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk]
+                                                        Project [d_date_sk]
+                                                          Filter [d_dom,d_year,d_date_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.date_dim [d_date_sk,d_year,d_dom]
+                                              Project [s_store_sk]
+                                                Filter [s_county,s_store_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.store [s_store_sk,s_county]
+                                    Project [hd_demo_sk]
+                                      Filter [hd_vehicle_count,hd_dep_count,hd_buy_potential,hd_demo_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count]
+              Filter [c_customer_sk]
+                ColumnarToRow
+                  InputAdapter
+                    Scan parquet default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q73/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q73/explain.txt
new file mode 100644
index 0000000000000..5ec772ca3d638
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q73/explain.txt
@@ -0,0 +1,214 @@
+== Physical Plan ==
+* Sort (36)
++- Exchange (35)
+   +- * Project (34)
+      +- * BroadcastHashJoin Inner BuildRight (33)
+         :- * Filter (28)
+         :  +- * HashAggregate (27)
+         :     +- Exchange (26)
+         :        +- * HashAggregate (25)
+         :           +- * Project (24)
+         :              +- * BroadcastHashJoin Inner BuildRight (23)
+         :                 :- * Project (17)
+         :                 :  +- * BroadcastHashJoin Inner BuildRight (16)
+         :                 :     :- * Project (10)
+         :                 :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+         :                 :     :     :- * Filter (3)
+         :                 :     :     :  +- * ColumnarToRow (2)
+         :                 :     :     :     +- Scan parquet default.store_sales (1)
+         :                 :     :     +- BroadcastExchange (8)
+         :                 :     :        +- * Project (7)
+         :                 :     :           +- * Filter (6)
+         :                 :     :              +- * ColumnarToRow (5)
+         :                 :     :                 +- Scan parquet default.date_dim (4)
+         :                 :     +- BroadcastExchange (15)
+         :                 :        +- * Project (14)
+         :                 :           +- * Filter (13)
+         :                 :              +- * ColumnarToRow (12)
+         :                 :                 +- Scan parquet default.store (11)
+         :                 +- BroadcastExchange (22)
+         :                    +- * Project (21)
+         :                       +- * Filter (20)
+         :                          +- * ColumnarToRow (19)
+         :                             +- Scan parquet default.household_demographics (18)
+         +- BroadcastExchange (32)
+            +- * Filter (31)
+               +- * ColumnarToRow (30)
+                  +- Scan parquet default.customer (29)
+
+
+(1) Scan parquet default.store_sales
+Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [ss_sold_date_sk#5 INSET (2451790,2451119,2451180,2451454,2450874,2450906,2450967,2451485,2451850,2451514,2451270,2451758,2451028,2451546,2450997,2450996,2451393,2451667,2451453,2451819,2450905,2451331,2451577,2451089,2451301,2451545,2451605,2451851,2451181,2451149,2451820,2451362,2451392,2451240,2450935,2451637,2451484,2451058,2451300,2451727,2451759,2450815,2451698,2451150,2451332,2451606,2451666,2451211,2450846,2450875,2450966,2450936,2451361,2451212,2451880,2451059,2451789,2451423,2451576,2450816,2451088,2451728,2451027,2451120,2451881,2451697,2450847,2451271,2451636,2451515,2451424,2451239), isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_hdemo_sk:int,ss_store_sk:int,ss_ticket_number:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+
+(3) Filter [codegen id : 4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#7, d_year#8, d_dom#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1998,1999,2000]), In(d_date_sk, [2451790,2451119,2451180,2451454,2450874,2450906,2450967,2451485,2451850,2451514,2451270,2451758,2451028,2451546,2450997,2450996,2451393,2451667,2451453,2451819,2450905,2451331,2451577,2451089,2451301,2451545,2451605,2451851,2451181,2451149,2451820,2451362,2451392,2451240,2450935,2451637,2451484,2451058,2451300,2451727,2451759,2450815,2451698,2451150,2451332,2451606,2451666,2451211,2450846,2450875,2450966,2450936,2451361,2451212,2451880,2451059,2451789,2451423,2451576,2450816,2451088,2451728,2451027,2451120,2451881,2451697,2450847,2451271,2451636,2451515,2451424,2451239]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_dom:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+Condition : (((((isnotnull(d_dom#9) AND (d_dom#9 >= 1)) AND (d_dom#9 <= 2)) AND d_year#8 IN (1998,1999,2000)) AND d_date_sk#7 INSET (2451790,2451119,2451180,2451454,2450874,2450906,2450967,2451485,2451850,2451514,2451270,2451758,2451028,2451546,2450997,2450996,2451393,2451667,2451453,2451819,2450905,2451331,2451577,2451089,2451301,2451545,2451605,2451851,2451181,2451149,2451820,2451362,2451392,2451240,2450935,2451637,2451484,2451058,2451300,2451727,2451759,2450815,2451698,2451150,2451332,2451606,2451666,2451211,2450846,2450875,2450966,2450936,2451361,2451212,2451880,2451059,2451789,2451423,2451576,2450816,2451088,2451728,2451027,2451120,2451881,2451697,2450847,2451271,2451636,2451515,2451424,2451239)) AND isnotnull(d_date_sk#7))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#7]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#5]
+Right keys [1]: [d_date_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4]
+Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#7]
+
+(11) Scan parquet default.store
+Output [2]: [s_store_sk#11, s_county#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [In(s_county, [Fairfield County,Ziebach County,Bronx County,Barrow County]), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_county:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [s_store_sk#11, s_county#12]
+
+(13) Filter [codegen id : 2]
+Input [2]: [s_store_sk#11, s_county#12]
+Condition : (s_county#12 IN (Fairfield County,Ziebach County,Bronx County,Barrow County) AND isnotnull(s_store_sk#11))
+
+(14) Project [codegen id : 2]
+Output [1]: [s_store_sk#11]
+Input [2]: [s_store_sk#11, s_county#12]
+
+(15) BroadcastExchange
+Input [1]: [s_store_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#11]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#11]
+
+(18) Scan parquet default.household_demographics
+Output [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_vehicle_count), IsNotNull(hd_dep_count), Or(EqualTo(hd_buy_potential,>10000         ),EqualTo(hd_buy_potential,Unknown        )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_buy_potential:string,hd_dep_count:int,hd_vehicle_count:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+
+(20) Filter [codegen id : 3]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+Condition : (((((isnotnull(hd_vehicle_count#17) AND isnotnull(hd_dep_count#16)) AND ((hd_buy_potential#15 = >10000         ) OR (hd_buy_potential#15 = Unknown        ))) AND (hd_vehicle_count#17 > 0)) AND ((cast(hd_dep_count#16 as double) / cast(hd_vehicle_count#17 as double)) > 1.0)) AND isnotnull(hd_demo_sk#14))
+
+(21) Project [codegen id : 3]
+Output [1]: [hd_demo_sk#14]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+
+(22) BroadcastExchange
+Input [1]: [hd_demo_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(23) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#14]
+Join condition: None
+
+(24) Project [codegen id : 4]
+Output [2]: [ss_customer_sk#1, ss_ticket_number#4]
+Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#14]
+
+(25) HashAggregate [codegen id : 4]
+Input [2]: [ss_customer_sk#1, ss_ticket_number#4]
+Keys [2]: [ss_ticket_number#4, ss_customer_sk#1]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#19]
+Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+
+(26) Exchange
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(27) HashAggregate [codegen id : 6]
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+Keys [2]: [ss_ticket_number#4, ss_customer_sk#1]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#22]
+Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count(1)#22 AS cnt#23]
+
+(28) Filter [codegen id : 6]
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23]
+Condition : ((cnt#23 >= 1) AND (cnt#23 <= 5))
+
+(29) Scan parquet default.customer
+Output [5]: [c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_salutation:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string>
+
+(30) ColumnarToRow [codegen id : 5]
+Input [5]: [c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+
+(31) Filter [codegen id : 5]
+Input [5]: [c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+Condition : isnotnull(c_customer_sk#24)
+
+(32) BroadcastExchange
+Input [5]: [c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#29]
+
+(33) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#24]
+Join condition: None
+
+(34) Project [codegen id : 6]
+Output [6]: [c_last_name#27, c_first_name#26, c_salutation#25, c_preferred_cust_flag#28, ss_ticket_number#4, cnt#23]
+Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23, c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+
+(35) Exchange
+Input [6]: [c_last_name#27, c_first_name#26, c_salutation#25, c_preferred_cust_flag#28, ss_ticket_number#4, cnt#23]
+Arguments: rangepartitioning(cnt#23 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(36) Sort [codegen id : 7]
+Input [6]: [c_last_name#27, c_first_name#26, c_salutation#25, c_preferred_cust_flag#28, ss_ticket_number#4, cnt#23]
+Arguments: [cnt#23 DESC NULLS LAST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (37)
+
+
+(37) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#7]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q73/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q73/simplified.txt
new file mode 100644
index 0000000000000..5bb7daa2bda9c
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q73/simplified.txt
@@ -0,0 +1,56 @@
+WholeStageCodegen (7)
+  Sort [cnt]
+    InputAdapter
+      Exchange [cnt] #1
+        WholeStageCodegen (6)
+          Project [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt]
+            BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+              Filter [cnt]
+                HashAggregate [ss_ticket_number,ss_customer_sk,count] [count(1),cnt,count]
+                  InputAdapter
+                    Exchange [ss_ticket_number,ss_customer_sk] #2
+                      WholeStageCodegen (4)
+                        HashAggregate [ss_ticket_number,ss_customer_sk] [count,count]
+                          Project [ss_customer_sk,ss_ticket_number]
+                            BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                              Project [ss_customer_sk,ss_hdemo_sk,ss_ticket_number]
+                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                  Project [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number]
+                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                      Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #1
+                                                ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        BroadcastExchange #3
+                                          WholeStageCodegen (1)
+                                            Project [d_date_sk]
+                                              Filter [d_dom,d_year,d_date_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_date_sk,d_year,d_dom]
+                                  InputAdapter
+                                    BroadcastExchange #4
+                                      WholeStageCodegen (2)
+                                        Project [s_store_sk]
+                                          Filter [s_county,s_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store [s_store_sk,s_county]
+                              InputAdapter
+                                BroadcastExchange #5
+                                  WholeStageCodegen (3)
+                                    Project [hd_demo_sk]
+                                      Filter [hd_vehicle_count,hd_dep_count,hd_buy_potential,hd_demo_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count]
+              InputAdapter
+                BroadcastExchange #6
+                  WholeStageCodegen (5)
+                    Filter [c_customer_sk]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q79.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q79.sf100/explain.txt
new file mode 100644
index 0000000000000..b06961eafc6e7
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q79.sf100/explain.txt
@@ -0,0 +1,219 @@
+== Physical Plan ==
+TakeOrderedAndProject (37)
++- * Project (36)
+   +- * SortMergeJoin Inner (35)
+      :- * Sort (29)
+      :  +- Exchange (28)
+      :     +- * HashAggregate (27)
+      :        +- Exchange (26)
+      :           +- * HashAggregate (25)
+      :              +- * Project (24)
+      :                 +- * BroadcastHashJoin Inner BuildRight (23)
+      :                    :- * Project (17)
+      :                    :  +- * BroadcastHashJoin Inner BuildRight (16)
+      :                    :     :- * Project (10)
+      :                    :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+      :                    :     :     :- * Filter (3)
+      :                    :     :     :  +- * ColumnarToRow (2)
+      :                    :     :     :     +- Scan parquet default.store_sales (1)
+      :                    :     :     +- BroadcastExchange (8)
+      :                    :     :        +- * Project (7)
+      :                    :     :           +- * Filter (6)
+      :                    :     :              +- * ColumnarToRow (5)
+      :                    :     :                 +- Scan parquet default.date_dim (4)
+      :                    :     +- BroadcastExchange (15)
+      :                    :        +- * Project (14)
+      :                    :           +- * Filter (13)
+      :                    :              +- * ColumnarToRow (12)
+      :                    :                 +- Scan parquet default.household_demographics (11)
+      :                    +- BroadcastExchange (22)
+      :                       +- * Project (21)
+      :                          +- * Filter (20)
+      :                             +- * ColumnarToRow (19)
+      :                                +- Scan parquet default.store (18)
+      +- * Sort (34)
+         +- Exchange (33)
+            +- * Filter (32)
+               +- * ColumnarToRow (31)
+                  +- Scan parquet default.customer (30)
+
+
+(1) Scan parquet default.store_sales
+Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), (ss_sold_date_sk#8 >= 2450819), (ss_sold_date_sk#8 <= 2451904), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_hdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_coupon_amt:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8]
+
+(3) Filter [codegen id : 4]
+Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8]
+Condition : ((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#10, d_year#11, d_dow#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_dow), EqualTo(d_dow,1), In(d_year, [1998,1999,2000]), GreaterThanOrEqual(d_date_sk,2450819), LessThanOrEqual(d_date_sk,2451904), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_dow:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#10, d_year#11, d_dow#12]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#10, d_year#11, d_dow#12]
+Condition : (((((isnotnull(d_dow#12) AND (d_dow#12 = 1)) AND d_year#11 IN (1998,1999,2000)) AND (d_date_sk#10 >= 2450819)) AND (d_date_sk#10 <= 2451904)) AND isnotnull(d_date_sk#10))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#10]
+Input [3]: [d_date_sk#10, d_year#11, d_dow#12]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7]
+Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#10]
+
+(11) Scan parquet default.household_demographics
+Output [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [Or(EqualTo(hd_dep_count,8),GreaterThan(hd_vehicle_count,0)), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_dep_count:int,hd_vehicle_count:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16]
+
+(13) Filter [codegen id : 2]
+Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16]
+Condition : (((hd_dep_count#15 = 8) OR (hd_vehicle_count#16 > 0)) AND isnotnull(hd_demo_sk#14))
+
+(14) Project [codegen id : 2]
+Output [1]: [hd_demo_sk#14]
+Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16]
+
+(15) BroadcastExchange
+Input [1]: [hd_demo_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#14]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7]
+Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, hd_demo_sk#14]
+
+(18) Scan parquet default.store
+Output [3]: [s_store_sk#18, s_number_employees#19, s_city#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_number_employees), GreaterThanOrEqual(s_number_employees,200), LessThanOrEqual(s_number_employees,295), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_number_employees:int,s_city:string>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [3]: [s_store_sk#18, s_number_employees#19, s_city#20]
+
+(20) Filter [codegen id : 3]
+Input [3]: [s_store_sk#18, s_number_employees#19, s_city#20]
+Condition : (((isnotnull(s_number_employees#19) AND (s_number_employees#19 >= 200)) AND (s_number_employees#19 <= 295)) AND isnotnull(s_store_sk#18))
+
+(21) Project [codegen id : 3]
+Output [2]: [s_store_sk#18, s_city#20]
+Input [3]: [s_store_sk#18, s_number_employees#19, s_city#20]
+
+(22) BroadcastExchange
+Input [2]: [s_store_sk#18, s_city#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(23) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#4]
+Right keys [1]: [s_store_sk#18]
+Join condition: None
+
+(24) Project [codegen id : 4]
+Output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#20]
+Input [8]: [ss_customer_sk#1, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#18, s_city#20]
+
+(25) HashAggregate [codegen id : 4]
+Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#20]
+Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#20]
+Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))]
+Aggregate Attributes [2]: [sum#22, sum#23]
+Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#20, sum#24, sum#25]
+
+(26) Exchange
+Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#20, sum#24, sum#25]
+Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#20, 5), ENSURE_REQUIREMENTS, [id=#26]
+
+(27) HashAggregate [codegen id : 5]
+Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#20, sum#24, sum#25]
+Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#20]
+Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#27, sum(UnscaledValue(ss_net_profit#7))#28]
+Results [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#20, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#27,17,2) AS amt#29, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#28,17,2) AS profit#30]
+
+(28) Exchange
+Input [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#20, amt#29, profit#30]
+Arguments: hashpartitioning(ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(29) Sort [codegen id : 6]
+Input [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#20, amt#29, profit#30]
+Arguments: [ss_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(30) Scan parquet default.customer
+Output [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_first_name:string,c_last_name:string>
+
+(31) ColumnarToRow [codegen id : 7]
+Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34]
+
+(32) Filter [codegen id : 7]
+Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34]
+Condition : isnotnull(c_customer_sk#32)
+
+(33) Exchange
+Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34]
+Arguments: hashpartitioning(c_customer_sk#32, 5), ENSURE_REQUIREMENTS, [id=#35]
+
+(34) Sort [codegen id : 8]
+Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34]
+Arguments: [c_customer_sk#32 ASC NULLS FIRST], false, 0
+
+(35) SortMergeJoin [codegen id : 9]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#32]
+Join condition: None
+
+(36) Project [codegen id : 9]
+Output [7]: [c_last_name#34, c_first_name#33, substr(s_city#20, 1, 30) AS substr(s_city, 1, 30)#36, ss_ticket_number#5, amt#29, profit#30, s_city#20]
+Input [8]: [ss_ticket_number#5, ss_customer_sk#1, s_city#20, amt#29, profit#30, c_customer_sk#32, c_first_name#33, c_last_name#34]
+
+(37) TakeOrderedAndProject
+Input [7]: [c_last_name#34, c_first_name#33, substr(s_city, 1, 30)#36, ss_ticket_number#5, amt#29, profit#30, s_city#20]
+Arguments: 100, [c_last_name#34 ASC NULLS FIRST, c_first_name#33 ASC NULLS FIRST, substr(s_city#20, 1, 30) ASC NULLS FIRST, profit#30 ASC NULLS FIRST], [c_last_name#34, c_first_name#33, substr(s_city, 1, 30)#36, ss_ticket_number#5, amt#29, profit#30]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (38)
+
+
+(38) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#10]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q79.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q79.sf100/simplified.txt
new file mode 100644
index 0000000000000..bac3b763658f8
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q79.sf100/simplified.txt
@@ -0,0 +1,61 @@
+TakeOrderedAndProject [c_last_name,c_first_name,s_city,profit,substr(s_city, 1, 30),ss_ticket_number,amt]
+  WholeStageCodegen (9)
+    Project [c_last_name,c_first_name,s_city,ss_ticket_number,amt,profit]
+      SortMergeJoin [ss_customer_sk,c_customer_sk]
+        InputAdapter
+          WholeStageCodegen (6)
+            Sort [ss_customer_sk]
+              InputAdapter
+                Exchange [ss_customer_sk] #1
+                  WholeStageCodegen (5)
+                    HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,sum,sum] [sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit)),amt,profit,sum,sum]
+                      InputAdapter
+                        Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city] #2
+                          WholeStageCodegen (4)
+                            HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,ss_coupon_amt,ss_net_profit] [sum,sum,sum,sum]
+                              Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city]
+                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                  Project [ss_customer_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit]
+                                    BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                      Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit]
+                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                          Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk]
+                                                  SubqueryBroadcast [d_date_sk] #1
+                                                    ReusedExchange [d_date_sk] #3
+                                          InputAdapter
+                                            BroadcastExchange #3
+                                              WholeStageCodegen (1)
+                                                Project [d_date_sk]
+                                                  Filter [d_dow,d_year,d_date_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.date_dim [d_date_sk,d_year,d_dow]
+                                      InputAdapter
+                                        BroadcastExchange #4
+                                          WholeStageCodegen (2)
+                                            Project [hd_demo_sk]
+                                              Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count]
+                                  InputAdapter
+                                    BroadcastExchange #5
+                                      WholeStageCodegen (3)
+                                        Project [s_store_sk,s_city]
+                                          Filter [s_number_employees,s_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store [s_store_sk,s_number_employees,s_city]
+        InputAdapter
+          WholeStageCodegen (8)
+            Sort [c_customer_sk]
+              InputAdapter
+                Exchange [c_customer_sk] #6
+                  WholeStageCodegen (7)
+                    Filter [c_customer_sk]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.customer [c_customer_sk,c_first_name,c_last_name]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q79/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q79/explain.txt
new file mode 100644
index 0000000000000..8162673563ea2
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q79/explain.txt
@@ -0,0 +1,204 @@
+== Physical Plan ==
+TakeOrderedAndProject (34)
++- * Project (33)
+   +- * BroadcastHashJoin Inner BuildRight (32)
+      :- * HashAggregate (27)
+      :  +- Exchange (26)
+      :     +- * HashAggregate (25)
+      :        +- * Project (24)
+      :           +- * BroadcastHashJoin Inner BuildRight (23)
+      :              :- * Project (17)
+      :              :  +- * BroadcastHashJoin Inner BuildRight (16)
+      :              :     :- * Project (10)
+      :              :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+      :              :     :     :- * Filter (3)
+      :              :     :     :  +- * ColumnarToRow (2)
+      :              :     :     :     +- Scan parquet default.store_sales (1)
+      :              :     :     +- BroadcastExchange (8)
+      :              :     :        +- * Project (7)
+      :              :     :           +- * Filter (6)
+      :              :     :              +- * ColumnarToRow (5)
+      :              :     :                 +- Scan parquet default.date_dim (4)
+      :              :     +- BroadcastExchange (15)
+      :              :        +- * Project (14)
+      :              :           +- * Filter (13)
+      :              :              +- * ColumnarToRow (12)
+      :              :                 +- Scan parquet default.store (11)
+      :              +- BroadcastExchange (22)
+      :                 +- * Project (21)
+      :                    +- * Filter (20)
+      :                       +- * ColumnarToRow (19)
+      :                          +- Scan parquet default.household_demographics (18)
+      +- BroadcastExchange (31)
+         +- * Filter (30)
+            +- * ColumnarToRow (29)
+               +- Scan parquet default.customer (28)
+
+
+(1) Scan parquet default.store_sales
+Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), (ss_sold_date_sk#8 >= 2450819), (ss_sold_date_sk#8 <= 2451904), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_hdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_coupon_amt:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8]
+
+(3) Filter [codegen id : 4]
+Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8]
+Condition : ((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#10, d_year#11, d_dow#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_dow), EqualTo(d_dow,1), In(d_year, [1998,1999,2000]), GreaterThanOrEqual(d_date_sk,2450819), LessThanOrEqual(d_date_sk,2451904), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_dow:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#10, d_year#11, d_dow#12]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#10, d_year#11, d_dow#12]
+Condition : (((((isnotnull(d_dow#12) AND (d_dow#12 = 1)) AND d_year#11 IN (1998,1999,2000)) AND (d_date_sk#10 >= 2450819)) AND (d_date_sk#10 <= 2451904)) AND isnotnull(d_date_sk#10))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#10]
+Input [3]: [d_date_sk#10, d_year#11, d_dow#12]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7]
+Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#10]
+
+(11) Scan parquet default.store
+Output [3]: [s_store_sk#14, s_number_employees#15, s_city#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_number_employees), GreaterThanOrEqual(s_number_employees,200), LessThanOrEqual(s_number_employees,295), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_number_employees:int,s_city:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [s_store_sk#14, s_number_employees#15, s_city#16]
+
+(13) Filter [codegen id : 2]
+Input [3]: [s_store_sk#14, s_number_employees#15, s_city#16]
+Condition : (((isnotnull(s_number_employees#15) AND (s_number_employees#15 >= 200)) AND (s_number_employees#15 <= 295)) AND isnotnull(s_store_sk#14))
+
+(14) Project [codegen id : 2]
+Output [2]: [s_store_sk#14, s_city#16]
+Input [3]: [s_store_sk#14, s_number_employees#15, s_city#16]
+
+(15) BroadcastExchange
+Input [2]: [s_store_sk#14, s_city#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#4]
+Right keys [1]: [s_store_sk#14]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#16]
+Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#14, s_city#16]
+
+(18) Scan parquet default.household_demographics
+Output [3]: [hd_demo_sk#18, hd_dep_count#19, hd_vehicle_count#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [Or(EqualTo(hd_dep_count,8),GreaterThan(hd_vehicle_count,0)), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_dep_count:int,hd_vehicle_count:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [3]: [hd_demo_sk#18, hd_dep_count#19, hd_vehicle_count#20]
+
+(20) Filter [codegen id : 3]
+Input [3]: [hd_demo_sk#18, hd_dep_count#19, hd_vehicle_count#20]
+Condition : (((hd_dep_count#19 = 8) OR (hd_vehicle_count#20 > 0)) AND isnotnull(hd_demo_sk#18))
+
+(21) Project [codegen id : 3]
+Output [1]: [hd_demo_sk#18]
+Input [3]: [hd_demo_sk#18, hd_dep_count#19, hd_vehicle_count#20]
+
+(22) BroadcastExchange
+Input [1]: [hd_demo_sk#18]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(23) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#18]
+Join condition: None
+
+(24) Project [codegen id : 4]
+Output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#16]
+Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#16, hd_demo_sk#18]
+
+(25) HashAggregate [codegen id : 4]
+Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#16]
+Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#16]
+Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))]
+Aggregate Attributes [2]: [sum#22, sum#23]
+Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#16, sum#24, sum#25]
+
+(26) Exchange
+Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#16, sum#24, sum#25]
+Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#16, 5), ENSURE_REQUIREMENTS, [id=#26]
+
+(27) HashAggregate [codegen id : 6]
+Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#16, sum#24, sum#25]
+Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#16]
+Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#27, sum(UnscaledValue(ss_net_profit#7))#28]
+Results [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#16, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#27,17,2) AS amt#29, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#28,17,2) AS profit#30]
+
+(28) Scan parquet default.customer
+Output [3]: [c_customer_sk#31, c_first_name#32, c_last_name#33]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_first_name:string,c_last_name:string>
+
+(29) ColumnarToRow [codegen id : 5]
+Input [3]: [c_customer_sk#31, c_first_name#32, c_last_name#33]
+
+(30) Filter [codegen id : 5]
+Input [3]: [c_customer_sk#31, c_first_name#32, c_last_name#33]
+Condition : isnotnull(c_customer_sk#31)
+
+(31) BroadcastExchange
+Input [3]: [c_customer_sk#31, c_first_name#32, c_last_name#33]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#34]
+
+(32) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#31]
+Join condition: None
+
+(33) Project [codegen id : 6]
+Output [7]: [c_last_name#33, c_first_name#32, substr(s_city#16, 1, 30) AS substr(s_city, 1, 30)#35, ss_ticket_number#5, amt#29, profit#30, s_city#16]
+Input [8]: [ss_ticket_number#5, ss_customer_sk#1, s_city#16, amt#29, profit#30, c_customer_sk#31, c_first_name#32, c_last_name#33]
+
+(34) TakeOrderedAndProject
+Input [7]: [c_last_name#33, c_first_name#32, substr(s_city, 1, 30)#35, ss_ticket_number#5, amt#29, profit#30, s_city#16]
+Arguments: 100, [c_last_name#33 ASC NULLS FIRST, c_first_name#32 ASC NULLS FIRST, substr(s_city#16, 1, 30) ASC NULLS FIRST, profit#30 ASC NULLS FIRST], [c_last_name#33, c_first_name#32, substr(s_city, 1, 30)#35, ss_ticket_number#5, amt#29, profit#30]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (35)
+
+
+(35) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#10]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q79/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q79/simplified.txt
new file mode 100644
index 0000000000000..6432bc55bd8f3
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q79/simplified.txt
@@ -0,0 +1,52 @@
+TakeOrderedAndProject [c_last_name,c_first_name,s_city,profit,substr(s_city, 1, 30),ss_ticket_number,amt]
+  WholeStageCodegen (6)
+    Project [c_last_name,c_first_name,s_city,ss_ticket_number,amt,profit]
+      BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+        HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,sum,sum] [sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit)),amt,profit,sum,sum]
+          InputAdapter
+            Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city] #1
+              WholeStageCodegen (4)
+                HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,ss_coupon_amt,ss_net_profit] [sum,sum,sum,sum]
+                  Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city]
+                    BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                      Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city]
+                        BroadcastHashJoin [ss_store_sk,s_store_sk]
+                          Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit]
+                            BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                              Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk]
+                                      SubqueryBroadcast [d_date_sk] #1
+                                        ReusedExchange [d_date_sk] #2
+                              InputAdapter
+                                BroadcastExchange #2
+                                  WholeStageCodegen (1)
+                                    Project [d_date_sk]
+                                      Filter [d_dow,d_year,d_date_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.date_dim [d_date_sk,d_year,d_dow]
+                          InputAdapter
+                            BroadcastExchange #3
+                              WholeStageCodegen (2)
+                                Project [s_store_sk,s_city]
+                                  Filter [s_number_employees,s_store_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.store [s_store_sk,s_number_employees,s_city]
+                      InputAdapter
+                        BroadcastExchange #4
+                          WholeStageCodegen (3)
+                            Project [hd_demo_sk]
+                              Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count]
+        InputAdapter
+          BroadcastExchange #5
+            WholeStageCodegen (5)
+              Filter [c_customer_sk]
+                ColumnarToRow
+                  InputAdapter
+                    Scan parquet default.customer [c_customer_sk,c_first_name,c_last_name]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q89.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q89.sf100/explain.txt
new file mode 100644
index 0000000000000..a325bf820a4f9
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q89.sf100/explain.txt
@@ -0,0 +1,186 @@
+== Physical Plan ==
+TakeOrderedAndProject (31)
++- * Project (30)
+   +- * Filter (29)
+      +- Window (28)
+         +- * Sort (27)
+            +- Exchange (26)
+               +- * HashAggregate (25)
+                  +- Exchange (24)
+                     +- * HashAggregate (23)
+                        +- * Project (22)
+                           +- * BroadcastHashJoin Inner BuildRight (21)
+                              :- * Project (16)
+                              :  +- * BroadcastHashJoin Inner BuildRight (15)
+                              :     :- * Project (10)
+                              :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+                              :     :     :- * Filter (3)
+                              :     :     :  +- * ColumnarToRow (2)
+                              :     :     :     +- Scan parquet default.store_sales (1)
+                              :     :     +- BroadcastExchange (8)
+                              :     :        +- * Project (7)
+                              :     :           +- * Filter (6)
+                              :     :              +- * ColumnarToRow (5)
+                              :     :                 +- Scan parquet default.date_dim (4)
+                              :     +- BroadcastExchange (14)
+                              :        +- * Filter (13)
+                              :           +- * ColumnarToRow (12)
+                              :              +- Scan parquet default.store (11)
+                              +- BroadcastExchange (20)
+                                 +- * Filter (19)
+                                    +- * ColumnarToRow (18)
+                                       +- Scan parquet default.item (17)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), (ss_sold_date_sk#4 >= 2451545), (ss_sold_date_sk#4 <= 2451910), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 4]
+Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, ss_sold_date_sk#4]
+Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), GreaterThanOrEqual(d_date_sk,2451545), LessThanOrEqual(d_date_sk,2451910), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Condition : ((((isnotnull(d_year#7) AND (d_year#7 = 2000)) AND (d_date_sk#6 >= 2451545)) AND (d_date_sk#6 <= 2451910)) AND isnotnull(d_date_sk#6))
+
+(7) Project [codegen id : 1]
+Output [2]: [d_date_sk#6, d_moy#8]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+(8) BroadcastExchange
+Input [2]: [d_date_sk#6, d_moy#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#9]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, d_moy#8]
+Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, ss_sold_date_sk#4, d_date_sk#6, d_moy#8]
+
+(11) Scan parquet default.store
+Output [3]: [s_store_sk#10, s_store_name#11, s_company_name#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_company_name:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [s_store_sk#10, s_store_name#11, s_company_name#12]
+
+(13) Filter [codegen id : 2]
+Input [3]: [s_store_sk#10, s_store_name#11, s_company_name#12]
+Condition : isnotnull(s_store_sk#10)
+
+(14) BroadcastExchange
+Input [3]: [s_store_sk#10, s_store_name#11, s_company_name#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#13]
+
+(15) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#2]
+Right keys [1]: [s_store_sk#10]
+Join condition: None
+
+(16) Project [codegen id : 4]
+Output [5]: [ss_item_sk#1, ss_sales_price#3, d_moy#8, s_store_name#11, s_company_name#12]
+Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, d_moy#8, s_store_sk#10, s_store_name#11, s_company_name#12]
+
+(17) Scan parquet default.item
+Output [4]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [Or(And(In(i_category, [Home                                              ,Books                                             ,Electronics                                       ]),In(i_class, [wallpaper                                         ,parenting                                         ,musical                                           ])),And(In(i_category, [Shoes                                             ,Jewelry                                           ,Men                                               ]),In(i_class, [womens                                            ,birdal                                            ,pants                                             ]))), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_class:string,i_category:string>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [4]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17]
+
+(19) Filter [codegen id : 3]
+Input [4]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17]
+Condition : (((i_category#17 IN (Home                                              ,Books                                             ,Electronics                                       ) AND i_class#16 IN (wallpaper                                         ,parenting                                         ,musical                                           )) OR (i_category#17 IN (Shoes                                             ,Jewelry                                           ,Men                                               ) AND i_class#16 IN (womens                                            ,birdal                                            ,pants                                             ))) AND isnotnull(i_item_sk#14))
+
+(20) BroadcastExchange
+Input [4]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#18]
+
+(21) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#14]
+Join condition: None
+
+(22) Project [codegen id : 4]
+Output [7]: [i_brand#15, i_class#16, i_category#17, ss_sales_price#3, d_moy#8, s_store_name#11, s_company_name#12]
+Input [9]: [ss_item_sk#1, ss_sales_price#3, d_moy#8, s_store_name#11, s_company_name#12, i_item_sk#14, i_brand#15, i_class#16, i_category#17]
+
+(23) HashAggregate [codegen id : 4]
+Input [7]: [i_brand#15, i_class#16, i_category#17, ss_sales_price#3, d_moy#8, s_store_name#11, s_company_name#12]
+Keys [6]: [i_category#17, i_class#16, i_brand#15, s_store_name#11, s_company_name#12, d_moy#8]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#3))]
+Aggregate Attributes [1]: [sum#19]
+Results [7]: [i_category#17, i_class#16, i_brand#15, s_store_name#11, s_company_name#12, d_moy#8, sum#20]
+
+(24) Exchange
+Input [7]: [i_category#17, i_class#16, i_brand#15, s_store_name#11, s_company_name#12, d_moy#8, sum#20]
+Arguments: hashpartitioning(i_category#17, i_class#16, i_brand#15, s_store_name#11, s_company_name#12, d_moy#8, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(25) HashAggregate [codegen id : 5]
+Input [7]: [i_category#17, i_class#16, i_brand#15, s_store_name#11, s_company_name#12, d_moy#8, sum#20]
+Keys [6]: [i_category#17, i_class#16, i_brand#15, s_store_name#11, s_company_name#12, d_moy#8]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#3))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#3))#22]
+Results [8]: [i_category#17, i_class#16, i_brand#15, s_store_name#11, s_company_name#12, d_moy#8, MakeDecimal(sum(UnscaledValue(ss_sales_price#3))#22,17,2) AS sum_sales#23, MakeDecimal(sum(UnscaledValue(ss_sales_price#3))#22,17,2) AS _w0#24]
+
+(26) Exchange
+Input [8]: [i_category#17, i_class#16, i_brand#15, s_store_name#11, s_company_name#12, d_moy#8, sum_sales#23, _w0#24]
+Arguments: hashpartitioning(i_category#17, i_brand#15, s_store_name#11, s_company_name#12, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(27) Sort [codegen id : 6]
+Input [8]: [i_category#17, i_class#16, i_brand#15, s_store_name#11, s_company_name#12, d_moy#8, sum_sales#23, _w0#24]
+Arguments: [i_category#17 ASC NULLS FIRST, i_brand#15 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, s_company_name#12 ASC NULLS FIRST], false, 0
+
+(28) Window
+Input [8]: [i_category#17, i_class#16, i_brand#15, s_store_name#11, s_company_name#12, d_moy#8, sum_sales#23, _w0#24]
+Arguments: [avg(_w0#24) windowspecdefinition(i_category#17, i_brand#15, s_store_name#11, s_company_name#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#26], [i_category#17, i_brand#15, s_store_name#11, s_company_name#12]
+
+(29) Filter [codegen id : 7]
+Input [9]: [i_category#17, i_class#16, i_brand#15, s_store_name#11, s_company_name#12, d_moy#8, sum_sales#23, _w0#24, avg_monthly_sales#26]
+Condition : (isnotnull(avg_monthly_sales#26) AND (NOT (avg_monthly_sales#26 = 0.000000) AND (CheckOverflow((promote_precision(abs(CheckOverflow((promote_precision(cast(sum_sales#23 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#26 as decimal(22,6)))), DecimalType(22,6), true), false)) / promote_precision(cast(avg_monthly_sales#26 as decimal(22,6)))), DecimalType(38,16), true) > 0.1000000000000000)))
+
+(30) Project [codegen id : 7]
+Output [8]: [i_category#17, i_class#16, i_brand#15, s_store_name#11, s_company_name#12, d_moy#8, sum_sales#23, avg_monthly_sales#26]
+Input [9]: [i_category#17, i_class#16, i_brand#15, s_store_name#11, s_company_name#12, d_moy#8, sum_sales#23, _w0#24, avg_monthly_sales#26]
+
+(31) TakeOrderedAndProject
+Input [8]: [i_category#17, i_class#16, i_brand#15, s_store_name#11, s_company_name#12, d_moy#8, sum_sales#23, avg_monthly_sales#26]
+Arguments: 100, [CheckOverflow((promote_precision(cast(sum_sales#23 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#26 as decimal(22,6)))), DecimalType(22,6), true) ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST], [i_category#17, i_class#16, i_brand#15, s_store_name#11, s_company_name#12, d_moy#8, sum_sales#23, avg_monthly_sales#26]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (32)
+
+
+(32) ReusedExchange [Reuses operator id: 8]
+Output [2]: [d_date_sk#6, d_moy#8]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q89.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q89.sf100/simplified.txt
new file mode 100644
index 0000000000000..c77c34f4408d9
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q89.sf100/simplified.txt
@@ -0,0 +1,50 @@
+TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_class,i_brand,s_company_name,d_moy]
+  WholeStageCodegen (7)
+    Project [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,avg_monthly_sales]
+      Filter [avg_monthly_sales,sum_sales]
+        InputAdapter
+          Window [_w0,i_category,i_brand,s_store_name,s_company_name]
+            WholeStageCodegen (6)
+              Sort [i_category,i_brand,s_store_name,s_company_name]
+                InputAdapter
+                  Exchange [i_category,i_brand,s_store_name,s_company_name] #1
+                    WholeStageCodegen (5)
+                      HashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum]
+                        InputAdapter
+                          Exchange [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy] #2
+                            WholeStageCodegen (4)
+                              HashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,ss_sales_price] [sum,sum]
+                                Project [i_brand,i_class,i_category,ss_sales_price,d_moy,s_store_name,s_company_name]
+                                  BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                    Project [ss_item_sk,ss_sales_price,d_moy,s_store_name,s_company_name]
+                                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                        Project [ss_item_sk,ss_store_sk,ss_sales_price,d_moy]
+                                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                            Filter [ss_item_sk,ss_store_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #1
+                                                      ReusedExchange [d_date_sk,d_moy] #3
+                                            InputAdapter
+                                              BroadcastExchange #3
+                                                WholeStageCodegen (1)
+                                                  Project [d_date_sk,d_moy]
+                                                    Filter [d_year,d_date_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                        InputAdapter
+                                          BroadcastExchange #4
+                                            WholeStageCodegen (2)
+                                              Filter [s_store_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.store [s_store_sk,s_store_name,s_company_name]
+                                    InputAdapter
+                                      BroadcastExchange #5
+                                        WholeStageCodegen (3)
+                                          Filter [i_category,i_class,i_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.item [i_item_sk,i_brand,i_class,i_category]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q89/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q89/explain.txt
new file mode 100644
index 0000000000000..2ec1ea034361a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q89/explain.txt
@@ -0,0 +1,186 @@
+== Physical Plan ==
+TakeOrderedAndProject (31)
++- * Project (30)
+   +- * Filter (29)
+      +- Window (28)
+         +- * Sort (27)
+            +- Exchange (26)
+               +- * HashAggregate (25)
+                  +- Exchange (24)
+                     +- * HashAggregate (23)
+                        +- * Project (22)
+                           +- * BroadcastHashJoin Inner BuildRight (21)
+                              :- * Project (16)
+                              :  +- * BroadcastHashJoin Inner BuildRight (15)
+                              :     :- * Project (9)
+                              :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+                              :     :     :- * Filter (3)
+                              :     :     :  +- * ColumnarToRow (2)
+                              :     :     :     +- Scan parquet default.item (1)
+                              :     :     +- BroadcastExchange (7)
+                              :     :        +- * Filter (6)
+                              :     :           +- * ColumnarToRow (5)
+                              :     :              +- Scan parquet default.store_sales (4)
+                              :     +- BroadcastExchange (14)
+                              :        +- * Project (13)
+                              :           +- * Filter (12)
+                              :              +- * ColumnarToRow (11)
+                              :                 +- Scan parquet default.date_dim (10)
+                              +- BroadcastExchange (20)
+                                 +- * Filter (19)
+                                    +- * ColumnarToRow (18)
+                                       +- Scan parquet default.store (17)
+
+
+(1) Scan parquet default.item
+Output [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [Or(And(In(i_category, [Home                                              ,Books                                             ,Electronics                                       ]),In(i_class, [wallpaper                                         ,parenting                                         ,musical                                           ])),And(In(i_category, [Shoes                                             ,Jewelry                                           ,Men                                               ]),In(i_class, [womens                                            ,birdal                                            ,pants                                             ]))), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_class:string,i_category:string>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4]
+
+(3) Filter [codegen id : 4]
+Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4]
+Condition : (((i_category#4 IN (Home                                              ,Books                                             ,Electronics                                       ) AND i_class#3 IN (wallpaper                                         ,parenting                                         ,musical                                           )) OR (i_category#4 IN (Shoes                                             ,Jewelry                                           ,Men                                               ) AND i_class#3 IN (womens                                            ,birdal                                            ,pants                                             ))) AND isnotnull(i_item_sk#1))
+
+(4) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), (ss_sold_date_sk#8 >= 2451545), (ss_sold_date_sk#8 <= 2451910), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8]
+
+(6) Filter [codegen id : 1]
+Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8]
+Condition : (isnotnull(ss_item_sk#5) AND isnotnull(ss_store_sk#6))
+
+(7) BroadcastExchange
+Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#10]
+
+(8) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [ss_item_sk#5]
+Join condition: None
+
+(9) Project [codegen id : 4]
+Output [6]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8]
+Input [8]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8]
+
+(10) Scan parquet default.date_dim
+Output [3]: [d_date_sk#11, d_year#12, d_moy#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), GreaterThanOrEqual(d_date_sk,2451545), LessThanOrEqual(d_date_sk,2451910), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [3]: [d_date_sk#11, d_year#12, d_moy#13]
+
+(12) Filter [codegen id : 2]
+Input [3]: [d_date_sk#11, d_year#12, d_moy#13]
+Condition : ((((isnotnull(d_year#12) AND (d_year#12 = 2000)) AND (d_date_sk#11 >= 2451545)) AND (d_date_sk#11 <= 2451910)) AND isnotnull(d_date_sk#11))
+
+(13) Project [codegen id : 2]
+Output [2]: [d_date_sk#11, d_moy#13]
+Input [3]: [d_date_sk#11, d_year#12, d_moy#13]
+
+(14) BroadcastExchange
+Input [2]: [d_date_sk#11, d_moy#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(15) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#11]
+Join condition: None
+
+(16) Project [codegen id : 4]
+Output [6]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#13]
+Input [8]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8, d_date_sk#11, d_moy#13]
+
+(17) Scan parquet default.store
+Output [3]: [s_store_sk#15, s_store_name#16, s_company_name#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_company_name:string>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17]
+
+(19) Filter [codegen id : 3]
+Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17]
+Condition : isnotnull(s_store_sk#15)
+
+(20) BroadcastExchange
+Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#18]
+
+(21) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#6]
+Right keys [1]: [s_store_sk#15]
+Join condition: None
+
+(22) Project [codegen id : 4]
+Output [7]: [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#13, s_store_name#16, s_company_name#17]
+Input [9]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#13, s_store_sk#15, s_store_name#16, s_company_name#17]
+
+(23) HashAggregate [codegen id : 4]
+Input [7]: [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#13, s_store_name#16, s_company_name#17]
+Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#7))]
+Aggregate Attributes [1]: [sum#19]
+Results [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum#20]
+
+(24) Exchange
+Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum#20]
+Arguments: hashpartitioning(i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(25) HashAggregate [codegen id : 5]
+Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum#20]
+Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#7))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#7))#22]
+Results [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, MakeDecimal(sum(UnscaledValue(ss_sales_price#7))#22,17,2) AS sum_sales#23, MakeDecimal(sum(UnscaledValue(ss_sales_price#7))#22,17,2) AS _w0#24]
+
+(26) Exchange
+Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum_sales#23, _w0#24]
+Arguments: hashpartitioning(i_category#4, i_brand#2, s_store_name#16, s_company_name#17, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(27) Sort [codegen id : 6]
+Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum_sales#23, _w0#24]
+Arguments: [i_category#4 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#16 ASC NULLS FIRST, s_company_name#17 ASC NULLS FIRST], false, 0
+
+(28) Window
+Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum_sales#23, _w0#24]
+Arguments: [avg(_w0#24) windowspecdefinition(i_category#4, i_brand#2, s_store_name#16, s_company_name#17, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#26], [i_category#4, i_brand#2, s_store_name#16, s_company_name#17]
+
+(29) Filter [codegen id : 7]
+Input [9]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum_sales#23, _w0#24, avg_monthly_sales#26]
+Condition : (isnotnull(avg_monthly_sales#26) AND (NOT (avg_monthly_sales#26 = 0.000000) AND (CheckOverflow((promote_precision(abs(CheckOverflow((promote_precision(cast(sum_sales#23 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#26 as decimal(22,6)))), DecimalType(22,6), true), false)) / promote_precision(cast(avg_monthly_sales#26 as decimal(22,6)))), DecimalType(38,16), true) > 0.1000000000000000)))
+
+(30) Project [codegen id : 7]
+Output [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum_sales#23, avg_monthly_sales#26]
+Input [9]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum_sales#23, _w0#24, avg_monthly_sales#26]
+
+(31) TakeOrderedAndProject
+Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum_sales#23, avg_monthly_sales#26]
+Arguments: 100, [CheckOverflow((promote_precision(cast(sum_sales#23 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#26 as decimal(22,6)))), DecimalType(22,6), true) ASC NULLS FIRST, s_store_name#16 ASC NULLS FIRST], [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum_sales#23, avg_monthly_sales#26]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (32)
+
+
+(32) ReusedExchange [Reuses operator id: 14]
+Output [2]: [d_date_sk#11, d_moy#13]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q89/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q89/simplified.txt
new file mode 100644
index 0000000000000..b96445eec6223
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q89/simplified.txt
@@ -0,0 +1,50 @@
+TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_class,i_brand,s_company_name,d_moy]
+  WholeStageCodegen (7)
+    Project [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,avg_monthly_sales]
+      Filter [avg_monthly_sales,sum_sales]
+        InputAdapter
+          Window [_w0,i_category,i_brand,s_store_name,s_company_name]
+            WholeStageCodegen (6)
+              Sort [i_category,i_brand,s_store_name,s_company_name]
+                InputAdapter
+                  Exchange [i_category,i_brand,s_store_name,s_company_name] #1
+                    WholeStageCodegen (5)
+                      HashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum]
+                        InputAdapter
+                          Exchange [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy] #2
+                            WholeStageCodegen (4)
+                              HashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,ss_sales_price] [sum,sum]
+                                Project [i_brand,i_class,i_category,ss_sales_price,d_moy,s_store_name,s_company_name]
+                                  BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                    Project [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy]
+                                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                        Project [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                          BroadcastHashJoin [i_item_sk,ss_item_sk]
+                                            Filter [i_category,i_class,i_item_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.item [i_item_sk,i_brand,i_class,i_category]
+                                            InputAdapter
+                                              BroadcastExchange #3
+                                                WholeStageCodegen (1)
+                                                  Filter [ss_item_sk,ss_store_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                                          SubqueryBroadcast [d_date_sk] #1
+                                                            ReusedExchange [d_date_sk,d_moy] #4
+                                        InputAdapter
+                                          BroadcastExchange #4
+                                            WholeStageCodegen (2)
+                                              Project [d_date_sk,d_moy]
+                                                Filter [d_year,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                    InputAdapter
+                                      BroadcastExchange #5
+                                        WholeStageCodegen (3)
+                                          Filter [s_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store [s_store_sk,s_store_name,s_company_name]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q98.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q98.sf100/explain.txt
new file mode 100644
index 0000000000000..9d369b5d211fc
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q98.sf100/explain.txt
@@ -0,0 +1,173 @@
+== Physical Plan ==
+* Project (29)
++- * Sort (28)
+   +- Exchange (27)
+      +- * Project (26)
+         +- Window (25)
+            +- * Sort (24)
+               +- Exchange (23)
+                  +- * HashAggregate (22)
+                     +- Exchange (21)
+                        +- * HashAggregate (20)
+                           +- * Project (19)
+                              +- * SortMergeJoin Inner (18)
+                                 :- * Sort (12)
+                                 :  +- Exchange (11)
+                                 :     +- * Project (10)
+                                 :        +- * BroadcastHashJoin Inner BuildRight (9)
+                                 :           :- * Filter (3)
+                                 :           :  +- * ColumnarToRow (2)
+                                 :           :     +- Scan parquet default.store_sales (1)
+                                 :           +- BroadcastExchange (8)
+                                 :              +- * Project (7)
+                                 :                 +- * Filter (6)
+                                 :                    +- * ColumnarToRow (5)
+                                 :                       +- Scan parquet default.date_dim (4)
+                                 +- * Sort (17)
+                                    +- Exchange (16)
+                                       +- * Filter (15)
+                                          +- * ColumnarToRow (14)
+                                             +- Scan parquet default.item (13)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), (ss_sold_date_sk#3 >= 2451911), (ss_sold_date_sk#3 <= 2451941), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 2]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_item_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#5, d_date#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2001-01-01), LessThanOrEqual(d_date,2001-01-31), GreaterThanOrEqual(d_date_sk,2451911), LessThanOrEqual(d_date_sk,2451941), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#5, d_date#6]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#5, d_date#6]
+Condition : (((((isnotnull(d_date#6) AND (d_date#6 >= 2001-01-01)) AND (d_date#6 <= 2001-01-31)) AND (d_date_sk#5 >= 2451911)) AND (d_date_sk#5 <= 2451941)) AND isnotnull(d_date_sk#5))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#5]
+Input [2]: [d_date_sk#5, d_date#6]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#5]
+Join condition: None
+
+(10) Project [codegen id : 2]
+Output [2]: [ss_item_sk#1, ss_ext_sales_price#2]
+Input [4]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, d_date_sk#5]
+
+(11) Exchange
+Input [2]: [ss_item_sk#1, ss_ext_sales_price#2]
+Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#8]
+
+(12) Sort [codegen id : 3]
+Input [2]: [ss_item_sk#1, ss_ext_sales_price#2]
+Arguments: [ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(13) Scan parquet default.item
+Output [6]: [i_item_sk#9, i_item_id#10, i_item_desc#11, i_current_price#12, i_class#13, i_category#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [In(i_category, [Jewelry                                           ,Sports                                            ,Books                                             ]), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_item_desc:string,i_current_price:decimal(7,2),i_class:string,i_category:string>
+
+(14) ColumnarToRow [codegen id : 4]
+Input [6]: [i_item_sk#9, i_item_id#10, i_item_desc#11, i_current_price#12, i_class#13, i_category#14]
+
+(15) Filter [codegen id : 4]
+Input [6]: [i_item_sk#9, i_item_id#10, i_item_desc#11, i_current_price#12, i_class#13, i_category#14]
+Condition : (i_category#14 IN (Jewelry                                           ,Sports                                            ,Books                                             ) AND isnotnull(i_item_sk#9))
+
+(16) Exchange
+Input [6]: [i_item_sk#9, i_item_id#10, i_item_desc#11, i_current_price#12, i_class#13, i_category#14]
+Arguments: hashpartitioning(i_item_sk#9, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(17) Sort [codegen id : 5]
+Input [6]: [i_item_sk#9, i_item_id#10, i_item_desc#11, i_current_price#12, i_class#13, i_category#14]
+Arguments: [i_item_sk#9 ASC NULLS FIRST], false, 0
+
+(18) SortMergeJoin [codegen id : 6]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#9]
+Join condition: None
+
+(19) Project [codegen id : 6]
+Output [6]: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#11, i_current_price#12, i_class#13, i_category#14]
+Input [8]: [ss_item_sk#1, ss_ext_sales_price#2, i_item_sk#9, i_item_id#10, i_item_desc#11, i_current_price#12, i_class#13, i_category#14]
+
+(20) HashAggregate [codegen id : 6]
+Input [6]: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#11, i_current_price#12, i_class#13, i_category#14]
+Keys [5]: [i_item_id#10, i_item_desc#11, i_category#14, i_class#13, i_current_price#12]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum#16]
+Results [6]: [i_item_id#10, i_item_desc#11, i_category#14, i_class#13, i_current_price#12, sum#17]
+
+(21) Exchange
+Input [6]: [i_item_id#10, i_item_desc#11, i_category#14, i_class#13, i_current_price#12, sum#17]
+Arguments: hashpartitioning(i_item_id#10, i_item_desc#11, i_category#14, i_class#13, i_current_price#12, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(22) HashAggregate [codegen id : 7]
+Input [6]: [i_item_id#10, i_item_desc#11, i_category#14, i_class#13, i_current_price#12, sum#17]
+Keys [5]: [i_item_id#10, i_item_desc#11, i_category#14, i_class#13, i_current_price#12]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#19]
+Results [8]: [i_item_desc#11, i_category#14, i_class#13, i_current_price#12, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#19,17,2) AS itemrevenue#20, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#19,17,2) AS _w0#21, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#19,17,2) AS _w1#22, i_item_id#10]
+
+(23) Exchange
+Input [8]: [i_item_desc#11, i_category#14, i_class#13, i_current_price#12, itemrevenue#20, _w0#21, _w1#22, i_item_id#10]
+Arguments: hashpartitioning(i_class#13, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(24) Sort [codegen id : 8]
+Input [8]: [i_item_desc#11, i_category#14, i_class#13, i_current_price#12, itemrevenue#20, _w0#21, _w1#22, i_item_id#10]
+Arguments: [i_class#13 ASC NULLS FIRST], false, 0
+
+(25) Window
+Input [8]: [i_item_desc#11, i_category#14, i_class#13, i_current_price#12, itemrevenue#20, _w0#21, _w1#22, i_item_id#10]
+Arguments: [sum(_w1#22) windowspecdefinition(i_class#13, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#24], [i_class#13]
+
+(26) Project [codegen id : 9]
+Output [7]: [i_item_desc#11, i_category#14, i_class#13, i_current_price#12, itemrevenue#20, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(_w0#21) * 100.00), DecimalType(21,2), true) as decimal(27,2))) / promote_precision(_we0#24)), DecimalType(38,17), true) AS revenueratio#25, i_item_id#10]
+Input [9]: [i_item_desc#11, i_category#14, i_class#13, i_current_price#12, itemrevenue#20, _w0#21, _w1#22, i_item_id#10, _we0#24]
+
+(27) Exchange
+Input [7]: [i_item_desc#11, i_category#14, i_class#13, i_current_price#12, itemrevenue#20, revenueratio#25, i_item_id#10]
+Arguments: rangepartitioning(i_category#14 ASC NULLS FIRST, i_class#13 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#11 ASC NULLS FIRST, revenueratio#25 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [id=#26]
+
+(28) Sort [codegen id : 10]
+Input [7]: [i_item_desc#11, i_category#14, i_class#13, i_current_price#12, itemrevenue#20, revenueratio#25, i_item_id#10]
+Arguments: [i_category#14 ASC NULLS FIRST, i_class#13 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#11 ASC NULLS FIRST, revenueratio#25 ASC NULLS FIRST], true, 0
+
+(29) Project [codegen id : 10]
+Output [6]: [i_item_desc#11, i_category#14, i_class#13, i_current_price#12, itemrevenue#20, revenueratio#25]
+Input [7]: [i_item_desc#11, i_category#14, i_class#13, i_current_price#12, itemrevenue#20, revenueratio#25, i_item_id#10]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (30)
+
+
+(30) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#5]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q98.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q98.sf100/simplified.txt
new file mode 100644
index 0000000000000..872ab90b8c3c8
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q98.sf100/simplified.txt
@@ -0,0 +1,53 @@
+WholeStageCodegen (10)
+  Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio]
+    Sort [i_category,i_class,i_item_id,i_item_desc,revenueratio]
+      InputAdapter
+        Exchange [i_category,i_class,i_item_id,i_item_desc,revenueratio] #1
+          WholeStageCodegen (9)
+            Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id]
+              InputAdapter
+                Window [_w1,i_class]
+                  WholeStageCodegen (8)
+                    Sort [i_class]
+                      InputAdapter
+                        Exchange [i_class] #2
+                          WholeStageCodegen (7)
+                            HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ss_ext_sales_price)),itemrevenue,_w0,_w1,sum]
+                              InputAdapter
+                                Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3
+                                  WholeStageCodegen (6)
+                                    HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ss_ext_sales_price] [sum,sum]
+                                      Project [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                        SortMergeJoin [ss_item_sk,i_item_sk]
+                                          InputAdapter
+                                            WholeStageCodegen (3)
+                                              Sort [ss_item_sk]
+                                                InputAdapter
+                                                  Exchange [ss_item_sk] #4
+                                                    WholeStageCodegen (2)
+                                                      Project [ss_item_sk,ss_ext_sales_price]
+                                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                          Filter [ss_item_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                                                  SubqueryBroadcast [d_date_sk] #1
+                                                                    ReusedExchange [d_date_sk] #5
+                                                          InputAdapter
+                                                            BroadcastExchange #5
+                                                              WholeStageCodegen (1)
+                                                                Project [d_date_sk]
+                                                                  Filter [d_date,d_date_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.date_dim [d_date_sk,d_date]
+                                          InputAdapter
+                                            WholeStageCodegen (5)
+                                              Sort [i_item_sk]
+                                                InputAdapter
+                                                  Exchange [i_item_sk] #6
+                                                    WholeStageCodegen (4)
+                                                      Filter [i_category,i_item_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q98/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q98/explain.txt
new file mode 100644
index 0000000000000..e31c029b4932e
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q98/explain.txt
@@ -0,0 +1,158 @@
+== Physical Plan ==
+* Project (26)
++- * Sort (25)
+   +- Exchange (24)
+      +- * Project (23)
+         +- Window (22)
+            +- * Sort (21)
+               +- Exchange (20)
+                  +- * HashAggregate (19)
+                     +- Exchange (18)
+                        +- * HashAggregate (17)
+                           +- * Project (16)
+                              +- * BroadcastHashJoin Inner BuildRight (15)
+                                 :- * Project (9)
+                                 :  +- * BroadcastHashJoin Inner BuildRight (8)
+                                 :     :- * Filter (3)
+                                 :     :  +- * ColumnarToRow (2)
+                                 :     :     +- Scan parquet default.store_sales (1)
+                                 :     +- BroadcastExchange (7)
+                                 :        +- * Filter (6)
+                                 :           +- * ColumnarToRow (5)
+                                 :              +- Scan parquet default.item (4)
+                                 +- BroadcastExchange (14)
+                                    +- * Project (13)
+                                       +- * Filter (12)
+                                          +- * ColumnarToRow (11)
+                                             +- Scan parquet default.date_dim (10)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), (ss_sold_date_sk#3 >= 2451911), (ss_sold_date_sk#3 <= 2451941), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_item_sk#1)
+
+(4) Scan parquet default.item
+Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [In(i_category, [Jewelry                                           ,Sports                                            ,Books                                             ]), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_item_desc:string,i_current_price:decimal(7,2),i_class:string,i_category:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+
+(6) Filter [codegen id : 1]
+Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Condition : (i_category#10 IN (Jewelry                                           ,Sports                                            ,Books                                             ) AND isnotnull(i_item_sk#5))
+
+(7) BroadcastExchange
+Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(8) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#5]
+Join condition: None
+
+(9) Project [codegen id : 3]
+Output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+
+(10) Scan parquet default.date_dim
+Output [2]: [d_date_sk#12, d_date#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2001-01-01), LessThanOrEqual(d_date,2001-01-31), GreaterThanOrEqual(d_date_sk,2451911), LessThanOrEqual(d_date_sk,2451941), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#12, d_date#13]
+
+(12) Filter [codegen id : 2]
+Input [2]: [d_date_sk#12, d_date#13]
+Condition : (((((isnotnull(d_date#13) AND (d_date#13 >= 2001-01-01)) AND (d_date#13 <= 2001-01-31)) AND (d_date_sk#12 >= 2451911)) AND (d_date_sk#12 <= 2451941)) AND isnotnull(d_date_sk#12))
+
+(13) Project [codegen id : 2]
+Output [1]: [d_date_sk#12]
+Input [2]: [d_date_sk#12, d_date#13]
+
+(14) BroadcastExchange
+Input [1]: [d_date_sk#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#12]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [6]: [ss_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10, d_date_sk#12]
+
+(17) HashAggregate [codegen id : 3]
+Input [6]: [ss_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum#15]
+Results [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#16]
+
+(18) Exchange
+Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#16]
+Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(19) HashAggregate [codegen id : 4]
+Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#16]
+Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#18]
+Results [8]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#18,17,2) AS itemrevenue#19, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#18,17,2) AS _w0#20, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#18,17,2) AS _w1#21, i_item_id#6]
+
+(20) Exchange
+Input [8]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21, i_item_id#6]
+Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(21) Sort [codegen id : 5]
+Input [8]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21, i_item_id#6]
+Arguments: [i_class#9 ASC NULLS FIRST], false, 0
+
+(22) Window
+Input [8]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21, i_item_id#6]
+Arguments: [sum(_w1#21) windowspecdefinition(i_class#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#23], [i_class#9]
+
+(23) Project [codegen id : 6]
+Output [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(_w0#20) * 100.00), DecimalType(21,2), true) as decimal(27,2))) / promote_precision(_we0#23)), DecimalType(38,17), true) AS revenueratio#24, i_item_id#6]
+Input [9]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21, i_item_id#6, _we0#23]
+
+(24) Exchange
+Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, revenueratio#24, i_item_id#6]
+Arguments: rangepartitioning(i_category#10 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_item_id#6 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#24 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(25) Sort [codegen id : 7]
+Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, revenueratio#24, i_item_id#6]
+Arguments: [i_category#10 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_item_id#6 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#24 ASC NULLS FIRST], true, 0
+
+(26) Project [codegen id : 7]
+Output [6]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, revenueratio#24]
+Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, revenueratio#24, i_item_id#6]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (27)
+
+
+(27) ReusedExchange [Reuses operator id: 14]
+Output [1]: [d_date_sk#12]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q98/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q98/simplified.txt
new file mode 100644
index 0000000000000..67205dff3ad35
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q98/simplified.txt
@@ -0,0 +1,44 @@
+WholeStageCodegen (7)
+  Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio]
+    Sort [i_category,i_class,i_item_id,i_item_desc,revenueratio]
+      InputAdapter
+        Exchange [i_category,i_class,i_item_id,i_item_desc,revenueratio] #1
+          WholeStageCodegen (6)
+            Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id]
+              InputAdapter
+                Window [_w1,i_class]
+                  WholeStageCodegen (5)
+                    Sort [i_class]
+                      InputAdapter
+                        Exchange [i_class] #2
+                          WholeStageCodegen (4)
+                            HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ss_ext_sales_price)),itemrevenue,_w0,_w1,sum]
+                              InputAdapter
+                                Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3
+                                  WholeStageCodegen (3)
+                                    HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ss_ext_sales_price] [sum,sum]
+                                      Project [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                          Project [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                            BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                              Filter [ss_item_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                                      SubqueryBroadcast [d_date_sk] #1
+                                                        ReusedExchange [d_date_sk] #4
+                                              InputAdapter
+                                                BroadcastExchange #5
+                                                  WholeStageCodegen (1)
+                                                    Filter [i_category,i_item_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                          InputAdapter
+                                            BroadcastExchange #4
+                                              WholeStageCodegen (2)
+                                                Project [d_date_sk]
+                                                  Filter [d_date,d_date_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.date_dim [d_date_sk,d_date]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/ss_max.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/ss_max.sf100/explain.txt
new file mode 100644
index 0000000000000..1ebc9a69a3865
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/ss_max.sf100/explain.txt
@@ -0,0 +1,56 @@
+== Physical Plan ==
+* HashAggregate (8)
++- Exchange (7)
+   +- * HashAggregate (6)
+      +- * HashAggregate (5)
+         +- Exchange (4)
+            +- * HashAggregate (3)
+               +- * ColumnarToRow (2)
+                  +- Scan parquet default.store_sales (1)
+
+
+(1) Scan parquet default.store_sales
+Output [9]: [ss_sold_time_sk#1, ss_item_sk#2, ss_customer_sk#3, ss_cdemo_sk#4, ss_hdemo_sk#5, ss_addr_sk#6, ss_store_sk#7, ss_promo_sk#8, ss_sold_date_sk#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+ReadSchema: struct<ss_sold_time_sk:int,ss_item_sk:int,ss_customer_sk:int,ss_cdemo_sk:int,ss_hdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_promo_sk:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [9]: [ss_sold_time_sk#1, ss_item_sk#2, ss_customer_sk#3, ss_cdemo_sk#4, ss_hdemo_sk#5, ss_addr_sk#6, ss_store_sk#7, ss_promo_sk#8, ss_sold_date_sk#9]
+
+(3) HashAggregate [codegen id : 1]
+Input [9]: [ss_sold_time_sk#1, ss_item_sk#2, ss_customer_sk#3, ss_cdemo_sk#4, ss_hdemo_sk#5, ss_addr_sk#6, ss_store_sk#7, ss_promo_sk#8, ss_sold_date_sk#9]
+Keys [1]: [ss_sold_date_sk#9]
+Functions [11]: [partial_count(1), partial_count(ss_sold_date_sk#9), partial_max(ss_sold_date_sk#9), partial_max(ss_sold_time_sk#1), partial_max(ss_item_sk#2), partial_max(ss_customer_sk#3), partial_max(ss_cdemo_sk#4), partial_max(ss_hdemo_sk#5), partial_max(ss_addr_sk#6), partial_max(ss_store_sk#7), partial_max(ss_promo_sk#8)]
+Aggregate Attributes [11]: [count(1)#10, count(ss_sold_date_sk#9)#11, max(ss_sold_date_sk#9)#12, max(ss_sold_time_sk#1)#13, max(ss_item_sk#2)#14, max(ss_customer_sk#3)#15, max(ss_cdemo_sk#4)#16, max(ss_hdemo_sk#5)#17, max(ss_addr_sk#6)#18, max(ss_store_sk#7)#19, max(ss_promo_sk#8)#20]
+Results [12]: [ss_sold_date_sk#9, count#21, count#22, max#23, max#24, max#25, max#26, max#27, max#28, max#29, max#30, max#31]
+
+(4) Exchange
+Input [12]: [ss_sold_date_sk#9, count#21, count#22, max#23, max#24, max#25, max#26, max#27, max#28, max#29, max#30, max#31]
+Arguments: hashpartitioning(ss_sold_date_sk#9, 5), ENSURE_REQUIREMENTS, [id=#32]
+
+(5) HashAggregate [codegen id : 2]
+Input [12]: [ss_sold_date_sk#9, count#21, count#22, max#23, max#24, max#25, max#26, max#27, max#28, max#29, max#30, max#31]
+Keys [1]: [ss_sold_date_sk#9]
+Functions [11]: [merge_count(1), merge_count(ss_sold_date_sk#9), merge_max(ss_sold_date_sk#9), merge_max(ss_sold_time_sk#1), merge_max(ss_item_sk#2), merge_max(ss_customer_sk#3), merge_max(ss_cdemo_sk#4), merge_max(ss_hdemo_sk#5), merge_max(ss_addr_sk#6), merge_max(ss_store_sk#7), merge_max(ss_promo_sk#8)]
+Aggregate Attributes [11]: [count(1)#10, count(ss_sold_date_sk#9)#11, max(ss_sold_date_sk#9)#12, max(ss_sold_time_sk#1)#13, max(ss_item_sk#2)#14, max(ss_customer_sk#3)#15, max(ss_cdemo_sk#4)#16, max(ss_hdemo_sk#5)#17, max(ss_addr_sk#6)#18, max(ss_store_sk#7)#19, max(ss_promo_sk#8)#20]
+Results [12]: [ss_sold_date_sk#9, count#21, count#22, max#23, max#24, max#25, max#26, max#27, max#28, max#29, max#30, max#31]
+
+(6) HashAggregate [codegen id : 2]
+Input [12]: [ss_sold_date_sk#9, count#21, count#22, max#23, max#24, max#25, max#26, max#27, max#28, max#29, max#30, max#31]
+Keys: []
+Functions [12]: [merge_count(1), merge_count(ss_sold_date_sk#9), merge_max(ss_sold_date_sk#9), merge_max(ss_sold_time_sk#1), merge_max(ss_item_sk#2), merge_max(ss_customer_sk#3), merge_max(ss_cdemo_sk#4), merge_max(ss_hdemo_sk#5), merge_max(ss_addr_sk#6), merge_max(ss_store_sk#7), merge_max(ss_promo_sk#8), partial_count(distinct ss_sold_date_sk#9)]
+Aggregate Attributes [12]: [count(1)#10, count(ss_sold_date_sk#9)#11, max(ss_sold_date_sk#9)#12, max(ss_sold_time_sk#1)#13, max(ss_item_sk#2)#14, max(ss_customer_sk#3)#15, max(ss_cdemo_sk#4)#16, max(ss_hdemo_sk#5)#17, max(ss_addr_sk#6)#18, max(ss_store_sk#7)#19, max(ss_promo_sk#8)#20, count(ss_sold_date_sk#9)#33]
+Results [12]: [count#21, count#22, max#23, max#24, max#25, max#26, max#27, max#28, max#29, max#30, max#31, count#34]
+
+(7) Exchange
+Input [12]: [count#21, count#22, max#23, max#24, max#25, max#26, max#27, max#28, max#29, max#30, max#31, count#34]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#35]
+
+(8) HashAggregate [codegen id : 3]
+Input [12]: [count#21, count#22, max#23, max#24, max#25, max#26, max#27, max#28, max#29, max#30, max#31, count#34]
+Keys: []
+Functions [12]: [count(1), count(ss_sold_date_sk#9), max(ss_sold_date_sk#9), max(ss_sold_time_sk#1), max(ss_item_sk#2), max(ss_customer_sk#3), max(ss_cdemo_sk#4), max(ss_hdemo_sk#5), max(ss_addr_sk#6), max(ss_store_sk#7), max(ss_promo_sk#8), count(distinct ss_sold_date_sk#9)]
+Aggregate Attributes [12]: [count(1)#10, count(ss_sold_date_sk#9)#11, max(ss_sold_date_sk#9)#12, max(ss_sold_time_sk#1)#13, max(ss_item_sk#2)#14, max(ss_customer_sk#3)#15, max(ss_cdemo_sk#4)#16, max(ss_hdemo_sk#5)#17, max(ss_addr_sk#6)#18, max(ss_store_sk#7)#19, max(ss_promo_sk#8)#20, count(ss_sold_date_sk#9)#33]
+Results [12]: [count(1)#10 AS total#36, count(ss_sold_date_sk#9)#11 AS not_null_total#37, count(ss_sold_date_sk#9)#33 AS unique_days#38, max(ss_sold_date_sk#9)#12 AS max_ss_sold_date_sk#39, max(ss_sold_time_sk#1)#13 AS max_ss_sold_time_sk#40, max(ss_item_sk#2)#14 AS max_ss_item_sk#41, max(ss_customer_sk#3)#15 AS max_ss_customer_sk#42, max(ss_cdemo_sk#4)#16 AS max_ss_cdemo_sk#43, max(ss_hdemo_sk#5)#17 AS max_ss_hdemo_sk#44, max(ss_addr_sk#6)#18 AS max_ss_addr_sk#45, max(ss_store_sk#7)#19 AS max_ss_store_sk#46, max(ss_promo_sk#8)#20 AS max_ss_promo_sk#47]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/ss_max.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/ss_max.sf100/simplified.txt
new file mode 100644
index 0000000000000..0d132018b90e2
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/ss_max.sf100/simplified.txt
@@ -0,0 +1,14 @@
+WholeStageCodegen (3)
+  HashAggregate [count,count,max,max,max,max,max,max,max,max,max,count] [count(1),count(ss_sold_date_sk),max(ss_sold_date_sk),max(ss_sold_time_sk),max(ss_item_sk),max(ss_customer_sk),max(ss_cdemo_sk),max(ss_hdemo_sk),max(ss_addr_sk),max(ss_store_sk),max(ss_promo_sk),count(ss_sold_date_sk),total,not_null_total,unique_days,max_ss_sold_date_sk,max_ss_sold_time_sk,max_ss_item_sk,max_ss_customer_sk,max_ss_cdemo_sk,max_ss_hdemo_sk,max_ss_addr_sk,max_ss_store_sk,max_ss_promo_sk,count,count,max,max,max,max,max,max,max,max,max,count]
+    InputAdapter
+      Exchange #1
+        WholeStageCodegen (2)
+          HashAggregate [ss_sold_date_sk] [count(1),count(ss_sold_date_sk),max(ss_sold_date_sk),max(ss_sold_time_sk),max(ss_item_sk),max(ss_customer_sk),max(ss_cdemo_sk),max(ss_hdemo_sk),max(ss_addr_sk),max(ss_store_sk),max(ss_promo_sk),count(ss_sold_date_sk),count,count,max,max,max,max,max,max,max,max,max,count,count,count,max,max,max,max,max,max,max,max,max,count]
+            HashAggregate [ss_sold_date_sk] [count(1),count(ss_sold_date_sk),max(ss_sold_date_sk),max(ss_sold_time_sk),max(ss_item_sk),max(ss_customer_sk),max(ss_cdemo_sk),max(ss_hdemo_sk),max(ss_addr_sk),max(ss_store_sk),max(ss_promo_sk),count,count,max,max,max,max,max,max,max,max,max,count,count,max,max,max,max,max,max,max,max,max]
+              InputAdapter
+                Exchange [ss_sold_date_sk] #2
+                  WholeStageCodegen (1)
+                    HashAggregate [ss_sold_date_sk,ss_sold_time_sk,ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk] [count(1),count(ss_sold_date_sk),max(ss_sold_date_sk),max(ss_sold_time_sk),max(ss_item_sk),max(ss_customer_sk),max(ss_cdemo_sk),max(ss_hdemo_sk),max(ss_addr_sk),max(ss_store_sk),max(ss_promo_sk),count,count,max,max,max,max,max,max,max,max,max,count,count,max,max,max,max,max,max,max,max,max]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_sold_time_sk,ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_sold_date_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/ss_max/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/ss_max/explain.txt
new file mode 100644
index 0000000000000..1ebc9a69a3865
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/ss_max/explain.txt
@@ -0,0 +1,56 @@
+== Physical Plan ==
+* HashAggregate (8)
++- Exchange (7)
+   +- * HashAggregate (6)
+      +- * HashAggregate (5)
+         +- Exchange (4)
+            +- * HashAggregate (3)
+               +- * ColumnarToRow (2)
+                  +- Scan parquet default.store_sales (1)
+
+
+(1) Scan parquet default.store_sales
+Output [9]: [ss_sold_time_sk#1, ss_item_sk#2, ss_customer_sk#3, ss_cdemo_sk#4, ss_hdemo_sk#5, ss_addr_sk#6, ss_store_sk#7, ss_promo_sk#8, ss_sold_date_sk#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+ReadSchema: struct<ss_sold_time_sk:int,ss_item_sk:int,ss_customer_sk:int,ss_cdemo_sk:int,ss_hdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_promo_sk:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [9]: [ss_sold_time_sk#1, ss_item_sk#2, ss_customer_sk#3, ss_cdemo_sk#4, ss_hdemo_sk#5, ss_addr_sk#6, ss_store_sk#7, ss_promo_sk#8, ss_sold_date_sk#9]
+
+(3) HashAggregate [codegen id : 1]
+Input [9]: [ss_sold_time_sk#1, ss_item_sk#2, ss_customer_sk#3, ss_cdemo_sk#4, ss_hdemo_sk#5, ss_addr_sk#6, ss_store_sk#7, ss_promo_sk#8, ss_sold_date_sk#9]
+Keys [1]: [ss_sold_date_sk#9]
+Functions [11]: [partial_count(1), partial_count(ss_sold_date_sk#9), partial_max(ss_sold_date_sk#9), partial_max(ss_sold_time_sk#1), partial_max(ss_item_sk#2), partial_max(ss_customer_sk#3), partial_max(ss_cdemo_sk#4), partial_max(ss_hdemo_sk#5), partial_max(ss_addr_sk#6), partial_max(ss_store_sk#7), partial_max(ss_promo_sk#8)]
+Aggregate Attributes [11]: [count(1)#10, count(ss_sold_date_sk#9)#11, max(ss_sold_date_sk#9)#12, max(ss_sold_time_sk#1)#13, max(ss_item_sk#2)#14, max(ss_customer_sk#3)#15, max(ss_cdemo_sk#4)#16, max(ss_hdemo_sk#5)#17, max(ss_addr_sk#6)#18, max(ss_store_sk#7)#19, max(ss_promo_sk#8)#20]
+Results [12]: [ss_sold_date_sk#9, count#21, count#22, max#23, max#24, max#25, max#26, max#27, max#28, max#29, max#30, max#31]
+
+(4) Exchange
+Input [12]: [ss_sold_date_sk#9, count#21, count#22, max#23, max#24, max#25, max#26, max#27, max#28, max#29, max#30, max#31]
+Arguments: hashpartitioning(ss_sold_date_sk#9, 5), ENSURE_REQUIREMENTS, [id=#32]
+
+(5) HashAggregate [codegen id : 2]
+Input [12]: [ss_sold_date_sk#9, count#21, count#22, max#23, max#24, max#25, max#26, max#27, max#28, max#29, max#30, max#31]
+Keys [1]: [ss_sold_date_sk#9]
+Functions [11]: [merge_count(1), merge_count(ss_sold_date_sk#9), merge_max(ss_sold_date_sk#9), merge_max(ss_sold_time_sk#1), merge_max(ss_item_sk#2), merge_max(ss_customer_sk#3), merge_max(ss_cdemo_sk#4), merge_max(ss_hdemo_sk#5), merge_max(ss_addr_sk#6), merge_max(ss_store_sk#7), merge_max(ss_promo_sk#8)]
+Aggregate Attributes [11]: [count(1)#10, count(ss_sold_date_sk#9)#11, max(ss_sold_date_sk#9)#12, max(ss_sold_time_sk#1)#13, max(ss_item_sk#2)#14, max(ss_customer_sk#3)#15, max(ss_cdemo_sk#4)#16, max(ss_hdemo_sk#5)#17, max(ss_addr_sk#6)#18, max(ss_store_sk#7)#19, max(ss_promo_sk#8)#20]
+Results [12]: [ss_sold_date_sk#9, count#21, count#22, max#23, max#24, max#25, max#26, max#27, max#28, max#29, max#30, max#31]
+
+(6) HashAggregate [codegen id : 2]
+Input [12]: [ss_sold_date_sk#9, count#21, count#22, max#23, max#24, max#25, max#26, max#27, max#28, max#29, max#30, max#31]
+Keys: []
+Functions [12]: [merge_count(1), merge_count(ss_sold_date_sk#9), merge_max(ss_sold_date_sk#9), merge_max(ss_sold_time_sk#1), merge_max(ss_item_sk#2), merge_max(ss_customer_sk#3), merge_max(ss_cdemo_sk#4), merge_max(ss_hdemo_sk#5), merge_max(ss_addr_sk#6), merge_max(ss_store_sk#7), merge_max(ss_promo_sk#8), partial_count(distinct ss_sold_date_sk#9)]
+Aggregate Attributes [12]: [count(1)#10, count(ss_sold_date_sk#9)#11, max(ss_sold_date_sk#9)#12, max(ss_sold_time_sk#1)#13, max(ss_item_sk#2)#14, max(ss_customer_sk#3)#15, max(ss_cdemo_sk#4)#16, max(ss_hdemo_sk#5)#17, max(ss_addr_sk#6)#18, max(ss_store_sk#7)#19, max(ss_promo_sk#8)#20, count(ss_sold_date_sk#9)#33]
+Results [12]: [count#21, count#22, max#23, max#24, max#25, max#26, max#27, max#28, max#29, max#30, max#31, count#34]
+
+(7) Exchange
+Input [12]: [count#21, count#22, max#23, max#24, max#25, max#26, max#27, max#28, max#29, max#30, max#31, count#34]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#35]
+
+(8) HashAggregate [codegen id : 3]
+Input [12]: [count#21, count#22, max#23, max#24, max#25, max#26, max#27, max#28, max#29, max#30, max#31, count#34]
+Keys: []
+Functions [12]: [count(1), count(ss_sold_date_sk#9), max(ss_sold_date_sk#9), max(ss_sold_time_sk#1), max(ss_item_sk#2), max(ss_customer_sk#3), max(ss_cdemo_sk#4), max(ss_hdemo_sk#5), max(ss_addr_sk#6), max(ss_store_sk#7), max(ss_promo_sk#8), count(distinct ss_sold_date_sk#9)]
+Aggregate Attributes [12]: [count(1)#10, count(ss_sold_date_sk#9)#11, max(ss_sold_date_sk#9)#12, max(ss_sold_time_sk#1)#13, max(ss_item_sk#2)#14, max(ss_customer_sk#3)#15, max(ss_cdemo_sk#4)#16, max(ss_hdemo_sk#5)#17, max(ss_addr_sk#6)#18, max(ss_store_sk#7)#19, max(ss_promo_sk#8)#20, count(ss_sold_date_sk#9)#33]
+Results [12]: [count(1)#10 AS total#36, count(ss_sold_date_sk#9)#11 AS not_null_total#37, count(ss_sold_date_sk#9)#33 AS unique_days#38, max(ss_sold_date_sk#9)#12 AS max_ss_sold_date_sk#39, max(ss_sold_time_sk#1)#13 AS max_ss_sold_time_sk#40, max(ss_item_sk#2)#14 AS max_ss_item_sk#41, max(ss_customer_sk#3)#15 AS max_ss_customer_sk#42, max(ss_cdemo_sk#4)#16 AS max_ss_cdemo_sk#43, max(ss_hdemo_sk#5)#17 AS max_ss_hdemo_sk#44, max(ss_addr_sk#6)#18 AS max_ss_addr_sk#45, max(ss_store_sk#7)#19 AS max_ss_store_sk#46, max(ss_promo_sk#8)#20 AS max_ss_promo_sk#47]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/ss_max/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/ss_max/simplified.txt
new file mode 100644
index 0000000000000..0d132018b90e2
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/ss_max/simplified.txt
@@ -0,0 +1,14 @@
+WholeStageCodegen (3)
+  HashAggregate [count,count,max,max,max,max,max,max,max,max,max,count] [count(1),count(ss_sold_date_sk),max(ss_sold_date_sk),max(ss_sold_time_sk),max(ss_item_sk),max(ss_customer_sk),max(ss_cdemo_sk),max(ss_hdemo_sk),max(ss_addr_sk),max(ss_store_sk),max(ss_promo_sk),count(ss_sold_date_sk),total,not_null_total,unique_days,max_ss_sold_date_sk,max_ss_sold_time_sk,max_ss_item_sk,max_ss_customer_sk,max_ss_cdemo_sk,max_ss_hdemo_sk,max_ss_addr_sk,max_ss_store_sk,max_ss_promo_sk,count,count,max,max,max,max,max,max,max,max,max,count]
+    InputAdapter
+      Exchange #1
+        WholeStageCodegen (2)
+          HashAggregate [ss_sold_date_sk] [count(1),count(ss_sold_date_sk),max(ss_sold_date_sk),max(ss_sold_time_sk),max(ss_item_sk),max(ss_customer_sk),max(ss_cdemo_sk),max(ss_hdemo_sk),max(ss_addr_sk),max(ss_store_sk),max(ss_promo_sk),count(ss_sold_date_sk),count,count,max,max,max,max,max,max,max,max,max,count,count,count,max,max,max,max,max,max,max,max,max,count]
+            HashAggregate [ss_sold_date_sk] [count(1),count(ss_sold_date_sk),max(ss_sold_date_sk),max(ss_sold_time_sk),max(ss_item_sk),max(ss_customer_sk),max(ss_cdemo_sk),max(ss_hdemo_sk),max(ss_addr_sk),max(ss_store_sk),max(ss_promo_sk),count,count,max,max,max,max,max,max,max,max,max,count,count,max,max,max,max,max,max,max,max,max]
+              InputAdapter
+                Exchange [ss_sold_date_sk] #2
+                  WholeStageCodegen (1)
+                    HashAggregate [ss_sold_date_sk,ss_sold_time_sk,ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk] [count(1),count(ss_sold_date_sk),max(ss_sold_date_sk),max(ss_sold_time_sk),max(ss_item_sk),max(ss_customer_sk),max(ss_cdemo_sk),max(ss_hdemo_sk),max(ss_addr_sk),max(ss_store_sk),max(ss_promo_sk),count,count,max,max,max,max,max,max,max,max,max,count,count,max,max,max,max,max,max,max,max,max]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_sold_time_sk,ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_sold_date_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.sf100/explain.txt
new file mode 100644
index 0000000000000..47c4db9ee29c1
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.sf100/explain.txt
@@ -0,0 +1,284 @@
+== Physical Plan ==
+TakeOrderedAndProject (47)
++- * Project (46)
+   +- * SortMergeJoin Inner (45)
+      :- * Sort (39)
+      :  +- Exchange (38)
+      :     +- * Project (37)
+      :        +- * BroadcastHashJoin Inner BuildRight (36)
+      :           :- * Project (30)
+      :           :  +- * BroadcastHashJoin Inner BuildRight (29)
+      :           :     :- * Filter (14)
+      :           :     :  +- * HashAggregate (13)
+      :           :     :     +- Exchange (12)
+      :           :     :        +- * HashAggregate (11)
+      :           :     :           +- * Project (10)
+      :           :     :              +- * BroadcastHashJoin Inner BuildRight (9)
+      :           :     :                 :- * Filter (3)
+      :           :     :                 :  +- * ColumnarToRow (2)
+      :           :     :                 :     +- Scan parquet default.store_returns (1)
+      :           :     :                 +- BroadcastExchange (8)
+      :           :     :                    +- * Project (7)
+      :           :     :                       +- * Filter (6)
+      :           :     :                          +- * ColumnarToRow (5)
+      :           :     :                             +- Scan parquet default.date_dim (4)
+      :           :     +- BroadcastExchange (28)
+      :           :        +- * Filter (27)
+      :           :           +- * HashAggregate (26)
+      :           :              +- Exchange (25)
+      :           :                 +- * HashAggregate (24)
+      :           :                    +- * HashAggregate (23)
+      :           :                       +- Exchange (22)
+      :           :                          +- * HashAggregate (21)
+      :           :                             +- * Project (20)
+      :           :                                +- * BroadcastHashJoin Inner BuildRight (19)
+      :           :                                   :- * Filter (17)
+      :           :                                   :  +- * ColumnarToRow (16)
+      :           :                                   :     +- Scan parquet default.store_returns (15)
+      :           :                                   +- ReusedExchange (18)
+      :           +- BroadcastExchange (35)
+      :              +- * Project (34)
+      :                 +- * Filter (33)
+      :                    +- * ColumnarToRow (32)
+      :                       +- Scan parquet default.store (31)
+      +- * Sort (44)
+         +- Exchange (43)
+            +- * Filter (42)
+               +- * ColumnarToRow (41)
+                  +- Scan parquet default.customer (40)
+
+
+(1) Scan parquet default.store_returns
+Output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(sr_returned_date_sk#4), dynamicpruningexpression(sr_returned_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(sr_store_sk), IsNotNull(sr_customer_sk)]
+ReadSchema: struct<sr_customer_sk:int,sr_store_sk:int,sr_return_amt:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4]
+
+(3) Filter [codegen id : 2]
+Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4]
+Condition : (isnotnull(sr_store_sk#2) AND isnotnull(sr_customer_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#6, d_year#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#6, d_year#7]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#6, d_year#7]
+Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2000)) AND isnotnull(d_date_sk#6))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#6]
+Input [2]: [d_date_sk#6, d_year#7]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#8]
+
+(9) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [sr_returned_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 2]
+Output [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3]
+Input [5]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4, d_date_sk#6]
+
+(11) HashAggregate [codegen id : 2]
+Input [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3]
+Keys [2]: [sr_customer_sk#1, sr_store_sk#2]
+Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#3))]
+Aggregate Attributes [1]: [sum#9]
+Results [3]: [sr_customer_sk#1, sr_store_sk#2, sum#10]
+
+(12) Exchange
+Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#10]
+Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, [id=#11]
+
+(13) HashAggregate [codegen id : 8]
+Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#10]
+Keys [2]: [sr_customer_sk#1, sr_store_sk#2]
+Functions [1]: [sum(UnscaledValue(sr_return_amt#3))]
+Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#3))#12]
+Results [3]: [sr_customer_sk#1 AS ctr_customer_sk#13, sr_store_sk#2 AS ctr_store_sk#14, MakeDecimal(sum(UnscaledValue(sr_return_amt#3))#12,17,2) AS ctr_total_return#15]
+
+(14) Filter [codegen id : 8]
+Input [3]: [ctr_customer_sk#13, ctr_store_sk#14, ctr_total_return#15]
+Condition : isnotnull(ctr_total_return#15)
+
+(15) Scan parquet default.store_returns
+Output [4]: [sr_customer_sk#16, sr_store_sk#17, sr_return_amt#18, sr_returned_date_sk#19]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(sr_returned_date_sk#19), dynamicpruningexpression(sr_returned_date_sk#19 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(sr_store_sk)]
+ReadSchema: struct<sr_customer_sk:int,sr_store_sk:int,sr_return_amt:decimal(7,2)>
+
+(16) ColumnarToRow [codegen id : 4]
+Input [4]: [sr_customer_sk#16, sr_store_sk#17, sr_return_amt#18, sr_returned_date_sk#19]
+
+(17) Filter [codegen id : 4]
+Input [4]: [sr_customer_sk#16, sr_store_sk#17, sr_return_amt#18, sr_returned_date_sk#19]
+Condition : isnotnull(sr_store_sk#17)
+
+(18) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#20]
+
+(19) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [sr_returned_date_sk#19]
+Right keys [1]: [d_date_sk#20]
+Join condition: None
+
+(20) Project [codegen id : 4]
+Output [3]: [sr_customer_sk#16, sr_store_sk#17, sr_return_amt#18]
+Input [5]: [sr_customer_sk#16, sr_store_sk#17, sr_return_amt#18, sr_returned_date_sk#19, d_date_sk#20]
+
+(21) HashAggregate [codegen id : 4]
+Input [3]: [sr_customer_sk#16, sr_store_sk#17, sr_return_amt#18]
+Keys [2]: [sr_customer_sk#16, sr_store_sk#17]
+Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#18))]
+Aggregate Attributes [1]: [sum#21]
+Results [3]: [sr_customer_sk#16, sr_store_sk#17, sum#22]
+
+(22) Exchange
+Input [3]: [sr_customer_sk#16, sr_store_sk#17, sum#22]
+Arguments: hashpartitioning(sr_customer_sk#16, sr_store_sk#17, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(23) HashAggregate [codegen id : 5]
+Input [3]: [sr_customer_sk#16, sr_store_sk#17, sum#22]
+Keys [2]: [sr_customer_sk#16, sr_store_sk#17]
+Functions [1]: [sum(UnscaledValue(sr_return_amt#18))]
+Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#18))#24]
+Results [2]: [sr_store_sk#17 AS ctr_store_sk#14, MakeDecimal(sum(UnscaledValue(sr_return_amt#18))#24,17,2) AS ctr_total_return#15]
+
+(24) HashAggregate [codegen id : 5]
+Input [2]: [ctr_store_sk#14, ctr_total_return#15]
+Keys [1]: [ctr_store_sk#14]
+Functions [1]: [partial_avg(ctr_total_return#15)]
+Aggregate Attributes [2]: [sum#25, count#26]
+Results [3]: [ctr_store_sk#14, sum#27, count#28]
+
+(25) Exchange
+Input [3]: [ctr_store_sk#14, sum#27, count#28]
+Arguments: hashpartitioning(ctr_store_sk#14, 5), ENSURE_REQUIREMENTS, [id=#29]
+
+(26) HashAggregate [codegen id : 6]
+Input [3]: [ctr_store_sk#14, sum#27, count#28]
+Keys [1]: [ctr_store_sk#14]
+Functions [1]: [avg(ctr_total_return#15)]
+Aggregate Attributes [1]: [avg(ctr_total_return#15)#30]
+Results [2]: [CheckOverflow((promote_precision(avg(ctr_total_return#15)#30) * 1.200000), DecimalType(24,7), true) AS (avg(ctr_total_return) * 1.2)#31, ctr_store_sk#14 AS ctr_store_sk#14#32]
+
+(27) Filter [codegen id : 6]
+Input [2]: [(avg(ctr_total_return) * 1.2)#31, ctr_store_sk#14#32]
+Condition : isnotnull((avg(ctr_total_return) * 1.2)#31)
+
+(28) BroadcastExchange
+Input [2]: [(avg(ctr_total_return) * 1.2)#31, ctr_store_sk#14#32]
+Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [id=#33]
+
+(29) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ctr_store_sk#14]
+Right keys [1]: [ctr_store_sk#14#32]
+Join condition: (cast(ctr_total_return#15 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#31)
+
+(30) Project [codegen id : 8]
+Output [2]: [ctr_customer_sk#13, ctr_store_sk#14]
+Input [5]: [ctr_customer_sk#13, ctr_store_sk#14, ctr_total_return#15, (avg(ctr_total_return) * 1.2)#31, ctr_store_sk#14#32]
+
+(31) Scan parquet default.store
+Output [2]: [s_store_sk#34, s_state#35]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_state:string>
+
+(32) ColumnarToRow [codegen id : 7]
+Input [2]: [s_store_sk#34, s_state#35]
+
+(33) Filter [codegen id : 7]
+Input [2]: [s_store_sk#34, s_state#35]
+Condition : ((isnotnull(s_state#35) AND (s_state#35 = TN)) AND isnotnull(s_store_sk#34))
+
+(34) Project [codegen id : 7]
+Output [1]: [s_store_sk#34]
+Input [2]: [s_store_sk#34, s_state#35]
+
+(35) BroadcastExchange
+Input [1]: [s_store_sk#34]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#36]
+
+(36) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ctr_store_sk#14]
+Right keys [1]: [s_store_sk#34]
+Join condition: None
+
+(37) Project [codegen id : 8]
+Output [1]: [ctr_customer_sk#13]
+Input [3]: [ctr_customer_sk#13, ctr_store_sk#14, s_store_sk#34]
+
+(38) Exchange
+Input [1]: [ctr_customer_sk#13]
+Arguments: hashpartitioning(ctr_customer_sk#13, 5), ENSURE_REQUIREMENTS, [id=#37]
+
+(39) Sort [codegen id : 9]
+Input [1]: [ctr_customer_sk#13]
+Arguments: [ctr_customer_sk#13 ASC NULLS FIRST], false, 0
+
+(40) Scan parquet default.customer
+Output [2]: [c_customer_sk#38, c_customer_id#39]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string>
+
+(41) ColumnarToRow [codegen id : 10]
+Input [2]: [c_customer_sk#38, c_customer_id#39]
+
+(42) Filter [codegen id : 10]
+Input [2]: [c_customer_sk#38, c_customer_id#39]
+Condition : isnotnull(c_customer_sk#38)
+
+(43) Exchange
+Input [2]: [c_customer_sk#38, c_customer_id#39]
+Arguments: hashpartitioning(c_customer_sk#38, 5), ENSURE_REQUIREMENTS, [id=#40]
+
+(44) Sort [codegen id : 11]
+Input [2]: [c_customer_sk#38, c_customer_id#39]
+Arguments: [c_customer_sk#38 ASC NULLS FIRST], false, 0
+
+(45) SortMergeJoin [codegen id : 12]
+Left keys [1]: [ctr_customer_sk#13]
+Right keys [1]: [c_customer_sk#38]
+Join condition: None
+
+(46) Project [codegen id : 12]
+Output [1]: [c_customer_id#39]
+Input [3]: [ctr_customer_sk#13, c_customer_sk#38, c_customer_id#39]
+
+(47) TakeOrderedAndProject
+Input [1]: [c_customer_id#39]
+Arguments: 100, [c_customer_id#39 ASC NULLS FIRST], [c_customer_id#39]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = sr_returned_date_sk#4 IN dynamicpruning#5
+ReusedExchange (48)
+
+
+(48) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#6]
+
+Subquery:2 Hosting operator id = 15 Hosting Expression = sr_returned_date_sk#19 IN dynamicpruning#5
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.sf100/simplified.txt
new file mode 100644
index 0000000000000..dea7ed93e7a1a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.sf100/simplified.txt
@@ -0,0 +1,77 @@
+TakeOrderedAndProject [c_customer_id]
+  WholeStageCodegen (12)
+    Project [c_customer_id]
+      SortMergeJoin [ctr_customer_sk,c_customer_sk]
+        InputAdapter
+          WholeStageCodegen (9)
+            Sort [ctr_customer_sk]
+              InputAdapter
+                Exchange [ctr_customer_sk] #1
+                  WholeStageCodegen (8)
+                    Project [ctr_customer_sk]
+                      BroadcastHashJoin [ctr_store_sk,s_store_sk]
+                        Project [ctr_customer_sk,ctr_store_sk]
+                          BroadcastHashJoin [ctr_store_sk,ctr_store_sk,ctr_total_return,(avg(ctr_total_return) * 1.2)]
+                            Filter [ctr_total_return]
+                              HashAggregate [sr_customer_sk,sr_store_sk,sum] [sum(UnscaledValue(sr_return_amt)),ctr_customer_sk,ctr_store_sk,ctr_total_return,sum]
+                                InputAdapter
+                                  Exchange [sr_customer_sk,sr_store_sk] #2
+                                    WholeStageCodegen (2)
+                                      HashAggregate [sr_customer_sk,sr_store_sk,sr_return_amt] [sum,sum]
+                                        Project [sr_customer_sk,sr_store_sk,sr_return_amt]
+                                          BroadcastHashJoin [sr_returned_date_sk,d_date_sk]
+                                            Filter [sr_store_sk,sr_customer_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #1
+                                                      ReusedExchange [d_date_sk] #3
+                                            InputAdapter
+                                              BroadcastExchange #3
+                                                WholeStageCodegen (1)
+                                                  Project [d_date_sk]
+                                                    Filter [d_year,d_date_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.date_dim [d_date_sk,d_year]
+                            InputAdapter
+                              BroadcastExchange #4
+                                WholeStageCodegen (6)
+                                  Filter [(avg(ctr_total_return) * 1.2)]
+                                    HashAggregate [ctr_store_sk,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),ctr_store_sk,sum,count]
+                                      InputAdapter
+                                        Exchange [ctr_store_sk] #5
+                                          WholeStageCodegen (5)
+                                            HashAggregate [ctr_store_sk,ctr_total_return] [sum,count,sum,count]
+                                              HashAggregate [sr_customer_sk,sr_store_sk,sum] [sum(UnscaledValue(sr_return_amt)),ctr_store_sk,ctr_total_return,sum]
+                                                InputAdapter
+                                                  Exchange [sr_customer_sk,sr_store_sk] #6
+                                                    WholeStageCodegen (4)
+                                                      HashAggregate [sr_customer_sk,sr_store_sk,sr_return_amt] [sum,sum]
+                                                        Project [sr_customer_sk,sr_store_sk,sr_return_amt]
+                                                          BroadcastHashJoin [sr_returned_date_sk,d_date_sk]
+                                                            Filter [sr_store_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk]
+                                                                    ReusedSubquery [d_date_sk] #1
+                                                            InputAdapter
+                                                              ReusedExchange [d_date_sk] #3
+                        InputAdapter
+                          BroadcastExchange #7
+                            WholeStageCodegen (7)
+                              Project [s_store_sk]
+                                Filter [s_state,s_store_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store [s_store_sk,s_state]
+        InputAdapter
+          WholeStageCodegen (11)
+            Sort [c_customer_sk]
+              InputAdapter
+                Exchange [c_customer_sk] #8
+                  WholeStageCodegen (10)
+                    Filter [c_customer_sk]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.customer [c_customer_sk,c_customer_id]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/explain.txt
new file mode 100644
index 0000000000000..21f0534974c01
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/explain.txt
@@ -0,0 +1,269 @@
+== Physical Plan ==
+TakeOrderedAndProject (44)
++- * Project (43)
+   +- * BroadcastHashJoin Inner BuildRight (42)
+      :- * Project (37)
+      :  +- * BroadcastHashJoin Inner BuildRight (36)
+      :     :- * Project (30)
+      :     :  +- * BroadcastHashJoin Inner BuildRight (29)
+      :     :     :- * Filter (14)
+      :     :     :  +- * HashAggregate (13)
+      :     :     :     +- Exchange (12)
+      :     :     :        +- * HashAggregate (11)
+      :     :     :           +- * Project (10)
+      :     :     :              +- * BroadcastHashJoin Inner BuildRight (9)
+      :     :     :                 :- * Filter (3)
+      :     :     :                 :  +- * ColumnarToRow (2)
+      :     :     :                 :     +- Scan parquet default.store_returns (1)
+      :     :     :                 +- BroadcastExchange (8)
+      :     :     :                    +- * Project (7)
+      :     :     :                       +- * Filter (6)
+      :     :     :                          +- * ColumnarToRow (5)
+      :     :     :                             +- Scan parquet default.date_dim (4)
+      :     :     +- BroadcastExchange (28)
+      :     :        +- * Filter (27)
+      :     :           +- * HashAggregate (26)
+      :     :              +- Exchange (25)
+      :     :                 +- * HashAggregate (24)
+      :     :                    +- * HashAggregate (23)
+      :     :                       +- Exchange (22)
+      :     :                          +- * HashAggregate (21)
+      :     :                             +- * Project (20)
+      :     :                                +- * BroadcastHashJoin Inner BuildRight (19)
+      :     :                                   :- * Filter (17)
+      :     :                                   :  +- * ColumnarToRow (16)
+      :     :                                   :     +- Scan parquet default.store_returns (15)
+      :     :                                   +- ReusedExchange (18)
+      :     +- BroadcastExchange (35)
+      :        +- * Project (34)
+      :           +- * Filter (33)
+      :              +- * ColumnarToRow (32)
+      :                 +- Scan parquet default.store (31)
+      +- BroadcastExchange (41)
+         +- * Filter (40)
+            +- * ColumnarToRow (39)
+               +- Scan parquet default.customer (38)
+
+
+(1) Scan parquet default.store_returns
+Output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(sr_returned_date_sk#4), dynamicpruningexpression(sr_returned_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(sr_store_sk), IsNotNull(sr_customer_sk)]
+ReadSchema: struct<sr_customer_sk:int,sr_store_sk:int,sr_return_amt:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4]
+
+(3) Filter [codegen id : 2]
+Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4]
+Condition : (isnotnull(sr_store_sk#2) AND isnotnull(sr_customer_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#6, d_year#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#6, d_year#7]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#6, d_year#7]
+Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2000)) AND isnotnull(d_date_sk#6))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#6]
+Input [2]: [d_date_sk#6, d_year#7]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#8]
+
+(9) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [sr_returned_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 2]
+Output [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3]
+Input [5]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4, d_date_sk#6]
+
+(11) HashAggregate [codegen id : 2]
+Input [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3]
+Keys [2]: [sr_customer_sk#1, sr_store_sk#2]
+Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#3))]
+Aggregate Attributes [1]: [sum#9]
+Results [3]: [sr_customer_sk#1, sr_store_sk#2, sum#10]
+
+(12) Exchange
+Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#10]
+Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, [id=#11]
+
+(13) HashAggregate [codegen id : 9]
+Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#10]
+Keys [2]: [sr_customer_sk#1, sr_store_sk#2]
+Functions [1]: [sum(UnscaledValue(sr_return_amt#3))]
+Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#3))#12]
+Results [3]: [sr_customer_sk#1 AS ctr_customer_sk#13, sr_store_sk#2 AS ctr_store_sk#14, MakeDecimal(sum(UnscaledValue(sr_return_amt#3))#12,17,2) AS ctr_total_return#15]
+
+(14) Filter [codegen id : 9]
+Input [3]: [ctr_customer_sk#13, ctr_store_sk#14, ctr_total_return#15]
+Condition : isnotnull(ctr_total_return#15)
+
+(15) Scan parquet default.store_returns
+Output [4]: [sr_customer_sk#16, sr_store_sk#17, sr_return_amt#18, sr_returned_date_sk#19]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(sr_returned_date_sk#19), dynamicpruningexpression(sr_returned_date_sk#19 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(sr_store_sk)]
+ReadSchema: struct<sr_customer_sk:int,sr_store_sk:int,sr_return_amt:decimal(7,2)>
+
+(16) ColumnarToRow [codegen id : 4]
+Input [4]: [sr_customer_sk#16, sr_store_sk#17, sr_return_amt#18, sr_returned_date_sk#19]
+
+(17) Filter [codegen id : 4]
+Input [4]: [sr_customer_sk#16, sr_store_sk#17, sr_return_amt#18, sr_returned_date_sk#19]
+Condition : isnotnull(sr_store_sk#17)
+
+(18) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#20]
+
+(19) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [sr_returned_date_sk#19]
+Right keys [1]: [d_date_sk#20]
+Join condition: None
+
+(20) Project [codegen id : 4]
+Output [3]: [sr_customer_sk#16, sr_store_sk#17, sr_return_amt#18]
+Input [5]: [sr_customer_sk#16, sr_store_sk#17, sr_return_amt#18, sr_returned_date_sk#19, d_date_sk#20]
+
+(21) HashAggregate [codegen id : 4]
+Input [3]: [sr_customer_sk#16, sr_store_sk#17, sr_return_amt#18]
+Keys [2]: [sr_customer_sk#16, sr_store_sk#17]
+Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#18))]
+Aggregate Attributes [1]: [sum#21]
+Results [3]: [sr_customer_sk#16, sr_store_sk#17, sum#22]
+
+(22) Exchange
+Input [3]: [sr_customer_sk#16, sr_store_sk#17, sum#22]
+Arguments: hashpartitioning(sr_customer_sk#16, sr_store_sk#17, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(23) HashAggregate [codegen id : 5]
+Input [3]: [sr_customer_sk#16, sr_store_sk#17, sum#22]
+Keys [2]: [sr_customer_sk#16, sr_store_sk#17]
+Functions [1]: [sum(UnscaledValue(sr_return_amt#18))]
+Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#18))#24]
+Results [2]: [sr_store_sk#17 AS ctr_store_sk#14, MakeDecimal(sum(UnscaledValue(sr_return_amt#18))#24,17,2) AS ctr_total_return#15]
+
+(24) HashAggregate [codegen id : 5]
+Input [2]: [ctr_store_sk#14, ctr_total_return#15]
+Keys [1]: [ctr_store_sk#14]
+Functions [1]: [partial_avg(ctr_total_return#15)]
+Aggregate Attributes [2]: [sum#25, count#26]
+Results [3]: [ctr_store_sk#14, sum#27, count#28]
+
+(25) Exchange
+Input [3]: [ctr_store_sk#14, sum#27, count#28]
+Arguments: hashpartitioning(ctr_store_sk#14, 5), ENSURE_REQUIREMENTS, [id=#29]
+
+(26) HashAggregate [codegen id : 6]
+Input [3]: [ctr_store_sk#14, sum#27, count#28]
+Keys [1]: [ctr_store_sk#14]
+Functions [1]: [avg(ctr_total_return#15)]
+Aggregate Attributes [1]: [avg(ctr_total_return#15)#30]
+Results [2]: [CheckOverflow((promote_precision(avg(ctr_total_return#15)#30) * 1.200000), DecimalType(24,7), true) AS (avg(ctr_total_return) * 1.2)#31, ctr_store_sk#14 AS ctr_store_sk#14#32]
+
+(27) Filter [codegen id : 6]
+Input [2]: [(avg(ctr_total_return) * 1.2)#31, ctr_store_sk#14#32]
+Condition : isnotnull((avg(ctr_total_return) * 1.2)#31)
+
+(28) BroadcastExchange
+Input [2]: [(avg(ctr_total_return) * 1.2)#31, ctr_store_sk#14#32]
+Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [id=#33]
+
+(29) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ctr_store_sk#14]
+Right keys [1]: [ctr_store_sk#14#32]
+Join condition: (cast(ctr_total_return#15 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#31)
+
+(30) Project [codegen id : 9]
+Output [2]: [ctr_customer_sk#13, ctr_store_sk#14]
+Input [5]: [ctr_customer_sk#13, ctr_store_sk#14, ctr_total_return#15, (avg(ctr_total_return) * 1.2)#31, ctr_store_sk#14#32]
+
+(31) Scan parquet default.store
+Output [2]: [s_store_sk#34, s_state#35]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_state:string>
+
+(32) ColumnarToRow [codegen id : 7]
+Input [2]: [s_store_sk#34, s_state#35]
+
+(33) Filter [codegen id : 7]
+Input [2]: [s_store_sk#34, s_state#35]
+Condition : ((isnotnull(s_state#35) AND (s_state#35 = TN)) AND isnotnull(s_store_sk#34))
+
+(34) Project [codegen id : 7]
+Output [1]: [s_store_sk#34]
+Input [2]: [s_store_sk#34, s_state#35]
+
+(35) BroadcastExchange
+Input [1]: [s_store_sk#34]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#36]
+
+(36) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ctr_store_sk#14]
+Right keys [1]: [s_store_sk#34]
+Join condition: None
+
+(37) Project [codegen id : 9]
+Output [1]: [ctr_customer_sk#13]
+Input [3]: [ctr_customer_sk#13, ctr_store_sk#14, s_store_sk#34]
+
+(38) Scan parquet default.customer
+Output [2]: [c_customer_sk#37, c_customer_id#38]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string>
+
+(39) ColumnarToRow [codegen id : 8]
+Input [2]: [c_customer_sk#37, c_customer_id#38]
+
+(40) Filter [codegen id : 8]
+Input [2]: [c_customer_sk#37, c_customer_id#38]
+Condition : isnotnull(c_customer_sk#37)
+
+(41) BroadcastExchange
+Input [2]: [c_customer_sk#37, c_customer_id#38]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#39]
+
+(42) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ctr_customer_sk#13]
+Right keys [1]: [c_customer_sk#37]
+Join condition: None
+
+(43) Project [codegen id : 9]
+Output [1]: [c_customer_id#38]
+Input [3]: [ctr_customer_sk#13, c_customer_sk#37, c_customer_id#38]
+
+(44) TakeOrderedAndProject
+Input [1]: [c_customer_id#38]
+Arguments: 100, [c_customer_id#38 ASC NULLS FIRST], [c_customer_id#38]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = sr_returned_date_sk#4 IN dynamicpruning#5
+ReusedExchange (45)
+
+
+(45) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#6]
+
+Subquery:2 Hosting operator id = 15 Hosting Expression = sr_returned_date_sk#19 IN dynamicpruning#5
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/simplified.txt
new file mode 100644
index 0000000000000..1a7ee26015049
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/simplified.txt
@@ -0,0 +1,68 @@
+TakeOrderedAndProject [c_customer_id]
+  WholeStageCodegen (9)
+    Project [c_customer_id]
+      BroadcastHashJoin [ctr_customer_sk,c_customer_sk]
+        Project [ctr_customer_sk]
+          BroadcastHashJoin [ctr_store_sk,s_store_sk]
+            Project [ctr_customer_sk,ctr_store_sk]
+              BroadcastHashJoin [ctr_store_sk,ctr_store_sk,ctr_total_return,(avg(ctr_total_return) * 1.2)]
+                Filter [ctr_total_return]
+                  HashAggregate [sr_customer_sk,sr_store_sk,sum] [sum(UnscaledValue(sr_return_amt)),ctr_customer_sk,ctr_store_sk,ctr_total_return,sum]
+                    InputAdapter
+                      Exchange [sr_customer_sk,sr_store_sk] #1
+                        WholeStageCodegen (2)
+                          HashAggregate [sr_customer_sk,sr_store_sk,sr_return_amt] [sum,sum]
+                            Project [sr_customer_sk,sr_store_sk,sr_return_amt]
+                              BroadcastHashJoin [sr_returned_date_sk,d_date_sk]
+                                Filter [sr_store_sk,sr_customer_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk]
+                                        SubqueryBroadcast [d_date_sk] #1
+                                          ReusedExchange [d_date_sk] #2
+                                InputAdapter
+                                  BroadcastExchange #2
+                                    WholeStageCodegen (1)
+                                      Project [d_date_sk]
+                                        Filter [d_year,d_date_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.date_dim [d_date_sk,d_year]
+                InputAdapter
+                  BroadcastExchange #3
+                    WholeStageCodegen (6)
+                      Filter [(avg(ctr_total_return) * 1.2)]
+                        HashAggregate [ctr_store_sk,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),ctr_store_sk,sum,count]
+                          InputAdapter
+                            Exchange [ctr_store_sk] #4
+                              WholeStageCodegen (5)
+                                HashAggregate [ctr_store_sk,ctr_total_return] [sum,count,sum,count]
+                                  HashAggregate [sr_customer_sk,sr_store_sk,sum] [sum(UnscaledValue(sr_return_amt)),ctr_store_sk,ctr_total_return,sum]
+                                    InputAdapter
+                                      Exchange [sr_customer_sk,sr_store_sk] #5
+                                        WholeStageCodegen (4)
+                                          HashAggregate [sr_customer_sk,sr_store_sk,sr_return_amt] [sum,sum]
+                                            Project [sr_customer_sk,sr_store_sk,sr_return_amt]
+                                              BroadcastHashJoin [sr_returned_date_sk,d_date_sk]
+                                                Filter [sr_store_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk]
+                                                        ReusedSubquery [d_date_sk] #1
+                                                InputAdapter
+                                                  ReusedExchange [d_date_sk] #2
+            InputAdapter
+              BroadcastExchange #6
+                WholeStageCodegen (7)
+                  Project [s_store_sk]
+                    Filter [s_state,s_store_sk]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store [s_store_sk,s_state]
+        InputAdapter
+          BroadcastExchange #7
+            WholeStageCodegen (8)
+              Filter [c_customer_sk]
+                ColumnarToRow
+                  InputAdapter
+                    Scan parquet default.customer [c_customer_sk,c_customer_id]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.sf100/explain.txt
new file mode 100644
index 0000000000000..b7f3535bd0ea2
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.sf100/explain.txt
@@ -0,0 +1,318 @@
+== Physical Plan ==
+TakeOrderedAndProject (55)
++- * HashAggregate (54)
+   +- Exchange (53)
+      +- * HashAggregate (52)
+         +- * Project (51)
+            +- * SortMergeJoin Inner (50)
+               :- * Sort (44)
+               :  +- Exchange (43)
+               :     +- * Project (42)
+               :        +- * BroadcastHashJoin Inner BuildRight (41)
+               :           :- * Project (35)
+               :           :  +- * Filter (34)
+               :           :     +- SortMergeJoin ExistenceJoin(exists#1) (33)
+               :           :        :- SortMergeJoin ExistenceJoin(exists#2) (25)
+               :           :        :  :- * SortMergeJoin LeftSemi (17)
+               :           :        :  :  :- * Sort (5)
+               :           :        :  :  :  +- Exchange (4)
+               :           :        :  :  :     +- * Filter (3)
+               :           :        :  :  :        +- * ColumnarToRow (2)
+               :           :        :  :  :           +- Scan parquet default.customer (1)
+               :           :        :  :  +- * Sort (16)
+               :           :        :  :     +- Exchange (15)
+               :           :        :  :        +- * Project (14)
+               :           :        :  :           +- * BroadcastHashJoin Inner BuildRight (13)
+               :           :        :  :              :- * ColumnarToRow (7)
+               :           :        :  :              :  +- Scan parquet default.store_sales (6)
+               :           :        :  :              +- BroadcastExchange (12)
+               :           :        :  :                 +- * Project (11)
+               :           :        :  :                    +- * Filter (10)
+               :           :        :  :                       +- * ColumnarToRow (9)
+               :           :        :  :                          +- Scan parquet default.date_dim (8)
+               :           :        :  +- * Sort (24)
+               :           :        :     +- Exchange (23)
+               :           :        :        +- * Project (22)
+               :           :        :           +- * BroadcastHashJoin Inner BuildRight (21)
+               :           :        :              :- * ColumnarToRow (19)
+               :           :        :              :  +- Scan parquet default.web_sales (18)
+               :           :        :              +- ReusedExchange (20)
+               :           :        +- * Sort (32)
+               :           :           +- Exchange (31)
+               :           :              +- * Project (30)
+               :           :                 +- * BroadcastHashJoin Inner BuildRight (29)
+               :           :                    :- * ColumnarToRow (27)
+               :           :                    :  +- Scan parquet default.catalog_sales (26)
+               :           :                    +- ReusedExchange (28)
+               :           +- BroadcastExchange (40)
+               :              +- * Project (39)
+               :                 +- * Filter (38)
+               :                    +- * ColumnarToRow (37)
+               :                       +- Scan parquet default.customer_address (36)
+               +- * Sort (49)
+                  +- Exchange (48)
+                     +- * Filter (47)
+                        +- * ColumnarToRow (46)
+                           +- Scan parquet default.customer_demographics (45)
+
+
+(1) Scan parquet default.customer
+Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_addr_sk:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5]
+
+(3) Filter [codegen id : 1]
+Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5]
+Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4))
+
+(4) Exchange
+Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5]
+Arguments: hashpartitioning(c_customer_sk#3, 5), ENSURE_REQUIREMENTS, [id=#6]
+
+(5) Sort [codegen id : 2]
+Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5]
+Arguments: [c_customer_sk#3 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.store_sales
+Output [2]: [ss_customer_sk#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+ReadSchema: struct<ss_customer_sk:int>
+
+(7) ColumnarToRow [codegen id : 4]
+Input [2]: [ss_customer_sk#7, ss_sold_date_sk#8]
+
+(8) Scan parquet default.date_dim
+Output [3]: [d_date_sk#10, d_year#11, d_moy#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,1), LessThanOrEqual(d_moy,4), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(9) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+
+(10) Filter [codegen id : 3]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+Condition : (((((isnotnull(d_year#11) AND isnotnull(d_moy#12)) AND (d_year#11 = 2002)) AND (d_moy#12 >= 1)) AND (d_moy#12 <= 4)) AND isnotnull(d_date_sk#10))
+
+(11) Project [codegen id : 3]
+Output [1]: [d_date_sk#10]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+
+(12) BroadcastExchange
+Input [1]: [d_date_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(13) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(14) Project [codegen id : 4]
+Output [1]: [ss_customer_sk#7]
+Input [3]: [ss_customer_sk#7, ss_sold_date_sk#8, d_date_sk#10]
+
+(15) Exchange
+Input [1]: [ss_customer_sk#7]
+Arguments: hashpartitioning(ss_customer_sk#7, 5), ENSURE_REQUIREMENTS, [id=#14]
+
+(16) Sort [codegen id : 5]
+Input [1]: [ss_customer_sk#7]
+Arguments: [ss_customer_sk#7 ASC NULLS FIRST], false, 0
+
+(17) SortMergeJoin [codegen id : 6]
+Left keys [1]: [c_customer_sk#3]
+Right keys [1]: [ss_customer_sk#7]
+Join condition: None
+
+(18) Scan parquet default.web_sales
+Output [2]: [ws_bill_customer_sk#15, ws_sold_date_sk#16]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#16), dynamicpruningexpression(ws_sold_date_sk#16 IN dynamicpruning#9)]
+ReadSchema: struct<ws_bill_customer_sk:int>
+
+(19) ColumnarToRow [codegen id : 8]
+Input [2]: [ws_bill_customer_sk#15, ws_sold_date_sk#16]
+
+(20) ReusedExchange [Reuses operator id: 12]
+Output [1]: [d_date_sk#17]
+
+(21) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ws_sold_date_sk#16]
+Right keys [1]: [d_date_sk#17]
+Join condition: None
+
+(22) Project [codegen id : 8]
+Output [1]: [ws_bill_customer_sk#15]
+Input [3]: [ws_bill_customer_sk#15, ws_sold_date_sk#16, d_date_sk#17]
+
+(23) Exchange
+Input [1]: [ws_bill_customer_sk#15]
+Arguments: hashpartitioning(ws_bill_customer_sk#15, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(24) Sort [codegen id : 9]
+Input [1]: [ws_bill_customer_sk#15]
+Arguments: [ws_bill_customer_sk#15 ASC NULLS FIRST], false, 0
+
+(25) SortMergeJoin
+Left keys [1]: [c_customer_sk#3]
+Right keys [1]: [ws_bill_customer_sk#15]
+Join condition: None
+
+(26) Scan parquet default.catalog_sales
+Output [2]: [cs_ship_customer_sk#19, cs_sold_date_sk#20]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#20), dynamicpruningexpression(cs_sold_date_sk#20 IN dynamicpruning#9)]
+ReadSchema: struct<cs_ship_customer_sk:int>
+
+(27) ColumnarToRow [codegen id : 11]
+Input [2]: [cs_ship_customer_sk#19, cs_sold_date_sk#20]
+
+(28) ReusedExchange [Reuses operator id: 12]
+Output [1]: [d_date_sk#21]
+
+(29) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_sold_date_sk#20]
+Right keys [1]: [d_date_sk#21]
+Join condition: None
+
+(30) Project [codegen id : 11]
+Output [1]: [cs_ship_customer_sk#19]
+Input [3]: [cs_ship_customer_sk#19, cs_sold_date_sk#20, d_date_sk#21]
+
+(31) Exchange
+Input [1]: [cs_ship_customer_sk#19]
+Arguments: hashpartitioning(cs_ship_customer_sk#19, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(32) Sort [codegen id : 12]
+Input [1]: [cs_ship_customer_sk#19]
+Arguments: [cs_ship_customer_sk#19 ASC NULLS FIRST], false, 0
+
+(33) SortMergeJoin
+Left keys [1]: [c_customer_sk#3]
+Right keys [1]: [cs_ship_customer_sk#19]
+Join condition: None
+
+(34) Filter [codegen id : 14]
+Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1]
+Condition : (exists#2 OR exists#1)
+
+(35) Project [codegen id : 14]
+Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5]
+Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1]
+
+(36) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#23, ca_county#24]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [In(ca_county, [Rush County,Toole County,Jefferson County,Dona Ana County,La Porte County]), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_county:string>
+
+(37) ColumnarToRow [codegen id : 13]
+Input [2]: [ca_address_sk#23, ca_county#24]
+
+(38) Filter [codegen id : 13]
+Input [2]: [ca_address_sk#23, ca_county#24]
+Condition : (ca_county#24 IN (Rush County,Toole County,Jefferson County,Dona Ana County,La Porte County) AND isnotnull(ca_address_sk#23))
+
+(39) Project [codegen id : 13]
+Output [1]: [ca_address_sk#23]
+Input [2]: [ca_address_sk#23, ca_county#24]
+
+(40) BroadcastExchange
+Input [1]: [ca_address_sk#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#25]
+
+(41) BroadcastHashJoin [codegen id : 14]
+Left keys [1]: [c_current_addr_sk#5]
+Right keys [1]: [ca_address_sk#23]
+Join condition: None
+
+(42) Project [codegen id : 14]
+Output [1]: [c_current_cdemo_sk#4]
+Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#23]
+
+(43) Exchange
+Input [1]: [c_current_cdemo_sk#4]
+Arguments: hashpartitioning(c_current_cdemo_sk#4, 5), ENSURE_REQUIREMENTS, [id=#26]
+
+(44) Sort [codegen id : 15]
+Input [1]: [c_current_cdemo_sk#4]
+Arguments: [c_current_cdemo_sk#4 ASC NULLS FIRST], false, 0
+
+(45) Scan parquet default.customer_demographics
+Output [9]: [cd_demo_sk#27, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_education_status:string,cd_purchase_estimate:int,cd_credit_rating:string,cd_dep_count:int,cd_dep_employed_count:int,cd_dep_college_count:int>
+
+(46) ColumnarToRow [codegen id : 16]
+Input [9]: [cd_demo_sk#27, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35]
+
+(47) Filter [codegen id : 16]
+Input [9]: [cd_demo_sk#27, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35]
+Condition : isnotnull(cd_demo_sk#27)
+
+(48) Exchange
+Input [9]: [cd_demo_sk#27, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35]
+Arguments: hashpartitioning(cd_demo_sk#27, 5), ENSURE_REQUIREMENTS, [id=#36]
+
+(49) Sort [codegen id : 17]
+Input [9]: [cd_demo_sk#27, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35]
+Arguments: [cd_demo_sk#27 ASC NULLS FIRST], false, 0
+
+(50) SortMergeJoin [codegen id : 18]
+Left keys [1]: [c_current_cdemo_sk#4]
+Right keys [1]: [cd_demo_sk#27]
+Join condition: None
+
+(51) Project [codegen id : 18]
+Output [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35]
+Input [10]: [c_current_cdemo_sk#4, cd_demo_sk#27, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35]
+
+(52) HashAggregate [codegen id : 18]
+Input [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35]
+Keys [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#37]
+Results [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35, count#38]
+
+(53) Exchange
+Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35, count#38]
+Arguments: hashpartitioning(cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35, 5), ENSURE_REQUIREMENTS, [id=#39]
+
+(54) HashAggregate [codegen id : 19]
+Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35, count#38]
+Keys [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#31, cd_credit_rating#32, cd_dep_count#33, cd_dep_employed_count#34, cd_dep_college_count#35]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#40]
+Results [14]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, count(1)#40 AS cnt1#41, cd_purchase_estimate#31, count(1)#40 AS cnt2#42, cd_credit_rating#32, count(1)#40 AS cnt3#43, cd_dep_count#33, count(1)#40 AS cnt4#44, cd_dep_employed_count#34, count(1)#40 AS cnt5#45, cd_dep_college_count#35, count(1)#40 AS cnt6#46]
+
+(55) TakeOrderedAndProject
+Input [14]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#41, cd_purchase_estimate#31, cnt2#42, cd_credit_rating#32, cnt3#43, cd_dep_count#33, cnt4#44, cd_dep_employed_count#34, cnt5#45, cd_dep_college_count#35, cnt6#46]
+Arguments: 100, [cd_gender#28 ASC NULLS FIRST, cd_marital_status#29 ASC NULLS FIRST, cd_education_status#30 ASC NULLS FIRST, cd_purchase_estimate#31 ASC NULLS FIRST, cd_credit_rating#32 ASC NULLS FIRST, cd_dep_count#33 ASC NULLS FIRST, cd_dep_employed_count#34 ASC NULLS FIRST, cd_dep_college_count#35 ASC NULLS FIRST], [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#41, cd_purchase_estimate#31, cnt2#42, cd_credit_rating#32, cnt3#43, cd_dep_count#33, cnt4#44, cd_dep_employed_count#34, cnt5#45, cd_dep_college_count#35, cnt6#46]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 6 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (56)
+
+
+(56) ReusedExchange [Reuses operator id: 12]
+Output [1]: [d_date_sk#10]
+
+Subquery:2 Hosting operator id = 18 Hosting Expression = ws_sold_date_sk#16 IN dynamicpruning#9
+
+Subquery:3 Hosting operator id = 26 Hosting Expression = cs_sold_date_sk#20 IN dynamicpruning#9
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.sf100/simplified.txt
new file mode 100644
index 0000000000000..a40043d68deeb
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.sf100/simplified.txt
@@ -0,0 +1,99 @@
+TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,cnt2,cnt3,cnt4,cnt5,cnt6]
+  WholeStageCodegen (19)
+    HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] [count(1),cnt1,cnt2,cnt3,cnt4,cnt5,cnt6,count]
+      InputAdapter
+        Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1
+          WholeStageCodegen (18)
+            HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count]
+              Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count]
+                SortMergeJoin [c_current_cdemo_sk,cd_demo_sk]
+                  InputAdapter
+                    WholeStageCodegen (15)
+                      Sort [c_current_cdemo_sk]
+                        InputAdapter
+                          Exchange [c_current_cdemo_sk] #2
+                            WholeStageCodegen (14)
+                              Project [c_current_cdemo_sk]
+                                BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                                  Project [c_current_cdemo_sk,c_current_addr_sk]
+                                    Filter [exists,exists]
+                                      InputAdapter
+                                        SortMergeJoin [c_customer_sk,cs_ship_customer_sk]
+                                          SortMergeJoin [c_customer_sk,ws_bill_customer_sk]
+                                            WholeStageCodegen (6)
+                                              SortMergeJoin [c_customer_sk,ss_customer_sk]
+                                                InputAdapter
+                                                  WholeStageCodegen (2)
+                                                    Sort [c_customer_sk]
+                                                      InputAdapter
+                                                        Exchange [c_customer_sk] #3
+                                                          WholeStageCodegen (1)
+                                                            Filter [c_current_addr_sk,c_current_cdemo_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk]
+                                                InputAdapter
+                                                  WholeStageCodegen (5)
+                                                    Sort [ss_customer_sk]
+                                                      InputAdapter
+                                                        Exchange [ss_customer_sk] #4
+                                                          WholeStageCodegen (4)
+                                                            Project [ss_customer_sk]
+                                                              BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.store_sales [ss_customer_sk,ss_sold_date_sk]
+                                                                      SubqueryBroadcast [d_date_sk] #1
+                                                                        ReusedExchange [d_date_sk] #5
+                                                                InputAdapter
+                                                                  BroadcastExchange #5
+                                                                    WholeStageCodegen (3)
+                                                                      Project [d_date_sk]
+                                                                        Filter [d_year,d_moy,d_date_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                            WholeStageCodegen (9)
+                                              Sort [ws_bill_customer_sk]
+                                                InputAdapter
+                                                  Exchange [ws_bill_customer_sk] #6
+                                                    WholeStageCodegen (8)
+                                                      Project [ws_bill_customer_sk]
+                                                        BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.web_sales [ws_bill_customer_sk,ws_sold_date_sk]
+                                                                ReusedSubquery [d_date_sk] #1
+                                                          InputAdapter
+                                                            ReusedExchange [d_date_sk] #5
+                                          WholeStageCodegen (12)
+                                            Sort [cs_ship_customer_sk]
+                                              InputAdapter
+                                                Exchange [cs_ship_customer_sk] #7
+                                                  WholeStageCodegen (11)
+                                                    Project [cs_ship_customer_sk]
+                                                      BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk]
+                                                              ReusedSubquery [d_date_sk] #1
+                                                        InputAdapter
+                                                          ReusedExchange [d_date_sk] #5
+                                  InputAdapter
+                                    BroadcastExchange #8
+                                      WholeStageCodegen (13)
+                                        Project [ca_address_sk]
+                                          Filter [ca_county,ca_address_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.customer_address [ca_address_sk,ca_county]
+                  InputAdapter
+                    WholeStageCodegen (17)
+                      Sort [cd_demo_sk]
+                        InputAdapter
+                          Exchange [cd_demo_sk] #9
+                            WholeStageCodegen (16)
+                              Filter [cd_demo_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/explain.txt
new file mode 100644
index 0000000000000..370551a2de652
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/explain.txt
@@ -0,0 +1,278 @@
+== Physical Plan ==
+TakeOrderedAndProject (47)
++- * HashAggregate (46)
+   +- Exchange (45)
+      +- * HashAggregate (44)
+         +- * Project (43)
+            +- * BroadcastHashJoin Inner BuildRight (42)
+               :- * Project (37)
+               :  +- * BroadcastHashJoin Inner BuildRight (36)
+               :     :- * Project (30)
+               :     :  +- * Filter (29)
+               :     :     +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (28)
+               :     :        :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (21)
+               :     :        :  :- * BroadcastHashJoin LeftSemi BuildRight (14)
+               :     :        :  :  :- * Filter (3)
+               :     :        :  :  :  +- * ColumnarToRow (2)
+               :     :        :  :  :     +- Scan parquet default.customer (1)
+               :     :        :  :  +- BroadcastExchange (13)
+               :     :        :  :     +- * Project (12)
+               :     :        :  :        +- * BroadcastHashJoin Inner BuildRight (11)
+               :     :        :  :           :- * ColumnarToRow (5)
+               :     :        :  :           :  +- Scan parquet default.store_sales (4)
+               :     :        :  :           +- BroadcastExchange (10)
+               :     :        :  :              +- * Project (9)
+               :     :        :  :                 +- * Filter (8)
+               :     :        :  :                    +- * ColumnarToRow (7)
+               :     :        :  :                       +- Scan parquet default.date_dim (6)
+               :     :        :  +- BroadcastExchange (20)
+               :     :        :     +- * Project (19)
+               :     :        :        +- * BroadcastHashJoin Inner BuildRight (18)
+               :     :        :           :- * ColumnarToRow (16)
+               :     :        :           :  +- Scan parquet default.web_sales (15)
+               :     :        :           +- ReusedExchange (17)
+               :     :        +- BroadcastExchange (27)
+               :     :           +- * Project (26)
+               :     :              +- * BroadcastHashJoin Inner BuildRight (25)
+               :     :                 :- * ColumnarToRow (23)
+               :     :                 :  +- Scan parquet default.catalog_sales (22)
+               :     :                 +- ReusedExchange (24)
+               :     +- BroadcastExchange (35)
+               :        +- * Project (34)
+               :           +- * Filter (33)
+               :              +- * ColumnarToRow (32)
+               :                 +- Scan parquet default.customer_address (31)
+               +- BroadcastExchange (41)
+                  +- * Filter (40)
+                     +- * ColumnarToRow (39)
+                        +- Scan parquet default.customer_demographics (38)
+
+
+(1) Scan parquet default.customer
+Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_addr_sk:int>
+
+(2) ColumnarToRow [codegen id : 9]
+Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5]
+
+(3) Filter [codegen id : 9]
+Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5]
+Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4))
+
+(4) Scan parquet default.store_sales
+Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)]
+ReadSchema: struct<ss_customer_sk:int>
+
+(5) ColumnarToRow [codegen id : 2]
+Input [2]: [ss_customer_sk#6, ss_sold_date_sk#7]
+
+(6) Scan parquet default.date_dim
+Output [3]: [d_date_sk#9, d_year#10, d_moy#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,1), LessThanOrEqual(d_moy,4), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(7) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#9, d_year#10, d_moy#11]
+
+(8) Filter [codegen id : 1]
+Input [3]: [d_date_sk#9, d_year#10, d_moy#11]
+Condition : (((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 2002)) AND (d_moy#11 >= 1)) AND (d_moy#11 <= 4)) AND isnotnull(d_date_sk#9))
+
+(9) Project [codegen id : 1]
+Output [1]: [d_date_sk#9]
+Input [3]: [d_date_sk#9, d_year#10, d_moy#11]
+
+(10) BroadcastExchange
+Input [1]: [d_date_sk#9]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(11) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#7]
+Right keys [1]: [d_date_sk#9]
+Join condition: None
+
+(12) Project [codegen id : 2]
+Output [1]: [ss_customer_sk#6]
+Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#9]
+
+(13) BroadcastExchange
+Input [1]: [ss_customer_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(14) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_customer_sk#3]
+Right keys [1]: [ss_customer_sk#6]
+Join condition: None
+
+(15) Scan parquet default.web_sales
+Output [2]: [ws_bill_customer_sk#14, ws_sold_date_sk#15]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#15), dynamicpruningexpression(ws_sold_date_sk#15 IN dynamicpruning#8)]
+ReadSchema: struct<ws_bill_customer_sk:int>
+
+(16) ColumnarToRow [codegen id : 4]
+Input [2]: [ws_bill_customer_sk#14, ws_sold_date_sk#15]
+
+(17) ReusedExchange [Reuses operator id: 10]
+Output [1]: [d_date_sk#16]
+
+(18) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ws_sold_date_sk#15]
+Right keys [1]: [d_date_sk#16]
+Join condition: None
+
+(19) Project [codegen id : 4]
+Output [1]: [ws_bill_customer_sk#14]
+Input [3]: [ws_bill_customer_sk#14, ws_sold_date_sk#15, d_date_sk#16]
+
+(20) BroadcastExchange
+Input [1]: [ws_bill_customer_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(21) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_customer_sk#3]
+Right keys [1]: [ws_bill_customer_sk#14]
+Join condition: None
+
+(22) Scan parquet default.catalog_sales
+Output [2]: [cs_ship_customer_sk#18, cs_sold_date_sk#19]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#19), dynamicpruningexpression(cs_sold_date_sk#19 IN dynamicpruning#8)]
+ReadSchema: struct<cs_ship_customer_sk:int>
+
+(23) ColumnarToRow [codegen id : 6]
+Input [2]: [cs_ship_customer_sk#18, cs_sold_date_sk#19]
+
+(24) ReusedExchange [Reuses operator id: 10]
+Output [1]: [d_date_sk#20]
+
+(25) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [cs_sold_date_sk#19]
+Right keys [1]: [d_date_sk#20]
+Join condition: None
+
+(26) Project [codegen id : 6]
+Output [1]: [cs_ship_customer_sk#18]
+Input [3]: [cs_ship_customer_sk#18, cs_sold_date_sk#19, d_date_sk#20]
+
+(27) BroadcastExchange
+Input [1]: [cs_ship_customer_sk#18]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(28) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_customer_sk#3]
+Right keys [1]: [cs_ship_customer_sk#18]
+Join condition: None
+
+(29) Filter [codegen id : 9]
+Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1]
+Condition : (exists#2 OR exists#1)
+
+(30) Project [codegen id : 9]
+Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5]
+Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1]
+
+(31) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#22, ca_county#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [In(ca_county, [Rush County,Toole County,Jefferson County,Dona Ana County,La Porte County]), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_county:string>
+
+(32) ColumnarToRow [codegen id : 7]
+Input [2]: [ca_address_sk#22, ca_county#23]
+
+(33) Filter [codegen id : 7]
+Input [2]: [ca_address_sk#22, ca_county#23]
+Condition : (ca_county#23 IN (Rush County,Toole County,Jefferson County,Dona Ana County,La Porte County) AND isnotnull(ca_address_sk#22))
+
+(34) Project [codegen id : 7]
+Output [1]: [ca_address_sk#22]
+Input [2]: [ca_address_sk#22, ca_county#23]
+
+(35) BroadcastExchange
+Input [1]: [ca_address_sk#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#24]
+
+(36) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_current_addr_sk#5]
+Right keys [1]: [ca_address_sk#22]
+Join condition: None
+
+(37) Project [codegen id : 9]
+Output [1]: [c_current_cdemo_sk#4]
+Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#22]
+
+(38) Scan parquet default.customer_demographics
+Output [9]: [cd_demo_sk#25, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_education_status:string,cd_purchase_estimate:int,cd_credit_rating:string,cd_dep_count:int,cd_dep_employed_count:int,cd_dep_college_count:int>
+
+(39) ColumnarToRow [codegen id : 8]
+Input [9]: [cd_demo_sk#25, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+
+(40) Filter [codegen id : 8]
+Input [9]: [cd_demo_sk#25, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Condition : isnotnull(cd_demo_sk#25)
+
+(41) BroadcastExchange
+Input [9]: [cd_demo_sk#25, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#34]
+
+(42) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_current_cdemo_sk#4]
+Right keys [1]: [cd_demo_sk#25]
+Join condition: None
+
+(43) Project [codegen id : 9]
+Output [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Input [10]: [c_current_cdemo_sk#4, cd_demo_sk#25, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+
+(44) HashAggregate [codegen id : 9]
+Input [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Keys [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#35]
+Results [9]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33, count#36]
+
+(45) Exchange
+Input [9]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33, count#36]
+Arguments: hashpartitioning(cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33, 5), ENSURE_REQUIREMENTS, [id=#37]
+
+(46) HashAggregate [codegen id : 10]
+Input [9]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33, count#36]
+Keys [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#38]
+Results [14]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, count(1)#38 AS cnt1#39, cd_purchase_estimate#29, count(1)#38 AS cnt2#40, cd_credit_rating#30, count(1)#38 AS cnt3#41, cd_dep_count#31, count(1)#38 AS cnt4#42, cd_dep_employed_count#32, count(1)#38 AS cnt5#43, cd_dep_college_count#33, count(1)#38 AS cnt6#44]
+
+(47) TakeOrderedAndProject
+Input [14]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cnt1#39, cd_purchase_estimate#29, cnt2#40, cd_credit_rating#30, cnt3#41, cd_dep_count#31, cnt4#42, cd_dep_employed_count#32, cnt5#43, cd_dep_college_count#33, cnt6#44]
+Arguments: 100, [cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_education_status#28 ASC NULLS FIRST, cd_purchase_estimate#29 ASC NULLS FIRST, cd_credit_rating#30 ASC NULLS FIRST, cd_dep_count#31 ASC NULLS FIRST, cd_dep_employed_count#32 ASC NULLS FIRST, cd_dep_college_count#33 ASC NULLS FIRST], [cd_gender#26, cd_marital_status#27, cd_education_status#28, cnt1#39, cd_purchase_estimate#29, cnt2#40, cd_credit_rating#30, cnt3#41, cd_dep_count#31, cnt4#42, cd_dep_employed_count#32, cnt5#43, cd_dep_college_count#33, cnt6#44]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (48)
+
+
+(48) ReusedExchange [Reuses operator id: 10]
+Output [1]: [d_date_sk#9]
+
+Subquery:2 Hosting operator id = 15 Hosting Expression = ws_sold_date_sk#15 IN dynamicpruning#8
+
+Subquery:3 Hosting operator id = 22 Hosting Expression = cs_sold_date_sk#19 IN dynamicpruning#8
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/simplified.txt
new file mode 100644
index 0000000000000..261bb6152b22a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/simplified.txt
@@ -0,0 +1,75 @@
+TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,cnt2,cnt3,cnt4,cnt5,cnt6]
+  WholeStageCodegen (10)
+    HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] [count(1),cnt1,cnt2,cnt3,cnt4,cnt5,cnt6,count]
+      InputAdapter
+        Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1
+          WholeStageCodegen (9)
+            HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count]
+              Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count]
+                BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk]
+                  Project [c_current_cdemo_sk]
+                    BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                      Project [c_current_cdemo_sk,c_current_addr_sk]
+                        Filter [exists,exists]
+                          BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk]
+                            BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk]
+                              BroadcastHashJoin [c_customer_sk,ss_customer_sk]
+                                Filter [c_current_addr_sk,c_current_cdemo_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk]
+                                InputAdapter
+                                  BroadcastExchange #2
+                                    WholeStageCodegen (2)
+                                      Project [ss_customer_sk]
+                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.store_sales [ss_customer_sk,ss_sold_date_sk]
+                                                SubqueryBroadcast [d_date_sk] #1
+                                                  ReusedExchange [d_date_sk] #3
+                                          InputAdapter
+                                            BroadcastExchange #3
+                                              WholeStageCodegen (1)
+                                                Project [d_date_sk]
+                                                  Filter [d_year,d_moy,d_date_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                              InputAdapter
+                                BroadcastExchange #4
+                                  WholeStageCodegen (4)
+                                    Project [ws_bill_customer_sk]
+                                      BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.web_sales [ws_bill_customer_sk,ws_sold_date_sk]
+                                              ReusedSubquery [d_date_sk] #1
+                                        InputAdapter
+                                          ReusedExchange [d_date_sk] #3
+                            InputAdapter
+                              BroadcastExchange #5
+                                WholeStageCodegen (6)
+                                  Project [cs_ship_customer_sk]
+                                    BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk]
+                                            ReusedSubquery [d_date_sk] #1
+                                      InputAdapter
+                                        ReusedExchange [d_date_sk] #3
+                      InputAdapter
+                        BroadcastExchange #6
+                          WholeStageCodegen (7)
+                            Project [ca_address_sk]
+                              Filter [ca_county,ca_address_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.customer_address [ca_address_sk,ca_county]
+                  InputAdapter
+                    BroadcastExchange #7
+                      WholeStageCodegen (8)
+                        Filter [cd_demo_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.sf100/explain.txt
new file mode 100644
index 0000000000000..92e8e38634ea6
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.sf100/explain.txt
@@ -0,0 +1,507 @@
+== Physical Plan ==
+TakeOrderedAndProject (87)
++- * Project (86)
+   +- * SortMergeJoin Inner (85)
+      :- * Project (67)
+      :  +- * SortMergeJoin Inner (66)
+      :     :- * Project (46)
+      :     :  +- * SortMergeJoin Inner (45)
+      :     :     :- * Sort (24)
+      :     :     :  +- Exchange (23)
+      :     :     :     +- * Filter (22)
+      :     :     :        +- * HashAggregate (21)
+      :     :     :           +- Exchange (20)
+      :     :     :              +- * HashAggregate (19)
+      :     :     :                 +- * Project (18)
+      :     :     :                    +- * SortMergeJoin Inner (17)
+      :     :     :                       :- * Sort (11)
+      :     :     :                       :  +- Exchange (10)
+      :     :     :                       :     +- * Project (9)
+      :     :     :                       :        +- * BroadcastHashJoin Inner BuildRight (8)
+      :     :     :                       :           :- * Filter (3)
+      :     :     :                       :           :  +- * ColumnarToRow (2)
+      :     :     :                       :           :     +- Scan parquet default.store_sales (1)
+      :     :     :                       :           +- BroadcastExchange (7)
+      :     :     :                       :              +- * Filter (6)
+      :     :     :                       :                 +- * ColumnarToRow (5)
+      :     :     :                       :                    +- Scan parquet default.date_dim (4)
+      :     :     :                       +- * Sort (16)
+      :     :     :                          +- Exchange (15)
+      :     :     :                             +- * Filter (14)
+      :     :     :                                +- * ColumnarToRow (13)
+      :     :     :                                   +- Scan parquet default.customer (12)
+      :     :     +- * Sort (44)
+      :     :        +- Exchange (43)
+      :     :           +- * HashAggregate (42)
+      :     :              +- Exchange (41)
+      :     :                 +- * HashAggregate (40)
+      :     :                    +- * Project (39)
+      :     :                       +- * SortMergeJoin Inner (38)
+      :     :                          :- * Sort (35)
+      :     :                          :  +- Exchange (34)
+      :     :                          :     +- * Project (33)
+      :     :                          :        +- * BroadcastHashJoin Inner BuildRight (32)
+      :     :                          :           :- * Filter (27)
+      :     :                          :           :  +- * ColumnarToRow (26)
+      :     :                          :           :     +- Scan parquet default.store_sales (25)
+      :     :                          :           +- BroadcastExchange (31)
+      :     :                          :              +- * Filter (30)
+      :     :                          :                 +- * ColumnarToRow (29)
+      :     :                          :                    +- Scan parquet default.date_dim (28)
+      :     :                          +- * Sort (37)
+      :     :                             +- ReusedExchange (36)
+      :     +- * Sort (65)
+      :        +- Exchange (64)
+      :           +- * Project (63)
+      :              +- * Filter (62)
+      :                 +- * HashAggregate (61)
+      :                    +- Exchange (60)
+      :                       +- * HashAggregate (59)
+      :                          +- * Project (58)
+      :                             +- * SortMergeJoin Inner (57)
+      :                                :- * Sort (54)
+      :                                :  +- Exchange (53)
+      :                                :     +- * Project (52)
+      :                                :        +- * BroadcastHashJoin Inner BuildRight (51)
+      :                                :           :- * Filter (49)
+      :                                :           :  +- * ColumnarToRow (48)
+      :                                :           :     +- Scan parquet default.web_sales (47)
+      :                                :           +- ReusedExchange (50)
+      :                                +- * Sort (56)
+      :                                   +- ReusedExchange (55)
+      +- * Sort (84)
+         +- Exchange (83)
+            +- * HashAggregate (82)
+               +- Exchange (81)
+                  +- * HashAggregate (80)
+                     +- * Project (79)
+                        +- * SortMergeJoin Inner (78)
+                           :- * Sort (75)
+                           :  +- Exchange (74)
+                           :     +- * Project (73)
+                           :        +- * BroadcastHashJoin Inner BuildRight (72)
+                           :           :- * Filter (70)
+                           :           :  +- * ColumnarToRow (69)
+                           :           :     +- Scan parquet default.web_sales (68)
+                           :           +- ReusedExchange (71)
+                           +- * Sort (77)
+                              +- ReusedExchange (76)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_customer_sk#1, ss_ext_discount_amt#2, ss_ext_list_price#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_ext_discount_amt:decimal(7,2),ss_ext_list_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [4]: [ss_customer_sk#1, ss_ext_discount_amt#2, ss_ext_list_price#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 2]
+Input [4]: [ss_customer_sk#1, ss_ext_discount_amt#2, ss_ext_list_price#3, ss_sold_date_sk#4]
+Condition : isnotnull(ss_customer_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#6, d_year#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#6, d_year#7]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#6, d_year#7]
+Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2001)) AND isnotnull(d_date_sk#6))
+
+(7) BroadcastExchange
+Input [2]: [d_date_sk#6, d_year#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#8]
+
+(8) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(9) Project [codegen id : 2]
+Output [4]: [ss_customer_sk#1, ss_ext_discount_amt#2, ss_ext_list_price#3, d_year#7]
+Input [6]: [ss_customer_sk#1, ss_ext_discount_amt#2, ss_ext_list_price#3, ss_sold_date_sk#4, d_date_sk#6, d_year#7]
+
+(10) Exchange
+Input [4]: [ss_customer_sk#1, ss_ext_discount_amt#2, ss_ext_list_price#3, d_year#7]
+Arguments: hashpartitioning(ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#9]
+
+(11) Sort [codegen id : 3]
+Input [4]: [ss_customer_sk#1, ss_ext_discount_amt#2, ss_ext_list_price#3, d_year#7]
+Arguments: [ss_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(12) Scan parquet default.customer
+Output [8]: [c_customer_sk#10, c_customer_id#11, c_first_name#12, c_last_name#13, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string,c_birth_country:string,c_login:string,c_email_address:string>
+
+(13) ColumnarToRow [codegen id : 4]
+Input [8]: [c_customer_sk#10, c_customer_id#11, c_first_name#12, c_last_name#13, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17]
+
+(14) Filter [codegen id : 4]
+Input [8]: [c_customer_sk#10, c_customer_id#11, c_first_name#12, c_last_name#13, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17]
+Condition : (isnotnull(c_customer_sk#10) AND isnotnull(c_customer_id#11))
+
+(15) Exchange
+Input [8]: [c_customer_sk#10, c_customer_id#11, c_first_name#12, c_last_name#13, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17]
+Arguments: hashpartitioning(c_customer_sk#10, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(16) Sort [codegen id : 5]
+Input [8]: [c_customer_sk#10, c_customer_id#11, c_first_name#12, c_last_name#13, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17]
+Arguments: [c_customer_sk#10 ASC NULLS FIRST], false, 0
+
+(17) SortMergeJoin [codegen id : 6]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#10]
+Join condition: None
+
+(18) Project [codegen id : 6]
+Output [10]: [c_customer_id#11, c_first_name#12, c_last_name#13, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17, ss_ext_discount_amt#2, ss_ext_list_price#3, d_year#7]
+Input [12]: [ss_customer_sk#1, ss_ext_discount_amt#2, ss_ext_list_price#3, d_year#7, c_customer_sk#10, c_customer_id#11, c_first_name#12, c_last_name#13, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17]
+
+(19) HashAggregate [codegen id : 6]
+Input [10]: [c_customer_id#11, c_first_name#12, c_last_name#13, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17, ss_ext_discount_amt#2, ss_ext_list_price#3, d_year#7]
+Keys [8]: [c_customer_id#11, c_first_name#12, c_last_name#13, d_year#7, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17]
+Functions [1]: [partial_sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#3 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#2 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum#19]
+Results [9]: [c_customer_id#11, c_first_name#12, c_last_name#13, d_year#7, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17, sum#20]
+
+(20) Exchange
+Input [9]: [c_customer_id#11, c_first_name#12, c_last_name#13, d_year#7, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17, sum#20]
+Arguments: hashpartitioning(c_customer_id#11, c_first_name#12, c_last_name#13, d_year#7, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(21) HashAggregate [codegen id : 7]
+Input [9]: [c_customer_id#11, c_first_name#12, c_last_name#13, d_year#7, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17, sum#20]
+Keys [8]: [c_customer_id#11, c_first_name#12, c_last_name#13, d_year#7, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17]
+Functions [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#3 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#2 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#3 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#2 as decimal(8,2)))), DecimalType(8,2), true)))#22]
+Results [2]: [c_customer_id#11 AS customer_id#23, MakeDecimal(sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#3 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#2 as decimal(8,2)))), DecimalType(8,2), true)))#22,18,2) AS year_total#24]
+
+(22) Filter [codegen id : 7]
+Input [2]: [customer_id#23, year_total#24]
+Condition : (isnotnull(year_total#24) AND (year_total#24 > 0.00))
+
+(23) Exchange
+Input [2]: [customer_id#23, year_total#24]
+Arguments: hashpartitioning(customer_id#23, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(24) Sort [codegen id : 8]
+Input [2]: [customer_id#23, year_total#24]
+Arguments: [customer_id#23 ASC NULLS FIRST], false, 0
+
+(25) Scan parquet default.store_sales
+Output [4]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#29), dynamicpruningexpression(ss_sold_date_sk#29 IN dynamicpruning#30)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_ext_discount_amt:decimal(7,2),ss_ext_list_price:decimal(7,2)>
+
+(26) ColumnarToRow [codegen id : 10]
+Input [4]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29]
+
+(27) Filter [codegen id : 10]
+Input [4]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29]
+Condition : isnotnull(ss_customer_sk#26)
+
+(28) Scan parquet default.date_dim
+Output [2]: [d_date_sk#31, d_year#32]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(29) ColumnarToRow [codegen id : 9]
+Input [2]: [d_date_sk#31, d_year#32]
+
+(30) Filter [codegen id : 9]
+Input [2]: [d_date_sk#31, d_year#32]
+Condition : ((isnotnull(d_year#32) AND (d_year#32 = 2002)) AND isnotnull(d_date_sk#31))
+
+(31) BroadcastExchange
+Input [2]: [d_date_sk#31, d_year#32]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#33]
+
+(32) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_sold_date_sk#29]
+Right keys [1]: [d_date_sk#31]
+Join condition: None
+
+(33) Project [codegen id : 10]
+Output [4]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#32]
+Input [6]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29, d_date_sk#31, d_year#32]
+
+(34) Exchange
+Input [4]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#32]
+Arguments: hashpartitioning(ss_customer_sk#26, 5), ENSURE_REQUIREMENTS, [id=#34]
+
+(35) Sort [codegen id : 11]
+Input [4]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#32]
+Arguments: [ss_customer_sk#26 ASC NULLS FIRST], false, 0
+
+(36) ReusedExchange [Reuses operator id: 15]
+Output [8]: [c_customer_sk#35, c_customer_id#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_country#40, c_login#41, c_email_address#42]
+
+(37) Sort [codegen id : 13]
+Input [8]: [c_customer_sk#35, c_customer_id#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_country#40, c_login#41, c_email_address#42]
+Arguments: [c_customer_sk#35 ASC NULLS FIRST], false, 0
+
+(38) SortMergeJoin [codegen id : 14]
+Left keys [1]: [ss_customer_sk#26]
+Right keys [1]: [c_customer_sk#35]
+Join condition: None
+
+(39) Project [codegen id : 14]
+Output [10]: [c_customer_id#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_country#40, c_login#41, c_email_address#42, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#32]
+Input [12]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#32, c_customer_sk#35, c_customer_id#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_country#40, c_login#41, c_email_address#42]
+
+(40) HashAggregate [codegen id : 14]
+Input [10]: [c_customer_id#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_country#40, c_login#41, c_email_address#42, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#32]
+Keys [8]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#32, c_preferred_cust_flag#39, c_birth_country#40, c_login#41, c_email_address#42]
+Functions [1]: [partial_sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#28 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#27 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum#43]
+Results [9]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#32, c_preferred_cust_flag#39, c_birth_country#40, c_login#41, c_email_address#42, sum#44]
+
+(41) Exchange
+Input [9]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#32, c_preferred_cust_flag#39, c_birth_country#40, c_login#41, c_email_address#42, sum#44]
+Arguments: hashpartitioning(c_customer_id#36, c_first_name#37, c_last_name#38, d_year#32, c_preferred_cust_flag#39, c_birth_country#40, c_login#41, c_email_address#42, 5), ENSURE_REQUIREMENTS, [id=#45]
+
+(42) HashAggregate [codegen id : 15]
+Input [9]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#32, c_preferred_cust_flag#39, c_birth_country#40, c_login#41, c_email_address#42, sum#44]
+Keys [8]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#32, c_preferred_cust_flag#39, c_birth_country#40, c_login#41, c_email_address#42]
+Functions [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#28 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#27 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#28 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#27 as decimal(8,2)))), DecimalType(8,2), true)))#46]
+Results [3]: [c_customer_id#36 AS customer_id#47, c_preferred_cust_flag#39 AS customer_preferred_cust_flag#48, MakeDecimal(sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#28 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#27 as decimal(8,2)))), DecimalType(8,2), true)))#46,18,2) AS year_total#49]
+
+(43) Exchange
+Input [3]: [customer_id#47, customer_preferred_cust_flag#48, year_total#49]
+Arguments: hashpartitioning(customer_id#47, 5), ENSURE_REQUIREMENTS, [id=#50]
+
+(44) Sort [codegen id : 16]
+Input [3]: [customer_id#47, customer_preferred_cust_flag#48, year_total#49]
+Arguments: [customer_id#47 ASC NULLS FIRST], false, 0
+
+(45) SortMergeJoin [codegen id : 17]
+Left keys [1]: [customer_id#23]
+Right keys [1]: [customer_id#47]
+Join condition: None
+
+(46) Project [codegen id : 17]
+Output [4]: [customer_id#23, year_total#24, customer_preferred_cust_flag#48, year_total#49]
+Input [5]: [customer_id#23, year_total#24, customer_id#47, customer_preferred_cust_flag#48, year_total#49]
+
+(47) Scan parquet default.web_sales
+Output [4]: [ws_bill_customer_sk#51, ws_ext_discount_amt#52, ws_ext_list_price#53, ws_sold_date_sk#54]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#54), dynamicpruningexpression(ws_sold_date_sk#54 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int,ws_ext_discount_amt:decimal(7,2),ws_ext_list_price:decimal(7,2)>
+
+(48) ColumnarToRow [codegen id : 19]
+Input [4]: [ws_bill_customer_sk#51, ws_ext_discount_amt#52, ws_ext_list_price#53, ws_sold_date_sk#54]
+
+(49) Filter [codegen id : 19]
+Input [4]: [ws_bill_customer_sk#51, ws_ext_discount_amt#52, ws_ext_list_price#53, ws_sold_date_sk#54]
+Condition : isnotnull(ws_bill_customer_sk#51)
+
+(50) ReusedExchange [Reuses operator id: 7]
+Output [2]: [d_date_sk#55, d_year#56]
+
+(51) BroadcastHashJoin [codegen id : 19]
+Left keys [1]: [ws_sold_date_sk#54]
+Right keys [1]: [d_date_sk#55]
+Join condition: None
+
+(52) Project [codegen id : 19]
+Output [4]: [ws_bill_customer_sk#51, ws_ext_discount_amt#52, ws_ext_list_price#53, d_year#56]
+Input [6]: [ws_bill_customer_sk#51, ws_ext_discount_amt#52, ws_ext_list_price#53, ws_sold_date_sk#54, d_date_sk#55, d_year#56]
+
+(53) Exchange
+Input [4]: [ws_bill_customer_sk#51, ws_ext_discount_amt#52, ws_ext_list_price#53, d_year#56]
+Arguments: hashpartitioning(ws_bill_customer_sk#51, 5), ENSURE_REQUIREMENTS, [id=#57]
+
+(54) Sort [codegen id : 20]
+Input [4]: [ws_bill_customer_sk#51, ws_ext_discount_amt#52, ws_ext_list_price#53, d_year#56]
+Arguments: [ws_bill_customer_sk#51 ASC NULLS FIRST], false, 0
+
+(55) ReusedExchange [Reuses operator id: 15]
+Output [8]: [c_customer_sk#58, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65]
+
+(56) Sort [codegen id : 22]
+Input [8]: [c_customer_sk#58, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65]
+Arguments: [c_customer_sk#58 ASC NULLS FIRST], false, 0
+
+(57) SortMergeJoin [codegen id : 23]
+Left keys [1]: [ws_bill_customer_sk#51]
+Right keys [1]: [c_customer_sk#58]
+Join condition: None
+
+(58) Project [codegen id : 23]
+Output [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#52, ws_ext_list_price#53, d_year#56]
+Input [12]: [ws_bill_customer_sk#51, ws_ext_discount_amt#52, ws_ext_list_price#53, d_year#56, c_customer_sk#58, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65]
+
+(59) HashAggregate [codegen id : 23]
+Input [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#52, ws_ext_list_price#53, d_year#56]
+Keys [8]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#56]
+Functions [1]: [partial_sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#53 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#52 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum#66]
+Results [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#56, sum#67]
+
+(60) Exchange
+Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#56, sum#67]
+Arguments: hashpartitioning(c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#56, 5), ENSURE_REQUIREMENTS, [id=#68]
+
+(61) HashAggregate [codegen id : 24]
+Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#56, sum#67]
+Keys [8]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#56]
+Functions [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#53 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#52 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#53 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#52 as decimal(8,2)))), DecimalType(8,2), true)))#69]
+Results [2]: [c_customer_id#59 AS customer_id#70, MakeDecimal(sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#53 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#52 as decimal(8,2)))), DecimalType(8,2), true)))#69,18,2) AS year_total#71]
+
+(62) Filter [codegen id : 24]
+Input [2]: [customer_id#70, year_total#71]
+Condition : (isnotnull(year_total#71) AND (year_total#71 > 0.00))
+
+(63) Project [codegen id : 24]
+Output [2]: [customer_id#70 AS customer_id#72, year_total#71 AS year_total#73]
+Input [2]: [customer_id#70, year_total#71]
+
+(64) Exchange
+Input [2]: [customer_id#72, year_total#73]
+Arguments: hashpartitioning(customer_id#72, 5), ENSURE_REQUIREMENTS, [id=#74]
+
+(65) Sort [codegen id : 25]
+Input [2]: [customer_id#72, year_total#73]
+Arguments: [customer_id#72 ASC NULLS FIRST], false, 0
+
+(66) SortMergeJoin [codegen id : 26]
+Left keys [1]: [customer_id#23]
+Right keys [1]: [customer_id#72]
+Join condition: None
+
+(67) Project [codegen id : 26]
+Output [5]: [customer_id#23, year_total#24, customer_preferred_cust_flag#48, year_total#49, year_total#73]
+Input [6]: [customer_id#23, year_total#24, customer_preferred_cust_flag#48, year_total#49, customer_id#72, year_total#73]
+
+(68) Scan parquet default.web_sales
+Output [4]: [ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#78), dynamicpruningexpression(ws_sold_date_sk#78 IN dynamicpruning#30)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int,ws_ext_discount_amt:decimal(7,2),ws_ext_list_price:decimal(7,2)>
+
+(69) ColumnarToRow [codegen id : 28]
+Input [4]: [ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78]
+
+(70) Filter [codegen id : 28]
+Input [4]: [ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78]
+Condition : isnotnull(ws_bill_customer_sk#75)
+
+(71) ReusedExchange [Reuses operator id: 31]
+Output [2]: [d_date_sk#79, d_year#80]
+
+(72) BroadcastHashJoin [codegen id : 28]
+Left keys [1]: [ws_sold_date_sk#78]
+Right keys [1]: [d_date_sk#79]
+Join condition: None
+
+(73) Project [codegen id : 28]
+Output [4]: [ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, d_year#80]
+Input [6]: [ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78, d_date_sk#79, d_year#80]
+
+(74) Exchange
+Input [4]: [ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, d_year#80]
+Arguments: hashpartitioning(ws_bill_customer_sk#75, 5), ENSURE_REQUIREMENTS, [id=#81]
+
+(75) Sort [codegen id : 29]
+Input [4]: [ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, d_year#80]
+Arguments: [ws_bill_customer_sk#75 ASC NULLS FIRST], false, 0
+
+(76) ReusedExchange [Reuses operator id: 15]
+Output [8]: [c_customer_sk#82, c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#87, c_login#88, c_email_address#89]
+
+(77) Sort [codegen id : 31]
+Input [8]: [c_customer_sk#82, c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#87, c_login#88, c_email_address#89]
+Arguments: [c_customer_sk#82 ASC NULLS FIRST], false, 0
+
+(78) SortMergeJoin [codegen id : 32]
+Left keys [1]: [ws_bill_customer_sk#75]
+Right keys [1]: [c_customer_sk#82]
+Join condition: None
+
+(79) Project [codegen id : 32]
+Output [10]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#87, c_login#88, c_email_address#89, ws_ext_discount_amt#76, ws_ext_list_price#77, d_year#80]
+Input [12]: [ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, d_year#80, c_customer_sk#82, c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#87, c_login#88, c_email_address#89]
+
+(80) HashAggregate [codegen id : 32]
+Input [10]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#87, c_login#88, c_email_address#89, ws_ext_discount_amt#76, ws_ext_list_price#77, d_year#80]
+Keys [8]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#87, c_login#88, c_email_address#89, d_year#80]
+Functions [1]: [partial_sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#77 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#76 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum#90]
+Results [9]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#87, c_login#88, c_email_address#89, d_year#80, sum#91]
+
+(81) Exchange
+Input [9]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#87, c_login#88, c_email_address#89, d_year#80, sum#91]
+Arguments: hashpartitioning(c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#87, c_login#88, c_email_address#89, d_year#80, 5), ENSURE_REQUIREMENTS, [id=#92]
+
+(82) HashAggregate [codegen id : 33]
+Input [9]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#87, c_login#88, c_email_address#89, d_year#80, sum#91]
+Keys [8]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#87, c_login#88, c_email_address#89, d_year#80]
+Functions [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#77 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#76 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#77 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#76 as decimal(8,2)))), DecimalType(8,2), true)))#93]
+Results [2]: [c_customer_id#83 AS customer_id#94, MakeDecimal(sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#77 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#76 as decimal(8,2)))), DecimalType(8,2), true)))#93,18,2) AS year_total#95]
+
+(83) Exchange
+Input [2]: [customer_id#94, year_total#95]
+Arguments: hashpartitioning(customer_id#94, 5), ENSURE_REQUIREMENTS, [id=#96]
+
+(84) Sort [codegen id : 34]
+Input [2]: [customer_id#94, year_total#95]
+Arguments: [customer_id#94 ASC NULLS FIRST], false, 0
+
+(85) SortMergeJoin [codegen id : 35]
+Left keys [1]: [customer_id#23]
+Right keys [1]: [customer_id#94]
+Join condition: (CASE WHEN (year_total#73 > 0.00) THEN CheckOverflow((promote_precision(year_total#95) / promote_precision(year_total#73)), DecimalType(38,20), true) ELSE null END > CASE WHEN (year_total#24 > 0.00) THEN CheckOverflow((promote_precision(year_total#49) / promote_precision(year_total#24)), DecimalType(38,20), true) ELSE null END)
+
+(86) Project [codegen id : 35]
+Output [1]: [customer_preferred_cust_flag#48]
+Input [7]: [customer_id#23, year_total#24, customer_preferred_cust_flag#48, year_total#49, year_total#73, customer_id#94, year_total#95]
+
+(87) TakeOrderedAndProject
+Input [1]: [customer_preferred_cust_flag#48]
+Arguments: 100, [customer_preferred_cust_flag#48 ASC NULLS FIRST], [customer_preferred_cust_flag#48]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (88)
+
+
+(88) ReusedExchange [Reuses operator id: 7]
+Output [2]: [d_date_sk#6, d_year#7]
+
+Subquery:2 Hosting operator id = 25 Hosting Expression = ss_sold_date_sk#29 IN dynamicpruning#30
+ReusedExchange (89)
+
+
+(89) ReusedExchange [Reuses operator id: 31]
+Output [2]: [d_date_sk#31, d_year#32]
+
+Subquery:3 Hosting operator id = 47 Hosting Expression = ws_sold_date_sk#54 IN dynamicpruning#5
+
+Subquery:4 Hosting operator id = 68 Hosting Expression = ws_sold_date_sk#78 IN dynamicpruning#30
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.sf100/simplified.txt
new file mode 100644
index 0000000000000..006f6a9f018ce
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.sf100/simplified.txt
@@ -0,0 +1,164 @@
+TakeOrderedAndProject [customer_preferred_cust_flag]
+  WholeStageCodegen (35)
+    Project [customer_preferred_cust_flag]
+      SortMergeJoin [customer_id,customer_id,year_total,year_total,year_total,year_total]
+        InputAdapter
+          WholeStageCodegen (26)
+            Project [customer_id,year_total,customer_preferred_cust_flag,year_total,year_total]
+              SortMergeJoin [customer_id,customer_id]
+                InputAdapter
+                  WholeStageCodegen (17)
+                    Project [customer_id,year_total,customer_preferred_cust_flag,year_total]
+                      SortMergeJoin [customer_id,customer_id]
+                        InputAdapter
+                          WholeStageCodegen (8)
+                            Sort [customer_id]
+                              InputAdapter
+                                Exchange [customer_id] #1
+                                  WholeStageCodegen (7)
+                                    Filter [year_total]
+                                      HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt as decimal(8,2)))), DecimalType(8,2), true))),customer_id,year_total,sum]
+                                        InputAdapter
+                                          Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #2
+                                            WholeStageCodegen (6)
+                                              HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum]
+                                                Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year]
+                                                  SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                                    InputAdapter
+                                                      WholeStageCodegen (3)
+                                                        Sort [ss_customer_sk]
+                                                          InputAdapter
+                                                            Exchange [ss_customer_sk] #3
+                                                              WholeStageCodegen (2)
+                                                                Project [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,d_year]
+                                                                  BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                    Filter [ss_customer_sk]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk]
+                                                                            SubqueryBroadcast [d_date_sk] #1
+                                                                              ReusedExchange [d_date_sk,d_year] #4
+                                                                    InputAdapter
+                                                                      BroadcastExchange #4
+                                                                        WholeStageCodegen (1)
+                                                                          Filter [d_year,d_date_sk]
+                                                                            ColumnarToRow
+                                                                              InputAdapter
+                                                                                Scan parquet default.date_dim [d_date_sk,d_year]
+                                                    InputAdapter
+                                                      WholeStageCodegen (5)
+                                                        Sort [c_customer_sk]
+                                                          InputAdapter
+                                                            Exchange [c_customer_sk] #5
+                                                              WholeStageCodegen (4)
+                                                                Filter [c_customer_sk,c_customer_id]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address]
+                        InputAdapter
+                          WholeStageCodegen (16)
+                            Sort [customer_id]
+                              InputAdapter
+                                Exchange [customer_id] #6
+                                  WholeStageCodegen (15)
+                                    HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt as decimal(8,2)))), DecimalType(8,2), true))),customer_id,customer_preferred_cust_flag,year_total,sum]
+                                      InputAdapter
+                                        Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #7
+                                          WholeStageCodegen (14)
+                                            HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum]
+                                              Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year]
+                                                SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (11)
+                                                      Sort [ss_customer_sk]
+                                                        InputAdapter
+                                                          Exchange [ss_customer_sk] #8
+                                                            WholeStageCodegen (10)
+                                                              Project [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,d_year]
+                                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                  Filter [ss_customer_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk]
+                                                                          SubqueryBroadcast [d_date_sk] #2
+                                                                            ReusedExchange [d_date_sk,d_year] #9
+                                                                  InputAdapter
+                                                                    BroadcastExchange #9
+                                                                      WholeStageCodegen (9)
+                                                                        Filter [d_year,d_date_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.date_dim [d_date_sk,d_year]
+                                                  InputAdapter
+                                                    WholeStageCodegen (13)
+                                                      Sort [c_customer_sk]
+                                                        InputAdapter
+                                                          ReusedExchange [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5
+                InputAdapter
+                  WholeStageCodegen (25)
+                    Sort [customer_id]
+                      InputAdapter
+                        Exchange [customer_id] #10
+                          WholeStageCodegen (24)
+                            Project [customer_id,year_total]
+                              Filter [year_total]
+                                HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt as decimal(8,2)))), DecimalType(8,2), true))),customer_id,year_total,sum]
+                                  InputAdapter
+                                    Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11
+                                      WholeStageCodegen (23)
+                                        HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum]
+                                          Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year]
+                                            SortMergeJoin [ws_bill_customer_sk,c_customer_sk]
+                                              InputAdapter
+                                                WholeStageCodegen (20)
+                                                  Sort [ws_bill_customer_sk]
+                                                    InputAdapter
+                                                      Exchange [ws_bill_customer_sk] #12
+                                                        WholeStageCodegen (19)
+                                                          Project [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,d_year]
+                                                            BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                              Filter [ws_bill_customer_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk]
+                                                                      ReusedSubquery [d_date_sk] #1
+                                                              InputAdapter
+                                                                ReusedExchange [d_date_sk,d_year] #4
+                                              InputAdapter
+                                                WholeStageCodegen (22)
+                                                  Sort [c_customer_sk]
+                                                    InputAdapter
+                                                      ReusedExchange [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5
+        InputAdapter
+          WholeStageCodegen (34)
+            Sort [customer_id]
+              InputAdapter
+                Exchange [customer_id] #13
+                  WholeStageCodegen (33)
+                    HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt as decimal(8,2)))), DecimalType(8,2), true))),customer_id,year_total,sum]
+                      InputAdapter
+                        Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14
+                          WholeStageCodegen (32)
+                            HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum]
+                              Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year]
+                                SortMergeJoin [ws_bill_customer_sk,c_customer_sk]
+                                  InputAdapter
+                                    WholeStageCodegen (29)
+                                      Sort [ws_bill_customer_sk]
+                                        InputAdapter
+                                          Exchange [ws_bill_customer_sk] #15
+                                            WholeStageCodegen (28)
+                                              Project [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,d_year]
+                                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                  Filter [ws_bill_customer_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk]
+                                                          ReusedSubquery [d_date_sk] #2
+                                                  InputAdapter
+                                                    ReusedExchange [d_date_sk,d_year] #9
+                                  InputAdapter
+                                    WholeStageCodegen (31)
+                                      Sort [c_customer_sk]
+                                        InputAdapter
+                                          ReusedExchange [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/explain.txt
new file mode 100644
index 0000000000000..4081f69732168
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/explain.txt
@@ -0,0 +1,476 @@
+== Physical Plan ==
+TakeOrderedAndProject (79)
++- * Project (78)
+   +- * BroadcastHashJoin Inner BuildRight (77)
+      :- * Project (60)
+      :  +- * BroadcastHashJoin Inner BuildRight (59)
+      :     :- * Project (40)
+      :     :  +- * BroadcastHashJoin Inner BuildRight (39)
+      :     :     :- * Filter (19)
+      :     :     :  +- * HashAggregate (18)
+      :     :     :     +- Exchange (17)
+      :     :     :        +- * HashAggregate (16)
+      :     :     :           +- * Project (15)
+      :     :     :              +- * BroadcastHashJoin Inner BuildRight (14)
+      :     :     :                 :- * Project (9)
+      :     :     :                 :  +- * BroadcastHashJoin Inner BuildRight (8)
+      :     :     :                 :     :- * Filter (3)
+      :     :     :                 :     :  +- * ColumnarToRow (2)
+      :     :     :                 :     :     +- Scan parquet default.customer (1)
+      :     :     :                 :     +- BroadcastExchange (7)
+      :     :     :                 :        +- * Filter (6)
+      :     :     :                 :           +- * ColumnarToRow (5)
+      :     :     :                 :              +- Scan parquet default.store_sales (4)
+      :     :     :                 +- BroadcastExchange (13)
+      :     :     :                    +- * Filter (12)
+      :     :     :                       +- * ColumnarToRow (11)
+      :     :     :                          +- Scan parquet default.date_dim (10)
+      :     :     +- BroadcastExchange (38)
+      :     :        +- * HashAggregate (37)
+      :     :           +- Exchange (36)
+      :     :              +- * HashAggregate (35)
+      :     :                 +- * Project (34)
+      :     :                    +- * BroadcastHashJoin Inner BuildRight (33)
+      :     :                       :- * Project (28)
+      :     :                       :  +- * BroadcastHashJoin Inner BuildRight (27)
+      :     :                       :     :- * Filter (22)
+      :     :                       :     :  +- * ColumnarToRow (21)
+      :     :                       :     :     +- Scan parquet default.customer (20)
+      :     :                       :     +- BroadcastExchange (26)
+      :     :                       :        +- * Filter (25)
+      :     :                       :           +- * ColumnarToRow (24)
+      :     :                       :              +- Scan parquet default.store_sales (23)
+      :     :                       +- BroadcastExchange (32)
+      :     :                          +- * Filter (31)
+      :     :                             +- * ColumnarToRow (30)
+      :     :                                +- Scan parquet default.date_dim (29)
+      :     +- BroadcastExchange (58)
+      :        +- * Project (57)
+      :           +- * Filter (56)
+      :              +- * HashAggregate (55)
+      :                 +- Exchange (54)
+      :                    +- * HashAggregate (53)
+      :                       +- * Project (52)
+      :                          +- * BroadcastHashJoin Inner BuildRight (51)
+      :                             :- * Project (49)
+      :                             :  +- * BroadcastHashJoin Inner BuildRight (48)
+      :                             :     :- * Filter (43)
+      :                             :     :  +- * ColumnarToRow (42)
+      :                             :     :     +- Scan parquet default.customer (41)
+      :                             :     +- BroadcastExchange (47)
+      :                             :        +- * Filter (46)
+      :                             :           +- * ColumnarToRow (45)
+      :                             :              +- Scan parquet default.web_sales (44)
+      :                             +- ReusedExchange (50)
+      +- BroadcastExchange (76)
+         +- * HashAggregate (75)
+            +- Exchange (74)
+               +- * HashAggregate (73)
+                  +- * Project (72)
+                     +- * BroadcastHashJoin Inner BuildRight (71)
+                        :- * Project (69)
+                        :  +- * BroadcastHashJoin Inner BuildRight (68)
+                        :     :- * Filter (63)
+                        :     :  +- * ColumnarToRow (62)
+                        :     :     +- Scan parquet default.customer (61)
+                        :     +- BroadcastExchange (67)
+                        :        +- * Filter (66)
+                        :           +- * ColumnarToRow (65)
+                        :              +- Scan parquet default.web_sales (64)
+                        +- ReusedExchange (70)
+
+
+(1) Scan parquet default.customer
+Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string,c_birth_country:string,c_login:string,c_email_address:string>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8]
+
+(3) Filter [codegen id : 3]
+Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8]
+Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2))
+
+(4) Scan parquet default.store_sales
+Output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_sold_date_sk#12 IN dynamicpruning#13)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_ext_discount_amt:decimal(7,2),ss_ext_list_price:decimal(7,2)>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12]
+
+(6) Filter [codegen id : 1]
+Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12]
+Condition : isnotnull(ss_customer_sk#9)
+
+(7) BroadcastExchange
+Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#14]
+
+(8) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [ss_customer_sk#9]
+Join condition: None
+
+(9) Project [codegen id : 3]
+Output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12]
+Input [12]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12]
+
+(10) Scan parquet default.date_dim
+Output [2]: [d_date_sk#15, d_year#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#15, d_year#16]
+
+(12) Filter [codegen id : 2]
+Input [2]: [d_date_sk#15, d_year#16]
+Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15))
+
+(13) BroadcastExchange
+Input [2]: [d_date_sk#15, d_year#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#17]
+
+(14) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#12]
+Right keys [1]: [d_date_sk#15]
+Join condition: None
+
+(15) Project [codegen id : 3]
+Output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#16]
+Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12, d_date_sk#15, d_year#16]
+
+(16) HashAggregate [codegen id : 3]
+Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#16]
+Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#16, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8]
+Functions [1]: [partial_sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#11 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#10 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum#18]
+Results [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#16, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#19]
+
+(17) Exchange
+Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#16, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#19]
+Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#16, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(18) HashAggregate [codegen id : 16]
+Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#16, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#19]
+Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#16, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8]
+Functions [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#11 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#10 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#11 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#10 as decimal(8,2)))), DecimalType(8,2), true)))#21]
+Results [2]: [c_customer_id#2 AS customer_id#22, MakeDecimal(sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#11 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#10 as decimal(8,2)))), DecimalType(8,2), true)))#21,18,2) AS year_total#23]
+
+(19) Filter [codegen id : 16]
+Input [2]: [customer_id#22, year_total#23]
+Condition : (isnotnull(year_total#23) AND (year_total#23 > 0.00))
+
+(20) Scan parquet default.customer
+Output [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string,c_birth_country:string,c_login:string,c_email_address:string>
+
+(21) ColumnarToRow [codegen id : 6]
+Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31]
+
+(22) Filter [codegen id : 6]
+Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31]
+Condition : (isnotnull(c_customer_sk#24) AND isnotnull(c_customer_id#25))
+
+(23) Scan parquet default.store_sales
+Output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#35), dynamicpruningexpression(ss_sold_date_sk#35 IN dynamicpruning#36)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_ext_discount_amt:decimal(7,2),ss_ext_list_price:decimal(7,2)>
+
+(24) ColumnarToRow [codegen id : 4]
+Input [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35]
+
+(25) Filter [codegen id : 4]
+Input [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35]
+Condition : isnotnull(ss_customer_sk#32)
+
+(26) BroadcastExchange
+Input [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#37]
+
+(27) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [c_customer_sk#24]
+Right keys [1]: [ss_customer_sk#32]
+Join condition: None
+
+(28) Project [codegen id : 6]
+Output [10]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35]
+Input [12]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35]
+
+(29) Scan parquet default.date_dim
+Output [2]: [d_date_sk#38, d_year#39]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(30) ColumnarToRow [codegen id : 5]
+Input [2]: [d_date_sk#38, d_year#39]
+
+(31) Filter [codegen id : 5]
+Input [2]: [d_date_sk#38, d_year#39]
+Condition : ((isnotnull(d_year#39) AND (d_year#39 = 2002)) AND isnotnull(d_date_sk#38))
+
+(32) BroadcastExchange
+Input [2]: [d_date_sk#38, d_year#39]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#40]
+
+(33) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_sold_date_sk#35]
+Right keys [1]: [d_date_sk#38]
+Join condition: None
+
+(34) Project [codegen id : 6]
+Output [10]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#39]
+Input [12]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35, d_date_sk#38, d_year#39]
+
+(35) HashAggregate [codegen id : 6]
+Input [10]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#39]
+Keys [8]: [c_customer_id#25, c_first_name#26, c_last_name#27, d_year#39, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31]
+Functions [1]: [partial_sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#34 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#33 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum#41]
+Results [9]: [c_customer_id#25, c_first_name#26, c_last_name#27, d_year#39, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, sum#42]
+
+(36) Exchange
+Input [9]: [c_customer_id#25, c_first_name#26, c_last_name#27, d_year#39, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, sum#42]
+Arguments: hashpartitioning(c_customer_id#25, c_first_name#26, c_last_name#27, d_year#39, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, 5), ENSURE_REQUIREMENTS, [id=#43]
+
+(37) HashAggregate [codegen id : 7]
+Input [9]: [c_customer_id#25, c_first_name#26, c_last_name#27, d_year#39, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, sum#42]
+Keys [8]: [c_customer_id#25, c_first_name#26, c_last_name#27, d_year#39, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31]
+Functions [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#34 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#33 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#34 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#33 as decimal(8,2)))), DecimalType(8,2), true)))#44]
+Results [3]: [c_customer_id#25 AS customer_id#45, c_preferred_cust_flag#28 AS customer_preferred_cust_flag#46, MakeDecimal(sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#34 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#33 as decimal(8,2)))), DecimalType(8,2), true)))#44,18,2) AS year_total#47]
+
+(38) BroadcastExchange
+Input [3]: [customer_id#45, customer_preferred_cust_flag#46, year_total#47]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#48]
+
+(39) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [customer_id#22]
+Right keys [1]: [customer_id#45]
+Join condition: None
+
+(40) Project [codegen id : 16]
+Output [4]: [customer_id#22, year_total#23, customer_preferred_cust_flag#46, year_total#47]
+Input [5]: [customer_id#22, year_total#23, customer_id#45, customer_preferred_cust_flag#46, year_total#47]
+
+(41) Scan parquet default.customer
+Output [8]: [c_customer_sk#49, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string,c_birth_country:string,c_login:string,c_email_address:string>
+
+(42) ColumnarToRow [codegen id : 10]
+Input [8]: [c_customer_sk#49, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56]
+
+(43) Filter [codegen id : 10]
+Input [8]: [c_customer_sk#49, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56]
+Condition : (isnotnull(c_customer_sk#49) AND isnotnull(c_customer_id#50))
+
+(44) Scan parquet default.web_sales
+Output [4]: [ws_bill_customer_sk#57, ws_ext_discount_amt#58, ws_ext_list_price#59, ws_sold_date_sk#60]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#60), dynamicpruningexpression(ws_sold_date_sk#60 IN dynamicpruning#13)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int,ws_ext_discount_amt:decimal(7,2),ws_ext_list_price:decimal(7,2)>
+
+(45) ColumnarToRow [codegen id : 8]
+Input [4]: [ws_bill_customer_sk#57, ws_ext_discount_amt#58, ws_ext_list_price#59, ws_sold_date_sk#60]
+
+(46) Filter [codegen id : 8]
+Input [4]: [ws_bill_customer_sk#57, ws_ext_discount_amt#58, ws_ext_list_price#59, ws_sold_date_sk#60]
+Condition : isnotnull(ws_bill_customer_sk#57)
+
+(47) BroadcastExchange
+Input [4]: [ws_bill_customer_sk#57, ws_ext_discount_amt#58, ws_ext_list_price#59, ws_sold_date_sk#60]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#61]
+
+(48) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [c_customer_sk#49]
+Right keys [1]: [ws_bill_customer_sk#57]
+Join condition: None
+
+(49) Project [codegen id : 10]
+Output [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, ws_ext_discount_amt#58, ws_ext_list_price#59, ws_sold_date_sk#60]
+Input [12]: [c_customer_sk#49, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, ws_bill_customer_sk#57, ws_ext_discount_amt#58, ws_ext_list_price#59, ws_sold_date_sk#60]
+
+(50) ReusedExchange [Reuses operator id: 13]
+Output [2]: [d_date_sk#62, d_year#63]
+
+(51) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ws_sold_date_sk#60]
+Right keys [1]: [d_date_sk#62]
+Join condition: None
+
+(52) Project [codegen id : 10]
+Output [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, ws_ext_discount_amt#58, ws_ext_list_price#59, d_year#63]
+Input [12]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, ws_ext_discount_amt#58, ws_ext_list_price#59, ws_sold_date_sk#60, d_date_sk#62, d_year#63]
+
+(53) HashAggregate [codegen id : 10]
+Input [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, ws_ext_discount_amt#58, ws_ext_list_price#59, d_year#63]
+Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, d_year#63]
+Functions [1]: [partial_sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#59 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#58 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum#64]
+Results [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, d_year#63, sum#65]
+
+(54) Exchange
+Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, d_year#63, sum#65]
+Arguments: hashpartitioning(c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, d_year#63, 5), ENSURE_REQUIREMENTS, [id=#66]
+
+(55) HashAggregate [codegen id : 11]
+Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, d_year#63, sum#65]
+Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, d_year#63]
+Functions [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#59 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#58 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#59 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#58 as decimal(8,2)))), DecimalType(8,2), true)))#67]
+Results [2]: [c_customer_id#50 AS customer_id#68, MakeDecimal(sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#59 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#58 as decimal(8,2)))), DecimalType(8,2), true)))#67,18,2) AS year_total#69]
+
+(56) Filter [codegen id : 11]
+Input [2]: [customer_id#68, year_total#69]
+Condition : (isnotnull(year_total#69) AND (year_total#69 > 0.00))
+
+(57) Project [codegen id : 11]
+Output [2]: [customer_id#68 AS customer_id#70, year_total#69 AS year_total#71]
+Input [2]: [customer_id#68, year_total#69]
+
+(58) BroadcastExchange
+Input [2]: [customer_id#70, year_total#71]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#72]
+
+(59) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [customer_id#22]
+Right keys [1]: [customer_id#70]
+Join condition: None
+
+(60) Project [codegen id : 16]
+Output [5]: [customer_id#22, year_total#23, customer_preferred_cust_flag#46, year_total#47, year_total#71]
+Input [6]: [customer_id#22, year_total#23, customer_preferred_cust_flag#46, year_total#47, customer_id#70, year_total#71]
+
+(61) Scan parquet default.customer
+Output [8]: [c_customer_sk#73, c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string,c_birth_country:string,c_login:string,c_email_address:string>
+
+(62) ColumnarToRow [codegen id : 14]
+Input [8]: [c_customer_sk#73, c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80]
+
+(63) Filter [codegen id : 14]
+Input [8]: [c_customer_sk#73, c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80]
+Condition : (isnotnull(c_customer_sk#73) AND isnotnull(c_customer_id#74))
+
+(64) Scan parquet default.web_sales
+Output [4]: [ws_bill_customer_sk#81, ws_ext_discount_amt#82, ws_ext_list_price#83, ws_sold_date_sk#84]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#84), dynamicpruningexpression(ws_sold_date_sk#84 IN dynamicpruning#36)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int,ws_ext_discount_amt:decimal(7,2),ws_ext_list_price:decimal(7,2)>
+
+(65) ColumnarToRow [codegen id : 12]
+Input [4]: [ws_bill_customer_sk#81, ws_ext_discount_amt#82, ws_ext_list_price#83, ws_sold_date_sk#84]
+
+(66) Filter [codegen id : 12]
+Input [4]: [ws_bill_customer_sk#81, ws_ext_discount_amt#82, ws_ext_list_price#83, ws_sold_date_sk#84]
+Condition : isnotnull(ws_bill_customer_sk#81)
+
+(67) BroadcastExchange
+Input [4]: [ws_bill_customer_sk#81, ws_ext_discount_amt#82, ws_ext_list_price#83, ws_sold_date_sk#84]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#85]
+
+(68) BroadcastHashJoin [codegen id : 14]
+Left keys [1]: [c_customer_sk#73]
+Right keys [1]: [ws_bill_customer_sk#81]
+Join condition: None
+
+(69) Project [codegen id : 14]
+Output [10]: [c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80, ws_ext_discount_amt#82, ws_ext_list_price#83, ws_sold_date_sk#84]
+Input [12]: [c_customer_sk#73, c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80, ws_bill_customer_sk#81, ws_ext_discount_amt#82, ws_ext_list_price#83, ws_sold_date_sk#84]
+
+(70) ReusedExchange [Reuses operator id: 32]
+Output [2]: [d_date_sk#86, d_year#87]
+
+(71) BroadcastHashJoin [codegen id : 14]
+Left keys [1]: [ws_sold_date_sk#84]
+Right keys [1]: [d_date_sk#86]
+Join condition: None
+
+(72) Project [codegen id : 14]
+Output [10]: [c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80, ws_ext_discount_amt#82, ws_ext_list_price#83, d_year#87]
+Input [12]: [c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80, ws_ext_discount_amt#82, ws_ext_list_price#83, ws_sold_date_sk#84, d_date_sk#86, d_year#87]
+
+(73) HashAggregate [codegen id : 14]
+Input [10]: [c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80, ws_ext_discount_amt#82, ws_ext_list_price#83, d_year#87]
+Keys [8]: [c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80, d_year#87]
+Functions [1]: [partial_sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#83 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#82 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum#88]
+Results [9]: [c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80, d_year#87, sum#89]
+
+(74) Exchange
+Input [9]: [c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80, d_year#87, sum#89]
+Arguments: hashpartitioning(c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80, d_year#87, 5), ENSURE_REQUIREMENTS, [id=#90]
+
+(75) HashAggregate [codegen id : 15]
+Input [9]: [c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80, d_year#87, sum#89]
+Keys [8]: [c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80, d_year#87]
+Functions [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#83 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#82 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#83 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#82 as decimal(8,2)))), DecimalType(8,2), true)))#91]
+Results [2]: [c_customer_id#74 AS customer_id#92, MakeDecimal(sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#83 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#82 as decimal(8,2)))), DecimalType(8,2), true)))#91,18,2) AS year_total#93]
+
+(76) BroadcastExchange
+Input [2]: [customer_id#92, year_total#93]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#94]
+
+(77) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [customer_id#22]
+Right keys [1]: [customer_id#92]
+Join condition: (CASE WHEN (year_total#71 > 0.00) THEN CheckOverflow((promote_precision(year_total#93) / promote_precision(year_total#71)), DecimalType(38,20), true) ELSE null END > CASE WHEN (year_total#23 > 0.00) THEN CheckOverflow((promote_precision(year_total#47) / promote_precision(year_total#23)), DecimalType(38,20), true) ELSE null END)
+
+(78) Project [codegen id : 16]
+Output [1]: [customer_preferred_cust_flag#46]
+Input [7]: [customer_id#22, year_total#23, customer_preferred_cust_flag#46, year_total#47, year_total#71, customer_id#92, year_total#93]
+
+(79) TakeOrderedAndProject
+Input [1]: [customer_preferred_cust_flag#46]
+Arguments: 100, [customer_preferred_cust_flag#46 ASC NULLS FIRST], [customer_preferred_cust_flag#46]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13
+ReusedExchange (80)
+
+
+(80) ReusedExchange [Reuses operator id: 13]
+Output [2]: [d_date_sk#15, d_year#16]
+
+Subquery:2 Hosting operator id = 23 Hosting Expression = ss_sold_date_sk#35 IN dynamicpruning#36
+ReusedExchange (81)
+
+
+(81) ReusedExchange [Reuses operator id: 32]
+Output [2]: [d_date_sk#38, d_year#39]
+
+Subquery:3 Hosting operator id = 44 Hosting Expression = ws_sold_date_sk#60 IN dynamicpruning#13
+
+Subquery:4 Hosting operator id = 64 Hosting Expression = ws_sold_date_sk#84 IN dynamicpruning#36
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/simplified.txt
new file mode 100644
index 0000000000000..6dbf8b3bf93f3
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/simplified.txt
@@ -0,0 +1,124 @@
+TakeOrderedAndProject [customer_preferred_cust_flag]
+  WholeStageCodegen (16)
+    Project [customer_preferred_cust_flag]
+      BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total]
+        Project [customer_id,year_total,customer_preferred_cust_flag,year_total,year_total]
+          BroadcastHashJoin [customer_id,customer_id]
+            Project [customer_id,year_total,customer_preferred_cust_flag,year_total]
+              BroadcastHashJoin [customer_id,customer_id]
+                Filter [year_total]
+                  HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt as decimal(8,2)))), DecimalType(8,2), true))),customer_id,year_total,sum]
+                    InputAdapter
+                      Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1
+                        WholeStageCodegen (3)
+                          HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum]
+                            Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year]
+                              BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk]
+                                  BroadcastHashJoin [c_customer_sk,ss_customer_sk]
+                                    Filter [c_customer_sk,c_customer_id]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address]
+                                    InputAdapter
+                                      BroadcastExchange #2
+                                        WholeStageCodegen (1)
+                                          Filter [ss_customer_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk]
+                                                  SubqueryBroadcast [d_date_sk] #1
+                                                    ReusedExchange [d_date_sk,d_year] #3
+                                InputAdapter
+                                  BroadcastExchange #3
+                                    WholeStageCodegen (2)
+                                      Filter [d_year,d_date_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.date_dim [d_date_sk,d_year]
+                InputAdapter
+                  BroadcastExchange #4
+                    WholeStageCodegen (7)
+                      HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt as decimal(8,2)))), DecimalType(8,2), true))),customer_id,customer_preferred_cust_flag,year_total,sum]
+                        InputAdapter
+                          Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5
+                            WholeStageCodegen (6)
+                              HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum]
+                                Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year]
+                                  BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                    Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk]
+                                      BroadcastHashJoin [c_customer_sk,ss_customer_sk]
+                                        Filter [c_customer_sk,c_customer_id]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address]
+                                        InputAdapter
+                                          BroadcastExchange #6
+                                            WholeStageCodegen (4)
+                                              Filter [ss_customer_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk]
+                                                      SubqueryBroadcast [d_date_sk] #2
+                                                        ReusedExchange [d_date_sk,d_year] #7
+                                    InputAdapter
+                                      BroadcastExchange #7
+                                        WholeStageCodegen (5)
+                                          Filter [d_year,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_year]
+            InputAdapter
+              BroadcastExchange #8
+                WholeStageCodegen (11)
+                  Project [customer_id,year_total]
+                    Filter [year_total]
+                      HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt as decimal(8,2)))), DecimalType(8,2), true))),customer_id,year_total,sum]
+                        InputAdapter
+                          Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #9
+                            WholeStageCodegen (10)
+                              HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum]
+                                Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year]
+                                  BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                    Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk]
+                                      BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk]
+                                        Filter [c_customer_sk,c_customer_id]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address]
+                                        InputAdapter
+                                          BroadcastExchange #10
+                                            WholeStageCodegen (8)
+                                              Filter [ws_bill_customer_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk]
+                                                      ReusedSubquery [d_date_sk] #1
+                                    InputAdapter
+                                      ReusedExchange [d_date_sk,d_year] #3
+        InputAdapter
+          BroadcastExchange #11
+            WholeStageCodegen (15)
+              HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt as decimal(8,2)))), DecimalType(8,2), true))),customer_id,year_total,sum]
+                InputAdapter
+                  Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #12
+                    WholeStageCodegen (14)
+                      HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum]
+                        Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year]
+                          BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                            Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk]
+                              BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk]
+                                Filter [c_customer_sk,c_customer_id]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address]
+                                InputAdapter
+                                  BroadcastExchange #13
+                                    WholeStageCodegen (12)
+                                      Filter [ws_bill_customer_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk]
+                                              ReusedSubquery [d_date_sk] #2
+                            InputAdapter
+                              ReusedExchange [d_date_sk,d_year] #7
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.sf100/explain.txt
new file mode 100644
index 0000000000000..7c414d9080734
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.sf100/explain.txt
@@ -0,0 +1,163 @@
+== Physical Plan ==
+TakeOrderedAndProject (27)
++- * Project (26)
+   +- Window (25)
+      +- * Sort (24)
+         +- Exchange (23)
+            +- * HashAggregate (22)
+               +- Exchange (21)
+                  +- * HashAggregate (20)
+                     +- * Project (19)
+                        +- * BroadcastHashJoin Inner BuildRight (18)
+                           :- * Project (12)
+                           :  +- * SortMergeJoin Inner (11)
+                           :     :- * Sort (5)
+                           :     :  +- Exchange (4)
+                           :     :     +- * Filter (3)
+                           :     :        +- * ColumnarToRow (2)
+                           :     :           +- Scan parquet default.web_sales (1)
+                           :     +- * Sort (10)
+                           :        +- Exchange (9)
+                           :           +- * Filter (8)
+                           :              +- * ColumnarToRow (7)
+                           :                 +- Scan parquet default.item (6)
+                           +- BroadcastExchange (17)
+                              +- * Project (16)
+                                 +- * Filter (15)
+                                    +- * ColumnarToRow (14)
+                                       +- Scan parquet default.date_dim (13)
+
+
+(1) Scan parquet default.web_sales
+Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3]
+
+(3) Filter [codegen id : 1]
+Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3]
+Condition : isnotnull(ws_item_sk#1)
+
+(4) Exchange
+Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3]
+Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#5]
+
+(5) Sort [codegen id : 2]
+Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3]
+Arguments: [ws_item_sk#1 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.item
+Output [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [In(i_category, [Sports                                            ,Books                                             ,Home                                              ]), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_item_desc:string,i_current_price:decimal(7,2),i_class:string,i_category:string>
+
+(7) ColumnarToRow [codegen id : 3]
+Input [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+
+(8) Filter [codegen id : 3]
+Input [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Condition : (i_category#11 IN (Sports                                            ,Books                                             ,Home                                              ) AND isnotnull(i_item_sk#6))
+
+(9) Exchange
+Input [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Arguments: hashpartitioning(i_item_sk#6, 5), ENSURE_REQUIREMENTS, [id=#12]
+
+(10) Sort [codegen id : 4]
+Input [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Arguments: [i_item_sk#6 ASC NULLS FIRST], false, 0
+
+(11) SortMergeJoin [codegen id : 6]
+Left keys [1]: [ws_item_sk#1]
+Right keys [1]: [i_item_sk#6]
+Join condition: None
+
+(12) Project [codegen id : 6]
+Output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+
+(13) Scan parquet default.date_dim
+Output [2]: [d_date_sk#13, d_date#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(14) ColumnarToRow [codegen id : 5]
+Input [2]: [d_date_sk#13, d_date#14]
+
+(15) Filter [codegen id : 5]
+Input [2]: [d_date_sk#13, d_date#14]
+Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13))
+
+(16) Project [codegen id : 5]
+Output [1]: [d_date_sk#13]
+Input [2]: [d_date_sk#13, d_date#14]
+
+(17) BroadcastExchange
+Input [1]: [d_date_sk#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#15]
+
+(18) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ws_sold_date_sk#3]
+Right keys [1]: [d_date_sk#13]
+Join condition: None
+
+(19) Project [codegen id : 6]
+Output [6]: [ws_ext_sales_price#2, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11, d_date_sk#13]
+
+(20) HashAggregate [codegen id : 6]
+Input [6]: [ws_ext_sales_price#2, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Keys [5]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9]
+Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum#16]
+Results [6]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, sum#17]
+
+(21) Exchange
+Input [6]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, sum#17]
+Arguments: hashpartitioning(i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(22) HashAggregate [codegen id : 7]
+Input [6]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, sum#17]
+Keys [5]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9]
+Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#2))#19]
+Results [8]: [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#19,17,2) AS itemrevenue#20, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#19,17,2) AS _w0#21, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#19,17,2) AS _w1#22, i_item_id#7]
+
+(23) Exchange
+Input [8]: [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, _w0#21, _w1#22, i_item_id#7]
+Arguments: hashpartitioning(i_class#10, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(24) Sort [codegen id : 8]
+Input [8]: [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, _w0#21, _w1#22, i_item_id#7]
+Arguments: [i_class#10 ASC NULLS FIRST], false, 0
+
+(25) Window
+Input [8]: [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, _w0#21, _w1#22, i_item_id#7]
+Arguments: [sum(_w1#22) windowspecdefinition(i_class#10, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#24], [i_class#10]
+
+(26) Project [codegen id : 9]
+Output [7]: [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(_w0#21) * 100.00), DecimalType(21,2), true) as decimal(27,2))) / promote_precision(_we0#24)), DecimalType(38,17), true) AS revenueratio#25, i_item_id#7]
+Input [9]: [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, _w0#21, _w1#22, i_item_id#7, _we0#24]
+
+(27) TakeOrderedAndProject
+Input [7]: [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, revenueratio#25, i_item_id#7]
+Arguments: 100, [i_category#11 ASC NULLS FIRST, i_class#10 ASC NULLS FIRST, i_item_id#7 ASC NULLS FIRST, i_item_desc#8 ASC NULLS FIRST, revenueratio#25 ASC NULLS FIRST], [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, revenueratio#25]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (28)
+
+
+(28) ReusedExchange [Reuses operator id: 17]
+Output [1]: [d_date_sk#13]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.sf100/simplified.txt
new file mode 100644
index 0000000000000..e981328e2d9b4
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.sf100/simplified.txt
@@ -0,0 +1,49 @@
+TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue]
+  WholeStageCodegen (9)
+    Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id]
+      InputAdapter
+        Window [_w1,i_class]
+          WholeStageCodegen (8)
+            Sort [i_class]
+              InputAdapter
+                Exchange [i_class] #1
+                  WholeStageCodegen (7)
+                    HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ws_ext_sales_price)),itemrevenue,_w0,_w1,sum]
+                      InputAdapter
+                        Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2
+                          WholeStageCodegen (6)
+                            HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ws_ext_sales_price] [sum,sum]
+                              Project [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                  Project [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                    SortMergeJoin [ws_item_sk,i_item_sk]
+                                      InputAdapter
+                                        WholeStageCodegen (2)
+                                          Sort [ws_item_sk]
+                                            InputAdapter
+                                              Exchange [ws_item_sk] #3
+                                                WholeStageCodegen (1)
+                                                  Filter [ws_item_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk]
+                                                          SubqueryBroadcast [d_date_sk] #1
+                                                            ReusedExchange [d_date_sk] #4
+                                      InputAdapter
+                                        WholeStageCodegen (4)
+                                          Sort [i_item_sk]
+                                            InputAdapter
+                                              Exchange [i_item_sk] #5
+                                                WholeStageCodegen (3)
+                                                  Filter [i_category,i_item_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                  InputAdapter
+                                    BroadcastExchange #4
+                                      WholeStageCodegen (5)
+                                        Project [d_date_sk]
+                                          Filter [d_date,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_date]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/explain.txt
new file mode 100644
index 0000000000000..0c22f6333e5a9
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/explain.txt
@@ -0,0 +1,148 @@
+== Physical Plan ==
+TakeOrderedAndProject (24)
++- * Project (23)
+   +- Window (22)
+      +- * Sort (21)
+         +- Exchange (20)
+            +- * HashAggregate (19)
+               +- Exchange (18)
+                  +- * HashAggregate (17)
+                     +- * Project (16)
+                        +- * BroadcastHashJoin Inner BuildRight (15)
+                           :- * Project (9)
+                           :  +- * BroadcastHashJoin Inner BuildRight (8)
+                           :     :- * Filter (3)
+                           :     :  +- * ColumnarToRow (2)
+                           :     :     +- Scan parquet default.web_sales (1)
+                           :     +- BroadcastExchange (7)
+                           :        +- * Filter (6)
+                           :           +- * ColumnarToRow (5)
+                           :              +- Scan parquet default.item (4)
+                           +- BroadcastExchange (14)
+                              +- * Project (13)
+                                 +- * Filter (12)
+                                    +- * ColumnarToRow (11)
+                                       +- Scan parquet default.date_dim (10)
+
+
+(1) Scan parquet default.web_sales
+Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3]
+Condition : isnotnull(ws_item_sk#1)
+
+(4) Scan parquet default.item
+Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [In(i_category, [Sports                                            ,Books                                             ,Home                                              ]), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_item_desc:string,i_current_price:decimal(7,2),i_class:string,i_category:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+
+(6) Filter [codegen id : 1]
+Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Condition : (i_category#10 IN (Sports                                            ,Books                                             ,Home                                              ) AND isnotnull(i_item_sk#5))
+
+(7) BroadcastExchange
+Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(8) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ws_item_sk#1]
+Right keys [1]: [i_item_sk#5]
+Join condition: None
+
+(9) Project [codegen id : 3]
+Output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+
+(10) Scan parquet default.date_dim
+Output [2]: [d_date_sk#12, d_date#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#12, d_date#13]
+
+(12) Filter [codegen id : 2]
+Input [2]: [d_date_sk#12, d_date#13]
+Condition : (((isnotnull(d_date#13) AND (d_date#13 >= 1999-02-22)) AND (d_date#13 <= 1999-03-24)) AND isnotnull(d_date_sk#12))
+
+(13) Project [codegen id : 2]
+Output [1]: [d_date_sk#12]
+Input [2]: [d_date_sk#12, d_date#13]
+
+(14) BroadcastExchange
+Input [1]: [d_date_sk#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ws_sold_date_sk#3]
+Right keys [1]: [d_date_sk#12]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [6]: [ws_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10, d_date_sk#12]
+
+(17) HashAggregate [codegen id : 3]
+Input [6]: [ws_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8]
+Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum#15]
+Results [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#16]
+
+(18) Exchange
+Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#16]
+Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(19) HashAggregate [codegen id : 4]
+Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#16]
+Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8]
+Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#2))#18]
+Results [8]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#18,17,2) AS itemrevenue#19, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#18,17,2) AS _w0#20, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#18,17,2) AS _w1#21, i_item_id#6]
+
+(20) Exchange
+Input [8]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21, i_item_id#6]
+Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(21) Sort [codegen id : 5]
+Input [8]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21, i_item_id#6]
+Arguments: [i_class#9 ASC NULLS FIRST], false, 0
+
+(22) Window
+Input [8]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21, i_item_id#6]
+Arguments: [sum(_w1#21) windowspecdefinition(i_class#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#23], [i_class#9]
+
+(23) Project [codegen id : 6]
+Output [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(_w0#20) * 100.00), DecimalType(21,2), true) as decimal(27,2))) / promote_precision(_we0#23)), DecimalType(38,17), true) AS revenueratio#24, i_item_id#6]
+Input [9]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21, i_item_id#6, _we0#23]
+
+(24) TakeOrderedAndProject
+Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, revenueratio#24, i_item_id#6]
+Arguments: 100, [i_category#10 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_item_id#6 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#24 ASC NULLS FIRST], [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, revenueratio#24]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (25)
+
+
+(25) ReusedExchange [Reuses operator id: 14]
+Output [1]: [d_date_sk#12]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/simplified.txt
new file mode 100644
index 0000000000000..0f193b514eff2
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/simplified.txt
@@ -0,0 +1,40 @@
+TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue]
+  WholeStageCodegen (6)
+    Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id]
+      InputAdapter
+        Window [_w1,i_class]
+          WholeStageCodegen (5)
+            Sort [i_class]
+              InputAdapter
+                Exchange [i_class] #1
+                  WholeStageCodegen (4)
+                    HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ws_ext_sales_price)),itemrevenue,_w0,_w1,sum]
+                      InputAdapter
+                        Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2
+                          WholeStageCodegen (3)
+                            HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ws_ext_sales_price] [sum,sum]
+                              Project [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                  Project [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                    BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                      Filter [ws_item_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #1
+                                                ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        BroadcastExchange #4
+                                          WholeStageCodegen (1)
+                                            Filter [i_category,i_item_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                  InputAdapter
+                                    BroadcastExchange #3
+                                      WholeStageCodegen (2)
+                                        Project [d_date_sk]
+                                          Filter [d_date,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_date]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.sf100/explain.txt
new file mode 100644
index 0000000000000..7d562331218b9
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.sf100/explain.txt
@@ -0,0 +1,227 @@
+== Physical Plan ==
+* HashAggregate (38)
++- Exchange (37)
+   +- * HashAggregate (36)
+      +- * Project (35)
+         +- * BroadcastHashJoin Inner BuildRight (34)
+            :- * Project (28)
+            :  +- * BroadcastHashJoin Inner BuildRight (27)
+            :     :- * Project (22)
+            :     :  +- * BroadcastHashJoin Inner BuildRight (21)
+            :     :     :- * Project (15)
+            :     :     :  +- * BroadcastHashJoin Inner BuildRight (14)
+            :     :     :     :- * Project (9)
+            :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+            :     :     :     :     :- * Filter (3)
+            :     :     :     :     :  +- * ColumnarToRow (2)
+            :     :     :     :     :     +- Scan parquet default.store_sales (1)
+            :     :     :     :     +- BroadcastExchange (7)
+            :     :     :     :        +- * Filter (6)
+            :     :     :     :           +- * ColumnarToRow (5)
+            :     :     :     :              +- Scan parquet default.customer_demographics (4)
+            :     :     :     +- BroadcastExchange (13)
+            :     :     :        +- * Filter (12)
+            :     :     :           +- * ColumnarToRow (11)
+            :     :     :              +- Scan parquet default.household_demographics (10)
+            :     :     +- BroadcastExchange (20)
+            :     :        +- * Project (19)
+            :     :           +- * Filter (18)
+            :     :              +- * ColumnarToRow (17)
+            :     :                 +- Scan parquet default.date_dim (16)
+            :     +- BroadcastExchange (26)
+            :        +- * Filter (25)
+            :           +- * ColumnarToRow (24)
+            :              +- Scan parquet default.store (23)
+            +- BroadcastExchange (33)
+               +- * Project (32)
+                  +- * Filter (31)
+                     +- * ColumnarToRow (30)
+                        +- Scan parquet default.customer_address (29)
+
+
+(1) Scan parquet default.store_sales
+Output [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#10), dynamicpruningexpression(ss_sold_date_sk#10 IN dynamicpruning#11)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_hdemo_sk), Or(Or(And(GreaterThanOrEqual(ss_net_profit,100.00),LessThanOrEqual(ss_net_profit,200.00)),And(GreaterThanOrEqual(ss_net_profit,150.00),LessThanOrEqual(ss_net_profit,300.00))),And(GreaterThanOrEqual(ss_net_profit,50.00),LessThanOrEqual(ss_net_profit,250.00))), Or(Or(And(GreaterThanOrEqual(ss_sales_price,100.00),LessThanOrEqual(ss_sales_price,150.00)),And(GreaterThanOrEqual(ss_sales_price,50.00),LessThanOrEqual(ss_sales_price,100.00))),And(GreaterThanOrEqual(ss_sales_price,150.00),LessThanOrEqual(ss_sales_price,200.00)))]
+ReadSchema: struct<ss_cdemo_sk:int,ss_hdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_quantity:int,ss_sales_price:decimal(7,2),ss_ext_sales_price:decimal(7,2),ss_ext_wholesale_cost:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 6]
+Input [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10]
+
+(3) Filter [codegen id : 6]
+Input [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10]
+Condition : (((((isnotnull(ss_store_sk#4) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_hdemo_sk#2)) AND ((((ss_net_profit#9 >= 100.00) AND (ss_net_profit#9 <= 200.00)) OR ((ss_net_profit#9 >= 150.00) AND (ss_net_profit#9 <= 300.00))) OR ((ss_net_profit#9 >= 50.00) AND (ss_net_profit#9 <= 250.00)))) AND ((((ss_sales_price#6 >= 100.00) AND (ss_sales_price#6 <= 150.00)) OR ((ss_sales_price#6 >= 50.00) AND (ss_sales_price#6 <= 100.00))) OR ((ss_sales_price#6 >= 150.00) AND (ss_sales_price#6 <= 200.00))))
+
+(4) Scan parquet default.customer_demographics
+Output [3]: [cd_demo_sk#12, cd_marital_status#13, cd_education_status#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk), Or(Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,Advanced Degree     )),And(EqualTo(cd_marital_status,S),EqualTo(cd_education_status,College             ))),And(EqualTo(cd_marital_status,W),EqualTo(cd_education_status,2 yr Degree         )))]
+ReadSchema: struct<cd_demo_sk:int,cd_marital_status:string,cd_education_status:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [cd_demo_sk#12, cd_marital_status#13, cd_education_status#14]
+
+(6) Filter [codegen id : 1]
+Input [3]: [cd_demo_sk#12, cd_marital_status#13, cd_education_status#14]
+Condition : (isnotnull(cd_demo_sk#12) AND ((((cd_marital_status#13 = M) AND (cd_education_status#14 = Advanced Degree     )) OR ((cd_marital_status#13 = S) AND (cd_education_status#14 = College             ))) OR ((cd_marital_status#13 = W) AND (cd_education_status#14 = 2 yr Degree         ))))
+
+(7) BroadcastExchange
+Input [3]: [cd_demo_sk#12, cd_marital_status#13, cd_education_status#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#15]
+
+(8) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_cdemo_sk#1]
+Right keys [1]: [cd_demo_sk#12]
+Join condition: ((((((cd_marital_status#13 = M) AND (cd_education_status#14 = Advanced Degree     )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((cd_marital_status#13 = S) AND (cd_education_status#14 = College             )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((cd_marital_status#13 = W) AND (cd_education_status#14 = 2 yr Degree         )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)))
+
+(9) Project [codegen id : 6]
+Output [11]: [ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, cd_marital_status#13, cd_education_status#14]
+Input [13]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, cd_demo_sk#12, cd_marital_status#13, cd_education_status#14]
+
+(10) Scan parquet default.household_demographics
+Output [2]: [hd_demo_sk#16, hd_dep_count#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_demo_sk), Or(Or(EqualTo(hd_dep_count,3),EqualTo(hd_dep_count,1)),EqualTo(hd_dep_count,1))]
+ReadSchema: struct<hd_demo_sk:int,hd_dep_count:int>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [hd_demo_sk#16, hd_dep_count#17]
+
+(12) Filter [codegen id : 2]
+Input [2]: [hd_demo_sk#16, hd_dep_count#17]
+Condition : (isnotnull(hd_demo_sk#16) AND (((hd_dep_count#17 = 3) OR (hd_dep_count#17 = 1)) OR (hd_dep_count#17 = 1)))
+
+(13) BroadcastExchange
+Input [2]: [hd_demo_sk#16, hd_dep_count#17]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#18]
+
+(14) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#16]
+Join condition: (((((((cd_marital_status#13 = M) AND (cd_education_status#14 = Advanced Degree     )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) AND (hd_dep_count#17 = 3)) OR (((((cd_marital_status#13 = S) AND (cd_education_status#14 = College             )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00)) AND (hd_dep_count#17 = 1))) OR (((((cd_marital_status#13 = W) AND (cd_education_status#14 = 2 yr Degree         )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)) AND (hd_dep_count#17 = 1)))
+
+(15) Project [codegen id : 6]
+Output [7]: [ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10]
+Input [13]: [ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, cd_marital_status#13, cd_education_status#14, hd_demo_sk#16, hd_dep_count#17]
+
+(16) Scan parquet default.date_dim
+Output [2]: [d_date_sk#19, d_year#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(17) ColumnarToRow [codegen id : 3]
+Input [2]: [d_date_sk#19, d_year#20]
+
+(18) Filter [codegen id : 3]
+Input [2]: [d_date_sk#19, d_year#20]
+Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19))
+
+(19) Project [codegen id : 3]
+Output [1]: [d_date_sk#19]
+Input [2]: [d_date_sk#19, d_year#20]
+
+(20) BroadcastExchange
+Input [1]: [d_date_sk#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(21) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_sold_date_sk#10]
+Right keys [1]: [d_date_sk#19]
+Join condition: None
+
+(22) Project [codegen id : 6]
+Output [6]: [ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9]
+Input [8]: [ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, d_date_sk#19]
+
+(23) Scan parquet default.store
+Output [1]: [s_store_sk#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int>
+
+(24) ColumnarToRow [codegen id : 4]
+Input [1]: [s_store_sk#22]
+
+(25) Filter [codegen id : 4]
+Input [1]: [s_store_sk#22]
+Condition : isnotnull(s_store_sk#22)
+
+(26) BroadcastExchange
+Input [1]: [s_store_sk#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#23]
+
+(27) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_store_sk#4]
+Right keys [1]: [s_store_sk#22]
+Join condition: None
+
+(28) Project [codegen id : 6]
+Output [5]: [ss_addr_sk#3, ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9]
+Input [7]: [ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, s_store_sk#22]
+
+(29) Scan parquet default.customer_address
+Output [3]: [ca_address_sk#24, ca_state#25, ca_country#26]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk), Or(Or(In(ca_state, [TX,OH]),In(ca_state, [OR,NM,KY])),In(ca_state, [VA,TX,MS]))]
+ReadSchema: struct<ca_address_sk:int,ca_state:string,ca_country:string>
+
+(30) ColumnarToRow [codegen id : 5]
+Input [3]: [ca_address_sk#24, ca_state#25, ca_country#26]
+
+(31) Filter [codegen id : 5]
+Input [3]: [ca_address_sk#24, ca_state#25, ca_country#26]
+Condition : (((isnotnull(ca_country#26) AND (ca_country#26 = United States)) AND isnotnull(ca_address_sk#24)) AND ((ca_state#25 IN (TX,OH) OR ca_state#25 IN (OR,NM,KY)) OR ca_state#25 IN (VA,TX,MS)))
+
+(32) Project [codegen id : 5]
+Output [2]: [ca_address_sk#24, ca_state#25]
+Input [3]: [ca_address_sk#24, ca_state#25, ca_country#26]
+
+(33) BroadcastExchange
+Input [2]: [ca_address_sk#24, ca_state#25]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#27]
+
+(34) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_addr_sk#3]
+Right keys [1]: [ca_address_sk#24]
+Join condition: ((((ca_state#25 IN (TX,OH) AND (ss_net_profit#9 >= 100.00)) AND (ss_net_profit#9 <= 200.00)) OR ((ca_state#25 IN (OR,NM,KY) AND (ss_net_profit#9 >= 150.00)) AND (ss_net_profit#9 <= 300.00))) OR ((ca_state#25 IN (VA,TX,MS) AND (ss_net_profit#9 >= 50.00)) AND (ss_net_profit#9 <= 250.00)))
+
+(35) Project [codegen id : 6]
+Output [3]: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8]
+Input [7]: [ss_addr_sk#3, ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ca_address_sk#24, ca_state#25]
+
+(36) HashAggregate [codegen id : 6]
+Input [3]: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8]
+Keys: []
+Functions [4]: [partial_avg(ss_quantity#5), partial_avg(UnscaledValue(ss_ext_sales_price#7)), partial_avg(UnscaledValue(ss_ext_wholesale_cost#8)), partial_sum(UnscaledValue(ss_ext_wholesale_cost#8))]
+Aggregate Attributes [7]: [sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34]
+Results [7]: [sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41]
+
+(37) Exchange
+Input [7]: [sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#42]
+
+(38) HashAggregate [codegen id : 7]
+Input [7]: [sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41]
+Keys: []
+Functions [4]: [avg(ss_quantity#5), avg(UnscaledValue(ss_ext_sales_price#7)), avg(UnscaledValue(ss_ext_wholesale_cost#8)), sum(UnscaledValue(ss_ext_wholesale_cost#8))]
+Aggregate Attributes [4]: [avg(ss_quantity#5)#43, avg(UnscaledValue(ss_ext_sales_price#7))#44, avg(UnscaledValue(ss_ext_wholesale_cost#8))#45, sum(UnscaledValue(ss_ext_wholesale_cost#8))#46]
+Results [4]: [avg(ss_quantity#5)#43 AS avg(ss_quantity)#47, cast((avg(UnscaledValue(ss_ext_sales_price#7))#44 / 100.0) as decimal(11,6)) AS avg(ss_ext_sales_price)#48, cast((avg(UnscaledValue(ss_ext_wholesale_cost#8))#45 / 100.0) as decimal(11,6)) AS avg(ss_ext_wholesale_cost)#49, MakeDecimal(sum(UnscaledValue(ss_ext_wholesale_cost#8))#46,17,2) AS sum(ss_ext_wholesale_cost)#50]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#10 IN dynamicpruning#11
+ReusedExchange (39)
+
+
+(39) ReusedExchange [Reuses operator id: 20]
+Output [1]: [d_date_sk#19]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.sf100/simplified.txt
new file mode 100644
index 0000000000000..26be2b52e400a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.sf100/simplified.txt
@@ -0,0 +1,59 @@
+WholeStageCodegen (7)
+  HashAggregate [sum,count,sum,count,sum,count,sum] [avg(ss_quantity),avg(UnscaledValue(ss_ext_sales_price)),avg(UnscaledValue(ss_ext_wholesale_cost)),sum(UnscaledValue(ss_ext_wholesale_cost)),avg(ss_quantity),avg(ss_ext_sales_price),avg(ss_ext_wholesale_cost),sum(ss_ext_wholesale_cost),sum,count,sum,count,sum,count,sum]
+    InputAdapter
+      Exchange #1
+        WholeStageCodegen (6)
+          HashAggregate [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] [sum,count,sum,count,sum,count,sum,sum,count,sum,count,sum,count,sum]
+            Project [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost]
+              BroadcastHashJoin [ss_addr_sk,ca_address_sk,ca_state,ss_net_profit]
+                Project [ss_addr_sk,ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit]
+                  BroadcastHashJoin [ss_store_sk,s_store_sk]
+                    Project [ss_addr_sk,ss_store_sk,ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit]
+                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                        Project [ss_addr_sk,ss_store_sk,ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk]
+                          BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk,cd_marital_status,cd_education_status,ss_sales_price,hd_dep_count]
+                            Project [ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,cd_marital_status,cd_education_status]
+                              BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status,ss_sales_price]
+                                Filter [ss_store_sk,ss_addr_sk,ss_cdemo_sk,ss_hdemo_sk,ss_net_profit,ss_sales_price]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_sales [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk]
+                                        SubqueryBroadcast [d_date_sk] #1
+                                          ReusedExchange [d_date_sk] #2
+                                InputAdapter
+                                  BroadcastExchange #3
+                                    WholeStageCodegen (1)
+                                      Filter [cd_demo_sk,cd_marital_status,cd_education_status]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status]
+                            InputAdapter
+                              BroadcastExchange #4
+                                WholeStageCodegen (2)
+                                  Filter [hd_demo_sk,hd_dep_count]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.household_demographics [hd_demo_sk,hd_dep_count]
+                        InputAdapter
+                          BroadcastExchange #2
+                            WholeStageCodegen (3)
+                              Project [d_date_sk]
+                                Filter [d_year,d_date_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.date_dim [d_date_sk,d_year]
+                    InputAdapter
+                      BroadcastExchange #5
+                        WholeStageCodegen (4)
+                          Filter [s_store_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.store [s_store_sk]
+                InputAdapter
+                  BroadcastExchange #6
+                    WholeStageCodegen (5)
+                      Project [ca_address_sk,ca_state]
+                        Filter [ca_country,ca_address_sk,ca_state]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.customer_address [ca_address_sk,ca_state,ca_country]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/explain.txt
new file mode 100644
index 0000000000000..027fde38ff7fa
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/explain.txt
@@ -0,0 +1,227 @@
+== Physical Plan ==
+* HashAggregate (38)
++- Exchange (37)
+   +- * HashAggregate (36)
+      +- * Project (35)
+         +- * BroadcastHashJoin Inner BuildRight (34)
+            :- * Project (29)
+            :  +- * BroadcastHashJoin Inner BuildRight (28)
+            :     :- * Project (23)
+            :     :  +- * BroadcastHashJoin Inner BuildRight (22)
+            :     :     :- * Project (16)
+            :     :     :  +- * BroadcastHashJoin Inner BuildRight (15)
+            :     :     :     :- * Project (9)
+            :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+            :     :     :     :     :- * Filter (3)
+            :     :     :     :     :  +- * ColumnarToRow (2)
+            :     :     :     :     :     +- Scan parquet default.store_sales (1)
+            :     :     :     :     +- BroadcastExchange (7)
+            :     :     :     :        +- * Filter (6)
+            :     :     :     :           +- * ColumnarToRow (5)
+            :     :     :     :              +- Scan parquet default.store (4)
+            :     :     :     +- BroadcastExchange (14)
+            :     :     :        +- * Project (13)
+            :     :     :           +- * Filter (12)
+            :     :     :              +- * ColumnarToRow (11)
+            :     :     :                 +- Scan parquet default.customer_address (10)
+            :     :     +- BroadcastExchange (21)
+            :     :        +- * Project (20)
+            :     :           +- * Filter (19)
+            :     :              +- * ColumnarToRow (18)
+            :     :                 +- Scan parquet default.date_dim (17)
+            :     +- BroadcastExchange (27)
+            :        +- * Filter (26)
+            :           +- * ColumnarToRow (25)
+            :              +- Scan parquet default.customer_demographics (24)
+            +- BroadcastExchange (33)
+               +- * Filter (32)
+                  +- * ColumnarToRow (31)
+                     +- Scan parquet default.household_demographics (30)
+
+
+(1) Scan parquet default.store_sales
+Output [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#10), dynamicpruningexpression(ss_sold_date_sk#10 IN dynamicpruning#11)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_hdemo_sk), Or(Or(And(GreaterThanOrEqual(ss_net_profit,100.00),LessThanOrEqual(ss_net_profit,200.00)),And(GreaterThanOrEqual(ss_net_profit,150.00),LessThanOrEqual(ss_net_profit,300.00))),And(GreaterThanOrEqual(ss_net_profit,50.00),LessThanOrEqual(ss_net_profit,250.00))), Or(Or(And(GreaterThanOrEqual(ss_sales_price,100.00),LessThanOrEqual(ss_sales_price,150.00)),And(GreaterThanOrEqual(ss_sales_price,50.00),LessThanOrEqual(ss_sales_price,100.00))),And(GreaterThanOrEqual(ss_sales_price,150.00),LessThanOrEqual(ss_sales_price,200.00)))]
+ReadSchema: struct<ss_cdemo_sk:int,ss_hdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_quantity:int,ss_sales_price:decimal(7,2),ss_ext_sales_price:decimal(7,2),ss_ext_wholesale_cost:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 6]
+Input [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10]
+
+(3) Filter [codegen id : 6]
+Input [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10]
+Condition : (((((isnotnull(ss_store_sk#4) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_hdemo_sk#2)) AND ((((ss_net_profit#9 >= 100.00) AND (ss_net_profit#9 <= 200.00)) OR ((ss_net_profit#9 >= 150.00) AND (ss_net_profit#9 <= 300.00))) OR ((ss_net_profit#9 >= 50.00) AND (ss_net_profit#9 <= 250.00)))) AND ((((ss_sales_price#6 >= 100.00) AND (ss_sales_price#6 <= 150.00)) OR ((ss_sales_price#6 >= 50.00) AND (ss_sales_price#6 <= 100.00))) OR ((ss_sales_price#6 >= 150.00) AND (ss_sales_price#6 <= 200.00))))
+
+(4) Scan parquet default.store
+Output [1]: [s_store_sk#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [1]: [s_store_sk#12]
+
+(6) Filter [codegen id : 1]
+Input [1]: [s_store_sk#12]
+Condition : isnotnull(s_store_sk#12)
+
+(7) BroadcastExchange
+Input [1]: [s_store_sk#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#13]
+
+(8) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_store_sk#4]
+Right keys [1]: [s_store_sk#12]
+Join condition: None
+
+(9) Project [codegen id : 6]
+Output [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10]
+Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, s_store_sk#12]
+
+(10) Scan parquet default.customer_address
+Output [3]: [ca_address_sk#14, ca_state#15, ca_country#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk), Or(Or(In(ca_state, [TX,OH]),In(ca_state, [OR,NM,KY])),In(ca_state, [VA,TX,MS]))]
+ReadSchema: struct<ca_address_sk:int,ca_state:string,ca_country:string>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [3]: [ca_address_sk#14, ca_state#15, ca_country#16]
+
+(12) Filter [codegen id : 2]
+Input [3]: [ca_address_sk#14, ca_state#15, ca_country#16]
+Condition : (((isnotnull(ca_country#16) AND (ca_country#16 = United States)) AND isnotnull(ca_address_sk#14)) AND ((ca_state#15 IN (TX,OH) OR ca_state#15 IN (OR,NM,KY)) OR ca_state#15 IN (VA,TX,MS)))
+
+(13) Project [codegen id : 2]
+Output [2]: [ca_address_sk#14, ca_state#15]
+Input [3]: [ca_address_sk#14, ca_state#15, ca_country#16]
+
+(14) BroadcastExchange
+Input [2]: [ca_address_sk#14, ca_state#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(15) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_addr_sk#3]
+Right keys [1]: [ca_address_sk#14]
+Join condition: ((((ca_state#15 IN (TX,OH) AND (ss_net_profit#9 >= 100.00)) AND (ss_net_profit#9 <= 200.00)) OR ((ca_state#15 IN (OR,NM,KY) AND (ss_net_profit#9 >= 150.00)) AND (ss_net_profit#9 <= 300.00))) OR ((ca_state#15 IN (VA,TX,MS) AND (ss_net_profit#9 >= 50.00)) AND (ss_net_profit#9 <= 250.00)))
+
+(16) Project [codegen id : 6]
+Output [7]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10]
+Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#14, ca_state#15]
+
+(17) Scan parquet default.date_dim
+Output [2]: [d_date_sk#18, d_year#19]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [2]: [d_date_sk#18, d_year#19]
+
+(19) Filter [codegen id : 3]
+Input [2]: [d_date_sk#18, d_year#19]
+Condition : ((isnotnull(d_year#19) AND (d_year#19 = 2001)) AND isnotnull(d_date_sk#18))
+
+(20) Project [codegen id : 3]
+Output [1]: [d_date_sk#18]
+Input [2]: [d_date_sk#18, d_year#19]
+
+(21) BroadcastExchange
+Input [1]: [d_date_sk#18]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#20]
+
+(22) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_sold_date_sk#10]
+Right keys [1]: [d_date_sk#18]
+Join condition: None
+
+(23) Project [codegen id : 6]
+Output [6]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8]
+Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#18]
+
+(24) Scan parquet default.customer_demographics
+Output [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk), Or(Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,Advanced Degree     )),And(EqualTo(cd_marital_status,S),EqualTo(cd_education_status,College             ))),And(EqualTo(cd_marital_status,W),EqualTo(cd_education_status,2 yr Degree         )))]
+ReadSchema: struct<cd_demo_sk:int,cd_marital_status:string,cd_education_status:string>
+
+(25) ColumnarToRow [codegen id : 4]
+Input [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23]
+
+(26) Filter [codegen id : 4]
+Input [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23]
+Condition : (isnotnull(cd_demo_sk#21) AND ((((cd_marital_status#22 = M) AND (cd_education_status#23 = Advanced Degree     )) OR ((cd_marital_status#22 = S) AND (cd_education_status#23 = College             ))) OR ((cd_marital_status#22 = W) AND (cd_education_status#23 = 2 yr Degree         ))))
+
+(27) BroadcastExchange
+Input [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#24]
+
+(28) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_cdemo_sk#1]
+Right keys [1]: [cd_demo_sk#21]
+Join condition: ((((((cd_marital_status#22 = M) AND (cd_education_status#23 = Advanced Degree     )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((cd_marital_status#22 = S) AND (cd_education_status#23 = College             )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((cd_marital_status#22 = W) AND (cd_education_status#23 = 2 yr Degree         )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)))
+
+(29) Project [codegen id : 6]
+Output [7]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#22, cd_education_status#23]
+Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#21, cd_marital_status#22, cd_education_status#23]
+
+(30) Scan parquet default.household_demographics
+Output [2]: [hd_demo_sk#25, hd_dep_count#26]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_demo_sk), Or(Or(EqualTo(hd_dep_count,3),EqualTo(hd_dep_count,1)),EqualTo(hd_dep_count,1))]
+ReadSchema: struct<hd_demo_sk:int,hd_dep_count:int>
+
+(31) ColumnarToRow [codegen id : 5]
+Input [2]: [hd_demo_sk#25, hd_dep_count#26]
+
+(32) Filter [codegen id : 5]
+Input [2]: [hd_demo_sk#25, hd_dep_count#26]
+Condition : (isnotnull(hd_demo_sk#25) AND (((hd_dep_count#26 = 3) OR (hd_dep_count#26 = 1)) OR (hd_dep_count#26 = 1)))
+
+(33) BroadcastExchange
+Input [2]: [hd_demo_sk#25, hd_dep_count#26]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#27]
+
+(34) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#25]
+Join condition: (((((((cd_marital_status#22 = M) AND (cd_education_status#23 = Advanced Degree     )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) AND (hd_dep_count#26 = 3)) OR (((((cd_marital_status#22 = S) AND (cd_education_status#23 = College             )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00)) AND (hd_dep_count#26 = 1))) OR (((((cd_marital_status#22 = W) AND (cd_education_status#23 = 2 yr Degree         )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)) AND (hd_dep_count#26 = 1)))
+
+(35) Project [codegen id : 6]
+Output [3]: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8]
+Input [9]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#22, cd_education_status#23, hd_demo_sk#25, hd_dep_count#26]
+
+(36) HashAggregate [codegen id : 6]
+Input [3]: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8]
+Keys: []
+Functions [4]: [partial_avg(ss_quantity#5), partial_avg(UnscaledValue(ss_ext_sales_price#7)), partial_avg(UnscaledValue(ss_ext_wholesale_cost#8)), partial_sum(UnscaledValue(ss_ext_wholesale_cost#8))]
+Aggregate Attributes [7]: [sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34]
+Results [7]: [sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41]
+
+(37) Exchange
+Input [7]: [sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#42]
+
+(38) HashAggregate [codegen id : 7]
+Input [7]: [sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41]
+Keys: []
+Functions [4]: [avg(ss_quantity#5), avg(UnscaledValue(ss_ext_sales_price#7)), avg(UnscaledValue(ss_ext_wholesale_cost#8)), sum(UnscaledValue(ss_ext_wholesale_cost#8))]
+Aggregate Attributes [4]: [avg(ss_quantity#5)#43, avg(UnscaledValue(ss_ext_sales_price#7))#44, avg(UnscaledValue(ss_ext_wholesale_cost#8))#45, sum(UnscaledValue(ss_ext_wholesale_cost#8))#46]
+Results [4]: [avg(ss_quantity#5)#43 AS avg(ss_quantity)#47, cast((avg(UnscaledValue(ss_ext_sales_price#7))#44 / 100.0) as decimal(11,6)) AS avg(ss_ext_sales_price)#48, cast((avg(UnscaledValue(ss_ext_wholesale_cost#8))#45 / 100.0) as decimal(11,6)) AS avg(ss_ext_wholesale_cost)#49, MakeDecimal(sum(UnscaledValue(ss_ext_wholesale_cost#8))#46,17,2) AS sum(ss_ext_wholesale_cost)#50]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#10 IN dynamicpruning#11
+ReusedExchange (39)
+
+
+(39) ReusedExchange [Reuses operator id: 21]
+Output [1]: [d_date_sk#18]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/simplified.txt
new file mode 100644
index 0000000000000..679cd0add2b33
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/simplified.txt
@@ -0,0 +1,59 @@
+WholeStageCodegen (7)
+  HashAggregate [sum,count,sum,count,sum,count,sum] [avg(ss_quantity),avg(UnscaledValue(ss_ext_sales_price)),avg(UnscaledValue(ss_ext_wholesale_cost)),sum(UnscaledValue(ss_ext_wholesale_cost)),avg(ss_quantity),avg(ss_ext_sales_price),avg(ss_ext_wholesale_cost),sum(ss_ext_wholesale_cost),sum,count,sum,count,sum,count,sum]
+    InputAdapter
+      Exchange #1
+        WholeStageCodegen (6)
+          HashAggregate [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] [sum,count,sum,count,sum,count,sum,sum,count,sum,count,sum,count,sum]
+            Project [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost]
+              BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk,cd_marital_status,cd_education_status,ss_sales_price,hd_dep_count]
+                Project [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status]
+                  BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status,ss_sales_price]
+                    Project [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost]
+                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                        Project [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_sold_date_sk]
+                          BroadcastHashJoin [ss_addr_sk,ca_address_sk,ca_state,ss_net_profit]
+                            Project [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk]
+                              BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                Filter [ss_store_sk,ss_addr_sk,ss_cdemo_sk,ss_hdemo_sk,ss_net_profit,ss_sales_price]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_sales [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk]
+                                        SubqueryBroadcast [d_date_sk] #1
+                                          ReusedExchange [d_date_sk] #2
+                                InputAdapter
+                                  BroadcastExchange #3
+                                    WholeStageCodegen (1)
+                                      Filter [s_store_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store [s_store_sk]
+                            InputAdapter
+                              BroadcastExchange #4
+                                WholeStageCodegen (2)
+                                  Project [ca_address_sk,ca_state]
+                                    Filter [ca_country,ca_address_sk,ca_state]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.customer_address [ca_address_sk,ca_state,ca_country]
+                        InputAdapter
+                          BroadcastExchange #2
+                            WholeStageCodegen (3)
+                              Project [d_date_sk]
+                                Filter [d_year,d_date_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.date_dim [d_date_sk,d_year]
+                    InputAdapter
+                      BroadcastExchange #5
+                        WholeStageCodegen (4)
+                          Filter [cd_demo_sk,cd_marital_status,cd_education_status]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status]
+                InputAdapter
+                  BroadcastExchange #6
+                    WholeStageCodegen (5)
+                      Filter [hd_demo_sk,hd_dep_count]
+                        ColumnarToRow
+                          InputAdapter
+                            Scan parquet default.household_demographics [hd_demo_sk,hd_dep_count]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.sf100/explain.txt
new file mode 100644
index 0000000000000..8828920519371
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.sf100/explain.txt
@@ -0,0 +1,917 @@
+== Physical Plan ==
+TakeOrderedAndProject (134)
++- * HashAggregate (133)
+   +- Exchange (132)
+      +- * HashAggregate (131)
+         +- * Expand (130)
+            +- Union (129)
+               :- * Project (90)
+               :  +- * Filter (89)
+               :     +- * HashAggregate (88)
+               :        +- Exchange (87)
+               :           +- * HashAggregate (86)
+               :              +- * Project (85)
+               :                 +- * BroadcastHashJoin Inner BuildRight (84)
+               :                    :- * Project (74)
+               :                    :  +- * BroadcastHashJoin Inner BuildRight (73)
+               :                    :     :- * SortMergeJoin LeftSemi (67)
+               :                    :     :  :- * Sort (5)
+               :                    :     :  :  +- Exchange (4)
+               :                    :     :  :     +- * Filter (3)
+               :                    :     :  :        +- * ColumnarToRow (2)
+               :                    :     :  :           +- Scan parquet default.store_sales (1)
+               :                    :     :  +- * Sort (66)
+               :                    :     :     +- Exchange (65)
+               :                    :     :        +- * Project (64)
+               :                    :     :           +- * BroadcastHashJoin Inner BuildRight (63)
+               :                    :     :              :- * Filter (8)
+               :                    :     :              :  +- * ColumnarToRow (7)
+               :                    :     :              :     +- Scan parquet default.item (6)
+               :                    :     :              +- BroadcastExchange (62)
+               :                    :     :                 +- * HashAggregate (61)
+               :                    :     :                    +- Exchange (60)
+               :                    :     :                       +- * HashAggregate (59)
+               :                    :     :                          +- * SortMergeJoin LeftSemi (58)
+               :                    :     :                             :- * Sort (46)
+               :                    :     :                             :  +- Exchange (45)
+               :                    :     :                             :     +- * HashAggregate (44)
+               :                    :     :                             :        +- Exchange (43)
+               :                    :     :                             :           +- * HashAggregate (42)
+               :                    :     :                             :              +- * Project (41)
+               :                    :     :                             :                 +- * BroadcastHashJoin Inner BuildRight (40)
+               :                    :     :                             :                    :- * Project (18)
+               :                    :     :                             :                    :  +- * BroadcastHashJoin Inner BuildRight (17)
+               :                    :     :                             :                    :     :- * Filter (11)
+               :                    :     :                             :                    :     :  +- * ColumnarToRow (10)
+               :                    :     :                             :                    :     :     +- Scan parquet default.store_sales (9)
+               :                    :     :                             :                    :     +- BroadcastExchange (16)
+               :                    :     :                             :                    :        +- * Project (15)
+               :                    :     :                             :                    :           +- * Filter (14)
+               :                    :     :                             :                    :              +- * ColumnarToRow (13)
+               :                    :     :                             :                    :                 +- Scan parquet default.date_dim (12)
+               :                    :     :                             :                    +- BroadcastExchange (39)
+               :                    :     :                             :                       +- * SortMergeJoin LeftSemi (38)
+               :                    :     :                             :                          :- * Sort (23)
+               :                    :     :                             :                          :  +- Exchange (22)
+               :                    :     :                             :                          :     +- * Filter (21)
+               :                    :     :                             :                          :        +- * ColumnarToRow (20)
+               :                    :     :                             :                          :           +- Scan parquet default.item (19)
+               :                    :     :                             :                          +- * Sort (37)
+               :                    :     :                             :                             +- Exchange (36)
+               :                    :     :                             :                                +- * Project (35)
+               :                    :     :                             :                                   +- * BroadcastHashJoin Inner BuildRight (34)
+               :                    :     :                             :                                      :- * Project (29)
+               :                    :     :                             :                                      :  +- * BroadcastHashJoin Inner BuildRight (28)
+               :                    :     :                             :                                      :     :- * Filter (26)
+               :                    :     :                             :                                      :     :  +- * ColumnarToRow (25)
+               :                    :     :                             :                                      :     :     +- Scan parquet default.catalog_sales (24)
+               :                    :     :                             :                                      :     +- ReusedExchange (27)
+               :                    :     :                             :                                      +- BroadcastExchange (33)
+               :                    :     :                             :                                         +- * Filter (32)
+               :                    :     :                             :                                            +- * ColumnarToRow (31)
+               :                    :     :                             :                                               +- Scan parquet default.item (30)
+               :                    :     :                             +- * Sort (57)
+               :                    :     :                                +- Exchange (56)
+               :                    :     :                                   +- * Project (55)
+               :                    :     :                                      +- * BroadcastHashJoin Inner BuildRight (54)
+               :                    :     :                                         :- * Project (52)
+               :                    :     :                                         :  +- * BroadcastHashJoin Inner BuildRight (51)
+               :                    :     :                                         :     :- * Filter (49)
+               :                    :     :                                         :     :  +- * ColumnarToRow (48)
+               :                    :     :                                         :     :     +- Scan parquet default.web_sales (47)
+               :                    :     :                                         :     +- ReusedExchange (50)
+               :                    :     :                                         +- ReusedExchange (53)
+               :                    :     +- BroadcastExchange (72)
+               :                    :        +- * Project (71)
+               :                    :           +- * Filter (70)
+               :                    :              +- * ColumnarToRow (69)
+               :                    :                 +- Scan parquet default.date_dim (68)
+               :                    +- BroadcastExchange (83)
+               :                       +- * SortMergeJoin LeftSemi (82)
+               :                          :- * Sort (79)
+               :                          :  +- Exchange (78)
+               :                          :     +- * Filter (77)
+               :                          :        +- * ColumnarToRow (76)
+               :                          :           +- Scan parquet default.item (75)
+               :                          +- * Sort (81)
+               :                             +- ReusedExchange (80)
+               :- * Project (109)
+               :  +- * Filter (108)
+               :     +- * HashAggregate (107)
+               :        +- Exchange (106)
+               :           +- * HashAggregate (105)
+               :              +- * Project (104)
+               :                 +- * BroadcastHashJoin Inner BuildRight (103)
+               :                    :- * Project (101)
+               :                    :  +- * BroadcastHashJoin Inner BuildRight (100)
+               :                    :     :- * SortMergeJoin LeftSemi (98)
+               :                    :     :  :- * Sort (95)
+               :                    :     :  :  +- Exchange (94)
+               :                    :     :  :     +- * Filter (93)
+               :                    :     :  :        +- * ColumnarToRow (92)
+               :                    :     :  :           +- Scan parquet default.catalog_sales (91)
+               :                    :     :  +- * Sort (97)
+               :                    :     :     +- ReusedExchange (96)
+               :                    :     +- ReusedExchange (99)
+               :                    +- ReusedExchange (102)
+               +- * Project (128)
+                  +- * Filter (127)
+                     +- * HashAggregate (126)
+                        +- Exchange (125)
+                           +- * HashAggregate (124)
+                              +- * Project (123)
+                                 +- * BroadcastHashJoin Inner BuildRight (122)
+                                    :- * Project (120)
+                                    :  +- * BroadcastHashJoin Inner BuildRight (119)
+                                    :     :- * SortMergeJoin LeftSemi (117)
+                                    :     :  :- * Sort (114)
+                                    :     :  :  +- Exchange (113)
+                                    :     :  :     +- * Filter (112)
+                                    :     :  :        +- * ColumnarToRow (111)
+                                    :     :  :           +- Scan parquet default.web_sales (110)
+                                    :     :  +- * Sort (116)
+                                    :     :     +- ReusedExchange (115)
+                                    :     +- ReusedExchange (118)
+                                    +- ReusedExchange (121)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_quantity:int,ss_list_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 1]
+Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+Condition : isnotnull(ss_item_sk#1)
+
+(4) Exchange
+Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#6]
+
+(5) Sort [codegen id : 2]
+Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+Arguments: [ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.item
+Output [4]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(7) ColumnarToRow [codegen id : 20]
+Input [4]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10]
+
+(8) Filter [codegen id : 20]
+Input [4]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10]
+Condition : ((isnotnull(i_brand_id#8) AND isnotnull(i_class_id#9)) AND isnotnull(i_category_id#10))
+
+(9) Scan parquet default.store_sales
+Output [2]: [ss_item_sk#11, ss_sold_date_sk#12]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_sold_date_sk#12 IN dynamicpruning#13)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int>
+
+(10) ColumnarToRow [codegen id : 11]
+Input [2]: [ss_item_sk#11, ss_sold_date_sk#12]
+
+(11) Filter [codegen id : 11]
+Input [2]: [ss_item_sk#11, ss_sold_date_sk#12]
+Condition : isnotnull(ss_item_sk#11)
+
+(12) Scan parquet default.date_dim
+Output [2]: [d_date_sk#14, d_year#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(13) ColumnarToRow [codegen id : 3]
+Input [2]: [d_date_sk#14, d_year#15]
+
+(14) Filter [codegen id : 3]
+Input [2]: [d_date_sk#14, d_year#15]
+Condition : (((isnotnull(d_year#15) AND (d_year#15 >= 1999)) AND (d_year#15 <= 2001)) AND isnotnull(d_date_sk#14))
+
+(15) Project [codegen id : 3]
+Output [1]: [d_date_sk#14]
+Input [2]: [d_date_sk#14, d_year#15]
+
+(16) BroadcastExchange
+Input [1]: [d_date_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#16]
+
+(17) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ss_sold_date_sk#12]
+Right keys [1]: [d_date_sk#14]
+Join condition: None
+
+(18) Project [codegen id : 11]
+Output [1]: [ss_item_sk#11]
+Input [3]: [ss_item_sk#11, ss_sold_date_sk#12, d_date_sk#14]
+
+(19) Scan parquet default.item
+Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(20) ColumnarToRow [codegen id : 4]
+Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+
+(21) Filter [codegen id : 4]
+Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+Condition : (((isnotnull(i_item_sk#17) AND isnotnull(i_brand_id#18)) AND isnotnull(i_class_id#19)) AND isnotnull(i_category_id#20))
+
+(22) Exchange
+Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+Arguments: hashpartitioning(coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20), 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(23) Sort [codegen id : 5]
+Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+Arguments: [coalesce(i_brand_id#18, 0) ASC NULLS FIRST, isnull(i_brand_id#18) ASC NULLS FIRST, coalesce(i_class_id#19, 0) ASC NULLS FIRST, isnull(i_class_id#19) ASC NULLS FIRST, coalesce(i_category_id#20, 0) ASC NULLS FIRST, isnull(i_category_id#20) ASC NULLS FIRST], false, 0
+
+(24) Scan parquet default.catalog_sales
+Output [2]: [cs_item_sk#22, cs_sold_date_sk#23]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#23), dynamicpruningexpression(cs_sold_date_sk#23 IN dynamicpruning#13)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int>
+
+(25) ColumnarToRow [codegen id : 8]
+Input [2]: [cs_item_sk#22, cs_sold_date_sk#23]
+
+(26) Filter [codegen id : 8]
+Input [2]: [cs_item_sk#22, cs_sold_date_sk#23]
+Condition : isnotnull(cs_item_sk#22)
+
+(27) ReusedExchange [Reuses operator id: 16]
+Output [1]: [d_date_sk#24]
+
+(28) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [cs_sold_date_sk#23]
+Right keys [1]: [d_date_sk#24]
+Join condition: None
+
+(29) Project [codegen id : 8]
+Output [1]: [cs_item_sk#22]
+Input [3]: [cs_item_sk#22, cs_sold_date_sk#23, d_date_sk#24]
+
+(30) Scan parquet default.item
+Output [4]: [i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(31) ColumnarToRow [codegen id : 7]
+Input [4]: [i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28]
+
+(32) Filter [codegen id : 7]
+Input [4]: [i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28]
+Condition : isnotnull(i_item_sk#25)
+
+(33) BroadcastExchange
+Input [4]: [i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#29]
+
+(34) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [cs_item_sk#22]
+Right keys [1]: [i_item_sk#25]
+Join condition: None
+
+(35) Project [codegen id : 8]
+Output [3]: [i_brand_id#26, i_class_id#27, i_category_id#28]
+Input [5]: [cs_item_sk#22, i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28]
+
+(36) Exchange
+Input [3]: [i_brand_id#26, i_class_id#27, i_category_id#28]
+Arguments: hashpartitioning(coalesce(i_brand_id#26, 0), isnull(i_brand_id#26), coalesce(i_class_id#27, 0), isnull(i_class_id#27), coalesce(i_category_id#28, 0), isnull(i_category_id#28), 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(37) Sort [codegen id : 9]
+Input [3]: [i_brand_id#26, i_class_id#27, i_category_id#28]
+Arguments: [coalesce(i_brand_id#26, 0) ASC NULLS FIRST, isnull(i_brand_id#26) ASC NULLS FIRST, coalesce(i_class_id#27, 0) ASC NULLS FIRST, isnull(i_class_id#27) ASC NULLS FIRST, coalesce(i_category_id#28, 0) ASC NULLS FIRST, isnull(i_category_id#28) ASC NULLS FIRST], false, 0
+
+(38) SortMergeJoin [codegen id : 10]
+Left keys [6]: [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)]
+Right keys [6]: [coalesce(i_brand_id#26, 0), isnull(i_brand_id#26), coalesce(i_class_id#27, 0), isnull(i_class_id#27), coalesce(i_category_id#28, 0), isnull(i_category_id#28)]
+Join condition: None
+
+(39) BroadcastExchange
+Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#31]
+
+(40) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ss_item_sk#11]
+Right keys [1]: [i_item_sk#17]
+Join condition: None
+
+(41) Project [codegen id : 11]
+Output [3]: [i_brand_id#18 AS brand_id#32, i_class_id#19 AS class_id#33, i_category_id#20 AS category_id#34]
+Input [5]: [ss_item_sk#11, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+
+(42) HashAggregate [codegen id : 11]
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Keys [3]: [brand_id#32, class_id#33, category_id#34]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#32, class_id#33, category_id#34]
+
+(43) Exchange
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Arguments: hashpartitioning(brand_id#32, class_id#33, category_id#34, 5), ENSURE_REQUIREMENTS, [id=#35]
+
+(44) HashAggregate [codegen id : 12]
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Keys [3]: [brand_id#32, class_id#33, category_id#34]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#32, class_id#33, category_id#34]
+
+(45) Exchange
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Arguments: hashpartitioning(coalesce(brand_id#32, 0), isnull(brand_id#32), coalesce(class_id#33, 0), isnull(class_id#33), coalesce(category_id#34, 0), isnull(category_id#34), 5), ENSURE_REQUIREMENTS, [id=#36]
+
+(46) Sort [codegen id : 13]
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Arguments: [coalesce(brand_id#32, 0) ASC NULLS FIRST, isnull(brand_id#32) ASC NULLS FIRST, coalesce(class_id#33, 0) ASC NULLS FIRST, isnull(class_id#33) ASC NULLS FIRST, coalesce(category_id#34, 0) ASC NULLS FIRST, isnull(category_id#34) ASC NULLS FIRST], false, 0
+
+(47) Scan parquet default.web_sales
+Output [2]: [ws_item_sk#37, ws_sold_date_sk#38]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#38), dynamicpruningexpression(ws_sold_date_sk#38 IN dynamicpruning#13)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int>
+
+(48) ColumnarToRow [codegen id : 16]
+Input [2]: [ws_item_sk#37, ws_sold_date_sk#38]
+
+(49) Filter [codegen id : 16]
+Input [2]: [ws_item_sk#37, ws_sold_date_sk#38]
+Condition : isnotnull(ws_item_sk#37)
+
+(50) ReusedExchange [Reuses operator id: 16]
+Output [1]: [d_date_sk#39]
+
+(51) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [ws_sold_date_sk#38]
+Right keys [1]: [d_date_sk#39]
+Join condition: None
+
+(52) Project [codegen id : 16]
+Output [1]: [ws_item_sk#37]
+Input [3]: [ws_item_sk#37, ws_sold_date_sk#38, d_date_sk#39]
+
+(53) ReusedExchange [Reuses operator id: 33]
+Output [4]: [i_item_sk#40, i_brand_id#41, i_class_id#42, i_category_id#43]
+
+(54) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [ws_item_sk#37]
+Right keys [1]: [i_item_sk#40]
+Join condition: None
+
+(55) Project [codegen id : 16]
+Output [3]: [i_brand_id#41, i_class_id#42, i_category_id#43]
+Input [5]: [ws_item_sk#37, i_item_sk#40, i_brand_id#41, i_class_id#42, i_category_id#43]
+
+(56) Exchange
+Input [3]: [i_brand_id#41, i_class_id#42, i_category_id#43]
+Arguments: hashpartitioning(coalesce(i_brand_id#41, 0), isnull(i_brand_id#41), coalesce(i_class_id#42, 0), isnull(i_class_id#42), coalesce(i_category_id#43, 0), isnull(i_category_id#43), 5), ENSURE_REQUIREMENTS, [id=#44]
+
+(57) Sort [codegen id : 17]
+Input [3]: [i_brand_id#41, i_class_id#42, i_category_id#43]
+Arguments: [coalesce(i_brand_id#41, 0) ASC NULLS FIRST, isnull(i_brand_id#41) ASC NULLS FIRST, coalesce(i_class_id#42, 0) ASC NULLS FIRST, isnull(i_class_id#42) ASC NULLS FIRST, coalesce(i_category_id#43, 0) ASC NULLS FIRST, isnull(i_category_id#43) ASC NULLS FIRST], false, 0
+
+(58) SortMergeJoin [codegen id : 18]
+Left keys [6]: [coalesce(brand_id#32, 0), isnull(brand_id#32), coalesce(class_id#33, 0), isnull(class_id#33), coalesce(category_id#34, 0), isnull(category_id#34)]
+Right keys [6]: [coalesce(i_brand_id#41, 0), isnull(i_brand_id#41), coalesce(i_class_id#42, 0), isnull(i_class_id#42), coalesce(i_category_id#43, 0), isnull(i_category_id#43)]
+Join condition: None
+
+(59) HashAggregate [codegen id : 18]
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Keys [3]: [brand_id#32, class_id#33, category_id#34]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#32, class_id#33, category_id#34]
+
+(60) Exchange
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Arguments: hashpartitioning(brand_id#32, class_id#33, category_id#34, 5), ENSURE_REQUIREMENTS, [id=#45]
+
+(61) HashAggregate [codegen id : 19]
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Keys [3]: [brand_id#32, class_id#33, category_id#34]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#32, class_id#33, category_id#34]
+
+(62) BroadcastExchange
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Arguments: HashedRelationBroadcastMode(List(input[0, int, true], input[1, int, true], input[2, int, true]),false), [id=#46]
+
+(63) BroadcastHashJoin [codegen id : 20]
+Left keys [3]: [i_brand_id#8, i_class_id#9, i_category_id#10]
+Right keys [3]: [brand_id#32, class_id#33, category_id#34]
+Join condition: None
+
+(64) Project [codegen id : 20]
+Output [1]: [i_item_sk#7 AS ss_item_sk#47]
+Input [7]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, brand_id#32, class_id#33, category_id#34]
+
+(65) Exchange
+Input [1]: [ss_item_sk#47]
+Arguments: hashpartitioning(ss_item_sk#47, 5), ENSURE_REQUIREMENTS, [id=#48]
+
+(66) Sort [codegen id : 21]
+Input [1]: [ss_item_sk#47]
+Arguments: [ss_item_sk#47 ASC NULLS FIRST], false, 0
+
+(67) SortMergeJoin [codegen id : 45]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [ss_item_sk#47]
+Join condition: None
+
+(68) Scan parquet default.date_dim
+Output [3]: [d_date_sk#49, d_year#50, d_moy#51]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,11), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(69) ColumnarToRow [codegen id : 22]
+Input [3]: [d_date_sk#49, d_year#50, d_moy#51]
+
+(70) Filter [codegen id : 22]
+Input [3]: [d_date_sk#49, d_year#50, d_moy#51]
+Condition : ((((isnotnull(d_year#50) AND isnotnull(d_moy#51)) AND (d_year#50 = 2001)) AND (d_moy#51 = 11)) AND isnotnull(d_date_sk#49))
+
+(71) Project [codegen id : 22]
+Output [1]: [d_date_sk#49]
+Input [3]: [d_date_sk#49, d_year#50, d_moy#51]
+
+(72) BroadcastExchange
+Input [1]: [d_date_sk#49]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#52]
+
+(73) BroadcastHashJoin [codegen id : 45]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#49]
+Join condition: None
+
+(74) Project [codegen id : 45]
+Output [3]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3]
+Input [5]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, d_date_sk#49]
+
+(75) Scan parquet default.item
+Output [4]: [i_item_sk#53, i_brand_id#54, i_class_id#55, i_category_id#56]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(76) ColumnarToRow [codegen id : 23]
+Input [4]: [i_item_sk#53, i_brand_id#54, i_class_id#55, i_category_id#56]
+
+(77) Filter [codegen id : 23]
+Input [4]: [i_item_sk#53, i_brand_id#54, i_class_id#55, i_category_id#56]
+Condition : isnotnull(i_item_sk#53)
+
+(78) Exchange
+Input [4]: [i_item_sk#53, i_brand_id#54, i_class_id#55, i_category_id#56]
+Arguments: hashpartitioning(i_item_sk#53, 5), ENSURE_REQUIREMENTS, [id=#57]
+
+(79) Sort [codegen id : 24]
+Input [4]: [i_item_sk#53, i_brand_id#54, i_class_id#55, i_category_id#56]
+Arguments: [i_item_sk#53 ASC NULLS FIRST], false, 0
+
+(80) ReusedExchange [Reuses operator id: unknown]
+Output [1]: [ss_item_sk#47]
+
+(81) Sort [codegen id : 43]
+Input [1]: [ss_item_sk#47]
+Arguments: [ss_item_sk#47 ASC NULLS FIRST], false, 0
+
+(82) SortMergeJoin [codegen id : 44]
+Left keys [1]: [i_item_sk#53]
+Right keys [1]: [ss_item_sk#47]
+Join condition: None
+
+(83) BroadcastExchange
+Input [4]: [i_item_sk#53, i_brand_id#54, i_class_id#55, i_category_id#56]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#58]
+
+(84) BroadcastHashJoin [codegen id : 45]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#53]
+Join condition: None
+
+(85) Project [codegen id : 45]
+Output [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#54, i_class_id#55, i_category_id#56]
+Input [7]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, i_item_sk#53, i_brand_id#54, i_class_id#55, i_category_id#56]
+
+(86) HashAggregate [codegen id : 45]
+Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#54, i_class_id#55, i_category_id#56]
+Keys [3]: [i_brand_id#54, i_class_id#55, i_category_id#56]
+Functions [2]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true)), partial_count(1)]
+Aggregate Attributes [3]: [sum#59, isEmpty#60, count#61]
+Results [6]: [i_brand_id#54, i_class_id#55, i_category_id#56, sum#62, isEmpty#63, count#64]
+
+(87) Exchange
+Input [6]: [i_brand_id#54, i_class_id#55, i_category_id#56, sum#62, isEmpty#63, count#64]
+Arguments: hashpartitioning(i_brand_id#54, i_class_id#55, i_category_id#56, 5), ENSURE_REQUIREMENTS, [id=#65]
+
+(88) HashAggregate [codegen id : 46]
+Input [6]: [i_brand_id#54, i_class_id#55, i_category_id#56, sum#62, isEmpty#63, count#64]
+Keys [3]: [i_brand_id#54, i_class_id#55, i_category_id#56]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#66, count(1)#67]
+Results [6]: [i_brand_id#54, i_class_id#55, i_category_id#56, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#66 AS sales#68, count(1)#67 AS number_sales#69, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#66 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#70]
+
+(89) Filter [codegen id : 46]
+Input [6]: [i_brand_id#54, i_class_id#55, i_category_id#56, sales#68, number_sales#69, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#70]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#70) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#70 as decimal(32,6)) > cast(Subquery scalar-subquery#71, [id=#72] as decimal(32,6))))
+
+(90) Project [codegen id : 46]
+Output [6]: [sales#68, number_sales#69, store AS channel#73, i_brand_id#54, i_class_id#55, i_category_id#56]
+Input [6]: [i_brand_id#54, i_class_id#55, i_category_id#56, sales#68, number_sales#69, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#70]
+
+(91) Scan parquet default.catalog_sales
+Output [4]: [cs_item_sk#74, cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#77), dynamicpruningexpression(cs_sold_date_sk#77 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int,cs_quantity:int,cs_list_price:decimal(7,2)>
+
+(92) ColumnarToRow [codegen id : 47]
+Input [4]: [cs_item_sk#74, cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77]
+
+(93) Filter [codegen id : 47]
+Input [4]: [cs_item_sk#74, cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77]
+Condition : isnotnull(cs_item_sk#74)
+
+(94) Exchange
+Input [4]: [cs_item_sk#74, cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77]
+Arguments: hashpartitioning(cs_item_sk#74, 5), ENSURE_REQUIREMENTS, [id=#78]
+
+(95) Sort [codegen id : 48]
+Input [4]: [cs_item_sk#74, cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77]
+Arguments: [cs_item_sk#74 ASC NULLS FIRST], false, 0
+
+(96) ReusedExchange [Reuses operator id: unknown]
+Output [1]: [ss_item_sk#47]
+
+(97) Sort [codegen id : 67]
+Input [1]: [ss_item_sk#47]
+Arguments: [ss_item_sk#47 ASC NULLS FIRST], false, 0
+
+(98) SortMergeJoin [codegen id : 91]
+Left keys [1]: [cs_item_sk#74]
+Right keys [1]: [ss_item_sk#47]
+Join condition: None
+
+(99) ReusedExchange [Reuses operator id: 72]
+Output [1]: [d_date_sk#79]
+
+(100) BroadcastHashJoin [codegen id : 91]
+Left keys [1]: [cs_sold_date_sk#77]
+Right keys [1]: [d_date_sk#79]
+Join condition: None
+
+(101) Project [codegen id : 91]
+Output [3]: [cs_item_sk#74, cs_quantity#75, cs_list_price#76]
+Input [5]: [cs_item_sk#74, cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77, d_date_sk#79]
+
+(102) ReusedExchange [Reuses operator id: 83]
+Output [4]: [i_item_sk#80, i_brand_id#81, i_class_id#82, i_category_id#83]
+
+(103) BroadcastHashJoin [codegen id : 91]
+Left keys [1]: [cs_item_sk#74]
+Right keys [1]: [i_item_sk#80]
+Join condition: None
+
+(104) Project [codegen id : 91]
+Output [5]: [cs_quantity#75, cs_list_price#76, i_brand_id#81, i_class_id#82, i_category_id#83]
+Input [7]: [cs_item_sk#74, cs_quantity#75, cs_list_price#76, i_item_sk#80, i_brand_id#81, i_class_id#82, i_category_id#83]
+
+(105) HashAggregate [codegen id : 91]
+Input [5]: [cs_quantity#75, cs_list_price#76, i_brand_id#81, i_class_id#82, i_category_id#83]
+Keys [3]: [i_brand_id#81, i_class_id#82, i_category_id#83]
+Functions [2]: [partial_sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#76 as decimal(12,2)))), DecimalType(18,2), true)), partial_count(1)]
+Aggregate Attributes [3]: [sum#84, isEmpty#85, count#86]
+Results [6]: [i_brand_id#81, i_class_id#82, i_category_id#83, sum#87, isEmpty#88, count#89]
+
+(106) Exchange
+Input [6]: [i_brand_id#81, i_class_id#82, i_category_id#83, sum#87, isEmpty#88, count#89]
+Arguments: hashpartitioning(i_brand_id#81, i_class_id#82, i_category_id#83, 5), ENSURE_REQUIREMENTS, [id=#90]
+
+(107) HashAggregate [codegen id : 92]
+Input [6]: [i_brand_id#81, i_class_id#82, i_category_id#83, sum#87, isEmpty#88, count#89]
+Keys [3]: [i_brand_id#81, i_class_id#82, i_category_id#83]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#76 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#76 as decimal(12,2)))), DecimalType(18,2), true))#91, count(1)#92]
+Results [6]: [i_brand_id#81, i_class_id#82, i_category_id#83, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#76 as decimal(12,2)))), DecimalType(18,2), true))#91 AS sales#93, count(1)#92 AS number_sales#94, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#76 as decimal(12,2)))), DecimalType(18,2), true))#91 AS sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#76 as decimal(12,2)))), DecimalType(18,2), true))#95]
+
+(108) Filter [codegen id : 92]
+Input [6]: [i_brand_id#81, i_class_id#82, i_category_id#83, sales#93, number_sales#94, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#76 as decimal(12,2)))), DecimalType(18,2), true))#95]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#76 as decimal(12,2)))), DecimalType(18,2), true))#95) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#76 as decimal(12,2)))), DecimalType(18,2), true))#95 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#71, [id=#72] as decimal(32,6))))
+
+(109) Project [codegen id : 92]
+Output [6]: [sales#93, number_sales#94, catalog AS channel#96, i_brand_id#81, i_class_id#82, i_category_id#83]
+Input [6]: [i_brand_id#81, i_class_id#82, i_category_id#83, sales#93, number_sales#94, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#76 as decimal(12,2)))), DecimalType(18,2), true))#95]
+
+(110) Scan parquet default.web_sales
+Output [4]: [ws_item_sk#97, ws_quantity#98, ws_list_price#99, ws_sold_date_sk#100]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#100), dynamicpruningexpression(ws_sold_date_sk#100 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_quantity:int,ws_list_price:decimal(7,2)>
+
+(111) ColumnarToRow [codegen id : 93]
+Input [4]: [ws_item_sk#97, ws_quantity#98, ws_list_price#99, ws_sold_date_sk#100]
+
+(112) Filter [codegen id : 93]
+Input [4]: [ws_item_sk#97, ws_quantity#98, ws_list_price#99, ws_sold_date_sk#100]
+Condition : isnotnull(ws_item_sk#97)
+
+(113) Exchange
+Input [4]: [ws_item_sk#97, ws_quantity#98, ws_list_price#99, ws_sold_date_sk#100]
+Arguments: hashpartitioning(ws_item_sk#97, 5), ENSURE_REQUIREMENTS, [id=#101]
+
+(114) Sort [codegen id : 94]
+Input [4]: [ws_item_sk#97, ws_quantity#98, ws_list_price#99, ws_sold_date_sk#100]
+Arguments: [ws_item_sk#97 ASC NULLS FIRST], false, 0
+
+(115) ReusedExchange [Reuses operator id: unknown]
+Output [1]: [ss_item_sk#47]
+
+(116) Sort [codegen id : 113]
+Input [1]: [ss_item_sk#47]
+Arguments: [ss_item_sk#47 ASC NULLS FIRST], false, 0
+
+(117) SortMergeJoin [codegen id : 137]
+Left keys [1]: [ws_item_sk#97]
+Right keys [1]: [ss_item_sk#47]
+Join condition: None
+
+(118) ReusedExchange [Reuses operator id: 72]
+Output [1]: [d_date_sk#102]
+
+(119) BroadcastHashJoin [codegen id : 137]
+Left keys [1]: [ws_sold_date_sk#100]
+Right keys [1]: [d_date_sk#102]
+Join condition: None
+
+(120) Project [codegen id : 137]
+Output [3]: [ws_item_sk#97, ws_quantity#98, ws_list_price#99]
+Input [5]: [ws_item_sk#97, ws_quantity#98, ws_list_price#99, ws_sold_date_sk#100, d_date_sk#102]
+
+(121) ReusedExchange [Reuses operator id: 83]
+Output [4]: [i_item_sk#103, i_brand_id#104, i_class_id#105, i_category_id#106]
+
+(122) BroadcastHashJoin [codegen id : 137]
+Left keys [1]: [ws_item_sk#97]
+Right keys [1]: [i_item_sk#103]
+Join condition: None
+
+(123) Project [codegen id : 137]
+Output [5]: [ws_quantity#98, ws_list_price#99, i_brand_id#104, i_class_id#105, i_category_id#106]
+Input [7]: [ws_item_sk#97, ws_quantity#98, ws_list_price#99, i_item_sk#103, i_brand_id#104, i_class_id#105, i_category_id#106]
+
+(124) HashAggregate [codegen id : 137]
+Input [5]: [ws_quantity#98, ws_list_price#99, i_brand_id#104, i_class_id#105, i_category_id#106]
+Keys [3]: [i_brand_id#104, i_class_id#105, i_category_id#106]
+Functions [2]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#98 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#99 as decimal(12,2)))), DecimalType(18,2), true)), partial_count(1)]
+Aggregate Attributes [3]: [sum#107, isEmpty#108, count#109]
+Results [6]: [i_brand_id#104, i_class_id#105, i_category_id#106, sum#110, isEmpty#111, count#112]
+
+(125) Exchange
+Input [6]: [i_brand_id#104, i_class_id#105, i_category_id#106, sum#110, isEmpty#111, count#112]
+Arguments: hashpartitioning(i_brand_id#104, i_class_id#105, i_category_id#106, 5), ENSURE_REQUIREMENTS, [id=#113]
+
+(126) HashAggregate [codegen id : 138]
+Input [6]: [i_brand_id#104, i_class_id#105, i_category_id#106, sum#110, isEmpty#111, count#112]
+Keys [3]: [i_brand_id#104, i_class_id#105, i_category_id#106]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#98 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#99 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#98 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#99 as decimal(12,2)))), DecimalType(18,2), true))#114, count(1)#115]
+Results [6]: [i_brand_id#104, i_class_id#105, i_category_id#106, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#98 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#99 as decimal(12,2)))), DecimalType(18,2), true))#114 AS sales#116, count(1)#115 AS number_sales#117, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#98 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#99 as decimal(12,2)))), DecimalType(18,2), true))#114 AS sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#98 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#99 as decimal(12,2)))), DecimalType(18,2), true))#118]
+
+(127) Filter [codegen id : 138]
+Input [6]: [i_brand_id#104, i_class_id#105, i_category_id#106, sales#116, number_sales#117, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#98 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#99 as decimal(12,2)))), DecimalType(18,2), true))#118]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#98 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#99 as decimal(12,2)))), DecimalType(18,2), true))#118) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#98 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#99 as decimal(12,2)))), DecimalType(18,2), true))#118 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#71, [id=#72] as decimal(32,6))))
+
+(128) Project [codegen id : 138]
+Output [6]: [sales#116, number_sales#117, web AS channel#119, i_brand_id#104, i_class_id#105, i_category_id#106]
+Input [6]: [i_brand_id#104, i_class_id#105, i_category_id#106, sales#116, number_sales#117, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#98 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#99 as decimal(12,2)))), DecimalType(18,2), true))#118]
+
+(129) Union
+
+(130) Expand [codegen id : 139]
+Input [6]: [sales#68, number_sales#69, channel#73, i_brand_id#54, i_class_id#55, i_category_id#56]
+Arguments: [ArrayBuffer(sales#68, number_sales#69, channel#73, i_brand_id#54, i_class_id#55, i_category_id#56, 0), ArrayBuffer(sales#68, number_sales#69, channel#73, i_brand_id#54, i_class_id#55, null, 1), ArrayBuffer(sales#68, number_sales#69, channel#73, i_brand_id#54, null, null, 3), ArrayBuffer(sales#68, number_sales#69, channel#73, null, null, null, 7), ArrayBuffer(sales#68, number_sales#69, null, null, null, null, 15)], [sales#68, number_sales#69, channel#120, i_brand_id#121, i_class_id#122, i_category_id#123, spark_grouping_id#124]
+
+(131) HashAggregate [codegen id : 139]
+Input [7]: [sales#68, number_sales#69, channel#120, i_brand_id#121, i_class_id#122, i_category_id#123, spark_grouping_id#124]
+Keys [5]: [channel#120, i_brand_id#121, i_class_id#122, i_category_id#123, spark_grouping_id#124]
+Functions [2]: [partial_sum(sales#68), partial_sum(number_sales#69)]
+Aggregate Attributes [3]: [sum#125, isEmpty#126, sum#127]
+Results [8]: [channel#120, i_brand_id#121, i_class_id#122, i_category_id#123, spark_grouping_id#124, sum#128, isEmpty#129, sum#130]
+
+(132) Exchange
+Input [8]: [channel#120, i_brand_id#121, i_class_id#122, i_category_id#123, spark_grouping_id#124, sum#128, isEmpty#129, sum#130]
+Arguments: hashpartitioning(channel#120, i_brand_id#121, i_class_id#122, i_category_id#123, spark_grouping_id#124, 5), ENSURE_REQUIREMENTS, [id=#131]
+
+(133) HashAggregate [codegen id : 140]
+Input [8]: [channel#120, i_brand_id#121, i_class_id#122, i_category_id#123, spark_grouping_id#124, sum#128, isEmpty#129, sum#130]
+Keys [5]: [channel#120, i_brand_id#121, i_class_id#122, i_category_id#123, spark_grouping_id#124]
+Functions [2]: [sum(sales#68), sum(number_sales#69)]
+Aggregate Attributes [2]: [sum(sales#68)#132, sum(number_sales#69)#133]
+Results [6]: [channel#120, i_brand_id#121, i_class_id#122, i_category_id#123, sum(sales#68)#132 AS sum(sales)#134, sum(number_sales#69)#133 AS sum(number_sales)#135]
+
+(134) TakeOrderedAndProject
+Input [6]: [channel#120, i_brand_id#121, i_class_id#122, i_category_id#123, sum(sales)#134, sum(number_sales)#135]
+Arguments: 100, [channel#120 ASC NULLS FIRST, i_brand_id#121 ASC NULLS FIRST, i_class_id#122 ASC NULLS FIRST, i_category_id#123 ASC NULLS FIRST], [channel#120, i_brand_id#121, i_class_id#122, i_category_id#123, sum(sales)#134, sum(number_sales)#135]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 89 Hosting Expression = Subquery scalar-subquery#71, [id=#72]
+* HashAggregate (157)
++- Exchange (156)
+   +- * HashAggregate (155)
+      +- Union (154)
+         :- * Project (143)
+         :  +- * BroadcastHashJoin Inner BuildRight (142)
+         :     :- * ColumnarToRow (136)
+         :     :  +- Scan parquet default.store_sales (135)
+         :     +- BroadcastExchange (141)
+         :        +- * Project (140)
+         :           +- * Filter (139)
+         :              +- * ColumnarToRow (138)
+         :                 +- Scan parquet default.date_dim (137)
+         :- * Project (148)
+         :  +- * BroadcastHashJoin Inner BuildRight (147)
+         :     :- * ColumnarToRow (145)
+         :     :  +- Scan parquet default.catalog_sales (144)
+         :     +- ReusedExchange (146)
+         +- * Project (153)
+            +- * BroadcastHashJoin Inner BuildRight (152)
+               :- * ColumnarToRow (150)
+               :  +- Scan parquet default.web_sales (149)
+               +- ReusedExchange (151)
+
+
+(135) Scan parquet default.store_sales
+Output [3]: [ss_quantity#136, ss_list_price#137, ss_sold_date_sk#138]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#138), dynamicpruningexpression(ss_sold_date_sk#138 IN dynamicpruning#139)]
+ReadSchema: struct<ss_quantity:int,ss_list_price:decimal(7,2)>
+
+(136) ColumnarToRow [codegen id : 2]
+Input [3]: [ss_quantity#136, ss_list_price#137, ss_sold_date_sk#138]
+
+(137) Scan parquet default.date_dim
+Output [2]: [d_date_sk#140, d_year#141]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(138) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#140, d_year#141]
+
+(139) Filter [codegen id : 1]
+Input [2]: [d_date_sk#140, d_year#141]
+Condition : (((isnotnull(d_year#141) AND (d_year#141 >= 1999)) AND (d_year#141 <= 2001)) AND isnotnull(d_date_sk#140))
+
+(140) Project [codegen id : 1]
+Output [1]: [d_date_sk#140]
+Input [2]: [d_date_sk#140, d_year#141]
+
+(141) BroadcastExchange
+Input [1]: [d_date_sk#140]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#142]
+
+(142) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#138]
+Right keys [1]: [d_date_sk#140]
+Join condition: None
+
+(143) Project [codegen id : 2]
+Output [2]: [ss_quantity#136 AS quantity#143, ss_list_price#137 AS list_price#144]
+Input [4]: [ss_quantity#136, ss_list_price#137, ss_sold_date_sk#138, d_date_sk#140]
+
+(144) Scan parquet default.catalog_sales
+Output [3]: [cs_quantity#145, cs_list_price#146, cs_sold_date_sk#147]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#147), dynamicpruningexpression(cs_sold_date_sk#147 IN dynamicpruning#139)]
+ReadSchema: struct<cs_quantity:int,cs_list_price:decimal(7,2)>
+
+(145) ColumnarToRow [codegen id : 4]
+Input [3]: [cs_quantity#145, cs_list_price#146, cs_sold_date_sk#147]
+
+(146) ReusedExchange [Reuses operator id: 141]
+Output [1]: [d_date_sk#148]
+
+(147) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_sold_date_sk#147]
+Right keys [1]: [d_date_sk#148]
+Join condition: None
+
+(148) Project [codegen id : 4]
+Output [2]: [cs_quantity#145 AS quantity#149, cs_list_price#146 AS list_price#150]
+Input [4]: [cs_quantity#145, cs_list_price#146, cs_sold_date_sk#147, d_date_sk#148]
+
+(149) Scan parquet default.web_sales
+Output [3]: [ws_quantity#151, ws_list_price#152, ws_sold_date_sk#153]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#153), dynamicpruningexpression(ws_sold_date_sk#153 IN dynamicpruning#139)]
+ReadSchema: struct<ws_quantity:int,ws_list_price:decimal(7,2)>
+
+(150) ColumnarToRow [codegen id : 6]
+Input [3]: [ws_quantity#151, ws_list_price#152, ws_sold_date_sk#153]
+
+(151) ReusedExchange [Reuses operator id: 141]
+Output [1]: [d_date_sk#154]
+
+(152) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ws_sold_date_sk#153]
+Right keys [1]: [d_date_sk#154]
+Join condition: None
+
+(153) Project [codegen id : 6]
+Output [2]: [ws_quantity#151 AS quantity#155, ws_list_price#152 AS list_price#156]
+Input [4]: [ws_quantity#151, ws_list_price#152, ws_sold_date_sk#153, d_date_sk#154]
+
+(154) Union
+
+(155) HashAggregate [codegen id : 7]
+Input [2]: [quantity#143, list_price#144]
+Keys: []
+Functions [1]: [partial_avg(CheckOverflow((promote_precision(cast(cast(quantity#143 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#144 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [2]: [sum#157, count#158]
+Results [2]: [sum#159, count#160]
+
+(156) Exchange
+Input [2]: [sum#159, count#160]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#161]
+
+(157) HashAggregate [codegen id : 8]
+Input [2]: [sum#159, count#160]
+Keys: []
+Functions [1]: [avg(CheckOverflow((promote_precision(cast(cast(quantity#143 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#144 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [avg(CheckOverflow((promote_precision(cast(cast(quantity#143 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#144 as decimal(12,2)))), DecimalType(18,2), true))#162]
+Results [1]: [avg(CheckOverflow((promote_precision(cast(cast(quantity#143 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#144 as decimal(12,2)))), DecimalType(18,2), true))#162 AS average_sales#163]
+
+Subquery:2 Hosting operator id = 135 Hosting Expression = ss_sold_date_sk#138 IN dynamicpruning#139
+ReusedExchange (158)
+
+
+(158) ReusedExchange [Reuses operator id: 141]
+Output [1]: [d_date_sk#140]
+
+Subquery:3 Hosting operator id = 144 Hosting Expression = cs_sold_date_sk#147 IN dynamicpruning#139
+
+Subquery:4 Hosting operator id = 149 Hosting Expression = ws_sold_date_sk#153 IN dynamicpruning#139
+
+Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (159)
+
+
+(159) ReusedExchange [Reuses operator id: 72]
+Output [1]: [d_date_sk#49]
+
+Subquery:6 Hosting operator id = 9 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13
+ReusedExchange (160)
+
+
+(160) ReusedExchange [Reuses operator id: 16]
+Output [1]: [d_date_sk#14]
+
+Subquery:7 Hosting operator id = 24 Hosting Expression = cs_sold_date_sk#23 IN dynamicpruning#13
+
+Subquery:8 Hosting operator id = 47 Hosting Expression = ws_sold_date_sk#38 IN dynamicpruning#13
+
+Subquery:9 Hosting operator id = 108 Hosting Expression = ReusedSubquery Subquery scalar-subquery#71, [id=#72]
+
+Subquery:10 Hosting operator id = 91 Hosting Expression = cs_sold_date_sk#77 IN dynamicpruning#5
+
+Subquery:11 Hosting operator id = 127 Hosting Expression = ReusedSubquery Subquery scalar-subquery#71, [id=#72]
+
+Subquery:12 Hosting operator id = 110 Hosting Expression = ws_sold_date_sk#100 IN dynamicpruning#5
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.sf100/simplified.txt
new file mode 100644
index 0000000000000..bf3b205bb8c58
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.sf100/simplified.txt
@@ -0,0 +1,280 @@
+TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales)]
+  WholeStageCodegen (140)
+    HashAggregate [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum(sales),sum(number_sales),sum,isEmpty,sum]
+      InputAdapter
+        Exchange [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] #1
+          WholeStageCodegen (139)
+            HashAggregate [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum]
+              Expand [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id]
+                InputAdapter
+                  Union
+                    WholeStageCodegen (46)
+                      Project [sales,number_sales,i_brand_id,i_class_id,i_category_id]
+                        Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                          Subquery #3
+                            WholeStageCodegen (8)
+                              HashAggregate [sum,count] [avg(CheckOverflow((promote_precision(cast(cast(quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price as decimal(12,2)))), DecimalType(18,2), true)),average_sales,sum,count]
+                                InputAdapter
+                                  Exchange #19
+                                    WholeStageCodegen (7)
+                                      HashAggregate [quantity,list_price] [sum,count,sum,count]
+                                        InputAdapter
+                                          Union
+                                            WholeStageCodegen (2)
+                                              Project [ss_quantity,ss_list_price]
+                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk]
+                                                        SubqueryBroadcast [d_date_sk] #4
+                                                          ReusedExchange [d_date_sk] #20
+                                                  InputAdapter
+                                                    BroadcastExchange #20
+                                                      WholeStageCodegen (1)
+                                                        Project [d_date_sk]
+                                                          Filter [d_year,d_date_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.date_dim [d_date_sk,d_year]
+                                            WholeStageCodegen (4)
+                                              Project [cs_quantity,cs_list_price]
+                                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk]
+                                                        ReusedSubquery [d_date_sk] #4
+                                                  InputAdapter
+                                                    ReusedExchange [d_date_sk] #20
+                                            WholeStageCodegen (6)
+                                              Project [ws_quantity,ws_list_price]
+                                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk]
+                                                        ReusedSubquery [d_date_sk] #4
+                                                  InputAdapter
+                                                    ReusedExchange [d_date_sk] #20
+                          HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                            InputAdapter
+                              Exchange [i_brand_id,i_class_id,i_category_id] #2
+                                WholeStageCodegen (45)
+                                  HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count]
+                                    Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id]
+                                      BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                        Project [ss_item_sk,ss_quantity,ss_list_price]
+                                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                            SortMergeJoin [ss_item_sk,ss_item_sk]
+                                              InputAdapter
+                                                WholeStageCodegen (2)
+                                                  Sort [ss_item_sk]
+                                                    InputAdapter
+                                                      Exchange [ss_item_sk] #3
+                                                        WholeStageCodegen (1)
+                                                          Filter [ss_item_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk]
+                                                                  SubqueryBroadcast [d_date_sk] #1
+                                                                    ReusedExchange [d_date_sk] #4
+                                              InputAdapter
+                                                WholeStageCodegen (21)
+                                                  Sort [ss_item_sk]
+                                                    InputAdapter
+                                                      Exchange [ss_item_sk] #5
+                                                        WholeStageCodegen (20)
+                                                          Project [i_item_sk]
+                                                            BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id]
+                                                              Filter [i_brand_id,i_class_id,i_category_id]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                              InputAdapter
+                                                                BroadcastExchange #6
+                                                                  WholeStageCodegen (19)
+                                                                    HashAggregate [brand_id,class_id,category_id]
+                                                                      InputAdapter
+                                                                        Exchange [brand_id,class_id,category_id] #7
+                                                                          WholeStageCodegen (18)
+                                                                            HashAggregate [brand_id,class_id,category_id]
+                                                                              SortMergeJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id]
+                                                                                InputAdapter
+                                                                                  WholeStageCodegen (13)
+                                                                                    Sort [brand_id,class_id,category_id]
+                                                                                      InputAdapter
+                                                                                        Exchange [brand_id,class_id,category_id] #8
+                                                                                          WholeStageCodegen (12)
+                                                                                            HashAggregate [brand_id,class_id,category_id]
+                                                                                              InputAdapter
+                                                                                                Exchange [brand_id,class_id,category_id] #9
+                                                                                                  WholeStageCodegen (11)
+                                                                                                    HashAggregate [brand_id,class_id,category_id]
+                                                                                                      Project [i_brand_id,i_class_id,i_category_id]
+                                                                                                        BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                                                                          Project [ss_item_sk]
+                                                                                                            BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                                              Filter [ss_item_sk]
+                                                                                                                ColumnarToRow
+                                                                                                                  InputAdapter
+                                                                                                                    Scan parquet default.store_sales [ss_item_sk,ss_sold_date_sk]
+                                                                                                                      SubqueryBroadcast [d_date_sk] #2
+                                                                                                                        ReusedExchange [d_date_sk] #10
+                                                                                                              InputAdapter
+                                                                                                                BroadcastExchange #10
+                                                                                                                  WholeStageCodegen (3)
+                                                                                                                    Project [d_date_sk]
+                                                                                                                      Filter [d_year,d_date_sk]
+                                                                                                                        ColumnarToRow
+                                                                                                                          InputAdapter
+                                                                                                                            Scan parquet default.date_dim [d_date_sk,d_year]
+                                                                                                          InputAdapter
+                                                                                                            BroadcastExchange #11
+                                                                                                              WholeStageCodegen (10)
+                                                                                                                SortMergeJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id]
+                                                                                                                  InputAdapter
+                                                                                                                    WholeStageCodegen (5)
+                                                                                                                      Sort [i_brand_id,i_class_id,i_category_id]
+                                                                                                                        InputAdapter
+                                                                                                                          Exchange [i_brand_id,i_class_id,i_category_id] #12
+                                                                                                                            WholeStageCodegen (4)
+                                                                                                                              Filter [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                                                                ColumnarToRow
+                                                                                                                                  InputAdapter
+                                                                                                                                    Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                                                  InputAdapter
+                                                                                                                    WholeStageCodegen (9)
+                                                                                                                      Sort [i_brand_id,i_class_id,i_category_id]
+                                                                                                                        InputAdapter
+                                                                                                                          Exchange [i_brand_id,i_class_id,i_category_id] #13
+                                                                                                                            WholeStageCodegen (8)
+                                                                                                                              Project [i_brand_id,i_class_id,i_category_id]
+                                                                                                                                BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                                                                                                                  Project [cs_item_sk]
+                                                                                                                                    BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                                                                                      Filter [cs_item_sk]
+                                                                                                                                        ColumnarToRow
+                                                                                                                                          InputAdapter
+                                                                                                                                            Scan parquet default.catalog_sales [cs_item_sk,cs_sold_date_sk]
+                                                                                                                                              ReusedSubquery [d_date_sk] #2
+                                                                                                                                      InputAdapter
+                                                                                                                                        ReusedExchange [d_date_sk] #10
+                                                                                                                                  InputAdapter
+                                                                                                                                    BroadcastExchange #14
+                                                                                                                                      WholeStageCodegen (7)
+                                                                                                                                        Filter [i_item_sk]
+                                                                                                                                          ColumnarToRow
+                                                                                                                                            InputAdapter
+                                                                                                                                              Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                InputAdapter
+                                                                                  WholeStageCodegen (17)
+                                                                                    Sort [i_brand_id,i_class_id,i_category_id]
+                                                                                      InputAdapter
+                                                                                        Exchange [i_brand_id,i_class_id,i_category_id] #15
+                                                                                          WholeStageCodegen (16)
+                                                                                            Project [i_brand_id,i_class_id,i_category_id]
+                                                                                              BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                                                                                Project [ws_item_sk]
+                                                                                                  BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                                                    Filter [ws_item_sk]
+                                                                                                      ColumnarToRow
+                                                                                                        InputAdapter
+                                                                                                          Scan parquet default.web_sales [ws_item_sk,ws_sold_date_sk]
+                                                                                                            ReusedSubquery [d_date_sk] #2
+                                                                                                    InputAdapter
+                                                                                                      ReusedExchange [d_date_sk] #10
+                                                                                                InputAdapter
+                                                                                                  ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14
+                                            InputAdapter
+                                              BroadcastExchange #4
+                                                WholeStageCodegen (22)
+                                                  Project [d_date_sk]
+                                                    Filter [d_year,d_moy,d_date_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                        InputAdapter
+                                          BroadcastExchange #16
+                                            WholeStageCodegen (44)
+                                              SortMergeJoin [i_item_sk,ss_item_sk]
+                                                InputAdapter
+                                                  WholeStageCodegen (24)
+                                                    Sort [i_item_sk]
+                                                      InputAdapter
+                                                        Exchange [i_item_sk] #17
+                                                          WholeStageCodegen (23)
+                                                            Filter [i_item_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                InputAdapter
+                                                  WholeStageCodegen (43)
+                                                    Sort [ss_item_sk]
+                                                      InputAdapter
+                                                        ReusedExchange [ss_item_sk] #18
+                    WholeStageCodegen (92)
+                      Project [sales,number_sales,i_brand_id,i_class_id,i_category_id]
+                        Filter [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                          ReusedSubquery [average_sales] #3
+                          HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                            InputAdapter
+                              Exchange [i_brand_id,i_class_id,i_category_id] #21
+                                WholeStageCodegen (91)
+                                  HashAggregate [i_brand_id,i_class_id,i_category_id,cs_quantity,cs_list_price] [sum,isEmpty,count,sum,isEmpty,count]
+                                    Project [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id]
+                                      BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                        Project [cs_item_sk,cs_quantity,cs_list_price]
+                                          BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                            SortMergeJoin [cs_item_sk,ss_item_sk]
+                                              InputAdapter
+                                                WholeStageCodegen (48)
+                                                  Sort [cs_item_sk]
+                                                    InputAdapter
+                                                      Exchange [cs_item_sk] #22
+                                                        WholeStageCodegen (47)
+                                                          Filter [cs_item_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk]
+                                                                  ReusedSubquery [d_date_sk] #1
+                                              InputAdapter
+                                                WholeStageCodegen (67)
+                                                  Sort [ss_item_sk]
+                                                    InputAdapter
+                                                      ReusedExchange [ss_item_sk] #18
+                                            InputAdapter
+                                              ReusedExchange [d_date_sk] #4
+                                        InputAdapter
+                                          ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #16
+                    WholeStageCodegen (138)
+                      Project [sales,number_sales,i_brand_id,i_class_id,i_category_id]
+                        Filter [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                          ReusedSubquery [average_sales] #3
+                          HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                            InputAdapter
+                              Exchange [i_brand_id,i_class_id,i_category_id] #23
+                                WholeStageCodegen (137)
+                                  HashAggregate [i_brand_id,i_class_id,i_category_id,ws_quantity,ws_list_price] [sum,isEmpty,count,sum,isEmpty,count]
+                                    Project [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id]
+                                      BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                        Project [ws_item_sk,ws_quantity,ws_list_price]
+                                          BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                            SortMergeJoin [ws_item_sk,ss_item_sk]
+                                              InputAdapter
+                                                WholeStageCodegen (94)
+                                                  Sort [ws_item_sk]
+                                                    InputAdapter
+                                                      Exchange [ws_item_sk] #24
+                                                        WholeStageCodegen (93)
+                                                          Filter [ws_item_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk]
+                                                                  ReusedSubquery [d_date_sk] #1
+                                              InputAdapter
+                                                WholeStageCodegen (113)
+                                                  Sort [ss_item_sk]
+                                                    InputAdapter
+                                                      ReusedExchange [ss_item_sk] #18
+                                            InputAdapter
+                                              ReusedExchange [d_date_sk] #4
+                                        InputAdapter
+                                          ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #16
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/explain.txt
new file mode 100644
index 0000000000000..ffcbef4ce1602
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/explain.txt
@@ -0,0 +1,822 @@
+== Physical Plan ==
+TakeOrderedAndProject (115)
++- * HashAggregate (114)
+   +- Exchange (113)
+      +- * HashAggregate (112)
+         +- * Expand (111)
+            +- Union (110)
+               :- * Project (77)
+               :  +- * Filter (76)
+               :     +- * HashAggregate (75)
+               :        +- Exchange (74)
+               :           +- * HashAggregate (73)
+               :              +- * Project (72)
+               :                 +- * BroadcastHashJoin Inner BuildRight (71)
+               :                    :- * Project (65)
+               :                    :  +- * BroadcastHashJoin Inner BuildRight (64)
+               :                    :     :- * BroadcastHashJoin LeftSemi BuildRight (57)
+               :                    :     :  :- * Filter (3)
+               :                    :     :  :  +- * ColumnarToRow (2)
+               :                    :     :  :     +- Scan parquet default.store_sales (1)
+               :                    :     :  +- BroadcastExchange (56)
+               :                    :     :     +- * Project (55)
+               :                    :     :        +- * BroadcastHashJoin Inner BuildRight (54)
+               :                    :     :           :- * Filter (6)
+               :                    :     :           :  +- * ColumnarToRow (5)
+               :                    :     :           :     +- Scan parquet default.item (4)
+               :                    :     :           +- BroadcastExchange (53)
+               :                    :     :              +- * HashAggregate (52)
+               :                    :     :                 +- * HashAggregate (51)
+               :                    :     :                    +- * BroadcastHashJoin LeftSemi BuildRight (50)
+               :                    :     :                       :- * HashAggregate (39)
+               :                    :     :                       :  +- Exchange (38)
+               :                    :     :                       :     +- * HashAggregate (37)
+               :                    :     :                       :        +- * Project (36)
+               :                    :     :                       :           +- * BroadcastHashJoin Inner BuildRight (35)
+               :                    :     :                       :              :- * Project (33)
+               :                    :     :                       :              :  +- * BroadcastHashJoin Inner BuildRight (32)
+               :                    :     :                       :              :     :- * Filter (9)
+               :                    :     :                       :              :     :  +- * ColumnarToRow (8)
+               :                    :     :                       :              :     :     +- Scan parquet default.store_sales (7)
+               :                    :     :                       :              :     +- BroadcastExchange (31)
+               :                    :     :                       :              :        +- * BroadcastHashJoin LeftSemi BuildRight (30)
+               :                    :     :                       :              :           :- * Filter (12)
+               :                    :     :                       :              :           :  +- * ColumnarToRow (11)
+               :                    :     :                       :              :           :     +- Scan parquet default.item (10)
+               :                    :     :                       :              :           +- BroadcastExchange (29)
+               :                    :     :                       :              :              +- * Project (28)
+               :                    :     :                       :              :                 +- * BroadcastHashJoin Inner BuildRight (27)
+               :                    :     :                       :              :                    :- * Project (21)
+               :                    :     :                       :              :                    :  +- * BroadcastHashJoin Inner BuildRight (20)
+               :                    :     :                       :              :                    :     :- * Filter (15)
+               :                    :     :                       :              :                    :     :  +- * ColumnarToRow (14)
+               :                    :     :                       :              :                    :     :     +- Scan parquet default.catalog_sales (13)
+               :                    :     :                       :              :                    :     +- BroadcastExchange (19)
+               :                    :     :                       :              :                    :        +- * Filter (18)
+               :                    :     :                       :              :                    :           +- * ColumnarToRow (17)
+               :                    :     :                       :              :                    :              +- Scan parquet default.item (16)
+               :                    :     :                       :              :                    +- BroadcastExchange (26)
+               :                    :     :                       :              :                       +- * Project (25)
+               :                    :     :                       :              :                          +- * Filter (24)
+               :                    :     :                       :              :                             +- * ColumnarToRow (23)
+               :                    :     :                       :              :                                +- Scan parquet default.date_dim (22)
+               :                    :     :                       :              +- ReusedExchange (34)
+               :                    :     :                       +- BroadcastExchange (49)
+               :                    :     :                          +- * Project (48)
+               :                    :     :                             +- * BroadcastHashJoin Inner BuildRight (47)
+               :                    :     :                                :- * Project (45)
+               :                    :     :                                :  +- * BroadcastHashJoin Inner BuildRight (44)
+               :                    :     :                                :     :- * Filter (42)
+               :                    :     :                                :     :  +- * ColumnarToRow (41)
+               :                    :     :                                :     :     +- Scan parquet default.web_sales (40)
+               :                    :     :                                :     +- ReusedExchange (43)
+               :                    :     :                                +- ReusedExchange (46)
+               :                    :     +- BroadcastExchange (63)
+               :                    :        +- * BroadcastHashJoin LeftSemi BuildRight (62)
+               :                    :           :- * Filter (60)
+               :                    :           :  +- * ColumnarToRow (59)
+               :                    :           :     +- Scan parquet default.item (58)
+               :                    :           +- ReusedExchange (61)
+               :                    +- BroadcastExchange (70)
+               :                       +- * Project (69)
+               :                          +- * Filter (68)
+               :                             +- * ColumnarToRow (67)
+               :                                +- Scan parquet default.date_dim (66)
+               :- * Project (93)
+               :  +- * Filter (92)
+               :     +- * HashAggregate (91)
+               :        +- Exchange (90)
+               :           +- * HashAggregate (89)
+               :              +- * Project (88)
+               :                 +- * BroadcastHashJoin Inner BuildRight (87)
+               :                    :- * Project (85)
+               :                    :  +- * BroadcastHashJoin Inner BuildRight (84)
+               :                    :     :- * BroadcastHashJoin LeftSemi BuildRight (82)
+               :                    :     :  :- * Filter (80)
+               :                    :     :  :  +- * ColumnarToRow (79)
+               :                    :     :  :     +- Scan parquet default.catalog_sales (78)
+               :                    :     :  +- ReusedExchange (81)
+               :                    :     +- ReusedExchange (83)
+               :                    +- ReusedExchange (86)
+               +- * Project (109)
+                  +- * Filter (108)
+                     +- * HashAggregate (107)
+                        +- Exchange (106)
+                           +- * HashAggregate (105)
+                              +- * Project (104)
+                                 +- * BroadcastHashJoin Inner BuildRight (103)
+                                    :- * Project (101)
+                                    :  +- * BroadcastHashJoin Inner BuildRight (100)
+                                    :     :- * BroadcastHashJoin LeftSemi BuildRight (98)
+                                    :     :  :- * Filter (96)
+                                    :     :  :  +- * ColumnarToRow (95)
+                                    :     :  :     +- Scan parquet default.web_sales (94)
+                                    :     :  +- ReusedExchange (97)
+                                    :     +- ReusedExchange (99)
+                                    +- ReusedExchange (102)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_quantity:int,ss_list_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 25]
+Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 25]
+Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+Condition : isnotnull(ss_item_sk#1)
+
+(4) Scan parquet default.item
+Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(5) ColumnarToRow [codegen id : 11]
+Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9]
+
+(6) Filter [codegen id : 11]
+Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9]
+Condition : ((isnotnull(i_brand_id#7) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9))
+
+(7) Scan parquet default.store_sales
+Output [2]: [ss_item_sk#10, ss_sold_date_sk#11]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#11), dynamicpruningexpression(ss_sold_date_sk#11 IN dynamicpruning#12)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int>
+
+(8) ColumnarToRow [codegen id : 6]
+Input [2]: [ss_item_sk#10, ss_sold_date_sk#11]
+
+(9) Filter [codegen id : 6]
+Input [2]: [ss_item_sk#10, ss_sold_date_sk#11]
+Condition : isnotnull(ss_item_sk#10)
+
+(10) Scan parquet default.item
+Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(11) ColumnarToRow [codegen id : 4]
+Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16]
+
+(12) Filter [codegen id : 4]
+Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16]
+Condition : (((isnotnull(i_item_sk#13) AND isnotnull(i_brand_id#14)) AND isnotnull(i_class_id#15)) AND isnotnull(i_category_id#16))
+
+(13) Scan parquet default.catalog_sales
+Output [2]: [cs_item_sk#17, cs_sold_date_sk#18]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_sold_date_sk#18 IN dynamicpruning#12)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int>
+
+(14) ColumnarToRow [codegen id : 3]
+Input [2]: [cs_item_sk#17, cs_sold_date_sk#18]
+
+(15) Filter [codegen id : 3]
+Input [2]: [cs_item_sk#17, cs_sold_date_sk#18]
+Condition : isnotnull(cs_item_sk#17)
+
+(16) Scan parquet default.item
+Output [4]: [i_item_sk#19, i_brand_id#20, i_class_id#21, i_category_id#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(17) ColumnarToRow [codegen id : 1]
+Input [4]: [i_item_sk#19, i_brand_id#20, i_class_id#21, i_category_id#22]
+
+(18) Filter [codegen id : 1]
+Input [4]: [i_item_sk#19, i_brand_id#20, i_class_id#21, i_category_id#22]
+Condition : isnotnull(i_item_sk#19)
+
+(19) BroadcastExchange
+Input [4]: [i_item_sk#19, i_brand_id#20, i_class_id#21, i_category_id#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#23]
+
+(20) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_item_sk#17]
+Right keys [1]: [i_item_sk#19]
+Join condition: None
+
+(21) Project [codegen id : 3]
+Output [4]: [cs_sold_date_sk#18, i_brand_id#20, i_class_id#21, i_category_id#22]
+Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#19, i_brand_id#20, i_class_id#21, i_category_id#22]
+
+(22) Scan parquet default.date_dim
+Output [2]: [d_date_sk#24, d_year#25]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(23) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#24, d_year#25]
+
+(24) Filter [codegen id : 2]
+Input [2]: [d_date_sk#24, d_year#25]
+Condition : (((isnotnull(d_year#25) AND (d_year#25 >= 1999)) AND (d_year#25 <= 2001)) AND isnotnull(d_date_sk#24))
+
+(25) Project [codegen id : 2]
+Output [1]: [d_date_sk#24]
+Input [2]: [d_date_sk#24, d_year#25]
+
+(26) BroadcastExchange
+Input [1]: [d_date_sk#24]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#26]
+
+(27) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_sold_date_sk#18]
+Right keys [1]: [d_date_sk#24]
+Join condition: None
+
+(28) Project [codegen id : 3]
+Output [3]: [i_brand_id#20, i_class_id#21, i_category_id#22]
+Input [5]: [cs_sold_date_sk#18, i_brand_id#20, i_class_id#21, i_category_id#22, d_date_sk#24]
+
+(29) BroadcastExchange
+Input [3]: [i_brand_id#20, i_class_id#21, i_category_id#22]
+Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [id=#27]
+
+(30) BroadcastHashJoin [codegen id : 4]
+Left keys [6]: [coalesce(i_brand_id#14, 0), isnull(i_brand_id#14), coalesce(i_class_id#15, 0), isnull(i_class_id#15), coalesce(i_category_id#16, 0), isnull(i_category_id#16)]
+Right keys [6]: [coalesce(i_brand_id#20, 0), isnull(i_brand_id#20), coalesce(i_class_id#21, 0), isnull(i_class_id#21), coalesce(i_category_id#22, 0), isnull(i_category_id#22)]
+Join condition: None
+
+(31) BroadcastExchange
+Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#28]
+
+(32) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_item_sk#10]
+Right keys [1]: [i_item_sk#13]
+Join condition: None
+
+(33) Project [codegen id : 6]
+Output [4]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16]
+Input [6]: [ss_item_sk#10, ss_sold_date_sk#11, i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16]
+
+(34) ReusedExchange [Reuses operator id: 26]
+Output [1]: [d_date_sk#29]
+
+(35) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_sold_date_sk#11]
+Right keys [1]: [d_date_sk#29]
+Join condition: None
+
+(36) Project [codegen id : 6]
+Output [3]: [i_brand_id#14 AS brand_id#30, i_class_id#15 AS class_id#31, i_category_id#16 AS category_id#32]
+Input [5]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16, d_date_sk#29]
+
+(37) HashAggregate [codegen id : 6]
+Input [3]: [brand_id#30, class_id#31, category_id#32]
+Keys [3]: [brand_id#30, class_id#31, category_id#32]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#30, class_id#31, category_id#32]
+
+(38) Exchange
+Input [3]: [brand_id#30, class_id#31, category_id#32]
+Arguments: hashpartitioning(brand_id#30, class_id#31, category_id#32, 5), ENSURE_REQUIREMENTS, [id=#33]
+
+(39) HashAggregate [codegen id : 10]
+Input [3]: [brand_id#30, class_id#31, category_id#32]
+Keys [3]: [brand_id#30, class_id#31, category_id#32]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#30, class_id#31, category_id#32]
+
+(40) Scan parquet default.web_sales
+Output [2]: [ws_item_sk#34, ws_sold_date_sk#35]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#35), dynamicpruningexpression(ws_sold_date_sk#35 IN dynamicpruning#12)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int>
+
+(41) ColumnarToRow [codegen id : 9]
+Input [2]: [ws_item_sk#34, ws_sold_date_sk#35]
+
+(42) Filter [codegen id : 9]
+Input [2]: [ws_item_sk#34, ws_sold_date_sk#35]
+Condition : isnotnull(ws_item_sk#34)
+
+(43) ReusedExchange [Reuses operator id: 19]
+Output [4]: [i_item_sk#36, i_brand_id#37, i_class_id#38, i_category_id#39]
+
+(44) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ws_item_sk#34]
+Right keys [1]: [i_item_sk#36]
+Join condition: None
+
+(45) Project [codegen id : 9]
+Output [4]: [ws_sold_date_sk#35, i_brand_id#37, i_class_id#38, i_category_id#39]
+Input [6]: [ws_item_sk#34, ws_sold_date_sk#35, i_item_sk#36, i_brand_id#37, i_class_id#38, i_category_id#39]
+
+(46) ReusedExchange [Reuses operator id: 26]
+Output [1]: [d_date_sk#40]
+
+(47) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ws_sold_date_sk#35]
+Right keys [1]: [d_date_sk#40]
+Join condition: None
+
+(48) Project [codegen id : 9]
+Output [3]: [i_brand_id#37, i_class_id#38, i_category_id#39]
+Input [5]: [ws_sold_date_sk#35, i_brand_id#37, i_class_id#38, i_category_id#39, d_date_sk#40]
+
+(49) BroadcastExchange
+Input [3]: [i_brand_id#37, i_class_id#38, i_category_id#39]
+Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [id=#41]
+
+(50) BroadcastHashJoin [codegen id : 10]
+Left keys [6]: [coalesce(brand_id#30, 0), isnull(brand_id#30), coalesce(class_id#31, 0), isnull(class_id#31), coalesce(category_id#32, 0), isnull(category_id#32)]
+Right keys [6]: [coalesce(i_brand_id#37, 0), isnull(i_brand_id#37), coalesce(i_class_id#38, 0), isnull(i_class_id#38), coalesce(i_category_id#39, 0), isnull(i_category_id#39)]
+Join condition: None
+
+(51) HashAggregate [codegen id : 10]
+Input [3]: [brand_id#30, class_id#31, category_id#32]
+Keys [3]: [brand_id#30, class_id#31, category_id#32]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#30, class_id#31, category_id#32]
+
+(52) HashAggregate [codegen id : 10]
+Input [3]: [brand_id#30, class_id#31, category_id#32]
+Keys [3]: [brand_id#30, class_id#31, category_id#32]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#30, class_id#31, category_id#32]
+
+(53) BroadcastExchange
+Input [3]: [brand_id#30, class_id#31, category_id#32]
+Arguments: HashedRelationBroadcastMode(List(input[0, int, true], input[1, int, true], input[2, int, true]),false), [id=#42]
+
+(54) BroadcastHashJoin [codegen id : 11]
+Left keys [3]: [i_brand_id#7, i_class_id#8, i_category_id#9]
+Right keys [3]: [brand_id#30, class_id#31, category_id#32]
+Join condition: None
+
+(55) Project [codegen id : 11]
+Output [1]: [i_item_sk#6 AS ss_item_sk#43]
+Input [7]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, brand_id#30, class_id#31, category_id#32]
+
+(56) BroadcastExchange
+Input [1]: [ss_item_sk#43]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#44]
+
+(57) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [ss_item_sk#43]
+Join condition: None
+
+(58) Scan parquet default.item
+Output [4]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(59) ColumnarToRow [codegen id : 23]
+Input [4]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48]
+
+(60) Filter [codegen id : 23]
+Input [4]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48]
+Condition : isnotnull(i_item_sk#45)
+
+(61) ReusedExchange [Reuses operator id: unknown]
+Output [1]: [ss_item_sk#43]
+
+(62) BroadcastHashJoin [codegen id : 23]
+Left keys [1]: [i_item_sk#45]
+Right keys [1]: [ss_item_sk#43]
+Join condition: None
+
+(63) BroadcastExchange
+Input [4]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#49]
+
+(64) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#45]
+Join condition: None
+
+(65) Project [codegen id : 25]
+Output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#46, i_class_id#47, i_category_id#48]
+Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48]
+
+(66) Scan parquet default.date_dim
+Output [3]: [d_date_sk#50, d_year#51, d_moy#52]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,11), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(67) ColumnarToRow [codegen id : 24]
+Input [3]: [d_date_sk#50, d_year#51, d_moy#52]
+
+(68) Filter [codegen id : 24]
+Input [3]: [d_date_sk#50, d_year#51, d_moy#52]
+Condition : ((((isnotnull(d_year#51) AND isnotnull(d_moy#52)) AND (d_year#51 = 2001)) AND (d_moy#52 = 11)) AND isnotnull(d_date_sk#50))
+
+(69) Project [codegen id : 24]
+Output [1]: [d_date_sk#50]
+Input [3]: [d_date_sk#50, d_year#51, d_moy#52]
+
+(70) BroadcastExchange
+Input [1]: [d_date_sk#50]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#53]
+
+(71) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#50]
+Join condition: None
+
+(72) Project [codegen id : 25]
+Output [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#46, i_class_id#47, i_category_id#48]
+Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#46, i_class_id#47, i_category_id#48, d_date_sk#50]
+
+(73) HashAggregate [codegen id : 25]
+Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#46, i_class_id#47, i_category_id#48]
+Keys [3]: [i_brand_id#46, i_class_id#47, i_category_id#48]
+Functions [2]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true)), partial_count(1)]
+Aggregate Attributes [3]: [sum#54, isEmpty#55, count#56]
+Results [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sum#57, isEmpty#58, count#59]
+
+(74) Exchange
+Input [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sum#57, isEmpty#58, count#59]
+Arguments: hashpartitioning(i_brand_id#46, i_class_id#47, i_category_id#48, 5), ENSURE_REQUIREMENTS, [id=#60]
+
+(75) HashAggregate [codegen id : 26]
+Input [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sum#57, isEmpty#58, count#59]
+Keys [3]: [i_brand_id#46, i_class_id#47, i_category_id#48]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#61, count(1)#62]
+Results [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#61 AS sales#63, count(1)#62 AS number_sales#64, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#61 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#65]
+
+(76) Filter [codegen id : 26]
+Input [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sales#63, number_sales#64, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#65]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#65) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#65 as decimal(32,6)) > cast(Subquery scalar-subquery#66, [id=#67] as decimal(32,6))))
+
+(77) Project [codegen id : 26]
+Output [6]: [sales#63, number_sales#64, store AS channel#68, i_brand_id#46, i_class_id#47, i_category_id#48]
+Input [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sales#63, number_sales#64, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#65]
+
+(78) Scan parquet default.catalog_sales
+Output [4]: [cs_item_sk#69, cs_quantity#70, cs_list_price#71, cs_sold_date_sk#72]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#72), dynamicpruningexpression(cs_sold_date_sk#72 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int,cs_quantity:int,cs_list_price:decimal(7,2)>
+
+(79) ColumnarToRow [codegen id : 51]
+Input [4]: [cs_item_sk#69, cs_quantity#70, cs_list_price#71, cs_sold_date_sk#72]
+
+(80) Filter [codegen id : 51]
+Input [4]: [cs_item_sk#69, cs_quantity#70, cs_list_price#71, cs_sold_date_sk#72]
+Condition : isnotnull(cs_item_sk#69)
+
+(81) ReusedExchange [Reuses operator id: unknown]
+Output [1]: [ss_item_sk#43]
+
+(82) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [cs_item_sk#69]
+Right keys [1]: [ss_item_sk#43]
+Join condition: None
+
+(83) ReusedExchange [Reuses operator id: 63]
+Output [4]: [i_item_sk#73, i_brand_id#74, i_class_id#75, i_category_id#76]
+
+(84) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [cs_item_sk#69]
+Right keys [1]: [i_item_sk#73]
+Join condition: None
+
+(85) Project [codegen id : 51]
+Output [6]: [cs_quantity#70, cs_list_price#71, cs_sold_date_sk#72, i_brand_id#74, i_class_id#75, i_category_id#76]
+Input [8]: [cs_item_sk#69, cs_quantity#70, cs_list_price#71, cs_sold_date_sk#72, i_item_sk#73, i_brand_id#74, i_class_id#75, i_category_id#76]
+
+(86) ReusedExchange [Reuses operator id: 70]
+Output [1]: [d_date_sk#77]
+
+(87) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [cs_sold_date_sk#72]
+Right keys [1]: [d_date_sk#77]
+Join condition: None
+
+(88) Project [codegen id : 51]
+Output [5]: [cs_quantity#70, cs_list_price#71, i_brand_id#74, i_class_id#75, i_category_id#76]
+Input [7]: [cs_quantity#70, cs_list_price#71, cs_sold_date_sk#72, i_brand_id#74, i_class_id#75, i_category_id#76, d_date_sk#77]
+
+(89) HashAggregate [codegen id : 51]
+Input [5]: [cs_quantity#70, cs_list_price#71, i_brand_id#74, i_class_id#75, i_category_id#76]
+Keys [3]: [i_brand_id#74, i_class_id#75, i_category_id#76]
+Functions [2]: [partial_sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#70 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#71 as decimal(12,2)))), DecimalType(18,2), true)), partial_count(1)]
+Aggregate Attributes [3]: [sum#78, isEmpty#79, count#80]
+Results [6]: [i_brand_id#74, i_class_id#75, i_category_id#76, sum#81, isEmpty#82, count#83]
+
+(90) Exchange
+Input [6]: [i_brand_id#74, i_class_id#75, i_category_id#76, sum#81, isEmpty#82, count#83]
+Arguments: hashpartitioning(i_brand_id#74, i_class_id#75, i_category_id#76, 5), ENSURE_REQUIREMENTS, [id=#84]
+
+(91) HashAggregate [codegen id : 52]
+Input [6]: [i_brand_id#74, i_class_id#75, i_category_id#76, sum#81, isEmpty#82, count#83]
+Keys [3]: [i_brand_id#74, i_class_id#75, i_category_id#76]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#70 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#71 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#70 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#71 as decimal(12,2)))), DecimalType(18,2), true))#85, count(1)#86]
+Results [6]: [i_brand_id#74, i_class_id#75, i_category_id#76, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#70 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#71 as decimal(12,2)))), DecimalType(18,2), true))#85 AS sales#87, count(1)#86 AS number_sales#88, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#70 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#71 as decimal(12,2)))), DecimalType(18,2), true))#85 AS sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#70 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#71 as decimal(12,2)))), DecimalType(18,2), true))#89]
+
+(92) Filter [codegen id : 52]
+Input [6]: [i_brand_id#74, i_class_id#75, i_category_id#76, sales#87, number_sales#88, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#70 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#71 as decimal(12,2)))), DecimalType(18,2), true))#89]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#70 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#71 as decimal(12,2)))), DecimalType(18,2), true))#89) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#70 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#71 as decimal(12,2)))), DecimalType(18,2), true))#89 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#66, [id=#67] as decimal(32,6))))
+
+(93) Project [codegen id : 52]
+Output [6]: [sales#87, number_sales#88, catalog AS channel#90, i_brand_id#74, i_class_id#75, i_category_id#76]
+Input [6]: [i_brand_id#74, i_class_id#75, i_category_id#76, sales#87, number_sales#88, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#70 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#71 as decimal(12,2)))), DecimalType(18,2), true))#89]
+
+(94) Scan parquet default.web_sales
+Output [4]: [ws_item_sk#91, ws_quantity#92, ws_list_price#93, ws_sold_date_sk#94]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#94), dynamicpruningexpression(ws_sold_date_sk#94 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_quantity:int,ws_list_price:decimal(7,2)>
+
+(95) ColumnarToRow [codegen id : 77]
+Input [4]: [ws_item_sk#91, ws_quantity#92, ws_list_price#93, ws_sold_date_sk#94]
+
+(96) Filter [codegen id : 77]
+Input [4]: [ws_item_sk#91, ws_quantity#92, ws_list_price#93, ws_sold_date_sk#94]
+Condition : isnotnull(ws_item_sk#91)
+
+(97) ReusedExchange [Reuses operator id: unknown]
+Output [1]: [ss_item_sk#43]
+
+(98) BroadcastHashJoin [codegen id : 77]
+Left keys [1]: [ws_item_sk#91]
+Right keys [1]: [ss_item_sk#43]
+Join condition: None
+
+(99) ReusedExchange [Reuses operator id: 63]
+Output [4]: [i_item_sk#95, i_brand_id#96, i_class_id#97, i_category_id#98]
+
+(100) BroadcastHashJoin [codegen id : 77]
+Left keys [1]: [ws_item_sk#91]
+Right keys [1]: [i_item_sk#95]
+Join condition: None
+
+(101) Project [codegen id : 77]
+Output [6]: [ws_quantity#92, ws_list_price#93, ws_sold_date_sk#94, i_brand_id#96, i_class_id#97, i_category_id#98]
+Input [8]: [ws_item_sk#91, ws_quantity#92, ws_list_price#93, ws_sold_date_sk#94, i_item_sk#95, i_brand_id#96, i_class_id#97, i_category_id#98]
+
+(102) ReusedExchange [Reuses operator id: 70]
+Output [1]: [d_date_sk#99]
+
+(103) BroadcastHashJoin [codegen id : 77]
+Left keys [1]: [ws_sold_date_sk#94]
+Right keys [1]: [d_date_sk#99]
+Join condition: None
+
+(104) Project [codegen id : 77]
+Output [5]: [ws_quantity#92, ws_list_price#93, i_brand_id#96, i_class_id#97, i_category_id#98]
+Input [7]: [ws_quantity#92, ws_list_price#93, ws_sold_date_sk#94, i_brand_id#96, i_class_id#97, i_category_id#98, d_date_sk#99]
+
+(105) HashAggregate [codegen id : 77]
+Input [5]: [ws_quantity#92, ws_list_price#93, i_brand_id#96, i_class_id#97, i_category_id#98]
+Keys [3]: [i_brand_id#96, i_class_id#97, i_category_id#98]
+Functions [2]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#92 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#93 as decimal(12,2)))), DecimalType(18,2), true)), partial_count(1)]
+Aggregate Attributes [3]: [sum#100, isEmpty#101, count#102]
+Results [6]: [i_brand_id#96, i_class_id#97, i_category_id#98, sum#103, isEmpty#104, count#105]
+
+(106) Exchange
+Input [6]: [i_brand_id#96, i_class_id#97, i_category_id#98, sum#103, isEmpty#104, count#105]
+Arguments: hashpartitioning(i_brand_id#96, i_class_id#97, i_category_id#98, 5), ENSURE_REQUIREMENTS, [id=#106]
+
+(107) HashAggregate [codegen id : 78]
+Input [6]: [i_brand_id#96, i_class_id#97, i_category_id#98, sum#103, isEmpty#104, count#105]
+Keys [3]: [i_brand_id#96, i_class_id#97, i_category_id#98]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#92 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#93 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#92 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#93 as decimal(12,2)))), DecimalType(18,2), true))#107, count(1)#108]
+Results [6]: [i_brand_id#96, i_class_id#97, i_category_id#98, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#92 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#93 as decimal(12,2)))), DecimalType(18,2), true))#107 AS sales#109, count(1)#108 AS number_sales#110, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#92 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#93 as decimal(12,2)))), DecimalType(18,2), true))#107 AS sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#92 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#93 as decimal(12,2)))), DecimalType(18,2), true))#111]
+
+(108) Filter [codegen id : 78]
+Input [6]: [i_brand_id#96, i_class_id#97, i_category_id#98, sales#109, number_sales#110, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#92 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#93 as decimal(12,2)))), DecimalType(18,2), true))#111]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#92 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#93 as decimal(12,2)))), DecimalType(18,2), true))#111) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#92 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#93 as decimal(12,2)))), DecimalType(18,2), true))#111 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#66, [id=#67] as decimal(32,6))))
+
+(109) Project [codegen id : 78]
+Output [6]: [sales#109, number_sales#110, web AS channel#112, i_brand_id#96, i_class_id#97, i_category_id#98]
+Input [6]: [i_brand_id#96, i_class_id#97, i_category_id#98, sales#109, number_sales#110, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#92 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#93 as decimal(12,2)))), DecimalType(18,2), true))#111]
+
+(110) Union
+
+(111) Expand [codegen id : 79]
+Input [6]: [sales#63, number_sales#64, channel#68, i_brand_id#46, i_class_id#47, i_category_id#48]
+Arguments: [ArrayBuffer(sales#63, number_sales#64, channel#68, i_brand_id#46, i_class_id#47, i_category_id#48, 0), ArrayBuffer(sales#63, number_sales#64, channel#68, i_brand_id#46, i_class_id#47, null, 1), ArrayBuffer(sales#63, number_sales#64, channel#68, i_brand_id#46, null, null, 3), ArrayBuffer(sales#63, number_sales#64, channel#68, null, null, null, 7), ArrayBuffer(sales#63, number_sales#64, null, null, null, null, 15)], [sales#63, number_sales#64, channel#113, i_brand_id#114, i_class_id#115, i_category_id#116, spark_grouping_id#117]
+
+(112) HashAggregate [codegen id : 79]
+Input [7]: [sales#63, number_sales#64, channel#113, i_brand_id#114, i_class_id#115, i_category_id#116, spark_grouping_id#117]
+Keys [5]: [channel#113, i_brand_id#114, i_class_id#115, i_category_id#116, spark_grouping_id#117]
+Functions [2]: [partial_sum(sales#63), partial_sum(number_sales#64)]
+Aggregate Attributes [3]: [sum#118, isEmpty#119, sum#120]
+Results [8]: [channel#113, i_brand_id#114, i_class_id#115, i_category_id#116, spark_grouping_id#117, sum#121, isEmpty#122, sum#123]
+
+(113) Exchange
+Input [8]: [channel#113, i_brand_id#114, i_class_id#115, i_category_id#116, spark_grouping_id#117, sum#121, isEmpty#122, sum#123]
+Arguments: hashpartitioning(channel#113, i_brand_id#114, i_class_id#115, i_category_id#116, spark_grouping_id#117, 5), ENSURE_REQUIREMENTS, [id=#124]
+
+(114) HashAggregate [codegen id : 80]
+Input [8]: [channel#113, i_brand_id#114, i_class_id#115, i_category_id#116, spark_grouping_id#117, sum#121, isEmpty#122, sum#123]
+Keys [5]: [channel#113, i_brand_id#114, i_class_id#115, i_category_id#116, spark_grouping_id#117]
+Functions [2]: [sum(sales#63), sum(number_sales#64)]
+Aggregate Attributes [2]: [sum(sales#63)#125, sum(number_sales#64)#126]
+Results [6]: [channel#113, i_brand_id#114, i_class_id#115, i_category_id#116, sum(sales#63)#125 AS sum(sales)#127, sum(number_sales#64)#126 AS sum(number_sales)#128]
+
+(115) TakeOrderedAndProject
+Input [6]: [channel#113, i_brand_id#114, i_class_id#115, i_category_id#116, sum(sales)#127, sum(number_sales)#128]
+Arguments: 100, [channel#113 ASC NULLS FIRST, i_brand_id#114 ASC NULLS FIRST, i_class_id#115 ASC NULLS FIRST, i_category_id#116 ASC NULLS FIRST], [channel#113, i_brand_id#114, i_class_id#115, i_category_id#116, sum(sales)#127, sum(number_sales)#128]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 76 Hosting Expression = Subquery scalar-subquery#66, [id=#67]
+* HashAggregate (138)
++- Exchange (137)
+   +- * HashAggregate (136)
+      +- Union (135)
+         :- * Project (124)
+         :  +- * BroadcastHashJoin Inner BuildRight (123)
+         :     :- * ColumnarToRow (117)
+         :     :  +- Scan parquet default.store_sales (116)
+         :     +- BroadcastExchange (122)
+         :        +- * Project (121)
+         :           +- * Filter (120)
+         :              +- * ColumnarToRow (119)
+         :                 +- Scan parquet default.date_dim (118)
+         :- * Project (129)
+         :  +- * BroadcastHashJoin Inner BuildRight (128)
+         :     :- * ColumnarToRow (126)
+         :     :  +- Scan parquet default.catalog_sales (125)
+         :     +- ReusedExchange (127)
+         +- * Project (134)
+            +- * BroadcastHashJoin Inner BuildRight (133)
+               :- * ColumnarToRow (131)
+               :  +- Scan parquet default.web_sales (130)
+               +- ReusedExchange (132)
+
+
+(116) Scan parquet default.store_sales
+Output [3]: [ss_quantity#129, ss_list_price#130, ss_sold_date_sk#131]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#131), dynamicpruningexpression(ss_sold_date_sk#131 IN dynamicpruning#132)]
+ReadSchema: struct<ss_quantity:int,ss_list_price:decimal(7,2)>
+
+(117) ColumnarToRow [codegen id : 2]
+Input [3]: [ss_quantity#129, ss_list_price#130, ss_sold_date_sk#131]
+
+(118) Scan parquet default.date_dim
+Output [2]: [d_date_sk#133, d_year#134]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(119) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#133, d_year#134]
+
+(120) Filter [codegen id : 1]
+Input [2]: [d_date_sk#133, d_year#134]
+Condition : (((isnotnull(d_year#134) AND (d_year#134 >= 1999)) AND (d_year#134 <= 2001)) AND isnotnull(d_date_sk#133))
+
+(121) Project [codegen id : 1]
+Output [1]: [d_date_sk#133]
+Input [2]: [d_date_sk#133, d_year#134]
+
+(122) BroadcastExchange
+Input [1]: [d_date_sk#133]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#135]
+
+(123) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#131]
+Right keys [1]: [d_date_sk#133]
+Join condition: None
+
+(124) Project [codegen id : 2]
+Output [2]: [ss_quantity#129 AS quantity#136, ss_list_price#130 AS list_price#137]
+Input [4]: [ss_quantity#129, ss_list_price#130, ss_sold_date_sk#131, d_date_sk#133]
+
+(125) Scan parquet default.catalog_sales
+Output [3]: [cs_quantity#138, cs_list_price#139, cs_sold_date_sk#140]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#140), dynamicpruningexpression(cs_sold_date_sk#140 IN dynamicpruning#132)]
+ReadSchema: struct<cs_quantity:int,cs_list_price:decimal(7,2)>
+
+(126) ColumnarToRow [codegen id : 4]
+Input [3]: [cs_quantity#138, cs_list_price#139, cs_sold_date_sk#140]
+
+(127) ReusedExchange [Reuses operator id: 122]
+Output [1]: [d_date_sk#141]
+
+(128) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_sold_date_sk#140]
+Right keys [1]: [d_date_sk#141]
+Join condition: None
+
+(129) Project [codegen id : 4]
+Output [2]: [cs_quantity#138 AS quantity#142, cs_list_price#139 AS list_price#143]
+Input [4]: [cs_quantity#138, cs_list_price#139, cs_sold_date_sk#140, d_date_sk#141]
+
+(130) Scan parquet default.web_sales
+Output [3]: [ws_quantity#144, ws_list_price#145, ws_sold_date_sk#146]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#146), dynamicpruningexpression(ws_sold_date_sk#146 IN dynamicpruning#132)]
+ReadSchema: struct<ws_quantity:int,ws_list_price:decimal(7,2)>
+
+(131) ColumnarToRow [codegen id : 6]
+Input [3]: [ws_quantity#144, ws_list_price#145, ws_sold_date_sk#146]
+
+(132) ReusedExchange [Reuses operator id: 122]
+Output [1]: [d_date_sk#147]
+
+(133) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ws_sold_date_sk#146]
+Right keys [1]: [d_date_sk#147]
+Join condition: None
+
+(134) Project [codegen id : 6]
+Output [2]: [ws_quantity#144 AS quantity#148, ws_list_price#145 AS list_price#149]
+Input [4]: [ws_quantity#144, ws_list_price#145, ws_sold_date_sk#146, d_date_sk#147]
+
+(135) Union
+
+(136) HashAggregate [codegen id : 7]
+Input [2]: [quantity#136, list_price#137]
+Keys: []
+Functions [1]: [partial_avg(CheckOverflow((promote_precision(cast(cast(quantity#136 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#137 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [2]: [sum#150, count#151]
+Results [2]: [sum#152, count#153]
+
+(137) Exchange
+Input [2]: [sum#152, count#153]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#154]
+
+(138) HashAggregate [codegen id : 8]
+Input [2]: [sum#152, count#153]
+Keys: []
+Functions [1]: [avg(CheckOverflow((promote_precision(cast(cast(quantity#136 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#137 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [avg(CheckOverflow((promote_precision(cast(cast(quantity#136 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#137 as decimal(12,2)))), DecimalType(18,2), true))#155]
+Results [1]: [avg(CheckOverflow((promote_precision(cast(cast(quantity#136 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#137 as decimal(12,2)))), DecimalType(18,2), true))#155 AS average_sales#156]
+
+Subquery:2 Hosting operator id = 116 Hosting Expression = ss_sold_date_sk#131 IN dynamicpruning#132
+ReusedExchange (139)
+
+
+(139) ReusedExchange [Reuses operator id: 122]
+Output [1]: [d_date_sk#133]
+
+Subquery:3 Hosting operator id = 125 Hosting Expression = cs_sold_date_sk#140 IN dynamicpruning#132
+
+Subquery:4 Hosting operator id = 130 Hosting Expression = ws_sold_date_sk#146 IN dynamicpruning#132
+
+Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (140)
+
+
+(140) ReusedExchange [Reuses operator id: 70]
+Output [1]: [d_date_sk#50]
+
+Subquery:6 Hosting operator id = 7 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12
+ReusedExchange (141)
+
+
+(141) ReusedExchange [Reuses operator id: 26]
+Output [1]: [d_date_sk#29]
+
+Subquery:7 Hosting operator id = 13 Hosting Expression = cs_sold_date_sk#18 IN dynamicpruning#12
+
+Subquery:8 Hosting operator id = 40 Hosting Expression = ws_sold_date_sk#35 IN dynamicpruning#12
+
+Subquery:9 Hosting operator id = 92 Hosting Expression = ReusedSubquery Subquery scalar-subquery#66, [id=#67]
+
+Subquery:10 Hosting operator id = 78 Hosting Expression = cs_sold_date_sk#72 IN dynamicpruning#5
+
+Subquery:11 Hosting operator id = 108 Hosting Expression = ReusedSubquery Subquery scalar-subquery#66, [id=#67]
+
+Subquery:12 Hosting operator id = 94 Hosting Expression = ws_sold_date_sk#94 IN dynamicpruning#5
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/simplified.txt
new file mode 100644
index 0000000000000..9ae1a0e30e90e
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/simplified.txt
@@ -0,0 +1,223 @@
+TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales)]
+  WholeStageCodegen (80)
+    HashAggregate [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum(sales),sum(number_sales),sum,isEmpty,sum]
+      InputAdapter
+        Exchange [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] #1
+          WholeStageCodegen (79)
+            HashAggregate [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum]
+              Expand [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id]
+                InputAdapter
+                  Union
+                    WholeStageCodegen (26)
+                      Project [sales,number_sales,i_brand_id,i_class_id,i_category_id]
+                        Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                          Subquery #3
+                            WholeStageCodegen (8)
+                              HashAggregate [sum,count] [avg(CheckOverflow((promote_precision(cast(cast(quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price as decimal(12,2)))), DecimalType(18,2), true)),average_sales,sum,count]
+                                InputAdapter
+                                  Exchange #14
+                                    WholeStageCodegen (7)
+                                      HashAggregate [quantity,list_price] [sum,count,sum,count]
+                                        InputAdapter
+                                          Union
+                                            WholeStageCodegen (2)
+                                              Project [ss_quantity,ss_list_price]
+                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk]
+                                                        SubqueryBroadcast [d_date_sk] #4
+                                                          ReusedExchange [d_date_sk] #15
+                                                  InputAdapter
+                                                    BroadcastExchange #15
+                                                      WholeStageCodegen (1)
+                                                        Project [d_date_sk]
+                                                          Filter [d_year,d_date_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.date_dim [d_date_sk,d_year]
+                                            WholeStageCodegen (4)
+                                              Project [cs_quantity,cs_list_price]
+                                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk]
+                                                        ReusedSubquery [d_date_sk] #4
+                                                  InputAdapter
+                                                    ReusedExchange [d_date_sk] #15
+                                            WholeStageCodegen (6)
+                                              Project [ws_quantity,ws_list_price]
+                                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk]
+                                                        ReusedSubquery [d_date_sk] #4
+                                                  InputAdapter
+                                                    ReusedExchange [d_date_sk] #15
+                          HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                            InputAdapter
+                              Exchange [i_brand_id,i_class_id,i_category_id] #2
+                                WholeStageCodegen (25)
+                                  HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count]
+                                    Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id]
+                                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                        Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id]
+                                          BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                            BroadcastHashJoin [ss_item_sk,ss_item_sk]
+                                              Filter [ss_item_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk]
+                                                      SubqueryBroadcast [d_date_sk] #1
+                                                        ReusedExchange [d_date_sk] #3
+                                              InputAdapter
+                                                BroadcastExchange #4
+                                                  WholeStageCodegen (11)
+                                                    Project [i_item_sk]
+                                                      BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id]
+                                                        Filter [i_brand_id,i_class_id,i_category_id]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                        InputAdapter
+                                                          BroadcastExchange #5
+                                                            WholeStageCodegen (10)
+                                                              HashAggregate [brand_id,class_id,category_id]
+                                                                HashAggregate [brand_id,class_id,category_id]
+                                                                  BroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id]
+                                                                    HashAggregate [brand_id,class_id,category_id]
+                                                                      InputAdapter
+                                                                        Exchange [brand_id,class_id,category_id] #6
+                                                                          WholeStageCodegen (6)
+                                                                            HashAggregate [brand_id,class_id,category_id]
+                                                                              Project [i_brand_id,i_class_id,i_category_id]
+                                                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                  Project [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                                                      Filter [ss_item_sk]
+                                                                                        ColumnarToRow
+                                                                                          InputAdapter
+                                                                                            Scan parquet default.store_sales [ss_item_sk,ss_sold_date_sk]
+                                                                                              SubqueryBroadcast [d_date_sk] #2
+                                                                                                ReusedExchange [d_date_sk] #7
+                                                                                      InputAdapter
+                                                                                        BroadcastExchange #8
+                                                                                          WholeStageCodegen (4)
+                                                                                            BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id]
+                                                                                              Filter [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                                ColumnarToRow
+                                                                                                  InputAdapter
+                                                                                                    Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                              InputAdapter
+                                                                                                BroadcastExchange #9
+                                                                                                  WholeStageCodegen (3)
+                                                                                                    Project [i_brand_id,i_class_id,i_category_id]
+                                                                                                      BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                                                        Project [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                                          BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                                                                                            Filter [cs_item_sk]
+                                                                                                              ColumnarToRow
+                                                                                                                InputAdapter
+                                                                                                                  Scan parquet default.catalog_sales [cs_item_sk,cs_sold_date_sk]
+                                                                                                                    ReusedSubquery [d_date_sk] #2
+                                                                                                            InputAdapter
+                                                                                                              BroadcastExchange #10
+                                                                                                                WholeStageCodegen (1)
+                                                                                                                  Filter [i_item_sk]
+                                                                                                                    ColumnarToRow
+                                                                                                                      InputAdapter
+                                                                                                                        Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                                        InputAdapter
+                                                                                                          BroadcastExchange #7
+                                                                                                            WholeStageCodegen (2)
+                                                                                                              Project [d_date_sk]
+                                                                                                                Filter [d_year,d_date_sk]
+                                                                                                                  ColumnarToRow
+                                                                                                                    InputAdapter
+                                                                                                                      Scan parquet default.date_dim [d_date_sk,d_year]
+                                                                                  InputAdapter
+                                                                                    ReusedExchange [d_date_sk] #7
+                                                                    InputAdapter
+                                                                      BroadcastExchange #11
+                                                                        WholeStageCodegen (9)
+                                                                          Project [i_brand_id,i_class_id,i_category_id]
+                                                                            BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                              Project [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                                                                  Filter [ws_item_sk]
+                                                                                    ColumnarToRow
+                                                                                      InputAdapter
+                                                                                        Scan parquet default.web_sales [ws_item_sk,ws_sold_date_sk]
+                                                                                          ReusedSubquery [d_date_sk] #2
+                                                                                  InputAdapter
+                                                                                    ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10
+                                                                              InputAdapter
+                                                                                ReusedExchange [d_date_sk] #7
+                                            InputAdapter
+                                              BroadcastExchange #12
+                                                WholeStageCodegen (23)
+                                                  BroadcastHashJoin [i_item_sk,ss_item_sk]
+                                                    Filter [i_item_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                    InputAdapter
+                                                      ReusedExchange [ss_item_sk] #13
+                                        InputAdapter
+                                          BroadcastExchange #3
+                                            WholeStageCodegen (24)
+                                              Project [d_date_sk]
+                                                Filter [d_year,d_moy,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                    WholeStageCodegen (52)
+                      Project [sales,number_sales,i_brand_id,i_class_id,i_category_id]
+                        Filter [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                          ReusedSubquery [average_sales] #3
+                          HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                            InputAdapter
+                              Exchange [i_brand_id,i_class_id,i_category_id] #16
+                                WholeStageCodegen (51)
+                                  HashAggregate [i_brand_id,i_class_id,i_category_id,cs_quantity,cs_list_price] [sum,isEmpty,count,sum,isEmpty,count]
+                                    Project [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id]
+                                      BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                        Project [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id]
+                                          BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                            BroadcastHashJoin [cs_item_sk,ss_item_sk]
+                                              Filter [cs_item_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk]
+                                                      ReusedSubquery [d_date_sk] #1
+                                              InputAdapter
+                                                ReusedExchange [ss_item_sk] #13
+                                            InputAdapter
+                                              ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12
+                                        InputAdapter
+                                          ReusedExchange [d_date_sk] #3
+                    WholeStageCodegen (78)
+                      Project [sales,number_sales,i_brand_id,i_class_id,i_category_id]
+                        Filter [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                          ReusedSubquery [average_sales] #3
+                          HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                            InputAdapter
+                              Exchange [i_brand_id,i_class_id,i_category_id] #17
+                                WholeStageCodegen (77)
+                                  HashAggregate [i_brand_id,i_class_id,i_category_id,ws_quantity,ws_list_price] [sum,isEmpty,count,sum,isEmpty,count]
+                                    Project [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id]
+                                      BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                        Project [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id]
+                                          BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                            BroadcastHashJoin [ws_item_sk,ss_item_sk]
+                                              Filter [ws_item_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk]
+                                                      ReusedSubquery [d_date_sk] #1
+                                              InputAdapter
+                                                ReusedExchange [ss_item_sk] #13
+                                            InputAdapter
+                                              ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12
+                                        InputAdapter
+                                          ReusedExchange [d_date_sk] #3
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.sf100/explain.txt
new file mode 100644
index 0000000000000..d0bdd41048367
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.sf100/explain.txt
@@ -0,0 +1,869 @@
+== Physical Plan ==
+TakeOrderedAndProject (116)
++- * BroadcastHashJoin Inner BuildRight (115)
+   :- * Project (90)
+   :  +- * Filter (89)
+   :     +- * HashAggregate (88)
+   :        +- Exchange (87)
+   :           +- * HashAggregate (86)
+   :              +- * Project (85)
+   :                 +- * BroadcastHashJoin Inner BuildRight (84)
+   :                    :- * Project (74)
+   :                    :  +- * BroadcastHashJoin Inner BuildRight (73)
+   :                    :     :- * SortMergeJoin LeftSemi (67)
+   :                    :     :  :- * Sort (5)
+   :                    :     :  :  +- Exchange (4)
+   :                    :     :  :     +- * Filter (3)
+   :                    :     :  :        +- * ColumnarToRow (2)
+   :                    :     :  :           +- Scan parquet default.store_sales (1)
+   :                    :     :  +- * Sort (66)
+   :                    :     :     +- Exchange (65)
+   :                    :     :        +- * Project (64)
+   :                    :     :           +- * BroadcastHashJoin Inner BuildRight (63)
+   :                    :     :              :- * Filter (8)
+   :                    :     :              :  +- * ColumnarToRow (7)
+   :                    :     :              :     +- Scan parquet default.item (6)
+   :                    :     :              +- BroadcastExchange (62)
+   :                    :     :                 +- * HashAggregate (61)
+   :                    :     :                    +- Exchange (60)
+   :                    :     :                       +- * HashAggregate (59)
+   :                    :     :                          +- * SortMergeJoin LeftSemi (58)
+   :                    :     :                             :- * Sort (46)
+   :                    :     :                             :  +- Exchange (45)
+   :                    :     :                             :     +- * HashAggregate (44)
+   :                    :     :                             :        +- Exchange (43)
+   :                    :     :                             :           +- * HashAggregate (42)
+   :                    :     :                             :              +- * Project (41)
+   :                    :     :                             :                 +- * BroadcastHashJoin Inner BuildRight (40)
+   :                    :     :                             :                    :- * Project (18)
+   :                    :     :                             :                    :  +- * BroadcastHashJoin Inner BuildRight (17)
+   :                    :     :                             :                    :     :- * Filter (11)
+   :                    :     :                             :                    :     :  +- * ColumnarToRow (10)
+   :                    :     :                             :                    :     :     +- Scan parquet default.store_sales (9)
+   :                    :     :                             :                    :     +- BroadcastExchange (16)
+   :                    :     :                             :                    :        +- * Project (15)
+   :                    :     :                             :                    :           +- * Filter (14)
+   :                    :     :                             :                    :              +- * ColumnarToRow (13)
+   :                    :     :                             :                    :                 +- Scan parquet default.date_dim (12)
+   :                    :     :                             :                    +- BroadcastExchange (39)
+   :                    :     :                             :                       +- * SortMergeJoin LeftSemi (38)
+   :                    :     :                             :                          :- * Sort (23)
+   :                    :     :                             :                          :  +- Exchange (22)
+   :                    :     :                             :                          :     +- * Filter (21)
+   :                    :     :                             :                          :        +- * ColumnarToRow (20)
+   :                    :     :                             :                          :           +- Scan parquet default.item (19)
+   :                    :     :                             :                          +- * Sort (37)
+   :                    :     :                             :                             +- Exchange (36)
+   :                    :     :                             :                                +- * Project (35)
+   :                    :     :                             :                                   +- * BroadcastHashJoin Inner BuildRight (34)
+   :                    :     :                             :                                      :- * Project (29)
+   :                    :     :                             :                                      :  +- * BroadcastHashJoin Inner BuildRight (28)
+   :                    :     :                             :                                      :     :- * Filter (26)
+   :                    :     :                             :                                      :     :  +- * ColumnarToRow (25)
+   :                    :     :                             :                                      :     :     +- Scan parquet default.catalog_sales (24)
+   :                    :     :                             :                                      :     +- ReusedExchange (27)
+   :                    :     :                             :                                      +- BroadcastExchange (33)
+   :                    :     :                             :                                         +- * Filter (32)
+   :                    :     :                             :                                            +- * ColumnarToRow (31)
+   :                    :     :                             :                                               +- Scan parquet default.item (30)
+   :                    :     :                             +- * Sort (57)
+   :                    :     :                                +- Exchange (56)
+   :                    :     :                                   +- * Project (55)
+   :                    :     :                                      +- * BroadcastHashJoin Inner BuildRight (54)
+   :                    :     :                                         :- * Project (52)
+   :                    :     :                                         :  +- * BroadcastHashJoin Inner BuildRight (51)
+   :                    :     :                                         :     :- * Filter (49)
+   :                    :     :                                         :     :  +- * ColumnarToRow (48)
+   :                    :     :                                         :     :     +- Scan parquet default.web_sales (47)
+   :                    :     :                                         :     +- ReusedExchange (50)
+   :                    :     :                                         +- ReusedExchange (53)
+   :                    :     +- BroadcastExchange (72)
+   :                    :        +- * Project (71)
+   :                    :           +- * Filter (70)
+   :                    :              +- * ColumnarToRow (69)
+   :                    :                 +- Scan parquet default.date_dim (68)
+   :                    +- BroadcastExchange (83)
+   :                       +- * SortMergeJoin LeftSemi (82)
+   :                          :- * Sort (79)
+   :                          :  +- Exchange (78)
+   :                          :     +- * Filter (77)
+   :                          :        +- * ColumnarToRow (76)
+   :                          :           +- Scan parquet default.item (75)
+   :                          +- * Sort (81)
+   :                             +- ReusedExchange (80)
+   +- BroadcastExchange (114)
+      +- * Project (113)
+         +- * Filter (112)
+            +- * HashAggregate (111)
+               +- Exchange (110)
+                  +- * HashAggregate (109)
+                     +- * Project (108)
+                        +- * BroadcastHashJoin Inner BuildRight (107)
+                           :- * Project (105)
+                           :  +- * BroadcastHashJoin Inner BuildRight (104)
+                           :     :- * SortMergeJoin LeftSemi (98)
+                           :     :  :- * Sort (95)
+                           :     :  :  +- Exchange (94)
+                           :     :  :     +- * Filter (93)
+                           :     :  :        +- * ColumnarToRow (92)
+                           :     :  :           +- Scan parquet default.store_sales (91)
+                           :     :  +- * Sort (97)
+                           :     :     +- ReusedExchange (96)
+                           :     +- BroadcastExchange (103)
+                           :        +- * Project (102)
+                           :           +- * Filter (101)
+                           :              +- * ColumnarToRow (100)
+                           :                 +- Scan parquet default.date_dim (99)
+                           +- ReusedExchange (106)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_quantity:int,ss_list_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 1]
+Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+Condition : isnotnull(ss_item_sk#1)
+
+(4) Exchange
+Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#6]
+
+(5) Sort [codegen id : 2]
+Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+Arguments: [ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.item
+Output [4]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(7) ColumnarToRow [codegen id : 20]
+Input [4]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10]
+
+(8) Filter [codegen id : 20]
+Input [4]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10]
+Condition : ((isnotnull(i_brand_id#8) AND isnotnull(i_class_id#9)) AND isnotnull(i_category_id#10))
+
+(9) Scan parquet default.store_sales
+Output [2]: [ss_item_sk#11, ss_sold_date_sk#12]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_sold_date_sk#12 IN dynamicpruning#13)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int>
+
+(10) ColumnarToRow [codegen id : 11]
+Input [2]: [ss_item_sk#11, ss_sold_date_sk#12]
+
+(11) Filter [codegen id : 11]
+Input [2]: [ss_item_sk#11, ss_sold_date_sk#12]
+Condition : isnotnull(ss_item_sk#11)
+
+(12) Scan parquet default.date_dim
+Output [2]: [d_date_sk#14, d_year#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(13) ColumnarToRow [codegen id : 3]
+Input [2]: [d_date_sk#14, d_year#15]
+
+(14) Filter [codegen id : 3]
+Input [2]: [d_date_sk#14, d_year#15]
+Condition : (((isnotnull(d_year#15) AND (d_year#15 >= 1999)) AND (d_year#15 <= 2001)) AND isnotnull(d_date_sk#14))
+
+(15) Project [codegen id : 3]
+Output [1]: [d_date_sk#14]
+Input [2]: [d_date_sk#14, d_year#15]
+
+(16) BroadcastExchange
+Input [1]: [d_date_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#16]
+
+(17) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ss_sold_date_sk#12]
+Right keys [1]: [d_date_sk#14]
+Join condition: None
+
+(18) Project [codegen id : 11]
+Output [1]: [ss_item_sk#11]
+Input [3]: [ss_item_sk#11, ss_sold_date_sk#12, d_date_sk#14]
+
+(19) Scan parquet default.item
+Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(20) ColumnarToRow [codegen id : 4]
+Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+
+(21) Filter [codegen id : 4]
+Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+Condition : (((isnotnull(i_item_sk#17) AND isnotnull(i_brand_id#18)) AND isnotnull(i_class_id#19)) AND isnotnull(i_category_id#20))
+
+(22) Exchange
+Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+Arguments: hashpartitioning(coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20), 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(23) Sort [codegen id : 5]
+Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+Arguments: [coalesce(i_brand_id#18, 0) ASC NULLS FIRST, isnull(i_brand_id#18) ASC NULLS FIRST, coalesce(i_class_id#19, 0) ASC NULLS FIRST, isnull(i_class_id#19) ASC NULLS FIRST, coalesce(i_category_id#20, 0) ASC NULLS FIRST, isnull(i_category_id#20) ASC NULLS FIRST], false, 0
+
+(24) Scan parquet default.catalog_sales
+Output [2]: [cs_item_sk#22, cs_sold_date_sk#23]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#23), dynamicpruningexpression(cs_sold_date_sk#23 IN dynamicpruning#13)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int>
+
+(25) ColumnarToRow [codegen id : 8]
+Input [2]: [cs_item_sk#22, cs_sold_date_sk#23]
+
+(26) Filter [codegen id : 8]
+Input [2]: [cs_item_sk#22, cs_sold_date_sk#23]
+Condition : isnotnull(cs_item_sk#22)
+
+(27) ReusedExchange [Reuses operator id: 16]
+Output [1]: [d_date_sk#24]
+
+(28) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [cs_sold_date_sk#23]
+Right keys [1]: [d_date_sk#24]
+Join condition: None
+
+(29) Project [codegen id : 8]
+Output [1]: [cs_item_sk#22]
+Input [3]: [cs_item_sk#22, cs_sold_date_sk#23, d_date_sk#24]
+
+(30) Scan parquet default.item
+Output [4]: [i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(31) ColumnarToRow [codegen id : 7]
+Input [4]: [i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28]
+
+(32) Filter [codegen id : 7]
+Input [4]: [i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28]
+Condition : isnotnull(i_item_sk#25)
+
+(33) BroadcastExchange
+Input [4]: [i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#29]
+
+(34) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [cs_item_sk#22]
+Right keys [1]: [i_item_sk#25]
+Join condition: None
+
+(35) Project [codegen id : 8]
+Output [3]: [i_brand_id#26, i_class_id#27, i_category_id#28]
+Input [5]: [cs_item_sk#22, i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28]
+
+(36) Exchange
+Input [3]: [i_brand_id#26, i_class_id#27, i_category_id#28]
+Arguments: hashpartitioning(coalesce(i_brand_id#26, 0), isnull(i_brand_id#26), coalesce(i_class_id#27, 0), isnull(i_class_id#27), coalesce(i_category_id#28, 0), isnull(i_category_id#28), 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(37) Sort [codegen id : 9]
+Input [3]: [i_brand_id#26, i_class_id#27, i_category_id#28]
+Arguments: [coalesce(i_brand_id#26, 0) ASC NULLS FIRST, isnull(i_brand_id#26) ASC NULLS FIRST, coalesce(i_class_id#27, 0) ASC NULLS FIRST, isnull(i_class_id#27) ASC NULLS FIRST, coalesce(i_category_id#28, 0) ASC NULLS FIRST, isnull(i_category_id#28) ASC NULLS FIRST], false, 0
+
+(38) SortMergeJoin [codegen id : 10]
+Left keys [6]: [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)]
+Right keys [6]: [coalesce(i_brand_id#26, 0), isnull(i_brand_id#26), coalesce(i_class_id#27, 0), isnull(i_class_id#27), coalesce(i_category_id#28, 0), isnull(i_category_id#28)]
+Join condition: None
+
+(39) BroadcastExchange
+Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#31]
+
+(40) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ss_item_sk#11]
+Right keys [1]: [i_item_sk#17]
+Join condition: None
+
+(41) Project [codegen id : 11]
+Output [3]: [i_brand_id#18 AS brand_id#32, i_class_id#19 AS class_id#33, i_category_id#20 AS category_id#34]
+Input [5]: [ss_item_sk#11, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+
+(42) HashAggregate [codegen id : 11]
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Keys [3]: [brand_id#32, class_id#33, category_id#34]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#32, class_id#33, category_id#34]
+
+(43) Exchange
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Arguments: hashpartitioning(brand_id#32, class_id#33, category_id#34, 5), ENSURE_REQUIREMENTS, [id=#35]
+
+(44) HashAggregate [codegen id : 12]
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Keys [3]: [brand_id#32, class_id#33, category_id#34]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#32, class_id#33, category_id#34]
+
+(45) Exchange
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Arguments: hashpartitioning(coalesce(brand_id#32, 0), isnull(brand_id#32), coalesce(class_id#33, 0), isnull(class_id#33), coalesce(category_id#34, 0), isnull(category_id#34), 5), ENSURE_REQUIREMENTS, [id=#36]
+
+(46) Sort [codegen id : 13]
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Arguments: [coalesce(brand_id#32, 0) ASC NULLS FIRST, isnull(brand_id#32) ASC NULLS FIRST, coalesce(class_id#33, 0) ASC NULLS FIRST, isnull(class_id#33) ASC NULLS FIRST, coalesce(category_id#34, 0) ASC NULLS FIRST, isnull(category_id#34) ASC NULLS FIRST], false, 0
+
+(47) Scan parquet default.web_sales
+Output [2]: [ws_item_sk#37, ws_sold_date_sk#38]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#38), dynamicpruningexpression(ws_sold_date_sk#38 IN dynamicpruning#13)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int>
+
+(48) ColumnarToRow [codegen id : 16]
+Input [2]: [ws_item_sk#37, ws_sold_date_sk#38]
+
+(49) Filter [codegen id : 16]
+Input [2]: [ws_item_sk#37, ws_sold_date_sk#38]
+Condition : isnotnull(ws_item_sk#37)
+
+(50) ReusedExchange [Reuses operator id: 16]
+Output [1]: [d_date_sk#39]
+
+(51) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [ws_sold_date_sk#38]
+Right keys [1]: [d_date_sk#39]
+Join condition: None
+
+(52) Project [codegen id : 16]
+Output [1]: [ws_item_sk#37]
+Input [3]: [ws_item_sk#37, ws_sold_date_sk#38, d_date_sk#39]
+
+(53) ReusedExchange [Reuses operator id: 33]
+Output [4]: [i_item_sk#40, i_brand_id#41, i_class_id#42, i_category_id#43]
+
+(54) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [ws_item_sk#37]
+Right keys [1]: [i_item_sk#40]
+Join condition: None
+
+(55) Project [codegen id : 16]
+Output [3]: [i_brand_id#41, i_class_id#42, i_category_id#43]
+Input [5]: [ws_item_sk#37, i_item_sk#40, i_brand_id#41, i_class_id#42, i_category_id#43]
+
+(56) Exchange
+Input [3]: [i_brand_id#41, i_class_id#42, i_category_id#43]
+Arguments: hashpartitioning(coalesce(i_brand_id#41, 0), isnull(i_brand_id#41), coalesce(i_class_id#42, 0), isnull(i_class_id#42), coalesce(i_category_id#43, 0), isnull(i_category_id#43), 5), ENSURE_REQUIREMENTS, [id=#44]
+
+(57) Sort [codegen id : 17]
+Input [3]: [i_brand_id#41, i_class_id#42, i_category_id#43]
+Arguments: [coalesce(i_brand_id#41, 0) ASC NULLS FIRST, isnull(i_brand_id#41) ASC NULLS FIRST, coalesce(i_class_id#42, 0) ASC NULLS FIRST, isnull(i_class_id#42) ASC NULLS FIRST, coalesce(i_category_id#43, 0) ASC NULLS FIRST, isnull(i_category_id#43) ASC NULLS FIRST], false, 0
+
+(58) SortMergeJoin [codegen id : 18]
+Left keys [6]: [coalesce(brand_id#32, 0), isnull(brand_id#32), coalesce(class_id#33, 0), isnull(class_id#33), coalesce(category_id#34, 0), isnull(category_id#34)]
+Right keys [6]: [coalesce(i_brand_id#41, 0), isnull(i_brand_id#41), coalesce(i_class_id#42, 0), isnull(i_class_id#42), coalesce(i_category_id#43, 0), isnull(i_category_id#43)]
+Join condition: None
+
+(59) HashAggregate [codegen id : 18]
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Keys [3]: [brand_id#32, class_id#33, category_id#34]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#32, class_id#33, category_id#34]
+
+(60) Exchange
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Arguments: hashpartitioning(brand_id#32, class_id#33, category_id#34, 5), ENSURE_REQUIREMENTS, [id=#45]
+
+(61) HashAggregate [codegen id : 19]
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Keys [3]: [brand_id#32, class_id#33, category_id#34]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#32, class_id#33, category_id#34]
+
+(62) BroadcastExchange
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Arguments: HashedRelationBroadcastMode(List(input[0, int, true], input[1, int, true], input[2, int, true]),false), [id=#46]
+
+(63) BroadcastHashJoin [codegen id : 20]
+Left keys [3]: [i_brand_id#8, i_class_id#9, i_category_id#10]
+Right keys [3]: [brand_id#32, class_id#33, category_id#34]
+Join condition: None
+
+(64) Project [codegen id : 20]
+Output [1]: [i_item_sk#7 AS ss_item_sk#47]
+Input [7]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, brand_id#32, class_id#33, category_id#34]
+
+(65) Exchange
+Input [1]: [ss_item_sk#47]
+Arguments: hashpartitioning(ss_item_sk#47, 5), ENSURE_REQUIREMENTS, [id=#48]
+
+(66) Sort [codegen id : 21]
+Input [1]: [ss_item_sk#47]
+Arguments: [ss_item_sk#47 ASC NULLS FIRST], false, 0
+
+(67) SortMergeJoin [codegen id : 45]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [ss_item_sk#47]
+Join condition: None
+
+(68) Scan parquet default.date_dim
+Output [2]: [d_date_sk#49, d_week_seq#50]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_week_seq:int>
+
+(69) ColumnarToRow [codegen id : 22]
+Input [2]: [d_date_sk#49, d_week_seq#50]
+
+(70) Filter [codegen id : 22]
+Input [2]: [d_date_sk#49, d_week_seq#50]
+Condition : ((isnotnull(d_week_seq#50) AND (d_week_seq#50 = Subquery scalar-subquery#51, [id=#52])) AND isnotnull(d_date_sk#49))
+
+(71) Project [codegen id : 22]
+Output [1]: [d_date_sk#49]
+Input [2]: [d_date_sk#49, d_week_seq#50]
+
+(72) BroadcastExchange
+Input [1]: [d_date_sk#49]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#53]
+
+(73) BroadcastHashJoin [codegen id : 45]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#49]
+Join condition: None
+
+(74) Project [codegen id : 45]
+Output [3]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3]
+Input [5]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, d_date_sk#49]
+
+(75) Scan parquet default.item
+Output [4]: [i_item_sk#54, i_brand_id#55, i_class_id#56, i_category_id#57]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(76) ColumnarToRow [codegen id : 23]
+Input [4]: [i_item_sk#54, i_brand_id#55, i_class_id#56, i_category_id#57]
+
+(77) Filter [codegen id : 23]
+Input [4]: [i_item_sk#54, i_brand_id#55, i_class_id#56, i_category_id#57]
+Condition : (((isnotnull(i_item_sk#54) AND isnotnull(i_brand_id#55)) AND isnotnull(i_class_id#56)) AND isnotnull(i_category_id#57))
+
+(78) Exchange
+Input [4]: [i_item_sk#54, i_brand_id#55, i_class_id#56, i_category_id#57]
+Arguments: hashpartitioning(i_item_sk#54, 5), ENSURE_REQUIREMENTS, [id=#58]
+
+(79) Sort [codegen id : 24]
+Input [4]: [i_item_sk#54, i_brand_id#55, i_class_id#56, i_category_id#57]
+Arguments: [i_item_sk#54 ASC NULLS FIRST], false, 0
+
+(80) ReusedExchange [Reuses operator id: unknown]
+Output [1]: [ss_item_sk#47]
+
+(81) Sort [codegen id : 43]
+Input [1]: [ss_item_sk#47]
+Arguments: [ss_item_sk#47 ASC NULLS FIRST], false, 0
+
+(82) SortMergeJoin [codegen id : 44]
+Left keys [1]: [i_item_sk#54]
+Right keys [1]: [ss_item_sk#47]
+Join condition: None
+
+(83) BroadcastExchange
+Input [4]: [i_item_sk#54, i_brand_id#55, i_class_id#56, i_category_id#57]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#59]
+
+(84) BroadcastHashJoin [codegen id : 45]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#54]
+Join condition: None
+
+(85) Project [codegen id : 45]
+Output [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#55, i_class_id#56, i_category_id#57]
+Input [7]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, i_item_sk#54, i_brand_id#55, i_class_id#56, i_category_id#57]
+
+(86) HashAggregate [codegen id : 45]
+Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#55, i_class_id#56, i_category_id#57]
+Keys [3]: [i_brand_id#55, i_class_id#56, i_category_id#57]
+Functions [2]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true)), partial_count(1)]
+Aggregate Attributes [3]: [sum#60, isEmpty#61, count#62]
+Results [6]: [i_brand_id#55, i_class_id#56, i_category_id#57, sum#63, isEmpty#64, count#65]
+
+(87) Exchange
+Input [6]: [i_brand_id#55, i_class_id#56, i_category_id#57, sum#63, isEmpty#64, count#65]
+Arguments: hashpartitioning(i_brand_id#55, i_class_id#56, i_category_id#57, 5), ENSURE_REQUIREMENTS, [id=#66]
+
+(88) HashAggregate [codegen id : 92]
+Input [6]: [i_brand_id#55, i_class_id#56, i_category_id#57, sum#63, isEmpty#64, count#65]
+Keys [3]: [i_brand_id#55, i_class_id#56, i_category_id#57]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#67, count(1)#68]
+Results [6]: [i_brand_id#55, i_class_id#56, i_category_id#57, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#67 AS sales#69, count(1)#68 AS number_sales#70, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#67 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#71]
+
+(89) Filter [codegen id : 92]
+Input [6]: [i_brand_id#55, i_class_id#56, i_category_id#57, sales#69, number_sales#70, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#71]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#71) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#71 as decimal(32,6)) > cast(Subquery scalar-subquery#72, [id=#73] as decimal(32,6))))
+
+(90) Project [codegen id : 92]
+Output [6]: [store AS channel#74, i_brand_id#55, i_class_id#56, i_category_id#57, sales#69, number_sales#70]
+Input [6]: [i_brand_id#55, i_class_id#56, i_category_id#57, sales#69, number_sales#70, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#71]
+
+(91) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#75, ss_quantity#76, ss_list_price#77, ss_sold_date_sk#78]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#78), dynamicpruningexpression(ss_sold_date_sk#78 IN dynamicpruning#79)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_quantity:int,ss_list_price:decimal(7,2)>
+
+(92) ColumnarToRow [codegen id : 46]
+Input [4]: [ss_item_sk#75, ss_quantity#76, ss_list_price#77, ss_sold_date_sk#78]
+
+(93) Filter [codegen id : 46]
+Input [4]: [ss_item_sk#75, ss_quantity#76, ss_list_price#77, ss_sold_date_sk#78]
+Condition : isnotnull(ss_item_sk#75)
+
+(94) Exchange
+Input [4]: [ss_item_sk#75, ss_quantity#76, ss_list_price#77, ss_sold_date_sk#78]
+Arguments: hashpartitioning(ss_item_sk#75, 5), ENSURE_REQUIREMENTS, [id=#80]
+
+(95) Sort [codegen id : 47]
+Input [4]: [ss_item_sk#75, ss_quantity#76, ss_list_price#77, ss_sold_date_sk#78]
+Arguments: [ss_item_sk#75 ASC NULLS FIRST], false, 0
+
+(96) ReusedExchange [Reuses operator id: unknown]
+Output [1]: [ss_item_sk#47]
+
+(97) Sort [codegen id : 66]
+Input [1]: [ss_item_sk#47]
+Arguments: [ss_item_sk#47 ASC NULLS FIRST], false, 0
+
+(98) SortMergeJoin [codegen id : 90]
+Left keys [1]: [ss_item_sk#75]
+Right keys [1]: [ss_item_sk#47]
+Join condition: None
+
+(99) Scan parquet default.date_dim
+Output [2]: [d_date_sk#81, d_week_seq#82]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_week_seq:int>
+
+(100) ColumnarToRow [codegen id : 67]
+Input [2]: [d_date_sk#81, d_week_seq#82]
+
+(101) Filter [codegen id : 67]
+Input [2]: [d_date_sk#81, d_week_seq#82]
+Condition : ((isnotnull(d_week_seq#82) AND (d_week_seq#82 = Subquery scalar-subquery#83, [id=#84])) AND isnotnull(d_date_sk#81))
+
+(102) Project [codegen id : 67]
+Output [1]: [d_date_sk#81]
+Input [2]: [d_date_sk#81, d_week_seq#82]
+
+(103) BroadcastExchange
+Input [1]: [d_date_sk#81]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#85]
+
+(104) BroadcastHashJoin [codegen id : 90]
+Left keys [1]: [ss_sold_date_sk#78]
+Right keys [1]: [d_date_sk#81]
+Join condition: None
+
+(105) Project [codegen id : 90]
+Output [3]: [ss_item_sk#75, ss_quantity#76, ss_list_price#77]
+Input [5]: [ss_item_sk#75, ss_quantity#76, ss_list_price#77, ss_sold_date_sk#78, d_date_sk#81]
+
+(106) ReusedExchange [Reuses operator id: 83]
+Output [4]: [i_item_sk#86, i_brand_id#87, i_class_id#88, i_category_id#89]
+
+(107) BroadcastHashJoin [codegen id : 90]
+Left keys [1]: [ss_item_sk#75]
+Right keys [1]: [i_item_sk#86]
+Join condition: None
+
+(108) Project [codegen id : 90]
+Output [5]: [ss_quantity#76, ss_list_price#77, i_brand_id#87, i_class_id#88, i_category_id#89]
+Input [7]: [ss_item_sk#75, ss_quantity#76, ss_list_price#77, i_item_sk#86, i_brand_id#87, i_class_id#88, i_category_id#89]
+
+(109) HashAggregate [codegen id : 90]
+Input [5]: [ss_quantity#76, ss_list_price#77, i_brand_id#87, i_class_id#88, i_category_id#89]
+Keys [3]: [i_brand_id#87, i_class_id#88, i_category_id#89]
+Functions [2]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#76 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#77 as decimal(12,2)))), DecimalType(18,2), true)), partial_count(1)]
+Aggregate Attributes [3]: [sum#90, isEmpty#91, count#92]
+Results [6]: [i_brand_id#87, i_class_id#88, i_category_id#89, sum#93, isEmpty#94, count#95]
+
+(110) Exchange
+Input [6]: [i_brand_id#87, i_class_id#88, i_category_id#89, sum#93, isEmpty#94, count#95]
+Arguments: hashpartitioning(i_brand_id#87, i_class_id#88, i_category_id#89, 5), ENSURE_REQUIREMENTS, [id=#96]
+
+(111) HashAggregate [codegen id : 91]
+Input [6]: [i_brand_id#87, i_class_id#88, i_category_id#89, sum#93, isEmpty#94, count#95]
+Keys [3]: [i_brand_id#87, i_class_id#88, i_category_id#89]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#76 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#77 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#76 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#77 as decimal(12,2)))), DecimalType(18,2), true))#97, count(1)#98]
+Results [6]: [i_brand_id#87, i_class_id#88, i_category_id#89, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#76 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#77 as decimal(12,2)))), DecimalType(18,2), true))#97 AS sales#99, count(1)#98 AS number_sales#100, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#76 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#77 as decimal(12,2)))), DecimalType(18,2), true))#97 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#76 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#77 as decimal(12,2)))), DecimalType(18,2), true))#101]
+
+(112) Filter [codegen id : 91]
+Input [6]: [i_brand_id#87, i_class_id#88, i_category_id#89, sales#99, number_sales#100, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#76 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#77 as decimal(12,2)))), DecimalType(18,2), true))#101]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#76 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#77 as decimal(12,2)))), DecimalType(18,2), true))#101) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#76 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#77 as decimal(12,2)))), DecimalType(18,2), true))#101 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#72, [id=#73] as decimal(32,6))))
+
+(113) Project [codegen id : 91]
+Output [6]: [store AS channel#102, i_brand_id#87, i_class_id#88, i_category_id#89, sales#99, number_sales#100]
+Input [6]: [i_brand_id#87, i_class_id#88, i_category_id#89, sales#99, number_sales#100, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#76 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#77 as decimal(12,2)))), DecimalType(18,2), true))#101]
+
+(114) BroadcastExchange
+Input [6]: [channel#102, i_brand_id#87, i_class_id#88, i_category_id#89, sales#99, number_sales#100]
+Arguments: HashedRelationBroadcastMode(List(input[1, int, true], input[2, int, true], input[3, int, true]),false), [id=#103]
+
+(115) BroadcastHashJoin [codegen id : 92]
+Left keys [3]: [i_brand_id#55, i_class_id#56, i_category_id#57]
+Right keys [3]: [i_brand_id#87, i_class_id#88, i_category_id#89]
+Join condition: None
+
+(116) TakeOrderedAndProject
+Input [12]: [channel#74, i_brand_id#55, i_class_id#56, i_category_id#57, sales#69, number_sales#70, channel#102, i_brand_id#87, i_class_id#88, i_category_id#89, sales#99, number_sales#100]
+Arguments: 100, [i_brand_id#55 ASC NULLS FIRST, i_class_id#56 ASC NULLS FIRST, i_category_id#57 ASC NULLS FIRST], [channel#74, i_brand_id#55, i_class_id#56, i_category_id#57, sales#69, number_sales#70, channel#102, i_brand_id#87, i_class_id#88, i_category_id#89, sales#99, number_sales#100]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 89 Hosting Expression = Subquery scalar-subquery#72, [id=#73]
+* HashAggregate (139)
++- Exchange (138)
+   +- * HashAggregate (137)
+      +- Union (136)
+         :- * Project (125)
+         :  +- * BroadcastHashJoin Inner BuildRight (124)
+         :     :- * ColumnarToRow (118)
+         :     :  +- Scan parquet default.store_sales (117)
+         :     +- BroadcastExchange (123)
+         :        +- * Project (122)
+         :           +- * Filter (121)
+         :              +- * ColumnarToRow (120)
+         :                 +- Scan parquet default.date_dim (119)
+         :- * Project (130)
+         :  +- * BroadcastHashJoin Inner BuildRight (129)
+         :     :- * ColumnarToRow (127)
+         :     :  +- Scan parquet default.catalog_sales (126)
+         :     +- ReusedExchange (128)
+         +- * Project (135)
+            +- * BroadcastHashJoin Inner BuildRight (134)
+               :- * ColumnarToRow (132)
+               :  +- Scan parquet default.web_sales (131)
+               +- ReusedExchange (133)
+
+
+(117) Scan parquet default.store_sales
+Output [3]: [ss_quantity#104, ss_list_price#105, ss_sold_date_sk#106]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#106), dynamicpruningexpression(ss_sold_date_sk#106 IN dynamicpruning#107)]
+ReadSchema: struct<ss_quantity:int,ss_list_price:decimal(7,2)>
+
+(118) ColumnarToRow [codegen id : 2]
+Input [3]: [ss_quantity#104, ss_list_price#105, ss_sold_date_sk#106]
+
+(119) Scan parquet default.date_dim
+Output [2]: [d_date_sk#108, d_year#109]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(120) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#108, d_year#109]
+
+(121) Filter [codegen id : 1]
+Input [2]: [d_date_sk#108, d_year#109]
+Condition : (((isnotnull(d_year#109) AND (d_year#109 >= 1999)) AND (d_year#109 <= 2001)) AND isnotnull(d_date_sk#108))
+
+(122) Project [codegen id : 1]
+Output [1]: [d_date_sk#108]
+Input [2]: [d_date_sk#108, d_year#109]
+
+(123) BroadcastExchange
+Input [1]: [d_date_sk#108]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#110]
+
+(124) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#106]
+Right keys [1]: [d_date_sk#108]
+Join condition: None
+
+(125) Project [codegen id : 2]
+Output [2]: [ss_quantity#104 AS quantity#111, ss_list_price#105 AS list_price#112]
+Input [4]: [ss_quantity#104, ss_list_price#105, ss_sold_date_sk#106, d_date_sk#108]
+
+(126) Scan parquet default.catalog_sales
+Output [3]: [cs_quantity#113, cs_list_price#114, cs_sold_date_sk#115]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#115), dynamicpruningexpression(cs_sold_date_sk#115 IN dynamicpruning#107)]
+ReadSchema: struct<cs_quantity:int,cs_list_price:decimal(7,2)>
+
+(127) ColumnarToRow [codegen id : 4]
+Input [3]: [cs_quantity#113, cs_list_price#114, cs_sold_date_sk#115]
+
+(128) ReusedExchange [Reuses operator id: 123]
+Output [1]: [d_date_sk#116]
+
+(129) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_sold_date_sk#115]
+Right keys [1]: [d_date_sk#116]
+Join condition: None
+
+(130) Project [codegen id : 4]
+Output [2]: [cs_quantity#113 AS quantity#117, cs_list_price#114 AS list_price#118]
+Input [4]: [cs_quantity#113, cs_list_price#114, cs_sold_date_sk#115, d_date_sk#116]
+
+(131) Scan parquet default.web_sales
+Output [3]: [ws_quantity#119, ws_list_price#120, ws_sold_date_sk#121]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#121), dynamicpruningexpression(ws_sold_date_sk#121 IN dynamicpruning#107)]
+ReadSchema: struct<ws_quantity:int,ws_list_price:decimal(7,2)>
+
+(132) ColumnarToRow [codegen id : 6]
+Input [3]: [ws_quantity#119, ws_list_price#120, ws_sold_date_sk#121]
+
+(133) ReusedExchange [Reuses operator id: 123]
+Output [1]: [d_date_sk#122]
+
+(134) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ws_sold_date_sk#121]
+Right keys [1]: [d_date_sk#122]
+Join condition: None
+
+(135) Project [codegen id : 6]
+Output [2]: [ws_quantity#119 AS quantity#123, ws_list_price#120 AS list_price#124]
+Input [4]: [ws_quantity#119, ws_list_price#120, ws_sold_date_sk#121, d_date_sk#122]
+
+(136) Union
+
+(137) HashAggregate [codegen id : 7]
+Input [2]: [quantity#111, list_price#112]
+Keys: []
+Functions [1]: [partial_avg(CheckOverflow((promote_precision(cast(cast(quantity#111 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#112 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [2]: [sum#125, count#126]
+Results [2]: [sum#127, count#128]
+
+(138) Exchange
+Input [2]: [sum#127, count#128]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#129]
+
+(139) HashAggregate [codegen id : 8]
+Input [2]: [sum#127, count#128]
+Keys: []
+Functions [1]: [avg(CheckOverflow((promote_precision(cast(cast(quantity#111 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#112 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [avg(CheckOverflow((promote_precision(cast(cast(quantity#111 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#112 as decimal(12,2)))), DecimalType(18,2), true))#130]
+Results [1]: [avg(CheckOverflow((promote_precision(cast(cast(quantity#111 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#112 as decimal(12,2)))), DecimalType(18,2), true))#130 AS average_sales#131]
+
+Subquery:2 Hosting operator id = 117 Hosting Expression = ss_sold_date_sk#106 IN dynamicpruning#107
+ReusedExchange (140)
+
+
+(140) ReusedExchange [Reuses operator id: 123]
+Output [1]: [d_date_sk#108]
+
+Subquery:3 Hosting operator id = 126 Hosting Expression = cs_sold_date_sk#115 IN dynamicpruning#107
+
+Subquery:4 Hosting operator id = 131 Hosting Expression = ws_sold_date_sk#121 IN dynamicpruning#107
+
+Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (141)
+
+
+(141) ReusedExchange [Reuses operator id: 72]
+Output [1]: [d_date_sk#49]
+
+Subquery:6 Hosting operator id = 9 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13
+ReusedExchange (142)
+
+
+(142) ReusedExchange [Reuses operator id: 16]
+Output [1]: [d_date_sk#14]
+
+Subquery:7 Hosting operator id = 24 Hosting Expression = cs_sold_date_sk#23 IN dynamicpruning#13
+
+Subquery:8 Hosting operator id = 47 Hosting Expression = ws_sold_date_sk#38 IN dynamicpruning#13
+
+Subquery:9 Hosting operator id = 70 Hosting Expression = Subquery scalar-subquery#51, [id=#52]
+* Project (146)
++- * Filter (145)
+   +- * ColumnarToRow (144)
+      +- Scan parquet default.date_dim (143)
+
+
+(143) Scan parquet default.date_dim
+Output [4]: [d_week_seq#132, d_year#133, d_moy#134, d_dom#135]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,2000), EqualTo(d_moy,12), EqualTo(d_dom,11)]
+ReadSchema: struct<d_week_seq:int,d_year:int,d_moy:int,d_dom:int>
+
+(144) ColumnarToRow [codegen id : 1]
+Input [4]: [d_week_seq#132, d_year#133, d_moy#134, d_dom#135]
+
+(145) Filter [codegen id : 1]
+Input [4]: [d_week_seq#132, d_year#133, d_moy#134, d_dom#135]
+Condition : (((((isnotnull(d_year#133) AND isnotnull(d_moy#134)) AND isnotnull(d_dom#135)) AND (d_year#133 = 2000)) AND (d_moy#134 = 12)) AND (d_dom#135 = 11))
+
+(146) Project [codegen id : 1]
+Output [1]: [d_week_seq#132]
+Input [4]: [d_week_seq#132, d_year#133, d_moy#134, d_dom#135]
+
+Subquery:10 Hosting operator id = 112 Hosting Expression = ReusedSubquery Subquery scalar-subquery#72, [id=#73]
+
+Subquery:11 Hosting operator id = 91 Hosting Expression = ss_sold_date_sk#78 IN dynamicpruning#79
+ReusedExchange (147)
+
+
+(147) ReusedExchange [Reuses operator id: 103]
+Output [1]: [d_date_sk#81]
+
+Subquery:12 Hosting operator id = 101 Hosting Expression = Subquery scalar-subquery#83, [id=#84]
+* Project (151)
++- * Filter (150)
+   +- * ColumnarToRow (149)
+      +- Scan parquet default.date_dim (148)
+
+
+(148) Scan parquet default.date_dim
+Output [4]: [d_week_seq#136, d_year#137, d_moy#138, d_dom#139]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,11)]
+ReadSchema: struct<d_week_seq:int,d_year:int,d_moy:int,d_dom:int>
+
+(149) ColumnarToRow [codegen id : 1]
+Input [4]: [d_week_seq#136, d_year#137, d_moy#138, d_dom#139]
+
+(150) Filter [codegen id : 1]
+Input [4]: [d_week_seq#136, d_year#137, d_moy#138, d_dom#139]
+Condition : (((((isnotnull(d_year#137) AND isnotnull(d_moy#138)) AND isnotnull(d_dom#139)) AND (d_year#137 = 1999)) AND (d_moy#138 = 12)) AND (d_dom#139 = 11))
+
+(151) Project [codegen id : 1]
+Output [1]: [d_week_seq#136]
+Input [4]: [d_week_seq#136, d_year#137, d_moy#138, d_dom#139]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.sf100/simplified.txt
new file mode 100644
index 0000000000000..6a8fcada7a61b
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.sf100/simplified.txt
@@ -0,0 +1,261 @@
+TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales]
+  WholeStageCodegen (92)
+    BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id]
+      Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+        Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+          Subquery #4
+            WholeStageCodegen (8)
+              HashAggregate [sum,count] [avg(CheckOverflow((promote_precision(cast(cast(quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price as decimal(12,2)))), DecimalType(18,2), true)),average_sales,sum,count]
+                InputAdapter
+                  Exchange #18
+                    WholeStageCodegen (7)
+                      HashAggregate [quantity,list_price] [sum,count,sum,count]
+                        InputAdapter
+                          Union
+                            WholeStageCodegen (2)
+                              Project [ss_quantity,ss_list_price]
+                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk]
+                                        SubqueryBroadcast [d_date_sk] #5
+                                          ReusedExchange [d_date_sk] #19
+                                  InputAdapter
+                                    BroadcastExchange #19
+                                      WholeStageCodegen (1)
+                                        Project [d_date_sk]
+                                          Filter [d_year,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_year]
+                            WholeStageCodegen (4)
+                              Project [cs_quantity,cs_list_price]
+                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk]
+                                        ReusedSubquery [d_date_sk] #5
+                                  InputAdapter
+                                    ReusedExchange [d_date_sk] #19
+                            WholeStageCodegen (6)
+                              Project [ws_quantity,ws_list_price]
+                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk]
+                                        ReusedSubquery [d_date_sk] #5
+                                  InputAdapter
+                                    ReusedExchange [d_date_sk] #19
+          HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+            InputAdapter
+              Exchange [i_brand_id,i_class_id,i_category_id] #1
+                WholeStageCodegen (45)
+                  HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count]
+                    Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id]
+                      BroadcastHashJoin [ss_item_sk,i_item_sk]
+                        Project [ss_item_sk,ss_quantity,ss_list_price]
+                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                            SortMergeJoin [ss_item_sk,ss_item_sk]
+                              InputAdapter
+                                WholeStageCodegen (2)
+                                  Sort [ss_item_sk]
+                                    InputAdapter
+                                      Exchange [ss_item_sk] #2
+                                        WholeStageCodegen (1)
+                                          Filter [ss_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk]
+                                                  SubqueryBroadcast [d_date_sk] #1
+                                                    ReusedExchange [d_date_sk] #3
+                              InputAdapter
+                                WholeStageCodegen (21)
+                                  Sort [ss_item_sk]
+                                    InputAdapter
+                                      Exchange [ss_item_sk] #4
+                                        WholeStageCodegen (20)
+                                          Project [i_item_sk]
+                                            BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id]
+                                              Filter [i_brand_id,i_class_id,i_category_id]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                              InputAdapter
+                                                BroadcastExchange #5
+                                                  WholeStageCodegen (19)
+                                                    HashAggregate [brand_id,class_id,category_id]
+                                                      InputAdapter
+                                                        Exchange [brand_id,class_id,category_id] #6
+                                                          WholeStageCodegen (18)
+                                                            HashAggregate [brand_id,class_id,category_id]
+                                                              SortMergeJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id]
+                                                                InputAdapter
+                                                                  WholeStageCodegen (13)
+                                                                    Sort [brand_id,class_id,category_id]
+                                                                      InputAdapter
+                                                                        Exchange [brand_id,class_id,category_id] #7
+                                                                          WholeStageCodegen (12)
+                                                                            HashAggregate [brand_id,class_id,category_id]
+                                                                              InputAdapter
+                                                                                Exchange [brand_id,class_id,category_id] #8
+                                                                                  WholeStageCodegen (11)
+                                                                                    HashAggregate [brand_id,class_id,category_id]
+                                                                                      Project [i_brand_id,i_class_id,i_category_id]
+                                                                                        BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                                                          Project [ss_item_sk]
+                                                                                            BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                              Filter [ss_item_sk]
+                                                                                                ColumnarToRow
+                                                                                                  InputAdapter
+                                                                                                    Scan parquet default.store_sales [ss_item_sk,ss_sold_date_sk]
+                                                                                                      SubqueryBroadcast [d_date_sk] #2
+                                                                                                        ReusedExchange [d_date_sk] #9
+                                                                                              InputAdapter
+                                                                                                BroadcastExchange #9
+                                                                                                  WholeStageCodegen (3)
+                                                                                                    Project [d_date_sk]
+                                                                                                      Filter [d_year,d_date_sk]
+                                                                                                        ColumnarToRow
+                                                                                                          InputAdapter
+                                                                                                            Scan parquet default.date_dim [d_date_sk,d_year]
+                                                                                          InputAdapter
+                                                                                            BroadcastExchange #10
+                                                                                              WholeStageCodegen (10)
+                                                                                                SortMergeJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id]
+                                                                                                  InputAdapter
+                                                                                                    WholeStageCodegen (5)
+                                                                                                      Sort [i_brand_id,i_class_id,i_category_id]
+                                                                                                        InputAdapter
+                                                                                                          Exchange [i_brand_id,i_class_id,i_category_id] #11
+                                                                                                            WholeStageCodegen (4)
+                                                                                                              Filter [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                                                ColumnarToRow
+                                                                                                                  InputAdapter
+                                                                                                                    Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                                  InputAdapter
+                                                                                                    WholeStageCodegen (9)
+                                                                                                      Sort [i_brand_id,i_class_id,i_category_id]
+                                                                                                        InputAdapter
+                                                                                                          Exchange [i_brand_id,i_class_id,i_category_id] #12
+                                                                                                            WholeStageCodegen (8)
+                                                                                                              Project [i_brand_id,i_class_id,i_category_id]
+                                                                                                                BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                                                                                                  Project [cs_item_sk]
+                                                                                                                    BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                                                                      Filter [cs_item_sk]
+                                                                                                                        ColumnarToRow
+                                                                                                                          InputAdapter
+                                                                                                                            Scan parquet default.catalog_sales [cs_item_sk,cs_sold_date_sk]
+                                                                                                                              ReusedSubquery [d_date_sk] #2
+                                                                                                                      InputAdapter
+                                                                                                                        ReusedExchange [d_date_sk] #9
+                                                                                                                  InputAdapter
+                                                                                                                    BroadcastExchange #13
+                                                                                                                      WholeStageCodegen (7)
+                                                                                                                        Filter [i_item_sk]
+                                                                                                                          ColumnarToRow
+                                                                                                                            InputAdapter
+                                                                                                                              Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                InputAdapter
+                                                                  WholeStageCodegen (17)
+                                                                    Sort [i_brand_id,i_class_id,i_category_id]
+                                                                      InputAdapter
+                                                                        Exchange [i_brand_id,i_class_id,i_category_id] #14
+                                                                          WholeStageCodegen (16)
+                                                                            Project [i_brand_id,i_class_id,i_category_id]
+                                                                              BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                                                                Project [ws_item_sk]
+                                                                                  BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                                    Filter [ws_item_sk]
+                                                                                      ColumnarToRow
+                                                                                        InputAdapter
+                                                                                          Scan parquet default.web_sales [ws_item_sk,ws_sold_date_sk]
+                                                                                            ReusedSubquery [d_date_sk] #2
+                                                                                    InputAdapter
+                                                                                      ReusedExchange [d_date_sk] #9
+                                                                                InputAdapter
+                                                                                  ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13
+                            InputAdapter
+                              BroadcastExchange #3
+                                WholeStageCodegen (22)
+                                  Project [d_date_sk]
+                                    Filter [d_week_seq,d_date_sk]
+                                      Subquery #3
+                                        WholeStageCodegen (1)
+                                          Project [d_week_seq]
+                                            Filter [d_year,d_moy,d_dom]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.date_dim [d_week_seq,d_year,d_moy,d_dom]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.date_dim [d_date_sk,d_week_seq]
+                        InputAdapter
+                          BroadcastExchange #15
+                            WholeStageCodegen (44)
+                              SortMergeJoin [i_item_sk,ss_item_sk]
+                                InputAdapter
+                                  WholeStageCodegen (24)
+                                    Sort [i_item_sk]
+                                      InputAdapter
+                                        Exchange [i_item_sk] #16
+                                          WholeStageCodegen (23)
+                                            Filter [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                InputAdapter
+                                  WholeStageCodegen (43)
+                                    Sort [ss_item_sk]
+                                      InputAdapter
+                                        ReusedExchange [ss_item_sk] #17
+      InputAdapter
+        BroadcastExchange #20
+          WholeStageCodegen (91)
+            Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+              Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                ReusedSubquery [average_sales] #4
+                HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                  InputAdapter
+                    Exchange [i_brand_id,i_class_id,i_category_id] #21
+                      WholeStageCodegen (90)
+                        HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count]
+                          Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id]
+                            BroadcastHashJoin [ss_item_sk,i_item_sk]
+                              Project [ss_item_sk,ss_quantity,ss_list_price]
+                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                  SortMergeJoin [ss_item_sk,ss_item_sk]
+                                    InputAdapter
+                                      WholeStageCodegen (47)
+                                        Sort [ss_item_sk]
+                                          InputAdapter
+                                            Exchange [ss_item_sk] #22
+                                              WholeStageCodegen (46)
+                                                Filter [ss_item_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk]
+                                                        SubqueryBroadcast [d_date_sk] #6
+                                                          ReusedExchange [d_date_sk] #23
+                                    InputAdapter
+                                      WholeStageCodegen (66)
+                                        Sort [ss_item_sk]
+                                          InputAdapter
+                                            ReusedExchange [ss_item_sk] #17
+                                  InputAdapter
+                                    BroadcastExchange #23
+                                      WholeStageCodegen (67)
+                                        Project [d_date_sk]
+                                          Filter [d_week_seq,d_date_sk]
+                                            Subquery #7
+                                              WholeStageCodegen (1)
+                                                Project [d_week_seq]
+                                                  Filter [d_year,d_moy,d_dom]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.date_dim [d_week_seq,d_year,d_moy,d_dom]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_week_seq]
+                              InputAdapter
+                                ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #15
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/explain.txt
new file mode 100644
index 0000000000000..ae653b43971db
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/explain.txt
@@ -0,0 +1,789 @@
+== Physical Plan ==
+TakeOrderedAndProject (100)
++- * BroadcastHashJoin Inner BuildRight (99)
+   :- * Project (77)
+   :  +- * Filter (76)
+   :     +- * HashAggregate (75)
+   :        +- Exchange (74)
+   :           +- * HashAggregate (73)
+   :              +- * Project (72)
+   :                 +- * BroadcastHashJoin Inner BuildRight (71)
+   :                    :- * Project (65)
+   :                    :  +- * BroadcastHashJoin Inner BuildRight (64)
+   :                    :     :- * BroadcastHashJoin LeftSemi BuildRight (57)
+   :                    :     :  :- * Filter (3)
+   :                    :     :  :  +- * ColumnarToRow (2)
+   :                    :     :  :     +- Scan parquet default.store_sales (1)
+   :                    :     :  +- BroadcastExchange (56)
+   :                    :     :     +- * Project (55)
+   :                    :     :        +- * BroadcastHashJoin Inner BuildRight (54)
+   :                    :     :           :- * Filter (6)
+   :                    :     :           :  +- * ColumnarToRow (5)
+   :                    :     :           :     +- Scan parquet default.item (4)
+   :                    :     :           +- BroadcastExchange (53)
+   :                    :     :              +- * HashAggregate (52)
+   :                    :     :                 +- * HashAggregate (51)
+   :                    :     :                    +- * BroadcastHashJoin LeftSemi BuildRight (50)
+   :                    :     :                       :- * HashAggregate (39)
+   :                    :     :                       :  +- Exchange (38)
+   :                    :     :                       :     +- * HashAggregate (37)
+   :                    :     :                       :        +- * Project (36)
+   :                    :     :                       :           +- * BroadcastHashJoin Inner BuildRight (35)
+   :                    :     :                       :              :- * Project (33)
+   :                    :     :                       :              :  +- * BroadcastHashJoin Inner BuildRight (32)
+   :                    :     :                       :              :     :- * Filter (9)
+   :                    :     :                       :              :     :  +- * ColumnarToRow (8)
+   :                    :     :                       :              :     :     +- Scan parquet default.store_sales (7)
+   :                    :     :                       :              :     +- BroadcastExchange (31)
+   :                    :     :                       :              :        +- * BroadcastHashJoin LeftSemi BuildRight (30)
+   :                    :     :                       :              :           :- * Filter (12)
+   :                    :     :                       :              :           :  +- * ColumnarToRow (11)
+   :                    :     :                       :              :           :     +- Scan parquet default.item (10)
+   :                    :     :                       :              :           +- BroadcastExchange (29)
+   :                    :     :                       :              :              +- * Project (28)
+   :                    :     :                       :              :                 +- * BroadcastHashJoin Inner BuildRight (27)
+   :                    :     :                       :              :                    :- * Project (21)
+   :                    :     :                       :              :                    :  +- * BroadcastHashJoin Inner BuildRight (20)
+   :                    :     :                       :              :                    :     :- * Filter (15)
+   :                    :     :                       :              :                    :     :  +- * ColumnarToRow (14)
+   :                    :     :                       :              :                    :     :     +- Scan parquet default.catalog_sales (13)
+   :                    :     :                       :              :                    :     +- BroadcastExchange (19)
+   :                    :     :                       :              :                    :        +- * Filter (18)
+   :                    :     :                       :              :                    :           +- * ColumnarToRow (17)
+   :                    :     :                       :              :                    :              +- Scan parquet default.item (16)
+   :                    :     :                       :              :                    +- BroadcastExchange (26)
+   :                    :     :                       :              :                       +- * Project (25)
+   :                    :     :                       :              :                          +- * Filter (24)
+   :                    :     :                       :              :                             +- * ColumnarToRow (23)
+   :                    :     :                       :              :                                +- Scan parquet default.date_dim (22)
+   :                    :     :                       :              +- ReusedExchange (34)
+   :                    :     :                       +- BroadcastExchange (49)
+   :                    :     :                          +- * Project (48)
+   :                    :     :                             +- * BroadcastHashJoin Inner BuildRight (47)
+   :                    :     :                                :- * Project (45)
+   :                    :     :                                :  +- * BroadcastHashJoin Inner BuildRight (44)
+   :                    :     :                                :     :- * Filter (42)
+   :                    :     :                                :     :  +- * ColumnarToRow (41)
+   :                    :     :                                :     :     +- Scan parquet default.web_sales (40)
+   :                    :     :                                :     +- ReusedExchange (43)
+   :                    :     :                                +- ReusedExchange (46)
+   :                    :     +- BroadcastExchange (63)
+   :                    :        +- * BroadcastHashJoin LeftSemi BuildRight (62)
+   :                    :           :- * Filter (60)
+   :                    :           :  +- * ColumnarToRow (59)
+   :                    :           :     +- Scan parquet default.item (58)
+   :                    :           +- ReusedExchange (61)
+   :                    +- BroadcastExchange (70)
+   :                       +- * Project (69)
+   :                          +- * Filter (68)
+   :                             +- * ColumnarToRow (67)
+   :                                +- Scan parquet default.date_dim (66)
+   +- BroadcastExchange (98)
+      +- * Project (97)
+         +- * Filter (96)
+            +- * HashAggregate (95)
+               +- Exchange (94)
+                  +- * HashAggregate (93)
+                     +- * Project (92)
+                        +- * BroadcastHashJoin Inner BuildRight (91)
+                           :- * Project (85)
+                           :  +- * BroadcastHashJoin Inner BuildRight (84)
+                           :     :- * BroadcastHashJoin LeftSemi BuildRight (82)
+                           :     :  :- * Filter (80)
+                           :     :  :  +- * ColumnarToRow (79)
+                           :     :  :     +- Scan parquet default.store_sales (78)
+                           :     :  +- ReusedExchange (81)
+                           :     +- ReusedExchange (83)
+                           +- BroadcastExchange (90)
+                              +- * Project (89)
+                                 +- * Filter (88)
+                                    +- * ColumnarToRow (87)
+                                       +- Scan parquet default.date_dim (86)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_quantity:int,ss_list_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 25]
+Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 25]
+Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+Condition : isnotnull(ss_item_sk#1)
+
+(4) Scan parquet default.item
+Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(5) ColumnarToRow [codegen id : 11]
+Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9]
+
+(6) Filter [codegen id : 11]
+Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9]
+Condition : ((isnotnull(i_brand_id#7) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9))
+
+(7) Scan parquet default.store_sales
+Output [2]: [ss_item_sk#10, ss_sold_date_sk#11]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#11), dynamicpruningexpression(ss_sold_date_sk#11 IN dynamicpruning#12)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int>
+
+(8) ColumnarToRow [codegen id : 6]
+Input [2]: [ss_item_sk#10, ss_sold_date_sk#11]
+
+(9) Filter [codegen id : 6]
+Input [2]: [ss_item_sk#10, ss_sold_date_sk#11]
+Condition : isnotnull(ss_item_sk#10)
+
+(10) Scan parquet default.item
+Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(11) ColumnarToRow [codegen id : 4]
+Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16]
+
+(12) Filter [codegen id : 4]
+Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16]
+Condition : (((isnotnull(i_item_sk#13) AND isnotnull(i_brand_id#14)) AND isnotnull(i_class_id#15)) AND isnotnull(i_category_id#16))
+
+(13) Scan parquet default.catalog_sales
+Output [2]: [cs_item_sk#17, cs_sold_date_sk#18]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_sold_date_sk#18 IN dynamicpruning#12)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int>
+
+(14) ColumnarToRow [codegen id : 3]
+Input [2]: [cs_item_sk#17, cs_sold_date_sk#18]
+
+(15) Filter [codegen id : 3]
+Input [2]: [cs_item_sk#17, cs_sold_date_sk#18]
+Condition : isnotnull(cs_item_sk#17)
+
+(16) Scan parquet default.item
+Output [4]: [i_item_sk#19, i_brand_id#20, i_class_id#21, i_category_id#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(17) ColumnarToRow [codegen id : 1]
+Input [4]: [i_item_sk#19, i_brand_id#20, i_class_id#21, i_category_id#22]
+
+(18) Filter [codegen id : 1]
+Input [4]: [i_item_sk#19, i_brand_id#20, i_class_id#21, i_category_id#22]
+Condition : isnotnull(i_item_sk#19)
+
+(19) BroadcastExchange
+Input [4]: [i_item_sk#19, i_brand_id#20, i_class_id#21, i_category_id#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#23]
+
+(20) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_item_sk#17]
+Right keys [1]: [i_item_sk#19]
+Join condition: None
+
+(21) Project [codegen id : 3]
+Output [4]: [cs_sold_date_sk#18, i_brand_id#20, i_class_id#21, i_category_id#22]
+Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#19, i_brand_id#20, i_class_id#21, i_category_id#22]
+
+(22) Scan parquet default.date_dim
+Output [2]: [d_date_sk#24, d_year#25]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(23) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#24, d_year#25]
+
+(24) Filter [codegen id : 2]
+Input [2]: [d_date_sk#24, d_year#25]
+Condition : (((isnotnull(d_year#25) AND (d_year#25 >= 1999)) AND (d_year#25 <= 2001)) AND isnotnull(d_date_sk#24))
+
+(25) Project [codegen id : 2]
+Output [1]: [d_date_sk#24]
+Input [2]: [d_date_sk#24, d_year#25]
+
+(26) BroadcastExchange
+Input [1]: [d_date_sk#24]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#26]
+
+(27) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_sold_date_sk#18]
+Right keys [1]: [d_date_sk#24]
+Join condition: None
+
+(28) Project [codegen id : 3]
+Output [3]: [i_brand_id#20, i_class_id#21, i_category_id#22]
+Input [5]: [cs_sold_date_sk#18, i_brand_id#20, i_class_id#21, i_category_id#22, d_date_sk#24]
+
+(29) BroadcastExchange
+Input [3]: [i_brand_id#20, i_class_id#21, i_category_id#22]
+Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [id=#27]
+
+(30) BroadcastHashJoin [codegen id : 4]
+Left keys [6]: [coalesce(i_brand_id#14, 0), isnull(i_brand_id#14), coalesce(i_class_id#15, 0), isnull(i_class_id#15), coalesce(i_category_id#16, 0), isnull(i_category_id#16)]
+Right keys [6]: [coalesce(i_brand_id#20, 0), isnull(i_brand_id#20), coalesce(i_class_id#21, 0), isnull(i_class_id#21), coalesce(i_category_id#22, 0), isnull(i_category_id#22)]
+Join condition: None
+
+(31) BroadcastExchange
+Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#28]
+
+(32) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_item_sk#10]
+Right keys [1]: [i_item_sk#13]
+Join condition: None
+
+(33) Project [codegen id : 6]
+Output [4]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16]
+Input [6]: [ss_item_sk#10, ss_sold_date_sk#11, i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16]
+
+(34) ReusedExchange [Reuses operator id: 26]
+Output [1]: [d_date_sk#29]
+
+(35) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_sold_date_sk#11]
+Right keys [1]: [d_date_sk#29]
+Join condition: None
+
+(36) Project [codegen id : 6]
+Output [3]: [i_brand_id#14 AS brand_id#30, i_class_id#15 AS class_id#31, i_category_id#16 AS category_id#32]
+Input [5]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16, d_date_sk#29]
+
+(37) HashAggregate [codegen id : 6]
+Input [3]: [brand_id#30, class_id#31, category_id#32]
+Keys [3]: [brand_id#30, class_id#31, category_id#32]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#30, class_id#31, category_id#32]
+
+(38) Exchange
+Input [3]: [brand_id#30, class_id#31, category_id#32]
+Arguments: hashpartitioning(brand_id#30, class_id#31, category_id#32, 5), ENSURE_REQUIREMENTS, [id=#33]
+
+(39) HashAggregate [codegen id : 10]
+Input [3]: [brand_id#30, class_id#31, category_id#32]
+Keys [3]: [brand_id#30, class_id#31, category_id#32]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#30, class_id#31, category_id#32]
+
+(40) Scan parquet default.web_sales
+Output [2]: [ws_item_sk#34, ws_sold_date_sk#35]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#35), dynamicpruningexpression(ws_sold_date_sk#35 IN dynamicpruning#12)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int>
+
+(41) ColumnarToRow [codegen id : 9]
+Input [2]: [ws_item_sk#34, ws_sold_date_sk#35]
+
+(42) Filter [codegen id : 9]
+Input [2]: [ws_item_sk#34, ws_sold_date_sk#35]
+Condition : isnotnull(ws_item_sk#34)
+
+(43) ReusedExchange [Reuses operator id: 19]
+Output [4]: [i_item_sk#36, i_brand_id#37, i_class_id#38, i_category_id#39]
+
+(44) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ws_item_sk#34]
+Right keys [1]: [i_item_sk#36]
+Join condition: None
+
+(45) Project [codegen id : 9]
+Output [4]: [ws_sold_date_sk#35, i_brand_id#37, i_class_id#38, i_category_id#39]
+Input [6]: [ws_item_sk#34, ws_sold_date_sk#35, i_item_sk#36, i_brand_id#37, i_class_id#38, i_category_id#39]
+
+(46) ReusedExchange [Reuses operator id: 26]
+Output [1]: [d_date_sk#40]
+
+(47) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ws_sold_date_sk#35]
+Right keys [1]: [d_date_sk#40]
+Join condition: None
+
+(48) Project [codegen id : 9]
+Output [3]: [i_brand_id#37, i_class_id#38, i_category_id#39]
+Input [5]: [ws_sold_date_sk#35, i_brand_id#37, i_class_id#38, i_category_id#39, d_date_sk#40]
+
+(49) BroadcastExchange
+Input [3]: [i_brand_id#37, i_class_id#38, i_category_id#39]
+Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [id=#41]
+
+(50) BroadcastHashJoin [codegen id : 10]
+Left keys [6]: [coalesce(brand_id#30, 0), isnull(brand_id#30), coalesce(class_id#31, 0), isnull(class_id#31), coalesce(category_id#32, 0), isnull(category_id#32)]
+Right keys [6]: [coalesce(i_brand_id#37, 0), isnull(i_brand_id#37), coalesce(i_class_id#38, 0), isnull(i_class_id#38), coalesce(i_category_id#39, 0), isnull(i_category_id#39)]
+Join condition: None
+
+(51) HashAggregate [codegen id : 10]
+Input [3]: [brand_id#30, class_id#31, category_id#32]
+Keys [3]: [brand_id#30, class_id#31, category_id#32]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#30, class_id#31, category_id#32]
+
+(52) HashAggregate [codegen id : 10]
+Input [3]: [brand_id#30, class_id#31, category_id#32]
+Keys [3]: [brand_id#30, class_id#31, category_id#32]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#30, class_id#31, category_id#32]
+
+(53) BroadcastExchange
+Input [3]: [brand_id#30, class_id#31, category_id#32]
+Arguments: HashedRelationBroadcastMode(List(input[0, int, true], input[1, int, true], input[2, int, true]),false), [id=#42]
+
+(54) BroadcastHashJoin [codegen id : 11]
+Left keys [3]: [i_brand_id#7, i_class_id#8, i_category_id#9]
+Right keys [3]: [brand_id#30, class_id#31, category_id#32]
+Join condition: None
+
+(55) Project [codegen id : 11]
+Output [1]: [i_item_sk#6 AS ss_item_sk#43]
+Input [7]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, brand_id#30, class_id#31, category_id#32]
+
+(56) BroadcastExchange
+Input [1]: [ss_item_sk#43]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#44]
+
+(57) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [ss_item_sk#43]
+Join condition: None
+
+(58) Scan parquet default.item
+Output [4]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(59) ColumnarToRow [codegen id : 23]
+Input [4]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48]
+
+(60) Filter [codegen id : 23]
+Input [4]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48]
+Condition : (((isnotnull(i_item_sk#45) AND isnotnull(i_brand_id#46)) AND isnotnull(i_class_id#47)) AND isnotnull(i_category_id#48))
+
+(61) ReusedExchange [Reuses operator id: unknown]
+Output [1]: [ss_item_sk#43]
+
+(62) BroadcastHashJoin [codegen id : 23]
+Left keys [1]: [i_item_sk#45]
+Right keys [1]: [ss_item_sk#43]
+Join condition: None
+
+(63) BroadcastExchange
+Input [4]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#49]
+
+(64) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#45]
+Join condition: None
+
+(65) Project [codegen id : 25]
+Output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#46, i_class_id#47, i_category_id#48]
+Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48]
+
+(66) Scan parquet default.date_dim
+Output [2]: [d_date_sk#50, d_week_seq#51]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_week_seq:int>
+
+(67) ColumnarToRow [codegen id : 24]
+Input [2]: [d_date_sk#50, d_week_seq#51]
+
+(68) Filter [codegen id : 24]
+Input [2]: [d_date_sk#50, d_week_seq#51]
+Condition : ((isnotnull(d_week_seq#51) AND (d_week_seq#51 = Subquery scalar-subquery#52, [id=#53])) AND isnotnull(d_date_sk#50))
+
+(69) Project [codegen id : 24]
+Output [1]: [d_date_sk#50]
+Input [2]: [d_date_sk#50, d_week_seq#51]
+
+(70) BroadcastExchange
+Input [1]: [d_date_sk#50]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#54]
+
+(71) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#50]
+Join condition: None
+
+(72) Project [codegen id : 25]
+Output [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#46, i_class_id#47, i_category_id#48]
+Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#46, i_class_id#47, i_category_id#48, d_date_sk#50]
+
+(73) HashAggregate [codegen id : 25]
+Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#46, i_class_id#47, i_category_id#48]
+Keys [3]: [i_brand_id#46, i_class_id#47, i_category_id#48]
+Functions [2]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true)), partial_count(1)]
+Aggregate Attributes [3]: [sum#55, isEmpty#56, count#57]
+Results [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sum#58, isEmpty#59, count#60]
+
+(74) Exchange
+Input [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sum#58, isEmpty#59, count#60]
+Arguments: hashpartitioning(i_brand_id#46, i_class_id#47, i_category_id#48, 5), ENSURE_REQUIREMENTS, [id=#61]
+
+(75) HashAggregate [codegen id : 52]
+Input [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sum#58, isEmpty#59, count#60]
+Keys [3]: [i_brand_id#46, i_class_id#47, i_category_id#48]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#62, count(1)#63]
+Results [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#62 AS sales#64, count(1)#63 AS number_sales#65, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#62 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#66]
+
+(76) Filter [codegen id : 52]
+Input [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sales#64, number_sales#65, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#66]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#66) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#66 as decimal(32,6)) > cast(Subquery scalar-subquery#67, [id=#68] as decimal(32,6))))
+
+(77) Project [codegen id : 52]
+Output [6]: [store AS channel#69, i_brand_id#46, i_class_id#47, i_category_id#48, sales#64, number_sales#65]
+Input [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sales#64, number_sales#65, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#66]
+
+(78) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#70, ss_quantity#71, ss_list_price#72, ss_sold_date_sk#73]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#73), dynamicpruningexpression(ss_sold_date_sk#73 IN dynamicpruning#74)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_quantity:int,ss_list_price:decimal(7,2)>
+
+(79) ColumnarToRow [codegen id : 50]
+Input [4]: [ss_item_sk#70, ss_quantity#71, ss_list_price#72, ss_sold_date_sk#73]
+
+(80) Filter [codegen id : 50]
+Input [4]: [ss_item_sk#70, ss_quantity#71, ss_list_price#72, ss_sold_date_sk#73]
+Condition : isnotnull(ss_item_sk#70)
+
+(81) ReusedExchange [Reuses operator id: unknown]
+Output [1]: [ss_item_sk#43]
+
+(82) BroadcastHashJoin [codegen id : 50]
+Left keys [1]: [ss_item_sk#70]
+Right keys [1]: [ss_item_sk#43]
+Join condition: None
+
+(83) ReusedExchange [Reuses operator id: 63]
+Output [4]: [i_item_sk#75, i_brand_id#76, i_class_id#77, i_category_id#78]
+
+(84) BroadcastHashJoin [codegen id : 50]
+Left keys [1]: [ss_item_sk#70]
+Right keys [1]: [i_item_sk#75]
+Join condition: None
+
+(85) Project [codegen id : 50]
+Output [6]: [ss_quantity#71, ss_list_price#72, ss_sold_date_sk#73, i_brand_id#76, i_class_id#77, i_category_id#78]
+Input [8]: [ss_item_sk#70, ss_quantity#71, ss_list_price#72, ss_sold_date_sk#73, i_item_sk#75, i_brand_id#76, i_class_id#77, i_category_id#78]
+
+(86) Scan parquet default.date_dim
+Output [2]: [d_date_sk#79, d_week_seq#80]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_week_seq:int>
+
+(87) ColumnarToRow [codegen id : 49]
+Input [2]: [d_date_sk#79, d_week_seq#80]
+
+(88) Filter [codegen id : 49]
+Input [2]: [d_date_sk#79, d_week_seq#80]
+Condition : ((isnotnull(d_week_seq#80) AND (d_week_seq#80 = Subquery scalar-subquery#81, [id=#82])) AND isnotnull(d_date_sk#79))
+
+(89) Project [codegen id : 49]
+Output [1]: [d_date_sk#79]
+Input [2]: [d_date_sk#79, d_week_seq#80]
+
+(90) BroadcastExchange
+Input [1]: [d_date_sk#79]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#83]
+
+(91) BroadcastHashJoin [codegen id : 50]
+Left keys [1]: [ss_sold_date_sk#73]
+Right keys [1]: [d_date_sk#79]
+Join condition: None
+
+(92) Project [codegen id : 50]
+Output [5]: [ss_quantity#71, ss_list_price#72, i_brand_id#76, i_class_id#77, i_category_id#78]
+Input [7]: [ss_quantity#71, ss_list_price#72, ss_sold_date_sk#73, i_brand_id#76, i_class_id#77, i_category_id#78, d_date_sk#79]
+
+(93) HashAggregate [codegen id : 50]
+Input [5]: [ss_quantity#71, ss_list_price#72, i_brand_id#76, i_class_id#77, i_category_id#78]
+Keys [3]: [i_brand_id#76, i_class_id#77, i_category_id#78]
+Functions [2]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#71 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#72 as decimal(12,2)))), DecimalType(18,2), true)), partial_count(1)]
+Aggregate Attributes [3]: [sum#84, isEmpty#85, count#86]
+Results [6]: [i_brand_id#76, i_class_id#77, i_category_id#78, sum#87, isEmpty#88, count#89]
+
+(94) Exchange
+Input [6]: [i_brand_id#76, i_class_id#77, i_category_id#78, sum#87, isEmpty#88, count#89]
+Arguments: hashpartitioning(i_brand_id#76, i_class_id#77, i_category_id#78, 5), ENSURE_REQUIREMENTS, [id=#90]
+
+(95) HashAggregate [codegen id : 51]
+Input [6]: [i_brand_id#76, i_class_id#77, i_category_id#78, sum#87, isEmpty#88, count#89]
+Keys [3]: [i_brand_id#76, i_class_id#77, i_category_id#78]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#71 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#72 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#71 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#72 as decimal(12,2)))), DecimalType(18,2), true))#91, count(1)#92]
+Results [6]: [i_brand_id#76, i_class_id#77, i_category_id#78, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#71 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#72 as decimal(12,2)))), DecimalType(18,2), true))#91 AS sales#93, count(1)#92 AS number_sales#94, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#71 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#72 as decimal(12,2)))), DecimalType(18,2), true))#91 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#71 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#72 as decimal(12,2)))), DecimalType(18,2), true))#95]
+
+(96) Filter [codegen id : 51]
+Input [6]: [i_brand_id#76, i_class_id#77, i_category_id#78, sales#93, number_sales#94, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#71 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#72 as decimal(12,2)))), DecimalType(18,2), true))#95]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#71 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#72 as decimal(12,2)))), DecimalType(18,2), true))#95) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#71 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#72 as decimal(12,2)))), DecimalType(18,2), true))#95 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#67, [id=#68] as decimal(32,6))))
+
+(97) Project [codegen id : 51]
+Output [6]: [store AS channel#96, i_brand_id#76, i_class_id#77, i_category_id#78, sales#93, number_sales#94]
+Input [6]: [i_brand_id#76, i_class_id#77, i_category_id#78, sales#93, number_sales#94, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#71 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#72 as decimal(12,2)))), DecimalType(18,2), true))#95]
+
+(98) BroadcastExchange
+Input [6]: [channel#96, i_brand_id#76, i_class_id#77, i_category_id#78, sales#93, number_sales#94]
+Arguments: HashedRelationBroadcastMode(List(input[1, int, true], input[2, int, true], input[3, int, true]),false), [id=#97]
+
+(99) BroadcastHashJoin [codegen id : 52]
+Left keys [3]: [i_brand_id#46, i_class_id#47, i_category_id#48]
+Right keys [3]: [i_brand_id#76, i_class_id#77, i_category_id#78]
+Join condition: None
+
+(100) TakeOrderedAndProject
+Input [12]: [channel#69, i_brand_id#46, i_class_id#47, i_category_id#48, sales#64, number_sales#65, channel#96, i_brand_id#76, i_class_id#77, i_category_id#78, sales#93, number_sales#94]
+Arguments: 100, [i_brand_id#46 ASC NULLS FIRST, i_class_id#47 ASC NULLS FIRST, i_category_id#48 ASC NULLS FIRST], [channel#69, i_brand_id#46, i_class_id#47, i_category_id#48, sales#64, number_sales#65, channel#96, i_brand_id#76, i_class_id#77, i_category_id#78, sales#93, number_sales#94]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 76 Hosting Expression = Subquery scalar-subquery#67, [id=#68]
+* HashAggregate (123)
++- Exchange (122)
+   +- * HashAggregate (121)
+      +- Union (120)
+         :- * Project (109)
+         :  +- * BroadcastHashJoin Inner BuildRight (108)
+         :     :- * ColumnarToRow (102)
+         :     :  +- Scan parquet default.store_sales (101)
+         :     +- BroadcastExchange (107)
+         :        +- * Project (106)
+         :           +- * Filter (105)
+         :              +- * ColumnarToRow (104)
+         :                 +- Scan parquet default.date_dim (103)
+         :- * Project (114)
+         :  +- * BroadcastHashJoin Inner BuildRight (113)
+         :     :- * ColumnarToRow (111)
+         :     :  +- Scan parquet default.catalog_sales (110)
+         :     +- ReusedExchange (112)
+         +- * Project (119)
+            +- * BroadcastHashJoin Inner BuildRight (118)
+               :- * ColumnarToRow (116)
+               :  +- Scan parquet default.web_sales (115)
+               +- ReusedExchange (117)
+
+
+(101) Scan parquet default.store_sales
+Output [3]: [ss_quantity#98, ss_list_price#99, ss_sold_date_sk#100]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#100), dynamicpruningexpression(ss_sold_date_sk#100 IN dynamicpruning#101)]
+ReadSchema: struct<ss_quantity:int,ss_list_price:decimal(7,2)>
+
+(102) ColumnarToRow [codegen id : 2]
+Input [3]: [ss_quantity#98, ss_list_price#99, ss_sold_date_sk#100]
+
+(103) Scan parquet default.date_dim
+Output [2]: [d_date_sk#102, d_year#103]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(104) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#102, d_year#103]
+
+(105) Filter [codegen id : 1]
+Input [2]: [d_date_sk#102, d_year#103]
+Condition : (((isnotnull(d_year#103) AND (d_year#103 >= 1999)) AND (d_year#103 <= 2001)) AND isnotnull(d_date_sk#102))
+
+(106) Project [codegen id : 1]
+Output [1]: [d_date_sk#102]
+Input [2]: [d_date_sk#102, d_year#103]
+
+(107) BroadcastExchange
+Input [1]: [d_date_sk#102]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#104]
+
+(108) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#100]
+Right keys [1]: [d_date_sk#102]
+Join condition: None
+
+(109) Project [codegen id : 2]
+Output [2]: [ss_quantity#98 AS quantity#105, ss_list_price#99 AS list_price#106]
+Input [4]: [ss_quantity#98, ss_list_price#99, ss_sold_date_sk#100, d_date_sk#102]
+
+(110) Scan parquet default.catalog_sales
+Output [3]: [cs_quantity#107, cs_list_price#108, cs_sold_date_sk#109]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#109), dynamicpruningexpression(cs_sold_date_sk#109 IN dynamicpruning#101)]
+ReadSchema: struct<cs_quantity:int,cs_list_price:decimal(7,2)>
+
+(111) ColumnarToRow [codegen id : 4]
+Input [3]: [cs_quantity#107, cs_list_price#108, cs_sold_date_sk#109]
+
+(112) ReusedExchange [Reuses operator id: 107]
+Output [1]: [d_date_sk#110]
+
+(113) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_sold_date_sk#109]
+Right keys [1]: [d_date_sk#110]
+Join condition: None
+
+(114) Project [codegen id : 4]
+Output [2]: [cs_quantity#107 AS quantity#111, cs_list_price#108 AS list_price#112]
+Input [4]: [cs_quantity#107, cs_list_price#108, cs_sold_date_sk#109, d_date_sk#110]
+
+(115) Scan parquet default.web_sales
+Output [3]: [ws_quantity#113, ws_list_price#114, ws_sold_date_sk#115]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#115), dynamicpruningexpression(ws_sold_date_sk#115 IN dynamicpruning#101)]
+ReadSchema: struct<ws_quantity:int,ws_list_price:decimal(7,2)>
+
+(116) ColumnarToRow [codegen id : 6]
+Input [3]: [ws_quantity#113, ws_list_price#114, ws_sold_date_sk#115]
+
+(117) ReusedExchange [Reuses operator id: 107]
+Output [1]: [d_date_sk#116]
+
+(118) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ws_sold_date_sk#115]
+Right keys [1]: [d_date_sk#116]
+Join condition: None
+
+(119) Project [codegen id : 6]
+Output [2]: [ws_quantity#113 AS quantity#117, ws_list_price#114 AS list_price#118]
+Input [4]: [ws_quantity#113, ws_list_price#114, ws_sold_date_sk#115, d_date_sk#116]
+
+(120) Union
+
+(121) HashAggregate [codegen id : 7]
+Input [2]: [quantity#105, list_price#106]
+Keys: []
+Functions [1]: [partial_avg(CheckOverflow((promote_precision(cast(cast(quantity#105 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#106 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [2]: [sum#119, count#120]
+Results [2]: [sum#121, count#122]
+
+(122) Exchange
+Input [2]: [sum#121, count#122]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#123]
+
+(123) HashAggregate [codegen id : 8]
+Input [2]: [sum#121, count#122]
+Keys: []
+Functions [1]: [avg(CheckOverflow((promote_precision(cast(cast(quantity#105 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#106 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [avg(CheckOverflow((promote_precision(cast(cast(quantity#105 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#106 as decimal(12,2)))), DecimalType(18,2), true))#124]
+Results [1]: [avg(CheckOverflow((promote_precision(cast(cast(quantity#105 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#106 as decimal(12,2)))), DecimalType(18,2), true))#124 AS average_sales#125]
+
+Subquery:2 Hosting operator id = 101 Hosting Expression = ss_sold_date_sk#100 IN dynamicpruning#101
+ReusedExchange (124)
+
+
+(124) ReusedExchange [Reuses operator id: 107]
+Output [1]: [d_date_sk#102]
+
+Subquery:3 Hosting operator id = 110 Hosting Expression = cs_sold_date_sk#109 IN dynamicpruning#101
+
+Subquery:4 Hosting operator id = 115 Hosting Expression = ws_sold_date_sk#115 IN dynamicpruning#101
+
+Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (125)
+
+
+(125) ReusedExchange [Reuses operator id: 70]
+Output [1]: [d_date_sk#50]
+
+Subquery:6 Hosting operator id = 7 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12
+ReusedExchange (126)
+
+
+(126) ReusedExchange [Reuses operator id: 26]
+Output [1]: [d_date_sk#29]
+
+Subquery:7 Hosting operator id = 13 Hosting Expression = cs_sold_date_sk#18 IN dynamicpruning#12
+
+Subquery:8 Hosting operator id = 40 Hosting Expression = ws_sold_date_sk#35 IN dynamicpruning#12
+
+Subquery:9 Hosting operator id = 68 Hosting Expression = Subquery scalar-subquery#52, [id=#53]
+* Project (130)
++- * Filter (129)
+   +- * ColumnarToRow (128)
+      +- Scan parquet default.date_dim (127)
+
+
+(127) Scan parquet default.date_dim
+Output [4]: [d_week_seq#126, d_year#127, d_moy#128, d_dom#129]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,2000), EqualTo(d_moy,12), EqualTo(d_dom,11)]
+ReadSchema: struct<d_week_seq:int,d_year:int,d_moy:int,d_dom:int>
+
+(128) ColumnarToRow [codegen id : 1]
+Input [4]: [d_week_seq#126, d_year#127, d_moy#128, d_dom#129]
+
+(129) Filter [codegen id : 1]
+Input [4]: [d_week_seq#126, d_year#127, d_moy#128, d_dom#129]
+Condition : (((((isnotnull(d_year#127) AND isnotnull(d_moy#128)) AND isnotnull(d_dom#129)) AND (d_year#127 = 2000)) AND (d_moy#128 = 12)) AND (d_dom#129 = 11))
+
+(130) Project [codegen id : 1]
+Output [1]: [d_week_seq#126]
+Input [4]: [d_week_seq#126, d_year#127, d_moy#128, d_dom#129]
+
+Subquery:10 Hosting operator id = 96 Hosting Expression = ReusedSubquery Subquery scalar-subquery#67, [id=#68]
+
+Subquery:11 Hosting operator id = 78 Hosting Expression = ss_sold_date_sk#73 IN dynamicpruning#74
+ReusedExchange (131)
+
+
+(131) ReusedExchange [Reuses operator id: 90]
+Output [1]: [d_date_sk#79]
+
+Subquery:12 Hosting operator id = 88 Hosting Expression = Subquery scalar-subquery#81, [id=#82]
+* Project (135)
++- * Filter (134)
+   +- * ColumnarToRow (133)
+      +- Scan parquet default.date_dim (132)
+
+
+(132) Scan parquet default.date_dim
+Output [4]: [d_week_seq#130, d_year#131, d_moy#132, d_dom#133]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,11)]
+ReadSchema: struct<d_week_seq:int,d_year:int,d_moy:int,d_dom:int>
+
+(133) ColumnarToRow [codegen id : 1]
+Input [4]: [d_week_seq#130, d_year#131, d_moy#132, d_dom#133]
+
+(134) Filter [codegen id : 1]
+Input [4]: [d_week_seq#130, d_year#131, d_moy#132, d_dom#133]
+Condition : (((((isnotnull(d_year#131) AND isnotnull(d_moy#132)) AND isnotnull(d_dom#133)) AND (d_year#131 = 1999)) AND (d_moy#132 = 12)) AND (d_dom#133 = 11))
+
+(135) Project [codegen id : 1]
+Output [1]: [d_week_seq#130]
+Input [4]: [d_week_seq#130, d_year#131, d_moy#132, d_dom#133]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/simplified.txt
new file mode 100644
index 0000000000000..cb9b410ea8568
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/simplified.txt
@@ -0,0 +1,213 @@
+TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales]
+  WholeStageCodegen (52)
+    BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id]
+      Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+        Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+          Subquery #4
+            WholeStageCodegen (8)
+              HashAggregate [sum,count] [avg(CheckOverflow((promote_precision(cast(cast(quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price as decimal(12,2)))), DecimalType(18,2), true)),average_sales,sum,count]
+                InputAdapter
+                  Exchange #13
+                    WholeStageCodegen (7)
+                      HashAggregate [quantity,list_price] [sum,count,sum,count]
+                        InputAdapter
+                          Union
+                            WholeStageCodegen (2)
+                              Project [ss_quantity,ss_list_price]
+                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk]
+                                        SubqueryBroadcast [d_date_sk] #5
+                                          ReusedExchange [d_date_sk] #14
+                                  InputAdapter
+                                    BroadcastExchange #14
+                                      WholeStageCodegen (1)
+                                        Project [d_date_sk]
+                                          Filter [d_year,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_year]
+                            WholeStageCodegen (4)
+                              Project [cs_quantity,cs_list_price]
+                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk]
+                                        ReusedSubquery [d_date_sk] #5
+                                  InputAdapter
+                                    ReusedExchange [d_date_sk] #14
+                            WholeStageCodegen (6)
+                              Project [ws_quantity,ws_list_price]
+                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk]
+                                        ReusedSubquery [d_date_sk] #5
+                                  InputAdapter
+                                    ReusedExchange [d_date_sk] #14
+          HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+            InputAdapter
+              Exchange [i_brand_id,i_class_id,i_category_id] #1
+                WholeStageCodegen (25)
+                  HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count]
+                    Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id]
+                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                        Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id]
+                          BroadcastHashJoin [ss_item_sk,i_item_sk]
+                            BroadcastHashJoin [ss_item_sk,ss_item_sk]
+                              Filter [ss_item_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk]
+                                      SubqueryBroadcast [d_date_sk] #1
+                                        ReusedExchange [d_date_sk] #2
+                              InputAdapter
+                                BroadcastExchange #3
+                                  WholeStageCodegen (11)
+                                    Project [i_item_sk]
+                                      BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id]
+                                        Filter [i_brand_id,i_class_id,i_category_id]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                        InputAdapter
+                                          BroadcastExchange #4
+                                            WholeStageCodegen (10)
+                                              HashAggregate [brand_id,class_id,category_id]
+                                                HashAggregate [brand_id,class_id,category_id]
+                                                  BroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id]
+                                                    HashAggregate [brand_id,class_id,category_id]
+                                                      InputAdapter
+                                                        Exchange [brand_id,class_id,category_id] #5
+                                                          WholeStageCodegen (6)
+                                                            HashAggregate [brand_id,class_id,category_id]
+                                                              Project [i_brand_id,i_class_id,i_category_id]
+                                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                  Project [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id]
+                                                                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                                      Filter [ss_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.store_sales [ss_item_sk,ss_sold_date_sk]
+                                                                              SubqueryBroadcast [d_date_sk] #2
+                                                                                ReusedExchange [d_date_sk] #6
+                                                                      InputAdapter
+                                                                        BroadcastExchange #7
+                                                                          WholeStageCodegen (4)
+                                                                            BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id]
+                                                                              Filter [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                              InputAdapter
+                                                                                BroadcastExchange #8
+                                                                                  WholeStageCodegen (3)
+                                                                                    Project [i_brand_id,i_class_id,i_category_id]
+                                                                                      BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                                        Project [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                          BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                                                                            Filter [cs_item_sk]
+                                                                                              ColumnarToRow
+                                                                                                InputAdapter
+                                                                                                  Scan parquet default.catalog_sales [cs_item_sk,cs_sold_date_sk]
+                                                                                                    ReusedSubquery [d_date_sk] #2
+                                                                                            InputAdapter
+                                                                                              BroadcastExchange #9
+                                                                                                WholeStageCodegen (1)
+                                                                                                  Filter [i_item_sk]
+                                                                                                    ColumnarToRow
+                                                                                                      InputAdapter
+                                                                                                        Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                        InputAdapter
+                                                                                          BroadcastExchange #6
+                                                                                            WholeStageCodegen (2)
+                                                                                              Project [d_date_sk]
+                                                                                                Filter [d_year,d_date_sk]
+                                                                                                  ColumnarToRow
+                                                                                                    InputAdapter
+                                                                                                      Scan parquet default.date_dim [d_date_sk,d_year]
+                                                                  InputAdapter
+                                                                    ReusedExchange [d_date_sk] #6
+                                                    InputAdapter
+                                                      BroadcastExchange #10
+                                                        WholeStageCodegen (9)
+                                                          Project [i_brand_id,i_class_id,i_category_id]
+                                                            BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                              Project [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id]
+                                                                BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                                                  Filter [ws_item_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.web_sales [ws_item_sk,ws_sold_date_sk]
+                                                                          ReusedSubquery [d_date_sk] #2
+                                                                  InputAdapter
+                                                                    ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9
+                                                              InputAdapter
+                                                                ReusedExchange [d_date_sk] #6
+                            InputAdapter
+                              BroadcastExchange #11
+                                WholeStageCodegen (23)
+                                  BroadcastHashJoin [i_item_sk,ss_item_sk]
+                                    Filter [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                    InputAdapter
+                                      ReusedExchange [ss_item_sk] #12
+                        InputAdapter
+                          BroadcastExchange #2
+                            WholeStageCodegen (24)
+                              Project [d_date_sk]
+                                Filter [d_week_seq,d_date_sk]
+                                  Subquery #3
+                                    WholeStageCodegen (1)
+                                      Project [d_week_seq]
+                                        Filter [d_year,d_moy,d_dom]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.date_dim [d_week_seq,d_year,d_moy,d_dom]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.date_dim [d_date_sk,d_week_seq]
+      InputAdapter
+        BroadcastExchange #15
+          WholeStageCodegen (51)
+            Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+              Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                ReusedSubquery [average_sales] #4
+                HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                  InputAdapter
+                    Exchange [i_brand_id,i_class_id,i_category_id] #16
+                      WholeStageCodegen (50)
+                        HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count]
+                          Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id]
+                            BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                              Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id]
+                                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                  BroadcastHashJoin [ss_item_sk,ss_item_sk]
+                                    Filter [ss_item_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk]
+                                            SubqueryBroadcast [d_date_sk] #6
+                                              ReusedExchange [d_date_sk] #17
+                                    InputAdapter
+                                      ReusedExchange [ss_item_sk] #12
+                                  InputAdapter
+                                    ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11
+                              InputAdapter
+                                BroadcastExchange #17
+                                  WholeStageCodegen (49)
+                                    Project [d_date_sk]
+                                      Filter [d_week_seq,d_date_sk]
+                                        Subquery #7
+                                          WholeStageCodegen (1)
+                                            Project [d_week_seq]
+                                              Filter [d_year,d_moy,d_dom]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_week_seq,d_year,d_moy,d_dom]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.date_dim [d_date_sk,d_week_seq]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.sf100/explain.txt
new file mode 100644
index 0000000000000..4725a7a60a3ce
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.sf100/explain.txt
@@ -0,0 +1,191 @@
+== Physical Plan ==
+TakeOrderedAndProject (32)
++- * HashAggregate (31)
+   +- Exchange (30)
+      +- * HashAggregate (29)
+         +- * Project (28)
+            +- * SortMergeJoin Inner (27)
+               :- * Sort (12)
+               :  +- Exchange (11)
+               :     +- * Project (10)
+               :        +- * BroadcastHashJoin Inner BuildRight (9)
+               :           :- * Filter (3)
+               :           :  +- * ColumnarToRow (2)
+               :           :     +- Scan parquet default.catalog_sales (1)
+               :           +- BroadcastExchange (8)
+               :              +- * Project (7)
+               :                 +- * Filter (6)
+               :                    +- * ColumnarToRow (5)
+               :                       +- Scan parquet default.date_dim (4)
+               +- * Sort (26)
+                  +- Exchange (25)
+                     +- * Project (24)
+                        +- * SortMergeJoin Inner (23)
+                           :- * Sort (17)
+                           :  +- Exchange (16)
+                           :     +- * Filter (15)
+                           :        +- * ColumnarToRow (14)
+                           :           +- Scan parquet default.customer (13)
+                           +- * Sort (22)
+                              +- Exchange (21)
+                                 +- * Filter (20)
+                                    +- * ColumnarToRow (19)
+                                       +- Scan parquet default.customer_address (18)
+
+
+(1) Scan parquet default.catalog_sales
+Output [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(cs_bill_customer_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3]
+
+(3) Filter [codegen id : 2]
+Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3]
+Condition : isnotnull(cs_bill_customer_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_qoy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+Condition : ((((isnotnull(d_qoy#7) AND isnotnull(d_year#6)) AND (d_qoy#7 = 2)) AND (d_year#6 = 2001)) AND isnotnull(d_date_sk#5))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#5]
+Input [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#8]
+
+(9) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [cs_sold_date_sk#3]
+Right keys [1]: [d_date_sk#5]
+Join condition: None
+
+(10) Project [codegen id : 2]
+Output [2]: [cs_bill_customer_sk#1, cs_sales_price#2]
+Input [4]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3, d_date_sk#5]
+
+(11) Exchange
+Input [2]: [cs_bill_customer_sk#1, cs_sales_price#2]
+Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#9]
+
+(12) Sort [codegen id : 3]
+Input [2]: [cs_bill_customer_sk#1, cs_sales_price#2]
+Arguments: [cs_bill_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(13) Scan parquet default.customer
+Output [2]: [c_customer_sk#10, c_current_addr_sk#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int>
+
+(14) ColumnarToRow [codegen id : 4]
+Input [2]: [c_customer_sk#10, c_current_addr_sk#11]
+
+(15) Filter [codegen id : 4]
+Input [2]: [c_customer_sk#10, c_current_addr_sk#11]
+Condition : (isnotnull(c_customer_sk#10) AND isnotnull(c_current_addr_sk#11))
+
+(16) Exchange
+Input [2]: [c_customer_sk#10, c_current_addr_sk#11]
+Arguments: hashpartitioning(c_current_addr_sk#11, 5), ENSURE_REQUIREMENTS, [id=#12]
+
+(17) Sort [codegen id : 5]
+Input [2]: [c_customer_sk#10, c_current_addr_sk#11]
+Arguments: [c_current_addr_sk#11 ASC NULLS FIRST], false, 0
+
+(18) Scan parquet default.customer_address
+Output [3]: [ca_address_sk#13, ca_state#14, ca_zip#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string,ca_zip:string>
+
+(19) ColumnarToRow [codegen id : 6]
+Input [3]: [ca_address_sk#13, ca_state#14, ca_zip#15]
+
+(20) Filter [codegen id : 6]
+Input [3]: [ca_address_sk#13, ca_state#14, ca_zip#15]
+Condition : isnotnull(ca_address_sk#13)
+
+(21) Exchange
+Input [3]: [ca_address_sk#13, ca_state#14, ca_zip#15]
+Arguments: hashpartitioning(ca_address_sk#13, 5), ENSURE_REQUIREMENTS, [id=#16]
+
+(22) Sort [codegen id : 7]
+Input [3]: [ca_address_sk#13, ca_state#14, ca_zip#15]
+Arguments: [ca_address_sk#13 ASC NULLS FIRST], false, 0
+
+(23) SortMergeJoin [codegen id : 8]
+Left keys [1]: [c_current_addr_sk#11]
+Right keys [1]: [ca_address_sk#13]
+Join condition: None
+
+(24) Project [codegen id : 8]
+Output [3]: [c_customer_sk#10, ca_state#14, ca_zip#15]
+Input [5]: [c_customer_sk#10, c_current_addr_sk#11, ca_address_sk#13, ca_state#14, ca_zip#15]
+
+(25) Exchange
+Input [3]: [c_customer_sk#10, ca_state#14, ca_zip#15]
+Arguments: hashpartitioning(c_customer_sk#10, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(26) Sort [codegen id : 9]
+Input [3]: [c_customer_sk#10, ca_state#14, ca_zip#15]
+Arguments: [c_customer_sk#10 ASC NULLS FIRST], false, 0
+
+(27) SortMergeJoin [codegen id : 10]
+Left keys [1]: [cs_bill_customer_sk#1]
+Right keys [1]: [c_customer_sk#10]
+Join condition: ((substr(ca_zip#15, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR ca_state#14 IN (CA,WA,GA)) OR (cs_sales_price#2 > 500.00))
+
+(28) Project [codegen id : 10]
+Output [2]: [cs_sales_price#2, ca_zip#15]
+Input [5]: [cs_bill_customer_sk#1, cs_sales_price#2, c_customer_sk#10, ca_state#14, ca_zip#15]
+
+(29) HashAggregate [codegen id : 10]
+Input [2]: [cs_sales_price#2, ca_zip#15]
+Keys [1]: [ca_zip#15]
+Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#2))]
+Aggregate Attributes [1]: [sum#18]
+Results [2]: [ca_zip#15, sum#19]
+
+(30) Exchange
+Input [2]: [ca_zip#15, sum#19]
+Arguments: hashpartitioning(ca_zip#15, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(31) HashAggregate [codegen id : 11]
+Input [2]: [ca_zip#15, sum#19]
+Keys [1]: [ca_zip#15]
+Functions [1]: [sum(UnscaledValue(cs_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#2))#21]
+Results [2]: [ca_zip#15, MakeDecimal(sum(UnscaledValue(cs_sales_price#2))#21,17,2) AS sum(cs_sales_price)#22]
+
+(32) TakeOrderedAndProject
+Input [2]: [ca_zip#15, sum(cs_sales_price)#22]
+Arguments: 100, [ca_zip#15 ASC NULLS FIRST], [ca_zip#15, sum(cs_sales_price)#22]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (33)
+
+
+(33) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#5]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.sf100/simplified.txt
new file mode 100644
index 0000000000000..f8d15ca97ac7a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.sf100/simplified.txt
@@ -0,0 +1,59 @@
+TakeOrderedAndProject [ca_zip,sum(cs_sales_price)]
+  WholeStageCodegen (11)
+    HashAggregate [ca_zip,sum] [sum(UnscaledValue(cs_sales_price)),sum(cs_sales_price),sum]
+      InputAdapter
+        Exchange [ca_zip] #1
+          WholeStageCodegen (10)
+            HashAggregate [ca_zip,cs_sales_price] [sum,sum]
+              Project [cs_sales_price,ca_zip]
+                SortMergeJoin [cs_bill_customer_sk,c_customer_sk,ca_zip,ca_state,cs_sales_price]
+                  InputAdapter
+                    WholeStageCodegen (3)
+                      Sort [cs_bill_customer_sk]
+                        InputAdapter
+                          Exchange [cs_bill_customer_sk] #2
+                            WholeStageCodegen (2)
+                              Project [cs_bill_customer_sk,cs_sales_price]
+                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                  Filter [cs_bill_customer_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk]
+                                          SubqueryBroadcast [d_date_sk] #1
+                                            ReusedExchange [d_date_sk] #3
+                                  InputAdapter
+                                    BroadcastExchange #3
+                                      WholeStageCodegen (1)
+                                        Project [d_date_sk]
+                                          Filter [d_qoy,d_year,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_year,d_qoy]
+                  InputAdapter
+                    WholeStageCodegen (9)
+                      Sort [c_customer_sk]
+                        InputAdapter
+                          Exchange [c_customer_sk] #4
+                            WholeStageCodegen (8)
+                              Project [c_customer_sk,ca_state,ca_zip]
+                                SortMergeJoin [c_current_addr_sk,ca_address_sk]
+                                  InputAdapter
+                                    WholeStageCodegen (5)
+                                      Sort [c_current_addr_sk]
+                                        InputAdapter
+                                          Exchange [c_current_addr_sk] #5
+                                            WholeStageCodegen (4)
+                                              Filter [c_customer_sk,c_current_addr_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.customer [c_customer_sk,c_current_addr_sk]
+                                  InputAdapter
+                                    WholeStageCodegen (7)
+                                      Sort [ca_address_sk]
+                                        InputAdapter
+                                          Exchange [ca_address_sk] #6
+                                            WholeStageCodegen (6)
+                                              Filter [ca_address_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.customer_address [ca_address_sk,ca_state,ca_zip]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/explain.txt
new file mode 100644
index 0000000000000..d360e7aa6a6fe
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/explain.txt
@@ -0,0 +1,161 @@
+== Physical Plan ==
+TakeOrderedAndProject (26)
++- * HashAggregate (25)
+   +- Exchange (24)
+      +- * HashAggregate (23)
+         +- * Project (22)
+            +- * BroadcastHashJoin Inner BuildRight (21)
+               :- * Project (15)
+               :  +- * BroadcastHashJoin Inner BuildRight (14)
+               :     :- * Project (9)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+               :     :     :- * Filter (3)
+               :     :     :  +- * ColumnarToRow (2)
+               :     :     :     +- Scan parquet default.catalog_sales (1)
+               :     :     +- BroadcastExchange (7)
+               :     :        +- * Filter (6)
+               :     :           +- * ColumnarToRow (5)
+               :     :              +- Scan parquet default.customer (4)
+               :     +- BroadcastExchange (13)
+               :        +- * Filter (12)
+               :           +- * ColumnarToRow (11)
+               :              +- Scan parquet default.customer_address (10)
+               +- BroadcastExchange (20)
+                  +- * Project (19)
+                     +- * Filter (18)
+                        +- * ColumnarToRow (17)
+                           +- Scan parquet default.date_dim (16)
+
+
+(1) Scan parquet default.catalog_sales
+Output [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(cs_bill_customer_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3]
+
+(3) Filter [codegen id : 4]
+Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3]
+Condition : isnotnull(cs_bill_customer_sk#1)
+
+(4) Scan parquet default.customer
+Output [2]: [c_customer_sk#5, c_current_addr_sk#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [c_customer_sk#5, c_current_addr_sk#6]
+
+(6) Filter [codegen id : 1]
+Input [2]: [c_customer_sk#5, c_current_addr_sk#6]
+Condition : (isnotnull(c_customer_sk#5) AND isnotnull(c_current_addr_sk#6))
+
+(7) BroadcastExchange
+Input [2]: [c_customer_sk#5, c_current_addr_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#7]
+
+(8) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_bill_customer_sk#1]
+Right keys [1]: [c_customer_sk#5]
+Join condition: None
+
+(9) Project [codegen id : 4]
+Output [3]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6]
+Input [5]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3, c_customer_sk#5, c_current_addr_sk#6]
+
+(10) Scan parquet default.customer_address
+Output [3]: [ca_address_sk#8, ca_state#9, ca_zip#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string,ca_zip:string>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [3]: [ca_address_sk#8, ca_state#9, ca_zip#10]
+
+(12) Filter [codegen id : 2]
+Input [3]: [ca_address_sk#8, ca_state#9, ca_zip#10]
+Condition : isnotnull(ca_address_sk#8)
+
+(13) BroadcastExchange
+Input [3]: [ca_address_sk#8, ca_state#9, ca_zip#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(14) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [c_current_addr_sk#6]
+Right keys [1]: [ca_address_sk#8]
+Join condition: ((substr(ca_zip#10, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR ca_state#9 IN (CA,WA,GA)) OR (cs_sales_price#2 > 500.00))
+
+(15) Project [codegen id : 4]
+Output [3]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10]
+Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6, ca_address_sk#8, ca_state#9, ca_zip#10]
+
+(16) Scan parquet default.date_dim
+Output [3]: [d_date_sk#12, d_year#13, d_qoy#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_qoy:int>
+
+(17) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#12, d_year#13, d_qoy#14]
+
+(18) Filter [codegen id : 3]
+Input [3]: [d_date_sk#12, d_year#13, d_qoy#14]
+Condition : ((((isnotnull(d_qoy#14) AND isnotnull(d_year#13)) AND (d_qoy#14 = 2)) AND (d_year#13 = 2001)) AND isnotnull(d_date_sk#12))
+
+(19) Project [codegen id : 3]
+Output [1]: [d_date_sk#12]
+Input [3]: [d_date_sk#12, d_year#13, d_qoy#14]
+
+(20) BroadcastExchange
+Input [1]: [d_date_sk#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#15]
+
+(21) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_sold_date_sk#3]
+Right keys [1]: [d_date_sk#12]
+Join condition: None
+
+(22) Project [codegen id : 4]
+Output [2]: [cs_sales_price#2, ca_zip#10]
+Input [4]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10, d_date_sk#12]
+
+(23) HashAggregate [codegen id : 4]
+Input [2]: [cs_sales_price#2, ca_zip#10]
+Keys [1]: [ca_zip#10]
+Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#2))]
+Aggregate Attributes [1]: [sum#16]
+Results [2]: [ca_zip#10, sum#17]
+
+(24) Exchange
+Input [2]: [ca_zip#10, sum#17]
+Arguments: hashpartitioning(ca_zip#10, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(25) HashAggregate [codegen id : 5]
+Input [2]: [ca_zip#10, sum#17]
+Keys [1]: [ca_zip#10]
+Functions [1]: [sum(UnscaledValue(cs_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#2))#19]
+Results [2]: [ca_zip#10, MakeDecimal(sum(UnscaledValue(cs_sales_price#2))#19,17,2) AS sum(cs_sales_price)#20]
+
+(26) TakeOrderedAndProject
+Input [2]: [ca_zip#10, sum(cs_sales_price)#20]
+Arguments: 100, [ca_zip#10 ASC NULLS FIRST], [ca_zip#10, sum(cs_sales_price)#20]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (27)
+
+
+(27) ReusedExchange [Reuses operator id: 20]
+Output [1]: [d_date_sk#12]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/simplified.txt
new file mode 100644
index 0000000000000..79b0d4b0cad21
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/simplified.txt
@@ -0,0 +1,41 @@
+TakeOrderedAndProject [ca_zip,sum(cs_sales_price)]
+  WholeStageCodegen (5)
+    HashAggregate [ca_zip,sum] [sum(UnscaledValue(cs_sales_price)),sum(cs_sales_price),sum]
+      InputAdapter
+        Exchange [ca_zip] #1
+          WholeStageCodegen (4)
+            HashAggregate [ca_zip,cs_sales_price] [sum,sum]
+              Project [cs_sales_price,ca_zip]
+                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                  Project [cs_sales_price,cs_sold_date_sk,ca_zip]
+                    BroadcastHashJoin [c_current_addr_sk,ca_address_sk,ca_zip,ca_state,cs_sales_price]
+                      Project [cs_sales_price,cs_sold_date_sk,c_current_addr_sk]
+                        BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk]
+                          Filter [cs_bill_customer_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk]
+                                  SubqueryBroadcast [d_date_sk] #1
+                                    ReusedExchange [d_date_sk] #2
+                          InputAdapter
+                            BroadcastExchange #3
+                              WholeStageCodegen (1)
+                                Filter [c_customer_sk,c_current_addr_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.customer [c_customer_sk,c_current_addr_sk]
+                      InputAdapter
+                        BroadcastExchange #4
+                          WholeStageCodegen (2)
+                            Filter [ca_address_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.customer_address [ca_address_sk,ca_state,ca_zip]
+                  InputAdapter
+                    BroadcastExchange #2
+                      WholeStageCodegen (3)
+                        Project [d_date_sk]
+                          Filter [d_qoy,d_year,d_date_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.date_dim [d_date_sk,d_year,d_qoy]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.sf100/explain.txt
new file mode 100644
index 0000000000000..f9ab964739273
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.sf100/explain.txt
@@ -0,0 +1,260 @@
+== Physical Plan ==
+* Sort (46)
++- * HashAggregate (45)
+   +- Exchange (44)
+      +- * HashAggregate (43)
+         +- * HashAggregate (42)
+            +- * HashAggregate (41)
+               +- * Project (40)
+                  +- * BroadcastHashJoin Inner BuildRight (39)
+                     :- * Project (33)
+                     :  +- * BroadcastHashJoin Inner BuildRight (32)
+                     :     :- * Project (26)
+                     :     :  +- * BroadcastHashJoin Inner BuildRight (25)
+                     :     :     :- SortMergeJoin LeftAnti (19)
+                     :     :     :  :- * Project (13)
+                     :     :     :  :  +- * SortMergeJoin LeftSemi (12)
+                     :     :     :  :     :- * Sort (6)
+                     :     :     :  :     :  +- Exchange (5)
+                     :     :     :  :     :     +- * Project (4)
+                     :     :     :  :     :        +- * Filter (3)
+                     :     :     :  :     :           +- * ColumnarToRow (2)
+                     :     :     :  :     :              +- Scan parquet default.catalog_sales (1)
+                     :     :     :  :     +- * Sort (11)
+                     :     :     :  :        +- Exchange (10)
+                     :     :     :  :           +- * Project (9)
+                     :     :     :  :              +- * ColumnarToRow (8)
+                     :     :     :  :                 +- Scan parquet default.catalog_sales (7)
+                     :     :     :  +- * Sort (18)
+                     :     :     :     +- Exchange (17)
+                     :     :     :        +- * Project (16)
+                     :     :     :           +- * ColumnarToRow (15)
+                     :     :     :              +- Scan parquet default.catalog_returns (14)
+                     :     :     +- BroadcastExchange (24)
+                     :     :        +- * Project (23)
+                     :     :           +- * Filter (22)
+                     :     :              +- * ColumnarToRow (21)
+                     :     :                 +- Scan parquet default.customer_address (20)
+                     :     +- BroadcastExchange (31)
+                     :        +- * Project (30)
+                     :           +- * Filter (29)
+                     :              +- * ColumnarToRow (28)
+                     :                 +- Scan parquet default.call_center (27)
+                     +- BroadcastExchange (38)
+                        +- * Project (37)
+                           +- * Filter (36)
+                              +- * ColumnarToRow (35)
+                                 +- Scan parquet default.date_dim (34)
+
+
+(1) Scan parquet default.catalog_sales
+Output [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_sales]
+PushedFilters: [IsNotNull(cs_ship_date_sk), IsNotNull(cs_ship_addr_sk), IsNotNull(cs_call_center_sk)]
+ReadSchema: struct<cs_ship_date_sk:int,cs_ship_addr_sk:int,cs_call_center_sk:int,cs_warehouse_sk:int,cs_order_number:int,cs_ext_ship_cost:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8]
+
+(3) Filter [codegen id : 1]
+Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8]
+Condition : ((isnotnull(cs_ship_date_sk#1) AND isnotnull(cs_ship_addr_sk#2)) AND isnotnull(cs_call_center_sk#3))
+
+(4) Project [codegen id : 1]
+Output [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7]
+Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8]
+
+(5) Exchange
+Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7]
+Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, [id=#9]
+
+(6) Sort [codegen id : 2]
+Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7]
+Arguments: [cs_order_number#5 ASC NULLS FIRST], false, 0
+
+(7) Scan parquet default.catalog_sales
+Output [3]: [cs_warehouse_sk#10, cs_order_number#11, cs_sold_date_sk#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_sales]
+ReadSchema: struct<cs_warehouse_sk:int,cs_order_number:int>
+
+(8) ColumnarToRow [codegen id : 3]
+Input [3]: [cs_warehouse_sk#10, cs_order_number#11, cs_sold_date_sk#12]
+
+(9) Project [codegen id : 3]
+Output [2]: [cs_warehouse_sk#10, cs_order_number#11]
+Input [3]: [cs_warehouse_sk#10, cs_order_number#11, cs_sold_date_sk#12]
+
+(10) Exchange
+Input [2]: [cs_warehouse_sk#10, cs_order_number#11]
+Arguments: hashpartitioning(cs_order_number#11, 5), ENSURE_REQUIREMENTS, [id=#13]
+
+(11) Sort [codegen id : 4]
+Input [2]: [cs_warehouse_sk#10, cs_order_number#11]
+Arguments: [cs_order_number#11 ASC NULLS FIRST], false, 0
+
+(12) SortMergeJoin [codegen id : 5]
+Left keys [1]: [cs_order_number#5]
+Right keys [1]: [cs_order_number#11]
+Join condition: NOT (cs_warehouse_sk#4 = cs_warehouse_sk#10)
+
+(13) Project [codegen id : 5]
+Output [6]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7]
+Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7]
+
+(14) Scan parquet default.catalog_returns
+Output [2]: [cr_order_number#14, cr_returned_date_sk#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+ReadSchema: struct<cr_order_number:int>
+
+(15) ColumnarToRow [codegen id : 6]
+Input [2]: [cr_order_number#14, cr_returned_date_sk#15]
+
+(16) Project [codegen id : 6]
+Output [1]: [cr_order_number#14]
+Input [2]: [cr_order_number#14, cr_returned_date_sk#15]
+
+(17) Exchange
+Input [1]: [cr_order_number#14]
+Arguments: hashpartitioning(cr_order_number#14, 5), ENSURE_REQUIREMENTS, [id=#16]
+
+(18) Sort [codegen id : 7]
+Input [1]: [cr_order_number#14]
+Arguments: [cr_order_number#14 ASC NULLS FIRST], false, 0
+
+(19) SortMergeJoin
+Left keys [1]: [cs_order_number#5]
+Right keys [1]: [cr_order_number#14]
+Join condition: None
+
+(20) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#17, ca_state#18]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(21) ColumnarToRow [codegen id : 8]
+Input [2]: [ca_address_sk#17, ca_state#18]
+
+(22) Filter [codegen id : 8]
+Input [2]: [ca_address_sk#17, ca_state#18]
+Condition : ((isnotnull(ca_state#18) AND (ca_state#18 = GA)) AND isnotnull(ca_address_sk#17))
+
+(23) Project [codegen id : 8]
+Output [1]: [ca_address_sk#17]
+Input [2]: [ca_address_sk#17, ca_state#18]
+
+(24) BroadcastExchange
+Input [1]: [ca_address_sk#17]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#19]
+
+(25) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_ship_addr_sk#2]
+Right keys [1]: [ca_address_sk#17]
+Join condition: None
+
+(26) Project [codegen id : 11]
+Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7]
+Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, ca_address_sk#17]
+
+(27) Scan parquet default.call_center
+Output [2]: [cc_call_center_sk#20, cc_county#21]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/call_center]
+PushedFilters: [IsNotNull(cc_county), EqualTo(cc_county,Williamson County), IsNotNull(cc_call_center_sk)]
+ReadSchema: struct<cc_call_center_sk:int,cc_county:string>
+
+(28) ColumnarToRow [codegen id : 9]
+Input [2]: [cc_call_center_sk#20, cc_county#21]
+
+(29) Filter [codegen id : 9]
+Input [2]: [cc_call_center_sk#20, cc_county#21]
+Condition : ((isnotnull(cc_county#21) AND (cc_county#21 = Williamson County)) AND isnotnull(cc_call_center_sk#20))
+
+(30) Project [codegen id : 9]
+Output [1]: [cc_call_center_sk#20]
+Input [2]: [cc_call_center_sk#20, cc_county#21]
+
+(31) BroadcastExchange
+Input [1]: [cc_call_center_sk#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#22]
+
+(32) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_call_center_sk#3]
+Right keys [1]: [cc_call_center_sk#20]
+Join condition: None
+
+(33) Project [codegen id : 11]
+Output [4]: [cs_ship_date_sk#1, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7]
+Input [6]: [cs_ship_date_sk#1, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cc_call_center_sk#20]
+
+(34) Scan parquet default.date_dim
+Output [2]: [d_date_sk#23, d_date#24]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2002-02-01), LessThanOrEqual(d_date,2002-04-02), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(35) ColumnarToRow [codegen id : 10]
+Input [2]: [d_date_sk#23, d_date#24]
+
+(36) Filter [codegen id : 10]
+Input [2]: [d_date_sk#23, d_date#24]
+Condition : (((isnotnull(d_date#24) AND (d_date#24 >= 2002-02-01)) AND (d_date#24 <= 2002-04-02)) AND isnotnull(d_date_sk#23))
+
+(37) Project [codegen id : 10]
+Output [1]: [d_date_sk#23]
+Input [2]: [d_date_sk#23, d_date#24]
+
+(38) BroadcastExchange
+Input [1]: [d_date_sk#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#25]
+
+(39) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_ship_date_sk#1]
+Right keys [1]: [d_date_sk#23]
+Join condition: None
+
+(40) Project [codegen id : 11]
+Output [3]: [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7]
+Input [5]: [cs_ship_date_sk#1, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, d_date_sk#23]
+
+(41) HashAggregate [codegen id : 11]
+Input [3]: [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7]
+Keys [1]: [cs_order_number#5]
+Functions [2]: [partial_sum(UnscaledValue(cs_ext_ship_cost#6)), partial_sum(UnscaledValue(cs_net_profit#7))]
+Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_ship_cost#6))#26, sum(UnscaledValue(cs_net_profit#7))#27]
+Results [3]: [cs_order_number#5, sum#28, sum#29]
+
+(42) HashAggregate [codegen id : 11]
+Input [3]: [cs_order_number#5, sum#28, sum#29]
+Keys [1]: [cs_order_number#5]
+Functions [2]: [merge_sum(UnscaledValue(cs_ext_ship_cost#6)), merge_sum(UnscaledValue(cs_net_profit#7))]
+Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_ship_cost#6))#26, sum(UnscaledValue(cs_net_profit#7))#27]
+Results [3]: [cs_order_number#5, sum#28, sum#29]
+
+(43) HashAggregate [codegen id : 11]
+Input [3]: [cs_order_number#5, sum#28, sum#29]
+Keys: []
+Functions [3]: [merge_sum(UnscaledValue(cs_ext_ship_cost#6)), merge_sum(UnscaledValue(cs_net_profit#7)), partial_count(distinct cs_order_number#5)]
+Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#26, sum(UnscaledValue(cs_net_profit#7))#27, count(cs_order_number#5)#30]
+Results [3]: [sum#28, sum#29, count#31]
+
+(44) Exchange
+Input [3]: [sum#28, sum#29, count#31]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#32]
+
+(45) HashAggregate [codegen id : 12]
+Input [3]: [sum#28, sum#29, count#31]
+Keys: []
+Functions [3]: [sum(UnscaledValue(cs_ext_ship_cost#6)), sum(UnscaledValue(cs_net_profit#7)), count(distinct cs_order_number#5)]
+Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#26, sum(UnscaledValue(cs_net_profit#7))#27, count(cs_order_number#5)#30]
+Results [3]: [count(cs_order_number#5)#30 AS order count #33, MakeDecimal(sum(UnscaledValue(cs_ext_ship_cost#6))#26,17,2) AS total shipping cost #34, MakeDecimal(sum(UnscaledValue(cs_net_profit#7))#27,17,2) AS total net profit #35]
+
+(46) Sort [codegen id : 12]
+Input [3]: [order count #33, total shipping cost #34, total net profit #35]
+Arguments: [order count #33 ASC NULLS FIRST], true, 0
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.sf100/simplified.txt
new file mode 100644
index 0000000000000..489200f5201eb
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.sf100/simplified.txt
@@ -0,0 +1,74 @@
+WholeStageCodegen (12)
+  Sort [order count ]
+    HashAggregate [sum,sum,count] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count]
+      InputAdapter
+        Exchange #1
+          WholeStageCodegen (11)
+            HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),sum,sum,count,sum,sum,count]
+              HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum]
+                HashAggregate [cs_order_number,cs_ext_ship_cost,cs_net_profit] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum]
+                  Project [cs_order_number,cs_ext_ship_cost,cs_net_profit]
+                    BroadcastHashJoin [cs_ship_date_sk,d_date_sk]
+                      Project [cs_ship_date_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit]
+                        BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk]
+                          Project [cs_ship_date_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit]
+                            BroadcastHashJoin [cs_ship_addr_sk,ca_address_sk]
+                              InputAdapter
+                                SortMergeJoin [cs_order_number,cr_order_number]
+                                  WholeStageCodegen (5)
+                                    Project [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit]
+                                      SortMergeJoin [cs_order_number,cs_order_number,cs_warehouse_sk,cs_warehouse_sk]
+                                        InputAdapter
+                                          WholeStageCodegen (2)
+                                            Sort [cs_order_number]
+                                              InputAdapter
+                                                Exchange [cs_order_number] #2
+                                                  WholeStageCodegen (1)
+                                                    Project [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit]
+                                                      Filter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk]
+                                        InputAdapter
+                                          WholeStageCodegen (4)
+                                            Sort [cs_order_number]
+                                              InputAdapter
+                                                Exchange [cs_order_number] #3
+                                                  WholeStageCodegen (3)
+                                                    Project [cs_warehouse_sk,cs_order_number]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk]
+                                  WholeStageCodegen (7)
+                                    Sort [cr_order_number]
+                                      InputAdapter
+                                        Exchange [cr_order_number] #4
+                                          WholeStageCodegen (6)
+                                            Project [cr_order_number]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.catalog_returns [cr_order_number,cr_returned_date_sk]
+                              InputAdapter
+                                BroadcastExchange #5
+                                  WholeStageCodegen (8)
+                                    Project [ca_address_sk]
+                                      Filter [ca_state,ca_address_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.customer_address [ca_address_sk,ca_state]
+                          InputAdapter
+                            BroadcastExchange #6
+                              WholeStageCodegen (9)
+                                Project [cc_call_center_sk]
+                                  Filter [cc_county,cc_call_center_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.call_center [cc_call_center_sk,cc_county]
+                      InputAdapter
+                        BroadcastExchange #7
+                          WholeStageCodegen (10)
+                            Project [d_date_sk]
+                              Filter [d_date,d_date_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.date_dim [d_date_sk,d_date]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/explain.txt
new file mode 100644
index 0000000000000..647824d3a9d75
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/explain.txt
@@ -0,0 +1,260 @@
+== Physical Plan ==
+* Sort (46)
++- * HashAggregate (45)
+   +- Exchange (44)
+      +- * HashAggregate (43)
+         +- * HashAggregate (42)
+            +- * HashAggregate (41)
+               +- * Project (40)
+                  +- * BroadcastHashJoin Inner BuildRight (39)
+                     :- * Project (33)
+                     :  +- * BroadcastHashJoin Inner BuildRight (32)
+                     :     :- * Project (26)
+                     :     :  +- * BroadcastHashJoin Inner BuildRight (25)
+                     :     :     :- SortMergeJoin LeftAnti (19)
+                     :     :     :  :- * Project (13)
+                     :     :     :  :  +- * SortMergeJoin LeftSemi (12)
+                     :     :     :  :     :- * Sort (6)
+                     :     :     :  :     :  +- Exchange (5)
+                     :     :     :  :     :     +- * Project (4)
+                     :     :     :  :     :        +- * Filter (3)
+                     :     :     :  :     :           +- * ColumnarToRow (2)
+                     :     :     :  :     :              +- Scan parquet default.catalog_sales (1)
+                     :     :     :  :     +- * Sort (11)
+                     :     :     :  :        +- Exchange (10)
+                     :     :     :  :           +- * Project (9)
+                     :     :     :  :              +- * ColumnarToRow (8)
+                     :     :     :  :                 +- Scan parquet default.catalog_sales (7)
+                     :     :     :  +- * Sort (18)
+                     :     :     :     +- Exchange (17)
+                     :     :     :        +- * Project (16)
+                     :     :     :           +- * ColumnarToRow (15)
+                     :     :     :              +- Scan parquet default.catalog_returns (14)
+                     :     :     +- BroadcastExchange (24)
+                     :     :        +- * Project (23)
+                     :     :           +- * Filter (22)
+                     :     :              +- * ColumnarToRow (21)
+                     :     :                 +- Scan parquet default.date_dim (20)
+                     :     +- BroadcastExchange (31)
+                     :        +- * Project (30)
+                     :           +- * Filter (29)
+                     :              +- * ColumnarToRow (28)
+                     :                 +- Scan parquet default.customer_address (27)
+                     +- BroadcastExchange (38)
+                        +- * Project (37)
+                           +- * Filter (36)
+                              +- * ColumnarToRow (35)
+                                 +- Scan parquet default.call_center (34)
+
+
+(1) Scan parquet default.catalog_sales
+Output [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_sales]
+PushedFilters: [IsNotNull(cs_ship_date_sk), IsNotNull(cs_ship_addr_sk), IsNotNull(cs_call_center_sk)]
+ReadSchema: struct<cs_ship_date_sk:int,cs_ship_addr_sk:int,cs_call_center_sk:int,cs_warehouse_sk:int,cs_order_number:int,cs_ext_ship_cost:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8]
+
+(3) Filter [codegen id : 1]
+Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8]
+Condition : ((isnotnull(cs_ship_date_sk#1) AND isnotnull(cs_ship_addr_sk#2)) AND isnotnull(cs_call_center_sk#3))
+
+(4) Project [codegen id : 1]
+Output [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7]
+Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8]
+
+(5) Exchange
+Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7]
+Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, [id=#9]
+
+(6) Sort [codegen id : 2]
+Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7]
+Arguments: [cs_order_number#5 ASC NULLS FIRST], false, 0
+
+(7) Scan parquet default.catalog_sales
+Output [3]: [cs_warehouse_sk#10, cs_order_number#11, cs_sold_date_sk#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_sales]
+ReadSchema: struct<cs_warehouse_sk:int,cs_order_number:int>
+
+(8) ColumnarToRow [codegen id : 3]
+Input [3]: [cs_warehouse_sk#10, cs_order_number#11, cs_sold_date_sk#12]
+
+(9) Project [codegen id : 3]
+Output [2]: [cs_warehouse_sk#10, cs_order_number#11]
+Input [3]: [cs_warehouse_sk#10, cs_order_number#11, cs_sold_date_sk#12]
+
+(10) Exchange
+Input [2]: [cs_warehouse_sk#10, cs_order_number#11]
+Arguments: hashpartitioning(cs_order_number#11, 5), ENSURE_REQUIREMENTS, [id=#13]
+
+(11) Sort [codegen id : 4]
+Input [2]: [cs_warehouse_sk#10, cs_order_number#11]
+Arguments: [cs_order_number#11 ASC NULLS FIRST], false, 0
+
+(12) SortMergeJoin [codegen id : 5]
+Left keys [1]: [cs_order_number#5]
+Right keys [1]: [cs_order_number#11]
+Join condition: NOT (cs_warehouse_sk#4 = cs_warehouse_sk#10)
+
+(13) Project [codegen id : 5]
+Output [6]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7]
+Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7]
+
+(14) Scan parquet default.catalog_returns
+Output [2]: [cr_order_number#14, cr_returned_date_sk#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+ReadSchema: struct<cr_order_number:int>
+
+(15) ColumnarToRow [codegen id : 6]
+Input [2]: [cr_order_number#14, cr_returned_date_sk#15]
+
+(16) Project [codegen id : 6]
+Output [1]: [cr_order_number#14]
+Input [2]: [cr_order_number#14, cr_returned_date_sk#15]
+
+(17) Exchange
+Input [1]: [cr_order_number#14]
+Arguments: hashpartitioning(cr_order_number#14, 5), ENSURE_REQUIREMENTS, [id=#16]
+
+(18) Sort [codegen id : 7]
+Input [1]: [cr_order_number#14]
+Arguments: [cr_order_number#14 ASC NULLS FIRST], false, 0
+
+(19) SortMergeJoin
+Left keys [1]: [cs_order_number#5]
+Right keys [1]: [cr_order_number#14]
+Join condition: None
+
+(20) Scan parquet default.date_dim
+Output [2]: [d_date_sk#17, d_date#18]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2002-02-01), LessThanOrEqual(d_date,2002-04-02), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(21) ColumnarToRow [codegen id : 8]
+Input [2]: [d_date_sk#17, d_date#18]
+
+(22) Filter [codegen id : 8]
+Input [2]: [d_date_sk#17, d_date#18]
+Condition : (((isnotnull(d_date#18) AND (d_date#18 >= 2002-02-01)) AND (d_date#18 <= 2002-04-02)) AND isnotnull(d_date_sk#17))
+
+(23) Project [codegen id : 8]
+Output [1]: [d_date_sk#17]
+Input [2]: [d_date_sk#17, d_date#18]
+
+(24) BroadcastExchange
+Input [1]: [d_date_sk#17]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#19]
+
+(25) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_ship_date_sk#1]
+Right keys [1]: [d_date_sk#17]
+Join condition: None
+
+(26) Project [codegen id : 11]
+Output [5]: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7]
+Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, d_date_sk#17]
+
+(27) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#20, ca_state#21]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(28) ColumnarToRow [codegen id : 9]
+Input [2]: [ca_address_sk#20, ca_state#21]
+
+(29) Filter [codegen id : 9]
+Input [2]: [ca_address_sk#20, ca_state#21]
+Condition : ((isnotnull(ca_state#21) AND (ca_state#21 = GA)) AND isnotnull(ca_address_sk#20))
+
+(30) Project [codegen id : 9]
+Output [1]: [ca_address_sk#20]
+Input [2]: [ca_address_sk#20, ca_state#21]
+
+(31) BroadcastExchange
+Input [1]: [ca_address_sk#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#22]
+
+(32) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_ship_addr_sk#2]
+Right keys [1]: [ca_address_sk#20]
+Join condition: None
+
+(33) Project [codegen id : 11]
+Output [4]: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7]
+Input [6]: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, ca_address_sk#20]
+
+(34) Scan parquet default.call_center
+Output [2]: [cc_call_center_sk#23, cc_county#24]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/call_center]
+PushedFilters: [IsNotNull(cc_county), EqualTo(cc_county,Williamson County), IsNotNull(cc_call_center_sk)]
+ReadSchema: struct<cc_call_center_sk:int,cc_county:string>
+
+(35) ColumnarToRow [codegen id : 10]
+Input [2]: [cc_call_center_sk#23, cc_county#24]
+
+(36) Filter [codegen id : 10]
+Input [2]: [cc_call_center_sk#23, cc_county#24]
+Condition : ((isnotnull(cc_county#24) AND (cc_county#24 = Williamson County)) AND isnotnull(cc_call_center_sk#23))
+
+(37) Project [codegen id : 10]
+Output [1]: [cc_call_center_sk#23]
+Input [2]: [cc_call_center_sk#23, cc_county#24]
+
+(38) BroadcastExchange
+Input [1]: [cc_call_center_sk#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#25]
+
+(39) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_call_center_sk#3]
+Right keys [1]: [cc_call_center_sk#23]
+Join condition: None
+
+(40) Project [codegen id : 11]
+Output [3]: [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7]
+Input [5]: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cc_call_center_sk#23]
+
+(41) HashAggregate [codegen id : 11]
+Input [3]: [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7]
+Keys [1]: [cs_order_number#5]
+Functions [2]: [partial_sum(UnscaledValue(cs_ext_ship_cost#6)), partial_sum(UnscaledValue(cs_net_profit#7))]
+Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_ship_cost#6))#26, sum(UnscaledValue(cs_net_profit#7))#27]
+Results [3]: [cs_order_number#5, sum#28, sum#29]
+
+(42) HashAggregate [codegen id : 11]
+Input [3]: [cs_order_number#5, sum#28, sum#29]
+Keys [1]: [cs_order_number#5]
+Functions [2]: [merge_sum(UnscaledValue(cs_ext_ship_cost#6)), merge_sum(UnscaledValue(cs_net_profit#7))]
+Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_ship_cost#6))#26, sum(UnscaledValue(cs_net_profit#7))#27]
+Results [3]: [cs_order_number#5, sum#28, sum#29]
+
+(43) HashAggregate [codegen id : 11]
+Input [3]: [cs_order_number#5, sum#28, sum#29]
+Keys: []
+Functions [3]: [merge_sum(UnscaledValue(cs_ext_ship_cost#6)), merge_sum(UnscaledValue(cs_net_profit#7)), partial_count(distinct cs_order_number#5)]
+Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#26, sum(UnscaledValue(cs_net_profit#7))#27, count(cs_order_number#5)#30]
+Results [3]: [sum#28, sum#29, count#31]
+
+(44) Exchange
+Input [3]: [sum#28, sum#29, count#31]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#32]
+
+(45) HashAggregate [codegen id : 12]
+Input [3]: [sum#28, sum#29, count#31]
+Keys: []
+Functions [3]: [sum(UnscaledValue(cs_ext_ship_cost#6)), sum(UnscaledValue(cs_net_profit#7)), count(distinct cs_order_number#5)]
+Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#26, sum(UnscaledValue(cs_net_profit#7))#27, count(cs_order_number#5)#30]
+Results [3]: [count(cs_order_number#5)#30 AS order count #33, MakeDecimal(sum(UnscaledValue(cs_ext_ship_cost#6))#26,17,2) AS total shipping cost #34, MakeDecimal(sum(UnscaledValue(cs_net_profit#7))#27,17,2) AS total net profit #35]
+
+(46) Sort [codegen id : 12]
+Input [3]: [order count #33, total shipping cost #34, total net profit #35]
+Arguments: [order count #33 ASC NULLS FIRST], true, 0
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/simplified.txt
new file mode 100644
index 0000000000000..c7ead9a46797a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/simplified.txt
@@ -0,0 +1,74 @@
+WholeStageCodegen (12)
+  Sort [order count ]
+    HashAggregate [sum,sum,count] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count]
+      InputAdapter
+        Exchange #1
+          WholeStageCodegen (11)
+            HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),sum,sum,count,sum,sum,count]
+              HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum]
+                HashAggregate [cs_order_number,cs_ext_ship_cost,cs_net_profit] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum]
+                  Project [cs_order_number,cs_ext_ship_cost,cs_net_profit]
+                    BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk]
+                      Project [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit]
+                        BroadcastHashJoin [cs_ship_addr_sk,ca_address_sk]
+                          Project [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit]
+                            BroadcastHashJoin [cs_ship_date_sk,d_date_sk]
+                              InputAdapter
+                                SortMergeJoin [cs_order_number,cr_order_number]
+                                  WholeStageCodegen (5)
+                                    Project [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit]
+                                      SortMergeJoin [cs_order_number,cs_order_number,cs_warehouse_sk,cs_warehouse_sk]
+                                        InputAdapter
+                                          WholeStageCodegen (2)
+                                            Sort [cs_order_number]
+                                              InputAdapter
+                                                Exchange [cs_order_number] #2
+                                                  WholeStageCodegen (1)
+                                                    Project [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit]
+                                                      Filter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk]
+                                        InputAdapter
+                                          WholeStageCodegen (4)
+                                            Sort [cs_order_number]
+                                              InputAdapter
+                                                Exchange [cs_order_number] #3
+                                                  WholeStageCodegen (3)
+                                                    Project [cs_warehouse_sk,cs_order_number]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk]
+                                  WholeStageCodegen (7)
+                                    Sort [cr_order_number]
+                                      InputAdapter
+                                        Exchange [cr_order_number] #4
+                                          WholeStageCodegen (6)
+                                            Project [cr_order_number]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.catalog_returns [cr_order_number,cr_returned_date_sk]
+                              InputAdapter
+                                BroadcastExchange #5
+                                  WholeStageCodegen (8)
+                                    Project [d_date_sk]
+                                      Filter [d_date,d_date_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.date_dim [d_date_sk,d_date]
+                          InputAdapter
+                            BroadcastExchange #6
+                              WholeStageCodegen (9)
+                                Project [ca_address_sk]
+                                  Filter [ca_state,ca_address_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.customer_address [ca_address_sk,ca_state]
+                      InputAdapter
+                        BroadcastExchange #7
+                          WholeStageCodegen (10)
+                            Project [cc_call_center_sk]
+                              Filter [cc_county,cc_call_center_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.call_center [cc_call_center_sk,cc_county]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.sf100/explain.txt
new file mode 100644
index 0000000000000..cc6b8786e2002
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.sf100/explain.txt
@@ -0,0 +1,336 @@
+== Physical Plan ==
+TakeOrderedAndProject (57)
++- * HashAggregate (56)
+   +- Exchange (55)
+      +- * HashAggregate (54)
+         +- * Project (53)
+            +- * SortMergeJoin Inner (52)
+               :- * Sort (43)
+               :  +- Exchange (42)
+               :     +- * Project (41)
+               :        +- * SortMergeJoin Inner (40)
+               :           :- * Sort (27)
+               :           :  +- Exchange (26)
+               :           :     +- * Project (25)
+               :           :        +- * SortMergeJoin Inner (24)
+               :           :           :- * Sort (18)
+               :           :           :  +- Exchange (17)
+               :           :           :     +- * Project (16)
+               :           :           :        +- * BroadcastHashJoin Inner BuildRight (15)
+               :           :           :           :- * Project (10)
+               :           :           :           :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :           :           :           :     :- * Filter (3)
+               :           :           :           :     :  +- * ColumnarToRow (2)
+               :           :           :           :     :     +- Scan parquet default.store_sales (1)
+               :           :           :           :     +- BroadcastExchange (8)
+               :           :           :           :        +- * Project (7)
+               :           :           :           :           +- * Filter (6)
+               :           :           :           :              +- * ColumnarToRow (5)
+               :           :           :           :                 +- Scan parquet default.date_dim (4)
+               :           :           :           +- BroadcastExchange (14)
+               :           :           :              +- * Filter (13)
+               :           :           :                 +- * ColumnarToRow (12)
+               :           :           :                    +- Scan parquet default.store (11)
+               :           :           +- * Sort (23)
+               :           :              +- Exchange (22)
+               :           :                 +- * Filter (21)
+               :           :                    +- * ColumnarToRow (20)
+               :           :                       +- Scan parquet default.item (19)
+               :           +- * Sort (39)
+               :              +- Exchange (38)
+               :                 +- * Project (37)
+               :                    +- * BroadcastHashJoin Inner BuildRight (36)
+               :                       :- * Filter (30)
+               :                       :  +- * ColumnarToRow (29)
+               :                       :     +- Scan parquet default.store_returns (28)
+               :                       +- BroadcastExchange (35)
+               :                          +- * Project (34)
+               :                             +- * Filter (33)
+               :                                +- * ColumnarToRow (32)
+               :                                   +- Scan parquet default.date_dim (31)
+               +- * Sort (51)
+                  +- Exchange (50)
+                     +- * Project (49)
+                        +- * BroadcastHashJoin Inner BuildRight (48)
+                           :- * Filter (46)
+                           :  +- * ColumnarToRow (45)
+                           :     +- Scan parquet default.catalog_sales (44)
+                           +- ReusedExchange (47)
+
+
+(1) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_quantity:int>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6]
+
+(3) Filter [codegen id : 3]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6]
+Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3))
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#8, d_quarter_name#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_quarter_name), EqualTo(d_quarter_name,2001Q1), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_quarter_name:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#8, d_quarter_name#9]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#8, d_quarter_name#9]
+Condition : ((isnotnull(d_quarter_name#9) AND (d_quarter_name#9 = 2001Q1)) AND isnotnull(d_date_sk#8))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#8]
+Input [2]: [d_date_sk#8, d_quarter_name#9]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#6]
+Right keys [1]: [d_date_sk#8]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5]
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, d_date_sk#8]
+
+(11) Scan parquet default.store
+Output [2]: [s_store_sk#11, s_state#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_state:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [s_store_sk#11, s_state#12]
+
+(13) Filter [codegen id : 2]
+Input [2]: [s_store_sk#11, s_state#12]
+Condition : isnotnull(s_store_sk#11)
+
+(14) BroadcastExchange
+Input [2]: [s_store_sk#11, s_state#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#13]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#11]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#4, ss_quantity#5, s_state#12]
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, s_store_sk#11, s_state#12]
+
+(17) Exchange
+Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#4, ss_quantity#5, s_state#12]
+Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#14]
+
+(18) Sort [codegen id : 4]
+Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#4, ss_quantity#5, s_state#12]
+Arguments: [ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(19) Scan parquet default.item
+Output [3]: [i_item_sk#15, i_item_id#16, i_item_desc#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_item_desc:string>
+
+(20) ColumnarToRow [codegen id : 5]
+Input [3]: [i_item_sk#15, i_item_id#16, i_item_desc#17]
+
+(21) Filter [codegen id : 5]
+Input [3]: [i_item_sk#15, i_item_id#16, i_item_desc#17]
+Condition : isnotnull(i_item_sk#15)
+
+(22) Exchange
+Input [3]: [i_item_sk#15, i_item_id#16, i_item_desc#17]
+Arguments: hashpartitioning(i_item_sk#15, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(23) Sort [codegen id : 6]
+Input [3]: [i_item_sk#15, i_item_id#16, i_item_desc#17]
+Arguments: [i_item_sk#15 ASC NULLS FIRST], false, 0
+
+(24) SortMergeJoin [codegen id : 7]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#15]
+Join condition: None
+
+(25) Project [codegen id : 7]
+Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#4, ss_quantity#5, s_state#12, i_item_id#16, i_item_desc#17]
+Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#4, ss_quantity#5, s_state#12, i_item_sk#15, i_item_id#16, i_item_desc#17]
+
+(26) Exchange
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#4, ss_quantity#5, s_state#12, i_item_id#16, i_item_desc#17]
+Arguments: hashpartitioning(ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, [id=#19]
+
+(27) Sort [codegen id : 8]
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#4, ss_quantity#5, s_state#12, i_item_id#16, i_item_desc#17]
+Arguments: [ss_customer_sk#2 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST], false, 0
+
+(28) Scan parquet default.store_returns
+Output [5]: [sr_item_sk#20, sr_customer_sk#21, sr_ticket_number#22, sr_return_quantity#23, sr_returned_date_sk#24]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(sr_returned_date_sk#24), dynamicpruningexpression(sr_returned_date_sk#24 IN dynamicpruning#25)]
+PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)]
+ReadSchema: struct<sr_item_sk:int,sr_customer_sk:int,sr_ticket_number:int,sr_return_quantity:int>
+
+(29) ColumnarToRow [codegen id : 10]
+Input [5]: [sr_item_sk#20, sr_customer_sk#21, sr_ticket_number#22, sr_return_quantity#23, sr_returned_date_sk#24]
+
+(30) Filter [codegen id : 10]
+Input [5]: [sr_item_sk#20, sr_customer_sk#21, sr_ticket_number#22, sr_return_quantity#23, sr_returned_date_sk#24]
+Condition : ((isnotnull(sr_customer_sk#21) AND isnotnull(sr_item_sk#20)) AND isnotnull(sr_ticket_number#22))
+
+(31) Scan parquet default.date_dim
+Output [2]: [d_date_sk#26, d_quarter_name#27]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [In(d_quarter_name, [2001Q1,2001Q2,2001Q3]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_quarter_name:string>
+
+(32) ColumnarToRow [codegen id : 9]
+Input [2]: [d_date_sk#26, d_quarter_name#27]
+
+(33) Filter [codegen id : 9]
+Input [2]: [d_date_sk#26, d_quarter_name#27]
+Condition : (d_quarter_name#27 IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#26))
+
+(34) Project [codegen id : 9]
+Output [1]: [d_date_sk#26]
+Input [2]: [d_date_sk#26, d_quarter_name#27]
+
+(35) BroadcastExchange
+Input [1]: [d_date_sk#26]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#28]
+
+(36) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [sr_returned_date_sk#24]
+Right keys [1]: [d_date_sk#26]
+Join condition: None
+
+(37) Project [codegen id : 10]
+Output [4]: [sr_item_sk#20, sr_customer_sk#21, sr_ticket_number#22, sr_return_quantity#23]
+Input [6]: [sr_item_sk#20, sr_customer_sk#21, sr_ticket_number#22, sr_return_quantity#23, sr_returned_date_sk#24, d_date_sk#26]
+
+(38) Exchange
+Input [4]: [sr_item_sk#20, sr_customer_sk#21, sr_ticket_number#22, sr_return_quantity#23]
+Arguments: hashpartitioning(sr_customer_sk#21, sr_item_sk#20, sr_ticket_number#22, 5), ENSURE_REQUIREMENTS, [id=#29]
+
+(39) Sort [codegen id : 11]
+Input [4]: [sr_item_sk#20, sr_customer_sk#21, sr_ticket_number#22, sr_return_quantity#23]
+Arguments: [sr_customer_sk#21 ASC NULLS FIRST, sr_item_sk#20 ASC NULLS FIRST, sr_ticket_number#22 ASC NULLS FIRST], false, 0
+
+(40) SortMergeJoin [codegen id : 12]
+Left keys [3]: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4]
+Right keys [3]: [sr_customer_sk#21, sr_item_sk#20, sr_ticket_number#22]
+Join condition: None
+
+(41) Project [codegen id : 12]
+Output [7]: [ss_quantity#5, s_state#12, i_item_id#16, i_item_desc#17, sr_item_sk#20, sr_customer_sk#21, sr_return_quantity#23]
+Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#4, ss_quantity#5, s_state#12, i_item_id#16, i_item_desc#17, sr_item_sk#20, sr_customer_sk#21, sr_ticket_number#22, sr_return_quantity#23]
+
+(42) Exchange
+Input [7]: [ss_quantity#5, s_state#12, i_item_id#16, i_item_desc#17, sr_item_sk#20, sr_customer_sk#21, sr_return_quantity#23]
+Arguments: hashpartitioning(sr_customer_sk#21, sr_item_sk#20, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(43) Sort [codegen id : 13]
+Input [7]: [ss_quantity#5, s_state#12, i_item_id#16, i_item_desc#17, sr_item_sk#20, sr_customer_sk#21, sr_return_quantity#23]
+Arguments: [sr_customer_sk#21 ASC NULLS FIRST, sr_item_sk#20 ASC NULLS FIRST], false, 0
+
+(44) Scan parquet default.catalog_sales
+Output [4]: [cs_bill_customer_sk#31, cs_item_sk#32, cs_quantity#33, cs_sold_date_sk#34]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#34), dynamicpruningexpression(cs_sold_date_sk#34 IN dynamicpruning#25)]
+PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_item_sk:int,cs_quantity:int>
+
+(45) ColumnarToRow [codegen id : 15]
+Input [4]: [cs_bill_customer_sk#31, cs_item_sk#32, cs_quantity#33, cs_sold_date_sk#34]
+
+(46) Filter [codegen id : 15]
+Input [4]: [cs_bill_customer_sk#31, cs_item_sk#32, cs_quantity#33, cs_sold_date_sk#34]
+Condition : (isnotnull(cs_bill_customer_sk#31) AND isnotnull(cs_item_sk#32))
+
+(47) ReusedExchange [Reuses operator id: 35]
+Output [1]: [d_date_sk#35]
+
+(48) BroadcastHashJoin [codegen id : 15]
+Left keys [1]: [cs_sold_date_sk#34]
+Right keys [1]: [d_date_sk#35]
+Join condition: None
+
+(49) Project [codegen id : 15]
+Output [3]: [cs_bill_customer_sk#31, cs_item_sk#32, cs_quantity#33]
+Input [5]: [cs_bill_customer_sk#31, cs_item_sk#32, cs_quantity#33, cs_sold_date_sk#34, d_date_sk#35]
+
+(50) Exchange
+Input [3]: [cs_bill_customer_sk#31, cs_item_sk#32, cs_quantity#33]
+Arguments: hashpartitioning(cs_bill_customer_sk#31, cs_item_sk#32, 5), ENSURE_REQUIREMENTS, [id=#36]
+
+(51) Sort [codegen id : 16]
+Input [3]: [cs_bill_customer_sk#31, cs_item_sk#32, cs_quantity#33]
+Arguments: [cs_bill_customer_sk#31 ASC NULLS FIRST, cs_item_sk#32 ASC NULLS FIRST], false, 0
+
+(52) SortMergeJoin [codegen id : 17]
+Left keys [2]: [sr_customer_sk#21, sr_item_sk#20]
+Right keys [2]: [cs_bill_customer_sk#31, cs_item_sk#32]
+Join condition: None
+
+(53) Project [codegen id : 17]
+Output [6]: [ss_quantity#5, sr_return_quantity#23, cs_quantity#33, s_state#12, i_item_id#16, i_item_desc#17]
+Input [10]: [ss_quantity#5, s_state#12, i_item_id#16, i_item_desc#17, sr_item_sk#20, sr_customer_sk#21, sr_return_quantity#23, cs_bill_customer_sk#31, cs_item_sk#32, cs_quantity#33]
+
+(54) HashAggregate [codegen id : 17]
+Input [6]: [ss_quantity#5, sr_return_quantity#23, cs_quantity#33, s_state#12, i_item_id#16, i_item_desc#17]
+Keys [3]: [i_item_id#16, i_item_desc#17, s_state#12]
+Functions [9]: [partial_count(ss_quantity#5), partial_avg(ss_quantity#5), partial_stddev_samp(cast(ss_quantity#5 as double)), partial_count(sr_return_quantity#23), partial_avg(sr_return_quantity#23), partial_stddev_samp(cast(sr_return_quantity#23 as double)), partial_count(cs_quantity#33), partial_avg(cs_quantity#33), partial_stddev_samp(cast(cs_quantity#33 as double))]
+Aggregate Attributes [18]: [count#37, sum#38, count#39, n#40, avg#41, m2#42, count#43, sum#44, count#45, n#46, avg#47, m2#48, count#49, sum#50, count#51, n#52, avg#53, m2#54]
+Results [21]: [i_item_id#16, i_item_desc#17, s_state#12, count#55, sum#56, count#57, n#58, avg#59, m2#60, count#61, sum#62, count#63, n#64, avg#65, m2#66, count#67, sum#68, count#69, n#70, avg#71, m2#72]
+
+(55) Exchange
+Input [21]: [i_item_id#16, i_item_desc#17, s_state#12, count#55, sum#56, count#57, n#58, avg#59, m2#60, count#61, sum#62, count#63, n#64, avg#65, m2#66, count#67, sum#68, count#69, n#70, avg#71, m2#72]
+Arguments: hashpartitioning(i_item_id#16, i_item_desc#17, s_state#12, 5), ENSURE_REQUIREMENTS, [id=#73]
+
+(56) HashAggregate [codegen id : 18]
+Input [21]: [i_item_id#16, i_item_desc#17, s_state#12, count#55, sum#56, count#57, n#58, avg#59, m2#60, count#61, sum#62, count#63, n#64, avg#65, m2#66, count#67, sum#68, count#69, n#70, avg#71, m2#72]
+Keys [3]: [i_item_id#16, i_item_desc#17, s_state#12]
+Functions [9]: [count(ss_quantity#5), avg(ss_quantity#5), stddev_samp(cast(ss_quantity#5 as double)), count(sr_return_quantity#23), avg(sr_return_quantity#23), stddev_samp(cast(sr_return_quantity#23 as double)), count(cs_quantity#33), avg(cs_quantity#33), stddev_samp(cast(cs_quantity#33 as double))]
+Aggregate Attributes [9]: [count(ss_quantity#5)#74, avg(ss_quantity#5)#75, stddev_samp(cast(ss_quantity#5 as double))#76, count(sr_return_quantity#23)#77, avg(sr_return_quantity#23)#78, stddev_samp(cast(sr_return_quantity#23 as double))#79, count(cs_quantity#33)#80, avg(cs_quantity#33)#81, stddev_samp(cast(cs_quantity#33 as double))#82]
+Results [15]: [i_item_id#16, i_item_desc#17, s_state#12, count(ss_quantity#5)#74 AS store_sales_quantitycount#83, avg(ss_quantity#5)#75 AS store_sales_quantityave#84, stddev_samp(cast(ss_quantity#5 as double))#76 AS store_sales_quantitystdev#85, (stddev_samp(cast(ss_quantity#5 as double))#76 / avg(ss_quantity#5)#75) AS store_sales_quantitycov#86, count(sr_return_quantity#23)#77 AS as_store_returns_quantitycount#87, avg(sr_return_quantity#23)#78 AS as_store_returns_quantityave#88, stddev_samp(cast(sr_return_quantity#23 as double))#79 AS as_store_returns_quantitystdev#89, (stddev_samp(cast(sr_return_quantity#23 as double))#79 / avg(sr_return_quantity#23)#78) AS store_returns_quantitycov#90, count(cs_quantity#33)#80 AS catalog_sales_quantitycount#91, avg(cs_quantity#33)#81 AS catalog_sales_quantityave#92, (stddev_samp(cast(cs_quantity#33 as double))#82 / avg(cs_quantity#33)#81) AS catalog_sales_quantitystdev#93, (stddev_samp(cast(cs_quantity#33 as double))#82 / avg(cs_quantity#33)#81) AS catalog_sales_quantitycov#94]
+
+(57) TakeOrderedAndProject
+Input [15]: [i_item_id#16, i_item_desc#17, s_state#12, store_sales_quantitycount#83, store_sales_quantityave#84, store_sales_quantitystdev#85, store_sales_quantitycov#86, as_store_returns_quantitycount#87, as_store_returns_quantityave#88, as_store_returns_quantitystdev#89, store_returns_quantitycov#90, catalog_sales_quantitycount#91, catalog_sales_quantityave#92, catalog_sales_quantitystdev#93, catalog_sales_quantitycov#94]
+Arguments: 100, [i_item_id#16 ASC NULLS FIRST, i_item_desc#17 ASC NULLS FIRST, s_state#12 ASC NULLS FIRST], [i_item_id#16, i_item_desc#17, s_state#12, store_sales_quantitycount#83, store_sales_quantityave#84, store_sales_quantitystdev#85, store_sales_quantitycov#86, as_store_returns_quantitycount#87, as_store_returns_quantityave#88, as_store_returns_quantitystdev#89, store_returns_quantitycov#90, catalog_sales_quantitycount#91, catalog_sales_quantityave#92, catalog_sales_quantitystdev#93, catalog_sales_quantitycov#94]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7
+ReusedExchange (58)
+
+
+(58) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#8]
+
+Subquery:2 Hosting operator id = 28 Hosting Expression = sr_returned_date_sk#24 IN dynamicpruning#25
+ReusedExchange (59)
+
+
+(59) ReusedExchange [Reuses operator id: 35]
+Output [1]: [d_date_sk#26]
+
+Subquery:3 Hosting operator id = 44 Hosting Expression = cs_sold_date_sk#34 IN dynamicpruning#25
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.sf100/simplified.txt
new file mode 100644
index 0000000000000..cda77e94f5b7f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.sf100/simplified.txt
@@ -0,0 +1,103 @@
+TakeOrderedAndProject [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov]
+  WholeStageCodegen (18)
+    HashAggregate [i_item_id,i_item_desc,s_state,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] [count(ss_quantity),avg(ss_quantity),stddev_samp(cast(ss_quantity as double)),count(sr_return_quantity),avg(sr_return_quantity),stddev_samp(cast(sr_return_quantity as double)),count(cs_quantity),avg(cs_quantity),stddev_samp(cast(cs_quantity as double)),store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2]
+      InputAdapter
+        Exchange [i_item_id,i_item_desc,s_state] #1
+          WholeStageCodegen (17)
+            HashAggregate [i_item_id,i_item_desc,s_state,ss_quantity,sr_return_quantity,cs_quantity] [count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2]
+              Project [ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_id,i_item_desc]
+                SortMergeJoin [sr_customer_sk,sr_item_sk,cs_bill_customer_sk,cs_item_sk]
+                  InputAdapter
+                    WholeStageCodegen (13)
+                      Sort [sr_customer_sk,sr_item_sk]
+                        InputAdapter
+                          Exchange [sr_customer_sk,sr_item_sk] #2
+                            WholeStageCodegen (12)
+                              Project [ss_quantity,s_state,i_item_id,i_item_desc,sr_item_sk,sr_customer_sk,sr_return_quantity]
+                                SortMergeJoin [ss_customer_sk,ss_item_sk,ss_ticket_number,sr_customer_sk,sr_item_sk,sr_ticket_number]
+                                  InputAdapter
+                                    WholeStageCodegen (8)
+                                      Sort [ss_customer_sk,ss_item_sk,ss_ticket_number]
+                                        InputAdapter
+                                          Exchange [ss_customer_sk,ss_item_sk,ss_ticket_number] #3
+                                            WholeStageCodegen (7)
+                                              Project [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,s_state,i_item_id,i_item_desc]
+                                                SortMergeJoin [ss_item_sk,i_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (4)
+                                                      Sort [ss_item_sk]
+                                                        InputAdapter
+                                                          Exchange [ss_item_sk] #4
+                                                            WholeStageCodegen (3)
+                                                              Project [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,s_state]
+                                                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                                  Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity]
+                                                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                      Filter [ss_customer_sk,ss_item_sk,ss_ticket_number,ss_store_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk]
+                                                                              SubqueryBroadcast [d_date_sk] #1
+                                                                                ReusedExchange [d_date_sk] #5
+                                                                      InputAdapter
+                                                                        BroadcastExchange #5
+                                                                          WholeStageCodegen (1)
+                                                                            Project [d_date_sk]
+                                                                              Filter [d_quarter_name,d_date_sk]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.date_dim [d_date_sk,d_quarter_name]
+                                                                  InputAdapter
+                                                                    BroadcastExchange #6
+                                                                      WholeStageCodegen (2)
+                                                                        Filter [s_store_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.store [s_store_sk,s_state]
+                                                  InputAdapter
+                                                    WholeStageCodegen (6)
+                                                      Sort [i_item_sk]
+                                                        InputAdapter
+                                                          Exchange [i_item_sk] #7
+                                                            WholeStageCodegen (5)
+                                                              Filter [i_item_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.item [i_item_sk,i_item_id,i_item_desc]
+                                  InputAdapter
+                                    WholeStageCodegen (11)
+                                      Sort [sr_customer_sk,sr_item_sk,sr_ticket_number]
+                                        InputAdapter
+                                          Exchange [sr_customer_sk,sr_item_sk,sr_ticket_number] #8
+                                            WholeStageCodegen (10)
+                                              Project [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity]
+                                                BroadcastHashJoin [sr_returned_date_sk,d_date_sk]
+                                                  Filter [sr_customer_sk,sr_item_sk,sr_ticket_number]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk]
+                                                          SubqueryBroadcast [d_date_sk] #2
+                                                            ReusedExchange [d_date_sk] #9
+                                                  InputAdapter
+                                                    BroadcastExchange #9
+                                                      WholeStageCodegen (9)
+                                                        Project [d_date_sk]
+                                                          Filter [d_quarter_name,d_date_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.date_dim [d_date_sk,d_quarter_name]
+                  InputAdapter
+                    WholeStageCodegen (16)
+                      Sort [cs_bill_customer_sk,cs_item_sk]
+                        InputAdapter
+                          Exchange [cs_bill_customer_sk,cs_item_sk] #10
+                            WholeStageCodegen (15)
+                              Project [cs_bill_customer_sk,cs_item_sk,cs_quantity]
+                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                  Filter [cs_bill_customer_sk,cs_item_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk]
+                                          ReusedSubquery [d_date_sk] #2
+                                  InputAdapter
+                                    ReusedExchange [d_date_sk] #9
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/explain.txt
new file mode 100644
index 0000000000000..467bcc3b429c9
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/explain.txt
@@ -0,0 +1,291 @@
+== Physical Plan ==
+TakeOrderedAndProject (48)
++- * HashAggregate (47)
+   +- Exchange (46)
+      +- * HashAggregate (45)
+         +- * Project (44)
+            +- * BroadcastHashJoin Inner BuildRight (43)
+               :- * Project (38)
+               :  +- * BroadcastHashJoin Inner BuildRight (37)
+               :     :- * Project (32)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (31)
+               :     :     :- * Project (29)
+               :     :     :  +- * BroadcastHashJoin Inner BuildRight (28)
+               :     :     :     :- * Project (22)
+               :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (21)
+               :     :     :     :     :- * Project (15)
+               :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (14)
+               :     :     :     :     :     :- * Project (9)
+               :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+               :     :     :     :     :     :     :- * Filter (3)
+               :     :     :     :     :     :     :  +- * ColumnarToRow (2)
+               :     :     :     :     :     :     :     +- Scan parquet default.store_sales (1)
+               :     :     :     :     :     :     +- BroadcastExchange (7)
+               :     :     :     :     :     :        +- * Filter (6)
+               :     :     :     :     :     :           +- * ColumnarToRow (5)
+               :     :     :     :     :     :              +- Scan parquet default.store_returns (4)
+               :     :     :     :     :     +- BroadcastExchange (13)
+               :     :     :     :     :        +- * Filter (12)
+               :     :     :     :     :           +- * ColumnarToRow (11)
+               :     :     :     :     :              +- Scan parquet default.catalog_sales (10)
+               :     :     :     :     +- BroadcastExchange (20)
+               :     :     :     :        +- * Project (19)
+               :     :     :     :           +- * Filter (18)
+               :     :     :     :              +- * ColumnarToRow (17)
+               :     :     :     :                 +- Scan parquet default.date_dim (16)
+               :     :     :     +- BroadcastExchange (27)
+               :     :     :        +- * Project (26)
+               :     :     :           +- * Filter (25)
+               :     :     :              +- * ColumnarToRow (24)
+               :     :     :                 +- Scan parquet default.date_dim (23)
+               :     :     +- ReusedExchange (30)
+               :     +- BroadcastExchange (36)
+               :        +- * Filter (35)
+               :           +- * ColumnarToRow (34)
+               :              +- Scan parquet default.store (33)
+               +- BroadcastExchange (42)
+                  +- * Filter (41)
+                     +- * ColumnarToRow (40)
+                        +- Scan parquet default.item (39)
+
+
+(1) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_quantity:int>
+
+(2) ColumnarToRow [codegen id : 8]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6]
+
+(3) Filter [codegen id : 8]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6]
+Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3))
+
+(4) Scan parquet default.store_returns
+Output [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(sr_returned_date_sk#12), dynamicpruningexpression(sr_returned_date_sk#12 IN dynamicpruning#13)]
+PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)]
+ReadSchema: struct<sr_item_sk:int,sr_customer_sk:int,sr_ticket_number:int,sr_return_quantity:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12]
+
+(6) Filter [codegen id : 1]
+Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12]
+Condition : ((isnotnull(sr_customer_sk#9) AND isnotnull(sr_item_sk#8)) AND isnotnull(sr_ticket_number#10))
+
+(7) BroadcastExchange
+Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12]
+Arguments: HashedRelationBroadcastMode(List(input[1, int, false], input[0, int, false], input[2, int, false]),false), [id=#14]
+
+(8) BroadcastHashJoin [codegen id : 8]
+Left keys [3]: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4]
+Right keys [3]: [sr_customer_sk#9, sr_item_sk#8, sr_ticket_number#10]
+Join condition: None
+
+(9) Project [codegen id : 8]
+Output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12]
+Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12]
+
+(10) Scan parquet default.catalog_sales
+Output [4]: [cs_bill_customer_sk#15, cs_item_sk#16, cs_quantity#17, cs_sold_date_sk#18]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_sold_date_sk#18 IN dynamicpruning#13)]
+PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_item_sk:int,cs_quantity:int>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [4]: [cs_bill_customer_sk#15, cs_item_sk#16, cs_quantity#17, cs_sold_date_sk#18]
+
+(12) Filter [codegen id : 2]
+Input [4]: [cs_bill_customer_sk#15, cs_item_sk#16, cs_quantity#17, cs_sold_date_sk#18]
+Condition : (isnotnull(cs_bill_customer_sk#15) AND isnotnull(cs_item_sk#16))
+
+(13) BroadcastExchange
+Input [4]: [cs_bill_customer_sk#15, cs_item_sk#16, cs_quantity#17, cs_sold_date_sk#18]
+Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[1, int, false] as bigint) & 4294967295))),false), [id=#19]
+
+(14) BroadcastHashJoin [codegen id : 8]
+Left keys [2]: [sr_customer_sk#9, sr_item_sk#8]
+Right keys [2]: [cs_bill_customer_sk#15, cs_item_sk#16]
+Join condition: None
+
+(15) Project [codegen id : 8]
+Output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#17, cs_sold_date_sk#18]
+Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12, cs_bill_customer_sk#15, cs_item_sk#16, cs_quantity#17, cs_sold_date_sk#18]
+
+(16) Scan parquet default.date_dim
+Output [2]: [d_date_sk#20, d_quarter_name#21]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_quarter_name), EqualTo(d_quarter_name,2001Q1), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_quarter_name:string>
+
+(17) ColumnarToRow [codegen id : 3]
+Input [2]: [d_date_sk#20, d_quarter_name#21]
+
+(18) Filter [codegen id : 3]
+Input [2]: [d_date_sk#20, d_quarter_name#21]
+Condition : ((isnotnull(d_quarter_name#21) AND (d_quarter_name#21 = 2001Q1)) AND isnotnull(d_date_sk#20))
+
+(19) Project [codegen id : 3]
+Output [1]: [d_date_sk#20]
+Input [2]: [d_date_sk#20, d_quarter_name#21]
+
+(20) BroadcastExchange
+Input [1]: [d_date_sk#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#22]
+
+(21) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_sold_date_sk#6]
+Right keys [1]: [d_date_sk#20]
+Join condition: None
+
+(22) Project [codegen id : 8]
+Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#17, cs_sold_date_sk#18]
+Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#17, cs_sold_date_sk#18, d_date_sk#20]
+
+(23) Scan parquet default.date_dim
+Output [2]: [d_date_sk#23, d_quarter_name#24]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [In(d_quarter_name, [2001Q1,2001Q2,2001Q3]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_quarter_name:string>
+
+(24) ColumnarToRow [codegen id : 4]
+Input [2]: [d_date_sk#23, d_quarter_name#24]
+
+(25) Filter [codegen id : 4]
+Input [2]: [d_date_sk#23, d_quarter_name#24]
+Condition : (d_quarter_name#24 IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#23))
+
+(26) Project [codegen id : 4]
+Output [1]: [d_date_sk#23]
+Input [2]: [d_date_sk#23, d_quarter_name#24]
+
+(27) BroadcastExchange
+Input [1]: [d_date_sk#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#25]
+
+(28) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [sr_returned_date_sk#12]
+Right keys [1]: [d_date_sk#23]
+Join condition: None
+
+(29) Project [codegen id : 8]
+Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#17, cs_sold_date_sk#18]
+Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#17, cs_sold_date_sk#18, d_date_sk#23]
+
+(30) ReusedExchange [Reuses operator id: 27]
+Output [1]: [d_date_sk#26]
+
+(31) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [cs_sold_date_sk#18]
+Right keys [1]: [d_date_sk#26]
+Join condition: None
+
+(32) Project [codegen id : 8]
+Output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#17]
+Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#17, cs_sold_date_sk#18, d_date_sk#26]
+
+(33) Scan parquet default.store
+Output [2]: [s_store_sk#27, s_state#28]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_state:string>
+
+(34) ColumnarToRow [codegen id : 6]
+Input [2]: [s_store_sk#27, s_state#28]
+
+(35) Filter [codegen id : 6]
+Input [2]: [s_store_sk#27, s_state#28]
+Condition : isnotnull(s_store_sk#27)
+
+(36) BroadcastExchange
+Input [2]: [s_store_sk#27, s_state#28]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#29]
+
+(37) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#27]
+Join condition: None
+
+(38) Project [codegen id : 8]
+Output [5]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#17, s_state#28]
+Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#17, s_store_sk#27, s_state#28]
+
+(39) Scan parquet default.item
+Output [3]: [i_item_sk#30, i_item_id#31, i_item_desc#32]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_item_desc:string>
+
+(40) ColumnarToRow [codegen id : 7]
+Input [3]: [i_item_sk#30, i_item_id#31, i_item_desc#32]
+
+(41) Filter [codegen id : 7]
+Input [3]: [i_item_sk#30, i_item_id#31, i_item_desc#32]
+Condition : isnotnull(i_item_sk#30)
+
+(42) BroadcastExchange
+Input [3]: [i_item_sk#30, i_item_id#31, i_item_desc#32]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#33]
+
+(43) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#30]
+Join condition: None
+
+(44) Project [codegen id : 8]
+Output [6]: [ss_quantity#5, sr_return_quantity#11, cs_quantity#17, s_state#28, i_item_id#31, i_item_desc#32]
+Input [8]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#17, s_state#28, i_item_sk#30, i_item_id#31, i_item_desc#32]
+
+(45) HashAggregate [codegen id : 8]
+Input [6]: [ss_quantity#5, sr_return_quantity#11, cs_quantity#17, s_state#28, i_item_id#31, i_item_desc#32]
+Keys [3]: [i_item_id#31, i_item_desc#32, s_state#28]
+Functions [9]: [partial_count(ss_quantity#5), partial_avg(ss_quantity#5), partial_stddev_samp(cast(ss_quantity#5 as double)), partial_count(sr_return_quantity#11), partial_avg(sr_return_quantity#11), partial_stddev_samp(cast(sr_return_quantity#11 as double)), partial_count(cs_quantity#17), partial_avg(cs_quantity#17), partial_stddev_samp(cast(cs_quantity#17 as double))]
+Aggregate Attributes [18]: [count#34, sum#35, count#36, n#37, avg#38, m2#39, count#40, sum#41, count#42, n#43, avg#44, m2#45, count#46, sum#47, count#48, n#49, avg#50, m2#51]
+Results [21]: [i_item_id#31, i_item_desc#32, s_state#28, count#52, sum#53, count#54, n#55, avg#56, m2#57, count#58, sum#59, count#60, n#61, avg#62, m2#63, count#64, sum#65, count#66, n#67, avg#68, m2#69]
+
+(46) Exchange
+Input [21]: [i_item_id#31, i_item_desc#32, s_state#28, count#52, sum#53, count#54, n#55, avg#56, m2#57, count#58, sum#59, count#60, n#61, avg#62, m2#63, count#64, sum#65, count#66, n#67, avg#68, m2#69]
+Arguments: hashpartitioning(i_item_id#31, i_item_desc#32, s_state#28, 5), ENSURE_REQUIREMENTS, [id=#70]
+
+(47) HashAggregate [codegen id : 9]
+Input [21]: [i_item_id#31, i_item_desc#32, s_state#28, count#52, sum#53, count#54, n#55, avg#56, m2#57, count#58, sum#59, count#60, n#61, avg#62, m2#63, count#64, sum#65, count#66, n#67, avg#68, m2#69]
+Keys [3]: [i_item_id#31, i_item_desc#32, s_state#28]
+Functions [9]: [count(ss_quantity#5), avg(ss_quantity#5), stddev_samp(cast(ss_quantity#5 as double)), count(sr_return_quantity#11), avg(sr_return_quantity#11), stddev_samp(cast(sr_return_quantity#11 as double)), count(cs_quantity#17), avg(cs_quantity#17), stddev_samp(cast(cs_quantity#17 as double))]
+Aggregate Attributes [9]: [count(ss_quantity#5)#71, avg(ss_quantity#5)#72, stddev_samp(cast(ss_quantity#5 as double))#73, count(sr_return_quantity#11)#74, avg(sr_return_quantity#11)#75, stddev_samp(cast(sr_return_quantity#11 as double))#76, count(cs_quantity#17)#77, avg(cs_quantity#17)#78, stddev_samp(cast(cs_quantity#17 as double))#79]
+Results [15]: [i_item_id#31, i_item_desc#32, s_state#28, count(ss_quantity#5)#71 AS store_sales_quantitycount#80, avg(ss_quantity#5)#72 AS store_sales_quantityave#81, stddev_samp(cast(ss_quantity#5 as double))#73 AS store_sales_quantitystdev#82, (stddev_samp(cast(ss_quantity#5 as double))#73 / avg(ss_quantity#5)#72) AS store_sales_quantitycov#83, count(sr_return_quantity#11)#74 AS as_store_returns_quantitycount#84, avg(sr_return_quantity#11)#75 AS as_store_returns_quantityave#85, stddev_samp(cast(sr_return_quantity#11 as double))#76 AS as_store_returns_quantitystdev#86, (stddev_samp(cast(sr_return_quantity#11 as double))#76 / avg(sr_return_quantity#11)#75) AS store_returns_quantitycov#87, count(cs_quantity#17)#77 AS catalog_sales_quantitycount#88, avg(cs_quantity#17)#78 AS catalog_sales_quantityave#89, (stddev_samp(cast(cs_quantity#17 as double))#79 / avg(cs_quantity#17)#78) AS catalog_sales_quantitystdev#90, (stddev_samp(cast(cs_quantity#17 as double))#79 / avg(cs_quantity#17)#78) AS catalog_sales_quantitycov#91]
+
+(48) TakeOrderedAndProject
+Input [15]: [i_item_id#31, i_item_desc#32, s_state#28, store_sales_quantitycount#80, store_sales_quantityave#81, store_sales_quantitystdev#82, store_sales_quantitycov#83, as_store_returns_quantitycount#84, as_store_returns_quantityave#85, as_store_returns_quantitystdev#86, store_returns_quantitycov#87, catalog_sales_quantitycount#88, catalog_sales_quantityave#89, catalog_sales_quantitystdev#90, catalog_sales_quantitycov#91]
+Arguments: 100, [i_item_id#31 ASC NULLS FIRST, i_item_desc#32 ASC NULLS FIRST, s_state#28 ASC NULLS FIRST], [i_item_id#31, i_item_desc#32, s_state#28, store_sales_quantitycount#80, store_sales_quantityave#81, store_sales_quantitystdev#82, store_sales_quantitycov#83, as_store_returns_quantitycount#84, as_store_returns_quantityave#85, as_store_returns_quantitystdev#86, store_returns_quantitycov#87, catalog_sales_quantitycount#88, catalog_sales_quantityave#89, catalog_sales_quantitystdev#90, catalog_sales_quantitycov#91]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7
+ReusedExchange (49)
+
+
+(49) ReusedExchange [Reuses operator id: 20]
+Output [1]: [d_date_sk#20]
+
+Subquery:2 Hosting operator id = 4 Hosting Expression = sr_returned_date_sk#12 IN dynamicpruning#13
+ReusedExchange (50)
+
+
+(50) ReusedExchange [Reuses operator id: 27]
+Output [1]: [d_date_sk#23]
+
+Subquery:3 Hosting operator id = 10 Hosting Expression = cs_sold_date_sk#18 IN dynamicpruning#13
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/simplified.txt
new file mode 100644
index 0000000000000..6d6fab87255f4
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/simplified.txt
@@ -0,0 +1,76 @@
+TakeOrderedAndProject [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov]
+  WholeStageCodegen (9)
+    HashAggregate [i_item_id,i_item_desc,s_state,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] [count(ss_quantity),avg(ss_quantity),stddev_samp(cast(ss_quantity as double)),count(sr_return_quantity),avg(sr_return_quantity),stddev_samp(cast(sr_return_quantity as double)),count(cs_quantity),avg(cs_quantity),stddev_samp(cast(cs_quantity as double)),store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2]
+      InputAdapter
+        Exchange [i_item_id,i_item_desc,s_state] #1
+          WholeStageCodegen (8)
+            HashAggregate [i_item_id,i_item_desc,s_state,ss_quantity,sr_return_quantity,cs_quantity] [count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2]
+              Project [ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_id,i_item_desc]
+                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                  Project [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state]
+                    BroadcastHashJoin [ss_store_sk,s_store_sk]
+                      Project [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity]
+                        BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                          Project [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk]
+                            BroadcastHashJoin [sr_returned_date_sk,d_date_sk]
+                              Project [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk]
+                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                  Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk]
+                                    BroadcastHashJoin [sr_customer_sk,sr_item_sk,cs_bill_customer_sk,cs_item_sk]
+                                      Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk]
+                                        BroadcastHashJoin [ss_customer_sk,ss_item_sk,ss_ticket_number,sr_customer_sk,sr_item_sk,sr_ticket_number]
+                                          Filter [ss_customer_sk,ss_item_sk,ss_ticket_number,ss_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk]
+                                                  SubqueryBroadcast [d_date_sk] #1
+                                                    ReusedExchange [d_date_sk] #2
+                                          InputAdapter
+                                            BroadcastExchange #3
+                                              WholeStageCodegen (1)
+                                                Filter [sr_customer_sk,sr_item_sk,sr_ticket_number]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk]
+                                                        SubqueryBroadcast [d_date_sk] #2
+                                                          ReusedExchange [d_date_sk] #4
+                                      InputAdapter
+                                        BroadcastExchange #5
+                                          WholeStageCodegen (2)
+                                            Filter [cs_bill_customer_sk,cs_item_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk]
+                                                    ReusedSubquery [d_date_sk] #2
+                                  InputAdapter
+                                    BroadcastExchange #2
+                                      WholeStageCodegen (3)
+                                        Project [d_date_sk]
+                                          Filter [d_quarter_name,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_quarter_name]
+                              InputAdapter
+                                BroadcastExchange #4
+                                  WholeStageCodegen (4)
+                                    Project [d_date_sk]
+                                      Filter [d_quarter_name,d_date_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.date_dim [d_date_sk,d_quarter_name]
+                          InputAdapter
+                            ReusedExchange [d_date_sk] #4
+                      InputAdapter
+                        BroadcastExchange #6
+                          WholeStageCodegen (6)
+                            Filter [s_store_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.store [s_store_sk,s_state]
+                  InputAdapter
+                    BroadcastExchange #7
+                      WholeStageCodegen (7)
+                        Filter [i_item_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.item [i_item_sk,i_item_id,i_item_desc]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.sf100/explain.txt
new file mode 100644
index 0000000000000..f7927aad003a2
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.sf100/explain.txt
@@ -0,0 +1,305 @@
+== Physical Plan ==
+TakeOrderedAndProject (53)
++- * HashAggregate (52)
+   +- Exchange (51)
+      +- * HashAggregate (50)
+         +- * Expand (49)
+            +- * Project (48)
+               +- * SortMergeJoin Inner (47)
+                  :- * Sort (25)
+                  :  +- Exchange (24)
+                  :     +- * Project (23)
+                  :        +- * BroadcastHashJoin Inner BuildRight (22)
+                  :           :- * Project (17)
+                  :           :  +- * BroadcastHashJoin Inner BuildRight (16)
+                  :           :     :- * Project (10)
+                  :           :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+                  :           :     :     :- * Filter (3)
+                  :           :     :     :  +- * ColumnarToRow (2)
+                  :           :     :     :     +- Scan parquet default.catalog_sales (1)
+                  :           :     :     +- BroadcastExchange (8)
+                  :           :     :        +- * Project (7)
+                  :           :     :           +- * Filter (6)
+                  :           :     :              +- * ColumnarToRow (5)
+                  :           :     :                 +- Scan parquet default.customer_demographics (4)
+                  :           :     +- BroadcastExchange (15)
+                  :           :        +- * Project (14)
+                  :           :           +- * Filter (13)
+                  :           :              +- * ColumnarToRow (12)
+                  :           :                 +- Scan parquet default.date_dim (11)
+                  :           +- BroadcastExchange (21)
+                  :              +- * Filter (20)
+                  :                 +- * ColumnarToRow (19)
+                  :                    +- Scan parquet default.item (18)
+                  +- * Sort (46)
+                     +- Exchange (45)
+                        +- * Project (44)
+                           +- * SortMergeJoin Inner (43)
+                              :- * Sort (37)
+                              :  +- Exchange (36)
+                              :     +- * Project (35)
+                              :        +- * BroadcastHashJoin Inner BuildRight (34)
+                              :           :- * Project (29)
+                              :           :  +- * Filter (28)
+                              :           :     +- * ColumnarToRow (27)
+                              :           :        +- Scan parquet default.customer (26)
+                              :           +- BroadcastExchange (33)
+                              :              +- * Filter (32)
+                              :                 +- * ColumnarToRow (31)
+                              :                    +- Scan parquet default.customer_address (30)
+                              +- * Sort (42)
+                                 +- Exchange (41)
+                                    +- * Filter (40)
+                                       +- * ColumnarToRow (39)
+                                          +- Scan parquet default.customer_demographics (38)
+
+
+(1) Scan parquet default.catalog_sales
+Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#9), dynamicpruningexpression(cs_sold_date_sk#9 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_bill_cdemo_sk:int,cs_item_sk:int,cs_quantity:int,cs_list_price:decimal(7,2),cs_sales_price:decimal(7,2),cs_coupon_amt:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9]
+
+(3) Filter [codegen id : 4]
+Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9]
+Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3))
+
+(4) Scan parquet default.customer_demographics
+Output [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_education_status), EqualTo(cd_gender,F), EqualTo(cd_education_status,Unknown             ), IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_education_status:string,cd_dep_count:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14]
+
+(6) Filter [codegen id : 1]
+Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14]
+Condition : ((((isnotnull(cd_gender#12) AND isnotnull(cd_education_status#13)) AND (cd_gender#12 = F)) AND (cd_education_status#13 = Unknown             )) AND isnotnull(cd_demo_sk#11))
+
+(7) Project [codegen id : 1]
+Output [2]: [cd_demo_sk#11, cd_dep_count#14]
+Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14]
+
+(8) BroadcastExchange
+Input [2]: [cd_demo_sk#11, cd_dep_count#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#15]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_bill_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#11]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14]
+Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#11, cd_dep_count#14]
+
+(11) Scan parquet default.date_dim
+Output [2]: [d_date_sk#16, d_year#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#16, d_year#17]
+
+(13) Filter [codegen id : 2]
+Input [2]: [d_date_sk#16, d_year#17]
+Condition : ((isnotnull(d_year#17) AND (d_year#17 = 1998)) AND isnotnull(d_date_sk#16))
+
+(14) Project [codegen id : 2]
+Output [1]: [d_date_sk#16]
+Input [2]: [d_date_sk#16, d_year#17]
+
+(15) BroadcastExchange
+Input [1]: [d_date_sk#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_sold_date_sk#9]
+Right keys [1]: [d_date_sk#16]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [8]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14]
+Input [10]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, d_date_sk#16]
+
+(18) Scan parquet default.item
+Output [2]: [i_item_sk#19, i_item_id#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [2]: [i_item_sk#19, i_item_id#20]
+
+(20) Filter [codegen id : 3]
+Input [2]: [i_item_sk#19, i_item_id#20]
+Condition : isnotnull(i_item_sk#19)
+
+(21) BroadcastExchange
+Input [2]: [i_item_sk#19, i_item_id#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#21]
+
+(22) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_item_sk#3]
+Right keys [1]: [i_item_sk#19]
+Join condition: None
+
+(23) Project [codegen id : 4]
+Output [8]: [cs_bill_customer_sk#1, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, i_item_id#20]
+Input [10]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, i_item_sk#19, i_item_id#20]
+
+(24) Exchange
+Input [8]: [cs_bill_customer_sk#1, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, i_item_id#20]
+Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(25) Sort [codegen id : 5]
+Input [8]: [cs_bill_customer_sk#1, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, i_item_id#20]
+Arguments: [cs_bill_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(26) Scan parquet default.customer
+Output [5]: [c_customer_sk#23, c_current_cdemo_sk#24, c_current_addr_sk#25, c_birth_month#26, c_birth_year#27]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [In(c_birth_month, [1,6,8,9,12,2]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_addr_sk:int,c_birth_month:int,c_birth_year:int>
+
+(27) ColumnarToRow [codegen id : 7]
+Input [5]: [c_customer_sk#23, c_current_cdemo_sk#24, c_current_addr_sk#25, c_birth_month#26, c_birth_year#27]
+
+(28) Filter [codegen id : 7]
+Input [5]: [c_customer_sk#23, c_current_cdemo_sk#24, c_current_addr_sk#25, c_birth_month#26, c_birth_year#27]
+Condition : (((c_birth_month#26 IN (1,6,8,9,12,2) AND isnotnull(c_customer_sk#23)) AND isnotnull(c_current_cdemo_sk#24)) AND isnotnull(c_current_addr_sk#25))
+
+(29) Project [codegen id : 7]
+Output [4]: [c_customer_sk#23, c_current_cdemo_sk#24, c_current_addr_sk#25, c_birth_year#27]
+Input [5]: [c_customer_sk#23, c_current_cdemo_sk#24, c_current_addr_sk#25, c_birth_month#26, c_birth_year#27]
+
+(30) Scan parquet default.customer_address
+Output [4]: [ca_address_sk#28, ca_county#29, ca_state#30, ca_country#31]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [In(ca_state, [MS,IN,ND,OK,NM,VA]), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_county:string,ca_state:string,ca_country:string>
+
+(31) ColumnarToRow [codegen id : 6]
+Input [4]: [ca_address_sk#28, ca_county#29, ca_state#30, ca_country#31]
+
+(32) Filter [codegen id : 6]
+Input [4]: [ca_address_sk#28, ca_county#29, ca_state#30, ca_country#31]
+Condition : (ca_state#30 IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#28))
+
+(33) BroadcastExchange
+Input [4]: [ca_address_sk#28, ca_county#29, ca_state#30, ca_country#31]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#32]
+
+(34) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [c_current_addr_sk#25]
+Right keys [1]: [ca_address_sk#28]
+Join condition: None
+
+(35) Project [codegen id : 7]
+Output [6]: [c_customer_sk#23, c_current_cdemo_sk#24, c_birth_year#27, ca_county#29, ca_state#30, ca_country#31]
+Input [8]: [c_customer_sk#23, c_current_cdemo_sk#24, c_current_addr_sk#25, c_birth_year#27, ca_address_sk#28, ca_county#29, ca_state#30, ca_country#31]
+
+(36) Exchange
+Input [6]: [c_customer_sk#23, c_current_cdemo_sk#24, c_birth_year#27, ca_county#29, ca_state#30, ca_country#31]
+Arguments: hashpartitioning(c_current_cdemo_sk#24, 5), ENSURE_REQUIREMENTS, [id=#33]
+
+(37) Sort [codegen id : 8]
+Input [6]: [c_customer_sk#23, c_current_cdemo_sk#24, c_birth_year#27, ca_county#29, ca_state#30, ca_country#31]
+Arguments: [c_current_cdemo_sk#24 ASC NULLS FIRST], false, 0
+
+(38) Scan parquet default.customer_demographics
+Output [1]: [cd_demo_sk#34]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int>
+
+(39) ColumnarToRow [codegen id : 9]
+Input [1]: [cd_demo_sk#34]
+
+(40) Filter [codegen id : 9]
+Input [1]: [cd_demo_sk#34]
+Condition : isnotnull(cd_demo_sk#34)
+
+(41) Exchange
+Input [1]: [cd_demo_sk#34]
+Arguments: hashpartitioning(cd_demo_sk#34, 5), ENSURE_REQUIREMENTS, [id=#35]
+
+(42) Sort [codegen id : 10]
+Input [1]: [cd_demo_sk#34]
+Arguments: [cd_demo_sk#34 ASC NULLS FIRST], false, 0
+
+(43) SortMergeJoin [codegen id : 11]
+Left keys [1]: [c_current_cdemo_sk#24]
+Right keys [1]: [cd_demo_sk#34]
+Join condition: None
+
+(44) Project [codegen id : 11]
+Output [5]: [c_customer_sk#23, c_birth_year#27, ca_county#29, ca_state#30, ca_country#31]
+Input [7]: [c_customer_sk#23, c_current_cdemo_sk#24, c_birth_year#27, ca_county#29, ca_state#30, ca_country#31, cd_demo_sk#34]
+
+(45) Exchange
+Input [5]: [c_customer_sk#23, c_birth_year#27, ca_county#29, ca_state#30, ca_country#31]
+Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, [id=#36]
+
+(46) Sort [codegen id : 12]
+Input [5]: [c_customer_sk#23, c_birth_year#27, ca_county#29, ca_state#30, ca_country#31]
+Arguments: [c_customer_sk#23 ASC NULLS FIRST], false, 0
+
+(47) SortMergeJoin [codegen id : 13]
+Left keys [1]: [cs_bill_customer_sk#1]
+Right keys [1]: [c_customer_sk#23]
+Join condition: None
+
+(48) Project [codegen id : 13]
+Output [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#27, i_item_id#20, ca_country#31, ca_state#30, ca_county#29]
+Input [13]: [cs_bill_customer_sk#1, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, i_item_id#20, c_customer_sk#23, c_birth_year#27, ca_county#29, ca_state#30, ca_country#31]
+
+(49) Expand [codegen id : 13]
+Input [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#27, i_item_id#20, ca_country#31, ca_state#30, ca_county#29]
+Arguments: [List(cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#27, i_item_id#20, ca_country#31, ca_state#30, ca_county#29, 0), List(cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#27, i_item_id#20, ca_country#31, ca_state#30, null, 1), List(cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#27, i_item_id#20, ca_country#31, null, null, 3), List(cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#27, i_item_id#20, null, null, null, 7), List(cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#27, null, null, null, null, 15)], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#27, i_item_id#37, ca_country#38, ca_state#39, ca_county#40, spark_grouping_id#41]
+
+(50) HashAggregate [codegen id : 13]
+Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#27, i_item_id#37, ca_country#38, ca_state#39, ca_county#40, spark_grouping_id#41]
+Keys [5]: [i_item_id#37, ca_country#38, ca_state#39, ca_county#40, spark_grouping_id#41]
+Functions [7]: [partial_avg(cast(cs_quantity#4 as decimal(12,2))), partial_avg(cast(cs_list_price#5 as decimal(12,2))), partial_avg(cast(cs_coupon_amt#7 as decimal(12,2))), partial_avg(cast(cs_sales_price#6 as decimal(12,2))), partial_avg(cast(cs_net_profit#8 as decimal(12,2))), partial_avg(cast(c_birth_year#27 as decimal(12,2))), partial_avg(cast(cd_dep_count#14 as decimal(12,2)))]
+Aggregate Attributes [14]: [sum#42, count#43, sum#44, count#45, sum#46, count#47, sum#48, count#49, sum#50, count#51, sum#52, count#53, sum#54, count#55]
+Results [19]: [i_item_id#37, ca_country#38, ca_state#39, ca_county#40, spark_grouping_id#41, sum#56, count#57, sum#58, count#59, sum#60, count#61, sum#62, count#63, sum#64, count#65, sum#66, count#67, sum#68, count#69]
+
+(51) Exchange
+Input [19]: [i_item_id#37, ca_country#38, ca_state#39, ca_county#40, spark_grouping_id#41, sum#56, count#57, sum#58, count#59, sum#60, count#61, sum#62, count#63, sum#64, count#65, sum#66, count#67, sum#68, count#69]
+Arguments: hashpartitioning(i_item_id#37, ca_country#38, ca_state#39, ca_county#40, spark_grouping_id#41, 5), ENSURE_REQUIREMENTS, [id=#70]
+
+(52) HashAggregate [codegen id : 14]
+Input [19]: [i_item_id#37, ca_country#38, ca_state#39, ca_county#40, spark_grouping_id#41, sum#56, count#57, sum#58, count#59, sum#60, count#61, sum#62, count#63, sum#64, count#65, sum#66, count#67, sum#68, count#69]
+Keys [5]: [i_item_id#37, ca_country#38, ca_state#39, ca_county#40, spark_grouping_id#41]
+Functions [7]: [avg(cast(cs_quantity#4 as decimal(12,2))), avg(cast(cs_list_price#5 as decimal(12,2))), avg(cast(cs_coupon_amt#7 as decimal(12,2))), avg(cast(cs_sales_price#6 as decimal(12,2))), avg(cast(cs_net_profit#8 as decimal(12,2))), avg(cast(c_birth_year#27 as decimal(12,2))), avg(cast(cd_dep_count#14 as decimal(12,2)))]
+Aggregate Attributes [7]: [avg(cast(cs_quantity#4 as decimal(12,2)))#71, avg(cast(cs_list_price#5 as decimal(12,2)))#72, avg(cast(cs_coupon_amt#7 as decimal(12,2)))#73, avg(cast(cs_sales_price#6 as decimal(12,2)))#74, avg(cast(cs_net_profit#8 as decimal(12,2)))#75, avg(cast(c_birth_year#27 as decimal(12,2)))#76, avg(cast(cd_dep_count#14 as decimal(12,2)))#77]
+Results [11]: [i_item_id#37, ca_country#38, ca_state#39, ca_county#40, avg(cast(cs_quantity#4 as decimal(12,2)))#71 AS agg1#78, avg(cast(cs_list_price#5 as decimal(12,2)))#72 AS agg2#79, avg(cast(cs_coupon_amt#7 as decimal(12,2)))#73 AS agg3#80, avg(cast(cs_sales_price#6 as decimal(12,2)))#74 AS agg4#81, avg(cast(cs_net_profit#8 as decimal(12,2)))#75 AS agg5#82, avg(cast(c_birth_year#27 as decimal(12,2)))#76 AS agg6#83, avg(cast(cd_dep_count#14 as decimal(12,2)))#77 AS agg7#84]
+
+(53) TakeOrderedAndProject
+Input [11]: [i_item_id#37, ca_country#38, ca_state#39, ca_county#40, agg1#78, agg2#79, agg3#80, agg4#81, agg5#82, agg6#83, agg7#84]
+Arguments: 100, [ca_country#38 ASC NULLS FIRST, ca_state#39 ASC NULLS FIRST, ca_county#40 ASC NULLS FIRST, i_item_id#37 ASC NULLS FIRST], [i_item_id#37, ca_country#38, ca_state#39, ca_county#40, agg1#78, agg2#79, agg3#80, agg4#81, agg5#82, agg6#83, agg7#84]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#9 IN dynamicpruning#10
+ReusedExchange (54)
+
+
+(54) ReusedExchange [Reuses operator id: 15]
+Output [1]: [d_date_sk#16]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.sf100/simplified.txt
new file mode 100644
index 0000000000000..29f977c87cb10
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.sf100/simplified.txt
@@ -0,0 +1,89 @@
+TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7]
+  WholeStageCodegen (14)
+    HashAggregate [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(cast(cs_quantity as decimal(12,2))),avg(cast(cs_list_price as decimal(12,2))),avg(cast(cs_coupon_amt as decimal(12,2))),avg(cast(cs_sales_price as decimal(12,2))),avg(cast(cs_net_profit as decimal(12,2))),avg(cast(c_birth_year as decimal(12,2))),avg(cast(cd_dep_count as decimal(12,2))),agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+      InputAdapter
+        Exchange [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] #1
+          WholeStageCodegen (13)
+            HashAggregate [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+              Expand [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county]
+                Project [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county]
+                  SortMergeJoin [cs_bill_customer_sk,c_customer_sk]
+                    InputAdapter
+                      WholeStageCodegen (5)
+                        Sort [cs_bill_customer_sk]
+                          InputAdapter
+                            Exchange [cs_bill_customer_sk] #2
+                              WholeStageCodegen (4)
+                                Project [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,i_item_id]
+                                  BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                    Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count]
+                                      BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                        Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count]
+                                          BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk]
+                                            Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #1
+                                                      ReusedExchange [d_date_sk] #3
+                                            InputAdapter
+                                              BroadcastExchange #4
+                                                WholeStageCodegen (1)
+                                                  Project [cd_demo_sk,cd_dep_count]
+                                                    Filter [cd_gender,cd_education_status,cd_demo_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count]
+                                        InputAdapter
+                                          BroadcastExchange #3
+                                            WholeStageCodegen (2)
+                                              Project [d_date_sk]
+                                                Filter [d_year,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_year]
+                                    InputAdapter
+                                      BroadcastExchange #5
+                                        WholeStageCodegen (3)
+                                          Filter [i_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.item [i_item_sk,i_item_id]
+                    InputAdapter
+                      WholeStageCodegen (12)
+                        Sort [c_customer_sk]
+                          InputAdapter
+                            Exchange [c_customer_sk] #6
+                              WholeStageCodegen (11)
+                                Project [c_customer_sk,c_birth_year,ca_county,ca_state,ca_country]
+                                  SortMergeJoin [c_current_cdemo_sk,cd_demo_sk]
+                                    InputAdapter
+                                      WholeStageCodegen (8)
+                                        Sort [c_current_cdemo_sk]
+                                          InputAdapter
+                                            Exchange [c_current_cdemo_sk] #7
+                                              WholeStageCodegen (7)
+                                                Project [c_customer_sk,c_current_cdemo_sk,c_birth_year,ca_county,ca_state,ca_country]
+                                                  BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                                                    Project [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year]
+                                                      Filter [c_birth_month,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year]
+                                                    InputAdapter
+                                                      BroadcastExchange #8
+                                                        WholeStageCodegen (6)
+                                                          Filter [ca_state,ca_address_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.customer_address [ca_address_sk,ca_county,ca_state,ca_country]
+                                    InputAdapter
+                                      WholeStageCodegen (10)
+                                        Sort [cd_demo_sk]
+                                          InputAdapter
+                                            Exchange [cd_demo_sk] #9
+                                              WholeStageCodegen (9)
+                                                Filter [cd_demo_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.customer_demographics [cd_demo_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/explain.txt
new file mode 100644
index 0000000000000..ed68f03734b13
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/explain.txt
@@ -0,0 +1,275 @@
+== Physical Plan ==
+TakeOrderedAndProject (47)
++- * HashAggregate (46)
+   +- Exchange (45)
+      +- * HashAggregate (44)
+         +- * Expand (43)
+            +- * Project (42)
+               +- * BroadcastHashJoin Inner BuildRight (41)
+                  :- * Project (36)
+                  :  +- * BroadcastHashJoin Inner BuildRight (35)
+                  :     :- * Project (29)
+                  :     :  +- * BroadcastHashJoin Inner BuildRight (28)
+                  :     :     :- * Project (23)
+                  :     :     :  +- * BroadcastHashJoin Inner BuildRight (22)
+                  :     :     :     :- * Project (17)
+                  :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (16)
+                  :     :     :     :     :- * Project (10)
+                  :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+                  :     :     :     :     :     :- * Filter (3)
+                  :     :     :     :     :     :  +- * ColumnarToRow (2)
+                  :     :     :     :     :     :     +- Scan parquet default.catalog_sales (1)
+                  :     :     :     :     :     +- BroadcastExchange (8)
+                  :     :     :     :     :        +- * Project (7)
+                  :     :     :     :     :           +- * Filter (6)
+                  :     :     :     :     :              +- * ColumnarToRow (5)
+                  :     :     :     :     :                 +- Scan parquet default.customer_demographics (4)
+                  :     :     :     :     +- BroadcastExchange (15)
+                  :     :     :     :        +- * Project (14)
+                  :     :     :     :           +- * Filter (13)
+                  :     :     :     :              +- * ColumnarToRow (12)
+                  :     :     :     :                 +- Scan parquet default.customer (11)
+                  :     :     :     +- BroadcastExchange (21)
+                  :     :     :        +- * Filter (20)
+                  :     :     :           +- * ColumnarToRow (19)
+                  :     :     :              +- Scan parquet default.customer_demographics (18)
+                  :     :     +- BroadcastExchange (27)
+                  :     :        +- * Filter (26)
+                  :     :           +- * ColumnarToRow (25)
+                  :     :              +- Scan parquet default.customer_address (24)
+                  :     +- BroadcastExchange (34)
+                  :        +- * Project (33)
+                  :           +- * Filter (32)
+                  :              +- * ColumnarToRow (31)
+                  :                 +- Scan parquet default.date_dim (30)
+                  +- BroadcastExchange (40)
+                     +- * Filter (39)
+                        +- * ColumnarToRow (38)
+                           +- Scan parquet default.item (37)
+
+
+(1) Scan parquet default.catalog_sales
+Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#9), dynamicpruningexpression(cs_sold_date_sk#9 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_bill_cdemo_sk:int,cs_item_sk:int,cs_quantity:int,cs_list_price:decimal(7,2),cs_sales_price:decimal(7,2),cs_coupon_amt:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 7]
+Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9]
+
+(3) Filter [codegen id : 7]
+Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9]
+Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3))
+
+(4) Scan parquet default.customer_demographics
+Output [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_education_status), EqualTo(cd_gender,F), EqualTo(cd_education_status,Unknown             ), IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_education_status:string,cd_dep_count:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14]
+
+(6) Filter [codegen id : 1]
+Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14]
+Condition : ((((isnotnull(cd_gender#12) AND isnotnull(cd_education_status#13)) AND (cd_gender#12 = F)) AND (cd_education_status#13 = Unknown             )) AND isnotnull(cd_demo_sk#11))
+
+(7) Project [codegen id : 1]
+Output [2]: [cd_demo_sk#11, cd_dep_count#14]
+Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14]
+
+(8) BroadcastExchange
+Input [2]: [cd_demo_sk#11, cd_dep_count#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#15]
+
+(9) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [cs_bill_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#11]
+Join condition: None
+
+(10) Project [codegen id : 7]
+Output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14]
+Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#11, cd_dep_count#14]
+
+(11) Scan parquet default.customer
+Output [5]: [c_customer_sk#16, c_current_cdemo_sk#17, c_current_addr_sk#18, c_birth_month#19, c_birth_year#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [In(c_birth_month, [1,6,8,9,12,2]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_addr_sk:int,c_birth_month:int,c_birth_year:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [5]: [c_customer_sk#16, c_current_cdemo_sk#17, c_current_addr_sk#18, c_birth_month#19, c_birth_year#20]
+
+(13) Filter [codegen id : 2]
+Input [5]: [c_customer_sk#16, c_current_cdemo_sk#17, c_current_addr_sk#18, c_birth_month#19, c_birth_year#20]
+Condition : (((c_birth_month#19 IN (1,6,8,9,12,2) AND isnotnull(c_customer_sk#16)) AND isnotnull(c_current_cdemo_sk#17)) AND isnotnull(c_current_addr_sk#18))
+
+(14) Project [codegen id : 2]
+Output [4]: [c_customer_sk#16, c_current_cdemo_sk#17, c_current_addr_sk#18, c_birth_year#20]
+Input [5]: [c_customer_sk#16, c_current_cdemo_sk#17, c_current_addr_sk#18, c_birth_month#19, c_birth_year#20]
+
+(15) BroadcastExchange
+Input [4]: [c_customer_sk#16, c_current_cdemo_sk#17, c_current_addr_sk#18, c_birth_year#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(16) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [cs_bill_customer_sk#1]
+Right keys [1]: [c_customer_sk#16]
+Join condition: None
+
+(17) Project [codegen id : 7]
+Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#17, c_current_addr_sk#18, c_birth_year#20]
+Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_customer_sk#16, c_current_cdemo_sk#17, c_current_addr_sk#18, c_birth_year#20]
+
+(18) Scan parquet default.customer_demographics
+Output [1]: [cd_demo_sk#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [1]: [cd_demo_sk#22]
+
+(20) Filter [codegen id : 3]
+Input [1]: [cd_demo_sk#22]
+Condition : isnotnull(cd_demo_sk#22)
+
+(21) BroadcastExchange
+Input [1]: [cd_demo_sk#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#23]
+
+(22) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [c_current_cdemo_sk#17]
+Right keys [1]: [cd_demo_sk#22]
+Join condition: None
+
+(23) Project [codegen id : 7]
+Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#18, c_birth_year#20]
+Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#17, c_current_addr_sk#18, c_birth_year#20, cd_demo_sk#22]
+
+(24) Scan parquet default.customer_address
+Output [4]: [ca_address_sk#24, ca_county#25, ca_state#26, ca_country#27]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [In(ca_state, [MS,IN,ND,OK,NM,VA]), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_county:string,ca_state:string,ca_country:string>
+
+(25) ColumnarToRow [codegen id : 4]
+Input [4]: [ca_address_sk#24, ca_county#25, ca_state#26, ca_country#27]
+
+(26) Filter [codegen id : 4]
+Input [4]: [ca_address_sk#24, ca_county#25, ca_state#26, ca_country#27]
+Condition : (ca_state#26 IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#24))
+
+(27) BroadcastExchange
+Input [4]: [ca_address_sk#24, ca_county#25, ca_state#26, ca_country#27]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#28]
+
+(28) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [c_current_addr_sk#18]
+Right keys [1]: [ca_address_sk#24]
+Join condition: None
+
+(29) Project [codegen id : 7]
+Output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#20, ca_county#25, ca_state#26, ca_country#27]
+Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#18, c_birth_year#20, ca_address_sk#24, ca_county#25, ca_state#26, ca_country#27]
+
+(30) Scan parquet default.date_dim
+Output [2]: [d_date_sk#29, d_year#30]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(31) ColumnarToRow [codegen id : 5]
+Input [2]: [d_date_sk#29, d_year#30]
+
+(32) Filter [codegen id : 5]
+Input [2]: [d_date_sk#29, d_year#30]
+Condition : ((isnotnull(d_year#30) AND (d_year#30 = 1998)) AND isnotnull(d_date_sk#29))
+
+(33) Project [codegen id : 5]
+Output [1]: [d_date_sk#29]
+Input [2]: [d_date_sk#29, d_year#30]
+
+(34) BroadcastExchange
+Input [1]: [d_date_sk#29]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#31]
+
+(35) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [cs_sold_date_sk#9]
+Right keys [1]: [d_date_sk#29]
+Join condition: None
+
+(36) Project [codegen id : 7]
+Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#20, ca_county#25, ca_state#26, ca_country#27]
+Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#20, ca_county#25, ca_state#26, ca_country#27, d_date_sk#29]
+
+(37) Scan parquet default.item
+Output [2]: [i_item_sk#32, i_item_id#33]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(38) ColumnarToRow [codegen id : 6]
+Input [2]: [i_item_sk#32, i_item_id#33]
+
+(39) Filter [codegen id : 6]
+Input [2]: [i_item_sk#32, i_item_id#33]
+Condition : isnotnull(i_item_sk#32)
+
+(40) BroadcastExchange
+Input [2]: [i_item_sk#32, i_item_id#33]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#34]
+
+(41) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [cs_item_sk#3]
+Right keys [1]: [i_item_sk#32]
+Join condition: None
+
+(42) Project [codegen id : 7]
+Output [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#20, i_item_id#33, ca_country#27, ca_state#26, ca_county#25]
+Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#20, ca_county#25, ca_state#26, ca_country#27, i_item_sk#32, i_item_id#33]
+
+(43) Expand [codegen id : 7]
+Input [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#20, i_item_id#33, ca_country#27, ca_state#26, ca_county#25]
+Arguments: [List(cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#20, i_item_id#33, ca_country#27, ca_state#26, ca_county#25, 0), List(cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#20, i_item_id#33, ca_country#27, ca_state#26, null, 1), List(cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#20, i_item_id#33, ca_country#27, null, null, 3), List(cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#20, i_item_id#33, null, null, null, 7), List(cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#20, null, null, null, null, 15)], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#20, i_item_id#35, ca_country#36, ca_state#37, ca_county#38, spark_grouping_id#39]
+
+(44) HashAggregate [codegen id : 7]
+Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#20, i_item_id#35, ca_country#36, ca_state#37, ca_county#38, spark_grouping_id#39]
+Keys [5]: [i_item_id#35, ca_country#36, ca_state#37, ca_county#38, spark_grouping_id#39]
+Functions [7]: [partial_avg(cast(cs_quantity#4 as decimal(12,2))), partial_avg(cast(cs_list_price#5 as decimal(12,2))), partial_avg(cast(cs_coupon_amt#7 as decimal(12,2))), partial_avg(cast(cs_sales_price#6 as decimal(12,2))), partial_avg(cast(cs_net_profit#8 as decimal(12,2))), partial_avg(cast(c_birth_year#20 as decimal(12,2))), partial_avg(cast(cd_dep_count#14 as decimal(12,2)))]
+Aggregate Attributes [14]: [sum#40, count#41, sum#42, count#43, sum#44, count#45, sum#46, count#47, sum#48, count#49, sum#50, count#51, sum#52, count#53]
+Results [19]: [i_item_id#35, ca_country#36, ca_state#37, ca_county#38, spark_grouping_id#39, sum#54, count#55, sum#56, count#57, sum#58, count#59, sum#60, count#61, sum#62, count#63, sum#64, count#65, sum#66, count#67]
+
+(45) Exchange
+Input [19]: [i_item_id#35, ca_country#36, ca_state#37, ca_county#38, spark_grouping_id#39, sum#54, count#55, sum#56, count#57, sum#58, count#59, sum#60, count#61, sum#62, count#63, sum#64, count#65, sum#66, count#67]
+Arguments: hashpartitioning(i_item_id#35, ca_country#36, ca_state#37, ca_county#38, spark_grouping_id#39, 5), ENSURE_REQUIREMENTS, [id=#68]
+
+(46) HashAggregate [codegen id : 8]
+Input [19]: [i_item_id#35, ca_country#36, ca_state#37, ca_county#38, spark_grouping_id#39, sum#54, count#55, sum#56, count#57, sum#58, count#59, sum#60, count#61, sum#62, count#63, sum#64, count#65, sum#66, count#67]
+Keys [5]: [i_item_id#35, ca_country#36, ca_state#37, ca_county#38, spark_grouping_id#39]
+Functions [7]: [avg(cast(cs_quantity#4 as decimal(12,2))), avg(cast(cs_list_price#5 as decimal(12,2))), avg(cast(cs_coupon_amt#7 as decimal(12,2))), avg(cast(cs_sales_price#6 as decimal(12,2))), avg(cast(cs_net_profit#8 as decimal(12,2))), avg(cast(c_birth_year#20 as decimal(12,2))), avg(cast(cd_dep_count#14 as decimal(12,2)))]
+Aggregate Attributes [7]: [avg(cast(cs_quantity#4 as decimal(12,2)))#69, avg(cast(cs_list_price#5 as decimal(12,2)))#70, avg(cast(cs_coupon_amt#7 as decimal(12,2)))#71, avg(cast(cs_sales_price#6 as decimal(12,2)))#72, avg(cast(cs_net_profit#8 as decimal(12,2)))#73, avg(cast(c_birth_year#20 as decimal(12,2)))#74, avg(cast(cd_dep_count#14 as decimal(12,2)))#75]
+Results [11]: [i_item_id#35, ca_country#36, ca_state#37, ca_county#38, avg(cast(cs_quantity#4 as decimal(12,2)))#69 AS agg1#76, avg(cast(cs_list_price#5 as decimal(12,2)))#70 AS agg2#77, avg(cast(cs_coupon_amt#7 as decimal(12,2)))#71 AS agg3#78, avg(cast(cs_sales_price#6 as decimal(12,2)))#72 AS agg4#79, avg(cast(cs_net_profit#8 as decimal(12,2)))#73 AS agg5#80, avg(cast(c_birth_year#20 as decimal(12,2)))#74 AS agg6#81, avg(cast(cd_dep_count#14 as decimal(12,2)))#75 AS agg7#82]
+
+(47) TakeOrderedAndProject
+Input [11]: [i_item_id#35, ca_country#36, ca_state#37, ca_county#38, agg1#76, agg2#77, agg3#78, agg4#79, agg5#80, agg6#81, agg7#82]
+Arguments: 100, [ca_country#36 ASC NULLS FIRST, ca_state#37 ASC NULLS FIRST, ca_county#38 ASC NULLS FIRST, i_item_id#35 ASC NULLS FIRST], [i_item_id#35, ca_country#36, ca_state#37, ca_county#38, agg1#76, agg2#77, agg3#78, agg4#79, agg5#80, agg6#81, agg7#82]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#9 IN dynamicpruning#10
+ReusedExchange (48)
+
+
+(48) ReusedExchange [Reuses operator id: 34]
+Output [1]: [d_date_sk#29]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/simplified.txt
new file mode 100644
index 0000000000000..d8dcd9db5ceef
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/simplified.txt
@@ -0,0 +1,71 @@
+TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7]
+  WholeStageCodegen (8)
+    HashAggregate [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(cast(cs_quantity as decimal(12,2))),avg(cast(cs_list_price as decimal(12,2))),avg(cast(cs_coupon_amt as decimal(12,2))),avg(cast(cs_sales_price as decimal(12,2))),avg(cast(cs_net_profit as decimal(12,2))),avg(cast(c_birth_year as decimal(12,2))),avg(cast(cd_dep_count as decimal(12,2))),agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+      InputAdapter
+        Exchange [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] #1
+          WholeStageCodegen (7)
+            HashAggregate [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+              Expand [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county]
+                Project [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county]
+                  BroadcastHashJoin [cs_item_sk,i_item_sk]
+                    Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country]
+                      BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                        Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country]
+                          BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                            Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year]
+                              BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk]
+                                Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year]
+                                  BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk]
+                                    Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count]
+                                      BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk]
+                                        Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk]
+                                                SubqueryBroadcast [d_date_sk] #1
+                                                  ReusedExchange [d_date_sk] #2
+                                        InputAdapter
+                                          BroadcastExchange #3
+                                            WholeStageCodegen (1)
+                                              Project [cd_demo_sk,cd_dep_count]
+                                                Filter [cd_gender,cd_education_status,cd_demo_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count]
+                                    InputAdapter
+                                      BroadcastExchange #4
+                                        WholeStageCodegen (2)
+                                          Project [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year]
+                                            Filter [c_birth_month,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year]
+                                InputAdapter
+                                  BroadcastExchange #5
+                                    WholeStageCodegen (3)
+                                      Filter [cd_demo_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.customer_demographics [cd_demo_sk]
+                            InputAdapter
+                              BroadcastExchange #6
+                                WholeStageCodegen (4)
+                                  Filter [ca_state,ca_address_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.customer_address [ca_address_sk,ca_county,ca_state,ca_country]
+                        InputAdapter
+                          BroadcastExchange #2
+                            WholeStageCodegen (5)
+                              Project [d_date_sk]
+                                Filter [d_year,d_date_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.date_dim [d_date_sk,d_year]
+                    InputAdapter
+                      BroadcastExchange #7
+                        WholeStageCodegen (6)
+                          Filter [i_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.item [i_item_sk,i_item_id]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.sf100/explain.txt
new file mode 100644
index 0000000000000..febae3b0df780
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.sf100/explain.txt
@@ -0,0 +1,262 @@
+== Physical Plan ==
+TakeOrderedAndProject (45)
++- * HashAggregate (44)
+   +- Exchange (43)
+      +- * HashAggregate (42)
+         +- * Project (41)
+            +- * SortMergeJoin Inner (40)
+               :- * Sort (25)
+               :  +- Exchange (24)
+               :     +- * Project (23)
+               :        +- * BroadcastHashJoin Inner BuildRight (22)
+               :           :- * Project (17)
+               :           :  +- * BroadcastHashJoin Inner BuildRight (16)
+               :           :     :- * Project (10)
+               :           :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :           :     :     :- * Filter (3)
+               :           :     :     :  +- * ColumnarToRow (2)
+               :           :     :     :     +- Scan parquet default.store_sales (1)
+               :           :     :     +- BroadcastExchange (8)
+               :           :     :        +- * Project (7)
+               :           :     :           +- * Filter (6)
+               :           :     :              +- * ColumnarToRow (5)
+               :           :     :                 +- Scan parquet default.item (4)
+               :           :     +- BroadcastExchange (15)
+               :           :        +- * Project (14)
+               :           :           +- * Filter (13)
+               :           :              +- * ColumnarToRow (12)
+               :           :                 +- Scan parquet default.date_dim (11)
+               :           +- BroadcastExchange (21)
+               :              +- * Filter (20)
+               :                 +- * ColumnarToRow (19)
+               :                    +- Scan parquet default.store (18)
+               +- * Sort (39)
+                  +- Exchange (38)
+                     +- * Project (37)
+                        +- * SortMergeJoin Inner (36)
+                           :- * Sort (30)
+                           :  +- Exchange (29)
+                           :     +- * Filter (28)
+                           :        +- * ColumnarToRow (27)
+                           :           +- Scan parquet default.customer (26)
+                           +- * Sort (35)
+                              +- Exchange (34)
+                                 +- * Filter (33)
+                                    +- * ColumnarToRow (32)
+                                       +- Scan parquet default.customer_address (31)
+
+
+(1) Scan parquet default.store_sales
+Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ext_sales_price#4, ss_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ext_sales_price#4, ss_sold_date_sk#5]
+
+(3) Filter [codegen id : 4]
+Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ext_sales_price#4, ss_sold_date_sk#5]
+Condition : ((isnotnull(ss_item_sk#1) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_store_sk#3))
+
+(4) Scan parquet default.item
+Output [6]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10, i_manufact#11, i_manager_id#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,8), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_brand:string,i_manufact_id:int,i_manufact:string,i_manager_id:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [6]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10, i_manufact#11, i_manager_id#12]
+
+(6) Filter [codegen id : 1]
+Input [6]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10, i_manufact#11, i_manager_id#12]
+Condition : ((isnotnull(i_manager_id#12) AND (i_manager_id#12 = 8)) AND isnotnull(i_item_sk#7))
+
+(7) Project [codegen id : 1]
+Output [5]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10, i_manufact#11]
+Input [6]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10, i_manufact#11, i_manager_id#12]
+
+(8) BroadcastExchange
+Input [5]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10, i_manufact#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [8]: [ss_customer_sk#2, ss_store_sk#3, ss_ext_sales_price#4, ss_sold_date_sk#5, i_brand_id#8, i_brand#9, i_manufact_id#10, i_manufact#11]
+Input [10]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ext_sales_price#4, ss_sold_date_sk#5, i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10, i_manufact#11]
+
+(11) Scan parquet default.date_dim
+Output [3]: [d_date_sk#14, d_year#15, d_moy#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1998), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [d_date_sk#14, d_year#15, d_moy#16]
+
+(13) Filter [codegen id : 2]
+Input [3]: [d_date_sk#14, d_year#15, d_moy#16]
+Condition : ((((isnotnull(d_moy#16) AND isnotnull(d_year#15)) AND (d_moy#16 = 11)) AND (d_year#15 = 1998)) AND isnotnull(d_date_sk#14))
+
+(14) Project [codegen id : 2]
+Output [1]: [d_date_sk#14]
+Input [3]: [d_date_sk#14, d_year#15, d_moy#16]
+
+(15) BroadcastExchange
+Input [1]: [d_date_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#5]
+Right keys [1]: [d_date_sk#14]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [7]: [ss_customer_sk#2, ss_store_sk#3, ss_ext_sales_price#4, i_brand_id#8, i_brand#9, i_manufact_id#10, i_manufact#11]
+Input [9]: [ss_customer_sk#2, ss_store_sk#3, ss_ext_sales_price#4, ss_sold_date_sk#5, i_brand_id#8, i_brand#9, i_manufact_id#10, i_manufact#11, d_date_sk#14]
+
+(18) Scan parquet default.store
+Output [2]: [s_store_sk#18, s_zip#19]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_zip), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_zip:string>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [2]: [s_store_sk#18, s_zip#19]
+
+(20) Filter [codegen id : 3]
+Input [2]: [s_store_sk#18, s_zip#19]
+Condition : (isnotnull(s_zip#19) AND isnotnull(s_store_sk#18))
+
+(21) BroadcastExchange
+Input [2]: [s_store_sk#18, s_zip#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#20]
+
+(22) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#18]
+Join condition: None
+
+(23) Project [codegen id : 4]
+Output [7]: [ss_customer_sk#2, ss_ext_sales_price#4, i_brand_id#8, i_brand#9, i_manufact_id#10, i_manufact#11, s_zip#19]
+Input [9]: [ss_customer_sk#2, ss_store_sk#3, ss_ext_sales_price#4, i_brand_id#8, i_brand#9, i_manufact_id#10, i_manufact#11, s_store_sk#18, s_zip#19]
+
+(24) Exchange
+Input [7]: [ss_customer_sk#2, ss_ext_sales_price#4, i_brand_id#8, i_brand#9, i_manufact_id#10, i_manufact#11, s_zip#19]
+Arguments: hashpartitioning(ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(25) Sort [codegen id : 5]
+Input [7]: [ss_customer_sk#2, ss_ext_sales_price#4, i_brand_id#8, i_brand#9, i_manufact_id#10, i_manufact#11, s_zip#19]
+Arguments: [ss_customer_sk#2 ASC NULLS FIRST], false, 0
+
+(26) Scan parquet default.customer
+Output [2]: [c_customer_sk#22, c_current_addr_sk#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int>
+
+(27) ColumnarToRow [codegen id : 6]
+Input [2]: [c_customer_sk#22, c_current_addr_sk#23]
+
+(28) Filter [codegen id : 6]
+Input [2]: [c_customer_sk#22, c_current_addr_sk#23]
+Condition : (isnotnull(c_customer_sk#22) AND isnotnull(c_current_addr_sk#23))
+
+(29) Exchange
+Input [2]: [c_customer_sk#22, c_current_addr_sk#23]
+Arguments: hashpartitioning(c_current_addr_sk#23, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(30) Sort [codegen id : 7]
+Input [2]: [c_customer_sk#22, c_current_addr_sk#23]
+Arguments: [c_current_addr_sk#23 ASC NULLS FIRST], false, 0
+
+(31) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#25, ca_zip#26]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_zip)]
+ReadSchema: struct<ca_address_sk:int,ca_zip:string>
+
+(32) ColumnarToRow [codegen id : 8]
+Input [2]: [ca_address_sk#25, ca_zip#26]
+
+(33) Filter [codegen id : 8]
+Input [2]: [ca_address_sk#25, ca_zip#26]
+Condition : (isnotnull(ca_address_sk#25) AND isnotnull(ca_zip#26))
+
+(34) Exchange
+Input [2]: [ca_address_sk#25, ca_zip#26]
+Arguments: hashpartitioning(ca_address_sk#25, 5), ENSURE_REQUIREMENTS, [id=#27]
+
+(35) Sort [codegen id : 9]
+Input [2]: [ca_address_sk#25, ca_zip#26]
+Arguments: [ca_address_sk#25 ASC NULLS FIRST], false, 0
+
+(36) SortMergeJoin [codegen id : 10]
+Left keys [1]: [c_current_addr_sk#23]
+Right keys [1]: [ca_address_sk#25]
+Join condition: None
+
+(37) Project [codegen id : 10]
+Output [2]: [c_customer_sk#22, ca_zip#26]
+Input [4]: [c_customer_sk#22, c_current_addr_sk#23, ca_address_sk#25, ca_zip#26]
+
+(38) Exchange
+Input [2]: [c_customer_sk#22, ca_zip#26]
+Arguments: hashpartitioning(c_customer_sk#22, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(39) Sort [codegen id : 11]
+Input [2]: [c_customer_sk#22, ca_zip#26]
+Arguments: [c_customer_sk#22 ASC NULLS FIRST], false, 0
+
+(40) SortMergeJoin [codegen id : 12]
+Left keys [1]: [ss_customer_sk#2]
+Right keys [1]: [c_customer_sk#22]
+Join condition: NOT (substr(ca_zip#26, 1, 5) = substr(s_zip#19, 1, 5))
+
+(41) Project [codegen id : 12]
+Output [5]: [ss_ext_sales_price#4, i_brand_id#8, i_brand#9, i_manufact_id#10, i_manufact#11]
+Input [9]: [ss_customer_sk#2, ss_ext_sales_price#4, i_brand_id#8, i_brand#9, i_manufact_id#10, i_manufact#11, s_zip#19, c_customer_sk#22, ca_zip#26]
+
+(42) HashAggregate [codegen id : 12]
+Input [5]: [ss_ext_sales_price#4, i_brand_id#8, i_brand#9, i_manufact_id#10, i_manufact#11]
+Keys [4]: [i_brand#9, i_brand_id#8, i_manufact_id#10, i_manufact#11]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#4))]
+Aggregate Attributes [1]: [sum#29]
+Results [5]: [i_brand#9, i_brand_id#8, i_manufact_id#10, i_manufact#11, sum#30]
+
+(43) Exchange
+Input [5]: [i_brand#9, i_brand_id#8, i_manufact_id#10, i_manufact#11, sum#30]
+Arguments: hashpartitioning(i_brand#9, i_brand_id#8, i_manufact_id#10, i_manufact#11, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(44) HashAggregate [codegen id : 13]
+Input [5]: [i_brand#9, i_brand_id#8, i_manufact_id#10, i_manufact#11, sum#30]
+Keys [4]: [i_brand#9, i_brand_id#8, i_manufact_id#10, i_manufact#11]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#4))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#4))#32]
+Results [5]: [i_brand_id#8 AS brand_id#33, i_brand#9 AS brand#34, i_manufact_id#10, i_manufact#11, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#4))#32,17,2) AS ext_price#35]
+
+(45) TakeOrderedAndProject
+Input [5]: [brand_id#33, brand#34, i_manufact_id#10, i_manufact#11, ext_price#35]
+Arguments: 100, [ext_price#35 DESC NULLS LAST, brand#34 ASC NULLS FIRST, brand_id#33 ASC NULLS FIRST, i_manufact_id#10 ASC NULLS FIRST, i_manufact#11 ASC NULLS FIRST], [brand_id#33, brand#34, i_manufact_id#10, i_manufact#11, ext_price#35]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (46)
+
+
+(46) ReusedExchange [Reuses operator id: 15]
+Output [1]: [d_date_sk#14]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.sf100/simplified.txt
new file mode 100644
index 0000000000000..a563ccd2a0106
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.sf100/simplified.txt
@@ -0,0 +1,78 @@
+TakeOrderedAndProject [ext_price,brand,brand_id,i_manufact_id,i_manufact]
+  WholeStageCodegen (13)
+    HashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum]
+      InputAdapter
+        Exchange [i_brand,i_brand_id,i_manufact_id,i_manufact] #1
+          WholeStageCodegen (12)
+            HashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,ss_ext_sales_price] [sum,sum]
+              Project [ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact]
+                SortMergeJoin [ss_customer_sk,c_customer_sk,ca_zip,s_zip]
+                  InputAdapter
+                    WholeStageCodegen (5)
+                      Sort [ss_customer_sk]
+                        InputAdapter
+                          Exchange [ss_customer_sk] #2
+                            WholeStageCodegen (4)
+                              Project [ss_customer_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,s_zip]
+                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                  Project [ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact]
+                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                      Project [ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_brand,i_manufact_id,i_manufact]
+                                        BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                          Filter [ss_item_sk,ss_customer_sk,ss_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                                  SubqueryBroadcast [d_date_sk] #1
+                                                    ReusedExchange [d_date_sk] #3
+                                          InputAdapter
+                                            BroadcastExchange #4
+                                              WholeStageCodegen (1)
+                                                Project [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact]
+                                                  Filter [i_manager_id,i_item_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id]
+                                      InputAdapter
+                                        BroadcastExchange #3
+                                          WholeStageCodegen (2)
+                                            Project [d_date_sk]
+                                              Filter [d_moy,d_year,d_date_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                  InputAdapter
+                                    BroadcastExchange #5
+                                      WholeStageCodegen (3)
+                                        Filter [s_zip,s_store_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.store [s_store_sk,s_zip]
+                  InputAdapter
+                    WholeStageCodegen (11)
+                      Sort [c_customer_sk]
+                        InputAdapter
+                          Exchange [c_customer_sk] #6
+                            WholeStageCodegen (10)
+                              Project [c_customer_sk,ca_zip]
+                                SortMergeJoin [c_current_addr_sk,ca_address_sk]
+                                  InputAdapter
+                                    WholeStageCodegen (7)
+                                      Sort [c_current_addr_sk]
+                                        InputAdapter
+                                          Exchange [c_current_addr_sk] #7
+                                            WholeStageCodegen (6)
+                                              Filter [c_customer_sk,c_current_addr_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.customer [c_customer_sk,c_current_addr_sk]
+                                  InputAdapter
+                                    WholeStageCodegen (9)
+                                      Sort [ca_address_sk]
+                                        InputAdapter
+                                          Exchange [ca_address_sk] #8
+                                            WholeStageCodegen (8)
+                                              Filter [ca_address_sk,ca_zip]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.customer_address [ca_address_sk,ca_zip]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/explain.txt
new file mode 100644
index 0000000000000..e321ddc156860
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/explain.txt
@@ -0,0 +1,222 @@
+== Physical Plan ==
+TakeOrderedAndProject (39)
++- * HashAggregate (38)
+   +- Exchange (37)
+      +- * HashAggregate (36)
+         +- * Project (35)
+            +- * BroadcastHashJoin Inner BuildRight (34)
+               :- * Project (29)
+               :  +- * BroadcastHashJoin Inner BuildRight (28)
+               :     :- * Project (23)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (22)
+               :     :     :- * Project (17)
+               :     :     :  +- * BroadcastHashJoin Inner BuildRight (16)
+               :     :     :     :- * Project (10)
+               :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :     :     :     :- * Project (4)
+               :     :     :     :     :  +- * Filter (3)
+               :     :     :     :     :     +- * ColumnarToRow (2)
+               :     :     :     :     :        +- Scan parquet default.date_dim (1)
+               :     :     :     :     +- BroadcastExchange (8)
+               :     :     :     :        +- * Filter (7)
+               :     :     :     :           +- * ColumnarToRow (6)
+               :     :     :     :              +- Scan parquet default.store_sales (5)
+               :     :     :     +- BroadcastExchange (15)
+               :     :     :        +- * Project (14)
+               :     :     :           +- * Filter (13)
+               :     :     :              +- * ColumnarToRow (12)
+               :     :     :                 +- Scan parquet default.item (11)
+               :     :     +- BroadcastExchange (21)
+               :     :        +- * Filter (20)
+               :     :           +- * ColumnarToRow (19)
+               :     :              +- Scan parquet default.customer (18)
+               :     +- BroadcastExchange (27)
+               :        +- * Filter (26)
+               :           +- * ColumnarToRow (25)
+               :              +- Scan parquet default.customer_address (24)
+               +- BroadcastExchange (33)
+                  +- * Filter (32)
+                     +- * ColumnarToRow (31)
+                        +- Scan parquet default.store (30)
+
+
+(1) Scan parquet default.date_dim
+Output [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1998), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(2) ColumnarToRow [codegen id : 6]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(3) Filter [codegen id : 6]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 1998)) AND isnotnull(d_date_sk#1))
+
+(4) Project [codegen id : 6]
+Output [1]: [d_date_sk#1]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(5) Scan parquet default.store_sales
+Output [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8]
+
+(7) Filter [codegen id : 1]
+Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8]
+Condition : ((isnotnull(ss_item_sk#4) AND isnotnull(ss_customer_sk#5)) AND isnotnull(ss_store_sk#6))
+
+(8) BroadcastExchange
+Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[4, int, true] as bigint)),false), [id=#9]
+
+(9) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [d_date_sk#1]
+Right keys [1]: [ss_sold_date_sk#8]
+Join condition: None
+
+(10) Project [codegen id : 6]
+Output [4]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7]
+Input [6]: [d_date_sk#1, ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8]
+
+(11) Scan parquet default.item
+Output [6]: [i_item_sk#10, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14, i_manager_id#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,8), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_brand:string,i_manufact_id:int,i_manufact:string,i_manager_id:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [6]: [i_item_sk#10, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14, i_manager_id#15]
+
+(13) Filter [codegen id : 2]
+Input [6]: [i_item_sk#10, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14, i_manager_id#15]
+Condition : ((isnotnull(i_manager_id#15) AND (i_manager_id#15 = 8)) AND isnotnull(i_item_sk#10))
+
+(14) Project [codegen id : 2]
+Output [5]: [i_item_sk#10, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14]
+Input [6]: [i_item_sk#10, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14, i_manager_id#15]
+
+(15) BroadcastExchange
+Input [5]: [i_item_sk#10, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#16]
+
+(16) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_item_sk#4]
+Right keys [1]: [i_item_sk#10]
+Join condition: None
+
+(17) Project [codegen id : 6]
+Output [7]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14]
+Input [9]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_item_sk#10, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14]
+
+(18) Scan parquet default.customer
+Output [2]: [c_customer_sk#17, c_current_addr_sk#18]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [2]: [c_customer_sk#17, c_current_addr_sk#18]
+
+(20) Filter [codegen id : 3]
+Input [2]: [c_customer_sk#17, c_current_addr_sk#18]
+Condition : (isnotnull(c_customer_sk#17) AND isnotnull(c_current_addr_sk#18))
+
+(21) BroadcastExchange
+Input [2]: [c_customer_sk#17, c_current_addr_sk#18]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#19]
+
+(22) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_customer_sk#5]
+Right keys [1]: [c_customer_sk#17]
+Join condition: None
+
+(23) Project [codegen id : 6]
+Output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14, c_current_addr_sk#18]
+Input [9]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14, c_customer_sk#17, c_current_addr_sk#18]
+
+(24) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#20, ca_zip#21]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_zip)]
+ReadSchema: struct<ca_address_sk:int,ca_zip:string>
+
+(25) ColumnarToRow [codegen id : 4]
+Input [2]: [ca_address_sk#20, ca_zip#21]
+
+(26) Filter [codegen id : 4]
+Input [2]: [ca_address_sk#20, ca_zip#21]
+Condition : (isnotnull(ca_address_sk#20) AND isnotnull(ca_zip#21))
+
+(27) BroadcastExchange
+Input [2]: [ca_address_sk#20, ca_zip#21]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#22]
+
+(28) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [c_current_addr_sk#18]
+Right keys [1]: [ca_address_sk#20]
+Join condition: None
+
+(29) Project [codegen id : 6]
+Output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14, ca_zip#21]
+Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14, c_current_addr_sk#18, ca_address_sk#20, ca_zip#21]
+
+(30) Scan parquet default.store
+Output [2]: [s_store_sk#23, s_zip#24]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_zip), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_zip:string>
+
+(31) ColumnarToRow [codegen id : 5]
+Input [2]: [s_store_sk#23, s_zip#24]
+
+(32) Filter [codegen id : 5]
+Input [2]: [s_store_sk#23, s_zip#24]
+Condition : (isnotnull(s_zip#24) AND isnotnull(s_store_sk#23))
+
+(33) BroadcastExchange
+Input [2]: [s_store_sk#23, s_zip#24]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#25]
+
+(34) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_store_sk#6]
+Right keys [1]: [s_store_sk#23]
+Join condition: NOT (substr(ca_zip#21, 1, 5) = substr(s_zip#24, 1, 5))
+
+(35) Project [codegen id : 6]
+Output [5]: [ss_ext_sales_price#7, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14]
+Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14, ca_zip#21, s_store_sk#23, s_zip#24]
+
+(36) HashAggregate [codegen id : 6]
+Input [5]: [ss_ext_sales_price#7, i_brand_id#11, i_brand#12, i_manufact_id#13, i_manufact#14]
+Keys [4]: [i_brand#12, i_brand_id#11, i_manufact_id#13, i_manufact#14]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#7))]
+Aggregate Attributes [1]: [sum#26]
+Results [5]: [i_brand#12, i_brand_id#11, i_manufact_id#13, i_manufact#14, sum#27]
+
+(37) Exchange
+Input [5]: [i_brand#12, i_brand_id#11, i_manufact_id#13, i_manufact#14, sum#27]
+Arguments: hashpartitioning(i_brand#12, i_brand_id#11, i_manufact_id#13, i_manufact#14, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(38) HashAggregate [codegen id : 7]
+Input [5]: [i_brand#12, i_brand_id#11, i_manufact_id#13, i_manufact#14, sum#27]
+Keys [4]: [i_brand#12, i_brand_id#11, i_manufact_id#13, i_manufact#14]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#7))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#7))#29]
+Results [5]: [i_brand_id#11 AS brand_id#30, i_brand#12 AS brand#31, i_manufact_id#13, i_manufact#14, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#7))#29,17,2) AS ext_price#32]
+
+(39) TakeOrderedAndProject
+Input [5]: [brand_id#30, brand#31, i_manufact_id#13, i_manufact#14, ext_price#32]
+Arguments: 100, [ext_price#32 DESC NULLS LAST, brand#31 ASC NULLS FIRST, brand_id#30 ASC NULLS FIRST, i_manufact_id#13 ASC NULLS FIRST, i_manufact#14 ASC NULLS FIRST], [brand_id#30, brand#31, i_manufact_id#13, i_manufact#14, ext_price#32]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/simplified.txt
new file mode 100644
index 0000000000000..4e00ccb014571
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/simplified.txt
@@ -0,0 +1,58 @@
+TakeOrderedAndProject [ext_price,brand,brand_id,i_manufact_id,i_manufact]
+  WholeStageCodegen (7)
+    HashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum]
+      InputAdapter
+        Exchange [i_brand,i_brand_id,i_manufact_id,i_manufact] #1
+          WholeStageCodegen (6)
+            HashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,ss_ext_sales_price] [sum,sum]
+              Project [ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact]
+                BroadcastHashJoin [ss_store_sk,s_store_sk,ca_zip,s_zip]
+                  Project [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip]
+                    BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                      Project [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_current_addr_sk]
+                        BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+                          Project [ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact]
+                            BroadcastHashJoin [ss_item_sk,i_item_sk]
+                              Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price]
+                                BroadcastHashJoin [d_date_sk,ss_sold_date_sk]
+                                  Project [d_date_sk]
+                                    Filter [d_moy,d_year,d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                  InputAdapter
+                                    BroadcastExchange #2
+                                      WholeStageCodegen (1)
+                                        Filter [ss_item_sk,ss_customer_sk,ss_store_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk]
+                              InputAdapter
+                                BroadcastExchange #3
+                                  WholeStageCodegen (2)
+                                    Project [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact]
+                                      Filter [i_manager_id,i_item_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id]
+                          InputAdapter
+                            BroadcastExchange #4
+                              WholeStageCodegen (3)
+                                Filter [c_customer_sk,c_current_addr_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.customer [c_customer_sk,c_current_addr_sk]
+                      InputAdapter
+                        BroadcastExchange #5
+                          WholeStageCodegen (4)
+                            Filter [ca_address_sk,ca_zip]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.customer_address [ca_address_sk,ca_zip]
+                  InputAdapter
+                    BroadcastExchange #6
+                      WholeStageCodegen (5)
+                        Filter [s_zip,s_store_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.store [s_store_sk,s_zip]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.sf100/explain.txt
new file mode 100644
index 0000000000000..c31bb7470648f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.sf100/explain.txt
@@ -0,0 +1,208 @@
+== Physical Plan ==
+* Sort (37)
++- Exchange (36)
+   +- * Project (35)
+      +- * BroadcastHashJoin Inner BuildRight (34)
+         :- * Project (23)
+         :  +- * BroadcastHashJoin Inner BuildRight (22)
+         :     :- * HashAggregate (16)
+         :     :  +- Exchange (15)
+         :     :     +- * HashAggregate (14)
+         :     :        +- * Project (13)
+         :     :           +- * BroadcastHashJoin Inner BuildRight (12)
+         :     :              :- Union (7)
+         :     :              :  :- * Project (3)
+         :     :              :  :  +- * ColumnarToRow (2)
+         :     :              :  :     +- Scan parquet default.web_sales (1)
+         :     :              :  +- * Project (6)
+         :     :              :     +- * ColumnarToRow (5)
+         :     :              :        +- Scan parquet default.catalog_sales (4)
+         :     :              +- BroadcastExchange (11)
+         :     :                 +- * Filter (10)
+         :     :                    +- * ColumnarToRow (9)
+         :     :                       +- Scan parquet default.date_dim (8)
+         :     +- BroadcastExchange (21)
+         :        +- * Project (20)
+         :           +- * Filter (19)
+         :              +- * ColumnarToRow (18)
+         :                 +- Scan parquet default.date_dim (17)
+         +- BroadcastExchange (33)
+            +- * Project (32)
+               +- * BroadcastHashJoin Inner BuildRight (31)
+                  :- * HashAggregate (25)
+                  :  +- ReusedExchange (24)
+                  +- BroadcastExchange (30)
+                     +- * Project (29)
+                        +- * Filter (28)
+                           +- * ColumnarToRow (27)
+                              +- Scan parquet default.date_dim (26)
+
+
+(1) Scan parquet default.web_sales
+Output [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#2)]
+ReadSchema: struct<ws_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2]
+
+(3) Project [codegen id : 1]
+Output [2]: [ws_sold_date_sk#2 AS sold_date_sk#3, ws_ext_sales_price#1 AS sales_price#4]
+Input [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2]
+
+(4) Scan parquet default.catalog_sales
+Output [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#6)]
+ReadSchema: struct<cs_ext_sales_price:decimal(7,2)>
+
+(5) ColumnarToRow [codegen id : 2]
+Input [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6]
+
+(6) Project [codegen id : 2]
+Output [2]: [cs_sold_date_sk#6 AS sold_date_sk#7, cs_ext_sales_price#5 AS sales_price#8]
+Input [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6]
+
+(7) Union
+
+(8) Scan parquet default.date_dim
+Output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date_sk), IsNotNull(d_week_seq)]
+ReadSchema: struct<d_date_sk:int,d_week_seq:int,d_day_name:string>
+
+(9) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11]
+
+(10) Filter [codegen id : 3]
+Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11]
+Condition : (isnotnull(d_date_sk#9) AND isnotnull(d_week_seq#10))
+
+(11) BroadcastExchange
+Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#12]
+
+(12) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [sold_date_sk#3]
+Right keys [1]: [d_date_sk#9]
+Join condition: None
+
+(13) Project [codegen id : 4]
+Output [3]: [sales_price#4, d_week_seq#10, d_day_name#11]
+Input [5]: [sold_date_sk#3, sales_price#4, d_date_sk#9, d_week_seq#10, d_day_name#11]
+
+(14) HashAggregate [codegen id : 4]
+Input [3]: [sales_price#4, d_week_seq#10, d_day_name#11]
+Keys [1]: [d_week_seq#10]
+Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday   ) THEN sales_price#4 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday   ) THEN sales_price#4 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday  ) THEN sales_price#4 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday   ) THEN sales_price#4 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 ELSE null END))]
+Aggregate Attributes [7]: [sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19]
+Results [8]: [d_week_seq#10, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25, sum#26]
+
+(15) Exchange
+Input [8]: [d_week_seq#10, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25, sum#26]
+Arguments: hashpartitioning(d_week_seq#10, 5), ENSURE_REQUIREMENTS, [id=#27]
+
+(16) HashAggregate [codegen id : 12]
+Input [8]: [d_week_seq#10, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25, sum#26]
+Keys [1]: [d_week_seq#10]
+Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday   ) THEN sales_price#4 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday   ) THEN sales_price#4 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday  ) THEN sales_price#4 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday   ) THEN sales_price#4 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 ELSE null END))]
+Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday   ) THEN sales_price#4 ELSE null END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday   ) THEN sales_price#4 ELSE null END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday  ) THEN sales_price#4 ELSE null END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 ELSE null END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 ELSE null END))#32, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday   ) THEN sales_price#4 ELSE null END))#33, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 ELSE null END))#34]
+Results [8]: [d_week_seq#10, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday   ) THEN sales_price#4 ELSE null END))#28,17,2) AS sun_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday   ) THEN sales_price#4 ELSE null END))#29,17,2) AS mon_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday  ) THEN sales_price#4 ELSE null END))#30,17,2) AS tue_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 ELSE null END))#31,17,2) AS wed_sales#38, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 ELSE null END))#32,17,2) AS thu_sales#39, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday   ) THEN sales_price#4 ELSE null END))#33,17,2) AS fri_sales#40, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 ELSE null END))#34,17,2) AS sat_sales#41]
+
+(17) Scan parquet default.date_dim
+Output [2]: [d_week_seq#42, d_year#43]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_week_seq)]
+ReadSchema: struct<d_week_seq:int,d_year:int>
+
+(18) ColumnarToRow [codegen id : 5]
+Input [2]: [d_week_seq#42, d_year#43]
+
+(19) Filter [codegen id : 5]
+Input [2]: [d_week_seq#42, d_year#43]
+Condition : ((isnotnull(d_year#43) AND (d_year#43 = 2001)) AND isnotnull(d_week_seq#42))
+
+(20) Project [codegen id : 5]
+Output [1]: [d_week_seq#42]
+Input [2]: [d_week_seq#42, d_year#43]
+
+(21) BroadcastExchange
+Input [1]: [d_week_seq#42]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#44]
+
+(22) BroadcastHashJoin [codegen id : 12]
+Left keys [1]: [d_week_seq#10]
+Right keys [1]: [d_week_seq#42]
+Join condition: None
+
+(23) Project [codegen id : 12]
+Output [8]: [d_week_seq#10 AS d_week_seq1#45, sun_sales#35 AS sun_sales1#46, mon_sales#36 AS mon_sales1#47, tue_sales#37 AS tue_sales1#48, wed_sales#38 AS wed_sales1#49, thu_sales#39 AS thu_sales1#50, fri_sales#40 AS fri_sales1#51, sat_sales#41 AS sat_sales1#52]
+Input [9]: [d_week_seq#10, sun_sales#35, mon_sales#36, tue_sales#37, wed_sales#38, thu_sales#39, fri_sales#40, sat_sales#41, d_week_seq#42]
+
+(24) ReusedExchange [Reuses operator id: 15]
+Output [8]: [d_week_seq#53, sum#54, sum#55, sum#56, sum#57, sum#58, sum#59, sum#60]
+
+(25) HashAggregate [codegen id : 11]
+Input [8]: [d_week_seq#53, sum#54, sum#55, sum#56, sum#57, sum#58, sum#59, sum#60]
+Keys [1]: [d_week_seq#53]
+Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#61 = Sunday   ) THEN sales_price#4 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#61 = Monday   ) THEN sales_price#4 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#61 = Tuesday  ) THEN sales_price#4 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#61 = Wednesday) THEN sales_price#4 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#61 = Thursday ) THEN sales_price#4 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#61 = Friday   ) THEN sales_price#4 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#61 = Saturday ) THEN sales_price#4 ELSE null END))]
+Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#61 = Sunday   ) THEN sales_price#4 ELSE null END))#62, sum(UnscaledValue(CASE WHEN (d_day_name#61 = Monday   ) THEN sales_price#4 ELSE null END))#63, sum(UnscaledValue(CASE WHEN (d_day_name#61 = Tuesday  ) THEN sales_price#4 ELSE null END))#64, sum(UnscaledValue(CASE WHEN (d_day_name#61 = Wednesday) THEN sales_price#4 ELSE null END))#65, sum(UnscaledValue(CASE WHEN (d_day_name#61 = Thursday ) THEN sales_price#4 ELSE null END))#66, sum(UnscaledValue(CASE WHEN (d_day_name#61 = Friday   ) THEN sales_price#4 ELSE null END))#67, sum(UnscaledValue(CASE WHEN (d_day_name#61 = Saturday ) THEN sales_price#4 ELSE null END))#68]
+Results [8]: [d_week_seq#53, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#61 = Sunday   ) THEN sales_price#4 ELSE null END))#62,17,2) AS sun_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#61 = Monday   ) THEN sales_price#4 ELSE null END))#63,17,2) AS mon_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#61 = Tuesday  ) THEN sales_price#4 ELSE null END))#64,17,2) AS tue_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#61 = Wednesday) THEN sales_price#4 ELSE null END))#65,17,2) AS wed_sales#38, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#61 = Thursday ) THEN sales_price#4 ELSE null END))#66,17,2) AS thu_sales#39, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#61 = Friday   ) THEN sales_price#4 ELSE null END))#67,17,2) AS fri_sales#40, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#61 = Saturday ) THEN sales_price#4 ELSE null END))#68,17,2) AS sat_sales#41]
+
+(26) Scan parquet default.date_dim
+Output [2]: [d_week_seq#69, d_year#70]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_week_seq)]
+ReadSchema: struct<d_week_seq:int,d_year:int>
+
+(27) ColumnarToRow [codegen id : 10]
+Input [2]: [d_week_seq#69, d_year#70]
+
+(28) Filter [codegen id : 10]
+Input [2]: [d_week_seq#69, d_year#70]
+Condition : ((isnotnull(d_year#70) AND (d_year#70 = 2002)) AND isnotnull(d_week_seq#69))
+
+(29) Project [codegen id : 10]
+Output [1]: [d_week_seq#69]
+Input [2]: [d_week_seq#69, d_year#70]
+
+(30) BroadcastExchange
+Input [1]: [d_week_seq#69]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#71]
+
+(31) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [d_week_seq#53]
+Right keys [1]: [d_week_seq#69]
+Join condition: None
+
+(32) Project [codegen id : 11]
+Output [8]: [d_week_seq#53 AS d_week_seq2#72, sun_sales#35 AS sun_sales2#73, mon_sales#36 AS mon_sales2#74, tue_sales#37 AS tue_sales2#75, wed_sales#38 AS wed_sales2#76, thu_sales#39 AS thu_sales2#77, fri_sales#40 AS fri_sales2#78, sat_sales#41 AS sat_sales2#79]
+Input [9]: [d_week_seq#53, sun_sales#35, mon_sales#36, tue_sales#37, wed_sales#38, thu_sales#39, fri_sales#40, sat_sales#41, d_week_seq#69]
+
+(33) BroadcastExchange
+Input [8]: [d_week_seq2#72, sun_sales2#73, mon_sales2#74, tue_sales2#75, wed_sales2#76, thu_sales2#77, fri_sales2#78, sat_sales2#79]
+Arguments: HashedRelationBroadcastMode(List(cast((input[0, int, true] - 53) as bigint)),false), [id=#80]
+
+(34) BroadcastHashJoin [codegen id : 12]
+Left keys [1]: [d_week_seq1#45]
+Right keys [1]: [(d_week_seq2#72 - 53)]
+Join condition: None
+
+(35) Project [codegen id : 12]
+Output [8]: [d_week_seq1#45, round(CheckOverflow((promote_precision(sun_sales1#46) / promote_precision(sun_sales2#73)), DecimalType(37,20), true), 2) AS round((sun_sales1 / sun_sales2), 2)#81, round(CheckOverflow((promote_precision(mon_sales1#47) / promote_precision(mon_sales2#74)), DecimalType(37,20), true), 2) AS round((mon_sales1 / mon_sales2), 2)#82, round(CheckOverflow((promote_precision(tue_sales1#48) / promote_precision(tue_sales2#75)), DecimalType(37,20), true), 2) AS round((tue_sales1 / tue_sales2), 2)#83, round(CheckOverflow((promote_precision(wed_sales1#49) / promote_precision(wed_sales2#76)), DecimalType(37,20), true), 2) AS round((wed_sales1 / wed_sales2), 2)#84, round(CheckOverflow((promote_precision(thu_sales1#50) / promote_precision(thu_sales2#77)), DecimalType(37,20), true), 2) AS round((thu_sales1 / thu_sales2), 2)#85, round(CheckOverflow((promote_precision(fri_sales1#51) / promote_precision(fri_sales2#78)), DecimalType(37,20), true), 2) AS round((fri_sales1 / fri_sales2), 2)#86, round(CheckOverflow((promote_precision(sat_sales1#52) / promote_precision(sat_sales2#79)), DecimalType(37,20), true), 2) AS round((sat_sales1 / sat_sales2), 2)#87]
+Input [16]: [d_week_seq1#45, sun_sales1#46, mon_sales1#47, tue_sales1#48, wed_sales1#49, thu_sales1#50, fri_sales1#51, sat_sales1#52, d_week_seq2#72, sun_sales2#73, mon_sales2#74, tue_sales2#75, wed_sales2#76, thu_sales2#77, fri_sales2#78, sat_sales2#79]
+
+(36) Exchange
+Input [8]: [d_week_seq1#45, round((sun_sales1 / sun_sales2), 2)#81, round((mon_sales1 / mon_sales2), 2)#82, round((tue_sales1 / tue_sales2), 2)#83, round((wed_sales1 / wed_sales2), 2)#84, round((thu_sales1 / thu_sales2), 2)#85, round((fri_sales1 / fri_sales2), 2)#86, round((sat_sales1 / sat_sales2), 2)#87]
+Arguments: rangepartitioning(d_week_seq1#45 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [id=#88]
+
+(37) Sort [codegen id : 13]
+Input [8]: [d_week_seq1#45, round((sun_sales1 / sun_sales2), 2)#81, round((mon_sales1 / mon_sales2), 2)#82, round((tue_sales1 / tue_sales2), 2)#83, round((wed_sales1 / wed_sales2), 2)#84, round((thu_sales1 / thu_sales2), 2)#85, round((fri_sales1 / fri_sales2), 2)#86, round((sat_sales1 / sat_sales2), 2)#87]
+Arguments: [d_week_seq1#45 ASC NULLS FIRST], true, 0
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.sf100/simplified.txt
new file mode 100644
index 0000000000000..036f9e8f9caa5
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.sf100/simplified.txt
@@ -0,0 +1,59 @@
+WholeStageCodegen (13)
+  Sort [d_week_seq1]
+    InputAdapter
+      Exchange [d_week_seq1] #1
+        WholeStageCodegen (12)
+          Project [d_week_seq1,sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2]
+            BroadcastHashJoin [d_week_seq1,d_week_seq2]
+              Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales]
+                BroadcastHashJoin [d_week_seq,d_week_seq]
+                  HashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday   ) THEN sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday   ) THEN sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday  ) THEN sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday   ) THEN sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price ELSE null END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum]
+                    InputAdapter
+                      Exchange [d_week_seq] #2
+                        WholeStageCodegen (4)
+                          HashAggregate [d_week_seq,d_day_name,sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum]
+                            Project [sales_price,d_week_seq,d_day_name]
+                              BroadcastHashJoin [sold_date_sk,d_date_sk]
+                                InputAdapter
+                                  Union
+                                    WholeStageCodegen (1)
+                                      Project [ws_sold_date_sk,ws_ext_sales_price]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.web_sales [ws_ext_sales_price,ws_sold_date_sk]
+                                    WholeStageCodegen (2)
+                                      Project [cs_sold_date_sk,cs_ext_sales_price]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk]
+                                InputAdapter
+                                  BroadcastExchange #3
+                                    WholeStageCodegen (3)
+                                      Filter [d_date_sk,d_week_seq]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.date_dim [d_date_sk,d_week_seq,d_day_name]
+                  InputAdapter
+                    BroadcastExchange #4
+                      WholeStageCodegen (5)
+                        Project [d_week_seq]
+                          Filter [d_year,d_week_seq]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.date_dim [d_week_seq,d_year]
+              InputAdapter
+                BroadcastExchange #5
+                  WholeStageCodegen (11)
+                    Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales]
+                      BroadcastHashJoin [d_week_seq,d_week_seq]
+                        HashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday   ) THEN sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday   ) THEN sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday  ) THEN sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday   ) THEN sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price ELSE null END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum]
+                          InputAdapter
+                            ReusedExchange [d_week_seq,sum,sum,sum,sum,sum,sum,sum] #2
+                        InputAdapter
+                          BroadcastExchange #6
+                            WholeStageCodegen (10)
+                              Project [d_week_seq]
+                                Filter [d_year,d_week_seq]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.date_dim [d_week_seq,d_year]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/explain.txt
new file mode 100644
index 0000000000000..c31bb7470648f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/explain.txt
@@ -0,0 +1,208 @@
+== Physical Plan ==
+* Sort (37)
++- Exchange (36)
+   +- * Project (35)
+      +- * BroadcastHashJoin Inner BuildRight (34)
+         :- * Project (23)
+         :  +- * BroadcastHashJoin Inner BuildRight (22)
+         :     :- * HashAggregate (16)
+         :     :  +- Exchange (15)
+         :     :     +- * HashAggregate (14)
+         :     :        +- * Project (13)
+         :     :           +- * BroadcastHashJoin Inner BuildRight (12)
+         :     :              :- Union (7)
+         :     :              :  :- * Project (3)
+         :     :              :  :  +- * ColumnarToRow (2)
+         :     :              :  :     +- Scan parquet default.web_sales (1)
+         :     :              :  +- * Project (6)
+         :     :              :     +- * ColumnarToRow (5)
+         :     :              :        +- Scan parquet default.catalog_sales (4)
+         :     :              +- BroadcastExchange (11)
+         :     :                 +- * Filter (10)
+         :     :                    +- * ColumnarToRow (9)
+         :     :                       +- Scan parquet default.date_dim (8)
+         :     +- BroadcastExchange (21)
+         :        +- * Project (20)
+         :           +- * Filter (19)
+         :              +- * ColumnarToRow (18)
+         :                 +- Scan parquet default.date_dim (17)
+         +- BroadcastExchange (33)
+            +- * Project (32)
+               +- * BroadcastHashJoin Inner BuildRight (31)
+                  :- * HashAggregate (25)
+                  :  +- ReusedExchange (24)
+                  +- BroadcastExchange (30)
+                     +- * Project (29)
+                        +- * Filter (28)
+                           +- * ColumnarToRow (27)
+                              +- Scan parquet default.date_dim (26)
+
+
+(1) Scan parquet default.web_sales
+Output [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#2)]
+ReadSchema: struct<ws_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2]
+
+(3) Project [codegen id : 1]
+Output [2]: [ws_sold_date_sk#2 AS sold_date_sk#3, ws_ext_sales_price#1 AS sales_price#4]
+Input [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2]
+
+(4) Scan parquet default.catalog_sales
+Output [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#6)]
+ReadSchema: struct<cs_ext_sales_price:decimal(7,2)>
+
+(5) ColumnarToRow [codegen id : 2]
+Input [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6]
+
+(6) Project [codegen id : 2]
+Output [2]: [cs_sold_date_sk#6 AS sold_date_sk#7, cs_ext_sales_price#5 AS sales_price#8]
+Input [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6]
+
+(7) Union
+
+(8) Scan parquet default.date_dim
+Output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date_sk), IsNotNull(d_week_seq)]
+ReadSchema: struct<d_date_sk:int,d_week_seq:int,d_day_name:string>
+
+(9) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11]
+
+(10) Filter [codegen id : 3]
+Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11]
+Condition : (isnotnull(d_date_sk#9) AND isnotnull(d_week_seq#10))
+
+(11) BroadcastExchange
+Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#12]
+
+(12) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [sold_date_sk#3]
+Right keys [1]: [d_date_sk#9]
+Join condition: None
+
+(13) Project [codegen id : 4]
+Output [3]: [sales_price#4, d_week_seq#10, d_day_name#11]
+Input [5]: [sold_date_sk#3, sales_price#4, d_date_sk#9, d_week_seq#10, d_day_name#11]
+
+(14) HashAggregate [codegen id : 4]
+Input [3]: [sales_price#4, d_week_seq#10, d_day_name#11]
+Keys [1]: [d_week_seq#10]
+Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday   ) THEN sales_price#4 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday   ) THEN sales_price#4 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday  ) THEN sales_price#4 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday   ) THEN sales_price#4 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 ELSE null END))]
+Aggregate Attributes [7]: [sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19]
+Results [8]: [d_week_seq#10, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25, sum#26]
+
+(15) Exchange
+Input [8]: [d_week_seq#10, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25, sum#26]
+Arguments: hashpartitioning(d_week_seq#10, 5), ENSURE_REQUIREMENTS, [id=#27]
+
+(16) HashAggregate [codegen id : 12]
+Input [8]: [d_week_seq#10, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25, sum#26]
+Keys [1]: [d_week_seq#10]
+Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday   ) THEN sales_price#4 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday   ) THEN sales_price#4 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday  ) THEN sales_price#4 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday   ) THEN sales_price#4 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 ELSE null END))]
+Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday   ) THEN sales_price#4 ELSE null END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday   ) THEN sales_price#4 ELSE null END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday  ) THEN sales_price#4 ELSE null END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 ELSE null END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 ELSE null END))#32, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday   ) THEN sales_price#4 ELSE null END))#33, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 ELSE null END))#34]
+Results [8]: [d_week_seq#10, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday   ) THEN sales_price#4 ELSE null END))#28,17,2) AS sun_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday   ) THEN sales_price#4 ELSE null END))#29,17,2) AS mon_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday  ) THEN sales_price#4 ELSE null END))#30,17,2) AS tue_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 ELSE null END))#31,17,2) AS wed_sales#38, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 ELSE null END))#32,17,2) AS thu_sales#39, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday   ) THEN sales_price#4 ELSE null END))#33,17,2) AS fri_sales#40, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 ELSE null END))#34,17,2) AS sat_sales#41]
+
+(17) Scan parquet default.date_dim
+Output [2]: [d_week_seq#42, d_year#43]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_week_seq)]
+ReadSchema: struct<d_week_seq:int,d_year:int>
+
+(18) ColumnarToRow [codegen id : 5]
+Input [2]: [d_week_seq#42, d_year#43]
+
+(19) Filter [codegen id : 5]
+Input [2]: [d_week_seq#42, d_year#43]
+Condition : ((isnotnull(d_year#43) AND (d_year#43 = 2001)) AND isnotnull(d_week_seq#42))
+
+(20) Project [codegen id : 5]
+Output [1]: [d_week_seq#42]
+Input [2]: [d_week_seq#42, d_year#43]
+
+(21) BroadcastExchange
+Input [1]: [d_week_seq#42]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#44]
+
+(22) BroadcastHashJoin [codegen id : 12]
+Left keys [1]: [d_week_seq#10]
+Right keys [1]: [d_week_seq#42]
+Join condition: None
+
+(23) Project [codegen id : 12]
+Output [8]: [d_week_seq#10 AS d_week_seq1#45, sun_sales#35 AS sun_sales1#46, mon_sales#36 AS mon_sales1#47, tue_sales#37 AS tue_sales1#48, wed_sales#38 AS wed_sales1#49, thu_sales#39 AS thu_sales1#50, fri_sales#40 AS fri_sales1#51, sat_sales#41 AS sat_sales1#52]
+Input [9]: [d_week_seq#10, sun_sales#35, mon_sales#36, tue_sales#37, wed_sales#38, thu_sales#39, fri_sales#40, sat_sales#41, d_week_seq#42]
+
+(24) ReusedExchange [Reuses operator id: 15]
+Output [8]: [d_week_seq#53, sum#54, sum#55, sum#56, sum#57, sum#58, sum#59, sum#60]
+
+(25) HashAggregate [codegen id : 11]
+Input [8]: [d_week_seq#53, sum#54, sum#55, sum#56, sum#57, sum#58, sum#59, sum#60]
+Keys [1]: [d_week_seq#53]
+Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#61 = Sunday   ) THEN sales_price#4 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#61 = Monday   ) THEN sales_price#4 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#61 = Tuesday  ) THEN sales_price#4 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#61 = Wednesday) THEN sales_price#4 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#61 = Thursday ) THEN sales_price#4 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#61 = Friday   ) THEN sales_price#4 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#61 = Saturday ) THEN sales_price#4 ELSE null END))]
+Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#61 = Sunday   ) THEN sales_price#4 ELSE null END))#62, sum(UnscaledValue(CASE WHEN (d_day_name#61 = Monday   ) THEN sales_price#4 ELSE null END))#63, sum(UnscaledValue(CASE WHEN (d_day_name#61 = Tuesday  ) THEN sales_price#4 ELSE null END))#64, sum(UnscaledValue(CASE WHEN (d_day_name#61 = Wednesday) THEN sales_price#4 ELSE null END))#65, sum(UnscaledValue(CASE WHEN (d_day_name#61 = Thursday ) THEN sales_price#4 ELSE null END))#66, sum(UnscaledValue(CASE WHEN (d_day_name#61 = Friday   ) THEN sales_price#4 ELSE null END))#67, sum(UnscaledValue(CASE WHEN (d_day_name#61 = Saturday ) THEN sales_price#4 ELSE null END))#68]
+Results [8]: [d_week_seq#53, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#61 = Sunday   ) THEN sales_price#4 ELSE null END))#62,17,2) AS sun_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#61 = Monday   ) THEN sales_price#4 ELSE null END))#63,17,2) AS mon_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#61 = Tuesday  ) THEN sales_price#4 ELSE null END))#64,17,2) AS tue_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#61 = Wednesday) THEN sales_price#4 ELSE null END))#65,17,2) AS wed_sales#38, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#61 = Thursday ) THEN sales_price#4 ELSE null END))#66,17,2) AS thu_sales#39, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#61 = Friday   ) THEN sales_price#4 ELSE null END))#67,17,2) AS fri_sales#40, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#61 = Saturday ) THEN sales_price#4 ELSE null END))#68,17,2) AS sat_sales#41]
+
+(26) Scan parquet default.date_dim
+Output [2]: [d_week_seq#69, d_year#70]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_week_seq)]
+ReadSchema: struct<d_week_seq:int,d_year:int>
+
+(27) ColumnarToRow [codegen id : 10]
+Input [2]: [d_week_seq#69, d_year#70]
+
+(28) Filter [codegen id : 10]
+Input [2]: [d_week_seq#69, d_year#70]
+Condition : ((isnotnull(d_year#70) AND (d_year#70 = 2002)) AND isnotnull(d_week_seq#69))
+
+(29) Project [codegen id : 10]
+Output [1]: [d_week_seq#69]
+Input [2]: [d_week_seq#69, d_year#70]
+
+(30) BroadcastExchange
+Input [1]: [d_week_seq#69]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#71]
+
+(31) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [d_week_seq#53]
+Right keys [1]: [d_week_seq#69]
+Join condition: None
+
+(32) Project [codegen id : 11]
+Output [8]: [d_week_seq#53 AS d_week_seq2#72, sun_sales#35 AS sun_sales2#73, mon_sales#36 AS mon_sales2#74, tue_sales#37 AS tue_sales2#75, wed_sales#38 AS wed_sales2#76, thu_sales#39 AS thu_sales2#77, fri_sales#40 AS fri_sales2#78, sat_sales#41 AS sat_sales2#79]
+Input [9]: [d_week_seq#53, sun_sales#35, mon_sales#36, tue_sales#37, wed_sales#38, thu_sales#39, fri_sales#40, sat_sales#41, d_week_seq#69]
+
+(33) BroadcastExchange
+Input [8]: [d_week_seq2#72, sun_sales2#73, mon_sales2#74, tue_sales2#75, wed_sales2#76, thu_sales2#77, fri_sales2#78, sat_sales2#79]
+Arguments: HashedRelationBroadcastMode(List(cast((input[0, int, true] - 53) as bigint)),false), [id=#80]
+
+(34) BroadcastHashJoin [codegen id : 12]
+Left keys [1]: [d_week_seq1#45]
+Right keys [1]: [(d_week_seq2#72 - 53)]
+Join condition: None
+
+(35) Project [codegen id : 12]
+Output [8]: [d_week_seq1#45, round(CheckOverflow((promote_precision(sun_sales1#46) / promote_precision(sun_sales2#73)), DecimalType(37,20), true), 2) AS round((sun_sales1 / sun_sales2), 2)#81, round(CheckOverflow((promote_precision(mon_sales1#47) / promote_precision(mon_sales2#74)), DecimalType(37,20), true), 2) AS round((mon_sales1 / mon_sales2), 2)#82, round(CheckOverflow((promote_precision(tue_sales1#48) / promote_precision(tue_sales2#75)), DecimalType(37,20), true), 2) AS round((tue_sales1 / tue_sales2), 2)#83, round(CheckOverflow((promote_precision(wed_sales1#49) / promote_precision(wed_sales2#76)), DecimalType(37,20), true), 2) AS round((wed_sales1 / wed_sales2), 2)#84, round(CheckOverflow((promote_precision(thu_sales1#50) / promote_precision(thu_sales2#77)), DecimalType(37,20), true), 2) AS round((thu_sales1 / thu_sales2), 2)#85, round(CheckOverflow((promote_precision(fri_sales1#51) / promote_precision(fri_sales2#78)), DecimalType(37,20), true), 2) AS round((fri_sales1 / fri_sales2), 2)#86, round(CheckOverflow((promote_precision(sat_sales1#52) / promote_precision(sat_sales2#79)), DecimalType(37,20), true), 2) AS round((sat_sales1 / sat_sales2), 2)#87]
+Input [16]: [d_week_seq1#45, sun_sales1#46, mon_sales1#47, tue_sales1#48, wed_sales1#49, thu_sales1#50, fri_sales1#51, sat_sales1#52, d_week_seq2#72, sun_sales2#73, mon_sales2#74, tue_sales2#75, wed_sales2#76, thu_sales2#77, fri_sales2#78, sat_sales2#79]
+
+(36) Exchange
+Input [8]: [d_week_seq1#45, round((sun_sales1 / sun_sales2), 2)#81, round((mon_sales1 / mon_sales2), 2)#82, round((tue_sales1 / tue_sales2), 2)#83, round((wed_sales1 / wed_sales2), 2)#84, round((thu_sales1 / thu_sales2), 2)#85, round((fri_sales1 / fri_sales2), 2)#86, round((sat_sales1 / sat_sales2), 2)#87]
+Arguments: rangepartitioning(d_week_seq1#45 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [id=#88]
+
+(37) Sort [codegen id : 13]
+Input [8]: [d_week_seq1#45, round((sun_sales1 / sun_sales2), 2)#81, round((mon_sales1 / mon_sales2), 2)#82, round((tue_sales1 / tue_sales2), 2)#83, round((wed_sales1 / wed_sales2), 2)#84, round((thu_sales1 / thu_sales2), 2)#85, round((fri_sales1 / fri_sales2), 2)#86, round((sat_sales1 / sat_sales2), 2)#87]
+Arguments: [d_week_seq1#45 ASC NULLS FIRST], true, 0
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/simplified.txt
new file mode 100644
index 0000000000000..036f9e8f9caa5
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/simplified.txt
@@ -0,0 +1,59 @@
+WholeStageCodegen (13)
+  Sort [d_week_seq1]
+    InputAdapter
+      Exchange [d_week_seq1] #1
+        WholeStageCodegen (12)
+          Project [d_week_seq1,sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2]
+            BroadcastHashJoin [d_week_seq1,d_week_seq2]
+              Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales]
+                BroadcastHashJoin [d_week_seq,d_week_seq]
+                  HashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday   ) THEN sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday   ) THEN sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday  ) THEN sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday   ) THEN sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price ELSE null END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum]
+                    InputAdapter
+                      Exchange [d_week_seq] #2
+                        WholeStageCodegen (4)
+                          HashAggregate [d_week_seq,d_day_name,sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum]
+                            Project [sales_price,d_week_seq,d_day_name]
+                              BroadcastHashJoin [sold_date_sk,d_date_sk]
+                                InputAdapter
+                                  Union
+                                    WholeStageCodegen (1)
+                                      Project [ws_sold_date_sk,ws_ext_sales_price]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.web_sales [ws_ext_sales_price,ws_sold_date_sk]
+                                    WholeStageCodegen (2)
+                                      Project [cs_sold_date_sk,cs_ext_sales_price]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk]
+                                InputAdapter
+                                  BroadcastExchange #3
+                                    WholeStageCodegen (3)
+                                      Filter [d_date_sk,d_week_seq]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.date_dim [d_date_sk,d_week_seq,d_day_name]
+                  InputAdapter
+                    BroadcastExchange #4
+                      WholeStageCodegen (5)
+                        Project [d_week_seq]
+                          Filter [d_year,d_week_seq]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.date_dim [d_week_seq,d_year]
+              InputAdapter
+                BroadcastExchange #5
+                  WholeStageCodegen (11)
+                    Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales]
+                      BroadcastHashJoin [d_week_seq,d_week_seq]
+                        HashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday   ) THEN sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday   ) THEN sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday  ) THEN sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday   ) THEN sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price ELSE null END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum]
+                          InputAdapter
+                            ReusedExchange [d_week_seq,sum,sum,sum,sum,sum,sum,sum] #2
+                        InputAdapter
+                          BroadcastExchange #6
+                            WholeStageCodegen (10)
+                              Project [d_week_seq]
+                                Filter [d_year,d_week_seq]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.date_dim [d_week_seq,d_year]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.sf100/explain.txt
new file mode 100644
index 0000000000000..d6bbbfc515205
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.sf100/explain.txt
@@ -0,0 +1,163 @@
+== Physical Plan ==
+TakeOrderedAndProject (27)
++- * Project (26)
+   +- Window (25)
+      +- * Sort (24)
+         +- Exchange (23)
+            +- * HashAggregate (22)
+               +- Exchange (21)
+                  +- * HashAggregate (20)
+                     +- * Project (19)
+                        +- * BroadcastHashJoin Inner BuildRight (18)
+                           :- * Project (12)
+                           :  +- * SortMergeJoin Inner (11)
+                           :     :- * Sort (5)
+                           :     :  +- Exchange (4)
+                           :     :     +- * Filter (3)
+                           :     :        +- * ColumnarToRow (2)
+                           :     :           +- Scan parquet default.catalog_sales (1)
+                           :     +- * Sort (10)
+                           :        +- Exchange (9)
+                           :           +- * Filter (8)
+                           :              +- * ColumnarToRow (7)
+                           :                 +- Scan parquet default.item (6)
+                           +- BroadcastExchange (17)
+                              +- * Project (16)
+                                 +- * Filter (15)
+                                    +- * ColumnarToRow (14)
+                                       +- Scan parquet default.date_dim (13)
+
+
+(1) Scan parquet default.catalog_sales
+Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int,cs_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3]
+
+(3) Filter [codegen id : 1]
+Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3]
+Condition : isnotnull(cs_item_sk#1)
+
+(4) Exchange
+Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3]
+Arguments: hashpartitioning(cs_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#5]
+
+(5) Sort [codegen id : 2]
+Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3]
+Arguments: [cs_item_sk#1 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.item
+Output [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [In(i_category, [Sports                                            ,Books                                             ,Home                                              ]), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_item_desc:string,i_current_price:decimal(7,2),i_class:string,i_category:string>
+
+(7) ColumnarToRow [codegen id : 3]
+Input [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+
+(8) Filter [codegen id : 3]
+Input [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Condition : (i_category#11 IN (Sports                                            ,Books                                             ,Home                                              ) AND isnotnull(i_item_sk#6))
+
+(9) Exchange
+Input [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Arguments: hashpartitioning(i_item_sk#6, 5), ENSURE_REQUIREMENTS, [id=#12]
+
+(10) Sort [codegen id : 4]
+Input [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Arguments: [i_item_sk#6 ASC NULLS FIRST], false, 0
+
+(11) SortMergeJoin [codegen id : 6]
+Left keys [1]: [cs_item_sk#1]
+Right keys [1]: [i_item_sk#6]
+Join condition: None
+
+(12) Project [codegen id : 6]
+Output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+
+(13) Scan parquet default.date_dim
+Output [2]: [d_date_sk#13, d_date#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(14) ColumnarToRow [codegen id : 5]
+Input [2]: [d_date_sk#13, d_date#14]
+
+(15) Filter [codegen id : 5]
+Input [2]: [d_date_sk#13, d_date#14]
+Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13))
+
+(16) Project [codegen id : 5]
+Output [1]: [d_date_sk#13]
+Input [2]: [d_date_sk#13, d_date#14]
+
+(17) BroadcastExchange
+Input [1]: [d_date_sk#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#15]
+
+(18) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [cs_sold_date_sk#3]
+Right keys [1]: [d_date_sk#13]
+Join condition: None
+
+(19) Project [codegen id : 6]
+Output [6]: [cs_ext_sales_price#2, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11, d_date_sk#13]
+
+(20) HashAggregate [codegen id : 6]
+Input [6]: [cs_ext_sales_price#2, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Keys [5]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9]
+Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum#16]
+Results [6]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, sum#17]
+
+(21) Exchange
+Input [6]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, sum#17]
+Arguments: hashpartitioning(i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(22) HashAggregate [codegen id : 7]
+Input [6]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, sum#17]
+Keys [5]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9]
+Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#2))#19]
+Results [8]: [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#19,17,2) AS itemrevenue#20, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#19,17,2) AS _w0#21, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#19,17,2) AS _w1#22, i_item_id#7]
+
+(23) Exchange
+Input [8]: [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, _w0#21, _w1#22, i_item_id#7]
+Arguments: hashpartitioning(i_class#10, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(24) Sort [codegen id : 8]
+Input [8]: [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, _w0#21, _w1#22, i_item_id#7]
+Arguments: [i_class#10 ASC NULLS FIRST], false, 0
+
+(25) Window
+Input [8]: [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, _w0#21, _w1#22, i_item_id#7]
+Arguments: [sum(_w1#22) windowspecdefinition(i_class#10, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#24], [i_class#10]
+
+(26) Project [codegen id : 9]
+Output [7]: [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(_w0#21) * 100.00), DecimalType(21,2), true) as decimal(27,2))) / promote_precision(_we0#24)), DecimalType(38,17), true) AS revenueratio#25, i_item_id#7]
+Input [9]: [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, _w0#21, _w1#22, i_item_id#7, _we0#24]
+
+(27) TakeOrderedAndProject
+Input [7]: [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, revenueratio#25, i_item_id#7]
+Arguments: 100, [i_category#11 ASC NULLS FIRST, i_class#10 ASC NULLS FIRST, i_item_id#7 ASC NULLS FIRST, i_item_desc#8 ASC NULLS FIRST, revenueratio#25 ASC NULLS FIRST], [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, revenueratio#25]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (28)
+
+
+(28) ReusedExchange [Reuses operator id: 17]
+Output [1]: [d_date_sk#13]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.sf100/simplified.txt
new file mode 100644
index 0000000000000..bc14161be8ee2
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.sf100/simplified.txt
@@ -0,0 +1,49 @@
+TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue]
+  WholeStageCodegen (9)
+    Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id]
+      InputAdapter
+        Window [_w1,i_class]
+          WholeStageCodegen (8)
+            Sort [i_class]
+              InputAdapter
+                Exchange [i_class] #1
+                  WholeStageCodegen (7)
+                    HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(cs_ext_sales_price)),itemrevenue,_w0,_w1,sum]
+                      InputAdapter
+                        Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2
+                          WholeStageCodegen (6)
+                            HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,cs_ext_sales_price] [sum,sum]
+                              Project [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                  Project [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                    SortMergeJoin [cs_item_sk,i_item_sk]
+                                      InputAdapter
+                                        WholeStageCodegen (2)
+                                          Sort [cs_item_sk]
+                                            InputAdapter
+                                              Exchange [cs_item_sk] #3
+                                                WholeStageCodegen (1)
+                                                  Filter [cs_item_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk]
+                                                          SubqueryBroadcast [d_date_sk] #1
+                                                            ReusedExchange [d_date_sk] #4
+                                      InputAdapter
+                                        WholeStageCodegen (4)
+                                          Sort [i_item_sk]
+                                            InputAdapter
+                                              Exchange [i_item_sk] #5
+                                                WholeStageCodegen (3)
+                                                  Filter [i_category,i_item_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                  InputAdapter
+                                    BroadcastExchange #4
+                                      WholeStageCodegen (5)
+                                        Project [d_date_sk]
+                                          Filter [d_date,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_date]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/explain.txt
new file mode 100644
index 0000000000000..4f5eebc0c300c
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/explain.txt
@@ -0,0 +1,148 @@
+== Physical Plan ==
+TakeOrderedAndProject (24)
++- * Project (23)
+   +- Window (22)
+      +- * Sort (21)
+         +- Exchange (20)
+            +- * HashAggregate (19)
+               +- Exchange (18)
+                  +- * HashAggregate (17)
+                     +- * Project (16)
+                        +- * BroadcastHashJoin Inner BuildRight (15)
+                           :- * Project (9)
+                           :  +- * BroadcastHashJoin Inner BuildRight (8)
+                           :     :- * Filter (3)
+                           :     :  +- * ColumnarToRow (2)
+                           :     :     +- Scan parquet default.catalog_sales (1)
+                           :     +- BroadcastExchange (7)
+                           :        +- * Filter (6)
+                           :           +- * ColumnarToRow (5)
+                           :              +- Scan parquet default.item (4)
+                           +- BroadcastExchange (14)
+                              +- * Project (13)
+                                 +- * Filter (12)
+                                    +- * ColumnarToRow (11)
+                                       +- Scan parquet default.date_dim (10)
+
+
+(1) Scan parquet default.catalog_sales
+Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int,cs_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3]
+Condition : isnotnull(cs_item_sk#1)
+
+(4) Scan parquet default.item
+Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [In(i_category, [Sports                                            ,Books                                             ,Home                                              ]), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_item_desc:string,i_current_price:decimal(7,2),i_class:string,i_category:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+
+(6) Filter [codegen id : 1]
+Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Condition : (i_category#10 IN (Sports                                            ,Books                                             ,Home                                              ) AND isnotnull(i_item_sk#5))
+
+(7) BroadcastExchange
+Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(8) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_item_sk#1]
+Right keys [1]: [i_item_sk#5]
+Join condition: None
+
+(9) Project [codegen id : 3]
+Output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+
+(10) Scan parquet default.date_dim
+Output [2]: [d_date_sk#12, d_date#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#12, d_date#13]
+
+(12) Filter [codegen id : 2]
+Input [2]: [d_date_sk#12, d_date#13]
+Condition : (((isnotnull(d_date#13) AND (d_date#13 >= 1999-02-22)) AND (d_date#13 <= 1999-03-24)) AND isnotnull(d_date_sk#12))
+
+(13) Project [codegen id : 2]
+Output [1]: [d_date_sk#12]
+Input [2]: [d_date_sk#12, d_date#13]
+
+(14) BroadcastExchange
+Input [1]: [d_date_sk#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_sold_date_sk#3]
+Right keys [1]: [d_date_sk#12]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [6]: [cs_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10, d_date_sk#12]
+
+(17) HashAggregate [codegen id : 3]
+Input [6]: [cs_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8]
+Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum#15]
+Results [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#16]
+
+(18) Exchange
+Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#16]
+Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(19) HashAggregate [codegen id : 4]
+Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#16]
+Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8]
+Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#2))#18]
+Results [8]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#18,17,2) AS itemrevenue#19, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#18,17,2) AS _w0#20, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#18,17,2) AS _w1#21, i_item_id#6]
+
+(20) Exchange
+Input [8]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21, i_item_id#6]
+Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(21) Sort [codegen id : 5]
+Input [8]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21, i_item_id#6]
+Arguments: [i_class#9 ASC NULLS FIRST], false, 0
+
+(22) Window
+Input [8]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21, i_item_id#6]
+Arguments: [sum(_w1#21) windowspecdefinition(i_class#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#23], [i_class#9]
+
+(23) Project [codegen id : 6]
+Output [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(_w0#20) * 100.00), DecimalType(21,2), true) as decimal(27,2))) / promote_precision(_we0#23)), DecimalType(38,17), true) AS revenueratio#24, i_item_id#6]
+Input [9]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21, i_item_id#6, _we0#23]
+
+(24) TakeOrderedAndProject
+Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, revenueratio#24, i_item_id#6]
+Arguments: 100, [i_category#10 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_item_id#6 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#24 ASC NULLS FIRST], [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, revenueratio#24]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (25)
+
+
+(25) ReusedExchange [Reuses operator id: 14]
+Output [1]: [d_date_sk#12]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/simplified.txt
new file mode 100644
index 0000000000000..d971de651c863
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/simplified.txt
@@ -0,0 +1,40 @@
+TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue]
+  WholeStageCodegen (6)
+    Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id]
+      InputAdapter
+        Window [_w1,i_class]
+          WholeStageCodegen (5)
+            Sort [i_class]
+              InputAdapter
+                Exchange [i_class] #1
+                  WholeStageCodegen (4)
+                    HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(cs_ext_sales_price)),itemrevenue,_w0,_w1,sum]
+                      InputAdapter
+                        Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2
+                          WholeStageCodegen (3)
+                            HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,cs_ext_sales_price] [sum,sum]
+                              Project [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                  Project [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                    BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                      Filter [cs_item_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #1
+                                                ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        BroadcastExchange #4
+                                          WholeStageCodegen (1)
+                                            Filter [i_category,i_item_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                  InputAdapter
+                                    BroadcastExchange #3
+                                      WholeStageCodegen (2)
+                                        Project [d_date_sk]
+                                          Filter [d_date,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_date]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.sf100/explain.txt
new file mode 100644
index 0000000000000..efb45b5ccdb7f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.sf100/explain.txt
@@ -0,0 +1,166 @@
+== Physical Plan ==
+TakeOrderedAndProject (27)
++- * Filter (26)
+   +- * HashAggregate (25)
+      +- Exchange (24)
+         +- * HashAggregate (23)
+            +- * Project (22)
+               +- * BroadcastHashJoin Inner BuildRight (21)
+                  :- * Project (16)
+                  :  +- * BroadcastHashJoin Inner BuildRight (15)
+                  :     :- * Project (10)
+                  :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+                  :     :     :- * Filter (3)
+                  :     :     :  +- * ColumnarToRow (2)
+                  :     :     :     +- Scan parquet default.inventory (1)
+                  :     :     +- BroadcastExchange (8)
+                  :     :        +- * Project (7)
+                  :     :           +- * Filter (6)
+                  :     :              +- * ColumnarToRow (5)
+                  :     :                 +- Scan parquet default.item (4)
+                  :     +- BroadcastExchange (14)
+                  :        +- * Filter (13)
+                  :           +- * ColumnarToRow (12)
+                  :              +- Scan parquet default.date_dim (11)
+                  +- BroadcastExchange (20)
+                     +- * Filter (19)
+                        +- * ColumnarToRow (18)
+                           +- Scan parquet default.warehouse (17)
+
+
+(1) Scan parquet default.inventory
+Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(inv_warehouse_sk), IsNotNull(inv_item_sk)]
+ReadSchema: struct<inv_item_sk:int,inv_warehouse_sk:int,inv_quantity_on_hand:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+
+(3) Filter [codegen id : 4]
+Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+Condition : (isnotnull(inv_warehouse_sk#2) AND isnotnull(inv_item_sk#1))
+
+(4) Scan parquet default.item
+Output [3]: [i_item_sk#6, i_item_id#7, i_current_price#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,0.99), LessThanOrEqual(i_current_price,1.49), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_current_price:decimal(7,2)>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [i_item_sk#6, i_item_id#7, i_current_price#8]
+
+(6) Filter [codegen id : 1]
+Input [3]: [i_item_sk#6, i_item_id#7, i_current_price#8]
+Condition : (((isnotnull(i_current_price#8) AND (i_current_price#8 >= 0.99)) AND (i_current_price#8 <= 1.49)) AND isnotnull(i_item_sk#6))
+
+(7) Project [codegen id : 1]
+Output [2]: [i_item_sk#6, i_item_id#7]
+Input [3]: [i_item_sk#6, i_item_id#7, i_current_price#8]
+
+(8) BroadcastExchange
+Input [2]: [i_item_sk#6, i_item_id#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#9]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_item_sk#1]
+Right keys [1]: [i_item_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_id#7]
+Input [6]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, i_item_id#7]
+
+(11) Scan parquet default.date_dim
+Output [2]: [d_date_sk#10, d_date#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#10, d_date#11]
+
+(13) Filter [codegen id : 2]
+Input [2]: [d_date_sk#10, d_date#11]
+Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-02-10)) AND (d_date#11 <= 2000-04-10)) AND isnotnull(d_date_sk#10))
+
+(14) BroadcastExchange
+Input [2]: [d_date_sk#10, d_date#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#12]
+
+(15) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_date_sk#4]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(16) Project [codegen id : 4]
+Output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_id#7, d_date#11]
+Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_id#7, d_date_sk#10, d_date#11]
+
+(17) Scan parquet default.warehouse
+Output [2]: [w_warehouse_sk#13, w_warehouse_name#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/warehouse]
+PushedFilters: [IsNotNull(w_warehouse_sk)]
+ReadSchema: struct<w_warehouse_sk:int,w_warehouse_name:string>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [2]: [w_warehouse_sk#13, w_warehouse_name#14]
+
+(19) Filter [codegen id : 3]
+Input [2]: [w_warehouse_sk#13, w_warehouse_name#14]
+Condition : isnotnull(w_warehouse_sk#13)
+
+(20) BroadcastExchange
+Input [2]: [w_warehouse_sk#13, w_warehouse_name#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#15]
+
+(21) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_warehouse_sk#2]
+Right keys [1]: [w_warehouse_sk#13]
+Join condition: None
+
+(22) Project [codegen id : 4]
+Output [4]: [inv_quantity_on_hand#3, w_warehouse_name#14, i_item_id#7, d_date#11]
+Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_id#7, d_date#11, w_warehouse_sk#13, w_warehouse_name#14]
+
+(23) HashAggregate [codegen id : 4]
+Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#14, i_item_id#7, d_date#11]
+Keys [2]: [w_warehouse_name#14, i_item_id#7]
+Functions [2]: [partial_sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), partial_sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)]
+Aggregate Attributes [2]: [sum#16, sum#17]
+Results [4]: [w_warehouse_name#14, i_item_id#7, sum#18, sum#19]
+
+(24) Exchange
+Input [4]: [w_warehouse_name#14, i_item_id#7, sum#18, sum#19]
+Arguments: hashpartitioning(w_warehouse_name#14, i_item_id#7, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(25) HashAggregate [codegen id : 5]
+Input [4]: [w_warehouse_name#14, i_item_id#7, sum#18, sum#19]
+Keys [2]: [w_warehouse_name#14, i_item_id#7]
+Functions [2]: [sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)]
+Aggregate Attributes [2]: [sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#21, sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#22]
+Results [4]: [w_warehouse_name#14, i_item_id#7, sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#21 AS inv_before#23, sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#22 AS inv_after#24]
+
+(26) Filter [codegen id : 5]
+Input [4]: [w_warehouse_name#14, i_item_id#7, inv_before#23, inv_after#24]
+Condition : ((isnotnull(inv_before#23) AND isnotnull(inv_after#24)) AND (((inv_before#23 > 0) AND ((cast(inv_after#24 as double) / cast(inv_before#23 as double)) >= 0.666667)) AND ((cast(inv_after#24 as double) / cast(inv_before#23 as double)) <= 1.5)))
+
+(27) TakeOrderedAndProject
+Input [4]: [w_warehouse_name#14, i_item_id#7, inv_before#23, inv_after#24]
+Arguments: 100, [w_warehouse_name#14 ASC NULLS FIRST, i_item_id#7 ASC NULLS FIRST], [w_warehouse_name#14, i_item_id#7, inv_before#23, inv_after#24]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5
+ReusedExchange (28)
+
+
+(28) ReusedExchange [Reuses operator id: 14]
+Output [2]: [d_date_sk#10, d_date#11]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.sf100/simplified.txt
new file mode 100644
index 0000000000000..6cecd2af48359
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.sf100/simplified.txt
@@ -0,0 +1,42 @@
+TakeOrderedAndProject [w_warehouse_name,i_item_id,inv_before,inv_after]
+  WholeStageCodegen (5)
+    Filter [inv_before,inv_after]
+      HashAggregate [w_warehouse_name,i_item_id,sum,sum] [sum(CASE WHEN (d_date < 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),inv_before,inv_after,sum,sum]
+        InputAdapter
+          Exchange [w_warehouse_name,i_item_id] #1
+            WholeStageCodegen (4)
+              HashAggregate [w_warehouse_name,i_item_id,d_date,inv_quantity_on_hand] [sum,sum,sum,sum]
+                Project [inv_quantity_on_hand,w_warehouse_name,i_item_id,d_date]
+                  BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk]
+                    Project [inv_warehouse_sk,inv_quantity_on_hand,i_item_id,d_date]
+                      BroadcastHashJoin [inv_date_sk,d_date_sk]
+                        Project [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_id]
+                          BroadcastHashJoin [inv_item_sk,i_item_sk]
+                            Filter [inv_warehouse_sk,inv_item_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk]
+                                    SubqueryBroadcast [d_date_sk] #1
+                                      ReusedExchange [d_date_sk,d_date] #2
+                            InputAdapter
+                              BroadcastExchange #3
+                                WholeStageCodegen (1)
+                                  Project [i_item_sk,i_item_id]
+                                    Filter [i_current_price,i_item_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.item [i_item_sk,i_item_id,i_current_price]
+                        InputAdapter
+                          BroadcastExchange #2
+                            WholeStageCodegen (2)
+                              Filter [d_date,d_date_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.date_dim [d_date_sk,d_date]
+                    InputAdapter
+                      BroadcastExchange #4
+                        WholeStageCodegen (3)
+                          Filter [w_warehouse_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.warehouse [w_warehouse_sk,w_warehouse_name]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/explain.txt
new file mode 100644
index 0000000000000..7fcbe1befa6b4
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/explain.txt
@@ -0,0 +1,166 @@
+== Physical Plan ==
+TakeOrderedAndProject (27)
++- * Filter (26)
+   +- * HashAggregate (25)
+      +- Exchange (24)
+         +- * HashAggregate (23)
+            +- * Project (22)
+               +- * BroadcastHashJoin Inner BuildRight (21)
+                  :- * Project (16)
+                  :  +- * BroadcastHashJoin Inner BuildRight (15)
+                  :     :- * Project (9)
+                  :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+                  :     :     :- * Filter (3)
+                  :     :     :  +- * ColumnarToRow (2)
+                  :     :     :     +- Scan parquet default.inventory (1)
+                  :     :     +- BroadcastExchange (7)
+                  :     :        +- * Filter (6)
+                  :     :           +- * ColumnarToRow (5)
+                  :     :              +- Scan parquet default.warehouse (4)
+                  :     +- BroadcastExchange (14)
+                  :        +- * Project (13)
+                  :           +- * Filter (12)
+                  :              +- * ColumnarToRow (11)
+                  :                 +- Scan parquet default.item (10)
+                  +- BroadcastExchange (20)
+                     +- * Filter (19)
+                        +- * ColumnarToRow (18)
+                           +- Scan parquet default.date_dim (17)
+
+
+(1) Scan parquet default.inventory
+Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(inv_warehouse_sk), IsNotNull(inv_item_sk)]
+ReadSchema: struct<inv_item_sk:int,inv_warehouse_sk:int,inv_quantity_on_hand:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+
+(3) Filter [codegen id : 4]
+Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+Condition : (isnotnull(inv_warehouse_sk#2) AND isnotnull(inv_item_sk#1))
+
+(4) Scan parquet default.warehouse
+Output [2]: [w_warehouse_sk#6, w_warehouse_name#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/warehouse]
+PushedFilters: [IsNotNull(w_warehouse_sk)]
+ReadSchema: struct<w_warehouse_sk:int,w_warehouse_name:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [w_warehouse_sk#6, w_warehouse_name#7]
+
+(6) Filter [codegen id : 1]
+Input [2]: [w_warehouse_sk#6, w_warehouse_name#7]
+Condition : isnotnull(w_warehouse_sk#6)
+
+(7) BroadcastExchange
+Input [2]: [w_warehouse_sk#6, w_warehouse_name#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#8]
+
+(8) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_warehouse_sk#2]
+Right keys [1]: [w_warehouse_sk#6]
+Join condition: None
+
+(9) Project [codegen id : 4]
+Output [4]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7]
+Input [6]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_sk#6, w_warehouse_name#7]
+
+(10) Scan parquet default.item
+Output [3]: [i_item_sk#9, i_item_id#10, i_current_price#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,0.99), LessThanOrEqual(i_current_price,1.49), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_current_price:decimal(7,2)>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [3]: [i_item_sk#9, i_item_id#10, i_current_price#11]
+
+(12) Filter [codegen id : 2]
+Input [3]: [i_item_sk#9, i_item_id#10, i_current_price#11]
+Condition : (((isnotnull(i_current_price#11) AND (i_current_price#11 >= 0.99)) AND (i_current_price#11 <= 1.49)) AND isnotnull(i_item_sk#9))
+
+(13) Project [codegen id : 2]
+Output [2]: [i_item_sk#9, i_item_id#10]
+Input [3]: [i_item_sk#9, i_item_id#10, i_current_price#11]
+
+(14) BroadcastExchange
+Input [2]: [i_item_sk#9, i_item_id#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(15) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_item_sk#1]
+Right keys [1]: [i_item_sk#9]
+Join condition: None
+
+(16) Project [codegen id : 4]
+Output [4]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7, i_item_id#10]
+Input [6]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7, i_item_sk#9, i_item_id#10]
+
+(17) Scan parquet default.date_dim
+Output [2]: [d_date_sk#13, d_date#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [2]: [d_date_sk#13, d_date#14]
+
+(19) Filter [codegen id : 3]
+Input [2]: [d_date_sk#13, d_date#14]
+Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 2000-02-10)) AND (d_date#14 <= 2000-04-10)) AND isnotnull(d_date_sk#13))
+
+(20) BroadcastExchange
+Input [2]: [d_date_sk#13, d_date#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#15]
+
+(21) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_date_sk#4]
+Right keys [1]: [d_date_sk#13]
+Join condition: None
+
+(22) Project [codegen id : 4]
+Output [4]: [inv_quantity_on_hand#3, w_warehouse_name#7, i_item_id#10, d_date#14]
+Input [6]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7, i_item_id#10, d_date_sk#13, d_date#14]
+
+(23) HashAggregate [codegen id : 4]
+Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#7, i_item_id#10, d_date#14]
+Keys [2]: [w_warehouse_name#7, i_item_id#10]
+Functions [2]: [partial_sum(CASE WHEN (d_date#14 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), partial_sum(CASE WHEN (d_date#14 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)]
+Aggregate Attributes [2]: [sum#16, sum#17]
+Results [4]: [w_warehouse_name#7, i_item_id#10, sum#18, sum#19]
+
+(24) Exchange
+Input [4]: [w_warehouse_name#7, i_item_id#10, sum#18, sum#19]
+Arguments: hashpartitioning(w_warehouse_name#7, i_item_id#10, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(25) HashAggregate [codegen id : 5]
+Input [4]: [w_warehouse_name#7, i_item_id#10, sum#18, sum#19]
+Keys [2]: [w_warehouse_name#7, i_item_id#10]
+Functions [2]: [sum(CASE WHEN (d_date#14 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), sum(CASE WHEN (d_date#14 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)]
+Aggregate Attributes [2]: [sum(CASE WHEN (d_date#14 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#21, sum(CASE WHEN (d_date#14 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#22]
+Results [4]: [w_warehouse_name#7, i_item_id#10, sum(CASE WHEN (d_date#14 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#21 AS inv_before#23, sum(CASE WHEN (d_date#14 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#22 AS inv_after#24]
+
+(26) Filter [codegen id : 5]
+Input [4]: [w_warehouse_name#7, i_item_id#10, inv_before#23, inv_after#24]
+Condition : ((isnotnull(inv_before#23) AND isnotnull(inv_after#24)) AND (((inv_before#23 > 0) AND ((cast(inv_after#24 as double) / cast(inv_before#23 as double)) >= 0.666667)) AND ((cast(inv_after#24 as double) / cast(inv_before#23 as double)) <= 1.5)))
+
+(27) TakeOrderedAndProject
+Input [4]: [w_warehouse_name#7, i_item_id#10, inv_before#23, inv_after#24]
+Arguments: 100, [w_warehouse_name#7 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST], [w_warehouse_name#7, i_item_id#10, inv_before#23, inv_after#24]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5
+ReusedExchange (28)
+
+
+(28) ReusedExchange [Reuses operator id: 20]
+Output [2]: [d_date_sk#13, d_date#14]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/simplified.txt
new file mode 100644
index 0000000000000..7431f803c13b7
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/simplified.txt
@@ -0,0 +1,42 @@
+TakeOrderedAndProject [w_warehouse_name,i_item_id,inv_before,inv_after]
+  WholeStageCodegen (5)
+    Filter [inv_before,inv_after]
+      HashAggregate [w_warehouse_name,i_item_id,sum,sum] [sum(CASE WHEN (d_date < 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),inv_before,inv_after,sum,sum]
+        InputAdapter
+          Exchange [w_warehouse_name,i_item_id] #1
+            WholeStageCodegen (4)
+              HashAggregate [w_warehouse_name,i_item_id,d_date,inv_quantity_on_hand] [sum,sum,sum,sum]
+                Project [inv_quantity_on_hand,w_warehouse_name,i_item_id,d_date]
+                  BroadcastHashJoin [inv_date_sk,d_date_sk]
+                    Project [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id]
+                      BroadcastHashJoin [inv_item_sk,i_item_sk]
+                        Project [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name]
+                          BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk]
+                            Filter [inv_warehouse_sk,inv_item_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk]
+                                    SubqueryBroadcast [d_date_sk] #1
+                                      ReusedExchange [d_date_sk,d_date] #2
+                            InputAdapter
+                              BroadcastExchange #3
+                                WholeStageCodegen (1)
+                                  Filter [w_warehouse_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.warehouse [w_warehouse_sk,w_warehouse_name]
+                        InputAdapter
+                          BroadcastExchange #4
+                            WholeStageCodegen (2)
+                              Project [i_item_sk,i_item_id]
+                                Filter [i_current_price,i_item_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.item [i_item_sk,i_item_id,i_current_price]
+                    InputAdapter
+                      BroadcastExchange #2
+                        WholeStageCodegen (3)
+                          Filter [d_date,d_date_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.date_dim [d_date_sk,d_date]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.sf100/explain.txt
new file mode 100644
index 0000000000000..db9e7c70fbc21
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.sf100/explain.txt
@@ -0,0 +1,181 @@
+== Physical Plan ==
+TakeOrderedAndProject (30)
++- * HashAggregate (29)
+   +- Exchange (28)
+      +- * HashAggregate (27)
+         +- * Expand (26)
+            +- * Project (25)
+               +- * SortMergeJoin Inner (24)
+                  :- * Sort (18)
+                  :  +- Exchange (17)
+                  :     +- * Project (16)
+                  :        +- * BroadcastHashJoin Inner BuildRight (15)
+                  :           :- * Project (9)
+                  :           :  +- * BroadcastHashJoin Inner BuildRight (8)
+                  :           :     :- * Filter (3)
+                  :           :     :  +- * ColumnarToRow (2)
+                  :           :     :     +- Scan parquet default.inventory (1)
+                  :           :     +- BroadcastExchange (7)
+                  :           :        +- * Filter (6)
+                  :           :           +- * ColumnarToRow (5)
+                  :           :              +- Scan parquet default.warehouse (4)
+                  :           +- BroadcastExchange (14)
+                  :              +- * Project (13)
+                  :                 +- * Filter (12)
+                  :                    +- * ColumnarToRow (11)
+                  :                       +- Scan parquet default.date_dim (10)
+                  +- * Sort (23)
+                     +- Exchange (22)
+                        +- * Filter (21)
+                           +- * ColumnarToRow (20)
+                              +- Scan parquet default.item (19)
+
+
+(1) Scan parquet default.inventory
+Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)]
+ReadSchema: struct<inv_item_sk:int,inv_warehouse_sk:int,inv_quantity_on_hand:int>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+
+(3) Filter [codegen id : 3]
+Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2))
+
+(4) Scan parquet default.warehouse
+Output [1]: [w_warehouse_sk#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/warehouse]
+PushedFilters: [IsNotNull(w_warehouse_sk)]
+ReadSchema: struct<w_warehouse_sk:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [1]: [w_warehouse_sk#6]
+
+(6) Filter [codegen id : 1]
+Input [1]: [w_warehouse_sk#6]
+Condition : isnotnull(w_warehouse_sk#6)
+
+(7) BroadcastExchange
+Input [1]: [w_warehouse_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#7]
+
+(8) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [inv_warehouse_sk#2]
+Right keys [1]: [w_warehouse_sk#6]
+Join condition: None
+
+(9) Project [codegen id : 3]
+Output [3]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4]
+Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_sk#6]
+
+(10) Scan parquet default.date_dim
+Output [2]: [d_date_sk#8, d_month_seq#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#8, d_month_seq#9]
+
+(12) Filter [codegen id : 2]
+Input [2]: [d_date_sk#8, d_month_seq#9]
+Condition : (((isnotnull(d_month_seq#9) AND (d_month_seq#9 >= 1200)) AND (d_month_seq#9 <= 1211)) AND isnotnull(d_date_sk#8))
+
+(13) Project [codegen id : 2]
+Output [1]: [d_date_sk#8]
+Input [2]: [d_date_sk#8, d_month_seq#9]
+
+(14) BroadcastExchange
+Input [1]: [d_date_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [inv_date_sk#4]
+Right keys [1]: [d_date_sk#8]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [2]: [inv_item_sk#1, inv_quantity_on_hand#3]
+Input [4]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#8]
+
+(17) Exchange
+Input [2]: [inv_item_sk#1, inv_quantity_on_hand#3]
+Arguments: hashpartitioning(inv_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#11]
+
+(18) Sort [codegen id : 4]
+Input [2]: [inv_item_sk#1, inv_quantity_on_hand#3]
+Arguments: [inv_item_sk#1 ASC NULLS FIRST], false, 0
+
+(19) Scan parquet default.item
+Output [5]: [i_item_sk#12, i_brand#13, i_class#14, i_category#15, i_product_name#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_class:string,i_category:string,i_product_name:string>
+
+(20) ColumnarToRow [codegen id : 5]
+Input [5]: [i_item_sk#12, i_brand#13, i_class#14, i_category#15, i_product_name#16]
+
+(21) Filter [codegen id : 5]
+Input [5]: [i_item_sk#12, i_brand#13, i_class#14, i_category#15, i_product_name#16]
+Condition : isnotnull(i_item_sk#12)
+
+(22) Exchange
+Input [5]: [i_item_sk#12, i_brand#13, i_class#14, i_category#15, i_product_name#16]
+Arguments: hashpartitioning(i_item_sk#12, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(23) Sort [codegen id : 6]
+Input [5]: [i_item_sk#12, i_brand#13, i_class#14, i_category#15, i_product_name#16]
+Arguments: [i_item_sk#12 ASC NULLS FIRST], false, 0
+
+(24) SortMergeJoin [codegen id : 7]
+Left keys [1]: [inv_item_sk#1]
+Right keys [1]: [i_item_sk#12]
+Join condition: None
+
+(25) Project [codegen id : 7]
+Output [5]: [inv_quantity_on_hand#3, i_product_name#16, i_brand#13, i_class#14, i_category#15]
+Input [7]: [inv_item_sk#1, inv_quantity_on_hand#3, i_item_sk#12, i_brand#13, i_class#14, i_category#15, i_product_name#16]
+
+(26) Expand [codegen id : 7]
+Input [5]: [inv_quantity_on_hand#3, i_product_name#16, i_brand#13, i_class#14, i_category#15]
+Arguments: [List(inv_quantity_on_hand#3, i_product_name#16, i_brand#13, i_class#14, i_category#15, 0), List(inv_quantity_on_hand#3, i_product_name#16, i_brand#13, i_class#14, null, 1), List(inv_quantity_on_hand#3, i_product_name#16, i_brand#13, null, null, 3), List(inv_quantity_on_hand#3, i_product_name#16, null, null, null, 7), List(inv_quantity_on_hand#3, null, null, null, null, 15)], [inv_quantity_on_hand#3, i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22]
+
+(27) HashAggregate [codegen id : 7]
+Input [6]: [inv_quantity_on_hand#3, i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22]
+Keys [5]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22]
+Functions [1]: [partial_avg(inv_quantity_on_hand#3)]
+Aggregate Attributes [2]: [sum#23, count#24]
+Results [7]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22, sum#25, count#26]
+
+(28) Exchange
+Input [7]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22, sum#25, count#26]
+Arguments: hashpartitioning(i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22, 5), ENSURE_REQUIREMENTS, [id=#27]
+
+(29) HashAggregate [codegen id : 8]
+Input [7]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22, sum#25, count#26]
+Keys [5]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22]
+Functions [1]: [avg(inv_quantity_on_hand#3)]
+Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#28]
+Results [5]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, avg(inv_quantity_on_hand#3)#28 AS qoh#29]
+
+(30) TakeOrderedAndProject
+Input [5]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, qoh#29]
+Arguments: 100, [qoh#29 ASC NULLS FIRST, i_product_name#18 ASC NULLS FIRST, i_brand#19 ASC NULLS FIRST, i_class#20 ASC NULLS FIRST, i_category#21 ASC NULLS FIRST], [i_product_name#18, i_brand#19, i_class#20, i_category#21, qoh#29]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5
+ReusedExchange (31)
+
+
+(31) ReusedExchange [Reuses operator id: 14]
+Output [1]: [d_date_sk#8]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.sf100/simplified.txt
new file mode 100644
index 0000000000000..1bcae4201335b
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.sf100/simplified.txt
@@ -0,0 +1,51 @@
+TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category]
+  WholeStageCodegen (8)
+    HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count]
+      InputAdapter
+        Exchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1
+          WholeStageCodegen (7)
+            HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count]
+              Expand [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category]
+                Project [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category]
+                  SortMergeJoin [inv_item_sk,i_item_sk]
+                    InputAdapter
+                      WholeStageCodegen (4)
+                        Sort [inv_item_sk]
+                          InputAdapter
+                            Exchange [inv_item_sk] #2
+                              WholeStageCodegen (3)
+                                Project [inv_item_sk,inv_quantity_on_hand]
+                                  BroadcastHashJoin [inv_date_sk,d_date_sk]
+                                    Project [inv_item_sk,inv_quantity_on_hand,inv_date_sk]
+                                      BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk]
+                                        Filter [inv_item_sk,inv_warehouse_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk]
+                                                SubqueryBroadcast [d_date_sk] #1
+                                                  ReusedExchange [d_date_sk] #3
+                                        InputAdapter
+                                          BroadcastExchange #4
+                                            WholeStageCodegen (1)
+                                              Filter [w_warehouse_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.warehouse [w_warehouse_sk]
+                                    InputAdapter
+                                      BroadcastExchange #3
+                                        WholeStageCodegen (2)
+                                          Project [d_date_sk]
+                                            Filter [d_month_seq,d_date_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.date_dim [d_date_sk,d_month_seq]
+                    InputAdapter
+                      WholeStageCodegen (6)
+                        Sort [i_item_sk]
+                          InputAdapter
+                            Exchange [i_item_sk] #5
+                              WholeStageCodegen (5)
+                                Filter [i_item_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.item [i_item_sk,i_brand,i_class,i_category,i_product_name]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22/explain.txt
new file mode 100644
index 0000000000000..fa775d7c43bdc
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22/explain.txt
@@ -0,0 +1,166 @@
+== Physical Plan ==
+TakeOrderedAndProject (27)
++- * HashAggregate (26)
+   +- Exchange (25)
+      +- * HashAggregate (24)
+         +- * Expand (23)
+            +- * Project (22)
+               +- * BroadcastHashJoin Inner BuildRight (21)
+                  :- * Project (16)
+                  :  +- * BroadcastHashJoin Inner BuildRight (15)
+                  :     :- * Project (10)
+                  :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+                  :     :     :- * Filter (3)
+                  :     :     :  +- * ColumnarToRow (2)
+                  :     :     :     +- Scan parquet default.inventory (1)
+                  :     :     +- BroadcastExchange (8)
+                  :     :        +- * Project (7)
+                  :     :           +- * Filter (6)
+                  :     :              +- * ColumnarToRow (5)
+                  :     :                 +- Scan parquet default.date_dim (4)
+                  :     +- BroadcastExchange (14)
+                  :        +- * Filter (13)
+                  :           +- * ColumnarToRow (12)
+                  :              +- Scan parquet default.item (11)
+                  +- BroadcastExchange (20)
+                     +- * Filter (19)
+                        +- * ColumnarToRow (18)
+                           +- Scan parquet default.warehouse (17)
+
+
+(1) Scan parquet default.inventory
+Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)]
+ReadSchema: struct<inv_item_sk:int,inv_warehouse_sk:int,inv_quantity_on_hand:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+
+(3) Filter [codegen id : 4]
+Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2))
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#6, d_month_seq#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#6, d_month_seq#7]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#6, d_month_seq#7]
+Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#6))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#6]
+Input [2]: [d_date_sk#6, d_month_seq#7]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#8]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3]
+Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#6]
+
+(11) Scan parquet default.item
+Output [5]: [i_item_sk#9, i_brand#10, i_class#11, i_category#12, i_product_name#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_class:string,i_category:string,i_product_name:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [5]: [i_item_sk#9, i_brand#10, i_class#11, i_category#12, i_product_name#13]
+
+(13) Filter [codegen id : 2]
+Input [5]: [i_item_sk#9, i_brand#10, i_class#11, i_category#12, i_product_name#13]
+Condition : isnotnull(i_item_sk#9)
+
+(14) BroadcastExchange
+Input [5]: [i_item_sk#9, i_brand#10, i_class#11, i_category#12, i_product_name#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#14]
+
+(15) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_item_sk#1]
+Right keys [1]: [i_item_sk#9]
+Join condition: None
+
+(16) Project [codegen id : 4]
+Output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#10, i_class#11, i_category#12, i_product_name#13]
+Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#9, i_brand#10, i_class#11, i_category#12, i_product_name#13]
+
+(17) Scan parquet default.warehouse
+Output [1]: [w_warehouse_sk#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/warehouse]
+PushedFilters: [IsNotNull(w_warehouse_sk)]
+ReadSchema: struct<w_warehouse_sk:int>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [1]: [w_warehouse_sk#15]
+
+(19) Filter [codegen id : 3]
+Input [1]: [w_warehouse_sk#15]
+Condition : isnotnull(w_warehouse_sk#15)
+
+(20) BroadcastExchange
+Input [1]: [w_warehouse_sk#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#16]
+
+(21) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_warehouse_sk#2]
+Right keys [1]: [w_warehouse_sk#15]
+Join condition: None
+
+(22) Project [codegen id : 4]
+Output [5]: [inv_quantity_on_hand#3, i_product_name#13, i_brand#10, i_class#11, i_category#12]
+Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#10, i_class#11, i_category#12, i_product_name#13, w_warehouse_sk#15]
+
+(23) Expand [codegen id : 4]
+Input [5]: [inv_quantity_on_hand#3, i_product_name#13, i_brand#10, i_class#11, i_category#12]
+Arguments: [List(inv_quantity_on_hand#3, i_product_name#13, i_brand#10, i_class#11, i_category#12, 0), List(inv_quantity_on_hand#3, i_product_name#13, i_brand#10, i_class#11, null, 1), List(inv_quantity_on_hand#3, i_product_name#13, i_brand#10, null, null, 3), List(inv_quantity_on_hand#3, i_product_name#13, null, null, null, 7), List(inv_quantity_on_hand#3, null, null, null, null, 15)], [inv_quantity_on_hand#3, i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21]
+
+(24) HashAggregate [codegen id : 4]
+Input [6]: [inv_quantity_on_hand#3, i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21]
+Keys [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21]
+Functions [1]: [partial_avg(inv_quantity_on_hand#3)]
+Aggregate Attributes [2]: [sum#22, count#23]
+Results [7]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, sum#24, count#25]
+
+(25) Exchange
+Input [7]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, sum#24, count#25]
+Arguments: hashpartitioning(i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, 5), ENSURE_REQUIREMENTS, [id=#26]
+
+(26) HashAggregate [codegen id : 5]
+Input [7]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, sum#24, count#25]
+Keys [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21]
+Functions [1]: [avg(inv_quantity_on_hand#3)]
+Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#27]
+Results [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, avg(inv_quantity_on_hand#3)#27 AS qoh#28]
+
+(27) TakeOrderedAndProject
+Input [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#28]
+Arguments: 100, [qoh#28 ASC NULLS FIRST, i_product_name#17 ASC NULLS FIRST, i_brand#18 ASC NULLS FIRST, i_class#19 ASC NULLS FIRST, i_category#20 ASC NULLS FIRST], [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#28]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5
+ReusedExchange (28)
+
+
+(28) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#6]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22/simplified.txt
new file mode 100644
index 0000000000000..e0b0d3f84a169
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22/simplified.txt
@@ -0,0 +1,42 @@
+TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category]
+  WholeStageCodegen (5)
+    HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count]
+      InputAdapter
+        Exchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1
+          WholeStageCodegen (4)
+            HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count]
+              Expand [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category]
+                Project [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category]
+                  BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk]
+                    Project [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name]
+                      BroadcastHashJoin [inv_item_sk,i_item_sk]
+                        Project [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand]
+                          BroadcastHashJoin [inv_date_sk,d_date_sk]
+                            Filter [inv_item_sk,inv_warehouse_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk]
+                                    SubqueryBroadcast [d_date_sk] #1
+                                      ReusedExchange [d_date_sk] #2
+                            InputAdapter
+                              BroadcastExchange #2
+                                WholeStageCodegen (1)
+                                  Project [d_date_sk]
+                                    Filter [d_month_seq,d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.date_dim [d_date_sk,d_month_seq]
+                        InputAdapter
+                          BroadcastExchange #3
+                            WholeStageCodegen (2)
+                              Filter [i_item_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.item [i_item_sk,i_brand,i_class,i_category,i_product_name]
+                    InputAdapter
+                      BroadcastExchange #4
+                        WholeStageCodegen (3)
+                          Filter [w_warehouse_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.warehouse [w_warehouse_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.sf100/explain.txt
new file mode 100644
index 0000000000000..c61606d124ccb
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.sf100/explain.txt
@@ -0,0 +1,716 @@
+== Physical Plan ==
+* HashAggregate (98)
++- Exchange (97)
+   +- * HashAggregate (96)
+      +- Union (95)
+         :- * Project (59)
+         :  +- * BroadcastHashJoin Inner BuildRight (58)
+         :     :- * Project (52)
+         :     :  +- * SortMergeJoin LeftSemi (51)
+         :     :     :- * Sort (32)
+         :     :     :  +- Exchange (31)
+         :     :     :     +- * Project (30)
+         :     :     :        +- * SortMergeJoin LeftSemi (29)
+         :     :     :           :- * Sort (4)
+         :     :     :           :  +- Exchange (3)
+         :     :     :           :     +- * ColumnarToRow (2)
+         :     :     :           :        +- Scan parquet default.catalog_sales (1)
+         :     :     :           +- * Sort (28)
+         :     :     :              +- * Project (27)
+         :     :     :                 +- * Filter (26)
+         :     :     :                    +- * HashAggregate (25)
+         :     :     :                       +- * HashAggregate (24)
+         :     :     :                          +- * Project (23)
+         :     :     :                             +- * SortMergeJoin Inner (22)
+         :     :     :                                :- * Sort (16)
+         :     :     :                                :  +- Exchange (15)
+         :     :     :                                :     +- * Project (14)
+         :     :     :                                :        +- * BroadcastHashJoin Inner BuildRight (13)
+         :     :     :                                :           :- * Filter (7)
+         :     :     :                                :           :  +- * ColumnarToRow (6)
+         :     :     :                                :           :     +- Scan parquet default.store_sales (5)
+         :     :     :                                :           +- BroadcastExchange (12)
+         :     :     :                                :              +- * Project (11)
+         :     :     :                                :                 +- * Filter (10)
+         :     :     :                                :                    +- * ColumnarToRow (9)
+         :     :     :                                :                       +- Scan parquet default.date_dim (8)
+         :     :     :                                +- * Sort (21)
+         :     :     :                                   +- Exchange (20)
+         :     :     :                                      +- * Filter (19)
+         :     :     :                                         +- * ColumnarToRow (18)
+         :     :     :                                            +- Scan parquet default.item (17)
+         :     :     +- * Sort (50)
+         :     :        +- * Project (49)
+         :     :           +- * Filter (48)
+         :     :              +- * HashAggregate (47)
+         :     :                 +- * HashAggregate (46)
+         :     :                    +- * Project (45)
+         :     :                       +- * SortMergeJoin Inner (44)
+         :     :                          :- * Sort (38)
+         :     :                          :  +- Exchange (37)
+         :     :                          :     +- * Project (36)
+         :     :                          :        +- * Filter (35)
+         :     :                          :           +- * ColumnarToRow (34)
+         :     :                          :              +- Scan parquet default.store_sales (33)
+         :     :                          +- * Sort (43)
+         :     :                             +- Exchange (42)
+         :     :                                +- * Filter (41)
+         :     :                                   +- * ColumnarToRow (40)
+         :     :                                      +- Scan parquet default.customer (39)
+         :     +- BroadcastExchange (57)
+         :        +- * Project (56)
+         :           +- * Filter (55)
+         :              +- * ColumnarToRow (54)
+         :                 +- Scan parquet default.date_dim (53)
+         +- * Project (94)
+            +- * BroadcastHashJoin Inner BuildRight (93)
+               :- * Project (91)
+               :  +- * SortMergeJoin LeftSemi (90)
+               :     :- * Sort (78)
+               :     :  +- Exchange (77)
+               :     :     +- * Project (76)
+               :     :        +- * SortMergeJoin LeftSemi (75)
+               :     :           :- * Sort (63)
+               :     :           :  +- Exchange (62)
+               :     :           :     +- * ColumnarToRow (61)
+               :     :           :        +- Scan parquet default.web_sales (60)
+               :     :           +- * Sort (74)
+               :     :              +- * Project (73)
+               :     :                 +- * Filter (72)
+               :     :                    +- * HashAggregate (71)
+               :     :                       +- * HashAggregate (70)
+               :     :                          +- * Project (69)
+               :     :                             +- * SortMergeJoin Inner (68)
+               :     :                                :- * Sort (65)
+               :     :                                :  +- ReusedExchange (64)
+               :     :                                +- * Sort (67)
+               :     :                                   +- ReusedExchange (66)
+               :     +- * Sort (89)
+               :        +- * Project (88)
+               :           +- * Filter (87)
+               :              +- * HashAggregate (86)
+               :                 +- * HashAggregate (85)
+               :                    +- * Project (84)
+               :                       +- * SortMergeJoin Inner (83)
+               :                          :- * Sort (80)
+               :                          :  +- ReusedExchange (79)
+               :                          +- * Sort (82)
+               :                             +- ReusedExchange (81)
+               +- ReusedExchange (92)
+
+
+(1) Scan parquet default.catalog_sales
+Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sold_date_sk#5 IN dynamicpruning#6)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_item_sk:int,cs_quantity:int,cs_list_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+
+(3) Exchange
+Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+Arguments: hashpartitioning(cs_item_sk#2, 5), ENSURE_REQUIREMENTS, [id=#7]
+
+(4) Sort [codegen id : 2]
+Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+Arguments: [cs_item_sk#2 ASC NULLS FIRST], false, 0
+
+(5) Scan parquet default.store_sales
+Output [2]: [ss_item_sk#8, ss_sold_date_sk#9]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#9), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int>
+
+(6) ColumnarToRow [codegen id : 4]
+Input [2]: [ss_item_sk#8, ss_sold_date_sk#9]
+
+(7) Filter [codegen id : 4]
+Input [2]: [ss_item_sk#8, ss_sold_date_sk#9]
+Condition : isnotnull(ss_item_sk#8)
+
+(8) Scan parquet default.date_dim
+Output [3]: [d_date_sk#11, d_date#12, d_year#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date,d_year:int>
+
+(9) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#11, d_date#12, d_year#13]
+
+(10) Filter [codegen id : 3]
+Input [3]: [d_date_sk#11, d_date#12, d_year#13]
+Condition : (d_year#13 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#11))
+
+(11) Project [codegen id : 3]
+Output [2]: [d_date_sk#11, d_date#12]
+Input [3]: [d_date_sk#11, d_date#12, d_year#13]
+
+(12) BroadcastExchange
+Input [2]: [d_date_sk#11, d_date#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(13) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#9]
+Right keys [1]: [d_date_sk#11]
+Join condition: None
+
+(14) Project [codegen id : 4]
+Output [2]: [ss_item_sk#8, d_date#12]
+Input [4]: [ss_item_sk#8, ss_sold_date_sk#9, d_date_sk#11, d_date#12]
+
+(15) Exchange
+Input [2]: [ss_item_sk#8, d_date#12]
+Arguments: hashpartitioning(ss_item_sk#8, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(16) Sort [codegen id : 5]
+Input [2]: [ss_item_sk#8, d_date#12]
+Arguments: [ss_item_sk#8 ASC NULLS FIRST], false, 0
+
+(17) Scan parquet default.item
+Output [2]: [i_item_sk#16, i_item_desc#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_desc:string>
+
+(18) ColumnarToRow [codegen id : 6]
+Input [2]: [i_item_sk#16, i_item_desc#17]
+
+(19) Filter [codegen id : 6]
+Input [2]: [i_item_sk#16, i_item_desc#17]
+Condition : isnotnull(i_item_sk#16)
+
+(20) Exchange
+Input [2]: [i_item_sk#16, i_item_desc#17]
+Arguments: hashpartitioning(i_item_sk#16, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(21) Sort [codegen id : 7]
+Input [2]: [i_item_sk#16, i_item_desc#17]
+Arguments: [i_item_sk#16 ASC NULLS FIRST], false, 0
+
+(22) SortMergeJoin [codegen id : 8]
+Left keys [1]: [ss_item_sk#8]
+Right keys [1]: [i_item_sk#16]
+Join condition: None
+
+(23) Project [codegen id : 8]
+Output [3]: [d_date#12, i_item_sk#16, substr(i_item_desc#17, 1, 30) AS _groupingexpression#19]
+Input [4]: [ss_item_sk#8, d_date#12, i_item_sk#16, i_item_desc#17]
+
+(24) HashAggregate [codegen id : 8]
+Input [3]: [d_date#12, i_item_sk#16, _groupingexpression#19]
+Keys [3]: [_groupingexpression#19, i_item_sk#16, d_date#12]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#20]
+Results [4]: [_groupingexpression#19, i_item_sk#16, d_date#12, count#21]
+
+(25) HashAggregate [codegen id : 8]
+Input [4]: [_groupingexpression#19, i_item_sk#16, d_date#12, count#21]
+Keys [3]: [_groupingexpression#19, i_item_sk#16, d_date#12]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#22]
+Results [2]: [i_item_sk#16 AS item_sk#23, count(1)#22 AS count(1)#24]
+
+(26) Filter [codegen id : 8]
+Input [2]: [item_sk#23, count(1)#24]
+Condition : (count(1)#24 > 4)
+
+(27) Project [codegen id : 8]
+Output [1]: [item_sk#23]
+Input [2]: [item_sk#23, count(1)#24]
+
+(28) Sort [codegen id : 8]
+Input [1]: [item_sk#23]
+Arguments: [item_sk#23 ASC NULLS FIRST], false, 0
+
+(29) SortMergeJoin [codegen id : 9]
+Left keys [1]: [cs_item_sk#2]
+Right keys [1]: [item_sk#23]
+Join condition: None
+
+(30) Project [codegen id : 9]
+Output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+
+(31) Exchange
+Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(32) Sort [codegen id : 10]
+Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+Arguments: [cs_bill_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(33) Scan parquet default.store_sales
+Output [4]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28, ss_sold_date_sk#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_quantity:int,ss_sales_price:decimal(7,2)>
+
+(34) ColumnarToRow [codegen id : 11]
+Input [4]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28, ss_sold_date_sk#29]
+
+(35) Filter [codegen id : 11]
+Input [4]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28, ss_sold_date_sk#29]
+Condition : isnotnull(ss_customer_sk#26)
+
+(36) Project [codegen id : 11]
+Output [3]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28]
+Input [4]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28, ss_sold_date_sk#29]
+
+(37) Exchange
+Input [3]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28]
+Arguments: hashpartitioning(ss_customer_sk#26, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(38) Sort [codegen id : 12]
+Input [3]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28]
+Arguments: [ss_customer_sk#26 ASC NULLS FIRST], false, 0
+
+(39) Scan parquet default.customer
+Output [1]: [c_customer_sk#31]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int>
+
+(40) ColumnarToRow [codegen id : 13]
+Input [1]: [c_customer_sk#31]
+
+(41) Filter [codegen id : 13]
+Input [1]: [c_customer_sk#31]
+Condition : isnotnull(c_customer_sk#31)
+
+(42) Exchange
+Input [1]: [c_customer_sk#31]
+Arguments: hashpartitioning(c_customer_sk#31, 5), ENSURE_REQUIREMENTS, [id=#32]
+
+(43) Sort [codegen id : 14]
+Input [1]: [c_customer_sk#31]
+Arguments: [c_customer_sk#31 ASC NULLS FIRST], false, 0
+
+(44) SortMergeJoin [codegen id : 15]
+Left keys [1]: [ss_customer_sk#26]
+Right keys [1]: [c_customer_sk#31]
+Join condition: None
+
+(45) Project [codegen id : 15]
+Output [3]: [ss_quantity#27, ss_sales_price#28, c_customer_sk#31]
+Input [4]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28, c_customer_sk#31]
+
+(46) HashAggregate [codegen id : 15]
+Input [3]: [ss_quantity#27, ss_sales_price#28, c_customer_sk#31]
+Keys [1]: [c_customer_sk#31]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#27 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#28 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [2]: [sum#33, isEmpty#34]
+Results [3]: [c_customer_sk#31, sum#35, isEmpty#36]
+
+(47) HashAggregate [codegen id : 15]
+Input [3]: [c_customer_sk#31, sum#35, isEmpty#36]
+Keys [1]: [c_customer_sk#31]
+Functions [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#27 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#28 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#27 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#28 as decimal(12,2)))), DecimalType(18,2), true))#37]
+Results [2]: [c_customer_sk#31, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#27 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#28 as decimal(12,2)))), DecimalType(18,2), true))#37 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40]
+
+(48) Filter [codegen id : 15]
+Input [2]: [c_customer_sk#31, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40 as decimal(38,8)) > CheckOverflow((0.500000 * promote_precision(cast(Subquery scalar-subquery#41, [id=#42] as decimal(32,6)))), DecimalType(38,8), true)))
+
+(49) Project [codegen id : 15]
+Output [1]: [c_customer_sk#31]
+Input [2]: [c_customer_sk#31, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40]
+
+(50) Sort [codegen id : 15]
+Input [1]: [c_customer_sk#31]
+Arguments: [c_customer_sk#31 ASC NULLS FIRST], false, 0
+
+(51) SortMergeJoin [codegen id : 17]
+Left keys [1]: [cs_bill_customer_sk#1]
+Right keys [1]: [c_customer_sk#31]
+Join condition: None
+
+(52) Project [codegen id : 17]
+Output [3]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+
+(53) Scan parquet default.date_dim
+Output [3]: [d_date_sk#43, d_year#44, d_moy#45]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(54) ColumnarToRow [codegen id : 16]
+Input [3]: [d_date_sk#43, d_year#44, d_moy#45]
+
+(55) Filter [codegen id : 16]
+Input [3]: [d_date_sk#43, d_year#44, d_moy#45]
+Condition : ((((isnotnull(d_year#44) AND isnotnull(d_moy#45)) AND (d_year#44 = 2000)) AND (d_moy#45 = 2)) AND isnotnull(d_date_sk#43))
+
+(56) Project [codegen id : 16]
+Output [1]: [d_date_sk#43]
+Input [3]: [d_date_sk#43, d_year#44, d_moy#45]
+
+(57) BroadcastExchange
+Input [1]: [d_date_sk#43]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#46]
+
+(58) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [cs_sold_date_sk#5]
+Right keys [1]: [d_date_sk#43]
+Join condition: None
+
+(59) Project [codegen id : 17]
+Output [1]: [CheckOverflow((promote_precision(cast(cast(cs_quantity#3 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#4 as decimal(12,2)))), DecimalType(18,2), true) AS sales#47]
+Input [4]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, d_date_sk#43]
+
+(60) Scan parquet default.web_sales
+Output [5]: [ws_item_sk#48, ws_bill_customer_sk#49, ws_quantity#50, ws_list_price#51, ws_sold_date_sk#52]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#52), dynamicpruningexpression(ws_sold_date_sk#52 IN dynamicpruning#6)]
+ReadSchema: struct<ws_item_sk:int,ws_bill_customer_sk:int,ws_quantity:int,ws_list_price:decimal(7,2)>
+
+(61) ColumnarToRow [codegen id : 18]
+Input [5]: [ws_item_sk#48, ws_bill_customer_sk#49, ws_quantity#50, ws_list_price#51, ws_sold_date_sk#52]
+
+(62) Exchange
+Input [5]: [ws_item_sk#48, ws_bill_customer_sk#49, ws_quantity#50, ws_list_price#51, ws_sold_date_sk#52]
+Arguments: hashpartitioning(ws_item_sk#48, 5), ENSURE_REQUIREMENTS, [id=#53]
+
+(63) Sort [codegen id : 19]
+Input [5]: [ws_item_sk#48, ws_bill_customer_sk#49, ws_quantity#50, ws_list_price#51, ws_sold_date_sk#52]
+Arguments: [ws_item_sk#48 ASC NULLS FIRST], false, 0
+
+(64) ReusedExchange [Reuses operator id: unknown]
+Output [2]: [ss_item_sk#54, d_date#55]
+
+(65) Sort [codegen id : 22]
+Input [2]: [ss_item_sk#54, d_date#55]
+Arguments: [ss_item_sk#54 ASC NULLS FIRST], false, 0
+
+(66) ReusedExchange [Reuses operator id: 20]
+Output [2]: [i_item_sk#56, i_item_desc#57]
+
+(67) Sort [codegen id : 24]
+Input [2]: [i_item_sk#56, i_item_desc#57]
+Arguments: [i_item_sk#56 ASC NULLS FIRST], false, 0
+
+(68) SortMergeJoin [codegen id : 25]
+Left keys [1]: [ss_item_sk#54]
+Right keys [1]: [i_item_sk#56]
+Join condition: None
+
+(69) Project [codegen id : 25]
+Output [3]: [d_date#55, i_item_sk#56, substr(i_item_desc#57, 1, 30) AS _groupingexpression#58]
+Input [4]: [ss_item_sk#54, d_date#55, i_item_sk#56, i_item_desc#57]
+
+(70) HashAggregate [codegen id : 25]
+Input [3]: [d_date#55, i_item_sk#56, _groupingexpression#58]
+Keys [3]: [_groupingexpression#58, i_item_sk#56, d_date#55]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#59]
+Results [4]: [_groupingexpression#58, i_item_sk#56, d_date#55, count#60]
+
+(71) HashAggregate [codegen id : 25]
+Input [4]: [_groupingexpression#58, i_item_sk#56, d_date#55, count#60]
+Keys [3]: [_groupingexpression#58, i_item_sk#56, d_date#55]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#61]
+Results [2]: [i_item_sk#56 AS item_sk#23, count(1)#61 AS count(1)#62]
+
+(72) Filter [codegen id : 25]
+Input [2]: [item_sk#23, count(1)#62]
+Condition : (count(1)#62 > 4)
+
+(73) Project [codegen id : 25]
+Output [1]: [item_sk#23]
+Input [2]: [item_sk#23, count(1)#62]
+
+(74) Sort [codegen id : 25]
+Input [1]: [item_sk#23]
+Arguments: [item_sk#23 ASC NULLS FIRST], false, 0
+
+(75) SortMergeJoin [codegen id : 26]
+Left keys [1]: [ws_item_sk#48]
+Right keys [1]: [item_sk#23]
+Join condition: None
+
+(76) Project [codegen id : 26]
+Output [4]: [ws_bill_customer_sk#49, ws_quantity#50, ws_list_price#51, ws_sold_date_sk#52]
+Input [5]: [ws_item_sk#48, ws_bill_customer_sk#49, ws_quantity#50, ws_list_price#51, ws_sold_date_sk#52]
+
+(77) Exchange
+Input [4]: [ws_bill_customer_sk#49, ws_quantity#50, ws_list_price#51, ws_sold_date_sk#52]
+Arguments: hashpartitioning(ws_bill_customer_sk#49, 5), ENSURE_REQUIREMENTS, [id=#63]
+
+(78) Sort [codegen id : 27]
+Input [4]: [ws_bill_customer_sk#49, ws_quantity#50, ws_list_price#51, ws_sold_date_sk#52]
+Arguments: [ws_bill_customer_sk#49 ASC NULLS FIRST], false, 0
+
+(79) ReusedExchange [Reuses operator id: 37]
+Output [3]: [ss_customer_sk#64, ss_quantity#65, ss_sales_price#66]
+
+(80) Sort [codegen id : 29]
+Input [3]: [ss_customer_sk#64, ss_quantity#65, ss_sales_price#66]
+Arguments: [ss_customer_sk#64 ASC NULLS FIRST], false, 0
+
+(81) ReusedExchange [Reuses operator id: 42]
+Output [1]: [c_customer_sk#67]
+
+(82) Sort [codegen id : 31]
+Input [1]: [c_customer_sk#67]
+Arguments: [c_customer_sk#67 ASC NULLS FIRST], false, 0
+
+(83) SortMergeJoin [codegen id : 32]
+Left keys [1]: [ss_customer_sk#64]
+Right keys [1]: [c_customer_sk#67]
+Join condition: None
+
+(84) Project [codegen id : 32]
+Output [3]: [ss_quantity#65, ss_sales_price#66, c_customer_sk#67]
+Input [4]: [ss_customer_sk#64, ss_quantity#65, ss_sales_price#66, c_customer_sk#67]
+
+(85) HashAggregate [codegen id : 32]
+Input [3]: [ss_quantity#65, ss_sales_price#66, c_customer_sk#67]
+Keys [1]: [c_customer_sk#67]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#65 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#66 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [2]: [sum#68, isEmpty#69]
+Results [3]: [c_customer_sk#67, sum#70, isEmpty#71]
+
+(86) HashAggregate [codegen id : 32]
+Input [3]: [c_customer_sk#67, sum#70, isEmpty#71]
+Keys [1]: [c_customer_sk#67]
+Functions [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#65 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#66 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#65 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#66 as decimal(12,2)))), DecimalType(18,2), true))#72]
+Results [2]: [c_customer_sk#67, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#65 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#66 as decimal(12,2)))), DecimalType(18,2), true))#72 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#73]
+
+(87) Filter [codegen id : 32]
+Input [2]: [c_customer_sk#67, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#73]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#73) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#73 as decimal(38,8)) > CheckOverflow((0.500000 * promote_precision(cast(ReusedSubquery Subquery scalar-subquery#41, [id=#42] as decimal(32,6)))), DecimalType(38,8), true)))
+
+(88) Project [codegen id : 32]
+Output [1]: [c_customer_sk#67]
+Input [2]: [c_customer_sk#67, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#73]
+
+(89) Sort [codegen id : 32]
+Input [1]: [c_customer_sk#67]
+Arguments: [c_customer_sk#67 ASC NULLS FIRST], false, 0
+
+(90) SortMergeJoin [codegen id : 34]
+Left keys [1]: [ws_bill_customer_sk#49]
+Right keys [1]: [c_customer_sk#67]
+Join condition: None
+
+(91) Project [codegen id : 34]
+Output [3]: [ws_quantity#50, ws_list_price#51, ws_sold_date_sk#52]
+Input [4]: [ws_bill_customer_sk#49, ws_quantity#50, ws_list_price#51, ws_sold_date_sk#52]
+
+(92) ReusedExchange [Reuses operator id: 57]
+Output [1]: [d_date_sk#74]
+
+(93) BroadcastHashJoin [codegen id : 34]
+Left keys [1]: [ws_sold_date_sk#52]
+Right keys [1]: [d_date_sk#74]
+Join condition: None
+
+(94) Project [codegen id : 34]
+Output [1]: [CheckOverflow((promote_precision(cast(cast(ws_quantity#50 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#51 as decimal(12,2)))), DecimalType(18,2), true) AS sales#75]
+Input [4]: [ws_quantity#50, ws_list_price#51, ws_sold_date_sk#52, d_date_sk#74]
+
+(95) Union
+
+(96) HashAggregate [codegen id : 35]
+Input [1]: [sales#47]
+Keys: []
+Functions [1]: [partial_sum(sales#47)]
+Aggregate Attributes [2]: [sum#76, isEmpty#77]
+Results [2]: [sum#78, isEmpty#79]
+
+(97) Exchange
+Input [2]: [sum#78, isEmpty#79]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#80]
+
+(98) HashAggregate [codegen id : 36]
+Input [2]: [sum#78, isEmpty#79]
+Keys: []
+Functions [1]: [sum(sales#47)]
+Aggregate Attributes [1]: [sum(sales#47)#81]
+Results [1]: [sum(sales#47)#81 AS sum(sales)#82]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (99)
+
+
+(99) ReusedExchange [Reuses operator id: 57]
+Output [1]: [d_date_sk#43]
+
+Subquery:2 Hosting operator id = 5 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10
+ReusedExchange (100)
+
+
+(100) ReusedExchange [Reuses operator id: 12]
+Output [2]: [d_date_sk#11, d_date#12]
+
+Subquery:3 Hosting operator id = 48 Hosting Expression = Subquery scalar-subquery#41, [id=#42]
+* HashAggregate (124)
++- Exchange (123)
+   +- * HashAggregate (122)
+      +- * HashAggregate (121)
+         +- * HashAggregate (120)
+            +- * Project (119)
+               +- * SortMergeJoin Inner (118)
+                  :- * Sort (112)
+                  :  +- Exchange (111)
+                  :     +- * Project (110)
+                  :        +- * BroadcastHashJoin Inner BuildRight (109)
+                  :           :- * Filter (103)
+                  :           :  +- * ColumnarToRow (102)
+                  :           :     +- Scan parquet default.store_sales (101)
+                  :           +- BroadcastExchange (108)
+                  :              +- * Project (107)
+                  :                 +- * Filter (106)
+                  :                    +- * ColumnarToRow (105)
+                  :                       +- Scan parquet default.date_dim (104)
+                  +- * Sort (117)
+                     +- Exchange (116)
+                        +- * Filter (115)
+                           +- * ColumnarToRow (114)
+                              +- Scan parquet default.customer (113)
+
+
+(101) Scan parquet default.store_sales
+Output [4]: [ss_customer_sk#83, ss_quantity#84, ss_sales_price#85, ss_sold_date_sk#86]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#86), dynamicpruningexpression(ss_sold_date_sk#86 IN dynamicpruning#87)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_quantity:int,ss_sales_price:decimal(7,2)>
+
+(102) ColumnarToRow [codegen id : 2]
+Input [4]: [ss_customer_sk#83, ss_quantity#84, ss_sales_price#85, ss_sold_date_sk#86]
+
+(103) Filter [codegen id : 2]
+Input [4]: [ss_customer_sk#83, ss_quantity#84, ss_sales_price#85, ss_sold_date_sk#86]
+Condition : isnotnull(ss_customer_sk#83)
+
+(104) Scan parquet default.date_dim
+Output [2]: [d_date_sk#88, d_year#89]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(105) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#88, d_year#89]
+
+(106) Filter [codegen id : 1]
+Input [2]: [d_date_sk#88, d_year#89]
+Condition : (d_year#89 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#88))
+
+(107) Project [codegen id : 1]
+Output [1]: [d_date_sk#88]
+Input [2]: [d_date_sk#88, d_year#89]
+
+(108) BroadcastExchange
+Input [1]: [d_date_sk#88]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#90]
+
+(109) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#86]
+Right keys [1]: [d_date_sk#88]
+Join condition: None
+
+(110) Project [codegen id : 2]
+Output [3]: [ss_customer_sk#83, ss_quantity#84, ss_sales_price#85]
+Input [5]: [ss_customer_sk#83, ss_quantity#84, ss_sales_price#85, ss_sold_date_sk#86, d_date_sk#88]
+
+(111) Exchange
+Input [3]: [ss_customer_sk#83, ss_quantity#84, ss_sales_price#85]
+Arguments: hashpartitioning(ss_customer_sk#83, 5), ENSURE_REQUIREMENTS, [id=#91]
+
+(112) Sort [codegen id : 3]
+Input [3]: [ss_customer_sk#83, ss_quantity#84, ss_sales_price#85]
+Arguments: [ss_customer_sk#83 ASC NULLS FIRST], false, 0
+
+(113) Scan parquet default.customer
+Output [1]: [c_customer_sk#92]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int>
+
+(114) ColumnarToRow [codegen id : 4]
+Input [1]: [c_customer_sk#92]
+
+(115) Filter [codegen id : 4]
+Input [1]: [c_customer_sk#92]
+Condition : isnotnull(c_customer_sk#92)
+
+(116) Exchange
+Input [1]: [c_customer_sk#92]
+Arguments: hashpartitioning(c_customer_sk#92, 5), ENSURE_REQUIREMENTS, [id=#93]
+
+(117) Sort [codegen id : 5]
+Input [1]: [c_customer_sk#92]
+Arguments: [c_customer_sk#92 ASC NULLS FIRST], false, 0
+
+(118) SortMergeJoin [codegen id : 6]
+Left keys [1]: [ss_customer_sk#83]
+Right keys [1]: [c_customer_sk#92]
+Join condition: None
+
+(119) Project [codegen id : 6]
+Output [3]: [ss_quantity#84, ss_sales_price#85, c_customer_sk#92]
+Input [4]: [ss_customer_sk#83, ss_quantity#84, ss_sales_price#85, c_customer_sk#92]
+
+(120) HashAggregate [codegen id : 6]
+Input [3]: [ss_quantity#84, ss_sales_price#85, c_customer_sk#92]
+Keys [1]: [c_customer_sk#92]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#84 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#85 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [2]: [sum#94, isEmpty#95]
+Results [3]: [c_customer_sk#92, sum#96, isEmpty#97]
+
+(121) HashAggregate [codegen id : 6]
+Input [3]: [c_customer_sk#92, sum#96, isEmpty#97]
+Keys [1]: [c_customer_sk#92]
+Functions [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#84 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#85 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#84 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#85 as decimal(12,2)))), DecimalType(18,2), true))#98]
+Results [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#84 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#85 as decimal(12,2)))), DecimalType(18,2), true))#98 AS csales#99]
+
+(122) HashAggregate [codegen id : 6]
+Input [1]: [csales#99]
+Keys: []
+Functions [1]: [partial_max(csales#99)]
+Aggregate Attributes [1]: [max#100]
+Results [1]: [max#101]
+
+(123) Exchange
+Input [1]: [max#101]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#102]
+
+(124) HashAggregate [codegen id : 7]
+Input [1]: [max#101]
+Keys: []
+Functions [1]: [max(csales#99)]
+Aggregate Attributes [1]: [max(csales#99)#103]
+Results [1]: [max(csales#99)#103 AS tpcds_cmax#104]
+
+Subquery:4 Hosting operator id = 101 Hosting Expression = ss_sold_date_sk#86 IN dynamicpruning#87
+ReusedExchange (125)
+
+
+(125) ReusedExchange [Reuses operator id: 108]
+Output [1]: [d_date_sk#88]
+
+Subquery:5 Hosting operator id = 60 Hosting Expression = ws_sold_date_sk#52 IN dynamicpruning#6
+
+Subquery:6 Hosting operator id = 87 Hosting Expression = ReusedSubquery Subquery scalar-subquery#41, [id=#42]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.sf100/simplified.txt
new file mode 100644
index 0000000000000..0e8b0977ae088
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.sf100/simplified.txt
@@ -0,0 +1,218 @@
+WholeStageCodegen (36)
+  HashAggregate [sum,isEmpty] [sum(sales),sum(sales),sum,isEmpty]
+    InputAdapter
+      Exchange #1
+        WholeStageCodegen (35)
+          HashAggregate [sales] [sum,isEmpty,sum,isEmpty]
+            InputAdapter
+              Union
+                WholeStageCodegen (17)
+                  Project [cs_quantity,cs_list_price]
+                    BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                      Project [cs_quantity,cs_list_price,cs_sold_date_sk]
+                        SortMergeJoin [cs_bill_customer_sk,c_customer_sk]
+                          InputAdapter
+                            WholeStageCodegen (10)
+                              Sort [cs_bill_customer_sk]
+                                InputAdapter
+                                  Exchange [cs_bill_customer_sk] #2
+                                    WholeStageCodegen (9)
+                                      Project [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk]
+                                        SortMergeJoin [cs_item_sk,item_sk]
+                                          InputAdapter
+                                            WholeStageCodegen (2)
+                                              Sort [cs_item_sk]
+                                                InputAdapter
+                                                  Exchange [cs_item_sk] #3
+                                                    WholeStageCodegen (1)
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk]
+                                                            SubqueryBroadcast [d_date_sk] #1
+                                                              ReusedExchange [d_date_sk] #4
+                                          InputAdapter
+                                            WholeStageCodegen (8)
+                                              Sort [item_sk]
+                                                Project [item_sk]
+                                                  Filter [count(1)]
+                                                    HashAggregate [_groupingexpression,i_item_sk,d_date,count] [count(1),item_sk,count(1),count]
+                                                      HashAggregate [_groupingexpression,i_item_sk,d_date] [count,count]
+                                                        Project [d_date,i_item_sk,i_item_desc]
+                                                          SortMergeJoin [ss_item_sk,i_item_sk]
+                                                            InputAdapter
+                                                              WholeStageCodegen (5)
+                                                                Sort [ss_item_sk]
+                                                                  InputAdapter
+                                                                    Exchange [ss_item_sk] #5
+                                                                      WholeStageCodegen (4)
+                                                                        Project [ss_item_sk,d_date]
+                                                                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                            Filter [ss_item_sk]
+                                                                              ColumnarToRow
+                                                                                InputAdapter
+                                                                                  Scan parquet default.store_sales [ss_item_sk,ss_sold_date_sk]
+                                                                                    SubqueryBroadcast [d_date_sk] #2
+                                                                                      ReusedExchange [d_date_sk,d_date] #6
+                                                                            InputAdapter
+                                                                              BroadcastExchange #6
+                                                                                WholeStageCodegen (3)
+                                                                                  Project [d_date_sk,d_date]
+                                                                                    Filter [d_year,d_date_sk]
+                                                                                      ColumnarToRow
+                                                                                        InputAdapter
+                                                                                          Scan parquet default.date_dim [d_date_sk,d_date,d_year]
+                                                            InputAdapter
+                                                              WholeStageCodegen (7)
+                                                                Sort [i_item_sk]
+                                                                  InputAdapter
+                                                                    Exchange [i_item_sk] #7
+                                                                      WholeStageCodegen (6)
+                                                                        Filter [i_item_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.item [i_item_sk,i_item_desc]
+                          InputAdapter
+                            WholeStageCodegen (15)
+                              Sort [c_customer_sk]
+                                Project [c_customer_sk]
+                                  Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                    Subquery #3
+                                      WholeStageCodegen (7)
+                                        HashAggregate [max] [max(csales),tpcds_cmax,max]
+                                          InputAdapter
+                                            Exchange #10
+                                              WholeStageCodegen (6)
+                                                HashAggregate [csales] [max,max]
+                                                  HashAggregate [c_customer_sk,sum,isEmpty] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),csales,sum,isEmpty]
+                                                    HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty]
+                                                      Project [ss_quantity,ss_sales_price,c_customer_sk]
+                                                        SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                                          InputAdapter
+                                                            WholeStageCodegen (3)
+                                                              Sort [ss_customer_sk]
+                                                                InputAdapter
+                                                                  Exchange [ss_customer_sk] #11
+                                                                    WholeStageCodegen (2)
+                                                                      Project [ss_customer_sk,ss_quantity,ss_sales_price]
+                                                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                          Filter [ss_customer_sk]
+                                                                            ColumnarToRow
+                                                                              InputAdapter
+                                                                                Scan parquet default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk]
+                                                                                  SubqueryBroadcast [d_date_sk] #4
+                                                                                    ReusedExchange [d_date_sk] #12
+                                                                          InputAdapter
+                                                                            BroadcastExchange #12
+                                                                              WholeStageCodegen (1)
+                                                                                Project [d_date_sk]
+                                                                                  Filter [d_year,d_date_sk]
+                                                                                    ColumnarToRow
+                                                                                      InputAdapter
+                                                                                        Scan parquet default.date_dim [d_date_sk,d_year]
+                                                          InputAdapter
+                                                            WholeStageCodegen (5)
+                                                              Sort [c_customer_sk]
+                                                                InputAdapter
+                                                                  Exchange [c_customer_sk] #13
+                                                                    WholeStageCodegen (4)
+                                                                      Filter [c_customer_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.customer [c_customer_sk]
+                                    HashAggregate [c_customer_sk,sum,isEmpty] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty]
+                                      HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty]
+                                        Project [ss_quantity,ss_sales_price,c_customer_sk]
+                                          SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                            InputAdapter
+                                              WholeStageCodegen (12)
+                                                Sort [ss_customer_sk]
+                                                  InputAdapter
+                                                    Exchange [ss_customer_sk] #8
+                                                      WholeStageCodegen (11)
+                                                        Project [ss_customer_sk,ss_quantity,ss_sales_price]
+                                                          Filter [ss_customer_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk]
+                                            InputAdapter
+                                              WholeStageCodegen (14)
+                                                Sort [c_customer_sk]
+                                                  InputAdapter
+                                                    Exchange [c_customer_sk] #9
+                                                      WholeStageCodegen (13)
+                                                        Filter [c_customer_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.customer [c_customer_sk]
+                      InputAdapter
+                        BroadcastExchange #4
+                          WholeStageCodegen (16)
+                            Project [d_date_sk]
+                              Filter [d_year,d_moy,d_date_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                WholeStageCodegen (34)
+                  Project [ws_quantity,ws_list_price]
+                    BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                      Project [ws_quantity,ws_list_price,ws_sold_date_sk]
+                        SortMergeJoin [ws_bill_customer_sk,c_customer_sk]
+                          InputAdapter
+                            WholeStageCodegen (27)
+                              Sort [ws_bill_customer_sk]
+                                InputAdapter
+                                  Exchange [ws_bill_customer_sk] #14
+                                    WholeStageCodegen (26)
+                                      Project [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk]
+                                        SortMergeJoin [ws_item_sk,item_sk]
+                                          InputAdapter
+                                            WholeStageCodegen (19)
+                                              Sort [ws_item_sk]
+                                                InputAdapter
+                                                  Exchange [ws_item_sk] #15
+                                                    WholeStageCodegen (18)
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk]
+                                                            ReusedSubquery [d_date_sk] #1
+                                          InputAdapter
+                                            WholeStageCodegen (25)
+                                              Sort [item_sk]
+                                                Project [item_sk]
+                                                  Filter [count(1)]
+                                                    HashAggregate [_groupingexpression,i_item_sk,d_date,count] [count(1),item_sk,count(1),count]
+                                                      HashAggregate [_groupingexpression,i_item_sk,d_date] [count,count]
+                                                        Project [d_date,i_item_sk,i_item_desc]
+                                                          SortMergeJoin [ss_item_sk,i_item_sk]
+                                                            InputAdapter
+                                                              WholeStageCodegen (22)
+                                                                Sort [ss_item_sk]
+                                                                  InputAdapter
+                                                                    ReusedExchange [ss_item_sk,d_date] #16
+                                                            InputAdapter
+                                                              WholeStageCodegen (24)
+                                                                Sort [i_item_sk]
+                                                                  InputAdapter
+                                                                    ReusedExchange [i_item_sk,i_item_desc] #7
+                          InputAdapter
+                            WholeStageCodegen (32)
+                              Sort [c_customer_sk]
+                                Project [c_customer_sk]
+                                  Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                    ReusedSubquery [tpcds_cmax] #3
+                                    HashAggregate [c_customer_sk,sum,isEmpty] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty]
+                                      HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty]
+                                        Project [ss_quantity,ss_sales_price,c_customer_sk]
+                                          SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                            InputAdapter
+                                              WholeStageCodegen (29)
+                                                Sort [ss_customer_sk]
+                                                  InputAdapter
+                                                    ReusedExchange [ss_customer_sk,ss_quantity,ss_sales_price] #8
+                                            InputAdapter
+                                              WholeStageCodegen (31)
+                                                Sort [c_customer_sk]
+                                                  InputAdapter
+                                                    ReusedExchange [c_customer_sk] #9
+                      InputAdapter
+                        ReusedExchange [d_date_sk] #4
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/explain.txt
new file mode 100644
index 0000000000000..2bce3e4d9bb60
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/explain.txt
@@ -0,0 +1,577 @@
+== Physical Plan ==
+* HashAggregate (74)
++- Exchange (73)
+   +- * HashAggregate (72)
+      +- Union (71)
+         :- * Project (53)
+         :  +- * BroadcastHashJoin Inner BuildRight (52)
+         :     :- * Project (46)
+         :     :  +- * SortMergeJoin LeftSemi (45)
+         :     :     :- * Sort (28)
+         :     :     :  +- Exchange (27)
+         :     :     :     +- * Project (26)
+         :     :     :        +- * BroadcastHashJoin LeftSemi BuildRight (25)
+         :     :     :           :- * ColumnarToRow (2)
+         :     :     :           :  +- Scan parquet default.catalog_sales (1)
+         :     :     :           +- BroadcastExchange (24)
+         :     :     :              +- * Project (23)
+         :     :     :                 +- * Filter (22)
+         :     :     :                    +- * HashAggregate (21)
+         :     :     :                       +- Exchange (20)
+         :     :     :                          +- * HashAggregate (19)
+         :     :     :                             +- * Project (18)
+         :     :     :                                +- * BroadcastHashJoin Inner BuildRight (17)
+         :     :     :                                   :- * Project (12)
+         :     :     :                                   :  +- * BroadcastHashJoin Inner BuildRight (11)
+         :     :     :                                   :     :- * Filter (5)
+         :     :     :                                   :     :  +- * ColumnarToRow (4)
+         :     :     :                                   :     :     +- Scan parquet default.store_sales (3)
+         :     :     :                                   :     +- BroadcastExchange (10)
+         :     :     :                                   :        +- * Project (9)
+         :     :     :                                   :           +- * Filter (8)
+         :     :     :                                   :              +- * ColumnarToRow (7)
+         :     :     :                                   :                 +- Scan parquet default.date_dim (6)
+         :     :     :                                   +- BroadcastExchange (16)
+         :     :     :                                      +- * Filter (15)
+         :     :     :                                         +- * ColumnarToRow (14)
+         :     :     :                                            +- Scan parquet default.item (13)
+         :     :     +- * Sort (44)
+         :     :        +- * Project (43)
+         :     :           +- * Filter (42)
+         :     :              +- * HashAggregate (41)
+         :     :                 +- Exchange (40)
+         :     :                    +- * HashAggregate (39)
+         :     :                       +- * Project (38)
+         :     :                          +- * BroadcastHashJoin Inner BuildRight (37)
+         :     :                             :- * Project (32)
+         :     :                             :  +- * Filter (31)
+         :     :                             :     +- * ColumnarToRow (30)
+         :     :                             :        +- Scan parquet default.store_sales (29)
+         :     :                             +- BroadcastExchange (36)
+         :     :                                +- * Filter (35)
+         :     :                                   +- * ColumnarToRow (34)
+         :     :                                      +- Scan parquet default.customer (33)
+         :     +- BroadcastExchange (51)
+         :        +- * Project (50)
+         :           +- * Filter (49)
+         :              +- * ColumnarToRow (48)
+         :                 +- Scan parquet default.date_dim (47)
+         +- * Project (70)
+            +- * BroadcastHashJoin Inner BuildRight (69)
+               :- * Project (67)
+               :  +- * SortMergeJoin LeftSemi (66)
+               :     :- * Sort (60)
+               :     :  +- Exchange (59)
+               :     :     +- * Project (58)
+               :     :        +- * BroadcastHashJoin LeftSemi BuildRight (57)
+               :     :           :- * ColumnarToRow (55)
+               :     :           :  +- Scan parquet default.web_sales (54)
+               :     :           +- ReusedExchange (56)
+               :     +- * Sort (65)
+               :        +- * Project (64)
+               :           +- * Filter (63)
+               :              +- * HashAggregate (62)
+               :                 +- ReusedExchange (61)
+               +- ReusedExchange (68)
+
+
+(1) Scan parquet default.catalog_sales
+Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sold_date_sk#5 IN dynamicpruning#6)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_item_sk:int,cs_quantity:int,cs_list_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+
+(3) Scan parquet default.store_sales
+Output [2]: [ss_item_sk#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int>
+
+(4) ColumnarToRow [codegen id : 3]
+Input [2]: [ss_item_sk#7, ss_sold_date_sk#8]
+
+(5) Filter [codegen id : 3]
+Input [2]: [ss_item_sk#7, ss_sold_date_sk#8]
+Condition : isnotnull(ss_item_sk#7)
+
+(6) Scan parquet default.date_dim
+Output [3]: [d_date_sk#10, d_date#11, d_year#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date,d_year:int>
+
+(7) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#10, d_date#11, d_year#12]
+
+(8) Filter [codegen id : 1]
+Input [3]: [d_date_sk#10, d_date#11, d_year#12]
+Condition : (d_year#12 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#10))
+
+(9) Project [codegen id : 1]
+Output [2]: [d_date_sk#10, d_date#11]
+Input [3]: [d_date_sk#10, d_date#11, d_year#12]
+
+(10) BroadcastExchange
+Input [2]: [d_date_sk#10, d_date#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(11) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(12) Project [codegen id : 3]
+Output [2]: [ss_item_sk#7, d_date#11]
+Input [4]: [ss_item_sk#7, ss_sold_date_sk#8, d_date_sk#10, d_date#11]
+
+(13) Scan parquet default.item
+Output [2]: [i_item_sk#14, i_item_desc#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_desc:string>
+
+(14) ColumnarToRow [codegen id : 2]
+Input [2]: [i_item_sk#14, i_item_desc#15]
+
+(15) Filter [codegen id : 2]
+Input [2]: [i_item_sk#14, i_item_desc#15]
+Condition : isnotnull(i_item_sk#14)
+
+(16) BroadcastExchange
+Input [2]: [i_item_sk#14, i_item_desc#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#16]
+
+(17) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_item_sk#7]
+Right keys [1]: [i_item_sk#14]
+Join condition: None
+
+(18) Project [codegen id : 3]
+Output [3]: [d_date#11, i_item_sk#14, substr(i_item_desc#15, 1, 30) AS _groupingexpression#17]
+Input [4]: [ss_item_sk#7, d_date#11, i_item_sk#14, i_item_desc#15]
+
+(19) HashAggregate [codegen id : 3]
+Input [3]: [d_date#11, i_item_sk#14, _groupingexpression#17]
+Keys [3]: [_groupingexpression#17, i_item_sk#14, d_date#11]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#18]
+Results [4]: [_groupingexpression#17, i_item_sk#14, d_date#11, count#19]
+
+(20) Exchange
+Input [4]: [_groupingexpression#17, i_item_sk#14, d_date#11, count#19]
+Arguments: hashpartitioning(_groupingexpression#17, i_item_sk#14, d_date#11, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(21) HashAggregate [codegen id : 4]
+Input [4]: [_groupingexpression#17, i_item_sk#14, d_date#11, count#19]
+Keys [3]: [_groupingexpression#17, i_item_sk#14, d_date#11]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#21]
+Results [2]: [i_item_sk#14 AS item_sk#22, count(1)#21 AS count(1)#23]
+
+(22) Filter [codegen id : 4]
+Input [2]: [item_sk#22, count(1)#23]
+Condition : (count(1)#23 > 4)
+
+(23) Project [codegen id : 4]
+Output [1]: [item_sk#22]
+Input [2]: [item_sk#22, count(1)#23]
+
+(24) BroadcastExchange
+Input [1]: [item_sk#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#24]
+
+(25) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [cs_item_sk#2]
+Right keys [1]: [item_sk#22]
+Join condition: None
+
+(26) Project [codegen id : 5]
+Output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+
+(27) Exchange
+Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(28) Sort [codegen id : 6]
+Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+Arguments: [cs_bill_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(29) Scan parquet default.store_sales
+Output [4]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28, ss_sold_date_sk#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_quantity:int,ss_sales_price:decimal(7,2)>
+
+(30) ColumnarToRow [codegen id : 8]
+Input [4]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28, ss_sold_date_sk#29]
+
+(31) Filter [codegen id : 8]
+Input [4]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28, ss_sold_date_sk#29]
+Condition : isnotnull(ss_customer_sk#26)
+
+(32) Project [codegen id : 8]
+Output [3]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28]
+Input [4]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28, ss_sold_date_sk#29]
+
+(33) Scan parquet default.customer
+Output [1]: [c_customer_sk#30]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int>
+
+(34) ColumnarToRow [codegen id : 7]
+Input [1]: [c_customer_sk#30]
+
+(35) Filter [codegen id : 7]
+Input [1]: [c_customer_sk#30]
+Condition : isnotnull(c_customer_sk#30)
+
+(36) BroadcastExchange
+Input [1]: [c_customer_sk#30]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#31]
+
+(37) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_customer_sk#26]
+Right keys [1]: [c_customer_sk#30]
+Join condition: None
+
+(38) Project [codegen id : 8]
+Output [3]: [ss_quantity#27, ss_sales_price#28, c_customer_sk#30]
+Input [4]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28, c_customer_sk#30]
+
+(39) HashAggregate [codegen id : 8]
+Input [3]: [ss_quantity#27, ss_sales_price#28, c_customer_sk#30]
+Keys [1]: [c_customer_sk#30]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#27 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#28 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [2]: [sum#32, isEmpty#33]
+Results [3]: [c_customer_sk#30, sum#34, isEmpty#35]
+
+(40) Exchange
+Input [3]: [c_customer_sk#30, sum#34, isEmpty#35]
+Arguments: hashpartitioning(c_customer_sk#30, 5), ENSURE_REQUIREMENTS, [id=#36]
+
+(41) HashAggregate [codegen id : 9]
+Input [3]: [c_customer_sk#30, sum#34, isEmpty#35]
+Keys [1]: [c_customer_sk#30]
+Functions [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#27 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#28 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#27 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#28 as decimal(12,2)))), DecimalType(18,2), true))#37]
+Results [2]: [c_customer_sk#30, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#27 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#28 as decimal(12,2)))), DecimalType(18,2), true))#37 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40]
+
+(42) Filter [codegen id : 9]
+Input [2]: [c_customer_sk#30, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40 as decimal(38,8)) > CheckOverflow((0.500000 * promote_precision(cast(Subquery scalar-subquery#41, [id=#42] as decimal(32,6)))), DecimalType(38,8), true)))
+
+(43) Project [codegen id : 9]
+Output [1]: [c_customer_sk#30]
+Input [2]: [c_customer_sk#30, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40]
+
+(44) Sort [codegen id : 9]
+Input [1]: [c_customer_sk#30]
+Arguments: [c_customer_sk#30 ASC NULLS FIRST], false, 0
+
+(45) SortMergeJoin [codegen id : 11]
+Left keys [1]: [cs_bill_customer_sk#1]
+Right keys [1]: [c_customer_sk#30]
+Join condition: None
+
+(46) Project [codegen id : 11]
+Output [3]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+
+(47) Scan parquet default.date_dim
+Output [3]: [d_date_sk#43, d_year#44, d_moy#45]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(48) ColumnarToRow [codegen id : 10]
+Input [3]: [d_date_sk#43, d_year#44, d_moy#45]
+
+(49) Filter [codegen id : 10]
+Input [3]: [d_date_sk#43, d_year#44, d_moy#45]
+Condition : ((((isnotnull(d_year#44) AND isnotnull(d_moy#45)) AND (d_year#44 = 2000)) AND (d_moy#45 = 2)) AND isnotnull(d_date_sk#43))
+
+(50) Project [codegen id : 10]
+Output [1]: [d_date_sk#43]
+Input [3]: [d_date_sk#43, d_year#44, d_moy#45]
+
+(51) BroadcastExchange
+Input [1]: [d_date_sk#43]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#46]
+
+(52) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_sold_date_sk#5]
+Right keys [1]: [d_date_sk#43]
+Join condition: None
+
+(53) Project [codegen id : 11]
+Output [1]: [CheckOverflow((promote_precision(cast(cast(cs_quantity#3 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#4 as decimal(12,2)))), DecimalType(18,2), true) AS sales#47]
+Input [4]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, d_date_sk#43]
+
+(54) Scan parquet default.web_sales
+Output [5]: [ws_item_sk#48, ws_bill_customer_sk#49, ws_quantity#50, ws_list_price#51, ws_sold_date_sk#52]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#52), dynamicpruningexpression(ws_sold_date_sk#52 IN dynamicpruning#6)]
+ReadSchema: struct<ws_item_sk:int,ws_bill_customer_sk:int,ws_quantity:int,ws_list_price:decimal(7,2)>
+
+(55) ColumnarToRow [codegen id : 16]
+Input [5]: [ws_item_sk#48, ws_bill_customer_sk#49, ws_quantity#50, ws_list_price#51, ws_sold_date_sk#52]
+
+(56) ReusedExchange [Reuses operator id: unknown]
+Output [1]: [item_sk#22]
+
+(57) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [ws_item_sk#48]
+Right keys [1]: [item_sk#22]
+Join condition: None
+
+(58) Project [codegen id : 16]
+Output [4]: [ws_bill_customer_sk#49, ws_quantity#50, ws_list_price#51, ws_sold_date_sk#52]
+Input [5]: [ws_item_sk#48, ws_bill_customer_sk#49, ws_quantity#50, ws_list_price#51, ws_sold_date_sk#52]
+
+(59) Exchange
+Input [4]: [ws_bill_customer_sk#49, ws_quantity#50, ws_list_price#51, ws_sold_date_sk#52]
+Arguments: hashpartitioning(ws_bill_customer_sk#49, 5), ENSURE_REQUIREMENTS, [id=#53]
+
+(60) Sort [codegen id : 17]
+Input [4]: [ws_bill_customer_sk#49, ws_quantity#50, ws_list_price#51, ws_sold_date_sk#52]
+Arguments: [ws_bill_customer_sk#49 ASC NULLS FIRST], false, 0
+
+(61) ReusedExchange [Reuses operator id: 40]
+Output [3]: [c_customer_sk#54, sum#55, isEmpty#56]
+
+(62) HashAggregate [codegen id : 20]
+Input [3]: [c_customer_sk#54, sum#55, isEmpty#56]
+Keys [1]: [c_customer_sk#54]
+Functions [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#57 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#58 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#57 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#58 as decimal(12,2)))), DecimalType(18,2), true))#59]
+Results [2]: [c_customer_sk#54, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#57 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#58 as decimal(12,2)))), DecimalType(18,2), true))#59 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#60]
+
+(63) Filter [codegen id : 20]
+Input [2]: [c_customer_sk#54, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#60]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#60) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#60 as decimal(38,8)) > CheckOverflow((0.500000 * promote_precision(cast(ReusedSubquery Subquery scalar-subquery#41, [id=#42] as decimal(32,6)))), DecimalType(38,8), true)))
+
+(64) Project [codegen id : 20]
+Output [1]: [c_customer_sk#54]
+Input [2]: [c_customer_sk#54, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#60]
+
+(65) Sort [codegen id : 20]
+Input [1]: [c_customer_sk#54]
+Arguments: [c_customer_sk#54 ASC NULLS FIRST], false, 0
+
+(66) SortMergeJoin [codegen id : 22]
+Left keys [1]: [ws_bill_customer_sk#49]
+Right keys [1]: [c_customer_sk#54]
+Join condition: None
+
+(67) Project [codegen id : 22]
+Output [3]: [ws_quantity#50, ws_list_price#51, ws_sold_date_sk#52]
+Input [4]: [ws_bill_customer_sk#49, ws_quantity#50, ws_list_price#51, ws_sold_date_sk#52]
+
+(68) ReusedExchange [Reuses operator id: 51]
+Output [1]: [d_date_sk#61]
+
+(69) BroadcastHashJoin [codegen id : 22]
+Left keys [1]: [ws_sold_date_sk#52]
+Right keys [1]: [d_date_sk#61]
+Join condition: None
+
+(70) Project [codegen id : 22]
+Output [1]: [CheckOverflow((promote_precision(cast(cast(ws_quantity#50 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#51 as decimal(12,2)))), DecimalType(18,2), true) AS sales#62]
+Input [4]: [ws_quantity#50, ws_list_price#51, ws_sold_date_sk#52, d_date_sk#61]
+
+(71) Union
+
+(72) HashAggregate [codegen id : 23]
+Input [1]: [sales#47]
+Keys: []
+Functions [1]: [partial_sum(sales#47)]
+Aggregate Attributes [2]: [sum#63, isEmpty#64]
+Results [2]: [sum#65, isEmpty#66]
+
+(73) Exchange
+Input [2]: [sum#65, isEmpty#66]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#67]
+
+(74) HashAggregate [codegen id : 24]
+Input [2]: [sum#65, isEmpty#66]
+Keys: []
+Functions [1]: [sum(sales#47)]
+Aggregate Attributes [1]: [sum(sales#47)#68]
+Results [1]: [sum(sales#47)#68 AS sum(sales)#69]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (75)
+
+
+(75) ReusedExchange [Reuses operator id: 51]
+Output [1]: [d_date_sk#43]
+
+Subquery:2 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (76)
+
+
+(76) ReusedExchange [Reuses operator id: 10]
+Output [2]: [d_date_sk#10, d_date#11]
+
+Subquery:3 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#41, [id=#42]
+* HashAggregate (98)
++- Exchange (97)
+   +- * HashAggregate (96)
+      +- * HashAggregate (95)
+         +- Exchange (94)
+            +- * HashAggregate (93)
+               +- * Project (92)
+                  +- * BroadcastHashJoin Inner BuildRight (91)
+                     :- * Project (85)
+                     :  +- * BroadcastHashJoin Inner BuildRight (84)
+                     :     :- * Filter (79)
+                     :     :  +- * ColumnarToRow (78)
+                     :     :     +- Scan parquet default.store_sales (77)
+                     :     +- BroadcastExchange (83)
+                     :        +- * Filter (82)
+                     :           +- * ColumnarToRow (81)
+                     :              +- Scan parquet default.customer (80)
+                     +- BroadcastExchange (90)
+                        +- * Project (89)
+                           +- * Filter (88)
+                              +- * ColumnarToRow (87)
+                                 +- Scan parquet default.date_dim (86)
+
+
+(77) Scan parquet default.store_sales
+Output [4]: [ss_customer_sk#70, ss_quantity#71, ss_sales_price#72, ss_sold_date_sk#73]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#73), dynamicpruningexpression(ss_sold_date_sk#73 IN dynamicpruning#74)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_quantity:int,ss_sales_price:decimal(7,2)>
+
+(78) ColumnarToRow [codegen id : 3]
+Input [4]: [ss_customer_sk#70, ss_quantity#71, ss_sales_price#72, ss_sold_date_sk#73]
+
+(79) Filter [codegen id : 3]
+Input [4]: [ss_customer_sk#70, ss_quantity#71, ss_sales_price#72, ss_sold_date_sk#73]
+Condition : isnotnull(ss_customer_sk#70)
+
+(80) Scan parquet default.customer
+Output [1]: [c_customer_sk#75]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int>
+
+(81) ColumnarToRow [codegen id : 1]
+Input [1]: [c_customer_sk#75]
+
+(82) Filter [codegen id : 1]
+Input [1]: [c_customer_sk#75]
+Condition : isnotnull(c_customer_sk#75)
+
+(83) BroadcastExchange
+Input [1]: [c_customer_sk#75]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#76]
+
+(84) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_customer_sk#70]
+Right keys [1]: [c_customer_sk#75]
+Join condition: None
+
+(85) Project [codegen id : 3]
+Output [4]: [ss_quantity#71, ss_sales_price#72, ss_sold_date_sk#73, c_customer_sk#75]
+Input [5]: [ss_customer_sk#70, ss_quantity#71, ss_sales_price#72, ss_sold_date_sk#73, c_customer_sk#75]
+
+(86) Scan parquet default.date_dim
+Output [2]: [d_date_sk#77, d_year#78]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(87) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#77, d_year#78]
+
+(88) Filter [codegen id : 2]
+Input [2]: [d_date_sk#77, d_year#78]
+Condition : (d_year#78 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#77))
+
+(89) Project [codegen id : 2]
+Output [1]: [d_date_sk#77]
+Input [2]: [d_date_sk#77, d_year#78]
+
+(90) BroadcastExchange
+Input [1]: [d_date_sk#77]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#79]
+
+(91) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#73]
+Right keys [1]: [d_date_sk#77]
+Join condition: None
+
+(92) Project [codegen id : 3]
+Output [3]: [ss_quantity#71, ss_sales_price#72, c_customer_sk#75]
+Input [5]: [ss_quantity#71, ss_sales_price#72, ss_sold_date_sk#73, c_customer_sk#75, d_date_sk#77]
+
+(93) HashAggregate [codegen id : 3]
+Input [3]: [ss_quantity#71, ss_sales_price#72, c_customer_sk#75]
+Keys [1]: [c_customer_sk#75]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#71 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#72 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [2]: [sum#80, isEmpty#81]
+Results [3]: [c_customer_sk#75, sum#82, isEmpty#83]
+
+(94) Exchange
+Input [3]: [c_customer_sk#75, sum#82, isEmpty#83]
+Arguments: hashpartitioning(c_customer_sk#75, 5), ENSURE_REQUIREMENTS, [id=#84]
+
+(95) HashAggregate [codegen id : 4]
+Input [3]: [c_customer_sk#75, sum#82, isEmpty#83]
+Keys [1]: [c_customer_sk#75]
+Functions [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#71 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#72 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#71 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#72 as decimal(12,2)))), DecimalType(18,2), true))#85]
+Results [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#71 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#72 as decimal(12,2)))), DecimalType(18,2), true))#85 AS csales#86]
+
+(96) HashAggregate [codegen id : 4]
+Input [1]: [csales#86]
+Keys: []
+Functions [1]: [partial_max(csales#86)]
+Aggregate Attributes [1]: [max#87]
+Results [1]: [max#88]
+
+(97) Exchange
+Input [1]: [max#88]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#89]
+
+(98) HashAggregate [codegen id : 5]
+Input [1]: [max#88]
+Keys: []
+Functions [1]: [max(csales#86)]
+Aggregate Attributes [1]: [max(csales#86)#90]
+Results [1]: [max(csales#86)#90 AS tpcds_cmax#91]
+
+Subquery:4 Hosting operator id = 77 Hosting Expression = ss_sold_date_sk#73 IN dynamicpruning#74
+ReusedExchange (99)
+
+
+(99) ReusedExchange [Reuses operator id: 90]
+Output [1]: [d_date_sk#77]
+
+Subquery:5 Hosting operator id = 54 Hosting Expression = ws_sold_date_sk#52 IN dynamicpruning#6
+
+Subquery:6 Hosting operator id = 63 Hosting Expression = ReusedSubquery Subquery scalar-subquery#41, [id=#42]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/simplified.txt
new file mode 100644
index 0000000000000..697c1f497c399
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/simplified.txt
@@ -0,0 +1,160 @@
+WholeStageCodegen (24)
+  HashAggregate [sum,isEmpty] [sum(sales),sum(sales),sum,isEmpty]
+    InputAdapter
+      Exchange #1
+        WholeStageCodegen (23)
+          HashAggregate [sales] [sum,isEmpty,sum,isEmpty]
+            InputAdapter
+              Union
+                WholeStageCodegen (11)
+                  Project [cs_quantity,cs_list_price]
+                    BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                      Project [cs_quantity,cs_list_price,cs_sold_date_sk]
+                        SortMergeJoin [cs_bill_customer_sk,c_customer_sk]
+                          InputAdapter
+                            WholeStageCodegen (6)
+                              Sort [cs_bill_customer_sk]
+                                InputAdapter
+                                  Exchange [cs_bill_customer_sk] #2
+                                    WholeStageCodegen (5)
+                                      Project [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk]
+                                        BroadcastHashJoin [cs_item_sk,item_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk]
+                                                SubqueryBroadcast [d_date_sk] #1
+                                                  ReusedExchange [d_date_sk] #3
+                                          InputAdapter
+                                            BroadcastExchange #4
+                                              WholeStageCodegen (4)
+                                                Project [item_sk]
+                                                  Filter [count(1)]
+                                                    HashAggregate [_groupingexpression,i_item_sk,d_date,count] [count(1),item_sk,count(1),count]
+                                                      InputAdapter
+                                                        Exchange [_groupingexpression,i_item_sk,d_date] #5
+                                                          WholeStageCodegen (3)
+                                                            HashAggregate [_groupingexpression,i_item_sk,d_date] [count,count]
+                                                              Project [d_date,i_item_sk,i_item_desc]
+                                                                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                                  Project [ss_item_sk,d_date]
+                                                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                      Filter [ss_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.store_sales [ss_item_sk,ss_sold_date_sk]
+                                                                              SubqueryBroadcast [d_date_sk] #2
+                                                                                ReusedExchange [d_date_sk,d_date] #6
+                                                                      InputAdapter
+                                                                        BroadcastExchange #6
+                                                                          WholeStageCodegen (1)
+                                                                            Project [d_date_sk,d_date]
+                                                                              Filter [d_year,d_date_sk]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.date_dim [d_date_sk,d_date,d_year]
+                                                                  InputAdapter
+                                                                    BroadcastExchange #7
+                                                                      WholeStageCodegen (2)
+                                                                        Filter [i_item_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.item [i_item_sk,i_item_desc]
+                          InputAdapter
+                            WholeStageCodegen (9)
+                              Sort [c_customer_sk]
+                                Project [c_customer_sk]
+                                  Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                    Subquery #3
+                                      WholeStageCodegen (5)
+                                        HashAggregate [max] [max(csales),tpcds_cmax,max]
+                                          InputAdapter
+                                            Exchange #10
+                                              WholeStageCodegen (4)
+                                                HashAggregate [csales] [max,max]
+                                                  HashAggregate [c_customer_sk,sum,isEmpty] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),csales,sum,isEmpty]
+                                                    InputAdapter
+                                                      Exchange [c_customer_sk] #11
+                                                        WholeStageCodegen (3)
+                                                          HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty]
+                                                            Project [ss_quantity,ss_sales_price,c_customer_sk]
+                                                              BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                Project [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk]
+                                                                  BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+                                                                    Filter [ss_customer_sk]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk]
+                                                                            SubqueryBroadcast [d_date_sk] #4
+                                                                              ReusedExchange [d_date_sk] #12
+                                                                    InputAdapter
+                                                                      BroadcastExchange #13
+                                                                        WholeStageCodegen (1)
+                                                                          Filter [c_customer_sk]
+                                                                            ColumnarToRow
+                                                                              InputAdapter
+                                                                                Scan parquet default.customer [c_customer_sk]
+                                                                InputAdapter
+                                                                  BroadcastExchange #12
+                                                                    WholeStageCodegen (2)
+                                                                      Project [d_date_sk]
+                                                                        Filter [d_year,d_date_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.date_dim [d_date_sk,d_year]
+                                    HashAggregate [c_customer_sk,sum,isEmpty] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty]
+                                      InputAdapter
+                                        Exchange [c_customer_sk] #8
+                                          WholeStageCodegen (8)
+                                            HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty]
+                                              Project [ss_quantity,ss_sales_price,c_customer_sk]
+                                                BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+                                                  Project [ss_customer_sk,ss_quantity,ss_sales_price]
+                                                    Filter [ss_customer_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk]
+                                                  InputAdapter
+                                                    BroadcastExchange #9
+                                                      WholeStageCodegen (7)
+                                                        Filter [c_customer_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.customer [c_customer_sk]
+                      InputAdapter
+                        BroadcastExchange #3
+                          WholeStageCodegen (10)
+                            Project [d_date_sk]
+                              Filter [d_year,d_moy,d_date_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                WholeStageCodegen (22)
+                  Project [ws_quantity,ws_list_price]
+                    BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                      Project [ws_quantity,ws_list_price,ws_sold_date_sk]
+                        SortMergeJoin [ws_bill_customer_sk,c_customer_sk]
+                          InputAdapter
+                            WholeStageCodegen (17)
+                              Sort [ws_bill_customer_sk]
+                                InputAdapter
+                                  Exchange [ws_bill_customer_sk] #14
+                                    WholeStageCodegen (16)
+                                      Project [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk]
+                                        BroadcastHashJoin [ws_item_sk,item_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk]
+                                                ReusedSubquery [d_date_sk] #1
+                                          InputAdapter
+                                            ReusedExchange [item_sk] #15
+                          InputAdapter
+                            WholeStageCodegen (20)
+                              Sort [c_customer_sk]
+                                Project [c_customer_sk]
+                                  Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                    ReusedSubquery [tpcds_cmax] #3
+                                    HashAggregate [c_customer_sk,sum,isEmpty] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty]
+                                      InputAdapter
+                                        ReusedExchange [c_customer_sk,sum,isEmpty] #8
+                      InputAdapter
+                        ReusedExchange [d_date_sk] #3
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.sf100/explain.txt
new file mode 100644
index 0000000000000..c5b8ad4607553
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.sf100/explain.txt
@@ -0,0 +1,938 @@
+== Physical Plan ==
+TakeOrderedAndProject (137)
++- Union (136)
+   :- * HashAggregate (81)
+   :  +- Exchange (80)
+   :     +- * HashAggregate (79)
+   :        +- * Project (78)
+   :           +- * SortMergeJoin Inner (77)
+   :              :- * Project (59)
+   :              :  +- * BroadcastHashJoin Inner BuildRight (58)
+   :              :     :- * SortMergeJoin LeftSemi (52)
+   :              :     :  :- * Sort (33)
+   :              :     :  :  +- Exchange (32)
+   :              :     :  :     +- * Project (31)
+   :              :     :  :        +- * SortMergeJoin LeftSemi (30)
+   :              :     :  :           :- * Sort (5)
+   :              :     :  :           :  +- Exchange (4)
+   :              :     :  :           :     +- * Filter (3)
+   :              :     :  :           :        +- * ColumnarToRow (2)
+   :              :     :  :           :           +- Scan parquet default.catalog_sales (1)
+   :              :     :  :           +- * Sort (29)
+   :              :     :  :              +- * Project (28)
+   :              :     :  :                 +- * Filter (27)
+   :              :     :  :                    +- * HashAggregate (26)
+   :              :     :  :                       +- * HashAggregate (25)
+   :              :     :  :                          +- * Project (24)
+   :              :     :  :                             +- * SortMergeJoin Inner (23)
+   :              :     :  :                                :- * Sort (17)
+   :              :     :  :                                :  +- Exchange (16)
+   :              :     :  :                                :     +- * Project (15)
+   :              :     :  :                                :        +- * BroadcastHashJoin Inner BuildRight (14)
+   :              :     :  :                                :           :- * Filter (8)
+   :              :     :  :                                :           :  +- * ColumnarToRow (7)
+   :              :     :  :                                :           :     +- Scan parquet default.store_sales (6)
+   :              :     :  :                                :           +- BroadcastExchange (13)
+   :              :     :  :                                :              +- * Project (12)
+   :              :     :  :                                :                 +- * Filter (11)
+   :              :     :  :                                :                    +- * ColumnarToRow (10)
+   :              :     :  :                                :                       +- Scan parquet default.date_dim (9)
+   :              :     :  :                                +- * Sort (22)
+   :              :     :  :                                   +- Exchange (21)
+   :              :     :  :                                      +- * Filter (20)
+   :              :     :  :                                         +- * ColumnarToRow (19)
+   :              :     :  :                                            +- Scan parquet default.item (18)
+   :              :     :  +- * Sort (51)
+   :              :     :     +- * Project (50)
+   :              :     :        +- * Filter (49)
+   :              :     :           +- * HashAggregate (48)
+   :              :     :              +- * HashAggregate (47)
+   :              :     :                 +- * Project (46)
+   :              :     :                    +- * SortMergeJoin Inner (45)
+   :              :     :                       :- * Sort (39)
+   :              :     :                       :  +- Exchange (38)
+   :              :     :                       :     +- * Project (37)
+   :              :     :                       :        +- * Filter (36)
+   :              :     :                       :           +- * ColumnarToRow (35)
+   :              :     :                       :              +- Scan parquet default.store_sales (34)
+   :              :     :                       +- * Sort (44)
+   :              :     :                          +- Exchange (43)
+   :              :     :                             +- * Filter (42)
+   :              :     :                                +- * ColumnarToRow (41)
+   :              :     :                                   +- Scan parquet default.customer (40)
+   :              :     +- BroadcastExchange (57)
+   :              :        +- * Project (56)
+   :              :           +- * Filter (55)
+   :              :              +- * ColumnarToRow (54)
+   :              :                 +- Scan parquet default.date_dim (53)
+   :              +- * SortMergeJoin LeftSemi (76)
+   :                 :- * Sort (64)
+   :                 :  +- Exchange (63)
+   :                 :     +- * Filter (62)
+   :                 :        +- * ColumnarToRow (61)
+   :                 :           +- Scan parquet default.customer (60)
+   :                 +- * Sort (75)
+   :                    +- * Project (74)
+   :                       +- * Filter (73)
+   :                          +- * HashAggregate (72)
+   :                             +- * HashAggregate (71)
+   :                                +- * Project (70)
+   :                                   +- * SortMergeJoin Inner (69)
+   :                                      :- * Sort (66)
+   :                                      :  +- ReusedExchange (65)
+   :                                      +- * Sort (68)
+   :                                         +- ReusedExchange (67)
+   +- * HashAggregate (135)
+      +- Exchange (134)
+         +- * HashAggregate (133)
+            +- * Project (132)
+               +- * SortMergeJoin Inner (131)
+                  :- * Project (116)
+                  :  +- * BroadcastHashJoin Inner BuildRight (115)
+                  :     :- * SortMergeJoin LeftSemi (113)
+                  :     :  :- * Sort (101)
+                  :     :  :  +- Exchange (100)
+                  :     :  :     +- * Project (99)
+                  :     :  :        +- * SortMergeJoin LeftSemi (98)
+                  :     :  :           :- * Sort (86)
+                  :     :  :           :  +- Exchange (85)
+                  :     :  :           :     +- * Filter (84)
+                  :     :  :           :        +- * ColumnarToRow (83)
+                  :     :  :           :           +- Scan parquet default.web_sales (82)
+                  :     :  :           +- * Sort (97)
+                  :     :  :              +- * Project (96)
+                  :     :  :                 +- * Filter (95)
+                  :     :  :                    +- * HashAggregate (94)
+                  :     :  :                       +- * HashAggregate (93)
+                  :     :  :                          +- * Project (92)
+                  :     :  :                             +- * SortMergeJoin Inner (91)
+                  :     :  :                                :- * Sort (88)
+                  :     :  :                                :  +- ReusedExchange (87)
+                  :     :  :                                +- * Sort (90)
+                  :     :  :                                   +- ReusedExchange (89)
+                  :     :  +- * Sort (112)
+                  :     :     +- * Project (111)
+                  :     :        +- * Filter (110)
+                  :     :           +- * HashAggregate (109)
+                  :     :              +- * HashAggregate (108)
+                  :     :                 +- * Project (107)
+                  :     :                    +- * SortMergeJoin Inner (106)
+                  :     :                       :- * Sort (103)
+                  :     :                       :  +- ReusedExchange (102)
+                  :     :                       +- * Sort (105)
+                  :     :                          +- ReusedExchange (104)
+                  :     +- ReusedExchange (114)
+                  +- * SortMergeJoin LeftSemi (130)
+                     :- * Sort (118)
+                     :  +- ReusedExchange (117)
+                     +- * Sort (129)
+                        +- * Project (128)
+                           +- * Filter (127)
+                              +- * HashAggregate (126)
+                                 +- * HashAggregate (125)
+                                    +- * Project (124)
+                                       +- * SortMergeJoin Inner (123)
+                                          :- * Sort (120)
+                                          :  +- ReusedExchange (119)
+                                          +- * Sort (122)
+                                             +- ReusedExchange (121)
+
+
+(1) Scan parquet default.catalog_sales
+Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(cs_bill_customer_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_item_sk:int,cs_quantity:int,cs_list_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+
+(3) Filter [codegen id : 1]
+Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+Condition : isnotnull(cs_bill_customer_sk#1)
+
+(4) Exchange
+Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+Arguments: hashpartitioning(cs_item_sk#2, 5), ENSURE_REQUIREMENTS, [id=#7]
+
+(5) Sort [codegen id : 2]
+Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+Arguments: [cs_item_sk#2 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.store_sales
+Output [2]: [ss_item_sk#8, ss_sold_date_sk#9]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#9), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int>
+
+(7) ColumnarToRow [codegen id : 4]
+Input [2]: [ss_item_sk#8, ss_sold_date_sk#9]
+
+(8) Filter [codegen id : 4]
+Input [2]: [ss_item_sk#8, ss_sold_date_sk#9]
+Condition : isnotnull(ss_item_sk#8)
+
+(9) Scan parquet default.date_dim
+Output [3]: [d_date_sk#11, d_date#12, d_year#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date,d_year:int>
+
+(10) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#11, d_date#12, d_year#13]
+
+(11) Filter [codegen id : 3]
+Input [3]: [d_date_sk#11, d_date#12, d_year#13]
+Condition : (d_year#13 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#11))
+
+(12) Project [codegen id : 3]
+Output [2]: [d_date_sk#11, d_date#12]
+Input [3]: [d_date_sk#11, d_date#12, d_year#13]
+
+(13) BroadcastExchange
+Input [2]: [d_date_sk#11, d_date#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(14) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#9]
+Right keys [1]: [d_date_sk#11]
+Join condition: None
+
+(15) Project [codegen id : 4]
+Output [2]: [ss_item_sk#8, d_date#12]
+Input [4]: [ss_item_sk#8, ss_sold_date_sk#9, d_date_sk#11, d_date#12]
+
+(16) Exchange
+Input [2]: [ss_item_sk#8, d_date#12]
+Arguments: hashpartitioning(ss_item_sk#8, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(17) Sort [codegen id : 5]
+Input [2]: [ss_item_sk#8, d_date#12]
+Arguments: [ss_item_sk#8 ASC NULLS FIRST], false, 0
+
+(18) Scan parquet default.item
+Output [2]: [i_item_sk#16, i_item_desc#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_desc:string>
+
+(19) ColumnarToRow [codegen id : 6]
+Input [2]: [i_item_sk#16, i_item_desc#17]
+
+(20) Filter [codegen id : 6]
+Input [2]: [i_item_sk#16, i_item_desc#17]
+Condition : isnotnull(i_item_sk#16)
+
+(21) Exchange
+Input [2]: [i_item_sk#16, i_item_desc#17]
+Arguments: hashpartitioning(i_item_sk#16, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(22) Sort [codegen id : 7]
+Input [2]: [i_item_sk#16, i_item_desc#17]
+Arguments: [i_item_sk#16 ASC NULLS FIRST], false, 0
+
+(23) SortMergeJoin [codegen id : 8]
+Left keys [1]: [ss_item_sk#8]
+Right keys [1]: [i_item_sk#16]
+Join condition: None
+
+(24) Project [codegen id : 8]
+Output [3]: [d_date#12, i_item_sk#16, substr(i_item_desc#17, 1, 30) AS _groupingexpression#19]
+Input [4]: [ss_item_sk#8, d_date#12, i_item_sk#16, i_item_desc#17]
+
+(25) HashAggregate [codegen id : 8]
+Input [3]: [d_date#12, i_item_sk#16, _groupingexpression#19]
+Keys [3]: [_groupingexpression#19, i_item_sk#16, d_date#12]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#20]
+Results [4]: [_groupingexpression#19, i_item_sk#16, d_date#12, count#21]
+
+(26) HashAggregate [codegen id : 8]
+Input [4]: [_groupingexpression#19, i_item_sk#16, d_date#12, count#21]
+Keys [3]: [_groupingexpression#19, i_item_sk#16, d_date#12]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#22]
+Results [2]: [i_item_sk#16 AS item_sk#23, count(1)#22 AS count(1)#24]
+
+(27) Filter [codegen id : 8]
+Input [2]: [item_sk#23, count(1)#24]
+Condition : (count(1)#24 > 4)
+
+(28) Project [codegen id : 8]
+Output [1]: [item_sk#23]
+Input [2]: [item_sk#23, count(1)#24]
+
+(29) Sort [codegen id : 8]
+Input [1]: [item_sk#23]
+Arguments: [item_sk#23 ASC NULLS FIRST], false, 0
+
+(30) SortMergeJoin [codegen id : 9]
+Left keys [1]: [cs_item_sk#2]
+Right keys [1]: [item_sk#23]
+Join condition: None
+
+(31) Project [codegen id : 9]
+Output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+
+(32) Exchange
+Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(33) Sort [codegen id : 10]
+Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+Arguments: [cs_bill_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(34) Scan parquet default.store_sales
+Output [4]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28, ss_sold_date_sk#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_quantity:int,ss_sales_price:decimal(7,2)>
+
+(35) ColumnarToRow [codegen id : 11]
+Input [4]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28, ss_sold_date_sk#29]
+
+(36) Filter [codegen id : 11]
+Input [4]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28, ss_sold_date_sk#29]
+Condition : isnotnull(ss_customer_sk#26)
+
+(37) Project [codegen id : 11]
+Output [3]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28]
+Input [4]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28, ss_sold_date_sk#29]
+
+(38) Exchange
+Input [3]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28]
+Arguments: hashpartitioning(ss_customer_sk#26, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(39) Sort [codegen id : 12]
+Input [3]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28]
+Arguments: [ss_customer_sk#26 ASC NULLS FIRST], false, 0
+
+(40) Scan parquet default.customer
+Output [1]: [c_customer_sk#31]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int>
+
+(41) ColumnarToRow [codegen id : 13]
+Input [1]: [c_customer_sk#31]
+
+(42) Filter [codegen id : 13]
+Input [1]: [c_customer_sk#31]
+Condition : isnotnull(c_customer_sk#31)
+
+(43) Exchange
+Input [1]: [c_customer_sk#31]
+Arguments: hashpartitioning(c_customer_sk#31, 5), ENSURE_REQUIREMENTS, [id=#32]
+
+(44) Sort [codegen id : 14]
+Input [1]: [c_customer_sk#31]
+Arguments: [c_customer_sk#31 ASC NULLS FIRST], false, 0
+
+(45) SortMergeJoin [codegen id : 15]
+Left keys [1]: [ss_customer_sk#26]
+Right keys [1]: [c_customer_sk#31]
+Join condition: None
+
+(46) Project [codegen id : 15]
+Output [3]: [ss_quantity#27, ss_sales_price#28, c_customer_sk#31]
+Input [4]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28, c_customer_sk#31]
+
+(47) HashAggregate [codegen id : 15]
+Input [3]: [ss_quantity#27, ss_sales_price#28, c_customer_sk#31]
+Keys [1]: [c_customer_sk#31]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#27 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#28 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [2]: [sum#33, isEmpty#34]
+Results [3]: [c_customer_sk#31, sum#35, isEmpty#36]
+
+(48) HashAggregate [codegen id : 15]
+Input [3]: [c_customer_sk#31, sum#35, isEmpty#36]
+Keys [1]: [c_customer_sk#31]
+Functions [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#27 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#28 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#27 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#28 as decimal(12,2)))), DecimalType(18,2), true))#37]
+Results [2]: [c_customer_sk#31, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#27 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#28 as decimal(12,2)))), DecimalType(18,2), true))#37 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40]
+
+(49) Filter [codegen id : 15]
+Input [2]: [c_customer_sk#31, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40 as decimal(38,8)) > CheckOverflow((0.500000 * promote_precision(cast(Subquery scalar-subquery#41, [id=#42] as decimal(32,6)))), DecimalType(38,8), true)))
+
+(50) Project [codegen id : 15]
+Output [1]: [c_customer_sk#31]
+Input [2]: [c_customer_sk#31, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40]
+
+(51) Sort [codegen id : 15]
+Input [1]: [c_customer_sk#31]
+Arguments: [c_customer_sk#31 ASC NULLS FIRST], false, 0
+
+(52) SortMergeJoin [codegen id : 17]
+Left keys [1]: [cs_bill_customer_sk#1]
+Right keys [1]: [c_customer_sk#31]
+Join condition: None
+
+(53) Scan parquet default.date_dim
+Output [3]: [d_date_sk#43, d_year#44, d_moy#45]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(54) ColumnarToRow [codegen id : 16]
+Input [3]: [d_date_sk#43, d_year#44, d_moy#45]
+
+(55) Filter [codegen id : 16]
+Input [3]: [d_date_sk#43, d_year#44, d_moy#45]
+Condition : ((((isnotnull(d_year#44) AND isnotnull(d_moy#45)) AND (d_year#44 = 2000)) AND (d_moy#45 = 2)) AND isnotnull(d_date_sk#43))
+
+(56) Project [codegen id : 16]
+Output [1]: [d_date_sk#43]
+Input [3]: [d_date_sk#43, d_year#44, d_moy#45]
+
+(57) BroadcastExchange
+Input [1]: [d_date_sk#43]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#46]
+
+(58) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [cs_sold_date_sk#5]
+Right keys [1]: [d_date_sk#43]
+Join condition: None
+
+(59) Project [codegen id : 17]
+Output [3]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4]
+Input [5]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, d_date_sk#43]
+
+(60) Scan parquet default.customer
+Output [3]: [c_customer_sk#47, c_first_name#48, c_last_name#49]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_first_name:string,c_last_name:string>
+
+(61) ColumnarToRow [codegen id : 18]
+Input [3]: [c_customer_sk#47, c_first_name#48, c_last_name#49]
+
+(62) Filter [codegen id : 18]
+Input [3]: [c_customer_sk#47, c_first_name#48, c_last_name#49]
+Condition : isnotnull(c_customer_sk#47)
+
+(63) Exchange
+Input [3]: [c_customer_sk#47, c_first_name#48, c_last_name#49]
+Arguments: hashpartitioning(c_customer_sk#47, 5), ENSURE_REQUIREMENTS, [id=#50]
+
+(64) Sort [codegen id : 19]
+Input [3]: [c_customer_sk#47, c_first_name#48, c_last_name#49]
+Arguments: [c_customer_sk#47 ASC NULLS FIRST], false, 0
+
+(65) ReusedExchange [Reuses operator id: 38]
+Output [3]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28]
+
+(66) Sort [codegen id : 21]
+Input [3]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28]
+Arguments: [ss_customer_sk#26 ASC NULLS FIRST], false, 0
+
+(67) ReusedExchange [Reuses operator id: 43]
+Output [1]: [c_customer_sk#31]
+
+(68) Sort [codegen id : 23]
+Input [1]: [c_customer_sk#31]
+Arguments: [c_customer_sk#31 ASC NULLS FIRST], false, 0
+
+(69) SortMergeJoin [codegen id : 24]
+Left keys [1]: [ss_customer_sk#26]
+Right keys [1]: [c_customer_sk#31]
+Join condition: None
+
+(70) Project [codegen id : 24]
+Output [3]: [ss_quantity#27, ss_sales_price#28, c_customer_sk#31]
+Input [4]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28, c_customer_sk#31]
+
+(71) HashAggregate [codegen id : 24]
+Input [3]: [ss_quantity#27, ss_sales_price#28, c_customer_sk#31]
+Keys [1]: [c_customer_sk#31]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#27 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#28 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [2]: [sum#33, isEmpty#34]
+Results [3]: [c_customer_sk#31, sum#35, isEmpty#36]
+
+(72) HashAggregate [codegen id : 24]
+Input [3]: [c_customer_sk#31, sum#35, isEmpty#36]
+Keys [1]: [c_customer_sk#31]
+Functions [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#27 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#28 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#27 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#28 as decimal(12,2)))), DecimalType(18,2), true))#37]
+Results [2]: [c_customer_sk#31, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#27 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#28 as decimal(12,2)))), DecimalType(18,2), true))#37 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40]
+
+(73) Filter [codegen id : 24]
+Input [2]: [c_customer_sk#31, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40 as decimal(38,8)) > CheckOverflow((0.500000 * promote_precision(cast(ReusedSubquery Subquery scalar-subquery#41, [id=#42] as decimal(32,6)))), DecimalType(38,8), true)))
+
+(74) Project [codegen id : 24]
+Output [1]: [c_customer_sk#31]
+Input [2]: [c_customer_sk#31, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40]
+
+(75) Sort [codegen id : 24]
+Input [1]: [c_customer_sk#31]
+Arguments: [c_customer_sk#31 ASC NULLS FIRST], false, 0
+
+(76) SortMergeJoin [codegen id : 25]
+Left keys [1]: [c_customer_sk#47]
+Right keys [1]: [c_customer_sk#31]
+Join condition: None
+
+(77) SortMergeJoin [codegen id : 26]
+Left keys [1]: [cs_bill_customer_sk#1]
+Right keys [1]: [c_customer_sk#47]
+Join condition: None
+
+(78) Project [codegen id : 26]
+Output [4]: [cs_quantity#3, cs_list_price#4, c_first_name#48, c_last_name#49]
+Input [6]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, c_customer_sk#47, c_first_name#48, c_last_name#49]
+
+(79) HashAggregate [codegen id : 26]
+Input [4]: [cs_quantity#3, cs_list_price#4, c_first_name#48, c_last_name#49]
+Keys [2]: [c_last_name#49, c_first_name#48]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#3 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#4 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [2]: [sum#51, isEmpty#52]
+Results [4]: [c_last_name#49, c_first_name#48, sum#53, isEmpty#54]
+
+(80) Exchange
+Input [4]: [c_last_name#49, c_first_name#48, sum#53, isEmpty#54]
+Arguments: hashpartitioning(c_last_name#49, c_first_name#48, 5), ENSURE_REQUIREMENTS, [id=#55]
+
+(81) HashAggregate [codegen id : 27]
+Input [4]: [c_last_name#49, c_first_name#48, sum#53, isEmpty#54]
+Keys [2]: [c_last_name#49, c_first_name#48]
+Functions [1]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#3 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#4 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#3 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#4 as decimal(12,2)))), DecimalType(18,2), true))#56]
+Results [3]: [c_last_name#49, c_first_name#48, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#3 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#4 as decimal(12,2)))), DecimalType(18,2), true))#56 AS sales#57]
+
+(82) Scan parquet default.web_sales
+Output [5]: [ws_item_sk#58, ws_bill_customer_sk#59, ws_quantity#60, ws_list_price#61, ws_sold_date_sk#62]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#62), dynamicpruningexpression(ws_sold_date_sk#62 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_bill_customer_sk:int,ws_quantity:int,ws_list_price:decimal(7,2)>
+
+(83) ColumnarToRow [codegen id : 28]
+Input [5]: [ws_item_sk#58, ws_bill_customer_sk#59, ws_quantity#60, ws_list_price#61, ws_sold_date_sk#62]
+
+(84) Filter [codegen id : 28]
+Input [5]: [ws_item_sk#58, ws_bill_customer_sk#59, ws_quantity#60, ws_list_price#61, ws_sold_date_sk#62]
+Condition : isnotnull(ws_bill_customer_sk#59)
+
+(85) Exchange
+Input [5]: [ws_item_sk#58, ws_bill_customer_sk#59, ws_quantity#60, ws_list_price#61, ws_sold_date_sk#62]
+Arguments: hashpartitioning(ws_item_sk#58, 5), ENSURE_REQUIREMENTS, [id=#63]
+
+(86) Sort [codegen id : 29]
+Input [5]: [ws_item_sk#58, ws_bill_customer_sk#59, ws_quantity#60, ws_list_price#61, ws_sold_date_sk#62]
+Arguments: [ws_item_sk#58 ASC NULLS FIRST], false, 0
+
+(87) ReusedExchange [Reuses operator id: unknown]
+Output [2]: [ss_item_sk#64, d_date#65]
+
+(88) Sort [codegen id : 32]
+Input [2]: [ss_item_sk#64, d_date#65]
+Arguments: [ss_item_sk#64 ASC NULLS FIRST], false, 0
+
+(89) ReusedExchange [Reuses operator id: 21]
+Output [2]: [i_item_sk#66, i_item_desc#67]
+
+(90) Sort [codegen id : 34]
+Input [2]: [i_item_sk#66, i_item_desc#67]
+Arguments: [i_item_sk#66 ASC NULLS FIRST], false, 0
+
+(91) SortMergeJoin [codegen id : 35]
+Left keys [1]: [ss_item_sk#64]
+Right keys [1]: [i_item_sk#66]
+Join condition: None
+
+(92) Project [codegen id : 35]
+Output [3]: [d_date#65, i_item_sk#66, substr(i_item_desc#67, 1, 30) AS _groupingexpression#68]
+Input [4]: [ss_item_sk#64, d_date#65, i_item_sk#66, i_item_desc#67]
+
+(93) HashAggregate [codegen id : 35]
+Input [3]: [d_date#65, i_item_sk#66, _groupingexpression#68]
+Keys [3]: [_groupingexpression#68, i_item_sk#66, d_date#65]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#69]
+Results [4]: [_groupingexpression#68, i_item_sk#66, d_date#65, count#70]
+
+(94) HashAggregate [codegen id : 35]
+Input [4]: [_groupingexpression#68, i_item_sk#66, d_date#65, count#70]
+Keys [3]: [_groupingexpression#68, i_item_sk#66, d_date#65]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#71]
+Results [2]: [i_item_sk#66 AS item_sk#23, count(1)#71 AS count(1)#72]
+
+(95) Filter [codegen id : 35]
+Input [2]: [item_sk#23, count(1)#72]
+Condition : (count(1)#72 > 4)
+
+(96) Project [codegen id : 35]
+Output [1]: [item_sk#23]
+Input [2]: [item_sk#23, count(1)#72]
+
+(97) Sort [codegen id : 35]
+Input [1]: [item_sk#23]
+Arguments: [item_sk#23 ASC NULLS FIRST], false, 0
+
+(98) SortMergeJoin [codegen id : 36]
+Left keys [1]: [ws_item_sk#58]
+Right keys [1]: [item_sk#23]
+Join condition: None
+
+(99) Project [codegen id : 36]
+Output [4]: [ws_bill_customer_sk#59, ws_quantity#60, ws_list_price#61, ws_sold_date_sk#62]
+Input [5]: [ws_item_sk#58, ws_bill_customer_sk#59, ws_quantity#60, ws_list_price#61, ws_sold_date_sk#62]
+
+(100) Exchange
+Input [4]: [ws_bill_customer_sk#59, ws_quantity#60, ws_list_price#61, ws_sold_date_sk#62]
+Arguments: hashpartitioning(ws_bill_customer_sk#59, 5), ENSURE_REQUIREMENTS, [id=#73]
+
+(101) Sort [codegen id : 37]
+Input [4]: [ws_bill_customer_sk#59, ws_quantity#60, ws_list_price#61, ws_sold_date_sk#62]
+Arguments: [ws_bill_customer_sk#59 ASC NULLS FIRST], false, 0
+
+(102) ReusedExchange [Reuses operator id: 38]
+Output [3]: [ss_customer_sk#74, ss_quantity#75, ss_sales_price#76]
+
+(103) Sort [codegen id : 39]
+Input [3]: [ss_customer_sk#74, ss_quantity#75, ss_sales_price#76]
+Arguments: [ss_customer_sk#74 ASC NULLS FIRST], false, 0
+
+(104) ReusedExchange [Reuses operator id: 43]
+Output [1]: [c_customer_sk#77]
+
+(105) Sort [codegen id : 41]
+Input [1]: [c_customer_sk#77]
+Arguments: [c_customer_sk#77 ASC NULLS FIRST], false, 0
+
+(106) SortMergeJoin [codegen id : 42]
+Left keys [1]: [ss_customer_sk#74]
+Right keys [1]: [c_customer_sk#77]
+Join condition: None
+
+(107) Project [codegen id : 42]
+Output [3]: [ss_quantity#75, ss_sales_price#76, c_customer_sk#77]
+Input [4]: [ss_customer_sk#74, ss_quantity#75, ss_sales_price#76, c_customer_sk#77]
+
+(108) HashAggregate [codegen id : 42]
+Input [3]: [ss_quantity#75, ss_sales_price#76, c_customer_sk#77]
+Keys [1]: [c_customer_sk#77]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#76 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [2]: [sum#78, isEmpty#79]
+Results [3]: [c_customer_sk#77, sum#80, isEmpty#81]
+
+(109) HashAggregate [codegen id : 42]
+Input [3]: [c_customer_sk#77, sum#80, isEmpty#81]
+Keys [1]: [c_customer_sk#77]
+Functions [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#76 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#76 as decimal(12,2)))), DecimalType(18,2), true))#82]
+Results [2]: [c_customer_sk#77, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#76 as decimal(12,2)))), DecimalType(18,2), true))#82 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#83]
+
+(110) Filter [codegen id : 42]
+Input [2]: [c_customer_sk#77, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#83]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#83) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#83 as decimal(38,8)) > CheckOverflow((0.500000 * promote_precision(cast(ReusedSubquery Subquery scalar-subquery#41, [id=#42] as decimal(32,6)))), DecimalType(38,8), true)))
+
+(111) Project [codegen id : 42]
+Output [1]: [c_customer_sk#77]
+Input [2]: [c_customer_sk#77, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#83]
+
+(112) Sort [codegen id : 42]
+Input [1]: [c_customer_sk#77]
+Arguments: [c_customer_sk#77 ASC NULLS FIRST], false, 0
+
+(113) SortMergeJoin [codegen id : 44]
+Left keys [1]: [ws_bill_customer_sk#59]
+Right keys [1]: [c_customer_sk#77]
+Join condition: None
+
+(114) ReusedExchange [Reuses operator id: 57]
+Output [1]: [d_date_sk#84]
+
+(115) BroadcastHashJoin [codegen id : 44]
+Left keys [1]: [ws_sold_date_sk#62]
+Right keys [1]: [d_date_sk#84]
+Join condition: None
+
+(116) Project [codegen id : 44]
+Output [3]: [ws_bill_customer_sk#59, ws_quantity#60, ws_list_price#61]
+Input [5]: [ws_bill_customer_sk#59, ws_quantity#60, ws_list_price#61, ws_sold_date_sk#62, d_date_sk#84]
+
+(117) ReusedExchange [Reuses operator id: 63]
+Output [3]: [c_customer_sk#85, c_first_name#86, c_last_name#87]
+
+(118) Sort [codegen id : 46]
+Input [3]: [c_customer_sk#85, c_first_name#86, c_last_name#87]
+Arguments: [c_customer_sk#85 ASC NULLS FIRST], false, 0
+
+(119) ReusedExchange [Reuses operator id: 38]
+Output [3]: [ss_customer_sk#74, ss_quantity#75, ss_sales_price#76]
+
+(120) Sort [codegen id : 48]
+Input [3]: [ss_customer_sk#74, ss_quantity#75, ss_sales_price#76]
+Arguments: [ss_customer_sk#74 ASC NULLS FIRST], false, 0
+
+(121) ReusedExchange [Reuses operator id: 43]
+Output [1]: [c_customer_sk#77]
+
+(122) Sort [codegen id : 50]
+Input [1]: [c_customer_sk#77]
+Arguments: [c_customer_sk#77 ASC NULLS FIRST], false, 0
+
+(123) SortMergeJoin [codegen id : 51]
+Left keys [1]: [ss_customer_sk#74]
+Right keys [1]: [c_customer_sk#77]
+Join condition: None
+
+(124) Project [codegen id : 51]
+Output [3]: [ss_quantity#75, ss_sales_price#76, c_customer_sk#77]
+Input [4]: [ss_customer_sk#74, ss_quantity#75, ss_sales_price#76, c_customer_sk#77]
+
+(125) HashAggregate [codegen id : 51]
+Input [3]: [ss_quantity#75, ss_sales_price#76, c_customer_sk#77]
+Keys [1]: [c_customer_sk#77]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#76 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [2]: [sum#78, isEmpty#79]
+Results [3]: [c_customer_sk#77, sum#80, isEmpty#81]
+
+(126) HashAggregate [codegen id : 51]
+Input [3]: [c_customer_sk#77, sum#80, isEmpty#81]
+Keys [1]: [c_customer_sk#77]
+Functions [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#76 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#76 as decimal(12,2)))), DecimalType(18,2), true))#82]
+Results [2]: [c_customer_sk#77, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#76 as decimal(12,2)))), DecimalType(18,2), true))#82 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#83]
+
+(127) Filter [codegen id : 51]
+Input [2]: [c_customer_sk#77, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#83]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#83) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#83 as decimal(38,8)) > CheckOverflow((0.500000 * promote_precision(cast(ReusedSubquery Subquery scalar-subquery#41, [id=#42] as decimal(32,6)))), DecimalType(38,8), true)))
+
+(128) Project [codegen id : 51]
+Output [1]: [c_customer_sk#77]
+Input [2]: [c_customer_sk#77, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#83]
+
+(129) Sort [codegen id : 51]
+Input [1]: [c_customer_sk#77]
+Arguments: [c_customer_sk#77 ASC NULLS FIRST], false, 0
+
+(130) SortMergeJoin [codegen id : 52]
+Left keys [1]: [c_customer_sk#85]
+Right keys [1]: [c_customer_sk#77]
+Join condition: None
+
+(131) SortMergeJoin [codegen id : 53]
+Left keys [1]: [ws_bill_customer_sk#59]
+Right keys [1]: [c_customer_sk#85]
+Join condition: None
+
+(132) Project [codegen id : 53]
+Output [4]: [ws_quantity#60, ws_list_price#61, c_first_name#86, c_last_name#87]
+Input [6]: [ws_bill_customer_sk#59, ws_quantity#60, ws_list_price#61, c_customer_sk#85, c_first_name#86, c_last_name#87]
+
+(133) HashAggregate [codegen id : 53]
+Input [4]: [ws_quantity#60, ws_list_price#61, c_first_name#86, c_last_name#87]
+Keys [2]: [c_last_name#87, c_first_name#86]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#60 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#61 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [2]: [sum#88, isEmpty#89]
+Results [4]: [c_last_name#87, c_first_name#86, sum#90, isEmpty#91]
+
+(134) Exchange
+Input [4]: [c_last_name#87, c_first_name#86, sum#90, isEmpty#91]
+Arguments: hashpartitioning(c_last_name#87, c_first_name#86, 5), ENSURE_REQUIREMENTS, [id=#92]
+
+(135) HashAggregate [codegen id : 54]
+Input [4]: [c_last_name#87, c_first_name#86, sum#90, isEmpty#91]
+Keys [2]: [c_last_name#87, c_first_name#86]
+Functions [1]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#60 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#61 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#60 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#61 as decimal(12,2)))), DecimalType(18,2), true))#93]
+Results [3]: [c_last_name#87, c_first_name#86, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#60 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#61 as decimal(12,2)))), DecimalType(18,2), true))#93 AS sales#94]
+
+(136) Union
+
+(137) TakeOrderedAndProject
+Input [3]: [c_last_name#49, c_first_name#48, sales#57]
+Arguments: 100, [c_last_name#49 ASC NULLS FIRST, c_first_name#48 ASC NULLS FIRST, sales#57 ASC NULLS FIRST], [c_last_name#49, c_first_name#48, sales#57]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (138)
+
+
+(138) ReusedExchange [Reuses operator id: 57]
+Output [1]: [d_date_sk#43]
+
+Subquery:2 Hosting operator id = 6 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10
+ReusedExchange (139)
+
+
+(139) ReusedExchange [Reuses operator id: 13]
+Output [2]: [d_date_sk#11, d_date#12]
+
+Subquery:3 Hosting operator id = 49 Hosting Expression = Subquery scalar-subquery#41, [id=#42]
+* HashAggregate (163)
++- Exchange (162)
+   +- * HashAggregate (161)
+      +- * HashAggregate (160)
+         +- * HashAggregate (159)
+            +- * Project (158)
+               +- * SortMergeJoin Inner (157)
+                  :- * Sort (151)
+                  :  +- Exchange (150)
+                  :     +- * Project (149)
+                  :        +- * BroadcastHashJoin Inner BuildRight (148)
+                  :           :- * Filter (142)
+                  :           :  +- * ColumnarToRow (141)
+                  :           :     +- Scan parquet default.store_sales (140)
+                  :           +- BroadcastExchange (147)
+                  :              +- * Project (146)
+                  :                 +- * Filter (145)
+                  :                    +- * ColumnarToRow (144)
+                  :                       +- Scan parquet default.date_dim (143)
+                  +- * Sort (156)
+                     +- Exchange (155)
+                        +- * Filter (154)
+                           +- * ColumnarToRow (153)
+                              +- Scan parquet default.customer (152)
+
+
+(140) Scan parquet default.store_sales
+Output [4]: [ss_customer_sk#95, ss_quantity#96, ss_sales_price#97, ss_sold_date_sk#98]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#98), dynamicpruningexpression(ss_sold_date_sk#98 IN dynamicpruning#99)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_quantity:int,ss_sales_price:decimal(7,2)>
+
+(141) ColumnarToRow [codegen id : 2]
+Input [4]: [ss_customer_sk#95, ss_quantity#96, ss_sales_price#97, ss_sold_date_sk#98]
+
+(142) Filter [codegen id : 2]
+Input [4]: [ss_customer_sk#95, ss_quantity#96, ss_sales_price#97, ss_sold_date_sk#98]
+Condition : isnotnull(ss_customer_sk#95)
+
+(143) Scan parquet default.date_dim
+Output [2]: [d_date_sk#100, d_year#101]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(144) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#100, d_year#101]
+
+(145) Filter [codegen id : 1]
+Input [2]: [d_date_sk#100, d_year#101]
+Condition : (d_year#101 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#100))
+
+(146) Project [codegen id : 1]
+Output [1]: [d_date_sk#100]
+Input [2]: [d_date_sk#100, d_year#101]
+
+(147) BroadcastExchange
+Input [1]: [d_date_sk#100]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#102]
+
+(148) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#98]
+Right keys [1]: [d_date_sk#100]
+Join condition: None
+
+(149) Project [codegen id : 2]
+Output [3]: [ss_customer_sk#95, ss_quantity#96, ss_sales_price#97]
+Input [5]: [ss_customer_sk#95, ss_quantity#96, ss_sales_price#97, ss_sold_date_sk#98, d_date_sk#100]
+
+(150) Exchange
+Input [3]: [ss_customer_sk#95, ss_quantity#96, ss_sales_price#97]
+Arguments: hashpartitioning(ss_customer_sk#95, 5), ENSURE_REQUIREMENTS, [id=#103]
+
+(151) Sort [codegen id : 3]
+Input [3]: [ss_customer_sk#95, ss_quantity#96, ss_sales_price#97]
+Arguments: [ss_customer_sk#95 ASC NULLS FIRST], false, 0
+
+(152) Scan parquet default.customer
+Output [1]: [c_customer_sk#104]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int>
+
+(153) ColumnarToRow [codegen id : 4]
+Input [1]: [c_customer_sk#104]
+
+(154) Filter [codegen id : 4]
+Input [1]: [c_customer_sk#104]
+Condition : isnotnull(c_customer_sk#104)
+
+(155) Exchange
+Input [1]: [c_customer_sk#104]
+Arguments: hashpartitioning(c_customer_sk#104, 5), ENSURE_REQUIREMENTS, [id=#105]
+
+(156) Sort [codegen id : 5]
+Input [1]: [c_customer_sk#104]
+Arguments: [c_customer_sk#104 ASC NULLS FIRST], false, 0
+
+(157) SortMergeJoin [codegen id : 6]
+Left keys [1]: [ss_customer_sk#95]
+Right keys [1]: [c_customer_sk#104]
+Join condition: None
+
+(158) Project [codegen id : 6]
+Output [3]: [ss_quantity#96, ss_sales_price#97, c_customer_sk#104]
+Input [4]: [ss_customer_sk#95, ss_quantity#96, ss_sales_price#97, c_customer_sk#104]
+
+(159) HashAggregate [codegen id : 6]
+Input [3]: [ss_quantity#96, ss_sales_price#97, c_customer_sk#104]
+Keys [1]: [c_customer_sk#104]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#96 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#97 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [2]: [sum#106, isEmpty#107]
+Results [3]: [c_customer_sk#104, sum#108, isEmpty#109]
+
+(160) HashAggregate [codegen id : 6]
+Input [3]: [c_customer_sk#104, sum#108, isEmpty#109]
+Keys [1]: [c_customer_sk#104]
+Functions [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#96 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#97 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#96 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#97 as decimal(12,2)))), DecimalType(18,2), true))#110]
+Results [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#96 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#97 as decimal(12,2)))), DecimalType(18,2), true))#110 AS csales#111]
+
+(161) HashAggregate [codegen id : 6]
+Input [1]: [csales#111]
+Keys: []
+Functions [1]: [partial_max(csales#111)]
+Aggregate Attributes [1]: [max#112]
+Results [1]: [max#113]
+
+(162) Exchange
+Input [1]: [max#113]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#114]
+
+(163) HashAggregate [codegen id : 7]
+Input [1]: [max#113]
+Keys: []
+Functions [1]: [max(csales#111)]
+Aggregate Attributes [1]: [max(csales#111)#115]
+Results [1]: [max(csales#111)#115 AS tpcds_cmax#116]
+
+Subquery:4 Hosting operator id = 140 Hosting Expression = ss_sold_date_sk#98 IN dynamicpruning#99
+ReusedExchange (164)
+
+
+(164) ReusedExchange [Reuses operator id: 147]
+Output [1]: [d_date_sk#100]
+
+Subquery:5 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#41, [id=#42]
+
+Subquery:6 Hosting operator id = 82 Hosting Expression = ws_sold_date_sk#62 IN dynamicpruning#6
+
+Subquery:7 Hosting operator id = 110 Hosting Expression = ReusedSubquery Subquery scalar-subquery#41, [id=#42]
+
+Subquery:8 Hosting operator id = 127 Hosting Expression = ReusedSubquery Subquery scalar-subquery#41, [id=#42]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.sf100/simplified.txt
new file mode 100644
index 0000000000000..98848b4ed0676
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.sf100/simplified.txt
@@ -0,0 +1,291 @@
+TakeOrderedAndProject [c_last_name,c_first_name,sales]
+  Union
+    WholeStageCodegen (27)
+      HashAggregate [c_last_name,c_first_name,sum,isEmpty] [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),sales,sum,isEmpty]
+        InputAdapter
+          Exchange [c_last_name,c_first_name] #1
+            WholeStageCodegen (26)
+              HashAggregate [c_last_name,c_first_name,cs_quantity,cs_list_price] [sum,isEmpty,sum,isEmpty]
+                Project [cs_quantity,cs_list_price,c_first_name,c_last_name]
+                  SortMergeJoin [cs_bill_customer_sk,c_customer_sk]
+                    InputAdapter
+                      WholeStageCodegen (17)
+                        Project [cs_bill_customer_sk,cs_quantity,cs_list_price]
+                          BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                            SortMergeJoin [cs_bill_customer_sk,c_customer_sk]
+                              InputAdapter
+                                WholeStageCodegen (10)
+                                  Sort [cs_bill_customer_sk]
+                                    InputAdapter
+                                      Exchange [cs_bill_customer_sk] #2
+                                        WholeStageCodegen (9)
+                                          Project [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk]
+                                            SortMergeJoin [cs_item_sk,item_sk]
+                                              InputAdapter
+                                                WholeStageCodegen (2)
+                                                  Sort [cs_item_sk]
+                                                    InputAdapter
+                                                      Exchange [cs_item_sk] #3
+                                                        WholeStageCodegen (1)
+                                                          Filter [cs_bill_customer_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk]
+                                                                  SubqueryBroadcast [d_date_sk] #1
+                                                                    ReusedExchange [d_date_sk] #4
+                                              InputAdapter
+                                                WholeStageCodegen (8)
+                                                  Sort [item_sk]
+                                                    Project [item_sk]
+                                                      Filter [count(1)]
+                                                        HashAggregate [_groupingexpression,i_item_sk,d_date,count] [count(1),item_sk,count(1),count]
+                                                          HashAggregate [_groupingexpression,i_item_sk,d_date] [count,count]
+                                                            Project [d_date,i_item_sk,i_item_desc]
+                                                              SortMergeJoin [ss_item_sk,i_item_sk]
+                                                                InputAdapter
+                                                                  WholeStageCodegen (5)
+                                                                    Sort [ss_item_sk]
+                                                                      InputAdapter
+                                                                        Exchange [ss_item_sk] #5
+                                                                          WholeStageCodegen (4)
+                                                                            Project [ss_item_sk,d_date]
+                                                                              BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                Filter [ss_item_sk]
+                                                                                  ColumnarToRow
+                                                                                    InputAdapter
+                                                                                      Scan parquet default.store_sales [ss_item_sk,ss_sold_date_sk]
+                                                                                        SubqueryBroadcast [d_date_sk] #2
+                                                                                          ReusedExchange [d_date_sk,d_date] #6
+                                                                                InputAdapter
+                                                                                  BroadcastExchange #6
+                                                                                    WholeStageCodegen (3)
+                                                                                      Project [d_date_sk,d_date]
+                                                                                        Filter [d_year,d_date_sk]
+                                                                                          ColumnarToRow
+                                                                                            InputAdapter
+                                                                                              Scan parquet default.date_dim [d_date_sk,d_date,d_year]
+                                                                InputAdapter
+                                                                  WholeStageCodegen (7)
+                                                                    Sort [i_item_sk]
+                                                                      InputAdapter
+                                                                        Exchange [i_item_sk] #7
+                                                                          WholeStageCodegen (6)
+                                                                            Filter [i_item_sk]
+                                                                              ColumnarToRow
+                                                                                InputAdapter
+                                                                                  Scan parquet default.item [i_item_sk,i_item_desc]
+                              InputAdapter
+                                WholeStageCodegen (15)
+                                  Sort [c_customer_sk]
+                                    Project [c_customer_sk]
+                                      Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                        Subquery #3
+                                          WholeStageCodegen (7)
+                                            HashAggregate [max] [max(csales),tpcds_cmax,max]
+                                              InputAdapter
+                                                Exchange #10
+                                                  WholeStageCodegen (6)
+                                                    HashAggregate [csales] [max,max]
+                                                      HashAggregate [c_customer_sk,sum,isEmpty] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),csales,sum,isEmpty]
+                                                        HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty]
+                                                          Project [ss_quantity,ss_sales_price,c_customer_sk]
+                                                            SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                                              InputAdapter
+                                                                WholeStageCodegen (3)
+                                                                  Sort [ss_customer_sk]
+                                                                    InputAdapter
+                                                                      Exchange [ss_customer_sk] #11
+                                                                        WholeStageCodegen (2)
+                                                                          Project [ss_customer_sk,ss_quantity,ss_sales_price]
+                                                                            BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                              Filter [ss_customer_sk]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk]
+                                                                                      SubqueryBroadcast [d_date_sk] #4
+                                                                                        ReusedExchange [d_date_sk] #12
+                                                                              InputAdapter
+                                                                                BroadcastExchange #12
+                                                                                  WholeStageCodegen (1)
+                                                                                    Project [d_date_sk]
+                                                                                      Filter [d_year,d_date_sk]
+                                                                                        ColumnarToRow
+                                                                                          InputAdapter
+                                                                                            Scan parquet default.date_dim [d_date_sk,d_year]
+                                                              InputAdapter
+                                                                WholeStageCodegen (5)
+                                                                  Sort [c_customer_sk]
+                                                                    InputAdapter
+                                                                      Exchange [c_customer_sk] #13
+                                                                        WholeStageCodegen (4)
+                                                                          Filter [c_customer_sk]
+                                                                            ColumnarToRow
+                                                                              InputAdapter
+                                                                                Scan parquet default.customer [c_customer_sk]
+                                        HashAggregate [c_customer_sk,sum,isEmpty] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty]
+                                          HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty]
+                                            Project [ss_quantity,ss_sales_price,c_customer_sk]
+                                              SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                                InputAdapter
+                                                  WholeStageCodegen (12)
+                                                    Sort [ss_customer_sk]
+                                                      InputAdapter
+                                                        Exchange [ss_customer_sk] #8
+                                                          WholeStageCodegen (11)
+                                                            Project [ss_customer_sk,ss_quantity,ss_sales_price]
+                                                              Filter [ss_customer_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk]
+                                                InputAdapter
+                                                  WholeStageCodegen (14)
+                                                    Sort [c_customer_sk]
+                                                      InputAdapter
+                                                        Exchange [c_customer_sk] #9
+                                                          WholeStageCodegen (13)
+                                                            Filter [c_customer_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.customer [c_customer_sk]
+                            InputAdapter
+                              BroadcastExchange #4
+                                WholeStageCodegen (16)
+                                  Project [d_date_sk]
+                                    Filter [d_year,d_moy,d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                    InputAdapter
+                      WholeStageCodegen (25)
+                        SortMergeJoin [c_customer_sk,c_customer_sk]
+                          InputAdapter
+                            WholeStageCodegen (19)
+                              Sort [c_customer_sk]
+                                InputAdapter
+                                  Exchange [c_customer_sk] #14
+                                    WholeStageCodegen (18)
+                                      Filter [c_customer_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.customer [c_customer_sk,c_first_name,c_last_name]
+                          InputAdapter
+                            WholeStageCodegen (24)
+                              Sort [c_customer_sk]
+                                Project [c_customer_sk]
+                                  Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                    ReusedSubquery [tpcds_cmax] #3
+                                    HashAggregate [c_customer_sk,sum,isEmpty] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty]
+                                      HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty]
+                                        Project [ss_quantity,ss_sales_price,c_customer_sk]
+                                          SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                            InputAdapter
+                                              WholeStageCodegen (21)
+                                                Sort [ss_customer_sk]
+                                                  InputAdapter
+                                                    ReusedExchange [ss_customer_sk,ss_quantity,ss_sales_price] #8
+                                            InputAdapter
+                                              WholeStageCodegen (23)
+                                                Sort [c_customer_sk]
+                                                  InputAdapter
+                                                    ReusedExchange [c_customer_sk] #9
+    WholeStageCodegen (54)
+      HashAggregate [c_last_name,c_first_name,sum,isEmpty] [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),sales,sum,isEmpty]
+        InputAdapter
+          Exchange [c_last_name,c_first_name] #15
+            WholeStageCodegen (53)
+              HashAggregate [c_last_name,c_first_name,ws_quantity,ws_list_price] [sum,isEmpty,sum,isEmpty]
+                Project [ws_quantity,ws_list_price,c_first_name,c_last_name]
+                  SortMergeJoin [ws_bill_customer_sk,c_customer_sk]
+                    InputAdapter
+                      WholeStageCodegen (44)
+                        Project [ws_bill_customer_sk,ws_quantity,ws_list_price]
+                          BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                            SortMergeJoin [ws_bill_customer_sk,c_customer_sk]
+                              InputAdapter
+                                WholeStageCodegen (37)
+                                  Sort [ws_bill_customer_sk]
+                                    InputAdapter
+                                      Exchange [ws_bill_customer_sk] #16
+                                        WholeStageCodegen (36)
+                                          Project [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk]
+                                            SortMergeJoin [ws_item_sk,item_sk]
+                                              InputAdapter
+                                                WholeStageCodegen (29)
+                                                  Sort [ws_item_sk]
+                                                    InputAdapter
+                                                      Exchange [ws_item_sk] #17
+                                                        WholeStageCodegen (28)
+                                                          Filter [ws_bill_customer_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk]
+                                                                  ReusedSubquery [d_date_sk] #1
+                                              InputAdapter
+                                                WholeStageCodegen (35)
+                                                  Sort [item_sk]
+                                                    Project [item_sk]
+                                                      Filter [count(1)]
+                                                        HashAggregate [_groupingexpression,i_item_sk,d_date,count] [count(1),item_sk,count(1),count]
+                                                          HashAggregate [_groupingexpression,i_item_sk,d_date] [count,count]
+                                                            Project [d_date,i_item_sk,i_item_desc]
+                                                              SortMergeJoin [ss_item_sk,i_item_sk]
+                                                                InputAdapter
+                                                                  WholeStageCodegen (32)
+                                                                    Sort [ss_item_sk]
+                                                                      InputAdapter
+                                                                        ReusedExchange [ss_item_sk,d_date] #18
+                                                                InputAdapter
+                                                                  WholeStageCodegen (34)
+                                                                    Sort [i_item_sk]
+                                                                      InputAdapter
+                                                                        ReusedExchange [i_item_sk,i_item_desc] #7
+                              InputAdapter
+                                WholeStageCodegen (42)
+                                  Sort [c_customer_sk]
+                                    Project [c_customer_sk]
+                                      Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                        ReusedSubquery [tpcds_cmax] #3
+                                        HashAggregate [c_customer_sk,sum,isEmpty] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty]
+                                          HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty]
+                                            Project [ss_quantity,ss_sales_price,c_customer_sk]
+                                              SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                                InputAdapter
+                                                  WholeStageCodegen (39)
+                                                    Sort [ss_customer_sk]
+                                                      InputAdapter
+                                                        ReusedExchange [ss_customer_sk,ss_quantity,ss_sales_price] #8
+                                                InputAdapter
+                                                  WholeStageCodegen (41)
+                                                    Sort [c_customer_sk]
+                                                      InputAdapter
+                                                        ReusedExchange [c_customer_sk] #9
+                            InputAdapter
+                              ReusedExchange [d_date_sk] #4
+                    InputAdapter
+                      WholeStageCodegen (52)
+                        SortMergeJoin [c_customer_sk,c_customer_sk]
+                          InputAdapter
+                            WholeStageCodegen (46)
+                              Sort [c_customer_sk]
+                                InputAdapter
+                                  ReusedExchange [c_customer_sk,c_first_name,c_last_name] #14
+                          InputAdapter
+                            WholeStageCodegen (51)
+                              Sort [c_customer_sk]
+                                Project [c_customer_sk]
+                                  Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                    ReusedSubquery [tpcds_cmax] #3
+                                    HashAggregate [c_customer_sk,sum,isEmpty] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty]
+                                      HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty]
+                                        Project [ss_quantity,ss_sales_price,c_customer_sk]
+                                          SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                            InputAdapter
+                                              WholeStageCodegen (48)
+                                                Sort [ss_customer_sk]
+                                                  InputAdapter
+                                                    ReusedExchange [ss_customer_sk,ss_quantity,ss_sales_price] #8
+                                            InputAdapter
+                                              WholeStageCodegen (50)
+                                                Sort [c_customer_sk]
+                                                  InputAdapter
+                                                    ReusedExchange [c_customer_sk] #9
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt
new file mode 100644
index 0000000000000..a7d797dbb545e
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt
@@ -0,0 +1,743 @@
+== Physical Plan ==
+TakeOrderedAndProject (103)
++- Union (102)
+   :- * HashAggregate (70)
+   :  +- Exchange (69)
+   :     +- * HashAggregate (68)
+   :        +- * Project (67)
+   :           +- * BroadcastHashJoin Inner BuildRight (66)
+   :              :- * Project (60)
+   :              :  +- * BroadcastHashJoin Inner BuildRight (59)
+   :              :     :- * SortMergeJoin LeftSemi (46)
+   :              :     :  :- * Sort (29)
+   :              :     :  :  +- Exchange (28)
+   :              :     :  :     +- * Project (27)
+   :              :     :  :        +- * BroadcastHashJoin LeftSemi BuildRight (26)
+   :              :     :  :           :- * Filter (3)
+   :              :     :  :           :  +- * ColumnarToRow (2)
+   :              :     :  :           :     +- Scan parquet default.catalog_sales (1)
+   :              :     :  :           +- BroadcastExchange (25)
+   :              :     :  :              +- * Project (24)
+   :              :     :  :                 +- * Filter (23)
+   :              :     :  :                    +- * HashAggregate (22)
+   :              :     :  :                       +- Exchange (21)
+   :              :     :  :                          +- * HashAggregate (20)
+   :              :     :  :                             +- * Project (19)
+   :              :     :  :                                +- * BroadcastHashJoin Inner BuildRight (18)
+   :              :     :  :                                   :- * Project (13)
+   :              :     :  :                                   :  +- * BroadcastHashJoin Inner BuildRight (12)
+   :              :     :  :                                   :     :- * Filter (6)
+   :              :     :  :                                   :     :  +- * ColumnarToRow (5)
+   :              :     :  :                                   :     :     +- Scan parquet default.store_sales (4)
+   :              :     :  :                                   :     +- BroadcastExchange (11)
+   :              :     :  :                                   :        +- * Project (10)
+   :              :     :  :                                   :           +- * Filter (9)
+   :              :     :  :                                   :              +- * ColumnarToRow (8)
+   :              :     :  :                                   :                 +- Scan parquet default.date_dim (7)
+   :              :     :  :                                   +- BroadcastExchange (17)
+   :              :     :  :                                      +- * Filter (16)
+   :              :     :  :                                         +- * ColumnarToRow (15)
+   :              :     :  :                                            +- Scan parquet default.item (14)
+   :              :     :  +- * Sort (45)
+   :              :     :     +- * Project (44)
+   :              :     :        +- * Filter (43)
+   :              :     :           +- * HashAggregate (42)
+   :              :     :              +- Exchange (41)
+   :              :     :                 +- * HashAggregate (40)
+   :              :     :                    +- * Project (39)
+   :              :     :                       +- * BroadcastHashJoin Inner BuildRight (38)
+   :              :     :                          :- * Project (33)
+   :              :     :                          :  +- * Filter (32)
+   :              :     :                          :     +- * ColumnarToRow (31)
+   :              :     :                          :        +- Scan parquet default.store_sales (30)
+   :              :     :                          +- BroadcastExchange (37)
+   :              :     :                             +- * Filter (36)
+   :              :     :                                +- * ColumnarToRow (35)
+   :              :     :                                   +- Scan parquet default.customer (34)
+   :              :     +- BroadcastExchange (58)
+   :              :        +- * SortMergeJoin LeftSemi (57)
+   :              :           :- * Sort (51)
+   :              :           :  +- Exchange (50)
+   :              :           :     +- * Filter (49)
+   :              :           :        +- * ColumnarToRow (48)
+   :              :           :           +- Scan parquet default.customer (47)
+   :              :           +- * Sort (56)
+   :              :              +- * Project (55)
+   :              :                 +- * Filter (54)
+   :              :                    +- * HashAggregate (53)
+   :              :                       +- ReusedExchange (52)
+   :              +- BroadcastExchange (65)
+   :                 +- * Project (64)
+   :                    +- * Filter (63)
+   :                       +- * ColumnarToRow (62)
+   :                          +- Scan parquet default.date_dim (61)
+   +- * HashAggregate (101)
+      +- Exchange (100)
+         +- * HashAggregate (99)
+            +- * Project (98)
+               +- * BroadcastHashJoin Inner BuildRight (97)
+                  :- * Project (95)
+                  :  +- * BroadcastHashJoin Inner BuildRight (94)
+                  :     :- * SortMergeJoin LeftSemi (84)
+                  :     :  :- * Sort (78)
+                  :     :  :  +- Exchange (77)
+                  :     :  :     +- * Project (76)
+                  :     :  :        +- * BroadcastHashJoin LeftSemi BuildRight (75)
+                  :     :  :           :- * Filter (73)
+                  :     :  :           :  +- * ColumnarToRow (72)
+                  :     :  :           :     +- Scan parquet default.web_sales (71)
+                  :     :  :           +- ReusedExchange (74)
+                  :     :  +- * Sort (83)
+                  :     :     +- * Project (82)
+                  :     :        +- * Filter (81)
+                  :     :           +- * HashAggregate (80)
+                  :     :              +- ReusedExchange (79)
+                  :     +- BroadcastExchange (93)
+                  :        +- * SortMergeJoin LeftSemi (92)
+                  :           :- * Sort (86)
+                  :           :  +- ReusedExchange (85)
+                  :           +- * Sort (91)
+                  :              +- * Project (90)
+                  :                 +- * Filter (89)
+                  :                    +- * HashAggregate (88)
+                  :                       +- ReusedExchange (87)
+                  +- ReusedExchange (96)
+
+
+(1) Scan parquet default.catalog_sales
+Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(cs_bill_customer_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_item_sk:int,cs_quantity:int,cs_list_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+
+(3) Filter [codegen id : 5]
+Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+Condition : isnotnull(cs_bill_customer_sk#1)
+
+(4) Scan parquet default.store_sales
+Output [2]: [ss_item_sk#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int>
+
+(5) ColumnarToRow [codegen id : 3]
+Input [2]: [ss_item_sk#7, ss_sold_date_sk#8]
+
+(6) Filter [codegen id : 3]
+Input [2]: [ss_item_sk#7, ss_sold_date_sk#8]
+Condition : isnotnull(ss_item_sk#7)
+
+(7) Scan parquet default.date_dim
+Output [3]: [d_date_sk#10, d_date#11, d_year#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date,d_year:int>
+
+(8) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#10, d_date#11, d_year#12]
+
+(9) Filter [codegen id : 1]
+Input [3]: [d_date_sk#10, d_date#11, d_year#12]
+Condition : (d_year#12 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#10))
+
+(10) Project [codegen id : 1]
+Output [2]: [d_date_sk#10, d_date#11]
+Input [3]: [d_date_sk#10, d_date#11, d_year#12]
+
+(11) BroadcastExchange
+Input [2]: [d_date_sk#10, d_date#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(12) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(13) Project [codegen id : 3]
+Output [2]: [ss_item_sk#7, d_date#11]
+Input [4]: [ss_item_sk#7, ss_sold_date_sk#8, d_date_sk#10, d_date#11]
+
+(14) Scan parquet default.item
+Output [2]: [i_item_sk#14, i_item_desc#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_desc:string>
+
+(15) ColumnarToRow [codegen id : 2]
+Input [2]: [i_item_sk#14, i_item_desc#15]
+
+(16) Filter [codegen id : 2]
+Input [2]: [i_item_sk#14, i_item_desc#15]
+Condition : isnotnull(i_item_sk#14)
+
+(17) BroadcastExchange
+Input [2]: [i_item_sk#14, i_item_desc#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#16]
+
+(18) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_item_sk#7]
+Right keys [1]: [i_item_sk#14]
+Join condition: None
+
+(19) Project [codegen id : 3]
+Output [3]: [d_date#11, i_item_sk#14, substr(i_item_desc#15, 1, 30) AS _groupingexpression#17]
+Input [4]: [ss_item_sk#7, d_date#11, i_item_sk#14, i_item_desc#15]
+
+(20) HashAggregate [codegen id : 3]
+Input [3]: [d_date#11, i_item_sk#14, _groupingexpression#17]
+Keys [3]: [_groupingexpression#17, i_item_sk#14, d_date#11]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#18]
+Results [4]: [_groupingexpression#17, i_item_sk#14, d_date#11, count#19]
+
+(21) Exchange
+Input [4]: [_groupingexpression#17, i_item_sk#14, d_date#11, count#19]
+Arguments: hashpartitioning(_groupingexpression#17, i_item_sk#14, d_date#11, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(22) HashAggregate [codegen id : 4]
+Input [4]: [_groupingexpression#17, i_item_sk#14, d_date#11, count#19]
+Keys [3]: [_groupingexpression#17, i_item_sk#14, d_date#11]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#21]
+Results [2]: [i_item_sk#14 AS item_sk#22, count(1)#21 AS count(1)#23]
+
+(23) Filter [codegen id : 4]
+Input [2]: [item_sk#22, count(1)#23]
+Condition : (count(1)#23 > 4)
+
+(24) Project [codegen id : 4]
+Output [1]: [item_sk#22]
+Input [2]: [item_sk#22, count(1)#23]
+
+(25) BroadcastExchange
+Input [1]: [item_sk#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#24]
+
+(26) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [cs_item_sk#2]
+Right keys [1]: [item_sk#22]
+Join condition: None
+
+(27) Project [codegen id : 5]
+Output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+
+(28) Exchange
+Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(29) Sort [codegen id : 6]
+Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5]
+Arguments: [cs_bill_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(30) Scan parquet default.store_sales
+Output [4]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28, ss_sold_date_sk#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_quantity:int,ss_sales_price:decimal(7,2)>
+
+(31) ColumnarToRow [codegen id : 8]
+Input [4]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28, ss_sold_date_sk#29]
+
+(32) Filter [codegen id : 8]
+Input [4]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28, ss_sold_date_sk#29]
+Condition : isnotnull(ss_customer_sk#26)
+
+(33) Project [codegen id : 8]
+Output [3]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28]
+Input [4]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28, ss_sold_date_sk#29]
+
+(34) Scan parquet default.customer
+Output [1]: [c_customer_sk#30]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int>
+
+(35) ColumnarToRow [codegen id : 7]
+Input [1]: [c_customer_sk#30]
+
+(36) Filter [codegen id : 7]
+Input [1]: [c_customer_sk#30]
+Condition : isnotnull(c_customer_sk#30)
+
+(37) BroadcastExchange
+Input [1]: [c_customer_sk#30]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#31]
+
+(38) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_customer_sk#26]
+Right keys [1]: [c_customer_sk#30]
+Join condition: None
+
+(39) Project [codegen id : 8]
+Output [3]: [ss_quantity#27, ss_sales_price#28, c_customer_sk#30]
+Input [4]: [ss_customer_sk#26, ss_quantity#27, ss_sales_price#28, c_customer_sk#30]
+
+(40) HashAggregate [codegen id : 8]
+Input [3]: [ss_quantity#27, ss_sales_price#28, c_customer_sk#30]
+Keys [1]: [c_customer_sk#30]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#27 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#28 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [2]: [sum#32, isEmpty#33]
+Results [3]: [c_customer_sk#30, sum#34, isEmpty#35]
+
+(41) Exchange
+Input [3]: [c_customer_sk#30, sum#34, isEmpty#35]
+Arguments: hashpartitioning(c_customer_sk#30, 5), ENSURE_REQUIREMENTS, [id=#36]
+
+(42) HashAggregate [codegen id : 9]
+Input [3]: [c_customer_sk#30, sum#34, isEmpty#35]
+Keys [1]: [c_customer_sk#30]
+Functions [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#27 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#28 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#27 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#28 as decimal(12,2)))), DecimalType(18,2), true))#37]
+Results [2]: [c_customer_sk#30, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#27 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#28 as decimal(12,2)))), DecimalType(18,2), true))#37 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40]
+
+(43) Filter [codegen id : 9]
+Input [2]: [c_customer_sk#30, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40 as decimal(38,8)) > CheckOverflow((0.500000 * promote_precision(cast(Subquery scalar-subquery#41, [id=#42] as decimal(32,6)))), DecimalType(38,8), true)))
+
+(44) Project [codegen id : 9]
+Output [1]: [c_customer_sk#30]
+Input [2]: [c_customer_sk#30, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40]
+
+(45) Sort [codegen id : 9]
+Input [1]: [c_customer_sk#30]
+Arguments: [c_customer_sk#30 ASC NULLS FIRST], false, 0
+
+(46) SortMergeJoin [codegen id : 17]
+Left keys [1]: [cs_bill_customer_sk#1]
+Right keys [1]: [c_customer_sk#30]
+Join condition: None
+
+(47) Scan parquet default.customer
+Output [3]: [c_customer_sk#43, c_first_name#44, c_last_name#45]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_first_name:string,c_last_name:string>
+
+(48) ColumnarToRow [codegen id : 10]
+Input [3]: [c_customer_sk#43, c_first_name#44, c_last_name#45]
+
+(49) Filter [codegen id : 10]
+Input [3]: [c_customer_sk#43, c_first_name#44, c_last_name#45]
+Condition : isnotnull(c_customer_sk#43)
+
+(50) Exchange
+Input [3]: [c_customer_sk#43, c_first_name#44, c_last_name#45]
+Arguments: hashpartitioning(c_customer_sk#43, 5), ENSURE_REQUIREMENTS, [id=#46]
+
+(51) Sort [codegen id : 11]
+Input [3]: [c_customer_sk#43, c_first_name#44, c_last_name#45]
+Arguments: [c_customer_sk#43 ASC NULLS FIRST], false, 0
+
+(52) ReusedExchange [Reuses operator id: 41]
+Output [3]: [c_customer_sk#30, sum#34, isEmpty#35]
+
+(53) HashAggregate [codegen id : 14]
+Input [3]: [c_customer_sk#30, sum#34, isEmpty#35]
+Keys [1]: [c_customer_sk#30]
+Functions [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#27 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#28 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#27 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#28 as decimal(12,2)))), DecimalType(18,2), true))#37]
+Results [2]: [c_customer_sk#30, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#27 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#28 as decimal(12,2)))), DecimalType(18,2), true))#37 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40]
+
+(54) Filter [codegen id : 14]
+Input [2]: [c_customer_sk#30, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40 as decimal(38,8)) > CheckOverflow((0.500000 * promote_precision(cast(ReusedSubquery Subquery scalar-subquery#41, [id=#42] as decimal(32,6)))), DecimalType(38,8), true)))
+
+(55) Project [codegen id : 14]
+Output [1]: [c_customer_sk#30]
+Input [2]: [c_customer_sk#30, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#40]
+
+(56) Sort [codegen id : 14]
+Input [1]: [c_customer_sk#30]
+Arguments: [c_customer_sk#30 ASC NULLS FIRST], false, 0
+
+(57) SortMergeJoin [codegen id : 15]
+Left keys [1]: [c_customer_sk#43]
+Right keys [1]: [c_customer_sk#30]
+Join condition: None
+
+(58) BroadcastExchange
+Input [3]: [c_customer_sk#43, c_first_name#44, c_last_name#45]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#47]
+
+(59) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [cs_bill_customer_sk#1]
+Right keys [1]: [c_customer_sk#43]
+Join condition: None
+
+(60) Project [codegen id : 17]
+Output [5]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#44, c_last_name#45]
+Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#43, c_first_name#44, c_last_name#45]
+
+(61) Scan parquet default.date_dim
+Output [3]: [d_date_sk#48, d_year#49, d_moy#50]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(62) ColumnarToRow [codegen id : 16]
+Input [3]: [d_date_sk#48, d_year#49, d_moy#50]
+
+(63) Filter [codegen id : 16]
+Input [3]: [d_date_sk#48, d_year#49, d_moy#50]
+Condition : ((((isnotnull(d_year#49) AND isnotnull(d_moy#50)) AND (d_year#49 = 2000)) AND (d_moy#50 = 2)) AND isnotnull(d_date_sk#48))
+
+(64) Project [codegen id : 16]
+Output [1]: [d_date_sk#48]
+Input [3]: [d_date_sk#48, d_year#49, d_moy#50]
+
+(65) BroadcastExchange
+Input [1]: [d_date_sk#48]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#51]
+
+(66) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [cs_sold_date_sk#5]
+Right keys [1]: [d_date_sk#48]
+Join condition: None
+
+(67) Project [codegen id : 17]
+Output [4]: [cs_quantity#3, cs_list_price#4, c_first_name#44, c_last_name#45]
+Input [6]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#44, c_last_name#45, d_date_sk#48]
+
+(68) HashAggregate [codegen id : 17]
+Input [4]: [cs_quantity#3, cs_list_price#4, c_first_name#44, c_last_name#45]
+Keys [2]: [c_last_name#45, c_first_name#44]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#3 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#4 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [2]: [sum#52, isEmpty#53]
+Results [4]: [c_last_name#45, c_first_name#44, sum#54, isEmpty#55]
+
+(69) Exchange
+Input [4]: [c_last_name#45, c_first_name#44, sum#54, isEmpty#55]
+Arguments: hashpartitioning(c_last_name#45, c_first_name#44, 5), ENSURE_REQUIREMENTS, [id=#56]
+
+(70) HashAggregate [codegen id : 18]
+Input [4]: [c_last_name#45, c_first_name#44, sum#54, isEmpty#55]
+Keys [2]: [c_last_name#45, c_first_name#44]
+Functions [1]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#3 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#4 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#3 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#4 as decimal(12,2)))), DecimalType(18,2), true))#57]
+Results [3]: [c_last_name#45, c_first_name#44, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#3 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#4 as decimal(12,2)))), DecimalType(18,2), true))#57 AS sales#58]
+
+(71) Scan parquet default.web_sales
+Output [5]: [ws_item_sk#59, ws_bill_customer_sk#60, ws_quantity#61, ws_list_price#62, ws_sold_date_sk#63]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#63), dynamicpruningexpression(ws_sold_date_sk#63 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_bill_customer_sk:int,ws_quantity:int,ws_list_price:decimal(7,2)>
+
+(72) ColumnarToRow [codegen id : 23]
+Input [5]: [ws_item_sk#59, ws_bill_customer_sk#60, ws_quantity#61, ws_list_price#62, ws_sold_date_sk#63]
+
+(73) Filter [codegen id : 23]
+Input [5]: [ws_item_sk#59, ws_bill_customer_sk#60, ws_quantity#61, ws_list_price#62, ws_sold_date_sk#63]
+Condition : isnotnull(ws_bill_customer_sk#60)
+
+(74) ReusedExchange [Reuses operator id: unknown]
+Output [1]: [item_sk#22]
+
+(75) BroadcastHashJoin [codegen id : 23]
+Left keys [1]: [ws_item_sk#59]
+Right keys [1]: [item_sk#22]
+Join condition: None
+
+(76) Project [codegen id : 23]
+Output [4]: [ws_bill_customer_sk#60, ws_quantity#61, ws_list_price#62, ws_sold_date_sk#63]
+Input [5]: [ws_item_sk#59, ws_bill_customer_sk#60, ws_quantity#61, ws_list_price#62, ws_sold_date_sk#63]
+
+(77) Exchange
+Input [4]: [ws_bill_customer_sk#60, ws_quantity#61, ws_list_price#62, ws_sold_date_sk#63]
+Arguments: hashpartitioning(ws_bill_customer_sk#60, 5), ENSURE_REQUIREMENTS, [id=#64]
+
+(78) Sort [codegen id : 24]
+Input [4]: [ws_bill_customer_sk#60, ws_quantity#61, ws_list_price#62, ws_sold_date_sk#63]
+Arguments: [ws_bill_customer_sk#60 ASC NULLS FIRST], false, 0
+
+(79) ReusedExchange [Reuses operator id: 41]
+Output [3]: [c_customer_sk#65, sum#66, isEmpty#67]
+
+(80) HashAggregate [codegen id : 27]
+Input [3]: [c_customer_sk#65, sum#66, isEmpty#67]
+Keys [1]: [c_customer_sk#65]
+Functions [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#68 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#69 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#68 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#69 as decimal(12,2)))), DecimalType(18,2), true))#70]
+Results [2]: [c_customer_sk#65, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#68 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#69 as decimal(12,2)))), DecimalType(18,2), true))#70 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#71]
+
+(81) Filter [codegen id : 27]
+Input [2]: [c_customer_sk#65, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#71]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#71) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#71 as decimal(38,8)) > CheckOverflow((0.500000 * promote_precision(cast(ReusedSubquery Subquery scalar-subquery#41, [id=#42] as decimal(32,6)))), DecimalType(38,8), true)))
+
+(82) Project [codegen id : 27]
+Output [1]: [c_customer_sk#65]
+Input [2]: [c_customer_sk#65, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#71]
+
+(83) Sort [codegen id : 27]
+Input [1]: [c_customer_sk#65]
+Arguments: [c_customer_sk#65 ASC NULLS FIRST], false, 0
+
+(84) SortMergeJoin [codegen id : 35]
+Left keys [1]: [ws_bill_customer_sk#60]
+Right keys [1]: [c_customer_sk#65]
+Join condition: None
+
+(85) ReusedExchange [Reuses operator id: 50]
+Output [3]: [c_customer_sk#72, c_first_name#73, c_last_name#74]
+
+(86) Sort [codegen id : 29]
+Input [3]: [c_customer_sk#72, c_first_name#73, c_last_name#74]
+Arguments: [c_customer_sk#72 ASC NULLS FIRST], false, 0
+
+(87) ReusedExchange [Reuses operator id: 41]
+Output [3]: [c_customer_sk#65, sum#66, isEmpty#67]
+
+(88) HashAggregate [codegen id : 32]
+Input [3]: [c_customer_sk#65, sum#66, isEmpty#67]
+Keys [1]: [c_customer_sk#65]
+Functions [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#68 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#69 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#68 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#69 as decimal(12,2)))), DecimalType(18,2), true))#70]
+Results [2]: [c_customer_sk#65, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#68 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#69 as decimal(12,2)))), DecimalType(18,2), true))#70 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#71]
+
+(89) Filter [codegen id : 32]
+Input [2]: [c_customer_sk#65, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#71]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#71) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#71 as decimal(38,8)) > CheckOverflow((0.500000 * promote_precision(cast(ReusedSubquery Subquery scalar-subquery#41, [id=#42] as decimal(32,6)))), DecimalType(38,8), true)))
+
+(90) Project [codegen id : 32]
+Output [1]: [c_customer_sk#65]
+Input [2]: [c_customer_sk#65, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#38 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#39 as decimal(12,2)))), DecimalType(18,2), true))#71]
+
+(91) Sort [codegen id : 32]
+Input [1]: [c_customer_sk#65]
+Arguments: [c_customer_sk#65 ASC NULLS FIRST], false, 0
+
+(92) SortMergeJoin [codegen id : 33]
+Left keys [1]: [c_customer_sk#72]
+Right keys [1]: [c_customer_sk#65]
+Join condition: None
+
+(93) BroadcastExchange
+Input [3]: [c_customer_sk#72, c_first_name#73, c_last_name#74]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#75]
+
+(94) BroadcastHashJoin [codegen id : 35]
+Left keys [1]: [ws_bill_customer_sk#60]
+Right keys [1]: [c_customer_sk#72]
+Join condition: None
+
+(95) Project [codegen id : 35]
+Output [5]: [ws_quantity#61, ws_list_price#62, ws_sold_date_sk#63, c_first_name#73, c_last_name#74]
+Input [7]: [ws_bill_customer_sk#60, ws_quantity#61, ws_list_price#62, ws_sold_date_sk#63, c_customer_sk#72, c_first_name#73, c_last_name#74]
+
+(96) ReusedExchange [Reuses operator id: 65]
+Output [1]: [d_date_sk#76]
+
+(97) BroadcastHashJoin [codegen id : 35]
+Left keys [1]: [ws_sold_date_sk#63]
+Right keys [1]: [d_date_sk#76]
+Join condition: None
+
+(98) Project [codegen id : 35]
+Output [4]: [ws_quantity#61, ws_list_price#62, c_first_name#73, c_last_name#74]
+Input [6]: [ws_quantity#61, ws_list_price#62, ws_sold_date_sk#63, c_first_name#73, c_last_name#74, d_date_sk#76]
+
+(99) HashAggregate [codegen id : 35]
+Input [4]: [ws_quantity#61, ws_list_price#62, c_first_name#73, c_last_name#74]
+Keys [2]: [c_last_name#74, c_first_name#73]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#61 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#62 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [2]: [sum#77, isEmpty#78]
+Results [4]: [c_last_name#74, c_first_name#73, sum#79, isEmpty#80]
+
+(100) Exchange
+Input [4]: [c_last_name#74, c_first_name#73, sum#79, isEmpty#80]
+Arguments: hashpartitioning(c_last_name#74, c_first_name#73, 5), ENSURE_REQUIREMENTS, [id=#81]
+
+(101) HashAggregate [codegen id : 36]
+Input [4]: [c_last_name#74, c_first_name#73, sum#79, isEmpty#80]
+Keys [2]: [c_last_name#74, c_first_name#73]
+Functions [1]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#61 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#62 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#61 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#62 as decimal(12,2)))), DecimalType(18,2), true))#82]
+Results [3]: [c_last_name#74, c_first_name#73, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#61 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#62 as decimal(12,2)))), DecimalType(18,2), true))#82 AS sales#83]
+
+(102) Union
+
+(103) TakeOrderedAndProject
+Input [3]: [c_last_name#45, c_first_name#44, sales#58]
+Arguments: 100, [c_last_name#45 ASC NULLS FIRST, c_first_name#44 ASC NULLS FIRST, sales#58 ASC NULLS FIRST], [c_last_name#45, c_first_name#44, sales#58]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (104)
+
+
+(104) ReusedExchange [Reuses operator id: 65]
+Output [1]: [d_date_sk#48]
+
+Subquery:2 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (105)
+
+
+(105) ReusedExchange [Reuses operator id: 11]
+Output [2]: [d_date_sk#10, d_date#11]
+
+Subquery:3 Hosting operator id = 43 Hosting Expression = Subquery scalar-subquery#41, [id=#42]
+* HashAggregate (127)
++- Exchange (126)
+   +- * HashAggregate (125)
+      +- * HashAggregate (124)
+         +- Exchange (123)
+            +- * HashAggregate (122)
+               +- * Project (121)
+                  +- * BroadcastHashJoin Inner BuildRight (120)
+                     :- * Project (114)
+                     :  +- * BroadcastHashJoin Inner BuildRight (113)
+                     :     :- * Filter (108)
+                     :     :  +- * ColumnarToRow (107)
+                     :     :     +- Scan parquet default.store_sales (106)
+                     :     +- BroadcastExchange (112)
+                     :        +- * Filter (111)
+                     :           +- * ColumnarToRow (110)
+                     :              +- Scan parquet default.customer (109)
+                     +- BroadcastExchange (119)
+                        +- * Project (118)
+                           +- * Filter (117)
+                              +- * ColumnarToRow (116)
+                                 +- Scan parquet default.date_dim (115)
+
+
+(106) Scan parquet default.store_sales
+Output [4]: [ss_customer_sk#84, ss_quantity#85, ss_sales_price#86, ss_sold_date_sk#87]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#87), dynamicpruningexpression(ss_sold_date_sk#87 IN dynamicpruning#88)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_quantity:int,ss_sales_price:decimal(7,2)>
+
+(107) ColumnarToRow [codegen id : 3]
+Input [4]: [ss_customer_sk#84, ss_quantity#85, ss_sales_price#86, ss_sold_date_sk#87]
+
+(108) Filter [codegen id : 3]
+Input [4]: [ss_customer_sk#84, ss_quantity#85, ss_sales_price#86, ss_sold_date_sk#87]
+Condition : isnotnull(ss_customer_sk#84)
+
+(109) Scan parquet default.customer
+Output [1]: [c_customer_sk#89]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int>
+
+(110) ColumnarToRow [codegen id : 1]
+Input [1]: [c_customer_sk#89]
+
+(111) Filter [codegen id : 1]
+Input [1]: [c_customer_sk#89]
+Condition : isnotnull(c_customer_sk#89)
+
+(112) BroadcastExchange
+Input [1]: [c_customer_sk#89]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#90]
+
+(113) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_customer_sk#84]
+Right keys [1]: [c_customer_sk#89]
+Join condition: None
+
+(114) Project [codegen id : 3]
+Output [4]: [ss_quantity#85, ss_sales_price#86, ss_sold_date_sk#87, c_customer_sk#89]
+Input [5]: [ss_customer_sk#84, ss_quantity#85, ss_sales_price#86, ss_sold_date_sk#87, c_customer_sk#89]
+
+(115) Scan parquet default.date_dim
+Output [2]: [d_date_sk#91, d_year#92]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(116) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#91, d_year#92]
+
+(117) Filter [codegen id : 2]
+Input [2]: [d_date_sk#91, d_year#92]
+Condition : (d_year#92 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#91))
+
+(118) Project [codegen id : 2]
+Output [1]: [d_date_sk#91]
+Input [2]: [d_date_sk#91, d_year#92]
+
+(119) BroadcastExchange
+Input [1]: [d_date_sk#91]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#93]
+
+(120) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#87]
+Right keys [1]: [d_date_sk#91]
+Join condition: None
+
+(121) Project [codegen id : 3]
+Output [3]: [ss_quantity#85, ss_sales_price#86, c_customer_sk#89]
+Input [5]: [ss_quantity#85, ss_sales_price#86, ss_sold_date_sk#87, c_customer_sk#89, d_date_sk#91]
+
+(122) HashAggregate [codegen id : 3]
+Input [3]: [ss_quantity#85, ss_sales_price#86, c_customer_sk#89]
+Keys [1]: [c_customer_sk#89]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#85 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#86 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [2]: [sum#94, isEmpty#95]
+Results [3]: [c_customer_sk#89, sum#96, isEmpty#97]
+
+(123) Exchange
+Input [3]: [c_customer_sk#89, sum#96, isEmpty#97]
+Arguments: hashpartitioning(c_customer_sk#89, 5), ENSURE_REQUIREMENTS, [id=#98]
+
+(124) HashAggregate [codegen id : 4]
+Input [3]: [c_customer_sk#89, sum#96, isEmpty#97]
+Keys [1]: [c_customer_sk#89]
+Functions [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#85 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#86 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#85 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#86 as decimal(12,2)))), DecimalType(18,2), true))#99]
+Results [1]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#85 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#86 as decimal(12,2)))), DecimalType(18,2), true))#99 AS csales#100]
+
+(125) HashAggregate [codegen id : 4]
+Input [1]: [csales#100]
+Keys: []
+Functions [1]: [partial_max(csales#100)]
+Aggregate Attributes [1]: [max#101]
+Results [1]: [max#102]
+
+(126) Exchange
+Input [1]: [max#102]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#103]
+
+(127) HashAggregate [codegen id : 5]
+Input [1]: [max#102]
+Keys: []
+Functions [1]: [max(csales#100)]
+Aggregate Attributes [1]: [max(csales#100)#104]
+Results [1]: [max(csales#100)#104 AS tpcds_cmax#105]
+
+Subquery:4 Hosting operator id = 106 Hosting Expression = ss_sold_date_sk#87 IN dynamicpruning#88
+ReusedExchange (128)
+
+
+(128) ReusedExchange [Reuses operator id: 119]
+Output [1]: [d_date_sk#91]
+
+Subquery:5 Hosting operator id = 54 Hosting Expression = ReusedSubquery Subquery scalar-subquery#41, [id=#42]
+
+Subquery:6 Hosting operator id = 71 Hosting Expression = ws_sold_date_sk#63 IN dynamicpruning#6
+
+Subquery:7 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#41, [id=#42]
+
+Subquery:8 Hosting operator id = 89 Hosting Expression = ReusedSubquery Subquery scalar-subquery#41, [id=#42]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt
new file mode 100644
index 0000000000000..8ef321be35bf0
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt
@@ -0,0 +1,209 @@
+TakeOrderedAndProject [c_last_name,c_first_name,sales]
+  Union
+    WholeStageCodegen (18)
+      HashAggregate [c_last_name,c_first_name,sum,isEmpty] [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),sales,sum,isEmpty]
+        InputAdapter
+          Exchange [c_last_name,c_first_name] #1
+            WholeStageCodegen (17)
+              HashAggregate [c_last_name,c_first_name,cs_quantity,cs_list_price] [sum,isEmpty,sum,isEmpty]
+                Project [cs_quantity,cs_list_price,c_first_name,c_last_name]
+                  BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                    Project [cs_quantity,cs_list_price,cs_sold_date_sk,c_first_name,c_last_name]
+                      BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk]
+                        SortMergeJoin [cs_bill_customer_sk,c_customer_sk]
+                          InputAdapter
+                            WholeStageCodegen (6)
+                              Sort [cs_bill_customer_sk]
+                                InputAdapter
+                                  Exchange [cs_bill_customer_sk] #2
+                                    WholeStageCodegen (5)
+                                      Project [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk]
+                                        BroadcastHashJoin [cs_item_sk,item_sk]
+                                          Filter [cs_bill_customer_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk]
+                                                  SubqueryBroadcast [d_date_sk] #1
+                                                    ReusedExchange [d_date_sk] #3
+                                          InputAdapter
+                                            BroadcastExchange #4
+                                              WholeStageCodegen (4)
+                                                Project [item_sk]
+                                                  Filter [count(1)]
+                                                    HashAggregate [_groupingexpression,i_item_sk,d_date,count] [count(1),item_sk,count(1),count]
+                                                      InputAdapter
+                                                        Exchange [_groupingexpression,i_item_sk,d_date] #5
+                                                          WholeStageCodegen (3)
+                                                            HashAggregate [_groupingexpression,i_item_sk,d_date] [count,count]
+                                                              Project [d_date,i_item_sk,i_item_desc]
+                                                                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                                  Project [ss_item_sk,d_date]
+                                                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                      Filter [ss_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.store_sales [ss_item_sk,ss_sold_date_sk]
+                                                                              SubqueryBroadcast [d_date_sk] #2
+                                                                                ReusedExchange [d_date_sk,d_date] #6
+                                                                      InputAdapter
+                                                                        BroadcastExchange #6
+                                                                          WholeStageCodegen (1)
+                                                                            Project [d_date_sk,d_date]
+                                                                              Filter [d_year,d_date_sk]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.date_dim [d_date_sk,d_date,d_year]
+                                                                  InputAdapter
+                                                                    BroadcastExchange #7
+                                                                      WholeStageCodegen (2)
+                                                                        Filter [i_item_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.item [i_item_sk,i_item_desc]
+                          InputAdapter
+                            WholeStageCodegen (9)
+                              Sort [c_customer_sk]
+                                Project [c_customer_sk]
+                                  Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                    Subquery #3
+                                      WholeStageCodegen (5)
+                                        HashAggregate [max] [max(csales),tpcds_cmax,max]
+                                          InputAdapter
+                                            Exchange #10
+                                              WholeStageCodegen (4)
+                                                HashAggregate [csales] [max,max]
+                                                  HashAggregate [c_customer_sk,sum,isEmpty] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),csales,sum,isEmpty]
+                                                    InputAdapter
+                                                      Exchange [c_customer_sk] #11
+                                                        WholeStageCodegen (3)
+                                                          HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty]
+                                                            Project [ss_quantity,ss_sales_price,c_customer_sk]
+                                                              BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                Project [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk]
+                                                                  BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+                                                                    Filter [ss_customer_sk]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk]
+                                                                            SubqueryBroadcast [d_date_sk] #4
+                                                                              ReusedExchange [d_date_sk] #12
+                                                                    InputAdapter
+                                                                      BroadcastExchange #13
+                                                                        WholeStageCodegen (1)
+                                                                          Filter [c_customer_sk]
+                                                                            ColumnarToRow
+                                                                              InputAdapter
+                                                                                Scan parquet default.customer [c_customer_sk]
+                                                                InputAdapter
+                                                                  BroadcastExchange #12
+                                                                    WholeStageCodegen (2)
+                                                                      Project [d_date_sk]
+                                                                        Filter [d_year,d_date_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.date_dim [d_date_sk,d_year]
+                                    HashAggregate [c_customer_sk,sum,isEmpty] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty]
+                                      InputAdapter
+                                        Exchange [c_customer_sk] #8
+                                          WholeStageCodegen (8)
+                                            HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty]
+                                              Project [ss_quantity,ss_sales_price,c_customer_sk]
+                                                BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+                                                  Project [ss_customer_sk,ss_quantity,ss_sales_price]
+                                                    Filter [ss_customer_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk]
+                                                  InputAdapter
+                                                    BroadcastExchange #9
+                                                      WholeStageCodegen (7)
+                                                        Filter [c_customer_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.customer [c_customer_sk]
+                        InputAdapter
+                          BroadcastExchange #14
+                            WholeStageCodegen (15)
+                              SortMergeJoin [c_customer_sk,c_customer_sk]
+                                InputAdapter
+                                  WholeStageCodegen (11)
+                                    Sort [c_customer_sk]
+                                      InputAdapter
+                                        Exchange [c_customer_sk] #15
+                                          WholeStageCodegen (10)
+                                            Filter [c_customer_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.customer [c_customer_sk,c_first_name,c_last_name]
+                                InputAdapter
+                                  WholeStageCodegen (14)
+                                    Sort [c_customer_sk]
+                                      Project [c_customer_sk]
+                                        Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                          ReusedSubquery [tpcds_cmax] #3
+                                          HashAggregate [c_customer_sk,sum,isEmpty] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty]
+                                            InputAdapter
+                                              ReusedExchange [c_customer_sk,sum,isEmpty] #8
+                    InputAdapter
+                      BroadcastExchange #3
+                        WholeStageCodegen (16)
+                          Project [d_date_sk]
+                            Filter [d_year,d_moy,d_date_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+    WholeStageCodegen (36)
+      HashAggregate [c_last_name,c_first_name,sum,isEmpty] [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),sales,sum,isEmpty]
+        InputAdapter
+          Exchange [c_last_name,c_first_name] #16
+            WholeStageCodegen (35)
+              HashAggregate [c_last_name,c_first_name,ws_quantity,ws_list_price] [sum,isEmpty,sum,isEmpty]
+                Project [ws_quantity,ws_list_price,c_first_name,c_last_name]
+                  BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                    Project [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name]
+                      BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk]
+                        SortMergeJoin [ws_bill_customer_sk,c_customer_sk]
+                          InputAdapter
+                            WholeStageCodegen (24)
+                              Sort [ws_bill_customer_sk]
+                                InputAdapter
+                                  Exchange [ws_bill_customer_sk] #17
+                                    WholeStageCodegen (23)
+                                      Project [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk]
+                                        BroadcastHashJoin [ws_item_sk,item_sk]
+                                          Filter [ws_bill_customer_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk]
+                                                  ReusedSubquery [d_date_sk] #1
+                                          InputAdapter
+                                            ReusedExchange [item_sk] #18
+                          InputAdapter
+                            WholeStageCodegen (27)
+                              Sort [c_customer_sk]
+                                Project [c_customer_sk]
+                                  Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                    ReusedSubquery [tpcds_cmax] #3
+                                    HashAggregate [c_customer_sk,sum,isEmpty] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty]
+                                      InputAdapter
+                                        ReusedExchange [c_customer_sk,sum,isEmpty] #8
+                        InputAdapter
+                          BroadcastExchange #19
+                            WholeStageCodegen (33)
+                              SortMergeJoin [c_customer_sk,c_customer_sk]
+                                InputAdapter
+                                  WholeStageCodegen (29)
+                                    Sort [c_customer_sk]
+                                      InputAdapter
+                                        ReusedExchange [c_customer_sk,c_first_name,c_last_name] #15
+                                InputAdapter
+                                  WholeStageCodegen (32)
+                                    Sort [c_customer_sk]
+                                      Project [c_customer_sk]
+                                        Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                          ReusedSubquery [tpcds_cmax] #3
+                                          HashAggregate [c_customer_sk,sum,isEmpty] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty]
+                                            InputAdapter
+                                              ReusedExchange [c_customer_sk,sum,isEmpty] #8
+                    InputAdapter
+                      ReusedExchange [d_date_sk] #3
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.sf100/explain.txt
new file mode 100644
index 0000000000000..23bfec44556cc
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.sf100/explain.txt
@@ -0,0 +1,587 @@
+== Physical Plan ==
+* Project (50)
++- * Filter (49)
+   +- * HashAggregate (48)
+      +- Exchange (47)
+         +- * HashAggregate (46)
+            +- * HashAggregate (45)
+               +- Exchange (44)
+                  +- * HashAggregate (43)
+                     +- * Project (42)
+                        +- * BroadcastHashJoin Inner BuildRight (41)
+                           :- * Project (29)
+                           :  +- * SortMergeJoin Inner (28)
+                           :     :- * Sort (21)
+                           :     :  +- Exchange (20)
+                           :     :     +- * Project (19)
+                           :     :        +- * SortMergeJoin Inner (18)
+                           :     :           :- * Sort (12)
+                           :     :           :  +- Exchange (11)
+                           :     :           :     +- * Project (10)
+                           :     :           :        +- * BroadcastHashJoin Inner BuildRight (9)
+                           :     :           :           :- * Project (4)
+                           :     :           :           :  +- * Filter (3)
+                           :     :           :           :     +- * ColumnarToRow (2)
+                           :     :           :           :        +- Scan parquet default.store_sales (1)
+                           :     :           :           +- BroadcastExchange (8)
+                           :     :           :              +- * Filter (7)
+                           :     :           :                 +- * ColumnarToRow (6)
+                           :     :           :                    +- Scan parquet default.item (5)
+                           :     :           +- * Sort (17)
+                           :     :              +- Exchange (16)
+                           :     :                 +- * Filter (15)
+                           :     :                    +- * ColumnarToRow (14)
+                           :     :                       +- Scan parquet default.customer (13)
+                           :     +- * Sort (27)
+                           :        +- Exchange (26)
+                           :           +- * Project (25)
+                           :              +- * Filter (24)
+                           :                 +- * ColumnarToRow (23)
+                           :                    +- Scan parquet default.store_returns (22)
+                           +- BroadcastExchange (40)
+                              +- * Project (39)
+                                 +- * BroadcastHashJoin Inner BuildLeft (38)
+                                    :- BroadcastExchange (34)
+                                    :  +- * Project (33)
+                                    :     +- * Filter (32)
+                                    :        +- * ColumnarToRow (31)
+                                    :           +- Scan parquet default.store (30)
+                                    +- * Filter (37)
+                                       +- * ColumnarToRow (36)
+                                          +- Scan parquet default.customer_address (35)
+
+
+(1) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_net_paid:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6]
+
+(3) Filter [codegen id : 2]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6]
+Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2))
+
+(4) Project [codegen id : 2]
+Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6]
+
+(5) Scan parquet default.item
+Output [6]: [i_item_sk#7, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_color), EqualTo(i_color,pale                ), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_current_price:decimal(7,2),i_size:string,i_color:string,i_units:string,i_manager_id:int>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [6]: [i_item_sk#7, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12]
+
+(7) Filter [codegen id : 1]
+Input [6]: [i_item_sk#7, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12]
+Condition : ((isnotnull(i_color#10) AND (i_color#10 = pale                )) AND isnotnull(i_item_sk#7))
+
+(8) BroadcastExchange
+Input [6]: [i_item_sk#7, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#13]
+
+(9) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 2]
+Output [10]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12]
+Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, i_item_sk#7, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12]
+
+(11) Exchange
+Input [10]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12]
+Arguments: hashpartitioning(ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [id=#14]
+
+(12) Sort [codegen id : 3]
+Input [10]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12]
+Arguments: [ss_customer_sk#2 ASC NULLS FIRST], false, 0
+
+(13) Scan parquet default.customer
+Output [4]: [c_customer_sk#15, c_first_name#16, c_last_name#17, c_birth_country#18]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_birth_country)]
+ReadSchema: struct<c_customer_sk:int,c_first_name:string,c_last_name:string,c_birth_country:string>
+
+(14) ColumnarToRow [codegen id : 4]
+Input [4]: [c_customer_sk#15, c_first_name#16, c_last_name#17, c_birth_country#18]
+
+(15) Filter [codegen id : 4]
+Input [4]: [c_customer_sk#15, c_first_name#16, c_last_name#17, c_birth_country#18]
+Condition : (isnotnull(c_customer_sk#15) AND isnotnull(c_birth_country#18))
+
+(16) Exchange
+Input [4]: [c_customer_sk#15, c_first_name#16, c_last_name#17, c_birth_country#18]
+Arguments: hashpartitioning(c_customer_sk#15, 5), ENSURE_REQUIREMENTS, [id=#19]
+
+(17) Sort [codegen id : 5]
+Input [4]: [c_customer_sk#15, c_first_name#16, c_last_name#17, c_birth_country#18]
+Arguments: [c_customer_sk#15 ASC NULLS FIRST], false, 0
+
+(18) SortMergeJoin [codegen id : 6]
+Left keys [1]: [ss_customer_sk#2]
+Right keys [1]: [c_customer_sk#15]
+Join condition: None
+
+(19) Project [codegen id : 6]
+Output [12]: [ss_item_sk#1, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12, c_first_name#16, c_last_name#17, c_birth_country#18]
+Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12, c_customer_sk#15, c_first_name#16, c_last_name#17, c_birth_country#18]
+
+(20) Exchange
+Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12, c_first_name#16, c_last_name#17, c_birth_country#18]
+Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(21) Sort [codegen id : 7]
+Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12, c_first_name#16, c_last_name#17, c_birth_country#18]
+Arguments: [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(22) Scan parquet default.store_returns
+Output [3]: [sr_item_sk#21, sr_ticket_number#22, sr_returned_date_sk#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int>
+
+(23) ColumnarToRow [codegen id : 8]
+Input [3]: [sr_item_sk#21, sr_ticket_number#22, sr_returned_date_sk#23]
+
+(24) Filter [codegen id : 8]
+Input [3]: [sr_item_sk#21, sr_ticket_number#22, sr_returned_date_sk#23]
+Condition : (isnotnull(sr_ticket_number#22) AND isnotnull(sr_item_sk#21))
+
+(25) Project [codegen id : 8]
+Output [2]: [sr_item_sk#21, sr_ticket_number#22]
+Input [3]: [sr_item_sk#21, sr_ticket_number#22, sr_returned_date_sk#23]
+
+(26) Exchange
+Input [2]: [sr_item_sk#21, sr_ticket_number#22]
+Arguments: hashpartitioning(sr_ticket_number#22, sr_item_sk#21, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(27) Sort [codegen id : 9]
+Input [2]: [sr_item_sk#21, sr_ticket_number#22]
+Arguments: [sr_ticket_number#22 ASC NULLS FIRST, sr_item_sk#21 ASC NULLS FIRST], false, 0
+
+(28) SortMergeJoin [codegen id : 12]
+Left keys [2]: [ss_ticket_number#4, ss_item_sk#1]
+Right keys [2]: [sr_ticket_number#22, sr_item_sk#21]
+Join condition: None
+
+(29) Project [codegen id : 12]
+Output [10]: [ss_store_sk#3, ss_net_paid#5, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12, c_first_name#16, c_last_name#17, c_birth_country#18]
+Input [14]: [ss_item_sk#1, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12, c_first_name#16, c_last_name#17, c_birth_country#18, sr_item_sk#21, sr_ticket_number#22]
+
+(30) Scan parquet default.store
+Output [5]: [s_store_sk#25, s_store_name#26, s_market_id#27, s_state#28, s_zip#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_market_id:int,s_state:string,s_zip:string>
+
+(31) ColumnarToRow [codegen id : 10]
+Input [5]: [s_store_sk#25, s_store_name#26, s_market_id#27, s_state#28, s_zip#29]
+
+(32) Filter [codegen id : 10]
+Input [5]: [s_store_sk#25, s_store_name#26, s_market_id#27, s_state#28, s_zip#29]
+Condition : (((isnotnull(s_market_id#27) AND (s_market_id#27 = 8)) AND isnotnull(s_store_sk#25)) AND isnotnull(s_zip#29))
+
+(33) Project [codegen id : 10]
+Output [4]: [s_store_sk#25, s_store_name#26, s_state#28, s_zip#29]
+Input [5]: [s_store_sk#25, s_store_name#26, s_market_id#27, s_state#28, s_zip#29]
+
+(34) BroadcastExchange
+Input [4]: [s_store_sk#25, s_store_name#26, s_state#28, s_zip#29]
+Arguments: HashedRelationBroadcastMode(List(input[3, string, true]),false), [id=#30]
+
+(35) Scan parquet default.customer_address
+Output [3]: [ca_state#31, ca_zip#32, ca_country#33]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_country), IsNotNull(ca_zip)]
+ReadSchema: struct<ca_state:string,ca_zip:string,ca_country:string>
+
+(36) ColumnarToRow
+Input [3]: [ca_state#31, ca_zip#32, ca_country#33]
+
+(37) Filter
+Input [3]: [ca_state#31, ca_zip#32, ca_country#33]
+Condition : (isnotnull(ca_country#33) AND isnotnull(ca_zip#32))
+
+(38) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [s_zip#29]
+Right keys [1]: [ca_zip#32]
+Join condition: None
+
+(39) Project [codegen id : 11]
+Output [5]: [s_store_sk#25, s_store_name#26, s_state#28, ca_state#31, ca_country#33]
+Input [7]: [s_store_sk#25, s_store_name#26, s_state#28, s_zip#29, ca_state#31, ca_zip#32, ca_country#33]
+
+(40) BroadcastExchange
+Input [5]: [s_store_sk#25, s_store_name#26, s_state#28, ca_state#31, ca_country#33]
+Arguments: HashedRelationBroadcastMode(List(input[0, int, true], upper(input[4, string, true])),false), [id=#34]
+
+(41) BroadcastHashJoin [codegen id : 12]
+Left keys [2]: [ss_store_sk#3, c_birth_country#18]
+Right keys [2]: [s_store_sk#25, upper(ca_country#33)]
+Join condition: None
+
+(42) Project [codegen id : 12]
+Output [11]: [ss_net_paid#5, s_store_name#26, s_state#28, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12, c_first_name#16, c_last_name#17, ca_state#31]
+Input [15]: [ss_store_sk#3, ss_net_paid#5, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12, c_first_name#16, c_last_name#17, c_birth_country#18, s_store_sk#25, s_store_name#26, s_state#28, ca_state#31, ca_country#33]
+
+(43) HashAggregate [codegen id : 12]
+Input [11]: [ss_net_paid#5, s_store_name#26, s_state#28, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12, c_first_name#16, c_last_name#17, ca_state#31]
+Keys [10]: [c_last_name#17, c_first_name#16, s_store_name#26, ca_state#31, s_state#28, i_color#10, i_current_price#8, i_manager_id#12, i_units#11, i_size#9]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))]
+Aggregate Attributes [1]: [sum#35]
+Results [11]: [c_last_name#17, c_first_name#16, s_store_name#26, ca_state#31, s_state#28, i_color#10, i_current_price#8, i_manager_id#12, i_units#11, i_size#9, sum#36]
+
+(44) Exchange
+Input [11]: [c_last_name#17, c_first_name#16, s_store_name#26, ca_state#31, s_state#28, i_color#10, i_current_price#8, i_manager_id#12, i_units#11, i_size#9, sum#36]
+Arguments: hashpartitioning(c_last_name#17, c_first_name#16, s_store_name#26, ca_state#31, s_state#28, i_color#10, i_current_price#8, i_manager_id#12, i_units#11, i_size#9, 5), ENSURE_REQUIREMENTS, [id=#37]
+
+(45) HashAggregate [codegen id : 13]
+Input [11]: [c_last_name#17, c_first_name#16, s_store_name#26, ca_state#31, s_state#28, i_color#10, i_current_price#8, i_manager_id#12, i_units#11, i_size#9, sum#36]
+Keys [10]: [c_last_name#17, c_first_name#16, s_store_name#26, ca_state#31, s_state#28, i_color#10, i_current_price#8, i_manager_id#12, i_units#11, i_size#9]
+Functions [1]: [sum(UnscaledValue(ss_net_paid#5))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#38]
+Results [4]: [c_last_name#17, c_first_name#16, s_store_name#26, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#38,17,2) AS netpaid#39]
+
+(46) HashAggregate [codegen id : 13]
+Input [4]: [c_last_name#17, c_first_name#16, s_store_name#26, netpaid#39]
+Keys [3]: [c_last_name#17, c_first_name#16, s_store_name#26]
+Functions [1]: [partial_sum(netpaid#39)]
+Aggregate Attributes [2]: [sum#40, isEmpty#41]
+Results [5]: [c_last_name#17, c_first_name#16, s_store_name#26, sum#42, isEmpty#43]
+
+(47) Exchange
+Input [5]: [c_last_name#17, c_first_name#16, s_store_name#26, sum#42, isEmpty#43]
+Arguments: hashpartitioning(c_last_name#17, c_first_name#16, s_store_name#26, 5), ENSURE_REQUIREMENTS, [id=#44]
+
+(48) HashAggregate [codegen id : 14]
+Input [5]: [c_last_name#17, c_first_name#16, s_store_name#26, sum#42, isEmpty#43]
+Keys [3]: [c_last_name#17, c_first_name#16, s_store_name#26]
+Functions [1]: [sum(netpaid#39)]
+Aggregate Attributes [1]: [sum(netpaid#39)#45]
+Results [5]: [c_last_name#17, c_first_name#16, s_store_name#26, sum(netpaid#39)#45 AS paid#46, sum(netpaid#39)#45 AS sum(netpaid#39)#47]
+
+(49) Filter [codegen id : 14]
+Input [5]: [c_last_name#17, c_first_name#16, s_store_name#26, paid#46, sum(netpaid#39)#47]
+Condition : (isnotnull(sum(netpaid#39)#47) AND (cast(sum(netpaid#39)#47 as decimal(33,8)) > cast(Subquery scalar-subquery#48, [id=#49] as decimal(33,8))))
+
+(50) Project [codegen id : 14]
+Output [4]: [c_last_name#17, c_first_name#16, s_store_name#26, paid#46]
+Input [5]: [c_last_name#17, c_first_name#16, s_store_name#26, paid#46, sum(netpaid#39)#47]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 49 Hosting Expression = Subquery scalar-subquery#48, [id=#49]
+* HashAggregate (104)
++- Exchange (103)
+   +- * HashAggregate (102)
+      +- * HashAggregate (101)
+         +- Exchange (100)
+            +- * HashAggregate (99)
+               +- * Project (98)
+                  +- * SortMergeJoin Inner (97)
+                     :- * Sort (91)
+                     :  +- Exchange (90)
+                     :     +- * Project (89)
+                     :        +- * SortMergeJoin Inner (88)
+                     :           :- * Sort (81)
+                     :           :  +- Exchange (80)
+                     :           :     +- * Project (79)
+                     :           :        +- * SortMergeJoin Inner (78)
+                     :           :           :- * Sort (72)
+                     :           :           :  +- Exchange (71)
+                     :           :           :     +- * Project (70)
+                     :           :           :        +- * SortMergeJoin Inner (69)
+                     :           :           :           :- * Sort (63)
+                     :           :           :           :  +- Exchange (62)
+                     :           :           :           :     +- * Project (61)
+                     :           :           :           :        +- * BroadcastHashJoin Inner BuildRight (60)
+                     :           :           :           :           :- * Project (54)
+                     :           :           :           :           :  +- * Filter (53)
+                     :           :           :           :           :     +- * ColumnarToRow (52)
+                     :           :           :           :           :        +- Scan parquet default.store_sales (51)
+                     :           :           :           :           +- BroadcastExchange (59)
+                     :           :           :           :              +- * Project (58)
+                     :           :           :           :                 +- * Filter (57)
+                     :           :           :           :                    +- * ColumnarToRow (56)
+                     :           :           :           :                       +- Scan parquet default.store (55)
+                     :           :           :           +- * Sort (68)
+                     :           :           :              +- Exchange (67)
+                     :           :           :                 +- * Filter (66)
+                     :           :           :                    +- * ColumnarToRow (65)
+                     :           :           :                       +- Scan parquet default.item (64)
+                     :           :           +- * Sort (77)
+                     :           :              +- Exchange (76)
+                     :           :                 +- * Filter (75)
+                     :           :                    +- * ColumnarToRow (74)
+                     :           :                       +- Scan parquet default.customer (73)
+                     :           +- * Sort (87)
+                     :              +- Exchange (86)
+                     :                 +- * Project (85)
+                     :                    +- * Filter (84)
+                     :                       +- * ColumnarToRow (83)
+                     :                          +- Scan parquet default.store_returns (82)
+                     +- * Sort (96)
+                        +- Exchange (95)
+                           +- * Filter (94)
+                              +- * ColumnarToRow (93)
+                                 +- Scan parquet default.customer_address (92)
+
+
+(51) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#50, ss_customer_sk#51, ss_store_sk#52, ss_ticket_number#53, ss_net_paid#54, ss_sold_date_sk#55]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_net_paid:decimal(7,2)>
+
+(52) ColumnarToRow [codegen id : 2]
+Input [6]: [ss_item_sk#50, ss_customer_sk#51, ss_store_sk#52, ss_ticket_number#53, ss_net_paid#54, ss_sold_date_sk#55]
+
+(53) Filter [codegen id : 2]
+Input [6]: [ss_item_sk#50, ss_customer_sk#51, ss_store_sk#52, ss_ticket_number#53, ss_net_paid#54, ss_sold_date_sk#55]
+Condition : (((isnotnull(ss_ticket_number#53) AND isnotnull(ss_item_sk#50)) AND isnotnull(ss_store_sk#52)) AND isnotnull(ss_customer_sk#51))
+
+(54) Project [codegen id : 2]
+Output [5]: [ss_item_sk#50, ss_customer_sk#51, ss_store_sk#52, ss_ticket_number#53, ss_net_paid#54]
+Input [6]: [ss_item_sk#50, ss_customer_sk#51, ss_store_sk#52, ss_ticket_number#53, ss_net_paid#54, ss_sold_date_sk#55]
+
+(55) Scan parquet default.store
+Output [5]: [s_store_sk#56, s_store_name#57, s_market_id#58, s_state#59, s_zip#60]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_market_id:int,s_state:string,s_zip:string>
+
+(56) ColumnarToRow [codegen id : 1]
+Input [5]: [s_store_sk#56, s_store_name#57, s_market_id#58, s_state#59, s_zip#60]
+
+(57) Filter [codegen id : 1]
+Input [5]: [s_store_sk#56, s_store_name#57, s_market_id#58, s_state#59, s_zip#60]
+Condition : (((isnotnull(s_market_id#58) AND (s_market_id#58 = 8)) AND isnotnull(s_store_sk#56)) AND isnotnull(s_zip#60))
+
+(58) Project [codegen id : 1]
+Output [4]: [s_store_sk#56, s_store_name#57, s_state#59, s_zip#60]
+Input [5]: [s_store_sk#56, s_store_name#57, s_market_id#58, s_state#59, s_zip#60]
+
+(59) BroadcastExchange
+Input [4]: [s_store_sk#56, s_store_name#57, s_state#59, s_zip#60]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#61]
+
+(60) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_store_sk#52]
+Right keys [1]: [s_store_sk#56]
+Join condition: None
+
+(61) Project [codegen id : 2]
+Output [7]: [ss_item_sk#50, ss_customer_sk#51, ss_ticket_number#53, ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60]
+Input [9]: [ss_item_sk#50, ss_customer_sk#51, ss_store_sk#52, ss_ticket_number#53, ss_net_paid#54, s_store_sk#56, s_store_name#57, s_state#59, s_zip#60]
+
+(62) Exchange
+Input [7]: [ss_item_sk#50, ss_customer_sk#51, ss_ticket_number#53, ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60]
+Arguments: hashpartitioning(ss_item_sk#50, 5), ENSURE_REQUIREMENTS, [id=#62]
+
+(63) Sort [codegen id : 3]
+Input [7]: [ss_item_sk#50, ss_customer_sk#51, ss_ticket_number#53, ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60]
+Arguments: [ss_item_sk#50 ASC NULLS FIRST], false, 0
+
+(64) Scan parquet default.item
+Output [6]: [i_item_sk#63, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_current_price:decimal(7,2),i_size:string,i_color:string,i_units:string,i_manager_id:int>
+
+(65) ColumnarToRow [codegen id : 4]
+Input [6]: [i_item_sk#63, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68]
+
+(66) Filter [codegen id : 4]
+Input [6]: [i_item_sk#63, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68]
+Condition : isnotnull(i_item_sk#63)
+
+(67) Exchange
+Input [6]: [i_item_sk#63, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68]
+Arguments: hashpartitioning(i_item_sk#63, 5), ENSURE_REQUIREMENTS, [id=#69]
+
+(68) Sort [codegen id : 5]
+Input [6]: [i_item_sk#63, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68]
+Arguments: [i_item_sk#63 ASC NULLS FIRST], false, 0
+
+(69) SortMergeJoin [codegen id : 6]
+Left keys [1]: [ss_item_sk#50]
+Right keys [1]: [i_item_sk#63]
+Join condition: None
+
+(70) Project [codegen id : 6]
+Output [12]: [ss_item_sk#50, ss_customer_sk#51, ss_ticket_number#53, ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68]
+Input [13]: [ss_item_sk#50, ss_customer_sk#51, ss_ticket_number#53, ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_item_sk#63, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68]
+
+(71) Exchange
+Input [12]: [ss_item_sk#50, ss_customer_sk#51, ss_ticket_number#53, ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68]
+Arguments: hashpartitioning(ss_customer_sk#51, 5), ENSURE_REQUIREMENTS, [id=#70]
+
+(72) Sort [codegen id : 7]
+Input [12]: [ss_item_sk#50, ss_customer_sk#51, ss_ticket_number#53, ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68]
+Arguments: [ss_customer_sk#51 ASC NULLS FIRST], false, 0
+
+(73) Scan parquet default.customer
+Output [4]: [c_customer_sk#71, c_first_name#72, c_last_name#73, c_birth_country#74]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_birth_country)]
+ReadSchema: struct<c_customer_sk:int,c_first_name:string,c_last_name:string,c_birth_country:string>
+
+(74) ColumnarToRow [codegen id : 8]
+Input [4]: [c_customer_sk#71, c_first_name#72, c_last_name#73, c_birth_country#74]
+
+(75) Filter [codegen id : 8]
+Input [4]: [c_customer_sk#71, c_first_name#72, c_last_name#73, c_birth_country#74]
+Condition : (isnotnull(c_customer_sk#71) AND isnotnull(c_birth_country#74))
+
+(76) Exchange
+Input [4]: [c_customer_sk#71, c_first_name#72, c_last_name#73, c_birth_country#74]
+Arguments: hashpartitioning(c_customer_sk#71, 5), ENSURE_REQUIREMENTS, [id=#75]
+
+(77) Sort [codegen id : 9]
+Input [4]: [c_customer_sk#71, c_first_name#72, c_last_name#73, c_birth_country#74]
+Arguments: [c_customer_sk#71 ASC NULLS FIRST], false, 0
+
+(78) SortMergeJoin [codegen id : 10]
+Left keys [1]: [ss_customer_sk#51]
+Right keys [1]: [c_customer_sk#71]
+Join condition: None
+
+(79) Project [codegen id : 10]
+Output [14]: [ss_item_sk#50, ss_ticket_number#53, ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68, c_first_name#72, c_last_name#73, c_birth_country#74]
+Input [16]: [ss_item_sk#50, ss_customer_sk#51, ss_ticket_number#53, ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68, c_customer_sk#71, c_first_name#72, c_last_name#73, c_birth_country#74]
+
+(80) Exchange
+Input [14]: [ss_item_sk#50, ss_ticket_number#53, ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68, c_first_name#72, c_last_name#73, c_birth_country#74]
+Arguments: hashpartitioning(ss_ticket_number#53, ss_item_sk#50, 5), ENSURE_REQUIREMENTS, [id=#76]
+
+(81) Sort [codegen id : 11]
+Input [14]: [ss_item_sk#50, ss_ticket_number#53, ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68, c_first_name#72, c_last_name#73, c_birth_country#74]
+Arguments: [ss_ticket_number#53 ASC NULLS FIRST, ss_item_sk#50 ASC NULLS FIRST], false, 0
+
+(82) Scan parquet default.store_returns
+Output [3]: [sr_item_sk#77, sr_ticket_number#78, sr_returned_date_sk#79]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int>
+
+(83) ColumnarToRow [codegen id : 12]
+Input [3]: [sr_item_sk#77, sr_ticket_number#78, sr_returned_date_sk#79]
+
+(84) Filter [codegen id : 12]
+Input [3]: [sr_item_sk#77, sr_ticket_number#78, sr_returned_date_sk#79]
+Condition : (isnotnull(sr_ticket_number#78) AND isnotnull(sr_item_sk#77))
+
+(85) Project [codegen id : 12]
+Output [2]: [sr_item_sk#77, sr_ticket_number#78]
+Input [3]: [sr_item_sk#77, sr_ticket_number#78, sr_returned_date_sk#79]
+
+(86) Exchange
+Input [2]: [sr_item_sk#77, sr_ticket_number#78]
+Arguments: hashpartitioning(sr_ticket_number#78, sr_item_sk#77, 5), ENSURE_REQUIREMENTS, [id=#80]
+
+(87) Sort [codegen id : 13]
+Input [2]: [sr_item_sk#77, sr_ticket_number#78]
+Arguments: [sr_ticket_number#78 ASC NULLS FIRST, sr_item_sk#77 ASC NULLS FIRST], false, 0
+
+(88) SortMergeJoin [codegen id : 14]
+Left keys [2]: [ss_ticket_number#53, ss_item_sk#50]
+Right keys [2]: [sr_ticket_number#78, sr_item_sk#77]
+Join condition: None
+
+(89) Project [codegen id : 14]
+Output [12]: [ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68, c_first_name#72, c_last_name#73, c_birth_country#74]
+Input [16]: [ss_item_sk#50, ss_ticket_number#53, ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68, c_first_name#72, c_last_name#73, c_birth_country#74, sr_item_sk#77, sr_ticket_number#78]
+
+(90) Exchange
+Input [12]: [ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68, c_first_name#72, c_last_name#73, c_birth_country#74]
+Arguments: hashpartitioning(c_birth_country#74, s_zip#60, 5), ENSURE_REQUIREMENTS, [id=#81]
+
+(91) Sort [codegen id : 15]
+Input [12]: [ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68, c_first_name#72, c_last_name#73, c_birth_country#74]
+Arguments: [c_birth_country#74 ASC NULLS FIRST, s_zip#60 ASC NULLS FIRST], false, 0
+
+(92) Scan parquet default.customer_address
+Output [3]: [ca_state#82, ca_zip#83, ca_country#84]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_country), IsNotNull(ca_zip)]
+ReadSchema: struct<ca_state:string,ca_zip:string,ca_country:string>
+
+(93) ColumnarToRow [codegen id : 16]
+Input [3]: [ca_state#82, ca_zip#83, ca_country#84]
+
+(94) Filter [codegen id : 16]
+Input [3]: [ca_state#82, ca_zip#83, ca_country#84]
+Condition : (isnotnull(ca_country#84) AND isnotnull(ca_zip#83))
+
+(95) Exchange
+Input [3]: [ca_state#82, ca_zip#83, ca_country#84]
+Arguments: hashpartitioning(upper(ca_country#84), ca_zip#83, 5), ENSURE_REQUIREMENTS, [id=#85]
+
+(96) Sort [codegen id : 17]
+Input [3]: [ca_state#82, ca_zip#83, ca_country#84]
+Arguments: [upper(ca_country#84) ASC NULLS FIRST, ca_zip#83 ASC NULLS FIRST], false, 0
+
+(97) SortMergeJoin [codegen id : 18]
+Left keys [2]: [c_birth_country#74, s_zip#60]
+Right keys [2]: [upper(ca_country#84), ca_zip#83]
+Join condition: None
+
+(98) Project [codegen id : 18]
+Output [11]: [ss_net_paid#54, s_store_name#57, s_state#59, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68, c_first_name#72, c_last_name#73, ca_state#82]
+Input [15]: [ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68, c_first_name#72, c_last_name#73, c_birth_country#74, ca_state#82, ca_zip#83, ca_country#84]
+
+(99) HashAggregate [codegen id : 18]
+Input [11]: [ss_net_paid#54, s_store_name#57, s_state#59, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68, c_first_name#72, c_last_name#73, ca_state#82]
+Keys [10]: [c_last_name#73, c_first_name#72, s_store_name#57, ca_state#82, s_state#59, i_color#66, i_current_price#64, i_manager_id#68, i_units#67, i_size#65]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#54))]
+Aggregate Attributes [1]: [sum#86]
+Results [11]: [c_last_name#73, c_first_name#72, s_store_name#57, ca_state#82, s_state#59, i_color#66, i_current_price#64, i_manager_id#68, i_units#67, i_size#65, sum#87]
+
+(100) Exchange
+Input [11]: [c_last_name#73, c_first_name#72, s_store_name#57, ca_state#82, s_state#59, i_color#66, i_current_price#64, i_manager_id#68, i_units#67, i_size#65, sum#87]
+Arguments: hashpartitioning(c_last_name#73, c_first_name#72, s_store_name#57, ca_state#82, s_state#59, i_color#66, i_current_price#64, i_manager_id#68, i_units#67, i_size#65, 5), ENSURE_REQUIREMENTS, [id=#88]
+
+(101) HashAggregate [codegen id : 19]
+Input [11]: [c_last_name#73, c_first_name#72, s_store_name#57, ca_state#82, s_state#59, i_color#66, i_current_price#64, i_manager_id#68, i_units#67, i_size#65, sum#87]
+Keys [10]: [c_last_name#73, c_first_name#72, s_store_name#57, ca_state#82, s_state#59, i_color#66, i_current_price#64, i_manager_id#68, i_units#67, i_size#65]
+Functions [1]: [sum(UnscaledValue(ss_net_paid#54))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#54))#89]
+Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#54))#89,17,2) AS netpaid#39]
+
+(102) HashAggregate [codegen id : 19]
+Input [1]: [netpaid#39]
+Keys: []
+Functions [1]: [partial_avg(netpaid#39)]
+Aggregate Attributes [2]: [sum#90, count#91]
+Results [2]: [sum#92, count#93]
+
+(103) Exchange
+Input [2]: [sum#92, count#93]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#94]
+
+(104) HashAggregate [codegen id : 20]
+Input [2]: [sum#92, count#93]
+Keys: []
+Functions [1]: [avg(netpaid#39)]
+Aggregate Attributes [1]: [avg(netpaid#39)#95]
+Results [1]: [CheckOverflow((0.050000 * promote_precision(avg(netpaid#39)#95)), DecimalType(24,8), true) AS (0.05 * avg(netpaid))#96]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.sf100/simplified.txt
new file mode 100644
index 0000000000000..46fa83fcfed4b
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.sf100/simplified.txt
@@ -0,0 +1,183 @@
+WholeStageCodegen (14)
+  Project [c_last_name,c_first_name,s_store_name,paid]
+    Filter [sum(netpaid)]
+      Subquery #1
+        WholeStageCodegen (20)
+          HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count]
+            InputAdapter
+              Exchange #10
+                WholeStageCodegen (19)
+                  HashAggregate [netpaid] [sum,count,sum,count]
+                    HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum]
+                      InputAdapter
+                        Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #11
+                          WholeStageCodegen (18)
+                            HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum]
+                              Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state]
+                                SortMergeJoin [c_birth_country,s_zip,ca_country,ca_zip]
+                                  InputAdapter
+                                    WholeStageCodegen (15)
+                                      Sort [c_birth_country,s_zip]
+                                        InputAdapter
+                                          Exchange [c_birth_country,s_zip] #12
+                                            WholeStageCodegen (14)
+                                              Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country]
+                                                SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (11)
+                                                      Sort [ss_ticket_number,ss_item_sk]
+                                                        InputAdapter
+                                                          Exchange [ss_ticket_number,ss_item_sk] #13
+                                                            WholeStageCodegen (10)
+                                                              Project [ss_item_sk,ss_ticket_number,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country]
+                                                                SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                                                  InputAdapter
+                                                                    WholeStageCodegen (7)
+                                                                      Sort [ss_customer_sk]
+                                                                        InputAdapter
+                                                                          Exchange [ss_customer_sk] #14
+                                                                            WholeStageCodegen (6)
+                                                                              Project [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id]
+                                                                                SortMergeJoin [ss_item_sk,i_item_sk]
+                                                                                  InputAdapter
+                                                                                    WholeStageCodegen (3)
+                                                                                      Sort [ss_item_sk]
+                                                                                        InputAdapter
+                                                                                          Exchange [ss_item_sk] #15
+                                                                                            WholeStageCodegen (2)
+                                                                                              Project [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_net_paid,s_store_name,s_state,s_zip]
+                                                                                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                                                                  Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid]
+                                                                                                    Filter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk]
+                                                                                                      ColumnarToRow
+                                                                                                        InputAdapter
+                                                                                                          Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk]
+                                                                                                  InputAdapter
+                                                                                                    BroadcastExchange #16
+                                                                                                      WholeStageCodegen (1)
+                                                                                                        Project [s_store_sk,s_store_name,s_state,s_zip]
+                                                                                                          Filter [s_market_id,s_store_sk,s_zip]
+                                                                                                            ColumnarToRow
+                                                                                                              InputAdapter
+                                                                                                                Scan parquet default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip]
+                                                                                  InputAdapter
+                                                                                    WholeStageCodegen (5)
+                                                                                      Sort [i_item_sk]
+                                                                                        InputAdapter
+                                                                                          Exchange [i_item_sk] #17
+                                                                                            WholeStageCodegen (4)
+                                                                                              Filter [i_item_sk]
+                                                                                                ColumnarToRow
+                                                                                                  InputAdapter
+                                                                                                    Scan parquet default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id]
+                                                                  InputAdapter
+                                                                    WholeStageCodegen (9)
+                                                                      Sort [c_customer_sk]
+                                                                        InputAdapter
+                                                                          Exchange [c_customer_sk] #18
+                                                                            WholeStageCodegen (8)
+                                                                              Filter [c_customer_sk,c_birth_country]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country]
+                                                  InputAdapter
+                                                    WholeStageCodegen (13)
+                                                      Sort [sr_ticket_number,sr_item_sk]
+                                                        InputAdapter
+                                                          Exchange [sr_ticket_number,sr_item_sk] #19
+                                                            WholeStageCodegen (12)
+                                                              Project [sr_item_sk,sr_ticket_number]
+                                                                Filter [sr_ticket_number,sr_item_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk]
+                                  InputAdapter
+                                    WholeStageCodegen (17)
+                                      Sort [ca_country,ca_zip]
+                                        InputAdapter
+                                          Exchange [ca_country,ca_zip] #20
+                                            WholeStageCodegen (16)
+                                              Filter [ca_country,ca_zip]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.customer_address [ca_state,ca_zip,ca_country]
+      HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum(netpaid),sum,isEmpty]
+        InputAdapter
+          Exchange [c_last_name,c_first_name,s_store_name] #1
+            WholeStageCodegen (13)
+              HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty]
+                HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum]
+                  InputAdapter
+                    Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2
+                      WholeStageCodegen (12)
+                        HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum]
+                          Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state]
+                            BroadcastHashJoin [ss_store_sk,c_birth_country,s_store_sk,ca_country]
+                              Project [ss_store_sk,ss_net_paid,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country]
+                                SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                  InputAdapter
+                                    WholeStageCodegen (7)
+                                      Sort [ss_ticket_number,ss_item_sk]
+                                        InputAdapter
+                                          Exchange [ss_ticket_number,ss_item_sk] #3
+                                            WholeStageCodegen (6)
+                                              Project [ss_item_sk,ss_store_sk,ss_ticket_number,ss_net_paid,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country]
+                                                SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (3)
+                                                      Sort [ss_customer_sk]
+                                                        InputAdapter
+                                                          Exchange [ss_customer_sk] #4
+                                                            WholeStageCodegen (2)
+                                                              Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,i_current_price,i_size,i_color,i_units,i_manager_id]
+                                                                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                                  Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid]
+                                                                    Filter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk]
+                                                                  InputAdapter
+                                                                    BroadcastExchange #5
+                                                                      WholeStageCodegen (1)
+                                                                        Filter [i_color,i_item_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id]
+                                                  InputAdapter
+                                                    WholeStageCodegen (5)
+                                                      Sort [c_customer_sk]
+                                                        InputAdapter
+                                                          Exchange [c_customer_sk] #6
+                                                            WholeStageCodegen (4)
+                                                              Filter [c_customer_sk,c_birth_country]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country]
+                                  InputAdapter
+                                    WholeStageCodegen (9)
+                                      Sort [sr_ticket_number,sr_item_sk]
+                                        InputAdapter
+                                          Exchange [sr_ticket_number,sr_item_sk] #7
+                                            WholeStageCodegen (8)
+                                              Project [sr_item_sk,sr_ticket_number]
+                                                Filter [sr_ticket_number,sr_item_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk]
+                              InputAdapter
+                                BroadcastExchange #8
+                                  WholeStageCodegen (11)
+                                    Project [s_store_sk,s_store_name,s_state,ca_state,ca_country]
+                                      BroadcastHashJoin [s_zip,ca_zip]
+                                        InputAdapter
+                                          BroadcastExchange #9
+                                            WholeStageCodegen (10)
+                                              Project [s_store_sk,s_store_name,s_state,s_zip]
+                                                Filter [s_market_id,s_store_sk,s_zip]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip]
+                                        Filter [ca_country,ca_zip]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer_address [ca_state,ca_zip,ca_country]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/explain.txt
new file mode 100644
index 0000000000000..527913016c998
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/explain.txt
@@ -0,0 +1,527 @@
+== Physical Plan ==
+* Project (47)
++- * Filter (46)
+   +- * HashAggregate (45)
+      +- Exchange (44)
+         +- * HashAggregate (43)
+            +- * HashAggregate (42)
+               +- Exchange (41)
+                  +- * HashAggregate (40)
+                     +- * Project (39)
+                        +- * BroadcastHashJoin Inner BuildRight (38)
+                           :- * Project (33)
+                           :  +- * BroadcastHashJoin Inner BuildRight (32)
+                           :     :- * Project (27)
+                           :     :  +- * BroadcastHashJoin Inner BuildRight (26)
+                           :     :     :- * Project (21)
+                           :     :     :  +- * BroadcastHashJoin Inner BuildRight (20)
+                           :     :     :     :- * Project (14)
+                           :     :     :     :  +- * SortMergeJoin Inner (13)
+                           :     :     :     :     :- * Sort (6)
+                           :     :     :     :     :  +- Exchange (5)
+                           :     :     :     :     :     +- * Project (4)
+                           :     :     :     :     :        +- * Filter (3)
+                           :     :     :     :     :           +- * ColumnarToRow (2)
+                           :     :     :     :     :              +- Scan parquet default.store_sales (1)
+                           :     :     :     :     +- * Sort (12)
+                           :     :     :     :        +- Exchange (11)
+                           :     :     :     :           +- * Project (10)
+                           :     :     :     :              +- * Filter (9)
+                           :     :     :     :                 +- * ColumnarToRow (8)
+                           :     :     :     :                    +- Scan parquet default.store_returns (7)
+                           :     :     :     +- BroadcastExchange (19)
+                           :     :     :        +- * Project (18)
+                           :     :     :           +- * Filter (17)
+                           :     :     :              +- * ColumnarToRow (16)
+                           :     :     :                 +- Scan parquet default.store (15)
+                           :     :     +- BroadcastExchange (25)
+                           :     :        +- * Filter (24)
+                           :     :           +- * ColumnarToRow (23)
+                           :     :              +- Scan parquet default.item (22)
+                           :     +- BroadcastExchange (31)
+                           :        +- * Filter (30)
+                           :           +- * ColumnarToRow (29)
+                           :              +- Scan parquet default.customer (28)
+                           +- BroadcastExchange (37)
+                              +- * Filter (36)
+                                 +- * ColumnarToRow (35)
+                                    +- Scan parquet default.customer_address (34)
+
+
+(1) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_net_paid:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6]
+
+(3) Filter [codegen id : 1]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6]
+Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2))
+
+(4) Project [codegen id : 1]
+Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6]
+
+(5) Exchange
+Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5]
+Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#7]
+
+(6) Sort [codegen id : 2]
+Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5]
+Arguments: [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(7) Scan parquet default.store_returns
+Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int>
+
+(8) ColumnarToRow [codegen id : 3]
+Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10]
+
+(9) Filter [codegen id : 3]
+Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10]
+Condition : (isnotnull(sr_ticket_number#9) AND isnotnull(sr_item_sk#8))
+
+(10) Project [codegen id : 3]
+Output [2]: [sr_item_sk#8, sr_ticket_number#9]
+Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10]
+
+(11) Exchange
+Input [2]: [sr_item_sk#8, sr_ticket_number#9]
+Arguments: hashpartitioning(sr_ticket_number#9, sr_item_sk#8, 5), ENSURE_REQUIREMENTS, [id=#11]
+
+(12) Sort [codegen id : 4]
+Input [2]: [sr_item_sk#8, sr_ticket_number#9]
+Arguments: [sr_ticket_number#9 ASC NULLS FIRST, sr_item_sk#8 ASC NULLS FIRST], false, 0
+
+(13) SortMergeJoin [codegen id : 9]
+Left keys [2]: [ss_ticket_number#4, ss_item_sk#1]
+Right keys [2]: [sr_ticket_number#9, sr_item_sk#8]
+Join condition: None
+
+(14) Project [codegen id : 9]
+Output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5]
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#8, sr_ticket_number#9]
+
+(15) Scan parquet default.store
+Output [5]: [s_store_sk#12, s_store_name#13, s_market_id#14, s_state#15, s_zip#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_market_id:int,s_state:string,s_zip:string>
+
+(16) ColumnarToRow [codegen id : 5]
+Input [5]: [s_store_sk#12, s_store_name#13, s_market_id#14, s_state#15, s_zip#16]
+
+(17) Filter [codegen id : 5]
+Input [5]: [s_store_sk#12, s_store_name#13, s_market_id#14, s_state#15, s_zip#16]
+Condition : (((isnotnull(s_market_id#14) AND (s_market_id#14 = 8)) AND isnotnull(s_store_sk#12)) AND isnotnull(s_zip#16))
+
+(18) Project [codegen id : 5]
+Output [4]: [s_store_sk#12, s_store_name#13, s_state#15, s_zip#16]
+Input [5]: [s_store_sk#12, s_store_name#13, s_market_id#14, s_state#15, s_zip#16]
+
+(19) BroadcastExchange
+Input [4]: [s_store_sk#12, s_store_name#13, s_state#15, s_zip#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(20) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#12]
+Join condition: None
+
+(21) Project [codegen id : 9]
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#13, s_state#15, s_zip#16]
+Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#12, s_store_name#13, s_state#15, s_zip#16]
+
+(22) Scan parquet default.item
+Output [6]: [i_item_sk#18, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_color), EqualTo(i_color,pale                ), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_current_price:decimal(7,2),i_size:string,i_color:string,i_units:string,i_manager_id:int>
+
+(23) ColumnarToRow [codegen id : 6]
+Input [6]: [i_item_sk#18, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23]
+
+(24) Filter [codegen id : 6]
+Input [6]: [i_item_sk#18, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23]
+Condition : ((isnotnull(i_color#21) AND (i_color#21 = pale                )) AND isnotnull(i_item_sk#18))
+
+(25) BroadcastExchange
+Input [6]: [i_item_sk#18, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#24]
+
+(26) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#18]
+Join condition: None
+
+(27) Project [codegen id : 9]
+Output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#13, s_state#15, s_zip#16, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23]
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#13, s_state#15, s_zip#16, i_item_sk#18, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23]
+
+(28) Scan parquet default.customer
+Output [4]: [c_customer_sk#25, c_first_name#26, c_last_name#27, c_birth_country#28]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_birth_country)]
+ReadSchema: struct<c_customer_sk:int,c_first_name:string,c_last_name:string,c_birth_country:string>
+
+(29) ColumnarToRow [codegen id : 7]
+Input [4]: [c_customer_sk#25, c_first_name#26, c_last_name#27, c_birth_country#28]
+
+(30) Filter [codegen id : 7]
+Input [4]: [c_customer_sk#25, c_first_name#26, c_last_name#27, c_birth_country#28]
+Condition : (isnotnull(c_customer_sk#25) AND isnotnull(c_birth_country#28))
+
+(31) BroadcastExchange
+Input [4]: [c_customer_sk#25, c_first_name#26, c_last_name#27, c_birth_country#28]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#29]
+
+(32) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_customer_sk#2]
+Right keys [1]: [c_customer_sk#25]
+Join condition: None
+
+(33) Project [codegen id : 9]
+Output [12]: [ss_net_paid#5, s_store_name#13, s_state#15, s_zip#16, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23, c_first_name#26, c_last_name#27, c_birth_country#28]
+Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#13, s_state#15, s_zip#16, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23, c_customer_sk#25, c_first_name#26, c_last_name#27, c_birth_country#28]
+
+(34) Scan parquet default.customer_address
+Output [3]: [ca_state#30, ca_zip#31, ca_country#32]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_country), IsNotNull(ca_zip)]
+ReadSchema: struct<ca_state:string,ca_zip:string,ca_country:string>
+
+(35) ColumnarToRow [codegen id : 8]
+Input [3]: [ca_state#30, ca_zip#31, ca_country#32]
+
+(36) Filter [codegen id : 8]
+Input [3]: [ca_state#30, ca_zip#31, ca_country#32]
+Condition : (isnotnull(ca_country#32) AND isnotnull(ca_zip#31))
+
+(37) BroadcastExchange
+Input [3]: [ca_state#30, ca_zip#31, ca_country#32]
+Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [id=#33]
+
+(38) BroadcastHashJoin [codegen id : 9]
+Left keys [2]: [c_birth_country#28, s_zip#16]
+Right keys [2]: [upper(ca_country#32), ca_zip#31]
+Join condition: None
+
+(39) Project [codegen id : 9]
+Output [11]: [ss_net_paid#5, s_store_name#13, s_state#15, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23, c_first_name#26, c_last_name#27, ca_state#30]
+Input [15]: [ss_net_paid#5, s_store_name#13, s_state#15, s_zip#16, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23, c_first_name#26, c_last_name#27, c_birth_country#28, ca_state#30, ca_zip#31, ca_country#32]
+
+(40) HashAggregate [codegen id : 9]
+Input [11]: [ss_net_paid#5, s_store_name#13, s_state#15, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23, c_first_name#26, c_last_name#27, ca_state#30]
+Keys [10]: [c_last_name#27, c_first_name#26, s_store_name#13, ca_state#30, s_state#15, i_color#21, i_current_price#19, i_manager_id#23, i_units#22, i_size#20]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))]
+Aggregate Attributes [1]: [sum#34]
+Results [11]: [c_last_name#27, c_first_name#26, s_store_name#13, ca_state#30, s_state#15, i_color#21, i_current_price#19, i_manager_id#23, i_units#22, i_size#20, sum#35]
+
+(41) Exchange
+Input [11]: [c_last_name#27, c_first_name#26, s_store_name#13, ca_state#30, s_state#15, i_color#21, i_current_price#19, i_manager_id#23, i_units#22, i_size#20, sum#35]
+Arguments: hashpartitioning(c_last_name#27, c_first_name#26, s_store_name#13, ca_state#30, s_state#15, i_color#21, i_current_price#19, i_manager_id#23, i_units#22, i_size#20, 5), ENSURE_REQUIREMENTS, [id=#36]
+
+(42) HashAggregate [codegen id : 10]
+Input [11]: [c_last_name#27, c_first_name#26, s_store_name#13, ca_state#30, s_state#15, i_color#21, i_current_price#19, i_manager_id#23, i_units#22, i_size#20, sum#35]
+Keys [10]: [c_last_name#27, c_first_name#26, s_store_name#13, ca_state#30, s_state#15, i_color#21, i_current_price#19, i_manager_id#23, i_units#22, i_size#20]
+Functions [1]: [sum(UnscaledValue(ss_net_paid#5))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#37]
+Results [4]: [c_last_name#27, c_first_name#26, s_store_name#13, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#37,17,2) AS netpaid#38]
+
+(43) HashAggregate [codegen id : 10]
+Input [4]: [c_last_name#27, c_first_name#26, s_store_name#13, netpaid#38]
+Keys [3]: [c_last_name#27, c_first_name#26, s_store_name#13]
+Functions [1]: [partial_sum(netpaid#38)]
+Aggregate Attributes [2]: [sum#39, isEmpty#40]
+Results [5]: [c_last_name#27, c_first_name#26, s_store_name#13, sum#41, isEmpty#42]
+
+(44) Exchange
+Input [5]: [c_last_name#27, c_first_name#26, s_store_name#13, sum#41, isEmpty#42]
+Arguments: hashpartitioning(c_last_name#27, c_first_name#26, s_store_name#13, 5), ENSURE_REQUIREMENTS, [id=#43]
+
+(45) HashAggregate [codegen id : 11]
+Input [5]: [c_last_name#27, c_first_name#26, s_store_name#13, sum#41, isEmpty#42]
+Keys [3]: [c_last_name#27, c_first_name#26, s_store_name#13]
+Functions [1]: [sum(netpaid#38)]
+Aggregate Attributes [1]: [sum(netpaid#38)#44]
+Results [5]: [c_last_name#27, c_first_name#26, s_store_name#13, sum(netpaid#38)#44 AS paid#45, sum(netpaid#38)#44 AS sum(netpaid#38)#46]
+
+(46) Filter [codegen id : 11]
+Input [5]: [c_last_name#27, c_first_name#26, s_store_name#13, paid#45, sum(netpaid#38)#46]
+Condition : (isnotnull(sum(netpaid#38)#46) AND (cast(sum(netpaid#38)#46 as decimal(33,8)) > cast(Subquery scalar-subquery#47, [id=#48] as decimal(33,8))))
+
+(47) Project [codegen id : 11]
+Output [4]: [c_last_name#27, c_first_name#26, s_store_name#13, paid#45]
+Input [5]: [c_last_name#27, c_first_name#26, s_store_name#13, paid#45, sum(netpaid#38)#46]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#47, [id=#48]
+* HashAggregate (92)
++- Exchange (91)
+   +- * HashAggregate (90)
+      +- * HashAggregate (89)
+         +- Exchange (88)
+            +- * HashAggregate (87)
+               +- * Project (86)
+                  +- * BroadcastHashJoin Inner BuildRight (85)
+                     :- * Project (80)
+                     :  +- * BroadcastHashJoin Inner BuildRight (79)
+                     :     :- * Project (74)
+                     :     :  +- * BroadcastHashJoin Inner BuildRight (73)
+                     :     :     :- * Project (68)
+                     :     :     :  +- * BroadcastHashJoin Inner BuildRight (67)
+                     :     :     :     :- * Project (61)
+                     :     :     :     :  +- * SortMergeJoin Inner (60)
+                     :     :     :     :     :- * Sort (53)
+                     :     :     :     :     :  +- Exchange (52)
+                     :     :     :     :     :     +- * Project (51)
+                     :     :     :     :     :        +- * Filter (50)
+                     :     :     :     :     :           +- * ColumnarToRow (49)
+                     :     :     :     :     :              +- Scan parquet default.store_sales (48)
+                     :     :     :     :     +- * Sort (59)
+                     :     :     :     :        +- Exchange (58)
+                     :     :     :     :           +- * Project (57)
+                     :     :     :     :              +- * Filter (56)
+                     :     :     :     :                 +- * ColumnarToRow (55)
+                     :     :     :     :                    +- Scan parquet default.store_returns (54)
+                     :     :     :     +- BroadcastExchange (66)
+                     :     :     :        +- * Project (65)
+                     :     :     :           +- * Filter (64)
+                     :     :     :              +- * ColumnarToRow (63)
+                     :     :     :                 +- Scan parquet default.store (62)
+                     :     :     +- BroadcastExchange (72)
+                     :     :        +- * Filter (71)
+                     :     :           +- * ColumnarToRow (70)
+                     :     :              +- Scan parquet default.item (69)
+                     :     +- BroadcastExchange (78)
+                     :        +- * Filter (77)
+                     :           +- * ColumnarToRow (76)
+                     :              +- Scan parquet default.customer (75)
+                     +- BroadcastExchange (84)
+                        +- * Filter (83)
+                           +- * ColumnarToRow (82)
+                              +- Scan parquet default.customer_address (81)
+
+
+(48) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53, ss_sold_date_sk#54]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_net_paid:decimal(7,2)>
+
+(49) ColumnarToRow [codegen id : 1]
+Input [6]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53, ss_sold_date_sk#54]
+
+(50) Filter [codegen id : 1]
+Input [6]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53, ss_sold_date_sk#54]
+Condition : (((isnotnull(ss_ticket_number#52) AND isnotnull(ss_item_sk#49)) AND isnotnull(ss_store_sk#51)) AND isnotnull(ss_customer_sk#50))
+
+(51) Project [codegen id : 1]
+Output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53]
+Input [6]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53, ss_sold_date_sk#54]
+
+(52) Exchange
+Input [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53]
+Arguments: hashpartitioning(ss_ticket_number#52, ss_item_sk#49, 5), ENSURE_REQUIREMENTS, [id=#55]
+
+(53) Sort [codegen id : 2]
+Input [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53]
+Arguments: [ss_ticket_number#52 ASC NULLS FIRST, ss_item_sk#49 ASC NULLS FIRST], false, 0
+
+(54) Scan parquet default.store_returns
+Output [3]: [sr_item_sk#56, sr_ticket_number#57, sr_returned_date_sk#58]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int>
+
+(55) ColumnarToRow [codegen id : 3]
+Input [3]: [sr_item_sk#56, sr_ticket_number#57, sr_returned_date_sk#58]
+
+(56) Filter [codegen id : 3]
+Input [3]: [sr_item_sk#56, sr_ticket_number#57, sr_returned_date_sk#58]
+Condition : (isnotnull(sr_ticket_number#57) AND isnotnull(sr_item_sk#56))
+
+(57) Project [codegen id : 3]
+Output [2]: [sr_item_sk#56, sr_ticket_number#57]
+Input [3]: [sr_item_sk#56, sr_ticket_number#57, sr_returned_date_sk#58]
+
+(58) Exchange
+Input [2]: [sr_item_sk#56, sr_ticket_number#57]
+Arguments: hashpartitioning(sr_ticket_number#57, sr_item_sk#56, 5), ENSURE_REQUIREMENTS, [id=#59]
+
+(59) Sort [codegen id : 4]
+Input [2]: [sr_item_sk#56, sr_ticket_number#57]
+Arguments: [sr_ticket_number#57 ASC NULLS FIRST, sr_item_sk#56 ASC NULLS FIRST], false, 0
+
+(60) SortMergeJoin [codegen id : 9]
+Left keys [2]: [ss_ticket_number#52, ss_item_sk#49]
+Right keys [2]: [sr_ticket_number#57, sr_item_sk#56]
+Join condition: None
+
+(61) Project [codegen id : 9]
+Output [4]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53]
+Input [7]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53, sr_item_sk#56, sr_ticket_number#57]
+
+(62) Scan parquet default.store
+Output [5]: [s_store_sk#60, s_store_name#61, s_market_id#62, s_state#63, s_zip#64]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_market_id:int,s_state:string,s_zip:string>
+
+(63) ColumnarToRow [codegen id : 5]
+Input [5]: [s_store_sk#60, s_store_name#61, s_market_id#62, s_state#63, s_zip#64]
+
+(64) Filter [codegen id : 5]
+Input [5]: [s_store_sk#60, s_store_name#61, s_market_id#62, s_state#63, s_zip#64]
+Condition : (((isnotnull(s_market_id#62) AND (s_market_id#62 = 8)) AND isnotnull(s_store_sk#60)) AND isnotnull(s_zip#64))
+
+(65) Project [codegen id : 5]
+Output [4]: [s_store_sk#60, s_store_name#61, s_state#63, s_zip#64]
+Input [5]: [s_store_sk#60, s_store_name#61, s_market_id#62, s_state#63, s_zip#64]
+
+(66) BroadcastExchange
+Input [4]: [s_store_sk#60, s_store_name#61, s_state#63, s_zip#64]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#65]
+
+(67) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_store_sk#51]
+Right keys [1]: [s_store_sk#60]
+Join condition: None
+
+(68) Project [codegen id : 9]
+Output [6]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#61, s_state#63, s_zip#64]
+Input [8]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53, s_store_sk#60, s_store_name#61, s_state#63, s_zip#64]
+
+(69) Scan parquet default.item
+Output [6]: [i_item_sk#66, i_current_price#67, i_size#68, i_color#69, i_units#70, i_manager_id#71]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_current_price:decimal(7,2),i_size:string,i_color:string,i_units:string,i_manager_id:int>
+
+(70) ColumnarToRow [codegen id : 6]
+Input [6]: [i_item_sk#66, i_current_price#67, i_size#68, i_color#69, i_units#70, i_manager_id#71]
+
+(71) Filter [codegen id : 6]
+Input [6]: [i_item_sk#66, i_current_price#67, i_size#68, i_color#69, i_units#70, i_manager_id#71]
+Condition : isnotnull(i_item_sk#66)
+
+(72) BroadcastExchange
+Input [6]: [i_item_sk#66, i_current_price#67, i_size#68, i_color#69, i_units#70, i_manager_id#71]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#72]
+
+(73) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_item_sk#49]
+Right keys [1]: [i_item_sk#66]
+Join condition: None
+
+(74) Project [codegen id : 9]
+Output [10]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#61, s_state#63, s_zip#64, i_current_price#67, i_size#68, i_color#69, i_units#70, i_manager_id#71]
+Input [12]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#61, s_state#63, s_zip#64, i_item_sk#66, i_current_price#67, i_size#68, i_color#69, i_units#70, i_manager_id#71]
+
+(75) Scan parquet default.customer
+Output [4]: [c_customer_sk#73, c_first_name#74, c_last_name#75, c_birth_country#76]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_birth_country)]
+ReadSchema: struct<c_customer_sk:int,c_first_name:string,c_last_name:string,c_birth_country:string>
+
+(76) ColumnarToRow [codegen id : 7]
+Input [4]: [c_customer_sk#73, c_first_name#74, c_last_name#75, c_birth_country#76]
+
+(77) Filter [codegen id : 7]
+Input [4]: [c_customer_sk#73, c_first_name#74, c_last_name#75, c_birth_country#76]
+Condition : (isnotnull(c_customer_sk#73) AND isnotnull(c_birth_country#76))
+
+(78) BroadcastExchange
+Input [4]: [c_customer_sk#73, c_first_name#74, c_last_name#75, c_birth_country#76]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#77]
+
+(79) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_customer_sk#50]
+Right keys [1]: [c_customer_sk#73]
+Join condition: None
+
+(80) Project [codegen id : 9]
+Output [12]: [ss_net_paid#53, s_store_name#61, s_state#63, s_zip#64, i_current_price#67, i_size#68, i_color#69, i_units#70, i_manager_id#71, c_first_name#74, c_last_name#75, c_birth_country#76]
+Input [14]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#61, s_state#63, s_zip#64, i_current_price#67, i_size#68, i_color#69, i_units#70, i_manager_id#71, c_customer_sk#73, c_first_name#74, c_last_name#75, c_birth_country#76]
+
+(81) Scan parquet default.customer_address
+Output [3]: [ca_state#78, ca_zip#79, ca_country#80]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_country), IsNotNull(ca_zip)]
+ReadSchema: struct<ca_state:string,ca_zip:string,ca_country:string>
+
+(82) ColumnarToRow [codegen id : 8]
+Input [3]: [ca_state#78, ca_zip#79, ca_country#80]
+
+(83) Filter [codegen id : 8]
+Input [3]: [ca_state#78, ca_zip#79, ca_country#80]
+Condition : (isnotnull(ca_country#80) AND isnotnull(ca_zip#79))
+
+(84) BroadcastExchange
+Input [3]: [ca_state#78, ca_zip#79, ca_country#80]
+Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [id=#81]
+
+(85) BroadcastHashJoin [codegen id : 9]
+Left keys [2]: [c_birth_country#76, s_zip#64]
+Right keys [2]: [upper(ca_country#80), ca_zip#79]
+Join condition: None
+
+(86) Project [codegen id : 9]
+Output [11]: [ss_net_paid#53, s_store_name#61, s_state#63, i_current_price#67, i_size#68, i_color#69, i_units#70, i_manager_id#71, c_first_name#74, c_last_name#75, ca_state#78]
+Input [15]: [ss_net_paid#53, s_store_name#61, s_state#63, s_zip#64, i_current_price#67, i_size#68, i_color#69, i_units#70, i_manager_id#71, c_first_name#74, c_last_name#75, c_birth_country#76, ca_state#78, ca_zip#79, ca_country#80]
+
+(87) HashAggregate [codegen id : 9]
+Input [11]: [ss_net_paid#53, s_store_name#61, s_state#63, i_current_price#67, i_size#68, i_color#69, i_units#70, i_manager_id#71, c_first_name#74, c_last_name#75, ca_state#78]
+Keys [10]: [c_last_name#75, c_first_name#74, s_store_name#61, ca_state#78, s_state#63, i_color#69, i_current_price#67, i_manager_id#71, i_units#70, i_size#68]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#53))]
+Aggregate Attributes [1]: [sum#82]
+Results [11]: [c_last_name#75, c_first_name#74, s_store_name#61, ca_state#78, s_state#63, i_color#69, i_current_price#67, i_manager_id#71, i_units#70, i_size#68, sum#83]
+
+(88) Exchange
+Input [11]: [c_last_name#75, c_first_name#74, s_store_name#61, ca_state#78, s_state#63, i_color#69, i_current_price#67, i_manager_id#71, i_units#70, i_size#68, sum#83]
+Arguments: hashpartitioning(c_last_name#75, c_first_name#74, s_store_name#61, ca_state#78, s_state#63, i_color#69, i_current_price#67, i_manager_id#71, i_units#70, i_size#68, 5), ENSURE_REQUIREMENTS, [id=#84]
+
+(89) HashAggregate [codegen id : 10]
+Input [11]: [c_last_name#75, c_first_name#74, s_store_name#61, ca_state#78, s_state#63, i_color#69, i_current_price#67, i_manager_id#71, i_units#70, i_size#68, sum#83]
+Keys [10]: [c_last_name#75, c_first_name#74, s_store_name#61, ca_state#78, s_state#63, i_color#69, i_current_price#67, i_manager_id#71, i_units#70, i_size#68]
+Functions [1]: [sum(UnscaledValue(ss_net_paid#53))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#53))#85]
+Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#53))#85,17,2) AS netpaid#38]
+
+(90) HashAggregate [codegen id : 10]
+Input [1]: [netpaid#38]
+Keys: []
+Functions [1]: [partial_avg(netpaid#38)]
+Aggregate Attributes [2]: [sum#86, count#87]
+Results [2]: [sum#88, count#89]
+
+(91) Exchange
+Input [2]: [sum#88, count#89]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#90]
+
+(92) HashAggregate [codegen id : 11]
+Input [2]: [sum#88, count#89]
+Keys: []
+Functions [1]: [avg(netpaid#38)]
+Aggregate Attributes [1]: [avg(netpaid#38)#91]
+Results [1]: [CheckOverflow((0.050000 * promote_precision(avg(netpaid#38)#91)), DecimalType(24,8), true) AS (0.05 * avg(netpaid))#92]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/simplified.txt
new file mode 100644
index 0000000000000..6d50a1e3f35a0
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/simplified.txt
@@ -0,0 +1,147 @@
+WholeStageCodegen (11)
+  Project [c_last_name,c_first_name,s_store_name,paid]
+    Filter [sum(netpaid)]
+      Subquery #1
+        WholeStageCodegen (11)
+          HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count]
+            InputAdapter
+              Exchange #9
+                WholeStageCodegen (10)
+                  HashAggregate [netpaid] [sum,count,sum,count]
+                    HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum]
+                      InputAdapter
+                        Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10
+                          WholeStageCodegen (9)
+                            HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum]
+                              Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state]
+                                BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip]
+                                  Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country]
+                                    BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+                                      Project [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id]
+                                        BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                          Project [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip]
+                                            BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                              Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid]
+                                                SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (2)
+                                                      Sort [ss_ticket_number,ss_item_sk]
+                                                        InputAdapter
+                                                          Exchange [ss_ticket_number,ss_item_sk] #11
+                                                            WholeStageCodegen (1)
+                                                              Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid]
+                                                                Filter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (4)
+                                                      Sort [sr_ticket_number,sr_item_sk]
+                                                        InputAdapter
+                                                          Exchange [sr_ticket_number,sr_item_sk] #12
+                                                            WholeStageCodegen (3)
+                                                              Project [sr_item_sk,sr_ticket_number]
+                                                                Filter [sr_ticket_number,sr_item_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk]
+                                              InputAdapter
+                                                BroadcastExchange #13
+                                                  WholeStageCodegen (5)
+                                                    Project [s_store_sk,s_store_name,s_state,s_zip]
+                                                      Filter [s_market_id,s_store_sk,s_zip]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip]
+                                          InputAdapter
+                                            BroadcastExchange #14
+                                              WholeStageCodegen (6)
+                                                Filter [i_item_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id]
+                                      InputAdapter
+                                        BroadcastExchange #15
+                                          WholeStageCodegen (7)
+                                            Filter [c_customer_sk,c_birth_country]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country]
+                                  InputAdapter
+                                    BroadcastExchange #16
+                                      WholeStageCodegen (8)
+                                        Filter [ca_country,ca_zip]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer_address [ca_state,ca_zip,ca_country]
+      HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum(netpaid),sum,isEmpty]
+        InputAdapter
+          Exchange [c_last_name,c_first_name,s_store_name] #1
+            WholeStageCodegen (10)
+              HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty]
+                HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum]
+                  InputAdapter
+                    Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2
+                      WholeStageCodegen (9)
+                        HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum]
+                          Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state]
+                            BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip]
+                              Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country]
+                                BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+                                  Project [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id]
+                                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                      Project [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip]
+                                        BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                          Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid]
+                                            SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                              InputAdapter
+                                                WholeStageCodegen (2)
+                                                  Sort [ss_ticket_number,ss_item_sk]
+                                                    InputAdapter
+                                                      Exchange [ss_ticket_number,ss_item_sk] #3
+                                                        WholeStageCodegen (1)
+                                                          Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid]
+                                                            Filter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk]
+                                              InputAdapter
+                                                WholeStageCodegen (4)
+                                                  Sort [sr_ticket_number,sr_item_sk]
+                                                    InputAdapter
+                                                      Exchange [sr_ticket_number,sr_item_sk] #4
+                                                        WholeStageCodegen (3)
+                                                          Project [sr_item_sk,sr_ticket_number]
+                                                            Filter [sr_ticket_number,sr_item_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk]
+                                          InputAdapter
+                                            BroadcastExchange #5
+                                              WholeStageCodegen (5)
+                                                Project [s_store_sk,s_store_name,s_state,s_zip]
+                                                  Filter [s_market_id,s_store_sk,s_zip]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip]
+                                      InputAdapter
+                                        BroadcastExchange #6
+                                          WholeStageCodegen (6)
+                                            Filter [i_color,i_item_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id]
+                                  InputAdapter
+                                    BroadcastExchange #7
+                                      WholeStageCodegen (7)
+                                        Filter [c_customer_sk,c_birth_country]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country]
+                              InputAdapter
+                                BroadcastExchange #8
+                                  WholeStageCodegen (8)
+                                    Filter [ca_country,ca_zip]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.customer_address [ca_state,ca_zip,ca_country]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.sf100/explain.txt
new file mode 100644
index 0000000000000..1663ffd6dc09e
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.sf100/explain.txt
@@ -0,0 +1,587 @@
+== Physical Plan ==
+* Project (50)
++- * Filter (49)
+   +- * HashAggregate (48)
+      +- Exchange (47)
+         +- * HashAggregate (46)
+            +- * HashAggregate (45)
+               +- Exchange (44)
+                  +- * HashAggregate (43)
+                     +- * Project (42)
+                        +- * BroadcastHashJoin Inner BuildRight (41)
+                           :- * Project (29)
+                           :  +- * SortMergeJoin Inner (28)
+                           :     :- * Sort (21)
+                           :     :  +- Exchange (20)
+                           :     :     +- * Project (19)
+                           :     :        +- * SortMergeJoin Inner (18)
+                           :     :           :- * Sort (12)
+                           :     :           :  +- Exchange (11)
+                           :     :           :     +- * Project (10)
+                           :     :           :        +- * BroadcastHashJoin Inner BuildRight (9)
+                           :     :           :           :- * Project (4)
+                           :     :           :           :  +- * Filter (3)
+                           :     :           :           :     +- * ColumnarToRow (2)
+                           :     :           :           :        +- Scan parquet default.store_sales (1)
+                           :     :           :           +- BroadcastExchange (8)
+                           :     :           :              +- * Filter (7)
+                           :     :           :                 +- * ColumnarToRow (6)
+                           :     :           :                    +- Scan parquet default.item (5)
+                           :     :           +- * Sort (17)
+                           :     :              +- Exchange (16)
+                           :     :                 +- * Filter (15)
+                           :     :                    +- * ColumnarToRow (14)
+                           :     :                       +- Scan parquet default.customer (13)
+                           :     +- * Sort (27)
+                           :        +- Exchange (26)
+                           :           +- * Project (25)
+                           :              +- * Filter (24)
+                           :                 +- * ColumnarToRow (23)
+                           :                    +- Scan parquet default.store_returns (22)
+                           +- BroadcastExchange (40)
+                              +- * Project (39)
+                                 +- * BroadcastHashJoin Inner BuildLeft (38)
+                                    :- BroadcastExchange (34)
+                                    :  +- * Project (33)
+                                    :     +- * Filter (32)
+                                    :        +- * ColumnarToRow (31)
+                                    :           +- Scan parquet default.store (30)
+                                    +- * Filter (37)
+                                       +- * ColumnarToRow (36)
+                                          +- Scan parquet default.customer_address (35)
+
+
+(1) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_net_paid:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6]
+
+(3) Filter [codegen id : 2]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6]
+Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2))
+
+(4) Project [codegen id : 2]
+Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6]
+
+(5) Scan parquet default.item
+Output [6]: [i_item_sk#7, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_color), EqualTo(i_color,chiffon             ), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_current_price:decimal(7,2),i_size:string,i_color:string,i_units:string,i_manager_id:int>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [6]: [i_item_sk#7, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12]
+
+(7) Filter [codegen id : 1]
+Input [6]: [i_item_sk#7, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12]
+Condition : ((isnotnull(i_color#10) AND (i_color#10 = chiffon             )) AND isnotnull(i_item_sk#7))
+
+(8) BroadcastExchange
+Input [6]: [i_item_sk#7, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#13]
+
+(9) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 2]
+Output [10]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12]
+Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, i_item_sk#7, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12]
+
+(11) Exchange
+Input [10]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12]
+Arguments: hashpartitioning(ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [id=#14]
+
+(12) Sort [codegen id : 3]
+Input [10]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12]
+Arguments: [ss_customer_sk#2 ASC NULLS FIRST], false, 0
+
+(13) Scan parquet default.customer
+Output [4]: [c_customer_sk#15, c_first_name#16, c_last_name#17, c_birth_country#18]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_birth_country)]
+ReadSchema: struct<c_customer_sk:int,c_first_name:string,c_last_name:string,c_birth_country:string>
+
+(14) ColumnarToRow [codegen id : 4]
+Input [4]: [c_customer_sk#15, c_first_name#16, c_last_name#17, c_birth_country#18]
+
+(15) Filter [codegen id : 4]
+Input [4]: [c_customer_sk#15, c_first_name#16, c_last_name#17, c_birth_country#18]
+Condition : (isnotnull(c_customer_sk#15) AND isnotnull(c_birth_country#18))
+
+(16) Exchange
+Input [4]: [c_customer_sk#15, c_first_name#16, c_last_name#17, c_birth_country#18]
+Arguments: hashpartitioning(c_customer_sk#15, 5), ENSURE_REQUIREMENTS, [id=#19]
+
+(17) Sort [codegen id : 5]
+Input [4]: [c_customer_sk#15, c_first_name#16, c_last_name#17, c_birth_country#18]
+Arguments: [c_customer_sk#15 ASC NULLS FIRST], false, 0
+
+(18) SortMergeJoin [codegen id : 6]
+Left keys [1]: [ss_customer_sk#2]
+Right keys [1]: [c_customer_sk#15]
+Join condition: None
+
+(19) Project [codegen id : 6]
+Output [12]: [ss_item_sk#1, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12, c_first_name#16, c_last_name#17, c_birth_country#18]
+Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12, c_customer_sk#15, c_first_name#16, c_last_name#17, c_birth_country#18]
+
+(20) Exchange
+Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12, c_first_name#16, c_last_name#17, c_birth_country#18]
+Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(21) Sort [codegen id : 7]
+Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12, c_first_name#16, c_last_name#17, c_birth_country#18]
+Arguments: [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(22) Scan parquet default.store_returns
+Output [3]: [sr_item_sk#21, sr_ticket_number#22, sr_returned_date_sk#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int>
+
+(23) ColumnarToRow [codegen id : 8]
+Input [3]: [sr_item_sk#21, sr_ticket_number#22, sr_returned_date_sk#23]
+
+(24) Filter [codegen id : 8]
+Input [3]: [sr_item_sk#21, sr_ticket_number#22, sr_returned_date_sk#23]
+Condition : (isnotnull(sr_ticket_number#22) AND isnotnull(sr_item_sk#21))
+
+(25) Project [codegen id : 8]
+Output [2]: [sr_item_sk#21, sr_ticket_number#22]
+Input [3]: [sr_item_sk#21, sr_ticket_number#22, sr_returned_date_sk#23]
+
+(26) Exchange
+Input [2]: [sr_item_sk#21, sr_ticket_number#22]
+Arguments: hashpartitioning(sr_ticket_number#22, sr_item_sk#21, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(27) Sort [codegen id : 9]
+Input [2]: [sr_item_sk#21, sr_ticket_number#22]
+Arguments: [sr_ticket_number#22 ASC NULLS FIRST, sr_item_sk#21 ASC NULLS FIRST], false, 0
+
+(28) SortMergeJoin [codegen id : 12]
+Left keys [2]: [ss_ticket_number#4, ss_item_sk#1]
+Right keys [2]: [sr_ticket_number#22, sr_item_sk#21]
+Join condition: None
+
+(29) Project [codegen id : 12]
+Output [10]: [ss_store_sk#3, ss_net_paid#5, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12, c_first_name#16, c_last_name#17, c_birth_country#18]
+Input [14]: [ss_item_sk#1, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12, c_first_name#16, c_last_name#17, c_birth_country#18, sr_item_sk#21, sr_ticket_number#22]
+
+(30) Scan parquet default.store
+Output [5]: [s_store_sk#25, s_store_name#26, s_market_id#27, s_state#28, s_zip#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_market_id:int,s_state:string,s_zip:string>
+
+(31) ColumnarToRow [codegen id : 10]
+Input [5]: [s_store_sk#25, s_store_name#26, s_market_id#27, s_state#28, s_zip#29]
+
+(32) Filter [codegen id : 10]
+Input [5]: [s_store_sk#25, s_store_name#26, s_market_id#27, s_state#28, s_zip#29]
+Condition : (((isnotnull(s_market_id#27) AND (s_market_id#27 = 8)) AND isnotnull(s_store_sk#25)) AND isnotnull(s_zip#29))
+
+(33) Project [codegen id : 10]
+Output [4]: [s_store_sk#25, s_store_name#26, s_state#28, s_zip#29]
+Input [5]: [s_store_sk#25, s_store_name#26, s_market_id#27, s_state#28, s_zip#29]
+
+(34) BroadcastExchange
+Input [4]: [s_store_sk#25, s_store_name#26, s_state#28, s_zip#29]
+Arguments: HashedRelationBroadcastMode(List(input[3, string, true]),false), [id=#30]
+
+(35) Scan parquet default.customer_address
+Output [3]: [ca_state#31, ca_zip#32, ca_country#33]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_country), IsNotNull(ca_zip)]
+ReadSchema: struct<ca_state:string,ca_zip:string,ca_country:string>
+
+(36) ColumnarToRow
+Input [3]: [ca_state#31, ca_zip#32, ca_country#33]
+
+(37) Filter
+Input [3]: [ca_state#31, ca_zip#32, ca_country#33]
+Condition : (isnotnull(ca_country#33) AND isnotnull(ca_zip#32))
+
+(38) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [s_zip#29]
+Right keys [1]: [ca_zip#32]
+Join condition: None
+
+(39) Project [codegen id : 11]
+Output [5]: [s_store_sk#25, s_store_name#26, s_state#28, ca_state#31, ca_country#33]
+Input [7]: [s_store_sk#25, s_store_name#26, s_state#28, s_zip#29, ca_state#31, ca_zip#32, ca_country#33]
+
+(40) BroadcastExchange
+Input [5]: [s_store_sk#25, s_store_name#26, s_state#28, ca_state#31, ca_country#33]
+Arguments: HashedRelationBroadcastMode(List(input[0, int, true], upper(input[4, string, true])),false), [id=#34]
+
+(41) BroadcastHashJoin [codegen id : 12]
+Left keys [2]: [ss_store_sk#3, c_birth_country#18]
+Right keys [2]: [s_store_sk#25, upper(ca_country#33)]
+Join condition: None
+
+(42) Project [codegen id : 12]
+Output [11]: [ss_net_paid#5, s_store_name#26, s_state#28, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12, c_first_name#16, c_last_name#17, ca_state#31]
+Input [15]: [ss_store_sk#3, ss_net_paid#5, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12, c_first_name#16, c_last_name#17, c_birth_country#18, s_store_sk#25, s_store_name#26, s_state#28, ca_state#31, ca_country#33]
+
+(43) HashAggregate [codegen id : 12]
+Input [11]: [ss_net_paid#5, s_store_name#26, s_state#28, i_current_price#8, i_size#9, i_color#10, i_units#11, i_manager_id#12, c_first_name#16, c_last_name#17, ca_state#31]
+Keys [10]: [c_last_name#17, c_first_name#16, s_store_name#26, ca_state#31, s_state#28, i_color#10, i_current_price#8, i_manager_id#12, i_units#11, i_size#9]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))]
+Aggregate Attributes [1]: [sum#35]
+Results [11]: [c_last_name#17, c_first_name#16, s_store_name#26, ca_state#31, s_state#28, i_color#10, i_current_price#8, i_manager_id#12, i_units#11, i_size#9, sum#36]
+
+(44) Exchange
+Input [11]: [c_last_name#17, c_first_name#16, s_store_name#26, ca_state#31, s_state#28, i_color#10, i_current_price#8, i_manager_id#12, i_units#11, i_size#9, sum#36]
+Arguments: hashpartitioning(c_last_name#17, c_first_name#16, s_store_name#26, ca_state#31, s_state#28, i_color#10, i_current_price#8, i_manager_id#12, i_units#11, i_size#9, 5), ENSURE_REQUIREMENTS, [id=#37]
+
+(45) HashAggregate [codegen id : 13]
+Input [11]: [c_last_name#17, c_first_name#16, s_store_name#26, ca_state#31, s_state#28, i_color#10, i_current_price#8, i_manager_id#12, i_units#11, i_size#9, sum#36]
+Keys [10]: [c_last_name#17, c_first_name#16, s_store_name#26, ca_state#31, s_state#28, i_color#10, i_current_price#8, i_manager_id#12, i_units#11, i_size#9]
+Functions [1]: [sum(UnscaledValue(ss_net_paid#5))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#38]
+Results [4]: [c_last_name#17, c_first_name#16, s_store_name#26, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#38,17,2) AS netpaid#39]
+
+(46) HashAggregate [codegen id : 13]
+Input [4]: [c_last_name#17, c_first_name#16, s_store_name#26, netpaid#39]
+Keys [3]: [c_last_name#17, c_first_name#16, s_store_name#26]
+Functions [1]: [partial_sum(netpaid#39)]
+Aggregate Attributes [2]: [sum#40, isEmpty#41]
+Results [5]: [c_last_name#17, c_first_name#16, s_store_name#26, sum#42, isEmpty#43]
+
+(47) Exchange
+Input [5]: [c_last_name#17, c_first_name#16, s_store_name#26, sum#42, isEmpty#43]
+Arguments: hashpartitioning(c_last_name#17, c_first_name#16, s_store_name#26, 5), ENSURE_REQUIREMENTS, [id=#44]
+
+(48) HashAggregate [codegen id : 14]
+Input [5]: [c_last_name#17, c_first_name#16, s_store_name#26, sum#42, isEmpty#43]
+Keys [3]: [c_last_name#17, c_first_name#16, s_store_name#26]
+Functions [1]: [sum(netpaid#39)]
+Aggregate Attributes [1]: [sum(netpaid#39)#45]
+Results [5]: [c_last_name#17, c_first_name#16, s_store_name#26, sum(netpaid#39)#45 AS paid#46, sum(netpaid#39)#45 AS sum(netpaid#39)#47]
+
+(49) Filter [codegen id : 14]
+Input [5]: [c_last_name#17, c_first_name#16, s_store_name#26, paid#46, sum(netpaid#39)#47]
+Condition : (isnotnull(sum(netpaid#39)#47) AND (cast(sum(netpaid#39)#47 as decimal(33,8)) > cast(Subquery scalar-subquery#48, [id=#49] as decimal(33,8))))
+
+(50) Project [codegen id : 14]
+Output [4]: [c_last_name#17, c_first_name#16, s_store_name#26, paid#46]
+Input [5]: [c_last_name#17, c_first_name#16, s_store_name#26, paid#46, sum(netpaid#39)#47]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 49 Hosting Expression = Subquery scalar-subquery#48, [id=#49]
+* HashAggregate (104)
++- Exchange (103)
+   +- * HashAggregate (102)
+      +- * HashAggregate (101)
+         +- Exchange (100)
+            +- * HashAggregate (99)
+               +- * Project (98)
+                  +- * SortMergeJoin Inner (97)
+                     :- * Sort (91)
+                     :  +- Exchange (90)
+                     :     +- * Project (89)
+                     :        +- * SortMergeJoin Inner (88)
+                     :           :- * Sort (81)
+                     :           :  +- Exchange (80)
+                     :           :     +- * Project (79)
+                     :           :        +- * SortMergeJoin Inner (78)
+                     :           :           :- * Sort (72)
+                     :           :           :  +- Exchange (71)
+                     :           :           :     +- * Project (70)
+                     :           :           :        +- * SortMergeJoin Inner (69)
+                     :           :           :           :- * Sort (63)
+                     :           :           :           :  +- Exchange (62)
+                     :           :           :           :     +- * Project (61)
+                     :           :           :           :        +- * BroadcastHashJoin Inner BuildRight (60)
+                     :           :           :           :           :- * Project (54)
+                     :           :           :           :           :  +- * Filter (53)
+                     :           :           :           :           :     +- * ColumnarToRow (52)
+                     :           :           :           :           :        +- Scan parquet default.store_sales (51)
+                     :           :           :           :           +- BroadcastExchange (59)
+                     :           :           :           :              +- * Project (58)
+                     :           :           :           :                 +- * Filter (57)
+                     :           :           :           :                    +- * ColumnarToRow (56)
+                     :           :           :           :                       +- Scan parquet default.store (55)
+                     :           :           :           +- * Sort (68)
+                     :           :           :              +- Exchange (67)
+                     :           :           :                 +- * Filter (66)
+                     :           :           :                    +- * ColumnarToRow (65)
+                     :           :           :                       +- Scan parquet default.item (64)
+                     :           :           +- * Sort (77)
+                     :           :              +- Exchange (76)
+                     :           :                 +- * Filter (75)
+                     :           :                    +- * ColumnarToRow (74)
+                     :           :                       +- Scan parquet default.customer (73)
+                     :           +- * Sort (87)
+                     :              +- Exchange (86)
+                     :                 +- * Project (85)
+                     :                    +- * Filter (84)
+                     :                       +- * ColumnarToRow (83)
+                     :                          +- Scan parquet default.store_returns (82)
+                     +- * Sort (96)
+                        +- Exchange (95)
+                           +- * Filter (94)
+                              +- * ColumnarToRow (93)
+                                 +- Scan parquet default.customer_address (92)
+
+
+(51) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#50, ss_customer_sk#51, ss_store_sk#52, ss_ticket_number#53, ss_net_paid#54, ss_sold_date_sk#55]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_net_paid:decimal(7,2)>
+
+(52) ColumnarToRow [codegen id : 2]
+Input [6]: [ss_item_sk#50, ss_customer_sk#51, ss_store_sk#52, ss_ticket_number#53, ss_net_paid#54, ss_sold_date_sk#55]
+
+(53) Filter [codegen id : 2]
+Input [6]: [ss_item_sk#50, ss_customer_sk#51, ss_store_sk#52, ss_ticket_number#53, ss_net_paid#54, ss_sold_date_sk#55]
+Condition : (((isnotnull(ss_ticket_number#53) AND isnotnull(ss_item_sk#50)) AND isnotnull(ss_store_sk#52)) AND isnotnull(ss_customer_sk#51))
+
+(54) Project [codegen id : 2]
+Output [5]: [ss_item_sk#50, ss_customer_sk#51, ss_store_sk#52, ss_ticket_number#53, ss_net_paid#54]
+Input [6]: [ss_item_sk#50, ss_customer_sk#51, ss_store_sk#52, ss_ticket_number#53, ss_net_paid#54, ss_sold_date_sk#55]
+
+(55) Scan parquet default.store
+Output [5]: [s_store_sk#56, s_store_name#57, s_market_id#58, s_state#59, s_zip#60]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_market_id:int,s_state:string,s_zip:string>
+
+(56) ColumnarToRow [codegen id : 1]
+Input [5]: [s_store_sk#56, s_store_name#57, s_market_id#58, s_state#59, s_zip#60]
+
+(57) Filter [codegen id : 1]
+Input [5]: [s_store_sk#56, s_store_name#57, s_market_id#58, s_state#59, s_zip#60]
+Condition : (((isnotnull(s_market_id#58) AND (s_market_id#58 = 8)) AND isnotnull(s_store_sk#56)) AND isnotnull(s_zip#60))
+
+(58) Project [codegen id : 1]
+Output [4]: [s_store_sk#56, s_store_name#57, s_state#59, s_zip#60]
+Input [5]: [s_store_sk#56, s_store_name#57, s_market_id#58, s_state#59, s_zip#60]
+
+(59) BroadcastExchange
+Input [4]: [s_store_sk#56, s_store_name#57, s_state#59, s_zip#60]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#61]
+
+(60) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_store_sk#52]
+Right keys [1]: [s_store_sk#56]
+Join condition: None
+
+(61) Project [codegen id : 2]
+Output [7]: [ss_item_sk#50, ss_customer_sk#51, ss_ticket_number#53, ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60]
+Input [9]: [ss_item_sk#50, ss_customer_sk#51, ss_store_sk#52, ss_ticket_number#53, ss_net_paid#54, s_store_sk#56, s_store_name#57, s_state#59, s_zip#60]
+
+(62) Exchange
+Input [7]: [ss_item_sk#50, ss_customer_sk#51, ss_ticket_number#53, ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60]
+Arguments: hashpartitioning(ss_item_sk#50, 5), ENSURE_REQUIREMENTS, [id=#62]
+
+(63) Sort [codegen id : 3]
+Input [7]: [ss_item_sk#50, ss_customer_sk#51, ss_ticket_number#53, ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60]
+Arguments: [ss_item_sk#50 ASC NULLS FIRST], false, 0
+
+(64) Scan parquet default.item
+Output [6]: [i_item_sk#63, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_current_price:decimal(7,2),i_size:string,i_color:string,i_units:string,i_manager_id:int>
+
+(65) ColumnarToRow [codegen id : 4]
+Input [6]: [i_item_sk#63, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68]
+
+(66) Filter [codegen id : 4]
+Input [6]: [i_item_sk#63, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68]
+Condition : isnotnull(i_item_sk#63)
+
+(67) Exchange
+Input [6]: [i_item_sk#63, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68]
+Arguments: hashpartitioning(i_item_sk#63, 5), ENSURE_REQUIREMENTS, [id=#69]
+
+(68) Sort [codegen id : 5]
+Input [6]: [i_item_sk#63, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68]
+Arguments: [i_item_sk#63 ASC NULLS FIRST], false, 0
+
+(69) SortMergeJoin [codegen id : 6]
+Left keys [1]: [ss_item_sk#50]
+Right keys [1]: [i_item_sk#63]
+Join condition: None
+
+(70) Project [codegen id : 6]
+Output [12]: [ss_item_sk#50, ss_customer_sk#51, ss_ticket_number#53, ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68]
+Input [13]: [ss_item_sk#50, ss_customer_sk#51, ss_ticket_number#53, ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_item_sk#63, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68]
+
+(71) Exchange
+Input [12]: [ss_item_sk#50, ss_customer_sk#51, ss_ticket_number#53, ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68]
+Arguments: hashpartitioning(ss_customer_sk#51, 5), ENSURE_REQUIREMENTS, [id=#70]
+
+(72) Sort [codegen id : 7]
+Input [12]: [ss_item_sk#50, ss_customer_sk#51, ss_ticket_number#53, ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68]
+Arguments: [ss_customer_sk#51 ASC NULLS FIRST], false, 0
+
+(73) Scan parquet default.customer
+Output [4]: [c_customer_sk#71, c_first_name#72, c_last_name#73, c_birth_country#74]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_birth_country)]
+ReadSchema: struct<c_customer_sk:int,c_first_name:string,c_last_name:string,c_birth_country:string>
+
+(74) ColumnarToRow [codegen id : 8]
+Input [4]: [c_customer_sk#71, c_first_name#72, c_last_name#73, c_birth_country#74]
+
+(75) Filter [codegen id : 8]
+Input [4]: [c_customer_sk#71, c_first_name#72, c_last_name#73, c_birth_country#74]
+Condition : (isnotnull(c_customer_sk#71) AND isnotnull(c_birth_country#74))
+
+(76) Exchange
+Input [4]: [c_customer_sk#71, c_first_name#72, c_last_name#73, c_birth_country#74]
+Arguments: hashpartitioning(c_customer_sk#71, 5), ENSURE_REQUIREMENTS, [id=#75]
+
+(77) Sort [codegen id : 9]
+Input [4]: [c_customer_sk#71, c_first_name#72, c_last_name#73, c_birth_country#74]
+Arguments: [c_customer_sk#71 ASC NULLS FIRST], false, 0
+
+(78) SortMergeJoin [codegen id : 10]
+Left keys [1]: [ss_customer_sk#51]
+Right keys [1]: [c_customer_sk#71]
+Join condition: None
+
+(79) Project [codegen id : 10]
+Output [14]: [ss_item_sk#50, ss_ticket_number#53, ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68, c_first_name#72, c_last_name#73, c_birth_country#74]
+Input [16]: [ss_item_sk#50, ss_customer_sk#51, ss_ticket_number#53, ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68, c_customer_sk#71, c_first_name#72, c_last_name#73, c_birth_country#74]
+
+(80) Exchange
+Input [14]: [ss_item_sk#50, ss_ticket_number#53, ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68, c_first_name#72, c_last_name#73, c_birth_country#74]
+Arguments: hashpartitioning(ss_ticket_number#53, ss_item_sk#50, 5), ENSURE_REQUIREMENTS, [id=#76]
+
+(81) Sort [codegen id : 11]
+Input [14]: [ss_item_sk#50, ss_ticket_number#53, ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68, c_first_name#72, c_last_name#73, c_birth_country#74]
+Arguments: [ss_ticket_number#53 ASC NULLS FIRST, ss_item_sk#50 ASC NULLS FIRST], false, 0
+
+(82) Scan parquet default.store_returns
+Output [3]: [sr_item_sk#77, sr_ticket_number#78, sr_returned_date_sk#79]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int>
+
+(83) ColumnarToRow [codegen id : 12]
+Input [3]: [sr_item_sk#77, sr_ticket_number#78, sr_returned_date_sk#79]
+
+(84) Filter [codegen id : 12]
+Input [3]: [sr_item_sk#77, sr_ticket_number#78, sr_returned_date_sk#79]
+Condition : (isnotnull(sr_ticket_number#78) AND isnotnull(sr_item_sk#77))
+
+(85) Project [codegen id : 12]
+Output [2]: [sr_item_sk#77, sr_ticket_number#78]
+Input [3]: [sr_item_sk#77, sr_ticket_number#78, sr_returned_date_sk#79]
+
+(86) Exchange
+Input [2]: [sr_item_sk#77, sr_ticket_number#78]
+Arguments: hashpartitioning(sr_ticket_number#78, sr_item_sk#77, 5), ENSURE_REQUIREMENTS, [id=#80]
+
+(87) Sort [codegen id : 13]
+Input [2]: [sr_item_sk#77, sr_ticket_number#78]
+Arguments: [sr_ticket_number#78 ASC NULLS FIRST, sr_item_sk#77 ASC NULLS FIRST], false, 0
+
+(88) SortMergeJoin [codegen id : 14]
+Left keys [2]: [ss_ticket_number#53, ss_item_sk#50]
+Right keys [2]: [sr_ticket_number#78, sr_item_sk#77]
+Join condition: None
+
+(89) Project [codegen id : 14]
+Output [12]: [ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68, c_first_name#72, c_last_name#73, c_birth_country#74]
+Input [16]: [ss_item_sk#50, ss_ticket_number#53, ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68, c_first_name#72, c_last_name#73, c_birth_country#74, sr_item_sk#77, sr_ticket_number#78]
+
+(90) Exchange
+Input [12]: [ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68, c_first_name#72, c_last_name#73, c_birth_country#74]
+Arguments: hashpartitioning(c_birth_country#74, s_zip#60, 5), ENSURE_REQUIREMENTS, [id=#81]
+
+(91) Sort [codegen id : 15]
+Input [12]: [ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68, c_first_name#72, c_last_name#73, c_birth_country#74]
+Arguments: [c_birth_country#74 ASC NULLS FIRST, s_zip#60 ASC NULLS FIRST], false, 0
+
+(92) Scan parquet default.customer_address
+Output [3]: [ca_state#82, ca_zip#83, ca_country#84]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_country), IsNotNull(ca_zip)]
+ReadSchema: struct<ca_state:string,ca_zip:string,ca_country:string>
+
+(93) ColumnarToRow [codegen id : 16]
+Input [3]: [ca_state#82, ca_zip#83, ca_country#84]
+
+(94) Filter [codegen id : 16]
+Input [3]: [ca_state#82, ca_zip#83, ca_country#84]
+Condition : (isnotnull(ca_country#84) AND isnotnull(ca_zip#83))
+
+(95) Exchange
+Input [3]: [ca_state#82, ca_zip#83, ca_country#84]
+Arguments: hashpartitioning(upper(ca_country#84), ca_zip#83, 5), ENSURE_REQUIREMENTS, [id=#85]
+
+(96) Sort [codegen id : 17]
+Input [3]: [ca_state#82, ca_zip#83, ca_country#84]
+Arguments: [upper(ca_country#84) ASC NULLS FIRST, ca_zip#83 ASC NULLS FIRST], false, 0
+
+(97) SortMergeJoin [codegen id : 18]
+Left keys [2]: [c_birth_country#74, s_zip#60]
+Right keys [2]: [upper(ca_country#84), ca_zip#83]
+Join condition: None
+
+(98) Project [codegen id : 18]
+Output [11]: [ss_net_paid#54, s_store_name#57, s_state#59, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68, c_first_name#72, c_last_name#73, ca_state#82]
+Input [15]: [ss_net_paid#54, s_store_name#57, s_state#59, s_zip#60, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68, c_first_name#72, c_last_name#73, c_birth_country#74, ca_state#82, ca_zip#83, ca_country#84]
+
+(99) HashAggregate [codegen id : 18]
+Input [11]: [ss_net_paid#54, s_store_name#57, s_state#59, i_current_price#64, i_size#65, i_color#66, i_units#67, i_manager_id#68, c_first_name#72, c_last_name#73, ca_state#82]
+Keys [10]: [c_last_name#73, c_first_name#72, s_store_name#57, ca_state#82, s_state#59, i_color#66, i_current_price#64, i_manager_id#68, i_units#67, i_size#65]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#54))]
+Aggregate Attributes [1]: [sum#86]
+Results [11]: [c_last_name#73, c_first_name#72, s_store_name#57, ca_state#82, s_state#59, i_color#66, i_current_price#64, i_manager_id#68, i_units#67, i_size#65, sum#87]
+
+(100) Exchange
+Input [11]: [c_last_name#73, c_first_name#72, s_store_name#57, ca_state#82, s_state#59, i_color#66, i_current_price#64, i_manager_id#68, i_units#67, i_size#65, sum#87]
+Arguments: hashpartitioning(c_last_name#73, c_first_name#72, s_store_name#57, ca_state#82, s_state#59, i_color#66, i_current_price#64, i_manager_id#68, i_units#67, i_size#65, 5), ENSURE_REQUIREMENTS, [id=#88]
+
+(101) HashAggregate [codegen id : 19]
+Input [11]: [c_last_name#73, c_first_name#72, s_store_name#57, ca_state#82, s_state#59, i_color#66, i_current_price#64, i_manager_id#68, i_units#67, i_size#65, sum#87]
+Keys [10]: [c_last_name#73, c_first_name#72, s_store_name#57, ca_state#82, s_state#59, i_color#66, i_current_price#64, i_manager_id#68, i_units#67, i_size#65]
+Functions [1]: [sum(UnscaledValue(ss_net_paid#54))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#54))#89]
+Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#54))#89,17,2) AS netpaid#39]
+
+(102) HashAggregate [codegen id : 19]
+Input [1]: [netpaid#39]
+Keys: []
+Functions [1]: [partial_avg(netpaid#39)]
+Aggregate Attributes [2]: [sum#90, count#91]
+Results [2]: [sum#92, count#93]
+
+(103) Exchange
+Input [2]: [sum#92, count#93]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#94]
+
+(104) HashAggregate [codegen id : 20]
+Input [2]: [sum#92, count#93]
+Keys: []
+Functions [1]: [avg(netpaid#39)]
+Aggregate Attributes [1]: [avg(netpaid#39)#95]
+Results [1]: [CheckOverflow((0.050000 * promote_precision(avg(netpaid#39)#95)), DecimalType(24,8), true) AS (0.05 * avg(netpaid))#96]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.sf100/simplified.txt
new file mode 100644
index 0000000000000..46fa83fcfed4b
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.sf100/simplified.txt
@@ -0,0 +1,183 @@
+WholeStageCodegen (14)
+  Project [c_last_name,c_first_name,s_store_name,paid]
+    Filter [sum(netpaid)]
+      Subquery #1
+        WholeStageCodegen (20)
+          HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count]
+            InputAdapter
+              Exchange #10
+                WholeStageCodegen (19)
+                  HashAggregate [netpaid] [sum,count,sum,count]
+                    HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum]
+                      InputAdapter
+                        Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #11
+                          WholeStageCodegen (18)
+                            HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum]
+                              Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state]
+                                SortMergeJoin [c_birth_country,s_zip,ca_country,ca_zip]
+                                  InputAdapter
+                                    WholeStageCodegen (15)
+                                      Sort [c_birth_country,s_zip]
+                                        InputAdapter
+                                          Exchange [c_birth_country,s_zip] #12
+                                            WholeStageCodegen (14)
+                                              Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country]
+                                                SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (11)
+                                                      Sort [ss_ticket_number,ss_item_sk]
+                                                        InputAdapter
+                                                          Exchange [ss_ticket_number,ss_item_sk] #13
+                                                            WholeStageCodegen (10)
+                                                              Project [ss_item_sk,ss_ticket_number,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country]
+                                                                SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                                                  InputAdapter
+                                                                    WholeStageCodegen (7)
+                                                                      Sort [ss_customer_sk]
+                                                                        InputAdapter
+                                                                          Exchange [ss_customer_sk] #14
+                                                                            WholeStageCodegen (6)
+                                                                              Project [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id]
+                                                                                SortMergeJoin [ss_item_sk,i_item_sk]
+                                                                                  InputAdapter
+                                                                                    WholeStageCodegen (3)
+                                                                                      Sort [ss_item_sk]
+                                                                                        InputAdapter
+                                                                                          Exchange [ss_item_sk] #15
+                                                                                            WholeStageCodegen (2)
+                                                                                              Project [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_net_paid,s_store_name,s_state,s_zip]
+                                                                                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                                                                  Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid]
+                                                                                                    Filter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk]
+                                                                                                      ColumnarToRow
+                                                                                                        InputAdapter
+                                                                                                          Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk]
+                                                                                                  InputAdapter
+                                                                                                    BroadcastExchange #16
+                                                                                                      WholeStageCodegen (1)
+                                                                                                        Project [s_store_sk,s_store_name,s_state,s_zip]
+                                                                                                          Filter [s_market_id,s_store_sk,s_zip]
+                                                                                                            ColumnarToRow
+                                                                                                              InputAdapter
+                                                                                                                Scan parquet default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip]
+                                                                                  InputAdapter
+                                                                                    WholeStageCodegen (5)
+                                                                                      Sort [i_item_sk]
+                                                                                        InputAdapter
+                                                                                          Exchange [i_item_sk] #17
+                                                                                            WholeStageCodegen (4)
+                                                                                              Filter [i_item_sk]
+                                                                                                ColumnarToRow
+                                                                                                  InputAdapter
+                                                                                                    Scan parquet default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id]
+                                                                  InputAdapter
+                                                                    WholeStageCodegen (9)
+                                                                      Sort [c_customer_sk]
+                                                                        InputAdapter
+                                                                          Exchange [c_customer_sk] #18
+                                                                            WholeStageCodegen (8)
+                                                                              Filter [c_customer_sk,c_birth_country]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country]
+                                                  InputAdapter
+                                                    WholeStageCodegen (13)
+                                                      Sort [sr_ticket_number,sr_item_sk]
+                                                        InputAdapter
+                                                          Exchange [sr_ticket_number,sr_item_sk] #19
+                                                            WholeStageCodegen (12)
+                                                              Project [sr_item_sk,sr_ticket_number]
+                                                                Filter [sr_ticket_number,sr_item_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk]
+                                  InputAdapter
+                                    WholeStageCodegen (17)
+                                      Sort [ca_country,ca_zip]
+                                        InputAdapter
+                                          Exchange [ca_country,ca_zip] #20
+                                            WholeStageCodegen (16)
+                                              Filter [ca_country,ca_zip]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.customer_address [ca_state,ca_zip,ca_country]
+      HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum(netpaid),sum,isEmpty]
+        InputAdapter
+          Exchange [c_last_name,c_first_name,s_store_name] #1
+            WholeStageCodegen (13)
+              HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty]
+                HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum]
+                  InputAdapter
+                    Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2
+                      WholeStageCodegen (12)
+                        HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum]
+                          Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state]
+                            BroadcastHashJoin [ss_store_sk,c_birth_country,s_store_sk,ca_country]
+                              Project [ss_store_sk,ss_net_paid,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country]
+                                SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                  InputAdapter
+                                    WholeStageCodegen (7)
+                                      Sort [ss_ticket_number,ss_item_sk]
+                                        InputAdapter
+                                          Exchange [ss_ticket_number,ss_item_sk] #3
+                                            WholeStageCodegen (6)
+                                              Project [ss_item_sk,ss_store_sk,ss_ticket_number,ss_net_paid,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country]
+                                                SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (3)
+                                                      Sort [ss_customer_sk]
+                                                        InputAdapter
+                                                          Exchange [ss_customer_sk] #4
+                                                            WholeStageCodegen (2)
+                                                              Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,i_current_price,i_size,i_color,i_units,i_manager_id]
+                                                                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                                  Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid]
+                                                                    Filter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk]
+                                                                  InputAdapter
+                                                                    BroadcastExchange #5
+                                                                      WholeStageCodegen (1)
+                                                                        Filter [i_color,i_item_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id]
+                                                  InputAdapter
+                                                    WholeStageCodegen (5)
+                                                      Sort [c_customer_sk]
+                                                        InputAdapter
+                                                          Exchange [c_customer_sk] #6
+                                                            WholeStageCodegen (4)
+                                                              Filter [c_customer_sk,c_birth_country]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country]
+                                  InputAdapter
+                                    WholeStageCodegen (9)
+                                      Sort [sr_ticket_number,sr_item_sk]
+                                        InputAdapter
+                                          Exchange [sr_ticket_number,sr_item_sk] #7
+                                            WholeStageCodegen (8)
+                                              Project [sr_item_sk,sr_ticket_number]
+                                                Filter [sr_ticket_number,sr_item_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk]
+                              InputAdapter
+                                BroadcastExchange #8
+                                  WholeStageCodegen (11)
+                                    Project [s_store_sk,s_store_name,s_state,ca_state,ca_country]
+                                      BroadcastHashJoin [s_zip,ca_zip]
+                                        InputAdapter
+                                          BroadcastExchange #9
+                                            WholeStageCodegen (10)
+                                              Project [s_store_sk,s_store_name,s_state,s_zip]
+                                                Filter [s_market_id,s_store_sk,s_zip]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip]
+                                        Filter [ca_country,ca_zip]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer_address [ca_state,ca_zip,ca_country]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/explain.txt
new file mode 100644
index 0000000000000..2b6c25252d1df
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/explain.txt
@@ -0,0 +1,527 @@
+== Physical Plan ==
+* Project (47)
++- * Filter (46)
+   +- * HashAggregate (45)
+      +- Exchange (44)
+         +- * HashAggregate (43)
+            +- * HashAggregate (42)
+               +- Exchange (41)
+                  +- * HashAggregate (40)
+                     +- * Project (39)
+                        +- * BroadcastHashJoin Inner BuildRight (38)
+                           :- * Project (33)
+                           :  +- * BroadcastHashJoin Inner BuildRight (32)
+                           :     :- * Project (27)
+                           :     :  +- * BroadcastHashJoin Inner BuildRight (26)
+                           :     :     :- * Project (21)
+                           :     :     :  +- * BroadcastHashJoin Inner BuildRight (20)
+                           :     :     :     :- * Project (14)
+                           :     :     :     :  +- * SortMergeJoin Inner (13)
+                           :     :     :     :     :- * Sort (6)
+                           :     :     :     :     :  +- Exchange (5)
+                           :     :     :     :     :     +- * Project (4)
+                           :     :     :     :     :        +- * Filter (3)
+                           :     :     :     :     :           +- * ColumnarToRow (2)
+                           :     :     :     :     :              +- Scan parquet default.store_sales (1)
+                           :     :     :     :     +- * Sort (12)
+                           :     :     :     :        +- Exchange (11)
+                           :     :     :     :           +- * Project (10)
+                           :     :     :     :              +- * Filter (9)
+                           :     :     :     :                 +- * ColumnarToRow (8)
+                           :     :     :     :                    +- Scan parquet default.store_returns (7)
+                           :     :     :     +- BroadcastExchange (19)
+                           :     :     :        +- * Project (18)
+                           :     :     :           +- * Filter (17)
+                           :     :     :              +- * ColumnarToRow (16)
+                           :     :     :                 +- Scan parquet default.store (15)
+                           :     :     +- BroadcastExchange (25)
+                           :     :        +- * Filter (24)
+                           :     :           +- * ColumnarToRow (23)
+                           :     :              +- Scan parquet default.item (22)
+                           :     +- BroadcastExchange (31)
+                           :        +- * Filter (30)
+                           :           +- * ColumnarToRow (29)
+                           :              +- Scan parquet default.customer (28)
+                           +- BroadcastExchange (37)
+                              +- * Filter (36)
+                                 +- * ColumnarToRow (35)
+                                    +- Scan parquet default.customer_address (34)
+
+
+(1) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_net_paid:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6]
+
+(3) Filter [codegen id : 1]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6]
+Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2))
+
+(4) Project [codegen id : 1]
+Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6]
+
+(5) Exchange
+Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5]
+Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#7]
+
+(6) Sort [codegen id : 2]
+Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5]
+Arguments: [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(7) Scan parquet default.store_returns
+Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int>
+
+(8) ColumnarToRow [codegen id : 3]
+Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10]
+
+(9) Filter [codegen id : 3]
+Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10]
+Condition : (isnotnull(sr_ticket_number#9) AND isnotnull(sr_item_sk#8))
+
+(10) Project [codegen id : 3]
+Output [2]: [sr_item_sk#8, sr_ticket_number#9]
+Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10]
+
+(11) Exchange
+Input [2]: [sr_item_sk#8, sr_ticket_number#9]
+Arguments: hashpartitioning(sr_ticket_number#9, sr_item_sk#8, 5), ENSURE_REQUIREMENTS, [id=#11]
+
+(12) Sort [codegen id : 4]
+Input [2]: [sr_item_sk#8, sr_ticket_number#9]
+Arguments: [sr_ticket_number#9 ASC NULLS FIRST, sr_item_sk#8 ASC NULLS FIRST], false, 0
+
+(13) SortMergeJoin [codegen id : 9]
+Left keys [2]: [ss_ticket_number#4, ss_item_sk#1]
+Right keys [2]: [sr_ticket_number#9, sr_item_sk#8]
+Join condition: None
+
+(14) Project [codegen id : 9]
+Output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5]
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#8, sr_ticket_number#9]
+
+(15) Scan parquet default.store
+Output [5]: [s_store_sk#12, s_store_name#13, s_market_id#14, s_state#15, s_zip#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_market_id:int,s_state:string,s_zip:string>
+
+(16) ColumnarToRow [codegen id : 5]
+Input [5]: [s_store_sk#12, s_store_name#13, s_market_id#14, s_state#15, s_zip#16]
+
+(17) Filter [codegen id : 5]
+Input [5]: [s_store_sk#12, s_store_name#13, s_market_id#14, s_state#15, s_zip#16]
+Condition : (((isnotnull(s_market_id#14) AND (s_market_id#14 = 8)) AND isnotnull(s_store_sk#12)) AND isnotnull(s_zip#16))
+
+(18) Project [codegen id : 5]
+Output [4]: [s_store_sk#12, s_store_name#13, s_state#15, s_zip#16]
+Input [5]: [s_store_sk#12, s_store_name#13, s_market_id#14, s_state#15, s_zip#16]
+
+(19) BroadcastExchange
+Input [4]: [s_store_sk#12, s_store_name#13, s_state#15, s_zip#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(20) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#12]
+Join condition: None
+
+(21) Project [codegen id : 9]
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#13, s_state#15, s_zip#16]
+Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#12, s_store_name#13, s_state#15, s_zip#16]
+
+(22) Scan parquet default.item
+Output [6]: [i_item_sk#18, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_color), EqualTo(i_color,chiffon             ), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_current_price:decimal(7,2),i_size:string,i_color:string,i_units:string,i_manager_id:int>
+
+(23) ColumnarToRow [codegen id : 6]
+Input [6]: [i_item_sk#18, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23]
+
+(24) Filter [codegen id : 6]
+Input [6]: [i_item_sk#18, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23]
+Condition : ((isnotnull(i_color#21) AND (i_color#21 = chiffon             )) AND isnotnull(i_item_sk#18))
+
+(25) BroadcastExchange
+Input [6]: [i_item_sk#18, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#24]
+
+(26) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#18]
+Join condition: None
+
+(27) Project [codegen id : 9]
+Output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#13, s_state#15, s_zip#16, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23]
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#13, s_state#15, s_zip#16, i_item_sk#18, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23]
+
+(28) Scan parquet default.customer
+Output [4]: [c_customer_sk#25, c_first_name#26, c_last_name#27, c_birth_country#28]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_birth_country)]
+ReadSchema: struct<c_customer_sk:int,c_first_name:string,c_last_name:string,c_birth_country:string>
+
+(29) ColumnarToRow [codegen id : 7]
+Input [4]: [c_customer_sk#25, c_first_name#26, c_last_name#27, c_birth_country#28]
+
+(30) Filter [codegen id : 7]
+Input [4]: [c_customer_sk#25, c_first_name#26, c_last_name#27, c_birth_country#28]
+Condition : (isnotnull(c_customer_sk#25) AND isnotnull(c_birth_country#28))
+
+(31) BroadcastExchange
+Input [4]: [c_customer_sk#25, c_first_name#26, c_last_name#27, c_birth_country#28]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#29]
+
+(32) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_customer_sk#2]
+Right keys [1]: [c_customer_sk#25]
+Join condition: None
+
+(33) Project [codegen id : 9]
+Output [12]: [ss_net_paid#5, s_store_name#13, s_state#15, s_zip#16, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23, c_first_name#26, c_last_name#27, c_birth_country#28]
+Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#13, s_state#15, s_zip#16, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23, c_customer_sk#25, c_first_name#26, c_last_name#27, c_birth_country#28]
+
+(34) Scan parquet default.customer_address
+Output [3]: [ca_state#30, ca_zip#31, ca_country#32]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_country), IsNotNull(ca_zip)]
+ReadSchema: struct<ca_state:string,ca_zip:string,ca_country:string>
+
+(35) ColumnarToRow [codegen id : 8]
+Input [3]: [ca_state#30, ca_zip#31, ca_country#32]
+
+(36) Filter [codegen id : 8]
+Input [3]: [ca_state#30, ca_zip#31, ca_country#32]
+Condition : (isnotnull(ca_country#32) AND isnotnull(ca_zip#31))
+
+(37) BroadcastExchange
+Input [3]: [ca_state#30, ca_zip#31, ca_country#32]
+Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [id=#33]
+
+(38) BroadcastHashJoin [codegen id : 9]
+Left keys [2]: [c_birth_country#28, s_zip#16]
+Right keys [2]: [upper(ca_country#32), ca_zip#31]
+Join condition: None
+
+(39) Project [codegen id : 9]
+Output [11]: [ss_net_paid#5, s_store_name#13, s_state#15, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23, c_first_name#26, c_last_name#27, ca_state#30]
+Input [15]: [ss_net_paid#5, s_store_name#13, s_state#15, s_zip#16, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23, c_first_name#26, c_last_name#27, c_birth_country#28, ca_state#30, ca_zip#31, ca_country#32]
+
+(40) HashAggregate [codegen id : 9]
+Input [11]: [ss_net_paid#5, s_store_name#13, s_state#15, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23, c_first_name#26, c_last_name#27, ca_state#30]
+Keys [10]: [c_last_name#27, c_first_name#26, s_store_name#13, ca_state#30, s_state#15, i_color#21, i_current_price#19, i_manager_id#23, i_units#22, i_size#20]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))]
+Aggregate Attributes [1]: [sum#34]
+Results [11]: [c_last_name#27, c_first_name#26, s_store_name#13, ca_state#30, s_state#15, i_color#21, i_current_price#19, i_manager_id#23, i_units#22, i_size#20, sum#35]
+
+(41) Exchange
+Input [11]: [c_last_name#27, c_first_name#26, s_store_name#13, ca_state#30, s_state#15, i_color#21, i_current_price#19, i_manager_id#23, i_units#22, i_size#20, sum#35]
+Arguments: hashpartitioning(c_last_name#27, c_first_name#26, s_store_name#13, ca_state#30, s_state#15, i_color#21, i_current_price#19, i_manager_id#23, i_units#22, i_size#20, 5), ENSURE_REQUIREMENTS, [id=#36]
+
+(42) HashAggregate [codegen id : 10]
+Input [11]: [c_last_name#27, c_first_name#26, s_store_name#13, ca_state#30, s_state#15, i_color#21, i_current_price#19, i_manager_id#23, i_units#22, i_size#20, sum#35]
+Keys [10]: [c_last_name#27, c_first_name#26, s_store_name#13, ca_state#30, s_state#15, i_color#21, i_current_price#19, i_manager_id#23, i_units#22, i_size#20]
+Functions [1]: [sum(UnscaledValue(ss_net_paid#5))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#37]
+Results [4]: [c_last_name#27, c_first_name#26, s_store_name#13, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#37,17,2) AS netpaid#38]
+
+(43) HashAggregate [codegen id : 10]
+Input [4]: [c_last_name#27, c_first_name#26, s_store_name#13, netpaid#38]
+Keys [3]: [c_last_name#27, c_first_name#26, s_store_name#13]
+Functions [1]: [partial_sum(netpaid#38)]
+Aggregate Attributes [2]: [sum#39, isEmpty#40]
+Results [5]: [c_last_name#27, c_first_name#26, s_store_name#13, sum#41, isEmpty#42]
+
+(44) Exchange
+Input [5]: [c_last_name#27, c_first_name#26, s_store_name#13, sum#41, isEmpty#42]
+Arguments: hashpartitioning(c_last_name#27, c_first_name#26, s_store_name#13, 5), ENSURE_REQUIREMENTS, [id=#43]
+
+(45) HashAggregate [codegen id : 11]
+Input [5]: [c_last_name#27, c_first_name#26, s_store_name#13, sum#41, isEmpty#42]
+Keys [3]: [c_last_name#27, c_first_name#26, s_store_name#13]
+Functions [1]: [sum(netpaid#38)]
+Aggregate Attributes [1]: [sum(netpaid#38)#44]
+Results [5]: [c_last_name#27, c_first_name#26, s_store_name#13, sum(netpaid#38)#44 AS paid#45, sum(netpaid#38)#44 AS sum(netpaid#38)#46]
+
+(46) Filter [codegen id : 11]
+Input [5]: [c_last_name#27, c_first_name#26, s_store_name#13, paid#45, sum(netpaid#38)#46]
+Condition : (isnotnull(sum(netpaid#38)#46) AND (cast(sum(netpaid#38)#46 as decimal(33,8)) > cast(Subquery scalar-subquery#47, [id=#48] as decimal(33,8))))
+
+(47) Project [codegen id : 11]
+Output [4]: [c_last_name#27, c_first_name#26, s_store_name#13, paid#45]
+Input [5]: [c_last_name#27, c_first_name#26, s_store_name#13, paid#45, sum(netpaid#38)#46]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#47, [id=#48]
+* HashAggregate (92)
++- Exchange (91)
+   +- * HashAggregate (90)
+      +- * HashAggregate (89)
+         +- Exchange (88)
+            +- * HashAggregate (87)
+               +- * Project (86)
+                  +- * BroadcastHashJoin Inner BuildRight (85)
+                     :- * Project (80)
+                     :  +- * BroadcastHashJoin Inner BuildRight (79)
+                     :     :- * Project (74)
+                     :     :  +- * BroadcastHashJoin Inner BuildRight (73)
+                     :     :     :- * Project (68)
+                     :     :     :  +- * BroadcastHashJoin Inner BuildRight (67)
+                     :     :     :     :- * Project (61)
+                     :     :     :     :  +- * SortMergeJoin Inner (60)
+                     :     :     :     :     :- * Sort (53)
+                     :     :     :     :     :  +- Exchange (52)
+                     :     :     :     :     :     +- * Project (51)
+                     :     :     :     :     :        +- * Filter (50)
+                     :     :     :     :     :           +- * ColumnarToRow (49)
+                     :     :     :     :     :              +- Scan parquet default.store_sales (48)
+                     :     :     :     :     +- * Sort (59)
+                     :     :     :     :        +- Exchange (58)
+                     :     :     :     :           +- * Project (57)
+                     :     :     :     :              +- * Filter (56)
+                     :     :     :     :                 +- * ColumnarToRow (55)
+                     :     :     :     :                    +- Scan parquet default.store_returns (54)
+                     :     :     :     +- BroadcastExchange (66)
+                     :     :     :        +- * Project (65)
+                     :     :     :           +- * Filter (64)
+                     :     :     :              +- * ColumnarToRow (63)
+                     :     :     :                 +- Scan parquet default.store (62)
+                     :     :     +- BroadcastExchange (72)
+                     :     :        +- * Filter (71)
+                     :     :           +- * ColumnarToRow (70)
+                     :     :              +- Scan parquet default.item (69)
+                     :     +- BroadcastExchange (78)
+                     :        +- * Filter (77)
+                     :           +- * ColumnarToRow (76)
+                     :              +- Scan parquet default.customer (75)
+                     +- BroadcastExchange (84)
+                        +- * Filter (83)
+                           +- * ColumnarToRow (82)
+                              +- Scan parquet default.customer_address (81)
+
+
+(48) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53, ss_sold_date_sk#54]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_net_paid:decimal(7,2)>
+
+(49) ColumnarToRow [codegen id : 1]
+Input [6]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53, ss_sold_date_sk#54]
+
+(50) Filter [codegen id : 1]
+Input [6]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53, ss_sold_date_sk#54]
+Condition : (((isnotnull(ss_ticket_number#52) AND isnotnull(ss_item_sk#49)) AND isnotnull(ss_store_sk#51)) AND isnotnull(ss_customer_sk#50))
+
+(51) Project [codegen id : 1]
+Output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53]
+Input [6]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53, ss_sold_date_sk#54]
+
+(52) Exchange
+Input [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53]
+Arguments: hashpartitioning(ss_ticket_number#52, ss_item_sk#49, 5), ENSURE_REQUIREMENTS, [id=#55]
+
+(53) Sort [codegen id : 2]
+Input [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53]
+Arguments: [ss_ticket_number#52 ASC NULLS FIRST, ss_item_sk#49 ASC NULLS FIRST], false, 0
+
+(54) Scan parquet default.store_returns
+Output [3]: [sr_item_sk#56, sr_ticket_number#57, sr_returned_date_sk#58]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int>
+
+(55) ColumnarToRow [codegen id : 3]
+Input [3]: [sr_item_sk#56, sr_ticket_number#57, sr_returned_date_sk#58]
+
+(56) Filter [codegen id : 3]
+Input [3]: [sr_item_sk#56, sr_ticket_number#57, sr_returned_date_sk#58]
+Condition : (isnotnull(sr_ticket_number#57) AND isnotnull(sr_item_sk#56))
+
+(57) Project [codegen id : 3]
+Output [2]: [sr_item_sk#56, sr_ticket_number#57]
+Input [3]: [sr_item_sk#56, sr_ticket_number#57, sr_returned_date_sk#58]
+
+(58) Exchange
+Input [2]: [sr_item_sk#56, sr_ticket_number#57]
+Arguments: hashpartitioning(sr_ticket_number#57, sr_item_sk#56, 5), ENSURE_REQUIREMENTS, [id=#59]
+
+(59) Sort [codegen id : 4]
+Input [2]: [sr_item_sk#56, sr_ticket_number#57]
+Arguments: [sr_ticket_number#57 ASC NULLS FIRST, sr_item_sk#56 ASC NULLS FIRST], false, 0
+
+(60) SortMergeJoin [codegen id : 9]
+Left keys [2]: [ss_ticket_number#52, ss_item_sk#49]
+Right keys [2]: [sr_ticket_number#57, sr_item_sk#56]
+Join condition: None
+
+(61) Project [codegen id : 9]
+Output [4]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53]
+Input [7]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53, sr_item_sk#56, sr_ticket_number#57]
+
+(62) Scan parquet default.store
+Output [5]: [s_store_sk#60, s_store_name#61, s_market_id#62, s_state#63, s_zip#64]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_market_id:int,s_state:string,s_zip:string>
+
+(63) ColumnarToRow [codegen id : 5]
+Input [5]: [s_store_sk#60, s_store_name#61, s_market_id#62, s_state#63, s_zip#64]
+
+(64) Filter [codegen id : 5]
+Input [5]: [s_store_sk#60, s_store_name#61, s_market_id#62, s_state#63, s_zip#64]
+Condition : (((isnotnull(s_market_id#62) AND (s_market_id#62 = 8)) AND isnotnull(s_store_sk#60)) AND isnotnull(s_zip#64))
+
+(65) Project [codegen id : 5]
+Output [4]: [s_store_sk#60, s_store_name#61, s_state#63, s_zip#64]
+Input [5]: [s_store_sk#60, s_store_name#61, s_market_id#62, s_state#63, s_zip#64]
+
+(66) BroadcastExchange
+Input [4]: [s_store_sk#60, s_store_name#61, s_state#63, s_zip#64]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#65]
+
+(67) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_store_sk#51]
+Right keys [1]: [s_store_sk#60]
+Join condition: None
+
+(68) Project [codegen id : 9]
+Output [6]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#61, s_state#63, s_zip#64]
+Input [8]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53, s_store_sk#60, s_store_name#61, s_state#63, s_zip#64]
+
+(69) Scan parquet default.item
+Output [6]: [i_item_sk#66, i_current_price#67, i_size#68, i_color#69, i_units#70, i_manager_id#71]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_current_price:decimal(7,2),i_size:string,i_color:string,i_units:string,i_manager_id:int>
+
+(70) ColumnarToRow [codegen id : 6]
+Input [6]: [i_item_sk#66, i_current_price#67, i_size#68, i_color#69, i_units#70, i_manager_id#71]
+
+(71) Filter [codegen id : 6]
+Input [6]: [i_item_sk#66, i_current_price#67, i_size#68, i_color#69, i_units#70, i_manager_id#71]
+Condition : isnotnull(i_item_sk#66)
+
+(72) BroadcastExchange
+Input [6]: [i_item_sk#66, i_current_price#67, i_size#68, i_color#69, i_units#70, i_manager_id#71]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#72]
+
+(73) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_item_sk#49]
+Right keys [1]: [i_item_sk#66]
+Join condition: None
+
+(74) Project [codegen id : 9]
+Output [10]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#61, s_state#63, s_zip#64, i_current_price#67, i_size#68, i_color#69, i_units#70, i_manager_id#71]
+Input [12]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#61, s_state#63, s_zip#64, i_item_sk#66, i_current_price#67, i_size#68, i_color#69, i_units#70, i_manager_id#71]
+
+(75) Scan parquet default.customer
+Output [4]: [c_customer_sk#73, c_first_name#74, c_last_name#75, c_birth_country#76]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_birth_country)]
+ReadSchema: struct<c_customer_sk:int,c_first_name:string,c_last_name:string,c_birth_country:string>
+
+(76) ColumnarToRow [codegen id : 7]
+Input [4]: [c_customer_sk#73, c_first_name#74, c_last_name#75, c_birth_country#76]
+
+(77) Filter [codegen id : 7]
+Input [4]: [c_customer_sk#73, c_first_name#74, c_last_name#75, c_birth_country#76]
+Condition : (isnotnull(c_customer_sk#73) AND isnotnull(c_birth_country#76))
+
+(78) BroadcastExchange
+Input [4]: [c_customer_sk#73, c_first_name#74, c_last_name#75, c_birth_country#76]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#77]
+
+(79) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_customer_sk#50]
+Right keys [1]: [c_customer_sk#73]
+Join condition: None
+
+(80) Project [codegen id : 9]
+Output [12]: [ss_net_paid#53, s_store_name#61, s_state#63, s_zip#64, i_current_price#67, i_size#68, i_color#69, i_units#70, i_manager_id#71, c_first_name#74, c_last_name#75, c_birth_country#76]
+Input [14]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#61, s_state#63, s_zip#64, i_current_price#67, i_size#68, i_color#69, i_units#70, i_manager_id#71, c_customer_sk#73, c_first_name#74, c_last_name#75, c_birth_country#76]
+
+(81) Scan parquet default.customer_address
+Output [3]: [ca_state#78, ca_zip#79, ca_country#80]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_country), IsNotNull(ca_zip)]
+ReadSchema: struct<ca_state:string,ca_zip:string,ca_country:string>
+
+(82) ColumnarToRow [codegen id : 8]
+Input [3]: [ca_state#78, ca_zip#79, ca_country#80]
+
+(83) Filter [codegen id : 8]
+Input [3]: [ca_state#78, ca_zip#79, ca_country#80]
+Condition : (isnotnull(ca_country#80) AND isnotnull(ca_zip#79))
+
+(84) BroadcastExchange
+Input [3]: [ca_state#78, ca_zip#79, ca_country#80]
+Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [id=#81]
+
+(85) BroadcastHashJoin [codegen id : 9]
+Left keys [2]: [c_birth_country#76, s_zip#64]
+Right keys [2]: [upper(ca_country#80), ca_zip#79]
+Join condition: None
+
+(86) Project [codegen id : 9]
+Output [11]: [ss_net_paid#53, s_store_name#61, s_state#63, i_current_price#67, i_size#68, i_color#69, i_units#70, i_manager_id#71, c_first_name#74, c_last_name#75, ca_state#78]
+Input [15]: [ss_net_paid#53, s_store_name#61, s_state#63, s_zip#64, i_current_price#67, i_size#68, i_color#69, i_units#70, i_manager_id#71, c_first_name#74, c_last_name#75, c_birth_country#76, ca_state#78, ca_zip#79, ca_country#80]
+
+(87) HashAggregate [codegen id : 9]
+Input [11]: [ss_net_paid#53, s_store_name#61, s_state#63, i_current_price#67, i_size#68, i_color#69, i_units#70, i_manager_id#71, c_first_name#74, c_last_name#75, ca_state#78]
+Keys [10]: [c_last_name#75, c_first_name#74, s_store_name#61, ca_state#78, s_state#63, i_color#69, i_current_price#67, i_manager_id#71, i_units#70, i_size#68]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#53))]
+Aggregate Attributes [1]: [sum#82]
+Results [11]: [c_last_name#75, c_first_name#74, s_store_name#61, ca_state#78, s_state#63, i_color#69, i_current_price#67, i_manager_id#71, i_units#70, i_size#68, sum#83]
+
+(88) Exchange
+Input [11]: [c_last_name#75, c_first_name#74, s_store_name#61, ca_state#78, s_state#63, i_color#69, i_current_price#67, i_manager_id#71, i_units#70, i_size#68, sum#83]
+Arguments: hashpartitioning(c_last_name#75, c_first_name#74, s_store_name#61, ca_state#78, s_state#63, i_color#69, i_current_price#67, i_manager_id#71, i_units#70, i_size#68, 5), ENSURE_REQUIREMENTS, [id=#84]
+
+(89) HashAggregate [codegen id : 10]
+Input [11]: [c_last_name#75, c_first_name#74, s_store_name#61, ca_state#78, s_state#63, i_color#69, i_current_price#67, i_manager_id#71, i_units#70, i_size#68, sum#83]
+Keys [10]: [c_last_name#75, c_first_name#74, s_store_name#61, ca_state#78, s_state#63, i_color#69, i_current_price#67, i_manager_id#71, i_units#70, i_size#68]
+Functions [1]: [sum(UnscaledValue(ss_net_paid#53))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#53))#85]
+Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#53))#85,17,2) AS netpaid#38]
+
+(90) HashAggregate [codegen id : 10]
+Input [1]: [netpaid#38]
+Keys: []
+Functions [1]: [partial_avg(netpaid#38)]
+Aggregate Attributes [2]: [sum#86, count#87]
+Results [2]: [sum#88, count#89]
+
+(91) Exchange
+Input [2]: [sum#88, count#89]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#90]
+
+(92) HashAggregate [codegen id : 11]
+Input [2]: [sum#88, count#89]
+Keys: []
+Functions [1]: [avg(netpaid#38)]
+Aggregate Attributes [1]: [avg(netpaid#38)#91]
+Results [1]: [CheckOverflow((0.050000 * promote_precision(avg(netpaid#38)#91)), DecimalType(24,8), true) AS (0.05 * avg(netpaid))#92]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/simplified.txt
new file mode 100644
index 0000000000000..6d50a1e3f35a0
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/simplified.txt
@@ -0,0 +1,147 @@
+WholeStageCodegen (11)
+  Project [c_last_name,c_first_name,s_store_name,paid]
+    Filter [sum(netpaid)]
+      Subquery #1
+        WholeStageCodegen (11)
+          HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count]
+            InputAdapter
+              Exchange #9
+                WholeStageCodegen (10)
+                  HashAggregate [netpaid] [sum,count,sum,count]
+                    HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum]
+                      InputAdapter
+                        Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10
+                          WholeStageCodegen (9)
+                            HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum]
+                              Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state]
+                                BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip]
+                                  Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country]
+                                    BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+                                      Project [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id]
+                                        BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                          Project [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip]
+                                            BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                              Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid]
+                                                SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (2)
+                                                      Sort [ss_ticket_number,ss_item_sk]
+                                                        InputAdapter
+                                                          Exchange [ss_ticket_number,ss_item_sk] #11
+                                                            WholeStageCodegen (1)
+                                                              Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid]
+                                                                Filter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (4)
+                                                      Sort [sr_ticket_number,sr_item_sk]
+                                                        InputAdapter
+                                                          Exchange [sr_ticket_number,sr_item_sk] #12
+                                                            WholeStageCodegen (3)
+                                                              Project [sr_item_sk,sr_ticket_number]
+                                                                Filter [sr_ticket_number,sr_item_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk]
+                                              InputAdapter
+                                                BroadcastExchange #13
+                                                  WholeStageCodegen (5)
+                                                    Project [s_store_sk,s_store_name,s_state,s_zip]
+                                                      Filter [s_market_id,s_store_sk,s_zip]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip]
+                                          InputAdapter
+                                            BroadcastExchange #14
+                                              WholeStageCodegen (6)
+                                                Filter [i_item_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id]
+                                      InputAdapter
+                                        BroadcastExchange #15
+                                          WholeStageCodegen (7)
+                                            Filter [c_customer_sk,c_birth_country]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country]
+                                  InputAdapter
+                                    BroadcastExchange #16
+                                      WholeStageCodegen (8)
+                                        Filter [ca_country,ca_zip]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer_address [ca_state,ca_zip,ca_country]
+      HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum(netpaid),sum,isEmpty]
+        InputAdapter
+          Exchange [c_last_name,c_first_name,s_store_name] #1
+            WholeStageCodegen (10)
+              HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty]
+                HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum]
+                  InputAdapter
+                    Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2
+                      WholeStageCodegen (9)
+                        HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum]
+                          Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state]
+                            BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip]
+                              Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country]
+                                BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+                                  Project [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id]
+                                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                      Project [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip]
+                                        BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                          Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid]
+                                            SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                              InputAdapter
+                                                WholeStageCodegen (2)
+                                                  Sort [ss_ticket_number,ss_item_sk]
+                                                    InputAdapter
+                                                      Exchange [ss_ticket_number,ss_item_sk] #3
+                                                        WholeStageCodegen (1)
+                                                          Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid]
+                                                            Filter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk]
+                                              InputAdapter
+                                                WholeStageCodegen (4)
+                                                  Sort [sr_ticket_number,sr_item_sk]
+                                                    InputAdapter
+                                                      Exchange [sr_ticket_number,sr_item_sk] #4
+                                                        WholeStageCodegen (3)
+                                                          Project [sr_item_sk,sr_ticket_number]
+                                                            Filter [sr_ticket_number,sr_item_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk]
+                                          InputAdapter
+                                            BroadcastExchange #5
+                                              WholeStageCodegen (5)
+                                                Project [s_store_sk,s_store_name,s_state,s_zip]
+                                                  Filter [s_market_id,s_store_sk,s_zip]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip]
+                                      InputAdapter
+                                        BroadcastExchange #6
+                                          WholeStageCodegen (6)
+                                            Filter [i_color,i_item_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id]
+                                  InputAdapter
+                                    BroadcastExchange #7
+                                      WholeStageCodegen (7)
+                                        Filter [c_customer_sk,c_birth_country]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country]
+                              InputAdapter
+                                BroadcastExchange #8
+                                  WholeStageCodegen (8)
+                                    Filter [ca_country,ca_zip]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.customer_address [ca_state,ca_zip,ca_country]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.sf100/explain.txt
new file mode 100644
index 0000000000000..60be358241b9e
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.sf100/explain.txt
@@ -0,0 +1,336 @@
+== Physical Plan ==
+TakeOrderedAndProject (57)
++- * HashAggregate (56)
+   +- Exchange (55)
+      +- * HashAggregate (54)
+         +- * Project (53)
+            +- * SortMergeJoin Inner (52)
+               :- * Sort (43)
+               :  +- Exchange (42)
+               :     +- * Project (41)
+               :        +- * SortMergeJoin Inner (40)
+               :           :- * Sort (27)
+               :           :  +- Exchange (26)
+               :           :     +- * Project (25)
+               :           :        +- * SortMergeJoin Inner (24)
+               :           :           :- * Sort (18)
+               :           :           :  +- Exchange (17)
+               :           :           :     +- * Project (16)
+               :           :           :        +- * BroadcastHashJoin Inner BuildRight (15)
+               :           :           :           :- * Project (10)
+               :           :           :           :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :           :           :           :     :- * Filter (3)
+               :           :           :           :     :  +- * ColumnarToRow (2)
+               :           :           :           :     :     +- Scan parquet default.store_sales (1)
+               :           :           :           :     +- BroadcastExchange (8)
+               :           :           :           :        +- * Project (7)
+               :           :           :           :           +- * Filter (6)
+               :           :           :           :              +- * ColumnarToRow (5)
+               :           :           :           :                 +- Scan parquet default.date_dim (4)
+               :           :           :           +- BroadcastExchange (14)
+               :           :           :              +- * Filter (13)
+               :           :           :                 +- * ColumnarToRow (12)
+               :           :           :                    +- Scan parquet default.store (11)
+               :           :           +- * Sort (23)
+               :           :              +- Exchange (22)
+               :           :                 +- * Filter (21)
+               :           :                    +- * ColumnarToRow (20)
+               :           :                       +- Scan parquet default.item (19)
+               :           +- * Sort (39)
+               :              +- Exchange (38)
+               :                 +- * Project (37)
+               :                    +- * BroadcastHashJoin Inner BuildRight (36)
+               :                       :- * Filter (30)
+               :                       :  +- * ColumnarToRow (29)
+               :                       :     +- Scan parquet default.store_returns (28)
+               :                       +- BroadcastExchange (35)
+               :                          +- * Project (34)
+               :                             +- * Filter (33)
+               :                                +- * ColumnarToRow (32)
+               :                                   +- Scan parquet default.date_dim (31)
+               +- * Sort (51)
+                  +- Exchange (50)
+                     +- * Project (49)
+                        +- * BroadcastHashJoin Inner BuildRight (48)
+                           :- * Filter (46)
+                           :  +- * ColumnarToRow (45)
+                           :     +- Scan parquet default.catalog_sales (44)
+                           +- ReusedExchange (47)
+
+
+(1) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6]
+
+(3) Filter [codegen id : 3]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6]
+Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#8, d_year#9, d_moy#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,4), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#8, d_year#9, d_moy#10]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#8, d_year#9, d_moy#10]
+Condition : ((((isnotnull(d_moy#10) AND isnotnull(d_year#9)) AND (d_moy#10 = 4)) AND (d_year#9 = 2001)) AND isnotnull(d_date_sk#8))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#8]
+Input [3]: [d_date_sk#8, d_year#9, d_moy#10]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#11]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#6]
+Right keys [1]: [d_date_sk#8]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5]
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6, d_date_sk#8]
+
+(11) Scan parquet default.store
+Output [3]: [s_store_sk#12, s_store_id#13, s_store_name#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string,s_store_name:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [s_store_sk#12, s_store_id#13, s_store_name#14]
+
+(13) Filter [codegen id : 2]
+Input [3]: [s_store_sk#12, s_store_id#13, s_store_name#14]
+Condition : isnotnull(s_store_sk#12)
+
+(14) BroadcastExchange
+Input [3]: [s_store_sk#12, s_store_id#13, s_store_name#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#15]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#12]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#4, ss_net_profit#5, s_store_id#13, s_store_name#14]
+Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, s_store_sk#12, s_store_id#13, s_store_name#14]
+
+(17) Exchange
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#4, ss_net_profit#5, s_store_id#13, s_store_name#14]
+Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#16]
+
+(18) Sort [codegen id : 4]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#4, ss_net_profit#5, s_store_id#13, s_store_name#14]
+Arguments: [ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(19) Scan parquet default.item
+Output [3]: [i_item_sk#17, i_item_id#18, i_item_desc#19]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_item_desc:string>
+
+(20) ColumnarToRow [codegen id : 5]
+Input [3]: [i_item_sk#17, i_item_id#18, i_item_desc#19]
+
+(21) Filter [codegen id : 5]
+Input [3]: [i_item_sk#17, i_item_id#18, i_item_desc#19]
+Condition : isnotnull(i_item_sk#17)
+
+(22) Exchange
+Input [3]: [i_item_sk#17, i_item_id#18, i_item_desc#19]
+Arguments: hashpartitioning(i_item_sk#17, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(23) Sort [codegen id : 6]
+Input [3]: [i_item_sk#17, i_item_id#18, i_item_desc#19]
+Arguments: [i_item_sk#17 ASC NULLS FIRST], false, 0
+
+(24) SortMergeJoin [codegen id : 7]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#17]
+Join condition: None
+
+(25) Project [codegen id : 7]
+Output [8]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#4, ss_net_profit#5, s_store_id#13, s_store_name#14, i_item_id#18, i_item_desc#19]
+Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#4, ss_net_profit#5, s_store_id#13, s_store_name#14, i_item_sk#17, i_item_id#18, i_item_desc#19]
+
+(26) Exchange
+Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#4, ss_net_profit#5, s_store_id#13, s_store_name#14, i_item_id#18, i_item_desc#19]
+Arguments: hashpartitioning(ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(27) Sort [codegen id : 8]
+Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#4, ss_net_profit#5, s_store_id#13, s_store_name#14, i_item_id#18, i_item_desc#19]
+Arguments: [ss_customer_sk#2 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST], false, 0
+
+(28) Scan parquet default.store_returns
+Output [5]: [sr_item_sk#22, sr_customer_sk#23, sr_ticket_number#24, sr_net_loss#25, sr_returned_date_sk#26]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(sr_returned_date_sk#26), dynamicpruningexpression(sr_returned_date_sk#26 IN dynamicpruning#27)]
+PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)]
+ReadSchema: struct<sr_item_sk:int,sr_customer_sk:int,sr_ticket_number:int,sr_net_loss:decimal(7,2)>
+
+(29) ColumnarToRow [codegen id : 10]
+Input [5]: [sr_item_sk#22, sr_customer_sk#23, sr_ticket_number#24, sr_net_loss#25, sr_returned_date_sk#26]
+
+(30) Filter [codegen id : 10]
+Input [5]: [sr_item_sk#22, sr_customer_sk#23, sr_ticket_number#24, sr_net_loss#25, sr_returned_date_sk#26]
+Condition : ((isnotnull(sr_customer_sk#23) AND isnotnull(sr_item_sk#22)) AND isnotnull(sr_ticket_number#24))
+
+(31) Scan parquet default.date_dim
+Output [3]: [d_date_sk#28, d_year#29, d_moy#30]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,10), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(32) ColumnarToRow [codegen id : 9]
+Input [3]: [d_date_sk#28, d_year#29, d_moy#30]
+
+(33) Filter [codegen id : 9]
+Input [3]: [d_date_sk#28, d_year#29, d_moy#30]
+Condition : (((((isnotnull(d_moy#30) AND isnotnull(d_year#29)) AND (d_moy#30 >= 4)) AND (d_moy#30 <= 10)) AND (d_year#29 = 2001)) AND isnotnull(d_date_sk#28))
+
+(34) Project [codegen id : 9]
+Output [1]: [d_date_sk#28]
+Input [3]: [d_date_sk#28, d_year#29, d_moy#30]
+
+(35) BroadcastExchange
+Input [1]: [d_date_sk#28]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#31]
+
+(36) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [sr_returned_date_sk#26]
+Right keys [1]: [d_date_sk#28]
+Join condition: None
+
+(37) Project [codegen id : 10]
+Output [4]: [sr_item_sk#22, sr_customer_sk#23, sr_ticket_number#24, sr_net_loss#25]
+Input [6]: [sr_item_sk#22, sr_customer_sk#23, sr_ticket_number#24, sr_net_loss#25, sr_returned_date_sk#26, d_date_sk#28]
+
+(38) Exchange
+Input [4]: [sr_item_sk#22, sr_customer_sk#23, sr_ticket_number#24, sr_net_loss#25]
+Arguments: hashpartitioning(sr_customer_sk#23, sr_item_sk#22, sr_ticket_number#24, 5), ENSURE_REQUIREMENTS, [id=#32]
+
+(39) Sort [codegen id : 11]
+Input [4]: [sr_item_sk#22, sr_customer_sk#23, sr_ticket_number#24, sr_net_loss#25]
+Arguments: [sr_customer_sk#23 ASC NULLS FIRST, sr_item_sk#22 ASC NULLS FIRST, sr_ticket_number#24 ASC NULLS FIRST], false, 0
+
+(40) SortMergeJoin [codegen id : 12]
+Left keys [3]: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4]
+Right keys [3]: [sr_customer_sk#23, sr_item_sk#22, sr_ticket_number#24]
+Join condition: None
+
+(41) Project [codegen id : 12]
+Output [8]: [ss_net_profit#5, s_store_id#13, s_store_name#14, i_item_id#18, i_item_desc#19, sr_item_sk#22, sr_customer_sk#23, sr_net_loss#25]
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#4, ss_net_profit#5, s_store_id#13, s_store_name#14, i_item_id#18, i_item_desc#19, sr_item_sk#22, sr_customer_sk#23, sr_ticket_number#24, sr_net_loss#25]
+
+(42) Exchange
+Input [8]: [ss_net_profit#5, s_store_id#13, s_store_name#14, i_item_id#18, i_item_desc#19, sr_item_sk#22, sr_customer_sk#23, sr_net_loss#25]
+Arguments: hashpartitioning(sr_customer_sk#23, sr_item_sk#22, 5), ENSURE_REQUIREMENTS, [id=#33]
+
+(43) Sort [codegen id : 13]
+Input [8]: [ss_net_profit#5, s_store_id#13, s_store_name#14, i_item_id#18, i_item_desc#19, sr_item_sk#22, sr_customer_sk#23, sr_net_loss#25]
+Arguments: [sr_customer_sk#23 ASC NULLS FIRST, sr_item_sk#22 ASC NULLS FIRST], false, 0
+
+(44) Scan parquet default.catalog_sales
+Output [4]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_net_profit#36, cs_sold_date_sk#37]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#37), dynamicpruningexpression(cs_sold_date_sk#37 IN dynamicpruning#27)]
+PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_item_sk:int,cs_net_profit:decimal(7,2)>
+
+(45) ColumnarToRow [codegen id : 15]
+Input [4]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_net_profit#36, cs_sold_date_sk#37]
+
+(46) Filter [codegen id : 15]
+Input [4]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_net_profit#36, cs_sold_date_sk#37]
+Condition : (isnotnull(cs_bill_customer_sk#34) AND isnotnull(cs_item_sk#35))
+
+(47) ReusedExchange [Reuses operator id: 35]
+Output [1]: [d_date_sk#38]
+
+(48) BroadcastHashJoin [codegen id : 15]
+Left keys [1]: [cs_sold_date_sk#37]
+Right keys [1]: [d_date_sk#38]
+Join condition: None
+
+(49) Project [codegen id : 15]
+Output [3]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_net_profit#36]
+Input [5]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_net_profit#36, cs_sold_date_sk#37, d_date_sk#38]
+
+(50) Exchange
+Input [3]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_net_profit#36]
+Arguments: hashpartitioning(cs_bill_customer_sk#34, cs_item_sk#35, 5), ENSURE_REQUIREMENTS, [id=#39]
+
+(51) Sort [codegen id : 16]
+Input [3]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_net_profit#36]
+Arguments: [cs_bill_customer_sk#34 ASC NULLS FIRST, cs_item_sk#35 ASC NULLS FIRST], false, 0
+
+(52) SortMergeJoin [codegen id : 17]
+Left keys [2]: [sr_customer_sk#23, sr_item_sk#22]
+Right keys [2]: [cs_bill_customer_sk#34, cs_item_sk#35]
+Join condition: None
+
+(53) Project [codegen id : 17]
+Output [7]: [ss_net_profit#5, sr_net_loss#25, cs_net_profit#36, s_store_id#13, s_store_name#14, i_item_id#18, i_item_desc#19]
+Input [11]: [ss_net_profit#5, s_store_id#13, s_store_name#14, i_item_id#18, i_item_desc#19, sr_item_sk#22, sr_customer_sk#23, sr_net_loss#25, cs_bill_customer_sk#34, cs_item_sk#35, cs_net_profit#36]
+
+(54) HashAggregate [codegen id : 17]
+Input [7]: [ss_net_profit#5, sr_net_loss#25, cs_net_profit#36, s_store_id#13, s_store_name#14, i_item_id#18, i_item_desc#19]
+Keys [4]: [i_item_id#18, i_item_desc#19, s_store_id#13, s_store_name#14]
+Functions [3]: [partial_sum(UnscaledValue(ss_net_profit#5)), partial_sum(UnscaledValue(sr_net_loss#25)), partial_sum(UnscaledValue(cs_net_profit#36))]
+Aggregate Attributes [3]: [sum#40, sum#41, sum#42]
+Results [7]: [i_item_id#18, i_item_desc#19, s_store_id#13, s_store_name#14, sum#43, sum#44, sum#45]
+
+(55) Exchange
+Input [7]: [i_item_id#18, i_item_desc#19, s_store_id#13, s_store_name#14, sum#43, sum#44, sum#45]
+Arguments: hashpartitioning(i_item_id#18, i_item_desc#19, s_store_id#13, s_store_name#14, 5), ENSURE_REQUIREMENTS, [id=#46]
+
+(56) HashAggregate [codegen id : 18]
+Input [7]: [i_item_id#18, i_item_desc#19, s_store_id#13, s_store_name#14, sum#43, sum#44, sum#45]
+Keys [4]: [i_item_id#18, i_item_desc#19, s_store_id#13, s_store_name#14]
+Functions [3]: [sum(UnscaledValue(ss_net_profit#5)), sum(UnscaledValue(sr_net_loss#25)), sum(UnscaledValue(cs_net_profit#36))]
+Aggregate Attributes [3]: [sum(UnscaledValue(ss_net_profit#5))#47, sum(UnscaledValue(sr_net_loss#25))#48, sum(UnscaledValue(cs_net_profit#36))#49]
+Results [7]: [i_item_id#18, i_item_desc#19, s_store_id#13, s_store_name#14, MakeDecimal(sum(UnscaledValue(ss_net_profit#5))#47,17,2) AS store_sales_profit#50, MakeDecimal(sum(UnscaledValue(sr_net_loss#25))#48,17,2) AS store_returns_loss#51, MakeDecimal(sum(UnscaledValue(cs_net_profit#36))#49,17,2) AS catalog_sales_profit#52]
+
+(57) TakeOrderedAndProject
+Input [7]: [i_item_id#18, i_item_desc#19, s_store_id#13, s_store_name#14, store_sales_profit#50, store_returns_loss#51, catalog_sales_profit#52]
+Arguments: 100, [i_item_id#18 ASC NULLS FIRST, i_item_desc#19 ASC NULLS FIRST, s_store_id#13 ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST], [i_item_id#18, i_item_desc#19, s_store_id#13, s_store_name#14, store_sales_profit#50, store_returns_loss#51, catalog_sales_profit#52]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7
+ReusedExchange (58)
+
+
+(58) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#8]
+
+Subquery:2 Hosting operator id = 28 Hosting Expression = sr_returned_date_sk#26 IN dynamicpruning#27
+ReusedExchange (59)
+
+
+(59) ReusedExchange [Reuses operator id: 35]
+Output [1]: [d_date_sk#28]
+
+Subquery:3 Hosting operator id = 44 Hosting Expression = cs_sold_date_sk#37 IN dynamicpruning#27
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.sf100/simplified.txt
new file mode 100644
index 0000000000000..5d7e47650d1b5
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.sf100/simplified.txt
@@ -0,0 +1,103 @@
+TakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_profit,store_returns_loss,catalog_sales_profit]
+  WholeStageCodegen (18)
+    HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(sr_net_loss)),sum(UnscaledValue(cs_net_profit)),store_sales_profit,store_returns_loss,catalog_sales_profit,sum,sum,sum]
+      InputAdapter
+        Exchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1
+          WholeStageCodegen (17)
+            HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,ss_net_profit,sr_net_loss,cs_net_profit] [sum,sum,sum,sum,sum,sum]
+              Project [ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_id,i_item_desc]
+                SortMergeJoin [sr_customer_sk,sr_item_sk,cs_bill_customer_sk,cs_item_sk]
+                  InputAdapter
+                    WholeStageCodegen (13)
+                      Sort [sr_customer_sk,sr_item_sk]
+                        InputAdapter
+                          Exchange [sr_customer_sk,sr_item_sk] #2
+                            WholeStageCodegen (12)
+                              Project [ss_net_profit,s_store_id,s_store_name,i_item_id,i_item_desc,sr_item_sk,sr_customer_sk,sr_net_loss]
+                                SortMergeJoin [ss_customer_sk,ss_item_sk,ss_ticket_number,sr_customer_sk,sr_item_sk,sr_ticket_number]
+                                  InputAdapter
+                                    WholeStageCodegen (8)
+                                      Sort [ss_customer_sk,ss_item_sk,ss_ticket_number]
+                                        InputAdapter
+                                          Exchange [ss_customer_sk,ss_item_sk,ss_ticket_number] #3
+                                            WholeStageCodegen (7)
+                                              Project [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_net_profit,s_store_id,s_store_name,i_item_id,i_item_desc]
+                                                SortMergeJoin [ss_item_sk,i_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (4)
+                                                      Sort [ss_item_sk]
+                                                        InputAdapter
+                                                          Exchange [ss_item_sk] #4
+                                                            WholeStageCodegen (3)
+                                                              Project [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_net_profit,s_store_id,s_store_name]
+                                                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                                  Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit]
+                                                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                      Filter [ss_customer_sk,ss_item_sk,ss_ticket_number,ss_store_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk]
+                                                                              SubqueryBroadcast [d_date_sk] #1
+                                                                                ReusedExchange [d_date_sk] #5
+                                                                      InputAdapter
+                                                                        BroadcastExchange #5
+                                                                          WholeStageCodegen (1)
+                                                                            Project [d_date_sk]
+                                                                              Filter [d_moy,d_year,d_date_sk]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                                                  InputAdapter
+                                                                    BroadcastExchange #6
+                                                                      WholeStageCodegen (2)
+                                                                        Filter [s_store_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.store [s_store_sk,s_store_id,s_store_name]
+                                                  InputAdapter
+                                                    WholeStageCodegen (6)
+                                                      Sort [i_item_sk]
+                                                        InputAdapter
+                                                          Exchange [i_item_sk] #7
+                                                            WholeStageCodegen (5)
+                                                              Filter [i_item_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.item [i_item_sk,i_item_id,i_item_desc]
+                                  InputAdapter
+                                    WholeStageCodegen (11)
+                                      Sort [sr_customer_sk,sr_item_sk,sr_ticket_number]
+                                        InputAdapter
+                                          Exchange [sr_customer_sk,sr_item_sk,sr_ticket_number] #8
+                                            WholeStageCodegen (10)
+                                              Project [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss]
+                                                BroadcastHashJoin [sr_returned_date_sk,d_date_sk]
+                                                  Filter [sr_customer_sk,sr_item_sk,sr_ticket_number]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk]
+                                                          SubqueryBroadcast [d_date_sk] #2
+                                                            ReusedExchange [d_date_sk] #9
+                                                  InputAdapter
+                                                    BroadcastExchange #9
+                                                      WholeStageCodegen (9)
+                                                        Project [d_date_sk]
+                                                          Filter [d_moy,d_year,d_date_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                  InputAdapter
+                    WholeStageCodegen (16)
+                      Sort [cs_bill_customer_sk,cs_item_sk]
+                        InputAdapter
+                          Exchange [cs_bill_customer_sk,cs_item_sk] #10
+                            WholeStageCodegen (15)
+                              Project [cs_bill_customer_sk,cs_item_sk,cs_net_profit]
+                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                  Filter [cs_bill_customer_sk,cs_item_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk]
+                                          ReusedSubquery [d_date_sk] #2
+                                  InputAdapter
+                                    ReusedExchange [d_date_sk] #9
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/explain.txt
new file mode 100644
index 0000000000000..184dd07f7d950
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/explain.txt
@@ -0,0 +1,291 @@
+== Physical Plan ==
+TakeOrderedAndProject (48)
++- * HashAggregate (47)
+   +- Exchange (46)
+      +- * HashAggregate (45)
+         +- * Project (44)
+            +- * BroadcastHashJoin Inner BuildRight (43)
+               :- * Project (38)
+               :  +- * BroadcastHashJoin Inner BuildRight (37)
+               :     :- * Project (32)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (31)
+               :     :     :- * Project (29)
+               :     :     :  +- * BroadcastHashJoin Inner BuildRight (28)
+               :     :     :     :- * Project (22)
+               :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (21)
+               :     :     :     :     :- * Project (15)
+               :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (14)
+               :     :     :     :     :     :- * Project (9)
+               :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+               :     :     :     :     :     :     :- * Filter (3)
+               :     :     :     :     :     :     :  +- * ColumnarToRow (2)
+               :     :     :     :     :     :     :     +- Scan parquet default.store_sales (1)
+               :     :     :     :     :     :     +- BroadcastExchange (7)
+               :     :     :     :     :     :        +- * Filter (6)
+               :     :     :     :     :     :           +- * ColumnarToRow (5)
+               :     :     :     :     :     :              +- Scan parquet default.store_returns (4)
+               :     :     :     :     :     +- BroadcastExchange (13)
+               :     :     :     :     :        +- * Filter (12)
+               :     :     :     :     :           +- * ColumnarToRow (11)
+               :     :     :     :     :              +- Scan parquet default.catalog_sales (10)
+               :     :     :     :     +- BroadcastExchange (20)
+               :     :     :     :        +- * Project (19)
+               :     :     :     :           +- * Filter (18)
+               :     :     :     :              +- * ColumnarToRow (17)
+               :     :     :     :                 +- Scan parquet default.date_dim (16)
+               :     :     :     +- BroadcastExchange (27)
+               :     :     :        +- * Project (26)
+               :     :     :           +- * Filter (25)
+               :     :     :              +- * ColumnarToRow (24)
+               :     :     :                 +- Scan parquet default.date_dim (23)
+               :     :     +- ReusedExchange (30)
+               :     +- BroadcastExchange (36)
+               :        +- * Filter (35)
+               :           +- * ColumnarToRow (34)
+               :              +- Scan parquet default.store (33)
+               +- BroadcastExchange (42)
+                  +- * Filter (41)
+                     +- * ColumnarToRow (40)
+                        +- Scan parquet default.item (39)
+
+
+(1) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 8]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6]
+
+(3) Filter [codegen id : 8]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6]
+Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3))
+
+(4) Scan parquet default.store_returns
+Output [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(sr_returned_date_sk#12), dynamicpruningexpression(sr_returned_date_sk#12 IN dynamicpruning#13)]
+PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)]
+ReadSchema: struct<sr_item_sk:int,sr_customer_sk:int,sr_ticket_number:int,sr_net_loss:decimal(7,2)>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12]
+
+(6) Filter [codegen id : 1]
+Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12]
+Condition : ((isnotnull(sr_customer_sk#9) AND isnotnull(sr_item_sk#8)) AND isnotnull(sr_ticket_number#10))
+
+(7) BroadcastExchange
+Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12]
+Arguments: HashedRelationBroadcastMode(List(input[1, int, false], input[0, int, false], input[2, int, false]),false), [id=#14]
+
+(8) BroadcastHashJoin [codegen id : 8]
+Left keys [3]: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4]
+Right keys [3]: [sr_customer_sk#9, sr_item_sk#8, sr_ticket_number#10]
+Join condition: None
+
+(9) Project [codegen id : 8]
+Output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_net_loss#11, sr_returned_date_sk#12]
+Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12]
+
+(10) Scan parquet default.catalog_sales
+Output [4]: [cs_bill_customer_sk#15, cs_item_sk#16, cs_net_profit#17, cs_sold_date_sk#18]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_sold_date_sk#18 IN dynamicpruning#13)]
+PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_item_sk:int,cs_net_profit:decimal(7,2)>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [4]: [cs_bill_customer_sk#15, cs_item_sk#16, cs_net_profit#17, cs_sold_date_sk#18]
+
+(12) Filter [codegen id : 2]
+Input [4]: [cs_bill_customer_sk#15, cs_item_sk#16, cs_net_profit#17, cs_sold_date_sk#18]
+Condition : (isnotnull(cs_bill_customer_sk#15) AND isnotnull(cs_item_sk#16))
+
+(13) BroadcastExchange
+Input [4]: [cs_bill_customer_sk#15, cs_item_sk#16, cs_net_profit#17, cs_sold_date_sk#18]
+Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[1, int, false] as bigint) & 4294967295))),false), [id=#19]
+
+(14) BroadcastHashJoin [codegen id : 8]
+Left keys [2]: [sr_customer_sk#9, sr_item_sk#8]
+Right keys [2]: [cs_bill_customer_sk#15, cs_item_sk#16]
+Join condition: None
+
+(15) Project [codegen id : 8]
+Output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#17, cs_sold_date_sk#18]
+Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_net_loss#11, sr_returned_date_sk#12, cs_bill_customer_sk#15, cs_item_sk#16, cs_net_profit#17, cs_sold_date_sk#18]
+
+(16) Scan parquet default.date_dim
+Output [3]: [d_date_sk#20, d_year#21, d_moy#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,4), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(17) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#20, d_year#21, d_moy#22]
+
+(18) Filter [codegen id : 3]
+Input [3]: [d_date_sk#20, d_year#21, d_moy#22]
+Condition : ((((isnotnull(d_moy#22) AND isnotnull(d_year#21)) AND (d_moy#22 = 4)) AND (d_year#21 = 2001)) AND isnotnull(d_date_sk#20))
+
+(19) Project [codegen id : 3]
+Output [1]: [d_date_sk#20]
+Input [3]: [d_date_sk#20, d_year#21, d_moy#22]
+
+(20) BroadcastExchange
+Input [1]: [d_date_sk#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#23]
+
+(21) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_sold_date_sk#6]
+Right keys [1]: [d_date_sk#20]
+Join condition: None
+
+(22) Project [codegen id : 8]
+Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#17, cs_sold_date_sk#18]
+Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#17, cs_sold_date_sk#18, d_date_sk#20]
+
+(23) Scan parquet default.date_dim
+Output [3]: [d_date_sk#24, d_year#25, d_moy#26]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,10), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(24) ColumnarToRow [codegen id : 4]
+Input [3]: [d_date_sk#24, d_year#25, d_moy#26]
+
+(25) Filter [codegen id : 4]
+Input [3]: [d_date_sk#24, d_year#25, d_moy#26]
+Condition : (((((isnotnull(d_moy#26) AND isnotnull(d_year#25)) AND (d_moy#26 >= 4)) AND (d_moy#26 <= 10)) AND (d_year#25 = 2001)) AND isnotnull(d_date_sk#24))
+
+(26) Project [codegen id : 4]
+Output [1]: [d_date_sk#24]
+Input [3]: [d_date_sk#24, d_year#25, d_moy#26]
+
+(27) BroadcastExchange
+Input [1]: [d_date_sk#24]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#27]
+
+(28) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [sr_returned_date_sk#12]
+Right keys [1]: [d_date_sk#24]
+Join condition: None
+
+(29) Project [codegen id : 8]
+Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#17, cs_sold_date_sk#18]
+Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#17, cs_sold_date_sk#18, d_date_sk#24]
+
+(30) ReusedExchange [Reuses operator id: 27]
+Output [1]: [d_date_sk#28]
+
+(31) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [cs_sold_date_sk#18]
+Right keys [1]: [d_date_sk#28]
+Join condition: None
+
+(32) Project [codegen id : 8]
+Output [5]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#17]
+Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#17, cs_sold_date_sk#18, d_date_sk#28]
+
+(33) Scan parquet default.store
+Output [3]: [s_store_sk#29, s_store_id#30, s_store_name#31]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string,s_store_name:string>
+
+(34) ColumnarToRow [codegen id : 6]
+Input [3]: [s_store_sk#29, s_store_id#30, s_store_name#31]
+
+(35) Filter [codegen id : 6]
+Input [3]: [s_store_sk#29, s_store_id#30, s_store_name#31]
+Condition : isnotnull(s_store_sk#29)
+
+(36) BroadcastExchange
+Input [3]: [s_store_sk#29, s_store_id#30, s_store_name#31]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#32]
+
+(37) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#29]
+Join condition: None
+
+(38) Project [codegen id : 8]
+Output [6]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#11, cs_net_profit#17, s_store_id#30, s_store_name#31]
+Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#17, s_store_sk#29, s_store_id#30, s_store_name#31]
+
+(39) Scan parquet default.item
+Output [3]: [i_item_sk#33, i_item_id#34, i_item_desc#35]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_item_desc:string>
+
+(40) ColumnarToRow [codegen id : 7]
+Input [3]: [i_item_sk#33, i_item_id#34, i_item_desc#35]
+
+(41) Filter [codegen id : 7]
+Input [3]: [i_item_sk#33, i_item_id#34, i_item_desc#35]
+Condition : isnotnull(i_item_sk#33)
+
+(42) BroadcastExchange
+Input [3]: [i_item_sk#33, i_item_id#34, i_item_desc#35]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#36]
+
+(43) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#33]
+Join condition: None
+
+(44) Project [codegen id : 8]
+Output [7]: [ss_net_profit#5, sr_net_loss#11, cs_net_profit#17, s_store_id#30, s_store_name#31, i_item_id#34, i_item_desc#35]
+Input [9]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#11, cs_net_profit#17, s_store_id#30, s_store_name#31, i_item_sk#33, i_item_id#34, i_item_desc#35]
+
+(45) HashAggregate [codegen id : 8]
+Input [7]: [ss_net_profit#5, sr_net_loss#11, cs_net_profit#17, s_store_id#30, s_store_name#31, i_item_id#34, i_item_desc#35]
+Keys [4]: [i_item_id#34, i_item_desc#35, s_store_id#30, s_store_name#31]
+Functions [3]: [partial_sum(UnscaledValue(ss_net_profit#5)), partial_sum(UnscaledValue(sr_net_loss#11)), partial_sum(UnscaledValue(cs_net_profit#17))]
+Aggregate Attributes [3]: [sum#37, sum#38, sum#39]
+Results [7]: [i_item_id#34, i_item_desc#35, s_store_id#30, s_store_name#31, sum#40, sum#41, sum#42]
+
+(46) Exchange
+Input [7]: [i_item_id#34, i_item_desc#35, s_store_id#30, s_store_name#31, sum#40, sum#41, sum#42]
+Arguments: hashpartitioning(i_item_id#34, i_item_desc#35, s_store_id#30, s_store_name#31, 5), ENSURE_REQUIREMENTS, [id=#43]
+
+(47) HashAggregate [codegen id : 9]
+Input [7]: [i_item_id#34, i_item_desc#35, s_store_id#30, s_store_name#31, sum#40, sum#41, sum#42]
+Keys [4]: [i_item_id#34, i_item_desc#35, s_store_id#30, s_store_name#31]
+Functions [3]: [sum(UnscaledValue(ss_net_profit#5)), sum(UnscaledValue(sr_net_loss#11)), sum(UnscaledValue(cs_net_profit#17))]
+Aggregate Attributes [3]: [sum(UnscaledValue(ss_net_profit#5))#44, sum(UnscaledValue(sr_net_loss#11))#45, sum(UnscaledValue(cs_net_profit#17))#46]
+Results [7]: [i_item_id#34, i_item_desc#35, s_store_id#30, s_store_name#31, MakeDecimal(sum(UnscaledValue(ss_net_profit#5))#44,17,2) AS store_sales_profit#47, MakeDecimal(sum(UnscaledValue(sr_net_loss#11))#45,17,2) AS store_returns_loss#48, MakeDecimal(sum(UnscaledValue(cs_net_profit#17))#46,17,2) AS catalog_sales_profit#49]
+
+(48) TakeOrderedAndProject
+Input [7]: [i_item_id#34, i_item_desc#35, s_store_id#30, s_store_name#31, store_sales_profit#47, store_returns_loss#48, catalog_sales_profit#49]
+Arguments: 100, [i_item_id#34 ASC NULLS FIRST, i_item_desc#35 ASC NULLS FIRST, s_store_id#30 ASC NULLS FIRST, s_store_name#31 ASC NULLS FIRST], [i_item_id#34, i_item_desc#35, s_store_id#30, s_store_name#31, store_sales_profit#47, store_returns_loss#48, catalog_sales_profit#49]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7
+ReusedExchange (49)
+
+
+(49) ReusedExchange [Reuses operator id: 20]
+Output [1]: [d_date_sk#20]
+
+Subquery:2 Hosting operator id = 4 Hosting Expression = sr_returned_date_sk#12 IN dynamicpruning#13
+ReusedExchange (50)
+
+
+(50) ReusedExchange [Reuses operator id: 27]
+Output [1]: [d_date_sk#24]
+
+Subquery:3 Hosting operator id = 10 Hosting Expression = cs_sold_date_sk#18 IN dynamicpruning#13
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/simplified.txt
new file mode 100644
index 0000000000000..6b22b6392eade
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/simplified.txt
@@ -0,0 +1,76 @@
+TakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_profit,store_returns_loss,catalog_sales_profit]
+  WholeStageCodegen (9)
+    HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(sr_net_loss)),sum(UnscaledValue(cs_net_profit)),store_sales_profit,store_returns_loss,catalog_sales_profit,sum,sum,sum]
+      InputAdapter
+        Exchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1
+          WholeStageCodegen (8)
+            HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,ss_net_profit,sr_net_loss,cs_net_profit] [sum,sum,sum,sum,sum,sum]
+              Project [ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_id,i_item_desc]
+                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                  Project [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name]
+                    BroadcastHashJoin [ss_store_sk,s_store_sk]
+                      Project [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit]
+                        BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                          Project [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,cs_sold_date_sk]
+                            BroadcastHashJoin [sr_returned_date_sk,d_date_sk]
+                              Project [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk]
+                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                  Project [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk]
+                                    BroadcastHashJoin [sr_customer_sk,sr_item_sk,cs_bill_customer_sk,cs_item_sk]
+                                      Project [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_net_loss,sr_returned_date_sk]
+                                        BroadcastHashJoin [ss_customer_sk,ss_item_sk,ss_ticket_number,sr_customer_sk,sr_item_sk,sr_ticket_number]
+                                          Filter [ss_customer_sk,ss_item_sk,ss_ticket_number,ss_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk]
+                                                  SubqueryBroadcast [d_date_sk] #1
+                                                    ReusedExchange [d_date_sk] #2
+                                          InputAdapter
+                                            BroadcastExchange #3
+                                              WholeStageCodegen (1)
+                                                Filter [sr_customer_sk,sr_item_sk,sr_ticket_number]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk]
+                                                        SubqueryBroadcast [d_date_sk] #2
+                                                          ReusedExchange [d_date_sk] #4
+                                      InputAdapter
+                                        BroadcastExchange #5
+                                          WholeStageCodegen (2)
+                                            Filter [cs_bill_customer_sk,cs_item_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk]
+                                                    ReusedSubquery [d_date_sk] #2
+                                  InputAdapter
+                                    BroadcastExchange #2
+                                      WholeStageCodegen (3)
+                                        Project [d_date_sk]
+                                          Filter [d_moy,d_year,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                              InputAdapter
+                                BroadcastExchange #4
+                                  WholeStageCodegen (4)
+                                    Project [d_date_sk]
+                                      Filter [d_moy,d_year,d_date_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                          InputAdapter
+                            ReusedExchange [d_date_sk] #4
+                      InputAdapter
+                        BroadcastExchange #6
+                          WholeStageCodegen (6)
+                            Filter [s_store_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.store [s_store_sk,s_store_id,s_store_name]
+                  InputAdapter
+                    BroadcastExchange #7
+                      WholeStageCodegen (7)
+                        Filter [i_item_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.item [i_item_sk,i_item_id,i_item_desc]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.sf100/explain.txt
new file mode 100644
index 0000000000000..40565dc76a7ba
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.sf100/explain.txt
@@ -0,0 +1,204 @@
+== Physical Plan ==
+TakeOrderedAndProject (34)
++- * HashAggregate (33)
+   +- Exchange (32)
+      +- * HashAggregate (31)
+         +- * Project (30)
+            +- * BroadcastHashJoin Inner BuildRight (29)
+               :- * Project (24)
+               :  +- * BroadcastHashJoin Inner BuildRight (23)
+               :     :- * Project (17)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (16)
+               :     :     :- * Project (10)
+               :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :     :     :- * Filter (3)
+               :     :     :     :  +- * ColumnarToRow (2)
+               :     :     :     :     +- Scan parquet default.catalog_sales (1)
+               :     :     :     +- BroadcastExchange (8)
+               :     :     :        +- * Project (7)
+               :     :     :           +- * Filter (6)
+               :     :     :              +- * ColumnarToRow (5)
+               :     :     :                 +- Scan parquet default.customer_demographics (4)
+               :     :     +- BroadcastExchange (15)
+               :     :        +- * Project (14)
+               :     :           +- * Filter (13)
+               :     :              +- * ColumnarToRow (12)
+               :     :                 +- Scan parquet default.promotion (11)
+               :     +- BroadcastExchange (22)
+               :        +- * Project (21)
+               :           +- * Filter (20)
+               :              +- * ColumnarToRow (19)
+               :                 +- Scan parquet default.date_dim (18)
+               +- BroadcastExchange (28)
+                  +- * Filter (27)
+                     +- * ColumnarToRow (26)
+                        +- Scan parquet default.item (25)
+
+
+(1) Scan parquet default.catalog_sales
+Output [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#8), dynamicpruningexpression(cs_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)]
+ReadSchema: struct<cs_bill_cdemo_sk:int,cs_item_sk:int,cs_promo_sk:int,cs_quantity:int,cs_list_price:decimal(7,2),cs_sales_price:decimal(7,2),cs_coupon_amt:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8]
+
+(3) Filter [codegen id : 5]
+Input [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8]
+Condition : ((isnotnull(cs_bill_cdemo_sk#1) AND isnotnull(cs_item_sk#2)) AND isnotnull(cs_promo_sk#3))
+
+(4) Scan parquet default.customer_demographics
+Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_marital_status,S), EqualTo(cd_education_status,College             ), IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_education_status:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+
+(6) Filter [codegen id : 1]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (cd_gender#11 = M)) AND (cd_marital_status#12 = S)) AND (cd_education_status#13 = College             )) AND isnotnull(cd_demo_sk#10))
+
+(7) Project [codegen id : 1]
+Output [1]: [cd_demo_sk#10]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+
+(8) BroadcastExchange
+Input [1]: [cd_demo_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [cs_bill_cdemo_sk#1]
+Right keys [1]: [cd_demo_sk#10]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [7]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8]
+Input [9]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, cd_demo_sk#10]
+
+(11) Scan parquet default.promotion
+Output [3]: [p_promo_sk#15, p_channel_email#16, p_channel_event#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/promotion]
+PushedFilters: [Or(EqualTo(p_channel_email,N),EqualTo(p_channel_event,N)), IsNotNull(p_promo_sk)]
+ReadSchema: struct<p_promo_sk:int,p_channel_email:string,p_channel_event:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [p_promo_sk#15, p_channel_email#16, p_channel_event#17]
+
+(13) Filter [codegen id : 2]
+Input [3]: [p_promo_sk#15, p_channel_email#16, p_channel_event#17]
+Condition : (((p_channel_email#16 = N) OR (p_channel_event#17 = N)) AND isnotnull(p_promo_sk#15))
+
+(14) Project [codegen id : 2]
+Output [1]: [p_promo_sk#15]
+Input [3]: [p_promo_sk#15, p_channel_email#16, p_channel_event#17]
+
+(15) BroadcastExchange
+Input [1]: [p_promo_sk#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(16) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [cs_promo_sk#3]
+Right keys [1]: [p_promo_sk#15]
+Join condition: None
+
+(17) Project [codegen id : 5]
+Output [6]: [cs_item_sk#2, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8]
+Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, p_promo_sk#15]
+
+(18) Scan parquet default.date_dim
+Output [2]: [d_date_sk#19, d_year#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [2]: [d_date_sk#19, d_year#20]
+
+(20) Filter [codegen id : 3]
+Input [2]: [d_date_sk#19, d_year#20]
+Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2000)) AND isnotnull(d_date_sk#19))
+
+(21) Project [codegen id : 3]
+Output [1]: [d_date_sk#19]
+Input [2]: [d_date_sk#19, d_year#20]
+
+(22) BroadcastExchange
+Input [1]: [d_date_sk#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(23) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [cs_sold_date_sk#8]
+Right keys [1]: [d_date_sk#19]
+Join condition: None
+
+(24) Project [codegen id : 5]
+Output [5]: [cs_item_sk#2, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7]
+Input [7]: [cs_item_sk#2, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, d_date_sk#19]
+
+(25) Scan parquet default.item
+Output [2]: [i_item_sk#22, i_item_id#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(26) ColumnarToRow [codegen id : 4]
+Input [2]: [i_item_sk#22, i_item_id#23]
+
+(27) Filter [codegen id : 4]
+Input [2]: [i_item_sk#22, i_item_id#23]
+Condition : isnotnull(i_item_sk#22)
+
+(28) BroadcastExchange
+Input [2]: [i_item_sk#22, i_item_id#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#24]
+
+(29) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [cs_item_sk#2]
+Right keys [1]: [i_item_sk#22]
+Join condition: None
+
+(30) Project [codegen id : 5]
+Output [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#23]
+Input [7]: [cs_item_sk#2, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_sk#22, i_item_id#23]
+
+(31) HashAggregate [codegen id : 5]
+Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#23]
+Keys [1]: [i_item_id#23]
+Functions [4]: [partial_avg(cs_quantity#4), partial_avg(UnscaledValue(cs_list_price#5)), partial_avg(UnscaledValue(cs_coupon_amt#7)), partial_avg(UnscaledValue(cs_sales_price#6))]
+Aggregate Attributes [8]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31, count#32]
+Results [9]: [i_item_id#23, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40]
+
+(32) Exchange
+Input [9]: [i_item_id#23, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40]
+Arguments: hashpartitioning(i_item_id#23, 5), ENSURE_REQUIREMENTS, [id=#41]
+
+(33) HashAggregate [codegen id : 6]
+Input [9]: [i_item_id#23, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40]
+Keys [1]: [i_item_id#23]
+Functions [4]: [avg(cs_quantity#4), avg(UnscaledValue(cs_list_price#5)), avg(UnscaledValue(cs_coupon_amt#7)), avg(UnscaledValue(cs_sales_price#6))]
+Aggregate Attributes [4]: [avg(cs_quantity#4)#42, avg(UnscaledValue(cs_list_price#5))#43, avg(UnscaledValue(cs_coupon_amt#7))#44, avg(UnscaledValue(cs_sales_price#6))#45]
+Results [5]: [i_item_id#23, avg(cs_quantity#4)#42 AS agg1#46, cast((avg(UnscaledValue(cs_list_price#5))#43 / 100.0) as decimal(11,6)) AS agg2#47, cast((avg(UnscaledValue(cs_coupon_amt#7))#44 / 100.0) as decimal(11,6)) AS agg3#48, cast((avg(UnscaledValue(cs_sales_price#6))#45 / 100.0) as decimal(11,6)) AS agg4#49]
+
+(34) TakeOrderedAndProject
+Input [5]: [i_item_id#23, agg1#46, agg2#47, agg3#48, agg4#49]
+Arguments: 100, [i_item_id#23 ASC NULLS FIRST], [i_item_id#23, agg1#46, agg2#47, agg3#48, agg4#49]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (35)
+
+
+(35) ReusedExchange [Reuses operator id: 22]
+Output [1]: [d_date_sk#19]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.sf100/simplified.txt
new file mode 100644
index 0000000000000..f5af2ee1a4aa4
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.sf100/simplified.txt
@@ -0,0 +1,52 @@
+TakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4]
+  WholeStageCodegen (6)
+    HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(cs_quantity),avg(UnscaledValue(cs_list_price)),avg(UnscaledValue(cs_coupon_amt)),avg(UnscaledValue(cs_sales_price)),agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count]
+      InputAdapter
+        Exchange [i_item_id] #1
+          WholeStageCodegen (5)
+            HashAggregate [i_item_id,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+              Project [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id]
+                BroadcastHashJoin [cs_item_sk,i_item_sk]
+                  Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt]
+                    BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                      Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk]
+                        BroadcastHashJoin [cs_promo_sk,p_promo_sk]
+                          Project [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk]
+                            BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk]
+                              Filter [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.catalog_sales [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk]
+                                      SubqueryBroadcast [d_date_sk] #1
+                                        ReusedExchange [d_date_sk] #2
+                              InputAdapter
+                                BroadcastExchange #3
+                                  WholeStageCodegen (1)
+                                    Project [cd_demo_sk]
+                                      Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status]
+                          InputAdapter
+                            BroadcastExchange #4
+                              WholeStageCodegen (2)
+                                Project [p_promo_sk]
+                                  Filter [p_channel_email,p_channel_event,p_promo_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.promotion [p_promo_sk,p_channel_email,p_channel_event]
+                      InputAdapter
+                        BroadcastExchange #2
+                          WholeStageCodegen (3)
+                            Project [d_date_sk]
+                              Filter [d_year,d_date_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.date_dim [d_date_sk,d_year]
+                  InputAdapter
+                    BroadcastExchange #5
+                      WholeStageCodegen (4)
+                        Filter [i_item_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.item [i_item_sk,i_item_id]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/explain.txt
new file mode 100644
index 0000000000000..c86f121ad35a9
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/explain.txt
@@ -0,0 +1,204 @@
+== Physical Plan ==
+TakeOrderedAndProject (34)
++- * HashAggregate (33)
+   +- Exchange (32)
+      +- * HashAggregate (31)
+         +- * Project (30)
+            +- * BroadcastHashJoin Inner BuildRight (29)
+               :- * Project (23)
+               :  +- * BroadcastHashJoin Inner BuildRight (22)
+               :     :- * Project (17)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (16)
+               :     :     :- * Project (10)
+               :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :     :     :- * Filter (3)
+               :     :     :     :  +- * ColumnarToRow (2)
+               :     :     :     :     +- Scan parquet default.catalog_sales (1)
+               :     :     :     +- BroadcastExchange (8)
+               :     :     :        +- * Project (7)
+               :     :     :           +- * Filter (6)
+               :     :     :              +- * ColumnarToRow (5)
+               :     :     :                 +- Scan parquet default.customer_demographics (4)
+               :     :     +- BroadcastExchange (15)
+               :     :        +- * Project (14)
+               :     :           +- * Filter (13)
+               :     :              +- * ColumnarToRow (12)
+               :     :                 +- Scan parquet default.date_dim (11)
+               :     +- BroadcastExchange (21)
+               :        +- * Filter (20)
+               :           +- * ColumnarToRow (19)
+               :              +- Scan parquet default.item (18)
+               +- BroadcastExchange (28)
+                  +- * Project (27)
+                     +- * Filter (26)
+                        +- * ColumnarToRow (25)
+                           +- Scan parquet default.promotion (24)
+
+
+(1) Scan parquet default.catalog_sales
+Output [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#8), dynamicpruningexpression(cs_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)]
+ReadSchema: struct<cs_bill_cdemo_sk:int,cs_item_sk:int,cs_promo_sk:int,cs_quantity:int,cs_list_price:decimal(7,2),cs_sales_price:decimal(7,2),cs_coupon_amt:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8]
+
+(3) Filter [codegen id : 5]
+Input [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8]
+Condition : ((isnotnull(cs_bill_cdemo_sk#1) AND isnotnull(cs_item_sk#2)) AND isnotnull(cs_promo_sk#3))
+
+(4) Scan parquet default.customer_demographics
+Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_marital_status,S), EqualTo(cd_education_status,College             ), IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_education_status:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+
+(6) Filter [codegen id : 1]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (cd_gender#11 = M)) AND (cd_marital_status#12 = S)) AND (cd_education_status#13 = College             )) AND isnotnull(cd_demo_sk#10))
+
+(7) Project [codegen id : 1]
+Output [1]: [cd_demo_sk#10]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+
+(8) BroadcastExchange
+Input [1]: [cd_demo_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [cs_bill_cdemo_sk#1]
+Right keys [1]: [cd_demo_sk#10]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [7]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8]
+Input [9]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, cd_demo_sk#10]
+
+(11) Scan parquet default.date_dim
+Output [2]: [d_date_sk#15, d_year#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#15, d_year#16]
+
+(13) Filter [codegen id : 2]
+Input [2]: [d_date_sk#15, d_year#16]
+Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2000)) AND isnotnull(d_date_sk#15))
+
+(14) Project [codegen id : 2]
+Output [1]: [d_date_sk#15]
+Input [2]: [d_date_sk#15, d_year#16]
+
+(15) BroadcastExchange
+Input [1]: [d_date_sk#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(16) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [cs_sold_date_sk#8]
+Right keys [1]: [d_date_sk#15]
+Join condition: None
+
+(17) Project [codegen id : 5]
+Output [6]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7]
+Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, d_date_sk#15]
+
+(18) Scan parquet default.item
+Output [2]: [i_item_sk#18, i_item_id#19]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [2]: [i_item_sk#18, i_item_id#19]
+
+(20) Filter [codegen id : 3]
+Input [2]: [i_item_sk#18, i_item_id#19]
+Condition : isnotnull(i_item_sk#18)
+
+(21) BroadcastExchange
+Input [2]: [i_item_sk#18, i_item_id#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#20]
+
+(22) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [cs_item_sk#2]
+Right keys [1]: [i_item_sk#18]
+Join condition: None
+
+(23) Project [codegen id : 5]
+Output [6]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#19]
+Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_sk#18, i_item_id#19]
+
+(24) Scan parquet default.promotion
+Output [3]: [p_promo_sk#21, p_channel_email#22, p_channel_event#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/promotion]
+PushedFilters: [Or(EqualTo(p_channel_email,N),EqualTo(p_channel_event,N)), IsNotNull(p_promo_sk)]
+ReadSchema: struct<p_promo_sk:int,p_channel_email:string,p_channel_event:string>
+
+(25) ColumnarToRow [codegen id : 4]
+Input [3]: [p_promo_sk#21, p_channel_email#22, p_channel_event#23]
+
+(26) Filter [codegen id : 4]
+Input [3]: [p_promo_sk#21, p_channel_email#22, p_channel_event#23]
+Condition : (((p_channel_email#22 = N) OR (p_channel_event#23 = N)) AND isnotnull(p_promo_sk#21))
+
+(27) Project [codegen id : 4]
+Output [1]: [p_promo_sk#21]
+Input [3]: [p_promo_sk#21, p_channel_email#22, p_channel_event#23]
+
+(28) BroadcastExchange
+Input [1]: [p_promo_sk#21]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#24]
+
+(29) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [cs_promo_sk#3]
+Right keys [1]: [p_promo_sk#21]
+Join condition: None
+
+(30) Project [codegen id : 5]
+Output [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#19]
+Input [7]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#19, p_promo_sk#21]
+
+(31) HashAggregate [codegen id : 5]
+Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#19]
+Keys [1]: [i_item_id#19]
+Functions [4]: [partial_avg(cs_quantity#4), partial_avg(UnscaledValue(cs_list_price#5)), partial_avg(UnscaledValue(cs_coupon_amt#7)), partial_avg(UnscaledValue(cs_sales_price#6))]
+Aggregate Attributes [8]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31, count#32]
+Results [9]: [i_item_id#19, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40]
+
+(32) Exchange
+Input [9]: [i_item_id#19, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40]
+Arguments: hashpartitioning(i_item_id#19, 5), ENSURE_REQUIREMENTS, [id=#41]
+
+(33) HashAggregate [codegen id : 6]
+Input [9]: [i_item_id#19, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40]
+Keys [1]: [i_item_id#19]
+Functions [4]: [avg(cs_quantity#4), avg(UnscaledValue(cs_list_price#5)), avg(UnscaledValue(cs_coupon_amt#7)), avg(UnscaledValue(cs_sales_price#6))]
+Aggregate Attributes [4]: [avg(cs_quantity#4)#42, avg(UnscaledValue(cs_list_price#5))#43, avg(UnscaledValue(cs_coupon_amt#7))#44, avg(UnscaledValue(cs_sales_price#6))#45]
+Results [5]: [i_item_id#19, avg(cs_quantity#4)#42 AS agg1#46, cast((avg(UnscaledValue(cs_list_price#5))#43 / 100.0) as decimal(11,6)) AS agg2#47, cast((avg(UnscaledValue(cs_coupon_amt#7))#44 / 100.0) as decimal(11,6)) AS agg3#48, cast((avg(UnscaledValue(cs_sales_price#6))#45 / 100.0) as decimal(11,6)) AS agg4#49]
+
+(34) TakeOrderedAndProject
+Input [5]: [i_item_id#19, agg1#46, agg2#47, agg3#48, agg4#49]
+Arguments: 100, [i_item_id#19 ASC NULLS FIRST], [i_item_id#19, agg1#46, agg2#47, agg3#48, agg4#49]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (35)
+
+
+(35) ReusedExchange [Reuses operator id: 15]
+Output [1]: [d_date_sk#15]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/simplified.txt
new file mode 100644
index 0000000000000..685ecb9d6370e
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/simplified.txt
@@ -0,0 +1,52 @@
+TakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4]
+  WholeStageCodegen (6)
+    HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(cs_quantity),avg(UnscaledValue(cs_list_price)),avg(UnscaledValue(cs_coupon_amt)),avg(UnscaledValue(cs_sales_price)),agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count]
+      InputAdapter
+        Exchange [i_item_id] #1
+          WholeStageCodegen (5)
+            HashAggregate [i_item_id,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+              Project [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id]
+                BroadcastHashJoin [cs_promo_sk,p_promo_sk]
+                  Project [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id]
+                    BroadcastHashJoin [cs_item_sk,i_item_sk]
+                      Project [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt]
+                        BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                          Project [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk]
+                            BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk]
+                              Filter [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.catalog_sales [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk]
+                                      SubqueryBroadcast [d_date_sk] #1
+                                        ReusedExchange [d_date_sk] #2
+                              InputAdapter
+                                BroadcastExchange #3
+                                  WholeStageCodegen (1)
+                                    Project [cd_demo_sk]
+                                      Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status]
+                          InputAdapter
+                            BroadcastExchange #2
+                              WholeStageCodegen (2)
+                                Project [d_date_sk]
+                                  Filter [d_year,d_date_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.date_dim [d_date_sk,d_year]
+                      InputAdapter
+                        BroadcastExchange #4
+                          WholeStageCodegen (3)
+                            Filter [i_item_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.item [i_item_sk,i_item_id]
+                  InputAdapter
+                    BroadcastExchange #5
+                      WholeStageCodegen (4)
+                        Project [p_promo_sk]
+                          Filter [p_channel_email,p_channel_event,p_promo_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.promotion [p_promo_sk,p_channel_email,p_channel_event]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.sf100/explain.txt
new file mode 100644
index 0000000000000..8c530f5a37885
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.sf100/explain.txt
@@ -0,0 +1,204 @@
+== Physical Plan ==
+TakeOrderedAndProject (34)
++- * HashAggregate (33)
+   +- Exchange (32)
+      +- * HashAggregate (31)
+         +- * Expand (30)
+            +- * Project (29)
+               +- * BroadcastHashJoin Inner BuildRight (28)
+                  :- * Project (23)
+                  :  +- * BroadcastHashJoin Inner BuildRight (22)
+                  :     :- * Project (17)
+                  :     :  +- * BroadcastHashJoin Inner BuildRight (16)
+                  :     :     :- * Project (10)
+                  :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+                  :     :     :     :- * Filter (3)
+                  :     :     :     :  +- * ColumnarToRow (2)
+                  :     :     :     :     +- Scan parquet default.store_sales (1)
+                  :     :     :     +- BroadcastExchange (8)
+                  :     :     :        +- * Project (7)
+                  :     :     :           +- * Filter (6)
+                  :     :     :              +- * ColumnarToRow (5)
+                  :     :     :                 +- Scan parquet default.customer_demographics (4)
+                  :     :     +- BroadcastExchange (15)
+                  :     :        +- * Project (14)
+                  :     :           +- * Filter (13)
+                  :     :              +- * ColumnarToRow (12)
+                  :     :                 +- Scan parquet default.date_dim (11)
+                  :     +- BroadcastExchange (21)
+                  :        +- * Filter (20)
+                  :           +- * ColumnarToRow (19)
+                  :              +- Scan parquet default.store (18)
+                  +- BroadcastExchange (27)
+                     +- * Filter (26)
+                        +- * ColumnarToRow (25)
+                           +- Scan parquet default.item (24)
+
+
+(1) Scan parquet default.store_sales
+Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_cdemo_sk:int,ss_store_sk:int,ss_quantity:int,ss_list_price:decimal(7,2),ss_sales_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+
+(3) Filter [codegen id : 5]
+Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1))
+
+(4) Scan parquet default.customer_demographics
+Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_marital_status,S), EqualTo(cd_education_status,College             ), IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_education_status:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+
+(6) Filter [codegen id : 1]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (cd_gender#11 = M)) AND (cd_marital_status#12 = S)) AND (cd_education_status#13 = College             )) AND isnotnull(cd_demo_sk#10))
+
+(7) Project [codegen id : 1]
+Output [1]: [cd_demo_sk#10]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+
+(8) BroadcastExchange
+Input [1]: [cd_demo_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#10]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#10]
+
+(11) Scan parquet default.date_dim
+Output [2]: [d_date_sk#15, d_year#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#15, d_year#16]
+
+(13) Filter [codegen id : 2]
+Input [2]: [d_date_sk#15, d_year#16]
+Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2002)) AND isnotnull(d_date_sk#15))
+
+(14) Project [codegen id : 2]
+Output [1]: [d_date_sk#15]
+Input [2]: [d_date_sk#15, d_year#16]
+
+(15) BroadcastExchange
+Input [1]: [d_date_sk#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(16) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#15]
+Join condition: None
+
+(17) Project [codegen id : 5]
+Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7]
+Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#15]
+
+(18) Scan parquet default.store
+Output [2]: [s_store_sk#18, s_state#19]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_state:string>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [2]: [s_store_sk#18, s_state#19]
+
+(20) Filter [codegen id : 3]
+Input [2]: [s_store_sk#18, s_state#19]
+Condition : ((isnotnull(s_state#19) AND (s_state#19 = TN)) AND isnotnull(s_store_sk#18))
+
+(21) BroadcastExchange
+Input [2]: [s_store_sk#18, s_state#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#20]
+
+(22) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#18]
+Join condition: None
+
+(23) Project [codegen id : 5]
+Output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#19]
+Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#18, s_state#19]
+
+(24) Scan parquet default.item
+Output [2]: [i_item_sk#21, i_item_id#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(25) ColumnarToRow [codegen id : 4]
+Input [2]: [i_item_sk#21, i_item_id#22]
+
+(26) Filter [codegen id : 4]
+Input [2]: [i_item_sk#21, i_item_id#22]
+Condition : isnotnull(i_item_sk#21)
+
+(27) BroadcastExchange
+Input [2]: [i_item_sk#21, i_item_id#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#23]
+
+(28) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#21]
+Join condition: None
+
+(29) Project [codegen id : 5]
+Output [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#22, s_state#19]
+Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#19, i_item_sk#21, i_item_id#22]
+
+(30) Expand [codegen id : 5]
+Input [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#22, s_state#19]
+Arguments: [List(ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#22, s_state#19, 0), List(ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#22, null, 1), List(ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, null, null, 3)], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#24, s_state#25, spark_grouping_id#26]
+
+(31) HashAggregate [codegen id : 5]
+Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#24, s_state#25, spark_grouping_id#26]
+Keys [3]: [i_item_id#24, s_state#25, spark_grouping_id#26]
+Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))]
+Aggregate Attributes [8]: [sum#27, count#28, sum#29, count#30, sum#31, count#32, sum#33, count#34]
+Results [11]: [i_item_id#24, s_state#25, spark_grouping_id#26, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42]
+
+(32) Exchange
+Input [11]: [i_item_id#24, s_state#25, spark_grouping_id#26, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42]
+Arguments: hashpartitioning(i_item_id#24, s_state#25, spark_grouping_id#26, 5), ENSURE_REQUIREMENTS, [id=#43]
+
+(33) HashAggregate [codegen id : 6]
+Input [11]: [i_item_id#24, s_state#25, spark_grouping_id#26, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42]
+Keys [3]: [i_item_id#24, s_state#25, spark_grouping_id#26]
+Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))]
+Aggregate Attributes [4]: [avg(ss_quantity#4)#44, avg(UnscaledValue(ss_list_price#5))#45, avg(UnscaledValue(ss_coupon_amt#7))#46, avg(UnscaledValue(ss_sales_price#6))#47]
+Results [7]: [i_item_id#24, s_state#25, cast((shiftright(spark_grouping_id#26, 0) & 1) as tinyint) AS g_state#48, avg(ss_quantity#4)#44 AS agg1#49, cast((avg(UnscaledValue(ss_list_price#5))#45 / 100.0) as decimal(11,6)) AS agg2#50, cast((avg(UnscaledValue(ss_coupon_amt#7))#46 / 100.0) as decimal(11,6)) AS agg3#51, cast((avg(UnscaledValue(ss_sales_price#6))#47 / 100.0) as decimal(11,6)) AS agg4#52]
+
+(34) TakeOrderedAndProject
+Input [7]: [i_item_id#24, s_state#25, g_state#48, agg1#49, agg2#50, agg3#51, agg4#52]
+Arguments: 100, [i_item_id#24 ASC NULLS FIRST, s_state#25 ASC NULLS FIRST], [i_item_id#24, s_state#25, g_state#48, agg1#49, agg2#50, agg3#51, agg4#52]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (35)
+
+
+(35) ReusedExchange [Reuses operator id: 15]
+Output [1]: [d_date_sk#15]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.sf100/simplified.txt
new file mode 100644
index 0000000000000..6724eceb18711
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.sf100/simplified.txt
@@ -0,0 +1,52 @@
+TakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4]
+  WholeStageCodegen (6)
+    HashAggregate [i_item_id,s_state,spark_grouping_id,sum,count,sum,count,sum,count,sum,count] [avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price)),g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count]
+      InputAdapter
+        Exchange [i_item_id,s_state,spark_grouping_id] #1
+          WholeStageCodegen (5)
+            HashAggregate [i_item_id,s_state,spark_grouping_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+              Expand [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state]
+                Project [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state]
+                  BroadcastHashJoin [ss_item_sk,i_item_sk]
+                    Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state]
+                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                        Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                            Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                              BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk]
+                                Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                                        SubqueryBroadcast [d_date_sk] #1
+                                          ReusedExchange [d_date_sk] #2
+                                InputAdapter
+                                  BroadcastExchange #3
+                                    WholeStageCodegen (1)
+                                      Project [cd_demo_sk]
+                                        Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status]
+                            InputAdapter
+                              BroadcastExchange #2
+                                WholeStageCodegen (2)
+                                  Project [d_date_sk]
+                                    Filter [d_year,d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.date_dim [d_date_sk,d_year]
+                        InputAdapter
+                          BroadcastExchange #4
+                            WholeStageCodegen (3)
+                              Filter [s_state,s_store_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.store [s_store_sk,s_state]
+                    InputAdapter
+                      BroadcastExchange #5
+                        WholeStageCodegen (4)
+                          Filter [i_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.item [i_item_sk,i_item_id]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27/explain.txt
new file mode 100644
index 0000000000000..8c530f5a37885
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27/explain.txt
@@ -0,0 +1,204 @@
+== Physical Plan ==
+TakeOrderedAndProject (34)
++- * HashAggregate (33)
+   +- Exchange (32)
+      +- * HashAggregate (31)
+         +- * Expand (30)
+            +- * Project (29)
+               +- * BroadcastHashJoin Inner BuildRight (28)
+                  :- * Project (23)
+                  :  +- * BroadcastHashJoin Inner BuildRight (22)
+                  :     :- * Project (17)
+                  :     :  +- * BroadcastHashJoin Inner BuildRight (16)
+                  :     :     :- * Project (10)
+                  :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+                  :     :     :     :- * Filter (3)
+                  :     :     :     :  +- * ColumnarToRow (2)
+                  :     :     :     :     +- Scan parquet default.store_sales (1)
+                  :     :     :     +- BroadcastExchange (8)
+                  :     :     :        +- * Project (7)
+                  :     :     :           +- * Filter (6)
+                  :     :     :              +- * ColumnarToRow (5)
+                  :     :     :                 +- Scan parquet default.customer_demographics (4)
+                  :     :     +- BroadcastExchange (15)
+                  :     :        +- * Project (14)
+                  :     :           +- * Filter (13)
+                  :     :              +- * ColumnarToRow (12)
+                  :     :                 +- Scan parquet default.date_dim (11)
+                  :     +- BroadcastExchange (21)
+                  :        +- * Filter (20)
+                  :           +- * ColumnarToRow (19)
+                  :              +- Scan parquet default.store (18)
+                  +- BroadcastExchange (27)
+                     +- * Filter (26)
+                        +- * ColumnarToRow (25)
+                           +- Scan parquet default.item (24)
+
+
+(1) Scan parquet default.store_sales
+Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_cdemo_sk:int,ss_store_sk:int,ss_quantity:int,ss_list_price:decimal(7,2),ss_sales_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+
+(3) Filter [codegen id : 5]
+Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1))
+
+(4) Scan parquet default.customer_demographics
+Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_marital_status,S), EqualTo(cd_education_status,College             ), IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_education_status:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+
+(6) Filter [codegen id : 1]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (cd_gender#11 = M)) AND (cd_marital_status#12 = S)) AND (cd_education_status#13 = College             )) AND isnotnull(cd_demo_sk#10))
+
+(7) Project [codegen id : 1]
+Output [1]: [cd_demo_sk#10]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+
+(8) BroadcastExchange
+Input [1]: [cd_demo_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#10]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#10]
+
+(11) Scan parquet default.date_dim
+Output [2]: [d_date_sk#15, d_year#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#15, d_year#16]
+
+(13) Filter [codegen id : 2]
+Input [2]: [d_date_sk#15, d_year#16]
+Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2002)) AND isnotnull(d_date_sk#15))
+
+(14) Project [codegen id : 2]
+Output [1]: [d_date_sk#15]
+Input [2]: [d_date_sk#15, d_year#16]
+
+(15) BroadcastExchange
+Input [1]: [d_date_sk#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(16) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#15]
+Join condition: None
+
+(17) Project [codegen id : 5]
+Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7]
+Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#15]
+
+(18) Scan parquet default.store
+Output [2]: [s_store_sk#18, s_state#19]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_state:string>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [2]: [s_store_sk#18, s_state#19]
+
+(20) Filter [codegen id : 3]
+Input [2]: [s_store_sk#18, s_state#19]
+Condition : ((isnotnull(s_state#19) AND (s_state#19 = TN)) AND isnotnull(s_store_sk#18))
+
+(21) BroadcastExchange
+Input [2]: [s_store_sk#18, s_state#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#20]
+
+(22) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#18]
+Join condition: None
+
+(23) Project [codegen id : 5]
+Output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#19]
+Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#18, s_state#19]
+
+(24) Scan parquet default.item
+Output [2]: [i_item_sk#21, i_item_id#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(25) ColumnarToRow [codegen id : 4]
+Input [2]: [i_item_sk#21, i_item_id#22]
+
+(26) Filter [codegen id : 4]
+Input [2]: [i_item_sk#21, i_item_id#22]
+Condition : isnotnull(i_item_sk#21)
+
+(27) BroadcastExchange
+Input [2]: [i_item_sk#21, i_item_id#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#23]
+
+(28) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#21]
+Join condition: None
+
+(29) Project [codegen id : 5]
+Output [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#22, s_state#19]
+Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#19, i_item_sk#21, i_item_id#22]
+
+(30) Expand [codegen id : 5]
+Input [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#22, s_state#19]
+Arguments: [List(ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#22, s_state#19, 0), List(ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#22, null, 1), List(ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, null, null, 3)], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#24, s_state#25, spark_grouping_id#26]
+
+(31) HashAggregate [codegen id : 5]
+Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#24, s_state#25, spark_grouping_id#26]
+Keys [3]: [i_item_id#24, s_state#25, spark_grouping_id#26]
+Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))]
+Aggregate Attributes [8]: [sum#27, count#28, sum#29, count#30, sum#31, count#32, sum#33, count#34]
+Results [11]: [i_item_id#24, s_state#25, spark_grouping_id#26, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42]
+
+(32) Exchange
+Input [11]: [i_item_id#24, s_state#25, spark_grouping_id#26, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42]
+Arguments: hashpartitioning(i_item_id#24, s_state#25, spark_grouping_id#26, 5), ENSURE_REQUIREMENTS, [id=#43]
+
+(33) HashAggregate [codegen id : 6]
+Input [11]: [i_item_id#24, s_state#25, spark_grouping_id#26, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42]
+Keys [3]: [i_item_id#24, s_state#25, spark_grouping_id#26]
+Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))]
+Aggregate Attributes [4]: [avg(ss_quantity#4)#44, avg(UnscaledValue(ss_list_price#5))#45, avg(UnscaledValue(ss_coupon_amt#7))#46, avg(UnscaledValue(ss_sales_price#6))#47]
+Results [7]: [i_item_id#24, s_state#25, cast((shiftright(spark_grouping_id#26, 0) & 1) as tinyint) AS g_state#48, avg(ss_quantity#4)#44 AS agg1#49, cast((avg(UnscaledValue(ss_list_price#5))#45 / 100.0) as decimal(11,6)) AS agg2#50, cast((avg(UnscaledValue(ss_coupon_amt#7))#46 / 100.0) as decimal(11,6)) AS agg3#51, cast((avg(UnscaledValue(ss_sales_price#6))#47 / 100.0) as decimal(11,6)) AS agg4#52]
+
+(34) TakeOrderedAndProject
+Input [7]: [i_item_id#24, s_state#25, g_state#48, agg1#49, agg2#50, agg3#51, agg4#52]
+Arguments: 100, [i_item_id#24 ASC NULLS FIRST, s_state#25 ASC NULLS FIRST], [i_item_id#24, s_state#25, g_state#48, agg1#49, agg2#50, agg3#51, agg4#52]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (35)
+
+
+(35) ReusedExchange [Reuses operator id: 15]
+Output [1]: [d_date_sk#15]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27/simplified.txt
new file mode 100644
index 0000000000000..6724eceb18711
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27/simplified.txt
@@ -0,0 +1,52 @@
+TakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4]
+  WholeStageCodegen (6)
+    HashAggregate [i_item_id,s_state,spark_grouping_id,sum,count,sum,count,sum,count,sum,count] [avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price)),g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count]
+      InputAdapter
+        Exchange [i_item_id,s_state,spark_grouping_id] #1
+          WholeStageCodegen (5)
+            HashAggregate [i_item_id,s_state,spark_grouping_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+              Expand [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state]
+                Project [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state]
+                  BroadcastHashJoin [ss_item_sk,i_item_sk]
+                    Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state]
+                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                        Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                            Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                              BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk]
+                                Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                                        SubqueryBroadcast [d_date_sk] #1
+                                          ReusedExchange [d_date_sk] #2
+                                InputAdapter
+                                  BroadcastExchange #3
+                                    WholeStageCodegen (1)
+                                      Project [cd_demo_sk]
+                                        Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status]
+                            InputAdapter
+                              BroadcastExchange #2
+                                WholeStageCodegen (2)
+                                  Project [d_date_sk]
+                                    Filter [d_year,d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.date_dim [d_date_sk,d_year]
+                        InputAdapter
+                          BroadcastExchange #4
+                            WholeStageCodegen (3)
+                              Filter [s_state,s_store_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.store [s_store_sk,s_state]
+                    InputAdapter
+                      BroadcastExchange #5
+                        WholeStageCodegen (4)
+                          Filter [i_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.item [i_item_sk,i_item_id]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.sf100/explain.txt
new file mode 100644
index 0000000000000..eec45ea549531
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.sf100/explain.txt
@@ -0,0 +1,432 @@
+== Physical Plan ==
+* BroadcastNestedLoopJoin Inner BuildRight (70)
+:- * BroadcastNestedLoopJoin Inner BuildRight (58)
+:  :- * BroadcastNestedLoopJoin Inner BuildRight (46)
+:  :  :- * BroadcastNestedLoopJoin Inner BuildRight (34)
+:  :  :  :- * BroadcastNestedLoopJoin Inner BuildRight (22)
+:  :  :  :  :- * HashAggregate (10)
+:  :  :  :  :  +- Exchange (9)
+:  :  :  :  :     +- * HashAggregate (8)
+:  :  :  :  :        +- * HashAggregate (7)
+:  :  :  :  :           +- Exchange (6)
+:  :  :  :  :              +- * HashAggregate (5)
+:  :  :  :  :                 +- * Project (4)
+:  :  :  :  :                    +- * Filter (3)
+:  :  :  :  :                       +- * ColumnarToRow (2)
+:  :  :  :  :                          +- Scan parquet default.store_sales (1)
+:  :  :  :  +- BroadcastExchange (21)
+:  :  :  :     +- * HashAggregate (20)
+:  :  :  :        +- Exchange (19)
+:  :  :  :           +- * HashAggregate (18)
+:  :  :  :              +- * HashAggregate (17)
+:  :  :  :                 +- Exchange (16)
+:  :  :  :                    +- * HashAggregate (15)
+:  :  :  :                       +- * Project (14)
+:  :  :  :                          +- * Filter (13)
+:  :  :  :                             +- * ColumnarToRow (12)
+:  :  :  :                                +- Scan parquet default.store_sales (11)
+:  :  :  +- BroadcastExchange (33)
+:  :  :     +- * HashAggregate (32)
+:  :  :        +- Exchange (31)
+:  :  :           +- * HashAggregate (30)
+:  :  :              +- * HashAggregate (29)
+:  :  :                 +- Exchange (28)
+:  :  :                    +- * HashAggregate (27)
+:  :  :                       +- * Project (26)
+:  :  :                          +- * Filter (25)
+:  :  :                             +- * ColumnarToRow (24)
+:  :  :                                +- Scan parquet default.store_sales (23)
+:  :  +- BroadcastExchange (45)
+:  :     +- * HashAggregate (44)
+:  :        +- Exchange (43)
+:  :           +- * HashAggregate (42)
+:  :              +- * HashAggregate (41)
+:  :                 +- Exchange (40)
+:  :                    +- * HashAggregate (39)
+:  :                       +- * Project (38)
+:  :                          +- * Filter (37)
+:  :                             +- * ColumnarToRow (36)
+:  :                                +- Scan parquet default.store_sales (35)
+:  +- BroadcastExchange (57)
+:     +- * HashAggregate (56)
+:        +- Exchange (55)
+:           +- * HashAggregate (54)
+:              +- * HashAggregate (53)
+:                 +- Exchange (52)
+:                    +- * HashAggregate (51)
+:                       +- * Project (50)
+:                          +- * Filter (49)
+:                             +- * ColumnarToRow (48)
+:                                +- Scan parquet default.store_sales (47)
++- BroadcastExchange (69)
+   +- * HashAggregate (68)
+      +- Exchange (67)
+         +- * HashAggregate (66)
+            +- * HashAggregate (65)
+               +- Exchange (64)
+                  +- * HashAggregate (63)
+                     +- * Project (62)
+                        +- * Filter (61)
+                           +- * ColumnarToRow (60)
+                              +- Scan parquet default.store_sales (59)
+
+
+(1) Scan parquet default.store_sales
+Output [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,0), LessThanOrEqual(ss_quantity,5), Or(Or(And(GreaterThanOrEqual(ss_list_price,8.00),LessThanOrEqual(ss_list_price,18.00)),And(GreaterThanOrEqual(ss_coupon_amt,459.00),LessThanOrEqual(ss_coupon_amt,1459.00))),And(GreaterThanOrEqual(ss_wholesale_cost,57.00),LessThanOrEqual(ss_wholesale_cost,77.00)))]
+ReadSchema: struct<ss_quantity:int,ss_wholesale_cost:decimal(7,2),ss_list_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5]
+
+(3) Filter [codegen id : 1]
+Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5]
+Condition : (((isnotnull(ss_quantity#1) AND (ss_quantity#1 >= 0)) AND (ss_quantity#1 <= 5)) AND ((((ss_list_price#3 >= 8.00) AND (ss_list_price#3 <= 18.00)) OR ((ss_coupon_amt#4 >= 459.00) AND (ss_coupon_amt#4 <= 1459.00))) OR ((ss_wholesale_cost#2 >= 57.00) AND (ss_wholesale_cost#2 <= 77.00))))
+
+(4) Project [codegen id : 1]
+Output [1]: [ss_list_price#3]
+Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5]
+
+(5) HashAggregate [codegen id : 1]
+Input [1]: [ss_list_price#3]
+Keys [1]: [ss_list_price#3]
+Functions [2]: [partial_avg(UnscaledValue(ss_list_price#3)), partial_count(ss_list_price#3)]
+Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7]
+Results [4]: [ss_list_price#3, sum#8, count#9, count#10]
+
+(6) Exchange
+Input [4]: [ss_list_price#3, sum#8, count#9, count#10]
+Arguments: hashpartitioning(ss_list_price#3, 5), ENSURE_REQUIREMENTS, [id=#11]
+
+(7) HashAggregate [codegen id : 2]
+Input [4]: [ss_list_price#3, sum#8, count#9, count#10]
+Keys [1]: [ss_list_price#3]
+Functions [2]: [merge_avg(UnscaledValue(ss_list_price#3)), merge_count(ss_list_price#3)]
+Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7]
+Results [4]: [ss_list_price#3, sum#8, count#9, count#10]
+
+(8) HashAggregate [codegen id : 2]
+Input [4]: [ss_list_price#3, sum#8, count#9, count#10]
+Keys: []
+Functions [3]: [merge_avg(UnscaledValue(ss_list_price#3)), merge_count(ss_list_price#3), partial_count(distinct ss_list_price#3)]
+Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7, count(ss_list_price#3)#12]
+Results [4]: [sum#8, count#9, count#10, count#13]
+
+(9) Exchange
+Input [4]: [sum#8, count#9, count#10, count#13]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#14]
+
+(10) HashAggregate [codegen id : 18]
+Input [4]: [sum#8, count#9, count#10, count#13]
+Keys: []
+Functions [3]: [avg(UnscaledValue(ss_list_price#3)), count(ss_list_price#3), count(distinct ss_list_price#3)]
+Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7, count(ss_list_price#3)#12]
+Results [3]: [cast((avg(UnscaledValue(ss_list_price#3))#6 / 100.0) as decimal(11,6)) AS B1_LP#15, count(ss_list_price#3)#7 AS B1_CNT#16, count(ss_list_price#3)#12 AS B1_CNTD#17]
+
+(11) Scan parquet default.store_sales
+Output [5]: [ss_quantity#18, ss_wholesale_cost#19, ss_list_price#20, ss_coupon_amt#21, ss_sold_date_sk#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,6), LessThanOrEqual(ss_quantity,10), Or(Or(And(GreaterThanOrEqual(ss_list_price,90.00),LessThanOrEqual(ss_list_price,100.00)),And(GreaterThanOrEqual(ss_coupon_amt,2323.00),LessThanOrEqual(ss_coupon_amt,3323.00))),And(GreaterThanOrEqual(ss_wholesale_cost,31.00),LessThanOrEqual(ss_wholesale_cost,51.00)))]
+ReadSchema: struct<ss_quantity:int,ss_wholesale_cost:decimal(7,2),ss_list_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(12) ColumnarToRow [codegen id : 3]
+Input [5]: [ss_quantity#18, ss_wholesale_cost#19, ss_list_price#20, ss_coupon_amt#21, ss_sold_date_sk#22]
+
+(13) Filter [codegen id : 3]
+Input [5]: [ss_quantity#18, ss_wholesale_cost#19, ss_list_price#20, ss_coupon_amt#21, ss_sold_date_sk#22]
+Condition : (((isnotnull(ss_quantity#18) AND (ss_quantity#18 >= 6)) AND (ss_quantity#18 <= 10)) AND ((((ss_list_price#20 >= 90.00) AND (ss_list_price#20 <= 100.00)) OR ((ss_coupon_amt#21 >= 2323.00) AND (ss_coupon_amt#21 <= 3323.00))) OR ((ss_wholesale_cost#19 >= 31.00) AND (ss_wholesale_cost#19 <= 51.00))))
+
+(14) Project [codegen id : 3]
+Output [1]: [ss_list_price#20]
+Input [5]: [ss_quantity#18, ss_wholesale_cost#19, ss_list_price#20, ss_coupon_amt#21, ss_sold_date_sk#22]
+
+(15) HashAggregate [codegen id : 3]
+Input [1]: [ss_list_price#20]
+Keys [1]: [ss_list_price#20]
+Functions [2]: [partial_avg(UnscaledValue(ss_list_price#20)), partial_count(ss_list_price#20)]
+Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#20))#23, count(ss_list_price#20)#24]
+Results [4]: [ss_list_price#20, sum#25, count#26, count#27]
+
+(16) Exchange
+Input [4]: [ss_list_price#20, sum#25, count#26, count#27]
+Arguments: hashpartitioning(ss_list_price#20, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(17) HashAggregate [codegen id : 4]
+Input [4]: [ss_list_price#20, sum#25, count#26, count#27]
+Keys [1]: [ss_list_price#20]
+Functions [2]: [merge_avg(UnscaledValue(ss_list_price#20)), merge_count(ss_list_price#20)]
+Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#20))#23, count(ss_list_price#20)#24]
+Results [4]: [ss_list_price#20, sum#25, count#26, count#27]
+
+(18) HashAggregate [codegen id : 4]
+Input [4]: [ss_list_price#20, sum#25, count#26, count#27]
+Keys: []
+Functions [3]: [merge_avg(UnscaledValue(ss_list_price#20)), merge_count(ss_list_price#20), partial_count(distinct ss_list_price#20)]
+Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#20))#23, count(ss_list_price#20)#24, count(ss_list_price#20)#29]
+Results [4]: [sum#25, count#26, count#27, count#30]
+
+(19) Exchange
+Input [4]: [sum#25, count#26, count#27, count#30]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#31]
+
+(20) HashAggregate [codegen id : 5]
+Input [4]: [sum#25, count#26, count#27, count#30]
+Keys: []
+Functions [3]: [avg(UnscaledValue(ss_list_price#20)), count(ss_list_price#20), count(distinct ss_list_price#20)]
+Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#20))#23, count(ss_list_price#20)#24, count(ss_list_price#20)#29]
+Results [3]: [cast((avg(UnscaledValue(ss_list_price#20))#23 / 100.0) as decimal(11,6)) AS B2_LP#32, count(ss_list_price#20)#24 AS B2_CNT#33, count(ss_list_price#20)#29 AS B2_CNTD#34]
+
+(21) BroadcastExchange
+Input [3]: [B2_LP#32, B2_CNT#33, B2_CNTD#34]
+Arguments: IdentityBroadcastMode, [id=#35]
+
+(22) BroadcastNestedLoopJoin [codegen id : 18]
+Join condition: None
+
+(23) Scan parquet default.store_sales
+Output [5]: [ss_quantity#36, ss_wholesale_cost#37, ss_list_price#38, ss_coupon_amt#39, ss_sold_date_sk#40]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,11), LessThanOrEqual(ss_quantity,15), Or(Or(And(GreaterThanOrEqual(ss_list_price,142.00),LessThanOrEqual(ss_list_price,152.00)),And(GreaterThanOrEqual(ss_coupon_amt,12214.00),LessThanOrEqual(ss_coupon_amt,13214.00))),And(GreaterThanOrEqual(ss_wholesale_cost,79.00),LessThanOrEqual(ss_wholesale_cost,99.00)))]
+ReadSchema: struct<ss_quantity:int,ss_wholesale_cost:decimal(7,2),ss_list_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(24) ColumnarToRow [codegen id : 6]
+Input [5]: [ss_quantity#36, ss_wholesale_cost#37, ss_list_price#38, ss_coupon_amt#39, ss_sold_date_sk#40]
+
+(25) Filter [codegen id : 6]
+Input [5]: [ss_quantity#36, ss_wholesale_cost#37, ss_list_price#38, ss_coupon_amt#39, ss_sold_date_sk#40]
+Condition : (((isnotnull(ss_quantity#36) AND (ss_quantity#36 >= 11)) AND (ss_quantity#36 <= 15)) AND ((((ss_list_price#38 >= 142.00) AND (ss_list_price#38 <= 152.00)) OR ((ss_coupon_amt#39 >= 12214.00) AND (ss_coupon_amt#39 <= 13214.00))) OR ((ss_wholesale_cost#37 >= 79.00) AND (ss_wholesale_cost#37 <= 99.00))))
+
+(26) Project [codegen id : 6]
+Output [1]: [ss_list_price#38]
+Input [5]: [ss_quantity#36, ss_wholesale_cost#37, ss_list_price#38, ss_coupon_amt#39, ss_sold_date_sk#40]
+
+(27) HashAggregate [codegen id : 6]
+Input [1]: [ss_list_price#38]
+Keys [1]: [ss_list_price#38]
+Functions [2]: [partial_avg(UnscaledValue(ss_list_price#38)), partial_count(ss_list_price#38)]
+Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#38))#41, count(ss_list_price#38)#42]
+Results [4]: [ss_list_price#38, sum#43, count#44, count#45]
+
+(28) Exchange
+Input [4]: [ss_list_price#38, sum#43, count#44, count#45]
+Arguments: hashpartitioning(ss_list_price#38, 5), ENSURE_REQUIREMENTS, [id=#46]
+
+(29) HashAggregate [codegen id : 7]
+Input [4]: [ss_list_price#38, sum#43, count#44, count#45]
+Keys [1]: [ss_list_price#38]
+Functions [2]: [merge_avg(UnscaledValue(ss_list_price#38)), merge_count(ss_list_price#38)]
+Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#38))#41, count(ss_list_price#38)#42]
+Results [4]: [ss_list_price#38, sum#43, count#44, count#45]
+
+(30) HashAggregate [codegen id : 7]
+Input [4]: [ss_list_price#38, sum#43, count#44, count#45]
+Keys: []
+Functions [3]: [merge_avg(UnscaledValue(ss_list_price#38)), merge_count(ss_list_price#38), partial_count(distinct ss_list_price#38)]
+Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#38))#41, count(ss_list_price#38)#42, count(ss_list_price#38)#47]
+Results [4]: [sum#43, count#44, count#45, count#48]
+
+(31) Exchange
+Input [4]: [sum#43, count#44, count#45, count#48]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#49]
+
+(32) HashAggregate [codegen id : 8]
+Input [4]: [sum#43, count#44, count#45, count#48]
+Keys: []
+Functions [3]: [avg(UnscaledValue(ss_list_price#38)), count(ss_list_price#38), count(distinct ss_list_price#38)]
+Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#38))#41, count(ss_list_price#38)#42, count(ss_list_price#38)#47]
+Results [3]: [cast((avg(UnscaledValue(ss_list_price#38))#41 / 100.0) as decimal(11,6)) AS B3_LP#50, count(ss_list_price#38)#42 AS B3_CNT#51, count(ss_list_price#38)#47 AS B3_CNTD#52]
+
+(33) BroadcastExchange
+Input [3]: [B3_LP#50, B3_CNT#51, B3_CNTD#52]
+Arguments: IdentityBroadcastMode, [id=#53]
+
+(34) BroadcastNestedLoopJoin [codegen id : 18]
+Join condition: None
+
+(35) Scan parquet default.store_sales
+Output [5]: [ss_quantity#54, ss_wholesale_cost#55, ss_list_price#56, ss_coupon_amt#57, ss_sold_date_sk#58]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,16), LessThanOrEqual(ss_quantity,20), Or(Or(And(GreaterThanOrEqual(ss_list_price,135.00),LessThanOrEqual(ss_list_price,145.00)),And(GreaterThanOrEqual(ss_coupon_amt,6071.00),LessThanOrEqual(ss_coupon_amt,7071.00))),And(GreaterThanOrEqual(ss_wholesale_cost,38.00),LessThanOrEqual(ss_wholesale_cost,58.00)))]
+ReadSchema: struct<ss_quantity:int,ss_wholesale_cost:decimal(7,2),ss_list_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(36) ColumnarToRow [codegen id : 9]
+Input [5]: [ss_quantity#54, ss_wholesale_cost#55, ss_list_price#56, ss_coupon_amt#57, ss_sold_date_sk#58]
+
+(37) Filter [codegen id : 9]
+Input [5]: [ss_quantity#54, ss_wholesale_cost#55, ss_list_price#56, ss_coupon_amt#57, ss_sold_date_sk#58]
+Condition : (((isnotnull(ss_quantity#54) AND (ss_quantity#54 >= 16)) AND (ss_quantity#54 <= 20)) AND ((((ss_list_price#56 >= 135.00) AND (ss_list_price#56 <= 145.00)) OR ((ss_coupon_amt#57 >= 6071.00) AND (ss_coupon_amt#57 <= 7071.00))) OR ((ss_wholesale_cost#55 >= 38.00) AND (ss_wholesale_cost#55 <= 58.00))))
+
+(38) Project [codegen id : 9]
+Output [1]: [ss_list_price#56]
+Input [5]: [ss_quantity#54, ss_wholesale_cost#55, ss_list_price#56, ss_coupon_amt#57, ss_sold_date_sk#58]
+
+(39) HashAggregate [codegen id : 9]
+Input [1]: [ss_list_price#56]
+Keys [1]: [ss_list_price#56]
+Functions [2]: [partial_avg(UnscaledValue(ss_list_price#56)), partial_count(ss_list_price#56)]
+Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#56))#59, count(ss_list_price#56)#60]
+Results [4]: [ss_list_price#56, sum#61, count#62, count#63]
+
+(40) Exchange
+Input [4]: [ss_list_price#56, sum#61, count#62, count#63]
+Arguments: hashpartitioning(ss_list_price#56, 5), ENSURE_REQUIREMENTS, [id=#64]
+
+(41) HashAggregate [codegen id : 10]
+Input [4]: [ss_list_price#56, sum#61, count#62, count#63]
+Keys [1]: [ss_list_price#56]
+Functions [2]: [merge_avg(UnscaledValue(ss_list_price#56)), merge_count(ss_list_price#56)]
+Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#56))#59, count(ss_list_price#56)#60]
+Results [4]: [ss_list_price#56, sum#61, count#62, count#63]
+
+(42) HashAggregate [codegen id : 10]
+Input [4]: [ss_list_price#56, sum#61, count#62, count#63]
+Keys: []
+Functions [3]: [merge_avg(UnscaledValue(ss_list_price#56)), merge_count(ss_list_price#56), partial_count(distinct ss_list_price#56)]
+Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#56))#59, count(ss_list_price#56)#60, count(ss_list_price#56)#65]
+Results [4]: [sum#61, count#62, count#63, count#66]
+
+(43) Exchange
+Input [4]: [sum#61, count#62, count#63, count#66]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#67]
+
+(44) HashAggregate [codegen id : 11]
+Input [4]: [sum#61, count#62, count#63, count#66]
+Keys: []
+Functions [3]: [avg(UnscaledValue(ss_list_price#56)), count(ss_list_price#56), count(distinct ss_list_price#56)]
+Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#56))#59, count(ss_list_price#56)#60, count(ss_list_price#56)#65]
+Results [3]: [cast((avg(UnscaledValue(ss_list_price#56))#59 / 100.0) as decimal(11,6)) AS B4_LP#68, count(ss_list_price#56)#60 AS B4_CNT#69, count(ss_list_price#56)#65 AS B4_CNTD#70]
+
+(45) BroadcastExchange
+Input [3]: [B4_LP#68, B4_CNT#69, B4_CNTD#70]
+Arguments: IdentityBroadcastMode, [id=#71]
+
+(46) BroadcastNestedLoopJoin [codegen id : 18]
+Join condition: None
+
+(47) Scan parquet default.store_sales
+Output [5]: [ss_quantity#72, ss_wholesale_cost#73, ss_list_price#74, ss_coupon_amt#75, ss_sold_date_sk#76]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,25), Or(Or(And(GreaterThanOrEqual(ss_list_price,122.00),LessThanOrEqual(ss_list_price,132.00)),And(GreaterThanOrEqual(ss_coupon_amt,836.00),LessThanOrEqual(ss_coupon_amt,1836.00))),And(GreaterThanOrEqual(ss_wholesale_cost,17.00),LessThanOrEqual(ss_wholesale_cost,37.00)))]
+ReadSchema: struct<ss_quantity:int,ss_wholesale_cost:decimal(7,2),ss_list_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(48) ColumnarToRow [codegen id : 12]
+Input [5]: [ss_quantity#72, ss_wholesale_cost#73, ss_list_price#74, ss_coupon_amt#75, ss_sold_date_sk#76]
+
+(49) Filter [codegen id : 12]
+Input [5]: [ss_quantity#72, ss_wholesale_cost#73, ss_list_price#74, ss_coupon_amt#75, ss_sold_date_sk#76]
+Condition : (((isnotnull(ss_quantity#72) AND (ss_quantity#72 >= 21)) AND (ss_quantity#72 <= 25)) AND ((((ss_list_price#74 >= 122.00) AND (ss_list_price#74 <= 132.00)) OR ((ss_coupon_amt#75 >= 836.00) AND (ss_coupon_amt#75 <= 1836.00))) OR ((ss_wholesale_cost#73 >= 17.00) AND (ss_wholesale_cost#73 <= 37.00))))
+
+(50) Project [codegen id : 12]
+Output [1]: [ss_list_price#74]
+Input [5]: [ss_quantity#72, ss_wholesale_cost#73, ss_list_price#74, ss_coupon_amt#75, ss_sold_date_sk#76]
+
+(51) HashAggregate [codegen id : 12]
+Input [1]: [ss_list_price#74]
+Keys [1]: [ss_list_price#74]
+Functions [2]: [partial_avg(UnscaledValue(ss_list_price#74)), partial_count(ss_list_price#74)]
+Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#74))#77, count(ss_list_price#74)#78]
+Results [4]: [ss_list_price#74, sum#79, count#80, count#81]
+
+(52) Exchange
+Input [4]: [ss_list_price#74, sum#79, count#80, count#81]
+Arguments: hashpartitioning(ss_list_price#74, 5), ENSURE_REQUIREMENTS, [id=#82]
+
+(53) HashAggregate [codegen id : 13]
+Input [4]: [ss_list_price#74, sum#79, count#80, count#81]
+Keys [1]: [ss_list_price#74]
+Functions [2]: [merge_avg(UnscaledValue(ss_list_price#74)), merge_count(ss_list_price#74)]
+Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#74))#77, count(ss_list_price#74)#78]
+Results [4]: [ss_list_price#74, sum#79, count#80, count#81]
+
+(54) HashAggregate [codegen id : 13]
+Input [4]: [ss_list_price#74, sum#79, count#80, count#81]
+Keys: []
+Functions [3]: [merge_avg(UnscaledValue(ss_list_price#74)), merge_count(ss_list_price#74), partial_count(distinct ss_list_price#74)]
+Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#74))#77, count(ss_list_price#74)#78, count(ss_list_price#74)#83]
+Results [4]: [sum#79, count#80, count#81, count#84]
+
+(55) Exchange
+Input [4]: [sum#79, count#80, count#81, count#84]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#85]
+
+(56) HashAggregate [codegen id : 14]
+Input [4]: [sum#79, count#80, count#81, count#84]
+Keys: []
+Functions [3]: [avg(UnscaledValue(ss_list_price#74)), count(ss_list_price#74), count(distinct ss_list_price#74)]
+Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#74))#77, count(ss_list_price#74)#78, count(ss_list_price#74)#83]
+Results [3]: [cast((avg(UnscaledValue(ss_list_price#74))#77 / 100.0) as decimal(11,6)) AS B5_LP#86, count(ss_list_price#74)#78 AS B5_CNT#87, count(ss_list_price#74)#83 AS B5_CNTD#88]
+
+(57) BroadcastExchange
+Input [3]: [B5_LP#86, B5_CNT#87, B5_CNTD#88]
+Arguments: IdentityBroadcastMode, [id=#89]
+
+(58) BroadcastNestedLoopJoin [codegen id : 18]
+Join condition: None
+
+(59) Scan parquet default.store_sales
+Output [5]: [ss_quantity#90, ss_wholesale_cost#91, ss_list_price#92, ss_coupon_amt#93, ss_sold_date_sk#94]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,26), LessThanOrEqual(ss_quantity,30), Or(Or(And(GreaterThanOrEqual(ss_list_price,154.00),LessThanOrEqual(ss_list_price,164.00)),And(GreaterThanOrEqual(ss_coupon_amt,7326.00),LessThanOrEqual(ss_coupon_amt,8326.00))),And(GreaterThanOrEqual(ss_wholesale_cost,7.00),LessThanOrEqual(ss_wholesale_cost,27.00)))]
+ReadSchema: struct<ss_quantity:int,ss_wholesale_cost:decimal(7,2),ss_list_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(60) ColumnarToRow [codegen id : 15]
+Input [5]: [ss_quantity#90, ss_wholesale_cost#91, ss_list_price#92, ss_coupon_amt#93, ss_sold_date_sk#94]
+
+(61) Filter [codegen id : 15]
+Input [5]: [ss_quantity#90, ss_wholesale_cost#91, ss_list_price#92, ss_coupon_amt#93, ss_sold_date_sk#94]
+Condition : (((isnotnull(ss_quantity#90) AND (ss_quantity#90 >= 26)) AND (ss_quantity#90 <= 30)) AND ((((ss_list_price#92 >= 154.00) AND (ss_list_price#92 <= 164.00)) OR ((ss_coupon_amt#93 >= 7326.00) AND (ss_coupon_amt#93 <= 8326.00))) OR ((ss_wholesale_cost#91 >= 7.00) AND (ss_wholesale_cost#91 <= 27.00))))
+
+(62) Project [codegen id : 15]
+Output [1]: [ss_list_price#92]
+Input [5]: [ss_quantity#90, ss_wholesale_cost#91, ss_list_price#92, ss_coupon_amt#93, ss_sold_date_sk#94]
+
+(63) HashAggregate [codegen id : 15]
+Input [1]: [ss_list_price#92]
+Keys [1]: [ss_list_price#92]
+Functions [2]: [partial_avg(UnscaledValue(ss_list_price#92)), partial_count(ss_list_price#92)]
+Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#92))#95, count(ss_list_price#92)#96]
+Results [4]: [ss_list_price#92, sum#97, count#98, count#99]
+
+(64) Exchange
+Input [4]: [ss_list_price#92, sum#97, count#98, count#99]
+Arguments: hashpartitioning(ss_list_price#92, 5), ENSURE_REQUIREMENTS, [id=#100]
+
+(65) HashAggregate [codegen id : 16]
+Input [4]: [ss_list_price#92, sum#97, count#98, count#99]
+Keys [1]: [ss_list_price#92]
+Functions [2]: [merge_avg(UnscaledValue(ss_list_price#92)), merge_count(ss_list_price#92)]
+Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#92))#95, count(ss_list_price#92)#96]
+Results [4]: [ss_list_price#92, sum#97, count#98, count#99]
+
+(66) HashAggregate [codegen id : 16]
+Input [4]: [ss_list_price#92, sum#97, count#98, count#99]
+Keys: []
+Functions [3]: [merge_avg(UnscaledValue(ss_list_price#92)), merge_count(ss_list_price#92), partial_count(distinct ss_list_price#92)]
+Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#92))#95, count(ss_list_price#92)#96, count(ss_list_price#92)#101]
+Results [4]: [sum#97, count#98, count#99, count#102]
+
+(67) Exchange
+Input [4]: [sum#97, count#98, count#99, count#102]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#103]
+
+(68) HashAggregate [codegen id : 17]
+Input [4]: [sum#97, count#98, count#99, count#102]
+Keys: []
+Functions [3]: [avg(UnscaledValue(ss_list_price#92)), count(ss_list_price#92), count(distinct ss_list_price#92)]
+Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#92))#95, count(ss_list_price#92)#96, count(ss_list_price#92)#101]
+Results [3]: [cast((avg(UnscaledValue(ss_list_price#92))#95 / 100.0) as decimal(11,6)) AS B6_LP#104, count(ss_list_price#92)#96 AS B6_CNT#105, count(ss_list_price#92)#101 AS B6_CNTD#106]
+
+(69) BroadcastExchange
+Input [3]: [B6_LP#104, B6_CNT#105, B6_CNTD#106]
+Arguments: IdentityBroadcastMode, [id=#107]
+
+(70) BroadcastNestedLoopJoin [codegen id : 18]
+Join condition: None
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.sf100/simplified.txt
new file mode 100644
index 0000000000000..25317a944a6f7
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.sf100/simplified.txt
@@ -0,0 +1,111 @@
+WholeStageCodegen (18)
+  BroadcastNestedLoopJoin
+    BroadcastNestedLoopJoin
+      BroadcastNestedLoopJoin
+        BroadcastNestedLoopJoin
+          BroadcastNestedLoopJoin
+            HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B1_LP,B1_CNT,B1_CNTD,sum,count,count,count]
+              InputAdapter
+                Exchange #1
+                  WholeStageCodegen (2)
+                    HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count]
+                      HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
+                        InputAdapter
+                          Exchange [ss_list_price] #2
+                            WholeStageCodegen (1)
+                              HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
+                                Project [ss_list_price]
+                                  Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+            InputAdapter
+              BroadcastExchange #3
+                WholeStageCodegen (5)
+                  HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B2_LP,B2_CNT,B2_CNTD,sum,count,count,count]
+                    InputAdapter
+                      Exchange #4
+                        WholeStageCodegen (4)
+                          HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count]
+                            HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
+                              InputAdapter
+                                Exchange [ss_list_price] #5
+                                  WholeStageCodegen (3)
+                                    HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
+                                      Project [ss_list_price]
+                                        Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+          InputAdapter
+            BroadcastExchange #6
+              WholeStageCodegen (8)
+                HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B3_LP,B3_CNT,B3_CNTD,sum,count,count,count]
+                  InputAdapter
+                    Exchange #7
+                      WholeStageCodegen (7)
+                        HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count]
+                          HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
+                            InputAdapter
+                              Exchange [ss_list_price] #8
+                                WholeStageCodegen (6)
+                                  HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
+                                    Project [ss_list_price]
+                                      Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+        InputAdapter
+          BroadcastExchange #9
+            WholeStageCodegen (11)
+              HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B4_LP,B4_CNT,B4_CNTD,sum,count,count,count]
+                InputAdapter
+                  Exchange #10
+                    WholeStageCodegen (10)
+                      HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count]
+                        HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
+                          InputAdapter
+                            Exchange [ss_list_price] #11
+                              WholeStageCodegen (9)
+                                HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
+                                  Project [ss_list_price]
+                                    Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+      InputAdapter
+        BroadcastExchange #12
+          WholeStageCodegen (14)
+            HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B5_LP,B5_CNT,B5_CNTD,sum,count,count,count]
+              InputAdapter
+                Exchange #13
+                  WholeStageCodegen (13)
+                    HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count]
+                      HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
+                        InputAdapter
+                          Exchange [ss_list_price] #14
+                            WholeStageCodegen (12)
+                              HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
+                                Project [ss_list_price]
+                                  Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+    InputAdapter
+      BroadcastExchange #15
+        WholeStageCodegen (17)
+          HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B6_LP,B6_CNT,B6_CNTD,sum,count,count,count]
+            InputAdapter
+              Exchange #16
+                WholeStageCodegen (16)
+                  HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count]
+                    HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
+                      InputAdapter
+                        Exchange [ss_list_price] #17
+                          WholeStageCodegen (15)
+                            HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
+                              Project [ss_list_price]
+                                Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28/explain.txt
new file mode 100644
index 0000000000000..eec45ea549531
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28/explain.txt
@@ -0,0 +1,432 @@
+== Physical Plan ==
+* BroadcastNestedLoopJoin Inner BuildRight (70)
+:- * BroadcastNestedLoopJoin Inner BuildRight (58)
+:  :- * BroadcastNestedLoopJoin Inner BuildRight (46)
+:  :  :- * BroadcastNestedLoopJoin Inner BuildRight (34)
+:  :  :  :- * BroadcastNestedLoopJoin Inner BuildRight (22)
+:  :  :  :  :- * HashAggregate (10)
+:  :  :  :  :  +- Exchange (9)
+:  :  :  :  :     +- * HashAggregate (8)
+:  :  :  :  :        +- * HashAggregate (7)
+:  :  :  :  :           +- Exchange (6)
+:  :  :  :  :              +- * HashAggregate (5)
+:  :  :  :  :                 +- * Project (4)
+:  :  :  :  :                    +- * Filter (3)
+:  :  :  :  :                       +- * ColumnarToRow (2)
+:  :  :  :  :                          +- Scan parquet default.store_sales (1)
+:  :  :  :  +- BroadcastExchange (21)
+:  :  :  :     +- * HashAggregate (20)
+:  :  :  :        +- Exchange (19)
+:  :  :  :           +- * HashAggregate (18)
+:  :  :  :              +- * HashAggregate (17)
+:  :  :  :                 +- Exchange (16)
+:  :  :  :                    +- * HashAggregate (15)
+:  :  :  :                       +- * Project (14)
+:  :  :  :                          +- * Filter (13)
+:  :  :  :                             +- * ColumnarToRow (12)
+:  :  :  :                                +- Scan parquet default.store_sales (11)
+:  :  :  +- BroadcastExchange (33)
+:  :  :     +- * HashAggregate (32)
+:  :  :        +- Exchange (31)
+:  :  :           +- * HashAggregate (30)
+:  :  :              +- * HashAggregate (29)
+:  :  :                 +- Exchange (28)
+:  :  :                    +- * HashAggregate (27)
+:  :  :                       +- * Project (26)
+:  :  :                          +- * Filter (25)
+:  :  :                             +- * ColumnarToRow (24)
+:  :  :                                +- Scan parquet default.store_sales (23)
+:  :  +- BroadcastExchange (45)
+:  :     +- * HashAggregate (44)
+:  :        +- Exchange (43)
+:  :           +- * HashAggregate (42)
+:  :              +- * HashAggregate (41)
+:  :                 +- Exchange (40)
+:  :                    +- * HashAggregate (39)
+:  :                       +- * Project (38)
+:  :                          +- * Filter (37)
+:  :                             +- * ColumnarToRow (36)
+:  :                                +- Scan parquet default.store_sales (35)
+:  +- BroadcastExchange (57)
+:     +- * HashAggregate (56)
+:        +- Exchange (55)
+:           +- * HashAggregate (54)
+:              +- * HashAggregate (53)
+:                 +- Exchange (52)
+:                    +- * HashAggregate (51)
+:                       +- * Project (50)
+:                          +- * Filter (49)
+:                             +- * ColumnarToRow (48)
+:                                +- Scan parquet default.store_sales (47)
++- BroadcastExchange (69)
+   +- * HashAggregate (68)
+      +- Exchange (67)
+         +- * HashAggregate (66)
+            +- * HashAggregate (65)
+               +- Exchange (64)
+                  +- * HashAggregate (63)
+                     +- * Project (62)
+                        +- * Filter (61)
+                           +- * ColumnarToRow (60)
+                              +- Scan parquet default.store_sales (59)
+
+
+(1) Scan parquet default.store_sales
+Output [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,0), LessThanOrEqual(ss_quantity,5), Or(Or(And(GreaterThanOrEqual(ss_list_price,8.00),LessThanOrEqual(ss_list_price,18.00)),And(GreaterThanOrEqual(ss_coupon_amt,459.00),LessThanOrEqual(ss_coupon_amt,1459.00))),And(GreaterThanOrEqual(ss_wholesale_cost,57.00),LessThanOrEqual(ss_wholesale_cost,77.00)))]
+ReadSchema: struct<ss_quantity:int,ss_wholesale_cost:decimal(7,2),ss_list_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5]
+
+(3) Filter [codegen id : 1]
+Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5]
+Condition : (((isnotnull(ss_quantity#1) AND (ss_quantity#1 >= 0)) AND (ss_quantity#1 <= 5)) AND ((((ss_list_price#3 >= 8.00) AND (ss_list_price#3 <= 18.00)) OR ((ss_coupon_amt#4 >= 459.00) AND (ss_coupon_amt#4 <= 1459.00))) OR ((ss_wholesale_cost#2 >= 57.00) AND (ss_wholesale_cost#2 <= 77.00))))
+
+(4) Project [codegen id : 1]
+Output [1]: [ss_list_price#3]
+Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5]
+
+(5) HashAggregate [codegen id : 1]
+Input [1]: [ss_list_price#3]
+Keys [1]: [ss_list_price#3]
+Functions [2]: [partial_avg(UnscaledValue(ss_list_price#3)), partial_count(ss_list_price#3)]
+Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7]
+Results [4]: [ss_list_price#3, sum#8, count#9, count#10]
+
+(6) Exchange
+Input [4]: [ss_list_price#3, sum#8, count#9, count#10]
+Arguments: hashpartitioning(ss_list_price#3, 5), ENSURE_REQUIREMENTS, [id=#11]
+
+(7) HashAggregate [codegen id : 2]
+Input [4]: [ss_list_price#3, sum#8, count#9, count#10]
+Keys [1]: [ss_list_price#3]
+Functions [2]: [merge_avg(UnscaledValue(ss_list_price#3)), merge_count(ss_list_price#3)]
+Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7]
+Results [4]: [ss_list_price#3, sum#8, count#9, count#10]
+
+(8) HashAggregate [codegen id : 2]
+Input [4]: [ss_list_price#3, sum#8, count#9, count#10]
+Keys: []
+Functions [3]: [merge_avg(UnscaledValue(ss_list_price#3)), merge_count(ss_list_price#3), partial_count(distinct ss_list_price#3)]
+Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7, count(ss_list_price#3)#12]
+Results [4]: [sum#8, count#9, count#10, count#13]
+
+(9) Exchange
+Input [4]: [sum#8, count#9, count#10, count#13]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#14]
+
+(10) HashAggregate [codegen id : 18]
+Input [4]: [sum#8, count#9, count#10, count#13]
+Keys: []
+Functions [3]: [avg(UnscaledValue(ss_list_price#3)), count(ss_list_price#3), count(distinct ss_list_price#3)]
+Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7, count(ss_list_price#3)#12]
+Results [3]: [cast((avg(UnscaledValue(ss_list_price#3))#6 / 100.0) as decimal(11,6)) AS B1_LP#15, count(ss_list_price#3)#7 AS B1_CNT#16, count(ss_list_price#3)#12 AS B1_CNTD#17]
+
+(11) Scan parquet default.store_sales
+Output [5]: [ss_quantity#18, ss_wholesale_cost#19, ss_list_price#20, ss_coupon_amt#21, ss_sold_date_sk#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,6), LessThanOrEqual(ss_quantity,10), Or(Or(And(GreaterThanOrEqual(ss_list_price,90.00),LessThanOrEqual(ss_list_price,100.00)),And(GreaterThanOrEqual(ss_coupon_amt,2323.00),LessThanOrEqual(ss_coupon_amt,3323.00))),And(GreaterThanOrEqual(ss_wholesale_cost,31.00),LessThanOrEqual(ss_wholesale_cost,51.00)))]
+ReadSchema: struct<ss_quantity:int,ss_wholesale_cost:decimal(7,2),ss_list_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(12) ColumnarToRow [codegen id : 3]
+Input [5]: [ss_quantity#18, ss_wholesale_cost#19, ss_list_price#20, ss_coupon_amt#21, ss_sold_date_sk#22]
+
+(13) Filter [codegen id : 3]
+Input [5]: [ss_quantity#18, ss_wholesale_cost#19, ss_list_price#20, ss_coupon_amt#21, ss_sold_date_sk#22]
+Condition : (((isnotnull(ss_quantity#18) AND (ss_quantity#18 >= 6)) AND (ss_quantity#18 <= 10)) AND ((((ss_list_price#20 >= 90.00) AND (ss_list_price#20 <= 100.00)) OR ((ss_coupon_amt#21 >= 2323.00) AND (ss_coupon_amt#21 <= 3323.00))) OR ((ss_wholesale_cost#19 >= 31.00) AND (ss_wholesale_cost#19 <= 51.00))))
+
+(14) Project [codegen id : 3]
+Output [1]: [ss_list_price#20]
+Input [5]: [ss_quantity#18, ss_wholesale_cost#19, ss_list_price#20, ss_coupon_amt#21, ss_sold_date_sk#22]
+
+(15) HashAggregate [codegen id : 3]
+Input [1]: [ss_list_price#20]
+Keys [1]: [ss_list_price#20]
+Functions [2]: [partial_avg(UnscaledValue(ss_list_price#20)), partial_count(ss_list_price#20)]
+Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#20))#23, count(ss_list_price#20)#24]
+Results [4]: [ss_list_price#20, sum#25, count#26, count#27]
+
+(16) Exchange
+Input [4]: [ss_list_price#20, sum#25, count#26, count#27]
+Arguments: hashpartitioning(ss_list_price#20, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(17) HashAggregate [codegen id : 4]
+Input [4]: [ss_list_price#20, sum#25, count#26, count#27]
+Keys [1]: [ss_list_price#20]
+Functions [2]: [merge_avg(UnscaledValue(ss_list_price#20)), merge_count(ss_list_price#20)]
+Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#20))#23, count(ss_list_price#20)#24]
+Results [4]: [ss_list_price#20, sum#25, count#26, count#27]
+
+(18) HashAggregate [codegen id : 4]
+Input [4]: [ss_list_price#20, sum#25, count#26, count#27]
+Keys: []
+Functions [3]: [merge_avg(UnscaledValue(ss_list_price#20)), merge_count(ss_list_price#20), partial_count(distinct ss_list_price#20)]
+Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#20))#23, count(ss_list_price#20)#24, count(ss_list_price#20)#29]
+Results [4]: [sum#25, count#26, count#27, count#30]
+
+(19) Exchange
+Input [4]: [sum#25, count#26, count#27, count#30]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#31]
+
+(20) HashAggregate [codegen id : 5]
+Input [4]: [sum#25, count#26, count#27, count#30]
+Keys: []
+Functions [3]: [avg(UnscaledValue(ss_list_price#20)), count(ss_list_price#20), count(distinct ss_list_price#20)]
+Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#20))#23, count(ss_list_price#20)#24, count(ss_list_price#20)#29]
+Results [3]: [cast((avg(UnscaledValue(ss_list_price#20))#23 / 100.0) as decimal(11,6)) AS B2_LP#32, count(ss_list_price#20)#24 AS B2_CNT#33, count(ss_list_price#20)#29 AS B2_CNTD#34]
+
+(21) BroadcastExchange
+Input [3]: [B2_LP#32, B2_CNT#33, B2_CNTD#34]
+Arguments: IdentityBroadcastMode, [id=#35]
+
+(22) BroadcastNestedLoopJoin [codegen id : 18]
+Join condition: None
+
+(23) Scan parquet default.store_sales
+Output [5]: [ss_quantity#36, ss_wholesale_cost#37, ss_list_price#38, ss_coupon_amt#39, ss_sold_date_sk#40]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,11), LessThanOrEqual(ss_quantity,15), Or(Or(And(GreaterThanOrEqual(ss_list_price,142.00),LessThanOrEqual(ss_list_price,152.00)),And(GreaterThanOrEqual(ss_coupon_amt,12214.00),LessThanOrEqual(ss_coupon_amt,13214.00))),And(GreaterThanOrEqual(ss_wholesale_cost,79.00),LessThanOrEqual(ss_wholesale_cost,99.00)))]
+ReadSchema: struct<ss_quantity:int,ss_wholesale_cost:decimal(7,2),ss_list_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(24) ColumnarToRow [codegen id : 6]
+Input [5]: [ss_quantity#36, ss_wholesale_cost#37, ss_list_price#38, ss_coupon_amt#39, ss_sold_date_sk#40]
+
+(25) Filter [codegen id : 6]
+Input [5]: [ss_quantity#36, ss_wholesale_cost#37, ss_list_price#38, ss_coupon_amt#39, ss_sold_date_sk#40]
+Condition : (((isnotnull(ss_quantity#36) AND (ss_quantity#36 >= 11)) AND (ss_quantity#36 <= 15)) AND ((((ss_list_price#38 >= 142.00) AND (ss_list_price#38 <= 152.00)) OR ((ss_coupon_amt#39 >= 12214.00) AND (ss_coupon_amt#39 <= 13214.00))) OR ((ss_wholesale_cost#37 >= 79.00) AND (ss_wholesale_cost#37 <= 99.00))))
+
+(26) Project [codegen id : 6]
+Output [1]: [ss_list_price#38]
+Input [5]: [ss_quantity#36, ss_wholesale_cost#37, ss_list_price#38, ss_coupon_amt#39, ss_sold_date_sk#40]
+
+(27) HashAggregate [codegen id : 6]
+Input [1]: [ss_list_price#38]
+Keys [1]: [ss_list_price#38]
+Functions [2]: [partial_avg(UnscaledValue(ss_list_price#38)), partial_count(ss_list_price#38)]
+Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#38))#41, count(ss_list_price#38)#42]
+Results [4]: [ss_list_price#38, sum#43, count#44, count#45]
+
+(28) Exchange
+Input [4]: [ss_list_price#38, sum#43, count#44, count#45]
+Arguments: hashpartitioning(ss_list_price#38, 5), ENSURE_REQUIREMENTS, [id=#46]
+
+(29) HashAggregate [codegen id : 7]
+Input [4]: [ss_list_price#38, sum#43, count#44, count#45]
+Keys [1]: [ss_list_price#38]
+Functions [2]: [merge_avg(UnscaledValue(ss_list_price#38)), merge_count(ss_list_price#38)]
+Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#38))#41, count(ss_list_price#38)#42]
+Results [4]: [ss_list_price#38, sum#43, count#44, count#45]
+
+(30) HashAggregate [codegen id : 7]
+Input [4]: [ss_list_price#38, sum#43, count#44, count#45]
+Keys: []
+Functions [3]: [merge_avg(UnscaledValue(ss_list_price#38)), merge_count(ss_list_price#38), partial_count(distinct ss_list_price#38)]
+Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#38))#41, count(ss_list_price#38)#42, count(ss_list_price#38)#47]
+Results [4]: [sum#43, count#44, count#45, count#48]
+
+(31) Exchange
+Input [4]: [sum#43, count#44, count#45, count#48]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#49]
+
+(32) HashAggregate [codegen id : 8]
+Input [4]: [sum#43, count#44, count#45, count#48]
+Keys: []
+Functions [3]: [avg(UnscaledValue(ss_list_price#38)), count(ss_list_price#38), count(distinct ss_list_price#38)]
+Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#38))#41, count(ss_list_price#38)#42, count(ss_list_price#38)#47]
+Results [3]: [cast((avg(UnscaledValue(ss_list_price#38))#41 / 100.0) as decimal(11,6)) AS B3_LP#50, count(ss_list_price#38)#42 AS B3_CNT#51, count(ss_list_price#38)#47 AS B3_CNTD#52]
+
+(33) BroadcastExchange
+Input [3]: [B3_LP#50, B3_CNT#51, B3_CNTD#52]
+Arguments: IdentityBroadcastMode, [id=#53]
+
+(34) BroadcastNestedLoopJoin [codegen id : 18]
+Join condition: None
+
+(35) Scan parquet default.store_sales
+Output [5]: [ss_quantity#54, ss_wholesale_cost#55, ss_list_price#56, ss_coupon_amt#57, ss_sold_date_sk#58]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,16), LessThanOrEqual(ss_quantity,20), Or(Or(And(GreaterThanOrEqual(ss_list_price,135.00),LessThanOrEqual(ss_list_price,145.00)),And(GreaterThanOrEqual(ss_coupon_amt,6071.00),LessThanOrEqual(ss_coupon_amt,7071.00))),And(GreaterThanOrEqual(ss_wholesale_cost,38.00),LessThanOrEqual(ss_wholesale_cost,58.00)))]
+ReadSchema: struct<ss_quantity:int,ss_wholesale_cost:decimal(7,2),ss_list_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(36) ColumnarToRow [codegen id : 9]
+Input [5]: [ss_quantity#54, ss_wholesale_cost#55, ss_list_price#56, ss_coupon_amt#57, ss_sold_date_sk#58]
+
+(37) Filter [codegen id : 9]
+Input [5]: [ss_quantity#54, ss_wholesale_cost#55, ss_list_price#56, ss_coupon_amt#57, ss_sold_date_sk#58]
+Condition : (((isnotnull(ss_quantity#54) AND (ss_quantity#54 >= 16)) AND (ss_quantity#54 <= 20)) AND ((((ss_list_price#56 >= 135.00) AND (ss_list_price#56 <= 145.00)) OR ((ss_coupon_amt#57 >= 6071.00) AND (ss_coupon_amt#57 <= 7071.00))) OR ((ss_wholesale_cost#55 >= 38.00) AND (ss_wholesale_cost#55 <= 58.00))))
+
+(38) Project [codegen id : 9]
+Output [1]: [ss_list_price#56]
+Input [5]: [ss_quantity#54, ss_wholesale_cost#55, ss_list_price#56, ss_coupon_amt#57, ss_sold_date_sk#58]
+
+(39) HashAggregate [codegen id : 9]
+Input [1]: [ss_list_price#56]
+Keys [1]: [ss_list_price#56]
+Functions [2]: [partial_avg(UnscaledValue(ss_list_price#56)), partial_count(ss_list_price#56)]
+Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#56))#59, count(ss_list_price#56)#60]
+Results [4]: [ss_list_price#56, sum#61, count#62, count#63]
+
+(40) Exchange
+Input [4]: [ss_list_price#56, sum#61, count#62, count#63]
+Arguments: hashpartitioning(ss_list_price#56, 5), ENSURE_REQUIREMENTS, [id=#64]
+
+(41) HashAggregate [codegen id : 10]
+Input [4]: [ss_list_price#56, sum#61, count#62, count#63]
+Keys [1]: [ss_list_price#56]
+Functions [2]: [merge_avg(UnscaledValue(ss_list_price#56)), merge_count(ss_list_price#56)]
+Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#56))#59, count(ss_list_price#56)#60]
+Results [4]: [ss_list_price#56, sum#61, count#62, count#63]
+
+(42) HashAggregate [codegen id : 10]
+Input [4]: [ss_list_price#56, sum#61, count#62, count#63]
+Keys: []
+Functions [3]: [merge_avg(UnscaledValue(ss_list_price#56)), merge_count(ss_list_price#56), partial_count(distinct ss_list_price#56)]
+Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#56))#59, count(ss_list_price#56)#60, count(ss_list_price#56)#65]
+Results [4]: [sum#61, count#62, count#63, count#66]
+
+(43) Exchange
+Input [4]: [sum#61, count#62, count#63, count#66]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#67]
+
+(44) HashAggregate [codegen id : 11]
+Input [4]: [sum#61, count#62, count#63, count#66]
+Keys: []
+Functions [3]: [avg(UnscaledValue(ss_list_price#56)), count(ss_list_price#56), count(distinct ss_list_price#56)]
+Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#56))#59, count(ss_list_price#56)#60, count(ss_list_price#56)#65]
+Results [3]: [cast((avg(UnscaledValue(ss_list_price#56))#59 / 100.0) as decimal(11,6)) AS B4_LP#68, count(ss_list_price#56)#60 AS B4_CNT#69, count(ss_list_price#56)#65 AS B4_CNTD#70]
+
+(45) BroadcastExchange
+Input [3]: [B4_LP#68, B4_CNT#69, B4_CNTD#70]
+Arguments: IdentityBroadcastMode, [id=#71]
+
+(46) BroadcastNestedLoopJoin [codegen id : 18]
+Join condition: None
+
+(47) Scan parquet default.store_sales
+Output [5]: [ss_quantity#72, ss_wholesale_cost#73, ss_list_price#74, ss_coupon_amt#75, ss_sold_date_sk#76]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,25), Or(Or(And(GreaterThanOrEqual(ss_list_price,122.00),LessThanOrEqual(ss_list_price,132.00)),And(GreaterThanOrEqual(ss_coupon_amt,836.00),LessThanOrEqual(ss_coupon_amt,1836.00))),And(GreaterThanOrEqual(ss_wholesale_cost,17.00),LessThanOrEqual(ss_wholesale_cost,37.00)))]
+ReadSchema: struct<ss_quantity:int,ss_wholesale_cost:decimal(7,2),ss_list_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(48) ColumnarToRow [codegen id : 12]
+Input [5]: [ss_quantity#72, ss_wholesale_cost#73, ss_list_price#74, ss_coupon_amt#75, ss_sold_date_sk#76]
+
+(49) Filter [codegen id : 12]
+Input [5]: [ss_quantity#72, ss_wholesale_cost#73, ss_list_price#74, ss_coupon_amt#75, ss_sold_date_sk#76]
+Condition : (((isnotnull(ss_quantity#72) AND (ss_quantity#72 >= 21)) AND (ss_quantity#72 <= 25)) AND ((((ss_list_price#74 >= 122.00) AND (ss_list_price#74 <= 132.00)) OR ((ss_coupon_amt#75 >= 836.00) AND (ss_coupon_amt#75 <= 1836.00))) OR ((ss_wholesale_cost#73 >= 17.00) AND (ss_wholesale_cost#73 <= 37.00))))
+
+(50) Project [codegen id : 12]
+Output [1]: [ss_list_price#74]
+Input [5]: [ss_quantity#72, ss_wholesale_cost#73, ss_list_price#74, ss_coupon_amt#75, ss_sold_date_sk#76]
+
+(51) HashAggregate [codegen id : 12]
+Input [1]: [ss_list_price#74]
+Keys [1]: [ss_list_price#74]
+Functions [2]: [partial_avg(UnscaledValue(ss_list_price#74)), partial_count(ss_list_price#74)]
+Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#74))#77, count(ss_list_price#74)#78]
+Results [4]: [ss_list_price#74, sum#79, count#80, count#81]
+
+(52) Exchange
+Input [4]: [ss_list_price#74, sum#79, count#80, count#81]
+Arguments: hashpartitioning(ss_list_price#74, 5), ENSURE_REQUIREMENTS, [id=#82]
+
+(53) HashAggregate [codegen id : 13]
+Input [4]: [ss_list_price#74, sum#79, count#80, count#81]
+Keys [1]: [ss_list_price#74]
+Functions [2]: [merge_avg(UnscaledValue(ss_list_price#74)), merge_count(ss_list_price#74)]
+Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#74))#77, count(ss_list_price#74)#78]
+Results [4]: [ss_list_price#74, sum#79, count#80, count#81]
+
+(54) HashAggregate [codegen id : 13]
+Input [4]: [ss_list_price#74, sum#79, count#80, count#81]
+Keys: []
+Functions [3]: [merge_avg(UnscaledValue(ss_list_price#74)), merge_count(ss_list_price#74), partial_count(distinct ss_list_price#74)]
+Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#74))#77, count(ss_list_price#74)#78, count(ss_list_price#74)#83]
+Results [4]: [sum#79, count#80, count#81, count#84]
+
+(55) Exchange
+Input [4]: [sum#79, count#80, count#81, count#84]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#85]
+
+(56) HashAggregate [codegen id : 14]
+Input [4]: [sum#79, count#80, count#81, count#84]
+Keys: []
+Functions [3]: [avg(UnscaledValue(ss_list_price#74)), count(ss_list_price#74), count(distinct ss_list_price#74)]
+Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#74))#77, count(ss_list_price#74)#78, count(ss_list_price#74)#83]
+Results [3]: [cast((avg(UnscaledValue(ss_list_price#74))#77 / 100.0) as decimal(11,6)) AS B5_LP#86, count(ss_list_price#74)#78 AS B5_CNT#87, count(ss_list_price#74)#83 AS B5_CNTD#88]
+
+(57) BroadcastExchange
+Input [3]: [B5_LP#86, B5_CNT#87, B5_CNTD#88]
+Arguments: IdentityBroadcastMode, [id=#89]
+
+(58) BroadcastNestedLoopJoin [codegen id : 18]
+Join condition: None
+
+(59) Scan parquet default.store_sales
+Output [5]: [ss_quantity#90, ss_wholesale_cost#91, ss_list_price#92, ss_coupon_amt#93, ss_sold_date_sk#94]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,26), LessThanOrEqual(ss_quantity,30), Or(Or(And(GreaterThanOrEqual(ss_list_price,154.00),LessThanOrEqual(ss_list_price,164.00)),And(GreaterThanOrEqual(ss_coupon_amt,7326.00),LessThanOrEqual(ss_coupon_amt,8326.00))),And(GreaterThanOrEqual(ss_wholesale_cost,7.00),LessThanOrEqual(ss_wholesale_cost,27.00)))]
+ReadSchema: struct<ss_quantity:int,ss_wholesale_cost:decimal(7,2),ss_list_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(60) ColumnarToRow [codegen id : 15]
+Input [5]: [ss_quantity#90, ss_wholesale_cost#91, ss_list_price#92, ss_coupon_amt#93, ss_sold_date_sk#94]
+
+(61) Filter [codegen id : 15]
+Input [5]: [ss_quantity#90, ss_wholesale_cost#91, ss_list_price#92, ss_coupon_amt#93, ss_sold_date_sk#94]
+Condition : (((isnotnull(ss_quantity#90) AND (ss_quantity#90 >= 26)) AND (ss_quantity#90 <= 30)) AND ((((ss_list_price#92 >= 154.00) AND (ss_list_price#92 <= 164.00)) OR ((ss_coupon_amt#93 >= 7326.00) AND (ss_coupon_amt#93 <= 8326.00))) OR ((ss_wholesale_cost#91 >= 7.00) AND (ss_wholesale_cost#91 <= 27.00))))
+
+(62) Project [codegen id : 15]
+Output [1]: [ss_list_price#92]
+Input [5]: [ss_quantity#90, ss_wholesale_cost#91, ss_list_price#92, ss_coupon_amt#93, ss_sold_date_sk#94]
+
+(63) HashAggregate [codegen id : 15]
+Input [1]: [ss_list_price#92]
+Keys [1]: [ss_list_price#92]
+Functions [2]: [partial_avg(UnscaledValue(ss_list_price#92)), partial_count(ss_list_price#92)]
+Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#92))#95, count(ss_list_price#92)#96]
+Results [4]: [ss_list_price#92, sum#97, count#98, count#99]
+
+(64) Exchange
+Input [4]: [ss_list_price#92, sum#97, count#98, count#99]
+Arguments: hashpartitioning(ss_list_price#92, 5), ENSURE_REQUIREMENTS, [id=#100]
+
+(65) HashAggregate [codegen id : 16]
+Input [4]: [ss_list_price#92, sum#97, count#98, count#99]
+Keys [1]: [ss_list_price#92]
+Functions [2]: [merge_avg(UnscaledValue(ss_list_price#92)), merge_count(ss_list_price#92)]
+Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#92))#95, count(ss_list_price#92)#96]
+Results [4]: [ss_list_price#92, sum#97, count#98, count#99]
+
+(66) HashAggregate [codegen id : 16]
+Input [4]: [ss_list_price#92, sum#97, count#98, count#99]
+Keys: []
+Functions [3]: [merge_avg(UnscaledValue(ss_list_price#92)), merge_count(ss_list_price#92), partial_count(distinct ss_list_price#92)]
+Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#92))#95, count(ss_list_price#92)#96, count(ss_list_price#92)#101]
+Results [4]: [sum#97, count#98, count#99, count#102]
+
+(67) Exchange
+Input [4]: [sum#97, count#98, count#99, count#102]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#103]
+
+(68) HashAggregate [codegen id : 17]
+Input [4]: [sum#97, count#98, count#99, count#102]
+Keys: []
+Functions [3]: [avg(UnscaledValue(ss_list_price#92)), count(ss_list_price#92), count(distinct ss_list_price#92)]
+Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#92))#95, count(ss_list_price#92)#96, count(ss_list_price#92)#101]
+Results [3]: [cast((avg(UnscaledValue(ss_list_price#92))#95 / 100.0) as decimal(11,6)) AS B6_LP#104, count(ss_list_price#92)#96 AS B6_CNT#105, count(ss_list_price#92)#101 AS B6_CNTD#106]
+
+(69) BroadcastExchange
+Input [3]: [B6_LP#104, B6_CNT#105, B6_CNTD#106]
+Arguments: IdentityBroadcastMode, [id=#107]
+
+(70) BroadcastNestedLoopJoin [codegen id : 18]
+Join condition: None
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28/simplified.txt
new file mode 100644
index 0000000000000..25317a944a6f7
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28/simplified.txt
@@ -0,0 +1,111 @@
+WholeStageCodegen (18)
+  BroadcastNestedLoopJoin
+    BroadcastNestedLoopJoin
+      BroadcastNestedLoopJoin
+        BroadcastNestedLoopJoin
+          BroadcastNestedLoopJoin
+            HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B1_LP,B1_CNT,B1_CNTD,sum,count,count,count]
+              InputAdapter
+                Exchange #1
+                  WholeStageCodegen (2)
+                    HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count]
+                      HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
+                        InputAdapter
+                          Exchange [ss_list_price] #2
+                            WholeStageCodegen (1)
+                              HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
+                                Project [ss_list_price]
+                                  Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+            InputAdapter
+              BroadcastExchange #3
+                WholeStageCodegen (5)
+                  HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B2_LP,B2_CNT,B2_CNTD,sum,count,count,count]
+                    InputAdapter
+                      Exchange #4
+                        WholeStageCodegen (4)
+                          HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count]
+                            HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
+                              InputAdapter
+                                Exchange [ss_list_price] #5
+                                  WholeStageCodegen (3)
+                                    HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
+                                      Project [ss_list_price]
+                                        Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+          InputAdapter
+            BroadcastExchange #6
+              WholeStageCodegen (8)
+                HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B3_LP,B3_CNT,B3_CNTD,sum,count,count,count]
+                  InputAdapter
+                    Exchange #7
+                      WholeStageCodegen (7)
+                        HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count]
+                          HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
+                            InputAdapter
+                              Exchange [ss_list_price] #8
+                                WholeStageCodegen (6)
+                                  HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
+                                    Project [ss_list_price]
+                                      Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+        InputAdapter
+          BroadcastExchange #9
+            WholeStageCodegen (11)
+              HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B4_LP,B4_CNT,B4_CNTD,sum,count,count,count]
+                InputAdapter
+                  Exchange #10
+                    WholeStageCodegen (10)
+                      HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count]
+                        HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
+                          InputAdapter
+                            Exchange [ss_list_price] #11
+                              WholeStageCodegen (9)
+                                HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
+                                  Project [ss_list_price]
+                                    Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+      InputAdapter
+        BroadcastExchange #12
+          WholeStageCodegen (14)
+            HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B5_LP,B5_CNT,B5_CNTD,sum,count,count,count]
+              InputAdapter
+                Exchange #13
+                  WholeStageCodegen (13)
+                    HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count]
+                      HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
+                        InputAdapter
+                          Exchange [ss_list_price] #14
+                            WholeStageCodegen (12)
+                              HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
+                                Project [ss_list_price]
+                                  Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+    InputAdapter
+      BroadcastExchange #15
+        WholeStageCodegen (17)
+          HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B6_LP,B6_CNT,B6_CNTD,sum,count,count,count]
+            InputAdapter
+              Exchange #16
+                WholeStageCodegen (16)
+                  HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count]
+                    HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
+                      InputAdapter
+                        Exchange [ss_list_price] #17
+                          WholeStageCodegen (15)
+                            HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
+                              Project [ss_list_price]
+                                Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.sf100/explain.txt
new file mode 100644
index 0000000000000..d07d84fda4259
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.sf100/explain.txt
@@ -0,0 +1,364 @@
+== Physical Plan ==
+TakeOrderedAndProject (61)
++- * HashAggregate (60)
+   +- Exchange (59)
+      +- * HashAggregate (58)
+         +- * Project (57)
+            +- * SortMergeJoin Inner (56)
+               :- * Sort (43)
+               :  +- Exchange (42)
+               :     +- * Project (41)
+               :        +- * SortMergeJoin Inner (40)
+               :           :- * Sort (27)
+               :           :  +- Exchange (26)
+               :           :     +- * Project (25)
+               :           :        +- * SortMergeJoin Inner (24)
+               :           :           :- * Sort (18)
+               :           :           :  +- Exchange (17)
+               :           :           :     +- * Project (16)
+               :           :           :        +- * BroadcastHashJoin Inner BuildRight (15)
+               :           :           :           :- * Project (10)
+               :           :           :           :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :           :           :           :     :- * Filter (3)
+               :           :           :           :     :  +- * ColumnarToRow (2)
+               :           :           :           :     :     +- Scan parquet default.store_sales (1)
+               :           :           :           :     +- BroadcastExchange (8)
+               :           :           :           :        +- * Project (7)
+               :           :           :           :           +- * Filter (6)
+               :           :           :           :              +- * ColumnarToRow (5)
+               :           :           :           :                 +- Scan parquet default.date_dim (4)
+               :           :           :           +- BroadcastExchange (14)
+               :           :           :              +- * Filter (13)
+               :           :           :                 +- * ColumnarToRow (12)
+               :           :           :                    +- Scan parquet default.store (11)
+               :           :           +- * Sort (23)
+               :           :              +- Exchange (22)
+               :           :                 +- * Filter (21)
+               :           :                    +- * ColumnarToRow (20)
+               :           :                       +- Scan parquet default.item (19)
+               :           +- * Sort (39)
+               :              +- Exchange (38)
+               :                 +- * Project (37)
+               :                    +- * BroadcastHashJoin Inner BuildRight (36)
+               :                       :- * Filter (30)
+               :                       :  +- * ColumnarToRow (29)
+               :                       :     +- Scan parquet default.store_returns (28)
+               :                       +- BroadcastExchange (35)
+               :                          +- * Project (34)
+               :                             +- * Filter (33)
+               :                                +- * ColumnarToRow (32)
+               :                                   +- Scan parquet default.date_dim (31)
+               +- * Sort (55)
+                  +- Exchange (54)
+                     +- * Project (53)
+                        +- * BroadcastHashJoin Inner BuildRight (52)
+                           :- * Filter (46)
+                           :  +- * ColumnarToRow (45)
+                           :     +- Scan parquet default.catalog_sales (44)
+                           +- BroadcastExchange (51)
+                              +- * Project (50)
+                                 +- * Filter (49)
+                                    +- * ColumnarToRow (48)
+                                       +- Scan parquet default.date_dim (47)
+
+
+(1) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_quantity:int>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6]
+
+(3) Filter [codegen id : 3]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6]
+Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#8, d_year#9, d_moy#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,9), EqualTo(d_year,1999), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#8, d_year#9, d_moy#10]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#8, d_year#9, d_moy#10]
+Condition : ((((isnotnull(d_moy#10) AND isnotnull(d_year#9)) AND (d_moy#10 = 9)) AND (d_year#9 = 1999)) AND isnotnull(d_date_sk#8))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#8]
+Input [3]: [d_date_sk#8, d_year#9, d_moy#10]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#11]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#6]
+Right keys [1]: [d_date_sk#8]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5]
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, d_date_sk#8]
+
+(11) Scan parquet default.store
+Output [3]: [s_store_sk#12, s_store_id#13, s_store_name#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string,s_store_name:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [s_store_sk#12, s_store_id#13, s_store_name#14]
+
+(13) Filter [codegen id : 2]
+Input [3]: [s_store_sk#12, s_store_id#13, s_store_name#14]
+Condition : isnotnull(s_store_sk#12)
+
+(14) BroadcastExchange
+Input [3]: [s_store_sk#12, s_store_id#13, s_store_name#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#15]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#12]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#4, ss_quantity#5, s_store_id#13, s_store_name#14]
+Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, s_store_sk#12, s_store_id#13, s_store_name#14]
+
+(17) Exchange
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#4, ss_quantity#5, s_store_id#13, s_store_name#14]
+Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#16]
+
+(18) Sort [codegen id : 4]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#4, ss_quantity#5, s_store_id#13, s_store_name#14]
+Arguments: [ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(19) Scan parquet default.item
+Output [3]: [i_item_sk#17, i_item_id#18, i_item_desc#19]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_item_desc:string>
+
+(20) ColumnarToRow [codegen id : 5]
+Input [3]: [i_item_sk#17, i_item_id#18, i_item_desc#19]
+
+(21) Filter [codegen id : 5]
+Input [3]: [i_item_sk#17, i_item_id#18, i_item_desc#19]
+Condition : isnotnull(i_item_sk#17)
+
+(22) Exchange
+Input [3]: [i_item_sk#17, i_item_id#18, i_item_desc#19]
+Arguments: hashpartitioning(i_item_sk#17, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(23) Sort [codegen id : 6]
+Input [3]: [i_item_sk#17, i_item_id#18, i_item_desc#19]
+Arguments: [i_item_sk#17 ASC NULLS FIRST], false, 0
+
+(24) SortMergeJoin [codegen id : 7]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#17]
+Join condition: None
+
+(25) Project [codegen id : 7]
+Output [8]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#4, ss_quantity#5, s_store_id#13, s_store_name#14, i_item_id#18, i_item_desc#19]
+Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#4, ss_quantity#5, s_store_id#13, s_store_name#14, i_item_sk#17, i_item_id#18, i_item_desc#19]
+
+(26) Exchange
+Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#4, ss_quantity#5, s_store_id#13, s_store_name#14, i_item_id#18, i_item_desc#19]
+Arguments: hashpartitioning(ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(27) Sort [codegen id : 8]
+Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#4, ss_quantity#5, s_store_id#13, s_store_name#14, i_item_id#18, i_item_desc#19]
+Arguments: [ss_customer_sk#2 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST], false, 0
+
+(28) Scan parquet default.store_returns
+Output [5]: [sr_item_sk#22, sr_customer_sk#23, sr_ticket_number#24, sr_return_quantity#25, sr_returned_date_sk#26]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(sr_returned_date_sk#26), dynamicpruningexpression(sr_returned_date_sk#26 IN dynamicpruning#27)]
+PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)]
+ReadSchema: struct<sr_item_sk:int,sr_customer_sk:int,sr_ticket_number:int,sr_return_quantity:int>
+
+(29) ColumnarToRow [codegen id : 10]
+Input [5]: [sr_item_sk#22, sr_customer_sk#23, sr_ticket_number#24, sr_return_quantity#25, sr_returned_date_sk#26]
+
+(30) Filter [codegen id : 10]
+Input [5]: [sr_item_sk#22, sr_customer_sk#23, sr_ticket_number#24, sr_return_quantity#25, sr_returned_date_sk#26]
+Condition : ((isnotnull(sr_customer_sk#23) AND isnotnull(sr_item_sk#22)) AND isnotnull(sr_ticket_number#24))
+
+(31) Scan parquet default.date_dim
+Output [3]: [d_date_sk#28, d_year#29, d_moy#30]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,9), LessThanOrEqual(d_moy,12), EqualTo(d_year,1999), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(32) ColumnarToRow [codegen id : 9]
+Input [3]: [d_date_sk#28, d_year#29, d_moy#30]
+
+(33) Filter [codegen id : 9]
+Input [3]: [d_date_sk#28, d_year#29, d_moy#30]
+Condition : (((((isnotnull(d_moy#30) AND isnotnull(d_year#29)) AND (d_moy#30 >= 9)) AND (d_moy#30 <= 12)) AND (d_year#29 = 1999)) AND isnotnull(d_date_sk#28))
+
+(34) Project [codegen id : 9]
+Output [1]: [d_date_sk#28]
+Input [3]: [d_date_sk#28, d_year#29, d_moy#30]
+
+(35) BroadcastExchange
+Input [1]: [d_date_sk#28]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#31]
+
+(36) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [sr_returned_date_sk#26]
+Right keys [1]: [d_date_sk#28]
+Join condition: None
+
+(37) Project [codegen id : 10]
+Output [4]: [sr_item_sk#22, sr_customer_sk#23, sr_ticket_number#24, sr_return_quantity#25]
+Input [6]: [sr_item_sk#22, sr_customer_sk#23, sr_ticket_number#24, sr_return_quantity#25, sr_returned_date_sk#26, d_date_sk#28]
+
+(38) Exchange
+Input [4]: [sr_item_sk#22, sr_customer_sk#23, sr_ticket_number#24, sr_return_quantity#25]
+Arguments: hashpartitioning(sr_customer_sk#23, sr_item_sk#22, sr_ticket_number#24, 5), ENSURE_REQUIREMENTS, [id=#32]
+
+(39) Sort [codegen id : 11]
+Input [4]: [sr_item_sk#22, sr_customer_sk#23, sr_ticket_number#24, sr_return_quantity#25]
+Arguments: [sr_customer_sk#23 ASC NULLS FIRST, sr_item_sk#22 ASC NULLS FIRST, sr_ticket_number#24 ASC NULLS FIRST], false, 0
+
+(40) SortMergeJoin [codegen id : 12]
+Left keys [3]: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4]
+Right keys [3]: [sr_customer_sk#23, sr_item_sk#22, sr_ticket_number#24]
+Join condition: None
+
+(41) Project [codegen id : 12]
+Output [8]: [ss_quantity#5, s_store_id#13, s_store_name#14, i_item_id#18, i_item_desc#19, sr_item_sk#22, sr_customer_sk#23, sr_return_quantity#25]
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#4, ss_quantity#5, s_store_id#13, s_store_name#14, i_item_id#18, i_item_desc#19, sr_item_sk#22, sr_customer_sk#23, sr_ticket_number#24, sr_return_quantity#25]
+
+(42) Exchange
+Input [8]: [ss_quantity#5, s_store_id#13, s_store_name#14, i_item_id#18, i_item_desc#19, sr_item_sk#22, sr_customer_sk#23, sr_return_quantity#25]
+Arguments: hashpartitioning(sr_customer_sk#23, sr_item_sk#22, 5), ENSURE_REQUIREMENTS, [id=#33]
+
+(43) Sort [codegen id : 13]
+Input [8]: [ss_quantity#5, s_store_id#13, s_store_name#14, i_item_id#18, i_item_desc#19, sr_item_sk#22, sr_customer_sk#23, sr_return_quantity#25]
+Arguments: [sr_customer_sk#23 ASC NULLS FIRST, sr_item_sk#22 ASC NULLS FIRST], false, 0
+
+(44) Scan parquet default.catalog_sales
+Output [4]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#36, cs_sold_date_sk#37]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#37), dynamicpruningexpression(cs_sold_date_sk#37 IN dynamicpruning#38)]
+PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_item_sk:int,cs_quantity:int>
+
+(45) ColumnarToRow [codegen id : 15]
+Input [4]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#36, cs_sold_date_sk#37]
+
+(46) Filter [codegen id : 15]
+Input [4]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#36, cs_sold_date_sk#37]
+Condition : (isnotnull(cs_bill_customer_sk#34) AND isnotnull(cs_item_sk#35))
+
+(47) Scan parquet default.date_dim
+Output [2]: [d_date_sk#39, d_year#40]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(48) ColumnarToRow [codegen id : 14]
+Input [2]: [d_date_sk#39, d_year#40]
+
+(49) Filter [codegen id : 14]
+Input [2]: [d_date_sk#39, d_year#40]
+Condition : (d_year#40 IN (1999,2000,2001) AND isnotnull(d_date_sk#39))
+
+(50) Project [codegen id : 14]
+Output [1]: [d_date_sk#39]
+Input [2]: [d_date_sk#39, d_year#40]
+
+(51) BroadcastExchange
+Input [1]: [d_date_sk#39]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#41]
+
+(52) BroadcastHashJoin [codegen id : 15]
+Left keys [1]: [cs_sold_date_sk#37]
+Right keys [1]: [d_date_sk#39]
+Join condition: None
+
+(53) Project [codegen id : 15]
+Output [3]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#36]
+Input [5]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#36, cs_sold_date_sk#37, d_date_sk#39]
+
+(54) Exchange
+Input [3]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#36]
+Arguments: hashpartitioning(cs_bill_customer_sk#34, cs_item_sk#35, 5), ENSURE_REQUIREMENTS, [id=#42]
+
+(55) Sort [codegen id : 16]
+Input [3]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#36]
+Arguments: [cs_bill_customer_sk#34 ASC NULLS FIRST, cs_item_sk#35 ASC NULLS FIRST], false, 0
+
+(56) SortMergeJoin [codegen id : 17]
+Left keys [2]: [sr_customer_sk#23, sr_item_sk#22]
+Right keys [2]: [cs_bill_customer_sk#34, cs_item_sk#35]
+Join condition: None
+
+(57) Project [codegen id : 17]
+Output [7]: [ss_quantity#5, sr_return_quantity#25, cs_quantity#36, s_store_id#13, s_store_name#14, i_item_id#18, i_item_desc#19]
+Input [11]: [ss_quantity#5, s_store_id#13, s_store_name#14, i_item_id#18, i_item_desc#19, sr_item_sk#22, sr_customer_sk#23, sr_return_quantity#25, cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#36]
+
+(58) HashAggregate [codegen id : 17]
+Input [7]: [ss_quantity#5, sr_return_quantity#25, cs_quantity#36, s_store_id#13, s_store_name#14, i_item_id#18, i_item_desc#19]
+Keys [4]: [i_item_id#18, i_item_desc#19, s_store_id#13, s_store_name#14]
+Functions [3]: [partial_sum(ss_quantity#5), partial_sum(sr_return_quantity#25), partial_sum(cs_quantity#36)]
+Aggregate Attributes [3]: [sum#43, sum#44, sum#45]
+Results [7]: [i_item_id#18, i_item_desc#19, s_store_id#13, s_store_name#14, sum#46, sum#47, sum#48]
+
+(59) Exchange
+Input [7]: [i_item_id#18, i_item_desc#19, s_store_id#13, s_store_name#14, sum#46, sum#47, sum#48]
+Arguments: hashpartitioning(i_item_id#18, i_item_desc#19, s_store_id#13, s_store_name#14, 5), ENSURE_REQUIREMENTS, [id=#49]
+
+(60) HashAggregate [codegen id : 18]
+Input [7]: [i_item_id#18, i_item_desc#19, s_store_id#13, s_store_name#14, sum#46, sum#47, sum#48]
+Keys [4]: [i_item_id#18, i_item_desc#19, s_store_id#13, s_store_name#14]
+Functions [3]: [sum(ss_quantity#5), sum(sr_return_quantity#25), sum(cs_quantity#36)]
+Aggregate Attributes [3]: [sum(ss_quantity#5)#50, sum(sr_return_quantity#25)#51, sum(cs_quantity#36)#52]
+Results [7]: [i_item_id#18, i_item_desc#19, s_store_id#13, s_store_name#14, sum(ss_quantity#5)#50 AS store_sales_quantity#53, sum(sr_return_quantity#25)#51 AS store_returns_quantity#54, sum(cs_quantity#36)#52 AS catalog_sales_quantity#55]
+
+(61) TakeOrderedAndProject
+Input [7]: [i_item_id#18, i_item_desc#19, s_store_id#13, s_store_name#14, store_sales_quantity#53, store_returns_quantity#54, catalog_sales_quantity#55]
+Arguments: 100, [i_item_id#18 ASC NULLS FIRST, i_item_desc#19 ASC NULLS FIRST, s_store_id#13 ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST], [i_item_id#18, i_item_desc#19, s_store_id#13, s_store_name#14, store_sales_quantity#53, store_returns_quantity#54, catalog_sales_quantity#55]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7
+ReusedExchange (62)
+
+
+(62) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#8]
+
+Subquery:2 Hosting operator id = 28 Hosting Expression = sr_returned_date_sk#26 IN dynamicpruning#27
+ReusedExchange (63)
+
+
+(63) ReusedExchange [Reuses operator id: 35]
+Output [1]: [d_date_sk#28]
+
+Subquery:3 Hosting operator id = 44 Hosting Expression = cs_sold_date_sk#37 IN dynamicpruning#38
+ReusedExchange (64)
+
+
+(64) ReusedExchange [Reuses operator id: 51]
+Output [1]: [d_date_sk#39]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.sf100/simplified.txt
new file mode 100644
index 0000000000000..d4c30cb4a2283
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.sf100/simplified.txt
@@ -0,0 +1,110 @@
+TakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity]
+  WholeStageCodegen (18)
+    HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] [sum(ss_quantity),sum(sr_return_quantity),sum(cs_quantity),store_sales_quantity,store_returns_quantity,catalog_sales_quantity,sum,sum,sum]
+      InputAdapter
+        Exchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1
+          WholeStageCodegen (17)
+            HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,ss_quantity,sr_return_quantity,cs_quantity] [sum,sum,sum,sum,sum,sum]
+              Project [ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_id,i_item_desc]
+                SortMergeJoin [sr_customer_sk,sr_item_sk,cs_bill_customer_sk,cs_item_sk]
+                  InputAdapter
+                    WholeStageCodegen (13)
+                      Sort [sr_customer_sk,sr_item_sk]
+                        InputAdapter
+                          Exchange [sr_customer_sk,sr_item_sk] #2
+                            WholeStageCodegen (12)
+                              Project [ss_quantity,s_store_id,s_store_name,i_item_id,i_item_desc,sr_item_sk,sr_customer_sk,sr_return_quantity]
+                                SortMergeJoin [ss_customer_sk,ss_item_sk,ss_ticket_number,sr_customer_sk,sr_item_sk,sr_ticket_number]
+                                  InputAdapter
+                                    WholeStageCodegen (8)
+                                      Sort [ss_customer_sk,ss_item_sk,ss_ticket_number]
+                                        InputAdapter
+                                          Exchange [ss_customer_sk,ss_item_sk,ss_ticket_number] #3
+                                            WholeStageCodegen (7)
+                                              Project [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,s_store_id,s_store_name,i_item_id,i_item_desc]
+                                                SortMergeJoin [ss_item_sk,i_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (4)
+                                                      Sort [ss_item_sk]
+                                                        InputAdapter
+                                                          Exchange [ss_item_sk] #4
+                                                            WholeStageCodegen (3)
+                                                              Project [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,s_store_id,s_store_name]
+                                                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                                  Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity]
+                                                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                      Filter [ss_customer_sk,ss_item_sk,ss_ticket_number,ss_store_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk]
+                                                                              SubqueryBroadcast [d_date_sk] #1
+                                                                                ReusedExchange [d_date_sk] #5
+                                                                      InputAdapter
+                                                                        BroadcastExchange #5
+                                                                          WholeStageCodegen (1)
+                                                                            Project [d_date_sk]
+                                                                              Filter [d_moy,d_year,d_date_sk]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                                                  InputAdapter
+                                                                    BroadcastExchange #6
+                                                                      WholeStageCodegen (2)
+                                                                        Filter [s_store_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.store [s_store_sk,s_store_id,s_store_name]
+                                                  InputAdapter
+                                                    WholeStageCodegen (6)
+                                                      Sort [i_item_sk]
+                                                        InputAdapter
+                                                          Exchange [i_item_sk] #7
+                                                            WholeStageCodegen (5)
+                                                              Filter [i_item_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.item [i_item_sk,i_item_id,i_item_desc]
+                                  InputAdapter
+                                    WholeStageCodegen (11)
+                                      Sort [sr_customer_sk,sr_item_sk,sr_ticket_number]
+                                        InputAdapter
+                                          Exchange [sr_customer_sk,sr_item_sk,sr_ticket_number] #8
+                                            WholeStageCodegen (10)
+                                              Project [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity]
+                                                BroadcastHashJoin [sr_returned_date_sk,d_date_sk]
+                                                  Filter [sr_customer_sk,sr_item_sk,sr_ticket_number]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk]
+                                                          SubqueryBroadcast [d_date_sk] #2
+                                                            ReusedExchange [d_date_sk] #9
+                                                  InputAdapter
+                                                    BroadcastExchange #9
+                                                      WholeStageCodegen (9)
+                                                        Project [d_date_sk]
+                                                          Filter [d_moy,d_year,d_date_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                  InputAdapter
+                    WholeStageCodegen (16)
+                      Sort [cs_bill_customer_sk,cs_item_sk]
+                        InputAdapter
+                          Exchange [cs_bill_customer_sk,cs_item_sk] #10
+                            WholeStageCodegen (15)
+                              Project [cs_bill_customer_sk,cs_item_sk,cs_quantity]
+                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                  Filter [cs_bill_customer_sk,cs_item_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk]
+                                          SubqueryBroadcast [d_date_sk] #3
+                                            ReusedExchange [d_date_sk] #11
+                                  InputAdapter
+                                    BroadcastExchange #11
+                                      WholeStageCodegen (14)
+                                        Project [d_date_sk]
+                                          Filter [d_year,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_year]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29/explain.txt
new file mode 100644
index 0000000000000..45a02422e92b1
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29/explain.txt
@@ -0,0 +1,319 @@
+== Physical Plan ==
+TakeOrderedAndProject (52)
++- * HashAggregate (51)
+   +- Exchange (50)
+      +- * HashAggregate (49)
+         +- * Project (48)
+            +- * BroadcastHashJoin Inner BuildRight (47)
+               :- * Project (42)
+               :  +- * BroadcastHashJoin Inner BuildRight (41)
+               :     :- * Project (36)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (35)
+               :     :     :- * Project (29)
+               :     :     :  +- * BroadcastHashJoin Inner BuildRight (28)
+               :     :     :     :- * Project (22)
+               :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (21)
+               :     :     :     :     :- * Project (15)
+               :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (14)
+               :     :     :     :     :     :- * Project (9)
+               :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+               :     :     :     :     :     :     :- * Filter (3)
+               :     :     :     :     :     :     :  +- * ColumnarToRow (2)
+               :     :     :     :     :     :     :     +- Scan parquet default.store_sales (1)
+               :     :     :     :     :     :     +- BroadcastExchange (7)
+               :     :     :     :     :     :        +- * Filter (6)
+               :     :     :     :     :     :           +- * ColumnarToRow (5)
+               :     :     :     :     :     :              +- Scan parquet default.store_returns (4)
+               :     :     :     :     :     +- BroadcastExchange (13)
+               :     :     :     :     :        +- * Filter (12)
+               :     :     :     :     :           +- * ColumnarToRow (11)
+               :     :     :     :     :              +- Scan parquet default.catalog_sales (10)
+               :     :     :     :     +- BroadcastExchange (20)
+               :     :     :     :        +- * Project (19)
+               :     :     :     :           +- * Filter (18)
+               :     :     :     :              +- * ColumnarToRow (17)
+               :     :     :     :                 +- Scan parquet default.date_dim (16)
+               :     :     :     +- BroadcastExchange (27)
+               :     :     :        +- * Project (26)
+               :     :     :           +- * Filter (25)
+               :     :     :              +- * ColumnarToRow (24)
+               :     :     :                 +- Scan parquet default.date_dim (23)
+               :     :     +- BroadcastExchange (34)
+               :     :        +- * Project (33)
+               :     :           +- * Filter (32)
+               :     :              +- * ColumnarToRow (31)
+               :     :                 +- Scan parquet default.date_dim (30)
+               :     +- BroadcastExchange (40)
+               :        +- * Filter (39)
+               :           +- * ColumnarToRow (38)
+               :              +- Scan parquet default.store (37)
+               +- BroadcastExchange (46)
+                  +- * Filter (45)
+                     +- * ColumnarToRow (44)
+                        +- Scan parquet default.item (43)
+
+
+(1) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_quantity:int>
+
+(2) ColumnarToRow [codegen id : 8]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6]
+
+(3) Filter [codegen id : 8]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6]
+Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3))
+
+(4) Scan parquet default.store_returns
+Output [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(sr_returned_date_sk#12), dynamicpruningexpression(sr_returned_date_sk#12 IN dynamicpruning#13)]
+PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)]
+ReadSchema: struct<sr_item_sk:int,sr_customer_sk:int,sr_ticket_number:int,sr_return_quantity:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12]
+
+(6) Filter [codegen id : 1]
+Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12]
+Condition : ((isnotnull(sr_customer_sk#9) AND isnotnull(sr_item_sk#8)) AND isnotnull(sr_ticket_number#10))
+
+(7) BroadcastExchange
+Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12]
+Arguments: HashedRelationBroadcastMode(List(input[1, int, false], input[0, int, false], input[2, int, false]),false), [id=#14]
+
+(8) BroadcastHashJoin [codegen id : 8]
+Left keys [3]: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4]
+Right keys [3]: [sr_customer_sk#9, sr_item_sk#8, sr_ticket_number#10]
+Join condition: None
+
+(9) Project [codegen id : 8]
+Output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12]
+Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12]
+
+(10) Scan parquet default.catalog_sales
+Output [4]: [cs_bill_customer_sk#15, cs_item_sk#16, cs_quantity#17, cs_sold_date_sk#18]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_sold_date_sk#18 IN dynamicpruning#19)]
+PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_item_sk:int,cs_quantity:int>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [4]: [cs_bill_customer_sk#15, cs_item_sk#16, cs_quantity#17, cs_sold_date_sk#18]
+
+(12) Filter [codegen id : 2]
+Input [4]: [cs_bill_customer_sk#15, cs_item_sk#16, cs_quantity#17, cs_sold_date_sk#18]
+Condition : (isnotnull(cs_bill_customer_sk#15) AND isnotnull(cs_item_sk#16))
+
+(13) BroadcastExchange
+Input [4]: [cs_bill_customer_sk#15, cs_item_sk#16, cs_quantity#17, cs_sold_date_sk#18]
+Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[1, int, false] as bigint) & 4294967295))),false), [id=#20]
+
+(14) BroadcastHashJoin [codegen id : 8]
+Left keys [2]: [sr_customer_sk#9, sr_item_sk#8]
+Right keys [2]: [cs_bill_customer_sk#15, cs_item_sk#16]
+Join condition: None
+
+(15) Project [codegen id : 8]
+Output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#17, cs_sold_date_sk#18]
+Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12, cs_bill_customer_sk#15, cs_item_sk#16, cs_quantity#17, cs_sold_date_sk#18]
+
+(16) Scan parquet default.date_dim
+Output [3]: [d_date_sk#21, d_year#22, d_moy#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,9), EqualTo(d_year,1999), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(17) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#21, d_year#22, d_moy#23]
+
+(18) Filter [codegen id : 3]
+Input [3]: [d_date_sk#21, d_year#22, d_moy#23]
+Condition : ((((isnotnull(d_moy#23) AND isnotnull(d_year#22)) AND (d_moy#23 = 9)) AND (d_year#22 = 1999)) AND isnotnull(d_date_sk#21))
+
+(19) Project [codegen id : 3]
+Output [1]: [d_date_sk#21]
+Input [3]: [d_date_sk#21, d_year#22, d_moy#23]
+
+(20) BroadcastExchange
+Input [1]: [d_date_sk#21]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#24]
+
+(21) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_sold_date_sk#6]
+Right keys [1]: [d_date_sk#21]
+Join condition: None
+
+(22) Project [codegen id : 8]
+Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#17, cs_sold_date_sk#18]
+Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#17, cs_sold_date_sk#18, d_date_sk#21]
+
+(23) Scan parquet default.date_dim
+Output [3]: [d_date_sk#25, d_year#26, d_moy#27]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,9), LessThanOrEqual(d_moy,12), EqualTo(d_year,1999), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(24) ColumnarToRow [codegen id : 4]
+Input [3]: [d_date_sk#25, d_year#26, d_moy#27]
+
+(25) Filter [codegen id : 4]
+Input [3]: [d_date_sk#25, d_year#26, d_moy#27]
+Condition : (((((isnotnull(d_moy#27) AND isnotnull(d_year#26)) AND (d_moy#27 >= 9)) AND (d_moy#27 <= 12)) AND (d_year#26 = 1999)) AND isnotnull(d_date_sk#25))
+
+(26) Project [codegen id : 4]
+Output [1]: [d_date_sk#25]
+Input [3]: [d_date_sk#25, d_year#26, d_moy#27]
+
+(27) BroadcastExchange
+Input [1]: [d_date_sk#25]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#28]
+
+(28) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [sr_returned_date_sk#12]
+Right keys [1]: [d_date_sk#25]
+Join condition: None
+
+(29) Project [codegen id : 8]
+Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#17, cs_sold_date_sk#18]
+Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#17, cs_sold_date_sk#18, d_date_sk#25]
+
+(30) Scan parquet default.date_dim
+Output [2]: [d_date_sk#29, d_year#30]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(31) ColumnarToRow [codegen id : 5]
+Input [2]: [d_date_sk#29, d_year#30]
+
+(32) Filter [codegen id : 5]
+Input [2]: [d_date_sk#29, d_year#30]
+Condition : (d_year#30 IN (1999,2000,2001) AND isnotnull(d_date_sk#29))
+
+(33) Project [codegen id : 5]
+Output [1]: [d_date_sk#29]
+Input [2]: [d_date_sk#29, d_year#30]
+
+(34) BroadcastExchange
+Input [1]: [d_date_sk#29]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#31]
+
+(35) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [cs_sold_date_sk#18]
+Right keys [1]: [d_date_sk#29]
+Join condition: None
+
+(36) Project [codegen id : 8]
+Output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#17]
+Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#17, cs_sold_date_sk#18, d_date_sk#29]
+
+(37) Scan parquet default.store
+Output [3]: [s_store_sk#32, s_store_id#33, s_store_name#34]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string,s_store_name:string>
+
+(38) ColumnarToRow [codegen id : 6]
+Input [3]: [s_store_sk#32, s_store_id#33, s_store_name#34]
+
+(39) Filter [codegen id : 6]
+Input [3]: [s_store_sk#32, s_store_id#33, s_store_name#34]
+Condition : isnotnull(s_store_sk#32)
+
+(40) BroadcastExchange
+Input [3]: [s_store_sk#32, s_store_id#33, s_store_name#34]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#35]
+
+(41) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#32]
+Join condition: None
+
+(42) Project [codegen id : 8]
+Output [6]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#17, s_store_id#33, s_store_name#34]
+Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#17, s_store_sk#32, s_store_id#33, s_store_name#34]
+
+(43) Scan parquet default.item
+Output [3]: [i_item_sk#36, i_item_id#37, i_item_desc#38]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_item_desc:string>
+
+(44) ColumnarToRow [codegen id : 7]
+Input [3]: [i_item_sk#36, i_item_id#37, i_item_desc#38]
+
+(45) Filter [codegen id : 7]
+Input [3]: [i_item_sk#36, i_item_id#37, i_item_desc#38]
+Condition : isnotnull(i_item_sk#36)
+
+(46) BroadcastExchange
+Input [3]: [i_item_sk#36, i_item_id#37, i_item_desc#38]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#39]
+
+(47) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#36]
+Join condition: None
+
+(48) Project [codegen id : 8]
+Output [7]: [ss_quantity#5, sr_return_quantity#11, cs_quantity#17, s_store_id#33, s_store_name#34, i_item_id#37, i_item_desc#38]
+Input [9]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#17, s_store_id#33, s_store_name#34, i_item_sk#36, i_item_id#37, i_item_desc#38]
+
+(49) HashAggregate [codegen id : 8]
+Input [7]: [ss_quantity#5, sr_return_quantity#11, cs_quantity#17, s_store_id#33, s_store_name#34, i_item_id#37, i_item_desc#38]
+Keys [4]: [i_item_id#37, i_item_desc#38, s_store_id#33, s_store_name#34]
+Functions [3]: [partial_sum(ss_quantity#5), partial_sum(sr_return_quantity#11), partial_sum(cs_quantity#17)]
+Aggregate Attributes [3]: [sum#40, sum#41, sum#42]
+Results [7]: [i_item_id#37, i_item_desc#38, s_store_id#33, s_store_name#34, sum#43, sum#44, sum#45]
+
+(50) Exchange
+Input [7]: [i_item_id#37, i_item_desc#38, s_store_id#33, s_store_name#34, sum#43, sum#44, sum#45]
+Arguments: hashpartitioning(i_item_id#37, i_item_desc#38, s_store_id#33, s_store_name#34, 5), ENSURE_REQUIREMENTS, [id=#46]
+
+(51) HashAggregate [codegen id : 9]
+Input [7]: [i_item_id#37, i_item_desc#38, s_store_id#33, s_store_name#34, sum#43, sum#44, sum#45]
+Keys [4]: [i_item_id#37, i_item_desc#38, s_store_id#33, s_store_name#34]
+Functions [3]: [sum(ss_quantity#5), sum(sr_return_quantity#11), sum(cs_quantity#17)]
+Aggregate Attributes [3]: [sum(ss_quantity#5)#47, sum(sr_return_quantity#11)#48, sum(cs_quantity#17)#49]
+Results [7]: [i_item_id#37, i_item_desc#38, s_store_id#33, s_store_name#34, sum(ss_quantity#5)#47 AS store_sales_quantity#50, sum(sr_return_quantity#11)#48 AS store_returns_quantity#51, sum(cs_quantity#17)#49 AS catalog_sales_quantity#52]
+
+(52) TakeOrderedAndProject
+Input [7]: [i_item_id#37, i_item_desc#38, s_store_id#33, s_store_name#34, store_sales_quantity#50, store_returns_quantity#51, catalog_sales_quantity#52]
+Arguments: 100, [i_item_id#37 ASC NULLS FIRST, i_item_desc#38 ASC NULLS FIRST, s_store_id#33 ASC NULLS FIRST, s_store_name#34 ASC NULLS FIRST], [i_item_id#37, i_item_desc#38, s_store_id#33, s_store_name#34, store_sales_quantity#50, store_returns_quantity#51, catalog_sales_quantity#52]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7
+ReusedExchange (53)
+
+
+(53) ReusedExchange [Reuses operator id: 20]
+Output [1]: [d_date_sk#21]
+
+Subquery:2 Hosting operator id = 4 Hosting Expression = sr_returned_date_sk#12 IN dynamicpruning#13
+ReusedExchange (54)
+
+
+(54) ReusedExchange [Reuses operator id: 27]
+Output [1]: [d_date_sk#25]
+
+Subquery:3 Hosting operator id = 10 Hosting Expression = cs_sold_date_sk#18 IN dynamicpruning#19
+ReusedExchange (55)
+
+
+(55) ReusedExchange [Reuses operator id: 34]
+Output [1]: [d_date_sk#29]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29/simplified.txt
new file mode 100644
index 0000000000000..a5ad345ea77c8
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29/simplified.txt
@@ -0,0 +1,83 @@
+TakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity]
+  WholeStageCodegen (9)
+    HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] [sum(ss_quantity),sum(sr_return_quantity),sum(cs_quantity),store_sales_quantity,store_returns_quantity,catalog_sales_quantity,sum,sum,sum]
+      InputAdapter
+        Exchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1
+          WholeStageCodegen (8)
+            HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,ss_quantity,sr_return_quantity,cs_quantity] [sum,sum,sum,sum,sum,sum]
+              Project [ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_id,i_item_desc]
+                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                  Project [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name]
+                    BroadcastHashJoin [ss_store_sk,s_store_sk]
+                      Project [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity]
+                        BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                          Project [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk]
+                            BroadcastHashJoin [sr_returned_date_sk,d_date_sk]
+                              Project [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk]
+                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                  Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk]
+                                    BroadcastHashJoin [sr_customer_sk,sr_item_sk,cs_bill_customer_sk,cs_item_sk]
+                                      Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk]
+                                        BroadcastHashJoin [ss_customer_sk,ss_item_sk,ss_ticket_number,sr_customer_sk,sr_item_sk,sr_ticket_number]
+                                          Filter [ss_customer_sk,ss_item_sk,ss_ticket_number,ss_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk]
+                                                  SubqueryBroadcast [d_date_sk] #1
+                                                    ReusedExchange [d_date_sk] #2
+                                          InputAdapter
+                                            BroadcastExchange #3
+                                              WholeStageCodegen (1)
+                                                Filter [sr_customer_sk,sr_item_sk,sr_ticket_number]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk]
+                                                        SubqueryBroadcast [d_date_sk] #2
+                                                          ReusedExchange [d_date_sk] #4
+                                      InputAdapter
+                                        BroadcastExchange #5
+                                          WholeStageCodegen (2)
+                                            Filter [cs_bill_customer_sk,cs_item_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #3
+                                                      ReusedExchange [d_date_sk] #6
+                                  InputAdapter
+                                    BroadcastExchange #2
+                                      WholeStageCodegen (3)
+                                        Project [d_date_sk]
+                                          Filter [d_moy,d_year,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                              InputAdapter
+                                BroadcastExchange #4
+                                  WholeStageCodegen (4)
+                                    Project [d_date_sk]
+                                      Filter [d_moy,d_year,d_date_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                          InputAdapter
+                            BroadcastExchange #6
+                              WholeStageCodegen (5)
+                                Project [d_date_sk]
+                                  Filter [d_year,d_date_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.date_dim [d_date_sk,d_year]
+                      InputAdapter
+                        BroadcastExchange #7
+                          WholeStageCodegen (6)
+                            Filter [s_store_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.store [s_store_sk,s_store_id,s_store_name]
+                  InputAdapter
+                    BroadcastExchange #8
+                      WholeStageCodegen (7)
+                        Filter [i_item_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.item [i_item_sk,i_item_id,i_item_desc]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.sf100/explain.txt
new file mode 100644
index 0000000000000..d94a1f3b14b4e
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.sf100/explain.txt
@@ -0,0 +1,133 @@
+== Physical Plan ==
+TakeOrderedAndProject (21)
++- * HashAggregate (20)
+   +- Exchange (19)
+      +- * HashAggregate (18)
+         +- * Project (17)
+            +- * BroadcastHashJoin Inner BuildRight (16)
+               :- * Project (10)
+               :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :- * Filter (3)
+               :     :  +- * ColumnarToRow (2)
+               :     :     +- Scan parquet default.store_sales (1)
+               :     +- BroadcastExchange (8)
+               :        +- * Project (7)
+               :           +- * Filter (6)
+               :              +- * ColumnarToRow (5)
+               :                 +- Scan parquet default.item (4)
+               +- BroadcastExchange (15)
+                  +- * Project (14)
+                     +- * Filter (13)
+                        +- * ColumnarToRow (12)
+                           +- Scan parquet default.date_dim (11)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_item_sk#1)
+
+(4) Scan parquet default.item
+Output [4]: [i_item_sk#5, i_brand_id#6, i_brand#7, i_manufact_id#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,128), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_brand:string,i_manufact_id:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [i_item_sk#5, i_brand_id#6, i_brand#7, i_manufact_id#8]
+
+(6) Filter [codegen id : 1]
+Input [4]: [i_item_sk#5, i_brand_id#6, i_brand#7, i_manufact_id#8]
+Condition : ((isnotnull(i_manufact_id#8) AND (i_manufact_id#8 = 128)) AND isnotnull(i_item_sk#5))
+
+(7) Project [codegen id : 1]
+Output [3]: [i_item_sk#5, i_brand_id#6, i_brand#7]
+Input [4]: [i_item_sk#5, i_brand_id#6, i_brand#7, i_manufact_id#8]
+
+(8) BroadcastExchange
+Input [3]: [i_item_sk#5, i_brand_id#6, i_brand#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#9]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#5]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [4]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_brand_id#6, i_brand#7]
+Input [6]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_brand_id#6, i_brand#7]
+
+(11) Scan parquet default.date_dim
+Output [3]: [d_date_sk#10, d_year#11, d_moy#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), EqualTo(d_moy,11), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+
+(13) Filter [codegen id : 2]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+Condition : ((isnotnull(d_moy#12) AND (d_moy#12 = 11)) AND isnotnull(d_date_sk#10))
+
+(14) Project [codegen id : 2]
+Output [2]: [d_date_sk#10, d_year#11]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+
+(15) BroadcastExchange
+Input [2]: [d_date_sk#10, d_year#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(16) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(17) Project [codegen id : 3]
+Output [4]: [d_year#11, ss_ext_sales_price#2, i_brand_id#6, i_brand#7]
+Input [6]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_brand_id#6, i_brand#7, d_date_sk#10, d_year#11]
+
+(18) HashAggregate [codegen id : 3]
+Input [4]: [d_year#11, ss_ext_sales_price#2, i_brand_id#6, i_brand#7]
+Keys [3]: [d_year#11, i_brand#7, i_brand_id#6]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum#14]
+Results [4]: [d_year#11, i_brand#7, i_brand_id#6, sum#15]
+
+(19) Exchange
+Input [4]: [d_year#11, i_brand#7, i_brand_id#6, sum#15]
+Arguments: hashpartitioning(d_year#11, i_brand#7, i_brand_id#6, 5), ENSURE_REQUIREMENTS, [id=#16]
+
+(20) HashAggregate [codegen id : 4]
+Input [4]: [d_year#11, i_brand#7, i_brand_id#6, sum#15]
+Keys [3]: [d_year#11, i_brand#7, i_brand_id#6]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#17]
+Results [4]: [d_year#11, i_brand_id#6 AS brand_id#18, i_brand#7 AS brand#19, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#17,17,2) AS sum_agg#20]
+
+(21) TakeOrderedAndProject
+Input [4]: [d_year#11, brand_id#18, brand#19, sum_agg#20]
+Arguments: 100, [d_year#11 ASC NULLS FIRST, sum_agg#20 DESC NULLS LAST, brand_id#18 ASC NULLS FIRST], [d_year#11, brand_id#18, brand#19, sum_agg#20]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (22)
+
+
+(22) ReusedExchange [Reuses operator id: 15]
+Output [2]: [d_date_sk#10, d_year#11]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.sf100/simplified.txt
new file mode 100644
index 0000000000000..9a4679289d519
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.sf100/simplified.txt
@@ -0,0 +1,33 @@
+TakeOrderedAndProject [d_year,sum_agg,brand_id,brand]
+  WholeStageCodegen (4)
+    HashAggregate [d_year,i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,sum_agg,sum]
+      InputAdapter
+        Exchange [d_year,i_brand,i_brand_id] #1
+          WholeStageCodegen (3)
+            HashAggregate [d_year,i_brand,i_brand_id,ss_ext_sales_price] [sum,sum]
+              Project [d_year,ss_ext_sales_price,i_brand_id,i_brand]
+                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                  Project [ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_brand]
+                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                      Filter [ss_item_sk]
+                        ColumnarToRow
+                          InputAdapter
+                            Scan parquet default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk]
+                              SubqueryBroadcast [d_date_sk] #1
+                                ReusedExchange [d_date_sk,d_year] #2
+                      InputAdapter
+                        BroadcastExchange #3
+                          WholeStageCodegen (1)
+                            Project [i_item_sk,i_brand_id,i_brand]
+                              Filter [i_manufact_id,i_item_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id]
+                  InputAdapter
+                    BroadcastExchange #2
+                      WholeStageCodegen (2)
+                        Project [d_date_sk,d_year]
+                          Filter [d_moy,d_date_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/explain.txt
new file mode 100644
index 0000000000000..77eb7fefd4d05
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/explain.txt
@@ -0,0 +1,123 @@
+== Physical Plan ==
+TakeOrderedAndProject (21)
++- * HashAggregate (20)
+   +- Exchange (19)
+      +- * HashAggregate (18)
+         +- * Project (17)
+            +- * BroadcastHashJoin Inner BuildRight (16)
+               :- * Project (10)
+               :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :- * Project (4)
+               :     :  +- * Filter (3)
+               :     :     +- * ColumnarToRow (2)
+               :     :        +- Scan parquet default.date_dim (1)
+               :     +- BroadcastExchange (8)
+               :        +- * Filter (7)
+               :           +- * ColumnarToRow (6)
+               :              +- Scan parquet default.store_sales (5)
+               +- BroadcastExchange (15)
+                  +- * Project (14)
+                     +- * Filter (13)
+                        +- * ColumnarToRow (12)
+                           +- Scan parquet default.item (11)
+
+
+(1) Scan parquet default.date_dim
+Output [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), EqualTo(d_moy,11), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Condition : ((isnotnull(d_moy#3) AND (d_moy#3 = 11)) AND isnotnull(d_date_sk#1))
+
+(4) Project [codegen id : 3]
+Output [2]: [d_date_sk#1, d_year#2]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(5) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+
+(7) Filter [codegen id : 1]
+Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Condition : isnotnull(ss_item_sk#4)
+
+(8) BroadcastExchange
+Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [d_date_sk#1]
+Right keys [1]: [ss_sold_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5]
+Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+
+(11) Scan parquet default.item
+Output [4]: [i_item_sk#8, i_brand_id#9, i_brand#10, i_manufact_id#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,128), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_brand:string,i_manufact_id:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [4]: [i_item_sk#8, i_brand_id#9, i_brand#10, i_manufact_id#11]
+
+(13) Filter [codegen id : 2]
+Input [4]: [i_item_sk#8, i_brand_id#9, i_brand#10, i_manufact_id#11]
+Condition : ((isnotnull(i_manufact_id#11) AND (i_manufact_id#11 = 128)) AND isnotnull(i_item_sk#8))
+
+(14) Project [codegen id : 2]
+Output [3]: [i_item_sk#8, i_brand_id#9, i_brand#10]
+Input [4]: [i_item_sk#8, i_brand_id#9, i_brand#10, i_manufact_id#11]
+
+(15) BroadcastExchange
+Input [3]: [i_item_sk#8, i_brand_id#9, i_brand#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(16) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_item_sk#4]
+Right keys [1]: [i_item_sk#8]
+Join condition: None
+
+(17) Project [codegen id : 3]
+Output [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#9, i_brand#10]
+Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#8, i_brand_id#9, i_brand#10]
+
+(18) HashAggregate [codegen id : 3]
+Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#9, i_brand#10]
+Keys [3]: [d_year#2, i_brand#10, i_brand_id#9]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))]
+Aggregate Attributes [1]: [sum#13]
+Results [4]: [d_year#2, i_brand#10, i_brand_id#9, sum#14]
+
+(19) Exchange
+Input [4]: [d_year#2, i_brand#10, i_brand_id#9, sum#14]
+Arguments: hashpartitioning(d_year#2, i_brand#10, i_brand_id#9, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(20) HashAggregate [codegen id : 4]
+Input [4]: [d_year#2, i_brand#10, i_brand_id#9, sum#14]
+Keys [3]: [d_year#2, i_brand#10, i_brand_id#9]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#16]
+Results [4]: [d_year#2, i_brand_id#9 AS brand_id#17, i_brand#10 AS brand#18, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#16,17,2) AS sum_agg#19]
+
+(21) TakeOrderedAndProject
+Input [4]: [d_year#2, brand_id#17, brand#18, sum_agg#19]
+Arguments: 100, [d_year#2 ASC NULLS FIRST, sum_agg#19 DESC NULLS LAST, brand_id#17 ASC NULLS FIRST], [d_year#2, brand_id#17, brand#18, sum_agg#19]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/simplified.txt
new file mode 100644
index 0000000000000..e05ba42f4e0ee
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/simplified.txt
@@ -0,0 +1,31 @@
+TakeOrderedAndProject [d_year,sum_agg,brand_id,brand]
+  WholeStageCodegen (4)
+    HashAggregate [d_year,i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,sum_agg,sum]
+      InputAdapter
+        Exchange [d_year,i_brand,i_brand_id] #1
+          WholeStageCodegen (3)
+            HashAggregate [d_year,i_brand,i_brand_id,ss_ext_sales_price] [sum,sum]
+              Project [d_year,ss_ext_sales_price,i_brand_id,i_brand]
+                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                  Project [d_year,ss_item_sk,ss_ext_sales_price]
+                    BroadcastHashJoin [d_date_sk,ss_sold_date_sk]
+                      Project [d_date_sk,d_year]
+                        Filter [d_moy,d_date_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                      InputAdapter
+                        BroadcastExchange #2
+                          WholeStageCodegen (1)
+                            Filter [ss_item_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk]
+                  InputAdapter
+                    BroadcastExchange #3
+                      WholeStageCodegen (2)
+                        Project [i_item_sk,i_brand_id,i_brand]
+                          Filter [i_manufact_id,i_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.sf100/explain.txt
new file mode 100644
index 0000000000000..6fdac15976957
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.sf100/explain.txt
@@ -0,0 +1,347 @@
+== Physical Plan ==
+TakeOrderedAndProject (59)
++- * Project (58)
+   +- * BroadcastHashJoin Inner BuildRight (57)
+      :- * Project (36)
+      :  +- * BroadcastHashJoin Inner BuildLeft (35)
+      :     :- BroadcastExchange (11)
+      :     :  +- * Project (10)
+      :     :     +- * BroadcastHashJoin Inner BuildRight (9)
+      :     :        :- * Filter (3)
+      :     :        :  +- * ColumnarToRow (2)
+      :     :        :     +- Scan parquet default.customer (1)
+      :     :        +- BroadcastExchange (8)
+      :     :           +- * Project (7)
+      :     :              +- * Filter (6)
+      :     :                 +- * ColumnarToRow (5)
+      :     :                    +- Scan parquet default.customer_address (4)
+      :     +- * Filter (34)
+      :        +- * HashAggregate (33)
+      :           +- Exchange (32)
+      :              +- * HashAggregate (31)
+      :                 +- * Project (30)
+      :                    +- * SortMergeJoin Inner (29)
+      :                       :- * Sort (23)
+      :                       :  +- Exchange (22)
+      :                       :     +- * Project (21)
+      :                       :        +- * BroadcastHashJoin Inner BuildRight (20)
+      :                       :           :- * Filter (14)
+      :                       :           :  +- * ColumnarToRow (13)
+      :                       :           :     +- Scan parquet default.web_returns (12)
+      :                       :           +- BroadcastExchange (19)
+      :                       :              +- * Project (18)
+      :                       :                 +- * Filter (17)
+      :                       :                    +- * ColumnarToRow (16)
+      :                       :                       +- Scan parquet default.date_dim (15)
+      :                       +- * Sort (28)
+      :                          +- Exchange (27)
+      :                             +- * Filter (26)
+      :                                +- * ColumnarToRow (25)
+      :                                   +- Scan parquet default.customer_address (24)
+      +- BroadcastExchange (56)
+         +- * Filter (55)
+            +- * HashAggregate (54)
+               +- Exchange (53)
+                  +- * HashAggregate (52)
+                     +- * HashAggregate (51)
+                        +- Exchange (50)
+                           +- * HashAggregate (49)
+                              +- * Project (48)
+                                 +- * SortMergeJoin Inner (47)
+                                    :- * Sort (44)
+                                    :  +- Exchange (43)
+                                    :     +- * Project (42)
+                                    :        +- * BroadcastHashJoin Inner BuildRight (41)
+                                    :           :- * Filter (39)
+                                    :           :  +- * ColumnarToRow (38)
+                                    :           :     +- Scan parquet default.web_returns (37)
+                                    :           +- ReusedExchange (40)
+                                    +- * Sort (46)
+                                       +- ReusedExchange (45)
+
+
+(1) Scan parquet default.customer
+Output [14]: [c_customer_sk#1, c_customer_id#2, c_current_addr_sk#3, c_salutation#4, c_first_name#5, c_last_name#6, c_preferred_cust_flag#7, c_birth_day#8, c_birth_month#9, c_birth_year#10, c_birth_country#11, c_login#12, c_email_address#13, c_last_review_date#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_current_addr_sk:int,c_salutation:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string,c_birth_day:int,c_birth_month:int,c_birth_year:int,c_birth_country:string,c_login:string,c_email_address:string,c_last_review_date:int>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [14]: [c_customer_sk#1, c_customer_id#2, c_current_addr_sk#3, c_salutation#4, c_first_name#5, c_last_name#6, c_preferred_cust_flag#7, c_birth_day#8, c_birth_month#9, c_birth_year#10, c_birth_country#11, c_login#12, c_email_address#13, c_last_review_date#14]
+
+(3) Filter [codegen id : 2]
+Input [14]: [c_customer_sk#1, c_customer_id#2, c_current_addr_sk#3, c_salutation#4, c_first_name#5, c_last_name#6, c_preferred_cust_flag#7, c_birth_day#8, c_birth_month#9, c_birth_year#10, c_birth_country#11, c_login#12, c_email_address#13, c_last_review_date#14]
+Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_current_addr_sk#3))
+
+(4) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#15, ca_state#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [ca_address_sk#15, ca_state#16]
+
+(6) Filter [codegen id : 1]
+Input [2]: [ca_address_sk#15, ca_state#16]
+Condition : ((isnotnull(ca_state#16) AND (ca_state#16 = GA)) AND isnotnull(ca_address_sk#15))
+
+(7) Project [codegen id : 1]
+Output [1]: [ca_address_sk#15]
+Input [2]: [ca_address_sk#15, ca_state#16]
+
+(8) BroadcastExchange
+Input [1]: [ca_address_sk#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(9) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [c_current_addr_sk#3]
+Right keys [1]: [ca_address_sk#15]
+Join condition: None
+
+(10) Project [codegen id : 2]
+Output [13]: [c_customer_sk#1, c_customer_id#2, c_salutation#4, c_first_name#5, c_last_name#6, c_preferred_cust_flag#7, c_birth_day#8, c_birth_month#9, c_birth_year#10, c_birth_country#11, c_login#12, c_email_address#13, c_last_review_date#14]
+Input [15]: [c_customer_sk#1, c_customer_id#2, c_current_addr_sk#3, c_salutation#4, c_first_name#5, c_last_name#6, c_preferred_cust_flag#7, c_birth_day#8, c_birth_month#9, c_birth_year#10, c_birth_country#11, c_login#12, c_email_address#13, c_last_review_date#14, ca_address_sk#15]
+
+(11) BroadcastExchange
+Input [13]: [c_customer_sk#1, c_customer_id#2, c_salutation#4, c_first_name#5, c_last_name#6, c_preferred_cust_flag#7, c_birth_day#8, c_birth_month#9, c_birth_year#10, c_birth_country#11, c_login#12, c_email_address#13, c_last_review_date#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(12) Scan parquet default.web_returns
+Output [4]: [wr_returning_customer_sk#19, wr_returning_addr_sk#20, wr_return_amt#21, wr_returned_date_sk#22]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(wr_returned_date_sk#22), dynamicpruningexpression(wr_returned_date_sk#22 IN dynamicpruning#23)]
+PushedFilters: [IsNotNull(wr_returning_addr_sk), IsNotNull(wr_returning_customer_sk)]
+ReadSchema: struct<wr_returning_customer_sk:int,wr_returning_addr_sk:int,wr_return_amt:decimal(7,2)>
+
+(13) ColumnarToRow [codegen id : 4]
+Input [4]: [wr_returning_customer_sk#19, wr_returning_addr_sk#20, wr_return_amt#21, wr_returned_date_sk#22]
+
+(14) Filter [codegen id : 4]
+Input [4]: [wr_returning_customer_sk#19, wr_returning_addr_sk#20, wr_return_amt#21, wr_returned_date_sk#22]
+Condition : (isnotnull(wr_returning_addr_sk#20) AND isnotnull(wr_returning_customer_sk#19))
+
+(15) Scan parquet default.date_dim
+Output [2]: [d_date_sk#24, d_year#25]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(16) ColumnarToRow [codegen id : 3]
+Input [2]: [d_date_sk#24, d_year#25]
+
+(17) Filter [codegen id : 3]
+Input [2]: [d_date_sk#24, d_year#25]
+Condition : ((isnotnull(d_year#25) AND (d_year#25 = 2002)) AND isnotnull(d_date_sk#24))
+
+(18) Project [codegen id : 3]
+Output [1]: [d_date_sk#24]
+Input [2]: [d_date_sk#24, d_year#25]
+
+(19) BroadcastExchange
+Input [1]: [d_date_sk#24]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#26]
+
+(20) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [wr_returned_date_sk#22]
+Right keys [1]: [d_date_sk#24]
+Join condition: None
+
+(21) Project [codegen id : 4]
+Output [3]: [wr_returning_customer_sk#19, wr_returning_addr_sk#20, wr_return_amt#21]
+Input [5]: [wr_returning_customer_sk#19, wr_returning_addr_sk#20, wr_return_amt#21, wr_returned_date_sk#22, d_date_sk#24]
+
+(22) Exchange
+Input [3]: [wr_returning_customer_sk#19, wr_returning_addr_sk#20, wr_return_amt#21]
+Arguments: hashpartitioning(wr_returning_addr_sk#20, 5), ENSURE_REQUIREMENTS, [id=#27]
+
+(23) Sort [codegen id : 5]
+Input [3]: [wr_returning_customer_sk#19, wr_returning_addr_sk#20, wr_return_amt#21]
+Arguments: [wr_returning_addr_sk#20 ASC NULLS FIRST], false, 0
+
+(24) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#28, ca_state#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_state)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(25) ColumnarToRow [codegen id : 6]
+Input [2]: [ca_address_sk#28, ca_state#29]
+
+(26) Filter [codegen id : 6]
+Input [2]: [ca_address_sk#28, ca_state#29]
+Condition : (isnotnull(ca_address_sk#28) AND isnotnull(ca_state#29))
+
+(27) Exchange
+Input [2]: [ca_address_sk#28, ca_state#29]
+Arguments: hashpartitioning(ca_address_sk#28, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(28) Sort [codegen id : 7]
+Input [2]: [ca_address_sk#28, ca_state#29]
+Arguments: [ca_address_sk#28 ASC NULLS FIRST], false, 0
+
+(29) SortMergeJoin [codegen id : 8]
+Left keys [1]: [wr_returning_addr_sk#20]
+Right keys [1]: [ca_address_sk#28]
+Join condition: None
+
+(30) Project [codegen id : 8]
+Output [3]: [wr_returning_customer_sk#19, wr_return_amt#21, ca_state#29]
+Input [5]: [wr_returning_customer_sk#19, wr_returning_addr_sk#20, wr_return_amt#21, ca_address_sk#28, ca_state#29]
+
+(31) HashAggregate [codegen id : 8]
+Input [3]: [wr_returning_customer_sk#19, wr_return_amt#21, ca_state#29]
+Keys [2]: [wr_returning_customer_sk#19, ca_state#29]
+Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#21))]
+Aggregate Attributes [1]: [sum#31]
+Results [3]: [wr_returning_customer_sk#19, ca_state#29, sum#32]
+
+(32) Exchange
+Input [3]: [wr_returning_customer_sk#19, ca_state#29, sum#32]
+Arguments: hashpartitioning(wr_returning_customer_sk#19, ca_state#29, 5), ENSURE_REQUIREMENTS, [id=#33]
+
+(33) HashAggregate
+Input [3]: [wr_returning_customer_sk#19, ca_state#29, sum#32]
+Keys [2]: [wr_returning_customer_sk#19, ca_state#29]
+Functions [1]: [sum(UnscaledValue(wr_return_amt#21))]
+Aggregate Attributes [1]: [sum(UnscaledValue(wr_return_amt#21))#34]
+Results [3]: [wr_returning_customer_sk#19 AS ctr_customer_sk#35, ca_state#29 AS ctr_state#36, MakeDecimal(sum(UnscaledValue(wr_return_amt#21))#34,17,2) AS ctr_total_return#37]
+
+(34) Filter
+Input [3]: [ctr_customer_sk#35, ctr_state#36, ctr_total_return#37]
+Condition : isnotnull(ctr_total_return#37)
+
+(35) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [ctr_customer_sk#35]
+Join condition: None
+
+(36) Project [codegen id : 17]
+Output [14]: [c_customer_id#2, c_salutation#4, c_first_name#5, c_last_name#6, c_preferred_cust_flag#7, c_birth_day#8, c_birth_month#9, c_birth_year#10, c_birth_country#11, c_login#12, c_email_address#13, c_last_review_date#14, ctr_state#36, ctr_total_return#37]
+Input [16]: [c_customer_sk#1, c_customer_id#2, c_salutation#4, c_first_name#5, c_last_name#6, c_preferred_cust_flag#7, c_birth_day#8, c_birth_month#9, c_birth_year#10, c_birth_country#11, c_login#12, c_email_address#13, c_last_review_date#14, ctr_customer_sk#35, ctr_state#36, ctr_total_return#37]
+
+(37) Scan parquet default.web_returns
+Output [4]: [wr_returning_customer_sk#38, wr_returning_addr_sk#39, wr_return_amt#40, wr_returned_date_sk#41]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(wr_returned_date_sk#41), dynamicpruningexpression(wr_returned_date_sk#41 IN dynamicpruning#23)]
+PushedFilters: [IsNotNull(wr_returning_addr_sk)]
+ReadSchema: struct<wr_returning_customer_sk:int,wr_returning_addr_sk:int,wr_return_amt:decimal(7,2)>
+
+(38) ColumnarToRow [codegen id : 10]
+Input [4]: [wr_returning_customer_sk#38, wr_returning_addr_sk#39, wr_return_amt#40, wr_returned_date_sk#41]
+
+(39) Filter [codegen id : 10]
+Input [4]: [wr_returning_customer_sk#38, wr_returning_addr_sk#39, wr_return_amt#40, wr_returned_date_sk#41]
+Condition : isnotnull(wr_returning_addr_sk#39)
+
+(40) ReusedExchange [Reuses operator id: 19]
+Output [1]: [d_date_sk#42]
+
+(41) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [wr_returned_date_sk#41]
+Right keys [1]: [d_date_sk#42]
+Join condition: None
+
+(42) Project [codegen id : 10]
+Output [3]: [wr_returning_customer_sk#38, wr_returning_addr_sk#39, wr_return_amt#40]
+Input [5]: [wr_returning_customer_sk#38, wr_returning_addr_sk#39, wr_return_amt#40, wr_returned_date_sk#41, d_date_sk#42]
+
+(43) Exchange
+Input [3]: [wr_returning_customer_sk#38, wr_returning_addr_sk#39, wr_return_amt#40]
+Arguments: hashpartitioning(wr_returning_addr_sk#39, 5), ENSURE_REQUIREMENTS, [id=#43]
+
+(44) Sort [codegen id : 11]
+Input [3]: [wr_returning_customer_sk#38, wr_returning_addr_sk#39, wr_return_amt#40]
+Arguments: [wr_returning_addr_sk#39 ASC NULLS FIRST], false, 0
+
+(45) ReusedExchange [Reuses operator id: 27]
+Output [2]: [ca_address_sk#44, ca_state#45]
+
+(46) Sort [codegen id : 13]
+Input [2]: [ca_address_sk#44, ca_state#45]
+Arguments: [ca_address_sk#44 ASC NULLS FIRST], false, 0
+
+(47) SortMergeJoin [codegen id : 14]
+Left keys [1]: [wr_returning_addr_sk#39]
+Right keys [1]: [ca_address_sk#44]
+Join condition: None
+
+(48) Project [codegen id : 14]
+Output [3]: [wr_returning_customer_sk#38, wr_return_amt#40, ca_state#45]
+Input [5]: [wr_returning_customer_sk#38, wr_returning_addr_sk#39, wr_return_amt#40, ca_address_sk#44, ca_state#45]
+
+(49) HashAggregate [codegen id : 14]
+Input [3]: [wr_returning_customer_sk#38, wr_return_amt#40, ca_state#45]
+Keys [2]: [wr_returning_customer_sk#38, ca_state#45]
+Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#40))]
+Aggregate Attributes [1]: [sum#46]
+Results [3]: [wr_returning_customer_sk#38, ca_state#45, sum#47]
+
+(50) Exchange
+Input [3]: [wr_returning_customer_sk#38, ca_state#45, sum#47]
+Arguments: hashpartitioning(wr_returning_customer_sk#38, ca_state#45, 5), ENSURE_REQUIREMENTS, [id=#48]
+
+(51) HashAggregate [codegen id : 15]
+Input [3]: [wr_returning_customer_sk#38, ca_state#45, sum#47]
+Keys [2]: [wr_returning_customer_sk#38, ca_state#45]
+Functions [1]: [sum(UnscaledValue(wr_return_amt#40))]
+Aggregate Attributes [1]: [sum(UnscaledValue(wr_return_amt#40))#49]
+Results [2]: [ca_state#45 AS ctr_state#36, MakeDecimal(sum(UnscaledValue(wr_return_amt#40))#49,17,2) AS ctr_total_return#37]
+
+(52) HashAggregate [codegen id : 15]
+Input [2]: [ctr_state#36, ctr_total_return#37]
+Keys [1]: [ctr_state#36]
+Functions [1]: [partial_avg(ctr_total_return#37)]
+Aggregate Attributes [2]: [sum#50, count#51]
+Results [3]: [ctr_state#36, sum#52, count#53]
+
+(53) Exchange
+Input [3]: [ctr_state#36, sum#52, count#53]
+Arguments: hashpartitioning(ctr_state#36, 5), ENSURE_REQUIREMENTS, [id=#54]
+
+(54) HashAggregate [codegen id : 16]
+Input [3]: [ctr_state#36, sum#52, count#53]
+Keys [1]: [ctr_state#36]
+Functions [1]: [avg(ctr_total_return#37)]
+Aggregate Attributes [1]: [avg(ctr_total_return#37)#55]
+Results [2]: [CheckOverflow((promote_precision(avg(ctr_total_return#37)#55) * 1.200000), DecimalType(24,7), true) AS (avg(ctr_total_return) * 1.2)#56, ctr_state#36 AS ctr_state#36#57]
+
+(55) Filter [codegen id : 16]
+Input [2]: [(avg(ctr_total_return) * 1.2)#56, ctr_state#36#57]
+Condition : isnotnull((avg(ctr_total_return) * 1.2)#56)
+
+(56) BroadcastExchange
+Input [2]: [(avg(ctr_total_return) * 1.2)#56, ctr_state#36#57]
+Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [id=#58]
+
+(57) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ctr_state#36]
+Right keys [1]: [ctr_state#36#57]
+Join condition: (cast(ctr_total_return#37 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#56)
+
+(58) Project [codegen id : 17]
+Output [13]: [c_customer_id#2, c_salutation#4, c_first_name#5, c_last_name#6, c_preferred_cust_flag#7, c_birth_day#8, c_birth_month#9, c_birth_year#10, c_birth_country#11, c_login#12, c_email_address#13, c_last_review_date#14, ctr_total_return#37]
+Input [16]: [c_customer_id#2, c_salutation#4, c_first_name#5, c_last_name#6, c_preferred_cust_flag#7, c_birth_day#8, c_birth_month#9, c_birth_year#10, c_birth_country#11, c_login#12, c_email_address#13, c_last_review_date#14, ctr_state#36, ctr_total_return#37, (avg(ctr_total_return) * 1.2)#56, ctr_state#36#57]
+
+(59) TakeOrderedAndProject
+Input [13]: [c_customer_id#2, c_salutation#4, c_first_name#5, c_last_name#6, c_preferred_cust_flag#7, c_birth_day#8, c_birth_month#9, c_birth_year#10, c_birth_country#11, c_login#12, c_email_address#13, c_last_review_date#14, ctr_total_return#37]
+Arguments: 100, [c_customer_id#2 ASC NULLS FIRST, c_salutation#4 ASC NULLS FIRST, c_first_name#5 ASC NULLS FIRST, c_last_name#6 ASC NULLS FIRST, c_preferred_cust_flag#7 ASC NULLS FIRST, c_birth_day#8 ASC NULLS FIRST, c_birth_month#9 ASC NULLS FIRST, c_birth_year#10 ASC NULLS FIRST, c_birth_country#11 ASC NULLS FIRST, c_login#12 ASC NULLS FIRST, c_email_address#13 ASC NULLS FIRST, c_last_review_date#14 ASC NULLS FIRST, ctr_total_return#37 ASC NULLS FIRST], [c_customer_id#2, c_salutation#4, c_first_name#5, c_last_name#6, c_preferred_cust_flag#7, c_birth_day#8, c_birth_month#9, c_birth_year#10, c_birth_country#11, c_login#12, c_email_address#13, c_last_review_date#14, ctr_total_return#37]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 12 Hosting Expression = wr_returned_date_sk#22 IN dynamicpruning#23
+ReusedExchange (60)
+
+
+(60) ReusedExchange [Reuses operator id: 19]
+Output [1]: [d_date_sk#24]
+
+Subquery:2 Hosting operator id = 37 Hosting Expression = wr_returned_date_sk#41 IN dynamicpruning#23
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.sf100/simplified.txt
new file mode 100644
index 0000000000000..e8c44d275d745
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.sf100/simplified.txt
@@ -0,0 +1,99 @@
+TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return]
+  WholeStageCodegen (17)
+    Project [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return]
+      BroadcastHashJoin [ctr_state,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2)]
+        Project [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_state,ctr_total_return]
+          BroadcastHashJoin [c_customer_sk,ctr_customer_sk]
+            InputAdapter
+              BroadcastExchange #1
+                WholeStageCodegen (2)
+                  Project [c_customer_sk,c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date]
+                    BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                      Filter [c_customer_sk,c_current_addr_sk]
+                        ColumnarToRow
+                          InputAdapter
+                            Scan parquet default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date]
+                      InputAdapter
+                        BroadcastExchange #2
+                          WholeStageCodegen (1)
+                            Project [ca_address_sk]
+                              Filter [ca_state,ca_address_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.customer_address [ca_address_sk,ca_state]
+            Filter [ctr_total_return]
+              HashAggregate [wr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(wr_return_amt)),ctr_customer_sk,ctr_state,ctr_total_return,sum]
+                InputAdapter
+                  Exchange [wr_returning_customer_sk,ca_state] #3
+                    WholeStageCodegen (8)
+                      HashAggregate [wr_returning_customer_sk,ca_state,wr_return_amt] [sum,sum]
+                        Project [wr_returning_customer_sk,wr_return_amt,ca_state]
+                          SortMergeJoin [wr_returning_addr_sk,ca_address_sk]
+                            InputAdapter
+                              WholeStageCodegen (5)
+                                Sort [wr_returning_addr_sk]
+                                  InputAdapter
+                                    Exchange [wr_returning_addr_sk] #4
+                                      WholeStageCodegen (4)
+                                        Project [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt]
+                                          BroadcastHashJoin [wr_returned_date_sk,d_date_sk]
+                                            Filter [wr_returning_addr_sk,wr_returning_customer_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #1
+                                                      ReusedExchange [d_date_sk] #5
+                                            InputAdapter
+                                              BroadcastExchange #5
+                                                WholeStageCodegen (3)
+                                                  Project [d_date_sk]
+                                                    Filter [d_year,d_date_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.date_dim [d_date_sk,d_year]
+                            InputAdapter
+                              WholeStageCodegen (7)
+                                Sort [ca_address_sk]
+                                  InputAdapter
+                                    Exchange [ca_address_sk] #6
+                                      WholeStageCodegen (6)
+                                        Filter [ca_address_sk,ca_state]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer_address [ca_address_sk,ca_state]
+        InputAdapter
+          BroadcastExchange #7
+            WholeStageCodegen (16)
+              Filter [(avg(ctr_total_return) * 1.2)]
+                HashAggregate [ctr_state,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),ctr_state,sum,count]
+                  InputAdapter
+                    Exchange [ctr_state] #8
+                      WholeStageCodegen (15)
+                        HashAggregate [ctr_state,ctr_total_return] [sum,count,sum,count]
+                          HashAggregate [wr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(wr_return_amt)),ctr_state,ctr_total_return,sum]
+                            InputAdapter
+                              Exchange [wr_returning_customer_sk,ca_state] #9
+                                WholeStageCodegen (14)
+                                  HashAggregate [wr_returning_customer_sk,ca_state,wr_return_amt] [sum,sum]
+                                    Project [wr_returning_customer_sk,wr_return_amt,ca_state]
+                                      SortMergeJoin [wr_returning_addr_sk,ca_address_sk]
+                                        InputAdapter
+                                          WholeStageCodegen (11)
+                                            Sort [wr_returning_addr_sk]
+                                              InputAdapter
+                                                Exchange [wr_returning_addr_sk] #10
+                                                  WholeStageCodegen (10)
+                                                    Project [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt]
+                                                      BroadcastHashJoin [wr_returned_date_sk,d_date_sk]
+                                                        Filter [wr_returning_addr_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk]
+                                                                ReusedSubquery [d_date_sk] #1
+                                                        InputAdapter
+                                                          ReusedExchange [d_date_sk] #5
+                                        InputAdapter
+                                          WholeStageCodegen (13)
+                                            Sort [ca_address_sk]
+                                              InputAdapter
+                                                ReusedExchange [ca_address_sk,ca_state] #6
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/explain.txt
new file mode 100644
index 0000000000000..ca8b8d966ea92
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/explain.txt
@@ -0,0 +1,317 @@
+== Physical Plan ==
+TakeOrderedAndProject (53)
++- * Project (52)
+   +- * BroadcastHashJoin Inner BuildRight (51)
+      :- * Project (45)
+      :  +- * BroadcastHashJoin Inner BuildRight (44)
+      :     :- * Project (39)
+      :     :  +- * BroadcastHashJoin Inner BuildRight (38)
+      :     :     :- * Filter (20)
+      :     :     :  +- * HashAggregate (19)
+      :     :     :     +- Exchange (18)
+      :     :     :        +- * HashAggregate (17)
+      :     :     :           +- * Project (16)
+      :     :     :              +- * BroadcastHashJoin Inner BuildRight (15)
+      :     :     :                 :- * Project (10)
+      :     :     :                 :  +- * BroadcastHashJoin Inner BuildRight (9)
+      :     :     :                 :     :- * Filter (3)
+      :     :     :                 :     :  +- * ColumnarToRow (2)
+      :     :     :                 :     :     +- Scan parquet default.web_returns (1)
+      :     :     :                 :     +- BroadcastExchange (8)
+      :     :     :                 :        +- * Project (7)
+      :     :     :                 :           +- * Filter (6)
+      :     :     :                 :              +- * ColumnarToRow (5)
+      :     :     :                 :                 +- Scan parquet default.date_dim (4)
+      :     :     :                 +- BroadcastExchange (14)
+      :     :     :                    +- * Filter (13)
+      :     :     :                       +- * ColumnarToRow (12)
+      :     :     :                          +- Scan parquet default.customer_address (11)
+      :     :     +- BroadcastExchange (37)
+      :     :        +- * Filter (36)
+      :     :           +- * HashAggregate (35)
+      :     :              +- Exchange (34)
+      :     :                 +- * HashAggregate (33)
+      :     :                    +- * HashAggregate (32)
+      :     :                       +- Exchange (31)
+      :     :                          +- * HashAggregate (30)
+      :     :                             +- * Project (29)
+      :     :                                +- * BroadcastHashJoin Inner BuildRight (28)
+      :     :                                   :- * Project (26)
+      :     :                                   :  +- * BroadcastHashJoin Inner BuildRight (25)
+      :     :                                   :     :- * Filter (23)
+      :     :                                   :     :  +- * ColumnarToRow (22)
+      :     :                                   :     :     +- Scan parquet default.web_returns (21)
+      :     :                                   :     +- ReusedExchange (24)
+      :     :                                   +- ReusedExchange (27)
+      :     +- BroadcastExchange (43)
+      :        +- * Filter (42)
+      :           +- * ColumnarToRow (41)
+      :              +- Scan parquet default.customer (40)
+      +- BroadcastExchange (50)
+         +- * Project (49)
+            +- * Filter (48)
+               +- * ColumnarToRow (47)
+                  +- Scan parquet default.customer_address (46)
+
+
+(1) Scan parquet default.web_returns
+Output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(wr_returned_date_sk#4), dynamicpruningexpression(wr_returned_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(wr_returning_addr_sk), IsNotNull(wr_returning_customer_sk)]
+ReadSchema: struct<wr_returning_customer_sk:int,wr_returning_addr_sk:int,wr_return_amt:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4]
+
+(3) Filter [codegen id : 3]
+Input [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4]
+Condition : (isnotnull(wr_returning_addr_sk#2) AND isnotnull(wr_returning_customer_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#6, d_year#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#6, d_year#7]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#6, d_year#7]
+Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2002)) AND isnotnull(d_date_sk#6))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#6]
+Input [2]: [d_date_sk#6, d_year#7]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#8]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [wr_returned_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [3]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3]
+Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4, d_date_sk#6]
+
+(11) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#9, ca_state#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_state)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [ca_address_sk#9, ca_state#10]
+
+(13) Filter [codegen id : 2]
+Input [2]: [ca_address_sk#9, ca_state#10]
+Condition : (isnotnull(ca_address_sk#9) AND isnotnull(ca_state#10))
+
+(14) BroadcastExchange
+Input [2]: [ca_address_sk#9, ca_state#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [wr_returning_addr_sk#2]
+Right keys [1]: [ca_address_sk#9]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#10]
+Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#9, ca_state#10]
+
+(17) HashAggregate [codegen id : 3]
+Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#10]
+Keys [2]: [wr_returning_customer_sk#1, ca_state#10]
+Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#3))]
+Aggregate Attributes [1]: [sum#12]
+Results [3]: [wr_returning_customer_sk#1, ca_state#10, sum#13]
+
+(18) Exchange
+Input [3]: [wr_returning_customer_sk#1, ca_state#10, sum#13]
+Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#10, 5), ENSURE_REQUIREMENTS, [id=#14]
+
+(19) HashAggregate [codegen id : 11]
+Input [3]: [wr_returning_customer_sk#1, ca_state#10, sum#13]
+Keys [2]: [wr_returning_customer_sk#1, ca_state#10]
+Functions [1]: [sum(UnscaledValue(wr_return_amt#3))]
+Aggregate Attributes [1]: [sum(UnscaledValue(wr_return_amt#3))#15]
+Results [3]: [wr_returning_customer_sk#1 AS ctr_customer_sk#16, ca_state#10 AS ctr_state#17, MakeDecimal(sum(UnscaledValue(wr_return_amt#3))#15,17,2) AS ctr_total_return#18]
+
+(20) Filter [codegen id : 11]
+Input [3]: [ctr_customer_sk#16, ctr_state#17, ctr_total_return#18]
+Condition : isnotnull(ctr_total_return#18)
+
+(21) Scan parquet default.web_returns
+Output [4]: [wr_returning_customer_sk#19, wr_returning_addr_sk#20, wr_return_amt#21, wr_returned_date_sk#22]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(wr_returned_date_sk#22), dynamicpruningexpression(wr_returned_date_sk#22 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(wr_returning_addr_sk)]
+ReadSchema: struct<wr_returning_customer_sk:int,wr_returning_addr_sk:int,wr_return_amt:decimal(7,2)>
+
+(22) ColumnarToRow [codegen id : 6]
+Input [4]: [wr_returning_customer_sk#19, wr_returning_addr_sk#20, wr_return_amt#21, wr_returned_date_sk#22]
+
+(23) Filter [codegen id : 6]
+Input [4]: [wr_returning_customer_sk#19, wr_returning_addr_sk#20, wr_return_amt#21, wr_returned_date_sk#22]
+Condition : isnotnull(wr_returning_addr_sk#20)
+
+(24) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#23]
+
+(25) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [wr_returned_date_sk#22]
+Right keys [1]: [d_date_sk#23]
+Join condition: None
+
+(26) Project [codegen id : 6]
+Output [3]: [wr_returning_customer_sk#19, wr_returning_addr_sk#20, wr_return_amt#21]
+Input [5]: [wr_returning_customer_sk#19, wr_returning_addr_sk#20, wr_return_amt#21, wr_returned_date_sk#22, d_date_sk#23]
+
+(27) ReusedExchange [Reuses operator id: 14]
+Output [2]: [ca_address_sk#24, ca_state#25]
+
+(28) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [wr_returning_addr_sk#20]
+Right keys [1]: [ca_address_sk#24]
+Join condition: None
+
+(29) Project [codegen id : 6]
+Output [3]: [wr_returning_customer_sk#19, wr_return_amt#21, ca_state#25]
+Input [5]: [wr_returning_customer_sk#19, wr_returning_addr_sk#20, wr_return_amt#21, ca_address_sk#24, ca_state#25]
+
+(30) HashAggregate [codegen id : 6]
+Input [3]: [wr_returning_customer_sk#19, wr_return_amt#21, ca_state#25]
+Keys [2]: [wr_returning_customer_sk#19, ca_state#25]
+Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#21))]
+Aggregate Attributes [1]: [sum#26]
+Results [3]: [wr_returning_customer_sk#19, ca_state#25, sum#27]
+
+(31) Exchange
+Input [3]: [wr_returning_customer_sk#19, ca_state#25, sum#27]
+Arguments: hashpartitioning(wr_returning_customer_sk#19, ca_state#25, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(32) HashAggregate [codegen id : 7]
+Input [3]: [wr_returning_customer_sk#19, ca_state#25, sum#27]
+Keys [2]: [wr_returning_customer_sk#19, ca_state#25]
+Functions [1]: [sum(UnscaledValue(wr_return_amt#21))]
+Aggregate Attributes [1]: [sum(UnscaledValue(wr_return_amt#21))#29]
+Results [2]: [ca_state#25 AS ctr_state#17, MakeDecimal(sum(UnscaledValue(wr_return_amt#21))#29,17,2) AS ctr_total_return#18]
+
+(33) HashAggregate [codegen id : 7]
+Input [2]: [ctr_state#17, ctr_total_return#18]
+Keys [1]: [ctr_state#17]
+Functions [1]: [partial_avg(ctr_total_return#18)]
+Aggregate Attributes [2]: [sum#30, count#31]
+Results [3]: [ctr_state#17, sum#32, count#33]
+
+(34) Exchange
+Input [3]: [ctr_state#17, sum#32, count#33]
+Arguments: hashpartitioning(ctr_state#17, 5), ENSURE_REQUIREMENTS, [id=#34]
+
+(35) HashAggregate [codegen id : 8]
+Input [3]: [ctr_state#17, sum#32, count#33]
+Keys [1]: [ctr_state#17]
+Functions [1]: [avg(ctr_total_return#18)]
+Aggregate Attributes [1]: [avg(ctr_total_return#18)#35]
+Results [2]: [CheckOverflow((promote_precision(avg(ctr_total_return#18)#35) * 1.200000), DecimalType(24,7), true) AS (avg(ctr_total_return) * 1.2)#36, ctr_state#17 AS ctr_state#17#37]
+
+(36) Filter [codegen id : 8]
+Input [2]: [(avg(ctr_total_return) * 1.2)#36, ctr_state#17#37]
+Condition : isnotnull((avg(ctr_total_return) * 1.2)#36)
+
+(37) BroadcastExchange
+Input [2]: [(avg(ctr_total_return) * 1.2)#36, ctr_state#17#37]
+Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [id=#38]
+
+(38) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ctr_state#17]
+Right keys [1]: [ctr_state#17#37]
+Join condition: (cast(ctr_total_return#18 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#36)
+
+(39) Project [codegen id : 11]
+Output [2]: [ctr_customer_sk#16, ctr_total_return#18]
+Input [5]: [ctr_customer_sk#16, ctr_state#17, ctr_total_return#18, (avg(ctr_total_return) * 1.2)#36, ctr_state#17#37]
+
+(40) Scan parquet default.customer
+Output [14]: [c_customer_sk#39, c_customer_id#40, c_current_addr_sk#41, c_salutation#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_day#46, c_birth_month#47, c_birth_year#48, c_birth_country#49, c_login#50, c_email_address#51, c_last_review_date#52]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_current_addr_sk:int,c_salutation:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string,c_birth_day:int,c_birth_month:int,c_birth_year:int,c_birth_country:string,c_login:string,c_email_address:string,c_last_review_date:int>
+
+(41) ColumnarToRow [codegen id : 9]
+Input [14]: [c_customer_sk#39, c_customer_id#40, c_current_addr_sk#41, c_salutation#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_day#46, c_birth_month#47, c_birth_year#48, c_birth_country#49, c_login#50, c_email_address#51, c_last_review_date#52]
+
+(42) Filter [codegen id : 9]
+Input [14]: [c_customer_sk#39, c_customer_id#40, c_current_addr_sk#41, c_salutation#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_day#46, c_birth_month#47, c_birth_year#48, c_birth_country#49, c_login#50, c_email_address#51, c_last_review_date#52]
+Condition : (isnotnull(c_customer_sk#39) AND isnotnull(c_current_addr_sk#41))
+
+(43) BroadcastExchange
+Input [14]: [c_customer_sk#39, c_customer_id#40, c_current_addr_sk#41, c_salutation#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_day#46, c_birth_month#47, c_birth_year#48, c_birth_country#49, c_login#50, c_email_address#51, c_last_review_date#52]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#53]
+
+(44) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ctr_customer_sk#16]
+Right keys [1]: [c_customer_sk#39]
+Join condition: None
+
+(45) Project [codegen id : 11]
+Output [14]: [ctr_total_return#18, c_customer_id#40, c_current_addr_sk#41, c_salutation#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_day#46, c_birth_month#47, c_birth_year#48, c_birth_country#49, c_login#50, c_email_address#51, c_last_review_date#52]
+Input [16]: [ctr_customer_sk#16, ctr_total_return#18, c_customer_sk#39, c_customer_id#40, c_current_addr_sk#41, c_salutation#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_day#46, c_birth_month#47, c_birth_year#48, c_birth_country#49, c_login#50, c_email_address#51, c_last_review_date#52]
+
+(46) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#54, ca_state#55]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(47) ColumnarToRow [codegen id : 10]
+Input [2]: [ca_address_sk#54, ca_state#55]
+
+(48) Filter [codegen id : 10]
+Input [2]: [ca_address_sk#54, ca_state#55]
+Condition : ((isnotnull(ca_state#55) AND (ca_state#55 = GA)) AND isnotnull(ca_address_sk#54))
+
+(49) Project [codegen id : 10]
+Output [1]: [ca_address_sk#54]
+Input [2]: [ca_address_sk#54, ca_state#55]
+
+(50) BroadcastExchange
+Input [1]: [ca_address_sk#54]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#56]
+
+(51) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [c_current_addr_sk#41]
+Right keys [1]: [ca_address_sk#54]
+Join condition: None
+
+(52) Project [codegen id : 11]
+Output [13]: [c_customer_id#40, c_salutation#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_day#46, c_birth_month#47, c_birth_year#48, c_birth_country#49, c_login#50, c_email_address#51, c_last_review_date#52, ctr_total_return#18]
+Input [15]: [ctr_total_return#18, c_customer_id#40, c_current_addr_sk#41, c_salutation#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_day#46, c_birth_month#47, c_birth_year#48, c_birth_country#49, c_login#50, c_email_address#51, c_last_review_date#52, ca_address_sk#54]
+
+(53) TakeOrderedAndProject
+Input [13]: [c_customer_id#40, c_salutation#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_day#46, c_birth_month#47, c_birth_year#48, c_birth_country#49, c_login#50, c_email_address#51, c_last_review_date#52, ctr_total_return#18]
+Arguments: 100, [c_customer_id#40 ASC NULLS FIRST, c_salutation#42 ASC NULLS FIRST, c_first_name#43 ASC NULLS FIRST, c_last_name#44 ASC NULLS FIRST, c_preferred_cust_flag#45 ASC NULLS FIRST, c_birth_day#46 ASC NULLS FIRST, c_birth_month#47 ASC NULLS FIRST, c_birth_year#48 ASC NULLS FIRST, c_birth_country#49 ASC NULLS FIRST, c_login#50 ASC NULLS FIRST, c_email_address#51 ASC NULLS FIRST, c_last_review_date#52 ASC NULLS FIRST, ctr_total_return#18 ASC NULLS FIRST], [c_customer_id#40, c_salutation#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_day#46, c_birth_month#47, c_birth_year#48, c_birth_country#49, c_login#50, c_email_address#51, c_last_review_date#52, ctr_total_return#18]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = wr_returned_date_sk#4 IN dynamicpruning#5
+ReusedExchange (54)
+
+
+(54) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#6]
+
+Subquery:2 Hosting operator id = 21 Hosting Expression = wr_returned_date_sk#22 IN dynamicpruning#5
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/simplified.txt
new file mode 100644
index 0000000000000..59584ba2262bd
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/simplified.txt
@@ -0,0 +1,81 @@
+TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return]
+  WholeStageCodegen (11)
+    Project [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return]
+      BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+        Project [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date]
+          BroadcastHashJoin [ctr_customer_sk,c_customer_sk]
+            Project [ctr_customer_sk,ctr_total_return]
+              BroadcastHashJoin [ctr_state,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2)]
+                Filter [ctr_total_return]
+                  HashAggregate [wr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(wr_return_amt)),ctr_customer_sk,ctr_state,ctr_total_return,sum]
+                    InputAdapter
+                      Exchange [wr_returning_customer_sk,ca_state] #1
+                        WholeStageCodegen (3)
+                          HashAggregate [wr_returning_customer_sk,ca_state,wr_return_amt] [sum,sum]
+                            Project [wr_returning_customer_sk,wr_return_amt,ca_state]
+                              BroadcastHashJoin [wr_returning_addr_sk,ca_address_sk]
+                                Project [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt]
+                                  BroadcastHashJoin [wr_returned_date_sk,d_date_sk]
+                                    Filter [wr_returning_addr_sk,wr_returning_customer_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk]
+                                            SubqueryBroadcast [d_date_sk] #1
+                                              ReusedExchange [d_date_sk] #2
+                                    InputAdapter
+                                      BroadcastExchange #2
+                                        WholeStageCodegen (1)
+                                          Project [d_date_sk]
+                                            Filter [d_year,d_date_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.date_dim [d_date_sk,d_year]
+                                InputAdapter
+                                  BroadcastExchange #3
+                                    WholeStageCodegen (2)
+                                      Filter [ca_address_sk,ca_state]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.customer_address [ca_address_sk,ca_state]
+                InputAdapter
+                  BroadcastExchange #4
+                    WholeStageCodegen (8)
+                      Filter [(avg(ctr_total_return) * 1.2)]
+                        HashAggregate [ctr_state,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),ctr_state,sum,count]
+                          InputAdapter
+                            Exchange [ctr_state] #5
+                              WholeStageCodegen (7)
+                                HashAggregate [ctr_state,ctr_total_return] [sum,count,sum,count]
+                                  HashAggregate [wr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(wr_return_amt)),ctr_state,ctr_total_return,sum]
+                                    InputAdapter
+                                      Exchange [wr_returning_customer_sk,ca_state] #6
+                                        WholeStageCodegen (6)
+                                          HashAggregate [wr_returning_customer_sk,ca_state,wr_return_amt] [sum,sum]
+                                            Project [wr_returning_customer_sk,wr_return_amt,ca_state]
+                                              BroadcastHashJoin [wr_returning_addr_sk,ca_address_sk]
+                                                Project [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt]
+                                                  BroadcastHashJoin [wr_returned_date_sk,d_date_sk]
+                                                    Filter [wr_returning_addr_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk]
+                                                            ReusedSubquery [d_date_sk] #1
+                                                    InputAdapter
+                                                      ReusedExchange [d_date_sk] #2
+                                                InputAdapter
+                                                  ReusedExchange [ca_address_sk,ca_state] #3
+            InputAdapter
+              BroadcastExchange #7
+                WholeStageCodegen (9)
+                  Filter [c_customer_sk,c_current_addr_sk]
+                    ColumnarToRow
+                      InputAdapter
+                        Scan parquet default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date]
+        InputAdapter
+          BroadcastExchange #8
+            WholeStageCodegen (10)
+              Project [ca_address_sk]
+                Filter [ca_state,ca_address_sk]
+                  ColumnarToRow
+                    InputAdapter
+                      Scan parquet default.customer_address [ca_address_sk,ca_state]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.sf100/explain.txt
new file mode 100644
index 0000000000000..1703da84e7914
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.sf100/explain.txt
@@ -0,0 +1,699 @@
+== Physical Plan ==
+* Sort (119)
++- Exchange (118)
+   +- * Project (117)
+      +- * BroadcastHashJoin Inner BuildRight (116)
+         :- * Project (63)
+         :  +- * BroadcastHashJoin Inner BuildRight (62)
+         :     :- * Project (42)
+         :     :  +- * BroadcastHashJoin Inner BuildRight (41)
+         :     :     :- * HashAggregate (21)
+         :     :     :  +- Exchange (20)
+         :     :     :     +- * HashAggregate (19)
+         :     :     :        +- * Project (18)
+         :     :     :           +- * SortMergeJoin Inner (17)
+         :     :     :              :- * Sort (11)
+         :     :     :              :  +- Exchange (10)
+         :     :     :              :     +- * Project (9)
+         :     :     :              :        +- * BroadcastHashJoin Inner BuildRight (8)
+         :     :     :              :           :- * Filter (3)
+         :     :     :              :           :  +- * ColumnarToRow (2)
+         :     :     :              :           :     +- Scan parquet default.store_sales (1)
+         :     :     :              :           +- BroadcastExchange (7)
+         :     :     :              :              +- * Filter (6)
+         :     :     :              :                 +- * ColumnarToRow (5)
+         :     :     :              :                    +- Scan parquet default.date_dim (4)
+         :     :     :              +- * Sort (16)
+         :     :     :                 +- Exchange (15)
+         :     :     :                    +- * Filter (14)
+         :     :     :                       +- * ColumnarToRow (13)
+         :     :     :                          +- Scan parquet default.customer_address (12)
+         :     :     +- BroadcastExchange (40)
+         :     :        +- * HashAggregate (39)
+         :     :           +- Exchange (38)
+         :     :              +- * HashAggregate (37)
+         :     :                 +- * Project (36)
+         :     :                    +- * SortMergeJoin Inner (35)
+         :     :                       :- * Sort (32)
+         :     :                       :  +- Exchange (31)
+         :     :                       :     +- * Project (30)
+         :     :                       :        +- * BroadcastHashJoin Inner BuildRight (29)
+         :     :                       :           :- * Filter (24)
+         :     :                       :           :  +- * ColumnarToRow (23)
+         :     :                       :           :     +- Scan parquet default.store_sales (22)
+         :     :                       :           +- BroadcastExchange (28)
+         :     :                       :              +- * Filter (27)
+         :     :                       :                 +- * ColumnarToRow (26)
+         :     :                       :                    +- Scan parquet default.date_dim (25)
+         :     :                       +- * Sort (34)
+         :     :                          +- ReusedExchange (33)
+         :     +- BroadcastExchange (61)
+         :        +- * HashAggregate (60)
+         :           +- Exchange (59)
+         :              +- * HashAggregate (58)
+         :                 +- * Project (57)
+         :                    +- * SortMergeJoin Inner (56)
+         :                       :- * Sort (53)
+         :                       :  +- Exchange (52)
+         :                       :     +- * Project (51)
+         :                       :        +- * BroadcastHashJoin Inner BuildRight (50)
+         :                       :           :- * Filter (45)
+         :                       :           :  +- * ColumnarToRow (44)
+         :                       :           :     +- Scan parquet default.store_sales (43)
+         :                       :           +- BroadcastExchange (49)
+         :                       :              +- * Filter (48)
+         :                       :                 +- * ColumnarToRow (47)
+         :                       :                    +- Scan parquet default.date_dim (46)
+         :                       +- * Sort (55)
+         :                          +- ReusedExchange (54)
+         +- BroadcastExchange (115)
+            +- * Project (114)
+               +- * BroadcastHashJoin Inner BuildRight (113)
+                  :- * Project (96)
+                  :  +- * BroadcastHashJoin Inner BuildRight (95)
+                  :     :- * HashAggregate (78)
+                  :     :  +- Exchange (77)
+                  :     :     +- * HashAggregate (76)
+                  :     :        +- * Project (75)
+                  :     :           +- * SortMergeJoin Inner (74)
+                  :     :              :- * Sort (71)
+                  :     :              :  +- Exchange (70)
+                  :     :              :     +- * Project (69)
+                  :     :              :        +- * BroadcastHashJoin Inner BuildRight (68)
+                  :     :              :           :- * Filter (66)
+                  :     :              :           :  +- * ColumnarToRow (65)
+                  :     :              :           :     +- Scan parquet default.web_sales (64)
+                  :     :              :           +- ReusedExchange (67)
+                  :     :              +- * Sort (73)
+                  :     :                 +- ReusedExchange (72)
+                  :     +- BroadcastExchange (94)
+                  :        +- * HashAggregate (93)
+                  :           +- Exchange (92)
+                  :              +- * HashAggregate (91)
+                  :                 +- * Project (90)
+                  :                    +- * SortMergeJoin Inner (89)
+                  :                       :- * Sort (86)
+                  :                       :  +- Exchange (85)
+                  :                       :     +- * Project (84)
+                  :                       :        +- * BroadcastHashJoin Inner BuildRight (83)
+                  :                       :           :- * Filter (81)
+                  :                       :           :  +- * ColumnarToRow (80)
+                  :                       :           :     +- Scan parquet default.web_sales (79)
+                  :                       :           +- ReusedExchange (82)
+                  :                       +- * Sort (88)
+                  :                          +- ReusedExchange (87)
+                  +- BroadcastExchange (112)
+                     +- * HashAggregate (111)
+                        +- Exchange (110)
+                           +- * HashAggregate (109)
+                              +- * Project (108)
+                                 +- * SortMergeJoin Inner (107)
+                                    :- * Sort (104)
+                                    :  +- Exchange (103)
+                                    :     +- * Project (102)
+                                    :        +- * BroadcastHashJoin Inner BuildRight (101)
+                                    :           :- * Filter (99)
+                                    :           :  +- * ColumnarToRow (98)
+                                    :           :     +- Scan parquet default.web_sales (97)
+                                    :           +- ReusedExchange (100)
+                                    +- * Sort (106)
+                                       +- ReusedExchange (105)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_addr_sk)]
+ReadSchema: struct<ss_addr_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 2]
+Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_addr_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_qoy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+Condition : ((((isnotnull(d_qoy#7) AND isnotnull(d_year#6)) AND (d_qoy#7 = 2)) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5))
+
+(7) BroadcastExchange
+Input [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#8]
+
+(8) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#5]
+Join condition: None
+
+(9) Project [codegen id : 2]
+Output [4]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7]
+Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, d_date_sk#5, d_year#6, d_qoy#7]
+
+(10) Exchange
+Input [4]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7]
+Arguments: hashpartitioning(ss_addr_sk#1, 5), ENSURE_REQUIREMENTS, [id=#9]
+
+(11) Sort [codegen id : 3]
+Input [4]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7]
+Arguments: [ss_addr_sk#1 ASC NULLS FIRST], false, 0
+
+(12) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#10, ca_county#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)]
+ReadSchema: struct<ca_address_sk:int,ca_county:string>
+
+(13) ColumnarToRow [codegen id : 4]
+Input [2]: [ca_address_sk#10, ca_county#11]
+
+(14) Filter [codegen id : 4]
+Input [2]: [ca_address_sk#10, ca_county#11]
+Condition : (isnotnull(ca_address_sk#10) AND isnotnull(ca_county#11))
+
+(15) Exchange
+Input [2]: [ca_address_sk#10, ca_county#11]
+Arguments: hashpartitioning(ca_address_sk#10, 5), ENSURE_REQUIREMENTS, [id=#12]
+
+(16) Sort [codegen id : 5]
+Input [2]: [ca_address_sk#10, ca_county#11]
+Arguments: [ca_address_sk#10 ASC NULLS FIRST], false, 0
+
+(17) SortMergeJoin [codegen id : 6]
+Left keys [1]: [ss_addr_sk#1]
+Right keys [1]: [ca_address_sk#10]
+Join condition: None
+
+(18) Project [codegen id : 6]
+Output [4]: [ss_ext_sales_price#2, d_year#6, d_qoy#7, ca_county#11]
+Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7, ca_address_sk#10, ca_county#11]
+
+(19) HashAggregate [codegen id : 6]
+Input [4]: [ss_ext_sales_price#2, d_year#6, d_qoy#7, ca_county#11]
+Keys [3]: [ca_county#11, d_qoy#7, d_year#6]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum#13]
+Results [4]: [ca_county#11, d_qoy#7, d_year#6, sum#14]
+
+(20) Exchange
+Input [4]: [ca_county#11, d_qoy#7, d_year#6, sum#14]
+Arguments: hashpartitioning(ca_county#11, d_qoy#7, d_year#6, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(21) HashAggregate [codegen id : 42]
+Input [4]: [ca_county#11, d_qoy#7, d_year#6, sum#14]
+Keys [3]: [ca_county#11, d_qoy#7, d_year#6]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#16]
+Results [2]: [ca_county#11, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#16,17,2) AS store_sales#17]
+
+(22) Scan parquet default.store_sales
+Output [3]: [ss_addr_sk#18, ss_ext_sales_price#19, ss_sold_date_sk#20]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#20), dynamicpruningexpression(ss_sold_date_sk#20 IN dynamicpruning#21)]
+PushedFilters: [IsNotNull(ss_addr_sk)]
+ReadSchema: struct<ss_addr_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(23) ColumnarToRow [codegen id : 8]
+Input [3]: [ss_addr_sk#18, ss_ext_sales_price#19, ss_sold_date_sk#20]
+
+(24) Filter [codegen id : 8]
+Input [3]: [ss_addr_sk#18, ss_ext_sales_price#19, ss_sold_date_sk#20]
+Condition : isnotnull(ss_addr_sk#18)
+
+(25) Scan parquet default.date_dim
+Output [3]: [d_date_sk#22, d_year#23, d_qoy#24]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,3), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_qoy:int>
+
+(26) ColumnarToRow [codegen id : 7]
+Input [3]: [d_date_sk#22, d_year#23, d_qoy#24]
+
+(27) Filter [codegen id : 7]
+Input [3]: [d_date_sk#22, d_year#23, d_qoy#24]
+Condition : ((((isnotnull(d_qoy#24) AND isnotnull(d_year#23)) AND (d_qoy#24 = 3)) AND (d_year#23 = 2000)) AND isnotnull(d_date_sk#22))
+
+(28) BroadcastExchange
+Input [3]: [d_date_sk#22, d_year#23, d_qoy#24]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#25]
+
+(29) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_sold_date_sk#20]
+Right keys [1]: [d_date_sk#22]
+Join condition: None
+
+(30) Project [codegen id : 8]
+Output [4]: [ss_addr_sk#18, ss_ext_sales_price#19, d_year#23, d_qoy#24]
+Input [6]: [ss_addr_sk#18, ss_ext_sales_price#19, ss_sold_date_sk#20, d_date_sk#22, d_year#23, d_qoy#24]
+
+(31) Exchange
+Input [4]: [ss_addr_sk#18, ss_ext_sales_price#19, d_year#23, d_qoy#24]
+Arguments: hashpartitioning(ss_addr_sk#18, 5), ENSURE_REQUIREMENTS, [id=#26]
+
+(32) Sort [codegen id : 9]
+Input [4]: [ss_addr_sk#18, ss_ext_sales_price#19, d_year#23, d_qoy#24]
+Arguments: [ss_addr_sk#18 ASC NULLS FIRST], false, 0
+
+(33) ReusedExchange [Reuses operator id: 15]
+Output [2]: [ca_address_sk#27, ca_county#28]
+
+(34) Sort [codegen id : 11]
+Input [2]: [ca_address_sk#27, ca_county#28]
+Arguments: [ca_address_sk#27 ASC NULLS FIRST], false, 0
+
+(35) SortMergeJoin [codegen id : 12]
+Left keys [1]: [ss_addr_sk#18]
+Right keys [1]: [ca_address_sk#27]
+Join condition: None
+
+(36) Project [codegen id : 12]
+Output [4]: [ss_ext_sales_price#19, d_year#23, d_qoy#24, ca_county#28]
+Input [6]: [ss_addr_sk#18, ss_ext_sales_price#19, d_year#23, d_qoy#24, ca_address_sk#27, ca_county#28]
+
+(37) HashAggregate [codegen id : 12]
+Input [4]: [ss_ext_sales_price#19, d_year#23, d_qoy#24, ca_county#28]
+Keys [3]: [ca_county#28, d_qoy#24, d_year#23]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#19))]
+Aggregate Attributes [1]: [sum#29]
+Results [4]: [ca_county#28, d_qoy#24, d_year#23, sum#30]
+
+(38) Exchange
+Input [4]: [ca_county#28, d_qoy#24, d_year#23, sum#30]
+Arguments: hashpartitioning(ca_county#28, d_qoy#24, d_year#23, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(39) HashAggregate [codegen id : 13]
+Input [4]: [ca_county#28, d_qoy#24, d_year#23, sum#30]
+Keys [3]: [ca_county#28, d_qoy#24, d_year#23]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#19))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#19))#32]
+Results [2]: [ca_county#28, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#19))#32,17,2) AS store_sales#33]
+
+(40) BroadcastExchange
+Input [2]: [ca_county#28, store_sales#33]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#34]
+
+(41) BroadcastHashJoin [codegen id : 42]
+Left keys [1]: [ca_county#11]
+Right keys [1]: [ca_county#28]
+Join condition: None
+
+(42) Project [codegen id : 42]
+Output [3]: [ca_county#11, store_sales#17, store_sales#33]
+Input [4]: [ca_county#11, store_sales#17, ca_county#28, store_sales#33]
+
+(43) Scan parquet default.store_sales
+Output [3]: [ss_addr_sk#35, ss_ext_sales_price#36, ss_sold_date_sk#37]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#37), dynamicpruningexpression(ss_sold_date_sk#37 IN dynamicpruning#38)]
+PushedFilters: [IsNotNull(ss_addr_sk)]
+ReadSchema: struct<ss_addr_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(44) ColumnarToRow [codegen id : 15]
+Input [3]: [ss_addr_sk#35, ss_ext_sales_price#36, ss_sold_date_sk#37]
+
+(45) Filter [codegen id : 15]
+Input [3]: [ss_addr_sk#35, ss_ext_sales_price#36, ss_sold_date_sk#37]
+Condition : isnotnull(ss_addr_sk#35)
+
+(46) Scan parquet default.date_dim
+Output [3]: [d_date_sk#39, d_year#40, d_qoy#41]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,1), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_qoy:int>
+
+(47) ColumnarToRow [codegen id : 14]
+Input [3]: [d_date_sk#39, d_year#40, d_qoy#41]
+
+(48) Filter [codegen id : 14]
+Input [3]: [d_date_sk#39, d_year#40, d_qoy#41]
+Condition : ((((isnotnull(d_qoy#41) AND isnotnull(d_year#40)) AND (d_qoy#41 = 1)) AND (d_year#40 = 2000)) AND isnotnull(d_date_sk#39))
+
+(49) BroadcastExchange
+Input [3]: [d_date_sk#39, d_year#40, d_qoy#41]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#42]
+
+(50) BroadcastHashJoin [codegen id : 15]
+Left keys [1]: [ss_sold_date_sk#37]
+Right keys [1]: [d_date_sk#39]
+Join condition: None
+
+(51) Project [codegen id : 15]
+Output [4]: [ss_addr_sk#35, ss_ext_sales_price#36, d_year#40, d_qoy#41]
+Input [6]: [ss_addr_sk#35, ss_ext_sales_price#36, ss_sold_date_sk#37, d_date_sk#39, d_year#40, d_qoy#41]
+
+(52) Exchange
+Input [4]: [ss_addr_sk#35, ss_ext_sales_price#36, d_year#40, d_qoy#41]
+Arguments: hashpartitioning(ss_addr_sk#35, 5), ENSURE_REQUIREMENTS, [id=#43]
+
+(53) Sort [codegen id : 16]
+Input [4]: [ss_addr_sk#35, ss_ext_sales_price#36, d_year#40, d_qoy#41]
+Arguments: [ss_addr_sk#35 ASC NULLS FIRST], false, 0
+
+(54) ReusedExchange [Reuses operator id: 15]
+Output [2]: [ca_address_sk#44, ca_county#45]
+
+(55) Sort [codegen id : 18]
+Input [2]: [ca_address_sk#44, ca_county#45]
+Arguments: [ca_address_sk#44 ASC NULLS FIRST], false, 0
+
+(56) SortMergeJoin [codegen id : 19]
+Left keys [1]: [ss_addr_sk#35]
+Right keys [1]: [ca_address_sk#44]
+Join condition: None
+
+(57) Project [codegen id : 19]
+Output [4]: [ss_ext_sales_price#36, d_year#40, d_qoy#41, ca_county#45]
+Input [6]: [ss_addr_sk#35, ss_ext_sales_price#36, d_year#40, d_qoy#41, ca_address_sk#44, ca_county#45]
+
+(58) HashAggregate [codegen id : 19]
+Input [4]: [ss_ext_sales_price#36, d_year#40, d_qoy#41, ca_county#45]
+Keys [3]: [ca_county#45, d_qoy#41, d_year#40]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#36))]
+Aggregate Attributes [1]: [sum#46]
+Results [4]: [ca_county#45, d_qoy#41, d_year#40, sum#47]
+
+(59) Exchange
+Input [4]: [ca_county#45, d_qoy#41, d_year#40, sum#47]
+Arguments: hashpartitioning(ca_county#45, d_qoy#41, d_year#40, 5), ENSURE_REQUIREMENTS, [id=#48]
+
+(60) HashAggregate [codegen id : 20]
+Input [4]: [ca_county#45, d_qoy#41, d_year#40, sum#47]
+Keys [3]: [ca_county#45, d_qoy#41, d_year#40]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#36))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#36))#49]
+Results [3]: [ca_county#45, d_year#40, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#36))#49,17,2) AS store_sales#50]
+
+(61) BroadcastExchange
+Input [3]: [ca_county#45, d_year#40, store_sales#50]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#51]
+
+(62) BroadcastHashJoin [codegen id : 42]
+Left keys [1]: [ca_county#11]
+Right keys [1]: [ca_county#45]
+Join condition: None
+
+(63) Project [codegen id : 42]
+Output [5]: [store_sales#17, store_sales#33, ca_county#45, d_year#40, store_sales#50]
+Input [6]: [ca_county#11, store_sales#17, store_sales#33, ca_county#45, d_year#40, store_sales#50]
+
+(64) Scan parquet default.web_sales
+Output [3]: [ws_bill_addr_sk#52, ws_ext_sales_price#53, ws_sold_date_sk#54]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#54), dynamicpruningexpression(ws_sold_date_sk#54 IN dynamicpruning#38)]
+PushedFilters: [IsNotNull(ws_bill_addr_sk)]
+ReadSchema: struct<ws_bill_addr_sk:int,ws_ext_sales_price:decimal(7,2)>
+
+(65) ColumnarToRow [codegen id : 22]
+Input [3]: [ws_bill_addr_sk#52, ws_ext_sales_price#53, ws_sold_date_sk#54]
+
+(66) Filter [codegen id : 22]
+Input [3]: [ws_bill_addr_sk#52, ws_ext_sales_price#53, ws_sold_date_sk#54]
+Condition : isnotnull(ws_bill_addr_sk#52)
+
+(67) ReusedExchange [Reuses operator id: 49]
+Output [3]: [d_date_sk#55, d_year#56, d_qoy#57]
+
+(68) BroadcastHashJoin [codegen id : 22]
+Left keys [1]: [ws_sold_date_sk#54]
+Right keys [1]: [d_date_sk#55]
+Join condition: None
+
+(69) Project [codegen id : 22]
+Output [4]: [ws_bill_addr_sk#52, ws_ext_sales_price#53, d_year#56, d_qoy#57]
+Input [6]: [ws_bill_addr_sk#52, ws_ext_sales_price#53, ws_sold_date_sk#54, d_date_sk#55, d_year#56, d_qoy#57]
+
+(70) Exchange
+Input [4]: [ws_bill_addr_sk#52, ws_ext_sales_price#53, d_year#56, d_qoy#57]
+Arguments: hashpartitioning(ws_bill_addr_sk#52, 5), ENSURE_REQUIREMENTS, [id=#58]
+
+(71) Sort [codegen id : 23]
+Input [4]: [ws_bill_addr_sk#52, ws_ext_sales_price#53, d_year#56, d_qoy#57]
+Arguments: [ws_bill_addr_sk#52 ASC NULLS FIRST], false, 0
+
+(72) ReusedExchange [Reuses operator id: 15]
+Output [2]: [ca_address_sk#59, ca_county#60]
+
+(73) Sort [codegen id : 25]
+Input [2]: [ca_address_sk#59, ca_county#60]
+Arguments: [ca_address_sk#59 ASC NULLS FIRST], false, 0
+
+(74) SortMergeJoin [codegen id : 26]
+Left keys [1]: [ws_bill_addr_sk#52]
+Right keys [1]: [ca_address_sk#59]
+Join condition: None
+
+(75) Project [codegen id : 26]
+Output [4]: [ws_ext_sales_price#53, d_year#56, d_qoy#57, ca_county#60]
+Input [6]: [ws_bill_addr_sk#52, ws_ext_sales_price#53, d_year#56, d_qoy#57, ca_address_sk#59, ca_county#60]
+
+(76) HashAggregate [codegen id : 26]
+Input [4]: [ws_ext_sales_price#53, d_year#56, d_qoy#57, ca_county#60]
+Keys [3]: [ca_county#60, d_qoy#57, d_year#56]
+Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#53))]
+Aggregate Attributes [1]: [sum#61]
+Results [4]: [ca_county#60, d_qoy#57, d_year#56, sum#62]
+
+(77) Exchange
+Input [4]: [ca_county#60, d_qoy#57, d_year#56, sum#62]
+Arguments: hashpartitioning(ca_county#60, d_qoy#57, d_year#56, 5), ENSURE_REQUIREMENTS, [id=#63]
+
+(78) HashAggregate [codegen id : 41]
+Input [4]: [ca_county#60, d_qoy#57, d_year#56, sum#62]
+Keys [3]: [ca_county#60, d_qoy#57, d_year#56]
+Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#53))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#53))#64]
+Results [2]: [ca_county#60, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#53))#64,17,2) AS web_sales#65]
+
+(79) Scan parquet default.web_sales
+Output [3]: [ws_bill_addr_sk#66, ws_ext_sales_price#67, ws_sold_date_sk#68]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#68), dynamicpruningexpression(ws_sold_date_sk#68 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ws_bill_addr_sk)]
+ReadSchema: struct<ws_bill_addr_sk:int,ws_ext_sales_price:decimal(7,2)>
+
+(80) ColumnarToRow [codegen id : 28]
+Input [3]: [ws_bill_addr_sk#66, ws_ext_sales_price#67, ws_sold_date_sk#68]
+
+(81) Filter [codegen id : 28]
+Input [3]: [ws_bill_addr_sk#66, ws_ext_sales_price#67, ws_sold_date_sk#68]
+Condition : isnotnull(ws_bill_addr_sk#66)
+
+(82) ReusedExchange [Reuses operator id: 7]
+Output [3]: [d_date_sk#69, d_year#70, d_qoy#71]
+
+(83) BroadcastHashJoin [codegen id : 28]
+Left keys [1]: [ws_sold_date_sk#68]
+Right keys [1]: [d_date_sk#69]
+Join condition: None
+
+(84) Project [codegen id : 28]
+Output [4]: [ws_bill_addr_sk#66, ws_ext_sales_price#67, d_year#70, d_qoy#71]
+Input [6]: [ws_bill_addr_sk#66, ws_ext_sales_price#67, ws_sold_date_sk#68, d_date_sk#69, d_year#70, d_qoy#71]
+
+(85) Exchange
+Input [4]: [ws_bill_addr_sk#66, ws_ext_sales_price#67, d_year#70, d_qoy#71]
+Arguments: hashpartitioning(ws_bill_addr_sk#66, 5), ENSURE_REQUIREMENTS, [id=#72]
+
+(86) Sort [codegen id : 29]
+Input [4]: [ws_bill_addr_sk#66, ws_ext_sales_price#67, d_year#70, d_qoy#71]
+Arguments: [ws_bill_addr_sk#66 ASC NULLS FIRST], false, 0
+
+(87) ReusedExchange [Reuses operator id: 15]
+Output [2]: [ca_address_sk#73, ca_county#74]
+
+(88) Sort [codegen id : 31]
+Input [2]: [ca_address_sk#73, ca_county#74]
+Arguments: [ca_address_sk#73 ASC NULLS FIRST], false, 0
+
+(89) SortMergeJoin [codegen id : 32]
+Left keys [1]: [ws_bill_addr_sk#66]
+Right keys [1]: [ca_address_sk#73]
+Join condition: None
+
+(90) Project [codegen id : 32]
+Output [4]: [ws_ext_sales_price#67, d_year#70, d_qoy#71, ca_county#74]
+Input [6]: [ws_bill_addr_sk#66, ws_ext_sales_price#67, d_year#70, d_qoy#71, ca_address_sk#73, ca_county#74]
+
+(91) HashAggregate [codegen id : 32]
+Input [4]: [ws_ext_sales_price#67, d_year#70, d_qoy#71, ca_county#74]
+Keys [3]: [ca_county#74, d_qoy#71, d_year#70]
+Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#67))]
+Aggregate Attributes [1]: [sum#75]
+Results [4]: [ca_county#74, d_qoy#71, d_year#70, sum#76]
+
+(92) Exchange
+Input [4]: [ca_county#74, d_qoy#71, d_year#70, sum#76]
+Arguments: hashpartitioning(ca_county#74, d_qoy#71, d_year#70, 5), ENSURE_REQUIREMENTS, [id=#77]
+
+(93) HashAggregate [codegen id : 33]
+Input [4]: [ca_county#74, d_qoy#71, d_year#70, sum#76]
+Keys [3]: [ca_county#74, d_qoy#71, d_year#70]
+Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#67))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#67))#78]
+Results [2]: [ca_county#74, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#67))#78,17,2) AS web_sales#79]
+
+(94) BroadcastExchange
+Input [2]: [ca_county#74, web_sales#79]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#80]
+
+(95) BroadcastHashJoin [codegen id : 41]
+Left keys [1]: [ca_county#60]
+Right keys [1]: [ca_county#74]
+Join condition: None
+
+(96) Project [codegen id : 41]
+Output [3]: [ca_county#60, web_sales#65, web_sales#79]
+Input [4]: [ca_county#60, web_sales#65, ca_county#74, web_sales#79]
+
+(97) Scan parquet default.web_sales
+Output [3]: [ws_bill_addr_sk#81, ws_ext_sales_price#82, ws_sold_date_sk#83]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#83), dynamicpruningexpression(ws_sold_date_sk#83 IN dynamicpruning#21)]
+PushedFilters: [IsNotNull(ws_bill_addr_sk)]
+ReadSchema: struct<ws_bill_addr_sk:int,ws_ext_sales_price:decimal(7,2)>
+
+(98) ColumnarToRow [codegen id : 35]
+Input [3]: [ws_bill_addr_sk#81, ws_ext_sales_price#82, ws_sold_date_sk#83]
+
+(99) Filter [codegen id : 35]
+Input [3]: [ws_bill_addr_sk#81, ws_ext_sales_price#82, ws_sold_date_sk#83]
+Condition : isnotnull(ws_bill_addr_sk#81)
+
+(100) ReusedExchange [Reuses operator id: 28]
+Output [3]: [d_date_sk#84, d_year#85, d_qoy#86]
+
+(101) BroadcastHashJoin [codegen id : 35]
+Left keys [1]: [ws_sold_date_sk#83]
+Right keys [1]: [d_date_sk#84]
+Join condition: None
+
+(102) Project [codegen id : 35]
+Output [4]: [ws_bill_addr_sk#81, ws_ext_sales_price#82, d_year#85, d_qoy#86]
+Input [6]: [ws_bill_addr_sk#81, ws_ext_sales_price#82, ws_sold_date_sk#83, d_date_sk#84, d_year#85, d_qoy#86]
+
+(103) Exchange
+Input [4]: [ws_bill_addr_sk#81, ws_ext_sales_price#82, d_year#85, d_qoy#86]
+Arguments: hashpartitioning(ws_bill_addr_sk#81, 5), ENSURE_REQUIREMENTS, [id=#87]
+
+(104) Sort [codegen id : 36]
+Input [4]: [ws_bill_addr_sk#81, ws_ext_sales_price#82, d_year#85, d_qoy#86]
+Arguments: [ws_bill_addr_sk#81 ASC NULLS FIRST], false, 0
+
+(105) ReusedExchange [Reuses operator id: 15]
+Output [2]: [ca_address_sk#88, ca_county#89]
+
+(106) Sort [codegen id : 38]
+Input [2]: [ca_address_sk#88, ca_county#89]
+Arguments: [ca_address_sk#88 ASC NULLS FIRST], false, 0
+
+(107) SortMergeJoin [codegen id : 39]
+Left keys [1]: [ws_bill_addr_sk#81]
+Right keys [1]: [ca_address_sk#88]
+Join condition: None
+
+(108) Project [codegen id : 39]
+Output [4]: [ws_ext_sales_price#82, d_year#85, d_qoy#86, ca_county#89]
+Input [6]: [ws_bill_addr_sk#81, ws_ext_sales_price#82, d_year#85, d_qoy#86, ca_address_sk#88, ca_county#89]
+
+(109) HashAggregate [codegen id : 39]
+Input [4]: [ws_ext_sales_price#82, d_year#85, d_qoy#86, ca_county#89]
+Keys [3]: [ca_county#89, d_qoy#86, d_year#85]
+Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#82))]
+Aggregate Attributes [1]: [sum#90]
+Results [4]: [ca_county#89, d_qoy#86, d_year#85, sum#91]
+
+(110) Exchange
+Input [4]: [ca_county#89, d_qoy#86, d_year#85, sum#91]
+Arguments: hashpartitioning(ca_county#89, d_qoy#86, d_year#85, 5), ENSURE_REQUIREMENTS, [id=#92]
+
+(111) HashAggregate [codegen id : 40]
+Input [4]: [ca_county#89, d_qoy#86, d_year#85, sum#91]
+Keys [3]: [ca_county#89, d_qoy#86, d_year#85]
+Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#82))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#82))#93]
+Results [2]: [ca_county#89, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#82))#93,17,2) AS web_sales#94]
+
+(112) BroadcastExchange
+Input [2]: [ca_county#89, web_sales#94]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#95]
+
+(113) BroadcastHashJoin [codegen id : 41]
+Left keys [1]: [ca_county#60]
+Right keys [1]: [ca_county#89]
+Join condition: None
+
+(114) Project [codegen id : 41]
+Output [4]: [ca_county#60, web_sales#65, web_sales#79, web_sales#94]
+Input [5]: [ca_county#60, web_sales#65, web_sales#79, ca_county#89, web_sales#94]
+
+(115) BroadcastExchange
+Input [4]: [ca_county#60, web_sales#65, web_sales#79, web_sales#94]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#96]
+
+(116) BroadcastHashJoin [codegen id : 42]
+Left keys [1]: [ca_county#45]
+Right keys [1]: [ca_county#60]
+Join condition: ((CASE WHEN (web_sales#65 > 0.00) THEN CheckOverflow((promote_precision(web_sales#79) / promote_precision(web_sales#65)), DecimalType(37,20), true) ELSE null END > CASE WHEN (store_sales#50 > 0.00) THEN CheckOverflow((promote_precision(store_sales#17) / promote_precision(store_sales#50)), DecimalType(37,20), true) ELSE null END) AND (CASE WHEN (web_sales#79 > 0.00) THEN CheckOverflow((promote_precision(web_sales#94) / promote_precision(web_sales#79)), DecimalType(37,20), true) ELSE null END > CASE WHEN (store_sales#17 > 0.00) THEN CheckOverflow((promote_precision(store_sales#33) / promote_precision(store_sales#17)), DecimalType(37,20), true) ELSE null END))
+
+(117) Project [codegen id : 42]
+Output [6]: [ca_county#45, d_year#40, CheckOverflow((promote_precision(web_sales#79) / promote_precision(web_sales#65)), DecimalType(37,20), true) AS web_q1_q2_increase#97, CheckOverflow((promote_precision(store_sales#17) / promote_precision(store_sales#50)), DecimalType(37,20), true) AS store_q1_q2_increase#98, CheckOverflow((promote_precision(web_sales#94) / promote_precision(web_sales#79)), DecimalType(37,20), true) AS web_q2_q3_increase#99, CheckOverflow((promote_precision(store_sales#33) / promote_precision(store_sales#17)), DecimalType(37,20), true) AS store_q2_q3_increase#100]
+Input [9]: [store_sales#17, store_sales#33, ca_county#45, d_year#40, store_sales#50, ca_county#60, web_sales#65, web_sales#79, web_sales#94]
+
+(118) Exchange
+Input [6]: [ca_county#45, d_year#40, web_q1_q2_increase#97, store_q1_q2_increase#98, web_q2_q3_increase#99, store_q2_q3_increase#100]
+Arguments: rangepartitioning(ca_county#45 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [id=#101]
+
+(119) Sort [codegen id : 43]
+Input [6]: [ca_county#45, d_year#40, web_q1_q2_increase#97, store_q1_q2_increase#98, web_q2_q3_increase#99, store_q2_q3_increase#100]
+Arguments: [ca_county#45 ASC NULLS FIRST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (120)
+
+
+(120) ReusedExchange [Reuses operator id: 7]
+Output [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+
+Subquery:2 Hosting operator id = 22 Hosting Expression = ss_sold_date_sk#20 IN dynamicpruning#21
+ReusedExchange (121)
+
+
+(121) ReusedExchange [Reuses operator id: 28]
+Output [3]: [d_date_sk#22, d_year#23, d_qoy#24]
+
+Subquery:3 Hosting operator id = 43 Hosting Expression = ss_sold_date_sk#37 IN dynamicpruning#38
+ReusedExchange (122)
+
+
+(122) ReusedExchange [Reuses operator id: 49]
+Output [3]: [d_date_sk#39, d_year#40, d_qoy#41]
+
+Subquery:4 Hosting operator id = 64 Hosting Expression = ws_sold_date_sk#54 IN dynamicpruning#38
+
+Subquery:5 Hosting operator id = 79 Hosting Expression = ws_sold_date_sk#68 IN dynamicpruning#4
+
+Subquery:6 Hosting operator id = 97 Hosting Expression = ws_sold_date_sk#83 IN dynamicpruning#21
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.sf100/simplified.txt
new file mode 100644
index 0000000000000..0f83c5b264666
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.sf100/simplified.txt
@@ -0,0 +1,215 @@
+WholeStageCodegen (43)
+  Sort [ca_county]
+    InputAdapter
+      Exchange [ca_county] #1
+        WholeStageCodegen (42)
+          Project [ca_county,d_year,web_sales,web_sales,store_sales,store_sales,web_sales,store_sales]
+            BroadcastHashJoin [ca_county,ca_county,web_sales,web_sales,store_sales,store_sales,web_sales,store_sales]
+              Project [store_sales,store_sales,ca_county,d_year,store_sales]
+                BroadcastHashJoin [ca_county,ca_county]
+                  Project [ca_county,store_sales,store_sales]
+                    BroadcastHashJoin [ca_county,ca_county]
+                      HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum]
+                        InputAdapter
+                          Exchange [ca_county,d_qoy,d_year] #2
+                            WholeStageCodegen (6)
+                              HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum]
+                                Project [ss_ext_sales_price,d_year,d_qoy,ca_county]
+                                  SortMergeJoin [ss_addr_sk,ca_address_sk]
+                                    InputAdapter
+                                      WholeStageCodegen (3)
+                                        Sort [ss_addr_sk]
+                                          InputAdapter
+                                            Exchange [ss_addr_sk] #3
+                                              WholeStageCodegen (2)
+                                                Project [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy]
+                                                  BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                    Filter [ss_addr_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                                            SubqueryBroadcast [d_date_sk] #1
+                                                              ReusedExchange [d_date_sk,d_year,d_qoy] #4
+                                                    InputAdapter
+                                                      BroadcastExchange #4
+                                                        WholeStageCodegen (1)
+                                                          Filter [d_qoy,d_year,d_date_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.date_dim [d_date_sk,d_year,d_qoy]
+                                    InputAdapter
+                                      WholeStageCodegen (5)
+                                        Sort [ca_address_sk]
+                                          InputAdapter
+                                            Exchange [ca_address_sk] #5
+                                              WholeStageCodegen (4)
+                                                Filter [ca_address_sk,ca_county]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.customer_address [ca_address_sk,ca_county]
+                      InputAdapter
+                        BroadcastExchange #6
+                          WholeStageCodegen (13)
+                            HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum]
+                              InputAdapter
+                                Exchange [ca_county,d_qoy,d_year] #7
+                                  WholeStageCodegen (12)
+                                    HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum]
+                                      Project [ss_ext_sales_price,d_year,d_qoy,ca_county]
+                                        SortMergeJoin [ss_addr_sk,ca_address_sk]
+                                          InputAdapter
+                                            WholeStageCodegen (9)
+                                              Sort [ss_addr_sk]
+                                                InputAdapter
+                                                  Exchange [ss_addr_sk] #8
+                                                    WholeStageCodegen (8)
+                                                      Project [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy]
+                                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                          Filter [ss_addr_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                                                  SubqueryBroadcast [d_date_sk] #2
+                                                                    ReusedExchange [d_date_sk,d_year,d_qoy] #9
+                                                          InputAdapter
+                                                            BroadcastExchange #9
+                                                              WholeStageCodegen (7)
+                                                                Filter [d_qoy,d_year,d_date_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.date_dim [d_date_sk,d_year,d_qoy]
+                                          InputAdapter
+                                            WholeStageCodegen (11)
+                                              Sort [ca_address_sk]
+                                                InputAdapter
+                                                  ReusedExchange [ca_address_sk,ca_county] #5
+                  InputAdapter
+                    BroadcastExchange #10
+                      WholeStageCodegen (20)
+                        HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum]
+                          InputAdapter
+                            Exchange [ca_county,d_qoy,d_year] #11
+                              WholeStageCodegen (19)
+                                HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum]
+                                  Project [ss_ext_sales_price,d_year,d_qoy,ca_county]
+                                    SortMergeJoin [ss_addr_sk,ca_address_sk]
+                                      InputAdapter
+                                        WholeStageCodegen (16)
+                                          Sort [ss_addr_sk]
+                                            InputAdapter
+                                              Exchange [ss_addr_sk] #12
+                                                WholeStageCodegen (15)
+                                                  Project [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy]
+                                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                      Filter [ss_addr_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                                              SubqueryBroadcast [d_date_sk] #3
+                                                                ReusedExchange [d_date_sk,d_year,d_qoy] #13
+                                                      InputAdapter
+                                                        BroadcastExchange #13
+                                                          WholeStageCodegen (14)
+                                                            Filter [d_qoy,d_year,d_date_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.date_dim [d_date_sk,d_year,d_qoy]
+                                      InputAdapter
+                                        WholeStageCodegen (18)
+                                          Sort [ca_address_sk]
+                                            InputAdapter
+                                              ReusedExchange [ca_address_sk,ca_county] #5
+              InputAdapter
+                BroadcastExchange #14
+                  WholeStageCodegen (41)
+                    Project [ca_county,web_sales,web_sales,web_sales]
+                      BroadcastHashJoin [ca_county,ca_county]
+                        Project [ca_county,web_sales,web_sales]
+                          BroadcastHashJoin [ca_county,ca_county]
+                            HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum]
+                              InputAdapter
+                                Exchange [ca_county,d_qoy,d_year] #15
+                                  WholeStageCodegen (26)
+                                    HashAggregate [ca_county,d_qoy,d_year,ws_ext_sales_price] [sum,sum]
+                                      Project [ws_ext_sales_price,d_year,d_qoy,ca_county]
+                                        SortMergeJoin [ws_bill_addr_sk,ca_address_sk]
+                                          InputAdapter
+                                            WholeStageCodegen (23)
+                                              Sort [ws_bill_addr_sk]
+                                                InputAdapter
+                                                  Exchange [ws_bill_addr_sk] #16
+                                                    WholeStageCodegen (22)
+                                                      Project [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy]
+                                                        BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                          Filter [ws_bill_addr_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk]
+                                                                  ReusedSubquery [d_date_sk] #3
+                                                          InputAdapter
+                                                            ReusedExchange [d_date_sk,d_year,d_qoy] #13
+                                          InputAdapter
+                                            WholeStageCodegen (25)
+                                              Sort [ca_address_sk]
+                                                InputAdapter
+                                                  ReusedExchange [ca_address_sk,ca_county] #5
+                            InputAdapter
+                              BroadcastExchange #17
+                                WholeStageCodegen (33)
+                                  HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum]
+                                    InputAdapter
+                                      Exchange [ca_county,d_qoy,d_year] #18
+                                        WholeStageCodegen (32)
+                                          HashAggregate [ca_county,d_qoy,d_year,ws_ext_sales_price] [sum,sum]
+                                            Project [ws_ext_sales_price,d_year,d_qoy,ca_county]
+                                              SortMergeJoin [ws_bill_addr_sk,ca_address_sk]
+                                                InputAdapter
+                                                  WholeStageCodegen (29)
+                                                    Sort [ws_bill_addr_sk]
+                                                      InputAdapter
+                                                        Exchange [ws_bill_addr_sk] #19
+                                                          WholeStageCodegen (28)
+                                                            Project [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy]
+                                                              BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                Filter [ws_bill_addr_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk]
+                                                                        ReusedSubquery [d_date_sk] #1
+                                                                InputAdapter
+                                                                  ReusedExchange [d_date_sk,d_year,d_qoy] #4
+                                                InputAdapter
+                                                  WholeStageCodegen (31)
+                                                    Sort [ca_address_sk]
+                                                      InputAdapter
+                                                        ReusedExchange [ca_address_sk,ca_county] #5
+                        InputAdapter
+                          BroadcastExchange #20
+                            WholeStageCodegen (40)
+                              HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum]
+                                InputAdapter
+                                  Exchange [ca_county,d_qoy,d_year] #21
+                                    WholeStageCodegen (39)
+                                      HashAggregate [ca_county,d_qoy,d_year,ws_ext_sales_price] [sum,sum]
+                                        Project [ws_ext_sales_price,d_year,d_qoy,ca_county]
+                                          SortMergeJoin [ws_bill_addr_sk,ca_address_sk]
+                                            InputAdapter
+                                              WholeStageCodegen (36)
+                                                Sort [ws_bill_addr_sk]
+                                                  InputAdapter
+                                                    Exchange [ws_bill_addr_sk] #22
+                                                      WholeStageCodegen (35)
+                                                        Project [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy]
+                                                          BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                            Filter [ws_bill_addr_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk]
+                                                                    ReusedSubquery [d_date_sk] #2
+                                                            InputAdapter
+                                                              ReusedExchange [d_date_sk,d_year,d_qoy] #9
+                                            InputAdapter
+                                              WholeStageCodegen (38)
+                                                Sort [ca_address_sk]
+                                                  InputAdapter
+                                                    ReusedExchange [ca_address_sk,ca_county] #5
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/explain.txt
new file mode 100644
index 0000000000000..f47a7af5a15e3
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/explain.txt
@@ -0,0 +1,599 @@
+== Physical Plan ==
+* Sort (99)
++- Exchange (98)
+   +- * Project (97)
+      +- * BroadcastHashJoin Inner BuildRight (96)
+         :- * Project (82)
+         :  +- * BroadcastHashJoin Inner BuildRight (81)
+         :     :- * BroadcastHashJoin Inner BuildRight (67)
+         :     :  :- * Project (53)
+         :     :  :  +- * BroadcastHashJoin Inner BuildRight (52)
+         :     :  :     :- * BroadcastHashJoin Inner BuildRight (35)
+         :     :  :     :  :- * HashAggregate (18)
+         :     :  :     :  :  +- Exchange (17)
+         :     :  :     :  :     +- * HashAggregate (16)
+         :     :  :     :  :        +- * Project (15)
+         :     :  :     :  :           +- * BroadcastHashJoin Inner BuildRight (14)
+         :     :  :     :  :              :- * Project (9)
+         :     :  :     :  :              :  +- * BroadcastHashJoin Inner BuildRight (8)
+         :     :  :     :  :              :     :- * Filter (3)
+         :     :  :     :  :              :     :  +- * ColumnarToRow (2)
+         :     :  :     :  :              :     :     +- Scan parquet default.store_sales (1)
+         :     :  :     :  :              :     +- BroadcastExchange (7)
+         :     :  :     :  :              :        +- * Filter (6)
+         :     :  :     :  :              :           +- * ColumnarToRow (5)
+         :     :  :     :  :              :              +- Scan parquet default.date_dim (4)
+         :     :  :     :  :              +- BroadcastExchange (13)
+         :     :  :     :  :                 +- * Filter (12)
+         :     :  :     :  :                    +- * ColumnarToRow (11)
+         :     :  :     :  :                       +- Scan parquet default.customer_address (10)
+         :     :  :     :  +- BroadcastExchange (34)
+         :     :  :     :     +- * HashAggregate (33)
+         :     :  :     :        +- Exchange (32)
+         :     :  :     :           +- * HashAggregate (31)
+         :     :  :     :              +- * Project (30)
+         :     :  :     :                 +- * BroadcastHashJoin Inner BuildRight (29)
+         :     :  :     :                    :- * Project (27)
+         :     :  :     :                    :  +- * BroadcastHashJoin Inner BuildRight (26)
+         :     :  :     :                    :     :- * Filter (21)
+         :     :  :     :                    :     :  +- * ColumnarToRow (20)
+         :     :  :     :                    :     :     +- Scan parquet default.store_sales (19)
+         :     :  :     :                    :     +- BroadcastExchange (25)
+         :     :  :     :                    :        +- * Filter (24)
+         :     :  :     :                    :           +- * ColumnarToRow (23)
+         :     :  :     :                    :              +- Scan parquet default.date_dim (22)
+         :     :  :     :                    +- ReusedExchange (28)
+         :     :  :     +- BroadcastExchange (51)
+         :     :  :        +- * HashAggregate (50)
+         :     :  :           +- Exchange (49)
+         :     :  :              +- * HashAggregate (48)
+         :     :  :                 +- * Project (47)
+         :     :  :                    +- * BroadcastHashJoin Inner BuildRight (46)
+         :     :  :                       :- * Project (44)
+         :     :  :                       :  +- * BroadcastHashJoin Inner BuildRight (43)
+         :     :  :                       :     :- * Filter (38)
+         :     :  :                       :     :  +- * ColumnarToRow (37)
+         :     :  :                       :     :     +- Scan parquet default.store_sales (36)
+         :     :  :                       :     +- BroadcastExchange (42)
+         :     :  :                       :        +- * Filter (41)
+         :     :  :                       :           +- * ColumnarToRow (40)
+         :     :  :                       :              +- Scan parquet default.date_dim (39)
+         :     :  :                       +- ReusedExchange (45)
+         :     :  +- BroadcastExchange (66)
+         :     :     +- * HashAggregate (65)
+         :     :        +- Exchange (64)
+         :     :           +- * HashAggregate (63)
+         :     :              +- * Project (62)
+         :     :                 +- * BroadcastHashJoin Inner BuildRight (61)
+         :     :                    :- * Project (59)
+         :     :                    :  +- * BroadcastHashJoin Inner BuildRight (58)
+         :     :                    :     :- * Filter (56)
+         :     :                    :     :  +- * ColumnarToRow (55)
+         :     :                    :     :     +- Scan parquet default.web_sales (54)
+         :     :                    :     +- ReusedExchange (57)
+         :     :                    +- ReusedExchange (60)
+         :     +- BroadcastExchange (80)
+         :        +- * HashAggregate (79)
+         :           +- Exchange (78)
+         :              +- * HashAggregate (77)
+         :                 +- * Project (76)
+         :                    +- * BroadcastHashJoin Inner BuildRight (75)
+         :                       :- * Project (73)
+         :                       :  +- * BroadcastHashJoin Inner BuildRight (72)
+         :                       :     :- * Filter (70)
+         :                       :     :  +- * ColumnarToRow (69)
+         :                       :     :     +- Scan parquet default.web_sales (68)
+         :                       :     +- ReusedExchange (71)
+         :                       +- ReusedExchange (74)
+         +- BroadcastExchange (95)
+            +- * HashAggregate (94)
+               +- Exchange (93)
+                  +- * HashAggregate (92)
+                     +- * Project (91)
+                        +- * BroadcastHashJoin Inner BuildRight (90)
+                           :- * Project (88)
+                           :  +- * BroadcastHashJoin Inner BuildRight (87)
+                           :     :- * Filter (85)
+                           :     :  +- * ColumnarToRow (84)
+                           :     :     +- Scan parquet default.web_sales (83)
+                           :     +- ReusedExchange (86)
+                           +- ReusedExchange (89)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_addr_sk)]
+ReadSchema: struct<ss_addr_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_addr_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,1), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_qoy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+Condition : ((((isnotnull(d_qoy#7) AND isnotnull(d_year#6)) AND (d_qoy#7 = 1)) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5))
+
+(7) BroadcastExchange
+Input [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#8]
+
+(8) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#5]
+Join condition: None
+
+(9) Project [codegen id : 3]
+Output [4]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7]
+Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, d_date_sk#5, d_year#6, d_qoy#7]
+
+(10) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#9, ca_county#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)]
+ReadSchema: struct<ca_address_sk:int,ca_county:string>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [ca_address_sk#9, ca_county#10]
+
+(12) Filter [codegen id : 2]
+Input [2]: [ca_address_sk#9, ca_county#10]
+Condition : (isnotnull(ca_address_sk#9) AND isnotnull(ca_county#10))
+
+(13) BroadcastExchange
+Input [2]: [ca_address_sk#9, ca_county#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(14) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_addr_sk#1]
+Right keys [1]: [ca_address_sk#9]
+Join condition: None
+
+(15) Project [codegen id : 3]
+Output [4]: [ss_ext_sales_price#2, d_year#6, d_qoy#7, ca_county#10]
+Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7, ca_address_sk#9, ca_county#10]
+
+(16) HashAggregate [codegen id : 3]
+Input [4]: [ss_ext_sales_price#2, d_year#6, d_qoy#7, ca_county#10]
+Keys [3]: [ca_county#10, d_qoy#7, d_year#6]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum#12]
+Results [4]: [ca_county#10, d_qoy#7, d_year#6, sum#13]
+
+(17) Exchange
+Input [4]: [ca_county#10, d_qoy#7, d_year#6, sum#13]
+Arguments: hashpartitioning(ca_county#10, d_qoy#7, d_year#6, 5), ENSURE_REQUIREMENTS, [id=#14]
+
+(18) HashAggregate [codegen id : 24]
+Input [4]: [ca_county#10, d_qoy#7, d_year#6, sum#13]
+Keys [3]: [ca_county#10, d_qoy#7, d_year#6]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#15]
+Results [3]: [ca_county#10, d_year#6, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#15,17,2) AS store_sales#16]
+
+(19) Scan parquet default.store_sales
+Output [3]: [ss_addr_sk#17, ss_ext_sales_price#18, ss_sold_date_sk#19]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#19), dynamicpruningexpression(ss_sold_date_sk#19 IN dynamicpruning#20)]
+PushedFilters: [IsNotNull(ss_addr_sk)]
+ReadSchema: struct<ss_addr_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(20) ColumnarToRow [codegen id : 6]
+Input [3]: [ss_addr_sk#17, ss_ext_sales_price#18, ss_sold_date_sk#19]
+
+(21) Filter [codegen id : 6]
+Input [3]: [ss_addr_sk#17, ss_ext_sales_price#18, ss_sold_date_sk#19]
+Condition : isnotnull(ss_addr_sk#17)
+
+(22) Scan parquet default.date_dim
+Output [3]: [d_date_sk#21, d_year#22, d_qoy#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_qoy:int>
+
+(23) ColumnarToRow [codegen id : 4]
+Input [3]: [d_date_sk#21, d_year#22, d_qoy#23]
+
+(24) Filter [codegen id : 4]
+Input [3]: [d_date_sk#21, d_year#22, d_qoy#23]
+Condition : ((((isnotnull(d_qoy#23) AND isnotnull(d_year#22)) AND (d_qoy#23 = 2)) AND (d_year#22 = 2000)) AND isnotnull(d_date_sk#21))
+
+(25) BroadcastExchange
+Input [3]: [d_date_sk#21, d_year#22, d_qoy#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#24]
+
+(26) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_sold_date_sk#19]
+Right keys [1]: [d_date_sk#21]
+Join condition: None
+
+(27) Project [codegen id : 6]
+Output [4]: [ss_addr_sk#17, ss_ext_sales_price#18, d_year#22, d_qoy#23]
+Input [6]: [ss_addr_sk#17, ss_ext_sales_price#18, ss_sold_date_sk#19, d_date_sk#21, d_year#22, d_qoy#23]
+
+(28) ReusedExchange [Reuses operator id: 13]
+Output [2]: [ca_address_sk#25, ca_county#26]
+
+(29) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_addr_sk#17]
+Right keys [1]: [ca_address_sk#25]
+Join condition: None
+
+(30) Project [codegen id : 6]
+Output [4]: [ss_ext_sales_price#18, d_year#22, d_qoy#23, ca_county#26]
+Input [6]: [ss_addr_sk#17, ss_ext_sales_price#18, d_year#22, d_qoy#23, ca_address_sk#25, ca_county#26]
+
+(31) HashAggregate [codegen id : 6]
+Input [4]: [ss_ext_sales_price#18, d_year#22, d_qoy#23, ca_county#26]
+Keys [3]: [ca_county#26, d_qoy#23, d_year#22]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#18))]
+Aggregate Attributes [1]: [sum#27]
+Results [4]: [ca_county#26, d_qoy#23, d_year#22, sum#28]
+
+(32) Exchange
+Input [4]: [ca_county#26, d_qoy#23, d_year#22, sum#28]
+Arguments: hashpartitioning(ca_county#26, d_qoy#23, d_year#22, 5), ENSURE_REQUIREMENTS, [id=#29]
+
+(33) HashAggregate [codegen id : 7]
+Input [4]: [ca_county#26, d_qoy#23, d_year#22, sum#28]
+Keys [3]: [ca_county#26, d_qoy#23, d_year#22]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#18))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#18))#30]
+Results [2]: [ca_county#26, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#18))#30,17,2) AS store_sales#31]
+
+(34) BroadcastExchange
+Input [2]: [ca_county#26, store_sales#31]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#32]
+
+(35) BroadcastHashJoin [codegen id : 24]
+Left keys [1]: [ca_county#10]
+Right keys [1]: [ca_county#26]
+Join condition: None
+
+(36) Scan parquet default.store_sales
+Output [3]: [ss_addr_sk#33, ss_ext_sales_price#34, ss_sold_date_sk#35]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#35), dynamicpruningexpression(ss_sold_date_sk#35 IN dynamicpruning#36)]
+PushedFilters: [IsNotNull(ss_addr_sk)]
+ReadSchema: struct<ss_addr_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(37) ColumnarToRow [codegen id : 10]
+Input [3]: [ss_addr_sk#33, ss_ext_sales_price#34, ss_sold_date_sk#35]
+
+(38) Filter [codegen id : 10]
+Input [3]: [ss_addr_sk#33, ss_ext_sales_price#34, ss_sold_date_sk#35]
+Condition : isnotnull(ss_addr_sk#33)
+
+(39) Scan parquet default.date_dim
+Output [3]: [d_date_sk#37, d_year#38, d_qoy#39]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,3), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_qoy:int>
+
+(40) ColumnarToRow [codegen id : 8]
+Input [3]: [d_date_sk#37, d_year#38, d_qoy#39]
+
+(41) Filter [codegen id : 8]
+Input [3]: [d_date_sk#37, d_year#38, d_qoy#39]
+Condition : ((((isnotnull(d_qoy#39) AND isnotnull(d_year#38)) AND (d_qoy#39 = 3)) AND (d_year#38 = 2000)) AND isnotnull(d_date_sk#37))
+
+(42) BroadcastExchange
+Input [3]: [d_date_sk#37, d_year#38, d_qoy#39]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#40]
+
+(43) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_sold_date_sk#35]
+Right keys [1]: [d_date_sk#37]
+Join condition: None
+
+(44) Project [codegen id : 10]
+Output [4]: [ss_addr_sk#33, ss_ext_sales_price#34, d_year#38, d_qoy#39]
+Input [6]: [ss_addr_sk#33, ss_ext_sales_price#34, ss_sold_date_sk#35, d_date_sk#37, d_year#38, d_qoy#39]
+
+(45) ReusedExchange [Reuses operator id: 13]
+Output [2]: [ca_address_sk#41, ca_county#42]
+
+(46) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_addr_sk#33]
+Right keys [1]: [ca_address_sk#41]
+Join condition: None
+
+(47) Project [codegen id : 10]
+Output [4]: [ss_ext_sales_price#34, d_year#38, d_qoy#39, ca_county#42]
+Input [6]: [ss_addr_sk#33, ss_ext_sales_price#34, d_year#38, d_qoy#39, ca_address_sk#41, ca_county#42]
+
+(48) HashAggregate [codegen id : 10]
+Input [4]: [ss_ext_sales_price#34, d_year#38, d_qoy#39, ca_county#42]
+Keys [3]: [ca_county#42, d_qoy#39, d_year#38]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#34))]
+Aggregate Attributes [1]: [sum#43]
+Results [4]: [ca_county#42, d_qoy#39, d_year#38, sum#44]
+
+(49) Exchange
+Input [4]: [ca_county#42, d_qoy#39, d_year#38, sum#44]
+Arguments: hashpartitioning(ca_county#42, d_qoy#39, d_year#38, 5), ENSURE_REQUIREMENTS, [id=#45]
+
+(50) HashAggregate [codegen id : 11]
+Input [4]: [ca_county#42, d_qoy#39, d_year#38, sum#44]
+Keys [3]: [ca_county#42, d_qoy#39, d_year#38]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#34))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#34))#46]
+Results [2]: [ca_county#42, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#34))#46,17,2) AS store_sales#47]
+
+(51) BroadcastExchange
+Input [2]: [ca_county#42, store_sales#47]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#48]
+
+(52) BroadcastHashJoin [codegen id : 24]
+Left keys [1]: [ca_county#26]
+Right keys [1]: [ca_county#42]
+Join condition: None
+
+(53) Project [codegen id : 24]
+Output [5]: [ca_county#10, d_year#6, store_sales#16, store_sales#31, store_sales#47]
+Input [7]: [ca_county#10, d_year#6, store_sales#16, ca_county#26, store_sales#31, ca_county#42, store_sales#47]
+
+(54) Scan parquet default.web_sales
+Output [3]: [ws_bill_addr_sk#49, ws_ext_sales_price#50, ws_sold_date_sk#51]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#51), dynamicpruningexpression(ws_sold_date_sk#51 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ws_bill_addr_sk)]
+ReadSchema: struct<ws_bill_addr_sk:int,ws_ext_sales_price:decimal(7,2)>
+
+(55) ColumnarToRow [codegen id : 14]
+Input [3]: [ws_bill_addr_sk#49, ws_ext_sales_price#50, ws_sold_date_sk#51]
+
+(56) Filter [codegen id : 14]
+Input [3]: [ws_bill_addr_sk#49, ws_ext_sales_price#50, ws_sold_date_sk#51]
+Condition : isnotnull(ws_bill_addr_sk#49)
+
+(57) ReusedExchange [Reuses operator id: 7]
+Output [3]: [d_date_sk#52, d_year#53, d_qoy#54]
+
+(58) BroadcastHashJoin [codegen id : 14]
+Left keys [1]: [ws_sold_date_sk#51]
+Right keys [1]: [d_date_sk#52]
+Join condition: None
+
+(59) Project [codegen id : 14]
+Output [4]: [ws_bill_addr_sk#49, ws_ext_sales_price#50, d_year#53, d_qoy#54]
+Input [6]: [ws_bill_addr_sk#49, ws_ext_sales_price#50, ws_sold_date_sk#51, d_date_sk#52, d_year#53, d_qoy#54]
+
+(60) ReusedExchange [Reuses operator id: 13]
+Output [2]: [ca_address_sk#55, ca_county#56]
+
+(61) BroadcastHashJoin [codegen id : 14]
+Left keys [1]: [ws_bill_addr_sk#49]
+Right keys [1]: [ca_address_sk#55]
+Join condition: None
+
+(62) Project [codegen id : 14]
+Output [4]: [ws_ext_sales_price#50, d_year#53, d_qoy#54, ca_county#56]
+Input [6]: [ws_bill_addr_sk#49, ws_ext_sales_price#50, d_year#53, d_qoy#54, ca_address_sk#55, ca_county#56]
+
+(63) HashAggregate [codegen id : 14]
+Input [4]: [ws_ext_sales_price#50, d_year#53, d_qoy#54, ca_county#56]
+Keys [3]: [ca_county#56, d_qoy#54, d_year#53]
+Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#50))]
+Aggregate Attributes [1]: [sum#57]
+Results [4]: [ca_county#56, d_qoy#54, d_year#53, sum#58]
+
+(64) Exchange
+Input [4]: [ca_county#56, d_qoy#54, d_year#53, sum#58]
+Arguments: hashpartitioning(ca_county#56, d_qoy#54, d_year#53, 5), ENSURE_REQUIREMENTS, [id=#59]
+
+(65) HashAggregate [codegen id : 15]
+Input [4]: [ca_county#56, d_qoy#54, d_year#53, sum#58]
+Keys [3]: [ca_county#56, d_qoy#54, d_year#53]
+Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#50))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#50))#60]
+Results [2]: [ca_county#56, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#50))#60,17,2) AS web_sales#61]
+
+(66) BroadcastExchange
+Input [2]: [ca_county#56, web_sales#61]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#62]
+
+(67) BroadcastHashJoin [codegen id : 24]
+Left keys [1]: [ca_county#10]
+Right keys [1]: [ca_county#56]
+Join condition: None
+
+(68) Scan parquet default.web_sales
+Output [3]: [ws_bill_addr_sk#63, ws_ext_sales_price#64, ws_sold_date_sk#65]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#65), dynamicpruningexpression(ws_sold_date_sk#65 IN dynamicpruning#20)]
+PushedFilters: [IsNotNull(ws_bill_addr_sk)]
+ReadSchema: struct<ws_bill_addr_sk:int,ws_ext_sales_price:decimal(7,2)>
+
+(69) ColumnarToRow [codegen id : 18]
+Input [3]: [ws_bill_addr_sk#63, ws_ext_sales_price#64, ws_sold_date_sk#65]
+
+(70) Filter [codegen id : 18]
+Input [3]: [ws_bill_addr_sk#63, ws_ext_sales_price#64, ws_sold_date_sk#65]
+Condition : isnotnull(ws_bill_addr_sk#63)
+
+(71) ReusedExchange [Reuses operator id: 25]
+Output [3]: [d_date_sk#66, d_year#67, d_qoy#68]
+
+(72) BroadcastHashJoin [codegen id : 18]
+Left keys [1]: [ws_sold_date_sk#65]
+Right keys [1]: [d_date_sk#66]
+Join condition: None
+
+(73) Project [codegen id : 18]
+Output [4]: [ws_bill_addr_sk#63, ws_ext_sales_price#64, d_year#67, d_qoy#68]
+Input [6]: [ws_bill_addr_sk#63, ws_ext_sales_price#64, ws_sold_date_sk#65, d_date_sk#66, d_year#67, d_qoy#68]
+
+(74) ReusedExchange [Reuses operator id: 13]
+Output [2]: [ca_address_sk#69, ca_county#70]
+
+(75) BroadcastHashJoin [codegen id : 18]
+Left keys [1]: [ws_bill_addr_sk#63]
+Right keys [1]: [ca_address_sk#69]
+Join condition: None
+
+(76) Project [codegen id : 18]
+Output [4]: [ws_ext_sales_price#64, d_year#67, d_qoy#68, ca_county#70]
+Input [6]: [ws_bill_addr_sk#63, ws_ext_sales_price#64, d_year#67, d_qoy#68, ca_address_sk#69, ca_county#70]
+
+(77) HashAggregate [codegen id : 18]
+Input [4]: [ws_ext_sales_price#64, d_year#67, d_qoy#68, ca_county#70]
+Keys [3]: [ca_county#70, d_qoy#68, d_year#67]
+Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#64))]
+Aggregate Attributes [1]: [sum#71]
+Results [4]: [ca_county#70, d_qoy#68, d_year#67, sum#72]
+
+(78) Exchange
+Input [4]: [ca_county#70, d_qoy#68, d_year#67, sum#72]
+Arguments: hashpartitioning(ca_county#70, d_qoy#68, d_year#67, 5), ENSURE_REQUIREMENTS, [id=#73]
+
+(79) HashAggregate [codegen id : 19]
+Input [4]: [ca_county#70, d_qoy#68, d_year#67, sum#72]
+Keys [3]: [ca_county#70, d_qoy#68, d_year#67]
+Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#64))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#64))#74]
+Results [2]: [ca_county#70, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#64))#74,17,2) AS web_sales#75]
+
+(80) BroadcastExchange
+Input [2]: [ca_county#70, web_sales#75]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#76]
+
+(81) BroadcastHashJoin [codegen id : 24]
+Left keys [1]: [ca_county#56]
+Right keys [1]: [ca_county#70]
+Join condition: (CASE WHEN (web_sales#61 > 0.00) THEN CheckOverflow((promote_precision(web_sales#75) / promote_precision(web_sales#61)), DecimalType(37,20), true) ELSE null END > CASE WHEN (store_sales#16 > 0.00) THEN CheckOverflow((promote_precision(store_sales#31) / promote_precision(store_sales#16)), DecimalType(37,20), true) ELSE null END)
+
+(82) Project [codegen id : 24]
+Output [8]: [ca_county#10, d_year#6, store_sales#16, store_sales#31, store_sales#47, ca_county#56, web_sales#61, web_sales#75]
+Input [9]: [ca_county#10, d_year#6, store_sales#16, store_sales#31, store_sales#47, ca_county#56, web_sales#61, ca_county#70, web_sales#75]
+
+(83) Scan parquet default.web_sales
+Output [3]: [ws_bill_addr_sk#77, ws_ext_sales_price#78, ws_sold_date_sk#79]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#79), dynamicpruningexpression(ws_sold_date_sk#79 IN dynamicpruning#36)]
+PushedFilters: [IsNotNull(ws_bill_addr_sk)]
+ReadSchema: struct<ws_bill_addr_sk:int,ws_ext_sales_price:decimal(7,2)>
+
+(84) ColumnarToRow [codegen id : 22]
+Input [3]: [ws_bill_addr_sk#77, ws_ext_sales_price#78, ws_sold_date_sk#79]
+
+(85) Filter [codegen id : 22]
+Input [3]: [ws_bill_addr_sk#77, ws_ext_sales_price#78, ws_sold_date_sk#79]
+Condition : isnotnull(ws_bill_addr_sk#77)
+
+(86) ReusedExchange [Reuses operator id: 42]
+Output [3]: [d_date_sk#80, d_year#81, d_qoy#82]
+
+(87) BroadcastHashJoin [codegen id : 22]
+Left keys [1]: [ws_sold_date_sk#79]
+Right keys [1]: [d_date_sk#80]
+Join condition: None
+
+(88) Project [codegen id : 22]
+Output [4]: [ws_bill_addr_sk#77, ws_ext_sales_price#78, d_year#81, d_qoy#82]
+Input [6]: [ws_bill_addr_sk#77, ws_ext_sales_price#78, ws_sold_date_sk#79, d_date_sk#80, d_year#81, d_qoy#82]
+
+(89) ReusedExchange [Reuses operator id: 13]
+Output [2]: [ca_address_sk#83, ca_county#84]
+
+(90) BroadcastHashJoin [codegen id : 22]
+Left keys [1]: [ws_bill_addr_sk#77]
+Right keys [1]: [ca_address_sk#83]
+Join condition: None
+
+(91) Project [codegen id : 22]
+Output [4]: [ws_ext_sales_price#78, d_year#81, d_qoy#82, ca_county#84]
+Input [6]: [ws_bill_addr_sk#77, ws_ext_sales_price#78, d_year#81, d_qoy#82, ca_address_sk#83, ca_county#84]
+
+(92) HashAggregate [codegen id : 22]
+Input [4]: [ws_ext_sales_price#78, d_year#81, d_qoy#82, ca_county#84]
+Keys [3]: [ca_county#84, d_qoy#82, d_year#81]
+Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#78))]
+Aggregate Attributes [1]: [sum#85]
+Results [4]: [ca_county#84, d_qoy#82, d_year#81, sum#86]
+
+(93) Exchange
+Input [4]: [ca_county#84, d_qoy#82, d_year#81, sum#86]
+Arguments: hashpartitioning(ca_county#84, d_qoy#82, d_year#81, 5), ENSURE_REQUIREMENTS, [id=#87]
+
+(94) HashAggregate [codegen id : 23]
+Input [4]: [ca_county#84, d_qoy#82, d_year#81, sum#86]
+Keys [3]: [ca_county#84, d_qoy#82, d_year#81]
+Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#78))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#78))#88]
+Results [2]: [ca_county#84, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#78))#88,17,2) AS web_sales#89]
+
+(95) BroadcastExchange
+Input [2]: [ca_county#84, web_sales#89]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#90]
+
+(96) BroadcastHashJoin [codegen id : 24]
+Left keys [1]: [ca_county#56]
+Right keys [1]: [ca_county#84]
+Join condition: (CASE WHEN (web_sales#75 > 0.00) THEN CheckOverflow((promote_precision(web_sales#89) / promote_precision(web_sales#75)), DecimalType(37,20), true) ELSE null END > CASE WHEN (store_sales#31 > 0.00) THEN CheckOverflow((promote_precision(store_sales#47) / promote_precision(store_sales#31)), DecimalType(37,20), true) ELSE null END)
+
+(97) Project [codegen id : 24]
+Output [6]: [ca_county#10, d_year#6, CheckOverflow((promote_precision(web_sales#75) / promote_precision(web_sales#61)), DecimalType(37,20), true) AS web_q1_q2_increase#91, CheckOverflow((promote_precision(store_sales#31) / promote_precision(store_sales#16)), DecimalType(37,20), true) AS store_q1_q2_increase#92, CheckOverflow((promote_precision(web_sales#89) / promote_precision(web_sales#75)), DecimalType(37,20), true) AS web_q2_q3_increase#93, CheckOverflow((promote_precision(store_sales#47) / promote_precision(store_sales#31)), DecimalType(37,20), true) AS store_q2_q3_increase#94]
+Input [10]: [ca_county#10, d_year#6, store_sales#16, store_sales#31, store_sales#47, ca_county#56, web_sales#61, web_sales#75, ca_county#84, web_sales#89]
+
+(98) Exchange
+Input [6]: [ca_county#10, d_year#6, web_q1_q2_increase#91, store_q1_q2_increase#92, web_q2_q3_increase#93, store_q2_q3_increase#94]
+Arguments: rangepartitioning(ca_county#10 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [id=#95]
+
+(99) Sort [codegen id : 25]
+Input [6]: [ca_county#10, d_year#6, web_q1_q2_increase#91, store_q1_q2_increase#92, web_q2_q3_increase#93, store_q2_q3_increase#94]
+Arguments: [ca_county#10 ASC NULLS FIRST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (100)
+
+
+(100) ReusedExchange [Reuses operator id: 7]
+Output [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+
+Subquery:2 Hosting operator id = 19 Hosting Expression = ss_sold_date_sk#19 IN dynamicpruning#20
+ReusedExchange (101)
+
+
+(101) ReusedExchange [Reuses operator id: 25]
+Output [3]: [d_date_sk#21, d_year#22, d_qoy#23]
+
+Subquery:3 Hosting operator id = 36 Hosting Expression = ss_sold_date_sk#35 IN dynamicpruning#36
+ReusedExchange (102)
+
+
+(102) ReusedExchange [Reuses operator id: 42]
+Output [3]: [d_date_sk#37, d_year#38, d_qoy#39]
+
+Subquery:4 Hosting operator id = 54 Hosting Expression = ws_sold_date_sk#51 IN dynamicpruning#4
+
+Subquery:5 Hosting operator id = 68 Hosting Expression = ws_sold_date_sk#65 IN dynamicpruning#20
+
+Subquery:6 Hosting operator id = 83 Hosting Expression = ws_sold_date_sk#79 IN dynamicpruning#36
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/simplified.txt
new file mode 100644
index 0000000000000..d7d6c5ee63cbc
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/simplified.txt
@@ -0,0 +1,159 @@
+WholeStageCodegen (25)
+  Sort [ca_county]
+    InputAdapter
+      Exchange [ca_county] #1
+        WholeStageCodegen (24)
+          Project [ca_county,d_year,web_sales,web_sales,store_sales,store_sales,web_sales,store_sales]
+            BroadcastHashJoin [ca_county,ca_county,web_sales,web_sales,store_sales,store_sales]
+              Project [ca_county,d_year,store_sales,store_sales,store_sales,ca_county,web_sales,web_sales]
+                BroadcastHashJoin [ca_county,ca_county,web_sales,web_sales,store_sales,store_sales]
+                  BroadcastHashJoin [ca_county,ca_county]
+                    Project [ca_county,d_year,store_sales,store_sales,store_sales]
+                      BroadcastHashJoin [ca_county,ca_county]
+                        BroadcastHashJoin [ca_county,ca_county]
+                          HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum]
+                            InputAdapter
+                              Exchange [ca_county,d_qoy,d_year] #2
+                                WholeStageCodegen (3)
+                                  HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum]
+                                    Project [ss_ext_sales_price,d_year,d_qoy,ca_county]
+                                      BroadcastHashJoin [ss_addr_sk,ca_address_sk]
+                                        Project [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy]
+                                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                            Filter [ss_addr_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #1
+                                                      ReusedExchange [d_date_sk,d_year,d_qoy] #3
+                                            InputAdapter
+                                              BroadcastExchange #3
+                                                WholeStageCodegen (1)
+                                                  Filter [d_qoy,d_year,d_date_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.date_dim [d_date_sk,d_year,d_qoy]
+                                        InputAdapter
+                                          BroadcastExchange #4
+                                            WholeStageCodegen (2)
+                                              Filter [ca_address_sk,ca_county]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.customer_address [ca_address_sk,ca_county]
+                          InputAdapter
+                            BroadcastExchange #5
+                              WholeStageCodegen (7)
+                                HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum]
+                                  InputAdapter
+                                    Exchange [ca_county,d_qoy,d_year] #6
+                                      WholeStageCodegen (6)
+                                        HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum]
+                                          Project [ss_ext_sales_price,d_year,d_qoy,ca_county]
+                                            BroadcastHashJoin [ss_addr_sk,ca_address_sk]
+                                              Project [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy]
+                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                  Filter [ss_addr_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                                          SubqueryBroadcast [d_date_sk] #2
+                                                            ReusedExchange [d_date_sk,d_year,d_qoy] #7
+                                                  InputAdapter
+                                                    BroadcastExchange #7
+                                                      WholeStageCodegen (4)
+                                                        Filter [d_qoy,d_year,d_date_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.date_dim [d_date_sk,d_year,d_qoy]
+                                              InputAdapter
+                                                ReusedExchange [ca_address_sk,ca_county] #4
+                        InputAdapter
+                          BroadcastExchange #8
+                            WholeStageCodegen (11)
+                              HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum]
+                                InputAdapter
+                                  Exchange [ca_county,d_qoy,d_year] #9
+                                    WholeStageCodegen (10)
+                                      HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum]
+                                        Project [ss_ext_sales_price,d_year,d_qoy,ca_county]
+                                          BroadcastHashJoin [ss_addr_sk,ca_address_sk]
+                                            Project [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy]
+                                              BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                Filter [ss_addr_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                                        SubqueryBroadcast [d_date_sk] #3
+                                                          ReusedExchange [d_date_sk,d_year,d_qoy] #10
+                                                InputAdapter
+                                                  BroadcastExchange #10
+                                                    WholeStageCodegen (8)
+                                                      Filter [d_qoy,d_year,d_date_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.date_dim [d_date_sk,d_year,d_qoy]
+                                            InputAdapter
+                                              ReusedExchange [ca_address_sk,ca_county] #4
+                    InputAdapter
+                      BroadcastExchange #11
+                        WholeStageCodegen (15)
+                          HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum]
+                            InputAdapter
+                              Exchange [ca_county,d_qoy,d_year] #12
+                                WholeStageCodegen (14)
+                                  HashAggregate [ca_county,d_qoy,d_year,ws_ext_sales_price] [sum,sum]
+                                    Project [ws_ext_sales_price,d_year,d_qoy,ca_county]
+                                      BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk]
+                                        Project [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy]
+                                          BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                            Filter [ws_bill_addr_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk]
+                                                    ReusedSubquery [d_date_sk] #1
+                                            InputAdapter
+                                              ReusedExchange [d_date_sk,d_year,d_qoy] #3
+                                        InputAdapter
+                                          ReusedExchange [ca_address_sk,ca_county] #4
+                  InputAdapter
+                    BroadcastExchange #13
+                      WholeStageCodegen (19)
+                        HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum]
+                          InputAdapter
+                            Exchange [ca_county,d_qoy,d_year] #14
+                              WholeStageCodegen (18)
+                                HashAggregate [ca_county,d_qoy,d_year,ws_ext_sales_price] [sum,sum]
+                                  Project [ws_ext_sales_price,d_year,d_qoy,ca_county]
+                                    BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk]
+                                      Project [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy]
+                                        BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                          Filter [ws_bill_addr_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk]
+                                                  ReusedSubquery [d_date_sk] #2
+                                          InputAdapter
+                                            ReusedExchange [d_date_sk,d_year,d_qoy] #7
+                                      InputAdapter
+                                        ReusedExchange [ca_address_sk,ca_county] #4
+              InputAdapter
+                BroadcastExchange #15
+                  WholeStageCodegen (23)
+                    HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum]
+                      InputAdapter
+                        Exchange [ca_county,d_qoy,d_year] #16
+                          WholeStageCodegen (22)
+                            HashAggregate [ca_county,d_qoy,d_year,ws_ext_sales_price] [sum,sum]
+                              Project [ws_ext_sales_price,d_year,d_qoy,ca_county]
+                                BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk]
+                                  Project [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy]
+                                    BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                      Filter [ws_bill_addr_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk]
+                                              ReusedSubquery [d_date_sk] #3
+                                      InputAdapter
+                                        ReusedExchange [d_date_sk,d_year,d_qoy] #10
+                                  InputAdapter
+                                    ReusedExchange [ca_address_sk,ca_county] #4
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.sf100/explain.txt
new file mode 100644
index 0000000000000..5d171e5f595b9
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.sf100/explain.txt
@@ -0,0 +1,189 @@
+== Physical Plan ==
+CollectLimit (31)
++- * Project (30)
+   +- * BroadcastHashJoin Inner BuildRight (29)
+      :- * Project (27)
+      :  +- * BroadcastHashJoin Inner BuildLeft (26)
+      :     :- BroadcastExchange (22)
+      :     :  +- * Project (21)
+      :     :     +- * BroadcastHashJoin Inner BuildLeft (20)
+      :     :        :- BroadcastExchange (5)
+      :     :        :  +- * Project (4)
+      :     :        :     +- * Filter (3)
+      :     :        :        +- * ColumnarToRow (2)
+      :     :        :           +- Scan parquet default.item (1)
+      :     :        +- * Filter (19)
+      :     :           +- * HashAggregate (18)
+      :     :              +- Exchange (17)
+      :     :                 +- * HashAggregate (16)
+      :     :                    +- * Project (15)
+      :     :                       +- * BroadcastHashJoin Inner BuildRight (14)
+      :     :                          :- * Filter (8)
+      :     :                          :  +- * ColumnarToRow (7)
+      :     :                          :     +- Scan parquet default.catalog_sales (6)
+      :     :                          +- BroadcastExchange (13)
+      :     :                             +- * Project (12)
+      :     :                                +- * Filter (11)
+      :     :                                   +- * ColumnarToRow (10)
+      :     :                                      +- Scan parquet default.date_dim (9)
+      :     +- * Filter (25)
+      :        +- * ColumnarToRow (24)
+      :           +- Scan parquet default.catalog_sales (23)
+      +- ReusedExchange (28)
+
+
+(1) Scan parquet default.item
+Output [2]: [i_item_sk#1, i_manufact_id#2]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,977), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_manufact_id:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [2]: [i_item_sk#1, i_manufact_id#2]
+
+(3) Filter [codegen id : 1]
+Input [2]: [i_item_sk#1, i_manufact_id#2]
+Condition : ((isnotnull(i_manufact_id#2) AND (i_manufact_id#2 = 977)) AND isnotnull(i_item_sk#1))
+
+(4) Project [codegen id : 1]
+Output [1]: [i_item_sk#1]
+Input [2]: [i_item_sk#1, i_manufact_id#2]
+
+(5) BroadcastExchange
+Input [1]: [i_item_sk#1]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#3]
+
+(6) Scan parquet default.catalog_sales
+Output [3]: [cs_item_sk#4, cs_ext_discount_amt#5, cs_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#6), dynamicpruningexpression(cs_sold_date_sk#6 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int,cs_ext_discount_amt:decimal(7,2)>
+
+(7) ColumnarToRow [codegen id : 3]
+Input [3]: [cs_item_sk#4, cs_ext_discount_amt#5, cs_sold_date_sk#6]
+
+(8) Filter [codegen id : 3]
+Input [3]: [cs_item_sk#4, cs_ext_discount_amt#5, cs_sold_date_sk#6]
+Condition : isnotnull(cs_item_sk#4)
+
+(9) Scan parquet default.date_dim
+Output [2]: [d_date_sk#8, d_date#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(10) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#8, d_date#9]
+
+(11) Filter [codegen id : 2]
+Input [2]: [d_date_sk#8, d_date#9]
+Condition : (((isnotnull(d_date#9) AND (d_date#9 >= 2000-01-27)) AND (d_date#9 <= 2000-04-26)) AND isnotnull(d_date_sk#8))
+
+(12) Project [codegen id : 2]
+Output [1]: [d_date_sk#8]
+Input [2]: [d_date_sk#8, d_date#9]
+
+(13) BroadcastExchange
+Input [1]: [d_date_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(14) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_sold_date_sk#6]
+Right keys [1]: [d_date_sk#8]
+Join condition: None
+
+(15) Project [codegen id : 3]
+Output [2]: [cs_item_sk#4, cs_ext_discount_amt#5]
+Input [4]: [cs_item_sk#4, cs_ext_discount_amt#5, cs_sold_date_sk#6, d_date_sk#8]
+
+(16) HashAggregate [codegen id : 3]
+Input [2]: [cs_item_sk#4, cs_ext_discount_amt#5]
+Keys [1]: [cs_item_sk#4]
+Functions [1]: [partial_avg(UnscaledValue(cs_ext_discount_amt#5))]
+Aggregate Attributes [2]: [sum#11, count#12]
+Results [3]: [cs_item_sk#4, sum#13, count#14]
+
+(17) Exchange
+Input [3]: [cs_item_sk#4, sum#13, count#14]
+Arguments: hashpartitioning(cs_item_sk#4, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(18) HashAggregate
+Input [3]: [cs_item_sk#4, sum#13, count#14]
+Keys [1]: [cs_item_sk#4]
+Functions [1]: [avg(UnscaledValue(cs_ext_discount_amt#5))]
+Aggregate Attributes [1]: [avg(UnscaledValue(cs_ext_discount_amt#5))#16]
+Results [2]: [CheckOverflow((1.300000 * promote_precision(cast((avg(UnscaledValue(cs_ext_discount_amt#5))#16 / 100.0) as decimal(11,6)))), DecimalType(14,7), true) AS (1.3 * avg(cs_ext_discount_amt))#17, cs_item_sk#4]
+
+(19) Filter
+Input [2]: [(1.3 * avg(cs_ext_discount_amt))#17, cs_item_sk#4]
+Condition : isnotnull((1.3 * avg(cs_ext_discount_amt))#17)
+
+(20) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [cs_item_sk#4]
+Join condition: None
+
+(21) Project [codegen id : 4]
+Output [2]: [i_item_sk#1, (1.3 * avg(cs_ext_discount_amt))#17]
+Input [3]: [i_item_sk#1, (1.3 * avg(cs_ext_discount_amt))#17, cs_item_sk#4]
+
+(22) BroadcastExchange
+Input [2]: [i_item_sk#1, (1.3 * avg(cs_ext_discount_amt))#17]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(23) Scan parquet default.catalog_sales
+Output [3]: [cs_item_sk#19, cs_ext_discount_amt#20, cs_sold_date_sk#21]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#21), dynamicpruningexpression(cs_sold_date_sk#21 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_ext_discount_amt)]
+ReadSchema: struct<cs_item_sk:int,cs_ext_discount_amt:decimal(7,2)>
+
+(24) ColumnarToRow
+Input [3]: [cs_item_sk#19, cs_ext_discount_amt#20, cs_sold_date_sk#21]
+
+(25) Filter
+Input [3]: [cs_item_sk#19, cs_ext_discount_amt#20, cs_sold_date_sk#21]
+Condition : (isnotnull(cs_item_sk#19) AND isnotnull(cs_ext_discount_amt#20))
+
+(26) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [cs_item_sk#19]
+Join condition: (cast(cs_ext_discount_amt#20 as decimal(14,7)) > (1.3 * avg(cs_ext_discount_amt))#17)
+
+(27) Project [codegen id : 6]
+Output [1]: [cs_sold_date_sk#21]
+Input [5]: [i_item_sk#1, (1.3 * avg(cs_ext_discount_amt))#17, cs_item_sk#19, cs_ext_discount_amt#20, cs_sold_date_sk#21]
+
+(28) ReusedExchange [Reuses operator id: 13]
+Output [1]: [d_date_sk#22]
+
+(29) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [cs_sold_date_sk#21]
+Right keys [1]: [d_date_sk#22]
+Join condition: None
+
+(30) Project [codegen id : 6]
+Output [1]: [1 AS excess discount amount #23]
+Input [2]: [cs_sold_date_sk#21, d_date_sk#22]
+
+(31) CollectLimit
+Input [1]: [excess discount amount #23]
+Arguments: 100
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 6 Hosting Expression = cs_sold_date_sk#6 IN dynamicpruning#7
+ReusedExchange (32)
+
+
+(32) ReusedExchange [Reuses operator id: 13]
+Output [1]: [d_date_sk#8]
+
+Subquery:2 Hosting operator id = 23 Hosting Expression = cs_sold_date_sk#21 IN dynamicpruning#7
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.sf100/simplified.txt
new file mode 100644
index 0000000000000..303bdf58604bf
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.sf100/simplified.txt
@@ -0,0 +1,48 @@
+CollectLimit
+  WholeStageCodegen (6)
+    Project
+      BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+        Project [cs_sold_date_sk]
+          BroadcastHashJoin [i_item_sk,cs_item_sk,cs_ext_discount_amt,(1.3 * avg(cs_ext_discount_amt))]
+            InputAdapter
+              BroadcastExchange #1
+                WholeStageCodegen (4)
+                  Project [i_item_sk,(1.3 * avg(cs_ext_discount_amt))]
+                    BroadcastHashJoin [i_item_sk,cs_item_sk]
+                      InputAdapter
+                        BroadcastExchange #2
+                          WholeStageCodegen (1)
+                            Project [i_item_sk]
+                              Filter [i_manufact_id,i_item_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.item [i_item_sk,i_manufact_id]
+                      Filter [(1.3 * avg(cs_ext_discount_amt))]
+                        HashAggregate [cs_item_sk,sum,count] [avg(UnscaledValue(cs_ext_discount_amt)),(1.3 * avg(cs_ext_discount_amt)),sum,count]
+                          InputAdapter
+                            Exchange [cs_item_sk] #3
+                              WholeStageCodegen (3)
+                                HashAggregate [cs_item_sk,cs_ext_discount_amt] [sum,count,sum,count]
+                                  Project [cs_item_sk,cs_ext_discount_amt]
+                                    BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                      Filter [cs_item_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #1
+                                                ReusedExchange [d_date_sk] #4
+                                      InputAdapter
+                                        BroadcastExchange #4
+                                          WholeStageCodegen (2)
+                                            Project [d_date_sk]
+                                              Filter [d_date,d_date_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_date_sk,d_date]
+            Filter [cs_item_sk,cs_ext_discount_amt]
+              ColumnarToRow
+                InputAdapter
+                  Scan parquet default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk]
+                    ReusedSubquery [d_date_sk] #1
+        InputAdapter
+          ReusedExchange [d_date_sk] #4
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/explain.txt
new file mode 100644
index 0000000000000..ad918310a918a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/explain.txt
@@ -0,0 +1,189 @@
+== Physical Plan ==
+CollectLimit (31)
++- * Project (30)
+   +- * BroadcastHashJoin Inner BuildRight (29)
+      :- * Project (27)
+      :  +- * BroadcastHashJoin Inner BuildRight (26)
+      :     :- * Project (10)
+      :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+      :     :     :- * Filter (3)
+      :     :     :  +- * ColumnarToRow (2)
+      :     :     :     +- Scan parquet default.catalog_sales (1)
+      :     :     +- BroadcastExchange (8)
+      :     :        +- * Project (7)
+      :     :           +- * Filter (6)
+      :     :              +- * ColumnarToRow (5)
+      :     :                 +- Scan parquet default.item (4)
+      :     +- BroadcastExchange (25)
+      :        +- * Filter (24)
+      :           +- * HashAggregate (23)
+      :              +- Exchange (22)
+      :                 +- * HashAggregate (21)
+      :                    +- * Project (20)
+      :                       +- * BroadcastHashJoin Inner BuildRight (19)
+      :                          :- * Filter (13)
+      :                          :  +- * ColumnarToRow (12)
+      :                          :     +- Scan parquet default.catalog_sales (11)
+      :                          +- BroadcastExchange (18)
+      :                             +- * Project (17)
+      :                                +- * Filter (16)
+      :                                   +- * ColumnarToRow (15)
+      :                                      +- Scan parquet default.date_dim (14)
+      +- ReusedExchange (28)
+
+
+(1) Scan parquet default.catalog_sales
+Output [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_ext_discount_amt)]
+ReadSchema: struct<cs_item_sk:int,cs_ext_discount_amt:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 6]
+Input [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3]
+
+(3) Filter [codegen id : 6]
+Input [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3]
+Condition : (isnotnull(cs_item_sk#1) AND isnotnull(cs_ext_discount_amt#2))
+
+(4) Scan parquet default.item
+Output [2]: [i_item_sk#5, i_manufact_id#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,977), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_manufact_id:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [i_item_sk#5, i_manufact_id#6]
+
+(6) Filter [codegen id : 1]
+Input [2]: [i_item_sk#5, i_manufact_id#6]
+Condition : ((isnotnull(i_manufact_id#6) AND (i_manufact_id#6 = 977)) AND isnotnull(i_item_sk#5))
+
+(7) Project [codegen id : 1]
+Output [1]: [i_item_sk#5]
+Input [2]: [i_item_sk#5, i_manufact_id#6]
+
+(8) BroadcastExchange
+Input [1]: [i_item_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [cs_item_sk#1]
+Right keys [1]: [i_item_sk#5]
+Join condition: None
+
+(10) Project [codegen id : 6]
+Output [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5]
+Input [4]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5]
+
+(11) Scan parquet default.catalog_sales
+Output [3]: [cs_item_sk#8, cs_ext_discount_amt#9, cs_sold_date_sk#10]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#10), dynamicpruningexpression(cs_sold_date_sk#10 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int,cs_ext_discount_amt:decimal(7,2)>
+
+(12) ColumnarToRow [codegen id : 3]
+Input [3]: [cs_item_sk#8, cs_ext_discount_amt#9, cs_sold_date_sk#10]
+
+(13) Filter [codegen id : 3]
+Input [3]: [cs_item_sk#8, cs_ext_discount_amt#9, cs_sold_date_sk#10]
+Condition : isnotnull(cs_item_sk#8)
+
+(14) Scan parquet default.date_dim
+Output [2]: [d_date_sk#11, d_date#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(15) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#11, d_date#12]
+
+(16) Filter [codegen id : 2]
+Input [2]: [d_date_sk#11, d_date#12]
+Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-01-27)) AND (d_date#12 <= 2000-04-26)) AND isnotnull(d_date_sk#11))
+
+(17) Project [codegen id : 2]
+Output [1]: [d_date_sk#11]
+Input [2]: [d_date_sk#11, d_date#12]
+
+(18) BroadcastExchange
+Input [1]: [d_date_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(19) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_sold_date_sk#10]
+Right keys [1]: [d_date_sk#11]
+Join condition: None
+
+(20) Project [codegen id : 3]
+Output [2]: [cs_item_sk#8, cs_ext_discount_amt#9]
+Input [4]: [cs_item_sk#8, cs_ext_discount_amt#9, cs_sold_date_sk#10, d_date_sk#11]
+
+(21) HashAggregate [codegen id : 3]
+Input [2]: [cs_item_sk#8, cs_ext_discount_amt#9]
+Keys [1]: [cs_item_sk#8]
+Functions [1]: [partial_avg(UnscaledValue(cs_ext_discount_amt#9))]
+Aggregate Attributes [2]: [sum#14, count#15]
+Results [3]: [cs_item_sk#8, sum#16, count#17]
+
+(22) Exchange
+Input [3]: [cs_item_sk#8, sum#16, count#17]
+Arguments: hashpartitioning(cs_item_sk#8, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(23) HashAggregate [codegen id : 4]
+Input [3]: [cs_item_sk#8, sum#16, count#17]
+Keys [1]: [cs_item_sk#8]
+Functions [1]: [avg(UnscaledValue(cs_ext_discount_amt#9))]
+Aggregate Attributes [1]: [avg(UnscaledValue(cs_ext_discount_amt#9))#19]
+Results [2]: [CheckOverflow((1.300000 * promote_precision(cast((avg(UnscaledValue(cs_ext_discount_amt#9))#19 / 100.0) as decimal(11,6)))), DecimalType(14,7), true) AS (1.3 * avg(cs_ext_discount_amt))#20, cs_item_sk#8]
+
+(24) Filter [codegen id : 4]
+Input [2]: [(1.3 * avg(cs_ext_discount_amt))#20, cs_item_sk#8]
+Condition : isnotnull((1.3 * avg(cs_ext_discount_amt))#20)
+
+(25) BroadcastExchange
+Input [2]: [(1.3 * avg(cs_ext_discount_amt))#20, cs_item_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [id=#21]
+
+(26) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [i_item_sk#5]
+Right keys [1]: [cs_item_sk#8]
+Join condition: (cast(cs_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(cs_ext_discount_amt))#20)
+
+(27) Project [codegen id : 6]
+Output [1]: [cs_sold_date_sk#3]
+Input [5]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5, (1.3 * avg(cs_ext_discount_amt))#20, cs_item_sk#8]
+
+(28) ReusedExchange [Reuses operator id: 18]
+Output [1]: [d_date_sk#22]
+
+(29) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [cs_sold_date_sk#3]
+Right keys [1]: [d_date_sk#22]
+Join condition: None
+
+(30) Project [codegen id : 6]
+Output [1]: [1 AS excess discount amount #23]
+Input [2]: [cs_sold_date_sk#3, d_date_sk#22]
+
+(31) CollectLimit
+Input [1]: [excess discount amount #23]
+Arguments: 100
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (32)
+
+
+(32) ReusedExchange [Reuses operator id: 18]
+Output [1]: [d_date_sk#22]
+
+Subquery:2 Hosting operator id = 11 Hosting Expression = cs_sold_date_sk#10 IN dynamicpruning#4
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/simplified.txt
new file mode 100644
index 0000000000000..f3dd6d0954046
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/simplified.txt
@@ -0,0 +1,48 @@
+CollectLimit
+  WholeStageCodegen (6)
+    Project
+      BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+        Project [cs_sold_date_sk]
+          BroadcastHashJoin [i_item_sk,cs_item_sk,cs_ext_discount_amt,(1.3 * avg(cs_ext_discount_amt))]
+            Project [cs_ext_discount_amt,cs_sold_date_sk,i_item_sk]
+              BroadcastHashJoin [cs_item_sk,i_item_sk]
+                Filter [cs_item_sk,cs_ext_discount_amt]
+                  ColumnarToRow
+                    InputAdapter
+                      Scan parquet default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk]
+                        SubqueryBroadcast [d_date_sk] #1
+                          ReusedExchange [d_date_sk] #1
+                InputAdapter
+                  BroadcastExchange #2
+                    WholeStageCodegen (1)
+                      Project [i_item_sk]
+                        Filter [i_manufact_id,i_item_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.item [i_item_sk,i_manufact_id]
+            InputAdapter
+              BroadcastExchange #3
+                WholeStageCodegen (4)
+                  Filter [(1.3 * avg(cs_ext_discount_amt))]
+                    HashAggregate [cs_item_sk,sum,count] [avg(UnscaledValue(cs_ext_discount_amt)),(1.3 * avg(cs_ext_discount_amt)),sum,count]
+                      InputAdapter
+                        Exchange [cs_item_sk] #4
+                          WholeStageCodegen (3)
+                            HashAggregate [cs_item_sk,cs_ext_discount_amt] [sum,count,sum,count]
+                              Project [cs_item_sk,cs_ext_discount_amt]
+                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                  Filter [cs_item_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk]
+                                          ReusedSubquery [d_date_sk] #1
+                                  InputAdapter
+                                    BroadcastExchange #1
+                                      WholeStageCodegen (2)
+                                        Project [d_date_sk]
+                                          Filter [d_date,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_date]
+        InputAdapter
+          ReusedExchange [d_date_sk] #1
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.sf100/explain.txt
new file mode 100644
index 0000000000000..2aa99626920ec
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.sf100/explain.txt
@@ -0,0 +1,395 @@
+== Physical Plan ==
+TakeOrderedAndProject (67)
++- * HashAggregate (66)
+   +- Exchange (65)
+      +- * HashAggregate (64)
+         +- Union (63)
+            :- * HashAggregate (32)
+            :  +- Exchange (31)
+            :     +- * HashAggregate (30)
+            :        +- * Project (29)
+            :           +- * BroadcastHashJoin Inner BuildRight (28)
+            :              :- * Project (22)
+            :              :  +- * BroadcastHashJoin Inner BuildRight (21)
+            :              :     :- * Project (10)
+            :              :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+            :              :     :     :- * Filter (3)
+            :              :     :     :  +- * ColumnarToRow (2)
+            :              :     :     :     +- Scan parquet default.store_sales (1)
+            :              :     :     +- BroadcastExchange (8)
+            :              :     :        +- * Project (7)
+            :              :     :           +- * Filter (6)
+            :              :     :              +- * ColumnarToRow (5)
+            :              :     :                 +- Scan parquet default.date_dim (4)
+            :              :     +- BroadcastExchange (20)
+            :              :        +- * BroadcastHashJoin LeftSemi BuildRight (19)
+            :              :           :- * Filter (13)
+            :              :           :  +- * ColumnarToRow (12)
+            :              :           :     +- Scan parquet default.item (11)
+            :              :           +- BroadcastExchange (18)
+            :              :              +- * Project (17)
+            :              :                 +- * Filter (16)
+            :              :                    +- * ColumnarToRow (15)
+            :              :                       +- Scan parquet default.item (14)
+            :              +- BroadcastExchange (27)
+            :                 +- * Project (26)
+            :                    +- * Filter (25)
+            :                       +- * ColumnarToRow (24)
+            :                          +- Scan parquet default.customer_address (23)
+            :- * HashAggregate (47)
+            :  +- Exchange (46)
+            :     +- * HashAggregate (45)
+            :        +- * Project (44)
+            :           +- * BroadcastHashJoin Inner BuildRight (43)
+            :              :- * Project (41)
+            :              :  +- * BroadcastHashJoin Inner BuildRight (40)
+            :              :     :- * Project (38)
+            :              :     :  +- * BroadcastHashJoin Inner BuildRight (37)
+            :              :     :     :- * Filter (35)
+            :              :     :     :  +- * ColumnarToRow (34)
+            :              :     :     :     +- Scan parquet default.catalog_sales (33)
+            :              :     :     +- ReusedExchange (36)
+            :              :     +- ReusedExchange (39)
+            :              +- ReusedExchange (42)
+            +- * HashAggregate (62)
+               +- Exchange (61)
+                  +- * HashAggregate (60)
+                     +- * Project (59)
+                        +- * BroadcastHashJoin Inner BuildRight (58)
+                           :- * Project (56)
+                           :  +- * BroadcastHashJoin Inner BuildRight (55)
+                           :     :- * Project (53)
+                           :     :  +- * BroadcastHashJoin Inner BuildRight (52)
+                           :     :     :- * Filter (50)
+                           :     :     :  +- * ColumnarToRow (49)
+                           :     :     :     +- Scan parquet default.web_sales (48)
+                           :     :     +- ReusedExchange (51)
+                           :     +- ReusedExchange (54)
+                           +- ReusedExchange (57)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_addr_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 5]
+Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4]
+Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,5), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 1998)) AND (d_moy#8 = 5)) AND isnotnull(d_date_sk#6))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#6]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#9]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3]
+Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#6]
+
+(11) Scan parquet default.item
+Output [2]: [i_item_sk#10, i_manufact_id#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_manufact_id:int>
+
+(12) ColumnarToRow [codegen id : 3]
+Input [2]: [i_item_sk#10, i_manufact_id#11]
+
+(13) Filter [codegen id : 3]
+Input [2]: [i_item_sk#10, i_manufact_id#11]
+Condition : isnotnull(i_item_sk#10)
+
+(14) Scan parquet default.item
+Output [2]: [i_category#12, i_manufact_id#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Electronics                                       )]
+ReadSchema: struct<i_category:string,i_manufact_id:int>
+
+(15) ColumnarToRow [codegen id : 2]
+Input [2]: [i_category#12, i_manufact_id#13]
+
+(16) Filter [codegen id : 2]
+Input [2]: [i_category#12, i_manufact_id#13]
+Condition : (isnotnull(i_category#12) AND (i_category#12 = Electronics                                       ))
+
+(17) Project [codegen id : 2]
+Output [1]: [i_manufact_id#13]
+Input [2]: [i_category#12, i_manufact_id#13]
+
+(18) BroadcastExchange
+Input [1]: [i_manufact_id#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(19) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [i_manufact_id#11]
+Right keys [1]: [i_manufact_id#13]
+Join condition: None
+
+(20) BroadcastExchange
+Input [2]: [i_item_sk#10, i_manufact_id#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#15]
+
+(21) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#10]
+Join condition: None
+
+(22) Project [codegen id : 5]
+Output [3]: [ss_addr_sk#2, ss_ext_sales_price#3, i_manufact_id#11]
+Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, i_item_sk#10, i_manufact_id#11]
+
+(23) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#16, ca_gmt_offset#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_gmt_offset:decimal(5,2)>
+
+(24) ColumnarToRow [codegen id : 4]
+Input [2]: [ca_address_sk#16, ca_gmt_offset#17]
+
+(25) Filter [codegen id : 4]
+Input [2]: [ca_address_sk#16, ca_gmt_offset#17]
+Condition : ((isnotnull(ca_gmt_offset#17) AND (ca_gmt_offset#17 = -5.00)) AND isnotnull(ca_address_sk#16))
+
+(26) Project [codegen id : 4]
+Output [1]: [ca_address_sk#16]
+Input [2]: [ca_address_sk#16, ca_gmt_offset#17]
+
+(27) BroadcastExchange
+Input [1]: [ca_address_sk#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(28) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_addr_sk#2]
+Right keys [1]: [ca_address_sk#16]
+Join condition: None
+
+(29) Project [codegen id : 5]
+Output [2]: [ss_ext_sales_price#3, i_manufact_id#11]
+Input [4]: [ss_addr_sk#2, ss_ext_sales_price#3, i_manufact_id#11, ca_address_sk#16]
+
+(30) HashAggregate [codegen id : 5]
+Input [2]: [ss_ext_sales_price#3, i_manufact_id#11]
+Keys [1]: [i_manufact_id#11]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))]
+Aggregate Attributes [1]: [sum#19]
+Results [2]: [i_manufact_id#11, sum#20]
+
+(31) Exchange
+Input [2]: [i_manufact_id#11, sum#20]
+Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(32) HashAggregate [codegen id : 6]
+Input [2]: [i_manufact_id#11, sum#20]
+Keys [1]: [i_manufact_id#11]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#22]
+Results [2]: [i_manufact_id#11, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#22,17,2) AS total_sales#23]
+
+(33) Scan parquet default.catalog_sales
+Output [4]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, cs_sold_date_sk#27]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#27), dynamicpruningexpression(cs_sold_date_sk#27 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_bill_addr_sk:int,cs_item_sk:int,cs_ext_sales_price:decimal(7,2)>
+
+(34) ColumnarToRow [codegen id : 11]
+Input [4]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, cs_sold_date_sk#27]
+
+(35) Filter [codegen id : 11]
+Input [4]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, cs_sold_date_sk#27]
+Condition : (isnotnull(cs_bill_addr_sk#24) AND isnotnull(cs_item_sk#25))
+
+(36) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#28]
+
+(37) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_sold_date_sk#27]
+Right keys [1]: [d_date_sk#28]
+Join condition: None
+
+(38) Project [codegen id : 11]
+Output [3]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26]
+Input [5]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, cs_sold_date_sk#27, d_date_sk#28]
+
+(39) ReusedExchange [Reuses operator id: 20]
+Output [2]: [i_item_sk#29, i_manufact_id#30]
+
+(40) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_item_sk#25]
+Right keys [1]: [i_item_sk#29]
+Join condition: None
+
+(41) Project [codegen id : 11]
+Output [3]: [cs_bill_addr_sk#24, cs_ext_sales_price#26, i_manufact_id#30]
+Input [5]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, i_item_sk#29, i_manufact_id#30]
+
+(42) ReusedExchange [Reuses operator id: 27]
+Output [1]: [ca_address_sk#31]
+
+(43) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_bill_addr_sk#24]
+Right keys [1]: [ca_address_sk#31]
+Join condition: None
+
+(44) Project [codegen id : 11]
+Output [2]: [cs_ext_sales_price#26, i_manufact_id#30]
+Input [4]: [cs_bill_addr_sk#24, cs_ext_sales_price#26, i_manufact_id#30, ca_address_sk#31]
+
+(45) HashAggregate [codegen id : 11]
+Input [2]: [cs_ext_sales_price#26, i_manufact_id#30]
+Keys [1]: [i_manufact_id#30]
+Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#26))]
+Aggregate Attributes [1]: [sum#32]
+Results [2]: [i_manufact_id#30, sum#33]
+
+(46) Exchange
+Input [2]: [i_manufact_id#30, sum#33]
+Arguments: hashpartitioning(i_manufact_id#30, 5), ENSURE_REQUIREMENTS, [id=#34]
+
+(47) HashAggregate [codegen id : 12]
+Input [2]: [i_manufact_id#30, sum#33]
+Keys [1]: [i_manufact_id#30]
+Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#26))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#26))#35]
+Results [2]: [i_manufact_id#30, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#26))#35,17,2) AS total_sales#36]
+
+(48) Scan parquet default.web_sales
+Output [4]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#40), dynamicpruningexpression(ws_sold_date_sk#40 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_bill_addr_sk:int,ws_ext_sales_price:decimal(7,2)>
+
+(49) ColumnarToRow [codegen id : 17]
+Input [4]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40]
+
+(50) Filter [codegen id : 17]
+Input [4]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40]
+Condition : (isnotnull(ws_bill_addr_sk#38) AND isnotnull(ws_item_sk#37))
+
+(51) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#41]
+
+(52) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_sold_date_sk#40]
+Right keys [1]: [d_date_sk#41]
+Join condition: None
+
+(53) Project [codegen id : 17]
+Output [3]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39]
+Input [5]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40, d_date_sk#41]
+
+(54) ReusedExchange [Reuses operator id: 20]
+Output [2]: [i_item_sk#42, i_manufact_id#43]
+
+(55) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_item_sk#37]
+Right keys [1]: [i_item_sk#42]
+Join condition: None
+
+(56) Project [codegen id : 17]
+Output [3]: [ws_bill_addr_sk#38, ws_ext_sales_price#39, i_manufact_id#43]
+Input [5]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, i_item_sk#42, i_manufact_id#43]
+
+(57) ReusedExchange [Reuses operator id: 27]
+Output [1]: [ca_address_sk#44]
+
+(58) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_bill_addr_sk#38]
+Right keys [1]: [ca_address_sk#44]
+Join condition: None
+
+(59) Project [codegen id : 17]
+Output [2]: [ws_ext_sales_price#39, i_manufact_id#43]
+Input [4]: [ws_bill_addr_sk#38, ws_ext_sales_price#39, i_manufact_id#43, ca_address_sk#44]
+
+(60) HashAggregate [codegen id : 17]
+Input [2]: [ws_ext_sales_price#39, i_manufact_id#43]
+Keys [1]: [i_manufact_id#43]
+Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#39))]
+Aggregate Attributes [1]: [sum#45]
+Results [2]: [i_manufact_id#43, sum#46]
+
+(61) Exchange
+Input [2]: [i_manufact_id#43, sum#46]
+Arguments: hashpartitioning(i_manufact_id#43, 5), ENSURE_REQUIREMENTS, [id=#47]
+
+(62) HashAggregate [codegen id : 18]
+Input [2]: [i_manufact_id#43, sum#46]
+Keys [1]: [i_manufact_id#43]
+Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#39))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#39))#48]
+Results [2]: [i_manufact_id#43, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#39))#48,17,2) AS total_sales#49]
+
+(63) Union
+
+(64) HashAggregate [codegen id : 19]
+Input [2]: [i_manufact_id#11, total_sales#23]
+Keys [1]: [i_manufact_id#11]
+Functions [1]: [partial_sum(total_sales#23)]
+Aggregate Attributes [2]: [sum#50, isEmpty#51]
+Results [3]: [i_manufact_id#11, sum#52, isEmpty#53]
+
+(65) Exchange
+Input [3]: [i_manufact_id#11, sum#52, isEmpty#53]
+Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [id=#54]
+
+(66) HashAggregate [codegen id : 20]
+Input [3]: [i_manufact_id#11, sum#52, isEmpty#53]
+Keys [1]: [i_manufact_id#11]
+Functions [1]: [sum(total_sales#23)]
+Aggregate Attributes [1]: [sum(total_sales#23)#55]
+Results [2]: [i_manufact_id#11, sum(total_sales#23)#55 AS total_sales#56]
+
+(67) TakeOrderedAndProject
+Input [2]: [i_manufact_id#11, total_sales#56]
+Arguments: 100, [total_sales#56 ASC NULLS FIRST], [i_manufact_id#11, total_sales#56]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (68)
+
+
+(68) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#6]
+
+Subquery:2 Hosting operator id = 33 Hosting Expression = cs_sold_date_sk#27 IN dynamicpruning#5
+
+Subquery:3 Hosting operator id = 48 Hosting Expression = ws_sold_date_sk#40 IN dynamicpruning#5
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.sf100/simplified.txt
new file mode 100644
index 0000000000000..1b29ee6b28a9c
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.sf100/simplified.txt
@@ -0,0 +1,105 @@
+TakeOrderedAndProject [total_sales,i_manufact_id]
+  WholeStageCodegen (20)
+    HashAggregate [i_manufact_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty]
+      InputAdapter
+        Exchange [i_manufact_id] #1
+          WholeStageCodegen (19)
+            HashAggregate [i_manufact_id,total_sales] [sum,isEmpty,sum,isEmpty]
+              InputAdapter
+                Union
+                  WholeStageCodegen (6)
+                    HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum]
+                      InputAdapter
+                        Exchange [i_manufact_id] #2
+                          WholeStageCodegen (5)
+                            HashAggregate [i_manufact_id,ss_ext_sales_price] [sum,sum]
+                              Project [ss_ext_sales_price,i_manufact_id]
+                                BroadcastHashJoin [ss_addr_sk,ca_address_sk]
+                                  Project [ss_addr_sk,ss_ext_sales_price,i_manufact_id]
+                                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                      Project [ss_item_sk,ss_addr_sk,ss_ext_sales_price]
+                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                          Filter [ss_addr_sk,ss_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                                  SubqueryBroadcast [d_date_sk] #1
+                                                    ReusedExchange [d_date_sk] #3
+                                          InputAdapter
+                                            BroadcastExchange #3
+                                              WholeStageCodegen (1)
+                                                Project [d_date_sk]
+                                                  Filter [d_year,d_moy,d_date_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                      InputAdapter
+                                        BroadcastExchange #4
+                                          WholeStageCodegen (3)
+                                            BroadcastHashJoin [i_manufact_id,i_manufact_id]
+                                              Filter [i_item_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.item [i_item_sk,i_manufact_id]
+                                              InputAdapter
+                                                BroadcastExchange #5
+                                                  WholeStageCodegen (2)
+                                                    Project [i_manufact_id]
+                                                      Filter [i_category]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.item [i_category,i_manufact_id]
+                                  InputAdapter
+                                    BroadcastExchange #6
+                                      WholeStageCodegen (4)
+                                        Project [ca_address_sk]
+                                          Filter [ca_gmt_offset,ca_address_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.customer_address [ca_address_sk,ca_gmt_offset]
+                  WholeStageCodegen (12)
+                    HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum]
+                      InputAdapter
+                        Exchange [i_manufact_id] #7
+                          WholeStageCodegen (11)
+                            HashAggregate [i_manufact_id,cs_ext_sales_price] [sum,sum]
+                              Project [cs_ext_sales_price,i_manufact_id]
+                                BroadcastHashJoin [cs_bill_addr_sk,ca_address_sk]
+                                  Project [cs_bill_addr_sk,cs_ext_sales_price,i_manufact_id]
+                                    BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                      Project [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price]
+                                        BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                          Filter [cs_bill_addr_sk,cs_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk]
+                                                  ReusedSubquery [d_date_sk] #1
+                                          InputAdapter
+                                            ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        ReusedExchange [i_item_sk,i_manufact_id] #4
+                                  InputAdapter
+                                    ReusedExchange [ca_address_sk] #6
+                  WholeStageCodegen (18)
+                    HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum]
+                      InputAdapter
+                        Exchange [i_manufact_id] #8
+                          WholeStageCodegen (17)
+                            HashAggregate [i_manufact_id,ws_ext_sales_price] [sum,sum]
+                              Project [ws_ext_sales_price,i_manufact_id]
+                                BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk]
+                                  Project [ws_bill_addr_sk,ws_ext_sales_price,i_manufact_id]
+                                    BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                      Project [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price]
+                                        BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                          Filter [ws_bill_addr_sk,ws_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk]
+                                                  ReusedSubquery [d_date_sk] #1
+                                          InputAdapter
+                                            ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        ReusedExchange [i_item_sk,i_manufact_id] #4
+                                  InputAdapter
+                                    ReusedExchange [ca_address_sk] #6
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/explain.txt
new file mode 100644
index 0000000000000..eeda8611876d6
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/explain.txt
@@ -0,0 +1,395 @@
+== Physical Plan ==
+TakeOrderedAndProject (67)
++- * HashAggregate (66)
+   +- Exchange (65)
+      +- * HashAggregate (64)
+         +- Union (63)
+            :- * HashAggregate (32)
+            :  +- Exchange (31)
+            :     +- * HashAggregate (30)
+            :        +- * Project (29)
+            :           +- * BroadcastHashJoin Inner BuildRight (28)
+            :              :- * Project (17)
+            :              :  +- * BroadcastHashJoin Inner BuildRight (16)
+            :              :     :- * Project (10)
+            :              :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+            :              :     :     :- * Filter (3)
+            :              :     :     :  +- * ColumnarToRow (2)
+            :              :     :     :     +- Scan parquet default.store_sales (1)
+            :              :     :     +- BroadcastExchange (8)
+            :              :     :        +- * Project (7)
+            :              :     :           +- * Filter (6)
+            :              :     :              +- * ColumnarToRow (5)
+            :              :     :                 +- Scan parquet default.date_dim (4)
+            :              :     +- BroadcastExchange (15)
+            :              :        +- * Project (14)
+            :              :           +- * Filter (13)
+            :              :              +- * ColumnarToRow (12)
+            :              :                 +- Scan parquet default.customer_address (11)
+            :              +- BroadcastExchange (27)
+            :                 +- * BroadcastHashJoin LeftSemi BuildRight (26)
+            :                    :- * Filter (20)
+            :                    :  +- * ColumnarToRow (19)
+            :                    :     +- Scan parquet default.item (18)
+            :                    +- BroadcastExchange (25)
+            :                       +- * Project (24)
+            :                          +- * Filter (23)
+            :                             +- * ColumnarToRow (22)
+            :                                +- Scan parquet default.item (21)
+            :- * HashAggregate (47)
+            :  +- Exchange (46)
+            :     +- * HashAggregate (45)
+            :        +- * Project (44)
+            :           +- * BroadcastHashJoin Inner BuildRight (43)
+            :              :- * Project (41)
+            :              :  +- * BroadcastHashJoin Inner BuildRight (40)
+            :              :     :- * Project (38)
+            :              :     :  +- * BroadcastHashJoin Inner BuildRight (37)
+            :              :     :     :- * Filter (35)
+            :              :     :     :  +- * ColumnarToRow (34)
+            :              :     :     :     +- Scan parquet default.catalog_sales (33)
+            :              :     :     +- ReusedExchange (36)
+            :              :     +- ReusedExchange (39)
+            :              +- ReusedExchange (42)
+            +- * HashAggregate (62)
+               +- Exchange (61)
+                  +- * HashAggregate (60)
+                     +- * Project (59)
+                        +- * BroadcastHashJoin Inner BuildRight (58)
+                           :- * Project (56)
+                           :  +- * BroadcastHashJoin Inner BuildRight (55)
+                           :     :- * Project (53)
+                           :     :  +- * BroadcastHashJoin Inner BuildRight (52)
+                           :     :     :- * Filter (50)
+                           :     :     :  +- * ColumnarToRow (49)
+                           :     :     :     +- Scan parquet default.web_sales (48)
+                           :     :     +- ReusedExchange (51)
+                           :     +- ReusedExchange (54)
+                           +- ReusedExchange (57)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_addr_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 5]
+Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4]
+Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,5), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 1998)) AND (d_moy#8 = 5)) AND isnotnull(d_date_sk#6))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#6]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#9]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3]
+Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#6]
+
+(11) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#10, ca_gmt_offset#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_gmt_offset:decimal(5,2)>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [ca_address_sk#10, ca_gmt_offset#11]
+
+(13) Filter [codegen id : 2]
+Input [2]: [ca_address_sk#10, ca_gmt_offset#11]
+Condition : ((isnotnull(ca_gmt_offset#11) AND (ca_gmt_offset#11 = -5.00)) AND isnotnull(ca_address_sk#10))
+
+(14) Project [codegen id : 2]
+Output [1]: [ca_address_sk#10]
+Input [2]: [ca_address_sk#10, ca_gmt_offset#11]
+
+(15) BroadcastExchange
+Input [1]: [ca_address_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(16) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_addr_sk#2]
+Right keys [1]: [ca_address_sk#10]
+Join condition: None
+
+(17) Project [codegen id : 5]
+Output [2]: [ss_item_sk#1, ss_ext_sales_price#3]
+Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#10]
+
+(18) Scan parquet default.item
+Output [2]: [i_item_sk#13, i_manufact_id#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_manufact_id:int>
+
+(19) ColumnarToRow [codegen id : 4]
+Input [2]: [i_item_sk#13, i_manufact_id#14]
+
+(20) Filter [codegen id : 4]
+Input [2]: [i_item_sk#13, i_manufact_id#14]
+Condition : isnotnull(i_item_sk#13)
+
+(21) Scan parquet default.item
+Output [2]: [i_category#15, i_manufact_id#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Electronics                                       )]
+ReadSchema: struct<i_category:string,i_manufact_id:int>
+
+(22) ColumnarToRow [codegen id : 3]
+Input [2]: [i_category#15, i_manufact_id#16]
+
+(23) Filter [codegen id : 3]
+Input [2]: [i_category#15, i_manufact_id#16]
+Condition : (isnotnull(i_category#15) AND (i_category#15 = Electronics                                       ))
+
+(24) Project [codegen id : 3]
+Output [1]: [i_manufact_id#16]
+Input [2]: [i_category#15, i_manufact_id#16]
+
+(25) BroadcastExchange
+Input [1]: [i_manufact_id#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(26) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [i_manufact_id#14]
+Right keys [1]: [i_manufact_id#16]
+Join condition: None
+
+(27) BroadcastExchange
+Input [2]: [i_item_sk#13, i_manufact_id#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#18]
+
+(28) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#13]
+Join condition: None
+
+(29) Project [codegen id : 5]
+Output [2]: [ss_ext_sales_price#3, i_manufact_id#14]
+Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#13, i_manufact_id#14]
+
+(30) HashAggregate [codegen id : 5]
+Input [2]: [ss_ext_sales_price#3, i_manufact_id#14]
+Keys [1]: [i_manufact_id#14]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))]
+Aggregate Attributes [1]: [sum#19]
+Results [2]: [i_manufact_id#14, sum#20]
+
+(31) Exchange
+Input [2]: [i_manufact_id#14, sum#20]
+Arguments: hashpartitioning(i_manufact_id#14, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(32) HashAggregate [codegen id : 6]
+Input [2]: [i_manufact_id#14, sum#20]
+Keys [1]: [i_manufact_id#14]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#22]
+Results [2]: [i_manufact_id#14, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#22,17,2) AS total_sales#23]
+
+(33) Scan parquet default.catalog_sales
+Output [4]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, cs_sold_date_sk#27]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#27), dynamicpruningexpression(cs_sold_date_sk#27 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_bill_addr_sk:int,cs_item_sk:int,cs_ext_sales_price:decimal(7,2)>
+
+(34) ColumnarToRow [codegen id : 11]
+Input [4]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, cs_sold_date_sk#27]
+
+(35) Filter [codegen id : 11]
+Input [4]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, cs_sold_date_sk#27]
+Condition : (isnotnull(cs_bill_addr_sk#24) AND isnotnull(cs_item_sk#25))
+
+(36) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#28]
+
+(37) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_sold_date_sk#27]
+Right keys [1]: [d_date_sk#28]
+Join condition: None
+
+(38) Project [codegen id : 11]
+Output [3]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26]
+Input [5]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, cs_sold_date_sk#27, d_date_sk#28]
+
+(39) ReusedExchange [Reuses operator id: 15]
+Output [1]: [ca_address_sk#29]
+
+(40) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_bill_addr_sk#24]
+Right keys [1]: [ca_address_sk#29]
+Join condition: None
+
+(41) Project [codegen id : 11]
+Output [2]: [cs_item_sk#25, cs_ext_sales_price#26]
+Input [4]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, ca_address_sk#29]
+
+(42) ReusedExchange [Reuses operator id: 27]
+Output [2]: [i_item_sk#30, i_manufact_id#31]
+
+(43) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_item_sk#25]
+Right keys [1]: [i_item_sk#30]
+Join condition: None
+
+(44) Project [codegen id : 11]
+Output [2]: [cs_ext_sales_price#26, i_manufact_id#31]
+Input [4]: [cs_item_sk#25, cs_ext_sales_price#26, i_item_sk#30, i_manufact_id#31]
+
+(45) HashAggregate [codegen id : 11]
+Input [2]: [cs_ext_sales_price#26, i_manufact_id#31]
+Keys [1]: [i_manufact_id#31]
+Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#26))]
+Aggregate Attributes [1]: [sum#32]
+Results [2]: [i_manufact_id#31, sum#33]
+
+(46) Exchange
+Input [2]: [i_manufact_id#31, sum#33]
+Arguments: hashpartitioning(i_manufact_id#31, 5), ENSURE_REQUIREMENTS, [id=#34]
+
+(47) HashAggregate [codegen id : 12]
+Input [2]: [i_manufact_id#31, sum#33]
+Keys [1]: [i_manufact_id#31]
+Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#26))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#26))#35]
+Results [2]: [i_manufact_id#31, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#26))#35,17,2) AS total_sales#36]
+
+(48) Scan parquet default.web_sales
+Output [4]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#40), dynamicpruningexpression(ws_sold_date_sk#40 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_bill_addr_sk:int,ws_ext_sales_price:decimal(7,2)>
+
+(49) ColumnarToRow [codegen id : 17]
+Input [4]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40]
+
+(50) Filter [codegen id : 17]
+Input [4]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40]
+Condition : (isnotnull(ws_bill_addr_sk#38) AND isnotnull(ws_item_sk#37))
+
+(51) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#41]
+
+(52) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_sold_date_sk#40]
+Right keys [1]: [d_date_sk#41]
+Join condition: None
+
+(53) Project [codegen id : 17]
+Output [3]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39]
+Input [5]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40, d_date_sk#41]
+
+(54) ReusedExchange [Reuses operator id: 15]
+Output [1]: [ca_address_sk#42]
+
+(55) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_bill_addr_sk#38]
+Right keys [1]: [ca_address_sk#42]
+Join condition: None
+
+(56) Project [codegen id : 17]
+Output [2]: [ws_item_sk#37, ws_ext_sales_price#39]
+Input [4]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ca_address_sk#42]
+
+(57) ReusedExchange [Reuses operator id: 27]
+Output [2]: [i_item_sk#43, i_manufact_id#44]
+
+(58) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_item_sk#37]
+Right keys [1]: [i_item_sk#43]
+Join condition: None
+
+(59) Project [codegen id : 17]
+Output [2]: [ws_ext_sales_price#39, i_manufact_id#44]
+Input [4]: [ws_item_sk#37, ws_ext_sales_price#39, i_item_sk#43, i_manufact_id#44]
+
+(60) HashAggregate [codegen id : 17]
+Input [2]: [ws_ext_sales_price#39, i_manufact_id#44]
+Keys [1]: [i_manufact_id#44]
+Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#39))]
+Aggregate Attributes [1]: [sum#45]
+Results [2]: [i_manufact_id#44, sum#46]
+
+(61) Exchange
+Input [2]: [i_manufact_id#44, sum#46]
+Arguments: hashpartitioning(i_manufact_id#44, 5), ENSURE_REQUIREMENTS, [id=#47]
+
+(62) HashAggregate [codegen id : 18]
+Input [2]: [i_manufact_id#44, sum#46]
+Keys [1]: [i_manufact_id#44]
+Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#39))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#39))#48]
+Results [2]: [i_manufact_id#44, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#39))#48,17,2) AS total_sales#49]
+
+(63) Union
+
+(64) HashAggregate [codegen id : 19]
+Input [2]: [i_manufact_id#14, total_sales#23]
+Keys [1]: [i_manufact_id#14]
+Functions [1]: [partial_sum(total_sales#23)]
+Aggregate Attributes [2]: [sum#50, isEmpty#51]
+Results [3]: [i_manufact_id#14, sum#52, isEmpty#53]
+
+(65) Exchange
+Input [3]: [i_manufact_id#14, sum#52, isEmpty#53]
+Arguments: hashpartitioning(i_manufact_id#14, 5), ENSURE_REQUIREMENTS, [id=#54]
+
+(66) HashAggregate [codegen id : 20]
+Input [3]: [i_manufact_id#14, sum#52, isEmpty#53]
+Keys [1]: [i_manufact_id#14]
+Functions [1]: [sum(total_sales#23)]
+Aggregate Attributes [1]: [sum(total_sales#23)#55]
+Results [2]: [i_manufact_id#14, sum(total_sales#23)#55 AS total_sales#56]
+
+(67) TakeOrderedAndProject
+Input [2]: [i_manufact_id#14, total_sales#56]
+Arguments: 100, [total_sales#56 ASC NULLS FIRST], [i_manufact_id#14, total_sales#56]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (68)
+
+
+(68) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#6]
+
+Subquery:2 Hosting operator id = 33 Hosting Expression = cs_sold_date_sk#27 IN dynamicpruning#5
+
+Subquery:3 Hosting operator id = 48 Hosting Expression = ws_sold_date_sk#40 IN dynamicpruning#5
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/simplified.txt
new file mode 100644
index 0000000000000..d1a115fa528d5
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/simplified.txt
@@ -0,0 +1,105 @@
+TakeOrderedAndProject [total_sales,i_manufact_id]
+  WholeStageCodegen (20)
+    HashAggregate [i_manufact_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty]
+      InputAdapter
+        Exchange [i_manufact_id] #1
+          WholeStageCodegen (19)
+            HashAggregate [i_manufact_id,total_sales] [sum,isEmpty,sum,isEmpty]
+              InputAdapter
+                Union
+                  WholeStageCodegen (6)
+                    HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum]
+                      InputAdapter
+                        Exchange [i_manufact_id] #2
+                          WholeStageCodegen (5)
+                            HashAggregate [i_manufact_id,ss_ext_sales_price] [sum,sum]
+                              Project [ss_ext_sales_price,i_manufact_id]
+                                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                  Project [ss_item_sk,ss_ext_sales_price]
+                                    BroadcastHashJoin [ss_addr_sk,ca_address_sk]
+                                      Project [ss_item_sk,ss_addr_sk,ss_ext_sales_price]
+                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                          Filter [ss_addr_sk,ss_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                                  SubqueryBroadcast [d_date_sk] #1
+                                                    ReusedExchange [d_date_sk] #3
+                                          InputAdapter
+                                            BroadcastExchange #3
+                                              WholeStageCodegen (1)
+                                                Project [d_date_sk]
+                                                  Filter [d_year,d_moy,d_date_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                      InputAdapter
+                                        BroadcastExchange #4
+                                          WholeStageCodegen (2)
+                                            Project [ca_address_sk]
+                                              Filter [ca_gmt_offset,ca_address_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.customer_address [ca_address_sk,ca_gmt_offset]
+                                  InputAdapter
+                                    BroadcastExchange #5
+                                      WholeStageCodegen (4)
+                                        BroadcastHashJoin [i_manufact_id,i_manufact_id]
+                                          Filter [i_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.item [i_item_sk,i_manufact_id]
+                                          InputAdapter
+                                            BroadcastExchange #6
+                                              WholeStageCodegen (3)
+                                                Project [i_manufact_id]
+                                                  Filter [i_category]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.item [i_category,i_manufact_id]
+                  WholeStageCodegen (12)
+                    HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum]
+                      InputAdapter
+                        Exchange [i_manufact_id] #7
+                          WholeStageCodegen (11)
+                            HashAggregate [i_manufact_id,cs_ext_sales_price] [sum,sum]
+                              Project [cs_ext_sales_price,i_manufact_id]
+                                BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                  Project [cs_item_sk,cs_ext_sales_price]
+                                    BroadcastHashJoin [cs_bill_addr_sk,ca_address_sk]
+                                      Project [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price]
+                                        BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                          Filter [cs_bill_addr_sk,cs_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk]
+                                                  ReusedSubquery [d_date_sk] #1
+                                          InputAdapter
+                                            ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        ReusedExchange [ca_address_sk] #4
+                                  InputAdapter
+                                    ReusedExchange [i_item_sk,i_manufact_id] #5
+                  WholeStageCodegen (18)
+                    HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum]
+                      InputAdapter
+                        Exchange [i_manufact_id] #8
+                          WholeStageCodegen (17)
+                            HashAggregate [i_manufact_id,ws_ext_sales_price] [sum,sum]
+                              Project [ws_ext_sales_price,i_manufact_id]
+                                BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                  Project [ws_item_sk,ws_ext_sales_price]
+                                    BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk]
+                                      Project [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price]
+                                        BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                          Filter [ws_bill_addr_sk,ws_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk]
+                                                  ReusedSubquery [d_date_sk] #1
+                                          InputAdapter
+                                            ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        ReusedExchange [ca_address_sk] #4
+                                  InputAdapter
+                                    ReusedExchange [i_item_sk,i_manufact_id] #5
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.sf100/explain.txt
new file mode 100644
index 0000000000000..ea9994910c3b9
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.sf100/explain.txt
@@ -0,0 +1,229 @@
+== Physical Plan ==
+* Sort (39)
++- Exchange (38)
+   +- * Project (37)
+      +- * SortMergeJoin Inner (36)
+         :- * Sort (30)
+         :  +- Exchange (29)
+         :     +- * Filter (28)
+         :        +- * HashAggregate (27)
+         :           +- Exchange (26)
+         :              +- * HashAggregate (25)
+         :                 +- * Project (24)
+         :                    +- * BroadcastHashJoin Inner BuildRight (23)
+         :                       :- * Project (17)
+         :                       :  +- * BroadcastHashJoin Inner BuildRight (16)
+         :                       :     :- * Project (10)
+         :                       :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+         :                       :     :     :- * Filter (3)
+         :                       :     :     :  +- * ColumnarToRow (2)
+         :                       :     :     :     +- Scan parquet default.store_sales (1)
+         :                       :     :     +- BroadcastExchange (8)
+         :                       :     :        +- * Project (7)
+         :                       :     :           +- * Filter (6)
+         :                       :     :              +- * ColumnarToRow (5)
+         :                       :     :                 +- Scan parquet default.date_dim (4)
+         :                       :     +- BroadcastExchange (15)
+         :                       :        +- * Project (14)
+         :                       :           +- * Filter (13)
+         :                       :              +- * ColumnarToRow (12)
+         :                       :                 +- Scan parquet default.store (11)
+         :                       +- BroadcastExchange (22)
+         :                          +- * Project (21)
+         :                             +- * Filter (20)
+         :                                +- * ColumnarToRow (19)
+         :                                   +- Scan parquet default.household_demographics (18)
+         +- * Sort (35)
+            +- Exchange (34)
+               +- * Filter (33)
+                  +- * ColumnarToRow (32)
+                     +- Scan parquet default.customer (31)
+
+
+(1) Scan parquet default.store_sales
+Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_hdemo_sk:int,ss_store_sk:int,ss_ticket_number:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+
+(3) Filter [codegen id : 4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#7, d_year#8, d_dom#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_dom:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+Condition : (((((d_dom#9 >= 1) AND (d_dom#9 <= 3)) OR ((d_dom#9 >= 25) AND (d_dom#9 <= 28))) AND d_year#8 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#7]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#5]
+Right keys [1]: [d_date_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4]
+Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#7]
+
+(11) Scan parquet default.store
+Output [2]: [s_store_sk#11, s_county#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_county), EqualTo(s_county,Williamson County), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_county:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [s_store_sk#11, s_county#12]
+
+(13) Filter [codegen id : 2]
+Input [2]: [s_store_sk#11, s_county#12]
+Condition : ((isnotnull(s_county#12) AND (s_county#12 = Williamson County)) AND isnotnull(s_store_sk#11))
+
+(14) Project [codegen id : 2]
+Output [1]: [s_store_sk#11]
+Input [2]: [s_store_sk#11, s_county#12]
+
+(15) BroadcastExchange
+Input [1]: [s_store_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#11]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#11]
+
+(18) Scan parquet default.household_demographics
+Output [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_vehicle_count), IsNotNull(hd_dep_count), Or(EqualTo(hd_buy_potential,>10000         ),EqualTo(hd_buy_potential,unknown        )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_buy_potential:string,hd_dep_count:int,hd_vehicle_count:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+
+(20) Filter [codegen id : 3]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+Condition : (((((isnotnull(hd_vehicle_count#17) AND isnotnull(hd_dep_count#16)) AND ((hd_buy_potential#15 = >10000         ) OR (hd_buy_potential#15 = unknown        ))) AND (hd_vehicle_count#17 > 0)) AND ((cast(hd_dep_count#16 as double) / cast(hd_vehicle_count#17 as double)) > 1.2)) AND isnotnull(hd_demo_sk#14))
+
+(21) Project [codegen id : 3]
+Output [1]: [hd_demo_sk#14]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+
+(22) BroadcastExchange
+Input [1]: [hd_demo_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(23) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#14]
+Join condition: None
+
+(24) Project [codegen id : 4]
+Output [2]: [ss_customer_sk#1, ss_ticket_number#4]
+Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#14]
+
+(25) HashAggregate [codegen id : 4]
+Input [2]: [ss_customer_sk#1, ss_ticket_number#4]
+Keys [2]: [ss_ticket_number#4, ss_customer_sk#1]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#19]
+Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+
+(26) Exchange
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(27) HashAggregate [codegen id : 5]
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+Keys [2]: [ss_ticket_number#4, ss_customer_sk#1]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#22]
+Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count(1)#22 AS cnt#23]
+
+(28) Filter [codegen id : 5]
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23]
+Condition : ((cnt#23 >= 15) AND (cnt#23 <= 20))
+
+(29) Exchange
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23]
+Arguments: hashpartitioning(ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(30) Sort [codegen id : 6]
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23]
+Arguments: [ss_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(31) Scan parquet default.customer
+Output [5]: [c_customer_sk#25, c_salutation#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_salutation:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string>
+
+(32) ColumnarToRow [codegen id : 7]
+Input [5]: [c_customer_sk#25, c_salutation#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29]
+
+(33) Filter [codegen id : 7]
+Input [5]: [c_customer_sk#25, c_salutation#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29]
+Condition : isnotnull(c_customer_sk#25)
+
+(34) Exchange
+Input [5]: [c_customer_sk#25, c_salutation#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29]
+Arguments: hashpartitioning(c_customer_sk#25, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(35) Sort [codegen id : 8]
+Input [5]: [c_customer_sk#25, c_salutation#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29]
+Arguments: [c_customer_sk#25 ASC NULLS FIRST], false, 0
+
+(36) SortMergeJoin [codegen id : 9]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#25]
+Join condition: None
+
+(37) Project [codegen id : 9]
+Output [6]: [c_last_name#28, c_first_name#27, c_salutation#26, c_preferred_cust_flag#29, ss_ticket_number#4, cnt#23]
+Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23, c_customer_sk#25, c_salutation#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29]
+
+(38) Exchange
+Input [6]: [c_last_name#28, c_first_name#27, c_salutation#26, c_preferred_cust_flag#29, ss_ticket_number#4, cnt#23]
+Arguments: rangepartitioning(c_last_name#28 ASC NULLS FIRST, c_first_name#27 ASC NULLS FIRST, c_salutation#26 ASC NULLS FIRST, c_preferred_cust_flag#29 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(39) Sort [codegen id : 10]
+Input [6]: [c_last_name#28, c_first_name#27, c_salutation#26, c_preferred_cust_flag#29, ss_ticket_number#4, cnt#23]
+Arguments: [c_last_name#28 ASC NULLS FIRST, c_first_name#27 ASC NULLS FIRST, c_salutation#26 ASC NULLS FIRST, c_preferred_cust_flag#29 DESC NULLS LAST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (40)
+
+
+(40) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#7]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.sf100/simplified.txt
new file mode 100644
index 0000000000000..24c5c1c256c26
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.sf100/simplified.txt
@@ -0,0 +1,65 @@
+WholeStageCodegen (10)
+  Sort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag]
+    InputAdapter
+      Exchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag] #1
+        WholeStageCodegen (9)
+          Project [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt]
+            SortMergeJoin [ss_customer_sk,c_customer_sk]
+              InputAdapter
+                WholeStageCodegen (6)
+                  Sort [ss_customer_sk]
+                    InputAdapter
+                      Exchange [ss_customer_sk] #2
+                        WholeStageCodegen (5)
+                          Filter [cnt]
+                            HashAggregate [ss_ticket_number,ss_customer_sk,count] [count(1),cnt,count]
+                              InputAdapter
+                                Exchange [ss_ticket_number,ss_customer_sk] #3
+                                  WholeStageCodegen (4)
+                                    HashAggregate [ss_ticket_number,ss_customer_sk] [count,count]
+                                      Project [ss_customer_sk,ss_ticket_number]
+                                        BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                          Project [ss_customer_sk,ss_hdemo_sk,ss_ticket_number]
+                                            BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                              Project [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number]
+                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                  Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk]
+                                                          SubqueryBroadcast [d_date_sk] #1
+                                                            ReusedExchange [d_date_sk] #4
+                                                  InputAdapter
+                                                    BroadcastExchange #4
+                                                      WholeStageCodegen (1)
+                                                        Project [d_date_sk]
+                                                          Filter [d_dom,d_year,d_date_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.date_dim [d_date_sk,d_year,d_dom]
+                                              InputAdapter
+                                                BroadcastExchange #5
+                                                  WholeStageCodegen (2)
+                                                    Project [s_store_sk]
+                                                      Filter [s_county,s_store_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.store [s_store_sk,s_county]
+                                          InputAdapter
+                                            BroadcastExchange #6
+                                              WholeStageCodegen (3)
+                                                Project [hd_demo_sk]
+                                                  Filter [hd_vehicle_count,hd_dep_count,hd_buy_potential,hd_demo_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count]
+              InputAdapter
+                WholeStageCodegen (8)
+                  Sort [c_customer_sk]
+                    InputAdapter
+                      Exchange [c_customer_sk] #7
+                        WholeStageCodegen (7)
+                          Filter [c_customer_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/explain.txt
new file mode 100644
index 0000000000000..df5b93da51771
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/explain.txt
@@ -0,0 +1,214 @@
+== Physical Plan ==
+* Sort (36)
++- Exchange (35)
+   +- * Project (34)
+      +- * BroadcastHashJoin Inner BuildRight (33)
+         :- * Filter (28)
+         :  +- * HashAggregate (27)
+         :     +- Exchange (26)
+         :        +- * HashAggregate (25)
+         :           +- * Project (24)
+         :              +- * BroadcastHashJoin Inner BuildRight (23)
+         :                 :- * Project (17)
+         :                 :  +- * BroadcastHashJoin Inner BuildRight (16)
+         :                 :     :- * Project (10)
+         :                 :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+         :                 :     :     :- * Filter (3)
+         :                 :     :     :  +- * ColumnarToRow (2)
+         :                 :     :     :     +- Scan parquet default.store_sales (1)
+         :                 :     :     +- BroadcastExchange (8)
+         :                 :     :        +- * Project (7)
+         :                 :     :           +- * Filter (6)
+         :                 :     :              +- * ColumnarToRow (5)
+         :                 :     :                 +- Scan parquet default.date_dim (4)
+         :                 :     +- BroadcastExchange (15)
+         :                 :        +- * Project (14)
+         :                 :           +- * Filter (13)
+         :                 :              +- * ColumnarToRow (12)
+         :                 :                 +- Scan parquet default.store (11)
+         :                 +- BroadcastExchange (22)
+         :                    +- * Project (21)
+         :                       +- * Filter (20)
+         :                          +- * ColumnarToRow (19)
+         :                             +- Scan parquet default.household_demographics (18)
+         +- BroadcastExchange (32)
+            +- * Filter (31)
+               +- * ColumnarToRow (30)
+                  +- Scan parquet default.customer (29)
+
+
+(1) Scan parquet default.store_sales
+Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_hdemo_sk:int,ss_store_sk:int,ss_ticket_number:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+
+(3) Filter [codegen id : 4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#7, d_year#8, d_dom#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_dom:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+Condition : (((((d_dom#9 >= 1) AND (d_dom#9 <= 3)) OR ((d_dom#9 >= 25) AND (d_dom#9 <= 28))) AND d_year#8 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#7]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#5]
+Right keys [1]: [d_date_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4]
+Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#7]
+
+(11) Scan parquet default.store
+Output [2]: [s_store_sk#11, s_county#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_county), EqualTo(s_county,Williamson County), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_county:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [s_store_sk#11, s_county#12]
+
+(13) Filter [codegen id : 2]
+Input [2]: [s_store_sk#11, s_county#12]
+Condition : ((isnotnull(s_county#12) AND (s_county#12 = Williamson County)) AND isnotnull(s_store_sk#11))
+
+(14) Project [codegen id : 2]
+Output [1]: [s_store_sk#11]
+Input [2]: [s_store_sk#11, s_county#12]
+
+(15) BroadcastExchange
+Input [1]: [s_store_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#11]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#11]
+
+(18) Scan parquet default.household_demographics
+Output [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_vehicle_count), IsNotNull(hd_dep_count), Or(EqualTo(hd_buy_potential,>10000         ),EqualTo(hd_buy_potential,unknown        )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_buy_potential:string,hd_dep_count:int,hd_vehicle_count:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+
+(20) Filter [codegen id : 3]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+Condition : (((((isnotnull(hd_vehicle_count#17) AND isnotnull(hd_dep_count#16)) AND ((hd_buy_potential#15 = >10000         ) OR (hd_buy_potential#15 = unknown        ))) AND (hd_vehicle_count#17 > 0)) AND ((cast(hd_dep_count#16 as double) / cast(hd_vehicle_count#17 as double)) > 1.2)) AND isnotnull(hd_demo_sk#14))
+
+(21) Project [codegen id : 3]
+Output [1]: [hd_demo_sk#14]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+
+(22) BroadcastExchange
+Input [1]: [hd_demo_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(23) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#14]
+Join condition: None
+
+(24) Project [codegen id : 4]
+Output [2]: [ss_customer_sk#1, ss_ticket_number#4]
+Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#14]
+
+(25) HashAggregate [codegen id : 4]
+Input [2]: [ss_customer_sk#1, ss_ticket_number#4]
+Keys [2]: [ss_ticket_number#4, ss_customer_sk#1]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#19]
+Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+
+(26) Exchange
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(27) HashAggregate [codegen id : 6]
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+Keys [2]: [ss_ticket_number#4, ss_customer_sk#1]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#22]
+Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count(1)#22 AS cnt#23]
+
+(28) Filter [codegen id : 6]
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23]
+Condition : ((cnt#23 >= 15) AND (cnt#23 <= 20))
+
+(29) Scan parquet default.customer
+Output [5]: [c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_salutation:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string>
+
+(30) ColumnarToRow [codegen id : 5]
+Input [5]: [c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+
+(31) Filter [codegen id : 5]
+Input [5]: [c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+Condition : isnotnull(c_customer_sk#24)
+
+(32) BroadcastExchange
+Input [5]: [c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#29]
+
+(33) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#24]
+Join condition: None
+
+(34) Project [codegen id : 6]
+Output [6]: [c_last_name#27, c_first_name#26, c_salutation#25, c_preferred_cust_flag#28, ss_ticket_number#4, cnt#23]
+Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23, c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+
+(35) Exchange
+Input [6]: [c_last_name#27, c_first_name#26, c_salutation#25, c_preferred_cust_flag#28, ss_ticket_number#4, cnt#23]
+Arguments: rangepartitioning(c_last_name#27 ASC NULLS FIRST, c_first_name#26 ASC NULLS FIRST, c_salutation#25 ASC NULLS FIRST, c_preferred_cust_flag#28 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(36) Sort [codegen id : 7]
+Input [6]: [c_last_name#27, c_first_name#26, c_salutation#25, c_preferred_cust_flag#28, ss_ticket_number#4, cnt#23]
+Arguments: [c_last_name#27 ASC NULLS FIRST, c_first_name#26 ASC NULLS FIRST, c_salutation#25 ASC NULLS FIRST, c_preferred_cust_flag#28 DESC NULLS LAST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (37)
+
+
+(37) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#7]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/simplified.txt
new file mode 100644
index 0000000000000..e4f89ab27658b
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/simplified.txt
@@ -0,0 +1,56 @@
+WholeStageCodegen (7)
+  Sort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag]
+    InputAdapter
+      Exchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag] #1
+        WholeStageCodegen (6)
+          Project [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt]
+            BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+              Filter [cnt]
+                HashAggregate [ss_ticket_number,ss_customer_sk,count] [count(1),cnt,count]
+                  InputAdapter
+                    Exchange [ss_ticket_number,ss_customer_sk] #2
+                      WholeStageCodegen (4)
+                        HashAggregate [ss_ticket_number,ss_customer_sk] [count,count]
+                          Project [ss_customer_sk,ss_ticket_number]
+                            BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                              Project [ss_customer_sk,ss_hdemo_sk,ss_ticket_number]
+                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                  Project [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number]
+                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                      Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #1
+                                                ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        BroadcastExchange #3
+                                          WholeStageCodegen (1)
+                                            Project [d_date_sk]
+                                              Filter [d_dom,d_year,d_date_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_date_sk,d_year,d_dom]
+                                  InputAdapter
+                                    BroadcastExchange #4
+                                      WholeStageCodegen (2)
+                                        Project [s_store_sk]
+                                          Filter [s_county,s_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store [s_store_sk,s_county]
+                              InputAdapter
+                                BroadcastExchange #5
+                                  WholeStageCodegen (3)
+                                    Project [hd_demo_sk]
+                                      Filter [hd_vehicle_count,hd_dep_count,hd_buy_potential,hd_demo_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count]
+              InputAdapter
+                BroadcastExchange #6
+                  WholeStageCodegen (5)
+                    Filter [c_customer_sk]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.sf100/explain.txt
new file mode 100644
index 0000000000000..9ed490449b008
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.sf100/explain.txt
@@ -0,0 +1,328 @@
+== Physical Plan ==
+TakeOrderedAndProject (57)
++- * HashAggregate (56)
+   +- Exchange (55)
+      +- * HashAggregate (54)
+         +- * Project (53)
+            +- * SortMergeJoin Inner (52)
+               :- * Sort (46)
+               :  +- Exchange (45)
+               :     +- * Project (44)
+               :        +- * SortMergeJoin Inner (43)
+               :           :- * Sort (37)
+               :           :  +- Exchange (36)
+               :           :     +- * Project (35)
+               :           :        +- * Filter (34)
+               :           :           +- SortMergeJoin ExistenceJoin(exists#1) (33)
+               :           :              :- SortMergeJoin ExistenceJoin(exists#2) (25)
+               :           :              :  :- * SortMergeJoin LeftSemi (17)
+               :           :              :  :  :- * Sort (5)
+               :           :              :  :  :  +- Exchange (4)
+               :           :              :  :  :     +- * Filter (3)
+               :           :              :  :  :        +- * ColumnarToRow (2)
+               :           :              :  :  :           +- Scan parquet default.customer (1)
+               :           :              :  :  +- * Sort (16)
+               :           :              :  :     +- Exchange (15)
+               :           :              :  :        +- * Project (14)
+               :           :              :  :           +- * BroadcastHashJoin Inner BuildRight (13)
+               :           :              :  :              :- * ColumnarToRow (7)
+               :           :              :  :              :  +- Scan parquet default.store_sales (6)
+               :           :              :  :              +- BroadcastExchange (12)
+               :           :              :  :                 +- * Project (11)
+               :           :              :  :                    +- * Filter (10)
+               :           :              :  :                       +- * ColumnarToRow (9)
+               :           :              :  :                          +- Scan parquet default.date_dim (8)
+               :           :              :  +- * Sort (24)
+               :           :              :     +- Exchange (23)
+               :           :              :        +- * Project (22)
+               :           :              :           +- * BroadcastHashJoin Inner BuildRight (21)
+               :           :              :              :- * ColumnarToRow (19)
+               :           :              :              :  +- Scan parquet default.web_sales (18)
+               :           :              :              +- ReusedExchange (20)
+               :           :              +- * Sort (32)
+               :           :                 +- Exchange (31)
+               :           :                    +- * Project (30)
+               :           :                       +- * BroadcastHashJoin Inner BuildRight (29)
+               :           :                          :- * ColumnarToRow (27)
+               :           :                          :  +- Scan parquet default.catalog_sales (26)
+               :           :                          +- ReusedExchange (28)
+               :           +- * Sort (42)
+               :              +- Exchange (41)
+               :                 +- * Filter (40)
+               :                    +- * ColumnarToRow (39)
+               :                       +- Scan parquet default.customer_address (38)
+               +- * Sort (51)
+                  +- Exchange (50)
+                     +- * Filter (49)
+                        +- * ColumnarToRow (48)
+                           +- Scan parquet default.customer_demographics (47)
+
+
+(1) Scan parquet default.customer
+Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_addr_sk:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5]
+
+(3) Filter [codegen id : 1]
+Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5]
+Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4))
+
+(4) Exchange
+Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5]
+Arguments: hashpartitioning(c_customer_sk#3, 5), ENSURE_REQUIREMENTS, [id=#6]
+
+(5) Sort [codegen id : 2]
+Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5]
+Arguments: [c_customer_sk#3 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.store_sales
+Output [2]: [ss_customer_sk#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+ReadSchema: struct<ss_customer_sk:int>
+
+(7) ColumnarToRow [codegen id : 4]
+Input [2]: [ss_customer_sk#7, ss_sold_date_sk#8]
+
+(8) Scan parquet default.date_dim
+Output [3]: [d_date_sk#10, d_year#11, d_qoy#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_qoy:int>
+
+(9) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#10, d_year#11, d_qoy#12]
+
+(10) Filter [codegen id : 3]
+Input [3]: [d_date_sk#10, d_year#11, d_qoy#12]
+Condition : ((((isnotnull(d_year#11) AND isnotnull(d_qoy#12)) AND (d_year#11 = 2002)) AND (d_qoy#12 < 4)) AND isnotnull(d_date_sk#10))
+
+(11) Project [codegen id : 3]
+Output [1]: [d_date_sk#10]
+Input [3]: [d_date_sk#10, d_year#11, d_qoy#12]
+
+(12) BroadcastExchange
+Input [1]: [d_date_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(13) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(14) Project [codegen id : 4]
+Output [1]: [ss_customer_sk#7]
+Input [3]: [ss_customer_sk#7, ss_sold_date_sk#8, d_date_sk#10]
+
+(15) Exchange
+Input [1]: [ss_customer_sk#7]
+Arguments: hashpartitioning(ss_customer_sk#7, 5), ENSURE_REQUIREMENTS, [id=#14]
+
+(16) Sort [codegen id : 5]
+Input [1]: [ss_customer_sk#7]
+Arguments: [ss_customer_sk#7 ASC NULLS FIRST], false, 0
+
+(17) SortMergeJoin [codegen id : 6]
+Left keys [1]: [c_customer_sk#3]
+Right keys [1]: [ss_customer_sk#7]
+Join condition: None
+
+(18) Scan parquet default.web_sales
+Output [2]: [ws_bill_customer_sk#15, ws_sold_date_sk#16]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#16), dynamicpruningexpression(ws_sold_date_sk#16 IN dynamicpruning#9)]
+ReadSchema: struct<ws_bill_customer_sk:int>
+
+(19) ColumnarToRow [codegen id : 8]
+Input [2]: [ws_bill_customer_sk#15, ws_sold_date_sk#16]
+
+(20) ReusedExchange [Reuses operator id: 12]
+Output [1]: [d_date_sk#17]
+
+(21) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ws_sold_date_sk#16]
+Right keys [1]: [d_date_sk#17]
+Join condition: None
+
+(22) Project [codegen id : 8]
+Output [1]: [ws_bill_customer_sk#15]
+Input [3]: [ws_bill_customer_sk#15, ws_sold_date_sk#16, d_date_sk#17]
+
+(23) Exchange
+Input [1]: [ws_bill_customer_sk#15]
+Arguments: hashpartitioning(ws_bill_customer_sk#15, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(24) Sort [codegen id : 9]
+Input [1]: [ws_bill_customer_sk#15]
+Arguments: [ws_bill_customer_sk#15 ASC NULLS FIRST], false, 0
+
+(25) SortMergeJoin
+Left keys [1]: [c_customer_sk#3]
+Right keys [1]: [ws_bill_customer_sk#15]
+Join condition: None
+
+(26) Scan parquet default.catalog_sales
+Output [2]: [cs_ship_customer_sk#19, cs_sold_date_sk#20]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#20), dynamicpruningexpression(cs_sold_date_sk#20 IN dynamicpruning#9)]
+ReadSchema: struct<cs_ship_customer_sk:int>
+
+(27) ColumnarToRow [codegen id : 11]
+Input [2]: [cs_ship_customer_sk#19, cs_sold_date_sk#20]
+
+(28) ReusedExchange [Reuses operator id: 12]
+Output [1]: [d_date_sk#21]
+
+(29) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_sold_date_sk#20]
+Right keys [1]: [d_date_sk#21]
+Join condition: None
+
+(30) Project [codegen id : 11]
+Output [1]: [cs_ship_customer_sk#19]
+Input [3]: [cs_ship_customer_sk#19, cs_sold_date_sk#20, d_date_sk#21]
+
+(31) Exchange
+Input [1]: [cs_ship_customer_sk#19]
+Arguments: hashpartitioning(cs_ship_customer_sk#19, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(32) Sort [codegen id : 12]
+Input [1]: [cs_ship_customer_sk#19]
+Arguments: [cs_ship_customer_sk#19 ASC NULLS FIRST], false, 0
+
+(33) SortMergeJoin
+Left keys [1]: [c_customer_sk#3]
+Right keys [1]: [cs_ship_customer_sk#19]
+Join condition: None
+
+(34) Filter [codegen id : 13]
+Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1]
+Condition : (exists#2 OR exists#1)
+
+(35) Project [codegen id : 13]
+Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5]
+Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1]
+
+(36) Exchange
+Input [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5]
+Arguments: hashpartitioning(c_current_addr_sk#5, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(37) Sort [codegen id : 14]
+Input [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5]
+Arguments: [c_current_addr_sk#5 ASC NULLS FIRST], false, 0
+
+(38) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#24, ca_state#25]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(39) ColumnarToRow [codegen id : 15]
+Input [2]: [ca_address_sk#24, ca_state#25]
+
+(40) Filter [codegen id : 15]
+Input [2]: [ca_address_sk#24, ca_state#25]
+Condition : isnotnull(ca_address_sk#24)
+
+(41) Exchange
+Input [2]: [ca_address_sk#24, ca_state#25]
+Arguments: hashpartitioning(ca_address_sk#24, 5), ENSURE_REQUIREMENTS, [id=#26]
+
+(42) Sort [codegen id : 16]
+Input [2]: [ca_address_sk#24, ca_state#25]
+Arguments: [ca_address_sk#24 ASC NULLS FIRST], false, 0
+
+(43) SortMergeJoin [codegen id : 17]
+Left keys [1]: [c_current_addr_sk#5]
+Right keys [1]: [ca_address_sk#24]
+Join condition: None
+
+(44) Project [codegen id : 17]
+Output [2]: [c_current_cdemo_sk#4, ca_state#25]
+Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#24, ca_state#25]
+
+(45) Exchange
+Input [2]: [c_current_cdemo_sk#4, ca_state#25]
+Arguments: hashpartitioning(c_current_cdemo_sk#4, 5), ENSURE_REQUIREMENTS, [id=#27]
+
+(46) Sort [codegen id : 18]
+Input [2]: [c_current_cdemo_sk#4, ca_state#25]
+Arguments: [c_current_cdemo_sk#4 ASC NULLS FIRST], false, 0
+
+(47) Scan parquet default.customer_demographics
+Output [6]: [cd_demo_sk#28, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_dep_count:int,cd_dep_employed_count:int,cd_dep_college_count:int>
+
+(48) ColumnarToRow [codegen id : 19]
+Input [6]: [cd_demo_sk#28, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+
+(49) Filter [codegen id : 19]
+Input [6]: [cd_demo_sk#28, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Condition : isnotnull(cd_demo_sk#28)
+
+(50) Exchange
+Input [6]: [cd_demo_sk#28, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Arguments: hashpartitioning(cd_demo_sk#28, 5), ENSURE_REQUIREMENTS, [id=#34]
+
+(51) Sort [codegen id : 20]
+Input [6]: [cd_demo_sk#28, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Arguments: [cd_demo_sk#28 ASC NULLS FIRST], false, 0
+
+(52) SortMergeJoin [codegen id : 21]
+Left keys [1]: [c_current_cdemo_sk#4]
+Right keys [1]: [cd_demo_sk#28]
+Join condition: None
+
+(53) Project [codegen id : 21]
+Output [6]: [ca_state#25, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Input [8]: [c_current_cdemo_sk#4, ca_state#25, cd_demo_sk#28, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+
+(54) HashAggregate [codegen id : 21]
+Input [6]: [ca_state#25, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Keys [6]: [ca_state#25, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Functions [10]: [partial_count(1), partial_min(cd_dep_count#31), partial_max(cd_dep_count#31), partial_avg(cd_dep_count#31), partial_min(cd_dep_employed_count#32), partial_max(cd_dep_employed_count#32), partial_avg(cd_dep_employed_count#32), partial_min(cd_dep_college_count#33), partial_max(cd_dep_college_count#33), partial_avg(cd_dep_college_count#33)]
+Aggregate Attributes [13]: [count#35, min#36, max#37, sum#38, count#39, min#40, max#41, sum#42, count#43, min#44, max#45, sum#46, count#47]
+Results [19]: [ca_state#25, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33, count#48, min#49, max#50, sum#51, count#52, min#53, max#54, sum#55, count#56, min#57, max#58, sum#59, count#60]
+
+(55) Exchange
+Input [19]: [ca_state#25, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33, count#48, min#49, max#50, sum#51, count#52, min#53, max#54, sum#55, count#56, min#57, max#58, sum#59, count#60]
+Arguments: hashpartitioning(ca_state#25, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33, 5), ENSURE_REQUIREMENTS, [id=#61]
+
+(56) HashAggregate [codegen id : 22]
+Input [19]: [ca_state#25, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33, count#48, min#49, max#50, sum#51, count#52, min#53, max#54, sum#55, count#56, min#57, max#58, sum#59, count#60]
+Keys [6]: [ca_state#25, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Functions [10]: [count(1), min(cd_dep_count#31), max(cd_dep_count#31), avg(cd_dep_count#31), min(cd_dep_employed_count#32), max(cd_dep_employed_count#32), avg(cd_dep_employed_count#32), min(cd_dep_college_count#33), max(cd_dep_college_count#33), avg(cd_dep_college_count#33)]
+Aggregate Attributes [10]: [count(1)#62, min(cd_dep_count#31)#63, max(cd_dep_count#31)#64, avg(cd_dep_count#31)#65, min(cd_dep_employed_count#32)#66, max(cd_dep_employed_count#32)#67, avg(cd_dep_employed_count#32)#68, min(cd_dep_college_count#33)#69, max(cd_dep_college_count#33)#70, avg(cd_dep_college_count#33)#71]
+Results [18]: [ca_state#25, cd_gender#29, cd_marital_status#30, count(1)#62 AS cnt1#72, min(cd_dep_count#31)#63 AS min(cd_dep_count)#73, max(cd_dep_count#31)#64 AS max(cd_dep_count)#74, avg(cd_dep_count#31)#65 AS avg(cd_dep_count)#75, cd_dep_employed_count#32, count(1)#62 AS cnt2#76, min(cd_dep_employed_count#32)#66 AS min(cd_dep_employed_count)#77, max(cd_dep_employed_count#32)#67 AS max(cd_dep_employed_count)#78, avg(cd_dep_employed_count#32)#68 AS avg(cd_dep_employed_count)#79, cd_dep_college_count#33, count(1)#62 AS cnt3#80, min(cd_dep_college_count#33)#69 AS min(cd_dep_college_count)#81, max(cd_dep_college_count#33)#70 AS max(cd_dep_college_count)#82, avg(cd_dep_college_count#33)#71 AS avg(cd_dep_college_count)#83, cd_dep_count#31 AS aggOrder#84]
+
+(57) TakeOrderedAndProject
+Input [18]: [ca_state#25, cd_gender#29, cd_marital_status#30, cnt1#72, min(cd_dep_count)#73, max(cd_dep_count)#74, avg(cd_dep_count)#75, cd_dep_employed_count#32, cnt2#76, min(cd_dep_employed_count)#77, max(cd_dep_employed_count)#78, avg(cd_dep_employed_count)#79, cd_dep_college_count#33, cnt3#80, min(cd_dep_college_count)#81, max(cd_dep_college_count)#82, avg(cd_dep_college_count)#83, aggOrder#84]
+Arguments: 100, [ca_state#25 ASC NULLS FIRST, cd_gender#29 ASC NULLS FIRST, cd_marital_status#30 ASC NULLS FIRST, aggOrder#84 ASC NULLS FIRST, cd_dep_employed_count#32 ASC NULLS FIRST, cd_dep_college_count#33 ASC NULLS FIRST], [ca_state#25, cd_gender#29, cd_marital_status#30, cnt1#72, min(cd_dep_count)#73, max(cd_dep_count)#74, avg(cd_dep_count)#75, cd_dep_employed_count#32, cnt2#76, min(cd_dep_employed_count)#77, max(cd_dep_employed_count)#78, avg(cd_dep_employed_count)#79, cd_dep_college_count#33, cnt3#80, min(cd_dep_college_count)#81, max(cd_dep_college_count)#82, avg(cd_dep_college_count)#83]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 6 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (58)
+
+
+(58) ReusedExchange [Reuses operator id: 12]
+Output [1]: [d_date_sk#10]
+
+Subquery:2 Hosting operator id = 18 Hosting Expression = ws_sold_date_sk#16 IN dynamicpruning#9
+
+Subquery:3 Hosting operator id = 26 Hosting Expression = cs_sold_date_sk#20 IN dynamicpruning#9
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.sf100/simplified.txt
new file mode 100644
index 0000000000000..820be2b5fd58a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.sf100/simplified.txt
@@ -0,0 +1,107 @@
+TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,aggOrder,cd_dep_employed_count,cd_dep_college_count,cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count)]
+  WholeStageCodegen (22)
+    HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] [count(1),min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),aggOrder,count,min,max,sum,count,min,max,sum,count,min,max,sum,count]
+      InputAdapter
+        Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1
+          WholeStageCodegen (21)
+            HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,min,max,sum,count,min,max,sum,count,min,max,sum,count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count]
+              Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count]
+                SortMergeJoin [c_current_cdemo_sk,cd_demo_sk]
+                  InputAdapter
+                    WholeStageCodegen (18)
+                      Sort [c_current_cdemo_sk]
+                        InputAdapter
+                          Exchange [c_current_cdemo_sk] #2
+                            WholeStageCodegen (17)
+                              Project [c_current_cdemo_sk,ca_state]
+                                SortMergeJoin [c_current_addr_sk,ca_address_sk]
+                                  InputAdapter
+                                    WholeStageCodegen (14)
+                                      Sort [c_current_addr_sk]
+                                        InputAdapter
+                                          Exchange [c_current_addr_sk] #3
+                                            WholeStageCodegen (13)
+                                              Project [c_current_cdemo_sk,c_current_addr_sk]
+                                                Filter [exists,exists]
+                                                  InputAdapter
+                                                    SortMergeJoin [c_customer_sk,cs_ship_customer_sk]
+                                                      SortMergeJoin [c_customer_sk,ws_bill_customer_sk]
+                                                        WholeStageCodegen (6)
+                                                          SortMergeJoin [c_customer_sk,ss_customer_sk]
+                                                            InputAdapter
+                                                              WholeStageCodegen (2)
+                                                                Sort [c_customer_sk]
+                                                                  InputAdapter
+                                                                    Exchange [c_customer_sk] #4
+                                                                      WholeStageCodegen (1)
+                                                                        Filter [c_current_addr_sk,c_current_cdemo_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk]
+                                                            InputAdapter
+                                                              WholeStageCodegen (5)
+                                                                Sort [ss_customer_sk]
+                                                                  InputAdapter
+                                                                    Exchange [ss_customer_sk] #5
+                                                                      WholeStageCodegen (4)
+                                                                        Project [ss_customer_sk]
+                                                                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                            ColumnarToRow
+                                                                              InputAdapter
+                                                                                Scan parquet default.store_sales [ss_customer_sk,ss_sold_date_sk]
+                                                                                  SubqueryBroadcast [d_date_sk] #1
+                                                                                    ReusedExchange [d_date_sk] #6
+                                                                            InputAdapter
+                                                                              BroadcastExchange #6
+                                                                                WholeStageCodegen (3)
+                                                                                  Project [d_date_sk]
+                                                                                    Filter [d_year,d_qoy,d_date_sk]
+                                                                                      ColumnarToRow
+                                                                                        InputAdapter
+                                                                                          Scan parquet default.date_dim [d_date_sk,d_year,d_qoy]
+                                                        WholeStageCodegen (9)
+                                                          Sort [ws_bill_customer_sk]
+                                                            InputAdapter
+                                                              Exchange [ws_bill_customer_sk] #7
+                                                                WholeStageCodegen (8)
+                                                                  Project [ws_bill_customer_sk]
+                                                                    BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.web_sales [ws_bill_customer_sk,ws_sold_date_sk]
+                                                                            ReusedSubquery [d_date_sk] #1
+                                                                      InputAdapter
+                                                                        ReusedExchange [d_date_sk] #6
+                                                      WholeStageCodegen (12)
+                                                        Sort [cs_ship_customer_sk]
+                                                          InputAdapter
+                                                            Exchange [cs_ship_customer_sk] #8
+                                                              WholeStageCodegen (11)
+                                                                Project [cs_ship_customer_sk]
+                                                                  BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk]
+                                                                          ReusedSubquery [d_date_sk] #1
+                                                                    InputAdapter
+                                                                      ReusedExchange [d_date_sk] #6
+                                  InputAdapter
+                                    WholeStageCodegen (16)
+                                      Sort [ca_address_sk]
+                                        InputAdapter
+                                          Exchange [ca_address_sk] #9
+                                            WholeStageCodegen (15)
+                                              Filter [ca_address_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.customer_address [ca_address_sk,ca_state]
+                  InputAdapter
+                    WholeStageCodegen (20)
+                      Sort [cd_demo_sk]
+                        InputAdapter
+                          Exchange [cd_demo_sk] #10
+                            WholeStageCodegen (19)
+                              Filter [cd_demo_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/explain.txt
new file mode 100644
index 0000000000000..48ae824834450
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/explain.txt
@@ -0,0 +1,273 @@
+== Physical Plan ==
+TakeOrderedAndProject (46)
++- * HashAggregate (45)
+   +- Exchange (44)
+      +- * HashAggregate (43)
+         +- * Project (42)
+            +- * BroadcastHashJoin Inner BuildRight (41)
+               :- * Project (36)
+               :  +- * BroadcastHashJoin Inner BuildRight (35)
+               :     :- * Project (30)
+               :     :  +- * Filter (29)
+               :     :     +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (28)
+               :     :        :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (21)
+               :     :        :  :- * BroadcastHashJoin LeftSemi BuildRight (14)
+               :     :        :  :  :- * Filter (3)
+               :     :        :  :  :  +- * ColumnarToRow (2)
+               :     :        :  :  :     +- Scan parquet default.customer (1)
+               :     :        :  :  +- BroadcastExchange (13)
+               :     :        :  :     +- * Project (12)
+               :     :        :  :        +- * BroadcastHashJoin Inner BuildRight (11)
+               :     :        :  :           :- * ColumnarToRow (5)
+               :     :        :  :           :  +- Scan parquet default.store_sales (4)
+               :     :        :  :           +- BroadcastExchange (10)
+               :     :        :  :              +- * Project (9)
+               :     :        :  :                 +- * Filter (8)
+               :     :        :  :                    +- * ColumnarToRow (7)
+               :     :        :  :                       +- Scan parquet default.date_dim (6)
+               :     :        :  +- BroadcastExchange (20)
+               :     :        :     +- * Project (19)
+               :     :        :        +- * BroadcastHashJoin Inner BuildRight (18)
+               :     :        :           :- * ColumnarToRow (16)
+               :     :        :           :  +- Scan parquet default.web_sales (15)
+               :     :        :           +- ReusedExchange (17)
+               :     :        +- BroadcastExchange (27)
+               :     :           +- * Project (26)
+               :     :              +- * BroadcastHashJoin Inner BuildRight (25)
+               :     :                 :- * ColumnarToRow (23)
+               :     :                 :  +- Scan parquet default.catalog_sales (22)
+               :     :                 +- ReusedExchange (24)
+               :     +- BroadcastExchange (34)
+               :        +- * Filter (33)
+               :           +- * ColumnarToRow (32)
+               :              +- Scan parquet default.customer_address (31)
+               +- BroadcastExchange (40)
+                  +- * Filter (39)
+                     +- * ColumnarToRow (38)
+                        +- Scan parquet default.customer_demographics (37)
+
+
+(1) Scan parquet default.customer
+Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_addr_sk:int>
+
+(2) ColumnarToRow [codegen id : 9]
+Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5]
+
+(3) Filter [codegen id : 9]
+Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5]
+Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4))
+
+(4) Scan parquet default.store_sales
+Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)]
+ReadSchema: struct<ss_customer_sk:int>
+
+(5) ColumnarToRow [codegen id : 2]
+Input [2]: [ss_customer_sk#6, ss_sold_date_sk#7]
+
+(6) Scan parquet default.date_dim
+Output [3]: [d_date_sk#9, d_year#10, d_qoy#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_qoy:int>
+
+(7) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#9, d_year#10, d_qoy#11]
+
+(8) Filter [codegen id : 1]
+Input [3]: [d_date_sk#9, d_year#10, d_qoy#11]
+Condition : ((((isnotnull(d_year#10) AND isnotnull(d_qoy#11)) AND (d_year#10 = 2002)) AND (d_qoy#11 < 4)) AND isnotnull(d_date_sk#9))
+
+(9) Project [codegen id : 1]
+Output [1]: [d_date_sk#9]
+Input [3]: [d_date_sk#9, d_year#10, d_qoy#11]
+
+(10) BroadcastExchange
+Input [1]: [d_date_sk#9]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(11) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#7]
+Right keys [1]: [d_date_sk#9]
+Join condition: None
+
+(12) Project [codegen id : 2]
+Output [1]: [ss_customer_sk#6]
+Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#9]
+
+(13) BroadcastExchange
+Input [1]: [ss_customer_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(14) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_customer_sk#3]
+Right keys [1]: [ss_customer_sk#6]
+Join condition: None
+
+(15) Scan parquet default.web_sales
+Output [2]: [ws_bill_customer_sk#14, ws_sold_date_sk#15]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#15), dynamicpruningexpression(ws_sold_date_sk#15 IN dynamicpruning#8)]
+ReadSchema: struct<ws_bill_customer_sk:int>
+
+(16) ColumnarToRow [codegen id : 4]
+Input [2]: [ws_bill_customer_sk#14, ws_sold_date_sk#15]
+
+(17) ReusedExchange [Reuses operator id: 10]
+Output [1]: [d_date_sk#16]
+
+(18) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ws_sold_date_sk#15]
+Right keys [1]: [d_date_sk#16]
+Join condition: None
+
+(19) Project [codegen id : 4]
+Output [1]: [ws_bill_customer_sk#14]
+Input [3]: [ws_bill_customer_sk#14, ws_sold_date_sk#15, d_date_sk#16]
+
+(20) BroadcastExchange
+Input [1]: [ws_bill_customer_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(21) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_customer_sk#3]
+Right keys [1]: [ws_bill_customer_sk#14]
+Join condition: None
+
+(22) Scan parquet default.catalog_sales
+Output [2]: [cs_ship_customer_sk#18, cs_sold_date_sk#19]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#19), dynamicpruningexpression(cs_sold_date_sk#19 IN dynamicpruning#8)]
+ReadSchema: struct<cs_ship_customer_sk:int>
+
+(23) ColumnarToRow [codegen id : 6]
+Input [2]: [cs_ship_customer_sk#18, cs_sold_date_sk#19]
+
+(24) ReusedExchange [Reuses operator id: 10]
+Output [1]: [d_date_sk#20]
+
+(25) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [cs_sold_date_sk#19]
+Right keys [1]: [d_date_sk#20]
+Join condition: None
+
+(26) Project [codegen id : 6]
+Output [1]: [cs_ship_customer_sk#18]
+Input [3]: [cs_ship_customer_sk#18, cs_sold_date_sk#19, d_date_sk#20]
+
+(27) BroadcastExchange
+Input [1]: [cs_ship_customer_sk#18]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(28) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_customer_sk#3]
+Right keys [1]: [cs_ship_customer_sk#18]
+Join condition: None
+
+(29) Filter [codegen id : 9]
+Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1]
+Condition : (exists#2 OR exists#1)
+
+(30) Project [codegen id : 9]
+Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5]
+Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1]
+
+(31) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#22, ca_state#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(32) ColumnarToRow [codegen id : 7]
+Input [2]: [ca_address_sk#22, ca_state#23]
+
+(33) Filter [codegen id : 7]
+Input [2]: [ca_address_sk#22, ca_state#23]
+Condition : isnotnull(ca_address_sk#22)
+
+(34) BroadcastExchange
+Input [2]: [ca_address_sk#22, ca_state#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#24]
+
+(35) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_current_addr_sk#5]
+Right keys [1]: [ca_address_sk#22]
+Join condition: None
+
+(36) Project [codegen id : 9]
+Output [2]: [c_current_cdemo_sk#4, ca_state#23]
+Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#22, ca_state#23]
+
+(37) Scan parquet default.customer_demographics
+Output [6]: [cd_demo_sk#25, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_dep_count:int,cd_dep_employed_count:int,cd_dep_college_count:int>
+
+(38) ColumnarToRow [codegen id : 8]
+Input [6]: [cd_demo_sk#25, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30]
+
+(39) Filter [codegen id : 8]
+Input [6]: [cd_demo_sk#25, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30]
+Condition : isnotnull(cd_demo_sk#25)
+
+(40) BroadcastExchange
+Input [6]: [cd_demo_sk#25, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#31]
+
+(41) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_current_cdemo_sk#4]
+Right keys [1]: [cd_demo_sk#25]
+Join condition: None
+
+(42) Project [codegen id : 9]
+Output [6]: [ca_state#23, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30]
+Input [8]: [c_current_cdemo_sk#4, ca_state#23, cd_demo_sk#25, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30]
+
+(43) HashAggregate [codegen id : 9]
+Input [6]: [ca_state#23, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30]
+Keys [6]: [ca_state#23, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30]
+Functions [10]: [partial_count(1), partial_min(cd_dep_count#28), partial_max(cd_dep_count#28), partial_avg(cd_dep_count#28), partial_min(cd_dep_employed_count#29), partial_max(cd_dep_employed_count#29), partial_avg(cd_dep_employed_count#29), partial_min(cd_dep_college_count#30), partial_max(cd_dep_college_count#30), partial_avg(cd_dep_college_count#30)]
+Aggregate Attributes [13]: [count#32, min#33, max#34, sum#35, count#36, min#37, max#38, sum#39, count#40, min#41, max#42, sum#43, count#44]
+Results [19]: [ca_state#23, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53, min#54, max#55, sum#56, count#57]
+
+(44) Exchange
+Input [19]: [ca_state#23, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53, min#54, max#55, sum#56, count#57]
+Arguments: hashpartitioning(ca_state#23, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, 5), ENSURE_REQUIREMENTS, [id=#58]
+
+(45) HashAggregate [codegen id : 10]
+Input [19]: [ca_state#23, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53, min#54, max#55, sum#56, count#57]
+Keys [6]: [ca_state#23, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30]
+Functions [10]: [count(1), min(cd_dep_count#28), max(cd_dep_count#28), avg(cd_dep_count#28), min(cd_dep_employed_count#29), max(cd_dep_employed_count#29), avg(cd_dep_employed_count#29), min(cd_dep_college_count#30), max(cd_dep_college_count#30), avg(cd_dep_college_count#30)]
+Aggregate Attributes [10]: [count(1)#59, min(cd_dep_count#28)#60, max(cd_dep_count#28)#61, avg(cd_dep_count#28)#62, min(cd_dep_employed_count#29)#63, max(cd_dep_employed_count#29)#64, avg(cd_dep_employed_count#29)#65, min(cd_dep_college_count#30)#66, max(cd_dep_college_count#30)#67, avg(cd_dep_college_count#30)#68]
+Results [18]: [ca_state#23, cd_gender#26, cd_marital_status#27, count(1)#59 AS cnt1#69, min(cd_dep_count#28)#60 AS min(cd_dep_count)#70, max(cd_dep_count#28)#61 AS max(cd_dep_count)#71, avg(cd_dep_count#28)#62 AS avg(cd_dep_count)#72, cd_dep_employed_count#29, count(1)#59 AS cnt2#73, min(cd_dep_employed_count#29)#63 AS min(cd_dep_employed_count)#74, max(cd_dep_employed_count#29)#64 AS max(cd_dep_employed_count)#75, avg(cd_dep_employed_count#29)#65 AS avg(cd_dep_employed_count)#76, cd_dep_college_count#30, count(1)#59 AS cnt3#77, min(cd_dep_college_count#30)#66 AS min(cd_dep_college_count)#78, max(cd_dep_college_count#30)#67 AS max(cd_dep_college_count)#79, avg(cd_dep_college_count#30)#68 AS avg(cd_dep_college_count)#80, cd_dep_count#28 AS aggOrder#81]
+
+(46) TakeOrderedAndProject
+Input [18]: [ca_state#23, cd_gender#26, cd_marital_status#27, cnt1#69, min(cd_dep_count)#70, max(cd_dep_count)#71, avg(cd_dep_count)#72, cd_dep_employed_count#29, cnt2#73, min(cd_dep_employed_count)#74, max(cd_dep_employed_count)#75, avg(cd_dep_employed_count)#76, cd_dep_college_count#30, cnt3#77, min(cd_dep_college_count)#78, max(cd_dep_college_count)#79, avg(cd_dep_college_count)#80, aggOrder#81]
+Arguments: 100, [ca_state#23 ASC NULLS FIRST, cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, aggOrder#81 ASC NULLS FIRST, cd_dep_employed_count#29 ASC NULLS FIRST, cd_dep_college_count#30 ASC NULLS FIRST], [ca_state#23, cd_gender#26, cd_marital_status#27, cnt1#69, min(cd_dep_count)#70, max(cd_dep_count)#71, avg(cd_dep_count)#72, cd_dep_employed_count#29, cnt2#73, min(cd_dep_employed_count)#74, max(cd_dep_employed_count)#75, avg(cd_dep_employed_count)#76, cd_dep_college_count#30, cnt3#77, min(cd_dep_college_count)#78, max(cd_dep_college_count)#79, avg(cd_dep_college_count)#80]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (47)
+
+
+(47) ReusedExchange [Reuses operator id: 10]
+Output [1]: [d_date_sk#9]
+
+Subquery:2 Hosting operator id = 15 Hosting Expression = ws_sold_date_sk#15 IN dynamicpruning#8
+
+Subquery:3 Hosting operator id = 22 Hosting Expression = cs_sold_date_sk#19 IN dynamicpruning#8
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/simplified.txt
new file mode 100644
index 0000000000000..2614f4f8ae881
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/simplified.txt
@@ -0,0 +1,74 @@
+TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,aggOrder,cd_dep_employed_count,cd_dep_college_count,cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count)]
+  WholeStageCodegen (10)
+    HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] [count(1),min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),aggOrder,count,min,max,sum,count,min,max,sum,count,min,max,sum,count]
+      InputAdapter
+        Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1
+          WholeStageCodegen (9)
+            HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,min,max,sum,count,min,max,sum,count,min,max,sum,count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count]
+              Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count]
+                BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk]
+                  Project [c_current_cdemo_sk,ca_state]
+                    BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                      Project [c_current_cdemo_sk,c_current_addr_sk]
+                        Filter [exists,exists]
+                          BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk]
+                            BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk]
+                              BroadcastHashJoin [c_customer_sk,ss_customer_sk]
+                                Filter [c_current_addr_sk,c_current_cdemo_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk]
+                                InputAdapter
+                                  BroadcastExchange #2
+                                    WholeStageCodegen (2)
+                                      Project [ss_customer_sk]
+                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.store_sales [ss_customer_sk,ss_sold_date_sk]
+                                                SubqueryBroadcast [d_date_sk] #1
+                                                  ReusedExchange [d_date_sk] #3
+                                          InputAdapter
+                                            BroadcastExchange #3
+                                              WholeStageCodegen (1)
+                                                Project [d_date_sk]
+                                                  Filter [d_year,d_qoy,d_date_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.date_dim [d_date_sk,d_year,d_qoy]
+                              InputAdapter
+                                BroadcastExchange #4
+                                  WholeStageCodegen (4)
+                                    Project [ws_bill_customer_sk]
+                                      BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.web_sales [ws_bill_customer_sk,ws_sold_date_sk]
+                                              ReusedSubquery [d_date_sk] #1
+                                        InputAdapter
+                                          ReusedExchange [d_date_sk] #3
+                            InputAdapter
+                              BroadcastExchange #5
+                                WholeStageCodegen (6)
+                                  Project [cs_ship_customer_sk]
+                                    BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk]
+                                            ReusedSubquery [d_date_sk] #1
+                                      InputAdapter
+                                        ReusedExchange [d_date_sk] #3
+                      InputAdapter
+                        BroadcastExchange #6
+                          WholeStageCodegen (7)
+                            Filter [ca_address_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.customer_address [ca_address_sk,ca_state]
+                  InputAdapter
+                    BroadcastExchange #7
+                      WholeStageCodegen (8)
+                        Filter [cd_demo_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.sf100/explain.txt
new file mode 100644
index 0000000000000..1c593f999ac02
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.sf100/explain.txt
@@ -0,0 +1,191 @@
+== Physical Plan ==
+TakeOrderedAndProject (32)
++- * Project (31)
+   +- Window (30)
+      +- * Sort (29)
+         +- Exchange (28)
+            +- * HashAggregate (27)
+               +- Exchange (26)
+                  +- * HashAggregate (25)
+                     +- * Expand (24)
+                        +- * Project (23)
+                           +- * BroadcastHashJoin Inner BuildRight (22)
+                              :- * Project (17)
+                              :  +- * BroadcastHashJoin Inner BuildRight (16)
+                              :     :- * Project (10)
+                              :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+                              :     :     :- * Filter (3)
+                              :     :     :  +- * ColumnarToRow (2)
+                              :     :     :     +- Scan parquet default.store_sales (1)
+                              :     :     +- BroadcastExchange (8)
+                              :     :        +- * Project (7)
+                              :     :           +- * Filter (6)
+                              :     :              +- * ColumnarToRow (5)
+                              :     :                 +- Scan parquet default.date_dim (4)
+                              :     +- BroadcastExchange (15)
+                              :        +- * Project (14)
+                              :           +- * Filter (13)
+                              :              +- * ColumnarToRow (12)
+                              :                 +- Scan parquet default.store (11)
+                              +- BroadcastExchange (21)
+                                 +- * Filter (20)
+                                    +- * ColumnarToRow (19)
+                                       +- Scan parquet default.item (18)
+
+
+(1) Scan parquet default.store_sales
+Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_ext_sales_price:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5]
+
+(3) Filter [codegen id : 4]
+Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5]
+Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2))
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#7, d_year#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#7, d_year#8]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#7, d_year#8]
+Condition : ((isnotnull(d_year#8) AND (d_year#8 = 2001)) AND isnotnull(d_date_sk#7))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#7]
+Input [2]: [d_date_sk#7, d_year#8]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#9]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#5]
+Right keys [1]: [d_date_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4]
+Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#7]
+
+(11) Scan parquet default.store
+Output [2]: [s_store_sk#10, s_state#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_state:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [s_store_sk#10, s_state#11]
+
+(13) Filter [codegen id : 2]
+Input [2]: [s_store_sk#10, s_state#11]
+Condition : ((isnotnull(s_state#11) AND (s_state#11 = TN)) AND isnotnull(s_store_sk#10))
+
+(14) Project [codegen id : 2]
+Output [1]: [s_store_sk#10]
+Input [2]: [s_store_sk#10, s_state#11]
+
+(15) BroadcastExchange
+Input [1]: [s_store_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#2]
+Right keys [1]: [s_store_sk#10]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [3]: [ss_item_sk#1, ss_ext_sales_price#3, ss_net_profit#4]
+Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, s_store_sk#10]
+
+(18) Scan parquet default.item
+Output [3]: [i_item_sk#13, i_class#14, i_category#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_class:string,i_category:string>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [3]: [i_item_sk#13, i_class#14, i_category#15]
+
+(20) Filter [codegen id : 3]
+Input [3]: [i_item_sk#13, i_class#14, i_category#15]
+Condition : isnotnull(i_item_sk#13)
+
+(21) BroadcastExchange
+Input [3]: [i_item_sk#13, i_class#14, i_category#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#16]
+
+(22) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#13]
+Join condition: None
+
+(23) Project [codegen id : 4]
+Output [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#15, i_class#14]
+Input [6]: [ss_item_sk#1, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#13, i_class#14, i_category#15]
+
+(24) Expand [codegen id : 4]
+Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#15, i_class#14]
+Arguments: [List(ss_ext_sales_price#3, ss_net_profit#4, i_category#15, i_class#14, 0), List(ss_ext_sales_price#3, ss_net_profit#4, i_category#15, null, 1), List(ss_ext_sales_price#3, ss_net_profit#4, null, null, 3)], [ss_ext_sales_price#3, ss_net_profit#4, i_category#17, i_class#18, spark_grouping_id#19]
+
+(25) HashAggregate [codegen id : 4]
+Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#17, i_class#18, spark_grouping_id#19]
+Keys [3]: [i_category#17, i_class#18, spark_grouping_id#19]
+Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))]
+Aggregate Attributes [2]: [sum#20, sum#21]
+Results [5]: [i_category#17, i_class#18, spark_grouping_id#19, sum#22, sum#23]
+
+(26) Exchange
+Input [5]: [i_category#17, i_class#18, spark_grouping_id#19, sum#22, sum#23]
+Arguments: hashpartitioning(i_category#17, i_class#18, spark_grouping_id#19, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(27) HashAggregate [codegen id : 5]
+Input [5]: [i_category#17, i_class#18, spark_grouping_id#19, sum#22, sum#23]
+Keys [3]: [i_category#17, i_class#18, spark_grouping_id#19]
+Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#25, sum(UnscaledValue(ss_ext_sales_price#3))#26]
+Results [7]: [CheckOverflow((promote_precision(MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#25,17,2)) / promote_precision(MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#26,17,2))), DecimalType(37,20), true) AS gross_margin#27, i_category#17, i_class#18, (cast((shiftright(spark_grouping_id#19, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#19, 0) & 1) as tinyint)) AS lochierarchy#28, (cast((shiftright(spark_grouping_id#19, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#19, 0) & 1) as tinyint)) AS _w1#29, CASE WHEN (cast((shiftright(spark_grouping_id#19, 0) & 1) as tinyint) = 0) THEN i_category#17 END AS _w2#30, CheckOverflow((promote_precision(MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#25,17,2)) / promote_precision(MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#26,17,2))), DecimalType(37,20), true) AS _w3#31]
+
+(28) Exchange
+Input [7]: [gross_margin#27, i_category#17, i_class#18, lochierarchy#28, _w1#29, _w2#30, _w3#31]
+Arguments: hashpartitioning(_w1#29, _w2#30, 5), ENSURE_REQUIREMENTS, [id=#32]
+
+(29) Sort [codegen id : 6]
+Input [7]: [gross_margin#27, i_category#17, i_class#18, lochierarchy#28, _w1#29, _w2#30, _w3#31]
+Arguments: [_w1#29 ASC NULLS FIRST, _w2#30 ASC NULLS FIRST, _w3#31 ASC NULLS FIRST], false, 0
+
+(30) Window
+Input [7]: [gross_margin#27, i_category#17, i_class#18, lochierarchy#28, _w1#29, _w2#30, _w3#31]
+Arguments: [rank(_w3#31) windowspecdefinition(_w1#29, _w2#30, _w3#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#33], [_w1#29, _w2#30], [_w3#31 ASC NULLS FIRST]
+
+(31) Project [codegen id : 7]
+Output [5]: [gross_margin#27, i_category#17, i_class#18, lochierarchy#28, rank_within_parent#33]
+Input [8]: [gross_margin#27, i_category#17, i_class#18, lochierarchy#28, _w1#29, _w2#30, _w3#31, rank_within_parent#33]
+
+(32) TakeOrderedAndProject
+Input [5]: [gross_margin#27, i_category#17, i_class#18, lochierarchy#28, rank_within_parent#33]
+Arguments: 100, [lochierarchy#28 DESC NULLS LAST, CASE WHEN (lochierarchy#28 = 0) THEN i_category#17 END ASC NULLS FIRST, rank_within_parent#33 ASC NULLS FIRST], [gross_margin#27, i_category#17, i_class#18, lochierarchy#28, rank_within_parent#33]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (33)
+
+
+(33) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#7]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.sf100/simplified.txt
new file mode 100644
index 0000000000000..00f3d7ab6d192
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.sf100/simplified.txt
@@ -0,0 +1,51 @@
+TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i_class]
+  WholeStageCodegen (7)
+    Project [gross_margin,i_category,i_class,lochierarchy,rank_within_parent]
+      InputAdapter
+        Window [_w3,_w1,_w2]
+          WholeStageCodegen (6)
+            Sort [_w1,_w2,_w3]
+              InputAdapter
+                Exchange [_w1,_w2] #1
+                  WholeStageCodegen (5)
+                    HashAggregate [i_category,i_class,spark_grouping_id,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),gross_margin,lochierarchy,_w1,_w2,_w3,sum,sum]
+                      InputAdapter
+                        Exchange [i_category,i_class,spark_grouping_id] #2
+                          WholeStageCodegen (4)
+                            HashAggregate [i_category,i_class,spark_grouping_id,ss_net_profit,ss_ext_sales_price] [sum,sum,sum,sum]
+                              Expand [ss_ext_sales_price,ss_net_profit,i_category,i_class]
+                                Project [ss_ext_sales_price,ss_net_profit,i_category,i_class]
+                                  BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                    Project [ss_item_sk,ss_ext_sales_price,ss_net_profit]
+                                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                        Project [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit]
+                                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                            Filter [ss_item_sk,ss_store_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #1
+                                                      ReusedExchange [d_date_sk] #3
+                                            InputAdapter
+                                              BroadcastExchange #3
+                                                WholeStageCodegen (1)
+                                                  Project [d_date_sk]
+                                                    Filter [d_year,d_date_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.date_dim [d_date_sk,d_year]
+                                        InputAdapter
+                                          BroadcastExchange #4
+                                            WholeStageCodegen (2)
+                                              Project [s_store_sk]
+                                                Filter [s_state,s_store_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.store [s_store_sk,s_state]
+                                    InputAdapter
+                                      BroadcastExchange #5
+                                        WholeStageCodegen (3)
+                                          Filter [i_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.item [i_item_sk,i_class,i_category]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/explain.txt
new file mode 100644
index 0000000000000..4300159e93dc5
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/explain.txt
@@ -0,0 +1,191 @@
+== Physical Plan ==
+TakeOrderedAndProject (32)
++- * Project (31)
+   +- Window (30)
+      +- * Sort (29)
+         +- Exchange (28)
+            +- * HashAggregate (27)
+               +- Exchange (26)
+                  +- * HashAggregate (25)
+                     +- * Expand (24)
+                        +- * Project (23)
+                           +- * BroadcastHashJoin Inner BuildRight (22)
+                              :- * Project (16)
+                              :  +- * BroadcastHashJoin Inner BuildRight (15)
+                              :     :- * Project (10)
+                              :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+                              :     :     :- * Filter (3)
+                              :     :     :  +- * ColumnarToRow (2)
+                              :     :     :     +- Scan parquet default.store_sales (1)
+                              :     :     +- BroadcastExchange (8)
+                              :     :        +- * Project (7)
+                              :     :           +- * Filter (6)
+                              :     :              +- * ColumnarToRow (5)
+                              :     :                 +- Scan parquet default.date_dim (4)
+                              :     +- BroadcastExchange (14)
+                              :        +- * Filter (13)
+                              :           +- * ColumnarToRow (12)
+                              :              +- Scan parquet default.item (11)
+                              +- BroadcastExchange (21)
+                                 +- * Project (20)
+                                    +- * Filter (19)
+                                       +- * ColumnarToRow (18)
+                                          +- Scan parquet default.store (17)
+
+
+(1) Scan parquet default.store_sales
+Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_ext_sales_price:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5]
+
+(3) Filter [codegen id : 4]
+Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5]
+Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2))
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#7, d_year#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#7, d_year#8]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#7, d_year#8]
+Condition : ((isnotnull(d_year#8) AND (d_year#8 = 2001)) AND isnotnull(d_date_sk#7))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#7]
+Input [2]: [d_date_sk#7, d_year#8]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#9]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#5]
+Right keys [1]: [d_date_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4]
+Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#7]
+
+(11) Scan parquet default.item
+Output [3]: [i_item_sk#10, i_class#11, i_category#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_class:string,i_category:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [i_item_sk#10, i_class#11, i_category#12]
+
+(13) Filter [codegen id : 2]
+Input [3]: [i_item_sk#10, i_class#11, i_category#12]
+Condition : isnotnull(i_item_sk#10)
+
+(14) BroadcastExchange
+Input [3]: [i_item_sk#10, i_class#11, i_category#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#13]
+
+(15) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#10]
+Join condition: None
+
+(16) Project [codegen id : 4]
+Output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12]
+Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#10, i_class#11, i_category#12]
+
+(17) Scan parquet default.store
+Output [2]: [s_store_sk#14, s_state#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_state:string>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [2]: [s_store_sk#14, s_state#15]
+
+(19) Filter [codegen id : 3]
+Input [2]: [s_store_sk#14, s_state#15]
+Condition : ((isnotnull(s_state#15) AND (s_state#15 = TN)) AND isnotnull(s_store_sk#14))
+
+(20) Project [codegen id : 3]
+Output [1]: [s_store_sk#14]
+Input [2]: [s_store_sk#14, s_state#15]
+
+(21) BroadcastExchange
+Input [1]: [s_store_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#16]
+
+(22) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#2]
+Right keys [1]: [s_store_sk#14]
+Join condition: None
+
+(23) Project [codegen id : 4]
+Output [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11]
+Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12, s_store_sk#14]
+
+(24) Expand [codegen id : 4]
+Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11]
+Arguments: [List(ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11, 0), List(ss_ext_sales_price#3, ss_net_profit#4, i_category#12, null, 1), List(ss_ext_sales_price#3, ss_net_profit#4, null, null, 3)], [ss_ext_sales_price#3, ss_net_profit#4, i_category#17, i_class#18, spark_grouping_id#19]
+
+(25) HashAggregate [codegen id : 4]
+Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#17, i_class#18, spark_grouping_id#19]
+Keys [3]: [i_category#17, i_class#18, spark_grouping_id#19]
+Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))]
+Aggregate Attributes [2]: [sum#20, sum#21]
+Results [5]: [i_category#17, i_class#18, spark_grouping_id#19, sum#22, sum#23]
+
+(26) Exchange
+Input [5]: [i_category#17, i_class#18, spark_grouping_id#19, sum#22, sum#23]
+Arguments: hashpartitioning(i_category#17, i_class#18, spark_grouping_id#19, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(27) HashAggregate [codegen id : 5]
+Input [5]: [i_category#17, i_class#18, spark_grouping_id#19, sum#22, sum#23]
+Keys [3]: [i_category#17, i_class#18, spark_grouping_id#19]
+Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#25, sum(UnscaledValue(ss_ext_sales_price#3))#26]
+Results [7]: [CheckOverflow((promote_precision(MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#25,17,2)) / promote_precision(MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#26,17,2))), DecimalType(37,20), true) AS gross_margin#27, i_category#17, i_class#18, (cast((shiftright(spark_grouping_id#19, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#19, 0) & 1) as tinyint)) AS lochierarchy#28, (cast((shiftright(spark_grouping_id#19, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#19, 0) & 1) as tinyint)) AS _w1#29, CASE WHEN (cast((shiftright(spark_grouping_id#19, 0) & 1) as tinyint) = 0) THEN i_category#17 END AS _w2#30, CheckOverflow((promote_precision(MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#25,17,2)) / promote_precision(MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#26,17,2))), DecimalType(37,20), true) AS _w3#31]
+
+(28) Exchange
+Input [7]: [gross_margin#27, i_category#17, i_class#18, lochierarchy#28, _w1#29, _w2#30, _w3#31]
+Arguments: hashpartitioning(_w1#29, _w2#30, 5), ENSURE_REQUIREMENTS, [id=#32]
+
+(29) Sort [codegen id : 6]
+Input [7]: [gross_margin#27, i_category#17, i_class#18, lochierarchy#28, _w1#29, _w2#30, _w3#31]
+Arguments: [_w1#29 ASC NULLS FIRST, _w2#30 ASC NULLS FIRST, _w3#31 ASC NULLS FIRST], false, 0
+
+(30) Window
+Input [7]: [gross_margin#27, i_category#17, i_class#18, lochierarchy#28, _w1#29, _w2#30, _w3#31]
+Arguments: [rank(_w3#31) windowspecdefinition(_w1#29, _w2#30, _w3#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#33], [_w1#29, _w2#30], [_w3#31 ASC NULLS FIRST]
+
+(31) Project [codegen id : 7]
+Output [5]: [gross_margin#27, i_category#17, i_class#18, lochierarchy#28, rank_within_parent#33]
+Input [8]: [gross_margin#27, i_category#17, i_class#18, lochierarchy#28, _w1#29, _w2#30, _w3#31, rank_within_parent#33]
+
+(32) TakeOrderedAndProject
+Input [5]: [gross_margin#27, i_category#17, i_class#18, lochierarchy#28, rank_within_parent#33]
+Arguments: 100, [lochierarchy#28 DESC NULLS LAST, CASE WHEN (lochierarchy#28 = 0) THEN i_category#17 END ASC NULLS FIRST, rank_within_parent#33 ASC NULLS FIRST], [gross_margin#27, i_category#17, i_class#18, lochierarchy#28, rank_within_parent#33]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (33)
+
+
+(33) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#7]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/simplified.txt
new file mode 100644
index 0000000000000..6e330ea7cff62
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/simplified.txt
@@ -0,0 +1,51 @@
+TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i_class]
+  WholeStageCodegen (7)
+    Project [gross_margin,i_category,i_class,lochierarchy,rank_within_parent]
+      InputAdapter
+        Window [_w3,_w1,_w2]
+          WholeStageCodegen (6)
+            Sort [_w1,_w2,_w3]
+              InputAdapter
+                Exchange [_w1,_w2] #1
+                  WholeStageCodegen (5)
+                    HashAggregate [i_category,i_class,spark_grouping_id,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),gross_margin,lochierarchy,_w1,_w2,_w3,sum,sum]
+                      InputAdapter
+                        Exchange [i_category,i_class,spark_grouping_id] #2
+                          WholeStageCodegen (4)
+                            HashAggregate [i_category,i_class,spark_grouping_id,ss_net_profit,ss_ext_sales_price] [sum,sum,sum,sum]
+                              Expand [ss_ext_sales_price,ss_net_profit,i_category,i_class]
+                                Project [ss_ext_sales_price,ss_net_profit,i_category,i_class]
+                                  BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                    Project [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category]
+                                      BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                        Project [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit]
+                                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                            Filter [ss_item_sk,ss_store_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #1
+                                                      ReusedExchange [d_date_sk] #3
+                                            InputAdapter
+                                              BroadcastExchange #3
+                                                WholeStageCodegen (1)
+                                                  Project [d_date_sk]
+                                                    Filter [d_year,d_date_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.date_dim [d_date_sk,d_year]
+                                        InputAdapter
+                                          BroadcastExchange #4
+                                            WholeStageCodegen (2)
+                                              Filter [i_item_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.item [i_item_sk,i_class,i_category]
+                                    InputAdapter
+                                      BroadcastExchange #5
+                                        WholeStageCodegen (3)
+                                          Project [s_store_sk]
+                                            Filter [s_state,s_store_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store [s_store_sk,s_state]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.sf100/explain.txt
new file mode 100644
index 0000000000000..1f2c169ba0921
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.sf100/explain.txt
@@ -0,0 +1,191 @@
+== Physical Plan ==
+TakeOrderedAndProject (32)
++- * HashAggregate (31)
+   +- Exchange (30)
+      +- * HashAggregate (29)
+         +- * Project (28)
+            +- * SortMergeJoin Inner (27)
+               :- * Sort (20)
+               :  +- Exchange (19)
+               :     +- * Project (18)
+               :        +- * BroadcastHashJoin Inner BuildRight (17)
+               :           :- * Project (11)
+               :           :  +- * BroadcastHashJoin Inner BuildLeft (10)
+               :           :     :- BroadcastExchange (5)
+               :           :     :  +- * Project (4)
+               :           :     :     +- * Filter (3)
+               :           :     :        +- * ColumnarToRow (2)
+               :           :     :           +- Scan parquet default.item (1)
+               :           :     +- * Project (9)
+               :           :        +- * Filter (8)
+               :           :           +- * ColumnarToRow (7)
+               :           :              +- Scan parquet default.inventory (6)
+               :           +- BroadcastExchange (16)
+               :              +- * Project (15)
+               :                 +- * Filter (14)
+               :                    +- * ColumnarToRow (13)
+               :                       +- Scan parquet default.date_dim (12)
+               +- * Sort (26)
+                  +- Exchange (25)
+                     +- * Project (24)
+                        +- * Filter (23)
+                           +- * ColumnarToRow (22)
+                              +- Scan parquet default.catalog_sales (21)
+
+
+(1) Scan parquet default.item
+Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,68.00), LessThanOrEqual(i_current_price,98.00), In(i_manufact_id, [677,940,694,808]), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_item_desc:string,i_current_price:decimal(7,2),i_manufact_id:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5]
+
+(3) Filter [codegen id : 1]
+Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5]
+Condition : ((((isnotnull(i_current_price#4) AND (i_current_price#4 >= 68.00)) AND (i_current_price#4 <= 98.00)) AND i_manufact_id#5 IN (677,940,694,808)) AND isnotnull(i_item_sk#1))
+
+(4) Project [codegen id : 1]
+Output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4]
+Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5]
+
+(5) BroadcastExchange
+Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#6]
+
+(6) Scan parquet default.inventory
+Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(inv_date_sk#9), dynamicpruningexpression(inv_date_sk#9 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)]
+ReadSchema: struct<inv_item_sk:int,inv_quantity_on_hand:int>
+
+(7) ColumnarToRow
+Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9]
+
+(8) Filter
+Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9]
+Condition : (((isnotnull(inv_quantity_on_hand#8) AND (inv_quantity_on_hand#8 >= 100)) AND (inv_quantity_on_hand#8 <= 500)) AND isnotnull(inv_item_sk#7))
+
+(9) Project
+Output [2]: [inv_item_sk#7, inv_date_sk#9]
+Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9]
+
+(10) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [inv_item_sk#7]
+Join condition: None
+
+(11) Project [codegen id : 3]
+Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#9]
+Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_item_sk#7, inv_date_sk#9]
+
+(12) Scan parquet default.date_dim
+Output [2]: [d_date_sk#11, d_date#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-01), LessThanOrEqual(d_date,2000-04-01), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(13) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#11, d_date#12]
+
+(14) Filter [codegen id : 2]
+Input [2]: [d_date_sk#11, d_date#12]
+Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-02-01)) AND (d_date#12 <= 2000-04-01)) AND isnotnull(d_date_sk#11))
+
+(15) Project [codegen id : 2]
+Output [1]: [d_date_sk#11]
+Input [2]: [d_date_sk#11, d_date#12]
+
+(16) BroadcastExchange
+Input [1]: [d_date_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(17) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [inv_date_sk#9]
+Right keys [1]: [d_date_sk#11]
+Join condition: None
+
+(18) Project [codegen id : 3]
+Output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4]
+Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#11]
+
+(19) Exchange
+Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4]
+Arguments: hashpartitioning(i_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#14]
+
+(20) Sort [codegen id : 4]
+Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4]
+Arguments: [i_item_sk#1 ASC NULLS FIRST], false, 0
+
+(21) Scan parquet default.catalog_sales
+Output [2]: [cs_item_sk#15, cs_sold_date_sk#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_sales]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int>
+
+(22) ColumnarToRow [codegen id : 5]
+Input [2]: [cs_item_sk#15, cs_sold_date_sk#16]
+
+(23) Filter [codegen id : 5]
+Input [2]: [cs_item_sk#15, cs_sold_date_sk#16]
+Condition : isnotnull(cs_item_sk#15)
+
+(24) Project [codegen id : 5]
+Output [1]: [cs_item_sk#15]
+Input [2]: [cs_item_sk#15, cs_sold_date_sk#16]
+
+(25) Exchange
+Input [1]: [cs_item_sk#15]
+Arguments: hashpartitioning(cs_item_sk#15, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(26) Sort [codegen id : 6]
+Input [1]: [cs_item_sk#15]
+Arguments: [cs_item_sk#15 ASC NULLS FIRST], false, 0
+
+(27) SortMergeJoin [codegen id : 7]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [cs_item_sk#15]
+Join condition: None
+
+(28) Project [codegen id : 7]
+Output [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, cs_item_sk#15]
+
+(29) HashAggregate [codegen id : 7]
+Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+
+(30) Exchange
+Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(31) HashAggregate [codegen id : 8]
+Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+
+(32) TakeOrderedAndProject
+Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Arguments: 100, [i_item_id#2 ASC NULLS FIRST], [i_item_id#2, i_item_desc#3, i_current_price#4]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 6 Hosting Expression = inv_date_sk#9 IN dynamicpruning#10
+ReusedExchange (33)
+
+
+(33) ReusedExchange [Reuses operator id: 16]
+Output [1]: [d_date_sk#11]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.sf100/simplified.txt
new file mode 100644
index 0000000000000..6ef4ec510e986
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.sf100/simplified.txt
@@ -0,0 +1,53 @@
+TakeOrderedAndProject [i_item_id,i_item_desc,i_current_price]
+  WholeStageCodegen (8)
+    HashAggregate [i_item_id,i_item_desc,i_current_price]
+      InputAdapter
+        Exchange [i_item_id,i_item_desc,i_current_price] #1
+          WholeStageCodegen (7)
+            HashAggregate [i_item_id,i_item_desc,i_current_price]
+              Project [i_item_id,i_item_desc,i_current_price]
+                SortMergeJoin [i_item_sk,cs_item_sk]
+                  InputAdapter
+                    WholeStageCodegen (4)
+                      Sort [i_item_sk]
+                        InputAdapter
+                          Exchange [i_item_sk] #2
+                            WholeStageCodegen (3)
+                              Project [i_item_sk,i_item_id,i_item_desc,i_current_price]
+                                BroadcastHashJoin [inv_date_sk,d_date_sk]
+                                  Project [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk]
+                                    BroadcastHashJoin [i_item_sk,inv_item_sk]
+                                      InputAdapter
+                                        BroadcastExchange #3
+                                          WholeStageCodegen (1)
+                                            Project [i_item_sk,i_item_id,i_item_desc,i_current_price]
+                                              Filter [i_current_price,i_manufact_id,i_item_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id]
+                                      Project [inv_item_sk,inv_date_sk]
+                                        Filter [inv_quantity_on_hand,inv_item_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk]
+                                                SubqueryBroadcast [d_date_sk] #1
+                                                  ReusedExchange [d_date_sk] #4
+                                  InputAdapter
+                                    BroadcastExchange #4
+                                      WholeStageCodegen (2)
+                                        Project [d_date_sk]
+                                          Filter [d_date,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_date]
+                  InputAdapter
+                    WholeStageCodegen (6)
+                      Sort [cs_item_sk]
+                        InputAdapter
+                          Exchange [cs_item_sk] #5
+                            WholeStageCodegen (5)
+                              Project [cs_item_sk]
+                                Filter [cs_item_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.catalog_sales [cs_item_sk,cs_sold_date_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/explain.txt
new file mode 100644
index 0000000000000..9cc89345f5a1c
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/explain.txt
@@ -0,0 +1,176 @@
+== Physical Plan ==
+TakeOrderedAndProject (29)
++- * HashAggregate (28)
+   +- Exchange (27)
+      +- * HashAggregate (26)
+         +- * Project (25)
+            +- * BroadcastHashJoin Inner BuildLeft (24)
+               :- BroadcastExchange (19)
+               :  +- * Project (18)
+               :     +- * BroadcastHashJoin Inner BuildRight (17)
+               :        :- * Project (11)
+               :        :  +- * BroadcastHashJoin Inner BuildRight (10)
+               :        :     :- * Project (4)
+               :        :     :  +- * Filter (3)
+               :        :     :     +- * ColumnarToRow (2)
+               :        :     :        +- Scan parquet default.item (1)
+               :        :     +- BroadcastExchange (9)
+               :        :        +- * Project (8)
+               :        :           +- * Filter (7)
+               :        :              +- * ColumnarToRow (6)
+               :        :                 +- Scan parquet default.inventory (5)
+               :        +- BroadcastExchange (16)
+               :           +- * Project (15)
+               :              +- * Filter (14)
+               :                 +- * ColumnarToRow (13)
+               :                    +- Scan parquet default.date_dim (12)
+               +- * Project (23)
+                  +- * Filter (22)
+                     +- * ColumnarToRow (21)
+                        +- Scan parquet default.catalog_sales (20)
+
+
+(1) Scan parquet default.item
+Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,68.00), LessThanOrEqual(i_current_price,98.00), In(i_manufact_id, [677,940,694,808]), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_item_desc:string,i_current_price:decimal(7,2),i_manufact_id:int>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5]
+
+(3) Filter [codegen id : 3]
+Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5]
+Condition : ((((isnotnull(i_current_price#4) AND (i_current_price#4 >= 68.00)) AND (i_current_price#4 <= 98.00)) AND i_manufact_id#5 IN (677,940,694,808)) AND isnotnull(i_item_sk#1))
+
+(4) Project [codegen id : 3]
+Output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4]
+Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5]
+
+(5) Scan parquet default.inventory
+Output [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(inv_date_sk#8), dynamicpruningexpression(inv_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)]
+ReadSchema: struct<inv_item_sk:int,inv_quantity_on_hand:int>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8]
+
+(7) Filter [codegen id : 1]
+Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8]
+Condition : (((isnotnull(inv_quantity_on_hand#7) AND (inv_quantity_on_hand#7 >= 100)) AND (inv_quantity_on_hand#7 <= 500)) AND isnotnull(inv_item_sk#6))
+
+(8) Project [codegen id : 1]
+Output [2]: [inv_item_sk#6, inv_date_sk#8]
+Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8]
+
+(9) BroadcastExchange
+Input [2]: [inv_item_sk#6, inv_date_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(10) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [inv_item_sk#6]
+Join condition: None
+
+(11) Project [codegen id : 3]
+Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8]
+Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_item_sk#6, inv_date_sk#8]
+
+(12) Scan parquet default.date_dim
+Output [2]: [d_date_sk#11, d_date#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-01), LessThanOrEqual(d_date,2000-04-01), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(13) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#11, d_date#12]
+
+(14) Filter [codegen id : 2]
+Input [2]: [d_date_sk#11, d_date#12]
+Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-02-01)) AND (d_date#12 <= 2000-04-01)) AND isnotnull(d_date_sk#11))
+
+(15) Project [codegen id : 2]
+Output [1]: [d_date_sk#11]
+Input [2]: [d_date_sk#11, d_date#12]
+
+(16) BroadcastExchange
+Input [1]: [d_date_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(17) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [inv_date_sk#8]
+Right keys [1]: [d_date_sk#11]
+Join condition: None
+
+(18) Project [codegen id : 3]
+Output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4]
+Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8, d_date_sk#11]
+
+(19) BroadcastExchange
+Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(20) Scan parquet default.catalog_sales
+Output [2]: [cs_item_sk#15, cs_sold_date_sk#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_sales]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int>
+
+(21) ColumnarToRow
+Input [2]: [cs_item_sk#15, cs_sold_date_sk#16]
+
+(22) Filter
+Input [2]: [cs_item_sk#15, cs_sold_date_sk#16]
+Condition : isnotnull(cs_item_sk#15)
+
+(23) Project
+Output [1]: [cs_item_sk#15]
+Input [2]: [cs_item_sk#15, cs_sold_date_sk#16]
+
+(24) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [cs_item_sk#15]
+Join condition: None
+
+(25) Project [codegen id : 4]
+Output [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, cs_item_sk#15]
+
+(26) HashAggregate [codegen id : 4]
+Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+
+(27) Exchange
+Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(28) HashAggregate [codegen id : 5]
+Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+
+(29) TakeOrderedAndProject
+Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Arguments: 100, [i_item_id#2 ASC NULLS FIRST], [i_item_id#2, i_item_desc#3, i_current_price#4]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 5 Hosting Expression = inv_date_sk#8 IN dynamicpruning#9
+ReusedExchange (30)
+
+
+(30) ReusedExchange [Reuses operator id: 16]
+Output [1]: [d_date_sk#11]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/simplified.txt
new file mode 100644
index 0000000000000..49ec7264dc423
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/simplified.txt
@@ -0,0 +1,44 @@
+TakeOrderedAndProject [i_item_id,i_item_desc,i_current_price]
+  WholeStageCodegen (5)
+    HashAggregate [i_item_id,i_item_desc,i_current_price]
+      InputAdapter
+        Exchange [i_item_id,i_item_desc,i_current_price] #1
+          WholeStageCodegen (4)
+            HashAggregate [i_item_id,i_item_desc,i_current_price]
+              Project [i_item_id,i_item_desc,i_current_price]
+                BroadcastHashJoin [i_item_sk,cs_item_sk]
+                  InputAdapter
+                    BroadcastExchange #2
+                      WholeStageCodegen (3)
+                        Project [i_item_sk,i_item_id,i_item_desc,i_current_price]
+                          BroadcastHashJoin [inv_date_sk,d_date_sk]
+                            Project [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk]
+                              BroadcastHashJoin [i_item_sk,inv_item_sk]
+                                Project [i_item_sk,i_item_id,i_item_desc,i_current_price]
+                                  Filter [i_current_price,i_manufact_id,i_item_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id]
+                                InputAdapter
+                                  BroadcastExchange #3
+                                    WholeStageCodegen (1)
+                                      Project [inv_item_sk,inv_date_sk]
+                                        Filter [inv_quantity_on_hand,inv_item_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk]
+                                                SubqueryBroadcast [d_date_sk] #1
+                                                  ReusedExchange [d_date_sk] #4
+                            InputAdapter
+                              BroadcastExchange #4
+                                WholeStageCodegen (2)
+                                  Project [d_date_sk]
+                                    Filter [d_date,d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.date_dim [d_date_sk,d_date]
+                  Project [cs_item_sk]
+                    Filter [cs_item_sk]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.catalog_sales [cs_item_sk,cs_sold_date_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.sf100/explain.txt
new file mode 100644
index 0000000000000..48edbc42d0ffb
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.sf100/explain.txt
@@ -0,0 +1,425 @@
+== Physical Plan ==
+* HashAggregate (71)
++- Exchange (70)
+   +- * HashAggregate (69)
+      +- * HashAggregate (68)
+         +- Exchange (67)
+            +- * HashAggregate (66)
+               +- * SortMergeJoin LeftSemi (65)
+                  :- * Sort (47)
+                  :  +- Exchange (46)
+                  :     +- * HashAggregate (45)
+                  :        +- Exchange (44)
+                  :           +- * HashAggregate (43)
+                  :              +- * SortMergeJoin LeftSemi (42)
+                  :                 :- * Sort (24)
+                  :                 :  +- Exchange (23)
+                  :                 :     +- * HashAggregate (22)
+                  :                 :        +- Exchange (21)
+                  :                 :           +- * HashAggregate (20)
+                  :                 :              +- * Project (19)
+                  :                 :                 +- * SortMergeJoin Inner (18)
+                  :                 :                    :- * Sort (12)
+                  :                 :                    :  +- Exchange (11)
+                  :                 :                    :     +- * Project (10)
+                  :                 :                    :        +- * BroadcastHashJoin Inner BuildRight (9)
+                  :                 :                    :           :- * Filter (3)
+                  :                 :                    :           :  +- * ColumnarToRow (2)
+                  :                 :                    :           :     +- Scan parquet default.store_sales (1)
+                  :                 :                    :           +- BroadcastExchange (8)
+                  :                 :                    :              +- * Project (7)
+                  :                 :                    :                 +- * Filter (6)
+                  :                 :                    :                    +- * ColumnarToRow (5)
+                  :                 :                    :                       +- Scan parquet default.date_dim (4)
+                  :                 :                    +- * Sort (17)
+                  :                 :                       +- Exchange (16)
+                  :                 :                          +- * Filter (15)
+                  :                 :                             +- * ColumnarToRow (14)
+                  :                 :                                +- Scan parquet default.customer (13)
+                  :                 +- * Sort (41)
+                  :                    +- Exchange (40)
+                  :                       +- * HashAggregate (39)
+                  :                          +- Exchange (38)
+                  :                             +- * HashAggregate (37)
+                  :                                +- * Project (36)
+                  :                                   +- * SortMergeJoin Inner (35)
+                  :                                      :- * Sort (32)
+                  :                                      :  +- Exchange (31)
+                  :                                      :     +- * Project (30)
+                  :                                      :        +- * BroadcastHashJoin Inner BuildRight (29)
+                  :                                      :           :- * Filter (27)
+                  :                                      :           :  +- * ColumnarToRow (26)
+                  :                                      :           :     +- Scan parquet default.catalog_sales (25)
+                  :                                      :           +- ReusedExchange (28)
+                  :                                      +- * Sort (34)
+                  :                                         +- ReusedExchange (33)
+                  +- * Sort (64)
+                     +- Exchange (63)
+                        +- * HashAggregate (62)
+                           +- Exchange (61)
+                              +- * HashAggregate (60)
+                                 +- * Project (59)
+                                    +- * SortMergeJoin Inner (58)
+                                       :- * Sort (55)
+                                       :  +- Exchange (54)
+                                       :     +- * Project (53)
+                                       :        +- * BroadcastHashJoin Inner BuildRight (52)
+                                       :           :- * Filter (50)
+                                       :           :  +- * ColumnarToRow (49)
+                                       :           :     +- Scan parquet default.web_sales (48)
+                                       :           +- ReusedExchange (51)
+                                       +- * Sort (57)
+                                          +- ReusedExchange (56)
+
+
+(1) Scan parquet default.store_sales
+Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#2), dynamicpruningexpression(ss_sold_date_sk#2 IN dynamicpruning#3)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2]
+
+(3) Filter [codegen id : 2]
+Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2]
+Condition : isnotnull(ss_customer_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date,d_month_seq:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6]
+Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4))
+
+(7) Project [codegen id : 1]
+Output [2]: [d_date_sk#4, d_date#5]
+Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6]
+
+(8) BroadcastExchange
+Input [2]: [d_date_sk#4, d_date#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#2]
+Right keys [1]: [d_date_sk#4]
+Join condition: None
+
+(10) Project [codegen id : 2]
+Output [2]: [ss_customer_sk#1, d_date#5]
+Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#4, d_date#5]
+
+(11) Exchange
+Input [2]: [ss_customer_sk#1, d_date#5]
+Arguments: hashpartitioning(ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#8]
+
+(12) Sort [codegen id : 3]
+Input [2]: [ss_customer_sk#1, d_date#5]
+Arguments: [ss_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(13) Scan parquet default.customer
+Output [3]: [c_customer_sk#9, c_first_name#10, c_last_name#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_first_name:string,c_last_name:string>
+
+(14) ColumnarToRow [codegen id : 4]
+Input [3]: [c_customer_sk#9, c_first_name#10, c_last_name#11]
+
+(15) Filter [codegen id : 4]
+Input [3]: [c_customer_sk#9, c_first_name#10, c_last_name#11]
+Condition : isnotnull(c_customer_sk#9)
+
+(16) Exchange
+Input [3]: [c_customer_sk#9, c_first_name#10, c_last_name#11]
+Arguments: hashpartitioning(c_customer_sk#9, 5), ENSURE_REQUIREMENTS, [id=#12]
+
+(17) Sort [codegen id : 5]
+Input [3]: [c_customer_sk#9, c_first_name#10, c_last_name#11]
+Arguments: [c_customer_sk#9 ASC NULLS FIRST], false, 0
+
+(18) SortMergeJoin [codegen id : 6]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#9]
+Join condition: None
+
+(19) Project [codegen id : 6]
+Output [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Input [5]: [ss_customer_sk#1, d_date#5, c_customer_sk#9, c_first_name#10, c_last_name#11]
+
+(20) HashAggregate [codegen id : 6]
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Keys [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#11, c_first_name#10, d_date#5]
+
+(21) Exchange
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Arguments: hashpartitioning(c_last_name#11, c_first_name#10, d_date#5, 5), ENSURE_REQUIREMENTS, [id=#13]
+
+(22) HashAggregate [codegen id : 7]
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Keys [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#11, c_first_name#10, d_date#5]
+
+(23) Exchange
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Arguments: hashpartitioning(coalesce(c_last_name#11, ), isnull(c_last_name#11), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#5, 1970-01-01), isnull(d_date#5), 5), ENSURE_REQUIREMENTS, [id=#14]
+
+(24) Sort [codegen id : 8]
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Arguments: [coalesce(c_last_name#11, ) ASC NULLS FIRST, isnull(c_last_name#11) ASC NULLS FIRST, coalesce(c_first_name#10, ) ASC NULLS FIRST, isnull(c_first_name#10) ASC NULLS FIRST, coalesce(d_date#5, 1970-01-01) ASC NULLS FIRST, isnull(d_date#5) ASC NULLS FIRST], false, 0
+
+(25) Scan parquet default.catalog_sales
+Output [2]: [cs_bill_customer_sk#15, cs_sold_date_sk#16]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#16), dynamicpruningexpression(cs_sold_date_sk#16 IN dynamicpruning#3)]
+PushedFilters: [IsNotNull(cs_bill_customer_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int>
+
+(26) ColumnarToRow [codegen id : 10]
+Input [2]: [cs_bill_customer_sk#15, cs_sold_date_sk#16]
+
+(27) Filter [codegen id : 10]
+Input [2]: [cs_bill_customer_sk#15, cs_sold_date_sk#16]
+Condition : isnotnull(cs_bill_customer_sk#15)
+
+(28) ReusedExchange [Reuses operator id: 8]
+Output [2]: [d_date_sk#17, d_date#18]
+
+(29) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_sold_date_sk#16]
+Right keys [1]: [d_date_sk#17]
+Join condition: None
+
+(30) Project [codegen id : 10]
+Output [2]: [cs_bill_customer_sk#15, d_date#18]
+Input [4]: [cs_bill_customer_sk#15, cs_sold_date_sk#16, d_date_sk#17, d_date#18]
+
+(31) Exchange
+Input [2]: [cs_bill_customer_sk#15, d_date#18]
+Arguments: hashpartitioning(cs_bill_customer_sk#15, 5), ENSURE_REQUIREMENTS, [id=#19]
+
+(32) Sort [codegen id : 11]
+Input [2]: [cs_bill_customer_sk#15, d_date#18]
+Arguments: [cs_bill_customer_sk#15 ASC NULLS FIRST], false, 0
+
+(33) ReusedExchange [Reuses operator id: 16]
+Output [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22]
+
+(34) Sort [codegen id : 13]
+Input [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22]
+Arguments: [c_customer_sk#20 ASC NULLS FIRST], false, 0
+
+(35) SortMergeJoin [codegen id : 14]
+Left keys [1]: [cs_bill_customer_sk#15]
+Right keys [1]: [c_customer_sk#20]
+Join condition: None
+
+(36) Project [codegen id : 14]
+Output [3]: [c_last_name#22, c_first_name#21, d_date#18]
+Input [5]: [cs_bill_customer_sk#15, d_date#18, c_customer_sk#20, c_first_name#21, c_last_name#22]
+
+(37) HashAggregate [codegen id : 14]
+Input [3]: [c_last_name#22, c_first_name#21, d_date#18]
+Keys [3]: [c_last_name#22, c_first_name#21, d_date#18]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#22, c_first_name#21, d_date#18]
+
+(38) Exchange
+Input [3]: [c_last_name#22, c_first_name#21, d_date#18]
+Arguments: hashpartitioning(c_last_name#22, c_first_name#21, d_date#18, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(39) HashAggregate [codegen id : 15]
+Input [3]: [c_last_name#22, c_first_name#21, d_date#18]
+Keys [3]: [c_last_name#22, c_first_name#21, d_date#18]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#22, c_first_name#21, d_date#18]
+
+(40) Exchange
+Input [3]: [c_last_name#22, c_first_name#21, d_date#18]
+Arguments: hashpartitioning(coalesce(c_last_name#22, ), isnull(c_last_name#22), coalesce(c_first_name#21, ), isnull(c_first_name#21), coalesce(d_date#18, 1970-01-01), isnull(d_date#18), 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(41) Sort [codegen id : 16]
+Input [3]: [c_last_name#22, c_first_name#21, d_date#18]
+Arguments: [coalesce(c_last_name#22, ) ASC NULLS FIRST, isnull(c_last_name#22) ASC NULLS FIRST, coalesce(c_first_name#21, ) ASC NULLS FIRST, isnull(c_first_name#21) ASC NULLS FIRST, coalesce(d_date#18, 1970-01-01) ASC NULLS FIRST, isnull(d_date#18) ASC NULLS FIRST], false, 0
+
+(42) SortMergeJoin [codegen id : 17]
+Left keys [6]: [coalesce(c_last_name#11, ), isnull(c_last_name#11), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#5, 1970-01-01), isnull(d_date#5)]
+Right keys [6]: [coalesce(c_last_name#22, ), isnull(c_last_name#22), coalesce(c_first_name#21, ), isnull(c_first_name#21), coalesce(d_date#18, 1970-01-01), isnull(d_date#18)]
+Join condition: None
+
+(43) HashAggregate [codegen id : 17]
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Keys [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#11, c_first_name#10, d_date#5]
+
+(44) Exchange
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Arguments: hashpartitioning(c_last_name#11, c_first_name#10, d_date#5, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(45) HashAggregate [codegen id : 18]
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Keys [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#11, c_first_name#10, d_date#5]
+
+(46) Exchange
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Arguments: hashpartitioning(coalesce(c_last_name#11, ), isnull(c_last_name#11), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#5, 1970-01-01), isnull(d_date#5), 5), ENSURE_REQUIREMENTS, [id=#26]
+
+(47) Sort [codegen id : 19]
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Arguments: [coalesce(c_last_name#11, ) ASC NULLS FIRST, isnull(c_last_name#11) ASC NULLS FIRST, coalesce(c_first_name#10, ) ASC NULLS FIRST, isnull(c_first_name#10) ASC NULLS FIRST, coalesce(d_date#5, 1970-01-01) ASC NULLS FIRST, isnull(d_date#5) ASC NULLS FIRST], false, 0
+
+(48) Scan parquet default.web_sales
+Output [2]: [ws_bill_customer_sk#27, ws_sold_date_sk#28]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#28), dynamicpruningexpression(ws_sold_date_sk#28 IN dynamicpruning#3)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int>
+
+(49) ColumnarToRow [codegen id : 21]
+Input [2]: [ws_bill_customer_sk#27, ws_sold_date_sk#28]
+
+(50) Filter [codegen id : 21]
+Input [2]: [ws_bill_customer_sk#27, ws_sold_date_sk#28]
+Condition : isnotnull(ws_bill_customer_sk#27)
+
+(51) ReusedExchange [Reuses operator id: 8]
+Output [2]: [d_date_sk#29, d_date#30]
+
+(52) BroadcastHashJoin [codegen id : 21]
+Left keys [1]: [ws_sold_date_sk#28]
+Right keys [1]: [d_date_sk#29]
+Join condition: None
+
+(53) Project [codegen id : 21]
+Output [2]: [ws_bill_customer_sk#27, d_date#30]
+Input [4]: [ws_bill_customer_sk#27, ws_sold_date_sk#28, d_date_sk#29, d_date#30]
+
+(54) Exchange
+Input [2]: [ws_bill_customer_sk#27, d_date#30]
+Arguments: hashpartitioning(ws_bill_customer_sk#27, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(55) Sort [codegen id : 22]
+Input [2]: [ws_bill_customer_sk#27, d_date#30]
+Arguments: [ws_bill_customer_sk#27 ASC NULLS FIRST], false, 0
+
+(56) ReusedExchange [Reuses operator id: 16]
+Output [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34]
+
+(57) Sort [codegen id : 24]
+Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34]
+Arguments: [c_customer_sk#32 ASC NULLS FIRST], false, 0
+
+(58) SortMergeJoin [codegen id : 25]
+Left keys [1]: [ws_bill_customer_sk#27]
+Right keys [1]: [c_customer_sk#32]
+Join condition: None
+
+(59) Project [codegen id : 25]
+Output [3]: [c_last_name#34, c_first_name#33, d_date#30]
+Input [5]: [ws_bill_customer_sk#27, d_date#30, c_customer_sk#32, c_first_name#33, c_last_name#34]
+
+(60) HashAggregate [codegen id : 25]
+Input [3]: [c_last_name#34, c_first_name#33, d_date#30]
+Keys [3]: [c_last_name#34, c_first_name#33, d_date#30]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#34, c_first_name#33, d_date#30]
+
+(61) Exchange
+Input [3]: [c_last_name#34, c_first_name#33, d_date#30]
+Arguments: hashpartitioning(c_last_name#34, c_first_name#33, d_date#30, 5), ENSURE_REQUIREMENTS, [id=#35]
+
+(62) HashAggregate [codegen id : 26]
+Input [3]: [c_last_name#34, c_first_name#33, d_date#30]
+Keys [3]: [c_last_name#34, c_first_name#33, d_date#30]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#34, c_first_name#33, d_date#30]
+
+(63) Exchange
+Input [3]: [c_last_name#34, c_first_name#33, d_date#30]
+Arguments: hashpartitioning(coalesce(c_last_name#34, ), isnull(c_last_name#34), coalesce(c_first_name#33, ), isnull(c_first_name#33), coalesce(d_date#30, 1970-01-01), isnull(d_date#30), 5), ENSURE_REQUIREMENTS, [id=#36]
+
+(64) Sort [codegen id : 27]
+Input [3]: [c_last_name#34, c_first_name#33, d_date#30]
+Arguments: [coalesce(c_last_name#34, ) ASC NULLS FIRST, isnull(c_last_name#34) ASC NULLS FIRST, coalesce(c_first_name#33, ) ASC NULLS FIRST, isnull(c_first_name#33) ASC NULLS FIRST, coalesce(d_date#30, 1970-01-01) ASC NULLS FIRST, isnull(d_date#30) ASC NULLS FIRST], false, 0
+
+(65) SortMergeJoin [codegen id : 28]
+Left keys [6]: [coalesce(c_last_name#11, ), isnull(c_last_name#11), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#5, 1970-01-01), isnull(d_date#5)]
+Right keys [6]: [coalesce(c_last_name#34, ), isnull(c_last_name#34), coalesce(c_first_name#33, ), isnull(c_first_name#33), coalesce(d_date#30, 1970-01-01), isnull(d_date#30)]
+Join condition: None
+
+(66) HashAggregate [codegen id : 28]
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Keys [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#11, c_first_name#10, d_date#5]
+
+(67) Exchange
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Arguments: hashpartitioning(c_last_name#11, c_first_name#10, d_date#5, 5), ENSURE_REQUIREMENTS, [id=#37]
+
+(68) HashAggregate [codegen id : 29]
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Keys [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Functions: []
+Aggregate Attributes: []
+Results: []
+
+(69) HashAggregate [codegen id : 29]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#38]
+Results [1]: [count#39]
+
+(70) Exchange
+Input [1]: [count#39]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#40]
+
+(71) HashAggregate [codegen id : 30]
+Input [1]: [count#39]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#41]
+Results [1]: [count(1)#41 AS count(1)#42]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#2 IN dynamicpruning#3
+ReusedExchange (72)
+
+
+(72) ReusedExchange [Reuses operator id: 8]
+Output [2]: [d_date_sk#4, d_date#5]
+
+Subquery:2 Hosting operator id = 25 Hosting Expression = cs_sold_date_sk#16 IN dynamicpruning#3
+
+Subquery:3 Hosting operator id = 48 Hosting Expression = ws_sold_date_sk#28 IN dynamicpruning#3
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.sf100/simplified.txt
new file mode 100644
index 0000000000000..421027136f3c0
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.sf100/simplified.txt
@@ -0,0 +1,135 @@
+WholeStageCodegen (30)
+  HashAggregate [count] [count(1),count(1),count]
+    InputAdapter
+      Exchange #1
+        WholeStageCodegen (29)
+          HashAggregate [count,count]
+            HashAggregate [c_last_name,c_first_name,d_date]
+              InputAdapter
+                Exchange [c_last_name,c_first_name,d_date] #2
+                  WholeStageCodegen (28)
+                    HashAggregate [c_last_name,c_first_name,d_date]
+                      SortMergeJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date]
+                        InputAdapter
+                          WholeStageCodegen (19)
+                            Sort [c_last_name,c_first_name,d_date]
+                              InputAdapter
+                                Exchange [c_last_name,c_first_name,d_date] #3
+                                  WholeStageCodegen (18)
+                                    HashAggregate [c_last_name,c_first_name,d_date]
+                                      InputAdapter
+                                        Exchange [c_last_name,c_first_name,d_date] #4
+                                          WholeStageCodegen (17)
+                                            HashAggregate [c_last_name,c_first_name,d_date]
+                                              SortMergeJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date]
+                                                InputAdapter
+                                                  WholeStageCodegen (8)
+                                                    Sort [c_last_name,c_first_name,d_date]
+                                                      InputAdapter
+                                                        Exchange [c_last_name,c_first_name,d_date] #5
+                                                          WholeStageCodegen (7)
+                                                            HashAggregate [c_last_name,c_first_name,d_date]
+                                                              InputAdapter
+                                                                Exchange [c_last_name,c_first_name,d_date] #6
+                                                                  WholeStageCodegen (6)
+                                                                    HashAggregate [c_last_name,c_first_name,d_date]
+                                                                      Project [c_last_name,c_first_name,d_date]
+                                                                        SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                                                          InputAdapter
+                                                                            WholeStageCodegen (3)
+                                                                              Sort [ss_customer_sk]
+                                                                                InputAdapter
+                                                                                  Exchange [ss_customer_sk] #7
+                                                                                    WholeStageCodegen (2)
+                                                                                      Project [ss_customer_sk,d_date]
+                                                                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                          Filter [ss_customer_sk]
+                                                                                            ColumnarToRow
+                                                                                              InputAdapter
+                                                                                                Scan parquet default.store_sales [ss_customer_sk,ss_sold_date_sk]
+                                                                                                  SubqueryBroadcast [d_date_sk] #1
+                                                                                                    ReusedExchange [d_date_sk,d_date] #8
+                                                                                          InputAdapter
+                                                                                            BroadcastExchange #8
+                                                                                              WholeStageCodegen (1)
+                                                                                                Project [d_date_sk,d_date]
+                                                                                                  Filter [d_month_seq,d_date_sk]
+                                                                                                    ColumnarToRow
+                                                                                                      InputAdapter
+                                                                                                        Scan parquet default.date_dim [d_date_sk,d_date,d_month_seq]
+                                                                          InputAdapter
+                                                                            WholeStageCodegen (5)
+                                                                              Sort [c_customer_sk]
+                                                                                InputAdapter
+                                                                                  Exchange [c_customer_sk] #9
+                                                                                    WholeStageCodegen (4)
+                                                                                      Filter [c_customer_sk]
+                                                                                        ColumnarToRow
+                                                                                          InputAdapter
+                                                                                            Scan parquet default.customer [c_customer_sk,c_first_name,c_last_name]
+                                                InputAdapter
+                                                  WholeStageCodegen (16)
+                                                    Sort [c_last_name,c_first_name,d_date]
+                                                      InputAdapter
+                                                        Exchange [c_last_name,c_first_name,d_date] #10
+                                                          WholeStageCodegen (15)
+                                                            HashAggregate [c_last_name,c_first_name,d_date]
+                                                              InputAdapter
+                                                                Exchange [c_last_name,c_first_name,d_date] #11
+                                                                  WholeStageCodegen (14)
+                                                                    HashAggregate [c_last_name,c_first_name,d_date]
+                                                                      Project [c_last_name,c_first_name,d_date]
+                                                                        SortMergeJoin [cs_bill_customer_sk,c_customer_sk]
+                                                                          InputAdapter
+                                                                            WholeStageCodegen (11)
+                                                                              Sort [cs_bill_customer_sk]
+                                                                                InputAdapter
+                                                                                  Exchange [cs_bill_customer_sk] #12
+                                                                                    WholeStageCodegen (10)
+                                                                                      Project [cs_bill_customer_sk,d_date]
+                                                                                        BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                                          Filter [cs_bill_customer_sk]
+                                                                                            ColumnarToRow
+                                                                                              InputAdapter
+                                                                                                Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk]
+                                                                                                  ReusedSubquery [d_date_sk] #1
+                                                                                          InputAdapter
+                                                                                            ReusedExchange [d_date_sk,d_date] #8
+                                                                          InputAdapter
+                                                                            WholeStageCodegen (13)
+                                                                              Sort [c_customer_sk]
+                                                                                InputAdapter
+                                                                                  ReusedExchange [c_customer_sk,c_first_name,c_last_name] #9
+                        InputAdapter
+                          WholeStageCodegen (27)
+                            Sort [c_last_name,c_first_name,d_date]
+                              InputAdapter
+                                Exchange [c_last_name,c_first_name,d_date] #13
+                                  WholeStageCodegen (26)
+                                    HashAggregate [c_last_name,c_first_name,d_date]
+                                      InputAdapter
+                                        Exchange [c_last_name,c_first_name,d_date] #14
+                                          WholeStageCodegen (25)
+                                            HashAggregate [c_last_name,c_first_name,d_date]
+                                              Project [c_last_name,c_first_name,d_date]
+                                                SortMergeJoin [ws_bill_customer_sk,c_customer_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (22)
+                                                      Sort [ws_bill_customer_sk]
+                                                        InputAdapter
+                                                          Exchange [ws_bill_customer_sk] #15
+                                                            WholeStageCodegen (21)
+                                                              Project [ws_bill_customer_sk,d_date]
+                                                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                  Filter [ws_bill_customer_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.web_sales [ws_bill_customer_sk,ws_sold_date_sk]
+                                                                          ReusedSubquery [d_date_sk] #1
+                                                                  InputAdapter
+                                                                    ReusedExchange [d_date_sk,d_date] #8
+                                                  InputAdapter
+                                                    WholeStageCodegen (24)
+                                                      Sort [c_customer_sk]
+                                                        InputAdapter
+                                                          ReusedExchange [c_customer_sk,c_first_name,c_last_name] #9
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/explain.txt
new file mode 100644
index 0000000000000..c4e679eb902e0
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/explain.txt
@@ -0,0 +1,340 @@
+== Physical Plan ==
+* HashAggregate (54)
++- Exchange (53)
+   +- * HashAggregate (52)
+      +- * HashAggregate (51)
+         +- * HashAggregate (50)
+            +- * BroadcastHashJoin LeftSemi BuildRight (49)
+               :- * HashAggregate (35)
+               :  +- * HashAggregate (34)
+               :     +- * BroadcastHashJoin LeftSemi BuildRight (33)
+               :        :- * HashAggregate (19)
+               :        :  +- Exchange (18)
+               :        :     +- * HashAggregate (17)
+               :        :        +- * Project (16)
+               :        :           +- * BroadcastHashJoin Inner BuildRight (15)
+               :        :              :- * Project (10)
+               :        :              :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :        :              :     :- * Filter (3)
+               :        :              :     :  +- * ColumnarToRow (2)
+               :        :              :     :     +- Scan parquet default.store_sales (1)
+               :        :              :     +- BroadcastExchange (8)
+               :        :              :        +- * Project (7)
+               :        :              :           +- * Filter (6)
+               :        :              :              +- * ColumnarToRow (5)
+               :        :              :                 +- Scan parquet default.date_dim (4)
+               :        :              +- BroadcastExchange (14)
+               :        :                 +- * Filter (13)
+               :        :                    +- * ColumnarToRow (12)
+               :        :                       +- Scan parquet default.customer (11)
+               :        +- BroadcastExchange (32)
+               :           +- * HashAggregate (31)
+               :              +- Exchange (30)
+               :                 +- * HashAggregate (29)
+               :                    +- * Project (28)
+               :                       +- * BroadcastHashJoin Inner BuildRight (27)
+               :                          :- * Project (25)
+               :                          :  +- * BroadcastHashJoin Inner BuildRight (24)
+               :                          :     :- * Filter (22)
+               :                          :     :  +- * ColumnarToRow (21)
+               :                          :     :     +- Scan parquet default.catalog_sales (20)
+               :                          :     +- ReusedExchange (23)
+               :                          +- ReusedExchange (26)
+               +- BroadcastExchange (48)
+                  +- * HashAggregate (47)
+                     +- Exchange (46)
+                        +- * HashAggregate (45)
+                           +- * Project (44)
+                              +- * BroadcastHashJoin Inner BuildRight (43)
+                                 :- * Project (41)
+                                 :  +- * BroadcastHashJoin Inner BuildRight (40)
+                                 :     :- * Filter (38)
+                                 :     :  +- * ColumnarToRow (37)
+                                 :     :     +- Scan parquet default.web_sales (36)
+                                 :     +- ReusedExchange (39)
+                                 +- ReusedExchange (42)
+
+
+(1) Scan parquet default.store_sales
+Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#2), dynamicpruningexpression(ss_sold_date_sk#2 IN dynamicpruning#3)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2]
+
+(3) Filter [codegen id : 3]
+Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2]
+Condition : isnotnull(ss_customer_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date,d_month_seq:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6]
+Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4))
+
+(7) Project [codegen id : 1]
+Output [2]: [d_date_sk#4, d_date#5]
+Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6]
+
+(8) BroadcastExchange
+Input [2]: [d_date_sk#4, d_date#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#2]
+Right keys [1]: [d_date_sk#4]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [2]: [ss_customer_sk#1, d_date#5]
+Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#4, d_date#5]
+
+(11) Scan parquet default.customer
+Output [3]: [c_customer_sk#8, c_first_name#9, c_last_name#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_first_name:string,c_last_name:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [c_customer_sk#8, c_first_name#9, c_last_name#10]
+
+(13) Filter [codegen id : 2]
+Input [3]: [c_customer_sk#8, c_first_name#9, c_last_name#10]
+Condition : isnotnull(c_customer_sk#8)
+
+(14) BroadcastExchange
+Input [3]: [c_customer_sk#8, c_first_name#9, c_last_name#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#8]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Input [5]: [ss_customer_sk#1, d_date#5, c_customer_sk#8, c_first_name#9, c_last_name#10]
+
+(17) HashAggregate [codegen id : 3]
+Input [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Keys [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#10, c_first_name#9, d_date#5]
+
+(18) Exchange
+Input [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Arguments: hashpartitioning(c_last_name#10, c_first_name#9, d_date#5, 5), ENSURE_REQUIREMENTS, [id=#12]
+
+(19) HashAggregate [codegen id : 12]
+Input [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Keys [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#10, c_first_name#9, d_date#5]
+
+(20) Scan parquet default.catalog_sales
+Output [2]: [cs_bill_customer_sk#13, cs_sold_date_sk#14]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#14), dynamicpruningexpression(cs_sold_date_sk#14 IN dynamicpruning#3)]
+PushedFilters: [IsNotNull(cs_bill_customer_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int>
+
+(21) ColumnarToRow [codegen id : 6]
+Input [2]: [cs_bill_customer_sk#13, cs_sold_date_sk#14]
+
+(22) Filter [codegen id : 6]
+Input [2]: [cs_bill_customer_sk#13, cs_sold_date_sk#14]
+Condition : isnotnull(cs_bill_customer_sk#13)
+
+(23) ReusedExchange [Reuses operator id: 8]
+Output [2]: [d_date_sk#15, d_date#16]
+
+(24) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [cs_sold_date_sk#14]
+Right keys [1]: [d_date_sk#15]
+Join condition: None
+
+(25) Project [codegen id : 6]
+Output [2]: [cs_bill_customer_sk#13, d_date#16]
+Input [4]: [cs_bill_customer_sk#13, cs_sold_date_sk#14, d_date_sk#15, d_date#16]
+
+(26) ReusedExchange [Reuses operator id: 14]
+Output [3]: [c_customer_sk#17, c_first_name#18, c_last_name#19]
+
+(27) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [cs_bill_customer_sk#13]
+Right keys [1]: [c_customer_sk#17]
+Join condition: None
+
+(28) Project [codegen id : 6]
+Output [3]: [c_last_name#19, c_first_name#18, d_date#16]
+Input [5]: [cs_bill_customer_sk#13, d_date#16, c_customer_sk#17, c_first_name#18, c_last_name#19]
+
+(29) HashAggregate [codegen id : 6]
+Input [3]: [c_last_name#19, c_first_name#18, d_date#16]
+Keys [3]: [c_last_name#19, c_first_name#18, d_date#16]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#19, c_first_name#18, d_date#16]
+
+(30) Exchange
+Input [3]: [c_last_name#19, c_first_name#18, d_date#16]
+Arguments: hashpartitioning(c_last_name#19, c_first_name#18, d_date#16, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(31) HashAggregate [codegen id : 7]
+Input [3]: [c_last_name#19, c_first_name#18, d_date#16]
+Keys [3]: [c_last_name#19, c_first_name#18, d_date#16]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#19, c_first_name#18, d_date#16]
+
+(32) BroadcastExchange
+Input [3]: [c_last_name#19, c_first_name#18, d_date#16]
+Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [id=#21]
+
+(33) BroadcastHashJoin [codegen id : 12]
+Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#5, 1970-01-01), isnull(d_date#5)]
+Right keys [6]: [coalesce(c_last_name#19, ), isnull(c_last_name#19), coalesce(c_first_name#18, ), isnull(c_first_name#18), coalesce(d_date#16, 1970-01-01), isnull(d_date#16)]
+Join condition: None
+
+(34) HashAggregate [codegen id : 12]
+Input [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Keys [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#10, c_first_name#9, d_date#5]
+
+(35) HashAggregate [codegen id : 12]
+Input [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Keys [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#10, c_first_name#9, d_date#5]
+
+(36) Scan parquet default.web_sales
+Output [2]: [ws_bill_customer_sk#22, ws_sold_date_sk#23]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#23), dynamicpruningexpression(ws_sold_date_sk#23 IN dynamicpruning#3)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int>
+
+(37) ColumnarToRow [codegen id : 10]
+Input [2]: [ws_bill_customer_sk#22, ws_sold_date_sk#23]
+
+(38) Filter [codegen id : 10]
+Input [2]: [ws_bill_customer_sk#22, ws_sold_date_sk#23]
+Condition : isnotnull(ws_bill_customer_sk#22)
+
+(39) ReusedExchange [Reuses operator id: 8]
+Output [2]: [d_date_sk#24, d_date#25]
+
+(40) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ws_sold_date_sk#23]
+Right keys [1]: [d_date_sk#24]
+Join condition: None
+
+(41) Project [codegen id : 10]
+Output [2]: [ws_bill_customer_sk#22, d_date#25]
+Input [4]: [ws_bill_customer_sk#22, ws_sold_date_sk#23, d_date_sk#24, d_date#25]
+
+(42) ReusedExchange [Reuses operator id: 14]
+Output [3]: [c_customer_sk#26, c_first_name#27, c_last_name#28]
+
+(43) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ws_bill_customer_sk#22]
+Right keys [1]: [c_customer_sk#26]
+Join condition: None
+
+(44) Project [codegen id : 10]
+Output [3]: [c_last_name#28, c_first_name#27, d_date#25]
+Input [5]: [ws_bill_customer_sk#22, d_date#25, c_customer_sk#26, c_first_name#27, c_last_name#28]
+
+(45) HashAggregate [codegen id : 10]
+Input [3]: [c_last_name#28, c_first_name#27, d_date#25]
+Keys [3]: [c_last_name#28, c_first_name#27, d_date#25]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#28, c_first_name#27, d_date#25]
+
+(46) Exchange
+Input [3]: [c_last_name#28, c_first_name#27, d_date#25]
+Arguments: hashpartitioning(c_last_name#28, c_first_name#27, d_date#25, 5), ENSURE_REQUIREMENTS, [id=#29]
+
+(47) HashAggregate [codegen id : 11]
+Input [3]: [c_last_name#28, c_first_name#27, d_date#25]
+Keys [3]: [c_last_name#28, c_first_name#27, d_date#25]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#28, c_first_name#27, d_date#25]
+
+(48) BroadcastExchange
+Input [3]: [c_last_name#28, c_first_name#27, d_date#25]
+Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [id=#30]
+
+(49) BroadcastHashJoin [codegen id : 12]
+Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#5, 1970-01-01), isnull(d_date#5)]
+Right keys [6]: [coalesce(c_last_name#28, ), isnull(c_last_name#28), coalesce(c_first_name#27, ), isnull(c_first_name#27), coalesce(d_date#25, 1970-01-01), isnull(d_date#25)]
+Join condition: None
+
+(50) HashAggregate [codegen id : 12]
+Input [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Keys [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#10, c_first_name#9, d_date#5]
+
+(51) HashAggregate [codegen id : 12]
+Input [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Keys [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Functions: []
+Aggregate Attributes: []
+Results: []
+
+(52) HashAggregate [codegen id : 12]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#31]
+Results [1]: [count#32]
+
+(53) Exchange
+Input [1]: [count#32]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#33]
+
+(54) HashAggregate [codegen id : 13]
+Input [1]: [count#32]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#34]
+Results [1]: [count(1)#34 AS count(1)#35]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#2 IN dynamicpruning#3
+ReusedExchange (55)
+
+
+(55) ReusedExchange [Reuses operator id: 8]
+Output [2]: [d_date_sk#4, d_date#5]
+
+Subquery:2 Hosting operator id = 20 Hosting Expression = cs_sold_date_sk#14 IN dynamicpruning#3
+
+Subquery:3 Hosting operator id = 36 Hosting Expression = ws_sold_date_sk#23 IN dynamicpruning#3
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/simplified.txt
new file mode 100644
index 0000000000000..ce0c20ab01d79
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/simplified.txt
@@ -0,0 +1,84 @@
+WholeStageCodegen (13)
+  HashAggregate [count] [count(1),count(1),count]
+    InputAdapter
+      Exchange #1
+        WholeStageCodegen (12)
+          HashAggregate [count,count]
+            HashAggregate [c_last_name,c_first_name,d_date]
+              HashAggregate [c_last_name,c_first_name,d_date]
+                BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date]
+                  HashAggregate [c_last_name,c_first_name,d_date]
+                    HashAggregate [c_last_name,c_first_name,d_date]
+                      BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date]
+                        HashAggregate [c_last_name,c_first_name,d_date]
+                          InputAdapter
+                            Exchange [c_last_name,c_first_name,d_date] #2
+                              WholeStageCodegen (3)
+                                HashAggregate [c_last_name,c_first_name,d_date]
+                                  Project [c_last_name,c_first_name,d_date]
+                                    BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+                                      Project [ss_customer_sk,d_date]
+                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                          Filter [ss_customer_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store_sales [ss_customer_sk,ss_sold_date_sk]
+                                                  SubqueryBroadcast [d_date_sk] #1
+                                                    ReusedExchange [d_date_sk,d_date] #3
+                                          InputAdapter
+                                            BroadcastExchange #3
+                                              WholeStageCodegen (1)
+                                                Project [d_date_sk,d_date]
+                                                  Filter [d_month_seq,d_date_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.date_dim [d_date_sk,d_date,d_month_seq]
+                                      InputAdapter
+                                        BroadcastExchange #4
+                                          WholeStageCodegen (2)
+                                            Filter [c_customer_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.customer [c_customer_sk,c_first_name,c_last_name]
+                        InputAdapter
+                          BroadcastExchange #5
+                            WholeStageCodegen (7)
+                              HashAggregate [c_last_name,c_first_name,d_date]
+                                InputAdapter
+                                  Exchange [c_last_name,c_first_name,d_date] #6
+                                    WholeStageCodegen (6)
+                                      HashAggregate [c_last_name,c_first_name,d_date]
+                                        Project [c_last_name,c_first_name,d_date]
+                                          BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk]
+                                            Project [cs_bill_customer_sk,d_date]
+                                              BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                Filter [cs_bill_customer_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk]
+                                                        ReusedSubquery [d_date_sk] #1
+                                                InputAdapter
+                                                  ReusedExchange [d_date_sk,d_date] #3
+                                            InputAdapter
+                                              ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4
+                  InputAdapter
+                    BroadcastExchange #7
+                      WholeStageCodegen (11)
+                        HashAggregate [c_last_name,c_first_name,d_date]
+                          InputAdapter
+                            Exchange [c_last_name,c_first_name,d_date] #8
+                              WholeStageCodegen (10)
+                                HashAggregate [c_last_name,c_first_name,d_date]
+                                  Project [c_last_name,c_first_name,d_date]
+                                    BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk]
+                                      Project [ws_bill_customer_sk,d_date]
+                                        BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                          Filter [ws_bill_customer_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.web_sales [ws_bill_customer_sk,ws_sold_date_sk]
+                                                  ReusedSubquery [d_date_sk] #1
+                                          InputAdapter
+                                            ReusedExchange [d_date_sk,d_date] #3
+                                      InputAdapter
+                                        ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.sf100/explain.txt
new file mode 100644
index 0000000000000..5eb63f2a046cf
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.sf100/explain.txt
@@ -0,0 +1,326 @@
+== Physical Plan ==
+* Sort (55)
++- Exchange (54)
+   +- * SortMergeJoin Inner (53)
+      :- * Sort (29)
+      :  +- Exchange (28)
+      :     +- * Project (27)
+      :        +- * Filter (26)
+      :           +- * HashAggregate (25)
+      :              +- Exchange (24)
+      :                 +- * HashAggregate (23)
+      :                    +- * Project (22)
+      :                       +- * BroadcastHashJoin Inner BuildRight (21)
+      :                          :- * Project (16)
+      :                          :  +- * BroadcastHashJoin Inner BuildRight (15)
+      :                          :     :- * Project (10)
+      :                          :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+      :                          :     :     :- * Filter (3)
+      :                          :     :     :  +- * ColumnarToRow (2)
+      :                          :     :     :     +- Scan parquet default.inventory (1)
+      :                          :     :     +- BroadcastExchange (8)
+      :                          :     :        +- * Project (7)
+      :                          :     :           +- * Filter (6)
+      :                          :     :              +- * ColumnarToRow (5)
+      :                          :     :                 +- Scan parquet default.date_dim (4)
+      :                          :     +- BroadcastExchange (14)
+      :                          :        +- * Filter (13)
+      :                          :           +- * ColumnarToRow (12)
+      :                          :              +- Scan parquet default.item (11)
+      :                          +- BroadcastExchange (20)
+      :                             +- * Filter (19)
+      :                                +- * ColumnarToRow (18)
+      :                                   +- Scan parquet default.warehouse (17)
+      +- * Sort (52)
+         +- Exchange (51)
+            +- * Project (50)
+               +- * Filter (49)
+                  +- * HashAggregate (48)
+                     +- Exchange (47)
+                        +- * HashAggregate (46)
+                           +- * Project (45)
+                              +- * BroadcastHashJoin Inner BuildRight (44)
+                                 :- * Project (42)
+                                 :  +- * BroadcastHashJoin Inner BuildRight (41)
+                                 :     :- * Project (39)
+                                 :     :  +- * BroadcastHashJoin Inner BuildRight (38)
+                                 :     :     :- * Filter (32)
+                                 :     :     :  +- * ColumnarToRow (31)
+                                 :     :     :     +- Scan parquet default.inventory (30)
+                                 :     :     +- BroadcastExchange (37)
+                                 :     :        +- * Project (36)
+                                 :     :           +- * Filter (35)
+                                 :     :              +- * ColumnarToRow (34)
+                                 :     :                 +- Scan parquet default.date_dim (33)
+                                 :     +- ReusedExchange (40)
+                                 +- ReusedExchange (43)
+
+
+(1) Scan parquet default.inventory
+Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)]
+ReadSchema: struct<inv_item_sk:int,inv_warehouse_sk:int,inv_quantity_on_hand:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+
+(3) Filter [codegen id : 4]
+Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2001)) AND (d_moy#8 = 1)) AND isnotnull(d_date_sk#6))
+
+(7) Project [codegen id : 1]
+Output [2]: [d_date_sk#6, d_moy#8]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+(8) BroadcastExchange
+Input [2]: [d_date_sk#6, d_moy#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#9]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, d_moy#8]
+Input [6]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#6, d_moy#8]
+
+(11) Scan parquet default.item
+Output [1]: [i_item_sk#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [1]: [i_item_sk#10]
+
+(13) Filter [codegen id : 2]
+Input [1]: [i_item_sk#10]
+Condition : isnotnull(i_item_sk#10)
+
+(14) BroadcastExchange
+Input [1]: [i_item_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(15) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_item_sk#1]
+Right keys [1]: [i_item_sk#10]
+Join condition: None
+
+(16) Project [codegen id : 4]
+Output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, d_moy#8, i_item_sk#10]
+Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, d_moy#8, i_item_sk#10]
+
+(17) Scan parquet default.warehouse
+Output [2]: [w_warehouse_sk#12, w_warehouse_name#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/warehouse]
+PushedFilters: [IsNotNull(w_warehouse_sk)]
+ReadSchema: struct<w_warehouse_sk:int,w_warehouse_name:string>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [2]: [w_warehouse_sk#12, w_warehouse_name#13]
+
+(19) Filter [codegen id : 3]
+Input [2]: [w_warehouse_sk#12, w_warehouse_name#13]
+Condition : isnotnull(w_warehouse_sk#12)
+
+(20) BroadcastExchange
+Input [2]: [w_warehouse_sk#12, w_warehouse_name#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#14]
+
+(21) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_warehouse_sk#2]
+Right keys [1]: [w_warehouse_sk#12]
+Join condition: None
+
+(22) Project [codegen id : 4]
+Output [5]: [inv_quantity_on_hand#3, i_item_sk#10, w_warehouse_sk#12, w_warehouse_name#13, d_moy#8]
+Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, d_moy#8, i_item_sk#10, w_warehouse_sk#12, w_warehouse_name#13]
+
+(23) HashAggregate [codegen id : 4]
+Input [5]: [inv_quantity_on_hand#3, i_item_sk#10, w_warehouse_sk#12, w_warehouse_name#13, d_moy#8]
+Keys [4]: [w_warehouse_name#13, w_warehouse_sk#12, i_item_sk#10, d_moy#8]
+Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)]
+Aggregate Attributes [5]: [n#15, avg#16, m2#17, sum#18, count#19]
+Results [9]: [w_warehouse_name#13, w_warehouse_sk#12, i_item_sk#10, d_moy#8, n#20, avg#21, m2#22, sum#23, count#24]
+
+(24) Exchange
+Input [9]: [w_warehouse_name#13, w_warehouse_sk#12, i_item_sk#10, d_moy#8, n#20, avg#21, m2#22, sum#23, count#24]
+Arguments: hashpartitioning(w_warehouse_name#13, w_warehouse_sk#12, i_item_sk#10, d_moy#8, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(25) HashAggregate [codegen id : 5]
+Input [9]: [w_warehouse_name#13, w_warehouse_sk#12, i_item_sk#10, d_moy#8, n#20, avg#21, m2#22, sum#23, count#24]
+Keys [4]: [w_warehouse_name#13, w_warehouse_sk#12, i_item_sk#10, d_moy#8]
+Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)]
+Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double))#26, avg(inv_quantity_on_hand#3)#27]
+Results [5]: [w_warehouse_sk#12, i_item_sk#10, d_moy#8, stddev_samp(cast(inv_quantity_on_hand#3 as double))#26 AS stdev#28, avg(inv_quantity_on_hand#3)#27 AS mean#29]
+
+(26) Filter [codegen id : 5]
+Input [5]: [w_warehouse_sk#12, i_item_sk#10, d_moy#8, stdev#28, mean#29]
+Condition : ((isnotnull(mean#29) AND isnotnull(stdev#28)) AND (NOT (mean#29 = 0.0) AND ((stdev#28 / mean#29) > 1.0)))
+
+(27) Project [codegen id : 5]
+Output [5]: [w_warehouse_sk#12, i_item_sk#10, d_moy#8, mean#29, CASE WHEN (mean#29 = 0.0) THEN null ELSE (stdev#28 / mean#29) END AS cov#30]
+Input [5]: [w_warehouse_sk#12, i_item_sk#10, d_moy#8, stdev#28, mean#29]
+
+(28) Exchange
+Input [5]: [w_warehouse_sk#12, i_item_sk#10, d_moy#8, mean#29, cov#30]
+Arguments: hashpartitioning(i_item_sk#10, w_warehouse_sk#12, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(29) Sort [codegen id : 6]
+Input [5]: [w_warehouse_sk#12, i_item_sk#10, d_moy#8, mean#29, cov#30]
+Arguments: [i_item_sk#10 ASC NULLS FIRST, w_warehouse_sk#12 ASC NULLS FIRST], false, 0
+
+(30) Scan parquet default.inventory
+Output [4]: [inv_item_sk#32, inv_warehouse_sk#33, inv_quantity_on_hand#34, inv_date_sk#35]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(inv_date_sk#35), dynamicpruningexpression(inv_date_sk#35 IN dynamicpruning#36)]
+PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)]
+ReadSchema: struct<inv_item_sk:int,inv_warehouse_sk:int,inv_quantity_on_hand:int>
+
+(31) ColumnarToRow [codegen id : 10]
+Input [4]: [inv_item_sk#32, inv_warehouse_sk#33, inv_quantity_on_hand#34, inv_date_sk#35]
+
+(32) Filter [codegen id : 10]
+Input [4]: [inv_item_sk#32, inv_warehouse_sk#33, inv_quantity_on_hand#34, inv_date_sk#35]
+Condition : (isnotnull(inv_item_sk#32) AND isnotnull(inv_warehouse_sk#33))
+
+(33) Scan parquet default.date_dim
+Output [3]: [d_date_sk#37, d_year#38, d_moy#39]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(34) ColumnarToRow [codegen id : 7]
+Input [3]: [d_date_sk#37, d_year#38, d_moy#39]
+
+(35) Filter [codegen id : 7]
+Input [3]: [d_date_sk#37, d_year#38, d_moy#39]
+Condition : ((((isnotnull(d_year#38) AND isnotnull(d_moy#39)) AND (d_year#38 = 2001)) AND (d_moy#39 = 2)) AND isnotnull(d_date_sk#37))
+
+(36) Project [codegen id : 7]
+Output [2]: [d_date_sk#37, d_moy#39]
+Input [3]: [d_date_sk#37, d_year#38, d_moy#39]
+
+(37) BroadcastExchange
+Input [2]: [d_date_sk#37, d_moy#39]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#40]
+
+(38) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [inv_date_sk#35]
+Right keys [1]: [d_date_sk#37]
+Join condition: None
+
+(39) Project [codegen id : 10]
+Output [4]: [inv_item_sk#32, inv_warehouse_sk#33, inv_quantity_on_hand#34, d_moy#39]
+Input [6]: [inv_item_sk#32, inv_warehouse_sk#33, inv_quantity_on_hand#34, inv_date_sk#35, d_date_sk#37, d_moy#39]
+
+(40) ReusedExchange [Reuses operator id: 14]
+Output [1]: [i_item_sk#41]
+
+(41) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [inv_item_sk#32]
+Right keys [1]: [i_item_sk#41]
+Join condition: None
+
+(42) Project [codegen id : 10]
+Output [4]: [inv_warehouse_sk#33, inv_quantity_on_hand#34, d_moy#39, i_item_sk#41]
+Input [5]: [inv_item_sk#32, inv_warehouse_sk#33, inv_quantity_on_hand#34, d_moy#39, i_item_sk#41]
+
+(43) ReusedExchange [Reuses operator id: 20]
+Output [2]: [w_warehouse_sk#42, w_warehouse_name#43]
+
+(44) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [inv_warehouse_sk#33]
+Right keys [1]: [w_warehouse_sk#42]
+Join condition: None
+
+(45) Project [codegen id : 10]
+Output [5]: [inv_quantity_on_hand#34, i_item_sk#41, w_warehouse_sk#42, w_warehouse_name#43, d_moy#39]
+Input [6]: [inv_warehouse_sk#33, inv_quantity_on_hand#34, d_moy#39, i_item_sk#41, w_warehouse_sk#42, w_warehouse_name#43]
+
+(46) HashAggregate [codegen id : 10]
+Input [5]: [inv_quantity_on_hand#34, i_item_sk#41, w_warehouse_sk#42, w_warehouse_name#43, d_moy#39]
+Keys [4]: [w_warehouse_name#43, w_warehouse_sk#42, i_item_sk#41, d_moy#39]
+Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#34 as double)), partial_avg(inv_quantity_on_hand#34)]
+Aggregate Attributes [5]: [n#44, avg#45, m2#46, sum#47, count#48]
+Results [9]: [w_warehouse_name#43, w_warehouse_sk#42, i_item_sk#41, d_moy#39, n#49, avg#50, m2#51, sum#52, count#53]
+
+(47) Exchange
+Input [9]: [w_warehouse_name#43, w_warehouse_sk#42, i_item_sk#41, d_moy#39, n#49, avg#50, m2#51, sum#52, count#53]
+Arguments: hashpartitioning(w_warehouse_name#43, w_warehouse_sk#42, i_item_sk#41, d_moy#39, 5), ENSURE_REQUIREMENTS, [id=#54]
+
+(48) HashAggregate [codegen id : 11]
+Input [9]: [w_warehouse_name#43, w_warehouse_sk#42, i_item_sk#41, d_moy#39, n#49, avg#50, m2#51, sum#52, count#53]
+Keys [4]: [w_warehouse_name#43, w_warehouse_sk#42, i_item_sk#41, d_moy#39]
+Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#34 as double)), avg(inv_quantity_on_hand#34)]
+Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#34 as double))#55, avg(inv_quantity_on_hand#34)#56]
+Results [5]: [w_warehouse_sk#42, i_item_sk#41, d_moy#39, stddev_samp(cast(inv_quantity_on_hand#34 as double))#55 AS stdev#57, avg(inv_quantity_on_hand#34)#56 AS mean#58]
+
+(49) Filter [codegen id : 11]
+Input [5]: [w_warehouse_sk#42, i_item_sk#41, d_moy#39, stdev#57, mean#58]
+Condition : ((isnotnull(mean#58) AND isnotnull(stdev#57)) AND (NOT (mean#58 = 0.0) AND ((stdev#57 / mean#58) > 1.0)))
+
+(50) Project [codegen id : 11]
+Output [5]: [w_warehouse_sk#42, i_item_sk#41, d_moy#39, mean#58, CASE WHEN (mean#58 = 0.0) THEN null ELSE (stdev#57 / mean#58) END AS cov#59]
+Input [5]: [w_warehouse_sk#42, i_item_sk#41, d_moy#39, stdev#57, mean#58]
+
+(51) Exchange
+Input [5]: [w_warehouse_sk#42, i_item_sk#41, d_moy#39, mean#58, cov#59]
+Arguments: hashpartitioning(i_item_sk#41, w_warehouse_sk#42, 5), ENSURE_REQUIREMENTS, [id=#60]
+
+(52) Sort [codegen id : 12]
+Input [5]: [w_warehouse_sk#42, i_item_sk#41, d_moy#39, mean#58, cov#59]
+Arguments: [i_item_sk#41 ASC NULLS FIRST, w_warehouse_sk#42 ASC NULLS FIRST], false, 0
+
+(53) SortMergeJoin [codegen id : 13]
+Left keys [2]: [i_item_sk#10, w_warehouse_sk#12]
+Right keys [2]: [i_item_sk#41, w_warehouse_sk#42]
+Join condition: None
+
+(54) Exchange
+Input [10]: [w_warehouse_sk#12, i_item_sk#10, d_moy#8, mean#29, cov#30, w_warehouse_sk#42, i_item_sk#41, d_moy#39, mean#58, cov#59]
+Arguments: rangepartitioning(w_warehouse_sk#12 ASC NULLS FIRST, i_item_sk#10 ASC NULLS FIRST, d_moy#8 ASC NULLS FIRST, mean#29 ASC NULLS FIRST, cov#30 ASC NULLS FIRST, d_moy#39 ASC NULLS FIRST, mean#58 ASC NULLS FIRST, cov#59 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [id=#61]
+
+(55) Sort [codegen id : 14]
+Input [10]: [w_warehouse_sk#12, i_item_sk#10, d_moy#8, mean#29, cov#30, w_warehouse_sk#42, i_item_sk#41, d_moy#39, mean#58, cov#59]
+Arguments: [w_warehouse_sk#12 ASC NULLS FIRST, i_item_sk#10 ASC NULLS FIRST, d_moy#8 ASC NULLS FIRST, mean#29 ASC NULLS FIRST, cov#30 ASC NULLS FIRST, d_moy#39 ASC NULLS FIRST, mean#58 ASC NULLS FIRST, cov#59 ASC NULLS FIRST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5
+ReusedExchange (56)
+
+
+(56) ReusedExchange [Reuses operator id: 8]
+Output [2]: [d_date_sk#6, d_moy#8]
+
+Subquery:2 Hosting operator id = 30 Hosting Expression = inv_date_sk#35 IN dynamicpruning#36
+ReusedExchange (57)
+
+
+(57) ReusedExchange [Reuses operator id: 37]
+Output [2]: [d_date_sk#37, d_moy#39]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.sf100/simplified.txt
new file mode 100644
index 0000000000000..c6fa5d7835a93
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.sf100/simplified.txt
@@ -0,0 +1,90 @@
+WholeStageCodegen (14)
+  Sort [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov]
+    InputAdapter
+      Exchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1
+        WholeStageCodegen (13)
+          SortMergeJoin [i_item_sk,w_warehouse_sk,i_item_sk,w_warehouse_sk]
+            InputAdapter
+              WholeStageCodegen (6)
+                Sort [i_item_sk,w_warehouse_sk]
+                  InputAdapter
+                    Exchange [i_item_sk,w_warehouse_sk] #2
+                      WholeStageCodegen (5)
+                        Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev]
+                          Filter [mean,stdev]
+                            HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count]
+                              InputAdapter
+                                Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #3
+                                  WholeStageCodegen (4)
+                                    HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count]
+                                      Project [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy]
+                                        BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk]
+                                          Project [inv_warehouse_sk,inv_quantity_on_hand,d_moy,i_item_sk]
+                                            BroadcastHashJoin [inv_item_sk,i_item_sk]
+                                              Project [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,d_moy]
+                                                BroadcastHashJoin [inv_date_sk,d_date_sk]
+                                                  Filter [inv_item_sk,inv_warehouse_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk]
+                                                          SubqueryBroadcast [d_date_sk] #1
+                                                            ReusedExchange [d_date_sk,d_moy] #4
+                                                  InputAdapter
+                                                    BroadcastExchange #4
+                                                      WholeStageCodegen (1)
+                                                        Project [d_date_sk,d_moy]
+                                                          Filter [d_year,d_moy,d_date_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                              InputAdapter
+                                                BroadcastExchange #5
+                                                  WholeStageCodegen (2)
+                                                    Filter [i_item_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.item [i_item_sk]
+                                          InputAdapter
+                                            BroadcastExchange #6
+                                              WholeStageCodegen (3)
+                                                Filter [w_warehouse_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.warehouse [w_warehouse_sk,w_warehouse_name]
+            InputAdapter
+              WholeStageCodegen (12)
+                Sort [i_item_sk,w_warehouse_sk]
+                  InputAdapter
+                    Exchange [i_item_sk,w_warehouse_sk] #7
+                      WholeStageCodegen (11)
+                        Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev]
+                          Filter [mean,stdev]
+                            HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count]
+                              InputAdapter
+                                Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #8
+                                  WholeStageCodegen (10)
+                                    HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count]
+                                      Project [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy]
+                                        BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk]
+                                          Project [inv_warehouse_sk,inv_quantity_on_hand,d_moy,i_item_sk]
+                                            BroadcastHashJoin [inv_item_sk,i_item_sk]
+                                              Project [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,d_moy]
+                                                BroadcastHashJoin [inv_date_sk,d_date_sk]
+                                                  Filter [inv_item_sk,inv_warehouse_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk]
+                                                          SubqueryBroadcast [d_date_sk] #2
+                                                            ReusedExchange [d_date_sk,d_moy] #9
+                                                  InputAdapter
+                                                    BroadcastExchange #9
+                                                      WholeStageCodegen (7)
+                                                        Project [d_date_sk,d_moy]
+                                                          Filter [d_year,d_moy,d_date_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                              InputAdapter
+                                                ReusedExchange [i_item_sk] #5
+                                          InputAdapter
+                                            ReusedExchange [w_warehouse_sk,w_warehouse_name] #6
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/explain.txt
new file mode 100644
index 0000000000000..e77de53c5adcb
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/explain.txt
@@ -0,0 +1,311 @@
+== Physical Plan ==
+* Sort (52)
++- Exchange (51)
+   +- * BroadcastHashJoin Inner BuildRight (50)
+      :- * Project (27)
+      :  +- * Filter (26)
+      :     +- * HashAggregate (25)
+      :        +- Exchange (24)
+      :           +- * HashAggregate (23)
+      :              +- * Project (22)
+      :                 +- * BroadcastHashJoin Inner BuildRight (21)
+      :                    :- * Project (15)
+      :                    :  +- * BroadcastHashJoin Inner BuildRight (14)
+      :                    :     :- * Project (9)
+      :                    :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+      :                    :     :     :- * Filter (3)
+      :                    :     :     :  +- * ColumnarToRow (2)
+      :                    :     :     :     +- Scan parquet default.inventory (1)
+      :                    :     :     +- BroadcastExchange (7)
+      :                    :     :        +- * Filter (6)
+      :                    :     :           +- * ColumnarToRow (5)
+      :                    :     :              +- Scan parquet default.item (4)
+      :                    :     +- BroadcastExchange (13)
+      :                    :        +- * Filter (12)
+      :                    :           +- * ColumnarToRow (11)
+      :                    :              +- Scan parquet default.warehouse (10)
+      :                    +- BroadcastExchange (20)
+      :                       +- * Project (19)
+      :                          +- * Filter (18)
+      :                             +- * ColumnarToRow (17)
+      :                                +- Scan parquet default.date_dim (16)
+      +- BroadcastExchange (49)
+         +- * Project (48)
+            +- * Filter (47)
+               +- * HashAggregate (46)
+                  +- Exchange (45)
+                     +- * HashAggregate (44)
+                        +- * Project (43)
+                           +- * BroadcastHashJoin Inner BuildRight (42)
+                              :- * Project (36)
+                              :  +- * BroadcastHashJoin Inner BuildRight (35)
+                              :     :- * Project (33)
+                              :     :  +- * BroadcastHashJoin Inner BuildRight (32)
+                              :     :     :- * Filter (30)
+                              :     :     :  +- * ColumnarToRow (29)
+                              :     :     :     +- Scan parquet default.inventory (28)
+                              :     :     +- ReusedExchange (31)
+                              :     +- ReusedExchange (34)
+                              +- BroadcastExchange (41)
+                                 +- * Project (40)
+                                    +- * Filter (39)
+                                       +- * ColumnarToRow (38)
+                                          +- Scan parquet default.date_dim (37)
+
+
+(1) Scan parquet default.inventory
+Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)]
+ReadSchema: struct<inv_item_sk:int,inv_warehouse_sk:int,inv_quantity_on_hand:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+
+(3) Filter [codegen id : 4]
+Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2))
+
+(4) Scan parquet default.item
+Output [1]: [i_item_sk#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [1]: [i_item_sk#6]
+
+(6) Filter [codegen id : 1]
+Input [1]: [i_item_sk#6]
+Condition : isnotnull(i_item_sk#6)
+
+(7) BroadcastExchange
+Input [1]: [i_item_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#7]
+
+(8) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_item_sk#1]
+Right keys [1]: [i_item_sk#6]
+Join condition: None
+
+(9) Project [codegen id : 4]
+Output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6]
+Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6]
+
+(10) Scan parquet default.warehouse
+Output [2]: [w_warehouse_sk#8, w_warehouse_name#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/warehouse]
+PushedFilters: [IsNotNull(w_warehouse_sk)]
+ReadSchema: struct<w_warehouse_sk:int,w_warehouse_name:string>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [w_warehouse_sk#8, w_warehouse_name#9]
+
+(12) Filter [codegen id : 2]
+Input [2]: [w_warehouse_sk#8, w_warehouse_name#9]
+Condition : isnotnull(w_warehouse_sk#8)
+
+(13) BroadcastExchange
+Input [2]: [w_warehouse_sk#8, w_warehouse_name#9]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#10]
+
+(14) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_warehouse_sk#2]
+Right keys [1]: [w_warehouse_sk#8]
+Join condition: None
+
+(15) Project [codegen id : 4]
+Output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#8, w_warehouse_name#9]
+Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#8, w_warehouse_name#9]
+
+(16) Scan parquet default.date_dim
+Output [3]: [d_date_sk#11, d_year#12, d_moy#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(17) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#11, d_year#12, d_moy#13]
+
+(18) Filter [codegen id : 3]
+Input [3]: [d_date_sk#11, d_year#12, d_moy#13]
+Condition : ((((isnotnull(d_year#12) AND isnotnull(d_moy#13)) AND (d_year#12 = 2001)) AND (d_moy#13 = 1)) AND isnotnull(d_date_sk#11))
+
+(19) Project [codegen id : 3]
+Output [2]: [d_date_sk#11, d_moy#13]
+Input [3]: [d_date_sk#11, d_year#12, d_moy#13]
+
+(20) BroadcastExchange
+Input [2]: [d_date_sk#11, d_moy#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(21) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_date_sk#4]
+Right keys [1]: [d_date_sk#11]
+Join condition: None
+
+(22) Project [codegen id : 4]
+Output [5]: [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#8, w_warehouse_name#9, d_moy#13]
+Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#8, w_warehouse_name#9, d_date_sk#11, d_moy#13]
+
+(23) HashAggregate [codegen id : 4]
+Input [5]: [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#8, w_warehouse_name#9, d_moy#13]
+Keys [4]: [w_warehouse_name#9, w_warehouse_sk#8, i_item_sk#6, d_moy#13]
+Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)]
+Aggregate Attributes [5]: [n#15, avg#16, m2#17, sum#18, count#19]
+Results [9]: [w_warehouse_name#9, w_warehouse_sk#8, i_item_sk#6, d_moy#13, n#20, avg#21, m2#22, sum#23, count#24]
+
+(24) Exchange
+Input [9]: [w_warehouse_name#9, w_warehouse_sk#8, i_item_sk#6, d_moy#13, n#20, avg#21, m2#22, sum#23, count#24]
+Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sk#8, i_item_sk#6, d_moy#13, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(25) HashAggregate [codegen id : 10]
+Input [9]: [w_warehouse_name#9, w_warehouse_sk#8, i_item_sk#6, d_moy#13, n#20, avg#21, m2#22, sum#23, count#24]
+Keys [4]: [w_warehouse_name#9, w_warehouse_sk#8, i_item_sk#6, d_moy#13]
+Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)]
+Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double))#26, avg(inv_quantity_on_hand#3)#27]
+Results [5]: [w_warehouse_sk#8, i_item_sk#6, d_moy#13, stddev_samp(cast(inv_quantity_on_hand#3 as double))#26 AS stdev#28, avg(inv_quantity_on_hand#3)#27 AS mean#29]
+
+(26) Filter [codegen id : 10]
+Input [5]: [w_warehouse_sk#8, i_item_sk#6, d_moy#13, stdev#28, mean#29]
+Condition : ((isnotnull(mean#29) AND isnotnull(stdev#28)) AND (NOT (mean#29 = 0.0) AND ((stdev#28 / mean#29) > 1.0)))
+
+(27) Project [codegen id : 10]
+Output [5]: [w_warehouse_sk#8, i_item_sk#6, d_moy#13, mean#29, CASE WHEN (mean#29 = 0.0) THEN null ELSE (stdev#28 / mean#29) END AS cov#30]
+Input [5]: [w_warehouse_sk#8, i_item_sk#6, d_moy#13, stdev#28, mean#29]
+
+(28) Scan parquet default.inventory
+Output [4]: [inv_item_sk#31, inv_warehouse_sk#32, inv_quantity_on_hand#33, inv_date_sk#34]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(inv_date_sk#34), dynamicpruningexpression(inv_date_sk#34 IN dynamicpruning#35)]
+PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)]
+ReadSchema: struct<inv_item_sk:int,inv_warehouse_sk:int,inv_quantity_on_hand:int>
+
+(29) ColumnarToRow [codegen id : 8]
+Input [4]: [inv_item_sk#31, inv_warehouse_sk#32, inv_quantity_on_hand#33, inv_date_sk#34]
+
+(30) Filter [codegen id : 8]
+Input [4]: [inv_item_sk#31, inv_warehouse_sk#32, inv_quantity_on_hand#33, inv_date_sk#34]
+Condition : (isnotnull(inv_item_sk#31) AND isnotnull(inv_warehouse_sk#32))
+
+(31) ReusedExchange [Reuses operator id: 7]
+Output [1]: [i_item_sk#36]
+
+(32) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [inv_item_sk#31]
+Right keys [1]: [i_item_sk#36]
+Join condition: None
+
+(33) Project [codegen id : 8]
+Output [4]: [inv_warehouse_sk#32, inv_quantity_on_hand#33, inv_date_sk#34, i_item_sk#36]
+Input [5]: [inv_item_sk#31, inv_warehouse_sk#32, inv_quantity_on_hand#33, inv_date_sk#34, i_item_sk#36]
+
+(34) ReusedExchange [Reuses operator id: 13]
+Output [2]: [w_warehouse_sk#37, w_warehouse_name#38]
+
+(35) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [inv_warehouse_sk#32]
+Right keys [1]: [w_warehouse_sk#37]
+Join condition: None
+
+(36) Project [codegen id : 8]
+Output [5]: [inv_quantity_on_hand#33, inv_date_sk#34, i_item_sk#36, w_warehouse_sk#37, w_warehouse_name#38]
+Input [6]: [inv_warehouse_sk#32, inv_quantity_on_hand#33, inv_date_sk#34, i_item_sk#36, w_warehouse_sk#37, w_warehouse_name#38]
+
+(37) Scan parquet default.date_dim
+Output [3]: [d_date_sk#39, d_year#40, d_moy#41]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(38) ColumnarToRow [codegen id : 7]
+Input [3]: [d_date_sk#39, d_year#40, d_moy#41]
+
+(39) Filter [codegen id : 7]
+Input [3]: [d_date_sk#39, d_year#40, d_moy#41]
+Condition : ((((isnotnull(d_year#40) AND isnotnull(d_moy#41)) AND (d_year#40 = 2001)) AND (d_moy#41 = 2)) AND isnotnull(d_date_sk#39))
+
+(40) Project [codegen id : 7]
+Output [2]: [d_date_sk#39, d_moy#41]
+Input [3]: [d_date_sk#39, d_year#40, d_moy#41]
+
+(41) BroadcastExchange
+Input [2]: [d_date_sk#39, d_moy#41]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#42]
+
+(42) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [inv_date_sk#34]
+Right keys [1]: [d_date_sk#39]
+Join condition: None
+
+(43) Project [codegen id : 8]
+Output [5]: [inv_quantity_on_hand#33, i_item_sk#36, w_warehouse_sk#37, w_warehouse_name#38, d_moy#41]
+Input [7]: [inv_quantity_on_hand#33, inv_date_sk#34, i_item_sk#36, w_warehouse_sk#37, w_warehouse_name#38, d_date_sk#39, d_moy#41]
+
+(44) HashAggregate [codegen id : 8]
+Input [5]: [inv_quantity_on_hand#33, i_item_sk#36, w_warehouse_sk#37, w_warehouse_name#38, d_moy#41]
+Keys [4]: [w_warehouse_name#38, w_warehouse_sk#37, i_item_sk#36, d_moy#41]
+Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#33 as double)), partial_avg(inv_quantity_on_hand#33)]
+Aggregate Attributes [5]: [n#43, avg#44, m2#45, sum#46, count#47]
+Results [9]: [w_warehouse_name#38, w_warehouse_sk#37, i_item_sk#36, d_moy#41, n#48, avg#49, m2#50, sum#51, count#52]
+
+(45) Exchange
+Input [9]: [w_warehouse_name#38, w_warehouse_sk#37, i_item_sk#36, d_moy#41, n#48, avg#49, m2#50, sum#51, count#52]
+Arguments: hashpartitioning(w_warehouse_name#38, w_warehouse_sk#37, i_item_sk#36, d_moy#41, 5), ENSURE_REQUIREMENTS, [id=#53]
+
+(46) HashAggregate [codegen id : 9]
+Input [9]: [w_warehouse_name#38, w_warehouse_sk#37, i_item_sk#36, d_moy#41, n#48, avg#49, m2#50, sum#51, count#52]
+Keys [4]: [w_warehouse_name#38, w_warehouse_sk#37, i_item_sk#36, d_moy#41]
+Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#33 as double)), avg(inv_quantity_on_hand#33)]
+Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#33 as double))#54, avg(inv_quantity_on_hand#33)#55]
+Results [5]: [w_warehouse_sk#37, i_item_sk#36, d_moy#41, stddev_samp(cast(inv_quantity_on_hand#33 as double))#54 AS stdev#56, avg(inv_quantity_on_hand#33)#55 AS mean#57]
+
+(47) Filter [codegen id : 9]
+Input [5]: [w_warehouse_sk#37, i_item_sk#36, d_moy#41, stdev#56, mean#57]
+Condition : ((isnotnull(mean#57) AND isnotnull(stdev#56)) AND (NOT (mean#57 = 0.0) AND ((stdev#56 / mean#57) > 1.0)))
+
+(48) Project [codegen id : 9]
+Output [5]: [w_warehouse_sk#37, i_item_sk#36, d_moy#41, mean#57, CASE WHEN (mean#57 = 0.0) THEN null ELSE (stdev#56 / mean#57) END AS cov#58]
+Input [5]: [w_warehouse_sk#37, i_item_sk#36, d_moy#41, stdev#56, mean#57]
+
+(49) BroadcastExchange
+Input [5]: [w_warehouse_sk#37, i_item_sk#36, d_moy#41, mean#57, cov#58]
+Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [id=#59]
+
+(50) BroadcastHashJoin [codegen id : 10]
+Left keys [2]: [i_item_sk#6, w_warehouse_sk#8]
+Right keys [2]: [i_item_sk#36, w_warehouse_sk#37]
+Join condition: None
+
+(51) Exchange
+Input [10]: [w_warehouse_sk#8, i_item_sk#6, d_moy#13, mean#29, cov#30, w_warehouse_sk#37, i_item_sk#36, d_moy#41, mean#57, cov#58]
+Arguments: rangepartitioning(w_warehouse_sk#8 ASC NULLS FIRST, i_item_sk#6 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST, mean#29 ASC NULLS FIRST, cov#30 ASC NULLS FIRST, d_moy#41 ASC NULLS FIRST, mean#57 ASC NULLS FIRST, cov#58 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [id=#60]
+
+(52) Sort [codegen id : 11]
+Input [10]: [w_warehouse_sk#8, i_item_sk#6, d_moy#13, mean#29, cov#30, w_warehouse_sk#37, i_item_sk#36, d_moy#41, mean#57, cov#58]
+Arguments: [w_warehouse_sk#8 ASC NULLS FIRST, i_item_sk#6 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST, mean#29 ASC NULLS FIRST, cov#30 ASC NULLS FIRST, d_moy#41 ASC NULLS FIRST, mean#57 ASC NULLS FIRST, cov#58 ASC NULLS FIRST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5
+ReusedExchange (53)
+
+
+(53) ReusedExchange [Reuses operator id: 20]
+Output [2]: [d_date_sk#11, d_moy#13]
+
+Subquery:2 Hosting operator id = 28 Hosting Expression = inv_date_sk#34 IN dynamicpruning#35
+ReusedExchange (54)
+
+
+(54) ReusedExchange [Reuses operator id: 41]
+Output [2]: [d_date_sk#39, d_moy#41]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/simplified.txt
new file mode 100644
index 0000000000000..ecd1313c1f5e1
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/simplified.txt
@@ -0,0 +1,81 @@
+WholeStageCodegen (11)
+  Sort [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov]
+    InputAdapter
+      Exchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1
+        WholeStageCodegen (10)
+          BroadcastHashJoin [i_item_sk,w_warehouse_sk,i_item_sk,w_warehouse_sk]
+            Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev]
+              Filter [mean,stdev]
+                HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count]
+                  InputAdapter
+                    Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2
+                      WholeStageCodegen (4)
+                        HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count]
+                          Project [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy]
+                            BroadcastHashJoin [inv_date_sk,d_date_sk]
+                              Project [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name]
+                                BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk]
+                                  Project [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk]
+                                    BroadcastHashJoin [inv_item_sk,i_item_sk]
+                                      Filter [inv_item_sk,inv_warehouse_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #1
+                                                ReusedExchange [d_date_sk,d_moy] #3
+                                      InputAdapter
+                                        BroadcastExchange #4
+                                          WholeStageCodegen (1)
+                                            Filter [i_item_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.item [i_item_sk]
+                                  InputAdapter
+                                    BroadcastExchange #5
+                                      WholeStageCodegen (2)
+                                        Filter [w_warehouse_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.warehouse [w_warehouse_sk,w_warehouse_name]
+                              InputAdapter
+                                BroadcastExchange #3
+                                  WholeStageCodegen (3)
+                                    Project [d_date_sk,d_moy]
+                                      Filter [d_year,d_moy,d_date_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+            InputAdapter
+              BroadcastExchange #6
+                WholeStageCodegen (9)
+                  Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev]
+                    Filter [mean,stdev]
+                      HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count]
+                        InputAdapter
+                          Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7
+                            WholeStageCodegen (8)
+                              HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count]
+                                Project [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy]
+                                  BroadcastHashJoin [inv_date_sk,d_date_sk]
+                                    Project [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name]
+                                      BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk]
+                                        Project [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk]
+                                          BroadcastHashJoin [inv_item_sk,i_item_sk]
+                                            Filter [inv_item_sk,inv_warehouse_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #2
+                                                      ReusedExchange [d_date_sk,d_moy] #8
+                                            InputAdapter
+                                              ReusedExchange [i_item_sk] #4
+                                        InputAdapter
+                                          ReusedExchange [w_warehouse_sk,w_warehouse_name] #5
+                                    InputAdapter
+                                      BroadcastExchange #8
+                                        WholeStageCodegen (7)
+                                          Project [d_date_sk,d_moy]
+                                            Filter [d_year,d_moy,d_date_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.sf100/explain.txt
new file mode 100644
index 0000000000000..e2e3760fe2d03
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.sf100/explain.txt
@@ -0,0 +1,326 @@
+== Physical Plan ==
+* Sort (55)
++- Exchange (54)
+   +- * SortMergeJoin Inner (53)
+      :- * Sort (29)
+      :  +- Exchange (28)
+      :     +- * Project (27)
+      :        +- * Filter (26)
+      :           +- * HashAggregate (25)
+      :              +- Exchange (24)
+      :                 +- * HashAggregate (23)
+      :                    +- * Project (22)
+      :                       +- * BroadcastHashJoin Inner BuildRight (21)
+      :                          :- * Project (16)
+      :                          :  +- * BroadcastHashJoin Inner BuildRight (15)
+      :                          :     :- * Project (10)
+      :                          :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+      :                          :     :     :- * Filter (3)
+      :                          :     :     :  +- * ColumnarToRow (2)
+      :                          :     :     :     +- Scan parquet default.inventory (1)
+      :                          :     :     +- BroadcastExchange (8)
+      :                          :     :        +- * Project (7)
+      :                          :     :           +- * Filter (6)
+      :                          :     :              +- * ColumnarToRow (5)
+      :                          :     :                 +- Scan parquet default.date_dim (4)
+      :                          :     +- BroadcastExchange (14)
+      :                          :        +- * Filter (13)
+      :                          :           +- * ColumnarToRow (12)
+      :                          :              +- Scan parquet default.item (11)
+      :                          +- BroadcastExchange (20)
+      :                             +- * Filter (19)
+      :                                +- * ColumnarToRow (18)
+      :                                   +- Scan parquet default.warehouse (17)
+      +- * Sort (52)
+         +- Exchange (51)
+            +- * Project (50)
+               +- * Filter (49)
+                  +- * HashAggregate (48)
+                     +- Exchange (47)
+                        +- * HashAggregate (46)
+                           +- * Project (45)
+                              +- * BroadcastHashJoin Inner BuildRight (44)
+                                 :- * Project (42)
+                                 :  +- * BroadcastHashJoin Inner BuildRight (41)
+                                 :     :- * Project (39)
+                                 :     :  +- * BroadcastHashJoin Inner BuildRight (38)
+                                 :     :     :- * Filter (32)
+                                 :     :     :  +- * ColumnarToRow (31)
+                                 :     :     :     +- Scan parquet default.inventory (30)
+                                 :     :     +- BroadcastExchange (37)
+                                 :     :        +- * Project (36)
+                                 :     :           +- * Filter (35)
+                                 :     :              +- * ColumnarToRow (34)
+                                 :     :                 +- Scan parquet default.date_dim (33)
+                                 :     +- ReusedExchange (40)
+                                 +- ReusedExchange (43)
+
+
+(1) Scan parquet default.inventory
+Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)]
+ReadSchema: struct<inv_item_sk:int,inv_warehouse_sk:int,inv_quantity_on_hand:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+
+(3) Filter [codegen id : 4]
+Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2001)) AND (d_moy#8 = 1)) AND isnotnull(d_date_sk#6))
+
+(7) Project [codegen id : 1]
+Output [2]: [d_date_sk#6, d_moy#8]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+(8) BroadcastExchange
+Input [2]: [d_date_sk#6, d_moy#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#9]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, d_moy#8]
+Input [6]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#6, d_moy#8]
+
+(11) Scan parquet default.item
+Output [1]: [i_item_sk#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [1]: [i_item_sk#10]
+
+(13) Filter [codegen id : 2]
+Input [1]: [i_item_sk#10]
+Condition : isnotnull(i_item_sk#10)
+
+(14) BroadcastExchange
+Input [1]: [i_item_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(15) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_item_sk#1]
+Right keys [1]: [i_item_sk#10]
+Join condition: None
+
+(16) Project [codegen id : 4]
+Output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, d_moy#8, i_item_sk#10]
+Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, d_moy#8, i_item_sk#10]
+
+(17) Scan parquet default.warehouse
+Output [2]: [w_warehouse_sk#12, w_warehouse_name#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/warehouse]
+PushedFilters: [IsNotNull(w_warehouse_sk)]
+ReadSchema: struct<w_warehouse_sk:int,w_warehouse_name:string>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [2]: [w_warehouse_sk#12, w_warehouse_name#13]
+
+(19) Filter [codegen id : 3]
+Input [2]: [w_warehouse_sk#12, w_warehouse_name#13]
+Condition : isnotnull(w_warehouse_sk#12)
+
+(20) BroadcastExchange
+Input [2]: [w_warehouse_sk#12, w_warehouse_name#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#14]
+
+(21) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_warehouse_sk#2]
+Right keys [1]: [w_warehouse_sk#12]
+Join condition: None
+
+(22) Project [codegen id : 4]
+Output [5]: [inv_quantity_on_hand#3, i_item_sk#10, w_warehouse_sk#12, w_warehouse_name#13, d_moy#8]
+Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, d_moy#8, i_item_sk#10, w_warehouse_sk#12, w_warehouse_name#13]
+
+(23) HashAggregate [codegen id : 4]
+Input [5]: [inv_quantity_on_hand#3, i_item_sk#10, w_warehouse_sk#12, w_warehouse_name#13, d_moy#8]
+Keys [4]: [w_warehouse_name#13, w_warehouse_sk#12, i_item_sk#10, d_moy#8]
+Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)]
+Aggregate Attributes [5]: [n#15, avg#16, m2#17, sum#18, count#19]
+Results [9]: [w_warehouse_name#13, w_warehouse_sk#12, i_item_sk#10, d_moy#8, n#20, avg#21, m2#22, sum#23, count#24]
+
+(24) Exchange
+Input [9]: [w_warehouse_name#13, w_warehouse_sk#12, i_item_sk#10, d_moy#8, n#20, avg#21, m2#22, sum#23, count#24]
+Arguments: hashpartitioning(w_warehouse_name#13, w_warehouse_sk#12, i_item_sk#10, d_moy#8, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(25) HashAggregate [codegen id : 5]
+Input [9]: [w_warehouse_name#13, w_warehouse_sk#12, i_item_sk#10, d_moy#8, n#20, avg#21, m2#22, sum#23, count#24]
+Keys [4]: [w_warehouse_name#13, w_warehouse_sk#12, i_item_sk#10, d_moy#8]
+Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)]
+Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double))#26, avg(inv_quantity_on_hand#3)#27]
+Results [5]: [w_warehouse_sk#12, i_item_sk#10, d_moy#8, stddev_samp(cast(inv_quantity_on_hand#3 as double))#26 AS stdev#28, avg(inv_quantity_on_hand#3)#27 AS mean#29]
+
+(26) Filter [codegen id : 5]
+Input [5]: [w_warehouse_sk#12, i_item_sk#10, d_moy#8, stdev#28, mean#29]
+Condition : ((isnotnull(mean#29) AND isnotnull(stdev#28)) AND ((NOT (mean#29 = 0.0) AND ((stdev#28 / mean#29) > 1.0)) AND ((stdev#28 / mean#29) > 1.5)))
+
+(27) Project [codegen id : 5]
+Output [5]: [w_warehouse_sk#12, i_item_sk#10, d_moy#8, mean#29, CASE WHEN (mean#29 = 0.0) THEN null ELSE (stdev#28 / mean#29) END AS cov#30]
+Input [5]: [w_warehouse_sk#12, i_item_sk#10, d_moy#8, stdev#28, mean#29]
+
+(28) Exchange
+Input [5]: [w_warehouse_sk#12, i_item_sk#10, d_moy#8, mean#29, cov#30]
+Arguments: hashpartitioning(i_item_sk#10, w_warehouse_sk#12, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(29) Sort [codegen id : 6]
+Input [5]: [w_warehouse_sk#12, i_item_sk#10, d_moy#8, mean#29, cov#30]
+Arguments: [i_item_sk#10 ASC NULLS FIRST, w_warehouse_sk#12 ASC NULLS FIRST], false, 0
+
+(30) Scan parquet default.inventory
+Output [4]: [inv_item_sk#32, inv_warehouse_sk#33, inv_quantity_on_hand#34, inv_date_sk#35]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(inv_date_sk#35), dynamicpruningexpression(inv_date_sk#35 IN dynamicpruning#36)]
+PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)]
+ReadSchema: struct<inv_item_sk:int,inv_warehouse_sk:int,inv_quantity_on_hand:int>
+
+(31) ColumnarToRow [codegen id : 10]
+Input [4]: [inv_item_sk#32, inv_warehouse_sk#33, inv_quantity_on_hand#34, inv_date_sk#35]
+
+(32) Filter [codegen id : 10]
+Input [4]: [inv_item_sk#32, inv_warehouse_sk#33, inv_quantity_on_hand#34, inv_date_sk#35]
+Condition : (isnotnull(inv_item_sk#32) AND isnotnull(inv_warehouse_sk#33))
+
+(33) Scan parquet default.date_dim
+Output [3]: [d_date_sk#37, d_year#38, d_moy#39]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(34) ColumnarToRow [codegen id : 7]
+Input [3]: [d_date_sk#37, d_year#38, d_moy#39]
+
+(35) Filter [codegen id : 7]
+Input [3]: [d_date_sk#37, d_year#38, d_moy#39]
+Condition : ((((isnotnull(d_year#38) AND isnotnull(d_moy#39)) AND (d_year#38 = 2001)) AND (d_moy#39 = 2)) AND isnotnull(d_date_sk#37))
+
+(36) Project [codegen id : 7]
+Output [2]: [d_date_sk#37, d_moy#39]
+Input [3]: [d_date_sk#37, d_year#38, d_moy#39]
+
+(37) BroadcastExchange
+Input [2]: [d_date_sk#37, d_moy#39]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#40]
+
+(38) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [inv_date_sk#35]
+Right keys [1]: [d_date_sk#37]
+Join condition: None
+
+(39) Project [codegen id : 10]
+Output [4]: [inv_item_sk#32, inv_warehouse_sk#33, inv_quantity_on_hand#34, d_moy#39]
+Input [6]: [inv_item_sk#32, inv_warehouse_sk#33, inv_quantity_on_hand#34, inv_date_sk#35, d_date_sk#37, d_moy#39]
+
+(40) ReusedExchange [Reuses operator id: 14]
+Output [1]: [i_item_sk#41]
+
+(41) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [inv_item_sk#32]
+Right keys [1]: [i_item_sk#41]
+Join condition: None
+
+(42) Project [codegen id : 10]
+Output [4]: [inv_warehouse_sk#33, inv_quantity_on_hand#34, d_moy#39, i_item_sk#41]
+Input [5]: [inv_item_sk#32, inv_warehouse_sk#33, inv_quantity_on_hand#34, d_moy#39, i_item_sk#41]
+
+(43) ReusedExchange [Reuses operator id: 20]
+Output [2]: [w_warehouse_sk#42, w_warehouse_name#43]
+
+(44) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [inv_warehouse_sk#33]
+Right keys [1]: [w_warehouse_sk#42]
+Join condition: None
+
+(45) Project [codegen id : 10]
+Output [5]: [inv_quantity_on_hand#34, i_item_sk#41, w_warehouse_sk#42, w_warehouse_name#43, d_moy#39]
+Input [6]: [inv_warehouse_sk#33, inv_quantity_on_hand#34, d_moy#39, i_item_sk#41, w_warehouse_sk#42, w_warehouse_name#43]
+
+(46) HashAggregate [codegen id : 10]
+Input [5]: [inv_quantity_on_hand#34, i_item_sk#41, w_warehouse_sk#42, w_warehouse_name#43, d_moy#39]
+Keys [4]: [w_warehouse_name#43, w_warehouse_sk#42, i_item_sk#41, d_moy#39]
+Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#34 as double)), partial_avg(inv_quantity_on_hand#34)]
+Aggregate Attributes [5]: [n#44, avg#45, m2#46, sum#47, count#48]
+Results [9]: [w_warehouse_name#43, w_warehouse_sk#42, i_item_sk#41, d_moy#39, n#49, avg#50, m2#51, sum#52, count#53]
+
+(47) Exchange
+Input [9]: [w_warehouse_name#43, w_warehouse_sk#42, i_item_sk#41, d_moy#39, n#49, avg#50, m2#51, sum#52, count#53]
+Arguments: hashpartitioning(w_warehouse_name#43, w_warehouse_sk#42, i_item_sk#41, d_moy#39, 5), ENSURE_REQUIREMENTS, [id=#54]
+
+(48) HashAggregate [codegen id : 11]
+Input [9]: [w_warehouse_name#43, w_warehouse_sk#42, i_item_sk#41, d_moy#39, n#49, avg#50, m2#51, sum#52, count#53]
+Keys [4]: [w_warehouse_name#43, w_warehouse_sk#42, i_item_sk#41, d_moy#39]
+Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#34 as double)), avg(inv_quantity_on_hand#34)]
+Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#34 as double))#55, avg(inv_quantity_on_hand#34)#56]
+Results [5]: [w_warehouse_sk#42, i_item_sk#41, d_moy#39, stddev_samp(cast(inv_quantity_on_hand#34 as double))#55 AS stdev#57, avg(inv_quantity_on_hand#34)#56 AS mean#58]
+
+(49) Filter [codegen id : 11]
+Input [5]: [w_warehouse_sk#42, i_item_sk#41, d_moy#39, stdev#57, mean#58]
+Condition : ((isnotnull(mean#58) AND isnotnull(stdev#57)) AND (NOT (mean#58 = 0.0) AND ((stdev#57 / mean#58) > 1.0)))
+
+(50) Project [codegen id : 11]
+Output [5]: [w_warehouse_sk#42, i_item_sk#41, d_moy#39, mean#58, CASE WHEN (mean#58 = 0.0) THEN null ELSE (stdev#57 / mean#58) END AS cov#59]
+Input [5]: [w_warehouse_sk#42, i_item_sk#41, d_moy#39, stdev#57, mean#58]
+
+(51) Exchange
+Input [5]: [w_warehouse_sk#42, i_item_sk#41, d_moy#39, mean#58, cov#59]
+Arguments: hashpartitioning(i_item_sk#41, w_warehouse_sk#42, 5), ENSURE_REQUIREMENTS, [id=#60]
+
+(52) Sort [codegen id : 12]
+Input [5]: [w_warehouse_sk#42, i_item_sk#41, d_moy#39, mean#58, cov#59]
+Arguments: [i_item_sk#41 ASC NULLS FIRST, w_warehouse_sk#42 ASC NULLS FIRST], false, 0
+
+(53) SortMergeJoin [codegen id : 13]
+Left keys [2]: [i_item_sk#10, w_warehouse_sk#12]
+Right keys [2]: [i_item_sk#41, w_warehouse_sk#42]
+Join condition: None
+
+(54) Exchange
+Input [10]: [w_warehouse_sk#12, i_item_sk#10, d_moy#8, mean#29, cov#30, w_warehouse_sk#42, i_item_sk#41, d_moy#39, mean#58, cov#59]
+Arguments: rangepartitioning(w_warehouse_sk#12 ASC NULLS FIRST, i_item_sk#10 ASC NULLS FIRST, d_moy#8 ASC NULLS FIRST, mean#29 ASC NULLS FIRST, cov#30 ASC NULLS FIRST, d_moy#39 ASC NULLS FIRST, mean#58 ASC NULLS FIRST, cov#59 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [id=#61]
+
+(55) Sort [codegen id : 14]
+Input [10]: [w_warehouse_sk#12, i_item_sk#10, d_moy#8, mean#29, cov#30, w_warehouse_sk#42, i_item_sk#41, d_moy#39, mean#58, cov#59]
+Arguments: [w_warehouse_sk#12 ASC NULLS FIRST, i_item_sk#10 ASC NULLS FIRST, d_moy#8 ASC NULLS FIRST, mean#29 ASC NULLS FIRST, cov#30 ASC NULLS FIRST, d_moy#39 ASC NULLS FIRST, mean#58 ASC NULLS FIRST, cov#59 ASC NULLS FIRST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5
+ReusedExchange (56)
+
+
+(56) ReusedExchange [Reuses operator id: 8]
+Output [2]: [d_date_sk#6, d_moy#8]
+
+Subquery:2 Hosting operator id = 30 Hosting Expression = inv_date_sk#35 IN dynamicpruning#36
+ReusedExchange (57)
+
+
+(57) ReusedExchange [Reuses operator id: 37]
+Output [2]: [d_date_sk#37, d_moy#39]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.sf100/simplified.txt
new file mode 100644
index 0000000000000..c6fa5d7835a93
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.sf100/simplified.txt
@@ -0,0 +1,90 @@
+WholeStageCodegen (14)
+  Sort [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov]
+    InputAdapter
+      Exchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1
+        WholeStageCodegen (13)
+          SortMergeJoin [i_item_sk,w_warehouse_sk,i_item_sk,w_warehouse_sk]
+            InputAdapter
+              WholeStageCodegen (6)
+                Sort [i_item_sk,w_warehouse_sk]
+                  InputAdapter
+                    Exchange [i_item_sk,w_warehouse_sk] #2
+                      WholeStageCodegen (5)
+                        Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev]
+                          Filter [mean,stdev]
+                            HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count]
+                              InputAdapter
+                                Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #3
+                                  WholeStageCodegen (4)
+                                    HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count]
+                                      Project [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy]
+                                        BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk]
+                                          Project [inv_warehouse_sk,inv_quantity_on_hand,d_moy,i_item_sk]
+                                            BroadcastHashJoin [inv_item_sk,i_item_sk]
+                                              Project [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,d_moy]
+                                                BroadcastHashJoin [inv_date_sk,d_date_sk]
+                                                  Filter [inv_item_sk,inv_warehouse_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk]
+                                                          SubqueryBroadcast [d_date_sk] #1
+                                                            ReusedExchange [d_date_sk,d_moy] #4
+                                                  InputAdapter
+                                                    BroadcastExchange #4
+                                                      WholeStageCodegen (1)
+                                                        Project [d_date_sk,d_moy]
+                                                          Filter [d_year,d_moy,d_date_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                              InputAdapter
+                                                BroadcastExchange #5
+                                                  WholeStageCodegen (2)
+                                                    Filter [i_item_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.item [i_item_sk]
+                                          InputAdapter
+                                            BroadcastExchange #6
+                                              WholeStageCodegen (3)
+                                                Filter [w_warehouse_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.warehouse [w_warehouse_sk,w_warehouse_name]
+            InputAdapter
+              WholeStageCodegen (12)
+                Sort [i_item_sk,w_warehouse_sk]
+                  InputAdapter
+                    Exchange [i_item_sk,w_warehouse_sk] #7
+                      WholeStageCodegen (11)
+                        Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev]
+                          Filter [mean,stdev]
+                            HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count]
+                              InputAdapter
+                                Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #8
+                                  WholeStageCodegen (10)
+                                    HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count]
+                                      Project [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy]
+                                        BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk]
+                                          Project [inv_warehouse_sk,inv_quantity_on_hand,d_moy,i_item_sk]
+                                            BroadcastHashJoin [inv_item_sk,i_item_sk]
+                                              Project [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,d_moy]
+                                                BroadcastHashJoin [inv_date_sk,d_date_sk]
+                                                  Filter [inv_item_sk,inv_warehouse_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk]
+                                                          SubqueryBroadcast [d_date_sk] #2
+                                                            ReusedExchange [d_date_sk,d_moy] #9
+                                                  InputAdapter
+                                                    BroadcastExchange #9
+                                                      WholeStageCodegen (7)
+                                                        Project [d_date_sk,d_moy]
+                                                          Filter [d_year,d_moy,d_date_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                              InputAdapter
+                                                ReusedExchange [i_item_sk] #5
+                                          InputAdapter
+                                            ReusedExchange [w_warehouse_sk,w_warehouse_name] #6
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b/explain.txt
new file mode 100644
index 0000000000000..d60cd37ce7bf1
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b/explain.txt
@@ -0,0 +1,311 @@
+== Physical Plan ==
+* Sort (52)
++- Exchange (51)
+   +- * BroadcastHashJoin Inner BuildRight (50)
+      :- * Project (27)
+      :  +- * Filter (26)
+      :     +- * HashAggregate (25)
+      :        +- Exchange (24)
+      :           +- * HashAggregate (23)
+      :              +- * Project (22)
+      :                 +- * BroadcastHashJoin Inner BuildRight (21)
+      :                    :- * Project (15)
+      :                    :  +- * BroadcastHashJoin Inner BuildRight (14)
+      :                    :     :- * Project (9)
+      :                    :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+      :                    :     :     :- * Filter (3)
+      :                    :     :     :  +- * ColumnarToRow (2)
+      :                    :     :     :     +- Scan parquet default.inventory (1)
+      :                    :     :     +- BroadcastExchange (7)
+      :                    :     :        +- * Filter (6)
+      :                    :     :           +- * ColumnarToRow (5)
+      :                    :     :              +- Scan parquet default.item (4)
+      :                    :     +- BroadcastExchange (13)
+      :                    :        +- * Filter (12)
+      :                    :           +- * ColumnarToRow (11)
+      :                    :              +- Scan parquet default.warehouse (10)
+      :                    +- BroadcastExchange (20)
+      :                       +- * Project (19)
+      :                          +- * Filter (18)
+      :                             +- * ColumnarToRow (17)
+      :                                +- Scan parquet default.date_dim (16)
+      +- BroadcastExchange (49)
+         +- * Project (48)
+            +- * Filter (47)
+               +- * HashAggregate (46)
+                  +- Exchange (45)
+                     +- * HashAggregate (44)
+                        +- * Project (43)
+                           +- * BroadcastHashJoin Inner BuildRight (42)
+                              :- * Project (36)
+                              :  +- * BroadcastHashJoin Inner BuildRight (35)
+                              :     :- * Project (33)
+                              :     :  +- * BroadcastHashJoin Inner BuildRight (32)
+                              :     :     :- * Filter (30)
+                              :     :     :  +- * ColumnarToRow (29)
+                              :     :     :     +- Scan parquet default.inventory (28)
+                              :     :     +- ReusedExchange (31)
+                              :     +- ReusedExchange (34)
+                              +- BroadcastExchange (41)
+                                 +- * Project (40)
+                                    +- * Filter (39)
+                                       +- * ColumnarToRow (38)
+                                          +- Scan parquet default.date_dim (37)
+
+
+(1) Scan parquet default.inventory
+Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)]
+ReadSchema: struct<inv_item_sk:int,inv_warehouse_sk:int,inv_quantity_on_hand:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+
+(3) Filter [codegen id : 4]
+Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2))
+
+(4) Scan parquet default.item
+Output [1]: [i_item_sk#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [1]: [i_item_sk#6]
+
+(6) Filter [codegen id : 1]
+Input [1]: [i_item_sk#6]
+Condition : isnotnull(i_item_sk#6)
+
+(7) BroadcastExchange
+Input [1]: [i_item_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#7]
+
+(8) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_item_sk#1]
+Right keys [1]: [i_item_sk#6]
+Join condition: None
+
+(9) Project [codegen id : 4]
+Output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6]
+Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6]
+
+(10) Scan parquet default.warehouse
+Output [2]: [w_warehouse_sk#8, w_warehouse_name#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/warehouse]
+PushedFilters: [IsNotNull(w_warehouse_sk)]
+ReadSchema: struct<w_warehouse_sk:int,w_warehouse_name:string>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [w_warehouse_sk#8, w_warehouse_name#9]
+
+(12) Filter [codegen id : 2]
+Input [2]: [w_warehouse_sk#8, w_warehouse_name#9]
+Condition : isnotnull(w_warehouse_sk#8)
+
+(13) BroadcastExchange
+Input [2]: [w_warehouse_sk#8, w_warehouse_name#9]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#10]
+
+(14) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_warehouse_sk#2]
+Right keys [1]: [w_warehouse_sk#8]
+Join condition: None
+
+(15) Project [codegen id : 4]
+Output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#8, w_warehouse_name#9]
+Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#8, w_warehouse_name#9]
+
+(16) Scan parquet default.date_dim
+Output [3]: [d_date_sk#11, d_year#12, d_moy#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(17) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#11, d_year#12, d_moy#13]
+
+(18) Filter [codegen id : 3]
+Input [3]: [d_date_sk#11, d_year#12, d_moy#13]
+Condition : ((((isnotnull(d_year#12) AND isnotnull(d_moy#13)) AND (d_year#12 = 2001)) AND (d_moy#13 = 1)) AND isnotnull(d_date_sk#11))
+
+(19) Project [codegen id : 3]
+Output [2]: [d_date_sk#11, d_moy#13]
+Input [3]: [d_date_sk#11, d_year#12, d_moy#13]
+
+(20) BroadcastExchange
+Input [2]: [d_date_sk#11, d_moy#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(21) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_date_sk#4]
+Right keys [1]: [d_date_sk#11]
+Join condition: None
+
+(22) Project [codegen id : 4]
+Output [5]: [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#8, w_warehouse_name#9, d_moy#13]
+Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#8, w_warehouse_name#9, d_date_sk#11, d_moy#13]
+
+(23) HashAggregate [codegen id : 4]
+Input [5]: [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#8, w_warehouse_name#9, d_moy#13]
+Keys [4]: [w_warehouse_name#9, w_warehouse_sk#8, i_item_sk#6, d_moy#13]
+Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)]
+Aggregate Attributes [5]: [n#15, avg#16, m2#17, sum#18, count#19]
+Results [9]: [w_warehouse_name#9, w_warehouse_sk#8, i_item_sk#6, d_moy#13, n#20, avg#21, m2#22, sum#23, count#24]
+
+(24) Exchange
+Input [9]: [w_warehouse_name#9, w_warehouse_sk#8, i_item_sk#6, d_moy#13, n#20, avg#21, m2#22, sum#23, count#24]
+Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sk#8, i_item_sk#6, d_moy#13, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(25) HashAggregate [codegen id : 10]
+Input [9]: [w_warehouse_name#9, w_warehouse_sk#8, i_item_sk#6, d_moy#13, n#20, avg#21, m2#22, sum#23, count#24]
+Keys [4]: [w_warehouse_name#9, w_warehouse_sk#8, i_item_sk#6, d_moy#13]
+Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)]
+Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double))#26, avg(inv_quantity_on_hand#3)#27]
+Results [5]: [w_warehouse_sk#8, i_item_sk#6, d_moy#13, stddev_samp(cast(inv_quantity_on_hand#3 as double))#26 AS stdev#28, avg(inv_quantity_on_hand#3)#27 AS mean#29]
+
+(26) Filter [codegen id : 10]
+Input [5]: [w_warehouse_sk#8, i_item_sk#6, d_moy#13, stdev#28, mean#29]
+Condition : ((isnotnull(mean#29) AND isnotnull(stdev#28)) AND ((NOT (mean#29 = 0.0) AND ((stdev#28 / mean#29) > 1.0)) AND ((stdev#28 / mean#29) > 1.5)))
+
+(27) Project [codegen id : 10]
+Output [5]: [w_warehouse_sk#8, i_item_sk#6, d_moy#13, mean#29, CASE WHEN (mean#29 = 0.0) THEN null ELSE (stdev#28 / mean#29) END AS cov#30]
+Input [5]: [w_warehouse_sk#8, i_item_sk#6, d_moy#13, stdev#28, mean#29]
+
+(28) Scan parquet default.inventory
+Output [4]: [inv_item_sk#31, inv_warehouse_sk#32, inv_quantity_on_hand#33, inv_date_sk#34]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(inv_date_sk#34), dynamicpruningexpression(inv_date_sk#34 IN dynamicpruning#35)]
+PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)]
+ReadSchema: struct<inv_item_sk:int,inv_warehouse_sk:int,inv_quantity_on_hand:int>
+
+(29) ColumnarToRow [codegen id : 8]
+Input [4]: [inv_item_sk#31, inv_warehouse_sk#32, inv_quantity_on_hand#33, inv_date_sk#34]
+
+(30) Filter [codegen id : 8]
+Input [4]: [inv_item_sk#31, inv_warehouse_sk#32, inv_quantity_on_hand#33, inv_date_sk#34]
+Condition : (isnotnull(inv_item_sk#31) AND isnotnull(inv_warehouse_sk#32))
+
+(31) ReusedExchange [Reuses operator id: 7]
+Output [1]: [i_item_sk#36]
+
+(32) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [inv_item_sk#31]
+Right keys [1]: [i_item_sk#36]
+Join condition: None
+
+(33) Project [codegen id : 8]
+Output [4]: [inv_warehouse_sk#32, inv_quantity_on_hand#33, inv_date_sk#34, i_item_sk#36]
+Input [5]: [inv_item_sk#31, inv_warehouse_sk#32, inv_quantity_on_hand#33, inv_date_sk#34, i_item_sk#36]
+
+(34) ReusedExchange [Reuses operator id: 13]
+Output [2]: [w_warehouse_sk#37, w_warehouse_name#38]
+
+(35) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [inv_warehouse_sk#32]
+Right keys [1]: [w_warehouse_sk#37]
+Join condition: None
+
+(36) Project [codegen id : 8]
+Output [5]: [inv_quantity_on_hand#33, inv_date_sk#34, i_item_sk#36, w_warehouse_sk#37, w_warehouse_name#38]
+Input [6]: [inv_warehouse_sk#32, inv_quantity_on_hand#33, inv_date_sk#34, i_item_sk#36, w_warehouse_sk#37, w_warehouse_name#38]
+
+(37) Scan parquet default.date_dim
+Output [3]: [d_date_sk#39, d_year#40, d_moy#41]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(38) ColumnarToRow [codegen id : 7]
+Input [3]: [d_date_sk#39, d_year#40, d_moy#41]
+
+(39) Filter [codegen id : 7]
+Input [3]: [d_date_sk#39, d_year#40, d_moy#41]
+Condition : ((((isnotnull(d_year#40) AND isnotnull(d_moy#41)) AND (d_year#40 = 2001)) AND (d_moy#41 = 2)) AND isnotnull(d_date_sk#39))
+
+(40) Project [codegen id : 7]
+Output [2]: [d_date_sk#39, d_moy#41]
+Input [3]: [d_date_sk#39, d_year#40, d_moy#41]
+
+(41) BroadcastExchange
+Input [2]: [d_date_sk#39, d_moy#41]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#42]
+
+(42) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [inv_date_sk#34]
+Right keys [1]: [d_date_sk#39]
+Join condition: None
+
+(43) Project [codegen id : 8]
+Output [5]: [inv_quantity_on_hand#33, i_item_sk#36, w_warehouse_sk#37, w_warehouse_name#38, d_moy#41]
+Input [7]: [inv_quantity_on_hand#33, inv_date_sk#34, i_item_sk#36, w_warehouse_sk#37, w_warehouse_name#38, d_date_sk#39, d_moy#41]
+
+(44) HashAggregate [codegen id : 8]
+Input [5]: [inv_quantity_on_hand#33, i_item_sk#36, w_warehouse_sk#37, w_warehouse_name#38, d_moy#41]
+Keys [4]: [w_warehouse_name#38, w_warehouse_sk#37, i_item_sk#36, d_moy#41]
+Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#33 as double)), partial_avg(inv_quantity_on_hand#33)]
+Aggregate Attributes [5]: [n#43, avg#44, m2#45, sum#46, count#47]
+Results [9]: [w_warehouse_name#38, w_warehouse_sk#37, i_item_sk#36, d_moy#41, n#48, avg#49, m2#50, sum#51, count#52]
+
+(45) Exchange
+Input [9]: [w_warehouse_name#38, w_warehouse_sk#37, i_item_sk#36, d_moy#41, n#48, avg#49, m2#50, sum#51, count#52]
+Arguments: hashpartitioning(w_warehouse_name#38, w_warehouse_sk#37, i_item_sk#36, d_moy#41, 5), ENSURE_REQUIREMENTS, [id=#53]
+
+(46) HashAggregate [codegen id : 9]
+Input [9]: [w_warehouse_name#38, w_warehouse_sk#37, i_item_sk#36, d_moy#41, n#48, avg#49, m2#50, sum#51, count#52]
+Keys [4]: [w_warehouse_name#38, w_warehouse_sk#37, i_item_sk#36, d_moy#41]
+Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#33 as double)), avg(inv_quantity_on_hand#33)]
+Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#33 as double))#54, avg(inv_quantity_on_hand#33)#55]
+Results [5]: [w_warehouse_sk#37, i_item_sk#36, d_moy#41, stddev_samp(cast(inv_quantity_on_hand#33 as double))#54 AS stdev#56, avg(inv_quantity_on_hand#33)#55 AS mean#57]
+
+(47) Filter [codegen id : 9]
+Input [5]: [w_warehouse_sk#37, i_item_sk#36, d_moy#41, stdev#56, mean#57]
+Condition : ((isnotnull(mean#57) AND isnotnull(stdev#56)) AND (NOT (mean#57 = 0.0) AND ((stdev#56 / mean#57) > 1.0)))
+
+(48) Project [codegen id : 9]
+Output [5]: [w_warehouse_sk#37, i_item_sk#36, d_moy#41, mean#57, CASE WHEN (mean#57 = 0.0) THEN null ELSE (stdev#56 / mean#57) END AS cov#58]
+Input [5]: [w_warehouse_sk#37, i_item_sk#36, d_moy#41, stdev#56, mean#57]
+
+(49) BroadcastExchange
+Input [5]: [w_warehouse_sk#37, i_item_sk#36, d_moy#41, mean#57, cov#58]
+Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [id=#59]
+
+(50) BroadcastHashJoin [codegen id : 10]
+Left keys [2]: [i_item_sk#6, w_warehouse_sk#8]
+Right keys [2]: [i_item_sk#36, w_warehouse_sk#37]
+Join condition: None
+
+(51) Exchange
+Input [10]: [w_warehouse_sk#8, i_item_sk#6, d_moy#13, mean#29, cov#30, w_warehouse_sk#37, i_item_sk#36, d_moy#41, mean#57, cov#58]
+Arguments: rangepartitioning(w_warehouse_sk#8 ASC NULLS FIRST, i_item_sk#6 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST, mean#29 ASC NULLS FIRST, cov#30 ASC NULLS FIRST, d_moy#41 ASC NULLS FIRST, mean#57 ASC NULLS FIRST, cov#58 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [id=#60]
+
+(52) Sort [codegen id : 11]
+Input [10]: [w_warehouse_sk#8, i_item_sk#6, d_moy#13, mean#29, cov#30, w_warehouse_sk#37, i_item_sk#36, d_moy#41, mean#57, cov#58]
+Arguments: [w_warehouse_sk#8 ASC NULLS FIRST, i_item_sk#6 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST, mean#29 ASC NULLS FIRST, cov#30 ASC NULLS FIRST, d_moy#41 ASC NULLS FIRST, mean#57 ASC NULLS FIRST, cov#58 ASC NULLS FIRST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5
+ReusedExchange (53)
+
+
+(53) ReusedExchange [Reuses operator id: 20]
+Output [2]: [d_date_sk#11, d_moy#13]
+
+Subquery:2 Hosting operator id = 28 Hosting Expression = inv_date_sk#34 IN dynamicpruning#35
+ReusedExchange (54)
+
+
+(54) ReusedExchange [Reuses operator id: 41]
+Output [2]: [d_date_sk#39, d_moy#41]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b/simplified.txt
new file mode 100644
index 0000000000000..ecd1313c1f5e1
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b/simplified.txt
@@ -0,0 +1,81 @@
+WholeStageCodegen (11)
+  Sort [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov]
+    InputAdapter
+      Exchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1
+        WholeStageCodegen (10)
+          BroadcastHashJoin [i_item_sk,w_warehouse_sk,i_item_sk,w_warehouse_sk]
+            Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev]
+              Filter [mean,stdev]
+                HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count]
+                  InputAdapter
+                    Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2
+                      WholeStageCodegen (4)
+                        HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count]
+                          Project [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy]
+                            BroadcastHashJoin [inv_date_sk,d_date_sk]
+                              Project [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name]
+                                BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk]
+                                  Project [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk]
+                                    BroadcastHashJoin [inv_item_sk,i_item_sk]
+                                      Filter [inv_item_sk,inv_warehouse_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #1
+                                                ReusedExchange [d_date_sk,d_moy] #3
+                                      InputAdapter
+                                        BroadcastExchange #4
+                                          WholeStageCodegen (1)
+                                            Filter [i_item_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.item [i_item_sk]
+                                  InputAdapter
+                                    BroadcastExchange #5
+                                      WholeStageCodegen (2)
+                                        Filter [w_warehouse_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.warehouse [w_warehouse_sk,w_warehouse_name]
+                              InputAdapter
+                                BroadcastExchange #3
+                                  WholeStageCodegen (3)
+                                    Project [d_date_sk,d_moy]
+                                      Filter [d_year,d_moy,d_date_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+            InputAdapter
+              BroadcastExchange #6
+                WholeStageCodegen (9)
+                  Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev]
+                    Filter [mean,stdev]
+                      HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count]
+                        InputAdapter
+                          Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7
+                            WholeStageCodegen (8)
+                              HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count]
+                                Project [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy]
+                                  BroadcastHashJoin [inv_date_sk,d_date_sk]
+                                    Project [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name]
+                                      BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk]
+                                        Project [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk]
+                                          BroadcastHashJoin [inv_item_sk,i_item_sk]
+                                            Filter [inv_item_sk,inv_warehouse_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #2
+                                                      ReusedExchange [d_date_sk,d_moy] #8
+                                            InputAdapter
+                                              ReusedExchange [i_item_sk] #4
+                                        InputAdapter
+                                          ReusedExchange [w_warehouse_sk,w_warehouse_name] #5
+                                    InputAdapter
+                                      BroadcastExchange #8
+                                        WholeStageCodegen (7)
+                                          Project [d_date_sk,d_moy]
+                                            Filter [d_year,d_moy,d_date_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.sf100/explain.txt
new file mode 100644
index 0000000000000..ce8b8bed5a26e
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.sf100/explain.txt
@@ -0,0 +1,726 @@
+== Physical Plan ==
+TakeOrderedAndProject (126)
++- * Project (125)
+   +- * SortMergeJoin Inner (124)
+      :- * Project (106)
+      :  +- * SortMergeJoin Inner (105)
+      :     :- * Project (85)
+      :     :  +- * SortMergeJoin Inner (84)
+      :     :     :- * Project (66)
+      :     :     :  +- * SortMergeJoin Inner (65)
+      :     :     :     :- * SortMergeJoin Inner (45)
+      :     :     :     :  :- * Sort (24)
+      :     :     :     :  :  +- Exchange (23)
+      :     :     :     :  :     +- * Filter (22)
+      :     :     :     :  :        +- * HashAggregate (21)
+      :     :     :     :  :           +- Exchange (20)
+      :     :     :     :  :              +- * HashAggregate (19)
+      :     :     :     :  :                 +- * Project (18)
+      :     :     :     :  :                    +- * SortMergeJoin Inner (17)
+      :     :     :     :  :                       :- * Sort (11)
+      :     :     :     :  :                       :  +- Exchange (10)
+      :     :     :     :  :                       :     +- * Project (9)
+      :     :     :     :  :                       :        +- * BroadcastHashJoin Inner BuildRight (8)
+      :     :     :     :  :                       :           :- * Filter (3)
+      :     :     :     :  :                       :           :  +- * ColumnarToRow (2)
+      :     :     :     :  :                       :           :     +- Scan parquet default.store_sales (1)
+      :     :     :     :  :                       :           +- BroadcastExchange (7)
+      :     :     :     :  :                       :              +- * Filter (6)
+      :     :     :     :  :                       :                 +- * ColumnarToRow (5)
+      :     :     :     :  :                       :                    +- Scan parquet default.date_dim (4)
+      :     :     :     :  :                       +- * Sort (16)
+      :     :     :     :  :                          +- Exchange (15)
+      :     :     :     :  :                             +- * Filter (14)
+      :     :     :     :  :                                +- * ColumnarToRow (13)
+      :     :     :     :  :                                   +- Scan parquet default.customer (12)
+      :     :     :     :  +- * Sort (44)
+      :     :     :     :     +- Exchange (43)
+      :     :     :     :        +- * HashAggregate (42)
+      :     :     :     :           +- Exchange (41)
+      :     :     :     :              +- * HashAggregate (40)
+      :     :     :     :                 +- * Project (39)
+      :     :     :     :                    +- * SortMergeJoin Inner (38)
+      :     :     :     :                       :- * Sort (35)
+      :     :     :     :                       :  +- Exchange (34)
+      :     :     :     :                       :     +- * Project (33)
+      :     :     :     :                       :        +- * BroadcastHashJoin Inner BuildRight (32)
+      :     :     :     :                       :           :- * Filter (27)
+      :     :     :     :                       :           :  +- * ColumnarToRow (26)
+      :     :     :     :                       :           :     +- Scan parquet default.store_sales (25)
+      :     :     :     :                       :           +- BroadcastExchange (31)
+      :     :     :     :                       :              +- * Filter (30)
+      :     :     :     :                       :                 +- * ColumnarToRow (29)
+      :     :     :     :                       :                    +- Scan parquet default.date_dim (28)
+      :     :     :     :                       +- * Sort (37)
+      :     :     :     :                          +- ReusedExchange (36)
+      :     :     :     +- * Sort (64)
+      :     :     :        +- Exchange (63)
+      :     :     :           +- * Project (62)
+      :     :     :              +- * Filter (61)
+      :     :     :                 +- * HashAggregate (60)
+      :     :     :                    +- Exchange (59)
+      :     :     :                       +- * HashAggregate (58)
+      :     :     :                          +- * Project (57)
+      :     :     :                             +- * SortMergeJoin Inner (56)
+      :     :     :                                :- * Sort (53)
+      :     :     :                                :  +- Exchange (52)
+      :     :     :                                :     +- * Project (51)
+      :     :     :                                :        +- * BroadcastHashJoin Inner BuildRight (50)
+      :     :     :                                :           :- * Filter (48)
+      :     :     :                                :           :  +- * ColumnarToRow (47)
+      :     :     :                                :           :     +- Scan parquet default.catalog_sales (46)
+      :     :     :                                :           +- ReusedExchange (49)
+      :     :     :                                +- * Sort (55)
+      :     :     :                                   +- ReusedExchange (54)
+      :     :     +- * Sort (83)
+      :     :        +- Exchange (82)
+      :     :           +- * HashAggregate (81)
+      :     :              +- Exchange (80)
+      :     :                 +- * HashAggregate (79)
+      :     :                    +- * Project (78)
+      :     :                       +- * SortMergeJoin Inner (77)
+      :     :                          :- * Sort (74)
+      :     :                          :  +- Exchange (73)
+      :     :                          :     +- * Project (72)
+      :     :                          :        +- * BroadcastHashJoin Inner BuildRight (71)
+      :     :                          :           :- * Filter (69)
+      :     :                          :           :  +- * ColumnarToRow (68)
+      :     :                          :           :     +- Scan parquet default.catalog_sales (67)
+      :     :                          :           +- ReusedExchange (70)
+      :     :                          +- * Sort (76)
+      :     :                             +- ReusedExchange (75)
+      :     +- * Sort (104)
+      :        +- Exchange (103)
+      :           +- * Project (102)
+      :              +- * Filter (101)
+      :                 +- * HashAggregate (100)
+      :                    +- Exchange (99)
+      :                       +- * HashAggregate (98)
+      :                          +- * Project (97)
+      :                             +- * SortMergeJoin Inner (96)
+      :                                :- * Sort (93)
+      :                                :  +- Exchange (92)
+      :                                :     +- * Project (91)
+      :                                :        +- * BroadcastHashJoin Inner BuildRight (90)
+      :                                :           :- * Filter (88)
+      :                                :           :  +- * ColumnarToRow (87)
+      :                                :           :     +- Scan parquet default.web_sales (86)
+      :                                :           +- ReusedExchange (89)
+      :                                +- * Sort (95)
+      :                                   +- ReusedExchange (94)
+      +- * Sort (123)
+         +- Exchange (122)
+            +- * HashAggregate (121)
+               +- Exchange (120)
+                  +- * HashAggregate (119)
+                     +- * Project (118)
+                        +- * SortMergeJoin Inner (117)
+                           :- * Sort (114)
+                           :  +- Exchange (113)
+                           :     +- * Project (112)
+                           :        +- * BroadcastHashJoin Inner BuildRight (111)
+                           :           :- * Filter (109)
+                           :           :  +- * ColumnarToRow (108)
+                           :           :     +- Scan parquet default.web_sales (107)
+                           :           +- ReusedExchange (110)
+                           +- * Sort (116)
+                              +- ReusedExchange (115)
+
+
+(1) Scan parquet default.store_sales
+Output [6]: [ss_customer_sk#1, ss_ext_discount_amt#2, ss_ext_sales_price#3, ss_ext_wholesale_cost#4, ss_ext_list_price#5, ss_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_ext_discount_amt:decimal(7,2),ss_ext_sales_price:decimal(7,2),ss_ext_wholesale_cost:decimal(7,2),ss_ext_list_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [6]: [ss_customer_sk#1, ss_ext_discount_amt#2, ss_ext_sales_price#3, ss_ext_wholesale_cost#4, ss_ext_list_price#5, ss_sold_date_sk#6]
+
+(3) Filter [codegen id : 2]
+Input [6]: [ss_customer_sk#1, ss_ext_discount_amt#2, ss_ext_sales_price#3, ss_ext_wholesale_cost#4, ss_ext_list_price#5, ss_sold_date_sk#6]
+Condition : isnotnull(ss_customer_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#8, d_year#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#8, d_year#9]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#8, d_year#9]
+Condition : ((isnotnull(d_year#9) AND (d_year#9 = 2001)) AND isnotnull(d_date_sk#8))
+
+(7) BroadcastExchange
+Input [2]: [d_date_sk#8, d_year#9]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#10]
+
+(8) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#6]
+Right keys [1]: [d_date_sk#8]
+Join condition: None
+
+(9) Project [codegen id : 2]
+Output [6]: [ss_customer_sk#1, ss_ext_discount_amt#2, ss_ext_sales_price#3, ss_ext_wholesale_cost#4, ss_ext_list_price#5, d_year#9]
+Input [8]: [ss_customer_sk#1, ss_ext_discount_amt#2, ss_ext_sales_price#3, ss_ext_wholesale_cost#4, ss_ext_list_price#5, ss_sold_date_sk#6, d_date_sk#8, d_year#9]
+
+(10) Exchange
+Input [6]: [ss_customer_sk#1, ss_ext_discount_amt#2, ss_ext_sales_price#3, ss_ext_wholesale_cost#4, ss_ext_list_price#5, d_year#9]
+Arguments: hashpartitioning(ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#11]
+
+(11) Sort [codegen id : 3]
+Input [6]: [ss_customer_sk#1, ss_ext_discount_amt#2, ss_ext_sales_price#3, ss_ext_wholesale_cost#4, ss_ext_list_price#5, d_year#9]
+Arguments: [ss_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(12) Scan parquet default.customer
+Output [8]: [c_customer_sk#12, c_customer_id#13, c_first_name#14, c_last_name#15, c_preferred_cust_flag#16, c_birth_country#17, c_login#18, c_email_address#19]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string,c_birth_country:string,c_login:string,c_email_address:string>
+
+(13) ColumnarToRow [codegen id : 4]
+Input [8]: [c_customer_sk#12, c_customer_id#13, c_first_name#14, c_last_name#15, c_preferred_cust_flag#16, c_birth_country#17, c_login#18, c_email_address#19]
+
+(14) Filter [codegen id : 4]
+Input [8]: [c_customer_sk#12, c_customer_id#13, c_first_name#14, c_last_name#15, c_preferred_cust_flag#16, c_birth_country#17, c_login#18, c_email_address#19]
+Condition : (isnotnull(c_customer_sk#12) AND isnotnull(c_customer_id#13))
+
+(15) Exchange
+Input [8]: [c_customer_sk#12, c_customer_id#13, c_first_name#14, c_last_name#15, c_preferred_cust_flag#16, c_birth_country#17, c_login#18, c_email_address#19]
+Arguments: hashpartitioning(c_customer_sk#12, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(16) Sort [codegen id : 5]
+Input [8]: [c_customer_sk#12, c_customer_id#13, c_first_name#14, c_last_name#15, c_preferred_cust_flag#16, c_birth_country#17, c_login#18, c_email_address#19]
+Arguments: [c_customer_sk#12 ASC NULLS FIRST], false, 0
+
+(17) SortMergeJoin [codegen id : 6]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#12]
+Join condition: None
+
+(18) Project [codegen id : 6]
+Output [12]: [c_customer_id#13, c_first_name#14, c_last_name#15, c_preferred_cust_flag#16, c_birth_country#17, c_login#18, c_email_address#19, ss_ext_discount_amt#2, ss_ext_sales_price#3, ss_ext_wholesale_cost#4, ss_ext_list_price#5, d_year#9]
+Input [14]: [ss_customer_sk#1, ss_ext_discount_amt#2, ss_ext_sales_price#3, ss_ext_wholesale_cost#4, ss_ext_list_price#5, d_year#9, c_customer_sk#12, c_customer_id#13, c_first_name#14, c_last_name#15, c_preferred_cust_flag#16, c_birth_country#17, c_login#18, c_email_address#19]
+
+(19) HashAggregate [codegen id : 6]
+Input [12]: [c_customer_id#13, c_first_name#14, c_last_name#15, c_preferred_cust_flag#16, c_birth_country#17, c_login#18, c_email_address#19, ss_ext_discount_amt#2, ss_ext_sales_price#3, ss_ext_wholesale_cost#4, ss_ext_list_price#5, d_year#9]
+Keys [8]: [c_customer_id#13, c_first_name#14, c_last_name#15, c_preferred_cust_flag#16, c_birth_country#17, c_login#18, c_email_address#19, d_year#9]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_ext_list_price#5 as decimal(8,2))) - promote_precision(cast(ss_ext_wholesale_cost#4 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ss_ext_discount_amt#2 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ss_ext_sales_price#3 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))]
+Aggregate Attributes [2]: [sum#21, isEmpty#22]
+Results [10]: [c_customer_id#13, c_first_name#14, c_last_name#15, c_preferred_cust_flag#16, c_birth_country#17, c_login#18, c_email_address#19, d_year#9, sum#23, isEmpty#24]
+
+(20) Exchange
+Input [10]: [c_customer_id#13, c_first_name#14, c_last_name#15, c_preferred_cust_flag#16, c_birth_country#17, c_login#18, c_email_address#19, d_year#9, sum#23, isEmpty#24]
+Arguments: hashpartitioning(c_customer_id#13, c_first_name#14, c_last_name#15, c_preferred_cust_flag#16, c_birth_country#17, c_login#18, c_email_address#19, d_year#9, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(21) HashAggregate [codegen id : 7]
+Input [10]: [c_customer_id#13, c_first_name#14, c_last_name#15, c_preferred_cust_flag#16, c_birth_country#17, c_login#18, c_email_address#19, d_year#9, sum#23, isEmpty#24]
+Keys [8]: [c_customer_id#13, c_first_name#14, c_last_name#15, c_preferred_cust_flag#16, c_birth_country#17, c_login#18, c_email_address#19, d_year#9]
+Functions [1]: [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_ext_list_price#5 as decimal(8,2))) - promote_precision(cast(ss_ext_wholesale_cost#4 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ss_ext_discount_amt#2 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ss_ext_sales_price#3 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_ext_list_price#5 as decimal(8,2))) - promote_precision(cast(ss_ext_wholesale_cost#4 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ss_ext_discount_amt#2 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ss_ext_sales_price#3 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))#26]
+Results [2]: [c_customer_id#13 AS customer_id#27, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_ext_list_price#5 as decimal(8,2))) - promote_precision(cast(ss_ext_wholesale_cost#4 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ss_ext_discount_amt#2 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ss_ext_sales_price#3 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))#26 AS year_total#28]
+
+(22) Filter [codegen id : 7]
+Input [2]: [customer_id#27, year_total#28]
+Condition : (isnotnull(year_total#28) AND (year_total#28 > 0.000000))
+
+(23) Exchange
+Input [2]: [customer_id#27, year_total#28]
+Arguments: hashpartitioning(customer_id#27, 5), ENSURE_REQUIREMENTS, [id=#29]
+
+(24) Sort [codegen id : 8]
+Input [2]: [customer_id#27, year_total#28]
+Arguments: [customer_id#27 ASC NULLS FIRST], false, 0
+
+(25) Scan parquet default.store_sales
+Output [6]: [ss_customer_sk#30, ss_ext_discount_amt#31, ss_ext_sales_price#32, ss_ext_wholesale_cost#33, ss_ext_list_price#34, ss_sold_date_sk#35]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#35), dynamicpruningexpression(ss_sold_date_sk#35 IN dynamicpruning#36)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_ext_discount_amt:decimal(7,2),ss_ext_sales_price:decimal(7,2),ss_ext_wholesale_cost:decimal(7,2),ss_ext_list_price:decimal(7,2)>
+
+(26) ColumnarToRow [codegen id : 10]
+Input [6]: [ss_customer_sk#30, ss_ext_discount_amt#31, ss_ext_sales_price#32, ss_ext_wholesale_cost#33, ss_ext_list_price#34, ss_sold_date_sk#35]
+
+(27) Filter [codegen id : 10]
+Input [6]: [ss_customer_sk#30, ss_ext_discount_amt#31, ss_ext_sales_price#32, ss_ext_wholesale_cost#33, ss_ext_list_price#34, ss_sold_date_sk#35]
+Condition : isnotnull(ss_customer_sk#30)
+
+(28) Scan parquet default.date_dim
+Output [2]: [d_date_sk#37, d_year#38]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(29) ColumnarToRow [codegen id : 9]
+Input [2]: [d_date_sk#37, d_year#38]
+
+(30) Filter [codegen id : 9]
+Input [2]: [d_date_sk#37, d_year#38]
+Condition : ((isnotnull(d_year#38) AND (d_year#38 = 2002)) AND isnotnull(d_date_sk#37))
+
+(31) BroadcastExchange
+Input [2]: [d_date_sk#37, d_year#38]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#39]
+
+(32) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_sold_date_sk#35]
+Right keys [1]: [d_date_sk#37]
+Join condition: None
+
+(33) Project [codegen id : 10]
+Output [6]: [ss_customer_sk#30, ss_ext_discount_amt#31, ss_ext_sales_price#32, ss_ext_wholesale_cost#33, ss_ext_list_price#34, d_year#38]
+Input [8]: [ss_customer_sk#30, ss_ext_discount_amt#31, ss_ext_sales_price#32, ss_ext_wholesale_cost#33, ss_ext_list_price#34, ss_sold_date_sk#35, d_date_sk#37, d_year#38]
+
+(34) Exchange
+Input [6]: [ss_customer_sk#30, ss_ext_discount_amt#31, ss_ext_sales_price#32, ss_ext_wholesale_cost#33, ss_ext_list_price#34, d_year#38]
+Arguments: hashpartitioning(ss_customer_sk#30, 5), ENSURE_REQUIREMENTS, [id=#40]
+
+(35) Sort [codegen id : 11]
+Input [6]: [ss_customer_sk#30, ss_ext_discount_amt#31, ss_ext_sales_price#32, ss_ext_wholesale_cost#33, ss_ext_list_price#34, d_year#38]
+Arguments: [ss_customer_sk#30 ASC NULLS FIRST], false, 0
+
+(36) ReusedExchange [Reuses operator id: 15]
+Output [8]: [c_customer_sk#41, c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48]
+
+(37) Sort [codegen id : 13]
+Input [8]: [c_customer_sk#41, c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48]
+Arguments: [c_customer_sk#41 ASC NULLS FIRST], false, 0
+
+(38) SortMergeJoin [codegen id : 14]
+Left keys [1]: [ss_customer_sk#30]
+Right keys [1]: [c_customer_sk#41]
+Join condition: None
+
+(39) Project [codegen id : 14]
+Output [12]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ss_ext_discount_amt#31, ss_ext_sales_price#32, ss_ext_wholesale_cost#33, ss_ext_list_price#34, d_year#38]
+Input [14]: [ss_customer_sk#30, ss_ext_discount_amt#31, ss_ext_sales_price#32, ss_ext_wholesale_cost#33, ss_ext_list_price#34, d_year#38, c_customer_sk#41, c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48]
+
+(40) HashAggregate [codegen id : 14]
+Input [12]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ss_ext_discount_amt#31, ss_ext_sales_price#32, ss_ext_wholesale_cost#33, ss_ext_list_price#34, d_year#38]
+Keys [8]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#38]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_ext_list_price#34 as decimal(8,2))) - promote_precision(cast(ss_ext_wholesale_cost#33 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ss_ext_discount_amt#31 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ss_ext_sales_price#32 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))]
+Aggregate Attributes [2]: [sum#49, isEmpty#50]
+Results [10]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#38, sum#51, isEmpty#52]
+
+(41) Exchange
+Input [10]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#38, sum#51, isEmpty#52]
+Arguments: hashpartitioning(c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#38, 5), ENSURE_REQUIREMENTS, [id=#53]
+
+(42) HashAggregate [codegen id : 15]
+Input [10]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#38, sum#51, isEmpty#52]
+Keys [8]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#38]
+Functions [1]: [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_ext_list_price#34 as decimal(8,2))) - promote_precision(cast(ss_ext_wholesale_cost#33 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ss_ext_discount_amt#31 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ss_ext_sales_price#32 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_ext_list_price#34 as decimal(8,2))) - promote_precision(cast(ss_ext_wholesale_cost#33 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ss_ext_discount_amt#31 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ss_ext_sales_price#32 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))#54]
+Results [8]: [c_customer_id#42 AS customer_id#55, c_first_name#43 AS customer_first_name#56, c_last_name#44 AS customer_last_name#57, c_preferred_cust_flag#45 AS customer_preferred_cust_flag#58, c_birth_country#46 AS customer_birth_country#59, c_login#47 AS customer_login#60, c_email_address#48 AS customer_email_address#61, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_ext_list_price#34 as decimal(8,2))) - promote_precision(cast(ss_ext_wholesale_cost#33 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ss_ext_discount_amt#31 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ss_ext_sales_price#32 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))#54 AS year_total#62]
+
+(43) Exchange
+Input [8]: [customer_id#55, customer_first_name#56, customer_last_name#57, customer_preferred_cust_flag#58, customer_birth_country#59, customer_login#60, customer_email_address#61, year_total#62]
+Arguments: hashpartitioning(customer_id#55, 5), ENSURE_REQUIREMENTS, [id=#63]
+
+(44) Sort [codegen id : 16]
+Input [8]: [customer_id#55, customer_first_name#56, customer_last_name#57, customer_preferred_cust_flag#58, customer_birth_country#59, customer_login#60, customer_email_address#61, year_total#62]
+Arguments: [customer_id#55 ASC NULLS FIRST], false, 0
+
+(45) SortMergeJoin [codegen id : 17]
+Left keys [1]: [customer_id#27]
+Right keys [1]: [customer_id#55]
+Join condition: None
+
+(46) Scan parquet default.catalog_sales
+Output [6]: [cs_bill_customer_sk#64, cs_ext_discount_amt#65, cs_ext_sales_price#66, cs_ext_wholesale_cost#67, cs_ext_list_price#68, cs_sold_date_sk#69]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#69), dynamicpruningexpression(cs_sold_date_sk#69 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(cs_bill_customer_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_ext_discount_amt:decimal(7,2),cs_ext_sales_price:decimal(7,2),cs_ext_wholesale_cost:decimal(7,2),cs_ext_list_price:decimal(7,2)>
+
+(47) ColumnarToRow [codegen id : 19]
+Input [6]: [cs_bill_customer_sk#64, cs_ext_discount_amt#65, cs_ext_sales_price#66, cs_ext_wholesale_cost#67, cs_ext_list_price#68, cs_sold_date_sk#69]
+
+(48) Filter [codegen id : 19]
+Input [6]: [cs_bill_customer_sk#64, cs_ext_discount_amt#65, cs_ext_sales_price#66, cs_ext_wholesale_cost#67, cs_ext_list_price#68, cs_sold_date_sk#69]
+Condition : isnotnull(cs_bill_customer_sk#64)
+
+(49) ReusedExchange [Reuses operator id: 7]
+Output [2]: [d_date_sk#70, d_year#71]
+
+(50) BroadcastHashJoin [codegen id : 19]
+Left keys [1]: [cs_sold_date_sk#69]
+Right keys [1]: [d_date_sk#70]
+Join condition: None
+
+(51) Project [codegen id : 19]
+Output [6]: [cs_bill_customer_sk#64, cs_ext_discount_amt#65, cs_ext_sales_price#66, cs_ext_wholesale_cost#67, cs_ext_list_price#68, d_year#71]
+Input [8]: [cs_bill_customer_sk#64, cs_ext_discount_amt#65, cs_ext_sales_price#66, cs_ext_wholesale_cost#67, cs_ext_list_price#68, cs_sold_date_sk#69, d_date_sk#70, d_year#71]
+
+(52) Exchange
+Input [6]: [cs_bill_customer_sk#64, cs_ext_discount_amt#65, cs_ext_sales_price#66, cs_ext_wholesale_cost#67, cs_ext_list_price#68, d_year#71]
+Arguments: hashpartitioning(cs_bill_customer_sk#64, 5), ENSURE_REQUIREMENTS, [id=#72]
+
+(53) Sort [codegen id : 20]
+Input [6]: [cs_bill_customer_sk#64, cs_ext_discount_amt#65, cs_ext_sales_price#66, cs_ext_wholesale_cost#67, cs_ext_list_price#68, d_year#71]
+Arguments: [cs_bill_customer_sk#64 ASC NULLS FIRST], false, 0
+
+(54) ReusedExchange [Reuses operator id: 15]
+Output [8]: [c_customer_sk#73, c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80]
+
+(55) Sort [codegen id : 22]
+Input [8]: [c_customer_sk#73, c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80]
+Arguments: [c_customer_sk#73 ASC NULLS FIRST], false, 0
+
+(56) SortMergeJoin [codegen id : 23]
+Left keys [1]: [cs_bill_customer_sk#64]
+Right keys [1]: [c_customer_sk#73]
+Join condition: None
+
+(57) Project [codegen id : 23]
+Output [12]: [c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80, cs_ext_discount_amt#65, cs_ext_sales_price#66, cs_ext_wholesale_cost#67, cs_ext_list_price#68, d_year#71]
+Input [14]: [cs_bill_customer_sk#64, cs_ext_discount_amt#65, cs_ext_sales_price#66, cs_ext_wholesale_cost#67, cs_ext_list_price#68, d_year#71, c_customer_sk#73, c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80]
+
+(58) HashAggregate [codegen id : 23]
+Input [12]: [c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80, cs_ext_discount_amt#65, cs_ext_sales_price#66, cs_ext_wholesale_cost#67, cs_ext_list_price#68, d_year#71]
+Keys [8]: [c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80, d_year#71]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cs_ext_list_price#68 as decimal(8,2))) - promote_precision(cast(cs_ext_wholesale_cost#67 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(cs_ext_discount_amt#65 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(cs_ext_sales_price#66 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))]
+Aggregate Attributes [2]: [sum#81, isEmpty#82]
+Results [10]: [c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80, d_year#71, sum#83, isEmpty#84]
+
+(59) Exchange
+Input [10]: [c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80, d_year#71, sum#83, isEmpty#84]
+Arguments: hashpartitioning(c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80, d_year#71, 5), ENSURE_REQUIREMENTS, [id=#85]
+
+(60) HashAggregate [codegen id : 24]
+Input [10]: [c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80, d_year#71, sum#83, isEmpty#84]
+Keys [8]: [c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80, d_year#71]
+Functions [1]: [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cs_ext_list_price#68 as decimal(8,2))) - promote_precision(cast(cs_ext_wholesale_cost#67 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(cs_ext_discount_amt#65 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(cs_ext_sales_price#66 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cs_ext_list_price#68 as decimal(8,2))) - promote_precision(cast(cs_ext_wholesale_cost#67 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(cs_ext_discount_amt#65 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(cs_ext_sales_price#66 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))#86]
+Results [2]: [c_customer_id#74 AS customer_id#87, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cs_ext_list_price#68 as decimal(8,2))) - promote_precision(cast(cs_ext_wholesale_cost#67 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(cs_ext_discount_amt#65 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(cs_ext_sales_price#66 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))#86 AS year_total#88]
+
+(61) Filter [codegen id : 24]
+Input [2]: [customer_id#87, year_total#88]
+Condition : (isnotnull(year_total#88) AND (year_total#88 > 0.000000))
+
+(62) Project [codegen id : 24]
+Output [2]: [customer_id#87 AS customer_id#89, year_total#88 AS year_total#90]
+Input [2]: [customer_id#87, year_total#88]
+
+(63) Exchange
+Input [2]: [customer_id#89, year_total#90]
+Arguments: hashpartitioning(customer_id#89, 5), ENSURE_REQUIREMENTS, [id=#91]
+
+(64) Sort [codegen id : 25]
+Input [2]: [customer_id#89, year_total#90]
+Arguments: [customer_id#89 ASC NULLS FIRST], false, 0
+
+(65) SortMergeJoin [codegen id : 26]
+Left keys [1]: [customer_id#27]
+Right keys [1]: [customer_id#89]
+Join condition: None
+
+(66) Project [codegen id : 26]
+Output [11]: [customer_id#27, year_total#28, customer_id#55, customer_first_name#56, customer_last_name#57, customer_preferred_cust_flag#58, customer_birth_country#59, customer_login#60, customer_email_address#61, year_total#62, year_total#90]
+Input [12]: [customer_id#27, year_total#28, customer_id#55, customer_first_name#56, customer_last_name#57, customer_preferred_cust_flag#58, customer_birth_country#59, customer_login#60, customer_email_address#61, year_total#62, customer_id#89, year_total#90]
+
+(67) Scan parquet default.catalog_sales
+Output [6]: [cs_bill_customer_sk#92, cs_ext_discount_amt#93, cs_ext_sales_price#94, cs_ext_wholesale_cost#95, cs_ext_list_price#96, cs_sold_date_sk#97]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#97), dynamicpruningexpression(cs_sold_date_sk#97 IN dynamicpruning#36)]
+PushedFilters: [IsNotNull(cs_bill_customer_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_ext_discount_amt:decimal(7,2),cs_ext_sales_price:decimal(7,2),cs_ext_wholesale_cost:decimal(7,2),cs_ext_list_price:decimal(7,2)>
+
+(68) ColumnarToRow [codegen id : 28]
+Input [6]: [cs_bill_customer_sk#92, cs_ext_discount_amt#93, cs_ext_sales_price#94, cs_ext_wholesale_cost#95, cs_ext_list_price#96, cs_sold_date_sk#97]
+
+(69) Filter [codegen id : 28]
+Input [6]: [cs_bill_customer_sk#92, cs_ext_discount_amt#93, cs_ext_sales_price#94, cs_ext_wholesale_cost#95, cs_ext_list_price#96, cs_sold_date_sk#97]
+Condition : isnotnull(cs_bill_customer_sk#92)
+
+(70) ReusedExchange [Reuses operator id: 31]
+Output [2]: [d_date_sk#98, d_year#99]
+
+(71) BroadcastHashJoin [codegen id : 28]
+Left keys [1]: [cs_sold_date_sk#97]
+Right keys [1]: [d_date_sk#98]
+Join condition: None
+
+(72) Project [codegen id : 28]
+Output [6]: [cs_bill_customer_sk#92, cs_ext_discount_amt#93, cs_ext_sales_price#94, cs_ext_wholesale_cost#95, cs_ext_list_price#96, d_year#99]
+Input [8]: [cs_bill_customer_sk#92, cs_ext_discount_amt#93, cs_ext_sales_price#94, cs_ext_wholesale_cost#95, cs_ext_list_price#96, cs_sold_date_sk#97, d_date_sk#98, d_year#99]
+
+(73) Exchange
+Input [6]: [cs_bill_customer_sk#92, cs_ext_discount_amt#93, cs_ext_sales_price#94, cs_ext_wholesale_cost#95, cs_ext_list_price#96, d_year#99]
+Arguments: hashpartitioning(cs_bill_customer_sk#92, 5), ENSURE_REQUIREMENTS, [id=#100]
+
+(74) Sort [codegen id : 29]
+Input [6]: [cs_bill_customer_sk#92, cs_ext_discount_amt#93, cs_ext_sales_price#94, cs_ext_wholesale_cost#95, cs_ext_list_price#96, d_year#99]
+Arguments: [cs_bill_customer_sk#92 ASC NULLS FIRST], false, 0
+
+(75) ReusedExchange [Reuses operator id: 15]
+Output [8]: [c_customer_sk#101, c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108]
+
+(76) Sort [codegen id : 31]
+Input [8]: [c_customer_sk#101, c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108]
+Arguments: [c_customer_sk#101 ASC NULLS FIRST], false, 0
+
+(77) SortMergeJoin [codegen id : 32]
+Left keys [1]: [cs_bill_customer_sk#92]
+Right keys [1]: [c_customer_sk#101]
+Join condition: None
+
+(78) Project [codegen id : 32]
+Output [12]: [c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, cs_ext_discount_amt#93, cs_ext_sales_price#94, cs_ext_wholesale_cost#95, cs_ext_list_price#96, d_year#99]
+Input [14]: [cs_bill_customer_sk#92, cs_ext_discount_amt#93, cs_ext_sales_price#94, cs_ext_wholesale_cost#95, cs_ext_list_price#96, d_year#99, c_customer_sk#101, c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108]
+
+(79) HashAggregate [codegen id : 32]
+Input [12]: [c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, cs_ext_discount_amt#93, cs_ext_sales_price#94, cs_ext_wholesale_cost#95, cs_ext_list_price#96, d_year#99]
+Keys [8]: [c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, d_year#99]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cs_ext_list_price#96 as decimal(8,2))) - promote_precision(cast(cs_ext_wholesale_cost#95 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(cs_ext_discount_amt#93 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(cs_ext_sales_price#94 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))]
+Aggregate Attributes [2]: [sum#109, isEmpty#110]
+Results [10]: [c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, d_year#99, sum#111, isEmpty#112]
+
+(80) Exchange
+Input [10]: [c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, d_year#99, sum#111, isEmpty#112]
+Arguments: hashpartitioning(c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, d_year#99, 5), ENSURE_REQUIREMENTS, [id=#113]
+
+(81) HashAggregate [codegen id : 33]
+Input [10]: [c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, d_year#99, sum#111, isEmpty#112]
+Keys [8]: [c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, d_year#99]
+Functions [1]: [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cs_ext_list_price#96 as decimal(8,2))) - promote_precision(cast(cs_ext_wholesale_cost#95 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(cs_ext_discount_amt#93 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(cs_ext_sales_price#94 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cs_ext_list_price#96 as decimal(8,2))) - promote_precision(cast(cs_ext_wholesale_cost#95 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(cs_ext_discount_amt#93 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(cs_ext_sales_price#94 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))#114]
+Results [2]: [c_customer_id#102 AS customer_id#115, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cs_ext_list_price#96 as decimal(8,2))) - promote_precision(cast(cs_ext_wholesale_cost#95 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(cs_ext_discount_amt#93 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(cs_ext_sales_price#94 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))#114 AS year_total#116]
+
+(82) Exchange
+Input [2]: [customer_id#115, year_total#116]
+Arguments: hashpartitioning(customer_id#115, 5), ENSURE_REQUIREMENTS, [id=#117]
+
+(83) Sort [codegen id : 34]
+Input [2]: [customer_id#115, year_total#116]
+Arguments: [customer_id#115 ASC NULLS FIRST], false, 0
+
+(84) SortMergeJoin [codegen id : 35]
+Left keys [1]: [customer_id#27]
+Right keys [1]: [customer_id#115]
+Join condition: (CASE WHEN (year_total#90 > 0.000000) THEN CheckOverflow((promote_precision(year_total#116) / promote_precision(year_total#90)), DecimalType(38,14), true) ELSE null END > CASE WHEN (year_total#28 > 0.000000) THEN CheckOverflow((promote_precision(year_total#62) / promote_precision(year_total#28)), DecimalType(38,14), true) ELSE null END)
+
+(85) Project [codegen id : 35]
+Output [10]: [customer_id#27, customer_id#55, customer_first_name#56, customer_last_name#57, customer_preferred_cust_flag#58, customer_birth_country#59, customer_login#60, customer_email_address#61, year_total#90, year_total#116]
+Input [13]: [customer_id#27, year_total#28, customer_id#55, customer_first_name#56, customer_last_name#57, customer_preferred_cust_flag#58, customer_birth_country#59, customer_login#60, customer_email_address#61, year_total#62, year_total#90, customer_id#115, year_total#116]
+
+(86) Scan parquet default.web_sales
+Output [6]: [ws_bill_customer_sk#118, ws_ext_discount_amt#119, ws_ext_sales_price#120, ws_ext_wholesale_cost#121, ws_ext_list_price#122, ws_sold_date_sk#123]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#123), dynamicpruningexpression(ws_sold_date_sk#123 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int,ws_ext_discount_amt:decimal(7,2),ws_ext_sales_price:decimal(7,2),ws_ext_wholesale_cost:decimal(7,2),ws_ext_list_price:decimal(7,2)>
+
+(87) ColumnarToRow [codegen id : 37]
+Input [6]: [ws_bill_customer_sk#118, ws_ext_discount_amt#119, ws_ext_sales_price#120, ws_ext_wholesale_cost#121, ws_ext_list_price#122, ws_sold_date_sk#123]
+
+(88) Filter [codegen id : 37]
+Input [6]: [ws_bill_customer_sk#118, ws_ext_discount_amt#119, ws_ext_sales_price#120, ws_ext_wholesale_cost#121, ws_ext_list_price#122, ws_sold_date_sk#123]
+Condition : isnotnull(ws_bill_customer_sk#118)
+
+(89) ReusedExchange [Reuses operator id: 7]
+Output [2]: [d_date_sk#124, d_year#125]
+
+(90) BroadcastHashJoin [codegen id : 37]
+Left keys [1]: [ws_sold_date_sk#123]
+Right keys [1]: [d_date_sk#124]
+Join condition: None
+
+(91) Project [codegen id : 37]
+Output [6]: [ws_bill_customer_sk#118, ws_ext_discount_amt#119, ws_ext_sales_price#120, ws_ext_wholesale_cost#121, ws_ext_list_price#122, d_year#125]
+Input [8]: [ws_bill_customer_sk#118, ws_ext_discount_amt#119, ws_ext_sales_price#120, ws_ext_wholesale_cost#121, ws_ext_list_price#122, ws_sold_date_sk#123, d_date_sk#124, d_year#125]
+
+(92) Exchange
+Input [6]: [ws_bill_customer_sk#118, ws_ext_discount_amt#119, ws_ext_sales_price#120, ws_ext_wholesale_cost#121, ws_ext_list_price#122, d_year#125]
+Arguments: hashpartitioning(ws_bill_customer_sk#118, 5), ENSURE_REQUIREMENTS, [id=#126]
+
+(93) Sort [codegen id : 38]
+Input [6]: [ws_bill_customer_sk#118, ws_ext_discount_amt#119, ws_ext_sales_price#120, ws_ext_wholesale_cost#121, ws_ext_list_price#122, d_year#125]
+Arguments: [ws_bill_customer_sk#118 ASC NULLS FIRST], false, 0
+
+(94) ReusedExchange [Reuses operator id: 15]
+Output [8]: [c_customer_sk#127, c_customer_id#128, c_first_name#129, c_last_name#130, c_preferred_cust_flag#131, c_birth_country#132, c_login#133, c_email_address#134]
+
+(95) Sort [codegen id : 40]
+Input [8]: [c_customer_sk#127, c_customer_id#128, c_first_name#129, c_last_name#130, c_preferred_cust_flag#131, c_birth_country#132, c_login#133, c_email_address#134]
+Arguments: [c_customer_sk#127 ASC NULLS FIRST], false, 0
+
+(96) SortMergeJoin [codegen id : 41]
+Left keys [1]: [ws_bill_customer_sk#118]
+Right keys [1]: [c_customer_sk#127]
+Join condition: None
+
+(97) Project [codegen id : 41]
+Output [12]: [c_customer_id#128, c_first_name#129, c_last_name#130, c_preferred_cust_flag#131, c_birth_country#132, c_login#133, c_email_address#134, ws_ext_discount_amt#119, ws_ext_sales_price#120, ws_ext_wholesale_cost#121, ws_ext_list_price#122, d_year#125]
+Input [14]: [ws_bill_customer_sk#118, ws_ext_discount_amt#119, ws_ext_sales_price#120, ws_ext_wholesale_cost#121, ws_ext_list_price#122, d_year#125, c_customer_sk#127, c_customer_id#128, c_first_name#129, c_last_name#130, c_preferred_cust_flag#131, c_birth_country#132, c_login#133, c_email_address#134]
+
+(98) HashAggregate [codegen id : 41]
+Input [12]: [c_customer_id#128, c_first_name#129, c_last_name#130, c_preferred_cust_flag#131, c_birth_country#132, c_login#133, c_email_address#134, ws_ext_discount_amt#119, ws_ext_sales_price#120, ws_ext_wholesale_cost#121, ws_ext_list_price#122, d_year#125]
+Keys [8]: [c_customer_id#128, c_first_name#129, c_last_name#130, c_preferred_cust_flag#131, c_birth_country#132, c_login#133, c_email_address#134, d_year#125]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ws_ext_list_price#122 as decimal(8,2))) - promote_precision(cast(ws_ext_wholesale_cost#121 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ws_ext_discount_amt#119 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ws_ext_sales_price#120 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))]
+Aggregate Attributes [2]: [sum#135, isEmpty#136]
+Results [10]: [c_customer_id#128, c_first_name#129, c_last_name#130, c_preferred_cust_flag#131, c_birth_country#132, c_login#133, c_email_address#134, d_year#125, sum#137, isEmpty#138]
+
+(99) Exchange
+Input [10]: [c_customer_id#128, c_first_name#129, c_last_name#130, c_preferred_cust_flag#131, c_birth_country#132, c_login#133, c_email_address#134, d_year#125, sum#137, isEmpty#138]
+Arguments: hashpartitioning(c_customer_id#128, c_first_name#129, c_last_name#130, c_preferred_cust_flag#131, c_birth_country#132, c_login#133, c_email_address#134, d_year#125, 5), ENSURE_REQUIREMENTS, [id=#139]
+
+(100) HashAggregate [codegen id : 42]
+Input [10]: [c_customer_id#128, c_first_name#129, c_last_name#130, c_preferred_cust_flag#131, c_birth_country#132, c_login#133, c_email_address#134, d_year#125, sum#137, isEmpty#138]
+Keys [8]: [c_customer_id#128, c_first_name#129, c_last_name#130, c_preferred_cust_flag#131, c_birth_country#132, c_login#133, c_email_address#134, d_year#125]
+Functions [1]: [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ws_ext_list_price#122 as decimal(8,2))) - promote_precision(cast(ws_ext_wholesale_cost#121 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ws_ext_discount_amt#119 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ws_ext_sales_price#120 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ws_ext_list_price#122 as decimal(8,2))) - promote_precision(cast(ws_ext_wholesale_cost#121 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ws_ext_discount_amt#119 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ws_ext_sales_price#120 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))#140]
+Results [2]: [c_customer_id#128 AS customer_id#141, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ws_ext_list_price#122 as decimal(8,2))) - promote_precision(cast(ws_ext_wholesale_cost#121 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ws_ext_discount_amt#119 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ws_ext_sales_price#120 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))#140 AS year_total#142]
+
+(101) Filter [codegen id : 42]
+Input [2]: [customer_id#141, year_total#142]
+Condition : (isnotnull(year_total#142) AND (year_total#142 > 0.000000))
+
+(102) Project [codegen id : 42]
+Output [2]: [customer_id#141 AS customer_id#143, year_total#142 AS year_total#144]
+Input [2]: [customer_id#141, year_total#142]
+
+(103) Exchange
+Input [2]: [customer_id#143, year_total#144]
+Arguments: hashpartitioning(customer_id#143, 5), ENSURE_REQUIREMENTS, [id=#145]
+
+(104) Sort [codegen id : 43]
+Input [2]: [customer_id#143, year_total#144]
+Arguments: [customer_id#143 ASC NULLS FIRST], false, 0
+
+(105) SortMergeJoin [codegen id : 44]
+Left keys [1]: [customer_id#27]
+Right keys [1]: [customer_id#143]
+Join condition: None
+
+(106) Project [codegen id : 44]
+Output [11]: [customer_id#27, customer_id#55, customer_first_name#56, customer_last_name#57, customer_preferred_cust_flag#58, customer_birth_country#59, customer_login#60, customer_email_address#61, year_total#90, year_total#116, year_total#144]
+Input [12]: [customer_id#27, customer_id#55, customer_first_name#56, customer_last_name#57, customer_preferred_cust_flag#58, customer_birth_country#59, customer_login#60, customer_email_address#61, year_total#90, year_total#116, customer_id#143, year_total#144]
+
+(107) Scan parquet default.web_sales
+Output [6]: [ws_bill_customer_sk#146, ws_ext_discount_amt#147, ws_ext_sales_price#148, ws_ext_wholesale_cost#149, ws_ext_list_price#150, ws_sold_date_sk#151]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#151), dynamicpruningexpression(ws_sold_date_sk#151 IN dynamicpruning#36)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int,ws_ext_discount_amt:decimal(7,2),ws_ext_sales_price:decimal(7,2),ws_ext_wholesale_cost:decimal(7,2),ws_ext_list_price:decimal(7,2)>
+
+(108) ColumnarToRow [codegen id : 46]
+Input [6]: [ws_bill_customer_sk#146, ws_ext_discount_amt#147, ws_ext_sales_price#148, ws_ext_wholesale_cost#149, ws_ext_list_price#150, ws_sold_date_sk#151]
+
+(109) Filter [codegen id : 46]
+Input [6]: [ws_bill_customer_sk#146, ws_ext_discount_amt#147, ws_ext_sales_price#148, ws_ext_wholesale_cost#149, ws_ext_list_price#150, ws_sold_date_sk#151]
+Condition : isnotnull(ws_bill_customer_sk#146)
+
+(110) ReusedExchange [Reuses operator id: 31]
+Output [2]: [d_date_sk#152, d_year#153]
+
+(111) BroadcastHashJoin [codegen id : 46]
+Left keys [1]: [ws_sold_date_sk#151]
+Right keys [1]: [d_date_sk#152]
+Join condition: None
+
+(112) Project [codegen id : 46]
+Output [6]: [ws_bill_customer_sk#146, ws_ext_discount_amt#147, ws_ext_sales_price#148, ws_ext_wholesale_cost#149, ws_ext_list_price#150, d_year#153]
+Input [8]: [ws_bill_customer_sk#146, ws_ext_discount_amt#147, ws_ext_sales_price#148, ws_ext_wholesale_cost#149, ws_ext_list_price#150, ws_sold_date_sk#151, d_date_sk#152, d_year#153]
+
+(113) Exchange
+Input [6]: [ws_bill_customer_sk#146, ws_ext_discount_amt#147, ws_ext_sales_price#148, ws_ext_wholesale_cost#149, ws_ext_list_price#150, d_year#153]
+Arguments: hashpartitioning(ws_bill_customer_sk#146, 5), ENSURE_REQUIREMENTS, [id=#154]
+
+(114) Sort [codegen id : 47]
+Input [6]: [ws_bill_customer_sk#146, ws_ext_discount_amt#147, ws_ext_sales_price#148, ws_ext_wholesale_cost#149, ws_ext_list_price#150, d_year#153]
+Arguments: [ws_bill_customer_sk#146 ASC NULLS FIRST], false, 0
+
+(115) ReusedExchange [Reuses operator id: 15]
+Output [8]: [c_customer_sk#155, c_customer_id#156, c_first_name#157, c_last_name#158, c_preferred_cust_flag#159, c_birth_country#160, c_login#161, c_email_address#162]
+
+(116) Sort [codegen id : 49]
+Input [8]: [c_customer_sk#155, c_customer_id#156, c_first_name#157, c_last_name#158, c_preferred_cust_flag#159, c_birth_country#160, c_login#161, c_email_address#162]
+Arguments: [c_customer_sk#155 ASC NULLS FIRST], false, 0
+
+(117) SortMergeJoin [codegen id : 50]
+Left keys [1]: [ws_bill_customer_sk#146]
+Right keys [1]: [c_customer_sk#155]
+Join condition: None
+
+(118) Project [codegen id : 50]
+Output [12]: [c_customer_id#156, c_first_name#157, c_last_name#158, c_preferred_cust_flag#159, c_birth_country#160, c_login#161, c_email_address#162, ws_ext_discount_amt#147, ws_ext_sales_price#148, ws_ext_wholesale_cost#149, ws_ext_list_price#150, d_year#153]
+Input [14]: [ws_bill_customer_sk#146, ws_ext_discount_amt#147, ws_ext_sales_price#148, ws_ext_wholesale_cost#149, ws_ext_list_price#150, d_year#153, c_customer_sk#155, c_customer_id#156, c_first_name#157, c_last_name#158, c_preferred_cust_flag#159, c_birth_country#160, c_login#161, c_email_address#162]
+
+(119) HashAggregate [codegen id : 50]
+Input [12]: [c_customer_id#156, c_first_name#157, c_last_name#158, c_preferred_cust_flag#159, c_birth_country#160, c_login#161, c_email_address#162, ws_ext_discount_amt#147, ws_ext_sales_price#148, ws_ext_wholesale_cost#149, ws_ext_list_price#150, d_year#153]
+Keys [8]: [c_customer_id#156, c_first_name#157, c_last_name#158, c_preferred_cust_flag#159, c_birth_country#160, c_login#161, c_email_address#162, d_year#153]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ws_ext_list_price#150 as decimal(8,2))) - promote_precision(cast(ws_ext_wholesale_cost#149 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ws_ext_discount_amt#147 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ws_ext_sales_price#148 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))]
+Aggregate Attributes [2]: [sum#163, isEmpty#164]
+Results [10]: [c_customer_id#156, c_first_name#157, c_last_name#158, c_preferred_cust_flag#159, c_birth_country#160, c_login#161, c_email_address#162, d_year#153, sum#165, isEmpty#166]
+
+(120) Exchange
+Input [10]: [c_customer_id#156, c_first_name#157, c_last_name#158, c_preferred_cust_flag#159, c_birth_country#160, c_login#161, c_email_address#162, d_year#153, sum#165, isEmpty#166]
+Arguments: hashpartitioning(c_customer_id#156, c_first_name#157, c_last_name#158, c_preferred_cust_flag#159, c_birth_country#160, c_login#161, c_email_address#162, d_year#153, 5), ENSURE_REQUIREMENTS, [id=#167]
+
+(121) HashAggregate [codegen id : 51]
+Input [10]: [c_customer_id#156, c_first_name#157, c_last_name#158, c_preferred_cust_flag#159, c_birth_country#160, c_login#161, c_email_address#162, d_year#153, sum#165, isEmpty#166]
+Keys [8]: [c_customer_id#156, c_first_name#157, c_last_name#158, c_preferred_cust_flag#159, c_birth_country#160, c_login#161, c_email_address#162, d_year#153]
+Functions [1]: [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ws_ext_list_price#150 as decimal(8,2))) - promote_precision(cast(ws_ext_wholesale_cost#149 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ws_ext_discount_amt#147 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ws_ext_sales_price#148 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ws_ext_list_price#150 as decimal(8,2))) - promote_precision(cast(ws_ext_wholesale_cost#149 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ws_ext_discount_amt#147 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ws_ext_sales_price#148 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))#168]
+Results [2]: [c_customer_id#156 AS customer_id#169, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ws_ext_list_price#150 as decimal(8,2))) - promote_precision(cast(ws_ext_wholesale_cost#149 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ws_ext_discount_amt#147 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ws_ext_sales_price#148 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))#168 AS year_total#170]
+
+(122) Exchange
+Input [2]: [customer_id#169, year_total#170]
+Arguments: hashpartitioning(customer_id#169, 5), ENSURE_REQUIREMENTS, [id=#171]
+
+(123) Sort [codegen id : 52]
+Input [2]: [customer_id#169, year_total#170]
+Arguments: [customer_id#169 ASC NULLS FIRST], false, 0
+
+(124) SortMergeJoin [codegen id : 53]
+Left keys [1]: [customer_id#27]
+Right keys [1]: [customer_id#169]
+Join condition: (CASE WHEN (year_total#90 > 0.000000) THEN CheckOverflow((promote_precision(year_total#116) / promote_precision(year_total#90)), DecimalType(38,14), true) ELSE null END > CASE WHEN (year_total#144 > 0.000000) THEN CheckOverflow((promote_precision(year_total#170) / promote_precision(year_total#144)), DecimalType(38,14), true) ELSE null END)
+
+(125) Project [codegen id : 53]
+Output [7]: [customer_id#55, customer_first_name#56, customer_last_name#57, customer_preferred_cust_flag#58, customer_birth_country#59, customer_login#60, customer_email_address#61]
+Input [13]: [customer_id#27, customer_id#55, customer_first_name#56, customer_last_name#57, customer_preferred_cust_flag#58, customer_birth_country#59, customer_login#60, customer_email_address#61, year_total#90, year_total#116, year_total#144, customer_id#169, year_total#170]
+
+(126) TakeOrderedAndProject
+Input [7]: [customer_id#55, customer_first_name#56, customer_last_name#57, customer_preferred_cust_flag#58, customer_birth_country#59, customer_login#60, customer_email_address#61]
+Arguments: 100, [customer_id#55 ASC NULLS FIRST, customer_first_name#56 ASC NULLS FIRST, customer_last_name#57 ASC NULLS FIRST, customer_preferred_cust_flag#58 ASC NULLS FIRST, customer_birth_country#59 ASC NULLS FIRST, customer_login#60 ASC NULLS FIRST, customer_email_address#61 ASC NULLS FIRST], [customer_id#55, customer_first_name#56, customer_last_name#57, customer_preferred_cust_flag#58, customer_birth_country#59, customer_login#60, customer_email_address#61]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7
+ReusedExchange (127)
+
+
+(127) ReusedExchange [Reuses operator id: 7]
+Output [2]: [d_date_sk#8, d_year#9]
+
+Subquery:2 Hosting operator id = 25 Hosting Expression = ss_sold_date_sk#35 IN dynamicpruning#36
+ReusedExchange (128)
+
+
+(128) ReusedExchange [Reuses operator id: 31]
+Output [2]: [d_date_sk#37, d_year#38]
+
+Subquery:3 Hosting operator id = 46 Hosting Expression = cs_sold_date_sk#69 IN dynamicpruning#7
+
+Subquery:4 Hosting operator id = 67 Hosting Expression = cs_sold_date_sk#97 IN dynamicpruning#36
+
+Subquery:5 Hosting operator id = 86 Hosting Expression = ws_sold_date_sk#123 IN dynamicpruning#7
+
+Subquery:6 Hosting operator id = 107 Hosting Expression = ws_sold_date_sk#151 IN dynamicpruning#36
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.sf100/simplified.txt
new file mode 100644
index 0000000000000..29d4199f5aff3
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.sf100/simplified.txt
@@ -0,0 +1,239 @@
+TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address]
+  WholeStageCodegen (53)
+    Project [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address]
+      SortMergeJoin [customer_id,customer_id,year_total,year_total,year_total,year_total]
+        InputAdapter
+          WholeStageCodegen (44)
+            Project [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,year_total]
+              SortMergeJoin [customer_id,customer_id]
+                InputAdapter
+                  WholeStageCodegen (35)
+                    Project [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total]
+                      SortMergeJoin [customer_id,customer_id,year_total,year_total,year_total,year_total]
+                        InputAdapter
+                          WholeStageCodegen (26)
+                            Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total]
+                              SortMergeJoin [customer_id,customer_id]
+                                InputAdapter
+                                  WholeStageCodegen (17)
+                                    SortMergeJoin [customer_id,customer_id]
+                                      InputAdapter
+                                        WholeStageCodegen (8)
+                                          Sort [customer_id]
+                                            InputAdapter
+                                              Exchange [customer_id] #1
+                                                WholeStageCodegen (7)
+                                                  Filter [year_total]
+                                                    HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_ext_list_price as decimal(8,2))) - promote_precision(cast(ss_ext_wholesale_cost as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ss_ext_discount_amt as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ss_ext_sales_price as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true)),customer_id,year_total,sum,isEmpty]
+                                                      InputAdapter
+                                                        Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #2
+                                                          WholeStageCodegen (6)
+                                                            HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty]
+                                                              Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year]
+                                                                SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                                                  InputAdapter
+                                                                    WholeStageCodegen (3)
+                                                                      Sort [ss_customer_sk]
+                                                                        InputAdapter
+                                                                          Exchange [ss_customer_sk] #3
+                                                                            WholeStageCodegen (2)
+                                                                              Project [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year]
+                                                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                  Filter [ss_customer_sk]
+                                                                                    ColumnarToRow
+                                                                                      InputAdapter
+                                                                                        Scan parquet default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk]
+                                                                                          SubqueryBroadcast [d_date_sk] #1
+                                                                                            ReusedExchange [d_date_sk,d_year] #4
+                                                                                  InputAdapter
+                                                                                    BroadcastExchange #4
+                                                                                      WholeStageCodegen (1)
+                                                                                        Filter [d_year,d_date_sk]
+                                                                                          ColumnarToRow
+                                                                                            InputAdapter
+                                                                                              Scan parquet default.date_dim [d_date_sk,d_year]
+                                                                  InputAdapter
+                                                                    WholeStageCodegen (5)
+                                                                      Sort [c_customer_sk]
+                                                                        InputAdapter
+                                                                          Exchange [c_customer_sk] #5
+                                                                            WholeStageCodegen (4)
+                                                                              Filter [c_customer_sk,c_customer_id]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address]
+                                      InputAdapter
+                                        WholeStageCodegen (16)
+                                          Sort [customer_id]
+                                            InputAdapter
+                                              Exchange [customer_id] #6
+                                                WholeStageCodegen (15)
+                                                  HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_ext_list_price as decimal(8,2))) - promote_precision(cast(ss_ext_wholesale_cost as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ss_ext_discount_amt as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ss_ext_sales_price as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true)),customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,sum,isEmpty]
+                                                    InputAdapter
+                                                      Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #7
+                                                        WholeStageCodegen (14)
+                                                          HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty]
+                                                            Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year]
+                                                              SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                                                InputAdapter
+                                                                  WholeStageCodegen (11)
+                                                                    Sort [ss_customer_sk]
+                                                                      InputAdapter
+                                                                        Exchange [ss_customer_sk] #8
+                                                                          WholeStageCodegen (10)
+                                                                            Project [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year]
+                                                                              BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                Filter [ss_customer_sk]
+                                                                                  ColumnarToRow
+                                                                                    InputAdapter
+                                                                                      Scan parquet default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk]
+                                                                                        SubqueryBroadcast [d_date_sk] #2
+                                                                                          ReusedExchange [d_date_sk,d_year] #9
+                                                                                InputAdapter
+                                                                                  BroadcastExchange #9
+                                                                                    WholeStageCodegen (9)
+                                                                                      Filter [d_year,d_date_sk]
+                                                                                        ColumnarToRow
+                                                                                          InputAdapter
+                                                                                            Scan parquet default.date_dim [d_date_sk,d_year]
+                                                                InputAdapter
+                                                                  WholeStageCodegen (13)
+                                                                    Sort [c_customer_sk]
+                                                                      InputAdapter
+                                                                        ReusedExchange [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5
+                                InputAdapter
+                                  WholeStageCodegen (25)
+                                    Sort [customer_id]
+                                      InputAdapter
+                                        Exchange [customer_id] #10
+                                          WholeStageCodegen (24)
+                                            Project [customer_id,year_total]
+                                              Filter [year_total]
+                                                HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cs_ext_list_price as decimal(8,2))) - promote_precision(cast(cs_ext_wholesale_cost as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(cs_ext_discount_amt as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(cs_ext_sales_price as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true)),customer_id,year_total,sum,isEmpty]
+                                                  InputAdapter
+                                                    Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11
+                                                      WholeStageCodegen (23)
+                                                        HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [sum,isEmpty,sum,isEmpty]
+                                                          Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year]
+                                                            SortMergeJoin [cs_bill_customer_sk,c_customer_sk]
+                                                              InputAdapter
+                                                                WholeStageCodegen (20)
+                                                                  Sort [cs_bill_customer_sk]
+                                                                    InputAdapter
+                                                                      Exchange [cs_bill_customer_sk] #12
+                                                                        WholeStageCodegen (19)
+                                                                          Project [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year]
+                                                                            BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                              Filter [cs_bill_customer_sk]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk]
+                                                                                      ReusedSubquery [d_date_sk] #1
+                                                                              InputAdapter
+                                                                                ReusedExchange [d_date_sk,d_year] #4
+                                                              InputAdapter
+                                                                WholeStageCodegen (22)
+                                                                  Sort [c_customer_sk]
+                                                                    InputAdapter
+                                                                      ReusedExchange [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5
+                        InputAdapter
+                          WholeStageCodegen (34)
+                            Sort [customer_id]
+                              InputAdapter
+                                Exchange [customer_id] #13
+                                  WholeStageCodegen (33)
+                                    HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cs_ext_list_price as decimal(8,2))) - promote_precision(cast(cs_ext_wholesale_cost as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(cs_ext_discount_amt as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(cs_ext_sales_price as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true)),customer_id,year_total,sum,isEmpty]
+                                      InputAdapter
+                                        Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14
+                                          WholeStageCodegen (32)
+                                            HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [sum,isEmpty,sum,isEmpty]
+                                              Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year]
+                                                SortMergeJoin [cs_bill_customer_sk,c_customer_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (29)
+                                                      Sort [cs_bill_customer_sk]
+                                                        InputAdapter
+                                                          Exchange [cs_bill_customer_sk] #15
+                                                            WholeStageCodegen (28)
+                                                              Project [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year]
+                                                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                  Filter [cs_bill_customer_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk]
+                                                                          ReusedSubquery [d_date_sk] #2
+                                                                  InputAdapter
+                                                                    ReusedExchange [d_date_sk,d_year] #9
+                                                  InputAdapter
+                                                    WholeStageCodegen (31)
+                                                      Sort [c_customer_sk]
+                                                        InputAdapter
+                                                          ReusedExchange [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5
+                InputAdapter
+                  WholeStageCodegen (43)
+                    Sort [customer_id]
+                      InputAdapter
+                        Exchange [customer_id] #16
+                          WholeStageCodegen (42)
+                            Project [customer_id,year_total]
+                              Filter [year_total]
+                                HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ws_ext_list_price as decimal(8,2))) - promote_precision(cast(ws_ext_wholesale_cost as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ws_ext_discount_amt as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ws_ext_sales_price as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true)),customer_id,year_total,sum,isEmpty]
+                                  InputAdapter
+                                    Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #17
+                                      WholeStageCodegen (41)
+                                        HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [sum,isEmpty,sum,isEmpty]
+                                          Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year]
+                                            SortMergeJoin [ws_bill_customer_sk,c_customer_sk]
+                                              InputAdapter
+                                                WholeStageCodegen (38)
+                                                  Sort [ws_bill_customer_sk]
+                                                    InputAdapter
+                                                      Exchange [ws_bill_customer_sk] #18
+                                                        WholeStageCodegen (37)
+                                                          Project [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year]
+                                                            BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                              Filter [ws_bill_customer_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk]
+                                                                      ReusedSubquery [d_date_sk] #1
+                                                              InputAdapter
+                                                                ReusedExchange [d_date_sk,d_year] #4
+                                              InputAdapter
+                                                WholeStageCodegen (40)
+                                                  Sort [c_customer_sk]
+                                                    InputAdapter
+                                                      ReusedExchange [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5
+        InputAdapter
+          WholeStageCodegen (52)
+            Sort [customer_id]
+              InputAdapter
+                Exchange [customer_id] #19
+                  WholeStageCodegen (51)
+                    HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ws_ext_list_price as decimal(8,2))) - promote_precision(cast(ws_ext_wholesale_cost as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ws_ext_discount_amt as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ws_ext_sales_price as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true)),customer_id,year_total,sum,isEmpty]
+                      InputAdapter
+                        Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #20
+                          WholeStageCodegen (50)
+                            HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [sum,isEmpty,sum,isEmpty]
+                              Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year]
+                                SortMergeJoin [ws_bill_customer_sk,c_customer_sk]
+                                  InputAdapter
+                                    WholeStageCodegen (47)
+                                      Sort [ws_bill_customer_sk]
+                                        InputAdapter
+                                          Exchange [ws_bill_customer_sk] #21
+                                            WholeStageCodegen (46)
+                                              Project [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year]
+                                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                  Filter [ws_bill_customer_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk]
+                                                          ReusedSubquery [d_date_sk] #2
+                                                  InputAdapter
+                                                    ReusedExchange [d_date_sk,d_year] #9
+                                  InputAdapter
+                                    WholeStageCodegen (49)
+                                      Sort [c_customer_sk]
+                                        InputAdapter
+                                          ReusedExchange [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/explain.txt
new file mode 100644
index 0000000000000..1cf27630d2994
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/explain.txt
@@ -0,0 +1,691 @@
+== Physical Plan ==
+TakeOrderedAndProject (116)
++- * Project (115)
+   +- * BroadcastHashJoin Inner BuildRight (114)
+      :- * Project (97)
+      :  +- * BroadcastHashJoin Inner BuildRight (96)
+      :     :- * Project (77)
+      :     :  +- * BroadcastHashJoin Inner BuildRight (76)
+      :     :     :- * Project (59)
+      :     :     :  +- * BroadcastHashJoin Inner BuildRight (58)
+      :     :     :     :- * BroadcastHashJoin Inner BuildRight (39)
+      :     :     :     :  :- * Filter (19)
+      :     :     :     :  :  +- * HashAggregate (18)
+      :     :     :     :  :     +- Exchange (17)
+      :     :     :     :  :        +- * HashAggregate (16)
+      :     :     :     :  :           +- * Project (15)
+      :     :     :     :  :              +- * BroadcastHashJoin Inner BuildRight (14)
+      :     :     :     :  :                 :- * Project (9)
+      :     :     :     :  :                 :  +- * BroadcastHashJoin Inner BuildRight (8)
+      :     :     :     :  :                 :     :- * Filter (3)
+      :     :     :     :  :                 :     :  +- * ColumnarToRow (2)
+      :     :     :     :  :                 :     :     +- Scan parquet default.customer (1)
+      :     :     :     :  :                 :     +- BroadcastExchange (7)
+      :     :     :     :  :                 :        +- * Filter (6)
+      :     :     :     :  :                 :           +- * ColumnarToRow (5)
+      :     :     :     :  :                 :              +- Scan parquet default.store_sales (4)
+      :     :     :     :  :                 +- BroadcastExchange (13)
+      :     :     :     :  :                    +- * Filter (12)
+      :     :     :     :  :                       +- * ColumnarToRow (11)
+      :     :     :     :  :                          +- Scan parquet default.date_dim (10)
+      :     :     :     :  +- BroadcastExchange (38)
+      :     :     :     :     +- * HashAggregate (37)
+      :     :     :     :        +- Exchange (36)
+      :     :     :     :           +- * HashAggregate (35)
+      :     :     :     :              +- * Project (34)
+      :     :     :     :                 +- * BroadcastHashJoin Inner BuildRight (33)
+      :     :     :     :                    :- * Project (28)
+      :     :     :     :                    :  +- * BroadcastHashJoin Inner BuildRight (27)
+      :     :     :     :                    :     :- * Filter (22)
+      :     :     :     :                    :     :  +- * ColumnarToRow (21)
+      :     :     :     :                    :     :     +- Scan parquet default.customer (20)
+      :     :     :     :                    :     +- BroadcastExchange (26)
+      :     :     :     :                    :        +- * Filter (25)
+      :     :     :     :                    :           +- * ColumnarToRow (24)
+      :     :     :     :                    :              +- Scan parquet default.store_sales (23)
+      :     :     :     :                    +- BroadcastExchange (32)
+      :     :     :     :                       +- * Filter (31)
+      :     :     :     :                          +- * ColumnarToRow (30)
+      :     :     :     :                             +- Scan parquet default.date_dim (29)
+      :     :     :     +- BroadcastExchange (57)
+      :     :     :        +- * Project (56)
+      :     :     :           +- * Filter (55)
+      :     :     :              +- * HashAggregate (54)
+      :     :     :                 +- Exchange (53)
+      :     :     :                    +- * HashAggregate (52)
+      :     :     :                       +- * Project (51)
+      :     :     :                          +- * BroadcastHashJoin Inner BuildRight (50)
+      :     :     :                             :- * Project (48)
+      :     :     :                             :  +- * BroadcastHashJoin Inner BuildRight (47)
+      :     :     :                             :     :- * Filter (42)
+      :     :     :                             :     :  +- * ColumnarToRow (41)
+      :     :     :                             :     :     +- Scan parquet default.customer (40)
+      :     :     :                             :     +- BroadcastExchange (46)
+      :     :     :                             :        +- * Filter (45)
+      :     :     :                             :           +- * ColumnarToRow (44)
+      :     :     :                             :              +- Scan parquet default.catalog_sales (43)
+      :     :     :                             +- ReusedExchange (49)
+      :     :     +- BroadcastExchange (75)
+      :     :        +- * HashAggregate (74)
+      :     :           +- Exchange (73)
+      :     :              +- * HashAggregate (72)
+      :     :                 +- * Project (71)
+      :     :                    +- * BroadcastHashJoin Inner BuildRight (70)
+      :     :                       :- * Project (68)
+      :     :                       :  +- * BroadcastHashJoin Inner BuildRight (67)
+      :     :                       :     :- * Filter (62)
+      :     :                       :     :  +- * ColumnarToRow (61)
+      :     :                       :     :     +- Scan parquet default.customer (60)
+      :     :                       :     +- BroadcastExchange (66)
+      :     :                       :        +- * Filter (65)
+      :     :                       :           +- * ColumnarToRow (64)
+      :     :                       :              +- Scan parquet default.catalog_sales (63)
+      :     :                       +- ReusedExchange (69)
+      :     +- BroadcastExchange (95)
+      :        +- * Project (94)
+      :           +- * Filter (93)
+      :              +- * HashAggregate (92)
+      :                 +- Exchange (91)
+      :                    +- * HashAggregate (90)
+      :                       +- * Project (89)
+      :                          +- * BroadcastHashJoin Inner BuildRight (88)
+      :                             :- * Project (86)
+      :                             :  +- * BroadcastHashJoin Inner BuildRight (85)
+      :                             :     :- * Filter (80)
+      :                             :     :  +- * ColumnarToRow (79)
+      :                             :     :     +- Scan parquet default.customer (78)
+      :                             :     +- BroadcastExchange (84)
+      :                             :        +- * Filter (83)
+      :                             :           +- * ColumnarToRow (82)
+      :                             :              +- Scan parquet default.web_sales (81)
+      :                             +- ReusedExchange (87)
+      +- BroadcastExchange (113)
+         +- * HashAggregate (112)
+            +- Exchange (111)
+               +- * HashAggregate (110)
+                  +- * Project (109)
+                     +- * BroadcastHashJoin Inner BuildRight (108)
+                        :- * Project (106)
+                        :  +- * BroadcastHashJoin Inner BuildRight (105)
+                        :     :- * Filter (100)
+                        :     :  +- * ColumnarToRow (99)
+                        :     :     +- Scan parquet default.customer (98)
+                        :     +- BroadcastExchange (104)
+                        :        +- * Filter (103)
+                        :           +- * ColumnarToRow (102)
+                        :              +- Scan parquet default.web_sales (101)
+                        +- ReusedExchange (107)
+
+
+(1) Scan parquet default.customer
+Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string,c_birth_country:string,c_login:string,c_email_address:string>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8]
+
+(3) Filter [codegen id : 3]
+Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8]
+Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2))
+
+(4) Scan parquet default.store_sales
+Output [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#14), dynamicpruningexpression(ss_sold_date_sk#14 IN dynamicpruning#15)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_ext_discount_amt:decimal(7,2),ss_ext_sales_price:decimal(7,2),ss_ext_wholesale_cost:decimal(7,2),ss_ext_list_price:decimal(7,2)>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14]
+
+(6) Filter [codegen id : 1]
+Input [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14]
+Condition : isnotnull(ss_customer_sk#9)
+
+(7) BroadcastExchange
+Input [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#16]
+
+(8) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [ss_customer_sk#9]
+Join condition: None
+
+(9) Project [codegen id : 3]
+Output [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14]
+Input [14]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14]
+
+(10) Scan parquet default.date_dim
+Output [2]: [d_date_sk#17, d_year#18]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#17, d_year#18]
+
+(12) Filter [codegen id : 2]
+Input [2]: [d_date_sk#17, d_year#18]
+Condition : ((isnotnull(d_year#18) AND (d_year#18 = 2001)) AND isnotnull(d_date_sk#17))
+
+(13) BroadcastExchange
+Input [2]: [d_date_sk#17, d_year#18]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#19]
+
+(14) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#14]
+Right keys [1]: [d_date_sk#17]
+Join condition: None
+
+(15) Project [codegen id : 3]
+Output [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, d_year#18]
+Input [14]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14, d_date_sk#17, d_year#18]
+
+(16) HashAggregate [codegen id : 3]
+Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, d_year#18]
+Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#18]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_ext_list_price#13 as decimal(8,2))) - promote_precision(cast(ss_ext_wholesale_cost#12 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ss_ext_discount_amt#10 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ss_ext_sales_price#11 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))]
+Aggregate Attributes [2]: [sum#20, isEmpty#21]
+Results [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#18, sum#22, isEmpty#23]
+
+(17) Exchange
+Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#18, sum#22, isEmpty#23]
+Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#18, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(18) HashAggregate [codegen id : 24]
+Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#18, sum#22, isEmpty#23]
+Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#18]
+Functions [1]: [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_ext_list_price#13 as decimal(8,2))) - promote_precision(cast(ss_ext_wholesale_cost#12 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ss_ext_discount_amt#10 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ss_ext_sales_price#11 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_ext_list_price#13 as decimal(8,2))) - promote_precision(cast(ss_ext_wholesale_cost#12 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ss_ext_discount_amt#10 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ss_ext_sales_price#11 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))#25]
+Results [2]: [c_customer_id#2 AS customer_id#26, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_ext_list_price#13 as decimal(8,2))) - promote_precision(cast(ss_ext_wholesale_cost#12 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ss_ext_discount_amt#10 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ss_ext_sales_price#11 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))#25 AS year_total#27]
+
+(19) Filter [codegen id : 24]
+Input [2]: [customer_id#26, year_total#27]
+Condition : (isnotnull(year_total#27) AND (year_total#27 > 0.000000))
+
+(20) Scan parquet default.customer
+Output [8]: [c_customer_sk#28, c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string,c_birth_country:string,c_login:string,c_email_address:string>
+
+(21) ColumnarToRow [codegen id : 6]
+Input [8]: [c_customer_sk#28, c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35]
+
+(22) Filter [codegen id : 6]
+Input [8]: [c_customer_sk#28, c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35]
+Condition : (isnotnull(c_customer_sk#28) AND isnotnull(c_customer_id#29))
+
+(23) Scan parquet default.store_sales
+Output [6]: [ss_customer_sk#36, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#41), dynamicpruningexpression(ss_sold_date_sk#41 IN dynamicpruning#42)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_ext_discount_amt:decimal(7,2),ss_ext_sales_price:decimal(7,2),ss_ext_wholesale_cost:decimal(7,2),ss_ext_list_price:decimal(7,2)>
+
+(24) ColumnarToRow [codegen id : 4]
+Input [6]: [ss_customer_sk#36, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41]
+
+(25) Filter [codegen id : 4]
+Input [6]: [ss_customer_sk#36, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41]
+Condition : isnotnull(ss_customer_sk#36)
+
+(26) BroadcastExchange
+Input [6]: [ss_customer_sk#36, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#43]
+
+(27) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [c_customer_sk#28]
+Right keys [1]: [ss_customer_sk#36]
+Join condition: None
+
+(28) Project [codegen id : 6]
+Output [12]: [c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41]
+Input [14]: [c_customer_sk#28, c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35, ss_customer_sk#36, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41]
+
+(29) Scan parquet default.date_dim
+Output [2]: [d_date_sk#44, d_year#45]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(30) ColumnarToRow [codegen id : 5]
+Input [2]: [d_date_sk#44, d_year#45]
+
+(31) Filter [codegen id : 5]
+Input [2]: [d_date_sk#44, d_year#45]
+Condition : ((isnotnull(d_year#45) AND (d_year#45 = 2002)) AND isnotnull(d_date_sk#44))
+
+(32) BroadcastExchange
+Input [2]: [d_date_sk#44, d_year#45]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#46]
+
+(33) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_sold_date_sk#41]
+Right keys [1]: [d_date_sk#44]
+Join condition: None
+
+(34) Project [codegen id : 6]
+Output [12]: [c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, d_year#45]
+Input [14]: [c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41, d_date_sk#44, d_year#45]
+
+(35) HashAggregate [codegen id : 6]
+Input [12]: [c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, d_year#45]
+Keys [8]: [c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35, d_year#45]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_ext_list_price#40 as decimal(8,2))) - promote_precision(cast(ss_ext_wholesale_cost#39 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ss_ext_discount_amt#37 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ss_ext_sales_price#38 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))]
+Aggregate Attributes [2]: [sum#47, isEmpty#48]
+Results [10]: [c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35, d_year#45, sum#49, isEmpty#50]
+
+(36) Exchange
+Input [10]: [c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35, d_year#45, sum#49, isEmpty#50]
+Arguments: hashpartitioning(c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35, d_year#45, 5), ENSURE_REQUIREMENTS, [id=#51]
+
+(37) HashAggregate [codegen id : 7]
+Input [10]: [c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35, d_year#45, sum#49, isEmpty#50]
+Keys [8]: [c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35, d_year#45]
+Functions [1]: [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_ext_list_price#40 as decimal(8,2))) - promote_precision(cast(ss_ext_wholesale_cost#39 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ss_ext_discount_amt#37 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ss_ext_sales_price#38 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_ext_list_price#40 as decimal(8,2))) - promote_precision(cast(ss_ext_wholesale_cost#39 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ss_ext_discount_amt#37 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ss_ext_sales_price#38 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))#52]
+Results [8]: [c_customer_id#29 AS customer_id#53, c_first_name#30 AS customer_first_name#54, c_last_name#31 AS customer_last_name#55, c_preferred_cust_flag#32 AS customer_preferred_cust_flag#56, c_birth_country#33 AS customer_birth_country#57, c_login#34 AS customer_login#58, c_email_address#35 AS customer_email_address#59, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_ext_list_price#40 as decimal(8,2))) - promote_precision(cast(ss_ext_wholesale_cost#39 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ss_ext_discount_amt#37 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ss_ext_sales_price#38 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))#52 AS year_total#60]
+
+(38) BroadcastExchange
+Input [8]: [customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59, year_total#60]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#61]
+
+(39) BroadcastHashJoin [codegen id : 24]
+Left keys [1]: [customer_id#26]
+Right keys [1]: [customer_id#53]
+Join condition: None
+
+(40) Scan parquet default.customer
+Output [8]: [c_customer_sk#62, c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#67, c_login#68, c_email_address#69]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string,c_birth_country:string,c_login:string,c_email_address:string>
+
+(41) ColumnarToRow [codegen id : 10]
+Input [8]: [c_customer_sk#62, c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#67, c_login#68, c_email_address#69]
+
+(42) Filter [codegen id : 10]
+Input [8]: [c_customer_sk#62, c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#67, c_login#68, c_email_address#69]
+Condition : (isnotnull(c_customer_sk#62) AND isnotnull(c_customer_id#63))
+
+(43) Scan parquet default.catalog_sales
+Output [6]: [cs_bill_customer_sk#70, cs_ext_discount_amt#71, cs_ext_sales_price#72, cs_ext_wholesale_cost#73, cs_ext_list_price#74, cs_sold_date_sk#75]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#75), dynamicpruningexpression(cs_sold_date_sk#75 IN dynamicpruning#15)]
+PushedFilters: [IsNotNull(cs_bill_customer_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_ext_discount_amt:decimal(7,2),cs_ext_sales_price:decimal(7,2),cs_ext_wholesale_cost:decimal(7,2),cs_ext_list_price:decimal(7,2)>
+
+(44) ColumnarToRow [codegen id : 8]
+Input [6]: [cs_bill_customer_sk#70, cs_ext_discount_amt#71, cs_ext_sales_price#72, cs_ext_wholesale_cost#73, cs_ext_list_price#74, cs_sold_date_sk#75]
+
+(45) Filter [codegen id : 8]
+Input [6]: [cs_bill_customer_sk#70, cs_ext_discount_amt#71, cs_ext_sales_price#72, cs_ext_wholesale_cost#73, cs_ext_list_price#74, cs_sold_date_sk#75]
+Condition : isnotnull(cs_bill_customer_sk#70)
+
+(46) BroadcastExchange
+Input [6]: [cs_bill_customer_sk#70, cs_ext_discount_amt#71, cs_ext_sales_price#72, cs_ext_wholesale_cost#73, cs_ext_list_price#74, cs_sold_date_sk#75]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#76]
+
+(47) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [c_customer_sk#62]
+Right keys [1]: [cs_bill_customer_sk#70]
+Join condition: None
+
+(48) Project [codegen id : 10]
+Output [12]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#67, c_login#68, c_email_address#69, cs_ext_discount_amt#71, cs_ext_sales_price#72, cs_ext_wholesale_cost#73, cs_ext_list_price#74, cs_sold_date_sk#75]
+Input [14]: [c_customer_sk#62, c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#67, c_login#68, c_email_address#69, cs_bill_customer_sk#70, cs_ext_discount_amt#71, cs_ext_sales_price#72, cs_ext_wholesale_cost#73, cs_ext_list_price#74, cs_sold_date_sk#75]
+
+(49) ReusedExchange [Reuses operator id: 13]
+Output [2]: [d_date_sk#77, d_year#78]
+
+(50) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_sold_date_sk#75]
+Right keys [1]: [d_date_sk#77]
+Join condition: None
+
+(51) Project [codegen id : 10]
+Output [12]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#67, c_login#68, c_email_address#69, cs_ext_discount_amt#71, cs_ext_sales_price#72, cs_ext_wholesale_cost#73, cs_ext_list_price#74, d_year#78]
+Input [14]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#67, c_login#68, c_email_address#69, cs_ext_discount_amt#71, cs_ext_sales_price#72, cs_ext_wholesale_cost#73, cs_ext_list_price#74, cs_sold_date_sk#75, d_date_sk#77, d_year#78]
+
+(52) HashAggregate [codegen id : 10]
+Input [12]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#67, c_login#68, c_email_address#69, cs_ext_discount_amt#71, cs_ext_sales_price#72, cs_ext_wholesale_cost#73, cs_ext_list_price#74, d_year#78]
+Keys [8]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#67, c_login#68, c_email_address#69, d_year#78]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cs_ext_list_price#74 as decimal(8,2))) - promote_precision(cast(cs_ext_wholesale_cost#73 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(cs_ext_discount_amt#71 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(cs_ext_sales_price#72 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))]
+Aggregate Attributes [2]: [sum#79, isEmpty#80]
+Results [10]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#67, c_login#68, c_email_address#69, d_year#78, sum#81, isEmpty#82]
+
+(53) Exchange
+Input [10]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#67, c_login#68, c_email_address#69, d_year#78, sum#81, isEmpty#82]
+Arguments: hashpartitioning(c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#67, c_login#68, c_email_address#69, d_year#78, 5), ENSURE_REQUIREMENTS, [id=#83]
+
+(54) HashAggregate [codegen id : 11]
+Input [10]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#67, c_login#68, c_email_address#69, d_year#78, sum#81, isEmpty#82]
+Keys [8]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#67, c_login#68, c_email_address#69, d_year#78]
+Functions [1]: [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cs_ext_list_price#74 as decimal(8,2))) - promote_precision(cast(cs_ext_wholesale_cost#73 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(cs_ext_discount_amt#71 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(cs_ext_sales_price#72 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cs_ext_list_price#74 as decimal(8,2))) - promote_precision(cast(cs_ext_wholesale_cost#73 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(cs_ext_discount_amt#71 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(cs_ext_sales_price#72 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))#84]
+Results [2]: [c_customer_id#63 AS customer_id#85, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cs_ext_list_price#74 as decimal(8,2))) - promote_precision(cast(cs_ext_wholesale_cost#73 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(cs_ext_discount_amt#71 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(cs_ext_sales_price#72 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))#84 AS year_total#86]
+
+(55) Filter [codegen id : 11]
+Input [2]: [customer_id#85, year_total#86]
+Condition : (isnotnull(year_total#86) AND (year_total#86 > 0.000000))
+
+(56) Project [codegen id : 11]
+Output [2]: [customer_id#85 AS customer_id#87, year_total#86 AS year_total#88]
+Input [2]: [customer_id#85, year_total#86]
+
+(57) BroadcastExchange
+Input [2]: [customer_id#87, year_total#88]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#89]
+
+(58) BroadcastHashJoin [codegen id : 24]
+Left keys [1]: [customer_id#26]
+Right keys [1]: [customer_id#87]
+Join condition: None
+
+(59) Project [codegen id : 24]
+Output [11]: [customer_id#26, year_total#27, customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59, year_total#60, year_total#88]
+Input [12]: [customer_id#26, year_total#27, customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59, year_total#60, customer_id#87, year_total#88]
+
+(60) Scan parquet default.customer
+Output [8]: [c_customer_sk#90, c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string,c_birth_country:string,c_login:string,c_email_address:string>
+
+(61) ColumnarToRow [codegen id : 14]
+Input [8]: [c_customer_sk#90, c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97]
+
+(62) Filter [codegen id : 14]
+Input [8]: [c_customer_sk#90, c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97]
+Condition : (isnotnull(c_customer_sk#90) AND isnotnull(c_customer_id#91))
+
+(63) Scan parquet default.catalog_sales
+Output [6]: [cs_bill_customer_sk#98, cs_ext_discount_amt#99, cs_ext_sales_price#100, cs_ext_wholesale_cost#101, cs_ext_list_price#102, cs_sold_date_sk#103]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#103), dynamicpruningexpression(cs_sold_date_sk#103 IN dynamicpruning#42)]
+PushedFilters: [IsNotNull(cs_bill_customer_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_ext_discount_amt:decimal(7,2),cs_ext_sales_price:decimal(7,2),cs_ext_wholesale_cost:decimal(7,2),cs_ext_list_price:decimal(7,2)>
+
+(64) ColumnarToRow [codegen id : 12]
+Input [6]: [cs_bill_customer_sk#98, cs_ext_discount_amt#99, cs_ext_sales_price#100, cs_ext_wholesale_cost#101, cs_ext_list_price#102, cs_sold_date_sk#103]
+
+(65) Filter [codegen id : 12]
+Input [6]: [cs_bill_customer_sk#98, cs_ext_discount_amt#99, cs_ext_sales_price#100, cs_ext_wholesale_cost#101, cs_ext_list_price#102, cs_sold_date_sk#103]
+Condition : isnotnull(cs_bill_customer_sk#98)
+
+(66) BroadcastExchange
+Input [6]: [cs_bill_customer_sk#98, cs_ext_discount_amt#99, cs_ext_sales_price#100, cs_ext_wholesale_cost#101, cs_ext_list_price#102, cs_sold_date_sk#103]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#104]
+
+(67) BroadcastHashJoin [codegen id : 14]
+Left keys [1]: [c_customer_sk#90]
+Right keys [1]: [cs_bill_customer_sk#98]
+Join condition: None
+
+(68) Project [codegen id : 14]
+Output [12]: [c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97, cs_ext_discount_amt#99, cs_ext_sales_price#100, cs_ext_wholesale_cost#101, cs_ext_list_price#102, cs_sold_date_sk#103]
+Input [14]: [c_customer_sk#90, c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97, cs_bill_customer_sk#98, cs_ext_discount_amt#99, cs_ext_sales_price#100, cs_ext_wholesale_cost#101, cs_ext_list_price#102, cs_sold_date_sk#103]
+
+(69) ReusedExchange [Reuses operator id: 32]
+Output [2]: [d_date_sk#105, d_year#106]
+
+(70) BroadcastHashJoin [codegen id : 14]
+Left keys [1]: [cs_sold_date_sk#103]
+Right keys [1]: [d_date_sk#105]
+Join condition: None
+
+(71) Project [codegen id : 14]
+Output [12]: [c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97, cs_ext_discount_amt#99, cs_ext_sales_price#100, cs_ext_wholesale_cost#101, cs_ext_list_price#102, d_year#106]
+Input [14]: [c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97, cs_ext_discount_amt#99, cs_ext_sales_price#100, cs_ext_wholesale_cost#101, cs_ext_list_price#102, cs_sold_date_sk#103, d_date_sk#105, d_year#106]
+
+(72) HashAggregate [codegen id : 14]
+Input [12]: [c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97, cs_ext_discount_amt#99, cs_ext_sales_price#100, cs_ext_wholesale_cost#101, cs_ext_list_price#102, d_year#106]
+Keys [8]: [c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97, d_year#106]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cs_ext_list_price#102 as decimal(8,2))) - promote_precision(cast(cs_ext_wholesale_cost#101 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(cs_ext_discount_amt#99 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(cs_ext_sales_price#100 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))]
+Aggregate Attributes [2]: [sum#107, isEmpty#108]
+Results [10]: [c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97, d_year#106, sum#109, isEmpty#110]
+
+(73) Exchange
+Input [10]: [c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97, d_year#106, sum#109, isEmpty#110]
+Arguments: hashpartitioning(c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97, d_year#106, 5), ENSURE_REQUIREMENTS, [id=#111]
+
+(74) HashAggregate [codegen id : 15]
+Input [10]: [c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97, d_year#106, sum#109, isEmpty#110]
+Keys [8]: [c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97, d_year#106]
+Functions [1]: [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cs_ext_list_price#102 as decimal(8,2))) - promote_precision(cast(cs_ext_wholesale_cost#101 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(cs_ext_discount_amt#99 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(cs_ext_sales_price#100 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cs_ext_list_price#102 as decimal(8,2))) - promote_precision(cast(cs_ext_wholesale_cost#101 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(cs_ext_discount_amt#99 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(cs_ext_sales_price#100 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))#112]
+Results [2]: [c_customer_id#91 AS customer_id#113, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cs_ext_list_price#102 as decimal(8,2))) - promote_precision(cast(cs_ext_wholesale_cost#101 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(cs_ext_discount_amt#99 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(cs_ext_sales_price#100 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))#112 AS year_total#114]
+
+(75) BroadcastExchange
+Input [2]: [customer_id#113, year_total#114]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#115]
+
+(76) BroadcastHashJoin [codegen id : 24]
+Left keys [1]: [customer_id#26]
+Right keys [1]: [customer_id#113]
+Join condition: (CASE WHEN (year_total#88 > 0.000000) THEN CheckOverflow((promote_precision(year_total#114) / promote_precision(year_total#88)), DecimalType(38,14), true) ELSE null END > CASE WHEN (year_total#27 > 0.000000) THEN CheckOverflow((promote_precision(year_total#60) / promote_precision(year_total#27)), DecimalType(38,14), true) ELSE null END)
+
+(77) Project [codegen id : 24]
+Output [10]: [customer_id#26, customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59, year_total#88, year_total#114]
+Input [13]: [customer_id#26, year_total#27, customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59, year_total#60, year_total#88, customer_id#113, year_total#114]
+
+(78) Scan parquet default.customer
+Output [8]: [c_customer_sk#116, c_customer_id#117, c_first_name#118, c_last_name#119, c_preferred_cust_flag#120, c_birth_country#121, c_login#122, c_email_address#123]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string,c_birth_country:string,c_login:string,c_email_address:string>
+
+(79) ColumnarToRow [codegen id : 18]
+Input [8]: [c_customer_sk#116, c_customer_id#117, c_first_name#118, c_last_name#119, c_preferred_cust_flag#120, c_birth_country#121, c_login#122, c_email_address#123]
+
+(80) Filter [codegen id : 18]
+Input [8]: [c_customer_sk#116, c_customer_id#117, c_first_name#118, c_last_name#119, c_preferred_cust_flag#120, c_birth_country#121, c_login#122, c_email_address#123]
+Condition : (isnotnull(c_customer_sk#116) AND isnotnull(c_customer_id#117))
+
+(81) Scan parquet default.web_sales
+Output [6]: [ws_bill_customer_sk#124, ws_ext_discount_amt#125, ws_ext_sales_price#126, ws_ext_wholesale_cost#127, ws_ext_list_price#128, ws_sold_date_sk#129]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#129), dynamicpruningexpression(ws_sold_date_sk#129 IN dynamicpruning#15)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int,ws_ext_discount_amt:decimal(7,2),ws_ext_sales_price:decimal(7,2),ws_ext_wholesale_cost:decimal(7,2),ws_ext_list_price:decimal(7,2)>
+
+(82) ColumnarToRow [codegen id : 16]
+Input [6]: [ws_bill_customer_sk#124, ws_ext_discount_amt#125, ws_ext_sales_price#126, ws_ext_wholesale_cost#127, ws_ext_list_price#128, ws_sold_date_sk#129]
+
+(83) Filter [codegen id : 16]
+Input [6]: [ws_bill_customer_sk#124, ws_ext_discount_amt#125, ws_ext_sales_price#126, ws_ext_wholesale_cost#127, ws_ext_list_price#128, ws_sold_date_sk#129]
+Condition : isnotnull(ws_bill_customer_sk#124)
+
+(84) BroadcastExchange
+Input [6]: [ws_bill_customer_sk#124, ws_ext_discount_amt#125, ws_ext_sales_price#126, ws_ext_wholesale_cost#127, ws_ext_list_price#128, ws_sold_date_sk#129]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#130]
+
+(85) BroadcastHashJoin [codegen id : 18]
+Left keys [1]: [c_customer_sk#116]
+Right keys [1]: [ws_bill_customer_sk#124]
+Join condition: None
+
+(86) Project [codegen id : 18]
+Output [12]: [c_customer_id#117, c_first_name#118, c_last_name#119, c_preferred_cust_flag#120, c_birth_country#121, c_login#122, c_email_address#123, ws_ext_discount_amt#125, ws_ext_sales_price#126, ws_ext_wholesale_cost#127, ws_ext_list_price#128, ws_sold_date_sk#129]
+Input [14]: [c_customer_sk#116, c_customer_id#117, c_first_name#118, c_last_name#119, c_preferred_cust_flag#120, c_birth_country#121, c_login#122, c_email_address#123, ws_bill_customer_sk#124, ws_ext_discount_amt#125, ws_ext_sales_price#126, ws_ext_wholesale_cost#127, ws_ext_list_price#128, ws_sold_date_sk#129]
+
+(87) ReusedExchange [Reuses operator id: 13]
+Output [2]: [d_date_sk#131, d_year#132]
+
+(88) BroadcastHashJoin [codegen id : 18]
+Left keys [1]: [ws_sold_date_sk#129]
+Right keys [1]: [d_date_sk#131]
+Join condition: None
+
+(89) Project [codegen id : 18]
+Output [12]: [c_customer_id#117, c_first_name#118, c_last_name#119, c_preferred_cust_flag#120, c_birth_country#121, c_login#122, c_email_address#123, ws_ext_discount_amt#125, ws_ext_sales_price#126, ws_ext_wholesale_cost#127, ws_ext_list_price#128, d_year#132]
+Input [14]: [c_customer_id#117, c_first_name#118, c_last_name#119, c_preferred_cust_flag#120, c_birth_country#121, c_login#122, c_email_address#123, ws_ext_discount_amt#125, ws_ext_sales_price#126, ws_ext_wholesale_cost#127, ws_ext_list_price#128, ws_sold_date_sk#129, d_date_sk#131, d_year#132]
+
+(90) HashAggregate [codegen id : 18]
+Input [12]: [c_customer_id#117, c_first_name#118, c_last_name#119, c_preferred_cust_flag#120, c_birth_country#121, c_login#122, c_email_address#123, ws_ext_discount_amt#125, ws_ext_sales_price#126, ws_ext_wholesale_cost#127, ws_ext_list_price#128, d_year#132]
+Keys [8]: [c_customer_id#117, c_first_name#118, c_last_name#119, c_preferred_cust_flag#120, c_birth_country#121, c_login#122, c_email_address#123, d_year#132]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ws_ext_list_price#128 as decimal(8,2))) - promote_precision(cast(ws_ext_wholesale_cost#127 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ws_ext_discount_amt#125 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ws_ext_sales_price#126 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))]
+Aggregate Attributes [2]: [sum#133, isEmpty#134]
+Results [10]: [c_customer_id#117, c_first_name#118, c_last_name#119, c_preferred_cust_flag#120, c_birth_country#121, c_login#122, c_email_address#123, d_year#132, sum#135, isEmpty#136]
+
+(91) Exchange
+Input [10]: [c_customer_id#117, c_first_name#118, c_last_name#119, c_preferred_cust_flag#120, c_birth_country#121, c_login#122, c_email_address#123, d_year#132, sum#135, isEmpty#136]
+Arguments: hashpartitioning(c_customer_id#117, c_first_name#118, c_last_name#119, c_preferred_cust_flag#120, c_birth_country#121, c_login#122, c_email_address#123, d_year#132, 5), ENSURE_REQUIREMENTS, [id=#137]
+
+(92) HashAggregate [codegen id : 19]
+Input [10]: [c_customer_id#117, c_first_name#118, c_last_name#119, c_preferred_cust_flag#120, c_birth_country#121, c_login#122, c_email_address#123, d_year#132, sum#135, isEmpty#136]
+Keys [8]: [c_customer_id#117, c_first_name#118, c_last_name#119, c_preferred_cust_flag#120, c_birth_country#121, c_login#122, c_email_address#123, d_year#132]
+Functions [1]: [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ws_ext_list_price#128 as decimal(8,2))) - promote_precision(cast(ws_ext_wholesale_cost#127 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ws_ext_discount_amt#125 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ws_ext_sales_price#126 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ws_ext_list_price#128 as decimal(8,2))) - promote_precision(cast(ws_ext_wholesale_cost#127 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ws_ext_discount_amt#125 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ws_ext_sales_price#126 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))#138]
+Results [2]: [c_customer_id#117 AS customer_id#139, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ws_ext_list_price#128 as decimal(8,2))) - promote_precision(cast(ws_ext_wholesale_cost#127 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ws_ext_discount_amt#125 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ws_ext_sales_price#126 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))#138 AS year_total#140]
+
+(93) Filter [codegen id : 19]
+Input [2]: [customer_id#139, year_total#140]
+Condition : (isnotnull(year_total#140) AND (year_total#140 > 0.000000))
+
+(94) Project [codegen id : 19]
+Output [2]: [customer_id#139 AS customer_id#141, year_total#140 AS year_total#142]
+Input [2]: [customer_id#139, year_total#140]
+
+(95) BroadcastExchange
+Input [2]: [customer_id#141, year_total#142]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#143]
+
+(96) BroadcastHashJoin [codegen id : 24]
+Left keys [1]: [customer_id#26]
+Right keys [1]: [customer_id#141]
+Join condition: None
+
+(97) Project [codegen id : 24]
+Output [11]: [customer_id#26, customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59, year_total#88, year_total#114, year_total#142]
+Input [12]: [customer_id#26, customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59, year_total#88, year_total#114, customer_id#141, year_total#142]
+
+(98) Scan parquet default.customer
+Output [8]: [c_customer_sk#144, c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#149, c_login#150, c_email_address#151]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string,c_birth_country:string,c_login:string,c_email_address:string>
+
+(99) ColumnarToRow [codegen id : 22]
+Input [8]: [c_customer_sk#144, c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#149, c_login#150, c_email_address#151]
+
+(100) Filter [codegen id : 22]
+Input [8]: [c_customer_sk#144, c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#149, c_login#150, c_email_address#151]
+Condition : (isnotnull(c_customer_sk#144) AND isnotnull(c_customer_id#145))
+
+(101) Scan parquet default.web_sales
+Output [6]: [ws_bill_customer_sk#152, ws_ext_discount_amt#153, ws_ext_sales_price#154, ws_ext_wholesale_cost#155, ws_ext_list_price#156, ws_sold_date_sk#157]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#157), dynamicpruningexpression(ws_sold_date_sk#157 IN dynamicpruning#42)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int,ws_ext_discount_amt:decimal(7,2),ws_ext_sales_price:decimal(7,2),ws_ext_wholesale_cost:decimal(7,2),ws_ext_list_price:decimal(7,2)>
+
+(102) ColumnarToRow [codegen id : 20]
+Input [6]: [ws_bill_customer_sk#152, ws_ext_discount_amt#153, ws_ext_sales_price#154, ws_ext_wholesale_cost#155, ws_ext_list_price#156, ws_sold_date_sk#157]
+
+(103) Filter [codegen id : 20]
+Input [6]: [ws_bill_customer_sk#152, ws_ext_discount_amt#153, ws_ext_sales_price#154, ws_ext_wholesale_cost#155, ws_ext_list_price#156, ws_sold_date_sk#157]
+Condition : isnotnull(ws_bill_customer_sk#152)
+
+(104) BroadcastExchange
+Input [6]: [ws_bill_customer_sk#152, ws_ext_discount_amt#153, ws_ext_sales_price#154, ws_ext_wholesale_cost#155, ws_ext_list_price#156, ws_sold_date_sk#157]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#158]
+
+(105) BroadcastHashJoin [codegen id : 22]
+Left keys [1]: [c_customer_sk#144]
+Right keys [1]: [ws_bill_customer_sk#152]
+Join condition: None
+
+(106) Project [codegen id : 22]
+Output [12]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#149, c_login#150, c_email_address#151, ws_ext_discount_amt#153, ws_ext_sales_price#154, ws_ext_wholesale_cost#155, ws_ext_list_price#156, ws_sold_date_sk#157]
+Input [14]: [c_customer_sk#144, c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#149, c_login#150, c_email_address#151, ws_bill_customer_sk#152, ws_ext_discount_amt#153, ws_ext_sales_price#154, ws_ext_wholesale_cost#155, ws_ext_list_price#156, ws_sold_date_sk#157]
+
+(107) ReusedExchange [Reuses operator id: 32]
+Output [2]: [d_date_sk#159, d_year#160]
+
+(108) BroadcastHashJoin [codegen id : 22]
+Left keys [1]: [ws_sold_date_sk#157]
+Right keys [1]: [d_date_sk#159]
+Join condition: None
+
+(109) Project [codegen id : 22]
+Output [12]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#149, c_login#150, c_email_address#151, ws_ext_discount_amt#153, ws_ext_sales_price#154, ws_ext_wholesale_cost#155, ws_ext_list_price#156, d_year#160]
+Input [14]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#149, c_login#150, c_email_address#151, ws_ext_discount_amt#153, ws_ext_sales_price#154, ws_ext_wholesale_cost#155, ws_ext_list_price#156, ws_sold_date_sk#157, d_date_sk#159, d_year#160]
+
+(110) HashAggregate [codegen id : 22]
+Input [12]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#149, c_login#150, c_email_address#151, ws_ext_discount_amt#153, ws_ext_sales_price#154, ws_ext_wholesale_cost#155, ws_ext_list_price#156, d_year#160]
+Keys [8]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#149, c_login#150, c_email_address#151, d_year#160]
+Functions [1]: [partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ws_ext_list_price#156 as decimal(8,2))) - promote_precision(cast(ws_ext_wholesale_cost#155 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ws_ext_discount_amt#153 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ws_ext_sales_price#154 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))]
+Aggregate Attributes [2]: [sum#161, isEmpty#162]
+Results [10]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#149, c_login#150, c_email_address#151, d_year#160, sum#163, isEmpty#164]
+
+(111) Exchange
+Input [10]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#149, c_login#150, c_email_address#151, d_year#160, sum#163, isEmpty#164]
+Arguments: hashpartitioning(c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#149, c_login#150, c_email_address#151, d_year#160, 5), ENSURE_REQUIREMENTS, [id=#165]
+
+(112) HashAggregate [codegen id : 23]
+Input [10]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#149, c_login#150, c_email_address#151, d_year#160, sum#163, isEmpty#164]
+Keys [8]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#149, c_login#150, c_email_address#151, d_year#160]
+Functions [1]: [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ws_ext_list_price#156 as decimal(8,2))) - promote_precision(cast(ws_ext_wholesale_cost#155 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ws_ext_discount_amt#153 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ws_ext_sales_price#154 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))]
+Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ws_ext_list_price#156 as decimal(8,2))) - promote_precision(cast(ws_ext_wholesale_cost#155 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ws_ext_discount_amt#153 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ws_ext_sales_price#154 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))#166]
+Results [2]: [c_customer_id#145 AS customer_id#167, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ws_ext_list_price#156 as decimal(8,2))) - promote_precision(cast(ws_ext_wholesale_cost#155 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ws_ext_discount_amt#153 as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ws_ext_sales_price#154 as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true))#166 AS year_total#168]
+
+(113) BroadcastExchange
+Input [2]: [customer_id#167, year_total#168]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#169]
+
+(114) BroadcastHashJoin [codegen id : 24]
+Left keys [1]: [customer_id#26]
+Right keys [1]: [customer_id#167]
+Join condition: (CASE WHEN (year_total#88 > 0.000000) THEN CheckOverflow((promote_precision(year_total#114) / promote_precision(year_total#88)), DecimalType(38,14), true) ELSE null END > CASE WHEN (year_total#142 > 0.000000) THEN CheckOverflow((promote_precision(year_total#168) / promote_precision(year_total#142)), DecimalType(38,14), true) ELSE null END)
+
+(115) Project [codegen id : 24]
+Output [7]: [customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59]
+Input [13]: [customer_id#26, customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59, year_total#88, year_total#114, year_total#142, customer_id#167, year_total#168]
+
+(116) TakeOrderedAndProject
+Input [7]: [customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59]
+Arguments: 100, [customer_id#53 ASC NULLS FIRST, customer_first_name#54 ASC NULLS FIRST, customer_last_name#55 ASC NULLS FIRST, customer_preferred_cust_flag#56 ASC NULLS FIRST, customer_birth_country#57 ASC NULLS FIRST, customer_login#58 ASC NULLS FIRST, customer_email_address#59 ASC NULLS FIRST], [customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#14 IN dynamicpruning#15
+ReusedExchange (117)
+
+
+(117) ReusedExchange [Reuses operator id: 13]
+Output [2]: [d_date_sk#17, d_year#18]
+
+Subquery:2 Hosting operator id = 23 Hosting Expression = ss_sold_date_sk#41 IN dynamicpruning#42
+ReusedExchange (118)
+
+
+(118) ReusedExchange [Reuses operator id: 32]
+Output [2]: [d_date_sk#44, d_year#45]
+
+Subquery:3 Hosting operator id = 43 Hosting Expression = cs_sold_date_sk#75 IN dynamicpruning#15
+
+Subquery:4 Hosting operator id = 63 Hosting Expression = cs_sold_date_sk#103 IN dynamicpruning#42
+
+Subquery:5 Hosting operator id = 81 Hosting Expression = ws_sold_date_sk#129 IN dynamicpruning#15
+
+Subquery:6 Hosting operator id = 101 Hosting Expression = ws_sold_date_sk#157 IN dynamicpruning#42
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/simplified.txt
new file mode 100644
index 0000000000000..fcfef6b7d85ed
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/simplified.txt
@@ -0,0 +1,181 @@
+TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address]
+  WholeStageCodegen (24)
+    Project [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address]
+      BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total]
+        Project [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,year_total]
+          BroadcastHashJoin [customer_id,customer_id]
+            Project [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total]
+              BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total]
+                Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total]
+                  BroadcastHashJoin [customer_id,customer_id]
+                    BroadcastHashJoin [customer_id,customer_id]
+                      Filter [year_total]
+                        HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_ext_list_price as decimal(8,2))) - promote_precision(cast(ss_ext_wholesale_cost as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ss_ext_discount_amt as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ss_ext_sales_price as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true)),customer_id,year_total,sum,isEmpty]
+                          InputAdapter
+                            Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #1
+                              WholeStageCodegen (3)
+                                HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty]
+                                  Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year]
+                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                      Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk]
+                                        BroadcastHashJoin [c_customer_sk,ss_customer_sk]
+                                          Filter [c_customer_sk,c_customer_id]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address]
+                                          InputAdapter
+                                            BroadcastExchange #2
+                                              WholeStageCodegen (1)
+                                                Filter [ss_customer_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk]
+                                                        SubqueryBroadcast [d_date_sk] #1
+                                                          ReusedExchange [d_date_sk,d_year] #3
+                                      InputAdapter
+                                        BroadcastExchange #3
+                                          WholeStageCodegen (2)
+                                            Filter [d_year,d_date_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.date_dim [d_date_sk,d_year]
+                      InputAdapter
+                        BroadcastExchange #4
+                          WholeStageCodegen (7)
+                            HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_ext_list_price as decimal(8,2))) - promote_precision(cast(ss_ext_wholesale_cost as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ss_ext_discount_amt as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ss_ext_sales_price as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true)),customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,sum,isEmpty]
+                              InputAdapter
+                                Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #5
+                                  WholeStageCodegen (6)
+                                    HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty]
+                                      Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year]
+                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                          Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk]
+                                            BroadcastHashJoin [c_customer_sk,ss_customer_sk]
+                                              Filter [c_customer_sk,c_customer_id]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address]
+                                              InputAdapter
+                                                BroadcastExchange #6
+                                                  WholeStageCodegen (4)
+                                                    Filter [ss_customer_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk]
+                                                            SubqueryBroadcast [d_date_sk] #2
+                                                              ReusedExchange [d_date_sk,d_year] #7
+                                          InputAdapter
+                                            BroadcastExchange #7
+                                              WholeStageCodegen (5)
+                                                Filter [d_year,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_year]
+                    InputAdapter
+                      BroadcastExchange #8
+                        WholeStageCodegen (11)
+                          Project [customer_id,year_total]
+                            Filter [year_total]
+                              HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cs_ext_list_price as decimal(8,2))) - promote_precision(cast(cs_ext_wholesale_cost as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(cs_ext_discount_amt as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(cs_ext_sales_price as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true)),customer_id,year_total,sum,isEmpty]
+                                InputAdapter
+                                  Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #9
+                                    WholeStageCodegen (10)
+                                      HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [sum,isEmpty,sum,isEmpty]
+                                        Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year]
+                                          BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                            Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk]
+                                              BroadcastHashJoin [c_customer_sk,cs_bill_customer_sk]
+                                                Filter [c_customer_sk,c_customer_id]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address]
+                                                InputAdapter
+                                                  BroadcastExchange #10
+                                                    WholeStageCodegen (8)
+                                                      Filter [cs_bill_customer_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk]
+                                                              ReusedSubquery [d_date_sk] #1
+                                            InputAdapter
+                                              ReusedExchange [d_date_sk,d_year] #3
+                InputAdapter
+                  BroadcastExchange #11
+                    WholeStageCodegen (15)
+                      HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cs_ext_list_price as decimal(8,2))) - promote_precision(cast(cs_ext_wholesale_cost as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(cs_ext_discount_amt as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(cs_ext_sales_price as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true)),customer_id,year_total,sum,isEmpty]
+                        InputAdapter
+                          Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #12
+                            WholeStageCodegen (14)
+                              HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [sum,isEmpty,sum,isEmpty]
+                                Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year]
+                                  BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                    Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk]
+                                      BroadcastHashJoin [c_customer_sk,cs_bill_customer_sk]
+                                        Filter [c_customer_sk,c_customer_id]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address]
+                                        InputAdapter
+                                          BroadcastExchange #13
+                                            WholeStageCodegen (12)
+                                              Filter [cs_bill_customer_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk]
+                                                      ReusedSubquery [d_date_sk] #2
+                                    InputAdapter
+                                      ReusedExchange [d_date_sk,d_year] #7
+            InputAdapter
+              BroadcastExchange #14
+                WholeStageCodegen (19)
+                  Project [customer_id,year_total]
+                    Filter [year_total]
+                      HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ws_ext_list_price as decimal(8,2))) - promote_precision(cast(ws_ext_wholesale_cost as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ws_ext_discount_amt as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ws_ext_sales_price as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true)),customer_id,year_total,sum,isEmpty]
+                        InputAdapter
+                          Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #15
+                            WholeStageCodegen (18)
+                              HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [sum,isEmpty,sum,isEmpty]
+                                Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year]
+                                  BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                    Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk]
+                                      BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk]
+                                        Filter [c_customer_sk,c_customer_id]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address]
+                                        InputAdapter
+                                          BroadcastExchange #16
+                                            WholeStageCodegen (16)
+                                              Filter [ws_bill_customer_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk]
+                                                      ReusedSubquery [d_date_sk] #1
+                                    InputAdapter
+                                      ReusedExchange [d_date_sk,d_year] #3
+        InputAdapter
+          BroadcastExchange #17
+            WholeStageCodegen (23)
+              HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ws_ext_list_price as decimal(8,2))) - promote_precision(cast(ws_ext_wholesale_cost as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) - promote_precision(cast(ws_ext_discount_amt as decimal(9,2)))), DecimalType(9,2), true) as decimal(10,2))) + promote_precision(cast(ws_ext_sales_price as decimal(10,2)))), DecimalType(10,2), true)) / 2.00), DecimalType(14,6), true)),customer_id,year_total,sum,isEmpty]
+                InputAdapter
+                  Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #18
+                    WholeStageCodegen (22)
+                      HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [sum,isEmpty,sum,isEmpty]
+                        Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year]
+                          BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                            Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk]
+                              BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk]
+                                Filter [c_customer_sk,c_customer_id]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address]
+                                InputAdapter
+                                  BroadcastExchange #19
+                                    WholeStageCodegen (20)
+                                      Filter [ws_bill_customer_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk]
+                                              ReusedSubquery [d_date_sk] #2
+                            InputAdapter
+                              ReusedExchange [d_date_sk,d_year] #7
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.sf100/explain.txt
new file mode 100644
index 0000000000000..fcd1b82886242
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.sf100/explain.txt
@@ -0,0 +1,214 @@
+== Physical Plan ==
+TakeOrderedAndProject (36)
++- * HashAggregate (35)
+   +- Exchange (34)
+      +- * HashAggregate (33)
+         +- * Project (32)
+            +- * BroadcastHashJoin Inner BuildRight (31)
+               :- * Project (26)
+               :  +- * BroadcastHashJoin Inner BuildRight (25)
+               :     :- * Project (20)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (19)
+               :     :     :- * Project (13)
+               :     :     :  +- * SortMergeJoin LeftOuter (12)
+               :     :     :     :- * Sort (5)
+               :     :     :     :  +- Exchange (4)
+               :     :     :     :     +- * Filter (3)
+               :     :     :     :        +- * ColumnarToRow (2)
+               :     :     :     :           +- Scan parquet default.catalog_sales (1)
+               :     :     :     +- * Sort (11)
+               :     :     :        +- Exchange (10)
+               :     :     :           +- * Project (9)
+               :     :     :              +- * Filter (8)
+               :     :     :                 +- * ColumnarToRow (7)
+               :     :     :                    +- Scan parquet default.catalog_returns (6)
+               :     :     +- BroadcastExchange (18)
+               :     :        +- * Project (17)
+               :     :           +- * Filter (16)
+               :     :              +- * ColumnarToRow (15)
+               :     :                 +- Scan parquet default.item (14)
+               :     +- BroadcastExchange (24)
+               :        +- * Filter (23)
+               :           +- * ColumnarToRow (22)
+               :              +- Scan parquet default.date_dim (21)
+               +- BroadcastExchange (30)
+                  +- * Filter (29)
+                     +- * ColumnarToRow (28)
+                        +- Scan parquet default.warehouse (27)
+
+
+(1) Scan parquet default.catalog_sales
+Output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_warehouse_sk:int,cs_item_sk:int,cs_order_number:int,cs_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5]
+
+(3) Filter [codegen id : 1]
+Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5]
+Condition : (isnotnull(cs_warehouse_sk#1) AND isnotnull(cs_item_sk#2))
+
+(4) Exchange
+Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5]
+Arguments: hashpartitioning(cs_order_number#3, cs_item_sk#2, 5), ENSURE_REQUIREMENTS, [id=#7]
+
+(5) Sort [codegen id : 2]
+Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5]
+Arguments: [cs_order_number#3 ASC NULLS FIRST, cs_item_sk#2 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.catalog_returns
+Output [4]: [cr_item_sk#8, cr_order_number#9, cr_refunded_cash#10, cr_returned_date_sk#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int,cr_refunded_cash:decimal(7,2)>
+
+(7) ColumnarToRow [codegen id : 3]
+Input [4]: [cr_item_sk#8, cr_order_number#9, cr_refunded_cash#10, cr_returned_date_sk#11]
+
+(8) Filter [codegen id : 3]
+Input [4]: [cr_item_sk#8, cr_order_number#9, cr_refunded_cash#10, cr_returned_date_sk#11]
+Condition : (isnotnull(cr_order_number#9) AND isnotnull(cr_item_sk#8))
+
+(9) Project [codegen id : 3]
+Output [3]: [cr_item_sk#8, cr_order_number#9, cr_refunded_cash#10]
+Input [4]: [cr_item_sk#8, cr_order_number#9, cr_refunded_cash#10, cr_returned_date_sk#11]
+
+(10) Exchange
+Input [3]: [cr_item_sk#8, cr_order_number#9, cr_refunded_cash#10]
+Arguments: hashpartitioning(cr_order_number#9, cr_item_sk#8, 5), ENSURE_REQUIREMENTS, [id=#12]
+
+(11) Sort [codegen id : 4]
+Input [3]: [cr_item_sk#8, cr_order_number#9, cr_refunded_cash#10]
+Arguments: [cr_order_number#9 ASC NULLS FIRST, cr_item_sk#8 ASC NULLS FIRST], false, 0
+
+(12) SortMergeJoin [codegen id : 8]
+Left keys [2]: [cs_order_number#3, cs_item_sk#2]
+Right keys [2]: [cr_order_number#9, cr_item_sk#8]
+Join condition: None
+
+(13) Project [codegen id : 8]
+Output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#10]
+Input [8]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5, cr_item_sk#8, cr_order_number#9, cr_refunded_cash#10]
+
+(14) Scan parquet default.item
+Output [3]: [i_item_sk#13, i_item_id#14, i_current_price#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,0.99), LessThanOrEqual(i_current_price,1.49), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_current_price:decimal(7,2)>
+
+(15) ColumnarToRow [codegen id : 5]
+Input [3]: [i_item_sk#13, i_item_id#14, i_current_price#15]
+
+(16) Filter [codegen id : 5]
+Input [3]: [i_item_sk#13, i_item_id#14, i_current_price#15]
+Condition : (((isnotnull(i_current_price#15) AND (i_current_price#15 >= 0.99)) AND (i_current_price#15 <= 1.49)) AND isnotnull(i_item_sk#13))
+
+(17) Project [codegen id : 5]
+Output [2]: [i_item_sk#13, i_item_id#14]
+Input [3]: [i_item_sk#13, i_item_id#14, i_current_price#15]
+
+(18) BroadcastExchange
+Input [2]: [i_item_sk#13, i_item_id#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#16]
+
+(19) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [cs_item_sk#2]
+Right keys [1]: [i_item_sk#13]
+Join condition: None
+
+(20) Project [codegen id : 8]
+Output [5]: [cs_warehouse_sk#1, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#10, i_item_id#14]
+Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#10, i_item_sk#13, i_item_id#14]
+
+(21) Scan parquet default.date_dim
+Output [2]: [d_date_sk#17, d_date#18]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(22) ColumnarToRow [codegen id : 6]
+Input [2]: [d_date_sk#17, d_date#18]
+
+(23) Filter [codegen id : 6]
+Input [2]: [d_date_sk#17, d_date#18]
+Condition : (((isnotnull(d_date#18) AND (d_date#18 >= 2000-02-10)) AND (d_date#18 <= 2000-04-10)) AND isnotnull(d_date_sk#17))
+
+(24) BroadcastExchange
+Input [2]: [d_date_sk#17, d_date#18]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#19]
+
+(25) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [cs_sold_date_sk#5]
+Right keys [1]: [d_date_sk#17]
+Join condition: None
+
+(26) Project [codegen id : 8]
+Output [5]: [cs_warehouse_sk#1, cs_sales_price#4, cr_refunded_cash#10, i_item_id#14, d_date#18]
+Input [7]: [cs_warehouse_sk#1, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#10, i_item_id#14, d_date_sk#17, d_date#18]
+
+(27) Scan parquet default.warehouse
+Output [2]: [w_warehouse_sk#20, w_state#21]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/warehouse]
+PushedFilters: [IsNotNull(w_warehouse_sk)]
+ReadSchema: struct<w_warehouse_sk:int,w_state:string>
+
+(28) ColumnarToRow [codegen id : 7]
+Input [2]: [w_warehouse_sk#20, w_state#21]
+
+(29) Filter [codegen id : 7]
+Input [2]: [w_warehouse_sk#20, w_state#21]
+Condition : isnotnull(w_warehouse_sk#20)
+
+(30) BroadcastExchange
+Input [2]: [w_warehouse_sk#20, w_state#21]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#22]
+
+(31) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [cs_warehouse_sk#1]
+Right keys [1]: [w_warehouse_sk#20]
+Join condition: None
+
+(32) Project [codegen id : 8]
+Output [5]: [cs_sales_price#4, cr_refunded_cash#10, w_state#21, i_item_id#14, d_date#18]
+Input [7]: [cs_warehouse_sk#1, cs_sales_price#4, cr_refunded_cash#10, i_item_id#14, d_date#18, w_warehouse_sk#20, w_state#21]
+
+(33) HashAggregate [codegen id : 8]
+Input [5]: [cs_sales_price#4, cr_refunded_cash#10, w_state#21, i_item_id#14, d_date#18]
+Keys [2]: [w_state#21, i_item_id#14]
+Functions [2]: [partial_sum(CASE WHEN (d_date#18 < 2000-03-11) THEN CheckOverflow((promote_precision(cast(cs_sales_price#4 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_refunded_cash#10 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_date#18 >= 2000-03-11) THEN CheckOverflow((promote_precision(cast(cs_sales_price#4 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_refunded_cash#10 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true) ELSE 0.00 END)]
+Aggregate Attributes [4]: [sum#23, isEmpty#24, sum#25, isEmpty#26]
+Results [6]: [w_state#21, i_item_id#14, sum#27, isEmpty#28, sum#29, isEmpty#30]
+
+(34) Exchange
+Input [6]: [w_state#21, i_item_id#14, sum#27, isEmpty#28, sum#29, isEmpty#30]
+Arguments: hashpartitioning(w_state#21, i_item_id#14, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(35) HashAggregate [codegen id : 9]
+Input [6]: [w_state#21, i_item_id#14, sum#27, isEmpty#28, sum#29, isEmpty#30]
+Keys [2]: [w_state#21, i_item_id#14]
+Functions [2]: [sum(CASE WHEN (d_date#18 < 2000-03-11) THEN CheckOverflow((promote_precision(cast(cs_sales_price#4 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_refunded_cash#10 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true) ELSE 0.00 END), sum(CASE WHEN (d_date#18 >= 2000-03-11) THEN CheckOverflow((promote_precision(cast(cs_sales_price#4 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_refunded_cash#10 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true) ELSE 0.00 END)]
+Aggregate Attributes [2]: [sum(CASE WHEN (d_date#18 < 2000-03-11) THEN CheckOverflow((promote_precision(cast(cs_sales_price#4 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_refunded_cash#10 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true) ELSE 0.00 END)#32, sum(CASE WHEN (d_date#18 >= 2000-03-11) THEN CheckOverflow((promote_precision(cast(cs_sales_price#4 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_refunded_cash#10 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true) ELSE 0.00 END)#33]
+Results [4]: [w_state#21, i_item_id#14, sum(CASE WHEN (d_date#18 < 2000-03-11) THEN CheckOverflow((promote_precision(cast(cs_sales_price#4 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_refunded_cash#10 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true) ELSE 0.00 END)#32 AS sales_before#34, sum(CASE WHEN (d_date#18 >= 2000-03-11) THEN CheckOverflow((promote_precision(cast(cs_sales_price#4 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_refunded_cash#10 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true) ELSE 0.00 END)#33 AS sales_after#35]
+
+(36) TakeOrderedAndProject
+Input [4]: [w_state#21, i_item_id#14, sales_before#34, sales_after#35]
+Arguments: 100, [w_state#21 ASC NULLS FIRST, i_item_id#14 ASC NULLS FIRST], [w_state#21, i_item_id#14, sales_before#34, sales_after#35]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (37)
+
+
+(37) ReusedExchange [Reuses operator id: 24]
+Output [2]: [d_date_sk#17, d_date#18]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.sf100/simplified.txt
new file mode 100644
index 0000000000000..53eafedcaa812
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.sf100/simplified.txt
@@ -0,0 +1,60 @@
+TakeOrderedAndProject [w_state,i_item_id,sales_before,sales_after]
+  WholeStageCodegen (9)
+    HashAggregate [w_state,i_item_id,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_date < 2000-03-11) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true) ELSE 0.00 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true) ELSE 0.00 END),sales_before,sales_after,sum,isEmpty,sum,isEmpty]
+      InputAdapter
+        Exchange [w_state,i_item_id] #1
+          WholeStageCodegen (8)
+            HashAggregate [w_state,i_item_id,d_date,cs_sales_price,cr_refunded_cash] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+              Project [cs_sales_price,cr_refunded_cash,w_state,i_item_id,d_date]
+                BroadcastHashJoin [cs_warehouse_sk,w_warehouse_sk]
+                  Project [cs_warehouse_sk,cs_sales_price,cr_refunded_cash,i_item_id,d_date]
+                    BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                      Project [cs_warehouse_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,i_item_id]
+                        BroadcastHashJoin [cs_item_sk,i_item_sk]
+                          Project [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash]
+                            SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk]
+                              InputAdapter
+                                WholeStageCodegen (2)
+                                  Sort [cs_order_number,cs_item_sk]
+                                    InputAdapter
+                                      Exchange [cs_order_number,cs_item_sk] #2
+                                        WholeStageCodegen (1)
+                                          Filter [cs_warehouse_sk,cs_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk]
+                                                  SubqueryBroadcast [d_date_sk] #1
+                                                    ReusedExchange [d_date_sk,d_date] #3
+                              InputAdapter
+                                WholeStageCodegen (4)
+                                  Sort [cr_order_number,cr_item_sk]
+                                    InputAdapter
+                                      Exchange [cr_order_number,cr_item_sk] #4
+                                        WholeStageCodegen (3)
+                                          Project [cr_item_sk,cr_order_number,cr_refunded_cash]
+                                            Filter [cr_order_number,cr_item_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk]
+                          InputAdapter
+                            BroadcastExchange #5
+                              WholeStageCodegen (5)
+                                Project [i_item_sk,i_item_id]
+                                  Filter [i_current_price,i_item_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.item [i_item_sk,i_item_id,i_current_price]
+                      InputAdapter
+                        BroadcastExchange #3
+                          WholeStageCodegen (6)
+                            Filter [d_date,d_date_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.date_dim [d_date_sk,d_date]
+                  InputAdapter
+                    BroadcastExchange #6
+                      WholeStageCodegen (7)
+                        Filter [w_warehouse_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.warehouse [w_warehouse_sk,w_state]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt
new file mode 100644
index 0000000000000..367d15e3b52f5
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt
@@ -0,0 +1,214 @@
+== Physical Plan ==
+TakeOrderedAndProject (36)
++- * HashAggregate (35)
+   +- Exchange (34)
+      +- * HashAggregate (33)
+         +- * Project (32)
+            +- * BroadcastHashJoin Inner BuildRight (31)
+               :- * Project (26)
+               :  +- * BroadcastHashJoin Inner BuildRight (25)
+               :     :- * Project (19)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (18)
+               :     :     :- * Project (13)
+               :     :     :  +- * SortMergeJoin LeftOuter (12)
+               :     :     :     :- * Sort (5)
+               :     :     :     :  +- Exchange (4)
+               :     :     :     :     +- * Filter (3)
+               :     :     :     :        +- * ColumnarToRow (2)
+               :     :     :     :           +- Scan parquet default.catalog_sales (1)
+               :     :     :     +- * Sort (11)
+               :     :     :        +- Exchange (10)
+               :     :     :           +- * Project (9)
+               :     :     :              +- * Filter (8)
+               :     :     :                 +- * ColumnarToRow (7)
+               :     :     :                    +- Scan parquet default.catalog_returns (6)
+               :     :     +- BroadcastExchange (17)
+               :     :        +- * Filter (16)
+               :     :           +- * ColumnarToRow (15)
+               :     :              +- Scan parquet default.warehouse (14)
+               :     +- BroadcastExchange (24)
+               :        +- * Project (23)
+               :           +- * Filter (22)
+               :              +- * ColumnarToRow (21)
+               :                 +- Scan parquet default.item (20)
+               +- BroadcastExchange (30)
+                  +- * Filter (29)
+                     +- * ColumnarToRow (28)
+                        +- Scan parquet default.date_dim (27)
+
+
+(1) Scan parquet default.catalog_sales
+Output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_warehouse_sk:int,cs_item_sk:int,cs_order_number:int,cs_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5]
+
+(3) Filter [codegen id : 1]
+Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5]
+Condition : (isnotnull(cs_warehouse_sk#1) AND isnotnull(cs_item_sk#2))
+
+(4) Exchange
+Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5]
+Arguments: hashpartitioning(cs_order_number#3, cs_item_sk#2, 5), ENSURE_REQUIREMENTS, [id=#7]
+
+(5) Sort [codegen id : 2]
+Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5]
+Arguments: [cs_order_number#3 ASC NULLS FIRST, cs_item_sk#2 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.catalog_returns
+Output [4]: [cr_item_sk#8, cr_order_number#9, cr_refunded_cash#10, cr_returned_date_sk#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int,cr_refunded_cash:decimal(7,2)>
+
+(7) ColumnarToRow [codegen id : 3]
+Input [4]: [cr_item_sk#8, cr_order_number#9, cr_refunded_cash#10, cr_returned_date_sk#11]
+
+(8) Filter [codegen id : 3]
+Input [4]: [cr_item_sk#8, cr_order_number#9, cr_refunded_cash#10, cr_returned_date_sk#11]
+Condition : (isnotnull(cr_order_number#9) AND isnotnull(cr_item_sk#8))
+
+(9) Project [codegen id : 3]
+Output [3]: [cr_item_sk#8, cr_order_number#9, cr_refunded_cash#10]
+Input [4]: [cr_item_sk#8, cr_order_number#9, cr_refunded_cash#10, cr_returned_date_sk#11]
+
+(10) Exchange
+Input [3]: [cr_item_sk#8, cr_order_number#9, cr_refunded_cash#10]
+Arguments: hashpartitioning(cr_order_number#9, cr_item_sk#8, 5), ENSURE_REQUIREMENTS, [id=#12]
+
+(11) Sort [codegen id : 4]
+Input [3]: [cr_item_sk#8, cr_order_number#9, cr_refunded_cash#10]
+Arguments: [cr_order_number#9 ASC NULLS FIRST, cr_item_sk#8 ASC NULLS FIRST], false, 0
+
+(12) SortMergeJoin [codegen id : 8]
+Left keys [2]: [cs_order_number#3, cs_item_sk#2]
+Right keys [2]: [cr_order_number#9, cr_item_sk#8]
+Join condition: None
+
+(13) Project [codegen id : 8]
+Output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#10]
+Input [8]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5, cr_item_sk#8, cr_order_number#9, cr_refunded_cash#10]
+
+(14) Scan parquet default.warehouse
+Output [2]: [w_warehouse_sk#13, w_state#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/warehouse]
+PushedFilters: [IsNotNull(w_warehouse_sk)]
+ReadSchema: struct<w_warehouse_sk:int,w_state:string>
+
+(15) ColumnarToRow [codegen id : 5]
+Input [2]: [w_warehouse_sk#13, w_state#14]
+
+(16) Filter [codegen id : 5]
+Input [2]: [w_warehouse_sk#13, w_state#14]
+Condition : isnotnull(w_warehouse_sk#13)
+
+(17) BroadcastExchange
+Input [2]: [w_warehouse_sk#13, w_state#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#15]
+
+(18) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [cs_warehouse_sk#1]
+Right keys [1]: [w_warehouse_sk#13]
+Join condition: None
+
+(19) Project [codegen id : 8]
+Output [5]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#10, w_state#14]
+Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#10, w_warehouse_sk#13, w_state#14]
+
+(20) Scan parquet default.item
+Output [3]: [i_item_sk#16, i_item_id#17, i_current_price#18]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,0.99), LessThanOrEqual(i_current_price,1.49), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_current_price:decimal(7,2)>
+
+(21) ColumnarToRow [codegen id : 6]
+Input [3]: [i_item_sk#16, i_item_id#17, i_current_price#18]
+
+(22) Filter [codegen id : 6]
+Input [3]: [i_item_sk#16, i_item_id#17, i_current_price#18]
+Condition : (((isnotnull(i_current_price#18) AND (i_current_price#18 >= 0.99)) AND (i_current_price#18 <= 1.49)) AND isnotnull(i_item_sk#16))
+
+(23) Project [codegen id : 6]
+Output [2]: [i_item_sk#16, i_item_id#17]
+Input [3]: [i_item_sk#16, i_item_id#17, i_current_price#18]
+
+(24) BroadcastExchange
+Input [2]: [i_item_sk#16, i_item_id#17]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#19]
+
+(25) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [cs_item_sk#2]
+Right keys [1]: [i_item_sk#16]
+Join condition: None
+
+(26) Project [codegen id : 8]
+Output [5]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#10, w_state#14, i_item_id#17]
+Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#10, w_state#14, i_item_sk#16, i_item_id#17]
+
+(27) Scan parquet default.date_dim
+Output [2]: [d_date_sk#20, d_date#21]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(28) ColumnarToRow [codegen id : 7]
+Input [2]: [d_date_sk#20, d_date#21]
+
+(29) Filter [codegen id : 7]
+Input [2]: [d_date_sk#20, d_date#21]
+Condition : (((isnotnull(d_date#21) AND (d_date#21 >= 2000-02-10)) AND (d_date#21 <= 2000-04-10)) AND isnotnull(d_date_sk#20))
+
+(30) BroadcastExchange
+Input [2]: [d_date_sk#20, d_date#21]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#22]
+
+(31) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [cs_sold_date_sk#5]
+Right keys [1]: [d_date_sk#20]
+Join condition: None
+
+(32) Project [codegen id : 8]
+Output [5]: [cs_sales_price#4, cr_refunded_cash#10, w_state#14, i_item_id#17, d_date#21]
+Input [7]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#10, w_state#14, i_item_id#17, d_date_sk#20, d_date#21]
+
+(33) HashAggregate [codegen id : 8]
+Input [5]: [cs_sales_price#4, cr_refunded_cash#10, w_state#14, i_item_id#17, d_date#21]
+Keys [2]: [w_state#14, i_item_id#17]
+Functions [2]: [partial_sum(CASE WHEN (d_date#21 < 2000-03-11) THEN CheckOverflow((promote_precision(cast(cs_sales_price#4 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_refunded_cash#10 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_date#21 >= 2000-03-11) THEN CheckOverflow((promote_precision(cast(cs_sales_price#4 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_refunded_cash#10 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true) ELSE 0.00 END)]
+Aggregate Attributes [4]: [sum#23, isEmpty#24, sum#25, isEmpty#26]
+Results [6]: [w_state#14, i_item_id#17, sum#27, isEmpty#28, sum#29, isEmpty#30]
+
+(34) Exchange
+Input [6]: [w_state#14, i_item_id#17, sum#27, isEmpty#28, sum#29, isEmpty#30]
+Arguments: hashpartitioning(w_state#14, i_item_id#17, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(35) HashAggregate [codegen id : 9]
+Input [6]: [w_state#14, i_item_id#17, sum#27, isEmpty#28, sum#29, isEmpty#30]
+Keys [2]: [w_state#14, i_item_id#17]
+Functions [2]: [sum(CASE WHEN (d_date#21 < 2000-03-11) THEN CheckOverflow((promote_precision(cast(cs_sales_price#4 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_refunded_cash#10 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true) ELSE 0.00 END), sum(CASE WHEN (d_date#21 >= 2000-03-11) THEN CheckOverflow((promote_precision(cast(cs_sales_price#4 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_refunded_cash#10 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true) ELSE 0.00 END)]
+Aggregate Attributes [2]: [sum(CASE WHEN (d_date#21 < 2000-03-11) THEN CheckOverflow((promote_precision(cast(cs_sales_price#4 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_refunded_cash#10 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true) ELSE 0.00 END)#32, sum(CASE WHEN (d_date#21 >= 2000-03-11) THEN CheckOverflow((promote_precision(cast(cs_sales_price#4 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_refunded_cash#10 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true) ELSE 0.00 END)#33]
+Results [4]: [w_state#14, i_item_id#17, sum(CASE WHEN (d_date#21 < 2000-03-11) THEN CheckOverflow((promote_precision(cast(cs_sales_price#4 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_refunded_cash#10 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true) ELSE 0.00 END)#32 AS sales_before#34, sum(CASE WHEN (d_date#21 >= 2000-03-11) THEN CheckOverflow((promote_precision(cast(cs_sales_price#4 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_refunded_cash#10 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true) ELSE 0.00 END)#33 AS sales_after#35]
+
+(36) TakeOrderedAndProject
+Input [4]: [w_state#14, i_item_id#17, sales_before#34, sales_after#35]
+Arguments: 100, [w_state#14 ASC NULLS FIRST, i_item_id#17 ASC NULLS FIRST], [w_state#14, i_item_id#17, sales_before#34, sales_after#35]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (37)
+
+
+(37) ReusedExchange [Reuses operator id: 30]
+Output [2]: [d_date_sk#20, d_date#21]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/simplified.txt
new file mode 100644
index 0000000000000..9f8f7b0c5fe45
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/simplified.txt
@@ -0,0 +1,60 @@
+TakeOrderedAndProject [w_state,i_item_id,sales_before,sales_after]
+  WholeStageCodegen (9)
+    HashAggregate [w_state,i_item_id,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_date < 2000-03-11) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true) ELSE 0.00 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true) ELSE 0.00 END),sales_before,sales_after,sum,isEmpty,sum,isEmpty]
+      InputAdapter
+        Exchange [w_state,i_item_id] #1
+          WholeStageCodegen (8)
+            HashAggregate [w_state,i_item_id,d_date,cs_sales_price,cr_refunded_cash] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+              Project [cs_sales_price,cr_refunded_cash,w_state,i_item_id,d_date]
+                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                  Project [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id]
+                    BroadcastHashJoin [cs_item_sk,i_item_sk]
+                      Project [cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state]
+                        BroadcastHashJoin [cs_warehouse_sk,w_warehouse_sk]
+                          Project [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash]
+                            SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk]
+                              InputAdapter
+                                WholeStageCodegen (2)
+                                  Sort [cs_order_number,cs_item_sk]
+                                    InputAdapter
+                                      Exchange [cs_order_number,cs_item_sk] #2
+                                        WholeStageCodegen (1)
+                                          Filter [cs_warehouse_sk,cs_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk]
+                                                  SubqueryBroadcast [d_date_sk] #1
+                                                    ReusedExchange [d_date_sk,d_date] #3
+                              InputAdapter
+                                WholeStageCodegen (4)
+                                  Sort [cr_order_number,cr_item_sk]
+                                    InputAdapter
+                                      Exchange [cr_order_number,cr_item_sk] #4
+                                        WholeStageCodegen (3)
+                                          Project [cr_item_sk,cr_order_number,cr_refunded_cash]
+                                            Filter [cr_order_number,cr_item_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk]
+                          InputAdapter
+                            BroadcastExchange #5
+                              WholeStageCodegen (5)
+                                Filter [w_warehouse_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.warehouse [w_warehouse_sk,w_state]
+                      InputAdapter
+                        BroadcastExchange #6
+                          WholeStageCodegen (6)
+                            Project [i_item_sk,i_item_id]
+                              Filter [i_current_price,i_item_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.item [i_item_sk,i_item_id,i_current_price]
+                  InputAdapter
+                    BroadcastExchange #3
+                      WholeStageCodegen (7)
+                        Filter [d_date,d_date_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.date_dim [d_date_sk,d_date]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.sf100/explain.txt
new file mode 100644
index 0000000000000..33c03d2b767dd
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.sf100/explain.txt
@@ -0,0 +1,120 @@
+== Physical Plan ==
+TakeOrderedAndProject (20)
++- * HashAggregate (19)
+   +- Exchange (18)
+      +- * HashAggregate (17)
+         +- * Project (16)
+            +- * BroadcastHashJoin Inner BuildRight (15)
+               :- * Project (4)
+               :  +- * Filter (3)
+               :     +- * ColumnarToRow (2)
+               :        +- Scan parquet default.item (1)
+               +- BroadcastExchange (14)
+                  +- * Project (13)
+                     +- * Filter (12)
+                        +- * HashAggregate (11)
+                           +- Exchange (10)
+                              +- * HashAggregate (9)
+                                 +- * Project (8)
+                                    +- * Filter (7)
+                                       +- * ColumnarToRow (6)
+                                          +- Scan parquet default.item (5)
+
+
+(1) Scan parquet default.item
+Output [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manufact_id), GreaterThanOrEqual(i_manufact_id,738), LessThanOrEqual(i_manufact_id,778), IsNotNull(i_manufact)]
+ReadSchema: struct<i_manufact_id:int,i_manufact:string,i_product_name:string>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3]
+Condition : (((isnotnull(i_manufact_id#1) AND (i_manufact_id#1 >= 738)) AND (i_manufact_id#1 <= 778)) AND isnotnull(i_manufact#2))
+
+(4) Project [codegen id : 3]
+Output [2]: [i_manufact#2, i_product_name#3]
+Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3]
+
+(5) Scan parquet default.item
+Output [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [Or(Or(And(EqualTo(i_category,Women                                             ),Or(And(And(Or(EqualTo(i_color,powder              ),EqualTo(i_color,khaki               )),Or(EqualTo(i_units,Ounce     ),EqualTo(i_units,Oz        ))),Or(EqualTo(i_size,medium              ),EqualTo(i_size,extra large         ))),And(And(Or(EqualTo(i_color,brown               ),EqualTo(i_color,honeydew            )),Or(EqualTo(i_units,Bunch     ),EqualTo(i_units,Ton       ))),Or(EqualTo(i_size,N/A                 ),EqualTo(i_size,small               ))))),And(EqualTo(i_category,Men                                               ),Or(And(And(Or(EqualTo(i_color,floral              ),EqualTo(i_color,deep                )),Or(EqualTo(i_units,N/A       ),EqualTo(i_units,Dozen     ))),Or(EqualTo(i_size,petite              ),EqualTo(i_size,large               ))),And(And(Or(EqualTo(i_color,light               ),EqualTo(i_color,cornflower          )),Or(EqualTo(i_units,Box       ),EqualTo(i_units,Pound     ))),Or(EqualTo(i_size,medium              ),EqualTo(i_size,extra large         )))))),Or(And(EqualTo(i_category,Women                                             ),Or(And(And(Or(EqualTo(i_color,midnight            ),EqualTo(i_color,snow                )),Or(EqualTo(i_units,Pallet    ),EqualTo(i_units,Gross     ))),Or(EqualTo(i_size,medium              ),EqualTo(i_size,extra large         ))),And(And(Or(EqualTo(i_color,cyan                ),EqualTo(i_color,papaya              )),Or(EqualTo(i_units,Cup       ),EqualTo(i_units,Dram      ))),Or(EqualTo(i_size,N/A                 ),EqualTo(i_size,small               ))))),And(EqualTo(i_category,Men                                               ),Or(And(And(Or(EqualTo(i_color,orange              ),EqualTo(i_color,frosted             )),Or(EqualTo(i_units,Each      ),EqualTo(i_units,Tbl       ))),Or(EqualTo(i_size,petite              ),EqualTo(i_size,large               ))),And(And(Or(EqualTo(i_color,forest              ),EqualTo(i_color,ghost               )),Or(EqualTo(i_units,Lb        ),EqualTo(i_units,Bundle    ))),Or(EqualTo(i_size,medium              ),EqualTo(i_size,extra large         ))))))), IsNotNull(i_manufact)]
+ReadSchema: struct<i_category:string,i_manufact:string,i_size:string,i_color:string,i_units:string>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8]
+
+(7) Filter [codegen id : 1]
+Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8]
+Condition : (((((i_category#4 = Women                                             ) AND (((((i_color#7 = powder              ) OR (i_color#7 = khaki               )) AND ((i_units#8 = Ounce     ) OR (i_units#8 = Oz        ))) AND ((i_size#6 = medium              ) OR (i_size#6 = extra large         ))) OR ((((i_color#7 = brown               ) OR (i_color#7 = honeydew            )) AND ((i_units#8 = Bunch     ) OR (i_units#8 = Ton       ))) AND ((i_size#6 = N/A                 ) OR (i_size#6 = small               ))))) OR ((i_category#4 = Men                                               ) AND (((((i_color#7 = floral              ) OR (i_color#7 = deep                )) AND ((i_units#8 = N/A       ) OR (i_units#8 = Dozen     ))) AND ((i_size#6 = petite              ) OR (i_size#6 = large               ))) OR ((((i_color#7 = light               ) OR (i_color#7 = cornflower          )) AND ((i_units#8 = Box       ) OR (i_units#8 = Pound     ))) AND ((i_size#6 = medium              ) OR (i_size#6 = extra large         )))))) OR (((i_category#4 = Women                                             ) AND (((((i_color#7 = midnight            ) OR (i_color#7 = snow                )) AND ((i_units#8 = Pallet    ) OR (i_units#8 = Gross     ))) AND ((i_size#6 = medium              ) OR (i_size#6 = extra large         ))) OR ((((i_color#7 = cyan                ) OR (i_color#7 = papaya              )) AND ((i_units#8 = Cup       ) OR (i_units#8 = Dram      ))) AND ((i_size#6 = N/A                 ) OR (i_size#6 = small               ))))) OR ((i_category#4 = Men                                               ) AND (((((i_color#7 = orange              ) OR (i_color#7 = frosted             )) AND ((i_units#8 = Each      ) OR (i_units#8 = Tbl       ))) AND ((i_size#6 = petite              ) OR (i_size#6 = large               ))) OR ((((i_color#7 = forest              ) OR (i_color#7 = ghost               )) AND ((i_units#8 = Lb        ) OR (i_units#8 = Bundle    ))) AND ((i_size#6 = medium              ) OR (i_size#6 = extra large         ))))))) AND isnotnull(i_manufact#5))
+
+(8) Project [codegen id : 1]
+Output [1]: [i_manufact#5]
+Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8]
+
+(9) HashAggregate [codegen id : 1]
+Input [1]: [i_manufact#5]
+Keys [1]: [i_manufact#5]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#9]
+Results [2]: [i_manufact#5, count#10]
+
+(10) Exchange
+Input [2]: [i_manufact#5, count#10]
+Arguments: hashpartitioning(i_manufact#5, 5), ENSURE_REQUIREMENTS, [id=#11]
+
+(11) HashAggregate [codegen id : 2]
+Input [2]: [i_manufact#5, count#10]
+Keys [1]: [i_manufact#5]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#12]
+Results [2]: [count(1)#12 AS item_cnt#13, i_manufact#5]
+
+(12) Filter [codegen id : 2]
+Input [2]: [item_cnt#13, i_manufact#5]
+Condition : (item_cnt#13 > 0)
+
+(13) Project [codegen id : 2]
+Output [1]: [i_manufact#5]
+Input [2]: [item_cnt#13, i_manufact#5]
+
+(14) BroadcastExchange
+Input [1]: [i_manufact#5]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#14]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [i_manufact#2]
+Right keys [1]: [i_manufact#5]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [1]: [i_product_name#3]
+Input [3]: [i_manufact#2, i_product_name#3, i_manufact#5]
+
+(17) HashAggregate [codegen id : 3]
+Input [1]: [i_product_name#3]
+Keys [1]: [i_product_name#3]
+Functions: []
+Aggregate Attributes: []
+Results [1]: [i_product_name#3]
+
+(18) Exchange
+Input [1]: [i_product_name#3]
+Arguments: hashpartitioning(i_product_name#3, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(19) HashAggregate [codegen id : 4]
+Input [1]: [i_product_name#3]
+Keys [1]: [i_product_name#3]
+Functions: []
+Aggregate Attributes: []
+Results [1]: [i_product_name#3]
+
+(20) TakeOrderedAndProject
+Input [1]: [i_product_name#3]
+Arguments: 100, [i_product_name#3 ASC NULLS FIRST], [i_product_name#3]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.sf100/simplified.txt
new file mode 100644
index 0000000000000..d36800823bb3f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.sf100/simplified.txt
@@ -0,0 +1,29 @@
+TakeOrderedAndProject [i_product_name]
+  WholeStageCodegen (4)
+    HashAggregate [i_product_name]
+      InputAdapter
+        Exchange [i_product_name] #1
+          WholeStageCodegen (3)
+            HashAggregate [i_product_name]
+              Project [i_product_name]
+                BroadcastHashJoin [i_manufact,i_manufact]
+                  Project [i_manufact,i_product_name]
+                    Filter [i_manufact_id,i_manufact]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.item [i_manufact_id,i_manufact,i_product_name]
+                  InputAdapter
+                    BroadcastExchange #2
+                      WholeStageCodegen (2)
+                        Project [i_manufact]
+                          Filter [item_cnt]
+                            HashAggregate [i_manufact,count] [count(1),item_cnt,count]
+                              InputAdapter
+                                Exchange [i_manufact] #3
+                                  WholeStageCodegen (1)
+                                    HashAggregate [i_manufact] [count,count]
+                                      Project [i_manufact]
+                                        Filter [i_category,i_color,i_units,i_size,i_manufact]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.item [i_category,i_manufact,i_size,i_color,i_units]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/explain.txt
new file mode 100644
index 0000000000000..33c03d2b767dd
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/explain.txt
@@ -0,0 +1,120 @@
+== Physical Plan ==
+TakeOrderedAndProject (20)
++- * HashAggregate (19)
+   +- Exchange (18)
+      +- * HashAggregate (17)
+         +- * Project (16)
+            +- * BroadcastHashJoin Inner BuildRight (15)
+               :- * Project (4)
+               :  +- * Filter (3)
+               :     +- * ColumnarToRow (2)
+               :        +- Scan parquet default.item (1)
+               +- BroadcastExchange (14)
+                  +- * Project (13)
+                     +- * Filter (12)
+                        +- * HashAggregate (11)
+                           +- Exchange (10)
+                              +- * HashAggregate (9)
+                                 +- * Project (8)
+                                    +- * Filter (7)
+                                       +- * ColumnarToRow (6)
+                                          +- Scan parquet default.item (5)
+
+
+(1) Scan parquet default.item
+Output [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manufact_id), GreaterThanOrEqual(i_manufact_id,738), LessThanOrEqual(i_manufact_id,778), IsNotNull(i_manufact)]
+ReadSchema: struct<i_manufact_id:int,i_manufact:string,i_product_name:string>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3]
+Condition : (((isnotnull(i_manufact_id#1) AND (i_manufact_id#1 >= 738)) AND (i_manufact_id#1 <= 778)) AND isnotnull(i_manufact#2))
+
+(4) Project [codegen id : 3]
+Output [2]: [i_manufact#2, i_product_name#3]
+Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3]
+
+(5) Scan parquet default.item
+Output [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [Or(Or(And(EqualTo(i_category,Women                                             ),Or(And(And(Or(EqualTo(i_color,powder              ),EqualTo(i_color,khaki               )),Or(EqualTo(i_units,Ounce     ),EqualTo(i_units,Oz        ))),Or(EqualTo(i_size,medium              ),EqualTo(i_size,extra large         ))),And(And(Or(EqualTo(i_color,brown               ),EqualTo(i_color,honeydew            )),Or(EqualTo(i_units,Bunch     ),EqualTo(i_units,Ton       ))),Or(EqualTo(i_size,N/A                 ),EqualTo(i_size,small               ))))),And(EqualTo(i_category,Men                                               ),Or(And(And(Or(EqualTo(i_color,floral              ),EqualTo(i_color,deep                )),Or(EqualTo(i_units,N/A       ),EqualTo(i_units,Dozen     ))),Or(EqualTo(i_size,petite              ),EqualTo(i_size,large               ))),And(And(Or(EqualTo(i_color,light               ),EqualTo(i_color,cornflower          )),Or(EqualTo(i_units,Box       ),EqualTo(i_units,Pound     ))),Or(EqualTo(i_size,medium              ),EqualTo(i_size,extra large         )))))),Or(And(EqualTo(i_category,Women                                             ),Or(And(And(Or(EqualTo(i_color,midnight            ),EqualTo(i_color,snow                )),Or(EqualTo(i_units,Pallet    ),EqualTo(i_units,Gross     ))),Or(EqualTo(i_size,medium              ),EqualTo(i_size,extra large         ))),And(And(Or(EqualTo(i_color,cyan                ),EqualTo(i_color,papaya              )),Or(EqualTo(i_units,Cup       ),EqualTo(i_units,Dram      ))),Or(EqualTo(i_size,N/A                 ),EqualTo(i_size,small               ))))),And(EqualTo(i_category,Men                                               ),Or(And(And(Or(EqualTo(i_color,orange              ),EqualTo(i_color,frosted             )),Or(EqualTo(i_units,Each      ),EqualTo(i_units,Tbl       ))),Or(EqualTo(i_size,petite              ),EqualTo(i_size,large               ))),And(And(Or(EqualTo(i_color,forest              ),EqualTo(i_color,ghost               )),Or(EqualTo(i_units,Lb        ),EqualTo(i_units,Bundle    ))),Or(EqualTo(i_size,medium              ),EqualTo(i_size,extra large         ))))))), IsNotNull(i_manufact)]
+ReadSchema: struct<i_category:string,i_manufact:string,i_size:string,i_color:string,i_units:string>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8]
+
+(7) Filter [codegen id : 1]
+Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8]
+Condition : (((((i_category#4 = Women                                             ) AND (((((i_color#7 = powder              ) OR (i_color#7 = khaki               )) AND ((i_units#8 = Ounce     ) OR (i_units#8 = Oz        ))) AND ((i_size#6 = medium              ) OR (i_size#6 = extra large         ))) OR ((((i_color#7 = brown               ) OR (i_color#7 = honeydew            )) AND ((i_units#8 = Bunch     ) OR (i_units#8 = Ton       ))) AND ((i_size#6 = N/A                 ) OR (i_size#6 = small               ))))) OR ((i_category#4 = Men                                               ) AND (((((i_color#7 = floral              ) OR (i_color#7 = deep                )) AND ((i_units#8 = N/A       ) OR (i_units#8 = Dozen     ))) AND ((i_size#6 = petite              ) OR (i_size#6 = large               ))) OR ((((i_color#7 = light               ) OR (i_color#7 = cornflower          )) AND ((i_units#8 = Box       ) OR (i_units#8 = Pound     ))) AND ((i_size#6 = medium              ) OR (i_size#6 = extra large         )))))) OR (((i_category#4 = Women                                             ) AND (((((i_color#7 = midnight            ) OR (i_color#7 = snow                )) AND ((i_units#8 = Pallet    ) OR (i_units#8 = Gross     ))) AND ((i_size#6 = medium              ) OR (i_size#6 = extra large         ))) OR ((((i_color#7 = cyan                ) OR (i_color#7 = papaya              )) AND ((i_units#8 = Cup       ) OR (i_units#8 = Dram      ))) AND ((i_size#6 = N/A                 ) OR (i_size#6 = small               ))))) OR ((i_category#4 = Men                                               ) AND (((((i_color#7 = orange              ) OR (i_color#7 = frosted             )) AND ((i_units#8 = Each      ) OR (i_units#8 = Tbl       ))) AND ((i_size#6 = petite              ) OR (i_size#6 = large               ))) OR ((((i_color#7 = forest              ) OR (i_color#7 = ghost               )) AND ((i_units#8 = Lb        ) OR (i_units#8 = Bundle    ))) AND ((i_size#6 = medium              ) OR (i_size#6 = extra large         ))))))) AND isnotnull(i_manufact#5))
+
+(8) Project [codegen id : 1]
+Output [1]: [i_manufact#5]
+Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8]
+
+(9) HashAggregate [codegen id : 1]
+Input [1]: [i_manufact#5]
+Keys [1]: [i_manufact#5]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#9]
+Results [2]: [i_manufact#5, count#10]
+
+(10) Exchange
+Input [2]: [i_manufact#5, count#10]
+Arguments: hashpartitioning(i_manufact#5, 5), ENSURE_REQUIREMENTS, [id=#11]
+
+(11) HashAggregate [codegen id : 2]
+Input [2]: [i_manufact#5, count#10]
+Keys [1]: [i_manufact#5]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#12]
+Results [2]: [count(1)#12 AS item_cnt#13, i_manufact#5]
+
+(12) Filter [codegen id : 2]
+Input [2]: [item_cnt#13, i_manufact#5]
+Condition : (item_cnt#13 > 0)
+
+(13) Project [codegen id : 2]
+Output [1]: [i_manufact#5]
+Input [2]: [item_cnt#13, i_manufact#5]
+
+(14) BroadcastExchange
+Input [1]: [i_manufact#5]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#14]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [i_manufact#2]
+Right keys [1]: [i_manufact#5]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [1]: [i_product_name#3]
+Input [3]: [i_manufact#2, i_product_name#3, i_manufact#5]
+
+(17) HashAggregate [codegen id : 3]
+Input [1]: [i_product_name#3]
+Keys [1]: [i_product_name#3]
+Functions: []
+Aggregate Attributes: []
+Results [1]: [i_product_name#3]
+
+(18) Exchange
+Input [1]: [i_product_name#3]
+Arguments: hashpartitioning(i_product_name#3, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(19) HashAggregate [codegen id : 4]
+Input [1]: [i_product_name#3]
+Keys [1]: [i_product_name#3]
+Functions: []
+Aggregate Attributes: []
+Results [1]: [i_product_name#3]
+
+(20) TakeOrderedAndProject
+Input [1]: [i_product_name#3]
+Arguments: 100, [i_product_name#3 ASC NULLS FIRST], [i_product_name#3]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/simplified.txt
new file mode 100644
index 0000000000000..d36800823bb3f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/simplified.txt
@@ -0,0 +1,29 @@
+TakeOrderedAndProject [i_product_name]
+  WholeStageCodegen (4)
+    HashAggregate [i_product_name]
+      InputAdapter
+        Exchange [i_product_name] #1
+          WholeStageCodegen (3)
+            HashAggregate [i_product_name]
+              Project [i_product_name]
+                BroadcastHashJoin [i_manufact,i_manufact]
+                  Project [i_manufact,i_product_name]
+                    Filter [i_manufact_id,i_manufact]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.item [i_manufact_id,i_manufact,i_product_name]
+                  InputAdapter
+                    BroadcastExchange #2
+                      WholeStageCodegen (2)
+                        Project [i_manufact]
+                          Filter [item_cnt]
+                            HashAggregate [i_manufact,count] [count(1),item_cnt,count]
+                              InputAdapter
+                                Exchange [i_manufact] #3
+                                  WholeStageCodegen (1)
+                                    HashAggregate [i_manufact] [count,count]
+                                      Project [i_manufact]
+                                        Filter [i_category,i_color,i_units,i_size,i_manufact]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.item [i_category,i_manufact,i_size,i_color,i_units]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.sf100/explain.txt
new file mode 100644
index 0000000000000..30a7a786e42ca
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.sf100/explain.txt
@@ -0,0 +1,133 @@
+== Physical Plan ==
+TakeOrderedAndProject (21)
++- * HashAggregate (20)
+   +- Exchange (19)
+      +- * HashAggregate (18)
+         +- * Project (17)
+            +- * BroadcastHashJoin Inner BuildRight (16)
+               :- * Project (10)
+               :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :- * Filter (3)
+               :     :  +- * ColumnarToRow (2)
+               :     :     +- Scan parquet default.store_sales (1)
+               :     +- BroadcastExchange (8)
+               :        +- * Project (7)
+               :           +- * Filter (6)
+               :              +- * ColumnarToRow (5)
+               :                 +- Scan parquet default.item (4)
+               +- BroadcastExchange (15)
+                  +- * Project (14)
+                     +- * Filter (13)
+                        +- * ColumnarToRow (12)
+                           +- Scan parquet default.date_dim (11)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_item_sk#1)
+
+(4) Scan parquet default.item
+Output [4]: [i_item_sk#5, i_category_id#6, i_category#7, i_manager_id#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_category_id:int,i_category:string,i_manager_id:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [i_item_sk#5, i_category_id#6, i_category#7, i_manager_id#8]
+
+(6) Filter [codegen id : 1]
+Input [4]: [i_item_sk#5, i_category_id#6, i_category#7, i_manager_id#8]
+Condition : ((isnotnull(i_manager_id#8) AND (i_manager_id#8 = 1)) AND isnotnull(i_item_sk#5))
+
+(7) Project [codegen id : 1]
+Output [3]: [i_item_sk#5, i_category_id#6, i_category#7]
+Input [4]: [i_item_sk#5, i_category_id#6, i_category#7, i_manager_id#8]
+
+(8) BroadcastExchange
+Input [3]: [i_item_sk#5, i_category_id#6, i_category#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#9]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#5]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [4]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_category_id#6, i_category#7]
+Input [6]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_category_id#6, i_category#7]
+
+(11) Scan parquet default.date_dim
+Output [3]: [d_date_sk#10, d_year#11, d_moy#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+
+(13) Filter [codegen id : 2]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+Condition : ((((isnotnull(d_moy#12) AND isnotnull(d_year#11)) AND (d_moy#12 = 11)) AND (d_year#11 = 2000)) AND isnotnull(d_date_sk#10))
+
+(14) Project [codegen id : 2]
+Output [2]: [d_date_sk#10, d_year#11]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+
+(15) BroadcastExchange
+Input [2]: [d_date_sk#10, d_year#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(16) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(17) Project [codegen id : 3]
+Output [4]: [d_year#11, ss_ext_sales_price#2, i_category_id#6, i_category#7]
+Input [6]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_category_id#6, i_category#7, d_date_sk#10, d_year#11]
+
+(18) HashAggregate [codegen id : 3]
+Input [4]: [d_year#11, ss_ext_sales_price#2, i_category_id#6, i_category#7]
+Keys [3]: [d_year#11, i_category_id#6, i_category#7]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum#14]
+Results [4]: [d_year#11, i_category_id#6, i_category#7, sum#15]
+
+(19) Exchange
+Input [4]: [d_year#11, i_category_id#6, i_category#7, sum#15]
+Arguments: hashpartitioning(d_year#11, i_category_id#6, i_category#7, 5), ENSURE_REQUIREMENTS, [id=#16]
+
+(20) HashAggregate [codegen id : 4]
+Input [4]: [d_year#11, i_category_id#6, i_category#7, sum#15]
+Keys [3]: [d_year#11, i_category_id#6, i_category#7]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#17]
+Results [4]: [d_year#11, i_category_id#6, i_category#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#17,17,2) AS sum(ss_ext_sales_price)#18]
+
+(21) TakeOrderedAndProject
+Input [4]: [d_year#11, i_category_id#6, i_category#7, sum(ss_ext_sales_price)#18]
+Arguments: 100, [sum(ss_ext_sales_price)#18 DESC NULLS LAST, d_year#11 ASC NULLS FIRST, i_category_id#6 ASC NULLS FIRST, i_category#7 ASC NULLS FIRST], [d_year#11, i_category_id#6, i_category#7, sum(ss_ext_sales_price)#18]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (22)
+
+
+(22) ReusedExchange [Reuses operator id: 15]
+Output [2]: [d_date_sk#10, d_year#11]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.sf100/simplified.txt
new file mode 100644
index 0000000000000..f1c1be7e7e41f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.sf100/simplified.txt
@@ -0,0 +1,33 @@
+TakeOrderedAndProject [sum(ss_ext_sales_price),d_year,i_category_id,i_category]
+  WholeStageCodegen (4)
+    HashAggregate [d_year,i_category_id,i_category,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(ss_ext_sales_price),sum]
+      InputAdapter
+        Exchange [d_year,i_category_id,i_category] #1
+          WholeStageCodegen (3)
+            HashAggregate [d_year,i_category_id,i_category,ss_ext_sales_price] [sum,sum]
+              Project [d_year,ss_ext_sales_price,i_category_id,i_category]
+                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                  Project [ss_ext_sales_price,ss_sold_date_sk,i_category_id,i_category]
+                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                      Filter [ss_item_sk]
+                        ColumnarToRow
+                          InputAdapter
+                            Scan parquet default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk]
+                              SubqueryBroadcast [d_date_sk] #1
+                                ReusedExchange [d_date_sk,d_year] #2
+                      InputAdapter
+                        BroadcastExchange #3
+                          WholeStageCodegen (1)
+                            Project [i_item_sk,i_category_id,i_category]
+                              Filter [i_manager_id,i_item_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.item [i_item_sk,i_category_id,i_category,i_manager_id]
+                  InputAdapter
+                    BroadcastExchange #2
+                      WholeStageCodegen (2)
+                        Project [d_date_sk,d_year]
+                          Filter [d_moy,d_year,d_date_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/explain.txt
new file mode 100644
index 0000000000000..d16b4bca7c3f4
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/explain.txt
@@ -0,0 +1,123 @@
+== Physical Plan ==
+TakeOrderedAndProject (21)
++- * HashAggregate (20)
+   +- Exchange (19)
+      +- * HashAggregate (18)
+         +- * Project (17)
+            +- * BroadcastHashJoin Inner BuildRight (16)
+               :- * Project (10)
+               :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :- * Project (4)
+               :     :  +- * Filter (3)
+               :     :     +- * ColumnarToRow (2)
+               :     :        +- Scan parquet default.date_dim (1)
+               :     +- BroadcastExchange (8)
+               :        +- * Filter (7)
+               :           +- * ColumnarToRow (6)
+               :              +- Scan parquet default.store_sales (5)
+               +- BroadcastExchange (15)
+                  +- * Project (14)
+                     +- * Filter (13)
+                        +- * ColumnarToRow (12)
+                           +- Scan parquet default.item (11)
+
+
+(1) Scan parquet default.date_dim
+Output [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk#1))
+
+(4) Project [codegen id : 3]
+Output [2]: [d_date_sk#1, d_year#2]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(5) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+
+(7) Filter [codegen id : 1]
+Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Condition : isnotnull(ss_item_sk#4)
+
+(8) BroadcastExchange
+Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [d_date_sk#1]
+Right keys [1]: [ss_sold_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5]
+Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+
+(11) Scan parquet default.item
+Output [4]: [i_item_sk#8, i_category_id#9, i_category#10, i_manager_id#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_category_id:int,i_category:string,i_manager_id:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [4]: [i_item_sk#8, i_category_id#9, i_category#10, i_manager_id#11]
+
+(13) Filter [codegen id : 2]
+Input [4]: [i_item_sk#8, i_category_id#9, i_category#10, i_manager_id#11]
+Condition : ((isnotnull(i_manager_id#11) AND (i_manager_id#11 = 1)) AND isnotnull(i_item_sk#8))
+
+(14) Project [codegen id : 2]
+Output [3]: [i_item_sk#8, i_category_id#9, i_category#10]
+Input [4]: [i_item_sk#8, i_category_id#9, i_category#10, i_manager_id#11]
+
+(15) BroadcastExchange
+Input [3]: [i_item_sk#8, i_category_id#9, i_category#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(16) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_item_sk#4]
+Right keys [1]: [i_item_sk#8]
+Join condition: None
+
+(17) Project [codegen id : 3]
+Output [4]: [d_year#2, ss_ext_sales_price#5, i_category_id#9, i_category#10]
+Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#8, i_category_id#9, i_category#10]
+
+(18) HashAggregate [codegen id : 3]
+Input [4]: [d_year#2, ss_ext_sales_price#5, i_category_id#9, i_category#10]
+Keys [3]: [d_year#2, i_category_id#9, i_category#10]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))]
+Aggregate Attributes [1]: [sum#13]
+Results [4]: [d_year#2, i_category_id#9, i_category#10, sum#14]
+
+(19) Exchange
+Input [4]: [d_year#2, i_category_id#9, i_category#10, sum#14]
+Arguments: hashpartitioning(d_year#2, i_category_id#9, i_category#10, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(20) HashAggregate [codegen id : 4]
+Input [4]: [d_year#2, i_category_id#9, i_category#10, sum#14]
+Keys [3]: [d_year#2, i_category_id#9, i_category#10]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#16]
+Results [4]: [d_year#2, i_category_id#9, i_category#10, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#16,17,2) AS sum(ss_ext_sales_price)#17]
+
+(21) TakeOrderedAndProject
+Input [4]: [d_year#2, i_category_id#9, i_category#10, sum(ss_ext_sales_price)#17]
+Arguments: 100, [sum(ss_ext_sales_price)#17 DESC NULLS LAST, d_year#2 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_category#10 ASC NULLS FIRST], [d_year#2, i_category_id#9, i_category#10, sum(ss_ext_sales_price)#17]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/simplified.txt
new file mode 100644
index 0000000000000..4806a9309bd90
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/simplified.txt
@@ -0,0 +1,31 @@
+TakeOrderedAndProject [sum(ss_ext_sales_price),d_year,i_category_id,i_category]
+  WholeStageCodegen (4)
+    HashAggregate [d_year,i_category_id,i_category,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(ss_ext_sales_price),sum]
+      InputAdapter
+        Exchange [d_year,i_category_id,i_category] #1
+          WholeStageCodegen (3)
+            HashAggregate [d_year,i_category_id,i_category,ss_ext_sales_price] [sum,sum]
+              Project [d_year,ss_ext_sales_price,i_category_id,i_category]
+                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                  Project [d_year,ss_item_sk,ss_ext_sales_price]
+                    BroadcastHashJoin [d_date_sk,ss_sold_date_sk]
+                      Project [d_date_sk,d_year]
+                        Filter [d_moy,d_year,d_date_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                      InputAdapter
+                        BroadcastExchange #2
+                          WholeStageCodegen (1)
+                            Filter [ss_item_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk]
+                  InputAdapter
+                    BroadcastExchange #3
+                      WholeStageCodegen (2)
+                        Project [i_item_sk,i_category_id,i_category]
+                          Filter [i_manager_id,i_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.item [i_item_sk,i_category_id,i_category,i_manager_id]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.sf100/explain.txt
new file mode 100644
index 0000000000000..d626333970af9
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.sf100/explain.txt
@@ -0,0 +1,133 @@
+== Physical Plan ==
+TakeOrderedAndProject (21)
++- * HashAggregate (20)
+   +- Exchange (19)
+      +- * HashAggregate (18)
+         +- * Project (17)
+            +- * BroadcastHashJoin Inner BuildRight (16)
+               :- * Project (10)
+               :  +- * BroadcastHashJoin Inner BuildLeft (9)
+               :     :- BroadcastExchange (5)
+               :     :  +- * Project (4)
+               :     :     +- * Filter (3)
+               :     :        +- * ColumnarToRow (2)
+               :     :           +- Scan parquet default.date_dim (1)
+               :     +- * Filter (8)
+               :        +- * ColumnarToRow (7)
+               :           +- Scan parquet default.store_sales (6)
+               +- BroadcastExchange (15)
+                  +- * Project (14)
+                     +- * Filter (13)
+                        +- * ColumnarToRow (12)
+                           +- Scan parquet default.store (11)
+
+
+(1) Scan parquet default.date_dim
+Output [3]: [d_date_sk#1, d_year#2, d_day_name#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_day_name:string>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#1, d_year#2, d_day_name#3]
+
+(3) Filter [codegen id : 1]
+Input [3]: [d_date_sk#1, d_year#2, d_day_name#3]
+Condition : ((isnotnull(d_year#2) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk#1))
+
+(4) Project [codegen id : 1]
+Output [2]: [d_date_sk#1, d_day_name#3]
+Input [3]: [d_date_sk#1, d_year#2, d_day_name#3]
+
+(5) BroadcastExchange
+Input [2]: [d_date_sk#1, d_day_name#3]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#4]
+
+(6) Scan parquet default.store_sales
+Output [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(7) ColumnarToRow
+Input [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7]
+
+(8) Filter
+Input [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7]
+Condition : isnotnull(ss_store_sk#5)
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [d_date_sk#1]
+Right keys [1]: [ss_sold_date_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [3]: [d_day_name#3, ss_store_sk#5, ss_sales_price#6]
+Input [5]: [d_date_sk#1, d_day_name#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7]
+
+(11) Scan parquet default.store
+Output [4]: [s_store_sk#9, s_store_id#10, s_store_name#11, s_gmt_offset#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_gmt_offset), EqualTo(s_gmt_offset,-5.00), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string,s_store_name:string,s_gmt_offset:decimal(5,2)>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [4]: [s_store_sk#9, s_store_id#10, s_store_name#11, s_gmt_offset#12]
+
+(13) Filter [codegen id : 2]
+Input [4]: [s_store_sk#9, s_store_id#10, s_store_name#11, s_gmt_offset#12]
+Condition : ((isnotnull(s_gmt_offset#12) AND (s_gmt_offset#12 = -5.00)) AND isnotnull(s_store_sk#9))
+
+(14) Project [codegen id : 2]
+Output [3]: [s_store_sk#9, s_store_id#10, s_store_name#11]
+Input [4]: [s_store_sk#9, s_store_id#10, s_store_name#11, s_gmt_offset#12]
+
+(15) BroadcastExchange
+Input [3]: [s_store_sk#9, s_store_id#10, s_store_name#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(16) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_store_sk#5]
+Right keys [1]: [s_store_sk#9]
+Join condition: None
+
+(17) Project [codegen id : 3]
+Output [4]: [d_day_name#3, ss_sales_price#6, s_store_id#10, s_store_name#11]
+Input [6]: [d_day_name#3, ss_store_sk#5, ss_sales_price#6, s_store_sk#9, s_store_id#10, s_store_name#11]
+
+(18) HashAggregate [codegen id : 3]
+Input [4]: [d_day_name#3, ss_sales_price#6, s_store_id#10, s_store_name#11]
+Keys [2]: [s_store_name#11, s_store_id#10]
+Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday   ) THEN ss_sales_price#6 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday   ) THEN ss_sales_price#6 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday  ) THEN ss_sales_price#6 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#6 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#6 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday   ) THEN ss_sales_price#6 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#6 ELSE null END))]
+Aggregate Attributes [7]: [sum#14, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20]
+Results [9]: [s_store_name#11, s_store_id#10, sum#21, sum#22, sum#23, sum#24, sum#25, sum#26, sum#27]
+
+(19) Exchange
+Input [9]: [s_store_name#11, s_store_id#10, sum#21, sum#22, sum#23, sum#24, sum#25, sum#26, sum#27]
+Arguments: hashpartitioning(s_store_name#11, s_store_id#10, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(20) HashAggregate [codegen id : 4]
+Input [9]: [s_store_name#11, s_store_id#10, sum#21, sum#22, sum#23, sum#24, sum#25, sum#26, sum#27]
+Keys [2]: [s_store_name#11, s_store_id#10]
+Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday   ) THEN ss_sales_price#6 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday   ) THEN ss_sales_price#6 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday  ) THEN ss_sales_price#6 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#6 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#6 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday   ) THEN ss_sales_price#6 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#6 ELSE null END))]
+Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday   ) THEN ss_sales_price#6 ELSE null END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday   ) THEN ss_sales_price#6 ELSE null END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday  ) THEN ss_sales_price#6 ELSE null END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#6 ELSE null END))#32, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#6 ELSE null END))#33, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday   ) THEN ss_sales_price#6 ELSE null END))#34, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#6 ELSE null END))#35]
+Results [9]: [s_store_name#11, s_store_id#10, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday   ) THEN ss_sales_price#6 ELSE null END))#29,17,2) AS sun_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday   ) THEN ss_sales_price#6 ELSE null END))#30,17,2) AS mon_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday  ) THEN ss_sales_price#6 ELSE null END))#31,17,2) AS tue_sales#38, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#6 ELSE null END))#32,17,2) AS wed_sales#39, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#6 ELSE null END))#33,17,2) AS thu_sales#40, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday   ) THEN ss_sales_price#6 ELSE null END))#34,17,2) AS fri_sales#41, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#6 ELSE null END))#35,17,2) AS sat_sales#42]
+
+(21) TakeOrderedAndProject
+Input [9]: [s_store_name#11, s_store_id#10, sun_sales#36, mon_sales#37, tue_sales#38, wed_sales#39, thu_sales#40, fri_sales#41, sat_sales#42]
+Arguments: 100, [s_store_name#11 ASC NULLS FIRST, s_store_id#10 ASC NULLS FIRST, sun_sales#36 ASC NULLS FIRST, mon_sales#37 ASC NULLS FIRST, tue_sales#38 ASC NULLS FIRST, wed_sales#39 ASC NULLS FIRST, thu_sales#40 ASC NULLS FIRST, fri_sales#41 ASC NULLS FIRST, sat_sales#42 ASC NULLS FIRST], [s_store_name#11, s_store_id#10, sun_sales#36, mon_sales#37, tue_sales#38, wed_sales#39, thu_sales#40, fri_sales#41, sat_sales#42]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 6 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (22)
+
+
+(22) ReusedExchange [Reuses operator id: 5]
+Output [2]: [d_date_sk#1, d_day_name#3]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.sf100/simplified.txt
new file mode 100644
index 0000000000000..1fbb5aa612fa9
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.sf100/simplified.txt
@@ -0,0 +1,33 @@
+TakeOrderedAndProject [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales]
+  WholeStageCodegen (4)
+    HashAggregate [s_store_name,s_store_id,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday  ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price ELSE null END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum]
+      InputAdapter
+        Exchange [s_store_name,s_store_id] #1
+          WholeStageCodegen (3)
+            HashAggregate [s_store_name,s_store_id,d_day_name,ss_sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum]
+              Project [d_day_name,ss_sales_price,s_store_id,s_store_name]
+                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                  Project [d_day_name,ss_store_sk,ss_sales_price]
+                    BroadcastHashJoin [d_date_sk,ss_sold_date_sk]
+                      InputAdapter
+                        BroadcastExchange #2
+                          WholeStageCodegen (1)
+                            Project [d_date_sk,d_day_name]
+                              Filter [d_year,d_date_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.date_dim [d_date_sk,d_year,d_day_name]
+                      Filter [ss_store_sk]
+                        ColumnarToRow
+                          InputAdapter
+                            Scan parquet default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                              SubqueryBroadcast [d_date_sk] #1
+                                ReusedExchange [d_date_sk,d_day_name] #2
+                  InputAdapter
+                    BroadcastExchange #3
+                      WholeStageCodegen (2)
+                        Project [s_store_sk,s_store_id,s_store_name]
+                          Filter [s_gmt_offset,s_store_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/explain.txt
new file mode 100644
index 0000000000000..24af5d2e813cb
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/explain.txt
@@ -0,0 +1,123 @@
+== Physical Plan ==
+TakeOrderedAndProject (21)
++- * HashAggregate (20)
+   +- Exchange (19)
+      +- * HashAggregate (18)
+         +- * Project (17)
+            +- * BroadcastHashJoin Inner BuildRight (16)
+               :- * Project (10)
+               :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :- * Project (4)
+               :     :  +- * Filter (3)
+               :     :     +- * ColumnarToRow (2)
+               :     :        +- Scan parquet default.date_dim (1)
+               :     +- BroadcastExchange (8)
+               :        +- * Filter (7)
+               :           +- * ColumnarToRow (6)
+               :              +- Scan parquet default.store_sales (5)
+               +- BroadcastExchange (15)
+                  +- * Project (14)
+                     +- * Filter (13)
+                        +- * ColumnarToRow (12)
+                           +- Scan parquet default.store (11)
+
+
+(1) Scan parquet default.date_dim
+Output [3]: [d_date_sk#1, d_year#2, d_day_name#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_day_name:string>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#1, d_year#2, d_day_name#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [d_date_sk#1, d_year#2, d_day_name#3]
+Condition : ((isnotnull(d_year#2) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk#1))
+
+(4) Project [codegen id : 3]
+Output [2]: [d_date_sk#1, d_day_name#3]
+Input [3]: [d_date_sk#1, d_year#2, d_day_name#3]
+
+(5) Scan parquet default.store_sales
+Output [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6]
+
+(7) Filter [codegen id : 1]
+Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6]
+Condition : isnotnull(ss_store_sk#4)
+
+(8) BroadcastExchange
+Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [d_date_sk#1]
+Right keys [1]: [ss_sold_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [3]: [d_day_name#3, ss_store_sk#4, ss_sales_price#5]
+Input [5]: [d_date_sk#1, d_day_name#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6]
+
+(11) Scan parquet default.store
+Output [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_gmt_offset), EqualTo(s_gmt_offset,-5.00), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string,s_store_name:string,s_gmt_offset:decimal(5,2)>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11]
+
+(13) Filter [codegen id : 2]
+Input [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11]
+Condition : ((isnotnull(s_gmt_offset#11) AND (s_gmt_offset#11 = -5.00)) AND isnotnull(s_store_sk#8))
+
+(14) Project [codegen id : 2]
+Output [3]: [s_store_sk#8, s_store_id#9, s_store_name#10]
+Input [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11]
+
+(15) BroadcastExchange
+Input [3]: [s_store_sk#8, s_store_id#9, s_store_name#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(16) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_store_sk#4]
+Right keys [1]: [s_store_sk#8]
+Join condition: None
+
+(17) Project [codegen id : 3]
+Output [4]: [d_day_name#3, ss_sales_price#5, s_store_id#9, s_store_name#10]
+Input [6]: [d_day_name#3, ss_store_sk#4, ss_sales_price#5, s_store_sk#8, s_store_id#9, s_store_name#10]
+
+(18) HashAggregate [codegen id : 3]
+Input [4]: [d_day_name#3, ss_sales_price#5, s_store_id#9, s_store_name#10]
+Keys [2]: [s_store_name#10, s_store_id#9]
+Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday   ) THEN ss_sales_price#5 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday   ) THEN ss_sales_price#5 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday  ) THEN ss_sales_price#5 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday   ) THEN ss_sales_price#5 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 ELSE null END))]
+Aggregate Attributes [7]: [sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19]
+Results [9]: [s_store_name#10, s_store_id#9, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25, sum#26]
+
+(19) Exchange
+Input [9]: [s_store_name#10, s_store_id#9, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25, sum#26]
+Arguments: hashpartitioning(s_store_name#10, s_store_id#9, 5), ENSURE_REQUIREMENTS, [id=#27]
+
+(20) HashAggregate [codegen id : 4]
+Input [9]: [s_store_name#10, s_store_id#9, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25, sum#26]
+Keys [2]: [s_store_name#10, s_store_id#9]
+Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday   ) THEN ss_sales_price#5 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday   ) THEN ss_sales_price#5 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday  ) THEN ss_sales_price#5 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday   ) THEN ss_sales_price#5 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 ELSE null END))]
+Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday   ) THEN ss_sales_price#5 ELSE null END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday   ) THEN ss_sales_price#5 ELSE null END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday  ) THEN ss_sales_price#5 ELSE null END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 ELSE null END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 ELSE null END))#32, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday   ) THEN ss_sales_price#5 ELSE null END))#33, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 ELSE null END))#34]
+Results [9]: [s_store_name#10, s_store_id#9, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday   ) THEN ss_sales_price#5 ELSE null END))#28,17,2) AS sun_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday   ) THEN ss_sales_price#5 ELSE null END))#29,17,2) AS mon_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday  ) THEN ss_sales_price#5 ELSE null END))#30,17,2) AS tue_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 ELSE null END))#31,17,2) AS wed_sales#38, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 ELSE null END))#32,17,2) AS thu_sales#39, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday   ) THEN ss_sales_price#5 ELSE null END))#33,17,2) AS fri_sales#40, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 ELSE null END))#34,17,2) AS sat_sales#41]
+
+(21) TakeOrderedAndProject
+Input [9]: [s_store_name#10, s_store_id#9, sun_sales#35, mon_sales#36, tue_sales#37, wed_sales#38, thu_sales#39, fri_sales#40, sat_sales#41]
+Arguments: 100, [s_store_name#10 ASC NULLS FIRST, s_store_id#9 ASC NULLS FIRST, sun_sales#35 ASC NULLS FIRST, mon_sales#36 ASC NULLS FIRST, tue_sales#37 ASC NULLS FIRST, wed_sales#38 ASC NULLS FIRST, thu_sales#39 ASC NULLS FIRST, fri_sales#40 ASC NULLS FIRST, sat_sales#41 ASC NULLS FIRST], [s_store_name#10, s_store_id#9, sun_sales#35, mon_sales#36, tue_sales#37, wed_sales#38, thu_sales#39, fri_sales#40, sat_sales#41]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/simplified.txt
new file mode 100644
index 0000000000000..2d292e81891af
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/simplified.txt
@@ -0,0 +1,31 @@
+TakeOrderedAndProject [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales]
+  WholeStageCodegen (4)
+    HashAggregate [s_store_name,s_store_id,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday  ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price ELSE null END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum]
+      InputAdapter
+        Exchange [s_store_name,s_store_id] #1
+          WholeStageCodegen (3)
+            HashAggregate [s_store_name,s_store_id,d_day_name,ss_sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum]
+              Project [d_day_name,ss_sales_price,s_store_id,s_store_name]
+                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                  Project [d_day_name,ss_store_sk,ss_sales_price]
+                    BroadcastHashJoin [d_date_sk,ss_sold_date_sk]
+                      Project [d_date_sk,d_day_name]
+                        Filter [d_year,d_date_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.date_dim [d_date_sk,d_year,d_day_name]
+                      InputAdapter
+                        BroadcastExchange #2
+                          WholeStageCodegen (1)
+                            Filter [ss_store_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                  InputAdapter
+                    BroadcastExchange #3
+                      WholeStageCodegen (2)
+                        Project [s_store_sk,s_store_id,s_store_name]
+                          Filter [s_gmt_offset,s_store_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.sf100/explain.txt
new file mode 100644
index 0000000000000..84a8547e3f6ef
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.sf100/explain.txt
@@ -0,0 +1,248 @@
+== Physical Plan ==
+TakeOrderedAndProject (36)
++- * Project (35)
+   +- * BroadcastHashJoin Inner BuildRight (34)
+      :- * Project (32)
+      :  +- * BroadcastHashJoin Inner BuildRight (31)
+      :     :- * Project (26)
+      :     :  +- * BroadcastHashJoin Inner BuildRight (25)
+      :     :     :- * Project (14)
+      :     :     :  +- * Filter (13)
+      :     :     :     +- Window (12)
+      :     :     :        +- * Sort (11)
+      :     :     :           +- Exchange (10)
+      :     :     :              +- * Project (9)
+      :     :     :                 +- * Filter (8)
+      :     :     :                    +- * HashAggregate (7)
+      :     :     :                       +- Exchange (6)
+      :     :     :                          +- * HashAggregate (5)
+      :     :     :                             +- * Project (4)
+      :     :     :                                +- * Filter (3)
+      :     :     :                                   +- * ColumnarToRow (2)
+      :     :     :                                      +- Scan parquet default.store_sales (1)
+      :     :     +- BroadcastExchange (24)
+      :     :        +- * Project (23)
+      :     :           +- * Filter (22)
+      :     :              +- Window (21)
+      :     :                 +- * Sort (20)
+      :     :                    +- Exchange (19)
+      :     :                       +- * Project (18)
+      :     :                          +- * Filter (17)
+      :     :                             +- * HashAggregate (16)
+      :     :                                +- ReusedExchange (15)
+      :     +- BroadcastExchange (30)
+      :        +- * Filter (29)
+      :           +- * ColumnarToRow (28)
+      :              +- Scan parquet default.item (27)
+      +- ReusedExchange (33)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_store_sk), EqualTo(ss_store_sk,4)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 1]
+Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4]
+Condition : (isnotnull(ss_store_sk#2) AND (ss_store_sk#2 = 4))
+
+(4) Project [codegen id : 1]
+Output [2]: [ss_item_sk#1, ss_net_profit#3]
+Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4]
+
+(5) HashAggregate [codegen id : 1]
+Input [2]: [ss_item_sk#1, ss_net_profit#3]
+Keys [1]: [ss_item_sk#1]
+Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#3))]
+Aggregate Attributes [2]: [sum#5, count#6]
+Results [3]: [ss_item_sk#1, sum#7, count#8]
+
+(6) Exchange
+Input [3]: [ss_item_sk#1, sum#7, count#8]
+Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#9]
+
+(7) HashAggregate [codegen id : 2]
+Input [3]: [ss_item_sk#1, sum#7, count#8]
+Keys [1]: [ss_item_sk#1]
+Functions [1]: [avg(UnscaledValue(ss_net_profit#3))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_profit#3))#10]
+Results [3]: [ss_item_sk#1 AS item_sk#11, cast((avg(UnscaledValue(ss_net_profit#3))#10 / 100.0) as decimal(11,6)) AS rank_col#12, cast((avg(UnscaledValue(ss_net_profit#3))#10 / 100.0) as decimal(11,6)) AS avg(ss_net_profit#3)#13]
+
+(8) Filter [codegen id : 2]
+Input [3]: [item_sk#11, rank_col#12, avg(ss_net_profit#3)#13]
+Condition : (isnotnull(avg(ss_net_profit#3)#13) AND (cast(avg(ss_net_profit#3)#13 as decimal(13,7)) > CheckOverflow((0.900000 * promote_precision(Subquery scalar-subquery#14, [id=#15])), DecimalType(13,7), true)))
+
+(9) Project [codegen id : 2]
+Output [2]: [item_sk#11, rank_col#12]
+Input [3]: [item_sk#11, rank_col#12, avg(ss_net_profit#3)#13]
+
+(10) Exchange
+Input [2]: [item_sk#11, rank_col#12]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#16]
+
+(11) Sort [codegen id : 3]
+Input [2]: [item_sk#11, rank_col#12]
+Arguments: [rank_col#12 ASC NULLS FIRST], false, 0
+
+(12) Window
+Input [2]: [item_sk#11, rank_col#12]
+Arguments: [rank(rank_col#12) windowspecdefinition(rank_col#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#17], [rank_col#12 ASC NULLS FIRST]
+
+(13) Filter [codegen id : 10]
+Input [3]: [item_sk#11, rank_col#12, rnk#17]
+Condition : ((rnk#17 < 11) AND isnotnull(item_sk#11))
+
+(14) Project [codegen id : 10]
+Output [2]: [item_sk#11, rnk#17]
+Input [3]: [item_sk#11, rank_col#12, rnk#17]
+
+(15) ReusedExchange [Reuses operator id: 6]
+Output [3]: [ss_item_sk#18, sum#19, count#20]
+
+(16) HashAggregate [codegen id : 5]
+Input [3]: [ss_item_sk#18, sum#19, count#20]
+Keys [1]: [ss_item_sk#18]
+Functions [1]: [avg(UnscaledValue(ss_net_profit#21))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_profit#21))#22]
+Results [3]: [ss_item_sk#18 AS item_sk#23, cast((avg(UnscaledValue(ss_net_profit#21))#22 / 100.0) as decimal(11,6)) AS rank_col#24, cast((avg(UnscaledValue(ss_net_profit#21))#22 / 100.0) as decimal(11,6)) AS avg(ss_net_profit#21)#25]
+
+(17) Filter [codegen id : 5]
+Input [3]: [item_sk#23, rank_col#24, avg(ss_net_profit#21)#25]
+Condition : (isnotnull(avg(ss_net_profit#21)#25) AND (cast(avg(ss_net_profit#21)#25 as decimal(13,7)) > CheckOverflow((0.900000 * promote_precision(ReusedSubquery Subquery scalar-subquery#14, [id=#15])), DecimalType(13,7), true)))
+
+(18) Project [codegen id : 5]
+Output [2]: [item_sk#23, rank_col#24]
+Input [3]: [item_sk#23, rank_col#24, avg(ss_net_profit#21)#25]
+
+(19) Exchange
+Input [2]: [item_sk#23, rank_col#24]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#26]
+
+(20) Sort [codegen id : 6]
+Input [2]: [item_sk#23, rank_col#24]
+Arguments: [rank_col#24 DESC NULLS LAST], false, 0
+
+(21) Window
+Input [2]: [item_sk#23, rank_col#24]
+Arguments: [rank(rank_col#24) windowspecdefinition(rank_col#24 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#27], [rank_col#24 DESC NULLS LAST]
+
+(22) Filter [codegen id : 7]
+Input [3]: [item_sk#23, rank_col#24, rnk#27]
+Condition : ((rnk#27 < 11) AND isnotnull(item_sk#23))
+
+(23) Project [codegen id : 7]
+Output [2]: [item_sk#23, rnk#27]
+Input [3]: [item_sk#23, rank_col#24, rnk#27]
+
+(24) BroadcastExchange
+Input [2]: [item_sk#23, rnk#27]
+Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [id=#28]
+
+(25) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [rnk#17]
+Right keys [1]: [rnk#27]
+Join condition: None
+
+(26) Project [codegen id : 10]
+Output [3]: [item_sk#11, rnk#17, item_sk#23]
+Input [4]: [item_sk#11, rnk#17, item_sk#23, rnk#27]
+
+(27) Scan parquet default.item
+Output [2]: [i_item_sk#29, i_product_name#30]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_product_name:string>
+
+(28) ColumnarToRow [codegen id : 8]
+Input [2]: [i_item_sk#29, i_product_name#30]
+
+(29) Filter [codegen id : 8]
+Input [2]: [i_item_sk#29, i_product_name#30]
+Condition : isnotnull(i_item_sk#29)
+
+(30) BroadcastExchange
+Input [2]: [i_item_sk#29, i_product_name#30]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#31]
+
+(31) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [item_sk#11]
+Right keys [1]: [i_item_sk#29]
+Join condition: None
+
+(32) Project [codegen id : 10]
+Output [3]: [rnk#17, item_sk#23, i_product_name#30]
+Input [5]: [item_sk#11, rnk#17, item_sk#23, i_item_sk#29, i_product_name#30]
+
+(33) ReusedExchange [Reuses operator id: 30]
+Output [2]: [i_item_sk#32, i_product_name#33]
+
+(34) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [item_sk#23]
+Right keys [1]: [i_item_sk#32]
+Join condition: None
+
+(35) Project [codegen id : 10]
+Output [3]: [rnk#17, i_product_name#30 AS best_performing#34, i_product_name#33 AS worst_performing#35]
+Input [5]: [rnk#17, item_sk#23, i_product_name#30, i_item_sk#32, i_product_name#33]
+
+(36) TakeOrderedAndProject
+Input [3]: [rnk#17, best_performing#34, worst_performing#35]
+Arguments: 100, [rnk#17 ASC NULLS FIRST], [rnk#17, best_performing#34, worst_performing#35]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 8 Hosting Expression = Subquery scalar-subquery#14, [id=#15]
+* HashAggregate (43)
++- Exchange (42)
+   +- * HashAggregate (41)
+      +- * Project (40)
+         +- * Filter (39)
+            +- * ColumnarToRow (38)
+               +- Scan parquet default.store_sales (37)
+
+
+(37) Scan parquet default.store_sales
+Output [4]: [ss_addr_sk#36, ss_store_sk#37, ss_net_profit#38, ss_sold_date_sk#39]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_store_sk), EqualTo(ss_store_sk,4), IsNull(ss_addr_sk)]
+ReadSchema: struct<ss_addr_sk:int,ss_store_sk:int,ss_net_profit:decimal(7,2)>
+
+(38) ColumnarToRow [codegen id : 1]
+Input [4]: [ss_addr_sk#36, ss_store_sk#37, ss_net_profit#38, ss_sold_date_sk#39]
+
+(39) Filter [codegen id : 1]
+Input [4]: [ss_addr_sk#36, ss_store_sk#37, ss_net_profit#38, ss_sold_date_sk#39]
+Condition : ((isnotnull(ss_store_sk#37) AND (ss_store_sk#37 = 4)) AND isnull(ss_addr_sk#36))
+
+(40) Project [codegen id : 1]
+Output [2]: [ss_store_sk#37, ss_net_profit#38]
+Input [4]: [ss_addr_sk#36, ss_store_sk#37, ss_net_profit#38, ss_sold_date_sk#39]
+
+(41) HashAggregate [codegen id : 1]
+Input [2]: [ss_store_sk#37, ss_net_profit#38]
+Keys [1]: [ss_store_sk#37]
+Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#38))]
+Aggregate Attributes [2]: [sum#40, count#41]
+Results [3]: [ss_store_sk#37, sum#42, count#43]
+
+(42) Exchange
+Input [3]: [ss_store_sk#37, sum#42, count#43]
+Arguments: hashpartitioning(ss_store_sk#37, 5), ENSURE_REQUIREMENTS, [id=#44]
+
+(43) HashAggregate [codegen id : 2]
+Input [3]: [ss_store_sk#37, sum#42, count#43]
+Keys [1]: [ss_store_sk#37]
+Functions [1]: [avg(UnscaledValue(ss_net_profit#38))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_profit#38))#45]
+Results [1]: [cast((avg(UnscaledValue(ss_net_profit#38))#45 / 100.0) as decimal(11,6)) AS rank_col#46]
+
+Subquery:2 Hosting operator id = 17 Hosting Expression = ReusedSubquery Subquery scalar-subquery#14, [id=#15]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.sf100/simplified.txt
new file mode 100644
index 0000000000000..73bc3a333b631
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.sf100/simplified.txt
@@ -0,0 +1,68 @@
+TakeOrderedAndProject [rnk,best_performing,worst_performing]
+  WholeStageCodegen (10)
+    Project [rnk,i_product_name,i_product_name]
+      BroadcastHashJoin [item_sk,i_item_sk]
+        Project [rnk,item_sk,i_product_name]
+          BroadcastHashJoin [item_sk,i_item_sk]
+            Project [item_sk,rnk,item_sk]
+              BroadcastHashJoin [rnk,rnk]
+                Project [item_sk,rnk]
+                  Filter [rnk,item_sk]
+                    InputAdapter
+                      Window [rank_col]
+                        WholeStageCodegen (3)
+                          Sort [rank_col]
+                            InputAdapter
+                              Exchange #1
+                                WholeStageCodegen (2)
+                                  Project [item_sk,rank_col]
+                                    Filter [avg(ss_net_profit)]
+                                      Subquery #1
+                                        WholeStageCodegen (2)
+                                          HashAggregate [ss_store_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),rank_col,sum,count]
+                                            InputAdapter
+                                              Exchange [ss_store_sk] #3
+                                                WholeStageCodegen (1)
+                                                  HashAggregate [ss_store_sk,ss_net_profit] [sum,count,sum,count]
+                                                    Project [ss_store_sk,ss_net_profit]
+                                                      Filter [ss_store_sk,ss_addr_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk]
+                                      HashAggregate [ss_item_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),item_sk,rank_col,avg(ss_net_profit),sum,count]
+                                        InputAdapter
+                                          Exchange [ss_item_sk] #2
+                                            WholeStageCodegen (1)
+                                              HashAggregate [ss_item_sk,ss_net_profit] [sum,count,sum,count]
+                                                Project [ss_item_sk,ss_net_profit]
+                                                  Filter [ss_store_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk]
+                InputAdapter
+                  BroadcastExchange #4
+                    WholeStageCodegen (7)
+                      Project [item_sk,rnk]
+                        Filter [rnk,item_sk]
+                          InputAdapter
+                            Window [rank_col]
+                              WholeStageCodegen (6)
+                                Sort [rank_col]
+                                  InputAdapter
+                                    Exchange #5
+                                      WholeStageCodegen (5)
+                                        Project [item_sk,rank_col]
+                                          Filter [avg(ss_net_profit)]
+                                            ReusedSubquery [rank_col] #1
+                                            HashAggregate [ss_item_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),item_sk,rank_col,avg(ss_net_profit),sum,count]
+                                              InputAdapter
+                                                ReusedExchange [ss_item_sk,sum,count] #2
+            InputAdapter
+              BroadcastExchange #6
+                WholeStageCodegen (8)
+                  Filter [i_item_sk]
+                    ColumnarToRow
+                      InputAdapter
+                        Scan parquet default.item [i_item_sk,i_product_name]
+        InputAdapter
+          ReusedExchange [i_item_sk,i_product_name] #6
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt
new file mode 100644
index 0000000000000..b42ddd8db7836
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt
@@ -0,0 +1,253 @@
+== Physical Plan ==
+TakeOrderedAndProject (37)
++- * Project (36)
+   +- * BroadcastHashJoin Inner BuildRight (35)
+      :- * Project (33)
+      :  +- * BroadcastHashJoin Inner BuildRight (32)
+      :     :- * Project (27)
+      :     :  +- * SortMergeJoin Inner (26)
+      :     :     :- * Sort (15)
+      :     :     :  +- * Project (14)
+      :     :     :     +- * Filter (13)
+      :     :     :        +- Window (12)
+      :     :     :           +- * Sort (11)
+      :     :     :              +- Exchange (10)
+      :     :     :                 +- * Project (9)
+      :     :     :                    +- * Filter (8)
+      :     :     :                       +- * HashAggregate (7)
+      :     :     :                          +- Exchange (6)
+      :     :     :                             +- * HashAggregate (5)
+      :     :     :                                +- * Project (4)
+      :     :     :                                   +- * Filter (3)
+      :     :     :                                      +- * ColumnarToRow (2)
+      :     :     :                                         +- Scan parquet default.store_sales (1)
+      :     :     +- * Sort (25)
+      :     :        +- * Project (24)
+      :     :           +- * Filter (23)
+      :     :              +- Window (22)
+      :     :                 +- * Sort (21)
+      :     :                    +- Exchange (20)
+      :     :                       +- * Project (19)
+      :     :                          +- * Filter (18)
+      :     :                             +- * HashAggregate (17)
+      :     :                                +- ReusedExchange (16)
+      :     +- BroadcastExchange (31)
+      :        +- * Filter (30)
+      :           +- * ColumnarToRow (29)
+      :              +- Scan parquet default.item (28)
+      +- ReusedExchange (34)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_store_sk), EqualTo(ss_store_sk,4)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 1]
+Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4]
+Condition : (isnotnull(ss_store_sk#2) AND (ss_store_sk#2 = 4))
+
+(4) Project [codegen id : 1]
+Output [2]: [ss_item_sk#1, ss_net_profit#3]
+Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4]
+
+(5) HashAggregate [codegen id : 1]
+Input [2]: [ss_item_sk#1, ss_net_profit#3]
+Keys [1]: [ss_item_sk#1]
+Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#3))]
+Aggregate Attributes [2]: [sum#5, count#6]
+Results [3]: [ss_item_sk#1, sum#7, count#8]
+
+(6) Exchange
+Input [3]: [ss_item_sk#1, sum#7, count#8]
+Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#9]
+
+(7) HashAggregate [codegen id : 2]
+Input [3]: [ss_item_sk#1, sum#7, count#8]
+Keys [1]: [ss_item_sk#1]
+Functions [1]: [avg(UnscaledValue(ss_net_profit#3))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_profit#3))#10]
+Results [3]: [ss_item_sk#1 AS item_sk#11, cast((avg(UnscaledValue(ss_net_profit#3))#10 / 100.0) as decimal(11,6)) AS rank_col#12, cast((avg(UnscaledValue(ss_net_profit#3))#10 / 100.0) as decimal(11,6)) AS avg(ss_net_profit#3)#13]
+
+(8) Filter [codegen id : 2]
+Input [3]: [item_sk#11, rank_col#12, avg(ss_net_profit#3)#13]
+Condition : (isnotnull(avg(ss_net_profit#3)#13) AND (cast(avg(ss_net_profit#3)#13 as decimal(13,7)) > CheckOverflow((0.900000 * promote_precision(Subquery scalar-subquery#14, [id=#15])), DecimalType(13,7), true)))
+
+(9) Project [codegen id : 2]
+Output [2]: [item_sk#11, rank_col#12]
+Input [3]: [item_sk#11, rank_col#12, avg(ss_net_profit#3)#13]
+
+(10) Exchange
+Input [2]: [item_sk#11, rank_col#12]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#16]
+
+(11) Sort [codegen id : 3]
+Input [2]: [item_sk#11, rank_col#12]
+Arguments: [rank_col#12 ASC NULLS FIRST], false, 0
+
+(12) Window
+Input [2]: [item_sk#11, rank_col#12]
+Arguments: [rank(rank_col#12) windowspecdefinition(rank_col#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#17], [rank_col#12 ASC NULLS FIRST]
+
+(13) Filter [codegen id : 4]
+Input [3]: [item_sk#11, rank_col#12, rnk#17]
+Condition : ((rnk#17 < 11) AND isnotnull(item_sk#11))
+
+(14) Project [codegen id : 4]
+Output [2]: [item_sk#11, rnk#17]
+Input [3]: [item_sk#11, rank_col#12, rnk#17]
+
+(15) Sort [codegen id : 4]
+Input [2]: [item_sk#11, rnk#17]
+Arguments: [rnk#17 ASC NULLS FIRST], false, 0
+
+(16) ReusedExchange [Reuses operator id: 6]
+Output [3]: [ss_item_sk#18, sum#19, count#20]
+
+(17) HashAggregate [codegen id : 6]
+Input [3]: [ss_item_sk#18, sum#19, count#20]
+Keys [1]: [ss_item_sk#18]
+Functions [1]: [avg(UnscaledValue(ss_net_profit#21))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_profit#21))#22]
+Results [3]: [ss_item_sk#18 AS item_sk#23, cast((avg(UnscaledValue(ss_net_profit#21))#22 / 100.0) as decimal(11,6)) AS rank_col#24, cast((avg(UnscaledValue(ss_net_profit#21))#22 / 100.0) as decimal(11,6)) AS avg(ss_net_profit#21)#25]
+
+(18) Filter [codegen id : 6]
+Input [3]: [item_sk#23, rank_col#24, avg(ss_net_profit#21)#25]
+Condition : (isnotnull(avg(ss_net_profit#21)#25) AND (cast(avg(ss_net_profit#21)#25 as decimal(13,7)) > CheckOverflow((0.900000 * promote_precision(ReusedSubquery Subquery scalar-subquery#14, [id=#15])), DecimalType(13,7), true)))
+
+(19) Project [codegen id : 6]
+Output [2]: [item_sk#23, rank_col#24]
+Input [3]: [item_sk#23, rank_col#24, avg(ss_net_profit#21)#25]
+
+(20) Exchange
+Input [2]: [item_sk#23, rank_col#24]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#26]
+
+(21) Sort [codegen id : 7]
+Input [2]: [item_sk#23, rank_col#24]
+Arguments: [rank_col#24 DESC NULLS LAST], false, 0
+
+(22) Window
+Input [2]: [item_sk#23, rank_col#24]
+Arguments: [rank(rank_col#24) windowspecdefinition(rank_col#24 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#27], [rank_col#24 DESC NULLS LAST]
+
+(23) Filter [codegen id : 8]
+Input [3]: [item_sk#23, rank_col#24, rnk#27]
+Condition : ((rnk#27 < 11) AND isnotnull(item_sk#23))
+
+(24) Project [codegen id : 8]
+Output [2]: [item_sk#23, rnk#27]
+Input [3]: [item_sk#23, rank_col#24, rnk#27]
+
+(25) Sort [codegen id : 8]
+Input [2]: [item_sk#23, rnk#27]
+Arguments: [rnk#27 ASC NULLS FIRST], false, 0
+
+(26) SortMergeJoin [codegen id : 11]
+Left keys [1]: [rnk#17]
+Right keys [1]: [rnk#27]
+Join condition: None
+
+(27) Project [codegen id : 11]
+Output [3]: [item_sk#11, rnk#17, item_sk#23]
+Input [4]: [item_sk#11, rnk#17, item_sk#23, rnk#27]
+
+(28) Scan parquet default.item
+Output [2]: [i_item_sk#28, i_product_name#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_product_name:string>
+
+(29) ColumnarToRow [codegen id : 9]
+Input [2]: [i_item_sk#28, i_product_name#29]
+
+(30) Filter [codegen id : 9]
+Input [2]: [i_item_sk#28, i_product_name#29]
+Condition : isnotnull(i_item_sk#28)
+
+(31) BroadcastExchange
+Input [2]: [i_item_sk#28, i_product_name#29]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#30]
+
+(32) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [item_sk#11]
+Right keys [1]: [i_item_sk#28]
+Join condition: None
+
+(33) Project [codegen id : 11]
+Output [3]: [rnk#17, item_sk#23, i_product_name#29]
+Input [5]: [item_sk#11, rnk#17, item_sk#23, i_item_sk#28, i_product_name#29]
+
+(34) ReusedExchange [Reuses operator id: 31]
+Output [2]: [i_item_sk#31, i_product_name#32]
+
+(35) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [item_sk#23]
+Right keys [1]: [i_item_sk#31]
+Join condition: None
+
+(36) Project [codegen id : 11]
+Output [3]: [rnk#17, i_product_name#29 AS best_performing#33, i_product_name#32 AS worst_performing#34]
+Input [5]: [rnk#17, item_sk#23, i_product_name#29, i_item_sk#31, i_product_name#32]
+
+(37) TakeOrderedAndProject
+Input [3]: [rnk#17, best_performing#33, worst_performing#34]
+Arguments: 100, [rnk#17 ASC NULLS FIRST], [rnk#17, best_performing#33, worst_performing#34]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 8 Hosting Expression = Subquery scalar-subquery#14, [id=#15]
+* HashAggregate (44)
++- Exchange (43)
+   +- * HashAggregate (42)
+      +- * Project (41)
+         +- * Filter (40)
+            +- * ColumnarToRow (39)
+               +- Scan parquet default.store_sales (38)
+
+
+(38) Scan parquet default.store_sales
+Output [4]: [ss_addr_sk#35, ss_store_sk#36, ss_net_profit#37, ss_sold_date_sk#38]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_store_sk), EqualTo(ss_store_sk,4), IsNull(ss_addr_sk)]
+ReadSchema: struct<ss_addr_sk:int,ss_store_sk:int,ss_net_profit:decimal(7,2)>
+
+(39) ColumnarToRow [codegen id : 1]
+Input [4]: [ss_addr_sk#35, ss_store_sk#36, ss_net_profit#37, ss_sold_date_sk#38]
+
+(40) Filter [codegen id : 1]
+Input [4]: [ss_addr_sk#35, ss_store_sk#36, ss_net_profit#37, ss_sold_date_sk#38]
+Condition : ((isnotnull(ss_store_sk#36) AND (ss_store_sk#36 = 4)) AND isnull(ss_addr_sk#35))
+
+(41) Project [codegen id : 1]
+Output [2]: [ss_store_sk#36, ss_net_profit#37]
+Input [4]: [ss_addr_sk#35, ss_store_sk#36, ss_net_profit#37, ss_sold_date_sk#38]
+
+(42) HashAggregate [codegen id : 1]
+Input [2]: [ss_store_sk#36, ss_net_profit#37]
+Keys [1]: [ss_store_sk#36]
+Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#37))]
+Aggregate Attributes [2]: [sum#39, count#40]
+Results [3]: [ss_store_sk#36, sum#41, count#42]
+
+(43) Exchange
+Input [3]: [ss_store_sk#36, sum#41, count#42]
+Arguments: hashpartitioning(ss_store_sk#36, 5), ENSURE_REQUIREMENTS, [id=#43]
+
+(44) HashAggregate [codegen id : 2]
+Input [3]: [ss_store_sk#36, sum#41, count#42]
+Keys [1]: [ss_store_sk#36]
+Functions [1]: [avg(UnscaledValue(ss_net_profit#37))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_profit#37))#44]
+Results [1]: [cast((avg(UnscaledValue(ss_net_profit#37))#44 / 100.0) as decimal(11,6)) AS rank_col#45]
+
+Subquery:2 Hosting operator id = 18 Hosting Expression = ReusedSubquery Subquery scalar-subquery#14, [id=#15]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/simplified.txt
new file mode 100644
index 0000000000000..10d3570c2dc8d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/simplified.txt
@@ -0,0 +1,71 @@
+TakeOrderedAndProject [rnk,best_performing,worst_performing]
+  WholeStageCodegen (11)
+    Project [rnk,i_product_name,i_product_name]
+      BroadcastHashJoin [item_sk,i_item_sk]
+        Project [rnk,item_sk,i_product_name]
+          BroadcastHashJoin [item_sk,i_item_sk]
+            Project [item_sk,rnk,item_sk]
+              SortMergeJoin [rnk,rnk]
+                InputAdapter
+                  WholeStageCodegen (4)
+                    Sort [rnk]
+                      Project [item_sk,rnk]
+                        Filter [rnk,item_sk]
+                          InputAdapter
+                            Window [rank_col]
+                              WholeStageCodegen (3)
+                                Sort [rank_col]
+                                  InputAdapter
+                                    Exchange #1
+                                      WholeStageCodegen (2)
+                                        Project [item_sk,rank_col]
+                                          Filter [avg(ss_net_profit)]
+                                            Subquery #1
+                                              WholeStageCodegen (2)
+                                                HashAggregate [ss_store_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),rank_col,sum,count]
+                                                  InputAdapter
+                                                    Exchange [ss_store_sk] #3
+                                                      WholeStageCodegen (1)
+                                                        HashAggregate [ss_store_sk,ss_net_profit] [sum,count,sum,count]
+                                                          Project [ss_store_sk,ss_net_profit]
+                                                            Filter [ss_store_sk,ss_addr_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk]
+                                            HashAggregate [ss_item_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),item_sk,rank_col,avg(ss_net_profit),sum,count]
+                                              InputAdapter
+                                                Exchange [ss_item_sk] #2
+                                                  WholeStageCodegen (1)
+                                                    HashAggregate [ss_item_sk,ss_net_profit] [sum,count,sum,count]
+                                                      Project [ss_item_sk,ss_net_profit]
+                                                        Filter [ss_store_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk]
+                InputAdapter
+                  WholeStageCodegen (8)
+                    Sort [rnk]
+                      Project [item_sk,rnk]
+                        Filter [rnk,item_sk]
+                          InputAdapter
+                            Window [rank_col]
+                              WholeStageCodegen (7)
+                                Sort [rank_col]
+                                  InputAdapter
+                                    Exchange #4
+                                      WholeStageCodegen (6)
+                                        Project [item_sk,rank_col]
+                                          Filter [avg(ss_net_profit)]
+                                            ReusedSubquery [rank_col] #1
+                                            HashAggregate [ss_item_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),item_sk,rank_col,avg(ss_net_profit),sum,count]
+                                              InputAdapter
+                                                ReusedExchange [ss_item_sk,sum,count] #2
+            InputAdapter
+              BroadcastExchange #5
+                WholeStageCodegen (9)
+                  Filter [i_item_sk]
+                    ColumnarToRow
+                      InputAdapter
+                        Scan parquet default.item [i_item_sk,i_product_name]
+        InputAdapter
+          ReusedExchange [i_item_sk,i_product_name] #5
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.sf100/explain.txt
new file mode 100644
index 0000000000000..701414b22eb80
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.sf100/explain.txt
@@ -0,0 +1,267 @@
+== Physical Plan ==
+TakeOrderedAndProject (46)
++- * HashAggregate (45)
+   +- Exchange (44)
+      +- * HashAggregate (43)
+         +- * Project (42)
+            +- * Filter (41)
+               +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (40)
+                  :- * Project (34)
+                  :  +- * SortMergeJoin Inner (33)
+                  :     :- * Sort (18)
+                  :     :  +- Exchange (17)
+                  :     :     +- * Project (16)
+                  :     :        +- * BroadcastHashJoin Inner BuildRight (15)
+                  :     :           :- * Project (10)
+                  :     :           :  +- * BroadcastHashJoin Inner BuildRight (9)
+                  :     :           :     :- * Filter (3)
+                  :     :           :     :  +- * ColumnarToRow (2)
+                  :     :           :     :     +- Scan parquet default.web_sales (1)
+                  :     :           :     +- BroadcastExchange (8)
+                  :     :           :        +- * Project (7)
+                  :     :           :           +- * Filter (6)
+                  :     :           :              +- * ColumnarToRow (5)
+                  :     :           :                 +- Scan parquet default.date_dim (4)
+                  :     :           +- BroadcastExchange (14)
+                  :     :              +- * Filter (13)
+                  :     :                 +- * ColumnarToRow (12)
+                  :     :                    +- Scan parquet default.item (11)
+                  :     +- * Sort (32)
+                  :        +- Exchange (31)
+                  :           +- * Project (30)
+                  :              +- * SortMergeJoin Inner (29)
+                  :                 :- * Sort (23)
+                  :                 :  +- Exchange (22)
+                  :                 :     +- * Filter (21)
+                  :                 :        +- * ColumnarToRow (20)
+                  :                 :           +- Scan parquet default.customer (19)
+                  :                 +- * Sort (28)
+                  :                    +- Exchange (27)
+                  :                       +- * Filter (26)
+                  :                          +- * ColumnarToRow (25)
+                  :                             +- Scan parquet default.customer_address (24)
+                  +- BroadcastExchange (39)
+                     +- * Project (38)
+                        +- * Filter (37)
+                           +- * ColumnarToRow (36)
+                              +- Scan parquet default.item (35)
+
+
+(1) Scan parquet default.web_sales
+Output [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#5), dynamicpruningexpression(ws_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk), IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_bill_customer_sk:int,ws_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5]
+
+(3) Filter [codegen id : 3]
+Input [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5]
+Condition : (isnotnull(ws_bill_customer_sk#3) AND isnotnull(ws_item_sk#2))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#7, d_year#8, d_qoy#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_qoy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_qoy#9]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_qoy#9]
+Condition : ((((isnotnull(d_qoy#9) AND isnotnull(d_year#8)) AND (d_qoy#9 = 2)) AND (d_year#8 = 2001)) AND isnotnull(d_date_sk#7))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#7]
+Input [3]: [d_date_sk#7, d_year#8, d_qoy#9]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ws_sold_date_sk#5]
+Right keys [1]: [d_date_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [3]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4]
+Input [5]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5, d_date_sk#7]
+
+(11) Scan parquet default.item
+Output [2]: [i_item_sk#11, i_item_id#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [i_item_sk#11, i_item_id#12]
+
+(13) Filter [codegen id : 2]
+Input [2]: [i_item_sk#11, i_item_id#12]
+Condition : isnotnull(i_item_sk#11)
+
+(14) BroadcastExchange
+Input [2]: [i_item_sk#11, i_item_id#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#13]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ws_item_sk#2]
+Right keys [1]: [i_item_sk#11]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [3]: [ws_bill_customer_sk#3, ws_sales_price#4, i_item_id#12]
+Input [5]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, i_item_sk#11, i_item_id#12]
+
+(17) Exchange
+Input [3]: [ws_bill_customer_sk#3, ws_sales_price#4, i_item_id#12]
+Arguments: hashpartitioning(ws_bill_customer_sk#3, 5), ENSURE_REQUIREMENTS, [id=#14]
+
+(18) Sort [codegen id : 4]
+Input [3]: [ws_bill_customer_sk#3, ws_sales_price#4, i_item_id#12]
+Arguments: [ws_bill_customer_sk#3 ASC NULLS FIRST], false, 0
+
+(19) Scan parquet default.customer
+Output [2]: [c_customer_sk#15, c_current_addr_sk#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int>
+
+(20) ColumnarToRow [codegen id : 5]
+Input [2]: [c_customer_sk#15, c_current_addr_sk#16]
+
+(21) Filter [codegen id : 5]
+Input [2]: [c_customer_sk#15, c_current_addr_sk#16]
+Condition : (isnotnull(c_customer_sk#15) AND isnotnull(c_current_addr_sk#16))
+
+(22) Exchange
+Input [2]: [c_customer_sk#15, c_current_addr_sk#16]
+Arguments: hashpartitioning(c_current_addr_sk#16, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(23) Sort [codegen id : 6]
+Input [2]: [c_customer_sk#15, c_current_addr_sk#16]
+Arguments: [c_current_addr_sk#16 ASC NULLS FIRST], false, 0
+
+(24) Scan parquet default.customer_address
+Output [3]: [ca_address_sk#18, ca_city#19, ca_zip#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_city:string,ca_zip:string>
+
+(25) ColumnarToRow [codegen id : 7]
+Input [3]: [ca_address_sk#18, ca_city#19, ca_zip#20]
+
+(26) Filter [codegen id : 7]
+Input [3]: [ca_address_sk#18, ca_city#19, ca_zip#20]
+Condition : isnotnull(ca_address_sk#18)
+
+(27) Exchange
+Input [3]: [ca_address_sk#18, ca_city#19, ca_zip#20]
+Arguments: hashpartitioning(ca_address_sk#18, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(28) Sort [codegen id : 8]
+Input [3]: [ca_address_sk#18, ca_city#19, ca_zip#20]
+Arguments: [ca_address_sk#18 ASC NULLS FIRST], false, 0
+
+(29) SortMergeJoin [codegen id : 9]
+Left keys [1]: [c_current_addr_sk#16]
+Right keys [1]: [ca_address_sk#18]
+Join condition: None
+
+(30) Project [codegen id : 9]
+Output [3]: [c_customer_sk#15, ca_city#19, ca_zip#20]
+Input [5]: [c_customer_sk#15, c_current_addr_sk#16, ca_address_sk#18, ca_city#19, ca_zip#20]
+
+(31) Exchange
+Input [3]: [c_customer_sk#15, ca_city#19, ca_zip#20]
+Arguments: hashpartitioning(c_customer_sk#15, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(32) Sort [codegen id : 10]
+Input [3]: [c_customer_sk#15, ca_city#19, ca_zip#20]
+Arguments: [c_customer_sk#15 ASC NULLS FIRST], false, 0
+
+(33) SortMergeJoin [codegen id : 12]
+Left keys [1]: [ws_bill_customer_sk#3]
+Right keys [1]: [c_customer_sk#15]
+Join condition: None
+
+(34) Project [codegen id : 12]
+Output [4]: [ws_sales_price#4, ca_city#19, ca_zip#20, i_item_id#12]
+Input [6]: [ws_bill_customer_sk#3, ws_sales_price#4, i_item_id#12, c_customer_sk#15, ca_city#19, ca_zip#20]
+
+(35) Scan parquet default.item
+Output [2]: [i_item_sk#23, i_item_id#24]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [In(i_item_sk, [2,3,5,7,11,13,17,19,23,29])]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(36) ColumnarToRow [codegen id : 11]
+Input [2]: [i_item_sk#23, i_item_id#24]
+
+(37) Filter [codegen id : 11]
+Input [2]: [i_item_sk#23, i_item_id#24]
+Condition : i_item_sk#23 IN (2,3,5,7,11,13,17,19,23,29)
+
+(38) Project [codegen id : 11]
+Output [1]: [i_item_id#24]
+Input [2]: [i_item_sk#23, i_item_id#24]
+
+(39) BroadcastExchange
+Input [1]: [i_item_id#24]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#25]
+
+(40) BroadcastHashJoin [codegen id : 12]
+Left keys [1]: [i_item_id#12]
+Right keys [1]: [i_item_id#24]
+Join condition: None
+
+(41) Filter [codegen id : 12]
+Input [5]: [ws_sales_price#4, ca_city#19, ca_zip#20, i_item_id#12, exists#1]
+Condition : (substr(ca_zip#20, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR exists#1)
+
+(42) Project [codegen id : 12]
+Output [3]: [ws_sales_price#4, ca_city#19, ca_zip#20]
+Input [5]: [ws_sales_price#4, ca_city#19, ca_zip#20, i_item_id#12, exists#1]
+
+(43) HashAggregate [codegen id : 12]
+Input [3]: [ws_sales_price#4, ca_city#19, ca_zip#20]
+Keys [2]: [ca_zip#20, ca_city#19]
+Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#4))]
+Aggregate Attributes [1]: [sum#26]
+Results [3]: [ca_zip#20, ca_city#19, sum#27]
+
+(44) Exchange
+Input [3]: [ca_zip#20, ca_city#19, sum#27]
+Arguments: hashpartitioning(ca_zip#20, ca_city#19, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(45) HashAggregate [codegen id : 13]
+Input [3]: [ca_zip#20, ca_city#19, sum#27]
+Keys [2]: [ca_zip#20, ca_city#19]
+Functions [1]: [sum(UnscaledValue(ws_sales_price#4))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#4))#29]
+Results [3]: [ca_zip#20, ca_city#19, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#29,17,2) AS sum(ws_sales_price)#30]
+
+(46) TakeOrderedAndProject
+Input [3]: [ca_zip#20, ca_city#19, sum(ws_sales_price)#30]
+Arguments: 100, [ca_zip#20 ASC NULLS FIRST, ca_city#19 ASC NULLS FIRST], [ca_zip#20, ca_city#19, sum(ws_sales_price)#30]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (47)
+
+
+(47) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#7]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.sf100/simplified.txt
new file mode 100644
index 0000000000000..fd36e3ade4099
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.sf100/simplified.txt
@@ -0,0 +1,79 @@
+TakeOrderedAndProject [ca_zip,ca_city,sum(ws_sales_price)]
+  WholeStageCodegen (13)
+    HashAggregate [ca_zip,ca_city,sum] [sum(UnscaledValue(ws_sales_price)),sum(ws_sales_price),sum]
+      InputAdapter
+        Exchange [ca_zip,ca_city] #1
+          WholeStageCodegen (12)
+            HashAggregate [ca_zip,ca_city,ws_sales_price] [sum,sum]
+              Project [ws_sales_price,ca_city,ca_zip]
+                Filter [ca_zip,exists]
+                  BroadcastHashJoin [i_item_id,i_item_id]
+                    Project [ws_sales_price,ca_city,ca_zip,i_item_id]
+                      SortMergeJoin [ws_bill_customer_sk,c_customer_sk]
+                        InputAdapter
+                          WholeStageCodegen (4)
+                            Sort [ws_bill_customer_sk]
+                              InputAdapter
+                                Exchange [ws_bill_customer_sk] #2
+                                  WholeStageCodegen (3)
+                                    Project [ws_bill_customer_sk,ws_sales_price,i_item_id]
+                                      BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                        Project [ws_item_sk,ws_bill_customer_sk,ws_sales_price]
+                                          BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                            Filter [ws_bill_customer_sk,ws_item_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #1
+                                                      ReusedExchange [d_date_sk] #3
+                                            InputAdapter
+                                              BroadcastExchange #3
+                                                WholeStageCodegen (1)
+                                                  Project [d_date_sk]
+                                                    Filter [d_qoy,d_year,d_date_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.date_dim [d_date_sk,d_year,d_qoy]
+                                        InputAdapter
+                                          BroadcastExchange #4
+                                            WholeStageCodegen (2)
+                                              Filter [i_item_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.item [i_item_sk,i_item_id]
+                        InputAdapter
+                          WholeStageCodegen (10)
+                            Sort [c_customer_sk]
+                              InputAdapter
+                                Exchange [c_customer_sk] #5
+                                  WholeStageCodegen (9)
+                                    Project [c_customer_sk,ca_city,ca_zip]
+                                      SortMergeJoin [c_current_addr_sk,ca_address_sk]
+                                        InputAdapter
+                                          WholeStageCodegen (6)
+                                            Sort [c_current_addr_sk]
+                                              InputAdapter
+                                                Exchange [c_current_addr_sk] #6
+                                                  WholeStageCodegen (5)
+                                                    Filter [c_customer_sk,c_current_addr_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.customer [c_customer_sk,c_current_addr_sk]
+                                        InputAdapter
+                                          WholeStageCodegen (8)
+                                            Sort [ca_address_sk]
+                                              InputAdapter
+                                                Exchange [ca_address_sk] #7
+                                                  WholeStageCodegen (7)
+                                                    Filter [ca_address_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.customer_address [ca_address_sk,ca_city,ca_zip]
+                    InputAdapter
+                      BroadcastExchange #8
+                        WholeStageCodegen (11)
+                          Project [i_item_id]
+                            Filter [i_item_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.item [i_item_sk,i_item_id]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/explain.txt
new file mode 100644
index 0000000000000..f3a37f9e8767e
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/explain.txt
@@ -0,0 +1,237 @@
+== Physical Plan ==
+TakeOrderedAndProject (40)
++- * HashAggregate (39)
+   +- Exchange (38)
+      +- * HashAggregate (37)
+         +- * Project (36)
+            +- * Filter (35)
+               +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (34)
+                  :- * Project (28)
+                  :  +- * BroadcastHashJoin Inner BuildRight (27)
+                  :     :- * Project (22)
+                  :     :  +- * BroadcastHashJoin Inner BuildRight (21)
+                  :     :     :- * Project (15)
+                  :     :     :  +- * BroadcastHashJoin Inner BuildRight (14)
+                  :     :     :     :- * Project (9)
+                  :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+                  :     :     :     :     :- * Filter (3)
+                  :     :     :     :     :  +- * ColumnarToRow (2)
+                  :     :     :     :     :     +- Scan parquet default.web_sales (1)
+                  :     :     :     :     +- BroadcastExchange (7)
+                  :     :     :     :        +- * Filter (6)
+                  :     :     :     :           +- * ColumnarToRow (5)
+                  :     :     :     :              +- Scan parquet default.customer (4)
+                  :     :     :     +- BroadcastExchange (13)
+                  :     :     :        +- * Filter (12)
+                  :     :     :           +- * ColumnarToRow (11)
+                  :     :     :              +- Scan parquet default.customer_address (10)
+                  :     :     +- BroadcastExchange (20)
+                  :     :        +- * Project (19)
+                  :     :           +- * Filter (18)
+                  :     :              +- * ColumnarToRow (17)
+                  :     :                 +- Scan parquet default.date_dim (16)
+                  :     +- BroadcastExchange (26)
+                  :        +- * Filter (25)
+                  :           +- * ColumnarToRow (24)
+                  :              +- Scan parquet default.item (23)
+                  +- BroadcastExchange (33)
+                     +- * Project (32)
+                        +- * Filter (31)
+                           +- * ColumnarToRow (30)
+                              +- Scan parquet default.item (29)
+
+
+(1) Scan parquet default.web_sales
+Output [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#5), dynamicpruningexpression(ws_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk), IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_bill_customer_sk:int,ws_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 6]
+Input [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5]
+
+(3) Filter [codegen id : 6]
+Input [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5]
+Condition : (isnotnull(ws_bill_customer_sk#3) AND isnotnull(ws_item_sk#2))
+
+(4) Scan parquet default.customer
+Output [2]: [c_customer_sk#7, c_current_addr_sk#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [c_customer_sk#7, c_current_addr_sk#8]
+
+(6) Filter [codegen id : 1]
+Input [2]: [c_customer_sk#7, c_current_addr_sk#8]
+Condition : (isnotnull(c_customer_sk#7) AND isnotnull(c_current_addr_sk#8))
+
+(7) BroadcastExchange
+Input [2]: [c_customer_sk#7, c_current_addr_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#9]
+
+(8) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ws_bill_customer_sk#3]
+Right keys [1]: [c_customer_sk#7]
+Join condition: None
+
+(9) Project [codegen id : 6]
+Output [4]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#8]
+Input [6]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5, c_customer_sk#7, c_current_addr_sk#8]
+
+(10) Scan parquet default.customer_address
+Output [3]: [ca_address_sk#10, ca_city#11, ca_zip#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_city:string,ca_zip:string>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [3]: [ca_address_sk#10, ca_city#11, ca_zip#12]
+
+(12) Filter [codegen id : 2]
+Input [3]: [ca_address_sk#10, ca_city#11, ca_zip#12]
+Condition : isnotnull(ca_address_sk#10)
+
+(13) BroadcastExchange
+Input [3]: [ca_address_sk#10, ca_city#11, ca_zip#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#13]
+
+(14) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [c_current_addr_sk#8]
+Right keys [1]: [ca_address_sk#10]
+Join condition: None
+
+(15) Project [codegen id : 6]
+Output [5]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#11, ca_zip#12]
+Input [7]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#8, ca_address_sk#10, ca_city#11, ca_zip#12]
+
+(16) Scan parquet default.date_dim
+Output [3]: [d_date_sk#14, d_year#15, d_qoy#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_qoy:int>
+
+(17) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#14, d_year#15, d_qoy#16]
+
+(18) Filter [codegen id : 3]
+Input [3]: [d_date_sk#14, d_year#15, d_qoy#16]
+Condition : ((((isnotnull(d_qoy#16) AND isnotnull(d_year#15)) AND (d_qoy#16 = 2)) AND (d_year#15 = 2001)) AND isnotnull(d_date_sk#14))
+
+(19) Project [codegen id : 3]
+Output [1]: [d_date_sk#14]
+Input [3]: [d_date_sk#14, d_year#15, d_qoy#16]
+
+(20) BroadcastExchange
+Input [1]: [d_date_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(21) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ws_sold_date_sk#5]
+Right keys [1]: [d_date_sk#14]
+Join condition: None
+
+(22) Project [codegen id : 6]
+Output [4]: [ws_item_sk#2, ws_sales_price#4, ca_city#11, ca_zip#12]
+Input [6]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#11, ca_zip#12, d_date_sk#14]
+
+(23) Scan parquet default.item
+Output [2]: [i_item_sk#18, i_item_id#19]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(24) ColumnarToRow [codegen id : 4]
+Input [2]: [i_item_sk#18, i_item_id#19]
+
+(25) Filter [codegen id : 4]
+Input [2]: [i_item_sk#18, i_item_id#19]
+Condition : isnotnull(i_item_sk#18)
+
+(26) BroadcastExchange
+Input [2]: [i_item_sk#18, i_item_id#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#20]
+
+(27) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ws_item_sk#2]
+Right keys [1]: [i_item_sk#18]
+Join condition: None
+
+(28) Project [codegen id : 6]
+Output [4]: [ws_sales_price#4, ca_city#11, ca_zip#12, i_item_id#19]
+Input [6]: [ws_item_sk#2, ws_sales_price#4, ca_city#11, ca_zip#12, i_item_sk#18, i_item_id#19]
+
+(29) Scan parquet default.item
+Output [2]: [i_item_sk#21, i_item_id#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [In(i_item_sk, [2,3,5,7,11,13,17,19,23,29])]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(30) ColumnarToRow [codegen id : 5]
+Input [2]: [i_item_sk#21, i_item_id#22]
+
+(31) Filter [codegen id : 5]
+Input [2]: [i_item_sk#21, i_item_id#22]
+Condition : i_item_sk#21 IN (2,3,5,7,11,13,17,19,23,29)
+
+(32) Project [codegen id : 5]
+Output [1]: [i_item_id#22]
+Input [2]: [i_item_sk#21, i_item_id#22]
+
+(33) BroadcastExchange
+Input [1]: [i_item_id#22]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#23]
+
+(34) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [i_item_id#19]
+Right keys [1]: [i_item_id#22]
+Join condition: None
+
+(35) Filter [codegen id : 6]
+Input [5]: [ws_sales_price#4, ca_city#11, ca_zip#12, i_item_id#19, exists#1]
+Condition : (substr(ca_zip#12, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR exists#1)
+
+(36) Project [codegen id : 6]
+Output [3]: [ws_sales_price#4, ca_city#11, ca_zip#12]
+Input [5]: [ws_sales_price#4, ca_city#11, ca_zip#12, i_item_id#19, exists#1]
+
+(37) HashAggregate [codegen id : 6]
+Input [3]: [ws_sales_price#4, ca_city#11, ca_zip#12]
+Keys [2]: [ca_zip#12, ca_city#11]
+Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#4))]
+Aggregate Attributes [1]: [sum#24]
+Results [3]: [ca_zip#12, ca_city#11, sum#25]
+
+(38) Exchange
+Input [3]: [ca_zip#12, ca_city#11, sum#25]
+Arguments: hashpartitioning(ca_zip#12, ca_city#11, 5), ENSURE_REQUIREMENTS, [id=#26]
+
+(39) HashAggregate [codegen id : 7]
+Input [3]: [ca_zip#12, ca_city#11, sum#25]
+Keys [2]: [ca_zip#12, ca_city#11]
+Functions [1]: [sum(UnscaledValue(ws_sales_price#4))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#4))#27]
+Results [3]: [ca_zip#12, ca_city#11, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#27,17,2) AS sum(ws_sales_price)#28]
+
+(40) TakeOrderedAndProject
+Input [3]: [ca_zip#12, ca_city#11, sum(ws_sales_price)#28]
+Arguments: 100, [ca_zip#12 ASC NULLS FIRST, ca_city#11 ASC NULLS FIRST], [ca_zip#12, ca_city#11, sum(ws_sales_price)#28]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (41)
+
+
+(41) ReusedExchange [Reuses operator id: 20]
+Output [1]: [d_date_sk#14]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/simplified.txt
new file mode 100644
index 0000000000000..5b2be557a96fa
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/simplified.txt
@@ -0,0 +1,61 @@
+TakeOrderedAndProject [ca_zip,ca_city,sum(ws_sales_price)]
+  WholeStageCodegen (7)
+    HashAggregate [ca_zip,ca_city,sum] [sum(UnscaledValue(ws_sales_price)),sum(ws_sales_price),sum]
+      InputAdapter
+        Exchange [ca_zip,ca_city] #1
+          WholeStageCodegen (6)
+            HashAggregate [ca_zip,ca_city,ws_sales_price] [sum,sum]
+              Project [ws_sales_price,ca_city,ca_zip]
+                Filter [ca_zip,exists]
+                  BroadcastHashJoin [i_item_id,i_item_id]
+                    Project [ws_sales_price,ca_city,ca_zip,i_item_id]
+                      BroadcastHashJoin [ws_item_sk,i_item_sk]
+                        Project [ws_item_sk,ws_sales_price,ca_city,ca_zip]
+                          BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                            Project [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip]
+                              BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                                Project [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk]
+                                  BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk]
+                                    Filter [ws_bill_customer_sk,ws_item_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk]
+                                            SubqueryBroadcast [d_date_sk] #1
+                                              ReusedExchange [d_date_sk] #2
+                                    InputAdapter
+                                      BroadcastExchange #3
+                                        WholeStageCodegen (1)
+                                          Filter [c_customer_sk,c_current_addr_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.customer [c_customer_sk,c_current_addr_sk]
+                                InputAdapter
+                                  BroadcastExchange #4
+                                    WholeStageCodegen (2)
+                                      Filter [ca_address_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.customer_address [ca_address_sk,ca_city,ca_zip]
+                            InputAdapter
+                              BroadcastExchange #2
+                                WholeStageCodegen (3)
+                                  Project [d_date_sk]
+                                    Filter [d_qoy,d_year,d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.date_dim [d_date_sk,d_year,d_qoy]
+                        InputAdapter
+                          BroadcastExchange #5
+                            WholeStageCodegen (4)
+                              Filter [i_item_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.item [i_item_sk,i_item_id]
+                    InputAdapter
+                      BroadcastExchange #6
+                        WholeStageCodegen (5)
+                          Project [i_item_id]
+                            Filter [i_item_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.item [i_item_sk,i_item_id]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.sf100/explain.txt
new file mode 100644
index 0000000000000..ac35640c7b004
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.sf100/explain.txt
@@ -0,0 +1,292 @@
+== Physical Plan ==
+TakeOrderedAndProject (51)
++- * Project (50)
+   +- * SortMergeJoin Inner (49)
+      :- * Sort (14)
+      :  +- Exchange (13)
+      :     +- * Project (12)
+      :        +- * SortMergeJoin Inner (11)
+      :           :- * Sort (5)
+      :           :  +- Exchange (4)
+      :           :     +- * Filter (3)
+      :           :        +- * ColumnarToRow (2)
+      :           :           +- Scan parquet default.customer (1)
+      :           +- * Sort (10)
+      :              +- Exchange (9)
+      :                 +- * Filter (8)
+      :                    +- * ColumnarToRow (7)
+      :                       +- Scan parquet default.customer_address (6)
+      +- * Sort (48)
+         +- Exchange (47)
+            +- * HashAggregate (46)
+               +- * HashAggregate (45)
+                  +- * Project (44)
+                     +- * SortMergeJoin Inner (43)
+                        :- * Sort (40)
+                        :  +- Exchange (39)
+                        :     +- * Project (38)
+                        :        +- * BroadcastHashJoin Inner BuildRight (37)
+                        :           :- * Project (31)
+                        :           :  +- * BroadcastHashJoin Inner BuildRight (30)
+                        :           :     :- * Project (24)
+                        :           :     :  +- * BroadcastHashJoin Inner BuildRight (23)
+                        :           :     :     :- * Filter (17)
+                        :           :     :     :  +- * ColumnarToRow (16)
+                        :           :     :     :     +- Scan parquet default.store_sales (15)
+                        :           :     :     +- BroadcastExchange (22)
+                        :           :     :        +- * Project (21)
+                        :           :     :           +- * Filter (20)
+                        :           :     :              +- * ColumnarToRow (19)
+                        :           :     :                 +- Scan parquet default.date_dim (18)
+                        :           :     +- BroadcastExchange (29)
+                        :           :        +- * Project (28)
+                        :           :           +- * Filter (27)
+                        :           :              +- * ColumnarToRow (26)
+                        :           :                 +- Scan parquet default.store (25)
+                        :           +- BroadcastExchange (36)
+                        :              +- * Project (35)
+                        :                 +- * Filter (34)
+                        :                    +- * ColumnarToRow (33)
+                        :                       +- Scan parquet default.household_demographics (32)
+                        +- * Sort (42)
+                           +- ReusedExchange (41)
+
+
+(1) Scan parquet default.customer
+Output [4]: [c_customer_sk#1, c_current_addr_sk#2, c_first_name#3, c_last_name#4]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int,c_first_name:string,c_last_name:string>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [4]: [c_customer_sk#1, c_current_addr_sk#2, c_first_name#3, c_last_name#4]
+
+(3) Filter [codegen id : 1]
+Input [4]: [c_customer_sk#1, c_current_addr_sk#2, c_first_name#3, c_last_name#4]
+Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_current_addr_sk#2))
+
+(4) Exchange
+Input [4]: [c_customer_sk#1, c_current_addr_sk#2, c_first_name#3, c_last_name#4]
+Arguments: hashpartitioning(c_current_addr_sk#2, 5), ENSURE_REQUIREMENTS, [id=#5]
+
+(5) Sort [codegen id : 2]
+Input [4]: [c_customer_sk#1, c_current_addr_sk#2, c_first_name#3, c_last_name#4]
+Arguments: [c_current_addr_sk#2 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#6, ca_city#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)]
+ReadSchema: struct<ca_address_sk:int,ca_city:string>
+
+(7) ColumnarToRow [codegen id : 3]
+Input [2]: [ca_address_sk#6, ca_city#7]
+
+(8) Filter [codegen id : 3]
+Input [2]: [ca_address_sk#6, ca_city#7]
+Condition : (isnotnull(ca_address_sk#6) AND isnotnull(ca_city#7))
+
+(9) Exchange
+Input [2]: [ca_address_sk#6, ca_city#7]
+Arguments: hashpartitioning(ca_address_sk#6, 5), ENSURE_REQUIREMENTS, [id=#8]
+
+(10) Sort [codegen id : 4]
+Input [2]: [ca_address_sk#6, ca_city#7]
+Arguments: [ca_address_sk#6 ASC NULLS FIRST], false, 0
+
+(11) SortMergeJoin [codegen id : 5]
+Left keys [1]: [c_current_addr_sk#2]
+Right keys [1]: [ca_address_sk#6]
+Join condition: None
+
+(12) Project [codegen id : 5]
+Output [4]: [c_customer_sk#1, c_first_name#3, c_last_name#4, ca_city#7]
+Input [6]: [c_customer_sk#1, c_current_addr_sk#2, c_first_name#3, c_last_name#4, ca_address_sk#6, ca_city#7]
+
+(13) Exchange
+Input [4]: [c_customer_sk#1, c_first_name#3, c_last_name#4, ca_city#7]
+Arguments: hashpartitioning(c_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#9]
+
+(14) Sort [codegen id : 6]
+Input [4]: [c_customer_sk#1, c_first_name#3, c_last_name#4, ca_city#7]
+Arguments: [c_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(15) Scan parquet default.store_sales
+Output [8]: [ss_customer_sk#10, ss_hdemo_sk#11, ss_addr_sk#12, ss_store_sk#13, ss_ticket_number#14, ss_coupon_amt#15, ss_net_profit#16, ss_sold_date_sk#17]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#17), dynamicpruningexpression(ss_sold_date_sk#17 IN dynamicpruning#18)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_hdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_coupon_amt:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(16) ColumnarToRow [codegen id : 10]
+Input [8]: [ss_customer_sk#10, ss_hdemo_sk#11, ss_addr_sk#12, ss_store_sk#13, ss_ticket_number#14, ss_coupon_amt#15, ss_net_profit#16, ss_sold_date_sk#17]
+
+(17) Filter [codegen id : 10]
+Input [8]: [ss_customer_sk#10, ss_hdemo_sk#11, ss_addr_sk#12, ss_store_sk#13, ss_ticket_number#14, ss_coupon_amt#15, ss_net_profit#16, ss_sold_date_sk#17]
+Condition : (((isnotnull(ss_store_sk#13) AND isnotnull(ss_hdemo_sk#11)) AND isnotnull(ss_addr_sk#12)) AND isnotnull(ss_customer_sk#10))
+
+(18) Scan parquet default.date_dim
+Output [3]: [d_date_sk#19, d_year#20, d_dow#21]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [In(d_dow, [6,0]), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_dow:int>
+
+(19) ColumnarToRow [codegen id : 7]
+Input [3]: [d_date_sk#19, d_year#20, d_dow#21]
+
+(20) Filter [codegen id : 7]
+Input [3]: [d_date_sk#19, d_year#20, d_dow#21]
+Condition : ((d_dow#21 IN (6,0) AND d_year#20 IN (1999,2000,2001)) AND isnotnull(d_date_sk#19))
+
+(21) Project [codegen id : 7]
+Output [1]: [d_date_sk#19]
+Input [3]: [d_date_sk#19, d_year#20, d_dow#21]
+
+(22) BroadcastExchange
+Input [1]: [d_date_sk#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#22]
+
+(23) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_sold_date_sk#17]
+Right keys [1]: [d_date_sk#19]
+Join condition: None
+
+(24) Project [codegen id : 10]
+Output [7]: [ss_customer_sk#10, ss_hdemo_sk#11, ss_addr_sk#12, ss_store_sk#13, ss_ticket_number#14, ss_coupon_amt#15, ss_net_profit#16]
+Input [9]: [ss_customer_sk#10, ss_hdemo_sk#11, ss_addr_sk#12, ss_store_sk#13, ss_ticket_number#14, ss_coupon_amt#15, ss_net_profit#16, ss_sold_date_sk#17, d_date_sk#19]
+
+(25) Scan parquet default.store
+Output [2]: [s_store_sk#23, s_city#24]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [In(s_city, [Fairview,Midway]), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_city:string>
+
+(26) ColumnarToRow [codegen id : 8]
+Input [2]: [s_store_sk#23, s_city#24]
+
+(27) Filter [codegen id : 8]
+Input [2]: [s_store_sk#23, s_city#24]
+Condition : (s_city#24 IN (Fairview,Midway) AND isnotnull(s_store_sk#23))
+
+(28) Project [codegen id : 8]
+Output [1]: [s_store_sk#23]
+Input [2]: [s_store_sk#23, s_city#24]
+
+(29) BroadcastExchange
+Input [1]: [s_store_sk#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#25]
+
+(30) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_store_sk#13]
+Right keys [1]: [s_store_sk#23]
+Join condition: None
+
+(31) Project [codegen id : 10]
+Output [6]: [ss_customer_sk#10, ss_hdemo_sk#11, ss_addr_sk#12, ss_ticket_number#14, ss_coupon_amt#15, ss_net_profit#16]
+Input [8]: [ss_customer_sk#10, ss_hdemo_sk#11, ss_addr_sk#12, ss_store_sk#13, ss_ticket_number#14, ss_coupon_amt#15, ss_net_profit#16, s_store_sk#23]
+
+(32) Scan parquet default.household_demographics
+Output [3]: [hd_demo_sk#26, hd_dep_count#27, hd_vehicle_count#28]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_dep_count:int,hd_vehicle_count:int>
+
+(33) ColumnarToRow [codegen id : 9]
+Input [3]: [hd_demo_sk#26, hd_dep_count#27, hd_vehicle_count#28]
+
+(34) Filter [codegen id : 9]
+Input [3]: [hd_demo_sk#26, hd_dep_count#27, hd_vehicle_count#28]
+Condition : (((hd_dep_count#27 = 4) OR (hd_vehicle_count#28 = 3)) AND isnotnull(hd_demo_sk#26))
+
+(35) Project [codegen id : 9]
+Output [1]: [hd_demo_sk#26]
+Input [3]: [hd_demo_sk#26, hd_dep_count#27, hd_vehicle_count#28]
+
+(36) BroadcastExchange
+Input [1]: [hd_demo_sk#26]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#29]
+
+(37) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_hdemo_sk#11]
+Right keys [1]: [hd_demo_sk#26]
+Join condition: None
+
+(38) Project [codegen id : 10]
+Output [5]: [ss_customer_sk#10, ss_addr_sk#12, ss_ticket_number#14, ss_coupon_amt#15, ss_net_profit#16]
+Input [7]: [ss_customer_sk#10, ss_hdemo_sk#11, ss_addr_sk#12, ss_ticket_number#14, ss_coupon_amt#15, ss_net_profit#16, hd_demo_sk#26]
+
+(39) Exchange
+Input [5]: [ss_customer_sk#10, ss_addr_sk#12, ss_ticket_number#14, ss_coupon_amt#15, ss_net_profit#16]
+Arguments: hashpartitioning(ss_addr_sk#12, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(40) Sort [codegen id : 11]
+Input [5]: [ss_customer_sk#10, ss_addr_sk#12, ss_ticket_number#14, ss_coupon_amt#15, ss_net_profit#16]
+Arguments: [ss_addr_sk#12 ASC NULLS FIRST], false, 0
+
+(41) ReusedExchange [Reuses operator id: 9]
+Output [2]: [ca_address_sk#31, ca_city#32]
+
+(42) Sort [codegen id : 13]
+Input [2]: [ca_address_sk#31, ca_city#32]
+Arguments: [ca_address_sk#31 ASC NULLS FIRST], false, 0
+
+(43) SortMergeJoin [codegen id : 14]
+Left keys [1]: [ss_addr_sk#12]
+Right keys [1]: [ca_address_sk#31]
+Join condition: None
+
+(44) Project [codegen id : 14]
+Output [6]: [ss_customer_sk#10, ss_addr_sk#12, ss_ticket_number#14, ss_coupon_amt#15, ss_net_profit#16, ca_city#32]
+Input [7]: [ss_customer_sk#10, ss_addr_sk#12, ss_ticket_number#14, ss_coupon_amt#15, ss_net_profit#16, ca_address_sk#31, ca_city#32]
+
+(45) HashAggregate [codegen id : 14]
+Input [6]: [ss_customer_sk#10, ss_addr_sk#12, ss_ticket_number#14, ss_coupon_amt#15, ss_net_profit#16, ca_city#32]
+Keys [4]: [ss_ticket_number#14, ss_customer_sk#10, ss_addr_sk#12, ca_city#32]
+Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#15)), partial_sum(UnscaledValue(ss_net_profit#16))]
+Aggregate Attributes [2]: [sum#33, sum#34]
+Results [6]: [ss_ticket_number#14, ss_customer_sk#10, ss_addr_sk#12, ca_city#32, sum#35, sum#36]
+
+(46) HashAggregate [codegen id : 14]
+Input [6]: [ss_ticket_number#14, ss_customer_sk#10, ss_addr_sk#12, ca_city#32, sum#35, sum#36]
+Keys [4]: [ss_ticket_number#14, ss_customer_sk#10, ss_addr_sk#12, ca_city#32]
+Functions [2]: [sum(UnscaledValue(ss_coupon_amt#15)), sum(UnscaledValue(ss_net_profit#16))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#15))#37, sum(UnscaledValue(ss_net_profit#16))#38]
+Results [5]: [ss_ticket_number#14, ss_customer_sk#10, ca_city#32 AS bought_city#39, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#15))#37,17,2) AS amt#40, MakeDecimal(sum(UnscaledValue(ss_net_profit#16))#38,17,2) AS profit#41]
+
+(47) Exchange
+Input [5]: [ss_ticket_number#14, ss_customer_sk#10, bought_city#39, amt#40, profit#41]
+Arguments: hashpartitioning(ss_customer_sk#10, 5), ENSURE_REQUIREMENTS, [id=#42]
+
+(48) Sort [codegen id : 15]
+Input [5]: [ss_ticket_number#14, ss_customer_sk#10, bought_city#39, amt#40, profit#41]
+Arguments: [ss_customer_sk#10 ASC NULLS FIRST], false, 0
+
+(49) SortMergeJoin [codegen id : 16]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [ss_customer_sk#10]
+Join condition: NOT (ca_city#7 = bought_city#39)
+
+(50) Project [codegen id : 16]
+Output [7]: [c_last_name#4, c_first_name#3, ca_city#7, bought_city#39, ss_ticket_number#14, amt#40, profit#41]
+Input [9]: [c_customer_sk#1, c_first_name#3, c_last_name#4, ca_city#7, ss_ticket_number#14, ss_customer_sk#10, bought_city#39, amt#40, profit#41]
+
+(51) TakeOrderedAndProject
+Input [7]: [c_last_name#4, c_first_name#3, ca_city#7, bought_city#39, ss_ticket_number#14, amt#40, profit#41]
+Arguments: 100, [c_last_name#4 ASC NULLS FIRST, c_first_name#3 ASC NULLS FIRST, ca_city#7 ASC NULLS FIRST, bought_city#39 ASC NULLS FIRST, ss_ticket_number#14 ASC NULLS FIRST], [c_last_name#4, c_first_name#3, ca_city#7, bought_city#39, ss_ticket_number#14, amt#40, profit#41]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 15 Hosting Expression = ss_sold_date_sk#17 IN dynamicpruning#18
+ReusedExchange (52)
+
+
+(52) ReusedExchange [Reuses operator id: 22]
+Output [1]: [d_date_sk#19]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.sf100/simplified.txt
new file mode 100644
index 0000000000000..f4e90335c78f9
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.sf100/simplified.txt
@@ -0,0 +1,89 @@
+TakeOrderedAndProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit]
+  WholeStageCodegen (16)
+    Project [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit]
+      SortMergeJoin [c_customer_sk,ss_customer_sk,ca_city,bought_city]
+        InputAdapter
+          WholeStageCodegen (6)
+            Sort [c_customer_sk]
+              InputAdapter
+                Exchange [c_customer_sk] #1
+                  WholeStageCodegen (5)
+                    Project [c_customer_sk,c_first_name,c_last_name,ca_city]
+                      SortMergeJoin [c_current_addr_sk,ca_address_sk]
+                        InputAdapter
+                          WholeStageCodegen (2)
+                            Sort [c_current_addr_sk]
+                              InputAdapter
+                                Exchange [c_current_addr_sk] #2
+                                  WholeStageCodegen (1)
+                                    Filter [c_customer_sk,c_current_addr_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name]
+                        InputAdapter
+                          WholeStageCodegen (4)
+                            Sort [ca_address_sk]
+                              InputAdapter
+                                Exchange [ca_address_sk] #3
+                                  WholeStageCodegen (3)
+                                    Filter [ca_address_sk,ca_city]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.customer_address [ca_address_sk,ca_city]
+        InputAdapter
+          WholeStageCodegen (15)
+            Sort [ss_customer_sk]
+              InputAdapter
+                Exchange [ss_customer_sk] #4
+                  WholeStageCodegen (14)
+                    HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum] [sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit)),bought_city,amt,profit,sum,sum]
+                      HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,ss_coupon_amt,ss_net_profit] [sum,sum,sum,sum]
+                        Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_city]
+                          SortMergeJoin [ss_addr_sk,ca_address_sk]
+                            InputAdapter
+                              WholeStageCodegen (11)
+                                Sort [ss_addr_sk]
+                                  InputAdapter
+                                    Exchange [ss_addr_sk] #5
+                                      WholeStageCodegen (10)
+                                        Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit]
+                                          BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                            Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit]
+                                              BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit]
+                                                  BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                    Filter [ss_store_sk,ss_hdemo_sk,ss_addr_sk,ss_customer_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk]
+                                                            SubqueryBroadcast [d_date_sk] #1
+                                                              ReusedExchange [d_date_sk] #6
+                                                    InputAdapter
+                                                      BroadcastExchange #6
+                                                        WholeStageCodegen (7)
+                                                          Project [d_date_sk]
+                                                            Filter [d_dow,d_year,d_date_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.date_dim [d_date_sk,d_year,d_dow]
+                                                InputAdapter
+                                                  BroadcastExchange #7
+                                                    WholeStageCodegen (8)
+                                                      Project [s_store_sk]
+                                                        Filter [s_city,s_store_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.store [s_store_sk,s_city]
+                                            InputAdapter
+                                              BroadcastExchange #8
+                                                WholeStageCodegen (9)
+                                                  Project [hd_demo_sk]
+                                                    Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count]
+                            InputAdapter
+                              WholeStageCodegen (13)
+                                Sort [ca_address_sk]
+                                  InputAdapter
+                                    ReusedExchange [ca_address_sk,ca_city] #3
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46/explain.txt
new file mode 100644
index 0000000000000..2532a14d7907f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46/explain.txt
@@ -0,0 +1,252 @@
+== Physical Plan ==
+TakeOrderedAndProject (43)
++- * Project (42)
+   +- * BroadcastHashJoin Inner BuildRight (41)
+      :- * Project (39)
+      :  +- * BroadcastHashJoin Inner BuildRight (38)
+      :     :- * HashAggregate (33)
+      :     :  +- Exchange (32)
+      :     :     +- * HashAggregate (31)
+      :     :        +- * Project (30)
+      :     :           +- * BroadcastHashJoin Inner BuildRight (29)
+      :     :              :- * Project (24)
+      :     :              :  +- * BroadcastHashJoin Inner BuildRight (23)
+      :     :              :     :- * Project (17)
+      :     :              :     :  +- * BroadcastHashJoin Inner BuildRight (16)
+      :     :              :     :     :- * Project (10)
+      :     :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+      :     :              :     :     :     :- * Filter (3)
+      :     :              :     :     :     :  +- * ColumnarToRow (2)
+      :     :              :     :     :     :     +- Scan parquet default.store_sales (1)
+      :     :              :     :     :     +- BroadcastExchange (8)
+      :     :              :     :     :        +- * Project (7)
+      :     :              :     :     :           +- * Filter (6)
+      :     :              :     :     :              +- * ColumnarToRow (5)
+      :     :              :     :     :                 +- Scan parquet default.date_dim (4)
+      :     :              :     :     +- BroadcastExchange (15)
+      :     :              :     :        +- * Project (14)
+      :     :              :     :           +- * Filter (13)
+      :     :              :     :              +- * ColumnarToRow (12)
+      :     :              :     :                 +- Scan parquet default.store (11)
+      :     :              :     +- BroadcastExchange (22)
+      :     :              :        +- * Project (21)
+      :     :              :           +- * Filter (20)
+      :     :              :              +- * ColumnarToRow (19)
+      :     :              :                 +- Scan parquet default.household_demographics (18)
+      :     :              +- BroadcastExchange (28)
+      :     :                 +- * Filter (27)
+      :     :                    +- * ColumnarToRow (26)
+      :     :                       +- Scan parquet default.customer_address (25)
+      :     +- BroadcastExchange (37)
+      :        +- * Filter (36)
+      :           +- * ColumnarToRow (35)
+      :              +- Scan parquet default.customer (34)
+      +- ReusedExchange (40)
+
+
+(1) Scan parquet default.store_sales
+Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_hdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_coupon_amt:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8]
+
+(3) Filter [codegen id : 5]
+Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8]
+Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#10, d_year#11, d_dow#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [In(d_dow, [6,0]), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_dow:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#10, d_year#11, d_dow#12]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#10, d_year#11, d_dow#12]
+Condition : ((d_dow#12 IN (6,0) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#10]
+Input [3]: [d_date_sk#10, d_year#11, d_dow#12]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7]
+Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#10]
+
+(11) Scan parquet default.store
+Output [2]: [s_store_sk#14, s_city#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [In(s_city, [Fairview,Midway]), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_city:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [s_store_sk#14, s_city#15]
+
+(13) Filter [codegen id : 2]
+Input [2]: [s_store_sk#14, s_city#15]
+Condition : (s_city#15 IN (Fairview,Midway) AND isnotnull(s_store_sk#14))
+
+(14) Project [codegen id : 2]
+Output [1]: [s_store_sk#14]
+Input [2]: [s_store_sk#14, s_city#15]
+
+(15) BroadcastExchange
+Input [1]: [s_store_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#16]
+
+(16) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_store_sk#4]
+Right keys [1]: [s_store_sk#14]
+Join condition: None
+
+(17) Project [codegen id : 5]
+Output [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7]
+Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#14]
+
+(18) Scan parquet default.household_demographics
+Output [3]: [hd_demo_sk#17, hd_dep_count#18, hd_vehicle_count#19]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_dep_count:int,hd_vehicle_count:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [3]: [hd_demo_sk#17, hd_dep_count#18, hd_vehicle_count#19]
+
+(20) Filter [codegen id : 3]
+Input [3]: [hd_demo_sk#17, hd_dep_count#18, hd_vehicle_count#19]
+Condition : (((hd_dep_count#18 = 4) OR (hd_vehicle_count#19 = 3)) AND isnotnull(hd_demo_sk#17))
+
+(21) Project [codegen id : 3]
+Output [1]: [hd_demo_sk#17]
+Input [3]: [hd_demo_sk#17, hd_dep_count#18, hd_vehicle_count#19]
+
+(22) BroadcastExchange
+Input [1]: [hd_demo_sk#17]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#20]
+
+(23) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#17]
+Join condition: None
+
+(24) Project [codegen id : 5]
+Output [5]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7]
+Input [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, hd_demo_sk#17]
+
+(25) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#21, ca_city#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)]
+ReadSchema: struct<ca_address_sk:int,ca_city:string>
+
+(26) ColumnarToRow [codegen id : 4]
+Input [2]: [ca_address_sk#21, ca_city#22]
+
+(27) Filter [codegen id : 4]
+Input [2]: [ca_address_sk#21, ca_city#22]
+Condition : (isnotnull(ca_address_sk#21) AND isnotnull(ca_city#22))
+
+(28) BroadcastExchange
+Input [2]: [ca_address_sk#21, ca_city#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#23]
+
+(29) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_addr_sk#3]
+Right keys [1]: [ca_address_sk#21]
+Join condition: None
+
+(30) Project [codegen id : 5]
+Output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#22]
+Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_address_sk#21, ca_city#22]
+
+(31) HashAggregate [codegen id : 5]
+Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#22]
+Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#22]
+Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))]
+Aggregate Attributes [2]: [sum#24, sum#25]
+Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#22, sum#26, sum#27]
+
+(32) Exchange
+Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#22, sum#26, sum#27]
+Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#22, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(33) HashAggregate [codegen id : 8]
+Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#22, sum#26, sum#27]
+Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#22]
+Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#29, sum(UnscaledValue(ss_net_profit#7))#30]
+Results [5]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#22 AS bought_city#31, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#29,17,2) AS amt#32, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#30,17,2) AS profit#33]
+
+(34) Scan parquet default.customer
+Output [4]: [c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int,c_first_name:string,c_last_name:string>
+
+(35) ColumnarToRow [codegen id : 6]
+Input [4]: [c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37]
+
+(36) Filter [codegen id : 6]
+Input [4]: [c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37]
+Condition : (isnotnull(c_customer_sk#34) AND isnotnull(c_current_addr_sk#35))
+
+(37) BroadcastExchange
+Input [4]: [c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#38]
+
+(38) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#34]
+Join condition: None
+
+(39) Project [codegen id : 8]
+Output [7]: [ss_ticket_number#5, bought_city#31, amt#32, profit#33, c_current_addr_sk#35, c_first_name#36, c_last_name#37]
+Input [9]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#31, amt#32, profit#33, c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37]
+
+(40) ReusedExchange [Reuses operator id: 28]
+Output [2]: [ca_address_sk#39, ca_city#40]
+
+(41) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [c_current_addr_sk#35]
+Right keys [1]: [ca_address_sk#39]
+Join condition: NOT (ca_city#40 = bought_city#31)
+
+(42) Project [codegen id : 8]
+Output [7]: [c_last_name#37, c_first_name#36, ca_city#40, bought_city#31, ss_ticket_number#5, amt#32, profit#33]
+Input [9]: [ss_ticket_number#5, bought_city#31, amt#32, profit#33, c_current_addr_sk#35, c_first_name#36, c_last_name#37, ca_address_sk#39, ca_city#40]
+
+(43) TakeOrderedAndProject
+Input [7]: [c_last_name#37, c_first_name#36, ca_city#40, bought_city#31, ss_ticket_number#5, amt#32, profit#33]
+Arguments: 100, [c_last_name#37 ASC NULLS FIRST, c_first_name#36 ASC NULLS FIRST, ca_city#40 ASC NULLS FIRST, bought_city#31 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#37, c_first_name#36, ca_city#40, bought_city#31, ss_ticket_number#5, amt#32, profit#33]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (44)
+
+
+(44) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#10]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46/simplified.txt
new file mode 100644
index 0000000000000..7f729f9874b53
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46/simplified.txt
@@ -0,0 +1,65 @@
+TakeOrderedAndProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit]
+  WholeStageCodegen (8)
+    Project [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit]
+      BroadcastHashJoin [c_current_addr_sk,ca_address_sk,ca_city,bought_city]
+        Project [ss_ticket_number,bought_city,amt,profit,c_current_addr_sk,c_first_name,c_last_name]
+          BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+            HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum] [sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit)),bought_city,amt,profit,sum,sum]
+              InputAdapter
+                Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1
+                  WholeStageCodegen (5)
+                    HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,ss_coupon_amt,ss_net_profit] [sum,sum,sum,sum]
+                      Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_city]
+                        BroadcastHashJoin [ss_addr_sk,ca_address_sk]
+                          Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit]
+                            BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                              Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit]
+                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                  Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit]
+                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                      Filter [ss_store_sk,ss_hdemo_sk,ss_addr_sk,ss_customer_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #1
+                                                ReusedExchange [d_date_sk] #2
+                                      InputAdapter
+                                        BroadcastExchange #2
+                                          WholeStageCodegen (1)
+                                            Project [d_date_sk]
+                                              Filter [d_dow,d_year,d_date_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_date_sk,d_year,d_dow]
+                                  InputAdapter
+                                    BroadcastExchange #3
+                                      WholeStageCodegen (2)
+                                        Project [s_store_sk]
+                                          Filter [s_city,s_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store [s_store_sk,s_city]
+                              InputAdapter
+                                BroadcastExchange #4
+                                  WholeStageCodegen (3)
+                                    Project [hd_demo_sk]
+                                      Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count]
+                          InputAdapter
+                            BroadcastExchange #5
+                              WholeStageCodegen (4)
+                                Filter [ca_address_sk,ca_city]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.customer_address [ca_address_sk,ca_city]
+            InputAdapter
+              BroadcastExchange #6
+                WholeStageCodegen (6)
+                  Filter [c_customer_sk,c_current_addr_sk]
+                    ColumnarToRow
+                      InputAdapter
+                        Scan parquet default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name]
+        InputAdapter
+          ReusedExchange [ca_address_sk,ca_city] #5
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.sf100/explain.txt
new file mode 100644
index 0000000000000..b7ffba01f92c6
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.sf100/explain.txt
@@ -0,0 +1,314 @@
+== Physical Plan ==
+TakeOrderedAndProject (56)
++- * Project (55)
+   +- * SortMergeJoin Inner (54)
+      :- * Project (47)
+      :  +- * SortMergeJoin Inner (46)
+      :     :- * Sort (37)
+      :     :  +- Exchange (36)
+      :     :     +- * Filter (35)
+      :     :        +- Window (34)
+      :     :           +- * Sort (33)
+      :     :              +- Exchange (32)
+      :     :                 +- * Project (31)
+      :     :                    +- Window (30)
+      :     :                       +- * Sort (29)
+      :     :                          +- Exchange (28)
+      :     :                             +- * HashAggregate (27)
+      :     :                                +- Exchange (26)
+      :     :                                   +- * HashAggregate (25)
+      :     :                                      +- * Project (24)
+      :     :                                         +- * SortMergeJoin Inner (23)
+      :     :                                            :- * Sort (17)
+      :     :                                            :  +- Exchange (16)
+      :     :                                            :     +- * Project (15)
+      :     :                                            :        +- * BroadcastHashJoin Inner BuildRight (14)
+      :     :                                            :           :- * Project (9)
+      :     :                                            :           :  +- * BroadcastHashJoin Inner BuildRight (8)
+      :     :                                            :           :     :- * Filter (3)
+      :     :                                            :           :     :  +- * ColumnarToRow (2)
+      :     :                                            :           :     :     +- Scan parquet default.store_sales (1)
+      :     :                                            :           :     +- BroadcastExchange (7)
+      :     :                                            :           :        +- * Filter (6)
+      :     :                                            :           :           +- * ColumnarToRow (5)
+      :     :                                            :           :              +- Scan parquet default.date_dim (4)
+      :     :                                            :           +- BroadcastExchange (13)
+      :     :                                            :              +- * Filter (12)
+      :     :                                            :                 +- * ColumnarToRow (11)
+      :     :                                            :                    +- Scan parquet default.store (10)
+      :     :                                            +- * Sort (22)
+      :     :                                               +- Exchange (21)
+      :     :                                                  +- * Filter (20)
+      :     :                                                     +- * ColumnarToRow (19)
+      :     :                                                        +- Scan parquet default.item (18)
+      :     +- * Sort (45)
+      :        +- Exchange (44)
+      :           +- * Project (43)
+      :              +- Window (42)
+      :                 +- * Sort (41)
+      :                    +- Exchange (40)
+      :                       +- * HashAggregate (39)
+      :                          +- ReusedExchange (38)
+      +- * Sort (53)
+         +- Exchange (52)
+            +- * Project (51)
+               +- Window (50)
+                  +- * Sort (49)
+                     +- ReusedExchange (48)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 3]
+Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, ss_sold_date_sk#4]
+Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Condition : ((((d_year#7 = 1999) OR ((d_year#7 = 1998) AND (d_moy#8 = 12))) OR ((d_year#7 = 2000) AND (d_moy#8 = 1))) AND isnotnull(d_date_sk#6))
+
+(7) BroadcastExchange
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#9]
+
+(8) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(9) Project [codegen id : 3]
+Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, d_year#7, d_moy#8]
+Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, ss_sold_date_sk#4, d_date_sk#6, d_year#7, d_moy#8]
+
+(10) Scan parquet default.store
+Output [3]: [s_store_sk#10, s_store_name#11, s_company_name#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_company_name:string>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [3]: [s_store_sk#10, s_store_name#11, s_company_name#12]
+
+(12) Filter [codegen id : 2]
+Input [3]: [s_store_sk#10, s_store_name#11, s_company_name#12]
+Condition : ((isnotnull(s_store_sk#10) AND isnotnull(s_store_name#11)) AND isnotnull(s_company_name#12))
+
+(13) BroadcastExchange
+Input [3]: [s_store_sk#10, s_store_name#11, s_company_name#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#13]
+
+(14) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_store_sk#2]
+Right keys [1]: [s_store_sk#10]
+Join condition: None
+
+(15) Project [codegen id : 3]
+Output [6]: [ss_item_sk#1, ss_sales_price#3, d_year#7, d_moy#8, s_store_name#11, s_company_name#12]
+Input [8]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, d_year#7, d_moy#8, s_store_sk#10, s_store_name#11, s_company_name#12]
+
+(16) Exchange
+Input [6]: [ss_item_sk#1, ss_sales_price#3, d_year#7, d_moy#8, s_store_name#11, s_company_name#12]
+Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#14]
+
+(17) Sort [codegen id : 4]
+Input [6]: [ss_item_sk#1, ss_sales_price#3, d_year#7, d_moy#8, s_store_name#11, s_company_name#12]
+Arguments: [ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(18) Scan parquet default.item
+Output [3]: [i_item_sk#15, i_brand#16, i_category#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_category:string>
+
+(19) ColumnarToRow [codegen id : 5]
+Input [3]: [i_item_sk#15, i_brand#16, i_category#17]
+
+(20) Filter [codegen id : 5]
+Input [3]: [i_item_sk#15, i_brand#16, i_category#17]
+Condition : ((isnotnull(i_item_sk#15) AND isnotnull(i_category#17)) AND isnotnull(i_brand#16))
+
+(21) Exchange
+Input [3]: [i_item_sk#15, i_brand#16, i_category#17]
+Arguments: hashpartitioning(i_item_sk#15, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(22) Sort [codegen id : 6]
+Input [3]: [i_item_sk#15, i_brand#16, i_category#17]
+Arguments: [i_item_sk#15 ASC NULLS FIRST], false, 0
+
+(23) SortMergeJoin [codegen id : 7]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#15]
+Join condition: None
+
+(24) Project [codegen id : 7]
+Output [7]: [i_brand#16, i_category#17, ss_sales_price#3, d_year#7, d_moy#8, s_store_name#11, s_company_name#12]
+Input [9]: [ss_item_sk#1, ss_sales_price#3, d_year#7, d_moy#8, s_store_name#11, s_company_name#12, i_item_sk#15, i_brand#16, i_category#17]
+
+(25) HashAggregate [codegen id : 7]
+Input [7]: [i_brand#16, i_category#17, ss_sales_price#3, d_year#7, d_moy#8, s_store_name#11, s_company_name#12]
+Keys [6]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#3))]
+Aggregate Attributes [1]: [sum#19]
+Results [7]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum#20]
+
+(26) Exchange
+Input [7]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum#20]
+Arguments: hashpartitioning(i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(27) HashAggregate [codegen id : 8]
+Input [7]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum#20]
+Keys [6]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#3))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#3))#22]
+Results [8]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, MakeDecimal(sum(UnscaledValue(ss_sales_price#3))#22,17,2) AS sum_sales#23, MakeDecimal(sum(UnscaledValue(ss_sales_price#3))#22,17,2) AS _w0#24]
+
+(28) Exchange
+Input [8]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, _w0#24]
+Arguments: hashpartitioning(i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(29) Sort [codegen id : 9]
+Input [8]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, _w0#24]
+Arguments: [i_category#17 ASC NULLS FIRST, i_brand#16 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, s_company_name#12 ASC NULLS FIRST, d_year#7 ASC NULLS FIRST], false, 0
+
+(30) Window
+Input [8]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, _w0#24]
+Arguments: [avg(_w0#24) windowspecdefinition(i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#26], [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7]
+
+(31) Project [codegen id : 10]
+Output [8]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, avg_monthly_sales#26]
+Input [9]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, _w0#24, avg_monthly_sales#26]
+
+(32) Exchange
+Input [8]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, avg_monthly_sales#26]
+Arguments: hashpartitioning(i_category#17, i_brand#16, s_store_name#11, s_company_name#12, 5), ENSURE_REQUIREMENTS, [id=#27]
+
+(33) Sort [codegen id : 11]
+Input [8]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, avg_monthly_sales#26]
+Arguments: [i_category#17 ASC NULLS FIRST, i_brand#16 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, s_company_name#12 ASC NULLS FIRST, d_year#7 ASC NULLS FIRST, d_moy#8 ASC NULLS FIRST], false, 0
+
+(34) Window
+Input [8]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, avg_monthly_sales#26]
+Arguments: [rank(d_year#7, d_moy#8) windowspecdefinition(i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7 ASC NULLS FIRST, d_moy#8 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#28], [i_category#17, i_brand#16, s_store_name#11, s_company_name#12], [d_year#7 ASC NULLS FIRST, d_moy#8 ASC NULLS FIRST]
+
+(35) Filter [codegen id : 12]
+Input [9]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, avg_monthly_sales#26, rn#28]
+Condition : ((((isnotnull(d_year#7) AND isnotnull(avg_monthly_sales#26)) AND (d_year#7 = 1999)) AND (avg_monthly_sales#26 > 0.000000)) AND (CheckOverflow((promote_precision(abs(CheckOverflow((promote_precision(cast(sum_sales#23 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#26 as decimal(22,6)))), DecimalType(22,6), true), false)) / promote_precision(cast(avg_monthly_sales#26 as decimal(22,6)))), DecimalType(38,16), true) > 0.1000000000000000))
+
+(36) Exchange
+Input [9]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, avg_monthly_sales#26, rn#28]
+Arguments: hashpartitioning(i_category#17, i_brand#16, s_store_name#11, s_company_name#12, rn#28, 5), ENSURE_REQUIREMENTS, [id=#29]
+
+(37) Sort [codegen id : 13]
+Input [9]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, avg_monthly_sales#26, rn#28]
+Arguments: [i_category#17 ASC NULLS FIRST, i_brand#16 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, s_company_name#12 ASC NULLS FIRST, rn#28 ASC NULLS FIRST], false, 0
+
+(38) ReusedExchange [Reuses operator id: unknown]
+Output [7]: [i_category#30, i_brand#31, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum#36]
+
+(39) HashAggregate [codegen id : 21]
+Input [7]: [i_category#30, i_brand#31, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum#36]
+Keys [6]: [i_category#30, i_brand#31, s_store_name#32, s_company_name#33, d_year#34, d_moy#35]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#37))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#37))#38]
+Results [7]: [i_category#30, i_brand#31, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, MakeDecimal(sum(UnscaledValue(ss_sales_price#37))#38,17,2) AS sum_sales#39]
+
+(40) Exchange
+Input [7]: [i_category#30, i_brand#31, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#39]
+Arguments: hashpartitioning(i_category#30, i_brand#31, s_store_name#32, s_company_name#33, 5), ENSURE_REQUIREMENTS, [id=#40]
+
+(41) Sort [codegen id : 22]
+Input [7]: [i_category#30, i_brand#31, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#39]
+Arguments: [i_category#30 ASC NULLS FIRST, i_brand#31 ASC NULLS FIRST, s_store_name#32 ASC NULLS FIRST, s_company_name#33 ASC NULLS FIRST, d_year#34 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST], false, 0
+
+(42) Window
+Input [7]: [i_category#30, i_brand#31, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#39]
+Arguments: [rank(d_year#34, d_moy#35) windowspecdefinition(i_category#30, i_brand#31, s_store_name#32, s_company_name#33, d_year#34 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#41], [i_category#30, i_brand#31, s_store_name#32, s_company_name#33], [d_year#34 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST]
+
+(43) Project [codegen id : 23]
+Output [6]: [i_category#30, i_brand#31, s_store_name#32, s_company_name#33, sum_sales#39, rn#41]
+Input [8]: [i_category#30, i_brand#31, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#39, rn#41]
+
+(44) Exchange
+Input [6]: [i_category#30, i_brand#31, s_store_name#32, s_company_name#33, sum_sales#39, rn#41]
+Arguments: hashpartitioning(i_category#30, i_brand#31, s_store_name#32, s_company_name#33, (rn#41 + 1), 5), ENSURE_REQUIREMENTS, [id=#42]
+
+(45) Sort [codegen id : 24]
+Input [6]: [i_category#30, i_brand#31, s_store_name#32, s_company_name#33, sum_sales#39, rn#41]
+Arguments: [i_category#30 ASC NULLS FIRST, i_brand#31 ASC NULLS FIRST, s_store_name#32 ASC NULLS FIRST, s_company_name#33 ASC NULLS FIRST, (rn#41 + 1) ASC NULLS FIRST], false, 0
+
+(46) SortMergeJoin [codegen id : 25]
+Left keys [5]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, rn#28]
+Right keys [5]: [i_category#30, i_brand#31, s_store_name#32, s_company_name#33, (rn#41 + 1)]
+Join condition: None
+
+(47) Project [codegen id : 25]
+Output [10]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, avg_monthly_sales#26, rn#28, sum_sales#39]
+Input [15]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, avg_monthly_sales#26, rn#28, i_category#30, i_brand#31, s_store_name#32, s_company_name#33, sum_sales#39, rn#41]
+
+(48) ReusedExchange [Reuses operator id: 40]
+Output [7]: [i_category#43, i_brand#44, s_store_name#45, s_company_name#46, d_year#47, d_moy#48, sum_sales#49]
+
+(49) Sort [codegen id : 34]
+Input [7]: [i_category#43, i_brand#44, s_store_name#45, s_company_name#46, d_year#47, d_moy#48, sum_sales#49]
+Arguments: [i_category#43 ASC NULLS FIRST, i_brand#44 ASC NULLS FIRST, s_store_name#45 ASC NULLS FIRST, s_company_name#46 ASC NULLS FIRST, d_year#47 ASC NULLS FIRST, d_moy#48 ASC NULLS FIRST], false, 0
+
+(50) Window
+Input [7]: [i_category#43, i_brand#44, s_store_name#45, s_company_name#46, d_year#47, d_moy#48, sum_sales#49]
+Arguments: [rank(d_year#47, d_moy#48) windowspecdefinition(i_category#43, i_brand#44, s_store_name#45, s_company_name#46, d_year#47 ASC NULLS FIRST, d_moy#48 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#50], [i_category#43, i_brand#44, s_store_name#45, s_company_name#46], [d_year#47 ASC NULLS FIRST, d_moy#48 ASC NULLS FIRST]
+
+(51) Project [codegen id : 35]
+Output [6]: [i_category#43, i_brand#44, s_store_name#45, s_company_name#46, sum_sales#49, rn#50]
+Input [8]: [i_category#43, i_brand#44, s_store_name#45, s_company_name#46, d_year#47, d_moy#48, sum_sales#49, rn#50]
+
+(52) Exchange
+Input [6]: [i_category#43, i_brand#44, s_store_name#45, s_company_name#46, sum_sales#49, rn#50]
+Arguments: hashpartitioning(i_category#43, i_brand#44, s_store_name#45, s_company_name#46, (rn#50 - 1), 5), ENSURE_REQUIREMENTS, [id=#51]
+
+(53) Sort [codegen id : 36]
+Input [6]: [i_category#43, i_brand#44, s_store_name#45, s_company_name#46, sum_sales#49, rn#50]
+Arguments: [i_category#43 ASC NULLS FIRST, i_brand#44 ASC NULLS FIRST, s_store_name#45 ASC NULLS FIRST, s_company_name#46 ASC NULLS FIRST, (rn#50 - 1) ASC NULLS FIRST], false, 0
+
+(54) SortMergeJoin [codegen id : 37]
+Left keys [5]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, rn#28]
+Right keys [5]: [i_category#43, i_brand#44, s_store_name#45, s_company_name#46, (rn#50 - 1)]
+Join condition: None
+
+(55) Project [codegen id : 37]
+Output [10]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, avg_monthly_sales#26, sum_sales#23, sum_sales#39 AS psum#52, sum_sales#49 AS nsum#53]
+Input [16]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, avg_monthly_sales#26, rn#28, sum_sales#39, i_category#43, i_brand#44, s_store_name#45, s_company_name#46, sum_sales#49, rn#50]
+
+(56) TakeOrderedAndProject
+Input [10]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, avg_monthly_sales#26, sum_sales#23, psum#52, nsum#53]
+Arguments: 100, [CheckOverflow((promote_precision(cast(sum_sales#23 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#26 as decimal(22,6)))), DecimalType(22,6), true) ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST], [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, avg_monthly_sales#26, sum_sales#23, psum#52, nsum#53]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (57)
+
+
+(57) ReusedExchange [Reuses operator id: 7]
+Output [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.sf100/simplified.txt
new file mode 100644
index 0000000000000..71c344d7e240a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.sf100/simplified.txt
@@ -0,0 +1,107 @@
+TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_brand,s_company_name,d_year,d_moy,psum,nsum]
+  WholeStageCodegen (37)
+    Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales]
+      SortMergeJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn]
+        InputAdapter
+          WholeStageCodegen (25)
+            Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales]
+              SortMergeJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn]
+                InputAdapter
+                  WholeStageCodegen (13)
+                    Sort [i_category,i_brand,s_store_name,s_company_name,rn]
+                      InputAdapter
+                        Exchange [i_category,i_brand,s_store_name,s_company_name,rn] #1
+                          WholeStageCodegen (12)
+                            Filter [d_year,avg_monthly_sales,sum_sales]
+                              InputAdapter
+                                Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name]
+                                  WholeStageCodegen (11)
+                                    Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy]
+                                      InputAdapter
+                                        Exchange [i_category,i_brand,s_store_name,s_company_name] #2
+                                          WholeStageCodegen (10)
+                                            Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales]
+                                              InputAdapter
+                                                Window [_w0,i_category,i_brand,s_store_name,s_company_name,d_year]
+                                                  WholeStageCodegen (9)
+                                                    Sort [i_category,i_brand,s_store_name,s_company_name,d_year]
+                                                      InputAdapter
+                                                        Exchange [i_category,i_brand,s_store_name,s_company_name,d_year] #3
+                                                          WholeStageCodegen (8)
+                                                            HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum]
+                                                              InputAdapter
+                                                                Exchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #4
+                                                                  WholeStageCodegen (7)
+                                                                    HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,ss_sales_price] [sum,sum]
+                                                                      Project [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name]
+                                                                        SortMergeJoin [ss_item_sk,i_item_sk]
+                                                                          InputAdapter
+                                                                            WholeStageCodegen (4)
+                                                                              Sort [ss_item_sk]
+                                                                                InputAdapter
+                                                                                  Exchange [ss_item_sk] #5
+                                                                                    WholeStageCodegen (3)
+                                                                                      Project [ss_item_sk,ss_sales_price,d_year,d_moy,s_store_name,s_company_name]
+                                                                                        BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                                                          Project [ss_item_sk,ss_store_sk,ss_sales_price,d_year,d_moy]
+                                                                                            BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                              Filter [ss_item_sk,ss_store_sk]
+                                                                                                ColumnarToRow
+                                                                                                  InputAdapter
+                                                                                                    Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                                                                                      SubqueryBroadcast [d_date_sk] #1
+                                                                                                        ReusedExchange [d_date_sk,d_year,d_moy] #6
+                                                                                              InputAdapter
+                                                                                                BroadcastExchange #6
+                                                                                                  WholeStageCodegen (1)
+                                                                                                    Filter [d_year,d_moy,d_date_sk]
+                                                                                                      ColumnarToRow
+                                                                                                        InputAdapter
+                                                                                                          Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                                                                          InputAdapter
+                                                                                            BroadcastExchange #7
+                                                                                              WholeStageCodegen (2)
+                                                                                                Filter [s_store_sk,s_store_name,s_company_name]
+                                                                                                  ColumnarToRow
+                                                                                                    InputAdapter
+                                                                                                      Scan parquet default.store [s_store_sk,s_store_name,s_company_name]
+                                                                          InputAdapter
+                                                                            WholeStageCodegen (6)
+                                                                              Sort [i_item_sk]
+                                                                                InputAdapter
+                                                                                  Exchange [i_item_sk] #8
+                                                                                    WholeStageCodegen (5)
+                                                                                      Filter [i_item_sk,i_category,i_brand]
+                                                                                        ColumnarToRow
+                                                                                          InputAdapter
+                                                                                            Scan parquet default.item [i_item_sk,i_brand,i_category]
+                InputAdapter
+                  WholeStageCodegen (24)
+                    Sort [i_category,i_brand,s_store_name,s_company_name,rn]
+                      InputAdapter
+                        Exchange [i_category,i_brand,s_store_name,s_company_name,rn] #9
+                          WholeStageCodegen (23)
+                            Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn]
+                              InputAdapter
+                                Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name]
+                                  WholeStageCodegen (22)
+                                    Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy]
+                                      InputAdapter
+                                        Exchange [i_category,i_brand,s_store_name,s_company_name] #10
+                                          WholeStageCodegen (21)
+                                            HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,sum]
+                                              InputAdapter
+                                                ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #11
+        InputAdapter
+          WholeStageCodegen (36)
+            Sort [i_category,i_brand,s_store_name,s_company_name,rn]
+              InputAdapter
+                Exchange [i_category,i_brand,s_store_name,s_company_name,rn] #12
+                  WholeStageCodegen (35)
+                    Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn]
+                      InputAdapter
+                        Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name]
+                          WholeStageCodegen (34)
+                            Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy]
+                              InputAdapter
+                                ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #10
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/explain.txt
new file mode 100644
index 0000000000000..917b21c247449
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/explain.txt
@@ -0,0 +1,279 @@
+== Physical Plan ==
+TakeOrderedAndProject (49)
++- * Project (48)
+   +- * BroadcastHashJoin Inner BuildRight (47)
+      :- * Project (41)
+      :  +- * BroadcastHashJoin Inner BuildRight (40)
+      :     :- * Filter (32)
+      :     :  +- Window (31)
+      :     :     +- * Sort (30)
+      :     :        +- Exchange (29)
+      :     :           +- * Project (28)
+      :     :              +- Window (27)
+      :     :                 +- * Sort (26)
+      :     :                    +- Exchange (25)
+      :     :                       +- * HashAggregate (24)
+      :     :                          +- Exchange (23)
+      :     :                             +- * HashAggregate (22)
+      :     :                                +- * Project (21)
+      :     :                                   +- * BroadcastHashJoin Inner BuildRight (20)
+      :     :                                      :- * Project (15)
+      :     :                                      :  +- * BroadcastHashJoin Inner BuildRight (14)
+      :     :                                      :     :- * Project (9)
+      :     :                                      :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+      :     :                                      :     :     :- * Filter (3)
+      :     :                                      :     :     :  +- * ColumnarToRow (2)
+      :     :                                      :     :     :     +- Scan parquet default.item (1)
+      :     :                                      :     :     +- BroadcastExchange (7)
+      :     :                                      :     :        +- * Filter (6)
+      :     :                                      :     :           +- * ColumnarToRow (5)
+      :     :                                      :     :              +- Scan parquet default.store_sales (4)
+      :     :                                      :     +- BroadcastExchange (13)
+      :     :                                      :        +- * Filter (12)
+      :     :                                      :           +- * ColumnarToRow (11)
+      :     :                                      :              +- Scan parquet default.date_dim (10)
+      :     :                                      +- BroadcastExchange (19)
+      :     :                                         +- * Filter (18)
+      :     :                                            +- * ColumnarToRow (17)
+      :     :                                               +- Scan parquet default.store (16)
+      :     +- BroadcastExchange (39)
+      :        +- * Project (38)
+      :           +- Window (37)
+      :              +- * Sort (36)
+      :                 +- Exchange (35)
+      :                    +- * HashAggregate (34)
+      :                       +- ReusedExchange (33)
+      +- BroadcastExchange (46)
+         +- * Project (45)
+            +- Window (44)
+               +- * Sort (43)
+                  +- ReusedExchange (42)
+
+
+(1) Scan parquet default.item
+Output [3]: [i_item_sk#1, i_brand#2, i_category#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_category:string>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [3]: [i_item_sk#1, i_brand#2, i_category#3]
+
+(3) Filter [codegen id : 4]
+Input [3]: [i_item_sk#1, i_brand#2, i_category#3]
+Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2))
+
+(4) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7]
+
+(6) Filter [codegen id : 1]
+Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7]
+Condition : (isnotnull(ss_item_sk#4) AND isnotnull(ss_store_sk#5))
+
+(7) BroadcastExchange
+Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#9]
+
+(8) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [ss_item_sk#4]
+Join condition: None
+
+(9) Project [codegen id : 4]
+Output [5]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7]
+Input [7]: [i_item_sk#1, i_brand#2, i_category#3, ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7]
+
+(10) Scan parquet default.date_dim
+Output [3]: [d_date_sk#10, d_year#11, d_moy#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+
+(12) Filter [codegen id : 2]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10))
+
+(13) BroadcastExchange
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#13]
+
+(14) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#7]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(15) Project [codegen id : 4]
+Output [6]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#11, d_moy#12]
+Input [8]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#10, d_year#11, d_moy#12]
+
+(16) Scan parquet default.store
+Output [3]: [s_store_sk#14, s_store_name#15, s_company_name#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_company_name:string>
+
+(17) ColumnarToRow [codegen id : 3]
+Input [3]: [s_store_sk#14, s_store_name#15, s_company_name#16]
+
+(18) Filter [codegen id : 3]
+Input [3]: [s_store_sk#14, s_store_name#15, s_company_name#16]
+Condition : ((isnotnull(s_store_sk#14) AND isnotnull(s_store_name#15)) AND isnotnull(s_company_name#16))
+
+(19) BroadcastExchange
+Input [3]: [s_store_sk#14, s_store_name#15, s_company_name#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#17]
+
+(20) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#5]
+Right keys [1]: [s_store_sk#14]
+Join condition: None
+
+(21) Project [codegen id : 4]
+Output [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#11, d_moy#12, s_store_name#15, s_company_name#16]
+Input [9]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#11, d_moy#12, s_store_sk#14, s_store_name#15, s_company_name#16]
+
+(22) HashAggregate [codegen id : 4]
+Input [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#11, d_moy#12, s_store_name#15, s_company_name#16]
+Keys [6]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#6))]
+Aggregate Attributes [1]: [sum#18]
+Results [7]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum#19]
+
+(23) Exchange
+Input [7]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum#19]
+Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(24) HashAggregate [codegen id : 5]
+Input [7]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum#19]
+Keys [6]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#6))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#6))#21]
+Results [8]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#21,17,2) AS sum_sales#22, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#21,17,2) AS _w0#23]
+
+(25) Exchange
+Input [8]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum_sales#22, _w0#23]
+Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(26) Sort [codegen id : 6]
+Input [8]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum_sales#22, _w0#23]
+Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#15 ASC NULLS FIRST, s_company_name#16 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST], false, 0
+
+(27) Window
+Input [8]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum_sales#22, _w0#23]
+Arguments: [avg(_w0#23) windowspecdefinition(i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#25], [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11]
+
+(28) Project [codegen id : 7]
+Output [8]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum_sales#22, avg_monthly_sales#25]
+Input [9]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum_sales#22, _w0#23, avg_monthly_sales#25]
+
+(29) Exchange
+Input [8]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum_sales#22, avg_monthly_sales#25]
+Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#15, s_company_name#16, 5), ENSURE_REQUIREMENTS, [id=#26]
+
+(30) Sort [codegen id : 8]
+Input [8]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum_sales#22, avg_monthly_sales#25]
+Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#15 ASC NULLS FIRST, s_company_name#16 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST], false, 0
+
+(31) Window
+Input [8]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum_sales#22, avg_monthly_sales#25]
+Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#3, i_brand#2, s_store_name#15, s_company_name#16], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST]
+
+(32) Filter [codegen id : 23]
+Input [9]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum_sales#22, avg_monthly_sales#25, rn#27]
+Condition : ((((isnotnull(d_year#11) AND isnotnull(avg_monthly_sales#25)) AND (d_year#11 = 1999)) AND (avg_monthly_sales#25 > 0.000000)) AND (CheckOverflow((promote_precision(abs(CheckOverflow((promote_precision(cast(sum_sales#22 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#25 as decimal(22,6)))), DecimalType(22,6), true), false)) / promote_precision(cast(avg_monthly_sales#25 as decimal(22,6)))), DecimalType(38,16), true) > 0.1000000000000000))
+
+(33) ReusedExchange [Reuses operator id: unknown]
+Output [7]: [i_category#28, i_brand#29, s_store_name#30, s_company_name#31, d_year#32, d_moy#33, sum#34]
+
+(34) HashAggregate [codegen id : 13]
+Input [7]: [i_category#28, i_brand#29, s_store_name#30, s_company_name#31, d_year#32, d_moy#33, sum#34]
+Keys [6]: [i_category#28, i_brand#29, s_store_name#30, s_company_name#31, d_year#32, d_moy#33]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#35))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#35))#36]
+Results [7]: [i_category#28, i_brand#29, s_store_name#30, s_company_name#31, d_year#32, d_moy#33, MakeDecimal(sum(UnscaledValue(ss_sales_price#35))#36,17,2) AS sum_sales#37]
+
+(35) Exchange
+Input [7]: [i_category#28, i_brand#29, s_store_name#30, s_company_name#31, d_year#32, d_moy#33, sum_sales#37]
+Arguments: hashpartitioning(i_category#28, i_brand#29, s_store_name#30, s_company_name#31, 5), ENSURE_REQUIREMENTS, [id=#38]
+
+(36) Sort [codegen id : 14]
+Input [7]: [i_category#28, i_brand#29, s_store_name#30, s_company_name#31, d_year#32, d_moy#33, sum_sales#37]
+Arguments: [i_category#28 ASC NULLS FIRST, i_brand#29 ASC NULLS FIRST, s_store_name#30 ASC NULLS FIRST, s_company_name#31 ASC NULLS FIRST, d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST], false, 0
+
+(37) Window
+Input [7]: [i_category#28, i_brand#29, s_store_name#30, s_company_name#31, d_year#32, d_moy#33, sum_sales#37]
+Arguments: [rank(d_year#32, d_moy#33) windowspecdefinition(i_category#28, i_brand#29, s_store_name#30, s_company_name#31, d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#39], [i_category#28, i_brand#29, s_store_name#30, s_company_name#31], [d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST]
+
+(38) Project [codegen id : 15]
+Output [6]: [i_category#28, i_brand#29, s_store_name#30, s_company_name#31, sum_sales#37, rn#39]
+Input [8]: [i_category#28, i_brand#29, s_store_name#30, s_company_name#31, d_year#32, d_moy#33, sum_sales#37, rn#39]
+
+(39) BroadcastExchange
+Input [6]: [i_category#28, i_brand#29, s_store_name#30, s_company_name#31, sum_sales#37, rn#39]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [id=#40]
+
+(40) BroadcastHashJoin [codegen id : 23]
+Left keys [5]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, rn#27]
+Right keys [5]: [i_category#28, i_brand#29, s_store_name#30, s_company_name#31, (rn#39 + 1)]
+Join condition: None
+
+(41) Project [codegen id : 23]
+Output [10]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum_sales#22, avg_monthly_sales#25, rn#27, sum_sales#37]
+Input [15]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum_sales#22, avg_monthly_sales#25, rn#27, i_category#28, i_brand#29, s_store_name#30, s_company_name#31, sum_sales#37, rn#39]
+
+(42) ReusedExchange [Reuses operator id: 35]
+Output [7]: [i_category#41, i_brand#42, s_store_name#43, s_company_name#44, d_year#45, d_moy#46, sum_sales#47]
+
+(43) Sort [codegen id : 21]
+Input [7]: [i_category#41, i_brand#42, s_store_name#43, s_company_name#44, d_year#45, d_moy#46, sum_sales#47]
+Arguments: [i_category#41 ASC NULLS FIRST, i_brand#42 ASC NULLS FIRST, s_store_name#43 ASC NULLS FIRST, s_company_name#44 ASC NULLS FIRST, d_year#45 ASC NULLS FIRST, d_moy#46 ASC NULLS FIRST], false, 0
+
+(44) Window
+Input [7]: [i_category#41, i_brand#42, s_store_name#43, s_company_name#44, d_year#45, d_moy#46, sum_sales#47]
+Arguments: [rank(d_year#45, d_moy#46) windowspecdefinition(i_category#41, i_brand#42, s_store_name#43, s_company_name#44, d_year#45 ASC NULLS FIRST, d_moy#46 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#48], [i_category#41, i_brand#42, s_store_name#43, s_company_name#44], [d_year#45 ASC NULLS FIRST, d_moy#46 ASC NULLS FIRST]
+
+(45) Project [codegen id : 22]
+Output [6]: [i_category#41, i_brand#42, s_store_name#43, s_company_name#44, sum_sales#47, rn#48]
+Input [8]: [i_category#41, i_brand#42, s_store_name#43, s_company_name#44, d_year#45, d_moy#46, sum_sales#47, rn#48]
+
+(46) BroadcastExchange
+Input [6]: [i_category#41, i_brand#42, s_store_name#43, s_company_name#44, sum_sales#47, rn#48]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [id=#49]
+
+(47) BroadcastHashJoin [codegen id : 23]
+Left keys [5]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, rn#27]
+Right keys [5]: [i_category#41, i_brand#42, s_store_name#43, s_company_name#44, (rn#48 - 1)]
+Join condition: None
+
+(48) Project [codegen id : 23]
+Output [10]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, avg_monthly_sales#25, sum_sales#22, sum_sales#37 AS psum#50, sum_sales#47 AS nsum#51]
+Input [16]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum_sales#22, avg_monthly_sales#25, rn#27, sum_sales#37, i_category#41, i_brand#42, s_store_name#43, s_company_name#44, sum_sales#47, rn#48]
+
+(49) TakeOrderedAndProject
+Input [10]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, avg_monthly_sales#25, sum_sales#22, psum#50, nsum#51]
+Arguments: 100, [CheckOverflow((promote_precision(cast(sum_sales#22 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#25 as decimal(22,6)))), DecimalType(22,6), true) ASC NULLS FIRST, s_store_name#15 ASC NULLS FIRST], [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, avg_monthly_sales#25, sum_sales#22, psum#50, nsum#51]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (50)
+
+
+(50) ReusedExchange [Reuses operator id: 13]
+Output [3]: [d_date_sk#10, d_year#11, d_moy#12]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/simplified.txt
new file mode 100644
index 0000000000000..ee3bf3f9890d3
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/simplified.txt
@@ -0,0 +1,84 @@
+TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_brand,s_company_name,d_year,d_moy,psum,nsum]
+  WholeStageCodegen (23)
+    Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales]
+      BroadcastHashJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn]
+        Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales]
+          BroadcastHashJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn]
+            Filter [d_year,avg_monthly_sales,sum_sales]
+              InputAdapter
+                Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name]
+                  WholeStageCodegen (8)
+                    Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy]
+                      InputAdapter
+                        Exchange [i_category,i_brand,s_store_name,s_company_name] #1
+                          WholeStageCodegen (7)
+                            Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales]
+                              InputAdapter
+                                Window [_w0,i_category,i_brand,s_store_name,s_company_name,d_year]
+                                  WholeStageCodegen (6)
+                                    Sort [i_category,i_brand,s_store_name,s_company_name,d_year]
+                                      InputAdapter
+                                        Exchange [i_category,i_brand,s_store_name,s_company_name,d_year] #2
+                                          WholeStageCodegen (5)
+                                            HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum]
+                                              InputAdapter
+                                                Exchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #3
+                                                  WholeStageCodegen (4)
+                                                    HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,ss_sales_price] [sum,sum]
+                                                      Project [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name]
+                                                        BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                          Project [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy]
+                                                            BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                              Project [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                                                BroadcastHashJoin [i_item_sk,ss_item_sk]
+                                                                  Filter [i_item_sk,i_category,i_brand]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.item [i_item_sk,i_brand,i_category]
+                                                                  InputAdapter
+                                                                    BroadcastExchange #4
+                                                                      WholeStageCodegen (1)
+                                                                        Filter [ss_item_sk,ss_store_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                                                                SubqueryBroadcast [d_date_sk] #1
+                                                                                  ReusedExchange [d_date_sk,d_year,d_moy] #5
+                                                              InputAdapter
+                                                                BroadcastExchange #5
+                                                                  WholeStageCodegen (2)
+                                                                    Filter [d_year,d_moy,d_date_sk]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                                          InputAdapter
+                                                            BroadcastExchange #6
+                                                              WholeStageCodegen (3)
+                                                                Filter [s_store_sk,s_store_name,s_company_name]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.store [s_store_sk,s_store_name,s_company_name]
+            InputAdapter
+              BroadcastExchange #7
+                WholeStageCodegen (15)
+                  Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn]
+                    InputAdapter
+                      Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name]
+                        WholeStageCodegen (14)
+                          Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy]
+                            InputAdapter
+                              Exchange [i_category,i_brand,s_store_name,s_company_name] #8
+                                WholeStageCodegen (13)
+                                  HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,sum]
+                                    InputAdapter
+                                      ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #9
+        InputAdapter
+          BroadcastExchange #10
+            WholeStageCodegen (22)
+              Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn]
+                InputAdapter
+                  Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name]
+                    WholeStageCodegen (21)
+                      Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy]
+                        InputAdapter
+                          ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #8
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.sf100/explain.txt
new file mode 100644
index 0000000000000..2f87af685f2e5
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.sf100/explain.txt
@@ -0,0 +1,194 @@
+== Physical Plan ==
+* HashAggregate (32)
++- Exchange (31)
+   +- * HashAggregate (30)
+      +- * Project (29)
+         +- * BroadcastHashJoin Inner BuildRight (28)
+            :- * Project (22)
+            :  +- * BroadcastHashJoin Inner BuildRight (21)
+            :     :- * Project (15)
+            :     :  +- * BroadcastHashJoin Inner BuildRight (14)
+            :     :     :- * Project (9)
+            :     :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+            :     :     :     :- * Filter (3)
+            :     :     :     :  +- * ColumnarToRow (2)
+            :     :     :     :     +- Scan parquet default.store_sales (1)
+            :     :     :     +- BroadcastExchange (7)
+            :     :     :        +- * Filter (6)
+            :     :     :           +- * ColumnarToRow (5)
+            :     :     :              +- Scan parquet default.store (4)
+            :     :     +- BroadcastExchange (13)
+            :     :        +- * Filter (12)
+            :     :           +- * ColumnarToRow (11)
+            :     :              +- Scan parquet default.customer_demographics (10)
+            :     +- BroadcastExchange (20)
+            :        +- * Project (19)
+            :           +- * Filter (18)
+            :              +- * ColumnarToRow (17)
+            :                 +- Scan parquet default.customer_address (16)
+            +- BroadcastExchange (27)
+               +- * Project (26)
+                  +- * Filter (25)
+                     +- * ColumnarToRow (24)
+                        +- Scan parquet default.date_dim (23)
+
+
+(1) Scan parquet default.store_sales
+Output [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_addr_sk), Or(Or(And(GreaterThanOrEqual(ss_sales_price,100.00),LessThanOrEqual(ss_sales_price,150.00)),And(GreaterThanOrEqual(ss_sales_price,50.00),LessThanOrEqual(ss_sales_price,100.00))),And(GreaterThanOrEqual(ss_sales_price,150.00),LessThanOrEqual(ss_sales_price,200.00))), Or(Or(And(GreaterThanOrEqual(ss_net_profit,0.00),LessThanOrEqual(ss_net_profit,2000.00)),And(GreaterThanOrEqual(ss_net_profit,150.00),LessThanOrEqual(ss_net_profit,3000.00))),And(GreaterThanOrEqual(ss_net_profit,50.00),LessThanOrEqual(ss_net_profit,25000.00)))]
+ReadSchema: struct<ss_cdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_quantity:int,ss_sales_price:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+
+(3) Filter [codegen id : 5]
+Input [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+Condition : ((((isnotnull(ss_store_sk#3) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_addr_sk#2)) AND ((((ss_sales_price#5 >= 100.00) AND (ss_sales_price#5 <= 150.00)) OR ((ss_sales_price#5 >= 50.00) AND (ss_sales_price#5 <= 100.00))) OR ((ss_sales_price#5 >= 150.00) AND (ss_sales_price#5 <= 200.00)))) AND ((((ss_net_profit#6 >= 0.00) AND (ss_net_profit#6 <= 2000.00)) OR ((ss_net_profit#6 >= 150.00) AND (ss_net_profit#6 <= 3000.00))) OR ((ss_net_profit#6 >= 50.00) AND (ss_net_profit#6 <= 25000.00))))
+
+(4) Scan parquet default.store
+Output [1]: [s_store_sk#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [1]: [s_store_sk#9]
+
+(6) Filter [codegen id : 1]
+Input [1]: [s_store_sk#9]
+Condition : isnotnull(s_store_sk#9)
+
+(7) BroadcastExchange
+Input [1]: [s_store_sk#9]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#10]
+
+(8) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#9]
+Join condition: None
+
+(9) Project [codegen id : 5]
+Output [6]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+Input [8]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, s_store_sk#9]
+
+(10) Scan parquet default.customer_demographics
+Output [3]: [cd_demo_sk#11, cd_marital_status#12, cd_education_status#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk), Or(Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,4 yr Degree         )),And(EqualTo(cd_marital_status,D),EqualTo(cd_education_status,2 yr Degree         ))),And(EqualTo(cd_marital_status,S),EqualTo(cd_education_status,College             )))]
+ReadSchema: struct<cd_demo_sk:int,cd_marital_status:string,cd_education_status:string>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [3]: [cd_demo_sk#11, cd_marital_status#12, cd_education_status#13]
+
+(12) Filter [codegen id : 2]
+Input [3]: [cd_demo_sk#11, cd_marital_status#12, cd_education_status#13]
+Condition : (isnotnull(cd_demo_sk#11) AND ((((cd_marital_status#12 = M) AND (cd_education_status#13 = 4 yr Degree         )) OR ((cd_marital_status#12 = D) AND (cd_education_status#13 = 2 yr Degree         ))) OR ((cd_marital_status#12 = S) AND (cd_education_status#13 = College             ))))
+
+(13) BroadcastExchange
+Input [3]: [cd_demo_sk#11, cd_marital_status#12, cd_education_status#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#14]
+
+(14) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_cdemo_sk#1]
+Right keys [1]: [cd_demo_sk#11]
+Join condition: ((((((cd_marital_status#12 = M) AND (cd_education_status#13 = 4 yr Degree         )) AND (ss_sales_price#5 >= 100.00)) AND (ss_sales_price#5 <= 150.00)) OR ((((cd_marital_status#12 = D) AND (cd_education_status#13 = 2 yr Degree         )) AND (ss_sales_price#5 >= 50.00)) AND (ss_sales_price#5 <= 100.00))) OR ((((cd_marital_status#12 = S) AND (cd_education_status#13 = College             )) AND (ss_sales_price#5 >= 150.00)) AND (ss_sales_price#5 <= 200.00)))
+
+(15) Project [codegen id : 5]
+Output [4]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7]
+Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#11, cd_marital_status#12, cd_education_status#13]
+
+(16) Scan parquet default.customer_address
+Output [3]: [ca_address_sk#15, ca_state#16, ca_country#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk), Or(Or(In(ca_state, [CO,OH,TX]),In(ca_state, [OR,MN,KY])),In(ca_state, [VA,CA,MS]))]
+ReadSchema: struct<ca_address_sk:int,ca_state:string,ca_country:string>
+
+(17) ColumnarToRow [codegen id : 3]
+Input [3]: [ca_address_sk#15, ca_state#16, ca_country#17]
+
+(18) Filter [codegen id : 3]
+Input [3]: [ca_address_sk#15, ca_state#16, ca_country#17]
+Condition : (((isnotnull(ca_country#17) AND (ca_country#17 = United States)) AND isnotnull(ca_address_sk#15)) AND ((ca_state#16 IN (CO,OH,TX) OR ca_state#16 IN (OR,MN,KY)) OR ca_state#16 IN (VA,CA,MS)))
+
+(19) Project [codegen id : 3]
+Output [2]: [ca_address_sk#15, ca_state#16]
+Input [3]: [ca_address_sk#15, ca_state#16, ca_country#17]
+
+(20) BroadcastExchange
+Input [2]: [ca_address_sk#15, ca_state#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(21) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_addr_sk#2]
+Right keys [1]: [ca_address_sk#15]
+Join condition: ((((ca_state#16 IN (CO,OH,TX) AND (ss_net_profit#6 >= 0.00)) AND (ss_net_profit#6 <= 2000.00)) OR ((ca_state#16 IN (OR,MN,KY) AND (ss_net_profit#6 >= 150.00)) AND (ss_net_profit#6 <= 3000.00))) OR ((ca_state#16 IN (VA,CA,MS) AND (ss_net_profit#6 >= 50.00)) AND (ss_net_profit#6 <= 25000.00)))
+
+(22) Project [codegen id : 5]
+Output [2]: [ss_quantity#4, ss_sold_date_sk#7]
+Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#15, ca_state#16]
+
+(23) Scan parquet default.date_dim
+Output [2]: [d_date_sk#19, d_year#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(24) ColumnarToRow [codegen id : 4]
+Input [2]: [d_date_sk#19, d_year#20]
+
+(25) Filter [codegen id : 4]
+Input [2]: [d_date_sk#19, d_year#20]
+Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19))
+
+(26) Project [codegen id : 4]
+Output [1]: [d_date_sk#19]
+Input [2]: [d_date_sk#19, d_year#20]
+
+(27) BroadcastExchange
+Input [1]: [d_date_sk#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(28) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_sold_date_sk#7]
+Right keys [1]: [d_date_sk#19]
+Join condition: None
+
+(29) Project [codegen id : 5]
+Output [1]: [ss_quantity#4]
+Input [3]: [ss_quantity#4, ss_sold_date_sk#7, d_date_sk#19]
+
+(30) HashAggregate [codegen id : 5]
+Input [1]: [ss_quantity#4]
+Keys: []
+Functions [1]: [partial_sum(ss_quantity#4)]
+Aggregate Attributes [1]: [sum#22]
+Results [1]: [sum#23]
+
+(31) Exchange
+Input [1]: [sum#23]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#24]
+
+(32) HashAggregate [codegen id : 6]
+Input [1]: [sum#23]
+Keys: []
+Functions [1]: [sum(ss_quantity#4)]
+Aggregate Attributes [1]: [sum(ss_quantity#4)#25]
+Results [1]: [sum(ss_quantity#4)#25 AS sum(ss_quantity)#26]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (33)
+
+
+(33) ReusedExchange [Reuses operator id: 27]
+Output [1]: [d_date_sk#19]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.sf100/simplified.txt
new file mode 100644
index 0000000000000..9a7fc188f3ded
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.sf100/simplified.txt
@@ -0,0 +1,50 @@
+WholeStageCodegen (6)
+  HashAggregate [sum] [sum(ss_quantity),sum(ss_quantity),sum]
+    InputAdapter
+      Exchange #1
+        WholeStageCodegen (5)
+          HashAggregate [ss_quantity] [sum,sum]
+            Project [ss_quantity]
+              BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                Project [ss_quantity,ss_sold_date_sk]
+                  BroadcastHashJoin [ss_addr_sk,ca_address_sk,ca_state,ss_net_profit]
+                    Project [ss_addr_sk,ss_quantity,ss_net_profit,ss_sold_date_sk]
+                      BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status,ss_sales_price]
+                        Project [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk]
+                          BroadcastHashJoin [ss_store_sk,s_store_sk]
+                            Filter [ss_store_sk,ss_cdemo_sk,ss_addr_sk,ss_sales_price,ss_net_profit]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.store_sales [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk]
+                                    SubqueryBroadcast [d_date_sk] #1
+                                      ReusedExchange [d_date_sk] #2
+                            InputAdapter
+                              BroadcastExchange #3
+                                WholeStageCodegen (1)
+                                  Filter [s_store_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.store [s_store_sk]
+                        InputAdapter
+                          BroadcastExchange #4
+                            WholeStageCodegen (2)
+                              Filter [cd_demo_sk,cd_marital_status,cd_education_status]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status]
+                    InputAdapter
+                      BroadcastExchange #5
+                        WholeStageCodegen (3)
+                          Project [ca_address_sk,ca_state]
+                            Filter [ca_country,ca_address_sk,ca_state]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.customer_address [ca_address_sk,ca_state,ca_country]
+                InputAdapter
+                  BroadcastExchange #2
+                    WholeStageCodegen (4)
+                      Project [d_date_sk]
+                        Filter [d_year,d_date_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.date_dim [d_date_sk,d_year]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/explain.txt
new file mode 100644
index 0000000000000..2f87af685f2e5
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/explain.txt
@@ -0,0 +1,194 @@
+== Physical Plan ==
+* HashAggregate (32)
++- Exchange (31)
+   +- * HashAggregate (30)
+      +- * Project (29)
+         +- * BroadcastHashJoin Inner BuildRight (28)
+            :- * Project (22)
+            :  +- * BroadcastHashJoin Inner BuildRight (21)
+            :     :- * Project (15)
+            :     :  +- * BroadcastHashJoin Inner BuildRight (14)
+            :     :     :- * Project (9)
+            :     :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+            :     :     :     :- * Filter (3)
+            :     :     :     :  +- * ColumnarToRow (2)
+            :     :     :     :     +- Scan parquet default.store_sales (1)
+            :     :     :     +- BroadcastExchange (7)
+            :     :     :        +- * Filter (6)
+            :     :     :           +- * ColumnarToRow (5)
+            :     :     :              +- Scan parquet default.store (4)
+            :     :     +- BroadcastExchange (13)
+            :     :        +- * Filter (12)
+            :     :           +- * ColumnarToRow (11)
+            :     :              +- Scan parquet default.customer_demographics (10)
+            :     +- BroadcastExchange (20)
+            :        +- * Project (19)
+            :           +- * Filter (18)
+            :              +- * ColumnarToRow (17)
+            :                 +- Scan parquet default.customer_address (16)
+            +- BroadcastExchange (27)
+               +- * Project (26)
+                  +- * Filter (25)
+                     +- * ColumnarToRow (24)
+                        +- Scan parquet default.date_dim (23)
+
+
+(1) Scan parquet default.store_sales
+Output [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_addr_sk), Or(Or(And(GreaterThanOrEqual(ss_sales_price,100.00),LessThanOrEqual(ss_sales_price,150.00)),And(GreaterThanOrEqual(ss_sales_price,50.00),LessThanOrEqual(ss_sales_price,100.00))),And(GreaterThanOrEqual(ss_sales_price,150.00),LessThanOrEqual(ss_sales_price,200.00))), Or(Or(And(GreaterThanOrEqual(ss_net_profit,0.00),LessThanOrEqual(ss_net_profit,2000.00)),And(GreaterThanOrEqual(ss_net_profit,150.00),LessThanOrEqual(ss_net_profit,3000.00))),And(GreaterThanOrEqual(ss_net_profit,50.00),LessThanOrEqual(ss_net_profit,25000.00)))]
+ReadSchema: struct<ss_cdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_quantity:int,ss_sales_price:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+
+(3) Filter [codegen id : 5]
+Input [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+Condition : ((((isnotnull(ss_store_sk#3) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_addr_sk#2)) AND ((((ss_sales_price#5 >= 100.00) AND (ss_sales_price#5 <= 150.00)) OR ((ss_sales_price#5 >= 50.00) AND (ss_sales_price#5 <= 100.00))) OR ((ss_sales_price#5 >= 150.00) AND (ss_sales_price#5 <= 200.00)))) AND ((((ss_net_profit#6 >= 0.00) AND (ss_net_profit#6 <= 2000.00)) OR ((ss_net_profit#6 >= 150.00) AND (ss_net_profit#6 <= 3000.00))) OR ((ss_net_profit#6 >= 50.00) AND (ss_net_profit#6 <= 25000.00))))
+
+(4) Scan parquet default.store
+Output [1]: [s_store_sk#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [1]: [s_store_sk#9]
+
+(6) Filter [codegen id : 1]
+Input [1]: [s_store_sk#9]
+Condition : isnotnull(s_store_sk#9)
+
+(7) BroadcastExchange
+Input [1]: [s_store_sk#9]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#10]
+
+(8) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#9]
+Join condition: None
+
+(9) Project [codegen id : 5]
+Output [6]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+Input [8]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, s_store_sk#9]
+
+(10) Scan parquet default.customer_demographics
+Output [3]: [cd_demo_sk#11, cd_marital_status#12, cd_education_status#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk), Or(Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,4 yr Degree         )),And(EqualTo(cd_marital_status,D),EqualTo(cd_education_status,2 yr Degree         ))),And(EqualTo(cd_marital_status,S),EqualTo(cd_education_status,College             )))]
+ReadSchema: struct<cd_demo_sk:int,cd_marital_status:string,cd_education_status:string>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [3]: [cd_demo_sk#11, cd_marital_status#12, cd_education_status#13]
+
+(12) Filter [codegen id : 2]
+Input [3]: [cd_demo_sk#11, cd_marital_status#12, cd_education_status#13]
+Condition : (isnotnull(cd_demo_sk#11) AND ((((cd_marital_status#12 = M) AND (cd_education_status#13 = 4 yr Degree         )) OR ((cd_marital_status#12 = D) AND (cd_education_status#13 = 2 yr Degree         ))) OR ((cd_marital_status#12 = S) AND (cd_education_status#13 = College             ))))
+
+(13) BroadcastExchange
+Input [3]: [cd_demo_sk#11, cd_marital_status#12, cd_education_status#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#14]
+
+(14) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_cdemo_sk#1]
+Right keys [1]: [cd_demo_sk#11]
+Join condition: ((((((cd_marital_status#12 = M) AND (cd_education_status#13 = 4 yr Degree         )) AND (ss_sales_price#5 >= 100.00)) AND (ss_sales_price#5 <= 150.00)) OR ((((cd_marital_status#12 = D) AND (cd_education_status#13 = 2 yr Degree         )) AND (ss_sales_price#5 >= 50.00)) AND (ss_sales_price#5 <= 100.00))) OR ((((cd_marital_status#12 = S) AND (cd_education_status#13 = College             )) AND (ss_sales_price#5 >= 150.00)) AND (ss_sales_price#5 <= 200.00)))
+
+(15) Project [codegen id : 5]
+Output [4]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7]
+Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#11, cd_marital_status#12, cd_education_status#13]
+
+(16) Scan parquet default.customer_address
+Output [3]: [ca_address_sk#15, ca_state#16, ca_country#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk), Or(Or(In(ca_state, [CO,OH,TX]),In(ca_state, [OR,MN,KY])),In(ca_state, [VA,CA,MS]))]
+ReadSchema: struct<ca_address_sk:int,ca_state:string,ca_country:string>
+
+(17) ColumnarToRow [codegen id : 3]
+Input [3]: [ca_address_sk#15, ca_state#16, ca_country#17]
+
+(18) Filter [codegen id : 3]
+Input [3]: [ca_address_sk#15, ca_state#16, ca_country#17]
+Condition : (((isnotnull(ca_country#17) AND (ca_country#17 = United States)) AND isnotnull(ca_address_sk#15)) AND ((ca_state#16 IN (CO,OH,TX) OR ca_state#16 IN (OR,MN,KY)) OR ca_state#16 IN (VA,CA,MS)))
+
+(19) Project [codegen id : 3]
+Output [2]: [ca_address_sk#15, ca_state#16]
+Input [3]: [ca_address_sk#15, ca_state#16, ca_country#17]
+
+(20) BroadcastExchange
+Input [2]: [ca_address_sk#15, ca_state#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(21) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_addr_sk#2]
+Right keys [1]: [ca_address_sk#15]
+Join condition: ((((ca_state#16 IN (CO,OH,TX) AND (ss_net_profit#6 >= 0.00)) AND (ss_net_profit#6 <= 2000.00)) OR ((ca_state#16 IN (OR,MN,KY) AND (ss_net_profit#6 >= 150.00)) AND (ss_net_profit#6 <= 3000.00))) OR ((ca_state#16 IN (VA,CA,MS) AND (ss_net_profit#6 >= 50.00)) AND (ss_net_profit#6 <= 25000.00)))
+
+(22) Project [codegen id : 5]
+Output [2]: [ss_quantity#4, ss_sold_date_sk#7]
+Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#15, ca_state#16]
+
+(23) Scan parquet default.date_dim
+Output [2]: [d_date_sk#19, d_year#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(24) ColumnarToRow [codegen id : 4]
+Input [2]: [d_date_sk#19, d_year#20]
+
+(25) Filter [codegen id : 4]
+Input [2]: [d_date_sk#19, d_year#20]
+Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19))
+
+(26) Project [codegen id : 4]
+Output [1]: [d_date_sk#19]
+Input [2]: [d_date_sk#19, d_year#20]
+
+(27) BroadcastExchange
+Input [1]: [d_date_sk#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(28) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_sold_date_sk#7]
+Right keys [1]: [d_date_sk#19]
+Join condition: None
+
+(29) Project [codegen id : 5]
+Output [1]: [ss_quantity#4]
+Input [3]: [ss_quantity#4, ss_sold_date_sk#7, d_date_sk#19]
+
+(30) HashAggregate [codegen id : 5]
+Input [1]: [ss_quantity#4]
+Keys: []
+Functions [1]: [partial_sum(ss_quantity#4)]
+Aggregate Attributes [1]: [sum#22]
+Results [1]: [sum#23]
+
+(31) Exchange
+Input [1]: [sum#23]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#24]
+
+(32) HashAggregate [codegen id : 6]
+Input [1]: [sum#23]
+Keys: []
+Functions [1]: [sum(ss_quantity#4)]
+Aggregate Attributes [1]: [sum(ss_quantity#4)#25]
+Results [1]: [sum(ss_quantity#4)#25 AS sum(ss_quantity)#26]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (33)
+
+
+(33) ReusedExchange [Reuses operator id: 27]
+Output [1]: [d_date_sk#19]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/simplified.txt
new file mode 100644
index 0000000000000..9a7fc188f3ded
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/simplified.txt
@@ -0,0 +1,50 @@
+WholeStageCodegen (6)
+  HashAggregate [sum] [sum(ss_quantity),sum(ss_quantity),sum]
+    InputAdapter
+      Exchange #1
+        WholeStageCodegen (5)
+          HashAggregate [ss_quantity] [sum,sum]
+            Project [ss_quantity]
+              BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                Project [ss_quantity,ss_sold_date_sk]
+                  BroadcastHashJoin [ss_addr_sk,ca_address_sk,ca_state,ss_net_profit]
+                    Project [ss_addr_sk,ss_quantity,ss_net_profit,ss_sold_date_sk]
+                      BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status,ss_sales_price]
+                        Project [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk]
+                          BroadcastHashJoin [ss_store_sk,s_store_sk]
+                            Filter [ss_store_sk,ss_cdemo_sk,ss_addr_sk,ss_sales_price,ss_net_profit]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.store_sales [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk]
+                                    SubqueryBroadcast [d_date_sk] #1
+                                      ReusedExchange [d_date_sk] #2
+                            InputAdapter
+                              BroadcastExchange #3
+                                WholeStageCodegen (1)
+                                  Filter [s_store_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.store [s_store_sk]
+                        InputAdapter
+                          BroadcastExchange #4
+                            WholeStageCodegen (2)
+                              Filter [cd_demo_sk,cd_marital_status,cd_education_status]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status]
+                    InputAdapter
+                      BroadcastExchange #5
+                        WholeStageCodegen (3)
+                          Project [ca_address_sk,ca_state]
+                            Filter [ca_country,ca_address_sk,ca_state]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.customer_address [ca_address_sk,ca_state,ca_country]
+                InputAdapter
+                  BroadcastExchange #2
+                    WholeStageCodegen (4)
+                      Project [d_date_sk]
+                        Filter [d_year,d_date_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.date_dim [d_date_sk,d_year]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.sf100/explain.txt
new file mode 100644
index 0000000000000..593d1ecf287e1
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.sf100/explain.txt
@@ -0,0 +1,510 @@
+== Physical Plan ==
+TakeOrderedAndProject (90)
++- * HashAggregate (89)
+   +- Exchange (88)
+      +- * HashAggregate (87)
+         +- Union (86)
+            :- * Project (31)
+            :  +- * Filter (30)
+            :     +- Window (29)
+            :        +- * Sort (28)
+            :           +- Window (27)
+            :              +- * Sort (26)
+            :                 +- Exchange (25)
+            :                    +- * HashAggregate (24)
+            :                       +- Exchange (23)
+            :                          +- * HashAggregate (22)
+            :                             +- * Project (21)
+            :                                +- * SortMergeJoin Inner (20)
+            :                                   :- * Sort (13)
+            :                                   :  +- Exchange (12)
+            :                                   :     +- * Project (11)
+            :                                   :        +- * BroadcastHashJoin Inner BuildRight (10)
+            :                                   :           :- * Project (4)
+            :                                   :           :  +- * Filter (3)
+            :                                   :           :     +- * ColumnarToRow (2)
+            :                                   :           :        +- Scan parquet default.web_sales (1)
+            :                                   :           +- BroadcastExchange (9)
+            :                                   :              +- * Project (8)
+            :                                   :                 +- * Filter (7)
+            :                                   :                    +- * ColumnarToRow (6)
+            :                                   :                       +- Scan parquet default.date_dim (5)
+            :                                   +- * Sort (19)
+            :                                      +- Exchange (18)
+            :                                         +- * Project (17)
+            :                                            +- * Filter (16)
+            :                                               +- * ColumnarToRow (15)
+            :                                                  +- Scan parquet default.web_returns (14)
+            :- * Project (58)
+            :  +- * Filter (57)
+            :     +- Window (56)
+            :        +- * Sort (55)
+            :           +- Window (54)
+            :              +- * Sort (53)
+            :                 +- Exchange (52)
+            :                    +- * HashAggregate (51)
+            :                       +- Exchange (50)
+            :                          +- * HashAggregate (49)
+            :                             +- * Project (48)
+            :                                +- * SortMergeJoin Inner (47)
+            :                                   :- * Sort (40)
+            :                                   :  +- Exchange (39)
+            :                                   :     +- * Project (38)
+            :                                   :        +- * BroadcastHashJoin Inner BuildRight (37)
+            :                                   :           :- * Project (35)
+            :                                   :           :  +- * Filter (34)
+            :                                   :           :     +- * ColumnarToRow (33)
+            :                                   :           :        +- Scan parquet default.catalog_sales (32)
+            :                                   :           +- ReusedExchange (36)
+            :                                   +- * Sort (46)
+            :                                      +- Exchange (45)
+            :                                         +- * Project (44)
+            :                                            +- * Filter (43)
+            :                                               +- * ColumnarToRow (42)
+            :                                                  +- Scan parquet default.catalog_returns (41)
+            +- * Project (85)
+               +- * Filter (84)
+                  +- Window (83)
+                     +- * Sort (82)
+                        +- Window (81)
+                           +- * Sort (80)
+                              +- Exchange (79)
+                                 +- * HashAggregate (78)
+                                    +- Exchange (77)
+                                       +- * HashAggregate (76)
+                                          +- * Project (75)
+                                             +- * SortMergeJoin Inner (74)
+                                                :- * Sort (67)
+                                                :  +- Exchange (66)
+                                                :     +- * Project (65)
+                                                :        +- * BroadcastHashJoin Inner BuildRight (64)
+                                                :           :- * Project (62)
+                                                :           :  +- * Filter (61)
+                                                :           :     +- * ColumnarToRow (60)
+                                                :           :        +- Scan parquet default.store_sales (59)
+                                                :           +- ReusedExchange (63)
+                                                +- * Sort (73)
+                                                   +- Exchange (72)
+                                                      +- * Project (71)
+                                                         +- * Filter (70)
+                                                            +- * ColumnarToRow (69)
+                                                               +- Scan parquet default.store_returns (68)
+
+
+(1) Scan parquet default.web_sales
+Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#6), dynamicpruningexpression(ws_sold_date_sk#6 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ws_net_profit), IsNotNull(ws_net_paid), IsNotNull(ws_quantity), GreaterThan(ws_net_profit,1.00), GreaterThan(ws_net_paid,0.00), GreaterThan(ws_quantity,0), IsNotNull(ws_order_number), IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_order_number:int,ws_quantity:int,ws_net_paid:decimal(7,2),ws_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6]
+
+(3) Filter [codegen id : 2]
+Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6]
+Condition : (((((((isnotnull(ws_net_profit#5) AND isnotnull(ws_net_paid#4)) AND isnotnull(ws_quantity#3)) AND (ws_net_profit#5 > 1.00)) AND (ws_net_paid#4 > 0.00)) AND (ws_quantity#3 > 0)) AND isnotnull(ws_order_number#2)) AND isnotnull(ws_item_sk#1))
+
+(4) Project [codegen id : 2]
+Output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6]
+Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6]
+
+(5) Scan parquet default.date_dim
+Output [3]: [d_date_sk#8, d_year#9, d_moy#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#8, d_year#9, d_moy#10]
+
+(7) Filter [codegen id : 1]
+Input [3]: [d_date_sk#8, d_year#9, d_moy#10]
+Condition : ((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2001)) AND (d_moy#10 = 12)) AND isnotnull(d_date_sk#8))
+
+(8) Project [codegen id : 1]
+Output [1]: [d_date_sk#8]
+Input [3]: [d_date_sk#8, d_year#9, d_moy#10]
+
+(9) BroadcastExchange
+Input [1]: [d_date_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#11]
+
+(10) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ws_sold_date_sk#6]
+Right keys [1]: [d_date_sk#8]
+Join condition: None
+
+(11) Project [codegen id : 2]
+Output [4]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4]
+Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, d_date_sk#8]
+
+(12) Exchange
+Input [4]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4]
+Arguments: hashpartitioning(ws_order_number#2, ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#12]
+
+(13) Sort [codegen id : 3]
+Input [4]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4]
+Arguments: [ws_order_number#2 ASC NULLS FIRST, ws_item_sk#1 ASC NULLS FIRST], false, 0
+
+(14) Scan parquet default.web_returns
+Output [5]: [wr_item_sk#13, wr_order_number#14, wr_return_quantity#15, wr_return_amt#16, wr_returned_date_sk#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_returns]
+PushedFilters: [IsNotNull(wr_return_amt), GreaterThan(wr_return_amt,10000.00), IsNotNull(wr_order_number), IsNotNull(wr_item_sk)]
+ReadSchema: struct<wr_item_sk:int,wr_order_number:int,wr_return_quantity:int,wr_return_amt:decimal(7,2)>
+
+(15) ColumnarToRow [codegen id : 4]
+Input [5]: [wr_item_sk#13, wr_order_number#14, wr_return_quantity#15, wr_return_amt#16, wr_returned_date_sk#17]
+
+(16) Filter [codegen id : 4]
+Input [5]: [wr_item_sk#13, wr_order_number#14, wr_return_quantity#15, wr_return_amt#16, wr_returned_date_sk#17]
+Condition : (((isnotnull(wr_return_amt#16) AND (wr_return_amt#16 > 10000.00)) AND isnotnull(wr_order_number#14)) AND isnotnull(wr_item_sk#13))
+
+(17) Project [codegen id : 4]
+Output [4]: [wr_item_sk#13, wr_order_number#14, wr_return_quantity#15, wr_return_amt#16]
+Input [5]: [wr_item_sk#13, wr_order_number#14, wr_return_quantity#15, wr_return_amt#16, wr_returned_date_sk#17]
+
+(18) Exchange
+Input [4]: [wr_item_sk#13, wr_order_number#14, wr_return_quantity#15, wr_return_amt#16]
+Arguments: hashpartitioning(wr_order_number#14, wr_item_sk#13, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(19) Sort [codegen id : 5]
+Input [4]: [wr_item_sk#13, wr_order_number#14, wr_return_quantity#15, wr_return_amt#16]
+Arguments: [wr_order_number#14 ASC NULLS FIRST, wr_item_sk#13 ASC NULLS FIRST], false, 0
+
+(20) SortMergeJoin [codegen id : 6]
+Left keys [2]: [ws_order_number#2, ws_item_sk#1]
+Right keys [2]: [wr_order_number#14, wr_item_sk#13]
+Join condition: None
+
+(21) Project [codegen id : 6]
+Output [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#15, wr_return_amt#16]
+Input [8]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, wr_item_sk#13, wr_order_number#14, wr_return_quantity#15, wr_return_amt#16]
+
+(22) HashAggregate [codegen id : 6]
+Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#15, wr_return_amt#16]
+Keys [1]: [ws_item_sk#1]
+Functions [4]: [partial_sum(coalesce(wr_return_quantity#15, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#16 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))]
+Aggregate Attributes [6]: [sum#19, sum#20, sum#21, isEmpty#22, sum#23, isEmpty#24]
+Results [7]: [ws_item_sk#1, sum#25, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30]
+
+(23) Exchange
+Input [7]: [ws_item_sk#1, sum#25, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30]
+Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(24) HashAggregate [codegen id : 7]
+Input [7]: [ws_item_sk#1, sum#25, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30]
+Keys [1]: [ws_item_sk#1]
+Functions [4]: [sum(coalesce(wr_return_quantity#15, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#16 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))]
+Aggregate Attributes [4]: [sum(coalesce(wr_return_quantity#15, 0))#32, sum(coalesce(ws_quantity#3, 0))#33, sum(coalesce(cast(wr_return_amt#16 as decimal(12,2)), 0.00))#34, sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#35]
+Results [3]: [ws_item_sk#1 AS item#36, CheckOverflow((promote_precision(cast(sum(coalesce(wr_return_quantity#15, 0))#32 as decimal(15,4))) / promote_precision(cast(sum(coalesce(ws_quantity#3, 0))#33 as decimal(15,4)))), DecimalType(35,20), true) AS return_ratio#37, CheckOverflow((promote_precision(cast(sum(coalesce(cast(wr_return_amt#16 as decimal(12,2)), 0.00))#34 as decimal(15,4))) / promote_precision(cast(sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#35 as decimal(15,4)))), DecimalType(35,20), true) AS currency_ratio#38]
+
+(25) Exchange
+Input [3]: [item#36, return_ratio#37, currency_ratio#38]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#39]
+
+(26) Sort [codegen id : 8]
+Input [3]: [item#36, return_ratio#37, currency_ratio#38]
+Arguments: [return_ratio#37 ASC NULLS FIRST], false, 0
+
+(27) Window
+Input [3]: [item#36, return_ratio#37, currency_ratio#38]
+Arguments: [rank(return_ratio#37) windowspecdefinition(return_ratio#37 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#40], [return_ratio#37 ASC NULLS FIRST]
+
+(28) Sort [codegen id : 9]
+Input [4]: [item#36, return_ratio#37, currency_ratio#38, return_rank#40]
+Arguments: [currency_ratio#38 ASC NULLS FIRST], false, 0
+
+(29) Window
+Input [4]: [item#36, return_ratio#37, currency_ratio#38, return_rank#40]
+Arguments: [rank(currency_ratio#38) windowspecdefinition(currency_ratio#38 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#41], [currency_ratio#38 ASC NULLS FIRST]
+
+(30) Filter [codegen id : 10]
+Input [5]: [item#36, return_ratio#37, currency_ratio#38, return_rank#40, currency_rank#41]
+Condition : ((return_rank#40 <= 10) OR (currency_rank#41 <= 10))
+
+(31) Project [codegen id : 10]
+Output [5]: [web AS channel#42, item#36, return_ratio#37, return_rank#40, currency_rank#41]
+Input [5]: [item#36, return_ratio#37, currency_ratio#38, return_rank#40, currency_rank#41]
+
+(32) Scan parquet default.catalog_sales
+Output [6]: [cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_net_paid#46, cs_net_profit#47, cs_sold_date_sk#48]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#48), dynamicpruningexpression(cs_sold_date_sk#48 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int,cs_order_number:int,cs_quantity:int,cs_net_paid:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(33) ColumnarToRow [codegen id : 12]
+Input [6]: [cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_net_paid#46, cs_net_profit#47, cs_sold_date_sk#48]
+
+(34) Filter [codegen id : 12]
+Input [6]: [cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_net_paid#46, cs_net_profit#47, cs_sold_date_sk#48]
+Condition : (((((((isnotnull(cs_net_profit#47) AND isnotnull(cs_net_paid#46)) AND isnotnull(cs_quantity#45)) AND (cs_net_profit#47 > 1.00)) AND (cs_net_paid#46 > 0.00)) AND (cs_quantity#45 > 0)) AND isnotnull(cs_order_number#44)) AND isnotnull(cs_item_sk#43))
+
+(35) Project [codegen id : 12]
+Output [5]: [cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_net_paid#46, cs_sold_date_sk#48]
+Input [6]: [cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_net_paid#46, cs_net_profit#47, cs_sold_date_sk#48]
+
+(36) ReusedExchange [Reuses operator id: 9]
+Output [1]: [d_date_sk#49]
+
+(37) BroadcastHashJoin [codegen id : 12]
+Left keys [1]: [cs_sold_date_sk#48]
+Right keys [1]: [d_date_sk#49]
+Join condition: None
+
+(38) Project [codegen id : 12]
+Output [4]: [cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_net_paid#46]
+Input [6]: [cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_net_paid#46, cs_sold_date_sk#48, d_date_sk#49]
+
+(39) Exchange
+Input [4]: [cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_net_paid#46]
+Arguments: hashpartitioning(cs_order_number#44, cs_item_sk#43, 5), ENSURE_REQUIREMENTS, [id=#50]
+
+(40) Sort [codegen id : 13]
+Input [4]: [cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_net_paid#46]
+Arguments: [cs_order_number#44 ASC NULLS FIRST, cs_item_sk#43 ASC NULLS FIRST], false, 0
+
+(41) Scan parquet default.catalog_returns
+Output [5]: [cr_item_sk#51, cr_order_number#52, cr_return_quantity#53, cr_return_amount#54, cr_returned_date_sk#55]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int,cr_return_quantity:int,cr_return_amount:decimal(7,2)>
+
+(42) ColumnarToRow [codegen id : 14]
+Input [5]: [cr_item_sk#51, cr_order_number#52, cr_return_quantity#53, cr_return_amount#54, cr_returned_date_sk#55]
+
+(43) Filter [codegen id : 14]
+Input [5]: [cr_item_sk#51, cr_order_number#52, cr_return_quantity#53, cr_return_amount#54, cr_returned_date_sk#55]
+Condition : (((isnotnull(cr_return_amount#54) AND (cr_return_amount#54 > 10000.00)) AND isnotnull(cr_order_number#52)) AND isnotnull(cr_item_sk#51))
+
+(44) Project [codegen id : 14]
+Output [4]: [cr_item_sk#51, cr_order_number#52, cr_return_quantity#53, cr_return_amount#54]
+Input [5]: [cr_item_sk#51, cr_order_number#52, cr_return_quantity#53, cr_return_amount#54, cr_returned_date_sk#55]
+
+(45) Exchange
+Input [4]: [cr_item_sk#51, cr_order_number#52, cr_return_quantity#53, cr_return_amount#54]
+Arguments: hashpartitioning(cr_order_number#52, cr_item_sk#51, 5), ENSURE_REQUIREMENTS, [id=#56]
+
+(46) Sort [codegen id : 15]
+Input [4]: [cr_item_sk#51, cr_order_number#52, cr_return_quantity#53, cr_return_amount#54]
+Arguments: [cr_order_number#52 ASC NULLS FIRST, cr_item_sk#51 ASC NULLS FIRST], false, 0
+
+(47) SortMergeJoin [codegen id : 16]
+Left keys [2]: [cs_order_number#44, cs_item_sk#43]
+Right keys [2]: [cr_order_number#52, cr_item_sk#51]
+Join condition: None
+
+(48) Project [codegen id : 16]
+Output [5]: [cs_item_sk#43, cs_quantity#45, cs_net_paid#46, cr_return_quantity#53, cr_return_amount#54]
+Input [8]: [cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_net_paid#46, cr_item_sk#51, cr_order_number#52, cr_return_quantity#53, cr_return_amount#54]
+
+(49) HashAggregate [codegen id : 16]
+Input [5]: [cs_item_sk#43, cs_quantity#45, cs_net_paid#46, cr_return_quantity#53, cr_return_amount#54]
+Keys [1]: [cs_item_sk#43]
+Functions [4]: [partial_sum(coalesce(cr_return_quantity#53, 0)), partial_sum(coalesce(cs_quantity#45, 0)), partial_sum(coalesce(cast(cr_return_amount#54 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#46 as decimal(12,2)), 0.00))]
+Aggregate Attributes [6]: [sum#57, sum#58, sum#59, isEmpty#60, sum#61, isEmpty#62]
+Results [7]: [cs_item_sk#43, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68]
+
+(50) Exchange
+Input [7]: [cs_item_sk#43, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68]
+Arguments: hashpartitioning(cs_item_sk#43, 5), ENSURE_REQUIREMENTS, [id=#69]
+
+(51) HashAggregate [codegen id : 17]
+Input [7]: [cs_item_sk#43, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68]
+Keys [1]: [cs_item_sk#43]
+Functions [4]: [sum(coalesce(cr_return_quantity#53, 0)), sum(coalesce(cs_quantity#45, 0)), sum(coalesce(cast(cr_return_amount#54 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#46 as decimal(12,2)), 0.00))]
+Aggregate Attributes [4]: [sum(coalesce(cr_return_quantity#53, 0))#70, sum(coalesce(cs_quantity#45, 0))#71, sum(coalesce(cast(cr_return_amount#54 as decimal(12,2)), 0.00))#72, sum(coalesce(cast(cs_net_paid#46 as decimal(12,2)), 0.00))#73]
+Results [3]: [cs_item_sk#43 AS item#74, CheckOverflow((promote_precision(cast(sum(coalesce(cr_return_quantity#53, 0))#70 as decimal(15,4))) / promote_precision(cast(sum(coalesce(cs_quantity#45, 0))#71 as decimal(15,4)))), DecimalType(35,20), true) AS return_ratio#75, CheckOverflow((promote_precision(cast(sum(coalesce(cast(cr_return_amount#54 as decimal(12,2)), 0.00))#72 as decimal(15,4))) / promote_precision(cast(sum(coalesce(cast(cs_net_paid#46 as decimal(12,2)), 0.00))#73 as decimal(15,4)))), DecimalType(35,20), true) AS currency_ratio#76]
+
+(52) Exchange
+Input [3]: [item#74, return_ratio#75, currency_ratio#76]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#77]
+
+(53) Sort [codegen id : 18]
+Input [3]: [item#74, return_ratio#75, currency_ratio#76]
+Arguments: [return_ratio#75 ASC NULLS FIRST], false, 0
+
+(54) Window
+Input [3]: [item#74, return_ratio#75, currency_ratio#76]
+Arguments: [rank(return_ratio#75) windowspecdefinition(return_ratio#75 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#78], [return_ratio#75 ASC NULLS FIRST]
+
+(55) Sort [codegen id : 19]
+Input [4]: [item#74, return_ratio#75, currency_ratio#76, return_rank#78]
+Arguments: [currency_ratio#76 ASC NULLS FIRST], false, 0
+
+(56) Window
+Input [4]: [item#74, return_ratio#75, currency_ratio#76, return_rank#78]
+Arguments: [rank(currency_ratio#76) windowspecdefinition(currency_ratio#76 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#79], [currency_ratio#76 ASC NULLS FIRST]
+
+(57) Filter [codegen id : 20]
+Input [5]: [item#74, return_ratio#75, currency_ratio#76, return_rank#78, currency_rank#79]
+Condition : ((return_rank#78 <= 10) OR (currency_rank#79 <= 10))
+
+(58) Project [codegen id : 20]
+Output [5]: [catalog AS channel#80, item#74, return_ratio#75, return_rank#78, currency_rank#79]
+Input [5]: [item#74, return_ratio#75, currency_ratio#76, return_rank#78, currency_rank#79]
+
+(59) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_net_paid#84, ss_net_profit#85, ss_sold_date_sk#86]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#86), dynamicpruningexpression(ss_sold_date_sk#86 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ticket_number:int,ss_quantity:int,ss_net_paid:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(60) ColumnarToRow [codegen id : 22]
+Input [6]: [ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_net_paid#84, ss_net_profit#85, ss_sold_date_sk#86]
+
+(61) Filter [codegen id : 22]
+Input [6]: [ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_net_paid#84, ss_net_profit#85, ss_sold_date_sk#86]
+Condition : (((((((isnotnull(ss_net_profit#85) AND isnotnull(ss_net_paid#84)) AND isnotnull(ss_quantity#83)) AND (ss_net_profit#85 > 1.00)) AND (ss_net_paid#84 > 0.00)) AND (ss_quantity#83 > 0)) AND isnotnull(ss_ticket_number#82)) AND isnotnull(ss_item_sk#81))
+
+(62) Project [codegen id : 22]
+Output [5]: [ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_net_paid#84, ss_sold_date_sk#86]
+Input [6]: [ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_net_paid#84, ss_net_profit#85, ss_sold_date_sk#86]
+
+(63) ReusedExchange [Reuses operator id: 9]
+Output [1]: [d_date_sk#87]
+
+(64) BroadcastHashJoin [codegen id : 22]
+Left keys [1]: [ss_sold_date_sk#86]
+Right keys [1]: [d_date_sk#87]
+Join condition: None
+
+(65) Project [codegen id : 22]
+Output [4]: [ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_net_paid#84]
+Input [6]: [ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_net_paid#84, ss_sold_date_sk#86, d_date_sk#87]
+
+(66) Exchange
+Input [4]: [ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_net_paid#84]
+Arguments: hashpartitioning(ss_ticket_number#82, ss_item_sk#81, 5), ENSURE_REQUIREMENTS, [id=#88]
+
+(67) Sort [codegen id : 23]
+Input [4]: [ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_net_paid#84]
+Arguments: [ss_ticket_number#82 ASC NULLS FIRST, ss_item_sk#81 ASC NULLS FIRST], false, 0
+
+(68) Scan parquet default.store_returns
+Output [5]: [sr_item_sk#89, sr_ticket_number#90, sr_return_quantity#91, sr_return_amt#92, sr_returned_date_sk#93]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int,sr_return_quantity:int,sr_return_amt:decimal(7,2)>
+
+(69) ColumnarToRow [codegen id : 24]
+Input [5]: [sr_item_sk#89, sr_ticket_number#90, sr_return_quantity#91, sr_return_amt#92, sr_returned_date_sk#93]
+
+(70) Filter [codegen id : 24]
+Input [5]: [sr_item_sk#89, sr_ticket_number#90, sr_return_quantity#91, sr_return_amt#92, sr_returned_date_sk#93]
+Condition : (((isnotnull(sr_return_amt#92) AND (sr_return_amt#92 > 10000.00)) AND isnotnull(sr_ticket_number#90)) AND isnotnull(sr_item_sk#89))
+
+(71) Project [codegen id : 24]
+Output [4]: [sr_item_sk#89, sr_ticket_number#90, sr_return_quantity#91, sr_return_amt#92]
+Input [5]: [sr_item_sk#89, sr_ticket_number#90, sr_return_quantity#91, sr_return_amt#92, sr_returned_date_sk#93]
+
+(72) Exchange
+Input [4]: [sr_item_sk#89, sr_ticket_number#90, sr_return_quantity#91, sr_return_amt#92]
+Arguments: hashpartitioning(sr_ticket_number#90, sr_item_sk#89, 5), ENSURE_REQUIREMENTS, [id=#94]
+
+(73) Sort [codegen id : 25]
+Input [4]: [sr_item_sk#89, sr_ticket_number#90, sr_return_quantity#91, sr_return_amt#92]
+Arguments: [sr_ticket_number#90 ASC NULLS FIRST, sr_item_sk#89 ASC NULLS FIRST], false, 0
+
+(74) SortMergeJoin [codegen id : 26]
+Left keys [2]: [ss_ticket_number#82, ss_item_sk#81]
+Right keys [2]: [sr_ticket_number#90, sr_item_sk#89]
+Join condition: None
+
+(75) Project [codegen id : 26]
+Output [5]: [ss_item_sk#81, ss_quantity#83, ss_net_paid#84, sr_return_quantity#91, sr_return_amt#92]
+Input [8]: [ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_net_paid#84, sr_item_sk#89, sr_ticket_number#90, sr_return_quantity#91, sr_return_amt#92]
+
+(76) HashAggregate [codegen id : 26]
+Input [5]: [ss_item_sk#81, ss_quantity#83, ss_net_paid#84, sr_return_quantity#91, sr_return_amt#92]
+Keys [1]: [ss_item_sk#81]
+Functions [4]: [partial_sum(coalesce(sr_return_quantity#91, 0)), partial_sum(coalesce(ss_quantity#83, 0)), partial_sum(coalesce(cast(sr_return_amt#92 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#84 as decimal(12,2)), 0.00))]
+Aggregate Attributes [6]: [sum#95, sum#96, sum#97, isEmpty#98, sum#99, isEmpty#100]
+Results [7]: [ss_item_sk#81, sum#101, sum#102, sum#103, isEmpty#104, sum#105, isEmpty#106]
+
+(77) Exchange
+Input [7]: [ss_item_sk#81, sum#101, sum#102, sum#103, isEmpty#104, sum#105, isEmpty#106]
+Arguments: hashpartitioning(ss_item_sk#81, 5), ENSURE_REQUIREMENTS, [id=#107]
+
+(78) HashAggregate [codegen id : 27]
+Input [7]: [ss_item_sk#81, sum#101, sum#102, sum#103, isEmpty#104, sum#105, isEmpty#106]
+Keys [1]: [ss_item_sk#81]
+Functions [4]: [sum(coalesce(sr_return_quantity#91, 0)), sum(coalesce(ss_quantity#83, 0)), sum(coalesce(cast(sr_return_amt#92 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#84 as decimal(12,2)), 0.00))]
+Aggregate Attributes [4]: [sum(coalesce(sr_return_quantity#91, 0))#108, sum(coalesce(ss_quantity#83, 0))#109, sum(coalesce(cast(sr_return_amt#92 as decimal(12,2)), 0.00))#110, sum(coalesce(cast(ss_net_paid#84 as decimal(12,2)), 0.00))#111]
+Results [3]: [ss_item_sk#81 AS item#112, CheckOverflow((promote_precision(cast(sum(coalesce(sr_return_quantity#91, 0))#108 as decimal(15,4))) / promote_precision(cast(sum(coalesce(ss_quantity#83, 0))#109 as decimal(15,4)))), DecimalType(35,20), true) AS return_ratio#113, CheckOverflow((promote_precision(cast(sum(coalesce(cast(sr_return_amt#92 as decimal(12,2)), 0.00))#110 as decimal(15,4))) / promote_precision(cast(sum(coalesce(cast(ss_net_paid#84 as decimal(12,2)), 0.00))#111 as decimal(15,4)))), DecimalType(35,20), true) AS currency_ratio#114]
+
+(79) Exchange
+Input [3]: [item#112, return_ratio#113, currency_ratio#114]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#115]
+
+(80) Sort [codegen id : 28]
+Input [3]: [item#112, return_ratio#113, currency_ratio#114]
+Arguments: [return_ratio#113 ASC NULLS FIRST], false, 0
+
+(81) Window
+Input [3]: [item#112, return_ratio#113, currency_ratio#114]
+Arguments: [rank(return_ratio#113) windowspecdefinition(return_ratio#113 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#116], [return_ratio#113 ASC NULLS FIRST]
+
+(82) Sort [codegen id : 29]
+Input [4]: [item#112, return_ratio#113, currency_ratio#114, return_rank#116]
+Arguments: [currency_ratio#114 ASC NULLS FIRST], false, 0
+
+(83) Window
+Input [4]: [item#112, return_ratio#113, currency_ratio#114, return_rank#116]
+Arguments: [rank(currency_ratio#114) windowspecdefinition(currency_ratio#114 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#117], [currency_ratio#114 ASC NULLS FIRST]
+
+(84) Filter [codegen id : 30]
+Input [5]: [item#112, return_ratio#113, currency_ratio#114, return_rank#116, currency_rank#117]
+Condition : ((return_rank#116 <= 10) OR (currency_rank#117 <= 10))
+
+(85) Project [codegen id : 30]
+Output [5]: [store AS channel#118, item#112, return_ratio#113, return_rank#116, currency_rank#117]
+Input [5]: [item#112, return_ratio#113, currency_ratio#114, return_rank#116, currency_rank#117]
+
+(86) Union
+
+(87) HashAggregate [codegen id : 31]
+Input [5]: [channel#42, item#36, return_ratio#37, return_rank#40, currency_rank#41]
+Keys [5]: [channel#42, item#36, return_ratio#37, return_rank#40, currency_rank#41]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [channel#42, item#36, return_ratio#37, return_rank#40, currency_rank#41]
+
+(88) Exchange
+Input [5]: [channel#42, item#36, return_ratio#37, return_rank#40, currency_rank#41]
+Arguments: hashpartitioning(channel#42, item#36, return_ratio#37, return_rank#40, currency_rank#41, 5), ENSURE_REQUIREMENTS, [id=#119]
+
+(89) HashAggregate [codegen id : 32]
+Input [5]: [channel#42, item#36, return_ratio#37, return_rank#40, currency_rank#41]
+Keys [5]: [channel#42, item#36, return_ratio#37, return_rank#40, currency_rank#41]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [channel#42, item#36, return_ratio#37, return_rank#40, currency_rank#41]
+
+(90) TakeOrderedAndProject
+Input [5]: [channel#42, item#36, return_ratio#37, return_rank#40, currency_rank#41]
+Arguments: 100, [channel#42 ASC NULLS FIRST, return_rank#40 ASC NULLS FIRST, currency_rank#41 ASC NULLS FIRST], [channel#42, item#36, return_ratio#37, return_rank#40, currency_rank#41]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#6 IN dynamicpruning#7
+ReusedExchange (91)
+
+
+(91) ReusedExchange [Reuses operator id: 9]
+Output [1]: [d_date_sk#8]
+
+Subquery:2 Hosting operator id = 32 Hosting Expression = cs_sold_date_sk#48 IN dynamicpruning#7
+
+Subquery:3 Hosting operator id = 59 Hosting Expression = ss_sold_date_sk#86 IN dynamicpruning#7
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.sf100/simplified.txt
new file mode 100644
index 0000000000000..60d16f8b606af
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.sf100/simplified.txt
@@ -0,0 +1,160 @@
+TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio]
+  WholeStageCodegen (32)
+    HashAggregate [channel,item,return_ratio,return_rank,currency_rank]
+      InputAdapter
+        Exchange [channel,item,return_ratio,return_rank,currency_rank] #1
+          WholeStageCodegen (31)
+            HashAggregate [channel,item,return_ratio,return_rank,currency_rank]
+              InputAdapter
+                Union
+                  WholeStageCodegen (10)
+                    Project [item,return_ratio,return_rank,currency_rank]
+                      Filter [return_rank,currency_rank]
+                        InputAdapter
+                          Window [currency_ratio]
+                            WholeStageCodegen (9)
+                              Sort [currency_ratio]
+                                InputAdapter
+                                  Window [return_ratio]
+                                    WholeStageCodegen (8)
+                                      Sort [return_ratio]
+                                        InputAdapter
+                                          Exchange #2
+                                            WholeStageCodegen (7)
+                                              HashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty]
+                                                InputAdapter
+                                                  Exchange [ws_item_sk] #3
+                                                    WholeStageCodegen (6)
+                                                      HashAggregate [ws_item_sk,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty]
+                                                        Project [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt]
+                                                          SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk]
+                                                            InputAdapter
+                                                              WholeStageCodegen (3)
+                                                                Sort [ws_order_number,ws_item_sk]
+                                                                  InputAdapter
+                                                                    Exchange [ws_order_number,ws_item_sk] #4
+                                                                      WholeStageCodegen (2)
+                                                                        Project [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid]
+                                                                          BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                            Project [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk]
+                                                                              Filter [ws_net_profit,ws_net_paid,ws_quantity,ws_order_number,ws_item_sk]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk]
+                                                                                      SubqueryBroadcast [d_date_sk] #1
+                                                                                        ReusedExchange [d_date_sk] #5
+                                                                            InputAdapter
+                                                                              BroadcastExchange #5
+                                                                                WholeStageCodegen (1)
+                                                                                  Project [d_date_sk]
+                                                                                    Filter [d_year,d_moy,d_date_sk]
+                                                                                      ColumnarToRow
+                                                                                        InputAdapter
+                                                                                          Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                                            InputAdapter
+                                                              WholeStageCodegen (5)
+                                                                Sort [wr_order_number,wr_item_sk]
+                                                                  InputAdapter
+                                                                    Exchange [wr_order_number,wr_item_sk] #6
+                                                                      WholeStageCodegen (4)
+                                                                        Project [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt]
+                                                                          Filter [wr_return_amt,wr_order_number,wr_item_sk]
+                                                                            ColumnarToRow
+                                                                              InputAdapter
+                                                                                Scan parquet default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk]
+                  WholeStageCodegen (20)
+                    Project [item,return_ratio,return_rank,currency_rank]
+                      Filter [return_rank,currency_rank]
+                        InputAdapter
+                          Window [currency_ratio]
+                            WholeStageCodegen (19)
+                              Sort [currency_ratio]
+                                InputAdapter
+                                  Window [return_ratio]
+                                    WholeStageCodegen (18)
+                                      Sort [return_ratio]
+                                        InputAdapter
+                                          Exchange #7
+                                            WholeStageCodegen (17)
+                                              HashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty]
+                                                InputAdapter
+                                                  Exchange [cs_item_sk] #8
+                                                    WholeStageCodegen (16)
+                                                      HashAggregate [cs_item_sk,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty]
+                                                        Project [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount]
+                                                          SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk]
+                                                            InputAdapter
+                                                              WholeStageCodegen (13)
+                                                                Sort [cs_order_number,cs_item_sk]
+                                                                  InputAdapter
+                                                                    Exchange [cs_order_number,cs_item_sk] #9
+                                                                      WholeStageCodegen (12)
+                                                                        Project [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid]
+                                                                          BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                            Project [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk]
+                                                                              Filter [cs_net_profit,cs_net_paid,cs_quantity,cs_order_number,cs_item_sk]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk]
+                                                                                      ReusedSubquery [d_date_sk] #1
+                                                                            InputAdapter
+                                                                              ReusedExchange [d_date_sk] #5
+                                                            InputAdapter
+                                                              WholeStageCodegen (15)
+                                                                Sort [cr_order_number,cr_item_sk]
+                                                                  InputAdapter
+                                                                    Exchange [cr_order_number,cr_item_sk] #10
+                                                                      WholeStageCodegen (14)
+                                                                        Project [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount]
+                                                                          Filter [cr_return_amount,cr_order_number,cr_item_sk]
+                                                                            ColumnarToRow
+                                                                              InputAdapter
+                                                                                Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk]
+                  WholeStageCodegen (30)
+                    Project [item,return_ratio,return_rank,currency_rank]
+                      Filter [return_rank,currency_rank]
+                        InputAdapter
+                          Window [currency_ratio]
+                            WholeStageCodegen (29)
+                              Sort [currency_ratio]
+                                InputAdapter
+                                  Window [return_ratio]
+                                    WholeStageCodegen (28)
+                                      Sort [return_ratio]
+                                        InputAdapter
+                                          Exchange #11
+                                            WholeStageCodegen (27)
+                                              HashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty]
+                                                InputAdapter
+                                                  Exchange [ss_item_sk] #12
+                                                    WholeStageCodegen (26)
+                                                      HashAggregate [ss_item_sk,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty]
+                                                        Project [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt]
+                                                          SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                                            InputAdapter
+                                                              WholeStageCodegen (23)
+                                                                Sort [ss_ticket_number,ss_item_sk]
+                                                                  InputAdapter
+                                                                    Exchange [ss_ticket_number,ss_item_sk] #13
+                                                                      WholeStageCodegen (22)
+                                                                        Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid]
+                                                                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                            Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk]
+                                                                              Filter [ss_net_profit,ss_net_paid,ss_quantity,ss_ticket_number,ss_item_sk]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk]
+                                                                                      ReusedSubquery [d_date_sk] #1
+                                                                            InputAdapter
+                                                                              ReusedExchange [d_date_sk] #5
+                                                            InputAdapter
+                                                              WholeStageCodegen (25)
+                                                                Sort [sr_ticket_number,sr_item_sk]
+                                                                  InputAdapter
+                                                                    Exchange [sr_ticket_number,sr_item_sk] #14
+                                                                      WholeStageCodegen (24)
+                                                                        Project [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt]
+                                                                          Filter [sr_return_amt,sr_ticket_number,sr_item_sk]
+                                                                            ColumnarToRow
+                                                                              InputAdapter
+                                                                                Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/explain.txt
new file mode 100644
index 0000000000000..f5411b5681aad
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/explain.txt
@@ -0,0 +1,465 @@
+== Physical Plan ==
+TakeOrderedAndProject (81)
++- * HashAggregate (80)
+   +- Exchange (79)
+      +- * HashAggregate (78)
+         +- Union (77)
+            :- * Project (28)
+            :  +- * Filter (27)
+            :     +- Window (26)
+            :        +- * Sort (25)
+            :           +- Window (24)
+            :              +- * Sort (23)
+            :                 +- Exchange (22)
+            :                    +- * HashAggregate (21)
+            :                       +- Exchange (20)
+            :                          +- * HashAggregate (19)
+            :                             +- * Project (18)
+            :                                +- * BroadcastHashJoin Inner BuildRight (17)
+            :                                   :- * Project (11)
+            :                                   :  +- * BroadcastHashJoin Inner BuildLeft (10)
+            :                                   :     :- BroadcastExchange (5)
+            :                                   :     :  +- * Project (4)
+            :                                   :     :     +- * Filter (3)
+            :                                   :     :        +- * ColumnarToRow (2)
+            :                                   :     :           +- Scan parquet default.web_sales (1)
+            :                                   :     +- * Project (9)
+            :                                   :        +- * Filter (8)
+            :                                   :           +- * ColumnarToRow (7)
+            :                                   :              +- Scan parquet default.web_returns (6)
+            :                                   +- BroadcastExchange (16)
+            :                                      +- * Project (15)
+            :                                         +- * Filter (14)
+            :                                            +- * ColumnarToRow (13)
+            :                                               +- Scan parquet default.date_dim (12)
+            :- * Project (52)
+            :  +- * Filter (51)
+            :     +- Window (50)
+            :        +- * Sort (49)
+            :           +- Window (48)
+            :              +- * Sort (47)
+            :                 +- Exchange (46)
+            :                    +- * HashAggregate (45)
+            :                       +- Exchange (44)
+            :                          +- * HashAggregate (43)
+            :                             +- * Project (42)
+            :                                +- * BroadcastHashJoin Inner BuildRight (41)
+            :                                   :- * Project (39)
+            :                                   :  +- * BroadcastHashJoin Inner BuildLeft (38)
+            :                                   :     :- BroadcastExchange (33)
+            :                                   :     :  +- * Project (32)
+            :                                   :     :     +- * Filter (31)
+            :                                   :     :        +- * ColumnarToRow (30)
+            :                                   :     :           +- Scan parquet default.catalog_sales (29)
+            :                                   :     +- * Project (37)
+            :                                   :        +- * Filter (36)
+            :                                   :           +- * ColumnarToRow (35)
+            :                                   :              +- Scan parquet default.catalog_returns (34)
+            :                                   +- ReusedExchange (40)
+            +- * Project (76)
+               +- * Filter (75)
+                  +- Window (74)
+                     +- * Sort (73)
+                        +- Window (72)
+                           +- * Sort (71)
+                              +- Exchange (70)
+                                 +- * HashAggregate (69)
+                                    +- Exchange (68)
+                                       +- * HashAggregate (67)
+                                          +- * Project (66)
+                                             +- * BroadcastHashJoin Inner BuildRight (65)
+                                                :- * Project (63)
+                                                :  +- * BroadcastHashJoin Inner BuildLeft (62)
+                                                :     :- BroadcastExchange (57)
+                                                :     :  +- * Project (56)
+                                                :     :     +- * Filter (55)
+                                                :     :        +- * ColumnarToRow (54)
+                                                :     :           +- Scan parquet default.store_sales (53)
+                                                :     +- * Project (61)
+                                                :        +- * Filter (60)
+                                                :           +- * ColumnarToRow (59)
+                                                :              +- Scan parquet default.store_returns (58)
+                                                +- ReusedExchange (64)
+
+
+(1) Scan parquet default.web_sales
+Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#6), dynamicpruningexpression(ws_sold_date_sk#6 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ws_net_profit), IsNotNull(ws_net_paid), IsNotNull(ws_quantity), GreaterThan(ws_net_profit,1.00), GreaterThan(ws_net_paid,0.00), GreaterThan(ws_quantity,0), IsNotNull(ws_order_number), IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_order_number:int,ws_quantity:int,ws_net_paid:decimal(7,2),ws_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6]
+
+(3) Filter [codegen id : 1]
+Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6]
+Condition : (((((((isnotnull(ws_net_profit#5) AND isnotnull(ws_net_paid#4)) AND isnotnull(ws_quantity#3)) AND (ws_net_profit#5 > 1.00)) AND (ws_net_paid#4 > 0.00)) AND (ws_quantity#3 > 0)) AND isnotnull(ws_order_number#2)) AND isnotnull(ws_item_sk#1))
+
+(4) Project [codegen id : 1]
+Output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6]
+Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6]
+
+(5) BroadcastExchange
+Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [id=#8]
+
+(6) Scan parquet default.web_returns
+Output [5]: [wr_item_sk#9, wr_order_number#10, wr_return_quantity#11, wr_return_amt#12, wr_returned_date_sk#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_returns]
+PushedFilters: [IsNotNull(wr_return_amt), GreaterThan(wr_return_amt,10000.00), IsNotNull(wr_order_number), IsNotNull(wr_item_sk)]
+ReadSchema: struct<wr_item_sk:int,wr_order_number:int,wr_return_quantity:int,wr_return_amt:decimal(7,2)>
+
+(7) ColumnarToRow
+Input [5]: [wr_item_sk#9, wr_order_number#10, wr_return_quantity#11, wr_return_amt#12, wr_returned_date_sk#13]
+
+(8) Filter
+Input [5]: [wr_item_sk#9, wr_order_number#10, wr_return_quantity#11, wr_return_amt#12, wr_returned_date_sk#13]
+Condition : (((isnotnull(wr_return_amt#12) AND (wr_return_amt#12 > 10000.00)) AND isnotnull(wr_order_number#10)) AND isnotnull(wr_item_sk#9))
+
+(9) Project
+Output [4]: [wr_item_sk#9, wr_order_number#10, wr_return_quantity#11, wr_return_amt#12]
+Input [5]: [wr_item_sk#9, wr_order_number#10, wr_return_quantity#11, wr_return_amt#12, wr_returned_date_sk#13]
+
+(10) BroadcastHashJoin [codegen id : 3]
+Left keys [2]: [ws_order_number#2, ws_item_sk#1]
+Right keys [2]: [wr_order_number#10, wr_item_sk#9]
+Join condition: None
+
+(11) Project [codegen id : 3]
+Output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#11, wr_return_amt#12]
+Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#9, wr_order_number#10, wr_return_quantity#11, wr_return_amt#12]
+
+(12) Scan parquet default.date_dim
+Output [3]: [d_date_sk#14, d_year#15, d_moy#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(13) ColumnarToRow [codegen id : 2]
+Input [3]: [d_date_sk#14, d_year#15, d_moy#16]
+
+(14) Filter [codegen id : 2]
+Input [3]: [d_date_sk#14, d_year#15, d_moy#16]
+Condition : ((((isnotnull(d_year#15) AND isnotnull(d_moy#16)) AND (d_year#15 = 2001)) AND (d_moy#16 = 12)) AND isnotnull(d_date_sk#14))
+
+(15) Project [codegen id : 2]
+Output [1]: [d_date_sk#14]
+Input [3]: [d_date_sk#14, d_year#15, d_moy#16]
+
+(16) BroadcastExchange
+Input [1]: [d_date_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(17) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ws_sold_date_sk#6]
+Right keys [1]: [d_date_sk#14]
+Join condition: None
+
+(18) Project [codegen id : 3]
+Output [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#11, wr_return_amt#12]
+Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#11, wr_return_amt#12, d_date_sk#14]
+
+(19) HashAggregate [codegen id : 3]
+Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#11, wr_return_amt#12]
+Keys [1]: [ws_item_sk#1]
+Functions [4]: [partial_sum(coalesce(wr_return_quantity#11, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#12 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))]
+Aggregate Attributes [6]: [sum#18, sum#19, sum#20, isEmpty#21, sum#22, isEmpty#23]
+Results [7]: [ws_item_sk#1, sum#24, sum#25, sum#26, isEmpty#27, sum#28, isEmpty#29]
+
+(20) Exchange
+Input [7]: [ws_item_sk#1, sum#24, sum#25, sum#26, isEmpty#27, sum#28, isEmpty#29]
+Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(21) HashAggregate [codegen id : 4]
+Input [7]: [ws_item_sk#1, sum#24, sum#25, sum#26, isEmpty#27, sum#28, isEmpty#29]
+Keys [1]: [ws_item_sk#1]
+Functions [4]: [sum(coalesce(wr_return_quantity#11, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#12 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))]
+Aggregate Attributes [4]: [sum(coalesce(wr_return_quantity#11, 0))#31, sum(coalesce(ws_quantity#3, 0))#32, sum(coalesce(cast(wr_return_amt#12 as decimal(12,2)), 0.00))#33, sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#34]
+Results [3]: [ws_item_sk#1 AS item#35, CheckOverflow((promote_precision(cast(sum(coalesce(wr_return_quantity#11, 0))#31 as decimal(15,4))) / promote_precision(cast(sum(coalesce(ws_quantity#3, 0))#32 as decimal(15,4)))), DecimalType(35,20), true) AS return_ratio#36, CheckOverflow((promote_precision(cast(sum(coalesce(cast(wr_return_amt#12 as decimal(12,2)), 0.00))#33 as decimal(15,4))) / promote_precision(cast(sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#34 as decimal(15,4)))), DecimalType(35,20), true) AS currency_ratio#37]
+
+(22) Exchange
+Input [3]: [item#35, return_ratio#36, currency_ratio#37]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#38]
+
+(23) Sort [codegen id : 5]
+Input [3]: [item#35, return_ratio#36, currency_ratio#37]
+Arguments: [return_ratio#36 ASC NULLS FIRST], false, 0
+
+(24) Window
+Input [3]: [item#35, return_ratio#36, currency_ratio#37]
+Arguments: [rank(return_ratio#36) windowspecdefinition(return_ratio#36 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#39], [return_ratio#36 ASC NULLS FIRST]
+
+(25) Sort [codegen id : 6]
+Input [4]: [item#35, return_ratio#36, currency_ratio#37, return_rank#39]
+Arguments: [currency_ratio#37 ASC NULLS FIRST], false, 0
+
+(26) Window
+Input [4]: [item#35, return_ratio#36, currency_ratio#37, return_rank#39]
+Arguments: [rank(currency_ratio#37) windowspecdefinition(currency_ratio#37 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#40], [currency_ratio#37 ASC NULLS FIRST]
+
+(27) Filter [codegen id : 7]
+Input [5]: [item#35, return_ratio#36, currency_ratio#37, return_rank#39, currency_rank#40]
+Condition : ((return_rank#39 <= 10) OR (currency_rank#40 <= 10))
+
+(28) Project [codegen id : 7]
+Output [5]: [web AS channel#41, item#35, return_ratio#36, return_rank#39, currency_rank#40]
+Input [5]: [item#35, return_ratio#36, currency_ratio#37, return_rank#39, currency_rank#40]
+
+(29) Scan parquet default.catalog_sales
+Output [6]: [cs_item_sk#42, cs_order_number#43, cs_quantity#44, cs_net_paid#45, cs_net_profit#46, cs_sold_date_sk#47]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#47), dynamicpruningexpression(cs_sold_date_sk#47 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int,cs_order_number:int,cs_quantity:int,cs_net_paid:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(30) ColumnarToRow [codegen id : 8]
+Input [6]: [cs_item_sk#42, cs_order_number#43, cs_quantity#44, cs_net_paid#45, cs_net_profit#46, cs_sold_date_sk#47]
+
+(31) Filter [codegen id : 8]
+Input [6]: [cs_item_sk#42, cs_order_number#43, cs_quantity#44, cs_net_paid#45, cs_net_profit#46, cs_sold_date_sk#47]
+Condition : (((((((isnotnull(cs_net_profit#46) AND isnotnull(cs_net_paid#45)) AND isnotnull(cs_quantity#44)) AND (cs_net_profit#46 > 1.00)) AND (cs_net_paid#45 > 0.00)) AND (cs_quantity#44 > 0)) AND isnotnull(cs_order_number#43)) AND isnotnull(cs_item_sk#42))
+
+(32) Project [codegen id : 8]
+Output [5]: [cs_item_sk#42, cs_order_number#43, cs_quantity#44, cs_net_paid#45, cs_sold_date_sk#47]
+Input [6]: [cs_item_sk#42, cs_order_number#43, cs_quantity#44, cs_net_paid#45, cs_net_profit#46, cs_sold_date_sk#47]
+
+(33) BroadcastExchange
+Input [5]: [cs_item_sk#42, cs_order_number#43, cs_quantity#44, cs_net_paid#45, cs_sold_date_sk#47]
+Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [id=#48]
+
+(34) Scan parquet default.catalog_returns
+Output [5]: [cr_item_sk#49, cr_order_number#50, cr_return_quantity#51, cr_return_amount#52, cr_returned_date_sk#53]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int,cr_return_quantity:int,cr_return_amount:decimal(7,2)>
+
+(35) ColumnarToRow
+Input [5]: [cr_item_sk#49, cr_order_number#50, cr_return_quantity#51, cr_return_amount#52, cr_returned_date_sk#53]
+
+(36) Filter
+Input [5]: [cr_item_sk#49, cr_order_number#50, cr_return_quantity#51, cr_return_amount#52, cr_returned_date_sk#53]
+Condition : (((isnotnull(cr_return_amount#52) AND (cr_return_amount#52 > 10000.00)) AND isnotnull(cr_order_number#50)) AND isnotnull(cr_item_sk#49))
+
+(37) Project
+Output [4]: [cr_item_sk#49, cr_order_number#50, cr_return_quantity#51, cr_return_amount#52]
+Input [5]: [cr_item_sk#49, cr_order_number#50, cr_return_quantity#51, cr_return_amount#52, cr_returned_date_sk#53]
+
+(38) BroadcastHashJoin [codegen id : 10]
+Left keys [2]: [cs_order_number#43, cs_item_sk#42]
+Right keys [2]: [cr_order_number#50, cr_item_sk#49]
+Join condition: None
+
+(39) Project [codegen id : 10]
+Output [6]: [cs_item_sk#42, cs_quantity#44, cs_net_paid#45, cs_sold_date_sk#47, cr_return_quantity#51, cr_return_amount#52]
+Input [9]: [cs_item_sk#42, cs_order_number#43, cs_quantity#44, cs_net_paid#45, cs_sold_date_sk#47, cr_item_sk#49, cr_order_number#50, cr_return_quantity#51, cr_return_amount#52]
+
+(40) ReusedExchange [Reuses operator id: 16]
+Output [1]: [d_date_sk#54]
+
+(41) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_sold_date_sk#47]
+Right keys [1]: [d_date_sk#54]
+Join condition: None
+
+(42) Project [codegen id : 10]
+Output [5]: [cs_item_sk#42, cs_quantity#44, cs_net_paid#45, cr_return_quantity#51, cr_return_amount#52]
+Input [7]: [cs_item_sk#42, cs_quantity#44, cs_net_paid#45, cs_sold_date_sk#47, cr_return_quantity#51, cr_return_amount#52, d_date_sk#54]
+
+(43) HashAggregate [codegen id : 10]
+Input [5]: [cs_item_sk#42, cs_quantity#44, cs_net_paid#45, cr_return_quantity#51, cr_return_amount#52]
+Keys [1]: [cs_item_sk#42]
+Functions [4]: [partial_sum(coalesce(cr_return_quantity#51, 0)), partial_sum(coalesce(cs_quantity#44, 0)), partial_sum(coalesce(cast(cr_return_amount#52 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#45 as decimal(12,2)), 0.00))]
+Aggregate Attributes [6]: [sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60]
+Results [7]: [cs_item_sk#42, sum#61, sum#62, sum#63, isEmpty#64, sum#65, isEmpty#66]
+
+(44) Exchange
+Input [7]: [cs_item_sk#42, sum#61, sum#62, sum#63, isEmpty#64, sum#65, isEmpty#66]
+Arguments: hashpartitioning(cs_item_sk#42, 5), ENSURE_REQUIREMENTS, [id=#67]
+
+(45) HashAggregate [codegen id : 11]
+Input [7]: [cs_item_sk#42, sum#61, sum#62, sum#63, isEmpty#64, sum#65, isEmpty#66]
+Keys [1]: [cs_item_sk#42]
+Functions [4]: [sum(coalesce(cr_return_quantity#51, 0)), sum(coalesce(cs_quantity#44, 0)), sum(coalesce(cast(cr_return_amount#52 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#45 as decimal(12,2)), 0.00))]
+Aggregate Attributes [4]: [sum(coalesce(cr_return_quantity#51, 0))#68, sum(coalesce(cs_quantity#44, 0))#69, sum(coalesce(cast(cr_return_amount#52 as decimal(12,2)), 0.00))#70, sum(coalesce(cast(cs_net_paid#45 as decimal(12,2)), 0.00))#71]
+Results [3]: [cs_item_sk#42 AS item#72, CheckOverflow((promote_precision(cast(sum(coalesce(cr_return_quantity#51, 0))#68 as decimal(15,4))) / promote_precision(cast(sum(coalesce(cs_quantity#44, 0))#69 as decimal(15,4)))), DecimalType(35,20), true) AS return_ratio#73, CheckOverflow((promote_precision(cast(sum(coalesce(cast(cr_return_amount#52 as decimal(12,2)), 0.00))#70 as decimal(15,4))) / promote_precision(cast(sum(coalesce(cast(cs_net_paid#45 as decimal(12,2)), 0.00))#71 as decimal(15,4)))), DecimalType(35,20), true) AS currency_ratio#74]
+
+(46) Exchange
+Input [3]: [item#72, return_ratio#73, currency_ratio#74]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#75]
+
+(47) Sort [codegen id : 12]
+Input [3]: [item#72, return_ratio#73, currency_ratio#74]
+Arguments: [return_ratio#73 ASC NULLS FIRST], false, 0
+
+(48) Window
+Input [3]: [item#72, return_ratio#73, currency_ratio#74]
+Arguments: [rank(return_ratio#73) windowspecdefinition(return_ratio#73 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#76], [return_ratio#73 ASC NULLS FIRST]
+
+(49) Sort [codegen id : 13]
+Input [4]: [item#72, return_ratio#73, currency_ratio#74, return_rank#76]
+Arguments: [currency_ratio#74 ASC NULLS FIRST], false, 0
+
+(50) Window
+Input [4]: [item#72, return_ratio#73, currency_ratio#74, return_rank#76]
+Arguments: [rank(currency_ratio#74) windowspecdefinition(currency_ratio#74 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#77], [currency_ratio#74 ASC NULLS FIRST]
+
+(51) Filter [codegen id : 14]
+Input [5]: [item#72, return_ratio#73, currency_ratio#74, return_rank#76, currency_rank#77]
+Condition : ((return_rank#76 <= 10) OR (currency_rank#77 <= 10))
+
+(52) Project [codegen id : 14]
+Output [5]: [catalog AS channel#78, item#72, return_ratio#73, return_rank#76, currency_rank#77]
+Input [5]: [item#72, return_ratio#73, currency_ratio#74, return_rank#76, currency_rank#77]
+
+(53) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_net_paid#82, ss_net_profit#83, ss_sold_date_sk#84]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#84), dynamicpruningexpression(ss_sold_date_sk#84 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ticket_number:int,ss_quantity:int,ss_net_paid:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(54) ColumnarToRow [codegen id : 15]
+Input [6]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_net_paid#82, ss_net_profit#83, ss_sold_date_sk#84]
+
+(55) Filter [codegen id : 15]
+Input [6]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_net_paid#82, ss_net_profit#83, ss_sold_date_sk#84]
+Condition : (((((((isnotnull(ss_net_profit#83) AND isnotnull(ss_net_paid#82)) AND isnotnull(ss_quantity#81)) AND (ss_net_profit#83 > 1.00)) AND (ss_net_paid#82 > 0.00)) AND (ss_quantity#81 > 0)) AND isnotnull(ss_ticket_number#80)) AND isnotnull(ss_item_sk#79))
+
+(56) Project [codegen id : 15]
+Output [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_net_paid#82, ss_sold_date_sk#84]
+Input [6]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_net_paid#82, ss_net_profit#83, ss_sold_date_sk#84]
+
+(57) BroadcastExchange
+Input [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_net_paid#82, ss_sold_date_sk#84]
+Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [id=#85]
+
+(58) Scan parquet default.store_returns
+Output [5]: [sr_item_sk#86, sr_ticket_number#87, sr_return_quantity#88, sr_return_amt#89, sr_returned_date_sk#90]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int,sr_return_quantity:int,sr_return_amt:decimal(7,2)>
+
+(59) ColumnarToRow
+Input [5]: [sr_item_sk#86, sr_ticket_number#87, sr_return_quantity#88, sr_return_amt#89, sr_returned_date_sk#90]
+
+(60) Filter
+Input [5]: [sr_item_sk#86, sr_ticket_number#87, sr_return_quantity#88, sr_return_amt#89, sr_returned_date_sk#90]
+Condition : (((isnotnull(sr_return_amt#89) AND (sr_return_amt#89 > 10000.00)) AND isnotnull(sr_ticket_number#87)) AND isnotnull(sr_item_sk#86))
+
+(61) Project
+Output [4]: [sr_item_sk#86, sr_ticket_number#87, sr_return_quantity#88, sr_return_amt#89]
+Input [5]: [sr_item_sk#86, sr_ticket_number#87, sr_return_quantity#88, sr_return_amt#89, sr_returned_date_sk#90]
+
+(62) BroadcastHashJoin [codegen id : 17]
+Left keys [2]: [ss_ticket_number#80, ss_item_sk#79]
+Right keys [2]: [sr_ticket_number#87, sr_item_sk#86]
+Join condition: None
+
+(63) Project [codegen id : 17]
+Output [6]: [ss_item_sk#79, ss_quantity#81, ss_net_paid#82, ss_sold_date_sk#84, sr_return_quantity#88, sr_return_amt#89]
+Input [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_net_paid#82, ss_sold_date_sk#84, sr_item_sk#86, sr_ticket_number#87, sr_return_quantity#88, sr_return_amt#89]
+
+(64) ReusedExchange [Reuses operator id: 16]
+Output [1]: [d_date_sk#91]
+
+(65) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ss_sold_date_sk#84]
+Right keys [1]: [d_date_sk#91]
+Join condition: None
+
+(66) Project [codegen id : 17]
+Output [5]: [ss_item_sk#79, ss_quantity#81, ss_net_paid#82, sr_return_quantity#88, sr_return_amt#89]
+Input [7]: [ss_item_sk#79, ss_quantity#81, ss_net_paid#82, ss_sold_date_sk#84, sr_return_quantity#88, sr_return_amt#89, d_date_sk#91]
+
+(67) HashAggregate [codegen id : 17]
+Input [5]: [ss_item_sk#79, ss_quantity#81, ss_net_paid#82, sr_return_quantity#88, sr_return_amt#89]
+Keys [1]: [ss_item_sk#79]
+Functions [4]: [partial_sum(coalesce(sr_return_quantity#88, 0)), partial_sum(coalesce(ss_quantity#81, 0)), partial_sum(coalesce(cast(sr_return_amt#89 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#82 as decimal(12,2)), 0.00))]
+Aggregate Attributes [6]: [sum#92, sum#93, sum#94, isEmpty#95, sum#96, isEmpty#97]
+Results [7]: [ss_item_sk#79, sum#98, sum#99, sum#100, isEmpty#101, sum#102, isEmpty#103]
+
+(68) Exchange
+Input [7]: [ss_item_sk#79, sum#98, sum#99, sum#100, isEmpty#101, sum#102, isEmpty#103]
+Arguments: hashpartitioning(ss_item_sk#79, 5), ENSURE_REQUIREMENTS, [id=#104]
+
+(69) HashAggregate [codegen id : 18]
+Input [7]: [ss_item_sk#79, sum#98, sum#99, sum#100, isEmpty#101, sum#102, isEmpty#103]
+Keys [1]: [ss_item_sk#79]
+Functions [4]: [sum(coalesce(sr_return_quantity#88, 0)), sum(coalesce(ss_quantity#81, 0)), sum(coalesce(cast(sr_return_amt#89 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#82 as decimal(12,2)), 0.00))]
+Aggregate Attributes [4]: [sum(coalesce(sr_return_quantity#88, 0))#105, sum(coalesce(ss_quantity#81, 0))#106, sum(coalesce(cast(sr_return_amt#89 as decimal(12,2)), 0.00))#107, sum(coalesce(cast(ss_net_paid#82 as decimal(12,2)), 0.00))#108]
+Results [3]: [ss_item_sk#79 AS item#109, CheckOverflow((promote_precision(cast(sum(coalesce(sr_return_quantity#88, 0))#105 as decimal(15,4))) / promote_precision(cast(sum(coalesce(ss_quantity#81, 0))#106 as decimal(15,4)))), DecimalType(35,20), true) AS return_ratio#110, CheckOverflow((promote_precision(cast(sum(coalesce(cast(sr_return_amt#89 as decimal(12,2)), 0.00))#107 as decimal(15,4))) / promote_precision(cast(sum(coalesce(cast(ss_net_paid#82 as decimal(12,2)), 0.00))#108 as decimal(15,4)))), DecimalType(35,20), true) AS currency_ratio#111]
+
+(70) Exchange
+Input [3]: [item#109, return_ratio#110, currency_ratio#111]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#112]
+
+(71) Sort [codegen id : 19]
+Input [3]: [item#109, return_ratio#110, currency_ratio#111]
+Arguments: [return_ratio#110 ASC NULLS FIRST], false, 0
+
+(72) Window
+Input [3]: [item#109, return_ratio#110, currency_ratio#111]
+Arguments: [rank(return_ratio#110) windowspecdefinition(return_ratio#110 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#113], [return_ratio#110 ASC NULLS FIRST]
+
+(73) Sort [codegen id : 20]
+Input [4]: [item#109, return_ratio#110, currency_ratio#111, return_rank#113]
+Arguments: [currency_ratio#111 ASC NULLS FIRST], false, 0
+
+(74) Window
+Input [4]: [item#109, return_ratio#110, currency_ratio#111, return_rank#113]
+Arguments: [rank(currency_ratio#111) windowspecdefinition(currency_ratio#111 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#114], [currency_ratio#111 ASC NULLS FIRST]
+
+(75) Filter [codegen id : 21]
+Input [5]: [item#109, return_ratio#110, currency_ratio#111, return_rank#113, currency_rank#114]
+Condition : ((return_rank#113 <= 10) OR (currency_rank#114 <= 10))
+
+(76) Project [codegen id : 21]
+Output [5]: [store AS channel#115, item#109, return_ratio#110, return_rank#113, currency_rank#114]
+Input [5]: [item#109, return_ratio#110, currency_ratio#111, return_rank#113, currency_rank#114]
+
+(77) Union
+
+(78) HashAggregate [codegen id : 22]
+Input [5]: [channel#41, item#35, return_ratio#36, return_rank#39, currency_rank#40]
+Keys [5]: [channel#41, item#35, return_ratio#36, return_rank#39, currency_rank#40]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [channel#41, item#35, return_ratio#36, return_rank#39, currency_rank#40]
+
+(79) Exchange
+Input [5]: [channel#41, item#35, return_ratio#36, return_rank#39, currency_rank#40]
+Arguments: hashpartitioning(channel#41, item#35, return_ratio#36, return_rank#39, currency_rank#40, 5), ENSURE_REQUIREMENTS, [id=#116]
+
+(80) HashAggregate [codegen id : 23]
+Input [5]: [channel#41, item#35, return_ratio#36, return_rank#39, currency_rank#40]
+Keys [5]: [channel#41, item#35, return_ratio#36, return_rank#39, currency_rank#40]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [channel#41, item#35, return_ratio#36, return_rank#39, currency_rank#40]
+
+(81) TakeOrderedAndProject
+Input [5]: [channel#41, item#35, return_ratio#36, return_rank#39, currency_rank#40]
+Arguments: 100, [channel#41 ASC NULLS FIRST, return_rank#39 ASC NULLS FIRST, currency_rank#40 ASC NULLS FIRST], [channel#41, item#35, return_ratio#36, return_rank#39, currency_rank#40]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#6 IN dynamicpruning#7
+ReusedExchange (82)
+
+
+(82) ReusedExchange [Reuses operator id: 16]
+Output [1]: [d_date_sk#14]
+
+Subquery:2 Hosting operator id = 29 Hosting Expression = cs_sold_date_sk#47 IN dynamicpruning#7
+
+Subquery:3 Hosting operator id = 53 Hosting Expression = ss_sold_date_sk#84 IN dynamicpruning#7
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/simplified.txt
new file mode 100644
index 0000000000000..0b548432d6fed
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/simplified.txt
@@ -0,0 +1,133 @@
+TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio]
+  WholeStageCodegen (23)
+    HashAggregate [channel,item,return_ratio,return_rank,currency_rank]
+      InputAdapter
+        Exchange [channel,item,return_ratio,return_rank,currency_rank] #1
+          WholeStageCodegen (22)
+            HashAggregate [channel,item,return_ratio,return_rank,currency_rank]
+              InputAdapter
+                Union
+                  WholeStageCodegen (7)
+                    Project [item,return_ratio,return_rank,currency_rank]
+                      Filter [return_rank,currency_rank]
+                        InputAdapter
+                          Window [currency_ratio]
+                            WholeStageCodegen (6)
+                              Sort [currency_ratio]
+                                InputAdapter
+                                  Window [return_ratio]
+                                    WholeStageCodegen (5)
+                                      Sort [return_ratio]
+                                        InputAdapter
+                                          Exchange #2
+                                            WholeStageCodegen (4)
+                                              HashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty]
+                                                InputAdapter
+                                                  Exchange [ws_item_sk] #3
+                                                    WholeStageCodegen (3)
+                                                      HashAggregate [ws_item_sk,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty]
+                                                        Project [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt]
+                                                          BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                            Project [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt]
+                                                              BroadcastHashJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk]
+                                                                InputAdapter
+                                                                  BroadcastExchange #4
+                                                                    WholeStageCodegen (1)
+                                                                      Project [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk]
+                                                                        Filter [ws_net_profit,ws_net_paid,ws_quantity,ws_order_number,ws_item_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk]
+                                                                                SubqueryBroadcast [d_date_sk] #1
+                                                                                  ReusedExchange [d_date_sk] #5
+                                                                Project [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt]
+                                                                  Filter [wr_return_amt,wr_order_number,wr_item_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk]
+                                                            InputAdapter
+                                                              BroadcastExchange #5
+                                                                WholeStageCodegen (2)
+                                                                  Project [d_date_sk]
+                                                                    Filter [d_year,d_moy,d_date_sk]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                  WholeStageCodegen (14)
+                    Project [item,return_ratio,return_rank,currency_rank]
+                      Filter [return_rank,currency_rank]
+                        InputAdapter
+                          Window [currency_ratio]
+                            WholeStageCodegen (13)
+                              Sort [currency_ratio]
+                                InputAdapter
+                                  Window [return_ratio]
+                                    WholeStageCodegen (12)
+                                      Sort [return_ratio]
+                                        InputAdapter
+                                          Exchange #6
+                                            WholeStageCodegen (11)
+                                              HashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty]
+                                                InputAdapter
+                                                  Exchange [cs_item_sk] #7
+                                                    WholeStageCodegen (10)
+                                                      HashAggregate [cs_item_sk,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty]
+                                                        Project [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount]
+                                                          BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                            Project [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount]
+                                                              BroadcastHashJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk]
+                                                                InputAdapter
+                                                                  BroadcastExchange #8
+                                                                    WholeStageCodegen (8)
+                                                                      Project [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk]
+                                                                        Filter [cs_net_profit,cs_net_paid,cs_quantity,cs_order_number,cs_item_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk]
+                                                                                ReusedSubquery [d_date_sk] #1
+                                                                Project [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount]
+                                                                  Filter [cr_return_amount,cr_order_number,cr_item_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk]
+                                                            InputAdapter
+                                                              ReusedExchange [d_date_sk] #5
+                  WholeStageCodegen (21)
+                    Project [item,return_ratio,return_rank,currency_rank]
+                      Filter [return_rank,currency_rank]
+                        InputAdapter
+                          Window [currency_ratio]
+                            WholeStageCodegen (20)
+                              Sort [currency_ratio]
+                                InputAdapter
+                                  Window [return_ratio]
+                                    WholeStageCodegen (19)
+                                      Sort [return_ratio]
+                                        InputAdapter
+                                          Exchange #9
+                                            WholeStageCodegen (18)
+                                              HashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty]
+                                                InputAdapter
+                                                  Exchange [ss_item_sk] #10
+                                                    WholeStageCodegen (17)
+                                                      HashAggregate [ss_item_sk,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty]
+                                                        Project [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt]
+                                                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                            Project [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt]
+                                                              BroadcastHashJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                                                InputAdapter
+                                                                  BroadcastExchange #11
+                                                                    WholeStageCodegen (15)
+                                                                      Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk]
+                                                                        Filter [ss_net_profit,ss_net_paid,ss_quantity,ss_ticket_number,ss_item_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk]
+                                                                                ReusedSubquery [d_date_sk] #1
+                                                                Project [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt]
+                                                                  Filter [sr_return_amt,sr_ticket_number,sr_item_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk]
+                                                            InputAdapter
+                                                              ReusedExchange [d_date_sk] #5
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.sf100/explain.txt
new file mode 100644
index 0000000000000..ec4a566d0e4c4
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.sf100/explain.txt
@@ -0,0 +1,476 @@
+== Physical Plan ==
+TakeOrderedAndProject (81)
++- * HashAggregate (80)
+   +- Exchange (79)
+      +- * HashAggregate (78)
+         +- * Expand (77)
+            +- Union (76)
+               :- * HashAggregate (25)
+               :  +- Exchange (24)
+               :     +- * HashAggregate (23)
+               :        +- * Project (22)
+               :           +- * BroadcastHashJoin Inner BuildRight (21)
+               :              :- * Project (15)
+               :              :  +- * BroadcastHashJoin Inner BuildRight (14)
+               :              :     :- Union (9)
+               :              :     :  :- * Project (4)
+               :              :     :  :  +- * Filter (3)
+               :              :     :  :     +- * ColumnarToRow (2)
+               :              :     :  :        +- Scan parquet default.store_sales (1)
+               :              :     :  +- * Project (8)
+               :              :     :     +- * Filter (7)
+               :              :     :        +- * ColumnarToRow (6)
+               :              :     :           +- Scan parquet default.store_returns (5)
+               :              :     +- BroadcastExchange (13)
+               :              :        +- * Filter (12)
+               :              :           +- * ColumnarToRow (11)
+               :              :              +- Scan parquet default.store (10)
+               :              +- BroadcastExchange (20)
+               :                 +- * Project (19)
+               :                    +- * Filter (18)
+               :                       +- * ColumnarToRow (17)
+               :                          +- Scan parquet default.date_dim (16)
+               :- * HashAggregate (46)
+               :  +- Exchange (45)
+               :     +- * HashAggregate (44)
+               :        +- * Project (43)
+               :           +- * BroadcastHashJoin Inner BuildRight (42)
+               :              :- * Project (40)
+               :              :  +- * BroadcastHashJoin Inner BuildRight (39)
+               :              :     :- Union (34)
+               :              :     :  :- * Project (29)
+               :              :     :  :  +- * Filter (28)
+               :              :     :  :     +- * ColumnarToRow (27)
+               :              :     :  :        +- Scan parquet default.catalog_sales (26)
+               :              :     :  +- * Project (33)
+               :              :     :     +- * Filter (32)
+               :              :     :        +- * ColumnarToRow (31)
+               :              :     :           +- Scan parquet default.catalog_returns (30)
+               :              :     +- BroadcastExchange (38)
+               :              :        +- * Filter (37)
+               :              :           +- * ColumnarToRow (36)
+               :              :              +- Scan parquet default.catalog_page (35)
+               :              +- ReusedExchange (41)
+               +- * HashAggregate (75)
+                  +- Exchange (74)
+                     +- * HashAggregate (73)
+                        +- * Project (72)
+                           +- * BroadcastHashJoin Inner BuildRight (71)
+                              :- * Project (69)
+                              :  +- * BroadcastHashJoin Inner BuildRight (68)
+                              :     :- Union (63)
+                              :     :  :- * Project (50)
+                              :     :  :  +- * Filter (49)
+                              :     :  :     +- * ColumnarToRow (48)
+                              :     :  :        +- Scan parquet default.web_sales (47)
+                              :     :  +- * Project (62)
+                              :     :     +- * SortMergeJoin Inner (61)
+                              :     :        :- * Sort (54)
+                              :     :        :  +- Exchange (53)
+                              :     :        :     +- * ColumnarToRow (52)
+                              :     :        :        +- Scan parquet default.web_returns (51)
+                              :     :        +- * Sort (60)
+                              :     :           +- Exchange (59)
+                              :     :              +- * Project (58)
+                              :     :                 +- * Filter (57)
+                              :     :                    +- * ColumnarToRow (56)
+                              :     :                       +- Scan parquet default.web_sales (55)
+                              :     +- BroadcastExchange (67)
+                              :        +- * Filter (66)
+                              :           +- * ColumnarToRow (65)
+                              :              +- Scan parquet default.web_site (64)
+                              +- ReusedExchange (70)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_ext_sales_price:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 1]
+Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+Condition : isnotnull(ss_store_sk#1)
+
+(4) Project [codegen id : 1]
+Output [6]: [ss_store_sk#1 AS store_sk#6, ss_sold_date_sk#4 AS date_sk#7, ss_ext_sales_price#2 AS sales_price#8, ss_net_profit#3 AS profit#9, 0.00 AS return_amt#10, 0.00 AS net_loss#11]
+Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+
+(5) Scan parquet default.store_returns
+Output [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(sr_returned_date_sk#15), dynamicpruningexpression(sr_returned_date_sk#15 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(sr_store_sk)]
+ReadSchema: struct<sr_store_sk:int,sr_return_amt:decimal(7,2),sr_net_loss:decimal(7,2)>
+
+(6) ColumnarToRow [codegen id : 2]
+Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15]
+
+(7) Filter [codegen id : 2]
+Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15]
+Condition : isnotnull(sr_store_sk#12)
+
+(8) Project [codegen id : 2]
+Output [6]: [sr_store_sk#12 AS store_sk#16, sr_returned_date_sk#15 AS date_sk#17, 0.00 AS sales_price#18, 0.00 AS profit#19, sr_return_amt#13 AS return_amt#20, sr_net_loss#14 AS net_loss#21]
+Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15]
+
+(9) Union
+
+(10) Scan parquet default.store
+Output [2]: [s_store_sk#22, s_store_id#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string>
+
+(11) ColumnarToRow [codegen id : 3]
+Input [2]: [s_store_sk#22, s_store_id#23]
+
+(12) Filter [codegen id : 3]
+Input [2]: [s_store_sk#22, s_store_id#23]
+Condition : isnotnull(s_store_sk#22)
+
+(13) BroadcastExchange
+Input [2]: [s_store_sk#22, s_store_id#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#24]
+
+(14) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [store_sk#6]
+Right keys [1]: [s_store_sk#22]
+Join condition: None
+
+(15) Project [codegen id : 5]
+Output [6]: [date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#23]
+Input [8]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_sk#22, s_store_id#23]
+
+(16) Scan parquet default.date_dim
+Output [2]: [d_date_sk#25, d_date#26]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-06), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(17) ColumnarToRow [codegen id : 4]
+Input [2]: [d_date_sk#25, d_date#26]
+
+(18) Filter [codegen id : 4]
+Input [2]: [d_date_sk#25, d_date#26]
+Condition : (((isnotnull(d_date#26) AND (d_date#26 >= 2000-08-23)) AND (d_date#26 <= 2000-09-06)) AND isnotnull(d_date_sk#25))
+
+(19) Project [codegen id : 4]
+Output [1]: [d_date_sk#25]
+Input [2]: [d_date_sk#25, d_date#26]
+
+(20) BroadcastExchange
+Input [1]: [d_date_sk#25]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#27]
+
+(21) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [date_sk#7]
+Right keys [1]: [d_date_sk#25]
+Join condition: None
+
+(22) Project [codegen id : 5]
+Output [5]: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#23]
+Input [7]: [date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#23, d_date_sk#25]
+
+(23) HashAggregate [codegen id : 5]
+Input [5]: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#23]
+Keys [1]: [s_store_id#23]
+Functions [4]: [partial_sum(UnscaledValue(sales_price#8)), partial_sum(UnscaledValue(return_amt#10)), partial_sum(UnscaledValue(profit#9)), partial_sum(UnscaledValue(net_loss#11))]
+Aggregate Attributes [4]: [sum#28, sum#29, sum#30, sum#31]
+Results [5]: [s_store_id#23, sum#32, sum#33, sum#34, sum#35]
+
+(24) Exchange
+Input [5]: [s_store_id#23, sum#32, sum#33, sum#34, sum#35]
+Arguments: hashpartitioning(s_store_id#23, 5), ENSURE_REQUIREMENTS, [id=#36]
+
+(25) HashAggregate [codegen id : 6]
+Input [5]: [s_store_id#23, sum#32, sum#33, sum#34, sum#35]
+Keys [1]: [s_store_id#23]
+Functions [4]: [sum(UnscaledValue(sales_price#8)), sum(UnscaledValue(return_amt#10)), sum(UnscaledValue(profit#9)), sum(UnscaledValue(net_loss#11))]
+Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#8))#37, sum(UnscaledValue(return_amt#10))#38, sum(UnscaledValue(profit#9))#39, sum(UnscaledValue(net_loss#11))#40]
+Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#8))#37,17,2) AS sales#41, MakeDecimal(sum(UnscaledValue(return_amt#10))#38,17,2) AS RETURNS#42, CheckOverflow((promote_precision(cast(MakeDecimal(sum(UnscaledValue(profit#9))#39,17,2) as decimal(18,2))) - promote_precision(cast(MakeDecimal(sum(UnscaledValue(net_loss#11))#40,17,2) as decimal(18,2)))), DecimalType(18,2), true) AS profit#43, store channel AS channel#44, concat(store, s_store_id#23) AS id#45]
+
+(26) Scan parquet default.catalog_sales
+Output [4]: [cs_catalog_page_sk#46, cs_ext_sales_price#47, cs_net_profit#48, cs_sold_date_sk#49]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#49), dynamicpruningexpression(cs_sold_date_sk#49 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(cs_catalog_page_sk)]
+ReadSchema: struct<cs_catalog_page_sk:int,cs_ext_sales_price:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(27) ColumnarToRow [codegen id : 7]
+Input [4]: [cs_catalog_page_sk#46, cs_ext_sales_price#47, cs_net_profit#48, cs_sold_date_sk#49]
+
+(28) Filter [codegen id : 7]
+Input [4]: [cs_catalog_page_sk#46, cs_ext_sales_price#47, cs_net_profit#48, cs_sold_date_sk#49]
+Condition : isnotnull(cs_catalog_page_sk#46)
+
+(29) Project [codegen id : 7]
+Output [6]: [cs_catalog_page_sk#46 AS page_sk#50, cs_sold_date_sk#49 AS date_sk#51, cs_ext_sales_price#47 AS sales_price#52, cs_net_profit#48 AS profit#53, 0.00 AS return_amt#54, 0.00 AS net_loss#55]
+Input [4]: [cs_catalog_page_sk#46, cs_ext_sales_price#47, cs_net_profit#48, cs_sold_date_sk#49]
+
+(30) Scan parquet default.catalog_returns
+Output [4]: [cr_catalog_page_sk#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cr_returned_date_sk#59), dynamicpruningexpression(cr_returned_date_sk#59 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(cr_catalog_page_sk)]
+ReadSchema: struct<cr_catalog_page_sk:int,cr_return_amount:decimal(7,2),cr_net_loss:decimal(7,2)>
+
+(31) ColumnarToRow [codegen id : 8]
+Input [4]: [cr_catalog_page_sk#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+
+(32) Filter [codegen id : 8]
+Input [4]: [cr_catalog_page_sk#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+Condition : isnotnull(cr_catalog_page_sk#56)
+
+(33) Project [codegen id : 8]
+Output [6]: [cr_catalog_page_sk#56 AS page_sk#60, cr_returned_date_sk#59 AS date_sk#61, 0.00 AS sales_price#62, 0.00 AS profit#63, cr_return_amount#57 AS return_amt#64, cr_net_loss#58 AS net_loss#65]
+Input [4]: [cr_catalog_page_sk#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+
+(34) Union
+
+(35) Scan parquet default.catalog_page
+Output [2]: [cp_catalog_page_sk#66, cp_catalog_page_id#67]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_page]
+PushedFilters: [IsNotNull(cp_catalog_page_sk)]
+ReadSchema: struct<cp_catalog_page_sk:int,cp_catalog_page_id:string>
+
+(36) ColumnarToRow [codegen id : 9]
+Input [2]: [cp_catalog_page_sk#66, cp_catalog_page_id#67]
+
+(37) Filter [codegen id : 9]
+Input [2]: [cp_catalog_page_sk#66, cp_catalog_page_id#67]
+Condition : isnotnull(cp_catalog_page_sk#66)
+
+(38) BroadcastExchange
+Input [2]: [cp_catalog_page_sk#66, cp_catalog_page_id#67]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#68]
+
+(39) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [page_sk#50]
+Right keys [1]: [cp_catalog_page_sk#66]
+Join condition: None
+
+(40) Project [codegen id : 11]
+Output [6]: [date_sk#51, sales_price#52, profit#53, return_amt#54, net_loss#55, cp_catalog_page_id#67]
+Input [8]: [page_sk#50, date_sk#51, sales_price#52, profit#53, return_amt#54, net_loss#55, cp_catalog_page_sk#66, cp_catalog_page_id#67]
+
+(41) ReusedExchange [Reuses operator id: 20]
+Output [1]: [d_date_sk#69]
+
+(42) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [date_sk#51]
+Right keys [1]: [d_date_sk#69]
+Join condition: None
+
+(43) Project [codegen id : 11]
+Output [5]: [sales_price#52, profit#53, return_amt#54, net_loss#55, cp_catalog_page_id#67]
+Input [7]: [date_sk#51, sales_price#52, profit#53, return_amt#54, net_loss#55, cp_catalog_page_id#67, d_date_sk#69]
+
+(44) HashAggregate [codegen id : 11]
+Input [5]: [sales_price#52, profit#53, return_amt#54, net_loss#55, cp_catalog_page_id#67]
+Keys [1]: [cp_catalog_page_id#67]
+Functions [4]: [partial_sum(UnscaledValue(sales_price#52)), partial_sum(UnscaledValue(return_amt#54)), partial_sum(UnscaledValue(profit#53)), partial_sum(UnscaledValue(net_loss#55))]
+Aggregate Attributes [4]: [sum#70, sum#71, sum#72, sum#73]
+Results [5]: [cp_catalog_page_id#67, sum#74, sum#75, sum#76, sum#77]
+
+(45) Exchange
+Input [5]: [cp_catalog_page_id#67, sum#74, sum#75, sum#76, sum#77]
+Arguments: hashpartitioning(cp_catalog_page_id#67, 5), ENSURE_REQUIREMENTS, [id=#78]
+
+(46) HashAggregate [codegen id : 12]
+Input [5]: [cp_catalog_page_id#67, sum#74, sum#75, sum#76, sum#77]
+Keys [1]: [cp_catalog_page_id#67]
+Functions [4]: [sum(UnscaledValue(sales_price#52)), sum(UnscaledValue(return_amt#54)), sum(UnscaledValue(profit#53)), sum(UnscaledValue(net_loss#55))]
+Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#52))#79, sum(UnscaledValue(return_amt#54))#80, sum(UnscaledValue(profit#53))#81, sum(UnscaledValue(net_loss#55))#82]
+Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#52))#79,17,2) AS sales#83, MakeDecimal(sum(UnscaledValue(return_amt#54))#80,17,2) AS RETURNS#84, CheckOverflow((promote_precision(cast(MakeDecimal(sum(UnscaledValue(profit#53))#81,17,2) as decimal(18,2))) - promote_precision(cast(MakeDecimal(sum(UnscaledValue(net_loss#55))#82,17,2) as decimal(18,2)))), DecimalType(18,2), true) AS profit#85, catalog channel AS channel#86, concat(catalog_page, cp_catalog_page_id#67) AS id#87]
+
+(47) Scan parquet default.web_sales
+Output [4]: [ws_web_site_sk#88, ws_ext_sales_price#89, ws_net_profit#90, ws_sold_date_sk#91]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#91), dynamicpruningexpression(ws_sold_date_sk#91 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ws_web_site_sk)]
+ReadSchema: struct<ws_web_site_sk:int,ws_ext_sales_price:decimal(7,2),ws_net_profit:decimal(7,2)>
+
+(48) ColumnarToRow [codegen id : 13]
+Input [4]: [ws_web_site_sk#88, ws_ext_sales_price#89, ws_net_profit#90, ws_sold_date_sk#91]
+
+(49) Filter [codegen id : 13]
+Input [4]: [ws_web_site_sk#88, ws_ext_sales_price#89, ws_net_profit#90, ws_sold_date_sk#91]
+Condition : isnotnull(ws_web_site_sk#88)
+
+(50) Project [codegen id : 13]
+Output [6]: [ws_web_site_sk#88 AS wsr_web_site_sk#92, ws_sold_date_sk#91 AS date_sk#93, ws_ext_sales_price#89 AS sales_price#94, ws_net_profit#90 AS profit#95, 0.00 AS return_amt#96, 0.00 AS net_loss#97]
+Input [4]: [ws_web_site_sk#88, ws_ext_sales_price#89, ws_net_profit#90, ws_sold_date_sk#91]
+
+(51) Scan parquet default.web_returns
+Output [5]: [wr_item_sk#98, wr_order_number#99, wr_return_amt#100, wr_net_loss#101, wr_returned_date_sk#102]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(wr_returned_date_sk#102), dynamicpruningexpression(wr_returned_date_sk#102 IN dynamicpruning#5)]
+ReadSchema: struct<wr_item_sk:int,wr_order_number:int,wr_return_amt:decimal(7,2),wr_net_loss:decimal(7,2)>
+
+(52) ColumnarToRow [codegen id : 14]
+Input [5]: [wr_item_sk#98, wr_order_number#99, wr_return_amt#100, wr_net_loss#101, wr_returned_date_sk#102]
+
+(53) Exchange
+Input [5]: [wr_item_sk#98, wr_order_number#99, wr_return_amt#100, wr_net_loss#101, wr_returned_date_sk#102]
+Arguments: hashpartitioning(wr_item_sk#98, wr_order_number#99, 5), ENSURE_REQUIREMENTS, [id=#103]
+
+(54) Sort [codegen id : 15]
+Input [5]: [wr_item_sk#98, wr_order_number#99, wr_return_amt#100, wr_net_loss#101, wr_returned_date_sk#102]
+Arguments: [wr_item_sk#98 ASC NULLS FIRST, wr_order_number#99 ASC NULLS FIRST], false, 0
+
+(55) Scan parquet default.web_sales
+Output [4]: [ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106, ws_sold_date_sk#107]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_sales]
+PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_web_site_sk:int,ws_order_number:int>
+
+(56) ColumnarToRow [codegen id : 16]
+Input [4]: [ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106, ws_sold_date_sk#107]
+
+(57) Filter [codegen id : 16]
+Input [4]: [ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106, ws_sold_date_sk#107]
+Condition : ((isnotnull(ws_item_sk#104) AND isnotnull(ws_order_number#106)) AND isnotnull(ws_web_site_sk#105))
+
+(58) Project [codegen id : 16]
+Output [3]: [ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106]
+Input [4]: [ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106, ws_sold_date_sk#107]
+
+(59) Exchange
+Input [3]: [ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106]
+Arguments: hashpartitioning(ws_item_sk#104, ws_order_number#106, 5), ENSURE_REQUIREMENTS, [id=#108]
+
+(60) Sort [codegen id : 17]
+Input [3]: [ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106]
+Arguments: [ws_item_sk#104 ASC NULLS FIRST, ws_order_number#106 ASC NULLS FIRST], false, 0
+
+(61) SortMergeJoin [codegen id : 18]
+Left keys [2]: [wr_item_sk#98, wr_order_number#99]
+Right keys [2]: [ws_item_sk#104, ws_order_number#106]
+Join condition: None
+
+(62) Project [codegen id : 18]
+Output [6]: [ws_web_site_sk#105 AS wsr_web_site_sk#109, wr_returned_date_sk#102 AS date_sk#110, 0.00 AS sales_price#111, 0.00 AS profit#112, wr_return_amt#100 AS return_amt#113, wr_net_loss#101 AS net_loss#114]
+Input [8]: [wr_item_sk#98, wr_order_number#99, wr_return_amt#100, wr_net_loss#101, wr_returned_date_sk#102, ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106]
+
+(63) Union
+
+(64) Scan parquet default.web_site
+Output [2]: [web_site_sk#115, web_site_id#116]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_site]
+PushedFilters: [IsNotNull(web_site_sk)]
+ReadSchema: struct<web_site_sk:int,web_site_id:string>
+
+(65) ColumnarToRow [codegen id : 19]
+Input [2]: [web_site_sk#115, web_site_id#116]
+
+(66) Filter [codegen id : 19]
+Input [2]: [web_site_sk#115, web_site_id#116]
+Condition : isnotnull(web_site_sk#115)
+
+(67) BroadcastExchange
+Input [2]: [web_site_sk#115, web_site_id#116]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#117]
+
+(68) BroadcastHashJoin [codegen id : 21]
+Left keys [1]: [wsr_web_site_sk#92]
+Right keys [1]: [web_site_sk#115]
+Join condition: None
+
+(69) Project [codegen id : 21]
+Output [6]: [date_sk#93, sales_price#94, profit#95, return_amt#96, net_loss#97, web_site_id#116]
+Input [8]: [wsr_web_site_sk#92, date_sk#93, sales_price#94, profit#95, return_amt#96, net_loss#97, web_site_sk#115, web_site_id#116]
+
+(70) ReusedExchange [Reuses operator id: 20]
+Output [1]: [d_date_sk#118]
+
+(71) BroadcastHashJoin [codegen id : 21]
+Left keys [1]: [date_sk#93]
+Right keys [1]: [d_date_sk#118]
+Join condition: None
+
+(72) Project [codegen id : 21]
+Output [5]: [sales_price#94, profit#95, return_amt#96, net_loss#97, web_site_id#116]
+Input [7]: [date_sk#93, sales_price#94, profit#95, return_amt#96, net_loss#97, web_site_id#116, d_date_sk#118]
+
+(73) HashAggregate [codegen id : 21]
+Input [5]: [sales_price#94, profit#95, return_amt#96, net_loss#97, web_site_id#116]
+Keys [1]: [web_site_id#116]
+Functions [4]: [partial_sum(UnscaledValue(sales_price#94)), partial_sum(UnscaledValue(return_amt#96)), partial_sum(UnscaledValue(profit#95)), partial_sum(UnscaledValue(net_loss#97))]
+Aggregate Attributes [4]: [sum#119, sum#120, sum#121, sum#122]
+Results [5]: [web_site_id#116, sum#123, sum#124, sum#125, sum#126]
+
+(74) Exchange
+Input [5]: [web_site_id#116, sum#123, sum#124, sum#125, sum#126]
+Arguments: hashpartitioning(web_site_id#116, 5), ENSURE_REQUIREMENTS, [id=#127]
+
+(75) HashAggregate [codegen id : 22]
+Input [5]: [web_site_id#116, sum#123, sum#124, sum#125, sum#126]
+Keys [1]: [web_site_id#116]
+Functions [4]: [sum(UnscaledValue(sales_price#94)), sum(UnscaledValue(return_amt#96)), sum(UnscaledValue(profit#95)), sum(UnscaledValue(net_loss#97))]
+Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#94))#128, sum(UnscaledValue(return_amt#96))#129, sum(UnscaledValue(profit#95))#130, sum(UnscaledValue(net_loss#97))#131]
+Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#94))#128,17,2) AS sales#132, MakeDecimal(sum(UnscaledValue(return_amt#96))#129,17,2) AS RETURNS#133, CheckOverflow((promote_precision(cast(MakeDecimal(sum(UnscaledValue(profit#95))#130,17,2) as decimal(18,2))) - promote_precision(cast(MakeDecimal(sum(UnscaledValue(net_loss#97))#131,17,2) as decimal(18,2)))), DecimalType(18,2), true) AS profit#134, web channel AS channel#135, concat(web_site, web_site_id#116) AS id#136]
+
+(76) Union
+
+(77) Expand [codegen id : 23]
+Input [5]: [sales#41, RETURNS#42, profit#43, channel#44, id#45]
+Arguments: [ArrayBuffer(sales#41, returns#42, profit#43, channel#44, id#45, 0), ArrayBuffer(sales#41, returns#42, profit#43, channel#44, null, 1), ArrayBuffer(sales#41, returns#42, profit#43, null, null, 3)], [sales#41, returns#42, profit#43, channel#137, id#138, spark_grouping_id#139]
+
+(78) HashAggregate [codegen id : 23]
+Input [6]: [sales#41, returns#42, profit#43, channel#137, id#138, spark_grouping_id#139]
+Keys [3]: [channel#137, id#138, spark_grouping_id#139]
+Functions [3]: [partial_sum(sales#41), partial_sum(returns#42), partial_sum(profit#43)]
+Aggregate Attributes [6]: [sum#140, isEmpty#141, sum#142, isEmpty#143, sum#144, isEmpty#145]
+Results [9]: [channel#137, id#138, spark_grouping_id#139, sum#146, isEmpty#147, sum#148, isEmpty#149, sum#150, isEmpty#151]
+
+(79) Exchange
+Input [9]: [channel#137, id#138, spark_grouping_id#139, sum#146, isEmpty#147, sum#148, isEmpty#149, sum#150, isEmpty#151]
+Arguments: hashpartitioning(channel#137, id#138, spark_grouping_id#139, 5), ENSURE_REQUIREMENTS, [id=#152]
+
+(80) HashAggregate [codegen id : 24]
+Input [9]: [channel#137, id#138, spark_grouping_id#139, sum#146, isEmpty#147, sum#148, isEmpty#149, sum#150, isEmpty#151]
+Keys [3]: [channel#137, id#138, spark_grouping_id#139]
+Functions [3]: [sum(sales#41), sum(returns#42), sum(profit#43)]
+Aggregate Attributes [3]: [sum(sales#41)#153, sum(returns#42)#154, sum(profit#43)#155]
+Results [5]: [channel#137, id#138, sum(sales#41)#153 AS sales#156, sum(returns#42)#154 AS returns#157, sum(profit#43)#155 AS profit#158]
+
+(81) TakeOrderedAndProject
+Input [5]: [channel#137, id#138, sales#156, returns#157, profit#158]
+Arguments: 100, [channel#137 ASC NULLS FIRST, id#138 ASC NULLS FIRST], [channel#137, id#138, sales#156, returns#157, profit#158]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (82)
+
+
+(82) ReusedExchange [Reuses operator id: 20]
+Output [1]: [d_date_sk#25]
+
+Subquery:2 Hosting operator id = 5 Hosting Expression = sr_returned_date_sk#15 IN dynamicpruning#5
+
+Subquery:3 Hosting operator id = 26 Hosting Expression = cs_sold_date_sk#49 IN dynamicpruning#5
+
+Subquery:4 Hosting operator id = 30 Hosting Expression = cr_returned_date_sk#59 IN dynamicpruning#5
+
+Subquery:5 Hosting operator id = 47 Hosting Expression = ws_sold_date_sk#91 IN dynamicpruning#5
+
+Subquery:6 Hosting operator id = 51 Hosting Expression = wr_returned_date_sk#102 IN dynamicpruning#5
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.sf100/simplified.txt
new file mode 100644
index 0000000000000..2b62a0f686b43
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.sf100/simplified.txt
@@ -0,0 +1,139 @@
+TakeOrderedAndProject [channel,id,sales,returns,profit]
+  WholeStageCodegen (24)
+    HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+      InputAdapter
+        Exchange [channel,id,spark_grouping_id] #1
+          WholeStageCodegen (23)
+            HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+              Expand [sales,returns,profit,channel,id]
+                InputAdapter
+                  Union
+                    WholeStageCodegen (6)
+                      HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,RETURNS,profit,channel,id,sum,sum,sum,sum]
+                        InputAdapter
+                          Exchange [s_store_id] #2
+                            WholeStageCodegen (5)
+                              HashAggregate [s_store_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum]
+                                Project [sales_price,profit,return_amt,net_loss,s_store_id]
+                                  BroadcastHashJoin [date_sk,d_date_sk]
+                                    Project [date_sk,sales_price,profit,return_amt,net_loss,s_store_id]
+                                      BroadcastHashJoin [store_sk,s_store_sk]
+                                        InputAdapter
+                                          Union
+                                            WholeStageCodegen (1)
+                                              Project [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit]
+                                                Filter [ss_store_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk]
+                                                        SubqueryBroadcast [d_date_sk] #1
+                                                          ReusedExchange [d_date_sk] #3
+                                            WholeStageCodegen (2)
+                                              Project [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss]
+                                                Filter [sr_store_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk]
+                                                        ReusedSubquery [d_date_sk] #1
+                                        InputAdapter
+                                          BroadcastExchange #4
+                                            WholeStageCodegen (3)
+                                              Filter [s_store_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.store [s_store_sk,s_store_id]
+                                    InputAdapter
+                                      BroadcastExchange #3
+                                        WholeStageCodegen (4)
+                                          Project [d_date_sk]
+                                            Filter [d_date,d_date_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.date_dim [d_date_sk,d_date]
+                    WholeStageCodegen (12)
+                      HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,RETURNS,profit,channel,id,sum,sum,sum,sum]
+                        InputAdapter
+                          Exchange [cp_catalog_page_id] #5
+                            WholeStageCodegen (11)
+                              HashAggregate [cp_catalog_page_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum]
+                                Project [sales_price,profit,return_amt,net_loss,cp_catalog_page_id]
+                                  BroadcastHashJoin [date_sk,d_date_sk]
+                                    Project [date_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_id]
+                                      BroadcastHashJoin [page_sk,cp_catalog_page_sk]
+                                        InputAdapter
+                                          Union
+                                            WholeStageCodegen (7)
+                                              Project [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit]
+                                                Filter [cs_catalog_page_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk]
+                                                        ReusedSubquery [d_date_sk] #1
+                                            WholeStageCodegen (8)
+                                              Project [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss]
+                                                Filter [cr_catalog_page_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk]
+                                                        ReusedSubquery [d_date_sk] #1
+                                        InputAdapter
+                                          BroadcastExchange #6
+                                            WholeStageCodegen (9)
+                                              Filter [cp_catalog_page_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id]
+                                    InputAdapter
+                                      ReusedExchange [d_date_sk] #3
+                    WholeStageCodegen (22)
+                      HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,RETURNS,profit,channel,id,sum,sum,sum,sum]
+                        InputAdapter
+                          Exchange [web_site_id] #7
+                            WholeStageCodegen (21)
+                              HashAggregate [web_site_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum]
+                                Project [sales_price,profit,return_amt,net_loss,web_site_id]
+                                  BroadcastHashJoin [date_sk,d_date_sk]
+                                    Project [date_sk,sales_price,profit,return_amt,net_loss,web_site_id]
+                                      BroadcastHashJoin [wsr_web_site_sk,web_site_sk]
+                                        InputAdapter
+                                          Union
+                                            WholeStageCodegen (13)
+                                              Project [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit]
+                                                Filter [ws_web_site_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk]
+                                                        ReusedSubquery [d_date_sk] #1
+                                            WholeStageCodegen (18)
+                                              Project [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss]
+                                                SortMergeJoin [wr_item_sk,wr_order_number,ws_item_sk,ws_order_number]
+                                                  InputAdapter
+                                                    WholeStageCodegen (15)
+                                                      Sort [wr_item_sk,wr_order_number]
+                                                        InputAdapter
+                                                          Exchange [wr_item_sk,wr_order_number] #8
+                                                            WholeStageCodegen (14)
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk]
+                                                                    ReusedSubquery [d_date_sk] #1
+                                                  InputAdapter
+                                                    WholeStageCodegen (17)
+                                                      Sort [ws_item_sk,ws_order_number]
+                                                        InputAdapter
+                                                          Exchange [ws_item_sk,ws_order_number] #9
+                                                            WholeStageCodegen (16)
+                                                              Project [ws_item_sk,ws_web_site_sk,ws_order_number]
+                                                                Filter [ws_item_sk,ws_order_number,ws_web_site_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk]
+                                        InputAdapter
+                                          BroadcastExchange #10
+                                            WholeStageCodegen (19)
+                                              Filter [web_site_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.web_site [web_site_sk,web_site_id]
+                                    InputAdapter
+                                      ReusedExchange [d_date_sk] #3
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/explain.txt
new file mode 100644
index 0000000000000..7c0804a51c860
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/explain.txt
@@ -0,0 +1,461 @@
+== Physical Plan ==
+TakeOrderedAndProject (78)
++- * HashAggregate (77)
+   +- Exchange (76)
+      +- * HashAggregate (75)
+         +- * Expand (74)
+            +- Union (73)
+               :- * HashAggregate (25)
+               :  +- Exchange (24)
+               :     +- * HashAggregate (23)
+               :        +- * Project (22)
+               :           +- * BroadcastHashJoin Inner BuildRight (21)
+               :              :- * Project (16)
+               :              :  +- * BroadcastHashJoin Inner BuildRight (15)
+               :              :     :- Union (9)
+               :              :     :  :- * Project (4)
+               :              :     :  :  +- * Filter (3)
+               :              :     :  :     +- * ColumnarToRow (2)
+               :              :     :  :        +- Scan parquet default.store_sales (1)
+               :              :     :  +- * Project (8)
+               :              :     :     +- * Filter (7)
+               :              :     :        +- * ColumnarToRow (6)
+               :              :     :           +- Scan parquet default.store_returns (5)
+               :              :     +- BroadcastExchange (14)
+               :              :        +- * Project (13)
+               :              :           +- * Filter (12)
+               :              :              +- * ColumnarToRow (11)
+               :              :                 +- Scan parquet default.date_dim (10)
+               :              +- BroadcastExchange (20)
+               :                 +- * Filter (19)
+               :                    +- * ColumnarToRow (18)
+               :                       +- Scan parquet default.store (17)
+               :- * HashAggregate (46)
+               :  +- Exchange (45)
+               :     +- * HashAggregate (44)
+               :        +- * Project (43)
+               :           +- * BroadcastHashJoin Inner BuildRight (42)
+               :              :- * Project (37)
+               :              :  +- * BroadcastHashJoin Inner BuildRight (36)
+               :              :     :- Union (34)
+               :              :     :  :- * Project (29)
+               :              :     :  :  +- * Filter (28)
+               :              :     :  :     +- * ColumnarToRow (27)
+               :              :     :  :        +- Scan parquet default.catalog_sales (26)
+               :              :     :  +- * Project (33)
+               :              :     :     +- * Filter (32)
+               :              :     :        +- * ColumnarToRow (31)
+               :              :     :           +- Scan parquet default.catalog_returns (30)
+               :              :     +- ReusedExchange (35)
+               :              +- BroadcastExchange (41)
+               :                 +- * Filter (40)
+               :                    +- * ColumnarToRow (39)
+               :                       +- Scan parquet default.catalog_page (38)
+               +- * HashAggregate (72)
+                  +- Exchange (71)
+                     +- * HashAggregate (70)
+                        +- * Project (69)
+                           +- * BroadcastHashJoin Inner BuildRight (68)
+                              :- * Project (63)
+                              :  +- * BroadcastHashJoin Inner BuildRight (62)
+                              :     :- Union (60)
+                              :     :  :- * Project (50)
+                              :     :  :  +- * Filter (49)
+                              :     :  :     +- * ColumnarToRow (48)
+                              :     :  :        +- Scan parquet default.web_sales (47)
+                              :     :  +- * Project (59)
+                              :     :     +- * BroadcastHashJoin Inner BuildLeft (58)
+                              :     :        :- BroadcastExchange (53)
+                              :     :        :  +- * ColumnarToRow (52)
+                              :     :        :     +- Scan parquet default.web_returns (51)
+                              :     :        +- * Project (57)
+                              :     :           +- * Filter (56)
+                              :     :              +- * ColumnarToRow (55)
+                              :     :                 +- Scan parquet default.web_sales (54)
+                              :     +- ReusedExchange (61)
+                              +- BroadcastExchange (67)
+                                 +- * Filter (66)
+                                    +- * ColumnarToRow (65)
+                                       +- Scan parquet default.web_site (64)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_ext_sales_price:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 1]
+Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+Condition : isnotnull(ss_store_sk#1)
+
+(4) Project [codegen id : 1]
+Output [6]: [ss_store_sk#1 AS store_sk#6, ss_sold_date_sk#4 AS date_sk#7, ss_ext_sales_price#2 AS sales_price#8, ss_net_profit#3 AS profit#9, 0.00 AS return_amt#10, 0.00 AS net_loss#11]
+Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+
+(5) Scan parquet default.store_returns
+Output [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(sr_returned_date_sk#15), dynamicpruningexpression(sr_returned_date_sk#15 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(sr_store_sk)]
+ReadSchema: struct<sr_store_sk:int,sr_return_amt:decimal(7,2),sr_net_loss:decimal(7,2)>
+
+(6) ColumnarToRow [codegen id : 2]
+Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15]
+
+(7) Filter [codegen id : 2]
+Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15]
+Condition : isnotnull(sr_store_sk#12)
+
+(8) Project [codegen id : 2]
+Output [6]: [sr_store_sk#12 AS store_sk#16, sr_returned_date_sk#15 AS date_sk#17, 0.00 AS sales_price#18, 0.00 AS profit#19, sr_return_amt#13 AS return_amt#20, sr_net_loss#14 AS net_loss#21]
+Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15]
+
+(9) Union
+
+(10) Scan parquet default.date_dim
+Output [2]: [d_date_sk#22, d_date#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-06), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(11) ColumnarToRow [codegen id : 3]
+Input [2]: [d_date_sk#22, d_date#23]
+
+(12) Filter [codegen id : 3]
+Input [2]: [d_date_sk#22, d_date#23]
+Condition : (((isnotnull(d_date#23) AND (d_date#23 >= 2000-08-23)) AND (d_date#23 <= 2000-09-06)) AND isnotnull(d_date_sk#22))
+
+(13) Project [codegen id : 3]
+Output [1]: [d_date_sk#22]
+Input [2]: [d_date_sk#22, d_date#23]
+
+(14) BroadcastExchange
+Input [1]: [d_date_sk#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#24]
+
+(15) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [date_sk#7]
+Right keys [1]: [d_date_sk#22]
+Join condition: None
+
+(16) Project [codegen id : 5]
+Output [5]: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11]
+Input [7]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, d_date_sk#22]
+
+(17) Scan parquet default.store
+Output [2]: [s_store_sk#25, s_store_id#26]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string>
+
+(18) ColumnarToRow [codegen id : 4]
+Input [2]: [s_store_sk#25, s_store_id#26]
+
+(19) Filter [codegen id : 4]
+Input [2]: [s_store_sk#25, s_store_id#26]
+Condition : isnotnull(s_store_sk#25)
+
+(20) BroadcastExchange
+Input [2]: [s_store_sk#25, s_store_id#26]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#27]
+
+(21) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [store_sk#6]
+Right keys [1]: [s_store_sk#25]
+Join condition: None
+
+(22) Project [codegen id : 5]
+Output [5]: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#26]
+Input [7]: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_sk#25, s_store_id#26]
+
+(23) HashAggregate [codegen id : 5]
+Input [5]: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#26]
+Keys [1]: [s_store_id#26]
+Functions [4]: [partial_sum(UnscaledValue(sales_price#8)), partial_sum(UnscaledValue(return_amt#10)), partial_sum(UnscaledValue(profit#9)), partial_sum(UnscaledValue(net_loss#11))]
+Aggregate Attributes [4]: [sum#28, sum#29, sum#30, sum#31]
+Results [5]: [s_store_id#26, sum#32, sum#33, sum#34, sum#35]
+
+(24) Exchange
+Input [5]: [s_store_id#26, sum#32, sum#33, sum#34, sum#35]
+Arguments: hashpartitioning(s_store_id#26, 5), ENSURE_REQUIREMENTS, [id=#36]
+
+(25) HashAggregate [codegen id : 6]
+Input [5]: [s_store_id#26, sum#32, sum#33, sum#34, sum#35]
+Keys [1]: [s_store_id#26]
+Functions [4]: [sum(UnscaledValue(sales_price#8)), sum(UnscaledValue(return_amt#10)), sum(UnscaledValue(profit#9)), sum(UnscaledValue(net_loss#11))]
+Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#8))#37, sum(UnscaledValue(return_amt#10))#38, sum(UnscaledValue(profit#9))#39, sum(UnscaledValue(net_loss#11))#40]
+Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#8))#37,17,2) AS sales#41, MakeDecimal(sum(UnscaledValue(return_amt#10))#38,17,2) AS RETURNS#42, CheckOverflow((promote_precision(cast(MakeDecimal(sum(UnscaledValue(profit#9))#39,17,2) as decimal(18,2))) - promote_precision(cast(MakeDecimal(sum(UnscaledValue(net_loss#11))#40,17,2) as decimal(18,2)))), DecimalType(18,2), true) AS profit#43, store channel AS channel#44, concat(store, s_store_id#26) AS id#45]
+
+(26) Scan parquet default.catalog_sales
+Output [4]: [cs_catalog_page_sk#46, cs_ext_sales_price#47, cs_net_profit#48, cs_sold_date_sk#49]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#49), dynamicpruningexpression(cs_sold_date_sk#49 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(cs_catalog_page_sk)]
+ReadSchema: struct<cs_catalog_page_sk:int,cs_ext_sales_price:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(27) ColumnarToRow [codegen id : 7]
+Input [4]: [cs_catalog_page_sk#46, cs_ext_sales_price#47, cs_net_profit#48, cs_sold_date_sk#49]
+
+(28) Filter [codegen id : 7]
+Input [4]: [cs_catalog_page_sk#46, cs_ext_sales_price#47, cs_net_profit#48, cs_sold_date_sk#49]
+Condition : isnotnull(cs_catalog_page_sk#46)
+
+(29) Project [codegen id : 7]
+Output [6]: [cs_catalog_page_sk#46 AS page_sk#50, cs_sold_date_sk#49 AS date_sk#51, cs_ext_sales_price#47 AS sales_price#52, cs_net_profit#48 AS profit#53, 0.00 AS return_amt#54, 0.00 AS net_loss#55]
+Input [4]: [cs_catalog_page_sk#46, cs_ext_sales_price#47, cs_net_profit#48, cs_sold_date_sk#49]
+
+(30) Scan parquet default.catalog_returns
+Output [4]: [cr_catalog_page_sk#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cr_returned_date_sk#59), dynamicpruningexpression(cr_returned_date_sk#59 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(cr_catalog_page_sk)]
+ReadSchema: struct<cr_catalog_page_sk:int,cr_return_amount:decimal(7,2),cr_net_loss:decimal(7,2)>
+
+(31) ColumnarToRow [codegen id : 8]
+Input [4]: [cr_catalog_page_sk#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+
+(32) Filter [codegen id : 8]
+Input [4]: [cr_catalog_page_sk#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+Condition : isnotnull(cr_catalog_page_sk#56)
+
+(33) Project [codegen id : 8]
+Output [6]: [cr_catalog_page_sk#56 AS page_sk#60, cr_returned_date_sk#59 AS date_sk#61, 0.00 AS sales_price#62, 0.00 AS profit#63, cr_return_amount#57 AS return_amt#64, cr_net_loss#58 AS net_loss#65]
+Input [4]: [cr_catalog_page_sk#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+
+(34) Union
+
+(35) ReusedExchange [Reuses operator id: 14]
+Output [1]: [d_date_sk#66]
+
+(36) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [date_sk#51]
+Right keys [1]: [d_date_sk#66]
+Join condition: None
+
+(37) Project [codegen id : 11]
+Output [5]: [page_sk#50, sales_price#52, profit#53, return_amt#54, net_loss#55]
+Input [7]: [page_sk#50, date_sk#51, sales_price#52, profit#53, return_amt#54, net_loss#55, d_date_sk#66]
+
+(38) Scan parquet default.catalog_page
+Output [2]: [cp_catalog_page_sk#67, cp_catalog_page_id#68]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_page]
+PushedFilters: [IsNotNull(cp_catalog_page_sk)]
+ReadSchema: struct<cp_catalog_page_sk:int,cp_catalog_page_id:string>
+
+(39) ColumnarToRow [codegen id : 10]
+Input [2]: [cp_catalog_page_sk#67, cp_catalog_page_id#68]
+
+(40) Filter [codegen id : 10]
+Input [2]: [cp_catalog_page_sk#67, cp_catalog_page_id#68]
+Condition : isnotnull(cp_catalog_page_sk#67)
+
+(41) BroadcastExchange
+Input [2]: [cp_catalog_page_sk#67, cp_catalog_page_id#68]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#69]
+
+(42) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [page_sk#50]
+Right keys [1]: [cp_catalog_page_sk#67]
+Join condition: None
+
+(43) Project [codegen id : 11]
+Output [5]: [sales_price#52, profit#53, return_amt#54, net_loss#55, cp_catalog_page_id#68]
+Input [7]: [page_sk#50, sales_price#52, profit#53, return_amt#54, net_loss#55, cp_catalog_page_sk#67, cp_catalog_page_id#68]
+
+(44) HashAggregate [codegen id : 11]
+Input [5]: [sales_price#52, profit#53, return_amt#54, net_loss#55, cp_catalog_page_id#68]
+Keys [1]: [cp_catalog_page_id#68]
+Functions [4]: [partial_sum(UnscaledValue(sales_price#52)), partial_sum(UnscaledValue(return_amt#54)), partial_sum(UnscaledValue(profit#53)), partial_sum(UnscaledValue(net_loss#55))]
+Aggregate Attributes [4]: [sum#70, sum#71, sum#72, sum#73]
+Results [5]: [cp_catalog_page_id#68, sum#74, sum#75, sum#76, sum#77]
+
+(45) Exchange
+Input [5]: [cp_catalog_page_id#68, sum#74, sum#75, sum#76, sum#77]
+Arguments: hashpartitioning(cp_catalog_page_id#68, 5), ENSURE_REQUIREMENTS, [id=#78]
+
+(46) HashAggregate [codegen id : 12]
+Input [5]: [cp_catalog_page_id#68, sum#74, sum#75, sum#76, sum#77]
+Keys [1]: [cp_catalog_page_id#68]
+Functions [4]: [sum(UnscaledValue(sales_price#52)), sum(UnscaledValue(return_amt#54)), sum(UnscaledValue(profit#53)), sum(UnscaledValue(net_loss#55))]
+Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#52))#79, sum(UnscaledValue(return_amt#54))#80, sum(UnscaledValue(profit#53))#81, sum(UnscaledValue(net_loss#55))#82]
+Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#52))#79,17,2) AS sales#83, MakeDecimal(sum(UnscaledValue(return_amt#54))#80,17,2) AS RETURNS#84, CheckOverflow((promote_precision(cast(MakeDecimal(sum(UnscaledValue(profit#53))#81,17,2) as decimal(18,2))) - promote_precision(cast(MakeDecimal(sum(UnscaledValue(net_loss#55))#82,17,2) as decimal(18,2)))), DecimalType(18,2), true) AS profit#85, catalog channel AS channel#86, concat(catalog_page, cp_catalog_page_id#68) AS id#87]
+
+(47) Scan parquet default.web_sales
+Output [4]: [ws_web_site_sk#88, ws_ext_sales_price#89, ws_net_profit#90, ws_sold_date_sk#91]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#91), dynamicpruningexpression(ws_sold_date_sk#91 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ws_web_site_sk)]
+ReadSchema: struct<ws_web_site_sk:int,ws_ext_sales_price:decimal(7,2),ws_net_profit:decimal(7,2)>
+
+(48) ColumnarToRow [codegen id : 13]
+Input [4]: [ws_web_site_sk#88, ws_ext_sales_price#89, ws_net_profit#90, ws_sold_date_sk#91]
+
+(49) Filter [codegen id : 13]
+Input [4]: [ws_web_site_sk#88, ws_ext_sales_price#89, ws_net_profit#90, ws_sold_date_sk#91]
+Condition : isnotnull(ws_web_site_sk#88)
+
+(50) Project [codegen id : 13]
+Output [6]: [ws_web_site_sk#88 AS wsr_web_site_sk#92, ws_sold_date_sk#91 AS date_sk#93, ws_ext_sales_price#89 AS sales_price#94, ws_net_profit#90 AS profit#95, 0.00 AS return_amt#96, 0.00 AS net_loss#97]
+Input [4]: [ws_web_site_sk#88, ws_ext_sales_price#89, ws_net_profit#90, ws_sold_date_sk#91]
+
+(51) Scan parquet default.web_returns
+Output [5]: [wr_item_sk#98, wr_order_number#99, wr_return_amt#100, wr_net_loss#101, wr_returned_date_sk#102]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(wr_returned_date_sk#102), dynamicpruningexpression(wr_returned_date_sk#102 IN dynamicpruning#5)]
+ReadSchema: struct<wr_item_sk:int,wr_order_number:int,wr_return_amt:decimal(7,2),wr_net_loss:decimal(7,2)>
+
+(52) ColumnarToRow [codegen id : 14]
+Input [5]: [wr_item_sk#98, wr_order_number#99, wr_return_amt#100, wr_net_loss#101, wr_returned_date_sk#102]
+
+(53) BroadcastExchange
+Input [5]: [wr_item_sk#98, wr_order_number#99, wr_return_amt#100, wr_net_loss#101, wr_returned_date_sk#102]
+Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, true] as bigint), 32) | (cast(input[1, int, true] as bigint) & 4294967295))),false), [id=#103]
+
+(54) Scan parquet default.web_sales
+Output [4]: [ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106, ws_sold_date_sk#107]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_sales]
+PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_web_site_sk:int,ws_order_number:int>
+
+(55) ColumnarToRow
+Input [4]: [ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106, ws_sold_date_sk#107]
+
+(56) Filter
+Input [4]: [ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106, ws_sold_date_sk#107]
+Condition : ((isnotnull(ws_item_sk#104) AND isnotnull(ws_order_number#106)) AND isnotnull(ws_web_site_sk#105))
+
+(57) Project
+Output [3]: [ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106]
+Input [4]: [ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106, ws_sold_date_sk#107]
+
+(58) BroadcastHashJoin [codegen id : 15]
+Left keys [2]: [wr_item_sk#98, wr_order_number#99]
+Right keys [2]: [ws_item_sk#104, ws_order_number#106]
+Join condition: None
+
+(59) Project [codegen id : 15]
+Output [6]: [ws_web_site_sk#105 AS wsr_web_site_sk#108, wr_returned_date_sk#102 AS date_sk#109, 0.00 AS sales_price#110, 0.00 AS profit#111, wr_return_amt#100 AS return_amt#112, wr_net_loss#101 AS net_loss#113]
+Input [8]: [wr_item_sk#98, wr_order_number#99, wr_return_amt#100, wr_net_loss#101, wr_returned_date_sk#102, ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106]
+
+(60) Union
+
+(61) ReusedExchange [Reuses operator id: 14]
+Output [1]: [d_date_sk#114]
+
+(62) BroadcastHashJoin [codegen id : 18]
+Left keys [1]: [date_sk#93]
+Right keys [1]: [d_date_sk#114]
+Join condition: None
+
+(63) Project [codegen id : 18]
+Output [5]: [wsr_web_site_sk#92, sales_price#94, profit#95, return_amt#96, net_loss#97]
+Input [7]: [wsr_web_site_sk#92, date_sk#93, sales_price#94, profit#95, return_amt#96, net_loss#97, d_date_sk#114]
+
+(64) Scan parquet default.web_site
+Output [2]: [web_site_sk#115, web_site_id#116]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_site]
+PushedFilters: [IsNotNull(web_site_sk)]
+ReadSchema: struct<web_site_sk:int,web_site_id:string>
+
+(65) ColumnarToRow [codegen id : 17]
+Input [2]: [web_site_sk#115, web_site_id#116]
+
+(66) Filter [codegen id : 17]
+Input [2]: [web_site_sk#115, web_site_id#116]
+Condition : isnotnull(web_site_sk#115)
+
+(67) BroadcastExchange
+Input [2]: [web_site_sk#115, web_site_id#116]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#117]
+
+(68) BroadcastHashJoin [codegen id : 18]
+Left keys [1]: [wsr_web_site_sk#92]
+Right keys [1]: [web_site_sk#115]
+Join condition: None
+
+(69) Project [codegen id : 18]
+Output [5]: [sales_price#94, profit#95, return_amt#96, net_loss#97, web_site_id#116]
+Input [7]: [wsr_web_site_sk#92, sales_price#94, profit#95, return_amt#96, net_loss#97, web_site_sk#115, web_site_id#116]
+
+(70) HashAggregate [codegen id : 18]
+Input [5]: [sales_price#94, profit#95, return_amt#96, net_loss#97, web_site_id#116]
+Keys [1]: [web_site_id#116]
+Functions [4]: [partial_sum(UnscaledValue(sales_price#94)), partial_sum(UnscaledValue(return_amt#96)), partial_sum(UnscaledValue(profit#95)), partial_sum(UnscaledValue(net_loss#97))]
+Aggregate Attributes [4]: [sum#118, sum#119, sum#120, sum#121]
+Results [5]: [web_site_id#116, sum#122, sum#123, sum#124, sum#125]
+
+(71) Exchange
+Input [5]: [web_site_id#116, sum#122, sum#123, sum#124, sum#125]
+Arguments: hashpartitioning(web_site_id#116, 5), ENSURE_REQUIREMENTS, [id=#126]
+
+(72) HashAggregate [codegen id : 19]
+Input [5]: [web_site_id#116, sum#122, sum#123, sum#124, sum#125]
+Keys [1]: [web_site_id#116]
+Functions [4]: [sum(UnscaledValue(sales_price#94)), sum(UnscaledValue(return_amt#96)), sum(UnscaledValue(profit#95)), sum(UnscaledValue(net_loss#97))]
+Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#94))#127, sum(UnscaledValue(return_amt#96))#128, sum(UnscaledValue(profit#95))#129, sum(UnscaledValue(net_loss#97))#130]
+Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#94))#127,17,2) AS sales#131, MakeDecimal(sum(UnscaledValue(return_amt#96))#128,17,2) AS RETURNS#132, CheckOverflow((promote_precision(cast(MakeDecimal(sum(UnscaledValue(profit#95))#129,17,2) as decimal(18,2))) - promote_precision(cast(MakeDecimal(sum(UnscaledValue(net_loss#97))#130,17,2) as decimal(18,2)))), DecimalType(18,2), true) AS profit#133, web channel AS channel#134, concat(web_site, web_site_id#116) AS id#135]
+
+(73) Union
+
+(74) Expand [codegen id : 20]
+Input [5]: [sales#41, RETURNS#42, profit#43, channel#44, id#45]
+Arguments: [ArrayBuffer(sales#41, returns#42, profit#43, channel#44, id#45, 0), ArrayBuffer(sales#41, returns#42, profit#43, channel#44, null, 1), ArrayBuffer(sales#41, returns#42, profit#43, null, null, 3)], [sales#41, returns#42, profit#43, channel#136, id#137, spark_grouping_id#138]
+
+(75) HashAggregate [codegen id : 20]
+Input [6]: [sales#41, returns#42, profit#43, channel#136, id#137, spark_grouping_id#138]
+Keys [3]: [channel#136, id#137, spark_grouping_id#138]
+Functions [3]: [partial_sum(sales#41), partial_sum(returns#42), partial_sum(profit#43)]
+Aggregate Attributes [6]: [sum#139, isEmpty#140, sum#141, isEmpty#142, sum#143, isEmpty#144]
+Results [9]: [channel#136, id#137, spark_grouping_id#138, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150]
+
+(76) Exchange
+Input [9]: [channel#136, id#137, spark_grouping_id#138, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150]
+Arguments: hashpartitioning(channel#136, id#137, spark_grouping_id#138, 5), ENSURE_REQUIREMENTS, [id=#151]
+
+(77) HashAggregate [codegen id : 21]
+Input [9]: [channel#136, id#137, spark_grouping_id#138, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150]
+Keys [3]: [channel#136, id#137, spark_grouping_id#138]
+Functions [3]: [sum(sales#41), sum(returns#42), sum(profit#43)]
+Aggregate Attributes [3]: [sum(sales#41)#152, sum(returns#42)#153, sum(profit#43)#154]
+Results [5]: [channel#136, id#137, sum(sales#41)#152 AS sales#155, sum(returns#42)#153 AS returns#156, sum(profit#43)#154 AS profit#157]
+
+(78) TakeOrderedAndProject
+Input [5]: [channel#136, id#137, sales#155, returns#156, profit#157]
+Arguments: 100, [channel#136 ASC NULLS FIRST, id#137 ASC NULLS FIRST], [channel#136, id#137, sales#155, returns#156, profit#157]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (79)
+
+
+(79) ReusedExchange [Reuses operator id: 14]
+Output [1]: [d_date_sk#22]
+
+Subquery:2 Hosting operator id = 5 Hosting Expression = sr_returned_date_sk#15 IN dynamicpruning#5
+
+Subquery:3 Hosting operator id = 26 Hosting Expression = cs_sold_date_sk#49 IN dynamicpruning#5
+
+Subquery:4 Hosting operator id = 30 Hosting Expression = cr_returned_date_sk#59 IN dynamicpruning#5
+
+Subquery:5 Hosting operator id = 47 Hosting Expression = ws_sold_date_sk#91 IN dynamicpruning#5
+
+Subquery:6 Hosting operator id = 51 Hosting Expression = wr_returned_date_sk#102 IN dynamicpruning#5
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/simplified.txt
new file mode 100644
index 0000000000000..99ad64a42ab28
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/simplified.txt
@@ -0,0 +1,130 @@
+TakeOrderedAndProject [channel,id,sales,returns,profit]
+  WholeStageCodegen (21)
+    HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+      InputAdapter
+        Exchange [channel,id,spark_grouping_id] #1
+          WholeStageCodegen (20)
+            HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+              Expand [sales,returns,profit,channel,id]
+                InputAdapter
+                  Union
+                    WholeStageCodegen (6)
+                      HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,RETURNS,profit,channel,id,sum,sum,sum,sum]
+                        InputAdapter
+                          Exchange [s_store_id] #2
+                            WholeStageCodegen (5)
+                              HashAggregate [s_store_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum]
+                                Project [sales_price,profit,return_amt,net_loss,s_store_id]
+                                  BroadcastHashJoin [store_sk,s_store_sk]
+                                    Project [store_sk,sales_price,profit,return_amt,net_loss]
+                                      BroadcastHashJoin [date_sk,d_date_sk]
+                                        InputAdapter
+                                          Union
+                                            WholeStageCodegen (1)
+                                              Project [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit]
+                                                Filter [ss_store_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk]
+                                                        SubqueryBroadcast [d_date_sk] #1
+                                                          ReusedExchange [d_date_sk] #3
+                                            WholeStageCodegen (2)
+                                              Project [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss]
+                                                Filter [sr_store_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk]
+                                                        ReusedSubquery [d_date_sk] #1
+                                        InputAdapter
+                                          BroadcastExchange #3
+                                            WholeStageCodegen (3)
+                                              Project [d_date_sk]
+                                                Filter [d_date,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_date]
+                                    InputAdapter
+                                      BroadcastExchange #4
+                                        WholeStageCodegen (4)
+                                          Filter [s_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store [s_store_sk,s_store_id]
+                    WholeStageCodegen (12)
+                      HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,RETURNS,profit,channel,id,sum,sum,sum,sum]
+                        InputAdapter
+                          Exchange [cp_catalog_page_id] #5
+                            WholeStageCodegen (11)
+                              HashAggregate [cp_catalog_page_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum]
+                                Project [sales_price,profit,return_amt,net_loss,cp_catalog_page_id]
+                                  BroadcastHashJoin [page_sk,cp_catalog_page_sk]
+                                    Project [page_sk,sales_price,profit,return_amt,net_loss]
+                                      BroadcastHashJoin [date_sk,d_date_sk]
+                                        InputAdapter
+                                          Union
+                                            WholeStageCodegen (7)
+                                              Project [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit]
+                                                Filter [cs_catalog_page_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk]
+                                                        ReusedSubquery [d_date_sk] #1
+                                            WholeStageCodegen (8)
+                                              Project [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss]
+                                                Filter [cr_catalog_page_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk]
+                                                        ReusedSubquery [d_date_sk] #1
+                                        InputAdapter
+                                          ReusedExchange [d_date_sk] #3
+                                    InputAdapter
+                                      BroadcastExchange #6
+                                        WholeStageCodegen (10)
+                                          Filter [cp_catalog_page_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id]
+                    WholeStageCodegen (19)
+                      HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,RETURNS,profit,channel,id,sum,sum,sum,sum]
+                        InputAdapter
+                          Exchange [web_site_id] #7
+                            WholeStageCodegen (18)
+                              HashAggregate [web_site_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum]
+                                Project [sales_price,profit,return_amt,net_loss,web_site_id]
+                                  BroadcastHashJoin [wsr_web_site_sk,web_site_sk]
+                                    Project [wsr_web_site_sk,sales_price,profit,return_amt,net_loss]
+                                      BroadcastHashJoin [date_sk,d_date_sk]
+                                        InputAdapter
+                                          Union
+                                            WholeStageCodegen (13)
+                                              Project [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit]
+                                                Filter [ws_web_site_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk]
+                                                        ReusedSubquery [d_date_sk] #1
+                                            WholeStageCodegen (15)
+                                              Project [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss]
+                                                BroadcastHashJoin [wr_item_sk,wr_order_number,ws_item_sk,ws_order_number]
+                                                  InputAdapter
+                                                    BroadcastExchange #8
+                                                      WholeStageCodegen (14)
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk]
+                                                              ReusedSubquery [d_date_sk] #1
+                                                  Project [ws_item_sk,ws_web_site_sk,ws_order_number]
+                                                    Filter [ws_item_sk,ws_order_number,ws_web_site_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk]
+                                        InputAdapter
+                                          ReusedExchange [d_date_sk] #3
+                                    InputAdapter
+                                      BroadcastExchange #9
+                                        WholeStageCodegen (17)
+                                          Filter [web_site_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.web_site [web_site_sk,web_site_id]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.sf100/explain.txt
new file mode 100644
index 0000000000000..ccb95a316cb28
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.sf100/explain.txt
@@ -0,0 +1,210 @@
+== Physical Plan ==
+TakeOrderedAndProject (35)
++- * HashAggregate (34)
+   +- Exchange (33)
+      +- * HashAggregate (32)
+         +- * Project (31)
+            +- * BroadcastHashJoin Inner BuildRight (30)
+               :- * Project (25)
+               :  +- * BroadcastHashJoin Inner BuildRight (24)
+               :     :- * Project (19)
+               :     :  +- * SortMergeJoin Inner (18)
+               :     :     :- * Sort (12)
+               :     :     :  +- Exchange (11)
+               :     :     :     +- * Project (10)
+               :     :     :        +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :     :           :- * Filter (3)
+               :     :     :           :  +- * ColumnarToRow (2)
+               :     :     :           :     +- Scan parquet default.store_returns (1)
+               :     :     :           +- BroadcastExchange (8)
+               :     :     :              +- * Project (7)
+               :     :     :                 +- * Filter (6)
+               :     :     :                    +- * ColumnarToRow (5)
+               :     :     :                       +- Scan parquet default.date_dim (4)
+               :     :     +- * Sort (17)
+               :     :        +- Exchange (16)
+               :     :           +- * Filter (15)
+               :     :              +- * ColumnarToRow (14)
+               :     :                 +- Scan parquet default.store_sales (13)
+               :     +- BroadcastExchange (23)
+               :        +- * Filter (22)
+               :           +- * ColumnarToRow (21)
+               :              +- Scan parquet default.date_dim (20)
+               +- BroadcastExchange (29)
+                  +- * Filter (28)
+                     +- * ColumnarToRow (27)
+                        +- Scan parquet default.store (26)
+
+
+(1) Scan parquet default.store_returns
+Output [4]: [sr_item_sk#1, sr_customer_sk#2, sr_ticket_number#3, sr_returned_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(sr_returned_date_sk#4), dynamicpruningexpression(sr_returned_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk), IsNotNull(sr_customer_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_customer_sk:int,sr_ticket_number:int>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [4]: [sr_item_sk#1, sr_customer_sk#2, sr_ticket_number#3, sr_returned_date_sk#4]
+
+(3) Filter [codegen id : 2]
+Input [4]: [sr_item_sk#1, sr_customer_sk#2, sr_ticket_number#3, sr_returned_date_sk#4]
+Condition : ((isnotnull(sr_ticket_number#3) AND isnotnull(sr_item_sk#1)) AND isnotnull(sr_customer_sk#2))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,8), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2001)) AND (d_moy#8 = 8)) AND isnotnull(d_date_sk#6))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#6]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#9]
+
+(9) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [sr_returned_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 2]
+Output [4]: [sr_item_sk#1, sr_customer_sk#2, sr_ticket_number#3, sr_returned_date_sk#4]
+Input [5]: [sr_item_sk#1, sr_customer_sk#2, sr_ticket_number#3, sr_returned_date_sk#4, d_date_sk#6]
+
+(11) Exchange
+Input [4]: [sr_item_sk#1, sr_customer_sk#2, sr_ticket_number#3, sr_returned_date_sk#4]
+Arguments: hashpartitioning(sr_ticket_number#3, sr_item_sk#1, sr_customer_sk#2, 5), ENSURE_REQUIREMENTS, [id=#10]
+
+(12) Sort [codegen id : 3]
+Input [4]: [sr_item_sk#1, sr_customer_sk#2, sr_ticket_number#3, sr_returned_date_sk#4]
+Arguments: [sr_ticket_number#3 ASC NULLS FIRST, sr_item_sk#1 ASC NULLS FIRST, sr_customer_sk#2 ASC NULLS FIRST], false, 0
+
+(13) Scan parquet default.store_sales
+Output [5]: [ss_item_sk#11, ss_customer_sk#12, ss_store_sk#13, ss_ticket_number#14, ss_sold_date_sk#15]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#15)]
+PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ticket_number:int>
+
+(14) ColumnarToRow [codegen id : 4]
+Input [5]: [ss_item_sk#11, ss_customer_sk#12, ss_store_sk#13, ss_ticket_number#14, ss_sold_date_sk#15]
+
+(15) Filter [codegen id : 4]
+Input [5]: [ss_item_sk#11, ss_customer_sk#12, ss_store_sk#13, ss_ticket_number#14, ss_sold_date_sk#15]
+Condition : (((isnotnull(ss_ticket_number#14) AND isnotnull(ss_item_sk#11)) AND isnotnull(ss_customer_sk#12)) AND isnotnull(ss_store_sk#13))
+
+(16) Exchange
+Input [5]: [ss_item_sk#11, ss_customer_sk#12, ss_store_sk#13, ss_ticket_number#14, ss_sold_date_sk#15]
+Arguments: hashpartitioning(ss_ticket_number#14, ss_item_sk#11, ss_customer_sk#12, 5), ENSURE_REQUIREMENTS, [id=#16]
+
+(17) Sort [codegen id : 5]
+Input [5]: [ss_item_sk#11, ss_customer_sk#12, ss_store_sk#13, ss_ticket_number#14, ss_sold_date_sk#15]
+Arguments: [ss_ticket_number#14 ASC NULLS FIRST, ss_item_sk#11 ASC NULLS FIRST, ss_customer_sk#12 ASC NULLS FIRST], false, 0
+
+(18) SortMergeJoin [codegen id : 8]
+Left keys [3]: [sr_ticket_number#3, sr_item_sk#1, sr_customer_sk#2]
+Right keys [3]: [ss_ticket_number#14, ss_item_sk#11, ss_customer_sk#12]
+Join condition: None
+
+(19) Project [codegen id : 8]
+Output [3]: [sr_returned_date_sk#4, ss_store_sk#13, ss_sold_date_sk#15]
+Input [9]: [sr_item_sk#1, sr_customer_sk#2, sr_ticket_number#3, sr_returned_date_sk#4, ss_item_sk#11, ss_customer_sk#12, ss_store_sk#13, ss_ticket_number#14, ss_sold_date_sk#15]
+
+(20) Scan parquet default.date_dim
+Output [1]: [d_date_sk#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int>
+
+(21) ColumnarToRow [codegen id : 6]
+Input [1]: [d_date_sk#17]
+
+(22) Filter [codegen id : 6]
+Input [1]: [d_date_sk#17]
+Condition : isnotnull(d_date_sk#17)
+
+(23) BroadcastExchange
+Input [1]: [d_date_sk#17]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#18]
+
+(24) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_sold_date_sk#15]
+Right keys [1]: [d_date_sk#17]
+Join condition: None
+
+(25) Project [codegen id : 8]
+Output [3]: [sr_returned_date_sk#4, ss_store_sk#13, ss_sold_date_sk#15]
+Input [4]: [sr_returned_date_sk#4, ss_store_sk#13, ss_sold_date_sk#15, d_date_sk#17]
+
+(26) Scan parquet default.store
+Output [11]: [s_store_sk#19, s_store_name#20, s_company_id#21, s_street_number#22, s_street_name#23, s_street_type#24, s_suite_number#25, s_city#26, s_county#27, s_state#28, s_zip#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_company_id:int,s_street_number:string,s_street_name:string,s_street_type:string,s_suite_number:string,s_city:string,s_county:string,s_state:string,s_zip:string>
+
+(27) ColumnarToRow [codegen id : 7]
+Input [11]: [s_store_sk#19, s_store_name#20, s_company_id#21, s_street_number#22, s_street_name#23, s_street_type#24, s_suite_number#25, s_city#26, s_county#27, s_state#28, s_zip#29]
+
+(28) Filter [codegen id : 7]
+Input [11]: [s_store_sk#19, s_store_name#20, s_company_id#21, s_street_number#22, s_street_name#23, s_street_type#24, s_suite_number#25, s_city#26, s_county#27, s_state#28, s_zip#29]
+Condition : isnotnull(s_store_sk#19)
+
+(29) BroadcastExchange
+Input [11]: [s_store_sk#19, s_store_name#20, s_company_id#21, s_street_number#22, s_street_name#23, s_street_type#24, s_suite_number#25, s_city#26, s_county#27, s_state#28, s_zip#29]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#30]
+
+(30) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_store_sk#13]
+Right keys [1]: [s_store_sk#19]
+Join condition: None
+
+(31) Project [codegen id : 8]
+Output [12]: [ss_sold_date_sk#15, sr_returned_date_sk#4, s_store_name#20, s_company_id#21, s_street_number#22, s_street_name#23, s_street_type#24, s_suite_number#25, s_city#26, s_county#27, s_state#28, s_zip#29]
+Input [14]: [sr_returned_date_sk#4, ss_store_sk#13, ss_sold_date_sk#15, s_store_sk#19, s_store_name#20, s_company_id#21, s_street_number#22, s_street_name#23, s_street_type#24, s_suite_number#25, s_city#26, s_county#27, s_state#28, s_zip#29]
+
+(32) HashAggregate [codegen id : 8]
+Input [12]: [ss_sold_date_sk#15, sr_returned_date_sk#4, s_store_name#20, s_company_id#21, s_street_number#22, s_street_name#23, s_street_type#24, s_suite_number#25, s_city#26, s_county#27, s_state#28, s_zip#29]
+Keys [10]: [s_store_name#20, s_company_id#21, s_street_number#22, s_street_name#23, s_street_type#24, s_suite_number#25, s_city#26, s_county#27, s_state#28, s_zip#29]
+Functions [5]: [partial_sum(CASE WHEN ((sr_returned_date_sk#4 - ss_sold_date_sk#15) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#4 - ss_sold_date_sk#15) > 30) AND ((sr_returned_date_sk#4 - ss_sold_date_sk#15) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#4 - ss_sold_date_sk#15) > 60) AND ((sr_returned_date_sk#4 - ss_sold_date_sk#15) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#4 - ss_sold_date_sk#15) > 90) AND ((sr_returned_date_sk#4 - ss_sold_date_sk#15) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((sr_returned_date_sk#4 - ss_sold_date_sk#15) > 120) THEN 1 ELSE 0 END)]
+Aggregate Attributes [5]: [sum#31, sum#32, sum#33, sum#34, sum#35]
+Results [15]: [s_store_name#20, s_company_id#21, s_street_number#22, s_street_name#23, s_street_type#24, s_suite_number#25, s_city#26, s_county#27, s_state#28, s_zip#29, sum#36, sum#37, sum#38, sum#39, sum#40]
+
+(33) Exchange
+Input [15]: [s_store_name#20, s_company_id#21, s_street_number#22, s_street_name#23, s_street_type#24, s_suite_number#25, s_city#26, s_county#27, s_state#28, s_zip#29, sum#36, sum#37, sum#38, sum#39, sum#40]
+Arguments: hashpartitioning(s_store_name#20, s_company_id#21, s_street_number#22, s_street_name#23, s_street_type#24, s_suite_number#25, s_city#26, s_county#27, s_state#28, s_zip#29, 5), ENSURE_REQUIREMENTS, [id=#41]
+
+(34) HashAggregate [codegen id : 9]
+Input [15]: [s_store_name#20, s_company_id#21, s_street_number#22, s_street_name#23, s_street_type#24, s_suite_number#25, s_city#26, s_county#27, s_state#28, s_zip#29, sum#36, sum#37, sum#38, sum#39, sum#40]
+Keys [10]: [s_store_name#20, s_company_id#21, s_street_number#22, s_street_name#23, s_street_type#24, s_suite_number#25, s_city#26, s_county#27, s_state#28, s_zip#29]
+Functions [5]: [sum(CASE WHEN ((sr_returned_date_sk#4 - ss_sold_date_sk#15) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#4 - ss_sold_date_sk#15) > 30) AND ((sr_returned_date_sk#4 - ss_sold_date_sk#15) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#4 - ss_sold_date_sk#15) > 60) AND ((sr_returned_date_sk#4 - ss_sold_date_sk#15) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#4 - ss_sold_date_sk#15) > 90) AND ((sr_returned_date_sk#4 - ss_sold_date_sk#15) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((sr_returned_date_sk#4 - ss_sold_date_sk#15) > 120) THEN 1 ELSE 0 END)]
+Aggregate Attributes [5]: [sum(CASE WHEN ((sr_returned_date_sk#4 - ss_sold_date_sk#15) <= 30) THEN 1 ELSE 0 END)#42, sum(CASE WHEN (((sr_returned_date_sk#4 - ss_sold_date_sk#15) > 30) AND ((sr_returned_date_sk#4 - ss_sold_date_sk#15) <= 60)) THEN 1 ELSE 0 END)#43, sum(CASE WHEN (((sr_returned_date_sk#4 - ss_sold_date_sk#15) > 60) AND ((sr_returned_date_sk#4 - ss_sold_date_sk#15) <= 90)) THEN 1 ELSE 0 END)#44, sum(CASE WHEN (((sr_returned_date_sk#4 - ss_sold_date_sk#15) > 90) AND ((sr_returned_date_sk#4 - ss_sold_date_sk#15) <= 120)) THEN 1 ELSE 0 END)#45, sum(CASE WHEN ((sr_returned_date_sk#4 - ss_sold_date_sk#15) > 120) THEN 1 ELSE 0 END)#46]
+Results [15]: [s_store_name#20, s_company_id#21, s_street_number#22, s_street_name#23, s_street_type#24, s_suite_number#25, s_city#26, s_county#27, s_state#28, s_zip#29, sum(CASE WHEN ((sr_returned_date_sk#4 - ss_sold_date_sk#15) <= 30) THEN 1 ELSE 0 END)#42 AS 30 days #47, sum(CASE WHEN (((sr_returned_date_sk#4 - ss_sold_date_sk#15) > 30) AND ((sr_returned_date_sk#4 - ss_sold_date_sk#15) <= 60)) THEN 1 ELSE 0 END)#43 AS 31 - 60 days #48, sum(CASE WHEN (((sr_returned_date_sk#4 - ss_sold_date_sk#15) > 60) AND ((sr_returned_date_sk#4 - ss_sold_date_sk#15) <= 90)) THEN 1 ELSE 0 END)#44 AS 61 - 90 days #49, sum(CASE WHEN (((sr_returned_date_sk#4 - ss_sold_date_sk#15) > 90) AND ((sr_returned_date_sk#4 - ss_sold_date_sk#15) <= 120)) THEN 1 ELSE 0 END)#45 AS 91 - 120 days #50, sum(CASE WHEN ((sr_returned_date_sk#4 - ss_sold_date_sk#15) > 120) THEN 1 ELSE 0 END)#46 AS >120 days #51]
+
+(35) TakeOrderedAndProject
+Input [15]: [s_store_name#20, s_company_id#21, s_street_number#22, s_street_name#23, s_street_type#24, s_suite_number#25, s_city#26, s_county#27, s_state#28, s_zip#29, 30 days #47, 31 - 60 days #48, 61 - 90 days #49, 91 - 120 days #50, >120 days #51]
+Arguments: 100, [s_store_name#20 ASC NULLS FIRST, s_company_id#21 ASC NULLS FIRST, s_street_number#22 ASC NULLS FIRST, s_street_name#23 ASC NULLS FIRST, s_street_type#24 ASC NULLS FIRST, s_suite_number#25 ASC NULLS FIRST, s_city#26 ASC NULLS FIRST, s_county#27 ASC NULLS FIRST, s_state#28 ASC NULLS FIRST, s_zip#29 ASC NULLS FIRST], [s_store_name#20, s_company_id#21, s_street_number#22, s_street_name#23, s_street_type#24, s_suite_number#25, s_city#26, s_county#27, s_state#28, s_zip#29, 30 days #47, 31 - 60 days #48, 61 - 90 days #49, 91 - 120 days #50, >120 days #51]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = sr_returned_date_sk#4 IN dynamicpruning#5
+ReusedExchange (36)
+
+
+(36) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#6]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.sf100/simplified.txt
new file mode 100644
index 0000000000000..441e0963f478c
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.sf100/simplified.txt
@@ -0,0 +1,59 @@
+TakeOrderedAndProject [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ]
+  WholeStageCodegen (9)
+    HashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sum,sum,sum,sum,sum] [sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 30) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 60) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 90) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) > 120) THEN 1 ELSE 0 END),30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum]
+      InputAdapter
+        Exchange [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #1
+          WholeStageCodegen (8)
+            HashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sr_returned_date_sk,ss_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum]
+              Project [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip]
+                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                  Project [sr_returned_date_sk,ss_store_sk,ss_sold_date_sk]
+                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                      Project [sr_returned_date_sk,ss_store_sk,ss_sold_date_sk]
+                        SortMergeJoin [sr_ticket_number,sr_item_sk,sr_customer_sk,ss_ticket_number,ss_item_sk,ss_customer_sk]
+                          InputAdapter
+                            WholeStageCodegen (3)
+                              Sort [sr_ticket_number,sr_item_sk,sr_customer_sk]
+                                InputAdapter
+                                  Exchange [sr_ticket_number,sr_item_sk,sr_customer_sk] #2
+                                    WholeStageCodegen (2)
+                                      Project [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk]
+                                        BroadcastHashJoin [sr_returned_date_sk,d_date_sk]
+                                          Filter [sr_ticket_number,sr_item_sk,sr_customer_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk]
+                                                  SubqueryBroadcast [d_date_sk] #1
+                                                    ReusedExchange [d_date_sk] #3
+                                          InputAdapter
+                                            BroadcastExchange #3
+                                              WholeStageCodegen (1)
+                                                Project [d_date_sk]
+                                                  Filter [d_year,d_moy,d_date_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                          InputAdapter
+                            WholeStageCodegen (5)
+                              Sort [ss_ticket_number,ss_item_sk,ss_customer_sk]
+                                InputAdapter
+                                  Exchange [ss_ticket_number,ss_item_sk,ss_customer_sk] #4
+                                    WholeStageCodegen (4)
+                                      Filter [ss_ticket_number,ss_item_sk,ss_customer_sk,ss_store_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk]
+                      InputAdapter
+                        BroadcastExchange #5
+                          WholeStageCodegen (6)
+                            Filter [d_date_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.date_dim [d_date_sk]
+                  InputAdapter
+                    BroadcastExchange #6
+                      WholeStageCodegen (7)
+                        Filter [s_store_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.store [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/explain.txt
new file mode 100644
index 0000000000000..0598c5b81f90a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/explain.txt
@@ -0,0 +1,195 @@
+== Physical Plan ==
+TakeOrderedAndProject (32)
++- * HashAggregate (31)
+   +- Exchange (30)
+      +- * HashAggregate (29)
+         +- * Project (28)
+            +- * BroadcastHashJoin Inner BuildRight (27)
+               :- * Project (21)
+               :  +- * BroadcastHashJoin Inner BuildRight (20)
+               :     :- * Project (15)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (14)
+               :     :     :- * Project (9)
+               :     :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+               :     :     :     :- * Filter (3)
+               :     :     :     :  +- * ColumnarToRow (2)
+               :     :     :     :     +- Scan parquet default.store_sales (1)
+               :     :     :     +- BroadcastExchange (7)
+               :     :     :        +- * Filter (6)
+               :     :     :           +- * ColumnarToRow (5)
+               :     :     :              +- Scan parquet default.store_returns (4)
+               :     :     +- BroadcastExchange (13)
+               :     :        +- * Filter (12)
+               :     :           +- * ColumnarToRow (11)
+               :     :              +- Scan parquet default.store (10)
+               :     +- BroadcastExchange (19)
+               :        +- * Filter (18)
+               :           +- * ColumnarToRow (17)
+               :              +- Scan parquet default.date_dim (16)
+               +- BroadcastExchange (26)
+                  +- * Project (25)
+                     +- * Filter (24)
+                        +- * ColumnarToRow (23)
+                           +- Scan parquet default.date_dim (22)
+
+
+(1) Scan parquet default.store_sales
+Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#5)]
+PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ticket_number:int>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+
+(3) Filter [codegen id : 5]
+Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_store_sk#3))
+
+(4) Scan parquet default.store_returns
+Output [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(sr_returned_date_sk#9), dynamicpruningexpression(sr_returned_date_sk#9 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk), IsNotNull(sr_customer_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_customer_sk:int,sr_ticket_number:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9]
+
+(6) Filter [codegen id : 1]
+Input [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9]
+Condition : ((isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#6)) AND isnotnull(sr_customer_sk#7))
+
+(7) BroadcastExchange
+Input [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9]
+Arguments: HashedRelationBroadcastMode(List(input[2, int, false], input[0, int, false], input[1, int, false]),false), [id=#11]
+
+(8) BroadcastHashJoin [codegen id : 5]
+Left keys [3]: [ss_ticket_number#4, ss_item_sk#1, ss_customer_sk#2]
+Right keys [3]: [sr_ticket_number#8, sr_item_sk#6, sr_customer_sk#7]
+Join condition: None
+
+(9) Project [codegen id : 5]
+Output [3]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9]
+Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9]
+
+(10) Scan parquet default.store
+Output [11]: [s_store_sk#12, s_store_name#13, s_company_id#14, s_street_number#15, s_street_name#16, s_street_type#17, s_suite_number#18, s_city#19, s_county#20, s_state#21, s_zip#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_company_id:int,s_street_number:string,s_street_name:string,s_street_type:string,s_suite_number:string,s_city:string,s_county:string,s_state:string,s_zip:string>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [11]: [s_store_sk#12, s_store_name#13, s_company_id#14, s_street_number#15, s_street_name#16, s_street_type#17, s_suite_number#18, s_city#19, s_county#20, s_state#21, s_zip#22]
+
+(12) Filter [codegen id : 2]
+Input [11]: [s_store_sk#12, s_store_name#13, s_company_id#14, s_street_number#15, s_street_name#16, s_street_type#17, s_suite_number#18, s_city#19, s_county#20, s_state#21, s_zip#22]
+Condition : isnotnull(s_store_sk#12)
+
+(13) BroadcastExchange
+Input [11]: [s_store_sk#12, s_store_name#13, s_company_id#14, s_street_number#15, s_street_name#16, s_street_type#17, s_suite_number#18, s_city#19, s_county#20, s_state#21, s_zip#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#23]
+
+(14) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#12]
+Join condition: None
+
+(15) Project [codegen id : 5]
+Output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#13, s_company_id#14, s_street_number#15, s_street_name#16, s_street_type#17, s_suite_number#18, s_city#19, s_county#20, s_state#21, s_zip#22]
+Input [14]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_sk#12, s_store_name#13, s_company_id#14, s_street_number#15, s_street_name#16, s_street_type#17, s_suite_number#18, s_city#19, s_county#20, s_state#21, s_zip#22]
+
+(16) Scan parquet default.date_dim
+Output [1]: [d_date_sk#24]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int>
+
+(17) ColumnarToRow [codegen id : 3]
+Input [1]: [d_date_sk#24]
+
+(18) Filter [codegen id : 3]
+Input [1]: [d_date_sk#24]
+Condition : isnotnull(d_date_sk#24)
+
+(19) BroadcastExchange
+Input [1]: [d_date_sk#24]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#25]
+
+(20) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_sold_date_sk#5]
+Right keys [1]: [d_date_sk#24]
+Join condition: None
+
+(21) Project [codegen id : 5]
+Output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#13, s_company_id#14, s_street_number#15, s_street_name#16, s_street_type#17, s_suite_number#18, s_city#19, s_county#20, s_state#21, s_zip#22]
+Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#13, s_company_id#14, s_street_number#15, s_street_name#16, s_street_type#17, s_suite_number#18, s_city#19, s_county#20, s_state#21, s_zip#22, d_date_sk#24]
+
+(22) Scan parquet default.date_dim
+Output [3]: [d_date_sk#26, d_year#27, d_moy#28]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,8), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(23) ColumnarToRow [codegen id : 4]
+Input [3]: [d_date_sk#26, d_year#27, d_moy#28]
+
+(24) Filter [codegen id : 4]
+Input [3]: [d_date_sk#26, d_year#27, d_moy#28]
+Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 8)) AND isnotnull(d_date_sk#26))
+
+(25) Project [codegen id : 4]
+Output [1]: [d_date_sk#26]
+Input [3]: [d_date_sk#26, d_year#27, d_moy#28]
+
+(26) BroadcastExchange
+Input [1]: [d_date_sk#26]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#29]
+
+(27) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [sr_returned_date_sk#9]
+Right keys [1]: [d_date_sk#26]
+Join condition: None
+
+(28) Project [codegen id : 5]
+Output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#13, s_company_id#14, s_street_number#15, s_street_name#16, s_street_type#17, s_suite_number#18, s_city#19, s_county#20, s_state#21, s_zip#22]
+Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#13, s_company_id#14, s_street_number#15, s_street_name#16, s_street_type#17, s_suite_number#18, s_city#19, s_county#20, s_state#21, s_zip#22, d_date_sk#26]
+
+(29) HashAggregate [codegen id : 5]
+Input [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#13, s_company_id#14, s_street_number#15, s_street_name#16, s_street_type#17, s_suite_number#18, s_city#19, s_county#20, s_state#21, s_zip#22]
+Keys [10]: [s_store_name#13, s_company_id#14, s_street_number#15, s_street_name#16, s_street_type#17, s_suite_number#18, s_city#19, s_county#20, s_state#21, s_zip#22]
+Functions [5]: [partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)]
+Aggregate Attributes [5]: [sum#30, sum#31, sum#32, sum#33, sum#34]
+Results [15]: [s_store_name#13, s_company_id#14, s_street_number#15, s_street_name#16, s_street_type#17, s_suite_number#18, s_city#19, s_county#20, s_state#21, s_zip#22, sum#35, sum#36, sum#37, sum#38, sum#39]
+
+(30) Exchange
+Input [15]: [s_store_name#13, s_company_id#14, s_street_number#15, s_street_name#16, s_street_type#17, s_suite_number#18, s_city#19, s_county#20, s_state#21, s_zip#22, sum#35, sum#36, sum#37, sum#38, sum#39]
+Arguments: hashpartitioning(s_store_name#13, s_company_id#14, s_street_number#15, s_street_name#16, s_street_type#17, s_suite_number#18, s_city#19, s_county#20, s_state#21, s_zip#22, 5), ENSURE_REQUIREMENTS, [id=#40]
+
+(31) HashAggregate [codegen id : 6]
+Input [15]: [s_store_name#13, s_company_id#14, s_street_number#15, s_street_name#16, s_street_type#17, s_suite_number#18, s_city#19, s_county#20, s_state#21, s_zip#22, sum#35, sum#36, sum#37, sum#38, sum#39]
+Keys [10]: [s_store_name#13, s_company_id#14, s_street_number#15, s_street_name#16, s_street_type#17, s_suite_number#18, s_city#19, s_county#20, s_state#21, s_zip#22]
+Functions [5]: [sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)]
+Aggregate Attributes [5]: [sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#41, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#42, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#43, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#44, sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#45]
+Results [15]: [s_store_name#13, s_company_id#14, s_street_number#15, s_street_name#16, s_street_type#17, s_suite_number#18, s_city#19, s_county#20, s_state#21, s_zip#22, sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#41 AS 30 days #46, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#42 AS 31 - 60 days #47, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#43 AS 61 - 90 days #48, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#44 AS 91 - 120 days #49, sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#45 AS >120 days #50]
+
+(32) TakeOrderedAndProject
+Input [15]: [s_store_name#13, s_company_id#14, s_street_number#15, s_street_name#16, s_street_type#17, s_suite_number#18, s_city#19, s_county#20, s_state#21, s_zip#22, 30 days #46, 31 - 60 days #47, 61 - 90 days #48, 91 - 120 days #49, >120 days #50]
+Arguments: 100, [s_store_name#13 ASC NULLS FIRST, s_company_id#14 ASC NULLS FIRST, s_street_number#15 ASC NULLS FIRST, s_street_name#16 ASC NULLS FIRST, s_street_type#17 ASC NULLS FIRST, s_suite_number#18 ASC NULLS FIRST, s_city#19 ASC NULLS FIRST, s_county#20 ASC NULLS FIRST, s_state#21 ASC NULLS FIRST, s_zip#22 ASC NULLS FIRST], [s_store_name#13, s_company_id#14, s_street_number#15, s_street_name#16, s_street_type#17, s_suite_number#18, s_city#19, s_county#20, s_state#21, s_zip#22, 30 days #46, 31 - 60 days #47, 61 - 90 days #48, 91 - 120 days #49, >120 days #50]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 4 Hosting Expression = sr_returned_date_sk#9 IN dynamicpruning#10
+ReusedExchange (33)
+
+
+(33) ReusedExchange [Reuses operator id: 26]
+Output [1]: [d_date_sk#26]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/simplified.txt
new file mode 100644
index 0000000000000..446b8705f3553
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/simplified.txt
@@ -0,0 +1,50 @@
+TakeOrderedAndProject [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ]
+  WholeStageCodegen (6)
+    HashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sum,sum,sum,sum,sum] [sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 30) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 60) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 90) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) > 120) THEN 1 ELSE 0 END),30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum]
+      InputAdapter
+        Exchange [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #1
+          WholeStageCodegen (5)
+            HashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sr_returned_date_sk,ss_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum]
+              Project [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip]
+                BroadcastHashJoin [sr_returned_date_sk,d_date_sk]
+                  Project [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip]
+                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                      Project [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip]
+                        BroadcastHashJoin [ss_store_sk,s_store_sk]
+                          Project [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk]
+                            BroadcastHashJoin [ss_ticket_number,ss_item_sk,ss_customer_sk,sr_ticket_number,sr_item_sk,sr_customer_sk]
+                              Filter [ss_ticket_number,ss_item_sk,ss_customer_sk,ss_store_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk]
+                              InputAdapter
+                                BroadcastExchange #2
+                                  WholeStageCodegen (1)
+                                    Filter [sr_ticket_number,sr_item_sk,sr_customer_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk]
+                                            SubqueryBroadcast [d_date_sk] #1
+                                              ReusedExchange [d_date_sk] #3
+                          InputAdapter
+                            BroadcastExchange #4
+                              WholeStageCodegen (2)
+                                Filter [s_store_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip]
+                      InputAdapter
+                        BroadcastExchange #5
+                          WholeStageCodegen (3)
+                            Filter [d_date_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.date_dim [d_date_sk]
+                  InputAdapter
+                    BroadcastExchange #3
+                      WholeStageCodegen (4)
+                        Project [d_date_sk]
+                          Filter [d_year,d_moy,d_date_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.sf100/explain.txt
new file mode 100644
index 0000000000000..8fef2a0b3f023
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.sf100/explain.txt
@@ -0,0 +1,242 @@
+== Physical Plan ==
+TakeOrderedAndProject (41)
++- * Filter (40)
+   +- Window (39)
+      +- * Sort (38)
+         +- Exchange (37)
+            +- * Project (36)
+               +- SortMergeJoin FullOuter (35)
+                  :- * Sort (19)
+                  :  +- Exchange (18)
+                  :     +- * Project (17)
+                  :        +- Window (16)
+                  :           +- * Sort (15)
+                  :              +- Exchange (14)
+                  :                 +- * HashAggregate (13)
+                  :                    +- Exchange (12)
+                  :                       +- * HashAggregate (11)
+                  :                          +- * Project (10)
+                  :                             +- * BroadcastHashJoin Inner BuildRight (9)
+                  :                                :- * Filter (3)
+                  :                                :  +- * ColumnarToRow (2)
+                  :                                :     +- Scan parquet default.web_sales (1)
+                  :                                +- BroadcastExchange (8)
+                  :                                   +- * Project (7)
+                  :                                      +- * Filter (6)
+                  :                                         +- * ColumnarToRow (5)
+                  :                                            +- Scan parquet default.date_dim (4)
+                  +- * Sort (34)
+                     +- Exchange (33)
+                        +- * Project (32)
+                           +- Window (31)
+                              +- * Sort (30)
+                                 +- Exchange (29)
+                                    +- * HashAggregate (28)
+                                       +- Exchange (27)
+                                          +- * HashAggregate (26)
+                                             +- * Project (25)
+                                                +- * BroadcastHashJoin Inner BuildRight (24)
+                                                   :- * Filter (22)
+                                                   :  +- * ColumnarToRow (21)
+                                                   :     +- Scan parquet default.store_sales (20)
+                                                   +- ReusedExchange (23)
+
+
+(1) Scan parquet default.web_sales
+Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3]
+
+(3) Filter [codegen id : 2]
+Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3]
+Condition : isnotnull(ws_item_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date,d_month_seq:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7]
+Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#5))
+
+(7) Project [codegen id : 1]
+Output [2]: [d_date_sk#5, d_date#6]
+Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7]
+
+(8) BroadcastExchange
+Input [2]: [d_date_sk#5, d_date#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#8]
+
+(9) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ws_sold_date_sk#3]
+Right keys [1]: [d_date_sk#5]
+Join condition: None
+
+(10) Project [codegen id : 2]
+Output [3]: [ws_item_sk#1, ws_sales_price#2, d_date#6]
+Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#5, d_date#6]
+
+(11) HashAggregate [codegen id : 2]
+Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#6]
+Keys [2]: [ws_item_sk#1, d_date#6]
+Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))]
+Aggregate Attributes [1]: [sum#9]
+Results [3]: [ws_item_sk#1, d_date#6, sum#10]
+
+(12) Exchange
+Input [3]: [ws_item_sk#1, d_date#6, sum#10]
+Arguments: hashpartitioning(ws_item_sk#1, d_date#6, 5), ENSURE_REQUIREMENTS, [id=#11]
+
+(13) HashAggregate [codegen id : 3]
+Input [3]: [ws_item_sk#1, d_date#6, sum#10]
+Keys [2]: [ws_item_sk#1, d_date#6]
+Functions [1]: [sum(UnscaledValue(ws_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#2))#12]
+Results [4]: [ws_item_sk#1 AS item_sk#13, d_date#6, MakeDecimal(sum(UnscaledValue(ws_sales_price#2))#12,17,2) AS _w0#14, ws_item_sk#1]
+
+(14) Exchange
+Input [4]: [item_sk#13, d_date#6, _w0#14, ws_item_sk#1]
+Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(15) Sort [codegen id : 4]
+Input [4]: [item_sk#13, d_date#6, _w0#14, ws_item_sk#1]
+Arguments: [ws_item_sk#1 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST], false, 0
+
+(16) Window
+Input [4]: [item_sk#13, d_date#6, _w0#14, ws_item_sk#1]
+Arguments: [sum(_w0#14) windowspecdefinition(ws_item_sk#1, d_date#6 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#16], [ws_item_sk#1], [d_date#6 ASC NULLS FIRST]
+
+(17) Project [codegen id : 5]
+Output [3]: [item_sk#13, d_date#6, cume_sales#16]
+Input [5]: [item_sk#13, d_date#6, _w0#14, ws_item_sk#1, cume_sales#16]
+
+(18) Exchange
+Input [3]: [item_sk#13, d_date#6, cume_sales#16]
+Arguments: hashpartitioning(item_sk#13, d_date#6, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(19) Sort [codegen id : 6]
+Input [3]: [item_sk#13, d_date#6, cume_sales#16]
+Arguments: [item_sk#13 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST], false, 0
+
+(20) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#18, ss_sales_price#19, ss_sold_date_sk#20]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#20), dynamicpruningexpression(ss_sold_date_sk#20 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_sales_price:decimal(7,2)>
+
+(21) ColumnarToRow [codegen id : 8]
+Input [3]: [ss_item_sk#18, ss_sales_price#19, ss_sold_date_sk#20]
+
+(22) Filter [codegen id : 8]
+Input [3]: [ss_item_sk#18, ss_sales_price#19, ss_sold_date_sk#20]
+Condition : isnotnull(ss_item_sk#18)
+
+(23) ReusedExchange [Reuses operator id: 8]
+Output [2]: [d_date_sk#21, d_date#22]
+
+(24) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_sold_date_sk#20]
+Right keys [1]: [d_date_sk#21]
+Join condition: None
+
+(25) Project [codegen id : 8]
+Output [3]: [ss_item_sk#18, ss_sales_price#19, d_date#22]
+Input [5]: [ss_item_sk#18, ss_sales_price#19, ss_sold_date_sk#20, d_date_sk#21, d_date#22]
+
+(26) HashAggregate [codegen id : 8]
+Input [3]: [ss_item_sk#18, ss_sales_price#19, d_date#22]
+Keys [2]: [ss_item_sk#18, d_date#22]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#19))]
+Aggregate Attributes [1]: [sum#23]
+Results [3]: [ss_item_sk#18, d_date#22, sum#24]
+
+(27) Exchange
+Input [3]: [ss_item_sk#18, d_date#22, sum#24]
+Arguments: hashpartitioning(ss_item_sk#18, d_date#22, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(28) HashAggregate [codegen id : 9]
+Input [3]: [ss_item_sk#18, d_date#22, sum#24]
+Keys [2]: [ss_item_sk#18, d_date#22]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#19))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#19))#26]
+Results [4]: [ss_item_sk#18 AS item_sk#27, d_date#22, MakeDecimal(sum(UnscaledValue(ss_sales_price#19))#26,17,2) AS _w0#28, ss_item_sk#18]
+
+(29) Exchange
+Input [4]: [item_sk#27, d_date#22, _w0#28, ss_item_sk#18]
+Arguments: hashpartitioning(ss_item_sk#18, 5), ENSURE_REQUIREMENTS, [id=#29]
+
+(30) Sort [codegen id : 10]
+Input [4]: [item_sk#27, d_date#22, _w0#28, ss_item_sk#18]
+Arguments: [ss_item_sk#18 ASC NULLS FIRST, d_date#22 ASC NULLS FIRST], false, 0
+
+(31) Window
+Input [4]: [item_sk#27, d_date#22, _w0#28, ss_item_sk#18]
+Arguments: [sum(_w0#28) windowspecdefinition(ss_item_sk#18, d_date#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#30], [ss_item_sk#18], [d_date#22 ASC NULLS FIRST]
+
+(32) Project [codegen id : 11]
+Output [3]: [item_sk#27, d_date#22, cume_sales#30]
+Input [5]: [item_sk#27, d_date#22, _w0#28, ss_item_sk#18, cume_sales#30]
+
+(33) Exchange
+Input [3]: [item_sk#27, d_date#22, cume_sales#30]
+Arguments: hashpartitioning(item_sk#27, d_date#22, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(34) Sort [codegen id : 12]
+Input [3]: [item_sk#27, d_date#22, cume_sales#30]
+Arguments: [item_sk#27 ASC NULLS FIRST, d_date#22 ASC NULLS FIRST], false, 0
+
+(35) SortMergeJoin
+Left keys [2]: [item_sk#13, d_date#6]
+Right keys [2]: [item_sk#27, d_date#22]
+Join condition: None
+
+(36) Project [codegen id : 13]
+Output [4]: [CASE WHEN isnotnull(item_sk#13) THEN item_sk#13 ELSE item_sk#27 END AS item_sk#32, CASE WHEN isnotnull(d_date#6) THEN d_date#6 ELSE d_date#22 END AS d_date#33, cume_sales#16 AS web_sales#34, cume_sales#30 AS store_sales#35]
+Input [6]: [item_sk#13, d_date#6, cume_sales#16, item_sk#27, d_date#22, cume_sales#30]
+
+(37) Exchange
+Input [4]: [item_sk#32, d_date#33, web_sales#34, store_sales#35]
+Arguments: hashpartitioning(item_sk#32, 5), ENSURE_REQUIREMENTS, [id=#36]
+
+(38) Sort [codegen id : 14]
+Input [4]: [item_sk#32, d_date#33, web_sales#34, store_sales#35]
+Arguments: [item_sk#32 ASC NULLS FIRST, d_date#33 ASC NULLS FIRST], false, 0
+
+(39) Window
+Input [4]: [item_sk#32, d_date#33, web_sales#34, store_sales#35]
+Arguments: [max(web_sales#34) windowspecdefinition(item_sk#32, d_date#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#37, max(store_sales#35) windowspecdefinition(item_sk#32, d_date#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#38], [item_sk#32], [d_date#33 ASC NULLS FIRST]
+
+(40) Filter [codegen id : 15]
+Input [6]: [item_sk#32, d_date#33, web_sales#34, store_sales#35, web_cumulative#37, store_cumulative#38]
+Condition : ((isnotnull(web_cumulative#37) AND isnotnull(store_cumulative#38)) AND (web_cumulative#37 > store_cumulative#38))
+
+(41) TakeOrderedAndProject
+Input [6]: [item_sk#32, d_date#33, web_sales#34, store_sales#35, web_cumulative#37, store_cumulative#38]
+Arguments: 100, [item_sk#32 ASC NULLS FIRST, d_date#33 ASC NULLS FIRST], [item_sk#32, d_date#33, web_sales#34, store_sales#35, web_cumulative#37, store_cumulative#38]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (42)
+
+
+(42) ReusedExchange [Reuses operator id: 8]
+Output [2]: [d_date_sk#5, d_date#6]
+
+Subquery:2 Hosting operator id = 20 Hosting Expression = ss_sold_date_sk#20 IN dynamicpruning#4
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.sf100/simplified.txt
new file mode 100644
index 0000000000000..d9831cd015452
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.sf100/simplified.txt
@@ -0,0 +1,74 @@
+TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative]
+  WholeStageCodegen (15)
+    Filter [web_cumulative,store_cumulative]
+      InputAdapter
+        Window [web_sales,item_sk,d_date,store_sales]
+          WholeStageCodegen (14)
+            Sort [item_sk,d_date]
+              InputAdapter
+                Exchange [item_sk] #1
+                  WholeStageCodegen (13)
+                    Project [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales]
+                      InputAdapter
+                        SortMergeJoin [item_sk,d_date,item_sk,d_date]
+                          WholeStageCodegen (6)
+                            Sort [item_sk,d_date]
+                              InputAdapter
+                                Exchange [item_sk,d_date] #2
+                                  WholeStageCodegen (5)
+                                    Project [item_sk,d_date,cume_sales]
+                                      InputAdapter
+                                        Window [_w0,ws_item_sk,d_date]
+                                          WholeStageCodegen (4)
+                                            Sort [ws_item_sk,d_date]
+                                              InputAdapter
+                                                Exchange [ws_item_sk] #3
+                                                  WholeStageCodegen (3)
+                                                    HashAggregate [ws_item_sk,d_date,sum] [sum(UnscaledValue(ws_sales_price)),item_sk,_w0,sum]
+                                                      InputAdapter
+                                                        Exchange [ws_item_sk,d_date] #4
+                                                          WholeStageCodegen (2)
+                                                            HashAggregate [ws_item_sk,d_date,ws_sales_price] [sum,sum]
+                                                              Project [ws_item_sk,ws_sales_price,d_date]
+                                                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                  Filter [ws_item_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk]
+                                                                          SubqueryBroadcast [d_date_sk] #1
+                                                                            ReusedExchange [d_date_sk,d_date] #5
+                                                                  InputAdapter
+                                                                    BroadcastExchange #5
+                                                                      WholeStageCodegen (1)
+                                                                        Project [d_date_sk,d_date]
+                                                                          Filter [d_month_seq,d_date_sk]
+                                                                            ColumnarToRow
+                                                                              InputAdapter
+                                                                                Scan parquet default.date_dim [d_date_sk,d_date,d_month_seq]
+                          WholeStageCodegen (12)
+                            Sort [item_sk,d_date]
+                              InputAdapter
+                                Exchange [item_sk,d_date] #6
+                                  WholeStageCodegen (11)
+                                    Project [item_sk,d_date,cume_sales]
+                                      InputAdapter
+                                        Window [_w0,ss_item_sk,d_date]
+                                          WholeStageCodegen (10)
+                                            Sort [ss_item_sk,d_date]
+                                              InputAdapter
+                                                Exchange [ss_item_sk] #7
+                                                  WholeStageCodegen (9)
+                                                    HashAggregate [ss_item_sk,d_date,sum] [sum(UnscaledValue(ss_sales_price)),item_sk,_w0,sum]
+                                                      InputAdapter
+                                                        Exchange [ss_item_sk,d_date] #8
+                                                          WholeStageCodegen (8)
+                                                            HashAggregate [ss_item_sk,d_date,ss_sales_price] [sum,sum]
+                                                              Project [ss_item_sk,ss_sales_price,d_date]
+                                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                  Filter [ss_item_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk]
+                                                                          ReusedSubquery [d_date_sk] #1
+                                                                  InputAdapter
+                                                                    ReusedExchange [d_date_sk,d_date] #5
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/explain.txt
new file mode 100644
index 0000000000000..8fef2a0b3f023
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/explain.txt
@@ -0,0 +1,242 @@
+== Physical Plan ==
+TakeOrderedAndProject (41)
++- * Filter (40)
+   +- Window (39)
+      +- * Sort (38)
+         +- Exchange (37)
+            +- * Project (36)
+               +- SortMergeJoin FullOuter (35)
+                  :- * Sort (19)
+                  :  +- Exchange (18)
+                  :     +- * Project (17)
+                  :        +- Window (16)
+                  :           +- * Sort (15)
+                  :              +- Exchange (14)
+                  :                 +- * HashAggregate (13)
+                  :                    +- Exchange (12)
+                  :                       +- * HashAggregate (11)
+                  :                          +- * Project (10)
+                  :                             +- * BroadcastHashJoin Inner BuildRight (9)
+                  :                                :- * Filter (3)
+                  :                                :  +- * ColumnarToRow (2)
+                  :                                :     +- Scan parquet default.web_sales (1)
+                  :                                +- BroadcastExchange (8)
+                  :                                   +- * Project (7)
+                  :                                      +- * Filter (6)
+                  :                                         +- * ColumnarToRow (5)
+                  :                                            +- Scan parquet default.date_dim (4)
+                  +- * Sort (34)
+                     +- Exchange (33)
+                        +- * Project (32)
+                           +- Window (31)
+                              +- * Sort (30)
+                                 +- Exchange (29)
+                                    +- * HashAggregate (28)
+                                       +- Exchange (27)
+                                          +- * HashAggregate (26)
+                                             +- * Project (25)
+                                                +- * BroadcastHashJoin Inner BuildRight (24)
+                                                   :- * Filter (22)
+                                                   :  +- * ColumnarToRow (21)
+                                                   :     +- Scan parquet default.store_sales (20)
+                                                   +- ReusedExchange (23)
+
+
+(1) Scan parquet default.web_sales
+Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3]
+
+(3) Filter [codegen id : 2]
+Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3]
+Condition : isnotnull(ws_item_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date,d_month_seq:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7]
+Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#5))
+
+(7) Project [codegen id : 1]
+Output [2]: [d_date_sk#5, d_date#6]
+Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7]
+
+(8) BroadcastExchange
+Input [2]: [d_date_sk#5, d_date#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#8]
+
+(9) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ws_sold_date_sk#3]
+Right keys [1]: [d_date_sk#5]
+Join condition: None
+
+(10) Project [codegen id : 2]
+Output [3]: [ws_item_sk#1, ws_sales_price#2, d_date#6]
+Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#5, d_date#6]
+
+(11) HashAggregate [codegen id : 2]
+Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#6]
+Keys [2]: [ws_item_sk#1, d_date#6]
+Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))]
+Aggregate Attributes [1]: [sum#9]
+Results [3]: [ws_item_sk#1, d_date#6, sum#10]
+
+(12) Exchange
+Input [3]: [ws_item_sk#1, d_date#6, sum#10]
+Arguments: hashpartitioning(ws_item_sk#1, d_date#6, 5), ENSURE_REQUIREMENTS, [id=#11]
+
+(13) HashAggregate [codegen id : 3]
+Input [3]: [ws_item_sk#1, d_date#6, sum#10]
+Keys [2]: [ws_item_sk#1, d_date#6]
+Functions [1]: [sum(UnscaledValue(ws_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#2))#12]
+Results [4]: [ws_item_sk#1 AS item_sk#13, d_date#6, MakeDecimal(sum(UnscaledValue(ws_sales_price#2))#12,17,2) AS _w0#14, ws_item_sk#1]
+
+(14) Exchange
+Input [4]: [item_sk#13, d_date#6, _w0#14, ws_item_sk#1]
+Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(15) Sort [codegen id : 4]
+Input [4]: [item_sk#13, d_date#6, _w0#14, ws_item_sk#1]
+Arguments: [ws_item_sk#1 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST], false, 0
+
+(16) Window
+Input [4]: [item_sk#13, d_date#6, _w0#14, ws_item_sk#1]
+Arguments: [sum(_w0#14) windowspecdefinition(ws_item_sk#1, d_date#6 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#16], [ws_item_sk#1], [d_date#6 ASC NULLS FIRST]
+
+(17) Project [codegen id : 5]
+Output [3]: [item_sk#13, d_date#6, cume_sales#16]
+Input [5]: [item_sk#13, d_date#6, _w0#14, ws_item_sk#1, cume_sales#16]
+
+(18) Exchange
+Input [3]: [item_sk#13, d_date#6, cume_sales#16]
+Arguments: hashpartitioning(item_sk#13, d_date#6, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(19) Sort [codegen id : 6]
+Input [3]: [item_sk#13, d_date#6, cume_sales#16]
+Arguments: [item_sk#13 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST], false, 0
+
+(20) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#18, ss_sales_price#19, ss_sold_date_sk#20]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#20), dynamicpruningexpression(ss_sold_date_sk#20 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_sales_price:decimal(7,2)>
+
+(21) ColumnarToRow [codegen id : 8]
+Input [3]: [ss_item_sk#18, ss_sales_price#19, ss_sold_date_sk#20]
+
+(22) Filter [codegen id : 8]
+Input [3]: [ss_item_sk#18, ss_sales_price#19, ss_sold_date_sk#20]
+Condition : isnotnull(ss_item_sk#18)
+
+(23) ReusedExchange [Reuses operator id: 8]
+Output [2]: [d_date_sk#21, d_date#22]
+
+(24) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_sold_date_sk#20]
+Right keys [1]: [d_date_sk#21]
+Join condition: None
+
+(25) Project [codegen id : 8]
+Output [3]: [ss_item_sk#18, ss_sales_price#19, d_date#22]
+Input [5]: [ss_item_sk#18, ss_sales_price#19, ss_sold_date_sk#20, d_date_sk#21, d_date#22]
+
+(26) HashAggregate [codegen id : 8]
+Input [3]: [ss_item_sk#18, ss_sales_price#19, d_date#22]
+Keys [2]: [ss_item_sk#18, d_date#22]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#19))]
+Aggregate Attributes [1]: [sum#23]
+Results [3]: [ss_item_sk#18, d_date#22, sum#24]
+
+(27) Exchange
+Input [3]: [ss_item_sk#18, d_date#22, sum#24]
+Arguments: hashpartitioning(ss_item_sk#18, d_date#22, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(28) HashAggregate [codegen id : 9]
+Input [3]: [ss_item_sk#18, d_date#22, sum#24]
+Keys [2]: [ss_item_sk#18, d_date#22]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#19))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#19))#26]
+Results [4]: [ss_item_sk#18 AS item_sk#27, d_date#22, MakeDecimal(sum(UnscaledValue(ss_sales_price#19))#26,17,2) AS _w0#28, ss_item_sk#18]
+
+(29) Exchange
+Input [4]: [item_sk#27, d_date#22, _w0#28, ss_item_sk#18]
+Arguments: hashpartitioning(ss_item_sk#18, 5), ENSURE_REQUIREMENTS, [id=#29]
+
+(30) Sort [codegen id : 10]
+Input [4]: [item_sk#27, d_date#22, _w0#28, ss_item_sk#18]
+Arguments: [ss_item_sk#18 ASC NULLS FIRST, d_date#22 ASC NULLS FIRST], false, 0
+
+(31) Window
+Input [4]: [item_sk#27, d_date#22, _w0#28, ss_item_sk#18]
+Arguments: [sum(_w0#28) windowspecdefinition(ss_item_sk#18, d_date#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#30], [ss_item_sk#18], [d_date#22 ASC NULLS FIRST]
+
+(32) Project [codegen id : 11]
+Output [3]: [item_sk#27, d_date#22, cume_sales#30]
+Input [5]: [item_sk#27, d_date#22, _w0#28, ss_item_sk#18, cume_sales#30]
+
+(33) Exchange
+Input [3]: [item_sk#27, d_date#22, cume_sales#30]
+Arguments: hashpartitioning(item_sk#27, d_date#22, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(34) Sort [codegen id : 12]
+Input [3]: [item_sk#27, d_date#22, cume_sales#30]
+Arguments: [item_sk#27 ASC NULLS FIRST, d_date#22 ASC NULLS FIRST], false, 0
+
+(35) SortMergeJoin
+Left keys [2]: [item_sk#13, d_date#6]
+Right keys [2]: [item_sk#27, d_date#22]
+Join condition: None
+
+(36) Project [codegen id : 13]
+Output [4]: [CASE WHEN isnotnull(item_sk#13) THEN item_sk#13 ELSE item_sk#27 END AS item_sk#32, CASE WHEN isnotnull(d_date#6) THEN d_date#6 ELSE d_date#22 END AS d_date#33, cume_sales#16 AS web_sales#34, cume_sales#30 AS store_sales#35]
+Input [6]: [item_sk#13, d_date#6, cume_sales#16, item_sk#27, d_date#22, cume_sales#30]
+
+(37) Exchange
+Input [4]: [item_sk#32, d_date#33, web_sales#34, store_sales#35]
+Arguments: hashpartitioning(item_sk#32, 5), ENSURE_REQUIREMENTS, [id=#36]
+
+(38) Sort [codegen id : 14]
+Input [4]: [item_sk#32, d_date#33, web_sales#34, store_sales#35]
+Arguments: [item_sk#32 ASC NULLS FIRST, d_date#33 ASC NULLS FIRST], false, 0
+
+(39) Window
+Input [4]: [item_sk#32, d_date#33, web_sales#34, store_sales#35]
+Arguments: [max(web_sales#34) windowspecdefinition(item_sk#32, d_date#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#37, max(store_sales#35) windowspecdefinition(item_sk#32, d_date#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#38], [item_sk#32], [d_date#33 ASC NULLS FIRST]
+
+(40) Filter [codegen id : 15]
+Input [6]: [item_sk#32, d_date#33, web_sales#34, store_sales#35, web_cumulative#37, store_cumulative#38]
+Condition : ((isnotnull(web_cumulative#37) AND isnotnull(store_cumulative#38)) AND (web_cumulative#37 > store_cumulative#38))
+
+(41) TakeOrderedAndProject
+Input [6]: [item_sk#32, d_date#33, web_sales#34, store_sales#35, web_cumulative#37, store_cumulative#38]
+Arguments: 100, [item_sk#32 ASC NULLS FIRST, d_date#33 ASC NULLS FIRST], [item_sk#32, d_date#33, web_sales#34, store_sales#35, web_cumulative#37, store_cumulative#38]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (42)
+
+
+(42) ReusedExchange [Reuses operator id: 8]
+Output [2]: [d_date_sk#5, d_date#6]
+
+Subquery:2 Hosting operator id = 20 Hosting Expression = ss_sold_date_sk#20 IN dynamicpruning#4
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/simplified.txt
new file mode 100644
index 0000000000000..d9831cd015452
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/simplified.txt
@@ -0,0 +1,74 @@
+TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative]
+  WholeStageCodegen (15)
+    Filter [web_cumulative,store_cumulative]
+      InputAdapter
+        Window [web_sales,item_sk,d_date,store_sales]
+          WholeStageCodegen (14)
+            Sort [item_sk,d_date]
+              InputAdapter
+                Exchange [item_sk] #1
+                  WholeStageCodegen (13)
+                    Project [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales]
+                      InputAdapter
+                        SortMergeJoin [item_sk,d_date,item_sk,d_date]
+                          WholeStageCodegen (6)
+                            Sort [item_sk,d_date]
+                              InputAdapter
+                                Exchange [item_sk,d_date] #2
+                                  WholeStageCodegen (5)
+                                    Project [item_sk,d_date,cume_sales]
+                                      InputAdapter
+                                        Window [_w0,ws_item_sk,d_date]
+                                          WholeStageCodegen (4)
+                                            Sort [ws_item_sk,d_date]
+                                              InputAdapter
+                                                Exchange [ws_item_sk] #3
+                                                  WholeStageCodegen (3)
+                                                    HashAggregate [ws_item_sk,d_date,sum] [sum(UnscaledValue(ws_sales_price)),item_sk,_w0,sum]
+                                                      InputAdapter
+                                                        Exchange [ws_item_sk,d_date] #4
+                                                          WholeStageCodegen (2)
+                                                            HashAggregate [ws_item_sk,d_date,ws_sales_price] [sum,sum]
+                                                              Project [ws_item_sk,ws_sales_price,d_date]
+                                                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                  Filter [ws_item_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk]
+                                                                          SubqueryBroadcast [d_date_sk] #1
+                                                                            ReusedExchange [d_date_sk,d_date] #5
+                                                                  InputAdapter
+                                                                    BroadcastExchange #5
+                                                                      WholeStageCodegen (1)
+                                                                        Project [d_date_sk,d_date]
+                                                                          Filter [d_month_seq,d_date_sk]
+                                                                            ColumnarToRow
+                                                                              InputAdapter
+                                                                                Scan parquet default.date_dim [d_date_sk,d_date,d_month_seq]
+                          WholeStageCodegen (12)
+                            Sort [item_sk,d_date]
+                              InputAdapter
+                                Exchange [item_sk,d_date] #6
+                                  WholeStageCodegen (11)
+                                    Project [item_sk,d_date,cume_sales]
+                                      InputAdapter
+                                        Window [_w0,ss_item_sk,d_date]
+                                          WholeStageCodegen (10)
+                                            Sort [ss_item_sk,d_date]
+                                              InputAdapter
+                                                Exchange [ss_item_sk] #7
+                                                  WholeStageCodegen (9)
+                                                    HashAggregate [ss_item_sk,d_date,sum] [sum(UnscaledValue(ss_sales_price)),item_sk,_w0,sum]
+                                                      InputAdapter
+                                                        Exchange [ss_item_sk,d_date] #8
+                                                          WholeStageCodegen (8)
+                                                            HashAggregate [ss_item_sk,d_date,ss_sales_price] [sum,sum]
+                                                              Project [ss_item_sk,ss_sales_price,d_date]
+                                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                  Filter [ss_item_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk]
+                                                                          ReusedSubquery [d_date_sk] #1
+                                                                  InputAdapter
+                                                                    ReusedExchange [d_date_sk,d_date] #5
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.sf100/explain.txt
new file mode 100644
index 0000000000000..aa943cd6a7473
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.sf100/explain.txt
@@ -0,0 +1,133 @@
+== Physical Plan ==
+TakeOrderedAndProject (21)
++- * HashAggregate (20)
+   +- Exchange (19)
+      +- * HashAggregate (18)
+         +- * Project (17)
+            +- * BroadcastHashJoin Inner BuildRight (16)
+               :- * Project (10)
+               :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :- * Filter (3)
+               :     :  +- * ColumnarToRow (2)
+               :     :     +- Scan parquet default.store_sales (1)
+               :     +- BroadcastExchange (8)
+               :        +- * Project (7)
+               :           +- * Filter (6)
+               :              +- * ColumnarToRow (5)
+               :                 +- Scan parquet default.item (4)
+               +- BroadcastExchange (15)
+                  +- * Project (14)
+                     +- * Filter (13)
+                        +- * ColumnarToRow (12)
+                           +- Scan parquet default.date_dim (11)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_item_sk#1)
+
+(4) Scan parquet default.item
+Output [4]: [i_item_sk#5, i_brand_id#6, i_brand#7, i_manager_id#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_brand:string,i_manager_id:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [i_item_sk#5, i_brand_id#6, i_brand#7, i_manager_id#8]
+
+(6) Filter [codegen id : 1]
+Input [4]: [i_item_sk#5, i_brand_id#6, i_brand#7, i_manager_id#8]
+Condition : ((isnotnull(i_manager_id#8) AND (i_manager_id#8 = 1)) AND isnotnull(i_item_sk#5))
+
+(7) Project [codegen id : 1]
+Output [3]: [i_item_sk#5, i_brand_id#6, i_brand#7]
+Input [4]: [i_item_sk#5, i_brand_id#6, i_brand#7, i_manager_id#8]
+
+(8) BroadcastExchange
+Input [3]: [i_item_sk#5, i_brand_id#6, i_brand#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#9]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#5]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [4]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_brand_id#6, i_brand#7]
+Input [6]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_brand_id#6, i_brand#7]
+
+(11) Scan parquet default.date_dim
+Output [3]: [d_date_sk#10, d_year#11, d_moy#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+
+(13) Filter [codegen id : 2]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+Condition : ((((isnotnull(d_moy#12) AND isnotnull(d_year#11)) AND (d_moy#12 = 11)) AND (d_year#11 = 2000)) AND isnotnull(d_date_sk#10))
+
+(14) Project [codegen id : 2]
+Output [2]: [d_date_sk#10, d_year#11]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+
+(15) BroadcastExchange
+Input [2]: [d_date_sk#10, d_year#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(16) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(17) Project [codegen id : 3]
+Output [4]: [d_year#11, ss_ext_sales_price#2, i_brand_id#6, i_brand#7]
+Input [6]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_brand_id#6, i_brand#7, d_date_sk#10, d_year#11]
+
+(18) HashAggregate [codegen id : 3]
+Input [4]: [d_year#11, ss_ext_sales_price#2, i_brand_id#6, i_brand#7]
+Keys [3]: [d_year#11, i_brand#7, i_brand_id#6]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum#14]
+Results [4]: [d_year#11, i_brand#7, i_brand_id#6, sum#15]
+
+(19) Exchange
+Input [4]: [d_year#11, i_brand#7, i_brand_id#6, sum#15]
+Arguments: hashpartitioning(d_year#11, i_brand#7, i_brand_id#6, 5), ENSURE_REQUIREMENTS, [id=#16]
+
+(20) HashAggregate [codegen id : 4]
+Input [4]: [d_year#11, i_brand#7, i_brand_id#6, sum#15]
+Keys [3]: [d_year#11, i_brand#7, i_brand_id#6]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#17]
+Results [4]: [d_year#11, i_brand_id#6 AS brand_id#18, i_brand#7 AS brand#19, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#17,17,2) AS ext_price#20]
+
+(21) TakeOrderedAndProject
+Input [4]: [d_year#11, brand_id#18, brand#19, ext_price#20]
+Arguments: 100, [d_year#11 ASC NULLS FIRST, ext_price#20 DESC NULLS LAST, brand_id#18 ASC NULLS FIRST], [d_year#11, brand_id#18, brand#19, ext_price#20]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (22)
+
+
+(22) ReusedExchange [Reuses operator id: 15]
+Output [2]: [d_date_sk#10, d_year#11]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.sf100/simplified.txt
new file mode 100644
index 0000000000000..c00005c8eb798
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.sf100/simplified.txt
@@ -0,0 +1,33 @@
+TakeOrderedAndProject [d_year,ext_price,brand_id,brand]
+  WholeStageCodegen (4)
+    HashAggregate [d_year,i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum]
+      InputAdapter
+        Exchange [d_year,i_brand,i_brand_id] #1
+          WholeStageCodegen (3)
+            HashAggregate [d_year,i_brand,i_brand_id,ss_ext_sales_price] [sum,sum]
+              Project [d_year,ss_ext_sales_price,i_brand_id,i_brand]
+                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                  Project [ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_brand]
+                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                      Filter [ss_item_sk]
+                        ColumnarToRow
+                          InputAdapter
+                            Scan parquet default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk]
+                              SubqueryBroadcast [d_date_sk] #1
+                                ReusedExchange [d_date_sk,d_year] #2
+                      InputAdapter
+                        BroadcastExchange #3
+                          WholeStageCodegen (1)
+                            Project [i_item_sk,i_brand_id,i_brand]
+                              Filter [i_manager_id,i_item_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.item [i_item_sk,i_brand_id,i_brand,i_manager_id]
+                  InputAdapter
+                    BroadcastExchange #2
+                      WholeStageCodegen (2)
+                        Project [d_date_sk,d_year]
+                          Filter [d_moy,d_year,d_date_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/explain.txt
new file mode 100644
index 0000000000000..e6e106706fb07
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/explain.txt
@@ -0,0 +1,123 @@
+== Physical Plan ==
+TakeOrderedAndProject (21)
++- * HashAggregate (20)
+   +- Exchange (19)
+      +- * HashAggregate (18)
+         +- * Project (17)
+            +- * BroadcastHashJoin Inner BuildRight (16)
+               :- * Project (10)
+               :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :- * Project (4)
+               :     :  +- * Filter (3)
+               :     :     +- * ColumnarToRow (2)
+               :     :        +- Scan parquet default.date_dim (1)
+               :     +- BroadcastExchange (8)
+               :        +- * Filter (7)
+               :           +- * ColumnarToRow (6)
+               :              +- Scan parquet default.store_sales (5)
+               +- BroadcastExchange (15)
+                  +- * Project (14)
+                     +- * Filter (13)
+                        +- * ColumnarToRow (12)
+                           +- Scan parquet default.item (11)
+
+
+(1) Scan parquet default.date_dim
+Output [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk#1))
+
+(4) Project [codegen id : 3]
+Output [2]: [d_date_sk#1, d_year#2]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(5) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+
+(7) Filter [codegen id : 1]
+Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Condition : isnotnull(ss_item_sk#4)
+
+(8) BroadcastExchange
+Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [d_date_sk#1]
+Right keys [1]: [ss_sold_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5]
+Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+
+(11) Scan parquet default.item
+Output [4]: [i_item_sk#8, i_brand_id#9, i_brand#10, i_manager_id#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_brand:string,i_manager_id:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [4]: [i_item_sk#8, i_brand_id#9, i_brand#10, i_manager_id#11]
+
+(13) Filter [codegen id : 2]
+Input [4]: [i_item_sk#8, i_brand_id#9, i_brand#10, i_manager_id#11]
+Condition : ((isnotnull(i_manager_id#11) AND (i_manager_id#11 = 1)) AND isnotnull(i_item_sk#8))
+
+(14) Project [codegen id : 2]
+Output [3]: [i_item_sk#8, i_brand_id#9, i_brand#10]
+Input [4]: [i_item_sk#8, i_brand_id#9, i_brand#10, i_manager_id#11]
+
+(15) BroadcastExchange
+Input [3]: [i_item_sk#8, i_brand_id#9, i_brand#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(16) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_item_sk#4]
+Right keys [1]: [i_item_sk#8]
+Join condition: None
+
+(17) Project [codegen id : 3]
+Output [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#9, i_brand#10]
+Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#8, i_brand_id#9, i_brand#10]
+
+(18) HashAggregate [codegen id : 3]
+Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#9, i_brand#10]
+Keys [3]: [d_year#2, i_brand#10, i_brand_id#9]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))]
+Aggregate Attributes [1]: [sum#13]
+Results [4]: [d_year#2, i_brand#10, i_brand_id#9, sum#14]
+
+(19) Exchange
+Input [4]: [d_year#2, i_brand#10, i_brand_id#9, sum#14]
+Arguments: hashpartitioning(d_year#2, i_brand#10, i_brand_id#9, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(20) HashAggregate [codegen id : 4]
+Input [4]: [d_year#2, i_brand#10, i_brand_id#9, sum#14]
+Keys [3]: [d_year#2, i_brand#10, i_brand_id#9]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#16]
+Results [4]: [d_year#2, i_brand_id#9 AS brand_id#17, i_brand#10 AS brand#18, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#16,17,2) AS ext_price#19]
+
+(21) TakeOrderedAndProject
+Input [4]: [d_year#2, brand_id#17, brand#18, ext_price#19]
+Arguments: 100, [d_year#2 ASC NULLS FIRST, ext_price#19 DESC NULLS LAST, brand_id#17 ASC NULLS FIRST], [d_year#2, brand_id#17, brand#18, ext_price#19]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/simplified.txt
new file mode 100644
index 0000000000000..d16dd603ec66a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/simplified.txt
@@ -0,0 +1,31 @@
+TakeOrderedAndProject [d_year,ext_price,brand_id,brand]
+  WholeStageCodegen (4)
+    HashAggregate [d_year,i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum]
+      InputAdapter
+        Exchange [d_year,i_brand,i_brand_id] #1
+          WholeStageCodegen (3)
+            HashAggregate [d_year,i_brand,i_brand_id,ss_ext_sales_price] [sum,sum]
+              Project [d_year,ss_ext_sales_price,i_brand_id,i_brand]
+                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                  Project [d_year,ss_item_sk,ss_ext_sales_price]
+                    BroadcastHashJoin [d_date_sk,ss_sold_date_sk]
+                      Project [d_date_sk,d_year]
+                        Filter [d_moy,d_year,d_date_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                      InputAdapter
+                        BroadcastExchange #2
+                          WholeStageCodegen (1)
+                            Filter [ss_item_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk]
+                  InputAdapter
+                    BroadcastExchange #3
+                      WholeStageCodegen (2)
+                        Project [i_item_sk,i_brand_id,i_brand]
+                          Filter [i_manager_id,i_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.item [i_item_sk,i_brand_id,i_brand,i_manager_id]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.sf100/explain.txt
new file mode 100644
index 0000000000000..e0cd0729e0e32
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.sf100/explain.txt
@@ -0,0 +1,191 @@
+== Physical Plan ==
+TakeOrderedAndProject (32)
++- * Project (31)
+   +- * Filter (30)
+      +- Window (29)
+         +- * Sort (28)
+            +- Exchange (27)
+               +- * HashAggregate (26)
+                  +- Exchange (25)
+                     +- * HashAggregate (24)
+                        +- * Project (23)
+                           +- * BroadcastHashJoin Inner BuildRight (22)
+                              :- * Project (16)
+                              :  +- * BroadcastHashJoin Inner BuildRight (15)
+                              :     :- * Project (10)
+                              :     :  +- * BroadcastHashJoin Inner BuildLeft (9)
+                              :     :     :- BroadcastExchange (5)
+                              :     :     :  +- * Project (4)
+                              :     :     :     +- * Filter (3)
+                              :     :     :        +- * ColumnarToRow (2)
+                              :     :     :           +- Scan parquet default.item (1)
+                              :     :     +- * Filter (8)
+                              :     :        +- * ColumnarToRow (7)
+                              :     :           +- Scan parquet default.store_sales (6)
+                              :     +- BroadcastExchange (14)
+                              :        +- * Filter (13)
+                              :           +- * ColumnarToRow (12)
+                              :              +- Scan parquet default.store (11)
+                              +- BroadcastExchange (21)
+                                 +- * Project (20)
+                                    +- * Filter (19)
+                                       +- * ColumnarToRow (18)
+                                          +- Scan parquet default.date_dim (17)
+
+
+(1) Scan parquet default.item
+Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [Or(And(And(In(i_category, [Books                                             ,Children                                          ,Electronics                                       ]),In(i_class, [personal                                          ,portable                                          ,reference                                         ,self-help                                         ])),In(i_brand, [scholaramalgamalg #6                             ,scholaramalgamalg #7                              ,exportiunivamalg #8                               ,scholaramalgamalg #8                              ])),And(And(In(i_category, [Women                                             ,Music                                             ,Men                                               ]),In(i_class, [accessories                                       ,classical                                         ,fragrances                                        ,pants                                             ])),In(i_brand, [amalgimporto #9                                   ,edu packscholar #9                                ,exportiimporto #9                                 ,importoamalg #9                                   ]))), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_class:string,i_category:string,i_manufact_id:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5]
+
+(3) Filter [codegen id : 1]
+Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5]
+Condition : ((((i_category#4 IN (Books                                             ,Children                                          ,Electronics                                       ) AND i_class#3 IN (personal                                          ,portable                                          ,reference                                         ,self-help                                         )) AND i_brand#2 IN (scholaramalgamalg #6                             ,scholaramalgamalg #7                              ,exportiunivamalg #8                               ,scholaramalgamalg #8                              )) OR ((i_category#4 IN (Women                                             ,Music                                             ,Men                                               ) AND i_class#3 IN (accessories                                       ,classical                                         ,fragrances                                        ,pants                                             )) AND i_brand#2 IN (amalgimporto #9                                   ,edu packscholar #9                                ,exportiimporto #9                                 ,importoamalg #9                                   ))) AND isnotnull(i_item_sk#1))
+
+(4) Project [codegen id : 1]
+Output [2]: [i_item_sk#1, i_manufact_id#5]
+Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5]
+
+(5) BroadcastExchange
+Input [2]: [i_item_sk#1, i_manufact_id#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(6) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#11, ss_store_sk#12, ss_sales_price#13, ss_sold_date_sk#14]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#14), dynamicpruningexpression(ss_sold_date_sk#14 IN dynamicpruning#15)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(7) ColumnarToRow
+Input [4]: [ss_item_sk#11, ss_store_sk#12, ss_sales_price#13, ss_sold_date_sk#14]
+
+(8) Filter
+Input [4]: [ss_item_sk#11, ss_store_sk#12, ss_sales_price#13, ss_sold_date_sk#14]
+Condition : (isnotnull(ss_item_sk#11) AND isnotnull(ss_store_sk#12))
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [ss_item_sk#11]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [4]: [i_manufact_id#5, ss_store_sk#12, ss_sales_price#13, ss_sold_date_sk#14]
+Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#11, ss_store_sk#12, ss_sales_price#13, ss_sold_date_sk#14]
+
+(11) Scan parquet default.store
+Output [1]: [s_store_sk#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [1]: [s_store_sk#16]
+
+(13) Filter [codegen id : 2]
+Input [1]: [s_store_sk#16]
+Condition : isnotnull(s_store_sk#16)
+
+(14) BroadcastExchange
+Input [1]: [s_store_sk#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#17]
+
+(15) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#12]
+Right keys [1]: [s_store_sk#16]
+Join condition: None
+
+(16) Project [codegen id : 4]
+Output [3]: [i_manufact_id#5, ss_sales_price#13, ss_sold_date_sk#14]
+Input [5]: [i_manufact_id#5, ss_store_sk#12, ss_sales_price#13, ss_sold_date_sk#14, s_store_sk#16]
+
+(17) Scan parquet default.date_dim
+Output [3]: [d_date_sk#18, d_month_seq#19, d_qoy#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [In(d_month_seq, [1200,1211,1205,1201,1206,1210,1207,1202,1209,1203,1208,1204]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int,d_qoy:int>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#18, d_month_seq#19, d_qoy#20]
+
+(19) Filter [codegen id : 3]
+Input [3]: [d_date_sk#18, d_month_seq#19, d_qoy#20]
+Condition : (d_month_seq#19 INSET (1200,1211,1205,1201,1206,1210,1207,1202,1209,1203,1208,1204) AND isnotnull(d_date_sk#18))
+
+(20) Project [codegen id : 3]
+Output [2]: [d_date_sk#18, d_qoy#20]
+Input [3]: [d_date_sk#18, d_month_seq#19, d_qoy#20]
+
+(21) BroadcastExchange
+Input [2]: [d_date_sk#18, d_qoy#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(22) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#14]
+Right keys [1]: [d_date_sk#18]
+Join condition: None
+
+(23) Project [codegen id : 4]
+Output [3]: [i_manufact_id#5, ss_sales_price#13, d_qoy#20]
+Input [5]: [i_manufact_id#5, ss_sales_price#13, ss_sold_date_sk#14, d_date_sk#18, d_qoy#20]
+
+(24) HashAggregate [codegen id : 4]
+Input [3]: [i_manufact_id#5, ss_sales_price#13, d_qoy#20]
+Keys [2]: [i_manufact_id#5, d_qoy#20]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#13))]
+Aggregate Attributes [1]: [sum#22]
+Results [3]: [i_manufact_id#5, d_qoy#20, sum#23]
+
+(25) Exchange
+Input [3]: [i_manufact_id#5, d_qoy#20, sum#23]
+Arguments: hashpartitioning(i_manufact_id#5, d_qoy#20, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(26) HashAggregate [codegen id : 5]
+Input [3]: [i_manufact_id#5, d_qoy#20, sum#23]
+Keys [2]: [i_manufact_id#5, d_qoy#20]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#13))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#13))#25]
+Results [3]: [i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#13))#25,17,2) AS sum_sales#26, MakeDecimal(sum(UnscaledValue(ss_sales_price#13))#25,17,2) AS _w0#27]
+
+(27) Exchange
+Input [3]: [i_manufact_id#5, sum_sales#26, _w0#27]
+Arguments: hashpartitioning(i_manufact_id#5, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(28) Sort [codegen id : 6]
+Input [3]: [i_manufact_id#5, sum_sales#26, _w0#27]
+Arguments: [i_manufact_id#5 ASC NULLS FIRST], false, 0
+
+(29) Window
+Input [3]: [i_manufact_id#5, sum_sales#26, _w0#27]
+Arguments: [avg(_w0#27) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#29], [i_manufact_id#5]
+
+(30) Filter [codegen id : 7]
+Input [4]: [i_manufact_id#5, sum_sales#26, _w0#27, avg_quarterly_sales#29]
+Condition : (isnotnull(avg_quarterly_sales#29) AND ((avg_quarterly_sales#29 > 0.000000) AND (CheckOverflow((promote_precision(abs(CheckOverflow((promote_precision(cast(sum_sales#26 as decimal(22,6))) - promote_precision(cast(avg_quarterly_sales#29 as decimal(22,6)))), DecimalType(22,6), true), false)) / promote_precision(cast(avg_quarterly_sales#29 as decimal(22,6)))), DecimalType(38,16), true) > 0.1000000000000000)))
+
+(31) Project [codegen id : 7]
+Output [3]: [i_manufact_id#5, sum_sales#26, avg_quarterly_sales#29]
+Input [4]: [i_manufact_id#5, sum_sales#26, _w0#27, avg_quarterly_sales#29]
+
+(32) TakeOrderedAndProject
+Input [3]: [i_manufact_id#5, sum_sales#26, avg_quarterly_sales#29]
+Arguments: 100, [avg_quarterly_sales#29 ASC NULLS FIRST, sum_sales#26 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#26, avg_quarterly_sales#29]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 6 Hosting Expression = ss_sold_date_sk#14 IN dynamicpruning#15
+ReusedExchange (33)
+
+
+(33) ReusedExchange [Reuses operator id: 21]
+Output [2]: [d_date_sk#18, d_qoy#20]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.sf100/simplified.txt
new file mode 100644
index 0000000000000..2e57ebbc0e801
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.sf100/simplified.txt
@@ -0,0 +1,51 @@
+TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id]
+  WholeStageCodegen (7)
+    Project [i_manufact_id,sum_sales,avg_quarterly_sales]
+      Filter [avg_quarterly_sales,sum_sales]
+        InputAdapter
+          Window [_w0,i_manufact_id]
+            WholeStageCodegen (6)
+              Sort [i_manufact_id]
+                InputAdapter
+                  Exchange [i_manufact_id] #1
+                    WholeStageCodegen (5)
+                      HashAggregate [i_manufact_id,d_qoy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum]
+                        InputAdapter
+                          Exchange [i_manufact_id,d_qoy] #2
+                            WholeStageCodegen (4)
+                              HashAggregate [i_manufact_id,d_qoy,ss_sales_price] [sum,sum]
+                                Project [i_manufact_id,ss_sales_price,d_qoy]
+                                  BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                    Project [i_manufact_id,ss_sales_price,ss_sold_date_sk]
+                                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                        Project [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                          BroadcastHashJoin [i_item_sk,ss_item_sk]
+                                            InputAdapter
+                                              BroadcastExchange #3
+                                                WholeStageCodegen (1)
+                                                  Project [i_item_sk,i_manufact_id]
+                                                    Filter [i_category,i_class,i_brand,i_item_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id]
+                                            Filter [ss_item_sk,ss_store_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #1
+                                                      ReusedExchange [d_date_sk,d_qoy] #4
+                                        InputAdapter
+                                          BroadcastExchange #5
+                                            WholeStageCodegen (2)
+                                              Filter [s_store_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.store [s_store_sk]
+                                    InputAdapter
+                                      BroadcastExchange #4
+                                        WholeStageCodegen (3)
+                                          Project [d_date_sk,d_qoy]
+                                            Filter [d_month_seq,d_date_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.date_dim [d_date_sk,d_month_seq,d_qoy]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/explain.txt
new file mode 100644
index 0000000000000..3bf06577e3212
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/explain.txt
@@ -0,0 +1,191 @@
+== Physical Plan ==
+TakeOrderedAndProject (32)
++- * Project (31)
+   +- * Filter (30)
+      +- Window (29)
+         +- * Sort (28)
+            +- Exchange (27)
+               +- * HashAggregate (26)
+                  +- Exchange (25)
+                     +- * HashAggregate (24)
+                        +- * Project (23)
+                           +- * BroadcastHashJoin Inner BuildRight (22)
+                              :- * Project (17)
+                              :  +- * BroadcastHashJoin Inner BuildRight (16)
+                              :     :- * Project (10)
+                              :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+                              :     :     :- * Project (4)
+                              :     :     :  +- * Filter (3)
+                              :     :     :     +- * ColumnarToRow (2)
+                              :     :     :        +- Scan parquet default.item (1)
+                              :     :     +- BroadcastExchange (8)
+                              :     :        +- * Filter (7)
+                              :     :           +- * ColumnarToRow (6)
+                              :     :              +- Scan parquet default.store_sales (5)
+                              :     +- BroadcastExchange (15)
+                              :        +- * Project (14)
+                              :           +- * Filter (13)
+                              :              +- * ColumnarToRow (12)
+                              :                 +- Scan parquet default.date_dim (11)
+                              +- BroadcastExchange (21)
+                                 +- * Filter (20)
+                                    +- * ColumnarToRow (19)
+                                       +- Scan parquet default.store (18)
+
+
+(1) Scan parquet default.item
+Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [Or(And(And(In(i_category, [Books                                             ,Children                                          ,Electronics                                       ]),In(i_class, [personal                                          ,portable                                          ,reference                                         ,self-help                                         ])),In(i_brand, [scholaramalgamalg #6                             ,scholaramalgamalg #7                              ,exportiunivamalg #8                               ,scholaramalgamalg #8                              ])),And(And(In(i_category, [Women                                             ,Music                                             ,Men                                               ]),In(i_class, [accessories                                       ,classical                                         ,fragrances                                        ,pants                                             ])),In(i_brand, [amalgimporto #9                                   ,edu packscholar #9                                ,exportiimporto #9                                 ,importoamalg #9                                   ]))), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_class:string,i_category:string,i_manufact_id:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5]
+
+(3) Filter [codegen id : 4]
+Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5]
+Condition : ((((i_category#4 IN (Books                                             ,Children                                          ,Electronics                                       ) AND i_class#3 IN (personal                                          ,portable                                          ,reference                                         ,self-help                                         )) AND i_brand#2 IN (scholaramalgamalg #6                             ,scholaramalgamalg #7                              ,exportiunivamalg #8                               ,scholaramalgamalg #8                              )) OR ((i_category#4 IN (Women                                             ,Music                                             ,Men                                               ) AND i_class#3 IN (accessories                                       ,classical                                         ,fragrances                                        ,pants                                             )) AND i_brand#2 IN (amalgimporto #9                                   ,edu packscholar #9                                ,exportiimporto #9                                 ,importoamalg #9                                   ))) AND isnotnull(i_item_sk#1))
+
+(4) Project [codegen id : 4]
+Output [2]: [i_item_sk#1, i_manufact_id#5]
+Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5]
+
+(5) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#13), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13]
+
+(7) Filter [codegen id : 1]
+Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13]
+Condition : (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11))
+
+(8) BroadcastExchange
+Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#15]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [ss_item_sk#10]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13]
+Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13]
+
+(11) Scan parquet default.date_dim
+Output [3]: [d_date_sk#16, d_month_seq#17, d_qoy#18]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [In(d_month_seq, [1200,1211,1205,1201,1206,1210,1207,1202,1209,1203,1208,1204]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int,d_qoy:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [d_date_sk#16, d_month_seq#17, d_qoy#18]
+
+(13) Filter [codegen id : 2]
+Input [3]: [d_date_sk#16, d_month_seq#17, d_qoy#18]
+Condition : (d_month_seq#17 INSET (1200,1211,1205,1201,1206,1210,1207,1202,1209,1203,1208,1204) AND isnotnull(d_date_sk#16))
+
+(14) Project [codegen id : 2]
+Output [2]: [d_date_sk#16, d_qoy#18]
+Input [3]: [d_date_sk#16, d_month_seq#17, d_qoy#18]
+
+(15) BroadcastExchange
+Input [2]: [d_date_sk#16, d_qoy#18]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#19]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#13]
+Right keys [1]: [d_date_sk#16]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#18]
+Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#16, d_qoy#18]
+
+(18) Scan parquet default.store
+Output [1]: [s_store_sk#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [1]: [s_store_sk#20]
+
+(20) Filter [codegen id : 3]
+Input [1]: [s_store_sk#20]
+Condition : isnotnull(s_store_sk#20)
+
+(21) BroadcastExchange
+Input [1]: [s_store_sk#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#21]
+
+(22) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#11]
+Right keys [1]: [s_store_sk#20]
+Join condition: None
+
+(23) Project [codegen id : 4]
+Output [3]: [i_manufact_id#5, ss_sales_price#12, d_qoy#18]
+Input [5]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#18, s_store_sk#20]
+
+(24) HashAggregate [codegen id : 4]
+Input [3]: [i_manufact_id#5, ss_sales_price#12, d_qoy#18]
+Keys [2]: [i_manufact_id#5, d_qoy#18]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))]
+Aggregate Attributes [1]: [sum#22]
+Results [3]: [i_manufact_id#5, d_qoy#18, sum#23]
+
+(25) Exchange
+Input [3]: [i_manufact_id#5, d_qoy#18, sum#23]
+Arguments: hashpartitioning(i_manufact_id#5, d_qoy#18, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(26) HashAggregate [codegen id : 5]
+Input [3]: [i_manufact_id#5, d_qoy#18, sum#23]
+Keys [2]: [i_manufact_id#5, d_qoy#18]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#12))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#25]
+Results [3]: [i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#25,17,2) AS sum_sales#26, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#25,17,2) AS _w0#27]
+
+(27) Exchange
+Input [3]: [i_manufact_id#5, sum_sales#26, _w0#27]
+Arguments: hashpartitioning(i_manufact_id#5, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(28) Sort [codegen id : 6]
+Input [3]: [i_manufact_id#5, sum_sales#26, _w0#27]
+Arguments: [i_manufact_id#5 ASC NULLS FIRST], false, 0
+
+(29) Window
+Input [3]: [i_manufact_id#5, sum_sales#26, _w0#27]
+Arguments: [avg(_w0#27) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#29], [i_manufact_id#5]
+
+(30) Filter [codegen id : 7]
+Input [4]: [i_manufact_id#5, sum_sales#26, _w0#27, avg_quarterly_sales#29]
+Condition : (isnotnull(avg_quarterly_sales#29) AND ((avg_quarterly_sales#29 > 0.000000) AND (CheckOverflow((promote_precision(abs(CheckOverflow((promote_precision(cast(sum_sales#26 as decimal(22,6))) - promote_precision(cast(avg_quarterly_sales#29 as decimal(22,6)))), DecimalType(22,6), true), false)) / promote_precision(cast(avg_quarterly_sales#29 as decimal(22,6)))), DecimalType(38,16), true) > 0.1000000000000000)))
+
+(31) Project [codegen id : 7]
+Output [3]: [i_manufact_id#5, sum_sales#26, avg_quarterly_sales#29]
+Input [4]: [i_manufact_id#5, sum_sales#26, _w0#27, avg_quarterly_sales#29]
+
+(32) TakeOrderedAndProject
+Input [3]: [i_manufact_id#5, sum_sales#26, avg_quarterly_sales#29]
+Arguments: 100, [avg_quarterly_sales#29 ASC NULLS FIRST, sum_sales#26 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#26, avg_quarterly_sales#29]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 5 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14
+ReusedExchange (33)
+
+
+(33) ReusedExchange [Reuses operator id: 15]
+Output [2]: [d_date_sk#16, d_qoy#18]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/simplified.txt
new file mode 100644
index 0000000000000..d67def8b728c1
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/simplified.txt
@@ -0,0 +1,51 @@
+TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id]
+  WholeStageCodegen (7)
+    Project [i_manufact_id,sum_sales,avg_quarterly_sales]
+      Filter [avg_quarterly_sales,sum_sales]
+        InputAdapter
+          Window [_w0,i_manufact_id]
+            WholeStageCodegen (6)
+              Sort [i_manufact_id]
+                InputAdapter
+                  Exchange [i_manufact_id] #1
+                    WholeStageCodegen (5)
+                      HashAggregate [i_manufact_id,d_qoy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum]
+                        InputAdapter
+                          Exchange [i_manufact_id,d_qoy] #2
+                            WholeStageCodegen (4)
+                              HashAggregate [i_manufact_id,d_qoy,ss_sales_price] [sum,sum]
+                                Project [i_manufact_id,ss_sales_price,d_qoy]
+                                  BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                    Project [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy]
+                                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                        Project [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                          BroadcastHashJoin [i_item_sk,ss_item_sk]
+                                            Project [i_item_sk,i_manufact_id]
+                                              Filter [i_category,i_class,i_brand,i_item_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id]
+                                            InputAdapter
+                                              BroadcastExchange #3
+                                                WholeStageCodegen (1)
+                                                  Filter [ss_item_sk,ss_store_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                                          SubqueryBroadcast [d_date_sk] #1
+                                                            ReusedExchange [d_date_sk,d_qoy] #4
+                                        InputAdapter
+                                          BroadcastExchange #4
+                                            WholeStageCodegen (2)
+                                              Project [d_date_sk,d_qoy]
+                                                Filter [d_month_seq,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_month_seq,d_qoy]
+                                    InputAdapter
+                                      BroadcastExchange #5
+                                        WholeStageCodegen (3)
+                                          Filter [s_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store [s_store_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.sf100/explain.txt
new file mode 100644
index 0000000000000..b149bdd3e1e3f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.sf100/explain.txt
@@ -0,0 +1,493 @@
+== Physical Plan ==
+TakeOrderedAndProject (67)
++- * HashAggregate (66)
+   +- Exchange (65)
+      +- * HashAggregate (64)
+         +- * HashAggregate (63)
+            +- * HashAggregate (62)
+               +- * Project (61)
+                  +- * SortMergeJoin Inner (60)
+                     :- * Sort (47)
+                     :  +- * Project (46)
+                     :     +- * BroadcastHashJoin Inner BuildLeft (45)
+                     :        :- BroadcastExchange (10)
+                     :        :  +- * Project (9)
+                     :        :     +- * BroadcastHashJoin Inner BuildRight (8)
+                     :        :        :- * Filter (3)
+                     :        :        :  +- * ColumnarToRow (2)
+                     :        :        :     +- Scan parquet default.customer_address (1)
+                     :        :        +- BroadcastExchange (7)
+                     :        :           +- * Filter (6)
+                     :        :              +- * ColumnarToRow (5)
+                     :        :                 +- Scan parquet default.store (4)
+                     :        +- * HashAggregate (44)
+                     :           +- * HashAggregate (43)
+                     :              +- * Project (42)
+                     :                 +- * SortMergeJoin Inner (41)
+                     :                    :- * Sort (35)
+                     :                    :  +- Exchange (34)
+                     :                    :     +- * Project (33)
+                     :                    :        +- * BroadcastHashJoin Inner BuildRight (32)
+                     :                    :           :- * Project (26)
+                     :                    :           :  +- * BroadcastHashJoin Inner BuildRight (25)
+                     :                    :           :     :- Union (19)
+                     :                    :           :     :  :- * Project (14)
+                     :                    :           :     :  :  +- * Filter (13)
+                     :                    :           :     :  :     +- * ColumnarToRow (12)
+                     :                    :           :     :  :        +- Scan parquet default.catalog_sales (11)
+                     :                    :           :     :  +- * Project (18)
+                     :                    :           :     :     +- * Filter (17)
+                     :                    :           :     :        +- * ColumnarToRow (16)
+                     :                    :           :     :           +- Scan parquet default.web_sales (15)
+                     :                    :           :     +- BroadcastExchange (24)
+                     :                    :           :        +- * Project (23)
+                     :                    :           :           +- * Filter (22)
+                     :                    :           :              +- * ColumnarToRow (21)
+                     :                    :           :                 +- Scan parquet default.date_dim (20)
+                     :                    :           +- BroadcastExchange (31)
+                     :                    :              +- * Project (30)
+                     :                    :                 +- * Filter (29)
+                     :                    :                    +- * ColumnarToRow (28)
+                     :                    :                       +- Scan parquet default.item (27)
+                     :                    +- * Sort (40)
+                     :                       +- Exchange (39)
+                     :                          +- * Filter (38)
+                     :                             +- * ColumnarToRow (37)
+                     :                                +- Scan parquet default.customer (36)
+                     +- * Sort (59)
+                        +- Exchange (58)
+                           +- * Project (57)
+                              +- * BroadcastHashJoin Inner BuildRight (56)
+                                 :- * Filter (50)
+                                 :  +- * ColumnarToRow (49)
+                                 :     +- Scan parquet default.store_sales (48)
+                                 +- BroadcastExchange (55)
+                                    +- * Project (54)
+                                       +- * Filter (53)
+                                          +- * ColumnarToRow (52)
+                                             +- Scan parquet default.date_dim (51)
+
+
+(1) Scan parquet default.customer_address
+Output [3]: [ca_address_sk#1, ca_county#2, ca_state#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county), IsNotNull(ca_state)]
+ReadSchema: struct<ca_address_sk:int,ca_county:string,ca_state:string>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [3]: [ca_address_sk#1, ca_county#2, ca_state#3]
+
+(3) Filter [codegen id : 2]
+Input [3]: [ca_address_sk#1, ca_county#2, ca_state#3]
+Condition : ((isnotnull(ca_address_sk#1) AND isnotnull(ca_county#2)) AND isnotnull(ca_state#3))
+
+(4) Scan parquet default.store
+Output [2]: [s_county#4, s_state#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_county), IsNotNull(s_state)]
+ReadSchema: struct<s_county:string,s_state:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [s_county#4, s_state#5]
+
+(6) Filter [codegen id : 1]
+Input [2]: [s_county#4, s_state#5]
+Condition : (isnotnull(s_county#4) AND isnotnull(s_state#5))
+
+(7) BroadcastExchange
+Input [2]: [s_county#4, s_state#5]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [id=#6]
+
+(8) BroadcastHashJoin [codegen id : 2]
+Left keys [2]: [ca_county#2, ca_state#3]
+Right keys [2]: [s_county#4, s_state#5]
+Join condition: None
+
+(9) Project [codegen id : 2]
+Output [1]: [ca_address_sk#1]
+Input [5]: [ca_address_sk#1, ca_county#2, ca_state#3, s_county#4, s_state#5]
+
+(10) BroadcastExchange
+Input [1]: [ca_address_sk#1]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7]
+
+(11) Scan parquet default.catalog_sales
+Output [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#10), dynamicpruningexpression(cs_sold_date_sk#10 IN dynamicpruning#11)]
+PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_item_sk:int>
+
+(12) ColumnarToRow [codegen id : 3]
+Input [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10]
+
+(13) Filter [codegen id : 3]
+Input [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10]
+Condition : (isnotnull(cs_item_sk#9) AND isnotnull(cs_bill_customer_sk#8))
+
+(14) Project [codegen id : 3]
+Output [3]: [cs_sold_date_sk#10 AS sold_date_sk#12, cs_bill_customer_sk#8 AS customer_sk#13, cs_item_sk#9 AS item_sk#14]
+Input [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10]
+
+(15) Scan parquet default.web_sales
+Output [3]: [ws_item_sk#15, ws_bill_customer_sk#16, ws_sold_date_sk#17]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#17), dynamicpruningexpression(ws_sold_date_sk#17 IN dynamicpruning#11)]
+PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_bill_customer_sk:int>
+
+(16) ColumnarToRow [codegen id : 4]
+Input [3]: [ws_item_sk#15, ws_bill_customer_sk#16, ws_sold_date_sk#17]
+
+(17) Filter [codegen id : 4]
+Input [3]: [ws_item_sk#15, ws_bill_customer_sk#16, ws_sold_date_sk#17]
+Condition : (isnotnull(ws_item_sk#15) AND isnotnull(ws_bill_customer_sk#16))
+
+(18) Project [codegen id : 4]
+Output [3]: [ws_sold_date_sk#17 AS sold_date_sk#18, ws_bill_customer_sk#16 AS customer_sk#19, ws_item_sk#15 AS item_sk#20]
+Input [3]: [ws_item_sk#15, ws_bill_customer_sk#16, ws_sold_date_sk#17]
+
+(19) Union
+
+(20) Scan parquet default.date_dim
+Output [3]: [d_date_sk#21, d_year#22, d_moy#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,12), EqualTo(d_year,1998), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(21) ColumnarToRow [codegen id : 5]
+Input [3]: [d_date_sk#21, d_year#22, d_moy#23]
+
+(22) Filter [codegen id : 5]
+Input [3]: [d_date_sk#21, d_year#22, d_moy#23]
+Condition : ((((isnotnull(d_moy#23) AND isnotnull(d_year#22)) AND (d_moy#23 = 12)) AND (d_year#22 = 1998)) AND isnotnull(d_date_sk#21))
+
+(23) Project [codegen id : 5]
+Output [1]: [d_date_sk#21]
+Input [3]: [d_date_sk#21, d_year#22, d_moy#23]
+
+(24) BroadcastExchange
+Input [1]: [d_date_sk#21]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#24]
+
+(25) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [sold_date_sk#12]
+Right keys [1]: [d_date_sk#21]
+Join condition: None
+
+(26) Project [codegen id : 7]
+Output [2]: [customer_sk#13, item_sk#14]
+Input [4]: [sold_date_sk#12, customer_sk#13, item_sk#14, d_date_sk#21]
+
+(27) Scan parquet default.item
+Output [3]: [i_item_sk#25, i_class#26, i_category#27]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_category), IsNotNull(i_class), EqualTo(i_category,Women                                             ), EqualTo(i_class,maternity                                         ), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_class:string,i_category:string>
+
+(28) ColumnarToRow [codegen id : 6]
+Input [3]: [i_item_sk#25, i_class#26, i_category#27]
+
+(29) Filter [codegen id : 6]
+Input [3]: [i_item_sk#25, i_class#26, i_category#27]
+Condition : ((((isnotnull(i_category#27) AND isnotnull(i_class#26)) AND (i_category#27 = Women                                             )) AND (i_class#26 = maternity                                         )) AND isnotnull(i_item_sk#25))
+
+(30) Project [codegen id : 6]
+Output [1]: [i_item_sk#25]
+Input [3]: [i_item_sk#25, i_class#26, i_category#27]
+
+(31) BroadcastExchange
+Input [1]: [i_item_sk#25]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#28]
+
+(32) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [item_sk#14]
+Right keys [1]: [i_item_sk#25]
+Join condition: None
+
+(33) Project [codegen id : 7]
+Output [1]: [customer_sk#13]
+Input [3]: [customer_sk#13, item_sk#14, i_item_sk#25]
+
+(34) Exchange
+Input [1]: [customer_sk#13]
+Arguments: hashpartitioning(customer_sk#13, 5), ENSURE_REQUIREMENTS, [id=#29]
+
+(35) Sort [codegen id : 8]
+Input [1]: [customer_sk#13]
+Arguments: [customer_sk#13 ASC NULLS FIRST], false, 0
+
+(36) Scan parquet default.customer
+Output [2]: [c_customer_sk#30, c_current_addr_sk#31]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int>
+
+(37) ColumnarToRow [codegen id : 9]
+Input [2]: [c_customer_sk#30, c_current_addr_sk#31]
+
+(38) Filter [codegen id : 9]
+Input [2]: [c_customer_sk#30, c_current_addr_sk#31]
+Condition : (isnotnull(c_customer_sk#30) AND isnotnull(c_current_addr_sk#31))
+
+(39) Exchange
+Input [2]: [c_customer_sk#30, c_current_addr_sk#31]
+Arguments: hashpartitioning(c_customer_sk#30, 5), ENSURE_REQUIREMENTS, [id=#32]
+
+(40) Sort [codegen id : 10]
+Input [2]: [c_customer_sk#30, c_current_addr_sk#31]
+Arguments: [c_customer_sk#30 ASC NULLS FIRST], false, 0
+
+(41) SortMergeJoin
+Left keys [1]: [customer_sk#13]
+Right keys [1]: [c_customer_sk#30]
+Join condition: None
+
+(42) Project
+Output [2]: [c_customer_sk#30, c_current_addr_sk#31]
+Input [3]: [customer_sk#13, c_customer_sk#30, c_current_addr_sk#31]
+
+(43) HashAggregate
+Input [2]: [c_customer_sk#30, c_current_addr_sk#31]
+Keys [2]: [c_customer_sk#30, c_current_addr_sk#31]
+Functions: []
+Aggregate Attributes: []
+Results [2]: [c_customer_sk#30, c_current_addr_sk#31]
+
+(44) HashAggregate
+Input [2]: [c_customer_sk#30, c_current_addr_sk#31]
+Keys [2]: [c_customer_sk#30, c_current_addr_sk#31]
+Functions: []
+Aggregate Attributes: []
+Results [2]: [c_customer_sk#30, c_current_addr_sk#31]
+
+(45) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ca_address_sk#1]
+Right keys [1]: [c_current_addr_sk#31]
+Join condition: None
+
+(46) Project [codegen id : 11]
+Output [1]: [c_customer_sk#30]
+Input [3]: [ca_address_sk#1, c_customer_sk#30, c_current_addr_sk#31]
+
+(47) Sort [codegen id : 11]
+Input [1]: [c_customer_sk#30]
+Arguments: [c_customer_sk#30 ASC NULLS FIRST], false, 0
+
+(48) Scan parquet default.store_sales
+Output [3]: [ss_customer_sk#33, ss_ext_sales_price#34, ss_sold_date_sk#35]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#35), dynamicpruningexpression(ss_sold_date_sk#35 IN dynamicpruning#36)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(49) ColumnarToRow [codegen id : 13]
+Input [3]: [ss_customer_sk#33, ss_ext_sales_price#34, ss_sold_date_sk#35]
+
+(50) Filter [codegen id : 13]
+Input [3]: [ss_customer_sk#33, ss_ext_sales_price#34, ss_sold_date_sk#35]
+Condition : isnotnull(ss_customer_sk#33)
+
+(51) Scan parquet default.date_dim
+Output [2]: [d_date_sk#37, d_month_seq#38]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(52) ColumnarToRow [codegen id : 12]
+Input [2]: [d_date_sk#37, d_month_seq#38]
+
+(53) Filter [codegen id : 12]
+Input [2]: [d_date_sk#37, d_month_seq#38]
+Condition : (((isnotnull(d_month_seq#38) AND (d_month_seq#38 >= Subquery scalar-subquery#39, [id=#40])) AND (d_month_seq#38 <= Subquery scalar-subquery#41, [id=#42])) AND isnotnull(d_date_sk#37))
+
+(54) Project [codegen id : 12]
+Output [1]: [d_date_sk#37]
+Input [2]: [d_date_sk#37, d_month_seq#38]
+
+(55) BroadcastExchange
+Input [1]: [d_date_sk#37]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#43]
+
+(56) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ss_sold_date_sk#35]
+Right keys [1]: [d_date_sk#37]
+Join condition: None
+
+(57) Project [codegen id : 13]
+Output [2]: [ss_customer_sk#33, ss_ext_sales_price#34]
+Input [4]: [ss_customer_sk#33, ss_ext_sales_price#34, ss_sold_date_sk#35, d_date_sk#37]
+
+(58) Exchange
+Input [2]: [ss_customer_sk#33, ss_ext_sales_price#34]
+Arguments: hashpartitioning(ss_customer_sk#33, 5), ENSURE_REQUIREMENTS, [id=#44]
+
+(59) Sort [codegen id : 14]
+Input [2]: [ss_customer_sk#33, ss_ext_sales_price#34]
+Arguments: [ss_customer_sk#33 ASC NULLS FIRST], false, 0
+
+(60) SortMergeJoin [codegen id : 15]
+Left keys [1]: [c_customer_sk#30]
+Right keys [1]: [ss_customer_sk#33]
+Join condition: None
+
+(61) Project [codegen id : 15]
+Output [2]: [c_customer_sk#30, ss_ext_sales_price#34]
+Input [3]: [c_customer_sk#30, ss_customer_sk#33, ss_ext_sales_price#34]
+
+(62) HashAggregate [codegen id : 15]
+Input [2]: [c_customer_sk#30, ss_ext_sales_price#34]
+Keys [1]: [c_customer_sk#30]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#34))]
+Aggregate Attributes [1]: [sum#45]
+Results [2]: [c_customer_sk#30, sum#46]
+
+(63) HashAggregate [codegen id : 15]
+Input [2]: [c_customer_sk#30, sum#46]
+Keys [1]: [c_customer_sk#30]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#34))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#34))#47]
+Results [1]: [cast(CheckOverflow((promote_precision(MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#34))#47,17,2)) / 50.00), DecimalType(21,6), true) as int) AS segment#48]
+
+(64) HashAggregate [codegen id : 15]
+Input [1]: [segment#48]
+Keys [1]: [segment#48]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#49]
+Results [2]: [segment#48, count#50]
+
+(65) Exchange
+Input [2]: [segment#48, count#50]
+Arguments: hashpartitioning(segment#48, 5), ENSURE_REQUIREMENTS, [id=#51]
+
+(66) HashAggregate [codegen id : 16]
+Input [2]: [segment#48, count#50]
+Keys [1]: [segment#48]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#52]
+Results [3]: [segment#48, count(1)#52 AS num_customers#53, (segment#48 * 50) AS segment_base#54]
+
+(67) TakeOrderedAndProject
+Input [3]: [segment#48, num_customers#53, segment_base#54]
+Arguments: 100, [segment#48 ASC NULLS FIRST, num_customers#53 ASC NULLS FIRST], [segment#48, num_customers#53, segment_base#54]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 11 Hosting Expression = cs_sold_date_sk#10 IN dynamicpruning#11
+ReusedExchange (68)
+
+
+(68) ReusedExchange [Reuses operator id: 24]
+Output [1]: [d_date_sk#21]
+
+Subquery:2 Hosting operator id = 15 Hosting Expression = ws_sold_date_sk#17 IN dynamicpruning#11
+
+Subquery:3 Hosting operator id = 48 Hosting Expression = ss_sold_date_sk#35 IN dynamicpruning#36
+ReusedExchange (69)
+
+
+(69) ReusedExchange [Reuses operator id: 55]
+Output [1]: [d_date_sk#37]
+
+Subquery:4 Hosting operator id = 53 Hosting Expression = Subquery scalar-subquery#39, [id=#40]
+* HashAggregate (76)
++- Exchange (75)
+   +- * HashAggregate (74)
+      +- * Project (73)
+         +- * Filter (72)
+            +- * ColumnarToRow (71)
+               +- Scan parquet default.date_dim (70)
+
+
+(70) Scan parquet default.date_dim
+Output [3]: [d_month_seq#55, d_year#56, d_moy#57]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)]
+ReadSchema: struct<d_month_seq:int,d_year:int,d_moy:int>
+
+(71) ColumnarToRow [codegen id : 1]
+Input [3]: [d_month_seq#55, d_year#56, d_moy#57]
+
+(72) Filter [codegen id : 1]
+Input [3]: [d_month_seq#55, d_year#56, d_moy#57]
+Condition : (((isnotnull(d_year#56) AND isnotnull(d_moy#57)) AND (d_year#56 = 1998)) AND (d_moy#57 = 12))
+
+(73) Project [codegen id : 1]
+Output [1]: [(d_month_seq#55 + 1) AS (d_month_seq + 1)#58]
+Input [3]: [d_month_seq#55, d_year#56, d_moy#57]
+
+(74) HashAggregate [codegen id : 1]
+Input [1]: [(d_month_seq + 1)#58]
+Keys [1]: [(d_month_seq + 1)#58]
+Functions: []
+Aggregate Attributes: []
+Results [1]: [(d_month_seq + 1)#58]
+
+(75) Exchange
+Input [1]: [(d_month_seq + 1)#58]
+Arguments: hashpartitioning((d_month_seq + 1)#58, 5), ENSURE_REQUIREMENTS, [id=#59]
+
+(76) HashAggregate [codegen id : 2]
+Input [1]: [(d_month_seq + 1)#58]
+Keys [1]: [(d_month_seq + 1)#58]
+Functions: []
+Aggregate Attributes: []
+Results [1]: [(d_month_seq + 1)#58]
+
+Subquery:5 Hosting operator id = 53 Hosting Expression = Subquery scalar-subquery#41, [id=#42]
+* HashAggregate (83)
++- Exchange (82)
+   +- * HashAggregate (81)
+      +- * Project (80)
+         +- * Filter (79)
+            +- * ColumnarToRow (78)
+               +- Scan parquet default.date_dim (77)
+
+
+(77) Scan parquet default.date_dim
+Output [3]: [d_month_seq#60, d_year#61, d_moy#62]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)]
+ReadSchema: struct<d_month_seq:int,d_year:int,d_moy:int>
+
+(78) ColumnarToRow [codegen id : 1]
+Input [3]: [d_month_seq#60, d_year#61, d_moy#62]
+
+(79) Filter [codegen id : 1]
+Input [3]: [d_month_seq#60, d_year#61, d_moy#62]
+Condition : (((isnotnull(d_year#61) AND isnotnull(d_moy#62)) AND (d_year#61 = 1998)) AND (d_moy#62 = 12))
+
+(80) Project [codegen id : 1]
+Output [1]: [(d_month_seq#60 + 3) AS (d_month_seq + 3)#63]
+Input [3]: [d_month_seq#60, d_year#61, d_moy#62]
+
+(81) HashAggregate [codegen id : 1]
+Input [1]: [(d_month_seq + 3)#63]
+Keys [1]: [(d_month_seq + 3)#63]
+Functions: []
+Aggregate Attributes: []
+Results [1]: [(d_month_seq + 3)#63]
+
+(82) Exchange
+Input [1]: [(d_month_seq + 3)#63]
+Arguments: hashpartitioning((d_month_seq + 3)#63, 5), ENSURE_REQUIREMENTS, [id=#64]
+
+(83) HashAggregate [codegen id : 2]
+Input [1]: [(d_month_seq + 3)#63]
+Keys [1]: [(d_month_seq + 3)#63]
+Functions: []
+Aggregate Attributes: []
+Results [1]: [(d_month_seq + 3)#63]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.sf100/simplified.txt
new file mode 100644
index 0000000000000..b800afb9a4c65
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.sf100/simplified.txt
@@ -0,0 +1,135 @@
+TakeOrderedAndProject [segment,num_customers,segment_base]
+  WholeStageCodegen (16)
+    HashAggregate [segment,count] [count(1),num_customers,segment_base,count]
+      InputAdapter
+        Exchange [segment] #1
+          WholeStageCodegen (15)
+            HashAggregate [segment] [count,count]
+              HashAggregate [c_customer_sk,sum] [sum(UnscaledValue(ss_ext_sales_price)),segment,sum]
+                HashAggregate [c_customer_sk,ss_ext_sales_price] [sum,sum]
+                  Project [c_customer_sk,ss_ext_sales_price]
+                    SortMergeJoin [c_customer_sk,ss_customer_sk]
+                      InputAdapter
+                        WholeStageCodegen (11)
+                          Sort [c_customer_sk]
+                            Project [c_customer_sk]
+                              BroadcastHashJoin [ca_address_sk,c_current_addr_sk]
+                                InputAdapter
+                                  BroadcastExchange #2
+                                    WholeStageCodegen (2)
+                                      Project [ca_address_sk]
+                                        BroadcastHashJoin [ca_county,ca_state,s_county,s_state]
+                                          Filter [ca_address_sk,ca_county,ca_state]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.customer_address [ca_address_sk,ca_county,ca_state]
+                                          InputAdapter
+                                            BroadcastExchange #3
+                                              WholeStageCodegen (1)
+                                                Filter [s_county,s_state]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.store [s_county,s_state]
+                                HashAggregate [c_customer_sk,c_current_addr_sk]
+                                  HashAggregate [c_customer_sk,c_current_addr_sk]
+                                    Project [c_customer_sk,c_current_addr_sk]
+                                      SortMergeJoin [customer_sk,c_customer_sk]
+                                        InputAdapter
+                                          WholeStageCodegen (8)
+                                            Sort [customer_sk]
+                                              InputAdapter
+                                                Exchange [customer_sk] #4
+                                                  WholeStageCodegen (7)
+                                                    Project [customer_sk]
+                                                      BroadcastHashJoin [item_sk,i_item_sk]
+                                                        Project [customer_sk,item_sk]
+                                                          BroadcastHashJoin [sold_date_sk,d_date_sk]
+                                                            InputAdapter
+                                                              Union
+                                                                WholeStageCodegen (3)
+                                                                  Project [cs_sold_date_sk,cs_bill_customer_sk,cs_item_sk]
+                                                                    Filter [cs_item_sk,cs_bill_customer_sk]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk]
+                                                                            SubqueryBroadcast [d_date_sk] #1
+                                                                              ReusedExchange [d_date_sk] #5
+                                                                WholeStageCodegen (4)
+                                                                  Project [ws_sold_date_sk,ws_bill_customer_sk,ws_item_sk]
+                                                                    Filter [ws_item_sk,ws_bill_customer_sk]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk]
+                                                                            ReusedSubquery [d_date_sk] #1
+                                                            InputAdapter
+                                                              BroadcastExchange #5
+                                                                WholeStageCodegen (5)
+                                                                  Project [d_date_sk]
+                                                                    Filter [d_moy,d_year,d_date_sk]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                                        InputAdapter
+                                                          BroadcastExchange #6
+                                                            WholeStageCodegen (6)
+                                                              Project [i_item_sk]
+                                                                Filter [i_category,i_class,i_item_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.item [i_item_sk,i_class,i_category]
+                                        InputAdapter
+                                          WholeStageCodegen (10)
+                                            Sort [c_customer_sk]
+                                              InputAdapter
+                                                Exchange [c_customer_sk] #7
+                                                  WholeStageCodegen (9)
+                                                    Filter [c_customer_sk,c_current_addr_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.customer [c_customer_sk,c_current_addr_sk]
+                      InputAdapter
+                        WholeStageCodegen (14)
+                          Sort [ss_customer_sk]
+                            InputAdapter
+                              Exchange [ss_customer_sk] #8
+                                WholeStageCodegen (13)
+                                  Project [ss_customer_sk,ss_ext_sales_price]
+                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                      Filter [ss_customer_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store_sales [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #2
+                                                ReusedExchange [d_date_sk] #9
+                                      InputAdapter
+                                        BroadcastExchange #9
+                                          WholeStageCodegen (12)
+                                            Project [d_date_sk]
+                                              Filter [d_month_seq,d_date_sk]
+                                                Subquery #3
+                                                  WholeStageCodegen (2)
+                                                    HashAggregate [(d_month_seq + 1)]
+                                                      InputAdapter
+                                                        Exchange [(d_month_seq + 1)] #10
+                                                          WholeStageCodegen (1)
+                                                            HashAggregate [(d_month_seq + 1)]
+                                                              Project [d_month_seq]
+                                                                Filter [d_year,d_moy]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.date_dim [d_month_seq,d_year,d_moy]
+                                                Subquery #4
+                                                  WholeStageCodegen (2)
+                                                    HashAggregate [(d_month_seq + 3)]
+                                                      InputAdapter
+                                                        Exchange [(d_month_seq + 3)] #11
+                                                          WholeStageCodegen (1)
+                                                            HashAggregate [(d_month_seq + 3)]
+                                                              Project [d_month_seq]
+                                                                Filter [d_year,d_moy]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.date_dim [d_month_seq,d_year,d_moy]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_date_sk,d_month_seq]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt
new file mode 100644
index 0000000000000..d0638d7606bb5
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt
@@ -0,0 +1,478 @@
+== Physical Plan ==
+TakeOrderedAndProject (64)
++- * HashAggregate (63)
+   +- Exchange (62)
+      +- * HashAggregate (61)
+         +- * HashAggregate (60)
+            +- Exchange (59)
+               +- * HashAggregate (58)
+                  +- * Project (57)
+                     +- * BroadcastHashJoin Inner BuildRight (56)
+                        :- * Project (50)
+                        :  +- * BroadcastHashJoin Inner BuildRight (49)
+                        :     :- * Project (44)
+                        :     :  +- * BroadcastHashJoin Inner BuildRight (43)
+                        :     :     :- * Project (38)
+                        :     :     :  +- * BroadcastHashJoin Inner BuildRight (37)
+                        :     :     :     :- * HashAggregate (32)
+                        :     :     :     :  +- Exchange (31)
+                        :     :     :     :     +- * HashAggregate (30)
+                        :     :     :     :        +- * Project (29)
+                        :     :     :     :           +- * BroadcastHashJoin Inner BuildRight (28)
+                        :     :     :     :              :- * Project (23)
+                        :     :     :     :              :  +- * BroadcastHashJoin Inner BuildRight (22)
+                        :     :     :     :              :     :- * Project (16)
+                        :     :     :     :              :     :  +- * BroadcastHashJoin Inner BuildRight (15)
+                        :     :     :     :              :     :     :- Union (9)
+                        :     :     :     :              :     :     :  :- * Project (4)
+                        :     :     :     :              :     :     :  :  +- * Filter (3)
+                        :     :     :     :              :     :     :  :     +- * ColumnarToRow (2)
+                        :     :     :     :              :     :     :  :        +- Scan parquet default.catalog_sales (1)
+                        :     :     :     :              :     :     :  +- * Project (8)
+                        :     :     :     :              :     :     :     +- * Filter (7)
+                        :     :     :     :              :     :     :        +- * ColumnarToRow (6)
+                        :     :     :     :              :     :     :           +- Scan parquet default.web_sales (5)
+                        :     :     :     :              :     :     +- BroadcastExchange (14)
+                        :     :     :     :              :     :        +- * Project (13)
+                        :     :     :     :              :     :           +- * Filter (12)
+                        :     :     :     :              :     :              +- * ColumnarToRow (11)
+                        :     :     :     :              :     :                 +- Scan parquet default.item (10)
+                        :     :     :     :              :     +- BroadcastExchange (21)
+                        :     :     :     :              :        +- * Project (20)
+                        :     :     :     :              :           +- * Filter (19)
+                        :     :     :     :              :              +- * ColumnarToRow (18)
+                        :     :     :     :              :                 +- Scan parquet default.date_dim (17)
+                        :     :     :     :              +- BroadcastExchange (27)
+                        :     :     :     :                 +- * Filter (26)
+                        :     :     :     :                    +- * ColumnarToRow (25)
+                        :     :     :     :                       +- Scan parquet default.customer (24)
+                        :     :     :     +- BroadcastExchange (36)
+                        :     :     :        +- * Filter (35)
+                        :     :     :           +- * ColumnarToRow (34)
+                        :     :     :              +- Scan parquet default.store_sales (33)
+                        :     :     +- BroadcastExchange (42)
+                        :     :        +- * Filter (41)
+                        :     :           +- * ColumnarToRow (40)
+                        :     :              +- Scan parquet default.customer_address (39)
+                        :     +- BroadcastExchange (48)
+                        :        +- * Filter (47)
+                        :           +- * ColumnarToRow (46)
+                        :              +- Scan parquet default.store (45)
+                        +- BroadcastExchange (55)
+                           +- * Project (54)
+                              +- * Filter (53)
+                                 +- * ColumnarToRow (52)
+                                    +- Scan parquet default.date_dim (51)
+
+
+(1) Scan parquet default.catalog_sales
+Output [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_item_sk:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3]
+
+(3) Filter [codegen id : 1]
+Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3]
+Condition : (isnotnull(cs_item_sk#2) AND isnotnull(cs_bill_customer_sk#1))
+
+(4) Project [codegen id : 1]
+Output [3]: [cs_sold_date_sk#3 AS sold_date_sk#5, cs_bill_customer_sk#1 AS customer_sk#6, cs_item_sk#2 AS item_sk#7]
+Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3]
+
+(5) Scan parquet default.web_sales
+Output [3]: [ws_item_sk#8, ws_bill_customer_sk#9, ws_sold_date_sk#10]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#10), dynamicpruningexpression(ws_sold_date_sk#10 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_bill_customer_sk:int>
+
+(6) ColumnarToRow [codegen id : 2]
+Input [3]: [ws_item_sk#8, ws_bill_customer_sk#9, ws_sold_date_sk#10]
+
+(7) Filter [codegen id : 2]
+Input [3]: [ws_item_sk#8, ws_bill_customer_sk#9, ws_sold_date_sk#10]
+Condition : (isnotnull(ws_item_sk#8) AND isnotnull(ws_bill_customer_sk#9))
+
+(8) Project [codegen id : 2]
+Output [3]: [ws_sold_date_sk#10 AS sold_date_sk#11, ws_bill_customer_sk#9 AS customer_sk#12, ws_item_sk#8 AS item_sk#13]
+Input [3]: [ws_item_sk#8, ws_bill_customer_sk#9, ws_sold_date_sk#10]
+
+(9) Union
+
+(10) Scan parquet default.item
+Output [3]: [i_item_sk#14, i_class#15, i_category#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_category), IsNotNull(i_class), EqualTo(i_category,Women                                             ), EqualTo(i_class,maternity                                         ), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_class:string,i_category:string>
+
+(11) ColumnarToRow [codegen id : 3]
+Input [3]: [i_item_sk#14, i_class#15, i_category#16]
+
+(12) Filter [codegen id : 3]
+Input [3]: [i_item_sk#14, i_class#15, i_category#16]
+Condition : ((((isnotnull(i_category#16) AND isnotnull(i_class#15)) AND (i_category#16 = Women                                             )) AND (i_class#15 = maternity                                         )) AND isnotnull(i_item_sk#14))
+
+(13) Project [codegen id : 3]
+Output [1]: [i_item_sk#14]
+Input [3]: [i_item_sk#14, i_class#15, i_category#16]
+
+(14) BroadcastExchange
+Input [1]: [i_item_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(15) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [item_sk#7]
+Right keys [1]: [i_item_sk#14]
+Join condition: None
+
+(16) Project [codegen id : 6]
+Output [2]: [sold_date_sk#5, customer_sk#6]
+Input [4]: [sold_date_sk#5, customer_sk#6, item_sk#7, i_item_sk#14]
+
+(17) Scan parquet default.date_dim
+Output [3]: [d_date_sk#18, d_year#19, d_moy#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,12), EqualTo(d_year,1998), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(18) ColumnarToRow [codegen id : 4]
+Input [3]: [d_date_sk#18, d_year#19, d_moy#20]
+
+(19) Filter [codegen id : 4]
+Input [3]: [d_date_sk#18, d_year#19, d_moy#20]
+Condition : ((((isnotnull(d_moy#20) AND isnotnull(d_year#19)) AND (d_moy#20 = 12)) AND (d_year#19 = 1998)) AND isnotnull(d_date_sk#18))
+
+(20) Project [codegen id : 4]
+Output [1]: [d_date_sk#18]
+Input [3]: [d_date_sk#18, d_year#19, d_moy#20]
+
+(21) BroadcastExchange
+Input [1]: [d_date_sk#18]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(22) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [sold_date_sk#5]
+Right keys [1]: [d_date_sk#18]
+Join condition: None
+
+(23) Project [codegen id : 6]
+Output [1]: [customer_sk#6]
+Input [3]: [sold_date_sk#5, customer_sk#6, d_date_sk#18]
+
+(24) Scan parquet default.customer
+Output [2]: [c_customer_sk#22, c_current_addr_sk#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int>
+
+(25) ColumnarToRow [codegen id : 5]
+Input [2]: [c_customer_sk#22, c_current_addr_sk#23]
+
+(26) Filter [codegen id : 5]
+Input [2]: [c_customer_sk#22, c_current_addr_sk#23]
+Condition : (isnotnull(c_customer_sk#22) AND isnotnull(c_current_addr_sk#23))
+
+(27) BroadcastExchange
+Input [2]: [c_customer_sk#22, c_current_addr_sk#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#24]
+
+(28) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [customer_sk#6]
+Right keys [1]: [c_customer_sk#22]
+Join condition: None
+
+(29) Project [codegen id : 6]
+Output [2]: [c_customer_sk#22, c_current_addr_sk#23]
+Input [3]: [customer_sk#6, c_customer_sk#22, c_current_addr_sk#23]
+
+(30) HashAggregate [codegen id : 6]
+Input [2]: [c_customer_sk#22, c_current_addr_sk#23]
+Keys [2]: [c_customer_sk#22, c_current_addr_sk#23]
+Functions: []
+Aggregate Attributes: []
+Results [2]: [c_customer_sk#22, c_current_addr_sk#23]
+
+(31) Exchange
+Input [2]: [c_customer_sk#22, c_current_addr_sk#23]
+Arguments: hashpartitioning(c_customer_sk#22, c_current_addr_sk#23, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(32) HashAggregate [codegen id : 11]
+Input [2]: [c_customer_sk#22, c_current_addr_sk#23]
+Keys [2]: [c_customer_sk#22, c_current_addr_sk#23]
+Functions: []
+Aggregate Attributes: []
+Results [2]: [c_customer_sk#22, c_current_addr_sk#23]
+
+(33) Scan parquet default.store_sales
+Output [3]: [ss_customer_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#28), dynamicpruningexpression(ss_sold_date_sk#28 IN dynamicpruning#29)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(34) ColumnarToRow [codegen id : 7]
+Input [3]: [ss_customer_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28]
+
+(35) Filter [codegen id : 7]
+Input [3]: [ss_customer_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28]
+Condition : isnotnull(ss_customer_sk#26)
+
+(36) BroadcastExchange
+Input [3]: [ss_customer_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#30]
+
+(37) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [c_customer_sk#22]
+Right keys [1]: [ss_customer_sk#26]
+Join condition: None
+
+(38) Project [codegen id : 11]
+Output [4]: [c_customer_sk#22, c_current_addr_sk#23, ss_ext_sales_price#27, ss_sold_date_sk#28]
+Input [5]: [c_customer_sk#22, c_current_addr_sk#23, ss_customer_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28]
+
+(39) Scan parquet default.customer_address
+Output [3]: [ca_address_sk#31, ca_county#32, ca_state#33]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county), IsNotNull(ca_state)]
+ReadSchema: struct<ca_address_sk:int,ca_county:string,ca_state:string>
+
+(40) ColumnarToRow [codegen id : 8]
+Input [3]: [ca_address_sk#31, ca_county#32, ca_state#33]
+
+(41) Filter [codegen id : 8]
+Input [3]: [ca_address_sk#31, ca_county#32, ca_state#33]
+Condition : ((isnotnull(ca_address_sk#31) AND isnotnull(ca_county#32)) AND isnotnull(ca_state#33))
+
+(42) BroadcastExchange
+Input [3]: [ca_address_sk#31, ca_county#32, ca_state#33]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#34]
+
+(43) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [c_current_addr_sk#23]
+Right keys [1]: [ca_address_sk#31]
+Join condition: None
+
+(44) Project [codegen id : 11]
+Output [5]: [c_customer_sk#22, ss_ext_sales_price#27, ss_sold_date_sk#28, ca_county#32, ca_state#33]
+Input [7]: [c_customer_sk#22, c_current_addr_sk#23, ss_ext_sales_price#27, ss_sold_date_sk#28, ca_address_sk#31, ca_county#32, ca_state#33]
+
+(45) Scan parquet default.store
+Output [2]: [s_county#35, s_state#36]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_county), IsNotNull(s_state)]
+ReadSchema: struct<s_county:string,s_state:string>
+
+(46) ColumnarToRow [codegen id : 9]
+Input [2]: [s_county#35, s_state#36]
+
+(47) Filter [codegen id : 9]
+Input [2]: [s_county#35, s_state#36]
+Condition : (isnotnull(s_county#35) AND isnotnull(s_state#36))
+
+(48) BroadcastExchange
+Input [2]: [s_county#35, s_state#36]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [id=#37]
+
+(49) BroadcastHashJoin [codegen id : 11]
+Left keys [2]: [ca_county#32, ca_state#33]
+Right keys [2]: [s_county#35, s_state#36]
+Join condition: None
+
+(50) Project [codegen id : 11]
+Output [3]: [c_customer_sk#22, ss_ext_sales_price#27, ss_sold_date_sk#28]
+Input [7]: [c_customer_sk#22, ss_ext_sales_price#27, ss_sold_date_sk#28, ca_county#32, ca_state#33, s_county#35, s_state#36]
+
+(51) Scan parquet default.date_dim
+Output [2]: [d_date_sk#38, d_month_seq#39]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(52) ColumnarToRow [codegen id : 10]
+Input [2]: [d_date_sk#38, d_month_seq#39]
+
+(53) Filter [codegen id : 10]
+Input [2]: [d_date_sk#38, d_month_seq#39]
+Condition : (((isnotnull(d_month_seq#39) AND (d_month_seq#39 >= Subquery scalar-subquery#40, [id=#41])) AND (d_month_seq#39 <= Subquery scalar-subquery#42, [id=#43])) AND isnotnull(d_date_sk#38))
+
+(54) Project [codegen id : 10]
+Output [1]: [d_date_sk#38]
+Input [2]: [d_date_sk#38, d_month_seq#39]
+
+(55) BroadcastExchange
+Input [1]: [d_date_sk#38]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#44]
+
+(56) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ss_sold_date_sk#28]
+Right keys [1]: [d_date_sk#38]
+Join condition: None
+
+(57) Project [codegen id : 11]
+Output [2]: [c_customer_sk#22, ss_ext_sales_price#27]
+Input [4]: [c_customer_sk#22, ss_ext_sales_price#27, ss_sold_date_sk#28, d_date_sk#38]
+
+(58) HashAggregate [codegen id : 11]
+Input [2]: [c_customer_sk#22, ss_ext_sales_price#27]
+Keys [1]: [c_customer_sk#22]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#27))]
+Aggregate Attributes [1]: [sum#45]
+Results [2]: [c_customer_sk#22, sum#46]
+
+(59) Exchange
+Input [2]: [c_customer_sk#22, sum#46]
+Arguments: hashpartitioning(c_customer_sk#22, 5), ENSURE_REQUIREMENTS, [id=#47]
+
+(60) HashAggregate [codegen id : 12]
+Input [2]: [c_customer_sk#22, sum#46]
+Keys [1]: [c_customer_sk#22]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#27))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#27))#48]
+Results [1]: [cast(CheckOverflow((promote_precision(MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#27))#48,17,2)) / 50.00), DecimalType(21,6), true) as int) AS segment#49]
+
+(61) HashAggregate [codegen id : 12]
+Input [1]: [segment#49]
+Keys [1]: [segment#49]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#50]
+Results [2]: [segment#49, count#51]
+
+(62) Exchange
+Input [2]: [segment#49, count#51]
+Arguments: hashpartitioning(segment#49, 5), ENSURE_REQUIREMENTS, [id=#52]
+
+(63) HashAggregate [codegen id : 13]
+Input [2]: [segment#49, count#51]
+Keys [1]: [segment#49]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#53]
+Results [3]: [segment#49, count(1)#53 AS num_customers#54, (segment#49 * 50) AS segment_base#55]
+
+(64) TakeOrderedAndProject
+Input [3]: [segment#49, num_customers#54, segment_base#55]
+Arguments: 100, [segment#49 ASC NULLS FIRST, num_customers#54 ASC NULLS FIRST], [segment#49, num_customers#54, segment_base#55]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (65)
+
+
+(65) ReusedExchange [Reuses operator id: 21]
+Output [1]: [d_date_sk#18]
+
+Subquery:2 Hosting operator id = 5 Hosting Expression = ws_sold_date_sk#10 IN dynamicpruning#4
+
+Subquery:3 Hosting operator id = 33 Hosting Expression = ss_sold_date_sk#28 IN dynamicpruning#29
+ReusedExchange (66)
+
+
+(66) ReusedExchange [Reuses operator id: 55]
+Output [1]: [d_date_sk#38]
+
+Subquery:4 Hosting operator id = 53 Hosting Expression = Subquery scalar-subquery#40, [id=#41]
+* HashAggregate (73)
++- Exchange (72)
+   +- * HashAggregate (71)
+      +- * Project (70)
+         +- * Filter (69)
+            +- * ColumnarToRow (68)
+               +- Scan parquet default.date_dim (67)
+
+
+(67) Scan parquet default.date_dim
+Output [3]: [d_month_seq#56, d_year#57, d_moy#58]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)]
+ReadSchema: struct<d_month_seq:int,d_year:int,d_moy:int>
+
+(68) ColumnarToRow [codegen id : 1]
+Input [3]: [d_month_seq#56, d_year#57, d_moy#58]
+
+(69) Filter [codegen id : 1]
+Input [3]: [d_month_seq#56, d_year#57, d_moy#58]
+Condition : (((isnotnull(d_year#57) AND isnotnull(d_moy#58)) AND (d_year#57 = 1998)) AND (d_moy#58 = 12))
+
+(70) Project [codegen id : 1]
+Output [1]: [(d_month_seq#56 + 1) AS (d_month_seq + 1)#59]
+Input [3]: [d_month_seq#56, d_year#57, d_moy#58]
+
+(71) HashAggregate [codegen id : 1]
+Input [1]: [(d_month_seq + 1)#59]
+Keys [1]: [(d_month_seq + 1)#59]
+Functions: []
+Aggregate Attributes: []
+Results [1]: [(d_month_seq + 1)#59]
+
+(72) Exchange
+Input [1]: [(d_month_seq + 1)#59]
+Arguments: hashpartitioning((d_month_seq + 1)#59, 5), ENSURE_REQUIREMENTS, [id=#60]
+
+(73) HashAggregate [codegen id : 2]
+Input [1]: [(d_month_seq + 1)#59]
+Keys [1]: [(d_month_seq + 1)#59]
+Functions: []
+Aggregate Attributes: []
+Results [1]: [(d_month_seq + 1)#59]
+
+Subquery:5 Hosting operator id = 53 Hosting Expression = Subquery scalar-subquery#42, [id=#43]
+* HashAggregate (80)
++- Exchange (79)
+   +- * HashAggregate (78)
+      +- * Project (77)
+         +- * Filter (76)
+            +- * ColumnarToRow (75)
+               +- Scan parquet default.date_dim (74)
+
+
+(74) Scan parquet default.date_dim
+Output [3]: [d_month_seq#61, d_year#62, d_moy#63]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)]
+ReadSchema: struct<d_month_seq:int,d_year:int,d_moy:int>
+
+(75) ColumnarToRow [codegen id : 1]
+Input [3]: [d_month_seq#61, d_year#62, d_moy#63]
+
+(76) Filter [codegen id : 1]
+Input [3]: [d_month_seq#61, d_year#62, d_moy#63]
+Condition : (((isnotnull(d_year#62) AND isnotnull(d_moy#63)) AND (d_year#62 = 1998)) AND (d_moy#63 = 12))
+
+(77) Project [codegen id : 1]
+Output [1]: [(d_month_seq#61 + 3) AS (d_month_seq + 3)#64]
+Input [3]: [d_month_seq#61, d_year#62, d_moy#63]
+
+(78) HashAggregate [codegen id : 1]
+Input [1]: [(d_month_seq + 3)#64]
+Keys [1]: [(d_month_seq + 3)#64]
+Functions: []
+Aggregate Attributes: []
+Results [1]: [(d_month_seq + 3)#64]
+
+(79) Exchange
+Input [1]: [(d_month_seq + 3)#64]
+Arguments: hashpartitioning((d_month_seq + 3)#64, 5), ENSURE_REQUIREMENTS, [id=#65]
+
+(80) HashAggregate [codegen id : 2]
+Input [1]: [(d_month_seq + 3)#64]
+Keys [1]: [(d_month_seq + 3)#64]
+Functions: []
+Aggregate Attributes: []
+Results [1]: [(d_month_seq + 3)#64]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/simplified.txt
new file mode 100644
index 0000000000000..d2db5d8da0a65
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/simplified.txt
@@ -0,0 +1,126 @@
+TakeOrderedAndProject [segment,num_customers,segment_base]
+  WholeStageCodegen (13)
+    HashAggregate [segment,count] [count(1),num_customers,segment_base,count]
+      InputAdapter
+        Exchange [segment] #1
+          WholeStageCodegen (12)
+            HashAggregate [segment] [count,count]
+              HashAggregate [c_customer_sk,sum] [sum(UnscaledValue(ss_ext_sales_price)),segment,sum]
+                InputAdapter
+                  Exchange [c_customer_sk] #2
+                    WholeStageCodegen (11)
+                      HashAggregate [c_customer_sk,ss_ext_sales_price] [sum,sum]
+                        Project [c_customer_sk,ss_ext_sales_price]
+                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                            Project [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk]
+                              BroadcastHashJoin [ca_county,ca_state,s_county,s_state]
+                                Project [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,ca_county,ca_state]
+                                  BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                                    Project [c_customer_sk,c_current_addr_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                      BroadcastHashJoin [c_customer_sk,ss_customer_sk]
+                                        HashAggregate [c_customer_sk,c_current_addr_sk]
+                                          InputAdapter
+                                            Exchange [c_customer_sk,c_current_addr_sk] #3
+                                              WholeStageCodegen (6)
+                                                HashAggregate [c_customer_sk,c_current_addr_sk]
+                                                  Project [c_customer_sk,c_current_addr_sk]
+                                                    BroadcastHashJoin [customer_sk,c_customer_sk]
+                                                      Project [customer_sk]
+                                                        BroadcastHashJoin [sold_date_sk,d_date_sk]
+                                                          Project [sold_date_sk,customer_sk]
+                                                            BroadcastHashJoin [item_sk,i_item_sk]
+                                                              InputAdapter
+                                                                Union
+                                                                  WholeStageCodegen (1)
+                                                                    Project [cs_sold_date_sk,cs_bill_customer_sk,cs_item_sk]
+                                                                      Filter [cs_item_sk,cs_bill_customer_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk]
+                                                                              SubqueryBroadcast [d_date_sk] #1
+                                                                                ReusedExchange [d_date_sk] #4
+                                                                  WholeStageCodegen (2)
+                                                                    Project [ws_sold_date_sk,ws_bill_customer_sk,ws_item_sk]
+                                                                      Filter [ws_item_sk,ws_bill_customer_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk]
+                                                                              ReusedSubquery [d_date_sk] #1
+                                                              InputAdapter
+                                                                BroadcastExchange #5
+                                                                  WholeStageCodegen (3)
+                                                                    Project [i_item_sk]
+                                                                      Filter [i_category,i_class,i_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.item [i_item_sk,i_class,i_category]
+                                                          InputAdapter
+                                                            BroadcastExchange #4
+                                                              WholeStageCodegen (4)
+                                                                Project [d_date_sk]
+                                                                  Filter [d_moy,d_year,d_date_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                                      InputAdapter
+                                                        BroadcastExchange #6
+                                                          WholeStageCodegen (5)
+                                                            Filter [c_customer_sk,c_current_addr_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.customer [c_customer_sk,c_current_addr_sk]
+                                        InputAdapter
+                                          BroadcastExchange #7
+                                            WholeStageCodegen (7)
+                                              Filter [ss_customer_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.store_sales [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                                      SubqueryBroadcast [d_date_sk] #2
+                                                        ReusedExchange [d_date_sk] #8
+                                    InputAdapter
+                                      BroadcastExchange #9
+                                        WholeStageCodegen (8)
+                                          Filter [ca_address_sk,ca_county,ca_state]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.customer_address [ca_address_sk,ca_county,ca_state]
+                                InputAdapter
+                                  BroadcastExchange #10
+                                    WholeStageCodegen (9)
+                                      Filter [s_county,s_state]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store [s_county,s_state]
+                            InputAdapter
+                              BroadcastExchange #8
+                                WholeStageCodegen (10)
+                                  Project [d_date_sk]
+                                    Filter [d_month_seq,d_date_sk]
+                                      Subquery #3
+                                        WholeStageCodegen (2)
+                                          HashAggregate [(d_month_seq + 1)]
+                                            InputAdapter
+                                              Exchange [(d_month_seq + 1)] #11
+                                                WholeStageCodegen (1)
+                                                  HashAggregate [(d_month_seq + 1)]
+                                                    Project [d_month_seq]
+                                                      Filter [d_year,d_moy]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.date_dim [d_month_seq,d_year,d_moy]
+                                      Subquery #4
+                                        WholeStageCodegen (2)
+                                          HashAggregate [(d_month_seq + 3)]
+                                            InputAdapter
+                                              Exchange [(d_month_seq + 3)] #12
+                                                WholeStageCodegen (1)
+                                                  HashAggregate [(d_month_seq + 3)]
+                                                    Project [d_month_seq]
+                                                      Filter [d_year,d_moy]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.date_dim [d_month_seq,d_year,d_moy]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.date_dim [d_date_sk,d_month_seq]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.sf100/explain.txt
new file mode 100644
index 0000000000000..c8d2a4748ba95
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.sf100/explain.txt
@@ -0,0 +1,133 @@
+== Physical Plan ==
+TakeOrderedAndProject (21)
++- * HashAggregate (20)
+   +- Exchange (19)
+      +- * HashAggregate (18)
+         +- * Project (17)
+            +- * BroadcastHashJoin Inner BuildRight (16)
+               :- * Project (10)
+               :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :- * Filter (3)
+               :     :  +- * ColumnarToRow (2)
+               :     :     +- Scan parquet default.store_sales (1)
+               :     +- BroadcastExchange (8)
+               :        +- * Project (7)
+               :           +- * Filter (6)
+               :              +- * ColumnarToRow (5)
+               :                 +- Scan parquet default.item (4)
+               +- BroadcastExchange (15)
+                  +- * Project (14)
+                     +- * Filter (13)
+                        +- * ColumnarToRow (12)
+                           +- Scan parquet default.date_dim (11)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_item_sk#1)
+
+(4) Scan parquet default.item
+Output [4]: [i_item_sk#5, i_brand_id#6, i_brand#7, i_manager_id#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,28), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_brand:string,i_manager_id:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [i_item_sk#5, i_brand_id#6, i_brand#7, i_manager_id#8]
+
+(6) Filter [codegen id : 1]
+Input [4]: [i_item_sk#5, i_brand_id#6, i_brand#7, i_manager_id#8]
+Condition : ((isnotnull(i_manager_id#8) AND (i_manager_id#8 = 28)) AND isnotnull(i_item_sk#5))
+
+(7) Project [codegen id : 1]
+Output [3]: [i_item_sk#5, i_brand_id#6, i_brand#7]
+Input [4]: [i_item_sk#5, i_brand_id#6, i_brand#7, i_manager_id#8]
+
+(8) BroadcastExchange
+Input [3]: [i_item_sk#5, i_brand_id#6, i_brand#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#9]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#5]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [4]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_brand_id#6, i_brand#7]
+Input [6]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_brand_id#6, i_brand#7]
+
+(11) Scan parquet default.date_dim
+Output [3]: [d_date_sk#10, d_year#11, d_moy#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+
+(13) Filter [codegen id : 2]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+Condition : ((((isnotnull(d_moy#12) AND isnotnull(d_year#11)) AND (d_moy#12 = 11)) AND (d_year#11 = 1999)) AND isnotnull(d_date_sk#10))
+
+(14) Project [codegen id : 2]
+Output [1]: [d_date_sk#10]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+
+(15) BroadcastExchange
+Input [1]: [d_date_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(16) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(17) Project [codegen id : 3]
+Output [3]: [ss_ext_sales_price#2, i_brand_id#6, i_brand#7]
+Input [5]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_brand_id#6, i_brand#7, d_date_sk#10]
+
+(18) HashAggregate [codegen id : 3]
+Input [3]: [ss_ext_sales_price#2, i_brand_id#6, i_brand#7]
+Keys [2]: [i_brand#7, i_brand_id#6]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum#14]
+Results [3]: [i_brand#7, i_brand_id#6, sum#15]
+
+(19) Exchange
+Input [3]: [i_brand#7, i_brand_id#6, sum#15]
+Arguments: hashpartitioning(i_brand#7, i_brand_id#6, 5), ENSURE_REQUIREMENTS, [id=#16]
+
+(20) HashAggregate [codegen id : 4]
+Input [3]: [i_brand#7, i_brand_id#6, sum#15]
+Keys [2]: [i_brand#7, i_brand_id#6]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#17]
+Results [3]: [i_brand_id#6 AS brand_id#18, i_brand#7 AS brand#19, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#17,17,2) AS ext_price#20]
+
+(21) TakeOrderedAndProject
+Input [3]: [brand_id#18, brand#19, ext_price#20]
+Arguments: 100, [ext_price#20 DESC NULLS LAST, brand_id#18 ASC NULLS FIRST], [brand_id#18, brand#19, ext_price#20]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (22)
+
+
+(22) ReusedExchange [Reuses operator id: 15]
+Output [1]: [d_date_sk#10]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.sf100/simplified.txt
new file mode 100644
index 0000000000000..4208d6ce88e97
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.sf100/simplified.txt
@@ -0,0 +1,33 @@
+TakeOrderedAndProject [ext_price,brand_id,brand]
+  WholeStageCodegen (4)
+    HashAggregate [i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum]
+      InputAdapter
+        Exchange [i_brand,i_brand_id] #1
+          WholeStageCodegen (3)
+            HashAggregate [i_brand,i_brand_id,ss_ext_sales_price] [sum,sum]
+              Project [ss_ext_sales_price,i_brand_id,i_brand]
+                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                  Project [ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_brand]
+                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                      Filter [ss_item_sk]
+                        ColumnarToRow
+                          InputAdapter
+                            Scan parquet default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk]
+                              SubqueryBroadcast [d_date_sk] #1
+                                ReusedExchange [d_date_sk] #2
+                      InputAdapter
+                        BroadcastExchange #3
+                          WholeStageCodegen (1)
+                            Project [i_item_sk,i_brand_id,i_brand]
+                              Filter [i_manager_id,i_item_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.item [i_item_sk,i_brand_id,i_brand,i_manager_id]
+                  InputAdapter
+                    BroadcastExchange #2
+                      WholeStageCodegen (2)
+                        Project [d_date_sk]
+                          Filter [d_moy,d_year,d_date_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/explain.txt
new file mode 100644
index 0000000000000..25b423382e332
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/explain.txt
@@ -0,0 +1,123 @@
+== Physical Plan ==
+TakeOrderedAndProject (21)
++- * HashAggregate (20)
+   +- Exchange (19)
+      +- * HashAggregate (18)
+         +- * Project (17)
+            +- * BroadcastHashJoin Inner BuildRight (16)
+               :- * Project (10)
+               :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :- * Project (4)
+               :     :  +- * Filter (3)
+               :     :     +- * ColumnarToRow (2)
+               :     :        +- Scan parquet default.date_dim (1)
+               :     +- BroadcastExchange (8)
+               :        +- * Filter (7)
+               :           +- * ColumnarToRow (6)
+               :              +- Scan parquet default.store_sales (5)
+               +- BroadcastExchange (15)
+                  +- * Project (14)
+                     +- * Filter (13)
+                        +- * ColumnarToRow (12)
+                           +- Scan parquet default.item (11)
+
+
+(1) Scan parquet default.date_dim
+Output [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 1999)) AND isnotnull(d_date_sk#1))
+
+(4) Project [codegen id : 3]
+Output [1]: [d_date_sk#1]
+Input [3]: [d_date_sk#1, d_year#2, d_moy#3]
+
+(5) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+
+(7) Filter [codegen id : 1]
+Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Condition : isnotnull(ss_item_sk#4)
+
+(8) BroadcastExchange
+Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [d_date_sk#1]
+Right keys [1]: [ss_sold_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [2]: [ss_item_sk#4, ss_ext_sales_price#5]
+Input [4]: [d_date_sk#1, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+
+(11) Scan parquet default.item
+Output [4]: [i_item_sk#8, i_brand_id#9, i_brand#10, i_manager_id#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,28), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_brand:string,i_manager_id:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [4]: [i_item_sk#8, i_brand_id#9, i_brand#10, i_manager_id#11]
+
+(13) Filter [codegen id : 2]
+Input [4]: [i_item_sk#8, i_brand_id#9, i_brand#10, i_manager_id#11]
+Condition : ((isnotnull(i_manager_id#11) AND (i_manager_id#11 = 28)) AND isnotnull(i_item_sk#8))
+
+(14) Project [codegen id : 2]
+Output [3]: [i_item_sk#8, i_brand_id#9, i_brand#10]
+Input [4]: [i_item_sk#8, i_brand_id#9, i_brand#10, i_manager_id#11]
+
+(15) BroadcastExchange
+Input [3]: [i_item_sk#8, i_brand_id#9, i_brand#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(16) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_item_sk#4]
+Right keys [1]: [i_item_sk#8]
+Join condition: None
+
+(17) Project [codegen id : 3]
+Output [3]: [ss_ext_sales_price#5, i_brand_id#9, i_brand#10]
+Input [5]: [ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#8, i_brand_id#9, i_brand#10]
+
+(18) HashAggregate [codegen id : 3]
+Input [3]: [ss_ext_sales_price#5, i_brand_id#9, i_brand#10]
+Keys [2]: [i_brand#10, i_brand_id#9]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))]
+Aggregate Attributes [1]: [sum#13]
+Results [3]: [i_brand#10, i_brand_id#9, sum#14]
+
+(19) Exchange
+Input [3]: [i_brand#10, i_brand_id#9, sum#14]
+Arguments: hashpartitioning(i_brand#10, i_brand_id#9, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(20) HashAggregate [codegen id : 4]
+Input [3]: [i_brand#10, i_brand_id#9, sum#14]
+Keys [2]: [i_brand#10, i_brand_id#9]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#16]
+Results [3]: [i_brand_id#9 AS brand_id#17, i_brand#10 AS brand#18, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#16,17,2) AS ext_price#19]
+
+(21) TakeOrderedAndProject
+Input [3]: [brand_id#17, brand#18, ext_price#19]
+Arguments: 100, [ext_price#19 DESC NULLS LAST, brand_id#17 ASC NULLS FIRST], [brand_id#17, brand#18, ext_price#19]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/simplified.txt
new file mode 100644
index 0000000000000..9157bbec2b06a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/simplified.txt
@@ -0,0 +1,31 @@
+TakeOrderedAndProject [ext_price,brand_id,brand]
+  WholeStageCodegen (4)
+    HashAggregate [i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum]
+      InputAdapter
+        Exchange [i_brand,i_brand_id] #1
+          WholeStageCodegen (3)
+            HashAggregate [i_brand,i_brand_id,ss_ext_sales_price] [sum,sum]
+              Project [ss_ext_sales_price,i_brand_id,i_brand]
+                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                  Project [ss_item_sk,ss_ext_sales_price]
+                    BroadcastHashJoin [d_date_sk,ss_sold_date_sk]
+                      Project [d_date_sk]
+                        Filter [d_moy,d_year,d_date_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                      InputAdapter
+                        BroadcastExchange #2
+                          WholeStageCodegen (1)
+                            Filter [ss_item_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk]
+                  InputAdapter
+                    BroadcastExchange #3
+                      WholeStageCodegen (2)
+                        Project [i_item_sk,i_brand_id,i_brand]
+                          Filter [i_manager_id,i_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.item [i_item_sk,i_brand_id,i_brand,i_manager_id]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.sf100/explain.txt
new file mode 100644
index 0000000000000..47eda1483a06c
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.sf100/explain.txt
@@ -0,0 +1,395 @@
+== Physical Plan ==
+TakeOrderedAndProject (67)
++- * HashAggregate (66)
+   +- Exchange (65)
+      +- * HashAggregate (64)
+         +- Union (63)
+            :- * HashAggregate (32)
+            :  +- Exchange (31)
+            :     +- * HashAggregate (30)
+            :        +- * Project (29)
+            :           +- * BroadcastHashJoin Inner BuildRight (28)
+            :              :- * Project (17)
+            :              :  +- * BroadcastHashJoin Inner BuildRight (16)
+            :              :     :- * Project (10)
+            :              :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+            :              :     :     :- * Filter (3)
+            :              :     :     :  +- * ColumnarToRow (2)
+            :              :     :     :     +- Scan parquet default.store_sales (1)
+            :              :     :     +- BroadcastExchange (8)
+            :              :     :        +- * Project (7)
+            :              :     :           +- * Filter (6)
+            :              :     :              +- * ColumnarToRow (5)
+            :              :     :                 +- Scan parquet default.date_dim (4)
+            :              :     +- BroadcastExchange (15)
+            :              :        +- * Project (14)
+            :              :           +- * Filter (13)
+            :              :              +- * ColumnarToRow (12)
+            :              :                 +- Scan parquet default.customer_address (11)
+            :              +- BroadcastExchange (27)
+            :                 +- * BroadcastHashJoin LeftSemi BuildRight (26)
+            :                    :- * Filter (20)
+            :                    :  +- * ColumnarToRow (19)
+            :                    :     +- Scan parquet default.item (18)
+            :                    +- BroadcastExchange (25)
+            :                       +- * Project (24)
+            :                          +- * Filter (23)
+            :                             +- * ColumnarToRow (22)
+            :                                +- Scan parquet default.item (21)
+            :- * HashAggregate (47)
+            :  +- Exchange (46)
+            :     +- * HashAggregate (45)
+            :        +- * Project (44)
+            :           +- * BroadcastHashJoin Inner BuildRight (43)
+            :              :- * Project (41)
+            :              :  +- * BroadcastHashJoin Inner BuildRight (40)
+            :              :     :- * Project (38)
+            :              :     :  +- * BroadcastHashJoin Inner BuildRight (37)
+            :              :     :     :- * Filter (35)
+            :              :     :     :  +- * ColumnarToRow (34)
+            :              :     :     :     +- Scan parquet default.catalog_sales (33)
+            :              :     :     +- ReusedExchange (36)
+            :              :     +- ReusedExchange (39)
+            :              +- ReusedExchange (42)
+            +- * HashAggregate (62)
+               +- Exchange (61)
+                  +- * HashAggregate (60)
+                     +- * Project (59)
+                        +- * BroadcastHashJoin Inner BuildRight (58)
+                           :- * Project (56)
+                           :  +- * BroadcastHashJoin Inner BuildRight (55)
+                           :     :- * Project (53)
+                           :     :  +- * BroadcastHashJoin Inner BuildRight (52)
+                           :     :     :- * Filter (50)
+                           :     :     :  +- * ColumnarToRow (49)
+                           :     :     :     +- Scan parquet default.web_sales (48)
+                           :     :     +- ReusedExchange (51)
+                           :     +- ReusedExchange (54)
+                           +- ReusedExchange (57)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_addr_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 5]
+Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4]
+Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2001)) AND (d_moy#8 = 2)) AND isnotnull(d_date_sk#6))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#6]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#9]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3]
+Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#6]
+
+(11) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#10, ca_gmt_offset#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_gmt_offset:decimal(5,2)>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [ca_address_sk#10, ca_gmt_offset#11]
+
+(13) Filter [codegen id : 2]
+Input [2]: [ca_address_sk#10, ca_gmt_offset#11]
+Condition : ((isnotnull(ca_gmt_offset#11) AND (ca_gmt_offset#11 = -5.00)) AND isnotnull(ca_address_sk#10))
+
+(14) Project [codegen id : 2]
+Output [1]: [ca_address_sk#10]
+Input [2]: [ca_address_sk#10, ca_gmt_offset#11]
+
+(15) BroadcastExchange
+Input [1]: [ca_address_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(16) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_addr_sk#2]
+Right keys [1]: [ca_address_sk#10]
+Join condition: None
+
+(17) Project [codegen id : 5]
+Output [2]: [ss_item_sk#1, ss_ext_sales_price#3]
+Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#10]
+
+(18) Scan parquet default.item
+Output [2]: [i_item_sk#13, i_item_id#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(19) ColumnarToRow [codegen id : 4]
+Input [2]: [i_item_sk#13, i_item_id#14]
+
+(20) Filter [codegen id : 4]
+Input [2]: [i_item_sk#13, i_item_id#14]
+Condition : isnotnull(i_item_sk#13)
+
+(21) Scan parquet default.item
+Output [2]: [i_item_id#15, i_color#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [In(i_color, [slate               ,blanched            ,burnished           ])]
+ReadSchema: struct<i_item_id:string,i_color:string>
+
+(22) ColumnarToRow [codegen id : 3]
+Input [2]: [i_item_id#15, i_color#16]
+
+(23) Filter [codegen id : 3]
+Input [2]: [i_item_id#15, i_color#16]
+Condition : i_color#16 IN (slate               ,blanched            ,burnished           )
+
+(24) Project [codegen id : 3]
+Output [1]: [i_item_id#15]
+Input [2]: [i_item_id#15, i_color#16]
+
+(25) BroadcastExchange
+Input [1]: [i_item_id#15]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#17]
+
+(26) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [i_item_id#14]
+Right keys [1]: [i_item_id#15]
+Join condition: None
+
+(27) BroadcastExchange
+Input [2]: [i_item_sk#13, i_item_id#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#18]
+
+(28) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#13]
+Join condition: None
+
+(29) Project [codegen id : 5]
+Output [2]: [ss_ext_sales_price#3, i_item_id#14]
+Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#13, i_item_id#14]
+
+(30) HashAggregate [codegen id : 5]
+Input [2]: [ss_ext_sales_price#3, i_item_id#14]
+Keys [1]: [i_item_id#14]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))]
+Aggregate Attributes [1]: [sum#19]
+Results [2]: [i_item_id#14, sum#20]
+
+(31) Exchange
+Input [2]: [i_item_id#14, sum#20]
+Arguments: hashpartitioning(i_item_id#14, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(32) HashAggregate [codegen id : 6]
+Input [2]: [i_item_id#14, sum#20]
+Keys [1]: [i_item_id#14]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#22]
+Results [2]: [i_item_id#14, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#22,17,2) AS total_sales#23]
+
+(33) Scan parquet default.catalog_sales
+Output [4]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, cs_sold_date_sk#27]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#27), dynamicpruningexpression(cs_sold_date_sk#27 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_bill_addr_sk:int,cs_item_sk:int,cs_ext_sales_price:decimal(7,2)>
+
+(34) ColumnarToRow [codegen id : 11]
+Input [4]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, cs_sold_date_sk#27]
+
+(35) Filter [codegen id : 11]
+Input [4]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, cs_sold_date_sk#27]
+Condition : (isnotnull(cs_bill_addr_sk#24) AND isnotnull(cs_item_sk#25))
+
+(36) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#28]
+
+(37) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_sold_date_sk#27]
+Right keys [1]: [d_date_sk#28]
+Join condition: None
+
+(38) Project [codegen id : 11]
+Output [3]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26]
+Input [5]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, cs_sold_date_sk#27, d_date_sk#28]
+
+(39) ReusedExchange [Reuses operator id: 15]
+Output [1]: [ca_address_sk#29]
+
+(40) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_bill_addr_sk#24]
+Right keys [1]: [ca_address_sk#29]
+Join condition: None
+
+(41) Project [codegen id : 11]
+Output [2]: [cs_item_sk#25, cs_ext_sales_price#26]
+Input [4]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, ca_address_sk#29]
+
+(42) ReusedExchange [Reuses operator id: 27]
+Output [2]: [i_item_sk#30, i_item_id#31]
+
+(43) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_item_sk#25]
+Right keys [1]: [i_item_sk#30]
+Join condition: None
+
+(44) Project [codegen id : 11]
+Output [2]: [cs_ext_sales_price#26, i_item_id#31]
+Input [4]: [cs_item_sk#25, cs_ext_sales_price#26, i_item_sk#30, i_item_id#31]
+
+(45) HashAggregate [codegen id : 11]
+Input [2]: [cs_ext_sales_price#26, i_item_id#31]
+Keys [1]: [i_item_id#31]
+Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#26))]
+Aggregate Attributes [1]: [sum#32]
+Results [2]: [i_item_id#31, sum#33]
+
+(46) Exchange
+Input [2]: [i_item_id#31, sum#33]
+Arguments: hashpartitioning(i_item_id#31, 5), ENSURE_REQUIREMENTS, [id=#34]
+
+(47) HashAggregate [codegen id : 12]
+Input [2]: [i_item_id#31, sum#33]
+Keys [1]: [i_item_id#31]
+Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#26))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#26))#35]
+Results [2]: [i_item_id#31, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#26))#35,17,2) AS total_sales#36]
+
+(48) Scan parquet default.web_sales
+Output [4]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#40), dynamicpruningexpression(ws_sold_date_sk#40 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_bill_addr_sk:int,ws_ext_sales_price:decimal(7,2)>
+
+(49) ColumnarToRow [codegen id : 17]
+Input [4]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40]
+
+(50) Filter [codegen id : 17]
+Input [4]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40]
+Condition : (isnotnull(ws_bill_addr_sk#38) AND isnotnull(ws_item_sk#37))
+
+(51) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#41]
+
+(52) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_sold_date_sk#40]
+Right keys [1]: [d_date_sk#41]
+Join condition: None
+
+(53) Project [codegen id : 17]
+Output [3]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39]
+Input [5]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40, d_date_sk#41]
+
+(54) ReusedExchange [Reuses operator id: 15]
+Output [1]: [ca_address_sk#42]
+
+(55) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_bill_addr_sk#38]
+Right keys [1]: [ca_address_sk#42]
+Join condition: None
+
+(56) Project [codegen id : 17]
+Output [2]: [ws_item_sk#37, ws_ext_sales_price#39]
+Input [4]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ca_address_sk#42]
+
+(57) ReusedExchange [Reuses operator id: 27]
+Output [2]: [i_item_sk#43, i_item_id#44]
+
+(58) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_item_sk#37]
+Right keys [1]: [i_item_sk#43]
+Join condition: None
+
+(59) Project [codegen id : 17]
+Output [2]: [ws_ext_sales_price#39, i_item_id#44]
+Input [4]: [ws_item_sk#37, ws_ext_sales_price#39, i_item_sk#43, i_item_id#44]
+
+(60) HashAggregate [codegen id : 17]
+Input [2]: [ws_ext_sales_price#39, i_item_id#44]
+Keys [1]: [i_item_id#44]
+Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#39))]
+Aggregate Attributes [1]: [sum#45]
+Results [2]: [i_item_id#44, sum#46]
+
+(61) Exchange
+Input [2]: [i_item_id#44, sum#46]
+Arguments: hashpartitioning(i_item_id#44, 5), ENSURE_REQUIREMENTS, [id=#47]
+
+(62) HashAggregate [codegen id : 18]
+Input [2]: [i_item_id#44, sum#46]
+Keys [1]: [i_item_id#44]
+Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#39))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#39))#48]
+Results [2]: [i_item_id#44, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#39))#48,17,2) AS total_sales#49]
+
+(63) Union
+
+(64) HashAggregate [codegen id : 19]
+Input [2]: [i_item_id#14, total_sales#23]
+Keys [1]: [i_item_id#14]
+Functions [1]: [partial_sum(total_sales#23)]
+Aggregate Attributes [2]: [sum#50, isEmpty#51]
+Results [3]: [i_item_id#14, sum#52, isEmpty#53]
+
+(65) Exchange
+Input [3]: [i_item_id#14, sum#52, isEmpty#53]
+Arguments: hashpartitioning(i_item_id#14, 5), ENSURE_REQUIREMENTS, [id=#54]
+
+(66) HashAggregate [codegen id : 20]
+Input [3]: [i_item_id#14, sum#52, isEmpty#53]
+Keys [1]: [i_item_id#14]
+Functions [1]: [sum(total_sales#23)]
+Aggregate Attributes [1]: [sum(total_sales#23)#55]
+Results [2]: [i_item_id#14, sum(total_sales#23)#55 AS total_sales#56]
+
+(67) TakeOrderedAndProject
+Input [2]: [i_item_id#14, total_sales#56]
+Arguments: 100, [total_sales#56 ASC NULLS FIRST], [i_item_id#14, total_sales#56]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (68)
+
+
+(68) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#6]
+
+Subquery:2 Hosting operator id = 33 Hosting Expression = cs_sold_date_sk#27 IN dynamicpruning#5
+
+Subquery:3 Hosting operator id = 48 Hosting Expression = ws_sold_date_sk#40 IN dynamicpruning#5
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.sf100/simplified.txt
new file mode 100644
index 0000000000000..2d21010c26b8d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.sf100/simplified.txt
@@ -0,0 +1,105 @@
+TakeOrderedAndProject [total_sales,i_item_id]
+  WholeStageCodegen (20)
+    HashAggregate [i_item_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty]
+      InputAdapter
+        Exchange [i_item_id] #1
+          WholeStageCodegen (19)
+            HashAggregate [i_item_id,total_sales] [sum,isEmpty,sum,isEmpty]
+              InputAdapter
+                Union
+                  WholeStageCodegen (6)
+                    HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum]
+                      InputAdapter
+                        Exchange [i_item_id] #2
+                          WholeStageCodegen (5)
+                            HashAggregate [i_item_id,ss_ext_sales_price] [sum,sum]
+                              Project [ss_ext_sales_price,i_item_id]
+                                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                  Project [ss_item_sk,ss_ext_sales_price]
+                                    BroadcastHashJoin [ss_addr_sk,ca_address_sk]
+                                      Project [ss_item_sk,ss_addr_sk,ss_ext_sales_price]
+                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                          Filter [ss_addr_sk,ss_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                                  SubqueryBroadcast [d_date_sk] #1
+                                                    ReusedExchange [d_date_sk] #3
+                                          InputAdapter
+                                            BroadcastExchange #3
+                                              WholeStageCodegen (1)
+                                                Project [d_date_sk]
+                                                  Filter [d_year,d_moy,d_date_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                      InputAdapter
+                                        BroadcastExchange #4
+                                          WholeStageCodegen (2)
+                                            Project [ca_address_sk]
+                                              Filter [ca_gmt_offset,ca_address_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.customer_address [ca_address_sk,ca_gmt_offset]
+                                  InputAdapter
+                                    BroadcastExchange #5
+                                      WholeStageCodegen (4)
+                                        BroadcastHashJoin [i_item_id,i_item_id]
+                                          Filter [i_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.item [i_item_sk,i_item_id]
+                                          InputAdapter
+                                            BroadcastExchange #6
+                                              WholeStageCodegen (3)
+                                                Project [i_item_id]
+                                                  Filter [i_color]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.item [i_item_id,i_color]
+                  WholeStageCodegen (12)
+                    HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum]
+                      InputAdapter
+                        Exchange [i_item_id] #7
+                          WholeStageCodegen (11)
+                            HashAggregate [i_item_id,cs_ext_sales_price] [sum,sum]
+                              Project [cs_ext_sales_price,i_item_id]
+                                BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                  Project [cs_item_sk,cs_ext_sales_price]
+                                    BroadcastHashJoin [cs_bill_addr_sk,ca_address_sk]
+                                      Project [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price]
+                                        BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                          Filter [cs_bill_addr_sk,cs_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk]
+                                                  ReusedSubquery [d_date_sk] #1
+                                          InputAdapter
+                                            ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        ReusedExchange [ca_address_sk] #4
+                                  InputAdapter
+                                    ReusedExchange [i_item_sk,i_item_id] #5
+                  WholeStageCodegen (18)
+                    HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum]
+                      InputAdapter
+                        Exchange [i_item_id] #8
+                          WholeStageCodegen (17)
+                            HashAggregate [i_item_id,ws_ext_sales_price] [sum,sum]
+                              Project [ws_ext_sales_price,i_item_id]
+                                BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                  Project [ws_item_sk,ws_ext_sales_price]
+                                    BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk]
+                                      Project [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price]
+                                        BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                          Filter [ws_bill_addr_sk,ws_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk]
+                                                  ReusedSubquery [d_date_sk] #1
+                                          InputAdapter
+                                            ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        ReusedExchange [ca_address_sk] #4
+                                  InputAdapter
+                                    ReusedExchange [i_item_sk,i_item_id] #5
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/explain.txt
new file mode 100644
index 0000000000000..47eda1483a06c
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/explain.txt
@@ -0,0 +1,395 @@
+== Physical Plan ==
+TakeOrderedAndProject (67)
++- * HashAggregate (66)
+   +- Exchange (65)
+      +- * HashAggregate (64)
+         +- Union (63)
+            :- * HashAggregate (32)
+            :  +- Exchange (31)
+            :     +- * HashAggregate (30)
+            :        +- * Project (29)
+            :           +- * BroadcastHashJoin Inner BuildRight (28)
+            :              :- * Project (17)
+            :              :  +- * BroadcastHashJoin Inner BuildRight (16)
+            :              :     :- * Project (10)
+            :              :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+            :              :     :     :- * Filter (3)
+            :              :     :     :  +- * ColumnarToRow (2)
+            :              :     :     :     +- Scan parquet default.store_sales (1)
+            :              :     :     +- BroadcastExchange (8)
+            :              :     :        +- * Project (7)
+            :              :     :           +- * Filter (6)
+            :              :     :              +- * ColumnarToRow (5)
+            :              :     :                 +- Scan parquet default.date_dim (4)
+            :              :     +- BroadcastExchange (15)
+            :              :        +- * Project (14)
+            :              :           +- * Filter (13)
+            :              :              +- * ColumnarToRow (12)
+            :              :                 +- Scan parquet default.customer_address (11)
+            :              +- BroadcastExchange (27)
+            :                 +- * BroadcastHashJoin LeftSemi BuildRight (26)
+            :                    :- * Filter (20)
+            :                    :  +- * ColumnarToRow (19)
+            :                    :     +- Scan parquet default.item (18)
+            :                    +- BroadcastExchange (25)
+            :                       +- * Project (24)
+            :                          +- * Filter (23)
+            :                             +- * ColumnarToRow (22)
+            :                                +- Scan parquet default.item (21)
+            :- * HashAggregate (47)
+            :  +- Exchange (46)
+            :     +- * HashAggregate (45)
+            :        +- * Project (44)
+            :           +- * BroadcastHashJoin Inner BuildRight (43)
+            :              :- * Project (41)
+            :              :  +- * BroadcastHashJoin Inner BuildRight (40)
+            :              :     :- * Project (38)
+            :              :     :  +- * BroadcastHashJoin Inner BuildRight (37)
+            :              :     :     :- * Filter (35)
+            :              :     :     :  +- * ColumnarToRow (34)
+            :              :     :     :     +- Scan parquet default.catalog_sales (33)
+            :              :     :     +- ReusedExchange (36)
+            :              :     +- ReusedExchange (39)
+            :              +- ReusedExchange (42)
+            +- * HashAggregate (62)
+               +- Exchange (61)
+                  +- * HashAggregate (60)
+                     +- * Project (59)
+                        +- * BroadcastHashJoin Inner BuildRight (58)
+                           :- * Project (56)
+                           :  +- * BroadcastHashJoin Inner BuildRight (55)
+                           :     :- * Project (53)
+                           :     :  +- * BroadcastHashJoin Inner BuildRight (52)
+                           :     :     :- * Filter (50)
+                           :     :     :  +- * ColumnarToRow (49)
+                           :     :     :     +- Scan parquet default.web_sales (48)
+                           :     :     +- ReusedExchange (51)
+                           :     +- ReusedExchange (54)
+                           +- ReusedExchange (57)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_addr_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 5]
+Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4]
+Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2001)) AND (d_moy#8 = 2)) AND isnotnull(d_date_sk#6))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#6]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#9]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3]
+Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#6]
+
+(11) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#10, ca_gmt_offset#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_gmt_offset:decimal(5,2)>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [ca_address_sk#10, ca_gmt_offset#11]
+
+(13) Filter [codegen id : 2]
+Input [2]: [ca_address_sk#10, ca_gmt_offset#11]
+Condition : ((isnotnull(ca_gmt_offset#11) AND (ca_gmt_offset#11 = -5.00)) AND isnotnull(ca_address_sk#10))
+
+(14) Project [codegen id : 2]
+Output [1]: [ca_address_sk#10]
+Input [2]: [ca_address_sk#10, ca_gmt_offset#11]
+
+(15) BroadcastExchange
+Input [1]: [ca_address_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(16) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_addr_sk#2]
+Right keys [1]: [ca_address_sk#10]
+Join condition: None
+
+(17) Project [codegen id : 5]
+Output [2]: [ss_item_sk#1, ss_ext_sales_price#3]
+Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#10]
+
+(18) Scan parquet default.item
+Output [2]: [i_item_sk#13, i_item_id#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(19) ColumnarToRow [codegen id : 4]
+Input [2]: [i_item_sk#13, i_item_id#14]
+
+(20) Filter [codegen id : 4]
+Input [2]: [i_item_sk#13, i_item_id#14]
+Condition : isnotnull(i_item_sk#13)
+
+(21) Scan parquet default.item
+Output [2]: [i_item_id#15, i_color#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [In(i_color, [slate               ,blanched            ,burnished           ])]
+ReadSchema: struct<i_item_id:string,i_color:string>
+
+(22) ColumnarToRow [codegen id : 3]
+Input [2]: [i_item_id#15, i_color#16]
+
+(23) Filter [codegen id : 3]
+Input [2]: [i_item_id#15, i_color#16]
+Condition : i_color#16 IN (slate               ,blanched            ,burnished           )
+
+(24) Project [codegen id : 3]
+Output [1]: [i_item_id#15]
+Input [2]: [i_item_id#15, i_color#16]
+
+(25) BroadcastExchange
+Input [1]: [i_item_id#15]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#17]
+
+(26) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [i_item_id#14]
+Right keys [1]: [i_item_id#15]
+Join condition: None
+
+(27) BroadcastExchange
+Input [2]: [i_item_sk#13, i_item_id#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#18]
+
+(28) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#13]
+Join condition: None
+
+(29) Project [codegen id : 5]
+Output [2]: [ss_ext_sales_price#3, i_item_id#14]
+Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#13, i_item_id#14]
+
+(30) HashAggregate [codegen id : 5]
+Input [2]: [ss_ext_sales_price#3, i_item_id#14]
+Keys [1]: [i_item_id#14]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))]
+Aggregate Attributes [1]: [sum#19]
+Results [2]: [i_item_id#14, sum#20]
+
+(31) Exchange
+Input [2]: [i_item_id#14, sum#20]
+Arguments: hashpartitioning(i_item_id#14, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(32) HashAggregate [codegen id : 6]
+Input [2]: [i_item_id#14, sum#20]
+Keys [1]: [i_item_id#14]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#22]
+Results [2]: [i_item_id#14, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#22,17,2) AS total_sales#23]
+
+(33) Scan parquet default.catalog_sales
+Output [4]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, cs_sold_date_sk#27]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#27), dynamicpruningexpression(cs_sold_date_sk#27 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_bill_addr_sk:int,cs_item_sk:int,cs_ext_sales_price:decimal(7,2)>
+
+(34) ColumnarToRow [codegen id : 11]
+Input [4]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, cs_sold_date_sk#27]
+
+(35) Filter [codegen id : 11]
+Input [4]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, cs_sold_date_sk#27]
+Condition : (isnotnull(cs_bill_addr_sk#24) AND isnotnull(cs_item_sk#25))
+
+(36) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#28]
+
+(37) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_sold_date_sk#27]
+Right keys [1]: [d_date_sk#28]
+Join condition: None
+
+(38) Project [codegen id : 11]
+Output [3]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26]
+Input [5]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, cs_sold_date_sk#27, d_date_sk#28]
+
+(39) ReusedExchange [Reuses operator id: 15]
+Output [1]: [ca_address_sk#29]
+
+(40) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_bill_addr_sk#24]
+Right keys [1]: [ca_address_sk#29]
+Join condition: None
+
+(41) Project [codegen id : 11]
+Output [2]: [cs_item_sk#25, cs_ext_sales_price#26]
+Input [4]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, ca_address_sk#29]
+
+(42) ReusedExchange [Reuses operator id: 27]
+Output [2]: [i_item_sk#30, i_item_id#31]
+
+(43) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_item_sk#25]
+Right keys [1]: [i_item_sk#30]
+Join condition: None
+
+(44) Project [codegen id : 11]
+Output [2]: [cs_ext_sales_price#26, i_item_id#31]
+Input [4]: [cs_item_sk#25, cs_ext_sales_price#26, i_item_sk#30, i_item_id#31]
+
+(45) HashAggregate [codegen id : 11]
+Input [2]: [cs_ext_sales_price#26, i_item_id#31]
+Keys [1]: [i_item_id#31]
+Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#26))]
+Aggregate Attributes [1]: [sum#32]
+Results [2]: [i_item_id#31, sum#33]
+
+(46) Exchange
+Input [2]: [i_item_id#31, sum#33]
+Arguments: hashpartitioning(i_item_id#31, 5), ENSURE_REQUIREMENTS, [id=#34]
+
+(47) HashAggregate [codegen id : 12]
+Input [2]: [i_item_id#31, sum#33]
+Keys [1]: [i_item_id#31]
+Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#26))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#26))#35]
+Results [2]: [i_item_id#31, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#26))#35,17,2) AS total_sales#36]
+
+(48) Scan parquet default.web_sales
+Output [4]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#40), dynamicpruningexpression(ws_sold_date_sk#40 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_bill_addr_sk:int,ws_ext_sales_price:decimal(7,2)>
+
+(49) ColumnarToRow [codegen id : 17]
+Input [4]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40]
+
+(50) Filter [codegen id : 17]
+Input [4]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40]
+Condition : (isnotnull(ws_bill_addr_sk#38) AND isnotnull(ws_item_sk#37))
+
+(51) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#41]
+
+(52) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_sold_date_sk#40]
+Right keys [1]: [d_date_sk#41]
+Join condition: None
+
+(53) Project [codegen id : 17]
+Output [3]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39]
+Input [5]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40, d_date_sk#41]
+
+(54) ReusedExchange [Reuses operator id: 15]
+Output [1]: [ca_address_sk#42]
+
+(55) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_bill_addr_sk#38]
+Right keys [1]: [ca_address_sk#42]
+Join condition: None
+
+(56) Project [codegen id : 17]
+Output [2]: [ws_item_sk#37, ws_ext_sales_price#39]
+Input [4]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ca_address_sk#42]
+
+(57) ReusedExchange [Reuses operator id: 27]
+Output [2]: [i_item_sk#43, i_item_id#44]
+
+(58) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_item_sk#37]
+Right keys [1]: [i_item_sk#43]
+Join condition: None
+
+(59) Project [codegen id : 17]
+Output [2]: [ws_ext_sales_price#39, i_item_id#44]
+Input [4]: [ws_item_sk#37, ws_ext_sales_price#39, i_item_sk#43, i_item_id#44]
+
+(60) HashAggregate [codegen id : 17]
+Input [2]: [ws_ext_sales_price#39, i_item_id#44]
+Keys [1]: [i_item_id#44]
+Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#39))]
+Aggregate Attributes [1]: [sum#45]
+Results [2]: [i_item_id#44, sum#46]
+
+(61) Exchange
+Input [2]: [i_item_id#44, sum#46]
+Arguments: hashpartitioning(i_item_id#44, 5), ENSURE_REQUIREMENTS, [id=#47]
+
+(62) HashAggregate [codegen id : 18]
+Input [2]: [i_item_id#44, sum#46]
+Keys [1]: [i_item_id#44]
+Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#39))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#39))#48]
+Results [2]: [i_item_id#44, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#39))#48,17,2) AS total_sales#49]
+
+(63) Union
+
+(64) HashAggregate [codegen id : 19]
+Input [2]: [i_item_id#14, total_sales#23]
+Keys [1]: [i_item_id#14]
+Functions [1]: [partial_sum(total_sales#23)]
+Aggregate Attributes [2]: [sum#50, isEmpty#51]
+Results [3]: [i_item_id#14, sum#52, isEmpty#53]
+
+(65) Exchange
+Input [3]: [i_item_id#14, sum#52, isEmpty#53]
+Arguments: hashpartitioning(i_item_id#14, 5), ENSURE_REQUIREMENTS, [id=#54]
+
+(66) HashAggregate [codegen id : 20]
+Input [3]: [i_item_id#14, sum#52, isEmpty#53]
+Keys [1]: [i_item_id#14]
+Functions [1]: [sum(total_sales#23)]
+Aggregate Attributes [1]: [sum(total_sales#23)#55]
+Results [2]: [i_item_id#14, sum(total_sales#23)#55 AS total_sales#56]
+
+(67) TakeOrderedAndProject
+Input [2]: [i_item_id#14, total_sales#56]
+Arguments: 100, [total_sales#56 ASC NULLS FIRST], [i_item_id#14, total_sales#56]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (68)
+
+
+(68) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#6]
+
+Subquery:2 Hosting operator id = 33 Hosting Expression = cs_sold_date_sk#27 IN dynamicpruning#5
+
+Subquery:3 Hosting operator id = 48 Hosting Expression = ws_sold_date_sk#40 IN dynamicpruning#5
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/simplified.txt
new file mode 100644
index 0000000000000..2d21010c26b8d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/simplified.txt
@@ -0,0 +1,105 @@
+TakeOrderedAndProject [total_sales,i_item_id]
+  WholeStageCodegen (20)
+    HashAggregate [i_item_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty]
+      InputAdapter
+        Exchange [i_item_id] #1
+          WholeStageCodegen (19)
+            HashAggregate [i_item_id,total_sales] [sum,isEmpty,sum,isEmpty]
+              InputAdapter
+                Union
+                  WholeStageCodegen (6)
+                    HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum]
+                      InputAdapter
+                        Exchange [i_item_id] #2
+                          WholeStageCodegen (5)
+                            HashAggregate [i_item_id,ss_ext_sales_price] [sum,sum]
+                              Project [ss_ext_sales_price,i_item_id]
+                                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                  Project [ss_item_sk,ss_ext_sales_price]
+                                    BroadcastHashJoin [ss_addr_sk,ca_address_sk]
+                                      Project [ss_item_sk,ss_addr_sk,ss_ext_sales_price]
+                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                          Filter [ss_addr_sk,ss_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                                  SubqueryBroadcast [d_date_sk] #1
+                                                    ReusedExchange [d_date_sk] #3
+                                          InputAdapter
+                                            BroadcastExchange #3
+                                              WholeStageCodegen (1)
+                                                Project [d_date_sk]
+                                                  Filter [d_year,d_moy,d_date_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                      InputAdapter
+                                        BroadcastExchange #4
+                                          WholeStageCodegen (2)
+                                            Project [ca_address_sk]
+                                              Filter [ca_gmt_offset,ca_address_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.customer_address [ca_address_sk,ca_gmt_offset]
+                                  InputAdapter
+                                    BroadcastExchange #5
+                                      WholeStageCodegen (4)
+                                        BroadcastHashJoin [i_item_id,i_item_id]
+                                          Filter [i_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.item [i_item_sk,i_item_id]
+                                          InputAdapter
+                                            BroadcastExchange #6
+                                              WholeStageCodegen (3)
+                                                Project [i_item_id]
+                                                  Filter [i_color]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.item [i_item_id,i_color]
+                  WholeStageCodegen (12)
+                    HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum]
+                      InputAdapter
+                        Exchange [i_item_id] #7
+                          WholeStageCodegen (11)
+                            HashAggregate [i_item_id,cs_ext_sales_price] [sum,sum]
+                              Project [cs_ext_sales_price,i_item_id]
+                                BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                  Project [cs_item_sk,cs_ext_sales_price]
+                                    BroadcastHashJoin [cs_bill_addr_sk,ca_address_sk]
+                                      Project [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price]
+                                        BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                          Filter [cs_bill_addr_sk,cs_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk]
+                                                  ReusedSubquery [d_date_sk] #1
+                                          InputAdapter
+                                            ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        ReusedExchange [ca_address_sk] #4
+                                  InputAdapter
+                                    ReusedExchange [i_item_sk,i_item_id] #5
+                  WholeStageCodegen (18)
+                    HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum]
+                      InputAdapter
+                        Exchange [i_item_id] #8
+                          WholeStageCodegen (17)
+                            HashAggregate [i_item_id,ws_ext_sales_price] [sum,sum]
+                              Project [ws_ext_sales_price,i_item_id]
+                                BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                  Project [ws_item_sk,ws_ext_sales_price]
+                                    BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk]
+                                      Project [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price]
+                                        BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                          Filter [ws_bill_addr_sk,ws_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk]
+                                                  ReusedSubquery [d_date_sk] #1
+                                          InputAdapter
+                                            ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        ReusedExchange [ca_address_sk] #4
+                                  InputAdapter
+                                    ReusedExchange [i_item_sk,i_item_id] #5
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.sf100/explain.txt
new file mode 100644
index 0000000000000..6e39745703215
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.sf100/explain.txt
@@ -0,0 +1,314 @@
+== Physical Plan ==
+TakeOrderedAndProject (56)
++- * Project (55)
+   +- * SortMergeJoin Inner (54)
+      :- * Project (47)
+      :  +- * SortMergeJoin Inner (46)
+      :     :- * Sort (37)
+      :     :  +- Exchange (36)
+      :     :     +- * Filter (35)
+      :     :        +- Window (34)
+      :     :           +- * Sort (33)
+      :     :              +- Exchange (32)
+      :     :                 +- * Project (31)
+      :     :                    +- Window (30)
+      :     :                       +- * Sort (29)
+      :     :                          +- Exchange (28)
+      :     :                             +- * HashAggregate (27)
+      :     :                                +- Exchange (26)
+      :     :                                   +- * HashAggregate (25)
+      :     :                                      +- * Project (24)
+      :     :                                         +- * SortMergeJoin Inner (23)
+      :     :                                            :- * Sort (17)
+      :     :                                            :  +- Exchange (16)
+      :     :                                            :     +- * Project (15)
+      :     :                                            :        +- * BroadcastHashJoin Inner BuildRight (14)
+      :     :                                            :           :- * Project (9)
+      :     :                                            :           :  +- * BroadcastHashJoin Inner BuildRight (8)
+      :     :                                            :           :     :- * Filter (3)
+      :     :                                            :           :     :  +- * ColumnarToRow (2)
+      :     :                                            :           :     :     +- Scan parquet default.catalog_sales (1)
+      :     :                                            :           :     +- BroadcastExchange (7)
+      :     :                                            :           :        +- * Filter (6)
+      :     :                                            :           :           +- * ColumnarToRow (5)
+      :     :                                            :           :              +- Scan parquet default.date_dim (4)
+      :     :                                            :           +- BroadcastExchange (13)
+      :     :                                            :              +- * Filter (12)
+      :     :                                            :                 +- * ColumnarToRow (11)
+      :     :                                            :                    +- Scan parquet default.call_center (10)
+      :     :                                            +- * Sort (22)
+      :     :                                               +- Exchange (21)
+      :     :                                                  +- * Filter (20)
+      :     :                                                     +- * ColumnarToRow (19)
+      :     :                                                        +- Scan parquet default.item (18)
+      :     +- * Sort (45)
+      :        +- Exchange (44)
+      :           +- * Project (43)
+      :              +- Window (42)
+      :                 +- * Sort (41)
+      :                    +- Exchange (40)
+      :                       +- * HashAggregate (39)
+      :                          +- ReusedExchange (38)
+      +- * Sort (53)
+         +- Exchange (52)
+            +- * Project (51)
+               +- Window (50)
+                  +- * Sort (49)
+                     +- ReusedExchange (48)
+
+
+(1) Scan parquet default.catalog_sales
+Output [4]: [cs_call_center_sk#1, cs_item_sk#2, cs_sales_price#3, cs_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#4), dynamicpruningexpression(cs_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)]
+ReadSchema: struct<cs_call_center_sk:int,cs_item_sk:int,cs_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [4]: [cs_call_center_sk#1, cs_item_sk#2, cs_sales_price#3, cs_sold_date_sk#4]
+
+(3) Filter [codegen id : 3]
+Input [4]: [cs_call_center_sk#1, cs_item_sk#2, cs_sales_price#3, cs_sold_date_sk#4]
+Condition : (isnotnull(cs_item_sk#2) AND isnotnull(cs_call_center_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Condition : ((((d_year#7 = 1999) OR ((d_year#7 = 1998) AND (d_moy#8 = 12))) OR ((d_year#7 = 2000) AND (d_moy#8 = 1))) AND isnotnull(d_date_sk#6))
+
+(7) BroadcastExchange
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#9]
+
+(8) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_sold_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(9) Project [codegen id : 3]
+Output [5]: [cs_call_center_sk#1, cs_item_sk#2, cs_sales_price#3, d_year#7, d_moy#8]
+Input [7]: [cs_call_center_sk#1, cs_item_sk#2, cs_sales_price#3, cs_sold_date_sk#4, d_date_sk#6, d_year#7, d_moy#8]
+
+(10) Scan parquet default.call_center
+Output [2]: [cc_call_center_sk#10, cc_name#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/call_center]
+PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)]
+ReadSchema: struct<cc_call_center_sk:int,cc_name:string>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [cc_call_center_sk#10, cc_name#11]
+
+(12) Filter [codegen id : 2]
+Input [2]: [cc_call_center_sk#10, cc_name#11]
+Condition : (isnotnull(cc_call_center_sk#10) AND isnotnull(cc_name#11))
+
+(13) BroadcastExchange
+Input [2]: [cc_call_center_sk#10, cc_name#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#12]
+
+(14) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_call_center_sk#1]
+Right keys [1]: [cc_call_center_sk#10]
+Join condition: None
+
+(15) Project [codegen id : 3]
+Output [5]: [cs_item_sk#2, cs_sales_price#3, d_year#7, d_moy#8, cc_name#11]
+Input [7]: [cs_call_center_sk#1, cs_item_sk#2, cs_sales_price#3, d_year#7, d_moy#8, cc_call_center_sk#10, cc_name#11]
+
+(16) Exchange
+Input [5]: [cs_item_sk#2, cs_sales_price#3, d_year#7, d_moy#8, cc_name#11]
+Arguments: hashpartitioning(cs_item_sk#2, 5), ENSURE_REQUIREMENTS, [id=#13]
+
+(17) Sort [codegen id : 4]
+Input [5]: [cs_item_sk#2, cs_sales_price#3, d_year#7, d_moy#8, cc_name#11]
+Arguments: [cs_item_sk#2 ASC NULLS FIRST], false, 0
+
+(18) Scan parquet default.item
+Output [3]: [i_item_sk#14, i_brand#15, i_category#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_category:string>
+
+(19) ColumnarToRow [codegen id : 5]
+Input [3]: [i_item_sk#14, i_brand#15, i_category#16]
+
+(20) Filter [codegen id : 5]
+Input [3]: [i_item_sk#14, i_brand#15, i_category#16]
+Condition : ((isnotnull(i_item_sk#14) AND isnotnull(i_category#16)) AND isnotnull(i_brand#15))
+
+(21) Exchange
+Input [3]: [i_item_sk#14, i_brand#15, i_category#16]
+Arguments: hashpartitioning(i_item_sk#14, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(22) Sort [codegen id : 6]
+Input [3]: [i_item_sk#14, i_brand#15, i_category#16]
+Arguments: [i_item_sk#14 ASC NULLS FIRST], false, 0
+
+(23) SortMergeJoin [codegen id : 7]
+Left keys [1]: [cs_item_sk#2]
+Right keys [1]: [i_item_sk#14]
+Join condition: None
+
+(24) Project [codegen id : 7]
+Output [6]: [i_brand#15, i_category#16, cs_sales_price#3, d_year#7, d_moy#8, cc_name#11]
+Input [8]: [cs_item_sk#2, cs_sales_price#3, d_year#7, d_moy#8, cc_name#11, i_item_sk#14, i_brand#15, i_category#16]
+
+(25) HashAggregate [codegen id : 7]
+Input [6]: [i_brand#15, i_category#16, cs_sales_price#3, d_year#7, d_moy#8, cc_name#11]
+Keys [5]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8]
+Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#3))]
+Aggregate Attributes [1]: [sum#18]
+Results [6]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum#19]
+
+(26) Exchange
+Input [6]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum#19]
+Arguments: hashpartitioning(i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(27) HashAggregate [codegen id : 8]
+Input [6]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum#19]
+Keys [5]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8]
+Functions [1]: [sum(UnscaledValue(cs_sales_price#3))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#3))#21]
+Results [7]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, MakeDecimal(sum(UnscaledValue(cs_sales_price#3))#21,17,2) AS sum_sales#22, MakeDecimal(sum(UnscaledValue(cs_sales_price#3))#21,17,2) AS _w0#23]
+
+(28) Exchange
+Input [7]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, _w0#23]
+Arguments: hashpartitioning(i_category#16, i_brand#15, cc_name#11, d_year#7, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(29) Sort [codegen id : 9]
+Input [7]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, _w0#23]
+Arguments: [i_category#16 ASC NULLS FIRST, i_brand#15 ASC NULLS FIRST, cc_name#11 ASC NULLS FIRST, d_year#7 ASC NULLS FIRST], false, 0
+
+(30) Window
+Input [7]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, _w0#23]
+Arguments: [avg(_w0#23) windowspecdefinition(i_category#16, i_brand#15, cc_name#11, d_year#7, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#25], [i_category#16, i_brand#15, cc_name#11, d_year#7]
+
+(31) Project [codegen id : 10]
+Output [7]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, avg_monthly_sales#25]
+Input [8]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, _w0#23, avg_monthly_sales#25]
+
+(32) Exchange
+Input [7]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, avg_monthly_sales#25]
+Arguments: hashpartitioning(i_category#16, i_brand#15, cc_name#11, 5), ENSURE_REQUIREMENTS, [id=#26]
+
+(33) Sort [codegen id : 11]
+Input [7]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, avg_monthly_sales#25]
+Arguments: [i_category#16 ASC NULLS FIRST, i_brand#15 ASC NULLS FIRST, cc_name#11 ASC NULLS FIRST, d_year#7 ASC NULLS FIRST, d_moy#8 ASC NULLS FIRST], false, 0
+
+(34) Window
+Input [7]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, avg_monthly_sales#25]
+Arguments: [rank(d_year#7, d_moy#8) windowspecdefinition(i_category#16, i_brand#15, cc_name#11, d_year#7 ASC NULLS FIRST, d_moy#8 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#16, i_brand#15, cc_name#11], [d_year#7 ASC NULLS FIRST, d_moy#8 ASC NULLS FIRST]
+
+(35) Filter [codegen id : 12]
+Input [8]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, avg_monthly_sales#25, rn#27]
+Condition : ((((isnotnull(d_year#7) AND isnotnull(avg_monthly_sales#25)) AND (d_year#7 = 1999)) AND (avg_monthly_sales#25 > 0.000000)) AND (CheckOverflow((promote_precision(abs(CheckOverflow((promote_precision(cast(sum_sales#22 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#25 as decimal(22,6)))), DecimalType(22,6), true), false)) / promote_precision(cast(avg_monthly_sales#25 as decimal(22,6)))), DecimalType(38,16), true) > 0.1000000000000000))
+
+(36) Exchange
+Input [8]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, avg_monthly_sales#25, rn#27]
+Arguments: hashpartitioning(i_category#16, i_brand#15, cc_name#11, rn#27, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(37) Sort [codegen id : 13]
+Input [8]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, avg_monthly_sales#25, rn#27]
+Arguments: [i_category#16 ASC NULLS FIRST, i_brand#15 ASC NULLS FIRST, cc_name#11 ASC NULLS FIRST, rn#27 ASC NULLS FIRST], false, 0
+
+(38) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum#34]
+
+(39) HashAggregate [codegen id : 21]
+Input [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum#34]
+Keys [5]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33]
+Functions [1]: [sum(UnscaledValue(cs_sales_price#35))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#35))#36]
+Results [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, MakeDecimal(sum(UnscaledValue(cs_sales_price#35))#36,17,2) AS sum_sales#37]
+
+(40) Exchange
+Input [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#37]
+Arguments: hashpartitioning(i_category#29, i_brand#30, cc_name#31, 5), ENSURE_REQUIREMENTS, [id=#38]
+
+(41) Sort [codegen id : 22]
+Input [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#37]
+Arguments: [i_category#29 ASC NULLS FIRST, i_brand#30 ASC NULLS FIRST, cc_name#31 ASC NULLS FIRST, d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST], false, 0
+
+(42) Window
+Input [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#37]
+Arguments: [rank(d_year#32, d_moy#33) windowspecdefinition(i_category#29, i_brand#30, cc_name#31, d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#39], [i_category#29, i_brand#30, cc_name#31], [d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST]
+
+(43) Project [codegen id : 23]
+Output [5]: [i_category#29, i_brand#30, cc_name#31, sum_sales#37, rn#39]
+Input [7]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#37, rn#39]
+
+(44) Exchange
+Input [5]: [i_category#29, i_brand#30, cc_name#31, sum_sales#37, rn#39]
+Arguments: hashpartitioning(i_category#29, i_brand#30, cc_name#31, (rn#39 + 1), 5), ENSURE_REQUIREMENTS, [id=#40]
+
+(45) Sort [codegen id : 24]
+Input [5]: [i_category#29, i_brand#30, cc_name#31, sum_sales#37, rn#39]
+Arguments: [i_category#29 ASC NULLS FIRST, i_brand#30 ASC NULLS FIRST, cc_name#31 ASC NULLS FIRST, (rn#39 + 1) ASC NULLS FIRST], false, 0
+
+(46) SortMergeJoin [codegen id : 25]
+Left keys [4]: [i_category#16, i_brand#15, cc_name#11, rn#27]
+Right keys [4]: [i_category#29, i_brand#30, cc_name#31, (rn#39 + 1)]
+Join condition: None
+
+(47) Project [codegen id : 25]
+Output [9]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, avg_monthly_sales#25, rn#27, sum_sales#37]
+Input [13]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, avg_monthly_sales#25, rn#27, i_category#29, i_brand#30, cc_name#31, sum_sales#37, rn#39]
+
+(48) ReusedExchange [Reuses operator id: 40]
+Output [6]: [i_category#41, i_brand#42, cc_name#43, d_year#44, d_moy#45, sum_sales#46]
+
+(49) Sort [codegen id : 34]
+Input [6]: [i_category#41, i_brand#42, cc_name#43, d_year#44, d_moy#45, sum_sales#46]
+Arguments: [i_category#41 ASC NULLS FIRST, i_brand#42 ASC NULLS FIRST, cc_name#43 ASC NULLS FIRST, d_year#44 ASC NULLS FIRST, d_moy#45 ASC NULLS FIRST], false, 0
+
+(50) Window
+Input [6]: [i_category#41, i_brand#42, cc_name#43, d_year#44, d_moy#45, sum_sales#46]
+Arguments: [rank(d_year#44, d_moy#45) windowspecdefinition(i_category#41, i_brand#42, cc_name#43, d_year#44 ASC NULLS FIRST, d_moy#45 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#47], [i_category#41, i_brand#42, cc_name#43], [d_year#44 ASC NULLS FIRST, d_moy#45 ASC NULLS FIRST]
+
+(51) Project [codegen id : 35]
+Output [5]: [i_category#41, i_brand#42, cc_name#43, sum_sales#46, rn#47]
+Input [7]: [i_category#41, i_brand#42, cc_name#43, d_year#44, d_moy#45, sum_sales#46, rn#47]
+
+(52) Exchange
+Input [5]: [i_category#41, i_brand#42, cc_name#43, sum_sales#46, rn#47]
+Arguments: hashpartitioning(i_category#41, i_brand#42, cc_name#43, (rn#47 - 1), 5), ENSURE_REQUIREMENTS, [id=#48]
+
+(53) Sort [codegen id : 36]
+Input [5]: [i_category#41, i_brand#42, cc_name#43, sum_sales#46, rn#47]
+Arguments: [i_category#41 ASC NULLS FIRST, i_brand#42 ASC NULLS FIRST, cc_name#43 ASC NULLS FIRST, (rn#47 - 1) ASC NULLS FIRST], false, 0
+
+(54) SortMergeJoin [codegen id : 37]
+Left keys [4]: [i_category#16, i_brand#15, cc_name#11, rn#27]
+Right keys [4]: [i_category#41, i_brand#42, cc_name#43, (rn#47 - 1)]
+Join condition: None
+
+(55) Project [codegen id : 37]
+Output [9]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, avg_monthly_sales#25, sum_sales#22, sum_sales#37 AS psum#49, sum_sales#46 AS nsum#50]
+Input [14]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, avg_monthly_sales#25, rn#27, sum_sales#37, i_category#41, i_brand#42, cc_name#43, sum_sales#46, rn#47]
+
+(56) TakeOrderedAndProject
+Input [9]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, avg_monthly_sales#25, sum_sales#22, psum#49, nsum#50]
+Arguments: 100, [CheckOverflow((promote_precision(cast(sum_sales#22 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#25 as decimal(22,6)))), DecimalType(22,6), true) ASC NULLS FIRST, cc_name#11 ASC NULLS FIRST], [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, avg_monthly_sales#25, sum_sales#22, psum#49, nsum#50]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (57)
+
+
+(57) ReusedExchange [Reuses operator id: 7]
+Output [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.sf100/simplified.txt
new file mode 100644
index 0000000000000..bd84b91e1ac12
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.sf100/simplified.txt
@@ -0,0 +1,107 @@
+TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_year,d_moy,psum,nsum]
+  WholeStageCodegen (37)
+    Project [i_category,i_brand,cc_name,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales]
+      SortMergeJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn]
+        InputAdapter
+          WholeStageCodegen (25)
+            Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales]
+              SortMergeJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn]
+                InputAdapter
+                  WholeStageCodegen (13)
+                    Sort [i_category,i_brand,cc_name,rn]
+                      InputAdapter
+                        Exchange [i_category,i_brand,cc_name,rn] #1
+                          WholeStageCodegen (12)
+                            Filter [d_year,avg_monthly_sales,sum_sales]
+                              InputAdapter
+                                Window [d_year,d_moy,i_category,i_brand,cc_name]
+                                  WholeStageCodegen (11)
+                                    Sort [i_category,i_brand,cc_name,d_year,d_moy]
+                                      InputAdapter
+                                        Exchange [i_category,i_brand,cc_name] #2
+                                          WholeStageCodegen (10)
+                                            Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales]
+                                              InputAdapter
+                                                Window [_w0,i_category,i_brand,cc_name,d_year]
+                                                  WholeStageCodegen (9)
+                                                    Sort [i_category,i_brand,cc_name,d_year]
+                                                      InputAdapter
+                                                        Exchange [i_category,i_brand,cc_name,d_year] #3
+                                                          WholeStageCodegen (8)
+                                                            HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,_w0,sum]
+                                                              InputAdapter
+                                                                Exchange [i_category,i_brand,cc_name,d_year,d_moy] #4
+                                                                  WholeStageCodegen (7)
+                                                                    HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,cs_sales_price] [sum,sum]
+                                                                      Project [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name]
+                                                                        SortMergeJoin [cs_item_sk,i_item_sk]
+                                                                          InputAdapter
+                                                                            WholeStageCodegen (4)
+                                                                              Sort [cs_item_sk]
+                                                                                InputAdapter
+                                                                                  Exchange [cs_item_sk] #5
+                                                                                    WholeStageCodegen (3)
+                                                                                      Project [cs_item_sk,cs_sales_price,d_year,d_moy,cc_name]
+                                                                                        BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk]
+                                                                                          Project [cs_call_center_sk,cs_item_sk,cs_sales_price,d_year,d_moy]
+                                                                                            BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                                              Filter [cs_item_sk,cs_call_center_sk]
+                                                                                                ColumnarToRow
+                                                                                                  InputAdapter
+                                                                                                    Scan parquet default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk]
+                                                                                                      SubqueryBroadcast [d_date_sk] #1
+                                                                                                        ReusedExchange [d_date_sk,d_year,d_moy] #6
+                                                                                              InputAdapter
+                                                                                                BroadcastExchange #6
+                                                                                                  WholeStageCodegen (1)
+                                                                                                    Filter [d_year,d_moy,d_date_sk]
+                                                                                                      ColumnarToRow
+                                                                                                        InputAdapter
+                                                                                                          Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                                                                          InputAdapter
+                                                                                            BroadcastExchange #7
+                                                                                              WholeStageCodegen (2)
+                                                                                                Filter [cc_call_center_sk,cc_name]
+                                                                                                  ColumnarToRow
+                                                                                                    InputAdapter
+                                                                                                      Scan parquet default.call_center [cc_call_center_sk,cc_name]
+                                                                          InputAdapter
+                                                                            WholeStageCodegen (6)
+                                                                              Sort [i_item_sk]
+                                                                                InputAdapter
+                                                                                  Exchange [i_item_sk] #8
+                                                                                    WholeStageCodegen (5)
+                                                                                      Filter [i_item_sk,i_category,i_brand]
+                                                                                        ColumnarToRow
+                                                                                          InputAdapter
+                                                                                            Scan parquet default.item [i_item_sk,i_brand,i_category]
+                InputAdapter
+                  WholeStageCodegen (24)
+                    Sort [i_category,i_brand,cc_name,rn]
+                      InputAdapter
+                        Exchange [i_category,i_brand,cc_name,rn] #9
+                          WholeStageCodegen (23)
+                            Project [i_category,i_brand,cc_name,sum_sales,rn]
+                              InputAdapter
+                                Window [d_year,d_moy,i_category,i_brand,cc_name]
+                                  WholeStageCodegen (22)
+                                    Sort [i_category,i_brand,cc_name,d_year,d_moy]
+                                      InputAdapter
+                                        Exchange [i_category,i_brand,cc_name] #10
+                                          WholeStageCodegen (21)
+                                            HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,sum]
+                                              InputAdapter
+                                                ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #11
+        InputAdapter
+          WholeStageCodegen (36)
+            Sort [i_category,i_brand,cc_name,rn]
+              InputAdapter
+                Exchange [i_category,i_brand,cc_name,rn] #12
+                  WholeStageCodegen (35)
+                    Project [i_category,i_brand,cc_name,sum_sales,rn]
+                      InputAdapter
+                        Window [d_year,d_moy,i_category,i_brand,cc_name]
+                          WholeStageCodegen (34)
+                            Sort [i_category,i_brand,cc_name,d_year,d_moy]
+                              InputAdapter
+                                ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #10
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/explain.txt
new file mode 100644
index 0000000000000..fa24f5ede5453
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/explain.txt
@@ -0,0 +1,279 @@
+== Physical Plan ==
+TakeOrderedAndProject (49)
++- * Project (48)
+   +- * BroadcastHashJoin Inner BuildRight (47)
+      :- * Project (41)
+      :  +- * BroadcastHashJoin Inner BuildRight (40)
+      :     :- * Filter (32)
+      :     :  +- Window (31)
+      :     :     +- * Sort (30)
+      :     :        +- Exchange (29)
+      :     :           +- * Project (28)
+      :     :              +- Window (27)
+      :     :                 +- * Sort (26)
+      :     :                    +- Exchange (25)
+      :     :                       +- * HashAggregate (24)
+      :     :                          +- Exchange (23)
+      :     :                             +- * HashAggregate (22)
+      :     :                                +- * Project (21)
+      :     :                                   +- * BroadcastHashJoin Inner BuildRight (20)
+      :     :                                      :- * Project (15)
+      :     :                                      :  +- * BroadcastHashJoin Inner BuildRight (14)
+      :     :                                      :     :- * Project (9)
+      :     :                                      :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+      :     :                                      :     :     :- * Filter (3)
+      :     :                                      :     :     :  +- * ColumnarToRow (2)
+      :     :                                      :     :     :     +- Scan parquet default.item (1)
+      :     :                                      :     :     +- BroadcastExchange (7)
+      :     :                                      :     :        +- * Filter (6)
+      :     :                                      :     :           +- * ColumnarToRow (5)
+      :     :                                      :     :              +- Scan parquet default.catalog_sales (4)
+      :     :                                      :     +- BroadcastExchange (13)
+      :     :                                      :        +- * Filter (12)
+      :     :                                      :           +- * ColumnarToRow (11)
+      :     :                                      :              +- Scan parquet default.date_dim (10)
+      :     :                                      +- BroadcastExchange (19)
+      :     :                                         +- * Filter (18)
+      :     :                                            +- * ColumnarToRow (17)
+      :     :                                               +- Scan parquet default.call_center (16)
+      :     +- BroadcastExchange (39)
+      :        +- * Project (38)
+      :           +- Window (37)
+      :              +- * Sort (36)
+      :                 +- Exchange (35)
+      :                    +- * HashAggregate (34)
+      :                       +- ReusedExchange (33)
+      +- BroadcastExchange (46)
+         +- * Project (45)
+            +- Window (44)
+               +- * Sort (43)
+                  +- ReusedExchange (42)
+
+
+(1) Scan parquet default.item
+Output [3]: [i_item_sk#1, i_brand#2, i_category#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_category:string>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [3]: [i_item_sk#1, i_brand#2, i_category#3]
+
+(3) Filter [codegen id : 4]
+Input [3]: [i_item_sk#1, i_brand#2, i_category#3]
+Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2))
+
+(4) Scan parquet default.catalog_sales
+Output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#7), dynamicpruningexpression(cs_sold_date_sk#7 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)]
+ReadSchema: struct<cs_call_center_sk:int,cs_item_sk:int,cs_sales_price:decimal(7,2)>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7]
+
+(6) Filter [codegen id : 1]
+Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7]
+Condition : (isnotnull(cs_item_sk#5) AND isnotnull(cs_call_center_sk#4))
+
+(7) BroadcastExchange
+Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [id=#9]
+
+(8) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [cs_item_sk#5]
+Join condition: None
+
+(9) Project [codegen id : 4]
+Output [5]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7]
+Input [7]: [i_item_sk#1, i_brand#2, i_category#3, cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7]
+
+(10) Scan parquet default.date_dim
+Output [3]: [d_date_sk#10, d_year#11, d_moy#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+
+(12) Filter [codegen id : 2]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10))
+
+(13) BroadcastExchange
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#13]
+
+(14) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_sold_date_sk#7]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(15) Project [codegen id : 4]
+Output [6]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#11, d_moy#12]
+Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7, d_date_sk#10, d_year#11, d_moy#12]
+
+(16) Scan parquet default.call_center
+Output [2]: [cc_call_center_sk#14, cc_name#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/call_center]
+PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)]
+ReadSchema: struct<cc_call_center_sk:int,cc_name:string>
+
+(17) ColumnarToRow [codegen id : 3]
+Input [2]: [cc_call_center_sk#14, cc_name#15]
+
+(18) Filter [codegen id : 3]
+Input [2]: [cc_call_center_sk#14, cc_name#15]
+Condition : (isnotnull(cc_call_center_sk#14) AND isnotnull(cc_name#15))
+
+(19) BroadcastExchange
+Input [2]: [cc_call_center_sk#14, cc_name#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#16]
+
+(20) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_call_center_sk#4]
+Right keys [1]: [cc_call_center_sk#14]
+Join condition: None
+
+(21) Project [codegen id : 4]
+Output [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#11, d_moy#12, cc_name#15]
+Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#11, d_moy#12, cc_call_center_sk#14, cc_name#15]
+
+(22) HashAggregate [codegen id : 4]
+Input [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#11, d_moy#12, cc_name#15]
+Keys [5]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12]
+Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#6))]
+Aggregate Attributes [1]: [sum#17]
+Results [6]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum#18]
+
+(23) Exchange
+Input [6]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum#18]
+Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, [id=#19]
+
+(24) HashAggregate [codegen id : 5]
+Input [6]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum#18]
+Keys [5]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12]
+Functions [1]: [sum(UnscaledValue(cs_sales_price#6))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#6))#20]
+Results [7]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#20,17,2) AS sum_sales#21, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#20,17,2) AS _w0#22]
+
+(25) Exchange
+Input [7]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum_sales#21, _w0#22]
+Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#15, d_year#11, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(26) Sort [codegen id : 6]
+Input [7]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum_sales#21, _w0#22]
+Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, cc_name#15 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST], false, 0
+
+(27) Window
+Input [7]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum_sales#21, _w0#22]
+Arguments: [avg(_w0#22) windowspecdefinition(i_category#3, i_brand#2, cc_name#15, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#24], [i_category#3, i_brand#2, cc_name#15, d_year#11]
+
+(28) Project [codegen id : 7]
+Output [7]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum_sales#21, avg_monthly_sales#24]
+Input [8]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum_sales#21, _w0#22, avg_monthly_sales#24]
+
+(29) Exchange
+Input [7]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum_sales#21, avg_monthly_sales#24]
+Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#15, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(30) Sort [codegen id : 8]
+Input [7]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum_sales#21, avg_monthly_sales#24]
+Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, cc_name#15 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST], false, 0
+
+(31) Window
+Input [7]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum_sales#21, avg_monthly_sales#24]
+Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#3, i_brand#2, cc_name#15, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#26], [i_category#3, i_brand#2, cc_name#15], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST]
+
+(32) Filter [codegen id : 23]
+Input [8]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum_sales#21, avg_monthly_sales#24, rn#26]
+Condition : ((((isnotnull(d_year#11) AND isnotnull(avg_monthly_sales#24)) AND (d_year#11 = 1999)) AND (avg_monthly_sales#24 > 0.000000)) AND (CheckOverflow((promote_precision(abs(CheckOverflow((promote_precision(cast(sum_sales#21 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#24 as decimal(22,6)))), DecimalType(22,6), true), false)) / promote_precision(cast(avg_monthly_sales#24 as decimal(22,6)))), DecimalType(38,16), true) > 0.1000000000000000))
+
+(33) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum#32]
+
+(34) HashAggregate [codegen id : 13]
+Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum#32]
+Keys [5]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31]
+Functions [1]: [sum(UnscaledValue(cs_sales_price#33))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#33))#34]
+Results [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, MakeDecimal(sum(UnscaledValue(cs_sales_price#33))#34,17,2) AS sum_sales#35]
+
+(35) Exchange
+Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#35]
+Arguments: hashpartitioning(i_category#27, i_brand#28, cc_name#29, 5), ENSURE_REQUIREMENTS, [id=#36]
+
+(36) Sort [codegen id : 14]
+Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#35]
+Arguments: [i_category#27 ASC NULLS FIRST, i_brand#28 ASC NULLS FIRST, cc_name#29 ASC NULLS FIRST, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST], false, 0
+
+(37) Window
+Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#35]
+Arguments: [rank(d_year#30, d_moy#31) windowspecdefinition(i_category#27, i_brand#28, cc_name#29, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#37], [i_category#27, i_brand#28, cc_name#29], [d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST]
+
+(38) Project [codegen id : 15]
+Output [5]: [i_category#27, i_brand#28, cc_name#29, sum_sales#35, rn#37]
+Input [7]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#35, rn#37]
+
+(39) BroadcastExchange
+Input [5]: [i_category#27, i_brand#28, cc_name#29, sum_sales#35, rn#37]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [id=#38]
+
+(40) BroadcastHashJoin [codegen id : 23]
+Left keys [4]: [i_category#3, i_brand#2, cc_name#15, rn#26]
+Right keys [4]: [i_category#27, i_brand#28, cc_name#29, (rn#37 + 1)]
+Join condition: None
+
+(41) Project [codegen id : 23]
+Output [9]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum_sales#21, avg_monthly_sales#24, rn#26, sum_sales#35]
+Input [13]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum_sales#21, avg_monthly_sales#24, rn#26, i_category#27, i_brand#28, cc_name#29, sum_sales#35, rn#37]
+
+(42) ReusedExchange [Reuses operator id: 35]
+Output [6]: [i_category#39, i_brand#40, cc_name#41, d_year#42, d_moy#43, sum_sales#44]
+
+(43) Sort [codegen id : 21]
+Input [6]: [i_category#39, i_brand#40, cc_name#41, d_year#42, d_moy#43, sum_sales#44]
+Arguments: [i_category#39 ASC NULLS FIRST, i_brand#40 ASC NULLS FIRST, cc_name#41 ASC NULLS FIRST, d_year#42 ASC NULLS FIRST, d_moy#43 ASC NULLS FIRST], false, 0
+
+(44) Window
+Input [6]: [i_category#39, i_brand#40, cc_name#41, d_year#42, d_moy#43, sum_sales#44]
+Arguments: [rank(d_year#42, d_moy#43) windowspecdefinition(i_category#39, i_brand#40, cc_name#41, d_year#42 ASC NULLS FIRST, d_moy#43 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#45], [i_category#39, i_brand#40, cc_name#41], [d_year#42 ASC NULLS FIRST, d_moy#43 ASC NULLS FIRST]
+
+(45) Project [codegen id : 22]
+Output [5]: [i_category#39, i_brand#40, cc_name#41, sum_sales#44, rn#45]
+Input [7]: [i_category#39, i_brand#40, cc_name#41, d_year#42, d_moy#43, sum_sales#44, rn#45]
+
+(46) BroadcastExchange
+Input [5]: [i_category#39, i_brand#40, cc_name#41, sum_sales#44, rn#45]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [id=#46]
+
+(47) BroadcastHashJoin [codegen id : 23]
+Left keys [4]: [i_category#3, i_brand#2, cc_name#15, rn#26]
+Right keys [4]: [i_category#39, i_brand#40, cc_name#41, (rn#45 - 1)]
+Join condition: None
+
+(48) Project [codegen id : 23]
+Output [9]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, avg_monthly_sales#24, sum_sales#21, sum_sales#35 AS psum#47, sum_sales#44 AS nsum#48]
+Input [14]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum_sales#21, avg_monthly_sales#24, rn#26, sum_sales#35, i_category#39, i_brand#40, cc_name#41, sum_sales#44, rn#45]
+
+(49) TakeOrderedAndProject
+Input [9]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, avg_monthly_sales#24, sum_sales#21, psum#47, nsum#48]
+Arguments: 100, [CheckOverflow((promote_precision(cast(sum_sales#21 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#24 as decimal(22,6)))), DecimalType(22,6), true) ASC NULLS FIRST, cc_name#15 ASC NULLS FIRST], [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, avg_monthly_sales#24, sum_sales#21, psum#47, nsum#48]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 4 Hosting Expression = cs_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (50)
+
+
+(50) ReusedExchange [Reuses operator id: 13]
+Output [3]: [d_date_sk#10, d_year#11, d_moy#12]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/simplified.txt
new file mode 100644
index 0000000000000..e9e7cf96303c3
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/simplified.txt
@@ -0,0 +1,84 @@
+TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_year,d_moy,psum,nsum]
+  WholeStageCodegen (23)
+    Project [i_category,i_brand,cc_name,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales]
+      BroadcastHashJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn]
+        Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales]
+          BroadcastHashJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn]
+            Filter [d_year,avg_monthly_sales,sum_sales]
+              InputAdapter
+                Window [d_year,d_moy,i_category,i_brand,cc_name]
+                  WholeStageCodegen (8)
+                    Sort [i_category,i_brand,cc_name,d_year,d_moy]
+                      InputAdapter
+                        Exchange [i_category,i_brand,cc_name] #1
+                          WholeStageCodegen (7)
+                            Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales]
+                              InputAdapter
+                                Window [_w0,i_category,i_brand,cc_name,d_year]
+                                  WholeStageCodegen (6)
+                                    Sort [i_category,i_brand,cc_name,d_year]
+                                      InputAdapter
+                                        Exchange [i_category,i_brand,cc_name,d_year] #2
+                                          WholeStageCodegen (5)
+                                            HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,_w0,sum]
+                                              InputAdapter
+                                                Exchange [i_category,i_brand,cc_name,d_year,d_moy] #3
+                                                  WholeStageCodegen (4)
+                                                    HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,cs_sales_price] [sum,sum]
+                                                      Project [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name]
+                                                        BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk]
+                                                          Project [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy]
+                                                            BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                              Project [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk]
+                                                                BroadcastHashJoin [i_item_sk,cs_item_sk]
+                                                                  Filter [i_item_sk,i_category,i_brand]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.item [i_item_sk,i_brand,i_category]
+                                                                  InputAdapter
+                                                                    BroadcastExchange #4
+                                                                      WholeStageCodegen (1)
+                                                                        Filter [cs_item_sk,cs_call_center_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk]
+                                                                                SubqueryBroadcast [d_date_sk] #1
+                                                                                  ReusedExchange [d_date_sk,d_year,d_moy] #5
+                                                              InputAdapter
+                                                                BroadcastExchange #5
+                                                                  WholeStageCodegen (2)
+                                                                    Filter [d_year,d_moy,d_date_sk]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                                          InputAdapter
+                                                            BroadcastExchange #6
+                                                              WholeStageCodegen (3)
+                                                                Filter [cc_call_center_sk,cc_name]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.call_center [cc_call_center_sk,cc_name]
+            InputAdapter
+              BroadcastExchange #7
+                WholeStageCodegen (15)
+                  Project [i_category,i_brand,cc_name,sum_sales,rn]
+                    InputAdapter
+                      Window [d_year,d_moy,i_category,i_brand,cc_name]
+                        WholeStageCodegen (14)
+                          Sort [i_category,i_brand,cc_name,d_year,d_moy]
+                            InputAdapter
+                              Exchange [i_category,i_brand,cc_name] #8
+                                WholeStageCodegen (13)
+                                  HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,sum]
+                                    InputAdapter
+                                      ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #9
+        InputAdapter
+          BroadcastExchange #10
+            WholeStageCodegen (22)
+              Project [i_category,i_brand,cc_name,sum_sales,rn]
+                InputAdapter
+                  Window [d_year,d_moy,i_category,i_brand,cc_name]
+                    WholeStageCodegen (21)
+                      Sort [i_category,i_brand,cc_name,d_year,d_moy]
+                        InputAdapter
+                          ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #8
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.sf100/explain.txt
new file mode 100644
index 0000000000000..b339df0707d2f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.sf100/explain.txt
@@ -0,0 +1,480 @@
+== Physical Plan ==
+TakeOrderedAndProject (79)
++- * Project (78)
+   +- * BroadcastHashJoin Inner BuildRight (77)
+      :- * Project (52)
+      :  +- * BroadcastHashJoin Inner BuildRight (51)
+      :     :- * Filter (26)
+      :     :  +- * HashAggregate (25)
+      :     :     +- Exchange (24)
+      :     :        +- * HashAggregate (23)
+      :     :           +- * Project (22)
+      :     :              +- * BroadcastHashJoin Inner BuildRight (21)
+      :     :                 :- * Project (16)
+      :     :                 :  +- * BroadcastHashJoin Inner BuildRight (15)
+      :     :                 :     :- * Filter (3)
+      :     :                 :     :  +- * ColumnarToRow (2)
+      :     :                 :     :     +- Scan parquet default.store_sales (1)
+      :     :                 :     +- BroadcastExchange (14)
+      :     :                 :        +- * Project (13)
+      :     :                 :           +- * BroadcastHashJoin LeftSemi BuildRight (12)
+      :     :                 :              :- * Filter (6)
+      :     :                 :              :  +- * ColumnarToRow (5)
+      :     :                 :              :     +- Scan parquet default.date_dim (4)
+      :     :                 :              +- BroadcastExchange (11)
+      :     :                 :                 +- * Project (10)
+      :     :                 :                    +- * Filter (9)
+      :     :                 :                       +- * ColumnarToRow (8)
+      :     :                 :                          +- Scan parquet default.date_dim (7)
+      :     :                 +- BroadcastExchange (20)
+      :     :                    +- * Filter (19)
+      :     :                       +- * ColumnarToRow (18)
+      :     :                          +- Scan parquet default.item (17)
+      :     +- BroadcastExchange (50)
+      :        +- * Filter (49)
+      :           +- * HashAggregate (48)
+      :              +- Exchange (47)
+      :                 +- * HashAggregate (46)
+      :                    +- * Project (45)
+      :                       +- * BroadcastHashJoin Inner BuildRight (44)
+      :                          :- * Project (42)
+      :                          :  +- * BroadcastHashJoin Inner BuildRight (41)
+      :                          :     :- * Filter (29)
+      :                          :     :  +- * ColumnarToRow (28)
+      :                          :     :     +- Scan parquet default.catalog_sales (27)
+      :                          :     +- BroadcastExchange (40)
+      :                          :        +- * Project (39)
+      :                          :           +- * BroadcastHashJoin LeftSemi BuildRight (38)
+      :                          :              :- * Filter (32)
+      :                          :              :  +- * ColumnarToRow (31)
+      :                          :              :     +- Scan parquet default.date_dim (30)
+      :                          :              +- BroadcastExchange (37)
+      :                          :                 +- * Project (36)
+      :                          :                    +- * Filter (35)
+      :                          :                       +- * ColumnarToRow (34)
+      :                          :                          +- Scan parquet default.date_dim (33)
+      :                          +- ReusedExchange (43)
+      +- BroadcastExchange (76)
+         +- * Filter (75)
+            +- * HashAggregate (74)
+               +- Exchange (73)
+                  +- * HashAggregate (72)
+                     +- * Project (71)
+                        +- * BroadcastHashJoin Inner BuildRight (70)
+                           :- * Project (68)
+                           :  +- * BroadcastHashJoin Inner BuildRight (67)
+                           :     :- * Filter (55)
+                           :     :  +- * ColumnarToRow (54)
+                           :     :     +- Scan parquet default.web_sales (53)
+                           :     +- BroadcastExchange (66)
+                           :        +- * Project (65)
+                           :           +- * BroadcastHashJoin LeftSemi BuildRight (64)
+                           :              :- * Filter (58)
+                           :              :  +- * ColumnarToRow (57)
+                           :              :     +- Scan parquet default.date_dim (56)
+                           :              +- BroadcastExchange (63)
+                           :                 +- * Project (62)
+                           :                    +- * Filter (61)
+                           :                       +- * ColumnarToRow (60)
+                           :                          +- Scan parquet default.date_dim (59)
+                           +- ReusedExchange (69)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 4]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_item_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#4, d_date#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(5) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#4, d_date#5]
+
+(6) Filter [codegen id : 2]
+Input [2]: [d_date_sk#4, d_date#5]
+Condition : isnotnull(d_date_sk#4)
+
+(7) Scan parquet default.date_dim
+Output [2]: [d_date#6, d_week_seq#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_week_seq)]
+ReadSchema: struct<d_date:date,d_week_seq:int>
+
+(8) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date#6, d_week_seq#7]
+
+(9) Filter [codegen id : 1]
+Input [2]: [d_date#6, d_week_seq#7]
+Condition : (isnotnull(d_week_seq#7) AND (d_week_seq#7 = Subquery scalar-subquery#8, [id=#9]))
+
+(10) Project [codegen id : 1]
+Output [1]: [d_date#6]
+Input [2]: [d_date#6, d_week_seq#7]
+
+(11) BroadcastExchange
+Input [1]: [d_date#6]
+Arguments: HashedRelationBroadcastMode(List(input[0, date, true]),false), [id=#10]
+
+(12) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [d_date#5]
+Right keys [1]: [d_date#6]
+Join condition: None
+
+(13) Project [codegen id : 2]
+Output [1]: [d_date_sk#4]
+Input [2]: [d_date_sk#4, d_date#5]
+
+(14) BroadcastExchange
+Input [1]: [d_date_sk#4]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#11]
+
+(15) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#4]
+Join condition: None
+
+(16) Project [codegen id : 4]
+Output [2]: [ss_item_sk#1, ss_ext_sales_price#2]
+Input [4]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, d_date_sk#4]
+
+(17) Scan parquet default.item
+Output [2]: [i_item_sk#12, i_item_id#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_item_id)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [2]: [i_item_sk#12, i_item_id#13]
+
+(19) Filter [codegen id : 3]
+Input [2]: [i_item_sk#12, i_item_id#13]
+Condition : (isnotnull(i_item_sk#12) AND isnotnull(i_item_id#13))
+
+(20) BroadcastExchange
+Input [2]: [i_item_sk#12, i_item_id#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#14]
+
+(21) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#12]
+Join condition: None
+
+(22) Project [codegen id : 4]
+Output [2]: [ss_ext_sales_price#2, i_item_id#13]
+Input [4]: [ss_item_sk#1, ss_ext_sales_price#2, i_item_sk#12, i_item_id#13]
+
+(23) HashAggregate [codegen id : 4]
+Input [2]: [ss_ext_sales_price#2, i_item_id#13]
+Keys [1]: [i_item_id#13]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum#15]
+Results [2]: [i_item_id#13, sum#16]
+
+(24) Exchange
+Input [2]: [i_item_id#13, sum#16]
+Arguments: hashpartitioning(i_item_id#13, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(25) HashAggregate [codegen id : 15]
+Input [2]: [i_item_id#13, sum#16]
+Keys [1]: [i_item_id#13]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#18]
+Results [2]: [i_item_id#13 AS item_id#19, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#18,17,2) AS ss_item_rev#20]
+
+(26) Filter [codegen id : 15]
+Input [2]: [item_id#19, ss_item_rev#20]
+Condition : isnotnull(ss_item_rev#20)
+
+(27) Scan parquet default.catalog_sales
+Output [3]: [cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#23), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int,cs_ext_sales_price:decimal(7,2)>
+
+(28) ColumnarToRow [codegen id : 8]
+Input [3]: [cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23]
+
+(29) Filter [codegen id : 8]
+Input [3]: [cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23]
+Condition : isnotnull(cs_item_sk#21)
+
+(30) Scan parquet default.date_dim
+Output [2]: [d_date_sk#24, d_date#25]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(31) ColumnarToRow [codegen id : 6]
+Input [2]: [d_date_sk#24, d_date#25]
+
+(32) Filter [codegen id : 6]
+Input [2]: [d_date_sk#24, d_date#25]
+Condition : isnotnull(d_date_sk#24)
+
+(33) Scan parquet default.date_dim
+Output [2]: [d_date#26, d_week_seq#27]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_week_seq)]
+ReadSchema: struct<d_date:date,d_week_seq:int>
+
+(34) ColumnarToRow [codegen id : 5]
+Input [2]: [d_date#26, d_week_seq#27]
+
+(35) Filter [codegen id : 5]
+Input [2]: [d_date#26, d_week_seq#27]
+Condition : (isnotnull(d_week_seq#27) AND (d_week_seq#27 = ReusedSubquery Subquery scalar-subquery#8, [id=#9]))
+
+(36) Project [codegen id : 5]
+Output [1]: [d_date#26]
+Input [2]: [d_date#26, d_week_seq#27]
+
+(37) BroadcastExchange
+Input [1]: [d_date#26]
+Arguments: HashedRelationBroadcastMode(List(input[0, date, true]),false), [id=#28]
+
+(38) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [d_date#25]
+Right keys [1]: [d_date#26]
+Join condition: None
+
+(39) Project [codegen id : 6]
+Output [1]: [d_date_sk#24]
+Input [2]: [d_date_sk#24, d_date#25]
+
+(40) BroadcastExchange
+Input [1]: [d_date_sk#24]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#29]
+
+(41) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [cs_sold_date_sk#23]
+Right keys [1]: [d_date_sk#24]
+Join condition: None
+
+(42) Project [codegen id : 8]
+Output [2]: [cs_item_sk#21, cs_ext_sales_price#22]
+Input [4]: [cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23, d_date_sk#24]
+
+(43) ReusedExchange [Reuses operator id: 20]
+Output [2]: [i_item_sk#30, i_item_id#31]
+
+(44) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [cs_item_sk#21]
+Right keys [1]: [i_item_sk#30]
+Join condition: None
+
+(45) Project [codegen id : 8]
+Output [2]: [cs_ext_sales_price#22, i_item_id#31]
+Input [4]: [cs_item_sk#21, cs_ext_sales_price#22, i_item_sk#30, i_item_id#31]
+
+(46) HashAggregate [codegen id : 8]
+Input [2]: [cs_ext_sales_price#22, i_item_id#31]
+Keys [1]: [i_item_id#31]
+Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#22))]
+Aggregate Attributes [1]: [sum#32]
+Results [2]: [i_item_id#31, sum#33]
+
+(47) Exchange
+Input [2]: [i_item_id#31, sum#33]
+Arguments: hashpartitioning(i_item_id#31, 5), ENSURE_REQUIREMENTS, [id=#34]
+
+(48) HashAggregate [codegen id : 9]
+Input [2]: [i_item_id#31, sum#33]
+Keys [1]: [i_item_id#31]
+Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#22))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#22))#35]
+Results [2]: [i_item_id#31 AS item_id#36, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#22))#35,17,2) AS cs_item_rev#37]
+
+(49) Filter [codegen id : 9]
+Input [2]: [item_id#36, cs_item_rev#37]
+Condition : isnotnull(cs_item_rev#37)
+
+(50) BroadcastExchange
+Input [2]: [item_id#36, cs_item_rev#37]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#38]
+
+(51) BroadcastHashJoin [codegen id : 15]
+Left keys [1]: [item_id#19]
+Right keys [1]: [item_id#36]
+Join condition: ((((cast(ss_item_rev#20 as decimal(19,3)) >= CheckOverflow((0.90 * promote_precision(cs_item_rev#37)), DecimalType(19,3), true)) AND (cast(ss_item_rev#20 as decimal(20,3)) <= CheckOverflow((1.10 * promote_precision(cs_item_rev#37)), DecimalType(20,3), true))) AND (cast(cs_item_rev#37 as decimal(19,3)) >= CheckOverflow((0.90 * promote_precision(ss_item_rev#20)), DecimalType(19,3), true))) AND (cast(cs_item_rev#37 as decimal(20,3)) <= CheckOverflow((1.10 * promote_precision(ss_item_rev#20)), DecimalType(20,3), true)))
+
+(52) Project [codegen id : 15]
+Output [3]: [item_id#19, ss_item_rev#20, cs_item_rev#37]
+Input [4]: [item_id#19, ss_item_rev#20, item_id#36, cs_item_rev#37]
+
+(53) Scan parquet default.web_sales
+Output [3]: [ws_item_sk#39, ws_ext_sales_price#40, ws_sold_date_sk#41]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#41), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_ext_sales_price:decimal(7,2)>
+
+(54) ColumnarToRow [codegen id : 13]
+Input [3]: [ws_item_sk#39, ws_ext_sales_price#40, ws_sold_date_sk#41]
+
+(55) Filter [codegen id : 13]
+Input [3]: [ws_item_sk#39, ws_ext_sales_price#40, ws_sold_date_sk#41]
+Condition : isnotnull(ws_item_sk#39)
+
+(56) Scan parquet default.date_dim
+Output [2]: [d_date_sk#42, d_date#43]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(57) ColumnarToRow [codegen id : 11]
+Input [2]: [d_date_sk#42, d_date#43]
+
+(58) Filter [codegen id : 11]
+Input [2]: [d_date_sk#42, d_date#43]
+Condition : isnotnull(d_date_sk#42)
+
+(59) Scan parquet default.date_dim
+Output [2]: [d_date#44, d_week_seq#45]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_week_seq)]
+ReadSchema: struct<d_date:date,d_week_seq:int>
+
+(60) ColumnarToRow [codegen id : 10]
+Input [2]: [d_date#44, d_week_seq#45]
+
+(61) Filter [codegen id : 10]
+Input [2]: [d_date#44, d_week_seq#45]
+Condition : (isnotnull(d_week_seq#45) AND (d_week_seq#45 = ReusedSubquery Subquery scalar-subquery#8, [id=#9]))
+
+(62) Project [codegen id : 10]
+Output [1]: [d_date#44]
+Input [2]: [d_date#44, d_week_seq#45]
+
+(63) BroadcastExchange
+Input [1]: [d_date#44]
+Arguments: HashedRelationBroadcastMode(List(input[0, date, true]),false), [id=#46]
+
+(64) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [d_date#43]
+Right keys [1]: [d_date#44]
+Join condition: None
+
+(65) Project [codegen id : 11]
+Output [1]: [d_date_sk#42]
+Input [2]: [d_date_sk#42, d_date#43]
+
+(66) BroadcastExchange
+Input [1]: [d_date_sk#42]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#47]
+
+(67) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ws_sold_date_sk#41]
+Right keys [1]: [d_date_sk#42]
+Join condition: None
+
+(68) Project [codegen id : 13]
+Output [2]: [ws_item_sk#39, ws_ext_sales_price#40]
+Input [4]: [ws_item_sk#39, ws_ext_sales_price#40, ws_sold_date_sk#41, d_date_sk#42]
+
+(69) ReusedExchange [Reuses operator id: 20]
+Output [2]: [i_item_sk#48, i_item_id#49]
+
+(70) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ws_item_sk#39]
+Right keys [1]: [i_item_sk#48]
+Join condition: None
+
+(71) Project [codegen id : 13]
+Output [2]: [ws_ext_sales_price#40, i_item_id#49]
+Input [4]: [ws_item_sk#39, ws_ext_sales_price#40, i_item_sk#48, i_item_id#49]
+
+(72) HashAggregate [codegen id : 13]
+Input [2]: [ws_ext_sales_price#40, i_item_id#49]
+Keys [1]: [i_item_id#49]
+Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#40))]
+Aggregate Attributes [1]: [sum#50]
+Results [2]: [i_item_id#49, sum#51]
+
+(73) Exchange
+Input [2]: [i_item_id#49, sum#51]
+Arguments: hashpartitioning(i_item_id#49, 5), ENSURE_REQUIREMENTS, [id=#52]
+
+(74) HashAggregate [codegen id : 14]
+Input [2]: [i_item_id#49, sum#51]
+Keys [1]: [i_item_id#49]
+Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#40))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#40))#53]
+Results [2]: [i_item_id#49 AS item_id#54, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#40))#53,17,2) AS ws_item_rev#55]
+
+(75) Filter [codegen id : 14]
+Input [2]: [item_id#54, ws_item_rev#55]
+Condition : isnotnull(ws_item_rev#55)
+
+(76) BroadcastExchange
+Input [2]: [item_id#54, ws_item_rev#55]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#56]
+
+(77) BroadcastHashJoin [codegen id : 15]
+Left keys [1]: [item_id#19]
+Right keys [1]: [item_id#54]
+Join condition: ((((((((cast(ss_item_rev#20 as decimal(19,3)) >= CheckOverflow((0.90 * promote_precision(ws_item_rev#55)), DecimalType(19,3), true)) AND (cast(ss_item_rev#20 as decimal(20,3)) <= CheckOverflow((1.10 * promote_precision(ws_item_rev#55)), DecimalType(20,3), true))) AND (cast(cs_item_rev#37 as decimal(19,3)) >= CheckOverflow((0.90 * promote_precision(ws_item_rev#55)), DecimalType(19,3), true))) AND (cast(cs_item_rev#37 as decimal(20,3)) <= CheckOverflow((1.10 * promote_precision(ws_item_rev#55)), DecimalType(20,3), true))) AND (cast(ws_item_rev#55 as decimal(19,3)) >= CheckOverflow((0.90 * promote_precision(ss_item_rev#20)), DecimalType(19,3), true))) AND (cast(ws_item_rev#55 as decimal(20,3)) <= CheckOverflow((1.10 * promote_precision(ss_item_rev#20)), DecimalType(20,3), true))) AND (cast(ws_item_rev#55 as decimal(19,3)) >= CheckOverflow((0.90 * promote_precision(cs_item_rev#37)), DecimalType(19,3), true))) AND (cast(ws_item_rev#55 as decimal(20,3)) <= CheckOverflow((1.10 * promote_precision(cs_item_rev#37)), DecimalType(20,3), true)))
+
+(78) Project [codegen id : 15]
+Output [8]: [item_id#19, ss_item_rev#20, CheckOverflow((promote_precision(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(ss_item_rev#20 as decimal(19,2))) / promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_item_rev#20 as decimal(18,2))) + promote_precision(cast(cs_item_rev#37 as decimal(18,2)))), DecimalType(18,2), true) as decimal(19,2))) + promote_precision(cast(ws_item_rev#55 as decimal(19,2)))), DecimalType(19,2), true))), DecimalType(38,21), true)) / 3.000000000000000000000), DecimalType(38,21), true)) * 100.000000000000000000000), DecimalType(38,17), true) AS ss_dev#57, cs_item_rev#37, CheckOverflow((promote_precision(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(cs_item_rev#37 as decimal(19,2))) / promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_item_rev#20 as decimal(18,2))) + promote_precision(cast(cs_item_rev#37 as decimal(18,2)))), DecimalType(18,2), true) as decimal(19,2))) + promote_precision(cast(ws_item_rev#55 as decimal(19,2)))), DecimalType(19,2), true))), DecimalType(38,21), true)) / 3.000000000000000000000), DecimalType(38,21), true)) * 100.000000000000000000000), DecimalType(38,17), true) AS cs_dev#58, ws_item_rev#55, CheckOverflow((promote_precision(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(ws_item_rev#55 as decimal(19,2))) / promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_item_rev#20 as decimal(18,2))) + promote_precision(cast(cs_item_rev#37 as decimal(18,2)))), DecimalType(18,2), true) as decimal(19,2))) + promote_precision(cast(ws_item_rev#55 as decimal(19,2)))), DecimalType(19,2), true))), DecimalType(38,21), true)) / 3.000000000000000000000), DecimalType(38,21), true)) * 100.000000000000000000000), DecimalType(38,17), true) AS ws_dev#59, CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_item_rev#20 as decimal(18,2))) + promote_precision(cast(cs_item_rev#37 as decimal(18,2)))), DecimalType(18,2), true) as decimal(19,2))) + promote_precision(cast(ws_item_rev#55 as decimal(19,2)))), DecimalType(19,2), true)) / 3.00), DecimalType(23,6), true) AS average#60]
+Input [5]: [item_id#19, ss_item_rev#20, cs_item_rev#37, item_id#54, ws_item_rev#55]
+
+(79) TakeOrderedAndProject
+Input [8]: [item_id#19, ss_item_rev#20, ss_dev#57, cs_item_rev#37, cs_dev#58, ws_item_rev#55, ws_dev#59, average#60]
+Arguments: 100, [item_id#19 ASC NULLS FIRST, ss_item_rev#20 ASC NULLS FIRST], [item_id#19, ss_item_rev#20, ss_dev#57, cs_item_rev#37, cs_dev#58, ws_item_rev#55, ws_dev#59, average#60]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 9 Hosting Expression = Subquery scalar-subquery#8, [id=#9]
+* Project (83)
++- * Filter (82)
+   +- * ColumnarToRow (81)
+      +- Scan parquet default.date_dim (80)
+
+
+(80) Scan parquet default.date_dim
+Output [2]: [d_date#61, d_week_seq#62]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), EqualTo(d_date,2000-01-03)]
+ReadSchema: struct<d_date:date,d_week_seq:int>
+
+(81) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date#61, d_week_seq#62]
+
+(82) Filter [codegen id : 1]
+Input [2]: [d_date#61, d_week_seq#62]
+Condition : (isnotnull(d_date#61) AND (d_date#61 = 2000-01-03))
+
+(83) Project [codegen id : 1]
+Output [1]: [d_week_seq#62]
+Input [2]: [d_date#61, d_week_seq#62]
+
+Subquery:2 Hosting operator id = 35 Hosting Expression = ReusedSubquery Subquery scalar-subquery#8, [id=#9]
+
+Subquery:3 Hosting operator id = 61 Hosting Expression = ReusedSubquery Subquery scalar-subquery#8, [id=#9]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.sf100/simplified.txt
new file mode 100644
index 0000000000000..5642c7dcbf91c
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.sf100/simplified.txt
@@ -0,0 +1,125 @@
+TakeOrderedAndProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev,ws_dev,average]
+  WholeStageCodegen (15)
+    Project [item_id,ss_item_rev,cs_item_rev,ws_item_rev]
+      BroadcastHashJoin [item_id,item_id,ss_item_rev,ws_item_rev,cs_item_rev]
+        Project [item_id,ss_item_rev,cs_item_rev]
+          BroadcastHashJoin [item_id,item_id,ss_item_rev,cs_item_rev]
+            Filter [ss_item_rev]
+              HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),item_id,ss_item_rev,sum]
+                InputAdapter
+                  Exchange [i_item_id] #1
+                    WholeStageCodegen (4)
+                      HashAggregate [i_item_id,ss_ext_sales_price] [sum,sum]
+                        Project [ss_ext_sales_price,i_item_id]
+                          BroadcastHashJoin [ss_item_sk,i_item_sk]
+                            Project [ss_item_sk,ss_ext_sales_price]
+                              BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                Filter [ss_item_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                InputAdapter
+                                  BroadcastExchange #2
+                                    WholeStageCodegen (2)
+                                      Project [d_date_sk]
+                                        BroadcastHashJoin [d_date,d_date]
+                                          Filter [d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_date]
+                                          InputAdapter
+                                            BroadcastExchange #3
+                                              WholeStageCodegen (1)
+                                                Project [d_date]
+                                                  Filter [d_week_seq]
+                                                    Subquery #1
+                                                      WholeStageCodegen (1)
+                                                        Project [d_week_seq]
+                                                          Filter [d_date]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.date_dim [d_date,d_week_seq]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.date_dim [d_date,d_week_seq]
+                            InputAdapter
+                              BroadcastExchange #4
+                                WholeStageCodegen (3)
+                                  Filter [i_item_sk,i_item_id]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.item [i_item_sk,i_item_id]
+            InputAdapter
+              BroadcastExchange #5
+                WholeStageCodegen (9)
+                  Filter [cs_item_rev]
+                    HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),item_id,cs_item_rev,sum]
+                      InputAdapter
+                        Exchange [i_item_id] #6
+                          WholeStageCodegen (8)
+                            HashAggregate [i_item_id,cs_ext_sales_price] [sum,sum]
+                              Project [cs_ext_sales_price,i_item_id]
+                                BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                  Project [cs_item_sk,cs_ext_sales_price]
+                                    BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                      Filter [cs_item_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk]
+                                      InputAdapter
+                                        BroadcastExchange #7
+                                          WholeStageCodegen (6)
+                                            Project [d_date_sk]
+                                              BroadcastHashJoin [d_date,d_date]
+                                                Filter [d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_date]
+                                                InputAdapter
+                                                  BroadcastExchange #8
+                                                    WholeStageCodegen (5)
+                                                      Project [d_date]
+                                                        Filter [d_week_seq]
+                                                          ReusedSubquery [d_week_seq] #1
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.date_dim [d_date,d_week_seq]
+                                  InputAdapter
+                                    ReusedExchange [i_item_sk,i_item_id] #4
+        InputAdapter
+          BroadcastExchange #9
+            WholeStageCodegen (14)
+              Filter [ws_item_rev]
+                HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),item_id,ws_item_rev,sum]
+                  InputAdapter
+                    Exchange [i_item_id] #10
+                      WholeStageCodegen (13)
+                        HashAggregate [i_item_id,ws_ext_sales_price] [sum,sum]
+                          Project [ws_ext_sales_price,i_item_id]
+                            BroadcastHashJoin [ws_item_sk,i_item_sk]
+                              Project [ws_item_sk,ws_ext_sales_price]
+                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                  Filter [ws_item_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk]
+                                  InputAdapter
+                                    BroadcastExchange #11
+                                      WholeStageCodegen (11)
+                                        Project [d_date_sk]
+                                          BroadcastHashJoin [d_date,d_date]
+                                            Filter [d_date_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.date_dim [d_date_sk,d_date]
+                                            InputAdapter
+                                              BroadcastExchange #12
+                                                WholeStageCodegen (10)
+                                                  Project [d_date]
+                                                    Filter [d_week_seq]
+                                                      ReusedSubquery [d_week_seq] #1
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.date_dim [d_date,d_week_seq]
+                              InputAdapter
+                                ReusedExchange [i_item_sk,i_item_id] #4
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/explain.txt
new file mode 100644
index 0000000000000..1f3e6853a3c41
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/explain.txt
@@ -0,0 +1,480 @@
+== Physical Plan ==
+TakeOrderedAndProject (79)
++- * Project (78)
+   +- * BroadcastHashJoin Inner BuildRight (77)
+      :- * Project (52)
+      :  +- * BroadcastHashJoin Inner BuildRight (51)
+      :     :- * Filter (26)
+      :     :  +- * HashAggregate (25)
+      :     :     +- Exchange (24)
+      :     :        +- * HashAggregate (23)
+      :     :           +- * Project (22)
+      :     :              +- * BroadcastHashJoin Inner BuildRight (21)
+      :     :                 :- * Project (9)
+      :     :                 :  +- * BroadcastHashJoin Inner BuildRight (8)
+      :     :                 :     :- * Filter (3)
+      :     :                 :     :  +- * ColumnarToRow (2)
+      :     :                 :     :     +- Scan parquet default.store_sales (1)
+      :     :                 :     +- BroadcastExchange (7)
+      :     :                 :        +- * Filter (6)
+      :     :                 :           +- * ColumnarToRow (5)
+      :     :                 :              +- Scan parquet default.item (4)
+      :     :                 +- BroadcastExchange (20)
+      :     :                    +- * Project (19)
+      :     :                       +- * BroadcastHashJoin LeftSemi BuildRight (18)
+      :     :                          :- * Filter (12)
+      :     :                          :  +- * ColumnarToRow (11)
+      :     :                          :     +- Scan parquet default.date_dim (10)
+      :     :                          +- BroadcastExchange (17)
+      :     :                             +- * Project (16)
+      :     :                                +- * Filter (15)
+      :     :                                   +- * ColumnarToRow (14)
+      :     :                                      +- Scan parquet default.date_dim (13)
+      :     +- BroadcastExchange (50)
+      :        +- * Filter (49)
+      :           +- * HashAggregate (48)
+      :              +- Exchange (47)
+      :                 +- * HashAggregate (46)
+      :                    +- * Project (45)
+      :                       +- * BroadcastHashJoin Inner BuildRight (44)
+      :                          :- * Project (32)
+      :                          :  +- * BroadcastHashJoin Inner BuildRight (31)
+      :                          :     :- * Filter (29)
+      :                          :     :  +- * ColumnarToRow (28)
+      :                          :     :     +- Scan parquet default.catalog_sales (27)
+      :                          :     +- ReusedExchange (30)
+      :                          +- BroadcastExchange (43)
+      :                             +- * Project (42)
+      :                                +- * BroadcastHashJoin LeftSemi BuildRight (41)
+      :                                   :- * Filter (35)
+      :                                   :  +- * ColumnarToRow (34)
+      :                                   :     +- Scan parquet default.date_dim (33)
+      :                                   +- BroadcastExchange (40)
+      :                                      +- * Project (39)
+      :                                         +- * Filter (38)
+      :                                            +- * ColumnarToRow (37)
+      :                                               +- Scan parquet default.date_dim (36)
+      +- BroadcastExchange (76)
+         +- * Filter (75)
+            +- * HashAggregate (74)
+               +- Exchange (73)
+                  +- * HashAggregate (72)
+                     +- * Project (71)
+                        +- * BroadcastHashJoin Inner BuildRight (70)
+                           :- * Project (58)
+                           :  +- * BroadcastHashJoin Inner BuildRight (57)
+                           :     :- * Filter (55)
+                           :     :  +- * ColumnarToRow (54)
+                           :     :     +- Scan parquet default.web_sales (53)
+                           :     +- ReusedExchange (56)
+                           +- BroadcastExchange (69)
+                              +- * Project (68)
+                                 +- * BroadcastHashJoin LeftSemi BuildRight (67)
+                                    :- * Filter (61)
+                                    :  +- * ColumnarToRow (60)
+                                    :     +- Scan parquet default.date_dim (59)
+                                    +- BroadcastExchange (66)
+                                       +- * Project (65)
+                                          +- * Filter (64)
+                                             +- * ColumnarToRow (63)
+                                                +- Scan parquet default.date_dim (62)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 4]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_item_sk#1)
+
+(4) Scan parquet default.item
+Output [2]: [i_item_sk#4, i_item_id#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_item_id)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [i_item_sk#4, i_item_id#5]
+
+(6) Filter [codegen id : 1]
+Input [2]: [i_item_sk#4, i_item_id#5]
+Condition : (isnotnull(i_item_sk#4) AND isnotnull(i_item_id#5))
+
+(7) BroadcastExchange
+Input [2]: [i_item_sk#4, i_item_id#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#6]
+
+(8) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#4]
+Join condition: None
+
+(9) Project [codegen id : 4]
+Output [3]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5]
+Input [5]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#5]
+
+(10) Scan parquet default.date_dim
+Output [2]: [d_date_sk#7, d_date#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(11) ColumnarToRow [codegen id : 3]
+Input [2]: [d_date_sk#7, d_date#8]
+
+(12) Filter [codegen id : 3]
+Input [2]: [d_date_sk#7, d_date#8]
+Condition : isnotnull(d_date_sk#7)
+
+(13) Scan parquet default.date_dim
+Output [2]: [d_date#9, d_week_seq#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_week_seq)]
+ReadSchema: struct<d_date:date,d_week_seq:int>
+
+(14) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date#9, d_week_seq#10]
+
+(15) Filter [codegen id : 2]
+Input [2]: [d_date#9, d_week_seq#10]
+Condition : (isnotnull(d_week_seq#10) AND (d_week_seq#10 = Subquery scalar-subquery#11, [id=#12]))
+
+(16) Project [codegen id : 2]
+Output [1]: [d_date#9]
+Input [2]: [d_date#9, d_week_seq#10]
+
+(17) BroadcastExchange
+Input [1]: [d_date#9]
+Arguments: HashedRelationBroadcastMode(List(input[0, date, true]),false), [id=#13]
+
+(18) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [d_date#8]
+Right keys [1]: [d_date#9]
+Join condition: None
+
+(19) Project [codegen id : 3]
+Output [1]: [d_date_sk#7]
+Input [2]: [d_date_sk#7, d_date#8]
+
+(20) BroadcastExchange
+Input [1]: [d_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(21) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#7]
+Join condition: None
+
+(22) Project [codegen id : 4]
+Output [2]: [ss_ext_sales_price#2, i_item_id#5]
+Input [4]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, d_date_sk#7]
+
+(23) HashAggregate [codegen id : 4]
+Input [2]: [ss_ext_sales_price#2, i_item_id#5]
+Keys [1]: [i_item_id#5]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum#15]
+Results [2]: [i_item_id#5, sum#16]
+
+(24) Exchange
+Input [2]: [i_item_id#5, sum#16]
+Arguments: hashpartitioning(i_item_id#5, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(25) HashAggregate [codegen id : 15]
+Input [2]: [i_item_id#5, sum#16]
+Keys [1]: [i_item_id#5]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#18]
+Results [2]: [i_item_id#5 AS item_id#19, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#18,17,2) AS ss_item_rev#20]
+
+(26) Filter [codegen id : 15]
+Input [2]: [item_id#19, ss_item_rev#20]
+Condition : isnotnull(ss_item_rev#20)
+
+(27) Scan parquet default.catalog_sales
+Output [3]: [cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#23), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int,cs_ext_sales_price:decimal(7,2)>
+
+(28) ColumnarToRow [codegen id : 8]
+Input [3]: [cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23]
+
+(29) Filter [codegen id : 8]
+Input [3]: [cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23]
+Condition : isnotnull(cs_item_sk#21)
+
+(30) ReusedExchange [Reuses operator id: 7]
+Output [2]: [i_item_sk#24, i_item_id#25]
+
+(31) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [cs_item_sk#21]
+Right keys [1]: [i_item_sk#24]
+Join condition: None
+
+(32) Project [codegen id : 8]
+Output [3]: [cs_ext_sales_price#22, cs_sold_date_sk#23, i_item_id#25]
+Input [5]: [cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23, i_item_sk#24, i_item_id#25]
+
+(33) Scan parquet default.date_dim
+Output [2]: [d_date_sk#26, d_date#27]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(34) ColumnarToRow [codegen id : 7]
+Input [2]: [d_date_sk#26, d_date#27]
+
+(35) Filter [codegen id : 7]
+Input [2]: [d_date_sk#26, d_date#27]
+Condition : isnotnull(d_date_sk#26)
+
+(36) Scan parquet default.date_dim
+Output [2]: [d_date#28, d_week_seq#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_week_seq)]
+ReadSchema: struct<d_date:date,d_week_seq:int>
+
+(37) ColumnarToRow [codegen id : 6]
+Input [2]: [d_date#28, d_week_seq#29]
+
+(38) Filter [codegen id : 6]
+Input [2]: [d_date#28, d_week_seq#29]
+Condition : (isnotnull(d_week_seq#29) AND (d_week_seq#29 = ReusedSubquery Subquery scalar-subquery#11, [id=#12]))
+
+(39) Project [codegen id : 6]
+Output [1]: [d_date#28]
+Input [2]: [d_date#28, d_week_seq#29]
+
+(40) BroadcastExchange
+Input [1]: [d_date#28]
+Arguments: HashedRelationBroadcastMode(List(input[0, date, true]),false), [id=#30]
+
+(41) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [d_date#27]
+Right keys [1]: [d_date#28]
+Join condition: None
+
+(42) Project [codegen id : 7]
+Output [1]: [d_date_sk#26]
+Input [2]: [d_date_sk#26, d_date#27]
+
+(43) BroadcastExchange
+Input [1]: [d_date_sk#26]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#31]
+
+(44) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [cs_sold_date_sk#23]
+Right keys [1]: [d_date_sk#26]
+Join condition: None
+
+(45) Project [codegen id : 8]
+Output [2]: [cs_ext_sales_price#22, i_item_id#25]
+Input [4]: [cs_ext_sales_price#22, cs_sold_date_sk#23, i_item_id#25, d_date_sk#26]
+
+(46) HashAggregate [codegen id : 8]
+Input [2]: [cs_ext_sales_price#22, i_item_id#25]
+Keys [1]: [i_item_id#25]
+Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#22))]
+Aggregate Attributes [1]: [sum#32]
+Results [2]: [i_item_id#25, sum#33]
+
+(47) Exchange
+Input [2]: [i_item_id#25, sum#33]
+Arguments: hashpartitioning(i_item_id#25, 5), ENSURE_REQUIREMENTS, [id=#34]
+
+(48) HashAggregate [codegen id : 9]
+Input [2]: [i_item_id#25, sum#33]
+Keys [1]: [i_item_id#25]
+Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#22))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#22))#35]
+Results [2]: [i_item_id#25 AS item_id#36, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#22))#35,17,2) AS cs_item_rev#37]
+
+(49) Filter [codegen id : 9]
+Input [2]: [item_id#36, cs_item_rev#37]
+Condition : isnotnull(cs_item_rev#37)
+
+(50) BroadcastExchange
+Input [2]: [item_id#36, cs_item_rev#37]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#38]
+
+(51) BroadcastHashJoin [codegen id : 15]
+Left keys [1]: [item_id#19]
+Right keys [1]: [item_id#36]
+Join condition: ((((cast(ss_item_rev#20 as decimal(19,3)) >= CheckOverflow((0.90 * promote_precision(cs_item_rev#37)), DecimalType(19,3), true)) AND (cast(ss_item_rev#20 as decimal(20,3)) <= CheckOverflow((1.10 * promote_precision(cs_item_rev#37)), DecimalType(20,3), true))) AND (cast(cs_item_rev#37 as decimal(19,3)) >= CheckOverflow((0.90 * promote_precision(ss_item_rev#20)), DecimalType(19,3), true))) AND (cast(cs_item_rev#37 as decimal(20,3)) <= CheckOverflow((1.10 * promote_precision(ss_item_rev#20)), DecimalType(20,3), true)))
+
+(52) Project [codegen id : 15]
+Output [3]: [item_id#19, ss_item_rev#20, cs_item_rev#37]
+Input [4]: [item_id#19, ss_item_rev#20, item_id#36, cs_item_rev#37]
+
+(53) Scan parquet default.web_sales
+Output [3]: [ws_item_sk#39, ws_ext_sales_price#40, ws_sold_date_sk#41]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#41), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_ext_sales_price:decimal(7,2)>
+
+(54) ColumnarToRow [codegen id : 13]
+Input [3]: [ws_item_sk#39, ws_ext_sales_price#40, ws_sold_date_sk#41]
+
+(55) Filter [codegen id : 13]
+Input [3]: [ws_item_sk#39, ws_ext_sales_price#40, ws_sold_date_sk#41]
+Condition : isnotnull(ws_item_sk#39)
+
+(56) ReusedExchange [Reuses operator id: 7]
+Output [2]: [i_item_sk#42, i_item_id#43]
+
+(57) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ws_item_sk#39]
+Right keys [1]: [i_item_sk#42]
+Join condition: None
+
+(58) Project [codegen id : 13]
+Output [3]: [ws_ext_sales_price#40, ws_sold_date_sk#41, i_item_id#43]
+Input [5]: [ws_item_sk#39, ws_ext_sales_price#40, ws_sold_date_sk#41, i_item_sk#42, i_item_id#43]
+
+(59) Scan parquet default.date_dim
+Output [2]: [d_date_sk#44, d_date#45]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(60) ColumnarToRow [codegen id : 12]
+Input [2]: [d_date_sk#44, d_date#45]
+
+(61) Filter [codegen id : 12]
+Input [2]: [d_date_sk#44, d_date#45]
+Condition : isnotnull(d_date_sk#44)
+
+(62) Scan parquet default.date_dim
+Output [2]: [d_date#46, d_week_seq#47]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_week_seq)]
+ReadSchema: struct<d_date:date,d_week_seq:int>
+
+(63) ColumnarToRow [codegen id : 11]
+Input [2]: [d_date#46, d_week_seq#47]
+
+(64) Filter [codegen id : 11]
+Input [2]: [d_date#46, d_week_seq#47]
+Condition : (isnotnull(d_week_seq#47) AND (d_week_seq#47 = ReusedSubquery Subquery scalar-subquery#11, [id=#12]))
+
+(65) Project [codegen id : 11]
+Output [1]: [d_date#46]
+Input [2]: [d_date#46, d_week_seq#47]
+
+(66) BroadcastExchange
+Input [1]: [d_date#46]
+Arguments: HashedRelationBroadcastMode(List(input[0, date, true]),false), [id=#48]
+
+(67) BroadcastHashJoin [codegen id : 12]
+Left keys [1]: [d_date#45]
+Right keys [1]: [d_date#46]
+Join condition: None
+
+(68) Project [codegen id : 12]
+Output [1]: [d_date_sk#44]
+Input [2]: [d_date_sk#44, d_date#45]
+
+(69) BroadcastExchange
+Input [1]: [d_date_sk#44]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#49]
+
+(70) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ws_sold_date_sk#41]
+Right keys [1]: [d_date_sk#44]
+Join condition: None
+
+(71) Project [codegen id : 13]
+Output [2]: [ws_ext_sales_price#40, i_item_id#43]
+Input [4]: [ws_ext_sales_price#40, ws_sold_date_sk#41, i_item_id#43, d_date_sk#44]
+
+(72) HashAggregate [codegen id : 13]
+Input [2]: [ws_ext_sales_price#40, i_item_id#43]
+Keys [1]: [i_item_id#43]
+Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#40))]
+Aggregate Attributes [1]: [sum#50]
+Results [2]: [i_item_id#43, sum#51]
+
+(73) Exchange
+Input [2]: [i_item_id#43, sum#51]
+Arguments: hashpartitioning(i_item_id#43, 5), ENSURE_REQUIREMENTS, [id=#52]
+
+(74) HashAggregate [codegen id : 14]
+Input [2]: [i_item_id#43, sum#51]
+Keys [1]: [i_item_id#43]
+Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#40))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#40))#53]
+Results [2]: [i_item_id#43 AS item_id#54, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#40))#53,17,2) AS ws_item_rev#55]
+
+(75) Filter [codegen id : 14]
+Input [2]: [item_id#54, ws_item_rev#55]
+Condition : isnotnull(ws_item_rev#55)
+
+(76) BroadcastExchange
+Input [2]: [item_id#54, ws_item_rev#55]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#56]
+
+(77) BroadcastHashJoin [codegen id : 15]
+Left keys [1]: [item_id#19]
+Right keys [1]: [item_id#54]
+Join condition: ((((((((cast(ss_item_rev#20 as decimal(19,3)) >= CheckOverflow((0.90 * promote_precision(ws_item_rev#55)), DecimalType(19,3), true)) AND (cast(ss_item_rev#20 as decimal(20,3)) <= CheckOverflow((1.10 * promote_precision(ws_item_rev#55)), DecimalType(20,3), true))) AND (cast(cs_item_rev#37 as decimal(19,3)) >= CheckOverflow((0.90 * promote_precision(ws_item_rev#55)), DecimalType(19,3), true))) AND (cast(cs_item_rev#37 as decimal(20,3)) <= CheckOverflow((1.10 * promote_precision(ws_item_rev#55)), DecimalType(20,3), true))) AND (cast(ws_item_rev#55 as decimal(19,3)) >= CheckOverflow((0.90 * promote_precision(ss_item_rev#20)), DecimalType(19,3), true))) AND (cast(ws_item_rev#55 as decimal(20,3)) <= CheckOverflow((1.10 * promote_precision(ss_item_rev#20)), DecimalType(20,3), true))) AND (cast(ws_item_rev#55 as decimal(19,3)) >= CheckOverflow((0.90 * promote_precision(cs_item_rev#37)), DecimalType(19,3), true))) AND (cast(ws_item_rev#55 as decimal(20,3)) <= CheckOverflow((1.10 * promote_precision(cs_item_rev#37)), DecimalType(20,3), true)))
+
+(78) Project [codegen id : 15]
+Output [8]: [item_id#19, ss_item_rev#20, CheckOverflow((promote_precision(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(ss_item_rev#20 as decimal(19,2))) / promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_item_rev#20 as decimal(18,2))) + promote_precision(cast(cs_item_rev#37 as decimal(18,2)))), DecimalType(18,2), true) as decimal(19,2))) + promote_precision(cast(ws_item_rev#55 as decimal(19,2)))), DecimalType(19,2), true))), DecimalType(38,21), true)) / 3.000000000000000000000), DecimalType(38,21), true)) * 100.000000000000000000000), DecimalType(38,17), true) AS ss_dev#57, cs_item_rev#37, CheckOverflow((promote_precision(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(cs_item_rev#37 as decimal(19,2))) / promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_item_rev#20 as decimal(18,2))) + promote_precision(cast(cs_item_rev#37 as decimal(18,2)))), DecimalType(18,2), true) as decimal(19,2))) + promote_precision(cast(ws_item_rev#55 as decimal(19,2)))), DecimalType(19,2), true))), DecimalType(38,21), true)) / 3.000000000000000000000), DecimalType(38,21), true)) * 100.000000000000000000000), DecimalType(38,17), true) AS cs_dev#58, ws_item_rev#55, CheckOverflow((promote_precision(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(ws_item_rev#55 as decimal(19,2))) / promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_item_rev#20 as decimal(18,2))) + promote_precision(cast(cs_item_rev#37 as decimal(18,2)))), DecimalType(18,2), true) as decimal(19,2))) + promote_precision(cast(ws_item_rev#55 as decimal(19,2)))), DecimalType(19,2), true))), DecimalType(38,21), true)) / 3.000000000000000000000), DecimalType(38,21), true)) * 100.000000000000000000000), DecimalType(38,17), true) AS ws_dev#59, CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(ss_item_rev#20 as decimal(18,2))) + promote_precision(cast(cs_item_rev#37 as decimal(18,2)))), DecimalType(18,2), true) as decimal(19,2))) + promote_precision(cast(ws_item_rev#55 as decimal(19,2)))), DecimalType(19,2), true)) / 3.00), DecimalType(23,6), true) AS average#60]
+Input [5]: [item_id#19, ss_item_rev#20, cs_item_rev#37, item_id#54, ws_item_rev#55]
+
+(79) TakeOrderedAndProject
+Input [8]: [item_id#19, ss_item_rev#20, ss_dev#57, cs_item_rev#37, cs_dev#58, ws_item_rev#55, ws_dev#59, average#60]
+Arguments: 100, [item_id#19 ASC NULLS FIRST, ss_item_rev#20 ASC NULLS FIRST], [item_id#19, ss_item_rev#20, ss_dev#57, cs_item_rev#37, cs_dev#58, ws_item_rev#55, ws_dev#59, average#60]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 15 Hosting Expression = Subquery scalar-subquery#11, [id=#12]
+* Project (83)
++- * Filter (82)
+   +- * ColumnarToRow (81)
+      +- Scan parquet default.date_dim (80)
+
+
+(80) Scan parquet default.date_dim
+Output [2]: [d_date#61, d_week_seq#62]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), EqualTo(d_date,2000-01-03)]
+ReadSchema: struct<d_date:date,d_week_seq:int>
+
+(81) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date#61, d_week_seq#62]
+
+(82) Filter [codegen id : 1]
+Input [2]: [d_date#61, d_week_seq#62]
+Condition : (isnotnull(d_date#61) AND (d_date#61 = 2000-01-03))
+
+(83) Project [codegen id : 1]
+Output [1]: [d_week_seq#62]
+Input [2]: [d_date#61, d_week_seq#62]
+
+Subquery:2 Hosting operator id = 38 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12]
+
+Subquery:3 Hosting operator id = 64 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/simplified.txt
new file mode 100644
index 0000000000000..b2e184c7e279d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/simplified.txt
@@ -0,0 +1,125 @@
+TakeOrderedAndProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev,ws_dev,average]
+  WholeStageCodegen (15)
+    Project [item_id,ss_item_rev,cs_item_rev,ws_item_rev]
+      BroadcastHashJoin [item_id,item_id,ss_item_rev,ws_item_rev,cs_item_rev]
+        Project [item_id,ss_item_rev,cs_item_rev]
+          BroadcastHashJoin [item_id,item_id,ss_item_rev,cs_item_rev]
+            Filter [ss_item_rev]
+              HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),item_id,ss_item_rev,sum]
+                InputAdapter
+                  Exchange [i_item_id] #1
+                    WholeStageCodegen (4)
+                      HashAggregate [i_item_id,ss_ext_sales_price] [sum,sum]
+                        Project [ss_ext_sales_price,i_item_id]
+                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                            Project [ss_ext_sales_price,ss_sold_date_sk,i_item_id]
+                              BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                Filter [ss_item_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                InputAdapter
+                                  BroadcastExchange #2
+                                    WholeStageCodegen (1)
+                                      Filter [i_item_sk,i_item_id]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.item [i_item_sk,i_item_id]
+                            InputAdapter
+                              BroadcastExchange #3
+                                WholeStageCodegen (3)
+                                  Project [d_date_sk]
+                                    BroadcastHashJoin [d_date,d_date]
+                                      Filter [d_date_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.date_dim [d_date_sk,d_date]
+                                      InputAdapter
+                                        BroadcastExchange #4
+                                          WholeStageCodegen (2)
+                                            Project [d_date]
+                                              Filter [d_week_seq]
+                                                Subquery #1
+                                                  WholeStageCodegen (1)
+                                                    Project [d_week_seq]
+                                                      Filter [d_date]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.date_dim [d_date,d_week_seq]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_date,d_week_seq]
+            InputAdapter
+              BroadcastExchange #5
+                WholeStageCodegen (9)
+                  Filter [cs_item_rev]
+                    HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),item_id,cs_item_rev,sum]
+                      InputAdapter
+                        Exchange [i_item_id] #6
+                          WholeStageCodegen (8)
+                            HashAggregate [i_item_id,cs_ext_sales_price] [sum,sum]
+                              Project [cs_ext_sales_price,i_item_id]
+                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                  Project [cs_ext_sales_price,cs_sold_date_sk,i_item_id]
+                                    BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                      Filter [cs_item_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk]
+                                      InputAdapter
+                                        ReusedExchange [i_item_sk,i_item_id] #2
+                                  InputAdapter
+                                    BroadcastExchange #7
+                                      WholeStageCodegen (7)
+                                        Project [d_date_sk]
+                                          BroadcastHashJoin [d_date,d_date]
+                                            Filter [d_date_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.date_dim [d_date_sk,d_date]
+                                            InputAdapter
+                                              BroadcastExchange #8
+                                                WholeStageCodegen (6)
+                                                  Project [d_date]
+                                                    Filter [d_week_seq]
+                                                      ReusedSubquery [d_week_seq] #1
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.date_dim [d_date,d_week_seq]
+        InputAdapter
+          BroadcastExchange #9
+            WholeStageCodegen (14)
+              Filter [ws_item_rev]
+                HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),item_id,ws_item_rev,sum]
+                  InputAdapter
+                    Exchange [i_item_id] #10
+                      WholeStageCodegen (13)
+                        HashAggregate [i_item_id,ws_ext_sales_price] [sum,sum]
+                          Project [ws_ext_sales_price,i_item_id]
+                            BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                              Project [ws_ext_sales_price,ws_sold_date_sk,i_item_id]
+                                BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                  Filter [ws_item_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk]
+                                  InputAdapter
+                                    ReusedExchange [i_item_sk,i_item_id] #2
+                              InputAdapter
+                                BroadcastExchange #11
+                                  WholeStageCodegen (12)
+                                    Project [d_date_sk]
+                                      BroadcastHashJoin [d_date,d_date]
+                                        Filter [d_date_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.date_dim [d_date_sk,d_date]
+                                        InputAdapter
+                                          BroadcastExchange #12
+                                            WholeStageCodegen (11)
+                                              Project [d_date]
+                                                Filter [d_week_seq]
+                                                  ReusedSubquery [d_week_seq] #1
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date,d_week_seq]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.sf100/explain.txt
new file mode 100644
index 0000000000000..cf22f178dd33f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.sf100/explain.txt
@@ -0,0 +1,250 @@
+== Physical Plan ==
+TakeOrderedAndProject (44)
++- * Project (43)
+   +- * BroadcastHashJoin Inner BuildRight (42)
+      :- * Project (25)
+      :  +- * BroadcastHashJoin Inner BuildRight (24)
+      :     :- * Project (18)
+      :     :  +- * BroadcastHashJoin Inner BuildRight (17)
+      :     :     :- * HashAggregate (12)
+      :     :     :  +- Exchange (11)
+      :     :     :     +- * HashAggregate (10)
+      :     :     :        +- * Project (9)
+      :     :     :           +- * BroadcastHashJoin Inner BuildRight (8)
+      :     :     :              :- * Filter (3)
+      :     :     :              :  +- * ColumnarToRow (2)
+      :     :     :              :     +- Scan parquet default.store_sales (1)
+      :     :     :              +- BroadcastExchange (7)
+      :     :     :                 +- * Filter (6)
+      :     :     :                    +- * ColumnarToRow (5)
+      :     :     :                       +- Scan parquet default.date_dim (4)
+      :     :     +- BroadcastExchange (16)
+      :     :        +- * Filter (15)
+      :     :           +- * ColumnarToRow (14)
+      :     :              +- Scan parquet default.store (13)
+      :     +- BroadcastExchange (23)
+      :        +- * Project (22)
+      :           +- * Filter (21)
+      :              +- * ColumnarToRow (20)
+      :                 +- Scan parquet default.date_dim (19)
+      +- BroadcastExchange (41)
+         +- * Project (40)
+            +- * BroadcastHashJoin Inner BuildRight (39)
+               :- * Project (33)
+               :  +- * BroadcastHashJoin Inner BuildRight (32)
+               :     :- * HashAggregate (27)
+               :     :  +- ReusedExchange (26)
+               :     +- BroadcastExchange (31)
+               :        +- * Filter (30)
+               :           +- * ColumnarToRow (29)
+               :              +- Scan parquet default.store (28)
+               +- BroadcastExchange (38)
+                  +- * Project (37)
+                     +- * Filter (36)
+                        +- * ColumnarToRow (35)
+                           +- Scan parquet default.date_dim (34)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 2]
+Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_store_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date_sk), IsNotNull(d_week_seq)]
+ReadSchema: struct<d_date_sk:int,d_week_seq:int,d_day_name:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6]
+Condition : (isnotnull(d_date_sk#4) AND isnotnull(d_week_seq#5))
+
+(7) BroadcastExchange
+Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#7]
+
+(8) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#4]
+Join condition: None
+
+(9) Project [codegen id : 2]
+Output [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6]
+Input [6]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_week_seq#5, d_day_name#6]
+
+(10) HashAggregate [codegen id : 2]
+Input [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6]
+Keys [2]: [d_week_seq#5, ss_store_sk#1]
+Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday   ) THEN ss_sales_price#2 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday   ) THEN ss_sales_price#2 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday  ) THEN ss_sales_price#2 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday   ) THEN ss_sales_price#2 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 ELSE null END))]
+Aggregate Attributes [7]: [sum#8, sum#9, sum#10, sum#11, sum#12, sum#13, sum#14]
+Results [9]: [d_week_seq#5, ss_store_sk#1, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20, sum#21]
+
+(11) Exchange
+Input [9]: [d_week_seq#5, ss_store_sk#1, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20, sum#21]
+Arguments: hashpartitioning(d_week_seq#5, ss_store_sk#1, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(12) HashAggregate [codegen id : 10]
+Input [9]: [d_week_seq#5, ss_store_sk#1, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20, sum#21]
+Keys [2]: [d_week_seq#5, ss_store_sk#1]
+Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday   ) THEN ss_sales_price#2 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday   ) THEN ss_sales_price#2 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday  ) THEN ss_sales_price#2 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday   ) THEN ss_sales_price#2 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 ELSE null END))]
+Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday   ) THEN ss_sales_price#2 ELSE null END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday   ) THEN ss_sales_price#2 ELSE null END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday  ) THEN ss_sales_price#2 ELSE null END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 ELSE null END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 ELSE null END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday   ) THEN ss_sales_price#2 ELSE null END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 ELSE null END))#29]
+Results [9]: [d_week_seq#5, ss_store_sk#1, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday   ) THEN ss_sales_price#2 ELSE null END))#23,17,2) AS sun_sales#30, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday   ) THEN ss_sales_price#2 ELSE null END))#24,17,2) AS mon_sales#31, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday  ) THEN ss_sales_price#2 ELSE null END))#25,17,2) AS tue_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 ELSE null END))#26,17,2) AS wed_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 ELSE null END))#27,17,2) AS thu_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday   ) THEN ss_sales_price#2 ELSE null END))#28,17,2) AS fri_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 ELSE null END))#29,17,2) AS sat_sales#36]
+
+(13) Scan parquet default.store
+Output [3]: [s_store_sk#37, s_store_id#38, s_store_name#39]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_id)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string,s_store_name:string>
+
+(14) ColumnarToRow [codegen id : 3]
+Input [3]: [s_store_sk#37, s_store_id#38, s_store_name#39]
+
+(15) Filter [codegen id : 3]
+Input [3]: [s_store_sk#37, s_store_id#38, s_store_name#39]
+Condition : (isnotnull(s_store_sk#37) AND isnotnull(s_store_id#38))
+
+(16) BroadcastExchange
+Input [3]: [s_store_sk#37, s_store_id#38, s_store_name#39]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#40]
+
+(17) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_store_sk#1]
+Right keys [1]: [s_store_sk#37]
+Join condition: None
+
+(18) Project [codegen id : 10]
+Output [10]: [d_week_seq#5, sun_sales#30, mon_sales#31, tue_sales#32, wed_sales#33, thu_sales#34, fri_sales#35, sat_sales#36, s_store_id#38, s_store_name#39]
+Input [12]: [d_week_seq#5, ss_store_sk#1, sun_sales#30, mon_sales#31, tue_sales#32, wed_sales#33, thu_sales#34, fri_sales#35, sat_sales#36, s_store_sk#37, s_store_id#38, s_store_name#39]
+
+(19) Scan parquet default.date_dim
+Output [2]: [d_month_seq#41, d_week_seq#42]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_week_seq)]
+ReadSchema: struct<d_month_seq:int,d_week_seq:int>
+
+(20) ColumnarToRow [codegen id : 4]
+Input [2]: [d_month_seq#41, d_week_seq#42]
+
+(21) Filter [codegen id : 4]
+Input [2]: [d_month_seq#41, d_week_seq#42]
+Condition : (((isnotnull(d_month_seq#41) AND (d_month_seq#41 >= 1212)) AND (d_month_seq#41 <= 1223)) AND isnotnull(d_week_seq#42))
+
+(22) Project [codegen id : 4]
+Output [1]: [d_week_seq#42]
+Input [2]: [d_month_seq#41, d_week_seq#42]
+
+(23) BroadcastExchange
+Input [1]: [d_week_seq#42]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#43]
+
+(24) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [d_week_seq#5]
+Right keys [1]: [d_week_seq#42]
+Join condition: None
+
+(25) Project [codegen id : 10]
+Output [10]: [s_store_name#39 AS s_store_name1#44, d_week_seq#5 AS d_week_seq1#45, s_store_id#38 AS s_store_id1#46, sun_sales#30 AS sun_sales1#47, mon_sales#31 AS mon_sales1#48, tue_sales#32 AS tue_sales1#49, wed_sales#33 AS wed_sales1#50, thu_sales#34 AS thu_sales1#51, fri_sales#35 AS fri_sales1#52, sat_sales#36 AS sat_sales1#53]
+Input [11]: [d_week_seq#5, sun_sales#30, mon_sales#31, tue_sales#32, wed_sales#33, thu_sales#34, fri_sales#35, sat_sales#36, s_store_id#38, s_store_name#39, d_week_seq#42]
+
+(26) ReusedExchange [Reuses operator id: 11]
+Output [9]: [d_week_seq#54, ss_store_sk#55, sum#56, sum#57, sum#58, sum#59, sum#60, sum#61, sum#62]
+
+(27) HashAggregate [codegen id : 9]
+Input [9]: [d_week_seq#54, ss_store_sk#55, sum#56, sum#57, sum#58, sum#59, sum#60, sum#61, sum#62]
+Keys [2]: [d_week_seq#54, ss_store_sk#55]
+Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#63 = Sunday   ) THEN ss_sales_price#64 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#63 = Monday   ) THEN ss_sales_price#64 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#63 = Tuesday  ) THEN ss_sales_price#64 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#63 = Wednesday) THEN ss_sales_price#64 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#63 = Thursday ) THEN ss_sales_price#64 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#63 = Friday   ) THEN ss_sales_price#64 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#63 = Saturday ) THEN ss_sales_price#64 ELSE null END))]
+Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#63 = Sunday   ) THEN ss_sales_price#64 ELSE null END))#65, sum(UnscaledValue(CASE WHEN (d_day_name#63 = Monday   ) THEN ss_sales_price#64 ELSE null END))#66, sum(UnscaledValue(CASE WHEN (d_day_name#63 = Tuesday  ) THEN ss_sales_price#64 ELSE null END))#67, sum(UnscaledValue(CASE WHEN (d_day_name#63 = Wednesday) THEN ss_sales_price#64 ELSE null END))#68, sum(UnscaledValue(CASE WHEN (d_day_name#63 = Thursday ) THEN ss_sales_price#64 ELSE null END))#69, sum(UnscaledValue(CASE WHEN (d_day_name#63 = Friday   ) THEN ss_sales_price#64 ELSE null END))#70, sum(UnscaledValue(CASE WHEN (d_day_name#63 = Saturday ) THEN ss_sales_price#64 ELSE null END))#71]
+Results [9]: [d_week_seq#54, ss_store_sk#55, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#63 = Sunday   ) THEN ss_sales_price#64 ELSE null END))#65,17,2) AS sun_sales#30, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#63 = Monday   ) THEN ss_sales_price#64 ELSE null END))#66,17,2) AS mon_sales#31, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#63 = Tuesday  ) THEN ss_sales_price#64 ELSE null END))#67,17,2) AS tue_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#63 = Wednesday) THEN ss_sales_price#64 ELSE null END))#68,17,2) AS wed_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#63 = Thursday ) THEN ss_sales_price#64 ELSE null END))#69,17,2) AS thu_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#63 = Friday   ) THEN ss_sales_price#64 ELSE null END))#70,17,2) AS fri_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#63 = Saturday ) THEN ss_sales_price#64 ELSE null END))#71,17,2) AS sat_sales#36]
+
+(28) Scan parquet default.store
+Output [2]: [s_store_sk#72, s_store_id#73]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_id)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string>
+
+(29) ColumnarToRow [codegen id : 7]
+Input [2]: [s_store_sk#72, s_store_id#73]
+
+(30) Filter [codegen id : 7]
+Input [2]: [s_store_sk#72, s_store_id#73]
+Condition : (isnotnull(s_store_sk#72) AND isnotnull(s_store_id#73))
+
+(31) BroadcastExchange
+Input [2]: [s_store_sk#72, s_store_id#73]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#74]
+
+(32) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_store_sk#55]
+Right keys [1]: [s_store_sk#72]
+Join condition: None
+
+(33) Project [codegen id : 9]
+Output [9]: [d_week_seq#54, sun_sales#30, mon_sales#31, tue_sales#32, wed_sales#33, thu_sales#34, fri_sales#35, sat_sales#36, s_store_id#73]
+Input [11]: [d_week_seq#54, ss_store_sk#55, sun_sales#30, mon_sales#31, tue_sales#32, wed_sales#33, thu_sales#34, fri_sales#35, sat_sales#36, s_store_sk#72, s_store_id#73]
+
+(34) Scan parquet default.date_dim
+Output [2]: [d_month_seq#75, d_week_seq#76]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1224), LessThanOrEqual(d_month_seq,1235), IsNotNull(d_week_seq)]
+ReadSchema: struct<d_month_seq:int,d_week_seq:int>
+
+(35) ColumnarToRow [codegen id : 8]
+Input [2]: [d_month_seq#75, d_week_seq#76]
+
+(36) Filter [codegen id : 8]
+Input [2]: [d_month_seq#75, d_week_seq#76]
+Condition : (((isnotnull(d_month_seq#75) AND (d_month_seq#75 >= 1224)) AND (d_month_seq#75 <= 1235)) AND isnotnull(d_week_seq#76))
+
+(37) Project [codegen id : 8]
+Output [1]: [d_week_seq#76]
+Input [2]: [d_month_seq#75, d_week_seq#76]
+
+(38) BroadcastExchange
+Input [1]: [d_week_seq#76]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#77]
+
+(39) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [d_week_seq#54]
+Right keys [1]: [d_week_seq#76]
+Join condition: None
+
+(40) Project [codegen id : 9]
+Output [9]: [d_week_seq#54 AS d_week_seq2#78, s_store_id#73 AS s_store_id2#79, sun_sales#30 AS sun_sales2#80, mon_sales#31 AS mon_sales2#81, tue_sales#32 AS tue_sales2#82, wed_sales#33 AS wed_sales2#83, thu_sales#34 AS thu_sales2#84, fri_sales#35 AS fri_sales2#85, sat_sales#36 AS sat_sales2#86]
+Input [10]: [d_week_seq#54, sun_sales#30, mon_sales#31, tue_sales#32, wed_sales#33, thu_sales#34, fri_sales#35, sat_sales#36, s_store_id#73, d_week_seq#76]
+
+(41) BroadcastExchange
+Input [9]: [d_week_seq2#78, s_store_id2#79, sun_sales2#80, mon_sales2#81, tue_sales2#82, wed_sales2#83, thu_sales2#84, fri_sales2#85, sat_sales2#86]
+Arguments: HashedRelationBroadcastMode(List(input[1, string, true], (input[0, int, true] - 52)),false), [id=#87]
+
+(42) BroadcastHashJoin [codegen id : 10]
+Left keys [2]: [s_store_id1#46, d_week_seq1#45]
+Right keys [2]: [s_store_id2#79, (d_week_seq2#78 - 52)]
+Join condition: None
+
+(43) Project [codegen id : 10]
+Output [10]: [s_store_name1#44, s_store_id1#46, d_week_seq1#45, CheckOverflow((promote_precision(sun_sales1#47) / promote_precision(sun_sales2#80)), DecimalType(37,20), true) AS (sun_sales1 / sun_sales2)#88, CheckOverflow((promote_precision(mon_sales1#48) / promote_precision(mon_sales2#81)), DecimalType(37,20), true) AS (mon_sales1 / mon_sales2)#89, CheckOverflow((promote_precision(tue_sales1#49) / promote_precision(tue_sales2#82)), DecimalType(37,20), true) AS (tue_sales1 / tue_sales2)#90, CheckOverflow((promote_precision(wed_sales1#50) / promote_precision(wed_sales2#83)), DecimalType(37,20), true) AS (wed_sales1 / wed_sales2)#91, CheckOverflow((promote_precision(thu_sales1#51) / promote_precision(thu_sales2#84)), DecimalType(37,20), true) AS (thu_sales1 / thu_sales2)#92, CheckOverflow((promote_precision(fri_sales1#52) / promote_precision(fri_sales2#85)), DecimalType(37,20), true) AS (fri_sales1 / fri_sales2)#93, CheckOverflow((promote_precision(sat_sales1#53) / promote_precision(sat_sales2#86)), DecimalType(37,20), true) AS (sat_sales1 / sat_sales2)#94]
+Input [19]: [s_store_name1#44, d_week_seq1#45, s_store_id1#46, sun_sales1#47, mon_sales1#48, tue_sales1#49, wed_sales1#50, thu_sales1#51, fri_sales1#52, sat_sales1#53, d_week_seq2#78, s_store_id2#79, sun_sales2#80, mon_sales2#81, tue_sales2#82, wed_sales2#83, thu_sales2#84, fri_sales2#85, sat_sales2#86]
+
+(44) TakeOrderedAndProject
+Input [10]: [s_store_name1#44, s_store_id1#46, d_week_seq1#45, (sun_sales1 / sun_sales2)#88, (mon_sales1 / mon_sales2)#89, (tue_sales1 / tue_sales2)#90, (wed_sales1 / wed_sales2)#91, (thu_sales1 / thu_sales2)#92, (fri_sales1 / fri_sales2)#93, (sat_sales1 / sat_sales2)#94]
+Arguments: 100, [s_store_name1#44 ASC NULLS FIRST, s_store_id1#46 ASC NULLS FIRST, d_week_seq1#45 ASC NULLS FIRST], [s_store_name1#44, s_store_id1#46, d_week_seq1#45, (sun_sales1 / sun_sales2)#88, (mon_sales1 / mon_sales2)#89, (tue_sales1 / tue_sales2)#90, (wed_sales1 / wed_sales2)#91, (thu_sales1 / thu_sales2)#92, (fri_sales1 / fri_sales2)#93, (sat_sales1 / sat_sales2)#94]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.sf100/simplified.txt
new file mode 100644
index 0000000000000..541817641b129
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.sf100/simplified.txt
@@ -0,0 +1,66 @@
+TakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_sales2),(mon_sales1 / mon_sales2),(tue_sales1 / tue_sales2),(wed_sales1 / wed_sales2),(thu_sales1 / thu_sales2),(fri_sales1 / fri_sales2),(sat_sales1 / sat_sales2)]
+  WholeStageCodegen (10)
+    Project [s_store_name1,s_store_id1,d_week_seq1,sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2]
+      BroadcastHashJoin [s_store_id1,d_week_seq1,s_store_id2,d_week_seq2]
+        Project [s_store_name,d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales]
+          BroadcastHashJoin [d_week_seq,d_week_seq]
+            Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,s_store_name]
+              BroadcastHashJoin [ss_store_sk,s_store_sk]
+                HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday  ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price ELSE null END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum]
+                  InputAdapter
+                    Exchange [d_week_seq,ss_store_sk] #1
+                      WholeStageCodegen (2)
+                        HashAggregate [d_week_seq,ss_store_sk,d_day_name,ss_sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum]
+                          Project [ss_store_sk,ss_sales_price,d_week_seq,d_day_name]
+                            BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                              Filter [ss_store_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                              InputAdapter
+                                BroadcastExchange #2
+                                  WholeStageCodegen (1)
+                                    Filter [d_date_sk,d_week_seq]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.date_dim [d_date_sk,d_week_seq,d_day_name]
+                InputAdapter
+                  BroadcastExchange #3
+                    WholeStageCodegen (3)
+                      Filter [s_store_sk,s_store_id]
+                        ColumnarToRow
+                          InputAdapter
+                            Scan parquet default.store [s_store_sk,s_store_id,s_store_name]
+            InputAdapter
+              BroadcastExchange #4
+                WholeStageCodegen (4)
+                  Project [d_week_seq]
+                    Filter [d_month_seq,d_week_seq]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.date_dim [d_month_seq,d_week_seq]
+        InputAdapter
+          BroadcastExchange #5
+            WholeStageCodegen (9)
+              Project [d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales]
+                BroadcastHashJoin [d_week_seq,d_week_seq]
+                  Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id]
+                    BroadcastHashJoin [ss_store_sk,s_store_sk]
+                      HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday  ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price ELSE null END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum]
+                        InputAdapter
+                          ReusedExchange [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] #1
+                      InputAdapter
+                        BroadcastExchange #6
+                          WholeStageCodegen (7)
+                            Filter [s_store_sk,s_store_id]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.store [s_store_sk,s_store_id]
+                  InputAdapter
+                    BroadcastExchange #7
+                      WholeStageCodegen (8)
+                        Project [d_week_seq]
+                          Filter [d_month_seq,d_week_seq]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.date_dim [d_month_seq,d_week_seq]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/explain.txt
new file mode 100644
index 0000000000000..cf22f178dd33f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/explain.txt
@@ -0,0 +1,250 @@
+== Physical Plan ==
+TakeOrderedAndProject (44)
++- * Project (43)
+   +- * BroadcastHashJoin Inner BuildRight (42)
+      :- * Project (25)
+      :  +- * BroadcastHashJoin Inner BuildRight (24)
+      :     :- * Project (18)
+      :     :  +- * BroadcastHashJoin Inner BuildRight (17)
+      :     :     :- * HashAggregate (12)
+      :     :     :  +- Exchange (11)
+      :     :     :     +- * HashAggregate (10)
+      :     :     :        +- * Project (9)
+      :     :     :           +- * BroadcastHashJoin Inner BuildRight (8)
+      :     :     :              :- * Filter (3)
+      :     :     :              :  +- * ColumnarToRow (2)
+      :     :     :              :     +- Scan parquet default.store_sales (1)
+      :     :     :              +- BroadcastExchange (7)
+      :     :     :                 +- * Filter (6)
+      :     :     :                    +- * ColumnarToRow (5)
+      :     :     :                       +- Scan parquet default.date_dim (4)
+      :     :     +- BroadcastExchange (16)
+      :     :        +- * Filter (15)
+      :     :           +- * ColumnarToRow (14)
+      :     :              +- Scan parquet default.store (13)
+      :     +- BroadcastExchange (23)
+      :        +- * Project (22)
+      :           +- * Filter (21)
+      :              +- * ColumnarToRow (20)
+      :                 +- Scan parquet default.date_dim (19)
+      +- BroadcastExchange (41)
+         +- * Project (40)
+            +- * BroadcastHashJoin Inner BuildRight (39)
+               :- * Project (33)
+               :  +- * BroadcastHashJoin Inner BuildRight (32)
+               :     :- * HashAggregate (27)
+               :     :  +- ReusedExchange (26)
+               :     +- BroadcastExchange (31)
+               :        +- * Filter (30)
+               :           +- * ColumnarToRow (29)
+               :              +- Scan parquet default.store (28)
+               +- BroadcastExchange (38)
+                  +- * Project (37)
+                     +- * Filter (36)
+                        +- * ColumnarToRow (35)
+                           +- Scan parquet default.date_dim (34)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 2]
+Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_store_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date_sk), IsNotNull(d_week_seq)]
+ReadSchema: struct<d_date_sk:int,d_week_seq:int,d_day_name:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6]
+Condition : (isnotnull(d_date_sk#4) AND isnotnull(d_week_seq#5))
+
+(7) BroadcastExchange
+Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#7]
+
+(8) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#4]
+Join condition: None
+
+(9) Project [codegen id : 2]
+Output [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6]
+Input [6]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_week_seq#5, d_day_name#6]
+
+(10) HashAggregate [codegen id : 2]
+Input [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6]
+Keys [2]: [d_week_seq#5, ss_store_sk#1]
+Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday   ) THEN ss_sales_price#2 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday   ) THEN ss_sales_price#2 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday  ) THEN ss_sales_price#2 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday   ) THEN ss_sales_price#2 ELSE null END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 ELSE null END))]
+Aggregate Attributes [7]: [sum#8, sum#9, sum#10, sum#11, sum#12, sum#13, sum#14]
+Results [9]: [d_week_seq#5, ss_store_sk#1, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20, sum#21]
+
+(11) Exchange
+Input [9]: [d_week_seq#5, ss_store_sk#1, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20, sum#21]
+Arguments: hashpartitioning(d_week_seq#5, ss_store_sk#1, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(12) HashAggregate [codegen id : 10]
+Input [9]: [d_week_seq#5, ss_store_sk#1, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20, sum#21]
+Keys [2]: [d_week_seq#5, ss_store_sk#1]
+Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday   ) THEN ss_sales_price#2 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday   ) THEN ss_sales_price#2 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday  ) THEN ss_sales_price#2 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday   ) THEN ss_sales_price#2 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 ELSE null END))]
+Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday   ) THEN ss_sales_price#2 ELSE null END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday   ) THEN ss_sales_price#2 ELSE null END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday  ) THEN ss_sales_price#2 ELSE null END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 ELSE null END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 ELSE null END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday   ) THEN ss_sales_price#2 ELSE null END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 ELSE null END))#29]
+Results [9]: [d_week_seq#5, ss_store_sk#1, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday   ) THEN ss_sales_price#2 ELSE null END))#23,17,2) AS sun_sales#30, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday   ) THEN ss_sales_price#2 ELSE null END))#24,17,2) AS mon_sales#31, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday  ) THEN ss_sales_price#2 ELSE null END))#25,17,2) AS tue_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 ELSE null END))#26,17,2) AS wed_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 ELSE null END))#27,17,2) AS thu_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday   ) THEN ss_sales_price#2 ELSE null END))#28,17,2) AS fri_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 ELSE null END))#29,17,2) AS sat_sales#36]
+
+(13) Scan parquet default.store
+Output [3]: [s_store_sk#37, s_store_id#38, s_store_name#39]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_id)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string,s_store_name:string>
+
+(14) ColumnarToRow [codegen id : 3]
+Input [3]: [s_store_sk#37, s_store_id#38, s_store_name#39]
+
+(15) Filter [codegen id : 3]
+Input [3]: [s_store_sk#37, s_store_id#38, s_store_name#39]
+Condition : (isnotnull(s_store_sk#37) AND isnotnull(s_store_id#38))
+
+(16) BroadcastExchange
+Input [3]: [s_store_sk#37, s_store_id#38, s_store_name#39]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#40]
+
+(17) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_store_sk#1]
+Right keys [1]: [s_store_sk#37]
+Join condition: None
+
+(18) Project [codegen id : 10]
+Output [10]: [d_week_seq#5, sun_sales#30, mon_sales#31, tue_sales#32, wed_sales#33, thu_sales#34, fri_sales#35, sat_sales#36, s_store_id#38, s_store_name#39]
+Input [12]: [d_week_seq#5, ss_store_sk#1, sun_sales#30, mon_sales#31, tue_sales#32, wed_sales#33, thu_sales#34, fri_sales#35, sat_sales#36, s_store_sk#37, s_store_id#38, s_store_name#39]
+
+(19) Scan parquet default.date_dim
+Output [2]: [d_month_seq#41, d_week_seq#42]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_week_seq)]
+ReadSchema: struct<d_month_seq:int,d_week_seq:int>
+
+(20) ColumnarToRow [codegen id : 4]
+Input [2]: [d_month_seq#41, d_week_seq#42]
+
+(21) Filter [codegen id : 4]
+Input [2]: [d_month_seq#41, d_week_seq#42]
+Condition : (((isnotnull(d_month_seq#41) AND (d_month_seq#41 >= 1212)) AND (d_month_seq#41 <= 1223)) AND isnotnull(d_week_seq#42))
+
+(22) Project [codegen id : 4]
+Output [1]: [d_week_seq#42]
+Input [2]: [d_month_seq#41, d_week_seq#42]
+
+(23) BroadcastExchange
+Input [1]: [d_week_seq#42]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#43]
+
+(24) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [d_week_seq#5]
+Right keys [1]: [d_week_seq#42]
+Join condition: None
+
+(25) Project [codegen id : 10]
+Output [10]: [s_store_name#39 AS s_store_name1#44, d_week_seq#5 AS d_week_seq1#45, s_store_id#38 AS s_store_id1#46, sun_sales#30 AS sun_sales1#47, mon_sales#31 AS mon_sales1#48, tue_sales#32 AS tue_sales1#49, wed_sales#33 AS wed_sales1#50, thu_sales#34 AS thu_sales1#51, fri_sales#35 AS fri_sales1#52, sat_sales#36 AS sat_sales1#53]
+Input [11]: [d_week_seq#5, sun_sales#30, mon_sales#31, tue_sales#32, wed_sales#33, thu_sales#34, fri_sales#35, sat_sales#36, s_store_id#38, s_store_name#39, d_week_seq#42]
+
+(26) ReusedExchange [Reuses operator id: 11]
+Output [9]: [d_week_seq#54, ss_store_sk#55, sum#56, sum#57, sum#58, sum#59, sum#60, sum#61, sum#62]
+
+(27) HashAggregate [codegen id : 9]
+Input [9]: [d_week_seq#54, ss_store_sk#55, sum#56, sum#57, sum#58, sum#59, sum#60, sum#61, sum#62]
+Keys [2]: [d_week_seq#54, ss_store_sk#55]
+Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#63 = Sunday   ) THEN ss_sales_price#64 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#63 = Monday   ) THEN ss_sales_price#64 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#63 = Tuesday  ) THEN ss_sales_price#64 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#63 = Wednesday) THEN ss_sales_price#64 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#63 = Thursday ) THEN ss_sales_price#64 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#63 = Friday   ) THEN ss_sales_price#64 ELSE null END)), sum(UnscaledValue(CASE WHEN (d_day_name#63 = Saturday ) THEN ss_sales_price#64 ELSE null END))]
+Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#63 = Sunday   ) THEN ss_sales_price#64 ELSE null END))#65, sum(UnscaledValue(CASE WHEN (d_day_name#63 = Monday   ) THEN ss_sales_price#64 ELSE null END))#66, sum(UnscaledValue(CASE WHEN (d_day_name#63 = Tuesday  ) THEN ss_sales_price#64 ELSE null END))#67, sum(UnscaledValue(CASE WHEN (d_day_name#63 = Wednesday) THEN ss_sales_price#64 ELSE null END))#68, sum(UnscaledValue(CASE WHEN (d_day_name#63 = Thursday ) THEN ss_sales_price#64 ELSE null END))#69, sum(UnscaledValue(CASE WHEN (d_day_name#63 = Friday   ) THEN ss_sales_price#64 ELSE null END))#70, sum(UnscaledValue(CASE WHEN (d_day_name#63 = Saturday ) THEN ss_sales_price#64 ELSE null END))#71]
+Results [9]: [d_week_seq#54, ss_store_sk#55, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#63 = Sunday   ) THEN ss_sales_price#64 ELSE null END))#65,17,2) AS sun_sales#30, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#63 = Monday   ) THEN ss_sales_price#64 ELSE null END))#66,17,2) AS mon_sales#31, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#63 = Tuesday  ) THEN ss_sales_price#64 ELSE null END))#67,17,2) AS tue_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#63 = Wednesday) THEN ss_sales_price#64 ELSE null END))#68,17,2) AS wed_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#63 = Thursday ) THEN ss_sales_price#64 ELSE null END))#69,17,2) AS thu_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#63 = Friday   ) THEN ss_sales_price#64 ELSE null END))#70,17,2) AS fri_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#63 = Saturday ) THEN ss_sales_price#64 ELSE null END))#71,17,2) AS sat_sales#36]
+
+(28) Scan parquet default.store
+Output [2]: [s_store_sk#72, s_store_id#73]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_id)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string>
+
+(29) ColumnarToRow [codegen id : 7]
+Input [2]: [s_store_sk#72, s_store_id#73]
+
+(30) Filter [codegen id : 7]
+Input [2]: [s_store_sk#72, s_store_id#73]
+Condition : (isnotnull(s_store_sk#72) AND isnotnull(s_store_id#73))
+
+(31) BroadcastExchange
+Input [2]: [s_store_sk#72, s_store_id#73]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#74]
+
+(32) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_store_sk#55]
+Right keys [1]: [s_store_sk#72]
+Join condition: None
+
+(33) Project [codegen id : 9]
+Output [9]: [d_week_seq#54, sun_sales#30, mon_sales#31, tue_sales#32, wed_sales#33, thu_sales#34, fri_sales#35, sat_sales#36, s_store_id#73]
+Input [11]: [d_week_seq#54, ss_store_sk#55, sun_sales#30, mon_sales#31, tue_sales#32, wed_sales#33, thu_sales#34, fri_sales#35, sat_sales#36, s_store_sk#72, s_store_id#73]
+
+(34) Scan parquet default.date_dim
+Output [2]: [d_month_seq#75, d_week_seq#76]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1224), LessThanOrEqual(d_month_seq,1235), IsNotNull(d_week_seq)]
+ReadSchema: struct<d_month_seq:int,d_week_seq:int>
+
+(35) ColumnarToRow [codegen id : 8]
+Input [2]: [d_month_seq#75, d_week_seq#76]
+
+(36) Filter [codegen id : 8]
+Input [2]: [d_month_seq#75, d_week_seq#76]
+Condition : (((isnotnull(d_month_seq#75) AND (d_month_seq#75 >= 1224)) AND (d_month_seq#75 <= 1235)) AND isnotnull(d_week_seq#76))
+
+(37) Project [codegen id : 8]
+Output [1]: [d_week_seq#76]
+Input [2]: [d_month_seq#75, d_week_seq#76]
+
+(38) BroadcastExchange
+Input [1]: [d_week_seq#76]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#77]
+
+(39) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [d_week_seq#54]
+Right keys [1]: [d_week_seq#76]
+Join condition: None
+
+(40) Project [codegen id : 9]
+Output [9]: [d_week_seq#54 AS d_week_seq2#78, s_store_id#73 AS s_store_id2#79, sun_sales#30 AS sun_sales2#80, mon_sales#31 AS mon_sales2#81, tue_sales#32 AS tue_sales2#82, wed_sales#33 AS wed_sales2#83, thu_sales#34 AS thu_sales2#84, fri_sales#35 AS fri_sales2#85, sat_sales#36 AS sat_sales2#86]
+Input [10]: [d_week_seq#54, sun_sales#30, mon_sales#31, tue_sales#32, wed_sales#33, thu_sales#34, fri_sales#35, sat_sales#36, s_store_id#73, d_week_seq#76]
+
+(41) BroadcastExchange
+Input [9]: [d_week_seq2#78, s_store_id2#79, sun_sales2#80, mon_sales2#81, tue_sales2#82, wed_sales2#83, thu_sales2#84, fri_sales2#85, sat_sales2#86]
+Arguments: HashedRelationBroadcastMode(List(input[1, string, true], (input[0, int, true] - 52)),false), [id=#87]
+
+(42) BroadcastHashJoin [codegen id : 10]
+Left keys [2]: [s_store_id1#46, d_week_seq1#45]
+Right keys [2]: [s_store_id2#79, (d_week_seq2#78 - 52)]
+Join condition: None
+
+(43) Project [codegen id : 10]
+Output [10]: [s_store_name1#44, s_store_id1#46, d_week_seq1#45, CheckOverflow((promote_precision(sun_sales1#47) / promote_precision(sun_sales2#80)), DecimalType(37,20), true) AS (sun_sales1 / sun_sales2)#88, CheckOverflow((promote_precision(mon_sales1#48) / promote_precision(mon_sales2#81)), DecimalType(37,20), true) AS (mon_sales1 / mon_sales2)#89, CheckOverflow((promote_precision(tue_sales1#49) / promote_precision(tue_sales2#82)), DecimalType(37,20), true) AS (tue_sales1 / tue_sales2)#90, CheckOverflow((promote_precision(wed_sales1#50) / promote_precision(wed_sales2#83)), DecimalType(37,20), true) AS (wed_sales1 / wed_sales2)#91, CheckOverflow((promote_precision(thu_sales1#51) / promote_precision(thu_sales2#84)), DecimalType(37,20), true) AS (thu_sales1 / thu_sales2)#92, CheckOverflow((promote_precision(fri_sales1#52) / promote_precision(fri_sales2#85)), DecimalType(37,20), true) AS (fri_sales1 / fri_sales2)#93, CheckOverflow((promote_precision(sat_sales1#53) / promote_precision(sat_sales2#86)), DecimalType(37,20), true) AS (sat_sales1 / sat_sales2)#94]
+Input [19]: [s_store_name1#44, d_week_seq1#45, s_store_id1#46, sun_sales1#47, mon_sales1#48, tue_sales1#49, wed_sales1#50, thu_sales1#51, fri_sales1#52, sat_sales1#53, d_week_seq2#78, s_store_id2#79, sun_sales2#80, mon_sales2#81, tue_sales2#82, wed_sales2#83, thu_sales2#84, fri_sales2#85, sat_sales2#86]
+
+(44) TakeOrderedAndProject
+Input [10]: [s_store_name1#44, s_store_id1#46, d_week_seq1#45, (sun_sales1 / sun_sales2)#88, (mon_sales1 / mon_sales2)#89, (tue_sales1 / tue_sales2)#90, (wed_sales1 / wed_sales2)#91, (thu_sales1 / thu_sales2)#92, (fri_sales1 / fri_sales2)#93, (sat_sales1 / sat_sales2)#94]
+Arguments: 100, [s_store_name1#44 ASC NULLS FIRST, s_store_id1#46 ASC NULLS FIRST, d_week_seq1#45 ASC NULLS FIRST], [s_store_name1#44, s_store_id1#46, d_week_seq1#45, (sun_sales1 / sun_sales2)#88, (mon_sales1 / mon_sales2)#89, (tue_sales1 / tue_sales2)#90, (wed_sales1 / wed_sales2)#91, (thu_sales1 / thu_sales2)#92, (fri_sales1 / fri_sales2)#93, (sat_sales1 / sat_sales2)#94]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/simplified.txt
new file mode 100644
index 0000000000000..541817641b129
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/simplified.txt
@@ -0,0 +1,66 @@
+TakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_sales2),(mon_sales1 / mon_sales2),(tue_sales1 / tue_sales2),(wed_sales1 / wed_sales2),(thu_sales1 / thu_sales2),(fri_sales1 / fri_sales2),(sat_sales1 / sat_sales2)]
+  WholeStageCodegen (10)
+    Project [s_store_name1,s_store_id1,d_week_seq1,sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2]
+      BroadcastHashJoin [s_store_id1,d_week_seq1,s_store_id2,d_week_seq2]
+        Project [s_store_name,d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales]
+          BroadcastHashJoin [d_week_seq,d_week_seq]
+            Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,s_store_name]
+              BroadcastHashJoin [ss_store_sk,s_store_sk]
+                HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday  ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price ELSE null END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum]
+                  InputAdapter
+                    Exchange [d_week_seq,ss_store_sk] #1
+                      WholeStageCodegen (2)
+                        HashAggregate [d_week_seq,ss_store_sk,d_day_name,ss_sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum]
+                          Project [ss_store_sk,ss_sales_price,d_week_seq,d_day_name]
+                            BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                              Filter [ss_store_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                              InputAdapter
+                                BroadcastExchange #2
+                                  WholeStageCodegen (1)
+                                    Filter [d_date_sk,d_week_seq]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.date_dim [d_date_sk,d_week_seq,d_day_name]
+                InputAdapter
+                  BroadcastExchange #3
+                    WholeStageCodegen (3)
+                      Filter [s_store_sk,s_store_id]
+                        ColumnarToRow
+                          InputAdapter
+                            Scan parquet default.store [s_store_sk,s_store_id,s_store_name]
+            InputAdapter
+              BroadcastExchange #4
+                WholeStageCodegen (4)
+                  Project [d_week_seq]
+                    Filter [d_month_seq,d_week_seq]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.date_dim [d_month_seq,d_week_seq]
+        InputAdapter
+          BroadcastExchange #5
+            WholeStageCodegen (9)
+              Project [d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales]
+                BroadcastHashJoin [d_week_seq,d_week_seq]
+                  Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id]
+                    BroadcastHashJoin [ss_store_sk,s_store_sk]
+                      HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday  ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday   ) THEN ss_sales_price ELSE null END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price ELSE null END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum]
+                        InputAdapter
+                          ReusedExchange [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] #1
+                      InputAdapter
+                        BroadcastExchange #6
+                          WholeStageCodegen (7)
+                            Filter [s_store_sk,s_store_id]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.store [s_store_sk,s_store_id]
+                  InputAdapter
+                    BroadcastExchange #7
+                      WholeStageCodegen (8)
+                        Project [d_week_seq]
+                          Filter [d_month_seq,d_week_seq]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.date_dim [d_month_seq,d_week_seq]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.sf100/explain.txt
new file mode 100644
index 0000000000000..7e82b4d5df296
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.sf100/explain.txt
@@ -0,0 +1,339 @@
+== Physical Plan ==
+TakeOrderedAndProject (50)
++- * Project (49)
+   +- * Filter (48)
+      +- * HashAggregate (47)
+         +- Exchange (46)
+            +- * HashAggregate (45)
+               +- * Project (44)
+                  +- * SortMergeJoin Inner (43)
+                     :- * Sort (28)
+                     :  +- Exchange (27)
+                     :     +- * Project (26)
+                     :        +- * BroadcastHashJoin Inner BuildRight (25)
+                     :           :- * Project (19)
+                     :           :  +- * BroadcastHashJoin Inner BuildRight (18)
+                     :           :     :- * Filter (3)
+                     :           :     :  +- * ColumnarToRow (2)
+                     :           :     :     +- Scan parquet default.store_sales (1)
+                     :           :     +- BroadcastExchange (17)
+                     :           :        +- * Project (16)
+                     :           :           +- * Filter (15)
+                     :           :              +- * BroadcastHashJoin LeftOuter BuildRight (14)
+                     :           :                 :- * Filter (6)
+                     :           :                 :  +- * ColumnarToRow (5)
+                     :           :                 :     +- Scan parquet default.item (4)
+                     :           :                 +- BroadcastExchange (13)
+                     :           :                    +- * HashAggregate (12)
+                     :           :                       +- Exchange (11)
+                     :           :                          +- * HashAggregate (10)
+                     :           :                             +- * Filter (9)
+                     :           :                                +- * ColumnarToRow (8)
+                     :           :                                   +- Scan parquet default.item (7)
+                     :           +- BroadcastExchange (24)
+                     :              +- * Project (23)
+                     :                 +- * Filter (22)
+                     :                    +- * ColumnarToRow (21)
+                     :                       +- Scan parquet default.date_dim (20)
+                     +- * Sort (42)
+                        +- Exchange (41)
+                           +- * Project (40)
+                              +- * SortMergeJoin Inner (39)
+                                 :- * Sort (33)
+                                 :  +- Exchange (32)
+                                 :     +- * Filter (31)
+                                 :        +- * ColumnarToRow (30)
+                                 :           +- Scan parquet default.customer_address (29)
+                                 +- * Sort (38)
+                                    +- Exchange (37)
+                                       +- * Filter (36)
+                                          +- * ColumnarToRow (35)
+                                             +- Scan parquet default.customer (34)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 5]
+Input [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3]
+Condition : (isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1))
+
+(4) Scan parquet default.item
+Output [3]: [i_item_sk#5, i_current_price#6, i_category#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_current_price:decimal(7,2),i_category:string>
+
+(5) ColumnarToRow [codegen id : 3]
+Input [3]: [i_item_sk#5, i_current_price#6, i_category#7]
+
+(6) Filter [codegen id : 3]
+Input [3]: [i_item_sk#5, i_current_price#6, i_category#7]
+Condition : (isnotnull(i_current_price#6) AND isnotnull(i_item_sk#5))
+
+(7) Scan parquet default.item
+Output [2]: [i_current_price#8, i_category#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_category)]
+ReadSchema: struct<i_current_price:decimal(7,2),i_category:string>
+
+(8) ColumnarToRow [codegen id : 1]
+Input [2]: [i_current_price#8, i_category#9]
+
+(9) Filter [codegen id : 1]
+Input [2]: [i_current_price#8, i_category#9]
+Condition : isnotnull(i_category#9)
+
+(10) HashAggregate [codegen id : 1]
+Input [2]: [i_current_price#8, i_category#9]
+Keys [1]: [i_category#9]
+Functions [1]: [partial_avg(UnscaledValue(i_current_price#8))]
+Aggregate Attributes [2]: [sum#10, count#11]
+Results [3]: [i_category#9, sum#12, count#13]
+
+(11) Exchange
+Input [3]: [i_category#9, sum#12, count#13]
+Arguments: hashpartitioning(i_category#9, 5), ENSURE_REQUIREMENTS, [id=#14]
+
+(12) HashAggregate [codegen id : 2]
+Input [3]: [i_category#9, sum#12, count#13]
+Keys [1]: [i_category#9]
+Functions [1]: [avg(UnscaledValue(i_current_price#8))]
+Aggregate Attributes [1]: [avg(UnscaledValue(i_current_price#8))#15]
+Results [2]: [cast((avg(UnscaledValue(i_current_price#8))#15 / 100.0) as decimal(11,6)) AS avg(i_current_price)#16, i_category#9]
+
+(13) BroadcastExchange
+Input [2]: [avg(i_current_price)#16, i_category#9]
+Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [id=#17]
+
+(14) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [i_category#7]
+Right keys [1]: [i_category#9]
+Join condition: None
+
+(15) Filter [codegen id : 3]
+Input [5]: [i_item_sk#5, i_current_price#6, i_category#7, avg(i_current_price)#16, i_category#9]
+Condition : (cast(i_current_price#6 as decimal(14,7)) > CheckOverflow((1.200000 * promote_precision(avg(i_current_price)#16)), DecimalType(14,7), true))
+
+(16) Project [codegen id : 3]
+Output [1]: [i_item_sk#5]
+Input [5]: [i_item_sk#5, i_current_price#6, i_category#7, avg(i_current_price)#16, i_category#9]
+
+(17) BroadcastExchange
+Input [1]: [i_item_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(18) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#5]
+Join condition: None
+
+(19) Project [codegen id : 5]
+Output [2]: [ss_customer_sk#2, ss_sold_date_sk#3]
+Input [4]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3, i_item_sk#5]
+
+(20) Scan parquet default.date_dim
+Output [2]: [d_date_sk#19, d_month_seq#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(21) ColumnarToRow [codegen id : 4]
+Input [2]: [d_date_sk#19, d_month_seq#20]
+
+(22) Filter [codegen id : 4]
+Input [2]: [d_date_sk#19, d_month_seq#20]
+Condition : ((isnotnull(d_month_seq#20) AND (d_month_seq#20 = Subquery scalar-subquery#21, [id=#22])) AND isnotnull(d_date_sk#19))
+
+(23) Project [codegen id : 4]
+Output [1]: [d_date_sk#19]
+Input [2]: [d_date_sk#19, d_month_seq#20]
+
+(24) BroadcastExchange
+Input [1]: [d_date_sk#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#23]
+
+(25) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#19]
+Join condition: None
+
+(26) Project [codegen id : 5]
+Output [1]: [ss_customer_sk#2]
+Input [3]: [ss_customer_sk#2, ss_sold_date_sk#3, d_date_sk#19]
+
+(27) Exchange
+Input [1]: [ss_customer_sk#2]
+Arguments: hashpartitioning(ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(28) Sort [codegen id : 6]
+Input [1]: [ss_customer_sk#2]
+Arguments: [ss_customer_sk#2 ASC NULLS FIRST], false, 0
+
+(29) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#25, ca_state#26]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(30) ColumnarToRow [codegen id : 7]
+Input [2]: [ca_address_sk#25, ca_state#26]
+
+(31) Filter [codegen id : 7]
+Input [2]: [ca_address_sk#25, ca_state#26]
+Condition : isnotnull(ca_address_sk#25)
+
+(32) Exchange
+Input [2]: [ca_address_sk#25, ca_state#26]
+Arguments: hashpartitioning(ca_address_sk#25, 5), ENSURE_REQUIREMENTS, [id=#27]
+
+(33) Sort [codegen id : 8]
+Input [2]: [ca_address_sk#25, ca_state#26]
+Arguments: [ca_address_sk#25 ASC NULLS FIRST], false, 0
+
+(34) Scan parquet default.customer
+Output [2]: [c_customer_sk#28, c_current_addr_sk#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int>
+
+(35) ColumnarToRow [codegen id : 9]
+Input [2]: [c_customer_sk#28, c_current_addr_sk#29]
+
+(36) Filter [codegen id : 9]
+Input [2]: [c_customer_sk#28, c_current_addr_sk#29]
+Condition : (isnotnull(c_current_addr_sk#29) AND isnotnull(c_customer_sk#28))
+
+(37) Exchange
+Input [2]: [c_customer_sk#28, c_current_addr_sk#29]
+Arguments: hashpartitioning(c_current_addr_sk#29, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(38) Sort [codegen id : 10]
+Input [2]: [c_customer_sk#28, c_current_addr_sk#29]
+Arguments: [c_current_addr_sk#29 ASC NULLS FIRST], false, 0
+
+(39) SortMergeJoin [codegen id : 11]
+Left keys [1]: [ca_address_sk#25]
+Right keys [1]: [c_current_addr_sk#29]
+Join condition: None
+
+(40) Project [codegen id : 11]
+Output [2]: [ca_state#26, c_customer_sk#28]
+Input [4]: [ca_address_sk#25, ca_state#26, c_customer_sk#28, c_current_addr_sk#29]
+
+(41) Exchange
+Input [2]: [ca_state#26, c_customer_sk#28]
+Arguments: hashpartitioning(c_customer_sk#28, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(42) Sort [codegen id : 12]
+Input [2]: [ca_state#26, c_customer_sk#28]
+Arguments: [c_customer_sk#28 ASC NULLS FIRST], false, 0
+
+(43) SortMergeJoin [codegen id : 13]
+Left keys [1]: [ss_customer_sk#2]
+Right keys [1]: [c_customer_sk#28]
+Join condition: None
+
+(44) Project [codegen id : 13]
+Output [1]: [ca_state#26]
+Input [3]: [ss_customer_sk#2, ca_state#26, c_customer_sk#28]
+
+(45) HashAggregate [codegen id : 13]
+Input [1]: [ca_state#26]
+Keys [1]: [ca_state#26]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#32]
+Results [2]: [ca_state#26, count#33]
+
+(46) Exchange
+Input [2]: [ca_state#26, count#33]
+Arguments: hashpartitioning(ca_state#26, 5), ENSURE_REQUIREMENTS, [id=#34]
+
+(47) HashAggregate [codegen id : 14]
+Input [2]: [ca_state#26, count#33]
+Keys [1]: [ca_state#26]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#35]
+Results [3]: [ca_state#26 AS state#36, count(1)#35 AS cnt#37, count(1)#35 AS count(1)#38]
+
+(48) Filter [codegen id : 14]
+Input [3]: [state#36, cnt#37, count(1)#38]
+Condition : (count(1)#38 >= 10)
+
+(49) Project [codegen id : 14]
+Output [2]: [state#36, cnt#37]
+Input [3]: [state#36, cnt#37, count(1)#38]
+
+(50) TakeOrderedAndProject
+Input [2]: [state#36, cnt#37]
+Arguments: 100, [cnt#37 ASC NULLS FIRST], [state#36, cnt#37]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (51)
+
+
+(51) ReusedExchange [Reuses operator id: 24]
+Output [1]: [d_date_sk#19]
+
+Subquery:2 Hosting operator id = 22 Hosting Expression = Subquery scalar-subquery#21, [id=#22]
+* HashAggregate (58)
++- Exchange (57)
+   +- * HashAggregate (56)
+      +- * Project (55)
+         +- * Filter (54)
+            +- * ColumnarToRow (53)
+               +- Scan parquet default.date_dim (52)
+
+
+(52) Scan parquet default.date_dim
+Output [3]: [d_month_seq#39, d_year#40, d_moy#41]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)]
+ReadSchema: struct<d_month_seq:int,d_year:int,d_moy:int>
+
+(53) ColumnarToRow [codegen id : 1]
+Input [3]: [d_month_seq#39, d_year#40, d_moy#41]
+
+(54) Filter [codegen id : 1]
+Input [3]: [d_month_seq#39, d_year#40, d_moy#41]
+Condition : (((isnotnull(d_year#40) AND isnotnull(d_moy#41)) AND (d_year#40 = 2000)) AND (d_moy#41 = 1))
+
+(55) Project [codegen id : 1]
+Output [1]: [d_month_seq#39]
+Input [3]: [d_month_seq#39, d_year#40, d_moy#41]
+
+(56) HashAggregate [codegen id : 1]
+Input [1]: [d_month_seq#39]
+Keys [1]: [d_month_seq#39]
+Functions: []
+Aggregate Attributes: []
+Results [1]: [d_month_seq#39]
+
+(57) Exchange
+Input [1]: [d_month_seq#39]
+Arguments: hashpartitioning(d_month_seq#39, 5), ENSURE_REQUIREMENTS, [id=#42]
+
+(58) HashAggregate [codegen id : 2]
+Input [1]: [d_month_seq#39]
+Keys [1]: [d_month_seq#39]
+Functions: []
+Aggregate Attributes: []
+Results [1]: [d_month_seq#39]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.sf100/simplified.txt
new file mode 100644
index 0000000000000..1cdb7a5df8e6f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.sf100/simplified.txt
@@ -0,0 +1,97 @@
+TakeOrderedAndProject [cnt,state]
+  WholeStageCodegen (14)
+    Project [state,cnt]
+      Filter [count(1)]
+        HashAggregate [ca_state,count] [count(1),state,cnt,count(1),count]
+          InputAdapter
+            Exchange [ca_state] #1
+              WholeStageCodegen (13)
+                HashAggregate [ca_state] [count,count]
+                  Project [ca_state]
+                    SortMergeJoin [ss_customer_sk,c_customer_sk]
+                      InputAdapter
+                        WholeStageCodegen (6)
+                          Sort [ss_customer_sk]
+                            InputAdapter
+                              Exchange [ss_customer_sk] #2
+                                WholeStageCodegen (5)
+                                  Project [ss_customer_sk]
+                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                      Project [ss_customer_sk,ss_sold_date_sk]
+                                        BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                          Filter [ss_customer_sk,ss_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk]
+                                                  SubqueryBroadcast [d_date_sk] #1
+                                                    ReusedExchange [d_date_sk] #3
+                                          InputAdapter
+                                            BroadcastExchange #4
+                                              WholeStageCodegen (3)
+                                                Project [i_item_sk]
+                                                  Filter [i_current_price,avg(i_current_price)]
+                                                    BroadcastHashJoin [i_category,i_category]
+                                                      Filter [i_current_price,i_item_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.item [i_item_sk,i_current_price,i_category]
+                                                      InputAdapter
+                                                        BroadcastExchange #5
+                                                          WholeStageCodegen (2)
+                                                            HashAggregate [i_category,sum,count] [avg(UnscaledValue(i_current_price)),avg(i_current_price),sum,count]
+                                                              InputAdapter
+                                                                Exchange [i_category] #6
+                                                                  WholeStageCodegen (1)
+                                                                    HashAggregate [i_category,i_current_price] [sum,count,sum,count]
+                                                                      Filter [i_category]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.item [i_current_price,i_category]
+                                      InputAdapter
+                                        BroadcastExchange #3
+                                          WholeStageCodegen (4)
+                                            Project [d_date_sk]
+                                              Filter [d_month_seq,d_date_sk]
+                                                Subquery #2
+                                                  WholeStageCodegen (2)
+                                                    HashAggregate [d_month_seq]
+                                                      InputAdapter
+                                                        Exchange [d_month_seq] #7
+                                                          WholeStageCodegen (1)
+                                                            HashAggregate [d_month_seq]
+                                                              Project [d_month_seq]
+                                                                Filter [d_year,d_moy]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.date_dim [d_month_seq,d_year,d_moy]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_date_sk,d_month_seq]
+                      InputAdapter
+                        WholeStageCodegen (12)
+                          Sort [c_customer_sk]
+                            InputAdapter
+                              Exchange [c_customer_sk] #8
+                                WholeStageCodegen (11)
+                                  Project [ca_state,c_customer_sk]
+                                    SortMergeJoin [ca_address_sk,c_current_addr_sk]
+                                      InputAdapter
+                                        WholeStageCodegen (8)
+                                          Sort [ca_address_sk]
+                                            InputAdapter
+                                              Exchange [ca_address_sk] #9
+                                                WholeStageCodegen (7)
+                                                  Filter [ca_address_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.customer_address [ca_address_sk,ca_state]
+                                      InputAdapter
+                                        WholeStageCodegen (10)
+                                          Sort [c_current_addr_sk]
+                                            InputAdapter
+                                              Exchange [c_current_addr_sk] #10
+                                                WholeStageCodegen (9)
+                                                  Filter [c_current_addr_sk,c_customer_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.customer [c_customer_sk,c_current_addr_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt
new file mode 100644
index 0000000000000..0f98039fc0f7f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt
@@ -0,0 +1,309 @@
+== Physical Plan ==
+TakeOrderedAndProject (44)
++- * Project (43)
+   +- * Filter (42)
+      +- * HashAggregate (41)
+         +- Exchange (40)
+            +- * HashAggregate (39)
+               +- * Project (38)
+                  +- * BroadcastHashJoin Inner BuildRight (37)
+                     :- * Project (22)
+                     :  +- * BroadcastHashJoin Inner BuildRight (21)
+                     :     :- * Project (15)
+                     :     :  +- * BroadcastHashJoin Inner BuildRight (14)
+                     :     :     :- * Project (9)
+                     :     :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+                     :     :     :     :- * Filter (3)
+                     :     :     :     :  +- * ColumnarToRow (2)
+                     :     :     :     :     +- Scan parquet default.customer_address (1)
+                     :     :     :     +- BroadcastExchange (7)
+                     :     :     :        +- * Filter (6)
+                     :     :     :           +- * ColumnarToRow (5)
+                     :     :     :              +- Scan parquet default.customer (4)
+                     :     :     +- BroadcastExchange (13)
+                     :     :        +- * Filter (12)
+                     :     :           +- * ColumnarToRow (11)
+                     :     :              +- Scan parquet default.store_sales (10)
+                     :     +- BroadcastExchange (20)
+                     :        +- * Project (19)
+                     :           +- * Filter (18)
+                     :              +- * ColumnarToRow (17)
+                     :                 +- Scan parquet default.date_dim (16)
+                     +- BroadcastExchange (36)
+                        +- * Project (35)
+                           +- * Filter (34)
+                              +- * BroadcastHashJoin LeftOuter BuildRight (33)
+                                 :- * Filter (25)
+                                 :  +- * ColumnarToRow (24)
+                                 :     +- Scan parquet default.item (23)
+                                 +- BroadcastExchange (32)
+                                    +- * HashAggregate (31)
+                                       +- Exchange (30)
+                                          +- * HashAggregate (29)
+                                             +- * Filter (28)
+                                                +- * ColumnarToRow (27)
+                                                   +- Scan parquet default.item (26)
+
+
+(1) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#1, ca_state#2]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(2) ColumnarToRow [codegen id : 7]
+Input [2]: [ca_address_sk#1, ca_state#2]
+
+(3) Filter [codegen id : 7]
+Input [2]: [ca_address_sk#1, ca_state#2]
+Condition : isnotnull(ca_address_sk#1)
+
+(4) Scan parquet default.customer
+Output [2]: [c_customer_sk#3, c_current_addr_sk#4]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [c_customer_sk#3, c_current_addr_sk#4]
+
+(6) Filter [codegen id : 1]
+Input [2]: [c_customer_sk#3, c_current_addr_sk#4]
+Condition : (isnotnull(c_current_addr_sk#4) AND isnotnull(c_customer_sk#3))
+
+(7) BroadcastExchange
+Input [2]: [c_customer_sk#3, c_current_addr_sk#4]
+Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [id=#5]
+
+(8) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [ca_address_sk#1]
+Right keys [1]: [c_current_addr_sk#4]
+Join condition: None
+
+(9) Project [codegen id : 7]
+Output [2]: [ca_state#2, c_customer_sk#3]
+Input [4]: [ca_address_sk#1, ca_state#2, c_customer_sk#3, c_current_addr_sk#4]
+
+(10) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8]
+
+(12) Filter [codegen id : 2]
+Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8]
+Condition : (isnotnull(ss_customer_sk#7) AND isnotnull(ss_item_sk#6))
+
+(13) BroadcastExchange
+Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [id=#10]
+
+(14) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [c_customer_sk#3]
+Right keys [1]: [ss_customer_sk#7]
+Join condition: None
+
+(15) Project [codegen id : 7]
+Output [3]: [ca_state#2, ss_item_sk#6, ss_sold_date_sk#8]
+Input [5]: [ca_state#2, c_customer_sk#3, ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8]
+
+(16) Scan parquet default.date_dim
+Output [2]: [d_date_sk#11, d_month_seq#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(17) ColumnarToRow [codegen id : 3]
+Input [2]: [d_date_sk#11, d_month_seq#12]
+
+(18) Filter [codegen id : 3]
+Input [2]: [d_date_sk#11, d_month_seq#12]
+Condition : ((isnotnull(d_month_seq#12) AND (d_month_seq#12 = Subquery scalar-subquery#13, [id=#14])) AND isnotnull(d_date_sk#11))
+
+(19) Project [codegen id : 3]
+Output [1]: [d_date_sk#11]
+Input [2]: [d_date_sk#11, d_month_seq#12]
+
+(20) BroadcastExchange
+Input [1]: [d_date_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#15]
+
+(21) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#11]
+Join condition: None
+
+(22) Project [codegen id : 7]
+Output [2]: [ca_state#2, ss_item_sk#6]
+Input [4]: [ca_state#2, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#11]
+
+(23) Scan parquet default.item
+Output [3]: [i_item_sk#16, i_current_price#17, i_category#18]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_current_price:decimal(7,2),i_category:string>
+
+(24) ColumnarToRow [codegen id : 6]
+Input [3]: [i_item_sk#16, i_current_price#17, i_category#18]
+
+(25) Filter [codegen id : 6]
+Input [3]: [i_item_sk#16, i_current_price#17, i_category#18]
+Condition : (isnotnull(i_current_price#17) AND isnotnull(i_item_sk#16))
+
+(26) Scan parquet default.item
+Output [2]: [i_current_price#19, i_category#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_category)]
+ReadSchema: struct<i_current_price:decimal(7,2),i_category:string>
+
+(27) ColumnarToRow [codegen id : 4]
+Input [2]: [i_current_price#19, i_category#20]
+
+(28) Filter [codegen id : 4]
+Input [2]: [i_current_price#19, i_category#20]
+Condition : isnotnull(i_category#20)
+
+(29) HashAggregate [codegen id : 4]
+Input [2]: [i_current_price#19, i_category#20]
+Keys [1]: [i_category#20]
+Functions [1]: [partial_avg(UnscaledValue(i_current_price#19))]
+Aggregate Attributes [2]: [sum#21, count#22]
+Results [3]: [i_category#20, sum#23, count#24]
+
+(30) Exchange
+Input [3]: [i_category#20, sum#23, count#24]
+Arguments: hashpartitioning(i_category#20, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(31) HashAggregate [codegen id : 5]
+Input [3]: [i_category#20, sum#23, count#24]
+Keys [1]: [i_category#20]
+Functions [1]: [avg(UnscaledValue(i_current_price#19))]
+Aggregate Attributes [1]: [avg(UnscaledValue(i_current_price#19))#26]
+Results [2]: [cast((avg(UnscaledValue(i_current_price#19))#26 / 100.0) as decimal(11,6)) AS avg(i_current_price)#27, i_category#20]
+
+(32) BroadcastExchange
+Input [2]: [avg(i_current_price)#27, i_category#20]
+Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [id=#28]
+
+(33) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [i_category#18]
+Right keys [1]: [i_category#20]
+Join condition: None
+
+(34) Filter [codegen id : 6]
+Input [5]: [i_item_sk#16, i_current_price#17, i_category#18, avg(i_current_price)#27, i_category#20]
+Condition : (cast(i_current_price#17 as decimal(14,7)) > CheckOverflow((1.200000 * promote_precision(avg(i_current_price)#27)), DecimalType(14,7), true))
+
+(35) Project [codegen id : 6]
+Output [1]: [i_item_sk#16]
+Input [5]: [i_item_sk#16, i_current_price#17, i_category#18, avg(i_current_price)#27, i_category#20]
+
+(36) BroadcastExchange
+Input [1]: [i_item_sk#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#29]
+
+(37) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [ss_item_sk#6]
+Right keys [1]: [i_item_sk#16]
+Join condition: None
+
+(38) Project [codegen id : 7]
+Output [1]: [ca_state#2]
+Input [3]: [ca_state#2, ss_item_sk#6, i_item_sk#16]
+
+(39) HashAggregate [codegen id : 7]
+Input [1]: [ca_state#2]
+Keys [1]: [ca_state#2]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#30]
+Results [2]: [ca_state#2, count#31]
+
+(40) Exchange
+Input [2]: [ca_state#2, count#31]
+Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, [id=#32]
+
+(41) HashAggregate [codegen id : 8]
+Input [2]: [ca_state#2, count#31]
+Keys [1]: [ca_state#2]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#33]
+Results [3]: [ca_state#2 AS state#34, count(1)#33 AS cnt#35, count(1)#33 AS count(1)#36]
+
+(42) Filter [codegen id : 8]
+Input [3]: [state#34, cnt#35, count(1)#36]
+Condition : (count(1)#36 >= 10)
+
+(43) Project [codegen id : 8]
+Output [2]: [state#34, cnt#35]
+Input [3]: [state#34, cnt#35, count(1)#36]
+
+(44) TakeOrderedAndProject
+Input [2]: [state#34, cnt#35]
+Arguments: 100, [cnt#35 ASC NULLS FIRST], [state#34, cnt#35]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 10 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (45)
+
+
+(45) ReusedExchange [Reuses operator id: 20]
+Output [1]: [d_date_sk#11]
+
+Subquery:2 Hosting operator id = 18 Hosting Expression = Subquery scalar-subquery#13, [id=#14]
+* HashAggregate (52)
++- Exchange (51)
+   +- * HashAggregate (50)
+      +- * Project (49)
+         +- * Filter (48)
+            +- * ColumnarToRow (47)
+               +- Scan parquet default.date_dim (46)
+
+
+(46) Scan parquet default.date_dim
+Output [3]: [d_month_seq#37, d_year#38, d_moy#39]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)]
+ReadSchema: struct<d_month_seq:int,d_year:int,d_moy:int>
+
+(47) ColumnarToRow [codegen id : 1]
+Input [3]: [d_month_seq#37, d_year#38, d_moy#39]
+
+(48) Filter [codegen id : 1]
+Input [3]: [d_month_seq#37, d_year#38, d_moy#39]
+Condition : (((isnotnull(d_year#38) AND isnotnull(d_moy#39)) AND (d_year#38 = 2000)) AND (d_moy#39 = 1))
+
+(49) Project [codegen id : 1]
+Output [1]: [d_month_seq#37]
+Input [3]: [d_month_seq#37, d_year#38, d_moy#39]
+
+(50) HashAggregate [codegen id : 1]
+Input [1]: [d_month_seq#37]
+Keys [1]: [d_month_seq#37]
+Functions: []
+Aggregate Attributes: []
+Results [1]: [d_month_seq#37]
+
+(51) Exchange
+Input [1]: [d_month_seq#37]
+Arguments: hashpartitioning(d_month_seq#37, 5), ENSURE_REQUIREMENTS, [id=#40]
+
+(52) HashAggregate [codegen id : 2]
+Input [1]: [d_month_seq#37]
+Keys [1]: [d_month_seq#37]
+Functions: []
+Aggregate Attributes: []
+Results [1]: [d_month_seq#37]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/simplified.txt
new file mode 100644
index 0000000000000..d7fad5948f64b
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/simplified.txt
@@ -0,0 +1,79 @@
+TakeOrderedAndProject [cnt,state]
+  WholeStageCodegen (8)
+    Project [state,cnt]
+      Filter [count(1)]
+        HashAggregate [ca_state,count] [count(1),state,cnt,count(1),count]
+          InputAdapter
+            Exchange [ca_state] #1
+              WholeStageCodegen (7)
+                HashAggregate [ca_state] [count,count]
+                  Project [ca_state]
+                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                      Project [ca_state,ss_item_sk]
+                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                          Project [ca_state,ss_item_sk,ss_sold_date_sk]
+                            BroadcastHashJoin [c_customer_sk,ss_customer_sk]
+                              Project [ca_state,c_customer_sk]
+                                BroadcastHashJoin [ca_address_sk,c_current_addr_sk]
+                                  Filter [ca_address_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.customer_address [ca_address_sk,ca_state]
+                                  InputAdapter
+                                    BroadcastExchange #2
+                                      WholeStageCodegen (1)
+                                        Filter [c_current_addr_sk,c_customer_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer [c_customer_sk,c_current_addr_sk]
+                              InputAdapter
+                                BroadcastExchange #3
+                                  WholeStageCodegen (2)
+                                    Filter [ss_customer_sk,ss_item_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk]
+                                            SubqueryBroadcast [d_date_sk] #1
+                                              ReusedExchange [d_date_sk] #4
+                          InputAdapter
+                            BroadcastExchange #4
+                              WholeStageCodegen (3)
+                                Project [d_date_sk]
+                                  Filter [d_month_seq,d_date_sk]
+                                    Subquery #2
+                                      WholeStageCodegen (2)
+                                        HashAggregate [d_month_seq]
+                                          InputAdapter
+                                            Exchange [d_month_seq] #5
+                                              WholeStageCodegen (1)
+                                                HashAggregate [d_month_seq]
+                                                  Project [d_month_seq]
+                                                    Filter [d_year,d_moy]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.date_dim [d_month_seq,d_year,d_moy]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.date_dim [d_date_sk,d_month_seq]
+                      InputAdapter
+                        BroadcastExchange #6
+                          WholeStageCodegen (6)
+                            Project [i_item_sk]
+                              Filter [i_current_price,avg(i_current_price)]
+                                BroadcastHashJoin [i_category,i_category]
+                                  Filter [i_current_price,i_item_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.item [i_item_sk,i_current_price,i_category]
+                                  InputAdapter
+                                    BroadcastExchange #7
+                                      WholeStageCodegen (5)
+                                        HashAggregate [i_category,sum,count] [avg(UnscaledValue(i_current_price)),avg(i_current_price),sum,count]
+                                          InputAdapter
+                                            Exchange [i_category] #8
+                                              WholeStageCodegen (4)
+                                                HashAggregate [i_category,i_current_price] [sum,count,sum,count]
+                                                  Filter [i_category]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.item [i_current_price,i_category]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.sf100/explain.txt
new file mode 100644
index 0000000000000..49cfd232239aa
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.sf100/explain.txt
@@ -0,0 +1,395 @@
+== Physical Plan ==
+TakeOrderedAndProject (67)
++- * HashAggregate (66)
+   +- Exchange (65)
+      +- * HashAggregate (64)
+         +- Union (63)
+            :- * HashAggregate (32)
+            :  +- Exchange (31)
+            :     +- * HashAggregate (30)
+            :        +- * Project (29)
+            :           +- * BroadcastHashJoin Inner BuildRight (28)
+            :              :- * Project (17)
+            :              :  +- * BroadcastHashJoin Inner BuildRight (16)
+            :              :     :- * Project (10)
+            :              :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+            :              :     :     :- * Filter (3)
+            :              :     :     :  +- * ColumnarToRow (2)
+            :              :     :     :     +- Scan parquet default.store_sales (1)
+            :              :     :     +- BroadcastExchange (8)
+            :              :     :        +- * Project (7)
+            :              :     :           +- * Filter (6)
+            :              :     :              +- * ColumnarToRow (5)
+            :              :     :                 +- Scan parquet default.date_dim (4)
+            :              :     +- BroadcastExchange (15)
+            :              :        +- * Project (14)
+            :              :           +- * Filter (13)
+            :              :              +- * ColumnarToRow (12)
+            :              :                 +- Scan parquet default.customer_address (11)
+            :              +- BroadcastExchange (27)
+            :                 +- * BroadcastHashJoin LeftSemi BuildRight (26)
+            :                    :- * Filter (20)
+            :                    :  +- * ColumnarToRow (19)
+            :                    :     +- Scan parquet default.item (18)
+            :                    +- BroadcastExchange (25)
+            :                       +- * Project (24)
+            :                          +- * Filter (23)
+            :                             +- * ColumnarToRow (22)
+            :                                +- Scan parquet default.item (21)
+            :- * HashAggregate (47)
+            :  +- Exchange (46)
+            :     +- * HashAggregate (45)
+            :        +- * Project (44)
+            :           +- * BroadcastHashJoin Inner BuildRight (43)
+            :              :- * Project (41)
+            :              :  +- * BroadcastHashJoin Inner BuildRight (40)
+            :              :     :- * Project (38)
+            :              :     :  +- * BroadcastHashJoin Inner BuildRight (37)
+            :              :     :     :- * Filter (35)
+            :              :     :     :  +- * ColumnarToRow (34)
+            :              :     :     :     +- Scan parquet default.catalog_sales (33)
+            :              :     :     +- ReusedExchange (36)
+            :              :     +- ReusedExchange (39)
+            :              +- ReusedExchange (42)
+            +- * HashAggregate (62)
+               +- Exchange (61)
+                  +- * HashAggregate (60)
+                     +- * Project (59)
+                        +- * BroadcastHashJoin Inner BuildRight (58)
+                           :- * Project (56)
+                           :  +- * BroadcastHashJoin Inner BuildRight (55)
+                           :     :- * Project (53)
+                           :     :  +- * BroadcastHashJoin Inner BuildRight (52)
+                           :     :     :- * Filter (50)
+                           :     :     :  +- * ColumnarToRow (49)
+                           :     :     :     +- Scan parquet default.web_sales (48)
+                           :     :     +- ReusedExchange (51)
+                           :     +- ReusedExchange (54)
+                           +- ReusedExchange (57)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_addr_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 5]
+Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4]
+Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,9), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 1998)) AND (d_moy#8 = 9)) AND isnotnull(d_date_sk#6))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#6]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#9]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3]
+Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#6]
+
+(11) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#10, ca_gmt_offset#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_gmt_offset:decimal(5,2)>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [ca_address_sk#10, ca_gmt_offset#11]
+
+(13) Filter [codegen id : 2]
+Input [2]: [ca_address_sk#10, ca_gmt_offset#11]
+Condition : ((isnotnull(ca_gmt_offset#11) AND (ca_gmt_offset#11 = -5.00)) AND isnotnull(ca_address_sk#10))
+
+(14) Project [codegen id : 2]
+Output [1]: [ca_address_sk#10]
+Input [2]: [ca_address_sk#10, ca_gmt_offset#11]
+
+(15) BroadcastExchange
+Input [1]: [ca_address_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(16) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_addr_sk#2]
+Right keys [1]: [ca_address_sk#10]
+Join condition: None
+
+(17) Project [codegen id : 5]
+Output [2]: [ss_item_sk#1, ss_ext_sales_price#3]
+Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#10]
+
+(18) Scan parquet default.item
+Output [2]: [i_item_sk#13, i_item_id#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(19) ColumnarToRow [codegen id : 4]
+Input [2]: [i_item_sk#13, i_item_id#14]
+
+(20) Filter [codegen id : 4]
+Input [2]: [i_item_sk#13, i_item_id#14]
+Condition : isnotnull(i_item_sk#13)
+
+(21) Scan parquet default.item
+Output [2]: [i_item_id#15, i_category#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Music                                             )]
+ReadSchema: struct<i_item_id:string,i_category:string>
+
+(22) ColumnarToRow [codegen id : 3]
+Input [2]: [i_item_id#15, i_category#16]
+
+(23) Filter [codegen id : 3]
+Input [2]: [i_item_id#15, i_category#16]
+Condition : (isnotnull(i_category#16) AND (i_category#16 = Music                                             ))
+
+(24) Project [codegen id : 3]
+Output [1]: [i_item_id#15]
+Input [2]: [i_item_id#15, i_category#16]
+
+(25) BroadcastExchange
+Input [1]: [i_item_id#15]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#17]
+
+(26) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [i_item_id#14]
+Right keys [1]: [i_item_id#15]
+Join condition: None
+
+(27) BroadcastExchange
+Input [2]: [i_item_sk#13, i_item_id#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#18]
+
+(28) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#13]
+Join condition: None
+
+(29) Project [codegen id : 5]
+Output [2]: [ss_ext_sales_price#3, i_item_id#14]
+Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#13, i_item_id#14]
+
+(30) HashAggregate [codegen id : 5]
+Input [2]: [ss_ext_sales_price#3, i_item_id#14]
+Keys [1]: [i_item_id#14]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))]
+Aggregate Attributes [1]: [sum#19]
+Results [2]: [i_item_id#14, sum#20]
+
+(31) Exchange
+Input [2]: [i_item_id#14, sum#20]
+Arguments: hashpartitioning(i_item_id#14, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(32) HashAggregate [codegen id : 6]
+Input [2]: [i_item_id#14, sum#20]
+Keys [1]: [i_item_id#14]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#22]
+Results [2]: [i_item_id#14, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#22,17,2) AS total_sales#23]
+
+(33) Scan parquet default.catalog_sales
+Output [4]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, cs_sold_date_sk#27]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#27), dynamicpruningexpression(cs_sold_date_sk#27 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_bill_addr_sk:int,cs_item_sk:int,cs_ext_sales_price:decimal(7,2)>
+
+(34) ColumnarToRow [codegen id : 11]
+Input [4]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, cs_sold_date_sk#27]
+
+(35) Filter [codegen id : 11]
+Input [4]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, cs_sold_date_sk#27]
+Condition : (isnotnull(cs_bill_addr_sk#24) AND isnotnull(cs_item_sk#25))
+
+(36) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#28]
+
+(37) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_sold_date_sk#27]
+Right keys [1]: [d_date_sk#28]
+Join condition: None
+
+(38) Project [codegen id : 11]
+Output [3]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26]
+Input [5]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, cs_sold_date_sk#27, d_date_sk#28]
+
+(39) ReusedExchange [Reuses operator id: 15]
+Output [1]: [ca_address_sk#29]
+
+(40) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_bill_addr_sk#24]
+Right keys [1]: [ca_address_sk#29]
+Join condition: None
+
+(41) Project [codegen id : 11]
+Output [2]: [cs_item_sk#25, cs_ext_sales_price#26]
+Input [4]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, ca_address_sk#29]
+
+(42) ReusedExchange [Reuses operator id: 27]
+Output [2]: [i_item_sk#30, i_item_id#31]
+
+(43) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_item_sk#25]
+Right keys [1]: [i_item_sk#30]
+Join condition: None
+
+(44) Project [codegen id : 11]
+Output [2]: [cs_ext_sales_price#26, i_item_id#31]
+Input [4]: [cs_item_sk#25, cs_ext_sales_price#26, i_item_sk#30, i_item_id#31]
+
+(45) HashAggregate [codegen id : 11]
+Input [2]: [cs_ext_sales_price#26, i_item_id#31]
+Keys [1]: [i_item_id#31]
+Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#26))]
+Aggregate Attributes [1]: [sum#32]
+Results [2]: [i_item_id#31, sum#33]
+
+(46) Exchange
+Input [2]: [i_item_id#31, sum#33]
+Arguments: hashpartitioning(i_item_id#31, 5), ENSURE_REQUIREMENTS, [id=#34]
+
+(47) HashAggregate [codegen id : 12]
+Input [2]: [i_item_id#31, sum#33]
+Keys [1]: [i_item_id#31]
+Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#26))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#26))#35]
+Results [2]: [i_item_id#31, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#26))#35,17,2) AS total_sales#36]
+
+(48) Scan parquet default.web_sales
+Output [4]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#40), dynamicpruningexpression(ws_sold_date_sk#40 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_bill_addr_sk:int,ws_ext_sales_price:decimal(7,2)>
+
+(49) ColumnarToRow [codegen id : 17]
+Input [4]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40]
+
+(50) Filter [codegen id : 17]
+Input [4]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40]
+Condition : (isnotnull(ws_bill_addr_sk#38) AND isnotnull(ws_item_sk#37))
+
+(51) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#41]
+
+(52) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_sold_date_sk#40]
+Right keys [1]: [d_date_sk#41]
+Join condition: None
+
+(53) Project [codegen id : 17]
+Output [3]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39]
+Input [5]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40, d_date_sk#41]
+
+(54) ReusedExchange [Reuses operator id: 15]
+Output [1]: [ca_address_sk#42]
+
+(55) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_bill_addr_sk#38]
+Right keys [1]: [ca_address_sk#42]
+Join condition: None
+
+(56) Project [codegen id : 17]
+Output [2]: [ws_item_sk#37, ws_ext_sales_price#39]
+Input [4]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ca_address_sk#42]
+
+(57) ReusedExchange [Reuses operator id: 27]
+Output [2]: [i_item_sk#43, i_item_id#44]
+
+(58) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_item_sk#37]
+Right keys [1]: [i_item_sk#43]
+Join condition: None
+
+(59) Project [codegen id : 17]
+Output [2]: [ws_ext_sales_price#39, i_item_id#44]
+Input [4]: [ws_item_sk#37, ws_ext_sales_price#39, i_item_sk#43, i_item_id#44]
+
+(60) HashAggregate [codegen id : 17]
+Input [2]: [ws_ext_sales_price#39, i_item_id#44]
+Keys [1]: [i_item_id#44]
+Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#39))]
+Aggregate Attributes [1]: [sum#45]
+Results [2]: [i_item_id#44, sum#46]
+
+(61) Exchange
+Input [2]: [i_item_id#44, sum#46]
+Arguments: hashpartitioning(i_item_id#44, 5), ENSURE_REQUIREMENTS, [id=#47]
+
+(62) HashAggregate [codegen id : 18]
+Input [2]: [i_item_id#44, sum#46]
+Keys [1]: [i_item_id#44]
+Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#39))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#39))#48]
+Results [2]: [i_item_id#44, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#39))#48,17,2) AS total_sales#49]
+
+(63) Union
+
+(64) HashAggregate [codegen id : 19]
+Input [2]: [i_item_id#14, total_sales#23]
+Keys [1]: [i_item_id#14]
+Functions [1]: [partial_sum(total_sales#23)]
+Aggregate Attributes [2]: [sum#50, isEmpty#51]
+Results [3]: [i_item_id#14, sum#52, isEmpty#53]
+
+(65) Exchange
+Input [3]: [i_item_id#14, sum#52, isEmpty#53]
+Arguments: hashpartitioning(i_item_id#14, 5), ENSURE_REQUIREMENTS, [id=#54]
+
+(66) HashAggregate [codegen id : 20]
+Input [3]: [i_item_id#14, sum#52, isEmpty#53]
+Keys [1]: [i_item_id#14]
+Functions [1]: [sum(total_sales#23)]
+Aggregate Attributes [1]: [sum(total_sales#23)#55]
+Results [2]: [i_item_id#14, sum(total_sales#23)#55 AS total_sales#56]
+
+(67) TakeOrderedAndProject
+Input [2]: [i_item_id#14, total_sales#56]
+Arguments: 100, [i_item_id#14 ASC NULLS FIRST, total_sales#56 ASC NULLS FIRST], [i_item_id#14, total_sales#56]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (68)
+
+
+(68) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#6]
+
+Subquery:2 Hosting operator id = 33 Hosting Expression = cs_sold_date_sk#27 IN dynamicpruning#5
+
+Subquery:3 Hosting operator id = 48 Hosting Expression = ws_sold_date_sk#40 IN dynamicpruning#5
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.sf100/simplified.txt
new file mode 100644
index 0000000000000..46775417cbb70
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.sf100/simplified.txt
@@ -0,0 +1,105 @@
+TakeOrderedAndProject [i_item_id,total_sales]
+  WholeStageCodegen (20)
+    HashAggregate [i_item_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty]
+      InputAdapter
+        Exchange [i_item_id] #1
+          WholeStageCodegen (19)
+            HashAggregate [i_item_id,total_sales] [sum,isEmpty,sum,isEmpty]
+              InputAdapter
+                Union
+                  WholeStageCodegen (6)
+                    HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum]
+                      InputAdapter
+                        Exchange [i_item_id] #2
+                          WholeStageCodegen (5)
+                            HashAggregate [i_item_id,ss_ext_sales_price] [sum,sum]
+                              Project [ss_ext_sales_price,i_item_id]
+                                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                  Project [ss_item_sk,ss_ext_sales_price]
+                                    BroadcastHashJoin [ss_addr_sk,ca_address_sk]
+                                      Project [ss_item_sk,ss_addr_sk,ss_ext_sales_price]
+                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                          Filter [ss_addr_sk,ss_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                                  SubqueryBroadcast [d_date_sk] #1
+                                                    ReusedExchange [d_date_sk] #3
+                                          InputAdapter
+                                            BroadcastExchange #3
+                                              WholeStageCodegen (1)
+                                                Project [d_date_sk]
+                                                  Filter [d_year,d_moy,d_date_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                      InputAdapter
+                                        BroadcastExchange #4
+                                          WholeStageCodegen (2)
+                                            Project [ca_address_sk]
+                                              Filter [ca_gmt_offset,ca_address_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.customer_address [ca_address_sk,ca_gmt_offset]
+                                  InputAdapter
+                                    BroadcastExchange #5
+                                      WholeStageCodegen (4)
+                                        BroadcastHashJoin [i_item_id,i_item_id]
+                                          Filter [i_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.item [i_item_sk,i_item_id]
+                                          InputAdapter
+                                            BroadcastExchange #6
+                                              WholeStageCodegen (3)
+                                                Project [i_item_id]
+                                                  Filter [i_category]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.item [i_item_id,i_category]
+                  WholeStageCodegen (12)
+                    HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum]
+                      InputAdapter
+                        Exchange [i_item_id] #7
+                          WholeStageCodegen (11)
+                            HashAggregate [i_item_id,cs_ext_sales_price] [sum,sum]
+                              Project [cs_ext_sales_price,i_item_id]
+                                BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                  Project [cs_item_sk,cs_ext_sales_price]
+                                    BroadcastHashJoin [cs_bill_addr_sk,ca_address_sk]
+                                      Project [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price]
+                                        BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                          Filter [cs_bill_addr_sk,cs_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk]
+                                                  ReusedSubquery [d_date_sk] #1
+                                          InputAdapter
+                                            ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        ReusedExchange [ca_address_sk] #4
+                                  InputAdapter
+                                    ReusedExchange [i_item_sk,i_item_id] #5
+                  WholeStageCodegen (18)
+                    HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum]
+                      InputAdapter
+                        Exchange [i_item_id] #8
+                          WholeStageCodegen (17)
+                            HashAggregate [i_item_id,ws_ext_sales_price] [sum,sum]
+                              Project [ws_ext_sales_price,i_item_id]
+                                BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                  Project [ws_item_sk,ws_ext_sales_price]
+                                    BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk]
+                                      Project [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price]
+                                        BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                          Filter [ws_bill_addr_sk,ws_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk]
+                                                  ReusedSubquery [d_date_sk] #1
+                                          InputAdapter
+                                            ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        ReusedExchange [ca_address_sk] #4
+                                  InputAdapter
+                                    ReusedExchange [i_item_sk,i_item_id] #5
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/explain.txt
new file mode 100644
index 0000000000000..49cfd232239aa
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/explain.txt
@@ -0,0 +1,395 @@
+== Physical Plan ==
+TakeOrderedAndProject (67)
++- * HashAggregate (66)
+   +- Exchange (65)
+      +- * HashAggregate (64)
+         +- Union (63)
+            :- * HashAggregate (32)
+            :  +- Exchange (31)
+            :     +- * HashAggregate (30)
+            :        +- * Project (29)
+            :           +- * BroadcastHashJoin Inner BuildRight (28)
+            :              :- * Project (17)
+            :              :  +- * BroadcastHashJoin Inner BuildRight (16)
+            :              :     :- * Project (10)
+            :              :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+            :              :     :     :- * Filter (3)
+            :              :     :     :  +- * ColumnarToRow (2)
+            :              :     :     :     +- Scan parquet default.store_sales (1)
+            :              :     :     +- BroadcastExchange (8)
+            :              :     :        +- * Project (7)
+            :              :     :           +- * Filter (6)
+            :              :     :              +- * ColumnarToRow (5)
+            :              :     :                 +- Scan parquet default.date_dim (4)
+            :              :     +- BroadcastExchange (15)
+            :              :        +- * Project (14)
+            :              :           +- * Filter (13)
+            :              :              +- * ColumnarToRow (12)
+            :              :                 +- Scan parquet default.customer_address (11)
+            :              +- BroadcastExchange (27)
+            :                 +- * BroadcastHashJoin LeftSemi BuildRight (26)
+            :                    :- * Filter (20)
+            :                    :  +- * ColumnarToRow (19)
+            :                    :     +- Scan parquet default.item (18)
+            :                    +- BroadcastExchange (25)
+            :                       +- * Project (24)
+            :                          +- * Filter (23)
+            :                             +- * ColumnarToRow (22)
+            :                                +- Scan parquet default.item (21)
+            :- * HashAggregate (47)
+            :  +- Exchange (46)
+            :     +- * HashAggregate (45)
+            :        +- * Project (44)
+            :           +- * BroadcastHashJoin Inner BuildRight (43)
+            :              :- * Project (41)
+            :              :  +- * BroadcastHashJoin Inner BuildRight (40)
+            :              :     :- * Project (38)
+            :              :     :  +- * BroadcastHashJoin Inner BuildRight (37)
+            :              :     :     :- * Filter (35)
+            :              :     :     :  +- * ColumnarToRow (34)
+            :              :     :     :     +- Scan parquet default.catalog_sales (33)
+            :              :     :     +- ReusedExchange (36)
+            :              :     +- ReusedExchange (39)
+            :              +- ReusedExchange (42)
+            +- * HashAggregate (62)
+               +- Exchange (61)
+                  +- * HashAggregate (60)
+                     +- * Project (59)
+                        +- * BroadcastHashJoin Inner BuildRight (58)
+                           :- * Project (56)
+                           :  +- * BroadcastHashJoin Inner BuildRight (55)
+                           :     :- * Project (53)
+                           :     :  +- * BroadcastHashJoin Inner BuildRight (52)
+                           :     :     :- * Filter (50)
+                           :     :     :  +- * ColumnarToRow (49)
+                           :     :     :     +- Scan parquet default.web_sales (48)
+                           :     :     +- ReusedExchange (51)
+                           :     +- ReusedExchange (54)
+                           +- ReusedExchange (57)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_addr_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 5]
+Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4]
+Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,9), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 1998)) AND (d_moy#8 = 9)) AND isnotnull(d_date_sk#6))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#6]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#9]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3]
+Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#6]
+
+(11) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#10, ca_gmt_offset#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_gmt_offset:decimal(5,2)>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [ca_address_sk#10, ca_gmt_offset#11]
+
+(13) Filter [codegen id : 2]
+Input [2]: [ca_address_sk#10, ca_gmt_offset#11]
+Condition : ((isnotnull(ca_gmt_offset#11) AND (ca_gmt_offset#11 = -5.00)) AND isnotnull(ca_address_sk#10))
+
+(14) Project [codegen id : 2]
+Output [1]: [ca_address_sk#10]
+Input [2]: [ca_address_sk#10, ca_gmt_offset#11]
+
+(15) BroadcastExchange
+Input [1]: [ca_address_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(16) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_addr_sk#2]
+Right keys [1]: [ca_address_sk#10]
+Join condition: None
+
+(17) Project [codegen id : 5]
+Output [2]: [ss_item_sk#1, ss_ext_sales_price#3]
+Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#10]
+
+(18) Scan parquet default.item
+Output [2]: [i_item_sk#13, i_item_id#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(19) ColumnarToRow [codegen id : 4]
+Input [2]: [i_item_sk#13, i_item_id#14]
+
+(20) Filter [codegen id : 4]
+Input [2]: [i_item_sk#13, i_item_id#14]
+Condition : isnotnull(i_item_sk#13)
+
+(21) Scan parquet default.item
+Output [2]: [i_item_id#15, i_category#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Music                                             )]
+ReadSchema: struct<i_item_id:string,i_category:string>
+
+(22) ColumnarToRow [codegen id : 3]
+Input [2]: [i_item_id#15, i_category#16]
+
+(23) Filter [codegen id : 3]
+Input [2]: [i_item_id#15, i_category#16]
+Condition : (isnotnull(i_category#16) AND (i_category#16 = Music                                             ))
+
+(24) Project [codegen id : 3]
+Output [1]: [i_item_id#15]
+Input [2]: [i_item_id#15, i_category#16]
+
+(25) BroadcastExchange
+Input [1]: [i_item_id#15]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#17]
+
+(26) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [i_item_id#14]
+Right keys [1]: [i_item_id#15]
+Join condition: None
+
+(27) BroadcastExchange
+Input [2]: [i_item_sk#13, i_item_id#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#18]
+
+(28) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#13]
+Join condition: None
+
+(29) Project [codegen id : 5]
+Output [2]: [ss_ext_sales_price#3, i_item_id#14]
+Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#13, i_item_id#14]
+
+(30) HashAggregate [codegen id : 5]
+Input [2]: [ss_ext_sales_price#3, i_item_id#14]
+Keys [1]: [i_item_id#14]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))]
+Aggregate Attributes [1]: [sum#19]
+Results [2]: [i_item_id#14, sum#20]
+
+(31) Exchange
+Input [2]: [i_item_id#14, sum#20]
+Arguments: hashpartitioning(i_item_id#14, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(32) HashAggregate [codegen id : 6]
+Input [2]: [i_item_id#14, sum#20]
+Keys [1]: [i_item_id#14]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#22]
+Results [2]: [i_item_id#14, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#22,17,2) AS total_sales#23]
+
+(33) Scan parquet default.catalog_sales
+Output [4]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, cs_sold_date_sk#27]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#27), dynamicpruningexpression(cs_sold_date_sk#27 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_bill_addr_sk:int,cs_item_sk:int,cs_ext_sales_price:decimal(7,2)>
+
+(34) ColumnarToRow [codegen id : 11]
+Input [4]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, cs_sold_date_sk#27]
+
+(35) Filter [codegen id : 11]
+Input [4]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, cs_sold_date_sk#27]
+Condition : (isnotnull(cs_bill_addr_sk#24) AND isnotnull(cs_item_sk#25))
+
+(36) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#28]
+
+(37) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_sold_date_sk#27]
+Right keys [1]: [d_date_sk#28]
+Join condition: None
+
+(38) Project [codegen id : 11]
+Output [3]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26]
+Input [5]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, cs_sold_date_sk#27, d_date_sk#28]
+
+(39) ReusedExchange [Reuses operator id: 15]
+Output [1]: [ca_address_sk#29]
+
+(40) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_bill_addr_sk#24]
+Right keys [1]: [ca_address_sk#29]
+Join condition: None
+
+(41) Project [codegen id : 11]
+Output [2]: [cs_item_sk#25, cs_ext_sales_price#26]
+Input [4]: [cs_bill_addr_sk#24, cs_item_sk#25, cs_ext_sales_price#26, ca_address_sk#29]
+
+(42) ReusedExchange [Reuses operator id: 27]
+Output [2]: [i_item_sk#30, i_item_id#31]
+
+(43) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_item_sk#25]
+Right keys [1]: [i_item_sk#30]
+Join condition: None
+
+(44) Project [codegen id : 11]
+Output [2]: [cs_ext_sales_price#26, i_item_id#31]
+Input [4]: [cs_item_sk#25, cs_ext_sales_price#26, i_item_sk#30, i_item_id#31]
+
+(45) HashAggregate [codegen id : 11]
+Input [2]: [cs_ext_sales_price#26, i_item_id#31]
+Keys [1]: [i_item_id#31]
+Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#26))]
+Aggregate Attributes [1]: [sum#32]
+Results [2]: [i_item_id#31, sum#33]
+
+(46) Exchange
+Input [2]: [i_item_id#31, sum#33]
+Arguments: hashpartitioning(i_item_id#31, 5), ENSURE_REQUIREMENTS, [id=#34]
+
+(47) HashAggregate [codegen id : 12]
+Input [2]: [i_item_id#31, sum#33]
+Keys [1]: [i_item_id#31]
+Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#26))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#26))#35]
+Results [2]: [i_item_id#31, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#26))#35,17,2) AS total_sales#36]
+
+(48) Scan parquet default.web_sales
+Output [4]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#40), dynamicpruningexpression(ws_sold_date_sk#40 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_bill_addr_sk:int,ws_ext_sales_price:decimal(7,2)>
+
+(49) ColumnarToRow [codegen id : 17]
+Input [4]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40]
+
+(50) Filter [codegen id : 17]
+Input [4]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40]
+Condition : (isnotnull(ws_bill_addr_sk#38) AND isnotnull(ws_item_sk#37))
+
+(51) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#41]
+
+(52) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_sold_date_sk#40]
+Right keys [1]: [d_date_sk#41]
+Join condition: None
+
+(53) Project [codegen id : 17]
+Output [3]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39]
+Input [5]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40, d_date_sk#41]
+
+(54) ReusedExchange [Reuses operator id: 15]
+Output [1]: [ca_address_sk#42]
+
+(55) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_bill_addr_sk#38]
+Right keys [1]: [ca_address_sk#42]
+Join condition: None
+
+(56) Project [codegen id : 17]
+Output [2]: [ws_item_sk#37, ws_ext_sales_price#39]
+Input [4]: [ws_item_sk#37, ws_bill_addr_sk#38, ws_ext_sales_price#39, ca_address_sk#42]
+
+(57) ReusedExchange [Reuses operator id: 27]
+Output [2]: [i_item_sk#43, i_item_id#44]
+
+(58) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_item_sk#37]
+Right keys [1]: [i_item_sk#43]
+Join condition: None
+
+(59) Project [codegen id : 17]
+Output [2]: [ws_ext_sales_price#39, i_item_id#44]
+Input [4]: [ws_item_sk#37, ws_ext_sales_price#39, i_item_sk#43, i_item_id#44]
+
+(60) HashAggregate [codegen id : 17]
+Input [2]: [ws_ext_sales_price#39, i_item_id#44]
+Keys [1]: [i_item_id#44]
+Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#39))]
+Aggregate Attributes [1]: [sum#45]
+Results [2]: [i_item_id#44, sum#46]
+
+(61) Exchange
+Input [2]: [i_item_id#44, sum#46]
+Arguments: hashpartitioning(i_item_id#44, 5), ENSURE_REQUIREMENTS, [id=#47]
+
+(62) HashAggregate [codegen id : 18]
+Input [2]: [i_item_id#44, sum#46]
+Keys [1]: [i_item_id#44]
+Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#39))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#39))#48]
+Results [2]: [i_item_id#44, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#39))#48,17,2) AS total_sales#49]
+
+(63) Union
+
+(64) HashAggregate [codegen id : 19]
+Input [2]: [i_item_id#14, total_sales#23]
+Keys [1]: [i_item_id#14]
+Functions [1]: [partial_sum(total_sales#23)]
+Aggregate Attributes [2]: [sum#50, isEmpty#51]
+Results [3]: [i_item_id#14, sum#52, isEmpty#53]
+
+(65) Exchange
+Input [3]: [i_item_id#14, sum#52, isEmpty#53]
+Arguments: hashpartitioning(i_item_id#14, 5), ENSURE_REQUIREMENTS, [id=#54]
+
+(66) HashAggregate [codegen id : 20]
+Input [3]: [i_item_id#14, sum#52, isEmpty#53]
+Keys [1]: [i_item_id#14]
+Functions [1]: [sum(total_sales#23)]
+Aggregate Attributes [1]: [sum(total_sales#23)#55]
+Results [2]: [i_item_id#14, sum(total_sales#23)#55 AS total_sales#56]
+
+(67) TakeOrderedAndProject
+Input [2]: [i_item_id#14, total_sales#56]
+Arguments: 100, [i_item_id#14 ASC NULLS FIRST, total_sales#56 ASC NULLS FIRST], [i_item_id#14, total_sales#56]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (68)
+
+
+(68) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#6]
+
+Subquery:2 Hosting operator id = 33 Hosting Expression = cs_sold_date_sk#27 IN dynamicpruning#5
+
+Subquery:3 Hosting operator id = 48 Hosting Expression = ws_sold_date_sk#40 IN dynamicpruning#5
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/simplified.txt
new file mode 100644
index 0000000000000..46775417cbb70
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/simplified.txt
@@ -0,0 +1,105 @@
+TakeOrderedAndProject [i_item_id,total_sales]
+  WholeStageCodegen (20)
+    HashAggregate [i_item_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty]
+      InputAdapter
+        Exchange [i_item_id] #1
+          WholeStageCodegen (19)
+            HashAggregate [i_item_id,total_sales] [sum,isEmpty,sum,isEmpty]
+              InputAdapter
+                Union
+                  WholeStageCodegen (6)
+                    HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum]
+                      InputAdapter
+                        Exchange [i_item_id] #2
+                          WholeStageCodegen (5)
+                            HashAggregate [i_item_id,ss_ext_sales_price] [sum,sum]
+                              Project [ss_ext_sales_price,i_item_id]
+                                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                  Project [ss_item_sk,ss_ext_sales_price]
+                                    BroadcastHashJoin [ss_addr_sk,ca_address_sk]
+                                      Project [ss_item_sk,ss_addr_sk,ss_ext_sales_price]
+                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                          Filter [ss_addr_sk,ss_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                                  SubqueryBroadcast [d_date_sk] #1
+                                                    ReusedExchange [d_date_sk] #3
+                                          InputAdapter
+                                            BroadcastExchange #3
+                                              WholeStageCodegen (1)
+                                                Project [d_date_sk]
+                                                  Filter [d_year,d_moy,d_date_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                      InputAdapter
+                                        BroadcastExchange #4
+                                          WholeStageCodegen (2)
+                                            Project [ca_address_sk]
+                                              Filter [ca_gmt_offset,ca_address_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.customer_address [ca_address_sk,ca_gmt_offset]
+                                  InputAdapter
+                                    BroadcastExchange #5
+                                      WholeStageCodegen (4)
+                                        BroadcastHashJoin [i_item_id,i_item_id]
+                                          Filter [i_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.item [i_item_sk,i_item_id]
+                                          InputAdapter
+                                            BroadcastExchange #6
+                                              WholeStageCodegen (3)
+                                                Project [i_item_id]
+                                                  Filter [i_category]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.item [i_item_id,i_category]
+                  WholeStageCodegen (12)
+                    HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum]
+                      InputAdapter
+                        Exchange [i_item_id] #7
+                          WholeStageCodegen (11)
+                            HashAggregate [i_item_id,cs_ext_sales_price] [sum,sum]
+                              Project [cs_ext_sales_price,i_item_id]
+                                BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                  Project [cs_item_sk,cs_ext_sales_price]
+                                    BroadcastHashJoin [cs_bill_addr_sk,ca_address_sk]
+                                      Project [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price]
+                                        BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                          Filter [cs_bill_addr_sk,cs_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk]
+                                                  ReusedSubquery [d_date_sk] #1
+                                          InputAdapter
+                                            ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        ReusedExchange [ca_address_sk] #4
+                                  InputAdapter
+                                    ReusedExchange [i_item_sk,i_item_id] #5
+                  WholeStageCodegen (18)
+                    HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum]
+                      InputAdapter
+                        Exchange [i_item_id] #8
+                          WholeStageCodegen (17)
+                            HashAggregate [i_item_id,ws_ext_sales_price] [sum,sum]
+                              Project [ws_ext_sales_price,i_item_id]
+                                BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                  Project [ws_item_sk,ws_ext_sales_price]
+                                    BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk]
+                                      Project [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price]
+                                        BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                          Filter [ws_bill_addr_sk,ws_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk]
+                                                  ReusedSubquery [d_date_sk] #1
+                                          InputAdapter
+                                            ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        ReusedExchange [ca_address_sk] #4
+                                  InputAdapter
+                                    ReusedExchange [i_item_sk,i_item_id] #5
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.sf100/explain.txt
new file mode 100644
index 0000000000000..410fd9bc3d4e7
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.sf100/explain.txt
@@ -0,0 +1,395 @@
+== Physical Plan ==
+* Sort (69)
++- * Project (68)
+   +- * BroadcastNestedLoopJoin Inner BuildRight (67)
+      :- * HashAggregate (47)
+      :  +- Exchange (46)
+      :     +- * HashAggregate (45)
+      :        +- * Project (44)
+      :           +- * BroadcastHashJoin Inner BuildRight (43)
+      :              :- * Project (31)
+      :              :  +- * BroadcastHashJoin Inner BuildRight (30)
+      :              :     :- * Project (24)
+      :              :     :  +- * BroadcastHashJoin Inner BuildRight (23)
+      :              :     :     :- * Project (17)
+      :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (16)
+      :              :     :     :     :- * Project (10)
+      :              :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+      :              :     :     :     :     :- * Filter (3)
+      :              :     :     :     :     :  +- * ColumnarToRow (2)
+      :              :     :     :     :     :     +- Scan parquet default.store_sales (1)
+      :              :     :     :     :     +- BroadcastExchange (8)
+      :              :     :     :     :        +- * Project (7)
+      :              :     :     :     :           +- * Filter (6)
+      :              :     :     :     :              +- * ColumnarToRow (5)
+      :              :     :     :     :                 +- Scan parquet default.date_dim (4)
+      :              :     :     :     +- BroadcastExchange (15)
+      :              :     :     :        +- * Project (14)
+      :              :     :     :           +- * Filter (13)
+      :              :     :     :              +- * ColumnarToRow (12)
+      :              :     :     :                 +- Scan parquet default.item (11)
+      :              :     :     +- BroadcastExchange (22)
+      :              :     :        +- * Project (21)
+      :              :     :           +- * Filter (20)
+      :              :     :              +- * ColumnarToRow (19)
+      :              :     :                 +- Scan parquet default.promotion (18)
+      :              :     +- BroadcastExchange (29)
+      :              :        +- * Project (28)
+      :              :           +- * Filter (27)
+      :              :              +- * ColumnarToRow (26)
+      :              :                 +- Scan parquet default.store (25)
+      :              +- BroadcastExchange (42)
+      :                 +- * Project (41)
+      :                    +- * BroadcastHashJoin Inner BuildRight (40)
+      :                       :- * Filter (34)
+      :                       :  +- * ColumnarToRow (33)
+      :                       :     +- Scan parquet default.customer (32)
+      :                       +- BroadcastExchange (39)
+      :                          +- * Project (38)
+      :                             +- * Filter (37)
+      :                                +- * ColumnarToRow (36)
+      :                                   +- Scan parquet default.customer_address (35)
+      +- BroadcastExchange (66)
+         +- * HashAggregate (65)
+            +- Exchange (64)
+               +- * HashAggregate (63)
+                  +- * Project (62)
+                     +- * BroadcastHashJoin Inner BuildRight (61)
+                        :- * Project (59)
+                        :  +- * BroadcastHashJoin Inner BuildRight (58)
+                        :     :- * Project (56)
+                        :     :  +- * BroadcastHashJoin Inner BuildRight (55)
+                        :     :     :- * Project (53)
+                        :     :     :  +- * BroadcastHashJoin Inner BuildRight (52)
+                        :     :     :     :- * Filter (50)
+                        :     :     :     :  +- * ColumnarToRow (49)
+                        :     :     :     :     +- Scan parquet default.store_sales (48)
+                        :     :     :     +- ReusedExchange (51)
+                        :     :     +- ReusedExchange (54)
+                        :     +- ReusedExchange (57)
+                        +- ReusedExchange (60)
+
+
+(1) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_promo_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 7]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+
+(3) Filter [codegen id : 7]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Condition : (((isnotnull(ss_store_sk#3) AND isnotnull(ss_promo_sk#4)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_item_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#8, d_year#9, d_moy#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#8, d_year#9, d_moy#10]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#8, d_year#9, d_moy#10]
+Condition : ((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 1998)) AND (d_moy#10 = 11)) AND isnotnull(d_date_sk#8))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#8]
+Input [3]: [d_date_sk#8, d_year#9, d_moy#10]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#11]
+
+(9) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [ss_sold_date_sk#6]
+Right keys [1]: [d_date_sk#8]
+Join condition: None
+
+(10) Project [codegen id : 7]
+Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5]
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, d_date_sk#8]
+
+(11) Scan parquet default.item
+Output [2]: [i_item_sk#12, i_category#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Jewelry                                           ), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_category:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [i_item_sk#12, i_category#13]
+
+(13) Filter [codegen id : 2]
+Input [2]: [i_item_sk#12, i_category#13]
+Condition : ((isnotnull(i_category#13) AND (i_category#13 = Jewelry                                           )) AND isnotnull(i_item_sk#12))
+
+(14) Project [codegen id : 2]
+Output [1]: [i_item_sk#12]
+Input [2]: [i_item_sk#12, i_category#13]
+
+(15) BroadcastExchange
+Input [1]: [i_item_sk#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(16) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#12]
+Join condition: None
+
+(17) Project [codegen id : 7]
+Output [4]: [ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, i_item_sk#12]
+
+(18) Scan parquet default.promotion
+Output [4]: [p_promo_sk#15, p_channel_dmail#16, p_channel_email#17, p_channel_tv#18]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/promotion]
+PushedFilters: [Or(Or(EqualTo(p_channel_dmail,Y),EqualTo(p_channel_email,Y)),EqualTo(p_channel_tv,Y)), IsNotNull(p_promo_sk)]
+ReadSchema: struct<p_promo_sk:int,p_channel_dmail:string,p_channel_email:string,p_channel_tv:string>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [4]: [p_promo_sk#15, p_channel_dmail#16, p_channel_email#17, p_channel_tv#18]
+
+(20) Filter [codegen id : 3]
+Input [4]: [p_promo_sk#15, p_channel_dmail#16, p_channel_email#17, p_channel_tv#18]
+Condition : ((((p_channel_dmail#16 = Y) OR (p_channel_email#17 = Y)) OR (p_channel_tv#18 = Y)) AND isnotnull(p_promo_sk#15))
+
+(21) Project [codegen id : 3]
+Output [1]: [p_promo_sk#15]
+Input [4]: [p_promo_sk#15, p_channel_dmail#16, p_channel_email#17, p_channel_tv#18]
+
+(22) BroadcastExchange
+Input [1]: [p_promo_sk#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#19]
+
+(23) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [ss_promo_sk#4]
+Right keys [1]: [p_promo_sk#15]
+Join condition: None
+
+(24) Project [codegen id : 7]
+Output [3]: [ss_customer_sk#2, ss_store_sk#3, ss_ext_sales_price#5]
+Input [5]: [ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, p_promo_sk#15]
+
+(25) Scan parquet default.store
+Output [2]: [s_store_sk#20, s_gmt_offset#21]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_gmt_offset), EqualTo(s_gmt_offset,-5.00), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_gmt_offset:decimal(5,2)>
+
+(26) ColumnarToRow [codegen id : 4]
+Input [2]: [s_store_sk#20, s_gmt_offset#21]
+
+(27) Filter [codegen id : 4]
+Input [2]: [s_store_sk#20, s_gmt_offset#21]
+Condition : ((isnotnull(s_gmt_offset#21) AND (s_gmt_offset#21 = -5.00)) AND isnotnull(s_store_sk#20))
+
+(28) Project [codegen id : 4]
+Output [1]: [s_store_sk#20]
+Input [2]: [s_store_sk#20, s_gmt_offset#21]
+
+(29) BroadcastExchange
+Input [1]: [s_store_sk#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#22]
+
+(30) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#20]
+Join condition: None
+
+(31) Project [codegen id : 7]
+Output [2]: [ss_customer_sk#2, ss_ext_sales_price#5]
+Input [4]: [ss_customer_sk#2, ss_store_sk#3, ss_ext_sales_price#5, s_store_sk#20]
+
+(32) Scan parquet default.customer
+Output [2]: [c_customer_sk#23, c_current_addr_sk#24]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int>
+
+(33) ColumnarToRow [codegen id : 6]
+Input [2]: [c_customer_sk#23, c_current_addr_sk#24]
+
+(34) Filter [codegen id : 6]
+Input [2]: [c_customer_sk#23, c_current_addr_sk#24]
+Condition : (isnotnull(c_customer_sk#23) AND isnotnull(c_current_addr_sk#24))
+
+(35) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#25, ca_gmt_offset#26]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_gmt_offset:decimal(5,2)>
+
+(36) ColumnarToRow [codegen id : 5]
+Input [2]: [ca_address_sk#25, ca_gmt_offset#26]
+
+(37) Filter [codegen id : 5]
+Input [2]: [ca_address_sk#25, ca_gmt_offset#26]
+Condition : ((isnotnull(ca_gmt_offset#26) AND (ca_gmt_offset#26 = -5.00)) AND isnotnull(ca_address_sk#25))
+
+(38) Project [codegen id : 5]
+Output [1]: [ca_address_sk#25]
+Input [2]: [ca_address_sk#25, ca_gmt_offset#26]
+
+(39) BroadcastExchange
+Input [1]: [ca_address_sk#25]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#27]
+
+(40) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [c_current_addr_sk#24]
+Right keys [1]: [ca_address_sk#25]
+Join condition: None
+
+(41) Project [codegen id : 6]
+Output [1]: [c_customer_sk#23]
+Input [3]: [c_customer_sk#23, c_current_addr_sk#24, ca_address_sk#25]
+
+(42) BroadcastExchange
+Input [1]: [c_customer_sk#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#28]
+
+(43) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [ss_customer_sk#2]
+Right keys [1]: [c_customer_sk#23]
+Join condition: None
+
+(44) Project [codegen id : 7]
+Output [1]: [ss_ext_sales_price#5]
+Input [3]: [ss_customer_sk#2, ss_ext_sales_price#5, c_customer_sk#23]
+
+(45) HashAggregate [codegen id : 7]
+Input [1]: [ss_ext_sales_price#5]
+Keys: []
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))]
+Aggregate Attributes [1]: [sum#29]
+Results [1]: [sum#30]
+
+(46) Exchange
+Input [1]: [sum#30]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#31]
+
+(47) HashAggregate [codegen id : 15]
+Input [1]: [sum#30]
+Keys: []
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#32]
+Results [1]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#32,17,2) AS promotions#33]
+
+(48) Scan parquet default.store_sales
+Output [5]: [ss_item_sk#34, ss_customer_sk#35, ss_store_sk#36, ss_ext_sales_price#37, ss_sold_date_sk#38]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#38), dynamicpruningexpression(ss_sold_date_sk#38 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(49) ColumnarToRow [codegen id : 13]
+Input [5]: [ss_item_sk#34, ss_customer_sk#35, ss_store_sk#36, ss_ext_sales_price#37, ss_sold_date_sk#38]
+
+(50) Filter [codegen id : 13]
+Input [5]: [ss_item_sk#34, ss_customer_sk#35, ss_store_sk#36, ss_ext_sales_price#37, ss_sold_date_sk#38]
+Condition : ((isnotnull(ss_store_sk#36) AND isnotnull(ss_customer_sk#35)) AND isnotnull(ss_item_sk#34))
+
+(51) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#39]
+
+(52) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ss_sold_date_sk#38]
+Right keys [1]: [d_date_sk#39]
+Join condition: None
+
+(53) Project [codegen id : 13]
+Output [4]: [ss_item_sk#34, ss_customer_sk#35, ss_store_sk#36, ss_ext_sales_price#37]
+Input [6]: [ss_item_sk#34, ss_customer_sk#35, ss_store_sk#36, ss_ext_sales_price#37, ss_sold_date_sk#38, d_date_sk#39]
+
+(54) ReusedExchange [Reuses operator id: 15]
+Output [1]: [i_item_sk#40]
+
+(55) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ss_item_sk#34]
+Right keys [1]: [i_item_sk#40]
+Join condition: None
+
+(56) Project [codegen id : 13]
+Output [3]: [ss_customer_sk#35, ss_store_sk#36, ss_ext_sales_price#37]
+Input [5]: [ss_item_sk#34, ss_customer_sk#35, ss_store_sk#36, ss_ext_sales_price#37, i_item_sk#40]
+
+(57) ReusedExchange [Reuses operator id: 29]
+Output [1]: [s_store_sk#41]
+
+(58) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ss_store_sk#36]
+Right keys [1]: [s_store_sk#41]
+Join condition: None
+
+(59) Project [codegen id : 13]
+Output [2]: [ss_customer_sk#35, ss_ext_sales_price#37]
+Input [4]: [ss_customer_sk#35, ss_store_sk#36, ss_ext_sales_price#37, s_store_sk#41]
+
+(60) ReusedExchange [Reuses operator id: 42]
+Output [1]: [c_customer_sk#42]
+
+(61) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ss_customer_sk#35]
+Right keys [1]: [c_customer_sk#42]
+Join condition: None
+
+(62) Project [codegen id : 13]
+Output [1]: [ss_ext_sales_price#37]
+Input [3]: [ss_customer_sk#35, ss_ext_sales_price#37, c_customer_sk#42]
+
+(63) HashAggregate [codegen id : 13]
+Input [1]: [ss_ext_sales_price#37]
+Keys: []
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#37))]
+Aggregate Attributes [1]: [sum#43]
+Results [1]: [sum#44]
+
+(64) Exchange
+Input [1]: [sum#44]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#45]
+
+(65) HashAggregate [codegen id : 14]
+Input [1]: [sum#44]
+Keys: []
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#37))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#37))#46]
+Results [1]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#37))#46,17,2) AS total#47]
+
+(66) BroadcastExchange
+Input [1]: [total#47]
+Arguments: IdentityBroadcastMode, [id=#48]
+
+(67) BroadcastNestedLoopJoin [codegen id : 15]
+Join condition: None
+
+(68) Project [codegen id : 15]
+Output [3]: [promotions#33, total#47, CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(promotions#33 as decimal(15,4))) / promote_precision(cast(total#47 as decimal(15,4)))), DecimalType(35,20), true)) * 100.00000000000000000000), DecimalType(38,19), true) AS ((CAST(promotions AS DECIMAL(15,4)) / CAST(total AS DECIMAL(15,4))) * 100)#49]
+Input [2]: [promotions#33, total#47]
+
+(69) Sort [codegen id : 15]
+Input [3]: [promotions#33, total#47, ((CAST(promotions AS DECIMAL(15,4)) / CAST(total AS DECIMAL(15,4))) * 100)#49]
+Arguments: [promotions#33 ASC NULLS FIRST, total#47 ASC NULLS FIRST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7
+ReusedExchange (70)
+
+
+(70) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#8]
+
+Subquery:2 Hosting operator id = 48 Hosting Expression = ss_sold_date_sk#38 IN dynamicpruning#7
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.sf100/simplified.txt
new file mode 100644
index 0000000000000..b4e4877c12ee1
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.sf100/simplified.txt
@@ -0,0 +1,103 @@
+WholeStageCodegen (15)
+  Sort [promotions,total]
+    Project [promotions,total]
+      BroadcastNestedLoopJoin
+        HashAggregate [sum] [sum(UnscaledValue(ss_ext_sales_price)),promotions,sum]
+          InputAdapter
+            Exchange #1
+              WholeStageCodegen (7)
+                HashAggregate [ss_ext_sales_price] [sum,sum]
+                  Project [ss_ext_sales_price]
+                    BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+                      Project [ss_customer_sk,ss_ext_sales_price]
+                        BroadcastHashJoin [ss_store_sk,s_store_sk]
+                          Project [ss_customer_sk,ss_store_sk,ss_ext_sales_price]
+                            BroadcastHashJoin [ss_promo_sk,p_promo_sk]
+                              Project [ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price]
+                                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                  Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price]
+                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                      Filter [ss_store_sk,ss_promo_sk,ss_customer_sk,ss_item_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #1
+                                                ReusedExchange [d_date_sk] #2
+                                      InputAdapter
+                                        BroadcastExchange #2
+                                          WholeStageCodegen (1)
+                                            Project [d_date_sk]
+                                              Filter [d_year,d_moy,d_date_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                  InputAdapter
+                                    BroadcastExchange #3
+                                      WholeStageCodegen (2)
+                                        Project [i_item_sk]
+                                          Filter [i_category,i_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.item [i_item_sk,i_category]
+                              InputAdapter
+                                BroadcastExchange #4
+                                  WholeStageCodegen (3)
+                                    Project [p_promo_sk]
+                                      Filter [p_channel_dmail,p_channel_email,p_channel_tv,p_promo_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.promotion [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv]
+                          InputAdapter
+                            BroadcastExchange #5
+                              WholeStageCodegen (4)
+                                Project [s_store_sk]
+                                  Filter [s_gmt_offset,s_store_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.store [s_store_sk,s_gmt_offset]
+                      InputAdapter
+                        BroadcastExchange #6
+                          WholeStageCodegen (6)
+                            Project [c_customer_sk]
+                              BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                                Filter [c_customer_sk,c_current_addr_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.customer [c_customer_sk,c_current_addr_sk]
+                                InputAdapter
+                                  BroadcastExchange #7
+                                    WholeStageCodegen (5)
+                                      Project [ca_address_sk]
+                                        Filter [ca_gmt_offset,ca_address_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer_address [ca_address_sk,ca_gmt_offset]
+        InputAdapter
+          BroadcastExchange #8
+            WholeStageCodegen (14)
+              HashAggregate [sum] [sum(UnscaledValue(ss_ext_sales_price)),total,sum]
+                InputAdapter
+                  Exchange #9
+                    WholeStageCodegen (13)
+                      HashAggregate [ss_ext_sales_price] [sum,sum]
+                        Project [ss_ext_sales_price]
+                          BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+                            Project [ss_customer_sk,ss_ext_sales_price]
+                              BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                Project [ss_customer_sk,ss_store_sk,ss_ext_sales_price]
+                                  BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                    Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price]
+                                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                        Filter [ss_store_sk,ss_customer_sk,ss_item_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                                ReusedSubquery [d_date_sk] #1
+                                        InputAdapter
+                                          ReusedExchange [d_date_sk] #2
+                                    InputAdapter
+                                      ReusedExchange [i_item_sk] #3
+                                InputAdapter
+                                  ReusedExchange [s_store_sk] #5
+                            InputAdapter
+                              ReusedExchange [c_customer_sk] #6
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/explain.txt
new file mode 100644
index 0000000000000..e0bee37e047cb
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/explain.txt
@@ -0,0 +1,410 @@
+== Physical Plan ==
+* Sort (72)
++- * Project (71)
+   +- * BroadcastNestedLoopJoin Inner BuildRight (70)
+      :- * HashAggregate (47)
+      :  +- Exchange (46)
+      :     +- * HashAggregate (45)
+      :        +- * Project (44)
+      :           +- * BroadcastHashJoin Inner BuildRight (43)
+      :              :- * Project (37)
+      :              :  +- * BroadcastHashJoin Inner BuildRight (36)
+      :              :     :- * Project (30)
+      :              :     :  +- * BroadcastHashJoin Inner BuildRight (29)
+      :              :     :     :- * Project (24)
+      :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (23)
+      :              :     :     :     :- * Project (17)
+      :              :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (16)
+      :              :     :     :     :     :- * Project (10)
+      :              :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+      :              :     :     :     :     :     :- * Filter (3)
+      :              :     :     :     :     :     :  +- * ColumnarToRow (2)
+      :              :     :     :     :     :     :     +- Scan parquet default.store_sales (1)
+      :              :     :     :     :     :     +- BroadcastExchange (8)
+      :              :     :     :     :     :        +- * Project (7)
+      :              :     :     :     :     :           +- * Filter (6)
+      :              :     :     :     :     :              +- * ColumnarToRow (5)
+      :              :     :     :     :     :                 +- Scan parquet default.store (4)
+      :              :     :     :     :     +- BroadcastExchange (15)
+      :              :     :     :     :        +- * Project (14)
+      :              :     :     :     :           +- * Filter (13)
+      :              :     :     :     :              +- * ColumnarToRow (12)
+      :              :     :     :     :                 +- Scan parquet default.promotion (11)
+      :              :     :     :     +- BroadcastExchange (22)
+      :              :     :     :        +- * Project (21)
+      :              :     :     :           +- * Filter (20)
+      :              :     :     :              +- * ColumnarToRow (19)
+      :              :     :     :                 +- Scan parquet default.date_dim (18)
+      :              :     :     +- BroadcastExchange (28)
+      :              :     :        +- * Filter (27)
+      :              :     :           +- * ColumnarToRow (26)
+      :              :     :              +- Scan parquet default.customer (25)
+      :              :     +- BroadcastExchange (35)
+      :              :        +- * Project (34)
+      :              :           +- * Filter (33)
+      :              :              +- * ColumnarToRow (32)
+      :              :                 +- Scan parquet default.customer_address (31)
+      :              +- BroadcastExchange (42)
+      :                 +- * Project (41)
+      :                    +- * Filter (40)
+      :                       +- * ColumnarToRow (39)
+      :                          +- Scan parquet default.item (38)
+      +- BroadcastExchange (69)
+         +- * HashAggregate (68)
+            +- Exchange (67)
+               +- * HashAggregate (66)
+                  +- * Project (65)
+                     +- * BroadcastHashJoin Inner BuildRight (64)
+                        :- * Project (62)
+                        :  +- * BroadcastHashJoin Inner BuildRight (61)
+                        :     :- * Project (59)
+                        :     :  +- * BroadcastHashJoin Inner BuildRight (58)
+                        :     :     :- * Project (56)
+                        :     :     :  +- * BroadcastHashJoin Inner BuildRight (55)
+                        :     :     :     :- * Project (53)
+                        :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (52)
+                        :     :     :     :     :- * Filter (50)
+                        :     :     :     :     :  +- * ColumnarToRow (49)
+                        :     :     :     :     :     +- Scan parquet default.store_sales (48)
+                        :     :     :     :     +- ReusedExchange (51)
+                        :     :     :     +- ReusedExchange (54)
+                        :     :     +- ReusedExchange (57)
+                        :     +- ReusedExchange (60)
+                        +- ReusedExchange (63)
+
+
+(1) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_promo_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 7]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+
+(3) Filter [codegen id : 7]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Condition : (((isnotnull(ss_store_sk#3) AND isnotnull(ss_promo_sk#4)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_item_sk#1))
+
+(4) Scan parquet default.store
+Output [2]: [s_store_sk#8, s_gmt_offset#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_gmt_offset), EqualTo(s_gmt_offset,-5.00), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_gmt_offset:decimal(5,2)>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [s_store_sk#8, s_gmt_offset#9]
+
+(6) Filter [codegen id : 1]
+Input [2]: [s_store_sk#8, s_gmt_offset#9]
+Condition : ((isnotnull(s_gmt_offset#9) AND (s_gmt_offset#9 = -5.00)) AND isnotnull(s_store_sk#8))
+
+(7) Project [codegen id : 1]
+Output [1]: [s_store_sk#8]
+Input [2]: [s_store_sk#8, s_gmt_offset#9]
+
+(8) BroadcastExchange
+Input [1]: [s_store_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(9) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#8]
+Join condition: None
+
+(10) Project [codegen id : 7]
+Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, s_store_sk#8]
+
+(11) Scan parquet default.promotion
+Output [4]: [p_promo_sk#11, p_channel_dmail#12, p_channel_email#13, p_channel_tv#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/promotion]
+PushedFilters: [Or(Or(EqualTo(p_channel_dmail,Y),EqualTo(p_channel_email,Y)),EqualTo(p_channel_tv,Y)), IsNotNull(p_promo_sk)]
+ReadSchema: struct<p_promo_sk:int,p_channel_dmail:string,p_channel_email:string,p_channel_tv:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [4]: [p_promo_sk#11, p_channel_dmail#12, p_channel_email#13, p_channel_tv#14]
+
+(13) Filter [codegen id : 2]
+Input [4]: [p_promo_sk#11, p_channel_dmail#12, p_channel_email#13, p_channel_tv#14]
+Condition : ((((p_channel_dmail#12 = Y) OR (p_channel_email#13 = Y)) OR (p_channel_tv#14 = Y)) AND isnotnull(p_promo_sk#11))
+
+(14) Project [codegen id : 2]
+Output [1]: [p_promo_sk#11]
+Input [4]: [p_promo_sk#11, p_channel_dmail#12, p_channel_email#13, p_channel_tv#14]
+
+(15) BroadcastExchange
+Input [1]: [p_promo_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#15]
+
+(16) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [ss_promo_sk#4]
+Right keys [1]: [p_promo_sk#11]
+Join condition: None
+
+(17) Project [codegen id : 7]
+Output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, p_promo_sk#11]
+
+(18) Scan parquet default.date_dim
+Output [3]: [d_date_sk#16, d_year#17, d_moy#18]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#16, d_year#17, d_moy#18]
+
+(20) Filter [codegen id : 3]
+Input [3]: [d_date_sk#16, d_year#17, d_moy#18]
+Condition : ((((isnotnull(d_year#17) AND isnotnull(d_moy#18)) AND (d_year#17 = 1998)) AND (d_moy#18 = 11)) AND isnotnull(d_date_sk#16))
+
+(21) Project [codegen id : 3]
+Output [1]: [d_date_sk#16]
+Input [3]: [d_date_sk#16, d_year#17, d_moy#18]
+
+(22) BroadcastExchange
+Input [1]: [d_date_sk#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#19]
+
+(23) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [ss_sold_date_sk#6]
+Right keys [1]: [d_date_sk#16]
+Join condition: None
+
+(24) Project [codegen id : 7]
+Output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5]
+Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6, d_date_sk#16]
+
+(25) Scan parquet default.customer
+Output [2]: [c_customer_sk#20, c_current_addr_sk#21]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int>
+
+(26) ColumnarToRow [codegen id : 4]
+Input [2]: [c_customer_sk#20, c_current_addr_sk#21]
+
+(27) Filter [codegen id : 4]
+Input [2]: [c_customer_sk#20, c_current_addr_sk#21]
+Condition : (isnotnull(c_customer_sk#20) AND isnotnull(c_current_addr_sk#21))
+
+(28) BroadcastExchange
+Input [2]: [c_customer_sk#20, c_current_addr_sk#21]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#22]
+
+(29) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [ss_customer_sk#2]
+Right keys [1]: [c_customer_sk#20]
+Join condition: None
+
+(30) Project [codegen id : 7]
+Output [3]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#21]
+Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, c_customer_sk#20, c_current_addr_sk#21]
+
+(31) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#23, ca_gmt_offset#24]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_gmt_offset:decimal(5,2)>
+
+(32) ColumnarToRow [codegen id : 5]
+Input [2]: [ca_address_sk#23, ca_gmt_offset#24]
+
+(33) Filter [codegen id : 5]
+Input [2]: [ca_address_sk#23, ca_gmt_offset#24]
+Condition : ((isnotnull(ca_gmt_offset#24) AND (ca_gmt_offset#24 = -5.00)) AND isnotnull(ca_address_sk#23))
+
+(34) Project [codegen id : 5]
+Output [1]: [ca_address_sk#23]
+Input [2]: [ca_address_sk#23, ca_gmt_offset#24]
+
+(35) BroadcastExchange
+Input [1]: [ca_address_sk#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#25]
+
+(36) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [c_current_addr_sk#21]
+Right keys [1]: [ca_address_sk#23]
+Join condition: None
+
+(37) Project [codegen id : 7]
+Output [2]: [ss_item_sk#1, ss_ext_sales_price#5]
+Input [4]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#21, ca_address_sk#23]
+
+(38) Scan parquet default.item
+Output [2]: [i_item_sk#26, i_category#27]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Jewelry                                           ), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_category:string>
+
+(39) ColumnarToRow [codegen id : 6]
+Input [2]: [i_item_sk#26, i_category#27]
+
+(40) Filter [codegen id : 6]
+Input [2]: [i_item_sk#26, i_category#27]
+Condition : ((isnotnull(i_category#27) AND (i_category#27 = Jewelry                                           )) AND isnotnull(i_item_sk#26))
+
+(41) Project [codegen id : 6]
+Output [1]: [i_item_sk#26]
+Input [2]: [i_item_sk#26, i_category#27]
+
+(42) BroadcastExchange
+Input [1]: [i_item_sk#26]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#28]
+
+(43) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#26]
+Join condition: None
+
+(44) Project [codegen id : 7]
+Output [1]: [ss_ext_sales_price#5]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#5, i_item_sk#26]
+
+(45) HashAggregate [codegen id : 7]
+Input [1]: [ss_ext_sales_price#5]
+Keys: []
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))]
+Aggregate Attributes [1]: [sum#29]
+Results [1]: [sum#30]
+
+(46) Exchange
+Input [1]: [sum#30]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#31]
+
+(47) HashAggregate [codegen id : 15]
+Input [1]: [sum#30]
+Keys: []
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#32]
+Results [1]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#32,17,2) AS promotions#33]
+
+(48) Scan parquet default.store_sales
+Output [5]: [ss_item_sk#34, ss_customer_sk#35, ss_store_sk#36, ss_ext_sales_price#37, ss_sold_date_sk#38]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#38), dynamicpruningexpression(ss_sold_date_sk#38 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(49) ColumnarToRow [codegen id : 13]
+Input [5]: [ss_item_sk#34, ss_customer_sk#35, ss_store_sk#36, ss_ext_sales_price#37, ss_sold_date_sk#38]
+
+(50) Filter [codegen id : 13]
+Input [5]: [ss_item_sk#34, ss_customer_sk#35, ss_store_sk#36, ss_ext_sales_price#37, ss_sold_date_sk#38]
+Condition : ((isnotnull(ss_store_sk#36) AND isnotnull(ss_customer_sk#35)) AND isnotnull(ss_item_sk#34))
+
+(51) ReusedExchange [Reuses operator id: 8]
+Output [1]: [s_store_sk#39]
+
+(52) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ss_store_sk#36]
+Right keys [1]: [s_store_sk#39]
+Join condition: None
+
+(53) Project [codegen id : 13]
+Output [4]: [ss_item_sk#34, ss_customer_sk#35, ss_ext_sales_price#37, ss_sold_date_sk#38]
+Input [6]: [ss_item_sk#34, ss_customer_sk#35, ss_store_sk#36, ss_ext_sales_price#37, ss_sold_date_sk#38, s_store_sk#39]
+
+(54) ReusedExchange [Reuses operator id: 22]
+Output [1]: [d_date_sk#40]
+
+(55) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ss_sold_date_sk#38]
+Right keys [1]: [d_date_sk#40]
+Join condition: None
+
+(56) Project [codegen id : 13]
+Output [3]: [ss_item_sk#34, ss_customer_sk#35, ss_ext_sales_price#37]
+Input [5]: [ss_item_sk#34, ss_customer_sk#35, ss_ext_sales_price#37, ss_sold_date_sk#38, d_date_sk#40]
+
+(57) ReusedExchange [Reuses operator id: 28]
+Output [2]: [c_customer_sk#41, c_current_addr_sk#42]
+
+(58) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ss_customer_sk#35]
+Right keys [1]: [c_customer_sk#41]
+Join condition: None
+
+(59) Project [codegen id : 13]
+Output [3]: [ss_item_sk#34, ss_ext_sales_price#37, c_current_addr_sk#42]
+Input [5]: [ss_item_sk#34, ss_customer_sk#35, ss_ext_sales_price#37, c_customer_sk#41, c_current_addr_sk#42]
+
+(60) ReusedExchange [Reuses operator id: 35]
+Output [1]: [ca_address_sk#43]
+
+(61) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [c_current_addr_sk#42]
+Right keys [1]: [ca_address_sk#43]
+Join condition: None
+
+(62) Project [codegen id : 13]
+Output [2]: [ss_item_sk#34, ss_ext_sales_price#37]
+Input [4]: [ss_item_sk#34, ss_ext_sales_price#37, c_current_addr_sk#42, ca_address_sk#43]
+
+(63) ReusedExchange [Reuses operator id: 42]
+Output [1]: [i_item_sk#44]
+
+(64) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ss_item_sk#34]
+Right keys [1]: [i_item_sk#44]
+Join condition: None
+
+(65) Project [codegen id : 13]
+Output [1]: [ss_ext_sales_price#37]
+Input [3]: [ss_item_sk#34, ss_ext_sales_price#37, i_item_sk#44]
+
+(66) HashAggregate [codegen id : 13]
+Input [1]: [ss_ext_sales_price#37]
+Keys: []
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#37))]
+Aggregate Attributes [1]: [sum#45]
+Results [1]: [sum#46]
+
+(67) Exchange
+Input [1]: [sum#46]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#47]
+
+(68) HashAggregate [codegen id : 14]
+Input [1]: [sum#46]
+Keys: []
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#37))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#37))#48]
+Results [1]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#37))#48,17,2) AS total#49]
+
+(69) BroadcastExchange
+Input [1]: [total#49]
+Arguments: IdentityBroadcastMode, [id=#50]
+
+(70) BroadcastNestedLoopJoin [codegen id : 15]
+Join condition: None
+
+(71) Project [codegen id : 15]
+Output [3]: [promotions#33, total#49, CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(promotions#33 as decimal(15,4))) / promote_precision(cast(total#49 as decimal(15,4)))), DecimalType(35,20), true)) * 100.00000000000000000000), DecimalType(38,19), true) AS ((CAST(promotions AS DECIMAL(15,4)) / CAST(total AS DECIMAL(15,4))) * 100)#51]
+Input [2]: [promotions#33, total#49]
+
+(72) Sort [codegen id : 15]
+Input [3]: [promotions#33, total#49, ((CAST(promotions AS DECIMAL(15,4)) / CAST(total AS DECIMAL(15,4))) * 100)#51]
+Arguments: [promotions#33 ASC NULLS FIRST, total#49 ASC NULLS FIRST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7
+ReusedExchange (73)
+
+
+(73) ReusedExchange [Reuses operator id: 22]
+Output [1]: [d_date_sk#16]
+
+Subquery:2 Hosting operator id = 48 Hosting Expression = ss_sold_date_sk#38 IN dynamicpruning#7
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/simplified.txt
new file mode 100644
index 0000000000000..5a96e4d5b4196
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/simplified.txt
@@ -0,0 +1,107 @@
+WholeStageCodegen (15)
+  Sort [promotions,total]
+    Project [promotions,total]
+      BroadcastNestedLoopJoin
+        HashAggregate [sum] [sum(UnscaledValue(ss_ext_sales_price)),promotions,sum]
+          InputAdapter
+            Exchange #1
+              WholeStageCodegen (7)
+                HashAggregate [ss_ext_sales_price] [sum,sum]
+                  Project [ss_ext_sales_price]
+                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                      Project [ss_item_sk,ss_ext_sales_price]
+                        BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                          Project [ss_item_sk,ss_ext_sales_price,c_current_addr_sk]
+                            BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+                              Project [ss_item_sk,ss_customer_sk,ss_ext_sales_price]
+                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                  Project [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                    BroadcastHashJoin [ss_promo_sk,p_promo_sk]
+                                      Project [ss_item_sk,ss_customer_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                        BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                          Filter [ss_store_sk,ss_promo_sk,ss_customer_sk,ss_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                                  SubqueryBroadcast [d_date_sk] #1
+                                                    ReusedExchange [d_date_sk] #2
+                                          InputAdapter
+                                            BroadcastExchange #3
+                                              WholeStageCodegen (1)
+                                                Project [s_store_sk]
+                                                  Filter [s_gmt_offset,s_store_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.store [s_store_sk,s_gmt_offset]
+                                      InputAdapter
+                                        BroadcastExchange #4
+                                          WholeStageCodegen (2)
+                                            Project [p_promo_sk]
+                                              Filter [p_channel_dmail,p_channel_email,p_channel_tv,p_promo_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.promotion [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv]
+                                  InputAdapter
+                                    BroadcastExchange #2
+                                      WholeStageCodegen (3)
+                                        Project [d_date_sk]
+                                          Filter [d_year,d_moy,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                              InputAdapter
+                                BroadcastExchange #5
+                                  WholeStageCodegen (4)
+                                    Filter [c_customer_sk,c_current_addr_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.customer [c_customer_sk,c_current_addr_sk]
+                          InputAdapter
+                            BroadcastExchange #6
+                              WholeStageCodegen (5)
+                                Project [ca_address_sk]
+                                  Filter [ca_gmt_offset,ca_address_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.customer_address [ca_address_sk,ca_gmt_offset]
+                      InputAdapter
+                        BroadcastExchange #7
+                          WholeStageCodegen (6)
+                            Project [i_item_sk]
+                              Filter [i_category,i_item_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.item [i_item_sk,i_category]
+        InputAdapter
+          BroadcastExchange #8
+            WholeStageCodegen (14)
+              HashAggregate [sum] [sum(UnscaledValue(ss_ext_sales_price)),total,sum]
+                InputAdapter
+                  Exchange #9
+                    WholeStageCodegen (13)
+                      HashAggregate [ss_ext_sales_price] [sum,sum]
+                        Project [ss_ext_sales_price]
+                          BroadcastHashJoin [ss_item_sk,i_item_sk]
+                            Project [ss_item_sk,ss_ext_sales_price]
+                              BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                                Project [ss_item_sk,ss_ext_sales_price,c_current_addr_sk]
+                                  BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+                                    Project [ss_item_sk,ss_customer_sk,ss_ext_sales_price]
+                                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                        Project [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                          BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                            Filter [ss_store_sk,ss_customer_sk,ss_item_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                                    ReusedSubquery [d_date_sk] #1
+                                            InputAdapter
+                                              ReusedExchange [s_store_sk] #3
+                                        InputAdapter
+                                          ReusedExchange [d_date_sk] #2
+                                    InputAdapter
+                                      ReusedExchange [c_customer_sk,c_current_addr_sk] #5
+                                InputAdapter
+                                  ReusedExchange [ca_address_sk] #6
+                            InputAdapter
+                              ReusedExchange [i_item_sk] #7
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.sf100/explain.txt
new file mode 100644
index 0000000000000..0c82f6182c240
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.sf100/explain.txt
@@ -0,0 +1,183 @@
+== Physical Plan ==
+TakeOrderedAndProject (32)
++- * HashAggregate (31)
+   +- Exchange (30)
+      +- * HashAggregate (29)
+         +- * Project (28)
+            +- * BroadcastHashJoin Inner BuildRight (27)
+               :- * Project (22)
+               :  +- * BroadcastHashJoin Inner BuildRight (21)
+               :     :- * Project (16)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (15)
+               :     :     :- * Project (10)
+               :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :     :     :- * Filter (3)
+               :     :     :     :  +- * ColumnarToRow (2)
+               :     :     :     :     +- Scan parquet default.web_sales (1)
+               :     :     :     +- BroadcastExchange (8)
+               :     :     :        +- * Project (7)
+               :     :     :           +- * Filter (6)
+               :     :     :              +- * ColumnarToRow (5)
+               :     :     :                 +- Scan parquet default.date_dim (4)
+               :     :     +- BroadcastExchange (14)
+               :     :        +- * Filter (13)
+               :     :           +- * ColumnarToRow (12)
+               :     :              +- Scan parquet default.web_site (11)
+               :     +- BroadcastExchange (20)
+               :        +- * Filter (19)
+               :           +- * ColumnarToRow (18)
+               :              +- Scan parquet default.ship_mode (17)
+               +- BroadcastExchange (26)
+                  +- * Filter (25)
+                     +- * ColumnarToRow (24)
+                        +- Scan parquet default.warehouse (23)
+
+
+(1) Scan parquet default.web_sales
+Output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_sales]
+PushedFilters: [IsNotNull(ws_warehouse_sk), IsNotNull(ws_ship_mode_sk), IsNotNull(ws_web_site_sk), IsNotNull(ws_ship_date_sk)]
+ReadSchema: struct<ws_ship_date_sk:int,ws_web_site_sk:int,ws_ship_mode_sk:int,ws_warehouse_sk:int>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5]
+
+(3) Filter [codegen id : 5]
+Input [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5]
+Condition : (((isnotnull(ws_warehouse_sk#4) AND isnotnull(ws_ship_mode_sk#3)) AND isnotnull(ws_web_site_sk#2)) AND isnotnull(ws_ship_date_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#6, d_month_seq#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#6, d_month_seq#7]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#6, d_month_seq#7]
+Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#6))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#6]
+Input [2]: [d_date_sk#6, d_month_seq#7]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#8]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ws_ship_date_sk#1]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5]
+Input [6]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5, d_date_sk#6]
+
+(11) Scan parquet default.web_site
+Output [2]: [web_site_sk#9, web_name#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_site]
+PushedFilters: [IsNotNull(web_site_sk)]
+ReadSchema: struct<web_site_sk:int,web_name:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [web_site_sk#9, web_name#10]
+
+(13) Filter [codegen id : 2]
+Input [2]: [web_site_sk#9, web_name#10]
+Condition : isnotnull(web_site_sk#9)
+
+(14) BroadcastExchange
+Input [2]: [web_site_sk#9, web_name#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(15) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ws_web_site_sk#2]
+Right keys [1]: [web_site_sk#9]
+Join condition: None
+
+(16) Project [codegen id : 5]
+Output [5]: [ws_ship_date_sk#1, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5, web_name#10]
+Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5, web_site_sk#9, web_name#10]
+
+(17) Scan parquet default.ship_mode
+Output [2]: [sm_ship_mode_sk#12, sm_type#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/ship_mode]
+PushedFilters: [IsNotNull(sm_ship_mode_sk)]
+ReadSchema: struct<sm_ship_mode_sk:int,sm_type:string>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [2]: [sm_ship_mode_sk#12, sm_type#13]
+
+(19) Filter [codegen id : 3]
+Input [2]: [sm_ship_mode_sk#12, sm_type#13]
+Condition : isnotnull(sm_ship_mode_sk#12)
+
+(20) BroadcastExchange
+Input [2]: [sm_ship_mode_sk#12, sm_type#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#14]
+
+(21) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ws_ship_mode_sk#3]
+Right keys [1]: [sm_ship_mode_sk#12]
+Join condition: None
+
+(22) Project [codegen id : 5]
+Output [5]: [ws_ship_date_sk#1, ws_warehouse_sk#4, ws_sold_date_sk#5, web_name#10, sm_type#13]
+Input [7]: [ws_ship_date_sk#1, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5, web_name#10, sm_ship_mode_sk#12, sm_type#13]
+
+(23) Scan parquet default.warehouse
+Output [2]: [w_warehouse_sk#15, w_warehouse_name#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/warehouse]
+PushedFilters: [IsNotNull(w_warehouse_sk)]
+ReadSchema: struct<w_warehouse_sk:int,w_warehouse_name:string>
+
+(24) ColumnarToRow [codegen id : 4]
+Input [2]: [w_warehouse_sk#15, w_warehouse_name#16]
+
+(25) Filter [codegen id : 4]
+Input [2]: [w_warehouse_sk#15, w_warehouse_name#16]
+Condition : isnotnull(w_warehouse_sk#15)
+
+(26) BroadcastExchange
+Input [2]: [w_warehouse_sk#15, w_warehouse_name#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#17]
+
+(27) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ws_warehouse_sk#4]
+Right keys [1]: [w_warehouse_sk#15]
+Join condition: None
+
+(28) Project [codegen id : 5]
+Output [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#13, web_name#10, substr(w_warehouse_name#16, 1, 20) AS _groupingexpression#18]
+Input [7]: [ws_ship_date_sk#1, ws_warehouse_sk#4, ws_sold_date_sk#5, web_name#10, sm_type#13, w_warehouse_sk#15, w_warehouse_name#16]
+
+(29) HashAggregate [codegen id : 5]
+Input [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#13, web_name#10, _groupingexpression#18]
+Keys [3]: [_groupingexpression#18, sm_type#13, web_name#10]
+Functions [5]: [partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)]
+Aggregate Attributes [5]: [sum#19, sum#20, sum#21, sum#22, sum#23]
+Results [8]: [_groupingexpression#18, sm_type#13, web_name#10, sum#24, sum#25, sum#26, sum#27, sum#28]
+
+(30) Exchange
+Input [8]: [_groupingexpression#18, sm_type#13, web_name#10, sum#24, sum#25, sum#26, sum#27, sum#28]
+Arguments: hashpartitioning(_groupingexpression#18, sm_type#13, web_name#10, 5), ENSURE_REQUIREMENTS, [id=#29]
+
+(31) HashAggregate [codegen id : 6]
+Input [8]: [_groupingexpression#18, sm_type#13, web_name#10, sum#24, sum#25, sum#26, sum#27, sum#28]
+Keys [3]: [_groupingexpression#18, sm_type#13, web_name#10]
+Functions [5]: [sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)]
+Aggregate Attributes [5]: [sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#30, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#31, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#32, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#33, sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#34]
+Results [8]: [_groupingexpression#18 AS substr(w_warehouse_name, 1, 20)#35, sm_type#13, web_name#10, sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#30 AS 30 days #36, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#31 AS 31 - 60 days #37, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#32 AS 61 - 90 days #38, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#33 AS 91 - 120 days #39, sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#34 AS >120 days #40]
+
+(32) TakeOrderedAndProject
+Input [8]: [substr(w_warehouse_name, 1, 20)#35, sm_type#13, web_name#10, 30 days #36, 31 - 60 days #37, 61 - 90 days #38, 91 - 120 days #39, >120 days #40]
+Arguments: 100, [substr(w_warehouse_name, 1, 20)#35 ASC NULLS FIRST, sm_type#13 ASC NULLS FIRST, web_name#10 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#35, sm_type#13, web_name#10, 30 days #36, 31 - 60 days #37, 61 - 90 days #38, 91 - 120 days #39, >120 days #40]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.sf100/simplified.txt
new file mode 100644
index 0000000000000..5d48a8701abe6
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.sf100/simplified.txt
@@ -0,0 +1,48 @@
+TakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ]
+  WholeStageCodegen (6)
+    HashAggregate [_groupingexpression,sm_type,web_name,sum,sum,sum,sum,sum] [sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 30) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 60) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 90) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) > 120) THEN 1 ELSE 0 END),substr(w_warehouse_name, 1, 20),30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum]
+      InputAdapter
+        Exchange [_groupingexpression,sm_type,web_name] #1
+          WholeStageCodegen (5)
+            HashAggregate [_groupingexpression,sm_type,web_name,ws_ship_date_sk,ws_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum]
+              Project [ws_ship_date_sk,ws_sold_date_sk,sm_type,web_name,w_warehouse_name]
+                BroadcastHashJoin [ws_warehouse_sk,w_warehouse_sk]
+                  Project [ws_ship_date_sk,ws_warehouse_sk,ws_sold_date_sk,web_name,sm_type]
+                    BroadcastHashJoin [ws_ship_mode_sk,sm_ship_mode_sk]
+                      Project [ws_ship_date_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk,web_name]
+                        BroadcastHashJoin [ws_web_site_sk,web_site_sk]
+                          Project [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk]
+                            BroadcastHashJoin [ws_ship_date_sk,d_date_sk]
+                              Filter [ws_warehouse_sk,ws_ship_mode_sk,ws_web_site_sk,ws_ship_date_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.web_sales [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk]
+                              InputAdapter
+                                BroadcastExchange #2
+                                  WholeStageCodegen (1)
+                                    Project [d_date_sk]
+                                      Filter [d_month_seq,d_date_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.date_dim [d_date_sk,d_month_seq]
+                          InputAdapter
+                            BroadcastExchange #3
+                              WholeStageCodegen (2)
+                                Filter [web_site_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.web_site [web_site_sk,web_name]
+                      InputAdapter
+                        BroadcastExchange #4
+                          WholeStageCodegen (3)
+                            Filter [sm_ship_mode_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.ship_mode [sm_ship_mode_sk,sm_type]
+                  InputAdapter
+                    BroadcastExchange #5
+                      WholeStageCodegen (4)
+                        Filter [w_warehouse_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.warehouse [w_warehouse_sk,w_warehouse_name]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62/explain.txt
new file mode 100644
index 0000000000000..752025ebea0a5
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62/explain.txt
@@ -0,0 +1,183 @@
+== Physical Plan ==
+TakeOrderedAndProject (32)
++- * HashAggregate (31)
+   +- Exchange (30)
+      +- * HashAggregate (29)
+         +- * Project (28)
+            +- * BroadcastHashJoin Inner BuildRight (27)
+               :- * Project (21)
+               :  +- * BroadcastHashJoin Inner BuildRight (20)
+               :     :- * Project (15)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (14)
+               :     :     :- * Project (9)
+               :     :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+               :     :     :     :- * Filter (3)
+               :     :     :     :  +- * ColumnarToRow (2)
+               :     :     :     :     +- Scan parquet default.web_sales (1)
+               :     :     :     +- BroadcastExchange (7)
+               :     :     :        +- * Filter (6)
+               :     :     :           +- * ColumnarToRow (5)
+               :     :     :              +- Scan parquet default.warehouse (4)
+               :     :     +- BroadcastExchange (13)
+               :     :        +- * Filter (12)
+               :     :           +- * ColumnarToRow (11)
+               :     :              +- Scan parquet default.ship_mode (10)
+               :     +- BroadcastExchange (19)
+               :        +- * Filter (18)
+               :           +- * ColumnarToRow (17)
+               :              +- Scan parquet default.web_site (16)
+               +- BroadcastExchange (26)
+                  +- * Project (25)
+                     +- * Filter (24)
+                        +- * ColumnarToRow (23)
+                           +- Scan parquet default.date_dim (22)
+
+
+(1) Scan parquet default.web_sales
+Output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_sales]
+PushedFilters: [IsNotNull(ws_warehouse_sk), IsNotNull(ws_ship_mode_sk), IsNotNull(ws_web_site_sk), IsNotNull(ws_ship_date_sk)]
+ReadSchema: struct<ws_ship_date_sk:int,ws_web_site_sk:int,ws_ship_mode_sk:int,ws_warehouse_sk:int>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5]
+
+(3) Filter [codegen id : 5]
+Input [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5]
+Condition : (((isnotnull(ws_warehouse_sk#4) AND isnotnull(ws_ship_mode_sk#3)) AND isnotnull(ws_web_site_sk#2)) AND isnotnull(ws_ship_date_sk#1))
+
+(4) Scan parquet default.warehouse
+Output [2]: [w_warehouse_sk#6, w_warehouse_name#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/warehouse]
+PushedFilters: [IsNotNull(w_warehouse_sk)]
+ReadSchema: struct<w_warehouse_sk:int,w_warehouse_name:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [w_warehouse_sk#6, w_warehouse_name#7]
+
+(6) Filter [codegen id : 1]
+Input [2]: [w_warehouse_sk#6, w_warehouse_name#7]
+Condition : isnotnull(w_warehouse_sk#6)
+
+(7) BroadcastExchange
+Input [2]: [w_warehouse_sk#6, w_warehouse_name#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#8]
+
+(8) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ws_warehouse_sk#4]
+Right keys [1]: [w_warehouse_sk#6]
+Join condition: None
+
+(9) Project [codegen id : 5]
+Output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7]
+Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5, w_warehouse_sk#6, w_warehouse_name#7]
+
+(10) Scan parquet default.ship_mode
+Output [2]: [sm_ship_mode_sk#9, sm_type#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/ship_mode]
+PushedFilters: [IsNotNull(sm_ship_mode_sk)]
+ReadSchema: struct<sm_ship_mode_sk:int,sm_type:string>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [sm_ship_mode_sk#9, sm_type#10]
+
+(12) Filter [codegen id : 2]
+Input [2]: [sm_ship_mode_sk#9, sm_type#10]
+Condition : isnotnull(sm_ship_mode_sk#9)
+
+(13) BroadcastExchange
+Input [2]: [sm_ship_mode_sk#9, sm_type#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(14) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ws_ship_mode_sk#3]
+Right keys [1]: [sm_ship_mode_sk#9]
+Join condition: None
+
+(15) Project [codegen id : 5]
+Output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10]
+Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#9, sm_type#10]
+
+(16) Scan parquet default.web_site
+Output [2]: [web_site_sk#12, web_name#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_site]
+PushedFilters: [IsNotNull(web_site_sk)]
+ReadSchema: struct<web_site_sk:int,web_name:string>
+
+(17) ColumnarToRow [codegen id : 3]
+Input [2]: [web_site_sk#12, web_name#13]
+
+(18) Filter [codegen id : 3]
+Input [2]: [web_site_sk#12, web_name#13]
+Condition : isnotnull(web_site_sk#12)
+
+(19) BroadcastExchange
+Input [2]: [web_site_sk#12, web_name#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#14]
+
+(20) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ws_web_site_sk#2]
+Right keys [1]: [web_site_sk#12]
+Join condition: None
+
+(21) Project [codegen id : 5]
+Output [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#13]
+Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_site_sk#12, web_name#13]
+
+(22) Scan parquet default.date_dim
+Output [2]: [d_date_sk#15, d_month_seq#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(23) ColumnarToRow [codegen id : 4]
+Input [2]: [d_date_sk#15, d_month_seq#16]
+
+(24) Filter [codegen id : 4]
+Input [2]: [d_date_sk#15, d_month_seq#16]
+Condition : (((isnotnull(d_month_seq#16) AND (d_month_seq#16 >= 1200)) AND (d_month_seq#16 <= 1211)) AND isnotnull(d_date_sk#15))
+
+(25) Project [codegen id : 4]
+Output [1]: [d_date_sk#15]
+Input [2]: [d_date_sk#15, d_month_seq#16]
+
+(26) BroadcastExchange
+Input [1]: [d_date_sk#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(27) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ws_ship_date_sk#1]
+Right keys [1]: [d_date_sk#15]
+Join condition: None
+
+(28) Project [codegen id : 5]
+Output [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#10, web_name#13, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#18]
+Input [6]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#13, d_date_sk#15]
+
+(29) HashAggregate [codegen id : 5]
+Input [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#10, web_name#13, _groupingexpression#18]
+Keys [3]: [_groupingexpression#18, sm_type#10, web_name#13]
+Functions [5]: [partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)]
+Aggregate Attributes [5]: [sum#19, sum#20, sum#21, sum#22, sum#23]
+Results [8]: [_groupingexpression#18, sm_type#10, web_name#13, sum#24, sum#25, sum#26, sum#27, sum#28]
+
+(30) Exchange
+Input [8]: [_groupingexpression#18, sm_type#10, web_name#13, sum#24, sum#25, sum#26, sum#27, sum#28]
+Arguments: hashpartitioning(_groupingexpression#18, sm_type#10, web_name#13, 5), ENSURE_REQUIREMENTS, [id=#29]
+
+(31) HashAggregate [codegen id : 6]
+Input [8]: [_groupingexpression#18, sm_type#10, web_name#13, sum#24, sum#25, sum#26, sum#27, sum#28]
+Keys [3]: [_groupingexpression#18, sm_type#10, web_name#13]
+Functions [5]: [sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)]
+Aggregate Attributes [5]: [sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#30, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#31, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#32, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#33, sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#34]
+Results [8]: [_groupingexpression#18 AS substr(w_warehouse_name, 1, 20)#35, sm_type#10, web_name#13, sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#30 AS 30 days #36, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#31 AS 31 - 60 days #37, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#32 AS 61 - 90 days #38, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#33 AS 91 - 120 days #39, sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#34 AS >120 days #40]
+
+(32) TakeOrderedAndProject
+Input [8]: [substr(w_warehouse_name, 1, 20)#35, sm_type#10, web_name#13, 30 days #36, 31 - 60 days #37, 61 - 90 days #38, 91 - 120 days #39, >120 days #40]
+Arguments: 100, [substr(w_warehouse_name, 1, 20)#35 ASC NULLS FIRST, sm_type#10 ASC NULLS FIRST, web_name#13 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#35, sm_type#10, web_name#13, 30 days #36, 31 - 60 days #37, 61 - 90 days #38, 91 - 120 days #39, >120 days #40]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62/simplified.txt
new file mode 100644
index 0000000000000..7b67c51892d9a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62/simplified.txt
@@ -0,0 +1,48 @@
+TakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ]
+  WholeStageCodegen (6)
+    HashAggregate [_groupingexpression,sm_type,web_name,sum,sum,sum,sum,sum] [sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 30) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 60) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 90) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) > 120) THEN 1 ELSE 0 END),substr(w_warehouse_name, 1, 20),30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum]
+      InputAdapter
+        Exchange [_groupingexpression,sm_type,web_name] #1
+          WholeStageCodegen (5)
+            HashAggregate [_groupingexpression,sm_type,web_name,ws_ship_date_sk,ws_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum]
+              Project [ws_ship_date_sk,ws_sold_date_sk,sm_type,web_name,w_warehouse_name]
+                BroadcastHashJoin [ws_ship_date_sk,d_date_sk]
+                  Project [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name]
+                    BroadcastHashJoin [ws_web_site_sk,web_site_sk]
+                      Project [ws_ship_date_sk,ws_web_site_sk,ws_sold_date_sk,w_warehouse_name,sm_type]
+                        BroadcastHashJoin [ws_ship_mode_sk,sm_ship_mode_sk]
+                          Project [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_sold_date_sk,w_warehouse_name]
+                            BroadcastHashJoin [ws_warehouse_sk,w_warehouse_sk]
+                              Filter [ws_warehouse_sk,ws_ship_mode_sk,ws_web_site_sk,ws_ship_date_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.web_sales [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk]
+                              InputAdapter
+                                BroadcastExchange #2
+                                  WholeStageCodegen (1)
+                                    Filter [w_warehouse_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.warehouse [w_warehouse_sk,w_warehouse_name]
+                          InputAdapter
+                            BroadcastExchange #3
+                              WholeStageCodegen (2)
+                                Filter [sm_ship_mode_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.ship_mode [sm_ship_mode_sk,sm_type]
+                      InputAdapter
+                        BroadcastExchange #4
+                          WholeStageCodegen (3)
+                            Filter [web_site_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.web_site [web_site_sk,web_name]
+                  InputAdapter
+                    BroadcastExchange #5
+                      WholeStageCodegen (4)
+                        Project [d_date_sk]
+                          Filter [d_month_seq,d_date_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.date_dim [d_date_sk,d_month_seq]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.sf100/explain.txt
new file mode 100644
index 0000000000000..2b8dbaddacad8
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.sf100/explain.txt
@@ -0,0 +1,191 @@
+== Physical Plan ==
+TakeOrderedAndProject (32)
++- * Project (31)
+   +- * Filter (30)
+      +- Window (29)
+         +- * Sort (28)
+            +- Exchange (27)
+               +- * HashAggregate (26)
+                  +- Exchange (25)
+                     +- * HashAggregate (24)
+                        +- * Project (23)
+                           +- * BroadcastHashJoin Inner BuildRight (22)
+                              :- * Project (16)
+                              :  +- * BroadcastHashJoin Inner BuildRight (15)
+                              :     :- * Project (10)
+                              :     :  +- * BroadcastHashJoin Inner BuildLeft (9)
+                              :     :     :- BroadcastExchange (5)
+                              :     :     :  +- * Project (4)
+                              :     :     :     +- * Filter (3)
+                              :     :     :        +- * ColumnarToRow (2)
+                              :     :     :           +- Scan parquet default.item (1)
+                              :     :     +- * Filter (8)
+                              :     :        +- * ColumnarToRow (7)
+                              :     :           +- Scan parquet default.store_sales (6)
+                              :     +- BroadcastExchange (14)
+                              :        +- * Filter (13)
+                              :           +- * ColumnarToRow (12)
+                              :              +- Scan parquet default.store (11)
+                              +- BroadcastExchange (21)
+                                 +- * Project (20)
+                                    +- * Filter (19)
+                                       +- * ColumnarToRow (18)
+                                          +- Scan parquet default.date_dim (17)
+
+
+(1) Scan parquet default.item
+Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [Or(And(And(In(i_category, [Books                                             ,Children                                          ,Electronics                                       ]),In(i_class, [personal                                          ,portable                                          ,refernece                                         ,self-help                                         ])),In(i_brand, [scholaramalgamalg #6                             ,scholaramalgamalg #7                              ,exportiunivamalg #8                               ,scholaramalgamalg #8                              ])),And(And(In(i_category, [Women                                             ,Music                                             ,Men                                               ]),In(i_class, [accessories                                       ,classical                                         ,fragrances                                        ,pants                                             ])),In(i_brand, [amalgimporto #9                                   ,edu packscholar #9                                ,exportiimporto #9                                 ,importoamalg #9                                   ]))), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_class:string,i_category:string,i_manager_id:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5]
+
+(3) Filter [codegen id : 1]
+Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5]
+Condition : ((((i_category#4 IN (Books                                             ,Children                                          ,Electronics                                       ) AND i_class#3 IN (personal                                          ,portable                                          ,refernece                                         ,self-help                                         )) AND i_brand#2 IN (scholaramalgamalg #6                             ,scholaramalgamalg #7                              ,exportiunivamalg #8                               ,scholaramalgamalg #8                              )) OR ((i_category#4 IN (Women                                             ,Music                                             ,Men                                               ) AND i_class#3 IN (accessories                                       ,classical                                         ,fragrances                                        ,pants                                             )) AND i_brand#2 IN (amalgimporto #9                                   ,edu packscholar #9                                ,exportiimporto #9                                 ,importoamalg #9                                   ))) AND isnotnull(i_item_sk#1))
+
+(4) Project [codegen id : 1]
+Output [2]: [i_item_sk#1, i_manager_id#5]
+Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5]
+
+(5) BroadcastExchange
+Input [2]: [i_item_sk#1, i_manager_id#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(6) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#11, ss_store_sk#12, ss_sales_price#13, ss_sold_date_sk#14]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#14), dynamicpruningexpression(ss_sold_date_sk#14 IN dynamicpruning#15)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(7) ColumnarToRow
+Input [4]: [ss_item_sk#11, ss_store_sk#12, ss_sales_price#13, ss_sold_date_sk#14]
+
+(8) Filter
+Input [4]: [ss_item_sk#11, ss_store_sk#12, ss_sales_price#13, ss_sold_date_sk#14]
+Condition : (isnotnull(ss_item_sk#11) AND isnotnull(ss_store_sk#12))
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [ss_item_sk#11]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [4]: [i_manager_id#5, ss_store_sk#12, ss_sales_price#13, ss_sold_date_sk#14]
+Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#11, ss_store_sk#12, ss_sales_price#13, ss_sold_date_sk#14]
+
+(11) Scan parquet default.store
+Output [1]: [s_store_sk#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [1]: [s_store_sk#16]
+
+(13) Filter [codegen id : 2]
+Input [1]: [s_store_sk#16]
+Condition : isnotnull(s_store_sk#16)
+
+(14) BroadcastExchange
+Input [1]: [s_store_sk#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#17]
+
+(15) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#12]
+Right keys [1]: [s_store_sk#16]
+Join condition: None
+
+(16) Project [codegen id : 4]
+Output [3]: [i_manager_id#5, ss_sales_price#13, ss_sold_date_sk#14]
+Input [5]: [i_manager_id#5, ss_store_sk#12, ss_sales_price#13, ss_sold_date_sk#14, s_store_sk#16]
+
+(17) Scan parquet default.date_dim
+Output [3]: [d_date_sk#18, d_month_seq#19, d_moy#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [In(d_month_seq, [1200,1211,1205,1201,1206,1210,1207,1202,1209,1203,1208,1204]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int,d_moy:int>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#18, d_month_seq#19, d_moy#20]
+
+(19) Filter [codegen id : 3]
+Input [3]: [d_date_sk#18, d_month_seq#19, d_moy#20]
+Condition : (d_month_seq#19 INSET (1200,1211,1205,1201,1206,1210,1207,1202,1209,1203,1208,1204) AND isnotnull(d_date_sk#18))
+
+(20) Project [codegen id : 3]
+Output [2]: [d_date_sk#18, d_moy#20]
+Input [3]: [d_date_sk#18, d_month_seq#19, d_moy#20]
+
+(21) BroadcastExchange
+Input [2]: [d_date_sk#18, d_moy#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(22) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#14]
+Right keys [1]: [d_date_sk#18]
+Join condition: None
+
+(23) Project [codegen id : 4]
+Output [3]: [i_manager_id#5, ss_sales_price#13, d_moy#20]
+Input [5]: [i_manager_id#5, ss_sales_price#13, ss_sold_date_sk#14, d_date_sk#18, d_moy#20]
+
+(24) HashAggregate [codegen id : 4]
+Input [3]: [i_manager_id#5, ss_sales_price#13, d_moy#20]
+Keys [2]: [i_manager_id#5, d_moy#20]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#13))]
+Aggregate Attributes [1]: [sum#22]
+Results [3]: [i_manager_id#5, d_moy#20, sum#23]
+
+(25) Exchange
+Input [3]: [i_manager_id#5, d_moy#20, sum#23]
+Arguments: hashpartitioning(i_manager_id#5, d_moy#20, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(26) HashAggregate [codegen id : 5]
+Input [3]: [i_manager_id#5, d_moy#20, sum#23]
+Keys [2]: [i_manager_id#5, d_moy#20]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#13))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#13))#25]
+Results [3]: [i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#13))#25,17,2) AS sum_sales#26, MakeDecimal(sum(UnscaledValue(ss_sales_price#13))#25,17,2) AS _w0#27]
+
+(27) Exchange
+Input [3]: [i_manager_id#5, sum_sales#26, _w0#27]
+Arguments: hashpartitioning(i_manager_id#5, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(28) Sort [codegen id : 6]
+Input [3]: [i_manager_id#5, sum_sales#26, _w0#27]
+Arguments: [i_manager_id#5 ASC NULLS FIRST], false, 0
+
+(29) Window
+Input [3]: [i_manager_id#5, sum_sales#26, _w0#27]
+Arguments: [avg(_w0#27) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#29], [i_manager_id#5]
+
+(30) Filter [codegen id : 7]
+Input [4]: [i_manager_id#5, sum_sales#26, _w0#27, avg_monthly_sales#29]
+Condition : (isnotnull(avg_monthly_sales#29) AND ((avg_monthly_sales#29 > 0.000000) AND (CheckOverflow((promote_precision(abs(CheckOverflow((promote_precision(cast(sum_sales#26 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#29 as decimal(22,6)))), DecimalType(22,6), true), false)) / promote_precision(cast(avg_monthly_sales#29 as decimal(22,6)))), DecimalType(38,16), true) > 0.1000000000000000)))
+
+(31) Project [codegen id : 7]
+Output [3]: [i_manager_id#5, sum_sales#26, avg_monthly_sales#29]
+Input [4]: [i_manager_id#5, sum_sales#26, _w0#27, avg_monthly_sales#29]
+
+(32) TakeOrderedAndProject
+Input [3]: [i_manager_id#5, sum_sales#26, avg_monthly_sales#29]
+Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#29 ASC NULLS FIRST, sum_sales#26 ASC NULLS FIRST], [i_manager_id#5, sum_sales#26, avg_monthly_sales#29]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 6 Hosting Expression = ss_sold_date_sk#14 IN dynamicpruning#15
+ReusedExchange (33)
+
+
+(33) ReusedExchange [Reuses operator id: 21]
+Output [2]: [d_date_sk#18, d_moy#20]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.sf100/simplified.txt
new file mode 100644
index 0000000000000..d8e6d0dcef768
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.sf100/simplified.txt
@@ -0,0 +1,51 @@
+TakeOrderedAndProject [i_manager_id,avg_monthly_sales,sum_sales]
+  WholeStageCodegen (7)
+    Project [i_manager_id,sum_sales,avg_monthly_sales]
+      Filter [avg_monthly_sales,sum_sales]
+        InputAdapter
+          Window [_w0,i_manager_id]
+            WholeStageCodegen (6)
+              Sort [i_manager_id]
+                InputAdapter
+                  Exchange [i_manager_id] #1
+                    WholeStageCodegen (5)
+                      HashAggregate [i_manager_id,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum]
+                        InputAdapter
+                          Exchange [i_manager_id,d_moy] #2
+                            WholeStageCodegen (4)
+                              HashAggregate [i_manager_id,d_moy,ss_sales_price] [sum,sum]
+                                Project [i_manager_id,ss_sales_price,d_moy]
+                                  BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                    Project [i_manager_id,ss_sales_price,ss_sold_date_sk]
+                                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                        Project [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                          BroadcastHashJoin [i_item_sk,ss_item_sk]
+                                            InputAdapter
+                                              BroadcastExchange #3
+                                                WholeStageCodegen (1)
+                                                  Project [i_item_sk,i_manager_id]
+                                                    Filter [i_category,i_class,i_brand,i_item_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.item [i_item_sk,i_brand,i_class,i_category,i_manager_id]
+                                            Filter [ss_item_sk,ss_store_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #1
+                                                      ReusedExchange [d_date_sk,d_moy] #4
+                                        InputAdapter
+                                          BroadcastExchange #5
+                                            WholeStageCodegen (2)
+                                              Filter [s_store_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.store [s_store_sk]
+                                    InputAdapter
+                                      BroadcastExchange #4
+                                        WholeStageCodegen (3)
+                                          Project [d_date_sk,d_moy]
+                                            Filter [d_month_seq,d_date_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.date_dim [d_date_sk,d_month_seq,d_moy]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/explain.txt
new file mode 100644
index 0000000000000..efb27d8cd8b11
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/explain.txt
@@ -0,0 +1,191 @@
+== Physical Plan ==
+TakeOrderedAndProject (32)
++- * Project (31)
+   +- * Filter (30)
+      +- Window (29)
+         +- * Sort (28)
+            +- Exchange (27)
+               +- * HashAggregate (26)
+                  +- Exchange (25)
+                     +- * HashAggregate (24)
+                        +- * Project (23)
+                           +- * BroadcastHashJoin Inner BuildRight (22)
+                              :- * Project (17)
+                              :  +- * BroadcastHashJoin Inner BuildRight (16)
+                              :     :- * Project (10)
+                              :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+                              :     :     :- * Project (4)
+                              :     :     :  +- * Filter (3)
+                              :     :     :     +- * ColumnarToRow (2)
+                              :     :     :        +- Scan parquet default.item (1)
+                              :     :     +- BroadcastExchange (8)
+                              :     :        +- * Filter (7)
+                              :     :           +- * ColumnarToRow (6)
+                              :     :              +- Scan parquet default.store_sales (5)
+                              :     +- BroadcastExchange (15)
+                              :        +- * Project (14)
+                              :           +- * Filter (13)
+                              :              +- * ColumnarToRow (12)
+                              :                 +- Scan parquet default.date_dim (11)
+                              +- BroadcastExchange (21)
+                                 +- * Filter (20)
+                                    +- * ColumnarToRow (19)
+                                       +- Scan parquet default.store (18)
+
+
+(1) Scan parquet default.item
+Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [Or(And(And(In(i_category, [Books                                             ,Children                                          ,Electronics                                       ]),In(i_class, [personal                                          ,portable                                          ,refernece                                         ,self-help                                         ])),In(i_brand, [scholaramalgamalg #6                             ,scholaramalgamalg #7                              ,exportiunivamalg #8                               ,scholaramalgamalg #8                              ])),And(And(In(i_category, [Women                                             ,Music                                             ,Men                                               ]),In(i_class, [accessories                                       ,classical                                         ,fragrances                                        ,pants                                             ])),In(i_brand, [amalgimporto #9                                   ,edu packscholar #9                                ,exportiimporto #9                                 ,importoamalg #9                                   ]))), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_class:string,i_category:string,i_manager_id:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5]
+
+(3) Filter [codegen id : 4]
+Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5]
+Condition : ((((i_category#4 IN (Books                                             ,Children                                          ,Electronics                                       ) AND i_class#3 IN (personal                                          ,portable                                          ,refernece                                         ,self-help                                         )) AND i_brand#2 IN (scholaramalgamalg #6                             ,scholaramalgamalg #7                              ,exportiunivamalg #8                               ,scholaramalgamalg #8                              )) OR ((i_category#4 IN (Women                                             ,Music                                             ,Men                                               ) AND i_class#3 IN (accessories                                       ,classical                                         ,fragrances                                        ,pants                                             )) AND i_brand#2 IN (amalgimporto #9                                   ,edu packscholar #9                                ,exportiimporto #9                                 ,importoamalg #9                                   ))) AND isnotnull(i_item_sk#1))
+
+(4) Project [codegen id : 4]
+Output [2]: [i_item_sk#1, i_manager_id#5]
+Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5]
+
+(5) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#13), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13]
+
+(7) Filter [codegen id : 1]
+Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13]
+Condition : (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11))
+
+(8) BroadcastExchange
+Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#15]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [ss_item_sk#10]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13]
+Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13]
+
+(11) Scan parquet default.date_dim
+Output [3]: [d_date_sk#16, d_month_seq#17, d_moy#18]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [In(d_month_seq, [1200,1211,1205,1201,1206,1210,1207,1202,1209,1203,1208,1204]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int,d_moy:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [d_date_sk#16, d_month_seq#17, d_moy#18]
+
+(13) Filter [codegen id : 2]
+Input [3]: [d_date_sk#16, d_month_seq#17, d_moy#18]
+Condition : (d_month_seq#17 INSET (1200,1211,1205,1201,1206,1210,1207,1202,1209,1203,1208,1204) AND isnotnull(d_date_sk#16))
+
+(14) Project [codegen id : 2]
+Output [2]: [d_date_sk#16, d_moy#18]
+Input [3]: [d_date_sk#16, d_month_seq#17, d_moy#18]
+
+(15) BroadcastExchange
+Input [2]: [d_date_sk#16, d_moy#18]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#19]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#13]
+Right keys [1]: [d_date_sk#16]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#18]
+Input [6]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#16, d_moy#18]
+
+(18) Scan parquet default.store
+Output [1]: [s_store_sk#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [1]: [s_store_sk#20]
+
+(20) Filter [codegen id : 3]
+Input [1]: [s_store_sk#20]
+Condition : isnotnull(s_store_sk#20)
+
+(21) BroadcastExchange
+Input [1]: [s_store_sk#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#21]
+
+(22) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#11]
+Right keys [1]: [s_store_sk#20]
+Join condition: None
+
+(23) Project [codegen id : 4]
+Output [3]: [i_manager_id#5, ss_sales_price#12, d_moy#18]
+Input [5]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#18, s_store_sk#20]
+
+(24) HashAggregate [codegen id : 4]
+Input [3]: [i_manager_id#5, ss_sales_price#12, d_moy#18]
+Keys [2]: [i_manager_id#5, d_moy#18]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))]
+Aggregate Attributes [1]: [sum#22]
+Results [3]: [i_manager_id#5, d_moy#18, sum#23]
+
+(25) Exchange
+Input [3]: [i_manager_id#5, d_moy#18, sum#23]
+Arguments: hashpartitioning(i_manager_id#5, d_moy#18, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(26) HashAggregate [codegen id : 5]
+Input [3]: [i_manager_id#5, d_moy#18, sum#23]
+Keys [2]: [i_manager_id#5, d_moy#18]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#12))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#25]
+Results [3]: [i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#25,17,2) AS sum_sales#26, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#25,17,2) AS _w0#27]
+
+(27) Exchange
+Input [3]: [i_manager_id#5, sum_sales#26, _w0#27]
+Arguments: hashpartitioning(i_manager_id#5, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(28) Sort [codegen id : 6]
+Input [3]: [i_manager_id#5, sum_sales#26, _w0#27]
+Arguments: [i_manager_id#5 ASC NULLS FIRST], false, 0
+
+(29) Window
+Input [3]: [i_manager_id#5, sum_sales#26, _w0#27]
+Arguments: [avg(_w0#27) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#29], [i_manager_id#5]
+
+(30) Filter [codegen id : 7]
+Input [4]: [i_manager_id#5, sum_sales#26, _w0#27, avg_monthly_sales#29]
+Condition : (isnotnull(avg_monthly_sales#29) AND ((avg_monthly_sales#29 > 0.000000) AND (CheckOverflow((promote_precision(abs(CheckOverflow((promote_precision(cast(sum_sales#26 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#29 as decimal(22,6)))), DecimalType(22,6), true), false)) / promote_precision(cast(avg_monthly_sales#29 as decimal(22,6)))), DecimalType(38,16), true) > 0.1000000000000000)))
+
+(31) Project [codegen id : 7]
+Output [3]: [i_manager_id#5, sum_sales#26, avg_monthly_sales#29]
+Input [4]: [i_manager_id#5, sum_sales#26, _w0#27, avg_monthly_sales#29]
+
+(32) TakeOrderedAndProject
+Input [3]: [i_manager_id#5, sum_sales#26, avg_monthly_sales#29]
+Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#29 ASC NULLS FIRST, sum_sales#26 ASC NULLS FIRST], [i_manager_id#5, sum_sales#26, avg_monthly_sales#29]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 5 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14
+ReusedExchange (33)
+
+
+(33) ReusedExchange [Reuses operator id: 15]
+Output [2]: [d_date_sk#16, d_moy#18]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/simplified.txt
new file mode 100644
index 0000000000000..4d85c6d1f57a3
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/simplified.txt
@@ -0,0 +1,51 @@
+TakeOrderedAndProject [i_manager_id,avg_monthly_sales,sum_sales]
+  WholeStageCodegen (7)
+    Project [i_manager_id,sum_sales,avg_monthly_sales]
+      Filter [avg_monthly_sales,sum_sales]
+        InputAdapter
+          Window [_w0,i_manager_id]
+            WholeStageCodegen (6)
+              Sort [i_manager_id]
+                InputAdapter
+                  Exchange [i_manager_id] #1
+                    WholeStageCodegen (5)
+                      HashAggregate [i_manager_id,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum]
+                        InputAdapter
+                          Exchange [i_manager_id,d_moy] #2
+                            WholeStageCodegen (4)
+                              HashAggregate [i_manager_id,d_moy,ss_sales_price] [sum,sum]
+                                Project [i_manager_id,ss_sales_price,d_moy]
+                                  BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                    Project [i_manager_id,ss_store_sk,ss_sales_price,d_moy]
+                                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                        Project [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                          BroadcastHashJoin [i_item_sk,ss_item_sk]
+                                            Project [i_item_sk,i_manager_id]
+                                              Filter [i_category,i_class,i_brand,i_item_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.item [i_item_sk,i_brand,i_class,i_category,i_manager_id]
+                                            InputAdapter
+                                              BroadcastExchange #3
+                                                WholeStageCodegen (1)
+                                                  Filter [ss_item_sk,ss_store_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                                          SubqueryBroadcast [d_date_sk] #1
+                                                            ReusedExchange [d_date_sk,d_moy] #4
+                                        InputAdapter
+                                          BroadcastExchange #4
+                                            WholeStageCodegen (2)
+                                              Project [d_date_sk,d_moy]
+                                                Filter [d_month_seq,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_month_seq,d_moy]
+                                    InputAdapter
+                                      BroadcastExchange #5
+                                        WholeStageCodegen (3)
+                                          Filter [s_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store [s_store_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.sf100/explain.txt
new file mode 100644
index 0000000000000..d8b825209f8bd
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.sf100/explain.txt
@@ -0,0 +1,1162 @@
+== Physical Plan ==
+* Sort (215)
++- Exchange (214)
+   +- * Project (213)
+      +- * SortMergeJoin Inner (212)
+         :- * Sort (131)
+         :  +- Exchange (130)
+         :     +- * HashAggregate (129)
+         :        +- Exchange (128)
+         :           +- * HashAggregate (127)
+         :              +- * Project (126)
+         :                 +- * BroadcastHashJoin Inner BuildRight (125)
+         :                    :- * Project (119)
+         :                    :  +- * BroadcastHashJoin Inner BuildRight (118)
+         :                    :     :- * Project (116)
+         :                    :     :  +- * BroadcastHashJoin Inner BuildRight (115)
+         :                    :     :     :- * Project (110)
+         :                    :     :     :  +- * SortMergeJoin Inner (109)
+         :                    :     :     :     :- * Sort (106)
+         :                    :     :     :     :  +- Exchange (105)
+         :                    :     :     :     :     +- * Project (104)
+         :                    :     :     :     :        +- * SortMergeJoin Inner (103)
+         :                    :     :     :     :           :- * Sort (97)
+         :                    :     :     :     :           :  +- Exchange (96)
+         :                    :     :     :     :           :     +- * Project (95)
+         :                    :     :     :     :           :        +- * BroadcastHashJoin Inner BuildRight (94)
+         :                    :     :     :     :           :           :- * Project (92)
+         :                    :     :     :     :           :           :  +- * BroadcastHashJoin Inner BuildRight (91)
+         :                    :     :     :     :           :           :     :- * Project (86)
+         :                    :     :     :     :           :           :     :  +- * BroadcastHashJoin Inner BuildRight (85)
+         :                    :     :     :     :           :           :     :     :- * Project (80)
+         :                    :     :     :     :           :           :     :     :  +- * SortMergeJoin Inner (79)
+         :                    :     :     :     :           :           :     :     :     :- * Sort (76)
+         :                    :     :     :     :           :           :     :     :     :  +- Exchange (75)
+         :                    :     :     :     :           :           :     :     :     :     +- * Project (74)
+         :                    :     :     :     :           :           :     :     :     :        +- * SortMergeJoin Inner (73)
+         :                    :     :     :     :           :           :     :     :     :           :- * Sort (67)
+         :                    :     :     :     :           :           :     :     :     :           :  +- Exchange (66)
+         :                    :     :     :     :           :           :     :     :     :           :     +- * Project (65)
+         :                    :     :     :     :           :           :     :     :     :           :        +- * BroadcastHashJoin Inner BuildRight (64)
+         :                    :     :     :     :           :           :     :     :     :           :           :- * Project (62)
+         :                    :     :     :     :           :           :     :     :     :           :           :  +- * BroadcastHashJoin Inner BuildRight (61)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :- * Project (56)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :  +- * SortMergeJoin Inner (55)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :- * Sort (49)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :  +- Exchange (48)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :     +- * Project (47)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :        +- * BroadcastHashJoin Inner BuildRight (46)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :- * Project (41)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :  +- * BroadcastHashJoin Inner BuildRight (40)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :- * Project (35)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :  +- * BroadcastHashJoin Inner BuildRight (34)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :- * Project (13)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :  +- * SortMergeJoin Inner (12)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :     :- * Sort (5)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :     :  +- Exchange (4)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :     :     +- * Filter (3)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :     :        +- * ColumnarToRow (2)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :     :           +- Scan parquet default.store_sales (1)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :     +- * Sort (11)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :        +- Exchange (10)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :           +- * Project (9)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :              +- * Filter (8)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :                 +- * ColumnarToRow (7)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :                    +- Scan parquet default.store_returns (6)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     +- BroadcastExchange (33)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :        +- * Project (32)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :           +- * Filter (31)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :              +- * HashAggregate (30)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                 +- Exchange (29)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                    +- * HashAggregate (28)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                       +- * Project (27)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                          +- * SortMergeJoin Inner (26)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                             :- * Sort (19)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                             :  +- Exchange (18)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                             :     +- * Project (17)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                             :        +- * Filter (16)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                             :           +- * ColumnarToRow (15)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                             :              +- Scan parquet default.catalog_sales (14)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                             +- * Sort (25)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                                +- Exchange (24)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                                   +- * Project (23)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                                      +- * Filter (22)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                                         +- * ColumnarToRow (21)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                                            +- Scan parquet default.catalog_returns (20)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     +- BroadcastExchange (39)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :        +- * Filter (38)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :           +- * ColumnarToRow (37)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :              +- Scan parquet default.date_dim (36)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           +- BroadcastExchange (45)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :              +- * Filter (44)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :                 +- * ColumnarToRow (43)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :                    +- Scan parquet default.store (42)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     +- * Sort (54)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :        +- Exchange (53)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :           +- * Filter (52)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :              +- * ColumnarToRow (51)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :                 +- Scan parquet default.customer (50)
+         :                    :     :     :     :           :           :     :     :     :           :           :     +- BroadcastExchange (60)
+         :                    :     :     :     :           :           :     :     :     :           :           :        +- * Filter (59)
+         :                    :     :     :     :           :           :     :     :     :           :           :           +- * ColumnarToRow (58)
+         :                    :     :     :     :           :           :     :     :     :           :           :              +- Scan parquet default.date_dim (57)
+         :                    :     :     :     :           :           :     :     :     :           :           +- ReusedExchange (63)
+         :                    :     :     :     :           :           :     :     :     :           +- * Sort (72)
+         :                    :     :     :     :           :           :     :     :     :              +- Exchange (71)
+         :                    :     :     :     :           :           :     :     :     :                 +- * Filter (70)
+         :                    :     :     :     :           :           :     :     :     :                    +- * ColumnarToRow (69)
+         :                    :     :     :     :           :           :     :     :     :                       +- Scan parquet default.customer_demographics (68)
+         :                    :     :     :     :           :           :     :     :     +- * Sort (78)
+         :                    :     :     :     :           :           :     :     :        +- ReusedExchange (77)
+         :                    :     :     :     :           :           :     :     +- BroadcastExchange (84)
+         :                    :     :     :     :           :           :     :        +- * Filter (83)
+         :                    :     :     :     :           :           :     :           +- * ColumnarToRow (82)
+         :                    :     :     :     :           :           :     :              +- Scan parquet default.promotion (81)
+         :                    :     :     :     :           :           :     +- BroadcastExchange (90)
+         :                    :     :     :     :           :           :        +- * Filter (89)
+         :                    :     :     :     :           :           :           +- * ColumnarToRow (88)
+         :                    :     :     :     :           :           :              +- Scan parquet default.household_demographics (87)
+         :                    :     :     :     :           :           +- ReusedExchange (93)
+         :                    :     :     :     :           +- * Sort (102)
+         :                    :     :     :     :              +- Exchange (101)
+         :                    :     :     :     :                 +- * Filter (100)
+         :                    :     :     :     :                    +- * ColumnarToRow (99)
+         :                    :     :     :     :                       +- Scan parquet default.customer_address (98)
+         :                    :     :     :     +- * Sort (108)
+         :                    :     :     :        +- ReusedExchange (107)
+         :                    :     :     +- BroadcastExchange (114)
+         :                    :     :        +- * Filter (113)
+         :                    :     :           +- * ColumnarToRow (112)
+         :                    :     :              +- Scan parquet default.income_band (111)
+         :                    :     +- ReusedExchange (117)
+         :                    +- BroadcastExchange (124)
+         :                       +- * Project (123)
+         :                          +- * Filter (122)
+         :                             +- * ColumnarToRow (121)
+         :                                +- Scan parquet default.item (120)
+         +- * Sort (211)
+            +- Exchange (210)
+               +- * HashAggregate (209)
+                  +- Exchange (208)
+                     +- * HashAggregate (207)
+                        +- * Project (206)
+                           +- * BroadcastHashJoin Inner BuildRight (205)
+                              :- * Project (203)
+                              :  +- * BroadcastHashJoin Inner BuildRight (202)
+                              :     :- * Project (200)
+                              :     :  +- * BroadcastHashJoin Inner BuildRight (199)
+                              :     :     :- * Project (197)
+                              :     :     :  +- * SortMergeJoin Inner (196)
+                              :     :     :     :- * Sort (193)
+                              :     :     :     :  +- Exchange (192)
+                              :     :     :     :     +- * Project (191)
+                              :     :     :     :        +- * SortMergeJoin Inner (190)
+                              :     :     :     :           :- * Sort (187)
+                              :     :     :     :           :  +- Exchange (186)
+                              :     :     :     :           :     +- * Project (185)
+                              :     :     :     :           :        +- * BroadcastHashJoin Inner BuildRight (184)
+                              :     :     :     :           :           :- * Project (182)
+                              :     :     :     :           :           :  +- * BroadcastHashJoin Inner BuildRight (181)
+                              :     :     :     :           :           :     :- * Project (179)
+                              :     :     :     :           :           :     :  +- * BroadcastHashJoin Inner BuildRight (178)
+                              :     :     :     :           :           :     :     :- * Project (176)
+                              :     :     :     :           :           :     :     :  +- * SortMergeJoin Inner (175)
+                              :     :     :     :           :           :     :     :     :- * Sort (172)
+                              :     :     :     :           :           :     :     :     :  +- Exchange (171)
+                              :     :     :     :           :           :     :     :     :     +- * Project (170)
+                              :     :     :     :           :           :     :     :     :        +- * SortMergeJoin Inner (169)
+                              :     :     :     :           :           :     :     :     :           :- * Sort (166)
+                              :     :     :     :           :           :     :     :     :           :  +- Exchange (165)
+                              :     :     :     :           :           :     :     :     :           :     +- * Project (164)
+                              :     :     :     :           :           :     :     :     :           :        +- * BroadcastHashJoin Inner BuildRight (163)
+                              :     :     :     :           :           :     :     :     :           :           :- * Project (161)
+                              :     :     :     :           :           :     :     :     :           :           :  +- * BroadcastHashJoin Inner BuildRight (160)
+                              :     :     :     :           :           :     :     :     :           :           :     :- * Project (158)
+                              :     :     :     :           :           :     :     :     :           :           :     :  +- * SortMergeJoin Inner (157)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :- * Sort (154)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :  +- Exchange (153)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :     +- * Project (152)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :        +- * BroadcastHashJoin Inner BuildRight (151)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :- * Project (149)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :  +- * BroadcastHashJoin Inner BuildRight (148)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     :- * Project (143)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     :  +- * BroadcastHashJoin Inner BuildRight (142)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :- * Project (140)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :  +- * SortMergeJoin Inner (139)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :     :- * Sort (136)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :     :  +- Exchange (135)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :     :     +- * Filter (134)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :     :        +- * ColumnarToRow (133)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :     :           +- Scan parquet default.store_sales (132)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :     +- * Sort (138)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :        +- ReusedExchange (137)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     +- ReusedExchange (141)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     +- BroadcastExchange (147)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :        +- * Filter (146)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :           +- * ColumnarToRow (145)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :              +- Scan parquet default.date_dim (144)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           +- ReusedExchange (150)
+                              :     :     :     :           :           :     :     :     :           :           :     :     +- * Sort (156)
+                              :     :     :     :           :           :     :     :     :           :           :     :        +- ReusedExchange (155)
+                              :     :     :     :           :           :     :     :     :           :           :     +- ReusedExchange (159)
+                              :     :     :     :           :           :     :     :     :           :           +- ReusedExchange (162)
+                              :     :     :     :           :           :     :     :     :           +- * Sort (168)
+                              :     :     :     :           :           :     :     :     :              +- ReusedExchange (167)
+                              :     :     :     :           :           :     :     :     +- * Sort (174)
+                              :     :     :     :           :           :     :     :        +- ReusedExchange (173)
+                              :     :     :     :           :           :     :     +- ReusedExchange (177)
+                              :     :     :     :           :           :     +- ReusedExchange (180)
+                              :     :     :     :           :           +- ReusedExchange (183)
+                              :     :     :     :           +- * Sort (189)
+                              :     :     :     :              +- ReusedExchange (188)
+                              :     :     :     +- * Sort (195)
+                              :     :     :        +- ReusedExchange (194)
+                              :     :     +- ReusedExchange (198)
+                              :     +- ReusedExchange (201)
+                              +- ReusedExchange (204)
+
+
+(1) Scan parquet default.store_sales
+Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_sold_date_sk#12 IN dynamicpruning#13)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_cdemo_sk:int,ss_hdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_promo_sk:int,ss_ticket_number:int,ss_wholesale_cost:decimal(7,2),ss_list_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+
+(3) Filter [codegen id : 1]
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AND isnotnull(ss_store_sk#6)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_cdemo_sk#3)) AND isnotnull(ss_promo_sk#7)) AND isnotnull(ss_hdemo_sk#4)) AND isnotnull(ss_addr_sk#5))
+
+(4) Exchange
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#8, 5), ENSURE_REQUIREMENTS, [id=#14]
+
+(5) Sort [codegen id : 2]
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Arguments: [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#8 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.store_returns
+Output [3]: [sr_item_sk#15, sr_ticket_number#16, sr_returned_date_sk#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int>
+
+(7) ColumnarToRow [codegen id : 3]
+Input [3]: [sr_item_sk#15, sr_ticket_number#16, sr_returned_date_sk#17]
+
+(8) Filter [codegen id : 3]
+Input [3]: [sr_item_sk#15, sr_ticket_number#16, sr_returned_date_sk#17]
+Condition : (isnotnull(sr_item_sk#15) AND isnotnull(sr_ticket_number#16))
+
+(9) Project [codegen id : 3]
+Output [2]: [sr_item_sk#15, sr_ticket_number#16]
+Input [3]: [sr_item_sk#15, sr_ticket_number#16, sr_returned_date_sk#17]
+
+(10) Exchange
+Input [2]: [sr_item_sk#15, sr_ticket_number#16]
+Arguments: hashpartitioning(sr_item_sk#15, sr_ticket_number#16, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(11) Sort [codegen id : 4]
+Input [2]: [sr_item_sk#15, sr_ticket_number#16]
+Arguments: [sr_item_sk#15 ASC NULLS FIRST, sr_ticket_number#16 ASC NULLS FIRST], false, 0
+
+(12) SortMergeJoin [codegen id : 13]
+Left keys [2]: [ss_item_sk#1, ss_ticket_number#8]
+Right keys [2]: [sr_item_sk#15, sr_ticket_number#16]
+Join condition: None
+
+(13) Project [codegen id : 13]
+Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#15, sr_ticket_number#16]
+
+(14) Scan parquet default.catalog_sales
+Output [4]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21, cs_sold_date_sk#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_sales]
+PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)]
+ReadSchema: struct<cs_item_sk:int,cs_order_number:int,cs_ext_list_price:decimal(7,2)>
+
+(15) ColumnarToRow [codegen id : 5]
+Input [4]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21, cs_sold_date_sk#22]
+
+(16) Filter [codegen id : 5]
+Input [4]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21, cs_sold_date_sk#22]
+Condition : (isnotnull(cs_item_sk#19) AND isnotnull(cs_order_number#20))
+
+(17) Project [codegen id : 5]
+Output [3]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21]
+Input [4]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21, cs_sold_date_sk#22]
+
+(18) Exchange
+Input [3]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21]
+Arguments: hashpartitioning(cs_item_sk#19, cs_order_number#20, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(19) Sort [codegen id : 6]
+Input [3]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21]
+Arguments: [cs_item_sk#19 ASC NULLS FIRST, cs_order_number#20 ASC NULLS FIRST], false, 0
+
+(20) Scan parquet default.catalog_returns
+Output [6]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28, cr_returned_date_sk#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int,cr_refunded_cash:decimal(7,2),cr_reversed_charge:decimal(7,2),cr_store_credit:decimal(7,2)>
+
+(21) ColumnarToRow [codegen id : 7]
+Input [6]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28, cr_returned_date_sk#29]
+
+(22) Filter [codegen id : 7]
+Input [6]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28, cr_returned_date_sk#29]
+Condition : (isnotnull(cr_item_sk#24) AND isnotnull(cr_order_number#25))
+
+(23) Project [codegen id : 7]
+Output [5]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28]
+Input [6]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28, cr_returned_date_sk#29]
+
+(24) Exchange
+Input [5]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28]
+Arguments: hashpartitioning(cr_item_sk#24, cr_order_number#25, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(25) Sort [codegen id : 8]
+Input [5]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28]
+Arguments: [cr_item_sk#24 ASC NULLS FIRST, cr_order_number#25 ASC NULLS FIRST], false, 0
+
+(26) SortMergeJoin [codegen id : 9]
+Left keys [2]: [cs_item_sk#19, cs_order_number#20]
+Right keys [2]: [cr_item_sk#24, cr_order_number#25]
+Join condition: None
+
+(27) Project [codegen id : 9]
+Output [5]: [cs_item_sk#19, cs_ext_list_price#21, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28]
+Input [8]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21, cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28]
+
+(28) HashAggregate [codegen id : 9]
+Input [5]: [cs_item_sk#19, cs_ext_list_price#21, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28]
+Keys [1]: [cs_item_sk#19]
+Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#21)), partial_sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))]
+Aggregate Attributes [3]: [sum#31, sum#32, isEmpty#33]
+Results [4]: [cs_item_sk#19, sum#34, sum#35, isEmpty#36]
+
+(29) Exchange
+Input [4]: [cs_item_sk#19, sum#34, sum#35, isEmpty#36]
+Arguments: hashpartitioning(cs_item_sk#19, 5), ENSURE_REQUIREMENTS, [id=#37]
+
+(30) HashAggregate [codegen id : 10]
+Input [4]: [cs_item_sk#19, sum#34, sum#35, isEmpty#36]
+Keys [1]: [cs_item_sk#19]
+Functions [2]: [sum(UnscaledValue(cs_ext_list_price#21)), sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))]
+Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#21))#38, sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))#39]
+Results [3]: [cs_item_sk#19, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#21))#38,17,2) AS sum(cs_ext_list_price#21)#40, sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))#39 AS sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))#41]
+
+(31) Filter [codegen id : 10]
+Input [3]: [cs_item_sk#19, sum(cs_ext_list_price#21)#40, sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))#41]
+Condition : (isnotnull(sum(cs_ext_list_price#21)#40) AND (cast(sum(cs_ext_list_price#21)#40 as decimal(21,2)) > CheckOverflow((2.00 * promote_precision(sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))#41)), DecimalType(21,2), true)))
+
+(32) Project [codegen id : 10]
+Output [1]: [cs_item_sk#19]
+Input [3]: [cs_item_sk#19, sum(cs_ext_list_price#21)#40, sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))#41]
+
+(33) BroadcastExchange
+Input [1]: [cs_item_sk#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#42]
+
+(34) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [cs_item_sk#19]
+Join condition: None
+
+(35) Project [codegen id : 13]
+Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#19]
+
+(36) Scan parquet default.date_dim
+Output [2]: [d_date_sk#43, d_year#44]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(37) ColumnarToRow [codegen id : 11]
+Input [2]: [d_date_sk#43, d_year#44]
+
+(38) Filter [codegen id : 11]
+Input [2]: [d_date_sk#43, d_year#44]
+Condition : ((isnotnull(d_year#44) AND (d_year#44 = 1999)) AND isnotnull(d_date_sk#43))
+
+(39) BroadcastExchange
+Input [2]: [d_date_sk#43, d_year#44]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#45]
+
+(40) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ss_sold_date_sk#12]
+Right keys [1]: [d_date_sk#43]
+Join condition: None
+
+(41) Project [codegen id : 13]
+Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44]
+Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#43, d_year#44]
+
+(42) Scan parquet default.store
+Output [3]: [s_store_sk#46, s_store_name#47, s_zip#48]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_zip)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_zip:string>
+
+(43) ColumnarToRow [codegen id : 12]
+Input [3]: [s_store_sk#46, s_store_name#47, s_zip#48]
+
+(44) Filter [codegen id : 12]
+Input [3]: [s_store_sk#46, s_store_name#47, s_zip#48]
+Condition : ((isnotnull(s_store_sk#46) AND isnotnull(s_store_name#47)) AND isnotnull(s_zip#48))
+
+(45) BroadcastExchange
+Input [3]: [s_store_sk#46, s_store_name#47, s_zip#48]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#49]
+
+(46) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ss_store_sk#6]
+Right keys [1]: [s_store_sk#46]
+Join condition: None
+
+(47) Project [codegen id : 13]
+Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48]
+Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_sk#46, s_store_name#47, s_zip#48]
+
+(48) Exchange
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48]
+Arguments: hashpartitioning(ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [id=#50]
+
+(49) Sort [codegen id : 14]
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48]
+Arguments: [ss_customer_sk#2 ASC NULLS FIRST], false, 0
+
+(50) Scan parquet default.customer
+Output [6]: [c_customer_sk#51, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, c_first_shipto_date_sk#55, c_first_sales_date_sk#56]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_hdemo_sk:int,c_current_addr_sk:int,c_first_shipto_date_sk:int,c_first_sales_date_sk:int>
+
+(51) ColumnarToRow [codegen id : 15]
+Input [6]: [c_customer_sk#51, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, c_first_shipto_date_sk#55, c_first_sales_date_sk#56]
+
+(52) Filter [codegen id : 15]
+Input [6]: [c_customer_sk#51, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, c_first_shipto_date_sk#55, c_first_sales_date_sk#56]
+Condition : (((((isnotnull(c_customer_sk#51) AND isnotnull(c_first_sales_date_sk#56)) AND isnotnull(c_first_shipto_date_sk#55)) AND isnotnull(c_current_cdemo_sk#52)) AND isnotnull(c_current_hdemo_sk#53)) AND isnotnull(c_current_addr_sk#54))
+
+(53) Exchange
+Input [6]: [c_customer_sk#51, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, c_first_shipto_date_sk#55, c_first_sales_date_sk#56]
+Arguments: hashpartitioning(c_customer_sk#51, 5), ENSURE_REQUIREMENTS, [id=#57]
+
+(54) Sort [codegen id : 16]
+Input [6]: [c_customer_sk#51, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, c_first_shipto_date_sk#55, c_first_sales_date_sk#56]
+Arguments: [c_customer_sk#51 ASC NULLS FIRST], false, 0
+
+(55) SortMergeJoin [codegen id : 19]
+Left keys [1]: [ss_customer_sk#2]
+Right keys [1]: [c_customer_sk#51]
+Join condition: None
+
+(56) Project [codegen id : 19]
+Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, c_first_shipto_date_sk#55, c_first_sales_date_sk#56]
+Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_customer_sk#51, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, c_first_shipto_date_sk#55, c_first_sales_date_sk#56]
+
+(57) Scan parquet default.date_dim
+Output [2]: [d_date_sk#58, d_year#59]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(58) ColumnarToRow [codegen id : 17]
+Input [2]: [d_date_sk#58, d_year#59]
+
+(59) Filter [codegen id : 17]
+Input [2]: [d_date_sk#58, d_year#59]
+Condition : isnotnull(d_date_sk#58)
+
+(60) BroadcastExchange
+Input [2]: [d_date_sk#58, d_year#59]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#60]
+
+(61) BroadcastHashJoin [codegen id : 19]
+Left keys [1]: [c_first_sales_date_sk#56]
+Right keys [1]: [d_date_sk#58]
+Join condition: None
+
+(62) Project [codegen id : 19]
+Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, c_first_shipto_date_sk#55, d_year#59]
+Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, c_first_shipto_date_sk#55, c_first_sales_date_sk#56, d_date_sk#58, d_year#59]
+
+(63) ReusedExchange [Reuses operator id: 60]
+Output [2]: [d_date_sk#61, d_year#62]
+
+(64) BroadcastHashJoin [codegen id : 19]
+Left keys [1]: [c_first_shipto_date_sk#55]
+Right keys [1]: [d_date_sk#61]
+Join condition: None
+
+(65) Project [codegen id : 19]
+Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62]
+Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, c_first_shipto_date_sk#55, d_year#59, d_date_sk#61, d_year#62]
+
+(66) Exchange
+Input [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62]
+Arguments: hashpartitioning(ss_cdemo_sk#3, 5), ENSURE_REQUIREMENTS, [id=#63]
+
+(67) Sort [codegen id : 20]
+Input [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62]
+Arguments: [ss_cdemo_sk#3 ASC NULLS FIRST], false, 0
+
+(68) Scan parquet default.customer_demographics
+Output [2]: [cd_demo_sk#64, cd_marital_status#65]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status)]
+ReadSchema: struct<cd_demo_sk:int,cd_marital_status:string>
+
+(69) ColumnarToRow [codegen id : 21]
+Input [2]: [cd_demo_sk#64, cd_marital_status#65]
+
+(70) Filter [codegen id : 21]
+Input [2]: [cd_demo_sk#64, cd_marital_status#65]
+Condition : (isnotnull(cd_demo_sk#64) AND isnotnull(cd_marital_status#65))
+
+(71) Exchange
+Input [2]: [cd_demo_sk#64, cd_marital_status#65]
+Arguments: hashpartitioning(cd_demo_sk#64, 5), ENSURE_REQUIREMENTS, [id=#66]
+
+(72) Sort [codegen id : 22]
+Input [2]: [cd_demo_sk#64, cd_marital_status#65]
+Arguments: [cd_demo_sk#64 ASC NULLS FIRST], false, 0
+
+(73) SortMergeJoin [codegen id : 23]
+Left keys [1]: [ss_cdemo_sk#3]
+Right keys [1]: [cd_demo_sk#64]
+Join condition: None
+
+(74) Project [codegen id : 23]
+Output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62, cd_marital_status#65]
+Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62, cd_demo_sk#64, cd_marital_status#65]
+
+(75) Exchange
+Input [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62, cd_marital_status#65]
+Arguments: hashpartitioning(c_current_cdemo_sk#52, 5), ENSURE_REQUIREMENTS, [id=#67]
+
+(76) Sort [codegen id : 24]
+Input [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62, cd_marital_status#65]
+Arguments: [c_current_cdemo_sk#52 ASC NULLS FIRST], false, 0
+
+(77) ReusedExchange [Reuses operator id: 71]
+Output [2]: [cd_demo_sk#68, cd_marital_status#69]
+
+(78) Sort [codegen id : 26]
+Input [2]: [cd_demo_sk#68, cd_marital_status#69]
+Arguments: [cd_demo_sk#68 ASC NULLS FIRST], false, 0
+
+(79) SortMergeJoin [codegen id : 30]
+Left keys [1]: [c_current_cdemo_sk#52]
+Right keys [1]: [cd_demo_sk#68]
+Join condition: NOT (cd_marital_status#65 = cd_marital_status#69)
+
+(80) Project [codegen id : 30]
+Output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62]
+Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62, cd_marital_status#65, cd_demo_sk#68, cd_marital_status#69]
+
+(81) Scan parquet default.promotion
+Output [1]: [p_promo_sk#70]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/promotion]
+PushedFilters: [IsNotNull(p_promo_sk)]
+ReadSchema: struct<p_promo_sk:int>
+
+(82) ColumnarToRow [codegen id : 27]
+Input [1]: [p_promo_sk#70]
+
+(83) Filter [codegen id : 27]
+Input [1]: [p_promo_sk#70]
+Condition : isnotnull(p_promo_sk#70)
+
+(84) BroadcastExchange
+Input [1]: [p_promo_sk#70]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#71]
+
+(85) BroadcastHashJoin [codegen id : 30]
+Left keys [1]: [ss_promo_sk#7]
+Right keys [1]: [p_promo_sk#70]
+Join condition: None
+
+(86) Project [codegen id : 30]
+Output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62]
+Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62, p_promo_sk#70]
+
+(87) Scan parquet default.household_demographics
+Output [2]: [hd_demo_sk#72, hd_income_band_sk#73]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_income_band_sk:int>
+
+(88) ColumnarToRow [codegen id : 28]
+Input [2]: [hd_demo_sk#72, hd_income_band_sk#73]
+
+(89) Filter [codegen id : 28]
+Input [2]: [hd_demo_sk#72, hd_income_band_sk#73]
+Condition : (isnotnull(hd_demo_sk#72) AND isnotnull(hd_income_band_sk#73))
+
+(90) BroadcastExchange
+Input [2]: [hd_demo_sk#72, hd_income_band_sk#73]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#74]
+
+(91) BroadcastHashJoin [codegen id : 30]
+Left keys [1]: [ss_hdemo_sk#4]
+Right keys [1]: [hd_demo_sk#72]
+Join condition: None
+
+(92) Project [codegen id : 30]
+Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62, hd_income_band_sk#73]
+Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62, hd_demo_sk#72, hd_income_band_sk#73]
+
+(93) ReusedExchange [Reuses operator id: 90]
+Output [2]: [hd_demo_sk#75, hd_income_band_sk#76]
+
+(94) BroadcastHashJoin [codegen id : 30]
+Left keys [1]: [c_current_hdemo_sk#53]
+Right keys [1]: [hd_demo_sk#75]
+Join condition: None
+
+(95) Project [codegen id : 30]
+Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_addr_sk#54, d_year#59, d_year#62, hd_income_band_sk#73, hd_income_band_sk#76]
+Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62, hd_income_band_sk#73, hd_demo_sk#75, hd_income_band_sk#76]
+
+(96) Exchange
+Input [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_addr_sk#54, d_year#59, d_year#62, hd_income_band_sk#73, hd_income_band_sk#76]
+Arguments: hashpartitioning(ss_addr_sk#5, 5), ENSURE_REQUIREMENTS, [id=#77]
+
+(97) Sort [codegen id : 31]
+Input [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_addr_sk#54, d_year#59, d_year#62, hd_income_band_sk#73, hd_income_band_sk#76]
+Arguments: [ss_addr_sk#5 ASC NULLS FIRST], false, 0
+
+(98) Scan parquet default.customer_address
+Output [5]: [ca_address_sk#78, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_street_number:string,ca_street_name:string,ca_city:string,ca_zip:string>
+
+(99) ColumnarToRow [codegen id : 32]
+Input [5]: [ca_address_sk#78, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82]
+
+(100) Filter [codegen id : 32]
+Input [5]: [ca_address_sk#78, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82]
+Condition : isnotnull(ca_address_sk#78)
+
+(101) Exchange
+Input [5]: [ca_address_sk#78, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82]
+Arguments: hashpartitioning(ca_address_sk#78, 5), ENSURE_REQUIREMENTS, [id=#83]
+
+(102) Sort [codegen id : 33]
+Input [5]: [ca_address_sk#78, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82]
+Arguments: [ca_address_sk#78 ASC NULLS FIRST], false, 0
+
+(103) SortMergeJoin [codegen id : 34]
+Left keys [1]: [ss_addr_sk#5]
+Right keys [1]: [ca_address_sk#78]
+Join condition: None
+
+(104) Project [codegen id : 34]
+Output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_addr_sk#54, d_year#59, d_year#62, hd_income_band_sk#73, hd_income_band_sk#76, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82]
+Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_addr_sk#54, d_year#59, d_year#62, hd_income_band_sk#73, hd_income_band_sk#76, ca_address_sk#78, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82]
+
+(105) Exchange
+Input [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_addr_sk#54, d_year#59, d_year#62, hd_income_band_sk#73, hd_income_band_sk#76, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82]
+Arguments: hashpartitioning(c_current_addr_sk#54, 5), ENSURE_REQUIREMENTS, [id=#84]
+
+(106) Sort [codegen id : 35]
+Input [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_addr_sk#54, d_year#59, d_year#62, hd_income_band_sk#73, hd_income_band_sk#76, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82]
+Arguments: [c_current_addr_sk#54 ASC NULLS FIRST], false, 0
+
+(107) ReusedExchange [Reuses operator id: 101]
+Output [5]: [ca_address_sk#85, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89]
+
+(108) Sort [codegen id : 37]
+Input [5]: [ca_address_sk#85, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89]
+Arguments: [ca_address_sk#85 ASC NULLS FIRST], false, 0
+
+(109) SortMergeJoin [codegen id : 41]
+Left keys [1]: [c_current_addr_sk#54]
+Right keys [1]: [ca_address_sk#85]
+Join condition: None
+
+(110) Project [codegen id : 41]
+Output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, d_year#59, d_year#62, hd_income_band_sk#73, hd_income_band_sk#76, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89]
+Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_addr_sk#54, d_year#59, d_year#62, hd_income_band_sk#73, hd_income_band_sk#76, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_address_sk#85, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89]
+
+(111) Scan parquet default.income_band
+Output [1]: [ib_income_band_sk#90]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/income_band]
+PushedFilters: [IsNotNull(ib_income_band_sk)]
+ReadSchema: struct<ib_income_band_sk:int>
+
+(112) ColumnarToRow [codegen id : 38]
+Input [1]: [ib_income_band_sk#90]
+
+(113) Filter [codegen id : 38]
+Input [1]: [ib_income_band_sk#90]
+Condition : isnotnull(ib_income_band_sk#90)
+
+(114) BroadcastExchange
+Input [1]: [ib_income_band_sk#90]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#91]
+
+(115) BroadcastHashJoin [codegen id : 41]
+Left keys [1]: [hd_income_band_sk#73]
+Right keys [1]: [ib_income_band_sk#90]
+Join condition: None
+
+(116) Project [codegen id : 41]
+Output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, d_year#59, d_year#62, hd_income_band_sk#76, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89]
+Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, d_year#59, d_year#62, hd_income_band_sk#73, hd_income_band_sk#76, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89, ib_income_band_sk#90]
+
+(117) ReusedExchange [Reuses operator id: 114]
+Output [1]: [ib_income_band_sk#92]
+
+(118) BroadcastHashJoin [codegen id : 41]
+Left keys [1]: [hd_income_band_sk#76]
+Right keys [1]: [ib_income_band_sk#92]
+Join condition: None
+
+(119) Project [codegen id : 41]
+Output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, d_year#59, d_year#62, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89]
+Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, d_year#59, d_year#62, hd_income_band_sk#76, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89, ib_income_band_sk#92]
+
+(120) Scan parquet default.item
+Output [4]: [i_item_sk#93, i_current_price#94, i_color#95, i_product_name#96]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_current_price), In(i_color, [purple              ,burlywood           ,indian              ,spring              ,floral              ,medium              ]), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_current_price:decimal(7,2),i_color:string,i_product_name:string>
+
+(121) ColumnarToRow [codegen id : 40]
+Input [4]: [i_item_sk#93, i_current_price#94, i_color#95, i_product_name#96]
+
+(122) Filter [codegen id : 40]
+Input [4]: [i_item_sk#93, i_current_price#94, i_color#95, i_product_name#96]
+Condition : ((((((isnotnull(i_current_price#94) AND i_color#95 IN (purple              ,burlywood           ,indian              ,spring              ,floral              ,medium              )) AND (i_current_price#94 >= 64.00)) AND (i_current_price#94 <= 74.00)) AND (i_current_price#94 >= 65.00)) AND (i_current_price#94 <= 79.00)) AND isnotnull(i_item_sk#93))
+
+(123) Project [codegen id : 40]
+Output [2]: [i_item_sk#93, i_product_name#96]
+Input [4]: [i_item_sk#93, i_current_price#94, i_color#95, i_product_name#96]
+
+(124) BroadcastExchange
+Input [2]: [i_item_sk#93, i_product_name#96]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#97]
+
+(125) BroadcastHashJoin [codegen id : 41]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#93]
+Join condition: None
+
+(126) Project [codegen id : 41]
+Output [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, d_year#59, d_year#62, s_store_name#47, s_zip#48, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89, i_item_sk#93, i_product_name#96]
+Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, d_year#59, d_year#62, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89, i_item_sk#93, i_product_name#96]
+
+(127) HashAggregate [codegen id : 41]
+Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, d_year#59, d_year#62, s_store_name#47, s_zip#48, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89, i_item_sk#93, i_product_name#96]
+Keys [15]: [i_product_name#96, i_item_sk#93, s_store_name#47, s_zip#48, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89, d_year#44, d_year#59, d_year#62]
+Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))]
+Aggregate Attributes [4]: [count#98, sum#99, sum#100, sum#101]
+Results [19]: [i_product_name#96, i_item_sk#93, s_store_name#47, s_zip#48, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89, d_year#44, d_year#59, d_year#62, count#102, sum#103, sum#104, sum#105]
+
+(128) Exchange
+Input [19]: [i_product_name#96, i_item_sk#93, s_store_name#47, s_zip#48, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89, d_year#44, d_year#59, d_year#62, count#102, sum#103, sum#104, sum#105]
+Arguments: hashpartitioning(i_product_name#96, i_item_sk#93, s_store_name#47, s_zip#48, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89, d_year#44, d_year#59, d_year#62, 5), ENSURE_REQUIREMENTS, [id=#106]
+
+(129) HashAggregate [codegen id : 42]
+Input [19]: [i_product_name#96, i_item_sk#93, s_store_name#47, s_zip#48, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89, d_year#44, d_year#59, d_year#62, count#102, sum#103, sum#104, sum#105]
+Keys [15]: [i_product_name#96, i_item_sk#93, s_store_name#47, s_zip#48, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89, d_year#44, d_year#59, d_year#62]
+Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))]
+Aggregate Attributes [4]: [count(1)#107, sum(UnscaledValue(ss_wholesale_cost#9))#108, sum(UnscaledValue(ss_list_price#10))#109, sum(UnscaledValue(ss_coupon_amt#11))#110]
+Results [17]: [i_product_name#96 AS product_name#111, i_item_sk#93 AS item_sk#112, s_store_name#47 AS store_name#113, s_zip#48 AS store_zip#114, ca_street_number#79 AS b_street_number#115, ca_street_name#80 AS b_streen_name#116, ca_city#81 AS b_city#117, ca_zip#82 AS b_zip#118, ca_street_number#86 AS c_street_number#119, ca_street_name#87 AS c_street_name#120, ca_city#88 AS c_city#121, ca_zip#89 AS c_zip#122, d_year#44 AS syear#123, count(1)#107 AS cnt#124, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#9))#108,17,2) AS s1#125, MakeDecimal(sum(UnscaledValue(ss_list_price#10))#109,17,2) AS s2#126, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#11))#110,17,2) AS s3#127]
+
+(130) Exchange
+Input [17]: [product_name#111, item_sk#112, store_name#113, store_zip#114, b_street_number#115, b_streen_name#116, b_city#117, b_zip#118, c_street_number#119, c_street_name#120, c_city#121, c_zip#122, syear#123, cnt#124, s1#125, s2#126, s3#127]
+Arguments: hashpartitioning(item_sk#112, store_name#113, store_zip#114, 5), ENSURE_REQUIREMENTS, [id=#128]
+
+(131) Sort [codegen id : 43]
+Input [17]: [product_name#111, item_sk#112, store_name#113, store_zip#114, b_street_number#115, b_streen_name#116, b_city#117, b_zip#118, c_street_number#119, c_street_name#120, c_city#121, c_zip#122, syear#123, cnt#124, s1#125, s2#126, s3#127]
+Arguments: [item_sk#112 ASC NULLS FIRST, store_name#113 ASC NULLS FIRST, store_zip#114 ASC NULLS FIRST], false, 0
+
+(132) Scan parquet default.store_sales
+Output [12]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_store_sk#134, ss_promo_sk#135, ss_ticket_number#136, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, ss_sold_date_sk#140]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#140), dynamicpruningexpression(ss_sold_date_sk#140 IN dynamicpruning#141)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_cdemo_sk:int,ss_hdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_promo_sk:int,ss_ticket_number:int,ss_wholesale_cost:decimal(7,2),ss_list_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(133) ColumnarToRow [codegen id : 44]
+Input [12]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_store_sk#134, ss_promo_sk#135, ss_ticket_number#136, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, ss_sold_date_sk#140]
+
+(134) Filter [codegen id : 44]
+Input [12]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_store_sk#134, ss_promo_sk#135, ss_ticket_number#136, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, ss_sold_date_sk#140]
+Condition : (((((((isnotnull(ss_item_sk#129) AND isnotnull(ss_ticket_number#136)) AND isnotnull(ss_store_sk#134)) AND isnotnull(ss_customer_sk#130)) AND isnotnull(ss_cdemo_sk#131)) AND isnotnull(ss_promo_sk#135)) AND isnotnull(ss_hdemo_sk#132)) AND isnotnull(ss_addr_sk#133))
+
+(135) Exchange
+Input [12]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_store_sk#134, ss_promo_sk#135, ss_ticket_number#136, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, ss_sold_date_sk#140]
+Arguments: hashpartitioning(ss_item_sk#129, ss_ticket_number#136, 5), ENSURE_REQUIREMENTS, [id=#142]
+
+(136) Sort [codegen id : 45]
+Input [12]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_store_sk#134, ss_promo_sk#135, ss_ticket_number#136, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, ss_sold_date_sk#140]
+Arguments: [ss_item_sk#129 ASC NULLS FIRST, ss_ticket_number#136 ASC NULLS FIRST], false, 0
+
+(137) ReusedExchange [Reuses operator id: 10]
+Output [2]: [sr_item_sk#143, sr_ticket_number#144]
+
+(138) Sort [codegen id : 47]
+Input [2]: [sr_item_sk#143, sr_ticket_number#144]
+Arguments: [sr_item_sk#143 ASC NULLS FIRST, sr_ticket_number#144 ASC NULLS FIRST], false, 0
+
+(139) SortMergeJoin [codegen id : 56]
+Left keys [2]: [ss_item_sk#129, ss_ticket_number#136]
+Right keys [2]: [sr_item_sk#143, sr_ticket_number#144]
+Join condition: None
+
+(140) Project [codegen id : 56]
+Output [11]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_store_sk#134, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, ss_sold_date_sk#140]
+Input [14]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_store_sk#134, ss_promo_sk#135, ss_ticket_number#136, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, ss_sold_date_sk#140, sr_item_sk#143, sr_ticket_number#144]
+
+(141) ReusedExchange [Reuses operator id: 33]
+Output [1]: [cs_item_sk#145]
+
+(142) BroadcastHashJoin [codegen id : 56]
+Left keys [1]: [ss_item_sk#129]
+Right keys [1]: [cs_item_sk#145]
+Join condition: None
+
+(143) Project [codegen id : 56]
+Output [11]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_store_sk#134, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, ss_sold_date_sk#140]
+Input [12]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_store_sk#134, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, ss_sold_date_sk#140, cs_item_sk#145]
+
+(144) Scan parquet default.date_dim
+Output [2]: [d_date_sk#146, d_year#147]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(145) ColumnarToRow [codegen id : 54]
+Input [2]: [d_date_sk#146, d_year#147]
+
+(146) Filter [codegen id : 54]
+Input [2]: [d_date_sk#146, d_year#147]
+Condition : ((isnotnull(d_year#147) AND (d_year#147 = 2000)) AND isnotnull(d_date_sk#146))
+
+(147) BroadcastExchange
+Input [2]: [d_date_sk#146, d_year#147]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#148]
+
+(148) BroadcastHashJoin [codegen id : 56]
+Left keys [1]: [ss_sold_date_sk#140]
+Right keys [1]: [d_date_sk#146]
+Join condition: None
+
+(149) Project [codegen id : 56]
+Output [11]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_store_sk#134, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147]
+Input [13]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_store_sk#134, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, ss_sold_date_sk#140, d_date_sk#146, d_year#147]
+
+(150) ReusedExchange [Reuses operator id: 45]
+Output [3]: [s_store_sk#149, s_store_name#150, s_zip#151]
+
+(151) BroadcastHashJoin [codegen id : 56]
+Left keys [1]: [ss_store_sk#134]
+Right keys [1]: [s_store_sk#149]
+Join condition: None
+
+(152) Project [codegen id : 56]
+Output [12]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151]
+Input [14]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_store_sk#134, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_sk#149, s_store_name#150, s_zip#151]
+
+(153) Exchange
+Input [12]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151]
+Arguments: hashpartitioning(ss_customer_sk#130, 5), ENSURE_REQUIREMENTS, [id=#152]
+
+(154) Sort [codegen id : 57]
+Input [12]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151]
+Arguments: [ss_customer_sk#130 ASC NULLS FIRST], false, 0
+
+(155) ReusedExchange [Reuses operator id: 53]
+Output [6]: [c_customer_sk#153, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, c_first_shipto_date_sk#157, c_first_sales_date_sk#158]
+
+(156) Sort [codegen id : 59]
+Input [6]: [c_customer_sk#153, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, c_first_shipto_date_sk#157, c_first_sales_date_sk#158]
+Arguments: [c_customer_sk#153 ASC NULLS FIRST], false, 0
+
+(157) SortMergeJoin [codegen id : 62]
+Left keys [1]: [ss_customer_sk#130]
+Right keys [1]: [c_customer_sk#153]
+Join condition: None
+
+(158) Project [codegen id : 62]
+Output [16]: [ss_item_sk#129, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, c_first_shipto_date_sk#157, c_first_sales_date_sk#158]
+Input [18]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_customer_sk#153, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, c_first_shipto_date_sk#157, c_first_sales_date_sk#158]
+
+(159) ReusedExchange [Reuses operator id: 60]
+Output [2]: [d_date_sk#159, d_year#160]
+
+(160) BroadcastHashJoin [codegen id : 62]
+Left keys [1]: [c_first_sales_date_sk#158]
+Right keys [1]: [d_date_sk#159]
+Join condition: None
+
+(161) Project [codegen id : 62]
+Output [16]: [ss_item_sk#129, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, c_first_shipto_date_sk#157, d_year#160]
+Input [18]: [ss_item_sk#129, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, c_first_shipto_date_sk#157, c_first_sales_date_sk#158, d_date_sk#159, d_year#160]
+
+(162) ReusedExchange [Reuses operator id: 60]
+Output [2]: [d_date_sk#161, d_year#162]
+
+(163) BroadcastHashJoin [codegen id : 62]
+Left keys [1]: [c_first_shipto_date_sk#157]
+Right keys [1]: [d_date_sk#161]
+Join condition: None
+
+(164) Project [codegen id : 62]
+Output [16]: [ss_item_sk#129, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162]
+Input [18]: [ss_item_sk#129, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, c_first_shipto_date_sk#157, d_year#160, d_date_sk#161, d_year#162]
+
+(165) Exchange
+Input [16]: [ss_item_sk#129, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162]
+Arguments: hashpartitioning(ss_cdemo_sk#131, 5), ENSURE_REQUIREMENTS, [id=#163]
+
+(166) Sort [codegen id : 63]
+Input [16]: [ss_item_sk#129, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162]
+Arguments: [ss_cdemo_sk#131 ASC NULLS FIRST], false, 0
+
+(167) ReusedExchange [Reuses operator id: 71]
+Output [2]: [cd_demo_sk#164, cd_marital_status#165]
+
+(168) Sort [codegen id : 65]
+Input [2]: [cd_demo_sk#164, cd_marital_status#165]
+Arguments: [cd_demo_sk#164 ASC NULLS FIRST], false, 0
+
+(169) SortMergeJoin [codegen id : 66]
+Left keys [1]: [ss_cdemo_sk#131]
+Right keys [1]: [cd_demo_sk#164]
+Join condition: None
+
+(170) Project [codegen id : 66]
+Output [16]: [ss_item_sk#129, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162, cd_marital_status#165]
+Input [18]: [ss_item_sk#129, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162, cd_demo_sk#164, cd_marital_status#165]
+
+(171) Exchange
+Input [16]: [ss_item_sk#129, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162, cd_marital_status#165]
+Arguments: hashpartitioning(c_current_cdemo_sk#154, 5), ENSURE_REQUIREMENTS, [id=#166]
+
+(172) Sort [codegen id : 67]
+Input [16]: [ss_item_sk#129, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162, cd_marital_status#165]
+Arguments: [c_current_cdemo_sk#154 ASC NULLS FIRST], false, 0
+
+(173) ReusedExchange [Reuses operator id: 71]
+Output [2]: [cd_demo_sk#167, cd_marital_status#168]
+
+(174) Sort [codegen id : 69]
+Input [2]: [cd_demo_sk#167, cd_marital_status#168]
+Arguments: [cd_demo_sk#167 ASC NULLS FIRST], false, 0
+
+(175) SortMergeJoin [codegen id : 73]
+Left keys [1]: [c_current_cdemo_sk#154]
+Right keys [1]: [cd_demo_sk#167]
+Join condition: NOT (cd_marital_status#165 = cd_marital_status#168)
+
+(176) Project [codegen id : 73]
+Output [14]: [ss_item_sk#129, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162]
+Input [18]: [ss_item_sk#129, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162, cd_marital_status#165, cd_demo_sk#167, cd_marital_status#168]
+
+(177) ReusedExchange [Reuses operator id: 84]
+Output [1]: [p_promo_sk#169]
+
+(178) BroadcastHashJoin [codegen id : 73]
+Left keys [1]: [ss_promo_sk#135]
+Right keys [1]: [p_promo_sk#169]
+Join condition: None
+
+(179) Project [codegen id : 73]
+Output [13]: [ss_item_sk#129, ss_hdemo_sk#132, ss_addr_sk#133, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162]
+Input [15]: [ss_item_sk#129, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162, p_promo_sk#169]
+
+(180) ReusedExchange [Reuses operator id: 90]
+Output [2]: [hd_demo_sk#170, hd_income_band_sk#171]
+
+(181) BroadcastHashJoin [codegen id : 73]
+Left keys [1]: [ss_hdemo_sk#132]
+Right keys [1]: [hd_demo_sk#170]
+Join condition: None
+
+(182) Project [codegen id : 73]
+Output [13]: [ss_item_sk#129, ss_addr_sk#133, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162, hd_income_band_sk#171]
+Input [15]: [ss_item_sk#129, ss_hdemo_sk#132, ss_addr_sk#133, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162, hd_demo_sk#170, hd_income_band_sk#171]
+
+(183) ReusedExchange [Reuses operator id: 90]
+Output [2]: [hd_demo_sk#172, hd_income_band_sk#173]
+
+(184) BroadcastHashJoin [codegen id : 73]
+Left keys [1]: [c_current_hdemo_sk#155]
+Right keys [1]: [hd_demo_sk#172]
+Join condition: None
+
+(185) Project [codegen id : 73]
+Output [13]: [ss_item_sk#129, ss_addr_sk#133, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_addr_sk#156, d_year#160, d_year#162, hd_income_band_sk#171, hd_income_band_sk#173]
+Input [15]: [ss_item_sk#129, ss_addr_sk#133, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162, hd_income_band_sk#171, hd_demo_sk#172, hd_income_band_sk#173]
+
+(186) Exchange
+Input [13]: [ss_item_sk#129, ss_addr_sk#133, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_addr_sk#156, d_year#160, d_year#162, hd_income_band_sk#171, hd_income_band_sk#173]
+Arguments: hashpartitioning(ss_addr_sk#133, 5), ENSURE_REQUIREMENTS, [id=#174]
+
+(187) Sort [codegen id : 74]
+Input [13]: [ss_item_sk#129, ss_addr_sk#133, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_addr_sk#156, d_year#160, d_year#162, hd_income_band_sk#171, hd_income_band_sk#173]
+Arguments: [ss_addr_sk#133 ASC NULLS FIRST], false, 0
+
+(188) ReusedExchange [Reuses operator id: 101]
+Output [5]: [ca_address_sk#175, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179]
+
+(189) Sort [codegen id : 76]
+Input [5]: [ca_address_sk#175, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179]
+Arguments: [ca_address_sk#175 ASC NULLS FIRST], false, 0
+
+(190) SortMergeJoin [codegen id : 77]
+Left keys [1]: [ss_addr_sk#133]
+Right keys [1]: [ca_address_sk#175]
+Join condition: None
+
+(191) Project [codegen id : 77]
+Output [16]: [ss_item_sk#129, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_addr_sk#156, d_year#160, d_year#162, hd_income_band_sk#171, hd_income_band_sk#173, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179]
+Input [18]: [ss_item_sk#129, ss_addr_sk#133, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_addr_sk#156, d_year#160, d_year#162, hd_income_band_sk#171, hd_income_band_sk#173, ca_address_sk#175, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179]
+
+(192) Exchange
+Input [16]: [ss_item_sk#129, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_addr_sk#156, d_year#160, d_year#162, hd_income_band_sk#171, hd_income_band_sk#173, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179]
+Arguments: hashpartitioning(c_current_addr_sk#156, 5), ENSURE_REQUIREMENTS, [id=#180]
+
+(193) Sort [codegen id : 78]
+Input [16]: [ss_item_sk#129, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_addr_sk#156, d_year#160, d_year#162, hd_income_band_sk#171, hd_income_band_sk#173, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179]
+Arguments: [c_current_addr_sk#156 ASC NULLS FIRST], false, 0
+
+(194) ReusedExchange [Reuses operator id: 101]
+Output [5]: [ca_address_sk#181, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185]
+
+(195) Sort [codegen id : 80]
+Input [5]: [ca_address_sk#181, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185]
+Arguments: [ca_address_sk#181 ASC NULLS FIRST], false, 0
+
+(196) SortMergeJoin [codegen id : 84]
+Left keys [1]: [c_current_addr_sk#156]
+Right keys [1]: [ca_address_sk#181]
+Join condition: None
+
+(197) Project [codegen id : 84]
+Output [19]: [ss_item_sk#129, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, d_year#160, d_year#162, hd_income_band_sk#171, hd_income_band_sk#173, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185]
+Input [21]: [ss_item_sk#129, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_addr_sk#156, d_year#160, d_year#162, hd_income_band_sk#171, hd_income_band_sk#173, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_address_sk#181, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185]
+
+(198) ReusedExchange [Reuses operator id: 114]
+Output [1]: [ib_income_band_sk#186]
+
+(199) BroadcastHashJoin [codegen id : 84]
+Left keys [1]: [hd_income_band_sk#171]
+Right keys [1]: [ib_income_band_sk#186]
+Join condition: None
+
+(200) Project [codegen id : 84]
+Output [18]: [ss_item_sk#129, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, d_year#160, d_year#162, hd_income_band_sk#173, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185]
+Input [20]: [ss_item_sk#129, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, d_year#160, d_year#162, hd_income_band_sk#171, hd_income_band_sk#173, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185, ib_income_band_sk#186]
+
+(201) ReusedExchange [Reuses operator id: 114]
+Output [1]: [ib_income_band_sk#187]
+
+(202) BroadcastHashJoin [codegen id : 84]
+Left keys [1]: [hd_income_band_sk#173]
+Right keys [1]: [ib_income_band_sk#187]
+Join condition: None
+
+(203) Project [codegen id : 84]
+Output [17]: [ss_item_sk#129, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, d_year#160, d_year#162, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185]
+Input [19]: [ss_item_sk#129, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, d_year#160, d_year#162, hd_income_band_sk#173, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185, ib_income_band_sk#187]
+
+(204) ReusedExchange [Reuses operator id: 124]
+Output [2]: [i_item_sk#188, i_product_name#189]
+
+(205) BroadcastHashJoin [codegen id : 84]
+Left keys [1]: [ss_item_sk#129]
+Right keys [1]: [i_item_sk#188]
+Join condition: None
+
+(206) Project [codegen id : 84]
+Output [18]: [ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, d_year#160, d_year#162, s_store_name#150, s_zip#151, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185, i_item_sk#188, i_product_name#189]
+Input [19]: [ss_item_sk#129, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, d_year#160, d_year#162, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185, i_item_sk#188, i_product_name#189]
+
+(207) HashAggregate [codegen id : 84]
+Input [18]: [ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, d_year#160, d_year#162, s_store_name#150, s_zip#151, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185, i_item_sk#188, i_product_name#189]
+Keys [15]: [i_product_name#189, i_item_sk#188, s_store_name#150, s_zip#151, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185, d_year#147, d_year#160, d_year#162]
+Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#137)), partial_sum(UnscaledValue(ss_list_price#138)), partial_sum(UnscaledValue(ss_coupon_amt#139))]
+Aggregate Attributes [4]: [count#190, sum#191, sum#192, sum#193]
+Results [19]: [i_product_name#189, i_item_sk#188, s_store_name#150, s_zip#151, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185, d_year#147, d_year#160, d_year#162, count#194, sum#195, sum#196, sum#197]
+
+(208) Exchange
+Input [19]: [i_product_name#189, i_item_sk#188, s_store_name#150, s_zip#151, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185, d_year#147, d_year#160, d_year#162, count#194, sum#195, sum#196, sum#197]
+Arguments: hashpartitioning(i_product_name#189, i_item_sk#188, s_store_name#150, s_zip#151, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185, d_year#147, d_year#160, d_year#162, 5), ENSURE_REQUIREMENTS, [id=#198]
+
+(209) HashAggregate [codegen id : 85]
+Input [19]: [i_product_name#189, i_item_sk#188, s_store_name#150, s_zip#151, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185, d_year#147, d_year#160, d_year#162, count#194, sum#195, sum#196, sum#197]
+Keys [15]: [i_product_name#189, i_item_sk#188, s_store_name#150, s_zip#151, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185, d_year#147, d_year#160, d_year#162]
+Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#137)), sum(UnscaledValue(ss_list_price#138)), sum(UnscaledValue(ss_coupon_amt#139))]
+Aggregate Attributes [4]: [count(1)#199, sum(UnscaledValue(ss_wholesale_cost#137))#200, sum(UnscaledValue(ss_list_price#138))#201, sum(UnscaledValue(ss_coupon_amt#139))#202]
+Results [8]: [i_item_sk#188 AS item_sk#203, s_store_name#150 AS store_name#204, s_zip#151 AS store_zip#205, d_year#147 AS syear#206, count(1)#199 AS cnt#207, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#137))#200,17,2) AS s1#208, MakeDecimal(sum(UnscaledValue(ss_list_price#138))#201,17,2) AS s2#209, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#139))#202,17,2) AS s3#210]
+
+(210) Exchange
+Input [8]: [item_sk#203, store_name#204, store_zip#205, syear#206, cnt#207, s1#208, s2#209, s3#210]
+Arguments: hashpartitioning(item_sk#203, store_name#204, store_zip#205, 5), ENSURE_REQUIREMENTS, [id=#211]
+
+(211) Sort [codegen id : 86]
+Input [8]: [item_sk#203, store_name#204, store_zip#205, syear#206, cnt#207, s1#208, s2#209, s3#210]
+Arguments: [item_sk#203 ASC NULLS FIRST, store_name#204 ASC NULLS FIRST, store_zip#205 ASC NULLS FIRST], false, 0
+
+(212) SortMergeJoin [codegen id : 87]
+Left keys [3]: [item_sk#112, store_name#113, store_zip#114]
+Right keys [3]: [item_sk#203, store_name#204, store_zip#205]
+Join condition: (cnt#207 <= cnt#124)
+
+(213) Project [codegen id : 87]
+Output [21]: [product_name#111, store_name#113, store_zip#114, b_street_number#115, b_streen_name#116, b_city#117, b_zip#118, c_street_number#119, c_street_name#120, c_city#121, c_zip#122, syear#123, cnt#124, s1#125, s2#126, s3#127, s1#208, s2#209, s3#210, syear#206, cnt#207]
+Input [25]: [product_name#111, item_sk#112, store_name#113, store_zip#114, b_street_number#115, b_streen_name#116, b_city#117, b_zip#118, c_street_number#119, c_street_name#120, c_city#121, c_zip#122, syear#123, cnt#124, s1#125, s2#126, s3#127, item_sk#203, store_name#204, store_zip#205, syear#206, cnt#207, s1#208, s2#209, s3#210]
+
+(214) Exchange
+Input [21]: [product_name#111, store_name#113, store_zip#114, b_street_number#115, b_streen_name#116, b_city#117, b_zip#118, c_street_number#119, c_street_name#120, c_city#121, c_zip#122, syear#123, cnt#124, s1#125, s2#126, s3#127, s1#208, s2#209, s3#210, syear#206, cnt#207]
+Arguments: rangepartitioning(product_name#111 ASC NULLS FIRST, store_name#113 ASC NULLS FIRST, cnt#207 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [id=#212]
+
+(215) Sort [codegen id : 88]
+Input [21]: [product_name#111, store_name#113, store_zip#114, b_street_number#115, b_streen_name#116, b_city#117, b_zip#118, c_street_number#119, c_street_name#120, c_city#121, c_zip#122, syear#123, cnt#124, s1#125, s2#126, s3#127, s1#208, s2#209, s3#210, syear#206, cnt#207]
+Arguments: [product_name#111 ASC NULLS FIRST, store_name#113 ASC NULLS FIRST, cnt#207 ASC NULLS FIRST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13
+ReusedExchange (216)
+
+
+(216) ReusedExchange [Reuses operator id: 39]
+Output [2]: [d_date_sk#43, d_year#44]
+
+Subquery:2 Hosting operator id = 132 Hosting Expression = ss_sold_date_sk#140 IN dynamicpruning#141
+ReusedExchange (217)
+
+
+(217) ReusedExchange [Reuses operator id: 147]
+Output [2]: [d_date_sk#146, d_year#147]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.sf100/simplified.txt
new file mode 100644
index 0000000000000..09023ded66475
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.sf100/simplified.txt
@@ -0,0 +1,379 @@
+WholeStageCodegen (88)
+  Sort [product_name,store_name,cnt]
+    InputAdapter
+      Exchange [product_name,store_name,cnt] #1
+        WholeStageCodegen (87)
+          Project [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt]
+            SortMergeJoin [item_sk,store_name,store_zip,item_sk,store_name,store_zip,cnt,cnt]
+              InputAdapter
+                WholeStageCodegen (43)
+                  Sort [item_sk,store_name,store_zip]
+                    InputAdapter
+                      Exchange [item_sk,store_name,store_zip] #2
+                        WholeStageCodegen (42)
+                          HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum]
+                            InputAdapter
+                              Exchange [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year] #3
+                                WholeStageCodegen (41)
+                                  HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum]
+                                    Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name]
+                                      BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                        Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                          BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk]
+                                            Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                              BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk]
+                                                Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                                  SortMergeJoin [c_current_addr_sk,ca_address_sk]
+                                                    InputAdapter
+                                                      WholeStageCodegen (35)
+                                                        Sort [c_current_addr_sk]
+                                                          InputAdapter
+                                                            Exchange [c_current_addr_sk] #4
+                                                              WholeStageCodegen (34)
+                                                                Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                                                  SortMergeJoin [ss_addr_sk,ca_address_sk]
+                                                                    InputAdapter
+                                                                      WholeStageCodegen (31)
+                                                                        Sort [ss_addr_sk]
+                                                                          InputAdapter
+                                                                            Exchange [ss_addr_sk] #5
+                                                                              WholeStageCodegen (30)
+                                                                                Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk]
+                                                                                  BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk]
+                                                                                    Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk]
+                                                                                      BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                                                                        Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year]
+                                                                                          BroadcastHashJoin [ss_promo_sk,p_promo_sk]
+                                                                                            Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year]
+                                                                                              SortMergeJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status]
+                                                                                                InputAdapter
+                                                                                                  WholeStageCodegen (24)
+                                                                                                    Sort [c_current_cdemo_sk]
+                                                                                                      InputAdapter
+                                                                                                        Exchange [c_current_cdemo_sk] #6
+                                                                                                          WholeStageCodegen (23)
+                                                                                                            Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status]
+                                                                                                              SortMergeJoin [ss_cdemo_sk,cd_demo_sk]
+                                                                                                                InputAdapter
+                                                                                                                  WholeStageCodegen (20)
+                                                                                                                    Sort [ss_cdemo_sk]
+                                                                                                                      InputAdapter
+                                                                                                                        Exchange [ss_cdemo_sk] #7
+                                                                                                                          WholeStageCodegen (19)
+                                                                                                                            Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year]
+                                                                                                                              BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk]
+                                                                                                                                Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year]
+                                                                                                                                  BroadcastHashJoin [c_first_sales_date_sk,d_date_sk]
+                                                                                                                                    Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk]
+                                                                                                                                      SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                                                                                                                        InputAdapter
+                                                                                                                                          WholeStageCodegen (14)
+                                                                                                                                            Sort [ss_customer_sk]
+                                                                                                                                              InputAdapter
+                                                                                                                                                Exchange [ss_customer_sk] #8
+                                                                                                                                                  WholeStageCodegen (13)
+                                                                                                                                                    Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip]
+                                                                                                                                                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                                                                                                                        Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year]
+                                                                                                                                                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                                                                                            Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+                                                                                                                                                              BroadcastHashJoin [ss_item_sk,cs_item_sk]
+                                                                                                                                                                Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+                                                                                                                                                                  SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number]
+                                                                                                                                                                    InputAdapter
+                                                                                                                                                                      WholeStageCodegen (2)
+                                                                                                                                                                        Sort [ss_item_sk,ss_ticket_number]
+                                                                                                                                                                          InputAdapter
+                                                                                                                                                                            Exchange [ss_item_sk,ss_ticket_number] #9
+                                                                                                                                                                              WholeStageCodegen (1)
+                                                                                                                                                                                Filter [ss_item_sk,ss_ticket_number,ss_store_sk,ss_customer_sk,ss_cdemo_sk,ss_promo_sk,ss_hdemo_sk,ss_addr_sk]
+                                                                                                                                                                                  ColumnarToRow
+                                                                                                                                                                                    InputAdapter
+                                                                                                                                                                                      Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+                                                                                                                                                                                        SubqueryBroadcast [d_date_sk] #1
+                                                                                                                                                                                          ReusedExchange [d_date_sk,d_year] #10
+                                                                                                                                                                    InputAdapter
+                                                                                                                                                                      WholeStageCodegen (4)
+                                                                                                                                                                        Sort [sr_item_sk,sr_ticket_number]
+                                                                                                                                                                          InputAdapter
+                                                                                                                                                                            Exchange [sr_item_sk,sr_ticket_number] #11
+                                                                                                                                                                              WholeStageCodegen (3)
+                                                                                                                                                                                Project [sr_item_sk,sr_ticket_number]
+                                                                                                                                                                                  Filter [sr_item_sk,sr_ticket_number]
+                                                                                                                                                                                    ColumnarToRow
+                                                                                                                                                                                      InputAdapter
+                                                                                                                                                                                        Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk]
+                                                                                                                                                                InputAdapter
+                                                                                                                                                                  BroadcastExchange #12
+                                                                                                                                                                    WholeStageCodegen (10)
+                                                                                                                                                                      Project [cs_item_sk]
+                                                                                                                                                                        Filter [sum(cs_ext_list_price),sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash as decimal(8,2))) + promote_precision(cast(cr_reversed_charge as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit as decimal(9,2)))), DecimalType(9,2), true))]
+                                                                                                                                                                          HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash as decimal(8,2))) + promote_precision(cast(cr_reversed_charge as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit as decimal(9,2)))), DecimalType(9,2), true)),sum(cs_ext_list_price),sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash as decimal(8,2))) + promote_precision(cast(cr_reversed_charge as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit as decimal(9,2)))), DecimalType(9,2), true)),sum,sum,isEmpty]
+                                                                                                                                                                            InputAdapter
+                                                                                                                                                                              Exchange [cs_item_sk] #13
+                                                                                                                                                                                WholeStageCodegen (9)
+                                                                                                                                                                                  HashAggregate [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [sum,sum,isEmpty,sum,sum,isEmpty]
+                                                                                                                                                                                    Project [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit]
+                                                                                                                                                                                      SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number]
+                                                                                                                                                                                        InputAdapter
+                                                                                                                                                                                          WholeStageCodegen (6)
+                                                                                                                                                                                            Sort [cs_item_sk,cs_order_number]
+                                                                                                                                                                                              InputAdapter
+                                                                                                                                                                                                Exchange [cs_item_sk,cs_order_number] #14
+                                                                                                                                                                                                  WholeStageCodegen (5)
+                                                                                                                                                                                                    Project [cs_item_sk,cs_order_number,cs_ext_list_price]
+                                                                                                                                                                                                      Filter [cs_item_sk,cs_order_number]
+                                                                                                                                                                                                        ColumnarToRow
+                                                                                                                                                                                                          InputAdapter
+                                                                                                                                                                                                            Scan parquet default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk]
+                                                                                                                                                                                        InputAdapter
+                                                                                                                                                                                          WholeStageCodegen (8)
+                                                                                                                                                                                            Sort [cr_item_sk,cr_order_number]
+                                                                                                                                                                                              InputAdapter
+                                                                                                                                                                                                Exchange [cr_item_sk,cr_order_number] #15
+                                                                                                                                                                                                  WholeStageCodegen (7)
+                                                                                                                                                                                                    Project [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit]
+                                                                                                                                                                                                      Filter [cr_item_sk,cr_order_number]
+                                                                                                                                                                                                        ColumnarToRow
+                                                                                                                                                                                                          InputAdapter
+                                                                                                                                                                                                            Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk]
+                                                                                                                                                            InputAdapter
+                                                                                                                                                              BroadcastExchange #10
+                                                                                                                                                                WholeStageCodegen (11)
+                                                                                                                                                                  Filter [d_year,d_date_sk]
+                                                                                                                                                                    ColumnarToRow
+                                                                                                                                                                      InputAdapter
+                                                                                                                                                                        Scan parquet default.date_dim [d_date_sk,d_year]
+                                                                                                                                                        InputAdapter
+                                                                                                                                                          BroadcastExchange #16
+                                                                                                                                                            WholeStageCodegen (12)
+                                                                                                                                                              Filter [s_store_sk,s_store_name,s_zip]
+                                                                                                                                                                ColumnarToRow
+                                                                                                                                                                  InputAdapter
+                                                                                                                                                                    Scan parquet default.store [s_store_sk,s_store_name,s_zip]
+                                                                                                                                        InputAdapter
+                                                                                                                                          WholeStageCodegen (16)
+                                                                                                                                            Sort [c_customer_sk]
+                                                                                                                                              InputAdapter
+                                                                                                                                                Exchange [c_customer_sk] #17
+                                                                                                                                                  WholeStageCodegen (15)
+                                                                                                                                                    Filter [c_customer_sk,c_first_sales_date_sk,c_first_shipto_date_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk]
+                                                                                                                                                      ColumnarToRow
+                                                                                                                                                        InputAdapter
+                                                                                                                                                          Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk]
+                                                                                                                                    InputAdapter
+                                                                                                                                      BroadcastExchange #18
+                                                                                                                                        WholeStageCodegen (17)
+                                                                                                                                          Filter [d_date_sk]
+                                                                                                                                            ColumnarToRow
+                                                                                                                                              InputAdapter
+                                                                                                                                                Scan parquet default.date_dim [d_date_sk,d_year]
+                                                                                                                                InputAdapter
+                                                                                                                                  ReusedExchange [d_date_sk,d_year] #18
+                                                                                                                InputAdapter
+                                                                                                                  WholeStageCodegen (22)
+                                                                                                                    Sort [cd_demo_sk]
+                                                                                                                      InputAdapter
+                                                                                                                        Exchange [cd_demo_sk] #19
+                                                                                                                          WholeStageCodegen (21)
+                                                                                                                            Filter [cd_demo_sk,cd_marital_status]
+                                                                                                                              ColumnarToRow
+                                                                                                                                InputAdapter
+                                                                                                                                  Scan parquet default.customer_demographics [cd_demo_sk,cd_marital_status]
+                                                                                                InputAdapter
+                                                                                                  WholeStageCodegen (26)
+                                                                                                    Sort [cd_demo_sk]
+                                                                                                      InputAdapter
+                                                                                                        ReusedExchange [cd_demo_sk,cd_marital_status] #19
+                                                                                            InputAdapter
+                                                                                              BroadcastExchange #20
+                                                                                                WholeStageCodegen (27)
+                                                                                                  Filter [p_promo_sk]
+                                                                                                    ColumnarToRow
+                                                                                                      InputAdapter
+                                                                                                        Scan parquet default.promotion [p_promo_sk]
+                                                                                        InputAdapter
+                                                                                          BroadcastExchange #21
+                                                                                            WholeStageCodegen (28)
+                                                                                              Filter [hd_demo_sk,hd_income_band_sk]
+                                                                                                ColumnarToRow
+                                                                                                  InputAdapter
+                                                                                                    Scan parquet default.household_demographics [hd_demo_sk,hd_income_band_sk]
+                                                                                    InputAdapter
+                                                                                      ReusedExchange [hd_demo_sk,hd_income_band_sk] #21
+                                                                    InputAdapter
+                                                                      WholeStageCodegen (33)
+                                                                        Sort [ca_address_sk]
+                                                                          InputAdapter
+                                                                            Exchange [ca_address_sk] #22
+                                                                              WholeStageCodegen (32)
+                                                                                Filter [ca_address_sk]
+                                                                                  ColumnarToRow
+                                                                                    InputAdapter
+                                                                                      Scan parquet default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                                    InputAdapter
+                                                      WholeStageCodegen (37)
+                                                        Sort [ca_address_sk]
+                                                          InputAdapter
+                                                            ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #22
+                                                InputAdapter
+                                                  BroadcastExchange #23
+                                                    WholeStageCodegen (38)
+                                                      Filter [ib_income_band_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.income_band [ib_income_band_sk]
+                                            InputAdapter
+                                              ReusedExchange [ib_income_band_sk] #23
+                                        InputAdapter
+                                          BroadcastExchange #24
+                                            WholeStageCodegen (40)
+                                              Project [i_item_sk,i_product_name]
+                                                Filter [i_current_price,i_color,i_item_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.item [i_item_sk,i_current_price,i_color,i_product_name]
+              InputAdapter
+                WholeStageCodegen (86)
+                  Sort [item_sk,store_name,store_zip]
+                    InputAdapter
+                      Exchange [item_sk,store_name,store_zip] #25
+                        WholeStageCodegen (85)
+                          HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum]
+                            InputAdapter
+                              Exchange [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year] #26
+                                WholeStageCodegen (84)
+                                  HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum]
+                                    Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name]
+                                      BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                        Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                          BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk]
+                                            Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                              BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk]
+                                                Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                                  SortMergeJoin [c_current_addr_sk,ca_address_sk]
+                                                    InputAdapter
+                                                      WholeStageCodegen (78)
+                                                        Sort [c_current_addr_sk]
+                                                          InputAdapter
+                                                            Exchange [c_current_addr_sk] #27
+                                                              WholeStageCodegen (77)
+                                                                Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                                                  SortMergeJoin [ss_addr_sk,ca_address_sk]
+                                                                    InputAdapter
+                                                                      WholeStageCodegen (74)
+                                                                        Sort [ss_addr_sk]
+                                                                          InputAdapter
+                                                                            Exchange [ss_addr_sk] #28
+                                                                              WholeStageCodegen (73)
+                                                                                Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk]
+                                                                                  BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk]
+                                                                                    Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk]
+                                                                                      BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                                                                        Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year]
+                                                                                          BroadcastHashJoin [ss_promo_sk,p_promo_sk]
+                                                                                            Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year]
+                                                                                              SortMergeJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status]
+                                                                                                InputAdapter
+                                                                                                  WholeStageCodegen (67)
+                                                                                                    Sort [c_current_cdemo_sk]
+                                                                                                      InputAdapter
+                                                                                                        Exchange [c_current_cdemo_sk] #29
+                                                                                                          WholeStageCodegen (66)
+                                                                                                            Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status]
+                                                                                                              SortMergeJoin [ss_cdemo_sk,cd_demo_sk]
+                                                                                                                InputAdapter
+                                                                                                                  WholeStageCodegen (63)
+                                                                                                                    Sort [ss_cdemo_sk]
+                                                                                                                      InputAdapter
+                                                                                                                        Exchange [ss_cdemo_sk] #30
+                                                                                                                          WholeStageCodegen (62)
+                                                                                                                            Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year]
+                                                                                                                              BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk]
+                                                                                                                                Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year]
+                                                                                                                                  BroadcastHashJoin [c_first_sales_date_sk,d_date_sk]
+                                                                                                                                    Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk]
+                                                                                                                                      SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                                                                                                                        InputAdapter
+                                                                                                                                          WholeStageCodegen (57)
+                                                                                                                                            Sort [ss_customer_sk]
+                                                                                                                                              InputAdapter
+                                                                                                                                                Exchange [ss_customer_sk] #31
+                                                                                                                                                  WholeStageCodegen (56)
+                                                                                                                                                    Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip]
+                                                                                                                                                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                                                                                                                        Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year]
+                                                                                                                                                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                                                                                            Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+                                                                                                                                                              BroadcastHashJoin [ss_item_sk,cs_item_sk]
+                                                                                                                                                                Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+                                                                                                                                                                  SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number]
+                                                                                                                                                                    InputAdapter
+                                                                                                                                                                      WholeStageCodegen (45)
+                                                                                                                                                                        Sort [ss_item_sk,ss_ticket_number]
+                                                                                                                                                                          InputAdapter
+                                                                                                                                                                            Exchange [ss_item_sk,ss_ticket_number] #32
+                                                                                                                                                                              WholeStageCodegen (44)
+                                                                                                                                                                                Filter [ss_item_sk,ss_ticket_number,ss_store_sk,ss_customer_sk,ss_cdemo_sk,ss_promo_sk,ss_hdemo_sk,ss_addr_sk]
+                                                                                                                                                                                  ColumnarToRow
+                                                                                                                                                                                    InputAdapter
+                                                                                                                                                                                      Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+                                                                                                                                                                                        SubqueryBroadcast [d_date_sk] #2
+                                                                                                                                                                                          ReusedExchange [d_date_sk,d_year] #33
+                                                                                                                                                                    InputAdapter
+                                                                                                                                                                      WholeStageCodegen (47)
+                                                                                                                                                                        Sort [sr_item_sk,sr_ticket_number]
+                                                                                                                                                                          InputAdapter
+                                                                                                                                                                            ReusedExchange [sr_item_sk,sr_ticket_number] #11
+                                                                                                                                                                InputAdapter
+                                                                                                                                                                  ReusedExchange [cs_item_sk] #12
+                                                                                                                                                            InputAdapter
+                                                                                                                                                              BroadcastExchange #33
+                                                                                                                                                                WholeStageCodegen (54)
+                                                                                                                                                                  Filter [d_year,d_date_sk]
+                                                                                                                                                                    ColumnarToRow
+                                                                                                                                                                      InputAdapter
+                                                                                                                                                                        Scan parquet default.date_dim [d_date_sk,d_year]
+                                                                                                                                                        InputAdapter
+                                                                                                                                                          ReusedExchange [s_store_sk,s_store_name,s_zip] #16
+                                                                                                                                        InputAdapter
+                                                                                                                                          WholeStageCodegen (59)
+                                                                                                                                            Sort [c_customer_sk]
+                                                                                                                                              InputAdapter
+                                                                                                                                                ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #17
+                                                                                                                                    InputAdapter
+                                                                                                                                      ReusedExchange [d_date_sk,d_year] #18
+                                                                                                                                InputAdapter
+                                                                                                                                  ReusedExchange [d_date_sk,d_year] #18
+                                                                                                                InputAdapter
+                                                                                                                  WholeStageCodegen (65)
+                                                                                                                    Sort [cd_demo_sk]
+                                                                                                                      InputAdapter
+                                                                                                                        ReusedExchange [cd_demo_sk,cd_marital_status] #19
+                                                                                                InputAdapter
+                                                                                                  WholeStageCodegen (69)
+                                                                                                    Sort [cd_demo_sk]
+                                                                                                      InputAdapter
+                                                                                                        ReusedExchange [cd_demo_sk,cd_marital_status] #19
+                                                                                            InputAdapter
+                                                                                              ReusedExchange [p_promo_sk] #20
+                                                                                        InputAdapter
+                                                                                          ReusedExchange [hd_demo_sk,hd_income_band_sk] #21
+                                                                                    InputAdapter
+                                                                                      ReusedExchange [hd_demo_sk,hd_income_band_sk] #21
+                                                                    InputAdapter
+                                                                      WholeStageCodegen (76)
+                                                                        Sort [ca_address_sk]
+                                                                          InputAdapter
+                                                                            ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #22
+                                                    InputAdapter
+                                                      WholeStageCodegen (80)
+                                                        Sort [ca_address_sk]
+                                                          InputAdapter
+                                                            ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #22
+                                                InputAdapter
+                                                  ReusedExchange [ib_income_band_sk] #23
+                                            InputAdapter
+                                              ReusedExchange [ib_income_band_sk] #23
+                                        InputAdapter
+                                          ReusedExchange [i_item_sk,i_product_name] #24
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/explain.txt
new file mode 100644
index 0000000000000..c1138bbbf315e
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/explain.txt
@@ -0,0 +1,1038 @@
+== Physical Plan ==
+* Sort (189)
++- Exchange (188)
+   +- * Project (187)
+      +- * SortMergeJoin Inner (186)
+         :- * Sort (114)
+         :  +- Exchange (113)
+         :     +- * HashAggregate (112)
+         :        +- * HashAggregate (111)
+         :           +- * Project (110)
+         :              +- * BroadcastHashJoin Inner BuildRight (109)
+         :                 :- * Project (103)
+         :                 :  +- * BroadcastHashJoin Inner BuildRight (102)
+         :                 :     :- * Project (100)
+         :                 :     :  +- * BroadcastHashJoin Inner BuildRight (99)
+         :                 :     :     :- * Project (94)
+         :                 :     :     :  +- * BroadcastHashJoin Inner BuildRight (93)
+         :                 :     :     :     :- * Project (91)
+         :                 :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (90)
+         :                 :     :     :     :     :- * Project (85)
+         :                 :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (84)
+         :                 :     :     :     :     :     :- * Project (82)
+         :                 :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (81)
+         :                 :     :     :     :     :     :     :- * Project (76)
+         :                 :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (75)
+         :                 :     :     :     :     :     :     :     :- * Project (70)
+         :                 :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (69)
+         :                 :     :     :     :     :     :     :     :     :- * Project (67)
+         :                 :     :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (66)
+         :                 :     :     :     :     :     :     :     :     :     :- * Project (61)
+         :                 :     :     :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (60)
+         :                 :     :     :     :     :     :     :     :     :     :     :- * Project (58)
+         :                 :     :     :     :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (57)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :- * Project (52)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (51)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :- * Project (46)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (45)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :- * Project (40)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (39)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :- * Project (34)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :  +- * SortMergeJoin Inner (33)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :- * Sort (12)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :  +- Exchange (11)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     +- * Project (10)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :        +- * BroadcastHashJoin Inner BuildLeft (9)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :           :- BroadcastExchange (4)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :           :  +- * Filter (3)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :           :     +- * ColumnarToRow (2)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :           :        +- Scan parquet default.store_sales (1)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :           +- * Project (8)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :              +- * Filter (7)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                 +- * ColumnarToRow (6)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                    +- Scan parquet default.store_returns (5)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     +- * Sort (32)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :        +- * Project (31)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :           +- * Filter (30)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :              +- * HashAggregate (29)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                 +- Exchange (28)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                    +- * HashAggregate (27)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                       +- * Project (26)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                          +- * SortMergeJoin Inner (25)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                             :- * Sort (18)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                             :  +- Exchange (17)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                             :     +- * Project (16)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                             :        +- * Filter (15)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                             :           +- * ColumnarToRow (14)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                             :              +- Scan parquet default.catalog_sales (13)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                             +- * Sort (24)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                                +- Exchange (23)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                                   +- * Project (22)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                                      +- * Filter (21)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                                         +- * ColumnarToRow (20)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                                            +- Scan parquet default.catalog_returns (19)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     +- BroadcastExchange (38)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :        +- * Filter (37)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :           +- * ColumnarToRow (36)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :              +- Scan parquet default.date_dim (35)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     +- BroadcastExchange (44)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :        +- * Filter (43)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :           +- * ColumnarToRow (42)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :              +- Scan parquet default.store (41)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     +- BroadcastExchange (50)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :        +- * Filter (49)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :           +- * ColumnarToRow (48)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :              +- Scan parquet default.customer (47)
+         :                 :     :     :     :     :     :     :     :     :     :     :     +- BroadcastExchange (56)
+         :                 :     :     :     :     :     :     :     :     :     :     :        +- * Filter (55)
+         :                 :     :     :     :     :     :     :     :     :     :     :           +- * ColumnarToRow (54)
+         :                 :     :     :     :     :     :     :     :     :     :     :              +- Scan parquet default.date_dim (53)
+         :                 :     :     :     :     :     :     :     :     :     :     +- ReusedExchange (59)
+         :                 :     :     :     :     :     :     :     :     :     +- BroadcastExchange (65)
+         :                 :     :     :     :     :     :     :     :     :        +- * Filter (64)
+         :                 :     :     :     :     :     :     :     :     :           +- * ColumnarToRow (63)
+         :                 :     :     :     :     :     :     :     :     :              +- Scan parquet default.customer_demographics (62)
+         :                 :     :     :     :     :     :     :     :     +- ReusedExchange (68)
+         :                 :     :     :     :     :     :     :     +- BroadcastExchange (74)
+         :                 :     :     :     :     :     :     :        +- * Filter (73)
+         :                 :     :     :     :     :     :     :           +- * ColumnarToRow (72)
+         :                 :     :     :     :     :     :     :              +- Scan parquet default.promotion (71)
+         :                 :     :     :     :     :     :     +- BroadcastExchange (80)
+         :                 :     :     :     :     :     :        +- * Filter (79)
+         :                 :     :     :     :     :     :           +- * ColumnarToRow (78)
+         :                 :     :     :     :     :     :              +- Scan parquet default.household_demographics (77)
+         :                 :     :     :     :     :     +- ReusedExchange (83)
+         :                 :     :     :     :     +- BroadcastExchange (89)
+         :                 :     :     :     :        +- * Filter (88)
+         :                 :     :     :     :           +- * ColumnarToRow (87)
+         :                 :     :     :     :              +- Scan parquet default.customer_address (86)
+         :                 :     :     :     +- ReusedExchange (92)
+         :                 :     :     +- BroadcastExchange (98)
+         :                 :     :        +- * Filter (97)
+         :                 :     :           +- * ColumnarToRow (96)
+         :                 :     :              +- Scan parquet default.income_band (95)
+         :                 :     +- ReusedExchange (101)
+         :                 +- BroadcastExchange (108)
+         :                    +- * Project (107)
+         :                       +- * Filter (106)
+         :                          +- * ColumnarToRow (105)
+         :                             +- Scan parquet default.item (104)
+         +- * Sort (185)
+            +- Exchange (184)
+               +- * HashAggregate (183)
+                  +- * HashAggregate (182)
+                     +- * Project (181)
+                        +- * BroadcastHashJoin Inner BuildRight (180)
+                           :- * Project (178)
+                           :  +- * BroadcastHashJoin Inner BuildRight (177)
+                           :     :- * Project (175)
+                           :     :  +- * BroadcastHashJoin Inner BuildRight (174)
+                           :     :     :- * Project (172)
+                           :     :     :  +- * BroadcastHashJoin Inner BuildRight (171)
+                           :     :     :     :- * Project (169)
+                           :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (168)
+                           :     :     :     :     :- * Project (166)
+                           :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (165)
+                           :     :     :     :     :     :- * Project (163)
+                           :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (162)
+                           :     :     :     :     :     :     :- * Project (160)
+                           :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (159)
+                           :     :     :     :     :     :     :     :- * Project (157)
+                           :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (156)
+                           :     :     :     :     :     :     :     :     :- * Project (154)
+                           :     :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (153)
+                           :     :     :     :     :     :     :     :     :     :- * Project (151)
+                           :     :     :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (150)
+                           :     :     :     :     :     :     :     :     :     :     :- * Project (148)
+                           :     :     :     :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (147)
+                           :     :     :     :     :     :     :     :     :     :     :     :- * Project (145)
+                           :     :     :     :     :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (144)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :- * Project (142)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (141)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :- * Project (139)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (138)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :- * Project (133)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :  +- * SortMergeJoin Inner (132)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :- * Sort (126)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :  +- Exchange (125)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     +- * Project (124)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :        +- * BroadcastHashJoin Inner BuildLeft (123)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :           :- BroadcastExchange (118)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :           :  +- * Filter (117)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :           :     +- * ColumnarToRow (116)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :           :        +- Scan parquet default.store_sales (115)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :           +- * Project (122)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :              +- * Filter (121)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                 +- * ColumnarToRow (120)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                    +- Scan parquet default.store_returns (119)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     +- * Sort (131)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :        +- * Project (130)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :           +- * Filter (129)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :              +- * HashAggregate (128)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                 +- ReusedExchange (127)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     +- BroadcastExchange (137)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :        +- * Filter (136)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :           +- * ColumnarToRow (135)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :              +- Scan parquet default.date_dim (134)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     +- ReusedExchange (140)
+                           :     :     :     :     :     :     :     :     :     :     :     :     +- ReusedExchange (143)
+                           :     :     :     :     :     :     :     :     :     :     :     +- ReusedExchange (146)
+                           :     :     :     :     :     :     :     :     :     :     +- ReusedExchange (149)
+                           :     :     :     :     :     :     :     :     :     +- ReusedExchange (152)
+                           :     :     :     :     :     :     :     :     +- ReusedExchange (155)
+                           :     :     :     :     :     :     :     +- ReusedExchange (158)
+                           :     :     :     :     :     :     +- ReusedExchange (161)
+                           :     :     :     :     :     +- ReusedExchange (164)
+                           :     :     :     :     +- ReusedExchange (167)
+                           :     :     :     +- ReusedExchange (170)
+                           :     :     +- ReusedExchange (173)
+                           :     +- ReusedExchange (176)
+                           +- ReusedExchange (179)
+
+
+(1) Scan parquet default.store_sales
+Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_sold_date_sk#12 IN dynamicpruning#13)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_cdemo_sk:int,ss_hdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_promo_sk:int,ss_ticket_number:int,ss_wholesale_cost:decimal(7,2),ss_list_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+
+(3) Filter [codegen id : 1]
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AND isnotnull(ss_store_sk#6)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_cdemo_sk#3)) AND isnotnull(ss_promo_sk#7)) AND isnotnull(ss_hdemo_sk#4)) AND isnotnull(ss_addr_sk#5))
+
+(4) BroadcastExchange
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[7, int, false] as bigint) & 4294967295))),false), [id=#14]
+
+(5) Scan parquet default.store_returns
+Output [3]: [sr_item_sk#15, sr_ticket_number#16, sr_returned_date_sk#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int>
+
+(6) ColumnarToRow
+Input [3]: [sr_item_sk#15, sr_ticket_number#16, sr_returned_date_sk#17]
+
+(7) Filter
+Input [3]: [sr_item_sk#15, sr_ticket_number#16, sr_returned_date_sk#17]
+Condition : (isnotnull(sr_item_sk#15) AND isnotnull(sr_ticket_number#16))
+
+(8) Project
+Output [2]: [sr_item_sk#15, sr_ticket_number#16]
+Input [3]: [sr_item_sk#15, sr_ticket_number#16, sr_returned_date_sk#17]
+
+(9) BroadcastHashJoin [codegen id : 2]
+Left keys [2]: [ss_item_sk#1, ss_ticket_number#8]
+Right keys [2]: [sr_item_sk#15, sr_ticket_number#16]
+Join condition: None
+
+(10) Project [codegen id : 2]
+Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#15, sr_ticket_number#16]
+
+(11) Exchange
+Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(12) Sort [codegen id : 3]
+Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Arguments: [ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(13) Scan parquet default.catalog_sales
+Output [4]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21, cs_sold_date_sk#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_sales]
+PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)]
+ReadSchema: struct<cs_item_sk:int,cs_order_number:int,cs_ext_list_price:decimal(7,2)>
+
+(14) ColumnarToRow [codegen id : 4]
+Input [4]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21, cs_sold_date_sk#22]
+
+(15) Filter [codegen id : 4]
+Input [4]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21, cs_sold_date_sk#22]
+Condition : (isnotnull(cs_item_sk#19) AND isnotnull(cs_order_number#20))
+
+(16) Project [codegen id : 4]
+Output [3]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21]
+Input [4]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21, cs_sold_date_sk#22]
+
+(17) Exchange
+Input [3]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21]
+Arguments: hashpartitioning(cs_item_sk#19, cs_order_number#20, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(18) Sort [codegen id : 5]
+Input [3]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21]
+Arguments: [cs_item_sk#19 ASC NULLS FIRST, cs_order_number#20 ASC NULLS FIRST], false, 0
+
+(19) Scan parquet default.catalog_returns
+Output [6]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28, cr_returned_date_sk#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int,cr_refunded_cash:decimal(7,2),cr_reversed_charge:decimal(7,2),cr_store_credit:decimal(7,2)>
+
+(20) ColumnarToRow [codegen id : 6]
+Input [6]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28, cr_returned_date_sk#29]
+
+(21) Filter [codegen id : 6]
+Input [6]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28, cr_returned_date_sk#29]
+Condition : (isnotnull(cr_item_sk#24) AND isnotnull(cr_order_number#25))
+
+(22) Project [codegen id : 6]
+Output [5]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28]
+Input [6]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28, cr_returned_date_sk#29]
+
+(23) Exchange
+Input [5]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28]
+Arguments: hashpartitioning(cr_item_sk#24, cr_order_number#25, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(24) Sort [codegen id : 7]
+Input [5]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28]
+Arguments: [cr_item_sk#24 ASC NULLS FIRST, cr_order_number#25 ASC NULLS FIRST], false, 0
+
+(25) SortMergeJoin [codegen id : 8]
+Left keys [2]: [cs_item_sk#19, cs_order_number#20]
+Right keys [2]: [cr_item_sk#24, cr_order_number#25]
+Join condition: None
+
+(26) Project [codegen id : 8]
+Output [5]: [cs_item_sk#19, cs_ext_list_price#21, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28]
+Input [8]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21, cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28]
+
+(27) HashAggregate [codegen id : 8]
+Input [5]: [cs_item_sk#19, cs_ext_list_price#21, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28]
+Keys [1]: [cs_item_sk#19]
+Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#21)), partial_sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))]
+Aggregate Attributes [3]: [sum#31, sum#32, isEmpty#33]
+Results [4]: [cs_item_sk#19, sum#34, sum#35, isEmpty#36]
+
+(28) Exchange
+Input [4]: [cs_item_sk#19, sum#34, sum#35, isEmpty#36]
+Arguments: hashpartitioning(cs_item_sk#19, 5), ENSURE_REQUIREMENTS, [id=#37]
+
+(29) HashAggregate [codegen id : 9]
+Input [4]: [cs_item_sk#19, sum#34, sum#35, isEmpty#36]
+Keys [1]: [cs_item_sk#19]
+Functions [2]: [sum(UnscaledValue(cs_ext_list_price#21)), sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))]
+Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#21))#38, sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))#39]
+Results [3]: [cs_item_sk#19, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#21))#38,17,2) AS sum(cs_ext_list_price#21)#40, sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))#39 AS sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))#41]
+
+(30) Filter [codegen id : 9]
+Input [3]: [cs_item_sk#19, sum(cs_ext_list_price#21)#40, sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))#41]
+Condition : (isnotnull(sum(cs_ext_list_price#21)#40) AND (cast(sum(cs_ext_list_price#21)#40 as decimal(21,2)) > CheckOverflow((2.00 * promote_precision(sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))#41)), DecimalType(21,2), true)))
+
+(31) Project [codegen id : 9]
+Output [1]: [cs_item_sk#19]
+Input [3]: [cs_item_sk#19, sum(cs_ext_list_price#21)#40, sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))#41]
+
+(32) Sort [codegen id : 9]
+Input [1]: [cs_item_sk#19]
+Arguments: [cs_item_sk#19 ASC NULLS FIRST], false, 0
+
+(33) SortMergeJoin [codegen id : 25]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [cs_item_sk#19]
+Join condition: None
+
+(34) Project [codegen id : 25]
+Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#19]
+
+(35) Scan parquet default.date_dim
+Output [2]: [d_date_sk#42, d_year#43]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(36) ColumnarToRow [codegen id : 10]
+Input [2]: [d_date_sk#42, d_year#43]
+
+(37) Filter [codegen id : 10]
+Input [2]: [d_date_sk#42, d_year#43]
+Condition : ((isnotnull(d_year#43) AND (d_year#43 = 1999)) AND isnotnull(d_date_sk#42))
+
+(38) BroadcastExchange
+Input [2]: [d_date_sk#42, d_year#43]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#44]
+
+(39) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_sold_date_sk#12]
+Right keys [1]: [d_date_sk#42]
+Join condition: None
+
+(40) Project [codegen id : 25]
+Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43]
+Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#42, d_year#43]
+
+(41) Scan parquet default.store
+Output [3]: [s_store_sk#45, s_store_name#46, s_zip#47]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_zip)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_zip:string>
+
+(42) ColumnarToRow [codegen id : 11]
+Input [3]: [s_store_sk#45, s_store_name#46, s_zip#47]
+
+(43) Filter [codegen id : 11]
+Input [3]: [s_store_sk#45, s_store_name#46, s_zip#47]
+Condition : ((isnotnull(s_store_sk#45) AND isnotnull(s_store_name#46)) AND isnotnull(s_zip#47))
+
+(44) BroadcastExchange
+Input [3]: [s_store_sk#45, s_store_name#46, s_zip#47]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#48]
+
+(45) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_store_sk#6]
+Right keys [1]: [s_store_sk#45]
+Join condition: None
+
+(46) Project [codegen id : 25]
+Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47]
+Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_sk#45, s_store_name#46, s_zip#47]
+
+(47) Scan parquet default.customer
+Output [6]: [c_customer_sk#49, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, c_first_shipto_date_sk#53, c_first_sales_date_sk#54]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_hdemo_sk:int,c_current_addr_sk:int,c_first_shipto_date_sk:int,c_first_sales_date_sk:int>
+
+(48) ColumnarToRow [codegen id : 12]
+Input [6]: [c_customer_sk#49, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, c_first_shipto_date_sk#53, c_first_sales_date_sk#54]
+
+(49) Filter [codegen id : 12]
+Input [6]: [c_customer_sk#49, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, c_first_shipto_date_sk#53, c_first_sales_date_sk#54]
+Condition : (((((isnotnull(c_customer_sk#49) AND isnotnull(c_first_sales_date_sk#54)) AND isnotnull(c_first_shipto_date_sk#53)) AND isnotnull(c_current_cdemo_sk#50)) AND isnotnull(c_current_hdemo_sk#51)) AND isnotnull(c_current_addr_sk#52))
+
+(50) BroadcastExchange
+Input [6]: [c_customer_sk#49, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, c_first_shipto_date_sk#53, c_first_sales_date_sk#54]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#55]
+
+(51) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_customer_sk#2]
+Right keys [1]: [c_customer_sk#49]
+Join condition: None
+
+(52) Project [codegen id : 25]
+Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, c_first_shipto_date_sk#53, c_first_sales_date_sk#54]
+Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_customer_sk#49, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, c_first_shipto_date_sk#53, c_first_sales_date_sk#54]
+
+(53) Scan parquet default.date_dim
+Output [2]: [d_date_sk#56, d_year#57]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(54) ColumnarToRow [codegen id : 13]
+Input [2]: [d_date_sk#56, d_year#57]
+
+(55) Filter [codegen id : 13]
+Input [2]: [d_date_sk#56, d_year#57]
+Condition : isnotnull(d_date_sk#56)
+
+(56) BroadcastExchange
+Input [2]: [d_date_sk#56, d_year#57]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#58]
+
+(57) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [c_first_sales_date_sk#54]
+Right keys [1]: [d_date_sk#56]
+Join condition: None
+
+(58) Project [codegen id : 25]
+Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, c_first_shipto_date_sk#53, d_year#57]
+Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, c_first_shipto_date_sk#53, c_first_sales_date_sk#54, d_date_sk#56, d_year#57]
+
+(59) ReusedExchange [Reuses operator id: 56]
+Output [2]: [d_date_sk#59, d_year#60]
+
+(60) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [c_first_shipto_date_sk#53]
+Right keys [1]: [d_date_sk#59]
+Join condition: None
+
+(61) Project [codegen id : 25]
+Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, d_year#57, d_year#60]
+Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, c_first_shipto_date_sk#53, d_year#57, d_date_sk#59, d_year#60]
+
+(62) Scan parquet default.customer_demographics
+Output [2]: [cd_demo_sk#61, cd_marital_status#62]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status)]
+ReadSchema: struct<cd_demo_sk:int,cd_marital_status:string>
+
+(63) ColumnarToRow [codegen id : 15]
+Input [2]: [cd_demo_sk#61, cd_marital_status#62]
+
+(64) Filter [codegen id : 15]
+Input [2]: [cd_demo_sk#61, cd_marital_status#62]
+Condition : (isnotnull(cd_demo_sk#61) AND isnotnull(cd_marital_status#62))
+
+(65) BroadcastExchange
+Input [2]: [cd_demo_sk#61, cd_marital_status#62]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#63]
+
+(66) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_cdemo_sk#3]
+Right keys [1]: [cd_demo_sk#61]
+Join condition: None
+
+(67) Project [codegen id : 25]
+Output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, d_year#57, d_year#60, cd_marital_status#62]
+Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, d_year#57, d_year#60, cd_demo_sk#61, cd_marital_status#62]
+
+(68) ReusedExchange [Reuses operator id: 65]
+Output [2]: [cd_demo_sk#64, cd_marital_status#65]
+
+(69) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [c_current_cdemo_sk#50]
+Right keys [1]: [cd_demo_sk#64]
+Join condition: NOT (cd_marital_status#62 = cd_marital_status#65)
+
+(70) Project [codegen id : 25]
+Output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_hdemo_sk#51, c_current_addr_sk#52, d_year#57, d_year#60]
+Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, d_year#57, d_year#60, cd_marital_status#62, cd_demo_sk#64, cd_marital_status#65]
+
+(71) Scan parquet default.promotion
+Output [1]: [p_promo_sk#66]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/promotion]
+PushedFilters: [IsNotNull(p_promo_sk)]
+ReadSchema: struct<p_promo_sk:int>
+
+(72) ColumnarToRow [codegen id : 17]
+Input [1]: [p_promo_sk#66]
+
+(73) Filter [codegen id : 17]
+Input [1]: [p_promo_sk#66]
+Condition : isnotnull(p_promo_sk#66)
+
+(74) BroadcastExchange
+Input [1]: [p_promo_sk#66]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#67]
+
+(75) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_promo_sk#7]
+Right keys [1]: [p_promo_sk#66]
+Join condition: None
+
+(76) Project [codegen id : 25]
+Output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_hdemo_sk#51, c_current_addr_sk#52, d_year#57, d_year#60]
+Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_hdemo_sk#51, c_current_addr_sk#52, d_year#57, d_year#60, p_promo_sk#66]
+
+(77) Scan parquet default.household_demographics
+Output [2]: [hd_demo_sk#68, hd_income_band_sk#69]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_income_band_sk:int>
+
+(78) ColumnarToRow [codegen id : 18]
+Input [2]: [hd_demo_sk#68, hd_income_band_sk#69]
+
+(79) Filter [codegen id : 18]
+Input [2]: [hd_demo_sk#68, hd_income_band_sk#69]
+Condition : (isnotnull(hd_demo_sk#68) AND isnotnull(hd_income_band_sk#69))
+
+(80) BroadcastExchange
+Input [2]: [hd_demo_sk#68, hd_income_band_sk#69]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#70]
+
+(81) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_hdemo_sk#4]
+Right keys [1]: [hd_demo_sk#68]
+Join condition: None
+
+(82) Project [codegen id : 25]
+Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_hdemo_sk#51, c_current_addr_sk#52, d_year#57, d_year#60, hd_income_band_sk#69]
+Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_hdemo_sk#51, c_current_addr_sk#52, d_year#57, d_year#60, hd_demo_sk#68, hd_income_band_sk#69]
+
+(83) ReusedExchange [Reuses operator id: 80]
+Output [2]: [hd_demo_sk#71, hd_income_band_sk#72]
+
+(84) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [c_current_hdemo_sk#51]
+Right keys [1]: [hd_demo_sk#71]
+Join condition: None
+
+(85) Project [codegen id : 25]
+Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_addr_sk#52, d_year#57, d_year#60, hd_income_band_sk#69, hd_income_band_sk#72]
+Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_hdemo_sk#51, c_current_addr_sk#52, d_year#57, d_year#60, hd_income_band_sk#69, hd_demo_sk#71, hd_income_band_sk#72]
+
+(86) Scan parquet default.customer_address
+Output [5]: [ca_address_sk#73, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_street_number:string,ca_street_name:string,ca_city:string,ca_zip:string>
+
+(87) ColumnarToRow [codegen id : 20]
+Input [5]: [ca_address_sk#73, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77]
+
+(88) Filter [codegen id : 20]
+Input [5]: [ca_address_sk#73, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77]
+Condition : isnotnull(ca_address_sk#73)
+
+(89) BroadcastExchange
+Input [5]: [ca_address_sk#73, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#78]
+
+(90) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_addr_sk#5]
+Right keys [1]: [ca_address_sk#73]
+Join condition: None
+
+(91) Project [codegen id : 25]
+Output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_addr_sk#52, d_year#57, d_year#60, hd_income_band_sk#69, hd_income_band_sk#72, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77]
+Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_addr_sk#52, d_year#57, d_year#60, hd_income_band_sk#69, hd_income_band_sk#72, ca_address_sk#73, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77]
+
+(92) ReusedExchange [Reuses operator id: 89]
+Output [5]: [ca_address_sk#79, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83]
+
+(93) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [c_current_addr_sk#52]
+Right keys [1]: [ca_address_sk#79]
+Join condition: None
+
+(94) Project [codegen id : 25]
+Output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, d_year#57, d_year#60, hd_income_band_sk#69, hd_income_band_sk#72, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83]
+Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_addr_sk#52, d_year#57, d_year#60, hd_income_band_sk#69, hd_income_band_sk#72, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_address_sk#79, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83]
+
+(95) Scan parquet default.income_band
+Output [1]: [ib_income_band_sk#84]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/income_band]
+PushedFilters: [IsNotNull(ib_income_band_sk)]
+ReadSchema: struct<ib_income_band_sk:int>
+
+(96) ColumnarToRow [codegen id : 22]
+Input [1]: [ib_income_band_sk#84]
+
+(97) Filter [codegen id : 22]
+Input [1]: [ib_income_band_sk#84]
+Condition : isnotnull(ib_income_band_sk#84)
+
+(98) BroadcastExchange
+Input [1]: [ib_income_band_sk#84]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#85]
+
+(99) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [hd_income_band_sk#69]
+Right keys [1]: [ib_income_band_sk#84]
+Join condition: None
+
+(100) Project [codegen id : 25]
+Output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, d_year#57, d_year#60, hd_income_band_sk#72, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83]
+Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, d_year#57, d_year#60, hd_income_band_sk#69, hd_income_band_sk#72, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83, ib_income_band_sk#84]
+
+(101) ReusedExchange [Reuses operator id: 98]
+Output [1]: [ib_income_band_sk#86]
+
+(102) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [hd_income_band_sk#72]
+Right keys [1]: [ib_income_band_sk#86]
+Join condition: None
+
+(103) Project [codegen id : 25]
+Output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, d_year#57, d_year#60, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83]
+Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, d_year#57, d_year#60, hd_income_band_sk#72, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83, ib_income_band_sk#86]
+
+(104) Scan parquet default.item
+Output [4]: [i_item_sk#87, i_current_price#88, i_color#89, i_product_name#90]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_current_price), In(i_color, [purple              ,burlywood           ,indian              ,spring              ,floral              ,medium              ]), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_current_price:decimal(7,2),i_color:string,i_product_name:string>
+
+(105) ColumnarToRow [codegen id : 24]
+Input [4]: [i_item_sk#87, i_current_price#88, i_color#89, i_product_name#90]
+
+(106) Filter [codegen id : 24]
+Input [4]: [i_item_sk#87, i_current_price#88, i_color#89, i_product_name#90]
+Condition : ((((((isnotnull(i_current_price#88) AND i_color#89 IN (purple              ,burlywood           ,indian              ,spring              ,floral              ,medium              )) AND (i_current_price#88 >= 64.00)) AND (i_current_price#88 <= 74.00)) AND (i_current_price#88 >= 65.00)) AND (i_current_price#88 <= 79.00)) AND isnotnull(i_item_sk#87))
+
+(107) Project [codegen id : 24]
+Output [2]: [i_item_sk#87, i_product_name#90]
+Input [4]: [i_item_sk#87, i_current_price#88, i_color#89, i_product_name#90]
+
+(108) BroadcastExchange
+Input [2]: [i_item_sk#87, i_product_name#90]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#91]
+
+(109) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#87]
+Join condition: None
+
+(110) Project [codegen id : 25]
+Output [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, d_year#57, d_year#60, s_store_name#46, s_zip#47, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83, i_item_sk#87, i_product_name#90]
+Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, d_year#57, d_year#60, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83, i_item_sk#87, i_product_name#90]
+
+(111) HashAggregate [codegen id : 25]
+Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, d_year#57, d_year#60, s_store_name#46, s_zip#47, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83, i_item_sk#87, i_product_name#90]
+Keys [15]: [i_product_name#90, i_item_sk#87, s_store_name#46, s_zip#47, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83, d_year#43, d_year#57, d_year#60]
+Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))]
+Aggregate Attributes [4]: [count#92, sum#93, sum#94, sum#95]
+Results [19]: [i_product_name#90, i_item_sk#87, s_store_name#46, s_zip#47, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83, d_year#43, d_year#57, d_year#60, count#96, sum#97, sum#98, sum#99]
+
+(112) HashAggregate [codegen id : 25]
+Input [19]: [i_product_name#90, i_item_sk#87, s_store_name#46, s_zip#47, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83, d_year#43, d_year#57, d_year#60, count#96, sum#97, sum#98, sum#99]
+Keys [15]: [i_product_name#90, i_item_sk#87, s_store_name#46, s_zip#47, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83, d_year#43, d_year#57, d_year#60]
+Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))]
+Aggregate Attributes [4]: [count(1)#100, sum(UnscaledValue(ss_wholesale_cost#9))#101, sum(UnscaledValue(ss_list_price#10))#102, sum(UnscaledValue(ss_coupon_amt#11))#103]
+Results [17]: [i_product_name#90 AS product_name#104, i_item_sk#87 AS item_sk#105, s_store_name#46 AS store_name#106, s_zip#47 AS store_zip#107, ca_street_number#74 AS b_street_number#108, ca_street_name#75 AS b_streen_name#109, ca_city#76 AS b_city#110, ca_zip#77 AS b_zip#111, ca_street_number#80 AS c_street_number#112, ca_street_name#81 AS c_street_name#113, ca_city#82 AS c_city#114, ca_zip#83 AS c_zip#115, d_year#43 AS syear#116, count(1)#100 AS cnt#117, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#9))#101,17,2) AS s1#118, MakeDecimal(sum(UnscaledValue(ss_list_price#10))#102,17,2) AS s2#119, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#11))#103,17,2) AS s3#120]
+
+(113) Exchange
+Input [17]: [product_name#104, item_sk#105, store_name#106, store_zip#107, b_street_number#108, b_streen_name#109, b_city#110, b_zip#111, c_street_number#112, c_street_name#113, c_city#114, c_zip#115, syear#116, cnt#117, s1#118, s2#119, s3#120]
+Arguments: hashpartitioning(item_sk#105, store_name#106, store_zip#107, 5), ENSURE_REQUIREMENTS, [id=#121]
+
+(114) Sort [codegen id : 26]
+Input [17]: [product_name#104, item_sk#105, store_name#106, store_zip#107, b_street_number#108, b_streen_name#109, b_city#110, b_zip#111, c_street_number#112, c_street_name#113, c_city#114, c_zip#115, syear#116, cnt#117, s1#118, s2#119, s3#120]
+Arguments: [item_sk#105 ASC NULLS FIRST, store_name#106 ASC NULLS FIRST, store_zip#107 ASC NULLS FIRST], false, 0
+
+(115) Scan parquet default.store_sales
+Output [12]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_ticket_number#129, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, ss_sold_date_sk#133]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#133), dynamicpruningexpression(ss_sold_date_sk#133 IN dynamicpruning#134)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_cdemo_sk:int,ss_hdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_promo_sk:int,ss_ticket_number:int,ss_wholesale_cost:decimal(7,2),ss_list_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(116) ColumnarToRow [codegen id : 27]
+Input [12]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_ticket_number#129, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, ss_sold_date_sk#133]
+
+(117) Filter [codegen id : 27]
+Input [12]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_ticket_number#129, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, ss_sold_date_sk#133]
+Condition : (((((((isnotnull(ss_item_sk#122) AND isnotnull(ss_ticket_number#129)) AND isnotnull(ss_store_sk#127)) AND isnotnull(ss_customer_sk#123)) AND isnotnull(ss_cdemo_sk#124)) AND isnotnull(ss_promo_sk#128)) AND isnotnull(ss_hdemo_sk#125)) AND isnotnull(ss_addr_sk#126))
+
+(118) BroadcastExchange
+Input [12]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_ticket_number#129, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, ss_sold_date_sk#133]
+Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[7, int, false] as bigint) & 4294967295))),false), [id=#135]
+
+(119) Scan parquet default.store_returns
+Output [3]: [sr_item_sk#136, sr_ticket_number#137, sr_returned_date_sk#138]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int>
+
+(120) ColumnarToRow
+Input [3]: [sr_item_sk#136, sr_ticket_number#137, sr_returned_date_sk#138]
+
+(121) Filter
+Input [3]: [sr_item_sk#136, sr_ticket_number#137, sr_returned_date_sk#138]
+Condition : (isnotnull(sr_item_sk#136) AND isnotnull(sr_ticket_number#137))
+
+(122) Project
+Output [2]: [sr_item_sk#136, sr_ticket_number#137]
+Input [3]: [sr_item_sk#136, sr_ticket_number#137, sr_returned_date_sk#138]
+
+(123) BroadcastHashJoin [codegen id : 28]
+Left keys [2]: [ss_item_sk#122, ss_ticket_number#129]
+Right keys [2]: [sr_item_sk#136, sr_ticket_number#137]
+Join condition: None
+
+(124) Project [codegen id : 28]
+Output [11]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, ss_sold_date_sk#133]
+Input [14]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_ticket_number#129, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, ss_sold_date_sk#133, sr_item_sk#136, sr_ticket_number#137]
+
+(125) Exchange
+Input [11]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, ss_sold_date_sk#133]
+Arguments: hashpartitioning(ss_item_sk#122, 5), ENSURE_REQUIREMENTS, [id=#139]
+
+(126) Sort [codegen id : 29]
+Input [11]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, ss_sold_date_sk#133]
+Arguments: [ss_item_sk#122 ASC NULLS FIRST], false, 0
+
+(127) ReusedExchange [Reuses operator id: 28]
+Output [4]: [cs_item_sk#140, sum#141, sum#142, isEmpty#143]
+
+(128) HashAggregate [codegen id : 35]
+Input [4]: [cs_item_sk#140, sum#141, sum#142, isEmpty#143]
+Keys [1]: [cs_item_sk#140]
+Functions [2]: [sum(UnscaledValue(cs_ext_list_price#144)), sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#145 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#146 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#147 as decimal(9,2)))), DecimalType(9,2), true))]
+Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#144))#148, sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#145 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#146 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#147 as decimal(9,2)))), DecimalType(9,2), true))#149]
+Results [3]: [cs_item_sk#140, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#144))#148,17,2) AS sum(cs_ext_list_price#144)#150, sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#145 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#146 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#147 as decimal(9,2)))), DecimalType(9,2), true))#149 AS sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#145 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#146 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#147 as decimal(9,2)))), DecimalType(9,2), true))#151]
+
+(129) Filter [codegen id : 35]
+Input [3]: [cs_item_sk#140, sum(cs_ext_list_price#144)#150, sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#145 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#146 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#147 as decimal(9,2)))), DecimalType(9,2), true))#151]
+Condition : (isnotnull(sum(cs_ext_list_price#144)#150) AND (cast(sum(cs_ext_list_price#144)#150 as decimal(21,2)) > CheckOverflow((2.00 * promote_precision(sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#145 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#146 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#147 as decimal(9,2)))), DecimalType(9,2), true))#151)), DecimalType(21,2), true)))
+
+(130) Project [codegen id : 35]
+Output [1]: [cs_item_sk#140]
+Input [3]: [cs_item_sk#140, sum(cs_ext_list_price#144)#150, sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#145 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#146 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#147 as decimal(9,2)))), DecimalType(9,2), true))#151]
+
+(131) Sort [codegen id : 35]
+Input [1]: [cs_item_sk#140]
+Arguments: [cs_item_sk#140 ASC NULLS FIRST], false, 0
+
+(132) SortMergeJoin [codegen id : 51]
+Left keys [1]: [ss_item_sk#122]
+Right keys [1]: [cs_item_sk#140]
+Join condition: None
+
+(133) Project [codegen id : 51]
+Output [11]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, ss_sold_date_sk#133]
+Input [12]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, ss_sold_date_sk#133, cs_item_sk#140]
+
+(134) Scan parquet default.date_dim
+Output [2]: [d_date_sk#152, d_year#153]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(135) ColumnarToRow [codegen id : 36]
+Input [2]: [d_date_sk#152, d_year#153]
+
+(136) Filter [codegen id : 36]
+Input [2]: [d_date_sk#152, d_year#153]
+Condition : ((isnotnull(d_year#153) AND (d_year#153 = 2000)) AND isnotnull(d_date_sk#152))
+
+(137) BroadcastExchange
+Input [2]: [d_date_sk#152, d_year#153]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#154]
+
+(138) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [ss_sold_date_sk#133]
+Right keys [1]: [d_date_sk#152]
+Join condition: None
+
+(139) Project [codegen id : 51]
+Output [11]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153]
+Input [13]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, ss_sold_date_sk#133, d_date_sk#152, d_year#153]
+
+(140) ReusedExchange [Reuses operator id: 44]
+Output [3]: [s_store_sk#155, s_store_name#156, s_zip#157]
+
+(141) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [ss_store_sk#127]
+Right keys [1]: [s_store_sk#155]
+Join condition: None
+
+(142) Project [codegen id : 51]
+Output [12]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157]
+Input [14]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_sk#155, s_store_name#156, s_zip#157]
+
+(143) ReusedExchange [Reuses operator id: 50]
+Output [6]: [c_customer_sk#158, c_current_cdemo_sk#159, c_current_hdemo_sk#160, c_current_addr_sk#161, c_first_shipto_date_sk#162, c_first_sales_date_sk#163]
+
+(144) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [ss_customer_sk#123]
+Right keys [1]: [c_customer_sk#158]
+Join condition: None
+
+(145) Project [codegen id : 51]
+Output [16]: [ss_item_sk#122, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_cdemo_sk#159, c_current_hdemo_sk#160, c_current_addr_sk#161, c_first_shipto_date_sk#162, c_first_sales_date_sk#163]
+Input [18]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_customer_sk#158, c_current_cdemo_sk#159, c_current_hdemo_sk#160, c_current_addr_sk#161, c_first_shipto_date_sk#162, c_first_sales_date_sk#163]
+
+(146) ReusedExchange [Reuses operator id: 56]
+Output [2]: [d_date_sk#164, d_year#165]
+
+(147) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [c_first_sales_date_sk#163]
+Right keys [1]: [d_date_sk#164]
+Join condition: None
+
+(148) Project [codegen id : 51]
+Output [16]: [ss_item_sk#122, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_cdemo_sk#159, c_current_hdemo_sk#160, c_current_addr_sk#161, c_first_shipto_date_sk#162, d_year#165]
+Input [18]: [ss_item_sk#122, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_cdemo_sk#159, c_current_hdemo_sk#160, c_current_addr_sk#161, c_first_shipto_date_sk#162, c_first_sales_date_sk#163, d_date_sk#164, d_year#165]
+
+(149) ReusedExchange [Reuses operator id: 56]
+Output [2]: [d_date_sk#166, d_year#167]
+
+(150) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [c_first_shipto_date_sk#162]
+Right keys [1]: [d_date_sk#166]
+Join condition: None
+
+(151) Project [codegen id : 51]
+Output [16]: [ss_item_sk#122, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_cdemo_sk#159, c_current_hdemo_sk#160, c_current_addr_sk#161, d_year#165, d_year#167]
+Input [18]: [ss_item_sk#122, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_cdemo_sk#159, c_current_hdemo_sk#160, c_current_addr_sk#161, c_first_shipto_date_sk#162, d_year#165, d_date_sk#166, d_year#167]
+
+(152) ReusedExchange [Reuses operator id: 65]
+Output [2]: [cd_demo_sk#168, cd_marital_status#169]
+
+(153) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [ss_cdemo_sk#124]
+Right keys [1]: [cd_demo_sk#168]
+Join condition: None
+
+(154) Project [codegen id : 51]
+Output [16]: [ss_item_sk#122, ss_hdemo_sk#125, ss_addr_sk#126, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_cdemo_sk#159, c_current_hdemo_sk#160, c_current_addr_sk#161, d_year#165, d_year#167, cd_marital_status#169]
+Input [18]: [ss_item_sk#122, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_cdemo_sk#159, c_current_hdemo_sk#160, c_current_addr_sk#161, d_year#165, d_year#167, cd_demo_sk#168, cd_marital_status#169]
+
+(155) ReusedExchange [Reuses operator id: 65]
+Output [2]: [cd_demo_sk#170, cd_marital_status#171]
+
+(156) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [c_current_cdemo_sk#159]
+Right keys [1]: [cd_demo_sk#170]
+Join condition: NOT (cd_marital_status#169 = cd_marital_status#171)
+
+(157) Project [codegen id : 51]
+Output [14]: [ss_item_sk#122, ss_hdemo_sk#125, ss_addr_sk#126, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_hdemo_sk#160, c_current_addr_sk#161, d_year#165, d_year#167]
+Input [18]: [ss_item_sk#122, ss_hdemo_sk#125, ss_addr_sk#126, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_cdemo_sk#159, c_current_hdemo_sk#160, c_current_addr_sk#161, d_year#165, d_year#167, cd_marital_status#169, cd_demo_sk#170, cd_marital_status#171]
+
+(158) ReusedExchange [Reuses operator id: 74]
+Output [1]: [p_promo_sk#172]
+
+(159) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [ss_promo_sk#128]
+Right keys [1]: [p_promo_sk#172]
+Join condition: None
+
+(160) Project [codegen id : 51]
+Output [13]: [ss_item_sk#122, ss_hdemo_sk#125, ss_addr_sk#126, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_hdemo_sk#160, c_current_addr_sk#161, d_year#165, d_year#167]
+Input [15]: [ss_item_sk#122, ss_hdemo_sk#125, ss_addr_sk#126, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_hdemo_sk#160, c_current_addr_sk#161, d_year#165, d_year#167, p_promo_sk#172]
+
+(161) ReusedExchange [Reuses operator id: 80]
+Output [2]: [hd_demo_sk#173, hd_income_band_sk#174]
+
+(162) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [ss_hdemo_sk#125]
+Right keys [1]: [hd_demo_sk#173]
+Join condition: None
+
+(163) Project [codegen id : 51]
+Output [13]: [ss_item_sk#122, ss_addr_sk#126, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_hdemo_sk#160, c_current_addr_sk#161, d_year#165, d_year#167, hd_income_band_sk#174]
+Input [15]: [ss_item_sk#122, ss_hdemo_sk#125, ss_addr_sk#126, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_hdemo_sk#160, c_current_addr_sk#161, d_year#165, d_year#167, hd_demo_sk#173, hd_income_band_sk#174]
+
+(164) ReusedExchange [Reuses operator id: 80]
+Output [2]: [hd_demo_sk#175, hd_income_band_sk#176]
+
+(165) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [c_current_hdemo_sk#160]
+Right keys [1]: [hd_demo_sk#175]
+Join condition: None
+
+(166) Project [codegen id : 51]
+Output [13]: [ss_item_sk#122, ss_addr_sk#126, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_addr_sk#161, d_year#165, d_year#167, hd_income_band_sk#174, hd_income_band_sk#176]
+Input [15]: [ss_item_sk#122, ss_addr_sk#126, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_hdemo_sk#160, c_current_addr_sk#161, d_year#165, d_year#167, hd_income_band_sk#174, hd_demo_sk#175, hd_income_band_sk#176]
+
+(167) ReusedExchange [Reuses operator id: 89]
+Output [5]: [ca_address_sk#177, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181]
+
+(168) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [ss_addr_sk#126]
+Right keys [1]: [ca_address_sk#177]
+Join condition: None
+
+(169) Project [codegen id : 51]
+Output [16]: [ss_item_sk#122, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_addr_sk#161, d_year#165, d_year#167, hd_income_band_sk#174, hd_income_band_sk#176, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181]
+Input [18]: [ss_item_sk#122, ss_addr_sk#126, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_addr_sk#161, d_year#165, d_year#167, hd_income_band_sk#174, hd_income_band_sk#176, ca_address_sk#177, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181]
+
+(170) ReusedExchange [Reuses operator id: 89]
+Output [5]: [ca_address_sk#182, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186]
+
+(171) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [c_current_addr_sk#161]
+Right keys [1]: [ca_address_sk#182]
+Join condition: None
+
+(172) Project [codegen id : 51]
+Output [19]: [ss_item_sk#122, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, d_year#165, d_year#167, hd_income_band_sk#174, hd_income_band_sk#176, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186]
+Input [21]: [ss_item_sk#122, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_addr_sk#161, d_year#165, d_year#167, hd_income_band_sk#174, hd_income_band_sk#176, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_address_sk#182, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186]
+
+(173) ReusedExchange [Reuses operator id: 98]
+Output [1]: [ib_income_band_sk#187]
+
+(174) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [hd_income_band_sk#174]
+Right keys [1]: [ib_income_band_sk#187]
+Join condition: None
+
+(175) Project [codegen id : 51]
+Output [18]: [ss_item_sk#122, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, d_year#165, d_year#167, hd_income_band_sk#176, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186]
+Input [20]: [ss_item_sk#122, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, d_year#165, d_year#167, hd_income_band_sk#174, hd_income_band_sk#176, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186, ib_income_band_sk#187]
+
+(176) ReusedExchange [Reuses operator id: 98]
+Output [1]: [ib_income_band_sk#188]
+
+(177) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [hd_income_band_sk#176]
+Right keys [1]: [ib_income_band_sk#188]
+Join condition: None
+
+(178) Project [codegen id : 51]
+Output [17]: [ss_item_sk#122, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, d_year#165, d_year#167, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186]
+Input [19]: [ss_item_sk#122, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, d_year#165, d_year#167, hd_income_band_sk#176, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186, ib_income_band_sk#188]
+
+(179) ReusedExchange [Reuses operator id: 108]
+Output [2]: [i_item_sk#189, i_product_name#190]
+
+(180) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [ss_item_sk#122]
+Right keys [1]: [i_item_sk#189]
+Join condition: None
+
+(181) Project [codegen id : 51]
+Output [18]: [ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, d_year#165, d_year#167, s_store_name#156, s_zip#157, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186, i_item_sk#189, i_product_name#190]
+Input [19]: [ss_item_sk#122, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, d_year#165, d_year#167, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186, i_item_sk#189, i_product_name#190]
+
+(182) HashAggregate [codegen id : 51]
+Input [18]: [ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, d_year#165, d_year#167, s_store_name#156, s_zip#157, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186, i_item_sk#189, i_product_name#190]
+Keys [15]: [i_product_name#190, i_item_sk#189, s_store_name#156, s_zip#157, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186, d_year#153, d_year#165, d_year#167]
+Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#130)), partial_sum(UnscaledValue(ss_list_price#131)), partial_sum(UnscaledValue(ss_coupon_amt#132))]
+Aggregate Attributes [4]: [count#191, sum#192, sum#193, sum#194]
+Results [19]: [i_product_name#190, i_item_sk#189, s_store_name#156, s_zip#157, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186, d_year#153, d_year#165, d_year#167, count#195, sum#196, sum#197, sum#198]
+
+(183) HashAggregate [codegen id : 51]
+Input [19]: [i_product_name#190, i_item_sk#189, s_store_name#156, s_zip#157, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186, d_year#153, d_year#165, d_year#167, count#195, sum#196, sum#197, sum#198]
+Keys [15]: [i_product_name#190, i_item_sk#189, s_store_name#156, s_zip#157, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186, d_year#153, d_year#165, d_year#167]
+Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#130)), sum(UnscaledValue(ss_list_price#131)), sum(UnscaledValue(ss_coupon_amt#132))]
+Aggregate Attributes [4]: [count(1)#199, sum(UnscaledValue(ss_wholesale_cost#130))#200, sum(UnscaledValue(ss_list_price#131))#201, sum(UnscaledValue(ss_coupon_amt#132))#202]
+Results [8]: [i_item_sk#189 AS item_sk#203, s_store_name#156 AS store_name#204, s_zip#157 AS store_zip#205, d_year#153 AS syear#206, count(1)#199 AS cnt#207, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#130))#200,17,2) AS s1#208, MakeDecimal(sum(UnscaledValue(ss_list_price#131))#201,17,2) AS s2#209, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#132))#202,17,2) AS s3#210]
+
+(184) Exchange
+Input [8]: [item_sk#203, store_name#204, store_zip#205, syear#206, cnt#207, s1#208, s2#209, s3#210]
+Arguments: hashpartitioning(item_sk#203, store_name#204, store_zip#205, 5), ENSURE_REQUIREMENTS, [id=#211]
+
+(185) Sort [codegen id : 52]
+Input [8]: [item_sk#203, store_name#204, store_zip#205, syear#206, cnt#207, s1#208, s2#209, s3#210]
+Arguments: [item_sk#203 ASC NULLS FIRST, store_name#204 ASC NULLS FIRST, store_zip#205 ASC NULLS FIRST], false, 0
+
+(186) SortMergeJoin [codegen id : 53]
+Left keys [3]: [item_sk#105, store_name#106, store_zip#107]
+Right keys [3]: [item_sk#203, store_name#204, store_zip#205]
+Join condition: (cnt#207 <= cnt#117)
+
+(187) Project [codegen id : 53]
+Output [21]: [product_name#104, store_name#106, store_zip#107, b_street_number#108, b_streen_name#109, b_city#110, b_zip#111, c_street_number#112, c_street_name#113, c_city#114, c_zip#115, syear#116, cnt#117, s1#118, s2#119, s3#120, s1#208, s2#209, s3#210, syear#206, cnt#207]
+Input [25]: [product_name#104, item_sk#105, store_name#106, store_zip#107, b_street_number#108, b_streen_name#109, b_city#110, b_zip#111, c_street_number#112, c_street_name#113, c_city#114, c_zip#115, syear#116, cnt#117, s1#118, s2#119, s3#120, item_sk#203, store_name#204, store_zip#205, syear#206, cnt#207, s1#208, s2#209, s3#210]
+
+(188) Exchange
+Input [21]: [product_name#104, store_name#106, store_zip#107, b_street_number#108, b_streen_name#109, b_city#110, b_zip#111, c_street_number#112, c_street_name#113, c_city#114, c_zip#115, syear#116, cnt#117, s1#118, s2#119, s3#120, s1#208, s2#209, s3#210, syear#206, cnt#207]
+Arguments: rangepartitioning(product_name#104 ASC NULLS FIRST, store_name#106 ASC NULLS FIRST, cnt#207 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [id=#212]
+
+(189) Sort [codegen id : 54]
+Input [21]: [product_name#104, store_name#106, store_zip#107, b_street_number#108, b_streen_name#109, b_city#110, b_zip#111, c_street_number#112, c_street_name#113, c_city#114, c_zip#115, syear#116, cnt#117, s1#118, s2#119, s3#120, s1#208, s2#209, s3#210, syear#206, cnt#207]
+Arguments: [product_name#104 ASC NULLS FIRST, store_name#106 ASC NULLS FIRST, cnt#207 ASC NULLS FIRST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13
+ReusedExchange (190)
+
+
+(190) ReusedExchange [Reuses operator id: 38]
+Output [2]: [d_date_sk#42, d_year#43]
+
+Subquery:2 Hosting operator id = 115 Hosting Expression = ss_sold_date_sk#133 IN dynamicpruning#134
+ReusedExchange (191)
+
+
+(191) ReusedExchange [Reuses operator id: 137]
+Output [2]: [d_date_sk#152, d_year#153]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/simplified.txt
new file mode 100644
index 0000000000000..4a0006b2db5ca
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/simplified.txt
@@ -0,0 +1,289 @@
+WholeStageCodegen (54)
+  Sort [product_name,store_name,cnt]
+    InputAdapter
+      Exchange [product_name,store_name,cnt] #1
+        WholeStageCodegen (53)
+          Project [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt]
+            SortMergeJoin [item_sk,store_name,store_zip,item_sk,store_name,store_zip,cnt,cnt]
+              InputAdapter
+                WholeStageCodegen (26)
+                  Sort [item_sk,store_name,store_zip]
+                    InputAdapter
+                      Exchange [item_sk,store_name,store_zip] #2
+                        WholeStageCodegen (25)
+                          HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum]
+                            HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum]
+                              Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name]
+                                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                  Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                    BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk]
+                                      Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                        BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk]
+                                          Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                            BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                                              Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                                BroadcastHashJoin [ss_addr_sk,ca_address_sk]
+                                                  Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk]
+                                                    BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk]
+                                                      Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk]
+                                                        BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                                          Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year]
+                                                            BroadcastHashJoin [ss_promo_sk,p_promo_sk]
+                                                              Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year]
+                                                                BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status]
+                                                                  Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status]
+                                                                    BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk]
+                                                                      Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year]
+                                                                        BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk]
+                                                                          Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year]
+                                                                            BroadcastHashJoin [c_first_sales_date_sk,d_date_sk]
+                                                                              Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk]
+                                                                                BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+                                                                                  Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip]
+                                                                                    BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                                                      Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year]
+                                                                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                          Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+                                                                                            SortMergeJoin [ss_item_sk,cs_item_sk]
+                                                                                              InputAdapter
+                                                                                                WholeStageCodegen (3)
+                                                                                                  Sort [ss_item_sk]
+                                                                                                    InputAdapter
+                                                                                                      Exchange [ss_item_sk] #3
+                                                                                                        WholeStageCodegen (2)
+                                                                                                          Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+                                                                                                            BroadcastHashJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number]
+                                                                                                              InputAdapter
+                                                                                                                BroadcastExchange #4
+                                                                                                                  WholeStageCodegen (1)
+                                                                                                                    Filter [ss_item_sk,ss_ticket_number,ss_store_sk,ss_customer_sk,ss_cdemo_sk,ss_promo_sk,ss_hdemo_sk,ss_addr_sk]
+                                                                                                                      ColumnarToRow
+                                                                                                                        InputAdapter
+                                                                                                                          Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+                                                                                                                            SubqueryBroadcast [d_date_sk] #1
+                                                                                                                              ReusedExchange [d_date_sk,d_year] #5
+                                                                                                              Project [sr_item_sk,sr_ticket_number]
+                                                                                                                Filter [sr_item_sk,sr_ticket_number]
+                                                                                                                  ColumnarToRow
+                                                                                                                    InputAdapter
+                                                                                                                      Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk]
+                                                                                              InputAdapter
+                                                                                                WholeStageCodegen (9)
+                                                                                                  Sort [cs_item_sk]
+                                                                                                    Project [cs_item_sk]
+                                                                                                      Filter [sum(cs_ext_list_price),sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash as decimal(8,2))) + promote_precision(cast(cr_reversed_charge as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit as decimal(9,2)))), DecimalType(9,2), true))]
+                                                                                                        HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash as decimal(8,2))) + promote_precision(cast(cr_reversed_charge as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit as decimal(9,2)))), DecimalType(9,2), true)),sum(cs_ext_list_price),sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash as decimal(8,2))) + promote_precision(cast(cr_reversed_charge as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit as decimal(9,2)))), DecimalType(9,2), true)),sum,sum,isEmpty]
+                                                                                                          InputAdapter
+                                                                                                            Exchange [cs_item_sk] #6
+                                                                                                              WholeStageCodegen (8)
+                                                                                                                HashAggregate [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [sum,sum,isEmpty,sum,sum,isEmpty]
+                                                                                                                  Project [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit]
+                                                                                                                    SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number]
+                                                                                                                      InputAdapter
+                                                                                                                        WholeStageCodegen (5)
+                                                                                                                          Sort [cs_item_sk,cs_order_number]
+                                                                                                                            InputAdapter
+                                                                                                                              Exchange [cs_item_sk,cs_order_number] #7
+                                                                                                                                WholeStageCodegen (4)
+                                                                                                                                  Project [cs_item_sk,cs_order_number,cs_ext_list_price]
+                                                                                                                                    Filter [cs_item_sk,cs_order_number]
+                                                                                                                                      ColumnarToRow
+                                                                                                                                        InputAdapter
+                                                                                                                                          Scan parquet default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk]
+                                                                                                                      InputAdapter
+                                                                                                                        WholeStageCodegen (7)
+                                                                                                                          Sort [cr_item_sk,cr_order_number]
+                                                                                                                            InputAdapter
+                                                                                                                              Exchange [cr_item_sk,cr_order_number] #8
+                                                                                                                                WholeStageCodegen (6)
+                                                                                                                                  Project [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit]
+                                                                                                                                    Filter [cr_item_sk,cr_order_number]
+                                                                                                                                      ColumnarToRow
+                                                                                                                                        InputAdapter
+                                                                                                                                          Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk]
+                                                                                          InputAdapter
+                                                                                            BroadcastExchange #5
+                                                                                              WholeStageCodegen (10)
+                                                                                                Filter [d_year,d_date_sk]
+                                                                                                  ColumnarToRow
+                                                                                                    InputAdapter
+                                                                                                      Scan parquet default.date_dim [d_date_sk,d_year]
+                                                                                      InputAdapter
+                                                                                        BroadcastExchange #9
+                                                                                          WholeStageCodegen (11)
+                                                                                            Filter [s_store_sk,s_store_name,s_zip]
+                                                                                              ColumnarToRow
+                                                                                                InputAdapter
+                                                                                                  Scan parquet default.store [s_store_sk,s_store_name,s_zip]
+                                                                                  InputAdapter
+                                                                                    BroadcastExchange #10
+                                                                                      WholeStageCodegen (12)
+                                                                                        Filter [c_customer_sk,c_first_sales_date_sk,c_first_shipto_date_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk]
+                                                                                          ColumnarToRow
+                                                                                            InputAdapter
+                                                                                              Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk]
+                                                                              InputAdapter
+                                                                                BroadcastExchange #11
+                                                                                  WholeStageCodegen (13)
+                                                                                    Filter [d_date_sk]
+                                                                                      ColumnarToRow
+                                                                                        InputAdapter
+                                                                                          Scan parquet default.date_dim [d_date_sk,d_year]
+                                                                          InputAdapter
+                                                                            ReusedExchange [d_date_sk,d_year] #11
+                                                                      InputAdapter
+                                                                        BroadcastExchange #12
+                                                                          WholeStageCodegen (15)
+                                                                            Filter [cd_demo_sk,cd_marital_status]
+                                                                              ColumnarToRow
+                                                                                InputAdapter
+                                                                                  Scan parquet default.customer_demographics [cd_demo_sk,cd_marital_status]
+                                                                  InputAdapter
+                                                                    ReusedExchange [cd_demo_sk,cd_marital_status] #12
+                                                              InputAdapter
+                                                                BroadcastExchange #13
+                                                                  WholeStageCodegen (17)
+                                                                    Filter [p_promo_sk]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.promotion [p_promo_sk]
+                                                          InputAdapter
+                                                            BroadcastExchange #14
+                                                              WholeStageCodegen (18)
+                                                                Filter [hd_demo_sk,hd_income_band_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.household_demographics [hd_demo_sk,hd_income_band_sk]
+                                                      InputAdapter
+                                                        ReusedExchange [hd_demo_sk,hd_income_band_sk] #14
+                                                  InputAdapter
+                                                    BroadcastExchange #15
+                                                      WholeStageCodegen (20)
+                                                        Filter [ca_address_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                              InputAdapter
+                                                ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15
+                                          InputAdapter
+                                            BroadcastExchange #16
+                                              WholeStageCodegen (22)
+                                                Filter [ib_income_band_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.income_band [ib_income_band_sk]
+                                      InputAdapter
+                                        ReusedExchange [ib_income_band_sk] #16
+                                  InputAdapter
+                                    BroadcastExchange #17
+                                      WholeStageCodegen (24)
+                                        Project [i_item_sk,i_product_name]
+                                          Filter [i_current_price,i_color,i_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.item [i_item_sk,i_current_price,i_color,i_product_name]
+              InputAdapter
+                WholeStageCodegen (52)
+                  Sort [item_sk,store_name,store_zip]
+                    InputAdapter
+                      Exchange [item_sk,store_name,store_zip] #18
+                        WholeStageCodegen (51)
+                          HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum]
+                            HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum]
+                              Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name]
+                                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                  Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                    BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk]
+                                      Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                        BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk]
+                                          Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                            BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                                              Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                                BroadcastHashJoin [ss_addr_sk,ca_address_sk]
+                                                  Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk]
+                                                    BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk]
+                                                      Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk]
+                                                        BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                                          Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year]
+                                                            BroadcastHashJoin [ss_promo_sk,p_promo_sk]
+                                                              Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year]
+                                                                BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status]
+                                                                  Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status]
+                                                                    BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk]
+                                                                      Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year]
+                                                                        BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk]
+                                                                          Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year]
+                                                                            BroadcastHashJoin [c_first_sales_date_sk,d_date_sk]
+                                                                              Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk]
+                                                                                BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+                                                                                  Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip]
+                                                                                    BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                                                      Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year]
+                                                                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                          Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+                                                                                            SortMergeJoin [ss_item_sk,cs_item_sk]
+                                                                                              InputAdapter
+                                                                                                WholeStageCodegen (29)
+                                                                                                  Sort [ss_item_sk]
+                                                                                                    InputAdapter
+                                                                                                      Exchange [ss_item_sk] #19
+                                                                                                        WholeStageCodegen (28)
+                                                                                                          Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+                                                                                                            BroadcastHashJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number]
+                                                                                                              InputAdapter
+                                                                                                                BroadcastExchange #20
+                                                                                                                  WholeStageCodegen (27)
+                                                                                                                    Filter [ss_item_sk,ss_ticket_number,ss_store_sk,ss_customer_sk,ss_cdemo_sk,ss_promo_sk,ss_hdemo_sk,ss_addr_sk]
+                                                                                                                      ColumnarToRow
+                                                                                                                        InputAdapter
+                                                                                                                          Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+                                                                                                                            SubqueryBroadcast [d_date_sk] #2
+                                                                                                                              ReusedExchange [d_date_sk,d_year] #21
+                                                                                                              Project [sr_item_sk,sr_ticket_number]
+                                                                                                                Filter [sr_item_sk,sr_ticket_number]
+                                                                                                                  ColumnarToRow
+                                                                                                                    InputAdapter
+                                                                                                                      Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk]
+                                                                                              InputAdapter
+                                                                                                WholeStageCodegen (35)
+                                                                                                  Sort [cs_item_sk]
+                                                                                                    Project [cs_item_sk]
+                                                                                                      Filter [sum(cs_ext_list_price),sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash as decimal(8,2))) + promote_precision(cast(cr_reversed_charge as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit as decimal(9,2)))), DecimalType(9,2), true))]
+                                                                                                        HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash as decimal(8,2))) + promote_precision(cast(cr_reversed_charge as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit as decimal(9,2)))), DecimalType(9,2), true)),sum(cs_ext_list_price),sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash as decimal(8,2))) + promote_precision(cast(cr_reversed_charge as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit as decimal(9,2)))), DecimalType(9,2), true)),sum,sum,isEmpty]
+                                                                                                          InputAdapter
+                                                                                                            ReusedExchange [cs_item_sk,sum,sum,isEmpty] #6
+                                                                                          InputAdapter
+                                                                                            BroadcastExchange #21
+                                                                                              WholeStageCodegen (36)
+                                                                                                Filter [d_year,d_date_sk]
+                                                                                                  ColumnarToRow
+                                                                                                    InputAdapter
+                                                                                                      Scan parquet default.date_dim [d_date_sk,d_year]
+                                                                                      InputAdapter
+                                                                                        ReusedExchange [s_store_sk,s_store_name,s_zip] #9
+                                                                                  InputAdapter
+                                                                                    ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10
+                                                                              InputAdapter
+                                                                                ReusedExchange [d_date_sk,d_year] #11
+                                                                          InputAdapter
+                                                                            ReusedExchange [d_date_sk,d_year] #11
+                                                                      InputAdapter
+                                                                        ReusedExchange [cd_demo_sk,cd_marital_status] #12
+                                                                  InputAdapter
+                                                                    ReusedExchange [cd_demo_sk,cd_marital_status] #12
+                                                              InputAdapter
+                                                                ReusedExchange [p_promo_sk] #13
+                                                          InputAdapter
+                                                            ReusedExchange [hd_demo_sk,hd_income_band_sk] #14
+                                                      InputAdapter
+                                                        ReusedExchange [hd_demo_sk,hd_income_band_sk] #14
+                                                  InputAdapter
+                                                    ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15
+                                              InputAdapter
+                                                ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15
+                                          InputAdapter
+                                            ReusedExchange [ib_income_band_sk] #16
+                                      InputAdapter
+                                        ReusedExchange [ib_income_band_sk] #16
+                                  InputAdapter
+                                    ReusedExchange [i_item_sk,i_product_name] #17
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.sf100/explain.txt
new file mode 100644
index 0000000000000..84e49ab9373e4
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.sf100/explain.txt
@@ -0,0 +1,274 @@
+== Physical Plan ==
+TakeOrderedAndProject (45)
++- * Project (44)
+   +- * SortMergeJoin Inner (43)
+      :- * Sort (37)
+      :  +- Exchange (36)
+      :     +- * Project (35)
+      :        +- * BroadcastHashJoin Inner BuildRight (34)
+      :           :- * Project (29)
+      :           :  +- * BroadcastHashJoin Inner BuildRight (28)
+      :           :     :- * Filter (14)
+      :           :     :  +- * HashAggregate (13)
+      :           :     :     +- Exchange (12)
+      :           :     :        +- * HashAggregate (11)
+      :           :     :           +- * Project (10)
+      :           :     :              +- * BroadcastHashJoin Inner BuildRight (9)
+      :           :     :                 :- * Filter (3)
+      :           :     :                 :  +- * ColumnarToRow (2)
+      :           :     :                 :     +- Scan parquet default.store_sales (1)
+      :           :     :                 +- BroadcastExchange (8)
+      :           :     :                    +- * Project (7)
+      :           :     :                       +- * Filter (6)
+      :           :     :                          +- * ColumnarToRow (5)
+      :           :     :                             +- Scan parquet default.date_dim (4)
+      :           :     +- BroadcastExchange (27)
+      :           :        +- * HashAggregate (26)
+      :           :           +- Exchange (25)
+      :           :              +- * HashAggregate (24)
+      :           :                 +- * HashAggregate (23)
+      :           :                    +- Exchange (22)
+      :           :                       +- * HashAggregate (21)
+      :           :                          +- * Project (20)
+      :           :                             +- * BroadcastHashJoin Inner BuildRight (19)
+      :           :                                :- * Filter (17)
+      :           :                                :  +- * ColumnarToRow (16)
+      :           :                                :     +- Scan parquet default.store_sales (15)
+      :           :                                +- ReusedExchange (18)
+      :           +- BroadcastExchange (33)
+      :              +- * Filter (32)
+      :                 +- * ColumnarToRow (31)
+      :                    +- Scan parquet default.store (30)
+      +- * Sort (42)
+         +- Exchange (41)
+            +- * Filter (40)
+               +- * ColumnarToRow (39)
+                  +- Scan parquet default.item (38)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 2]
+Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, ss_sold_date_sk#4]
+Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#6, d_month_seq#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1176), LessThanOrEqual(d_month_seq,1187), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#6, d_month_seq#7]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#6, d_month_seq#7]
+Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1176)) AND (d_month_seq#7 <= 1187)) AND isnotnull(d_date_sk#6))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#6]
+Input [2]: [d_date_sk#6, d_month_seq#7]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#8]
+
+(9) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 2]
+Output [3]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3]
+Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, ss_sold_date_sk#4, d_date_sk#6]
+
+(11) HashAggregate [codegen id : 2]
+Input [3]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3]
+Keys [2]: [ss_store_sk#2, ss_item_sk#1]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#3))]
+Aggregate Attributes [1]: [sum#9]
+Results [3]: [ss_store_sk#2, ss_item_sk#1, sum#10]
+
+(12) Exchange
+Input [3]: [ss_store_sk#2, ss_item_sk#1, sum#10]
+Arguments: hashpartitioning(ss_store_sk#2, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#11]
+
+(13) HashAggregate [codegen id : 8]
+Input [3]: [ss_store_sk#2, ss_item_sk#1, sum#10]
+Keys [2]: [ss_store_sk#2, ss_item_sk#1]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#3))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#3))#12]
+Results [3]: [ss_store_sk#2, ss_item_sk#1, MakeDecimal(sum(UnscaledValue(ss_sales_price#3))#12,17,2) AS revenue#13]
+
+(14) Filter [codegen id : 8]
+Input [3]: [ss_store_sk#2, ss_item_sk#1, revenue#13]
+Condition : isnotnull(revenue#13)
+
+(15) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#14, ss_store_sk#15, ss_sales_price#16, ss_sold_date_sk#17]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#17), dynamicpruningexpression(ss_sold_date_sk#17 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(16) ColumnarToRow [codegen id : 4]
+Input [4]: [ss_item_sk#14, ss_store_sk#15, ss_sales_price#16, ss_sold_date_sk#17]
+
+(17) Filter [codegen id : 4]
+Input [4]: [ss_item_sk#14, ss_store_sk#15, ss_sales_price#16, ss_sold_date_sk#17]
+Condition : isnotnull(ss_store_sk#15)
+
+(18) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#18]
+
+(19) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#17]
+Right keys [1]: [d_date_sk#18]
+Join condition: None
+
+(20) Project [codegen id : 4]
+Output [3]: [ss_item_sk#14, ss_store_sk#15, ss_sales_price#16]
+Input [5]: [ss_item_sk#14, ss_store_sk#15, ss_sales_price#16, ss_sold_date_sk#17, d_date_sk#18]
+
+(21) HashAggregate [codegen id : 4]
+Input [3]: [ss_item_sk#14, ss_store_sk#15, ss_sales_price#16]
+Keys [2]: [ss_store_sk#15, ss_item_sk#14]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#16))]
+Aggregate Attributes [1]: [sum#19]
+Results [3]: [ss_store_sk#15, ss_item_sk#14, sum#20]
+
+(22) Exchange
+Input [3]: [ss_store_sk#15, ss_item_sk#14, sum#20]
+Arguments: hashpartitioning(ss_store_sk#15, ss_item_sk#14, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(23) HashAggregate [codegen id : 5]
+Input [3]: [ss_store_sk#15, ss_item_sk#14, sum#20]
+Keys [2]: [ss_store_sk#15, ss_item_sk#14]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#16))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#16))#22]
+Results [2]: [ss_store_sk#15, MakeDecimal(sum(UnscaledValue(ss_sales_price#16))#22,17,2) AS revenue#23]
+
+(24) HashAggregate [codegen id : 5]
+Input [2]: [ss_store_sk#15, revenue#23]
+Keys [1]: [ss_store_sk#15]
+Functions [1]: [partial_avg(revenue#23)]
+Aggregate Attributes [2]: [sum#24, count#25]
+Results [3]: [ss_store_sk#15, sum#26, count#27]
+
+(25) Exchange
+Input [3]: [ss_store_sk#15, sum#26, count#27]
+Arguments: hashpartitioning(ss_store_sk#15, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(26) HashAggregate [codegen id : 6]
+Input [3]: [ss_store_sk#15, sum#26, count#27]
+Keys [1]: [ss_store_sk#15]
+Functions [1]: [avg(revenue#23)]
+Aggregate Attributes [1]: [avg(revenue#23)#29]
+Results [2]: [ss_store_sk#15, avg(revenue#23)#29 AS ave#30]
+
+(27) BroadcastExchange
+Input [2]: [ss_store_sk#15, ave#30]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#31]
+
+(28) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_store_sk#2]
+Right keys [1]: [ss_store_sk#15]
+Join condition: (cast(revenue#13 as decimal(23,7)) <= CheckOverflow((0.100000 * promote_precision(ave#30)), DecimalType(23,7), true))
+
+(29) Project [codegen id : 8]
+Output [3]: [ss_store_sk#2, ss_item_sk#1, revenue#13]
+Input [5]: [ss_store_sk#2, ss_item_sk#1, revenue#13, ss_store_sk#15, ave#30]
+
+(30) Scan parquet default.store
+Output [2]: [s_store_sk#32, s_store_name#33]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string>
+
+(31) ColumnarToRow [codegen id : 7]
+Input [2]: [s_store_sk#32, s_store_name#33]
+
+(32) Filter [codegen id : 7]
+Input [2]: [s_store_sk#32, s_store_name#33]
+Condition : isnotnull(s_store_sk#32)
+
+(33) BroadcastExchange
+Input [2]: [s_store_sk#32, s_store_name#33]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#34]
+
+(34) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_store_sk#2]
+Right keys [1]: [s_store_sk#32]
+Join condition: None
+
+(35) Project [codegen id : 8]
+Output [3]: [ss_item_sk#1, revenue#13, s_store_name#33]
+Input [5]: [ss_store_sk#2, ss_item_sk#1, revenue#13, s_store_sk#32, s_store_name#33]
+
+(36) Exchange
+Input [3]: [ss_item_sk#1, revenue#13, s_store_name#33]
+Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#35]
+
+(37) Sort [codegen id : 9]
+Input [3]: [ss_item_sk#1, revenue#13, s_store_name#33]
+Arguments: [ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(38) Scan parquet default.item
+Output [5]: [i_item_sk#36, i_item_desc#37, i_current_price#38, i_wholesale_cost#39, i_brand#40]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_desc:string,i_current_price:decimal(7,2),i_wholesale_cost:decimal(7,2),i_brand:string>
+
+(39) ColumnarToRow [codegen id : 10]
+Input [5]: [i_item_sk#36, i_item_desc#37, i_current_price#38, i_wholesale_cost#39, i_brand#40]
+
+(40) Filter [codegen id : 10]
+Input [5]: [i_item_sk#36, i_item_desc#37, i_current_price#38, i_wholesale_cost#39, i_brand#40]
+Condition : isnotnull(i_item_sk#36)
+
+(41) Exchange
+Input [5]: [i_item_sk#36, i_item_desc#37, i_current_price#38, i_wholesale_cost#39, i_brand#40]
+Arguments: hashpartitioning(i_item_sk#36, 5), ENSURE_REQUIREMENTS, [id=#41]
+
+(42) Sort [codegen id : 11]
+Input [5]: [i_item_sk#36, i_item_desc#37, i_current_price#38, i_wholesale_cost#39, i_brand#40]
+Arguments: [i_item_sk#36 ASC NULLS FIRST], false, 0
+
+(43) SortMergeJoin [codegen id : 12]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#36]
+Join condition: None
+
+(44) Project [codegen id : 12]
+Output [6]: [s_store_name#33, i_item_desc#37, revenue#13, i_current_price#38, i_wholesale_cost#39, i_brand#40]
+Input [8]: [ss_item_sk#1, revenue#13, s_store_name#33, i_item_sk#36, i_item_desc#37, i_current_price#38, i_wholesale_cost#39, i_brand#40]
+
+(45) TakeOrderedAndProject
+Input [6]: [s_store_name#33, i_item_desc#37, revenue#13, i_current_price#38, i_wholesale_cost#39, i_brand#40]
+Arguments: 100, [s_store_name#33 ASC NULLS FIRST, i_item_desc#37 ASC NULLS FIRST], [s_store_name#33, i_item_desc#37, revenue#13, i_current_price#38, i_wholesale_cost#39, i_brand#40]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (46)
+
+
+(46) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#6]
+
+Subquery:2 Hosting operator id = 15 Hosting Expression = ss_sold_date_sk#17 IN dynamicpruning#5
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.sf100/simplified.txt
new file mode 100644
index 0000000000000..aed4f13fecfd4
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.sf100/simplified.txt
@@ -0,0 +1,75 @@
+TakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand]
+  WholeStageCodegen (12)
+    Project [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand]
+      SortMergeJoin [ss_item_sk,i_item_sk]
+        InputAdapter
+          WholeStageCodegen (9)
+            Sort [ss_item_sk]
+              InputAdapter
+                Exchange [ss_item_sk] #1
+                  WholeStageCodegen (8)
+                    Project [ss_item_sk,revenue,s_store_name]
+                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                        Project [ss_store_sk,ss_item_sk,revenue]
+                          BroadcastHashJoin [ss_store_sk,ss_store_sk,revenue,ave]
+                            Filter [revenue]
+                              HashAggregate [ss_store_sk,ss_item_sk,sum] [sum(UnscaledValue(ss_sales_price)),revenue,sum]
+                                InputAdapter
+                                  Exchange [ss_store_sk,ss_item_sk] #2
+                                    WholeStageCodegen (2)
+                                      HashAggregate [ss_store_sk,ss_item_sk,ss_sales_price] [sum,sum]
+                                        Project [ss_item_sk,ss_store_sk,ss_sales_price]
+                                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                            Filter [ss_store_sk,ss_item_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #1
+                                                      ReusedExchange [d_date_sk] #3
+                                            InputAdapter
+                                              BroadcastExchange #3
+                                                WholeStageCodegen (1)
+                                                  Project [d_date_sk]
+                                                    Filter [d_month_seq,d_date_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.date_dim [d_date_sk,d_month_seq]
+                            InputAdapter
+                              BroadcastExchange #4
+                                WholeStageCodegen (6)
+                                  HashAggregate [ss_store_sk,sum,count] [avg(revenue),ave,sum,count]
+                                    InputAdapter
+                                      Exchange [ss_store_sk] #5
+                                        WholeStageCodegen (5)
+                                          HashAggregate [ss_store_sk,revenue] [sum,count,sum,count]
+                                            HashAggregate [ss_store_sk,ss_item_sk,sum] [sum(UnscaledValue(ss_sales_price)),revenue,sum]
+                                              InputAdapter
+                                                Exchange [ss_store_sk,ss_item_sk] #6
+                                                  WholeStageCodegen (4)
+                                                    HashAggregate [ss_store_sk,ss_item_sk,ss_sales_price] [sum,sum]
+                                                      Project [ss_item_sk,ss_store_sk,ss_sales_price]
+                                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                          Filter [ss_store_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                                                  ReusedSubquery [d_date_sk] #1
+                                                          InputAdapter
+                                                            ReusedExchange [d_date_sk] #3
+                        InputAdapter
+                          BroadcastExchange #7
+                            WholeStageCodegen (7)
+                              Filter [s_store_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.store [s_store_sk,s_store_name]
+        InputAdapter
+          WholeStageCodegen (11)
+            Sort [i_item_sk]
+              InputAdapter
+                Exchange [i_item_sk] #8
+                  WholeStageCodegen (10)
+                    Filter [i_item_sk]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/explain.txt
new file mode 100644
index 0000000000000..45c7c051601c5
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/explain.txt
@@ -0,0 +1,259 @@
+== Physical Plan ==
+TakeOrderedAndProject (42)
++- * Project (41)
+   +- * BroadcastHashJoin Inner BuildRight (40)
+      :- * Project (26)
+      :  +- * BroadcastHashJoin Inner BuildRight (25)
+      :     :- * Project (20)
+      :     :  +- * BroadcastHashJoin Inner BuildRight (19)
+      :     :     :- * Filter (3)
+      :     :     :  +- * ColumnarToRow (2)
+      :     :     :     +- Scan parquet default.store (1)
+      :     :     +- BroadcastExchange (18)
+      :     :        +- * Filter (17)
+      :     :           +- * HashAggregate (16)
+      :     :              +- Exchange (15)
+      :     :                 +- * HashAggregate (14)
+      :     :                    +- * Project (13)
+      :     :                       +- * BroadcastHashJoin Inner BuildRight (12)
+      :     :                          :- * Filter (6)
+      :     :                          :  +- * ColumnarToRow (5)
+      :     :                          :     +- Scan parquet default.store_sales (4)
+      :     :                          +- BroadcastExchange (11)
+      :     :                             +- * Project (10)
+      :     :                                +- * Filter (9)
+      :     :                                   +- * ColumnarToRow (8)
+      :     :                                      +- Scan parquet default.date_dim (7)
+      :     +- BroadcastExchange (24)
+      :        +- * Filter (23)
+      :           +- * ColumnarToRow (22)
+      :              +- Scan parquet default.item (21)
+      +- BroadcastExchange (39)
+         +- * HashAggregate (38)
+            +- Exchange (37)
+               +- * HashAggregate (36)
+                  +- * HashAggregate (35)
+                     +- Exchange (34)
+                        +- * HashAggregate (33)
+                           +- * Project (32)
+                              +- * BroadcastHashJoin Inner BuildRight (31)
+                                 :- * Filter (29)
+                                 :  +- * ColumnarToRow (28)
+                                 :     +- Scan parquet default.store_sales (27)
+                                 +- ReusedExchange (30)
+
+
+(1) Scan parquet default.store
+Output [2]: [s_store_sk#1, s_store_name#2]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string>
+
+(2) ColumnarToRow [codegen id : 9]
+Input [2]: [s_store_sk#1, s_store_name#2]
+
+(3) Filter [codegen id : 9]
+Input [2]: [s_store_sk#1, s_store_name#2]
+Condition : isnotnull(s_store_sk#1)
+
+(4) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(5) ColumnarToRow [codegen id : 2]
+Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6]
+
+(6) Filter [codegen id : 2]
+Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6]
+Condition : (isnotnull(ss_store_sk#4) AND isnotnull(ss_item_sk#3))
+
+(7) Scan parquet default.date_dim
+Output [2]: [d_date_sk#8, d_month_seq#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1176), LessThanOrEqual(d_month_seq,1187), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(8) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#8, d_month_seq#9]
+
+(9) Filter [codegen id : 1]
+Input [2]: [d_date_sk#8, d_month_seq#9]
+Condition : (((isnotnull(d_month_seq#9) AND (d_month_seq#9 >= 1176)) AND (d_month_seq#9 <= 1187)) AND isnotnull(d_date_sk#8))
+
+(10) Project [codegen id : 1]
+Output [1]: [d_date_sk#8]
+Input [2]: [d_date_sk#8, d_month_seq#9]
+
+(11) BroadcastExchange
+Input [1]: [d_date_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(12) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#6]
+Right keys [1]: [d_date_sk#8]
+Join condition: None
+
+(13) Project [codegen id : 2]
+Output [3]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5]
+Input [5]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6, d_date_sk#8]
+
+(14) HashAggregate [codegen id : 2]
+Input [3]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5]
+Keys [2]: [ss_store_sk#4, ss_item_sk#3]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#5))]
+Aggregate Attributes [1]: [sum#11]
+Results [3]: [ss_store_sk#4, ss_item_sk#3, sum#12]
+
+(15) Exchange
+Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#12]
+Arguments: hashpartitioning(ss_store_sk#4, ss_item_sk#3, 5), ENSURE_REQUIREMENTS, [id=#13]
+
+(16) HashAggregate [codegen id : 3]
+Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#12]
+Keys [2]: [ss_store_sk#4, ss_item_sk#3]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#5))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#5))#14]
+Results [3]: [ss_store_sk#4, ss_item_sk#3, MakeDecimal(sum(UnscaledValue(ss_sales_price#5))#14,17,2) AS revenue#15]
+
+(17) Filter [codegen id : 3]
+Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#15]
+Condition : isnotnull(revenue#15)
+
+(18) BroadcastExchange
+Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#16]
+
+(19) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [s_store_sk#1]
+Right keys [1]: [ss_store_sk#4]
+Join condition: None
+
+(20) Project [codegen id : 9]
+Output [4]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#15]
+Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#15]
+
+(21) Scan parquet default.item
+Output [5]: [i_item_sk#17, i_item_desc#18, i_current_price#19, i_wholesale_cost#20, i_brand#21]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_desc:string,i_current_price:decimal(7,2),i_wholesale_cost:decimal(7,2),i_brand:string>
+
+(22) ColumnarToRow [codegen id : 4]
+Input [5]: [i_item_sk#17, i_item_desc#18, i_current_price#19, i_wholesale_cost#20, i_brand#21]
+
+(23) Filter [codegen id : 4]
+Input [5]: [i_item_sk#17, i_item_desc#18, i_current_price#19, i_wholesale_cost#20, i_brand#21]
+Condition : isnotnull(i_item_sk#17)
+
+(24) BroadcastExchange
+Input [5]: [i_item_sk#17, i_item_desc#18, i_current_price#19, i_wholesale_cost#20, i_brand#21]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#22]
+
+(25) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_item_sk#3]
+Right keys [1]: [i_item_sk#17]
+Join condition: None
+
+(26) Project [codegen id : 9]
+Output [7]: [s_store_name#2, ss_store_sk#4, revenue#15, i_item_desc#18, i_current_price#19, i_wholesale_cost#20, i_brand#21]
+Input [9]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#15, i_item_sk#17, i_item_desc#18, i_current_price#19, i_wholesale_cost#20, i_brand#21]
+
+(27) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#23, ss_store_sk#24, ss_sales_price#25, ss_sold_date_sk#26]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#26), dynamicpruningexpression(ss_sold_date_sk#26 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(28) ColumnarToRow [codegen id : 6]
+Input [4]: [ss_item_sk#23, ss_store_sk#24, ss_sales_price#25, ss_sold_date_sk#26]
+
+(29) Filter [codegen id : 6]
+Input [4]: [ss_item_sk#23, ss_store_sk#24, ss_sales_price#25, ss_sold_date_sk#26]
+Condition : isnotnull(ss_store_sk#24)
+
+(30) ReusedExchange [Reuses operator id: 11]
+Output [1]: [d_date_sk#27]
+
+(31) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_sold_date_sk#26]
+Right keys [1]: [d_date_sk#27]
+Join condition: None
+
+(32) Project [codegen id : 6]
+Output [3]: [ss_item_sk#23, ss_store_sk#24, ss_sales_price#25]
+Input [5]: [ss_item_sk#23, ss_store_sk#24, ss_sales_price#25, ss_sold_date_sk#26, d_date_sk#27]
+
+(33) HashAggregate [codegen id : 6]
+Input [3]: [ss_item_sk#23, ss_store_sk#24, ss_sales_price#25]
+Keys [2]: [ss_store_sk#24, ss_item_sk#23]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#25))]
+Aggregate Attributes [1]: [sum#28]
+Results [3]: [ss_store_sk#24, ss_item_sk#23, sum#29]
+
+(34) Exchange
+Input [3]: [ss_store_sk#24, ss_item_sk#23, sum#29]
+Arguments: hashpartitioning(ss_store_sk#24, ss_item_sk#23, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(35) HashAggregate [codegen id : 7]
+Input [3]: [ss_store_sk#24, ss_item_sk#23, sum#29]
+Keys [2]: [ss_store_sk#24, ss_item_sk#23]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#25))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#25))#31]
+Results [2]: [ss_store_sk#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#25))#31,17,2) AS revenue#32]
+
+(36) HashAggregate [codegen id : 7]
+Input [2]: [ss_store_sk#24, revenue#32]
+Keys [1]: [ss_store_sk#24]
+Functions [1]: [partial_avg(revenue#32)]
+Aggregate Attributes [2]: [sum#33, count#34]
+Results [3]: [ss_store_sk#24, sum#35, count#36]
+
+(37) Exchange
+Input [3]: [ss_store_sk#24, sum#35, count#36]
+Arguments: hashpartitioning(ss_store_sk#24, 5), ENSURE_REQUIREMENTS, [id=#37]
+
+(38) HashAggregate [codegen id : 8]
+Input [3]: [ss_store_sk#24, sum#35, count#36]
+Keys [1]: [ss_store_sk#24]
+Functions [1]: [avg(revenue#32)]
+Aggregate Attributes [1]: [avg(revenue#32)#38]
+Results [2]: [ss_store_sk#24, avg(revenue#32)#38 AS ave#39]
+
+(39) BroadcastExchange
+Input [2]: [ss_store_sk#24, ave#39]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#40]
+
+(40) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_store_sk#4]
+Right keys [1]: [ss_store_sk#24]
+Join condition: (cast(revenue#15 as decimal(23,7)) <= CheckOverflow((0.100000 * promote_precision(ave#39)), DecimalType(23,7), true))
+
+(41) Project [codegen id : 9]
+Output [6]: [s_store_name#2, i_item_desc#18, revenue#15, i_current_price#19, i_wholesale_cost#20, i_brand#21]
+Input [9]: [s_store_name#2, ss_store_sk#4, revenue#15, i_item_desc#18, i_current_price#19, i_wholesale_cost#20, i_brand#21, ss_store_sk#24, ave#39]
+
+(42) TakeOrderedAndProject
+Input [6]: [s_store_name#2, i_item_desc#18, revenue#15, i_current_price#19, i_wholesale_cost#20, i_brand#21]
+Arguments: 100, [s_store_name#2 ASC NULLS FIRST, i_item_desc#18 ASC NULLS FIRST], [s_store_name#2, i_item_desc#18, revenue#15, i_current_price#19, i_wholesale_cost#20, i_brand#21]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7
+ReusedExchange (43)
+
+
+(43) ReusedExchange [Reuses operator id: 11]
+Output [1]: [d_date_sk#8]
+
+Subquery:2 Hosting operator id = 27 Hosting Expression = ss_sold_date_sk#26 IN dynamicpruning#7
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/simplified.txt
new file mode 100644
index 0000000000000..d7d461677be96
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/simplified.txt
@@ -0,0 +1,66 @@
+TakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand]
+  WholeStageCodegen (9)
+    Project [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand]
+      BroadcastHashJoin [ss_store_sk,ss_store_sk,revenue,ave]
+        Project [s_store_name,ss_store_sk,revenue,i_item_desc,i_current_price,i_wholesale_cost,i_brand]
+          BroadcastHashJoin [ss_item_sk,i_item_sk]
+            Project [s_store_name,ss_store_sk,ss_item_sk,revenue]
+              BroadcastHashJoin [s_store_sk,ss_store_sk]
+                Filter [s_store_sk]
+                  ColumnarToRow
+                    InputAdapter
+                      Scan parquet default.store [s_store_sk,s_store_name]
+                InputAdapter
+                  BroadcastExchange #1
+                    WholeStageCodegen (3)
+                      Filter [revenue]
+                        HashAggregate [ss_store_sk,ss_item_sk,sum] [sum(UnscaledValue(ss_sales_price)),revenue,sum]
+                          InputAdapter
+                            Exchange [ss_store_sk,ss_item_sk] #2
+                              WholeStageCodegen (2)
+                                HashAggregate [ss_store_sk,ss_item_sk,ss_sales_price] [sum,sum]
+                                  Project [ss_item_sk,ss_store_sk,ss_sales_price]
+                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                      Filter [ss_store_sk,ss_item_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #1
+                                                ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        BroadcastExchange #3
+                                          WholeStageCodegen (1)
+                                            Project [d_date_sk]
+                                              Filter [d_month_seq,d_date_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_date_sk,d_month_seq]
+            InputAdapter
+              BroadcastExchange #4
+                WholeStageCodegen (4)
+                  Filter [i_item_sk]
+                    ColumnarToRow
+                      InputAdapter
+                        Scan parquet default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand]
+        InputAdapter
+          BroadcastExchange #5
+            WholeStageCodegen (8)
+              HashAggregate [ss_store_sk,sum,count] [avg(revenue),ave,sum,count]
+                InputAdapter
+                  Exchange [ss_store_sk] #6
+                    WholeStageCodegen (7)
+                      HashAggregate [ss_store_sk,revenue] [sum,count,sum,count]
+                        HashAggregate [ss_store_sk,ss_item_sk,sum] [sum(UnscaledValue(ss_sales_price)),revenue,sum]
+                          InputAdapter
+                            Exchange [ss_store_sk,ss_item_sk] #7
+                              WholeStageCodegen (6)
+                                HashAggregate [ss_store_sk,ss_item_sk,ss_sales_price] [sum,sum]
+                                  Project [ss_item_sk,ss_store_sk,ss_sales_price]
+                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                      Filter [ss_store_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                              ReusedSubquery [d_date_sk] #1
+                                      InputAdapter
+                                        ReusedExchange [d_date_sk] #3
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.sf100/explain.txt
new file mode 100644
index 0000000000000..8c342961cf970
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.sf100/explain.txt
@@ -0,0 +1,324 @@
+== Physical Plan ==
+TakeOrderedAndProject (55)
++- * HashAggregate (54)
+   +- Exchange (53)
+      +- * HashAggregate (52)
+         +- Union (51)
+            :- * HashAggregate (32)
+            :  +- Exchange (31)
+            :     +- * HashAggregate (30)
+            :        +- * Project (29)
+            :           +- * BroadcastHashJoin Inner BuildRight (28)
+            :              :- * Project (23)
+            :              :  +- * BroadcastHashJoin Inner BuildRight (22)
+            :              :     :- * Project (17)
+            :              :     :  +- * BroadcastHashJoin Inner BuildRight (16)
+            :              :     :     :- * Project (10)
+            :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+            :              :     :     :     :- * Filter (3)
+            :              :     :     :     :  +- * ColumnarToRow (2)
+            :              :     :     :     :     +- Scan parquet default.web_sales (1)
+            :              :     :     :     +- BroadcastExchange (8)
+            :              :     :     :        +- * Project (7)
+            :              :     :     :           +- * Filter (6)
+            :              :     :     :              +- * ColumnarToRow (5)
+            :              :     :     :                 +- Scan parquet default.ship_mode (4)
+            :              :     :     +- BroadcastExchange (15)
+            :              :     :        +- * Project (14)
+            :              :     :           +- * Filter (13)
+            :              :     :              +- * ColumnarToRow (12)
+            :              :     :                 +- Scan parquet default.time_dim (11)
+            :              :     +- BroadcastExchange (21)
+            :              :        +- * Filter (20)
+            :              :           +- * ColumnarToRow (19)
+            :              :              +- Scan parquet default.date_dim (18)
+            :              +- BroadcastExchange (27)
+            :                 +- * Filter (26)
+            :                    +- * ColumnarToRow (25)
+            :                       +- Scan parquet default.warehouse (24)
+            +- * HashAggregate (50)
+               +- Exchange (49)
+                  +- * HashAggregate (48)
+                     +- * Project (47)
+                        +- * BroadcastHashJoin Inner BuildRight (46)
+                           :- * Project (44)
+                           :  +- * BroadcastHashJoin Inner BuildRight (43)
+                           :     :- * Project (41)
+                           :     :  +- * BroadcastHashJoin Inner BuildRight (40)
+                           :     :     :- * Project (38)
+                           :     :     :  +- * BroadcastHashJoin Inner BuildRight (37)
+                           :     :     :     :- * Filter (35)
+                           :     :     :     :  +- * ColumnarToRow (34)
+                           :     :     :     :     +- Scan parquet default.catalog_sales (33)
+                           :     :     :     +- ReusedExchange (36)
+                           :     :     +- ReusedExchange (39)
+                           :     +- ReusedExchange (42)
+                           +- ReusedExchange (45)
+
+
+(1) Scan parquet default.web_sales
+Output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#7), dynamicpruningexpression(ws_sold_date_sk#7 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ws_warehouse_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_ship_mode_sk)]
+ReadSchema: struct<ws_sold_time_sk:int,ws_ship_mode_sk:int,ws_warehouse_sk:int,ws_quantity:int,ws_ext_sales_price:decimal(7,2),ws_net_paid:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7]
+
+(3) Filter [codegen id : 5]
+Input [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7]
+Condition : ((isnotnull(ws_warehouse_sk#3) AND isnotnull(ws_sold_time_sk#1)) AND isnotnull(ws_ship_mode_sk#2))
+
+(4) Scan parquet default.ship_mode
+Output [2]: [sm_ship_mode_sk#9, sm_carrier#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/ship_mode]
+PushedFilters: [In(sm_carrier, [DHL                 ,BARIAN              ]), IsNotNull(sm_ship_mode_sk)]
+ReadSchema: struct<sm_ship_mode_sk:int,sm_carrier:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [sm_ship_mode_sk#9, sm_carrier#10]
+
+(6) Filter [codegen id : 1]
+Input [2]: [sm_ship_mode_sk#9, sm_carrier#10]
+Condition : (sm_carrier#10 IN (DHL                 ,BARIAN              ) AND isnotnull(sm_ship_mode_sk#9))
+
+(7) Project [codegen id : 1]
+Output [1]: [sm_ship_mode_sk#9]
+Input [2]: [sm_ship_mode_sk#9, sm_carrier#10]
+
+(8) BroadcastExchange
+Input [1]: [sm_ship_mode_sk#9]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#11]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ws_ship_mode_sk#2]
+Right keys [1]: [sm_ship_mode_sk#9]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [6]: [ws_sold_time_sk#1, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7]
+Input [8]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, sm_ship_mode_sk#9]
+
+(11) Scan parquet default.time_dim
+Output [2]: [t_time_sk#12, t_time#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [IsNotNull(t_time), GreaterThanOrEqual(t_time,30838), LessThanOrEqual(t_time,59638), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_time:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [t_time_sk#12, t_time#13]
+
+(13) Filter [codegen id : 2]
+Input [2]: [t_time_sk#12, t_time#13]
+Condition : (((isnotnull(t_time#13) AND (t_time#13 >= 30838)) AND (t_time#13 <= 59638)) AND isnotnull(t_time_sk#12))
+
+(14) Project [codegen id : 2]
+Output [1]: [t_time_sk#12]
+Input [2]: [t_time_sk#12, t_time#13]
+
+(15) BroadcastExchange
+Input [1]: [t_time_sk#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(16) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ws_sold_time_sk#1]
+Right keys [1]: [t_time_sk#12]
+Join condition: None
+
+(17) Project [codegen id : 5]
+Output [5]: [ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7]
+Input [7]: [ws_sold_time_sk#1, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, t_time_sk#12]
+
+(18) Scan parquet default.date_dim
+Output [3]: [d_date_sk#15, d_year#16, d_moy#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#15, d_year#16, d_moy#17]
+
+(20) Filter [codegen id : 3]
+Input [3]: [d_date_sk#15, d_year#16, d_moy#17]
+Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15))
+
+(21) BroadcastExchange
+Input [3]: [d_date_sk#15, d_year#16, d_moy#17]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#18]
+
+(22) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ws_sold_date_sk#7]
+Right keys [1]: [d_date_sk#15]
+Join condition: None
+
+(23) Project [codegen id : 5]
+Output [6]: [ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, d_year#16, d_moy#17]
+Input [8]: [ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, d_date_sk#15, d_year#16, d_moy#17]
+
+(24) Scan parquet default.warehouse
+Output [7]: [w_warehouse_sk#19, w_warehouse_name#20, w_warehouse_sq_ft#21, w_city#22, w_county#23, w_state#24, w_country#25]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/warehouse]
+PushedFilters: [IsNotNull(w_warehouse_sk)]
+ReadSchema: struct<w_warehouse_sk:int,w_warehouse_name:string,w_warehouse_sq_ft:int,w_city:string,w_county:string,w_state:string,w_country:string>
+
+(25) ColumnarToRow [codegen id : 4]
+Input [7]: [w_warehouse_sk#19, w_warehouse_name#20, w_warehouse_sq_ft#21, w_city#22, w_county#23, w_state#24, w_country#25]
+
+(26) Filter [codegen id : 4]
+Input [7]: [w_warehouse_sk#19, w_warehouse_name#20, w_warehouse_sq_ft#21, w_city#22, w_county#23, w_state#24, w_country#25]
+Condition : isnotnull(w_warehouse_sk#19)
+
+(27) BroadcastExchange
+Input [7]: [w_warehouse_sk#19, w_warehouse_name#20, w_warehouse_sq_ft#21, w_city#22, w_county#23, w_state#24, w_country#25]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#26]
+
+(28) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ws_warehouse_sk#3]
+Right keys [1]: [w_warehouse_sk#19]
+Join condition: None
+
+(29) Project [codegen id : 5]
+Output [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#20, w_warehouse_sq_ft#21, w_city#22, w_county#23, w_state#24, w_country#25, d_year#16, d_moy#17]
+Input [13]: [ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, d_year#16, d_moy#17, w_warehouse_sk#19, w_warehouse_name#20, w_warehouse_sq_ft#21, w_city#22, w_county#23, w_state#24, w_country#25]
+
+(30) HashAggregate [codegen id : 5]
+Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#20, w_warehouse_sq_ft#21, w_city#22, w_county#23, w_state#24, w_country#25, d_year#16, d_moy#17]
+Keys [7]: [w_warehouse_name#20, w_warehouse_sq_ft#21, w_city#22, w_county#23, w_state#24, w_country#25, d_year#16]
+Functions [24]: [partial_sum(CASE WHEN (d_moy#17 = 1) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 2) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 3) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 4) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 5) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 6) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 7) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 8) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 9) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 10) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 11) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 12) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 1) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 2) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 3) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 4) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 5) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 6) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 7) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 8) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 9) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 10) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 11) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 12) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)]
+Aggregate Attributes [48]: [sum#27, isEmpty#28, sum#29, isEmpty#30, sum#31, isEmpty#32, sum#33, isEmpty#34, sum#35, isEmpty#36, sum#37, isEmpty#38, sum#39, isEmpty#40, sum#41, isEmpty#42, sum#43, isEmpty#44, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, isEmpty#54, sum#55, isEmpty#56, sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68, sum#69, isEmpty#70, sum#71, isEmpty#72, sum#73, isEmpty#74]
+Results [55]: [w_warehouse_name#20, w_warehouse_sq_ft#21, w_city#22, w_county#23, w_state#24, w_country#25, d_year#16, sum#75, isEmpty#76, sum#77, isEmpty#78, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122]
+
+(31) Exchange
+Input [55]: [w_warehouse_name#20, w_warehouse_sq_ft#21, w_city#22, w_county#23, w_state#24, w_country#25, d_year#16, sum#75, isEmpty#76, sum#77, isEmpty#78, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122]
+Arguments: hashpartitioning(w_warehouse_name#20, w_warehouse_sq_ft#21, w_city#22, w_county#23, w_state#24, w_country#25, d_year#16, 5), ENSURE_REQUIREMENTS, [id=#123]
+
+(32) HashAggregate [codegen id : 6]
+Input [55]: [w_warehouse_name#20, w_warehouse_sq_ft#21, w_city#22, w_county#23, w_state#24, w_country#25, d_year#16, sum#75, isEmpty#76, sum#77, isEmpty#78, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122]
+Keys [7]: [w_warehouse_name#20, w_warehouse_sq_ft#21, w_city#22, w_county#23, w_state#24, w_country#25, d_year#16]
+Functions [24]: [sum(CASE WHEN (d_moy#17 = 1) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 2) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 3) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 4) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 5) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 6) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 7) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 8) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 9) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 10) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 11) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 12) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 1) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 2) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 3) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 4) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 5) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 6) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 7) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 8) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 9) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 10) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 11) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 12) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)]
+Aggregate Attributes [24]: [sum(CASE WHEN (d_moy#17 = 1) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#124, sum(CASE WHEN (d_moy#17 = 2) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#125, sum(CASE WHEN (d_moy#17 = 3) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#126, sum(CASE WHEN (d_moy#17 = 4) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#127, sum(CASE WHEN (d_moy#17 = 5) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#128, sum(CASE WHEN (d_moy#17 = 6) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#129, sum(CASE WHEN (d_moy#17 = 7) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#130, sum(CASE WHEN (d_moy#17 = 8) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#131, sum(CASE WHEN (d_moy#17 = 9) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#132, sum(CASE WHEN (d_moy#17 = 10) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#133, sum(CASE WHEN (d_moy#17 = 11) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#134, sum(CASE WHEN (d_moy#17 = 12) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#135, sum(CASE WHEN (d_moy#17 = 1) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#136, sum(CASE WHEN (d_moy#17 = 2) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#137, sum(CASE WHEN (d_moy#17 = 3) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#138, sum(CASE WHEN (d_moy#17 = 4) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#139, sum(CASE WHEN (d_moy#17 = 5) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#140, sum(CASE WHEN (d_moy#17 = 6) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#141, sum(CASE WHEN (d_moy#17 = 7) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#142, sum(CASE WHEN (d_moy#17 = 8) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#143, sum(CASE WHEN (d_moy#17 = 9) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#144, sum(CASE WHEN (d_moy#17 = 10) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#145, sum(CASE WHEN (d_moy#17 = 11) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#146, sum(CASE WHEN (d_moy#17 = 12) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#147]
+Results [32]: [w_warehouse_name#20, w_warehouse_sq_ft#21, w_city#22, w_county#23, w_state#24, w_country#25, DHL,BARIAN AS ship_carriers#148, d_year#16 AS year#149, sum(CASE WHEN (d_moy#17 = 1) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#124 AS jan_sales#150, sum(CASE WHEN (d_moy#17 = 2) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#125 AS feb_sales#151, sum(CASE WHEN (d_moy#17 = 3) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#126 AS mar_sales#152, sum(CASE WHEN (d_moy#17 = 4) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#127 AS apr_sales#153, sum(CASE WHEN (d_moy#17 = 5) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#128 AS may_sales#154, sum(CASE WHEN (d_moy#17 = 6) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#129 AS jun_sales#155, sum(CASE WHEN (d_moy#17 = 7) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#130 AS jul_sales#156, sum(CASE WHEN (d_moy#17 = 8) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#131 AS aug_sales#157, sum(CASE WHEN (d_moy#17 = 9) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#132 AS sep_sales#158, sum(CASE WHEN (d_moy#17 = 10) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#133 AS oct_sales#159, sum(CASE WHEN (d_moy#17 = 11) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#134 AS nov_sales#160, sum(CASE WHEN (d_moy#17 = 12) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#135 AS dec_sales#161, sum(CASE WHEN (d_moy#17 = 1) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#136 AS jan_net#162, sum(CASE WHEN (d_moy#17 = 2) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#137 AS feb_net#163, sum(CASE WHEN (d_moy#17 = 3) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#138 AS mar_net#164, sum(CASE WHEN (d_moy#17 = 4) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#139 AS apr_net#165, sum(CASE WHEN (d_moy#17 = 5) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#140 AS may_net#166, sum(CASE WHEN (d_moy#17 = 6) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#141 AS jun_net#167, sum(CASE WHEN (d_moy#17 = 7) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#142 AS jul_net#168, sum(CASE WHEN (d_moy#17 = 8) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#143 AS aug_net#169, sum(CASE WHEN (d_moy#17 = 9) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#144 AS sep_net#170, sum(CASE WHEN (d_moy#17 = 10) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#145 AS oct_net#171, sum(CASE WHEN (d_moy#17 = 11) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#146 AS nov_net#172, sum(CASE WHEN (d_moy#17 = 12) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#147 AS dec_net#173]
+
+(33) Scan parquet default.catalog_sales
+Output [7]: [cs_sold_time_sk#174, cs_ship_mode_sk#175, cs_warehouse_sk#176, cs_quantity#177, cs_sales_price#178, cs_net_paid_inc_tax#179, cs_sold_date_sk#180]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#180), dynamicpruningexpression(cs_sold_date_sk#180 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_sold_time_sk), IsNotNull(cs_ship_mode_sk)]
+ReadSchema: struct<cs_sold_time_sk:int,cs_ship_mode_sk:int,cs_warehouse_sk:int,cs_quantity:int,cs_sales_price:decimal(7,2),cs_net_paid_inc_tax:decimal(7,2)>
+
+(34) ColumnarToRow [codegen id : 11]
+Input [7]: [cs_sold_time_sk#174, cs_ship_mode_sk#175, cs_warehouse_sk#176, cs_quantity#177, cs_sales_price#178, cs_net_paid_inc_tax#179, cs_sold_date_sk#180]
+
+(35) Filter [codegen id : 11]
+Input [7]: [cs_sold_time_sk#174, cs_ship_mode_sk#175, cs_warehouse_sk#176, cs_quantity#177, cs_sales_price#178, cs_net_paid_inc_tax#179, cs_sold_date_sk#180]
+Condition : ((isnotnull(cs_warehouse_sk#176) AND isnotnull(cs_sold_time_sk#174)) AND isnotnull(cs_ship_mode_sk#175))
+
+(36) ReusedExchange [Reuses operator id: 8]
+Output [1]: [sm_ship_mode_sk#181]
+
+(37) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_ship_mode_sk#175]
+Right keys [1]: [sm_ship_mode_sk#181]
+Join condition: None
+
+(38) Project [codegen id : 11]
+Output [6]: [cs_sold_time_sk#174, cs_warehouse_sk#176, cs_quantity#177, cs_sales_price#178, cs_net_paid_inc_tax#179, cs_sold_date_sk#180]
+Input [8]: [cs_sold_time_sk#174, cs_ship_mode_sk#175, cs_warehouse_sk#176, cs_quantity#177, cs_sales_price#178, cs_net_paid_inc_tax#179, cs_sold_date_sk#180, sm_ship_mode_sk#181]
+
+(39) ReusedExchange [Reuses operator id: 15]
+Output [1]: [t_time_sk#182]
+
+(40) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_sold_time_sk#174]
+Right keys [1]: [t_time_sk#182]
+Join condition: None
+
+(41) Project [codegen id : 11]
+Output [5]: [cs_warehouse_sk#176, cs_quantity#177, cs_sales_price#178, cs_net_paid_inc_tax#179, cs_sold_date_sk#180]
+Input [7]: [cs_sold_time_sk#174, cs_warehouse_sk#176, cs_quantity#177, cs_sales_price#178, cs_net_paid_inc_tax#179, cs_sold_date_sk#180, t_time_sk#182]
+
+(42) ReusedExchange [Reuses operator id: 21]
+Output [3]: [d_date_sk#183, d_year#184, d_moy#185]
+
+(43) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_sold_date_sk#180]
+Right keys [1]: [d_date_sk#183]
+Join condition: None
+
+(44) Project [codegen id : 11]
+Output [6]: [cs_warehouse_sk#176, cs_quantity#177, cs_sales_price#178, cs_net_paid_inc_tax#179, d_year#184, d_moy#185]
+Input [8]: [cs_warehouse_sk#176, cs_quantity#177, cs_sales_price#178, cs_net_paid_inc_tax#179, cs_sold_date_sk#180, d_date_sk#183, d_year#184, d_moy#185]
+
+(45) ReusedExchange [Reuses operator id: 27]
+Output [7]: [w_warehouse_sk#186, w_warehouse_name#187, w_warehouse_sq_ft#188, w_city#189, w_county#190, w_state#191, w_country#192]
+
+(46) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_warehouse_sk#176]
+Right keys [1]: [w_warehouse_sk#186]
+Join condition: None
+
+(47) Project [codegen id : 11]
+Output [11]: [cs_quantity#177, cs_sales_price#178, cs_net_paid_inc_tax#179, w_warehouse_name#187, w_warehouse_sq_ft#188, w_city#189, w_county#190, w_state#191, w_country#192, d_year#184, d_moy#185]
+Input [13]: [cs_warehouse_sk#176, cs_quantity#177, cs_sales_price#178, cs_net_paid_inc_tax#179, d_year#184, d_moy#185, w_warehouse_sk#186, w_warehouse_name#187, w_warehouse_sq_ft#188, w_city#189, w_county#190, w_state#191, w_country#192]
+
+(48) HashAggregate [codegen id : 11]
+Input [11]: [cs_quantity#177, cs_sales_price#178, cs_net_paid_inc_tax#179, w_warehouse_name#187, w_warehouse_sq_ft#188, w_city#189, w_county#190, w_state#191, w_country#192, d_year#184, d_moy#185]
+Keys [7]: [w_warehouse_name#187, w_warehouse_sq_ft#188, w_city#189, w_county#190, w_state#191, w_country#192, d_year#184]
+Functions [24]: [partial_sum(CASE WHEN (d_moy#185 = 1) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#185 = 2) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#185 = 3) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#185 = 4) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#185 = 5) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#185 = 6) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#185 = 7) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#185 = 8) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#185 = 9) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#185 = 10) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#185 = 11) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#185 = 12) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#185 = 1) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#185 = 2) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#185 = 3) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#185 = 4) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#185 = 5) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#185 = 6) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#185 = 7) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#185 = 8) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#185 = 9) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#185 = 10) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#185 = 11) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#185 = 12) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)]
+Aggregate Attributes [48]: [sum#193, isEmpty#194, sum#195, isEmpty#196, sum#197, isEmpty#198, sum#199, isEmpty#200, sum#201, isEmpty#202, sum#203, isEmpty#204, sum#205, isEmpty#206, sum#207, isEmpty#208, sum#209, isEmpty#210, sum#211, isEmpty#212, sum#213, isEmpty#214, sum#215, isEmpty#216, sum#217, isEmpty#218, sum#219, isEmpty#220, sum#221, isEmpty#222, sum#223, isEmpty#224, sum#225, isEmpty#226, sum#227, isEmpty#228, sum#229, isEmpty#230, sum#231, isEmpty#232, sum#233, isEmpty#234, sum#235, isEmpty#236, sum#237, isEmpty#238, sum#239, isEmpty#240]
+Results [55]: [w_warehouse_name#187, w_warehouse_sq_ft#188, w_city#189, w_county#190, w_state#191, w_country#192, d_year#184, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260, sum#261, isEmpty#262, sum#263, isEmpty#264, sum#265, isEmpty#266, sum#267, isEmpty#268, sum#269, isEmpty#270, sum#271, isEmpty#272, sum#273, isEmpty#274, sum#275, isEmpty#276, sum#277, isEmpty#278, sum#279, isEmpty#280, sum#281, isEmpty#282, sum#283, isEmpty#284, sum#285, isEmpty#286, sum#287, isEmpty#288]
+
+(49) Exchange
+Input [55]: [w_warehouse_name#187, w_warehouse_sq_ft#188, w_city#189, w_county#190, w_state#191, w_country#192, d_year#184, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260, sum#261, isEmpty#262, sum#263, isEmpty#264, sum#265, isEmpty#266, sum#267, isEmpty#268, sum#269, isEmpty#270, sum#271, isEmpty#272, sum#273, isEmpty#274, sum#275, isEmpty#276, sum#277, isEmpty#278, sum#279, isEmpty#280, sum#281, isEmpty#282, sum#283, isEmpty#284, sum#285, isEmpty#286, sum#287, isEmpty#288]
+Arguments: hashpartitioning(w_warehouse_name#187, w_warehouse_sq_ft#188, w_city#189, w_county#190, w_state#191, w_country#192, d_year#184, 5), ENSURE_REQUIREMENTS, [id=#289]
+
+(50) HashAggregate [codegen id : 12]
+Input [55]: [w_warehouse_name#187, w_warehouse_sq_ft#188, w_city#189, w_county#190, w_state#191, w_country#192, d_year#184, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260, sum#261, isEmpty#262, sum#263, isEmpty#264, sum#265, isEmpty#266, sum#267, isEmpty#268, sum#269, isEmpty#270, sum#271, isEmpty#272, sum#273, isEmpty#274, sum#275, isEmpty#276, sum#277, isEmpty#278, sum#279, isEmpty#280, sum#281, isEmpty#282, sum#283, isEmpty#284, sum#285, isEmpty#286, sum#287, isEmpty#288]
+Keys [7]: [w_warehouse_name#187, w_warehouse_sq_ft#188, w_city#189, w_county#190, w_state#191, w_country#192, d_year#184]
+Functions [24]: [sum(CASE WHEN (d_moy#185 = 1) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#185 = 2) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#185 = 3) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#185 = 4) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#185 = 5) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#185 = 6) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#185 = 7) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#185 = 8) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#185 = 9) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#185 = 10) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#185 = 11) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#185 = 12) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#185 = 1) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#185 = 2) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#185 = 3) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#185 = 4) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#185 = 5) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#185 = 6) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#185 = 7) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#185 = 8) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#185 = 9) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#185 = 10) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#185 = 11) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#185 = 12) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)]
+Aggregate Attributes [24]: [sum(CASE WHEN (d_moy#185 = 1) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#290, sum(CASE WHEN (d_moy#185 = 2) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#291, sum(CASE WHEN (d_moy#185 = 3) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#292, sum(CASE WHEN (d_moy#185 = 4) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#293, sum(CASE WHEN (d_moy#185 = 5) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#294, sum(CASE WHEN (d_moy#185 = 6) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#295, sum(CASE WHEN (d_moy#185 = 7) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#296, sum(CASE WHEN (d_moy#185 = 8) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#297, sum(CASE WHEN (d_moy#185 = 9) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#298, sum(CASE WHEN (d_moy#185 = 10) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#299, sum(CASE WHEN (d_moy#185 = 11) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#300, sum(CASE WHEN (d_moy#185 = 12) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#301, sum(CASE WHEN (d_moy#185 = 1) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#302, sum(CASE WHEN (d_moy#185 = 2) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#303, sum(CASE WHEN (d_moy#185 = 3) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#304, sum(CASE WHEN (d_moy#185 = 4) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#305, sum(CASE WHEN (d_moy#185 = 5) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#306, sum(CASE WHEN (d_moy#185 = 6) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#307, sum(CASE WHEN (d_moy#185 = 7) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#308, sum(CASE WHEN (d_moy#185 = 8) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#309, sum(CASE WHEN (d_moy#185 = 9) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#310, sum(CASE WHEN (d_moy#185 = 10) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#311, sum(CASE WHEN (d_moy#185 = 11) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#312, sum(CASE WHEN (d_moy#185 = 12) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#313]
+Results [32]: [w_warehouse_name#187, w_warehouse_sq_ft#188, w_city#189, w_county#190, w_state#191, w_country#192, DHL,BARIAN AS ship_carriers#314, d_year#184 AS year#315, sum(CASE WHEN (d_moy#185 = 1) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#290 AS jan_sales#316, sum(CASE WHEN (d_moy#185 = 2) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#291 AS feb_sales#317, sum(CASE WHEN (d_moy#185 = 3) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#292 AS mar_sales#318, sum(CASE WHEN (d_moy#185 = 4) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#293 AS apr_sales#319, sum(CASE WHEN (d_moy#185 = 5) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#294 AS may_sales#320, sum(CASE WHEN (d_moy#185 = 6) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#295 AS jun_sales#321, sum(CASE WHEN (d_moy#185 = 7) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#296 AS jul_sales#322, sum(CASE WHEN (d_moy#185 = 8) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#297 AS aug_sales#323, sum(CASE WHEN (d_moy#185 = 9) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#298 AS sep_sales#324, sum(CASE WHEN (d_moy#185 = 10) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#299 AS oct_sales#325, sum(CASE WHEN (d_moy#185 = 11) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#300 AS nov_sales#326, sum(CASE WHEN (d_moy#185 = 12) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#301 AS dec_sales#327, sum(CASE WHEN (d_moy#185 = 1) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#302 AS jan_net#328, sum(CASE WHEN (d_moy#185 = 2) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#303 AS feb_net#329, sum(CASE WHEN (d_moy#185 = 3) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#304 AS mar_net#330, sum(CASE WHEN (d_moy#185 = 4) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#305 AS apr_net#331, sum(CASE WHEN (d_moy#185 = 5) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#306 AS may_net#332, sum(CASE WHEN (d_moy#185 = 6) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#307 AS jun_net#333, sum(CASE WHEN (d_moy#185 = 7) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#308 AS jul_net#334, sum(CASE WHEN (d_moy#185 = 8) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#309 AS aug_net#335, sum(CASE WHEN (d_moy#185 = 9) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#310 AS sep_net#336, sum(CASE WHEN (d_moy#185 = 10) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#311 AS oct_net#337, sum(CASE WHEN (d_moy#185 = 11) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#312 AS nov_net#338, sum(CASE WHEN (d_moy#185 = 12) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#313 AS dec_net#339]
+
+(51) Union
+
+(52) HashAggregate [codegen id : 13]
+Input [32]: [w_warehouse_name#20, w_warehouse_sq_ft#21, w_city#22, w_county#23, w_state#24, w_country#25, ship_carriers#148, year#149, jan_sales#150, feb_sales#151, mar_sales#152, apr_sales#153, may_sales#154, jun_sales#155, jul_sales#156, aug_sales#157, sep_sales#158, oct_sales#159, nov_sales#160, dec_sales#161, jan_net#162, feb_net#163, mar_net#164, apr_net#165, may_net#166, jun_net#167, jul_net#168, aug_net#169, sep_net#170, oct_net#171, nov_net#172, dec_net#173]
+Keys [8]: [w_warehouse_name#20, w_warehouse_sq_ft#21, w_city#22, w_county#23, w_state#24, w_country#25, ship_carriers#148, year#149]
+Functions [36]: [partial_sum(jan_sales#150), partial_sum(feb_sales#151), partial_sum(mar_sales#152), partial_sum(apr_sales#153), partial_sum(may_sales#154), partial_sum(jun_sales#155), partial_sum(jul_sales#156), partial_sum(aug_sales#157), partial_sum(sep_sales#158), partial_sum(oct_sales#159), partial_sum(nov_sales#160), partial_sum(dec_sales#161), partial_sum(CheckOverflow((promote_precision(jan_sales#150) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), partial_sum(CheckOverflow((promote_precision(feb_sales#151) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), partial_sum(CheckOverflow((promote_precision(mar_sales#152) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), partial_sum(CheckOverflow((promote_precision(apr_sales#153) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), partial_sum(CheckOverflow((promote_precision(may_sales#154) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), partial_sum(CheckOverflow((promote_precision(jun_sales#155) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), partial_sum(CheckOverflow((promote_precision(jul_sales#156) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), partial_sum(CheckOverflow((promote_precision(aug_sales#157) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), partial_sum(CheckOverflow((promote_precision(sep_sales#158) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), partial_sum(CheckOverflow((promote_precision(oct_sales#159) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), partial_sum(CheckOverflow((promote_precision(nov_sales#160) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), partial_sum(CheckOverflow((promote_precision(dec_sales#161) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), partial_sum(jan_net#162), partial_sum(feb_net#163), partial_sum(mar_net#164), partial_sum(apr_net#165), partial_sum(may_net#166), partial_sum(jun_net#167), partial_sum(jul_net#168), partial_sum(aug_net#169), partial_sum(sep_net#170), partial_sum(oct_net#171), partial_sum(nov_net#172), partial_sum(dec_net#173)]
+Aggregate Attributes [72]: [sum#340, isEmpty#341, sum#342, isEmpty#343, sum#344, isEmpty#345, sum#346, isEmpty#347, sum#348, isEmpty#349, sum#350, isEmpty#351, sum#352, isEmpty#353, sum#354, isEmpty#355, sum#356, isEmpty#357, sum#358, isEmpty#359, sum#360, isEmpty#361, sum#362, isEmpty#363, sum#364, isEmpty#365, sum#366, isEmpty#367, sum#368, isEmpty#369, sum#370, isEmpty#371, sum#372, isEmpty#373, sum#374, isEmpty#375, sum#376, isEmpty#377, sum#378, isEmpty#379, sum#380, isEmpty#381, sum#382, isEmpty#383, sum#384, isEmpty#385, sum#386, isEmpty#387, sum#388, isEmpty#389, sum#390, isEmpty#391, sum#392, isEmpty#393, sum#394, isEmpty#395, sum#396, isEmpty#397, sum#398, isEmpty#399, sum#400, isEmpty#401, sum#402, isEmpty#403, sum#404, isEmpty#405, sum#406, isEmpty#407, sum#408, isEmpty#409, sum#410, isEmpty#411]
+Results [80]: [w_warehouse_name#20, w_warehouse_sq_ft#21, w_city#22, w_county#23, w_state#24, w_country#25, ship_carriers#148, year#149, sum#412, isEmpty#413, sum#414, isEmpty#415, sum#416, isEmpty#417, sum#418, isEmpty#419, sum#420, isEmpty#421, sum#422, isEmpty#423, sum#424, isEmpty#425, sum#426, isEmpty#427, sum#428, isEmpty#429, sum#430, isEmpty#431, sum#432, isEmpty#433, sum#434, isEmpty#435, sum#436, isEmpty#437, sum#438, isEmpty#439, sum#440, isEmpty#441, sum#442, isEmpty#443, sum#444, isEmpty#445, sum#446, isEmpty#447, sum#448, isEmpty#449, sum#450, isEmpty#451, sum#452, isEmpty#453, sum#454, isEmpty#455, sum#456, isEmpty#457, sum#458, isEmpty#459, sum#460, isEmpty#461, sum#462, isEmpty#463, sum#464, isEmpty#465, sum#466, isEmpty#467, sum#468, isEmpty#469, sum#470, isEmpty#471, sum#472, isEmpty#473, sum#474, isEmpty#475, sum#476, isEmpty#477, sum#478, isEmpty#479, sum#480, isEmpty#481, sum#482, isEmpty#483]
+
+(53) Exchange
+Input [80]: [w_warehouse_name#20, w_warehouse_sq_ft#21, w_city#22, w_county#23, w_state#24, w_country#25, ship_carriers#148, year#149, sum#412, isEmpty#413, sum#414, isEmpty#415, sum#416, isEmpty#417, sum#418, isEmpty#419, sum#420, isEmpty#421, sum#422, isEmpty#423, sum#424, isEmpty#425, sum#426, isEmpty#427, sum#428, isEmpty#429, sum#430, isEmpty#431, sum#432, isEmpty#433, sum#434, isEmpty#435, sum#436, isEmpty#437, sum#438, isEmpty#439, sum#440, isEmpty#441, sum#442, isEmpty#443, sum#444, isEmpty#445, sum#446, isEmpty#447, sum#448, isEmpty#449, sum#450, isEmpty#451, sum#452, isEmpty#453, sum#454, isEmpty#455, sum#456, isEmpty#457, sum#458, isEmpty#459, sum#460, isEmpty#461, sum#462, isEmpty#463, sum#464, isEmpty#465, sum#466, isEmpty#467, sum#468, isEmpty#469, sum#470, isEmpty#471, sum#472, isEmpty#473, sum#474, isEmpty#475, sum#476, isEmpty#477, sum#478, isEmpty#479, sum#480, isEmpty#481, sum#482, isEmpty#483]
+Arguments: hashpartitioning(w_warehouse_name#20, w_warehouse_sq_ft#21, w_city#22, w_county#23, w_state#24, w_country#25, ship_carriers#148, year#149, 5), ENSURE_REQUIREMENTS, [id=#484]
+
+(54) HashAggregate [codegen id : 14]
+Input [80]: [w_warehouse_name#20, w_warehouse_sq_ft#21, w_city#22, w_county#23, w_state#24, w_country#25, ship_carriers#148, year#149, sum#412, isEmpty#413, sum#414, isEmpty#415, sum#416, isEmpty#417, sum#418, isEmpty#419, sum#420, isEmpty#421, sum#422, isEmpty#423, sum#424, isEmpty#425, sum#426, isEmpty#427, sum#428, isEmpty#429, sum#430, isEmpty#431, sum#432, isEmpty#433, sum#434, isEmpty#435, sum#436, isEmpty#437, sum#438, isEmpty#439, sum#440, isEmpty#441, sum#442, isEmpty#443, sum#444, isEmpty#445, sum#446, isEmpty#447, sum#448, isEmpty#449, sum#450, isEmpty#451, sum#452, isEmpty#453, sum#454, isEmpty#455, sum#456, isEmpty#457, sum#458, isEmpty#459, sum#460, isEmpty#461, sum#462, isEmpty#463, sum#464, isEmpty#465, sum#466, isEmpty#467, sum#468, isEmpty#469, sum#470, isEmpty#471, sum#472, isEmpty#473, sum#474, isEmpty#475, sum#476, isEmpty#477, sum#478, isEmpty#479, sum#480, isEmpty#481, sum#482, isEmpty#483]
+Keys [8]: [w_warehouse_name#20, w_warehouse_sq_ft#21, w_city#22, w_county#23, w_state#24, w_country#25, ship_carriers#148, year#149]
+Functions [36]: [sum(jan_sales#150), sum(feb_sales#151), sum(mar_sales#152), sum(apr_sales#153), sum(may_sales#154), sum(jun_sales#155), sum(jul_sales#156), sum(aug_sales#157), sum(sep_sales#158), sum(oct_sales#159), sum(nov_sales#160), sum(dec_sales#161), sum(CheckOverflow((promote_precision(jan_sales#150) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), sum(CheckOverflow((promote_precision(feb_sales#151) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), sum(CheckOverflow((promote_precision(mar_sales#152) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), sum(CheckOverflow((promote_precision(apr_sales#153) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), sum(CheckOverflow((promote_precision(may_sales#154) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), sum(CheckOverflow((promote_precision(jun_sales#155) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), sum(CheckOverflow((promote_precision(jul_sales#156) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), sum(CheckOverflow((promote_precision(aug_sales#157) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), sum(CheckOverflow((promote_precision(sep_sales#158) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), sum(CheckOverflow((promote_precision(oct_sales#159) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), sum(CheckOverflow((promote_precision(nov_sales#160) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), sum(CheckOverflow((promote_precision(dec_sales#161) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), sum(jan_net#162), sum(feb_net#163), sum(mar_net#164), sum(apr_net#165), sum(may_net#166), sum(jun_net#167), sum(jul_net#168), sum(aug_net#169), sum(sep_net#170), sum(oct_net#171), sum(nov_net#172), sum(dec_net#173)]
+Aggregate Attributes [36]: [sum(jan_sales#150)#485, sum(feb_sales#151)#486, sum(mar_sales#152)#487, sum(apr_sales#153)#488, sum(may_sales#154)#489, sum(jun_sales#155)#490, sum(jul_sales#156)#491, sum(aug_sales#157)#492, sum(sep_sales#158)#493, sum(oct_sales#159)#494, sum(nov_sales#160)#495, sum(dec_sales#161)#496, sum(CheckOverflow((promote_precision(jan_sales#150) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#497, sum(CheckOverflow((promote_precision(feb_sales#151) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#498, sum(CheckOverflow((promote_precision(mar_sales#152) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#499, sum(CheckOverflow((promote_precision(apr_sales#153) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#500, sum(CheckOverflow((promote_precision(may_sales#154) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#501, sum(CheckOverflow((promote_precision(jun_sales#155) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#502, sum(CheckOverflow((promote_precision(jul_sales#156) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#503, sum(CheckOverflow((promote_precision(aug_sales#157) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#504, sum(CheckOverflow((promote_precision(sep_sales#158) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#505, sum(CheckOverflow((promote_precision(oct_sales#159) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#506, sum(CheckOverflow((promote_precision(nov_sales#160) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#507, sum(CheckOverflow((promote_precision(dec_sales#161) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#508, sum(jan_net#162)#509, sum(feb_net#163)#510, sum(mar_net#164)#511, sum(apr_net#165)#512, sum(may_net#166)#513, sum(jun_net#167)#514, sum(jul_net#168)#515, sum(aug_net#169)#516, sum(sep_net#170)#517, sum(oct_net#171)#518, sum(nov_net#172)#519, sum(dec_net#173)#520]
+Results [44]: [w_warehouse_name#20, w_warehouse_sq_ft#21, w_city#22, w_county#23, w_state#24, w_country#25, ship_carriers#148, year#149, sum(jan_sales#150)#485 AS jan_sales#521, sum(feb_sales#151)#486 AS feb_sales#522, sum(mar_sales#152)#487 AS mar_sales#523, sum(apr_sales#153)#488 AS apr_sales#524, sum(may_sales#154)#489 AS may_sales#525, sum(jun_sales#155)#490 AS jun_sales#526, sum(jul_sales#156)#491 AS jul_sales#527, sum(aug_sales#157)#492 AS aug_sales#528, sum(sep_sales#158)#493 AS sep_sales#529, sum(oct_sales#159)#494 AS oct_sales#530, sum(nov_sales#160)#495 AS nov_sales#531, sum(dec_sales#161)#496 AS dec_sales#532, sum(CheckOverflow((promote_precision(jan_sales#150) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#497 AS jan_sales_per_sq_foot#533, sum(CheckOverflow((promote_precision(feb_sales#151) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#498 AS feb_sales_per_sq_foot#534, sum(CheckOverflow((promote_precision(mar_sales#152) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#499 AS mar_sales_per_sq_foot#535, sum(CheckOverflow((promote_precision(apr_sales#153) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#500 AS apr_sales_per_sq_foot#536, sum(CheckOverflow((promote_precision(may_sales#154) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#501 AS may_sales_per_sq_foot#537, sum(CheckOverflow((promote_precision(jun_sales#155) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#502 AS jun_sales_per_sq_foot#538, sum(CheckOverflow((promote_precision(jul_sales#156) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#503 AS jul_sales_per_sq_foot#539, sum(CheckOverflow((promote_precision(aug_sales#157) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#504 AS aug_sales_per_sq_foot#540, sum(CheckOverflow((promote_precision(sep_sales#158) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#505 AS sep_sales_per_sq_foot#541, sum(CheckOverflow((promote_precision(oct_sales#159) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#506 AS oct_sales_per_sq_foot#542, sum(CheckOverflow((promote_precision(nov_sales#160) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#507 AS nov_sales_per_sq_foot#543, sum(CheckOverflow((promote_precision(dec_sales#161) / promote_precision(cast(cast(w_warehouse_sq_ft#21 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#508 AS dec_sales_per_sq_foot#544, sum(jan_net#162)#509 AS jan_net#545, sum(feb_net#163)#510 AS feb_net#546, sum(mar_net#164)#511 AS mar_net#547, sum(apr_net#165)#512 AS apr_net#548, sum(may_net#166)#513 AS may_net#549, sum(jun_net#167)#514 AS jun_net#550, sum(jul_net#168)#515 AS jul_net#551, sum(aug_net#169)#516 AS aug_net#552, sum(sep_net#170)#517 AS sep_net#553, sum(oct_net#171)#518 AS oct_net#554, sum(nov_net#172)#519 AS nov_net#555, sum(dec_net#173)#520 AS dec_net#556]
+
+(55) TakeOrderedAndProject
+Input [44]: [w_warehouse_name#20, w_warehouse_sq_ft#21, w_city#22, w_county#23, w_state#24, w_country#25, ship_carriers#148, year#149, jan_sales#521, feb_sales#522, mar_sales#523, apr_sales#524, may_sales#525, jun_sales#526, jul_sales#527, aug_sales#528, sep_sales#529, oct_sales#530, nov_sales#531, dec_sales#532, jan_sales_per_sq_foot#533, feb_sales_per_sq_foot#534, mar_sales_per_sq_foot#535, apr_sales_per_sq_foot#536, may_sales_per_sq_foot#537, jun_sales_per_sq_foot#538, jul_sales_per_sq_foot#539, aug_sales_per_sq_foot#540, sep_sales_per_sq_foot#541, oct_sales_per_sq_foot#542, nov_sales_per_sq_foot#543, dec_sales_per_sq_foot#544, jan_net#545, feb_net#546, mar_net#547, apr_net#548, may_net#549, jun_net#550, jul_net#551, aug_net#552, sep_net#553, oct_net#554, nov_net#555, dec_net#556]
+Arguments: 100, [w_warehouse_name#20 ASC NULLS FIRST], [w_warehouse_name#20, w_warehouse_sq_ft#21, w_city#22, w_county#23, w_state#24, w_country#25, ship_carriers#148, year#149, jan_sales#521, feb_sales#522, mar_sales#523, apr_sales#524, may_sales#525, jun_sales#526, jul_sales#527, aug_sales#528, sep_sales#529, oct_sales#530, nov_sales#531, dec_sales#532, jan_sales_per_sq_foot#533, feb_sales_per_sq_foot#534, mar_sales_per_sq_foot#535, apr_sales_per_sq_foot#536, ... 20 more fields]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (56)
+
+
+(56) ReusedExchange [Reuses operator id: 21]
+Output [3]: [d_date_sk#15, d_year#16, d_moy#17]
+
+Subquery:2 Hosting operator id = 33 Hosting Expression = cs_sold_date_sk#180 IN dynamicpruning#8
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.sf100/simplified.txt
new file mode 100644
index 0000000000000..ae92c0623bd82
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.sf100/simplified.txt
@@ -0,0 +1,86 @@
+TakeOrderedAndProject [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net]
+  WholeStageCodegen (14)
+    HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(jan_sales),sum(feb_sales),sum(mar_sales),sum(apr_sales),sum(may_sales),sum(jun_sales),sum(jul_sales),sum(aug_sales),sum(sep_sales),sum(oct_sales),sum(nov_sales),sum(dec_sales),sum(CheckOverflow((promote_precision(jan_sales) / promote_precision(cast(cast(w_warehouse_sq_ft as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)),sum(CheckOverflow((promote_precision(feb_sales) / promote_precision(cast(cast(w_warehouse_sq_ft as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)),sum(CheckOverflow((promote_precision(mar_sales) / promote_precision(cast(cast(w_warehouse_sq_ft as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)),sum(CheckOverflow((promote_precision(apr_sales) / promote_precision(cast(cast(w_warehouse_sq_ft as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)),sum(CheckOverflow((promote_precision(may_sales) / promote_precision(cast(cast(w_warehouse_sq_ft as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)),sum(CheckOverflow((promote_precision(jun_sales) / promote_precision(cast(cast(w_warehouse_sq_ft as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)),sum(CheckOverflow((promote_precision(jul_sales) / promote_precision(cast(cast(w_warehouse_sq_ft as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)),sum(CheckOverflow((promote_precision(aug_sales) / promote_precision(cast(cast(w_warehouse_sq_ft as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)),sum(CheckOverflow((promote_precision(sep_sales) / promote_precision(cast(cast(w_warehouse_sq_ft as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)),sum(CheckOverflow((promote_precision(oct_sales) / promote_precision(cast(cast(w_warehouse_sq_ft as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)),sum(CheckOverflow((promote_precision(nov_sales) / promote_precision(cast(cast(w_warehouse_sq_ft as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)),sum(CheckOverflow((promote_precision(dec_sales) / promote_precision(cast(cast(w_warehouse_sq_ft as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)),sum(jan_net),sum(feb_net),sum(mar_net),sum(apr_net),sum(may_net),sum(jun_net),sum(jul_net),sum(aug_net),sum(sep_net),sum(oct_net),sum(nov_net),sum(dec_net),jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+      InputAdapter
+        Exchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year] #1
+          WholeStageCodegen (13)
+            HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+              InputAdapter
+                Union
+                  WholeStageCodegen (6)
+                    HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_moy = 1) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN CheckOverflow((promote_precision(cast(ws_net_paid as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN CheckOverflow((promote_precision(cast(ws_net_paid as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN CheckOverflow((promote_precision(cast(ws_net_paid as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN CheckOverflow((promote_precision(cast(ws_net_paid as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN CheckOverflow((promote_precision(cast(ws_net_paid as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN CheckOverflow((promote_precision(cast(ws_net_paid as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN CheckOverflow((promote_precision(cast(ws_net_paid as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN CheckOverflow((promote_precision(cast(ws_net_paid as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN CheckOverflow((promote_precision(cast(ws_net_paid as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN CheckOverflow((promote_precision(cast(ws_net_paid as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN CheckOverflow((promote_precision(cast(ws_net_paid as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN CheckOverflow((promote_precision(cast(ws_net_paid as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                      InputAdapter
+                        Exchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #2
+                          WholeStageCodegen (5)
+                            HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,ws_ext_sales_price,ws_quantity,ws_net_paid] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                              Project [ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy]
+                                BroadcastHashJoin [ws_warehouse_sk,w_warehouse_sk]
+                                  Project [ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,d_year,d_moy]
+                                    BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                      Project [ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk]
+                                        BroadcastHashJoin [ws_sold_time_sk,t_time_sk]
+                                          Project [ws_sold_time_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk]
+                                            BroadcastHashJoin [ws_ship_mode_sk,sm_ship_mode_sk]
+                                              Filter [ws_warehouse_sk,ws_sold_time_sk,ws_ship_mode_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.web_sales [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk]
+                                                      SubqueryBroadcast [d_date_sk] #1
+                                                        ReusedExchange [d_date_sk,d_year,d_moy] #3
+                                              InputAdapter
+                                                BroadcastExchange #4
+                                                  WholeStageCodegen (1)
+                                                    Project [sm_ship_mode_sk]
+                                                      Filter [sm_carrier,sm_ship_mode_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.ship_mode [sm_ship_mode_sk,sm_carrier]
+                                          InputAdapter
+                                            BroadcastExchange #5
+                                              WholeStageCodegen (2)
+                                                Project [t_time_sk]
+                                                  Filter [t_time,t_time_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.time_dim [t_time_sk,t_time]
+                                      InputAdapter
+                                        BroadcastExchange #3
+                                          WholeStageCodegen (3)
+                                            Filter [d_year,d_date_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                  InputAdapter
+                                    BroadcastExchange #6
+                                      WholeStageCodegen (4)
+                                        Filter [w_warehouse_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.warehouse [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country]
+                  WholeStageCodegen (12)
+                    HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_moy = 1) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                      InputAdapter
+                        Exchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #7
+                          WholeStageCodegen (11)
+                            HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,cs_sales_price,cs_quantity,cs_net_paid_inc_tax] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                              Project [cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy]
+                                BroadcastHashJoin [cs_warehouse_sk,w_warehouse_sk]
+                                  Project [cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,d_year,d_moy]
+                                    BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                      Project [cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk]
+                                        BroadcastHashJoin [cs_sold_time_sk,t_time_sk]
+                                          Project [cs_sold_time_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk]
+                                            BroadcastHashJoin [cs_ship_mode_sk,sm_ship_mode_sk]
+                                              Filter [cs_warehouse_sk,cs_sold_time_sk,cs_ship_mode_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.catalog_sales [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk]
+                                                      ReusedSubquery [d_date_sk] #1
+                                              InputAdapter
+                                                ReusedExchange [sm_ship_mode_sk] #4
+                                          InputAdapter
+                                            ReusedExchange [t_time_sk] #5
+                                      InputAdapter
+                                        ReusedExchange [d_date_sk,d_year,d_moy] #3
+                                  InputAdapter
+                                    ReusedExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #6
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/explain.txt
new file mode 100644
index 0000000000000..832965c1aaa31
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/explain.txt
@@ -0,0 +1,324 @@
+== Physical Plan ==
+TakeOrderedAndProject (55)
++- * HashAggregate (54)
+   +- Exchange (53)
+      +- * HashAggregate (52)
+         +- Union (51)
+            :- * HashAggregate (32)
+            :  +- Exchange (31)
+            :     +- * HashAggregate (30)
+            :        +- * Project (29)
+            :           +- * BroadcastHashJoin Inner BuildRight (28)
+            :              :- * Project (22)
+            :              :  +- * BroadcastHashJoin Inner BuildRight (21)
+            :              :     :- * Project (15)
+            :              :     :  +- * BroadcastHashJoin Inner BuildRight (14)
+            :              :     :     :- * Project (9)
+            :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+            :              :     :     :     :- * Filter (3)
+            :              :     :     :     :  +- * ColumnarToRow (2)
+            :              :     :     :     :     +- Scan parquet default.web_sales (1)
+            :              :     :     :     +- BroadcastExchange (7)
+            :              :     :     :        +- * Filter (6)
+            :              :     :     :           +- * ColumnarToRow (5)
+            :              :     :     :              +- Scan parquet default.warehouse (4)
+            :              :     :     +- BroadcastExchange (13)
+            :              :     :        +- * Filter (12)
+            :              :     :           +- * ColumnarToRow (11)
+            :              :     :              +- Scan parquet default.date_dim (10)
+            :              :     +- BroadcastExchange (20)
+            :              :        +- * Project (19)
+            :              :           +- * Filter (18)
+            :              :              +- * ColumnarToRow (17)
+            :              :                 +- Scan parquet default.time_dim (16)
+            :              +- BroadcastExchange (27)
+            :                 +- * Project (26)
+            :                    +- * Filter (25)
+            :                       +- * ColumnarToRow (24)
+            :                          +- Scan parquet default.ship_mode (23)
+            +- * HashAggregate (50)
+               +- Exchange (49)
+                  +- * HashAggregate (48)
+                     +- * Project (47)
+                        +- * BroadcastHashJoin Inner BuildRight (46)
+                           :- * Project (44)
+                           :  +- * BroadcastHashJoin Inner BuildRight (43)
+                           :     :- * Project (41)
+                           :     :  +- * BroadcastHashJoin Inner BuildRight (40)
+                           :     :     :- * Project (38)
+                           :     :     :  +- * BroadcastHashJoin Inner BuildRight (37)
+                           :     :     :     :- * Filter (35)
+                           :     :     :     :  +- * ColumnarToRow (34)
+                           :     :     :     :     +- Scan parquet default.catalog_sales (33)
+                           :     :     :     +- ReusedExchange (36)
+                           :     :     +- ReusedExchange (39)
+                           :     +- ReusedExchange (42)
+                           +- ReusedExchange (45)
+
+
+(1) Scan parquet default.web_sales
+Output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#7), dynamicpruningexpression(ws_sold_date_sk#7 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ws_warehouse_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_ship_mode_sk)]
+ReadSchema: struct<ws_sold_time_sk:int,ws_ship_mode_sk:int,ws_warehouse_sk:int,ws_quantity:int,ws_ext_sales_price:decimal(7,2),ws_net_paid:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7]
+
+(3) Filter [codegen id : 5]
+Input [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7]
+Condition : ((isnotnull(ws_warehouse_sk#3) AND isnotnull(ws_sold_time_sk#1)) AND isnotnull(ws_ship_mode_sk#2))
+
+(4) Scan parquet default.warehouse
+Output [7]: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/warehouse]
+PushedFilters: [IsNotNull(w_warehouse_sk)]
+ReadSchema: struct<w_warehouse_sk:int,w_warehouse_name:string,w_warehouse_sq_ft:int,w_city:string,w_county:string,w_state:string,w_country:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [7]: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15]
+
+(6) Filter [codegen id : 1]
+Input [7]: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15]
+Condition : isnotnull(w_warehouse_sk#9)
+
+(7) BroadcastExchange
+Input [7]: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#16]
+
+(8) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ws_warehouse_sk#3]
+Right keys [1]: [w_warehouse_sk#9]
+Join condition: None
+
+(9) Project [codegen id : 5]
+Output [12]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15]
+Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15]
+
+(10) Scan parquet default.date_dim
+Output [3]: [d_date_sk#17, d_year#18, d_moy#19]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [3]: [d_date_sk#17, d_year#18, d_moy#19]
+
+(12) Filter [codegen id : 2]
+Input [3]: [d_date_sk#17, d_year#18, d_moy#19]
+Condition : ((isnotnull(d_year#18) AND (d_year#18 = 2001)) AND isnotnull(d_date_sk#17))
+
+(13) BroadcastExchange
+Input [3]: [d_date_sk#17, d_year#18, d_moy#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#20]
+
+(14) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ws_sold_date_sk#7]
+Right keys [1]: [d_date_sk#17]
+Join condition: None
+
+(15) Project [codegen id : 5]
+Output [13]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#18, d_moy#19]
+Input [15]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_date_sk#17, d_year#18, d_moy#19]
+
+(16) Scan parquet default.time_dim
+Output [2]: [t_time_sk#21, t_time#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [IsNotNull(t_time), GreaterThanOrEqual(t_time,30838), LessThanOrEqual(t_time,59638), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_time:int>
+
+(17) ColumnarToRow [codegen id : 3]
+Input [2]: [t_time_sk#21, t_time#22]
+
+(18) Filter [codegen id : 3]
+Input [2]: [t_time_sk#21, t_time#22]
+Condition : (((isnotnull(t_time#22) AND (t_time#22 >= 30838)) AND (t_time#22 <= 59638)) AND isnotnull(t_time_sk#21))
+
+(19) Project [codegen id : 3]
+Output [1]: [t_time_sk#21]
+Input [2]: [t_time_sk#21, t_time#22]
+
+(20) BroadcastExchange
+Input [1]: [t_time_sk#21]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#23]
+
+(21) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ws_sold_time_sk#1]
+Right keys [1]: [t_time_sk#21]
+Join condition: None
+
+(22) Project [codegen id : 5]
+Output [12]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#18, d_moy#19]
+Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#18, d_moy#19, t_time_sk#21]
+
+(23) Scan parquet default.ship_mode
+Output [2]: [sm_ship_mode_sk#24, sm_carrier#25]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/ship_mode]
+PushedFilters: [In(sm_carrier, [DHL                 ,BARIAN              ]), IsNotNull(sm_ship_mode_sk)]
+ReadSchema: struct<sm_ship_mode_sk:int,sm_carrier:string>
+
+(24) ColumnarToRow [codegen id : 4]
+Input [2]: [sm_ship_mode_sk#24, sm_carrier#25]
+
+(25) Filter [codegen id : 4]
+Input [2]: [sm_ship_mode_sk#24, sm_carrier#25]
+Condition : (sm_carrier#25 IN (DHL                 ,BARIAN              ) AND isnotnull(sm_ship_mode_sk#24))
+
+(26) Project [codegen id : 4]
+Output [1]: [sm_ship_mode_sk#24]
+Input [2]: [sm_ship_mode_sk#24, sm_carrier#25]
+
+(27) BroadcastExchange
+Input [1]: [sm_ship_mode_sk#24]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#26]
+
+(28) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ws_ship_mode_sk#2]
+Right keys [1]: [sm_ship_mode_sk#24]
+Join condition: None
+
+(29) Project [codegen id : 5]
+Output [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#18, d_moy#19]
+Input [13]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#18, d_moy#19, sm_ship_mode_sk#24]
+
+(30) HashAggregate [codegen id : 5]
+Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#18, d_moy#19]
+Keys [7]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#18]
+Functions [24]: [partial_sum(CASE WHEN (d_moy#19 = 1) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 2) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 3) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 4) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 5) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 6) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 7) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 8) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 9) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 10) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 11) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 12) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 1) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 2) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 3) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 4) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 5) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 6) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 7) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 8) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 9) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 10) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 11) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 12) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)]
+Aggregate Attributes [48]: [sum#27, isEmpty#28, sum#29, isEmpty#30, sum#31, isEmpty#32, sum#33, isEmpty#34, sum#35, isEmpty#36, sum#37, isEmpty#38, sum#39, isEmpty#40, sum#41, isEmpty#42, sum#43, isEmpty#44, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, isEmpty#54, sum#55, isEmpty#56, sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68, sum#69, isEmpty#70, sum#71, isEmpty#72, sum#73, isEmpty#74]
+Results [55]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#18, sum#75, isEmpty#76, sum#77, isEmpty#78, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122]
+
+(31) Exchange
+Input [55]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#18, sum#75, isEmpty#76, sum#77, isEmpty#78, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122]
+Arguments: hashpartitioning(w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#18, 5), ENSURE_REQUIREMENTS, [id=#123]
+
+(32) HashAggregate [codegen id : 6]
+Input [55]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#18, sum#75, isEmpty#76, sum#77, isEmpty#78, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122]
+Keys [7]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#18]
+Functions [24]: [sum(CASE WHEN (d_moy#19 = 1) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 2) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 3) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 4) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 5) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 6) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 7) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 8) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 9) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 10) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 11) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 12) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 1) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 2) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 3) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 4) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 5) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 6) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 7) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 8) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 9) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 10) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 11) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 12) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)]
+Aggregate Attributes [24]: [sum(CASE WHEN (d_moy#19 = 1) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#124, sum(CASE WHEN (d_moy#19 = 2) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#125, sum(CASE WHEN (d_moy#19 = 3) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#126, sum(CASE WHEN (d_moy#19 = 4) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#127, sum(CASE WHEN (d_moy#19 = 5) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#128, sum(CASE WHEN (d_moy#19 = 6) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#129, sum(CASE WHEN (d_moy#19 = 7) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#130, sum(CASE WHEN (d_moy#19 = 8) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#131, sum(CASE WHEN (d_moy#19 = 9) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#132, sum(CASE WHEN (d_moy#19 = 10) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#133, sum(CASE WHEN (d_moy#19 = 11) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#134, sum(CASE WHEN (d_moy#19 = 12) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#135, sum(CASE WHEN (d_moy#19 = 1) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#136, sum(CASE WHEN (d_moy#19 = 2) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#137, sum(CASE WHEN (d_moy#19 = 3) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#138, sum(CASE WHEN (d_moy#19 = 4) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#139, sum(CASE WHEN (d_moy#19 = 5) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#140, sum(CASE WHEN (d_moy#19 = 6) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#141, sum(CASE WHEN (d_moy#19 = 7) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#142, sum(CASE WHEN (d_moy#19 = 8) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#143, sum(CASE WHEN (d_moy#19 = 9) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#144, sum(CASE WHEN (d_moy#19 = 10) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#145, sum(CASE WHEN (d_moy#19 = 11) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#146, sum(CASE WHEN (d_moy#19 = 12) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#147]
+Results [32]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, DHL,BARIAN AS ship_carriers#148, d_year#18 AS year#149, sum(CASE WHEN (d_moy#19 = 1) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#124 AS jan_sales#150, sum(CASE WHEN (d_moy#19 = 2) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#125 AS feb_sales#151, sum(CASE WHEN (d_moy#19 = 3) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#126 AS mar_sales#152, sum(CASE WHEN (d_moy#19 = 4) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#127 AS apr_sales#153, sum(CASE WHEN (d_moy#19 = 5) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#128 AS may_sales#154, sum(CASE WHEN (d_moy#19 = 6) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#129 AS jun_sales#155, sum(CASE WHEN (d_moy#19 = 7) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#130 AS jul_sales#156, sum(CASE WHEN (d_moy#19 = 8) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#131 AS aug_sales#157, sum(CASE WHEN (d_moy#19 = 9) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#132 AS sep_sales#158, sum(CASE WHEN (d_moy#19 = 10) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#133 AS oct_sales#159, sum(CASE WHEN (d_moy#19 = 11) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#134 AS nov_sales#160, sum(CASE WHEN (d_moy#19 = 12) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price#5 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#135 AS dec_sales#161, sum(CASE WHEN (d_moy#19 = 1) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#136 AS jan_net#162, sum(CASE WHEN (d_moy#19 = 2) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#137 AS feb_net#163, sum(CASE WHEN (d_moy#19 = 3) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#138 AS mar_net#164, sum(CASE WHEN (d_moy#19 = 4) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#139 AS apr_net#165, sum(CASE WHEN (d_moy#19 = 5) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#140 AS may_net#166, sum(CASE WHEN (d_moy#19 = 6) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#141 AS jun_net#167, sum(CASE WHEN (d_moy#19 = 7) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#142 AS jul_net#168, sum(CASE WHEN (d_moy#19 = 8) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#143 AS aug_net#169, sum(CASE WHEN (d_moy#19 = 9) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#144 AS sep_net#170, sum(CASE WHEN (d_moy#19 = 10) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#145 AS oct_net#171, sum(CASE WHEN (d_moy#19 = 11) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#146 AS nov_net#172, sum(CASE WHEN (d_moy#19 = 12) THEN CheckOverflow((promote_precision(cast(ws_net_paid#6 as decimal(12,2))) * promote_precision(cast(cast(ws_quantity#4 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#147 AS dec_net#173]
+
+(33) Scan parquet default.catalog_sales
+Output [7]: [cs_sold_time_sk#174, cs_ship_mode_sk#175, cs_warehouse_sk#176, cs_quantity#177, cs_sales_price#178, cs_net_paid_inc_tax#179, cs_sold_date_sk#180]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#180), dynamicpruningexpression(cs_sold_date_sk#180 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_sold_time_sk), IsNotNull(cs_ship_mode_sk)]
+ReadSchema: struct<cs_sold_time_sk:int,cs_ship_mode_sk:int,cs_warehouse_sk:int,cs_quantity:int,cs_sales_price:decimal(7,2),cs_net_paid_inc_tax:decimal(7,2)>
+
+(34) ColumnarToRow [codegen id : 11]
+Input [7]: [cs_sold_time_sk#174, cs_ship_mode_sk#175, cs_warehouse_sk#176, cs_quantity#177, cs_sales_price#178, cs_net_paid_inc_tax#179, cs_sold_date_sk#180]
+
+(35) Filter [codegen id : 11]
+Input [7]: [cs_sold_time_sk#174, cs_ship_mode_sk#175, cs_warehouse_sk#176, cs_quantity#177, cs_sales_price#178, cs_net_paid_inc_tax#179, cs_sold_date_sk#180]
+Condition : ((isnotnull(cs_warehouse_sk#176) AND isnotnull(cs_sold_time_sk#174)) AND isnotnull(cs_ship_mode_sk#175))
+
+(36) ReusedExchange [Reuses operator id: 7]
+Output [7]: [w_warehouse_sk#181, w_warehouse_name#182, w_warehouse_sq_ft#183, w_city#184, w_county#185, w_state#186, w_country#187]
+
+(37) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_warehouse_sk#176]
+Right keys [1]: [w_warehouse_sk#181]
+Join condition: None
+
+(38) Project [codegen id : 11]
+Output [12]: [cs_sold_time_sk#174, cs_ship_mode_sk#175, cs_quantity#177, cs_sales_price#178, cs_net_paid_inc_tax#179, cs_sold_date_sk#180, w_warehouse_name#182, w_warehouse_sq_ft#183, w_city#184, w_county#185, w_state#186, w_country#187]
+Input [14]: [cs_sold_time_sk#174, cs_ship_mode_sk#175, cs_warehouse_sk#176, cs_quantity#177, cs_sales_price#178, cs_net_paid_inc_tax#179, cs_sold_date_sk#180, w_warehouse_sk#181, w_warehouse_name#182, w_warehouse_sq_ft#183, w_city#184, w_county#185, w_state#186, w_country#187]
+
+(39) ReusedExchange [Reuses operator id: 13]
+Output [3]: [d_date_sk#188, d_year#189, d_moy#190]
+
+(40) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_sold_date_sk#180]
+Right keys [1]: [d_date_sk#188]
+Join condition: None
+
+(41) Project [codegen id : 11]
+Output [13]: [cs_sold_time_sk#174, cs_ship_mode_sk#175, cs_quantity#177, cs_sales_price#178, cs_net_paid_inc_tax#179, w_warehouse_name#182, w_warehouse_sq_ft#183, w_city#184, w_county#185, w_state#186, w_country#187, d_year#189, d_moy#190]
+Input [15]: [cs_sold_time_sk#174, cs_ship_mode_sk#175, cs_quantity#177, cs_sales_price#178, cs_net_paid_inc_tax#179, cs_sold_date_sk#180, w_warehouse_name#182, w_warehouse_sq_ft#183, w_city#184, w_county#185, w_state#186, w_country#187, d_date_sk#188, d_year#189, d_moy#190]
+
+(42) ReusedExchange [Reuses operator id: 20]
+Output [1]: [t_time_sk#191]
+
+(43) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_sold_time_sk#174]
+Right keys [1]: [t_time_sk#191]
+Join condition: None
+
+(44) Project [codegen id : 11]
+Output [12]: [cs_ship_mode_sk#175, cs_quantity#177, cs_sales_price#178, cs_net_paid_inc_tax#179, w_warehouse_name#182, w_warehouse_sq_ft#183, w_city#184, w_county#185, w_state#186, w_country#187, d_year#189, d_moy#190]
+Input [14]: [cs_sold_time_sk#174, cs_ship_mode_sk#175, cs_quantity#177, cs_sales_price#178, cs_net_paid_inc_tax#179, w_warehouse_name#182, w_warehouse_sq_ft#183, w_city#184, w_county#185, w_state#186, w_country#187, d_year#189, d_moy#190, t_time_sk#191]
+
+(45) ReusedExchange [Reuses operator id: 27]
+Output [1]: [sm_ship_mode_sk#192]
+
+(46) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_ship_mode_sk#175]
+Right keys [1]: [sm_ship_mode_sk#192]
+Join condition: None
+
+(47) Project [codegen id : 11]
+Output [11]: [cs_quantity#177, cs_sales_price#178, cs_net_paid_inc_tax#179, w_warehouse_name#182, w_warehouse_sq_ft#183, w_city#184, w_county#185, w_state#186, w_country#187, d_year#189, d_moy#190]
+Input [13]: [cs_ship_mode_sk#175, cs_quantity#177, cs_sales_price#178, cs_net_paid_inc_tax#179, w_warehouse_name#182, w_warehouse_sq_ft#183, w_city#184, w_county#185, w_state#186, w_country#187, d_year#189, d_moy#190, sm_ship_mode_sk#192]
+
+(48) HashAggregate [codegen id : 11]
+Input [11]: [cs_quantity#177, cs_sales_price#178, cs_net_paid_inc_tax#179, w_warehouse_name#182, w_warehouse_sq_ft#183, w_city#184, w_county#185, w_state#186, w_country#187, d_year#189, d_moy#190]
+Keys [7]: [w_warehouse_name#182, w_warehouse_sq_ft#183, w_city#184, w_county#185, w_state#186, w_country#187, d_year#189]
+Functions [24]: [partial_sum(CASE WHEN (d_moy#190 = 1) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#190 = 2) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#190 = 3) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#190 = 4) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#190 = 5) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#190 = 6) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#190 = 7) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#190 = 8) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#190 = 9) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#190 = 10) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#190 = 11) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#190 = 12) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#190 = 1) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#190 = 2) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#190 = 3) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#190 = 4) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#190 = 5) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#190 = 6) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#190 = 7) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#190 = 8) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#190 = 9) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#190 = 10) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#190 = 11) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#190 = 12) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)]
+Aggregate Attributes [48]: [sum#193, isEmpty#194, sum#195, isEmpty#196, sum#197, isEmpty#198, sum#199, isEmpty#200, sum#201, isEmpty#202, sum#203, isEmpty#204, sum#205, isEmpty#206, sum#207, isEmpty#208, sum#209, isEmpty#210, sum#211, isEmpty#212, sum#213, isEmpty#214, sum#215, isEmpty#216, sum#217, isEmpty#218, sum#219, isEmpty#220, sum#221, isEmpty#222, sum#223, isEmpty#224, sum#225, isEmpty#226, sum#227, isEmpty#228, sum#229, isEmpty#230, sum#231, isEmpty#232, sum#233, isEmpty#234, sum#235, isEmpty#236, sum#237, isEmpty#238, sum#239, isEmpty#240]
+Results [55]: [w_warehouse_name#182, w_warehouse_sq_ft#183, w_city#184, w_county#185, w_state#186, w_country#187, d_year#189, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260, sum#261, isEmpty#262, sum#263, isEmpty#264, sum#265, isEmpty#266, sum#267, isEmpty#268, sum#269, isEmpty#270, sum#271, isEmpty#272, sum#273, isEmpty#274, sum#275, isEmpty#276, sum#277, isEmpty#278, sum#279, isEmpty#280, sum#281, isEmpty#282, sum#283, isEmpty#284, sum#285, isEmpty#286, sum#287, isEmpty#288]
+
+(49) Exchange
+Input [55]: [w_warehouse_name#182, w_warehouse_sq_ft#183, w_city#184, w_county#185, w_state#186, w_country#187, d_year#189, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260, sum#261, isEmpty#262, sum#263, isEmpty#264, sum#265, isEmpty#266, sum#267, isEmpty#268, sum#269, isEmpty#270, sum#271, isEmpty#272, sum#273, isEmpty#274, sum#275, isEmpty#276, sum#277, isEmpty#278, sum#279, isEmpty#280, sum#281, isEmpty#282, sum#283, isEmpty#284, sum#285, isEmpty#286, sum#287, isEmpty#288]
+Arguments: hashpartitioning(w_warehouse_name#182, w_warehouse_sq_ft#183, w_city#184, w_county#185, w_state#186, w_country#187, d_year#189, 5), ENSURE_REQUIREMENTS, [id=#289]
+
+(50) HashAggregate [codegen id : 12]
+Input [55]: [w_warehouse_name#182, w_warehouse_sq_ft#183, w_city#184, w_county#185, w_state#186, w_country#187, d_year#189, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260, sum#261, isEmpty#262, sum#263, isEmpty#264, sum#265, isEmpty#266, sum#267, isEmpty#268, sum#269, isEmpty#270, sum#271, isEmpty#272, sum#273, isEmpty#274, sum#275, isEmpty#276, sum#277, isEmpty#278, sum#279, isEmpty#280, sum#281, isEmpty#282, sum#283, isEmpty#284, sum#285, isEmpty#286, sum#287, isEmpty#288]
+Keys [7]: [w_warehouse_name#182, w_warehouse_sq_ft#183, w_city#184, w_county#185, w_state#186, w_country#187, d_year#189]
+Functions [24]: [sum(CASE WHEN (d_moy#190 = 1) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#190 = 2) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#190 = 3) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#190 = 4) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#190 = 5) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#190 = 6) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#190 = 7) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#190 = 8) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#190 = 9) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#190 = 10) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#190 = 11) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#190 = 12) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#190 = 1) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#190 = 2) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#190 = 3) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#190 = 4) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#190 = 5) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#190 = 6) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#190 = 7) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#190 = 8) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#190 = 9) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#190 = 10) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#190 = 11) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END), sum(CASE WHEN (d_moy#190 = 12) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)]
+Aggregate Attributes [24]: [sum(CASE WHEN (d_moy#190 = 1) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#290, sum(CASE WHEN (d_moy#190 = 2) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#291, sum(CASE WHEN (d_moy#190 = 3) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#292, sum(CASE WHEN (d_moy#190 = 4) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#293, sum(CASE WHEN (d_moy#190 = 5) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#294, sum(CASE WHEN (d_moy#190 = 6) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#295, sum(CASE WHEN (d_moy#190 = 7) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#296, sum(CASE WHEN (d_moy#190 = 8) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#297, sum(CASE WHEN (d_moy#190 = 9) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#298, sum(CASE WHEN (d_moy#190 = 10) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#299, sum(CASE WHEN (d_moy#190 = 11) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#300, sum(CASE WHEN (d_moy#190 = 12) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#301, sum(CASE WHEN (d_moy#190 = 1) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#302, sum(CASE WHEN (d_moy#190 = 2) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#303, sum(CASE WHEN (d_moy#190 = 3) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#304, sum(CASE WHEN (d_moy#190 = 4) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#305, sum(CASE WHEN (d_moy#190 = 5) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#306, sum(CASE WHEN (d_moy#190 = 6) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#307, sum(CASE WHEN (d_moy#190 = 7) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#308, sum(CASE WHEN (d_moy#190 = 8) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#309, sum(CASE WHEN (d_moy#190 = 9) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#310, sum(CASE WHEN (d_moy#190 = 10) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#311, sum(CASE WHEN (d_moy#190 = 11) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#312, sum(CASE WHEN (d_moy#190 = 12) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#313]
+Results [32]: [w_warehouse_name#182, w_warehouse_sq_ft#183, w_city#184, w_county#185, w_state#186, w_country#187, DHL,BARIAN AS ship_carriers#314, d_year#189 AS year#315, sum(CASE WHEN (d_moy#190 = 1) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#290 AS jan_sales#316, sum(CASE WHEN (d_moy#190 = 2) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#291 AS feb_sales#317, sum(CASE WHEN (d_moy#190 = 3) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#292 AS mar_sales#318, sum(CASE WHEN (d_moy#190 = 4) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#293 AS apr_sales#319, sum(CASE WHEN (d_moy#190 = 5) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#294 AS may_sales#320, sum(CASE WHEN (d_moy#190 = 6) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#295 AS jun_sales#321, sum(CASE WHEN (d_moy#190 = 7) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#296 AS jul_sales#322, sum(CASE WHEN (d_moy#190 = 8) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#297 AS aug_sales#323, sum(CASE WHEN (d_moy#190 = 9) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#298 AS sep_sales#324, sum(CASE WHEN (d_moy#190 = 10) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#299 AS oct_sales#325, sum(CASE WHEN (d_moy#190 = 11) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#300 AS nov_sales#326, sum(CASE WHEN (d_moy#190 = 12) THEN CheckOverflow((promote_precision(cast(cs_sales_price#178 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#301 AS dec_sales#327, sum(CASE WHEN (d_moy#190 = 1) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#302 AS jan_net#328, sum(CASE WHEN (d_moy#190 = 2) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#303 AS feb_net#329, sum(CASE WHEN (d_moy#190 = 3) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#304 AS mar_net#330, sum(CASE WHEN (d_moy#190 = 4) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#305 AS apr_net#331, sum(CASE WHEN (d_moy#190 = 5) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#306 AS may_net#332, sum(CASE WHEN (d_moy#190 = 6) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#307 AS jun_net#333, sum(CASE WHEN (d_moy#190 = 7) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#308 AS jul_net#334, sum(CASE WHEN (d_moy#190 = 8) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#309 AS aug_net#335, sum(CASE WHEN (d_moy#190 = 9) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#310 AS sep_net#336, sum(CASE WHEN (d_moy#190 = 10) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#311 AS oct_net#337, sum(CASE WHEN (d_moy#190 = 11) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#312 AS nov_net#338, sum(CASE WHEN (d_moy#190 = 12) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax#179 as decimal(12,2))) * promote_precision(cast(cast(cs_quantity#177 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END)#313 AS dec_net#339]
+
+(51) Union
+
+(52) HashAggregate [codegen id : 13]
+Input [32]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#148, year#149, jan_sales#150, feb_sales#151, mar_sales#152, apr_sales#153, may_sales#154, jun_sales#155, jul_sales#156, aug_sales#157, sep_sales#158, oct_sales#159, nov_sales#160, dec_sales#161, jan_net#162, feb_net#163, mar_net#164, apr_net#165, may_net#166, jun_net#167, jul_net#168, aug_net#169, sep_net#170, oct_net#171, nov_net#172, dec_net#173]
+Keys [8]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#148, year#149]
+Functions [36]: [partial_sum(jan_sales#150), partial_sum(feb_sales#151), partial_sum(mar_sales#152), partial_sum(apr_sales#153), partial_sum(may_sales#154), partial_sum(jun_sales#155), partial_sum(jul_sales#156), partial_sum(aug_sales#157), partial_sum(sep_sales#158), partial_sum(oct_sales#159), partial_sum(nov_sales#160), partial_sum(dec_sales#161), partial_sum(CheckOverflow((promote_precision(jan_sales#150) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), partial_sum(CheckOverflow((promote_precision(feb_sales#151) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), partial_sum(CheckOverflow((promote_precision(mar_sales#152) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), partial_sum(CheckOverflow((promote_precision(apr_sales#153) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), partial_sum(CheckOverflow((promote_precision(may_sales#154) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), partial_sum(CheckOverflow((promote_precision(jun_sales#155) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), partial_sum(CheckOverflow((promote_precision(jul_sales#156) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), partial_sum(CheckOverflow((promote_precision(aug_sales#157) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), partial_sum(CheckOverflow((promote_precision(sep_sales#158) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), partial_sum(CheckOverflow((promote_precision(oct_sales#159) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), partial_sum(CheckOverflow((promote_precision(nov_sales#160) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), partial_sum(CheckOverflow((promote_precision(dec_sales#161) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), partial_sum(jan_net#162), partial_sum(feb_net#163), partial_sum(mar_net#164), partial_sum(apr_net#165), partial_sum(may_net#166), partial_sum(jun_net#167), partial_sum(jul_net#168), partial_sum(aug_net#169), partial_sum(sep_net#170), partial_sum(oct_net#171), partial_sum(nov_net#172), partial_sum(dec_net#173)]
+Aggregate Attributes [72]: [sum#340, isEmpty#341, sum#342, isEmpty#343, sum#344, isEmpty#345, sum#346, isEmpty#347, sum#348, isEmpty#349, sum#350, isEmpty#351, sum#352, isEmpty#353, sum#354, isEmpty#355, sum#356, isEmpty#357, sum#358, isEmpty#359, sum#360, isEmpty#361, sum#362, isEmpty#363, sum#364, isEmpty#365, sum#366, isEmpty#367, sum#368, isEmpty#369, sum#370, isEmpty#371, sum#372, isEmpty#373, sum#374, isEmpty#375, sum#376, isEmpty#377, sum#378, isEmpty#379, sum#380, isEmpty#381, sum#382, isEmpty#383, sum#384, isEmpty#385, sum#386, isEmpty#387, sum#388, isEmpty#389, sum#390, isEmpty#391, sum#392, isEmpty#393, sum#394, isEmpty#395, sum#396, isEmpty#397, sum#398, isEmpty#399, sum#400, isEmpty#401, sum#402, isEmpty#403, sum#404, isEmpty#405, sum#406, isEmpty#407, sum#408, isEmpty#409, sum#410, isEmpty#411]
+Results [80]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#148, year#149, sum#412, isEmpty#413, sum#414, isEmpty#415, sum#416, isEmpty#417, sum#418, isEmpty#419, sum#420, isEmpty#421, sum#422, isEmpty#423, sum#424, isEmpty#425, sum#426, isEmpty#427, sum#428, isEmpty#429, sum#430, isEmpty#431, sum#432, isEmpty#433, sum#434, isEmpty#435, sum#436, isEmpty#437, sum#438, isEmpty#439, sum#440, isEmpty#441, sum#442, isEmpty#443, sum#444, isEmpty#445, sum#446, isEmpty#447, sum#448, isEmpty#449, sum#450, isEmpty#451, sum#452, isEmpty#453, sum#454, isEmpty#455, sum#456, isEmpty#457, sum#458, isEmpty#459, sum#460, isEmpty#461, sum#462, isEmpty#463, sum#464, isEmpty#465, sum#466, isEmpty#467, sum#468, isEmpty#469, sum#470, isEmpty#471, sum#472, isEmpty#473, sum#474, isEmpty#475, sum#476, isEmpty#477, sum#478, isEmpty#479, sum#480, isEmpty#481, sum#482, isEmpty#483]
+
+(53) Exchange
+Input [80]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#148, year#149, sum#412, isEmpty#413, sum#414, isEmpty#415, sum#416, isEmpty#417, sum#418, isEmpty#419, sum#420, isEmpty#421, sum#422, isEmpty#423, sum#424, isEmpty#425, sum#426, isEmpty#427, sum#428, isEmpty#429, sum#430, isEmpty#431, sum#432, isEmpty#433, sum#434, isEmpty#435, sum#436, isEmpty#437, sum#438, isEmpty#439, sum#440, isEmpty#441, sum#442, isEmpty#443, sum#444, isEmpty#445, sum#446, isEmpty#447, sum#448, isEmpty#449, sum#450, isEmpty#451, sum#452, isEmpty#453, sum#454, isEmpty#455, sum#456, isEmpty#457, sum#458, isEmpty#459, sum#460, isEmpty#461, sum#462, isEmpty#463, sum#464, isEmpty#465, sum#466, isEmpty#467, sum#468, isEmpty#469, sum#470, isEmpty#471, sum#472, isEmpty#473, sum#474, isEmpty#475, sum#476, isEmpty#477, sum#478, isEmpty#479, sum#480, isEmpty#481, sum#482, isEmpty#483]
+Arguments: hashpartitioning(w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#148, year#149, 5), ENSURE_REQUIREMENTS, [id=#484]
+
+(54) HashAggregate [codegen id : 14]
+Input [80]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#148, year#149, sum#412, isEmpty#413, sum#414, isEmpty#415, sum#416, isEmpty#417, sum#418, isEmpty#419, sum#420, isEmpty#421, sum#422, isEmpty#423, sum#424, isEmpty#425, sum#426, isEmpty#427, sum#428, isEmpty#429, sum#430, isEmpty#431, sum#432, isEmpty#433, sum#434, isEmpty#435, sum#436, isEmpty#437, sum#438, isEmpty#439, sum#440, isEmpty#441, sum#442, isEmpty#443, sum#444, isEmpty#445, sum#446, isEmpty#447, sum#448, isEmpty#449, sum#450, isEmpty#451, sum#452, isEmpty#453, sum#454, isEmpty#455, sum#456, isEmpty#457, sum#458, isEmpty#459, sum#460, isEmpty#461, sum#462, isEmpty#463, sum#464, isEmpty#465, sum#466, isEmpty#467, sum#468, isEmpty#469, sum#470, isEmpty#471, sum#472, isEmpty#473, sum#474, isEmpty#475, sum#476, isEmpty#477, sum#478, isEmpty#479, sum#480, isEmpty#481, sum#482, isEmpty#483]
+Keys [8]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#148, year#149]
+Functions [36]: [sum(jan_sales#150), sum(feb_sales#151), sum(mar_sales#152), sum(apr_sales#153), sum(may_sales#154), sum(jun_sales#155), sum(jul_sales#156), sum(aug_sales#157), sum(sep_sales#158), sum(oct_sales#159), sum(nov_sales#160), sum(dec_sales#161), sum(CheckOverflow((promote_precision(jan_sales#150) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), sum(CheckOverflow((promote_precision(feb_sales#151) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), sum(CheckOverflow((promote_precision(mar_sales#152) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), sum(CheckOverflow((promote_precision(apr_sales#153) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), sum(CheckOverflow((promote_precision(may_sales#154) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), sum(CheckOverflow((promote_precision(jun_sales#155) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), sum(CheckOverflow((promote_precision(jul_sales#156) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), sum(CheckOverflow((promote_precision(aug_sales#157) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), sum(CheckOverflow((promote_precision(sep_sales#158) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), sum(CheckOverflow((promote_precision(oct_sales#159) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), sum(CheckOverflow((promote_precision(nov_sales#160) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), sum(CheckOverflow((promote_precision(dec_sales#161) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)), sum(jan_net#162), sum(feb_net#163), sum(mar_net#164), sum(apr_net#165), sum(may_net#166), sum(jun_net#167), sum(jul_net#168), sum(aug_net#169), sum(sep_net#170), sum(oct_net#171), sum(nov_net#172), sum(dec_net#173)]
+Aggregate Attributes [36]: [sum(jan_sales#150)#485, sum(feb_sales#151)#486, sum(mar_sales#152)#487, sum(apr_sales#153)#488, sum(may_sales#154)#489, sum(jun_sales#155)#490, sum(jul_sales#156)#491, sum(aug_sales#157)#492, sum(sep_sales#158)#493, sum(oct_sales#159)#494, sum(nov_sales#160)#495, sum(dec_sales#161)#496, sum(CheckOverflow((promote_precision(jan_sales#150) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#497, sum(CheckOverflow((promote_precision(feb_sales#151) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#498, sum(CheckOverflow((promote_precision(mar_sales#152) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#499, sum(CheckOverflow((promote_precision(apr_sales#153) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#500, sum(CheckOverflow((promote_precision(may_sales#154) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#501, sum(CheckOverflow((promote_precision(jun_sales#155) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#502, sum(CheckOverflow((promote_precision(jul_sales#156) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#503, sum(CheckOverflow((promote_precision(aug_sales#157) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#504, sum(CheckOverflow((promote_precision(sep_sales#158) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#505, sum(CheckOverflow((promote_precision(oct_sales#159) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#506, sum(CheckOverflow((promote_precision(nov_sales#160) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#507, sum(CheckOverflow((promote_precision(dec_sales#161) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#508, sum(jan_net#162)#509, sum(feb_net#163)#510, sum(mar_net#164)#511, sum(apr_net#165)#512, sum(may_net#166)#513, sum(jun_net#167)#514, sum(jul_net#168)#515, sum(aug_net#169)#516, sum(sep_net#170)#517, sum(oct_net#171)#518, sum(nov_net#172)#519, sum(dec_net#173)#520]
+Results [44]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#148, year#149, sum(jan_sales#150)#485 AS jan_sales#521, sum(feb_sales#151)#486 AS feb_sales#522, sum(mar_sales#152)#487 AS mar_sales#523, sum(apr_sales#153)#488 AS apr_sales#524, sum(may_sales#154)#489 AS may_sales#525, sum(jun_sales#155)#490 AS jun_sales#526, sum(jul_sales#156)#491 AS jul_sales#527, sum(aug_sales#157)#492 AS aug_sales#528, sum(sep_sales#158)#493 AS sep_sales#529, sum(oct_sales#159)#494 AS oct_sales#530, sum(nov_sales#160)#495 AS nov_sales#531, sum(dec_sales#161)#496 AS dec_sales#532, sum(CheckOverflow((promote_precision(jan_sales#150) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#497 AS jan_sales_per_sq_foot#533, sum(CheckOverflow((promote_precision(feb_sales#151) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#498 AS feb_sales_per_sq_foot#534, sum(CheckOverflow((promote_precision(mar_sales#152) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#499 AS mar_sales_per_sq_foot#535, sum(CheckOverflow((promote_precision(apr_sales#153) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#500 AS apr_sales_per_sq_foot#536, sum(CheckOverflow((promote_precision(may_sales#154) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#501 AS may_sales_per_sq_foot#537, sum(CheckOverflow((promote_precision(jun_sales#155) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#502 AS jun_sales_per_sq_foot#538, sum(CheckOverflow((promote_precision(jul_sales#156) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#503 AS jul_sales_per_sq_foot#539, sum(CheckOverflow((promote_precision(aug_sales#157) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#504 AS aug_sales_per_sq_foot#540, sum(CheckOverflow((promote_precision(sep_sales#158) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#505 AS sep_sales_per_sq_foot#541, sum(CheckOverflow((promote_precision(oct_sales#159) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#506 AS oct_sales_per_sq_foot#542, sum(CheckOverflow((promote_precision(nov_sales#160) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#507 AS nov_sales_per_sq_foot#543, sum(CheckOverflow((promote_precision(dec_sales#161) / promote_precision(cast(cast(w_warehouse_sq_ft#11 as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true))#508 AS dec_sales_per_sq_foot#544, sum(jan_net#162)#509 AS jan_net#545, sum(feb_net#163)#510 AS feb_net#546, sum(mar_net#164)#511 AS mar_net#547, sum(apr_net#165)#512 AS apr_net#548, sum(may_net#166)#513 AS may_net#549, sum(jun_net#167)#514 AS jun_net#550, sum(jul_net#168)#515 AS jul_net#551, sum(aug_net#169)#516 AS aug_net#552, sum(sep_net#170)#517 AS sep_net#553, sum(oct_net#171)#518 AS oct_net#554, sum(nov_net#172)#519 AS nov_net#555, sum(dec_net#173)#520 AS dec_net#556]
+
+(55) TakeOrderedAndProject
+Input [44]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#148, year#149, jan_sales#521, feb_sales#522, mar_sales#523, apr_sales#524, may_sales#525, jun_sales#526, jul_sales#527, aug_sales#528, sep_sales#529, oct_sales#530, nov_sales#531, dec_sales#532, jan_sales_per_sq_foot#533, feb_sales_per_sq_foot#534, mar_sales_per_sq_foot#535, apr_sales_per_sq_foot#536, may_sales_per_sq_foot#537, jun_sales_per_sq_foot#538, jul_sales_per_sq_foot#539, aug_sales_per_sq_foot#540, sep_sales_per_sq_foot#541, oct_sales_per_sq_foot#542, nov_sales_per_sq_foot#543, dec_sales_per_sq_foot#544, jan_net#545, feb_net#546, mar_net#547, apr_net#548, may_net#549, jun_net#550, jul_net#551, aug_net#552, sep_net#553, oct_net#554, nov_net#555, dec_net#556]
+Arguments: 100, [w_warehouse_name#10 ASC NULLS FIRST], [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#148, year#149, jan_sales#521, feb_sales#522, mar_sales#523, apr_sales#524, may_sales#525, jun_sales#526, jul_sales#527, aug_sales#528, sep_sales#529, oct_sales#530, nov_sales#531, dec_sales#532, jan_sales_per_sq_foot#533, feb_sales_per_sq_foot#534, mar_sales_per_sq_foot#535, apr_sales_per_sq_foot#536, ... 20 more fields]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (56)
+
+
+(56) ReusedExchange [Reuses operator id: 13]
+Output [3]: [d_date_sk#17, d_year#18, d_moy#19]
+
+Subquery:2 Hosting operator id = 33 Hosting Expression = cs_sold_date_sk#180 IN dynamicpruning#8
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/simplified.txt
new file mode 100644
index 0000000000000..ae0b56b8aa037
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/simplified.txt
@@ -0,0 +1,86 @@
+TakeOrderedAndProject [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net]
+  WholeStageCodegen (14)
+    HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(jan_sales),sum(feb_sales),sum(mar_sales),sum(apr_sales),sum(may_sales),sum(jun_sales),sum(jul_sales),sum(aug_sales),sum(sep_sales),sum(oct_sales),sum(nov_sales),sum(dec_sales),sum(CheckOverflow((promote_precision(jan_sales) / promote_precision(cast(cast(w_warehouse_sq_ft as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)),sum(CheckOverflow((promote_precision(feb_sales) / promote_precision(cast(cast(w_warehouse_sq_ft as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)),sum(CheckOverflow((promote_precision(mar_sales) / promote_precision(cast(cast(w_warehouse_sq_ft as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)),sum(CheckOverflow((promote_precision(apr_sales) / promote_precision(cast(cast(w_warehouse_sq_ft as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)),sum(CheckOverflow((promote_precision(may_sales) / promote_precision(cast(cast(w_warehouse_sq_ft as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)),sum(CheckOverflow((promote_precision(jun_sales) / promote_precision(cast(cast(w_warehouse_sq_ft as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)),sum(CheckOverflow((promote_precision(jul_sales) / promote_precision(cast(cast(w_warehouse_sq_ft as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)),sum(CheckOverflow((promote_precision(aug_sales) / promote_precision(cast(cast(w_warehouse_sq_ft as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)),sum(CheckOverflow((promote_precision(sep_sales) / promote_precision(cast(cast(w_warehouse_sq_ft as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)),sum(CheckOverflow((promote_precision(oct_sales) / promote_precision(cast(cast(w_warehouse_sq_ft as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)),sum(CheckOverflow((promote_precision(nov_sales) / promote_precision(cast(cast(w_warehouse_sq_ft as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)),sum(CheckOverflow((promote_precision(dec_sales) / promote_precision(cast(cast(w_warehouse_sq_ft as decimal(10,0)) as decimal(28,2)))), DecimalType(38,12), true)),sum(jan_net),sum(feb_net),sum(mar_net),sum(apr_net),sum(may_net),sum(jun_net),sum(jul_net),sum(aug_net),sum(sep_net),sum(oct_net),sum(nov_net),sum(dec_net),jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+      InputAdapter
+        Exchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year] #1
+          WholeStageCodegen (13)
+            HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+              InputAdapter
+                Union
+                  WholeStageCodegen (6)
+                    HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_moy = 1) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN CheckOverflow((promote_precision(cast(ws_ext_sales_price as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN CheckOverflow((promote_precision(cast(ws_net_paid as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN CheckOverflow((promote_precision(cast(ws_net_paid as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN CheckOverflow((promote_precision(cast(ws_net_paid as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN CheckOverflow((promote_precision(cast(ws_net_paid as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN CheckOverflow((promote_precision(cast(ws_net_paid as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN CheckOverflow((promote_precision(cast(ws_net_paid as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN CheckOverflow((promote_precision(cast(ws_net_paid as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN CheckOverflow((promote_precision(cast(ws_net_paid as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN CheckOverflow((promote_precision(cast(ws_net_paid as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN CheckOverflow((promote_precision(cast(ws_net_paid as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN CheckOverflow((promote_precision(cast(ws_net_paid as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN CheckOverflow((promote_precision(cast(ws_net_paid as decimal(12,2))) * promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                      InputAdapter
+                        Exchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #2
+                          WholeStageCodegen (5)
+                            HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,ws_ext_sales_price,ws_quantity,ws_net_paid] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                              Project [ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy]
+                                BroadcastHashJoin [ws_ship_mode_sk,sm_ship_mode_sk]
+                                  Project [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy]
+                                    BroadcastHashJoin [ws_sold_time_sk,t_time_sk]
+                                      Project [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy]
+                                        BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                          Project [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country]
+                                            BroadcastHashJoin [ws_warehouse_sk,w_warehouse_sk]
+                                              Filter [ws_warehouse_sk,ws_sold_time_sk,ws_ship_mode_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.web_sales [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk]
+                                                      SubqueryBroadcast [d_date_sk] #1
+                                                        ReusedExchange [d_date_sk,d_year,d_moy] #3
+                                              InputAdapter
+                                                BroadcastExchange #4
+                                                  WholeStageCodegen (1)
+                                                    Filter [w_warehouse_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.warehouse [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country]
+                                          InputAdapter
+                                            BroadcastExchange #3
+                                              WholeStageCodegen (2)
+                                                Filter [d_year,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                      InputAdapter
+                                        BroadcastExchange #5
+                                          WholeStageCodegen (3)
+                                            Project [t_time_sk]
+                                              Filter [t_time,t_time_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.time_dim [t_time_sk,t_time]
+                                  InputAdapter
+                                    BroadcastExchange #6
+                                      WholeStageCodegen (4)
+                                        Project [sm_ship_mode_sk]
+                                          Filter [sm_carrier,sm_ship_mode_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.ship_mode [sm_ship_mode_sk,sm_carrier]
+                  WholeStageCodegen (12)
+                    HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_moy = 1) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN CheckOverflow((promote_precision(cast(cs_sales_price as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN CheckOverflow((promote_precision(cast(cs_net_paid_inc_tax as decimal(12,2))) * promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true) ELSE 0.00 END),ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                      InputAdapter
+                        Exchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #7
+                          WholeStageCodegen (11)
+                            HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,cs_sales_price,cs_quantity,cs_net_paid_inc_tax] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                              Project [cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy]
+                                BroadcastHashJoin [cs_ship_mode_sk,sm_ship_mode_sk]
+                                  Project [cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy]
+                                    BroadcastHashJoin [cs_sold_time_sk,t_time_sk]
+                                      Project [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy]
+                                        BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                          Project [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country]
+                                            BroadcastHashJoin [cs_warehouse_sk,w_warehouse_sk]
+                                              Filter [cs_warehouse_sk,cs_sold_time_sk,cs_ship_mode_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.catalog_sales [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk]
+                                                      ReusedSubquery [d_date_sk] #1
+                                              InputAdapter
+                                                ReusedExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #4
+                                          InputAdapter
+                                            ReusedExchange [d_date_sk,d_year,d_moy] #3
+                                      InputAdapter
+                                        ReusedExchange [t_time_sk] #5
+                                  InputAdapter
+                                    ReusedExchange [sm_ship_mode_sk] #6
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.sf100/explain.txt
new file mode 100644
index 0000000000000..3a19c3923737d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.sf100/explain.txt
@@ -0,0 +1,201 @@
+== Physical Plan ==
+TakeOrderedAndProject (34)
++- * Filter (33)
+   +- Window (32)
+      +- * Sort (31)
+         +- Exchange (30)
+            +- * HashAggregate (29)
+               +- Exchange (28)
+                  +- * HashAggregate (27)
+                     +- * Expand (26)
+                        +- * Project (25)
+                           +- * SortMergeJoin Inner (24)
+                              :- * Sort (18)
+                              :  +- Exchange (17)
+                              :     +- * Project (16)
+                              :        +- * BroadcastHashJoin Inner BuildRight (15)
+                              :           :- * Project (10)
+                              :           :  +- * BroadcastHashJoin Inner BuildRight (9)
+                              :           :     :- * Filter (3)
+                              :           :     :  +- * ColumnarToRow (2)
+                              :           :     :     +- Scan parquet default.store_sales (1)
+                              :           :     +- BroadcastExchange (8)
+                              :           :        +- * Project (7)
+                              :           :           +- * Filter (6)
+                              :           :              +- * ColumnarToRow (5)
+                              :           :                 +- Scan parquet default.date_dim (4)
+                              :           +- BroadcastExchange (14)
+                              :              +- * Filter (13)
+                              :                 +- * ColumnarToRow (12)
+                              :                    +- Scan parquet default.store (11)
+                              +- * Sort (23)
+                                 +- Exchange (22)
+                                    +- * Filter (21)
+                                       +- * ColumnarToRow (20)
+                                          +- Scan parquet default.item (19)
+
+
+(1) Scan parquet default.store_sales
+Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_quantity:int,ss_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5]
+
+(3) Filter [codegen id : 3]
+Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5]
+Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int,d_year:int,d_moy:int,d_qoy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11]
+
+(6) Filter [codegen id : 1]
+Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11]
+Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1200)) AND (d_month_seq#8 <= 1211)) AND isnotnull(d_date_sk#7))
+
+(7) Project [codegen id : 1]
+Output [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11]
+Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11]
+
+(8) BroadcastExchange
+Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#5]
+Right keys [1]: [d_date_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11]
+Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#7, d_year#9, d_moy#10, d_qoy#11]
+
+(11) Scan parquet default.store
+Output [2]: [s_store_sk#13, s_store_id#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [s_store_sk#13, s_store_id#14]
+
+(13) Filter [codegen id : 2]
+Input [2]: [s_store_sk#13, s_store_id#14]
+Condition : isnotnull(s_store_sk#13)
+
+(14) BroadcastExchange
+Input [2]: [s_store_sk#13, s_store_id#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#15]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_store_sk#2]
+Right keys [1]: [s_store_sk#13]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14]
+Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_sk#13, s_store_id#14]
+
+(17) Exchange
+Input [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14]
+Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#16]
+
+(18) Sort [codegen id : 4]
+Input [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14]
+Arguments: [ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(19) Scan parquet default.item
+Output [5]: [i_item_sk#17, i_brand#18, i_class#19, i_category#20, i_product_name#21]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_class:string,i_category:string,i_product_name:string>
+
+(20) ColumnarToRow [codegen id : 5]
+Input [5]: [i_item_sk#17, i_brand#18, i_class#19, i_category#20, i_product_name#21]
+
+(21) Filter [codegen id : 5]
+Input [5]: [i_item_sk#17, i_brand#18, i_class#19, i_category#20, i_product_name#21]
+Condition : isnotnull(i_item_sk#17)
+
+(22) Exchange
+Input [5]: [i_item_sk#17, i_brand#18, i_class#19, i_category#20, i_product_name#21]
+Arguments: hashpartitioning(i_item_sk#17, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(23) Sort [codegen id : 6]
+Input [5]: [i_item_sk#17, i_brand#18, i_class#19, i_category#20, i_product_name#21]
+Arguments: [i_item_sk#17 ASC NULLS FIRST], false, 0
+
+(24) SortMergeJoin [codegen id : 7]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#17]
+Join condition: None
+
+(25) Project [codegen id : 7]
+Output [10]: [ss_quantity#3, ss_sales_price#4, i_category#20, i_class#19, i_brand#18, i_product_name#21, d_year#9, d_qoy#11, d_moy#10, s_store_id#14]
+Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14, i_item_sk#17, i_brand#18, i_class#19, i_category#20, i_product_name#21]
+
+(26) Expand [codegen id : 7]
+Input [10]: [ss_quantity#3, ss_sales_price#4, i_category#20, i_class#19, i_brand#18, i_product_name#21, d_year#9, d_qoy#11, d_moy#10, s_store_id#14]
+Arguments: [List(ss_quantity#3, ss_sales_price#4, i_category#20, i_class#19, i_brand#18, i_product_name#21, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, 0), List(ss_quantity#3, ss_sales_price#4, i_category#20, i_class#19, i_brand#18, i_product_name#21, d_year#9, d_qoy#11, d_moy#10, null, 1), List(ss_quantity#3, ss_sales_price#4, i_category#20, i_class#19, i_brand#18, i_product_name#21, d_year#9, d_qoy#11, null, null, 3), List(ss_quantity#3, ss_sales_price#4, i_category#20, i_class#19, i_brand#18, i_product_name#21, d_year#9, null, null, null, 7), List(ss_quantity#3, ss_sales_price#4, i_category#20, i_class#19, i_brand#18, i_product_name#21, null, null, null, null, 15), List(ss_quantity#3, ss_sales_price#4, i_category#20, i_class#19, i_brand#18, null, null, null, null, null, 31), List(ss_quantity#3, ss_sales_price#4, i_category#20, i_class#19, null, null, null, null, null, null, 63), List(ss_quantity#3, ss_sales_price#4, i_category#20, null, null, null, null, null, null, null, 127), List(ss_quantity#3, ss_sales_price#4, null, null, null, null, null, null, null, null, 255)], [ss_quantity#3, ss_sales_price#4, i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31]
+
+(27) HashAggregate [codegen id : 7]
+Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31]
+Keys [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31]
+Functions [1]: [partial_sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#4 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#3 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))]
+Aggregate Attributes [2]: [sum#32, isEmpty#33]
+Results [11]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, sum#34, isEmpty#35]
+
+(28) Exchange
+Input [11]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, sum#34, isEmpty#35]
+Arguments: hashpartitioning(i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, 5), ENSURE_REQUIREMENTS, [id=#36]
+
+(29) HashAggregate [codegen id : 8]
+Input [11]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, sum#34, isEmpty#35]
+Keys [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31]
+Functions [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#4 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#3 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))]
+Aggregate Attributes [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#4 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#3 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#37]
+Results [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#4 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#3 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#37 AS sumsales#38]
+
+(30) Exchange
+Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#38]
+Arguments: hashpartitioning(i_category#23, 5), ENSURE_REQUIREMENTS, [id=#39]
+
+(31) Sort [codegen id : 9]
+Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#38]
+Arguments: [i_category#23 ASC NULLS FIRST, sumsales#38 DESC NULLS LAST], false, 0
+
+(32) Window
+Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#38]
+Arguments: [rank(sumsales#38) windowspecdefinition(i_category#23, sumsales#38 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#40], [i_category#23], [sumsales#38 DESC NULLS LAST]
+
+(33) Filter [codegen id : 10]
+Input [10]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#38, rk#40]
+Condition : (rk#40 <= 100)
+
+(34) TakeOrderedAndProject
+Input [10]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#38, rk#40]
+Arguments: 100, [i_category#23 ASC NULLS FIRST, i_class#24 ASC NULLS FIRST, i_brand#25 ASC NULLS FIRST, i_product_name#26 ASC NULLS FIRST, d_year#27 ASC NULLS FIRST, d_qoy#28 ASC NULLS FIRST, d_moy#29 ASC NULLS FIRST, s_store_id#30 ASC NULLS FIRST, sumsales#38 ASC NULLS FIRST, rk#40 ASC NULLS FIRST], [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#38, rk#40]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (35)
+
+
+(35) ReusedExchange [Reuses operator id: 8]
+Output [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.sf100/simplified.txt
new file mode 100644
index 0000000000000..ceff615184818
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.sf100/simplified.txt
@@ -0,0 +1,59 @@
+TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,rk]
+  WholeStageCodegen (10)
+    Filter [rk]
+      InputAdapter
+        Window [sumsales,i_category]
+          WholeStageCodegen (9)
+            Sort [i_category,sumsales]
+              InputAdapter
+                Exchange [i_category] #1
+                  WholeStageCodegen (8)
+                    HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,sum,isEmpty] [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price as decimal(12,2))) * promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00)),sumsales,sum,isEmpty]
+                      InputAdapter
+                        Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] #2
+                          WholeStageCodegen (7)
+                            HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,ss_sales_price,ss_quantity] [sum,isEmpty,sum,isEmpty]
+                              Expand [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id]
+                                Project [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id]
+                                  SortMergeJoin [ss_item_sk,i_item_sk]
+                                    InputAdapter
+                                      WholeStageCodegen (4)
+                                        Sort [ss_item_sk]
+                                          InputAdapter
+                                            Exchange [ss_item_sk] #3
+                                              WholeStageCodegen (3)
+                                                Project [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id]
+                                                  BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                    Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy]
+                                                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                        Filter [ss_store_sk,ss_item_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk]
+                                                                SubqueryBroadcast [d_date_sk] #1
+                                                                  ReusedExchange [d_date_sk,d_year,d_moy,d_qoy] #4
+                                                        InputAdapter
+                                                          BroadcastExchange #4
+                                                            WholeStageCodegen (1)
+                                                              Project [d_date_sk,d_year,d_moy,d_qoy]
+                                                                Filter [d_month_seq,d_date_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy]
+                                                    InputAdapter
+                                                      BroadcastExchange #5
+                                                        WholeStageCodegen (2)
+                                                          Filter [s_store_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.store [s_store_sk,s_store_id]
+                                    InputAdapter
+                                      WholeStageCodegen (6)
+                                        Sort [i_item_sk]
+                                          InputAdapter
+                                            Exchange [i_item_sk] #6
+                                              WholeStageCodegen (5)
+                                                Filter [i_item_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.item [i_item_sk,i_brand,i_class,i_category,i_product_name]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/explain.txt
new file mode 100644
index 0000000000000..3d42f49415640
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/explain.txt
@@ -0,0 +1,186 @@
+== Physical Plan ==
+TakeOrderedAndProject (31)
++- * Filter (30)
+   +- Window (29)
+      +- * Sort (28)
+         +- Exchange (27)
+            +- * HashAggregate (26)
+               +- Exchange (25)
+                  +- * HashAggregate (24)
+                     +- * Expand (23)
+                        +- * Project (22)
+                           +- * BroadcastHashJoin Inner BuildRight (21)
+                              :- * Project (16)
+                              :  +- * BroadcastHashJoin Inner BuildRight (15)
+                              :     :- * Project (10)
+                              :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+                              :     :     :- * Filter (3)
+                              :     :     :  +- * ColumnarToRow (2)
+                              :     :     :     +- Scan parquet default.store_sales (1)
+                              :     :     +- BroadcastExchange (8)
+                              :     :        +- * Project (7)
+                              :     :           +- * Filter (6)
+                              :     :              +- * ColumnarToRow (5)
+                              :     :                 +- Scan parquet default.date_dim (4)
+                              :     +- BroadcastExchange (14)
+                              :        +- * Filter (13)
+                              :           +- * ColumnarToRow (12)
+                              :              +- Scan parquet default.store (11)
+                              +- BroadcastExchange (20)
+                                 +- * Filter (19)
+                                    +- * ColumnarToRow (18)
+                                       +- Scan parquet default.item (17)
+
+
+(1) Scan parquet default.store_sales
+Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_quantity:int,ss_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5]
+
+(3) Filter [codegen id : 4]
+Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5]
+Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int,d_year:int,d_moy:int,d_qoy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11]
+
+(6) Filter [codegen id : 1]
+Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11]
+Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1200)) AND (d_month_seq#8 <= 1211)) AND isnotnull(d_date_sk#7))
+
+(7) Project [codegen id : 1]
+Output [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11]
+Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11]
+
+(8) BroadcastExchange
+Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#5]
+Right keys [1]: [d_date_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11]
+Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#7, d_year#9, d_moy#10, d_qoy#11]
+
+(11) Scan parquet default.store
+Output [2]: [s_store_sk#13, s_store_id#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [s_store_sk#13, s_store_id#14]
+
+(13) Filter [codegen id : 2]
+Input [2]: [s_store_sk#13, s_store_id#14]
+Condition : isnotnull(s_store_sk#13)
+
+(14) BroadcastExchange
+Input [2]: [s_store_sk#13, s_store_id#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#15]
+
+(15) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#2]
+Right keys [1]: [s_store_sk#13]
+Join condition: None
+
+(16) Project [codegen id : 4]
+Output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14]
+Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_sk#13, s_store_id#14]
+
+(17) Scan parquet default.item
+Output [5]: [i_item_sk#16, i_brand#17, i_class#18, i_category#19, i_product_name#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_class:string,i_category:string,i_product_name:string>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [5]: [i_item_sk#16, i_brand#17, i_class#18, i_category#19, i_product_name#20]
+
+(19) Filter [codegen id : 3]
+Input [5]: [i_item_sk#16, i_brand#17, i_class#18, i_category#19, i_product_name#20]
+Condition : isnotnull(i_item_sk#16)
+
+(20) BroadcastExchange
+Input [5]: [i_item_sk#16, i_brand#17, i_class#18, i_category#19, i_product_name#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#21]
+
+(21) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#16]
+Join condition: None
+
+(22) Project [codegen id : 4]
+Output [10]: [ss_quantity#3, ss_sales_price#4, i_category#19, i_class#18, i_brand#17, i_product_name#20, d_year#9, d_qoy#11, d_moy#10, s_store_id#14]
+Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14, i_item_sk#16, i_brand#17, i_class#18, i_category#19, i_product_name#20]
+
+(23) Expand [codegen id : 4]
+Input [10]: [ss_quantity#3, ss_sales_price#4, i_category#19, i_class#18, i_brand#17, i_product_name#20, d_year#9, d_qoy#11, d_moy#10, s_store_id#14]
+Arguments: [List(ss_quantity#3, ss_sales_price#4, i_category#19, i_class#18, i_brand#17, i_product_name#20, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, 0), List(ss_quantity#3, ss_sales_price#4, i_category#19, i_class#18, i_brand#17, i_product_name#20, d_year#9, d_qoy#11, d_moy#10, null, 1), List(ss_quantity#3, ss_sales_price#4, i_category#19, i_class#18, i_brand#17, i_product_name#20, d_year#9, d_qoy#11, null, null, 3), List(ss_quantity#3, ss_sales_price#4, i_category#19, i_class#18, i_brand#17, i_product_name#20, d_year#9, null, null, null, 7), List(ss_quantity#3, ss_sales_price#4, i_category#19, i_class#18, i_brand#17, i_product_name#20, null, null, null, null, 15), List(ss_quantity#3, ss_sales_price#4, i_category#19, i_class#18, i_brand#17, null, null, null, null, null, 31), List(ss_quantity#3, ss_sales_price#4, i_category#19, i_class#18, null, null, null, null, null, null, 63), List(ss_quantity#3, ss_sales_price#4, i_category#19, null, null, null, null, null, null, null, 127), List(ss_quantity#3, ss_sales_price#4, null, null, null, null, null, null, null, null, 255)], [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#23, i_brand#24, i_product_name#25, d_year#26, d_qoy#27, d_moy#28, s_store_id#29, spark_grouping_id#30]
+
+(24) HashAggregate [codegen id : 4]
+Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#23, i_brand#24, i_product_name#25, d_year#26, d_qoy#27, d_moy#28, s_store_id#29, spark_grouping_id#30]
+Keys [9]: [i_category#22, i_class#23, i_brand#24, i_product_name#25, d_year#26, d_qoy#27, d_moy#28, s_store_id#29, spark_grouping_id#30]
+Functions [1]: [partial_sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#4 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#3 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))]
+Aggregate Attributes [2]: [sum#31, isEmpty#32]
+Results [11]: [i_category#22, i_class#23, i_brand#24, i_product_name#25, d_year#26, d_qoy#27, d_moy#28, s_store_id#29, spark_grouping_id#30, sum#33, isEmpty#34]
+
+(25) Exchange
+Input [11]: [i_category#22, i_class#23, i_brand#24, i_product_name#25, d_year#26, d_qoy#27, d_moy#28, s_store_id#29, spark_grouping_id#30, sum#33, isEmpty#34]
+Arguments: hashpartitioning(i_category#22, i_class#23, i_brand#24, i_product_name#25, d_year#26, d_qoy#27, d_moy#28, s_store_id#29, spark_grouping_id#30, 5), ENSURE_REQUIREMENTS, [id=#35]
+
+(26) HashAggregate [codegen id : 5]
+Input [11]: [i_category#22, i_class#23, i_brand#24, i_product_name#25, d_year#26, d_qoy#27, d_moy#28, s_store_id#29, spark_grouping_id#30, sum#33, isEmpty#34]
+Keys [9]: [i_category#22, i_class#23, i_brand#24, i_product_name#25, d_year#26, d_qoy#27, d_moy#28, s_store_id#29, spark_grouping_id#30]
+Functions [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#4 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#3 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))]
+Aggregate Attributes [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#4 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#3 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#36]
+Results [9]: [i_category#22, i_class#23, i_brand#24, i_product_name#25, d_year#26, d_qoy#27, d_moy#28, s_store_id#29, sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#4 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#3 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#36 AS sumsales#37]
+
+(27) Exchange
+Input [9]: [i_category#22, i_class#23, i_brand#24, i_product_name#25, d_year#26, d_qoy#27, d_moy#28, s_store_id#29, sumsales#37]
+Arguments: hashpartitioning(i_category#22, 5), ENSURE_REQUIREMENTS, [id=#38]
+
+(28) Sort [codegen id : 6]
+Input [9]: [i_category#22, i_class#23, i_brand#24, i_product_name#25, d_year#26, d_qoy#27, d_moy#28, s_store_id#29, sumsales#37]
+Arguments: [i_category#22 ASC NULLS FIRST, sumsales#37 DESC NULLS LAST], false, 0
+
+(29) Window
+Input [9]: [i_category#22, i_class#23, i_brand#24, i_product_name#25, d_year#26, d_qoy#27, d_moy#28, s_store_id#29, sumsales#37]
+Arguments: [rank(sumsales#37) windowspecdefinition(i_category#22, sumsales#37 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#39], [i_category#22], [sumsales#37 DESC NULLS LAST]
+
+(30) Filter [codegen id : 7]
+Input [10]: [i_category#22, i_class#23, i_brand#24, i_product_name#25, d_year#26, d_qoy#27, d_moy#28, s_store_id#29, sumsales#37, rk#39]
+Condition : (rk#39 <= 100)
+
+(31) TakeOrderedAndProject
+Input [10]: [i_category#22, i_class#23, i_brand#24, i_product_name#25, d_year#26, d_qoy#27, d_moy#28, s_store_id#29, sumsales#37, rk#39]
+Arguments: 100, [i_category#22 ASC NULLS FIRST, i_class#23 ASC NULLS FIRST, i_brand#24 ASC NULLS FIRST, i_product_name#25 ASC NULLS FIRST, d_year#26 ASC NULLS FIRST, d_qoy#27 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, s_store_id#29 ASC NULLS FIRST, sumsales#37 ASC NULLS FIRST, rk#39 ASC NULLS FIRST], [i_category#22, i_class#23, i_brand#24, i_product_name#25, d_year#26, d_qoy#27, d_moy#28, s_store_id#29, sumsales#37, rk#39]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (32)
+
+
+(32) ReusedExchange [Reuses operator id: 8]
+Output [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/simplified.txt
new file mode 100644
index 0000000000000..a7cc283af5aa1
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/simplified.txt
@@ -0,0 +1,50 @@
+TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,rk]
+  WholeStageCodegen (7)
+    Filter [rk]
+      InputAdapter
+        Window [sumsales,i_category]
+          WholeStageCodegen (6)
+            Sort [i_category,sumsales]
+              InputAdapter
+                Exchange [i_category] #1
+                  WholeStageCodegen (5)
+                    HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,sum,isEmpty] [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price as decimal(12,2))) * promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00)),sumsales,sum,isEmpty]
+                      InputAdapter
+                        Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] #2
+                          WholeStageCodegen (4)
+                            HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,ss_sales_price,ss_quantity] [sum,isEmpty,sum,isEmpty]
+                              Expand [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id]
+                                Project [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id]
+                                  BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                    Project [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id]
+                                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                        Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy]
+                                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                            Filter [ss_store_sk,ss_item_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #1
+                                                      ReusedExchange [d_date_sk,d_year,d_moy,d_qoy] #3
+                                            InputAdapter
+                                              BroadcastExchange #3
+                                                WholeStageCodegen (1)
+                                                  Project [d_date_sk,d_year,d_moy,d_qoy]
+                                                    Filter [d_month_seq,d_date_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy]
+                                        InputAdapter
+                                          BroadcastExchange #4
+                                            WholeStageCodegen (2)
+                                              Filter [s_store_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.store [s_store_sk,s_store_id]
+                                    InputAdapter
+                                      BroadcastExchange #5
+                                        WholeStageCodegen (3)
+                                          Filter [i_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.item [i_item_sk,i_brand,i_class,i_category,i_product_name]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.sf100/explain.txt
new file mode 100644
index 0000000000000..b6a609ec193b4
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.sf100/explain.txt
@@ -0,0 +1,292 @@
+== Physical Plan ==
+TakeOrderedAndProject (51)
++- * Project (50)
+   +- * SortMergeJoin Inner (49)
+      :- * Sort (14)
+      :  +- Exchange (13)
+      :     +- * Project (12)
+      :        +- * SortMergeJoin Inner (11)
+      :           :- * Sort (5)
+      :           :  +- Exchange (4)
+      :           :     +- * Filter (3)
+      :           :        +- * ColumnarToRow (2)
+      :           :           +- Scan parquet default.customer (1)
+      :           +- * Sort (10)
+      :              +- Exchange (9)
+      :                 +- * Filter (8)
+      :                    +- * ColumnarToRow (7)
+      :                       +- Scan parquet default.customer_address (6)
+      +- * Sort (48)
+         +- Exchange (47)
+            +- * HashAggregate (46)
+               +- * HashAggregate (45)
+                  +- * Project (44)
+                     +- * SortMergeJoin Inner (43)
+                        :- * Sort (40)
+                        :  +- Exchange (39)
+                        :     +- * Project (38)
+                        :        +- * BroadcastHashJoin Inner BuildRight (37)
+                        :           :- * Project (31)
+                        :           :  +- * BroadcastHashJoin Inner BuildRight (30)
+                        :           :     :- * Project (24)
+                        :           :     :  +- * BroadcastHashJoin Inner BuildRight (23)
+                        :           :     :     :- * Filter (17)
+                        :           :     :     :  +- * ColumnarToRow (16)
+                        :           :     :     :     +- Scan parquet default.store_sales (15)
+                        :           :     :     +- BroadcastExchange (22)
+                        :           :     :        +- * Project (21)
+                        :           :     :           +- * Filter (20)
+                        :           :     :              +- * ColumnarToRow (19)
+                        :           :     :                 +- Scan parquet default.date_dim (18)
+                        :           :     +- BroadcastExchange (29)
+                        :           :        +- * Project (28)
+                        :           :           +- * Filter (27)
+                        :           :              +- * ColumnarToRow (26)
+                        :           :                 +- Scan parquet default.store (25)
+                        :           +- BroadcastExchange (36)
+                        :              +- * Project (35)
+                        :                 +- * Filter (34)
+                        :                    +- * ColumnarToRow (33)
+                        :                       +- Scan parquet default.household_demographics (32)
+                        +- * Sort (42)
+                           +- ReusedExchange (41)
+
+
+(1) Scan parquet default.customer
+Output [4]: [c_customer_sk#1, c_current_addr_sk#2, c_first_name#3, c_last_name#4]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int,c_first_name:string,c_last_name:string>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [4]: [c_customer_sk#1, c_current_addr_sk#2, c_first_name#3, c_last_name#4]
+
+(3) Filter [codegen id : 1]
+Input [4]: [c_customer_sk#1, c_current_addr_sk#2, c_first_name#3, c_last_name#4]
+Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_current_addr_sk#2))
+
+(4) Exchange
+Input [4]: [c_customer_sk#1, c_current_addr_sk#2, c_first_name#3, c_last_name#4]
+Arguments: hashpartitioning(c_current_addr_sk#2, 5), ENSURE_REQUIREMENTS, [id=#5]
+
+(5) Sort [codegen id : 2]
+Input [4]: [c_customer_sk#1, c_current_addr_sk#2, c_first_name#3, c_last_name#4]
+Arguments: [c_current_addr_sk#2 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#6, ca_city#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)]
+ReadSchema: struct<ca_address_sk:int,ca_city:string>
+
+(7) ColumnarToRow [codegen id : 3]
+Input [2]: [ca_address_sk#6, ca_city#7]
+
+(8) Filter [codegen id : 3]
+Input [2]: [ca_address_sk#6, ca_city#7]
+Condition : (isnotnull(ca_address_sk#6) AND isnotnull(ca_city#7))
+
+(9) Exchange
+Input [2]: [ca_address_sk#6, ca_city#7]
+Arguments: hashpartitioning(ca_address_sk#6, 5), ENSURE_REQUIREMENTS, [id=#8]
+
+(10) Sort [codegen id : 4]
+Input [2]: [ca_address_sk#6, ca_city#7]
+Arguments: [ca_address_sk#6 ASC NULLS FIRST], false, 0
+
+(11) SortMergeJoin [codegen id : 5]
+Left keys [1]: [c_current_addr_sk#2]
+Right keys [1]: [ca_address_sk#6]
+Join condition: None
+
+(12) Project [codegen id : 5]
+Output [4]: [c_customer_sk#1, c_first_name#3, c_last_name#4, ca_city#7]
+Input [6]: [c_customer_sk#1, c_current_addr_sk#2, c_first_name#3, c_last_name#4, ca_address_sk#6, ca_city#7]
+
+(13) Exchange
+Input [4]: [c_customer_sk#1, c_first_name#3, c_last_name#4, ca_city#7]
+Arguments: hashpartitioning(c_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#9]
+
+(14) Sort [codegen id : 6]
+Input [4]: [c_customer_sk#1, c_first_name#3, c_last_name#4, ca_city#7]
+Arguments: [c_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(15) Scan parquet default.store_sales
+Output [9]: [ss_customer_sk#10, ss_hdemo_sk#11, ss_addr_sk#12, ss_store_sk#13, ss_ticket_number#14, ss_ext_sales_price#15, ss_ext_list_price#16, ss_ext_tax#17, ss_sold_date_sk#18]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#18), dynamicpruningexpression(ss_sold_date_sk#18 IN dynamicpruning#19)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_hdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_ext_sales_price:decimal(7,2),ss_ext_list_price:decimal(7,2),ss_ext_tax:decimal(7,2)>
+
+(16) ColumnarToRow [codegen id : 10]
+Input [9]: [ss_customer_sk#10, ss_hdemo_sk#11, ss_addr_sk#12, ss_store_sk#13, ss_ticket_number#14, ss_ext_sales_price#15, ss_ext_list_price#16, ss_ext_tax#17, ss_sold_date_sk#18]
+
+(17) Filter [codegen id : 10]
+Input [9]: [ss_customer_sk#10, ss_hdemo_sk#11, ss_addr_sk#12, ss_store_sk#13, ss_ticket_number#14, ss_ext_sales_price#15, ss_ext_list_price#16, ss_ext_tax#17, ss_sold_date_sk#18]
+Condition : (((isnotnull(ss_store_sk#13) AND isnotnull(ss_hdemo_sk#11)) AND isnotnull(ss_addr_sk#12)) AND isnotnull(ss_customer_sk#10))
+
+(18) Scan parquet default.date_dim
+Output [3]: [d_date_sk#20, d_year#21, d_dom#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_dom:int>
+
+(19) ColumnarToRow [codegen id : 7]
+Input [3]: [d_date_sk#20, d_year#21, d_dom#22]
+
+(20) Filter [codegen id : 7]
+Input [3]: [d_date_sk#20, d_year#21, d_dom#22]
+Condition : ((((isnotnull(d_dom#22) AND (d_dom#22 >= 1)) AND (d_dom#22 <= 2)) AND d_year#21 IN (1999,2000,2001)) AND isnotnull(d_date_sk#20))
+
+(21) Project [codegen id : 7]
+Output [1]: [d_date_sk#20]
+Input [3]: [d_date_sk#20, d_year#21, d_dom#22]
+
+(22) BroadcastExchange
+Input [1]: [d_date_sk#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#23]
+
+(23) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_sold_date_sk#18]
+Right keys [1]: [d_date_sk#20]
+Join condition: None
+
+(24) Project [codegen id : 10]
+Output [8]: [ss_customer_sk#10, ss_hdemo_sk#11, ss_addr_sk#12, ss_store_sk#13, ss_ticket_number#14, ss_ext_sales_price#15, ss_ext_list_price#16, ss_ext_tax#17]
+Input [10]: [ss_customer_sk#10, ss_hdemo_sk#11, ss_addr_sk#12, ss_store_sk#13, ss_ticket_number#14, ss_ext_sales_price#15, ss_ext_list_price#16, ss_ext_tax#17, ss_sold_date_sk#18, d_date_sk#20]
+
+(25) Scan parquet default.store
+Output [2]: [s_store_sk#24, s_city#25]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [In(s_city, [Midway,Fairview]), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_city:string>
+
+(26) ColumnarToRow [codegen id : 8]
+Input [2]: [s_store_sk#24, s_city#25]
+
+(27) Filter [codegen id : 8]
+Input [2]: [s_store_sk#24, s_city#25]
+Condition : (s_city#25 IN (Midway,Fairview) AND isnotnull(s_store_sk#24))
+
+(28) Project [codegen id : 8]
+Output [1]: [s_store_sk#24]
+Input [2]: [s_store_sk#24, s_city#25]
+
+(29) BroadcastExchange
+Input [1]: [s_store_sk#24]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#26]
+
+(30) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_store_sk#13]
+Right keys [1]: [s_store_sk#24]
+Join condition: None
+
+(31) Project [codegen id : 10]
+Output [7]: [ss_customer_sk#10, ss_hdemo_sk#11, ss_addr_sk#12, ss_ticket_number#14, ss_ext_sales_price#15, ss_ext_list_price#16, ss_ext_tax#17]
+Input [9]: [ss_customer_sk#10, ss_hdemo_sk#11, ss_addr_sk#12, ss_store_sk#13, ss_ticket_number#14, ss_ext_sales_price#15, ss_ext_list_price#16, ss_ext_tax#17, s_store_sk#24]
+
+(32) Scan parquet default.household_demographics
+Output [3]: [hd_demo_sk#27, hd_dep_count#28, hd_vehicle_count#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_dep_count:int,hd_vehicle_count:int>
+
+(33) ColumnarToRow [codegen id : 9]
+Input [3]: [hd_demo_sk#27, hd_dep_count#28, hd_vehicle_count#29]
+
+(34) Filter [codegen id : 9]
+Input [3]: [hd_demo_sk#27, hd_dep_count#28, hd_vehicle_count#29]
+Condition : (((hd_dep_count#28 = 4) OR (hd_vehicle_count#29 = 3)) AND isnotnull(hd_demo_sk#27))
+
+(35) Project [codegen id : 9]
+Output [1]: [hd_demo_sk#27]
+Input [3]: [hd_demo_sk#27, hd_dep_count#28, hd_vehicle_count#29]
+
+(36) BroadcastExchange
+Input [1]: [hd_demo_sk#27]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#30]
+
+(37) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_hdemo_sk#11]
+Right keys [1]: [hd_demo_sk#27]
+Join condition: None
+
+(38) Project [codegen id : 10]
+Output [6]: [ss_customer_sk#10, ss_addr_sk#12, ss_ticket_number#14, ss_ext_sales_price#15, ss_ext_list_price#16, ss_ext_tax#17]
+Input [8]: [ss_customer_sk#10, ss_hdemo_sk#11, ss_addr_sk#12, ss_ticket_number#14, ss_ext_sales_price#15, ss_ext_list_price#16, ss_ext_tax#17, hd_demo_sk#27]
+
+(39) Exchange
+Input [6]: [ss_customer_sk#10, ss_addr_sk#12, ss_ticket_number#14, ss_ext_sales_price#15, ss_ext_list_price#16, ss_ext_tax#17]
+Arguments: hashpartitioning(ss_addr_sk#12, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(40) Sort [codegen id : 11]
+Input [6]: [ss_customer_sk#10, ss_addr_sk#12, ss_ticket_number#14, ss_ext_sales_price#15, ss_ext_list_price#16, ss_ext_tax#17]
+Arguments: [ss_addr_sk#12 ASC NULLS FIRST], false, 0
+
+(41) ReusedExchange [Reuses operator id: 9]
+Output [2]: [ca_address_sk#32, ca_city#33]
+
+(42) Sort [codegen id : 13]
+Input [2]: [ca_address_sk#32, ca_city#33]
+Arguments: [ca_address_sk#32 ASC NULLS FIRST], false, 0
+
+(43) SortMergeJoin [codegen id : 14]
+Left keys [1]: [ss_addr_sk#12]
+Right keys [1]: [ca_address_sk#32]
+Join condition: None
+
+(44) Project [codegen id : 14]
+Output [7]: [ss_customer_sk#10, ss_addr_sk#12, ss_ticket_number#14, ss_ext_sales_price#15, ss_ext_list_price#16, ss_ext_tax#17, ca_city#33]
+Input [8]: [ss_customer_sk#10, ss_addr_sk#12, ss_ticket_number#14, ss_ext_sales_price#15, ss_ext_list_price#16, ss_ext_tax#17, ca_address_sk#32, ca_city#33]
+
+(45) HashAggregate [codegen id : 14]
+Input [7]: [ss_customer_sk#10, ss_addr_sk#12, ss_ticket_number#14, ss_ext_sales_price#15, ss_ext_list_price#16, ss_ext_tax#17, ca_city#33]
+Keys [4]: [ss_ticket_number#14, ss_customer_sk#10, ss_addr_sk#12, ca_city#33]
+Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#15)), partial_sum(UnscaledValue(ss_ext_list_price#16)), partial_sum(UnscaledValue(ss_ext_tax#17))]
+Aggregate Attributes [3]: [sum#34, sum#35, sum#36]
+Results [7]: [ss_ticket_number#14, ss_customer_sk#10, ss_addr_sk#12, ca_city#33, sum#37, sum#38, sum#39]
+
+(46) HashAggregate [codegen id : 14]
+Input [7]: [ss_ticket_number#14, ss_customer_sk#10, ss_addr_sk#12, ca_city#33, sum#37, sum#38, sum#39]
+Keys [4]: [ss_ticket_number#14, ss_customer_sk#10, ss_addr_sk#12, ca_city#33]
+Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#15)), sum(UnscaledValue(ss_ext_list_price#16)), sum(UnscaledValue(ss_ext_tax#17))]
+Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#15))#40, sum(UnscaledValue(ss_ext_list_price#16))#41, sum(UnscaledValue(ss_ext_tax#17))#42]
+Results [6]: [ss_ticket_number#14, ss_customer_sk#10, ca_city#33 AS bought_city#43, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#15))#40,17,2) AS extended_price#44, MakeDecimal(sum(UnscaledValue(ss_ext_list_price#16))#41,17,2) AS list_price#45, MakeDecimal(sum(UnscaledValue(ss_ext_tax#17))#42,17,2) AS extended_tax#46]
+
+(47) Exchange
+Input [6]: [ss_ticket_number#14, ss_customer_sk#10, bought_city#43, extended_price#44, list_price#45, extended_tax#46]
+Arguments: hashpartitioning(ss_customer_sk#10, 5), ENSURE_REQUIREMENTS, [id=#47]
+
+(48) Sort [codegen id : 15]
+Input [6]: [ss_ticket_number#14, ss_customer_sk#10, bought_city#43, extended_price#44, list_price#45, extended_tax#46]
+Arguments: [ss_customer_sk#10 ASC NULLS FIRST], false, 0
+
+(49) SortMergeJoin [codegen id : 16]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [ss_customer_sk#10]
+Join condition: NOT (ca_city#7 = bought_city#43)
+
+(50) Project [codegen id : 16]
+Output [8]: [c_last_name#4, c_first_name#3, ca_city#7, bought_city#43, ss_ticket_number#14, extended_price#44, extended_tax#46, list_price#45]
+Input [10]: [c_customer_sk#1, c_first_name#3, c_last_name#4, ca_city#7, ss_ticket_number#14, ss_customer_sk#10, bought_city#43, extended_price#44, list_price#45, extended_tax#46]
+
+(51) TakeOrderedAndProject
+Input [8]: [c_last_name#4, c_first_name#3, ca_city#7, bought_city#43, ss_ticket_number#14, extended_price#44, extended_tax#46, list_price#45]
+Arguments: 100, [c_last_name#4 ASC NULLS FIRST, ss_ticket_number#14 ASC NULLS FIRST], [c_last_name#4, c_first_name#3, ca_city#7, bought_city#43, ss_ticket_number#14, extended_price#44, extended_tax#46, list_price#45]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 15 Hosting Expression = ss_sold_date_sk#18 IN dynamicpruning#19
+ReusedExchange (52)
+
+
+(52) ReusedExchange [Reuses operator id: 22]
+Output [1]: [d_date_sk#20]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.sf100/simplified.txt
new file mode 100644
index 0000000000000..3c4e863923c5d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.sf100/simplified.txt
@@ -0,0 +1,89 @@
+TakeOrderedAndProject [c_last_name,ss_ticket_number,c_first_name,ca_city,bought_city,extended_price,extended_tax,list_price]
+  WholeStageCodegen (16)
+    Project [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,extended_price,extended_tax,list_price]
+      SortMergeJoin [c_customer_sk,ss_customer_sk,ca_city,bought_city]
+        InputAdapter
+          WholeStageCodegen (6)
+            Sort [c_customer_sk]
+              InputAdapter
+                Exchange [c_customer_sk] #1
+                  WholeStageCodegen (5)
+                    Project [c_customer_sk,c_first_name,c_last_name,ca_city]
+                      SortMergeJoin [c_current_addr_sk,ca_address_sk]
+                        InputAdapter
+                          WholeStageCodegen (2)
+                            Sort [c_current_addr_sk]
+                              InputAdapter
+                                Exchange [c_current_addr_sk] #2
+                                  WholeStageCodegen (1)
+                                    Filter [c_customer_sk,c_current_addr_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name]
+                        InputAdapter
+                          WholeStageCodegen (4)
+                            Sort [ca_address_sk]
+                              InputAdapter
+                                Exchange [ca_address_sk] #3
+                                  WholeStageCodegen (3)
+                                    Filter [ca_address_sk,ca_city]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.customer_address [ca_address_sk,ca_city]
+        InputAdapter
+          WholeStageCodegen (15)
+            Sort [ss_customer_sk]
+              InputAdapter
+                Exchange [ss_customer_sk] #4
+                  WholeStageCodegen (14)
+                    HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_ext_list_price)),sum(UnscaledValue(ss_ext_tax)),bought_city,extended_price,list_price,extended_tax,sum,sum,sum]
+                      HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] [sum,sum,sum,sum,sum,sum]
+                        Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_city]
+                          SortMergeJoin [ss_addr_sk,ca_address_sk]
+                            InputAdapter
+                              WholeStageCodegen (11)
+                                Sort [ss_addr_sk]
+                                  InputAdapter
+                                    Exchange [ss_addr_sk] #5
+                                      WholeStageCodegen (10)
+                                        Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax]
+                                          BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                            Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax]
+                                              BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax]
+                                                  BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                    Filter [ss_store_sk,ss_hdemo_sk,ss_addr_sk,ss_customer_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk]
+                                                            SubqueryBroadcast [d_date_sk] #1
+                                                              ReusedExchange [d_date_sk] #6
+                                                    InputAdapter
+                                                      BroadcastExchange #6
+                                                        WholeStageCodegen (7)
+                                                          Project [d_date_sk]
+                                                            Filter [d_dom,d_year,d_date_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.date_dim [d_date_sk,d_year,d_dom]
+                                                InputAdapter
+                                                  BroadcastExchange #7
+                                                    WholeStageCodegen (8)
+                                                      Project [s_store_sk]
+                                                        Filter [s_city,s_store_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.store [s_store_sk,s_city]
+                                            InputAdapter
+                                              BroadcastExchange #8
+                                                WholeStageCodegen (9)
+                                                  Project [hd_demo_sk]
+                                                    Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count]
+                            InputAdapter
+                              WholeStageCodegen (13)
+                                Sort [ca_address_sk]
+                                  InputAdapter
+                                    ReusedExchange [ca_address_sk,ca_city] #3
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68/explain.txt
new file mode 100644
index 0000000000000..101e4f272f98a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68/explain.txt
@@ -0,0 +1,252 @@
+== Physical Plan ==
+TakeOrderedAndProject (43)
++- * Project (42)
+   +- * BroadcastHashJoin Inner BuildRight (41)
+      :- * Project (39)
+      :  +- * BroadcastHashJoin Inner BuildRight (38)
+      :     :- * HashAggregate (33)
+      :     :  +- Exchange (32)
+      :     :     +- * HashAggregate (31)
+      :     :        +- * Project (30)
+      :     :           +- * BroadcastHashJoin Inner BuildRight (29)
+      :     :              :- * Project (24)
+      :     :              :  +- * BroadcastHashJoin Inner BuildRight (23)
+      :     :              :     :- * Project (17)
+      :     :              :     :  +- * BroadcastHashJoin Inner BuildRight (16)
+      :     :              :     :     :- * Project (10)
+      :     :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+      :     :              :     :     :     :- * Filter (3)
+      :     :              :     :     :     :  +- * ColumnarToRow (2)
+      :     :              :     :     :     :     +- Scan parquet default.store_sales (1)
+      :     :              :     :     :     +- BroadcastExchange (8)
+      :     :              :     :     :        +- * Project (7)
+      :     :              :     :     :           +- * Filter (6)
+      :     :              :     :     :              +- * ColumnarToRow (5)
+      :     :              :     :     :                 +- Scan parquet default.date_dim (4)
+      :     :              :     :     +- BroadcastExchange (15)
+      :     :              :     :        +- * Project (14)
+      :     :              :     :           +- * Filter (13)
+      :     :              :     :              +- * ColumnarToRow (12)
+      :     :              :     :                 +- Scan parquet default.store (11)
+      :     :              :     +- BroadcastExchange (22)
+      :     :              :        +- * Project (21)
+      :     :              :           +- * Filter (20)
+      :     :              :              +- * ColumnarToRow (19)
+      :     :              :                 +- Scan parquet default.household_demographics (18)
+      :     :              +- BroadcastExchange (28)
+      :     :                 +- * Filter (27)
+      :     :                    +- * ColumnarToRow (26)
+      :     :                       +- Scan parquet default.customer_address (25)
+      :     +- BroadcastExchange (37)
+      :        +- * Filter (36)
+      :           +- * ColumnarToRow (35)
+      :              +- Scan parquet default.customer (34)
+      +- ReusedExchange (40)
+
+
+(1) Scan parquet default.store_sales
+Output [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#9), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_hdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_ext_sales_price:decimal(7,2),ss_ext_list_price:decimal(7,2),ss_ext_tax:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9]
+
+(3) Filter [codegen id : 5]
+Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9]
+Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#11, d_year#12, d_dom#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_dom:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#11, d_year#12, d_dom#13]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#11, d_year#12, d_dom#13]
+Condition : ((((isnotnull(d_dom#13) AND (d_dom#13 >= 1)) AND (d_dom#13 <= 2)) AND d_year#12 IN (1999,2000,2001)) AND isnotnull(d_date_sk#11))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#11]
+Input [3]: [d_date_sk#11, d_year#12, d_dom#13]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_sold_date_sk#9]
+Right keys [1]: [d_date_sk#11]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8]
+Input [10]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9, d_date_sk#11]
+
+(11) Scan parquet default.store
+Output [2]: [s_store_sk#15, s_city#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [In(s_city, [Midway,Fairview]), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_city:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [s_store_sk#15, s_city#16]
+
+(13) Filter [codegen id : 2]
+Input [2]: [s_store_sk#15, s_city#16]
+Condition : (s_city#16 IN (Midway,Fairview) AND isnotnull(s_store_sk#15))
+
+(14) Project [codegen id : 2]
+Output [1]: [s_store_sk#15]
+Input [2]: [s_store_sk#15, s_city#16]
+
+(15) BroadcastExchange
+Input [1]: [s_store_sk#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(16) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_store_sk#4]
+Right keys [1]: [s_store_sk#15]
+Join condition: None
+
+(17) Project [codegen id : 5]
+Output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8]
+Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, s_store_sk#15]
+
+(18) Scan parquet default.household_demographics
+Output [3]: [hd_demo_sk#18, hd_dep_count#19, hd_vehicle_count#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_dep_count:int,hd_vehicle_count:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [3]: [hd_demo_sk#18, hd_dep_count#19, hd_vehicle_count#20]
+
+(20) Filter [codegen id : 3]
+Input [3]: [hd_demo_sk#18, hd_dep_count#19, hd_vehicle_count#20]
+Condition : (((hd_dep_count#19 = 4) OR (hd_vehicle_count#20 = 3)) AND isnotnull(hd_demo_sk#18))
+
+(21) Project [codegen id : 3]
+Output [1]: [hd_demo_sk#18]
+Input [3]: [hd_demo_sk#18, hd_dep_count#19, hd_vehicle_count#20]
+
+(22) BroadcastExchange
+Input [1]: [hd_demo_sk#18]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(23) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#18]
+Join condition: None
+
+(24) Project [codegen id : 5]
+Output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8]
+Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, hd_demo_sk#18]
+
+(25) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#22, ca_city#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)]
+ReadSchema: struct<ca_address_sk:int,ca_city:string>
+
+(26) ColumnarToRow [codegen id : 4]
+Input [2]: [ca_address_sk#22, ca_city#23]
+
+(27) Filter [codegen id : 4]
+Input [2]: [ca_address_sk#22, ca_city#23]
+Condition : (isnotnull(ca_address_sk#22) AND isnotnull(ca_city#23))
+
+(28) BroadcastExchange
+Input [2]: [ca_address_sk#22, ca_city#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#24]
+
+(29) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_addr_sk#3]
+Right keys [1]: [ca_address_sk#22]
+Join condition: None
+
+(30) Project [codegen id : 5]
+Output [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#23]
+Input [8]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_address_sk#22, ca_city#23]
+
+(31) HashAggregate [codegen id : 5]
+Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#23]
+Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#23]
+Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#6)), partial_sum(UnscaledValue(ss_ext_list_price#7)), partial_sum(UnscaledValue(ss_ext_tax#8))]
+Aggregate Attributes [3]: [sum#25, sum#26, sum#27]
+Results [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#23, sum#28, sum#29, sum#30]
+
+(32) Exchange
+Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#23, sum#28, sum#29, sum#30]
+Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#23, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(33) HashAggregate [codegen id : 8]
+Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#23, sum#28, sum#29, sum#30]
+Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#23]
+Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#6)), sum(UnscaledValue(ss_ext_list_price#7)), sum(UnscaledValue(ss_ext_tax#8))]
+Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#6))#32, sum(UnscaledValue(ss_ext_list_price#7))#33, sum(UnscaledValue(ss_ext_tax#8))#34]
+Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#23 AS bought_city#35, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#6))#32,17,2) AS extended_price#36, MakeDecimal(sum(UnscaledValue(ss_ext_list_price#7))#33,17,2) AS list_price#37, MakeDecimal(sum(UnscaledValue(ss_ext_tax#8))#34,17,2) AS extended_tax#38]
+
+(34) Scan parquet default.customer
+Output [4]: [c_customer_sk#39, c_current_addr_sk#40, c_first_name#41, c_last_name#42]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int,c_first_name:string,c_last_name:string>
+
+(35) ColumnarToRow [codegen id : 6]
+Input [4]: [c_customer_sk#39, c_current_addr_sk#40, c_first_name#41, c_last_name#42]
+
+(36) Filter [codegen id : 6]
+Input [4]: [c_customer_sk#39, c_current_addr_sk#40, c_first_name#41, c_last_name#42]
+Condition : (isnotnull(c_customer_sk#39) AND isnotnull(c_current_addr_sk#40))
+
+(37) BroadcastExchange
+Input [4]: [c_customer_sk#39, c_current_addr_sk#40, c_first_name#41, c_last_name#42]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#43]
+
+(38) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#39]
+Join condition: None
+
+(39) Project [codegen id : 8]
+Output [8]: [ss_ticket_number#5, bought_city#35, extended_price#36, list_price#37, extended_tax#38, c_current_addr_sk#40, c_first_name#41, c_last_name#42]
+Input [10]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#35, extended_price#36, list_price#37, extended_tax#38, c_customer_sk#39, c_current_addr_sk#40, c_first_name#41, c_last_name#42]
+
+(40) ReusedExchange [Reuses operator id: 28]
+Output [2]: [ca_address_sk#44, ca_city#45]
+
+(41) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [c_current_addr_sk#40]
+Right keys [1]: [ca_address_sk#44]
+Join condition: NOT (ca_city#45 = bought_city#35)
+
+(42) Project [codegen id : 8]
+Output [8]: [c_last_name#42, c_first_name#41, ca_city#45, bought_city#35, ss_ticket_number#5, extended_price#36, extended_tax#38, list_price#37]
+Input [10]: [ss_ticket_number#5, bought_city#35, extended_price#36, list_price#37, extended_tax#38, c_current_addr_sk#40, c_first_name#41, c_last_name#42, ca_address_sk#44, ca_city#45]
+
+(43) TakeOrderedAndProject
+Input [8]: [c_last_name#42, c_first_name#41, ca_city#45, bought_city#35, ss_ticket_number#5, extended_price#36, extended_tax#38, list_price#37]
+Arguments: 100, [c_last_name#42 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#42, c_first_name#41, ca_city#45, bought_city#35, ss_ticket_number#5, extended_price#36, extended_tax#38, list_price#37]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10
+ReusedExchange (44)
+
+
+(44) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#11]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68/simplified.txt
new file mode 100644
index 0000000000000..b4d72f1da879d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68/simplified.txt
@@ -0,0 +1,65 @@
+TakeOrderedAndProject [c_last_name,ss_ticket_number,c_first_name,ca_city,bought_city,extended_price,extended_tax,list_price]
+  WholeStageCodegen (8)
+    Project [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,extended_price,extended_tax,list_price]
+      BroadcastHashJoin [c_current_addr_sk,ca_address_sk,ca_city,bought_city]
+        Project [ss_ticket_number,bought_city,extended_price,list_price,extended_tax,c_current_addr_sk,c_first_name,c_last_name]
+          BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+            HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_ext_list_price)),sum(UnscaledValue(ss_ext_tax)),bought_city,extended_price,list_price,extended_tax,sum,sum,sum]
+              InputAdapter
+                Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1
+                  WholeStageCodegen (5)
+                    HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] [sum,sum,sum,sum,sum,sum]
+                      Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_city]
+                        BroadcastHashJoin [ss_addr_sk,ca_address_sk]
+                          Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax]
+                            BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                              Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax]
+                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                  Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax]
+                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                      Filter [ss_store_sk,ss_hdemo_sk,ss_addr_sk,ss_customer_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #1
+                                                ReusedExchange [d_date_sk] #2
+                                      InputAdapter
+                                        BroadcastExchange #2
+                                          WholeStageCodegen (1)
+                                            Project [d_date_sk]
+                                              Filter [d_dom,d_year,d_date_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_date_sk,d_year,d_dom]
+                                  InputAdapter
+                                    BroadcastExchange #3
+                                      WholeStageCodegen (2)
+                                        Project [s_store_sk]
+                                          Filter [s_city,s_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store [s_store_sk,s_city]
+                              InputAdapter
+                                BroadcastExchange #4
+                                  WholeStageCodegen (3)
+                                    Project [hd_demo_sk]
+                                      Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count]
+                          InputAdapter
+                            BroadcastExchange #5
+                              WholeStageCodegen (4)
+                                Filter [ca_address_sk,ca_city]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.customer_address [ca_address_sk,ca_city]
+            InputAdapter
+              BroadcastExchange #6
+                WholeStageCodegen (6)
+                  Filter [c_customer_sk,c_current_addr_sk]
+                    ColumnarToRow
+                      InputAdapter
+                        Scan parquet default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name]
+        InputAdapter
+          ReusedExchange [ca_address_sk,ca_city] #5
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.sf100/explain.txt
new file mode 100644
index 0000000000000..2cb4a1f42eb1d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.sf100/explain.txt
@@ -0,0 +1,298 @@
+== Physical Plan ==
+TakeOrderedAndProject (51)
++- * HashAggregate (50)
+   +- Exchange (49)
+      +- * HashAggregate (48)
+         +- * Project (47)
+            +- * BroadcastHashJoin Inner BuildLeft (46)
+               :- BroadcastExchange (42)
+               :  +- * Project (41)
+               :     +- * BroadcastHashJoin Inner BuildRight (40)
+               :        :- * Project (34)
+               :        :  +- SortMergeJoin LeftAnti (33)
+               :        :     :- SortMergeJoin LeftAnti (25)
+               :        :     :  :- * SortMergeJoin LeftSemi (17)
+               :        :     :  :  :- * Sort (5)
+               :        :     :  :  :  +- Exchange (4)
+               :        :     :  :  :     +- * Filter (3)
+               :        :     :  :  :        +- * ColumnarToRow (2)
+               :        :     :  :  :           +- Scan parquet default.customer (1)
+               :        :     :  :  +- * Sort (16)
+               :        :     :  :     +- Exchange (15)
+               :        :     :  :        +- * Project (14)
+               :        :     :  :           +- * BroadcastHashJoin Inner BuildRight (13)
+               :        :     :  :              :- * ColumnarToRow (7)
+               :        :     :  :              :  +- Scan parquet default.store_sales (6)
+               :        :     :  :              +- BroadcastExchange (12)
+               :        :     :  :                 +- * Project (11)
+               :        :     :  :                    +- * Filter (10)
+               :        :     :  :                       +- * ColumnarToRow (9)
+               :        :     :  :                          +- Scan parquet default.date_dim (8)
+               :        :     :  +- * Sort (24)
+               :        :     :     +- Exchange (23)
+               :        :     :        +- * Project (22)
+               :        :     :           +- * BroadcastHashJoin Inner BuildRight (21)
+               :        :     :              :- * ColumnarToRow (19)
+               :        :     :              :  +- Scan parquet default.web_sales (18)
+               :        :     :              +- ReusedExchange (20)
+               :        :     +- * Sort (32)
+               :        :        +- Exchange (31)
+               :        :           +- * Project (30)
+               :        :              +- * BroadcastHashJoin Inner BuildRight (29)
+               :        :                 :- * ColumnarToRow (27)
+               :        :                 :  +- Scan parquet default.catalog_sales (26)
+               :        :                 +- ReusedExchange (28)
+               :        +- BroadcastExchange (39)
+               :           +- * Project (38)
+               :              +- * Filter (37)
+               :                 +- * ColumnarToRow (36)
+               :                    +- Scan parquet default.customer_address (35)
+               +- * Filter (45)
+                  +- * ColumnarToRow (44)
+                     +- Scan parquet default.customer_demographics (43)
+
+
+(1) Scan parquet default.customer
+Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_addr_sk:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+
+(3) Filter [codegen id : 1]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2))
+
+(4) Exchange
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+Arguments: hashpartitioning(c_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#4]
+
+(5) Sort [codegen id : 2]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+Arguments: [c_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.store_sales
+Output [2]: [ss_customer_sk#5, ss_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)]
+ReadSchema: struct<ss_customer_sk:int>
+
+(7) ColumnarToRow [codegen id : 4]
+Input [2]: [ss_customer_sk#5, ss_sold_date_sk#6]
+
+(8) Scan parquet default.date_dim
+Output [3]: [d_date_sk#8, d_year#9, d_moy#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,6), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(9) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#8, d_year#9, d_moy#10]
+
+(10) Filter [codegen id : 3]
+Input [3]: [d_date_sk#8, d_year#9, d_moy#10]
+Condition : (((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2001)) AND (d_moy#10 >= 4)) AND (d_moy#10 <= 6)) AND isnotnull(d_date_sk#8))
+
+(11) Project [codegen id : 3]
+Output [1]: [d_date_sk#8]
+Input [3]: [d_date_sk#8, d_year#9, d_moy#10]
+
+(12) BroadcastExchange
+Input [1]: [d_date_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#11]
+
+(13) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#6]
+Right keys [1]: [d_date_sk#8]
+Join condition: None
+
+(14) Project [codegen id : 4]
+Output [1]: [ss_customer_sk#5]
+Input [3]: [ss_customer_sk#5, ss_sold_date_sk#6, d_date_sk#8]
+
+(15) Exchange
+Input [1]: [ss_customer_sk#5]
+Arguments: hashpartitioning(ss_customer_sk#5, 5), ENSURE_REQUIREMENTS, [id=#12]
+
+(16) Sort [codegen id : 5]
+Input [1]: [ss_customer_sk#5]
+Arguments: [ss_customer_sk#5 ASC NULLS FIRST], false, 0
+
+(17) SortMergeJoin [codegen id : 6]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [ss_customer_sk#5]
+Join condition: None
+
+(18) Scan parquet default.web_sales
+Output [2]: [ws_bill_customer_sk#13, ws_sold_date_sk#14]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#14), dynamicpruningexpression(ws_sold_date_sk#14 IN dynamicpruning#7)]
+ReadSchema: struct<ws_bill_customer_sk:int>
+
+(19) ColumnarToRow [codegen id : 8]
+Input [2]: [ws_bill_customer_sk#13, ws_sold_date_sk#14]
+
+(20) ReusedExchange [Reuses operator id: 12]
+Output [1]: [d_date_sk#15]
+
+(21) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ws_sold_date_sk#14]
+Right keys [1]: [d_date_sk#15]
+Join condition: None
+
+(22) Project [codegen id : 8]
+Output [1]: [ws_bill_customer_sk#13]
+Input [3]: [ws_bill_customer_sk#13, ws_sold_date_sk#14, d_date_sk#15]
+
+(23) Exchange
+Input [1]: [ws_bill_customer_sk#13]
+Arguments: hashpartitioning(ws_bill_customer_sk#13, 5), ENSURE_REQUIREMENTS, [id=#16]
+
+(24) Sort [codegen id : 9]
+Input [1]: [ws_bill_customer_sk#13]
+Arguments: [ws_bill_customer_sk#13 ASC NULLS FIRST], false, 0
+
+(25) SortMergeJoin
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [ws_bill_customer_sk#13]
+Join condition: None
+
+(26) Scan parquet default.catalog_sales
+Output [2]: [cs_ship_customer_sk#17, cs_sold_date_sk#18]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_sold_date_sk#18 IN dynamicpruning#7)]
+ReadSchema: struct<cs_ship_customer_sk:int>
+
+(27) ColumnarToRow [codegen id : 11]
+Input [2]: [cs_ship_customer_sk#17, cs_sold_date_sk#18]
+
+(28) ReusedExchange [Reuses operator id: 12]
+Output [1]: [d_date_sk#19]
+
+(29) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_sold_date_sk#18]
+Right keys [1]: [d_date_sk#19]
+Join condition: None
+
+(30) Project [codegen id : 11]
+Output [1]: [cs_ship_customer_sk#17]
+Input [3]: [cs_ship_customer_sk#17, cs_sold_date_sk#18, d_date_sk#19]
+
+(31) Exchange
+Input [1]: [cs_ship_customer_sk#17]
+Arguments: hashpartitioning(cs_ship_customer_sk#17, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(32) Sort [codegen id : 12]
+Input [1]: [cs_ship_customer_sk#17]
+Arguments: [cs_ship_customer_sk#17 ASC NULLS FIRST], false, 0
+
+(33) SortMergeJoin
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [cs_ship_customer_sk#17]
+Join condition: None
+
+(34) Project [codegen id : 14]
+Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+
+(35) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#21, ca_state#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [In(ca_state, [KY,GA,NM]), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(36) ColumnarToRow [codegen id : 13]
+Input [2]: [ca_address_sk#21, ca_state#22]
+
+(37) Filter [codegen id : 13]
+Input [2]: [ca_address_sk#21, ca_state#22]
+Condition : (ca_state#22 IN (KY,GA,NM) AND isnotnull(ca_address_sk#21))
+
+(38) Project [codegen id : 13]
+Output [1]: [ca_address_sk#21]
+Input [2]: [ca_address_sk#21, ca_state#22]
+
+(39) BroadcastExchange
+Input [1]: [ca_address_sk#21]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#23]
+
+(40) BroadcastHashJoin [codegen id : 14]
+Left keys [1]: [c_current_addr_sk#3]
+Right keys [1]: [ca_address_sk#21]
+Join condition: None
+
+(41) Project [codegen id : 14]
+Output [1]: [c_current_cdemo_sk#2]
+Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#21]
+
+(42) BroadcastExchange
+Input [1]: [c_current_cdemo_sk#2]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#24]
+
+(43) Scan parquet default.customer_demographics
+Output [6]: [cd_demo_sk#25, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_education_status:string,cd_purchase_estimate:int,cd_credit_rating:string>
+
+(44) ColumnarToRow
+Input [6]: [cd_demo_sk#25, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30]
+
+(45) Filter
+Input [6]: [cd_demo_sk#25, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30]
+Condition : isnotnull(cd_demo_sk#25)
+
+(46) BroadcastHashJoin [codegen id : 15]
+Left keys [1]: [c_current_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#25]
+Join condition: None
+
+(47) Project [codegen id : 15]
+Output [5]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30]
+Input [7]: [c_current_cdemo_sk#2, cd_demo_sk#25, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30]
+
+(48) HashAggregate [codegen id : 15]
+Input [5]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30]
+Keys [5]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#31]
+Results [6]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, count#32]
+
+(49) Exchange
+Input [6]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, count#32]
+Arguments: hashpartitioning(cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, 5), ENSURE_REQUIREMENTS, [id=#33]
+
+(50) HashAggregate [codegen id : 16]
+Input [6]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30, count#32]
+Keys [5]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#29, cd_credit_rating#30]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#34]
+Results [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, count(1)#34 AS cnt1#35, cd_purchase_estimate#29, count(1)#34 AS cnt2#36, cd_credit_rating#30, count(1)#34 AS cnt3#37]
+
+(51) TakeOrderedAndProject
+Input [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cnt1#35, cd_purchase_estimate#29, cnt2#36, cd_credit_rating#30, cnt3#37]
+Arguments: 100, [cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_education_status#28 ASC NULLS FIRST, cd_purchase_estimate#29 ASC NULLS FIRST, cd_credit_rating#30 ASC NULLS FIRST], [cd_gender#26, cd_marital_status#27, cd_education_status#28, cnt1#35, cd_purchase_estimate#29, cnt2#36, cd_credit_rating#30, cnt3#37]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 6 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7
+ReusedExchange (52)
+
+
+(52) ReusedExchange [Reuses operator id: 12]
+Output [1]: [d_date_sk#8]
+
+Subquery:2 Hosting operator id = 18 Hosting Expression = ws_sold_date_sk#14 IN dynamicpruning#7
+
+Subquery:3 Hosting operator id = 26 Hosting Expression = cs_sold_date_sk#18 IN dynamicpruning#7
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.sf100/simplified.txt
new file mode 100644
index 0000000000000..bdbf95bd10721
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.sf100/simplified.txt
@@ -0,0 +1,89 @@
+TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cnt1,cnt2,cnt3]
+  WholeStageCodegen (16)
+    HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,count] [count(1),cnt1,cnt2,cnt3,count]
+      InputAdapter
+        Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] #1
+          WholeStageCodegen (15)
+            HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] [count,count]
+              Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating]
+                BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk]
+                  InputAdapter
+                    BroadcastExchange #2
+                      WholeStageCodegen (14)
+                        Project [c_current_cdemo_sk]
+                          BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                            Project [c_current_cdemo_sk,c_current_addr_sk]
+                              InputAdapter
+                                SortMergeJoin [c_customer_sk,cs_ship_customer_sk]
+                                  SortMergeJoin [c_customer_sk,ws_bill_customer_sk]
+                                    WholeStageCodegen (6)
+                                      SortMergeJoin [c_customer_sk,ss_customer_sk]
+                                        InputAdapter
+                                          WholeStageCodegen (2)
+                                            Sort [c_customer_sk]
+                                              InputAdapter
+                                                Exchange [c_customer_sk] #3
+                                                  WholeStageCodegen (1)
+                                                    Filter [c_current_addr_sk,c_current_cdemo_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk]
+                                        InputAdapter
+                                          WholeStageCodegen (5)
+                                            Sort [ss_customer_sk]
+                                              InputAdapter
+                                                Exchange [ss_customer_sk] #4
+                                                  WholeStageCodegen (4)
+                                                    Project [ss_customer_sk]
+                                                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.store_sales [ss_customer_sk,ss_sold_date_sk]
+                                                              SubqueryBroadcast [d_date_sk] #1
+                                                                ReusedExchange [d_date_sk] #5
+                                                        InputAdapter
+                                                          BroadcastExchange #5
+                                                            WholeStageCodegen (3)
+                                                              Project [d_date_sk]
+                                                                Filter [d_year,d_moy,d_date_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                    WholeStageCodegen (9)
+                                      Sort [ws_bill_customer_sk]
+                                        InputAdapter
+                                          Exchange [ws_bill_customer_sk] #6
+                                            WholeStageCodegen (8)
+                                              Project [ws_bill_customer_sk]
+                                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.web_sales [ws_bill_customer_sk,ws_sold_date_sk]
+                                                        ReusedSubquery [d_date_sk] #1
+                                                  InputAdapter
+                                                    ReusedExchange [d_date_sk] #5
+                                  WholeStageCodegen (12)
+                                    Sort [cs_ship_customer_sk]
+                                      InputAdapter
+                                        Exchange [cs_ship_customer_sk] #7
+                                          WholeStageCodegen (11)
+                                            Project [cs_ship_customer_sk]
+                                              BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk]
+                                                      ReusedSubquery [d_date_sk] #1
+                                                InputAdapter
+                                                  ReusedExchange [d_date_sk] #5
+                            InputAdapter
+                              BroadcastExchange #8
+                                WholeStageCodegen (13)
+                                  Project [ca_address_sk]
+                                    Filter [ca_state,ca_address_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.customer_address [ca_address_sk,ca_state]
+                  Filter [cd_demo_sk]
+                    ColumnarToRow
+                      InputAdapter
+                        Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/explain.txt
new file mode 100644
index 0000000000000..d0b3c2231d997
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/explain.txt
@@ -0,0 +1,273 @@
+== Physical Plan ==
+TakeOrderedAndProject (46)
++- * HashAggregate (45)
+   +- Exchange (44)
+      +- * HashAggregate (43)
+         +- * Project (42)
+            +- * BroadcastHashJoin Inner BuildRight (41)
+               :- * Project (36)
+               :  +- * BroadcastHashJoin Inner BuildRight (35)
+               :     :- * Project (29)
+               :     :  +- * BroadcastHashJoin LeftAnti BuildRight (28)
+               :     :     :- * BroadcastHashJoin LeftAnti BuildRight (21)
+               :     :     :  :- * BroadcastHashJoin LeftSemi BuildRight (14)
+               :     :     :  :  :- * Filter (3)
+               :     :     :  :  :  +- * ColumnarToRow (2)
+               :     :     :  :  :     +- Scan parquet default.customer (1)
+               :     :     :  :  +- BroadcastExchange (13)
+               :     :     :  :     +- * Project (12)
+               :     :     :  :        +- * BroadcastHashJoin Inner BuildRight (11)
+               :     :     :  :           :- * ColumnarToRow (5)
+               :     :     :  :           :  +- Scan parquet default.store_sales (4)
+               :     :     :  :           +- BroadcastExchange (10)
+               :     :     :  :              +- * Project (9)
+               :     :     :  :                 +- * Filter (8)
+               :     :     :  :                    +- * ColumnarToRow (7)
+               :     :     :  :                       +- Scan parquet default.date_dim (6)
+               :     :     :  +- BroadcastExchange (20)
+               :     :     :     +- * Project (19)
+               :     :     :        +- * BroadcastHashJoin Inner BuildRight (18)
+               :     :     :           :- * ColumnarToRow (16)
+               :     :     :           :  +- Scan parquet default.web_sales (15)
+               :     :     :           +- ReusedExchange (17)
+               :     :     +- BroadcastExchange (27)
+               :     :        +- * Project (26)
+               :     :           +- * BroadcastHashJoin Inner BuildRight (25)
+               :     :              :- * ColumnarToRow (23)
+               :     :              :  +- Scan parquet default.catalog_sales (22)
+               :     :              +- ReusedExchange (24)
+               :     +- BroadcastExchange (34)
+               :        +- * Project (33)
+               :           +- * Filter (32)
+               :              +- * ColumnarToRow (31)
+               :                 +- Scan parquet default.customer_address (30)
+               +- BroadcastExchange (40)
+                  +- * Filter (39)
+                     +- * ColumnarToRow (38)
+                        +- Scan parquet default.customer_demographics (37)
+
+
+(1) Scan parquet default.customer
+Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_addr_sk:int>
+
+(2) ColumnarToRow [codegen id : 9]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+
+(3) Filter [codegen id : 9]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2))
+
+(4) Scan parquet default.store_sales
+Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)]
+ReadSchema: struct<ss_customer_sk:int>
+
+(5) ColumnarToRow [codegen id : 2]
+Input [2]: [ss_customer_sk#4, ss_sold_date_sk#5]
+
+(6) Scan parquet default.date_dim
+Output [3]: [d_date_sk#7, d_year#8, d_moy#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,6), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(7) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_moy#9]
+
+(8) Filter [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_moy#9]
+Condition : (((((isnotnull(d_year#8) AND isnotnull(d_moy#9)) AND (d_year#8 = 2001)) AND (d_moy#9 >= 4)) AND (d_moy#9 <= 6)) AND isnotnull(d_date_sk#7))
+
+(9) Project [codegen id : 1]
+Output [1]: [d_date_sk#7]
+Input [3]: [d_date_sk#7, d_year#8, d_moy#9]
+
+(10) BroadcastExchange
+Input [1]: [d_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(11) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#5]
+Right keys [1]: [d_date_sk#7]
+Join condition: None
+
+(12) Project [codegen id : 2]
+Output [1]: [ss_customer_sk#4]
+Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#7]
+
+(13) BroadcastExchange
+Input [1]: [ss_customer_sk#4]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#11]
+
+(14) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [ss_customer_sk#4]
+Join condition: None
+
+(15) Scan parquet default.web_sales
+Output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#13), dynamicpruningexpression(ws_sold_date_sk#13 IN dynamicpruning#6)]
+ReadSchema: struct<ws_bill_customer_sk:int>
+
+(16) ColumnarToRow [codegen id : 4]
+Input [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13]
+
+(17) ReusedExchange [Reuses operator id: 10]
+Output [1]: [d_date_sk#14]
+
+(18) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ws_sold_date_sk#13]
+Right keys [1]: [d_date_sk#14]
+Join condition: None
+
+(19) Project [codegen id : 4]
+Output [1]: [ws_bill_customer_sk#12]
+Input [3]: [ws_bill_customer_sk#12, ws_sold_date_sk#13, d_date_sk#14]
+
+(20) BroadcastExchange
+Input [1]: [ws_bill_customer_sk#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#15]
+
+(21) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [ws_bill_customer_sk#12]
+Join condition: None
+
+(22) Scan parquet default.catalog_sales
+Output [2]: [cs_ship_customer_sk#16, cs_sold_date_sk#17]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#17), dynamicpruningexpression(cs_sold_date_sk#17 IN dynamicpruning#6)]
+ReadSchema: struct<cs_ship_customer_sk:int>
+
+(23) ColumnarToRow [codegen id : 6]
+Input [2]: [cs_ship_customer_sk#16, cs_sold_date_sk#17]
+
+(24) ReusedExchange [Reuses operator id: 10]
+Output [1]: [d_date_sk#18]
+
+(25) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [cs_sold_date_sk#17]
+Right keys [1]: [d_date_sk#18]
+Join condition: None
+
+(26) Project [codegen id : 6]
+Output [1]: [cs_ship_customer_sk#16]
+Input [3]: [cs_ship_customer_sk#16, cs_sold_date_sk#17, d_date_sk#18]
+
+(27) BroadcastExchange
+Input [1]: [cs_ship_customer_sk#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#19]
+
+(28) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [cs_ship_customer_sk#16]
+Join condition: None
+
+(29) Project [codegen id : 9]
+Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+
+(30) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#20, ca_state#21]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [In(ca_state, [KY,GA,NM]), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(31) ColumnarToRow [codegen id : 7]
+Input [2]: [ca_address_sk#20, ca_state#21]
+
+(32) Filter [codegen id : 7]
+Input [2]: [ca_address_sk#20, ca_state#21]
+Condition : (ca_state#21 IN (KY,GA,NM) AND isnotnull(ca_address_sk#20))
+
+(33) Project [codegen id : 7]
+Output [1]: [ca_address_sk#20]
+Input [2]: [ca_address_sk#20, ca_state#21]
+
+(34) BroadcastExchange
+Input [1]: [ca_address_sk#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#22]
+
+(35) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_current_addr_sk#3]
+Right keys [1]: [ca_address_sk#20]
+Join condition: None
+
+(36) Project [codegen id : 9]
+Output [1]: [c_current_cdemo_sk#2]
+Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#20]
+
+(37) Scan parquet default.customer_demographics
+Output [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#27, cd_credit_rating#28]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_education_status:string,cd_purchase_estimate:int,cd_credit_rating:string>
+
+(38) ColumnarToRow [codegen id : 8]
+Input [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#27, cd_credit_rating#28]
+
+(39) Filter [codegen id : 8]
+Input [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#27, cd_credit_rating#28]
+Condition : isnotnull(cd_demo_sk#23)
+
+(40) BroadcastExchange
+Input [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#27, cd_credit_rating#28]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#29]
+
+(41) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_current_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#23]
+Join condition: None
+
+(42) Project [codegen id : 9]
+Output [5]: [cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#27, cd_credit_rating#28]
+Input [7]: [c_current_cdemo_sk#2, cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#27, cd_credit_rating#28]
+
+(43) HashAggregate [codegen id : 9]
+Input [5]: [cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#27, cd_credit_rating#28]
+Keys [5]: [cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#27, cd_credit_rating#28]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#30]
+Results [6]: [cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#27, cd_credit_rating#28, count#31]
+
+(44) Exchange
+Input [6]: [cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#27, cd_credit_rating#28, count#31]
+Arguments: hashpartitioning(cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#27, cd_credit_rating#28, 5), ENSURE_REQUIREMENTS, [id=#32]
+
+(45) HashAggregate [codegen id : 10]
+Input [6]: [cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#27, cd_credit_rating#28, count#31]
+Keys [5]: [cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#27, cd_credit_rating#28]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#33]
+Results [8]: [cd_gender#24, cd_marital_status#25, cd_education_status#26, count(1)#33 AS cnt1#34, cd_purchase_estimate#27, count(1)#33 AS cnt2#35, cd_credit_rating#28, count(1)#33 AS cnt3#36]
+
+(46) TakeOrderedAndProject
+Input [8]: [cd_gender#24, cd_marital_status#25, cd_education_status#26, cnt1#34, cd_purchase_estimate#27, cnt2#35, cd_credit_rating#28, cnt3#36]
+Arguments: 100, [cd_gender#24 ASC NULLS FIRST, cd_marital_status#25 ASC NULLS FIRST, cd_education_status#26 ASC NULLS FIRST, cd_purchase_estimate#27 ASC NULLS FIRST, cd_credit_rating#28 ASC NULLS FIRST], [cd_gender#24, cd_marital_status#25, cd_education_status#26, cnt1#34, cd_purchase_estimate#27, cnt2#35, cd_credit_rating#28, cnt3#36]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (47)
+
+
+(47) ReusedExchange [Reuses operator id: 10]
+Output [1]: [d_date_sk#7]
+
+Subquery:2 Hosting operator id = 15 Hosting Expression = ws_sold_date_sk#13 IN dynamicpruning#6
+
+Subquery:3 Hosting operator id = 22 Hosting Expression = cs_sold_date_sk#17 IN dynamicpruning#6
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/simplified.txt
new file mode 100644
index 0000000000000..957b9561b1752
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/simplified.txt
@@ -0,0 +1,74 @@
+TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cnt1,cnt2,cnt3]
+  WholeStageCodegen (10)
+    HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,count] [count(1),cnt1,cnt2,cnt3,count]
+      InputAdapter
+        Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] #1
+          WholeStageCodegen (9)
+            HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] [count,count]
+              Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating]
+                BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk]
+                  Project [c_current_cdemo_sk]
+                    BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                      Project [c_current_cdemo_sk,c_current_addr_sk]
+                        BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk]
+                          BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk]
+                            BroadcastHashJoin [c_customer_sk,ss_customer_sk]
+                              Filter [c_current_addr_sk,c_current_cdemo_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk]
+                              InputAdapter
+                                BroadcastExchange #2
+                                  WholeStageCodegen (2)
+                                    Project [ss_customer_sk]
+                                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store_sales [ss_customer_sk,ss_sold_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #1
+                                                ReusedExchange [d_date_sk] #3
+                                        InputAdapter
+                                          BroadcastExchange #3
+                                            WholeStageCodegen (1)
+                                              Project [d_date_sk]
+                                                Filter [d_year,d_moy,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                            InputAdapter
+                              BroadcastExchange #4
+                                WholeStageCodegen (4)
+                                  Project [ws_bill_customer_sk]
+                                    BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.web_sales [ws_bill_customer_sk,ws_sold_date_sk]
+                                            ReusedSubquery [d_date_sk] #1
+                                      InputAdapter
+                                        ReusedExchange [d_date_sk] #3
+                          InputAdapter
+                            BroadcastExchange #5
+                              WholeStageCodegen (6)
+                                Project [cs_ship_customer_sk]
+                                  BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk]
+                                          ReusedSubquery [d_date_sk] #1
+                                    InputAdapter
+                                      ReusedExchange [d_date_sk] #3
+                      InputAdapter
+                        BroadcastExchange #6
+                          WholeStageCodegen (7)
+                            Project [ca_address_sk]
+                              Filter [ca_state,ca_address_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.customer_address [ca_address_sk,ca_state]
+                  InputAdapter
+                    BroadcastExchange #7
+                      WholeStageCodegen (8)
+                        Filter [cd_demo_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.sf100/explain.txt
new file mode 100644
index 0000000000000..b138d059eaecb
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.sf100/explain.txt
@@ -0,0 +1,204 @@
+== Physical Plan ==
+TakeOrderedAndProject (34)
++- * HashAggregate (33)
+   +- Exchange (32)
+      +- * HashAggregate (31)
+         +- * Project (30)
+            +- * BroadcastHashJoin Inner BuildRight (29)
+               :- * Project (24)
+               :  +- * BroadcastHashJoin Inner BuildRight (23)
+               :     :- * Project (17)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (16)
+               :     :     :- * Project (10)
+               :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :     :     :- * Filter (3)
+               :     :     :     :  +- * ColumnarToRow (2)
+               :     :     :     :     +- Scan parquet default.store_sales (1)
+               :     :     :     +- BroadcastExchange (8)
+               :     :     :        +- * Project (7)
+               :     :     :           +- * Filter (6)
+               :     :     :              +- * ColumnarToRow (5)
+               :     :     :                 +- Scan parquet default.customer_demographics (4)
+               :     :     +- BroadcastExchange (15)
+               :     :        +- * Project (14)
+               :     :           +- * Filter (13)
+               :     :              +- * ColumnarToRow (12)
+               :     :                 +- Scan parquet default.promotion (11)
+               :     +- BroadcastExchange (22)
+               :        +- * Project (21)
+               :           +- * Filter (20)
+               :              +- * ColumnarToRow (19)
+               :                 +- Scan parquet default.date_dim (18)
+               +- BroadcastExchange (28)
+                  +- * Filter (27)
+                     +- * ColumnarToRow (26)
+                        +- Scan parquet default.item (25)
+
+
+(1) Scan parquet default.store_sales
+Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_cdemo_sk:int,ss_promo_sk:int,ss_quantity:int,ss_list_price:decimal(7,2),ss_sales_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+
+(3) Filter [codegen id : 5]
+Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3))
+
+(4) Scan parquet default.customer_demographics
+Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_marital_status,S), EqualTo(cd_education_status,College             ), IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_education_status:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+
+(6) Filter [codegen id : 1]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (cd_gender#11 = M)) AND (cd_marital_status#12 = S)) AND (cd_education_status#13 = College             )) AND isnotnull(cd_demo_sk#10))
+
+(7) Project [codegen id : 1]
+Output [1]: [cd_demo_sk#10]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+
+(8) BroadcastExchange
+Input [1]: [cd_demo_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#10]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#10]
+
+(11) Scan parquet default.promotion
+Output [3]: [p_promo_sk#15, p_channel_email#16, p_channel_event#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/promotion]
+PushedFilters: [Or(EqualTo(p_channel_email,N),EqualTo(p_channel_event,N)), IsNotNull(p_promo_sk)]
+ReadSchema: struct<p_promo_sk:int,p_channel_email:string,p_channel_event:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [p_promo_sk#15, p_channel_email#16, p_channel_event#17]
+
+(13) Filter [codegen id : 2]
+Input [3]: [p_promo_sk#15, p_channel_email#16, p_channel_event#17]
+Condition : (((p_channel_email#16 = N) OR (p_channel_event#17 = N)) AND isnotnull(p_promo_sk#15))
+
+(14) Project [codegen id : 2]
+Output [1]: [p_promo_sk#15]
+Input [3]: [p_promo_sk#15, p_channel_email#16, p_channel_event#17]
+
+(15) BroadcastExchange
+Input [1]: [p_promo_sk#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(16) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_promo_sk#3]
+Right keys [1]: [p_promo_sk#15]
+Join condition: None
+
+(17) Project [codegen id : 5]
+Output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, p_promo_sk#15]
+
+(18) Scan parquet default.date_dim
+Output [2]: [d_date_sk#19, d_year#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [2]: [d_date_sk#19, d_year#20]
+
+(20) Filter [codegen id : 3]
+Input [2]: [d_date_sk#19, d_year#20]
+Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2000)) AND isnotnull(d_date_sk#19))
+
+(21) Project [codegen id : 3]
+Output [1]: [d_date_sk#19]
+Input [2]: [d_date_sk#19, d_year#20]
+
+(22) BroadcastExchange
+Input [1]: [d_date_sk#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(23) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#19]
+Join condition: None
+
+(24) Project [codegen id : 5]
+Output [5]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7]
+Input [7]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#19]
+
+(25) Scan parquet default.item
+Output [2]: [i_item_sk#22, i_item_id#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(26) ColumnarToRow [codegen id : 4]
+Input [2]: [i_item_sk#22, i_item_id#23]
+
+(27) Filter [codegen id : 4]
+Input [2]: [i_item_sk#22, i_item_id#23]
+Condition : isnotnull(i_item_sk#22)
+
+(28) BroadcastExchange
+Input [2]: [i_item_sk#22, i_item_id#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#24]
+
+(29) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#22]
+Join condition: None
+
+(30) Project [codegen id : 5]
+Output [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#23]
+Input [7]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#22, i_item_id#23]
+
+(31) HashAggregate [codegen id : 5]
+Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#23]
+Keys [1]: [i_item_id#23]
+Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))]
+Aggregate Attributes [8]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31, count#32]
+Results [9]: [i_item_id#23, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40]
+
+(32) Exchange
+Input [9]: [i_item_id#23, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40]
+Arguments: hashpartitioning(i_item_id#23, 5), ENSURE_REQUIREMENTS, [id=#41]
+
+(33) HashAggregate [codegen id : 6]
+Input [9]: [i_item_id#23, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40]
+Keys [1]: [i_item_id#23]
+Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))]
+Aggregate Attributes [4]: [avg(ss_quantity#4)#42, avg(UnscaledValue(ss_list_price#5))#43, avg(UnscaledValue(ss_coupon_amt#7))#44, avg(UnscaledValue(ss_sales_price#6))#45]
+Results [5]: [i_item_id#23, avg(ss_quantity#4)#42 AS agg1#46, cast((avg(UnscaledValue(ss_list_price#5))#43 / 100.0) as decimal(11,6)) AS agg2#47, cast((avg(UnscaledValue(ss_coupon_amt#7))#44 / 100.0) as decimal(11,6)) AS agg3#48, cast((avg(UnscaledValue(ss_sales_price#6))#45 / 100.0) as decimal(11,6)) AS agg4#49]
+
+(34) TakeOrderedAndProject
+Input [5]: [i_item_id#23, agg1#46, agg2#47, agg3#48, agg4#49]
+Arguments: 100, [i_item_id#23 ASC NULLS FIRST], [i_item_id#23, agg1#46, agg2#47, agg3#48, agg4#49]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (35)
+
+
+(35) ReusedExchange [Reuses operator id: 22]
+Output [1]: [d_date_sk#19]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.sf100/simplified.txt
new file mode 100644
index 0000000000000..e5cf942ba37b5
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.sf100/simplified.txt
@@ -0,0 +1,52 @@
+TakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4]
+  WholeStageCodegen (6)
+    HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price)),agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count]
+      InputAdapter
+        Exchange [i_item_id] #1
+          WholeStageCodegen (5)
+            HashAggregate [i_item_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+              Project [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id]
+                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                  Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                      Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                        BroadcastHashJoin [ss_promo_sk,p_promo_sk]
+                          Project [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                            BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk]
+                              Filter [ss_cdemo_sk,ss_item_sk,ss_promo_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.store_sales [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                                      SubqueryBroadcast [d_date_sk] #1
+                                        ReusedExchange [d_date_sk] #2
+                              InputAdapter
+                                BroadcastExchange #3
+                                  WholeStageCodegen (1)
+                                    Project [cd_demo_sk]
+                                      Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status]
+                          InputAdapter
+                            BroadcastExchange #4
+                              WholeStageCodegen (2)
+                                Project [p_promo_sk]
+                                  Filter [p_channel_email,p_channel_event,p_promo_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.promotion [p_promo_sk,p_channel_email,p_channel_event]
+                      InputAdapter
+                        BroadcastExchange #2
+                          WholeStageCodegen (3)
+                            Project [d_date_sk]
+                              Filter [d_year,d_date_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.date_dim [d_date_sk,d_year]
+                  InputAdapter
+                    BroadcastExchange #5
+                      WholeStageCodegen (4)
+                        Filter [i_item_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.item [i_item_sk,i_item_id]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/explain.txt
new file mode 100644
index 0000000000000..5f6002b84f411
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/explain.txt
@@ -0,0 +1,204 @@
+== Physical Plan ==
+TakeOrderedAndProject (34)
++- * HashAggregate (33)
+   +- Exchange (32)
+      +- * HashAggregate (31)
+         +- * Project (30)
+            +- * BroadcastHashJoin Inner BuildRight (29)
+               :- * Project (23)
+               :  +- * BroadcastHashJoin Inner BuildRight (22)
+               :     :- * Project (17)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (16)
+               :     :     :- * Project (10)
+               :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :     :     :- * Filter (3)
+               :     :     :     :  +- * ColumnarToRow (2)
+               :     :     :     :     +- Scan parquet default.store_sales (1)
+               :     :     :     +- BroadcastExchange (8)
+               :     :     :        +- * Project (7)
+               :     :     :           +- * Filter (6)
+               :     :     :              +- * ColumnarToRow (5)
+               :     :     :                 +- Scan parquet default.customer_demographics (4)
+               :     :     +- BroadcastExchange (15)
+               :     :        +- * Project (14)
+               :     :           +- * Filter (13)
+               :     :              +- * ColumnarToRow (12)
+               :     :                 +- Scan parquet default.date_dim (11)
+               :     +- BroadcastExchange (21)
+               :        +- * Filter (20)
+               :           +- * ColumnarToRow (19)
+               :              +- Scan parquet default.item (18)
+               +- BroadcastExchange (28)
+                  +- * Project (27)
+                     +- * Filter (26)
+                        +- * ColumnarToRow (25)
+                           +- Scan parquet default.promotion (24)
+
+
+(1) Scan parquet default.store_sales
+Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_cdemo_sk:int,ss_promo_sk:int,ss_quantity:int,ss_list_price:decimal(7,2),ss_sales_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+
+(3) Filter [codegen id : 5]
+Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3))
+
+(4) Scan parquet default.customer_demographics
+Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_marital_status,S), EqualTo(cd_education_status,College             ), IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_education_status:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+
+(6) Filter [codegen id : 1]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (cd_gender#11 = M)) AND (cd_marital_status#12 = S)) AND (cd_education_status#13 = College             )) AND isnotnull(cd_demo_sk#10))
+
+(7) Project [codegen id : 1]
+Output [1]: [cd_demo_sk#10]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+
+(8) BroadcastExchange
+Input [1]: [cd_demo_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#10]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#10]
+
+(11) Scan parquet default.date_dim
+Output [2]: [d_date_sk#15, d_year#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#15, d_year#16]
+
+(13) Filter [codegen id : 2]
+Input [2]: [d_date_sk#15, d_year#16]
+Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2000)) AND isnotnull(d_date_sk#15))
+
+(14) Project [codegen id : 2]
+Output [1]: [d_date_sk#15]
+Input [2]: [d_date_sk#15, d_year#16]
+
+(15) BroadcastExchange
+Input [1]: [d_date_sk#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(16) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#15]
+Join condition: None
+
+(17) Project [codegen id : 5]
+Output [6]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7]
+Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#15]
+
+(18) Scan parquet default.item
+Output [2]: [i_item_sk#18, i_item_id#19]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [2]: [i_item_sk#18, i_item_id#19]
+
+(20) Filter [codegen id : 3]
+Input [2]: [i_item_sk#18, i_item_id#19]
+Condition : isnotnull(i_item_sk#18)
+
+(21) BroadcastExchange
+Input [2]: [i_item_sk#18, i_item_id#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#20]
+
+(22) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#18]
+Join condition: None
+
+(23) Project [codegen id : 5]
+Output [6]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19]
+Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#18, i_item_id#19]
+
+(24) Scan parquet default.promotion
+Output [3]: [p_promo_sk#21, p_channel_email#22, p_channel_event#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/promotion]
+PushedFilters: [Or(EqualTo(p_channel_email,N),EqualTo(p_channel_event,N)), IsNotNull(p_promo_sk)]
+ReadSchema: struct<p_promo_sk:int,p_channel_email:string,p_channel_event:string>
+
+(25) ColumnarToRow [codegen id : 4]
+Input [3]: [p_promo_sk#21, p_channel_email#22, p_channel_event#23]
+
+(26) Filter [codegen id : 4]
+Input [3]: [p_promo_sk#21, p_channel_email#22, p_channel_event#23]
+Condition : (((p_channel_email#22 = N) OR (p_channel_event#23 = N)) AND isnotnull(p_promo_sk#21))
+
+(27) Project [codegen id : 4]
+Output [1]: [p_promo_sk#21]
+Input [3]: [p_promo_sk#21, p_channel_email#22, p_channel_event#23]
+
+(28) BroadcastExchange
+Input [1]: [p_promo_sk#21]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#24]
+
+(29) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_promo_sk#3]
+Right keys [1]: [p_promo_sk#21]
+Join condition: None
+
+(30) Project [codegen id : 5]
+Output [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19]
+Input [7]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19, p_promo_sk#21]
+
+(31) HashAggregate [codegen id : 5]
+Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19]
+Keys [1]: [i_item_id#19]
+Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))]
+Aggregate Attributes [8]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31, count#32]
+Results [9]: [i_item_id#19, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40]
+
+(32) Exchange
+Input [9]: [i_item_id#19, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40]
+Arguments: hashpartitioning(i_item_id#19, 5), ENSURE_REQUIREMENTS, [id=#41]
+
+(33) HashAggregate [codegen id : 6]
+Input [9]: [i_item_id#19, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40]
+Keys [1]: [i_item_id#19]
+Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))]
+Aggregate Attributes [4]: [avg(ss_quantity#4)#42, avg(UnscaledValue(ss_list_price#5))#43, avg(UnscaledValue(ss_coupon_amt#7))#44, avg(UnscaledValue(ss_sales_price#6))#45]
+Results [5]: [i_item_id#19, avg(ss_quantity#4)#42 AS agg1#46, cast((avg(UnscaledValue(ss_list_price#5))#43 / 100.0) as decimal(11,6)) AS agg2#47, cast((avg(UnscaledValue(ss_coupon_amt#7))#44 / 100.0) as decimal(11,6)) AS agg3#48, cast((avg(UnscaledValue(ss_sales_price#6))#45 / 100.0) as decimal(11,6)) AS agg4#49]
+
+(34) TakeOrderedAndProject
+Input [5]: [i_item_id#19, agg1#46, agg2#47, agg3#48, agg4#49]
+Arguments: 100, [i_item_id#19 ASC NULLS FIRST], [i_item_id#19, agg1#46, agg2#47, agg3#48, agg4#49]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (35)
+
+
+(35) ReusedExchange [Reuses operator id: 15]
+Output [1]: [d_date_sk#15]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/simplified.txt
new file mode 100644
index 0000000000000..4de61c8e70519
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/simplified.txt
@@ -0,0 +1,52 @@
+TakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4]
+  WholeStageCodegen (6)
+    HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price)),agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count]
+      InputAdapter
+        Exchange [i_item_id] #1
+          WholeStageCodegen (5)
+            HashAggregate [i_item_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+              Project [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id]
+                BroadcastHashJoin [ss_promo_sk,p_promo_sk]
+                  Project [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id]
+                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                      Project [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                          Project [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                            BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk]
+                              Filter [ss_cdemo_sk,ss_item_sk,ss_promo_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.store_sales [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                                      SubqueryBroadcast [d_date_sk] #1
+                                        ReusedExchange [d_date_sk] #2
+                              InputAdapter
+                                BroadcastExchange #3
+                                  WholeStageCodegen (1)
+                                    Project [cd_demo_sk]
+                                      Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status]
+                          InputAdapter
+                            BroadcastExchange #2
+                              WholeStageCodegen (2)
+                                Project [d_date_sk]
+                                  Filter [d_year,d_date_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.date_dim [d_date_sk,d_year]
+                      InputAdapter
+                        BroadcastExchange #4
+                          WholeStageCodegen (3)
+                            Filter [i_item_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.item [i_item_sk,i_item_id]
+                  InputAdapter
+                    BroadcastExchange #5
+                      WholeStageCodegen (4)
+                        Project [p_promo_sk]
+                          Filter [p_channel_email,p_channel_event,p_promo_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.promotion [p_promo_sk,p_channel_email,p_channel_event]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.sf100/explain.txt
new file mode 100644
index 0000000000000..7b8445a46ac54
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.sf100/explain.txt
@@ -0,0 +1,278 @@
+== Physical Plan ==
+TakeOrderedAndProject (47)
++- * Project (46)
+   +- Window (45)
+      +- * Sort (44)
+         +- Exchange (43)
+            +- * HashAggregate (42)
+               +- Exchange (41)
+                  +- * HashAggregate (40)
+                     +- * Expand (39)
+                        +- * Project (38)
+                           +- * BroadcastHashJoin Inner BuildRight (37)
+                              :- * Project (10)
+                              :  +- * BroadcastHashJoin Inner BuildRight (9)
+                              :     :- * Filter (3)
+                              :     :  +- * ColumnarToRow (2)
+                              :     :     +- Scan parquet default.store_sales (1)
+                              :     +- BroadcastExchange (8)
+                              :        +- * Project (7)
+                              :           +- * Filter (6)
+                              :              +- * ColumnarToRow (5)
+                              :                 +- Scan parquet default.date_dim (4)
+                              +- BroadcastExchange (36)
+                                 +- * BroadcastHashJoin LeftSemi BuildRight (35)
+                                    :- * Filter (13)
+                                    :  +- * ColumnarToRow (12)
+                                    :     +- Scan parquet default.store (11)
+                                    +- BroadcastExchange (34)
+                                       +- * Project (33)
+                                          +- * Filter (32)
+                                             +- Window (31)
+                                                +- * Sort (30)
+                                                   +- Exchange (29)
+                                                      +- * HashAggregate (28)
+                                                         +- Exchange (27)
+                                                            +- * HashAggregate (26)
+                                                               +- * Project (25)
+                                                                  +- * BroadcastHashJoin Inner BuildRight (24)
+                                                                     :- * Project (19)
+                                                                     :  +- * BroadcastHashJoin Inner BuildRight (18)
+                                                                     :     :- * Filter (16)
+                                                                     :     :  +- * ColumnarToRow (15)
+                                                                     :     :     +- Scan parquet default.store_sales (14)
+                                                                     :     +- ReusedExchange (17)
+                                                                     +- BroadcastExchange (23)
+                                                                        +- * Filter (22)
+                                                                           +- * ColumnarToRow (21)
+                                                                              +- Scan parquet default.store (20)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 9]
+Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 9]
+Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_store_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#5, d_month_seq#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#5]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#5]
+Join condition: None
+
+(10) Project [codegen id : 9]
+Output [2]: [ss_store_sk#1, ss_net_profit#2]
+Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#5]
+
+(11) Scan parquet default.store
+Output [3]: [s_store_sk#8, s_county#9, s_state#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_county:string,s_state:string>
+
+(12) ColumnarToRow [codegen id : 8]
+Input [3]: [s_store_sk#8, s_county#9, s_state#10]
+
+(13) Filter [codegen id : 8]
+Input [3]: [s_store_sk#8, s_county#9, s_state#10]
+Condition : isnotnull(s_store_sk#8)
+
+(14) Scan parquet default.store_sales
+Output [3]: [ss_store_sk#11, ss_net_profit#12, ss_sold_date_sk#13]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#13), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_net_profit:decimal(7,2)>
+
+(15) ColumnarToRow [codegen id : 4]
+Input [3]: [ss_store_sk#11, ss_net_profit#12, ss_sold_date_sk#13]
+
+(16) Filter [codegen id : 4]
+Input [3]: [ss_store_sk#11, ss_net_profit#12, ss_sold_date_sk#13]
+Condition : isnotnull(ss_store_sk#11)
+
+(17) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#14]
+
+(18) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#13]
+Right keys [1]: [d_date_sk#14]
+Join condition: None
+
+(19) Project [codegen id : 4]
+Output [2]: [ss_store_sk#11, ss_net_profit#12]
+Input [4]: [ss_store_sk#11, ss_net_profit#12, ss_sold_date_sk#13, d_date_sk#14]
+
+(20) Scan parquet default.store
+Output [2]: [s_store_sk#15, s_state#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_state:string>
+
+(21) ColumnarToRow [codegen id : 3]
+Input [2]: [s_store_sk#15, s_state#16]
+
+(22) Filter [codegen id : 3]
+Input [2]: [s_store_sk#15, s_state#16]
+Condition : isnotnull(s_store_sk#15)
+
+(23) BroadcastExchange
+Input [2]: [s_store_sk#15, s_state#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#17]
+
+(24) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#11]
+Right keys [1]: [s_store_sk#15]
+Join condition: None
+
+(25) Project [codegen id : 4]
+Output [2]: [ss_net_profit#12, s_state#16]
+Input [4]: [ss_store_sk#11, ss_net_profit#12, s_store_sk#15, s_state#16]
+
+(26) HashAggregate [codegen id : 4]
+Input [2]: [ss_net_profit#12, s_state#16]
+Keys [1]: [s_state#16]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#12))]
+Aggregate Attributes [1]: [sum#18]
+Results [2]: [s_state#16, sum#19]
+
+(27) Exchange
+Input [2]: [s_state#16, sum#19]
+Arguments: hashpartitioning(s_state#16, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(28) HashAggregate [codegen id : 5]
+Input [2]: [s_state#16, sum#19]
+Keys [1]: [s_state#16]
+Functions [1]: [sum(UnscaledValue(ss_net_profit#12))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#12))#21]
+Results [3]: [s_state#16 AS s_state#22, s_state#16, MakeDecimal(sum(UnscaledValue(ss_net_profit#12))#21,17,2) AS _w2#23]
+
+(29) Exchange
+Input [3]: [s_state#22, s_state#16, _w2#23]
+Arguments: hashpartitioning(s_state#16, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(30) Sort [codegen id : 6]
+Input [3]: [s_state#22, s_state#16, _w2#23]
+Arguments: [s_state#16 ASC NULLS FIRST, _w2#23 DESC NULLS LAST], false, 0
+
+(31) Window
+Input [3]: [s_state#22, s_state#16, _w2#23]
+Arguments: [rank(_w2#23) windowspecdefinition(s_state#16, _w2#23 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#25], [s_state#16], [_w2#23 DESC NULLS LAST]
+
+(32) Filter [codegen id : 7]
+Input [4]: [s_state#22, s_state#16, _w2#23, ranking#25]
+Condition : (ranking#25 <= 5)
+
+(33) Project [codegen id : 7]
+Output [1]: [s_state#22]
+Input [4]: [s_state#22, s_state#16, _w2#23, ranking#25]
+
+(34) BroadcastExchange
+Input [1]: [s_state#22]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#26]
+
+(35) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [s_state#10]
+Right keys [1]: [s_state#22]
+Join condition: None
+
+(36) BroadcastExchange
+Input [3]: [s_store_sk#8, s_county#9, s_state#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#27]
+
+(37) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_store_sk#1]
+Right keys [1]: [s_store_sk#8]
+Join condition: None
+
+(38) Project [codegen id : 9]
+Output [3]: [ss_net_profit#2, s_state#10, s_county#9]
+Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#8, s_county#9, s_state#10]
+
+(39) Expand [codegen id : 9]
+Input [3]: [ss_net_profit#2, s_state#10, s_county#9]
+Arguments: [List(ss_net_profit#2, s_state#10, s_county#9, 0), List(ss_net_profit#2, s_state#10, null, 1), List(ss_net_profit#2, null, null, 3)], [ss_net_profit#2, s_state#28, s_county#29, spark_grouping_id#30]
+
+(40) HashAggregate [codegen id : 9]
+Input [4]: [ss_net_profit#2, s_state#28, s_county#29, spark_grouping_id#30]
+Keys [3]: [s_state#28, s_county#29, spark_grouping_id#30]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))]
+Aggregate Attributes [1]: [sum#31]
+Results [4]: [s_state#28, s_county#29, spark_grouping_id#30, sum#32]
+
+(41) Exchange
+Input [4]: [s_state#28, s_county#29, spark_grouping_id#30, sum#32]
+Arguments: hashpartitioning(s_state#28, s_county#29, spark_grouping_id#30, 5), ENSURE_REQUIREMENTS, [id=#33]
+
+(42) HashAggregate [codegen id : 10]
+Input [4]: [s_state#28, s_county#29, spark_grouping_id#30, sum#32]
+Keys [3]: [s_state#28, s_county#29, spark_grouping_id#30]
+Functions [1]: [sum(UnscaledValue(ss_net_profit#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#34]
+Results [7]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#34,17,2) AS total_sum#35, s_state#28, s_county#29, (cast((shiftright(spark_grouping_id#30, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#30, 0) & 1) as tinyint)) AS lochierarchy#36, (cast((shiftright(spark_grouping_id#30, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#30, 0) & 1) as tinyint)) AS _w1#37, CASE WHEN (cast((shiftright(spark_grouping_id#30, 0) & 1) as tinyint) = 0) THEN s_state#28 END AS _w2#38, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#34,17,2) AS _w3#39]
+
+(43) Exchange
+Input [7]: [total_sum#35, s_state#28, s_county#29, lochierarchy#36, _w1#37, _w2#38, _w3#39]
+Arguments: hashpartitioning(_w1#37, _w2#38, 5), ENSURE_REQUIREMENTS, [id=#40]
+
+(44) Sort [codegen id : 11]
+Input [7]: [total_sum#35, s_state#28, s_county#29, lochierarchy#36, _w1#37, _w2#38, _w3#39]
+Arguments: [_w1#37 ASC NULLS FIRST, _w2#38 ASC NULLS FIRST, _w3#39 DESC NULLS LAST], false, 0
+
+(45) Window
+Input [7]: [total_sum#35, s_state#28, s_county#29, lochierarchy#36, _w1#37, _w2#38, _w3#39]
+Arguments: [rank(_w3#39) windowspecdefinition(_w1#37, _w2#38, _w3#39 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#41], [_w1#37, _w2#38], [_w3#39 DESC NULLS LAST]
+
+(46) Project [codegen id : 12]
+Output [5]: [total_sum#35, s_state#28, s_county#29, lochierarchy#36, rank_within_parent#41]
+Input [8]: [total_sum#35, s_state#28, s_county#29, lochierarchy#36, _w1#37, _w2#38, _w3#39, rank_within_parent#41]
+
+(47) TakeOrderedAndProject
+Input [5]: [total_sum#35, s_state#28, s_county#29, lochierarchy#36, rank_within_parent#41]
+Arguments: 100, [lochierarchy#36 DESC NULLS LAST, CASE WHEN (lochierarchy#36 = 0) THEN s_state#28 END ASC NULLS FIRST, rank_within_parent#41 ASC NULLS FIRST], [total_sum#35, s_state#28, s_county#29, lochierarchy#36, rank_within_parent#41]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (48)
+
+
+(48) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#5]
+
+Subquery:2 Hosting operator id = 14 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#4
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.sf100/simplified.txt
new file mode 100644
index 0000000000000..7a5207362a5bd
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.sf100/simplified.txt
@@ -0,0 +1,77 @@
+TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_county]
+  WholeStageCodegen (12)
+    Project [total_sum,s_state,s_county,lochierarchy,rank_within_parent]
+      InputAdapter
+        Window [_w3,_w1,_w2]
+          WholeStageCodegen (11)
+            Sort [_w1,_w2,_w3]
+              InputAdapter
+                Exchange [_w1,_w2] #1
+                  WholeStageCodegen (10)
+                    HashAggregate [s_state,s_county,spark_grouping_id,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,lochierarchy,_w1,_w2,_w3,sum]
+                      InputAdapter
+                        Exchange [s_state,s_county,spark_grouping_id] #2
+                          WholeStageCodegen (9)
+                            HashAggregate [s_state,s_county,spark_grouping_id,ss_net_profit] [sum,sum]
+                              Expand [ss_net_profit,s_state,s_county]
+                                Project [ss_net_profit,s_state,s_county]
+                                  BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                    Project [ss_store_sk,ss_net_profit]
+                                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                        Filter [ss_store_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk]
+                                                SubqueryBroadcast [d_date_sk] #1
+                                                  ReusedExchange [d_date_sk] #3
+                                        InputAdapter
+                                          BroadcastExchange #3
+                                            WholeStageCodegen (1)
+                                              Project [d_date_sk]
+                                                Filter [d_month_seq,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_month_seq]
+                                    InputAdapter
+                                      BroadcastExchange #4
+                                        WholeStageCodegen (8)
+                                          BroadcastHashJoin [s_state,s_state]
+                                            Filter [s_store_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store [s_store_sk,s_county,s_state]
+                                            InputAdapter
+                                              BroadcastExchange #5
+                                                WholeStageCodegen (7)
+                                                  Project [s_state]
+                                                    Filter [ranking]
+                                                      InputAdapter
+                                                        Window [_w2,s_state]
+                                                          WholeStageCodegen (6)
+                                                            Sort [s_state,_w2]
+                                                              InputAdapter
+                                                                Exchange [s_state] #6
+                                                                  WholeStageCodegen (5)
+                                                                    HashAggregate [s_state,sum] [sum(UnscaledValue(ss_net_profit)),s_state,_w2,sum]
+                                                                      InputAdapter
+                                                                        Exchange [s_state] #7
+                                                                          WholeStageCodegen (4)
+                                                                            HashAggregate [s_state,ss_net_profit] [sum,sum]
+                                                                              Project [ss_net_profit,s_state]
+                                                                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                                                  Project [ss_store_sk,ss_net_profit]
+                                                                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                      Filter [ss_store_sk]
+                                                                                        ColumnarToRow
+                                                                                          InputAdapter
+                                                                                            Scan parquet default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk]
+                                                                                              ReusedSubquery [d_date_sk] #1
+                                                                                      InputAdapter
+                                                                                        ReusedExchange [d_date_sk] #3
+                                                                                  InputAdapter
+                                                                                    BroadcastExchange #8
+                                                                                      WholeStageCodegen (3)
+                                                                                        Filter [s_store_sk]
+                                                                                          ColumnarToRow
+                                                                                            InputAdapter
+                                                                                              Scan parquet default.store [s_store_sk,s_state]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/explain.txt
new file mode 100644
index 0000000000000..91cfba59bb5bc
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/explain.txt
@@ -0,0 +1,278 @@
+== Physical Plan ==
+TakeOrderedAndProject (47)
++- * Project (46)
+   +- Window (45)
+      +- * Sort (44)
+         +- Exchange (43)
+            +- * HashAggregate (42)
+               +- Exchange (41)
+                  +- * HashAggregate (40)
+                     +- * Expand (39)
+                        +- * Project (38)
+                           +- * BroadcastHashJoin Inner BuildRight (37)
+                              :- * Project (10)
+                              :  +- * BroadcastHashJoin Inner BuildRight (9)
+                              :     :- * Filter (3)
+                              :     :  +- * ColumnarToRow (2)
+                              :     :     +- Scan parquet default.store_sales (1)
+                              :     +- BroadcastExchange (8)
+                              :        +- * Project (7)
+                              :           +- * Filter (6)
+                              :              +- * ColumnarToRow (5)
+                              :                 +- Scan parquet default.date_dim (4)
+                              +- BroadcastExchange (36)
+                                 +- * BroadcastHashJoin LeftSemi BuildRight (35)
+                                    :- * Filter (13)
+                                    :  +- * ColumnarToRow (12)
+                                    :     +- Scan parquet default.store (11)
+                                    +- BroadcastExchange (34)
+                                       +- * Project (33)
+                                          +- * Filter (32)
+                                             +- Window (31)
+                                                +- * Sort (30)
+                                                   +- Exchange (29)
+                                                      +- * HashAggregate (28)
+                                                         +- Exchange (27)
+                                                            +- * HashAggregate (26)
+                                                               +- * Project (25)
+                                                                  +- * BroadcastHashJoin Inner BuildRight (24)
+                                                                     :- * Project (22)
+                                                                     :  +- * BroadcastHashJoin Inner BuildRight (21)
+                                                                     :     :- * Filter (16)
+                                                                     :     :  +- * ColumnarToRow (15)
+                                                                     :     :     +- Scan parquet default.store_sales (14)
+                                                                     :     +- BroadcastExchange (20)
+                                                                     :        +- * Filter (19)
+                                                                     :           +- * ColumnarToRow (18)
+                                                                     :              +- Scan parquet default.store (17)
+                                                                     +- ReusedExchange (23)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 9]
+Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 9]
+Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_store_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#5, d_month_seq#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#5]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#5]
+Join condition: None
+
+(10) Project [codegen id : 9]
+Output [2]: [ss_store_sk#1, ss_net_profit#2]
+Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#5]
+
+(11) Scan parquet default.store
+Output [3]: [s_store_sk#8, s_county#9, s_state#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_county:string,s_state:string>
+
+(12) ColumnarToRow [codegen id : 8]
+Input [3]: [s_store_sk#8, s_county#9, s_state#10]
+
+(13) Filter [codegen id : 8]
+Input [3]: [s_store_sk#8, s_county#9, s_state#10]
+Condition : isnotnull(s_store_sk#8)
+
+(14) Scan parquet default.store_sales
+Output [3]: [ss_store_sk#11, ss_net_profit#12, ss_sold_date_sk#13]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#13), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_net_profit:decimal(7,2)>
+
+(15) ColumnarToRow [codegen id : 4]
+Input [3]: [ss_store_sk#11, ss_net_profit#12, ss_sold_date_sk#13]
+
+(16) Filter [codegen id : 4]
+Input [3]: [ss_store_sk#11, ss_net_profit#12, ss_sold_date_sk#13]
+Condition : isnotnull(ss_store_sk#11)
+
+(17) Scan parquet default.store
+Output [2]: [s_store_sk#14, s_state#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_state:string>
+
+(18) ColumnarToRow [codegen id : 2]
+Input [2]: [s_store_sk#14, s_state#15]
+
+(19) Filter [codegen id : 2]
+Input [2]: [s_store_sk#14, s_state#15]
+Condition : isnotnull(s_store_sk#14)
+
+(20) BroadcastExchange
+Input [2]: [s_store_sk#14, s_state#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#16]
+
+(21) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#11]
+Right keys [1]: [s_store_sk#14]
+Join condition: None
+
+(22) Project [codegen id : 4]
+Output [3]: [ss_net_profit#12, ss_sold_date_sk#13, s_state#15]
+Input [5]: [ss_store_sk#11, ss_net_profit#12, ss_sold_date_sk#13, s_store_sk#14, s_state#15]
+
+(23) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#17]
+
+(24) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#13]
+Right keys [1]: [d_date_sk#17]
+Join condition: None
+
+(25) Project [codegen id : 4]
+Output [2]: [ss_net_profit#12, s_state#15]
+Input [4]: [ss_net_profit#12, ss_sold_date_sk#13, s_state#15, d_date_sk#17]
+
+(26) HashAggregate [codegen id : 4]
+Input [2]: [ss_net_profit#12, s_state#15]
+Keys [1]: [s_state#15]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#12))]
+Aggregate Attributes [1]: [sum#18]
+Results [2]: [s_state#15, sum#19]
+
+(27) Exchange
+Input [2]: [s_state#15, sum#19]
+Arguments: hashpartitioning(s_state#15, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(28) HashAggregate [codegen id : 5]
+Input [2]: [s_state#15, sum#19]
+Keys [1]: [s_state#15]
+Functions [1]: [sum(UnscaledValue(ss_net_profit#12))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#12))#21]
+Results [3]: [s_state#15 AS s_state#22, s_state#15, MakeDecimal(sum(UnscaledValue(ss_net_profit#12))#21,17,2) AS _w2#23]
+
+(29) Exchange
+Input [3]: [s_state#22, s_state#15, _w2#23]
+Arguments: hashpartitioning(s_state#15, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(30) Sort [codegen id : 6]
+Input [3]: [s_state#22, s_state#15, _w2#23]
+Arguments: [s_state#15 ASC NULLS FIRST, _w2#23 DESC NULLS LAST], false, 0
+
+(31) Window
+Input [3]: [s_state#22, s_state#15, _w2#23]
+Arguments: [rank(_w2#23) windowspecdefinition(s_state#15, _w2#23 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#25], [s_state#15], [_w2#23 DESC NULLS LAST]
+
+(32) Filter [codegen id : 7]
+Input [4]: [s_state#22, s_state#15, _w2#23, ranking#25]
+Condition : (ranking#25 <= 5)
+
+(33) Project [codegen id : 7]
+Output [1]: [s_state#22]
+Input [4]: [s_state#22, s_state#15, _w2#23, ranking#25]
+
+(34) BroadcastExchange
+Input [1]: [s_state#22]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#26]
+
+(35) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [s_state#10]
+Right keys [1]: [s_state#22]
+Join condition: None
+
+(36) BroadcastExchange
+Input [3]: [s_store_sk#8, s_county#9, s_state#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#27]
+
+(37) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_store_sk#1]
+Right keys [1]: [s_store_sk#8]
+Join condition: None
+
+(38) Project [codegen id : 9]
+Output [3]: [ss_net_profit#2, s_state#10, s_county#9]
+Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#8, s_county#9, s_state#10]
+
+(39) Expand [codegen id : 9]
+Input [3]: [ss_net_profit#2, s_state#10, s_county#9]
+Arguments: [List(ss_net_profit#2, s_state#10, s_county#9, 0), List(ss_net_profit#2, s_state#10, null, 1), List(ss_net_profit#2, null, null, 3)], [ss_net_profit#2, s_state#28, s_county#29, spark_grouping_id#30]
+
+(40) HashAggregate [codegen id : 9]
+Input [4]: [ss_net_profit#2, s_state#28, s_county#29, spark_grouping_id#30]
+Keys [3]: [s_state#28, s_county#29, spark_grouping_id#30]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))]
+Aggregate Attributes [1]: [sum#31]
+Results [4]: [s_state#28, s_county#29, spark_grouping_id#30, sum#32]
+
+(41) Exchange
+Input [4]: [s_state#28, s_county#29, spark_grouping_id#30, sum#32]
+Arguments: hashpartitioning(s_state#28, s_county#29, spark_grouping_id#30, 5), ENSURE_REQUIREMENTS, [id=#33]
+
+(42) HashAggregate [codegen id : 10]
+Input [4]: [s_state#28, s_county#29, spark_grouping_id#30, sum#32]
+Keys [3]: [s_state#28, s_county#29, spark_grouping_id#30]
+Functions [1]: [sum(UnscaledValue(ss_net_profit#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#34]
+Results [7]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#34,17,2) AS total_sum#35, s_state#28, s_county#29, (cast((shiftright(spark_grouping_id#30, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#30, 0) & 1) as tinyint)) AS lochierarchy#36, (cast((shiftright(spark_grouping_id#30, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#30, 0) & 1) as tinyint)) AS _w1#37, CASE WHEN (cast((shiftright(spark_grouping_id#30, 0) & 1) as tinyint) = 0) THEN s_state#28 END AS _w2#38, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#34,17,2) AS _w3#39]
+
+(43) Exchange
+Input [7]: [total_sum#35, s_state#28, s_county#29, lochierarchy#36, _w1#37, _w2#38, _w3#39]
+Arguments: hashpartitioning(_w1#37, _w2#38, 5), ENSURE_REQUIREMENTS, [id=#40]
+
+(44) Sort [codegen id : 11]
+Input [7]: [total_sum#35, s_state#28, s_county#29, lochierarchy#36, _w1#37, _w2#38, _w3#39]
+Arguments: [_w1#37 ASC NULLS FIRST, _w2#38 ASC NULLS FIRST, _w3#39 DESC NULLS LAST], false, 0
+
+(45) Window
+Input [7]: [total_sum#35, s_state#28, s_county#29, lochierarchy#36, _w1#37, _w2#38, _w3#39]
+Arguments: [rank(_w3#39) windowspecdefinition(_w1#37, _w2#38, _w3#39 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#41], [_w1#37, _w2#38], [_w3#39 DESC NULLS LAST]
+
+(46) Project [codegen id : 12]
+Output [5]: [total_sum#35, s_state#28, s_county#29, lochierarchy#36, rank_within_parent#41]
+Input [8]: [total_sum#35, s_state#28, s_county#29, lochierarchy#36, _w1#37, _w2#38, _w3#39, rank_within_parent#41]
+
+(47) TakeOrderedAndProject
+Input [5]: [total_sum#35, s_state#28, s_county#29, lochierarchy#36, rank_within_parent#41]
+Arguments: 100, [lochierarchy#36 DESC NULLS LAST, CASE WHEN (lochierarchy#36 = 0) THEN s_state#28 END ASC NULLS FIRST, rank_within_parent#41 ASC NULLS FIRST], [total_sum#35, s_state#28, s_county#29, lochierarchy#36, rank_within_parent#41]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (48)
+
+
+(48) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#5]
+
+Subquery:2 Hosting operator id = 14 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#4
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/simplified.txt
new file mode 100644
index 0000000000000..f52e7f9a11922
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/simplified.txt
@@ -0,0 +1,77 @@
+TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_county]
+  WholeStageCodegen (12)
+    Project [total_sum,s_state,s_county,lochierarchy,rank_within_parent]
+      InputAdapter
+        Window [_w3,_w1,_w2]
+          WholeStageCodegen (11)
+            Sort [_w1,_w2,_w3]
+              InputAdapter
+                Exchange [_w1,_w2] #1
+                  WholeStageCodegen (10)
+                    HashAggregate [s_state,s_county,spark_grouping_id,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,lochierarchy,_w1,_w2,_w3,sum]
+                      InputAdapter
+                        Exchange [s_state,s_county,spark_grouping_id] #2
+                          WholeStageCodegen (9)
+                            HashAggregate [s_state,s_county,spark_grouping_id,ss_net_profit] [sum,sum]
+                              Expand [ss_net_profit,s_state,s_county]
+                                Project [ss_net_profit,s_state,s_county]
+                                  BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                    Project [ss_store_sk,ss_net_profit]
+                                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                        Filter [ss_store_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk]
+                                                SubqueryBroadcast [d_date_sk] #1
+                                                  ReusedExchange [d_date_sk] #3
+                                        InputAdapter
+                                          BroadcastExchange #3
+                                            WholeStageCodegen (1)
+                                              Project [d_date_sk]
+                                                Filter [d_month_seq,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_month_seq]
+                                    InputAdapter
+                                      BroadcastExchange #4
+                                        WholeStageCodegen (8)
+                                          BroadcastHashJoin [s_state,s_state]
+                                            Filter [s_store_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store [s_store_sk,s_county,s_state]
+                                            InputAdapter
+                                              BroadcastExchange #5
+                                                WholeStageCodegen (7)
+                                                  Project [s_state]
+                                                    Filter [ranking]
+                                                      InputAdapter
+                                                        Window [_w2,s_state]
+                                                          WholeStageCodegen (6)
+                                                            Sort [s_state,_w2]
+                                                              InputAdapter
+                                                                Exchange [s_state] #6
+                                                                  WholeStageCodegen (5)
+                                                                    HashAggregate [s_state,sum] [sum(UnscaledValue(ss_net_profit)),s_state,_w2,sum]
+                                                                      InputAdapter
+                                                                        Exchange [s_state] #7
+                                                                          WholeStageCodegen (4)
+                                                                            HashAggregate [s_state,ss_net_profit] [sum,sum]
+                                                                              Project [ss_net_profit,s_state]
+                                                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                  Project [ss_net_profit,ss_sold_date_sk,s_state]
+                                                                                    BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                                                      Filter [ss_store_sk]
+                                                                                        ColumnarToRow
+                                                                                          InputAdapter
+                                                                                            Scan parquet default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk]
+                                                                                              ReusedSubquery [d_date_sk] #1
+                                                                                      InputAdapter
+                                                                                        BroadcastExchange #8
+                                                                                          WholeStageCodegen (2)
+                                                                                            Filter [s_store_sk]
+                                                                                              ColumnarToRow
+                                                                                                InputAdapter
+                                                                                                  Scan parquet default.store [s_store_sk,s_state]
+                                                                                  InputAdapter
+                                                                                    ReusedExchange [d_date_sk] #3
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.sf100/explain.txt
new file mode 100644
index 0000000000000..274d0c02b5583
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.sf100/explain.txt
@@ -0,0 +1,249 @@
+== Physical Plan ==
+* Sort (42)
++- Exchange (41)
+   +- * HashAggregate (40)
+      +- Exchange (39)
+         +- * HashAggregate (38)
+            +- * Project (37)
+               +- * BroadcastHashJoin Inner BuildRight (36)
+                  :- * Project (30)
+                  :  +- * BroadcastHashJoin Inner BuildLeft (29)
+                  :     :- BroadcastExchange (5)
+                  :     :  +- * Project (4)
+                  :     :     +- * Filter (3)
+                  :     :        +- * ColumnarToRow (2)
+                  :     :           +- Scan parquet default.item (1)
+                  :     +- Union (28)
+                  :        :- * Project (15)
+                  :        :  +- * BroadcastHashJoin Inner BuildRight (14)
+                  :        :     :- * Filter (8)
+                  :        :     :  +- * ColumnarToRow (7)
+                  :        :     :     +- Scan parquet default.web_sales (6)
+                  :        :     +- BroadcastExchange (13)
+                  :        :        +- * Project (12)
+                  :        :           +- * Filter (11)
+                  :        :              +- * ColumnarToRow (10)
+                  :        :                 +- Scan parquet default.date_dim (9)
+                  :        :- * Project (21)
+                  :        :  +- * BroadcastHashJoin Inner BuildRight (20)
+                  :        :     :- * Filter (18)
+                  :        :     :  +- * ColumnarToRow (17)
+                  :        :     :     +- Scan parquet default.catalog_sales (16)
+                  :        :     +- ReusedExchange (19)
+                  :        +- * Project (27)
+                  :           +- * BroadcastHashJoin Inner BuildRight (26)
+                  :              :- * Filter (24)
+                  :              :  +- * ColumnarToRow (23)
+                  :              :     +- Scan parquet default.store_sales (22)
+                  :              +- ReusedExchange (25)
+                  +- BroadcastExchange (35)
+                     +- * Project (34)
+                        +- * Filter (33)
+                           +- * ColumnarToRow (32)
+                              +- Scan parquet default.time_dim (31)
+
+
+(1) Scan parquet default.item
+Output [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_brand:string,i_manager_id:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4]
+
+(3) Filter [codegen id : 1]
+Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4]
+Condition : ((isnotnull(i_manager_id#4) AND (i_manager_id#4 = 1)) AND isnotnull(i_item_sk#1))
+
+(4) Project [codegen id : 1]
+Output [3]: [i_item_sk#1, i_brand_id#2, i_brand#3]
+Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4]
+
+(5) BroadcastExchange
+Input [3]: [i_item_sk#1, i_brand_id#2, i_brand#3]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#5]
+
+(6) Scan parquet default.web_sales
+Output [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#9), dynamicpruningexpression(ws_sold_date_sk#9 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_sold_time_sk)]
+ReadSchema: struct<ws_sold_time_sk:int,ws_item_sk:int,ws_ext_sales_price:decimal(7,2)>
+
+(7) ColumnarToRow [codegen id : 3]
+Input [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9]
+
+(8) Filter [codegen id : 3]
+Input [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9]
+Condition : (isnotnull(ws_item_sk#7) AND isnotnull(ws_sold_time_sk#6))
+
+(9) Scan parquet default.date_dim
+Output [3]: [d_date_sk#11, d_year#12, d_moy#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(10) ColumnarToRow [codegen id : 2]
+Input [3]: [d_date_sk#11, d_year#12, d_moy#13]
+
+(11) Filter [codegen id : 2]
+Input [3]: [d_date_sk#11, d_year#12, d_moy#13]
+Condition : ((((isnotnull(d_moy#13) AND isnotnull(d_year#12)) AND (d_moy#13 = 11)) AND (d_year#12 = 1999)) AND isnotnull(d_date_sk#11))
+
+(12) Project [codegen id : 2]
+Output [1]: [d_date_sk#11]
+Input [3]: [d_date_sk#11, d_year#12, d_moy#13]
+
+(13) BroadcastExchange
+Input [1]: [d_date_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(14) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ws_sold_date_sk#9]
+Right keys [1]: [d_date_sk#11]
+Join condition: None
+
+(15) Project [codegen id : 3]
+Output [3]: [ws_ext_sales_price#8 AS ext_price#15, ws_item_sk#7 AS sold_item_sk#16, ws_sold_time_sk#6 AS time_sk#17]
+Input [5]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9, d_date_sk#11]
+
+(16) Scan parquet default.catalog_sales
+Output [4]: [cs_sold_time_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#21), dynamicpruningexpression(cs_sold_date_sk#21 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_sold_time_sk)]
+ReadSchema: struct<cs_sold_time_sk:int,cs_item_sk:int,cs_ext_sales_price:decimal(7,2)>
+
+(17) ColumnarToRow [codegen id : 5]
+Input [4]: [cs_sold_time_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21]
+
+(18) Filter [codegen id : 5]
+Input [4]: [cs_sold_time_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21]
+Condition : (isnotnull(cs_item_sk#19) AND isnotnull(cs_sold_time_sk#18))
+
+(19) ReusedExchange [Reuses operator id: 13]
+Output [1]: [d_date_sk#22]
+
+(20) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [cs_sold_date_sk#21]
+Right keys [1]: [d_date_sk#22]
+Join condition: None
+
+(21) Project [codegen id : 5]
+Output [3]: [cs_ext_sales_price#20 AS ext_price#23, cs_item_sk#19 AS sold_item_sk#24, cs_sold_time_sk#18 AS time_sk#25]
+Input [5]: [cs_sold_time_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21, d_date_sk#22]
+
+(22) Scan parquet default.store_sales
+Output [4]: [ss_sold_time_sk#26, ss_item_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#29), dynamicpruningexpression(ss_sold_date_sk#29 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_sold_time_sk)]
+ReadSchema: struct<ss_sold_time_sk:int,ss_item_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(23) ColumnarToRow [codegen id : 7]
+Input [4]: [ss_sold_time_sk#26, ss_item_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29]
+
+(24) Filter [codegen id : 7]
+Input [4]: [ss_sold_time_sk#26, ss_item_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29]
+Condition : (isnotnull(ss_item_sk#27) AND isnotnull(ss_sold_time_sk#26))
+
+(25) ReusedExchange [Reuses operator id: 13]
+Output [1]: [d_date_sk#30]
+
+(26) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [ss_sold_date_sk#29]
+Right keys [1]: [d_date_sk#30]
+Join condition: None
+
+(27) Project [codegen id : 7]
+Output [3]: [ss_ext_sales_price#28 AS ext_price#31, ss_item_sk#27 AS sold_item_sk#32, ss_sold_time_sk#26 AS time_sk#33]
+Input [5]: [ss_sold_time_sk#26, ss_item_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29, d_date_sk#30]
+
+(28) Union
+
+(29) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [sold_item_sk#16]
+Join condition: None
+
+(30) Project [codegen id : 9]
+Output [4]: [i_brand_id#2, i_brand#3, ext_price#15, time_sk#17]
+Input [6]: [i_item_sk#1, i_brand_id#2, i_brand#3, ext_price#15, sold_item_sk#16, time_sk#17]
+
+(31) Scan parquet default.time_dim
+Output [4]: [t_time_sk#34, t_hour#35, t_minute#36, t_meal_time#37]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [Or(EqualTo(t_meal_time,breakfast           ),EqualTo(t_meal_time,dinner              )), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_hour:int,t_minute:int,t_meal_time:string>
+
+(32) ColumnarToRow [codegen id : 8]
+Input [4]: [t_time_sk#34, t_hour#35, t_minute#36, t_meal_time#37]
+
+(33) Filter [codegen id : 8]
+Input [4]: [t_time_sk#34, t_hour#35, t_minute#36, t_meal_time#37]
+Condition : (((t_meal_time#37 = breakfast           ) OR (t_meal_time#37 = dinner              )) AND isnotnull(t_time_sk#34))
+
+(34) Project [codegen id : 8]
+Output [3]: [t_time_sk#34, t_hour#35, t_minute#36]
+Input [4]: [t_time_sk#34, t_hour#35, t_minute#36, t_meal_time#37]
+
+(35) BroadcastExchange
+Input [3]: [t_time_sk#34, t_hour#35, t_minute#36]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#38]
+
+(36) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [time_sk#17]
+Right keys [1]: [t_time_sk#34]
+Join condition: None
+
+(37) Project [codegen id : 9]
+Output [5]: [i_brand_id#2, i_brand#3, ext_price#15, t_hour#35, t_minute#36]
+Input [7]: [i_brand_id#2, i_brand#3, ext_price#15, time_sk#17, t_time_sk#34, t_hour#35, t_minute#36]
+
+(38) HashAggregate [codegen id : 9]
+Input [5]: [i_brand_id#2, i_brand#3, ext_price#15, t_hour#35, t_minute#36]
+Keys [4]: [i_brand#3, i_brand_id#2, t_hour#35, t_minute#36]
+Functions [1]: [partial_sum(UnscaledValue(ext_price#15))]
+Aggregate Attributes [1]: [sum#39]
+Results [5]: [i_brand#3, i_brand_id#2, t_hour#35, t_minute#36, sum#40]
+
+(39) Exchange
+Input [5]: [i_brand#3, i_brand_id#2, t_hour#35, t_minute#36, sum#40]
+Arguments: hashpartitioning(i_brand#3, i_brand_id#2, t_hour#35, t_minute#36, 5), ENSURE_REQUIREMENTS, [id=#41]
+
+(40) HashAggregate [codegen id : 10]
+Input [5]: [i_brand#3, i_brand_id#2, t_hour#35, t_minute#36, sum#40]
+Keys [4]: [i_brand#3, i_brand_id#2, t_hour#35, t_minute#36]
+Functions [1]: [sum(UnscaledValue(ext_price#15))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ext_price#15))#42]
+Results [5]: [i_brand_id#2 AS brand_id#43, i_brand#3 AS brand#44, t_hour#35, t_minute#36, MakeDecimal(sum(UnscaledValue(ext_price#15))#42,17,2) AS ext_price#45]
+
+(41) Exchange
+Input [5]: [brand_id#43, brand#44, t_hour#35, t_minute#36, ext_price#45]
+Arguments: rangepartitioning(ext_price#45 DESC NULLS LAST, brand_id#43 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [id=#46]
+
+(42) Sort [codegen id : 11]
+Input [5]: [brand_id#43, brand#44, t_hour#35, t_minute#36, ext_price#45]
+Arguments: [ext_price#45 DESC NULLS LAST, brand_id#43 ASC NULLS FIRST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 6 Hosting Expression = ws_sold_date_sk#9 IN dynamicpruning#10
+ReusedExchange (43)
+
+
+(43) ReusedExchange [Reuses operator id: 13]
+Output [1]: [d_date_sk#11]
+
+Subquery:2 Hosting operator id = 16 Hosting Expression = cs_sold_date_sk#21 IN dynamicpruning#10
+
+Subquery:3 Hosting operator id = 22 Hosting Expression = ss_sold_date_sk#29 IN dynamicpruning#10
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.sf100/simplified.txt
new file mode 100644
index 0000000000000..eaa3121cbab39
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.sf100/simplified.txt
@@ -0,0 +1,69 @@
+WholeStageCodegen (11)
+  Sort [ext_price,brand_id]
+    InputAdapter
+      Exchange [ext_price,brand_id] #1
+        WholeStageCodegen (10)
+          HashAggregate [i_brand,i_brand_id,t_hour,t_minute,sum] [sum(UnscaledValue(ext_price)),brand_id,brand,ext_price,sum]
+            InputAdapter
+              Exchange [i_brand,i_brand_id,t_hour,t_minute] #2
+                WholeStageCodegen (9)
+                  HashAggregate [i_brand,i_brand_id,t_hour,t_minute,ext_price] [sum,sum]
+                    Project [i_brand_id,i_brand,ext_price,t_hour,t_minute]
+                      BroadcastHashJoin [time_sk,t_time_sk]
+                        Project [i_brand_id,i_brand,ext_price,time_sk]
+                          BroadcastHashJoin [i_item_sk,sold_item_sk]
+                            InputAdapter
+                              BroadcastExchange #3
+                                WholeStageCodegen (1)
+                                  Project [i_item_sk,i_brand_id,i_brand]
+                                    Filter [i_manager_id,i_item_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.item [i_item_sk,i_brand_id,i_brand,i_manager_id]
+                            InputAdapter
+                              Union
+                                WholeStageCodegen (3)
+                                  Project [ws_ext_sales_price,ws_item_sk,ws_sold_time_sk]
+                                    BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                      Filter [ws_item_sk,ws_sold_time_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.web_sales [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #1
+                                                ReusedExchange [d_date_sk] #4
+                                      InputAdapter
+                                        BroadcastExchange #4
+                                          WholeStageCodegen (2)
+                                            Project [d_date_sk]
+                                              Filter [d_moy,d_year,d_date_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                WholeStageCodegen (5)
+                                  Project [cs_ext_sales_price,cs_item_sk,cs_sold_time_sk]
+                                    BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                      Filter [cs_item_sk,cs_sold_time_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.catalog_sales [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk]
+                                              ReusedSubquery [d_date_sk] #1
+                                      InputAdapter
+                                        ReusedExchange [d_date_sk] #4
+                                WholeStageCodegen (7)
+                                  Project [ss_ext_sales_price,ss_item_sk,ss_sold_time_sk]
+                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                      Filter [ss_item_sk,ss_sold_time_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store_sales [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                              ReusedSubquery [d_date_sk] #1
+                                      InputAdapter
+                                        ReusedExchange [d_date_sk] #4
+                        InputAdapter
+                          BroadcastExchange #5
+                            WholeStageCodegen (8)
+                              Project [t_time_sk,t_hour,t_minute]
+                                Filter [t_meal_time,t_time_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.time_dim [t_time_sk,t_hour,t_minute,t_meal_time]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/explain.txt
new file mode 100644
index 0000000000000..274d0c02b5583
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/explain.txt
@@ -0,0 +1,249 @@
+== Physical Plan ==
+* Sort (42)
++- Exchange (41)
+   +- * HashAggregate (40)
+      +- Exchange (39)
+         +- * HashAggregate (38)
+            +- * Project (37)
+               +- * BroadcastHashJoin Inner BuildRight (36)
+                  :- * Project (30)
+                  :  +- * BroadcastHashJoin Inner BuildLeft (29)
+                  :     :- BroadcastExchange (5)
+                  :     :  +- * Project (4)
+                  :     :     +- * Filter (3)
+                  :     :        +- * ColumnarToRow (2)
+                  :     :           +- Scan parquet default.item (1)
+                  :     +- Union (28)
+                  :        :- * Project (15)
+                  :        :  +- * BroadcastHashJoin Inner BuildRight (14)
+                  :        :     :- * Filter (8)
+                  :        :     :  +- * ColumnarToRow (7)
+                  :        :     :     +- Scan parquet default.web_sales (6)
+                  :        :     +- BroadcastExchange (13)
+                  :        :        +- * Project (12)
+                  :        :           +- * Filter (11)
+                  :        :              +- * ColumnarToRow (10)
+                  :        :                 +- Scan parquet default.date_dim (9)
+                  :        :- * Project (21)
+                  :        :  +- * BroadcastHashJoin Inner BuildRight (20)
+                  :        :     :- * Filter (18)
+                  :        :     :  +- * ColumnarToRow (17)
+                  :        :     :     +- Scan parquet default.catalog_sales (16)
+                  :        :     +- ReusedExchange (19)
+                  :        +- * Project (27)
+                  :           +- * BroadcastHashJoin Inner BuildRight (26)
+                  :              :- * Filter (24)
+                  :              :  +- * ColumnarToRow (23)
+                  :              :     +- Scan parquet default.store_sales (22)
+                  :              +- ReusedExchange (25)
+                  +- BroadcastExchange (35)
+                     +- * Project (34)
+                        +- * Filter (33)
+                           +- * ColumnarToRow (32)
+                              +- Scan parquet default.time_dim (31)
+
+
+(1) Scan parquet default.item
+Output [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_brand:string,i_manager_id:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4]
+
+(3) Filter [codegen id : 1]
+Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4]
+Condition : ((isnotnull(i_manager_id#4) AND (i_manager_id#4 = 1)) AND isnotnull(i_item_sk#1))
+
+(4) Project [codegen id : 1]
+Output [3]: [i_item_sk#1, i_brand_id#2, i_brand#3]
+Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4]
+
+(5) BroadcastExchange
+Input [3]: [i_item_sk#1, i_brand_id#2, i_brand#3]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#5]
+
+(6) Scan parquet default.web_sales
+Output [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#9), dynamicpruningexpression(ws_sold_date_sk#9 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_sold_time_sk)]
+ReadSchema: struct<ws_sold_time_sk:int,ws_item_sk:int,ws_ext_sales_price:decimal(7,2)>
+
+(7) ColumnarToRow [codegen id : 3]
+Input [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9]
+
+(8) Filter [codegen id : 3]
+Input [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9]
+Condition : (isnotnull(ws_item_sk#7) AND isnotnull(ws_sold_time_sk#6))
+
+(9) Scan parquet default.date_dim
+Output [3]: [d_date_sk#11, d_year#12, d_moy#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(10) ColumnarToRow [codegen id : 2]
+Input [3]: [d_date_sk#11, d_year#12, d_moy#13]
+
+(11) Filter [codegen id : 2]
+Input [3]: [d_date_sk#11, d_year#12, d_moy#13]
+Condition : ((((isnotnull(d_moy#13) AND isnotnull(d_year#12)) AND (d_moy#13 = 11)) AND (d_year#12 = 1999)) AND isnotnull(d_date_sk#11))
+
+(12) Project [codegen id : 2]
+Output [1]: [d_date_sk#11]
+Input [3]: [d_date_sk#11, d_year#12, d_moy#13]
+
+(13) BroadcastExchange
+Input [1]: [d_date_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(14) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ws_sold_date_sk#9]
+Right keys [1]: [d_date_sk#11]
+Join condition: None
+
+(15) Project [codegen id : 3]
+Output [3]: [ws_ext_sales_price#8 AS ext_price#15, ws_item_sk#7 AS sold_item_sk#16, ws_sold_time_sk#6 AS time_sk#17]
+Input [5]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9, d_date_sk#11]
+
+(16) Scan parquet default.catalog_sales
+Output [4]: [cs_sold_time_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#21), dynamicpruningexpression(cs_sold_date_sk#21 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_sold_time_sk)]
+ReadSchema: struct<cs_sold_time_sk:int,cs_item_sk:int,cs_ext_sales_price:decimal(7,2)>
+
+(17) ColumnarToRow [codegen id : 5]
+Input [4]: [cs_sold_time_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21]
+
+(18) Filter [codegen id : 5]
+Input [4]: [cs_sold_time_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21]
+Condition : (isnotnull(cs_item_sk#19) AND isnotnull(cs_sold_time_sk#18))
+
+(19) ReusedExchange [Reuses operator id: 13]
+Output [1]: [d_date_sk#22]
+
+(20) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [cs_sold_date_sk#21]
+Right keys [1]: [d_date_sk#22]
+Join condition: None
+
+(21) Project [codegen id : 5]
+Output [3]: [cs_ext_sales_price#20 AS ext_price#23, cs_item_sk#19 AS sold_item_sk#24, cs_sold_time_sk#18 AS time_sk#25]
+Input [5]: [cs_sold_time_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21, d_date_sk#22]
+
+(22) Scan parquet default.store_sales
+Output [4]: [ss_sold_time_sk#26, ss_item_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#29), dynamicpruningexpression(ss_sold_date_sk#29 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_sold_time_sk)]
+ReadSchema: struct<ss_sold_time_sk:int,ss_item_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(23) ColumnarToRow [codegen id : 7]
+Input [4]: [ss_sold_time_sk#26, ss_item_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29]
+
+(24) Filter [codegen id : 7]
+Input [4]: [ss_sold_time_sk#26, ss_item_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29]
+Condition : (isnotnull(ss_item_sk#27) AND isnotnull(ss_sold_time_sk#26))
+
+(25) ReusedExchange [Reuses operator id: 13]
+Output [1]: [d_date_sk#30]
+
+(26) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [ss_sold_date_sk#29]
+Right keys [1]: [d_date_sk#30]
+Join condition: None
+
+(27) Project [codegen id : 7]
+Output [3]: [ss_ext_sales_price#28 AS ext_price#31, ss_item_sk#27 AS sold_item_sk#32, ss_sold_time_sk#26 AS time_sk#33]
+Input [5]: [ss_sold_time_sk#26, ss_item_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29, d_date_sk#30]
+
+(28) Union
+
+(29) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [sold_item_sk#16]
+Join condition: None
+
+(30) Project [codegen id : 9]
+Output [4]: [i_brand_id#2, i_brand#3, ext_price#15, time_sk#17]
+Input [6]: [i_item_sk#1, i_brand_id#2, i_brand#3, ext_price#15, sold_item_sk#16, time_sk#17]
+
+(31) Scan parquet default.time_dim
+Output [4]: [t_time_sk#34, t_hour#35, t_minute#36, t_meal_time#37]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [Or(EqualTo(t_meal_time,breakfast           ),EqualTo(t_meal_time,dinner              )), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_hour:int,t_minute:int,t_meal_time:string>
+
+(32) ColumnarToRow [codegen id : 8]
+Input [4]: [t_time_sk#34, t_hour#35, t_minute#36, t_meal_time#37]
+
+(33) Filter [codegen id : 8]
+Input [4]: [t_time_sk#34, t_hour#35, t_minute#36, t_meal_time#37]
+Condition : (((t_meal_time#37 = breakfast           ) OR (t_meal_time#37 = dinner              )) AND isnotnull(t_time_sk#34))
+
+(34) Project [codegen id : 8]
+Output [3]: [t_time_sk#34, t_hour#35, t_minute#36]
+Input [4]: [t_time_sk#34, t_hour#35, t_minute#36, t_meal_time#37]
+
+(35) BroadcastExchange
+Input [3]: [t_time_sk#34, t_hour#35, t_minute#36]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#38]
+
+(36) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [time_sk#17]
+Right keys [1]: [t_time_sk#34]
+Join condition: None
+
+(37) Project [codegen id : 9]
+Output [5]: [i_brand_id#2, i_brand#3, ext_price#15, t_hour#35, t_minute#36]
+Input [7]: [i_brand_id#2, i_brand#3, ext_price#15, time_sk#17, t_time_sk#34, t_hour#35, t_minute#36]
+
+(38) HashAggregate [codegen id : 9]
+Input [5]: [i_brand_id#2, i_brand#3, ext_price#15, t_hour#35, t_minute#36]
+Keys [4]: [i_brand#3, i_brand_id#2, t_hour#35, t_minute#36]
+Functions [1]: [partial_sum(UnscaledValue(ext_price#15))]
+Aggregate Attributes [1]: [sum#39]
+Results [5]: [i_brand#3, i_brand_id#2, t_hour#35, t_minute#36, sum#40]
+
+(39) Exchange
+Input [5]: [i_brand#3, i_brand_id#2, t_hour#35, t_minute#36, sum#40]
+Arguments: hashpartitioning(i_brand#3, i_brand_id#2, t_hour#35, t_minute#36, 5), ENSURE_REQUIREMENTS, [id=#41]
+
+(40) HashAggregate [codegen id : 10]
+Input [5]: [i_brand#3, i_brand_id#2, t_hour#35, t_minute#36, sum#40]
+Keys [4]: [i_brand#3, i_brand_id#2, t_hour#35, t_minute#36]
+Functions [1]: [sum(UnscaledValue(ext_price#15))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ext_price#15))#42]
+Results [5]: [i_brand_id#2 AS brand_id#43, i_brand#3 AS brand#44, t_hour#35, t_minute#36, MakeDecimal(sum(UnscaledValue(ext_price#15))#42,17,2) AS ext_price#45]
+
+(41) Exchange
+Input [5]: [brand_id#43, brand#44, t_hour#35, t_minute#36, ext_price#45]
+Arguments: rangepartitioning(ext_price#45 DESC NULLS LAST, brand_id#43 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [id=#46]
+
+(42) Sort [codegen id : 11]
+Input [5]: [brand_id#43, brand#44, t_hour#35, t_minute#36, ext_price#45]
+Arguments: [ext_price#45 DESC NULLS LAST, brand_id#43 ASC NULLS FIRST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 6 Hosting Expression = ws_sold_date_sk#9 IN dynamicpruning#10
+ReusedExchange (43)
+
+
+(43) ReusedExchange [Reuses operator id: 13]
+Output [1]: [d_date_sk#11]
+
+Subquery:2 Hosting operator id = 16 Hosting Expression = cs_sold_date_sk#21 IN dynamicpruning#10
+
+Subquery:3 Hosting operator id = 22 Hosting Expression = ss_sold_date_sk#29 IN dynamicpruning#10
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/simplified.txt
new file mode 100644
index 0000000000000..eaa3121cbab39
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/simplified.txt
@@ -0,0 +1,69 @@
+WholeStageCodegen (11)
+  Sort [ext_price,brand_id]
+    InputAdapter
+      Exchange [ext_price,brand_id] #1
+        WholeStageCodegen (10)
+          HashAggregate [i_brand,i_brand_id,t_hour,t_minute,sum] [sum(UnscaledValue(ext_price)),brand_id,brand,ext_price,sum]
+            InputAdapter
+              Exchange [i_brand,i_brand_id,t_hour,t_minute] #2
+                WholeStageCodegen (9)
+                  HashAggregate [i_brand,i_brand_id,t_hour,t_minute,ext_price] [sum,sum]
+                    Project [i_brand_id,i_brand,ext_price,t_hour,t_minute]
+                      BroadcastHashJoin [time_sk,t_time_sk]
+                        Project [i_brand_id,i_brand,ext_price,time_sk]
+                          BroadcastHashJoin [i_item_sk,sold_item_sk]
+                            InputAdapter
+                              BroadcastExchange #3
+                                WholeStageCodegen (1)
+                                  Project [i_item_sk,i_brand_id,i_brand]
+                                    Filter [i_manager_id,i_item_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.item [i_item_sk,i_brand_id,i_brand,i_manager_id]
+                            InputAdapter
+                              Union
+                                WholeStageCodegen (3)
+                                  Project [ws_ext_sales_price,ws_item_sk,ws_sold_time_sk]
+                                    BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                      Filter [ws_item_sk,ws_sold_time_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.web_sales [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #1
+                                                ReusedExchange [d_date_sk] #4
+                                      InputAdapter
+                                        BroadcastExchange #4
+                                          WholeStageCodegen (2)
+                                            Project [d_date_sk]
+                                              Filter [d_moy,d_year,d_date_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                WholeStageCodegen (5)
+                                  Project [cs_ext_sales_price,cs_item_sk,cs_sold_time_sk]
+                                    BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                      Filter [cs_item_sk,cs_sold_time_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.catalog_sales [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk]
+                                              ReusedSubquery [d_date_sk] #1
+                                      InputAdapter
+                                        ReusedExchange [d_date_sk] #4
+                                WholeStageCodegen (7)
+                                  Project [ss_ext_sales_price,ss_item_sk,ss_sold_time_sk]
+                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                      Filter [ss_item_sk,ss_sold_time_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store_sales [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                              ReusedSubquery [d_date_sk] #1
+                                      InputAdapter
+                                        ReusedExchange [d_date_sk] #4
+                        InputAdapter
+                          BroadcastExchange #5
+                            WholeStageCodegen (8)
+                              Project [t_time_sk,t_hour,t_minute]
+                                Filter [t_meal_time,t_time_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.time_dim [t_time_sk,t_hour,t_minute,t_meal_time]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.sf100/explain.txt
new file mode 100644
index 0000000000000..ac66894467451
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.sf100/explain.txt
@@ -0,0 +1,453 @@
+== Physical Plan ==
+TakeOrderedAndProject (80)
++- * HashAggregate (79)
+   +- Exchange (78)
+      +- * HashAggregate (77)
+         +- * Project (76)
+            +- * SortMergeJoin LeftOuter (75)
+               :- * Sort (68)
+               :  +- Exchange (67)
+               :     +- * Project (66)
+               :        +- * BroadcastHashJoin LeftOuter BuildRight (65)
+               :           :- * Project (60)
+               :           :  +- * SortMergeJoin Inner (59)
+               :           :     :- * Sort (47)
+               :           :     :  +- Exchange (46)
+               :           :     :     +- * Project (45)
+               :           :     :        +- * BroadcastHashJoin Inner BuildRight (44)
+               :           :     :           :- * Project (32)
+               :           :     :           :  +- * SortMergeJoin Inner (31)
+               :           :     :           :     :- * Sort (25)
+               :           :     :           :     :  +- Exchange (24)
+               :           :     :           :     :     +- * Project (23)
+               :           :     :           :     :        +- * BroadcastHashJoin Inner BuildRight (22)
+               :           :     :           :     :           :- * Project (17)
+               :           :     :           :     :           :  +- * BroadcastHashJoin Inner BuildRight (16)
+               :           :     :           :     :           :     :- * Project (10)
+               :           :     :           :     :           :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :           :     :           :     :           :     :     :- * Filter (3)
+               :           :     :           :     :           :     :     :  +- * ColumnarToRow (2)
+               :           :     :           :     :           :     :     :     +- Scan parquet default.catalog_sales (1)
+               :           :     :           :     :           :     :     +- BroadcastExchange (8)
+               :           :     :           :     :           :     :        +- * Project (7)
+               :           :     :           :     :           :     :           +- * Filter (6)
+               :           :     :           :     :           :     :              +- * ColumnarToRow (5)
+               :           :     :           :     :           :     :                 +- Scan parquet default.household_demographics (4)
+               :           :     :           :     :           :     +- BroadcastExchange (15)
+               :           :     :           :     :           :        +- * Project (14)
+               :           :     :           :     :           :           +- * Filter (13)
+               :           :     :           :     :           :              +- * ColumnarToRow (12)
+               :           :     :           :     :           :                 +- Scan parquet default.customer_demographics (11)
+               :           :     :           :     :           +- BroadcastExchange (21)
+               :           :     :           :     :              +- * Filter (20)
+               :           :     :           :     :                 +- * ColumnarToRow (19)
+               :           :     :           :     :                    +- Scan parquet default.date_dim (18)
+               :           :     :           :     +- * Sort (30)
+               :           :     :           :        +- Exchange (29)
+               :           :     :           :           +- * Filter (28)
+               :           :     :           :              +- * ColumnarToRow (27)
+               :           :     :           :                 +- Scan parquet default.item (26)
+               :           :     :           +- BroadcastExchange (43)
+               :           :     :              +- * Project (42)
+               :           :     :                 +- * BroadcastHashJoin Inner BuildLeft (41)
+               :           :     :                    :- BroadcastExchange (37)
+               :           :     :                    :  +- * Project (36)
+               :           :     :                    :     +- * Filter (35)
+               :           :     :                    :        +- * ColumnarToRow (34)
+               :           :     :                    :           +- Scan parquet default.date_dim (33)
+               :           :     :                    +- * Filter (40)
+               :           :     :                       +- * ColumnarToRow (39)
+               :           :     :                          +- Scan parquet default.date_dim (38)
+               :           :     +- * Sort (58)
+               :           :        +- Exchange (57)
+               :           :           +- * Project (56)
+               :           :              +- * BroadcastHashJoin Inner BuildRight (55)
+               :           :                 :- * Filter (50)
+               :           :                 :  +- * ColumnarToRow (49)
+               :           :                 :     +- Scan parquet default.inventory (48)
+               :           :                 +- BroadcastExchange (54)
+               :           :                    +- * Filter (53)
+               :           :                       +- * ColumnarToRow (52)
+               :           :                          +- Scan parquet default.warehouse (51)
+               :           +- BroadcastExchange (64)
+               :              +- * Filter (63)
+               :                 +- * ColumnarToRow (62)
+               :                    +- Scan parquet default.promotion (61)
+               +- * Sort (74)
+                  +- Exchange (73)
+                     +- * Project (72)
+                        +- * Filter (71)
+                           +- * ColumnarToRow (70)
+                              +- Scan parquet default.catalog_returns (69)
+
+
+(1) Scan parquet default.catalog_sales
+Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#8), dynamicpruningexpression(cs_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(cs_quantity), IsNotNull(cs_item_sk), IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_hdemo_sk), IsNotNull(cs_ship_date_sk)]
+ReadSchema: struct<cs_ship_date_sk:int,cs_bill_cdemo_sk:int,cs_bill_hdemo_sk:int,cs_item_sk:int,cs_promo_sk:int,cs_order_number:int,cs_quantity:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8]
+
+(3) Filter [codegen id : 4]
+Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8]
+Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1))
+
+(4) Scan parquet default.household_demographics
+Output [2]: [hd_demo_sk#10, hd_buy_potential#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_buy_potential), EqualTo(hd_buy_potential,>10000         ), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_buy_potential:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [hd_demo_sk#10, hd_buy_potential#11]
+
+(6) Filter [codegen id : 1]
+Input [2]: [hd_demo_sk#10, hd_buy_potential#11]
+Condition : ((isnotnull(hd_buy_potential#11) AND (hd_buy_potential#11 = >10000         )) AND isnotnull(hd_demo_sk#10))
+
+(7) Project [codegen id : 1]
+Output [1]: [hd_demo_sk#10]
+Input [2]: [hd_demo_sk#10, hd_buy_potential#11]
+
+(8) BroadcastExchange
+Input [1]: [hd_demo_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_bill_hdemo_sk#3]
+Right keys [1]: [hd_demo_sk#10]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [7]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8]
+Input [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, hd_demo_sk#10]
+
+(11) Scan parquet default.customer_demographics
+Output [2]: [cd_demo_sk#13, cd_marital_status#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_marital_status), EqualTo(cd_marital_status,D), IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_marital_status:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [cd_demo_sk#13, cd_marital_status#14]
+
+(13) Filter [codegen id : 2]
+Input [2]: [cd_demo_sk#13, cd_marital_status#14]
+Condition : ((isnotnull(cd_marital_status#14) AND (cd_marital_status#14 = D)) AND isnotnull(cd_demo_sk#13))
+
+(14) Project [codegen id : 2]
+Output [1]: [cd_demo_sk#13]
+Input [2]: [cd_demo_sk#13, cd_marital_status#14]
+
+(15) BroadcastExchange
+Input [1]: [cd_demo_sk#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#15]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_bill_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#13]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [6]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8]
+Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, cd_demo_sk#13]
+
+(18) Scan parquet default.date_dim
+Output [2]: [d_date_sk#16, d_date#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [2]: [d_date_sk#16, d_date#17]
+
+(20) Filter [codegen id : 3]
+Input [2]: [d_date_sk#16, d_date#17]
+Condition : (isnotnull(d_date#17) AND isnotnull(d_date_sk#16))
+
+(21) BroadcastExchange
+Input [2]: [d_date_sk#16, d_date#17]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#18]
+
+(22) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_ship_date_sk#1]
+Right keys [1]: [d_date_sk#16]
+Join condition: None
+
+(23) Project [codegen id : 4]
+Output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, d_date#17]
+Input [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, d_date_sk#16, d_date#17]
+
+(24) Exchange
+Input [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, d_date#17]
+Arguments: hashpartitioning(cs_item_sk#4, 5), ENSURE_REQUIREMENTS, [id=#19]
+
+(25) Sort [codegen id : 5]
+Input [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, d_date#17]
+Arguments: [cs_item_sk#4 ASC NULLS FIRST], false, 0
+
+(26) Scan parquet default.item
+Output [2]: [i_item_sk#20, i_item_desc#21]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_desc:string>
+
+(27) ColumnarToRow [codegen id : 6]
+Input [2]: [i_item_sk#20, i_item_desc#21]
+
+(28) Filter [codegen id : 6]
+Input [2]: [i_item_sk#20, i_item_desc#21]
+Condition : isnotnull(i_item_sk#20)
+
+(29) Exchange
+Input [2]: [i_item_sk#20, i_item_desc#21]
+Arguments: hashpartitioning(i_item_sk#20, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(30) Sort [codegen id : 7]
+Input [2]: [i_item_sk#20, i_item_desc#21]
+Arguments: [i_item_sk#20 ASC NULLS FIRST], false, 0
+
+(31) SortMergeJoin [codegen id : 10]
+Left keys [1]: [cs_item_sk#4]
+Right keys [1]: [i_item_sk#20]
+Join condition: None
+
+(32) Project [codegen id : 10]
+Output [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, d_date#17, i_item_desc#21]
+Input [8]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, d_date#17, i_item_sk#20, i_item_desc#21]
+
+(33) Scan parquet default.date_dim
+Output [4]: [d_date_sk#23, d_date#24, d_week_seq#25, d_year#26]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)]
+ReadSchema: struct<d_date_sk:int,d_date:date,d_week_seq:int,d_year:int>
+
+(34) ColumnarToRow [codegen id : 8]
+Input [4]: [d_date_sk#23, d_date#24, d_week_seq#25, d_year#26]
+
+(35) Filter [codegen id : 8]
+Input [4]: [d_date_sk#23, d_date#24, d_week_seq#25, d_year#26]
+Condition : ((((isnotnull(d_year#26) AND (d_year#26 = 1999)) AND isnotnull(d_date_sk#23)) AND isnotnull(d_week_seq#25)) AND isnotnull(d_date#24))
+
+(36) Project [codegen id : 8]
+Output [3]: [d_date_sk#23, d_date#24, d_week_seq#25]
+Input [4]: [d_date_sk#23, d_date#24, d_week_seq#25, d_year#26]
+
+(37) BroadcastExchange
+Input [3]: [d_date_sk#23, d_date#24, d_week_seq#25]
+Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [id=#27]
+
+(38) Scan parquet default.date_dim
+Output [2]: [d_date_sk#28, d_week_seq#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_week_seq:int>
+
+(39) ColumnarToRow
+Input [2]: [d_date_sk#28, d_week_seq#29]
+
+(40) Filter
+Input [2]: [d_date_sk#28, d_week_seq#29]
+Condition : (isnotnull(d_week_seq#29) AND isnotnull(d_date_sk#28))
+
+(41) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [d_week_seq#25]
+Right keys [1]: [d_week_seq#29]
+Join condition: None
+
+(42) Project [codegen id : 9]
+Output [4]: [d_date_sk#23, d_date#24, d_week_seq#25, d_date_sk#28]
+Input [5]: [d_date_sk#23, d_date#24, d_week_seq#25, d_date_sk#28, d_week_seq#29]
+
+(43) BroadcastExchange
+Input [4]: [d_date_sk#23, d_date#24, d_week_seq#25, d_date_sk#28]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#30]
+
+(44) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_sold_date_sk#8]
+Right keys [1]: [d_date_sk#23]
+Join condition: (d_date#17 > d_date#24 + 5 days)
+
+(45) Project [codegen id : 10]
+Output [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, i_item_desc#21, d_week_seq#25, d_date_sk#28]
+Input [11]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, d_date#17, i_item_desc#21, d_date_sk#23, d_date#24, d_week_seq#25, d_date_sk#28]
+
+(46) Exchange
+Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, i_item_desc#21, d_week_seq#25, d_date_sk#28]
+Arguments: hashpartitioning(cs_item_sk#4, d_date_sk#28, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(47) Sort [codegen id : 11]
+Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, i_item_desc#21, d_week_seq#25, d_date_sk#28]
+Arguments: [cs_item_sk#4 ASC NULLS FIRST, d_date_sk#28 ASC NULLS FIRST], false, 0
+
+(48) Scan parquet default.inventory
+Output [4]: [inv_item_sk#32, inv_warehouse_sk#33, inv_quantity_on_hand#34, inv_date_sk#35]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(inv_date_sk#35), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(inv_quantity_on_hand), IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)]
+ReadSchema: struct<inv_item_sk:int,inv_warehouse_sk:int,inv_quantity_on_hand:int>
+
+(49) ColumnarToRow [codegen id : 13]
+Input [4]: [inv_item_sk#32, inv_warehouse_sk#33, inv_quantity_on_hand#34, inv_date_sk#35]
+
+(50) Filter [codegen id : 13]
+Input [4]: [inv_item_sk#32, inv_warehouse_sk#33, inv_quantity_on_hand#34, inv_date_sk#35]
+Condition : ((isnotnull(inv_quantity_on_hand#34) AND isnotnull(inv_item_sk#32)) AND isnotnull(inv_warehouse_sk#33))
+
+(51) Scan parquet default.warehouse
+Output [2]: [w_warehouse_sk#36, w_warehouse_name#37]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/warehouse]
+PushedFilters: [IsNotNull(w_warehouse_sk)]
+ReadSchema: struct<w_warehouse_sk:int,w_warehouse_name:string>
+
+(52) ColumnarToRow [codegen id : 12]
+Input [2]: [w_warehouse_sk#36, w_warehouse_name#37]
+
+(53) Filter [codegen id : 12]
+Input [2]: [w_warehouse_sk#36, w_warehouse_name#37]
+Condition : isnotnull(w_warehouse_sk#36)
+
+(54) BroadcastExchange
+Input [2]: [w_warehouse_sk#36, w_warehouse_name#37]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#38]
+
+(55) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [inv_warehouse_sk#33]
+Right keys [1]: [w_warehouse_sk#36]
+Join condition: None
+
+(56) Project [codegen id : 13]
+Output [4]: [inv_item_sk#32, inv_quantity_on_hand#34, inv_date_sk#35, w_warehouse_name#37]
+Input [6]: [inv_item_sk#32, inv_warehouse_sk#33, inv_quantity_on_hand#34, inv_date_sk#35, w_warehouse_sk#36, w_warehouse_name#37]
+
+(57) Exchange
+Input [4]: [inv_item_sk#32, inv_quantity_on_hand#34, inv_date_sk#35, w_warehouse_name#37]
+Arguments: hashpartitioning(inv_item_sk#32, inv_date_sk#35, 5), ENSURE_REQUIREMENTS, [id=#39]
+
+(58) Sort [codegen id : 14]
+Input [4]: [inv_item_sk#32, inv_quantity_on_hand#34, inv_date_sk#35, w_warehouse_name#37]
+Arguments: [inv_item_sk#32 ASC NULLS FIRST, inv_date_sk#35 ASC NULLS FIRST], false, 0
+
+(59) SortMergeJoin [codegen id : 16]
+Left keys [2]: [cs_item_sk#4, d_date_sk#28]
+Right keys [2]: [inv_item_sk#32, inv_date_sk#35]
+Join condition: (inv_quantity_on_hand#34 < cs_quantity#7)
+
+(60) Project [codegen id : 16]
+Output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#37, i_item_desc#21, d_week_seq#25]
+Input [11]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, i_item_desc#21, d_week_seq#25, d_date_sk#28, inv_item_sk#32, inv_quantity_on_hand#34, inv_date_sk#35, w_warehouse_name#37]
+
+(61) Scan parquet default.promotion
+Output [1]: [p_promo_sk#40]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/promotion]
+PushedFilters: [IsNotNull(p_promo_sk)]
+ReadSchema: struct<p_promo_sk:int>
+
+(62) ColumnarToRow [codegen id : 15]
+Input [1]: [p_promo_sk#40]
+
+(63) Filter [codegen id : 15]
+Input [1]: [p_promo_sk#40]
+Condition : isnotnull(p_promo_sk#40)
+
+(64) BroadcastExchange
+Input [1]: [p_promo_sk#40]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#41]
+
+(65) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [cs_promo_sk#5]
+Right keys [1]: [p_promo_sk#40]
+Join condition: None
+
+(66) Project [codegen id : 16]
+Output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#37, i_item_desc#21, d_week_seq#25]
+Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#37, i_item_desc#21, d_week_seq#25, p_promo_sk#40]
+
+(67) Exchange
+Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#37, i_item_desc#21, d_week_seq#25]
+Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, [id=#42]
+
+(68) Sort [codegen id : 17]
+Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#37, i_item_desc#21, d_week_seq#25]
+Arguments: [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST], false, 0
+
+(69) Scan parquet default.catalog_returns
+Output [3]: [cr_item_sk#43, cr_order_number#44, cr_returned_date_sk#45]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int>
+
+(70) ColumnarToRow [codegen id : 18]
+Input [3]: [cr_item_sk#43, cr_order_number#44, cr_returned_date_sk#45]
+
+(71) Filter [codegen id : 18]
+Input [3]: [cr_item_sk#43, cr_order_number#44, cr_returned_date_sk#45]
+Condition : (isnotnull(cr_item_sk#43) AND isnotnull(cr_order_number#44))
+
+(72) Project [codegen id : 18]
+Output [2]: [cr_item_sk#43, cr_order_number#44]
+Input [3]: [cr_item_sk#43, cr_order_number#44, cr_returned_date_sk#45]
+
+(73) Exchange
+Input [2]: [cr_item_sk#43, cr_order_number#44]
+Arguments: hashpartitioning(cr_item_sk#43, cr_order_number#44, 5), ENSURE_REQUIREMENTS, [id=#46]
+
+(74) Sort [codegen id : 19]
+Input [2]: [cr_item_sk#43, cr_order_number#44]
+Arguments: [cr_item_sk#43 ASC NULLS FIRST, cr_order_number#44 ASC NULLS FIRST], false, 0
+
+(75) SortMergeJoin [codegen id : 20]
+Left keys [2]: [cs_item_sk#4, cs_order_number#6]
+Right keys [2]: [cr_item_sk#43, cr_order_number#44]
+Join condition: None
+
+(76) Project [codegen id : 20]
+Output [3]: [w_warehouse_name#37, i_item_desc#21, d_week_seq#25]
+Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#37, i_item_desc#21, d_week_seq#25, cr_item_sk#43, cr_order_number#44]
+
+(77) HashAggregate [codegen id : 20]
+Input [3]: [w_warehouse_name#37, i_item_desc#21, d_week_seq#25]
+Keys [3]: [i_item_desc#21, w_warehouse_name#37, d_week_seq#25]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#47]
+Results [4]: [i_item_desc#21, w_warehouse_name#37, d_week_seq#25, count#48]
+
+(78) Exchange
+Input [4]: [i_item_desc#21, w_warehouse_name#37, d_week_seq#25, count#48]
+Arguments: hashpartitioning(i_item_desc#21, w_warehouse_name#37, d_week_seq#25, 5), ENSURE_REQUIREMENTS, [id=#49]
+
+(79) HashAggregate [codegen id : 21]
+Input [4]: [i_item_desc#21, w_warehouse_name#37, d_week_seq#25, count#48]
+Keys [3]: [i_item_desc#21, w_warehouse_name#37, d_week_seq#25]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#50]
+Results [6]: [i_item_desc#21, w_warehouse_name#37, d_week_seq#25, count(1)#50 AS no_promo#51, count(1)#50 AS promo#52, count(1)#50 AS total_cnt#53]
+
+(80) TakeOrderedAndProject
+Input [6]: [i_item_desc#21, w_warehouse_name#37, d_week_seq#25, no_promo#51, promo#52, total_cnt#53]
+Arguments: 100, [total_cnt#53 DESC NULLS LAST, i_item_desc#21 ASC NULLS FIRST, w_warehouse_name#37 ASC NULLS FIRST, d_week_seq#25 ASC NULLS FIRST], [i_item_desc#21, w_warehouse_name#37, d_week_seq#25, no_promo#51, promo#52, total_cnt#53]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (81)
+
+
+(81) ReusedExchange [Reuses operator id: 43]
+Output [4]: [d_date_sk#23, d_date#24, d_week_seq#25, d_date_sk#28]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.sf100/simplified.txt
new file mode 100644
index 0000000000000..da6b79ca3c79d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.sf100/simplified.txt
@@ -0,0 +1,134 @@
+TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo]
+  WholeStageCodegen (21)
+    HashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] [count(1),no_promo,promo,total_cnt,count]
+      InputAdapter
+        Exchange [i_item_desc,w_warehouse_name,d_week_seq] #1
+          WholeStageCodegen (20)
+            HashAggregate [i_item_desc,w_warehouse_name,d_week_seq] [count,count]
+              Project [w_warehouse_name,i_item_desc,d_week_seq]
+                SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number]
+                  InputAdapter
+                    WholeStageCodegen (17)
+                      Sort [cs_item_sk,cs_order_number]
+                        InputAdapter
+                          Exchange [cs_item_sk,cs_order_number] #2
+                            WholeStageCodegen (16)
+                              Project [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq]
+                                BroadcastHashJoin [cs_promo_sk,p_promo_sk]
+                                  Project [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq]
+                                    SortMergeJoin [cs_item_sk,d_date_sk,inv_item_sk,inv_date_sk,inv_quantity_on_hand,cs_quantity]
+                                      InputAdapter
+                                        WholeStageCodegen (11)
+                                          Sort [cs_item_sk,d_date_sk]
+                                            InputAdapter
+                                              Exchange [cs_item_sk,d_date_sk] #3
+                                                WholeStageCodegen (10)
+                                                  Project [cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,i_item_desc,d_week_seq,d_date_sk]
+                                                    BroadcastHashJoin [cs_sold_date_sk,d_date_sk,d_date,d_date]
+                                                      Project [cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,d_date,i_item_desc]
+                                                        SortMergeJoin [cs_item_sk,i_item_sk]
+                                                          InputAdapter
+                                                            WholeStageCodegen (5)
+                                                              Sort [cs_item_sk]
+                                                                InputAdapter
+                                                                  Exchange [cs_item_sk] #4
+                                                                    WholeStageCodegen (4)
+                                                                      Project [cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,d_date]
+                                                                        BroadcastHashJoin [cs_ship_date_sk,d_date_sk]
+                                                                          Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk]
+                                                                            BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk]
+                                                                              Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk]
+                                                                                BroadcastHashJoin [cs_bill_hdemo_sk,hd_demo_sk]
+                                                                                  Filter [cs_quantity,cs_item_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_ship_date_sk]
+                                                                                    ColumnarToRow
+                                                                                      InputAdapter
+                                                                                        Scan parquet default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk]
+                                                                                          SubqueryBroadcast [d_date_sk] #1
+                                                                                            ReusedExchange [d_date_sk,d_date,d_week_seq,d_date_sk] #5
+                                                                                  InputAdapter
+                                                                                    BroadcastExchange #6
+                                                                                      WholeStageCodegen (1)
+                                                                                        Project [hd_demo_sk]
+                                                                                          Filter [hd_buy_potential,hd_demo_sk]
+                                                                                            ColumnarToRow
+                                                                                              InputAdapter
+                                                                                                Scan parquet default.household_demographics [hd_demo_sk,hd_buy_potential]
+                                                                              InputAdapter
+                                                                                BroadcastExchange #7
+                                                                                  WholeStageCodegen (2)
+                                                                                    Project [cd_demo_sk]
+                                                                                      Filter [cd_marital_status,cd_demo_sk]
+                                                                                        ColumnarToRow
+                                                                                          InputAdapter
+                                                                                            Scan parquet default.customer_demographics [cd_demo_sk,cd_marital_status]
+                                                                          InputAdapter
+                                                                            BroadcastExchange #8
+                                                                              WholeStageCodegen (3)
+                                                                                Filter [d_date,d_date_sk]
+                                                                                  ColumnarToRow
+                                                                                    InputAdapter
+                                                                                      Scan parquet default.date_dim [d_date_sk,d_date]
+                                                          InputAdapter
+                                                            WholeStageCodegen (7)
+                                                              Sort [i_item_sk]
+                                                                InputAdapter
+                                                                  Exchange [i_item_sk] #9
+                                                                    WholeStageCodegen (6)
+                                                                      Filter [i_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.item [i_item_sk,i_item_desc]
+                                                      InputAdapter
+                                                        BroadcastExchange #5
+                                                          WholeStageCodegen (9)
+                                                            Project [d_date_sk,d_date,d_week_seq,d_date_sk]
+                                                              BroadcastHashJoin [d_week_seq,d_week_seq]
+                                                                InputAdapter
+                                                                  BroadcastExchange #10
+                                                                    WholeStageCodegen (8)
+                                                                      Project [d_date_sk,d_date,d_week_seq]
+                                                                        Filter [d_year,d_date_sk,d_week_seq,d_date]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.date_dim [d_date_sk,d_date,d_week_seq,d_year]
+                                                                Filter [d_week_seq,d_date_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.date_dim [d_date_sk,d_week_seq]
+                                      InputAdapter
+                                        WholeStageCodegen (14)
+                                          Sort [inv_item_sk,inv_date_sk]
+                                            InputAdapter
+                                              Exchange [inv_item_sk,inv_date_sk] #11
+                                                WholeStageCodegen (13)
+                                                  Project [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name]
+                                                    BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk]
+                                                      Filter [inv_quantity_on_hand,inv_item_sk,inv_warehouse_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk]
+                                                      InputAdapter
+                                                        BroadcastExchange #12
+                                                          WholeStageCodegen (12)
+                                                            Filter [w_warehouse_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.warehouse [w_warehouse_sk,w_warehouse_name]
+                                  InputAdapter
+                                    BroadcastExchange #13
+                                      WholeStageCodegen (15)
+                                        Filter [p_promo_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.promotion [p_promo_sk]
+                  InputAdapter
+                    WholeStageCodegen (19)
+                      Sort [cr_item_sk,cr_order_number]
+                        InputAdapter
+                          Exchange [cr_item_sk,cr_order_number] #14
+                            WholeStageCodegen (18)
+                              Project [cr_item_sk,cr_order_number]
+                                Filter [cr_item_sk,cr_order_number]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/explain.txt
new file mode 100644
index 0000000000000..75a96ae5d2cdd
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/explain.txt
@@ -0,0 +1,423 @@
+== Physical Plan ==
+TakeOrderedAndProject (74)
++- * HashAggregate (73)
+   +- Exchange (72)
+      +- * HashAggregate (71)
+         +- * Project (70)
+            +- * SortMergeJoin LeftOuter (69)
+               :- * Sort (62)
+               :  +- Exchange (61)
+               :     +- * Project (60)
+               :        +- * BroadcastHashJoin LeftOuter BuildRight (59)
+               :           :- * Project (54)
+               :           :  +- * BroadcastHashJoin Inner BuildRight (53)
+               :           :     :- * Project (48)
+               :           :     :  +- * BroadcastHashJoin Inner BuildRight (47)
+               :           :     :     :- * Project (42)
+               :           :     :     :  +- * BroadcastHashJoin Inner BuildRight (41)
+               :           :     :     :     :- * Project (35)
+               :           :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (34)
+               :           :     :     :     :     :- * Project (28)
+               :           :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (27)
+               :           :     :     :     :     :     :- * Project (21)
+               :           :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (20)
+               :           :     :     :     :     :     :     :- * Project (15)
+               :           :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (14)
+               :           :     :     :     :     :     :     :     :- * Project (9)
+               :           :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+               :           :     :     :     :     :     :     :     :     :- * Filter (3)
+               :           :     :     :     :     :     :     :     :     :  +- * ColumnarToRow (2)
+               :           :     :     :     :     :     :     :     :     :     +- Scan parquet default.catalog_sales (1)
+               :           :     :     :     :     :     :     :     :     +- BroadcastExchange (7)
+               :           :     :     :     :     :     :     :     :        +- * Filter (6)
+               :           :     :     :     :     :     :     :     :           +- * ColumnarToRow (5)
+               :           :     :     :     :     :     :     :     :              +- Scan parquet default.inventory (4)
+               :           :     :     :     :     :     :     :     +- BroadcastExchange (13)
+               :           :     :     :     :     :     :     :        +- * Filter (12)
+               :           :     :     :     :     :     :     :           +- * ColumnarToRow (11)
+               :           :     :     :     :     :     :     :              +- Scan parquet default.warehouse (10)
+               :           :     :     :     :     :     :     +- BroadcastExchange (19)
+               :           :     :     :     :     :     :        +- * Filter (18)
+               :           :     :     :     :     :     :           +- * ColumnarToRow (17)
+               :           :     :     :     :     :     :              +- Scan parquet default.item (16)
+               :           :     :     :     :     :     +- BroadcastExchange (26)
+               :           :     :     :     :     :        +- * Project (25)
+               :           :     :     :     :     :           +- * Filter (24)
+               :           :     :     :     :     :              +- * ColumnarToRow (23)
+               :           :     :     :     :     :                 +- Scan parquet default.customer_demographics (22)
+               :           :     :     :     :     +- BroadcastExchange (33)
+               :           :     :     :     :        +- * Project (32)
+               :           :     :     :     :           +- * Filter (31)
+               :           :     :     :     :              +- * ColumnarToRow (30)
+               :           :     :     :     :                 +- Scan parquet default.household_demographics (29)
+               :           :     :     :     +- BroadcastExchange (40)
+               :           :     :     :        +- * Project (39)
+               :           :     :     :           +- * Filter (38)
+               :           :     :     :              +- * ColumnarToRow (37)
+               :           :     :     :                 +- Scan parquet default.date_dim (36)
+               :           :     :     +- BroadcastExchange (46)
+               :           :     :        +- * Filter (45)
+               :           :     :           +- * ColumnarToRow (44)
+               :           :     :              +- Scan parquet default.date_dim (43)
+               :           :     +- BroadcastExchange (52)
+               :           :        +- * Filter (51)
+               :           :           +- * ColumnarToRow (50)
+               :           :              +- Scan parquet default.date_dim (49)
+               :           +- BroadcastExchange (58)
+               :              +- * Filter (57)
+               :                 +- * ColumnarToRow (56)
+               :                    +- Scan parquet default.promotion (55)
+               +- * Sort (68)
+                  +- Exchange (67)
+                     +- * Project (66)
+                        +- * Filter (65)
+                           +- * ColumnarToRow (64)
+                              +- Scan parquet default.catalog_returns (63)
+
+
+(1) Scan parquet default.catalog_sales
+Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#8), dynamicpruningexpression(cs_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(cs_quantity), IsNotNull(cs_item_sk), IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_hdemo_sk), IsNotNull(cs_ship_date_sk)]
+ReadSchema: struct<cs_ship_date_sk:int,cs_bill_cdemo_sk:int,cs_bill_hdemo_sk:int,cs_item_sk:int,cs_promo_sk:int,cs_order_number:int,cs_quantity:int>
+
+(2) ColumnarToRow [codegen id : 10]
+Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8]
+
+(3) Filter [codegen id : 10]
+Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8]
+Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1))
+
+(4) Scan parquet default.inventory
+Output [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(inv_date_sk#13)]
+PushedFilters: [IsNotNull(inv_quantity_on_hand), IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)]
+ReadSchema: struct<inv_item_sk:int,inv_warehouse_sk:int,inv_quantity_on_hand:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13]
+
+(6) Filter [codegen id : 1]
+Input [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13]
+Condition : ((isnotnull(inv_quantity_on_hand#12) AND isnotnull(inv_item_sk#10)) AND isnotnull(inv_warehouse_sk#11))
+
+(7) BroadcastExchange
+Input [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#14]
+
+(8) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_item_sk#4]
+Right keys [1]: [inv_item_sk#10]
+Join condition: (inv_quantity_on_hand#12 < cs_quantity#7)
+
+(9) Project [codegen id : 10]
+Output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13]
+Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13]
+
+(10) Scan parquet default.warehouse
+Output [2]: [w_warehouse_sk#15, w_warehouse_name#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/warehouse]
+PushedFilters: [IsNotNull(w_warehouse_sk)]
+ReadSchema: struct<w_warehouse_sk:int,w_warehouse_name:string>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [w_warehouse_sk#15, w_warehouse_name#16]
+
+(12) Filter [codegen id : 2]
+Input [2]: [w_warehouse_sk#15, w_warehouse_name#16]
+Condition : isnotnull(w_warehouse_sk#15)
+
+(13) BroadcastExchange
+Input [2]: [w_warehouse_sk#15, w_warehouse_name#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#17]
+
+(14) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [inv_warehouse_sk#11]
+Right keys [1]: [w_warehouse_sk#15]
+Join condition: None
+
+(15) Project [codegen id : 10]
+Output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#16]
+Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13, w_warehouse_sk#15, w_warehouse_name#16]
+
+(16) Scan parquet default.item
+Output [2]: [i_item_sk#18, i_item_desc#19]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_desc:string>
+
+(17) ColumnarToRow [codegen id : 3]
+Input [2]: [i_item_sk#18, i_item_desc#19]
+
+(18) Filter [codegen id : 3]
+Input [2]: [i_item_sk#18, i_item_desc#19]
+Condition : isnotnull(i_item_sk#18)
+
+(19) BroadcastExchange
+Input [2]: [i_item_sk#18, i_item_desc#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#20]
+
+(20) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_item_sk#4]
+Right keys [1]: [i_item_sk#18]
+Join condition: None
+
+(21) Project [codegen id : 10]
+Output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#16, i_item_desc#19]
+Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#16, i_item_sk#18, i_item_desc#19]
+
+(22) Scan parquet default.customer_demographics
+Output [2]: [cd_demo_sk#21, cd_marital_status#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_marital_status), EqualTo(cd_marital_status,D), IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_marital_status:string>
+
+(23) ColumnarToRow [codegen id : 4]
+Input [2]: [cd_demo_sk#21, cd_marital_status#22]
+
+(24) Filter [codegen id : 4]
+Input [2]: [cd_demo_sk#21, cd_marital_status#22]
+Condition : ((isnotnull(cd_marital_status#22) AND (cd_marital_status#22 = D)) AND isnotnull(cd_demo_sk#21))
+
+(25) Project [codegen id : 4]
+Output [1]: [cd_demo_sk#21]
+Input [2]: [cd_demo_sk#21, cd_marital_status#22]
+
+(26) BroadcastExchange
+Input [1]: [cd_demo_sk#21]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#23]
+
+(27) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_bill_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#21]
+Join condition: None
+
+(28) Project [codegen id : 10]
+Output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#16, i_item_desc#19]
+Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#16, i_item_desc#19, cd_demo_sk#21]
+
+(29) Scan parquet default.household_demographics
+Output [2]: [hd_demo_sk#24, hd_buy_potential#25]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_buy_potential), EqualTo(hd_buy_potential,>10000         ), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_buy_potential:string>
+
+(30) ColumnarToRow [codegen id : 5]
+Input [2]: [hd_demo_sk#24, hd_buy_potential#25]
+
+(31) Filter [codegen id : 5]
+Input [2]: [hd_demo_sk#24, hd_buy_potential#25]
+Condition : ((isnotnull(hd_buy_potential#25) AND (hd_buy_potential#25 = >10000         )) AND isnotnull(hd_demo_sk#24))
+
+(32) Project [codegen id : 5]
+Output [1]: [hd_demo_sk#24]
+Input [2]: [hd_demo_sk#24, hd_buy_potential#25]
+
+(33) BroadcastExchange
+Input [1]: [hd_demo_sk#24]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#26]
+
+(34) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_bill_hdemo_sk#3]
+Right keys [1]: [hd_demo_sk#24]
+Join condition: None
+
+(35) Project [codegen id : 10]
+Output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#16, i_item_desc#19]
+Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#16, i_item_desc#19, hd_demo_sk#24]
+
+(36) Scan parquet default.date_dim
+Output [4]: [d_date_sk#27, d_date#28, d_week_seq#29, d_year#30]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)]
+ReadSchema: struct<d_date_sk:int,d_date:date,d_week_seq:int,d_year:int>
+
+(37) ColumnarToRow [codegen id : 6]
+Input [4]: [d_date_sk#27, d_date#28, d_week_seq#29, d_year#30]
+
+(38) Filter [codegen id : 6]
+Input [4]: [d_date_sk#27, d_date#28, d_week_seq#29, d_year#30]
+Condition : ((((isnotnull(d_year#30) AND (d_year#30 = 1999)) AND isnotnull(d_date_sk#27)) AND isnotnull(d_week_seq#29)) AND isnotnull(d_date#28))
+
+(39) Project [codegen id : 6]
+Output [3]: [d_date_sk#27, d_date#28, d_week_seq#29]
+Input [4]: [d_date_sk#27, d_date#28, d_week_seq#29, d_year#30]
+
+(40) BroadcastExchange
+Input [3]: [d_date_sk#27, d_date#28, d_week_seq#29]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#31]
+
+(41) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_sold_date_sk#8]
+Right keys [1]: [d_date_sk#27]
+Join condition: None
+
+(42) Project [codegen id : 10]
+Output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#16, i_item_desc#19, d_date#28, d_week_seq#29]
+Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#16, i_item_desc#19, d_date_sk#27, d_date#28, d_week_seq#29]
+
+(43) Scan parquet default.date_dim
+Output [2]: [d_date_sk#32, d_week_seq#33]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_week_seq:int>
+
+(44) ColumnarToRow [codegen id : 7]
+Input [2]: [d_date_sk#32, d_week_seq#33]
+
+(45) Filter [codegen id : 7]
+Input [2]: [d_date_sk#32, d_week_seq#33]
+Condition : (isnotnull(d_week_seq#33) AND isnotnull(d_date_sk#32))
+
+(46) BroadcastExchange
+Input [2]: [d_date_sk#32, d_week_seq#33]
+Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, false] as bigint), 32) | (cast(input[0, int, false] as bigint) & 4294967295))),false), [id=#34]
+
+(47) BroadcastHashJoin [codegen id : 10]
+Left keys [2]: [d_week_seq#29, inv_date_sk#13]
+Right keys [2]: [d_week_seq#33, d_date_sk#32]
+Join condition: None
+
+(48) Project [codegen id : 10]
+Output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#16, i_item_desc#19, d_date#28, d_week_seq#29]
+Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#16, i_item_desc#19, d_date#28, d_week_seq#29, d_date_sk#32, d_week_seq#33]
+
+(49) Scan parquet default.date_dim
+Output [2]: [d_date_sk#35, d_date#36]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(50) ColumnarToRow [codegen id : 8]
+Input [2]: [d_date_sk#35, d_date#36]
+
+(51) Filter [codegen id : 8]
+Input [2]: [d_date_sk#35, d_date#36]
+Condition : (isnotnull(d_date#36) AND isnotnull(d_date_sk#35))
+
+(52) BroadcastExchange
+Input [2]: [d_date_sk#35, d_date#36]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#37]
+
+(53) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_ship_date_sk#1]
+Right keys [1]: [d_date_sk#35]
+Join condition: (d_date#36 > d_date#28 + 5 days)
+
+(54) Project [codegen id : 10]
+Output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#16, i_item_desc#19, d_week_seq#29]
+Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#16, i_item_desc#19, d_date#28, d_week_seq#29, d_date_sk#35, d_date#36]
+
+(55) Scan parquet default.promotion
+Output [1]: [p_promo_sk#38]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/promotion]
+PushedFilters: [IsNotNull(p_promo_sk)]
+ReadSchema: struct<p_promo_sk:int>
+
+(56) ColumnarToRow [codegen id : 9]
+Input [1]: [p_promo_sk#38]
+
+(57) Filter [codegen id : 9]
+Input [1]: [p_promo_sk#38]
+Condition : isnotnull(p_promo_sk#38)
+
+(58) BroadcastExchange
+Input [1]: [p_promo_sk#38]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#39]
+
+(59) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_promo_sk#5]
+Right keys [1]: [p_promo_sk#38]
+Join condition: None
+
+(60) Project [codegen id : 10]
+Output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#16, i_item_desc#19, d_week_seq#29]
+Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#16, i_item_desc#19, d_week_seq#29, p_promo_sk#38]
+
+(61) Exchange
+Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#16, i_item_desc#19, d_week_seq#29]
+Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, [id=#40]
+
+(62) Sort [codegen id : 11]
+Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#16, i_item_desc#19, d_week_seq#29]
+Arguments: [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST], false, 0
+
+(63) Scan parquet default.catalog_returns
+Output [3]: [cr_item_sk#41, cr_order_number#42, cr_returned_date_sk#43]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int>
+
+(64) ColumnarToRow [codegen id : 12]
+Input [3]: [cr_item_sk#41, cr_order_number#42, cr_returned_date_sk#43]
+
+(65) Filter [codegen id : 12]
+Input [3]: [cr_item_sk#41, cr_order_number#42, cr_returned_date_sk#43]
+Condition : (isnotnull(cr_item_sk#41) AND isnotnull(cr_order_number#42))
+
+(66) Project [codegen id : 12]
+Output [2]: [cr_item_sk#41, cr_order_number#42]
+Input [3]: [cr_item_sk#41, cr_order_number#42, cr_returned_date_sk#43]
+
+(67) Exchange
+Input [2]: [cr_item_sk#41, cr_order_number#42]
+Arguments: hashpartitioning(cr_item_sk#41, cr_order_number#42, 5), ENSURE_REQUIREMENTS, [id=#44]
+
+(68) Sort [codegen id : 13]
+Input [2]: [cr_item_sk#41, cr_order_number#42]
+Arguments: [cr_item_sk#41 ASC NULLS FIRST, cr_order_number#42 ASC NULLS FIRST], false, 0
+
+(69) SortMergeJoin [codegen id : 14]
+Left keys [2]: [cs_item_sk#4, cs_order_number#6]
+Right keys [2]: [cr_item_sk#41, cr_order_number#42]
+Join condition: None
+
+(70) Project [codegen id : 14]
+Output [3]: [w_warehouse_name#16, i_item_desc#19, d_week_seq#29]
+Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#16, i_item_desc#19, d_week_seq#29, cr_item_sk#41, cr_order_number#42]
+
+(71) HashAggregate [codegen id : 14]
+Input [3]: [w_warehouse_name#16, i_item_desc#19, d_week_seq#29]
+Keys [3]: [i_item_desc#19, w_warehouse_name#16, d_week_seq#29]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#45]
+Results [4]: [i_item_desc#19, w_warehouse_name#16, d_week_seq#29, count#46]
+
+(72) Exchange
+Input [4]: [i_item_desc#19, w_warehouse_name#16, d_week_seq#29, count#46]
+Arguments: hashpartitioning(i_item_desc#19, w_warehouse_name#16, d_week_seq#29, 5), ENSURE_REQUIREMENTS, [id=#47]
+
+(73) HashAggregate [codegen id : 15]
+Input [4]: [i_item_desc#19, w_warehouse_name#16, d_week_seq#29, count#46]
+Keys [3]: [i_item_desc#19, w_warehouse_name#16, d_week_seq#29]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#48]
+Results [6]: [i_item_desc#19, w_warehouse_name#16, d_week_seq#29, count(1)#48 AS no_promo#49, count(1)#48 AS promo#50, count(1)#48 AS total_cnt#51]
+
+(74) TakeOrderedAndProject
+Input [6]: [i_item_desc#19, w_warehouse_name#16, d_week_seq#29, no_promo#49, promo#50, total_cnt#51]
+Arguments: 100, [total_cnt#51 DESC NULLS LAST, i_item_desc#19 ASC NULLS FIRST, w_warehouse_name#16 ASC NULLS FIRST, d_week_seq#29 ASC NULLS FIRST], [i_item_desc#19, w_warehouse_name#16, d_week_seq#29, no_promo#49, promo#50, total_cnt#51]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (75)
+
+
+(75) ReusedExchange [Reuses operator id: 40]
+Output [3]: [d_date_sk#27, d_date#28, d_week_seq#29]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/simplified.txt
new file mode 100644
index 0000000000000..92800d31b82f1
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/simplified.txt
@@ -0,0 +1,116 @@
+TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo]
+  WholeStageCodegen (15)
+    HashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] [count(1),no_promo,promo,total_cnt,count]
+      InputAdapter
+        Exchange [i_item_desc,w_warehouse_name,d_week_seq] #1
+          WholeStageCodegen (14)
+            HashAggregate [i_item_desc,w_warehouse_name,d_week_seq] [count,count]
+              Project [w_warehouse_name,i_item_desc,d_week_seq]
+                SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number]
+                  InputAdapter
+                    WholeStageCodegen (11)
+                      Sort [cs_item_sk,cs_order_number]
+                        InputAdapter
+                          Exchange [cs_item_sk,cs_order_number] #2
+                            WholeStageCodegen (10)
+                              Project [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq]
+                                BroadcastHashJoin [cs_promo_sk,p_promo_sk]
+                                  Project [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq]
+                                    BroadcastHashJoin [cs_ship_date_sk,d_date_sk,d_date,d_date]
+                                      Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq]
+                                        BroadcastHashJoin [d_week_seq,inv_date_sk,d_week_seq,d_date_sk]
+                                          Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq]
+                                            BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                              Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc]
+                                                BroadcastHashJoin [cs_bill_hdemo_sk,hd_demo_sk]
+                                                  Project [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc]
+                                                    BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk]
+                                                      Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc]
+                                                        BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                                          Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name]
+                                                            BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk]
+                                                              Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk]
+                                                                BroadcastHashJoin [cs_item_sk,inv_item_sk,inv_quantity_on_hand,cs_quantity]
+                                                                  Filter [cs_quantity,cs_item_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_ship_date_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk]
+                                                                          SubqueryBroadcast [d_date_sk] #1
+                                                                            ReusedExchange [d_date_sk,d_date,d_week_seq] #3
+                                                                  InputAdapter
+                                                                    BroadcastExchange #4
+                                                                      WholeStageCodegen (1)
+                                                                        Filter [inv_quantity_on_hand,inv_item_sk,inv_warehouse_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk]
+                                                              InputAdapter
+                                                                BroadcastExchange #5
+                                                                  WholeStageCodegen (2)
+                                                                    Filter [w_warehouse_sk]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.warehouse [w_warehouse_sk,w_warehouse_name]
+                                                          InputAdapter
+                                                            BroadcastExchange #6
+                                                              WholeStageCodegen (3)
+                                                                Filter [i_item_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.item [i_item_sk,i_item_desc]
+                                                      InputAdapter
+                                                        BroadcastExchange #7
+                                                          WholeStageCodegen (4)
+                                                            Project [cd_demo_sk]
+                                                              Filter [cd_marital_status,cd_demo_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.customer_demographics [cd_demo_sk,cd_marital_status]
+                                                  InputAdapter
+                                                    BroadcastExchange #8
+                                                      WholeStageCodegen (5)
+                                                        Project [hd_demo_sk]
+                                                          Filter [hd_buy_potential,hd_demo_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.household_demographics [hd_demo_sk,hd_buy_potential]
+                                              InputAdapter
+                                                BroadcastExchange #3
+                                                  WholeStageCodegen (6)
+                                                    Project [d_date_sk,d_date,d_week_seq]
+                                                      Filter [d_year,d_date_sk,d_week_seq,d_date]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.date_dim [d_date_sk,d_date,d_week_seq,d_year]
+                                          InputAdapter
+                                            BroadcastExchange #9
+                                              WholeStageCodegen (7)
+                                                Filter [d_week_seq,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_week_seq]
+                                      InputAdapter
+                                        BroadcastExchange #10
+                                          WholeStageCodegen (8)
+                                            Filter [d_date,d_date_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.date_dim [d_date_sk,d_date]
+                                  InputAdapter
+                                    BroadcastExchange #11
+                                      WholeStageCodegen (9)
+                                        Filter [p_promo_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.promotion [p_promo_sk]
+                  InputAdapter
+                    WholeStageCodegen (13)
+                      Sort [cr_item_sk,cr_order_number]
+                        InputAdapter
+                          Exchange [cr_item_sk,cr_order_number] #12
+                            WholeStageCodegen (12)
+                              Project [cr_item_sk,cr_order_number]
+                                Filter [cr_item_sk,cr_order_number]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.sf100/explain.txt
new file mode 100644
index 0000000000000..aec6d66c98fdd
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.sf100/explain.txt
@@ -0,0 +1,229 @@
+== Physical Plan ==
+* Sort (39)
++- Exchange (38)
+   +- * Project (37)
+      +- * SortMergeJoin Inner (36)
+         :- * Sort (30)
+         :  +- Exchange (29)
+         :     +- * Filter (28)
+         :        +- * HashAggregate (27)
+         :           +- Exchange (26)
+         :              +- * HashAggregate (25)
+         :                 +- * Project (24)
+         :                    +- * BroadcastHashJoin Inner BuildRight (23)
+         :                       :- * Project (17)
+         :                       :  +- * BroadcastHashJoin Inner BuildRight (16)
+         :                       :     :- * Project (10)
+         :                       :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+         :                       :     :     :- * Filter (3)
+         :                       :     :     :  +- * ColumnarToRow (2)
+         :                       :     :     :     +- Scan parquet default.store_sales (1)
+         :                       :     :     +- BroadcastExchange (8)
+         :                       :     :        +- * Project (7)
+         :                       :     :           +- * Filter (6)
+         :                       :     :              +- * ColumnarToRow (5)
+         :                       :     :                 +- Scan parquet default.date_dim (4)
+         :                       :     +- BroadcastExchange (15)
+         :                       :        +- * Project (14)
+         :                       :           +- * Filter (13)
+         :                       :              +- * ColumnarToRow (12)
+         :                       :                 +- Scan parquet default.store (11)
+         :                       +- BroadcastExchange (22)
+         :                          +- * Project (21)
+         :                             +- * Filter (20)
+         :                                +- * ColumnarToRow (19)
+         :                                   +- Scan parquet default.household_demographics (18)
+         +- * Sort (35)
+            +- Exchange (34)
+               +- * Filter (33)
+                  +- * ColumnarToRow (32)
+                     +- Scan parquet default.customer (31)
+
+
+(1) Scan parquet default.store_sales
+Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_hdemo_sk:int,ss_store_sk:int,ss_ticket_number:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+
+(3) Filter [codegen id : 4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#7, d_year#8, d_dom#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_dom:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+Condition : ((((isnotnull(d_dom#9) AND (d_dom#9 >= 1)) AND (d_dom#9 <= 2)) AND d_year#8 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#7]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#5]
+Right keys [1]: [d_date_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4]
+Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#7]
+
+(11) Scan parquet default.store
+Output [2]: [s_store_sk#11, s_county#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [In(s_county, [Williamson County,Franklin Parish,Bronx County,Orange County]), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_county:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [s_store_sk#11, s_county#12]
+
+(13) Filter [codegen id : 2]
+Input [2]: [s_store_sk#11, s_county#12]
+Condition : (s_county#12 IN (Williamson County,Franklin Parish,Bronx County,Orange County) AND isnotnull(s_store_sk#11))
+
+(14) Project [codegen id : 2]
+Output [1]: [s_store_sk#11]
+Input [2]: [s_store_sk#11, s_county#12]
+
+(15) BroadcastExchange
+Input [1]: [s_store_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#11]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#11]
+
+(18) Scan parquet default.household_demographics
+Output [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_vehicle_count), IsNotNull(hd_dep_count), Or(EqualTo(hd_buy_potential,>10000         ),EqualTo(hd_buy_potential,unknown        )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_buy_potential:string,hd_dep_count:int,hd_vehicle_count:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+
+(20) Filter [codegen id : 3]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+Condition : (((((isnotnull(hd_vehicle_count#17) AND isnotnull(hd_dep_count#16)) AND ((hd_buy_potential#15 = >10000         ) OR (hd_buy_potential#15 = unknown        ))) AND (hd_vehicle_count#17 > 0)) AND ((cast(hd_dep_count#16 as double) / cast(hd_vehicle_count#17 as double)) > 1.0)) AND isnotnull(hd_demo_sk#14))
+
+(21) Project [codegen id : 3]
+Output [1]: [hd_demo_sk#14]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+
+(22) BroadcastExchange
+Input [1]: [hd_demo_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(23) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#14]
+Join condition: None
+
+(24) Project [codegen id : 4]
+Output [2]: [ss_customer_sk#1, ss_ticket_number#4]
+Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#14]
+
+(25) HashAggregate [codegen id : 4]
+Input [2]: [ss_customer_sk#1, ss_ticket_number#4]
+Keys [2]: [ss_ticket_number#4, ss_customer_sk#1]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#19]
+Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+
+(26) Exchange
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(27) HashAggregate [codegen id : 5]
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+Keys [2]: [ss_ticket_number#4, ss_customer_sk#1]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#22]
+Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count(1)#22 AS cnt#23]
+
+(28) Filter [codegen id : 5]
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23]
+Condition : ((cnt#23 >= 1) AND (cnt#23 <= 5))
+
+(29) Exchange
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23]
+Arguments: hashpartitioning(ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(30) Sort [codegen id : 6]
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23]
+Arguments: [ss_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(31) Scan parquet default.customer
+Output [5]: [c_customer_sk#25, c_salutation#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_salutation:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string>
+
+(32) ColumnarToRow [codegen id : 7]
+Input [5]: [c_customer_sk#25, c_salutation#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29]
+
+(33) Filter [codegen id : 7]
+Input [5]: [c_customer_sk#25, c_salutation#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29]
+Condition : isnotnull(c_customer_sk#25)
+
+(34) Exchange
+Input [5]: [c_customer_sk#25, c_salutation#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29]
+Arguments: hashpartitioning(c_customer_sk#25, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(35) Sort [codegen id : 8]
+Input [5]: [c_customer_sk#25, c_salutation#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29]
+Arguments: [c_customer_sk#25 ASC NULLS FIRST], false, 0
+
+(36) SortMergeJoin [codegen id : 9]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#25]
+Join condition: None
+
+(37) Project [codegen id : 9]
+Output [6]: [c_last_name#28, c_first_name#27, c_salutation#26, c_preferred_cust_flag#29, ss_ticket_number#4, cnt#23]
+Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23, c_customer_sk#25, c_salutation#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29]
+
+(38) Exchange
+Input [6]: [c_last_name#28, c_first_name#27, c_salutation#26, c_preferred_cust_flag#29, ss_ticket_number#4, cnt#23]
+Arguments: rangepartitioning(cnt#23 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(39) Sort [codegen id : 10]
+Input [6]: [c_last_name#28, c_first_name#27, c_salutation#26, c_preferred_cust_flag#29, ss_ticket_number#4, cnt#23]
+Arguments: [cnt#23 DESC NULLS LAST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (40)
+
+
+(40) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#7]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.sf100/simplified.txt
new file mode 100644
index 0000000000000..5884cd70c28fc
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.sf100/simplified.txt
@@ -0,0 +1,65 @@
+WholeStageCodegen (10)
+  Sort [cnt]
+    InputAdapter
+      Exchange [cnt] #1
+        WholeStageCodegen (9)
+          Project [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt]
+            SortMergeJoin [ss_customer_sk,c_customer_sk]
+              InputAdapter
+                WholeStageCodegen (6)
+                  Sort [ss_customer_sk]
+                    InputAdapter
+                      Exchange [ss_customer_sk] #2
+                        WholeStageCodegen (5)
+                          Filter [cnt]
+                            HashAggregate [ss_ticket_number,ss_customer_sk,count] [count(1),cnt,count]
+                              InputAdapter
+                                Exchange [ss_ticket_number,ss_customer_sk] #3
+                                  WholeStageCodegen (4)
+                                    HashAggregate [ss_ticket_number,ss_customer_sk] [count,count]
+                                      Project [ss_customer_sk,ss_ticket_number]
+                                        BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                          Project [ss_customer_sk,ss_hdemo_sk,ss_ticket_number]
+                                            BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                              Project [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number]
+                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                  Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk]
+                                                          SubqueryBroadcast [d_date_sk] #1
+                                                            ReusedExchange [d_date_sk] #4
+                                                  InputAdapter
+                                                    BroadcastExchange #4
+                                                      WholeStageCodegen (1)
+                                                        Project [d_date_sk]
+                                                          Filter [d_dom,d_year,d_date_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.date_dim [d_date_sk,d_year,d_dom]
+                                              InputAdapter
+                                                BroadcastExchange #5
+                                                  WholeStageCodegen (2)
+                                                    Project [s_store_sk]
+                                                      Filter [s_county,s_store_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.store [s_store_sk,s_county]
+                                          InputAdapter
+                                            BroadcastExchange #6
+                                              WholeStageCodegen (3)
+                                                Project [hd_demo_sk]
+                                                  Filter [hd_vehicle_count,hd_dep_count,hd_buy_potential,hd_demo_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count]
+              InputAdapter
+                WholeStageCodegen (8)
+                  Sort [c_customer_sk]
+                    InputAdapter
+                      Exchange [c_customer_sk] #7
+                        WholeStageCodegen (7)
+                          Filter [c_customer_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/explain.txt
new file mode 100644
index 0000000000000..646a8fbc11a3a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/explain.txt
@@ -0,0 +1,214 @@
+== Physical Plan ==
+* Sort (36)
++- Exchange (35)
+   +- * Project (34)
+      +- * BroadcastHashJoin Inner BuildRight (33)
+         :- * Filter (28)
+         :  +- * HashAggregate (27)
+         :     +- Exchange (26)
+         :        +- * HashAggregate (25)
+         :           +- * Project (24)
+         :              +- * BroadcastHashJoin Inner BuildRight (23)
+         :                 :- * Project (17)
+         :                 :  +- * BroadcastHashJoin Inner BuildRight (16)
+         :                 :     :- * Project (10)
+         :                 :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+         :                 :     :     :- * Filter (3)
+         :                 :     :     :  +- * ColumnarToRow (2)
+         :                 :     :     :     +- Scan parquet default.store_sales (1)
+         :                 :     :     +- BroadcastExchange (8)
+         :                 :     :        +- * Project (7)
+         :                 :     :           +- * Filter (6)
+         :                 :     :              +- * ColumnarToRow (5)
+         :                 :     :                 +- Scan parquet default.date_dim (4)
+         :                 :     +- BroadcastExchange (15)
+         :                 :        +- * Project (14)
+         :                 :           +- * Filter (13)
+         :                 :              +- * ColumnarToRow (12)
+         :                 :                 +- Scan parquet default.store (11)
+         :                 +- BroadcastExchange (22)
+         :                    +- * Project (21)
+         :                       +- * Filter (20)
+         :                          +- * ColumnarToRow (19)
+         :                             +- Scan parquet default.household_demographics (18)
+         +- BroadcastExchange (32)
+            +- * Filter (31)
+               +- * ColumnarToRow (30)
+                  +- Scan parquet default.customer (29)
+
+
+(1) Scan parquet default.store_sales
+Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_hdemo_sk:int,ss_store_sk:int,ss_ticket_number:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+
+(3) Filter [codegen id : 4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#7, d_year#8, d_dom#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_dom:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+Condition : ((((isnotnull(d_dom#9) AND (d_dom#9 >= 1)) AND (d_dom#9 <= 2)) AND d_year#8 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#7]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#5]
+Right keys [1]: [d_date_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4]
+Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#7]
+
+(11) Scan parquet default.store
+Output [2]: [s_store_sk#11, s_county#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [In(s_county, [Williamson County,Franklin Parish,Bronx County,Orange County]), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_county:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [s_store_sk#11, s_county#12]
+
+(13) Filter [codegen id : 2]
+Input [2]: [s_store_sk#11, s_county#12]
+Condition : (s_county#12 IN (Williamson County,Franklin Parish,Bronx County,Orange County) AND isnotnull(s_store_sk#11))
+
+(14) Project [codegen id : 2]
+Output [1]: [s_store_sk#11]
+Input [2]: [s_store_sk#11, s_county#12]
+
+(15) BroadcastExchange
+Input [1]: [s_store_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#11]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#11]
+
+(18) Scan parquet default.household_demographics
+Output [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_vehicle_count), IsNotNull(hd_dep_count), Or(EqualTo(hd_buy_potential,>10000         ),EqualTo(hd_buy_potential,unknown        )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_buy_potential:string,hd_dep_count:int,hd_vehicle_count:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+
+(20) Filter [codegen id : 3]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+Condition : (((((isnotnull(hd_vehicle_count#17) AND isnotnull(hd_dep_count#16)) AND ((hd_buy_potential#15 = >10000         ) OR (hd_buy_potential#15 = unknown        ))) AND (hd_vehicle_count#17 > 0)) AND ((cast(hd_dep_count#16 as double) / cast(hd_vehicle_count#17 as double)) > 1.0)) AND isnotnull(hd_demo_sk#14))
+
+(21) Project [codegen id : 3]
+Output [1]: [hd_demo_sk#14]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+
+(22) BroadcastExchange
+Input [1]: [hd_demo_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(23) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#14]
+Join condition: None
+
+(24) Project [codegen id : 4]
+Output [2]: [ss_customer_sk#1, ss_ticket_number#4]
+Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#14]
+
+(25) HashAggregate [codegen id : 4]
+Input [2]: [ss_customer_sk#1, ss_ticket_number#4]
+Keys [2]: [ss_ticket_number#4, ss_customer_sk#1]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#19]
+Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+
+(26) Exchange
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(27) HashAggregate [codegen id : 6]
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+Keys [2]: [ss_ticket_number#4, ss_customer_sk#1]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#22]
+Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count(1)#22 AS cnt#23]
+
+(28) Filter [codegen id : 6]
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23]
+Condition : ((cnt#23 >= 1) AND (cnt#23 <= 5))
+
+(29) Scan parquet default.customer
+Output [5]: [c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_salutation:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string>
+
+(30) ColumnarToRow [codegen id : 5]
+Input [5]: [c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+
+(31) Filter [codegen id : 5]
+Input [5]: [c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+Condition : isnotnull(c_customer_sk#24)
+
+(32) BroadcastExchange
+Input [5]: [c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#29]
+
+(33) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#24]
+Join condition: None
+
+(34) Project [codegen id : 6]
+Output [6]: [c_last_name#27, c_first_name#26, c_salutation#25, c_preferred_cust_flag#28, ss_ticket_number#4, cnt#23]
+Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23, c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+
+(35) Exchange
+Input [6]: [c_last_name#27, c_first_name#26, c_salutation#25, c_preferred_cust_flag#28, ss_ticket_number#4, cnt#23]
+Arguments: rangepartitioning(cnt#23 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(36) Sort [codegen id : 7]
+Input [6]: [c_last_name#27, c_first_name#26, c_salutation#25, c_preferred_cust_flag#28, ss_ticket_number#4, cnt#23]
+Arguments: [cnt#23 DESC NULLS LAST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (37)
+
+
+(37) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#7]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/simplified.txt
new file mode 100644
index 0000000000000..5bb7daa2bda9c
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/simplified.txt
@@ -0,0 +1,56 @@
+WholeStageCodegen (7)
+  Sort [cnt]
+    InputAdapter
+      Exchange [cnt] #1
+        WholeStageCodegen (6)
+          Project [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt]
+            BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+              Filter [cnt]
+                HashAggregate [ss_ticket_number,ss_customer_sk,count] [count(1),cnt,count]
+                  InputAdapter
+                    Exchange [ss_ticket_number,ss_customer_sk] #2
+                      WholeStageCodegen (4)
+                        HashAggregate [ss_ticket_number,ss_customer_sk] [count,count]
+                          Project [ss_customer_sk,ss_ticket_number]
+                            BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                              Project [ss_customer_sk,ss_hdemo_sk,ss_ticket_number]
+                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                  Project [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number]
+                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                      Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #1
+                                                ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        BroadcastExchange #3
+                                          WholeStageCodegen (1)
+                                            Project [d_date_sk]
+                                              Filter [d_dom,d_year,d_date_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_date_sk,d_year,d_dom]
+                                  InputAdapter
+                                    BroadcastExchange #4
+                                      WholeStageCodegen (2)
+                                        Project [s_store_sk]
+                                          Filter [s_county,s_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store [s_store_sk,s_county]
+                              InputAdapter
+                                BroadcastExchange #5
+                                  WholeStageCodegen (3)
+                                    Project [hd_demo_sk]
+                                      Filter [hd_vehicle_count,hd_dep_count,hd_buy_potential,hd_demo_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count]
+              InputAdapter
+                BroadcastExchange #6
+                  WholeStageCodegen (5)
+                    Filter [c_customer_sk]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.sf100/explain.txt
new file mode 100644
index 0000000000000..9b2ead7ea96f7
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.sf100/explain.txt
@@ -0,0 +1,502 @@
+== Physical Plan ==
+TakeOrderedAndProject (86)
++- * Project (85)
+   +- * SortMergeJoin Inner (84)
+      :- * Project (66)
+      :  +- * SortMergeJoin Inner (65)
+      :     :- * SortMergeJoin Inner (45)
+      :     :  :- * Sort (24)
+      :     :  :  +- Exchange (23)
+      :     :  :     +- * Filter (22)
+      :     :  :        +- * HashAggregate (21)
+      :     :  :           +- Exchange (20)
+      :     :  :              +- * HashAggregate (19)
+      :     :  :                 +- * Project (18)
+      :     :  :                    +- * SortMergeJoin Inner (17)
+      :     :  :                       :- * Sort (11)
+      :     :  :                       :  +- Exchange (10)
+      :     :  :                       :     +- * Project (9)
+      :     :  :                       :        +- * BroadcastHashJoin Inner BuildRight (8)
+      :     :  :                       :           :- * Filter (3)
+      :     :  :                       :           :  +- * ColumnarToRow (2)
+      :     :  :                       :           :     +- Scan parquet default.store_sales (1)
+      :     :  :                       :           +- BroadcastExchange (7)
+      :     :  :                       :              +- * Filter (6)
+      :     :  :                       :                 +- * ColumnarToRow (5)
+      :     :  :                       :                    +- Scan parquet default.date_dim (4)
+      :     :  :                       +- * Sort (16)
+      :     :  :                          +- Exchange (15)
+      :     :  :                             +- * Filter (14)
+      :     :  :                                +- * ColumnarToRow (13)
+      :     :  :                                   +- Scan parquet default.customer (12)
+      :     :  +- * Sort (44)
+      :     :     +- Exchange (43)
+      :     :        +- * HashAggregate (42)
+      :     :           +- Exchange (41)
+      :     :              +- * HashAggregate (40)
+      :     :                 +- * Project (39)
+      :     :                    +- * SortMergeJoin Inner (38)
+      :     :                       :- * Sort (35)
+      :     :                       :  +- Exchange (34)
+      :     :                       :     +- * Project (33)
+      :     :                       :        +- * BroadcastHashJoin Inner BuildRight (32)
+      :     :                       :           :- * Filter (27)
+      :     :                       :           :  +- * ColumnarToRow (26)
+      :     :                       :           :     +- Scan parquet default.store_sales (25)
+      :     :                       :           +- BroadcastExchange (31)
+      :     :                       :              +- * Filter (30)
+      :     :                       :                 +- * ColumnarToRow (29)
+      :     :                       :                    +- Scan parquet default.date_dim (28)
+      :     :                       +- * Sort (37)
+      :     :                          +- ReusedExchange (36)
+      :     +- * Sort (64)
+      :        +- Exchange (63)
+      :           +- * Project (62)
+      :              +- * Filter (61)
+      :                 +- * HashAggregate (60)
+      :                    +- Exchange (59)
+      :                       +- * HashAggregate (58)
+      :                          +- * Project (57)
+      :                             +- * SortMergeJoin Inner (56)
+      :                                :- * Sort (53)
+      :                                :  +- Exchange (52)
+      :                                :     +- * Project (51)
+      :                                :        +- * BroadcastHashJoin Inner BuildRight (50)
+      :                                :           :- * Filter (48)
+      :                                :           :  +- * ColumnarToRow (47)
+      :                                :           :     +- Scan parquet default.web_sales (46)
+      :                                :           +- ReusedExchange (49)
+      :                                +- * Sort (55)
+      :                                   +- ReusedExchange (54)
+      +- * Sort (83)
+         +- Exchange (82)
+            +- * HashAggregate (81)
+               +- Exchange (80)
+                  +- * HashAggregate (79)
+                     +- * Project (78)
+                        +- * SortMergeJoin Inner (77)
+                           :- * Sort (74)
+                           :  +- Exchange (73)
+                           :     +- * Project (72)
+                           :        +- * BroadcastHashJoin Inner BuildRight (71)
+                           :           :- * Filter (69)
+                           :           :  +- * ColumnarToRow (68)
+                           :           :     +- Scan parquet default.web_sales (67)
+                           :           +- ReusedExchange (70)
+                           +- * Sort (76)
+                              +- ReusedExchange (75)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_customer_sk#1, ss_net_paid#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_net_paid:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [3]: [ss_customer_sk#1, ss_net_paid#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 2]
+Input [3]: [ss_customer_sk#1, ss_net_paid#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_customer_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#5, d_year#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#5, d_year#6]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#5, d_year#6]
+Condition : (((isnotnull(d_year#6) AND (d_year#6 = 2001)) AND d_year#6 IN (2001,2002)) AND isnotnull(d_date_sk#5))
+
+(7) BroadcastExchange
+Input [2]: [d_date_sk#5, d_year#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#7]
+
+(8) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#5]
+Join condition: None
+
+(9) Project [codegen id : 2]
+Output [3]: [ss_customer_sk#1, ss_net_paid#2, d_year#6]
+Input [5]: [ss_customer_sk#1, ss_net_paid#2, ss_sold_date_sk#3, d_date_sk#5, d_year#6]
+
+(10) Exchange
+Input [3]: [ss_customer_sk#1, ss_net_paid#2, d_year#6]
+Arguments: hashpartitioning(ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#8]
+
+(11) Sort [codegen id : 3]
+Input [3]: [ss_customer_sk#1, ss_net_paid#2, d_year#6]
+Arguments: [ss_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(12) Scan parquet default.customer
+Output [4]: [c_customer_sk#9, c_customer_id#10, c_first_name#11, c_last_name#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string>
+
+(13) ColumnarToRow [codegen id : 4]
+Input [4]: [c_customer_sk#9, c_customer_id#10, c_first_name#11, c_last_name#12]
+
+(14) Filter [codegen id : 4]
+Input [4]: [c_customer_sk#9, c_customer_id#10, c_first_name#11, c_last_name#12]
+Condition : (isnotnull(c_customer_sk#9) AND isnotnull(c_customer_id#10))
+
+(15) Exchange
+Input [4]: [c_customer_sk#9, c_customer_id#10, c_first_name#11, c_last_name#12]
+Arguments: hashpartitioning(c_customer_sk#9, 5), ENSURE_REQUIREMENTS, [id=#13]
+
+(16) Sort [codegen id : 5]
+Input [4]: [c_customer_sk#9, c_customer_id#10, c_first_name#11, c_last_name#12]
+Arguments: [c_customer_sk#9 ASC NULLS FIRST], false, 0
+
+(17) SortMergeJoin [codegen id : 6]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#9]
+Join condition: None
+
+(18) Project [codegen id : 6]
+Output [5]: [c_customer_id#10, c_first_name#11, c_last_name#12, ss_net_paid#2, d_year#6]
+Input [7]: [ss_customer_sk#1, ss_net_paid#2, d_year#6, c_customer_sk#9, c_customer_id#10, c_first_name#11, c_last_name#12]
+
+(19) HashAggregate [codegen id : 6]
+Input [5]: [c_customer_id#10, c_first_name#11, c_last_name#12, ss_net_paid#2, d_year#6]
+Keys [4]: [c_customer_id#10, c_first_name#11, c_last_name#12, d_year#6]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#2))]
+Aggregate Attributes [1]: [sum#14]
+Results [5]: [c_customer_id#10, c_first_name#11, c_last_name#12, d_year#6, sum#15]
+
+(20) Exchange
+Input [5]: [c_customer_id#10, c_first_name#11, c_last_name#12, d_year#6, sum#15]
+Arguments: hashpartitioning(c_customer_id#10, c_first_name#11, c_last_name#12, d_year#6, 5), ENSURE_REQUIREMENTS, [id=#16]
+
+(21) HashAggregate [codegen id : 7]
+Input [5]: [c_customer_id#10, c_first_name#11, c_last_name#12, d_year#6, sum#15]
+Keys [4]: [c_customer_id#10, c_first_name#11, c_last_name#12, d_year#6]
+Functions [1]: [sum(UnscaledValue(ss_net_paid#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#2))#17]
+Results [2]: [c_customer_id#10 AS customer_id#18, MakeDecimal(sum(UnscaledValue(ss_net_paid#2))#17,17,2) AS year_total#19]
+
+(22) Filter [codegen id : 7]
+Input [2]: [customer_id#18, year_total#19]
+Condition : (isnotnull(year_total#19) AND (year_total#19 > 0.00))
+
+(23) Exchange
+Input [2]: [customer_id#18, year_total#19]
+Arguments: hashpartitioning(customer_id#18, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(24) Sort [codegen id : 8]
+Input [2]: [customer_id#18, year_total#19]
+Arguments: [customer_id#18 ASC NULLS FIRST], false, 0
+
+(25) Scan parquet default.store_sales
+Output [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#23), dynamicpruningexpression(ss_sold_date_sk#23 IN dynamicpruning#24)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_net_paid:decimal(7,2)>
+
+(26) ColumnarToRow [codegen id : 10]
+Input [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23]
+
+(27) Filter [codegen id : 10]
+Input [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23]
+Condition : isnotnull(ss_customer_sk#21)
+
+(28) Scan parquet default.date_dim
+Output [2]: [d_date_sk#25, d_year#26]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(29) ColumnarToRow [codegen id : 9]
+Input [2]: [d_date_sk#25, d_year#26]
+
+(30) Filter [codegen id : 9]
+Input [2]: [d_date_sk#25, d_year#26]
+Condition : (((isnotnull(d_year#26) AND (d_year#26 = 2002)) AND d_year#26 IN (2001,2002)) AND isnotnull(d_date_sk#25))
+
+(31) BroadcastExchange
+Input [2]: [d_date_sk#25, d_year#26]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#27]
+
+(32) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_sold_date_sk#23]
+Right keys [1]: [d_date_sk#25]
+Join condition: None
+
+(33) Project [codegen id : 10]
+Output [3]: [ss_customer_sk#21, ss_net_paid#22, d_year#26]
+Input [5]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23, d_date_sk#25, d_year#26]
+
+(34) Exchange
+Input [3]: [ss_customer_sk#21, ss_net_paid#22, d_year#26]
+Arguments: hashpartitioning(ss_customer_sk#21, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(35) Sort [codegen id : 11]
+Input [3]: [ss_customer_sk#21, ss_net_paid#22, d_year#26]
+Arguments: [ss_customer_sk#21 ASC NULLS FIRST], false, 0
+
+(36) ReusedExchange [Reuses operator id: 15]
+Output [4]: [c_customer_sk#29, c_customer_id#30, c_first_name#31, c_last_name#32]
+
+(37) Sort [codegen id : 13]
+Input [4]: [c_customer_sk#29, c_customer_id#30, c_first_name#31, c_last_name#32]
+Arguments: [c_customer_sk#29 ASC NULLS FIRST], false, 0
+
+(38) SortMergeJoin [codegen id : 14]
+Left keys [1]: [ss_customer_sk#21]
+Right keys [1]: [c_customer_sk#29]
+Join condition: None
+
+(39) Project [codegen id : 14]
+Output [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, ss_net_paid#22, d_year#26]
+Input [7]: [ss_customer_sk#21, ss_net_paid#22, d_year#26, c_customer_sk#29, c_customer_id#30, c_first_name#31, c_last_name#32]
+
+(40) HashAggregate [codegen id : 14]
+Input [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, ss_net_paid#22, d_year#26]
+Keys [4]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#26]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#22))]
+Aggregate Attributes [1]: [sum#33]
+Results [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#26, sum#34]
+
+(41) Exchange
+Input [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#26, sum#34]
+Arguments: hashpartitioning(c_customer_id#30, c_first_name#31, c_last_name#32, d_year#26, 5), ENSURE_REQUIREMENTS, [id=#35]
+
+(42) HashAggregate [codegen id : 15]
+Input [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#26, sum#34]
+Keys [4]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#26]
+Functions [1]: [sum(UnscaledValue(ss_net_paid#22))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#22))#36]
+Results [4]: [c_customer_id#30 AS customer_id#37, c_first_name#31 AS customer_first_name#38, c_last_name#32 AS customer_last_name#39, MakeDecimal(sum(UnscaledValue(ss_net_paid#22))#36,17,2) AS year_total#40]
+
+(43) Exchange
+Input [4]: [customer_id#37, customer_first_name#38, customer_last_name#39, year_total#40]
+Arguments: hashpartitioning(customer_id#37, 5), ENSURE_REQUIREMENTS, [id=#41]
+
+(44) Sort [codegen id : 16]
+Input [4]: [customer_id#37, customer_first_name#38, customer_last_name#39, year_total#40]
+Arguments: [customer_id#37 ASC NULLS FIRST], false, 0
+
+(45) SortMergeJoin [codegen id : 17]
+Left keys [1]: [customer_id#18]
+Right keys [1]: [customer_id#37]
+Join condition: None
+
+(46) Scan parquet default.web_sales
+Output [3]: [ws_bill_customer_sk#42, ws_net_paid#43, ws_sold_date_sk#44]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#44), dynamicpruningexpression(ws_sold_date_sk#44 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int,ws_net_paid:decimal(7,2)>
+
+(47) ColumnarToRow [codegen id : 19]
+Input [3]: [ws_bill_customer_sk#42, ws_net_paid#43, ws_sold_date_sk#44]
+
+(48) Filter [codegen id : 19]
+Input [3]: [ws_bill_customer_sk#42, ws_net_paid#43, ws_sold_date_sk#44]
+Condition : isnotnull(ws_bill_customer_sk#42)
+
+(49) ReusedExchange [Reuses operator id: 7]
+Output [2]: [d_date_sk#45, d_year#46]
+
+(50) BroadcastHashJoin [codegen id : 19]
+Left keys [1]: [ws_sold_date_sk#44]
+Right keys [1]: [d_date_sk#45]
+Join condition: None
+
+(51) Project [codegen id : 19]
+Output [3]: [ws_bill_customer_sk#42, ws_net_paid#43, d_year#46]
+Input [5]: [ws_bill_customer_sk#42, ws_net_paid#43, ws_sold_date_sk#44, d_date_sk#45, d_year#46]
+
+(52) Exchange
+Input [3]: [ws_bill_customer_sk#42, ws_net_paid#43, d_year#46]
+Arguments: hashpartitioning(ws_bill_customer_sk#42, 5), ENSURE_REQUIREMENTS, [id=#47]
+
+(53) Sort [codegen id : 20]
+Input [3]: [ws_bill_customer_sk#42, ws_net_paid#43, d_year#46]
+Arguments: [ws_bill_customer_sk#42 ASC NULLS FIRST], false, 0
+
+(54) ReusedExchange [Reuses operator id: 15]
+Output [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51]
+
+(55) Sort [codegen id : 22]
+Input [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51]
+Arguments: [c_customer_sk#48 ASC NULLS FIRST], false, 0
+
+(56) SortMergeJoin [codegen id : 23]
+Left keys [1]: [ws_bill_customer_sk#42]
+Right keys [1]: [c_customer_sk#48]
+Join condition: None
+
+(57) Project [codegen id : 23]
+Output [5]: [c_customer_id#49, c_first_name#50, c_last_name#51, ws_net_paid#43, d_year#46]
+Input [7]: [ws_bill_customer_sk#42, ws_net_paid#43, d_year#46, c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51]
+
+(58) HashAggregate [codegen id : 23]
+Input [5]: [c_customer_id#49, c_first_name#50, c_last_name#51, ws_net_paid#43, d_year#46]
+Keys [4]: [c_customer_id#49, c_first_name#50, c_last_name#51, d_year#46]
+Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#43))]
+Aggregate Attributes [1]: [sum#52]
+Results [5]: [c_customer_id#49, c_first_name#50, c_last_name#51, d_year#46, sum#53]
+
+(59) Exchange
+Input [5]: [c_customer_id#49, c_first_name#50, c_last_name#51, d_year#46, sum#53]
+Arguments: hashpartitioning(c_customer_id#49, c_first_name#50, c_last_name#51, d_year#46, 5), ENSURE_REQUIREMENTS, [id=#54]
+
+(60) HashAggregate [codegen id : 24]
+Input [5]: [c_customer_id#49, c_first_name#50, c_last_name#51, d_year#46, sum#53]
+Keys [4]: [c_customer_id#49, c_first_name#50, c_last_name#51, d_year#46]
+Functions [1]: [sum(UnscaledValue(ws_net_paid#43))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#43))#55]
+Results [2]: [c_customer_id#49 AS customer_id#56, MakeDecimal(sum(UnscaledValue(ws_net_paid#43))#55,17,2) AS year_total#57]
+
+(61) Filter [codegen id : 24]
+Input [2]: [customer_id#56, year_total#57]
+Condition : (isnotnull(year_total#57) AND (year_total#57 > 0.00))
+
+(62) Project [codegen id : 24]
+Output [2]: [customer_id#56 AS customer_id#58, year_total#57 AS year_total#59]
+Input [2]: [customer_id#56, year_total#57]
+
+(63) Exchange
+Input [2]: [customer_id#58, year_total#59]
+Arguments: hashpartitioning(customer_id#58, 5), ENSURE_REQUIREMENTS, [id=#60]
+
+(64) Sort [codegen id : 25]
+Input [2]: [customer_id#58, year_total#59]
+Arguments: [customer_id#58 ASC NULLS FIRST], false, 0
+
+(65) SortMergeJoin [codegen id : 26]
+Left keys [1]: [customer_id#18]
+Right keys [1]: [customer_id#58]
+Join condition: None
+
+(66) Project [codegen id : 26]
+Output [7]: [customer_id#18, year_total#19, customer_id#37, customer_first_name#38, customer_last_name#39, year_total#40, year_total#59]
+Input [8]: [customer_id#18, year_total#19, customer_id#37, customer_first_name#38, customer_last_name#39, year_total#40, customer_id#58, year_total#59]
+
+(67) Scan parquet default.web_sales
+Output [3]: [ws_bill_customer_sk#61, ws_net_paid#62, ws_sold_date_sk#63]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#63), dynamicpruningexpression(ws_sold_date_sk#63 IN dynamicpruning#24)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int,ws_net_paid:decimal(7,2)>
+
+(68) ColumnarToRow [codegen id : 28]
+Input [3]: [ws_bill_customer_sk#61, ws_net_paid#62, ws_sold_date_sk#63]
+
+(69) Filter [codegen id : 28]
+Input [3]: [ws_bill_customer_sk#61, ws_net_paid#62, ws_sold_date_sk#63]
+Condition : isnotnull(ws_bill_customer_sk#61)
+
+(70) ReusedExchange [Reuses operator id: 31]
+Output [2]: [d_date_sk#64, d_year#65]
+
+(71) BroadcastHashJoin [codegen id : 28]
+Left keys [1]: [ws_sold_date_sk#63]
+Right keys [1]: [d_date_sk#64]
+Join condition: None
+
+(72) Project [codegen id : 28]
+Output [3]: [ws_bill_customer_sk#61, ws_net_paid#62, d_year#65]
+Input [5]: [ws_bill_customer_sk#61, ws_net_paid#62, ws_sold_date_sk#63, d_date_sk#64, d_year#65]
+
+(73) Exchange
+Input [3]: [ws_bill_customer_sk#61, ws_net_paid#62, d_year#65]
+Arguments: hashpartitioning(ws_bill_customer_sk#61, 5), ENSURE_REQUIREMENTS, [id=#66]
+
+(74) Sort [codegen id : 29]
+Input [3]: [ws_bill_customer_sk#61, ws_net_paid#62, d_year#65]
+Arguments: [ws_bill_customer_sk#61 ASC NULLS FIRST], false, 0
+
+(75) ReusedExchange [Reuses operator id: 15]
+Output [4]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70]
+
+(76) Sort [codegen id : 31]
+Input [4]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70]
+Arguments: [c_customer_sk#67 ASC NULLS FIRST], false, 0
+
+(77) SortMergeJoin [codegen id : 32]
+Left keys [1]: [ws_bill_customer_sk#61]
+Right keys [1]: [c_customer_sk#67]
+Join condition: None
+
+(78) Project [codegen id : 32]
+Output [5]: [c_customer_id#68, c_first_name#69, c_last_name#70, ws_net_paid#62, d_year#65]
+Input [7]: [ws_bill_customer_sk#61, ws_net_paid#62, d_year#65, c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70]
+
+(79) HashAggregate [codegen id : 32]
+Input [5]: [c_customer_id#68, c_first_name#69, c_last_name#70, ws_net_paid#62, d_year#65]
+Keys [4]: [c_customer_id#68, c_first_name#69, c_last_name#70, d_year#65]
+Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#62))]
+Aggregate Attributes [1]: [sum#71]
+Results [5]: [c_customer_id#68, c_first_name#69, c_last_name#70, d_year#65, sum#72]
+
+(80) Exchange
+Input [5]: [c_customer_id#68, c_first_name#69, c_last_name#70, d_year#65, sum#72]
+Arguments: hashpartitioning(c_customer_id#68, c_first_name#69, c_last_name#70, d_year#65, 5), ENSURE_REQUIREMENTS, [id=#73]
+
+(81) HashAggregate [codegen id : 33]
+Input [5]: [c_customer_id#68, c_first_name#69, c_last_name#70, d_year#65, sum#72]
+Keys [4]: [c_customer_id#68, c_first_name#69, c_last_name#70, d_year#65]
+Functions [1]: [sum(UnscaledValue(ws_net_paid#62))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#62))#74]
+Results [2]: [c_customer_id#68 AS customer_id#75, MakeDecimal(sum(UnscaledValue(ws_net_paid#62))#74,17,2) AS year_total#76]
+
+(82) Exchange
+Input [2]: [customer_id#75, year_total#76]
+Arguments: hashpartitioning(customer_id#75, 5), ENSURE_REQUIREMENTS, [id=#77]
+
+(83) Sort [codegen id : 34]
+Input [2]: [customer_id#75, year_total#76]
+Arguments: [customer_id#75 ASC NULLS FIRST], false, 0
+
+(84) SortMergeJoin [codegen id : 35]
+Left keys [1]: [customer_id#18]
+Right keys [1]: [customer_id#75]
+Join condition: (CASE WHEN (year_total#59 > 0.00) THEN CheckOverflow((promote_precision(year_total#76) / promote_precision(year_total#59)), DecimalType(37,20), true) ELSE null END > CASE WHEN (year_total#19 > 0.00) THEN CheckOverflow((promote_precision(year_total#40) / promote_precision(year_total#19)), DecimalType(37,20), true) ELSE null END)
+
+(85) Project [codegen id : 35]
+Output [3]: [customer_id#37, customer_first_name#38, customer_last_name#39]
+Input [9]: [customer_id#18, year_total#19, customer_id#37, customer_first_name#38, customer_last_name#39, year_total#40, year_total#59, customer_id#75, year_total#76]
+
+(86) TakeOrderedAndProject
+Input [3]: [customer_id#37, customer_first_name#38, customer_last_name#39]
+Arguments: 100, [customer_id#37 ASC NULLS FIRST, customer_id#37 ASC NULLS FIRST, customer_id#37 ASC NULLS FIRST], [customer_id#37, customer_first_name#38, customer_last_name#39]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (87)
+
+
+(87) ReusedExchange [Reuses operator id: 7]
+Output [2]: [d_date_sk#5, d_year#6]
+
+Subquery:2 Hosting operator id = 25 Hosting Expression = ss_sold_date_sk#23 IN dynamicpruning#24
+ReusedExchange (88)
+
+
+(88) ReusedExchange [Reuses operator id: 31]
+Output [2]: [d_date_sk#25, d_year#26]
+
+Subquery:3 Hosting operator id = 46 Hosting Expression = ws_sold_date_sk#44 IN dynamicpruning#4
+
+Subquery:4 Hosting operator id = 67 Hosting Expression = ws_sold_date_sk#63 IN dynamicpruning#24
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.sf100/simplified.txt
new file mode 100644
index 0000000000000..4a047351e7cdd
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.sf100/simplified.txt
@@ -0,0 +1,163 @@
+TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name]
+  WholeStageCodegen (35)
+    Project [customer_id,customer_first_name,customer_last_name]
+      SortMergeJoin [customer_id,customer_id,year_total,year_total,year_total,year_total]
+        InputAdapter
+          WholeStageCodegen (26)
+            Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total]
+              SortMergeJoin [customer_id,customer_id]
+                InputAdapter
+                  WholeStageCodegen (17)
+                    SortMergeJoin [customer_id,customer_id]
+                      InputAdapter
+                        WholeStageCodegen (8)
+                          Sort [customer_id]
+                            InputAdapter
+                              Exchange [customer_id] #1
+                                WholeStageCodegen (7)
+                                  Filter [year_total]
+                                    HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,year_total,sum]
+                                      InputAdapter
+                                        Exchange [c_customer_id,c_first_name,c_last_name,d_year] #2
+                                          WholeStageCodegen (6)
+                                            HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum]
+                                              Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year]
+                                                SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (3)
+                                                      Sort [ss_customer_sk]
+                                                        InputAdapter
+                                                          Exchange [ss_customer_sk] #3
+                                                            WholeStageCodegen (2)
+                                                              Project [ss_customer_sk,ss_net_paid,d_year]
+                                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                  Filter [ss_customer_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk]
+                                                                          SubqueryBroadcast [d_date_sk] #1
+                                                                            ReusedExchange [d_date_sk,d_year] #4
+                                                                  InputAdapter
+                                                                    BroadcastExchange #4
+                                                                      WholeStageCodegen (1)
+                                                                        Filter [d_year,d_date_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.date_dim [d_date_sk,d_year]
+                                                  InputAdapter
+                                                    WholeStageCodegen (5)
+                                                      Sort [c_customer_sk]
+                                                        InputAdapter
+                                                          Exchange [c_customer_sk] #5
+                                                            WholeStageCodegen (4)
+                                                              Filter [c_customer_sk,c_customer_id]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name]
+                      InputAdapter
+                        WholeStageCodegen (16)
+                          Sort [customer_id]
+                            InputAdapter
+                              Exchange [customer_id] #6
+                                WholeStageCodegen (15)
+                                  HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,customer_first_name,customer_last_name,year_total,sum]
+                                    InputAdapter
+                                      Exchange [c_customer_id,c_first_name,c_last_name,d_year] #7
+                                        WholeStageCodegen (14)
+                                          HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum]
+                                            Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year]
+                                              SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                                InputAdapter
+                                                  WholeStageCodegen (11)
+                                                    Sort [ss_customer_sk]
+                                                      InputAdapter
+                                                        Exchange [ss_customer_sk] #8
+                                                          WholeStageCodegen (10)
+                                                            Project [ss_customer_sk,ss_net_paid,d_year]
+                                                              BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                Filter [ss_customer_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk]
+                                                                        SubqueryBroadcast [d_date_sk] #2
+                                                                          ReusedExchange [d_date_sk,d_year] #9
+                                                                InputAdapter
+                                                                  BroadcastExchange #9
+                                                                    WholeStageCodegen (9)
+                                                                      Filter [d_year,d_date_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.date_dim [d_date_sk,d_year]
+                                                InputAdapter
+                                                  WholeStageCodegen (13)
+                                                    Sort [c_customer_sk]
+                                                      InputAdapter
+                                                        ReusedExchange [c_customer_sk,c_customer_id,c_first_name,c_last_name] #5
+                InputAdapter
+                  WholeStageCodegen (25)
+                    Sort [customer_id]
+                      InputAdapter
+                        Exchange [customer_id] #10
+                          WholeStageCodegen (24)
+                            Project [customer_id,year_total]
+                              Filter [year_total]
+                                HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum]
+                                  InputAdapter
+                                    Exchange [c_customer_id,c_first_name,c_last_name,d_year] #11
+                                      WholeStageCodegen (23)
+                                        HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum]
+                                          Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year]
+                                            SortMergeJoin [ws_bill_customer_sk,c_customer_sk]
+                                              InputAdapter
+                                                WholeStageCodegen (20)
+                                                  Sort [ws_bill_customer_sk]
+                                                    InputAdapter
+                                                      Exchange [ws_bill_customer_sk] #12
+                                                        WholeStageCodegen (19)
+                                                          Project [ws_bill_customer_sk,ws_net_paid,d_year]
+                                                            BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                              Filter [ws_bill_customer_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk]
+                                                                      ReusedSubquery [d_date_sk] #1
+                                                              InputAdapter
+                                                                ReusedExchange [d_date_sk,d_year] #4
+                                              InputAdapter
+                                                WholeStageCodegen (22)
+                                                  Sort [c_customer_sk]
+                                                    InputAdapter
+                                                      ReusedExchange [c_customer_sk,c_customer_id,c_first_name,c_last_name] #5
+        InputAdapter
+          WholeStageCodegen (34)
+            Sort [customer_id]
+              InputAdapter
+                Exchange [customer_id] #13
+                  WholeStageCodegen (33)
+                    HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum]
+                      InputAdapter
+                        Exchange [c_customer_id,c_first_name,c_last_name,d_year] #14
+                          WholeStageCodegen (32)
+                            HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum]
+                              Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year]
+                                SortMergeJoin [ws_bill_customer_sk,c_customer_sk]
+                                  InputAdapter
+                                    WholeStageCodegen (29)
+                                      Sort [ws_bill_customer_sk]
+                                        InputAdapter
+                                          Exchange [ws_bill_customer_sk] #15
+                                            WholeStageCodegen (28)
+                                              Project [ws_bill_customer_sk,ws_net_paid,d_year]
+                                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                  Filter [ws_bill_customer_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk]
+                                                          ReusedSubquery [d_date_sk] #2
+                                                  InputAdapter
+                                                    ReusedExchange [d_date_sk,d_year] #9
+                                  InputAdapter
+                                    WholeStageCodegen (31)
+                                      Sort [c_customer_sk]
+                                        InputAdapter
+                                          ReusedExchange [c_customer_sk,c_customer_id,c_first_name,c_last_name] #5
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/explain.txt
new file mode 100644
index 0000000000000..9fccc4c4ba66d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/explain.txt
@@ -0,0 +1,471 @@
+== Physical Plan ==
+TakeOrderedAndProject (78)
++- * Project (77)
+   +- * BroadcastHashJoin Inner BuildRight (76)
+      :- * Project (59)
+      :  +- * BroadcastHashJoin Inner BuildRight (58)
+      :     :- * BroadcastHashJoin Inner BuildRight (39)
+      :     :  :- * Filter (19)
+      :     :  :  +- * HashAggregate (18)
+      :     :  :     +- Exchange (17)
+      :     :  :        +- * HashAggregate (16)
+      :     :  :           +- * Project (15)
+      :     :  :              +- * BroadcastHashJoin Inner BuildRight (14)
+      :     :  :                 :- * Project (9)
+      :     :  :                 :  +- * BroadcastHashJoin Inner BuildRight (8)
+      :     :  :                 :     :- * Filter (3)
+      :     :  :                 :     :  +- * ColumnarToRow (2)
+      :     :  :                 :     :     +- Scan parquet default.customer (1)
+      :     :  :                 :     +- BroadcastExchange (7)
+      :     :  :                 :        +- * Filter (6)
+      :     :  :                 :           +- * ColumnarToRow (5)
+      :     :  :                 :              +- Scan parquet default.store_sales (4)
+      :     :  :                 +- BroadcastExchange (13)
+      :     :  :                    +- * Filter (12)
+      :     :  :                       +- * ColumnarToRow (11)
+      :     :  :                          +- Scan parquet default.date_dim (10)
+      :     :  +- BroadcastExchange (38)
+      :     :     +- * HashAggregate (37)
+      :     :        +- Exchange (36)
+      :     :           +- * HashAggregate (35)
+      :     :              +- * Project (34)
+      :     :                 +- * BroadcastHashJoin Inner BuildRight (33)
+      :     :                    :- * Project (28)
+      :     :                    :  +- * BroadcastHashJoin Inner BuildRight (27)
+      :     :                    :     :- * Filter (22)
+      :     :                    :     :  +- * ColumnarToRow (21)
+      :     :                    :     :     +- Scan parquet default.customer (20)
+      :     :                    :     +- BroadcastExchange (26)
+      :     :                    :        +- * Filter (25)
+      :     :                    :           +- * ColumnarToRow (24)
+      :     :                    :              +- Scan parquet default.store_sales (23)
+      :     :                    +- BroadcastExchange (32)
+      :     :                       +- * Filter (31)
+      :     :                          +- * ColumnarToRow (30)
+      :     :                             +- Scan parquet default.date_dim (29)
+      :     +- BroadcastExchange (57)
+      :        +- * Project (56)
+      :           +- * Filter (55)
+      :              +- * HashAggregate (54)
+      :                 +- Exchange (53)
+      :                    +- * HashAggregate (52)
+      :                       +- * Project (51)
+      :                          +- * BroadcastHashJoin Inner BuildRight (50)
+      :                             :- * Project (48)
+      :                             :  +- * BroadcastHashJoin Inner BuildRight (47)
+      :                             :     :- * Filter (42)
+      :                             :     :  +- * ColumnarToRow (41)
+      :                             :     :     +- Scan parquet default.customer (40)
+      :                             :     +- BroadcastExchange (46)
+      :                             :        +- * Filter (45)
+      :                             :           +- * ColumnarToRow (44)
+      :                             :              +- Scan parquet default.web_sales (43)
+      :                             +- ReusedExchange (49)
+      +- BroadcastExchange (75)
+         +- * HashAggregate (74)
+            +- Exchange (73)
+               +- * HashAggregate (72)
+                  +- * Project (71)
+                     +- * BroadcastHashJoin Inner BuildRight (70)
+                        :- * Project (68)
+                        :  +- * BroadcastHashJoin Inner BuildRight (67)
+                        :     :- * Filter (62)
+                        :     :  +- * ColumnarToRow (61)
+                        :     :     +- Scan parquet default.customer (60)
+                        :     +- BroadcastExchange (66)
+                        :        +- * Filter (65)
+                        :           +- * ColumnarToRow (64)
+                        :              +- Scan parquet default.web_sales (63)
+                        +- ReusedExchange (69)
+
+
+(1) Scan parquet default.customer
+Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4]
+
+(3) Filter [codegen id : 3]
+Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4]
+Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2))
+
+(4) Scan parquet default.store_sales
+Output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_net_paid:decimal(7,2)>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7]
+
+(6) Filter [codegen id : 1]
+Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7]
+Condition : isnotnull(ss_customer_sk#5)
+
+(7) BroadcastExchange
+Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#9]
+
+(8) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [ss_customer_sk#5]
+Join condition: None
+
+(9) Project [codegen id : 3]
+Output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7]
+Input [7]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7]
+
+(10) Scan parquet default.date_dim
+Output [2]: [d_date_sk#10, d_year#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#10, d_year#11]
+
+(12) Filter [codegen id : 2]
+Input [2]: [d_date_sk#10, d_year#11]
+Condition : (((isnotnull(d_year#11) AND (d_year#11 = 2001)) AND d_year#11 IN (2001,2002)) AND isnotnull(d_date_sk#10))
+
+(13) BroadcastExchange
+Input [2]: [d_date_sk#10, d_year#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#12]
+
+(14) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#7]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(15) Project [codegen id : 3]
+Output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#11]
+Input [7]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7, d_date_sk#10, d_year#11]
+
+(16) HashAggregate [codegen id : 3]
+Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#11]
+Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#11]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#6))]
+Aggregate Attributes [1]: [sum#13]
+Results [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#11, sum#14]
+
+(17) Exchange
+Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#11, sum#14]
+Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#11, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(18) HashAggregate [codegen id : 16]
+Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#11, sum#14]
+Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#11]
+Functions [1]: [sum(UnscaledValue(ss_net_paid#6))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#6))#16]
+Results [2]: [c_customer_id#2 AS customer_id#17, MakeDecimal(sum(UnscaledValue(ss_net_paid#6))#16,17,2) AS year_total#18]
+
+(19) Filter [codegen id : 16]
+Input [2]: [customer_id#17, year_total#18]
+Condition : (isnotnull(year_total#18) AND (year_total#18 > 0.00))
+
+(20) Scan parquet default.customer
+Output [4]: [c_customer_sk#19, c_customer_id#20, c_first_name#21, c_last_name#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string>
+
+(21) ColumnarToRow [codegen id : 6]
+Input [4]: [c_customer_sk#19, c_customer_id#20, c_first_name#21, c_last_name#22]
+
+(22) Filter [codegen id : 6]
+Input [4]: [c_customer_sk#19, c_customer_id#20, c_first_name#21, c_last_name#22]
+Condition : (isnotnull(c_customer_sk#19) AND isnotnull(c_customer_id#20))
+
+(23) Scan parquet default.store_sales
+Output [3]: [ss_customer_sk#23, ss_net_paid#24, ss_sold_date_sk#25]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#25), dynamicpruningexpression(ss_sold_date_sk#25 IN dynamicpruning#26)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_net_paid:decimal(7,2)>
+
+(24) ColumnarToRow [codegen id : 4]
+Input [3]: [ss_customer_sk#23, ss_net_paid#24, ss_sold_date_sk#25]
+
+(25) Filter [codegen id : 4]
+Input [3]: [ss_customer_sk#23, ss_net_paid#24, ss_sold_date_sk#25]
+Condition : isnotnull(ss_customer_sk#23)
+
+(26) BroadcastExchange
+Input [3]: [ss_customer_sk#23, ss_net_paid#24, ss_sold_date_sk#25]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#27]
+
+(27) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [c_customer_sk#19]
+Right keys [1]: [ss_customer_sk#23]
+Join condition: None
+
+(28) Project [codegen id : 6]
+Output [5]: [c_customer_id#20, c_first_name#21, c_last_name#22, ss_net_paid#24, ss_sold_date_sk#25]
+Input [7]: [c_customer_sk#19, c_customer_id#20, c_first_name#21, c_last_name#22, ss_customer_sk#23, ss_net_paid#24, ss_sold_date_sk#25]
+
+(29) Scan parquet default.date_dim
+Output [2]: [d_date_sk#28, d_year#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(30) ColumnarToRow [codegen id : 5]
+Input [2]: [d_date_sk#28, d_year#29]
+
+(31) Filter [codegen id : 5]
+Input [2]: [d_date_sk#28, d_year#29]
+Condition : (((isnotnull(d_year#29) AND (d_year#29 = 2002)) AND d_year#29 IN (2001,2002)) AND isnotnull(d_date_sk#28))
+
+(32) BroadcastExchange
+Input [2]: [d_date_sk#28, d_year#29]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#30]
+
+(33) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_sold_date_sk#25]
+Right keys [1]: [d_date_sk#28]
+Join condition: None
+
+(34) Project [codegen id : 6]
+Output [5]: [c_customer_id#20, c_first_name#21, c_last_name#22, ss_net_paid#24, d_year#29]
+Input [7]: [c_customer_id#20, c_first_name#21, c_last_name#22, ss_net_paid#24, ss_sold_date_sk#25, d_date_sk#28, d_year#29]
+
+(35) HashAggregate [codegen id : 6]
+Input [5]: [c_customer_id#20, c_first_name#21, c_last_name#22, ss_net_paid#24, d_year#29]
+Keys [4]: [c_customer_id#20, c_first_name#21, c_last_name#22, d_year#29]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#24))]
+Aggregate Attributes [1]: [sum#31]
+Results [5]: [c_customer_id#20, c_first_name#21, c_last_name#22, d_year#29, sum#32]
+
+(36) Exchange
+Input [5]: [c_customer_id#20, c_first_name#21, c_last_name#22, d_year#29, sum#32]
+Arguments: hashpartitioning(c_customer_id#20, c_first_name#21, c_last_name#22, d_year#29, 5), ENSURE_REQUIREMENTS, [id=#33]
+
+(37) HashAggregate [codegen id : 7]
+Input [5]: [c_customer_id#20, c_first_name#21, c_last_name#22, d_year#29, sum#32]
+Keys [4]: [c_customer_id#20, c_first_name#21, c_last_name#22, d_year#29]
+Functions [1]: [sum(UnscaledValue(ss_net_paid#24))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#24))#34]
+Results [4]: [c_customer_id#20 AS customer_id#35, c_first_name#21 AS customer_first_name#36, c_last_name#22 AS customer_last_name#37, MakeDecimal(sum(UnscaledValue(ss_net_paid#24))#34,17,2) AS year_total#38]
+
+(38) BroadcastExchange
+Input [4]: [customer_id#35, customer_first_name#36, customer_last_name#37, year_total#38]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#39]
+
+(39) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [customer_id#17]
+Right keys [1]: [customer_id#35]
+Join condition: None
+
+(40) Scan parquet default.customer
+Output [4]: [c_customer_sk#40, c_customer_id#41, c_first_name#42, c_last_name#43]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string>
+
+(41) ColumnarToRow [codegen id : 10]
+Input [4]: [c_customer_sk#40, c_customer_id#41, c_first_name#42, c_last_name#43]
+
+(42) Filter [codegen id : 10]
+Input [4]: [c_customer_sk#40, c_customer_id#41, c_first_name#42, c_last_name#43]
+Condition : (isnotnull(c_customer_sk#40) AND isnotnull(c_customer_id#41))
+
+(43) Scan parquet default.web_sales
+Output [3]: [ws_bill_customer_sk#44, ws_net_paid#45, ws_sold_date_sk#46]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#46), dynamicpruningexpression(ws_sold_date_sk#46 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int,ws_net_paid:decimal(7,2)>
+
+(44) ColumnarToRow [codegen id : 8]
+Input [3]: [ws_bill_customer_sk#44, ws_net_paid#45, ws_sold_date_sk#46]
+
+(45) Filter [codegen id : 8]
+Input [3]: [ws_bill_customer_sk#44, ws_net_paid#45, ws_sold_date_sk#46]
+Condition : isnotnull(ws_bill_customer_sk#44)
+
+(46) BroadcastExchange
+Input [3]: [ws_bill_customer_sk#44, ws_net_paid#45, ws_sold_date_sk#46]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#47]
+
+(47) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [c_customer_sk#40]
+Right keys [1]: [ws_bill_customer_sk#44]
+Join condition: None
+
+(48) Project [codegen id : 10]
+Output [5]: [c_customer_id#41, c_first_name#42, c_last_name#43, ws_net_paid#45, ws_sold_date_sk#46]
+Input [7]: [c_customer_sk#40, c_customer_id#41, c_first_name#42, c_last_name#43, ws_bill_customer_sk#44, ws_net_paid#45, ws_sold_date_sk#46]
+
+(49) ReusedExchange [Reuses operator id: 13]
+Output [2]: [d_date_sk#48, d_year#49]
+
+(50) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ws_sold_date_sk#46]
+Right keys [1]: [d_date_sk#48]
+Join condition: None
+
+(51) Project [codegen id : 10]
+Output [5]: [c_customer_id#41, c_first_name#42, c_last_name#43, ws_net_paid#45, d_year#49]
+Input [7]: [c_customer_id#41, c_first_name#42, c_last_name#43, ws_net_paid#45, ws_sold_date_sk#46, d_date_sk#48, d_year#49]
+
+(52) HashAggregate [codegen id : 10]
+Input [5]: [c_customer_id#41, c_first_name#42, c_last_name#43, ws_net_paid#45, d_year#49]
+Keys [4]: [c_customer_id#41, c_first_name#42, c_last_name#43, d_year#49]
+Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#45))]
+Aggregate Attributes [1]: [sum#50]
+Results [5]: [c_customer_id#41, c_first_name#42, c_last_name#43, d_year#49, sum#51]
+
+(53) Exchange
+Input [5]: [c_customer_id#41, c_first_name#42, c_last_name#43, d_year#49, sum#51]
+Arguments: hashpartitioning(c_customer_id#41, c_first_name#42, c_last_name#43, d_year#49, 5), ENSURE_REQUIREMENTS, [id=#52]
+
+(54) HashAggregate [codegen id : 11]
+Input [5]: [c_customer_id#41, c_first_name#42, c_last_name#43, d_year#49, sum#51]
+Keys [4]: [c_customer_id#41, c_first_name#42, c_last_name#43, d_year#49]
+Functions [1]: [sum(UnscaledValue(ws_net_paid#45))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#45))#53]
+Results [2]: [c_customer_id#41 AS customer_id#54, MakeDecimal(sum(UnscaledValue(ws_net_paid#45))#53,17,2) AS year_total#55]
+
+(55) Filter [codegen id : 11]
+Input [2]: [customer_id#54, year_total#55]
+Condition : (isnotnull(year_total#55) AND (year_total#55 > 0.00))
+
+(56) Project [codegen id : 11]
+Output [2]: [customer_id#54 AS customer_id#56, year_total#55 AS year_total#57]
+Input [2]: [customer_id#54, year_total#55]
+
+(57) BroadcastExchange
+Input [2]: [customer_id#56, year_total#57]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#58]
+
+(58) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [customer_id#17]
+Right keys [1]: [customer_id#56]
+Join condition: None
+
+(59) Project [codegen id : 16]
+Output [7]: [customer_id#17, year_total#18, customer_id#35, customer_first_name#36, customer_last_name#37, year_total#38, year_total#57]
+Input [8]: [customer_id#17, year_total#18, customer_id#35, customer_first_name#36, customer_last_name#37, year_total#38, customer_id#56, year_total#57]
+
+(60) Scan parquet default.customer
+Output [4]: [c_customer_sk#59, c_customer_id#60, c_first_name#61, c_last_name#62]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string>
+
+(61) ColumnarToRow [codegen id : 14]
+Input [4]: [c_customer_sk#59, c_customer_id#60, c_first_name#61, c_last_name#62]
+
+(62) Filter [codegen id : 14]
+Input [4]: [c_customer_sk#59, c_customer_id#60, c_first_name#61, c_last_name#62]
+Condition : (isnotnull(c_customer_sk#59) AND isnotnull(c_customer_id#60))
+
+(63) Scan parquet default.web_sales
+Output [3]: [ws_bill_customer_sk#63, ws_net_paid#64, ws_sold_date_sk#65]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#65), dynamicpruningexpression(ws_sold_date_sk#65 IN dynamicpruning#26)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int,ws_net_paid:decimal(7,2)>
+
+(64) ColumnarToRow [codegen id : 12]
+Input [3]: [ws_bill_customer_sk#63, ws_net_paid#64, ws_sold_date_sk#65]
+
+(65) Filter [codegen id : 12]
+Input [3]: [ws_bill_customer_sk#63, ws_net_paid#64, ws_sold_date_sk#65]
+Condition : isnotnull(ws_bill_customer_sk#63)
+
+(66) BroadcastExchange
+Input [3]: [ws_bill_customer_sk#63, ws_net_paid#64, ws_sold_date_sk#65]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#66]
+
+(67) BroadcastHashJoin [codegen id : 14]
+Left keys [1]: [c_customer_sk#59]
+Right keys [1]: [ws_bill_customer_sk#63]
+Join condition: None
+
+(68) Project [codegen id : 14]
+Output [5]: [c_customer_id#60, c_first_name#61, c_last_name#62, ws_net_paid#64, ws_sold_date_sk#65]
+Input [7]: [c_customer_sk#59, c_customer_id#60, c_first_name#61, c_last_name#62, ws_bill_customer_sk#63, ws_net_paid#64, ws_sold_date_sk#65]
+
+(69) ReusedExchange [Reuses operator id: 32]
+Output [2]: [d_date_sk#67, d_year#68]
+
+(70) BroadcastHashJoin [codegen id : 14]
+Left keys [1]: [ws_sold_date_sk#65]
+Right keys [1]: [d_date_sk#67]
+Join condition: None
+
+(71) Project [codegen id : 14]
+Output [5]: [c_customer_id#60, c_first_name#61, c_last_name#62, ws_net_paid#64, d_year#68]
+Input [7]: [c_customer_id#60, c_first_name#61, c_last_name#62, ws_net_paid#64, ws_sold_date_sk#65, d_date_sk#67, d_year#68]
+
+(72) HashAggregate [codegen id : 14]
+Input [5]: [c_customer_id#60, c_first_name#61, c_last_name#62, ws_net_paid#64, d_year#68]
+Keys [4]: [c_customer_id#60, c_first_name#61, c_last_name#62, d_year#68]
+Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#64))]
+Aggregate Attributes [1]: [sum#69]
+Results [5]: [c_customer_id#60, c_first_name#61, c_last_name#62, d_year#68, sum#70]
+
+(73) Exchange
+Input [5]: [c_customer_id#60, c_first_name#61, c_last_name#62, d_year#68, sum#70]
+Arguments: hashpartitioning(c_customer_id#60, c_first_name#61, c_last_name#62, d_year#68, 5), ENSURE_REQUIREMENTS, [id=#71]
+
+(74) HashAggregate [codegen id : 15]
+Input [5]: [c_customer_id#60, c_first_name#61, c_last_name#62, d_year#68, sum#70]
+Keys [4]: [c_customer_id#60, c_first_name#61, c_last_name#62, d_year#68]
+Functions [1]: [sum(UnscaledValue(ws_net_paid#64))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#64))#72]
+Results [2]: [c_customer_id#60 AS customer_id#73, MakeDecimal(sum(UnscaledValue(ws_net_paid#64))#72,17,2) AS year_total#74]
+
+(75) BroadcastExchange
+Input [2]: [customer_id#73, year_total#74]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#75]
+
+(76) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [customer_id#17]
+Right keys [1]: [customer_id#73]
+Join condition: (CASE WHEN (year_total#57 > 0.00) THEN CheckOverflow((promote_precision(year_total#74) / promote_precision(year_total#57)), DecimalType(37,20), true) ELSE null END > CASE WHEN (year_total#18 > 0.00) THEN CheckOverflow((promote_precision(year_total#38) / promote_precision(year_total#18)), DecimalType(37,20), true) ELSE null END)
+
+(77) Project [codegen id : 16]
+Output [3]: [customer_id#35, customer_first_name#36, customer_last_name#37]
+Input [9]: [customer_id#17, year_total#18, customer_id#35, customer_first_name#36, customer_last_name#37, year_total#38, year_total#57, customer_id#73, year_total#74]
+
+(78) TakeOrderedAndProject
+Input [3]: [customer_id#35, customer_first_name#36, customer_last_name#37]
+Arguments: 100, [customer_id#35 ASC NULLS FIRST, customer_id#35 ASC NULLS FIRST, customer_id#35 ASC NULLS FIRST], [customer_id#35, customer_first_name#36, customer_last_name#37]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (79)
+
+
+(79) ReusedExchange [Reuses operator id: 13]
+Output [2]: [d_date_sk#10, d_year#11]
+
+Subquery:2 Hosting operator id = 23 Hosting Expression = ss_sold_date_sk#25 IN dynamicpruning#26
+ReusedExchange (80)
+
+
+(80) ReusedExchange [Reuses operator id: 32]
+Output [2]: [d_date_sk#28, d_year#29]
+
+Subquery:3 Hosting operator id = 43 Hosting Expression = ws_sold_date_sk#46 IN dynamicpruning#8
+
+Subquery:4 Hosting operator id = 63 Hosting Expression = ws_sold_date_sk#65 IN dynamicpruning#26
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/simplified.txt
new file mode 100644
index 0000000000000..f1e2b9a595e64
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/simplified.txt
@@ -0,0 +1,123 @@
+TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name]
+  WholeStageCodegen (16)
+    Project [customer_id,customer_first_name,customer_last_name]
+      BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total]
+        Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total]
+          BroadcastHashJoin [customer_id,customer_id]
+            BroadcastHashJoin [customer_id,customer_id]
+              Filter [year_total]
+                HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,year_total,sum]
+                  InputAdapter
+                    Exchange [c_customer_id,c_first_name,c_last_name,d_year] #1
+                      WholeStageCodegen (3)
+                        HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum]
+                          Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year]
+                            BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                              Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk]
+                                BroadcastHashJoin [c_customer_sk,ss_customer_sk]
+                                  Filter [c_customer_sk,c_customer_id]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name]
+                                  InputAdapter
+                                    BroadcastExchange #2
+                                      WholeStageCodegen (1)
+                                        Filter [ss_customer_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk]
+                                                SubqueryBroadcast [d_date_sk] #1
+                                                  ReusedExchange [d_date_sk,d_year] #3
+                              InputAdapter
+                                BroadcastExchange #3
+                                  WholeStageCodegen (2)
+                                    Filter [d_year,d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.date_dim [d_date_sk,d_year]
+              InputAdapter
+                BroadcastExchange #4
+                  WholeStageCodegen (7)
+                    HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,customer_first_name,customer_last_name,year_total,sum]
+                      InputAdapter
+                        Exchange [c_customer_id,c_first_name,c_last_name,d_year] #5
+                          WholeStageCodegen (6)
+                            HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum]
+                              Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year]
+                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                  Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk]
+                                    BroadcastHashJoin [c_customer_sk,ss_customer_sk]
+                                      Filter [c_customer_sk,c_customer_id]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name]
+                                      InputAdapter
+                                        BroadcastExchange #6
+                                          WholeStageCodegen (4)
+                                            Filter [ss_customer_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #2
+                                                      ReusedExchange [d_date_sk,d_year] #7
+                                  InputAdapter
+                                    BroadcastExchange #7
+                                      WholeStageCodegen (5)
+                                        Filter [d_year,d_date_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.date_dim [d_date_sk,d_year]
+            InputAdapter
+              BroadcastExchange #8
+                WholeStageCodegen (11)
+                  Project [customer_id,year_total]
+                    Filter [year_total]
+                      HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum]
+                        InputAdapter
+                          Exchange [c_customer_id,c_first_name,c_last_name,d_year] #9
+                            WholeStageCodegen (10)
+                              HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum]
+                                Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year]
+                                  BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                    Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk]
+                                      BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk]
+                                        Filter [c_customer_sk,c_customer_id]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name]
+                                        InputAdapter
+                                          BroadcastExchange #10
+                                            WholeStageCodegen (8)
+                                              Filter [ws_bill_customer_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk]
+                                                      ReusedSubquery [d_date_sk] #1
+                                    InputAdapter
+                                      ReusedExchange [d_date_sk,d_year] #3
+        InputAdapter
+          BroadcastExchange #11
+            WholeStageCodegen (15)
+              HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum]
+                InputAdapter
+                  Exchange [c_customer_id,c_first_name,c_last_name,d_year] #12
+                    WholeStageCodegen (14)
+                      HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum]
+                        Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year]
+                          BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                            Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk]
+                              BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk]
+                                Filter [c_customer_sk,c_customer_id]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name]
+                                InputAdapter
+                                  BroadcastExchange #13
+                                    WholeStageCodegen (12)
+                                      Filter [ws_bill_customer_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk]
+                                              ReusedSubquery [d_date_sk] #2
+                            InputAdapter
+                              ReusedExchange [d_date_sk,d_year] #7
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.sf100/explain.txt
new file mode 100644
index 0000000000000..d2f110a58788f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.sf100/explain.txt
@@ -0,0 +1,750 @@
+== Physical Plan ==
+TakeOrderedAndProject (133)
++- * Project (132)
+   +- * SortMergeJoin Inner (131)
+      :- * Sort (73)
+      :  +- Exchange (72)
+      :     +- * HashAggregate (71)
+      :        +- Exchange (70)
+      :           +- * HashAggregate (69)
+      :              +- * HashAggregate (68)
+      :                 +- Exchange (67)
+      :                    +- * HashAggregate (66)
+      :                       +- Union (65)
+      :                          :- * Project (26)
+      :                          :  +- * SortMergeJoin LeftOuter (25)
+      :                          :     :- * Sort (18)
+      :                          :     :  +- Exchange (17)
+      :                          :     :     +- * Project (16)
+      :                          :     :        +- * BroadcastHashJoin Inner BuildRight (15)
+      :                          :     :           :- * Project (10)
+      :                          :     :           :  +- * BroadcastHashJoin Inner BuildRight (9)
+      :                          :     :           :     :- * Filter (3)
+      :                          :     :           :     :  +- * ColumnarToRow (2)
+      :                          :     :           :     :     +- Scan parquet default.catalog_sales (1)
+      :                          :     :           :     +- BroadcastExchange (8)
+      :                          :     :           :        +- * Project (7)
+      :                          :     :           :           +- * Filter (6)
+      :                          :     :           :              +- * ColumnarToRow (5)
+      :                          :     :           :                 +- Scan parquet default.item (4)
+      :                          :     :           +- BroadcastExchange (14)
+      :                          :     :              +- * Filter (13)
+      :                          :     :                 +- * ColumnarToRow (12)
+      :                          :     :                    +- Scan parquet default.date_dim (11)
+      :                          :     +- * Sort (24)
+      :                          :        +- Exchange (23)
+      :                          :           +- * Project (22)
+      :                          :              +- * Filter (21)
+      :                          :                 +- * ColumnarToRow (20)
+      :                          :                    +- Scan parquet default.catalog_returns (19)
+      :                          :- * Project (45)
+      :                          :  +- * SortMergeJoin LeftOuter (44)
+      :                          :     :- * Sort (37)
+      :                          :     :  +- Exchange (36)
+      :                          :     :     +- * Project (35)
+      :                          :     :        +- * BroadcastHashJoin Inner BuildRight (34)
+      :                          :     :           :- * Project (32)
+      :                          :     :           :  +- * BroadcastHashJoin Inner BuildRight (31)
+      :                          :     :           :     :- * Filter (29)
+      :                          :     :           :     :  +- * ColumnarToRow (28)
+      :                          :     :           :     :     +- Scan parquet default.store_sales (27)
+      :                          :     :           :     +- ReusedExchange (30)
+      :                          :     :           +- ReusedExchange (33)
+      :                          :     +- * Sort (43)
+      :                          :        +- Exchange (42)
+      :                          :           +- * Project (41)
+      :                          :              +- * Filter (40)
+      :                          :                 +- * ColumnarToRow (39)
+      :                          :                    +- Scan parquet default.store_returns (38)
+      :                          +- * Project (64)
+      :                             +- * SortMergeJoin LeftOuter (63)
+      :                                :- * Sort (56)
+      :                                :  +- Exchange (55)
+      :                                :     +- * Project (54)
+      :                                :        +- * BroadcastHashJoin Inner BuildRight (53)
+      :                                :           :- * Project (51)
+      :                                :           :  +- * BroadcastHashJoin Inner BuildRight (50)
+      :                                :           :     :- * Filter (48)
+      :                                :           :     :  +- * ColumnarToRow (47)
+      :                                :           :     :     +- Scan parquet default.web_sales (46)
+      :                                :           :     +- ReusedExchange (49)
+      :                                :           +- ReusedExchange (52)
+      :                                +- * Sort (62)
+      :                                   +- Exchange (61)
+      :                                      +- * Project (60)
+      :                                         +- * Filter (59)
+      :                                            +- * ColumnarToRow (58)
+      :                                               +- Scan parquet default.web_returns (57)
+      +- * Sort (130)
+         +- Exchange (129)
+            +- * HashAggregate (128)
+               +- Exchange (127)
+                  +- * HashAggregate (126)
+                     +- * HashAggregate (125)
+                        +- Exchange (124)
+                           +- * HashAggregate (123)
+                              +- Union (122)
+                                 :- * Project (91)
+                                 :  +- * SortMergeJoin LeftOuter (90)
+                                 :     :- * Sort (87)
+                                 :     :  +- Exchange (86)
+                                 :     :     +- * Project (85)
+                                 :     :        +- * BroadcastHashJoin Inner BuildRight (84)
+                                 :     :           :- * Project (79)
+                                 :     :           :  +- * BroadcastHashJoin Inner BuildRight (78)
+                                 :     :           :     :- * Filter (76)
+                                 :     :           :     :  +- * ColumnarToRow (75)
+                                 :     :           :     :     +- Scan parquet default.catalog_sales (74)
+                                 :     :           :     +- ReusedExchange (77)
+                                 :     :           +- BroadcastExchange (83)
+                                 :     :              +- * Filter (82)
+                                 :     :                 +- * ColumnarToRow (81)
+                                 :     :                    +- Scan parquet default.date_dim (80)
+                                 :     +- * Sort (89)
+                                 :        +- ReusedExchange (88)
+                                 :- * Project (106)
+                                 :  +- * SortMergeJoin LeftOuter (105)
+                                 :     :- * Sort (102)
+                                 :     :  +- Exchange (101)
+                                 :     :     +- * Project (100)
+                                 :     :        +- * BroadcastHashJoin Inner BuildRight (99)
+                                 :     :           :- * Project (97)
+                                 :     :           :  +- * BroadcastHashJoin Inner BuildRight (96)
+                                 :     :           :     :- * Filter (94)
+                                 :     :           :     :  +- * ColumnarToRow (93)
+                                 :     :           :     :     +- Scan parquet default.store_sales (92)
+                                 :     :           :     +- ReusedExchange (95)
+                                 :     :           +- ReusedExchange (98)
+                                 :     +- * Sort (104)
+                                 :        +- ReusedExchange (103)
+                                 +- * Project (121)
+                                    +- * SortMergeJoin LeftOuter (120)
+                                       :- * Sort (117)
+                                       :  +- Exchange (116)
+                                       :     +- * Project (115)
+                                       :        +- * BroadcastHashJoin Inner BuildRight (114)
+                                       :           :- * Project (112)
+                                       :           :  +- * BroadcastHashJoin Inner BuildRight (111)
+                                       :           :     :- * Filter (109)
+                                       :           :     :  +- * ColumnarToRow (108)
+                                       :           :     :     +- Scan parquet default.web_sales (107)
+                                       :           :     +- ReusedExchange (110)
+                                       :           +- ReusedExchange (113)
+                                       +- * Sort (119)
+                                          +- ReusedExchange (118)
+
+
+(1) Scan parquet default.catalog_sales
+Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int,cs_order_number:int,cs_quantity:int,cs_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5]
+
+(3) Filter [codegen id : 3]
+Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5]
+Condition : isnotnull(cs_item_sk#1)
+
+(4) Scan parquet default.item
+Output [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Books                                             ), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int,i_category:string,i_manufact_id:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12]
+
+(6) Filter [codegen id : 1]
+Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12]
+Condition : ((((((isnotnull(i_category#11) AND (i_category#11 = Books                                             )) AND isnotnull(i_item_sk#7)) AND isnotnull(i_brand_id#8)) AND isnotnull(i_class_id#9)) AND isnotnull(i_category_id#10)) AND isnotnull(i_manufact_id#12))
+
+(7) Project [codegen id : 1]
+Output [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12]
+
+(8) BroadcastExchange
+Input [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_item_sk#1]
+Right keys [1]: [i_item_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+
+(11) Scan parquet default.date_dim
+Output [2]: [d_date_sk#14, d_year#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#14, d_year#15]
+
+(13) Filter [codegen id : 2]
+Input [2]: [d_date_sk#14, d_year#15]
+Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2002)) AND isnotnull(d_date_sk#14))
+
+(14) BroadcastExchange
+Input [2]: [d_date_sk#14, d_year#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#16]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_sold_date_sk#5]
+Right keys [1]: [d_date_sk#14]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#15]
+Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_date_sk#14, d_year#15]
+
+(17) Exchange
+Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#15]
+Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(18) Sort [codegen id : 4]
+Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#15]
+Arguments: [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST], false, 0
+
+(19) Scan parquet default.catalog_returns
+Output [5]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21, cr_returned_date_sk#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int,cr_return_quantity:int,cr_return_amount:decimal(7,2)>
+
+(20) ColumnarToRow [codegen id : 5]
+Input [5]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21, cr_returned_date_sk#22]
+
+(21) Filter [codegen id : 5]
+Input [5]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21, cr_returned_date_sk#22]
+Condition : (isnotnull(cr_order_number#19) AND isnotnull(cr_item_sk#18))
+
+(22) Project [codegen id : 5]
+Output [4]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21]
+Input [5]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21, cr_returned_date_sk#22]
+
+(23) Exchange
+Input [4]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21]
+Arguments: hashpartitioning(cr_order_number#19, cr_item_sk#18, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(24) Sort [codegen id : 6]
+Input [4]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21]
+Arguments: [cr_order_number#19 ASC NULLS FIRST, cr_item_sk#18 ASC NULLS FIRST], false, 0
+
+(25) SortMergeJoin [codegen id : 7]
+Left keys [2]: [cs_order_number#2, cs_item_sk#1]
+Right keys [2]: [cr_order_number#19, cr_item_sk#18]
+Join condition: None
+
+(26) Project [codegen id : 7]
+Output [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, (cs_quantity#3 - coalesce(cr_return_quantity#20, 0)) AS sales_cnt#24, CheckOverflow((promote_precision(cast(cs_ext_sales_price#4 as decimal(8,2))) - promote_precision(cast(coalesce(cr_return_amount#21, 0.00) as decimal(8,2)))), DecimalType(8,2), true) AS sales_amt#25]
+Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#15, cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21]
+
+(27) Scan parquet default.store_sales
+Output [5]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, ss_sold_date_sk#30]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#30), dynamicpruningexpression(ss_sold_date_sk#30 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ticket_number:int,ss_quantity:int,ss_ext_sales_price:decimal(7,2)>
+
+(28) ColumnarToRow [codegen id : 10]
+Input [5]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, ss_sold_date_sk#30]
+
+(29) Filter [codegen id : 10]
+Input [5]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, ss_sold_date_sk#30]
+Condition : isnotnull(ss_item_sk#26)
+
+(30) ReusedExchange [Reuses operator id: 8]
+Output [5]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35]
+
+(31) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_item_sk#26]
+Right keys [1]: [i_item_sk#31]
+Join condition: None
+
+(32) Project [codegen id : 10]
+Output [9]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, ss_sold_date_sk#30, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35]
+Input [10]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, ss_sold_date_sk#30, i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35]
+
+(33) ReusedExchange [Reuses operator id: 14]
+Output [2]: [d_date_sk#36, d_year#37]
+
+(34) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_sold_date_sk#30]
+Right keys [1]: [d_date_sk#36]
+Join condition: None
+
+(35) Project [codegen id : 10]
+Output [9]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35, d_year#37]
+Input [11]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, ss_sold_date_sk#30, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35, d_date_sk#36, d_year#37]
+
+(36) Exchange
+Input [9]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35, d_year#37]
+Arguments: hashpartitioning(ss_ticket_number#27, ss_item_sk#26, 5), ENSURE_REQUIREMENTS, [id=#38]
+
+(37) Sort [codegen id : 11]
+Input [9]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35, d_year#37]
+Arguments: [ss_ticket_number#27 ASC NULLS FIRST, ss_item_sk#26 ASC NULLS FIRST], false, 0
+
+(38) Scan parquet default.store_returns
+Output [5]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42, sr_returned_date_sk#43]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int,sr_return_quantity:int,sr_return_amt:decimal(7,2)>
+
+(39) ColumnarToRow [codegen id : 12]
+Input [5]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42, sr_returned_date_sk#43]
+
+(40) Filter [codegen id : 12]
+Input [5]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42, sr_returned_date_sk#43]
+Condition : (isnotnull(sr_ticket_number#40) AND isnotnull(sr_item_sk#39))
+
+(41) Project [codegen id : 12]
+Output [4]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42]
+Input [5]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42, sr_returned_date_sk#43]
+
+(42) Exchange
+Input [4]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42]
+Arguments: hashpartitioning(sr_ticket_number#40, sr_item_sk#39, 5), ENSURE_REQUIREMENTS, [id=#44]
+
+(43) Sort [codegen id : 13]
+Input [4]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42]
+Arguments: [sr_ticket_number#40 ASC NULLS FIRST, sr_item_sk#39 ASC NULLS FIRST], false, 0
+
+(44) SortMergeJoin [codegen id : 14]
+Left keys [2]: [ss_ticket_number#27, ss_item_sk#26]
+Right keys [2]: [sr_ticket_number#40, sr_item_sk#39]
+Join condition: None
+
+(45) Project [codegen id : 14]
+Output [7]: [d_year#37, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35, (ss_quantity#28 - coalesce(sr_return_quantity#41, 0)) AS sales_cnt#45, CheckOverflow((promote_precision(cast(ss_ext_sales_price#29 as decimal(8,2))) - promote_precision(cast(coalesce(sr_return_amt#42, 0.00) as decimal(8,2)))), DecimalType(8,2), true) AS sales_amt#46]
+Input [13]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35, d_year#37, sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42]
+
+(46) Scan parquet default.web_sales
+Output [5]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, ws_sold_date_sk#51]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#51), dynamicpruningexpression(ws_sold_date_sk#51 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_order_number:int,ws_quantity:int,ws_ext_sales_price:decimal(7,2)>
+
+(47) ColumnarToRow [codegen id : 17]
+Input [5]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, ws_sold_date_sk#51]
+
+(48) Filter [codegen id : 17]
+Input [5]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, ws_sold_date_sk#51]
+Condition : isnotnull(ws_item_sk#47)
+
+(49) ReusedExchange [Reuses operator id: 8]
+Output [5]: [i_item_sk#52, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56]
+
+(50) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_item_sk#47]
+Right keys [1]: [i_item_sk#52]
+Join condition: None
+
+(51) Project [codegen id : 17]
+Output [9]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, ws_sold_date_sk#51, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56]
+Input [10]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, ws_sold_date_sk#51, i_item_sk#52, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56]
+
+(52) ReusedExchange [Reuses operator id: 14]
+Output [2]: [d_date_sk#57, d_year#58]
+
+(53) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_sold_date_sk#51]
+Right keys [1]: [d_date_sk#57]
+Join condition: None
+
+(54) Project [codegen id : 17]
+Output [9]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56, d_year#58]
+Input [11]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, ws_sold_date_sk#51, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56, d_date_sk#57, d_year#58]
+
+(55) Exchange
+Input [9]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56, d_year#58]
+Arguments: hashpartitioning(ws_order_number#48, ws_item_sk#47, 5), ENSURE_REQUIREMENTS, [id=#59]
+
+(56) Sort [codegen id : 18]
+Input [9]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56, d_year#58]
+Arguments: [ws_order_number#48 ASC NULLS FIRST, ws_item_sk#47 ASC NULLS FIRST], false, 0
+
+(57) Scan parquet default.web_returns
+Output [5]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63, wr_returned_date_sk#64]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_returns]
+PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)]
+ReadSchema: struct<wr_item_sk:int,wr_order_number:int,wr_return_quantity:int,wr_return_amt:decimal(7,2)>
+
+(58) ColumnarToRow [codegen id : 19]
+Input [5]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63, wr_returned_date_sk#64]
+
+(59) Filter [codegen id : 19]
+Input [5]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63, wr_returned_date_sk#64]
+Condition : (isnotnull(wr_order_number#61) AND isnotnull(wr_item_sk#60))
+
+(60) Project [codegen id : 19]
+Output [4]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63]
+Input [5]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63, wr_returned_date_sk#64]
+
+(61) Exchange
+Input [4]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63]
+Arguments: hashpartitioning(wr_order_number#61, wr_item_sk#60, 5), ENSURE_REQUIREMENTS, [id=#65]
+
+(62) Sort [codegen id : 20]
+Input [4]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63]
+Arguments: [wr_order_number#61 ASC NULLS FIRST, wr_item_sk#60 ASC NULLS FIRST], false, 0
+
+(63) SortMergeJoin [codegen id : 21]
+Left keys [2]: [ws_order_number#48, ws_item_sk#47]
+Right keys [2]: [wr_order_number#61, wr_item_sk#60]
+Join condition: None
+
+(64) Project [codegen id : 21]
+Output [7]: [d_year#58, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56, (ws_quantity#49 - coalesce(wr_return_quantity#62, 0)) AS sales_cnt#66, CheckOverflow((promote_precision(cast(ws_ext_sales_price#50 as decimal(8,2))) - promote_precision(cast(coalesce(wr_return_amt#63, 0.00) as decimal(8,2)))), DecimalType(8,2), true) AS sales_amt#67]
+Input [13]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56, d_year#58, wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63]
+
+(65) Union
+
+(66) HashAggregate [codegen id : 22]
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+Keys [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+Functions: []
+Aggregate Attributes: []
+Results [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+
+(67) Exchange
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+Arguments: hashpartitioning(d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25, 5), ENSURE_REQUIREMENTS, [id=#68]
+
+(68) HashAggregate [codegen id : 23]
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+Keys [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+Functions: []
+Aggregate Attributes: []
+Results [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+
+(69) HashAggregate [codegen id : 23]
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+Keys [5]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+Functions [2]: [partial_sum(sales_cnt#24), partial_sum(UnscaledValue(sales_amt#25))]
+Aggregate Attributes [2]: [sum#69, sum#70]
+Results [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#71, sum#72]
+
+(70) Exchange
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#71, sum#72]
+Arguments: hashpartitioning(d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [id=#73]
+
+(71) HashAggregate [codegen id : 24]
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#71, sum#72]
+Keys [5]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+Functions [2]: [sum(sales_cnt#24), sum(UnscaledValue(sales_amt#25))]
+Aggregate Attributes [2]: [sum(sales_cnt#24)#74, sum(UnscaledValue(sales_amt#25))#75]
+Results [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum(sales_cnt#24)#74 AS sales_cnt#76, MakeDecimal(sum(UnscaledValue(sales_amt#25))#75,18,2) AS sales_amt#77]
+
+(72) Exchange
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#76, sales_amt#77]
+Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [id=#78]
+
+(73) Sort [codegen id : 25]
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#76, sales_amt#77]
+Arguments: [i_brand_id#8 ASC NULLS FIRST, i_class_id#9 ASC NULLS FIRST, i_category_id#10 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST], false, 0
+
+(74) Scan parquet default.catalog_sales
+Output [5]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, cs_sold_date_sk#83]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#83), dynamicpruningexpression(cs_sold_date_sk#83 IN dynamicpruning#84)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int,cs_order_number:int,cs_quantity:int,cs_ext_sales_price:decimal(7,2)>
+
+(75) ColumnarToRow [codegen id : 28]
+Input [5]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, cs_sold_date_sk#83]
+
+(76) Filter [codegen id : 28]
+Input [5]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, cs_sold_date_sk#83]
+Condition : isnotnull(cs_item_sk#79)
+
+(77) ReusedExchange [Reuses operator id: 8]
+Output [5]: [i_item_sk#85, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89]
+
+(78) BroadcastHashJoin [codegen id : 28]
+Left keys [1]: [cs_item_sk#79]
+Right keys [1]: [i_item_sk#85]
+Join condition: None
+
+(79) Project [codegen id : 28]
+Output [9]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, cs_sold_date_sk#83, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89]
+Input [10]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, cs_sold_date_sk#83, i_item_sk#85, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89]
+
+(80) Scan parquet default.date_dim
+Output [2]: [d_date_sk#90, d_year#91]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(81) ColumnarToRow [codegen id : 27]
+Input [2]: [d_date_sk#90, d_year#91]
+
+(82) Filter [codegen id : 27]
+Input [2]: [d_date_sk#90, d_year#91]
+Condition : ((isnotnull(d_year#91) AND (d_year#91 = 2001)) AND isnotnull(d_date_sk#90))
+
+(83) BroadcastExchange
+Input [2]: [d_date_sk#90, d_year#91]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#92]
+
+(84) BroadcastHashJoin [codegen id : 28]
+Left keys [1]: [cs_sold_date_sk#83]
+Right keys [1]: [d_date_sk#90]
+Join condition: None
+
+(85) Project [codegen id : 28]
+Output [9]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, d_year#91]
+Input [11]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, cs_sold_date_sk#83, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, d_date_sk#90, d_year#91]
+
+(86) Exchange
+Input [9]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, d_year#91]
+Arguments: hashpartitioning(cs_order_number#80, cs_item_sk#79, 5), ENSURE_REQUIREMENTS, [id=#93]
+
+(87) Sort [codegen id : 29]
+Input [9]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, d_year#91]
+Arguments: [cs_order_number#80 ASC NULLS FIRST, cs_item_sk#79 ASC NULLS FIRST], false, 0
+
+(88) ReusedExchange [Reuses operator id: 23]
+Output [4]: [cr_item_sk#94, cr_order_number#95, cr_return_quantity#96, cr_return_amount#97]
+
+(89) Sort [codegen id : 31]
+Input [4]: [cr_item_sk#94, cr_order_number#95, cr_return_quantity#96, cr_return_amount#97]
+Arguments: [cr_order_number#95 ASC NULLS FIRST, cr_item_sk#94 ASC NULLS FIRST], false, 0
+
+(90) SortMergeJoin [codegen id : 32]
+Left keys [2]: [cs_order_number#80, cs_item_sk#79]
+Right keys [2]: [cr_order_number#95, cr_item_sk#94]
+Join condition: None
+
+(91) Project [codegen id : 32]
+Output [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, (cs_quantity#81 - coalesce(cr_return_quantity#96, 0)) AS sales_cnt#24, CheckOverflow((promote_precision(cast(cs_ext_sales_price#82 as decimal(8,2))) - promote_precision(cast(coalesce(cr_return_amount#97, 0.00) as decimal(8,2)))), DecimalType(8,2), true) AS sales_amt#25]
+Input [13]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, d_year#91, cr_item_sk#94, cr_order_number#95, cr_return_quantity#96, cr_return_amount#97]
+
+(92) Scan parquet default.store_sales
+Output [5]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, ss_sold_date_sk#102]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#102), dynamicpruningexpression(ss_sold_date_sk#102 IN dynamicpruning#84)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ticket_number:int,ss_quantity:int,ss_ext_sales_price:decimal(7,2)>
+
+(93) ColumnarToRow [codegen id : 35]
+Input [5]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, ss_sold_date_sk#102]
+
+(94) Filter [codegen id : 35]
+Input [5]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, ss_sold_date_sk#102]
+Condition : isnotnull(ss_item_sk#98)
+
+(95) ReusedExchange [Reuses operator id: 8]
+Output [5]: [i_item_sk#103, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107]
+
+(96) BroadcastHashJoin [codegen id : 35]
+Left keys [1]: [ss_item_sk#98]
+Right keys [1]: [i_item_sk#103]
+Join condition: None
+
+(97) Project [codegen id : 35]
+Output [9]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, ss_sold_date_sk#102, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107]
+Input [10]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, ss_sold_date_sk#102, i_item_sk#103, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107]
+
+(98) ReusedExchange [Reuses operator id: 83]
+Output [2]: [d_date_sk#108, d_year#109]
+
+(99) BroadcastHashJoin [codegen id : 35]
+Left keys [1]: [ss_sold_date_sk#102]
+Right keys [1]: [d_date_sk#108]
+Join condition: None
+
+(100) Project [codegen id : 35]
+Output [9]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107, d_year#109]
+Input [11]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, ss_sold_date_sk#102, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107, d_date_sk#108, d_year#109]
+
+(101) Exchange
+Input [9]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107, d_year#109]
+Arguments: hashpartitioning(ss_ticket_number#99, ss_item_sk#98, 5), ENSURE_REQUIREMENTS, [id=#110]
+
+(102) Sort [codegen id : 36]
+Input [9]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107, d_year#109]
+Arguments: [ss_ticket_number#99 ASC NULLS FIRST, ss_item_sk#98 ASC NULLS FIRST], false, 0
+
+(103) ReusedExchange [Reuses operator id: 42]
+Output [4]: [sr_item_sk#111, sr_ticket_number#112, sr_return_quantity#113, sr_return_amt#114]
+
+(104) Sort [codegen id : 38]
+Input [4]: [sr_item_sk#111, sr_ticket_number#112, sr_return_quantity#113, sr_return_amt#114]
+Arguments: [sr_ticket_number#112 ASC NULLS FIRST, sr_item_sk#111 ASC NULLS FIRST], false, 0
+
+(105) SortMergeJoin [codegen id : 39]
+Left keys [2]: [ss_ticket_number#99, ss_item_sk#98]
+Right keys [2]: [sr_ticket_number#112, sr_item_sk#111]
+Join condition: None
+
+(106) Project [codegen id : 39]
+Output [7]: [d_year#109, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107, (ss_quantity#100 - coalesce(sr_return_quantity#113, 0)) AS sales_cnt#45, CheckOverflow((promote_precision(cast(ss_ext_sales_price#101 as decimal(8,2))) - promote_precision(cast(coalesce(sr_return_amt#114, 0.00) as decimal(8,2)))), DecimalType(8,2), true) AS sales_amt#46]
+Input [13]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107, d_year#109, sr_item_sk#111, sr_ticket_number#112, sr_return_quantity#113, sr_return_amt#114]
+
+(107) Scan parquet default.web_sales
+Output [5]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, ws_sold_date_sk#119]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#119), dynamicpruningexpression(ws_sold_date_sk#119 IN dynamicpruning#84)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_order_number:int,ws_quantity:int,ws_ext_sales_price:decimal(7,2)>
+
+(108) ColumnarToRow [codegen id : 42]
+Input [5]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, ws_sold_date_sk#119]
+
+(109) Filter [codegen id : 42]
+Input [5]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, ws_sold_date_sk#119]
+Condition : isnotnull(ws_item_sk#115)
+
+(110) ReusedExchange [Reuses operator id: 8]
+Output [5]: [i_item_sk#120, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124]
+
+(111) BroadcastHashJoin [codegen id : 42]
+Left keys [1]: [ws_item_sk#115]
+Right keys [1]: [i_item_sk#120]
+Join condition: None
+
+(112) Project [codegen id : 42]
+Output [9]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, ws_sold_date_sk#119, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124]
+Input [10]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, ws_sold_date_sk#119, i_item_sk#120, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124]
+
+(113) ReusedExchange [Reuses operator id: 83]
+Output [2]: [d_date_sk#125, d_year#126]
+
+(114) BroadcastHashJoin [codegen id : 42]
+Left keys [1]: [ws_sold_date_sk#119]
+Right keys [1]: [d_date_sk#125]
+Join condition: None
+
+(115) Project [codegen id : 42]
+Output [9]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124, d_year#126]
+Input [11]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, ws_sold_date_sk#119, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124, d_date_sk#125, d_year#126]
+
+(116) Exchange
+Input [9]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124, d_year#126]
+Arguments: hashpartitioning(ws_order_number#116, ws_item_sk#115, 5), ENSURE_REQUIREMENTS, [id=#127]
+
+(117) Sort [codegen id : 43]
+Input [9]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124, d_year#126]
+Arguments: [ws_order_number#116 ASC NULLS FIRST, ws_item_sk#115 ASC NULLS FIRST], false, 0
+
+(118) ReusedExchange [Reuses operator id: 61]
+Output [4]: [wr_item_sk#128, wr_order_number#129, wr_return_quantity#130, wr_return_amt#131]
+
+(119) Sort [codegen id : 45]
+Input [4]: [wr_item_sk#128, wr_order_number#129, wr_return_quantity#130, wr_return_amt#131]
+Arguments: [wr_order_number#129 ASC NULLS FIRST, wr_item_sk#128 ASC NULLS FIRST], false, 0
+
+(120) SortMergeJoin [codegen id : 46]
+Left keys [2]: [ws_order_number#116, ws_item_sk#115]
+Right keys [2]: [wr_order_number#129, wr_item_sk#128]
+Join condition: None
+
+(121) Project [codegen id : 46]
+Output [7]: [d_year#126, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124, (ws_quantity#117 - coalesce(wr_return_quantity#130, 0)) AS sales_cnt#66, CheckOverflow((promote_precision(cast(ws_ext_sales_price#118 as decimal(8,2))) - promote_precision(cast(coalesce(wr_return_amt#131, 0.00) as decimal(8,2)))), DecimalType(8,2), true) AS sales_amt#67]
+Input [13]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124, d_year#126, wr_item_sk#128, wr_order_number#129, wr_return_quantity#130, wr_return_amt#131]
+
+(122) Union
+
+(123) HashAggregate [codegen id : 47]
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+Keys [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+Functions: []
+Aggregate Attributes: []
+Results [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+
+(124) Exchange
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+Arguments: hashpartitioning(d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25, 5), ENSURE_REQUIREMENTS, [id=#132]
+
+(125) HashAggregate [codegen id : 48]
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+Keys [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+Functions: []
+Aggregate Attributes: []
+Results [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+
+(126) HashAggregate [codegen id : 48]
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+Keys [5]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89]
+Functions [2]: [partial_sum(sales_cnt#24), partial_sum(UnscaledValue(sales_amt#25))]
+Aggregate Attributes [2]: [sum#133, sum#134]
+Results [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sum#135, sum#136]
+
+(127) Exchange
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sum#135, sum#136]
+Arguments: hashpartitioning(d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, 5), ENSURE_REQUIREMENTS, [id=#137]
+
+(128) HashAggregate [codegen id : 49]
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sum#135, sum#136]
+Keys [5]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89]
+Functions [2]: [sum(sales_cnt#24), sum(UnscaledValue(sales_amt#25))]
+Aggregate Attributes [2]: [sum(sales_cnt#24)#138, sum(UnscaledValue(sales_amt#25))#139]
+Results [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sum(sales_cnt#24)#138 AS sales_cnt#140, MakeDecimal(sum(UnscaledValue(sales_amt#25))#139,18,2) AS sales_amt#141]
+
+(129) Exchange
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#140, sales_amt#141]
+Arguments: hashpartitioning(i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, 5), ENSURE_REQUIREMENTS, [id=#142]
+
+(130) Sort [codegen id : 50]
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#140, sales_amt#141]
+Arguments: [i_brand_id#86 ASC NULLS FIRST, i_class_id#87 ASC NULLS FIRST, i_category_id#88 ASC NULLS FIRST, i_manufact_id#89 ASC NULLS FIRST], false, 0
+
+(131) SortMergeJoin [codegen id : 51]
+Left keys [4]: [i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+Right keys [4]: [i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89]
+Join condition: (CheckOverflow((promote_precision(cast(sales_cnt#76 as decimal(17,2))) / promote_precision(cast(sales_cnt#140 as decimal(17,2)))), DecimalType(37,20), true) < 0.90000000000000000000)
+
+(132) Project [codegen id : 51]
+Output [10]: [d_year#91 AS prev_year#143, d_year#15 AS year#144, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#140 AS prev_yr_cnt#145, sales_cnt#76 AS curr_yr_cnt#146, (sales_cnt#76 - sales_cnt#140) AS sales_cnt_diff#147, CheckOverflow((promote_precision(cast(sales_amt#77 as decimal(19,2))) - promote_precision(cast(sales_amt#141 as decimal(19,2)))), DecimalType(19,2), true) AS sales_amt_diff#148]
+Input [14]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#76, sales_amt#77, d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#140, sales_amt#141]
+
+(133) TakeOrderedAndProject
+Input [10]: [prev_year#143, year#144, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#145, curr_yr_cnt#146, sales_cnt_diff#147, sales_amt_diff#148]
+Arguments: 100, [sales_cnt_diff#147 ASC NULLS FIRST], [prev_year#143, year#144, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#145, curr_yr_cnt#146, sales_cnt_diff#147, sales_amt_diff#148]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (134)
+
+
+(134) ReusedExchange [Reuses operator id: 14]
+Output [2]: [d_date_sk#14, d_year#15]
+
+Subquery:2 Hosting operator id = 27 Hosting Expression = ss_sold_date_sk#30 IN dynamicpruning#6
+
+Subquery:3 Hosting operator id = 46 Hosting Expression = ws_sold_date_sk#51 IN dynamicpruning#6
+
+Subquery:4 Hosting operator id = 74 Hosting Expression = cs_sold_date_sk#83 IN dynamicpruning#84
+ReusedExchange (135)
+
+
+(135) ReusedExchange [Reuses operator id: 83]
+Output [2]: [d_date_sk#90, d_year#91]
+
+Subquery:5 Hosting operator id = 92 Hosting Expression = ss_sold_date_sk#102 IN dynamicpruning#84
+
+Subquery:6 Hosting operator id = 107 Hosting Expression = ws_sold_date_sk#119 IN dynamicpruning#84
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.sf100/simplified.txt
new file mode 100644
index 0000000000000..170dad0b5dadd
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.sf100/simplified.txt
@@ -0,0 +1,238 @@
+TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_amt_diff]
+  WholeStageCodegen (51)
+    Project [d_year,d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt,sales_amt,sales_amt]
+      SortMergeJoin [i_brand_id,i_class_id,i_category_id,i_manufact_id,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt]
+        InputAdapter
+          WholeStageCodegen (25)
+            Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id]
+              InputAdapter
+                Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1
+                  WholeStageCodegen (24)
+                    HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum]
+                      InputAdapter
+                        Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2
+                          WholeStageCodegen (23)
+                            HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum]
+                              HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt]
+                                InputAdapter
+                                  Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3
+                                    WholeStageCodegen (22)
+                                      HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt]
+                                        InputAdapter
+                                          Union
+                                            WholeStageCodegen (7)
+                                              Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount]
+                                                SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (4)
+                                                      Sort [cs_order_number,cs_item_sk]
+                                                        InputAdapter
+                                                          Exchange [cs_order_number,cs_item_sk] #4
+                                                            WholeStageCodegen (3)
+                                                              Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year]
+                                                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                  Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                    BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                                                      Filter [cs_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk]
+                                                                              SubqueryBroadcast [d_date_sk] #1
+                                                                                ReusedExchange [d_date_sk,d_year] #5
+                                                                      InputAdapter
+                                                                        BroadcastExchange #6
+                                                                          WholeStageCodegen (1)
+                                                                            Project [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                              Filter [i_category,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id]
+                                                                  InputAdapter
+                                                                    BroadcastExchange #5
+                                                                      WholeStageCodegen (2)
+                                                                        Filter [d_year,d_date_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.date_dim [d_date_sk,d_year]
+                                                  InputAdapter
+                                                    WholeStageCodegen (6)
+                                                      Sort [cr_order_number,cr_item_sk]
+                                                        InputAdapter
+                                                          Exchange [cr_order_number,cr_item_sk] #7
+                                                            WholeStageCodegen (5)
+                                                              Project [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount]
+                                                                Filter [cr_order_number,cr_item_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk]
+                                            WholeStageCodegen (14)
+                                              Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt]
+                                                SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (11)
+                                                      Sort [ss_ticket_number,ss_item_sk]
+                                                        InputAdapter
+                                                          Exchange [ss_ticket_number,ss_item_sk] #8
+                                                            WholeStageCodegen (10)
+                                                              Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year]
+                                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                  Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                                      Filter [ss_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk]
+                                                                              ReusedSubquery [d_date_sk] #1
+                                                                      InputAdapter
+                                                                        ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6
+                                                                  InputAdapter
+                                                                    ReusedExchange [d_date_sk,d_year] #5
+                                                  InputAdapter
+                                                    WholeStageCodegen (13)
+                                                      Sort [sr_ticket_number,sr_item_sk]
+                                                        InputAdapter
+                                                          Exchange [sr_ticket_number,sr_item_sk] #9
+                                                            WholeStageCodegen (12)
+                                                              Project [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt]
+                                                                Filter [sr_ticket_number,sr_item_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk]
+                                            WholeStageCodegen (21)
+                                              Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt]
+                                                SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (18)
+                                                      Sort [ws_order_number,ws_item_sk]
+                                                        InputAdapter
+                                                          Exchange [ws_order_number,ws_item_sk] #10
+                                                            WholeStageCodegen (17)
+                                                              Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year]
+                                                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                  Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                    BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                                                      Filter [ws_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk]
+                                                                              ReusedSubquery [d_date_sk] #1
+                                                                      InputAdapter
+                                                                        ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6
+                                                                  InputAdapter
+                                                                    ReusedExchange [d_date_sk,d_year] #5
+                                                  InputAdapter
+                                                    WholeStageCodegen (20)
+                                                      Sort [wr_order_number,wr_item_sk]
+                                                        InputAdapter
+                                                          Exchange [wr_order_number,wr_item_sk] #11
+                                                            WholeStageCodegen (19)
+                                                              Project [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt]
+                                                                Filter [wr_order_number,wr_item_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk]
+        InputAdapter
+          WholeStageCodegen (50)
+            Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id]
+              InputAdapter
+                Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12
+                  WholeStageCodegen (49)
+                    HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum]
+                      InputAdapter
+                        Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13
+                          WholeStageCodegen (48)
+                            HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum]
+                              HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt]
+                                InputAdapter
+                                  Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14
+                                    WholeStageCodegen (47)
+                                      HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt]
+                                        InputAdapter
+                                          Union
+                                            WholeStageCodegen (32)
+                                              Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount]
+                                                SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (29)
+                                                      Sort [cs_order_number,cs_item_sk]
+                                                        InputAdapter
+                                                          Exchange [cs_order_number,cs_item_sk] #15
+                                                            WholeStageCodegen (28)
+                                                              Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year]
+                                                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                  Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                    BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                                                      Filter [cs_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk]
+                                                                              SubqueryBroadcast [d_date_sk] #2
+                                                                                ReusedExchange [d_date_sk,d_year] #16
+                                                                      InputAdapter
+                                                                        ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6
+                                                                  InputAdapter
+                                                                    BroadcastExchange #16
+                                                                      WholeStageCodegen (27)
+                                                                        Filter [d_year,d_date_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.date_dim [d_date_sk,d_year]
+                                                  InputAdapter
+                                                    WholeStageCodegen (31)
+                                                      Sort [cr_order_number,cr_item_sk]
+                                                        InputAdapter
+                                                          ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7
+                                            WholeStageCodegen (39)
+                                              Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt]
+                                                SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (36)
+                                                      Sort [ss_ticket_number,ss_item_sk]
+                                                        InputAdapter
+                                                          Exchange [ss_ticket_number,ss_item_sk] #17
+                                                            WholeStageCodegen (35)
+                                                              Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year]
+                                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                  Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                                      Filter [ss_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk]
+                                                                              ReusedSubquery [d_date_sk] #2
+                                                                      InputAdapter
+                                                                        ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6
+                                                                  InputAdapter
+                                                                    ReusedExchange [d_date_sk,d_year] #16
+                                                  InputAdapter
+                                                    WholeStageCodegen (38)
+                                                      Sort [sr_ticket_number,sr_item_sk]
+                                                        InputAdapter
+                                                          ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9
+                                            WholeStageCodegen (46)
+                                              Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt]
+                                                SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (43)
+                                                      Sort [ws_order_number,ws_item_sk]
+                                                        InputAdapter
+                                                          Exchange [ws_order_number,ws_item_sk] #18
+                                                            WholeStageCodegen (42)
+                                                              Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year]
+                                                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                  Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                    BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                                                      Filter [ws_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk]
+                                                                              ReusedSubquery [d_date_sk] #2
+                                                                      InputAdapter
+                                                                        ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6
+                                                                  InputAdapter
+                                                                    ReusedExchange [d_date_sk,d_year] #16
+                                                  InputAdapter
+                                                    WholeStageCodegen (45)
+                                                      Sort [wr_order_number,wr_item_sk]
+                                                        InputAdapter
+                                                          ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt
new file mode 100644
index 0000000000000..d2f110a58788f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt
@@ -0,0 +1,750 @@
+== Physical Plan ==
+TakeOrderedAndProject (133)
++- * Project (132)
+   +- * SortMergeJoin Inner (131)
+      :- * Sort (73)
+      :  +- Exchange (72)
+      :     +- * HashAggregate (71)
+      :        +- Exchange (70)
+      :           +- * HashAggregate (69)
+      :              +- * HashAggregate (68)
+      :                 +- Exchange (67)
+      :                    +- * HashAggregate (66)
+      :                       +- Union (65)
+      :                          :- * Project (26)
+      :                          :  +- * SortMergeJoin LeftOuter (25)
+      :                          :     :- * Sort (18)
+      :                          :     :  +- Exchange (17)
+      :                          :     :     +- * Project (16)
+      :                          :     :        +- * BroadcastHashJoin Inner BuildRight (15)
+      :                          :     :           :- * Project (10)
+      :                          :     :           :  +- * BroadcastHashJoin Inner BuildRight (9)
+      :                          :     :           :     :- * Filter (3)
+      :                          :     :           :     :  +- * ColumnarToRow (2)
+      :                          :     :           :     :     +- Scan parquet default.catalog_sales (1)
+      :                          :     :           :     +- BroadcastExchange (8)
+      :                          :     :           :        +- * Project (7)
+      :                          :     :           :           +- * Filter (6)
+      :                          :     :           :              +- * ColumnarToRow (5)
+      :                          :     :           :                 +- Scan parquet default.item (4)
+      :                          :     :           +- BroadcastExchange (14)
+      :                          :     :              +- * Filter (13)
+      :                          :     :                 +- * ColumnarToRow (12)
+      :                          :     :                    +- Scan parquet default.date_dim (11)
+      :                          :     +- * Sort (24)
+      :                          :        +- Exchange (23)
+      :                          :           +- * Project (22)
+      :                          :              +- * Filter (21)
+      :                          :                 +- * ColumnarToRow (20)
+      :                          :                    +- Scan parquet default.catalog_returns (19)
+      :                          :- * Project (45)
+      :                          :  +- * SortMergeJoin LeftOuter (44)
+      :                          :     :- * Sort (37)
+      :                          :     :  +- Exchange (36)
+      :                          :     :     +- * Project (35)
+      :                          :     :        +- * BroadcastHashJoin Inner BuildRight (34)
+      :                          :     :           :- * Project (32)
+      :                          :     :           :  +- * BroadcastHashJoin Inner BuildRight (31)
+      :                          :     :           :     :- * Filter (29)
+      :                          :     :           :     :  +- * ColumnarToRow (28)
+      :                          :     :           :     :     +- Scan parquet default.store_sales (27)
+      :                          :     :           :     +- ReusedExchange (30)
+      :                          :     :           +- ReusedExchange (33)
+      :                          :     +- * Sort (43)
+      :                          :        +- Exchange (42)
+      :                          :           +- * Project (41)
+      :                          :              +- * Filter (40)
+      :                          :                 +- * ColumnarToRow (39)
+      :                          :                    +- Scan parquet default.store_returns (38)
+      :                          +- * Project (64)
+      :                             +- * SortMergeJoin LeftOuter (63)
+      :                                :- * Sort (56)
+      :                                :  +- Exchange (55)
+      :                                :     +- * Project (54)
+      :                                :        +- * BroadcastHashJoin Inner BuildRight (53)
+      :                                :           :- * Project (51)
+      :                                :           :  +- * BroadcastHashJoin Inner BuildRight (50)
+      :                                :           :     :- * Filter (48)
+      :                                :           :     :  +- * ColumnarToRow (47)
+      :                                :           :     :     +- Scan parquet default.web_sales (46)
+      :                                :           :     +- ReusedExchange (49)
+      :                                :           +- ReusedExchange (52)
+      :                                +- * Sort (62)
+      :                                   +- Exchange (61)
+      :                                      +- * Project (60)
+      :                                         +- * Filter (59)
+      :                                            +- * ColumnarToRow (58)
+      :                                               +- Scan parquet default.web_returns (57)
+      +- * Sort (130)
+         +- Exchange (129)
+            +- * HashAggregate (128)
+               +- Exchange (127)
+                  +- * HashAggregate (126)
+                     +- * HashAggregate (125)
+                        +- Exchange (124)
+                           +- * HashAggregate (123)
+                              +- Union (122)
+                                 :- * Project (91)
+                                 :  +- * SortMergeJoin LeftOuter (90)
+                                 :     :- * Sort (87)
+                                 :     :  +- Exchange (86)
+                                 :     :     +- * Project (85)
+                                 :     :        +- * BroadcastHashJoin Inner BuildRight (84)
+                                 :     :           :- * Project (79)
+                                 :     :           :  +- * BroadcastHashJoin Inner BuildRight (78)
+                                 :     :           :     :- * Filter (76)
+                                 :     :           :     :  +- * ColumnarToRow (75)
+                                 :     :           :     :     +- Scan parquet default.catalog_sales (74)
+                                 :     :           :     +- ReusedExchange (77)
+                                 :     :           +- BroadcastExchange (83)
+                                 :     :              +- * Filter (82)
+                                 :     :                 +- * ColumnarToRow (81)
+                                 :     :                    +- Scan parquet default.date_dim (80)
+                                 :     +- * Sort (89)
+                                 :        +- ReusedExchange (88)
+                                 :- * Project (106)
+                                 :  +- * SortMergeJoin LeftOuter (105)
+                                 :     :- * Sort (102)
+                                 :     :  +- Exchange (101)
+                                 :     :     +- * Project (100)
+                                 :     :        +- * BroadcastHashJoin Inner BuildRight (99)
+                                 :     :           :- * Project (97)
+                                 :     :           :  +- * BroadcastHashJoin Inner BuildRight (96)
+                                 :     :           :     :- * Filter (94)
+                                 :     :           :     :  +- * ColumnarToRow (93)
+                                 :     :           :     :     +- Scan parquet default.store_sales (92)
+                                 :     :           :     +- ReusedExchange (95)
+                                 :     :           +- ReusedExchange (98)
+                                 :     +- * Sort (104)
+                                 :        +- ReusedExchange (103)
+                                 +- * Project (121)
+                                    +- * SortMergeJoin LeftOuter (120)
+                                       :- * Sort (117)
+                                       :  +- Exchange (116)
+                                       :     +- * Project (115)
+                                       :        +- * BroadcastHashJoin Inner BuildRight (114)
+                                       :           :- * Project (112)
+                                       :           :  +- * BroadcastHashJoin Inner BuildRight (111)
+                                       :           :     :- * Filter (109)
+                                       :           :     :  +- * ColumnarToRow (108)
+                                       :           :     :     +- Scan parquet default.web_sales (107)
+                                       :           :     +- ReusedExchange (110)
+                                       :           +- ReusedExchange (113)
+                                       +- * Sort (119)
+                                          +- ReusedExchange (118)
+
+
+(1) Scan parquet default.catalog_sales
+Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int,cs_order_number:int,cs_quantity:int,cs_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5]
+
+(3) Filter [codegen id : 3]
+Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5]
+Condition : isnotnull(cs_item_sk#1)
+
+(4) Scan parquet default.item
+Output [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Books                                             ), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int,i_category:string,i_manufact_id:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12]
+
+(6) Filter [codegen id : 1]
+Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12]
+Condition : ((((((isnotnull(i_category#11) AND (i_category#11 = Books                                             )) AND isnotnull(i_item_sk#7)) AND isnotnull(i_brand_id#8)) AND isnotnull(i_class_id#9)) AND isnotnull(i_category_id#10)) AND isnotnull(i_manufact_id#12))
+
+(7) Project [codegen id : 1]
+Output [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12]
+
+(8) BroadcastExchange
+Input [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_item_sk#1]
+Right keys [1]: [i_item_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+
+(11) Scan parquet default.date_dim
+Output [2]: [d_date_sk#14, d_year#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#14, d_year#15]
+
+(13) Filter [codegen id : 2]
+Input [2]: [d_date_sk#14, d_year#15]
+Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2002)) AND isnotnull(d_date_sk#14))
+
+(14) BroadcastExchange
+Input [2]: [d_date_sk#14, d_year#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#16]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_sold_date_sk#5]
+Right keys [1]: [d_date_sk#14]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#15]
+Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_date_sk#14, d_year#15]
+
+(17) Exchange
+Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#15]
+Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(18) Sort [codegen id : 4]
+Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#15]
+Arguments: [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST], false, 0
+
+(19) Scan parquet default.catalog_returns
+Output [5]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21, cr_returned_date_sk#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int,cr_return_quantity:int,cr_return_amount:decimal(7,2)>
+
+(20) ColumnarToRow [codegen id : 5]
+Input [5]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21, cr_returned_date_sk#22]
+
+(21) Filter [codegen id : 5]
+Input [5]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21, cr_returned_date_sk#22]
+Condition : (isnotnull(cr_order_number#19) AND isnotnull(cr_item_sk#18))
+
+(22) Project [codegen id : 5]
+Output [4]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21]
+Input [5]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21, cr_returned_date_sk#22]
+
+(23) Exchange
+Input [4]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21]
+Arguments: hashpartitioning(cr_order_number#19, cr_item_sk#18, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(24) Sort [codegen id : 6]
+Input [4]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21]
+Arguments: [cr_order_number#19 ASC NULLS FIRST, cr_item_sk#18 ASC NULLS FIRST], false, 0
+
+(25) SortMergeJoin [codegen id : 7]
+Left keys [2]: [cs_order_number#2, cs_item_sk#1]
+Right keys [2]: [cr_order_number#19, cr_item_sk#18]
+Join condition: None
+
+(26) Project [codegen id : 7]
+Output [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, (cs_quantity#3 - coalesce(cr_return_quantity#20, 0)) AS sales_cnt#24, CheckOverflow((promote_precision(cast(cs_ext_sales_price#4 as decimal(8,2))) - promote_precision(cast(coalesce(cr_return_amount#21, 0.00) as decimal(8,2)))), DecimalType(8,2), true) AS sales_amt#25]
+Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#15, cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21]
+
+(27) Scan parquet default.store_sales
+Output [5]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, ss_sold_date_sk#30]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#30), dynamicpruningexpression(ss_sold_date_sk#30 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ticket_number:int,ss_quantity:int,ss_ext_sales_price:decimal(7,2)>
+
+(28) ColumnarToRow [codegen id : 10]
+Input [5]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, ss_sold_date_sk#30]
+
+(29) Filter [codegen id : 10]
+Input [5]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, ss_sold_date_sk#30]
+Condition : isnotnull(ss_item_sk#26)
+
+(30) ReusedExchange [Reuses operator id: 8]
+Output [5]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35]
+
+(31) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_item_sk#26]
+Right keys [1]: [i_item_sk#31]
+Join condition: None
+
+(32) Project [codegen id : 10]
+Output [9]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, ss_sold_date_sk#30, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35]
+Input [10]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, ss_sold_date_sk#30, i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35]
+
+(33) ReusedExchange [Reuses operator id: 14]
+Output [2]: [d_date_sk#36, d_year#37]
+
+(34) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_sold_date_sk#30]
+Right keys [1]: [d_date_sk#36]
+Join condition: None
+
+(35) Project [codegen id : 10]
+Output [9]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35, d_year#37]
+Input [11]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, ss_sold_date_sk#30, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35, d_date_sk#36, d_year#37]
+
+(36) Exchange
+Input [9]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35, d_year#37]
+Arguments: hashpartitioning(ss_ticket_number#27, ss_item_sk#26, 5), ENSURE_REQUIREMENTS, [id=#38]
+
+(37) Sort [codegen id : 11]
+Input [9]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35, d_year#37]
+Arguments: [ss_ticket_number#27 ASC NULLS FIRST, ss_item_sk#26 ASC NULLS FIRST], false, 0
+
+(38) Scan parquet default.store_returns
+Output [5]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42, sr_returned_date_sk#43]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int,sr_return_quantity:int,sr_return_amt:decimal(7,2)>
+
+(39) ColumnarToRow [codegen id : 12]
+Input [5]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42, sr_returned_date_sk#43]
+
+(40) Filter [codegen id : 12]
+Input [5]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42, sr_returned_date_sk#43]
+Condition : (isnotnull(sr_ticket_number#40) AND isnotnull(sr_item_sk#39))
+
+(41) Project [codegen id : 12]
+Output [4]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42]
+Input [5]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42, sr_returned_date_sk#43]
+
+(42) Exchange
+Input [4]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42]
+Arguments: hashpartitioning(sr_ticket_number#40, sr_item_sk#39, 5), ENSURE_REQUIREMENTS, [id=#44]
+
+(43) Sort [codegen id : 13]
+Input [4]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42]
+Arguments: [sr_ticket_number#40 ASC NULLS FIRST, sr_item_sk#39 ASC NULLS FIRST], false, 0
+
+(44) SortMergeJoin [codegen id : 14]
+Left keys [2]: [ss_ticket_number#27, ss_item_sk#26]
+Right keys [2]: [sr_ticket_number#40, sr_item_sk#39]
+Join condition: None
+
+(45) Project [codegen id : 14]
+Output [7]: [d_year#37, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35, (ss_quantity#28 - coalesce(sr_return_quantity#41, 0)) AS sales_cnt#45, CheckOverflow((promote_precision(cast(ss_ext_sales_price#29 as decimal(8,2))) - promote_precision(cast(coalesce(sr_return_amt#42, 0.00) as decimal(8,2)))), DecimalType(8,2), true) AS sales_amt#46]
+Input [13]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35, d_year#37, sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42]
+
+(46) Scan parquet default.web_sales
+Output [5]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, ws_sold_date_sk#51]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#51), dynamicpruningexpression(ws_sold_date_sk#51 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_order_number:int,ws_quantity:int,ws_ext_sales_price:decimal(7,2)>
+
+(47) ColumnarToRow [codegen id : 17]
+Input [5]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, ws_sold_date_sk#51]
+
+(48) Filter [codegen id : 17]
+Input [5]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, ws_sold_date_sk#51]
+Condition : isnotnull(ws_item_sk#47)
+
+(49) ReusedExchange [Reuses operator id: 8]
+Output [5]: [i_item_sk#52, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56]
+
+(50) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_item_sk#47]
+Right keys [1]: [i_item_sk#52]
+Join condition: None
+
+(51) Project [codegen id : 17]
+Output [9]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, ws_sold_date_sk#51, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56]
+Input [10]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, ws_sold_date_sk#51, i_item_sk#52, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56]
+
+(52) ReusedExchange [Reuses operator id: 14]
+Output [2]: [d_date_sk#57, d_year#58]
+
+(53) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_sold_date_sk#51]
+Right keys [1]: [d_date_sk#57]
+Join condition: None
+
+(54) Project [codegen id : 17]
+Output [9]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56, d_year#58]
+Input [11]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, ws_sold_date_sk#51, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56, d_date_sk#57, d_year#58]
+
+(55) Exchange
+Input [9]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56, d_year#58]
+Arguments: hashpartitioning(ws_order_number#48, ws_item_sk#47, 5), ENSURE_REQUIREMENTS, [id=#59]
+
+(56) Sort [codegen id : 18]
+Input [9]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56, d_year#58]
+Arguments: [ws_order_number#48 ASC NULLS FIRST, ws_item_sk#47 ASC NULLS FIRST], false, 0
+
+(57) Scan parquet default.web_returns
+Output [5]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63, wr_returned_date_sk#64]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_returns]
+PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)]
+ReadSchema: struct<wr_item_sk:int,wr_order_number:int,wr_return_quantity:int,wr_return_amt:decimal(7,2)>
+
+(58) ColumnarToRow [codegen id : 19]
+Input [5]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63, wr_returned_date_sk#64]
+
+(59) Filter [codegen id : 19]
+Input [5]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63, wr_returned_date_sk#64]
+Condition : (isnotnull(wr_order_number#61) AND isnotnull(wr_item_sk#60))
+
+(60) Project [codegen id : 19]
+Output [4]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63]
+Input [5]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63, wr_returned_date_sk#64]
+
+(61) Exchange
+Input [4]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63]
+Arguments: hashpartitioning(wr_order_number#61, wr_item_sk#60, 5), ENSURE_REQUIREMENTS, [id=#65]
+
+(62) Sort [codegen id : 20]
+Input [4]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63]
+Arguments: [wr_order_number#61 ASC NULLS FIRST, wr_item_sk#60 ASC NULLS FIRST], false, 0
+
+(63) SortMergeJoin [codegen id : 21]
+Left keys [2]: [ws_order_number#48, ws_item_sk#47]
+Right keys [2]: [wr_order_number#61, wr_item_sk#60]
+Join condition: None
+
+(64) Project [codegen id : 21]
+Output [7]: [d_year#58, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56, (ws_quantity#49 - coalesce(wr_return_quantity#62, 0)) AS sales_cnt#66, CheckOverflow((promote_precision(cast(ws_ext_sales_price#50 as decimal(8,2))) - promote_precision(cast(coalesce(wr_return_amt#63, 0.00) as decimal(8,2)))), DecimalType(8,2), true) AS sales_amt#67]
+Input [13]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56, d_year#58, wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63]
+
+(65) Union
+
+(66) HashAggregate [codegen id : 22]
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+Keys [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+Functions: []
+Aggregate Attributes: []
+Results [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+
+(67) Exchange
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+Arguments: hashpartitioning(d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25, 5), ENSURE_REQUIREMENTS, [id=#68]
+
+(68) HashAggregate [codegen id : 23]
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+Keys [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+Functions: []
+Aggregate Attributes: []
+Results [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+
+(69) HashAggregate [codegen id : 23]
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+Keys [5]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+Functions [2]: [partial_sum(sales_cnt#24), partial_sum(UnscaledValue(sales_amt#25))]
+Aggregate Attributes [2]: [sum#69, sum#70]
+Results [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#71, sum#72]
+
+(70) Exchange
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#71, sum#72]
+Arguments: hashpartitioning(d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [id=#73]
+
+(71) HashAggregate [codegen id : 24]
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#71, sum#72]
+Keys [5]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+Functions [2]: [sum(sales_cnt#24), sum(UnscaledValue(sales_amt#25))]
+Aggregate Attributes [2]: [sum(sales_cnt#24)#74, sum(UnscaledValue(sales_amt#25))#75]
+Results [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum(sales_cnt#24)#74 AS sales_cnt#76, MakeDecimal(sum(UnscaledValue(sales_amt#25))#75,18,2) AS sales_amt#77]
+
+(72) Exchange
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#76, sales_amt#77]
+Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [id=#78]
+
+(73) Sort [codegen id : 25]
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#76, sales_amt#77]
+Arguments: [i_brand_id#8 ASC NULLS FIRST, i_class_id#9 ASC NULLS FIRST, i_category_id#10 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST], false, 0
+
+(74) Scan parquet default.catalog_sales
+Output [5]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, cs_sold_date_sk#83]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#83), dynamicpruningexpression(cs_sold_date_sk#83 IN dynamicpruning#84)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int,cs_order_number:int,cs_quantity:int,cs_ext_sales_price:decimal(7,2)>
+
+(75) ColumnarToRow [codegen id : 28]
+Input [5]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, cs_sold_date_sk#83]
+
+(76) Filter [codegen id : 28]
+Input [5]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, cs_sold_date_sk#83]
+Condition : isnotnull(cs_item_sk#79)
+
+(77) ReusedExchange [Reuses operator id: 8]
+Output [5]: [i_item_sk#85, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89]
+
+(78) BroadcastHashJoin [codegen id : 28]
+Left keys [1]: [cs_item_sk#79]
+Right keys [1]: [i_item_sk#85]
+Join condition: None
+
+(79) Project [codegen id : 28]
+Output [9]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, cs_sold_date_sk#83, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89]
+Input [10]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, cs_sold_date_sk#83, i_item_sk#85, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89]
+
+(80) Scan parquet default.date_dim
+Output [2]: [d_date_sk#90, d_year#91]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(81) ColumnarToRow [codegen id : 27]
+Input [2]: [d_date_sk#90, d_year#91]
+
+(82) Filter [codegen id : 27]
+Input [2]: [d_date_sk#90, d_year#91]
+Condition : ((isnotnull(d_year#91) AND (d_year#91 = 2001)) AND isnotnull(d_date_sk#90))
+
+(83) BroadcastExchange
+Input [2]: [d_date_sk#90, d_year#91]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#92]
+
+(84) BroadcastHashJoin [codegen id : 28]
+Left keys [1]: [cs_sold_date_sk#83]
+Right keys [1]: [d_date_sk#90]
+Join condition: None
+
+(85) Project [codegen id : 28]
+Output [9]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, d_year#91]
+Input [11]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, cs_sold_date_sk#83, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, d_date_sk#90, d_year#91]
+
+(86) Exchange
+Input [9]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, d_year#91]
+Arguments: hashpartitioning(cs_order_number#80, cs_item_sk#79, 5), ENSURE_REQUIREMENTS, [id=#93]
+
+(87) Sort [codegen id : 29]
+Input [9]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, d_year#91]
+Arguments: [cs_order_number#80 ASC NULLS FIRST, cs_item_sk#79 ASC NULLS FIRST], false, 0
+
+(88) ReusedExchange [Reuses operator id: 23]
+Output [4]: [cr_item_sk#94, cr_order_number#95, cr_return_quantity#96, cr_return_amount#97]
+
+(89) Sort [codegen id : 31]
+Input [4]: [cr_item_sk#94, cr_order_number#95, cr_return_quantity#96, cr_return_amount#97]
+Arguments: [cr_order_number#95 ASC NULLS FIRST, cr_item_sk#94 ASC NULLS FIRST], false, 0
+
+(90) SortMergeJoin [codegen id : 32]
+Left keys [2]: [cs_order_number#80, cs_item_sk#79]
+Right keys [2]: [cr_order_number#95, cr_item_sk#94]
+Join condition: None
+
+(91) Project [codegen id : 32]
+Output [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, (cs_quantity#81 - coalesce(cr_return_quantity#96, 0)) AS sales_cnt#24, CheckOverflow((promote_precision(cast(cs_ext_sales_price#82 as decimal(8,2))) - promote_precision(cast(coalesce(cr_return_amount#97, 0.00) as decimal(8,2)))), DecimalType(8,2), true) AS sales_amt#25]
+Input [13]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, d_year#91, cr_item_sk#94, cr_order_number#95, cr_return_quantity#96, cr_return_amount#97]
+
+(92) Scan parquet default.store_sales
+Output [5]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, ss_sold_date_sk#102]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#102), dynamicpruningexpression(ss_sold_date_sk#102 IN dynamicpruning#84)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ticket_number:int,ss_quantity:int,ss_ext_sales_price:decimal(7,2)>
+
+(93) ColumnarToRow [codegen id : 35]
+Input [5]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, ss_sold_date_sk#102]
+
+(94) Filter [codegen id : 35]
+Input [5]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, ss_sold_date_sk#102]
+Condition : isnotnull(ss_item_sk#98)
+
+(95) ReusedExchange [Reuses operator id: 8]
+Output [5]: [i_item_sk#103, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107]
+
+(96) BroadcastHashJoin [codegen id : 35]
+Left keys [1]: [ss_item_sk#98]
+Right keys [1]: [i_item_sk#103]
+Join condition: None
+
+(97) Project [codegen id : 35]
+Output [9]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, ss_sold_date_sk#102, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107]
+Input [10]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, ss_sold_date_sk#102, i_item_sk#103, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107]
+
+(98) ReusedExchange [Reuses operator id: 83]
+Output [2]: [d_date_sk#108, d_year#109]
+
+(99) BroadcastHashJoin [codegen id : 35]
+Left keys [1]: [ss_sold_date_sk#102]
+Right keys [1]: [d_date_sk#108]
+Join condition: None
+
+(100) Project [codegen id : 35]
+Output [9]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107, d_year#109]
+Input [11]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, ss_sold_date_sk#102, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107, d_date_sk#108, d_year#109]
+
+(101) Exchange
+Input [9]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107, d_year#109]
+Arguments: hashpartitioning(ss_ticket_number#99, ss_item_sk#98, 5), ENSURE_REQUIREMENTS, [id=#110]
+
+(102) Sort [codegen id : 36]
+Input [9]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107, d_year#109]
+Arguments: [ss_ticket_number#99 ASC NULLS FIRST, ss_item_sk#98 ASC NULLS FIRST], false, 0
+
+(103) ReusedExchange [Reuses operator id: 42]
+Output [4]: [sr_item_sk#111, sr_ticket_number#112, sr_return_quantity#113, sr_return_amt#114]
+
+(104) Sort [codegen id : 38]
+Input [4]: [sr_item_sk#111, sr_ticket_number#112, sr_return_quantity#113, sr_return_amt#114]
+Arguments: [sr_ticket_number#112 ASC NULLS FIRST, sr_item_sk#111 ASC NULLS FIRST], false, 0
+
+(105) SortMergeJoin [codegen id : 39]
+Left keys [2]: [ss_ticket_number#99, ss_item_sk#98]
+Right keys [2]: [sr_ticket_number#112, sr_item_sk#111]
+Join condition: None
+
+(106) Project [codegen id : 39]
+Output [7]: [d_year#109, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107, (ss_quantity#100 - coalesce(sr_return_quantity#113, 0)) AS sales_cnt#45, CheckOverflow((promote_precision(cast(ss_ext_sales_price#101 as decimal(8,2))) - promote_precision(cast(coalesce(sr_return_amt#114, 0.00) as decimal(8,2)))), DecimalType(8,2), true) AS sales_amt#46]
+Input [13]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107, d_year#109, sr_item_sk#111, sr_ticket_number#112, sr_return_quantity#113, sr_return_amt#114]
+
+(107) Scan parquet default.web_sales
+Output [5]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, ws_sold_date_sk#119]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#119), dynamicpruningexpression(ws_sold_date_sk#119 IN dynamicpruning#84)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_order_number:int,ws_quantity:int,ws_ext_sales_price:decimal(7,2)>
+
+(108) ColumnarToRow [codegen id : 42]
+Input [5]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, ws_sold_date_sk#119]
+
+(109) Filter [codegen id : 42]
+Input [5]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, ws_sold_date_sk#119]
+Condition : isnotnull(ws_item_sk#115)
+
+(110) ReusedExchange [Reuses operator id: 8]
+Output [5]: [i_item_sk#120, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124]
+
+(111) BroadcastHashJoin [codegen id : 42]
+Left keys [1]: [ws_item_sk#115]
+Right keys [1]: [i_item_sk#120]
+Join condition: None
+
+(112) Project [codegen id : 42]
+Output [9]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, ws_sold_date_sk#119, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124]
+Input [10]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, ws_sold_date_sk#119, i_item_sk#120, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124]
+
+(113) ReusedExchange [Reuses operator id: 83]
+Output [2]: [d_date_sk#125, d_year#126]
+
+(114) BroadcastHashJoin [codegen id : 42]
+Left keys [1]: [ws_sold_date_sk#119]
+Right keys [1]: [d_date_sk#125]
+Join condition: None
+
+(115) Project [codegen id : 42]
+Output [9]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124, d_year#126]
+Input [11]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, ws_sold_date_sk#119, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124, d_date_sk#125, d_year#126]
+
+(116) Exchange
+Input [9]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124, d_year#126]
+Arguments: hashpartitioning(ws_order_number#116, ws_item_sk#115, 5), ENSURE_REQUIREMENTS, [id=#127]
+
+(117) Sort [codegen id : 43]
+Input [9]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124, d_year#126]
+Arguments: [ws_order_number#116 ASC NULLS FIRST, ws_item_sk#115 ASC NULLS FIRST], false, 0
+
+(118) ReusedExchange [Reuses operator id: 61]
+Output [4]: [wr_item_sk#128, wr_order_number#129, wr_return_quantity#130, wr_return_amt#131]
+
+(119) Sort [codegen id : 45]
+Input [4]: [wr_item_sk#128, wr_order_number#129, wr_return_quantity#130, wr_return_amt#131]
+Arguments: [wr_order_number#129 ASC NULLS FIRST, wr_item_sk#128 ASC NULLS FIRST], false, 0
+
+(120) SortMergeJoin [codegen id : 46]
+Left keys [2]: [ws_order_number#116, ws_item_sk#115]
+Right keys [2]: [wr_order_number#129, wr_item_sk#128]
+Join condition: None
+
+(121) Project [codegen id : 46]
+Output [7]: [d_year#126, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124, (ws_quantity#117 - coalesce(wr_return_quantity#130, 0)) AS sales_cnt#66, CheckOverflow((promote_precision(cast(ws_ext_sales_price#118 as decimal(8,2))) - promote_precision(cast(coalesce(wr_return_amt#131, 0.00) as decimal(8,2)))), DecimalType(8,2), true) AS sales_amt#67]
+Input [13]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124, d_year#126, wr_item_sk#128, wr_order_number#129, wr_return_quantity#130, wr_return_amt#131]
+
+(122) Union
+
+(123) HashAggregate [codegen id : 47]
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+Keys [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+Functions: []
+Aggregate Attributes: []
+Results [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+
+(124) Exchange
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+Arguments: hashpartitioning(d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25, 5), ENSURE_REQUIREMENTS, [id=#132]
+
+(125) HashAggregate [codegen id : 48]
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+Keys [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+Functions: []
+Aggregate Attributes: []
+Results [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+
+(126) HashAggregate [codegen id : 48]
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+Keys [5]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89]
+Functions [2]: [partial_sum(sales_cnt#24), partial_sum(UnscaledValue(sales_amt#25))]
+Aggregate Attributes [2]: [sum#133, sum#134]
+Results [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sum#135, sum#136]
+
+(127) Exchange
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sum#135, sum#136]
+Arguments: hashpartitioning(d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, 5), ENSURE_REQUIREMENTS, [id=#137]
+
+(128) HashAggregate [codegen id : 49]
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sum#135, sum#136]
+Keys [5]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89]
+Functions [2]: [sum(sales_cnt#24), sum(UnscaledValue(sales_amt#25))]
+Aggregate Attributes [2]: [sum(sales_cnt#24)#138, sum(UnscaledValue(sales_amt#25))#139]
+Results [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sum(sales_cnt#24)#138 AS sales_cnt#140, MakeDecimal(sum(UnscaledValue(sales_amt#25))#139,18,2) AS sales_amt#141]
+
+(129) Exchange
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#140, sales_amt#141]
+Arguments: hashpartitioning(i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, 5), ENSURE_REQUIREMENTS, [id=#142]
+
+(130) Sort [codegen id : 50]
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#140, sales_amt#141]
+Arguments: [i_brand_id#86 ASC NULLS FIRST, i_class_id#87 ASC NULLS FIRST, i_category_id#88 ASC NULLS FIRST, i_manufact_id#89 ASC NULLS FIRST], false, 0
+
+(131) SortMergeJoin [codegen id : 51]
+Left keys [4]: [i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+Right keys [4]: [i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89]
+Join condition: (CheckOverflow((promote_precision(cast(sales_cnt#76 as decimal(17,2))) / promote_precision(cast(sales_cnt#140 as decimal(17,2)))), DecimalType(37,20), true) < 0.90000000000000000000)
+
+(132) Project [codegen id : 51]
+Output [10]: [d_year#91 AS prev_year#143, d_year#15 AS year#144, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#140 AS prev_yr_cnt#145, sales_cnt#76 AS curr_yr_cnt#146, (sales_cnt#76 - sales_cnt#140) AS sales_cnt_diff#147, CheckOverflow((promote_precision(cast(sales_amt#77 as decimal(19,2))) - promote_precision(cast(sales_amt#141 as decimal(19,2)))), DecimalType(19,2), true) AS sales_amt_diff#148]
+Input [14]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#76, sales_amt#77, d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#140, sales_amt#141]
+
+(133) TakeOrderedAndProject
+Input [10]: [prev_year#143, year#144, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#145, curr_yr_cnt#146, sales_cnt_diff#147, sales_amt_diff#148]
+Arguments: 100, [sales_cnt_diff#147 ASC NULLS FIRST], [prev_year#143, year#144, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#145, curr_yr_cnt#146, sales_cnt_diff#147, sales_amt_diff#148]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (134)
+
+
+(134) ReusedExchange [Reuses operator id: 14]
+Output [2]: [d_date_sk#14, d_year#15]
+
+Subquery:2 Hosting operator id = 27 Hosting Expression = ss_sold_date_sk#30 IN dynamicpruning#6
+
+Subquery:3 Hosting operator id = 46 Hosting Expression = ws_sold_date_sk#51 IN dynamicpruning#6
+
+Subquery:4 Hosting operator id = 74 Hosting Expression = cs_sold_date_sk#83 IN dynamicpruning#84
+ReusedExchange (135)
+
+
+(135) ReusedExchange [Reuses operator id: 83]
+Output [2]: [d_date_sk#90, d_year#91]
+
+Subquery:5 Hosting operator id = 92 Hosting Expression = ss_sold_date_sk#102 IN dynamicpruning#84
+
+Subquery:6 Hosting operator id = 107 Hosting Expression = ws_sold_date_sk#119 IN dynamicpruning#84
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/simplified.txt
new file mode 100644
index 0000000000000..170dad0b5dadd
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/simplified.txt
@@ -0,0 +1,238 @@
+TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_amt_diff]
+  WholeStageCodegen (51)
+    Project [d_year,d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt,sales_amt,sales_amt]
+      SortMergeJoin [i_brand_id,i_class_id,i_category_id,i_manufact_id,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt]
+        InputAdapter
+          WholeStageCodegen (25)
+            Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id]
+              InputAdapter
+                Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1
+                  WholeStageCodegen (24)
+                    HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum]
+                      InputAdapter
+                        Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2
+                          WholeStageCodegen (23)
+                            HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum]
+                              HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt]
+                                InputAdapter
+                                  Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3
+                                    WholeStageCodegen (22)
+                                      HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt]
+                                        InputAdapter
+                                          Union
+                                            WholeStageCodegen (7)
+                                              Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount]
+                                                SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (4)
+                                                      Sort [cs_order_number,cs_item_sk]
+                                                        InputAdapter
+                                                          Exchange [cs_order_number,cs_item_sk] #4
+                                                            WholeStageCodegen (3)
+                                                              Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year]
+                                                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                  Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                    BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                                                      Filter [cs_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk]
+                                                                              SubqueryBroadcast [d_date_sk] #1
+                                                                                ReusedExchange [d_date_sk,d_year] #5
+                                                                      InputAdapter
+                                                                        BroadcastExchange #6
+                                                                          WholeStageCodegen (1)
+                                                                            Project [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                              Filter [i_category,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id]
+                                                                  InputAdapter
+                                                                    BroadcastExchange #5
+                                                                      WholeStageCodegen (2)
+                                                                        Filter [d_year,d_date_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.date_dim [d_date_sk,d_year]
+                                                  InputAdapter
+                                                    WholeStageCodegen (6)
+                                                      Sort [cr_order_number,cr_item_sk]
+                                                        InputAdapter
+                                                          Exchange [cr_order_number,cr_item_sk] #7
+                                                            WholeStageCodegen (5)
+                                                              Project [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount]
+                                                                Filter [cr_order_number,cr_item_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk]
+                                            WholeStageCodegen (14)
+                                              Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt]
+                                                SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (11)
+                                                      Sort [ss_ticket_number,ss_item_sk]
+                                                        InputAdapter
+                                                          Exchange [ss_ticket_number,ss_item_sk] #8
+                                                            WholeStageCodegen (10)
+                                                              Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year]
+                                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                  Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                                      Filter [ss_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk]
+                                                                              ReusedSubquery [d_date_sk] #1
+                                                                      InputAdapter
+                                                                        ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6
+                                                                  InputAdapter
+                                                                    ReusedExchange [d_date_sk,d_year] #5
+                                                  InputAdapter
+                                                    WholeStageCodegen (13)
+                                                      Sort [sr_ticket_number,sr_item_sk]
+                                                        InputAdapter
+                                                          Exchange [sr_ticket_number,sr_item_sk] #9
+                                                            WholeStageCodegen (12)
+                                                              Project [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt]
+                                                                Filter [sr_ticket_number,sr_item_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk]
+                                            WholeStageCodegen (21)
+                                              Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt]
+                                                SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (18)
+                                                      Sort [ws_order_number,ws_item_sk]
+                                                        InputAdapter
+                                                          Exchange [ws_order_number,ws_item_sk] #10
+                                                            WholeStageCodegen (17)
+                                                              Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year]
+                                                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                  Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                    BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                                                      Filter [ws_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk]
+                                                                              ReusedSubquery [d_date_sk] #1
+                                                                      InputAdapter
+                                                                        ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6
+                                                                  InputAdapter
+                                                                    ReusedExchange [d_date_sk,d_year] #5
+                                                  InputAdapter
+                                                    WholeStageCodegen (20)
+                                                      Sort [wr_order_number,wr_item_sk]
+                                                        InputAdapter
+                                                          Exchange [wr_order_number,wr_item_sk] #11
+                                                            WholeStageCodegen (19)
+                                                              Project [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt]
+                                                                Filter [wr_order_number,wr_item_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk]
+        InputAdapter
+          WholeStageCodegen (50)
+            Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id]
+              InputAdapter
+                Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12
+                  WholeStageCodegen (49)
+                    HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum]
+                      InputAdapter
+                        Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13
+                          WholeStageCodegen (48)
+                            HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum]
+                              HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt]
+                                InputAdapter
+                                  Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14
+                                    WholeStageCodegen (47)
+                                      HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt]
+                                        InputAdapter
+                                          Union
+                                            WholeStageCodegen (32)
+                                              Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount]
+                                                SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (29)
+                                                      Sort [cs_order_number,cs_item_sk]
+                                                        InputAdapter
+                                                          Exchange [cs_order_number,cs_item_sk] #15
+                                                            WholeStageCodegen (28)
+                                                              Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year]
+                                                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                  Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                    BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                                                      Filter [cs_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk]
+                                                                              SubqueryBroadcast [d_date_sk] #2
+                                                                                ReusedExchange [d_date_sk,d_year] #16
+                                                                      InputAdapter
+                                                                        ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6
+                                                                  InputAdapter
+                                                                    BroadcastExchange #16
+                                                                      WholeStageCodegen (27)
+                                                                        Filter [d_year,d_date_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.date_dim [d_date_sk,d_year]
+                                                  InputAdapter
+                                                    WholeStageCodegen (31)
+                                                      Sort [cr_order_number,cr_item_sk]
+                                                        InputAdapter
+                                                          ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7
+                                            WholeStageCodegen (39)
+                                              Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt]
+                                                SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (36)
+                                                      Sort [ss_ticket_number,ss_item_sk]
+                                                        InputAdapter
+                                                          Exchange [ss_ticket_number,ss_item_sk] #17
+                                                            WholeStageCodegen (35)
+                                                              Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year]
+                                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                  Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                                      Filter [ss_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk]
+                                                                              ReusedSubquery [d_date_sk] #2
+                                                                      InputAdapter
+                                                                        ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6
+                                                                  InputAdapter
+                                                                    ReusedExchange [d_date_sk,d_year] #16
+                                                  InputAdapter
+                                                    WholeStageCodegen (38)
+                                                      Sort [sr_ticket_number,sr_item_sk]
+                                                        InputAdapter
+                                                          ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9
+                                            WholeStageCodegen (46)
+                                              Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt]
+                                                SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (43)
+                                                      Sort [ws_order_number,ws_item_sk]
+                                                        InputAdapter
+                                                          Exchange [ws_order_number,ws_item_sk] #18
+                                                            WholeStageCodegen (42)
+                                                              Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year]
+                                                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                  Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                    BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                                                      Filter [ws_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk]
+                                                                              ReusedSubquery [d_date_sk] #2
+                                                                      InputAdapter
+                                                                        ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6
+                                                                  InputAdapter
+                                                                    ReusedExchange [d_date_sk,d_year] #16
+                                                  InputAdapter
+                                                    WholeStageCodegen (45)
+                                                      Sort [wr_order_number,wr_item_sk]
+                                                        InputAdapter
+                                                          ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.sf100/explain.txt
new file mode 100644
index 0000000000000..9284172139688
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.sf100/explain.txt
@@ -0,0 +1,248 @@
+== Physical Plan ==
+TakeOrderedAndProject (44)
++- * HashAggregate (43)
+   +- Exchange (42)
+      +- * HashAggregate (41)
+         +- Union (40)
+            :- * Project (15)
+            :  +- * BroadcastHashJoin Inner BuildRight (14)
+            :     :- * Project (9)
+            :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+            :     :     :- * Filter (3)
+            :     :     :  +- * ColumnarToRow (2)
+            :     :     :     +- Scan parquet default.store_sales (1)
+            :     :     +- BroadcastExchange (7)
+            :     :        +- * Filter (6)
+            :     :           +- * ColumnarToRow (5)
+            :     :              +- Scan parquet default.date_dim (4)
+            :     +- BroadcastExchange (13)
+            :        +- * Filter (12)
+            :           +- * ColumnarToRow (11)
+            :              +- Scan parquet default.item (10)
+            :- * Project (30)
+            :  +- * BroadcastHashJoin Inner BuildLeft (29)
+            :     :- BroadcastExchange (25)
+            :     :  +- * Project (24)
+            :     :     +- * BroadcastHashJoin Inner BuildLeft (23)
+            :     :        :- BroadcastExchange (19)
+            :     :        :  +- * Filter (18)
+            :     :        :     +- * ColumnarToRow (17)
+            :     :        :        +- Scan parquet default.web_sales (16)
+            :     :        +- * Filter (22)
+            :     :           +- * ColumnarToRow (21)
+            :     :              +- Scan parquet default.date_dim (20)
+            :     +- * Filter (28)
+            :        +- * ColumnarToRow (27)
+            :           +- Scan parquet default.item (26)
+            +- * Project (39)
+               +- * BroadcastHashJoin Inner BuildRight (38)
+                  :- * Project (36)
+                  :  +- * BroadcastHashJoin Inner BuildRight (35)
+                  :     :- * Filter (33)
+                  :     :  +- * ColumnarToRow (32)
+                  :     :     +- Scan parquet default.catalog_sales (31)
+                  :     +- ReusedExchange (34)
+                  +- ReusedExchange (37)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4)]
+PushedFilters: [IsNull(ss_store_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 3]
+Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4]
+Condition : (isnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_qoy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+Condition : isnotnull(d_date_sk#5)
+
+(7) BroadcastExchange
+Input [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#8]
+
+(8) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#5]
+Join condition: None
+
+(9) Project [codegen id : 3]
+Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, d_year#6, d_qoy#7]
+Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5, d_year#6, d_qoy#7]
+
+(10) Scan parquet default.item
+Output [2]: [i_item_sk#9, i_category#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_category:string>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [i_item_sk#9, i_category#10]
+
+(12) Filter [codegen id : 2]
+Input [2]: [i_item_sk#9, i_category#10]
+Condition : isnotnull(i_item_sk#9)
+
+(13) BroadcastExchange
+Input [2]: [i_item_sk#9, i_category#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(14) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#9]
+Join condition: None
+
+(15) Project [codegen id : 3]
+Output [6]: [store AS channel#12, ss_store_sk#2 AS col_name#13, d_year#6, d_qoy#7, i_category#10, ss_ext_sales_price#3 AS ext_sales_price#14]
+Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, d_year#6, d_qoy#7, i_item_sk#9, i_category#10]
+
+(16) Scan parquet default.web_sales
+Output [4]: [ws_item_sk#15, ws_ship_customer_sk#16, ws_ext_sales_price#17, ws_sold_date_sk#18]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#18)]
+PushedFilters: [IsNull(ws_ship_customer_sk), IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_ship_customer_sk:int,ws_ext_sales_price:decimal(7,2)>
+
+(17) ColumnarToRow [codegen id : 4]
+Input [4]: [ws_item_sk#15, ws_ship_customer_sk#16, ws_ext_sales_price#17, ws_sold_date_sk#18]
+
+(18) Filter [codegen id : 4]
+Input [4]: [ws_item_sk#15, ws_ship_customer_sk#16, ws_ext_sales_price#17, ws_sold_date_sk#18]
+Condition : (isnull(ws_ship_customer_sk#16) AND isnotnull(ws_item_sk#15))
+
+(19) BroadcastExchange
+Input [4]: [ws_item_sk#15, ws_ship_customer_sk#16, ws_ext_sales_price#17, ws_sold_date_sk#18]
+Arguments: HashedRelationBroadcastMode(List(cast(input[3, int, true] as bigint)),false), [id=#19]
+
+(20) Scan parquet default.date_dim
+Output [3]: [d_date_sk#20, d_year#21, d_qoy#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_qoy:int>
+
+(21) ColumnarToRow
+Input [3]: [d_date_sk#20, d_year#21, d_qoy#22]
+
+(22) Filter
+Input [3]: [d_date_sk#20, d_year#21, d_qoy#22]
+Condition : isnotnull(d_date_sk#20)
+
+(23) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ws_sold_date_sk#18]
+Right keys [1]: [d_date_sk#20]
+Join condition: None
+
+(24) Project [codegen id : 5]
+Output [5]: [ws_item_sk#15, ws_ship_customer_sk#16, ws_ext_sales_price#17, d_year#21, d_qoy#22]
+Input [7]: [ws_item_sk#15, ws_ship_customer_sk#16, ws_ext_sales_price#17, ws_sold_date_sk#18, d_date_sk#20, d_year#21, d_qoy#22]
+
+(25) BroadcastExchange
+Input [5]: [ws_item_sk#15, ws_ship_customer_sk#16, ws_ext_sales_price#17, d_year#21, d_qoy#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#23]
+
+(26) Scan parquet default.item
+Output [2]: [i_item_sk#24, i_category#25]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_category:string>
+
+(27) ColumnarToRow
+Input [2]: [i_item_sk#24, i_category#25]
+
+(28) Filter
+Input [2]: [i_item_sk#24, i_category#25]
+Condition : isnotnull(i_item_sk#24)
+
+(29) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ws_item_sk#15]
+Right keys [1]: [i_item_sk#24]
+Join condition: None
+
+(30) Project [codegen id : 6]
+Output [6]: [web AS channel#26, ws_ship_customer_sk#16 AS col_name#27, d_year#21, d_qoy#22, i_category#25, ws_ext_sales_price#17 AS ext_sales_price#28]
+Input [7]: [ws_item_sk#15, ws_ship_customer_sk#16, ws_ext_sales_price#17, d_year#21, d_qoy#22, i_item_sk#24, i_category#25]
+
+(31) Scan parquet default.catalog_sales
+Output [4]: [cs_ship_addr_sk#29, cs_item_sk#30, cs_ext_sales_price#31, cs_sold_date_sk#32]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#32)]
+PushedFilters: [IsNull(cs_ship_addr_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_ship_addr_sk:int,cs_item_sk:int,cs_ext_sales_price:decimal(7,2)>
+
+(32) ColumnarToRow [codegen id : 9]
+Input [4]: [cs_ship_addr_sk#29, cs_item_sk#30, cs_ext_sales_price#31, cs_sold_date_sk#32]
+
+(33) Filter [codegen id : 9]
+Input [4]: [cs_ship_addr_sk#29, cs_item_sk#30, cs_ext_sales_price#31, cs_sold_date_sk#32]
+Condition : (isnull(cs_ship_addr_sk#29) AND isnotnull(cs_item_sk#30))
+
+(34) ReusedExchange [Reuses operator id: 7]
+Output [3]: [d_date_sk#33, d_year#34, d_qoy#35]
+
+(35) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [cs_sold_date_sk#32]
+Right keys [1]: [d_date_sk#33]
+Join condition: None
+
+(36) Project [codegen id : 9]
+Output [5]: [cs_ship_addr_sk#29, cs_item_sk#30, cs_ext_sales_price#31, d_year#34, d_qoy#35]
+Input [7]: [cs_ship_addr_sk#29, cs_item_sk#30, cs_ext_sales_price#31, cs_sold_date_sk#32, d_date_sk#33, d_year#34, d_qoy#35]
+
+(37) ReusedExchange [Reuses operator id: 13]
+Output [2]: [i_item_sk#36, i_category#37]
+
+(38) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [cs_item_sk#30]
+Right keys [1]: [i_item_sk#36]
+Join condition: None
+
+(39) Project [codegen id : 9]
+Output [6]: [catalog AS channel#38, cs_ship_addr_sk#29 AS col_name#39, d_year#34, d_qoy#35, i_category#37, cs_ext_sales_price#31 AS ext_sales_price#40]
+Input [7]: [cs_ship_addr_sk#29, cs_item_sk#30, cs_ext_sales_price#31, d_year#34, d_qoy#35, i_item_sk#36, i_category#37]
+
+(40) Union
+
+(41) HashAggregate [codegen id : 10]
+Input [6]: [channel#12, col_name#13, d_year#6, d_qoy#7, i_category#10, ext_sales_price#14]
+Keys [5]: [channel#12, col_name#13, d_year#6, d_qoy#7, i_category#10]
+Functions [2]: [partial_count(1), partial_sum(UnscaledValue(ext_sales_price#14))]
+Aggregate Attributes [2]: [count#41, sum#42]
+Results [7]: [channel#12, col_name#13, d_year#6, d_qoy#7, i_category#10, count#43, sum#44]
+
+(42) Exchange
+Input [7]: [channel#12, col_name#13, d_year#6, d_qoy#7, i_category#10, count#43, sum#44]
+Arguments: hashpartitioning(channel#12, col_name#13, d_year#6, d_qoy#7, i_category#10, 5), ENSURE_REQUIREMENTS, [id=#45]
+
+(43) HashAggregate [codegen id : 11]
+Input [7]: [channel#12, col_name#13, d_year#6, d_qoy#7, i_category#10, count#43, sum#44]
+Keys [5]: [channel#12, col_name#13, d_year#6, d_qoy#7, i_category#10]
+Functions [2]: [count(1), sum(UnscaledValue(ext_sales_price#14))]
+Aggregate Attributes [2]: [count(1)#46, sum(UnscaledValue(ext_sales_price#14))#47]
+Results [7]: [channel#12, col_name#13, d_year#6, d_qoy#7, i_category#10, count(1)#46 AS sales_cnt#48, MakeDecimal(sum(UnscaledValue(ext_sales_price#14))#47,17,2) AS sales_amt#49]
+
+(44) TakeOrderedAndProject
+Input [7]: [channel#12, col_name#13, d_year#6, d_qoy#7, i_category#10, sales_cnt#48, sales_amt#49]
+Arguments: 100, [channel#12 ASC NULLS FIRST, col_name#13 ASC NULLS FIRST, d_year#6 ASC NULLS FIRST, d_qoy#7 ASC NULLS FIRST, i_category#10 ASC NULLS FIRST], [channel#12, col_name#13, d_year#6, d_qoy#7, i_category#10, sales_cnt#48, sales_amt#49]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.sf100/simplified.txt
new file mode 100644
index 0000000000000..c866897775ede
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.sf100/simplified.txt
@@ -0,0 +1,68 @@
+TakeOrderedAndProject [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_amt]
+  WholeStageCodegen (11)
+    HashAggregate [channel,col_name,d_year,d_qoy,i_category,count,sum] [count(1),sum(UnscaledValue(ext_sales_price)),sales_cnt,sales_amt,count,sum]
+      InputAdapter
+        Exchange [channel,col_name,d_year,d_qoy,i_category] #1
+          WholeStageCodegen (10)
+            HashAggregate [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] [count,sum,count,sum]
+              InputAdapter
+                Union
+                  WholeStageCodegen (3)
+                    Project [ss_store_sk,d_year,d_qoy,i_category,ss_ext_sales_price]
+                      BroadcastHashJoin [ss_item_sk,i_item_sk]
+                        Project [ss_item_sk,ss_store_sk,ss_ext_sales_price,d_year,d_qoy]
+                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                            Filter [ss_store_sk,ss_item_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk]
+                            InputAdapter
+                              BroadcastExchange #2
+                                WholeStageCodegen (1)
+                                  Filter [d_date_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.date_dim [d_date_sk,d_year,d_qoy]
+                        InputAdapter
+                          BroadcastExchange #3
+                            WholeStageCodegen (2)
+                              Filter [i_item_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.item [i_item_sk,i_category]
+                  WholeStageCodegen (6)
+                    Project [ws_ship_customer_sk,d_year,d_qoy,i_category,ws_ext_sales_price]
+                      BroadcastHashJoin [ws_item_sk,i_item_sk]
+                        InputAdapter
+                          BroadcastExchange #4
+                            WholeStageCodegen (5)
+                              Project [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,d_year,d_qoy]
+                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                  InputAdapter
+                                    BroadcastExchange #5
+                                      WholeStageCodegen (4)
+                                        Filter [ws_ship_customer_sk,ws_item_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.web_sales [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk]
+                                  Filter [d_date_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.date_dim [d_date_sk,d_year,d_qoy]
+                        Filter [i_item_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.item [i_item_sk,i_category]
+                  WholeStageCodegen (9)
+                    Project [cs_ship_addr_sk,d_year,d_qoy,i_category,cs_ext_sales_price]
+                      BroadcastHashJoin [cs_item_sk,i_item_sk]
+                        Project [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,d_year,d_qoy]
+                          BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                            Filter [cs_ship_addr_sk,cs_item_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.catalog_sales [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk]
+                            InputAdapter
+                              ReusedExchange [d_date_sk,d_year,d_qoy] #2
+                        InputAdapter
+                          ReusedExchange [i_item_sk,i_category] #3
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76/explain.txt
new file mode 100644
index 0000000000000..026c9396cd025
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76/explain.txt
@@ -0,0 +1,212 @@
+== Physical Plan ==
+TakeOrderedAndProject (38)
++- * HashAggregate (37)
+   +- Exchange (36)
+      +- * HashAggregate (35)
+         +- Union (34)
+            :- * Project (15)
+            :  +- * BroadcastHashJoin Inner BuildRight (14)
+            :     :- * Project (9)
+            :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+            :     :     :- * Filter (3)
+            :     :     :  +- * ColumnarToRow (2)
+            :     :     :     +- Scan parquet default.store_sales (1)
+            :     :     +- BroadcastExchange (7)
+            :     :        +- * Filter (6)
+            :     :           +- * ColumnarToRow (5)
+            :     :              +- Scan parquet default.item (4)
+            :     +- BroadcastExchange (13)
+            :        +- * Filter (12)
+            :           +- * ColumnarToRow (11)
+            :              +- Scan parquet default.date_dim (10)
+            :- * Project (24)
+            :  +- * BroadcastHashJoin Inner BuildRight (23)
+            :     :- * Project (21)
+            :     :  +- * BroadcastHashJoin Inner BuildRight (20)
+            :     :     :- * Filter (18)
+            :     :     :  +- * ColumnarToRow (17)
+            :     :     :     +- Scan parquet default.web_sales (16)
+            :     :     +- ReusedExchange (19)
+            :     +- ReusedExchange (22)
+            +- * Project (33)
+               +- * BroadcastHashJoin Inner BuildRight (32)
+                  :- * Project (30)
+                  :  +- * BroadcastHashJoin Inner BuildRight (29)
+                  :     :- * Filter (27)
+                  :     :  +- * ColumnarToRow (26)
+                  :     :     +- Scan parquet default.catalog_sales (25)
+                  :     +- ReusedExchange (28)
+                  +- ReusedExchange (31)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4)]
+PushedFilters: [IsNull(ss_store_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 3]
+Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4]
+Condition : (isnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1))
+
+(4) Scan parquet default.item
+Output [2]: [i_item_sk#5, i_category#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_category:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [i_item_sk#5, i_category#6]
+
+(6) Filter [codegen id : 1]
+Input [2]: [i_item_sk#5, i_category#6]
+Condition : isnotnull(i_item_sk#5)
+
+(7) BroadcastExchange
+Input [2]: [i_item_sk#5, i_category#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#7]
+
+(8) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#5]
+Join condition: None
+
+(9) Project [codegen id : 3]
+Output [4]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6]
+Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_item_sk#5, i_category#6]
+
+(10) Scan parquet default.date_dim
+Output [3]: [d_date_sk#8, d_year#9, d_qoy#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_qoy:int>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [3]: [d_date_sk#8, d_year#9, d_qoy#10]
+
+(12) Filter [codegen id : 2]
+Input [3]: [d_date_sk#8, d_year#9, d_qoy#10]
+Condition : isnotnull(d_date_sk#8)
+
+(13) BroadcastExchange
+Input [3]: [d_date_sk#8, d_year#9, d_qoy#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(14) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#8]
+Join condition: None
+
+(15) Project [codegen id : 3]
+Output [6]: [store AS channel#12, ss_store_sk#2 AS col_name#13, d_year#9, d_qoy#10, i_category#6, ss_ext_sales_price#3 AS ext_sales_price#14]
+Input [7]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6, d_date_sk#8, d_year#9, d_qoy#10]
+
+(16) Scan parquet default.web_sales
+Output [4]: [ws_item_sk#15, ws_ship_customer_sk#16, ws_ext_sales_price#17, ws_sold_date_sk#18]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#18)]
+PushedFilters: [IsNull(ws_ship_customer_sk), IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_ship_customer_sk:int,ws_ext_sales_price:decimal(7,2)>
+
+(17) ColumnarToRow [codegen id : 6]
+Input [4]: [ws_item_sk#15, ws_ship_customer_sk#16, ws_ext_sales_price#17, ws_sold_date_sk#18]
+
+(18) Filter [codegen id : 6]
+Input [4]: [ws_item_sk#15, ws_ship_customer_sk#16, ws_ext_sales_price#17, ws_sold_date_sk#18]
+Condition : (isnull(ws_ship_customer_sk#16) AND isnotnull(ws_item_sk#15))
+
+(19) ReusedExchange [Reuses operator id: 7]
+Output [2]: [i_item_sk#19, i_category#20]
+
+(20) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ws_item_sk#15]
+Right keys [1]: [i_item_sk#19]
+Join condition: None
+
+(21) Project [codegen id : 6]
+Output [4]: [ws_ship_customer_sk#16, ws_ext_sales_price#17, ws_sold_date_sk#18, i_category#20]
+Input [6]: [ws_item_sk#15, ws_ship_customer_sk#16, ws_ext_sales_price#17, ws_sold_date_sk#18, i_item_sk#19, i_category#20]
+
+(22) ReusedExchange [Reuses operator id: 13]
+Output [3]: [d_date_sk#21, d_year#22, d_qoy#23]
+
+(23) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ws_sold_date_sk#18]
+Right keys [1]: [d_date_sk#21]
+Join condition: None
+
+(24) Project [codegen id : 6]
+Output [6]: [web AS channel#24, ws_ship_customer_sk#16 AS col_name#25, d_year#22, d_qoy#23, i_category#20, ws_ext_sales_price#17 AS ext_sales_price#26]
+Input [7]: [ws_ship_customer_sk#16, ws_ext_sales_price#17, ws_sold_date_sk#18, i_category#20, d_date_sk#21, d_year#22, d_qoy#23]
+
+(25) Scan parquet default.catalog_sales
+Output [4]: [cs_ship_addr_sk#27, cs_item_sk#28, cs_ext_sales_price#29, cs_sold_date_sk#30]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#30)]
+PushedFilters: [IsNull(cs_ship_addr_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_ship_addr_sk:int,cs_item_sk:int,cs_ext_sales_price:decimal(7,2)>
+
+(26) ColumnarToRow [codegen id : 9]
+Input [4]: [cs_ship_addr_sk#27, cs_item_sk#28, cs_ext_sales_price#29, cs_sold_date_sk#30]
+
+(27) Filter [codegen id : 9]
+Input [4]: [cs_ship_addr_sk#27, cs_item_sk#28, cs_ext_sales_price#29, cs_sold_date_sk#30]
+Condition : (isnull(cs_ship_addr_sk#27) AND isnotnull(cs_item_sk#28))
+
+(28) ReusedExchange [Reuses operator id: 7]
+Output [2]: [i_item_sk#31, i_category#32]
+
+(29) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [cs_item_sk#28]
+Right keys [1]: [i_item_sk#31]
+Join condition: None
+
+(30) Project [codegen id : 9]
+Output [4]: [cs_ship_addr_sk#27, cs_ext_sales_price#29, cs_sold_date_sk#30, i_category#32]
+Input [6]: [cs_ship_addr_sk#27, cs_item_sk#28, cs_ext_sales_price#29, cs_sold_date_sk#30, i_item_sk#31, i_category#32]
+
+(31) ReusedExchange [Reuses operator id: 13]
+Output [3]: [d_date_sk#33, d_year#34, d_qoy#35]
+
+(32) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [cs_sold_date_sk#30]
+Right keys [1]: [d_date_sk#33]
+Join condition: None
+
+(33) Project [codegen id : 9]
+Output [6]: [catalog AS channel#36, cs_ship_addr_sk#27 AS col_name#37, d_year#34, d_qoy#35, i_category#32, cs_ext_sales_price#29 AS ext_sales_price#38]
+Input [7]: [cs_ship_addr_sk#27, cs_ext_sales_price#29, cs_sold_date_sk#30, i_category#32, d_date_sk#33, d_year#34, d_qoy#35]
+
+(34) Union
+
+(35) HashAggregate [codegen id : 10]
+Input [6]: [channel#12, col_name#13, d_year#9, d_qoy#10, i_category#6, ext_sales_price#14]
+Keys [5]: [channel#12, col_name#13, d_year#9, d_qoy#10, i_category#6]
+Functions [2]: [partial_count(1), partial_sum(UnscaledValue(ext_sales_price#14))]
+Aggregate Attributes [2]: [count#39, sum#40]
+Results [7]: [channel#12, col_name#13, d_year#9, d_qoy#10, i_category#6, count#41, sum#42]
+
+(36) Exchange
+Input [7]: [channel#12, col_name#13, d_year#9, d_qoy#10, i_category#6, count#41, sum#42]
+Arguments: hashpartitioning(channel#12, col_name#13, d_year#9, d_qoy#10, i_category#6, 5), ENSURE_REQUIREMENTS, [id=#43]
+
+(37) HashAggregate [codegen id : 11]
+Input [7]: [channel#12, col_name#13, d_year#9, d_qoy#10, i_category#6, count#41, sum#42]
+Keys [5]: [channel#12, col_name#13, d_year#9, d_qoy#10, i_category#6]
+Functions [2]: [count(1), sum(UnscaledValue(ext_sales_price#14))]
+Aggregate Attributes [2]: [count(1)#44, sum(UnscaledValue(ext_sales_price#14))#45]
+Results [7]: [channel#12, col_name#13, d_year#9, d_qoy#10, i_category#6, count(1)#44 AS sales_cnt#46, MakeDecimal(sum(UnscaledValue(ext_sales_price#14))#45,17,2) AS sales_amt#47]
+
+(38) TakeOrderedAndProject
+Input [7]: [channel#12, col_name#13, d_year#9, d_qoy#10, i_category#6, sales_cnt#46, sales_amt#47]
+Arguments: 100, [channel#12 ASC NULLS FIRST, col_name#13 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_qoy#10 ASC NULLS FIRST, i_category#6 ASC NULLS FIRST], [channel#12, col_name#13, d_year#9, d_qoy#10, i_category#6, sales_cnt#46, sales_amt#47]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76/simplified.txt
new file mode 100644
index 0000000000000..15a607c70b63f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76/simplified.txt
@@ -0,0 +1,58 @@
+TakeOrderedAndProject [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_amt]
+  WholeStageCodegen (11)
+    HashAggregate [channel,col_name,d_year,d_qoy,i_category,count,sum] [count(1),sum(UnscaledValue(ext_sales_price)),sales_cnt,sales_amt,count,sum]
+      InputAdapter
+        Exchange [channel,col_name,d_year,d_qoy,i_category] #1
+          WholeStageCodegen (10)
+            HashAggregate [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] [count,sum,count,sum]
+              InputAdapter
+                Union
+                  WholeStageCodegen (3)
+                    Project [ss_store_sk,d_year,d_qoy,i_category,ss_ext_sales_price]
+                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                        Project [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category]
+                          BroadcastHashJoin [ss_item_sk,i_item_sk]
+                            Filter [ss_store_sk,ss_item_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk]
+                            InputAdapter
+                              BroadcastExchange #2
+                                WholeStageCodegen (1)
+                                  Filter [i_item_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.item [i_item_sk,i_category]
+                        InputAdapter
+                          BroadcastExchange #3
+                            WholeStageCodegen (2)
+                              Filter [d_date_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.date_dim [d_date_sk,d_year,d_qoy]
+                  WholeStageCodegen (6)
+                    Project [ws_ship_customer_sk,d_year,d_qoy,i_category,ws_ext_sales_price]
+                      BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                        Project [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category]
+                          BroadcastHashJoin [ws_item_sk,i_item_sk]
+                            Filter [ws_ship_customer_sk,ws_item_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.web_sales [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk]
+                            InputAdapter
+                              ReusedExchange [i_item_sk,i_category] #2
+                        InputAdapter
+                          ReusedExchange [d_date_sk,d_year,d_qoy] #3
+                  WholeStageCodegen (9)
+                    Project [cs_ship_addr_sk,d_year,d_qoy,i_category,cs_ext_sales_price]
+                      BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                        Project [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category]
+                          BroadcastHashJoin [cs_item_sk,i_item_sk]
+                            Filter [cs_ship_addr_sk,cs_item_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.catalog_sales [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk]
+                            InputAdapter
+                              ReusedExchange [i_item_sk,i_category] #2
+                        InputAdapter
+                          ReusedExchange [d_date_sk,d_year,d_qoy] #3
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.sf100/explain.txt
new file mode 100644
index 0000000000000..34d2ec82f500d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.sf100/explain.txt
@@ -0,0 +1,534 @@
+== Physical Plan ==
+TakeOrderedAndProject (89)
++- * HashAggregate (88)
+   +- Exchange (87)
+      +- * HashAggregate (86)
+         +- * Expand (85)
+            +- Union (84)
+               :- * Project (34)
+               :  +- * BroadcastHashJoin LeftOuter BuildRight (33)
+               :     :- * HashAggregate (19)
+               :     :  +- Exchange (18)
+               :     :     +- * HashAggregate (17)
+               :     :        +- * Project (16)
+               :     :           +- * BroadcastHashJoin Inner BuildRight (15)
+               :     :              :- * Project (10)
+               :     :              :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :              :     :- * Filter (3)
+               :     :              :     :  +- * ColumnarToRow (2)
+               :     :              :     :     +- Scan parquet default.store_sales (1)
+               :     :              :     +- BroadcastExchange (8)
+               :     :              :        +- * Project (7)
+               :     :              :           +- * Filter (6)
+               :     :              :              +- * ColumnarToRow (5)
+               :     :              :                 +- Scan parquet default.date_dim (4)
+               :     :              +- BroadcastExchange (14)
+               :     :                 +- * Filter (13)
+               :     :                    +- * ColumnarToRow (12)
+               :     :                       +- Scan parquet default.store (11)
+               :     +- BroadcastExchange (32)
+               :        +- * HashAggregate (31)
+               :           +- Exchange (30)
+               :              +- * HashAggregate (29)
+               :                 +- * Project (28)
+               :                    +- * BroadcastHashJoin Inner BuildRight (27)
+               :                       :- * Project (25)
+               :                       :  +- * BroadcastHashJoin Inner BuildRight (24)
+               :                       :     :- * Filter (22)
+               :                       :     :  +- * ColumnarToRow (21)
+               :                       :     :     +- Scan parquet default.store_returns (20)
+               :                       :     +- ReusedExchange (23)
+               :                       +- ReusedExchange (26)
+               :- * Project (53)
+               :  +- * BroadcastNestedLoopJoin Inner BuildRight (52)
+               :     :- * HashAggregate (42)
+               :     :  +- Exchange (41)
+               :     :     +- * HashAggregate (40)
+               :     :        +- * Project (39)
+               :     :           +- * BroadcastHashJoin Inner BuildRight (38)
+               :     :              :- * ColumnarToRow (36)
+               :     :              :  +- Scan parquet default.catalog_sales (35)
+               :     :              +- ReusedExchange (37)
+               :     +- BroadcastExchange (51)
+               :        +- * HashAggregate (50)
+               :           +- Exchange (49)
+               :              +- * HashAggregate (48)
+               :                 +- * Project (47)
+               :                    +- * BroadcastHashJoin Inner BuildRight (46)
+               :                       :- * ColumnarToRow (44)
+               :                       :  +- Scan parquet default.catalog_returns (43)
+               :                       +- ReusedExchange (45)
+               +- * Project (83)
+                  +- * BroadcastHashJoin LeftOuter BuildRight (82)
+                     :- * HashAggregate (68)
+                     :  +- Exchange (67)
+                     :     +- * HashAggregate (66)
+                     :        +- * Project (65)
+                     :           +- * BroadcastHashJoin Inner BuildRight (64)
+                     :              :- * Project (59)
+                     :              :  +- * BroadcastHashJoin Inner BuildRight (58)
+                     :              :     :- * Filter (56)
+                     :              :     :  +- * ColumnarToRow (55)
+                     :              :     :     +- Scan parquet default.web_sales (54)
+                     :              :     +- ReusedExchange (57)
+                     :              +- BroadcastExchange (63)
+                     :                 +- * Filter (62)
+                     :                    +- * ColumnarToRow (61)
+                     :                       +- Scan parquet default.web_page (60)
+                     +- BroadcastExchange (81)
+                        +- * HashAggregate (80)
+                           +- Exchange (79)
+                              +- * HashAggregate (78)
+                                 +- * Project (77)
+                                    +- * BroadcastHashJoin Inner BuildRight (76)
+                                       :- * Project (74)
+                                       :  +- * BroadcastHashJoin Inner BuildRight (73)
+                                       :     :- * Filter (71)
+                                       :     :  +- * ColumnarToRow (70)
+                                       :     :     +- Scan parquet default.web_returns (69)
+                                       :     +- ReusedExchange (72)
+                                       +- ReusedExchange (75)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_ext_sales_price:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 3]
+Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+Condition : isnotnull(ss_store_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#6, d_date#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-03), LessThanOrEqual(d_date,2000-09-02), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#6, d_date#7]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#6, d_date#7]
+Condition : (((isnotnull(d_date#7) AND (d_date#7 >= 2000-08-03)) AND (d_date#7 <= 2000-09-02)) AND isnotnull(d_date_sk#6))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#6]
+Input [2]: [d_date_sk#6, d_date#7]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#8]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3]
+Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#6]
+
+(11) Scan parquet default.store
+Output [1]: [s_store_sk#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [1]: [s_store_sk#9]
+
+(13) Filter [codegen id : 2]
+Input [1]: [s_store_sk#9]
+Condition : isnotnull(s_store_sk#9)
+
+(14) BroadcastExchange
+Input [1]: [s_store_sk#9]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#10]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_store_sk#1]
+Right keys [1]: [s_store_sk#9]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#9]
+Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#9]
+
+(17) HashAggregate [codegen id : 3]
+Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#9]
+Keys [1]: [s_store_sk#9]
+Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))]
+Aggregate Attributes [2]: [sum#11, sum#12]
+Results [3]: [s_store_sk#9, sum#13, sum#14]
+
+(18) Exchange
+Input [3]: [s_store_sk#9, sum#13, sum#14]
+Arguments: hashpartitioning(s_store_sk#9, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(19) HashAggregate [codegen id : 8]
+Input [3]: [s_store_sk#9, sum#13, sum#14]
+Keys [1]: [s_store_sk#9]
+Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ss_ext_sales_price#2))#16, sum(UnscaledValue(ss_net_profit#3))#17]
+Results [3]: [s_store_sk#9, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#16,17,2) AS sales#18, MakeDecimal(sum(UnscaledValue(ss_net_profit#3))#17,17,2) AS profit#19]
+
+(20) Scan parquet default.store_returns
+Output [4]: [sr_store_sk#20, sr_return_amt#21, sr_net_loss#22, sr_returned_date_sk#23]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(sr_returned_date_sk#23), dynamicpruningexpression(sr_returned_date_sk#23 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(sr_store_sk)]
+ReadSchema: struct<sr_store_sk:int,sr_return_amt:decimal(7,2),sr_net_loss:decimal(7,2)>
+
+(21) ColumnarToRow [codegen id : 6]
+Input [4]: [sr_store_sk#20, sr_return_amt#21, sr_net_loss#22, sr_returned_date_sk#23]
+
+(22) Filter [codegen id : 6]
+Input [4]: [sr_store_sk#20, sr_return_amt#21, sr_net_loss#22, sr_returned_date_sk#23]
+Condition : isnotnull(sr_store_sk#20)
+
+(23) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#24]
+
+(24) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [sr_returned_date_sk#23]
+Right keys [1]: [d_date_sk#24]
+Join condition: None
+
+(25) Project [codegen id : 6]
+Output [3]: [sr_store_sk#20, sr_return_amt#21, sr_net_loss#22]
+Input [5]: [sr_store_sk#20, sr_return_amt#21, sr_net_loss#22, sr_returned_date_sk#23, d_date_sk#24]
+
+(26) ReusedExchange [Reuses operator id: 14]
+Output [1]: [s_store_sk#25]
+
+(27) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [sr_store_sk#20]
+Right keys [1]: [s_store_sk#25]
+Join condition: None
+
+(28) Project [codegen id : 6]
+Output [3]: [sr_return_amt#21, sr_net_loss#22, s_store_sk#25]
+Input [4]: [sr_store_sk#20, sr_return_amt#21, sr_net_loss#22, s_store_sk#25]
+
+(29) HashAggregate [codegen id : 6]
+Input [3]: [sr_return_amt#21, sr_net_loss#22, s_store_sk#25]
+Keys [1]: [s_store_sk#25]
+Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#21)), partial_sum(UnscaledValue(sr_net_loss#22))]
+Aggregate Attributes [2]: [sum#26, sum#27]
+Results [3]: [s_store_sk#25, sum#28, sum#29]
+
+(30) Exchange
+Input [3]: [s_store_sk#25, sum#28, sum#29]
+Arguments: hashpartitioning(s_store_sk#25, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(31) HashAggregate [codegen id : 7]
+Input [3]: [s_store_sk#25, sum#28, sum#29]
+Keys [1]: [s_store_sk#25]
+Functions [2]: [sum(UnscaledValue(sr_return_amt#21)), sum(UnscaledValue(sr_net_loss#22))]
+Aggregate Attributes [2]: [sum(UnscaledValue(sr_return_amt#21))#31, sum(UnscaledValue(sr_net_loss#22))#32]
+Results [3]: [s_store_sk#25, MakeDecimal(sum(UnscaledValue(sr_return_amt#21))#31,17,2) AS returns#33, MakeDecimal(sum(UnscaledValue(sr_net_loss#22))#32,17,2) AS profit_loss#34]
+
+(32) BroadcastExchange
+Input [3]: [s_store_sk#25, returns#33, profit_loss#34]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#35]
+
+(33) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [s_store_sk#9]
+Right keys [1]: [s_store_sk#25]
+Join condition: None
+
+(34) Project [codegen id : 8]
+Output [5]: [sales#18, coalesce(returns#33, 0.00) AS returns#36, CheckOverflow((promote_precision(cast(profit#19 as decimal(18,2))) - promote_precision(cast(coalesce(profit_loss#34, 0.00) as decimal(18,2)))), DecimalType(18,2), true) AS profit#37, store channel AS channel#38, s_store_sk#9 AS id#39]
+Input [6]: [s_store_sk#9, sales#18, profit#19, s_store_sk#25, returns#33, profit_loss#34]
+
+(35) Scan parquet default.catalog_sales
+Output [4]: [cs_call_center_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#43), dynamicpruningexpression(cs_sold_date_sk#43 IN dynamicpruning#5)]
+ReadSchema: struct<cs_call_center_sk:int,cs_ext_sales_price:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(36) ColumnarToRow [codegen id : 10]
+Input [4]: [cs_call_center_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43]
+
+(37) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#44]
+
+(38) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_sold_date_sk#43]
+Right keys [1]: [d_date_sk#44]
+Join condition: None
+
+(39) Project [codegen id : 10]
+Output [3]: [cs_call_center_sk#40, cs_ext_sales_price#41, cs_net_profit#42]
+Input [5]: [cs_call_center_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43, d_date_sk#44]
+
+(40) HashAggregate [codegen id : 10]
+Input [3]: [cs_call_center_sk#40, cs_ext_sales_price#41, cs_net_profit#42]
+Keys [1]: [cs_call_center_sk#40]
+Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#41)), partial_sum(UnscaledValue(cs_net_profit#42))]
+Aggregate Attributes [2]: [sum#45, sum#46]
+Results [3]: [cs_call_center_sk#40, sum#47, sum#48]
+
+(41) Exchange
+Input [3]: [cs_call_center_sk#40, sum#47, sum#48]
+Arguments: hashpartitioning(cs_call_center_sk#40, 5), ENSURE_REQUIREMENTS, [id=#49]
+
+(42) HashAggregate [codegen id : 14]
+Input [3]: [cs_call_center_sk#40, sum#47, sum#48]
+Keys [1]: [cs_call_center_sk#40]
+Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#41)), sum(UnscaledValue(cs_net_profit#42))]
+Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_sales_price#41))#50, sum(UnscaledValue(cs_net_profit#42))#51]
+Results [3]: [cs_call_center_sk#40, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#41))#50,17,2) AS sales#52, MakeDecimal(sum(UnscaledValue(cs_net_profit#42))#51,17,2) AS profit#53]
+
+(43) Scan parquet default.catalog_returns
+Output [3]: [cr_return_amount#54, cr_net_loss#55, cr_returned_date_sk#56]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cr_returned_date_sk#56), dynamicpruningexpression(cr_returned_date_sk#56 IN dynamicpruning#5)]
+ReadSchema: struct<cr_return_amount:decimal(7,2),cr_net_loss:decimal(7,2)>
+
+(44) ColumnarToRow [codegen id : 12]
+Input [3]: [cr_return_amount#54, cr_net_loss#55, cr_returned_date_sk#56]
+
+(45) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#57]
+
+(46) BroadcastHashJoin [codegen id : 12]
+Left keys [1]: [cr_returned_date_sk#56]
+Right keys [1]: [d_date_sk#57]
+Join condition: None
+
+(47) Project [codegen id : 12]
+Output [2]: [cr_return_amount#54, cr_net_loss#55]
+Input [4]: [cr_return_amount#54, cr_net_loss#55, cr_returned_date_sk#56, d_date_sk#57]
+
+(48) HashAggregate [codegen id : 12]
+Input [2]: [cr_return_amount#54, cr_net_loss#55]
+Keys: []
+Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#54)), partial_sum(UnscaledValue(cr_net_loss#55))]
+Aggregate Attributes [2]: [sum#58, sum#59]
+Results [2]: [sum#60, sum#61]
+
+(49) Exchange
+Input [2]: [sum#60, sum#61]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#62]
+
+(50) HashAggregate [codegen id : 13]
+Input [2]: [sum#60, sum#61]
+Keys: []
+Functions [2]: [sum(UnscaledValue(cr_return_amount#54)), sum(UnscaledValue(cr_net_loss#55))]
+Aggregate Attributes [2]: [sum(UnscaledValue(cr_return_amount#54))#63, sum(UnscaledValue(cr_net_loss#55))#64]
+Results [2]: [MakeDecimal(sum(UnscaledValue(cr_return_amount#54))#63,17,2) AS returns#65, MakeDecimal(sum(UnscaledValue(cr_net_loss#55))#64,17,2) AS profit_loss#66]
+
+(51) BroadcastExchange
+Input [2]: [returns#65, profit_loss#66]
+Arguments: IdentityBroadcastMode, [id=#67]
+
+(52) BroadcastNestedLoopJoin [codegen id : 14]
+Join condition: None
+
+(53) Project [codegen id : 14]
+Output [5]: [sales#52, returns#65, CheckOverflow((promote_precision(cast(profit#53 as decimal(18,2))) - promote_precision(cast(profit_loss#66 as decimal(18,2)))), DecimalType(18,2), true) AS profit#68, catalog channel AS channel#69, cs_call_center_sk#40 AS id#70]
+Input [5]: [cs_call_center_sk#40, sales#52, profit#53, returns#65, profit_loss#66]
+
+(54) Scan parquet default.web_sales
+Output [4]: [ws_web_page_sk#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#74), dynamicpruningexpression(ws_sold_date_sk#74 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ws_web_page_sk)]
+ReadSchema: struct<ws_web_page_sk:int,ws_ext_sales_price:decimal(7,2),ws_net_profit:decimal(7,2)>
+
+(55) ColumnarToRow [codegen id : 17]
+Input [4]: [ws_web_page_sk#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74]
+
+(56) Filter [codegen id : 17]
+Input [4]: [ws_web_page_sk#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74]
+Condition : isnotnull(ws_web_page_sk#71)
+
+(57) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#75]
+
+(58) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_sold_date_sk#74]
+Right keys [1]: [d_date_sk#75]
+Join condition: None
+
+(59) Project [codegen id : 17]
+Output [3]: [ws_web_page_sk#71, ws_ext_sales_price#72, ws_net_profit#73]
+Input [5]: [ws_web_page_sk#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, d_date_sk#75]
+
+(60) Scan parquet default.web_page
+Output [1]: [wp_web_page_sk#76]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_page]
+PushedFilters: [IsNotNull(wp_web_page_sk)]
+ReadSchema: struct<wp_web_page_sk:int>
+
+(61) ColumnarToRow [codegen id : 16]
+Input [1]: [wp_web_page_sk#76]
+
+(62) Filter [codegen id : 16]
+Input [1]: [wp_web_page_sk#76]
+Condition : isnotnull(wp_web_page_sk#76)
+
+(63) BroadcastExchange
+Input [1]: [wp_web_page_sk#76]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#77]
+
+(64) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_web_page_sk#71]
+Right keys [1]: [wp_web_page_sk#76]
+Join condition: None
+
+(65) Project [codegen id : 17]
+Output [3]: [ws_ext_sales_price#72, ws_net_profit#73, wp_web_page_sk#76]
+Input [4]: [ws_web_page_sk#71, ws_ext_sales_price#72, ws_net_profit#73, wp_web_page_sk#76]
+
+(66) HashAggregate [codegen id : 17]
+Input [3]: [ws_ext_sales_price#72, ws_net_profit#73, wp_web_page_sk#76]
+Keys [1]: [wp_web_page_sk#76]
+Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#72)), partial_sum(UnscaledValue(ws_net_profit#73))]
+Aggregate Attributes [2]: [sum#78, sum#79]
+Results [3]: [wp_web_page_sk#76, sum#80, sum#81]
+
+(67) Exchange
+Input [3]: [wp_web_page_sk#76, sum#80, sum#81]
+Arguments: hashpartitioning(wp_web_page_sk#76, 5), ENSURE_REQUIREMENTS, [id=#82]
+
+(68) HashAggregate [codegen id : 22]
+Input [3]: [wp_web_page_sk#76, sum#80, sum#81]
+Keys [1]: [wp_web_page_sk#76]
+Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#72)), sum(UnscaledValue(ws_net_profit#73))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_sales_price#72))#83, sum(UnscaledValue(ws_net_profit#73))#84]
+Results [3]: [wp_web_page_sk#76, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#72))#83,17,2) AS sales#85, MakeDecimal(sum(UnscaledValue(ws_net_profit#73))#84,17,2) AS profit#86]
+
+(69) Scan parquet default.web_returns
+Output [4]: [wr_web_page_sk#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(wr_returned_date_sk#90), dynamicpruningexpression(wr_returned_date_sk#90 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(wr_web_page_sk)]
+ReadSchema: struct<wr_web_page_sk:int,wr_return_amt:decimal(7,2),wr_net_loss:decimal(7,2)>
+
+(70) ColumnarToRow [codegen id : 20]
+Input [4]: [wr_web_page_sk#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90]
+
+(71) Filter [codegen id : 20]
+Input [4]: [wr_web_page_sk#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90]
+Condition : isnotnull(wr_web_page_sk#87)
+
+(72) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#91]
+
+(73) BroadcastHashJoin [codegen id : 20]
+Left keys [1]: [wr_returned_date_sk#90]
+Right keys [1]: [d_date_sk#91]
+Join condition: None
+
+(74) Project [codegen id : 20]
+Output [3]: [wr_web_page_sk#87, wr_return_amt#88, wr_net_loss#89]
+Input [5]: [wr_web_page_sk#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90, d_date_sk#91]
+
+(75) ReusedExchange [Reuses operator id: 63]
+Output [1]: [wp_web_page_sk#92]
+
+(76) BroadcastHashJoin [codegen id : 20]
+Left keys [1]: [wr_web_page_sk#87]
+Right keys [1]: [wp_web_page_sk#92]
+Join condition: None
+
+(77) Project [codegen id : 20]
+Output [3]: [wr_return_amt#88, wr_net_loss#89, wp_web_page_sk#92]
+Input [4]: [wr_web_page_sk#87, wr_return_amt#88, wr_net_loss#89, wp_web_page_sk#92]
+
+(78) HashAggregate [codegen id : 20]
+Input [3]: [wr_return_amt#88, wr_net_loss#89, wp_web_page_sk#92]
+Keys [1]: [wp_web_page_sk#92]
+Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#88)), partial_sum(UnscaledValue(wr_net_loss#89))]
+Aggregate Attributes [2]: [sum#93, sum#94]
+Results [3]: [wp_web_page_sk#92, sum#95, sum#96]
+
+(79) Exchange
+Input [3]: [wp_web_page_sk#92, sum#95, sum#96]
+Arguments: hashpartitioning(wp_web_page_sk#92, 5), ENSURE_REQUIREMENTS, [id=#97]
+
+(80) HashAggregate [codegen id : 21]
+Input [3]: [wp_web_page_sk#92, sum#95, sum#96]
+Keys [1]: [wp_web_page_sk#92]
+Functions [2]: [sum(UnscaledValue(wr_return_amt#88)), sum(UnscaledValue(wr_net_loss#89))]
+Aggregate Attributes [2]: [sum(UnscaledValue(wr_return_amt#88))#98, sum(UnscaledValue(wr_net_loss#89))#99]
+Results [3]: [wp_web_page_sk#92, MakeDecimal(sum(UnscaledValue(wr_return_amt#88))#98,17,2) AS returns#100, MakeDecimal(sum(UnscaledValue(wr_net_loss#89))#99,17,2) AS profit_loss#101]
+
+(81) BroadcastExchange
+Input [3]: [wp_web_page_sk#92, returns#100, profit_loss#101]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#102]
+
+(82) BroadcastHashJoin [codegen id : 22]
+Left keys [1]: [wp_web_page_sk#76]
+Right keys [1]: [wp_web_page_sk#92]
+Join condition: None
+
+(83) Project [codegen id : 22]
+Output [5]: [sales#85, coalesce(returns#100, 0.00) AS returns#103, CheckOverflow((promote_precision(cast(profit#86 as decimal(18,2))) - promote_precision(cast(coalesce(profit_loss#101, 0.00) as decimal(18,2)))), DecimalType(18,2), true) AS profit#104, web channel AS channel#105, wp_web_page_sk#76 AS id#106]
+Input [6]: [wp_web_page_sk#76, sales#85, profit#86, wp_web_page_sk#92, returns#100, profit_loss#101]
+
+(84) Union
+
+(85) Expand [codegen id : 23]
+Input [5]: [sales#18, returns#36, profit#37, channel#38, id#39]
+Arguments: [ArrayBuffer(sales#18, returns#36, profit#37, channel#38, id#39, 0), ArrayBuffer(sales#18, returns#36, profit#37, channel#38, null, 1), ArrayBuffer(sales#18, returns#36, profit#37, null, null, 3)], [sales#18, returns#36, profit#37, channel#107, id#108, spark_grouping_id#109]
+
+(86) HashAggregate [codegen id : 23]
+Input [6]: [sales#18, returns#36, profit#37, channel#107, id#108, spark_grouping_id#109]
+Keys [3]: [channel#107, id#108, spark_grouping_id#109]
+Functions [3]: [partial_sum(sales#18), partial_sum(returns#36), partial_sum(profit#37)]
+Aggregate Attributes [6]: [sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115]
+Results [9]: [channel#107, id#108, spark_grouping_id#109, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121]
+
+(87) Exchange
+Input [9]: [channel#107, id#108, spark_grouping_id#109, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121]
+Arguments: hashpartitioning(channel#107, id#108, spark_grouping_id#109, 5), ENSURE_REQUIREMENTS, [id=#122]
+
+(88) HashAggregate [codegen id : 24]
+Input [9]: [channel#107, id#108, spark_grouping_id#109, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121]
+Keys [3]: [channel#107, id#108, spark_grouping_id#109]
+Functions [3]: [sum(sales#18), sum(returns#36), sum(profit#37)]
+Aggregate Attributes [3]: [sum(sales#18)#123, sum(returns#36)#124, sum(profit#37)#125]
+Results [5]: [channel#107, id#108, sum(sales#18)#123 AS sales#126, sum(returns#36)#124 AS returns#127, sum(profit#37)#125 AS profit#128]
+
+(89) TakeOrderedAndProject
+Input [5]: [channel#107, id#108, sales#126, returns#127, profit#128]
+Arguments: 100, [channel#107 ASC NULLS FIRST, id#108 ASC NULLS FIRST], [channel#107, id#108, sales#126, returns#127, profit#128]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (90)
+
+
+(90) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#6]
+
+Subquery:2 Hosting operator id = 20 Hosting Expression = sr_returned_date_sk#23 IN dynamicpruning#5
+
+Subquery:3 Hosting operator id = 35 Hosting Expression = cs_sold_date_sk#43 IN dynamicpruning#5
+
+Subquery:4 Hosting operator id = 43 Hosting Expression = cr_returned_date_sk#56 IN dynamicpruning#5
+
+Subquery:5 Hosting operator id = 54 Hosting Expression = ws_sold_date_sk#74 IN dynamicpruning#5
+
+Subquery:6 Hosting operator id = 69 Hosting Expression = wr_returned_date_sk#90 IN dynamicpruning#5
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.sf100/simplified.txt
new file mode 100644
index 0000000000000..d2fc07d0112f0
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.sf100/simplified.txt
@@ -0,0 +1,143 @@
+TakeOrderedAndProject [channel,id,sales,returns,profit]
+  WholeStageCodegen (24)
+    HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+      InputAdapter
+        Exchange [channel,id,spark_grouping_id] #1
+          WholeStageCodegen (23)
+            HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+              Expand [sales,returns,profit,channel,id]
+                InputAdapter
+                  Union
+                    WholeStageCodegen (8)
+                      Project [sales,returns,profit,profit_loss,s_store_sk]
+                        BroadcastHashJoin [s_store_sk,s_store_sk]
+                          HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit)),sales,profit,sum,sum]
+                            InputAdapter
+                              Exchange [s_store_sk] #2
+                                WholeStageCodegen (3)
+                                  HashAggregate [s_store_sk,ss_ext_sales_price,ss_net_profit] [sum,sum,sum,sum]
+                                    Project [ss_ext_sales_price,ss_net_profit,s_store_sk]
+                                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                        Project [ss_store_sk,ss_ext_sales_price,ss_net_profit]
+                                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                            Filter [ss_store_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #1
+                                                      ReusedExchange [d_date_sk] #3
+                                            InputAdapter
+                                              BroadcastExchange #3
+                                                WholeStageCodegen (1)
+                                                  Project [d_date_sk]
+                                                    Filter [d_date,d_date_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.date_dim [d_date_sk,d_date]
+                                        InputAdapter
+                                          BroadcastExchange #4
+                                            WholeStageCodegen (2)
+                                              Filter [s_store_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.store [s_store_sk]
+                          InputAdapter
+                            BroadcastExchange #5
+                              WholeStageCodegen (7)
+                                HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss)),returns,profit_loss,sum,sum]
+                                  InputAdapter
+                                    Exchange [s_store_sk] #6
+                                      WholeStageCodegen (6)
+                                        HashAggregate [s_store_sk,sr_return_amt,sr_net_loss] [sum,sum,sum,sum]
+                                          Project [sr_return_amt,sr_net_loss,s_store_sk]
+                                            BroadcastHashJoin [sr_store_sk,s_store_sk]
+                                              Project [sr_store_sk,sr_return_amt,sr_net_loss]
+                                                BroadcastHashJoin [sr_returned_date_sk,d_date_sk]
+                                                  Filter [sr_store_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk]
+                                                          ReusedSubquery [d_date_sk] #1
+                                                  InputAdapter
+                                                    ReusedExchange [d_date_sk] #3
+                                              InputAdapter
+                                                ReusedExchange [s_store_sk] #4
+                    WholeStageCodegen (14)
+                      Project [sales,returns,profit,profit_loss,cs_call_center_sk]
+                        BroadcastNestedLoopJoin
+                          HashAggregate [cs_call_center_sk,sum,sum] [sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit)),sales,profit,sum,sum]
+                            InputAdapter
+                              Exchange [cs_call_center_sk] #7
+                                WholeStageCodegen (10)
+                                  HashAggregate [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] [sum,sum,sum,sum]
+                                    Project [cs_call_center_sk,cs_ext_sales_price,cs_net_profit]
+                                      BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk]
+                                              ReusedSubquery [d_date_sk] #1
+                                        InputAdapter
+                                          ReusedExchange [d_date_sk] #3
+                          InputAdapter
+                            BroadcastExchange #8
+                              WholeStageCodegen (13)
+                                HashAggregate [sum,sum] [sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss)),returns,profit_loss,sum,sum]
+                                  InputAdapter
+                                    Exchange #9
+                                      WholeStageCodegen (12)
+                                        HashAggregate [cr_return_amount,cr_net_loss] [sum,sum,sum,sum]
+                                          Project [cr_return_amount,cr_net_loss]
+                                            BroadcastHashJoin [cr_returned_date_sk,d_date_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk]
+                                                    ReusedSubquery [d_date_sk] #1
+                                              InputAdapter
+                                                ReusedExchange [d_date_sk] #3
+                    WholeStageCodegen (22)
+                      Project [sales,returns,profit,profit_loss,wp_web_page_sk]
+                        BroadcastHashJoin [wp_web_page_sk,wp_web_page_sk]
+                          HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit)),sales,profit,sum,sum]
+                            InputAdapter
+                              Exchange [wp_web_page_sk] #10
+                                WholeStageCodegen (17)
+                                  HashAggregate [wp_web_page_sk,ws_ext_sales_price,ws_net_profit] [sum,sum,sum,sum]
+                                    Project [ws_ext_sales_price,ws_net_profit,wp_web_page_sk]
+                                      BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk]
+                                        Project [ws_web_page_sk,ws_ext_sales_price,ws_net_profit]
+                                          BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                            Filter [ws_web_page_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk]
+                                                    ReusedSubquery [d_date_sk] #1
+                                            InputAdapter
+                                              ReusedExchange [d_date_sk] #3
+                                        InputAdapter
+                                          BroadcastExchange #11
+                                            WholeStageCodegen (16)
+                                              Filter [wp_web_page_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.web_page [wp_web_page_sk]
+                          InputAdapter
+                            BroadcastExchange #12
+                              WholeStageCodegen (21)
+                                HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss)),returns,profit_loss,sum,sum]
+                                  InputAdapter
+                                    Exchange [wp_web_page_sk] #13
+                                      WholeStageCodegen (20)
+                                        HashAggregate [wp_web_page_sk,wr_return_amt,wr_net_loss] [sum,sum,sum,sum]
+                                          Project [wr_return_amt,wr_net_loss,wp_web_page_sk]
+                                            BroadcastHashJoin [wr_web_page_sk,wp_web_page_sk]
+                                              Project [wr_web_page_sk,wr_return_amt,wr_net_loss]
+                                                BroadcastHashJoin [wr_returned_date_sk,d_date_sk]
+                                                  Filter [wr_web_page_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk]
+                                                          ReusedSubquery [d_date_sk] #1
+                                                  InputAdapter
+                                                    ReusedExchange [d_date_sk] #3
+                                              InputAdapter
+                                                ReusedExchange [wp_web_page_sk] #11
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/explain.txt
new file mode 100644
index 0000000000000..7c9c3589c275e
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/explain.txt
@@ -0,0 +1,534 @@
+== Physical Plan ==
+TakeOrderedAndProject (89)
++- * HashAggregate (88)
+   +- Exchange (87)
+      +- * HashAggregate (86)
+         +- * Expand (85)
+            +- Union (84)
+               :- * Project (34)
+               :  +- * BroadcastHashJoin LeftOuter BuildRight (33)
+               :     :- * HashAggregate (19)
+               :     :  +- Exchange (18)
+               :     :     +- * HashAggregate (17)
+               :     :        +- * Project (16)
+               :     :           +- * BroadcastHashJoin Inner BuildRight (15)
+               :     :              :- * Project (10)
+               :     :              :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :              :     :- * Filter (3)
+               :     :              :     :  +- * ColumnarToRow (2)
+               :     :              :     :     +- Scan parquet default.store_sales (1)
+               :     :              :     +- BroadcastExchange (8)
+               :     :              :        +- * Project (7)
+               :     :              :           +- * Filter (6)
+               :     :              :              +- * ColumnarToRow (5)
+               :     :              :                 +- Scan parquet default.date_dim (4)
+               :     :              +- BroadcastExchange (14)
+               :     :                 +- * Filter (13)
+               :     :                    +- * ColumnarToRow (12)
+               :     :                       +- Scan parquet default.store (11)
+               :     +- BroadcastExchange (32)
+               :        +- * HashAggregate (31)
+               :           +- Exchange (30)
+               :              +- * HashAggregate (29)
+               :                 +- * Project (28)
+               :                    +- * BroadcastHashJoin Inner BuildRight (27)
+               :                       :- * Project (25)
+               :                       :  +- * BroadcastHashJoin Inner BuildRight (24)
+               :                       :     :- * Filter (22)
+               :                       :     :  +- * ColumnarToRow (21)
+               :                       :     :     +- Scan parquet default.store_returns (20)
+               :                       :     +- ReusedExchange (23)
+               :                       +- ReusedExchange (26)
+               :- * Project (53)
+               :  +- * BroadcastNestedLoopJoin Inner BuildLeft (52)
+               :     :- BroadcastExchange (43)
+               :     :  +- * HashAggregate (42)
+               :     :     +- Exchange (41)
+               :     :        +- * HashAggregate (40)
+               :     :           +- * Project (39)
+               :     :              +- * BroadcastHashJoin Inner BuildRight (38)
+               :     :                 :- * ColumnarToRow (36)
+               :     :                 :  +- Scan parquet default.catalog_sales (35)
+               :     :                 +- ReusedExchange (37)
+               :     +- * HashAggregate (51)
+               :        +- Exchange (50)
+               :           +- * HashAggregate (49)
+               :              +- * Project (48)
+               :                 +- * BroadcastHashJoin Inner BuildRight (47)
+               :                    :- * ColumnarToRow (45)
+               :                    :  +- Scan parquet default.catalog_returns (44)
+               :                    +- ReusedExchange (46)
+               +- * Project (83)
+                  +- * BroadcastHashJoin LeftOuter BuildRight (82)
+                     :- * HashAggregate (68)
+                     :  +- Exchange (67)
+                     :     +- * HashAggregate (66)
+                     :        +- * Project (65)
+                     :           +- * BroadcastHashJoin Inner BuildRight (64)
+                     :              :- * Project (59)
+                     :              :  +- * BroadcastHashJoin Inner BuildRight (58)
+                     :              :     :- * Filter (56)
+                     :              :     :  +- * ColumnarToRow (55)
+                     :              :     :     +- Scan parquet default.web_sales (54)
+                     :              :     +- ReusedExchange (57)
+                     :              +- BroadcastExchange (63)
+                     :                 +- * Filter (62)
+                     :                    +- * ColumnarToRow (61)
+                     :                       +- Scan parquet default.web_page (60)
+                     +- BroadcastExchange (81)
+                        +- * HashAggregate (80)
+                           +- Exchange (79)
+                              +- * HashAggregate (78)
+                                 +- * Project (77)
+                                    +- * BroadcastHashJoin Inner BuildRight (76)
+                                       :- * Project (74)
+                                       :  +- * BroadcastHashJoin Inner BuildRight (73)
+                                       :     :- * Filter (71)
+                                       :     :  +- * ColumnarToRow (70)
+                                       :     :     +- Scan parquet default.web_returns (69)
+                                       :     +- ReusedExchange (72)
+                                       +- ReusedExchange (75)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_ext_sales_price:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 3]
+Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+Condition : isnotnull(ss_store_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#6, d_date#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-03), LessThanOrEqual(d_date,2000-09-02), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#6, d_date#7]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#6, d_date#7]
+Condition : (((isnotnull(d_date#7) AND (d_date#7 >= 2000-08-03)) AND (d_date#7 <= 2000-09-02)) AND isnotnull(d_date_sk#6))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#6]
+Input [2]: [d_date_sk#6, d_date#7]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#8]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3]
+Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#6]
+
+(11) Scan parquet default.store
+Output [1]: [s_store_sk#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [1]: [s_store_sk#9]
+
+(13) Filter [codegen id : 2]
+Input [1]: [s_store_sk#9]
+Condition : isnotnull(s_store_sk#9)
+
+(14) BroadcastExchange
+Input [1]: [s_store_sk#9]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#10]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_store_sk#1]
+Right keys [1]: [s_store_sk#9]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#9]
+Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#9]
+
+(17) HashAggregate [codegen id : 3]
+Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#9]
+Keys [1]: [s_store_sk#9]
+Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))]
+Aggregate Attributes [2]: [sum#11, sum#12]
+Results [3]: [s_store_sk#9, sum#13, sum#14]
+
+(18) Exchange
+Input [3]: [s_store_sk#9, sum#13, sum#14]
+Arguments: hashpartitioning(s_store_sk#9, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(19) HashAggregate [codegen id : 8]
+Input [3]: [s_store_sk#9, sum#13, sum#14]
+Keys [1]: [s_store_sk#9]
+Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ss_ext_sales_price#2))#16, sum(UnscaledValue(ss_net_profit#3))#17]
+Results [3]: [s_store_sk#9, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#16,17,2) AS sales#18, MakeDecimal(sum(UnscaledValue(ss_net_profit#3))#17,17,2) AS profit#19]
+
+(20) Scan parquet default.store_returns
+Output [4]: [sr_store_sk#20, sr_return_amt#21, sr_net_loss#22, sr_returned_date_sk#23]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(sr_returned_date_sk#23), dynamicpruningexpression(sr_returned_date_sk#23 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(sr_store_sk)]
+ReadSchema: struct<sr_store_sk:int,sr_return_amt:decimal(7,2),sr_net_loss:decimal(7,2)>
+
+(21) ColumnarToRow [codegen id : 6]
+Input [4]: [sr_store_sk#20, sr_return_amt#21, sr_net_loss#22, sr_returned_date_sk#23]
+
+(22) Filter [codegen id : 6]
+Input [4]: [sr_store_sk#20, sr_return_amt#21, sr_net_loss#22, sr_returned_date_sk#23]
+Condition : isnotnull(sr_store_sk#20)
+
+(23) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#24]
+
+(24) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [sr_returned_date_sk#23]
+Right keys [1]: [d_date_sk#24]
+Join condition: None
+
+(25) Project [codegen id : 6]
+Output [3]: [sr_store_sk#20, sr_return_amt#21, sr_net_loss#22]
+Input [5]: [sr_store_sk#20, sr_return_amt#21, sr_net_loss#22, sr_returned_date_sk#23, d_date_sk#24]
+
+(26) ReusedExchange [Reuses operator id: 14]
+Output [1]: [s_store_sk#25]
+
+(27) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [sr_store_sk#20]
+Right keys [1]: [s_store_sk#25]
+Join condition: None
+
+(28) Project [codegen id : 6]
+Output [3]: [sr_return_amt#21, sr_net_loss#22, s_store_sk#25]
+Input [4]: [sr_store_sk#20, sr_return_amt#21, sr_net_loss#22, s_store_sk#25]
+
+(29) HashAggregate [codegen id : 6]
+Input [3]: [sr_return_amt#21, sr_net_loss#22, s_store_sk#25]
+Keys [1]: [s_store_sk#25]
+Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#21)), partial_sum(UnscaledValue(sr_net_loss#22))]
+Aggregate Attributes [2]: [sum#26, sum#27]
+Results [3]: [s_store_sk#25, sum#28, sum#29]
+
+(30) Exchange
+Input [3]: [s_store_sk#25, sum#28, sum#29]
+Arguments: hashpartitioning(s_store_sk#25, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(31) HashAggregate [codegen id : 7]
+Input [3]: [s_store_sk#25, sum#28, sum#29]
+Keys [1]: [s_store_sk#25]
+Functions [2]: [sum(UnscaledValue(sr_return_amt#21)), sum(UnscaledValue(sr_net_loss#22))]
+Aggregate Attributes [2]: [sum(UnscaledValue(sr_return_amt#21))#31, sum(UnscaledValue(sr_net_loss#22))#32]
+Results [3]: [s_store_sk#25, MakeDecimal(sum(UnscaledValue(sr_return_amt#21))#31,17,2) AS returns#33, MakeDecimal(sum(UnscaledValue(sr_net_loss#22))#32,17,2) AS profit_loss#34]
+
+(32) BroadcastExchange
+Input [3]: [s_store_sk#25, returns#33, profit_loss#34]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#35]
+
+(33) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [s_store_sk#9]
+Right keys [1]: [s_store_sk#25]
+Join condition: None
+
+(34) Project [codegen id : 8]
+Output [5]: [sales#18, coalesce(returns#33, 0.00) AS returns#36, CheckOverflow((promote_precision(cast(profit#19 as decimal(18,2))) - promote_precision(cast(coalesce(profit_loss#34, 0.00) as decimal(18,2)))), DecimalType(18,2), true) AS profit#37, store channel AS channel#38, s_store_sk#9 AS id#39]
+Input [6]: [s_store_sk#9, sales#18, profit#19, s_store_sk#25, returns#33, profit_loss#34]
+
+(35) Scan parquet default.catalog_sales
+Output [4]: [cs_call_center_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#43), dynamicpruningexpression(cs_sold_date_sk#43 IN dynamicpruning#5)]
+ReadSchema: struct<cs_call_center_sk:int,cs_ext_sales_price:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(36) ColumnarToRow [codegen id : 10]
+Input [4]: [cs_call_center_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43]
+
+(37) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#44]
+
+(38) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_sold_date_sk#43]
+Right keys [1]: [d_date_sk#44]
+Join condition: None
+
+(39) Project [codegen id : 10]
+Output [3]: [cs_call_center_sk#40, cs_ext_sales_price#41, cs_net_profit#42]
+Input [5]: [cs_call_center_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43, d_date_sk#44]
+
+(40) HashAggregate [codegen id : 10]
+Input [3]: [cs_call_center_sk#40, cs_ext_sales_price#41, cs_net_profit#42]
+Keys [1]: [cs_call_center_sk#40]
+Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#41)), partial_sum(UnscaledValue(cs_net_profit#42))]
+Aggregate Attributes [2]: [sum#45, sum#46]
+Results [3]: [cs_call_center_sk#40, sum#47, sum#48]
+
+(41) Exchange
+Input [3]: [cs_call_center_sk#40, sum#47, sum#48]
+Arguments: hashpartitioning(cs_call_center_sk#40, 5), ENSURE_REQUIREMENTS, [id=#49]
+
+(42) HashAggregate [codegen id : 11]
+Input [3]: [cs_call_center_sk#40, sum#47, sum#48]
+Keys [1]: [cs_call_center_sk#40]
+Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#41)), sum(UnscaledValue(cs_net_profit#42))]
+Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_sales_price#41))#50, sum(UnscaledValue(cs_net_profit#42))#51]
+Results [3]: [cs_call_center_sk#40, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#41))#50,17,2) AS sales#52, MakeDecimal(sum(UnscaledValue(cs_net_profit#42))#51,17,2) AS profit#53]
+
+(43) BroadcastExchange
+Input [3]: [cs_call_center_sk#40, sales#52, profit#53]
+Arguments: IdentityBroadcastMode, [id=#54]
+
+(44) Scan parquet default.catalog_returns
+Output [3]: [cr_return_amount#55, cr_net_loss#56, cr_returned_date_sk#57]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cr_returned_date_sk#57), dynamicpruningexpression(cr_returned_date_sk#57 IN dynamicpruning#5)]
+ReadSchema: struct<cr_return_amount:decimal(7,2),cr_net_loss:decimal(7,2)>
+
+(45) ColumnarToRow [codegen id : 13]
+Input [3]: [cr_return_amount#55, cr_net_loss#56, cr_returned_date_sk#57]
+
+(46) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#58]
+
+(47) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [cr_returned_date_sk#57]
+Right keys [1]: [d_date_sk#58]
+Join condition: None
+
+(48) Project [codegen id : 13]
+Output [2]: [cr_return_amount#55, cr_net_loss#56]
+Input [4]: [cr_return_amount#55, cr_net_loss#56, cr_returned_date_sk#57, d_date_sk#58]
+
+(49) HashAggregate [codegen id : 13]
+Input [2]: [cr_return_amount#55, cr_net_loss#56]
+Keys: []
+Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#55)), partial_sum(UnscaledValue(cr_net_loss#56))]
+Aggregate Attributes [2]: [sum#59, sum#60]
+Results [2]: [sum#61, sum#62]
+
+(50) Exchange
+Input [2]: [sum#61, sum#62]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#63]
+
+(51) HashAggregate
+Input [2]: [sum#61, sum#62]
+Keys: []
+Functions [2]: [sum(UnscaledValue(cr_return_amount#55)), sum(UnscaledValue(cr_net_loss#56))]
+Aggregate Attributes [2]: [sum(UnscaledValue(cr_return_amount#55))#64, sum(UnscaledValue(cr_net_loss#56))#65]
+Results [2]: [MakeDecimal(sum(UnscaledValue(cr_return_amount#55))#64,17,2) AS returns#66, MakeDecimal(sum(UnscaledValue(cr_net_loss#56))#65,17,2) AS profit_loss#67]
+
+(52) BroadcastNestedLoopJoin [codegen id : 14]
+Join condition: None
+
+(53) Project [codegen id : 14]
+Output [5]: [sales#52, returns#66, CheckOverflow((promote_precision(cast(profit#53 as decimal(18,2))) - promote_precision(cast(profit_loss#67 as decimal(18,2)))), DecimalType(18,2), true) AS profit#68, catalog channel AS channel#69, cs_call_center_sk#40 AS id#70]
+Input [5]: [cs_call_center_sk#40, sales#52, profit#53, returns#66, profit_loss#67]
+
+(54) Scan parquet default.web_sales
+Output [4]: [ws_web_page_sk#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#74), dynamicpruningexpression(ws_sold_date_sk#74 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ws_web_page_sk)]
+ReadSchema: struct<ws_web_page_sk:int,ws_ext_sales_price:decimal(7,2),ws_net_profit:decimal(7,2)>
+
+(55) ColumnarToRow [codegen id : 17]
+Input [4]: [ws_web_page_sk#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74]
+
+(56) Filter [codegen id : 17]
+Input [4]: [ws_web_page_sk#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74]
+Condition : isnotnull(ws_web_page_sk#71)
+
+(57) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#75]
+
+(58) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_sold_date_sk#74]
+Right keys [1]: [d_date_sk#75]
+Join condition: None
+
+(59) Project [codegen id : 17]
+Output [3]: [ws_web_page_sk#71, ws_ext_sales_price#72, ws_net_profit#73]
+Input [5]: [ws_web_page_sk#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, d_date_sk#75]
+
+(60) Scan parquet default.web_page
+Output [1]: [wp_web_page_sk#76]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_page]
+PushedFilters: [IsNotNull(wp_web_page_sk)]
+ReadSchema: struct<wp_web_page_sk:int>
+
+(61) ColumnarToRow [codegen id : 16]
+Input [1]: [wp_web_page_sk#76]
+
+(62) Filter [codegen id : 16]
+Input [1]: [wp_web_page_sk#76]
+Condition : isnotnull(wp_web_page_sk#76)
+
+(63) BroadcastExchange
+Input [1]: [wp_web_page_sk#76]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#77]
+
+(64) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_web_page_sk#71]
+Right keys [1]: [wp_web_page_sk#76]
+Join condition: None
+
+(65) Project [codegen id : 17]
+Output [3]: [ws_ext_sales_price#72, ws_net_profit#73, wp_web_page_sk#76]
+Input [4]: [ws_web_page_sk#71, ws_ext_sales_price#72, ws_net_profit#73, wp_web_page_sk#76]
+
+(66) HashAggregate [codegen id : 17]
+Input [3]: [ws_ext_sales_price#72, ws_net_profit#73, wp_web_page_sk#76]
+Keys [1]: [wp_web_page_sk#76]
+Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#72)), partial_sum(UnscaledValue(ws_net_profit#73))]
+Aggregate Attributes [2]: [sum#78, sum#79]
+Results [3]: [wp_web_page_sk#76, sum#80, sum#81]
+
+(67) Exchange
+Input [3]: [wp_web_page_sk#76, sum#80, sum#81]
+Arguments: hashpartitioning(wp_web_page_sk#76, 5), ENSURE_REQUIREMENTS, [id=#82]
+
+(68) HashAggregate [codegen id : 22]
+Input [3]: [wp_web_page_sk#76, sum#80, sum#81]
+Keys [1]: [wp_web_page_sk#76]
+Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#72)), sum(UnscaledValue(ws_net_profit#73))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_sales_price#72))#83, sum(UnscaledValue(ws_net_profit#73))#84]
+Results [3]: [wp_web_page_sk#76, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#72))#83,17,2) AS sales#85, MakeDecimal(sum(UnscaledValue(ws_net_profit#73))#84,17,2) AS profit#86]
+
+(69) Scan parquet default.web_returns
+Output [4]: [wr_web_page_sk#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(wr_returned_date_sk#90), dynamicpruningexpression(wr_returned_date_sk#90 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(wr_web_page_sk)]
+ReadSchema: struct<wr_web_page_sk:int,wr_return_amt:decimal(7,2),wr_net_loss:decimal(7,2)>
+
+(70) ColumnarToRow [codegen id : 20]
+Input [4]: [wr_web_page_sk#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90]
+
+(71) Filter [codegen id : 20]
+Input [4]: [wr_web_page_sk#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90]
+Condition : isnotnull(wr_web_page_sk#87)
+
+(72) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#91]
+
+(73) BroadcastHashJoin [codegen id : 20]
+Left keys [1]: [wr_returned_date_sk#90]
+Right keys [1]: [d_date_sk#91]
+Join condition: None
+
+(74) Project [codegen id : 20]
+Output [3]: [wr_web_page_sk#87, wr_return_amt#88, wr_net_loss#89]
+Input [5]: [wr_web_page_sk#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90, d_date_sk#91]
+
+(75) ReusedExchange [Reuses operator id: 63]
+Output [1]: [wp_web_page_sk#92]
+
+(76) BroadcastHashJoin [codegen id : 20]
+Left keys [1]: [wr_web_page_sk#87]
+Right keys [1]: [wp_web_page_sk#92]
+Join condition: None
+
+(77) Project [codegen id : 20]
+Output [3]: [wr_return_amt#88, wr_net_loss#89, wp_web_page_sk#92]
+Input [4]: [wr_web_page_sk#87, wr_return_amt#88, wr_net_loss#89, wp_web_page_sk#92]
+
+(78) HashAggregate [codegen id : 20]
+Input [3]: [wr_return_amt#88, wr_net_loss#89, wp_web_page_sk#92]
+Keys [1]: [wp_web_page_sk#92]
+Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#88)), partial_sum(UnscaledValue(wr_net_loss#89))]
+Aggregate Attributes [2]: [sum#93, sum#94]
+Results [3]: [wp_web_page_sk#92, sum#95, sum#96]
+
+(79) Exchange
+Input [3]: [wp_web_page_sk#92, sum#95, sum#96]
+Arguments: hashpartitioning(wp_web_page_sk#92, 5), ENSURE_REQUIREMENTS, [id=#97]
+
+(80) HashAggregate [codegen id : 21]
+Input [3]: [wp_web_page_sk#92, sum#95, sum#96]
+Keys [1]: [wp_web_page_sk#92]
+Functions [2]: [sum(UnscaledValue(wr_return_amt#88)), sum(UnscaledValue(wr_net_loss#89))]
+Aggregate Attributes [2]: [sum(UnscaledValue(wr_return_amt#88))#98, sum(UnscaledValue(wr_net_loss#89))#99]
+Results [3]: [wp_web_page_sk#92, MakeDecimal(sum(UnscaledValue(wr_return_amt#88))#98,17,2) AS returns#100, MakeDecimal(sum(UnscaledValue(wr_net_loss#89))#99,17,2) AS profit_loss#101]
+
+(81) BroadcastExchange
+Input [3]: [wp_web_page_sk#92, returns#100, profit_loss#101]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#102]
+
+(82) BroadcastHashJoin [codegen id : 22]
+Left keys [1]: [wp_web_page_sk#76]
+Right keys [1]: [wp_web_page_sk#92]
+Join condition: None
+
+(83) Project [codegen id : 22]
+Output [5]: [sales#85, coalesce(returns#100, 0.00) AS returns#103, CheckOverflow((promote_precision(cast(profit#86 as decimal(18,2))) - promote_precision(cast(coalesce(profit_loss#101, 0.00) as decimal(18,2)))), DecimalType(18,2), true) AS profit#104, web channel AS channel#105, wp_web_page_sk#76 AS id#106]
+Input [6]: [wp_web_page_sk#76, sales#85, profit#86, wp_web_page_sk#92, returns#100, profit_loss#101]
+
+(84) Union
+
+(85) Expand [codegen id : 23]
+Input [5]: [sales#18, returns#36, profit#37, channel#38, id#39]
+Arguments: [ArrayBuffer(sales#18, returns#36, profit#37, channel#38, id#39, 0), ArrayBuffer(sales#18, returns#36, profit#37, channel#38, null, 1), ArrayBuffer(sales#18, returns#36, profit#37, null, null, 3)], [sales#18, returns#36, profit#37, channel#107, id#108, spark_grouping_id#109]
+
+(86) HashAggregate [codegen id : 23]
+Input [6]: [sales#18, returns#36, profit#37, channel#107, id#108, spark_grouping_id#109]
+Keys [3]: [channel#107, id#108, spark_grouping_id#109]
+Functions [3]: [partial_sum(sales#18), partial_sum(returns#36), partial_sum(profit#37)]
+Aggregate Attributes [6]: [sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115]
+Results [9]: [channel#107, id#108, spark_grouping_id#109, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121]
+
+(87) Exchange
+Input [9]: [channel#107, id#108, spark_grouping_id#109, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121]
+Arguments: hashpartitioning(channel#107, id#108, spark_grouping_id#109, 5), ENSURE_REQUIREMENTS, [id=#122]
+
+(88) HashAggregate [codegen id : 24]
+Input [9]: [channel#107, id#108, spark_grouping_id#109, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121]
+Keys [3]: [channel#107, id#108, spark_grouping_id#109]
+Functions [3]: [sum(sales#18), sum(returns#36), sum(profit#37)]
+Aggregate Attributes [3]: [sum(sales#18)#123, sum(returns#36)#124, sum(profit#37)#125]
+Results [5]: [channel#107, id#108, sum(sales#18)#123 AS sales#126, sum(returns#36)#124 AS returns#127, sum(profit#37)#125 AS profit#128]
+
+(89) TakeOrderedAndProject
+Input [5]: [channel#107, id#108, sales#126, returns#127, profit#128]
+Arguments: 100, [channel#107 ASC NULLS FIRST, id#108 ASC NULLS FIRST], [channel#107, id#108, sales#126, returns#127, profit#128]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (90)
+
+
+(90) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#6]
+
+Subquery:2 Hosting operator id = 20 Hosting Expression = sr_returned_date_sk#23 IN dynamicpruning#5
+
+Subquery:3 Hosting operator id = 35 Hosting Expression = cs_sold_date_sk#43 IN dynamicpruning#5
+
+Subquery:4 Hosting operator id = 44 Hosting Expression = cr_returned_date_sk#57 IN dynamicpruning#5
+
+Subquery:5 Hosting operator id = 54 Hosting Expression = ws_sold_date_sk#74 IN dynamicpruning#5
+
+Subquery:6 Hosting operator id = 69 Hosting Expression = wr_returned_date_sk#90 IN dynamicpruning#5
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/simplified.txt
new file mode 100644
index 0000000000000..7c20631f0086a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/simplified.txt
@@ -0,0 +1,143 @@
+TakeOrderedAndProject [channel,id,sales,returns,profit]
+  WholeStageCodegen (24)
+    HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+      InputAdapter
+        Exchange [channel,id,spark_grouping_id] #1
+          WholeStageCodegen (23)
+            HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+              Expand [sales,returns,profit,channel,id]
+                InputAdapter
+                  Union
+                    WholeStageCodegen (8)
+                      Project [sales,returns,profit,profit_loss,s_store_sk]
+                        BroadcastHashJoin [s_store_sk,s_store_sk]
+                          HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit)),sales,profit,sum,sum]
+                            InputAdapter
+                              Exchange [s_store_sk] #2
+                                WholeStageCodegen (3)
+                                  HashAggregate [s_store_sk,ss_ext_sales_price,ss_net_profit] [sum,sum,sum,sum]
+                                    Project [ss_ext_sales_price,ss_net_profit,s_store_sk]
+                                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                        Project [ss_store_sk,ss_ext_sales_price,ss_net_profit]
+                                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                            Filter [ss_store_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #1
+                                                      ReusedExchange [d_date_sk] #3
+                                            InputAdapter
+                                              BroadcastExchange #3
+                                                WholeStageCodegen (1)
+                                                  Project [d_date_sk]
+                                                    Filter [d_date,d_date_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.date_dim [d_date_sk,d_date]
+                                        InputAdapter
+                                          BroadcastExchange #4
+                                            WholeStageCodegen (2)
+                                              Filter [s_store_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.store [s_store_sk]
+                          InputAdapter
+                            BroadcastExchange #5
+                              WholeStageCodegen (7)
+                                HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss)),returns,profit_loss,sum,sum]
+                                  InputAdapter
+                                    Exchange [s_store_sk] #6
+                                      WholeStageCodegen (6)
+                                        HashAggregate [s_store_sk,sr_return_amt,sr_net_loss] [sum,sum,sum,sum]
+                                          Project [sr_return_amt,sr_net_loss,s_store_sk]
+                                            BroadcastHashJoin [sr_store_sk,s_store_sk]
+                                              Project [sr_store_sk,sr_return_amt,sr_net_loss]
+                                                BroadcastHashJoin [sr_returned_date_sk,d_date_sk]
+                                                  Filter [sr_store_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk]
+                                                          ReusedSubquery [d_date_sk] #1
+                                                  InputAdapter
+                                                    ReusedExchange [d_date_sk] #3
+                                              InputAdapter
+                                                ReusedExchange [s_store_sk] #4
+                    WholeStageCodegen (14)
+                      Project [sales,returns,profit,profit_loss,cs_call_center_sk]
+                        BroadcastNestedLoopJoin
+                          InputAdapter
+                            BroadcastExchange #7
+                              WholeStageCodegen (11)
+                                HashAggregate [cs_call_center_sk,sum,sum] [sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit)),sales,profit,sum,sum]
+                                  InputAdapter
+                                    Exchange [cs_call_center_sk] #8
+                                      WholeStageCodegen (10)
+                                        HashAggregate [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] [sum,sum,sum,sum]
+                                          Project [cs_call_center_sk,cs_ext_sales_price,cs_net_profit]
+                                            BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk]
+                                                    ReusedSubquery [d_date_sk] #1
+                                              InputAdapter
+                                                ReusedExchange [d_date_sk] #3
+                          HashAggregate [sum,sum] [sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss)),returns,profit_loss,sum,sum]
+                            InputAdapter
+                              Exchange #9
+                                WholeStageCodegen (13)
+                                  HashAggregate [cr_return_amount,cr_net_loss] [sum,sum,sum,sum]
+                                    Project [cr_return_amount,cr_net_loss]
+                                      BroadcastHashJoin [cr_returned_date_sk,d_date_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk]
+                                              ReusedSubquery [d_date_sk] #1
+                                        InputAdapter
+                                          ReusedExchange [d_date_sk] #3
+                    WholeStageCodegen (22)
+                      Project [sales,returns,profit,profit_loss,wp_web_page_sk]
+                        BroadcastHashJoin [wp_web_page_sk,wp_web_page_sk]
+                          HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit)),sales,profit,sum,sum]
+                            InputAdapter
+                              Exchange [wp_web_page_sk] #10
+                                WholeStageCodegen (17)
+                                  HashAggregate [wp_web_page_sk,ws_ext_sales_price,ws_net_profit] [sum,sum,sum,sum]
+                                    Project [ws_ext_sales_price,ws_net_profit,wp_web_page_sk]
+                                      BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk]
+                                        Project [ws_web_page_sk,ws_ext_sales_price,ws_net_profit]
+                                          BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                            Filter [ws_web_page_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk]
+                                                    ReusedSubquery [d_date_sk] #1
+                                            InputAdapter
+                                              ReusedExchange [d_date_sk] #3
+                                        InputAdapter
+                                          BroadcastExchange #11
+                                            WholeStageCodegen (16)
+                                              Filter [wp_web_page_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.web_page [wp_web_page_sk]
+                          InputAdapter
+                            BroadcastExchange #12
+                              WholeStageCodegen (21)
+                                HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss)),returns,profit_loss,sum,sum]
+                                  InputAdapter
+                                    Exchange [wp_web_page_sk] #13
+                                      WholeStageCodegen (20)
+                                        HashAggregate [wp_web_page_sk,wr_return_amt,wr_net_loss] [sum,sum,sum,sum]
+                                          Project [wr_return_amt,wr_net_loss,wp_web_page_sk]
+                                            BroadcastHashJoin [wr_web_page_sk,wp_web_page_sk]
+                                              Project [wr_web_page_sk,wr_return_amt,wr_net_loss]
+                                                BroadcastHashJoin [wr_returned_date_sk,d_date_sk]
+                                                  Filter [wr_web_page_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk]
+                                                          ReusedSubquery [d_date_sk] #1
+                                                  InputAdapter
+                                                    ReusedExchange [d_date_sk] #3
+                                              InputAdapter
+                                                ReusedExchange [wp_web_page_sk] #11
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.sf100/explain.txt
new file mode 100644
index 0000000000000..5ed24863c0e1d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.sf100/explain.txt
@@ -0,0 +1,423 @@
+== Physical Plan ==
+TakeOrderedAndProject (73)
++- * Project (72)
+   +- * SortMergeJoin Inner (71)
+      :- * Project (48)
+      :  +- * SortMergeJoin Inner (47)
+      :     :- * Sort (24)
+      :     :  +- * HashAggregate (23)
+      :     :     +- Exchange (22)
+      :     :        +- * HashAggregate (21)
+      :     :           +- * Project (20)
+      :     :              +- * BroadcastHashJoin Inner BuildRight (19)
+      :     :                 :- * Project (14)
+      :     :                 :  +- * Filter (13)
+      :     :                 :     +- * SortMergeJoin LeftOuter (12)
+      :     :                 :        :- * Sort (5)
+      :     :                 :        :  +- Exchange (4)
+      :     :                 :        :     +- * Filter (3)
+      :     :                 :        :        +- * ColumnarToRow (2)
+      :     :                 :        :           +- Scan parquet default.store_sales (1)
+      :     :                 :        +- * Sort (11)
+      :     :                 :           +- Exchange (10)
+      :     :                 :              +- * Project (9)
+      :     :                 :                 +- * Filter (8)
+      :     :                 :                    +- * ColumnarToRow (7)
+      :     :                 :                       +- Scan parquet default.store_returns (6)
+      :     :                 +- BroadcastExchange (18)
+      :     :                    +- * Filter (17)
+      :     :                       +- * ColumnarToRow (16)
+      :     :                          +- Scan parquet default.date_dim (15)
+      :     +- * Sort (46)
+      :        +- * Filter (45)
+      :           +- * HashAggregate (44)
+      :              +- Exchange (43)
+      :                 +- * HashAggregate (42)
+      :                    +- * Project (41)
+      :                       +- * BroadcastHashJoin Inner BuildRight (40)
+      :                          :- * Project (38)
+      :                          :  +- * Filter (37)
+      :                          :     +- * SortMergeJoin LeftOuter (36)
+      :                          :        :- * Sort (29)
+      :                          :        :  +- Exchange (28)
+      :                          :        :     +- * Filter (27)
+      :                          :        :        +- * ColumnarToRow (26)
+      :                          :        :           +- Scan parquet default.web_sales (25)
+      :                          :        +- * Sort (35)
+      :                          :           +- Exchange (34)
+      :                          :              +- * Project (33)
+      :                          :                 +- * Filter (32)
+      :                          :                    +- * ColumnarToRow (31)
+      :                          :                       +- Scan parquet default.web_returns (30)
+      :                          +- ReusedExchange (39)
+      +- * Sort (70)
+         +- * Filter (69)
+            +- * HashAggregate (68)
+               +- Exchange (67)
+                  +- * HashAggregate (66)
+                     +- * Project (65)
+                        +- * BroadcastHashJoin Inner BuildRight (64)
+                           :- * Project (62)
+                           :  +- * Filter (61)
+                           :     +- * SortMergeJoin LeftOuter (60)
+                           :        :- * Sort (53)
+                           :        :  +- Exchange (52)
+                           :        :     +- * Filter (51)
+                           :        :        +- * ColumnarToRow (50)
+                           :        :           +- Scan parquet default.catalog_sales (49)
+                           :        +- * Sort (59)
+                           :           +- Exchange (58)
+                           :              +- * Project (57)
+                           :                 +- * Filter (56)
+                           :                    +- * ColumnarToRow (55)
+                           :                       +- Scan parquet default.catalog_returns (54)
+                           +- ReusedExchange (63)
+
+
+(1) Scan parquet default.store_sales
+Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_ticket_number:int,ss_quantity:int,ss_wholesale_cost:decimal(7,2),ss_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7]
+
+(3) Filter [codegen id : 1]
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7]
+Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_customer_sk#2))
+
+(4) Exchange
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7]
+Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#9]
+
+(5) Sort [codegen id : 2]
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7]
+Arguments: [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.store_returns
+Output [3]: [sr_item_sk#10, sr_ticket_number#11, sr_returned_date_sk#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int>
+
+(7) ColumnarToRow [codegen id : 3]
+Input [3]: [sr_item_sk#10, sr_ticket_number#11, sr_returned_date_sk#12]
+
+(8) Filter [codegen id : 3]
+Input [3]: [sr_item_sk#10, sr_ticket_number#11, sr_returned_date_sk#12]
+Condition : (isnotnull(sr_ticket_number#11) AND isnotnull(sr_item_sk#10))
+
+(9) Project [codegen id : 3]
+Output [2]: [sr_item_sk#10, sr_ticket_number#11]
+Input [3]: [sr_item_sk#10, sr_ticket_number#11, sr_returned_date_sk#12]
+
+(10) Exchange
+Input [2]: [sr_item_sk#10, sr_ticket_number#11]
+Arguments: hashpartitioning(sr_ticket_number#11, sr_item_sk#10, 5), ENSURE_REQUIREMENTS, [id=#13]
+
+(11) Sort [codegen id : 4]
+Input [2]: [sr_item_sk#10, sr_ticket_number#11]
+Arguments: [sr_ticket_number#11 ASC NULLS FIRST, sr_item_sk#10 ASC NULLS FIRST], false, 0
+
+(12) SortMergeJoin [codegen id : 6]
+Left keys [2]: [ss_ticket_number#3, ss_item_sk#1]
+Right keys [2]: [sr_ticket_number#11, sr_item_sk#10]
+Join condition: None
+
+(13) Filter [codegen id : 6]
+Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#10, sr_ticket_number#11]
+Condition : isnull(sr_ticket_number#11)
+
+(14) Project [codegen id : 6]
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7]
+Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#10, sr_ticket_number#11]
+
+(15) Scan parquet default.date_dim
+Output [2]: [d_date_sk#14, d_year#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(16) ColumnarToRow [codegen id : 5]
+Input [2]: [d_date_sk#14, d_year#15]
+
+(17) Filter [codegen id : 5]
+Input [2]: [d_date_sk#14, d_year#15]
+Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2000)) AND isnotnull(d_date_sk#14))
+
+(18) BroadcastExchange
+Input [2]: [d_date_sk#14, d_year#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#16]
+
+(19) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_sold_date_sk#7]
+Right keys [1]: [d_date_sk#14]
+Join condition: None
+
+(20) Project [codegen id : 6]
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#15]
+Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#14, d_year#15]
+
+(21) HashAggregate [codegen id : 6]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#15]
+Keys [3]: [d_year#15, ss_item_sk#1, ss_customer_sk#2]
+Functions [3]: [partial_sum(ss_quantity#4), partial_sum(UnscaledValue(ss_wholesale_cost#5)), partial_sum(UnscaledValue(ss_sales_price#6))]
+Aggregate Attributes [3]: [sum#17, sum#18, sum#19]
+Results [6]: [d_year#15, ss_item_sk#1, ss_customer_sk#2, sum#20, sum#21, sum#22]
+
+(22) Exchange
+Input [6]: [d_year#15, ss_item_sk#1, ss_customer_sk#2, sum#20, sum#21, sum#22]
+Arguments: hashpartitioning(d_year#15, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(23) HashAggregate [codegen id : 7]
+Input [6]: [d_year#15, ss_item_sk#1, ss_customer_sk#2, sum#20, sum#21, sum#22]
+Keys [3]: [d_year#15, ss_item_sk#1, ss_customer_sk#2]
+Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum(UnscaledValue(ss_sales_price#6))]
+Aggregate Attributes [3]: [sum(ss_quantity#4)#24, sum(UnscaledValue(ss_wholesale_cost#5))#25, sum(UnscaledValue(ss_sales_price#6))#26]
+Results [6]: [d_year#15 AS ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2, sum(ss_quantity#4)#24 AS ss_qty#28, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#5))#25,17,2) AS ss_wc#29, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#26,17,2) AS ss_sp#30]
+
+(24) Sort [codegen id : 7]
+Input [6]: [ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2, ss_qty#28, ss_wc#29, ss_sp#30]
+Arguments: [ss_sold_year#27 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], false, 0
+
+(25) Scan parquet default.web_sales
+Output [7]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#37), dynamicpruningexpression(ws_sold_date_sk#37 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_bill_customer_sk:int,ws_order_number:int,ws_quantity:int,ws_wholesale_cost:decimal(7,2),ws_sales_price:decimal(7,2)>
+
+(26) ColumnarToRow [codegen id : 8]
+Input [7]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37]
+
+(27) Filter [codegen id : 8]
+Input [7]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37]
+Condition : (isnotnull(ws_item_sk#31) AND isnotnull(ws_bill_customer_sk#32))
+
+(28) Exchange
+Input [7]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37]
+Arguments: hashpartitioning(ws_order_number#33, ws_item_sk#31, 5), ENSURE_REQUIREMENTS, [id=#38]
+
+(29) Sort [codegen id : 9]
+Input [7]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37]
+Arguments: [ws_order_number#33 ASC NULLS FIRST, ws_item_sk#31 ASC NULLS FIRST], false, 0
+
+(30) Scan parquet default.web_returns
+Output [3]: [wr_item_sk#39, wr_order_number#40, wr_returned_date_sk#41]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_returns]
+PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)]
+ReadSchema: struct<wr_item_sk:int,wr_order_number:int>
+
+(31) ColumnarToRow [codegen id : 10]
+Input [3]: [wr_item_sk#39, wr_order_number#40, wr_returned_date_sk#41]
+
+(32) Filter [codegen id : 10]
+Input [3]: [wr_item_sk#39, wr_order_number#40, wr_returned_date_sk#41]
+Condition : (isnotnull(wr_order_number#40) AND isnotnull(wr_item_sk#39))
+
+(33) Project [codegen id : 10]
+Output [2]: [wr_item_sk#39, wr_order_number#40]
+Input [3]: [wr_item_sk#39, wr_order_number#40, wr_returned_date_sk#41]
+
+(34) Exchange
+Input [2]: [wr_item_sk#39, wr_order_number#40]
+Arguments: hashpartitioning(wr_order_number#40, wr_item_sk#39, 5), ENSURE_REQUIREMENTS, [id=#42]
+
+(35) Sort [codegen id : 11]
+Input [2]: [wr_item_sk#39, wr_order_number#40]
+Arguments: [wr_order_number#40 ASC NULLS FIRST, wr_item_sk#39 ASC NULLS FIRST], false, 0
+
+(36) SortMergeJoin [codegen id : 13]
+Left keys [2]: [ws_order_number#33, ws_item_sk#31]
+Right keys [2]: [wr_order_number#40, wr_item_sk#39]
+Join condition: None
+
+(37) Filter [codegen id : 13]
+Input [9]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37, wr_item_sk#39, wr_order_number#40]
+Condition : isnull(wr_order_number#40)
+
+(38) Project [codegen id : 13]
+Output [6]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37]
+Input [9]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37, wr_item_sk#39, wr_order_number#40]
+
+(39) ReusedExchange [Reuses operator id: 18]
+Output [2]: [d_date_sk#43, d_year#44]
+
+(40) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ws_sold_date_sk#37]
+Right keys [1]: [d_date_sk#43]
+Join condition: None
+
+(41) Project [codegen id : 13]
+Output [6]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, d_year#44]
+Input [8]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37, d_date_sk#43, d_year#44]
+
+(42) HashAggregate [codegen id : 13]
+Input [6]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, d_year#44]
+Keys [3]: [d_year#44, ws_item_sk#31, ws_bill_customer_sk#32]
+Functions [3]: [partial_sum(ws_quantity#34), partial_sum(UnscaledValue(ws_wholesale_cost#35)), partial_sum(UnscaledValue(ws_sales_price#36))]
+Aggregate Attributes [3]: [sum#45, sum#46, sum#47]
+Results [6]: [d_year#44, ws_item_sk#31, ws_bill_customer_sk#32, sum#48, sum#49, sum#50]
+
+(43) Exchange
+Input [6]: [d_year#44, ws_item_sk#31, ws_bill_customer_sk#32, sum#48, sum#49, sum#50]
+Arguments: hashpartitioning(d_year#44, ws_item_sk#31, ws_bill_customer_sk#32, 5), ENSURE_REQUIREMENTS, [id=#51]
+
+(44) HashAggregate [codegen id : 14]
+Input [6]: [d_year#44, ws_item_sk#31, ws_bill_customer_sk#32, sum#48, sum#49, sum#50]
+Keys [3]: [d_year#44, ws_item_sk#31, ws_bill_customer_sk#32]
+Functions [3]: [sum(ws_quantity#34), sum(UnscaledValue(ws_wholesale_cost#35)), sum(UnscaledValue(ws_sales_price#36))]
+Aggregate Attributes [3]: [sum(ws_quantity#34)#52, sum(UnscaledValue(ws_wholesale_cost#35))#53, sum(UnscaledValue(ws_sales_price#36))#54]
+Results [6]: [d_year#44 AS ws_sold_year#55, ws_item_sk#31, ws_bill_customer_sk#32 AS ws_customer_sk#56, sum(ws_quantity#34)#52 AS ws_qty#57, MakeDecimal(sum(UnscaledValue(ws_wholesale_cost#35))#53,17,2) AS ws_wc#58, MakeDecimal(sum(UnscaledValue(ws_sales_price#36))#54,17,2) AS ws_sp#59]
+
+(45) Filter [codegen id : 14]
+Input [6]: [ws_sold_year#55, ws_item_sk#31, ws_customer_sk#56, ws_qty#57, ws_wc#58, ws_sp#59]
+Condition : (coalesce(ws_qty#57, 0) > 0)
+
+(46) Sort [codegen id : 14]
+Input [6]: [ws_sold_year#55, ws_item_sk#31, ws_customer_sk#56, ws_qty#57, ws_wc#58, ws_sp#59]
+Arguments: [ws_sold_year#55 ASC NULLS FIRST, ws_item_sk#31 ASC NULLS FIRST, ws_customer_sk#56 ASC NULLS FIRST], false, 0
+
+(47) SortMergeJoin [codegen id : 15]
+Left keys [3]: [ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2]
+Right keys [3]: [ws_sold_year#55, ws_item_sk#31, ws_customer_sk#56]
+Join condition: None
+
+(48) Project [codegen id : 15]
+Output [9]: [ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2, ss_qty#28, ss_wc#29, ss_sp#30, ws_qty#57, ws_wc#58, ws_sp#59]
+Input [12]: [ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2, ss_qty#28, ss_wc#29, ss_sp#30, ws_sold_year#55, ws_item_sk#31, ws_customer_sk#56, ws_qty#57, ws_wc#58, ws_sp#59]
+
+(49) Scan parquet default.catalog_sales
+Output [7]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#66), dynamicpruningexpression(cs_sold_date_sk#66 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_item_sk:int,cs_order_number:int,cs_quantity:int,cs_wholesale_cost:decimal(7,2),cs_sales_price:decimal(7,2)>
+
+(50) ColumnarToRow [codegen id : 16]
+Input [7]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66]
+
+(51) Filter [codegen id : 16]
+Input [7]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66]
+Condition : (isnotnull(cs_item_sk#61) AND isnotnull(cs_bill_customer_sk#60))
+
+(52) Exchange
+Input [7]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66]
+Arguments: hashpartitioning(cs_order_number#62, cs_item_sk#61, 5), ENSURE_REQUIREMENTS, [id=#67]
+
+(53) Sort [codegen id : 17]
+Input [7]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66]
+Arguments: [cs_order_number#62 ASC NULLS FIRST, cs_item_sk#61 ASC NULLS FIRST], false, 0
+
+(54) Scan parquet default.catalog_returns
+Output [3]: [cr_item_sk#68, cr_order_number#69, cr_returned_date_sk#70]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int>
+
+(55) ColumnarToRow [codegen id : 18]
+Input [3]: [cr_item_sk#68, cr_order_number#69, cr_returned_date_sk#70]
+
+(56) Filter [codegen id : 18]
+Input [3]: [cr_item_sk#68, cr_order_number#69, cr_returned_date_sk#70]
+Condition : (isnotnull(cr_order_number#69) AND isnotnull(cr_item_sk#68))
+
+(57) Project [codegen id : 18]
+Output [2]: [cr_item_sk#68, cr_order_number#69]
+Input [3]: [cr_item_sk#68, cr_order_number#69, cr_returned_date_sk#70]
+
+(58) Exchange
+Input [2]: [cr_item_sk#68, cr_order_number#69]
+Arguments: hashpartitioning(cr_order_number#69, cr_item_sk#68, 5), ENSURE_REQUIREMENTS, [id=#71]
+
+(59) Sort [codegen id : 19]
+Input [2]: [cr_item_sk#68, cr_order_number#69]
+Arguments: [cr_order_number#69 ASC NULLS FIRST, cr_item_sk#68 ASC NULLS FIRST], false, 0
+
+(60) SortMergeJoin [codegen id : 21]
+Left keys [2]: [cs_order_number#62, cs_item_sk#61]
+Right keys [2]: [cr_order_number#69, cr_item_sk#68]
+Join condition: None
+
+(61) Filter [codegen id : 21]
+Input [9]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66, cr_item_sk#68, cr_order_number#69]
+Condition : isnull(cr_order_number#69)
+
+(62) Project [codegen id : 21]
+Output [6]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66]
+Input [9]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66, cr_item_sk#68, cr_order_number#69]
+
+(63) ReusedExchange [Reuses operator id: 18]
+Output [2]: [d_date_sk#72, d_year#73]
+
+(64) BroadcastHashJoin [codegen id : 21]
+Left keys [1]: [cs_sold_date_sk#66]
+Right keys [1]: [d_date_sk#72]
+Join condition: None
+
+(65) Project [codegen id : 21]
+Output [6]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, d_year#73]
+Input [8]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66, d_date_sk#72, d_year#73]
+
+(66) HashAggregate [codegen id : 21]
+Input [6]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, d_year#73]
+Keys [3]: [d_year#73, cs_item_sk#61, cs_bill_customer_sk#60]
+Functions [3]: [partial_sum(cs_quantity#63), partial_sum(UnscaledValue(cs_wholesale_cost#64)), partial_sum(UnscaledValue(cs_sales_price#65))]
+Aggregate Attributes [3]: [sum#74, sum#75, sum#76]
+Results [6]: [d_year#73, cs_item_sk#61, cs_bill_customer_sk#60, sum#77, sum#78, sum#79]
+
+(67) Exchange
+Input [6]: [d_year#73, cs_item_sk#61, cs_bill_customer_sk#60, sum#77, sum#78, sum#79]
+Arguments: hashpartitioning(d_year#73, cs_item_sk#61, cs_bill_customer_sk#60, 5), ENSURE_REQUIREMENTS, [id=#80]
+
+(68) HashAggregate [codegen id : 22]
+Input [6]: [d_year#73, cs_item_sk#61, cs_bill_customer_sk#60, sum#77, sum#78, sum#79]
+Keys [3]: [d_year#73, cs_item_sk#61, cs_bill_customer_sk#60]
+Functions [3]: [sum(cs_quantity#63), sum(UnscaledValue(cs_wholesale_cost#64)), sum(UnscaledValue(cs_sales_price#65))]
+Aggregate Attributes [3]: [sum(cs_quantity#63)#81, sum(UnscaledValue(cs_wholesale_cost#64))#82, sum(UnscaledValue(cs_sales_price#65))#83]
+Results [6]: [d_year#73 AS cs_sold_year#84, cs_item_sk#61, cs_bill_customer_sk#60 AS cs_customer_sk#85, sum(cs_quantity#63)#81 AS cs_qty#86, MakeDecimal(sum(UnscaledValue(cs_wholesale_cost#64))#82,17,2) AS cs_wc#87, MakeDecimal(sum(UnscaledValue(cs_sales_price#65))#83,17,2) AS cs_sp#88]
+
+(69) Filter [codegen id : 22]
+Input [6]: [cs_sold_year#84, cs_item_sk#61, cs_customer_sk#85, cs_qty#86, cs_wc#87, cs_sp#88]
+Condition : (coalesce(cs_qty#86, 0) > 0)
+
+(70) Sort [codegen id : 22]
+Input [6]: [cs_sold_year#84, cs_item_sk#61, cs_customer_sk#85, cs_qty#86, cs_wc#87, cs_sp#88]
+Arguments: [cs_sold_year#84 ASC NULLS FIRST, cs_item_sk#61 ASC NULLS FIRST, cs_customer_sk#85 ASC NULLS FIRST], false, 0
+
+(71) SortMergeJoin [codegen id : 23]
+Left keys [3]: [ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2]
+Right keys [3]: [cs_sold_year#84, cs_item_sk#61, cs_customer_sk#85]
+Join condition: None
+
+(72) Project [codegen id : 23]
+Output [12]: [round((cast(ss_qty#28 as double) / cast(coalesce((ws_qty#57 + cs_qty#86), 1) as double)), 2) AS ratio#89, ss_qty#28 AS store_qty#90, ss_wc#29 AS store_wholesale_cost#91, ss_sp#30 AS store_sales_price#92, (coalesce(ws_qty#57, 0) + coalesce(cs_qty#86, 0)) AS other_chan_qty#93, CheckOverflow((promote_precision(cast(coalesce(ws_wc#58, 0.00) as decimal(18,2))) + promote_precision(cast(coalesce(cs_wc#87, 0.00) as decimal(18,2)))), DecimalType(18,2), true) AS other_chan_wholesale_cost#94, CheckOverflow((promote_precision(cast(coalesce(ws_sp#59, 0.00) as decimal(18,2))) + promote_precision(cast(coalesce(cs_sp#88, 0.00) as decimal(18,2)))), DecimalType(18,2), true) AS other_chan_sales_price#95, ss_qty#28, ss_wc#29, ss_sp#30, ws_qty#57, cs_qty#86]
+Input [15]: [ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2, ss_qty#28, ss_wc#29, ss_sp#30, ws_qty#57, ws_wc#58, ws_sp#59, cs_sold_year#84, cs_item_sk#61, cs_customer_sk#85, cs_qty#86, cs_wc#87, cs_sp#88]
+
+(73) TakeOrderedAndProject
+Input [12]: [ratio#89, store_qty#90, store_wholesale_cost#91, store_sales_price#92, other_chan_qty#93, other_chan_wholesale_cost#94, other_chan_sales_price#95, ss_qty#28, ss_wc#29, ss_sp#30, ws_qty#57, cs_qty#86]
+Arguments: 100, [ratio#89 ASC NULLS FIRST, ss_qty#28 DESC NULLS LAST, ss_wc#29 DESC NULLS LAST, ss_sp#30 DESC NULLS LAST, other_chan_qty#93 ASC NULLS FIRST, other_chan_wholesale_cost#94 ASC NULLS FIRST, other_chan_sales_price#95 ASC NULLS FIRST, round((cast(ss_qty#28 as double) / cast(coalesce((ws_qty#57 + cs_qty#86), 1) as double)), 2) ASC NULLS FIRST], [ratio#89, store_qty#90, store_wholesale_cost#91, store_sales_price#92, other_chan_qty#93, other_chan_wholesale_cost#94, other_chan_sales_price#95]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (74)
+
+
+(74) ReusedExchange [Reuses operator id: 18]
+Output [2]: [d_date_sk#14, d_year#15]
+
+Subquery:2 Hosting operator id = 25 Hosting Expression = ws_sold_date_sk#37 IN dynamicpruning#8
+
+Subquery:3 Hosting operator id = 49 Hosting Expression = cs_sold_date_sk#66 IN dynamicpruning#8
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.sf100/simplified.txt
new file mode 100644
index 0000000000000..0aa69ab4645e4
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.sf100/simplified.txt
@@ -0,0 +1,127 @@
+TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholesale_cost,other_chan_sales_price,ws_qty,cs_qty,store_qty,store_wholesale_cost,store_sales_price]
+  WholeStageCodegen (23)
+    Project [ss_qty,ws_qty,cs_qty,ss_wc,ss_sp,ws_wc,cs_wc,ws_sp,cs_sp]
+      SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,cs_sold_year,cs_item_sk,cs_customer_sk]
+        InputAdapter
+          WholeStageCodegen (15)
+            Project [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp]
+              SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ws_sold_year,ws_item_sk,ws_customer_sk]
+                InputAdapter
+                  WholeStageCodegen (7)
+                    Sort [ss_sold_year,ss_item_sk,ss_customer_sk]
+                      HashAggregate [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] [sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price)),ss_sold_year,ss_qty,ss_wc,ss_sp,sum,sum,sum]
+                        InputAdapter
+                          Exchange [d_year,ss_item_sk,ss_customer_sk] #1
+                            WholeStageCodegen (6)
+                              HashAggregate [d_year,ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price] [sum,sum,sum,sum,sum,sum]
+                                Project [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year]
+                                  BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                    Project [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk]
+                                      Filter [sr_ticket_number]
+                                        SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                          InputAdapter
+                                            WholeStageCodegen (2)
+                                              Sort [ss_ticket_number,ss_item_sk]
+                                                InputAdapter
+                                                  Exchange [ss_ticket_number,ss_item_sk] #2
+                                                    WholeStageCodegen (1)
+                                                      Filter [ss_item_sk,ss_customer_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk]
+                                                              SubqueryBroadcast [d_date_sk] #1
+                                                                ReusedExchange [d_date_sk,d_year] #3
+                                          InputAdapter
+                                            WholeStageCodegen (4)
+                                              Sort [sr_ticket_number,sr_item_sk]
+                                                InputAdapter
+                                                  Exchange [sr_ticket_number,sr_item_sk] #4
+                                                    WholeStageCodegen (3)
+                                                      Project [sr_item_sk,sr_ticket_number]
+                                                        Filter [sr_ticket_number,sr_item_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk]
+                                    InputAdapter
+                                      BroadcastExchange #3
+                                        WholeStageCodegen (5)
+                                          Filter [d_year,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_year]
+                InputAdapter
+                  WholeStageCodegen (14)
+                    Sort [ws_sold_year,ws_item_sk,ws_customer_sk]
+                      Filter [ws_qty]
+                        HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] [sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price)),ws_sold_year,ws_customer_sk,ws_qty,ws_wc,ws_sp,sum,sum,sum]
+                          InputAdapter
+                            Exchange [d_year,ws_item_sk,ws_bill_customer_sk] #5
+                              WholeStageCodegen (13)
+                                HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price] [sum,sum,sum,sum,sum,sum]
+                                  Project [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year]
+                                    BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                      Project [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk]
+                                        Filter [wr_order_number]
+                                          SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk]
+                                            InputAdapter
+                                              WholeStageCodegen (9)
+                                                Sort [ws_order_number,ws_item_sk]
+                                                  InputAdapter
+                                                    Exchange [ws_order_number,ws_item_sk] #6
+                                                      WholeStageCodegen (8)
+                                                        Filter [ws_item_sk,ws_bill_customer_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk]
+                                                                ReusedSubquery [d_date_sk] #1
+                                            InputAdapter
+                                              WholeStageCodegen (11)
+                                                Sort [wr_order_number,wr_item_sk]
+                                                  InputAdapter
+                                                    Exchange [wr_order_number,wr_item_sk] #7
+                                                      WholeStageCodegen (10)
+                                                        Project [wr_item_sk,wr_order_number]
+                                                          Filter [wr_order_number,wr_item_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk]
+                                      InputAdapter
+                                        ReusedExchange [d_date_sk,d_year] #3
+        InputAdapter
+          WholeStageCodegen (22)
+            Sort [cs_sold_year,cs_item_sk,cs_customer_sk]
+              Filter [cs_qty]
+                HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] [sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price)),cs_sold_year,cs_customer_sk,cs_qty,cs_wc,cs_sp,sum,sum,sum]
+                  InputAdapter
+                    Exchange [d_year,cs_item_sk,cs_bill_customer_sk] #8
+                      WholeStageCodegen (21)
+                        HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,cs_quantity,cs_wholesale_cost,cs_sales_price] [sum,sum,sum,sum,sum,sum]
+                          Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year]
+                            BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                              Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk]
+                                Filter [cr_order_number]
+                                  SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk]
+                                    InputAdapter
+                                      WholeStageCodegen (17)
+                                        Sort [cs_order_number,cs_item_sk]
+                                          InputAdapter
+                                            Exchange [cs_order_number,cs_item_sk] #9
+                                              WholeStageCodegen (16)
+                                                Filter [cs_item_sk,cs_bill_customer_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk]
+                                                        ReusedSubquery [d_date_sk] #1
+                                    InputAdapter
+                                      WholeStageCodegen (19)
+                                        Sort [cr_order_number,cr_item_sk]
+                                          InputAdapter
+                                            Exchange [cr_order_number,cr_item_sk] #10
+                                              WholeStageCodegen (18)
+                                                Project [cr_item_sk,cr_order_number]
+                                                  Filter [cr_order_number,cr_item_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk]
+                              InputAdapter
+                                ReusedExchange [d_date_sk,d_year] #3
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/explain.txt
new file mode 100644
index 0000000000000..5ed24863c0e1d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/explain.txt
@@ -0,0 +1,423 @@
+== Physical Plan ==
+TakeOrderedAndProject (73)
++- * Project (72)
+   +- * SortMergeJoin Inner (71)
+      :- * Project (48)
+      :  +- * SortMergeJoin Inner (47)
+      :     :- * Sort (24)
+      :     :  +- * HashAggregate (23)
+      :     :     +- Exchange (22)
+      :     :        +- * HashAggregate (21)
+      :     :           +- * Project (20)
+      :     :              +- * BroadcastHashJoin Inner BuildRight (19)
+      :     :                 :- * Project (14)
+      :     :                 :  +- * Filter (13)
+      :     :                 :     +- * SortMergeJoin LeftOuter (12)
+      :     :                 :        :- * Sort (5)
+      :     :                 :        :  +- Exchange (4)
+      :     :                 :        :     +- * Filter (3)
+      :     :                 :        :        +- * ColumnarToRow (2)
+      :     :                 :        :           +- Scan parquet default.store_sales (1)
+      :     :                 :        +- * Sort (11)
+      :     :                 :           +- Exchange (10)
+      :     :                 :              +- * Project (9)
+      :     :                 :                 +- * Filter (8)
+      :     :                 :                    +- * ColumnarToRow (7)
+      :     :                 :                       +- Scan parquet default.store_returns (6)
+      :     :                 +- BroadcastExchange (18)
+      :     :                    +- * Filter (17)
+      :     :                       +- * ColumnarToRow (16)
+      :     :                          +- Scan parquet default.date_dim (15)
+      :     +- * Sort (46)
+      :        +- * Filter (45)
+      :           +- * HashAggregate (44)
+      :              +- Exchange (43)
+      :                 +- * HashAggregate (42)
+      :                    +- * Project (41)
+      :                       +- * BroadcastHashJoin Inner BuildRight (40)
+      :                          :- * Project (38)
+      :                          :  +- * Filter (37)
+      :                          :     +- * SortMergeJoin LeftOuter (36)
+      :                          :        :- * Sort (29)
+      :                          :        :  +- Exchange (28)
+      :                          :        :     +- * Filter (27)
+      :                          :        :        +- * ColumnarToRow (26)
+      :                          :        :           +- Scan parquet default.web_sales (25)
+      :                          :        +- * Sort (35)
+      :                          :           +- Exchange (34)
+      :                          :              +- * Project (33)
+      :                          :                 +- * Filter (32)
+      :                          :                    +- * ColumnarToRow (31)
+      :                          :                       +- Scan parquet default.web_returns (30)
+      :                          +- ReusedExchange (39)
+      +- * Sort (70)
+         +- * Filter (69)
+            +- * HashAggregate (68)
+               +- Exchange (67)
+                  +- * HashAggregate (66)
+                     +- * Project (65)
+                        +- * BroadcastHashJoin Inner BuildRight (64)
+                           :- * Project (62)
+                           :  +- * Filter (61)
+                           :     +- * SortMergeJoin LeftOuter (60)
+                           :        :- * Sort (53)
+                           :        :  +- Exchange (52)
+                           :        :     +- * Filter (51)
+                           :        :        +- * ColumnarToRow (50)
+                           :        :           +- Scan parquet default.catalog_sales (49)
+                           :        +- * Sort (59)
+                           :           +- Exchange (58)
+                           :              +- * Project (57)
+                           :                 +- * Filter (56)
+                           :                    +- * ColumnarToRow (55)
+                           :                       +- Scan parquet default.catalog_returns (54)
+                           +- ReusedExchange (63)
+
+
+(1) Scan parquet default.store_sales
+Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_ticket_number:int,ss_quantity:int,ss_wholesale_cost:decimal(7,2),ss_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7]
+
+(3) Filter [codegen id : 1]
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7]
+Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_customer_sk#2))
+
+(4) Exchange
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7]
+Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#9]
+
+(5) Sort [codegen id : 2]
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7]
+Arguments: [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.store_returns
+Output [3]: [sr_item_sk#10, sr_ticket_number#11, sr_returned_date_sk#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int>
+
+(7) ColumnarToRow [codegen id : 3]
+Input [3]: [sr_item_sk#10, sr_ticket_number#11, sr_returned_date_sk#12]
+
+(8) Filter [codegen id : 3]
+Input [3]: [sr_item_sk#10, sr_ticket_number#11, sr_returned_date_sk#12]
+Condition : (isnotnull(sr_ticket_number#11) AND isnotnull(sr_item_sk#10))
+
+(9) Project [codegen id : 3]
+Output [2]: [sr_item_sk#10, sr_ticket_number#11]
+Input [3]: [sr_item_sk#10, sr_ticket_number#11, sr_returned_date_sk#12]
+
+(10) Exchange
+Input [2]: [sr_item_sk#10, sr_ticket_number#11]
+Arguments: hashpartitioning(sr_ticket_number#11, sr_item_sk#10, 5), ENSURE_REQUIREMENTS, [id=#13]
+
+(11) Sort [codegen id : 4]
+Input [2]: [sr_item_sk#10, sr_ticket_number#11]
+Arguments: [sr_ticket_number#11 ASC NULLS FIRST, sr_item_sk#10 ASC NULLS FIRST], false, 0
+
+(12) SortMergeJoin [codegen id : 6]
+Left keys [2]: [ss_ticket_number#3, ss_item_sk#1]
+Right keys [2]: [sr_ticket_number#11, sr_item_sk#10]
+Join condition: None
+
+(13) Filter [codegen id : 6]
+Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#10, sr_ticket_number#11]
+Condition : isnull(sr_ticket_number#11)
+
+(14) Project [codegen id : 6]
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7]
+Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#10, sr_ticket_number#11]
+
+(15) Scan parquet default.date_dim
+Output [2]: [d_date_sk#14, d_year#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(16) ColumnarToRow [codegen id : 5]
+Input [2]: [d_date_sk#14, d_year#15]
+
+(17) Filter [codegen id : 5]
+Input [2]: [d_date_sk#14, d_year#15]
+Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2000)) AND isnotnull(d_date_sk#14))
+
+(18) BroadcastExchange
+Input [2]: [d_date_sk#14, d_year#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#16]
+
+(19) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_sold_date_sk#7]
+Right keys [1]: [d_date_sk#14]
+Join condition: None
+
+(20) Project [codegen id : 6]
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#15]
+Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#14, d_year#15]
+
+(21) HashAggregate [codegen id : 6]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#15]
+Keys [3]: [d_year#15, ss_item_sk#1, ss_customer_sk#2]
+Functions [3]: [partial_sum(ss_quantity#4), partial_sum(UnscaledValue(ss_wholesale_cost#5)), partial_sum(UnscaledValue(ss_sales_price#6))]
+Aggregate Attributes [3]: [sum#17, sum#18, sum#19]
+Results [6]: [d_year#15, ss_item_sk#1, ss_customer_sk#2, sum#20, sum#21, sum#22]
+
+(22) Exchange
+Input [6]: [d_year#15, ss_item_sk#1, ss_customer_sk#2, sum#20, sum#21, sum#22]
+Arguments: hashpartitioning(d_year#15, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(23) HashAggregate [codegen id : 7]
+Input [6]: [d_year#15, ss_item_sk#1, ss_customer_sk#2, sum#20, sum#21, sum#22]
+Keys [3]: [d_year#15, ss_item_sk#1, ss_customer_sk#2]
+Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum(UnscaledValue(ss_sales_price#6))]
+Aggregate Attributes [3]: [sum(ss_quantity#4)#24, sum(UnscaledValue(ss_wholesale_cost#5))#25, sum(UnscaledValue(ss_sales_price#6))#26]
+Results [6]: [d_year#15 AS ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2, sum(ss_quantity#4)#24 AS ss_qty#28, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#5))#25,17,2) AS ss_wc#29, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#26,17,2) AS ss_sp#30]
+
+(24) Sort [codegen id : 7]
+Input [6]: [ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2, ss_qty#28, ss_wc#29, ss_sp#30]
+Arguments: [ss_sold_year#27 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], false, 0
+
+(25) Scan parquet default.web_sales
+Output [7]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#37), dynamicpruningexpression(ws_sold_date_sk#37 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_bill_customer_sk:int,ws_order_number:int,ws_quantity:int,ws_wholesale_cost:decimal(7,2),ws_sales_price:decimal(7,2)>
+
+(26) ColumnarToRow [codegen id : 8]
+Input [7]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37]
+
+(27) Filter [codegen id : 8]
+Input [7]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37]
+Condition : (isnotnull(ws_item_sk#31) AND isnotnull(ws_bill_customer_sk#32))
+
+(28) Exchange
+Input [7]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37]
+Arguments: hashpartitioning(ws_order_number#33, ws_item_sk#31, 5), ENSURE_REQUIREMENTS, [id=#38]
+
+(29) Sort [codegen id : 9]
+Input [7]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37]
+Arguments: [ws_order_number#33 ASC NULLS FIRST, ws_item_sk#31 ASC NULLS FIRST], false, 0
+
+(30) Scan parquet default.web_returns
+Output [3]: [wr_item_sk#39, wr_order_number#40, wr_returned_date_sk#41]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_returns]
+PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)]
+ReadSchema: struct<wr_item_sk:int,wr_order_number:int>
+
+(31) ColumnarToRow [codegen id : 10]
+Input [3]: [wr_item_sk#39, wr_order_number#40, wr_returned_date_sk#41]
+
+(32) Filter [codegen id : 10]
+Input [3]: [wr_item_sk#39, wr_order_number#40, wr_returned_date_sk#41]
+Condition : (isnotnull(wr_order_number#40) AND isnotnull(wr_item_sk#39))
+
+(33) Project [codegen id : 10]
+Output [2]: [wr_item_sk#39, wr_order_number#40]
+Input [3]: [wr_item_sk#39, wr_order_number#40, wr_returned_date_sk#41]
+
+(34) Exchange
+Input [2]: [wr_item_sk#39, wr_order_number#40]
+Arguments: hashpartitioning(wr_order_number#40, wr_item_sk#39, 5), ENSURE_REQUIREMENTS, [id=#42]
+
+(35) Sort [codegen id : 11]
+Input [2]: [wr_item_sk#39, wr_order_number#40]
+Arguments: [wr_order_number#40 ASC NULLS FIRST, wr_item_sk#39 ASC NULLS FIRST], false, 0
+
+(36) SortMergeJoin [codegen id : 13]
+Left keys [2]: [ws_order_number#33, ws_item_sk#31]
+Right keys [2]: [wr_order_number#40, wr_item_sk#39]
+Join condition: None
+
+(37) Filter [codegen id : 13]
+Input [9]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37, wr_item_sk#39, wr_order_number#40]
+Condition : isnull(wr_order_number#40)
+
+(38) Project [codegen id : 13]
+Output [6]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37]
+Input [9]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37, wr_item_sk#39, wr_order_number#40]
+
+(39) ReusedExchange [Reuses operator id: 18]
+Output [2]: [d_date_sk#43, d_year#44]
+
+(40) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ws_sold_date_sk#37]
+Right keys [1]: [d_date_sk#43]
+Join condition: None
+
+(41) Project [codegen id : 13]
+Output [6]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, d_year#44]
+Input [8]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37, d_date_sk#43, d_year#44]
+
+(42) HashAggregate [codegen id : 13]
+Input [6]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, d_year#44]
+Keys [3]: [d_year#44, ws_item_sk#31, ws_bill_customer_sk#32]
+Functions [3]: [partial_sum(ws_quantity#34), partial_sum(UnscaledValue(ws_wholesale_cost#35)), partial_sum(UnscaledValue(ws_sales_price#36))]
+Aggregate Attributes [3]: [sum#45, sum#46, sum#47]
+Results [6]: [d_year#44, ws_item_sk#31, ws_bill_customer_sk#32, sum#48, sum#49, sum#50]
+
+(43) Exchange
+Input [6]: [d_year#44, ws_item_sk#31, ws_bill_customer_sk#32, sum#48, sum#49, sum#50]
+Arguments: hashpartitioning(d_year#44, ws_item_sk#31, ws_bill_customer_sk#32, 5), ENSURE_REQUIREMENTS, [id=#51]
+
+(44) HashAggregate [codegen id : 14]
+Input [6]: [d_year#44, ws_item_sk#31, ws_bill_customer_sk#32, sum#48, sum#49, sum#50]
+Keys [3]: [d_year#44, ws_item_sk#31, ws_bill_customer_sk#32]
+Functions [3]: [sum(ws_quantity#34), sum(UnscaledValue(ws_wholesale_cost#35)), sum(UnscaledValue(ws_sales_price#36))]
+Aggregate Attributes [3]: [sum(ws_quantity#34)#52, sum(UnscaledValue(ws_wholesale_cost#35))#53, sum(UnscaledValue(ws_sales_price#36))#54]
+Results [6]: [d_year#44 AS ws_sold_year#55, ws_item_sk#31, ws_bill_customer_sk#32 AS ws_customer_sk#56, sum(ws_quantity#34)#52 AS ws_qty#57, MakeDecimal(sum(UnscaledValue(ws_wholesale_cost#35))#53,17,2) AS ws_wc#58, MakeDecimal(sum(UnscaledValue(ws_sales_price#36))#54,17,2) AS ws_sp#59]
+
+(45) Filter [codegen id : 14]
+Input [6]: [ws_sold_year#55, ws_item_sk#31, ws_customer_sk#56, ws_qty#57, ws_wc#58, ws_sp#59]
+Condition : (coalesce(ws_qty#57, 0) > 0)
+
+(46) Sort [codegen id : 14]
+Input [6]: [ws_sold_year#55, ws_item_sk#31, ws_customer_sk#56, ws_qty#57, ws_wc#58, ws_sp#59]
+Arguments: [ws_sold_year#55 ASC NULLS FIRST, ws_item_sk#31 ASC NULLS FIRST, ws_customer_sk#56 ASC NULLS FIRST], false, 0
+
+(47) SortMergeJoin [codegen id : 15]
+Left keys [3]: [ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2]
+Right keys [3]: [ws_sold_year#55, ws_item_sk#31, ws_customer_sk#56]
+Join condition: None
+
+(48) Project [codegen id : 15]
+Output [9]: [ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2, ss_qty#28, ss_wc#29, ss_sp#30, ws_qty#57, ws_wc#58, ws_sp#59]
+Input [12]: [ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2, ss_qty#28, ss_wc#29, ss_sp#30, ws_sold_year#55, ws_item_sk#31, ws_customer_sk#56, ws_qty#57, ws_wc#58, ws_sp#59]
+
+(49) Scan parquet default.catalog_sales
+Output [7]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#66), dynamicpruningexpression(cs_sold_date_sk#66 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_item_sk:int,cs_order_number:int,cs_quantity:int,cs_wholesale_cost:decimal(7,2),cs_sales_price:decimal(7,2)>
+
+(50) ColumnarToRow [codegen id : 16]
+Input [7]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66]
+
+(51) Filter [codegen id : 16]
+Input [7]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66]
+Condition : (isnotnull(cs_item_sk#61) AND isnotnull(cs_bill_customer_sk#60))
+
+(52) Exchange
+Input [7]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66]
+Arguments: hashpartitioning(cs_order_number#62, cs_item_sk#61, 5), ENSURE_REQUIREMENTS, [id=#67]
+
+(53) Sort [codegen id : 17]
+Input [7]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66]
+Arguments: [cs_order_number#62 ASC NULLS FIRST, cs_item_sk#61 ASC NULLS FIRST], false, 0
+
+(54) Scan parquet default.catalog_returns
+Output [3]: [cr_item_sk#68, cr_order_number#69, cr_returned_date_sk#70]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int>
+
+(55) ColumnarToRow [codegen id : 18]
+Input [3]: [cr_item_sk#68, cr_order_number#69, cr_returned_date_sk#70]
+
+(56) Filter [codegen id : 18]
+Input [3]: [cr_item_sk#68, cr_order_number#69, cr_returned_date_sk#70]
+Condition : (isnotnull(cr_order_number#69) AND isnotnull(cr_item_sk#68))
+
+(57) Project [codegen id : 18]
+Output [2]: [cr_item_sk#68, cr_order_number#69]
+Input [3]: [cr_item_sk#68, cr_order_number#69, cr_returned_date_sk#70]
+
+(58) Exchange
+Input [2]: [cr_item_sk#68, cr_order_number#69]
+Arguments: hashpartitioning(cr_order_number#69, cr_item_sk#68, 5), ENSURE_REQUIREMENTS, [id=#71]
+
+(59) Sort [codegen id : 19]
+Input [2]: [cr_item_sk#68, cr_order_number#69]
+Arguments: [cr_order_number#69 ASC NULLS FIRST, cr_item_sk#68 ASC NULLS FIRST], false, 0
+
+(60) SortMergeJoin [codegen id : 21]
+Left keys [2]: [cs_order_number#62, cs_item_sk#61]
+Right keys [2]: [cr_order_number#69, cr_item_sk#68]
+Join condition: None
+
+(61) Filter [codegen id : 21]
+Input [9]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66, cr_item_sk#68, cr_order_number#69]
+Condition : isnull(cr_order_number#69)
+
+(62) Project [codegen id : 21]
+Output [6]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66]
+Input [9]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66, cr_item_sk#68, cr_order_number#69]
+
+(63) ReusedExchange [Reuses operator id: 18]
+Output [2]: [d_date_sk#72, d_year#73]
+
+(64) BroadcastHashJoin [codegen id : 21]
+Left keys [1]: [cs_sold_date_sk#66]
+Right keys [1]: [d_date_sk#72]
+Join condition: None
+
+(65) Project [codegen id : 21]
+Output [6]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, d_year#73]
+Input [8]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66, d_date_sk#72, d_year#73]
+
+(66) HashAggregate [codegen id : 21]
+Input [6]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, d_year#73]
+Keys [3]: [d_year#73, cs_item_sk#61, cs_bill_customer_sk#60]
+Functions [3]: [partial_sum(cs_quantity#63), partial_sum(UnscaledValue(cs_wholesale_cost#64)), partial_sum(UnscaledValue(cs_sales_price#65))]
+Aggregate Attributes [3]: [sum#74, sum#75, sum#76]
+Results [6]: [d_year#73, cs_item_sk#61, cs_bill_customer_sk#60, sum#77, sum#78, sum#79]
+
+(67) Exchange
+Input [6]: [d_year#73, cs_item_sk#61, cs_bill_customer_sk#60, sum#77, sum#78, sum#79]
+Arguments: hashpartitioning(d_year#73, cs_item_sk#61, cs_bill_customer_sk#60, 5), ENSURE_REQUIREMENTS, [id=#80]
+
+(68) HashAggregate [codegen id : 22]
+Input [6]: [d_year#73, cs_item_sk#61, cs_bill_customer_sk#60, sum#77, sum#78, sum#79]
+Keys [3]: [d_year#73, cs_item_sk#61, cs_bill_customer_sk#60]
+Functions [3]: [sum(cs_quantity#63), sum(UnscaledValue(cs_wholesale_cost#64)), sum(UnscaledValue(cs_sales_price#65))]
+Aggregate Attributes [3]: [sum(cs_quantity#63)#81, sum(UnscaledValue(cs_wholesale_cost#64))#82, sum(UnscaledValue(cs_sales_price#65))#83]
+Results [6]: [d_year#73 AS cs_sold_year#84, cs_item_sk#61, cs_bill_customer_sk#60 AS cs_customer_sk#85, sum(cs_quantity#63)#81 AS cs_qty#86, MakeDecimal(sum(UnscaledValue(cs_wholesale_cost#64))#82,17,2) AS cs_wc#87, MakeDecimal(sum(UnscaledValue(cs_sales_price#65))#83,17,2) AS cs_sp#88]
+
+(69) Filter [codegen id : 22]
+Input [6]: [cs_sold_year#84, cs_item_sk#61, cs_customer_sk#85, cs_qty#86, cs_wc#87, cs_sp#88]
+Condition : (coalesce(cs_qty#86, 0) > 0)
+
+(70) Sort [codegen id : 22]
+Input [6]: [cs_sold_year#84, cs_item_sk#61, cs_customer_sk#85, cs_qty#86, cs_wc#87, cs_sp#88]
+Arguments: [cs_sold_year#84 ASC NULLS FIRST, cs_item_sk#61 ASC NULLS FIRST, cs_customer_sk#85 ASC NULLS FIRST], false, 0
+
+(71) SortMergeJoin [codegen id : 23]
+Left keys [3]: [ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2]
+Right keys [3]: [cs_sold_year#84, cs_item_sk#61, cs_customer_sk#85]
+Join condition: None
+
+(72) Project [codegen id : 23]
+Output [12]: [round((cast(ss_qty#28 as double) / cast(coalesce((ws_qty#57 + cs_qty#86), 1) as double)), 2) AS ratio#89, ss_qty#28 AS store_qty#90, ss_wc#29 AS store_wholesale_cost#91, ss_sp#30 AS store_sales_price#92, (coalesce(ws_qty#57, 0) + coalesce(cs_qty#86, 0)) AS other_chan_qty#93, CheckOverflow((promote_precision(cast(coalesce(ws_wc#58, 0.00) as decimal(18,2))) + promote_precision(cast(coalesce(cs_wc#87, 0.00) as decimal(18,2)))), DecimalType(18,2), true) AS other_chan_wholesale_cost#94, CheckOverflow((promote_precision(cast(coalesce(ws_sp#59, 0.00) as decimal(18,2))) + promote_precision(cast(coalesce(cs_sp#88, 0.00) as decimal(18,2)))), DecimalType(18,2), true) AS other_chan_sales_price#95, ss_qty#28, ss_wc#29, ss_sp#30, ws_qty#57, cs_qty#86]
+Input [15]: [ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2, ss_qty#28, ss_wc#29, ss_sp#30, ws_qty#57, ws_wc#58, ws_sp#59, cs_sold_year#84, cs_item_sk#61, cs_customer_sk#85, cs_qty#86, cs_wc#87, cs_sp#88]
+
+(73) TakeOrderedAndProject
+Input [12]: [ratio#89, store_qty#90, store_wholesale_cost#91, store_sales_price#92, other_chan_qty#93, other_chan_wholesale_cost#94, other_chan_sales_price#95, ss_qty#28, ss_wc#29, ss_sp#30, ws_qty#57, cs_qty#86]
+Arguments: 100, [ratio#89 ASC NULLS FIRST, ss_qty#28 DESC NULLS LAST, ss_wc#29 DESC NULLS LAST, ss_sp#30 DESC NULLS LAST, other_chan_qty#93 ASC NULLS FIRST, other_chan_wholesale_cost#94 ASC NULLS FIRST, other_chan_sales_price#95 ASC NULLS FIRST, round((cast(ss_qty#28 as double) / cast(coalesce((ws_qty#57 + cs_qty#86), 1) as double)), 2) ASC NULLS FIRST], [ratio#89, store_qty#90, store_wholesale_cost#91, store_sales_price#92, other_chan_qty#93, other_chan_wholesale_cost#94, other_chan_sales_price#95]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (74)
+
+
+(74) ReusedExchange [Reuses operator id: 18]
+Output [2]: [d_date_sk#14, d_year#15]
+
+Subquery:2 Hosting operator id = 25 Hosting Expression = ws_sold_date_sk#37 IN dynamicpruning#8
+
+Subquery:3 Hosting operator id = 49 Hosting Expression = cs_sold_date_sk#66 IN dynamicpruning#8
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/simplified.txt
new file mode 100644
index 0000000000000..0aa69ab4645e4
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/simplified.txt
@@ -0,0 +1,127 @@
+TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholesale_cost,other_chan_sales_price,ws_qty,cs_qty,store_qty,store_wholesale_cost,store_sales_price]
+  WholeStageCodegen (23)
+    Project [ss_qty,ws_qty,cs_qty,ss_wc,ss_sp,ws_wc,cs_wc,ws_sp,cs_sp]
+      SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,cs_sold_year,cs_item_sk,cs_customer_sk]
+        InputAdapter
+          WholeStageCodegen (15)
+            Project [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp]
+              SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ws_sold_year,ws_item_sk,ws_customer_sk]
+                InputAdapter
+                  WholeStageCodegen (7)
+                    Sort [ss_sold_year,ss_item_sk,ss_customer_sk]
+                      HashAggregate [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] [sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price)),ss_sold_year,ss_qty,ss_wc,ss_sp,sum,sum,sum]
+                        InputAdapter
+                          Exchange [d_year,ss_item_sk,ss_customer_sk] #1
+                            WholeStageCodegen (6)
+                              HashAggregate [d_year,ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price] [sum,sum,sum,sum,sum,sum]
+                                Project [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year]
+                                  BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                    Project [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk]
+                                      Filter [sr_ticket_number]
+                                        SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                          InputAdapter
+                                            WholeStageCodegen (2)
+                                              Sort [ss_ticket_number,ss_item_sk]
+                                                InputAdapter
+                                                  Exchange [ss_ticket_number,ss_item_sk] #2
+                                                    WholeStageCodegen (1)
+                                                      Filter [ss_item_sk,ss_customer_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk]
+                                                              SubqueryBroadcast [d_date_sk] #1
+                                                                ReusedExchange [d_date_sk,d_year] #3
+                                          InputAdapter
+                                            WholeStageCodegen (4)
+                                              Sort [sr_ticket_number,sr_item_sk]
+                                                InputAdapter
+                                                  Exchange [sr_ticket_number,sr_item_sk] #4
+                                                    WholeStageCodegen (3)
+                                                      Project [sr_item_sk,sr_ticket_number]
+                                                        Filter [sr_ticket_number,sr_item_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk]
+                                    InputAdapter
+                                      BroadcastExchange #3
+                                        WholeStageCodegen (5)
+                                          Filter [d_year,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_year]
+                InputAdapter
+                  WholeStageCodegen (14)
+                    Sort [ws_sold_year,ws_item_sk,ws_customer_sk]
+                      Filter [ws_qty]
+                        HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] [sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price)),ws_sold_year,ws_customer_sk,ws_qty,ws_wc,ws_sp,sum,sum,sum]
+                          InputAdapter
+                            Exchange [d_year,ws_item_sk,ws_bill_customer_sk] #5
+                              WholeStageCodegen (13)
+                                HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price] [sum,sum,sum,sum,sum,sum]
+                                  Project [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year]
+                                    BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                      Project [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk]
+                                        Filter [wr_order_number]
+                                          SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk]
+                                            InputAdapter
+                                              WholeStageCodegen (9)
+                                                Sort [ws_order_number,ws_item_sk]
+                                                  InputAdapter
+                                                    Exchange [ws_order_number,ws_item_sk] #6
+                                                      WholeStageCodegen (8)
+                                                        Filter [ws_item_sk,ws_bill_customer_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk]
+                                                                ReusedSubquery [d_date_sk] #1
+                                            InputAdapter
+                                              WholeStageCodegen (11)
+                                                Sort [wr_order_number,wr_item_sk]
+                                                  InputAdapter
+                                                    Exchange [wr_order_number,wr_item_sk] #7
+                                                      WholeStageCodegen (10)
+                                                        Project [wr_item_sk,wr_order_number]
+                                                          Filter [wr_order_number,wr_item_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk]
+                                      InputAdapter
+                                        ReusedExchange [d_date_sk,d_year] #3
+        InputAdapter
+          WholeStageCodegen (22)
+            Sort [cs_sold_year,cs_item_sk,cs_customer_sk]
+              Filter [cs_qty]
+                HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] [sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price)),cs_sold_year,cs_customer_sk,cs_qty,cs_wc,cs_sp,sum,sum,sum]
+                  InputAdapter
+                    Exchange [d_year,cs_item_sk,cs_bill_customer_sk] #8
+                      WholeStageCodegen (21)
+                        HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,cs_quantity,cs_wholesale_cost,cs_sales_price] [sum,sum,sum,sum,sum,sum]
+                          Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year]
+                            BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                              Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk]
+                                Filter [cr_order_number]
+                                  SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk]
+                                    InputAdapter
+                                      WholeStageCodegen (17)
+                                        Sort [cs_order_number,cs_item_sk]
+                                          InputAdapter
+                                            Exchange [cs_order_number,cs_item_sk] #9
+                                              WholeStageCodegen (16)
+                                                Filter [cs_item_sk,cs_bill_customer_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk]
+                                                        ReusedSubquery [d_date_sk] #1
+                                    InputAdapter
+                                      WholeStageCodegen (19)
+                                        Sort [cr_order_number,cr_item_sk]
+                                          InputAdapter
+                                            Exchange [cr_order_number,cr_item_sk] #10
+                                              WholeStageCodegen (18)
+                                                Project [cr_item_sk,cr_order_number]
+                                                  Filter [cr_order_number,cr_item_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk]
+                              InputAdapter
+                                ReusedExchange [d_date_sk,d_year] #3
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.sf100/explain.txt
new file mode 100644
index 0000000000000..09a95c5df6e29
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.sf100/explain.txt
@@ -0,0 +1,219 @@
+== Physical Plan ==
+TakeOrderedAndProject (37)
++- * Project (36)
+   +- * SortMergeJoin Inner (35)
+      :- * Sort (29)
+      :  +- Exchange (28)
+      :     +- * HashAggregate (27)
+      :        +- Exchange (26)
+      :           +- * HashAggregate (25)
+      :              +- * Project (24)
+      :                 +- * BroadcastHashJoin Inner BuildRight (23)
+      :                    :- * Project (17)
+      :                    :  +- * BroadcastHashJoin Inner BuildRight (16)
+      :                    :     :- * Project (10)
+      :                    :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+      :                    :     :     :- * Filter (3)
+      :                    :     :     :  +- * ColumnarToRow (2)
+      :                    :     :     :     +- Scan parquet default.store_sales (1)
+      :                    :     :     +- BroadcastExchange (8)
+      :                    :     :        +- * Project (7)
+      :                    :     :           +- * Filter (6)
+      :                    :     :              +- * ColumnarToRow (5)
+      :                    :     :                 +- Scan parquet default.date_dim (4)
+      :                    :     +- BroadcastExchange (15)
+      :                    :        +- * Project (14)
+      :                    :           +- * Filter (13)
+      :                    :              +- * ColumnarToRow (12)
+      :                    :                 +- Scan parquet default.household_demographics (11)
+      :                    +- BroadcastExchange (22)
+      :                       +- * Project (21)
+      :                          +- * Filter (20)
+      :                             +- * ColumnarToRow (19)
+      :                                +- Scan parquet default.store (18)
+      +- * Sort (34)
+         +- Exchange (33)
+            +- * Filter (32)
+               +- * ColumnarToRow (31)
+                  +- Scan parquet default.customer (30)
+
+
+(1) Scan parquet default.store_sales
+Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_hdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_coupon_amt:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8]
+
+(3) Filter [codegen id : 4]
+Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8]
+Condition : ((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#10, d_year#11, d_dow#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_dow), EqualTo(d_dow,1), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_dow:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#10, d_year#11, d_dow#12]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#10, d_year#11, d_dow#12]
+Condition : (((isnotnull(d_dow#12) AND (d_dow#12 = 1)) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#10]
+Input [3]: [d_date_sk#10, d_year#11, d_dow#12]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7]
+Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#10]
+
+(11) Scan parquet default.household_demographics
+Output [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [Or(EqualTo(hd_dep_count,6),GreaterThan(hd_vehicle_count,2)), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_dep_count:int,hd_vehicle_count:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16]
+
+(13) Filter [codegen id : 2]
+Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16]
+Condition : (((hd_dep_count#15 = 6) OR (hd_vehicle_count#16 > 2)) AND isnotnull(hd_demo_sk#14))
+
+(14) Project [codegen id : 2]
+Output [1]: [hd_demo_sk#14]
+Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16]
+
+(15) BroadcastExchange
+Input [1]: [hd_demo_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#14]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7]
+Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, hd_demo_sk#14]
+
+(18) Scan parquet default.store
+Output [3]: [s_store_sk#18, s_number_employees#19, s_city#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_number_employees), GreaterThanOrEqual(s_number_employees,200), LessThanOrEqual(s_number_employees,295), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_number_employees:int,s_city:string>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [3]: [s_store_sk#18, s_number_employees#19, s_city#20]
+
+(20) Filter [codegen id : 3]
+Input [3]: [s_store_sk#18, s_number_employees#19, s_city#20]
+Condition : (((isnotnull(s_number_employees#19) AND (s_number_employees#19 >= 200)) AND (s_number_employees#19 <= 295)) AND isnotnull(s_store_sk#18))
+
+(21) Project [codegen id : 3]
+Output [2]: [s_store_sk#18, s_city#20]
+Input [3]: [s_store_sk#18, s_number_employees#19, s_city#20]
+
+(22) BroadcastExchange
+Input [2]: [s_store_sk#18, s_city#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(23) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#4]
+Right keys [1]: [s_store_sk#18]
+Join condition: None
+
+(24) Project [codegen id : 4]
+Output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#20]
+Input [8]: [ss_customer_sk#1, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#18, s_city#20]
+
+(25) HashAggregate [codegen id : 4]
+Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#20]
+Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#20]
+Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))]
+Aggregate Attributes [2]: [sum#22, sum#23]
+Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#20, sum#24, sum#25]
+
+(26) Exchange
+Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#20, sum#24, sum#25]
+Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#20, 5), ENSURE_REQUIREMENTS, [id=#26]
+
+(27) HashAggregate [codegen id : 5]
+Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#20, sum#24, sum#25]
+Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#20]
+Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#27, sum(UnscaledValue(ss_net_profit#7))#28]
+Results [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#20, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#27,17,2) AS amt#29, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#28,17,2) AS profit#30]
+
+(28) Exchange
+Input [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#20, amt#29, profit#30]
+Arguments: hashpartitioning(ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(29) Sort [codegen id : 6]
+Input [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#20, amt#29, profit#30]
+Arguments: [ss_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(30) Scan parquet default.customer
+Output [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_first_name:string,c_last_name:string>
+
+(31) ColumnarToRow [codegen id : 7]
+Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34]
+
+(32) Filter [codegen id : 7]
+Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34]
+Condition : isnotnull(c_customer_sk#32)
+
+(33) Exchange
+Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34]
+Arguments: hashpartitioning(c_customer_sk#32, 5), ENSURE_REQUIREMENTS, [id=#35]
+
+(34) Sort [codegen id : 8]
+Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34]
+Arguments: [c_customer_sk#32 ASC NULLS FIRST], false, 0
+
+(35) SortMergeJoin [codegen id : 9]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#32]
+Join condition: None
+
+(36) Project [codegen id : 9]
+Output [7]: [c_last_name#34, c_first_name#33, substr(s_city#20, 1, 30) AS substr(s_city, 1, 30)#36, ss_ticket_number#5, amt#29, profit#30, s_city#20]
+Input [8]: [ss_ticket_number#5, ss_customer_sk#1, s_city#20, amt#29, profit#30, c_customer_sk#32, c_first_name#33, c_last_name#34]
+
+(37) TakeOrderedAndProject
+Input [7]: [c_last_name#34, c_first_name#33, substr(s_city, 1, 30)#36, ss_ticket_number#5, amt#29, profit#30, s_city#20]
+Arguments: 100, [c_last_name#34 ASC NULLS FIRST, c_first_name#33 ASC NULLS FIRST, substr(s_city#20, 1, 30) ASC NULLS FIRST, profit#30 ASC NULLS FIRST], [c_last_name#34, c_first_name#33, substr(s_city, 1, 30)#36, ss_ticket_number#5, amt#29, profit#30]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (38)
+
+
+(38) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#10]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.sf100/simplified.txt
new file mode 100644
index 0000000000000..bac3b763658f8
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.sf100/simplified.txt
@@ -0,0 +1,61 @@
+TakeOrderedAndProject [c_last_name,c_first_name,s_city,profit,substr(s_city, 1, 30),ss_ticket_number,amt]
+  WholeStageCodegen (9)
+    Project [c_last_name,c_first_name,s_city,ss_ticket_number,amt,profit]
+      SortMergeJoin [ss_customer_sk,c_customer_sk]
+        InputAdapter
+          WholeStageCodegen (6)
+            Sort [ss_customer_sk]
+              InputAdapter
+                Exchange [ss_customer_sk] #1
+                  WholeStageCodegen (5)
+                    HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,sum,sum] [sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit)),amt,profit,sum,sum]
+                      InputAdapter
+                        Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city] #2
+                          WholeStageCodegen (4)
+                            HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,ss_coupon_amt,ss_net_profit] [sum,sum,sum,sum]
+                              Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city]
+                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                  Project [ss_customer_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit]
+                                    BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                      Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit]
+                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                          Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk]
+                                                  SubqueryBroadcast [d_date_sk] #1
+                                                    ReusedExchange [d_date_sk] #3
+                                          InputAdapter
+                                            BroadcastExchange #3
+                                              WholeStageCodegen (1)
+                                                Project [d_date_sk]
+                                                  Filter [d_dow,d_year,d_date_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.date_dim [d_date_sk,d_year,d_dow]
+                                      InputAdapter
+                                        BroadcastExchange #4
+                                          WholeStageCodegen (2)
+                                            Project [hd_demo_sk]
+                                              Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count]
+                                  InputAdapter
+                                    BroadcastExchange #5
+                                      WholeStageCodegen (3)
+                                        Project [s_store_sk,s_city]
+                                          Filter [s_number_employees,s_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store [s_store_sk,s_number_employees,s_city]
+        InputAdapter
+          WholeStageCodegen (8)
+            Sort [c_customer_sk]
+              InputAdapter
+                Exchange [c_customer_sk] #6
+                  WholeStageCodegen (7)
+                    Filter [c_customer_sk]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.customer [c_customer_sk,c_first_name,c_last_name]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79/explain.txt
new file mode 100644
index 0000000000000..5602a2fddfde7
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79/explain.txt
@@ -0,0 +1,204 @@
+== Physical Plan ==
+TakeOrderedAndProject (34)
++- * Project (33)
+   +- * BroadcastHashJoin Inner BuildRight (32)
+      :- * HashAggregate (27)
+      :  +- Exchange (26)
+      :     +- * HashAggregate (25)
+      :        +- * Project (24)
+      :           +- * BroadcastHashJoin Inner BuildRight (23)
+      :              :- * Project (17)
+      :              :  +- * BroadcastHashJoin Inner BuildRight (16)
+      :              :     :- * Project (10)
+      :              :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+      :              :     :     :- * Filter (3)
+      :              :     :     :  +- * ColumnarToRow (2)
+      :              :     :     :     +- Scan parquet default.store_sales (1)
+      :              :     :     +- BroadcastExchange (8)
+      :              :     :        +- * Project (7)
+      :              :     :           +- * Filter (6)
+      :              :     :              +- * ColumnarToRow (5)
+      :              :     :                 +- Scan parquet default.date_dim (4)
+      :              :     +- BroadcastExchange (15)
+      :              :        +- * Project (14)
+      :              :           +- * Filter (13)
+      :              :              +- * ColumnarToRow (12)
+      :              :                 +- Scan parquet default.store (11)
+      :              +- BroadcastExchange (22)
+      :                 +- * Project (21)
+      :                    +- * Filter (20)
+      :                       +- * ColumnarToRow (19)
+      :                          +- Scan parquet default.household_demographics (18)
+      +- BroadcastExchange (31)
+         +- * Filter (30)
+            +- * ColumnarToRow (29)
+               +- Scan parquet default.customer (28)
+
+
+(1) Scan parquet default.store_sales
+Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_hdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_coupon_amt:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8]
+
+(3) Filter [codegen id : 4]
+Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8]
+Condition : ((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#10, d_year#11, d_dow#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_dow), EqualTo(d_dow,1), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_dow:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#10, d_year#11, d_dow#12]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#10, d_year#11, d_dow#12]
+Condition : (((isnotnull(d_dow#12) AND (d_dow#12 = 1)) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#10]
+Input [3]: [d_date_sk#10, d_year#11, d_dow#12]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7]
+Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#10]
+
+(11) Scan parquet default.store
+Output [3]: [s_store_sk#14, s_number_employees#15, s_city#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_number_employees), GreaterThanOrEqual(s_number_employees,200), LessThanOrEqual(s_number_employees,295), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_number_employees:int,s_city:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [s_store_sk#14, s_number_employees#15, s_city#16]
+
+(13) Filter [codegen id : 2]
+Input [3]: [s_store_sk#14, s_number_employees#15, s_city#16]
+Condition : (((isnotnull(s_number_employees#15) AND (s_number_employees#15 >= 200)) AND (s_number_employees#15 <= 295)) AND isnotnull(s_store_sk#14))
+
+(14) Project [codegen id : 2]
+Output [2]: [s_store_sk#14, s_city#16]
+Input [3]: [s_store_sk#14, s_number_employees#15, s_city#16]
+
+(15) BroadcastExchange
+Input [2]: [s_store_sk#14, s_city#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#4]
+Right keys [1]: [s_store_sk#14]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#16]
+Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#14, s_city#16]
+
+(18) Scan parquet default.household_demographics
+Output [3]: [hd_demo_sk#18, hd_dep_count#19, hd_vehicle_count#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [Or(EqualTo(hd_dep_count,6),GreaterThan(hd_vehicle_count,2)), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_dep_count:int,hd_vehicle_count:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [3]: [hd_demo_sk#18, hd_dep_count#19, hd_vehicle_count#20]
+
+(20) Filter [codegen id : 3]
+Input [3]: [hd_demo_sk#18, hd_dep_count#19, hd_vehicle_count#20]
+Condition : (((hd_dep_count#19 = 6) OR (hd_vehicle_count#20 > 2)) AND isnotnull(hd_demo_sk#18))
+
+(21) Project [codegen id : 3]
+Output [1]: [hd_demo_sk#18]
+Input [3]: [hd_demo_sk#18, hd_dep_count#19, hd_vehicle_count#20]
+
+(22) BroadcastExchange
+Input [1]: [hd_demo_sk#18]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(23) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#18]
+Join condition: None
+
+(24) Project [codegen id : 4]
+Output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#16]
+Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#16, hd_demo_sk#18]
+
+(25) HashAggregate [codegen id : 4]
+Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#16]
+Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#16]
+Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))]
+Aggregate Attributes [2]: [sum#22, sum#23]
+Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#16, sum#24, sum#25]
+
+(26) Exchange
+Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#16, sum#24, sum#25]
+Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#16, 5), ENSURE_REQUIREMENTS, [id=#26]
+
+(27) HashAggregate [codegen id : 6]
+Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#16, sum#24, sum#25]
+Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#16]
+Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#27, sum(UnscaledValue(ss_net_profit#7))#28]
+Results [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#16, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#27,17,2) AS amt#29, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#28,17,2) AS profit#30]
+
+(28) Scan parquet default.customer
+Output [3]: [c_customer_sk#31, c_first_name#32, c_last_name#33]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_first_name:string,c_last_name:string>
+
+(29) ColumnarToRow [codegen id : 5]
+Input [3]: [c_customer_sk#31, c_first_name#32, c_last_name#33]
+
+(30) Filter [codegen id : 5]
+Input [3]: [c_customer_sk#31, c_first_name#32, c_last_name#33]
+Condition : isnotnull(c_customer_sk#31)
+
+(31) BroadcastExchange
+Input [3]: [c_customer_sk#31, c_first_name#32, c_last_name#33]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#34]
+
+(32) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#31]
+Join condition: None
+
+(33) Project [codegen id : 6]
+Output [7]: [c_last_name#33, c_first_name#32, substr(s_city#16, 1, 30) AS substr(s_city, 1, 30)#35, ss_ticket_number#5, amt#29, profit#30, s_city#16]
+Input [8]: [ss_ticket_number#5, ss_customer_sk#1, s_city#16, amt#29, profit#30, c_customer_sk#31, c_first_name#32, c_last_name#33]
+
+(34) TakeOrderedAndProject
+Input [7]: [c_last_name#33, c_first_name#32, substr(s_city, 1, 30)#35, ss_ticket_number#5, amt#29, profit#30, s_city#16]
+Arguments: 100, [c_last_name#33 ASC NULLS FIRST, c_first_name#32 ASC NULLS FIRST, substr(s_city#16, 1, 30) ASC NULLS FIRST, profit#30 ASC NULLS FIRST], [c_last_name#33, c_first_name#32, substr(s_city, 1, 30)#35, ss_ticket_number#5, amt#29, profit#30]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (35)
+
+
+(35) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#10]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79/simplified.txt
new file mode 100644
index 0000000000000..6432bc55bd8f3
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79/simplified.txt
@@ -0,0 +1,52 @@
+TakeOrderedAndProject [c_last_name,c_first_name,s_city,profit,substr(s_city, 1, 30),ss_ticket_number,amt]
+  WholeStageCodegen (6)
+    Project [c_last_name,c_first_name,s_city,ss_ticket_number,amt,profit]
+      BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+        HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,sum,sum] [sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit)),amt,profit,sum,sum]
+          InputAdapter
+            Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city] #1
+              WholeStageCodegen (4)
+                HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,ss_coupon_amt,ss_net_profit] [sum,sum,sum,sum]
+                  Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city]
+                    BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                      Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city]
+                        BroadcastHashJoin [ss_store_sk,s_store_sk]
+                          Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit]
+                            BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                              Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk]
+                                      SubqueryBroadcast [d_date_sk] #1
+                                        ReusedExchange [d_date_sk] #2
+                              InputAdapter
+                                BroadcastExchange #2
+                                  WholeStageCodegen (1)
+                                    Project [d_date_sk]
+                                      Filter [d_dow,d_year,d_date_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.date_dim [d_date_sk,d_year,d_dow]
+                          InputAdapter
+                            BroadcastExchange #3
+                              WholeStageCodegen (2)
+                                Project [s_store_sk,s_city]
+                                  Filter [s_number_employees,s_store_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.store [s_store_sk,s_number_employees,s_city]
+                      InputAdapter
+                        BroadcastExchange #4
+                          WholeStageCodegen (3)
+                            Project [hd_demo_sk]
+                              Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count]
+        InputAdapter
+          BroadcastExchange #5
+            WholeStageCodegen (5)
+              Filter [c_customer_sk]
+                ColumnarToRow
+                  InputAdapter
+                    Scan parquet default.customer [c_customer_sk,c_first_name,c_last_name]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.sf100/explain.txt
new file mode 100644
index 0000000000000..d13b9623d1b34
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.sf100/explain.txt
@@ -0,0 +1,313 @@
+== Physical Plan ==
+TakeOrderedAndProject (53)
++- * HashAggregate (52)
+   +- Exchange (51)
+      +- * HashAggregate (50)
+         +- * Project (49)
+            +- * SortMergeJoin Inner (48)
+               :- * Sort (18)
+               :  +- Exchange (17)
+               :     +- * Project (16)
+               :        +- * BroadcastHashJoin Inner BuildRight (15)
+               :           :- * Project (10)
+               :           :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :           :     :- * Filter (3)
+               :           :     :  +- * ColumnarToRow (2)
+               :           :     :     +- Scan parquet default.store_sales (1)
+               :           :     +- BroadcastExchange (8)
+               :           :        +- * Project (7)
+               :           :           +- * Filter (6)
+               :           :              +- * ColumnarToRow (5)
+               :           :                 +- Scan parquet default.date_dim (4)
+               :           +- BroadcastExchange (14)
+               :              +- * Filter (13)
+               :                 +- * ColumnarToRow (12)
+               :                    +- Scan parquet default.store (11)
+               +- * Sort (47)
+                  +- Exchange (46)
+                     +- * HashAggregate (45)
+                        +- Exchange (44)
+                           +- * HashAggregate (43)
+                              +- * Project (42)
+                                 +- * BroadcastHashJoin LeftSemi BuildRight (41)
+                                    :- * Filter (21)
+                                    :  +- * ColumnarToRow (20)
+                                    :     +- Scan parquet default.customer_address (19)
+                                    +- BroadcastExchange (40)
+                                       +- * Project (39)
+                                          +- * Filter (38)
+                                             +- * HashAggregate (37)
+                                                +- Exchange (36)
+                                                   +- * HashAggregate (35)
+                                                      +- * Project (34)
+                                                         +- * SortMergeJoin Inner (33)
+                                                            :- * Sort (26)
+                                                            :  +- Exchange (25)
+                                                            :     +- * Filter (24)
+                                                            :        +- * ColumnarToRow (23)
+                                                            :           +- Scan parquet default.customer_address (22)
+                                                            +- * Sort (32)
+                                                               +- Exchange (31)
+                                                                  +- * Project (30)
+                                                                     +- * Filter (29)
+                                                                        +- * ColumnarToRow (28)
+                                                                           +- Scan parquet default.customer (27)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_store_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,1998), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_qoy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+Condition : ((((isnotnull(d_qoy#7) AND isnotnull(d_year#6)) AND (d_qoy#7 = 2)) AND (d_year#6 = 1998)) AND isnotnull(d_date_sk#5))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#5]
+Input [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#8]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#5]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [2]: [ss_store_sk#1, ss_net_profit#2]
+Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#5]
+
+(11) Scan parquet default.store
+Output [3]: [s_store_sk#9, s_store_name#10, s_zip#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_zip)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_zip:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [s_store_sk#9, s_store_name#10, s_zip#11]
+
+(13) Filter [codegen id : 2]
+Input [3]: [s_store_sk#9, s_store_name#10, s_zip#11]
+Condition : (isnotnull(s_store_sk#9) AND isnotnull(s_zip#11))
+
+(14) BroadcastExchange
+Input [3]: [s_store_sk#9, s_store_name#10, s_zip#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#12]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_store_sk#1]
+Right keys [1]: [s_store_sk#9]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [3]: [ss_net_profit#2, s_store_name#10, s_zip#11]
+Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#9, s_store_name#10, s_zip#11]
+
+(17) Exchange
+Input [3]: [ss_net_profit#2, s_store_name#10, s_zip#11]
+Arguments: hashpartitioning(substr(s_zip#11, 1, 2), 5), ENSURE_REQUIREMENTS, [id=#13]
+
+(18) Sort [codegen id : 4]
+Input [3]: [ss_net_profit#2, s_store_name#10, s_zip#11]
+Arguments: [substr(s_zip#11, 1, 2) ASC NULLS FIRST], false, 0
+
+(19) Scan parquet default.customer_address
+Output [1]: [ca_zip#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+ReadSchema: struct<ca_zip:string>
+
+(20) ColumnarToRow [codegen id : 11]
+Input [1]: [ca_zip#14]
+
+(21) Filter [codegen id : 11]
+Input [1]: [ca_zip#14]
+Condition : (substr(ca_zip#14, 1, 5) INSET (56910,69952,63792,39371,74351,11101,25003,97189,57834,73134,62377,51200,32754,22752,86379,14171,91110,40162,98569,28709,13394,66162,25733,25782,26065,18383,51949,87343,50298,83849,33786,64528,23470,67030,46136,25280,46820,77721,99076,18426,31880,17871,98235,45748,49156,18652,72013,51622,43848,78567,41248,13695,44165,67853,54917,53179,64034,10567,71791,68908,55565,59402,64147,85816,57855,61547,27700,68100,28810,58263,15723,83933,51103,58058,90578,82276,81096,81426,96451,77556,38607,76638,18906,62971,57047,48425,35576,11928,30625,83444,73520,51650,57647,60099,30122,94983,24128,10445,41368,26233,26859,21756,24676,19849,36420,38193,58470,39127,13595,87501,24317,15455,69399,98025,81019,48033,11376,39516,67875,92712,14867,38122,29741,42961,30469,51211,56458,15559,16021,33123,33282,33515,72823,54601,76698,56240,72175,60279,20004,68806,72325,28488,43933,50412,45200,22246,78668,79777,96765,67301,73273,49448,82636,23932,47305,29839,39192,18799,61265,37125,58943,64457,88424,24610,84935,89360,68893,30431,28898,10336,90257,59166,46081,26105,96888,36634,86284,35258,39972,22927,73241,53268,24206,27385,99543,31671,14663,30903,39861,24996,63089,88086,83921,21076,67897,66708,45721,60576,25103,52867,30450,36233,30010,96576,73171,56571,56575,64544,13955,78451,43285,18119,16725,83041,76107,79994,54364,35942,56691,19769,63435,34102,18845,22744,13354,75691,45549,23968,31387,83144,13375,15765,28577,88190,19736,73650,37930,25989,83926,94898,51798,39736,22437,55253,38415,71256,18376,42029,25858,44438,19515,38935,51649,71954,15882,18767,63193,25486,49130,37126,40604,34425,17043,12305,11634,26653,94167,36446,10516,67473,66864,72425,63981,18842,22461,42666,47770,69035,70372,28587,45266,15371,15798,45375,90225,16807,31016,68014,21337,19505,50016,10144,84093,21286,19430,34322,91068,94945,72305,24671,58048,65084,28545,21195,20548,22245,77191,96976,48583,76231,15734,61810,11356,68621,68786,98359,41367,26689,69913,76614,68101,88885,50308,79077,18270,28915,29178,53672,62878,10390,14922,68341,56529,41766,68309,56616,15126,61860,97789,11489,45692,41918,72151,72550,27156,36495,70738,17879,53535,17920,68880,78890,35850,14089,58078,65164,27068,26231,13376,57665,32213,77610,87816,21309,15146,86198,91137,55307,67467,40558,94627,82136,22351,89091,20260,23006,91393,47537,62496,98294,18840,71286,81312,31029,70466,35458,14060,22685,28286,25631,19512,40081,63837,14328,35474,22152,76232,51061,86057,17183) AND isnotnull(substr(ca_zip#14, 1, 5)))
+
+(22) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#15, ca_zip#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_zip:string>
+
+(23) ColumnarToRow [codegen id : 5]
+Input [2]: [ca_address_sk#15, ca_zip#16]
+
+(24) Filter [codegen id : 5]
+Input [2]: [ca_address_sk#15, ca_zip#16]
+Condition : isnotnull(ca_address_sk#15)
+
+(25) Exchange
+Input [2]: [ca_address_sk#15, ca_zip#16]
+Arguments: hashpartitioning(ca_address_sk#15, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(26) Sort [codegen id : 6]
+Input [2]: [ca_address_sk#15, ca_zip#16]
+Arguments: [ca_address_sk#15 ASC NULLS FIRST], false, 0
+
+(27) Scan parquet default.customer
+Output [2]: [c_current_addr_sk#18, c_preferred_cust_flag#19]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_preferred_cust_flag), EqualTo(c_preferred_cust_flag,Y), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_current_addr_sk:int,c_preferred_cust_flag:string>
+
+(28) ColumnarToRow [codegen id : 7]
+Input [2]: [c_current_addr_sk#18, c_preferred_cust_flag#19]
+
+(29) Filter [codegen id : 7]
+Input [2]: [c_current_addr_sk#18, c_preferred_cust_flag#19]
+Condition : ((isnotnull(c_preferred_cust_flag#19) AND (c_preferred_cust_flag#19 = Y)) AND isnotnull(c_current_addr_sk#18))
+
+(30) Project [codegen id : 7]
+Output [1]: [c_current_addr_sk#18]
+Input [2]: [c_current_addr_sk#18, c_preferred_cust_flag#19]
+
+(31) Exchange
+Input [1]: [c_current_addr_sk#18]
+Arguments: hashpartitioning(c_current_addr_sk#18, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(32) Sort [codegen id : 8]
+Input [1]: [c_current_addr_sk#18]
+Arguments: [c_current_addr_sk#18 ASC NULLS FIRST], false, 0
+
+(33) SortMergeJoin [codegen id : 9]
+Left keys [1]: [ca_address_sk#15]
+Right keys [1]: [c_current_addr_sk#18]
+Join condition: None
+
+(34) Project [codegen id : 9]
+Output [1]: [ca_zip#16]
+Input [3]: [ca_address_sk#15, ca_zip#16, c_current_addr_sk#18]
+
+(35) HashAggregate [codegen id : 9]
+Input [1]: [ca_zip#16]
+Keys [1]: [ca_zip#16]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#21]
+Results [2]: [ca_zip#16, count#22]
+
+(36) Exchange
+Input [2]: [ca_zip#16, count#22]
+Arguments: hashpartitioning(ca_zip#16, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(37) HashAggregate [codegen id : 10]
+Input [2]: [ca_zip#16, count#22]
+Keys [1]: [ca_zip#16]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#24]
+Results [2]: [substr(ca_zip#16, 1, 5) AS ca_zip#25, count(1)#24 AS count(1)#26]
+
+(38) Filter [codegen id : 10]
+Input [2]: [ca_zip#25, count(1)#26]
+Condition : (count(1)#26 > 10)
+
+(39) Project [codegen id : 10]
+Output [1]: [ca_zip#25]
+Input [2]: [ca_zip#25, count(1)#26]
+
+(40) BroadcastExchange
+Input [1]: [ca_zip#25]
+Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true])),false), [id=#27]
+
+(41) BroadcastHashJoin [codegen id : 11]
+Left keys [2]: [coalesce(substr(ca_zip#14, 1, 5), ), isnull(substr(ca_zip#14, 1, 5))]
+Right keys [2]: [coalesce(ca_zip#25, ), isnull(ca_zip#25)]
+Join condition: None
+
+(42) Project [codegen id : 11]
+Output [1]: [substr(ca_zip#14, 1, 5) AS ca_zip#28]
+Input [1]: [ca_zip#14]
+
+(43) HashAggregate [codegen id : 11]
+Input [1]: [ca_zip#28]
+Keys [1]: [ca_zip#28]
+Functions: []
+Aggregate Attributes: []
+Results [1]: [ca_zip#28]
+
+(44) Exchange
+Input [1]: [ca_zip#28]
+Arguments: hashpartitioning(ca_zip#28, 5), ENSURE_REQUIREMENTS, [id=#29]
+
+(45) HashAggregate [codegen id : 12]
+Input [1]: [ca_zip#28]
+Keys [1]: [ca_zip#28]
+Functions: []
+Aggregate Attributes: []
+Results [1]: [ca_zip#28]
+
+(46) Exchange
+Input [1]: [ca_zip#28]
+Arguments: hashpartitioning(substr(ca_zip#28, 1, 2), 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(47) Sort [codegen id : 13]
+Input [1]: [ca_zip#28]
+Arguments: [substr(ca_zip#28, 1, 2) ASC NULLS FIRST], false, 0
+
+(48) SortMergeJoin [codegen id : 14]
+Left keys [1]: [substr(s_zip#11, 1, 2)]
+Right keys [1]: [substr(ca_zip#28, 1, 2)]
+Join condition: None
+
+(49) Project [codegen id : 14]
+Output [2]: [ss_net_profit#2, s_store_name#10]
+Input [4]: [ss_net_profit#2, s_store_name#10, s_zip#11, ca_zip#28]
+
+(50) HashAggregate [codegen id : 14]
+Input [2]: [ss_net_profit#2, s_store_name#10]
+Keys [1]: [s_store_name#10]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))]
+Aggregate Attributes [1]: [sum#31]
+Results [2]: [s_store_name#10, sum#32]
+
+(51) Exchange
+Input [2]: [s_store_name#10, sum#32]
+Arguments: hashpartitioning(s_store_name#10, 5), ENSURE_REQUIREMENTS, [id=#33]
+
+(52) HashAggregate [codegen id : 15]
+Input [2]: [s_store_name#10, sum#32]
+Keys [1]: [s_store_name#10]
+Functions [1]: [sum(UnscaledValue(ss_net_profit#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#34]
+Results [2]: [s_store_name#10, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#34,17,2) AS sum(ss_net_profit)#35]
+
+(53) TakeOrderedAndProject
+Input [2]: [s_store_name#10, sum(ss_net_profit)#35]
+Arguments: 100, [s_store_name#10 ASC NULLS FIRST], [s_store_name#10, sum(ss_net_profit)#35]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (54)
+
+
+(54) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#5]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.sf100/simplified.txt
new file mode 100644
index 0000000000000..88cc9c98e1ed7
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.sf100/simplified.txt
@@ -0,0 +1,90 @@
+TakeOrderedAndProject [s_store_name,sum(ss_net_profit)]
+  WholeStageCodegen (15)
+    HashAggregate [s_store_name,sum] [sum(UnscaledValue(ss_net_profit)),sum(ss_net_profit),sum]
+      InputAdapter
+        Exchange [s_store_name] #1
+          WholeStageCodegen (14)
+            HashAggregate [s_store_name,ss_net_profit] [sum,sum]
+              Project [ss_net_profit,s_store_name]
+                SortMergeJoin [s_zip,ca_zip]
+                  InputAdapter
+                    WholeStageCodegen (4)
+                      Sort [s_zip]
+                        InputAdapter
+                          Exchange [s_zip] #2
+                            WholeStageCodegen (3)
+                              Project [ss_net_profit,s_store_name,s_zip]
+                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                  Project [ss_store_sk,ss_net_profit]
+                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                      Filter [ss_store_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #1
+                                                ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        BroadcastExchange #3
+                                          WholeStageCodegen (1)
+                                            Project [d_date_sk]
+                                              Filter [d_qoy,d_year,d_date_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_date_sk,d_year,d_qoy]
+                                  InputAdapter
+                                    BroadcastExchange #4
+                                      WholeStageCodegen (2)
+                                        Filter [s_store_sk,s_zip]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.store [s_store_sk,s_store_name,s_zip]
+                  InputAdapter
+                    WholeStageCodegen (13)
+                      Sort [ca_zip]
+                        InputAdapter
+                          Exchange [ca_zip] #5
+                            WholeStageCodegen (12)
+                              HashAggregate [ca_zip]
+                                InputAdapter
+                                  Exchange [ca_zip] #6
+                                    WholeStageCodegen (11)
+                                      HashAggregate [ca_zip]
+                                        Project [ca_zip]
+                                          BroadcastHashJoin [ca_zip,ca_zip]
+                                            Filter [ca_zip]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.customer_address [ca_zip]
+                                            InputAdapter
+                                              BroadcastExchange #7
+                                                WholeStageCodegen (10)
+                                                  Project [ca_zip]
+                                                    Filter [count(1)]
+                                                      HashAggregate [ca_zip,count] [count(1),ca_zip,count(1),count]
+                                                        InputAdapter
+                                                          Exchange [ca_zip] #8
+                                                            WholeStageCodegen (9)
+                                                              HashAggregate [ca_zip] [count,count]
+                                                                Project [ca_zip]
+                                                                  SortMergeJoin [ca_address_sk,c_current_addr_sk]
+                                                                    InputAdapter
+                                                                      WholeStageCodegen (6)
+                                                                        Sort [ca_address_sk]
+                                                                          InputAdapter
+                                                                            Exchange [ca_address_sk] #9
+                                                                              WholeStageCodegen (5)
+                                                                                Filter [ca_address_sk]
+                                                                                  ColumnarToRow
+                                                                                    InputAdapter
+                                                                                      Scan parquet default.customer_address [ca_address_sk,ca_zip]
+                                                                    InputAdapter
+                                                                      WholeStageCodegen (8)
+                                                                        Sort [c_current_addr_sk]
+                                                                          InputAdapter
+                                                                            Exchange [c_current_addr_sk] #10
+                                                                              WholeStageCodegen (7)
+                                                                                Project [c_current_addr_sk]
+                                                                                  Filter [c_preferred_cust_flag,c_current_addr_sk]
+                                                                                    ColumnarToRow
+                                                                                      InputAdapter
+                                                                                        Scan parquet default.customer [c_current_addr_sk,c_preferred_cust_flag]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/explain.txt
new file mode 100644
index 0000000000000..dbb5e1f606a8d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/explain.txt
@@ -0,0 +1,283 @@
+== Physical Plan ==
+TakeOrderedAndProject (47)
++- * HashAggregate (46)
+   +- Exchange (45)
+      +- * HashAggregate (44)
+         +- * Project (43)
+            +- * BroadcastHashJoin Inner BuildRight (42)
+               :- * Project (16)
+               :  +- * BroadcastHashJoin Inner BuildRight (15)
+               :     :- * Project (10)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :     :- * Filter (3)
+               :     :     :  +- * ColumnarToRow (2)
+               :     :     :     +- Scan parquet default.store_sales (1)
+               :     :     +- BroadcastExchange (8)
+               :     :        +- * Project (7)
+               :     :           +- * Filter (6)
+               :     :              +- * ColumnarToRow (5)
+               :     :                 +- Scan parquet default.date_dim (4)
+               :     +- BroadcastExchange (14)
+               :        +- * Filter (13)
+               :           +- * ColumnarToRow (12)
+               :              +- Scan parquet default.store (11)
+               +- BroadcastExchange (41)
+                  +- * HashAggregate (40)
+                     +- Exchange (39)
+                        +- * HashAggregate (38)
+                           +- * Project (37)
+                              +- * BroadcastHashJoin LeftSemi BuildRight (36)
+                                 :- * Filter (19)
+                                 :  +- * ColumnarToRow (18)
+                                 :     +- Scan parquet default.customer_address (17)
+                                 +- BroadcastExchange (35)
+                                    +- * Project (34)
+                                       +- * Filter (33)
+                                          +- * HashAggregate (32)
+                                             +- Exchange (31)
+                                                +- * HashAggregate (30)
+                                                   +- * Project (29)
+                                                      +- * BroadcastHashJoin Inner BuildRight (28)
+                                                         :- * Filter (22)
+                                                         :  +- * ColumnarToRow (21)
+                                                         :     +- Scan parquet default.customer_address (20)
+                                                         +- BroadcastExchange (27)
+                                                            +- * Project (26)
+                                                               +- * Filter (25)
+                                                                  +- * ColumnarToRow (24)
+                                                                     +- Scan parquet default.customer (23)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 8]
+Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 8]
+Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_store_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,1998), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_qoy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+Condition : ((((isnotnull(d_qoy#7) AND isnotnull(d_year#6)) AND (d_qoy#7 = 2)) AND (d_year#6 = 1998)) AND isnotnull(d_date_sk#5))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#5]
+Input [3]: [d_date_sk#5, d_year#6, d_qoy#7]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#8]
+
+(9) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#5]
+Join condition: None
+
+(10) Project [codegen id : 8]
+Output [2]: [ss_store_sk#1, ss_net_profit#2]
+Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#5]
+
+(11) Scan parquet default.store
+Output [3]: [s_store_sk#9, s_store_name#10, s_zip#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_zip)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_zip:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [s_store_sk#9, s_store_name#10, s_zip#11]
+
+(13) Filter [codegen id : 2]
+Input [3]: [s_store_sk#9, s_store_name#10, s_zip#11]
+Condition : (isnotnull(s_store_sk#9) AND isnotnull(s_zip#11))
+
+(14) BroadcastExchange
+Input [3]: [s_store_sk#9, s_store_name#10, s_zip#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#12]
+
+(15) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_store_sk#1]
+Right keys [1]: [s_store_sk#9]
+Join condition: None
+
+(16) Project [codegen id : 8]
+Output [3]: [ss_net_profit#2, s_store_name#10, s_zip#11]
+Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#9, s_store_name#10, s_zip#11]
+
+(17) Scan parquet default.customer_address
+Output [1]: [ca_zip#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+ReadSchema: struct<ca_zip:string>
+
+(18) ColumnarToRow [codegen id : 6]
+Input [1]: [ca_zip#13]
+
+(19) Filter [codegen id : 6]
+Input [1]: [ca_zip#13]
+Condition : (substr(ca_zip#13, 1, 5) INSET (56910,69952,63792,39371,74351,11101,25003,97189,57834,73134,62377,51200,32754,22752,86379,14171,91110,40162,98569,28709,13394,66162,25733,25782,26065,18383,51949,87343,50298,83849,33786,64528,23470,67030,46136,25280,46820,77721,99076,18426,31880,17871,98235,45748,49156,18652,72013,51622,43848,78567,41248,13695,44165,67853,54917,53179,64034,10567,71791,68908,55565,59402,64147,85816,57855,61547,27700,68100,28810,58263,15723,83933,51103,58058,90578,82276,81096,81426,96451,77556,38607,76638,18906,62971,57047,48425,35576,11928,30625,83444,73520,51650,57647,60099,30122,94983,24128,10445,41368,26233,26859,21756,24676,19849,36420,38193,58470,39127,13595,87501,24317,15455,69399,98025,81019,48033,11376,39516,67875,92712,14867,38122,29741,42961,30469,51211,56458,15559,16021,33123,33282,33515,72823,54601,76698,56240,72175,60279,20004,68806,72325,28488,43933,50412,45200,22246,78668,79777,96765,67301,73273,49448,82636,23932,47305,29839,39192,18799,61265,37125,58943,64457,88424,24610,84935,89360,68893,30431,28898,10336,90257,59166,46081,26105,96888,36634,86284,35258,39972,22927,73241,53268,24206,27385,99543,31671,14663,30903,39861,24996,63089,88086,83921,21076,67897,66708,45721,60576,25103,52867,30450,36233,30010,96576,73171,56571,56575,64544,13955,78451,43285,18119,16725,83041,76107,79994,54364,35942,56691,19769,63435,34102,18845,22744,13354,75691,45549,23968,31387,83144,13375,15765,28577,88190,19736,73650,37930,25989,83926,94898,51798,39736,22437,55253,38415,71256,18376,42029,25858,44438,19515,38935,51649,71954,15882,18767,63193,25486,49130,37126,40604,34425,17043,12305,11634,26653,94167,36446,10516,67473,66864,72425,63981,18842,22461,42666,47770,69035,70372,28587,45266,15371,15798,45375,90225,16807,31016,68014,21337,19505,50016,10144,84093,21286,19430,34322,91068,94945,72305,24671,58048,65084,28545,21195,20548,22245,77191,96976,48583,76231,15734,61810,11356,68621,68786,98359,41367,26689,69913,76614,68101,88885,50308,79077,18270,28915,29178,53672,62878,10390,14922,68341,56529,41766,68309,56616,15126,61860,97789,11489,45692,41918,72151,72550,27156,36495,70738,17879,53535,17920,68880,78890,35850,14089,58078,65164,27068,26231,13376,57665,32213,77610,87816,21309,15146,86198,91137,55307,67467,40558,94627,82136,22351,89091,20260,23006,91393,47537,62496,98294,18840,71286,81312,31029,70466,35458,14060,22685,28286,25631,19512,40081,63837,14328,35474,22152,76232,51061,86057,17183) AND isnotnull(substr(ca_zip#13, 1, 5)))
+
+(20) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#14, ca_zip#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_zip:string>
+
+(21) ColumnarToRow [codegen id : 4]
+Input [2]: [ca_address_sk#14, ca_zip#15]
+
+(22) Filter [codegen id : 4]
+Input [2]: [ca_address_sk#14, ca_zip#15]
+Condition : isnotnull(ca_address_sk#14)
+
+(23) Scan parquet default.customer
+Output [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_preferred_cust_flag), EqualTo(c_preferred_cust_flag,Y), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_current_addr_sk:int,c_preferred_cust_flag:string>
+
+(24) ColumnarToRow [codegen id : 3]
+Input [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17]
+
+(25) Filter [codegen id : 3]
+Input [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17]
+Condition : ((isnotnull(c_preferred_cust_flag#17) AND (c_preferred_cust_flag#17 = Y)) AND isnotnull(c_current_addr_sk#16))
+
+(26) Project [codegen id : 3]
+Output [1]: [c_current_addr_sk#16]
+Input [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17]
+
+(27) BroadcastExchange
+Input [1]: [c_current_addr_sk#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(28) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ca_address_sk#14]
+Right keys [1]: [c_current_addr_sk#16]
+Join condition: None
+
+(29) Project [codegen id : 4]
+Output [1]: [ca_zip#15]
+Input [3]: [ca_address_sk#14, ca_zip#15, c_current_addr_sk#16]
+
+(30) HashAggregate [codegen id : 4]
+Input [1]: [ca_zip#15]
+Keys [1]: [ca_zip#15]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#19]
+Results [2]: [ca_zip#15, count#20]
+
+(31) Exchange
+Input [2]: [ca_zip#15, count#20]
+Arguments: hashpartitioning(ca_zip#15, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(32) HashAggregate [codegen id : 5]
+Input [2]: [ca_zip#15, count#20]
+Keys [1]: [ca_zip#15]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#22]
+Results [2]: [substr(ca_zip#15, 1, 5) AS ca_zip#23, count(1)#22 AS count(1)#24]
+
+(33) Filter [codegen id : 5]
+Input [2]: [ca_zip#23, count(1)#24]
+Condition : (count(1)#24 > 10)
+
+(34) Project [codegen id : 5]
+Output [1]: [ca_zip#23]
+Input [2]: [ca_zip#23, count(1)#24]
+
+(35) BroadcastExchange
+Input [1]: [ca_zip#23]
+Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true])),false), [id=#25]
+
+(36) BroadcastHashJoin [codegen id : 6]
+Left keys [2]: [coalesce(substr(ca_zip#13, 1, 5), ), isnull(substr(ca_zip#13, 1, 5))]
+Right keys [2]: [coalesce(ca_zip#23, ), isnull(ca_zip#23)]
+Join condition: None
+
+(37) Project [codegen id : 6]
+Output [1]: [substr(ca_zip#13, 1, 5) AS ca_zip#26]
+Input [1]: [ca_zip#13]
+
+(38) HashAggregate [codegen id : 6]
+Input [1]: [ca_zip#26]
+Keys [1]: [ca_zip#26]
+Functions: []
+Aggregate Attributes: []
+Results [1]: [ca_zip#26]
+
+(39) Exchange
+Input [1]: [ca_zip#26]
+Arguments: hashpartitioning(ca_zip#26, 5), ENSURE_REQUIREMENTS, [id=#27]
+
+(40) HashAggregate [codegen id : 7]
+Input [1]: [ca_zip#26]
+Keys [1]: [ca_zip#26]
+Functions: []
+Aggregate Attributes: []
+Results [1]: [ca_zip#26]
+
+(41) BroadcastExchange
+Input [1]: [ca_zip#26]
+Arguments: HashedRelationBroadcastMode(List(substr(input[0, string, true], 1, 2)),false), [id=#28]
+
+(42) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [substr(s_zip#11, 1, 2)]
+Right keys [1]: [substr(ca_zip#26, 1, 2)]
+Join condition: None
+
+(43) Project [codegen id : 8]
+Output [2]: [ss_net_profit#2, s_store_name#10]
+Input [4]: [ss_net_profit#2, s_store_name#10, s_zip#11, ca_zip#26]
+
+(44) HashAggregate [codegen id : 8]
+Input [2]: [ss_net_profit#2, s_store_name#10]
+Keys [1]: [s_store_name#10]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))]
+Aggregate Attributes [1]: [sum#29]
+Results [2]: [s_store_name#10, sum#30]
+
+(45) Exchange
+Input [2]: [s_store_name#10, sum#30]
+Arguments: hashpartitioning(s_store_name#10, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(46) HashAggregate [codegen id : 9]
+Input [2]: [s_store_name#10, sum#30]
+Keys [1]: [s_store_name#10]
+Functions [1]: [sum(UnscaledValue(ss_net_profit#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#32]
+Results [2]: [s_store_name#10, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#32,17,2) AS sum(ss_net_profit)#33]
+
+(47) TakeOrderedAndProject
+Input [2]: [s_store_name#10, sum(ss_net_profit)#33]
+Arguments: 100, [s_store_name#10 ASC NULLS FIRST], [s_store_name#10, sum(ss_net_profit)#33]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (48)
+
+
+(48) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#5]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/simplified.txt
new file mode 100644
index 0000000000000..294468c04eea3
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/simplified.txt
@@ -0,0 +1,72 @@
+TakeOrderedAndProject [s_store_name,sum(ss_net_profit)]
+  WholeStageCodegen (9)
+    HashAggregate [s_store_name,sum] [sum(UnscaledValue(ss_net_profit)),sum(ss_net_profit),sum]
+      InputAdapter
+        Exchange [s_store_name] #1
+          WholeStageCodegen (8)
+            HashAggregate [s_store_name,ss_net_profit] [sum,sum]
+              Project [ss_net_profit,s_store_name]
+                BroadcastHashJoin [s_zip,ca_zip]
+                  Project [ss_net_profit,s_store_name,s_zip]
+                    BroadcastHashJoin [ss_store_sk,s_store_sk]
+                      Project [ss_store_sk,ss_net_profit]
+                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                          Filter [ss_store_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk]
+                                  SubqueryBroadcast [d_date_sk] #1
+                                    ReusedExchange [d_date_sk] #2
+                          InputAdapter
+                            BroadcastExchange #2
+                              WholeStageCodegen (1)
+                                Project [d_date_sk]
+                                  Filter [d_qoy,d_year,d_date_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.date_dim [d_date_sk,d_year,d_qoy]
+                      InputAdapter
+                        BroadcastExchange #3
+                          WholeStageCodegen (2)
+                            Filter [s_store_sk,s_zip]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.store [s_store_sk,s_store_name,s_zip]
+                  InputAdapter
+                    BroadcastExchange #4
+                      WholeStageCodegen (7)
+                        HashAggregate [ca_zip]
+                          InputAdapter
+                            Exchange [ca_zip] #5
+                              WholeStageCodegen (6)
+                                HashAggregate [ca_zip]
+                                  Project [ca_zip]
+                                    BroadcastHashJoin [ca_zip,ca_zip]
+                                      Filter [ca_zip]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.customer_address [ca_zip]
+                                      InputAdapter
+                                        BroadcastExchange #6
+                                          WholeStageCodegen (5)
+                                            Project [ca_zip]
+                                              Filter [count(1)]
+                                                HashAggregate [ca_zip,count] [count(1),ca_zip,count(1),count]
+                                                  InputAdapter
+                                                    Exchange [ca_zip] #7
+                                                      WholeStageCodegen (4)
+                                                        HashAggregate [ca_zip] [count,count]
+                                                          Project [ca_zip]
+                                                            BroadcastHashJoin [ca_address_sk,c_current_addr_sk]
+                                                              Filter [ca_address_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.customer_address [ca_address_sk,ca_zip]
+                                                              InputAdapter
+                                                                BroadcastExchange #8
+                                                                  WholeStageCodegen (3)
+                                                                    Project [c_current_addr_sk]
+                                                                      Filter [c_preferred_cust_flag,c_current_addr_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.customer [c_current_addr_sk,c_preferred_cust_flag]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.sf100/explain.txt
new file mode 100644
index 0000000000000..d9794c33a767d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.sf100/explain.txt
@@ -0,0 +1,630 @@
+== Physical Plan ==
+TakeOrderedAndProject (111)
++- * HashAggregate (110)
+   +- Exchange (109)
+      +- * HashAggregate (108)
+         +- * Expand (107)
+            +- Union (106)
+               :- * HashAggregate (43)
+               :  +- Exchange (42)
+               :     +- * HashAggregate (41)
+               :        +- * Project (40)
+               :           +- * BroadcastHashJoin Inner BuildRight (39)
+               :              :- * Project (34)
+               :              :  +- * BroadcastHashJoin Inner BuildRight (33)
+               :              :     :- * Project (27)
+               :              :     :  +- * BroadcastHashJoin Inner BuildRight (26)
+               :              :     :     :- * Project (20)
+               :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (19)
+               :              :     :     :     :- * Project (13)
+               :              :     :     :     :  +- * SortMergeJoin LeftOuter (12)
+               :              :     :     :     :     :- * Sort (5)
+               :              :     :     :     :     :  +- Exchange (4)
+               :              :     :     :     :     :     +- * Filter (3)
+               :              :     :     :     :     :        +- * ColumnarToRow (2)
+               :              :     :     :     :     :           +- Scan parquet default.store_sales (1)
+               :              :     :     :     :     +- * Sort (11)
+               :              :     :     :     :        +- Exchange (10)
+               :              :     :     :     :           +- * Project (9)
+               :              :     :     :     :              +- * Filter (8)
+               :              :     :     :     :                 +- * ColumnarToRow (7)
+               :              :     :     :     :                    +- Scan parquet default.store_returns (6)
+               :              :     :     :     +- BroadcastExchange (18)
+               :              :     :     :        +- * Project (17)
+               :              :     :     :           +- * Filter (16)
+               :              :     :     :              +- * ColumnarToRow (15)
+               :              :     :     :                 +- Scan parquet default.item (14)
+               :              :     :     +- BroadcastExchange (25)
+               :              :     :        +- * Project (24)
+               :              :     :           +- * Filter (23)
+               :              :     :              +- * ColumnarToRow (22)
+               :              :     :                 +- Scan parquet default.promotion (21)
+               :              :     +- BroadcastExchange (32)
+               :              :        +- * Project (31)
+               :              :           +- * Filter (30)
+               :              :              +- * ColumnarToRow (29)
+               :              :                 +- Scan parquet default.date_dim (28)
+               :              +- BroadcastExchange (38)
+               :                 +- * Filter (37)
+               :                    +- * ColumnarToRow (36)
+               :                       +- Scan parquet default.store (35)
+               :- * HashAggregate (74)
+               :  +- Exchange (73)
+               :     +- * HashAggregate (72)
+               :        +- * Project (71)
+               :           +- * BroadcastHashJoin Inner BuildRight (70)
+               :              :- * Project (65)
+               :              :  +- * BroadcastHashJoin Inner BuildRight (64)
+               :              :     :- * Project (62)
+               :              :     :  +- * BroadcastHashJoin Inner BuildRight (61)
+               :              :     :     :- * Project (59)
+               :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (58)
+               :              :     :     :     :- * Project (56)
+               :              :     :     :     :  +- * SortMergeJoin LeftOuter (55)
+               :              :     :     :     :     :- * Sort (48)
+               :              :     :     :     :     :  +- Exchange (47)
+               :              :     :     :     :     :     +- * Filter (46)
+               :              :     :     :     :     :        +- * ColumnarToRow (45)
+               :              :     :     :     :     :           +- Scan parquet default.catalog_sales (44)
+               :              :     :     :     :     +- * Sort (54)
+               :              :     :     :     :        +- Exchange (53)
+               :              :     :     :     :           +- * Project (52)
+               :              :     :     :     :              +- * Filter (51)
+               :              :     :     :     :                 +- * ColumnarToRow (50)
+               :              :     :     :     :                    +- Scan parquet default.catalog_returns (49)
+               :              :     :     :     +- ReusedExchange (57)
+               :              :     :     +- ReusedExchange (60)
+               :              :     +- ReusedExchange (63)
+               :              +- BroadcastExchange (69)
+               :                 +- * Filter (68)
+               :                    +- * ColumnarToRow (67)
+               :                       +- Scan parquet default.catalog_page (66)
+               +- * HashAggregate (105)
+                  +- Exchange (104)
+                     +- * HashAggregate (103)
+                        +- * Project (102)
+                           +- * BroadcastHashJoin Inner BuildRight (101)
+                              :- * Project (96)
+                              :  +- * BroadcastHashJoin Inner BuildRight (95)
+                              :     :- * Project (93)
+                              :     :  +- * BroadcastHashJoin Inner BuildRight (92)
+                              :     :     :- * Project (90)
+                              :     :     :  +- * BroadcastHashJoin Inner BuildRight (89)
+                              :     :     :     :- * Project (87)
+                              :     :     :     :  +- * SortMergeJoin LeftOuter (86)
+                              :     :     :     :     :- * Sort (79)
+                              :     :     :     :     :  +- Exchange (78)
+                              :     :     :     :     :     +- * Filter (77)
+                              :     :     :     :     :        +- * ColumnarToRow (76)
+                              :     :     :     :     :           +- Scan parquet default.web_sales (75)
+                              :     :     :     :     +- * Sort (85)
+                              :     :     :     :        +- Exchange (84)
+                              :     :     :     :           +- * Project (83)
+                              :     :     :     :              +- * Filter (82)
+                              :     :     :     :                 +- * ColumnarToRow (81)
+                              :     :     :     :                    +- Scan parquet default.web_returns (80)
+                              :     :     :     +- ReusedExchange (88)
+                              :     :     +- ReusedExchange (91)
+                              :     +- ReusedExchange (94)
+                              +- BroadcastExchange (100)
+                                 +- * Filter (99)
+                                    +- * ColumnarToRow (98)
+                                       +- Scan parquet default.web_site (97)
+
+
+(1) Scan parquet default.store_sales
+Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_promo_sk:int,ss_ticket_number:int,ss_ext_sales_price:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+
+(3) Filter [codegen id : 1]
+Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3))
+
+(4) Exchange
+Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, [id=#9]
+
+(5) Sort [codegen id : 2]
+Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+Arguments: [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.store_returns
+Output [5]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int,sr_return_amt:decimal(7,2),sr_net_loss:decimal(7,2)>
+
+(7) ColumnarToRow [codegen id : 3]
+Input [5]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14]
+
+(8) Filter [codegen id : 3]
+Input [5]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14]
+Condition : (isnotnull(sr_item_sk#10) AND isnotnull(sr_ticket_number#11))
+
+(9) Project [codegen id : 3]
+Output [4]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13]
+Input [5]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14]
+
+(10) Exchange
+Input [4]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13]
+Arguments: hashpartitioning(sr_item_sk#10, sr_ticket_number#11, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(11) Sort [codegen id : 4]
+Input [4]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13]
+Arguments: [sr_item_sk#10 ASC NULLS FIRST, sr_ticket_number#11 ASC NULLS FIRST], false, 0
+
+(12) SortMergeJoin [codegen id : 9]
+Left keys [2]: [ss_item_sk#1, ss_ticket_number#4]
+Right keys [2]: [sr_item_sk#10, sr_ticket_number#11]
+Join condition: None
+
+(13) Project [codegen id : 9]
+Output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#12, sr_net_loss#13]
+Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13]
+
+(14) Scan parquet default.item
+Output [2]: [i_item_sk#16, i_current_price#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_current_price:decimal(7,2)>
+
+(15) ColumnarToRow [codegen id : 5]
+Input [2]: [i_item_sk#16, i_current_price#17]
+
+(16) Filter [codegen id : 5]
+Input [2]: [i_item_sk#16, i_current_price#17]
+Condition : ((isnotnull(i_current_price#17) AND (i_current_price#17 > 50.00)) AND isnotnull(i_item_sk#16))
+
+(17) Project [codegen id : 5]
+Output [1]: [i_item_sk#16]
+Input [2]: [i_item_sk#16, i_current_price#17]
+
+(18) BroadcastExchange
+Input [1]: [i_item_sk#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(19) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#16]
+Join condition: None
+
+(20) Project [codegen id : 9]
+Output [7]: [ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#12, sr_net_loss#13]
+Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#12, sr_net_loss#13, i_item_sk#16]
+
+(21) Scan parquet default.promotion
+Output [2]: [p_promo_sk#19, p_channel_tv#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/promotion]
+PushedFilters: [IsNotNull(p_channel_tv), EqualTo(p_channel_tv,N), IsNotNull(p_promo_sk)]
+ReadSchema: struct<p_promo_sk:int,p_channel_tv:string>
+
+(22) ColumnarToRow [codegen id : 6]
+Input [2]: [p_promo_sk#19, p_channel_tv#20]
+
+(23) Filter [codegen id : 6]
+Input [2]: [p_promo_sk#19, p_channel_tv#20]
+Condition : ((isnotnull(p_channel_tv#20) AND (p_channel_tv#20 = N)) AND isnotnull(p_promo_sk#19))
+
+(24) Project [codegen id : 6]
+Output [1]: [p_promo_sk#19]
+Input [2]: [p_promo_sk#19, p_channel_tv#20]
+
+(25) BroadcastExchange
+Input [1]: [p_promo_sk#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(26) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_promo_sk#3]
+Right keys [1]: [p_promo_sk#19]
+Join condition: None
+
+(27) Project [codegen id : 9]
+Output [6]: [ss_store_sk#2, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#12, sr_net_loss#13]
+Input [8]: [ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#12, sr_net_loss#13, p_promo_sk#19]
+
+(28) Scan parquet default.date_dim
+Output [2]: [d_date_sk#22, d_date#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-22), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(29) ColumnarToRow [codegen id : 7]
+Input [2]: [d_date_sk#22, d_date#23]
+
+(30) Filter [codegen id : 7]
+Input [2]: [d_date_sk#22, d_date#23]
+Condition : (((isnotnull(d_date#23) AND (d_date#23 >= 2000-08-23)) AND (d_date#23 <= 2000-09-22)) AND isnotnull(d_date_sk#22))
+
+(31) Project [codegen id : 7]
+Output [1]: [d_date_sk#22]
+Input [2]: [d_date_sk#22, d_date#23]
+
+(32) BroadcastExchange
+Input [1]: [d_date_sk#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#24]
+
+(33) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_sold_date_sk#7]
+Right keys [1]: [d_date_sk#22]
+Join condition: None
+
+(34) Project [codegen id : 9]
+Output [5]: [ss_store_sk#2, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#12, sr_net_loss#13]
+Input [7]: [ss_store_sk#2, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#12, sr_net_loss#13, d_date_sk#22]
+
+(35) Scan parquet default.store
+Output [2]: [s_store_sk#25, s_store_id#26]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string>
+
+(36) ColumnarToRow [codegen id : 8]
+Input [2]: [s_store_sk#25, s_store_id#26]
+
+(37) Filter [codegen id : 8]
+Input [2]: [s_store_sk#25, s_store_id#26]
+Condition : isnotnull(s_store_sk#25)
+
+(38) BroadcastExchange
+Input [2]: [s_store_sk#25, s_store_id#26]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#27]
+
+(39) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_store_sk#2]
+Right keys [1]: [s_store_sk#25]
+Join condition: None
+
+(40) Project [codegen id : 9]
+Output [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#12, sr_net_loss#13, s_store_id#26]
+Input [7]: [ss_store_sk#2, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#12, sr_net_loss#13, s_store_sk#25, s_store_id#26]
+
+(41) HashAggregate [codegen id : 9]
+Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#12, sr_net_loss#13, s_store_id#26]
+Keys [1]: [s_store_id#26]
+Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#12 as decimal(12,2)), 0.00)), partial_sum(CheckOverflow((promote_precision(cast(ss_net_profit#6 as decimal(13,2))) - promote_precision(cast(coalesce(cast(sr_net_loss#13 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))]
+Aggregate Attributes [5]: [sum#28, sum#29, isEmpty#30, sum#31, isEmpty#32]
+Results [6]: [s_store_id#26, sum#33, sum#34, isEmpty#35, sum#36, isEmpty#37]
+
+(42) Exchange
+Input [6]: [s_store_id#26, sum#33, sum#34, isEmpty#35, sum#36, isEmpty#37]
+Arguments: hashpartitioning(s_store_id#26, 5), ENSURE_REQUIREMENTS, [id=#38]
+
+(43) HashAggregate [codegen id : 10]
+Input [6]: [s_store_id#26, sum#33, sum#34, isEmpty#35, sum#36, isEmpty#37]
+Keys [1]: [s_store_id#26]
+Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#12 as decimal(12,2)), 0.00)), sum(CheckOverflow((promote_precision(cast(ss_net_profit#6 as decimal(13,2))) - promote_precision(cast(coalesce(cast(sr_net_loss#13 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))]
+Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#39, sum(coalesce(cast(sr_return_amt#12 as decimal(12,2)), 0.00))#40, sum(CheckOverflow((promote_precision(cast(ss_net_profit#6 as decimal(13,2))) - promote_precision(cast(coalesce(cast(sr_net_loss#13 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))#41]
+Results [5]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#39,17,2) AS sales#42, sum(coalesce(cast(sr_return_amt#12 as decimal(12,2)), 0.00))#40 AS returns#43, sum(CheckOverflow((promote_precision(cast(ss_net_profit#6 as decimal(13,2))) - promote_precision(cast(coalesce(cast(sr_net_loss#13 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))#41 AS profit#44, store channel AS channel#45, concat(store, s_store_id#26) AS id#46]
+
+(44) Scan parquet default.catalog_sales
+Output [7]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_order_number#50, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#53), dynamicpruningexpression(cs_sold_date_sk#53 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)]
+ReadSchema: struct<cs_catalog_page_sk:int,cs_item_sk:int,cs_promo_sk:int,cs_order_number:int,cs_ext_sales_price:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(45) ColumnarToRow [codegen id : 11]
+Input [7]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_order_number#50, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53]
+
+(46) Filter [codegen id : 11]
+Input [7]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_order_number#50, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53]
+Condition : ((isnotnull(cs_catalog_page_sk#47) AND isnotnull(cs_item_sk#48)) AND isnotnull(cs_promo_sk#49))
+
+(47) Exchange
+Input [7]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_order_number#50, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53]
+Arguments: hashpartitioning(cs_item_sk#48, cs_order_number#50, 5), ENSURE_REQUIREMENTS, [id=#54]
+
+(48) Sort [codegen id : 12]
+Input [7]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_order_number#50, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53]
+Arguments: [cs_item_sk#48 ASC NULLS FIRST, cs_order_number#50 ASC NULLS FIRST], false, 0
+
+(49) Scan parquet default.catalog_returns
+Output [5]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int,cr_return_amount:decimal(7,2),cr_net_loss:decimal(7,2)>
+
+(50) ColumnarToRow [codegen id : 13]
+Input [5]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+
+(51) Filter [codegen id : 13]
+Input [5]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+Condition : (isnotnull(cr_item_sk#55) AND isnotnull(cr_order_number#56))
+
+(52) Project [codegen id : 13]
+Output [4]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58]
+Input [5]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+
+(53) Exchange
+Input [4]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58]
+Arguments: hashpartitioning(cr_item_sk#55, cr_order_number#56, 5), ENSURE_REQUIREMENTS, [id=#60]
+
+(54) Sort [codegen id : 14]
+Input [4]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58]
+Arguments: [cr_item_sk#55 ASC NULLS FIRST, cr_order_number#56 ASC NULLS FIRST], false, 0
+
+(55) SortMergeJoin [codegen id : 19]
+Left keys [2]: [cs_item_sk#48, cs_order_number#50]
+Right keys [2]: [cr_item_sk#55, cr_order_number#56]
+Join condition: None
+
+(56) Project [codegen id : 19]
+Output [8]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53, cr_return_amount#57, cr_net_loss#58]
+Input [11]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_order_number#50, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53, cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58]
+
+(57) ReusedExchange [Reuses operator id: 18]
+Output [1]: [i_item_sk#61]
+
+(58) BroadcastHashJoin [codegen id : 19]
+Left keys [1]: [cs_item_sk#48]
+Right keys [1]: [i_item_sk#61]
+Join condition: None
+
+(59) Project [codegen id : 19]
+Output [7]: [cs_catalog_page_sk#47, cs_promo_sk#49, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53, cr_return_amount#57, cr_net_loss#58]
+Input [9]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53, cr_return_amount#57, cr_net_loss#58, i_item_sk#61]
+
+(60) ReusedExchange [Reuses operator id: 25]
+Output [1]: [p_promo_sk#62]
+
+(61) BroadcastHashJoin [codegen id : 19]
+Left keys [1]: [cs_promo_sk#49]
+Right keys [1]: [p_promo_sk#62]
+Join condition: None
+
+(62) Project [codegen id : 19]
+Output [6]: [cs_catalog_page_sk#47, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53, cr_return_amount#57, cr_net_loss#58]
+Input [8]: [cs_catalog_page_sk#47, cs_promo_sk#49, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53, cr_return_amount#57, cr_net_loss#58, p_promo_sk#62]
+
+(63) ReusedExchange [Reuses operator id: 32]
+Output [1]: [d_date_sk#63]
+
+(64) BroadcastHashJoin [codegen id : 19]
+Left keys [1]: [cs_sold_date_sk#53]
+Right keys [1]: [d_date_sk#63]
+Join condition: None
+
+(65) Project [codegen id : 19]
+Output [5]: [cs_catalog_page_sk#47, cs_ext_sales_price#51, cs_net_profit#52, cr_return_amount#57, cr_net_loss#58]
+Input [7]: [cs_catalog_page_sk#47, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53, cr_return_amount#57, cr_net_loss#58, d_date_sk#63]
+
+(66) Scan parquet default.catalog_page
+Output [2]: [cp_catalog_page_sk#64, cp_catalog_page_id#65]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_page]
+PushedFilters: [IsNotNull(cp_catalog_page_sk)]
+ReadSchema: struct<cp_catalog_page_sk:int,cp_catalog_page_id:string>
+
+(67) ColumnarToRow [codegen id : 18]
+Input [2]: [cp_catalog_page_sk#64, cp_catalog_page_id#65]
+
+(68) Filter [codegen id : 18]
+Input [2]: [cp_catalog_page_sk#64, cp_catalog_page_id#65]
+Condition : isnotnull(cp_catalog_page_sk#64)
+
+(69) BroadcastExchange
+Input [2]: [cp_catalog_page_sk#64, cp_catalog_page_id#65]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#66]
+
+(70) BroadcastHashJoin [codegen id : 19]
+Left keys [1]: [cs_catalog_page_sk#47]
+Right keys [1]: [cp_catalog_page_sk#64]
+Join condition: None
+
+(71) Project [codegen id : 19]
+Output [5]: [cs_ext_sales_price#51, cs_net_profit#52, cr_return_amount#57, cr_net_loss#58, cp_catalog_page_id#65]
+Input [7]: [cs_catalog_page_sk#47, cs_ext_sales_price#51, cs_net_profit#52, cr_return_amount#57, cr_net_loss#58, cp_catalog_page_sk#64, cp_catalog_page_id#65]
+
+(72) HashAggregate [codegen id : 19]
+Input [5]: [cs_ext_sales_price#51, cs_net_profit#52, cr_return_amount#57, cr_net_loss#58, cp_catalog_page_id#65]
+Keys [1]: [cp_catalog_page_id#65]
+Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#51)), partial_sum(coalesce(cast(cr_return_amount#57 as decimal(12,2)), 0.00)), partial_sum(CheckOverflow((promote_precision(cast(cs_net_profit#52 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_net_loss#58 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))]
+Aggregate Attributes [5]: [sum#67, sum#68, isEmpty#69, sum#70, isEmpty#71]
+Results [6]: [cp_catalog_page_id#65, sum#72, sum#73, isEmpty#74, sum#75, isEmpty#76]
+
+(73) Exchange
+Input [6]: [cp_catalog_page_id#65, sum#72, sum#73, isEmpty#74, sum#75, isEmpty#76]
+Arguments: hashpartitioning(cp_catalog_page_id#65, 5), ENSURE_REQUIREMENTS, [id=#77]
+
+(74) HashAggregate [codegen id : 20]
+Input [6]: [cp_catalog_page_id#65, sum#72, sum#73, isEmpty#74, sum#75, isEmpty#76]
+Keys [1]: [cp_catalog_page_id#65]
+Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#51)), sum(coalesce(cast(cr_return_amount#57 as decimal(12,2)), 0.00)), sum(CheckOverflow((promote_precision(cast(cs_net_profit#52 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_net_loss#58 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))]
+Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#51))#78, sum(coalesce(cast(cr_return_amount#57 as decimal(12,2)), 0.00))#79, sum(CheckOverflow((promote_precision(cast(cs_net_profit#52 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_net_loss#58 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))#80]
+Results [5]: [MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#51))#78,17,2) AS sales#81, sum(coalesce(cast(cr_return_amount#57 as decimal(12,2)), 0.00))#79 AS returns#82, sum(CheckOverflow((promote_precision(cast(cs_net_profit#52 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_net_loss#58 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))#80 AS profit#83, catalog channel AS channel#84, concat(catalog_page, cp_catalog_page_id#65) AS id#85]
+
+(75) Scan parquet default.web_sales
+Output [7]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_order_number#89, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#92), dynamicpruningexpression(ws_sold_date_sk#92 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_web_site_sk:int,ws_promo_sk:int,ws_order_number:int,ws_ext_sales_price:decimal(7,2),ws_net_profit:decimal(7,2)>
+
+(76) ColumnarToRow [codegen id : 21]
+Input [7]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_order_number#89, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92]
+
+(77) Filter [codegen id : 21]
+Input [7]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_order_number#89, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92]
+Condition : ((isnotnull(ws_web_site_sk#87) AND isnotnull(ws_item_sk#86)) AND isnotnull(ws_promo_sk#88))
+
+(78) Exchange
+Input [7]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_order_number#89, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92]
+Arguments: hashpartitioning(ws_item_sk#86, ws_order_number#89, 5), ENSURE_REQUIREMENTS, [id=#93]
+
+(79) Sort [codegen id : 22]
+Input [7]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_order_number#89, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92]
+Arguments: [ws_item_sk#86 ASC NULLS FIRST, ws_order_number#89 ASC NULLS FIRST], false, 0
+
+(80) Scan parquet default.web_returns
+Output [5]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_returns]
+PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)]
+ReadSchema: struct<wr_item_sk:int,wr_order_number:int,wr_return_amt:decimal(7,2),wr_net_loss:decimal(7,2)>
+
+(81) ColumnarToRow [codegen id : 23]
+Input [5]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98]
+
+(82) Filter [codegen id : 23]
+Input [5]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98]
+Condition : (isnotnull(wr_item_sk#94) AND isnotnull(wr_order_number#95))
+
+(83) Project [codegen id : 23]
+Output [4]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97]
+Input [5]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98]
+
+(84) Exchange
+Input [4]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97]
+Arguments: hashpartitioning(wr_item_sk#94, wr_order_number#95, 5), ENSURE_REQUIREMENTS, [id=#99]
+
+(85) Sort [codegen id : 24]
+Input [4]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97]
+Arguments: [wr_item_sk#94 ASC NULLS FIRST, wr_order_number#95 ASC NULLS FIRST], false, 0
+
+(86) SortMergeJoin [codegen id : 29]
+Left keys [2]: [ws_item_sk#86, ws_order_number#89]
+Right keys [2]: [wr_item_sk#94, wr_order_number#95]
+Join condition: None
+
+(87) Project [codegen id : 29]
+Output [8]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92, wr_return_amt#96, wr_net_loss#97]
+Input [11]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_order_number#89, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92, wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97]
+
+(88) ReusedExchange [Reuses operator id: 18]
+Output [1]: [i_item_sk#100]
+
+(89) BroadcastHashJoin [codegen id : 29]
+Left keys [1]: [ws_item_sk#86]
+Right keys [1]: [i_item_sk#100]
+Join condition: None
+
+(90) Project [codegen id : 29]
+Output [7]: [ws_web_site_sk#87, ws_promo_sk#88, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92, wr_return_amt#96, wr_net_loss#97]
+Input [9]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92, wr_return_amt#96, wr_net_loss#97, i_item_sk#100]
+
+(91) ReusedExchange [Reuses operator id: 25]
+Output [1]: [p_promo_sk#101]
+
+(92) BroadcastHashJoin [codegen id : 29]
+Left keys [1]: [ws_promo_sk#88]
+Right keys [1]: [p_promo_sk#101]
+Join condition: None
+
+(93) Project [codegen id : 29]
+Output [6]: [ws_web_site_sk#87, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92, wr_return_amt#96, wr_net_loss#97]
+Input [8]: [ws_web_site_sk#87, ws_promo_sk#88, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92, wr_return_amt#96, wr_net_loss#97, p_promo_sk#101]
+
+(94) ReusedExchange [Reuses operator id: 32]
+Output [1]: [d_date_sk#102]
+
+(95) BroadcastHashJoin [codegen id : 29]
+Left keys [1]: [ws_sold_date_sk#92]
+Right keys [1]: [d_date_sk#102]
+Join condition: None
+
+(96) Project [codegen id : 29]
+Output [5]: [ws_web_site_sk#87, ws_ext_sales_price#90, ws_net_profit#91, wr_return_amt#96, wr_net_loss#97]
+Input [7]: [ws_web_site_sk#87, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92, wr_return_amt#96, wr_net_loss#97, d_date_sk#102]
+
+(97) Scan parquet default.web_site
+Output [2]: [web_site_sk#103, web_site_id#104]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_site]
+PushedFilters: [IsNotNull(web_site_sk)]
+ReadSchema: struct<web_site_sk:int,web_site_id:string>
+
+(98) ColumnarToRow [codegen id : 28]
+Input [2]: [web_site_sk#103, web_site_id#104]
+
+(99) Filter [codegen id : 28]
+Input [2]: [web_site_sk#103, web_site_id#104]
+Condition : isnotnull(web_site_sk#103)
+
+(100) BroadcastExchange
+Input [2]: [web_site_sk#103, web_site_id#104]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#105]
+
+(101) BroadcastHashJoin [codegen id : 29]
+Left keys [1]: [ws_web_site_sk#87]
+Right keys [1]: [web_site_sk#103]
+Join condition: None
+
+(102) Project [codegen id : 29]
+Output [5]: [ws_ext_sales_price#90, ws_net_profit#91, wr_return_amt#96, wr_net_loss#97, web_site_id#104]
+Input [7]: [ws_web_site_sk#87, ws_ext_sales_price#90, ws_net_profit#91, wr_return_amt#96, wr_net_loss#97, web_site_sk#103, web_site_id#104]
+
+(103) HashAggregate [codegen id : 29]
+Input [5]: [ws_ext_sales_price#90, ws_net_profit#91, wr_return_amt#96, wr_net_loss#97, web_site_id#104]
+Keys [1]: [web_site_id#104]
+Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#90)), partial_sum(coalesce(cast(wr_return_amt#96 as decimal(12,2)), 0.00)), partial_sum(CheckOverflow((promote_precision(cast(ws_net_profit#91 as decimal(13,2))) - promote_precision(cast(coalesce(cast(wr_net_loss#97 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))]
+Aggregate Attributes [5]: [sum#106, sum#107, isEmpty#108, sum#109, isEmpty#110]
+Results [6]: [web_site_id#104, sum#111, sum#112, isEmpty#113, sum#114, isEmpty#115]
+
+(104) Exchange
+Input [6]: [web_site_id#104, sum#111, sum#112, isEmpty#113, sum#114, isEmpty#115]
+Arguments: hashpartitioning(web_site_id#104, 5), ENSURE_REQUIREMENTS, [id=#116]
+
+(105) HashAggregate [codegen id : 30]
+Input [6]: [web_site_id#104, sum#111, sum#112, isEmpty#113, sum#114, isEmpty#115]
+Keys [1]: [web_site_id#104]
+Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#90)), sum(coalesce(cast(wr_return_amt#96 as decimal(12,2)), 0.00)), sum(CheckOverflow((promote_precision(cast(ws_net_profit#91 as decimal(13,2))) - promote_precision(cast(coalesce(cast(wr_net_loss#97 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))]
+Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#90))#117, sum(coalesce(cast(wr_return_amt#96 as decimal(12,2)), 0.00))#118, sum(CheckOverflow((promote_precision(cast(ws_net_profit#91 as decimal(13,2))) - promote_precision(cast(coalesce(cast(wr_net_loss#97 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))#119]
+Results [5]: [MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#90))#117,17,2) AS sales#120, sum(coalesce(cast(wr_return_amt#96 as decimal(12,2)), 0.00))#118 AS returns#121, sum(CheckOverflow((promote_precision(cast(ws_net_profit#91 as decimal(13,2))) - promote_precision(cast(coalesce(cast(wr_net_loss#97 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))#119 AS profit#122, web channel AS channel#123, concat(web_site, web_site_id#104) AS id#124]
+
+(106) Union
+
+(107) Expand [codegen id : 31]
+Input [5]: [sales#42, returns#43, profit#44, channel#45, id#46]
+Arguments: [ArrayBuffer(sales#42, returns#43, profit#44, channel#45, id#46, 0), ArrayBuffer(sales#42, returns#43, profit#44, channel#45, null, 1), ArrayBuffer(sales#42, returns#43, profit#44, null, null, 3)], [sales#42, returns#43, profit#44, channel#125, id#126, spark_grouping_id#127]
+
+(108) HashAggregate [codegen id : 31]
+Input [6]: [sales#42, returns#43, profit#44, channel#125, id#126, spark_grouping_id#127]
+Keys [3]: [channel#125, id#126, spark_grouping_id#127]
+Functions [3]: [partial_sum(sales#42), partial_sum(returns#43), partial_sum(profit#44)]
+Aggregate Attributes [6]: [sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133]
+Results [9]: [channel#125, id#126, spark_grouping_id#127, sum#134, isEmpty#135, sum#136, isEmpty#137, sum#138, isEmpty#139]
+
+(109) Exchange
+Input [9]: [channel#125, id#126, spark_grouping_id#127, sum#134, isEmpty#135, sum#136, isEmpty#137, sum#138, isEmpty#139]
+Arguments: hashpartitioning(channel#125, id#126, spark_grouping_id#127, 5), ENSURE_REQUIREMENTS, [id=#140]
+
+(110) HashAggregate [codegen id : 32]
+Input [9]: [channel#125, id#126, spark_grouping_id#127, sum#134, isEmpty#135, sum#136, isEmpty#137, sum#138, isEmpty#139]
+Keys [3]: [channel#125, id#126, spark_grouping_id#127]
+Functions [3]: [sum(sales#42), sum(returns#43), sum(profit#44)]
+Aggregate Attributes [3]: [sum(sales#42)#141, sum(returns#43)#142, sum(profit#44)#143]
+Results [5]: [channel#125, id#126, sum(sales#42)#141 AS sales#144, sum(returns#43)#142 AS returns#145, sum(profit#44)#143 AS profit#146]
+
+(111) TakeOrderedAndProject
+Input [5]: [channel#125, id#126, sales#144, returns#145, profit#146]
+Arguments: 100, [channel#125 ASC NULLS FIRST, id#126 ASC NULLS FIRST], [channel#125, id#126, sales#144, returns#145, profit#146]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (112)
+
+
+(112) ReusedExchange [Reuses operator id: 32]
+Output [1]: [d_date_sk#22]
+
+Subquery:2 Hosting operator id = 44 Hosting Expression = cs_sold_date_sk#53 IN dynamicpruning#8
+
+Subquery:3 Hosting operator id = 75 Hosting Expression = ws_sold_date_sk#92 IN dynamicpruning#8
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.sf100/simplified.txt
new file mode 100644
index 0000000000000..4155b5176397e
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.sf100/simplified.txt
@@ -0,0 +1,182 @@
+TakeOrderedAndProject [channel,id,sales,returns,profit]
+  WholeStageCodegen (32)
+    HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+      InputAdapter
+        Exchange [channel,id,spark_grouping_id] #1
+          WholeStageCodegen (31)
+            HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+              Expand [sales,returns,profit,channel,id]
+                InputAdapter
+                  Union
+                    WholeStageCodegen (10)
+                      HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(CheckOverflow((promote_precision(cast(ss_net_profit as decimal(13,2))) - promote_precision(cast(coalesce(cast(sr_net_loss as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true)),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty]
+                        InputAdapter
+                          Exchange [s_store_id] #2
+                            WholeStageCodegen (9)
+                              HashAggregate [s_store_id,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty]
+                                Project [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id]
+                                  BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                    Project [ss_store_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss]
+                                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                        Project [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss]
+                                          BroadcastHashJoin [ss_promo_sk,p_promo_sk]
+                                            Project [ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss]
+                                              BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss]
+                                                  SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number]
+                                                    InputAdapter
+                                                      WholeStageCodegen (2)
+                                                        Sort [ss_item_sk,ss_ticket_number]
+                                                          InputAdapter
+                                                            Exchange [ss_item_sk,ss_ticket_number] #3
+                                                              WholeStageCodegen (1)
+                                                                Filter [ss_store_sk,ss_item_sk,ss_promo_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk]
+                                                                        SubqueryBroadcast [d_date_sk] #1
+                                                                          ReusedExchange [d_date_sk] #4
+                                                    InputAdapter
+                                                      WholeStageCodegen (4)
+                                                        Sort [sr_item_sk,sr_ticket_number]
+                                                          InputAdapter
+                                                            Exchange [sr_item_sk,sr_ticket_number] #5
+                                                              WholeStageCodegen (3)
+                                                                Project [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss]
+                                                                  Filter [sr_item_sk,sr_ticket_number]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk]
+                                                InputAdapter
+                                                  BroadcastExchange #6
+                                                    WholeStageCodegen (5)
+                                                      Project [i_item_sk]
+                                                        Filter [i_current_price,i_item_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.item [i_item_sk,i_current_price]
+                                            InputAdapter
+                                              BroadcastExchange #7
+                                                WholeStageCodegen (6)
+                                                  Project [p_promo_sk]
+                                                    Filter [p_channel_tv,p_promo_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.promotion [p_promo_sk,p_channel_tv]
+                                        InputAdapter
+                                          BroadcastExchange #4
+                                            WholeStageCodegen (7)
+                                              Project [d_date_sk]
+                                                Filter [d_date,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_date]
+                                    InputAdapter
+                                      BroadcastExchange #8
+                                        WholeStageCodegen (8)
+                                          Filter [s_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store [s_store_sk,s_store_id]
+                    WholeStageCodegen (20)
+                      HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(CheckOverflow((promote_precision(cast(cs_net_profit as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_net_loss as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true)),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty]
+                        InputAdapter
+                          Exchange [cp_catalog_page_id] #9
+                            WholeStageCodegen (19)
+                              HashAggregate [cp_catalog_page_id,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty]
+                                Project [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id]
+                                  BroadcastHashJoin [cs_catalog_page_sk,cp_catalog_page_sk]
+                                    Project [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss]
+                                      BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                        Project [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss]
+                                          BroadcastHashJoin [cs_promo_sk,p_promo_sk]
+                                            Project [cs_catalog_page_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss]
+                                              BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                                Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss]
+                                                  SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number]
+                                                    InputAdapter
+                                                      WholeStageCodegen (12)
+                                                        Sort [cs_item_sk,cs_order_number]
+                                                          InputAdapter
+                                                            Exchange [cs_item_sk,cs_order_number] #10
+                                                              WholeStageCodegen (11)
+                                                                Filter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk]
+                                                                        ReusedSubquery [d_date_sk] #1
+                                                    InputAdapter
+                                                      WholeStageCodegen (14)
+                                                        Sort [cr_item_sk,cr_order_number]
+                                                          InputAdapter
+                                                            Exchange [cr_item_sk,cr_order_number] #11
+                                                              WholeStageCodegen (13)
+                                                                Project [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss]
+                                                                  Filter [cr_item_sk,cr_order_number]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk]
+                                                InputAdapter
+                                                  ReusedExchange [i_item_sk] #6
+                                            InputAdapter
+                                              ReusedExchange [p_promo_sk] #7
+                                        InputAdapter
+                                          ReusedExchange [d_date_sk] #4
+                                    InputAdapter
+                                      BroadcastExchange #12
+                                        WholeStageCodegen (18)
+                                          Filter [cp_catalog_page_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id]
+                    WholeStageCodegen (30)
+                      HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(CheckOverflow((promote_precision(cast(ws_net_profit as decimal(13,2))) - promote_precision(cast(coalesce(cast(wr_net_loss as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true)),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty]
+                        InputAdapter
+                          Exchange [web_site_id] #13
+                            WholeStageCodegen (29)
+                              HashAggregate [web_site_id,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty]
+                                Project [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id]
+                                  BroadcastHashJoin [ws_web_site_sk,web_site_sk]
+                                    Project [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss]
+                                      BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                        Project [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss]
+                                          BroadcastHashJoin [ws_promo_sk,p_promo_sk]
+                                            Project [ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss]
+                                              BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                                Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss]
+                                                  SortMergeJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number]
+                                                    InputAdapter
+                                                      WholeStageCodegen (22)
+                                                        Sort [ws_item_sk,ws_order_number]
+                                                          InputAdapter
+                                                            Exchange [ws_item_sk,ws_order_number] #14
+                                                              WholeStageCodegen (21)
+                                                                Filter [ws_web_site_sk,ws_item_sk,ws_promo_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk]
+                                                                        ReusedSubquery [d_date_sk] #1
+                                                    InputAdapter
+                                                      WholeStageCodegen (24)
+                                                        Sort [wr_item_sk,wr_order_number]
+                                                          InputAdapter
+                                                            Exchange [wr_item_sk,wr_order_number] #15
+                                                              WholeStageCodegen (23)
+                                                                Project [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss]
+                                                                  Filter [wr_item_sk,wr_order_number]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk]
+                                                InputAdapter
+                                                  ReusedExchange [i_item_sk] #6
+                                            InputAdapter
+                                              ReusedExchange [p_promo_sk] #7
+                                        InputAdapter
+                                          ReusedExchange [d_date_sk] #4
+                                    InputAdapter
+                                      BroadcastExchange #16
+                                        WholeStageCodegen (28)
+                                          Filter [web_site_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.web_site [web_site_sk,web_site_id]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt
new file mode 100644
index 0000000000000..5d0c3b9ddbb31
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt
@@ -0,0 +1,630 @@
+== Physical Plan ==
+TakeOrderedAndProject (111)
++- * HashAggregate (110)
+   +- Exchange (109)
+      +- * HashAggregate (108)
+         +- * Expand (107)
+            +- Union (106)
+               :- * HashAggregate (43)
+               :  +- Exchange (42)
+               :     +- * HashAggregate (41)
+               :        +- * Project (40)
+               :           +- * BroadcastHashJoin Inner BuildRight (39)
+               :              :- * Project (33)
+               :              :  +- * BroadcastHashJoin Inner BuildRight (32)
+               :              :     :- * Project (26)
+               :              :     :  +- * BroadcastHashJoin Inner BuildRight (25)
+               :              :     :     :- * Project (20)
+               :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (19)
+               :              :     :     :     :- * Project (13)
+               :              :     :     :     :  +- * SortMergeJoin LeftOuter (12)
+               :              :     :     :     :     :- * Sort (5)
+               :              :     :     :     :     :  +- Exchange (4)
+               :              :     :     :     :     :     +- * Filter (3)
+               :              :     :     :     :     :        +- * ColumnarToRow (2)
+               :              :     :     :     :     :           +- Scan parquet default.store_sales (1)
+               :              :     :     :     :     +- * Sort (11)
+               :              :     :     :     :        +- Exchange (10)
+               :              :     :     :     :           +- * Project (9)
+               :              :     :     :     :              +- * Filter (8)
+               :              :     :     :     :                 +- * ColumnarToRow (7)
+               :              :     :     :     :                    +- Scan parquet default.store_returns (6)
+               :              :     :     :     +- BroadcastExchange (18)
+               :              :     :     :        +- * Project (17)
+               :              :     :     :           +- * Filter (16)
+               :              :     :     :              +- * ColumnarToRow (15)
+               :              :     :     :                 +- Scan parquet default.date_dim (14)
+               :              :     :     +- BroadcastExchange (24)
+               :              :     :        +- * Filter (23)
+               :              :     :           +- * ColumnarToRow (22)
+               :              :     :              +- Scan parquet default.store (21)
+               :              :     +- BroadcastExchange (31)
+               :              :        +- * Project (30)
+               :              :           +- * Filter (29)
+               :              :              +- * ColumnarToRow (28)
+               :              :                 +- Scan parquet default.item (27)
+               :              +- BroadcastExchange (38)
+               :                 +- * Project (37)
+               :                    +- * Filter (36)
+               :                       +- * ColumnarToRow (35)
+               :                          +- Scan parquet default.promotion (34)
+               :- * HashAggregate (74)
+               :  +- Exchange (73)
+               :     +- * HashAggregate (72)
+               :        +- * Project (71)
+               :           +- * BroadcastHashJoin Inner BuildRight (70)
+               :              :- * Project (68)
+               :              :  +- * BroadcastHashJoin Inner BuildRight (67)
+               :              :     :- * Project (65)
+               :              :     :  +- * BroadcastHashJoin Inner BuildRight (64)
+               :              :     :     :- * Project (59)
+               :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (58)
+               :              :     :     :     :- * Project (56)
+               :              :     :     :     :  +- * SortMergeJoin LeftOuter (55)
+               :              :     :     :     :     :- * Sort (48)
+               :              :     :     :     :     :  +- Exchange (47)
+               :              :     :     :     :     :     +- * Filter (46)
+               :              :     :     :     :     :        +- * ColumnarToRow (45)
+               :              :     :     :     :     :           +- Scan parquet default.catalog_sales (44)
+               :              :     :     :     :     +- * Sort (54)
+               :              :     :     :     :        +- Exchange (53)
+               :              :     :     :     :           +- * Project (52)
+               :              :     :     :     :              +- * Filter (51)
+               :              :     :     :     :                 +- * ColumnarToRow (50)
+               :              :     :     :     :                    +- Scan parquet default.catalog_returns (49)
+               :              :     :     :     +- ReusedExchange (57)
+               :              :     :     +- BroadcastExchange (63)
+               :              :     :        +- * Filter (62)
+               :              :     :           +- * ColumnarToRow (61)
+               :              :     :              +- Scan parquet default.catalog_page (60)
+               :              :     +- ReusedExchange (66)
+               :              +- ReusedExchange (69)
+               +- * HashAggregate (105)
+                  +- Exchange (104)
+                     +- * HashAggregate (103)
+                        +- * Project (102)
+                           +- * BroadcastHashJoin Inner BuildRight (101)
+                              :- * Project (99)
+                              :  +- * BroadcastHashJoin Inner BuildRight (98)
+                              :     :- * Project (96)
+                              :     :  +- * BroadcastHashJoin Inner BuildRight (95)
+                              :     :     :- * Project (90)
+                              :     :     :  +- * BroadcastHashJoin Inner BuildRight (89)
+                              :     :     :     :- * Project (87)
+                              :     :     :     :  +- * SortMergeJoin LeftOuter (86)
+                              :     :     :     :     :- * Sort (79)
+                              :     :     :     :     :  +- Exchange (78)
+                              :     :     :     :     :     +- * Filter (77)
+                              :     :     :     :     :        +- * ColumnarToRow (76)
+                              :     :     :     :     :           +- Scan parquet default.web_sales (75)
+                              :     :     :     :     +- * Sort (85)
+                              :     :     :     :        +- Exchange (84)
+                              :     :     :     :           +- * Project (83)
+                              :     :     :     :              +- * Filter (82)
+                              :     :     :     :                 +- * ColumnarToRow (81)
+                              :     :     :     :                    +- Scan parquet default.web_returns (80)
+                              :     :     :     +- ReusedExchange (88)
+                              :     :     +- BroadcastExchange (94)
+                              :     :        +- * Filter (93)
+                              :     :           +- * ColumnarToRow (92)
+                              :     :              +- Scan parquet default.web_site (91)
+                              :     +- ReusedExchange (97)
+                              +- ReusedExchange (100)
+
+
+(1) Scan parquet default.store_sales
+Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_promo_sk:int,ss_ticket_number:int,ss_ext_sales_price:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+
+(3) Filter [codegen id : 1]
+Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3))
+
+(4) Exchange
+Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, [id=#9]
+
+(5) Sort [codegen id : 2]
+Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+Arguments: [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.store_returns
+Output [5]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int,sr_return_amt:decimal(7,2),sr_net_loss:decimal(7,2)>
+
+(7) ColumnarToRow [codegen id : 3]
+Input [5]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14]
+
+(8) Filter [codegen id : 3]
+Input [5]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14]
+Condition : (isnotnull(sr_item_sk#10) AND isnotnull(sr_ticket_number#11))
+
+(9) Project [codegen id : 3]
+Output [4]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13]
+Input [5]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14]
+
+(10) Exchange
+Input [4]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13]
+Arguments: hashpartitioning(sr_item_sk#10, sr_ticket_number#11, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(11) Sort [codegen id : 4]
+Input [4]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13]
+Arguments: [sr_item_sk#10 ASC NULLS FIRST, sr_ticket_number#11 ASC NULLS FIRST], false, 0
+
+(12) SortMergeJoin [codegen id : 9]
+Left keys [2]: [ss_item_sk#1, ss_ticket_number#4]
+Right keys [2]: [sr_item_sk#10, sr_ticket_number#11]
+Join condition: None
+
+(13) Project [codegen id : 9]
+Output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#12, sr_net_loss#13]
+Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13]
+
+(14) Scan parquet default.date_dim
+Output [2]: [d_date_sk#16, d_date#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-22), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(15) ColumnarToRow [codegen id : 5]
+Input [2]: [d_date_sk#16, d_date#17]
+
+(16) Filter [codegen id : 5]
+Input [2]: [d_date_sk#16, d_date#17]
+Condition : (((isnotnull(d_date#17) AND (d_date#17 >= 2000-08-23)) AND (d_date#17 <= 2000-09-22)) AND isnotnull(d_date_sk#16))
+
+(17) Project [codegen id : 5]
+Output [1]: [d_date_sk#16]
+Input [2]: [d_date_sk#16, d_date#17]
+
+(18) BroadcastExchange
+Input [1]: [d_date_sk#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(19) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_sold_date_sk#7]
+Right keys [1]: [d_date_sk#16]
+Join condition: None
+
+(20) Project [codegen id : 9]
+Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#12, sr_net_loss#13]
+Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#12, sr_net_loss#13, d_date_sk#16]
+
+(21) Scan parquet default.store
+Output [2]: [s_store_sk#19, s_store_id#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string>
+
+(22) ColumnarToRow [codegen id : 6]
+Input [2]: [s_store_sk#19, s_store_id#20]
+
+(23) Filter [codegen id : 6]
+Input [2]: [s_store_sk#19, s_store_id#20]
+Condition : isnotnull(s_store_sk#19)
+
+(24) BroadcastExchange
+Input [2]: [s_store_sk#19, s_store_id#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#21]
+
+(25) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_store_sk#2]
+Right keys [1]: [s_store_sk#19]
+Join condition: None
+
+(26) Project [codegen id : 9]
+Output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#12, sr_net_loss#13, s_store_id#20]
+Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#12, sr_net_loss#13, s_store_sk#19, s_store_id#20]
+
+(27) Scan parquet default.item
+Output [2]: [i_item_sk#22, i_current_price#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_current_price:decimal(7,2)>
+
+(28) ColumnarToRow [codegen id : 7]
+Input [2]: [i_item_sk#22, i_current_price#23]
+
+(29) Filter [codegen id : 7]
+Input [2]: [i_item_sk#22, i_current_price#23]
+Condition : ((isnotnull(i_current_price#23) AND (i_current_price#23 > 50.00)) AND isnotnull(i_item_sk#22))
+
+(30) Project [codegen id : 7]
+Output [1]: [i_item_sk#22]
+Input [2]: [i_item_sk#22, i_current_price#23]
+
+(31) BroadcastExchange
+Input [1]: [i_item_sk#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#24]
+
+(32) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#22]
+Join condition: None
+
+(33) Project [codegen id : 9]
+Output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#12, sr_net_loss#13, s_store_id#20]
+Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#12, sr_net_loss#13, s_store_id#20, i_item_sk#22]
+
+(34) Scan parquet default.promotion
+Output [2]: [p_promo_sk#25, p_channel_tv#26]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/promotion]
+PushedFilters: [IsNotNull(p_channel_tv), EqualTo(p_channel_tv,N), IsNotNull(p_promo_sk)]
+ReadSchema: struct<p_promo_sk:int,p_channel_tv:string>
+
+(35) ColumnarToRow [codegen id : 8]
+Input [2]: [p_promo_sk#25, p_channel_tv#26]
+
+(36) Filter [codegen id : 8]
+Input [2]: [p_promo_sk#25, p_channel_tv#26]
+Condition : ((isnotnull(p_channel_tv#26) AND (p_channel_tv#26 = N)) AND isnotnull(p_promo_sk#25))
+
+(37) Project [codegen id : 8]
+Output [1]: [p_promo_sk#25]
+Input [2]: [p_promo_sk#25, p_channel_tv#26]
+
+(38) BroadcastExchange
+Input [1]: [p_promo_sk#25]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#27]
+
+(39) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_promo_sk#3]
+Right keys [1]: [p_promo_sk#25]
+Join condition: None
+
+(40) Project [codegen id : 9]
+Output [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#12, sr_net_loss#13, s_store_id#20]
+Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#12, sr_net_loss#13, s_store_id#20, p_promo_sk#25]
+
+(41) HashAggregate [codegen id : 9]
+Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#12, sr_net_loss#13, s_store_id#20]
+Keys [1]: [s_store_id#20]
+Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#12 as decimal(12,2)), 0.00)), partial_sum(CheckOverflow((promote_precision(cast(ss_net_profit#6 as decimal(13,2))) - promote_precision(cast(coalesce(cast(sr_net_loss#13 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))]
+Aggregate Attributes [5]: [sum#28, sum#29, isEmpty#30, sum#31, isEmpty#32]
+Results [6]: [s_store_id#20, sum#33, sum#34, isEmpty#35, sum#36, isEmpty#37]
+
+(42) Exchange
+Input [6]: [s_store_id#20, sum#33, sum#34, isEmpty#35, sum#36, isEmpty#37]
+Arguments: hashpartitioning(s_store_id#20, 5), ENSURE_REQUIREMENTS, [id=#38]
+
+(43) HashAggregate [codegen id : 10]
+Input [6]: [s_store_id#20, sum#33, sum#34, isEmpty#35, sum#36, isEmpty#37]
+Keys [1]: [s_store_id#20]
+Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#12 as decimal(12,2)), 0.00)), sum(CheckOverflow((promote_precision(cast(ss_net_profit#6 as decimal(13,2))) - promote_precision(cast(coalesce(cast(sr_net_loss#13 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))]
+Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#39, sum(coalesce(cast(sr_return_amt#12 as decimal(12,2)), 0.00))#40, sum(CheckOverflow((promote_precision(cast(ss_net_profit#6 as decimal(13,2))) - promote_precision(cast(coalesce(cast(sr_net_loss#13 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))#41]
+Results [5]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#39,17,2) AS sales#42, sum(coalesce(cast(sr_return_amt#12 as decimal(12,2)), 0.00))#40 AS returns#43, sum(CheckOverflow((promote_precision(cast(ss_net_profit#6 as decimal(13,2))) - promote_precision(cast(coalesce(cast(sr_net_loss#13 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))#41 AS profit#44, store channel AS channel#45, concat(store, s_store_id#20) AS id#46]
+
+(44) Scan parquet default.catalog_sales
+Output [7]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_order_number#50, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#53), dynamicpruningexpression(cs_sold_date_sk#53 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)]
+ReadSchema: struct<cs_catalog_page_sk:int,cs_item_sk:int,cs_promo_sk:int,cs_order_number:int,cs_ext_sales_price:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(45) ColumnarToRow [codegen id : 11]
+Input [7]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_order_number#50, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53]
+
+(46) Filter [codegen id : 11]
+Input [7]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_order_number#50, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53]
+Condition : ((isnotnull(cs_catalog_page_sk#47) AND isnotnull(cs_item_sk#48)) AND isnotnull(cs_promo_sk#49))
+
+(47) Exchange
+Input [7]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_order_number#50, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53]
+Arguments: hashpartitioning(cs_item_sk#48, cs_order_number#50, 5), ENSURE_REQUIREMENTS, [id=#54]
+
+(48) Sort [codegen id : 12]
+Input [7]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_order_number#50, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53]
+Arguments: [cs_item_sk#48 ASC NULLS FIRST, cs_order_number#50 ASC NULLS FIRST], false, 0
+
+(49) Scan parquet default.catalog_returns
+Output [5]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int,cr_return_amount:decimal(7,2),cr_net_loss:decimal(7,2)>
+
+(50) ColumnarToRow [codegen id : 13]
+Input [5]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+
+(51) Filter [codegen id : 13]
+Input [5]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+Condition : (isnotnull(cr_item_sk#55) AND isnotnull(cr_order_number#56))
+
+(52) Project [codegen id : 13]
+Output [4]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58]
+Input [5]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+
+(53) Exchange
+Input [4]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58]
+Arguments: hashpartitioning(cr_item_sk#55, cr_order_number#56, 5), ENSURE_REQUIREMENTS, [id=#60]
+
+(54) Sort [codegen id : 14]
+Input [4]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58]
+Arguments: [cr_item_sk#55 ASC NULLS FIRST, cr_order_number#56 ASC NULLS FIRST], false, 0
+
+(55) SortMergeJoin [codegen id : 19]
+Left keys [2]: [cs_item_sk#48, cs_order_number#50]
+Right keys [2]: [cr_item_sk#55, cr_order_number#56]
+Join condition: None
+
+(56) Project [codegen id : 19]
+Output [8]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53, cr_return_amount#57, cr_net_loss#58]
+Input [11]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_order_number#50, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53, cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58]
+
+(57) ReusedExchange [Reuses operator id: 18]
+Output [1]: [d_date_sk#61]
+
+(58) BroadcastHashJoin [codegen id : 19]
+Left keys [1]: [cs_sold_date_sk#53]
+Right keys [1]: [d_date_sk#61]
+Join condition: None
+
+(59) Project [codegen id : 19]
+Output [7]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_ext_sales_price#51, cs_net_profit#52, cr_return_amount#57, cr_net_loss#58]
+Input [9]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53, cr_return_amount#57, cr_net_loss#58, d_date_sk#61]
+
+(60) Scan parquet default.catalog_page
+Output [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_page]
+PushedFilters: [IsNotNull(cp_catalog_page_sk)]
+ReadSchema: struct<cp_catalog_page_sk:int,cp_catalog_page_id:string>
+
+(61) ColumnarToRow [codegen id : 16]
+Input [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63]
+
+(62) Filter [codegen id : 16]
+Input [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63]
+Condition : isnotnull(cp_catalog_page_sk#62)
+
+(63) BroadcastExchange
+Input [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#64]
+
+(64) BroadcastHashJoin [codegen id : 19]
+Left keys [1]: [cs_catalog_page_sk#47]
+Right keys [1]: [cp_catalog_page_sk#62]
+Join condition: None
+
+(65) Project [codegen id : 19]
+Output [7]: [cs_item_sk#48, cs_promo_sk#49, cs_ext_sales_price#51, cs_net_profit#52, cr_return_amount#57, cr_net_loss#58, cp_catalog_page_id#63]
+Input [9]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_ext_sales_price#51, cs_net_profit#52, cr_return_amount#57, cr_net_loss#58, cp_catalog_page_sk#62, cp_catalog_page_id#63]
+
+(66) ReusedExchange [Reuses operator id: 31]
+Output [1]: [i_item_sk#65]
+
+(67) BroadcastHashJoin [codegen id : 19]
+Left keys [1]: [cs_item_sk#48]
+Right keys [1]: [i_item_sk#65]
+Join condition: None
+
+(68) Project [codegen id : 19]
+Output [6]: [cs_promo_sk#49, cs_ext_sales_price#51, cs_net_profit#52, cr_return_amount#57, cr_net_loss#58, cp_catalog_page_id#63]
+Input [8]: [cs_item_sk#48, cs_promo_sk#49, cs_ext_sales_price#51, cs_net_profit#52, cr_return_amount#57, cr_net_loss#58, cp_catalog_page_id#63, i_item_sk#65]
+
+(69) ReusedExchange [Reuses operator id: 38]
+Output [1]: [p_promo_sk#66]
+
+(70) BroadcastHashJoin [codegen id : 19]
+Left keys [1]: [cs_promo_sk#49]
+Right keys [1]: [p_promo_sk#66]
+Join condition: None
+
+(71) Project [codegen id : 19]
+Output [5]: [cs_ext_sales_price#51, cs_net_profit#52, cr_return_amount#57, cr_net_loss#58, cp_catalog_page_id#63]
+Input [7]: [cs_promo_sk#49, cs_ext_sales_price#51, cs_net_profit#52, cr_return_amount#57, cr_net_loss#58, cp_catalog_page_id#63, p_promo_sk#66]
+
+(72) HashAggregate [codegen id : 19]
+Input [5]: [cs_ext_sales_price#51, cs_net_profit#52, cr_return_amount#57, cr_net_loss#58, cp_catalog_page_id#63]
+Keys [1]: [cp_catalog_page_id#63]
+Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#51)), partial_sum(coalesce(cast(cr_return_amount#57 as decimal(12,2)), 0.00)), partial_sum(CheckOverflow((promote_precision(cast(cs_net_profit#52 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_net_loss#58 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))]
+Aggregate Attributes [5]: [sum#67, sum#68, isEmpty#69, sum#70, isEmpty#71]
+Results [6]: [cp_catalog_page_id#63, sum#72, sum#73, isEmpty#74, sum#75, isEmpty#76]
+
+(73) Exchange
+Input [6]: [cp_catalog_page_id#63, sum#72, sum#73, isEmpty#74, sum#75, isEmpty#76]
+Arguments: hashpartitioning(cp_catalog_page_id#63, 5), ENSURE_REQUIREMENTS, [id=#77]
+
+(74) HashAggregate [codegen id : 20]
+Input [6]: [cp_catalog_page_id#63, sum#72, sum#73, isEmpty#74, sum#75, isEmpty#76]
+Keys [1]: [cp_catalog_page_id#63]
+Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#51)), sum(coalesce(cast(cr_return_amount#57 as decimal(12,2)), 0.00)), sum(CheckOverflow((promote_precision(cast(cs_net_profit#52 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_net_loss#58 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))]
+Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#51))#78, sum(coalesce(cast(cr_return_amount#57 as decimal(12,2)), 0.00))#79, sum(CheckOverflow((promote_precision(cast(cs_net_profit#52 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_net_loss#58 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))#80]
+Results [5]: [MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#51))#78,17,2) AS sales#81, sum(coalesce(cast(cr_return_amount#57 as decimal(12,2)), 0.00))#79 AS returns#82, sum(CheckOverflow((promote_precision(cast(cs_net_profit#52 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_net_loss#58 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))#80 AS profit#83, catalog channel AS channel#84, concat(catalog_page, cp_catalog_page_id#63) AS id#85]
+
+(75) Scan parquet default.web_sales
+Output [7]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_order_number#89, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#92), dynamicpruningexpression(ws_sold_date_sk#92 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_web_site_sk:int,ws_promo_sk:int,ws_order_number:int,ws_ext_sales_price:decimal(7,2),ws_net_profit:decimal(7,2)>
+
+(76) ColumnarToRow [codegen id : 21]
+Input [7]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_order_number#89, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92]
+
+(77) Filter [codegen id : 21]
+Input [7]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_order_number#89, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92]
+Condition : ((isnotnull(ws_web_site_sk#87) AND isnotnull(ws_item_sk#86)) AND isnotnull(ws_promo_sk#88))
+
+(78) Exchange
+Input [7]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_order_number#89, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92]
+Arguments: hashpartitioning(ws_item_sk#86, ws_order_number#89, 5), ENSURE_REQUIREMENTS, [id=#93]
+
+(79) Sort [codegen id : 22]
+Input [7]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_order_number#89, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92]
+Arguments: [ws_item_sk#86 ASC NULLS FIRST, ws_order_number#89 ASC NULLS FIRST], false, 0
+
+(80) Scan parquet default.web_returns
+Output [5]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_returns]
+PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)]
+ReadSchema: struct<wr_item_sk:int,wr_order_number:int,wr_return_amt:decimal(7,2),wr_net_loss:decimal(7,2)>
+
+(81) ColumnarToRow [codegen id : 23]
+Input [5]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98]
+
+(82) Filter [codegen id : 23]
+Input [5]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98]
+Condition : (isnotnull(wr_item_sk#94) AND isnotnull(wr_order_number#95))
+
+(83) Project [codegen id : 23]
+Output [4]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97]
+Input [5]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98]
+
+(84) Exchange
+Input [4]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97]
+Arguments: hashpartitioning(wr_item_sk#94, wr_order_number#95, 5), ENSURE_REQUIREMENTS, [id=#99]
+
+(85) Sort [codegen id : 24]
+Input [4]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97]
+Arguments: [wr_item_sk#94 ASC NULLS FIRST, wr_order_number#95 ASC NULLS FIRST], false, 0
+
+(86) SortMergeJoin [codegen id : 29]
+Left keys [2]: [ws_item_sk#86, ws_order_number#89]
+Right keys [2]: [wr_item_sk#94, wr_order_number#95]
+Join condition: None
+
+(87) Project [codegen id : 29]
+Output [8]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92, wr_return_amt#96, wr_net_loss#97]
+Input [11]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_order_number#89, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92, wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97]
+
+(88) ReusedExchange [Reuses operator id: 18]
+Output [1]: [d_date_sk#100]
+
+(89) BroadcastHashJoin [codegen id : 29]
+Left keys [1]: [ws_sold_date_sk#92]
+Right keys [1]: [d_date_sk#100]
+Join condition: None
+
+(90) Project [codegen id : 29]
+Output [7]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_ext_sales_price#90, ws_net_profit#91, wr_return_amt#96, wr_net_loss#97]
+Input [9]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92, wr_return_amt#96, wr_net_loss#97, d_date_sk#100]
+
+(91) Scan parquet default.web_site
+Output [2]: [web_site_sk#101, web_site_id#102]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_site]
+PushedFilters: [IsNotNull(web_site_sk)]
+ReadSchema: struct<web_site_sk:int,web_site_id:string>
+
+(92) ColumnarToRow [codegen id : 26]
+Input [2]: [web_site_sk#101, web_site_id#102]
+
+(93) Filter [codegen id : 26]
+Input [2]: [web_site_sk#101, web_site_id#102]
+Condition : isnotnull(web_site_sk#101)
+
+(94) BroadcastExchange
+Input [2]: [web_site_sk#101, web_site_id#102]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#103]
+
+(95) BroadcastHashJoin [codegen id : 29]
+Left keys [1]: [ws_web_site_sk#87]
+Right keys [1]: [web_site_sk#101]
+Join condition: None
+
+(96) Project [codegen id : 29]
+Output [7]: [ws_item_sk#86, ws_promo_sk#88, ws_ext_sales_price#90, ws_net_profit#91, wr_return_amt#96, wr_net_loss#97, web_site_id#102]
+Input [9]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_ext_sales_price#90, ws_net_profit#91, wr_return_amt#96, wr_net_loss#97, web_site_sk#101, web_site_id#102]
+
+(97) ReusedExchange [Reuses operator id: 31]
+Output [1]: [i_item_sk#104]
+
+(98) BroadcastHashJoin [codegen id : 29]
+Left keys [1]: [ws_item_sk#86]
+Right keys [1]: [i_item_sk#104]
+Join condition: None
+
+(99) Project [codegen id : 29]
+Output [6]: [ws_promo_sk#88, ws_ext_sales_price#90, ws_net_profit#91, wr_return_amt#96, wr_net_loss#97, web_site_id#102]
+Input [8]: [ws_item_sk#86, ws_promo_sk#88, ws_ext_sales_price#90, ws_net_profit#91, wr_return_amt#96, wr_net_loss#97, web_site_id#102, i_item_sk#104]
+
+(100) ReusedExchange [Reuses operator id: 38]
+Output [1]: [p_promo_sk#105]
+
+(101) BroadcastHashJoin [codegen id : 29]
+Left keys [1]: [ws_promo_sk#88]
+Right keys [1]: [p_promo_sk#105]
+Join condition: None
+
+(102) Project [codegen id : 29]
+Output [5]: [ws_ext_sales_price#90, ws_net_profit#91, wr_return_amt#96, wr_net_loss#97, web_site_id#102]
+Input [7]: [ws_promo_sk#88, ws_ext_sales_price#90, ws_net_profit#91, wr_return_amt#96, wr_net_loss#97, web_site_id#102, p_promo_sk#105]
+
+(103) HashAggregate [codegen id : 29]
+Input [5]: [ws_ext_sales_price#90, ws_net_profit#91, wr_return_amt#96, wr_net_loss#97, web_site_id#102]
+Keys [1]: [web_site_id#102]
+Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#90)), partial_sum(coalesce(cast(wr_return_amt#96 as decimal(12,2)), 0.00)), partial_sum(CheckOverflow((promote_precision(cast(ws_net_profit#91 as decimal(13,2))) - promote_precision(cast(coalesce(cast(wr_net_loss#97 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))]
+Aggregate Attributes [5]: [sum#106, sum#107, isEmpty#108, sum#109, isEmpty#110]
+Results [6]: [web_site_id#102, sum#111, sum#112, isEmpty#113, sum#114, isEmpty#115]
+
+(104) Exchange
+Input [6]: [web_site_id#102, sum#111, sum#112, isEmpty#113, sum#114, isEmpty#115]
+Arguments: hashpartitioning(web_site_id#102, 5), ENSURE_REQUIREMENTS, [id=#116]
+
+(105) HashAggregate [codegen id : 30]
+Input [6]: [web_site_id#102, sum#111, sum#112, isEmpty#113, sum#114, isEmpty#115]
+Keys [1]: [web_site_id#102]
+Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#90)), sum(coalesce(cast(wr_return_amt#96 as decimal(12,2)), 0.00)), sum(CheckOverflow((promote_precision(cast(ws_net_profit#91 as decimal(13,2))) - promote_precision(cast(coalesce(cast(wr_net_loss#97 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))]
+Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#90))#117, sum(coalesce(cast(wr_return_amt#96 as decimal(12,2)), 0.00))#118, sum(CheckOverflow((promote_precision(cast(ws_net_profit#91 as decimal(13,2))) - promote_precision(cast(coalesce(cast(wr_net_loss#97 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))#119]
+Results [5]: [MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#90))#117,17,2) AS sales#120, sum(coalesce(cast(wr_return_amt#96 as decimal(12,2)), 0.00))#118 AS returns#121, sum(CheckOverflow((promote_precision(cast(ws_net_profit#91 as decimal(13,2))) - promote_precision(cast(coalesce(cast(wr_net_loss#97 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))#119 AS profit#122, web channel AS channel#123, concat(web_site, web_site_id#102) AS id#124]
+
+(106) Union
+
+(107) Expand [codegen id : 31]
+Input [5]: [sales#42, returns#43, profit#44, channel#45, id#46]
+Arguments: [ArrayBuffer(sales#42, returns#43, profit#44, channel#45, id#46, 0), ArrayBuffer(sales#42, returns#43, profit#44, channel#45, null, 1), ArrayBuffer(sales#42, returns#43, profit#44, null, null, 3)], [sales#42, returns#43, profit#44, channel#125, id#126, spark_grouping_id#127]
+
+(108) HashAggregate [codegen id : 31]
+Input [6]: [sales#42, returns#43, profit#44, channel#125, id#126, spark_grouping_id#127]
+Keys [3]: [channel#125, id#126, spark_grouping_id#127]
+Functions [3]: [partial_sum(sales#42), partial_sum(returns#43), partial_sum(profit#44)]
+Aggregate Attributes [6]: [sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133]
+Results [9]: [channel#125, id#126, spark_grouping_id#127, sum#134, isEmpty#135, sum#136, isEmpty#137, sum#138, isEmpty#139]
+
+(109) Exchange
+Input [9]: [channel#125, id#126, spark_grouping_id#127, sum#134, isEmpty#135, sum#136, isEmpty#137, sum#138, isEmpty#139]
+Arguments: hashpartitioning(channel#125, id#126, spark_grouping_id#127, 5), ENSURE_REQUIREMENTS, [id=#140]
+
+(110) HashAggregate [codegen id : 32]
+Input [9]: [channel#125, id#126, spark_grouping_id#127, sum#134, isEmpty#135, sum#136, isEmpty#137, sum#138, isEmpty#139]
+Keys [3]: [channel#125, id#126, spark_grouping_id#127]
+Functions [3]: [sum(sales#42), sum(returns#43), sum(profit#44)]
+Aggregate Attributes [3]: [sum(sales#42)#141, sum(returns#43)#142, sum(profit#44)#143]
+Results [5]: [channel#125, id#126, sum(sales#42)#141 AS sales#144, sum(returns#43)#142 AS returns#145, sum(profit#44)#143 AS profit#146]
+
+(111) TakeOrderedAndProject
+Input [5]: [channel#125, id#126, sales#144, returns#145, profit#146]
+Arguments: 100, [channel#125 ASC NULLS FIRST, id#126 ASC NULLS FIRST], [channel#125, id#126, sales#144, returns#145, profit#146]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (112)
+
+
+(112) ReusedExchange [Reuses operator id: 18]
+Output [1]: [d_date_sk#16]
+
+Subquery:2 Hosting operator id = 44 Hosting Expression = cs_sold_date_sk#53 IN dynamicpruning#8
+
+Subquery:3 Hosting operator id = 75 Hosting Expression = ws_sold_date_sk#92 IN dynamicpruning#8
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/simplified.txt
new file mode 100644
index 0000000000000..5c8bd611e8dbc
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/simplified.txt
@@ -0,0 +1,182 @@
+TakeOrderedAndProject [channel,id,sales,returns,profit]
+  WholeStageCodegen (32)
+    HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+      InputAdapter
+        Exchange [channel,id,spark_grouping_id] #1
+          WholeStageCodegen (31)
+            HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+              Expand [sales,returns,profit,channel,id]
+                InputAdapter
+                  Union
+                    WholeStageCodegen (10)
+                      HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(CheckOverflow((promote_precision(cast(ss_net_profit as decimal(13,2))) - promote_precision(cast(coalesce(cast(sr_net_loss as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true)),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty]
+                        InputAdapter
+                          Exchange [s_store_id] #2
+                            WholeStageCodegen (9)
+                              HashAggregate [s_store_id,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty]
+                                Project [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id]
+                                  BroadcastHashJoin [ss_promo_sk,p_promo_sk]
+                                    Project [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id]
+                                      BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                        Project [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id]
+                                          BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                            Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss]
+                                              BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss]
+                                                  SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number]
+                                                    InputAdapter
+                                                      WholeStageCodegen (2)
+                                                        Sort [ss_item_sk,ss_ticket_number]
+                                                          InputAdapter
+                                                            Exchange [ss_item_sk,ss_ticket_number] #3
+                                                              WholeStageCodegen (1)
+                                                                Filter [ss_store_sk,ss_item_sk,ss_promo_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk]
+                                                                        SubqueryBroadcast [d_date_sk] #1
+                                                                          ReusedExchange [d_date_sk] #4
+                                                    InputAdapter
+                                                      WholeStageCodegen (4)
+                                                        Sort [sr_item_sk,sr_ticket_number]
+                                                          InputAdapter
+                                                            Exchange [sr_item_sk,sr_ticket_number] #5
+                                                              WholeStageCodegen (3)
+                                                                Project [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss]
+                                                                  Filter [sr_item_sk,sr_ticket_number]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk]
+                                                InputAdapter
+                                                  BroadcastExchange #4
+                                                    WholeStageCodegen (5)
+                                                      Project [d_date_sk]
+                                                        Filter [d_date,d_date_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.date_dim [d_date_sk,d_date]
+                                            InputAdapter
+                                              BroadcastExchange #6
+                                                WholeStageCodegen (6)
+                                                  Filter [s_store_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.store [s_store_sk,s_store_id]
+                                        InputAdapter
+                                          BroadcastExchange #7
+                                            WholeStageCodegen (7)
+                                              Project [i_item_sk]
+                                                Filter [i_current_price,i_item_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.item [i_item_sk,i_current_price]
+                                    InputAdapter
+                                      BroadcastExchange #8
+                                        WholeStageCodegen (8)
+                                          Project [p_promo_sk]
+                                            Filter [p_channel_tv,p_promo_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.promotion [p_promo_sk,p_channel_tv]
+                    WholeStageCodegen (20)
+                      HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(CheckOverflow((promote_precision(cast(cs_net_profit as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_net_loss as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true)),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty]
+                        InputAdapter
+                          Exchange [cp_catalog_page_id] #9
+                            WholeStageCodegen (19)
+                              HashAggregate [cp_catalog_page_id,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty]
+                                Project [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id]
+                                  BroadcastHashJoin [cs_promo_sk,p_promo_sk]
+                                    Project [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id]
+                                      BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                        Project [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id]
+                                          BroadcastHashJoin [cs_catalog_page_sk,cp_catalog_page_sk]
+                                            Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss]
+                                              BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss]
+                                                  SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number]
+                                                    InputAdapter
+                                                      WholeStageCodegen (12)
+                                                        Sort [cs_item_sk,cs_order_number]
+                                                          InputAdapter
+                                                            Exchange [cs_item_sk,cs_order_number] #10
+                                                              WholeStageCodegen (11)
+                                                                Filter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk]
+                                                                        ReusedSubquery [d_date_sk] #1
+                                                    InputAdapter
+                                                      WholeStageCodegen (14)
+                                                        Sort [cr_item_sk,cr_order_number]
+                                                          InputAdapter
+                                                            Exchange [cr_item_sk,cr_order_number] #11
+                                                              WholeStageCodegen (13)
+                                                                Project [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss]
+                                                                  Filter [cr_item_sk,cr_order_number]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk]
+                                                InputAdapter
+                                                  ReusedExchange [d_date_sk] #4
+                                            InputAdapter
+                                              BroadcastExchange #12
+                                                WholeStageCodegen (16)
+                                                  Filter [cp_catalog_page_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id]
+                                        InputAdapter
+                                          ReusedExchange [i_item_sk] #7
+                                    InputAdapter
+                                      ReusedExchange [p_promo_sk] #8
+                    WholeStageCodegen (30)
+                      HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(CheckOverflow((promote_precision(cast(ws_net_profit as decimal(13,2))) - promote_precision(cast(coalesce(cast(wr_net_loss as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true)),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty]
+                        InputAdapter
+                          Exchange [web_site_id] #13
+                            WholeStageCodegen (29)
+                              HashAggregate [web_site_id,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty]
+                                Project [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id]
+                                  BroadcastHashJoin [ws_promo_sk,p_promo_sk]
+                                    Project [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id]
+                                      BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                        Project [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id]
+                                          BroadcastHashJoin [ws_web_site_sk,web_site_sk]
+                                            Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss]
+                                              BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss]
+                                                  SortMergeJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number]
+                                                    InputAdapter
+                                                      WholeStageCodegen (22)
+                                                        Sort [ws_item_sk,ws_order_number]
+                                                          InputAdapter
+                                                            Exchange [ws_item_sk,ws_order_number] #14
+                                                              WholeStageCodegen (21)
+                                                                Filter [ws_web_site_sk,ws_item_sk,ws_promo_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk]
+                                                                        ReusedSubquery [d_date_sk] #1
+                                                    InputAdapter
+                                                      WholeStageCodegen (24)
+                                                        Sort [wr_item_sk,wr_order_number]
+                                                          InputAdapter
+                                                            Exchange [wr_item_sk,wr_order_number] #15
+                                                              WholeStageCodegen (23)
+                                                                Project [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss]
+                                                                  Filter [wr_item_sk,wr_order_number]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk]
+                                                InputAdapter
+                                                  ReusedExchange [d_date_sk] #4
+                                            InputAdapter
+                                              BroadcastExchange #16
+                                                WholeStageCodegen (26)
+                                                  Filter [web_site_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.web_site [web_site_sk,web_site_id]
+                                        InputAdapter
+                                          ReusedExchange [i_item_sk] #7
+                                    InputAdapter
+                                      ReusedExchange [p_promo_sk] #8
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.sf100/explain.txt
new file mode 100644
index 0000000000000..93cc597893dc2
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.sf100/explain.txt
@@ -0,0 +1,357 @@
+== Physical Plan ==
+TakeOrderedAndProject (61)
++- * Project (60)
+   +- * BroadcastHashJoin Inner BuildRight (59)
+      :- * Project (38)
+      :  +- * SortMergeJoin Inner (37)
+      :     :- * Sort (11)
+      :     :  +- Exchange (10)
+      :     :     +- * Project (9)
+      :     :        +- * BroadcastHashJoin Inner BuildRight (8)
+      :     :           :- * Filter (3)
+      :     :           :  +- * ColumnarToRow (2)
+      :     :           :     +- Scan parquet default.customer (1)
+      :     :           +- BroadcastExchange (7)
+      :     :              +- * Filter (6)
+      :     :                 +- * ColumnarToRow (5)
+      :     :                    +- Scan parquet default.customer_address (4)
+      :     +- * Sort (36)
+      :        +- Exchange (35)
+      :           +- * Filter (34)
+      :              +- * HashAggregate (33)
+      :                 +- Exchange (32)
+      :                    +- * HashAggregate (31)
+      :                       +- * Project (30)
+      :                          +- * SortMergeJoin Inner (29)
+      :                             :- * Sort (23)
+      :                             :  +- Exchange (22)
+      :                             :     +- * Project (21)
+      :                             :        +- * BroadcastHashJoin Inner BuildRight (20)
+      :                             :           :- * Filter (14)
+      :                             :           :  +- * ColumnarToRow (13)
+      :                             :           :     +- Scan parquet default.catalog_returns (12)
+      :                             :           +- BroadcastExchange (19)
+      :                             :              +- * Project (18)
+      :                             :                 +- * Filter (17)
+      :                             :                    +- * ColumnarToRow (16)
+      :                             :                       +- Scan parquet default.date_dim (15)
+      :                             +- * Sort (28)
+      :                                +- Exchange (27)
+      :                                   +- * Filter (26)
+      :                                      +- * ColumnarToRow (25)
+      :                                         +- Scan parquet default.customer_address (24)
+      +- BroadcastExchange (58)
+         +- * Filter (57)
+            +- * HashAggregate (56)
+               +- Exchange (55)
+                  +- * HashAggregate (54)
+                     +- * HashAggregate (53)
+                        +- Exchange (52)
+                           +- * HashAggregate (51)
+                              +- * Project (50)
+                                 +- * SortMergeJoin Inner (49)
+                                    :- * Sort (46)
+                                    :  +- Exchange (45)
+                                    :     +- * Project (44)
+                                    :        +- * BroadcastHashJoin Inner BuildRight (43)
+                                    :           :- * Filter (41)
+                                    :           :  +- * ColumnarToRow (40)
+                                    :           :     +- Scan parquet default.catalog_returns (39)
+                                    :           +- ReusedExchange (42)
+                                    +- * Sort (48)
+                                       +- ReusedExchange (47)
+
+
+(1) Scan parquet default.customer
+Output [6]: [c_customer_sk#1, c_customer_id#2, c_current_addr_sk#3, c_salutation#4, c_first_name#5, c_last_name#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_current_addr_sk:int,c_salutation:string,c_first_name:string,c_last_name:string>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [6]: [c_customer_sk#1, c_customer_id#2, c_current_addr_sk#3, c_salutation#4, c_first_name#5, c_last_name#6]
+
+(3) Filter [codegen id : 2]
+Input [6]: [c_customer_sk#1, c_customer_id#2, c_current_addr_sk#3, c_salutation#4, c_first_name#5, c_last_name#6]
+Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_current_addr_sk#3))
+
+(4) Scan parquet default.customer_address
+Output [12]: [ca_address_sk#7, ca_street_number#8, ca_street_name#9, ca_street_type#10, ca_suite_number#11, ca_city#12, ca_county#13, ca_state#14, ca_zip#15, ca_country#16, ca_gmt_offset#17, ca_location_type#18]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_street_number:string,ca_street_name:string,ca_street_type:string,ca_suite_number:string,ca_city:string,ca_county:string,ca_state:string,ca_zip:string,ca_country:string,ca_gmt_offset:decimal(5,2),ca_location_type:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [12]: [ca_address_sk#7, ca_street_number#8, ca_street_name#9, ca_street_type#10, ca_suite_number#11, ca_city#12, ca_county#13, ca_state#14, ca_zip#15, ca_country#16, ca_gmt_offset#17, ca_location_type#18]
+
+(6) Filter [codegen id : 1]
+Input [12]: [ca_address_sk#7, ca_street_number#8, ca_street_name#9, ca_street_type#10, ca_suite_number#11, ca_city#12, ca_county#13, ca_state#14, ca_zip#15, ca_country#16, ca_gmt_offset#17, ca_location_type#18]
+Condition : ((isnotnull(ca_state#14) AND (ca_state#14 = GA)) AND isnotnull(ca_address_sk#7))
+
+(7) BroadcastExchange
+Input [12]: [ca_address_sk#7, ca_street_number#8, ca_street_name#9, ca_street_type#10, ca_suite_number#11, ca_city#12, ca_county#13, ca_state#14, ca_zip#15, ca_country#16, ca_gmt_offset#17, ca_location_type#18]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#19]
+
+(8) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [c_current_addr_sk#3]
+Right keys [1]: [ca_address_sk#7]
+Join condition: None
+
+(9) Project [codegen id : 2]
+Output [16]: [c_customer_sk#1, c_customer_id#2, c_salutation#4, c_first_name#5, c_last_name#6, ca_street_number#8, ca_street_name#9, ca_street_type#10, ca_suite_number#11, ca_city#12, ca_county#13, ca_state#14, ca_zip#15, ca_country#16, ca_gmt_offset#17, ca_location_type#18]
+Input [18]: [c_customer_sk#1, c_customer_id#2, c_current_addr_sk#3, c_salutation#4, c_first_name#5, c_last_name#6, ca_address_sk#7, ca_street_number#8, ca_street_name#9, ca_street_type#10, ca_suite_number#11, ca_city#12, ca_county#13, ca_state#14, ca_zip#15, ca_country#16, ca_gmt_offset#17, ca_location_type#18]
+
+(10) Exchange
+Input [16]: [c_customer_sk#1, c_customer_id#2, c_salutation#4, c_first_name#5, c_last_name#6, ca_street_number#8, ca_street_name#9, ca_street_type#10, ca_suite_number#11, ca_city#12, ca_county#13, ca_state#14, ca_zip#15, ca_country#16, ca_gmt_offset#17, ca_location_type#18]
+Arguments: hashpartitioning(c_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(11) Sort [codegen id : 3]
+Input [16]: [c_customer_sk#1, c_customer_id#2, c_salutation#4, c_first_name#5, c_last_name#6, ca_street_number#8, ca_street_name#9, ca_street_type#10, ca_suite_number#11, ca_city#12, ca_county#13, ca_state#14, ca_zip#15, ca_country#16, ca_gmt_offset#17, ca_location_type#18]
+Arguments: [c_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(12) Scan parquet default.catalog_returns
+Output [4]: [cr_returning_customer_sk#21, cr_returning_addr_sk#22, cr_return_amt_inc_tax#23, cr_returned_date_sk#24]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cr_returned_date_sk#24), dynamicpruningexpression(cr_returned_date_sk#24 IN dynamicpruning#25)]
+PushedFilters: [IsNotNull(cr_returning_addr_sk), IsNotNull(cr_returning_customer_sk)]
+ReadSchema: struct<cr_returning_customer_sk:int,cr_returning_addr_sk:int,cr_return_amt_inc_tax:decimal(7,2)>
+
+(13) ColumnarToRow [codegen id : 5]
+Input [4]: [cr_returning_customer_sk#21, cr_returning_addr_sk#22, cr_return_amt_inc_tax#23, cr_returned_date_sk#24]
+
+(14) Filter [codegen id : 5]
+Input [4]: [cr_returning_customer_sk#21, cr_returning_addr_sk#22, cr_return_amt_inc_tax#23, cr_returned_date_sk#24]
+Condition : (isnotnull(cr_returning_addr_sk#22) AND isnotnull(cr_returning_customer_sk#21))
+
+(15) Scan parquet default.date_dim
+Output [2]: [d_date_sk#26, d_year#27]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(16) ColumnarToRow [codegen id : 4]
+Input [2]: [d_date_sk#26, d_year#27]
+
+(17) Filter [codegen id : 4]
+Input [2]: [d_date_sk#26, d_year#27]
+Condition : ((isnotnull(d_year#27) AND (d_year#27 = 2000)) AND isnotnull(d_date_sk#26))
+
+(18) Project [codegen id : 4]
+Output [1]: [d_date_sk#26]
+Input [2]: [d_date_sk#26, d_year#27]
+
+(19) BroadcastExchange
+Input [1]: [d_date_sk#26]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#28]
+
+(20) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [cr_returned_date_sk#24]
+Right keys [1]: [d_date_sk#26]
+Join condition: None
+
+(21) Project [codegen id : 5]
+Output [3]: [cr_returning_customer_sk#21, cr_returning_addr_sk#22, cr_return_amt_inc_tax#23]
+Input [5]: [cr_returning_customer_sk#21, cr_returning_addr_sk#22, cr_return_amt_inc_tax#23, cr_returned_date_sk#24, d_date_sk#26]
+
+(22) Exchange
+Input [3]: [cr_returning_customer_sk#21, cr_returning_addr_sk#22, cr_return_amt_inc_tax#23]
+Arguments: hashpartitioning(cr_returning_addr_sk#22, 5), ENSURE_REQUIREMENTS, [id=#29]
+
+(23) Sort [codegen id : 6]
+Input [3]: [cr_returning_customer_sk#21, cr_returning_addr_sk#22, cr_return_amt_inc_tax#23]
+Arguments: [cr_returning_addr_sk#22 ASC NULLS FIRST], false, 0
+
+(24) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#30, ca_state#31]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_state)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(25) ColumnarToRow [codegen id : 7]
+Input [2]: [ca_address_sk#30, ca_state#31]
+
+(26) Filter [codegen id : 7]
+Input [2]: [ca_address_sk#30, ca_state#31]
+Condition : (isnotnull(ca_address_sk#30) AND isnotnull(ca_state#31))
+
+(27) Exchange
+Input [2]: [ca_address_sk#30, ca_state#31]
+Arguments: hashpartitioning(ca_address_sk#30, 5), ENSURE_REQUIREMENTS, [id=#32]
+
+(28) Sort [codegen id : 8]
+Input [2]: [ca_address_sk#30, ca_state#31]
+Arguments: [ca_address_sk#30 ASC NULLS FIRST], false, 0
+
+(29) SortMergeJoin [codegen id : 9]
+Left keys [1]: [cr_returning_addr_sk#22]
+Right keys [1]: [ca_address_sk#30]
+Join condition: None
+
+(30) Project [codegen id : 9]
+Output [3]: [cr_returning_customer_sk#21, cr_return_amt_inc_tax#23, ca_state#31]
+Input [5]: [cr_returning_customer_sk#21, cr_returning_addr_sk#22, cr_return_amt_inc_tax#23, ca_address_sk#30, ca_state#31]
+
+(31) HashAggregate [codegen id : 9]
+Input [3]: [cr_returning_customer_sk#21, cr_return_amt_inc_tax#23, ca_state#31]
+Keys [2]: [cr_returning_customer_sk#21, ca_state#31]
+Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#23))]
+Aggregate Attributes [1]: [sum#33]
+Results [3]: [cr_returning_customer_sk#21, ca_state#31, sum#34]
+
+(32) Exchange
+Input [3]: [cr_returning_customer_sk#21, ca_state#31, sum#34]
+Arguments: hashpartitioning(cr_returning_customer_sk#21, ca_state#31, 5), ENSURE_REQUIREMENTS, [id=#35]
+
+(33) HashAggregate [codegen id : 10]
+Input [3]: [cr_returning_customer_sk#21, ca_state#31, sum#34]
+Keys [2]: [cr_returning_customer_sk#21, ca_state#31]
+Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#23))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#23))#36]
+Results [3]: [cr_returning_customer_sk#21 AS ctr_customer_sk#37, ca_state#31 AS ctr_state#38, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#23))#36,17,2) AS ctr_total_return#39]
+
+(34) Filter [codegen id : 10]
+Input [3]: [ctr_customer_sk#37, ctr_state#38, ctr_total_return#39]
+Condition : isnotnull(ctr_total_return#39)
+
+(35) Exchange
+Input [3]: [ctr_customer_sk#37, ctr_state#38, ctr_total_return#39]
+Arguments: hashpartitioning(ctr_customer_sk#37, 5), ENSURE_REQUIREMENTS, [id=#40]
+
+(36) Sort [codegen id : 11]
+Input [3]: [ctr_customer_sk#37, ctr_state#38, ctr_total_return#39]
+Arguments: [ctr_customer_sk#37 ASC NULLS FIRST], false, 0
+
+(37) SortMergeJoin [codegen id : 20]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [ctr_customer_sk#37]
+Join condition: None
+
+(38) Project [codegen id : 20]
+Output [17]: [c_customer_id#2, c_salutation#4, c_first_name#5, c_last_name#6, ca_street_number#8, ca_street_name#9, ca_street_type#10, ca_suite_number#11, ca_city#12, ca_county#13, ca_state#14, ca_zip#15, ca_country#16, ca_gmt_offset#17, ca_location_type#18, ctr_state#38, ctr_total_return#39]
+Input [19]: [c_customer_sk#1, c_customer_id#2, c_salutation#4, c_first_name#5, c_last_name#6, ca_street_number#8, ca_street_name#9, ca_street_type#10, ca_suite_number#11, ca_city#12, ca_county#13, ca_state#14, ca_zip#15, ca_country#16, ca_gmt_offset#17, ca_location_type#18, ctr_customer_sk#37, ctr_state#38, ctr_total_return#39]
+
+(39) Scan parquet default.catalog_returns
+Output [4]: [cr_returning_customer_sk#41, cr_returning_addr_sk#42, cr_return_amt_inc_tax#43, cr_returned_date_sk#44]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cr_returned_date_sk#44), dynamicpruningexpression(cr_returned_date_sk#44 IN dynamicpruning#25)]
+PushedFilters: [IsNotNull(cr_returning_addr_sk)]
+ReadSchema: struct<cr_returning_customer_sk:int,cr_returning_addr_sk:int,cr_return_amt_inc_tax:decimal(7,2)>
+
+(40) ColumnarToRow [codegen id : 13]
+Input [4]: [cr_returning_customer_sk#41, cr_returning_addr_sk#42, cr_return_amt_inc_tax#43, cr_returned_date_sk#44]
+
+(41) Filter [codegen id : 13]
+Input [4]: [cr_returning_customer_sk#41, cr_returning_addr_sk#42, cr_return_amt_inc_tax#43, cr_returned_date_sk#44]
+Condition : isnotnull(cr_returning_addr_sk#42)
+
+(42) ReusedExchange [Reuses operator id: 19]
+Output [1]: [d_date_sk#45]
+
+(43) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [cr_returned_date_sk#44]
+Right keys [1]: [d_date_sk#45]
+Join condition: None
+
+(44) Project [codegen id : 13]
+Output [3]: [cr_returning_customer_sk#41, cr_returning_addr_sk#42, cr_return_amt_inc_tax#43]
+Input [5]: [cr_returning_customer_sk#41, cr_returning_addr_sk#42, cr_return_amt_inc_tax#43, cr_returned_date_sk#44, d_date_sk#45]
+
+(45) Exchange
+Input [3]: [cr_returning_customer_sk#41, cr_returning_addr_sk#42, cr_return_amt_inc_tax#43]
+Arguments: hashpartitioning(cr_returning_addr_sk#42, 5), ENSURE_REQUIREMENTS, [id=#46]
+
+(46) Sort [codegen id : 14]
+Input [3]: [cr_returning_customer_sk#41, cr_returning_addr_sk#42, cr_return_amt_inc_tax#43]
+Arguments: [cr_returning_addr_sk#42 ASC NULLS FIRST], false, 0
+
+(47) ReusedExchange [Reuses operator id: 27]
+Output [2]: [ca_address_sk#47, ca_state#48]
+
+(48) Sort [codegen id : 16]
+Input [2]: [ca_address_sk#47, ca_state#48]
+Arguments: [ca_address_sk#47 ASC NULLS FIRST], false, 0
+
+(49) SortMergeJoin [codegen id : 17]
+Left keys [1]: [cr_returning_addr_sk#42]
+Right keys [1]: [ca_address_sk#47]
+Join condition: None
+
+(50) Project [codegen id : 17]
+Output [3]: [cr_returning_customer_sk#41, cr_return_amt_inc_tax#43, ca_state#48]
+Input [5]: [cr_returning_customer_sk#41, cr_returning_addr_sk#42, cr_return_amt_inc_tax#43, ca_address_sk#47, ca_state#48]
+
+(51) HashAggregate [codegen id : 17]
+Input [3]: [cr_returning_customer_sk#41, cr_return_amt_inc_tax#43, ca_state#48]
+Keys [2]: [cr_returning_customer_sk#41, ca_state#48]
+Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#43))]
+Aggregate Attributes [1]: [sum#49]
+Results [3]: [cr_returning_customer_sk#41, ca_state#48, sum#50]
+
+(52) Exchange
+Input [3]: [cr_returning_customer_sk#41, ca_state#48, sum#50]
+Arguments: hashpartitioning(cr_returning_customer_sk#41, ca_state#48, 5), ENSURE_REQUIREMENTS, [id=#51]
+
+(53) HashAggregate [codegen id : 18]
+Input [3]: [cr_returning_customer_sk#41, ca_state#48, sum#50]
+Keys [2]: [cr_returning_customer_sk#41, ca_state#48]
+Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#43))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#43))#52]
+Results [2]: [ca_state#48 AS ctr_state#38, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#43))#52,17,2) AS ctr_total_return#39]
+
+(54) HashAggregate [codegen id : 18]
+Input [2]: [ctr_state#38, ctr_total_return#39]
+Keys [1]: [ctr_state#38]
+Functions [1]: [partial_avg(ctr_total_return#39)]
+Aggregate Attributes [2]: [sum#53, count#54]
+Results [3]: [ctr_state#38, sum#55, count#56]
+
+(55) Exchange
+Input [3]: [ctr_state#38, sum#55, count#56]
+Arguments: hashpartitioning(ctr_state#38, 5), ENSURE_REQUIREMENTS, [id=#57]
+
+(56) HashAggregate [codegen id : 19]
+Input [3]: [ctr_state#38, sum#55, count#56]
+Keys [1]: [ctr_state#38]
+Functions [1]: [avg(ctr_total_return#39)]
+Aggregate Attributes [1]: [avg(ctr_total_return#39)#58]
+Results [2]: [CheckOverflow((promote_precision(avg(ctr_total_return#39)#58) * 1.200000), DecimalType(24,7), true) AS (avg(ctr_total_return) * 1.2)#59, ctr_state#38 AS ctr_state#38#60]
+
+(57) Filter [codegen id : 19]
+Input [2]: [(avg(ctr_total_return) * 1.2)#59, ctr_state#38#60]
+Condition : isnotnull((avg(ctr_total_return) * 1.2)#59)
+
+(58) BroadcastExchange
+Input [2]: [(avg(ctr_total_return) * 1.2)#59, ctr_state#38#60]
+Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [id=#61]
+
+(59) BroadcastHashJoin [codegen id : 20]
+Left keys [1]: [ctr_state#38]
+Right keys [1]: [ctr_state#38#60]
+Join condition: (cast(ctr_total_return#39 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#59)
+
+(60) Project [codegen id : 20]
+Output [16]: [c_customer_id#2, c_salutation#4, c_first_name#5, c_last_name#6, ca_street_number#8, ca_street_name#9, ca_street_type#10, ca_suite_number#11, ca_city#12, ca_county#13, ca_state#14, ca_zip#15, ca_country#16, ca_gmt_offset#17, ca_location_type#18, ctr_total_return#39]
+Input [19]: [c_customer_id#2, c_salutation#4, c_first_name#5, c_last_name#6, ca_street_number#8, ca_street_name#9, ca_street_type#10, ca_suite_number#11, ca_city#12, ca_county#13, ca_state#14, ca_zip#15, ca_country#16, ca_gmt_offset#17, ca_location_type#18, ctr_state#38, ctr_total_return#39, (avg(ctr_total_return) * 1.2)#59, ctr_state#38#60]
+
+(61) TakeOrderedAndProject
+Input [16]: [c_customer_id#2, c_salutation#4, c_first_name#5, c_last_name#6, ca_street_number#8, ca_street_name#9, ca_street_type#10, ca_suite_number#11, ca_city#12, ca_county#13, ca_state#14, ca_zip#15, ca_country#16, ca_gmt_offset#17, ca_location_type#18, ctr_total_return#39]
+Arguments: 100, [c_customer_id#2 ASC NULLS FIRST, c_salutation#4 ASC NULLS FIRST, c_first_name#5 ASC NULLS FIRST, c_last_name#6 ASC NULLS FIRST, ca_street_number#8 ASC NULLS FIRST, ca_street_name#9 ASC NULLS FIRST, ca_street_type#10 ASC NULLS FIRST, ca_suite_number#11 ASC NULLS FIRST, ca_city#12 ASC NULLS FIRST, ca_county#13 ASC NULLS FIRST, ca_state#14 ASC NULLS FIRST, ca_zip#15 ASC NULLS FIRST, ca_country#16 ASC NULLS FIRST, ca_gmt_offset#17 ASC NULLS FIRST, ca_location_type#18 ASC NULLS FIRST, ctr_total_return#39 ASC NULLS FIRST], [c_customer_id#2, c_salutation#4, c_first_name#5, c_last_name#6, ca_street_number#8, ca_street_name#9, ca_street_type#10, ca_suite_number#11, ca_city#12, ca_county#13, ca_state#14, ca_zip#15, ca_country#16, ca_gmt_offset#17, ca_location_type#18, ctr_total_return#39]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 12 Hosting Expression = cr_returned_date_sk#24 IN dynamicpruning#25
+ReusedExchange (62)
+
+
+(62) ReusedExchange [Reuses operator id: 19]
+Output [1]: [d_date_sk#26]
+
+Subquery:2 Hosting operator id = 39 Hosting Expression = cr_returned_date_sk#44 IN dynamicpruning#25
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.sf100/simplified.txt
new file mode 100644
index 0000000000000..cee20aeec3948
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.sf100/simplified.txt
@@ -0,0 +1,107 @@
+TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return]
+  WholeStageCodegen (20)
+    Project [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return]
+      BroadcastHashJoin [ctr_state,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2)]
+        Project [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_state,ctr_total_return]
+          SortMergeJoin [c_customer_sk,ctr_customer_sk]
+            InputAdapter
+              WholeStageCodegen (3)
+                Sort [c_customer_sk]
+                  InputAdapter
+                    Exchange [c_customer_sk] #1
+                      WholeStageCodegen (2)
+                        Project [c_customer_sk,c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type]
+                          BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                            Filter [c_customer_sk,c_current_addr_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name]
+                            InputAdapter
+                              BroadcastExchange #2
+                                WholeStageCodegen (1)
+                                  Filter [ca_state,ca_address_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type]
+            InputAdapter
+              WholeStageCodegen (11)
+                Sort [ctr_customer_sk]
+                  InputAdapter
+                    Exchange [ctr_customer_sk] #3
+                      WholeStageCodegen (10)
+                        Filter [ctr_total_return]
+                          HashAggregate [cr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(cr_return_amt_inc_tax)),ctr_customer_sk,ctr_state,ctr_total_return,sum]
+                            InputAdapter
+                              Exchange [cr_returning_customer_sk,ca_state] #4
+                                WholeStageCodegen (9)
+                                  HashAggregate [cr_returning_customer_sk,ca_state,cr_return_amt_inc_tax] [sum,sum]
+                                    Project [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state]
+                                      SortMergeJoin [cr_returning_addr_sk,ca_address_sk]
+                                        InputAdapter
+                                          WholeStageCodegen (6)
+                                            Sort [cr_returning_addr_sk]
+                                              InputAdapter
+                                                Exchange [cr_returning_addr_sk] #5
+                                                  WholeStageCodegen (5)
+                                                    Project [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax]
+                                                      BroadcastHashJoin [cr_returned_date_sk,d_date_sk]
+                                                        Filter [cr_returning_addr_sk,cr_returning_customer_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk]
+                                                                SubqueryBroadcast [d_date_sk] #1
+                                                                  ReusedExchange [d_date_sk] #6
+                                                        InputAdapter
+                                                          BroadcastExchange #6
+                                                            WholeStageCodegen (4)
+                                                              Project [d_date_sk]
+                                                                Filter [d_year,d_date_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.date_dim [d_date_sk,d_year]
+                                        InputAdapter
+                                          WholeStageCodegen (8)
+                                            Sort [ca_address_sk]
+                                              InputAdapter
+                                                Exchange [ca_address_sk] #7
+                                                  WholeStageCodegen (7)
+                                                    Filter [ca_address_sk,ca_state]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.customer_address [ca_address_sk,ca_state]
+        InputAdapter
+          BroadcastExchange #8
+            WholeStageCodegen (19)
+              Filter [(avg(ctr_total_return) * 1.2)]
+                HashAggregate [ctr_state,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),ctr_state,sum,count]
+                  InputAdapter
+                    Exchange [ctr_state] #9
+                      WholeStageCodegen (18)
+                        HashAggregate [ctr_state,ctr_total_return] [sum,count,sum,count]
+                          HashAggregate [cr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(cr_return_amt_inc_tax)),ctr_state,ctr_total_return,sum]
+                            InputAdapter
+                              Exchange [cr_returning_customer_sk,ca_state] #10
+                                WholeStageCodegen (17)
+                                  HashAggregate [cr_returning_customer_sk,ca_state,cr_return_amt_inc_tax] [sum,sum]
+                                    Project [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state]
+                                      SortMergeJoin [cr_returning_addr_sk,ca_address_sk]
+                                        InputAdapter
+                                          WholeStageCodegen (14)
+                                            Sort [cr_returning_addr_sk]
+                                              InputAdapter
+                                                Exchange [cr_returning_addr_sk] #11
+                                                  WholeStageCodegen (13)
+                                                    Project [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax]
+                                                      BroadcastHashJoin [cr_returned_date_sk,d_date_sk]
+                                                        Filter [cr_returning_addr_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk]
+                                                                ReusedSubquery [d_date_sk] #1
+                                                        InputAdapter
+                                                          ReusedExchange [d_date_sk] #6
+                                        InputAdapter
+                                          WholeStageCodegen (16)
+                                            Sort [ca_address_sk]
+                                              InputAdapter
+                                                ReusedExchange [ca_address_sk,ca_state] #7
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/explain.txt
new file mode 100644
index 0000000000000..04371a7f43d2a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/explain.txt
@@ -0,0 +1,312 @@
+== Physical Plan ==
+TakeOrderedAndProject (52)
++- * Project (51)
+   +- * BroadcastHashJoin Inner BuildRight (50)
+      :- * Project (45)
+      :  +- * BroadcastHashJoin Inner BuildRight (44)
+      :     :- * Project (39)
+      :     :  +- * BroadcastHashJoin Inner BuildRight (38)
+      :     :     :- * Filter (20)
+      :     :     :  +- * HashAggregate (19)
+      :     :     :     +- Exchange (18)
+      :     :     :        +- * HashAggregate (17)
+      :     :     :           +- * Project (16)
+      :     :     :              +- * BroadcastHashJoin Inner BuildRight (15)
+      :     :     :                 :- * Project (10)
+      :     :     :                 :  +- * BroadcastHashJoin Inner BuildRight (9)
+      :     :     :                 :     :- * Filter (3)
+      :     :     :                 :     :  +- * ColumnarToRow (2)
+      :     :     :                 :     :     +- Scan parquet default.catalog_returns (1)
+      :     :     :                 :     +- BroadcastExchange (8)
+      :     :     :                 :        +- * Project (7)
+      :     :     :                 :           +- * Filter (6)
+      :     :     :                 :              +- * ColumnarToRow (5)
+      :     :     :                 :                 +- Scan parquet default.date_dim (4)
+      :     :     :                 +- BroadcastExchange (14)
+      :     :     :                    +- * Filter (13)
+      :     :     :                       +- * ColumnarToRow (12)
+      :     :     :                          +- Scan parquet default.customer_address (11)
+      :     :     +- BroadcastExchange (37)
+      :     :        +- * Filter (36)
+      :     :           +- * HashAggregate (35)
+      :     :              +- Exchange (34)
+      :     :                 +- * HashAggregate (33)
+      :     :                    +- * HashAggregate (32)
+      :     :                       +- Exchange (31)
+      :     :                          +- * HashAggregate (30)
+      :     :                             +- * Project (29)
+      :     :                                +- * BroadcastHashJoin Inner BuildRight (28)
+      :     :                                   :- * Project (26)
+      :     :                                   :  +- * BroadcastHashJoin Inner BuildRight (25)
+      :     :                                   :     :- * Filter (23)
+      :     :                                   :     :  +- * ColumnarToRow (22)
+      :     :                                   :     :     +- Scan parquet default.catalog_returns (21)
+      :     :                                   :     +- ReusedExchange (24)
+      :     :                                   +- ReusedExchange (27)
+      :     +- BroadcastExchange (43)
+      :        +- * Filter (42)
+      :           +- * ColumnarToRow (41)
+      :              +- Scan parquet default.customer (40)
+      +- BroadcastExchange (49)
+         +- * Filter (48)
+            +- * ColumnarToRow (47)
+               +- Scan parquet default.customer_address (46)
+
+
+(1) Scan parquet default.catalog_returns
+Output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cr_returned_date_sk#4), dynamicpruningexpression(cr_returned_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(cr_returning_addr_sk), IsNotNull(cr_returning_customer_sk)]
+ReadSchema: struct<cr_returning_customer_sk:int,cr_returning_addr_sk:int,cr_return_amt_inc_tax:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4]
+
+(3) Filter [codegen id : 3]
+Input [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4]
+Condition : (isnotnull(cr_returning_addr_sk#2) AND isnotnull(cr_returning_customer_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#6, d_year#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#6, d_year#7]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#6, d_year#7]
+Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2000)) AND isnotnull(d_date_sk#6))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#6]
+Input [2]: [d_date_sk#6, d_year#7]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#8]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cr_returned_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [3]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3]
+Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4, d_date_sk#6]
+
+(11) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#9, ca_state#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_state)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [ca_address_sk#9, ca_state#10]
+
+(13) Filter [codegen id : 2]
+Input [2]: [ca_address_sk#9, ca_state#10]
+Condition : (isnotnull(ca_address_sk#9) AND isnotnull(ca_state#10))
+
+(14) BroadcastExchange
+Input [2]: [ca_address_sk#9, ca_state#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cr_returning_addr_sk#2]
+Right keys [1]: [ca_address_sk#9]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#10]
+Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#9, ca_state#10]
+
+(17) HashAggregate [codegen id : 3]
+Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#10]
+Keys [2]: [cr_returning_customer_sk#1, ca_state#10]
+Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#3))]
+Aggregate Attributes [1]: [sum#12]
+Results [3]: [cr_returning_customer_sk#1, ca_state#10, sum#13]
+
+(18) Exchange
+Input [3]: [cr_returning_customer_sk#1, ca_state#10, sum#13]
+Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#10, 5), ENSURE_REQUIREMENTS, [id=#14]
+
+(19) HashAggregate [codegen id : 11]
+Input [3]: [cr_returning_customer_sk#1, ca_state#10, sum#13]
+Keys [2]: [cr_returning_customer_sk#1, ca_state#10]
+Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))#15]
+Results [3]: [cr_returning_customer_sk#1 AS ctr_customer_sk#16, ca_state#10 AS ctr_state#17, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#3))#15,17,2) AS ctr_total_return#18]
+
+(20) Filter [codegen id : 11]
+Input [3]: [ctr_customer_sk#16, ctr_state#17, ctr_total_return#18]
+Condition : isnotnull(ctr_total_return#18)
+
+(21) Scan parquet default.catalog_returns
+Output [4]: [cr_returning_customer_sk#19, cr_returning_addr_sk#20, cr_return_amt_inc_tax#21, cr_returned_date_sk#22]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cr_returned_date_sk#22), dynamicpruningexpression(cr_returned_date_sk#22 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(cr_returning_addr_sk)]
+ReadSchema: struct<cr_returning_customer_sk:int,cr_returning_addr_sk:int,cr_return_amt_inc_tax:decimal(7,2)>
+
+(22) ColumnarToRow [codegen id : 6]
+Input [4]: [cr_returning_customer_sk#19, cr_returning_addr_sk#20, cr_return_amt_inc_tax#21, cr_returned_date_sk#22]
+
+(23) Filter [codegen id : 6]
+Input [4]: [cr_returning_customer_sk#19, cr_returning_addr_sk#20, cr_return_amt_inc_tax#21, cr_returned_date_sk#22]
+Condition : isnotnull(cr_returning_addr_sk#20)
+
+(24) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#23]
+
+(25) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [cr_returned_date_sk#22]
+Right keys [1]: [d_date_sk#23]
+Join condition: None
+
+(26) Project [codegen id : 6]
+Output [3]: [cr_returning_customer_sk#19, cr_returning_addr_sk#20, cr_return_amt_inc_tax#21]
+Input [5]: [cr_returning_customer_sk#19, cr_returning_addr_sk#20, cr_return_amt_inc_tax#21, cr_returned_date_sk#22, d_date_sk#23]
+
+(27) ReusedExchange [Reuses operator id: 14]
+Output [2]: [ca_address_sk#24, ca_state#25]
+
+(28) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [cr_returning_addr_sk#20]
+Right keys [1]: [ca_address_sk#24]
+Join condition: None
+
+(29) Project [codegen id : 6]
+Output [3]: [cr_returning_customer_sk#19, cr_return_amt_inc_tax#21, ca_state#25]
+Input [5]: [cr_returning_customer_sk#19, cr_returning_addr_sk#20, cr_return_amt_inc_tax#21, ca_address_sk#24, ca_state#25]
+
+(30) HashAggregate [codegen id : 6]
+Input [3]: [cr_returning_customer_sk#19, cr_return_amt_inc_tax#21, ca_state#25]
+Keys [2]: [cr_returning_customer_sk#19, ca_state#25]
+Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#21))]
+Aggregate Attributes [1]: [sum#26]
+Results [3]: [cr_returning_customer_sk#19, ca_state#25, sum#27]
+
+(31) Exchange
+Input [3]: [cr_returning_customer_sk#19, ca_state#25, sum#27]
+Arguments: hashpartitioning(cr_returning_customer_sk#19, ca_state#25, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(32) HashAggregate [codegen id : 7]
+Input [3]: [cr_returning_customer_sk#19, ca_state#25, sum#27]
+Keys [2]: [cr_returning_customer_sk#19, ca_state#25]
+Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#21))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#21))#29]
+Results [2]: [ca_state#25 AS ctr_state#17, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#21))#29,17,2) AS ctr_total_return#18]
+
+(33) HashAggregate [codegen id : 7]
+Input [2]: [ctr_state#17, ctr_total_return#18]
+Keys [1]: [ctr_state#17]
+Functions [1]: [partial_avg(ctr_total_return#18)]
+Aggregate Attributes [2]: [sum#30, count#31]
+Results [3]: [ctr_state#17, sum#32, count#33]
+
+(34) Exchange
+Input [3]: [ctr_state#17, sum#32, count#33]
+Arguments: hashpartitioning(ctr_state#17, 5), ENSURE_REQUIREMENTS, [id=#34]
+
+(35) HashAggregate [codegen id : 8]
+Input [3]: [ctr_state#17, sum#32, count#33]
+Keys [1]: [ctr_state#17]
+Functions [1]: [avg(ctr_total_return#18)]
+Aggregate Attributes [1]: [avg(ctr_total_return#18)#35]
+Results [2]: [CheckOverflow((promote_precision(avg(ctr_total_return#18)#35) * 1.200000), DecimalType(24,7), true) AS (avg(ctr_total_return) * 1.2)#36, ctr_state#17 AS ctr_state#17#37]
+
+(36) Filter [codegen id : 8]
+Input [2]: [(avg(ctr_total_return) * 1.2)#36, ctr_state#17#37]
+Condition : isnotnull((avg(ctr_total_return) * 1.2)#36)
+
+(37) BroadcastExchange
+Input [2]: [(avg(ctr_total_return) * 1.2)#36, ctr_state#17#37]
+Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [id=#38]
+
+(38) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ctr_state#17]
+Right keys [1]: [ctr_state#17#37]
+Join condition: (cast(ctr_total_return#18 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#36)
+
+(39) Project [codegen id : 11]
+Output [2]: [ctr_customer_sk#16, ctr_total_return#18]
+Input [5]: [ctr_customer_sk#16, ctr_state#17, ctr_total_return#18, (avg(ctr_total_return) * 1.2)#36, ctr_state#17#37]
+
+(40) Scan parquet default.customer
+Output [6]: [c_customer_sk#39, c_customer_id#40, c_current_addr_sk#41, c_salutation#42, c_first_name#43, c_last_name#44]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_current_addr_sk:int,c_salutation:string,c_first_name:string,c_last_name:string>
+
+(41) ColumnarToRow [codegen id : 9]
+Input [6]: [c_customer_sk#39, c_customer_id#40, c_current_addr_sk#41, c_salutation#42, c_first_name#43, c_last_name#44]
+
+(42) Filter [codegen id : 9]
+Input [6]: [c_customer_sk#39, c_customer_id#40, c_current_addr_sk#41, c_salutation#42, c_first_name#43, c_last_name#44]
+Condition : (isnotnull(c_customer_sk#39) AND isnotnull(c_current_addr_sk#41))
+
+(43) BroadcastExchange
+Input [6]: [c_customer_sk#39, c_customer_id#40, c_current_addr_sk#41, c_salutation#42, c_first_name#43, c_last_name#44]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#45]
+
+(44) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ctr_customer_sk#16]
+Right keys [1]: [c_customer_sk#39]
+Join condition: None
+
+(45) Project [codegen id : 11]
+Output [6]: [ctr_total_return#18, c_customer_id#40, c_current_addr_sk#41, c_salutation#42, c_first_name#43, c_last_name#44]
+Input [8]: [ctr_customer_sk#16, ctr_total_return#18, c_customer_sk#39, c_customer_id#40, c_current_addr_sk#41, c_salutation#42, c_first_name#43, c_last_name#44]
+
+(46) Scan parquet default.customer_address
+Output [12]: [ca_address_sk#46, ca_street_number#47, ca_street_name#48, ca_street_type#49, ca_suite_number#50, ca_city#51, ca_county#52, ca_state#53, ca_zip#54, ca_country#55, ca_gmt_offset#56, ca_location_type#57]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_street_number:string,ca_street_name:string,ca_street_type:string,ca_suite_number:string,ca_city:string,ca_county:string,ca_state:string,ca_zip:string,ca_country:string,ca_gmt_offset:decimal(5,2),ca_location_type:string>
+
+(47) ColumnarToRow [codegen id : 10]
+Input [12]: [ca_address_sk#46, ca_street_number#47, ca_street_name#48, ca_street_type#49, ca_suite_number#50, ca_city#51, ca_county#52, ca_state#53, ca_zip#54, ca_country#55, ca_gmt_offset#56, ca_location_type#57]
+
+(48) Filter [codegen id : 10]
+Input [12]: [ca_address_sk#46, ca_street_number#47, ca_street_name#48, ca_street_type#49, ca_suite_number#50, ca_city#51, ca_county#52, ca_state#53, ca_zip#54, ca_country#55, ca_gmt_offset#56, ca_location_type#57]
+Condition : ((isnotnull(ca_state#53) AND (ca_state#53 = GA)) AND isnotnull(ca_address_sk#46))
+
+(49) BroadcastExchange
+Input [12]: [ca_address_sk#46, ca_street_number#47, ca_street_name#48, ca_street_type#49, ca_suite_number#50, ca_city#51, ca_county#52, ca_state#53, ca_zip#54, ca_country#55, ca_gmt_offset#56, ca_location_type#57]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#58]
+
+(50) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [c_current_addr_sk#41]
+Right keys [1]: [ca_address_sk#46]
+Join condition: None
+
+(51) Project [codegen id : 11]
+Output [16]: [c_customer_id#40, c_salutation#42, c_first_name#43, c_last_name#44, ca_street_number#47, ca_street_name#48, ca_street_type#49, ca_suite_number#50, ca_city#51, ca_county#52, ca_state#53, ca_zip#54, ca_country#55, ca_gmt_offset#56, ca_location_type#57, ctr_total_return#18]
+Input [18]: [ctr_total_return#18, c_customer_id#40, c_current_addr_sk#41, c_salutation#42, c_first_name#43, c_last_name#44, ca_address_sk#46, ca_street_number#47, ca_street_name#48, ca_street_type#49, ca_suite_number#50, ca_city#51, ca_county#52, ca_state#53, ca_zip#54, ca_country#55, ca_gmt_offset#56, ca_location_type#57]
+
+(52) TakeOrderedAndProject
+Input [16]: [c_customer_id#40, c_salutation#42, c_first_name#43, c_last_name#44, ca_street_number#47, ca_street_name#48, ca_street_type#49, ca_suite_number#50, ca_city#51, ca_county#52, ca_state#53, ca_zip#54, ca_country#55, ca_gmt_offset#56, ca_location_type#57, ctr_total_return#18]
+Arguments: 100, [c_customer_id#40 ASC NULLS FIRST, c_salutation#42 ASC NULLS FIRST, c_first_name#43 ASC NULLS FIRST, c_last_name#44 ASC NULLS FIRST, ca_street_number#47 ASC NULLS FIRST, ca_street_name#48 ASC NULLS FIRST, ca_street_type#49 ASC NULLS FIRST, ca_suite_number#50 ASC NULLS FIRST, ca_city#51 ASC NULLS FIRST, ca_county#52 ASC NULLS FIRST, ca_state#53 ASC NULLS FIRST, ca_zip#54 ASC NULLS FIRST, ca_country#55 ASC NULLS FIRST, ca_gmt_offset#56 ASC NULLS FIRST, ca_location_type#57 ASC NULLS FIRST, ctr_total_return#18 ASC NULLS FIRST], [c_customer_id#40, c_salutation#42, c_first_name#43, c_last_name#44, ca_street_number#47, ca_street_name#48, ca_street_type#49, ca_suite_number#50, ca_city#51, ca_county#52, ca_state#53, ca_zip#54, ca_country#55, ca_gmt_offset#56, ca_location_type#57, ctr_total_return#18]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cr_returned_date_sk#4 IN dynamicpruning#5
+ReusedExchange (53)
+
+
+(53) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#6]
+
+Subquery:2 Hosting operator id = 21 Hosting Expression = cr_returned_date_sk#22 IN dynamicpruning#5
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/simplified.txt
new file mode 100644
index 0000000000000..767205a90cddf
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/simplified.txt
@@ -0,0 +1,80 @@
+TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return]
+  WholeStageCodegen (11)
+    Project [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return]
+      BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+        Project [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name]
+          BroadcastHashJoin [ctr_customer_sk,c_customer_sk]
+            Project [ctr_customer_sk,ctr_total_return]
+              BroadcastHashJoin [ctr_state,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2)]
+                Filter [ctr_total_return]
+                  HashAggregate [cr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(cr_return_amt_inc_tax)),ctr_customer_sk,ctr_state,ctr_total_return,sum]
+                    InputAdapter
+                      Exchange [cr_returning_customer_sk,ca_state] #1
+                        WholeStageCodegen (3)
+                          HashAggregate [cr_returning_customer_sk,ca_state,cr_return_amt_inc_tax] [sum,sum]
+                            Project [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state]
+                              BroadcastHashJoin [cr_returning_addr_sk,ca_address_sk]
+                                Project [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax]
+                                  BroadcastHashJoin [cr_returned_date_sk,d_date_sk]
+                                    Filter [cr_returning_addr_sk,cr_returning_customer_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk]
+                                            SubqueryBroadcast [d_date_sk] #1
+                                              ReusedExchange [d_date_sk] #2
+                                    InputAdapter
+                                      BroadcastExchange #2
+                                        WholeStageCodegen (1)
+                                          Project [d_date_sk]
+                                            Filter [d_year,d_date_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.date_dim [d_date_sk,d_year]
+                                InputAdapter
+                                  BroadcastExchange #3
+                                    WholeStageCodegen (2)
+                                      Filter [ca_address_sk,ca_state]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.customer_address [ca_address_sk,ca_state]
+                InputAdapter
+                  BroadcastExchange #4
+                    WholeStageCodegen (8)
+                      Filter [(avg(ctr_total_return) * 1.2)]
+                        HashAggregate [ctr_state,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),ctr_state,sum,count]
+                          InputAdapter
+                            Exchange [ctr_state] #5
+                              WholeStageCodegen (7)
+                                HashAggregate [ctr_state,ctr_total_return] [sum,count,sum,count]
+                                  HashAggregate [cr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(cr_return_amt_inc_tax)),ctr_state,ctr_total_return,sum]
+                                    InputAdapter
+                                      Exchange [cr_returning_customer_sk,ca_state] #6
+                                        WholeStageCodegen (6)
+                                          HashAggregate [cr_returning_customer_sk,ca_state,cr_return_amt_inc_tax] [sum,sum]
+                                            Project [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state]
+                                              BroadcastHashJoin [cr_returning_addr_sk,ca_address_sk]
+                                                Project [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax]
+                                                  BroadcastHashJoin [cr_returned_date_sk,d_date_sk]
+                                                    Filter [cr_returning_addr_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk]
+                                                            ReusedSubquery [d_date_sk] #1
+                                                    InputAdapter
+                                                      ReusedExchange [d_date_sk] #2
+                                                InputAdapter
+                                                  ReusedExchange [ca_address_sk,ca_state] #3
+            InputAdapter
+              BroadcastExchange #7
+                WholeStageCodegen (9)
+                  Filter [c_customer_sk,c_current_addr_sk]
+                    ColumnarToRow
+                      InputAdapter
+                        Scan parquet default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name]
+        InputAdapter
+          BroadcastExchange #8
+            WholeStageCodegen (10)
+              Filter [ca_state,ca_address_sk]
+                ColumnarToRow
+                  InputAdapter
+                    Scan parquet default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.sf100/explain.txt
new file mode 100644
index 0000000000000..6725e273a3acc
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.sf100/explain.txt
@@ -0,0 +1,191 @@
+== Physical Plan ==
+TakeOrderedAndProject (32)
++- * HashAggregate (31)
+   +- Exchange (30)
+      +- * HashAggregate (29)
+         +- * Project (28)
+            +- * SortMergeJoin Inner (27)
+               :- * Sort (20)
+               :  +- Exchange (19)
+               :     +- * Project (18)
+               :        +- * BroadcastHashJoin Inner BuildRight (17)
+               :           :- * Project (11)
+               :           :  +- * BroadcastHashJoin Inner BuildLeft (10)
+               :           :     :- BroadcastExchange (5)
+               :           :     :  +- * Project (4)
+               :           :     :     +- * Filter (3)
+               :           :     :        +- * ColumnarToRow (2)
+               :           :     :           +- Scan parquet default.item (1)
+               :           :     +- * Project (9)
+               :           :        +- * Filter (8)
+               :           :           +- * ColumnarToRow (7)
+               :           :              +- Scan parquet default.inventory (6)
+               :           +- BroadcastExchange (16)
+               :              +- * Project (15)
+               :                 +- * Filter (14)
+               :                    +- * ColumnarToRow (13)
+               :                       +- Scan parquet default.date_dim (12)
+               +- * Sort (26)
+                  +- Exchange (25)
+                     +- * Project (24)
+                        +- * Filter (23)
+                           +- * ColumnarToRow (22)
+                              +- Scan parquet default.store_sales (21)
+
+
+(1) Scan parquet default.item
+Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,62.00), LessThanOrEqual(i_current_price,92.00), In(i_manufact_id, [129,270,821,423]), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_item_desc:string,i_current_price:decimal(7,2),i_manufact_id:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5]
+
+(3) Filter [codegen id : 1]
+Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5]
+Condition : ((((isnotnull(i_current_price#4) AND (i_current_price#4 >= 62.00)) AND (i_current_price#4 <= 92.00)) AND i_manufact_id#5 IN (129,270,821,423)) AND isnotnull(i_item_sk#1))
+
+(4) Project [codegen id : 1]
+Output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4]
+Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5]
+
+(5) BroadcastExchange
+Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#6]
+
+(6) Scan parquet default.inventory
+Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(inv_date_sk#9), dynamicpruningexpression(inv_date_sk#9 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)]
+ReadSchema: struct<inv_item_sk:int,inv_quantity_on_hand:int>
+
+(7) ColumnarToRow
+Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9]
+
+(8) Filter
+Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9]
+Condition : (((isnotnull(inv_quantity_on_hand#8) AND (inv_quantity_on_hand#8 >= 100)) AND (inv_quantity_on_hand#8 <= 500)) AND isnotnull(inv_item_sk#7))
+
+(9) Project
+Output [2]: [inv_item_sk#7, inv_date_sk#9]
+Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9]
+
+(10) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [inv_item_sk#7]
+Join condition: None
+
+(11) Project [codegen id : 3]
+Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#9]
+Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_item_sk#7, inv_date_sk#9]
+
+(12) Scan parquet default.date_dim
+Output [2]: [d_date_sk#11, d_date#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-05-25), LessThanOrEqual(d_date,2000-07-24), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(13) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#11, d_date#12]
+
+(14) Filter [codegen id : 2]
+Input [2]: [d_date_sk#11, d_date#12]
+Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-05-25)) AND (d_date#12 <= 2000-07-24)) AND isnotnull(d_date_sk#11))
+
+(15) Project [codegen id : 2]
+Output [1]: [d_date_sk#11]
+Input [2]: [d_date_sk#11, d_date#12]
+
+(16) BroadcastExchange
+Input [1]: [d_date_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(17) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [inv_date_sk#9]
+Right keys [1]: [d_date_sk#11]
+Join condition: None
+
+(18) Project [codegen id : 3]
+Output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4]
+Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#11]
+
+(19) Exchange
+Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4]
+Arguments: hashpartitioning(i_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#14]
+
+(20) Sort [codegen id : 4]
+Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4]
+Arguments: [i_item_sk#1 ASC NULLS FIRST], false, 0
+
+(21) Scan parquet default.store_sales
+Output [2]: [ss_item_sk#15, ss_sold_date_sk#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int>
+
+(22) ColumnarToRow [codegen id : 5]
+Input [2]: [ss_item_sk#15, ss_sold_date_sk#16]
+
+(23) Filter [codegen id : 5]
+Input [2]: [ss_item_sk#15, ss_sold_date_sk#16]
+Condition : isnotnull(ss_item_sk#15)
+
+(24) Project [codegen id : 5]
+Output [1]: [ss_item_sk#15]
+Input [2]: [ss_item_sk#15, ss_sold_date_sk#16]
+
+(25) Exchange
+Input [1]: [ss_item_sk#15]
+Arguments: hashpartitioning(ss_item_sk#15, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(26) Sort [codegen id : 6]
+Input [1]: [ss_item_sk#15]
+Arguments: [ss_item_sk#15 ASC NULLS FIRST], false, 0
+
+(27) SortMergeJoin [codegen id : 7]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [ss_item_sk#15]
+Join condition: None
+
+(28) Project [codegen id : 7]
+Output [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, ss_item_sk#15]
+
+(29) HashAggregate [codegen id : 7]
+Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+
+(30) Exchange
+Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(31) HashAggregate [codegen id : 8]
+Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+
+(32) TakeOrderedAndProject
+Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Arguments: 100, [i_item_id#2 ASC NULLS FIRST], [i_item_id#2, i_item_desc#3, i_current_price#4]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 6 Hosting Expression = inv_date_sk#9 IN dynamicpruning#10
+ReusedExchange (33)
+
+
+(33) ReusedExchange [Reuses operator id: 16]
+Output [1]: [d_date_sk#11]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.sf100/simplified.txt
new file mode 100644
index 0000000000000..1a22750705b3c
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.sf100/simplified.txt
@@ -0,0 +1,53 @@
+TakeOrderedAndProject [i_item_id,i_item_desc,i_current_price]
+  WholeStageCodegen (8)
+    HashAggregate [i_item_id,i_item_desc,i_current_price]
+      InputAdapter
+        Exchange [i_item_id,i_item_desc,i_current_price] #1
+          WholeStageCodegen (7)
+            HashAggregate [i_item_id,i_item_desc,i_current_price]
+              Project [i_item_id,i_item_desc,i_current_price]
+                SortMergeJoin [i_item_sk,ss_item_sk]
+                  InputAdapter
+                    WholeStageCodegen (4)
+                      Sort [i_item_sk]
+                        InputAdapter
+                          Exchange [i_item_sk] #2
+                            WholeStageCodegen (3)
+                              Project [i_item_sk,i_item_id,i_item_desc,i_current_price]
+                                BroadcastHashJoin [inv_date_sk,d_date_sk]
+                                  Project [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk]
+                                    BroadcastHashJoin [i_item_sk,inv_item_sk]
+                                      InputAdapter
+                                        BroadcastExchange #3
+                                          WholeStageCodegen (1)
+                                            Project [i_item_sk,i_item_id,i_item_desc,i_current_price]
+                                              Filter [i_current_price,i_manufact_id,i_item_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id]
+                                      Project [inv_item_sk,inv_date_sk]
+                                        Filter [inv_quantity_on_hand,inv_item_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk]
+                                                SubqueryBroadcast [d_date_sk] #1
+                                                  ReusedExchange [d_date_sk] #4
+                                  InputAdapter
+                                    BroadcastExchange #4
+                                      WholeStageCodegen (2)
+                                        Project [d_date_sk]
+                                          Filter [d_date,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_date]
+                  InputAdapter
+                    WholeStageCodegen (6)
+                      Sort [ss_item_sk]
+                        InputAdapter
+                          Exchange [ss_item_sk] #5
+                            WholeStageCodegen (5)
+                              Project [ss_item_sk]
+                                Filter [ss_item_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_sales [ss_item_sk,ss_sold_date_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/explain.txt
new file mode 100644
index 0000000000000..a03333f7623cc
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/explain.txt
@@ -0,0 +1,176 @@
+== Physical Plan ==
+TakeOrderedAndProject (29)
++- * HashAggregate (28)
+   +- Exchange (27)
+      +- * HashAggregate (26)
+         +- * Project (25)
+            +- * BroadcastHashJoin Inner BuildLeft (24)
+               :- BroadcastExchange (19)
+               :  +- * Project (18)
+               :     +- * BroadcastHashJoin Inner BuildRight (17)
+               :        :- * Project (11)
+               :        :  +- * BroadcastHashJoin Inner BuildRight (10)
+               :        :     :- * Project (4)
+               :        :     :  +- * Filter (3)
+               :        :     :     +- * ColumnarToRow (2)
+               :        :     :        +- Scan parquet default.item (1)
+               :        :     +- BroadcastExchange (9)
+               :        :        +- * Project (8)
+               :        :           +- * Filter (7)
+               :        :              +- * ColumnarToRow (6)
+               :        :                 +- Scan parquet default.inventory (5)
+               :        +- BroadcastExchange (16)
+               :           +- * Project (15)
+               :              +- * Filter (14)
+               :                 +- * ColumnarToRow (13)
+               :                    +- Scan parquet default.date_dim (12)
+               +- * Project (23)
+                  +- * Filter (22)
+                     +- * ColumnarToRow (21)
+                        +- Scan parquet default.store_sales (20)
+
+
+(1) Scan parquet default.item
+Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,62.00), LessThanOrEqual(i_current_price,92.00), In(i_manufact_id, [129,270,821,423]), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_item_desc:string,i_current_price:decimal(7,2),i_manufact_id:int>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5]
+
+(3) Filter [codegen id : 3]
+Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5]
+Condition : ((((isnotnull(i_current_price#4) AND (i_current_price#4 >= 62.00)) AND (i_current_price#4 <= 92.00)) AND i_manufact_id#5 IN (129,270,821,423)) AND isnotnull(i_item_sk#1))
+
+(4) Project [codegen id : 3]
+Output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4]
+Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5]
+
+(5) Scan parquet default.inventory
+Output [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(inv_date_sk#8), dynamicpruningexpression(inv_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)]
+ReadSchema: struct<inv_item_sk:int,inv_quantity_on_hand:int>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8]
+
+(7) Filter [codegen id : 1]
+Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8]
+Condition : (((isnotnull(inv_quantity_on_hand#7) AND (inv_quantity_on_hand#7 >= 100)) AND (inv_quantity_on_hand#7 <= 500)) AND isnotnull(inv_item_sk#6))
+
+(8) Project [codegen id : 1]
+Output [2]: [inv_item_sk#6, inv_date_sk#8]
+Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8]
+
+(9) BroadcastExchange
+Input [2]: [inv_item_sk#6, inv_date_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(10) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [inv_item_sk#6]
+Join condition: None
+
+(11) Project [codegen id : 3]
+Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8]
+Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_item_sk#6, inv_date_sk#8]
+
+(12) Scan parquet default.date_dim
+Output [2]: [d_date_sk#11, d_date#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-05-25), LessThanOrEqual(d_date,2000-07-24), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(13) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#11, d_date#12]
+
+(14) Filter [codegen id : 2]
+Input [2]: [d_date_sk#11, d_date#12]
+Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-05-25)) AND (d_date#12 <= 2000-07-24)) AND isnotnull(d_date_sk#11))
+
+(15) Project [codegen id : 2]
+Output [1]: [d_date_sk#11]
+Input [2]: [d_date_sk#11, d_date#12]
+
+(16) BroadcastExchange
+Input [1]: [d_date_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(17) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [inv_date_sk#8]
+Right keys [1]: [d_date_sk#11]
+Join condition: None
+
+(18) Project [codegen id : 3]
+Output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4]
+Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8, d_date_sk#11]
+
+(19) BroadcastExchange
+Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(20) Scan parquet default.store_sales
+Output [2]: [ss_item_sk#15, ss_sold_date_sk#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int>
+
+(21) ColumnarToRow
+Input [2]: [ss_item_sk#15, ss_sold_date_sk#16]
+
+(22) Filter
+Input [2]: [ss_item_sk#15, ss_sold_date_sk#16]
+Condition : isnotnull(ss_item_sk#15)
+
+(23) Project
+Output [1]: [ss_item_sk#15]
+Input [2]: [ss_item_sk#15, ss_sold_date_sk#16]
+
+(24) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [ss_item_sk#15]
+Join condition: None
+
+(25) Project [codegen id : 4]
+Output [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, ss_item_sk#15]
+
+(26) HashAggregate [codegen id : 4]
+Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+
+(27) Exchange
+Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(28) HashAggregate [codegen id : 5]
+Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+
+(29) TakeOrderedAndProject
+Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4]
+Arguments: 100, [i_item_id#2 ASC NULLS FIRST], [i_item_id#2, i_item_desc#3, i_current_price#4]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 5 Hosting Expression = inv_date_sk#8 IN dynamicpruning#9
+ReusedExchange (30)
+
+
+(30) ReusedExchange [Reuses operator id: 16]
+Output [1]: [d_date_sk#11]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/simplified.txt
new file mode 100644
index 0000000000000..7cec23062ee50
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/simplified.txt
@@ -0,0 +1,44 @@
+TakeOrderedAndProject [i_item_id,i_item_desc,i_current_price]
+  WholeStageCodegen (5)
+    HashAggregate [i_item_id,i_item_desc,i_current_price]
+      InputAdapter
+        Exchange [i_item_id,i_item_desc,i_current_price] #1
+          WholeStageCodegen (4)
+            HashAggregate [i_item_id,i_item_desc,i_current_price]
+              Project [i_item_id,i_item_desc,i_current_price]
+                BroadcastHashJoin [i_item_sk,ss_item_sk]
+                  InputAdapter
+                    BroadcastExchange #2
+                      WholeStageCodegen (3)
+                        Project [i_item_sk,i_item_id,i_item_desc,i_current_price]
+                          BroadcastHashJoin [inv_date_sk,d_date_sk]
+                            Project [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk]
+                              BroadcastHashJoin [i_item_sk,inv_item_sk]
+                                Project [i_item_sk,i_item_id,i_item_desc,i_current_price]
+                                  Filter [i_current_price,i_manufact_id,i_item_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id]
+                                InputAdapter
+                                  BroadcastExchange #3
+                                    WholeStageCodegen (1)
+                                      Project [inv_item_sk,inv_date_sk]
+                                        Filter [inv_quantity_on_hand,inv_item_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk]
+                                                SubqueryBroadcast [d_date_sk] #1
+                                                  ReusedExchange [d_date_sk] #4
+                            InputAdapter
+                              BroadcastExchange #4
+                                WholeStageCodegen (2)
+                                  Project [d_date_sk]
+                                    Filter [d_date,d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.date_dim [d_date_sk,d_date]
+                  Project [ss_item_sk]
+                    Filter [ss_item_sk]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_item_sk,ss_sold_date_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.sf100/explain.txt
new file mode 100644
index 0000000000000..7f0868b3b4db2
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.sf100/explain.txt
@@ -0,0 +1,347 @@
+== Physical Plan ==
+TakeOrderedAndProject (61)
++- * Project (60)
+   +- * BroadcastHashJoin Inner BuildRight (59)
+      :- * Project (45)
+      :  +- * BroadcastHashJoin Inner BuildRight (44)
+      :     :- * HashAggregate (30)
+      :     :  +- Exchange (29)
+      :     :     +- * HashAggregate (28)
+      :     :        +- * Project (27)
+      :     :           +- * BroadcastHashJoin Inner BuildRight (26)
+      :     :              :- * Project (21)
+      :     :              :  +- * BroadcastHashJoin Inner BuildRight (20)
+      :     :              :     :- * Filter (3)
+      :     :              :     :  +- * ColumnarToRow (2)
+      :     :              :     :     +- Scan parquet default.store_returns (1)
+      :     :              :     +- BroadcastExchange (19)
+      :     :              :        +- * Project (18)
+      :     :              :           +- * BroadcastHashJoin LeftSemi BuildRight (17)
+      :     :              :              :- * Filter (6)
+      :     :              :              :  +- * ColumnarToRow (5)
+      :     :              :              :     +- Scan parquet default.date_dim (4)
+      :     :              :              +- BroadcastExchange (16)
+      :     :              :                 +- * Project (15)
+      :     :              :                    +- * BroadcastHashJoin LeftSemi BuildRight (14)
+      :     :              :                       :- * ColumnarToRow (8)
+      :     :              :                       :  +- Scan parquet default.date_dim (7)
+      :     :              :                       +- BroadcastExchange (13)
+      :     :              :                          +- * Project (12)
+      :     :              :                             +- * Filter (11)
+      :     :              :                                +- * ColumnarToRow (10)
+      :     :              :                                   +- Scan parquet default.date_dim (9)
+      :     :              +- BroadcastExchange (25)
+      :     :                 +- * Filter (24)
+      :     :                    +- * ColumnarToRow (23)
+      :     :                       +- Scan parquet default.item (22)
+      :     +- BroadcastExchange (43)
+      :        +- * HashAggregate (42)
+      :           +- Exchange (41)
+      :              +- * HashAggregate (40)
+      :                 +- * Project (39)
+      :                    +- * BroadcastHashJoin Inner BuildRight (38)
+      :                       :- * Project (36)
+      :                       :  +- * BroadcastHashJoin Inner BuildRight (35)
+      :                       :     :- * Filter (33)
+      :                       :     :  +- * ColumnarToRow (32)
+      :                       :     :     +- Scan parquet default.catalog_returns (31)
+      :                       :     +- ReusedExchange (34)
+      :                       +- ReusedExchange (37)
+      +- BroadcastExchange (58)
+         +- * HashAggregate (57)
+            +- Exchange (56)
+               +- * HashAggregate (55)
+                  +- * Project (54)
+                     +- * BroadcastHashJoin Inner BuildRight (53)
+                        :- * Project (51)
+                        :  +- * BroadcastHashJoin Inner BuildRight (50)
+                        :     :- * Filter (48)
+                        :     :  +- * ColumnarToRow (47)
+                        :     :     +- Scan parquet default.web_returns (46)
+                        :     +- ReusedExchange (49)
+                        +- ReusedExchange (52)
+
+
+(1) Scan parquet default.store_returns
+Output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(sr_returned_date_sk#3), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_return_quantity:int>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3]
+
+(3) Filter [codegen id : 5]
+Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3]
+Condition : isnotnull(sr_item_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#4, d_date#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(5) ColumnarToRow [codegen id : 3]
+Input [2]: [d_date_sk#4, d_date#5]
+
+(6) Filter [codegen id : 3]
+Input [2]: [d_date_sk#4, d_date#5]
+Condition : isnotnull(d_date_sk#4)
+
+(7) Scan parquet default.date_dim
+Output [2]: [d_date#6, d_week_seq#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+ReadSchema: struct<d_date:date,d_week_seq:int>
+
+(8) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date#6, d_week_seq#7]
+
+(9) Scan parquet default.date_dim
+Output [2]: [d_date#8, d_week_seq#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+ReadSchema: struct<d_date:date,d_week_seq:int>
+
+(10) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date#8, d_week_seq#9]
+
+(11) Filter [codegen id : 1]
+Input [2]: [d_date#8, d_week_seq#9]
+Condition : cast(d_date#8 as string) IN (2000-06-30,2000-09-27,2000-11-17)
+
+(12) Project [codegen id : 1]
+Output [1]: [d_week_seq#9]
+Input [2]: [d_date#8, d_week_seq#9]
+
+(13) BroadcastExchange
+Input [1]: [d_week_seq#9]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(14) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [d_week_seq#7]
+Right keys [1]: [d_week_seq#9]
+Join condition: None
+
+(15) Project [codegen id : 2]
+Output [1]: [d_date#6]
+Input [2]: [d_date#6, d_week_seq#7]
+
+(16) BroadcastExchange
+Input [1]: [d_date#6]
+Arguments: HashedRelationBroadcastMode(List(input[0, date, true]),false), [id=#11]
+
+(17) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [d_date#5]
+Right keys [1]: [d_date#6]
+Join condition: None
+
+(18) Project [codegen id : 3]
+Output [1]: [d_date_sk#4]
+Input [2]: [d_date_sk#4, d_date#5]
+
+(19) BroadcastExchange
+Input [1]: [d_date_sk#4]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(20) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [sr_returned_date_sk#3]
+Right keys [1]: [d_date_sk#4]
+Join condition: None
+
+(21) Project [codegen id : 5]
+Output [2]: [sr_item_sk#1, sr_return_quantity#2]
+Input [4]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, d_date_sk#4]
+
+(22) Scan parquet default.item
+Output [2]: [i_item_sk#13, i_item_id#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_item_id)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(23) ColumnarToRow [codegen id : 4]
+Input [2]: [i_item_sk#13, i_item_id#14]
+
+(24) Filter [codegen id : 4]
+Input [2]: [i_item_sk#13, i_item_id#14]
+Condition : (isnotnull(i_item_sk#13) AND isnotnull(i_item_id#14))
+
+(25) BroadcastExchange
+Input [2]: [i_item_sk#13, i_item_id#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#15]
+
+(26) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [sr_item_sk#1]
+Right keys [1]: [i_item_sk#13]
+Join condition: None
+
+(27) Project [codegen id : 5]
+Output [2]: [sr_return_quantity#2, i_item_id#14]
+Input [4]: [sr_item_sk#1, sr_return_quantity#2, i_item_sk#13, i_item_id#14]
+
+(28) HashAggregate [codegen id : 5]
+Input [2]: [sr_return_quantity#2, i_item_id#14]
+Keys [1]: [i_item_id#14]
+Functions [1]: [partial_sum(sr_return_quantity#2)]
+Aggregate Attributes [1]: [sum#16]
+Results [2]: [i_item_id#14, sum#17]
+
+(29) Exchange
+Input [2]: [i_item_id#14, sum#17]
+Arguments: hashpartitioning(i_item_id#14, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(30) HashAggregate [codegen id : 18]
+Input [2]: [i_item_id#14, sum#17]
+Keys [1]: [i_item_id#14]
+Functions [1]: [sum(sr_return_quantity#2)]
+Aggregate Attributes [1]: [sum(sr_return_quantity#2)#19]
+Results [2]: [i_item_id#14 AS item_id#20, sum(sr_return_quantity#2)#19 AS sr_item_qty#21]
+
+(31) Scan parquet default.catalog_returns
+Output [3]: [cr_item_sk#22, cr_return_quantity#23, cr_returned_date_sk#24]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cr_returned_date_sk#24), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(cr_item_sk)]
+ReadSchema: struct<cr_item_sk:int,cr_return_quantity:int>
+
+(32) ColumnarToRow [codegen id : 10]
+Input [3]: [cr_item_sk#22, cr_return_quantity#23, cr_returned_date_sk#24]
+
+(33) Filter [codegen id : 10]
+Input [3]: [cr_item_sk#22, cr_return_quantity#23, cr_returned_date_sk#24]
+Condition : isnotnull(cr_item_sk#22)
+
+(34) ReusedExchange [Reuses operator id: 19]
+Output [1]: [d_date_sk#25]
+
+(35) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cr_returned_date_sk#24]
+Right keys [1]: [d_date_sk#25]
+Join condition: None
+
+(36) Project [codegen id : 10]
+Output [2]: [cr_item_sk#22, cr_return_quantity#23]
+Input [4]: [cr_item_sk#22, cr_return_quantity#23, cr_returned_date_sk#24, d_date_sk#25]
+
+(37) ReusedExchange [Reuses operator id: 25]
+Output [2]: [i_item_sk#26, i_item_id#27]
+
+(38) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cr_item_sk#22]
+Right keys [1]: [i_item_sk#26]
+Join condition: None
+
+(39) Project [codegen id : 10]
+Output [2]: [cr_return_quantity#23, i_item_id#27]
+Input [4]: [cr_item_sk#22, cr_return_quantity#23, i_item_sk#26, i_item_id#27]
+
+(40) HashAggregate [codegen id : 10]
+Input [2]: [cr_return_quantity#23, i_item_id#27]
+Keys [1]: [i_item_id#27]
+Functions [1]: [partial_sum(cr_return_quantity#23)]
+Aggregate Attributes [1]: [sum#28]
+Results [2]: [i_item_id#27, sum#29]
+
+(41) Exchange
+Input [2]: [i_item_id#27, sum#29]
+Arguments: hashpartitioning(i_item_id#27, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(42) HashAggregate [codegen id : 11]
+Input [2]: [i_item_id#27, sum#29]
+Keys [1]: [i_item_id#27]
+Functions [1]: [sum(cr_return_quantity#23)]
+Aggregate Attributes [1]: [sum(cr_return_quantity#23)#31]
+Results [2]: [i_item_id#27 AS item_id#32, sum(cr_return_quantity#23)#31 AS cr_item_qty#33]
+
+(43) BroadcastExchange
+Input [2]: [item_id#32, cr_item_qty#33]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#34]
+
+(44) BroadcastHashJoin [codegen id : 18]
+Left keys [1]: [item_id#20]
+Right keys [1]: [item_id#32]
+Join condition: None
+
+(45) Project [codegen id : 18]
+Output [3]: [item_id#20, sr_item_qty#21, cr_item_qty#33]
+Input [4]: [item_id#20, sr_item_qty#21, item_id#32, cr_item_qty#33]
+
+(46) Scan parquet default.web_returns
+Output [3]: [wr_item_sk#35, wr_return_quantity#36, wr_returned_date_sk#37]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(wr_returned_date_sk#37), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(wr_item_sk)]
+ReadSchema: struct<wr_item_sk:int,wr_return_quantity:int>
+
+(47) ColumnarToRow [codegen id : 16]
+Input [3]: [wr_item_sk#35, wr_return_quantity#36, wr_returned_date_sk#37]
+
+(48) Filter [codegen id : 16]
+Input [3]: [wr_item_sk#35, wr_return_quantity#36, wr_returned_date_sk#37]
+Condition : isnotnull(wr_item_sk#35)
+
+(49) ReusedExchange [Reuses operator id: 19]
+Output [1]: [d_date_sk#38]
+
+(50) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [wr_returned_date_sk#37]
+Right keys [1]: [d_date_sk#38]
+Join condition: None
+
+(51) Project [codegen id : 16]
+Output [2]: [wr_item_sk#35, wr_return_quantity#36]
+Input [4]: [wr_item_sk#35, wr_return_quantity#36, wr_returned_date_sk#37, d_date_sk#38]
+
+(52) ReusedExchange [Reuses operator id: 25]
+Output [2]: [i_item_sk#39, i_item_id#40]
+
+(53) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [wr_item_sk#35]
+Right keys [1]: [i_item_sk#39]
+Join condition: None
+
+(54) Project [codegen id : 16]
+Output [2]: [wr_return_quantity#36, i_item_id#40]
+Input [4]: [wr_item_sk#35, wr_return_quantity#36, i_item_sk#39, i_item_id#40]
+
+(55) HashAggregate [codegen id : 16]
+Input [2]: [wr_return_quantity#36, i_item_id#40]
+Keys [1]: [i_item_id#40]
+Functions [1]: [partial_sum(wr_return_quantity#36)]
+Aggregate Attributes [1]: [sum#41]
+Results [2]: [i_item_id#40, sum#42]
+
+(56) Exchange
+Input [2]: [i_item_id#40, sum#42]
+Arguments: hashpartitioning(i_item_id#40, 5), ENSURE_REQUIREMENTS, [id=#43]
+
+(57) HashAggregate [codegen id : 17]
+Input [2]: [i_item_id#40, sum#42]
+Keys [1]: [i_item_id#40]
+Functions [1]: [sum(wr_return_quantity#36)]
+Aggregate Attributes [1]: [sum(wr_return_quantity#36)#44]
+Results [2]: [i_item_id#40 AS item_id#45, sum(wr_return_quantity#36)#44 AS wr_item_qty#46]
+
+(58) BroadcastExchange
+Input [2]: [item_id#45, wr_item_qty#46]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#47]
+
+(59) BroadcastHashJoin [codegen id : 18]
+Left keys [1]: [item_id#20]
+Right keys [1]: [item_id#45]
+Join condition: None
+
+(60) Project [codegen id : 18]
+Output [8]: [item_id#20, sr_item_qty#21, (((cast(sr_item_qty#21 as double) / cast(((sr_item_qty#21 + cr_item_qty#33) + wr_item_qty#46) as double)) / 3.0) * 100.0) AS sr_dev#48, cr_item_qty#33, (((cast(cr_item_qty#33 as double) / cast(((sr_item_qty#21 + cr_item_qty#33) + wr_item_qty#46) as double)) / 3.0) * 100.0) AS cr_dev#49, wr_item_qty#46, (((cast(wr_item_qty#46 as double) / cast(((sr_item_qty#21 + cr_item_qty#33) + wr_item_qty#46) as double)) / 3.0) * 100.0) AS wr_dev#50, CheckOverflow((promote_precision(cast(cast(((sr_item_qty#21 + cr_item_qty#33) + wr_item_qty#46) as decimal(20,0)) as decimal(21,1))) / 3.0), DecimalType(27,6), true) AS average#51]
+Input [5]: [item_id#20, sr_item_qty#21, cr_item_qty#33, item_id#45, wr_item_qty#46]
+
+(61) TakeOrderedAndProject
+Input [8]: [item_id#20, sr_item_qty#21, sr_dev#48, cr_item_qty#33, cr_dev#49, wr_item_qty#46, wr_dev#50, average#51]
+Arguments: 100, [item_id#20 ASC NULLS FIRST, sr_item_qty#21 ASC NULLS FIRST], [item_id#20, sr_item_qty#21, sr_dev#48, cr_item_qty#33, cr_dev#49, wr_item_qty#46, wr_dev#50, average#51]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.sf100/simplified.txt
new file mode 100644
index 0000000000000..b46131afd5ec7
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.sf100/simplified.txt
@@ -0,0 +1,91 @@
+TakeOrderedAndProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty,wr_dev,average]
+  WholeStageCodegen (18)
+    Project [item_id,sr_item_qty,cr_item_qty,wr_item_qty]
+      BroadcastHashJoin [item_id,item_id]
+        Project [item_id,sr_item_qty,cr_item_qty]
+          BroadcastHashJoin [item_id,item_id]
+            HashAggregate [i_item_id,sum] [sum(sr_return_quantity),item_id,sr_item_qty,sum]
+              InputAdapter
+                Exchange [i_item_id] #1
+                  WholeStageCodegen (5)
+                    HashAggregate [i_item_id,sr_return_quantity] [sum,sum]
+                      Project [sr_return_quantity,i_item_id]
+                        BroadcastHashJoin [sr_item_sk,i_item_sk]
+                          Project [sr_item_sk,sr_return_quantity]
+                            BroadcastHashJoin [sr_returned_date_sk,d_date_sk]
+                              Filter [sr_item_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.store_returns [sr_item_sk,sr_return_quantity,sr_returned_date_sk]
+                              InputAdapter
+                                BroadcastExchange #2
+                                  WholeStageCodegen (3)
+                                    Project [d_date_sk]
+                                      BroadcastHashJoin [d_date,d_date]
+                                        Filter [d_date_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.date_dim [d_date_sk,d_date]
+                                        InputAdapter
+                                          BroadcastExchange #3
+                                            WholeStageCodegen (2)
+                                              Project [d_date]
+                                                BroadcastHashJoin [d_week_seq,d_week_seq]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date,d_week_seq]
+                                                  InputAdapter
+                                                    BroadcastExchange #4
+                                                      WholeStageCodegen (1)
+                                                        Project [d_week_seq]
+                                                          Filter [d_date]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.date_dim [d_date,d_week_seq]
+                          InputAdapter
+                            BroadcastExchange #5
+                              WholeStageCodegen (4)
+                                Filter [i_item_sk,i_item_id]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.item [i_item_sk,i_item_id]
+            InputAdapter
+              BroadcastExchange #6
+                WholeStageCodegen (11)
+                  HashAggregate [i_item_id,sum] [sum(cr_return_quantity),item_id,cr_item_qty,sum]
+                    InputAdapter
+                      Exchange [i_item_id] #7
+                        WholeStageCodegen (10)
+                          HashAggregate [i_item_id,cr_return_quantity] [sum,sum]
+                            Project [cr_return_quantity,i_item_id]
+                              BroadcastHashJoin [cr_item_sk,i_item_sk]
+                                Project [cr_item_sk,cr_return_quantity]
+                                  BroadcastHashJoin [cr_returned_date_sk,d_date_sk]
+                                    Filter [cr_item_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.catalog_returns [cr_item_sk,cr_return_quantity,cr_returned_date_sk]
+                                    InputAdapter
+                                      ReusedExchange [d_date_sk] #2
+                                InputAdapter
+                                  ReusedExchange [i_item_sk,i_item_id] #5
+        InputAdapter
+          BroadcastExchange #8
+            WholeStageCodegen (17)
+              HashAggregate [i_item_id,sum] [sum(wr_return_quantity),item_id,wr_item_qty,sum]
+                InputAdapter
+                  Exchange [i_item_id] #9
+                    WholeStageCodegen (16)
+                      HashAggregate [i_item_id,wr_return_quantity] [sum,sum]
+                        Project [wr_return_quantity,i_item_id]
+                          BroadcastHashJoin [wr_item_sk,i_item_sk]
+                            Project [wr_item_sk,wr_return_quantity]
+                              BroadcastHashJoin [wr_returned_date_sk,d_date_sk]
+                                Filter [wr_item_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.web_returns [wr_item_sk,wr_return_quantity,wr_returned_date_sk]
+                                InputAdapter
+                                  ReusedExchange [d_date_sk] #2
+                            InputAdapter
+                              ReusedExchange [i_item_sk,i_item_id] #5
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/explain.txt
new file mode 100644
index 0000000000000..0764824706376
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/explain.txt
@@ -0,0 +1,347 @@
+== Physical Plan ==
+TakeOrderedAndProject (61)
++- * Project (60)
+   +- * BroadcastHashJoin Inner BuildRight (59)
+      :- * Project (45)
+      :  +- * BroadcastHashJoin Inner BuildRight (44)
+      :     :- * HashAggregate (30)
+      :     :  +- Exchange (29)
+      :     :     +- * HashAggregate (28)
+      :     :        +- * Project (27)
+      :     :           +- * BroadcastHashJoin Inner BuildRight (26)
+      :     :              :- * Project (9)
+      :     :              :  +- * BroadcastHashJoin Inner BuildRight (8)
+      :     :              :     :- * Filter (3)
+      :     :              :     :  +- * ColumnarToRow (2)
+      :     :              :     :     +- Scan parquet default.store_returns (1)
+      :     :              :     +- BroadcastExchange (7)
+      :     :              :        +- * Filter (6)
+      :     :              :           +- * ColumnarToRow (5)
+      :     :              :              +- Scan parquet default.item (4)
+      :     :              +- BroadcastExchange (25)
+      :     :                 +- * Project (24)
+      :     :                    +- * BroadcastHashJoin LeftSemi BuildRight (23)
+      :     :                       :- * Filter (12)
+      :     :                       :  +- * ColumnarToRow (11)
+      :     :                       :     +- Scan parquet default.date_dim (10)
+      :     :                       +- BroadcastExchange (22)
+      :     :                          +- * Project (21)
+      :     :                             +- * BroadcastHashJoin LeftSemi BuildRight (20)
+      :     :                                :- * ColumnarToRow (14)
+      :     :                                :  +- Scan parquet default.date_dim (13)
+      :     :                                +- BroadcastExchange (19)
+      :     :                                   +- * Project (18)
+      :     :                                      +- * Filter (17)
+      :     :                                         +- * ColumnarToRow (16)
+      :     :                                            +- Scan parquet default.date_dim (15)
+      :     +- BroadcastExchange (43)
+      :        +- * HashAggregate (42)
+      :           +- Exchange (41)
+      :              +- * HashAggregate (40)
+      :                 +- * Project (39)
+      :                    +- * BroadcastHashJoin Inner BuildRight (38)
+      :                       :- * Project (36)
+      :                       :  +- * BroadcastHashJoin Inner BuildRight (35)
+      :                       :     :- * Filter (33)
+      :                       :     :  +- * ColumnarToRow (32)
+      :                       :     :     +- Scan parquet default.catalog_returns (31)
+      :                       :     +- ReusedExchange (34)
+      :                       +- ReusedExchange (37)
+      +- BroadcastExchange (58)
+         +- * HashAggregate (57)
+            +- Exchange (56)
+               +- * HashAggregate (55)
+                  +- * Project (54)
+                     +- * BroadcastHashJoin Inner BuildRight (53)
+                        :- * Project (51)
+                        :  +- * BroadcastHashJoin Inner BuildRight (50)
+                        :     :- * Filter (48)
+                        :     :  +- * ColumnarToRow (47)
+                        :     :     +- Scan parquet default.web_returns (46)
+                        :     +- ReusedExchange (49)
+                        +- ReusedExchange (52)
+
+
+(1) Scan parquet default.store_returns
+Output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(sr_returned_date_sk#3), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_return_quantity:int>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3]
+
+(3) Filter [codegen id : 5]
+Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3]
+Condition : isnotnull(sr_item_sk#1)
+
+(4) Scan parquet default.item
+Output [2]: [i_item_sk#4, i_item_id#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_item_id)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [i_item_sk#4, i_item_id#5]
+
+(6) Filter [codegen id : 1]
+Input [2]: [i_item_sk#4, i_item_id#5]
+Condition : (isnotnull(i_item_sk#4) AND isnotnull(i_item_id#5))
+
+(7) BroadcastExchange
+Input [2]: [i_item_sk#4, i_item_id#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#6]
+
+(8) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [sr_item_sk#1]
+Right keys [1]: [i_item_sk#4]
+Join condition: None
+
+(9) Project [codegen id : 5]
+Output [3]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5]
+Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#4, i_item_id#5]
+
+(10) Scan parquet default.date_dim
+Output [2]: [d_date_sk#7, d_date#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(11) ColumnarToRow [codegen id : 4]
+Input [2]: [d_date_sk#7, d_date#8]
+
+(12) Filter [codegen id : 4]
+Input [2]: [d_date_sk#7, d_date#8]
+Condition : isnotnull(d_date_sk#7)
+
+(13) Scan parquet default.date_dim
+Output [2]: [d_date#9, d_week_seq#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+ReadSchema: struct<d_date:date,d_week_seq:int>
+
+(14) ColumnarToRow [codegen id : 3]
+Input [2]: [d_date#9, d_week_seq#10]
+
+(15) Scan parquet default.date_dim
+Output [2]: [d_date#11, d_week_seq#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+ReadSchema: struct<d_date:date,d_week_seq:int>
+
+(16) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date#11, d_week_seq#12]
+
+(17) Filter [codegen id : 2]
+Input [2]: [d_date#11, d_week_seq#12]
+Condition : cast(d_date#11 as string) IN (2000-06-30,2000-09-27,2000-11-17)
+
+(18) Project [codegen id : 2]
+Output [1]: [d_week_seq#12]
+Input [2]: [d_date#11, d_week_seq#12]
+
+(19) BroadcastExchange
+Input [1]: [d_week_seq#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(20) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [d_week_seq#10]
+Right keys [1]: [d_week_seq#12]
+Join condition: None
+
+(21) Project [codegen id : 3]
+Output [1]: [d_date#9]
+Input [2]: [d_date#9, d_week_seq#10]
+
+(22) BroadcastExchange
+Input [1]: [d_date#9]
+Arguments: HashedRelationBroadcastMode(List(input[0, date, true]),false), [id=#14]
+
+(23) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [d_date#8]
+Right keys [1]: [d_date#9]
+Join condition: None
+
+(24) Project [codegen id : 4]
+Output [1]: [d_date_sk#7]
+Input [2]: [d_date_sk#7, d_date#8]
+
+(25) BroadcastExchange
+Input [1]: [d_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#15]
+
+(26) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [sr_returned_date_sk#3]
+Right keys [1]: [d_date_sk#7]
+Join condition: None
+
+(27) Project [codegen id : 5]
+Output [2]: [sr_return_quantity#2, i_item_id#5]
+Input [4]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5, d_date_sk#7]
+
+(28) HashAggregate [codegen id : 5]
+Input [2]: [sr_return_quantity#2, i_item_id#5]
+Keys [1]: [i_item_id#5]
+Functions [1]: [partial_sum(sr_return_quantity#2)]
+Aggregate Attributes [1]: [sum#16]
+Results [2]: [i_item_id#5, sum#17]
+
+(29) Exchange
+Input [2]: [i_item_id#5, sum#17]
+Arguments: hashpartitioning(i_item_id#5, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(30) HashAggregate [codegen id : 18]
+Input [2]: [i_item_id#5, sum#17]
+Keys [1]: [i_item_id#5]
+Functions [1]: [sum(sr_return_quantity#2)]
+Aggregate Attributes [1]: [sum(sr_return_quantity#2)#19]
+Results [2]: [i_item_id#5 AS item_id#20, sum(sr_return_quantity#2)#19 AS sr_item_qty#21]
+
+(31) Scan parquet default.catalog_returns
+Output [3]: [cr_item_sk#22, cr_return_quantity#23, cr_returned_date_sk#24]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cr_returned_date_sk#24), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(cr_item_sk)]
+ReadSchema: struct<cr_item_sk:int,cr_return_quantity:int>
+
+(32) ColumnarToRow [codegen id : 10]
+Input [3]: [cr_item_sk#22, cr_return_quantity#23, cr_returned_date_sk#24]
+
+(33) Filter [codegen id : 10]
+Input [3]: [cr_item_sk#22, cr_return_quantity#23, cr_returned_date_sk#24]
+Condition : isnotnull(cr_item_sk#22)
+
+(34) ReusedExchange [Reuses operator id: 7]
+Output [2]: [i_item_sk#25, i_item_id#26]
+
+(35) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cr_item_sk#22]
+Right keys [1]: [i_item_sk#25]
+Join condition: None
+
+(36) Project [codegen id : 10]
+Output [3]: [cr_return_quantity#23, cr_returned_date_sk#24, i_item_id#26]
+Input [5]: [cr_item_sk#22, cr_return_quantity#23, cr_returned_date_sk#24, i_item_sk#25, i_item_id#26]
+
+(37) ReusedExchange [Reuses operator id: 25]
+Output [1]: [d_date_sk#27]
+
+(38) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cr_returned_date_sk#24]
+Right keys [1]: [d_date_sk#27]
+Join condition: None
+
+(39) Project [codegen id : 10]
+Output [2]: [cr_return_quantity#23, i_item_id#26]
+Input [4]: [cr_return_quantity#23, cr_returned_date_sk#24, i_item_id#26, d_date_sk#27]
+
+(40) HashAggregate [codegen id : 10]
+Input [2]: [cr_return_quantity#23, i_item_id#26]
+Keys [1]: [i_item_id#26]
+Functions [1]: [partial_sum(cr_return_quantity#23)]
+Aggregate Attributes [1]: [sum#28]
+Results [2]: [i_item_id#26, sum#29]
+
+(41) Exchange
+Input [2]: [i_item_id#26, sum#29]
+Arguments: hashpartitioning(i_item_id#26, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(42) HashAggregate [codegen id : 11]
+Input [2]: [i_item_id#26, sum#29]
+Keys [1]: [i_item_id#26]
+Functions [1]: [sum(cr_return_quantity#23)]
+Aggregate Attributes [1]: [sum(cr_return_quantity#23)#31]
+Results [2]: [i_item_id#26 AS item_id#32, sum(cr_return_quantity#23)#31 AS cr_item_qty#33]
+
+(43) BroadcastExchange
+Input [2]: [item_id#32, cr_item_qty#33]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#34]
+
+(44) BroadcastHashJoin [codegen id : 18]
+Left keys [1]: [item_id#20]
+Right keys [1]: [item_id#32]
+Join condition: None
+
+(45) Project [codegen id : 18]
+Output [3]: [item_id#20, sr_item_qty#21, cr_item_qty#33]
+Input [4]: [item_id#20, sr_item_qty#21, item_id#32, cr_item_qty#33]
+
+(46) Scan parquet default.web_returns
+Output [3]: [wr_item_sk#35, wr_return_quantity#36, wr_returned_date_sk#37]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(wr_returned_date_sk#37), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(wr_item_sk)]
+ReadSchema: struct<wr_item_sk:int,wr_return_quantity:int>
+
+(47) ColumnarToRow [codegen id : 16]
+Input [3]: [wr_item_sk#35, wr_return_quantity#36, wr_returned_date_sk#37]
+
+(48) Filter [codegen id : 16]
+Input [3]: [wr_item_sk#35, wr_return_quantity#36, wr_returned_date_sk#37]
+Condition : isnotnull(wr_item_sk#35)
+
+(49) ReusedExchange [Reuses operator id: 7]
+Output [2]: [i_item_sk#38, i_item_id#39]
+
+(50) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [wr_item_sk#35]
+Right keys [1]: [i_item_sk#38]
+Join condition: None
+
+(51) Project [codegen id : 16]
+Output [3]: [wr_return_quantity#36, wr_returned_date_sk#37, i_item_id#39]
+Input [5]: [wr_item_sk#35, wr_return_quantity#36, wr_returned_date_sk#37, i_item_sk#38, i_item_id#39]
+
+(52) ReusedExchange [Reuses operator id: 25]
+Output [1]: [d_date_sk#40]
+
+(53) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [wr_returned_date_sk#37]
+Right keys [1]: [d_date_sk#40]
+Join condition: None
+
+(54) Project [codegen id : 16]
+Output [2]: [wr_return_quantity#36, i_item_id#39]
+Input [4]: [wr_return_quantity#36, wr_returned_date_sk#37, i_item_id#39, d_date_sk#40]
+
+(55) HashAggregate [codegen id : 16]
+Input [2]: [wr_return_quantity#36, i_item_id#39]
+Keys [1]: [i_item_id#39]
+Functions [1]: [partial_sum(wr_return_quantity#36)]
+Aggregate Attributes [1]: [sum#41]
+Results [2]: [i_item_id#39, sum#42]
+
+(56) Exchange
+Input [2]: [i_item_id#39, sum#42]
+Arguments: hashpartitioning(i_item_id#39, 5), ENSURE_REQUIREMENTS, [id=#43]
+
+(57) HashAggregate [codegen id : 17]
+Input [2]: [i_item_id#39, sum#42]
+Keys [1]: [i_item_id#39]
+Functions [1]: [sum(wr_return_quantity#36)]
+Aggregate Attributes [1]: [sum(wr_return_quantity#36)#44]
+Results [2]: [i_item_id#39 AS item_id#45, sum(wr_return_quantity#36)#44 AS wr_item_qty#46]
+
+(58) BroadcastExchange
+Input [2]: [item_id#45, wr_item_qty#46]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#47]
+
+(59) BroadcastHashJoin [codegen id : 18]
+Left keys [1]: [item_id#20]
+Right keys [1]: [item_id#45]
+Join condition: None
+
+(60) Project [codegen id : 18]
+Output [8]: [item_id#20, sr_item_qty#21, (((cast(sr_item_qty#21 as double) / cast(((sr_item_qty#21 + cr_item_qty#33) + wr_item_qty#46) as double)) / 3.0) * 100.0) AS sr_dev#48, cr_item_qty#33, (((cast(cr_item_qty#33 as double) / cast(((sr_item_qty#21 + cr_item_qty#33) + wr_item_qty#46) as double)) / 3.0) * 100.0) AS cr_dev#49, wr_item_qty#46, (((cast(wr_item_qty#46 as double) / cast(((sr_item_qty#21 + cr_item_qty#33) + wr_item_qty#46) as double)) / 3.0) * 100.0) AS wr_dev#50, CheckOverflow((promote_precision(cast(cast(((sr_item_qty#21 + cr_item_qty#33) + wr_item_qty#46) as decimal(20,0)) as decimal(21,1))) / 3.0), DecimalType(27,6), true) AS average#51]
+Input [5]: [item_id#20, sr_item_qty#21, cr_item_qty#33, item_id#45, wr_item_qty#46]
+
+(61) TakeOrderedAndProject
+Input [8]: [item_id#20, sr_item_qty#21, sr_dev#48, cr_item_qty#33, cr_dev#49, wr_item_qty#46, wr_dev#50, average#51]
+Arguments: 100, [item_id#20 ASC NULLS FIRST, sr_item_qty#21 ASC NULLS FIRST], [item_id#20, sr_item_qty#21, sr_dev#48, cr_item_qty#33, cr_dev#49, wr_item_qty#46, wr_dev#50, average#51]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/simplified.txt
new file mode 100644
index 0000000000000..7fc930f2885fd
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/simplified.txt
@@ -0,0 +1,91 @@
+TakeOrderedAndProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty,wr_dev,average]
+  WholeStageCodegen (18)
+    Project [item_id,sr_item_qty,cr_item_qty,wr_item_qty]
+      BroadcastHashJoin [item_id,item_id]
+        Project [item_id,sr_item_qty,cr_item_qty]
+          BroadcastHashJoin [item_id,item_id]
+            HashAggregate [i_item_id,sum] [sum(sr_return_quantity),item_id,sr_item_qty,sum]
+              InputAdapter
+                Exchange [i_item_id] #1
+                  WholeStageCodegen (5)
+                    HashAggregate [i_item_id,sr_return_quantity] [sum,sum]
+                      Project [sr_return_quantity,i_item_id]
+                        BroadcastHashJoin [sr_returned_date_sk,d_date_sk]
+                          Project [sr_return_quantity,sr_returned_date_sk,i_item_id]
+                            BroadcastHashJoin [sr_item_sk,i_item_sk]
+                              Filter [sr_item_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.store_returns [sr_item_sk,sr_return_quantity,sr_returned_date_sk]
+                              InputAdapter
+                                BroadcastExchange #2
+                                  WholeStageCodegen (1)
+                                    Filter [i_item_sk,i_item_id]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.item [i_item_sk,i_item_id]
+                          InputAdapter
+                            BroadcastExchange #3
+                              WholeStageCodegen (4)
+                                Project [d_date_sk]
+                                  BroadcastHashJoin [d_date,d_date]
+                                    Filter [d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.date_dim [d_date_sk,d_date]
+                                    InputAdapter
+                                      BroadcastExchange #4
+                                        WholeStageCodegen (3)
+                                          Project [d_date]
+                                            BroadcastHashJoin [d_week_seq,d_week_seq]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.date_dim [d_date,d_week_seq]
+                                              InputAdapter
+                                                BroadcastExchange #5
+                                                  WholeStageCodegen (2)
+                                                    Project [d_week_seq]
+                                                      Filter [d_date]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.date_dim [d_date,d_week_seq]
+            InputAdapter
+              BroadcastExchange #6
+                WholeStageCodegen (11)
+                  HashAggregate [i_item_id,sum] [sum(cr_return_quantity),item_id,cr_item_qty,sum]
+                    InputAdapter
+                      Exchange [i_item_id] #7
+                        WholeStageCodegen (10)
+                          HashAggregate [i_item_id,cr_return_quantity] [sum,sum]
+                            Project [cr_return_quantity,i_item_id]
+                              BroadcastHashJoin [cr_returned_date_sk,d_date_sk]
+                                Project [cr_return_quantity,cr_returned_date_sk,i_item_id]
+                                  BroadcastHashJoin [cr_item_sk,i_item_sk]
+                                    Filter [cr_item_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.catalog_returns [cr_item_sk,cr_return_quantity,cr_returned_date_sk]
+                                    InputAdapter
+                                      ReusedExchange [i_item_sk,i_item_id] #2
+                                InputAdapter
+                                  ReusedExchange [d_date_sk] #3
+        InputAdapter
+          BroadcastExchange #8
+            WholeStageCodegen (17)
+              HashAggregate [i_item_id,sum] [sum(wr_return_quantity),item_id,wr_item_qty,sum]
+                InputAdapter
+                  Exchange [i_item_id] #9
+                    WholeStageCodegen (16)
+                      HashAggregate [i_item_id,wr_return_quantity] [sum,sum]
+                        Project [wr_return_quantity,i_item_id]
+                          BroadcastHashJoin [wr_returned_date_sk,d_date_sk]
+                            Project [wr_return_quantity,wr_returned_date_sk,i_item_id]
+                              BroadcastHashJoin [wr_item_sk,i_item_sk]
+                                Filter [wr_item_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.web_returns [wr_item_sk,wr_return_quantity,wr_returned_date_sk]
+                                InputAdapter
+                                  ReusedExchange [i_item_sk,i_item_id] #2
+                            InputAdapter
+                              ReusedExchange [d_date_sk] #3
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.sf100/explain.txt
new file mode 100644
index 0000000000000..9762d51e943e8
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.sf100/explain.txt
@@ -0,0 +1,205 @@
+== Physical Plan ==
+TakeOrderedAndProject (37)
++- * Project (36)
+   +- * BroadcastHashJoin Inner BuildLeft (35)
+      :- BroadcastExchange (30)
+      :  +- * Project (29)
+      :     +- * BroadcastHashJoin Inner BuildLeft (28)
+      :        :- BroadcastExchange (24)
+      :        :  +- * Project (23)
+      :        :     +- * BroadcastHashJoin Inner BuildRight (22)
+      :        :        :- * Project (10)
+      :        :        :  +- * BroadcastHashJoin Inner BuildRight (9)
+      :        :        :     :- * Filter (3)
+      :        :        :     :  +- * ColumnarToRow (2)
+      :        :        :     :     +- Scan parquet default.customer (1)
+      :        :        :     +- BroadcastExchange (8)
+      :        :        :        +- * Project (7)
+      :        :        :           +- * Filter (6)
+      :        :        :              +- * ColumnarToRow (5)
+      :        :        :                 +- Scan parquet default.customer_address (4)
+      :        :        +- BroadcastExchange (21)
+      :        :           +- * Project (20)
+      :        :              +- * BroadcastHashJoin Inner BuildRight (19)
+      :        :                 :- * Filter (13)
+      :        :                 :  +- * ColumnarToRow (12)
+      :        :                 :     +- Scan parquet default.household_demographics (11)
+      :        :                 +- BroadcastExchange (18)
+      :        :                    +- * Project (17)
+      :        :                       +- * Filter (16)
+      :        :                          +- * ColumnarToRow (15)
+      :        :                             +- Scan parquet default.income_band (14)
+      :        +- * Filter (27)
+      :           +- * ColumnarToRow (26)
+      :              +- Scan parquet default.customer_demographics (25)
+      +- * Project (34)
+         +- * Filter (33)
+            +- * ColumnarToRow (32)
+               +- Scan parquet default.store_returns (31)
+
+
+(1) Scan parquet default.customer
+Output [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk)]
+ReadSchema: struct<c_customer_id:string,c_current_cdemo_sk:int,c_current_hdemo_sk:int,c_current_addr_sk:int,c_first_name:string,c_last_name:string>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6]
+
+(3) Filter [codegen id : 4]
+Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6]
+Condition : ((isnotnull(c_current_addr_sk#4) AND isnotnull(c_current_cdemo_sk#2)) AND isnotnull(c_current_hdemo_sk#3))
+
+(4) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#7, ca_city#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_city), EqualTo(ca_city,Edgewood), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_city:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [ca_address_sk#7, ca_city#8]
+
+(6) Filter [codegen id : 1]
+Input [2]: [ca_address_sk#7, ca_city#8]
+Condition : ((isnotnull(ca_city#8) AND (ca_city#8 = Edgewood)) AND isnotnull(ca_address_sk#7))
+
+(7) Project [codegen id : 1]
+Output [1]: [ca_address_sk#7]
+Input [2]: [ca_address_sk#7, ca_city#8]
+
+(8) BroadcastExchange
+Input [1]: [ca_address_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#9]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [c_current_addr_sk#4]
+Right keys [1]: [ca_address_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [5]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6]
+Input [7]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6, ca_address_sk#7]
+
+(11) Scan parquet default.household_demographics
+Output [2]: [hd_demo_sk#10, hd_income_band_sk#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_income_band_sk:int>
+
+(12) ColumnarToRow [codegen id : 3]
+Input [2]: [hd_demo_sk#10, hd_income_band_sk#11]
+
+(13) Filter [codegen id : 3]
+Input [2]: [hd_demo_sk#10, hd_income_band_sk#11]
+Condition : (isnotnull(hd_demo_sk#10) AND isnotnull(hd_income_band_sk#11))
+
+(14) Scan parquet default.income_band
+Output [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/income_band]
+PushedFilters: [IsNotNull(ib_lower_bound), IsNotNull(ib_upper_bound), GreaterThanOrEqual(ib_lower_bound,38128), LessThanOrEqual(ib_upper_bound,88128), IsNotNull(ib_income_band_sk)]
+ReadSchema: struct<ib_income_band_sk:int,ib_lower_bound:int,ib_upper_bound:int>
+
+(15) ColumnarToRow [codegen id : 2]
+Input [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14]
+
+(16) Filter [codegen id : 2]
+Input [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14]
+Condition : ((((isnotnull(ib_lower_bound#13) AND isnotnull(ib_upper_bound#14)) AND (ib_lower_bound#13 >= 38128)) AND (ib_upper_bound#14 <= 88128)) AND isnotnull(ib_income_band_sk#12))
+
+(17) Project [codegen id : 2]
+Output [1]: [ib_income_band_sk#12]
+Input [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14]
+
+(18) BroadcastExchange
+Input [1]: [ib_income_band_sk#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#15]
+
+(19) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [hd_income_band_sk#11]
+Right keys [1]: [ib_income_band_sk#12]
+Join condition: None
+
+(20) Project [codegen id : 3]
+Output [1]: [hd_demo_sk#10]
+Input [3]: [hd_demo_sk#10, hd_income_band_sk#11, ib_income_band_sk#12]
+
+(21) BroadcastExchange
+Input [1]: [hd_demo_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#16]
+
+(22) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [c_current_hdemo_sk#3]
+Right keys [1]: [hd_demo_sk#10]
+Join condition: None
+
+(23) Project [codegen id : 4]
+Output [4]: [c_customer_id#1, c_current_cdemo_sk#2, c_first_name#5, c_last_name#6]
+Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, hd_demo_sk#10]
+
+(24) BroadcastExchange
+Input [4]: [c_customer_id#1, c_current_cdemo_sk#2, c_first_name#5, c_last_name#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [id=#17]
+
+(25) Scan parquet default.customer_demographics
+Output [1]: [cd_demo_sk#18]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int>
+
+(26) ColumnarToRow
+Input [1]: [cd_demo_sk#18]
+
+(27) Filter
+Input [1]: [cd_demo_sk#18]
+Condition : isnotnull(cd_demo_sk#18)
+
+(28) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [c_current_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#18]
+Join condition: None
+
+(29) Project [codegen id : 5]
+Output [4]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#18]
+Input [5]: [c_customer_id#1, c_current_cdemo_sk#2, c_first_name#5, c_last_name#6, cd_demo_sk#18]
+
+(30) BroadcastExchange
+Input [4]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#18]
+Arguments: HashedRelationBroadcastMode(List(cast(input[3, int, true] as bigint)),false), [id=#19]
+
+(31) Scan parquet default.store_returns
+Output [2]: [sr_cdemo_sk#20, sr_returned_date_sk#21]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_cdemo_sk)]
+ReadSchema: struct<sr_cdemo_sk:int>
+
+(32) ColumnarToRow
+Input [2]: [sr_cdemo_sk#20, sr_returned_date_sk#21]
+
+(33) Filter
+Input [2]: [sr_cdemo_sk#20, sr_returned_date_sk#21]
+Condition : isnotnull(sr_cdemo_sk#20)
+
+(34) Project
+Output [1]: [sr_cdemo_sk#20]
+Input [2]: [sr_cdemo_sk#20, sr_returned_date_sk#21]
+
+(35) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [cd_demo_sk#18]
+Right keys [1]: [sr_cdemo_sk#20]
+Join condition: None
+
+(36) Project [codegen id : 6]
+Output [3]: [c_customer_id#1 AS customer_id#22, concat(c_last_name#6, , , c_first_name#5) AS customername#23, c_customer_id#1]
+Input [5]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#18, sr_cdemo_sk#20]
+
+(37) TakeOrderedAndProject
+Input [3]: [customer_id#22, customername#23, c_customer_id#1]
+Arguments: 100, [c_customer_id#1 ASC NULLS FIRST], [customer_id#22, customername#23]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.sf100/simplified.txt
new file mode 100644
index 0000000000000..d44ebc0e5b05f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.sf100/simplified.txt
@@ -0,0 +1,54 @@
+TakeOrderedAndProject [c_customer_id,customer_id,customername]
+  WholeStageCodegen (6)
+    Project [c_customer_id,c_last_name,c_first_name]
+      BroadcastHashJoin [cd_demo_sk,sr_cdemo_sk]
+        InputAdapter
+          BroadcastExchange #1
+            WholeStageCodegen (5)
+              Project [c_customer_id,c_first_name,c_last_name,cd_demo_sk]
+                BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk]
+                  InputAdapter
+                    BroadcastExchange #2
+                      WholeStageCodegen (4)
+                        Project [c_customer_id,c_current_cdemo_sk,c_first_name,c_last_name]
+                          BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk]
+                            Project [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_first_name,c_last_name]
+                              BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                                Filter [c_current_addr_sk,c_current_cdemo_sk,c_current_hdemo_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.customer [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name]
+                                InputAdapter
+                                  BroadcastExchange #3
+                                    WholeStageCodegen (1)
+                                      Project [ca_address_sk]
+                                        Filter [ca_city,ca_address_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer_address [ca_address_sk,ca_city]
+                            InputAdapter
+                              BroadcastExchange #4
+                                WholeStageCodegen (3)
+                                  Project [hd_demo_sk]
+                                    BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk]
+                                      Filter [hd_demo_sk,hd_income_band_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.household_demographics [hd_demo_sk,hd_income_band_sk]
+                                      InputAdapter
+                                        BroadcastExchange #5
+                                          WholeStageCodegen (2)
+                                            Project [ib_income_band_sk]
+                                              Filter [ib_lower_bound,ib_upper_bound,ib_income_band_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.income_band [ib_income_band_sk,ib_lower_bound,ib_upper_bound]
+                  Filter [cd_demo_sk]
+                    ColumnarToRow
+                      InputAdapter
+                        Scan parquet default.customer_demographics [cd_demo_sk]
+        Project [sr_cdemo_sk]
+          Filter [sr_cdemo_sk]
+            ColumnarToRow
+              InputAdapter
+                Scan parquet default.store_returns [sr_cdemo_sk,sr_returned_date_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/explain.txt
new file mode 100644
index 0000000000000..d5b84f52f4b00
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/explain.txt
@@ -0,0 +1,205 @@
+== Physical Plan ==
+TakeOrderedAndProject (37)
++- * Project (36)
+   +- * BroadcastHashJoin Inner BuildLeft (35)
+      :- BroadcastExchange (30)
+      :  +- * Project (29)
+      :     +- * BroadcastHashJoin Inner BuildRight (28)
+      :        :- * Project (22)
+      :        :  +- * BroadcastHashJoin Inner BuildRight (21)
+      :        :     :- * Project (16)
+      :        :     :  +- * BroadcastHashJoin Inner BuildRight (15)
+      :        :     :     :- * Project (10)
+      :        :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+      :        :     :     :     :- * Filter (3)
+      :        :     :     :     :  +- * ColumnarToRow (2)
+      :        :     :     :     :     +- Scan parquet default.customer (1)
+      :        :     :     :     +- BroadcastExchange (8)
+      :        :     :     :        +- * Project (7)
+      :        :     :     :           +- * Filter (6)
+      :        :     :     :              +- * ColumnarToRow (5)
+      :        :     :     :                 +- Scan parquet default.customer_address (4)
+      :        :     :     +- BroadcastExchange (14)
+      :        :     :        +- * Filter (13)
+      :        :     :           +- * ColumnarToRow (12)
+      :        :     :              +- Scan parquet default.customer_demographics (11)
+      :        :     +- BroadcastExchange (20)
+      :        :        +- * Filter (19)
+      :        :           +- * ColumnarToRow (18)
+      :        :              +- Scan parquet default.household_demographics (17)
+      :        +- BroadcastExchange (27)
+      :           +- * Project (26)
+      :              +- * Filter (25)
+      :                 +- * ColumnarToRow (24)
+      :                    +- Scan parquet default.income_band (23)
+      +- * Project (34)
+         +- * Filter (33)
+            +- * ColumnarToRow (32)
+               +- Scan parquet default.store_returns (31)
+
+
+(1) Scan parquet default.customer
+Output [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk)]
+ReadSchema: struct<c_customer_id:string,c_current_cdemo_sk:int,c_current_hdemo_sk:int,c_current_addr_sk:int,c_first_name:string,c_last_name:string>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6]
+
+(3) Filter [codegen id : 5]
+Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6]
+Condition : ((isnotnull(c_current_addr_sk#4) AND isnotnull(c_current_cdemo_sk#2)) AND isnotnull(c_current_hdemo_sk#3))
+
+(4) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#7, ca_city#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_city), EqualTo(ca_city,Edgewood), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_city:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [ca_address_sk#7, ca_city#8]
+
+(6) Filter [codegen id : 1]
+Input [2]: [ca_address_sk#7, ca_city#8]
+Condition : ((isnotnull(ca_city#8) AND (ca_city#8 = Edgewood)) AND isnotnull(ca_address_sk#7))
+
+(7) Project [codegen id : 1]
+Output [1]: [ca_address_sk#7]
+Input [2]: [ca_address_sk#7, ca_city#8]
+
+(8) BroadcastExchange
+Input [1]: [ca_address_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#9]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [c_current_addr_sk#4]
+Right keys [1]: [ca_address_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [5]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6]
+Input [7]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6, ca_address_sk#7]
+
+(11) Scan parquet default.customer_demographics
+Output [1]: [cd_demo_sk#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [1]: [cd_demo_sk#10]
+
+(13) Filter [codegen id : 2]
+Input [1]: [cd_demo_sk#10]
+Condition : isnotnull(cd_demo_sk#10)
+
+(14) BroadcastExchange
+Input [1]: [cd_demo_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(15) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [c_current_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#10]
+Join condition: None
+
+(16) Project [codegen id : 5]
+Output [5]: [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#10]
+Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#10]
+
+(17) Scan parquet default.household_demographics
+Output [2]: [hd_demo_sk#12, hd_income_band_sk#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_income_band_sk:int>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [2]: [hd_demo_sk#12, hd_income_band_sk#13]
+
+(19) Filter [codegen id : 3]
+Input [2]: [hd_demo_sk#12, hd_income_band_sk#13]
+Condition : (isnotnull(hd_demo_sk#12) AND isnotnull(hd_income_band_sk#13))
+
+(20) BroadcastExchange
+Input [2]: [hd_demo_sk#12, hd_income_band_sk#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#14]
+
+(21) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [c_current_hdemo_sk#3]
+Right keys [1]: [hd_demo_sk#12]
+Join condition: None
+
+(22) Project [codegen id : 5]
+Output [5]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#10, hd_income_band_sk#13]
+Input [7]: [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#10, hd_demo_sk#12, hd_income_band_sk#13]
+
+(23) Scan parquet default.income_band
+Output [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/income_band]
+PushedFilters: [IsNotNull(ib_lower_bound), IsNotNull(ib_upper_bound), GreaterThanOrEqual(ib_lower_bound,38128), LessThanOrEqual(ib_upper_bound,88128), IsNotNull(ib_income_band_sk)]
+ReadSchema: struct<ib_income_band_sk:int,ib_lower_bound:int,ib_upper_bound:int>
+
+(24) ColumnarToRow [codegen id : 4]
+Input [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17]
+
+(25) Filter [codegen id : 4]
+Input [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17]
+Condition : ((((isnotnull(ib_lower_bound#16) AND isnotnull(ib_upper_bound#17)) AND (ib_lower_bound#16 >= 38128)) AND (ib_upper_bound#17 <= 88128)) AND isnotnull(ib_income_band_sk#15))
+
+(26) Project [codegen id : 4]
+Output [1]: [ib_income_band_sk#15]
+Input [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17]
+
+(27) BroadcastExchange
+Input [1]: [ib_income_band_sk#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(28) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [hd_income_band_sk#13]
+Right keys [1]: [ib_income_band_sk#15]
+Join condition: None
+
+(29) Project [codegen id : 5]
+Output [4]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#10]
+Input [6]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#10, hd_income_band_sk#13, ib_income_band_sk#15]
+
+(30) BroadcastExchange
+Input [4]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[3, int, true] as bigint)),false), [id=#19]
+
+(31) Scan parquet default.store_returns
+Output [2]: [sr_cdemo_sk#20, sr_returned_date_sk#21]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_cdemo_sk)]
+ReadSchema: struct<sr_cdemo_sk:int>
+
+(32) ColumnarToRow
+Input [2]: [sr_cdemo_sk#20, sr_returned_date_sk#21]
+
+(33) Filter
+Input [2]: [sr_cdemo_sk#20, sr_returned_date_sk#21]
+Condition : isnotnull(sr_cdemo_sk#20)
+
+(34) Project
+Output [1]: [sr_cdemo_sk#20]
+Input [2]: [sr_cdemo_sk#20, sr_returned_date_sk#21]
+
+(35) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [cd_demo_sk#10]
+Right keys [1]: [sr_cdemo_sk#20]
+Join condition: None
+
+(36) Project [codegen id : 6]
+Output [3]: [c_customer_id#1 AS customer_id#22, concat(c_last_name#6, , , c_first_name#5) AS customername#23, c_customer_id#1]
+Input [5]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#10, sr_cdemo_sk#20]
+
+(37) TakeOrderedAndProject
+Input [3]: [customer_id#22, customername#23, c_customer_id#1]
+Arguments: 100, [c_customer_id#1 ASC NULLS FIRST], [customer_id#22, customername#23]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/simplified.txt
new file mode 100644
index 0000000000000..f0116eb8d8df9
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/simplified.txt
@@ -0,0 +1,54 @@
+TakeOrderedAndProject [c_customer_id,customer_id,customername]
+  WholeStageCodegen (6)
+    Project [c_customer_id,c_last_name,c_first_name]
+      BroadcastHashJoin [cd_demo_sk,sr_cdemo_sk]
+        InputAdapter
+          BroadcastExchange #1
+            WholeStageCodegen (5)
+              Project [c_customer_id,c_first_name,c_last_name,cd_demo_sk]
+                BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk]
+                  Project [c_customer_id,c_first_name,c_last_name,cd_demo_sk,hd_income_band_sk]
+                    BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk]
+                      Project [c_customer_id,c_current_hdemo_sk,c_first_name,c_last_name,cd_demo_sk]
+                        BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk]
+                          Project [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_first_name,c_last_name]
+                            BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                              Filter [c_current_addr_sk,c_current_cdemo_sk,c_current_hdemo_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.customer [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name]
+                              InputAdapter
+                                BroadcastExchange #2
+                                  WholeStageCodegen (1)
+                                    Project [ca_address_sk]
+                                      Filter [ca_city,ca_address_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.customer_address [ca_address_sk,ca_city]
+                          InputAdapter
+                            BroadcastExchange #3
+                              WholeStageCodegen (2)
+                                Filter [cd_demo_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.customer_demographics [cd_demo_sk]
+                      InputAdapter
+                        BroadcastExchange #4
+                          WholeStageCodegen (3)
+                            Filter [hd_demo_sk,hd_income_band_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.household_demographics [hd_demo_sk,hd_income_band_sk]
+                  InputAdapter
+                    BroadcastExchange #5
+                      WholeStageCodegen (4)
+                        Project [ib_income_band_sk]
+                          Filter [ib_lower_bound,ib_upper_bound,ib_income_band_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.income_band [ib_income_band_sk,ib_lower_bound,ib_upper_bound]
+        Project [sr_cdemo_sk]
+          Filter [sr_cdemo_sk]
+            ColumnarToRow
+              InputAdapter
+                Scan parquet default.store_returns [sr_cdemo_sk,sr_returned_date_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.sf100/explain.txt
new file mode 100644
index 0000000000000..d002f987ff3fd
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.sf100/explain.txt
@@ -0,0 +1,333 @@
+== Physical Plan ==
+TakeOrderedAndProject (58)
++- * HashAggregate (57)
+   +- Exchange (56)
+      +- * HashAggregate (55)
+         +- * Project (54)
+            +- * BroadcastHashJoin Inner BuildRight (53)
+               :- * Project (48)
+               :  +- * BroadcastHashJoin Inner BuildRight (47)
+               :     :- * Project (41)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (40)
+               :     :     :- * Project (34)
+               :     :     :  +- * SortMergeJoin Inner (33)
+               :     :     :     :- * Sort (27)
+               :     :     :     :  +- Exchange (26)
+               :     :     :     :     +- * Project (25)
+               :     :     :     :        +- * BroadcastHashJoin Inner BuildRight (24)
+               :     :     :     :           :- * Project (19)
+               :     :     :     :           :  +- * SortMergeJoin Inner (18)
+               :     :     :     :           :     :- * Sort (11)
+               :     :     :     :           :     :  +- Exchange (10)
+               :     :     :     :           :     :     +- * Project (9)
+               :     :     :     :           :     :        +- * BroadcastHashJoin Inner BuildRight (8)
+               :     :     :     :           :     :           :- * Filter (3)
+               :     :     :     :           :     :           :  +- * ColumnarToRow (2)
+               :     :     :     :           :     :           :     +- Scan parquet default.web_sales (1)
+               :     :     :     :           :     :           +- BroadcastExchange (7)
+               :     :     :     :           :     :              +- * Filter (6)
+               :     :     :     :           :     :                 +- * ColumnarToRow (5)
+               :     :     :     :           :     :                    +- Scan parquet default.web_page (4)
+               :     :     :     :           :     +- * Sort (17)
+               :     :     :     :           :        +- Exchange (16)
+               :     :     :     :           :           +- * Project (15)
+               :     :     :     :           :              +- * Filter (14)
+               :     :     :     :           :                 +- * ColumnarToRow (13)
+               :     :     :     :           :                    +- Scan parquet default.web_returns (12)
+               :     :     :     :           +- BroadcastExchange (23)
+               :     :     :     :              +- * Filter (22)
+               :     :     :     :                 +- * ColumnarToRow (21)
+               :     :     :     :                    +- Scan parquet default.customer_demographics (20)
+               :     :     :     +- * Sort (32)
+               :     :     :        +- Exchange (31)
+               :     :     :           +- * Filter (30)
+               :     :     :              +- * ColumnarToRow (29)
+               :     :     :                 +- Scan parquet default.customer_demographics (28)
+               :     :     +- BroadcastExchange (39)
+               :     :        +- * Project (38)
+               :     :           +- * Filter (37)
+               :     :              +- * ColumnarToRow (36)
+               :     :                 +- Scan parquet default.customer_address (35)
+               :     +- BroadcastExchange (46)
+               :        +- * Project (45)
+               :           +- * Filter (44)
+               :              +- * ColumnarToRow (43)
+               :                 +- Scan parquet default.date_dim (42)
+               +- BroadcastExchange (52)
+                  +- * Filter (51)
+                     +- * ColumnarToRow (50)
+                        +- Scan parquet default.reason (49)
+
+
+(1) Scan parquet default.web_sales
+Output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#7), dynamicpruningexpression(ws_sold_date_sk#7 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_page_sk), Or(Or(And(GreaterThanOrEqual(ws_sales_price,100.00),LessThanOrEqual(ws_sales_price,150.00)),And(GreaterThanOrEqual(ws_sales_price,50.00),LessThanOrEqual(ws_sales_price,100.00))),And(GreaterThanOrEqual(ws_sales_price,150.00),LessThanOrEqual(ws_sales_price,200.00))), Or(Or(And(GreaterThanOrEqual(ws_net_profit,100.00),LessThanOrEqual(ws_net_profit,200.00)),And(GreaterThanOrEqual(ws_net_profit,150.00),LessThanOrEqual(ws_net_profit,300.00))),And(GreaterThanOrEqual(ws_net_profit,50.00),LessThanOrEqual(ws_net_profit,250.00)))]
+ReadSchema: struct<ws_item_sk:int,ws_web_page_sk:int,ws_order_number:int,ws_quantity:int,ws_sales_price:decimal(7,2),ws_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7]
+
+(3) Filter [codegen id : 2]
+Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7]
+Condition : ((((isnotnull(ws_item_sk#1) AND isnotnull(ws_order_number#3)) AND isnotnull(ws_web_page_sk#2)) AND ((((ws_sales_price#5 >= 100.00) AND (ws_sales_price#5 <= 150.00)) OR ((ws_sales_price#5 >= 50.00) AND (ws_sales_price#5 <= 100.00))) OR ((ws_sales_price#5 >= 150.00) AND (ws_sales_price#5 <= 200.00)))) AND ((((ws_net_profit#6 >= 100.00) AND (ws_net_profit#6 <= 200.00)) OR ((ws_net_profit#6 >= 150.00) AND (ws_net_profit#6 <= 300.00))) OR ((ws_net_profit#6 >= 50.00) AND (ws_net_profit#6 <= 250.00))))
+
+(4) Scan parquet default.web_page
+Output [1]: [wp_web_page_sk#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_page]
+PushedFilters: [IsNotNull(wp_web_page_sk)]
+ReadSchema: struct<wp_web_page_sk:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [1]: [wp_web_page_sk#9]
+
+(6) Filter [codegen id : 1]
+Input [1]: [wp_web_page_sk#9]
+Condition : isnotnull(wp_web_page_sk#9)
+
+(7) BroadcastExchange
+Input [1]: [wp_web_page_sk#9]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#10]
+
+(8) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ws_web_page_sk#2]
+Right keys [1]: [wp_web_page_sk#9]
+Join condition: None
+
+(9) Project [codegen id : 2]
+Output [6]: [ws_item_sk#1, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7]
+Input [8]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wp_web_page_sk#9]
+
+(10) Exchange
+Input [6]: [ws_item_sk#1, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7]
+Arguments: hashpartitioning(ws_item_sk#1, ws_order_number#3, 5), ENSURE_REQUIREMENTS, [id=#11]
+
+(11) Sort [codegen id : 3]
+Input [6]: [ws_item_sk#1, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7]
+Arguments: [ws_item_sk#1 ASC NULLS FIRST, ws_order_number#3 ASC NULLS FIRST], false, 0
+
+(12) Scan parquet default.web_returns
+Output [9]: [wr_item_sk#12, wr_refunded_cdemo_sk#13, wr_refunded_addr_sk#14, wr_returning_cdemo_sk#15, wr_reason_sk#16, wr_order_number#17, wr_fee#18, wr_refunded_cash#19, wr_returned_date_sk#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_returns]
+PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number), IsNotNull(wr_refunded_cdemo_sk), IsNotNull(wr_returning_cdemo_sk), IsNotNull(wr_refunded_addr_sk), IsNotNull(wr_reason_sk)]
+ReadSchema: struct<wr_item_sk:int,wr_refunded_cdemo_sk:int,wr_refunded_addr_sk:int,wr_returning_cdemo_sk:int,wr_reason_sk:int,wr_order_number:int,wr_fee:decimal(7,2),wr_refunded_cash:decimal(7,2)>
+
+(13) ColumnarToRow [codegen id : 4]
+Input [9]: [wr_item_sk#12, wr_refunded_cdemo_sk#13, wr_refunded_addr_sk#14, wr_returning_cdemo_sk#15, wr_reason_sk#16, wr_order_number#17, wr_fee#18, wr_refunded_cash#19, wr_returned_date_sk#20]
+
+(14) Filter [codegen id : 4]
+Input [9]: [wr_item_sk#12, wr_refunded_cdemo_sk#13, wr_refunded_addr_sk#14, wr_returning_cdemo_sk#15, wr_reason_sk#16, wr_order_number#17, wr_fee#18, wr_refunded_cash#19, wr_returned_date_sk#20]
+Condition : (((((isnotnull(wr_item_sk#12) AND isnotnull(wr_order_number#17)) AND isnotnull(wr_refunded_cdemo_sk#13)) AND isnotnull(wr_returning_cdemo_sk#15)) AND isnotnull(wr_refunded_addr_sk#14)) AND isnotnull(wr_reason_sk#16))
+
+(15) Project [codegen id : 4]
+Output [8]: [wr_item_sk#12, wr_refunded_cdemo_sk#13, wr_refunded_addr_sk#14, wr_returning_cdemo_sk#15, wr_reason_sk#16, wr_order_number#17, wr_fee#18, wr_refunded_cash#19]
+Input [9]: [wr_item_sk#12, wr_refunded_cdemo_sk#13, wr_refunded_addr_sk#14, wr_returning_cdemo_sk#15, wr_reason_sk#16, wr_order_number#17, wr_fee#18, wr_refunded_cash#19, wr_returned_date_sk#20]
+
+(16) Exchange
+Input [8]: [wr_item_sk#12, wr_refunded_cdemo_sk#13, wr_refunded_addr_sk#14, wr_returning_cdemo_sk#15, wr_reason_sk#16, wr_order_number#17, wr_fee#18, wr_refunded_cash#19]
+Arguments: hashpartitioning(wr_item_sk#12, wr_order_number#17, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(17) Sort [codegen id : 5]
+Input [8]: [wr_item_sk#12, wr_refunded_cdemo_sk#13, wr_refunded_addr_sk#14, wr_returning_cdemo_sk#15, wr_reason_sk#16, wr_order_number#17, wr_fee#18, wr_refunded_cash#19]
+Arguments: [wr_item_sk#12 ASC NULLS FIRST, wr_order_number#17 ASC NULLS FIRST], false, 0
+
+(18) SortMergeJoin [codegen id : 7]
+Left keys [2]: [ws_item_sk#1, ws_order_number#3]
+Right keys [2]: [wr_item_sk#12, wr_order_number#17]
+Join condition: None
+
+(19) Project [codegen id : 7]
+Output [10]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#13, wr_refunded_addr_sk#14, wr_returning_cdemo_sk#15, wr_reason_sk#16, wr_fee#18, wr_refunded_cash#19]
+Input [14]: [ws_item_sk#1, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_item_sk#12, wr_refunded_cdemo_sk#13, wr_refunded_addr_sk#14, wr_returning_cdemo_sk#15, wr_reason_sk#16, wr_order_number#17, wr_fee#18, wr_refunded_cash#19]
+
+(20) Scan parquet default.customer_demographics
+Output [3]: [cd_demo_sk#22, cd_marital_status#23, cd_education_status#24]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), Or(Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,Advanced Degree     )),And(EqualTo(cd_marital_status,S),EqualTo(cd_education_status,College             ))),And(EqualTo(cd_marital_status,W),EqualTo(cd_education_status,2 yr Degree         )))]
+ReadSchema: struct<cd_demo_sk:int,cd_marital_status:string,cd_education_status:string>
+
+(21) ColumnarToRow [codegen id : 6]
+Input [3]: [cd_demo_sk#22, cd_marital_status#23, cd_education_status#24]
+
+(22) Filter [codegen id : 6]
+Input [3]: [cd_demo_sk#22, cd_marital_status#23, cd_education_status#24]
+Condition : (((isnotnull(cd_demo_sk#22) AND isnotnull(cd_marital_status#23)) AND isnotnull(cd_education_status#24)) AND ((((cd_marital_status#23 = M) AND (cd_education_status#24 = Advanced Degree     )) OR ((cd_marital_status#23 = S) AND (cd_education_status#24 = College             ))) OR ((cd_marital_status#23 = W) AND (cd_education_status#24 = 2 yr Degree         ))))
+
+(23) BroadcastExchange
+Input [3]: [cd_demo_sk#22, cd_marital_status#23, cd_education_status#24]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#25]
+
+(24) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [wr_refunded_cdemo_sk#13]
+Right keys [1]: [cd_demo_sk#22]
+Join condition: ((((((cd_marital_status#23 = M) AND (cd_education_status#24 = Advanced Degree     )) AND (ws_sales_price#5 >= 100.00)) AND (ws_sales_price#5 <= 150.00)) OR ((((cd_marital_status#23 = S) AND (cd_education_status#24 = College             )) AND (ws_sales_price#5 >= 50.00)) AND (ws_sales_price#5 <= 100.00))) OR ((((cd_marital_status#23 = W) AND (cd_education_status#24 = 2 yr Degree         )) AND (ws_sales_price#5 >= 150.00)) AND (ws_sales_price#5 <= 200.00)))
+
+(25) Project [codegen id : 7]
+Output [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#14, wr_returning_cdemo_sk#15, wr_reason_sk#16, wr_fee#18, wr_refunded_cash#19, cd_marital_status#23, cd_education_status#24]
+Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#13, wr_refunded_addr_sk#14, wr_returning_cdemo_sk#15, wr_reason_sk#16, wr_fee#18, wr_refunded_cash#19, cd_demo_sk#22, cd_marital_status#23, cd_education_status#24]
+
+(26) Exchange
+Input [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#14, wr_returning_cdemo_sk#15, wr_reason_sk#16, wr_fee#18, wr_refunded_cash#19, cd_marital_status#23, cd_education_status#24]
+Arguments: hashpartitioning(wr_returning_cdemo_sk#15, cd_marital_status#23, cd_education_status#24, 5), ENSURE_REQUIREMENTS, [id=#26]
+
+(27) Sort [codegen id : 8]
+Input [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#14, wr_returning_cdemo_sk#15, wr_reason_sk#16, wr_fee#18, wr_refunded_cash#19, cd_marital_status#23, cd_education_status#24]
+Arguments: [wr_returning_cdemo_sk#15 ASC NULLS FIRST, cd_marital_status#23 ASC NULLS FIRST, cd_education_status#24 ASC NULLS FIRST], false, 0
+
+(28) Scan parquet default.customer_demographics
+Output [3]: [cd_demo_sk#27, cd_marital_status#28, cd_education_status#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status), IsNotNull(cd_education_status)]
+ReadSchema: struct<cd_demo_sk:int,cd_marital_status:string,cd_education_status:string>
+
+(29) ColumnarToRow [codegen id : 9]
+Input [3]: [cd_demo_sk#27, cd_marital_status#28, cd_education_status#29]
+
+(30) Filter [codegen id : 9]
+Input [3]: [cd_demo_sk#27, cd_marital_status#28, cd_education_status#29]
+Condition : ((isnotnull(cd_demo_sk#27) AND isnotnull(cd_marital_status#28)) AND isnotnull(cd_education_status#29))
+
+(31) Exchange
+Input [3]: [cd_demo_sk#27, cd_marital_status#28, cd_education_status#29]
+Arguments: hashpartitioning(cd_demo_sk#27, cd_marital_status#28, cd_education_status#29, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(32) Sort [codegen id : 10]
+Input [3]: [cd_demo_sk#27, cd_marital_status#28, cd_education_status#29]
+Arguments: [cd_demo_sk#27 ASC NULLS FIRST, cd_marital_status#28 ASC NULLS FIRST, cd_education_status#29 ASC NULLS FIRST], false, 0
+
+(33) SortMergeJoin [codegen id : 14]
+Left keys [3]: [wr_returning_cdemo_sk#15, cd_marital_status#23, cd_education_status#24]
+Right keys [3]: [cd_demo_sk#27, cd_marital_status#28, cd_education_status#29]
+Join condition: None
+
+(34) Project [codegen id : 14]
+Output [7]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#14, wr_reason_sk#16, wr_fee#18, wr_refunded_cash#19]
+Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#14, wr_returning_cdemo_sk#15, wr_reason_sk#16, wr_fee#18, wr_refunded_cash#19, cd_marital_status#23, cd_education_status#24, cd_demo_sk#27, cd_marital_status#28, cd_education_status#29]
+
+(35) Scan parquet default.customer_address
+Output [3]: [ca_address_sk#31, ca_state#32, ca_country#33]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk), Or(Or(In(ca_state, [IN,OH,NJ]),In(ca_state, [WI,CT,KY])),In(ca_state, [LA,IA,AR]))]
+ReadSchema: struct<ca_address_sk:int,ca_state:string,ca_country:string>
+
+(36) ColumnarToRow [codegen id : 11]
+Input [3]: [ca_address_sk#31, ca_state#32, ca_country#33]
+
+(37) Filter [codegen id : 11]
+Input [3]: [ca_address_sk#31, ca_state#32, ca_country#33]
+Condition : (((isnotnull(ca_country#33) AND (ca_country#33 = United States)) AND isnotnull(ca_address_sk#31)) AND ((ca_state#32 IN (IN,OH,NJ) OR ca_state#32 IN (WI,CT,KY)) OR ca_state#32 IN (LA,IA,AR)))
+
+(38) Project [codegen id : 11]
+Output [2]: [ca_address_sk#31, ca_state#32]
+Input [3]: [ca_address_sk#31, ca_state#32, ca_country#33]
+
+(39) BroadcastExchange
+Input [2]: [ca_address_sk#31, ca_state#32]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#34]
+
+(40) BroadcastHashJoin [codegen id : 14]
+Left keys [1]: [wr_refunded_addr_sk#14]
+Right keys [1]: [ca_address_sk#31]
+Join condition: ((((ca_state#32 IN (IN,OH,NJ) AND (ws_net_profit#6 >= 100.00)) AND (ws_net_profit#6 <= 200.00)) OR ((ca_state#32 IN (WI,CT,KY) AND (ws_net_profit#6 >= 150.00)) AND (ws_net_profit#6 <= 300.00))) OR ((ca_state#32 IN (LA,IA,AR) AND (ws_net_profit#6 >= 50.00)) AND (ws_net_profit#6 <= 250.00)))
+
+(41) Project [codegen id : 14]
+Output [5]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#16, wr_fee#18, wr_refunded_cash#19]
+Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#14, wr_reason_sk#16, wr_fee#18, wr_refunded_cash#19, ca_address_sk#31, ca_state#32]
+
+(42) Scan parquet default.date_dim
+Output [2]: [d_date_sk#35, d_year#36]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(43) ColumnarToRow [codegen id : 12]
+Input [2]: [d_date_sk#35, d_year#36]
+
+(44) Filter [codegen id : 12]
+Input [2]: [d_date_sk#35, d_year#36]
+Condition : ((isnotnull(d_year#36) AND (d_year#36 = 2000)) AND isnotnull(d_date_sk#35))
+
+(45) Project [codegen id : 12]
+Output [1]: [d_date_sk#35]
+Input [2]: [d_date_sk#35, d_year#36]
+
+(46) BroadcastExchange
+Input [1]: [d_date_sk#35]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#37]
+
+(47) BroadcastHashJoin [codegen id : 14]
+Left keys [1]: [ws_sold_date_sk#7]
+Right keys [1]: [d_date_sk#35]
+Join condition: None
+
+(48) Project [codegen id : 14]
+Output [4]: [ws_quantity#4, wr_reason_sk#16, wr_fee#18, wr_refunded_cash#19]
+Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#16, wr_fee#18, wr_refunded_cash#19, d_date_sk#35]
+
+(49) Scan parquet default.reason
+Output [2]: [r_reason_sk#38, r_reason_desc#39]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/reason]
+PushedFilters: [IsNotNull(r_reason_sk)]
+ReadSchema: struct<r_reason_sk:int,r_reason_desc:string>
+
+(50) ColumnarToRow [codegen id : 13]
+Input [2]: [r_reason_sk#38, r_reason_desc#39]
+
+(51) Filter [codegen id : 13]
+Input [2]: [r_reason_sk#38, r_reason_desc#39]
+Condition : isnotnull(r_reason_sk#38)
+
+(52) BroadcastExchange
+Input [2]: [r_reason_sk#38, r_reason_desc#39]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#40]
+
+(53) BroadcastHashJoin [codegen id : 14]
+Left keys [1]: [wr_reason_sk#16]
+Right keys [1]: [r_reason_sk#38]
+Join condition: None
+
+(54) Project [codegen id : 14]
+Output [4]: [ws_quantity#4, wr_fee#18, wr_refunded_cash#19, r_reason_desc#39]
+Input [6]: [ws_quantity#4, wr_reason_sk#16, wr_fee#18, wr_refunded_cash#19, r_reason_sk#38, r_reason_desc#39]
+
+(55) HashAggregate [codegen id : 14]
+Input [4]: [ws_quantity#4, wr_fee#18, wr_refunded_cash#19, r_reason_desc#39]
+Keys [1]: [r_reason_desc#39]
+Functions [3]: [partial_avg(ws_quantity#4), partial_avg(UnscaledValue(wr_refunded_cash#19)), partial_avg(UnscaledValue(wr_fee#18))]
+Aggregate Attributes [6]: [sum#41, count#42, sum#43, count#44, sum#45, count#46]
+Results [7]: [r_reason_desc#39, sum#47, count#48, sum#49, count#50, sum#51, count#52]
+
+(56) Exchange
+Input [7]: [r_reason_desc#39, sum#47, count#48, sum#49, count#50, sum#51, count#52]
+Arguments: hashpartitioning(r_reason_desc#39, 5), ENSURE_REQUIREMENTS, [id=#53]
+
+(57) HashAggregate [codegen id : 15]
+Input [7]: [r_reason_desc#39, sum#47, count#48, sum#49, count#50, sum#51, count#52]
+Keys [1]: [r_reason_desc#39]
+Functions [3]: [avg(ws_quantity#4), avg(UnscaledValue(wr_refunded_cash#19)), avg(UnscaledValue(wr_fee#18))]
+Aggregate Attributes [3]: [avg(ws_quantity#4)#54, avg(UnscaledValue(wr_refunded_cash#19))#55, avg(UnscaledValue(wr_fee#18))#56]
+Results [4]: [substr(r_reason_desc#39, 1, 20) AS substr(r_reason_desc, 1, 20)#57, avg(ws_quantity#4)#54 AS avg(ws_quantity)#58, cast((avg(UnscaledValue(wr_refunded_cash#19))#55 / 100.0) as decimal(11,6)) AS avg(wr_refunded_cash)#59, cast((avg(UnscaledValue(wr_fee#18))#56 / 100.0) as decimal(11,6)) AS avg(wr_fee)#60]
+
+(58) TakeOrderedAndProject
+Input [4]: [substr(r_reason_desc, 1, 20)#57, avg(ws_quantity)#58, avg(wr_refunded_cash)#59, avg(wr_fee)#60]
+Arguments: 100, [substr(r_reason_desc, 1, 20)#57 ASC NULLS FIRST, avg(ws_quantity)#58 ASC NULLS FIRST, avg(wr_refunded_cash)#59 ASC NULLS FIRST, avg(wr_fee)#60 ASC NULLS FIRST], [substr(r_reason_desc, 1, 20)#57, avg(ws_quantity)#58, avg(wr_refunded_cash)#59, avg(wr_fee)#60]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (59)
+
+
+(59) ReusedExchange [Reuses operator id: 46]
+Output [1]: [d_date_sk#35]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.sf100/simplified.txt
new file mode 100644
index 0000000000000..135c097574bd2
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.sf100/simplified.txt
@@ -0,0 +1,97 @@
+TakeOrderedAndProject [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee)]
+  WholeStageCodegen (15)
+    HashAggregate [r_reason_desc,sum,count,sum,count,sum,count] [avg(ws_quantity),avg(UnscaledValue(wr_refunded_cash)),avg(UnscaledValue(wr_fee)),substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee),sum,count,sum,count,sum,count]
+      InputAdapter
+        Exchange [r_reason_desc] #1
+          WholeStageCodegen (14)
+            HashAggregate [r_reason_desc,ws_quantity,wr_refunded_cash,wr_fee] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+              Project [ws_quantity,wr_fee,wr_refunded_cash,r_reason_desc]
+                BroadcastHashJoin [wr_reason_sk,r_reason_sk]
+                  Project [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash]
+                    BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                      Project [ws_quantity,ws_sold_date_sk,wr_reason_sk,wr_fee,wr_refunded_cash]
+                        BroadcastHashJoin [wr_refunded_addr_sk,ca_address_sk,ca_state,ws_net_profit]
+                          Project [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_reason_sk,wr_fee,wr_refunded_cash]
+                            SortMergeJoin [wr_returning_cdemo_sk,cd_marital_status,cd_education_status,cd_demo_sk,cd_marital_status,cd_education_status]
+                              InputAdapter
+                                WholeStageCodegen (8)
+                                  Sort [wr_returning_cdemo_sk,cd_marital_status,cd_education_status]
+                                    InputAdapter
+                                      Exchange [wr_returning_cdemo_sk,cd_marital_status,cd_education_status] #2
+                                        WholeStageCodegen (7)
+                                          Project [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_marital_status,cd_education_status]
+                                            BroadcastHashJoin [wr_refunded_cdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status,ws_sales_price]
+                                              Project [ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash]
+                                                SortMergeJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number]
+                                                  InputAdapter
+                                                    WholeStageCodegen (3)
+                                                      Sort [ws_item_sk,ws_order_number]
+                                                        InputAdapter
+                                                          Exchange [ws_item_sk,ws_order_number] #3
+                                                            WholeStageCodegen (2)
+                                                              Project [ws_item_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk]
+                                                                BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk]
+                                                                  Filter [ws_item_sk,ws_order_number,ws_web_page_sk,ws_sales_price,ws_net_profit]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.web_sales [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk]
+                                                                          SubqueryBroadcast [d_date_sk] #1
+                                                                            ReusedExchange [d_date_sk] #4
+                                                                  InputAdapter
+                                                                    BroadcastExchange #5
+                                                                      WholeStageCodegen (1)
+                                                                        Filter [wp_web_page_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.web_page [wp_web_page_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (5)
+                                                      Sort [wr_item_sk,wr_order_number]
+                                                        InputAdapter
+                                                          Exchange [wr_item_sk,wr_order_number] #6
+                                                            WholeStageCodegen (4)
+                                                              Project [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash]
+                                                                Filter [wr_item_sk,wr_order_number,wr_refunded_cdemo_sk,wr_returning_cdemo_sk,wr_refunded_addr_sk,wr_reason_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk]
+                                              InputAdapter
+                                                BroadcastExchange #7
+                                                  WholeStageCodegen (6)
+                                                    Filter [cd_demo_sk,cd_marital_status,cd_education_status]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status]
+                              InputAdapter
+                                WholeStageCodegen (10)
+                                  Sort [cd_demo_sk,cd_marital_status,cd_education_status]
+                                    InputAdapter
+                                      Exchange [cd_demo_sk,cd_marital_status,cd_education_status] #8
+                                        WholeStageCodegen (9)
+                                          Filter [cd_demo_sk,cd_marital_status,cd_education_status]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status]
+                          InputAdapter
+                            BroadcastExchange #9
+                              WholeStageCodegen (11)
+                                Project [ca_address_sk,ca_state]
+                                  Filter [ca_country,ca_address_sk,ca_state]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.customer_address [ca_address_sk,ca_state,ca_country]
+                      InputAdapter
+                        BroadcastExchange #4
+                          WholeStageCodegen (12)
+                            Project [d_date_sk]
+                              Filter [d_year,d_date_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.date_dim [d_date_sk,d_year]
+                  InputAdapter
+                    BroadcastExchange #10
+                      WholeStageCodegen (13)
+                        Filter [r_reason_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.reason [r_reason_sk,r_reason_desc]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/explain.txt
new file mode 100644
index 0000000000000..b244c6f3810c6
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/explain.txt
@@ -0,0 +1,303 @@
+== Physical Plan ==
+TakeOrderedAndProject (52)
++- * HashAggregate (51)
+   +- Exchange (50)
+      +- * HashAggregate (49)
+         +- * Project (48)
+            +- * BroadcastHashJoin Inner BuildRight (47)
+               :- * Project (42)
+               :  +- * BroadcastHashJoin Inner BuildRight (41)
+               :     :- * Project (35)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (34)
+               :     :     :- * Project (28)
+               :     :     :  +- * BroadcastHashJoin Inner BuildRight (27)
+               :     :     :     :- * Project (22)
+               :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (21)
+               :     :     :     :     :- * Project (16)
+               :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (15)
+               :     :     :     :     :     :- * Project (10)
+               :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildLeft (9)
+               :     :     :     :     :     :     :- BroadcastExchange (4)
+               :     :     :     :     :     :     :  +- * Filter (3)
+               :     :     :     :     :     :     :     +- * ColumnarToRow (2)
+               :     :     :     :     :     :     :        +- Scan parquet default.web_sales (1)
+               :     :     :     :     :     :     +- * Project (8)
+               :     :     :     :     :     :        +- * Filter (7)
+               :     :     :     :     :     :           +- * ColumnarToRow (6)
+               :     :     :     :     :     :              +- Scan parquet default.web_returns (5)
+               :     :     :     :     :     +- BroadcastExchange (14)
+               :     :     :     :     :        +- * Filter (13)
+               :     :     :     :     :           +- * ColumnarToRow (12)
+               :     :     :     :     :              +- Scan parquet default.web_page (11)
+               :     :     :     :     +- BroadcastExchange (20)
+               :     :     :     :        +- * Filter (19)
+               :     :     :     :           +- * ColumnarToRow (18)
+               :     :     :     :              +- Scan parquet default.customer_demographics (17)
+               :     :     :     +- BroadcastExchange (26)
+               :     :     :        +- * Filter (25)
+               :     :     :           +- * ColumnarToRow (24)
+               :     :     :              +- Scan parquet default.customer_demographics (23)
+               :     :     +- BroadcastExchange (33)
+               :     :        +- * Project (32)
+               :     :           +- * Filter (31)
+               :     :              +- * ColumnarToRow (30)
+               :     :                 +- Scan parquet default.customer_address (29)
+               :     +- BroadcastExchange (40)
+               :        +- * Project (39)
+               :           +- * Filter (38)
+               :              +- * ColumnarToRow (37)
+               :                 +- Scan parquet default.date_dim (36)
+               +- BroadcastExchange (46)
+                  +- * Filter (45)
+                     +- * ColumnarToRow (44)
+                        +- Scan parquet default.reason (43)
+
+
+(1) Scan parquet default.web_sales
+Output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#7), dynamicpruningexpression(ws_sold_date_sk#7 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_page_sk), Or(Or(And(GreaterThanOrEqual(ws_sales_price,100.00),LessThanOrEqual(ws_sales_price,150.00)),And(GreaterThanOrEqual(ws_sales_price,50.00),LessThanOrEqual(ws_sales_price,100.00))),And(GreaterThanOrEqual(ws_sales_price,150.00),LessThanOrEqual(ws_sales_price,200.00))), Or(Or(And(GreaterThanOrEqual(ws_net_profit,100.00),LessThanOrEqual(ws_net_profit,200.00)),And(GreaterThanOrEqual(ws_net_profit,150.00),LessThanOrEqual(ws_net_profit,300.00))),And(GreaterThanOrEqual(ws_net_profit,50.00),LessThanOrEqual(ws_net_profit,250.00)))]
+ReadSchema: struct<ws_item_sk:int,ws_web_page_sk:int,ws_order_number:int,ws_quantity:int,ws_sales_price:decimal(7,2),ws_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7]
+
+(3) Filter [codegen id : 1]
+Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7]
+Condition : ((((isnotnull(ws_item_sk#1) AND isnotnull(ws_order_number#3)) AND isnotnull(ws_web_page_sk#2)) AND ((((ws_sales_price#5 >= 100.00) AND (ws_sales_price#5 <= 150.00)) OR ((ws_sales_price#5 >= 50.00) AND (ws_sales_price#5 <= 100.00))) OR ((ws_sales_price#5 >= 150.00) AND (ws_sales_price#5 <= 200.00)))) AND ((((ws_net_profit#6 >= 100.00) AND (ws_net_profit#6 <= 200.00)) OR ((ws_net_profit#6 >= 150.00) AND (ws_net_profit#6 <= 300.00))) OR ((ws_net_profit#6 >= 50.00) AND (ws_net_profit#6 <= 250.00))))
+
+(4) BroadcastExchange
+Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[2, int, false] as bigint) & 4294967295))),false), [id=#9]
+
+(5) Scan parquet default.web_returns
+Output [9]: [wr_item_sk#10, wr_refunded_cdemo_sk#11, wr_refunded_addr_sk#12, wr_returning_cdemo_sk#13, wr_reason_sk#14, wr_order_number#15, wr_fee#16, wr_refunded_cash#17, wr_returned_date_sk#18]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_returns]
+PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number), IsNotNull(wr_refunded_cdemo_sk), IsNotNull(wr_returning_cdemo_sk), IsNotNull(wr_refunded_addr_sk), IsNotNull(wr_reason_sk)]
+ReadSchema: struct<wr_item_sk:int,wr_refunded_cdemo_sk:int,wr_refunded_addr_sk:int,wr_returning_cdemo_sk:int,wr_reason_sk:int,wr_order_number:int,wr_fee:decimal(7,2),wr_refunded_cash:decimal(7,2)>
+
+(6) ColumnarToRow
+Input [9]: [wr_item_sk#10, wr_refunded_cdemo_sk#11, wr_refunded_addr_sk#12, wr_returning_cdemo_sk#13, wr_reason_sk#14, wr_order_number#15, wr_fee#16, wr_refunded_cash#17, wr_returned_date_sk#18]
+
+(7) Filter
+Input [9]: [wr_item_sk#10, wr_refunded_cdemo_sk#11, wr_refunded_addr_sk#12, wr_returning_cdemo_sk#13, wr_reason_sk#14, wr_order_number#15, wr_fee#16, wr_refunded_cash#17, wr_returned_date_sk#18]
+Condition : (((((isnotnull(wr_item_sk#10) AND isnotnull(wr_order_number#15)) AND isnotnull(wr_refunded_cdemo_sk#11)) AND isnotnull(wr_returning_cdemo_sk#13)) AND isnotnull(wr_refunded_addr_sk#12)) AND isnotnull(wr_reason_sk#14))
+
+(8) Project
+Output [8]: [wr_item_sk#10, wr_refunded_cdemo_sk#11, wr_refunded_addr_sk#12, wr_returning_cdemo_sk#13, wr_reason_sk#14, wr_order_number#15, wr_fee#16, wr_refunded_cash#17]
+Input [9]: [wr_item_sk#10, wr_refunded_cdemo_sk#11, wr_refunded_addr_sk#12, wr_returning_cdemo_sk#13, wr_reason_sk#14, wr_order_number#15, wr_fee#16, wr_refunded_cash#17, wr_returned_date_sk#18]
+
+(9) BroadcastHashJoin [codegen id : 8]
+Left keys [2]: [ws_item_sk#1, ws_order_number#3]
+Right keys [2]: [wr_item_sk#10, wr_order_number#15]
+Join condition: None
+
+(10) Project [codegen id : 8]
+Output [11]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#11, wr_refunded_addr_sk#12, wr_returning_cdemo_sk#13, wr_reason_sk#14, wr_fee#16, wr_refunded_cash#17]
+Input [15]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_item_sk#10, wr_refunded_cdemo_sk#11, wr_refunded_addr_sk#12, wr_returning_cdemo_sk#13, wr_reason_sk#14, wr_order_number#15, wr_fee#16, wr_refunded_cash#17]
+
+(11) Scan parquet default.web_page
+Output [1]: [wp_web_page_sk#19]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_page]
+PushedFilters: [IsNotNull(wp_web_page_sk)]
+ReadSchema: struct<wp_web_page_sk:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [1]: [wp_web_page_sk#19]
+
+(13) Filter [codegen id : 2]
+Input [1]: [wp_web_page_sk#19]
+Condition : isnotnull(wp_web_page_sk#19)
+
+(14) BroadcastExchange
+Input [1]: [wp_web_page_sk#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#20]
+
+(15) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ws_web_page_sk#2]
+Right keys [1]: [wp_web_page_sk#19]
+Join condition: None
+
+(16) Project [codegen id : 8]
+Output [10]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#11, wr_refunded_addr_sk#12, wr_returning_cdemo_sk#13, wr_reason_sk#14, wr_fee#16, wr_refunded_cash#17]
+Input [12]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#11, wr_refunded_addr_sk#12, wr_returning_cdemo_sk#13, wr_reason_sk#14, wr_fee#16, wr_refunded_cash#17, wp_web_page_sk#19]
+
+(17) Scan parquet default.customer_demographics
+Output [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), Or(Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,Advanced Degree     )),And(EqualTo(cd_marital_status,S),EqualTo(cd_education_status,College             ))),And(EqualTo(cd_marital_status,W),EqualTo(cd_education_status,2 yr Degree         )))]
+ReadSchema: struct<cd_demo_sk:int,cd_marital_status:string,cd_education_status:string>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23]
+
+(19) Filter [codegen id : 3]
+Input [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23]
+Condition : (((isnotnull(cd_demo_sk#21) AND isnotnull(cd_marital_status#22)) AND isnotnull(cd_education_status#23)) AND ((((cd_marital_status#22 = M) AND (cd_education_status#23 = Advanced Degree     )) OR ((cd_marital_status#22 = S) AND (cd_education_status#23 = College             ))) OR ((cd_marital_status#22 = W) AND (cd_education_status#23 = 2 yr Degree         ))))
+
+(20) BroadcastExchange
+Input [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#24]
+
+(21) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [wr_refunded_cdemo_sk#11]
+Right keys [1]: [cd_demo_sk#21]
+Join condition: ((((((cd_marital_status#22 = M) AND (cd_education_status#23 = Advanced Degree     )) AND (ws_sales_price#5 >= 100.00)) AND (ws_sales_price#5 <= 150.00)) OR ((((cd_marital_status#22 = S) AND (cd_education_status#23 = College             )) AND (ws_sales_price#5 >= 50.00)) AND (ws_sales_price#5 <= 100.00))) OR ((((cd_marital_status#22 = W) AND (cd_education_status#23 = 2 yr Degree         )) AND (ws_sales_price#5 >= 150.00)) AND (ws_sales_price#5 <= 200.00)))
+
+(22) Project [codegen id : 8]
+Output [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#12, wr_returning_cdemo_sk#13, wr_reason_sk#14, wr_fee#16, wr_refunded_cash#17, cd_marital_status#22, cd_education_status#23]
+Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#11, wr_refunded_addr_sk#12, wr_returning_cdemo_sk#13, wr_reason_sk#14, wr_fee#16, wr_refunded_cash#17, cd_demo_sk#21, cd_marital_status#22, cd_education_status#23]
+
+(23) Scan parquet default.customer_demographics
+Output [3]: [cd_demo_sk#25, cd_marital_status#26, cd_education_status#27]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status), IsNotNull(cd_education_status)]
+ReadSchema: struct<cd_demo_sk:int,cd_marital_status:string,cd_education_status:string>
+
+(24) ColumnarToRow [codegen id : 4]
+Input [3]: [cd_demo_sk#25, cd_marital_status#26, cd_education_status#27]
+
+(25) Filter [codegen id : 4]
+Input [3]: [cd_demo_sk#25, cd_marital_status#26, cd_education_status#27]
+Condition : ((isnotnull(cd_demo_sk#25) AND isnotnull(cd_marital_status#26)) AND isnotnull(cd_education_status#27))
+
+(26) BroadcastExchange
+Input [3]: [cd_demo_sk#25, cd_marital_status#26, cd_education_status#27]
+Arguments: HashedRelationBroadcastMode(List(input[0, int, false], input[1, string, false], input[2, string, false]),false), [id=#28]
+
+(27) BroadcastHashJoin [codegen id : 8]
+Left keys [3]: [wr_returning_cdemo_sk#13, cd_marital_status#22, cd_education_status#23]
+Right keys [3]: [cd_demo_sk#25, cd_marital_status#26, cd_education_status#27]
+Join condition: None
+
+(28) Project [codegen id : 8]
+Output [7]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#12, wr_reason_sk#14, wr_fee#16, wr_refunded_cash#17]
+Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#12, wr_returning_cdemo_sk#13, wr_reason_sk#14, wr_fee#16, wr_refunded_cash#17, cd_marital_status#22, cd_education_status#23, cd_demo_sk#25, cd_marital_status#26, cd_education_status#27]
+
+(29) Scan parquet default.customer_address
+Output [3]: [ca_address_sk#29, ca_state#30, ca_country#31]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk), Or(Or(In(ca_state, [IN,OH,NJ]),In(ca_state, [WI,CT,KY])),In(ca_state, [LA,IA,AR]))]
+ReadSchema: struct<ca_address_sk:int,ca_state:string,ca_country:string>
+
+(30) ColumnarToRow [codegen id : 5]
+Input [3]: [ca_address_sk#29, ca_state#30, ca_country#31]
+
+(31) Filter [codegen id : 5]
+Input [3]: [ca_address_sk#29, ca_state#30, ca_country#31]
+Condition : (((isnotnull(ca_country#31) AND (ca_country#31 = United States)) AND isnotnull(ca_address_sk#29)) AND ((ca_state#30 IN (IN,OH,NJ) OR ca_state#30 IN (WI,CT,KY)) OR ca_state#30 IN (LA,IA,AR)))
+
+(32) Project [codegen id : 5]
+Output [2]: [ca_address_sk#29, ca_state#30]
+Input [3]: [ca_address_sk#29, ca_state#30, ca_country#31]
+
+(33) BroadcastExchange
+Input [2]: [ca_address_sk#29, ca_state#30]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#32]
+
+(34) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [wr_refunded_addr_sk#12]
+Right keys [1]: [ca_address_sk#29]
+Join condition: ((((ca_state#30 IN (IN,OH,NJ) AND (ws_net_profit#6 >= 100.00)) AND (ws_net_profit#6 <= 200.00)) OR ((ca_state#30 IN (WI,CT,KY) AND (ws_net_profit#6 >= 150.00)) AND (ws_net_profit#6 <= 300.00))) OR ((ca_state#30 IN (LA,IA,AR) AND (ws_net_profit#6 >= 50.00)) AND (ws_net_profit#6 <= 250.00)))
+
+(35) Project [codegen id : 8]
+Output [5]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#14, wr_fee#16, wr_refunded_cash#17]
+Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#12, wr_reason_sk#14, wr_fee#16, wr_refunded_cash#17, ca_address_sk#29, ca_state#30]
+
+(36) Scan parquet default.date_dim
+Output [2]: [d_date_sk#33, d_year#34]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(37) ColumnarToRow [codegen id : 6]
+Input [2]: [d_date_sk#33, d_year#34]
+
+(38) Filter [codegen id : 6]
+Input [2]: [d_date_sk#33, d_year#34]
+Condition : ((isnotnull(d_year#34) AND (d_year#34 = 2000)) AND isnotnull(d_date_sk#33))
+
+(39) Project [codegen id : 6]
+Output [1]: [d_date_sk#33]
+Input [2]: [d_date_sk#33, d_year#34]
+
+(40) BroadcastExchange
+Input [1]: [d_date_sk#33]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#35]
+
+(41) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ws_sold_date_sk#7]
+Right keys [1]: [d_date_sk#33]
+Join condition: None
+
+(42) Project [codegen id : 8]
+Output [4]: [ws_quantity#4, wr_reason_sk#14, wr_fee#16, wr_refunded_cash#17]
+Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#14, wr_fee#16, wr_refunded_cash#17, d_date_sk#33]
+
+(43) Scan parquet default.reason
+Output [2]: [r_reason_sk#36, r_reason_desc#37]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/reason]
+PushedFilters: [IsNotNull(r_reason_sk)]
+ReadSchema: struct<r_reason_sk:int,r_reason_desc:string>
+
+(44) ColumnarToRow [codegen id : 7]
+Input [2]: [r_reason_sk#36, r_reason_desc#37]
+
+(45) Filter [codegen id : 7]
+Input [2]: [r_reason_sk#36, r_reason_desc#37]
+Condition : isnotnull(r_reason_sk#36)
+
+(46) BroadcastExchange
+Input [2]: [r_reason_sk#36, r_reason_desc#37]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#38]
+
+(47) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [wr_reason_sk#14]
+Right keys [1]: [r_reason_sk#36]
+Join condition: None
+
+(48) Project [codegen id : 8]
+Output [4]: [ws_quantity#4, wr_fee#16, wr_refunded_cash#17, r_reason_desc#37]
+Input [6]: [ws_quantity#4, wr_reason_sk#14, wr_fee#16, wr_refunded_cash#17, r_reason_sk#36, r_reason_desc#37]
+
+(49) HashAggregate [codegen id : 8]
+Input [4]: [ws_quantity#4, wr_fee#16, wr_refunded_cash#17, r_reason_desc#37]
+Keys [1]: [r_reason_desc#37]
+Functions [3]: [partial_avg(ws_quantity#4), partial_avg(UnscaledValue(wr_refunded_cash#17)), partial_avg(UnscaledValue(wr_fee#16))]
+Aggregate Attributes [6]: [sum#39, count#40, sum#41, count#42, sum#43, count#44]
+Results [7]: [r_reason_desc#37, sum#45, count#46, sum#47, count#48, sum#49, count#50]
+
+(50) Exchange
+Input [7]: [r_reason_desc#37, sum#45, count#46, sum#47, count#48, sum#49, count#50]
+Arguments: hashpartitioning(r_reason_desc#37, 5), ENSURE_REQUIREMENTS, [id=#51]
+
+(51) HashAggregate [codegen id : 9]
+Input [7]: [r_reason_desc#37, sum#45, count#46, sum#47, count#48, sum#49, count#50]
+Keys [1]: [r_reason_desc#37]
+Functions [3]: [avg(ws_quantity#4), avg(UnscaledValue(wr_refunded_cash#17)), avg(UnscaledValue(wr_fee#16))]
+Aggregate Attributes [3]: [avg(ws_quantity#4)#52, avg(UnscaledValue(wr_refunded_cash#17))#53, avg(UnscaledValue(wr_fee#16))#54]
+Results [4]: [substr(r_reason_desc#37, 1, 20) AS substr(r_reason_desc, 1, 20)#55, avg(ws_quantity#4)#52 AS avg(ws_quantity)#56, cast((avg(UnscaledValue(wr_refunded_cash#17))#53 / 100.0) as decimal(11,6)) AS avg(wr_refunded_cash)#57, cast((avg(UnscaledValue(wr_fee#16))#54 / 100.0) as decimal(11,6)) AS avg(wr_fee)#58]
+
+(52) TakeOrderedAndProject
+Input [4]: [substr(r_reason_desc, 1, 20)#55, avg(ws_quantity)#56, avg(wr_refunded_cash)#57, avg(wr_fee)#58]
+Arguments: 100, [substr(r_reason_desc, 1, 20)#55 ASC NULLS FIRST, avg(ws_quantity)#56 ASC NULLS FIRST, avg(wr_refunded_cash)#57 ASC NULLS FIRST, avg(wr_fee)#58 ASC NULLS FIRST], [substr(r_reason_desc, 1, 20)#55, avg(ws_quantity)#56, avg(wr_refunded_cash)#57, avg(wr_fee)#58]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (53)
+
+
+(53) ReusedExchange [Reuses operator id: 40]
+Output [1]: [d_date_sk#33]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/simplified.txt
new file mode 100644
index 0000000000000..932ed71fbd57d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/simplified.txt
@@ -0,0 +1,79 @@
+TakeOrderedAndProject [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee)]
+  WholeStageCodegen (9)
+    HashAggregate [r_reason_desc,sum,count,sum,count,sum,count] [avg(ws_quantity),avg(UnscaledValue(wr_refunded_cash)),avg(UnscaledValue(wr_fee)),substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee),sum,count,sum,count,sum,count]
+      InputAdapter
+        Exchange [r_reason_desc] #1
+          WholeStageCodegen (8)
+            HashAggregate [r_reason_desc,ws_quantity,wr_refunded_cash,wr_fee] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+              Project [ws_quantity,wr_fee,wr_refunded_cash,r_reason_desc]
+                BroadcastHashJoin [wr_reason_sk,r_reason_sk]
+                  Project [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash]
+                    BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                      Project [ws_quantity,ws_sold_date_sk,wr_reason_sk,wr_fee,wr_refunded_cash]
+                        BroadcastHashJoin [wr_refunded_addr_sk,ca_address_sk,ca_state,ws_net_profit]
+                          Project [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_reason_sk,wr_fee,wr_refunded_cash]
+                            BroadcastHashJoin [wr_returning_cdemo_sk,cd_marital_status,cd_education_status,cd_demo_sk,cd_marital_status,cd_education_status]
+                              Project [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_marital_status,cd_education_status]
+                                BroadcastHashJoin [wr_refunded_cdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status,ws_sales_price]
+                                  Project [ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash]
+                                    BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk]
+                                      Project [ws_web_page_sk,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash]
+                                        BroadcastHashJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number]
+                                          InputAdapter
+                                            BroadcastExchange #2
+                                              WholeStageCodegen (1)
+                                                Filter [ws_item_sk,ws_order_number,ws_web_page_sk,ws_sales_price,ws_net_profit]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.web_sales [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk]
+                                                        SubqueryBroadcast [d_date_sk] #1
+                                                          ReusedExchange [d_date_sk] #3
+                                          Project [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash]
+                                            Filter [wr_item_sk,wr_order_number,wr_refunded_cdemo_sk,wr_returning_cdemo_sk,wr_refunded_addr_sk,wr_reason_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk]
+                                      InputAdapter
+                                        BroadcastExchange #4
+                                          WholeStageCodegen (2)
+                                            Filter [wp_web_page_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.web_page [wp_web_page_sk]
+                                  InputAdapter
+                                    BroadcastExchange #5
+                                      WholeStageCodegen (3)
+                                        Filter [cd_demo_sk,cd_marital_status,cd_education_status]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status]
+                              InputAdapter
+                                BroadcastExchange #6
+                                  WholeStageCodegen (4)
+                                    Filter [cd_demo_sk,cd_marital_status,cd_education_status]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status]
+                          InputAdapter
+                            BroadcastExchange #7
+                              WholeStageCodegen (5)
+                                Project [ca_address_sk,ca_state]
+                                  Filter [ca_country,ca_address_sk,ca_state]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.customer_address [ca_address_sk,ca_state,ca_country]
+                      InputAdapter
+                        BroadcastExchange #3
+                          WholeStageCodegen (6)
+                            Project [d_date_sk]
+                              Filter [d_year,d_date_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.date_dim [d_date_sk,d_year]
+                  InputAdapter
+                    BroadcastExchange #8
+                      WholeStageCodegen (7)
+                        Filter [r_reason_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.reason [r_reason_sk,r_reason_desc]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.sf100/explain.txt
new file mode 100644
index 0000000000000..447a40326a9c0
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.sf100/explain.txt
@@ -0,0 +1,153 @@
+== Physical Plan ==
+TakeOrderedAndProject (25)
++- * Project (24)
+   +- Window (23)
+      +- * Sort (22)
+         +- Exchange (21)
+            +- * HashAggregate (20)
+               +- Exchange (19)
+                  +- * HashAggregate (18)
+                     +- * Expand (17)
+                        +- * Project (16)
+                           +- * BroadcastHashJoin Inner BuildRight (15)
+                              :- * Project (10)
+                              :  +- * BroadcastHashJoin Inner BuildRight (9)
+                              :     :- * Filter (3)
+                              :     :  +- * ColumnarToRow (2)
+                              :     :     +- Scan parquet default.web_sales (1)
+                              :     +- BroadcastExchange (8)
+                              :        +- * Project (7)
+                              :           +- * Filter (6)
+                              :              +- * ColumnarToRow (5)
+                              :                 +- Scan parquet default.date_dim (4)
+                              +- BroadcastExchange (14)
+                                 +- * Filter (13)
+                                    +- * ColumnarToRow (12)
+                                       +- Scan parquet default.item (11)
+
+
+(1) Scan parquet default.web_sales
+Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_net_paid:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3]
+Condition : isnotnull(ws_item_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#5, d_month_seq#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#5]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ws_sold_date_sk#3]
+Right keys [1]: [d_date_sk#5]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [2]: [ws_item_sk#1, ws_net_paid#2]
+Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#5]
+
+(11) Scan parquet default.item
+Output [3]: [i_item_sk#8, i_class#9, i_category#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_class:string,i_category:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [i_item_sk#8, i_class#9, i_category#10]
+
+(13) Filter [codegen id : 2]
+Input [3]: [i_item_sk#8, i_class#9, i_category#10]
+Condition : isnotnull(i_item_sk#8)
+
+(14) BroadcastExchange
+Input [3]: [i_item_sk#8, i_class#9, i_category#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ws_item_sk#1]
+Right keys [1]: [i_item_sk#8]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [3]: [ws_net_paid#2, i_category#10, i_class#9]
+Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#8, i_class#9, i_category#10]
+
+(17) Expand [codegen id : 3]
+Input [3]: [ws_net_paid#2, i_category#10, i_class#9]
+Arguments: [List(ws_net_paid#2, i_category#10, i_class#9, 0), List(ws_net_paid#2, i_category#10, null, 1), List(ws_net_paid#2, null, null, 3)], [ws_net_paid#2, i_category#12, i_class#13, spark_grouping_id#14]
+
+(18) HashAggregate [codegen id : 3]
+Input [4]: [ws_net_paid#2, i_category#12, i_class#13, spark_grouping_id#14]
+Keys [3]: [i_category#12, i_class#13, spark_grouping_id#14]
+Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))]
+Aggregate Attributes [1]: [sum#15]
+Results [4]: [i_category#12, i_class#13, spark_grouping_id#14, sum#16]
+
+(19) Exchange
+Input [4]: [i_category#12, i_class#13, spark_grouping_id#14, sum#16]
+Arguments: hashpartitioning(i_category#12, i_class#13, spark_grouping_id#14, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(20) HashAggregate [codegen id : 4]
+Input [4]: [i_category#12, i_class#13, spark_grouping_id#14, sum#16]
+Keys [3]: [i_category#12, i_class#13, spark_grouping_id#14]
+Functions [1]: [sum(UnscaledValue(ws_net_paid#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#18]
+Results [7]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#18,17,2) AS total_sum#19, i_category#12, i_class#13, (cast((shiftright(spark_grouping_id#14, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#14, 0) & 1) as tinyint)) AS lochierarchy#20, (cast((shiftright(spark_grouping_id#14, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#14, 0) & 1) as tinyint)) AS _w1#21, CASE WHEN (cast((shiftright(spark_grouping_id#14, 0) & 1) as tinyint) = 0) THEN i_category#12 END AS _w2#22, MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#18,17,2) AS _w3#23]
+
+(21) Exchange
+Input [7]: [total_sum#19, i_category#12, i_class#13, lochierarchy#20, _w1#21, _w2#22, _w3#23]
+Arguments: hashpartitioning(_w1#21, _w2#22, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(22) Sort [codegen id : 5]
+Input [7]: [total_sum#19, i_category#12, i_class#13, lochierarchy#20, _w1#21, _w2#22, _w3#23]
+Arguments: [_w1#21 ASC NULLS FIRST, _w2#22 ASC NULLS FIRST, _w3#23 DESC NULLS LAST], false, 0
+
+(23) Window
+Input [7]: [total_sum#19, i_category#12, i_class#13, lochierarchy#20, _w1#21, _w2#22, _w3#23]
+Arguments: [rank(_w3#23) windowspecdefinition(_w1#21, _w2#22, _w3#23 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#25], [_w1#21, _w2#22], [_w3#23 DESC NULLS LAST]
+
+(24) Project [codegen id : 6]
+Output [5]: [total_sum#19, i_category#12, i_class#13, lochierarchy#20, rank_within_parent#25]
+Input [8]: [total_sum#19, i_category#12, i_class#13, lochierarchy#20, _w1#21, _w2#22, _w3#23, rank_within_parent#25]
+
+(25) TakeOrderedAndProject
+Input [5]: [total_sum#19, i_category#12, i_class#13, lochierarchy#20, rank_within_parent#25]
+Arguments: 100, [lochierarchy#20 DESC NULLS LAST, CASE WHEN (lochierarchy#20 = 0) THEN i_category#12 END ASC NULLS FIRST, rank_within_parent#25 ASC NULLS FIRST], [total_sum#19, i_category#12, i_class#13, lochierarchy#20, rank_within_parent#25]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (26)
+
+
+(26) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#5]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.sf100/simplified.txt
new file mode 100644
index 0000000000000..428ebafa39063
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.sf100/simplified.txt
@@ -0,0 +1,41 @@
+TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_class]
+  WholeStageCodegen (6)
+    Project [total_sum,i_category,i_class,lochierarchy,rank_within_parent]
+      InputAdapter
+        Window [_w3,_w1,_w2]
+          WholeStageCodegen (5)
+            Sort [_w1,_w2,_w3]
+              InputAdapter
+                Exchange [_w1,_w2] #1
+                  WholeStageCodegen (4)
+                    HashAggregate [i_category,i_class,spark_grouping_id,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,lochierarchy,_w1,_w2,_w3,sum]
+                      InputAdapter
+                        Exchange [i_category,i_class,spark_grouping_id] #2
+                          WholeStageCodegen (3)
+                            HashAggregate [i_category,i_class,spark_grouping_id,ws_net_paid] [sum,sum]
+                              Expand [ws_net_paid,i_category,i_class]
+                                Project [ws_net_paid,i_category,i_class]
+                                  BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                    Project [ws_item_sk,ws_net_paid]
+                                      BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                        Filter [ws_item_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk]
+                                                SubqueryBroadcast [d_date_sk] #1
+                                                  ReusedExchange [d_date_sk] #3
+                                        InputAdapter
+                                          BroadcastExchange #3
+                                            WholeStageCodegen (1)
+                                              Project [d_date_sk]
+                                                Filter [d_month_seq,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_month_seq]
+                                    InputAdapter
+                                      BroadcastExchange #4
+                                        WholeStageCodegen (2)
+                                          Filter [i_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.item [i_item_sk,i_class,i_category]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/explain.txt
new file mode 100644
index 0000000000000..447a40326a9c0
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/explain.txt
@@ -0,0 +1,153 @@
+== Physical Plan ==
+TakeOrderedAndProject (25)
++- * Project (24)
+   +- Window (23)
+      +- * Sort (22)
+         +- Exchange (21)
+            +- * HashAggregate (20)
+               +- Exchange (19)
+                  +- * HashAggregate (18)
+                     +- * Expand (17)
+                        +- * Project (16)
+                           +- * BroadcastHashJoin Inner BuildRight (15)
+                              :- * Project (10)
+                              :  +- * BroadcastHashJoin Inner BuildRight (9)
+                              :     :- * Filter (3)
+                              :     :  +- * ColumnarToRow (2)
+                              :     :     +- Scan parquet default.web_sales (1)
+                              :     +- BroadcastExchange (8)
+                              :        +- * Project (7)
+                              :           +- * Filter (6)
+                              :              +- * ColumnarToRow (5)
+                              :                 +- Scan parquet default.date_dim (4)
+                              +- BroadcastExchange (14)
+                                 +- * Filter (13)
+                                    +- * ColumnarToRow (12)
+                                       +- Scan parquet default.item (11)
+
+
+(1) Scan parquet default.web_sales
+Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_net_paid:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3]
+Condition : isnotnull(ws_item_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#5, d_month_seq#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#5]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ws_sold_date_sk#3]
+Right keys [1]: [d_date_sk#5]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [2]: [ws_item_sk#1, ws_net_paid#2]
+Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#5]
+
+(11) Scan parquet default.item
+Output [3]: [i_item_sk#8, i_class#9, i_category#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_class:string,i_category:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [i_item_sk#8, i_class#9, i_category#10]
+
+(13) Filter [codegen id : 2]
+Input [3]: [i_item_sk#8, i_class#9, i_category#10]
+Condition : isnotnull(i_item_sk#8)
+
+(14) BroadcastExchange
+Input [3]: [i_item_sk#8, i_class#9, i_category#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ws_item_sk#1]
+Right keys [1]: [i_item_sk#8]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [3]: [ws_net_paid#2, i_category#10, i_class#9]
+Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#8, i_class#9, i_category#10]
+
+(17) Expand [codegen id : 3]
+Input [3]: [ws_net_paid#2, i_category#10, i_class#9]
+Arguments: [List(ws_net_paid#2, i_category#10, i_class#9, 0), List(ws_net_paid#2, i_category#10, null, 1), List(ws_net_paid#2, null, null, 3)], [ws_net_paid#2, i_category#12, i_class#13, spark_grouping_id#14]
+
+(18) HashAggregate [codegen id : 3]
+Input [4]: [ws_net_paid#2, i_category#12, i_class#13, spark_grouping_id#14]
+Keys [3]: [i_category#12, i_class#13, spark_grouping_id#14]
+Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))]
+Aggregate Attributes [1]: [sum#15]
+Results [4]: [i_category#12, i_class#13, spark_grouping_id#14, sum#16]
+
+(19) Exchange
+Input [4]: [i_category#12, i_class#13, spark_grouping_id#14, sum#16]
+Arguments: hashpartitioning(i_category#12, i_class#13, spark_grouping_id#14, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(20) HashAggregate [codegen id : 4]
+Input [4]: [i_category#12, i_class#13, spark_grouping_id#14, sum#16]
+Keys [3]: [i_category#12, i_class#13, spark_grouping_id#14]
+Functions [1]: [sum(UnscaledValue(ws_net_paid#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#18]
+Results [7]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#18,17,2) AS total_sum#19, i_category#12, i_class#13, (cast((shiftright(spark_grouping_id#14, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#14, 0) & 1) as tinyint)) AS lochierarchy#20, (cast((shiftright(spark_grouping_id#14, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#14, 0) & 1) as tinyint)) AS _w1#21, CASE WHEN (cast((shiftright(spark_grouping_id#14, 0) & 1) as tinyint) = 0) THEN i_category#12 END AS _w2#22, MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#18,17,2) AS _w3#23]
+
+(21) Exchange
+Input [7]: [total_sum#19, i_category#12, i_class#13, lochierarchy#20, _w1#21, _w2#22, _w3#23]
+Arguments: hashpartitioning(_w1#21, _w2#22, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(22) Sort [codegen id : 5]
+Input [7]: [total_sum#19, i_category#12, i_class#13, lochierarchy#20, _w1#21, _w2#22, _w3#23]
+Arguments: [_w1#21 ASC NULLS FIRST, _w2#22 ASC NULLS FIRST, _w3#23 DESC NULLS LAST], false, 0
+
+(23) Window
+Input [7]: [total_sum#19, i_category#12, i_class#13, lochierarchy#20, _w1#21, _w2#22, _w3#23]
+Arguments: [rank(_w3#23) windowspecdefinition(_w1#21, _w2#22, _w3#23 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#25], [_w1#21, _w2#22], [_w3#23 DESC NULLS LAST]
+
+(24) Project [codegen id : 6]
+Output [5]: [total_sum#19, i_category#12, i_class#13, lochierarchy#20, rank_within_parent#25]
+Input [8]: [total_sum#19, i_category#12, i_class#13, lochierarchy#20, _w1#21, _w2#22, _w3#23, rank_within_parent#25]
+
+(25) TakeOrderedAndProject
+Input [5]: [total_sum#19, i_category#12, i_class#13, lochierarchy#20, rank_within_parent#25]
+Arguments: 100, [lochierarchy#20 DESC NULLS LAST, CASE WHEN (lochierarchy#20 = 0) THEN i_category#12 END ASC NULLS FIRST, rank_within_parent#25 ASC NULLS FIRST], [total_sum#19, i_category#12, i_class#13, lochierarchy#20, rank_within_parent#25]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (26)
+
+
+(26) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#5]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/simplified.txt
new file mode 100644
index 0000000000000..428ebafa39063
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/simplified.txt
@@ -0,0 +1,41 @@
+TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_class]
+  WholeStageCodegen (6)
+    Project [total_sum,i_category,i_class,lochierarchy,rank_within_parent]
+      InputAdapter
+        Window [_w3,_w1,_w2]
+          WholeStageCodegen (5)
+            Sort [_w1,_w2,_w3]
+              InputAdapter
+                Exchange [_w1,_w2] #1
+                  WholeStageCodegen (4)
+                    HashAggregate [i_category,i_class,spark_grouping_id,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,lochierarchy,_w1,_w2,_w3,sum]
+                      InputAdapter
+                        Exchange [i_category,i_class,spark_grouping_id] #2
+                          WholeStageCodegen (3)
+                            HashAggregate [i_category,i_class,spark_grouping_id,ws_net_paid] [sum,sum]
+                              Expand [ws_net_paid,i_category,i_class]
+                                Project [ws_net_paid,i_category,i_class]
+                                  BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                    Project [ws_item_sk,ws_net_paid]
+                                      BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                        Filter [ws_item_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk]
+                                                SubqueryBroadcast [d_date_sk] #1
+                                                  ReusedExchange [d_date_sk] #3
+                                        InputAdapter
+                                          BroadcastExchange #3
+                                            WholeStageCodegen (1)
+                                              Project [d_date_sk]
+                                                Filter [d_month_seq,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_month_seq]
+                                    InputAdapter
+                                      BroadcastExchange #4
+                                        WholeStageCodegen (2)
+                                          Filter [i_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.item [i_item_sk,i_class,i_category]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.sf100/explain.txt
new file mode 100644
index 0000000000000..92895cb566fd2
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.sf100/explain.txt
@@ -0,0 +1,425 @@
+== Physical Plan ==
+* HashAggregate (71)
++- Exchange (70)
+   +- * HashAggregate (69)
+      +- * HashAggregate (68)
+         +- Exchange (67)
+            +- * HashAggregate (66)
+               +- SortMergeJoin LeftAnti (65)
+                  :- * Sort (47)
+                  :  +- Exchange (46)
+                  :     +- * HashAggregate (45)
+                  :        +- Exchange (44)
+                  :           +- * HashAggregate (43)
+                  :              +- SortMergeJoin LeftAnti (42)
+                  :                 :- * Sort (24)
+                  :                 :  +- Exchange (23)
+                  :                 :     +- * HashAggregate (22)
+                  :                 :        +- Exchange (21)
+                  :                 :           +- * HashAggregate (20)
+                  :                 :              +- * Project (19)
+                  :                 :                 +- * SortMergeJoin Inner (18)
+                  :                 :                    :- * Sort (12)
+                  :                 :                    :  +- Exchange (11)
+                  :                 :                    :     +- * Project (10)
+                  :                 :                    :        +- * BroadcastHashJoin Inner BuildRight (9)
+                  :                 :                    :           :- * Filter (3)
+                  :                 :                    :           :  +- * ColumnarToRow (2)
+                  :                 :                    :           :     +- Scan parquet default.store_sales (1)
+                  :                 :                    :           +- BroadcastExchange (8)
+                  :                 :                    :              +- * Project (7)
+                  :                 :                    :                 +- * Filter (6)
+                  :                 :                    :                    +- * ColumnarToRow (5)
+                  :                 :                    :                       +- Scan parquet default.date_dim (4)
+                  :                 :                    +- * Sort (17)
+                  :                 :                       +- Exchange (16)
+                  :                 :                          +- * Filter (15)
+                  :                 :                             +- * ColumnarToRow (14)
+                  :                 :                                +- Scan parquet default.customer (13)
+                  :                 +- * Sort (41)
+                  :                    +- Exchange (40)
+                  :                       +- * HashAggregate (39)
+                  :                          +- Exchange (38)
+                  :                             +- * HashAggregate (37)
+                  :                                +- * Project (36)
+                  :                                   +- * SortMergeJoin Inner (35)
+                  :                                      :- * Sort (32)
+                  :                                      :  +- Exchange (31)
+                  :                                      :     +- * Project (30)
+                  :                                      :        +- * BroadcastHashJoin Inner BuildRight (29)
+                  :                                      :           :- * Filter (27)
+                  :                                      :           :  +- * ColumnarToRow (26)
+                  :                                      :           :     +- Scan parquet default.catalog_sales (25)
+                  :                                      :           +- ReusedExchange (28)
+                  :                                      +- * Sort (34)
+                  :                                         +- ReusedExchange (33)
+                  +- * Sort (64)
+                     +- Exchange (63)
+                        +- * HashAggregate (62)
+                           +- Exchange (61)
+                              +- * HashAggregate (60)
+                                 +- * Project (59)
+                                    +- * SortMergeJoin Inner (58)
+                                       :- * Sort (55)
+                                       :  +- Exchange (54)
+                                       :     +- * Project (53)
+                                       :        +- * BroadcastHashJoin Inner BuildRight (52)
+                                       :           :- * Filter (50)
+                                       :           :  +- * ColumnarToRow (49)
+                                       :           :     +- Scan parquet default.web_sales (48)
+                                       :           +- ReusedExchange (51)
+                                       +- * Sort (57)
+                                          +- ReusedExchange (56)
+
+
+(1) Scan parquet default.store_sales
+Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#2), dynamicpruningexpression(ss_sold_date_sk#2 IN dynamicpruning#3)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2]
+
+(3) Filter [codegen id : 2]
+Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2]
+Condition : isnotnull(ss_customer_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date,d_month_seq:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6]
+Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4))
+
+(7) Project [codegen id : 1]
+Output [2]: [d_date_sk#4, d_date#5]
+Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6]
+
+(8) BroadcastExchange
+Input [2]: [d_date_sk#4, d_date#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#2]
+Right keys [1]: [d_date_sk#4]
+Join condition: None
+
+(10) Project [codegen id : 2]
+Output [2]: [ss_customer_sk#1, d_date#5]
+Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#4, d_date#5]
+
+(11) Exchange
+Input [2]: [ss_customer_sk#1, d_date#5]
+Arguments: hashpartitioning(ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#8]
+
+(12) Sort [codegen id : 3]
+Input [2]: [ss_customer_sk#1, d_date#5]
+Arguments: [ss_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(13) Scan parquet default.customer
+Output [3]: [c_customer_sk#9, c_first_name#10, c_last_name#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_first_name:string,c_last_name:string>
+
+(14) ColumnarToRow [codegen id : 4]
+Input [3]: [c_customer_sk#9, c_first_name#10, c_last_name#11]
+
+(15) Filter [codegen id : 4]
+Input [3]: [c_customer_sk#9, c_first_name#10, c_last_name#11]
+Condition : isnotnull(c_customer_sk#9)
+
+(16) Exchange
+Input [3]: [c_customer_sk#9, c_first_name#10, c_last_name#11]
+Arguments: hashpartitioning(c_customer_sk#9, 5), ENSURE_REQUIREMENTS, [id=#12]
+
+(17) Sort [codegen id : 5]
+Input [3]: [c_customer_sk#9, c_first_name#10, c_last_name#11]
+Arguments: [c_customer_sk#9 ASC NULLS FIRST], false, 0
+
+(18) SortMergeJoin [codegen id : 6]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#9]
+Join condition: None
+
+(19) Project [codegen id : 6]
+Output [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Input [5]: [ss_customer_sk#1, d_date#5, c_customer_sk#9, c_first_name#10, c_last_name#11]
+
+(20) HashAggregate [codegen id : 6]
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Keys [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#11, c_first_name#10, d_date#5]
+
+(21) Exchange
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Arguments: hashpartitioning(c_last_name#11, c_first_name#10, d_date#5, 5), ENSURE_REQUIREMENTS, [id=#13]
+
+(22) HashAggregate [codegen id : 7]
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Keys [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#11, c_first_name#10, d_date#5]
+
+(23) Exchange
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Arguments: hashpartitioning(coalesce(c_last_name#11, ), isnull(c_last_name#11), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#5, 1970-01-01), isnull(d_date#5), 5), ENSURE_REQUIREMENTS, [id=#14]
+
+(24) Sort [codegen id : 8]
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Arguments: [coalesce(c_last_name#11, ) ASC NULLS FIRST, isnull(c_last_name#11) ASC NULLS FIRST, coalesce(c_first_name#10, ) ASC NULLS FIRST, isnull(c_first_name#10) ASC NULLS FIRST, coalesce(d_date#5, 1970-01-01) ASC NULLS FIRST, isnull(d_date#5) ASC NULLS FIRST], false, 0
+
+(25) Scan parquet default.catalog_sales
+Output [2]: [cs_bill_customer_sk#15, cs_sold_date_sk#16]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#16), dynamicpruningexpression(cs_sold_date_sk#16 IN dynamicpruning#3)]
+PushedFilters: [IsNotNull(cs_bill_customer_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int>
+
+(26) ColumnarToRow [codegen id : 10]
+Input [2]: [cs_bill_customer_sk#15, cs_sold_date_sk#16]
+
+(27) Filter [codegen id : 10]
+Input [2]: [cs_bill_customer_sk#15, cs_sold_date_sk#16]
+Condition : isnotnull(cs_bill_customer_sk#15)
+
+(28) ReusedExchange [Reuses operator id: 8]
+Output [2]: [d_date_sk#17, d_date#18]
+
+(29) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_sold_date_sk#16]
+Right keys [1]: [d_date_sk#17]
+Join condition: None
+
+(30) Project [codegen id : 10]
+Output [2]: [cs_bill_customer_sk#15, d_date#18]
+Input [4]: [cs_bill_customer_sk#15, cs_sold_date_sk#16, d_date_sk#17, d_date#18]
+
+(31) Exchange
+Input [2]: [cs_bill_customer_sk#15, d_date#18]
+Arguments: hashpartitioning(cs_bill_customer_sk#15, 5), ENSURE_REQUIREMENTS, [id=#19]
+
+(32) Sort [codegen id : 11]
+Input [2]: [cs_bill_customer_sk#15, d_date#18]
+Arguments: [cs_bill_customer_sk#15 ASC NULLS FIRST], false, 0
+
+(33) ReusedExchange [Reuses operator id: 16]
+Output [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22]
+
+(34) Sort [codegen id : 13]
+Input [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22]
+Arguments: [c_customer_sk#20 ASC NULLS FIRST], false, 0
+
+(35) SortMergeJoin [codegen id : 14]
+Left keys [1]: [cs_bill_customer_sk#15]
+Right keys [1]: [c_customer_sk#20]
+Join condition: None
+
+(36) Project [codegen id : 14]
+Output [3]: [c_last_name#22, c_first_name#21, d_date#18]
+Input [5]: [cs_bill_customer_sk#15, d_date#18, c_customer_sk#20, c_first_name#21, c_last_name#22]
+
+(37) HashAggregate [codegen id : 14]
+Input [3]: [c_last_name#22, c_first_name#21, d_date#18]
+Keys [3]: [c_last_name#22, c_first_name#21, d_date#18]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#22, c_first_name#21, d_date#18]
+
+(38) Exchange
+Input [3]: [c_last_name#22, c_first_name#21, d_date#18]
+Arguments: hashpartitioning(c_last_name#22, c_first_name#21, d_date#18, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(39) HashAggregate [codegen id : 15]
+Input [3]: [c_last_name#22, c_first_name#21, d_date#18]
+Keys [3]: [c_last_name#22, c_first_name#21, d_date#18]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#22, c_first_name#21, d_date#18]
+
+(40) Exchange
+Input [3]: [c_last_name#22, c_first_name#21, d_date#18]
+Arguments: hashpartitioning(coalesce(c_last_name#22, ), isnull(c_last_name#22), coalesce(c_first_name#21, ), isnull(c_first_name#21), coalesce(d_date#18, 1970-01-01), isnull(d_date#18), 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(41) Sort [codegen id : 16]
+Input [3]: [c_last_name#22, c_first_name#21, d_date#18]
+Arguments: [coalesce(c_last_name#22, ) ASC NULLS FIRST, isnull(c_last_name#22) ASC NULLS FIRST, coalesce(c_first_name#21, ) ASC NULLS FIRST, isnull(c_first_name#21) ASC NULLS FIRST, coalesce(d_date#18, 1970-01-01) ASC NULLS FIRST, isnull(d_date#18) ASC NULLS FIRST], false, 0
+
+(42) SortMergeJoin
+Left keys [6]: [coalesce(c_last_name#11, ), isnull(c_last_name#11), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#5, 1970-01-01), isnull(d_date#5)]
+Right keys [6]: [coalesce(c_last_name#22, ), isnull(c_last_name#22), coalesce(c_first_name#21, ), isnull(c_first_name#21), coalesce(d_date#18, 1970-01-01), isnull(d_date#18)]
+Join condition: None
+
+(43) HashAggregate [codegen id : 17]
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Keys [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#11, c_first_name#10, d_date#5]
+
+(44) Exchange
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Arguments: hashpartitioning(c_last_name#11, c_first_name#10, d_date#5, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(45) HashAggregate [codegen id : 18]
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Keys [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#11, c_first_name#10, d_date#5]
+
+(46) Exchange
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Arguments: hashpartitioning(coalesce(c_last_name#11, ), isnull(c_last_name#11), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#5, 1970-01-01), isnull(d_date#5), 5), ENSURE_REQUIREMENTS, [id=#26]
+
+(47) Sort [codegen id : 19]
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Arguments: [coalesce(c_last_name#11, ) ASC NULLS FIRST, isnull(c_last_name#11) ASC NULLS FIRST, coalesce(c_first_name#10, ) ASC NULLS FIRST, isnull(c_first_name#10) ASC NULLS FIRST, coalesce(d_date#5, 1970-01-01) ASC NULLS FIRST, isnull(d_date#5) ASC NULLS FIRST], false, 0
+
+(48) Scan parquet default.web_sales
+Output [2]: [ws_bill_customer_sk#27, ws_sold_date_sk#28]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#28), dynamicpruningexpression(ws_sold_date_sk#28 IN dynamicpruning#3)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int>
+
+(49) ColumnarToRow [codegen id : 21]
+Input [2]: [ws_bill_customer_sk#27, ws_sold_date_sk#28]
+
+(50) Filter [codegen id : 21]
+Input [2]: [ws_bill_customer_sk#27, ws_sold_date_sk#28]
+Condition : isnotnull(ws_bill_customer_sk#27)
+
+(51) ReusedExchange [Reuses operator id: 8]
+Output [2]: [d_date_sk#29, d_date#30]
+
+(52) BroadcastHashJoin [codegen id : 21]
+Left keys [1]: [ws_sold_date_sk#28]
+Right keys [1]: [d_date_sk#29]
+Join condition: None
+
+(53) Project [codegen id : 21]
+Output [2]: [ws_bill_customer_sk#27, d_date#30]
+Input [4]: [ws_bill_customer_sk#27, ws_sold_date_sk#28, d_date_sk#29, d_date#30]
+
+(54) Exchange
+Input [2]: [ws_bill_customer_sk#27, d_date#30]
+Arguments: hashpartitioning(ws_bill_customer_sk#27, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(55) Sort [codegen id : 22]
+Input [2]: [ws_bill_customer_sk#27, d_date#30]
+Arguments: [ws_bill_customer_sk#27 ASC NULLS FIRST], false, 0
+
+(56) ReusedExchange [Reuses operator id: 16]
+Output [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34]
+
+(57) Sort [codegen id : 24]
+Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34]
+Arguments: [c_customer_sk#32 ASC NULLS FIRST], false, 0
+
+(58) SortMergeJoin [codegen id : 25]
+Left keys [1]: [ws_bill_customer_sk#27]
+Right keys [1]: [c_customer_sk#32]
+Join condition: None
+
+(59) Project [codegen id : 25]
+Output [3]: [c_last_name#34, c_first_name#33, d_date#30]
+Input [5]: [ws_bill_customer_sk#27, d_date#30, c_customer_sk#32, c_first_name#33, c_last_name#34]
+
+(60) HashAggregate [codegen id : 25]
+Input [3]: [c_last_name#34, c_first_name#33, d_date#30]
+Keys [3]: [c_last_name#34, c_first_name#33, d_date#30]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#34, c_first_name#33, d_date#30]
+
+(61) Exchange
+Input [3]: [c_last_name#34, c_first_name#33, d_date#30]
+Arguments: hashpartitioning(c_last_name#34, c_first_name#33, d_date#30, 5), ENSURE_REQUIREMENTS, [id=#35]
+
+(62) HashAggregate [codegen id : 26]
+Input [3]: [c_last_name#34, c_first_name#33, d_date#30]
+Keys [3]: [c_last_name#34, c_first_name#33, d_date#30]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#34, c_first_name#33, d_date#30]
+
+(63) Exchange
+Input [3]: [c_last_name#34, c_first_name#33, d_date#30]
+Arguments: hashpartitioning(coalesce(c_last_name#34, ), isnull(c_last_name#34), coalesce(c_first_name#33, ), isnull(c_first_name#33), coalesce(d_date#30, 1970-01-01), isnull(d_date#30), 5), ENSURE_REQUIREMENTS, [id=#36]
+
+(64) Sort [codegen id : 27]
+Input [3]: [c_last_name#34, c_first_name#33, d_date#30]
+Arguments: [coalesce(c_last_name#34, ) ASC NULLS FIRST, isnull(c_last_name#34) ASC NULLS FIRST, coalesce(c_first_name#33, ) ASC NULLS FIRST, isnull(c_first_name#33) ASC NULLS FIRST, coalesce(d_date#30, 1970-01-01) ASC NULLS FIRST, isnull(d_date#30) ASC NULLS FIRST], false, 0
+
+(65) SortMergeJoin
+Left keys [6]: [coalesce(c_last_name#11, ), isnull(c_last_name#11), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#5, 1970-01-01), isnull(d_date#5)]
+Right keys [6]: [coalesce(c_last_name#34, ), isnull(c_last_name#34), coalesce(c_first_name#33, ), isnull(c_first_name#33), coalesce(d_date#30, 1970-01-01), isnull(d_date#30)]
+Join condition: None
+
+(66) HashAggregate [codegen id : 28]
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Keys [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#11, c_first_name#10, d_date#5]
+
+(67) Exchange
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Arguments: hashpartitioning(c_last_name#11, c_first_name#10, d_date#5, 5), ENSURE_REQUIREMENTS, [id=#37]
+
+(68) HashAggregate [codegen id : 29]
+Input [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Keys [3]: [c_last_name#11, c_first_name#10, d_date#5]
+Functions: []
+Aggregate Attributes: []
+Results: []
+
+(69) HashAggregate [codegen id : 29]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#38]
+Results [1]: [count#39]
+
+(70) Exchange
+Input [1]: [count#39]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#40]
+
+(71) HashAggregate [codegen id : 30]
+Input [1]: [count#39]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#41]
+Results [1]: [count(1)#41 AS count(1)#42]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#2 IN dynamicpruning#3
+ReusedExchange (72)
+
+
+(72) ReusedExchange [Reuses operator id: 8]
+Output [2]: [d_date_sk#4, d_date#5]
+
+Subquery:2 Hosting operator id = 25 Hosting Expression = cs_sold_date_sk#16 IN dynamicpruning#3
+
+Subquery:3 Hosting operator id = 48 Hosting Expression = ws_sold_date_sk#28 IN dynamicpruning#3
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.sf100/simplified.txt
new file mode 100644
index 0000000000000..2978f51532d83
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.sf100/simplified.txt
@@ -0,0 +1,133 @@
+WholeStageCodegen (30)
+  HashAggregate [count] [count(1),count(1),count]
+    InputAdapter
+      Exchange #1
+        WholeStageCodegen (29)
+          HashAggregate [count,count]
+            HashAggregate [c_last_name,c_first_name,d_date]
+              InputAdapter
+                Exchange [c_last_name,c_first_name,d_date] #2
+                  WholeStageCodegen (28)
+                    HashAggregate [c_last_name,c_first_name,d_date]
+                      InputAdapter
+                        SortMergeJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date]
+                          WholeStageCodegen (19)
+                            Sort [c_last_name,c_first_name,d_date]
+                              InputAdapter
+                                Exchange [c_last_name,c_first_name,d_date] #3
+                                  WholeStageCodegen (18)
+                                    HashAggregate [c_last_name,c_first_name,d_date]
+                                      InputAdapter
+                                        Exchange [c_last_name,c_first_name,d_date] #4
+                                          WholeStageCodegen (17)
+                                            HashAggregate [c_last_name,c_first_name,d_date]
+                                              InputAdapter
+                                                SortMergeJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date]
+                                                  WholeStageCodegen (8)
+                                                    Sort [c_last_name,c_first_name,d_date]
+                                                      InputAdapter
+                                                        Exchange [c_last_name,c_first_name,d_date] #5
+                                                          WholeStageCodegen (7)
+                                                            HashAggregate [c_last_name,c_first_name,d_date]
+                                                              InputAdapter
+                                                                Exchange [c_last_name,c_first_name,d_date] #6
+                                                                  WholeStageCodegen (6)
+                                                                    HashAggregate [c_last_name,c_first_name,d_date]
+                                                                      Project [c_last_name,c_first_name,d_date]
+                                                                        SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                                                          InputAdapter
+                                                                            WholeStageCodegen (3)
+                                                                              Sort [ss_customer_sk]
+                                                                                InputAdapter
+                                                                                  Exchange [ss_customer_sk] #7
+                                                                                    WholeStageCodegen (2)
+                                                                                      Project [ss_customer_sk,d_date]
+                                                                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                          Filter [ss_customer_sk]
+                                                                                            ColumnarToRow
+                                                                                              InputAdapter
+                                                                                                Scan parquet default.store_sales [ss_customer_sk,ss_sold_date_sk]
+                                                                                                  SubqueryBroadcast [d_date_sk] #1
+                                                                                                    ReusedExchange [d_date_sk,d_date] #8
+                                                                                          InputAdapter
+                                                                                            BroadcastExchange #8
+                                                                                              WholeStageCodegen (1)
+                                                                                                Project [d_date_sk,d_date]
+                                                                                                  Filter [d_month_seq,d_date_sk]
+                                                                                                    ColumnarToRow
+                                                                                                      InputAdapter
+                                                                                                        Scan parquet default.date_dim [d_date_sk,d_date,d_month_seq]
+                                                                          InputAdapter
+                                                                            WholeStageCodegen (5)
+                                                                              Sort [c_customer_sk]
+                                                                                InputAdapter
+                                                                                  Exchange [c_customer_sk] #9
+                                                                                    WholeStageCodegen (4)
+                                                                                      Filter [c_customer_sk]
+                                                                                        ColumnarToRow
+                                                                                          InputAdapter
+                                                                                            Scan parquet default.customer [c_customer_sk,c_first_name,c_last_name]
+                                                  WholeStageCodegen (16)
+                                                    Sort [c_last_name,c_first_name,d_date]
+                                                      InputAdapter
+                                                        Exchange [c_last_name,c_first_name,d_date] #10
+                                                          WholeStageCodegen (15)
+                                                            HashAggregate [c_last_name,c_first_name,d_date]
+                                                              InputAdapter
+                                                                Exchange [c_last_name,c_first_name,d_date] #11
+                                                                  WholeStageCodegen (14)
+                                                                    HashAggregate [c_last_name,c_first_name,d_date]
+                                                                      Project [c_last_name,c_first_name,d_date]
+                                                                        SortMergeJoin [cs_bill_customer_sk,c_customer_sk]
+                                                                          InputAdapter
+                                                                            WholeStageCodegen (11)
+                                                                              Sort [cs_bill_customer_sk]
+                                                                                InputAdapter
+                                                                                  Exchange [cs_bill_customer_sk] #12
+                                                                                    WholeStageCodegen (10)
+                                                                                      Project [cs_bill_customer_sk,d_date]
+                                                                                        BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                                          Filter [cs_bill_customer_sk]
+                                                                                            ColumnarToRow
+                                                                                              InputAdapter
+                                                                                                Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk]
+                                                                                                  ReusedSubquery [d_date_sk] #1
+                                                                                          InputAdapter
+                                                                                            ReusedExchange [d_date_sk,d_date] #8
+                                                                          InputAdapter
+                                                                            WholeStageCodegen (13)
+                                                                              Sort [c_customer_sk]
+                                                                                InputAdapter
+                                                                                  ReusedExchange [c_customer_sk,c_first_name,c_last_name] #9
+                          WholeStageCodegen (27)
+                            Sort [c_last_name,c_first_name,d_date]
+                              InputAdapter
+                                Exchange [c_last_name,c_first_name,d_date] #13
+                                  WholeStageCodegen (26)
+                                    HashAggregate [c_last_name,c_first_name,d_date]
+                                      InputAdapter
+                                        Exchange [c_last_name,c_first_name,d_date] #14
+                                          WholeStageCodegen (25)
+                                            HashAggregate [c_last_name,c_first_name,d_date]
+                                              Project [c_last_name,c_first_name,d_date]
+                                                SortMergeJoin [ws_bill_customer_sk,c_customer_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (22)
+                                                      Sort [ws_bill_customer_sk]
+                                                        InputAdapter
+                                                          Exchange [ws_bill_customer_sk] #15
+                                                            WholeStageCodegen (21)
+                                                              Project [ws_bill_customer_sk,d_date]
+                                                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                  Filter [ws_bill_customer_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.web_sales [ws_bill_customer_sk,ws_sold_date_sk]
+                                                                          ReusedSubquery [d_date_sk] #1
+                                                                  InputAdapter
+                                                                    ReusedExchange [d_date_sk,d_date] #8
+                                                  InputAdapter
+                                                    WholeStageCodegen (24)
+                                                      Sort [c_customer_sk]
+                                                        InputAdapter
+                                                          ReusedExchange [c_customer_sk,c_first_name,c_last_name] #9
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/explain.txt
new file mode 100644
index 0000000000000..27e16b75638a8
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/explain.txt
@@ -0,0 +1,340 @@
+== Physical Plan ==
+* HashAggregate (54)
++- Exchange (53)
+   +- * HashAggregate (52)
+      +- * HashAggregate (51)
+         +- * HashAggregate (50)
+            +- * BroadcastHashJoin LeftAnti BuildRight (49)
+               :- * HashAggregate (35)
+               :  +- * HashAggregate (34)
+               :     +- * BroadcastHashJoin LeftAnti BuildRight (33)
+               :        :- * HashAggregate (19)
+               :        :  +- Exchange (18)
+               :        :     +- * HashAggregate (17)
+               :        :        +- * Project (16)
+               :        :           +- * BroadcastHashJoin Inner BuildRight (15)
+               :        :              :- * Project (10)
+               :        :              :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :        :              :     :- * Filter (3)
+               :        :              :     :  +- * ColumnarToRow (2)
+               :        :              :     :     +- Scan parquet default.store_sales (1)
+               :        :              :     +- BroadcastExchange (8)
+               :        :              :        +- * Project (7)
+               :        :              :           +- * Filter (6)
+               :        :              :              +- * ColumnarToRow (5)
+               :        :              :                 +- Scan parquet default.date_dim (4)
+               :        :              +- BroadcastExchange (14)
+               :        :                 +- * Filter (13)
+               :        :                    +- * ColumnarToRow (12)
+               :        :                       +- Scan parquet default.customer (11)
+               :        +- BroadcastExchange (32)
+               :           +- * HashAggregate (31)
+               :              +- Exchange (30)
+               :                 +- * HashAggregate (29)
+               :                    +- * Project (28)
+               :                       +- * BroadcastHashJoin Inner BuildRight (27)
+               :                          :- * Project (25)
+               :                          :  +- * BroadcastHashJoin Inner BuildRight (24)
+               :                          :     :- * Filter (22)
+               :                          :     :  +- * ColumnarToRow (21)
+               :                          :     :     +- Scan parquet default.catalog_sales (20)
+               :                          :     +- ReusedExchange (23)
+               :                          +- ReusedExchange (26)
+               +- BroadcastExchange (48)
+                  +- * HashAggregate (47)
+                     +- Exchange (46)
+                        +- * HashAggregate (45)
+                           +- * Project (44)
+                              +- * BroadcastHashJoin Inner BuildRight (43)
+                                 :- * Project (41)
+                                 :  +- * BroadcastHashJoin Inner BuildRight (40)
+                                 :     :- * Filter (38)
+                                 :     :  +- * ColumnarToRow (37)
+                                 :     :     +- Scan parquet default.web_sales (36)
+                                 :     +- ReusedExchange (39)
+                                 +- ReusedExchange (42)
+
+
+(1) Scan parquet default.store_sales
+Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#2), dynamicpruningexpression(ss_sold_date_sk#2 IN dynamicpruning#3)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2]
+
+(3) Filter [codegen id : 3]
+Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2]
+Condition : isnotnull(ss_customer_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date,d_month_seq:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6]
+Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4))
+
+(7) Project [codegen id : 1]
+Output [2]: [d_date_sk#4, d_date#5]
+Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6]
+
+(8) BroadcastExchange
+Input [2]: [d_date_sk#4, d_date#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#2]
+Right keys [1]: [d_date_sk#4]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [2]: [ss_customer_sk#1, d_date#5]
+Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#4, d_date#5]
+
+(11) Scan parquet default.customer
+Output [3]: [c_customer_sk#8, c_first_name#9, c_last_name#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_first_name:string,c_last_name:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [c_customer_sk#8, c_first_name#9, c_last_name#10]
+
+(13) Filter [codegen id : 2]
+Input [3]: [c_customer_sk#8, c_first_name#9, c_last_name#10]
+Condition : isnotnull(c_customer_sk#8)
+
+(14) BroadcastExchange
+Input [3]: [c_customer_sk#8, c_first_name#9, c_last_name#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#8]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Input [5]: [ss_customer_sk#1, d_date#5, c_customer_sk#8, c_first_name#9, c_last_name#10]
+
+(17) HashAggregate [codegen id : 3]
+Input [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Keys [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#10, c_first_name#9, d_date#5]
+
+(18) Exchange
+Input [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Arguments: hashpartitioning(c_last_name#10, c_first_name#9, d_date#5, 5), ENSURE_REQUIREMENTS, [id=#12]
+
+(19) HashAggregate [codegen id : 12]
+Input [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Keys [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#10, c_first_name#9, d_date#5]
+
+(20) Scan parquet default.catalog_sales
+Output [2]: [cs_bill_customer_sk#13, cs_sold_date_sk#14]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#14), dynamicpruningexpression(cs_sold_date_sk#14 IN dynamicpruning#3)]
+PushedFilters: [IsNotNull(cs_bill_customer_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int>
+
+(21) ColumnarToRow [codegen id : 6]
+Input [2]: [cs_bill_customer_sk#13, cs_sold_date_sk#14]
+
+(22) Filter [codegen id : 6]
+Input [2]: [cs_bill_customer_sk#13, cs_sold_date_sk#14]
+Condition : isnotnull(cs_bill_customer_sk#13)
+
+(23) ReusedExchange [Reuses operator id: 8]
+Output [2]: [d_date_sk#15, d_date#16]
+
+(24) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [cs_sold_date_sk#14]
+Right keys [1]: [d_date_sk#15]
+Join condition: None
+
+(25) Project [codegen id : 6]
+Output [2]: [cs_bill_customer_sk#13, d_date#16]
+Input [4]: [cs_bill_customer_sk#13, cs_sold_date_sk#14, d_date_sk#15, d_date#16]
+
+(26) ReusedExchange [Reuses operator id: 14]
+Output [3]: [c_customer_sk#17, c_first_name#18, c_last_name#19]
+
+(27) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [cs_bill_customer_sk#13]
+Right keys [1]: [c_customer_sk#17]
+Join condition: None
+
+(28) Project [codegen id : 6]
+Output [3]: [c_last_name#19, c_first_name#18, d_date#16]
+Input [5]: [cs_bill_customer_sk#13, d_date#16, c_customer_sk#17, c_first_name#18, c_last_name#19]
+
+(29) HashAggregate [codegen id : 6]
+Input [3]: [c_last_name#19, c_first_name#18, d_date#16]
+Keys [3]: [c_last_name#19, c_first_name#18, d_date#16]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#19, c_first_name#18, d_date#16]
+
+(30) Exchange
+Input [3]: [c_last_name#19, c_first_name#18, d_date#16]
+Arguments: hashpartitioning(c_last_name#19, c_first_name#18, d_date#16, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(31) HashAggregate [codegen id : 7]
+Input [3]: [c_last_name#19, c_first_name#18, d_date#16]
+Keys [3]: [c_last_name#19, c_first_name#18, d_date#16]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#19, c_first_name#18, d_date#16]
+
+(32) BroadcastExchange
+Input [3]: [c_last_name#19, c_first_name#18, d_date#16]
+Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [id=#21]
+
+(33) BroadcastHashJoin [codegen id : 12]
+Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#5, 1970-01-01), isnull(d_date#5)]
+Right keys [6]: [coalesce(c_last_name#19, ), isnull(c_last_name#19), coalesce(c_first_name#18, ), isnull(c_first_name#18), coalesce(d_date#16, 1970-01-01), isnull(d_date#16)]
+Join condition: None
+
+(34) HashAggregate [codegen id : 12]
+Input [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Keys [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#10, c_first_name#9, d_date#5]
+
+(35) HashAggregate [codegen id : 12]
+Input [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Keys [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#10, c_first_name#9, d_date#5]
+
+(36) Scan parquet default.web_sales
+Output [2]: [ws_bill_customer_sk#22, ws_sold_date_sk#23]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#23), dynamicpruningexpression(ws_sold_date_sk#23 IN dynamicpruning#3)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int>
+
+(37) ColumnarToRow [codegen id : 10]
+Input [2]: [ws_bill_customer_sk#22, ws_sold_date_sk#23]
+
+(38) Filter [codegen id : 10]
+Input [2]: [ws_bill_customer_sk#22, ws_sold_date_sk#23]
+Condition : isnotnull(ws_bill_customer_sk#22)
+
+(39) ReusedExchange [Reuses operator id: 8]
+Output [2]: [d_date_sk#24, d_date#25]
+
+(40) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ws_sold_date_sk#23]
+Right keys [1]: [d_date_sk#24]
+Join condition: None
+
+(41) Project [codegen id : 10]
+Output [2]: [ws_bill_customer_sk#22, d_date#25]
+Input [4]: [ws_bill_customer_sk#22, ws_sold_date_sk#23, d_date_sk#24, d_date#25]
+
+(42) ReusedExchange [Reuses operator id: 14]
+Output [3]: [c_customer_sk#26, c_first_name#27, c_last_name#28]
+
+(43) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ws_bill_customer_sk#22]
+Right keys [1]: [c_customer_sk#26]
+Join condition: None
+
+(44) Project [codegen id : 10]
+Output [3]: [c_last_name#28, c_first_name#27, d_date#25]
+Input [5]: [ws_bill_customer_sk#22, d_date#25, c_customer_sk#26, c_first_name#27, c_last_name#28]
+
+(45) HashAggregate [codegen id : 10]
+Input [3]: [c_last_name#28, c_first_name#27, d_date#25]
+Keys [3]: [c_last_name#28, c_first_name#27, d_date#25]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#28, c_first_name#27, d_date#25]
+
+(46) Exchange
+Input [3]: [c_last_name#28, c_first_name#27, d_date#25]
+Arguments: hashpartitioning(c_last_name#28, c_first_name#27, d_date#25, 5), ENSURE_REQUIREMENTS, [id=#29]
+
+(47) HashAggregate [codegen id : 11]
+Input [3]: [c_last_name#28, c_first_name#27, d_date#25]
+Keys [3]: [c_last_name#28, c_first_name#27, d_date#25]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#28, c_first_name#27, d_date#25]
+
+(48) BroadcastExchange
+Input [3]: [c_last_name#28, c_first_name#27, d_date#25]
+Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [id=#30]
+
+(49) BroadcastHashJoin [codegen id : 12]
+Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#5, 1970-01-01), isnull(d_date#5)]
+Right keys [6]: [coalesce(c_last_name#28, ), isnull(c_last_name#28), coalesce(c_first_name#27, ), isnull(c_first_name#27), coalesce(d_date#25, 1970-01-01), isnull(d_date#25)]
+Join condition: None
+
+(50) HashAggregate [codegen id : 12]
+Input [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Keys [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [c_last_name#10, c_first_name#9, d_date#5]
+
+(51) HashAggregate [codegen id : 12]
+Input [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Keys [3]: [c_last_name#10, c_first_name#9, d_date#5]
+Functions: []
+Aggregate Attributes: []
+Results: []
+
+(52) HashAggregate [codegen id : 12]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#31]
+Results [1]: [count#32]
+
+(53) Exchange
+Input [1]: [count#32]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#33]
+
+(54) HashAggregate [codegen id : 13]
+Input [1]: [count#32]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#34]
+Results [1]: [count(1)#34 AS count(1)#35]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#2 IN dynamicpruning#3
+ReusedExchange (55)
+
+
+(55) ReusedExchange [Reuses operator id: 8]
+Output [2]: [d_date_sk#4, d_date#5]
+
+Subquery:2 Hosting operator id = 20 Hosting Expression = cs_sold_date_sk#14 IN dynamicpruning#3
+
+Subquery:3 Hosting operator id = 36 Hosting Expression = ws_sold_date_sk#23 IN dynamicpruning#3
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/simplified.txt
new file mode 100644
index 0000000000000..ce0c20ab01d79
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/simplified.txt
@@ -0,0 +1,84 @@
+WholeStageCodegen (13)
+  HashAggregate [count] [count(1),count(1),count]
+    InputAdapter
+      Exchange #1
+        WholeStageCodegen (12)
+          HashAggregate [count,count]
+            HashAggregate [c_last_name,c_first_name,d_date]
+              HashAggregate [c_last_name,c_first_name,d_date]
+                BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date]
+                  HashAggregate [c_last_name,c_first_name,d_date]
+                    HashAggregate [c_last_name,c_first_name,d_date]
+                      BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date]
+                        HashAggregate [c_last_name,c_first_name,d_date]
+                          InputAdapter
+                            Exchange [c_last_name,c_first_name,d_date] #2
+                              WholeStageCodegen (3)
+                                HashAggregate [c_last_name,c_first_name,d_date]
+                                  Project [c_last_name,c_first_name,d_date]
+                                    BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+                                      Project [ss_customer_sk,d_date]
+                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                          Filter [ss_customer_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store_sales [ss_customer_sk,ss_sold_date_sk]
+                                                  SubqueryBroadcast [d_date_sk] #1
+                                                    ReusedExchange [d_date_sk,d_date] #3
+                                          InputAdapter
+                                            BroadcastExchange #3
+                                              WholeStageCodegen (1)
+                                                Project [d_date_sk,d_date]
+                                                  Filter [d_month_seq,d_date_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.date_dim [d_date_sk,d_date,d_month_seq]
+                                      InputAdapter
+                                        BroadcastExchange #4
+                                          WholeStageCodegen (2)
+                                            Filter [c_customer_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.customer [c_customer_sk,c_first_name,c_last_name]
+                        InputAdapter
+                          BroadcastExchange #5
+                            WholeStageCodegen (7)
+                              HashAggregate [c_last_name,c_first_name,d_date]
+                                InputAdapter
+                                  Exchange [c_last_name,c_first_name,d_date] #6
+                                    WholeStageCodegen (6)
+                                      HashAggregate [c_last_name,c_first_name,d_date]
+                                        Project [c_last_name,c_first_name,d_date]
+                                          BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk]
+                                            Project [cs_bill_customer_sk,d_date]
+                                              BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                Filter [cs_bill_customer_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk]
+                                                        ReusedSubquery [d_date_sk] #1
+                                                InputAdapter
+                                                  ReusedExchange [d_date_sk,d_date] #3
+                                            InputAdapter
+                                              ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4
+                  InputAdapter
+                    BroadcastExchange #7
+                      WholeStageCodegen (11)
+                        HashAggregate [c_last_name,c_first_name,d_date]
+                          InputAdapter
+                            Exchange [c_last_name,c_first_name,d_date] #8
+                              WholeStageCodegen (10)
+                                HashAggregate [c_last_name,c_first_name,d_date]
+                                  Project [c_last_name,c_first_name,d_date]
+                                    BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk]
+                                      Project [ws_bill_customer_sk,d_date]
+                                        BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                          Filter [ws_bill_customer_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.web_sales [ws_bill_customer_sk,ws_sold_date_sk]
+                                                  ReusedSubquery [d_date_sk] #1
+                                          InputAdapter
+                                            ReusedExchange [d_date_sk,d_date] #3
+                                      InputAdapter
+                                        ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.sf100/explain.txt
new file mode 100644
index 0000000000000..e72928545d080
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.sf100/explain.txt
@@ -0,0 +1,1000 @@
+== Physical Plan ==
+* BroadcastNestedLoopJoin Inner BuildRight (182)
+:- * BroadcastNestedLoopJoin Inner BuildRight (160)
+:  :- * BroadcastNestedLoopJoin Inner BuildRight (138)
+:  :  :- * BroadcastNestedLoopJoin Inner BuildRight (116)
+:  :  :  :- * BroadcastNestedLoopJoin Inner BuildRight (94)
+:  :  :  :  :- * BroadcastNestedLoopJoin Inner BuildRight (72)
+:  :  :  :  :  :- * BroadcastNestedLoopJoin Inner BuildRight (50)
+:  :  :  :  :  :  :- * HashAggregate (28)
+:  :  :  :  :  :  :  +- Exchange (27)
+:  :  :  :  :  :  :     +- * HashAggregate (26)
+:  :  :  :  :  :  :        +- * Project (25)
+:  :  :  :  :  :  :           +- * BroadcastHashJoin Inner BuildRight (24)
+:  :  :  :  :  :  :              :- * Project (18)
+:  :  :  :  :  :  :              :  +- * BroadcastHashJoin Inner BuildRight (17)
+:  :  :  :  :  :  :              :     :- * Project (11)
+:  :  :  :  :  :  :              :     :  +- * BroadcastHashJoin Inner BuildRight (10)
+:  :  :  :  :  :  :              :     :     :- * Project (4)
+:  :  :  :  :  :  :              :     :     :  +- * Filter (3)
+:  :  :  :  :  :  :              :     :     :     +- * ColumnarToRow (2)
+:  :  :  :  :  :  :              :     :     :        +- Scan parquet default.store_sales (1)
+:  :  :  :  :  :  :              :     :     +- BroadcastExchange (9)
+:  :  :  :  :  :  :              :     :        +- * Project (8)
+:  :  :  :  :  :  :              :     :           +- * Filter (7)
+:  :  :  :  :  :  :              :     :              +- * ColumnarToRow (6)
+:  :  :  :  :  :  :              :     :                 +- Scan parquet default.time_dim (5)
+:  :  :  :  :  :  :              :     +- BroadcastExchange (16)
+:  :  :  :  :  :  :              :        +- * Project (15)
+:  :  :  :  :  :  :              :           +- * Filter (14)
+:  :  :  :  :  :  :              :              +- * ColumnarToRow (13)
+:  :  :  :  :  :  :              :                 +- Scan parquet default.store (12)
+:  :  :  :  :  :  :              +- BroadcastExchange (23)
+:  :  :  :  :  :  :                 +- * Project (22)
+:  :  :  :  :  :  :                    +- * Filter (21)
+:  :  :  :  :  :  :                       +- * ColumnarToRow (20)
+:  :  :  :  :  :  :                          +- Scan parquet default.household_demographics (19)
+:  :  :  :  :  :  +- BroadcastExchange (49)
+:  :  :  :  :  :     +- * HashAggregate (48)
+:  :  :  :  :  :        +- Exchange (47)
+:  :  :  :  :  :           +- * HashAggregate (46)
+:  :  :  :  :  :              +- * Project (45)
+:  :  :  :  :  :                 +- * BroadcastHashJoin Inner BuildRight (44)
+:  :  :  :  :  :                    :- * Project (42)
+:  :  :  :  :  :                    :  +- * BroadcastHashJoin Inner BuildRight (41)
+:  :  :  :  :  :                    :     :- * Project (39)
+:  :  :  :  :  :                    :     :  +- * BroadcastHashJoin Inner BuildRight (38)
+:  :  :  :  :  :                    :     :     :- * Project (32)
+:  :  :  :  :  :                    :     :     :  +- * Filter (31)
+:  :  :  :  :  :                    :     :     :     +- * ColumnarToRow (30)
+:  :  :  :  :  :                    :     :     :        +- Scan parquet default.store_sales (29)
+:  :  :  :  :  :                    :     :     +- BroadcastExchange (37)
+:  :  :  :  :  :                    :     :        +- * Project (36)
+:  :  :  :  :  :                    :     :           +- * Filter (35)
+:  :  :  :  :  :                    :     :              +- * ColumnarToRow (34)
+:  :  :  :  :  :                    :     :                 +- Scan parquet default.time_dim (33)
+:  :  :  :  :  :                    :     +- ReusedExchange (40)
+:  :  :  :  :  :                    +- ReusedExchange (43)
+:  :  :  :  :  +- BroadcastExchange (71)
+:  :  :  :  :     +- * HashAggregate (70)
+:  :  :  :  :        +- Exchange (69)
+:  :  :  :  :           +- * HashAggregate (68)
+:  :  :  :  :              +- * Project (67)
+:  :  :  :  :                 +- * BroadcastHashJoin Inner BuildRight (66)
+:  :  :  :  :                    :- * Project (64)
+:  :  :  :  :                    :  +- * BroadcastHashJoin Inner BuildRight (63)
+:  :  :  :  :                    :     :- * Project (61)
+:  :  :  :  :                    :     :  +- * BroadcastHashJoin Inner BuildRight (60)
+:  :  :  :  :                    :     :     :- * Project (54)
+:  :  :  :  :                    :     :     :  +- * Filter (53)
+:  :  :  :  :                    :     :     :     +- * ColumnarToRow (52)
+:  :  :  :  :                    :     :     :        +- Scan parquet default.store_sales (51)
+:  :  :  :  :                    :     :     +- BroadcastExchange (59)
+:  :  :  :  :                    :     :        +- * Project (58)
+:  :  :  :  :                    :     :           +- * Filter (57)
+:  :  :  :  :                    :     :              +- * ColumnarToRow (56)
+:  :  :  :  :                    :     :                 +- Scan parquet default.time_dim (55)
+:  :  :  :  :                    :     +- ReusedExchange (62)
+:  :  :  :  :                    +- ReusedExchange (65)
+:  :  :  :  +- BroadcastExchange (93)
+:  :  :  :     +- * HashAggregate (92)
+:  :  :  :        +- Exchange (91)
+:  :  :  :           +- * HashAggregate (90)
+:  :  :  :              +- * Project (89)
+:  :  :  :                 +- * BroadcastHashJoin Inner BuildRight (88)
+:  :  :  :                    :- * Project (86)
+:  :  :  :                    :  +- * BroadcastHashJoin Inner BuildRight (85)
+:  :  :  :                    :     :- * Project (83)
+:  :  :  :                    :     :  +- * BroadcastHashJoin Inner BuildRight (82)
+:  :  :  :                    :     :     :- * Project (76)
+:  :  :  :                    :     :     :  +- * Filter (75)
+:  :  :  :                    :     :     :     +- * ColumnarToRow (74)
+:  :  :  :                    :     :     :        +- Scan parquet default.store_sales (73)
+:  :  :  :                    :     :     +- BroadcastExchange (81)
+:  :  :  :                    :     :        +- * Project (80)
+:  :  :  :                    :     :           +- * Filter (79)
+:  :  :  :                    :     :              +- * ColumnarToRow (78)
+:  :  :  :                    :     :                 +- Scan parquet default.time_dim (77)
+:  :  :  :                    :     +- ReusedExchange (84)
+:  :  :  :                    +- ReusedExchange (87)
+:  :  :  +- BroadcastExchange (115)
+:  :  :     +- * HashAggregate (114)
+:  :  :        +- Exchange (113)
+:  :  :           +- * HashAggregate (112)
+:  :  :              +- * Project (111)
+:  :  :                 +- * BroadcastHashJoin Inner BuildRight (110)
+:  :  :                    :- * Project (108)
+:  :  :                    :  +- * BroadcastHashJoin Inner BuildRight (107)
+:  :  :                    :     :- * Project (105)
+:  :  :                    :     :  +- * BroadcastHashJoin Inner BuildRight (104)
+:  :  :                    :     :     :- * Project (98)
+:  :  :                    :     :     :  +- * Filter (97)
+:  :  :                    :     :     :     +- * ColumnarToRow (96)
+:  :  :                    :     :     :        +- Scan parquet default.store_sales (95)
+:  :  :                    :     :     +- BroadcastExchange (103)
+:  :  :                    :     :        +- * Project (102)
+:  :  :                    :     :           +- * Filter (101)
+:  :  :                    :     :              +- * ColumnarToRow (100)
+:  :  :                    :     :                 +- Scan parquet default.time_dim (99)
+:  :  :                    :     +- ReusedExchange (106)
+:  :  :                    +- ReusedExchange (109)
+:  :  +- BroadcastExchange (137)
+:  :     +- * HashAggregate (136)
+:  :        +- Exchange (135)
+:  :           +- * HashAggregate (134)
+:  :              +- * Project (133)
+:  :                 +- * BroadcastHashJoin Inner BuildRight (132)
+:  :                    :- * Project (130)
+:  :                    :  +- * BroadcastHashJoin Inner BuildRight (129)
+:  :                    :     :- * Project (127)
+:  :                    :     :  +- * BroadcastHashJoin Inner BuildRight (126)
+:  :                    :     :     :- * Project (120)
+:  :                    :     :     :  +- * Filter (119)
+:  :                    :     :     :     +- * ColumnarToRow (118)
+:  :                    :     :     :        +- Scan parquet default.store_sales (117)
+:  :                    :     :     +- BroadcastExchange (125)
+:  :                    :     :        +- * Project (124)
+:  :                    :     :           +- * Filter (123)
+:  :                    :     :              +- * ColumnarToRow (122)
+:  :                    :     :                 +- Scan parquet default.time_dim (121)
+:  :                    :     +- ReusedExchange (128)
+:  :                    +- ReusedExchange (131)
+:  +- BroadcastExchange (159)
+:     +- * HashAggregate (158)
+:        +- Exchange (157)
+:           +- * HashAggregate (156)
+:              +- * Project (155)
+:                 +- * BroadcastHashJoin Inner BuildRight (154)
+:                    :- * Project (152)
+:                    :  +- * BroadcastHashJoin Inner BuildRight (151)
+:                    :     :- * Project (149)
+:                    :     :  +- * BroadcastHashJoin Inner BuildRight (148)
+:                    :     :     :- * Project (142)
+:                    :     :     :  +- * Filter (141)
+:                    :     :     :     +- * ColumnarToRow (140)
+:                    :     :     :        +- Scan parquet default.store_sales (139)
+:                    :     :     +- BroadcastExchange (147)
+:                    :     :        +- * Project (146)
+:                    :     :           +- * Filter (145)
+:                    :     :              +- * ColumnarToRow (144)
+:                    :     :                 +- Scan parquet default.time_dim (143)
+:                    :     +- ReusedExchange (150)
+:                    +- ReusedExchange (153)
++- BroadcastExchange (181)
+   +- * HashAggregate (180)
+      +- Exchange (179)
+         +- * HashAggregate (178)
+            +- * Project (177)
+               +- * BroadcastHashJoin Inner BuildRight (176)
+                  :- * Project (174)
+                  :  +- * BroadcastHashJoin Inner BuildRight (173)
+                  :     :- * Project (171)
+                  :     :  +- * BroadcastHashJoin Inner BuildRight (170)
+                  :     :     :- * Project (164)
+                  :     :     :  +- * Filter (163)
+                  :     :     :     +- * ColumnarToRow (162)
+                  :     :     :        +- Scan parquet default.store_sales (161)
+                  :     :     +- BroadcastExchange (169)
+                  :     :        +- * Project (168)
+                  :     :           +- * Filter (167)
+                  :     :              +- * ColumnarToRow (166)
+                  :     :                 +- Scan parquet default.time_dim (165)
+                  :     +- ReusedExchange (172)
+                  +- ReusedExchange (175)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_sold_time_sk:int,ss_hdemo_sk:int,ss_store_sk:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 4]
+Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4]
+Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isnotnull(ss_store_sk#3))
+
+(4) Project [codegen id : 4]
+Output [3]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3]
+Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4]
+
+(5) Scan parquet default.time_dim
+Output [3]: [t_time_sk#5, t_hour#6, t_minute#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,8), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_hour:int,t_minute:int>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [3]: [t_time_sk#5, t_hour#6, t_minute#7]
+
+(7) Filter [codegen id : 1]
+Input [3]: [t_time_sk#5, t_hour#6, t_minute#7]
+Condition : ((((isnotnull(t_hour#6) AND isnotnull(t_minute#7)) AND (t_hour#6 = 8)) AND (t_minute#7 >= 30)) AND isnotnull(t_time_sk#5))
+
+(8) Project [codegen id : 1]
+Output [1]: [t_time_sk#5]
+Input [3]: [t_time_sk#5, t_hour#6, t_minute#7]
+
+(9) BroadcastExchange
+Input [1]: [t_time_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#8]
+
+(10) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_time_sk#1]
+Right keys [1]: [t_time_sk#5]
+Join condition: None
+
+(11) Project [codegen id : 4]
+Output [2]: [ss_hdemo_sk#2, ss_store_sk#3]
+Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, t_time_sk#5]
+
+(12) Scan parquet default.store
+Output [2]: [s_store_sk#9, s_store_name#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_name), EqualTo(s_store_name,ese), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string>
+
+(13) ColumnarToRow [codegen id : 2]
+Input [2]: [s_store_sk#9, s_store_name#10]
+
+(14) Filter [codegen id : 2]
+Input [2]: [s_store_sk#9, s_store_name#10]
+Condition : ((isnotnull(s_store_name#10) AND (s_store_name#10 = ese)) AND isnotnull(s_store_sk#9))
+
+(15) Project [codegen id : 2]
+Output [1]: [s_store_sk#9]
+Input [2]: [s_store_sk#9, s_store_name#10]
+
+(16) BroadcastExchange
+Input [1]: [s_store_sk#9]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#11]
+
+(17) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#9]
+Join condition: None
+
+(18) Project [codegen id : 4]
+Output [1]: [ss_hdemo_sk#2]
+Input [3]: [ss_hdemo_sk#2, ss_store_sk#3, s_store_sk#9]
+
+(19) Scan parquet default.household_demographics
+Output [3]: [hd_demo_sk#12, hd_dep_count#13, hd_vehicle_count#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [Or(Or(And(EqualTo(hd_dep_count,4),LessThanOrEqual(hd_vehicle_count,6)),And(EqualTo(hd_dep_count,2),LessThanOrEqual(hd_vehicle_count,4))),And(EqualTo(hd_dep_count,0),LessThanOrEqual(hd_vehicle_count,2))), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_dep_count:int,hd_vehicle_count:int>
+
+(20) ColumnarToRow [codegen id : 3]
+Input [3]: [hd_demo_sk#12, hd_dep_count#13, hd_vehicle_count#14]
+
+(21) Filter [codegen id : 3]
+Input [3]: [hd_demo_sk#12, hd_dep_count#13, hd_vehicle_count#14]
+Condition : (((((hd_dep_count#13 = 4) AND (hd_vehicle_count#14 <= 6)) OR ((hd_dep_count#13 = 2) AND (hd_vehicle_count#14 <= 4))) OR ((hd_dep_count#13 = 0) AND (hd_vehicle_count#14 <= 2))) AND isnotnull(hd_demo_sk#12))
+
+(22) Project [codegen id : 3]
+Output [1]: [hd_demo_sk#12]
+Input [3]: [hd_demo_sk#12, hd_dep_count#13, hd_vehicle_count#14]
+
+(23) BroadcastExchange
+Input [1]: [hd_demo_sk#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#15]
+
+(24) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#12]
+Join condition: None
+
+(25) Project [codegen id : 4]
+Output: []
+Input [2]: [ss_hdemo_sk#2, hd_demo_sk#12]
+
+(26) HashAggregate [codegen id : 4]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#16]
+Results [1]: [count#17]
+
+(27) Exchange
+Input [1]: [count#17]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#18]
+
+(28) HashAggregate [codegen id : 40]
+Input [1]: [count#17]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#19]
+Results [1]: [count(1)#19 AS h8_30_to_9#20]
+
+(29) Scan parquet default.store_sales
+Output [4]: [ss_sold_time_sk#21, ss_hdemo_sk#22, ss_store_sk#23, ss_sold_date_sk#24]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_sold_time_sk:int,ss_hdemo_sk:int,ss_store_sk:int>
+
+(30) ColumnarToRow [codegen id : 8]
+Input [4]: [ss_sold_time_sk#21, ss_hdemo_sk#22, ss_store_sk#23, ss_sold_date_sk#24]
+
+(31) Filter [codegen id : 8]
+Input [4]: [ss_sold_time_sk#21, ss_hdemo_sk#22, ss_store_sk#23, ss_sold_date_sk#24]
+Condition : ((isnotnull(ss_hdemo_sk#22) AND isnotnull(ss_sold_time_sk#21)) AND isnotnull(ss_store_sk#23))
+
+(32) Project [codegen id : 8]
+Output [3]: [ss_sold_time_sk#21, ss_hdemo_sk#22, ss_store_sk#23]
+Input [4]: [ss_sold_time_sk#21, ss_hdemo_sk#22, ss_store_sk#23, ss_sold_date_sk#24]
+
+(33) Scan parquet default.time_dim
+Output [3]: [t_time_sk#25, t_hour#26, t_minute#27]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,9), LessThan(t_minute,30), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_hour:int,t_minute:int>
+
+(34) ColumnarToRow [codegen id : 5]
+Input [3]: [t_time_sk#25, t_hour#26, t_minute#27]
+
+(35) Filter [codegen id : 5]
+Input [3]: [t_time_sk#25, t_hour#26, t_minute#27]
+Condition : ((((isnotnull(t_hour#26) AND isnotnull(t_minute#27)) AND (t_hour#26 = 9)) AND (t_minute#27 < 30)) AND isnotnull(t_time_sk#25))
+
+(36) Project [codegen id : 5]
+Output [1]: [t_time_sk#25]
+Input [3]: [t_time_sk#25, t_hour#26, t_minute#27]
+
+(37) BroadcastExchange
+Input [1]: [t_time_sk#25]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#28]
+
+(38) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_sold_time_sk#21]
+Right keys [1]: [t_time_sk#25]
+Join condition: None
+
+(39) Project [codegen id : 8]
+Output [2]: [ss_hdemo_sk#22, ss_store_sk#23]
+Input [4]: [ss_sold_time_sk#21, ss_hdemo_sk#22, ss_store_sk#23, t_time_sk#25]
+
+(40) ReusedExchange [Reuses operator id: 16]
+Output [1]: [s_store_sk#29]
+
+(41) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_store_sk#23]
+Right keys [1]: [s_store_sk#29]
+Join condition: None
+
+(42) Project [codegen id : 8]
+Output [1]: [ss_hdemo_sk#22]
+Input [3]: [ss_hdemo_sk#22, ss_store_sk#23, s_store_sk#29]
+
+(43) ReusedExchange [Reuses operator id: 23]
+Output [1]: [hd_demo_sk#30]
+
+(44) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_hdemo_sk#22]
+Right keys [1]: [hd_demo_sk#30]
+Join condition: None
+
+(45) Project [codegen id : 8]
+Output: []
+Input [2]: [ss_hdemo_sk#22, hd_demo_sk#30]
+
+(46) HashAggregate [codegen id : 8]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#31]
+Results [1]: [count#32]
+
+(47) Exchange
+Input [1]: [count#32]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#33]
+
+(48) HashAggregate [codegen id : 9]
+Input [1]: [count#32]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#34]
+Results [1]: [count(1)#34 AS h9_to_9_30#35]
+
+(49) BroadcastExchange
+Input [1]: [h9_to_9_30#35]
+Arguments: IdentityBroadcastMode, [id=#36]
+
+(50) BroadcastNestedLoopJoin [codegen id : 40]
+Join condition: None
+
+(51) Scan parquet default.store_sales
+Output [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_sold_time_sk:int,ss_hdemo_sk:int,ss_store_sk:int>
+
+(52) ColumnarToRow [codegen id : 13]
+Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40]
+
+(53) Filter [codegen id : 13]
+Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40]
+Condition : ((isnotnull(ss_hdemo_sk#38) AND isnotnull(ss_sold_time_sk#37)) AND isnotnull(ss_store_sk#39))
+
+(54) Project [codegen id : 13]
+Output [3]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39]
+Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40]
+
+(55) Scan parquet default.time_dim
+Output [3]: [t_time_sk#41, t_hour#42, t_minute#43]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,9), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_hour:int,t_minute:int>
+
+(56) ColumnarToRow [codegen id : 10]
+Input [3]: [t_time_sk#41, t_hour#42, t_minute#43]
+
+(57) Filter [codegen id : 10]
+Input [3]: [t_time_sk#41, t_hour#42, t_minute#43]
+Condition : ((((isnotnull(t_hour#42) AND isnotnull(t_minute#43)) AND (t_hour#42 = 9)) AND (t_minute#43 >= 30)) AND isnotnull(t_time_sk#41))
+
+(58) Project [codegen id : 10]
+Output [1]: [t_time_sk#41]
+Input [3]: [t_time_sk#41, t_hour#42, t_minute#43]
+
+(59) BroadcastExchange
+Input [1]: [t_time_sk#41]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#44]
+
+(60) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ss_sold_time_sk#37]
+Right keys [1]: [t_time_sk#41]
+Join condition: None
+
+(61) Project [codegen id : 13]
+Output [2]: [ss_hdemo_sk#38, ss_store_sk#39]
+Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, t_time_sk#41]
+
+(62) ReusedExchange [Reuses operator id: 16]
+Output [1]: [s_store_sk#45]
+
+(63) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ss_store_sk#39]
+Right keys [1]: [s_store_sk#45]
+Join condition: None
+
+(64) Project [codegen id : 13]
+Output [1]: [ss_hdemo_sk#38]
+Input [3]: [ss_hdemo_sk#38, ss_store_sk#39, s_store_sk#45]
+
+(65) ReusedExchange [Reuses operator id: 23]
+Output [1]: [hd_demo_sk#46]
+
+(66) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ss_hdemo_sk#38]
+Right keys [1]: [hd_demo_sk#46]
+Join condition: None
+
+(67) Project [codegen id : 13]
+Output: []
+Input [2]: [ss_hdemo_sk#38, hd_demo_sk#46]
+
+(68) HashAggregate [codegen id : 13]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#47]
+Results [1]: [count#48]
+
+(69) Exchange
+Input [1]: [count#48]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#49]
+
+(70) HashAggregate [codegen id : 14]
+Input [1]: [count#48]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#50]
+Results [1]: [count(1)#50 AS h9_30_to_10#51]
+
+(71) BroadcastExchange
+Input [1]: [h9_30_to_10#51]
+Arguments: IdentityBroadcastMode, [id=#52]
+
+(72) BroadcastNestedLoopJoin [codegen id : 40]
+Join condition: None
+
+(73) Scan parquet default.store_sales
+Output [4]: [ss_sold_time_sk#53, ss_hdemo_sk#54, ss_store_sk#55, ss_sold_date_sk#56]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_sold_time_sk:int,ss_hdemo_sk:int,ss_store_sk:int>
+
+(74) ColumnarToRow [codegen id : 18]
+Input [4]: [ss_sold_time_sk#53, ss_hdemo_sk#54, ss_store_sk#55, ss_sold_date_sk#56]
+
+(75) Filter [codegen id : 18]
+Input [4]: [ss_sold_time_sk#53, ss_hdemo_sk#54, ss_store_sk#55, ss_sold_date_sk#56]
+Condition : ((isnotnull(ss_hdemo_sk#54) AND isnotnull(ss_sold_time_sk#53)) AND isnotnull(ss_store_sk#55))
+
+(76) Project [codegen id : 18]
+Output [3]: [ss_sold_time_sk#53, ss_hdemo_sk#54, ss_store_sk#55]
+Input [4]: [ss_sold_time_sk#53, ss_hdemo_sk#54, ss_store_sk#55, ss_sold_date_sk#56]
+
+(77) Scan parquet default.time_dim
+Output [3]: [t_time_sk#57, t_hour#58, t_minute#59]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,10), LessThan(t_minute,30), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_hour:int,t_minute:int>
+
+(78) ColumnarToRow [codegen id : 15]
+Input [3]: [t_time_sk#57, t_hour#58, t_minute#59]
+
+(79) Filter [codegen id : 15]
+Input [3]: [t_time_sk#57, t_hour#58, t_minute#59]
+Condition : ((((isnotnull(t_hour#58) AND isnotnull(t_minute#59)) AND (t_hour#58 = 10)) AND (t_minute#59 < 30)) AND isnotnull(t_time_sk#57))
+
+(80) Project [codegen id : 15]
+Output [1]: [t_time_sk#57]
+Input [3]: [t_time_sk#57, t_hour#58, t_minute#59]
+
+(81) BroadcastExchange
+Input [1]: [t_time_sk#57]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#60]
+
+(82) BroadcastHashJoin [codegen id : 18]
+Left keys [1]: [ss_sold_time_sk#53]
+Right keys [1]: [t_time_sk#57]
+Join condition: None
+
+(83) Project [codegen id : 18]
+Output [2]: [ss_hdemo_sk#54, ss_store_sk#55]
+Input [4]: [ss_sold_time_sk#53, ss_hdemo_sk#54, ss_store_sk#55, t_time_sk#57]
+
+(84) ReusedExchange [Reuses operator id: 16]
+Output [1]: [s_store_sk#61]
+
+(85) BroadcastHashJoin [codegen id : 18]
+Left keys [1]: [ss_store_sk#55]
+Right keys [1]: [s_store_sk#61]
+Join condition: None
+
+(86) Project [codegen id : 18]
+Output [1]: [ss_hdemo_sk#54]
+Input [3]: [ss_hdemo_sk#54, ss_store_sk#55, s_store_sk#61]
+
+(87) ReusedExchange [Reuses operator id: 23]
+Output [1]: [hd_demo_sk#62]
+
+(88) BroadcastHashJoin [codegen id : 18]
+Left keys [1]: [ss_hdemo_sk#54]
+Right keys [1]: [hd_demo_sk#62]
+Join condition: None
+
+(89) Project [codegen id : 18]
+Output: []
+Input [2]: [ss_hdemo_sk#54, hd_demo_sk#62]
+
+(90) HashAggregate [codegen id : 18]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#63]
+Results [1]: [count#64]
+
+(91) Exchange
+Input [1]: [count#64]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#65]
+
+(92) HashAggregate [codegen id : 19]
+Input [1]: [count#64]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#66]
+Results [1]: [count(1)#66 AS h10_to_10_30#67]
+
+(93) BroadcastExchange
+Input [1]: [h10_to_10_30#67]
+Arguments: IdentityBroadcastMode, [id=#68]
+
+(94) BroadcastNestedLoopJoin [codegen id : 40]
+Join condition: None
+
+(95) Scan parquet default.store_sales
+Output [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, ss_sold_date_sk#72]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_sold_time_sk:int,ss_hdemo_sk:int,ss_store_sk:int>
+
+(96) ColumnarToRow [codegen id : 23]
+Input [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, ss_sold_date_sk#72]
+
+(97) Filter [codegen id : 23]
+Input [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, ss_sold_date_sk#72]
+Condition : ((isnotnull(ss_hdemo_sk#70) AND isnotnull(ss_sold_time_sk#69)) AND isnotnull(ss_store_sk#71))
+
+(98) Project [codegen id : 23]
+Output [3]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71]
+Input [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, ss_sold_date_sk#72]
+
+(99) Scan parquet default.time_dim
+Output [3]: [t_time_sk#73, t_hour#74, t_minute#75]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,10), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_hour:int,t_minute:int>
+
+(100) ColumnarToRow [codegen id : 20]
+Input [3]: [t_time_sk#73, t_hour#74, t_minute#75]
+
+(101) Filter [codegen id : 20]
+Input [3]: [t_time_sk#73, t_hour#74, t_minute#75]
+Condition : ((((isnotnull(t_hour#74) AND isnotnull(t_minute#75)) AND (t_hour#74 = 10)) AND (t_minute#75 >= 30)) AND isnotnull(t_time_sk#73))
+
+(102) Project [codegen id : 20]
+Output [1]: [t_time_sk#73]
+Input [3]: [t_time_sk#73, t_hour#74, t_minute#75]
+
+(103) BroadcastExchange
+Input [1]: [t_time_sk#73]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#76]
+
+(104) BroadcastHashJoin [codegen id : 23]
+Left keys [1]: [ss_sold_time_sk#69]
+Right keys [1]: [t_time_sk#73]
+Join condition: None
+
+(105) Project [codegen id : 23]
+Output [2]: [ss_hdemo_sk#70, ss_store_sk#71]
+Input [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, t_time_sk#73]
+
+(106) ReusedExchange [Reuses operator id: 16]
+Output [1]: [s_store_sk#77]
+
+(107) BroadcastHashJoin [codegen id : 23]
+Left keys [1]: [ss_store_sk#71]
+Right keys [1]: [s_store_sk#77]
+Join condition: None
+
+(108) Project [codegen id : 23]
+Output [1]: [ss_hdemo_sk#70]
+Input [3]: [ss_hdemo_sk#70, ss_store_sk#71, s_store_sk#77]
+
+(109) ReusedExchange [Reuses operator id: 23]
+Output [1]: [hd_demo_sk#78]
+
+(110) BroadcastHashJoin [codegen id : 23]
+Left keys [1]: [ss_hdemo_sk#70]
+Right keys [1]: [hd_demo_sk#78]
+Join condition: None
+
+(111) Project [codegen id : 23]
+Output: []
+Input [2]: [ss_hdemo_sk#70, hd_demo_sk#78]
+
+(112) HashAggregate [codegen id : 23]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#79]
+Results [1]: [count#80]
+
+(113) Exchange
+Input [1]: [count#80]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#81]
+
+(114) HashAggregate [codegen id : 24]
+Input [1]: [count#80]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#82]
+Results [1]: [count(1)#82 AS h10_30_to_11#83]
+
+(115) BroadcastExchange
+Input [1]: [h10_30_to_11#83]
+Arguments: IdentityBroadcastMode, [id=#84]
+
+(116) BroadcastNestedLoopJoin [codegen id : 40]
+Join condition: None
+
+(117) Scan parquet default.store_sales
+Output [4]: [ss_sold_time_sk#85, ss_hdemo_sk#86, ss_store_sk#87, ss_sold_date_sk#88]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_sold_time_sk:int,ss_hdemo_sk:int,ss_store_sk:int>
+
+(118) ColumnarToRow [codegen id : 28]
+Input [4]: [ss_sold_time_sk#85, ss_hdemo_sk#86, ss_store_sk#87, ss_sold_date_sk#88]
+
+(119) Filter [codegen id : 28]
+Input [4]: [ss_sold_time_sk#85, ss_hdemo_sk#86, ss_store_sk#87, ss_sold_date_sk#88]
+Condition : ((isnotnull(ss_hdemo_sk#86) AND isnotnull(ss_sold_time_sk#85)) AND isnotnull(ss_store_sk#87))
+
+(120) Project [codegen id : 28]
+Output [3]: [ss_sold_time_sk#85, ss_hdemo_sk#86, ss_store_sk#87]
+Input [4]: [ss_sold_time_sk#85, ss_hdemo_sk#86, ss_store_sk#87, ss_sold_date_sk#88]
+
+(121) Scan parquet default.time_dim
+Output [3]: [t_time_sk#89, t_hour#90, t_minute#91]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,11), LessThan(t_minute,30), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_hour:int,t_minute:int>
+
+(122) ColumnarToRow [codegen id : 25]
+Input [3]: [t_time_sk#89, t_hour#90, t_minute#91]
+
+(123) Filter [codegen id : 25]
+Input [3]: [t_time_sk#89, t_hour#90, t_minute#91]
+Condition : ((((isnotnull(t_hour#90) AND isnotnull(t_minute#91)) AND (t_hour#90 = 11)) AND (t_minute#91 < 30)) AND isnotnull(t_time_sk#89))
+
+(124) Project [codegen id : 25]
+Output [1]: [t_time_sk#89]
+Input [3]: [t_time_sk#89, t_hour#90, t_minute#91]
+
+(125) BroadcastExchange
+Input [1]: [t_time_sk#89]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#92]
+
+(126) BroadcastHashJoin [codegen id : 28]
+Left keys [1]: [ss_sold_time_sk#85]
+Right keys [1]: [t_time_sk#89]
+Join condition: None
+
+(127) Project [codegen id : 28]
+Output [2]: [ss_hdemo_sk#86, ss_store_sk#87]
+Input [4]: [ss_sold_time_sk#85, ss_hdemo_sk#86, ss_store_sk#87, t_time_sk#89]
+
+(128) ReusedExchange [Reuses operator id: 16]
+Output [1]: [s_store_sk#93]
+
+(129) BroadcastHashJoin [codegen id : 28]
+Left keys [1]: [ss_store_sk#87]
+Right keys [1]: [s_store_sk#93]
+Join condition: None
+
+(130) Project [codegen id : 28]
+Output [1]: [ss_hdemo_sk#86]
+Input [3]: [ss_hdemo_sk#86, ss_store_sk#87, s_store_sk#93]
+
+(131) ReusedExchange [Reuses operator id: 23]
+Output [1]: [hd_demo_sk#94]
+
+(132) BroadcastHashJoin [codegen id : 28]
+Left keys [1]: [ss_hdemo_sk#86]
+Right keys [1]: [hd_demo_sk#94]
+Join condition: None
+
+(133) Project [codegen id : 28]
+Output: []
+Input [2]: [ss_hdemo_sk#86, hd_demo_sk#94]
+
+(134) HashAggregate [codegen id : 28]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#95]
+Results [1]: [count#96]
+
+(135) Exchange
+Input [1]: [count#96]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#97]
+
+(136) HashAggregate [codegen id : 29]
+Input [1]: [count#96]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#98]
+Results [1]: [count(1)#98 AS h11_to_11_30#99]
+
+(137) BroadcastExchange
+Input [1]: [h11_to_11_30#99]
+Arguments: IdentityBroadcastMode, [id=#100]
+
+(138) BroadcastNestedLoopJoin [codegen id : 40]
+Join condition: None
+
+(139) Scan parquet default.store_sales
+Output [4]: [ss_sold_time_sk#101, ss_hdemo_sk#102, ss_store_sk#103, ss_sold_date_sk#104]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_sold_time_sk:int,ss_hdemo_sk:int,ss_store_sk:int>
+
+(140) ColumnarToRow [codegen id : 33]
+Input [4]: [ss_sold_time_sk#101, ss_hdemo_sk#102, ss_store_sk#103, ss_sold_date_sk#104]
+
+(141) Filter [codegen id : 33]
+Input [4]: [ss_sold_time_sk#101, ss_hdemo_sk#102, ss_store_sk#103, ss_sold_date_sk#104]
+Condition : ((isnotnull(ss_hdemo_sk#102) AND isnotnull(ss_sold_time_sk#101)) AND isnotnull(ss_store_sk#103))
+
+(142) Project [codegen id : 33]
+Output [3]: [ss_sold_time_sk#101, ss_hdemo_sk#102, ss_store_sk#103]
+Input [4]: [ss_sold_time_sk#101, ss_hdemo_sk#102, ss_store_sk#103, ss_sold_date_sk#104]
+
+(143) Scan parquet default.time_dim
+Output [3]: [t_time_sk#105, t_hour#106, t_minute#107]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,11), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_hour:int,t_minute:int>
+
+(144) ColumnarToRow [codegen id : 30]
+Input [3]: [t_time_sk#105, t_hour#106, t_minute#107]
+
+(145) Filter [codegen id : 30]
+Input [3]: [t_time_sk#105, t_hour#106, t_minute#107]
+Condition : ((((isnotnull(t_hour#106) AND isnotnull(t_minute#107)) AND (t_hour#106 = 11)) AND (t_minute#107 >= 30)) AND isnotnull(t_time_sk#105))
+
+(146) Project [codegen id : 30]
+Output [1]: [t_time_sk#105]
+Input [3]: [t_time_sk#105, t_hour#106, t_minute#107]
+
+(147) BroadcastExchange
+Input [1]: [t_time_sk#105]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#108]
+
+(148) BroadcastHashJoin [codegen id : 33]
+Left keys [1]: [ss_sold_time_sk#101]
+Right keys [1]: [t_time_sk#105]
+Join condition: None
+
+(149) Project [codegen id : 33]
+Output [2]: [ss_hdemo_sk#102, ss_store_sk#103]
+Input [4]: [ss_sold_time_sk#101, ss_hdemo_sk#102, ss_store_sk#103, t_time_sk#105]
+
+(150) ReusedExchange [Reuses operator id: 16]
+Output [1]: [s_store_sk#109]
+
+(151) BroadcastHashJoin [codegen id : 33]
+Left keys [1]: [ss_store_sk#103]
+Right keys [1]: [s_store_sk#109]
+Join condition: None
+
+(152) Project [codegen id : 33]
+Output [1]: [ss_hdemo_sk#102]
+Input [3]: [ss_hdemo_sk#102, ss_store_sk#103, s_store_sk#109]
+
+(153) ReusedExchange [Reuses operator id: 23]
+Output [1]: [hd_demo_sk#110]
+
+(154) BroadcastHashJoin [codegen id : 33]
+Left keys [1]: [ss_hdemo_sk#102]
+Right keys [1]: [hd_demo_sk#110]
+Join condition: None
+
+(155) Project [codegen id : 33]
+Output: []
+Input [2]: [ss_hdemo_sk#102, hd_demo_sk#110]
+
+(156) HashAggregate [codegen id : 33]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#111]
+Results [1]: [count#112]
+
+(157) Exchange
+Input [1]: [count#112]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#113]
+
+(158) HashAggregate [codegen id : 34]
+Input [1]: [count#112]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#114]
+Results [1]: [count(1)#114 AS h11_30_to_12#115]
+
+(159) BroadcastExchange
+Input [1]: [h11_30_to_12#115]
+Arguments: IdentityBroadcastMode, [id=#116]
+
+(160) BroadcastNestedLoopJoin [codegen id : 40]
+Join condition: None
+
+(161) Scan parquet default.store_sales
+Output [4]: [ss_sold_time_sk#117, ss_hdemo_sk#118, ss_store_sk#119, ss_sold_date_sk#120]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_sold_time_sk:int,ss_hdemo_sk:int,ss_store_sk:int>
+
+(162) ColumnarToRow [codegen id : 38]
+Input [4]: [ss_sold_time_sk#117, ss_hdemo_sk#118, ss_store_sk#119, ss_sold_date_sk#120]
+
+(163) Filter [codegen id : 38]
+Input [4]: [ss_sold_time_sk#117, ss_hdemo_sk#118, ss_store_sk#119, ss_sold_date_sk#120]
+Condition : ((isnotnull(ss_hdemo_sk#118) AND isnotnull(ss_sold_time_sk#117)) AND isnotnull(ss_store_sk#119))
+
+(164) Project [codegen id : 38]
+Output [3]: [ss_sold_time_sk#117, ss_hdemo_sk#118, ss_store_sk#119]
+Input [4]: [ss_sold_time_sk#117, ss_hdemo_sk#118, ss_store_sk#119, ss_sold_date_sk#120]
+
+(165) Scan parquet default.time_dim
+Output [3]: [t_time_sk#121, t_hour#122, t_minute#123]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,12), LessThan(t_minute,30), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_hour:int,t_minute:int>
+
+(166) ColumnarToRow [codegen id : 35]
+Input [3]: [t_time_sk#121, t_hour#122, t_minute#123]
+
+(167) Filter [codegen id : 35]
+Input [3]: [t_time_sk#121, t_hour#122, t_minute#123]
+Condition : ((((isnotnull(t_hour#122) AND isnotnull(t_minute#123)) AND (t_hour#122 = 12)) AND (t_minute#123 < 30)) AND isnotnull(t_time_sk#121))
+
+(168) Project [codegen id : 35]
+Output [1]: [t_time_sk#121]
+Input [3]: [t_time_sk#121, t_hour#122, t_minute#123]
+
+(169) BroadcastExchange
+Input [1]: [t_time_sk#121]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#124]
+
+(170) BroadcastHashJoin [codegen id : 38]
+Left keys [1]: [ss_sold_time_sk#117]
+Right keys [1]: [t_time_sk#121]
+Join condition: None
+
+(171) Project [codegen id : 38]
+Output [2]: [ss_hdemo_sk#118, ss_store_sk#119]
+Input [4]: [ss_sold_time_sk#117, ss_hdemo_sk#118, ss_store_sk#119, t_time_sk#121]
+
+(172) ReusedExchange [Reuses operator id: 16]
+Output [1]: [s_store_sk#125]
+
+(173) BroadcastHashJoin [codegen id : 38]
+Left keys [1]: [ss_store_sk#119]
+Right keys [1]: [s_store_sk#125]
+Join condition: None
+
+(174) Project [codegen id : 38]
+Output [1]: [ss_hdemo_sk#118]
+Input [3]: [ss_hdemo_sk#118, ss_store_sk#119, s_store_sk#125]
+
+(175) ReusedExchange [Reuses operator id: 23]
+Output [1]: [hd_demo_sk#126]
+
+(176) BroadcastHashJoin [codegen id : 38]
+Left keys [1]: [ss_hdemo_sk#118]
+Right keys [1]: [hd_demo_sk#126]
+Join condition: None
+
+(177) Project [codegen id : 38]
+Output: []
+Input [2]: [ss_hdemo_sk#118, hd_demo_sk#126]
+
+(178) HashAggregate [codegen id : 38]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#127]
+Results [1]: [count#128]
+
+(179) Exchange
+Input [1]: [count#128]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#129]
+
+(180) HashAggregate [codegen id : 39]
+Input [1]: [count#128]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#130]
+Results [1]: [count(1)#130 AS h12_to_12_30#131]
+
+(181) BroadcastExchange
+Input [1]: [h12_to_12_30#131]
+Arguments: IdentityBroadcastMode, [id=#132]
+
+(182) BroadcastNestedLoopJoin [codegen id : 40]
+Join condition: None
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.sf100/simplified.txt
new file mode 100644
index 0000000000000..41fef687d30be
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.sf100/simplified.txt
@@ -0,0 +1,265 @@
+WholeStageCodegen (40)
+  BroadcastNestedLoopJoin
+    BroadcastNestedLoopJoin
+      BroadcastNestedLoopJoin
+        BroadcastNestedLoopJoin
+          BroadcastNestedLoopJoin
+            BroadcastNestedLoopJoin
+              BroadcastNestedLoopJoin
+                HashAggregate [count] [count(1),h8_30_to_9,count]
+                  InputAdapter
+                    Exchange #1
+                      WholeStageCodegen (4)
+                        HashAggregate [count,count]
+                          Project
+                            BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                              Project [ss_hdemo_sk]
+                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                  Project [ss_hdemo_sk,ss_store_sk]
+                                    BroadcastHashJoin [ss_sold_time_sk,t_time_sk]
+                                      Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk]
+                                        Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk]
+                                      InputAdapter
+                                        BroadcastExchange #2
+                                          WholeStageCodegen (1)
+                                            Project [t_time_sk]
+                                              Filter [t_hour,t_minute,t_time_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.time_dim [t_time_sk,t_hour,t_minute]
+                                  InputAdapter
+                                    BroadcastExchange #3
+                                      WholeStageCodegen (2)
+                                        Project [s_store_sk]
+                                          Filter [s_store_name,s_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store [s_store_sk,s_store_name]
+                              InputAdapter
+                                BroadcastExchange #4
+                                  WholeStageCodegen (3)
+                                    Project [hd_demo_sk]
+                                      Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count]
+                InputAdapter
+                  BroadcastExchange #5
+                    WholeStageCodegen (9)
+                      HashAggregate [count] [count(1),h9_to_9_30,count]
+                        InputAdapter
+                          Exchange #6
+                            WholeStageCodegen (8)
+                              HashAggregate [count,count]
+                                Project
+                                  BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                    Project [ss_hdemo_sk]
+                                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                        Project [ss_hdemo_sk,ss_store_sk]
+                                          BroadcastHashJoin [ss_sold_time_sk,t_time_sk]
+                                            Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk]
+                                              Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk]
+                                            InputAdapter
+                                              BroadcastExchange #7
+                                                WholeStageCodegen (5)
+                                                  Project [t_time_sk]
+                                                    Filter [t_hour,t_minute,t_time_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.time_dim [t_time_sk,t_hour,t_minute]
+                                        InputAdapter
+                                          ReusedExchange [s_store_sk] #3
+                                    InputAdapter
+                                      ReusedExchange [hd_demo_sk] #4
+              InputAdapter
+                BroadcastExchange #8
+                  WholeStageCodegen (14)
+                    HashAggregate [count] [count(1),h9_30_to_10,count]
+                      InputAdapter
+                        Exchange #9
+                          WholeStageCodegen (13)
+                            HashAggregate [count,count]
+                              Project
+                                BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                  Project [ss_hdemo_sk]
+                                    BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                      Project [ss_hdemo_sk,ss_store_sk]
+                                        BroadcastHashJoin [ss_sold_time_sk,t_time_sk]
+                                          Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk]
+                                            Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk]
+                                          InputAdapter
+                                            BroadcastExchange #10
+                                              WholeStageCodegen (10)
+                                                Project [t_time_sk]
+                                                  Filter [t_hour,t_minute,t_time_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.time_dim [t_time_sk,t_hour,t_minute]
+                                      InputAdapter
+                                        ReusedExchange [s_store_sk] #3
+                                  InputAdapter
+                                    ReusedExchange [hd_demo_sk] #4
+            InputAdapter
+              BroadcastExchange #11
+                WholeStageCodegen (19)
+                  HashAggregate [count] [count(1),h10_to_10_30,count]
+                    InputAdapter
+                      Exchange #12
+                        WholeStageCodegen (18)
+                          HashAggregate [count,count]
+                            Project
+                              BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                Project [ss_hdemo_sk]
+                                  BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                    Project [ss_hdemo_sk,ss_store_sk]
+                                      BroadcastHashJoin [ss_sold_time_sk,t_time_sk]
+                                        Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk]
+                                          Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk]
+                                        InputAdapter
+                                          BroadcastExchange #13
+                                            WholeStageCodegen (15)
+                                              Project [t_time_sk]
+                                                Filter [t_hour,t_minute,t_time_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.time_dim [t_time_sk,t_hour,t_minute]
+                                    InputAdapter
+                                      ReusedExchange [s_store_sk] #3
+                                InputAdapter
+                                  ReusedExchange [hd_demo_sk] #4
+          InputAdapter
+            BroadcastExchange #14
+              WholeStageCodegen (24)
+                HashAggregate [count] [count(1),h10_30_to_11,count]
+                  InputAdapter
+                    Exchange #15
+                      WholeStageCodegen (23)
+                        HashAggregate [count,count]
+                          Project
+                            BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                              Project [ss_hdemo_sk]
+                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                  Project [ss_hdemo_sk,ss_store_sk]
+                                    BroadcastHashJoin [ss_sold_time_sk,t_time_sk]
+                                      Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk]
+                                        Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk]
+                                      InputAdapter
+                                        BroadcastExchange #16
+                                          WholeStageCodegen (20)
+                                            Project [t_time_sk]
+                                              Filter [t_hour,t_minute,t_time_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.time_dim [t_time_sk,t_hour,t_minute]
+                                  InputAdapter
+                                    ReusedExchange [s_store_sk] #3
+                              InputAdapter
+                                ReusedExchange [hd_demo_sk] #4
+        InputAdapter
+          BroadcastExchange #17
+            WholeStageCodegen (29)
+              HashAggregate [count] [count(1),h11_to_11_30,count]
+                InputAdapter
+                  Exchange #18
+                    WholeStageCodegen (28)
+                      HashAggregate [count,count]
+                        Project
+                          BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                            Project [ss_hdemo_sk]
+                              BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                Project [ss_hdemo_sk,ss_store_sk]
+                                  BroadcastHashJoin [ss_sold_time_sk,t_time_sk]
+                                    Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk]
+                                      Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk]
+                                    InputAdapter
+                                      BroadcastExchange #19
+                                        WholeStageCodegen (25)
+                                          Project [t_time_sk]
+                                            Filter [t_hour,t_minute,t_time_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.time_dim [t_time_sk,t_hour,t_minute]
+                                InputAdapter
+                                  ReusedExchange [s_store_sk] #3
+                            InputAdapter
+                              ReusedExchange [hd_demo_sk] #4
+      InputAdapter
+        BroadcastExchange #20
+          WholeStageCodegen (34)
+            HashAggregate [count] [count(1),h11_30_to_12,count]
+              InputAdapter
+                Exchange #21
+                  WholeStageCodegen (33)
+                    HashAggregate [count,count]
+                      Project
+                        BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                          Project [ss_hdemo_sk]
+                            BroadcastHashJoin [ss_store_sk,s_store_sk]
+                              Project [ss_hdemo_sk,ss_store_sk]
+                                BroadcastHashJoin [ss_sold_time_sk,t_time_sk]
+                                  Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk]
+                                    Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk]
+                                  InputAdapter
+                                    BroadcastExchange #22
+                                      WholeStageCodegen (30)
+                                        Project [t_time_sk]
+                                          Filter [t_hour,t_minute,t_time_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.time_dim [t_time_sk,t_hour,t_minute]
+                              InputAdapter
+                                ReusedExchange [s_store_sk] #3
+                          InputAdapter
+                            ReusedExchange [hd_demo_sk] #4
+    InputAdapter
+      BroadcastExchange #23
+        WholeStageCodegen (39)
+          HashAggregate [count] [count(1),h12_to_12_30,count]
+            InputAdapter
+              Exchange #24
+                WholeStageCodegen (38)
+                  HashAggregate [count,count]
+                    Project
+                      BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                        Project [ss_hdemo_sk]
+                          BroadcastHashJoin [ss_store_sk,s_store_sk]
+                            Project [ss_hdemo_sk,ss_store_sk]
+                              BroadcastHashJoin [ss_sold_time_sk,t_time_sk]
+                                Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk]
+                                  Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk]
+                                InputAdapter
+                                  BroadcastExchange #25
+                                    WholeStageCodegen (35)
+                                      Project [t_time_sk]
+                                        Filter [t_hour,t_minute,t_time_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.time_dim [t_time_sk,t_hour,t_minute]
+                            InputAdapter
+                              ReusedExchange [s_store_sk] #3
+                        InputAdapter
+                          ReusedExchange [hd_demo_sk] #4
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/explain.txt
new file mode 100644
index 0000000000000..9f56c71154a66
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/explain.txt
@@ -0,0 +1,1000 @@
+== Physical Plan ==
+* BroadcastNestedLoopJoin Inner BuildRight (182)
+:- * BroadcastNestedLoopJoin Inner BuildRight (160)
+:  :- * BroadcastNestedLoopJoin Inner BuildRight (138)
+:  :  :- * BroadcastNestedLoopJoin Inner BuildRight (116)
+:  :  :  :- * BroadcastNestedLoopJoin Inner BuildRight (94)
+:  :  :  :  :- * BroadcastNestedLoopJoin Inner BuildRight (72)
+:  :  :  :  :  :- * BroadcastNestedLoopJoin Inner BuildRight (50)
+:  :  :  :  :  :  :- * HashAggregate (28)
+:  :  :  :  :  :  :  +- Exchange (27)
+:  :  :  :  :  :  :     +- * HashAggregate (26)
+:  :  :  :  :  :  :        +- * Project (25)
+:  :  :  :  :  :  :           +- * BroadcastHashJoin Inner BuildRight (24)
+:  :  :  :  :  :  :              :- * Project (18)
+:  :  :  :  :  :  :              :  +- * BroadcastHashJoin Inner BuildRight (17)
+:  :  :  :  :  :  :              :     :- * Project (11)
+:  :  :  :  :  :  :              :     :  +- * BroadcastHashJoin Inner BuildRight (10)
+:  :  :  :  :  :  :              :     :     :- * Project (4)
+:  :  :  :  :  :  :              :     :     :  +- * Filter (3)
+:  :  :  :  :  :  :              :     :     :     +- * ColumnarToRow (2)
+:  :  :  :  :  :  :              :     :     :        +- Scan parquet default.store_sales (1)
+:  :  :  :  :  :  :              :     :     +- BroadcastExchange (9)
+:  :  :  :  :  :  :              :     :        +- * Project (8)
+:  :  :  :  :  :  :              :     :           +- * Filter (7)
+:  :  :  :  :  :  :              :     :              +- * ColumnarToRow (6)
+:  :  :  :  :  :  :              :     :                 +- Scan parquet default.household_demographics (5)
+:  :  :  :  :  :  :              :     +- BroadcastExchange (16)
+:  :  :  :  :  :  :              :        +- * Project (15)
+:  :  :  :  :  :  :              :           +- * Filter (14)
+:  :  :  :  :  :  :              :              +- * ColumnarToRow (13)
+:  :  :  :  :  :  :              :                 +- Scan parquet default.time_dim (12)
+:  :  :  :  :  :  :              +- BroadcastExchange (23)
+:  :  :  :  :  :  :                 +- * Project (22)
+:  :  :  :  :  :  :                    +- * Filter (21)
+:  :  :  :  :  :  :                       +- * ColumnarToRow (20)
+:  :  :  :  :  :  :                          +- Scan parquet default.store (19)
+:  :  :  :  :  :  +- BroadcastExchange (49)
+:  :  :  :  :  :     +- * HashAggregate (48)
+:  :  :  :  :  :        +- Exchange (47)
+:  :  :  :  :  :           +- * HashAggregate (46)
+:  :  :  :  :  :              +- * Project (45)
+:  :  :  :  :  :                 +- * BroadcastHashJoin Inner BuildRight (44)
+:  :  :  :  :  :                    :- * Project (42)
+:  :  :  :  :  :                    :  +- * BroadcastHashJoin Inner BuildRight (41)
+:  :  :  :  :  :                    :     :- * Project (35)
+:  :  :  :  :  :                    :     :  +- * BroadcastHashJoin Inner BuildRight (34)
+:  :  :  :  :  :                    :     :     :- * Project (32)
+:  :  :  :  :  :                    :     :     :  +- * Filter (31)
+:  :  :  :  :  :                    :     :     :     +- * ColumnarToRow (30)
+:  :  :  :  :  :                    :     :     :        +- Scan parquet default.store_sales (29)
+:  :  :  :  :  :                    :     :     +- ReusedExchange (33)
+:  :  :  :  :  :                    :     +- BroadcastExchange (40)
+:  :  :  :  :  :                    :        +- * Project (39)
+:  :  :  :  :  :                    :           +- * Filter (38)
+:  :  :  :  :  :                    :              +- * ColumnarToRow (37)
+:  :  :  :  :  :                    :                 +- Scan parquet default.time_dim (36)
+:  :  :  :  :  :                    +- ReusedExchange (43)
+:  :  :  :  :  +- BroadcastExchange (71)
+:  :  :  :  :     +- * HashAggregate (70)
+:  :  :  :  :        +- Exchange (69)
+:  :  :  :  :           +- * HashAggregate (68)
+:  :  :  :  :              +- * Project (67)
+:  :  :  :  :                 +- * BroadcastHashJoin Inner BuildRight (66)
+:  :  :  :  :                    :- * Project (64)
+:  :  :  :  :                    :  +- * BroadcastHashJoin Inner BuildRight (63)
+:  :  :  :  :                    :     :- * Project (57)
+:  :  :  :  :                    :     :  +- * BroadcastHashJoin Inner BuildRight (56)
+:  :  :  :  :                    :     :     :- * Project (54)
+:  :  :  :  :                    :     :     :  +- * Filter (53)
+:  :  :  :  :                    :     :     :     +- * ColumnarToRow (52)
+:  :  :  :  :                    :     :     :        +- Scan parquet default.store_sales (51)
+:  :  :  :  :                    :     :     +- ReusedExchange (55)
+:  :  :  :  :                    :     +- BroadcastExchange (62)
+:  :  :  :  :                    :        +- * Project (61)
+:  :  :  :  :                    :           +- * Filter (60)
+:  :  :  :  :                    :              +- * ColumnarToRow (59)
+:  :  :  :  :                    :                 +- Scan parquet default.time_dim (58)
+:  :  :  :  :                    +- ReusedExchange (65)
+:  :  :  :  +- BroadcastExchange (93)
+:  :  :  :     +- * HashAggregate (92)
+:  :  :  :        +- Exchange (91)
+:  :  :  :           +- * HashAggregate (90)
+:  :  :  :              +- * Project (89)
+:  :  :  :                 +- * BroadcastHashJoin Inner BuildRight (88)
+:  :  :  :                    :- * Project (86)
+:  :  :  :                    :  +- * BroadcastHashJoin Inner BuildRight (85)
+:  :  :  :                    :     :- * Project (79)
+:  :  :  :                    :     :  +- * BroadcastHashJoin Inner BuildRight (78)
+:  :  :  :                    :     :     :- * Project (76)
+:  :  :  :                    :     :     :  +- * Filter (75)
+:  :  :  :                    :     :     :     +- * ColumnarToRow (74)
+:  :  :  :                    :     :     :        +- Scan parquet default.store_sales (73)
+:  :  :  :                    :     :     +- ReusedExchange (77)
+:  :  :  :                    :     +- BroadcastExchange (84)
+:  :  :  :                    :        +- * Project (83)
+:  :  :  :                    :           +- * Filter (82)
+:  :  :  :                    :              +- * ColumnarToRow (81)
+:  :  :  :                    :                 +- Scan parquet default.time_dim (80)
+:  :  :  :                    +- ReusedExchange (87)
+:  :  :  +- BroadcastExchange (115)
+:  :  :     +- * HashAggregate (114)
+:  :  :        +- Exchange (113)
+:  :  :           +- * HashAggregate (112)
+:  :  :              +- * Project (111)
+:  :  :                 +- * BroadcastHashJoin Inner BuildRight (110)
+:  :  :                    :- * Project (108)
+:  :  :                    :  +- * BroadcastHashJoin Inner BuildRight (107)
+:  :  :                    :     :- * Project (101)
+:  :  :                    :     :  +- * BroadcastHashJoin Inner BuildRight (100)
+:  :  :                    :     :     :- * Project (98)
+:  :  :                    :     :     :  +- * Filter (97)
+:  :  :                    :     :     :     +- * ColumnarToRow (96)
+:  :  :                    :     :     :        +- Scan parquet default.store_sales (95)
+:  :  :                    :     :     +- ReusedExchange (99)
+:  :  :                    :     +- BroadcastExchange (106)
+:  :  :                    :        +- * Project (105)
+:  :  :                    :           +- * Filter (104)
+:  :  :                    :              +- * ColumnarToRow (103)
+:  :  :                    :                 +- Scan parquet default.time_dim (102)
+:  :  :                    +- ReusedExchange (109)
+:  :  +- BroadcastExchange (137)
+:  :     +- * HashAggregate (136)
+:  :        +- Exchange (135)
+:  :           +- * HashAggregate (134)
+:  :              +- * Project (133)
+:  :                 +- * BroadcastHashJoin Inner BuildRight (132)
+:  :                    :- * Project (130)
+:  :                    :  +- * BroadcastHashJoin Inner BuildRight (129)
+:  :                    :     :- * Project (123)
+:  :                    :     :  +- * BroadcastHashJoin Inner BuildRight (122)
+:  :                    :     :     :- * Project (120)
+:  :                    :     :     :  +- * Filter (119)
+:  :                    :     :     :     +- * ColumnarToRow (118)
+:  :                    :     :     :        +- Scan parquet default.store_sales (117)
+:  :                    :     :     +- ReusedExchange (121)
+:  :                    :     +- BroadcastExchange (128)
+:  :                    :        +- * Project (127)
+:  :                    :           +- * Filter (126)
+:  :                    :              +- * ColumnarToRow (125)
+:  :                    :                 +- Scan parquet default.time_dim (124)
+:  :                    +- ReusedExchange (131)
+:  +- BroadcastExchange (159)
+:     +- * HashAggregate (158)
+:        +- Exchange (157)
+:           +- * HashAggregate (156)
+:              +- * Project (155)
+:                 +- * BroadcastHashJoin Inner BuildRight (154)
+:                    :- * Project (152)
+:                    :  +- * BroadcastHashJoin Inner BuildRight (151)
+:                    :     :- * Project (145)
+:                    :     :  +- * BroadcastHashJoin Inner BuildRight (144)
+:                    :     :     :- * Project (142)
+:                    :     :     :  +- * Filter (141)
+:                    :     :     :     +- * ColumnarToRow (140)
+:                    :     :     :        +- Scan parquet default.store_sales (139)
+:                    :     :     +- ReusedExchange (143)
+:                    :     +- BroadcastExchange (150)
+:                    :        +- * Project (149)
+:                    :           +- * Filter (148)
+:                    :              +- * ColumnarToRow (147)
+:                    :                 +- Scan parquet default.time_dim (146)
+:                    +- ReusedExchange (153)
++- BroadcastExchange (181)
+   +- * HashAggregate (180)
+      +- Exchange (179)
+         +- * HashAggregate (178)
+            +- * Project (177)
+               +- * BroadcastHashJoin Inner BuildRight (176)
+                  :- * Project (174)
+                  :  +- * BroadcastHashJoin Inner BuildRight (173)
+                  :     :- * Project (167)
+                  :     :  +- * BroadcastHashJoin Inner BuildRight (166)
+                  :     :     :- * Project (164)
+                  :     :     :  +- * Filter (163)
+                  :     :     :     +- * ColumnarToRow (162)
+                  :     :     :        +- Scan parquet default.store_sales (161)
+                  :     :     +- ReusedExchange (165)
+                  :     +- BroadcastExchange (172)
+                  :        +- * Project (171)
+                  :           +- * Filter (170)
+                  :              +- * ColumnarToRow (169)
+                  :                 +- Scan parquet default.time_dim (168)
+                  +- ReusedExchange (175)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_sold_time_sk:int,ss_hdemo_sk:int,ss_store_sk:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 4]
+Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4]
+Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isnotnull(ss_store_sk#3))
+
+(4) Project [codegen id : 4]
+Output [3]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3]
+Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4]
+
+(5) Scan parquet default.household_demographics
+Output [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [Or(Or(And(EqualTo(hd_dep_count,4),LessThanOrEqual(hd_vehicle_count,6)),And(EqualTo(hd_dep_count,2),LessThanOrEqual(hd_vehicle_count,4))),And(EqualTo(hd_dep_count,0),LessThanOrEqual(hd_vehicle_count,2))), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_dep_count:int,hd_vehicle_count:int>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7]
+
+(7) Filter [codegen id : 1]
+Input [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7]
+Condition : (((((hd_dep_count#6 = 4) AND (hd_vehicle_count#7 <= 6)) OR ((hd_dep_count#6 = 2) AND (hd_vehicle_count#7 <= 4))) OR ((hd_dep_count#6 = 0) AND (hd_vehicle_count#7 <= 2))) AND isnotnull(hd_demo_sk#5))
+
+(8) Project [codegen id : 1]
+Output [1]: [hd_demo_sk#5]
+Input [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7]
+
+(9) BroadcastExchange
+Input [1]: [hd_demo_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#8]
+
+(10) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#5]
+Join condition: None
+
+(11) Project [codegen id : 4]
+Output [2]: [ss_sold_time_sk#1, ss_store_sk#3]
+Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5]
+
+(12) Scan parquet default.time_dim
+Output [3]: [t_time_sk#9, t_hour#10, t_minute#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,8), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_hour:int,t_minute:int>
+
+(13) ColumnarToRow [codegen id : 2]
+Input [3]: [t_time_sk#9, t_hour#10, t_minute#11]
+
+(14) Filter [codegen id : 2]
+Input [3]: [t_time_sk#9, t_hour#10, t_minute#11]
+Condition : ((((isnotnull(t_hour#10) AND isnotnull(t_minute#11)) AND (t_hour#10 = 8)) AND (t_minute#11 >= 30)) AND isnotnull(t_time_sk#9))
+
+(15) Project [codegen id : 2]
+Output [1]: [t_time_sk#9]
+Input [3]: [t_time_sk#9, t_hour#10, t_minute#11]
+
+(16) BroadcastExchange
+Input [1]: [t_time_sk#9]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(17) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_time_sk#1]
+Right keys [1]: [t_time_sk#9]
+Join condition: None
+
+(18) Project [codegen id : 4]
+Output [1]: [ss_store_sk#3]
+Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#9]
+
+(19) Scan parquet default.store
+Output [2]: [s_store_sk#13, s_store_name#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_name), EqualTo(s_store_name,ese), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string>
+
+(20) ColumnarToRow [codegen id : 3]
+Input [2]: [s_store_sk#13, s_store_name#14]
+
+(21) Filter [codegen id : 3]
+Input [2]: [s_store_sk#13, s_store_name#14]
+Condition : ((isnotnull(s_store_name#14) AND (s_store_name#14 = ese)) AND isnotnull(s_store_sk#13))
+
+(22) Project [codegen id : 3]
+Output [1]: [s_store_sk#13]
+Input [2]: [s_store_sk#13, s_store_name#14]
+
+(23) BroadcastExchange
+Input [1]: [s_store_sk#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#15]
+
+(24) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#13]
+Join condition: None
+
+(25) Project [codegen id : 4]
+Output: []
+Input [2]: [ss_store_sk#3, s_store_sk#13]
+
+(26) HashAggregate [codegen id : 4]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#16]
+Results [1]: [count#17]
+
+(27) Exchange
+Input [1]: [count#17]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#18]
+
+(28) HashAggregate [codegen id : 40]
+Input [1]: [count#17]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#19]
+Results [1]: [count(1)#19 AS h8_30_to_9#20]
+
+(29) Scan parquet default.store_sales
+Output [4]: [ss_sold_time_sk#21, ss_hdemo_sk#22, ss_store_sk#23, ss_sold_date_sk#24]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_sold_time_sk:int,ss_hdemo_sk:int,ss_store_sk:int>
+
+(30) ColumnarToRow [codegen id : 8]
+Input [4]: [ss_sold_time_sk#21, ss_hdemo_sk#22, ss_store_sk#23, ss_sold_date_sk#24]
+
+(31) Filter [codegen id : 8]
+Input [4]: [ss_sold_time_sk#21, ss_hdemo_sk#22, ss_store_sk#23, ss_sold_date_sk#24]
+Condition : ((isnotnull(ss_hdemo_sk#22) AND isnotnull(ss_sold_time_sk#21)) AND isnotnull(ss_store_sk#23))
+
+(32) Project [codegen id : 8]
+Output [3]: [ss_sold_time_sk#21, ss_hdemo_sk#22, ss_store_sk#23]
+Input [4]: [ss_sold_time_sk#21, ss_hdemo_sk#22, ss_store_sk#23, ss_sold_date_sk#24]
+
+(33) ReusedExchange [Reuses operator id: 9]
+Output [1]: [hd_demo_sk#25]
+
+(34) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_hdemo_sk#22]
+Right keys [1]: [hd_demo_sk#25]
+Join condition: None
+
+(35) Project [codegen id : 8]
+Output [2]: [ss_sold_time_sk#21, ss_store_sk#23]
+Input [4]: [ss_sold_time_sk#21, ss_hdemo_sk#22, ss_store_sk#23, hd_demo_sk#25]
+
+(36) Scan parquet default.time_dim
+Output [3]: [t_time_sk#26, t_hour#27, t_minute#28]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,9), LessThan(t_minute,30), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_hour:int,t_minute:int>
+
+(37) ColumnarToRow [codegen id : 6]
+Input [3]: [t_time_sk#26, t_hour#27, t_minute#28]
+
+(38) Filter [codegen id : 6]
+Input [3]: [t_time_sk#26, t_hour#27, t_minute#28]
+Condition : ((((isnotnull(t_hour#27) AND isnotnull(t_minute#28)) AND (t_hour#27 = 9)) AND (t_minute#28 < 30)) AND isnotnull(t_time_sk#26))
+
+(39) Project [codegen id : 6]
+Output [1]: [t_time_sk#26]
+Input [3]: [t_time_sk#26, t_hour#27, t_minute#28]
+
+(40) BroadcastExchange
+Input [1]: [t_time_sk#26]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#29]
+
+(41) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_sold_time_sk#21]
+Right keys [1]: [t_time_sk#26]
+Join condition: None
+
+(42) Project [codegen id : 8]
+Output [1]: [ss_store_sk#23]
+Input [3]: [ss_sold_time_sk#21, ss_store_sk#23, t_time_sk#26]
+
+(43) ReusedExchange [Reuses operator id: 23]
+Output [1]: [s_store_sk#30]
+
+(44) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ss_store_sk#23]
+Right keys [1]: [s_store_sk#30]
+Join condition: None
+
+(45) Project [codegen id : 8]
+Output: []
+Input [2]: [ss_store_sk#23, s_store_sk#30]
+
+(46) HashAggregate [codegen id : 8]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#31]
+Results [1]: [count#32]
+
+(47) Exchange
+Input [1]: [count#32]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#33]
+
+(48) HashAggregate [codegen id : 9]
+Input [1]: [count#32]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#34]
+Results [1]: [count(1)#34 AS h9_to_9_30#35]
+
+(49) BroadcastExchange
+Input [1]: [h9_to_9_30#35]
+Arguments: IdentityBroadcastMode, [id=#36]
+
+(50) BroadcastNestedLoopJoin [codegen id : 40]
+Join condition: None
+
+(51) Scan parquet default.store_sales
+Output [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_sold_time_sk:int,ss_hdemo_sk:int,ss_store_sk:int>
+
+(52) ColumnarToRow [codegen id : 13]
+Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40]
+
+(53) Filter [codegen id : 13]
+Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40]
+Condition : ((isnotnull(ss_hdemo_sk#38) AND isnotnull(ss_sold_time_sk#37)) AND isnotnull(ss_store_sk#39))
+
+(54) Project [codegen id : 13]
+Output [3]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39]
+Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40]
+
+(55) ReusedExchange [Reuses operator id: 9]
+Output [1]: [hd_demo_sk#41]
+
+(56) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ss_hdemo_sk#38]
+Right keys [1]: [hd_demo_sk#41]
+Join condition: None
+
+(57) Project [codegen id : 13]
+Output [2]: [ss_sold_time_sk#37, ss_store_sk#39]
+Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, hd_demo_sk#41]
+
+(58) Scan parquet default.time_dim
+Output [3]: [t_time_sk#42, t_hour#43, t_minute#44]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,9), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_hour:int,t_minute:int>
+
+(59) ColumnarToRow [codegen id : 11]
+Input [3]: [t_time_sk#42, t_hour#43, t_minute#44]
+
+(60) Filter [codegen id : 11]
+Input [3]: [t_time_sk#42, t_hour#43, t_minute#44]
+Condition : ((((isnotnull(t_hour#43) AND isnotnull(t_minute#44)) AND (t_hour#43 = 9)) AND (t_minute#44 >= 30)) AND isnotnull(t_time_sk#42))
+
+(61) Project [codegen id : 11]
+Output [1]: [t_time_sk#42]
+Input [3]: [t_time_sk#42, t_hour#43, t_minute#44]
+
+(62) BroadcastExchange
+Input [1]: [t_time_sk#42]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#45]
+
+(63) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ss_sold_time_sk#37]
+Right keys [1]: [t_time_sk#42]
+Join condition: None
+
+(64) Project [codegen id : 13]
+Output [1]: [ss_store_sk#39]
+Input [3]: [ss_sold_time_sk#37, ss_store_sk#39, t_time_sk#42]
+
+(65) ReusedExchange [Reuses operator id: 23]
+Output [1]: [s_store_sk#46]
+
+(66) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ss_store_sk#39]
+Right keys [1]: [s_store_sk#46]
+Join condition: None
+
+(67) Project [codegen id : 13]
+Output: []
+Input [2]: [ss_store_sk#39, s_store_sk#46]
+
+(68) HashAggregate [codegen id : 13]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#47]
+Results [1]: [count#48]
+
+(69) Exchange
+Input [1]: [count#48]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#49]
+
+(70) HashAggregate [codegen id : 14]
+Input [1]: [count#48]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#50]
+Results [1]: [count(1)#50 AS h9_30_to_10#51]
+
+(71) BroadcastExchange
+Input [1]: [h9_30_to_10#51]
+Arguments: IdentityBroadcastMode, [id=#52]
+
+(72) BroadcastNestedLoopJoin [codegen id : 40]
+Join condition: None
+
+(73) Scan parquet default.store_sales
+Output [4]: [ss_sold_time_sk#53, ss_hdemo_sk#54, ss_store_sk#55, ss_sold_date_sk#56]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_sold_time_sk:int,ss_hdemo_sk:int,ss_store_sk:int>
+
+(74) ColumnarToRow [codegen id : 18]
+Input [4]: [ss_sold_time_sk#53, ss_hdemo_sk#54, ss_store_sk#55, ss_sold_date_sk#56]
+
+(75) Filter [codegen id : 18]
+Input [4]: [ss_sold_time_sk#53, ss_hdemo_sk#54, ss_store_sk#55, ss_sold_date_sk#56]
+Condition : ((isnotnull(ss_hdemo_sk#54) AND isnotnull(ss_sold_time_sk#53)) AND isnotnull(ss_store_sk#55))
+
+(76) Project [codegen id : 18]
+Output [3]: [ss_sold_time_sk#53, ss_hdemo_sk#54, ss_store_sk#55]
+Input [4]: [ss_sold_time_sk#53, ss_hdemo_sk#54, ss_store_sk#55, ss_sold_date_sk#56]
+
+(77) ReusedExchange [Reuses operator id: 9]
+Output [1]: [hd_demo_sk#57]
+
+(78) BroadcastHashJoin [codegen id : 18]
+Left keys [1]: [ss_hdemo_sk#54]
+Right keys [1]: [hd_demo_sk#57]
+Join condition: None
+
+(79) Project [codegen id : 18]
+Output [2]: [ss_sold_time_sk#53, ss_store_sk#55]
+Input [4]: [ss_sold_time_sk#53, ss_hdemo_sk#54, ss_store_sk#55, hd_demo_sk#57]
+
+(80) Scan parquet default.time_dim
+Output [3]: [t_time_sk#58, t_hour#59, t_minute#60]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,10), LessThan(t_minute,30), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_hour:int,t_minute:int>
+
+(81) ColumnarToRow [codegen id : 16]
+Input [3]: [t_time_sk#58, t_hour#59, t_minute#60]
+
+(82) Filter [codegen id : 16]
+Input [3]: [t_time_sk#58, t_hour#59, t_minute#60]
+Condition : ((((isnotnull(t_hour#59) AND isnotnull(t_minute#60)) AND (t_hour#59 = 10)) AND (t_minute#60 < 30)) AND isnotnull(t_time_sk#58))
+
+(83) Project [codegen id : 16]
+Output [1]: [t_time_sk#58]
+Input [3]: [t_time_sk#58, t_hour#59, t_minute#60]
+
+(84) BroadcastExchange
+Input [1]: [t_time_sk#58]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#61]
+
+(85) BroadcastHashJoin [codegen id : 18]
+Left keys [1]: [ss_sold_time_sk#53]
+Right keys [1]: [t_time_sk#58]
+Join condition: None
+
+(86) Project [codegen id : 18]
+Output [1]: [ss_store_sk#55]
+Input [3]: [ss_sold_time_sk#53, ss_store_sk#55, t_time_sk#58]
+
+(87) ReusedExchange [Reuses operator id: 23]
+Output [1]: [s_store_sk#62]
+
+(88) BroadcastHashJoin [codegen id : 18]
+Left keys [1]: [ss_store_sk#55]
+Right keys [1]: [s_store_sk#62]
+Join condition: None
+
+(89) Project [codegen id : 18]
+Output: []
+Input [2]: [ss_store_sk#55, s_store_sk#62]
+
+(90) HashAggregate [codegen id : 18]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#63]
+Results [1]: [count#64]
+
+(91) Exchange
+Input [1]: [count#64]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#65]
+
+(92) HashAggregate [codegen id : 19]
+Input [1]: [count#64]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#66]
+Results [1]: [count(1)#66 AS h10_to_10_30#67]
+
+(93) BroadcastExchange
+Input [1]: [h10_to_10_30#67]
+Arguments: IdentityBroadcastMode, [id=#68]
+
+(94) BroadcastNestedLoopJoin [codegen id : 40]
+Join condition: None
+
+(95) Scan parquet default.store_sales
+Output [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, ss_sold_date_sk#72]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_sold_time_sk:int,ss_hdemo_sk:int,ss_store_sk:int>
+
+(96) ColumnarToRow [codegen id : 23]
+Input [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, ss_sold_date_sk#72]
+
+(97) Filter [codegen id : 23]
+Input [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, ss_sold_date_sk#72]
+Condition : ((isnotnull(ss_hdemo_sk#70) AND isnotnull(ss_sold_time_sk#69)) AND isnotnull(ss_store_sk#71))
+
+(98) Project [codegen id : 23]
+Output [3]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71]
+Input [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, ss_sold_date_sk#72]
+
+(99) ReusedExchange [Reuses operator id: 9]
+Output [1]: [hd_demo_sk#73]
+
+(100) BroadcastHashJoin [codegen id : 23]
+Left keys [1]: [ss_hdemo_sk#70]
+Right keys [1]: [hd_demo_sk#73]
+Join condition: None
+
+(101) Project [codegen id : 23]
+Output [2]: [ss_sold_time_sk#69, ss_store_sk#71]
+Input [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, hd_demo_sk#73]
+
+(102) Scan parquet default.time_dim
+Output [3]: [t_time_sk#74, t_hour#75, t_minute#76]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,10), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_hour:int,t_minute:int>
+
+(103) ColumnarToRow [codegen id : 21]
+Input [3]: [t_time_sk#74, t_hour#75, t_minute#76]
+
+(104) Filter [codegen id : 21]
+Input [3]: [t_time_sk#74, t_hour#75, t_minute#76]
+Condition : ((((isnotnull(t_hour#75) AND isnotnull(t_minute#76)) AND (t_hour#75 = 10)) AND (t_minute#76 >= 30)) AND isnotnull(t_time_sk#74))
+
+(105) Project [codegen id : 21]
+Output [1]: [t_time_sk#74]
+Input [3]: [t_time_sk#74, t_hour#75, t_minute#76]
+
+(106) BroadcastExchange
+Input [1]: [t_time_sk#74]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#77]
+
+(107) BroadcastHashJoin [codegen id : 23]
+Left keys [1]: [ss_sold_time_sk#69]
+Right keys [1]: [t_time_sk#74]
+Join condition: None
+
+(108) Project [codegen id : 23]
+Output [1]: [ss_store_sk#71]
+Input [3]: [ss_sold_time_sk#69, ss_store_sk#71, t_time_sk#74]
+
+(109) ReusedExchange [Reuses operator id: 23]
+Output [1]: [s_store_sk#78]
+
+(110) BroadcastHashJoin [codegen id : 23]
+Left keys [1]: [ss_store_sk#71]
+Right keys [1]: [s_store_sk#78]
+Join condition: None
+
+(111) Project [codegen id : 23]
+Output: []
+Input [2]: [ss_store_sk#71, s_store_sk#78]
+
+(112) HashAggregate [codegen id : 23]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#79]
+Results [1]: [count#80]
+
+(113) Exchange
+Input [1]: [count#80]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#81]
+
+(114) HashAggregate [codegen id : 24]
+Input [1]: [count#80]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#82]
+Results [1]: [count(1)#82 AS h10_30_to_11#83]
+
+(115) BroadcastExchange
+Input [1]: [h10_30_to_11#83]
+Arguments: IdentityBroadcastMode, [id=#84]
+
+(116) BroadcastNestedLoopJoin [codegen id : 40]
+Join condition: None
+
+(117) Scan parquet default.store_sales
+Output [4]: [ss_sold_time_sk#85, ss_hdemo_sk#86, ss_store_sk#87, ss_sold_date_sk#88]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_sold_time_sk:int,ss_hdemo_sk:int,ss_store_sk:int>
+
+(118) ColumnarToRow [codegen id : 28]
+Input [4]: [ss_sold_time_sk#85, ss_hdemo_sk#86, ss_store_sk#87, ss_sold_date_sk#88]
+
+(119) Filter [codegen id : 28]
+Input [4]: [ss_sold_time_sk#85, ss_hdemo_sk#86, ss_store_sk#87, ss_sold_date_sk#88]
+Condition : ((isnotnull(ss_hdemo_sk#86) AND isnotnull(ss_sold_time_sk#85)) AND isnotnull(ss_store_sk#87))
+
+(120) Project [codegen id : 28]
+Output [3]: [ss_sold_time_sk#85, ss_hdemo_sk#86, ss_store_sk#87]
+Input [4]: [ss_sold_time_sk#85, ss_hdemo_sk#86, ss_store_sk#87, ss_sold_date_sk#88]
+
+(121) ReusedExchange [Reuses operator id: 9]
+Output [1]: [hd_demo_sk#89]
+
+(122) BroadcastHashJoin [codegen id : 28]
+Left keys [1]: [ss_hdemo_sk#86]
+Right keys [1]: [hd_demo_sk#89]
+Join condition: None
+
+(123) Project [codegen id : 28]
+Output [2]: [ss_sold_time_sk#85, ss_store_sk#87]
+Input [4]: [ss_sold_time_sk#85, ss_hdemo_sk#86, ss_store_sk#87, hd_demo_sk#89]
+
+(124) Scan parquet default.time_dim
+Output [3]: [t_time_sk#90, t_hour#91, t_minute#92]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,11), LessThan(t_minute,30), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_hour:int,t_minute:int>
+
+(125) ColumnarToRow [codegen id : 26]
+Input [3]: [t_time_sk#90, t_hour#91, t_minute#92]
+
+(126) Filter [codegen id : 26]
+Input [3]: [t_time_sk#90, t_hour#91, t_minute#92]
+Condition : ((((isnotnull(t_hour#91) AND isnotnull(t_minute#92)) AND (t_hour#91 = 11)) AND (t_minute#92 < 30)) AND isnotnull(t_time_sk#90))
+
+(127) Project [codegen id : 26]
+Output [1]: [t_time_sk#90]
+Input [3]: [t_time_sk#90, t_hour#91, t_minute#92]
+
+(128) BroadcastExchange
+Input [1]: [t_time_sk#90]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#93]
+
+(129) BroadcastHashJoin [codegen id : 28]
+Left keys [1]: [ss_sold_time_sk#85]
+Right keys [1]: [t_time_sk#90]
+Join condition: None
+
+(130) Project [codegen id : 28]
+Output [1]: [ss_store_sk#87]
+Input [3]: [ss_sold_time_sk#85, ss_store_sk#87, t_time_sk#90]
+
+(131) ReusedExchange [Reuses operator id: 23]
+Output [1]: [s_store_sk#94]
+
+(132) BroadcastHashJoin [codegen id : 28]
+Left keys [1]: [ss_store_sk#87]
+Right keys [1]: [s_store_sk#94]
+Join condition: None
+
+(133) Project [codegen id : 28]
+Output: []
+Input [2]: [ss_store_sk#87, s_store_sk#94]
+
+(134) HashAggregate [codegen id : 28]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#95]
+Results [1]: [count#96]
+
+(135) Exchange
+Input [1]: [count#96]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#97]
+
+(136) HashAggregate [codegen id : 29]
+Input [1]: [count#96]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#98]
+Results [1]: [count(1)#98 AS h11_to_11_30#99]
+
+(137) BroadcastExchange
+Input [1]: [h11_to_11_30#99]
+Arguments: IdentityBroadcastMode, [id=#100]
+
+(138) BroadcastNestedLoopJoin [codegen id : 40]
+Join condition: None
+
+(139) Scan parquet default.store_sales
+Output [4]: [ss_sold_time_sk#101, ss_hdemo_sk#102, ss_store_sk#103, ss_sold_date_sk#104]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_sold_time_sk:int,ss_hdemo_sk:int,ss_store_sk:int>
+
+(140) ColumnarToRow [codegen id : 33]
+Input [4]: [ss_sold_time_sk#101, ss_hdemo_sk#102, ss_store_sk#103, ss_sold_date_sk#104]
+
+(141) Filter [codegen id : 33]
+Input [4]: [ss_sold_time_sk#101, ss_hdemo_sk#102, ss_store_sk#103, ss_sold_date_sk#104]
+Condition : ((isnotnull(ss_hdemo_sk#102) AND isnotnull(ss_sold_time_sk#101)) AND isnotnull(ss_store_sk#103))
+
+(142) Project [codegen id : 33]
+Output [3]: [ss_sold_time_sk#101, ss_hdemo_sk#102, ss_store_sk#103]
+Input [4]: [ss_sold_time_sk#101, ss_hdemo_sk#102, ss_store_sk#103, ss_sold_date_sk#104]
+
+(143) ReusedExchange [Reuses operator id: 9]
+Output [1]: [hd_demo_sk#105]
+
+(144) BroadcastHashJoin [codegen id : 33]
+Left keys [1]: [ss_hdemo_sk#102]
+Right keys [1]: [hd_demo_sk#105]
+Join condition: None
+
+(145) Project [codegen id : 33]
+Output [2]: [ss_sold_time_sk#101, ss_store_sk#103]
+Input [4]: [ss_sold_time_sk#101, ss_hdemo_sk#102, ss_store_sk#103, hd_demo_sk#105]
+
+(146) Scan parquet default.time_dim
+Output [3]: [t_time_sk#106, t_hour#107, t_minute#108]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,11), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_hour:int,t_minute:int>
+
+(147) ColumnarToRow [codegen id : 31]
+Input [3]: [t_time_sk#106, t_hour#107, t_minute#108]
+
+(148) Filter [codegen id : 31]
+Input [3]: [t_time_sk#106, t_hour#107, t_minute#108]
+Condition : ((((isnotnull(t_hour#107) AND isnotnull(t_minute#108)) AND (t_hour#107 = 11)) AND (t_minute#108 >= 30)) AND isnotnull(t_time_sk#106))
+
+(149) Project [codegen id : 31]
+Output [1]: [t_time_sk#106]
+Input [3]: [t_time_sk#106, t_hour#107, t_minute#108]
+
+(150) BroadcastExchange
+Input [1]: [t_time_sk#106]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#109]
+
+(151) BroadcastHashJoin [codegen id : 33]
+Left keys [1]: [ss_sold_time_sk#101]
+Right keys [1]: [t_time_sk#106]
+Join condition: None
+
+(152) Project [codegen id : 33]
+Output [1]: [ss_store_sk#103]
+Input [3]: [ss_sold_time_sk#101, ss_store_sk#103, t_time_sk#106]
+
+(153) ReusedExchange [Reuses operator id: 23]
+Output [1]: [s_store_sk#110]
+
+(154) BroadcastHashJoin [codegen id : 33]
+Left keys [1]: [ss_store_sk#103]
+Right keys [1]: [s_store_sk#110]
+Join condition: None
+
+(155) Project [codegen id : 33]
+Output: []
+Input [2]: [ss_store_sk#103, s_store_sk#110]
+
+(156) HashAggregate [codegen id : 33]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#111]
+Results [1]: [count#112]
+
+(157) Exchange
+Input [1]: [count#112]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#113]
+
+(158) HashAggregate [codegen id : 34]
+Input [1]: [count#112]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#114]
+Results [1]: [count(1)#114 AS h11_30_to_12#115]
+
+(159) BroadcastExchange
+Input [1]: [h11_30_to_12#115]
+Arguments: IdentityBroadcastMode, [id=#116]
+
+(160) BroadcastNestedLoopJoin [codegen id : 40]
+Join condition: None
+
+(161) Scan parquet default.store_sales
+Output [4]: [ss_sold_time_sk#117, ss_hdemo_sk#118, ss_store_sk#119, ss_sold_date_sk#120]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_sold_time_sk:int,ss_hdemo_sk:int,ss_store_sk:int>
+
+(162) ColumnarToRow [codegen id : 38]
+Input [4]: [ss_sold_time_sk#117, ss_hdemo_sk#118, ss_store_sk#119, ss_sold_date_sk#120]
+
+(163) Filter [codegen id : 38]
+Input [4]: [ss_sold_time_sk#117, ss_hdemo_sk#118, ss_store_sk#119, ss_sold_date_sk#120]
+Condition : ((isnotnull(ss_hdemo_sk#118) AND isnotnull(ss_sold_time_sk#117)) AND isnotnull(ss_store_sk#119))
+
+(164) Project [codegen id : 38]
+Output [3]: [ss_sold_time_sk#117, ss_hdemo_sk#118, ss_store_sk#119]
+Input [4]: [ss_sold_time_sk#117, ss_hdemo_sk#118, ss_store_sk#119, ss_sold_date_sk#120]
+
+(165) ReusedExchange [Reuses operator id: 9]
+Output [1]: [hd_demo_sk#121]
+
+(166) BroadcastHashJoin [codegen id : 38]
+Left keys [1]: [ss_hdemo_sk#118]
+Right keys [1]: [hd_demo_sk#121]
+Join condition: None
+
+(167) Project [codegen id : 38]
+Output [2]: [ss_sold_time_sk#117, ss_store_sk#119]
+Input [4]: [ss_sold_time_sk#117, ss_hdemo_sk#118, ss_store_sk#119, hd_demo_sk#121]
+
+(168) Scan parquet default.time_dim
+Output [3]: [t_time_sk#122, t_hour#123, t_minute#124]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,12), LessThan(t_minute,30), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_hour:int,t_minute:int>
+
+(169) ColumnarToRow [codegen id : 36]
+Input [3]: [t_time_sk#122, t_hour#123, t_minute#124]
+
+(170) Filter [codegen id : 36]
+Input [3]: [t_time_sk#122, t_hour#123, t_minute#124]
+Condition : ((((isnotnull(t_hour#123) AND isnotnull(t_minute#124)) AND (t_hour#123 = 12)) AND (t_minute#124 < 30)) AND isnotnull(t_time_sk#122))
+
+(171) Project [codegen id : 36]
+Output [1]: [t_time_sk#122]
+Input [3]: [t_time_sk#122, t_hour#123, t_minute#124]
+
+(172) BroadcastExchange
+Input [1]: [t_time_sk#122]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#125]
+
+(173) BroadcastHashJoin [codegen id : 38]
+Left keys [1]: [ss_sold_time_sk#117]
+Right keys [1]: [t_time_sk#122]
+Join condition: None
+
+(174) Project [codegen id : 38]
+Output [1]: [ss_store_sk#119]
+Input [3]: [ss_sold_time_sk#117, ss_store_sk#119, t_time_sk#122]
+
+(175) ReusedExchange [Reuses operator id: 23]
+Output [1]: [s_store_sk#126]
+
+(176) BroadcastHashJoin [codegen id : 38]
+Left keys [1]: [ss_store_sk#119]
+Right keys [1]: [s_store_sk#126]
+Join condition: None
+
+(177) Project [codegen id : 38]
+Output: []
+Input [2]: [ss_store_sk#119, s_store_sk#126]
+
+(178) HashAggregate [codegen id : 38]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#127]
+Results [1]: [count#128]
+
+(179) Exchange
+Input [1]: [count#128]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#129]
+
+(180) HashAggregate [codegen id : 39]
+Input [1]: [count#128]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#130]
+Results [1]: [count(1)#130 AS h12_to_12_30#131]
+
+(181) BroadcastExchange
+Input [1]: [h12_to_12_30#131]
+Arguments: IdentityBroadcastMode, [id=#132]
+
+(182) BroadcastNestedLoopJoin [codegen id : 40]
+Join condition: None
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/simplified.txt
new file mode 100644
index 0000000000000..4bbd80c7a884e
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/simplified.txt
@@ -0,0 +1,265 @@
+WholeStageCodegen (40)
+  BroadcastNestedLoopJoin
+    BroadcastNestedLoopJoin
+      BroadcastNestedLoopJoin
+        BroadcastNestedLoopJoin
+          BroadcastNestedLoopJoin
+            BroadcastNestedLoopJoin
+              BroadcastNestedLoopJoin
+                HashAggregate [count] [count(1),h8_30_to_9,count]
+                  InputAdapter
+                    Exchange #1
+                      WholeStageCodegen (4)
+                        HashAggregate [count,count]
+                          Project
+                            BroadcastHashJoin [ss_store_sk,s_store_sk]
+                              Project [ss_store_sk]
+                                BroadcastHashJoin [ss_sold_time_sk,t_time_sk]
+                                  Project [ss_sold_time_sk,ss_store_sk]
+                                    BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                      Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk]
+                                        Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk]
+                                      InputAdapter
+                                        BroadcastExchange #2
+                                          WholeStageCodegen (1)
+                                            Project [hd_demo_sk]
+                                              Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count]
+                                  InputAdapter
+                                    BroadcastExchange #3
+                                      WholeStageCodegen (2)
+                                        Project [t_time_sk]
+                                          Filter [t_hour,t_minute,t_time_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.time_dim [t_time_sk,t_hour,t_minute]
+                              InputAdapter
+                                BroadcastExchange #4
+                                  WholeStageCodegen (3)
+                                    Project [s_store_sk]
+                                      Filter [s_store_name,s_store_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store [s_store_sk,s_store_name]
+                InputAdapter
+                  BroadcastExchange #5
+                    WholeStageCodegen (9)
+                      HashAggregate [count] [count(1),h9_to_9_30,count]
+                        InputAdapter
+                          Exchange #6
+                            WholeStageCodegen (8)
+                              HashAggregate [count,count]
+                                Project
+                                  BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                    Project [ss_store_sk]
+                                      BroadcastHashJoin [ss_sold_time_sk,t_time_sk]
+                                        Project [ss_sold_time_sk,ss_store_sk]
+                                          BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                            Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk]
+                                              Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk]
+                                            InputAdapter
+                                              ReusedExchange [hd_demo_sk] #2
+                                        InputAdapter
+                                          BroadcastExchange #7
+                                            WholeStageCodegen (6)
+                                              Project [t_time_sk]
+                                                Filter [t_hour,t_minute,t_time_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.time_dim [t_time_sk,t_hour,t_minute]
+                                    InputAdapter
+                                      ReusedExchange [s_store_sk] #4
+              InputAdapter
+                BroadcastExchange #8
+                  WholeStageCodegen (14)
+                    HashAggregate [count] [count(1),h9_30_to_10,count]
+                      InputAdapter
+                        Exchange #9
+                          WholeStageCodegen (13)
+                            HashAggregate [count,count]
+                              Project
+                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                  Project [ss_store_sk]
+                                    BroadcastHashJoin [ss_sold_time_sk,t_time_sk]
+                                      Project [ss_sold_time_sk,ss_store_sk]
+                                        BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                          Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk]
+                                            Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk]
+                                          InputAdapter
+                                            ReusedExchange [hd_demo_sk] #2
+                                      InputAdapter
+                                        BroadcastExchange #10
+                                          WholeStageCodegen (11)
+                                            Project [t_time_sk]
+                                              Filter [t_hour,t_minute,t_time_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.time_dim [t_time_sk,t_hour,t_minute]
+                                  InputAdapter
+                                    ReusedExchange [s_store_sk] #4
+            InputAdapter
+              BroadcastExchange #11
+                WholeStageCodegen (19)
+                  HashAggregate [count] [count(1),h10_to_10_30,count]
+                    InputAdapter
+                      Exchange #12
+                        WholeStageCodegen (18)
+                          HashAggregate [count,count]
+                            Project
+                              BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                Project [ss_store_sk]
+                                  BroadcastHashJoin [ss_sold_time_sk,t_time_sk]
+                                    Project [ss_sold_time_sk,ss_store_sk]
+                                      BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                        Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk]
+                                          Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk]
+                                        InputAdapter
+                                          ReusedExchange [hd_demo_sk] #2
+                                    InputAdapter
+                                      BroadcastExchange #13
+                                        WholeStageCodegen (16)
+                                          Project [t_time_sk]
+                                            Filter [t_hour,t_minute,t_time_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.time_dim [t_time_sk,t_hour,t_minute]
+                                InputAdapter
+                                  ReusedExchange [s_store_sk] #4
+          InputAdapter
+            BroadcastExchange #14
+              WholeStageCodegen (24)
+                HashAggregate [count] [count(1),h10_30_to_11,count]
+                  InputAdapter
+                    Exchange #15
+                      WholeStageCodegen (23)
+                        HashAggregate [count,count]
+                          Project
+                            BroadcastHashJoin [ss_store_sk,s_store_sk]
+                              Project [ss_store_sk]
+                                BroadcastHashJoin [ss_sold_time_sk,t_time_sk]
+                                  Project [ss_sold_time_sk,ss_store_sk]
+                                    BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                      Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk]
+                                        Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk]
+                                      InputAdapter
+                                        ReusedExchange [hd_demo_sk] #2
+                                  InputAdapter
+                                    BroadcastExchange #16
+                                      WholeStageCodegen (21)
+                                        Project [t_time_sk]
+                                          Filter [t_hour,t_minute,t_time_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.time_dim [t_time_sk,t_hour,t_minute]
+                              InputAdapter
+                                ReusedExchange [s_store_sk] #4
+        InputAdapter
+          BroadcastExchange #17
+            WholeStageCodegen (29)
+              HashAggregate [count] [count(1),h11_to_11_30,count]
+                InputAdapter
+                  Exchange #18
+                    WholeStageCodegen (28)
+                      HashAggregate [count,count]
+                        Project
+                          BroadcastHashJoin [ss_store_sk,s_store_sk]
+                            Project [ss_store_sk]
+                              BroadcastHashJoin [ss_sold_time_sk,t_time_sk]
+                                Project [ss_sold_time_sk,ss_store_sk]
+                                  BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                    Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk]
+                                      Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk]
+                                    InputAdapter
+                                      ReusedExchange [hd_demo_sk] #2
+                                InputAdapter
+                                  BroadcastExchange #19
+                                    WholeStageCodegen (26)
+                                      Project [t_time_sk]
+                                        Filter [t_hour,t_minute,t_time_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.time_dim [t_time_sk,t_hour,t_minute]
+                            InputAdapter
+                              ReusedExchange [s_store_sk] #4
+      InputAdapter
+        BroadcastExchange #20
+          WholeStageCodegen (34)
+            HashAggregate [count] [count(1),h11_30_to_12,count]
+              InputAdapter
+                Exchange #21
+                  WholeStageCodegen (33)
+                    HashAggregate [count,count]
+                      Project
+                        BroadcastHashJoin [ss_store_sk,s_store_sk]
+                          Project [ss_store_sk]
+                            BroadcastHashJoin [ss_sold_time_sk,t_time_sk]
+                              Project [ss_sold_time_sk,ss_store_sk]
+                                BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                  Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk]
+                                    Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk]
+                                  InputAdapter
+                                    ReusedExchange [hd_demo_sk] #2
+                              InputAdapter
+                                BroadcastExchange #22
+                                  WholeStageCodegen (31)
+                                    Project [t_time_sk]
+                                      Filter [t_hour,t_minute,t_time_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.time_dim [t_time_sk,t_hour,t_minute]
+                          InputAdapter
+                            ReusedExchange [s_store_sk] #4
+    InputAdapter
+      BroadcastExchange #23
+        WholeStageCodegen (39)
+          HashAggregate [count] [count(1),h12_to_12_30,count]
+            InputAdapter
+              Exchange #24
+                WholeStageCodegen (38)
+                  HashAggregate [count,count]
+                    Project
+                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                        Project [ss_store_sk]
+                          BroadcastHashJoin [ss_sold_time_sk,t_time_sk]
+                            Project [ss_sold_time_sk,ss_store_sk]
+                              BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk]
+                                  Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk]
+                                InputAdapter
+                                  ReusedExchange [hd_demo_sk] #2
+                            InputAdapter
+                              BroadcastExchange #25
+                                WholeStageCodegen (36)
+                                  Project [t_time_sk]
+                                    Filter [t_hour,t_minute,t_time_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.time_dim [t_time_sk,t_hour,t_minute]
+                        InputAdapter
+                          ReusedExchange [s_store_sk] #4
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.sf100/explain.txt
new file mode 100644
index 0000000000000..b3d0c07cd80fe
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.sf100/explain.txt
@@ -0,0 +1,186 @@
+== Physical Plan ==
+TakeOrderedAndProject (31)
++- * Project (30)
+   +- * Filter (29)
+      +- Window (28)
+         +- * Sort (27)
+            +- Exchange (26)
+               +- * HashAggregate (25)
+                  +- Exchange (24)
+                     +- * HashAggregate (23)
+                        +- * Project (22)
+                           +- * BroadcastHashJoin Inner BuildRight (21)
+                              :- * Project (16)
+                              :  +- * BroadcastHashJoin Inner BuildRight (15)
+                              :     :- * Project (9)
+                              :     :  +- * BroadcastHashJoin Inner BuildLeft (8)
+                              :     :     :- BroadcastExchange (4)
+                              :     :     :  +- * Filter (3)
+                              :     :     :     +- * ColumnarToRow (2)
+                              :     :     :        +- Scan parquet default.item (1)
+                              :     :     +- * Filter (7)
+                              :     :        +- * ColumnarToRow (6)
+                              :     :           +- Scan parquet default.store_sales (5)
+                              :     +- BroadcastExchange (14)
+                              :        +- * Project (13)
+                              :           +- * Filter (12)
+                              :              +- * ColumnarToRow (11)
+                              :                 +- Scan parquet default.date_dim (10)
+                              +- BroadcastExchange (20)
+                                 +- * Filter (19)
+                                    +- * ColumnarToRow (18)
+                                       +- Scan parquet default.store (17)
+
+
+(1) Scan parquet default.item
+Output [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [Or(And(In(i_category, [Books                                             ,Electronics                                       ,Sports                                            ]),In(i_class, [computers                                         ,stereo                                            ,football                                          ])),And(In(i_category, [Men                                               ,Jewelry                                           ,Women                                             ]),In(i_class, [shirts                                            ,birdal                                            ,dresses                                           ]))), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_class:string,i_category:string>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4]
+
+(3) Filter [codegen id : 1]
+Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4]
+Condition : (((i_category#4 IN (Books                                             ,Electronics                                       ,Sports                                            ) AND i_class#3 IN (computers                                         ,stereo                                            ,football                                          )) OR (i_category#4 IN (Men                                               ,Jewelry                                           ,Women                                             ) AND i_class#3 IN (shirts                                            ,birdal                                            ,dresses                                           ))) AND isnotnull(i_item_sk#1))
+
+(4) BroadcastExchange
+Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#5]
+
+(5) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#9), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(6) ColumnarToRow
+Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9]
+
+(7) Filter
+Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9]
+Condition : (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7))
+
+(8) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [ss_item_sk#6]
+Join condition: None
+
+(9) Project [codegen id : 4]
+Output [6]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9]
+Input [8]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9]
+
+(10) Scan parquet default.date_dim
+Output [3]: [d_date_sk#11, d_year#12, d_moy#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [3]: [d_date_sk#11, d_year#12, d_moy#13]
+
+(12) Filter [codegen id : 2]
+Input [3]: [d_date_sk#11, d_year#12, d_moy#13]
+Condition : ((isnotnull(d_year#12) AND (d_year#12 = 1999)) AND isnotnull(d_date_sk#11))
+
+(13) Project [codegen id : 2]
+Output [2]: [d_date_sk#11, d_moy#13]
+Input [3]: [d_date_sk#11, d_year#12, d_moy#13]
+
+(14) BroadcastExchange
+Input [2]: [d_date_sk#11, d_moy#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(15) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#9]
+Right keys [1]: [d_date_sk#11]
+Join condition: None
+
+(16) Project [codegen id : 4]
+Output [6]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#7, ss_sales_price#8, d_moy#13]
+Input [8]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#11, d_moy#13]
+
+(17) Scan parquet default.store
+Output [3]: [s_store_sk#15, s_store_name#16, s_company_name#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_company_name:string>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17]
+
+(19) Filter [codegen id : 3]
+Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17]
+Condition : isnotnull(s_store_sk#15)
+
+(20) BroadcastExchange
+Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#18]
+
+(21) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#7]
+Right keys [1]: [s_store_sk#15]
+Join condition: None
+
+(22) Project [codegen id : 4]
+Output [7]: [i_brand#2, i_class#3, i_category#4, ss_sales_price#8, d_moy#13, s_store_name#16, s_company_name#17]
+Input [9]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#7, ss_sales_price#8, d_moy#13, s_store_sk#15, s_store_name#16, s_company_name#17]
+
+(23) HashAggregate [codegen id : 4]
+Input [7]: [i_brand#2, i_class#3, i_category#4, ss_sales_price#8, d_moy#13, s_store_name#16, s_company_name#17]
+Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#8))]
+Aggregate Attributes [1]: [sum#19]
+Results [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum#20]
+
+(24) Exchange
+Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum#20]
+Arguments: hashpartitioning(i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(25) HashAggregate [codegen id : 5]
+Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum#20]
+Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#8))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#8))#22]
+Results [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#22,17,2) AS sum_sales#23, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#22,17,2) AS _w0#24]
+
+(26) Exchange
+Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum_sales#23, _w0#24]
+Arguments: hashpartitioning(i_category#4, i_brand#2, s_store_name#16, s_company_name#17, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(27) Sort [codegen id : 6]
+Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum_sales#23, _w0#24]
+Arguments: [i_category#4 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#16 ASC NULLS FIRST, s_company_name#17 ASC NULLS FIRST], false, 0
+
+(28) Window
+Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum_sales#23, _w0#24]
+Arguments: [avg(_w0#24) windowspecdefinition(i_category#4, i_brand#2, s_store_name#16, s_company_name#17, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#26], [i_category#4, i_brand#2, s_store_name#16, s_company_name#17]
+
+(29) Filter [codegen id : 7]
+Input [9]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum_sales#23, _w0#24, avg_monthly_sales#26]
+Condition : (isnotnull(avg_monthly_sales#26) AND (NOT (avg_monthly_sales#26 = 0.000000) AND (CheckOverflow((promote_precision(abs(CheckOverflow((promote_precision(cast(sum_sales#23 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#26 as decimal(22,6)))), DecimalType(22,6), true), false)) / promote_precision(cast(avg_monthly_sales#26 as decimal(22,6)))), DecimalType(38,16), true) > 0.1000000000000000)))
+
+(30) Project [codegen id : 7]
+Output [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum_sales#23, avg_monthly_sales#26]
+Input [9]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum_sales#23, _w0#24, avg_monthly_sales#26]
+
+(31) TakeOrderedAndProject
+Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum_sales#23, avg_monthly_sales#26]
+Arguments: 100, [CheckOverflow((promote_precision(cast(sum_sales#23 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#26 as decimal(22,6)))), DecimalType(22,6), true) ASC NULLS FIRST, s_store_name#16 ASC NULLS FIRST], [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum_sales#23, avg_monthly_sales#26]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 5 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10
+ReusedExchange (32)
+
+
+(32) ReusedExchange [Reuses operator id: 14]
+Output [2]: [d_date_sk#11, d_moy#13]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.sf100/simplified.txt
new file mode 100644
index 0000000000000..e4a9b0edb9ded
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.sf100/simplified.txt
@@ -0,0 +1,50 @@
+TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_class,i_brand,s_company_name,d_moy]
+  WholeStageCodegen (7)
+    Project [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,avg_monthly_sales]
+      Filter [avg_monthly_sales,sum_sales]
+        InputAdapter
+          Window [_w0,i_category,i_brand,s_store_name,s_company_name]
+            WholeStageCodegen (6)
+              Sort [i_category,i_brand,s_store_name,s_company_name]
+                InputAdapter
+                  Exchange [i_category,i_brand,s_store_name,s_company_name] #1
+                    WholeStageCodegen (5)
+                      HashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum]
+                        InputAdapter
+                          Exchange [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy] #2
+                            WholeStageCodegen (4)
+                              HashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,ss_sales_price] [sum,sum]
+                                Project [i_brand,i_class,i_category,ss_sales_price,d_moy,s_store_name,s_company_name]
+                                  BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                    Project [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy]
+                                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                        Project [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                          BroadcastHashJoin [i_item_sk,ss_item_sk]
+                                            InputAdapter
+                                              BroadcastExchange #3
+                                                WholeStageCodegen (1)
+                                                  Filter [i_category,i_class,i_item_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.item [i_item_sk,i_brand,i_class,i_category]
+                                            Filter [ss_item_sk,ss_store_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #1
+                                                      ReusedExchange [d_date_sk,d_moy] #4
+                                        InputAdapter
+                                          BroadcastExchange #4
+                                            WholeStageCodegen (2)
+                                              Project [d_date_sk,d_moy]
+                                                Filter [d_year,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                    InputAdapter
+                                      BroadcastExchange #5
+                                        WholeStageCodegen (3)
+                                          Filter [s_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store [s_store_sk,s_store_name,s_company_name]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/explain.txt
new file mode 100644
index 0000000000000..f61c8e6945003
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/explain.txt
@@ -0,0 +1,186 @@
+== Physical Plan ==
+TakeOrderedAndProject (31)
++- * Project (30)
+   +- * Filter (29)
+      +- Window (28)
+         +- * Sort (27)
+            +- Exchange (26)
+               +- * HashAggregate (25)
+                  +- Exchange (24)
+                     +- * HashAggregate (23)
+                        +- * Project (22)
+                           +- * BroadcastHashJoin Inner BuildRight (21)
+                              :- * Project (16)
+                              :  +- * BroadcastHashJoin Inner BuildRight (15)
+                              :     :- * Project (9)
+                              :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+                              :     :     :- * Filter (3)
+                              :     :     :  +- * ColumnarToRow (2)
+                              :     :     :     +- Scan parquet default.item (1)
+                              :     :     +- BroadcastExchange (7)
+                              :     :        +- * Filter (6)
+                              :     :           +- * ColumnarToRow (5)
+                              :     :              +- Scan parquet default.store_sales (4)
+                              :     +- BroadcastExchange (14)
+                              :        +- * Project (13)
+                              :           +- * Filter (12)
+                              :              +- * ColumnarToRow (11)
+                              :                 +- Scan parquet default.date_dim (10)
+                              +- BroadcastExchange (20)
+                                 +- * Filter (19)
+                                    +- * ColumnarToRow (18)
+                                       +- Scan parquet default.store (17)
+
+
+(1) Scan parquet default.item
+Output [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [Or(And(In(i_category, [Books                                             ,Electronics                                       ,Sports                                            ]),In(i_class, [computers                                         ,stereo                                            ,football                                          ])),And(In(i_category, [Men                                               ,Jewelry                                           ,Women                                             ]),In(i_class, [shirts                                            ,birdal                                            ,dresses                                           ]))), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_class:string,i_category:string>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4]
+
+(3) Filter [codegen id : 4]
+Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4]
+Condition : (((i_category#4 IN (Books                                             ,Electronics                                       ,Sports                                            ) AND i_class#3 IN (computers                                         ,stereo                                            ,football                                          )) OR (i_category#4 IN (Men                                               ,Jewelry                                           ,Women                                             ) AND i_class#3 IN (shirts                                            ,birdal                                            ,dresses                                           ))) AND isnotnull(i_item_sk#1))
+
+(4) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8]
+
+(6) Filter [codegen id : 1]
+Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8]
+Condition : (isnotnull(ss_item_sk#5) AND isnotnull(ss_store_sk#6))
+
+(7) BroadcastExchange
+Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#10]
+
+(8) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [ss_item_sk#5]
+Join condition: None
+
+(9) Project [codegen id : 4]
+Output [6]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8]
+Input [8]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8]
+
+(10) Scan parquet default.date_dim
+Output [3]: [d_date_sk#11, d_year#12, d_moy#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [3]: [d_date_sk#11, d_year#12, d_moy#13]
+
+(12) Filter [codegen id : 2]
+Input [3]: [d_date_sk#11, d_year#12, d_moy#13]
+Condition : ((isnotnull(d_year#12) AND (d_year#12 = 1999)) AND isnotnull(d_date_sk#11))
+
+(13) Project [codegen id : 2]
+Output [2]: [d_date_sk#11, d_moy#13]
+Input [3]: [d_date_sk#11, d_year#12, d_moy#13]
+
+(14) BroadcastExchange
+Input [2]: [d_date_sk#11, d_moy#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(15) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#11]
+Join condition: None
+
+(16) Project [codegen id : 4]
+Output [6]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#13]
+Input [8]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8, d_date_sk#11, d_moy#13]
+
+(17) Scan parquet default.store
+Output [3]: [s_store_sk#15, s_store_name#16, s_company_name#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_company_name:string>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17]
+
+(19) Filter [codegen id : 3]
+Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17]
+Condition : isnotnull(s_store_sk#15)
+
+(20) BroadcastExchange
+Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#18]
+
+(21) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#6]
+Right keys [1]: [s_store_sk#15]
+Join condition: None
+
+(22) Project [codegen id : 4]
+Output [7]: [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#13, s_store_name#16, s_company_name#17]
+Input [9]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#13, s_store_sk#15, s_store_name#16, s_company_name#17]
+
+(23) HashAggregate [codegen id : 4]
+Input [7]: [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#13, s_store_name#16, s_company_name#17]
+Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#7))]
+Aggregate Attributes [1]: [sum#19]
+Results [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum#20]
+
+(24) Exchange
+Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum#20]
+Arguments: hashpartitioning(i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(25) HashAggregate [codegen id : 5]
+Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum#20]
+Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#7))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#7))#22]
+Results [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, MakeDecimal(sum(UnscaledValue(ss_sales_price#7))#22,17,2) AS sum_sales#23, MakeDecimal(sum(UnscaledValue(ss_sales_price#7))#22,17,2) AS _w0#24]
+
+(26) Exchange
+Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum_sales#23, _w0#24]
+Arguments: hashpartitioning(i_category#4, i_brand#2, s_store_name#16, s_company_name#17, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(27) Sort [codegen id : 6]
+Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum_sales#23, _w0#24]
+Arguments: [i_category#4 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#16 ASC NULLS FIRST, s_company_name#17 ASC NULLS FIRST], false, 0
+
+(28) Window
+Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum_sales#23, _w0#24]
+Arguments: [avg(_w0#24) windowspecdefinition(i_category#4, i_brand#2, s_store_name#16, s_company_name#17, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#26], [i_category#4, i_brand#2, s_store_name#16, s_company_name#17]
+
+(29) Filter [codegen id : 7]
+Input [9]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum_sales#23, _w0#24, avg_monthly_sales#26]
+Condition : (isnotnull(avg_monthly_sales#26) AND (NOT (avg_monthly_sales#26 = 0.000000) AND (CheckOverflow((promote_precision(abs(CheckOverflow((promote_precision(cast(sum_sales#23 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#26 as decimal(22,6)))), DecimalType(22,6), true), false)) / promote_precision(cast(avg_monthly_sales#26 as decimal(22,6)))), DecimalType(38,16), true) > 0.1000000000000000)))
+
+(30) Project [codegen id : 7]
+Output [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum_sales#23, avg_monthly_sales#26]
+Input [9]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum_sales#23, _w0#24, avg_monthly_sales#26]
+
+(31) TakeOrderedAndProject
+Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum_sales#23, avg_monthly_sales#26]
+Arguments: 100, [CheckOverflow((promote_precision(cast(sum_sales#23 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#26 as decimal(22,6)))), DecimalType(22,6), true) ASC NULLS FIRST, s_store_name#16 ASC NULLS FIRST], [i_category#4, i_class#3, i_brand#2, s_store_name#16, s_company_name#17, d_moy#13, sum_sales#23, avg_monthly_sales#26]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (32)
+
+
+(32) ReusedExchange [Reuses operator id: 14]
+Output [2]: [d_date_sk#11, d_moy#13]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/simplified.txt
new file mode 100644
index 0000000000000..b96445eec6223
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/simplified.txt
@@ -0,0 +1,50 @@
+TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_class,i_brand,s_company_name,d_moy]
+  WholeStageCodegen (7)
+    Project [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,avg_monthly_sales]
+      Filter [avg_monthly_sales,sum_sales]
+        InputAdapter
+          Window [_w0,i_category,i_brand,s_store_name,s_company_name]
+            WholeStageCodegen (6)
+              Sort [i_category,i_brand,s_store_name,s_company_name]
+                InputAdapter
+                  Exchange [i_category,i_brand,s_store_name,s_company_name] #1
+                    WholeStageCodegen (5)
+                      HashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum]
+                        InputAdapter
+                          Exchange [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy] #2
+                            WholeStageCodegen (4)
+                              HashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,ss_sales_price] [sum,sum]
+                                Project [i_brand,i_class,i_category,ss_sales_price,d_moy,s_store_name,s_company_name]
+                                  BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                    Project [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy]
+                                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                        Project [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                          BroadcastHashJoin [i_item_sk,ss_item_sk]
+                                            Filter [i_category,i_class,i_item_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.item [i_item_sk,i_brand,i_class,i_category]
+                                            InputAdapter
+                                              BroadcastExchange #3
+                                                WholeStageCodegen (1)
+                                                  Filter [ss_item_sk,ss_store_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                                          SubqueryBroadcast [d_date_sk] #1
+                                                            ReusedExchange [d_date_sk,d_moy] #4
+                                        InputAdapter
+                                          BroadcastExchange #4
+                                            WholeStageCodegen (2)
+                                              Project [d_date_sk,d_moy]
+                                                Filter [d_year,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                    InputAdapter
+                                      BroadcastExchange #5
+                                        WholeStageCodegen (3)
+                                          Filter [s_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store [s_store_sk,s_store_name,s_company_name]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.sf100/explain.txt
new file mode 100644
index 0000000000000..8736c9861a5ce
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.sf100/explain.txt
@@ -0,0 +1,718 @@
+== Physical Plan ==
+* Project (4)
++- * Filter (3)
+   +- * ColumnarToRow (2)
+      +- Scan parquet default.reason (1)
+
+
+(1) Scan parquet default.reason
+Output [1]: [r_reason_sk#1]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/reason]
+PushedFilters: [IsNotNull(r_reason_sk), EqualTo(r_reason_sk,1)]
+ReadSchema: struct<r_reason_sk:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [1]: [r_reason_sk#1]
+
+(3) Filter [codegen id : 1]
+Input [1]: [r_reason_sk#1]
+Condition : (isnotnull(r_reason_sk#1) AND (r_reason_sk#1 = 1))
+
+(4) Project [codegen id : 1]
+Output [5]: [CASE WHEN (Subquery scalar-subquery#2, [id=#3] > 62316685) THEN Subquery scalar-subquery#4, [id=#5] ELSE Subquery scalar-subquery#6, [id=#7] END AS bucket1#8, CASE WHEN (Subquery scalar-subquery#9, [id=#10] > 19045798) THEN Subquery scalar-subquery#11, [id=#12] ELSE Subquery scalar-subquery#13, [id=#14] END AS bucket2#15, CASE WHEN (Subquery scalar-subquery#16, [id=#17] > 365541424) THEN Subquery scalar-subquery#18, [id=#19] ELSE Subquery scalar-subquery#20, [id=#21] END AS bucket3#22, CASE WHEN (Subquery scalar-subquery#23, [id=#24] > 216357808) THEN Subquery scalar-subquery#25, [id=#26] ELSE Subquery scalar-subquery#27, [id=#28] END AS bucket4#29, CASE WHEN (Subquery scalar-subquery#30, [id=#31] > 184483884) THEN Subquery scalar-subquery#32, [id=#33] ELSE Subquery scalar-subquery#34, [id=#35] END AS bucket5#36]
+Input [1]: [r_reason_sk#1]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#2, [id=#3]
+* HashAggregate (11)
++- Exchange (10)
+   +- * HashAggregate (9)
+      +- * Project (8)
+         +- * Filter (7)
+            +- * ColumnarToRow (6)
+               +- Scan parquet default.store_sales (5)
+
+
+(5) Scan parquet default.store_sales
+Output [2]: [ss_quantity#37, ss_sold_date_sk#38]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,1), LessThanOrEqual(ss_quantity,20)]
+ReadSchema: struct<ss_quantity:int>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [2]: [ss_quantity#37, ss_sold_date_sk#38]
+
+(7) Filter [codegen id : 1]
+Input [2]: [ss_quantity#37, ss_sold_date_sk#38]
+Condition : ((isnotnull(ss_quantity#37) AND (ss_quantity#37 >= 1)) AND (ss_quantity#37 <= 20))
+
+(8) Project [codegen id : 1]
+Output: []
+Input [2]: [ss_quantity#37, ss_sold_date_sk#38]
+
+(9) HashAggregate [codegen id : 1]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#39]
+Results [1]: [count#40]
+
+(10) Exchange
+Input [1]: [count#40]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#41]
+
+(11) HashAggregate [codegen id : 2]
+Input [1]: [count#40]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#42]
+Results [1]: [count(1)#42 AS count(1)#43]
+
+Subquery:2 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#4, [id=#5]
+* HashAggregate (18)
++- Exchange (17)
+   +- * HashAggregate (16)
+      +- * Project (15)
+         +- * Filter (14)
+            +- * ColumnarToRow (13)
+               +- Scan parquet default.store_sales (12)
+
+
+(12) Scan parquet default.store_sales
+Output [3]: [ss_quantity#44, ss_ext_discount_amt#45, ss_sold_date_sk#46]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,1), LessThanOrEqual(ss_quantity,20)]
+ReadSchema: struct<ss_quantity:int,ss_ext_discount_amt:decimal(7,2)>
+
+(13) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_quantity#44, ss_ext_discount_amt#45, ss_sold_date_sk#46]
+
+(14) Filter [codegen id : 1]
+Input [3]: [ss_quantity#44, ss_ext_discount_amt#45, ss_sold_date_sk#46]
+Condition : ((isnotnull(ss_quantity#44) AND (ss_quantity#44 >= 1)) AND (ss_quantity#44 <= 20))
+
+(15) Project [codegen id : 1]
+Output [1]: [ss_ext_discount_amt#45]
+Input [3]: [ss_quantity#44, ss_ext_discount_amt#45, ss_sold_date_sk#46]
+
+(16) HashAggregate [codegen id : 1]
+Input [1]: [ss_ext_discount_amt#45]
+Keys: []
+Functions [1]: [partial_avg(UnscaledValue(ss_ext_discount_amt#45))]
+Aggregate Attributes [2]: [sum#47, count#48]
+Results [2]: [sum#49, count#50]
+
+(17) Exchange
+Input [2]: [sum#49, count#50]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#51]
+
+(18) HashAggregate [codegen id : 2]
+Input [2]: [sum#49, count#50]
+Keys: []
+Functions [1]: [avg(UnscaledValue(ss_ext_discount_amt#45))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_ext_discount_amt#45))#52]
+Results [1]: [cast((avg(UnscaledValue(ss_ext_discount_amt#45))#52 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#53]
+
+Subquery:3 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#6, [id=#7]
+* HashAggregate (25)
++- Exchange (24)
+   +- * HashAggregate (23)
+      +- * Project (22)
+         +- * Filter (21)
+            +- * ColumnarToRow (20)
+               +- Scan parquet default.store_sales (19)
+
+
+(19) Scan parquet default.store_sales
+Output [3]: [ss_quantity#54, ss_net_paid#55, ss_sold_date_sk#56]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,1), LessThanOrEqual(ss_quantity,20)]
+ReadSchema: struct<ss_quantity:int,ss_net_paid:decimal(7,2)>
+
+(20) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_quantity#54, ss_net_paid#55, ss_sold_date_sk#56]
+
+(21) Filter [codegen id : 1]
+Input [3]: [ss_quantity#54, ss_net_paid#55, ss_sold_date_sk#56]
+Condition : ((isnotnull(ss_quantity#54) AND (ss_quantity#54 >= 1)) AND (ss_quantity#54 <= 20))
+
+(22) Project [codegen id : 1]
+Output [1]: [ss_net_paid#55]
+Input [3]: [ss_quantity#54, ss_net_paid#55, ss_sold_date_sk#56]
+
+(23) HashAggregate [codegen id : 1]
+Input [1]: [ss_net_paid#55]
+Keys: []
+Functions [1]: [partial_avg(UnscaledValue(ss_net_paid#55))]
+Aggregate Attributes [2]: [sum#57, count#58]
+Results [2]: [sum#59, count#60]
+
+(24) Exchange
+Input [2]: [sum#59, count#60]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#61]
+
+(25) HashAggregate [codegen id : 2]
+Input [2]: [sum#59, count#60]
+Keys: []
+Functions [1]: [avg(UnscaledValue(ss_net_paid#55))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_paid#55))#62]
+Results [1]: [cast((avg(UnscaledValue(ss_net_paid#55))#62 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#63]
+
+Subquery:4 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#9, [id=#10]
+* HashAggregate (32)
++- Exchange (31)
+   +- * HashAggregate (30)
+      +- * Project (29)
+         +- * Filter (28)
+            +- * ColumnarToRow (27)
+               +- Scan parquet default.store_sales (26)
+
+
+(26) Scan parquet default.store_sales
+Output [2]: [ss_quantity#64, ss_sold_date_sk#65]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,40)]
+ReadSchema: struct<ss_quantity:int>
+
+(27) ColumnarToRow [codegen id : 1]
+Input [2]: [ss_quantity#64, ss_sold_date_sk#65]
+
+(28) Filter [codegen id : 1]
+Input [2]: [ss_quantity#64, ss_sold_date_sk#65]
+Condition : ((isnotnull(ss_quantity#64) AND (ss_quantity#64 >= 21)) AND (ss_quantity#64 <= 40))
+
+(29) Project [codegen id : 1]
+Output: []
+Input [2]: [ss_quantity#64, ss_sold_date_sk#65]
+
+(30) HashAggregate [codegen id : 1]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#66]
+Results [1]: [count#67]
+
+(31) Exchange
+Input [1]: [count#67]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#68]
+
+(32) HashAggregate [codegen id : 2]
+Input [1]: [count#67]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#69]
+Results [1]: [count(1)#69 AS count(1)#70]
+
+Subquery:5 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#11, [id=#12]
+* HashAggregate (39)
++- Exchange (38)
+   +- * HashAggregate (37)
+      +- * Project (36)
+         +- * Filter (35)
+            +- * ColumnarToRow (34)
+               +- Scan parquet default.store_sales (33)
+
+
+(33) Scan parquet default.store_sales
+Output [3]: [ss_quantity#71, ss_ext_discount_amt#72, ss_sold_date_sk#73]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,40)]
+ReadSchema: struct<ss_quantity:int,ss_ext_discount_amt:decimal(7,2)>
+
+(34) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_quantity#71, ss_ext_discount_amt#72, ss_sold_date_sk#73]
+
+(35) Filter [codegen id : 1]
+Input [3]: [ss_quantity#71, ss_ext_discount_amt#72, ss_sold_date_sk#73]
+Condition : ((isnotnull(ss_quantity#71) AND (ss_quantity#71 >= 21)) AND (ss_quantity#71 <= 40))
+
+(36) Project [codegen id : 1]
+Output [1]: [ss_ext_discount_amt#72]
+Input [3]: [ss_quantity#71, ss_ext_discount_amt#72, ss_sold_date_sk#73]
+
+(37) HashAggregate [codegen id : 1]
+Input [1]: [ss_ext_discount_amt#72]
+Keys: []
+Functions [1]: [partial_avg(UnscaledValue(ss_ext_discount_amt#72))]
+Aggregate Attributes [2]: [sum#74, count#75]
+Results [2]: [sum#76, count#77]
+
+(38) Exchange
+Input [2]: [sum#76, count#77]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#78]
+
+(39) HashAggregate [codegen id : 2]
+Input [2]: [sum#76, count#77]
+Keys: []
+Functions [1]: [avg(UnscaledValue(ss_ext_discount_amt#72))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_ext_discount_amt#72))#79]
+Results [1]: [cast((avg(UnscaledValue(ss_ext_discount_amt#72))#79 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#80]
+
+Subquery:6 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#13, [id=#14]
+* HashAggregate (46)
++- Exchange (45)
+   +- * HashAggregate (44)
+      +- * Project (43)
+         +- * Filter (42)
+            +- * ColumnarToRow (41)
+               +- Scan parquet default.store_sales (40)
+
+
+(40) Scan parquet default.store_sales
+Output [3]: [ss_quantity#81, ss_net_paid#82, ss_sold_date_sk#83]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,40)]
+ReadSchema: struct<ss_quantity:int,ss_net_paid:decimal(7,2)>
+
+(41) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_quantity#81, ss_net_paid#82, ss_sold_date_sk#83]
+
+(42) Filter [codegen id : 1]
+Input [3]: [ss_quantity#81, ss_net_paid#82, ss_sold_date_sk#83]
+Condition : ((isnotnull(ss_quantity#81) AND (ss_quantity#81 >= 21)) AND (ss_quantity#81 <= 40))
+
+(43) Project [codegen id : 1]
+Output [1]: [ss_net_paid#82]
+Input [3]: [ss_quantity#81, ss_net_paid#82, ss_sold_date_sk#83]
+
+(44) HashAggregate [codegen id : 1]
+Input [1]: [ss_net_paid#82]
+Keys: []
+Functions [1]: [partial_avg(UnscaledValue(ss_net_paid#82))]
+Aggregate Attributes [2]: [sum#84, count#85]
+Results [2]: [sum#86, count#87]
+
+(45) Exchange
+Input [2]: [sum#86, count#87]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#88]
+
+(46) HashAggregate [codegen id : 2]
+Input [2]: [sum#86, count#87]
+Keys: []
+Functions [1]: [avg(UnscaledValue(ss_net_paid#82))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_paid#82))#89]
+Results [1]: [cast((avg(UnscaledValue(ss_net_paid#82))#89 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#90]
+
+Subquery:7 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#16, [id=#17]
+* HashAggregate (53)
++- Exchange (52)
+   +- * HashAggregate (51)
+      +- * Project (50)
+         +- * Filter (49)
+            +- * ColumnarToRow (48)
+               +- Scan parquet default.store_sales (47)
+
+
+(47) Scan parquet default.store_sales
+Output [2]: [ss_quantity#91, ss_sold_date_sk#92]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,41), LessThanOrEqual(ss_quantity,60)]
+ReadSchema: struct<ss_quantity:int>
+
+(48) ColumnarToRow [codegen id : 1]
+Input [2]: [ss_quantity#91, ss_sold_date_sk#92]
+
+(49) Filter [codegen id : 1]
+Input [2]: [ss_quantity#91, ss_sold_date_sk#92]
+Condition : ((isnotnull(ss_quantity#91) AND (ss_quantity#91 >= 41)) AND (ss_quantity#91 <= 60))
+
+(50) Project [codegen id : 1]
+Output: []
+Input [2]: [ss_quantity#91, ss_sold_date_sk#92]
+
+(51) HashAggregate [codegen id : 1]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#93]
+Results [1]: [count#94]
+
+(52) Exchange
+Input [1]: [count#94]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#95]
+
+(53) HashAggregate [codegen id : 2]
+Input [1]: [count#94]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#96]
+Results [1]: [count(1)#96 AS count(1)#97]
+
+Subquery:8 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#18, [id=#19]
+* HashAggregate (60)
++- Exchange (59)
+   +- * HashAggregate (58)
+      +- * Project (57)
+         +- * Filter (56)
+            +- * ColumnarToRow (55)
+               +- Scan parquet default.store_sales (54)
+
+
+(54) Scan parquet default.store_sales
+Output [3]: [ss_quantity#98, ss_ext_discount_amt#99, ss_sold_date_sk#100]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,41), LessThanOrEqual(ss_quantity,60)]
+ReadSchema: struct<ss_quantity:int,ss_ext_discount_amt:decimal(7,2)>
+
+(55) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_quantity#98, ss_ext_discount_amt#99, ss_sold_date_sk#100]
+
+(56) Filter [codegen id : 1]
+Input [3]: [ss_quantity#98, ss_ext_discount_amt#99, ss_sold_date_sk#100]
+Condition : ((isnotnull(ss_quantity#98) AND (ss_quantity#98 >= 41)) AND (ss_quantity#98 <= 60))
+
+(57) Project [codegen id : 1]
+Output [1]: [ss_ext_discount_amt#99]
+Input [3]: [ss_quantity#98, ss_ext_discount_amt#99, ss_sold_date_sk#100]
+
+(58) HashAggregate [codegen id : 1]
+Input [1]: [ss_ext_discount_amt#99]
+Keys: []
+Functions [1]: [partial_avg(UnscaledValue(ss_ext_discount_amt#99))]
+Aggregate Attributes [2]: [sum#101, count#102]
+Results [2]: [sum#103, count#104]
+
+(59) Exchange
+Input [2]: [sum#103, count#104]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#105]
+
+(60) HashAggregate [codegen id : 2]
+Input [2]: [sum#103, count#104]
+Keys: []
+Functions [1]: [avg(UnscaledValue(ss_ext_discount_amt#99))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_ext_discount_amt#99))#106]
+Results [1]: [cast((avg(UnscaledValue(ss_ext_discount_amt#99))#106 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#107]
+
+Subquery:9 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#20, [id=#21]
+* HashAggregate (67)
++- Exchange (66)
+   +- * HashAggregate (65)
+      +- * Project (64)
+         +- * Filter (63)
+            +- * ColumnarToRow (62)
+               +- Scan parquet default.store_sales (61)
+
+
+(61) Scan parquet default.store_sales
+Output [3]: [ss_quantity#108, ss_net_paid#109, ss_sold_date_sk#110]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,41), LessThanOrEqual(ss_quantity,60)]
+ReadSchema: struct<ss_quantity:int,ss_net_paid:decimal(7,2)>
+
+(62) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_quantity#108, ss_net_paid#109, ss_sold_date_sk#110]
+
+(63) Filter [codegen id : 1]
+Input [3]: [ss_quantity#108, ss_net_paid#109, ss_sold_date_sk#110]
+Condition : ((isnotnull(ss_quantity#108) AND (ss_quantity#108 >= 41)) AND (ss_quantity#108 <= 60))
+
+(64) Project [codegen id : 1]
+Output [1]: [ss_net_paid#109]
+Input [3]: [ss_quantity#108, ss_net_paid#109, ss_sold_date_sk#110]
+
+(65) HashAggregate [codegen id : 1]
+Input [1]: [ss_net_paid#109]
+Keys: []
+Functions [1]: [partial_avg(UnscaledValue(ss_net_paid#109))]
+Aggregate Attributes [2]: [sum#111, count#112]
+Results [2]: [sum#113, count#114]
+
+(66) Exchange
+Input [2]: [sum#113, count#114]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#115]
+
+(67) HashAggregate [codegen id : 2]
+Input [2]: [sum#113, count#114]
+Keys: []
+Functions [1]: [avg(UnscaledValue(ss_net_paid#109))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_paid#109))#116]
+Results [1]: [cast((avg(UnscaledValue(ss_net_paid#109))#116 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#117]
+
+Subquery:10 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#23, [id=#24]
+* HashAggregate (74)
++- Exchange (73)
+   +- * HashAggregate (72)
+      +- * Project (71)
+         +- * Filter (70)
+            +- * ColumnarToRow (69)
+               +- Scan parquet default.store_sales (68)
+
+
+(68) Scan parquet default.store_sales
+Output [2]: [ss_quantity#118, ss_sold_date_sk#119]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,61), LessThanOrEqual(ss_quantity,80)]
+ReadSchema: struct<ss_quantity:int>
+
+(69) ColumnarToRow [codegen id : 1]
+Input [2]: [ss_quantity#118, ss_sold_date_sk#119]
+
+(70) Filter [codegen id : 1]
+Input [2]: [ss_quantity#118, ss_sold_date_sk#119]
+Condition : ((isnotnull(ss_quantity#118) AND (ss_quantity#118 >= 61)) AND (ss_quantity#118 <= 80))
+
+(71) Project [codegen id : 1]
+Output: []
+Input [2]: [ss_quantity#118, ss_sold_date_sk#119]
+
+(72) HashAggregate [codegen id : 1]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#120]
+Results [1]: [count#121]
+
+(73) Exchange
+Input [1]: [count#121]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#122]
+
+(74) HashAggregate [codegen id : 2]
+Input [1]: [count#121]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#123]
+Results [1]: [count(1)#123 AS count(1)#124]
+
+Subquery:11 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#25, [id=#26]
+* HashAggregate (81)
++- Exchange (80)
+   +- * HashAggregate (79)
+      +- * Project (78)
+         +- * Filter (77)
+            +- * ColumnarToRow (76)
+               +- Scan parquet default.store_sales (75)
+
+
+(75) Scan parquet default.store_sales
+Output [3]: [ss_quantity#125, ss_ext_discount_amt#126, ss_sold_date_sk#127]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,61), LessThanOrEqual(ss_quantity,80)]
+ReadSchema: struct<ss_quantity:int,ss_ext_discount_amt:decimal(7,2)>
+
+(76) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_quantity#125, ss_ext_discount_amt#126, ss_sold_date_sk#127]
+
+(77) Filter [codegen id : 1]
+Input [3]: [ss_quantity#125, ss_ext_discount_amt#126, ss_sold_date_sk#127]
+Condition : ((isnotnull(ss_quantity#125) AND (ss_quantity#125 >= 61)) AND (ss_quantity#125 <= 80))
+
+(78) Project [codegen id : 1]
+Output [1]: [ss_ext_discount_amt#126]
+Input [3]: [ss_quantity#125, ss_ext_discount_amt#126, ss_sold_date_sk#127]
+
+(79) HashAggregate [codegen id : 1]
+Input [1]: [ss_ext_discount_amt#126]
+Keys: []
+Functions [1]: [partial_avg(UnscaledValue(ss_ext_discount_amt#126))]
+Aggregate Attributes [2]: [sum#128, count#129]
+Results [2]: [sum#130, count#131]
+
+(80) Exchange
+Input [2]: [sum#130, count#131]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#132]
+
+(81) HashAggregate [codegen id : 2]
+Input [2]: [sum#130, count#131]
+Keys: []
+Functions [1]: [avg(UnscaledValue(ss_ext_discount_amt#126))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_ext_discount_amt#126))#133]
+Results [1]: [cast((avg(UnscaledValue(ss_ext_discount_amt#126))#133 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#134]
+
+Subquery:12 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#27, [id=#28]
+* HashAggregate (88)
++- Exchange (87)
+   +- * HashAggregate (86)
+      +- * Project (85)
+         +- * Filter (84)
+            +- * ColumnarToRow (83)
+               +- Scan parquet default.store_sales (82)
+
+
+(82) Scan parquet default.store_sales
+Output [3]: [ss_quantity#135, ss_net_paid#136, ss_sold_date_sk#137]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,61), LessThanOrEqual(ss_quantity,80)]
+ReadSchema: struct<ss_quantity:int,ss_net_paid:decimal(7,2)>
+
+(83) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_quantity#135, ss_net_paid#136, ss_sold_date_sk#137]
+
+(84) Filter [codegen id : 1]
+Input [3]: [ss_quantity#135, ss_net_paid#136, ss_sold_date_sk#137]
+Condition : ((isnotnull(ss_quantity#135) AND (ss_quantity#135 >= 61)) AND (ss_quantity#135 <= 80))
+
+(85) Project [codegen id : 1]
+Output [1]: [ss_net_paid#136]
+Input [3]: [ss_quantity#135, ss_net_paid#136, ss_sold_date_sk#137]
+
+(86) HashAggregate [codegen id : 1]
+Input [1]: [ss_net_paid#136]
+Keys: []
+Functions [1]: [partial_avg(UnscaledValue(ss_net_paid#136))]
+Aggregate Attributes [2]: [sum#138, count#139]
+Results [2]: [sum#140, count#141]
+
+(87) Exchange
+Input [2]: [sum#140, count#141]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#142]
+
+(88) HashAggregate [codegen id : 2]
+Input [2]: [sum#140, count#141]
+Keys: []
+Functions [1]: [avg(UnscaledValue(ss_net_paid#136))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_paid#136))#143]
+Results [1]: [cast((avg(UnscaledValue(ss_net_paid#136))#143 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#144]
+
+Subquery:13 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#30, [id=#31]
+* HashAggregate (95)
++- Exchange (94)
+   +- * HashAggregate (93)
+      +- * Project (92)
+         +- * Filter (91)
+            +- * ColumnarToRow (90)
+               +- Scan parquet default.store_sales (89)
+
+
+(89) Scan parquet default.store_sales
+Output [2]: [ss_quantity#145, ss_sold_date_sk#146]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,81), LessThanOrEqual(ss_quantity,100)]
+ReadSchema: struct<ss_quantity:int>
+
+(90) ColumnarToRow [codegen id : 1]
+Input [2]: [ss_quantity#145, ss_sold_date_sk#146]
+
+(91) Filter [codegen id : 1]
+Input [2]: [ss_quantity#145, ss_sold_date_sk#146]
+Condition : ((isnotnull(ss_quantity#145) AND (ss_quantity#145 >= 81)) AND (ss_quantity#145 <= 100))
+
+(92) Project [codegen id : 1]
+Output: []
+Input [2]: [ss_quantity#145, ss_sold_date_sk#146]
+
+(93) HashAggregate [codegen id : 1]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#147]
+Results [1]: [count#148]
+
+(94) Exchange
+Input [1]: [count#148]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#149]
+
+(95) HashAggregate [codegen id : 2]
+Input [1]: [count#148]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#150]
+Results [1]: [count(1)#150 AS count(1)#151]
+
+Subquery:14 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#32, [id=#33]
+* HashAggregate (102)
++- Exchange (101)
+   +- * HashAggregate (100)
+      +- * Project (99)
+         +- * Filter (98)
+            +- * ColumnarToRow (97)
+               +- Scan parquet default.store_sales (96)
+
+
+(96) Scan parquet default.store_sales
+Output [3]: [ss_quantity#152, ss_ext_discount_amt#153, ss_sold_date_sk#154]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,81), LessThanOrEqual(ss_quantity,100)]
+ReadSchema: struct<ss_quantity:int,ss_ext_discount_amt:decimal(7,2)>
+
+(97) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_quantity#152, ss_ext_discount_amt#153, ss_sold_date_sk#154]
+
+(98) Filter [codegen id : 1]
+Input [3]: [ss_quantity#152, ss_ext_discount_amt#153, ss_sold_date_sk#154]
+Condition : ((isnotnull(ss_quantity#152) AND (ss_quantity#152 >= 81)) AND (ss_quantity#152 <= 100))
+
+(99) Project [codegen id : 1]
+Output [1]: [ss_ext_discount_amt#153]
+Input [3]: [ss_quantity#152, ss_ext_discount_amt#153, ss_sold_date_sk#154]
+
+(100) HashAggregate [codegen id : 1]
+Input [1]: [ss_ext_discount_amt#153]
+Keys: []
+Functions [1]: [partial_avg(UnscaledValue(ss_ext_discount_amt#153))]
+Aggregate Attributes [2]: [sum#155, count#156]
+Results [2]: [sum#157, count#158]
+
+(101) Exchange
+Input [2]: [sum#157, count#158]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#159]
+
+(102) HashAggregate [codegen id : 2]
+Input [2]: [sum#157, count#158]
+Keys: []
+Functions [1]: [avg(UnscaledValue(ss_ext_discount_amt#153))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_ext_discount_amt#153))#160]
+Results [1]: [cast((avg(UnscaledValue(ss_ext_discount_amt#153))#160 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#161]
+
+Subquery:15 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#34, [id=#35]
+* HashAggregate (109)
++- Exchange (108)
+   +- * HashAggregate (107)
+      +- * Project (106)
+         +- * Filter (105)
+            +- * ColumnarToRow (104)
+               +- Scan parquet default.store_sales (103)
+
+
+(103) Scan parquet default.store_sales
+Output [3]: [ss_quantity#162, ss_net_paid#163, ss_sold_date_sk#164]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,81), LessThanOrEqual(ss_quantity,100)]
+ReadSchema: struct<ss_quantity:int,ss_net_paid:decimal(7,2)>
+
+(104) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_quantity#162, ss_net_paid#163, ss_sold_date_sk#164]
+
+(105) Filter [codegen id : 1]
+Input [3]: [ss_quantity#162, ss_net_paid#163, ss_sold_date_sk#164]
+Condition : ((isnotnull(ss_quantity#162) AND (ss_quantity#162 >= 81)) AND (ss_quantity#162 <= 100))
+
+(106) Project [codegen id : 1]
+Output [1]: [ss_net_paid#163]
+Input [3]: [ss_quantity#162, ss_net_paid#163, ss_sold_date_sk#164]
+
+(107) HashAggregate [codegen id : 1]
+Input [1]: [ss_net_paid#163]
+Keys: []
+Functions [1]: [partial_avg(UnscaledValue(ss_net_paid#163))]
+Aggregate Attributes [2]: [sum#165, count#166]
+Results [2]: [sum#167, count#168]
+
+(108) Exchange
+Input [2]: [sum#167, count#168]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#169]
+
+(109) HashAggregate [codegen id : 2]
+Input [2]: [sum#167, count#168]
+Keys: []
+Functions [1]: [avg(UnscaledValue(ss_net_paid#163))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_paid#163))#170]
+Results [1]: [cast((avg(UnscaledValue(ss_net_paid#163))#170 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#171]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.sf100/simplified.txt
new file mode 100644
index 0000000000000..1c42d8f2638c6
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.sf100/simplified.txt
@@ -0,0 +1,186 @@
+WholeStageCodegen (1)
+  Project
+    Subquery #1
+      WholeStageCodegen (2)
+        HashAggregate [count] [count(1),count(1),count]
+          InputAdapter
+            Exchange #1
+              WholeStageCodegen (1)
+                HashAggregate [count,count]
+                  Project
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_sold_date_sk]
+    Subquery #2
+      WholeStageCodegen (2)
+        HashAggregate [sum,count] [avg(UnscaledValue(ss_ext_discount_amt)),avg(ss_ext_discount_amt),sum,count]
+          InputAdapter
+            Exchange #2
+              WholeStageCodegen (1)
+                HashAggregate [ss_ext_discount_amt] [sum,count,sum,count]
+                  Project [ss_ext_discount_amt]
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_ext_discount_amt,ss_sold_date_sk]
+    Subquery #3
+      WholeStageCodegen (2)
+        HashAggregate [sum,count] [avg(UnscaledValue(ss_net_paid)),avg(ss_net_paid),sum,count]
+          InputAdapter
+            Exchange #3
+              WholeStageCodegen (1)
+                HashAggregate [ss_net_paid] [sum,count,sum,count]
+                  Project [ss_net_paid]
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_net_paid,ss_sold_date_sk]
+    Subquery #4
+      WholeStageCodegen (2)
+        HashAggregate [count] [count(1),count(1),count]
+          InputAdapter
+            Exchange #4
+              WholeStageCodegen (1)
+                HashAggregate [count,count]
+                  Project
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_sold_date_sk]
+    Subquery #5
+      WholeStageCodegen (2)
+        HashAggregate [sum,count] [avg(UnscaledValue(ss_ext_discount_amt)),avg(ss_ext_discount_amt),sum,count]
+          InputAdapter
+            Exchange #5
+              WholeStageCodegen (1)
+                HashAggregate [ss_ext_discount_amt] [sum,count,sum,count]
+                  Project [ss_ext_discount_amt]
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_ext_discount_amt,ss_sold_date_sk]
+    Subquery #6
+      WholeStageCodegen (2)
+        HashAggregate [sum,count] [avg(UnscaledValue(ss_net_paid)),avg(ss_net_paid),sum,count]
+          InputAdapter
+            Exchange #6
+              WholeStageCodegen (1)
+                HashAggregate [ss_net_paid] [sum,count,sum,count]
+                  Project [ss_net_paid]
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_net_paid,ss_sold_date_sk]
+    Subquery #7
+      WholeStageCodegen (2)
+        HashAggregate [count] [count(1),count(1),count]
+          InputAdapter
+            Exchange #7
+              WholeStageCodegen (1)
+                HashAggregate [count,count]
+                  Project
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_sold_date_sk]
+    Subquery #8
+      WholeStageCodegen (2)
+        HashAggregate [sum,count] [avg(UnscaledValue(ss_ext_discount_amt)),avg(ss_ext_discount_amt),sum,count]
+          InputAdapter
+            Exchange #8
+              WholeStageCodegen (1)
+                HashAggregate [ss_ext_discount_amt] [sum,count,sum,count]
+                  Project [ss_ext_discount_amt]
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_ext_discount_amt,ss_sold_date_sk]
+    Subquery #9
+      WholeStageCodegen (2)
+        HashAggregate [sum,count] [avg(UnscaledValue(ss_net_paid)),avg(ss_net_paid),sum,count]
+          InputAdapter
+            Exchange #9
+              WholeStageCodegen (1)
+                HashAggregate [ss_net_paid] [sum,count,sum,count]
+                  Project [ss_net_paid]
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_net_paid,ss_sold_date_sk]
+    Subquery #10
+      WholeStageCodegen (2)
+        HashAggregate [count] [count(1),count(1),count]
+          InputAdapter
+            Exchange #10
+              WholeStageCodegen (1)
+                HashAggregate [count,count]
+                  Project
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_sold_date_sk]
+    Subquery #11
+      WholeStageCodegen (2)
+        HashAggregate [sum,count] [avg(UnscaledValue(ss_ext_discount_amt)),avg(ss_ext_discount_amt),sum,count]
+          InputAdapter
+            Exchange #11
+              WholeStageCodegen (1)
+                HashAggregate [ss_ext_discount_amt] [sum,count,sum,count]
+                  Project [ss_ext_discount_amt]
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_ext_discount_amt,ss_sold_date_sk]
+    Subquery #12
+      WholeStageCodegen (2)
+        HashAggregate [sum,count] [avg(UnscaledValue(ss_net_paid)),avg(ss_net_paid),sum,count]
+          InputAdapter
+            Exchange #12
+              WholeStageCodegen (1)
+                HashAggregate [ss_net_paid] [sum,count,sum,count]
+                  Project [ss_net_paid]
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_net_paid,ss_sold_date_sk]
+    Subquery #13
+      WholeStageCodegen (2)
+        HashAggregate [count] [count(1),count(1),count]
+          InputAdapter
+            Exchange #13
+              WholeStageCodegen (1)
+                HashAggregate [count,count]
+                  Project
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_sold_date_sk]
+    Subquery #14
+      WholeStageCodegen (2)
+        HashAggregate [sum,count] [avg(UnscaledValue(ss_ext_discount_amt)),avg(ss_ext_discount_amt),sum,count]
+          InputAdapter
+            Exchange #14
+              WholeStageCodegen (1)
+                HashAggregate [ss_ext_discount_amt] [sum,count,sum,count]
+                  Project [ss_ext_discount_amt]
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_ext_discount_amt,ss_sold_date_sk]
+    Subquery #15
+      WholeStageCodegen (2)
+        HashAggregate [sum,count] [avg(UnscaledValue(ss_net_paid)),avg(ss_net_paid),sum,count]
+          InputAdapter
+            Exchange #15
+              WholeStageCodegen (1)
+                HashAggregate [ss_net_paid] [sum,count,sum,count]
+                  Project [ss_net_paid]
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_net_paid,ss_sold_date_sk]
+    Filter [r_reason_sk]
+      ColumnarToRow
+        InputAdapter
+          Scan parquet default.reason [r_reason_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/explain.txt
new file mode 100644
index 0000000000000..8736c9861a5ce
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/explain.txt
@@ -0,0 +1,718 @@
+== Physical Plan ==
+* Project (4)
++- * Filter (3)
+   +- * ColumnarToRow (2)
+      +- Scan parquet default.reason (1)
+
+
+(1) Scan parquet default.reason
+Output [1]: [r_reason_sk#1]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/reason]
+PushedFilters: [IsNotNull(r_reason_sk), EqualTo(r_reason_sk,1)]
+ReadSchema: struct<r_reason_sk:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [1]: [r_reason_sk#1]
+
+(3) Filter [codegen id : 1]
+Input [1]: [r_reason_sk#1]
+Condition : (isnotnull(r_reason_sk#1) AND (r_reason_sk#1 = 1))
+
+(4) Project [codegen id : 1]
+Output [5]: [CASE WHEN (Subquery scalar-subquery#2, [id=#3] > 62316685) THEN Subquery scalar-subquery#4, [id=#5] ELSE Subquery scalar-subquery#6, [id=#7] END AS bucket1#8, CASE WHEN (Subquery scalar-subquery#9, [id=#10] > 19045798) THEN Subquery scalar-subquery#11, [id=#12] ELSE Subquery scalar-subquery#13, [id=#14] END AS bucket2#15, CASE WHEN (Subquery scalar-subquery#16, [id=#17] > 365541424) THEN Subquery scalar-subquery#18, [id=#19] ELSE Subquery scalar-subquery#20, [id=#21] END AS bucket3#22, CASE WHEN (Subquery scalar-subquery#23, [id=#24] > 216357808) THEN Subquery scalar-subquery#25, [id=#26] ELSE Subquery scalar-subquery#27, [id=#28] END AS bucket4#29, CASE WHEN (Subquery scalar-subquery#30, [id=#31] > 184483884) THEN Subquery scalar-subquery#32, [id=#33] ELSE Subquery scalar-subquery#34, [id=#35] END AS bucket5#36]
+Input [1]: [r_reason_sk#1]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#2, [id=#3]
+* HashAggregate (11)
++- Exchange (10)
+   +- * HashAggregate (9)
+      +- * Project (8)
+         +- * Filter (7)
+            +- * ColumnarToRow (6)
+               +- Scan parquet default.store_sales (5)
+
+
+(5) Scan parquet default.store_sales
+Output [2]: [ss_quantity#37, ss_sold_date_sk#38]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,1), LessThanOrEqual(ss_quantity,20)]
+ReadSchema: struct<ss_quantity:int>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [2]: [ss_quantity#37, ss_sold_date_sk#38]
+
+(7) Filter [codegen id : 1]
+Input [2]: [ss_quantity#37, ss_sold_date_sk#38]
+Condition : ((isnotnull(ss_quantity#37) AND (ss_quantity#37 >= 1)) AND (ss_quantity#37 <= 20))
+
+(8) Project [codegen id : 1]
+Output: []
+Input [2]: [ss_quantity#37, ss_sold_date_sk#38]
+
+(9) HashAggregate [codegen id : 1]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#39]
+Results [1]: [count#40]
+
+(10) Exchange
+Input [1]: [count#40]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#41]
+
+(11) HashAggregate [codegen id : 2]
+Input [1]: [count#40]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#42]
+Results [1]: [count(1)#42 AS count(1)#43]
+
+Subquery:2 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#4, [id=#5]
+* HashAggregate (18)
++- Exchange (17)
+   +- * HashAggregate (16)
+      +- * Project (15)
+         +- * Filter (14)
+            +- * ColumnarToRow (13)
+               +- Scan parquet default.store_sales (12)
+
+
+(12) Scan parquet default.store_sales
+Output [3]: [ss_quantity#44, ss_ext_discount_amt#45, ss_sold_date_sk#46]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,1), LessThanOrEqual(ss_quantity,20)]
+ReadSchema: struct<ss_quantity:int,ss_ext_discount_amt:decimal(7,2)>
+
+(13) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_quantity#44, ss_ext_discount_amt#45, ss_sold_date_sk#46]
+
+(14) Filter [codegen id : 1]
+Input [3]: [ss_quantity#44, ss_ext_discount_amt#45, ss_sold_date_sk#46]
+Condition : ((isnotnull(ss_quantity#44) AND (ss_quantity#44 >= 1)) AND (ss_quantity#44 <= 20))
+
+(15) Project [codegen id : 1]
+Output [1]: [ss_ext_discount_amt#45]
+Input [3]: [ss_quantity#44, ss_ext_discount_amt#45, ss_sold_date_sk#46]
+
+(16) HashAggregate [codegen id : 1]
+Input [1]: [ss_ext_discount_amt#45]
+Keys: []
+Functions [1]: [partial_avg(UnscaledValue(ss_ext_discount_amt#45))]
+Aggregate Attributes [2]: [sum#47, count#48]
+Results [2]: [sum#49, count#50]
+
+(17) Exchange
+Input [2]: [sum#49, count#50]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#51]
+
+(18) HashAggregate [codegen id : 2]
+Input [2]: [sum#49, count#50]
+Keys: []
+Functions [1]: [avg(UnscaledValue(ss_ext_discount_amt#45))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_ext_discount_amt#45))#52]
+Results [1]: [cast((avg(UnscaledValue(ss_ext_discount_amt#45))#52 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#53]
+
+Subquery:3 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#6, [id=#7]
+* HashAggregate (25)
++- Exchange (24)
+   +- * HashAggregate (23)
+      +- * Project (22)
+         +- * Filter (21)
+            +- * ColumnarToRow (20)
+               +- Scan parquet default.store_sales (19)
+
+
+(19) Scan parquet default.store_sales
+Output [3]: [ss_quantity#54, ss_net_paid#55, ss_sold_date_sk#56]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,1), LessThanOrEqual(ss_quantity,20)]
+ReadSchema: struct<ss_quantity:int,ss_net_paid:decimal(7,2)>
+
+(20) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_quantity#54, ss_net_paid#55, ss_sold_date_sk#56]
+
+(21) Filter [codegen id : 1]
+Input [3]: [ss_quantity#54, ss_net_paid#55, ss_sold_date_sk#56]
+Condition : ((isnotnull(ss_quantity#54) AND (ss_quantity#54 >= 1)) AND (ss_quantity#54 <= 20))
+
+(22) Project [codegen id : 1]
+Output [1]: [ss_net_paid#55]
+Input [3]: [ss_quantity#54, ss_net_paid#55, ss_sold_date_sk#56]
+
+(23) HashAggregate [codegen id : 1]
+Input [1]: [ss_net_paid#55]
+Keys: []
+Functions [1]: [partial_avg(UnscaledValue(ss_net_paid#55))]
+Aggregate Attributes [2]: [sum#57, count#58]
+Results [2]: [sum#59, count#60]
+
+(24) Exchange
+Input [2]: [sum#59, count#60]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#61]
+
+(25) HashAggregate [codegen id : 2]
+Input [2]: [sum#59, count#60]
+Keys: []
+Functions [1]: [avg(UnscaledValue(ss_net_paid#55))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_paid#55))#62]
+Results [1]: [cast((avg(UnscaledValue(ss_net_paid#55))#62 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#63]
+
+Subquery:4 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#9, [id=#10]
+* HashAggregate (32)
++- Exchange (31)
+   +- * HashAggregate (30)
+      +- * Project (29)
+         +- * Filter (28)
+            +- * ColumnarToRow (27)
+               +- Scan parquet default.store_sales (26)
+
+
+(26) Scan parquet default.store_sales
+Output [2]: [ss_quantity#64, ss_sold_date_sk#65]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,40)]
+ReadSchema: struct<ss_quantity:int>
+
+(27) ColumnarToRow [codegen id : 1]
+Input [2]: [ss_quantity#64, ss_sold_date_sk#65]
+
+(28) Filter [codegen id : 1]
+Input [2]: [ss_quantity#64, ss_sold_date_sk#65]
+Condition : ((isnotnull(ss_quantity#64) AND (ss_quantity#64 >= 21)) AND (ss_quantity#64 <= 40))
+
+(29) Project [codegen id : 1]
+Output: []
+Input [2]: [ss_quantity#64, ss_sold_date_sk#65]
+
+(30) HashAggregate [codegen id : 1]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#66]
+Results [1]: [count#67]
+
+(31) Exchange
+Input [1]: [count#67]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#68]
+
+(32) HashAggregate [codegen id : 2]
+Input [1]: [count#67]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#69]
+Results [1]: [count(1)#69 AS count(1)#70]
+
+Subquery:5 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#11, [id=#12]
+* HashAggregate (39)
++- Exchange (38)
+   +- * HashAggregate (37)
+      +- * Project (36)
+         +- * Filter (35)
+            +- * ColumnarToRow (34)
+               +- Scan parquet default.store_sales (33)
+
+
+(33) Scan parquet default.store_sales
+Output [3]: [ss_quantity#71, ss_ext_discount_amt#72, ss_sold_date_sk#73]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,40)]
+ReadSchema: struct<ss_quantity:int,ss_ext_discount_amt:decimal(7,2)>
+
+(34) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_quantity#71, ss_ext_discount_amt#72, ss_sold_date_sk#73]
+
+(35) Filter [codegen id : 1]
+Input [3]: [ss_quantity#71, ss_ext_discount_amt#72, ss_sold_date_sk#73]
+Condition : ((isnotnull(ss_quantity#71) AND (ss_quantity#71 >= 21)) AND (ss_quantity#71 <= 40))
+
+(36) Project [codegen id : 1]
+Output [1]: [ss_ext_discount_amt#72]
+Input [3]: [ss_quantity#71, ss_ext_discount_amt#72, ss_sold_date_sk#73]
+
+(37) HashAggregate [codegen id : 1]
+Input [1]: [ss_ext_discount_amt#72]
+Keys: []
+Functions [1]: [partial_avg(UnscaledValue(ss_ext_discount_amt#72))]
+Aggregate Attributes [2]: [sum#74, count#75]
+Results [2]: [sum#76, count#77]
+
+(38) Exchange
+Input [2]: [sum#76, count#77]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#78]
+
+(39) HashAggregate [codegen id : 2]
+Input [2]: [sum#76, count#77]
+Keys: []
+Functions [1]: [avg(UnscaledValue(ss_ext_discount_amt#72))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_ext_discount_amt#72))#79]
+Results [1]: [cast((avg(UnscaledValue(ss_ext_discount_amt#72))#79 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#80]
+
+Subquery:6 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#13, [id=#14]
+* HashAggregate (46)
++- Exchange (45)
+   +- * HashAggregate (44)
+      +- * Project (43)
+         +- * Filter (42)
+            +- * ColumnarToRow (41)
+               +- Scan parquet default.store_sales (40)
+
+
+(40) Scan parquet default.store_sales
+Output [3]: [ss_quantity#81, ss_net_paid#82, ss_sold_date_sk#83]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,40)]
+ReadSchema: struct<ss_quantity:int,ss_net_paid:decimal(7,2)>
+
+(41) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_quantity#81, ss_net_paid#82, ss_sold_date_sk#83]
+
+(42) Filter [codegen id : 1]
+Input [3]: [ss_quantity#81, ss_net_paid#82, ss_sold_date_sk#83]
+Condition : ((isnotnull(ss_quantity#81) AND (ss_quantity#81 >= 21)) AND (ss_quantity#81 <= 40))
+
+(43) Project [codegen id : 1]
+Output [1]: [ss_net_paid#82]
+Input [3]: [ss_quantity#81, ss_net_paid#82, ss_sold_date_sk#83]
+
+(44) HashAggregate [codegen id : 1]
+Input [1]: [ss_net_paid#82]
+Keys: []
+Functions [1]: [partial_avg(UnscaledValue(ss_net_paid#82))]
+Aggregate Attributes [2]: [sum#84, count#85]
+Results [2]: [sum#86, count#87]
+
+(45) Exchange
+Input [2]: [sum#86, count#87]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#88]
+
+(46) HashAggregate [codegen id : 2]
+Input [2]: [sum#86, count#87]
+Keys: []
+Functions [1]: [avg(UnscaledValue(ss_net_paid#82))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_paid#82))#89]
+Results [1]: [cast((avg(UnscaledValue(ss_net_paid#82))#89 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#90]
+
+Subquery:7 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#16, [id=#17]
+* HashAggregate (53)
++- Exchange (52)
+   +- * HashAggregate (51)
+      +- * Project (50)
+         +- * Filter (49)
+            +- * ColumnarToRow (48)
+               +- Scan parquet default.store_sales (47)
+
+
+(47) Scan parquet default.store_sales
+Output [2]: [ss_quantity#91, ss_sold_date_sk#92]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,41), LessThanOrEqual(ss_quantity,60)]
+ReadSchema: struct<ss_quantity:int>
+
+(48) ColumnarToRow [codegen id : 1]
+Input [2]: [ss_quantity#91, ss_sold_date_sk#92]
+
+(49) Filter [codegen id : 1]
+Input [2]: [ss_quantity#91, ss_sold_date_sk#92]
+Condition : ((isnotnull(ss_quantity#91) AND (ss_quantity#91 >= 41)) AND (ss_quantity#91 <= 60))
+
+(50) Project [codegen id : 1]
+Output: []
+Input [2]: [ss_quantity#91, ss_sold_date_sk#92]
+
+(51) HashAggregate [codegen id : 1]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#93]
+Results [1]: [count#94]
+
+(52) Exchange
+Input [1]: [count#94]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#95]
+
+(53) HashAggregate [codegen id : 2]
+Input [1]: [count#94]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#96]
+Results [1]: [count(1)#96 AS count(1)#97]
+
+Subquery:8 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#18, [id=#19]
+* HashAggregate (60)
++- Exchange (59)
+   +- * HashAggregate (58)
+      +- * Project (57)
+         +- * Filter (56)
+            +- * ColumnarToRow (55)
+               +- Scan parquet default.store_sales (54)
+
+
+(54) Scan parquet default.store_sales
+Output [3]: [ss_quantity#98, ss_ext_discount_amt#99, ss_sold_date_sk#100]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,41), LessThanOrEqual(ss_quantity,60)]
+ReadSchema: struct<ss_quantity:int,ss_ext_discount_amt:decimal(7,2)>
+
+(55) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_quantity#98, ss_ext_discount_amt#99, ss_sold_date_sk#100]
+
+(56) Filter [codegen id : 1]
+Input [3]: [ss_quantity#98, ss_ext_discount_amt#99, ss_sold_date_sk#100]
+Condition : ((isnotnull(ss_quantity#98) AND (ss_quantity#98 >= 41)) AND (ss_quantity#98 <= 60))
+
+(57) Project [codegen id : 1]
+Output [1]: [ss_ext_discount_amt#99]
+Input [3]: [ss_quantity#98, ss_ext_discount_amt#99, ss_sold_date_sk#100]
+
+(58) HashAggregate [codegen id : 1]
+Input [1]: [ss_ext_discount_amt#99]
+Keys: []
+Functions [1]: [partial_avg(UnscaledValue(ss_ext_discount_amt#99))]
+Aggregate Attributes [2]: [sum#101, count#102]
+Results [2]: [sum#103, count#104]
+
+(59) Exchange
+Input [2]: [sum#103, count#104]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#105]
+
+(60) HashAggregate [codegen id : 2]
+Input [2]: [sum#103, count#104]
+Keys: []
+Functions [1]: [avg(UnscaledValue(ss_ext_discount_amt#99))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_ext_discount_amt#99))#106]
+Results [1]: [cast((avg(UnscaledValue(ss_ext_discount_amt#99))#106 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#107]
+
+Subquery:9 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#20, [id=#21]
+* HashAggregate (67)
++- Exchange (66)
+   +- * HashAggregate (65)
+      +- * Project (64)
+         +- * Filter (63)
+            +- * ColumnarToRow (62)
+               +- Scan parquet default.store_sales (61)
+
+
+(61) Scan parquet default.store_sales
+Output [3]: [ss_quantity#108, ss_net_paid#109, ss_sold_date_sk#110]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,41), LessThanOrEqual(ss_quantity,60)]
+ReadSchema: struct<ss_quantity:int,ss_net_paid:decimal(7,2)>
+
+(62) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_quantity#108, ss_net_paid#109, ss_sold_date_sk#110]
+
+(63) Filter [codegen id : 1]
+Input [3]: [ss_quantity#108, ss_net_paid#109, ss_sold_date_sk#110]
+Condition : ((isnotnull(ss_quantity#108) AND (ss_quantity#108 >= 41)) AND (ss_quantity#108 <= 60))
+
+(64) Project [codegen id : 1]
+Output [1]: [ss_net_paid#109]
+Input [3]: [ss_quantity#108, ss_net_paid#109, ss_sold_date_sk#110]
+
+(65) HashAggregate [codegen id : 1]
+Input [1]: [ss_net_paid#109]
+Keys: []
+Functions [1]: [partial_avg(UnscaledValue(ss_net_paid#109))]
+Aggregate Attributes [2]: [sum#111, count#112]
+Results [2]: [sum#113, count#114]
+
+(66) Exchange
+Input [2]: [sum#113, count#114]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#115]
+
+(67) HashAggregate [codegen id : 2]
+Input [2]: [sum#113, count#114]
+Keys: []
+Functions [1]: [avg(UnscaledValue(ss_net_paid#109))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_paid#109))#116]
+Results [1]: [cast((avg(UnscaledValue(ss_net_paid#109))#116 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#117]
+
+Subquery:10 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#23, [id=#24]
+* HashAggregate (74)
++- Exchange (73)
+   +- * HashAggregate (72)
+      +- * Project (71)
+         +- * Filter (70)
+            +- * ColumnarToRow (69)
+               +- Scan parquet default.store_sales (68)
+
+
+(68) Scan parquet default.store_sales
+Output [2]: [ss_quantity#118, ss_sold_date_sk#119]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,61), LessThanOrEqual(ss_quantity,80)]
+ReadSchema: struct<ss_quantity:int>
+
+(69) ColumnarToRow [codegen id : 1]
+Input [2]: [ss_quantity#118, ss_sold_date_sk#119]
+
+(70) Filter [codegen id : 1]
+Input [2]: [ss_quantity#118, ss_sold_date_sk#119]
+Condition : ((isnotnull(ss_quantity#118) AND (ss_quantity#118 >= 61)) AND (ss_quantity#118 <= 80))
+
+(71) Project [codegen id : 1]
+Output: []
+Input [2]: [ss_quantity#118, ss_sold_date_sk#119]
+
+(72) HashAggregate [codegen id : 1]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#120]
+Results [1]: [count#121]
+
+(73) Exchange
+Input [1]: [count#121]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#122]
+
+(74) HashAggregate [codegen id : 2]
+Input [1]: [count#121]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#123]
+Results [1]: [count(1)#123 AS count(1)#124]
+
+Subquery:11 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#25, [id=#26]
+* HashAggregate (81)
++- Exchange (80)
+   +- * HashAggregate (79)
+      +- * Project (78)
+         +- * Filter (77)
+            +- * ColumnarToRow (76)
+               +- Scan parquet default.store_sales (75)
+
+
+(75) Scan parquet default.store_sales
+Output [3]: [ss_quantity#125, ss_ext_discount_amt#126, ss_sold_date_sk#127]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,61), LessThanOrEqual(ss_quantity,80)]
+ReadSchema: struct<ss_quantity:int,ss_ext_discount_amt:decimal(7,2)>
+
+(76) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_quantity#125, ss_ext_discount_amt#126, ss_sold_date_sk#127]
+
+(77) Filter [codegen id : 1]
+Input [3]: [ss_quantity#125, ss_ext_discount_amt#126, ss_sold_date_sk#127]
+Condition : ((isnotnull(ss_quantity#125) AND (ss_quantity#125 >= 61)) AND (ss_quantity#125 <= 80))
+
+(78) Project [codegen id : 1]
+Output [1]: [ss_ext_discount_amt#126]
+Input [3]: [ss_quantity#125, ss_ext_discount_amt#126, ss_sold_date_sk#127]
+
+(79) HashAggregate [codegen id : 1]
+Input [1]: [ss_ext_discount_amt#126]
+Keys: []
+Functions [1]: [partial_avg(UnscaledValue(ss_ext_discount_amt#126))]
+Aggregate Attributes [2]: [sum#128, count#129]
+Results [2]: [sum#130, count#131]
+
+(80) Exchange
+Input [2]: [sum#130, count#131]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#132]
+
+(81) HashAggregate [codegen id : 2]
+Input [2]: [sum#130, count#131]
+Keys: []
+Functions [1]: [avg(UnscaledValue(ss_ext_discount_amt#126))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_ext_discount_amt#126))#133]
+Results [1]: [cast((avg(UnscaledValue(ss_ext_discount_amt#126))#133 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#134]
+
+Subquery:12 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#27, [id=#28]
+* HashAggregate (88)
++- Exchange (87)
+   +- * HashAggregate (86)
+      +- * Project (85)
+         +- * Filter (84)
+            +- * ColumnarToRow (83)
+               +- Scan parquet default.store_sales (82)
+
+
+(82) Scan parquet default.store_sales
+Output [3]: [ss_quantity#135, ss_net_paid#136, ss_sold_date_sk#137]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,61), LessThanOrEqual(ss_quantity,80)]
+ReadSchema: struct<ss_quantity:int,ss_net_paid:decimal(7,2)>
+
+(83) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_quantity#135, ss_net_paid#136, ss_sold_date_sk#137]
+
+(84) Filter [codegen id : 1]
+Input [3]: [ss_quantity#135, ss_net_paid#136, ss_sold_date_sk#137]
+Condition : ((isnotnull(ss_quantity#135) AND (ss_quantity#135 >= 61)) AND (ss_quantity#135 <= 80))
+
+(85) Project [codegen id : 1]
+Output [1]: [ss_net_paid#136]
+Input [3]: [ss_quantity#135, ss_net_paid#136, ss_sold_date_sk#137]
+
+(86) HashAggregate [codegen id : 1]
+Input [1]: [ss_net_paid#136]
+Keys: []
+Functions [1]: [partial_avg(UnscaledValue(ss_net_paid#136))]
+Aggregate Attributes [2]: [sum#138, count#139]
+Results [2]: [sum#140, count#141]
+
+(87) Exchange
+Input [2]: [sum#140, count#141]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#142]
+
+(88) HashAggregate [codegen id : 2]
+Input [2]: [sum#140, count#141]
+Keys: []
+Functions [1]: [avg(UnscaledValue(ss_net_paid#136))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_paid#136))#143]
+Results [1]: [cast((avg(UnscaledValue(ss_net_paid#136))#143 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#144]
+
+Subquery:13 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#30, [id=#31]
+* HashAggregate (95)
++- Exchange (94)
+   +- * HashAggregate (93)
+      +- * Project (92)
+         +- * Filter (91)
+            +- * ColumnarToRow (90)
+               +- Scan parquet default.store_sales (89)
+
+
+(89) Scan parquet default.store_sales
+Output [2]: [ss_quantity#145, ss_sold_date_sk#146]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,81), LessThanOrEqual(ss_quantity,100)]
+ReadSchema: struct<ss_quantity:int>
+
+(90) ColumnarToRow [codegen id : 1]
+Input [2]: [ss_quantity#145, ss_sold_date_sk#146]
+
+(91) Filter [codegen id : 1]
+Input [2]: [ss_quantity#145, ss_sold_date_sk#146]
+Condition : ((isnotnull(ss_quantity#145) AND (ss_quantity#145 >= 81)) AND (ss_quantity#145 <= 100))
+
+(92) Project [codegen id : 1]
+Output: []
+Input [2]: [ss_quantity#145, ss_sold_date_sk#146]
+
+(93) HashAggregate [codegen id : 1]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#147]
+Results [1]: [count#148]
+
+(94) Exchange
+Input [1]: [count#148]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#149]
+
+(95) HashAggregate [codegen id : 2]
+Input [1]: [count#148]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#150]
+Results [1]: [count(1)#150 AS count(1)#151]
+
+Subquery:14 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#32, [id=#33]
+* HashAggregate (102)
++- Exchange (101)
+   +- * HashAggregate (100)
+      +- * Project (99)
+         +- * Filter (98)
+            +- * ColumnarToRow (97)
+               +- Scan parquet default.store_sales (96)
+
+
+(96) Scan parquet default.store_sales
+Output [3]: [ss_quantity#152, ss_ext_discount_amt#153, ss_sold_date_sk#154]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,81), LessThanOrEqual(ss_quantity,100)]
+ReadSchema: struct<ss_quantity:int,ss_ext_discount_amt:decimal(7,2)>
+
+(97) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_quantity#152, ss_ext_discount_amt#153, ss_sold_date_sk#154]
+
+(98) Filter [codegen id : 1]
+Input [3]: [ss_quantity#152, ss_ext_discount_amt#153, ss_sold_date_sk#154]
+Condition : ((isnotnull(ss_quantity#152) AND (ss_quantity#152 >= 81)) AND (ss_quantity#152 <= 100))
+
+(99) Project [codegen id : 1]
+Output [1]: [ss_ext_discount_amt#153]
+Input [3]: [ss_quantity#152, ss_ext_discount_amt#153, ss_sold_date_sk#154]
+
+(100) HashAggregate [codegen id : 1]
+Input [1]: [ss_ext_discount_amt#153]
+Keys: []
+Functions [1]: [partial_avg(UnscaledValue(ss_ext_discount_amt#153))]
+Aggregate Attributes [2]: [sum#155, count#156]
+Results [2]: [sum#157, count#158]
+
+(101) Exchange
+Input [2]: [sum#157, count#158]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#159]
+
+(102) HashAggregate [codegen id : 2]
+Input [2]: [sum#157, count#158]
+Keys: []
+Functions [1]: [avg(UnscaledValue(ss_ext_discount_amt#153))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_ext_discount_amt#153))#160]
+Results [1]: [cast((avg(UnscaledValue(ss_ext_discount_amt#153))#160 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#161]
+
+Subquery:15 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#34, [id=#35]
+* HashAggregate (109)
++- Exchange (108)
+   +- * HashAggregate (107)
+      +- * Project (106)
+         +- * Filter (105)
+            +- * ColumnarToRow (104)
+               +- Scan parquet default.store_sales (103)
+
+
+(103) Scan parquet default.store_sales
+Output [3]: [ss_quantity#162, ss_net_paid#163, ss_sold_date_sk#164]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,81), LessThanOrEqual(ss_quantity,100)]
+ReadSchema: struct<ss_quantity:int,ss_net_paid:decimal(7,2)>
+
+(104) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_quantity#162, ss_net_paid#163, ss_sold_date_sk#164]
+
+(105) Filter [codegen id : 1]
+Input [3]: [ss_quantity#162, ss_net_paid#163, ss_sold_date_sk#164]
+Condition : ((isnotnull(ss_quantity#162) AND (ss_quantity#162 >= 81)) AND (ss_quantity#162 <= 100))
+
+(106) Project [codegen id : 1]
+Output [1]: [ss_net_paid#163]
+Input [3]: [ss_quantity#162, ss_net_paid#163, ss_sold_date_sk#164]
+
+(107) HashAggregate [codegen id : 1]
+Input [1]: [ss_net_paid#163]
+Keys: []
+Functions [1]: [partial_avg(UnscaledValue(ss_net_paid#163))]
+Aggregate Attributes [2]: [sum#165, count#166]
+Results [2]: [sum#167, count#168]
+
+(108) Exchange
+Input [2]: [sum#167, count#168]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#169]
+
+(109) HashAggregate [codegen id : 2]
+Input [2]: [sum#167, count#168]
+Keys: []
+Functions [1]: [avg(UnscaledValue(ss_net_paid#163))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_paid#163))#170]
+Results [1]: [cast((avg(UnscaledValue(ss_net_paid#163))#170 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#171]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/simplified.txt
new file mode 100644
index 0000000000000..1c42d8f2638c6
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/simplified.txt
@@ -0,0 +1,186 @@
+WholeStageCodegen (1)
+  Project
+    Subquery #1
+      WholeStageCodegen (2)
+        HashAggregate [count] [count(1),count(1),count]
+          InputAdapter
+            Exchange #1
+              WholeStageCodegen (1)
+                HashAggregate [count,count]
+                  Project
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_sold_date_sk]
+    Subquery #2
+      WholeStageCodegen (2)
+        HashAggregate [sum,count] [avg(UnscaledValue(ss_ext_discount_amt)),avg(ss_ext_discount_amt),sum,count]
+          InputAdapter
+            Exchange #2
+              WholeStageCodegen (1)
+                HashAggregate [ss_ext_discount_amt] [sum,count,sum,count]
+                  Project [ss_ext_discount_amt]
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_ext_discount_amt,ss_sold_date_sk]
+    Subquery #3
+      WholeStageCodegen (2)
+        HashAggregate [sum,count] [avg(UnscaledValue(ss_net_paid)),avg(ss_net_paid),sum,count]
+          InputAdapter
+            Exchange #3
+              WholeStageCodegen (1)
+                HashAggregate [ss_net_paid] [sum,count,sum,count]
+                  Project [ss_net_paid]
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_net_paid,ss_sold_date_sk]
+    Subquery #4
+      WholeStageCodegen (2)
+        HashAggregate [count] [count(1),count(1),count]
+          InputAdapter
+            Exchange #4
+              WholeStageCodegen (1)
+                HashAggregate [count,count]
+                  Project
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_sold_date_sk]
+    Subquery #5
+      WholeStageCodegen (2)
+        HashAggregate [sum,count] [avg(UnscaledValue(ss_ext_discount_amt)),avg(ss_ext_discount_amt),sum,count]
+          InputAdapter
+            Exchange #5
+              WholeStageCodegen (1)
+                HashAggregate [ss_ext_discount_amt] [sum,count,sum,count]
+                  Project [ss_ext_discount_amt]
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_ext_discount_amt,ss_sold_date_sk]
+    Subquery #6
+      WholeStageCodegen (2)
+        HashAggregate [sum,count] [avg(UnscaledValue(ss_net_paid)),avg(ss_net_paid),sum,count]
+          InputAdapter
+            Exchange #6
+              WholeStageCodegen (1)
+                HashAggregate [ss_net_paid] [sum,count,sum,count]
+                  Project [ss_net_paid]
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_net_paid,ss_sold_date_sk]
+    Subquery #7
+      WholeStageCodegen (2)
+        HashAggregate [count] [count(1),count(1),count]
+          InputAdapter
+            Exchange #7
+              WholeStageCodegen (1)
+                HashAggregate [count,count]
+                  Project
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_sold_date_sk]
+    Subquery #8
+      WholeStageCodegen (2)
+        HashAggregate [sum,count] [avg(UnscaledValue(ss_ext_discount_amt)),avg(ss_ext_discount_amt),sum,count]
+          InputAdapter
+            Exchange #8
+              WholeStageCodegen (1)
+                HashAggregate [ss_ext_discount_amt] [sum,count,sum,count]
+                  Project [ss_ext_discount_amt]
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_ext_discount_amt,ss_sold_date_sk]
+    Subquery #9
+      WholeStageCodegen (2)
+        HashAggregate [sum,count] [avg(UnscaledValue(ss_net_paid)),avg(ss_net_paid),sum,count]
+          InputAdapter
+            Exchange #9
+              WholeStageCodegen (1)
+                HashAggregate [ss_net_paid] [sum,count,sum,count]
+                  Project [ss_net_paid]
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_net_paid,ss_sold_date_sk]
+    Subquery #10
+      WholeStageCodegen (2)
+        HashAggregate [count] [count(1),count(1),count]
+          InputAdapter
+            Exchange #10
+              WholeStageCodegen (1)
+                HashAggregate [count,count]
+                  Project
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_sold_date_sk]
+    Subquery #11
+      WholeStageCodegen (2)
+        HashAggregate [sum,count] [avg(UnscaledValue(ss_ext_discount_amt)),avg(ss_ext_discount_amt),sum,count]
+          InputAdapter
+            Exchange #11
+              WholeStageCodegen (1)
+                HashAggregate [ss_ext_discount_amt] [sum,count,sum,count]
+                  Project [ss_ext_discount_amt]
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_ext_discount_amt,ss_sold_date_sk]
+    Subquery #12
+      WholeStageCodegen (2)
+        HashAggregate [sum,count] [avg(UnscaledValue(ss_net_paid)),avg(ss_net_paid),sum,count]
+          InputAdapter
+            Exchange #12
+              WholeStageCodegen (1)
+                HashAggregate [ss_net_paid] [sum,count,sum,count]
+                  Project [ss_net_paid]
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_net_paid,ss_sold_date_sk]
+    Subquery #13
+      WholeStageCodegen (2)
+        HashAggregate [count] [count(1),count(1),count]
+          InputAdapter
+            Exchange #13
+              WholeStageCodegen (1)
+                HashAggregate [count,count]
+                  Project
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_sold_date_sk]
+    Subquery #14
+      WholeStageCodegen (2)
+        HashAggregate [sum,count] [avg(UnscaledValue(ss_ext_discount_amt)),avg(ss_ext_discount_amt),sum,count]
+          InputAdapter
+            Exchange #14
+              WholeStageCodegen (1)
+                HashAggregate [ss_ext_discount_amt] [sum,count,sum,count]
+                  Project [ss_ext_discount_amt]
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_ext_discount_amt,ss_sold_date_sk]
+    Subquery #15
+      WholeStageCodegen (2)
+        HashAggregate [sum,count] [avg(UnscaledValue(ss_net_paid)),avg(ss_net_paid),sum,count]
+          InputAdapter
+            Exchange #15
+              WholeStageCodegen (1)
+                HashAggregate [ss_net_paid] [sum,count,sum,count]
+                  Project [ss_net_paid]
+                    Filter [ss_quantity]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.store_sales [ss_quantity,ss_net_paid,ss_sold_date_sk]
+    Filter [r_reason_sk]
+      ColumnarToRow
+        InputAdapter
+          Scan parquet default.reason [r_reason_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.sf100/explain.txt
new file mode 100644
index 0000000000000..5226fce4ef512
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.sf100/explain.txt
@@ -0,0 +1,290 @@
+== Physical Plan ==
+* Sort (52)
++- * Project (51)
+   +- * BroadcastNestedLoopJoin Inner BuildRight (50)
+      :- * HashAggregate (28)
+      :  +- Exchange (27)
+      :     +- * HashAggregate (26)
+      :        +- * Project (25)
+      :           +- * BroadcastHashJoin Inner BuildRight (24)
+      :              :- * Project (18)
+      :              :  +- * BroadcastHashJoin Inner BuildRight (17)
+      :              :     :- * Project (11)
+      :              :     :  +- * BroadcastHashJoin Inner BuildRight (10)
+      :              :     :     :- * Project (4)
+      :              :     :     :  +- * Filter (3)
+      :              :     :     :     +- * ColumnarToRow (2)
+      :              :     :     :        +- Scan parquet default.web_sales (1)
+      :              :     :     +- BroadcastExchange (9)
+      :              :     :        +- * Project (8)
+      :              :     :           +- * Filter (7)
+      :              :     :              +- * ColumnarToRow (6)
+      :              :     :                 +- Scan parquet default.web_page (5)
+      :              :     +- BroadcastExchange (16)
+      :              :        +- * Project (15)
+      :              :           +- * Filter (14)
+      :              :              +- * ColumnarToRow (13)
+      :              :                 +- Scan parquet default.household_demographics (12)
+      :              +- BroadcastExchange (23)
+      :                 +- * Project (22)
+      :                    +- * Filter (21)
+      :                       +- * ColumnarToRow (20)
+      :                          +- Scan parquet default.time_dim (19)
+      +- BroadcastExchange (49)
+         +- * HashAggregate (48)
+            +- Exchange (47)
+               +- * HashAggregate (46)
+                  +- * Project (45)
+                     +- * BroadcastHashJoin Inner BuildRight (44)
+                        :- * Project (38)
+                        :  +- * BroadcastHashJoin Inner BuildRight (37)
+                        :     :- * Project (35)
+                        :     :  +- * BroadcastHashJoin Inner BuildRight (34)
+                        :     :     :- * Project (32)
+                        :     :     :  +- * Filter (31)
+                        :     :     :     +- * ColumnarToRow (30)
+                        :     :     :        +- Scan parquet default.web_sales (29)
+                        :     :     +- ReusedExchange (33)
+                        :     +- ReusedExchange (36)
+                        +- BroadcastExchange (43)
+                           +- * Project (42)
+                              +- * Filter (41)
+                                 +- * ColumnarToRow (40)
+                                    +- Scan parquet default.time_dim (39)
+
+
+(1) Scan parquet default.web_sales
+Output [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_sales]
+PushedFilters: [IsNotNull(ws_ship_hdemo_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_web_page_sk)]
+ReadSchema: struct<ws_sold_time_sk:int,ws_ship_hdemo_sk:int,ws_web_page_sk:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4]
+
+(3) Filter [codegen id : 4]
+Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4]
+Condition : ((isnotnull(ws_ship_hdemo_sk#2) AND isnotnull(ws_sold_time_sk#1)) AND isnotnull(ws_web_page_sk#3))
+
+(4) Project [codegen id : 4]
+Output [3]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3]
+Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4]
+
+(5) Scan parquet default.web_page
+Output [2]: [wp_web_page_sk#5, wp_char_count#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_page]
+PushedFilters: [IsNotNull(wp_char_count), GreaterThanOrEqual(wp_char_count,5000), LessThanOrEqual(wp_char_count,5200), IsNotNull(wp_web_page_sk)]
+ReadSchema: struct<wp_web_page_sk:int,wp_char_count:int>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [2]: [wp_web_page_sk#5, wp_char_count#6]
+
+(7) Filter [codegen id : 1]
+Input [2]: [wp_web_page_sk#5, wp_char_count#6]
+Condition : (((isnotnull(wp_char_count#6) AND (wp_char_count#6 >= 5000)) AND (wp_char_count#6 <= 5200)) AND isnotnull(wp_web_page_sk#5))
+
+(8) Project [codegen id : 1]
+Output [1]: [wp_web_page_sk#5]
+Input [2]: [wp_web_page_sk#5, wp_char_count#6]
+
+(9) BroadcastExchange
+Input [1]: [wp_web_page_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7]
+
+(10) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ws_web_page_sk#3]
+Right keys [1]: [wp_web_page_sk#5]
+Join condition: None
+
+(11) Project [codegen id : 4]
+Output [2]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2]
+Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, wp_web_page_sk#5]
+
+(12) Scan parquet default.household_demographics
+Output [2]: [hd_demo_sk#8, hd_dep_count#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_dep_count), EqualTo(hd_dep_count,6), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_dep_count:int>
+
+(13) ColumnarToRow [codegen id : 2]
+Input [2]: [hd_demo_sk#8, hd_dep_count#9]
+
+(14) Filter [codegen id : 2]
+Input [2]: [hd_demo_sk#8, hd_dep_count#9]
+Condition : ((isnotnull(hd_dep_count#9) AND (hd_dep_count#9 = 6)) AND isnotnull(hd_demo_sk#8))
+
+(15) Project [codegen id : 2]
+Output [1]: [hd_demo_sk#8]
+Input [2]: [hd_demo_sk#8, hd_dep_count#9]
+
+(16) BroadcastExchange
+Input [1]: [hd_demo_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(17) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ws_ship_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#8]
+Join condition: None
+
+(18) Project [codegen id : 4]
+Output [1]: [ws_sold_time_sk#1]
+Input [3]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, hd_demo_sk#8]
+
+(19) Scan parquet default.time_dim
+Output [2]: [t_time_sk#11, t_hour#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [IsNotNull(t_hour), GreaterThanOrEqual(t_hour,8), LessThanOrEqual(t_hour,9), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_hour:int>
+
+(20) ColumnarToRow [codegen id : 3]
+Input [2]: [t_time_sk#11, t_hour#12]
+
+(21) Filter [codegen id : 3]
+Input [2]: [t_time_sk#11, t_hour#12]
+Condition : (((isnotnull(t_hour#12) AND (t_hour#12 >= 8)) AND (t_hour#12 <= 9)) AND isnotnull(t_time_sk#11))
+
+(22) Project [codegen id : 3]
+Output [1]: [t_time_sk#11]
+Input [2]: [t_time_sk#11, t_hour#12]
+
+(23) BroadcastExchange
+Input [1]: [t_time_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(24) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ws_sold_time_sk#1]
+Right keys [1]: [t_time_sk#11]
+Join condition: None
+
+(25) Project [codegen id : 4]
+Output: []
+Input [2]: [ws_sold_time_sk#1, t_time_sk#11]
+
+(26) HashAggregate [codegen id : 4]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#14]
+Results [1]: [count#15]
+
+(27) Exchange
+Input [1]: [count#15]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#16]
+
+(28) HashAggregate [codegen id : 10]
+Input [1]: [count#15]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#17]
+Results [1]: [count(1)#17 AS amc#18]
+
+(29) Scan parquet default.web_sales
+Output [4]: [ws_sold_time_sk#19, ws_ship_hdemo_sk#20, ws_web_page_sk#21, ws_sold_date_sk#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_sales]
+PushedFilters: [IsNotNull(ws_ship_hdemo_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_web_page_sk)]
+ReadSchema: struct<ws_sold_time_sk:int,ws_ship_hdemo_sk:int,ws_web_page_sk:int>
+
+(30) ColumnarToRow [codegen id : 8]
+Input [4]: [ws_sold_time_sk#19, ws_ship_hdemo_sk#20, ws_web_page_sk#21, ws_sold_date_sk#22]
+
+(31) Filter [codegen id : 8]
+Input [4]: [ws_sold_time_sk#19, ws_ship_hdemo_sk#20, ws_web_page_sk#21, ws_sold_date_sk#22]
+Condition : ((isnotnull(ws_ship_hdemo_sk#20) AND isnotnull(ws_sold_time_sk#19)) AND isnotnull(ws_web_page_sk#21))
+
+(32) Project [codegen id : 8]
+Output [3]: [ws_sold_time_sk#19, ws_ship_hdemo_sk#20, ws_web_page_sk#21]
+Input [4]: [ws_sold_time_sk#19, ws_ship_hdemo_sk#20, ws_web_page_sk#21, ws_sold_date_sk#22]
+
+(33) ReusedExchange [Reuses operator id: 9]
+Output [1]: [wp_web_page_sk#23]
+
+(34) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ws_web_page_sk#21]
+Right keys [1]: [wp_web_page_sk#23]
+Join condition: None
+
+(35) Project [codegen id : 8]
+Output [2]: [ws_sold_time_sk#19, ws_ship_hdemo_sk#20]
+Input [4]: [ws_sold_time_sk#19, ws_ship_hdemo_sk#20, ws_web_page_sk#21, wp_web_page_sk#23]
+
+(36) ReusedExchange [Reuses operator id: 16]
+Output [1]: [hd_demo_sk#24]
+
+(37) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ws_ship_hdemo_sk#20]
+Right keys [1]: [hd_demo_sk#24]
+Join condition: None
+
+(38) Project [codegen id : 8]
+Output [1]: [ws_sold_time_sk#19]
+Input [3]: [ws_sold_time_sk#19, ws_ship_hdemo_sk#20, hd_demo_sk#24]
+
+(39) Scan parquet default.time_dim
+Output [2]: [t_time_sk#25, t_hour#26]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [IsNotNull(t_hour), GreaterThanOrEqual(t_hour,19), LessThanOrEqual(t_hour,20), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_hour:int>
+
+(40) ColumnarToRow [codegen id : 7]
+Input [2]: [t_time_sk#25, t_hour#26]
+
+(41) Filter [codegen id : 7]
+Input [2]: [t_time_sk#25, t_hour#26]
+Condition : (((isnotnull(t_hour#26) AND (t_hour#26 >= 19)) AND (t_hour#26 <= 20)) AND isnotnull(t_time_sk#25))
+
+(42) Project [codegen id : 7]
+Output [1]: [t_time_sk#25]
+Input [2]: [t_time_sk#25, t_hour#26]
+
+(43) BroadcastExchange
+Input [1]: [t_time_sk#25]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#27]
+
+(44) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ws_sold_time_sk#19]
+Right keys [1]: [t_time_sk#25]
+Join condition: None
+
+(45) Project [codegen id : 8]
+Output: []
+Input [2]: [ws_sold_time_sk#19, t_time_sk#25]
+
+(46) HashAggregate [codegen id : 8]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#28]
+Results [1]: [count#29]
+
+(47) Exchange
+Input [1]: [count#29]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#30]
+
+(48) HashAggregate [codegen id : 9]
+Input [1]: [count#29]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#31]
+Results [1]: [count(1)#31 AS pmc#32]
+
+(49) BroadcastExchange
+Input [1]: [pmc#32]
+Arguments: IdentityBroadcastMode, [id=#33]
+
+(50) BroadcastNestedLoopJoin [codegen id : 10]
+Join condition: None
+
+(51) Project [codegen id : 10]
+Output [1]: [CheckOverflow((promote_precision(cast(amc#18 as decimal(15,4))) / promote_precision(cast(pmc#32 as decimal(15,4)))), DecimalType(35,20), true) AS am_pm_ratio#34]
+Input [2]: [amc#18, pmc#32]
+
+(52) Sort [codegen id : 10]
+Input [1]: [am_pm_ratio#34]
+Arguments: [am_pm_ratio#34 ASC NULLS FIRST], true, 0
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.sf100/simplified.txt
new file mode 100644
index 0000000000000..6fed86d7cea43
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.sf100/simplified.txt
@@ -0,0 +1,75 @@
+WholeStageCodegen (10)
+  Sort [am_pm_ratio]
+    Project [amc,pmc]
+      BroadcastNestedLoopJoin
+        HashAggregate [count] [count(1),amc,count]
+          InputAdapter
+            Exchange #1
+              WholeStageCodegen (4)
+                HashAggregate [count,count]
+                  Project
+                    BroadcastHashJoin [ws_sold_time_sk,t_time_sk]
+                      Project [ws_sold_time_sk]
+                        BroadcastHashJoin [ws_ship_hdemo_sk,hd_demo_sk]
+                          Project [ws_sold_time_sk,ws_ship_hdemo_sk]
+                            BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk]
+                              Project [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk]
+                                Filter [ws_ship_hdemo_sk,ws_sold_time_sk,ws_web_page_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk]
+                              InputAdapter
+                                BroadcastExchange #2
+                                  WholeStageCodegen (1)
+                                    Project [wp_web_page_sk]
+                                      Filter [wp_char_count,wp_web_page_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.web_page [wp_web_page_sk,wp_char_count]
+                          InputAdapter
+                            BroadcastExchange #3
+                              WholeStageCodegen (2)
+                                Project [hd_demo_sk]
+                                  Filter [hd_dep_count,hd_demo_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.household_demographics [hd_demo_sk,hd_dep_count]
+                      InputAdapter
+                        BroadcastExchange #4
+                          WholeStageCodegen (3)
+                            Project [t_time_sk]
+                              Filter [t_hour,t_time_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.time_dim [t_time_sk,t_hour]
+        InputAdapter
+          BroadcastExchange #5
+            WholeStageCodegen (9)
+              HashAggregate [count] [count(1),pmc,count]
+                InputAdapter
+                  Exchange #6
+                    WholeStageCodegen (8)
+                      HashAggregate [count,count]
+                        Project
+                          BroadcastHashJoin [ws_sold_time_sk,t_time_sk]
+                            Project [ws_sold_time_sk]
+                              BroadcastHashJoin [ws_ship_hdemo_sk,hd_demo_sk]
+                                Project [ws_sold_time_sk,ws_ship_hdemo_sk]
+                                  BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk]
+                                    Project [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk]
+                                      Filter [ws_ship_hdemo_sk,ws_sold_time_sk,ws_web_page_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk]
+                                    InputAdapter
+                                      ReusedExchange [wp_web_page_sk] #2
+                                InputAdapter
+                                  ReusedExchange [hd_demo_sk] #3
+                            InputAdapter
+                              BroadcastExchange #7
+                                WholeStageCodegen (7)
+                                  Project [t_time_sk]
+                                    Filter [t_hour,t_time_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.time_dim [t_time_sk,t_hour]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/explain.txt
new file mode 100644
index 0000000000000..e369a027040d3
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/explain.txt
@@ -0,0 +1,290 @@
+== Physical Plan ==
+* Sort (52)
++- * Project (51)
+   +- * BroadcastNestedLoopJoin Inner BuildRight (50)
+      :- * HashAggregate (28)
+      :  +- Exchange (27)
+      :     +- * HashAggregate (26)
+      :        +- * Project (25)
+      :           +- * BroadcastHashJoin Inner BuildRight (24)
+      :              :- * Project (18)
+      :              :  +- * BroadcastHashJoin Inner BuildRight (17)
+      :              :     :- * Project (11)
+      :              :     :  +- * BroadcastHashJoin Inner BuildRight (10)
+      :              :     :     :- * Project (4)
+      :              :     :     :  +- * Filter (3)
+      :              :     :     :     +- * ColumnarToRow (2)
+      :              :     :     :        +- Scan parquet default.web_sales (1)
+      :              :     :     +- BroadcastExchange (9)
+      :              :     :        +- * Project (8)
+      :              :     :           +- * Filter (7)
+      :              :     :              +- * ColumnarToRow (6)
+      :              :     :                 +- Scan parquet default.household_demographics (5)
+      :              :     +- BroadcastExchange (16)
+      :              :        +- * Project (15)
+      :              :           +- * Filter (14)
+      :              :              +- * ColumnarToRow (13)
+      :              :                 +- Scan parquet default.time_dim (12)
+      :              +- BroadcastExchange (23)
+      :                 +- * Project (22)
+      :                    +- * Filter (21)
+      :                       +- * ColumnarToRow (20)
+      :                          +- Scan parquet default.web_page (19)
+      +- BroadcastExchange (49)
+         +- * HashAggregate (48)
+            +- Exchange (47)
+               +- * HashAggregate (46)
+                  +- * Project (45)
+                     +- * BroadcastHashJoin Inner BuildRight (44)
+                        :- * Project (42)
+                        :  +- * BroadcastHashJoin Inner BuildRight (41)
+                        :     :- * Project (35)
+                        :     :  +- * BroadcastHashJoin Inner BuildRight (34)
+                        :     :     :- * Project (32)
+                        :     :     :  +- * Filter (31)
+                        :     :     :     +- * ColumnarToRow (30)
+                        :     :     :        +- Scan parquet default.web_sales (29)
+                        :     :     +- ReusedExchange (33)
+                        :     +- BroadcastExchange (40)
+                        :        +- * Project (39)
+                        :           +- * Filter (38)
+                        :              +- * ColumnarToRow (37)
+                        :                 +- Scan parquet default.time_dim (36)
+                        +- ReusedExchange (43)
+
+
+(1) Scan parquet default.web_sales
+Output [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_sales]
+PushedFilters: [IsNotNull(ws_ship_hdemo_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_web_page_sk)]
+ReadSchema: struct<ws_sold_time_sk:int,ws_ship_hdemo_sk:int,ws_web_page_sk:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4]
+
+(3) Filter [codegen id : 4]
+Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4]
+Condition : ((isnotnull(ws_ship_hdemo_sk#2) AND isnotnull(ws_sold_time_sk#1)) AND isnotnull(ws_web_page_sk#3))
+
+(4) Project [codegen id : 4]
+Output [3]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3]
+Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4]
+
+(5) Scan parquet default.household_demographics
+Output [2]: [hd_demo_sk#5, hd_dep_count#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_dep_count), EqualTo(hd_dep_count,6), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_dep_count:int>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [2]: [hd_demo_sk#5, hd_dep_count#6]
+
+(7) Filter [codegen id : 1]
+Input [2]: [hd_demo_sk#5, hd_dep_count#6]
+Condition : ((isnotnull(hd_dep_count#6) AND (hd_dep_count#6 = 6)) AND isnotnull(hd_demo_sk#5))
+
+(8) Project [codegen id : 1]
+Output [1]: [hd_demo_sk#5]
+Input [2]: [hd_demo_sk#5, hd_dep_count#6]
+
+(9) BroadcastExchange
+Input [1]: [hd_demo_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7]
+
+(10) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ws_ship_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#5]
+Join condition: None
+
+(11) Project [codegen id : 4]
+Output [2]: [ws_sold_time_sk#1, ws_web_page_sk#3]
+Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, hd_demo_sk#5]
+
+(12) Scan parquet default.time_dim
+Output [2]: [t_time_sk#8, t_hour#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [IsNotNull(t_hour), GreaterThanOrEqual(t_hour,8), LessThanOrEqual(t_hour,9), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_hour:int>
+
+(13) ColumnarToRow [codegen id : 2]
+Input [2]: [t_time_sk#8, t_hour#9]
+
+(14) Filter [codegen id : 2]
+Input [2]: [t_time_sk#8, t_hour#9]
+Condition : (((isnotnull(t_hour#9) AND (t_hour#9 >= 8)) AND (t_hour#9 <= 9)) AND isnotnull(t_time_sk#8))
+
+(15) Project [codegen id : 2]
+Output [1]: [t_time_sk#8]
+Input [2]: [t_time_sk#8, t_hour#9]
+
+(16) BroadcastExchange
+Input [1]: [t_time_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(17) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ws_sold_time_sk#1]
+Right keys [1]: [t_time_sk#8]
+Join condition: None
+
+(18) Project [codegen id : 4]
+Output [1]: [ws_web_page_sk#3]
+Input [3]: [ws_sold_time_sk#1, ws_web_page_sk#3, t_time_sk#8]
+
+(19) Scan parquet default.web_page
+Output [2]: [wp_web_page_sk#11, wp_char_count#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_page]
+PushedFilters: [IsNotNull(wp_char_count), GreaterThanOrEqual(wp_char_count,5000), LessThanOrEqual(wp_char_count,5200), IsNotNull(wp_web_page_sk)]
+ReadSchema: struct<wp_web_page_sk:int,wp_char_count:int>
+
+(20) ColumnarToRow [codegen id : 3]
+Input [2]: [wp_web_page_sk#11, wp_char_count#12]
+
+(21) Filter [codegen id : 3]
+Input [2]: [wp_web_page_sk#11, wp_char_count#12]
+Condition : (((isnotnull(wp_char_count#12) AND (wp_char_count#12 >= 5000)) AND (wp_char_count#12 <= 5200)) AND isnotnull(wp_web_page_sk#11))
+
+(22) Project [codegen id : 3]
+Output [1]: [wp_web_page_sk#11]
+Input [2]: [wp_web_page_sk#11, wp_char_count#12]
+
+(23) BroadcastExchange
+Input [1]: [wp_web_page_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(24) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ws_web_page_sk#3]
+Right keys [1]: [wp_web_page_sk#11]
+Join condition: None
+
+(25) Project [codegen id : 4]
+Output: []
+Input [2]: [ws_web_page_sk#3, wp_web_page_sk#11]
+
+(26) HashAggregate [codegen id : 4]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#14]
+Results [1]: [count#15]
+
+(27) Exchange
+Input [1]: [count#15]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#16]
+
+(28) HashAggregate [codegen id : 10]
+Input [1]: [count#15]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#17]
+Results [1]: [count(1)#17 AS amc#18]
+
+(29) Scan parquet default.web_sales
+Output [4]: [ws_sold_time_sk#19, ws_ship_hdemo_sk#20, ws_web_page_sk#21, ws_sold_date_sk#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_sales]
+PushedFilters: [IsNotNull(ws_ship_hdemo_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_web_page_sk)]
+ReadSchema: struct<ws_sold_time_sk:int,ws_ship_hdemo_sk:int,ws_web_page_sk:int>
+
+(30) ColumnarToRow [codegen id : 8]
+Input [4]: [ws_sold_time_sk#19, ws_ship_hdemo_sk#20, ws_web_page_sk#21, ws_sold_date_sk#22]
+
+(31) Filter [codegen id : 8]
+Input [4]: [ws_sold_time_sk#19, ws_ship_hdemo_sk#20, ws_web_page_sk#21, ws_sold_date_sk#22]
+Condition : ((isnotnull(ws_ship_hdemo_sk#20) AND isnotnull(ws_sold_time_sk#19)) AND isnotnull(ws_web_page_sk#21))
+
+(32) Project [codegen id : 8]
+Output [3]: [ws_sold_time_sk#19, ws_ship_hdemo_sk#20, ws_web_page_sk#21]
+Input [4]: [ws_sold_time_sk#19, ws_ship_hdemo_sk#20, ws_web_page_sk#21, ws_sold_date_sk#22]
+
+(33) ReusedExchange [Reuses operator id: 9]
+Output [1]: [hd_demo_sk#23]
+
+(34) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ws_ship_hdemo_sk#20]
+Right keys [1]: [hd_demo_sk#23]
+Join condition: None
+
+(35) Project [codegen id : 8]
+Output [2]: [ws_sold_time_sk#19, ws_web_page_sk#21]
+Input [4]: [ws_sold_time_sk#19, ws_ship_hdemo_sk#20, ws_web_page_sk#21, hd_demo_sk#23]
+
+(36) Scan parquet default.time_dim
+Output [2]: [t_time_sk#24, t_hour#25]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [IsNotNull(t_hour), GreaterThanOrEqual(t_hour,19), LessThanOrEqual(t_hour,20), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_hour:int>
+
+(37) ColumnarToRow [codegen id : 6]
+Input [2]: [t_time_sk#24, t_hour#25]
+
+(38) Filter [codegen id : 6]
+Input [2]: [t_time_sk#24, t_hour#25]
+Condition : (((isnotnull(t_hour#25) AND (t_hour#25 >= 19)) AND (t_hour#25 <= 20)) AND isnotnull(t_time_sk#24))
+
+(39) Project [codegen id : 6]
+Output [1]: [t_time_sk#24]
+Input [2]: [t_time_sk#24, t_hour#25]
+
+(40) BroadcastExchange
+Input [1]: [t_time_sk#24]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#26]
+
+(41) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ws_sold_time_sk#19]
+Right keys [1]: [t_time_sk#24]
+Join condition: None
+
+(42) Project [codegen id : 8]
+Output [1]: [ws_web_page_sk#21]
+Input [3]: [ws_sold_time_sk#19, ws_web_page_sk#21, t_time_sk#24]
+
+(43) ReusedExchange [Reuses operator id: 23]
+Output [1]: [wp_web_page_sk#27]
+
+(44) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ws_web_page_sk#21]
+Right keys [1]: [wp_web_page_sk#27]
+Join condition: None
+
+(45) Project [codegen id : 8]
+Output: []
+Input [2]: [ws_web_page_sk#21, wp_web_page_sk#27]
+
+(46) HashAggregate [codegen id : 8]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#28]
+Results [1]: [count#29]
+
+(47) Exchange
+Input [1]: [count#29]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#30]
+
+(48) HashAggregate [codegen id : 9]
+Input [1]: [count#29]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#31]
+Results [1]: [count(1)#31 AS pmc#32]
+
+(49) BroadcastExchange
+Input [1]: [pmc#32]
+Arguments: IdentityBroadcastMode, [id=#33]
+
+(50) BroadcastNestedLoopJoin [codegen id : 10]
+Join condition: None
+
+(51) Project [codegen id : 10]
+Output [1]: [CheckOverflow((promote_precision(cast(amc#18 as decimal(15,4))) / promote_precision(cast(pmc#32 as decimal(15,4)))), DecimalType(35,20), true) AS am_pm_ratio#34]
+Input [2]: [amc#18, pmc#32]
+
+(52) Sort [codegen id : 10]
+Input [1]: [am_pm_ratio#34]
+Arguments: [am_pm_ratio#34 ASC NULLS FIRST], true, 0
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/simplified.txt
new file mode 100644
index 0000000000000..0964189da3cfb
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/simplified.txt
@@ -0,0 +1,75 @@
+WholeStageCodegen (10)
+  Sort [am_pm_ratio]
+    Project [amc,pmc]
+      BroadcastNestedLoopJoin
+        HashAggregate [count] [count(1),amc,count]
+          InputAdapter
+            Exchange #1
+              WholeStageCodegen (4)
+                HashAggregate [count,count]
+                  Project
+                    BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk]
+                      Project [ws_web_page_sk]
+                        BroadcastHashJoin [ws_sold_time_sk,t_time_sk]
+                          Project [ws_sold_time_sk,ws_web_page_sk]
+                            BroadcastHashJoin [ws_ship_hdemo_sk,hd_demo_sk]
+                              Project [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk]
+                                Filter [ws_ship_hdemo_sk,ws_sold_time_sk,ws_web_page_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk]
+                              InputAdapter
+                                BroadcastExchange #2
+                                  WholeStageCodegen (1)
+                                    Project [hd_demo_sk]
+                                      Filter [hd_dep_count,hd_demo_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.household_demographics [hd_demo_sk,hd_dep_count]
+                          InputAdapter
+                            BroadcastExchange #3
+                              WholeStageCodegen (2)
+                                Project [t_time_sk]
+                                  Filter [t_hour,t_time_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.time_dim [t_time_sk,t_hour]
+                      InputAdapter
+                        BroadcastExchange #4
+                          WholeStageCodegen (3)
+                            Project [wp_web_page_sk]
+                              Filter [wp_char_count,wp_web_page_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.web_page [wp_web_page_sk,wp_char_count]
+        InputAdapter
+          BroadcastExchange #5
+            WholeStageCodegen (9)
+              HashAggregate [count] [count(1),pmc,count]
+                InputAdapter
+                  Exchange #6
+                    WholeStageCodegen (8)
+                      HashAggregate [count,count]
+                        Project
+                          BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk]
+                            Project [ws_web_page_sk]
+                              BroadcastHashJoin [ws_sold_time_sk,t_time_sk]
+                                Project [ws_sold_time_sk,ws_web_page_sk]
+                                  BroadcastHashJoin [ws_ship_hdemo_sk,hd_demo_sk]
+                                    Project [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk]
+                                      Filter [ws_ship_hdemo_sk,ws_sold_time_sk,ws_web_page_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk]
+                                    InputAdapter
+                                      ReusedExchange [hd_demo_sk] #2
+                                InputAdapter
+                                  BroadcastExchange #7
+                                    WholeStageCodegen (6)
+                                      Project [t_time_sk]
+                                        Filter [t_hour,t_time_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.time_dim [t_time_sk,t_hour]
+                            InputAdapter
+                              ReusedExchange [wp_web_page_sk] #4
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.sf100/explain.txt
new file mode 100644
index 0000000000000..ef2ddc2466f18
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.sf100/explain.txt
@@ -0,0 +1,275 @@
+== Physical Plan ==
+* Sort (47)
++- Exchange (46)
+   +- * HashAggregate (45)
+      +- Exchange (44)
+         +- * HashAggregate (43)
+            +- * Project (42)
+               +- * BroadcastHashJoin Inner BuildRight (41)
+                  :- * Project (36)
+                  :  +- * BroadcastHashJoin Inner BuildRight (35)
+                  :     :- * Project (23)
+                  :     :  +- * BroadcastHashJoin Inner BuildRight (22)
+                  :     :     :- * Project (16)
+                  :     :     :  +- * BroadcastHashJoin Inner BuildRight (15)
+                  :     :     :     :- * Project (9)
+                  :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+                  :     :     :     :     :- * Filter (3)
+                  :     :     :     :     :  +- * ColumnarToRow (2)
+                  :     :     :     :     :     +- Scan parquet default.customer (1)
+                  :     :     :     :     +- BroadcastExchange (7)
+                  :     :     :     :        +- * Filter (6)
+                  :     :     :     :           +- * ColumnarToRow (5)
+                  :     :     :     :              +- Scan parquet default.customer_demographics (4)
+                  :     :     :     +- BroadcastExchange (14)
+                  :     :     :        +- * Project (13)
+                  :     :     :           +- * Filter (12)
+                  :     :     :              +- * ColumnarToRow (11)
+                  :     :     :                 +- Scan parquet default.household_demographics (10)
+                  :     :     +- BroadcastExchange (21)
+                  :     :        +- * Project (20)
+                  :     :           +- * Filter (19)
+                  :     :              +- * ColumnarToRow (18)
+                  :     :                 +- Scan parquet default.customer_address (17)
+                  :     +- BroadcastExchange (34)
+                  :        +- * Project (33)
+                  :           +- * BroadcastHashJoin Inner BuildRight (32)
+                  :              :- * Filter (26)
+                  :              :  +- * ColumnarToRow (25)
+                  :              :     +- Scan parquet default.catalog_returns (24)
+                  :              +- BroadcastExchange (31)
+                  :                 +- * Project (30)
+                  :                    +- * Filter (29)
+                  :                       +- * ColumnarToRow (28)
+                  :                          +- Scan parquet default.date_dim (27)
+                  +- BroadcastExchange (40)
+                     +- * Filter (39)
+                        +- * ColumnarToRow (38)
+                           +- Scan parquet default.call_center (37)
+
+
+(1) Scan parquet default.customer
+Output [4]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_hdemo_sk:int,c_current_addr_sk:int>
+
+(2) ColumnarToRow [codegen id : 7]
+Input [4]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4]
+
+(3) Filter [codegen id : 7]
+Input [4]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4]
+Condition : (((isnotnull(c_customer_sk#1) AND isnotnull(c_current_addr_sk#4)) AND isnotnull(c_current_cdemo_sk#2)) AND isnotnull(c_current_hdemo_sk#3))
+
+(4) Scan parquet default.customer_demographics
+Output [3]: [cd_demo_sk#5, cd_marital_status#6, cd_education_status#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,Unknown             )),And(EqualTo(cd_marital_status,W),EqualTo(cd_education_status,Advanced Degree     ))), IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_marital_status:string,cd_education_status:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [cd_demo_sk#5, cd_marital_status#6, cd_education_status#7]
+
+(6) Filter [codegen id : 1]
+Input [3]: [cd_demo_sk#5, cd_marital_status#6, cd_education_status#7]
+Condition : ((((cd_marital_status#6 = M) AND (cd_education_status#7 = Unknown             )) OR ((cd_marital_status#6 = W) AND (cd_education_status#7 = Advanced Degree     ))) AND isnotnull(cd_demo_sk#5))
+
+(7) BroadcastExchange
+Input [3]: [cd_demo_sk#5, cd_marital_status#6, cd_education_status#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#8]
+
+(8) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [c_current_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#5]
+Join condition: None
+
+(9) Project [codegen id : 7]
+Output [5]: [c_customer_sk#1, c_current_hdemo_sk#3, c_current_addr_sk#4, cd_marital_status#6, cd_education_status#7]
+Input [7]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, cd_demo_sk#5, cd_marital_status#6, cd_education_status#7]
+
+(10) Scan parquet default.household_demographics
+Output [2]: [hd_demo_sk#9, hd_buy_potential#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_buy_potential), StringStartsWith(hd_buy_potential,Unknown), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_buy_potential:string>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [hd_demo_sk#9, hd_buy_potential#10]
+
+(12) Filter [codegen id : 2]
+Input [2]: [hd_demo_sk#9, hd_buy_potential#10]
+Condition : ((isnotnull(hd_buy_potential#10) AND StartsWith(hd_buy_potential#10, Unknown)) AND isnotnull(hd_demo_sk#9))
+
+(13) Project [codegen id : 2]
+Output [1]: [hd_demo_sk#9]
+Input [2]: [hd_demo_sk#9, hd_buy_potential#10]
+
+(14) BroadcastExchange
+Input [1]: [hd_demo_sk#9]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#11]
+
+(15) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [c_current_hdemo_sk#3]
+Right keys [1]: [hd_demo_sk#9]
+Join condition: None
+
+(16) Project [codegen id : 7]
+Output [4]: [c_customer_sk#1, c_current_addr_sk#4, cd_marital_status#6, cd_education_status#7]
+Input [6]: [c_customer_sk#1, c_current_hdemo_sk#3, c_current_addr_sk#4, cd_marital_status#6, cd_education_status#7, hd_demo_sk#9]
+
+(17) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#12, ca_gmt_offset#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-7.00), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_gmt_offset:decimal(5,2)>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [2]: [ca_address_sk#12, ca_gmt_offset#13]
+
+(19) Filter [codegen id : 3]
+Input [2]: [ca_address_sk#12, ca_gmt_offset#13]
+Condition : ((isnotnull(ca_gmt_offset#13) AND (ca_gmt_offset#13 = -7.00)) AND isnotnull(ca_address_sk#12))
+
+(20) Project [codegen id : 3]
+Output [1]: [ca_address_sk#12]
+Input [2]: [ca_address_sk#12, ca_gmt_offset#13]
+
+(21) BroadcastExchange
+Input [1]: [ca_address_sk#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(22) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [c_current_addr_sk#4]
+Right keys [1]: [ca_address_sk#12]
+Join condition: None
+
+(23) Project [codegen id : 7]
+Output [3]: [c_customer_sk#1, cd_marital_status#6, cd_education_status#7]
+Input [5]: [c_customer_sk#1, c_current_addr_sk#4, cd_marital_status#6, cd_education_status#7, ca_address_sk#12]
+
+(24) Scan parquet default.catalog_returns
+Output [4]: [cr_returning_customer_sk#15, cr_call_center_sk#16, cr_net_loss#17, cr_returned_date_sk#18]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cr_returned_date_sk#18), dynamicpruningexpression(cr_returned_date_sk#18 IN dynamicpruning#19)]
+PushedFilters: [IsNotNull(cr_call_center_sk), IsNotNull(cr_returning_customer_sk)]
+ReadSchema: struct<cr_returning_customer_sk:int,cr_call_center_sk:int,cr_net_loss:decimal(7,2)>
+
+(25) ColumnarToRow [codegen id : 5]
+Input [4]: [cr_returning_customer_sk#15, cr_call_center_sk#16, cr_net_loss#17, cr_returned_date_sk#18]
+
+(26) Filter [codegen id : 5]
+Input [4]: [cr_returning_customer_sk#15, cr_call_center_sk#16, cr_net_loss#17, cr_returned_date_sk#18]
+Condition : (isnotnull(cr_call_center_sk#16) AND isnotnull(cr_returning_customer_sk#15))
+
+(27) Scan parquet default.date_dim
+Output [3]: [d_date_sk#20, d_year#21, d_moy#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(28) ColumnarToRow [codegen id : 4]
+Input [3]: [d_date_sk#20, d_year#21, d_moy#22]
+
+(29) Filter [codegen id : 4]
+Input [3]: [d_date_sk#20, d_year#21, d_moy#22]
+Condition : ((((isnotnull(d_year#21) AND isnotnull(d_moy#22)) AND (d_year#21 = 1998)) AND (d_moy#22 = 11)) AND isnotnull(d_date_sk#20))
+
+(30) Project [codegen id : 4]
+Output [1]: [d_date_sk#20]
+Input [3]: [d_date_sk#20, d_year#21, d_moy#22]
+
+(31) BroadcastExchange
+Input [1]: [d_date_sk#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#23]
+
+(32) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [cr_returned_date_sk#18]
+Right keys [1]: [d_date_sk#20]
+Join condition: None
+
+(33) Project [codegen id : 5]
+Output [3]: [cr_returning_customer_sk#15, cr_call_center_sk#16, cr_net_loss#17]
+Input [5]: [cr_returning_customer_sk#15, cr_call_center_sk#16, cr_net_loss#17, cr_returned_date_sk#18, d_date_sk#20]
+
+(34) BroadcastExchange
+Input [3]: [cr_returning_customer_sk#15, cr_call_center_sk#16, cr_net_loss#17]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#24]
+
+(35) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [cr_returning_customer_sk#15]
+Join condition: None
+
+(36) Project [codegen id : 7]
+Output [4]: [cd_marital_status#6, cd_education_status#7, cr_call_center_sk#16, cr_net_loss#17]
+Input [6]: [c_customer_sk#1, cd_marital_status#6, cd_education_status#7, cr_returning_customer_sk#15, cr_call_center_sk#16, cr_net_loss#17]
+
+(37) Scan parquet default.call_center
+Output [4]: [cc_call_center_sk#25, cc_call_center_id#26, cc_name#27, cc_manager#28]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/call_center]
+PushedFilters: [IsNotNull(cc_call_center_sk)]
+ReadSchema: struct<cc_call_center_sk:int,cc_call_center_id:string,cc_name:string,cc_manager:string>
+
+(38) ColumnarToRow [codegen id : 6]
+Input [4]: [cc_call_center_sk#25, cc_call_center_id#26, cc_name#27, cc_manager#28]
+
+(39) Filter [codegen id : 6]
+Input [4]: [cc_call_center_sk#25, cc_call_center_id#26, cc_name#27, cc_manager#28]
+Condition : isnotnull(cc_call_center_sk#25)
+
+(40) BroadcastExchange
+Input [4]: [cc_call_center_sk#25, cc_call_center_id#26, cc_name#27, cc_manager#28]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#29]
+
+(41) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [cr_call_center_sk#16]
+Right keys [1]: [cc_call_center_sk#25]
+Join condition: None
+
+(42) Project [codegen id : 7]
+Output [6]: [cc_call_center_id#26, cc_name#27, cc_manager#28, cr_net_loss#17, cd_marital_status#6, cd_education_status#7]
+Input [8]: [cd_marital_status#6, cd_education_status#7, cr_call_center_sk#16, cr_net_loss#17, cc_call_center_sk#25, cc_call_center_id#26, cc_name#27, cc_manager#28]
+
+(43) HashAggregate [codegen id : 7]
+Input [6]: [cc_call_center_id#26, cc_name#27, cc_manager#28, cr_net_loss#17, cd_marital_status#6, cd_education_status#7]
+Keys [5]: [cc_call_center_id#26, cc_name#27, cc_manager#28, cd_marital_status#6, cd_education_status#7]
+Functions [1]: [partial_sum(UnscaledValue(cr_net_loss#17))]
+Aggregate Attributes [1]: [sum#30]
+Results [6]: [cc_call_center_id#26, cc_name#27, cc_manager#28, cd_marital_status#6, cd_education_status#7, sum#31]
+
+(44) Exchange
+Input [6]: [cc_call_center_id#26, cc_name#27, cc_manager#28, cd_marital_status#6, cd_education_status#7, sum#31]
+Arguments: hashpartitioning(cc_call_center_id#26, cc_name#27, cc_manager#28, cd_marital_status#6, cd_education_status#7, 5), ENSURE_REQUIREMENTS, [id=#32]
+
+(45) HashAggregate [codegen id : 8]
+Input [6]: [cc_call_center_id#26, cc_name#27, cc_manager#28, cd_marital_status#6, cd_education_status#7, sum#31]
+Keys [5]: [cc_call_center_id#26, cc_name#27, cc_manager#28, cd_marital_status#6, cd_education_status#7]
+Functions [1]: [sum(UnscaledValue(cr_net_loss#17))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cr_net_loss#17))#33]
+Results [4]: [cc_call_center_id#26 AS Call_Center#34, cc_name#27 AS Call_Center_Name#35, cc_manager#28 AS Manager#36, MakeDecimal(sum(UnscaledValue(cr_net_loss#17))#33,17,2) AS Returns_Loss#37]
+
+(46) Exchange
+Input [4]: [Call_Center#34, Call_Center_Name#35, Manager#36, Returns_Loss#37]
+Arguments: rangepartitioning(Returns_Loss#37 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [id=#38]
+
+(47) Sort [codegen id : 9]
+Input [4]: [Call_Center#34, Call_Center_Name#35, Manager#36, Returns_Loss#37]
+Arguments: [Returns_Loss#37 DESC NULLS LAST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 24 Hosting Expression = cr_returned_date_sk#18 IN dynamicpruning#19
+ReusedExchange (48)
+
+
+(48) ReusedExchange [Reuses operator id: 31]
+Output [1]: [d_date_sk#20]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.sf100/simplified.txt
new file mode 100644
index 0000000000000..03ef76bc63897
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.sf100/simplified.txt
@@ -0,0 +1,73 @@
+WholeStageCodegen (9)
+  Sort [Returns_Loss]
+    InputAdapter
+      Exchange [Returns_Loss] #1
+        WholeStageCodegen (8)
+          HashAggregate [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,sum] [sum(UnscaledValue(cr_net_loss)),Call_Center,Call_Center_Name,Manager,Returns_Loss,sum]
+            InputAdapter
+              Exchange [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status] #2
+                WholeStageCodegen (7)
+                  HashAggregate [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,cr_net_loss] [sum,sum]
+                    Project [cc_call_center_id,cc_name,cc_manager,cr_net_loss,cd_marital_status,cd_education_status]
+                      BroadcastHashJoin [cr_call_center_sk,cc_call_center_sk]
+                        Project [cd_marital_status,cd_education_status,cr_call_center_sk,cr_net_loss]
+                          BroadcastHashJoin [c_customer_sk,cr_returning_customer_sk]
+                            Project [c_customer_sk,cd_marital_status,cd_education_status]
+                              BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                                Project [c_customer_sk,c_current_addr_sk,cd_marital_status,cd_education_status]
+                                  BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk]
+                                    Project [c_customer_sk,c_current_hdemo_sk,c_current_addr_sk,cd_marital_status,cd_education_status]
+                                      BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk]
+                                        Filter [c_customer_sk,c_current_addr_sk,c_current_cdemo_sk,c_current_hdemo_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk]
+                                        InputAdapter
+                                          BroadcastExchange #3
+                                            WholeStageCodegen (1)
+                                              Filter [cd_marital_status,cd_education_status,cd_demo_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status]
+                                    InputAdapter
+                                      BroadcastExchange #4
+                                        WholeStageCodegen (2)
+                                          Project [hd_demo_sk]
+                                            Filter [hd_buy_potential,hd_demo_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.household_demographics [hd_demo_sk,hd_buy_potential]
+                                InputAdapter
+                                  BroadcastExchange #5
+                                    WholeStageCodegen (3)
+                                      Project [ca_address_sk]
+                                        Filter [ca_gmt_offset,ca_address_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer_address [ca_address_sk,ca_gmt_offset]
+                            InputAdapter
+                              BroadcastExchange #6
+                                WholeStageCodegen (5)
+                                  Project [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss]
+                                    BroadcastHashJoin [cr_returned_date_sk,d_date_sk]
+                                      Filter [cr_call_center_sk,cr_returning_customer_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #1
+                                                ReusedExchange [d_date_sk] #7
+                                      InputAdapter
+                                        BroadcastExchange #7
+                                          WholeStageCodegen (4)
+                                            Project [d_date_sk]
+                                              Filter [d_year,d_moy,d_date_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                        InputAdapter
+                          BroadcastExchange #8
+                            WholeStageCodegen (6)
+                              Filter [cc_call_center_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.call_center [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/explain.txt
new file mode 100644
index 0000000000000..fc0da4adf7b16
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/explain.txt
@@ -0,0 +1,275 @@
+== Physical Plan ==
+* Sort (47)
++- Exchange (46)
+   +- * HashAggregate (45)
+      +- Exchange (44)
+         +- * HashAggregate (43)
+            +- * Project (42)
+               +- * BroadcastHashJoin Inner BuildRight (41)
+                  :- * Project (35)
+                  :  +- * BroadcastHashJoin Inner BuildRight (34)
+                  :     :- * Project (29)
+                  :     :  +- * BroadcastHashJoin Inner BuildRight (28)
+                  :     :     :- * Project (22)
+                  :     :     :  +- * BroadcastHashJoin Inner BuildRight (21)
+                  :     :     :     :- * Project (16)
+                  :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (15)
+                  :     :     :     :     :- * Project (9)
+                  :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+                  :     :     :     :     :     :- * Filter (3)
+                  :     :     :     :     :     :  +- * ColumnarToRow (2)
+                  :     :     :     :     :     :     +- Scan parquet default.call_center (1)
+                  :     :     :     :     :     +- BroadcastExchange (7)
+                  :     :     :     :     :        +- * Filter (6)
+                  :     :     :     :     :           +- * ColumnarToRow (5)
+                  :     :     :     :     :              +- Scan parquet default.catalog_returns (4)
+                  :     :     :     :     +- BroadcastExchange (14)
+                  :     :     :     :        +- * Project (13)
+                  :     :     :     :           +- * Filter (12)
+                  :     :     :     :              +- * ColumnarToRow (11)
+                  :     :     :     :                 +- Scan parquet default.date_dim (10)
+                  :     :     :     +- BroadcastExchange (20)
+                  :     :     :        +- * Filter (19)
+                  :     :     :           +- * ColumnarToRow (18)
+                  :     :     :              +- Scan parquet default.customer (17)
+                  :     :     +- BroadcastExchange (27)
+                  :     :        +- * Project (26)
+                  :     :           +- * Filter (25)
+                  :     :              +- * ColumnarToRow (24)
+                  :     :                 +- Scan parquet default.customer_address (23)
+                  :     +- BroadcastExchange (33)
+                  :        +- * Filter (32)
+                  :           +- * ColumnarToRow (31)
+                  :              +- Scan parquet default.customer_demographics (30)
+                  +- BroadcastExchange (40)
+                     +- * Project (39)
+                        +- * Filter (38)
+                           +- * ColumnarToRow (37)
+                              +- Scan parquet default.household_demographics (36)
+
+
+(1) Scan parquet default.call_center
+Output [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/call_center]
+PushedFilters: [IsNotNull(cc_call_center_sk)]
+ReadSchema: struct<cc_call_center_sk:int,cc_call_center_id:string,cc_name:string,cc_manager:string>
+
+(2) ColumnarToRow [codegen id : 7]
+Input [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4]
+
+(3) Filter [codegen id : 7]
+Input [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4]
+Condition : isnotnull(cc_call_center_sk#1)
+
+(4) Scan parquet default.catalog_returns
+Output [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cr_returned_date_sk#8), dynamicpruningexpression(cr_returned_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(cr_call_center_sk), IsNotNull(cr_returning_customer_sk)]
+ReadSchema: struct<cr_returning_customer_sk:int,cr_call_center_sk:int,cr_net_loss:decimal(7,2)>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8]
+
+(6) Filter [codegen id : 1]
+Input [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8]
+Condition : (isnotnull(cr_call_center_sk#6) AND isnotnull(cr_returning_customer_sk#5))
+
+(7) BroadcastExchange
+Input [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [id=#10]
+
+(8) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [cc_call_center_sk#1]
+Right keys [1]: [cr_call_center_sk#6]
+Join condition: None
+
+(9) Project [codegen id : 7]
+Output [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8]
+Input [8]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8]
+
+(10) Scan parquet default.date_dim
+Output [3]: [d_date_sk#11, d_year#12, d_moy#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [3]: [d_date_sk#11, d_year#12, d_moy#13]
+
+(12) Filter [codegen id : 2]
+Input [3]: [d_date_sk#11, d_year#12, d_moy#13]
+Condition : ((((isnotnull(d_year#12) AND isnotnull(d_moy#13)) AND (d_year#12 = 1998)) AND (d_moy#13 = 11)) AND isnotnull(d_date_sk#11))
+
+(13) Project [codegen id : 2]
+Output [1]: [d_date_sk#11]
+Input [3]: [d_date_sk#11, d_year#12, d_moy#13]
+
+(14) BroadcastExchange
+Input [1]: [d_date_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(15) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [cr_returned_date_sk#8]
+Right keys [1]: [d_date_sk#11]
+Join condition: None
+
+(16) Project [codegen id : 7]
+Output [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7]
+Input [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8, d_date_sk#11]
+
+(17) Scan parquet default.customer
+Output [4]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_hdemo_sk#17, c_current_addr_sk#18]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_hdemo_sk:int,c_current_addr_sk:int>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [4]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_hdemo_sk#17, c_current_addr_sk#18]
+
+(19) Filter [codegen id : 3]
+Input [4]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_hdemo_sk#17, c_current_addr_sk#18]
+Condition : (((isnotnull(c_customer_sk#15) AND isnotnull(c_current_addr_sk#18)) AND isnotnull(c_current_cdemo_sk#16)) AND isnotnull(c_current_hdemo_sk#17))
+
+(20) BroadcastExchange
+Input [4]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_hdemo_sk#17, c_current_addr_sk#18]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#19]
+
+(21) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [cr_returning_customer_sk#5]
+Right keys [1]: [c_customer_sk#15]
+Join condition: None
+
+(22) Project [codegen id : 7]
+Output [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#16, c_current_hdemo_sk#17, c_current_addr_sk#18]
+Input [9]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, c_customer_sk#15, c_current_cdemo_sk#16, c_current_hdemo_sk#17, c_current_addr_sk#18]
+
+(23) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#20, ca_gmt_offset#21]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-7.00), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_gmt_offset:decimal(5,2)>
+
+(24) ColumnarToRow [codegen id : 4]
+Input [2]: [ca_address_sk#20, ca_gmt_offset#21]
+
+(25) Filter [codegen id : 4]
+Input [2]: [ca_address_sk#20, ca_gmt_offset#21]
+Condition : ((isnotnull(ca_gmt_offset#21) AND (ca_gmt_offset#21 = -7.00)) AND isnotnull(ca_address_sk#20))
+
+(26) Project [codegen id : 4]
+Output [1]: [ca_address_sk#20]
+Input [2]: [ca_address_sk#20, ca_gmt_offset#21]
+
+(27) BroadcastExchange
+Input [1]: [ca_address_sk#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#22]
+
+(28) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [c_current_addr_sk#18]
+Right keys [1]: [ca_address_sk#20]
+Join condition: None
+
+(29) Project [codegen id : 7]
+Output [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#16, c_current_hdemo_sk#17]
+Input [8]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#16, c_current_hdemo_sk#17, c_current_addr_sk#18, ca_address_sk#20]
+
+(30) Scan parquet default.customer_demographics
+Output [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,Unknown             )),And(EqualTo(cd_marital_status,W),EqualTo(cd_education_status,Advanced Degree     ))), IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_marital_status:string,cd_education_status:string>
+
+(31) ColumnarToRow [codegen id : 5]
+Input [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25]
+
+(32) Filter [codegen id : 5]
+Input [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25]
+Condition : ((((cd_marital_status#24 = M) AND (cd_education_status#25 = Unknown             )) OR ((cd_marital_status#24 = W) AND (cd_education_status#25 = Advanced Degree     ))) AND isnotnull(cd_demo_sk#23))
+
+(33) BroadcastExchange
+Input [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#26]
+
+(34) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [c_current_cdemo_sk#16]
+Right keys [1]: [cd_demo_sk#23]
+Join condition: None
+
+(35) Project [codegen id : 7]
+Output [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#17, cd_marital_status#24, cd_education_status#25]
+Input [9]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#16, c_current_hdemo_sk#17, cd_demo_sk#23, cd_marital_status#24, cd_education_status#25]
+
+(36) Scan parquet default.household_demographics
+Output [2]: [hd_demo_sk#27, hd_buy_potential#28]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_buy_potential), StringStartsWith(hd_buy_potential,Unknown), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_buy_potential:string>
+
+(37) ColumnarToRow [codegen id : 6]
+Input [2]: [hd_demo_sk#27, hd_buy_potential#28]
+
+(38) Filter [codegen id : 6]
+Input [2]: [hd_demo_sk#27, hd_buy_potential#28]
+Condition : ((isnotnull(hd_buy_potential#28) AND StartsWith(hd_buy_potential#28, Unknown)) AND isnotnull(hd_demo_sk#27))
+
+(39) Project [codegen id : 6]
+Output [1]: [hd_demo_sk#27]
+Input [2]: [hd_demo_sk#27, hd_buy_potential#28]
+
+(40) BroadcastExchange
+Input [1]: [hd_demo_sk#27]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#29]
+
+(41) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [c_current_hdemo_sk#17]
+Right keys [1]: [hd_demo_sk#27]
+Join condition: None
+
+(42) Project [codegen id : 7]
+Output [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_marital_status#24, cd_education_status#25]
+Input [8]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#17, cd_marital_status#24, cd_education_status#25, hd_demo_sk#27]
+
+(43) HashAggregate [codegen id : 7]
+Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_marital_status#24, cd_education_status#25]
+Keys [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#24, cd_education_status#25]
+Functions [1]: [partial_sum(UnscaledValue(cr_net_loss#7))]
+Aggregate Attributes [1]: [sum#30]
+Results [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#24, cd_education_status#25, sum#31]
+
+(44) Exchange
+Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#24, cd_education_status#25, sum#31]
+Arguments: hashpartitioning(cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#24, cd_education_status#25, 5), ENSURE_REQUIREMENTS, [id=#32]
+
+(45) HashAggregate [codegen id : 8]
+Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#24, cd_education_status#25, sum#31]
+Keys [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#24, cd_education_status#25]
+Functions [1]: [sum(UnscaledValue(cr_net_loss#7))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cr_net_loss#7))#33]
+Results [4]: [cc_call_center_id#2 AS Call_Center#34, cc_name#3 AS Call_Center_Name#35, cc_manager#4 AS Manager#36, MakeDecimal(sum(UnscaledValue(cr_net_loss#7))#33,17,2) AS Returns_Loss#37]
+
+(46) Exchange
+Input [4]: [Call_Center#34, Call_Center_Name#35, Manager#36, Returns_Loss#37]
+Arguments: rangepartitioning(Returns_Loss#37 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [id=#38]
+
+(47) Sort [codegen id : 9]
+Input [4]: [Call_Center#34, Call_Center_Name#35, Manager#36, Returns_Loss#37]
+Arguments: [Returns_Loss#37 DESC NULLS LAST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 4 Hosting Expression = cr_returned_date_sk#8 IN dynamicpruning#9
+ReusedExchange (48)
+
+
+(48) ReusedExchange [Reuses operator id: 14]
+Output [1]: [d_date_sk#11]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/simplified.txt
new file mode 100644
index 0000000000000..885d4127c6ff7
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/simplified.txt
@@ -0,0 +1,73 @@
+WholeStageCodegen (9)
+  Sort [Returns_Loss]
+    InputAdapter
+      Exchange [Returns_Loss] #1
+        WholeStageCodegen (8)
+          HashAggregate [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,sum] [sum(UnscaledValue(cr_net_loss)),Call_Center,Call_Center_Name,Manager,Returns_Loss,sum]
+            InputAdapter
+              Exchange [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status] #2
+                WholeStageCodegen (7)
+                  HashAggregate [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,cr_net_loss] [sum,sum]
+                    Project [cc_call_center_id,cc_name,cc_manager,cr_net_loss,cd_marital_status,cd_education_status]
+                      BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk]
+                        Project [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status]
+                          BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk]
+                            Project [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk]
+                              BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                                Project [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk]
+                                  BroadcastHashJoin [cr_returning_customer_sk,c_customer_sk]
+                                    Project [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss]
+                                      BroadcastHashJoin [cr_returned_date_sk,d_date_sk]
+                                        Project [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk]
+                                          BroadcastHashJoin [cc_call_center_sk,cr_call_center_sk]
+                                            Filter [cc_call_center_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.call_center [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager]
+                                            InputAdapter
+                                              BroadcastExchange #3
+                                                WholeStageCodegen (1)
+                                                  Filter [cr_call_center_sk,cr_returning_customer_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk]
+                                                          SubqueryBroadcast [d_date_sk] #1
+                                                            ReusedExchange [d_date_sk] #4
+                                        InputAdapter
+                                          BroadcastExchange #4
+                                            WholeStageCodegen (2)
+                                              Project [d_date_sk]
+                                                Filter [d_year,d_moy,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                    InputAdapter
+                                      BroadcastExchange #5
+                                        WholeStageCodegen (3)
+                                          Filter [c_customer_sk,c_current_addr_sk,c_current_cdemo_sk,c_current_hdemo_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk]
+                                InputAdapter
+                                  BroadcastExchange #6
+                                    WholeStageCodegen (4)
+                                      Project [ca_address_sk]
+                                        Filter [ca_gmt_offset,ca_address_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer_address [ca_address_sk,ca_gmt_offset]
+                            InputAdapter
+                              BroadcastExchange #7
+                                WholeStageCodegen (5)
+                                  Filter [cd_marital_status,cd_education_status,cd_demo_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status]
+                        InputAdapter
+                          BroadcastExchange #8
+                            WholeStageCodegen (6)
+                              Project [hd_demo_sk]
+                                Filter [hd_buy_potential,hd_demo_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.household_demographics [hd_demo_sk,hd_buy_potential]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.sf100/explain.txt
new file mode 100644
index 0000000000000..94e76ced527a0
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.sf100/explain.txt
@@ -0,0 +1,210 @@
+== Physical Plan ==
+* Sort (34)
++- * HashAggregate (33)
+   +- Exchange (32)
+      +- * HashAggregate (31)
+         +- * Project (30)
+            +- * BroadcastHashJoin Inner BuildRight (29)
+               :- * Project (27)
+               :  +- * BroadcastHashJoin Inner BuildLeft (26)
+               :     :- BroadcastExchange (22)
+               :     :  +- * Project (21)
+               :     :     +- * BroadcastHashJoin Inner BuildLeft (20)
+               :     :        :- BroadcastExchange (5)
+               :     :        :  +- * Project (4)
+               :     :        :     +- * Filter (3)
+               :     :        :        +- * ColumnarToRow (2)
+               :     :        :           +- Scan parquet default.item (1)
+               :     :        +- * Filter (19)
+               :     :           +- * HashAggregate (18)
+               :     :              +- Exchange (17)
+               :     :                 +- * HashAggregate (16)
+               :     :                    +- * Project (15)
+               :     :                       +- * BroadcastHashJoin Inner BuildRight (14)
+               :     :                          :- * Filter (8)
+               :     :                          :  +- * ColumnarToRow (7)
+               :     :                          :     +- Scan parquet default.web_sales (6)
+               :     :                          +- BroadcastExchange (13)
+               :     :                             +- * Project (12)
+               :     :                                +- * Filter (11)
+               :     :                                   +- * ColumnarToRow (10)
+               :     :                                      +- Scan parquet default.date_dim (9)
+               :     +- * Filter (25)
+               :        +- * ColumnarToRow (24)
+               :           +- Scan parquet default.web_sales (23)
+               +- ReusedExchange (28)
+
+
+(1) Scan parquet default.item
+Output [2]: [i_item_sk#1, i_manufact_id#2]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,350), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_manufact_id:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [2]: [i_item_sk#1, i_manufact_id#2]
+
+(3) Filter [codegen id : 1]
+Input [2]: [i_item_sk#1, i_manufact_id#2]
+Condition : ((isnotnull(i_manufact_id#2) AND (i_manufact_id#2 = 350)) AND isnotnull(i_item_sk#1))
+
+(4) Project [codegen id : 1]
+Output [1]: [i_item_sk#1]
+Input [2]: [i_item_sk#1, i_manufact_id#2]
+
+(5) BroadcastExchange
+Input [1]: [i_item_sk#1]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#3]
+
+(6) Scan parquet default.web_sales
+Output [3]: [ws_item_sk#4, ws_ext_discount_amt#5, ws_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#6), dynamicpruningexpression(ws_sold_date_sk#6 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_ext_discount_amt:decimal(7,2)>
+
+(7) ColumnarToRow [codegen id : 3]
+Input [3]: [ws_item_sk#4, ws_ext_discount_amt#5, ws_sold_date_sk#6]
+
+(8) Filter [codegen id : 3]
+Input [3]: [ws_item_sk#4, ws_ext_discount_amt#5, ws_sold_date_sk#6]
+Condition : isnotnull(ws_item_sk#4)
+
+(9) Scan parquet default.date_dim
+Output [2]: [d_date_sk#8, d_date#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(10) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#8, d_date#9]
+
+(11) Filter [codegen id : 2]
+Input [2]: [d_date_sk#8, d_date#9]
+Condition : (((isnotnull(d_date#9) AND (d_date#9 >= 2000-01-27)) AND (d_date#9 <= 2000-04-26)) AND isnotnull(d_date_sk#8))
+
+(12) Project [codegen id : 2]
+Output [1]: [d_date_sk#8]
+Input [2]: [d_date_sk#8, d_date#9]
+
+(13) BroadcastExchange
+Input [1]: [d_date_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(14) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ws_sold_date_sk#6]
+Right keys [1]: [d_date_sk#8]
+Join condition: None
+
+(15) Project [codegen id : 3]
+Output [2]: [ws_item_sk#4, ws_ext_discount_amt#5]
+Input [4]: [ws_item_sk#4, ws_ext_discount_amt#5, ws_sold_date_sk#6, d_date_sk#8]
+
+(16) HashAggregate [codegen id : 3]
+Input [2]: [ws_item_sk#4, ws_ext_discount_amt#5]
+Keys [1]: [ws_item_sk#4]
+Functions [1]: [partial_avg(UnscaledValue(ws_ext_discount_amt#5))]
+Aggregate Attributes [2]: [sum#11, count#12]
+Results [3]: [ws_item_sk#4, sum#13, count#14]
+
+(17) Exchange
+Input [3]: [ws_item_sk#4, sum#13, count#14]
+Arguments: hashpartitioning(ws_item_sk#4, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(18) HashAggregate
+Input [3]: [ws_item_sk#4, sum#13, count#14]
+Keys [1]: [ws_item_sk#4]
+Functions [1]: [avg(UnscaledValue(ws_ext_discount_amt#5))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ws_ext_discount_amt#5))#16]
+Results [2]: [CheckOverflow((1.300000 * promote_precision(cast((avg(UnscaledValue(ws_ext_discount_amt#5))#16 / 100.0) as decimal(11,6)))), DecimalType(14,7), true) AS (1.3 * avg(ws_ext_discount_amt))#17, ws_item_sk#4]
+
+(19) Filter
+Input [2]: [(1.3 * avg(ws_ext_discount_amt))#17, ws_item_sk#4]
+Condition : isnotnull((1.3 * avg(ws_ext_discount_amt))#17)
+
+(20) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [ws_item_sk#4]
+Join condition: None
+
+(21) Project [codegen id : 4]
+Output [2]: [i_item_sk#1, (1.3 * avg(ws_ext_discount_amt))#17]
+Input [3]: [i_item_sk#1, (1.3 * avg(ws_ext_discount_amt))#17, ws_item_sk#4]
+
+(22) BroadcastExchange
+Input [2]: [i_item_sk#1, (1.3 * avg(ws_ext_discount_amt))#17]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(23) Scan parquet default.web_sales
+Output [3]: [ws_item_sk#19, ws_ext_discount_amt#20, ws_sold_date_sk#21]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#21), dynamicpruningexpression(ws_sold_date_sk#21 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_ext_discount_amt)]
+ReadSchema: struct<ws_item_sk:int,ws_ext_discount_amt:decimal(7,2)>
+
+(24) ColumnarToRow
+Input [3]: [ws_item_sk#19, ws_ext_discount_amt#20, ws_sold_date_sk#21]
+
+(25) Filter
+Input [3]: [ws_item_sk#19, ws_ext_discount_amt#20, ws_sold_date_sk#21]
+Condition : (isnotnull(ws_item_sk#19) AND isnotnull(ws_ext_discount_amt#20))
+
+(26) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [ws_item_sk#19]
+Join condition: (cast(ws_ext_discount_amt#20 as decimal(14,7)) > (1.3 * avg(ws_ext_discount_amt))#17)
+
+(27) Project [codegen id : 6]
+Output [2]: [ws_ext_discount_amt#20, ws_sold_date_sk#21]
+Input [5]: [i_item_sk#1, (1.3 * avg(ws_ext_discount_amt))#17, ws_item_sk#19, ws_ext_discount_amt#20, ws_sold_date_sk#21]
+
+(28) ReusedExchange [Reuses operator id: 13]
+Output [1]: [d_date_sk#22]
+
+(29) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ws_sold_date_sk#21]
+Right keys [1]: [d_date_sk#22]
+Join condition: None
+
+(30) Project [codegen id : 6]
+Output [1]: [ws_ext_discount_amt#20]
+Input [3]: [ws_ext_discount_amt#20, ws_sold_date_sk#21, d_date_sk#22]
+
+(31) HashAggregate [codegen id : 6]
+Input [1]: [ws_ext_discount_amt#20]
+Keys: []
+Functions [1]: [partial_sum(UnscaledValue(ws_ext_discount_amt#20))]
+Aggregate Attributes [1]: [sum#23]
+Results [1]: [sum#24]
+
+(32) Exchange
+Input [1]: [sum#24]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#25]
+
+(33) HashAggregate [codegen id : 7]
+Input [1]: [sum#24]
+Keys: []
+Functions [1]: [sum(UnscaledValue(ws_ext_discount_amt#20))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_discount_amt#20))#26]
+Results [1]: [MakeDecimal(sum(UnscaledValue(ws_ext_discount_amt#20))#26,17,2) AS Excess Discount Amount #27]
+
+(34) Sort [codegen id : 7]
+Input [1]: [Excess Discount Amount #27]
+Arguments: [Excess Discount Amount #27 ASC NULLS FIRST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 6 Hosting Expression = ws_sold_date_sk#6 IN dynamicpruning#7
+ReusedExchange (35)
+
+
+(35) ReusedExchange [Reuses operator id: 13]
+Output [1]: [d_date_sk#8]
+
+Subquery:2 Hosting operator id = 23 Hosting Expression = ws_sold_date_sk#21 IN dynamicpruning#7
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.sf100/simplified.txt
new file mode 100644
index 0000000000000..e83a3e67e5c6f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.sf100/simplified.txt
@@ -0,0 +1,53 @@
+WholeStageCodegen (7)
+  Sort [Excess Discount Amount ]
+    HashAggregate [sum] [sum(UnscaledValue(ws_ext_discount_amt)),Excess Discount Amount ,sum]
+      InputAdapter
+        Exchange #1
+          WholeStageCodegen (6)
+            HashAggregate [ws_ext_discount_amt] [sum,sum]
+              Project [ws_ext_discount_amt]
+                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                  Project [ws_ext_discount_amt,ws_sold_date_sk]
+                    BroadcastHashJoin [i_item_sk,ws_item_sk,ws_ext_discount_amt,(1.3 * avg(ws_ext_discount_amt))]
+                      InputAdapter
+                        BroadcastExchange #2
+                          WholeStageCodegen (4)
+                            Project [i_item_sk,(1.3 * avg(ws_ext_discount_amt))]
+                              BroadcastHashJoin [i_item_sk,ws_item_sk]
+                                InputAdapter
+                                  BroadcastExchange #3
+                                    WholeStageCodegen (1)
+                                      Project [i_item_sk]
+                                        Filter [i_manufact_id,i_item_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.item [i_item_sk,i_manufact_id]
+                                Filter [(1.3 * avg(ws_ext_discount_amt))]
+                                  HashAggregate [ws_item_sk,sum,count] [avg(UnscaledValue(ws_ext_discount_amt)),(1.3 * avg(ws_ext_discount_amt)),sum,count]
+                                    InputAdapter
+                                      Exchange [ws_item_sk] #4
+                                        WholeStageCodegen (3)
+                                          HashAggregate [ws_item_sk,ws_ext_discount_amt] [sum,count,sum,count]
+                                            Project [ws_item_sk,ws_ext_discount_amt]
+                                              BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                Filter [ws_item_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk]
+                                                        SubqueryBroadcast [d_date_sk] #1
+                                                          ReusedExchange [d_date_sk] #5
+                                                InputAdapter
+                                                  BroadcastExchange #5
+                                                    WholeStageCodegen (2)
+                                                      Project [d_date_sk]
+                                                        Filter [d_date,d_date_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.date_dim [d_date_sk,d_date]
+                      Filter [ws_item_sk,ws_ext_discount_amt]
+                        ColumnarToRow
+                          InputAdapter
+                            Scan parquet default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk]
+                              ReusedSubquery [d_date_sk] #1
+                  InputAdapter
+                    ReusedExchange [d_date_sk] #5
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/explain.txt
new file mode 100644
index 0000000000000..46953fe184f28
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/explain.txt
@@ -0,0 +1,210 @@
+== Physical Plan ==
+* Sort (34)
++- * HashAggregate (33)
+   +- Exchange (32)
+      +- * HashAggregate (31)
+         +- * Project (30)
+            +- * BroadcastHashJoin Inner BuildRight (29)
+               :- * Project (27)
+               :  +- * BroadcastHashJoin Inner BuildRight (26)
+               :     :- * Project (10)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :     :- * Filter (3)
+               :     :     :  +- * ColumnarToRow (2)
+               :     :     :     +- Scan parquet default.web_sales (1)
+               :     :     +- BroadcastExchange (8)
+               :     :        +- * Project (7)
+               :     :           +- * Filter (6)
+               :     :              +- * ColumnarToRow (5)
+               :     :                 +- Scan parquet default.item (4)
+               :     +- BroadcastExchange (25)
+               :        +- * Filter (24)
+               :           +- * HashAggregate (23)
+               :              +- Exchange (22)
+               :                 +- * HashAggregate (21)
+               :                    +- * Project (20)
+               :                       +- * BroadcastHashJoin Inner BuildRight (19)
+               :                          :- * Filter (13)
+               :                          :  +- * ColumnarToRow (12)
+               :                          :     +- Scan parquet default.web_sales (11)
+               :                          +- BroadcastExchange (18)
+               :                             +- * Project (17)
+               :                                +- * Filter (16)
+               :                                   +- * ColumnarToRow (15)
+               :                                      +- Scan parquet default.date_dim (14)
+               +- ReusedExchange (28)
+
+
+(1) Scan parquet default.web_sales
+Output [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_ext_discount_amt)]
+ReadSchema: struct<ws_item_sk:int,ws_ext_discount_amt:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 6]
+Input [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3]
+
+(3) Filter [codegen id : 6]
+Input [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3]
+Condition : (isnotnull(ws_item_sk#1) AND isnotnull(ws_ext_discount_amt#2))
+
+(4) Scan parquet default.item
+Output [2]: [i_item_sk#5, i_manufact_id#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,350), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_manufact_id:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [i_item_sk#5, i_manufact_id#6]
+
+(6) Filter [codegen id : 1]
+Input [2]: [i_item_sk#5, i_manufact_id#6]
+Condition : ((isnotnull(i_manufact_id#6) AND (i_manufact_id#6 = 350)) AND isnotnull(i_item_sk#5))
+
+(7) Project [codegen id : 1]
+Output [1]: [i_item_sk#5]
+Input [2]: [i_item_sk#5, i_manufact_id#6]
+
+(8) BroadcastExchange
+Input [1]: [i_item_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ws_item_sk#1]
+Right keys [1]: [i_item_sk#5]
+Join condition: None
+
+(10) Project [codegen id : 6]
+Output [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5]
+Input [4]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5]
+
+(11) Scan parquet default.web_sales
+Output [3]: [ws_item_sk#8, ws_ext_discount_amt#9, ws_sold_date_sk#10]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#10), dynamicpruningexpression(ws_sold_date_sk#10 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_ext_discount_amt:decimal(7,2)>
+
+(12) ColumnarToRow [codegen id : 3]
+Input [3]: [ws_item_sk#8, ws_ext_discount_amt#9, ws_sold_date_sk#10]
+
+(13) Filter [codegen id : 3]
+Input [3]: [ws_item_sk#8, ws_ext_discount_amt#9, ws_sold_date_sk#10]
+Condition : isnotnull(ws_item_sk#8)
+
+(14) Scan parquet default.date_dim
+Output [2]: [d_date_sk#11, d_date#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(15) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#11, d_date#12]
+
+(16) Filter [codegen id : 2]
+Input [2]: [d_date_sk#11, d_date#12]
+Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-01-27)) AND (d_date#12 <= 2000-04-26)) AND isnotnull(d_date_sk#11))
+
+(17) Project [codegen id : 2]
+Output [1]: [d_date_sk#11]
+Input [2]: [d_date_sk#11, d_date#12]
+
+(18) BroadcastExchange
+Input [1]: [d_date_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(19) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ws_sold_date_sk#10]
+Right keys [1]: [d_date_sk#11]
+Join condition: None
+
+(20) Project [codegen id : 3]
+Output [2]: [ws_item_sk#8, ws_ext_discount_amt#9]
+Input [4]: [ws_item_sk#8, ws_ext_discount_amt#9, ws_sold_date_sk#10, d_date_sk#11]
+
+(21) HashAggregate [codegen id : 3]
+Input [2]: [ws_item_sk#8, ws_ext_discount_amt#9]
+Keys [1]: [ws_item_sk#8]
+Functions [1]: [partial_avg(UnscaledValue(ws_ext_discount_amt#9))]
+Aggregate Attributes [2]: [sum#14, count#15]
+Results [3]: [ws_item_sk#8, sum#16, count#17]
+
+(22) Exchange
+Input [3]: [ws_item_sk#8, sum#16, count#17]
+Arguments: hashpartitioning(ws_item_sk#8, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(23) HashAggregate [codegen id : 4]
+Input [3]: [ws_item_sk#8, sum#16, count#17]
+Keys [1]: [ws_item_sk#8]
+Functions [1]: [avg(UnscaledValue(ws_ext_discount_amt#9))]
+Aggregate Attributes [1]: [avg(UnscaledValue(ws_ext_discount_amt#9))#19]
+Results [2]: [CheckOverflow((1.300000 * promote_precision(cast((avg(UnscaledValue(ws_ext_discount_amt#9))#19 / 100.0) as decimal(11,6)))), DecimalType(14,7), true) AS (1.3 * avg(ws_ext_discount_amt))#20, ws_item_sk#8]
+
+(24) Filter [codegen id : 4]
+Input [2]: [(1.3 * avg(ws_ext_discount_amt))#20, ws_item_sk#8]
+Condition : isnotnull((1.3 * avg(ws_ext_discount_amt))#20)
+
+(25) BroadcastExchange
+Input [2]: [(1.3 * avg(ws_ext_discount_amt))#20, ws_item_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [id=#21]
+
+(26) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [i_item_sk#5]
+Right keys [1]: [ws_item_sk#8]
+Join condition: (cast(ws_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(ws_ext_discount_amt))#20)
+
+(27) Project [codegen id : 6]
+Output [2]: [ws_ext_discount_amt#2, ws_sold_date_sk#3]
+Input [5]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5, (1.3 * avg(ws_ext_discount_amt))#20, ws_item_sk#8]
+
+(28) ReusedExchange [Reuses operator id: 18]
+Output [1]: [d_date_sk#22]
+
+(29) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ws_sold_date_sk#3]
+Right keys [1]: [d_date_sk#22]
+Join condition: None
+
+(30) Project [codegen id : 6]
+Output [1]: [ws_ext_discount_amt#2]
+Input [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, d_date_sk#22]
+
+(31) HashAggregate [codegen id : 6]
+Input [1]: [ws_ext_discount_amt#2]
+Keys: []
+Functions [1]: [partial_sum(UnscaledValue(ws_ext_discount_amt#2))]
+Aggregate Attributes [1]: [sum#23]
+Results [1]: [sum#24]
+
+(32) Exchange
+Input [1]: [sum#24]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#25]
+
+(33) HashAggregate [codegen id : 7]
+Input [1]: [sum#24]
+Keys: []
+Functions [1]: [sum(UnscaledValue(ws_ext_discount_amt#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_discount_amt#2))#26]
+Results [1]: [MakeDecimal(sum(UnscaledValue(ws_ext_discount_amt#2))#26,17,2) AS Excess Discount Amount #27]
+
+(34) Sort [codegen id : 7]
+Input [1]: [Excess Discount Amount #27]
+Arguments: [Excess Discount Amount #27 ASC NULLS FIRST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (35)
+
+
+(35) ReusedExchange [Reuses operator id: 18]
+Output [1]: [d_date_sk#22]
+
+Subquery:2 Hosting operator id = 11 Hosting Expression = ws_sold_date_sk#10 IN dynamicpruning#4
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/simplified.txt
new file mode 100644
index 0000000000000..0df713c0e075e
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/simplified.txt
@@ -0,0 +1,53 @@
+WholeStageCodegen (7)
+  Sort [Excess Discount Amount ]
+    HashAggregate [sum] [sum(UnscaledValue(ws_ext_discount_amt)),Excess Discount Amount ,sum]
+      InputAdapter
+        Exchange #1
+          WholeStageCodegen (6)
+            HashAggregate [ws_ext_discount_amt] [sum,sum]
+              Project [ws_ext_discount_amt]
+                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                  Project [ws_ext_discount_amt,ws_sold_date_sk]
+                    BroadcastHashJoin [i_item_sk,ws_item_sk,ws_ext_discount_amt,(1.3 * avg(ws_ext_discount_amt))]
+                      Project [ws_ext_discount_amt,ws_sold_date_sk,i_item_sk]
+                        BroadcastHashJoin [ws_item_sk,i_item_sk]
+                          Filter [ws_item_sk,ws_ext_discount_amt]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk]
+                                  SubqueryBroadcast [d_date_sk] #1
+                                    ReusedExchange [d_date_sk] #2
+                          InputAdapter
+                            BroadcastExchange #3
+                              WholeStageCodegen (1)
+                                Project [i_item_sk]
+                                  Filter [i_manufact_id,i_item_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.item [i_item_sk,i_manufact_id]
+                      InputAdapter
+                        BroadcastExchange #4
+                          WholeStageCodegen (4)
+                            Filter [(1.3 * avg(ws_ext_discount_amt))]
+                              HashAggregate [ws_item_sk,sum,count] [avg(UnscaledValue(ws_ext_discount_amt)),(1.3 * avg(ws_ext_discount_amt)),sum,count]
+                                InputAdapter
+                                  Exchange [ws_item_sk] #5
+                                    WholeStageCodegen (3)
+                                      HashAggregate [ws_item_sk,ws_ext_discount_amt] [sum,count,sum,count]
+                                        Project [ws_item_sk,ws_ext_discount_amt]
+                                          BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                            Filter [ws_item_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk]
+                                                    ReusedSubquery [d_date_sk] #1
+                                            InputAdapter
+                                              BroadcastExchange #2
+                                                WholeStageCodegen (2)
+                                                  Project [d_date_sk]
+                                                    Filter [d_date,d_date_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.date_dim [d_date_sk,d_date]
+                  InputAdapter
+                    ReusedExchange [d_date_sk] #2
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.sf100/explain.txt
new file mode 100644
index 0000000000000..01b7b7f5e20c8
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.sf100/explain.txt
@@ -0,0 +1,136 @@
+== Physical Plan ==
+TakeOrderedAndProject (24)
++- * HashAggregate (23)
+   +- Exchange (22)
+      +- * HashAggregate (21)
+         +- * Project (20)
+            +- * SortMergeJoin Inner (19)
+               :- * Sort (13)
+               :  +- Exchange (12)
+               :     +- * Project (11)
+               :        +- * BroadcastHashJoin Inner BuildRight (10)
+               :           :- * Project (4)
+               :           :  +- * Filter (3)
+               :           :     +- * ColumnarToRow (2)
+               :           :        +- Scan parquet default.store_returns (1)
+               :           +- BroadcastExchange (9)
+               :              +- * Project (8)
+               :                 +- * Filter (7)
+               :                    +- * ColumnarToRow (6)
+               :                       +- Scan parquet default.reason (5)
+               +- * Sort (18)
+                  +- Exchange (17)
+                     +- * Project (16)
+                        +- * ColumnarToRow (15)
+                           +- Scan parquet default.store_sales (14)
+
+
+(1) Scan parquet default.store_returns
+Output [5]: [sr_item_sk#1, sr_reason_sk#2, sr_ticket_number#3, sr_return_quantity#4, sr_returned_date_sk#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number), IsNotNull(sr_reason_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_reason_sk:int,sr_ticket_number:int,sr_return_quantity:int>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [5]: [sr_item_sk#1, sr_reason_sk#2, sr_ticket_number#3, sr_return_quantity#4, sr_returned_date_sk#5]
+
+(3) Filter [codegen id : 2]
+Input [5]: [sr_item_sk#1, sr_reason_sk#2, sr_ticket_number#3, sr_return_quantity#4, sr_returned_date_sk#5]
+Condition : ((isnotnull(sr_item_sk#1) AND isnotnull(sr_ticket_number#3)) AND isnotnull(sr_reason_sk#2))
+
+(4) Project [codegen id : 2]
+Output [4]: [sr_item_sk#1, sr_reason_sk#2, sr_ticket_number#3, sr_return_quantity#4]
+Input [5]: [sr_item_sk#1, sr_reason_sk#2, sr_ticket_number#3, sr_return_quantity#4, sr_returned_date_sk#5]
+
+(5) Scan parquet default.reason
+Output [2]: [r_reason_sk#6, r_reason_desc#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/reason]
+PushedFilters: [IsNotNull(r_reason_desc), EqualTo(r_reason_desc,reason 28                                                                                           ), IsNotNull(r_reason_sk)]
+ReadSchema: struct<r_reason_sk:int,r_reason_desc:string>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [2]: [r_reason_sk#6, r_reason_desc#7]
+
+(7) Filter [codegen id : 1]
+Input [2]: [r_reason_sk#6, r_reason_desc#7]
+Condition : ((isnotnull(r_reason_desc#7) AND (r_reason_desc#7 = reason 28                                                                                           )) AND isnotnull(r_reason_sk#6))
+
+(8) Project [codegen id : 1]
+Output [1]: [r_reason_sk#6]
+Input [2]: [r_reason_sk#6, r_reason_desc#7]
+
+(9) BroadcastExchange
+Input [1]: [r_reason_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#8]
+
+(10) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [sr_reason_sk#2]
+Right keys [1]: [r_reason_sk#6]
+Join condition: None
+
+(11) Project [codegen id : 2]
+Output [3]: [sr_item_sk#1, sr_ticket_number#3, sr_return_quantity#4]
+Input [5]: [sr_item_sk#1, sr_reason_sk#2, sr_ticket_number#3, sr_return_quantity#4, r_reason_sk#6]
+
+(12) Exchange
+Input [3]: [sr_item_sk#1, sr_ticket_number#3, sr_return_quantity#4]
+Arguments: hashpartitioning(sr_item_sk#1, sr_ticket_number#3, 5), ENSURE_REQUIREMENTS, [id=#9]
+
+(13) Sort [codegen id : 3]
+Input [3]: [sr_item_sk#1, sr_ticket_number#3, sr_return_quantity#4]
+Arguments: [sr_item_sk#1 ASC NULLS FIRST, sr_ticket_number#3 ASC NULLS FIRST], false, 0
+
+(14) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#10, ss_customer_sk#11, ss_ticket_number#12, ss_quantity#13, ss_sales_price#14, ss_sold_date_sk#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_ticket_number:int,ss_quantity:int,ss_sales_price:decimal(7,2)>
+
+(15) ColumnarToRow [codegen id : 4]
+Input [6]: [ss_item_sk#10, ss_customer_sk#11, ss_ticket_number#12, ss_quantity#13, ss_sales_price#14, ss_sold_date_sk#15]
+
+(16) Project [codegen id : 4]
+Output [5]: [ss_item_sk#10, ss_customer_sk#11, ss_ticket_number#12, ss_quantity#13, ss_sales_price#14]
+Input [6]: [ss_item_sk#10, ss_customer_sk#11, ss_ticket_number#12, ss_quantity#13, ss_sales_price#14, ss_sold_date_sk#15]
+
+(17) Exchange
+Input [5]: [ss_item_sk#10, ss_customer_sk#11, ss_ticket_number#12, ss_quantity#13, ss_sales_price#14]
+Arguments: hashpartitioning(ss_item_sk#10, ss_ticket_number#12, 5), ENSURE_REQUIREMENTS, [id=#16]
+
+(18) Sort [codegen id : 5]
+Input [5]: [ss_item_sk#10, ss_customer_sk#11, ss_ticket_number#12, ss_quantity#13, ss_sales_price#14]
+Arguments: [ss_item_sk#10 ASC NULLS FIRST, ss_ticket_number#12 ASC NULLS FIRST], false, 0
+
+(19) SortMergeJoin [codegen id : 6]
+Left keys [2]: [sr_item_sk#1, sr_ticket_number#3]
+Right keys [2]: [ss_item_sk#10, ss_ticket_number#12]
+Join condition: None
+
+(20) Project [codegen id : 6]
+Output [2]: [ss_customer_sk#11, CASE WHEN isnotnull(sr_return_quantity#4) THEN CheckOverflow((promote_precision(cast(cast((ss_quantity#13 - sr_return_quantity#4) as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#14 as decimal(12,2)))), DecimalType(18,2), true) ELSE CheckOverflow((promote_precision(cast(cast(ss_quantity#13 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#14 as decimal(12,2)))), DecimalType(18,2), true) END AS act_sales#17]
+Input [8]: [sr_item_sk#1, sr_ticket_number#3, sr_return_quantity#4, ss_item_sk#10, ss_customer_sk#11, ss_ticket_number#12, ss_quantity#13, ss_sales_price#14]
+
+(21) HashAggregate [codegen id : 6]
+Input [2]: [ss_customer_sk#11, act_sales#17]
+Keys [1]: [ss_customer_sk#11]
+Functions [1]: [partial_sum(act_sales#17)]
+Aggregate Attributes [2]: [sum#18, isEmpty#19]
+Results [3]: [ss_customer_sk#11, sum#20, isEmpty#21]
+
+(22) Exchange
+Input [3]: [ss_customer_sk#11, sum#20, isEmpty#21]
+Arguments: hashpartitioning(ss_customer_sk#11, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(23) HashAggregate [codegen id : 7]
+Input [3]: [ss_customer_sk#11, sum#20, isEmpty#21]
+Keys [1]: [ss_customer_sk#11]
+Functions [1]: [sum(act_sales#17)]
+Aggregate Attributes [1]: [sum(act_sales#17)#23]
+Results [2]: [ss_customer_sk#11, sum(act_sales#17)#23 AS sumsales#24]
+
+(24) TakeOrderedAndProject
+Input [2]: [ss_customer_sk#11, sumsales#24]
+Arguments: 100, [sumsales#24 ASC NULLS FIRST, ss_customer_sk#11 ASC NULLS FIRST], [ss_customer_sk#11, sumsales#24]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.sf100/simplified.txt
new file mode 100644
index 0000000000000..17464e295b1dd
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.sf100/simplified.txt
@@ -0,0 +1,40 @@
+TakeOrderedAndProject [sumsales,ss_customer_sk]
+  WholeStageCodegen (7)
+    HashAggregate [ss_customer_sk,sum,isEmpty] [sum(act_sales),sumsales,sum,isEmpty]
+      InputAdapter
+        Exchange [ss_customer_sk] #1
+          WholeStageCodegen (6)
+            HashAggregate [ss_customer_sk,act_sales] [sum,isEmpty,sum,isEmpty]
+              Project [ss_customer_sk,sr_return_quantity,ss_quantity,ss_sales_price]
+                SortMergeJoin [sr_item_sk,sr_ticket_number,ss_item_sk,ss_ticket_number]
+                  InputAdapter
+                    WholeStageCodegen (3)
+                      Sort [sr_item_sk,sr_ticket_number]
+                        InputAdapter
+                          Exchange [sr_item_sk,sr_ticket_number] #2
+                            WholeStageCodegen (2)
+                              Project [sr_item_sk,sr_ticket_number,sr_return_quantity]
+                                BroadcastHashJoin [sr_reason_sk,r_reason_sk]
+                                  Project [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity]
+                                    Filter [sr_item_sk,sr_ticket_number,sr_reason_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.store_returns [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk]
+                                  InputAdapter
+                                    BroadcastExchange #3
+                                      WholeStageCodegen (1)
+                                        Project [r_reason_sk]
+                                          Filter [r_reason_desc,r_reason_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.reason [r_reason_sk,r_reason_desc]
+                  InputAdapter
+                    WholeStageCodegen (5)
+                      Sort [ss_item_sk,ss_ticket_number]
+                        InputAdapter
+                          Exchange [ss_item_sk,ss_ticket_number] #4
+                            WholeStageCodegen (4)
+                              Project [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/explain.txt
new file mode 100644
index 0000000000000..54b9ae752c7a0
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/explain.txt
@@ -0,0 +1,136 @@
+== Physical Plan ==
+TakeOrderedAndProject (24)
++- * HashAggregate (23)
+   +- Exchange (22)
+      +- * HashAggregate (21)
+         +- * Project (20)
+            +- * BroadcastHashJoin Inner BuildRight (19)
+               :- * Project (13)
+               :  +- * SortMergeJoin Inner (12)
+               :     :- * Sort (5)
+               :     :  +- Exchange (4)
+               :     :     +- * Project (3)
+               :     :        +- * ColumnarToRow (2)
+               :     :           +- Scan parquet default.store_sales (1)
+               :     +- * Sort (11)
+               :        +- Exchange (10)
+               :           +- * Project (9)
+               :              +- * Filter (8)
+               :                 +- * ColumnarToRow (7)
+               :                    +- Scan parquet default.store_returns (6)
+               +- BroadcastExchange (18)
+                  +- * Project (17)
+                     +- * Filter (16)
+                        +- * ColumnarToRow (15)
+                           +- Scan parquet default.reason (14)
+
+
+(1) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_ticket_number:int,ss_quantity:int,ss_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6]
+
+(3) Project [codegen id : 1]
+Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6]
+
+(4) Exchange
+Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5]
+Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#3, 5), ENSURE_REQUIREMENTS, [id=#7]
+
+(5) Sort [codegen id : 2]
+Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5]
+Arguments: [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#3 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.store_returns
+Output [5]: [sr_item_sk#8, sr_reason_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number), IsNotNull(sr_reason_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_reason_sk:int,sr_ticket_number:int,sr_return_quantity:int>
+
+(7) ColumnarToRow [codegen id : 3]
+Input [5]: [sr_item_sk#8, sr_reason_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12]
+
+(8) Filter [codegen id : 3]
+Input [5]: [sr_item_sk#8, sr_reason_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12]
+Condition : ((isnotnull(sr_item_sk#8) AND isnotnull(sr_ticket_number#10)) AND isnotnull(sr_reason_sk#9))
+
+(9) Project [codegen id : 3]
+Output [4]: [sr_item_sk#8, sr_reason_sk#9, sr_ticket_number#10, sr_return_quantity#11]
+Input [5]: [sr_item_sk#8, sr_reason_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12]
+
+(10) Exchange
+Input [4]: [sr_item_sk#8, sr_reason_sk#9, sr_ticket_number#10, sr_return_quantity#11]
+Arguments: hashpartitioning(sr_item_sk#8, sr_ticket_number#10, 5), ENSURE_REQUIREMENTS, [id=#13]
+
+(11) Sort [codegen id : 4]
+Input [4]: [sr_item_sk#8, sr_reason_sk#9, sr_ticket_number#10, sr_return_quantity#11]
+Arguments: [sr_item_sk#8 ASC NULLS FIRST, sr_ticket_number#10 ASC NULLS FIRST], false, 0
+
+(12) SortMergeJoin [codegen id : 6]
+Left keys [2]: [ss_item_sk#1, ss_ticket_number#3]
+Right keys [2]: [sr_item_sk#8, sr_ticket_number#10]
+Join condition: None
+
+(13) Project [codegen id : 6]
+Output [5]: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#9, sr_return_quantity#11]
+Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, sr_item_sk#8, sr_reason_sk#9, sr_ticket_number#10, sr_return_quantity#11]
+
+(14) Scan parquet default.reason
+Output [2]: [r_reason_sk#14, r_reason_desc#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/reason]
+PushedFilters: [IsNotNull(r_reason_desc), EqualTo(r_reason_desc,reason 28                                                                                           ), IsNotNull(r_reason_sk)]
+ReadSchema: struct<r_reason_sk:int,r_reason_desc:string>
+
+(15) ColumnarToRow [codegen id : 5]
+Input [2]: [r_reason_sk#14, r_reason_desc#15]
+
+(16) Filter [codegen id : 5]
+Input [2]: [r_reason_sk#14, r_reason_desc#15]
+Condition : ((isnotnull(r_reason_desc#15) AND (r_reason_desc#15 = reason 28                                                                                           )) AND isnotnull(r_reason_sk#14))
+
+(17) Project [codegen id : 5]
+Output [1]: [r_reason_sk#14]
+Input [2]: [r_reason_sk#14, r_reason_desc#15]
+
+(18) BroadcastExchange
+Input [1]: [r_reason_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#16]
+
+(19) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [sr_reason_sk#9]
+Right keys [1]: [r_reason_sk#14]
+Join condition: None
+
+(20) Project [codegen id : 6]
+Output [2]: [ss_customer_sk#2, CASE WHEN isnotnull(sr_return_quantity#11) THEN CheckOverflow((promote_precision(cast(cast((ss_quantity#4 - sr_return_quantity#11) as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#5 as decimal(12,2)))), DecimalType(18,2), true) ELSE CheckOverflow((promote_precision(cast(cast(ss_quantity#4 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#5 as decimal(12,2)))), DecimalType(18,2), true) END AS act_sales#17]
+Input [6]: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#9, sr_return_quantity#11, r_reason_sk#14]
+
+(21) HashAggregate [codegen id : 6]
+Input [2]: [ss_customer_sk#2, act_sales#17]
+Keys [1]: [ss_customer_sk#2]
+Functions [1]: [partial_sum(act_sales#17)]
+Aggregate Attributes [2]: [sum#18, isEmpty#19]
+Results [3]: [ss_customer_sk#2, sum#20, isEmpty#21]
+
+(22) Exchange
+Input [3]: [ss_customer_sk#2, sum#20, isEmpty#21]
+Arguments: hashpartitioning(ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(23) HashAggregate [codegen id : 7]
+Input [3]: [ss_customer_sk#2, sum#20, isEmpty#21]
+Keys [1]: [ss_customer_sk#2]
+Functions [1]: [sum(act_sales#17)]
+Aggregate Attributes [1]: [sum(act_sales#17)#23]
+Results [2]: [ss_customer_sk#2, sum(act_sales#17)#23 AS sumsales#24]
+
+(24) TakeOrderedAndProject
+Input [2]: [ss_customer_sk#2, sumsales#24]
+Arguments: 100, [sumsales#24 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], [ss_customer_sk#2, sumsales#24]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/simplified.txt
new file mode 100644
index 0000000000000..0d9aec90a2da4
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/simplified.txt
@@ -0,0 +1,40 @@
+TakeOrderedAndProject [sumsales,ss_customer_sk]
+  WholeStageCodegen (7)
+    HashAggregate [ss_customer_sk,sum,isEmpty] [sum(act_sales),sumsales,sum,isEmpty]
+      InputAdapter
+        Exchange [ss_customer_sk] #1
+          WholeStageCodegen (6)
+            HashAggregate [ss_customer_sk,act_sales] [sum,isEmpty,sum,isEmpty]
+              Project [ss_customer_sk,sr_return_quantity,ss_quantity,ss_sales_price]
+                BroadcastHashJoin [sr_reason_sk,r_reason_sk]
+                  Project [ss_customer_sk,ss_quantity,ss_sales_price,sr_reason_sk,sr_return_quantity]
+                    SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number]
+                      InputAdapter
+                        WholeStageCodegen (2)
+                          Sort [ss_item_sk,ss_ticket_number]
+                            InputAdapter
+                              Exchange [ss_item_sk,ss_ticket_number] #2
+                                WholeStageCodegen (1)
+                                  Project [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk]
+                      InputAdapter
+                        WholeStageCodegen (4)
+                          Sort [sr_item_sk,sr_ticket_number]
+                            InputAdapter
+                              Exchange [sr_item_sk,sr_ticket_number] #3
+                                WholeStageCodegen (3)
+                                  Project [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity]
+                                    Filter [sr_item_sk,sr_ticket_number,sr_reason_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.store_returns [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk]
+                  InputAdapter
+                    BroadcastExchange #4
+                      WholeStageCodegen (5)
+                        Project [r_reason_sk]
+                          Filter [r_reason_desc,r_reason_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.reason [r_reason_sk,r_reason_desc]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.sf100/explain.txt
new file mode 100644
index 0000000000000..e06516011e4ec
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.sf100/explain.txt
@@ -0,0 +1,260 @@
+== Physical Plan ==
+* Sort (46)
++- * HashAggregate (45)
+   +- Exchange (44)
+      +- * HashAggregate (43)
+         +- * HashAggregate (42)
+            +- * HashAggregate (41)
+               +- * Project (40)
+                  +- * BroadcastHashJoin Inner BuildRight (39)
+                     :- * Project (33)
+                     :  +- * BroadcastHashJoin Inner BuildRight (32)
+                     :     :- * Project (26)
+                     :     :  +- * BroadcastHashJoin Inner BuildRight (25)
+                     :     :     :- SortMergeJoin LeftAnti (19)
+                     :     :     :  :- * Project (13)
+                     :     :     :  :  +- * SortMergeJoin LeftSemi (12)
+                     :     :     :  :     :- * Sort (6)
+                     :     :     :  :     :  +- Exchange (5)
+                     :     :     :  :     :     +- * Project (4)
+                     :     :     :  :     :        +- * Filter (3)
+                     :     :     :  :     :           +- * ColumnarToRow (2)
+                     :     :     :  :     :              +- Scan parquet default.web_sales (1)
+                     :     :     :  :     +- * Sort (11)
+                     :     :     :  :        +- Exchange (10)
+                     :     :     :  :           +- * Project (9)
+                     :     :     :  :              +- * ColumnarToRow (8)
+                     :     :     :  :                 +- Scan parquet default.web_sales (7)
+                     :     :     :  +- * Sort (18)
+                     :     :     :     +- Exchange (17)
+                     :     :     :        +- * Project (16)
+                     :     :     :           +- * ColumnarToRow (15)
+                     :     :     :              +- Scan parquet default.web_returns (14)
+                     :     :     +- BroadcastExchange (24)
+                     :     :        +- * Project (23)
+                     :     :           +- * Filter (22)
+                     :     :              +- * ColumnarToRow (21)
+                     :     :                 +- Scan parquet default.customer_address (20)
+                     :     +- BroadcastExchange (31)
+                     :        +- * Project (30)
+                     :           +- * Filter (29)
+                     :              +- * ColumnarToRow (28)
+                     :                 +- Scan parquet default.web_site (27)
+                     +- BroadcastExchange (38)
+                        +- * Project (37)
+                           +- * Filter (36)
+                              +- * ColumnarToRow (35)
+                                 +- Scan parquet default.date_dim (34)
+
+
+(1) Scan parquet default.web_sales
+Output [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_sales]
+PushedFilters: [IsNotNull(ws_ship_date_sk), IsNotNull(ws_ship_addr_sk), IsNotNull(ws_web_site_sk)]
+ReadSchema: struct<ws_ship_date_sk:int,ws_ship_addr_sk:int,ws_web_site_sk:int,ws_warehouse_sk:int,ws_order_number:int,ws_ext_ship_cost:decimal(7,2),ws_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8]
+
+(3) Filter [codegen id : 1]
+Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8]
+Condition : ((isnotnull(ws_ship_date_sk#1) AND isnotnull(ws_ship_addr_sk#2)) AND isnotnull(ws_web_site_sk#3))
+
+(4) Project [codegen id : 1]
+Output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7]
+Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8]
+
+(5) Exchange
+Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7]
+Arguments: hashpartitioning(ws_order_number#5, 5), ENSURE_REQUIREMENTS, [id=#9]
+
+(6) Sort [codegen id : 2]
+Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7]
+Arguments: [ws_order_number#5 ASC NULLS FIRST], false, 0
+
+(7) Scan parquet default.web_sales
+Output [3]: [ws_warehouse_sk#10, ws_order_number#11, ws_sold_date_sk#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_sales]
+ReadSchema: struct<ws_warehouse_sk:int,ws_order_number:int>
+
+(8) ColumnarToRow [codegen id : 3]
+Input [3]: [ws_warehouse_sk#10, ws_order_number#11, ws_sold_date_sk#12]
+
+(9) Project [codegen id : 3]
+Output [2]: [ws_warehouse_sk#10, ws_order_number#11]
+Input [3]: [ws_warehouse_sk#10, ws_order_number#11, ws_sold_date_sk#12]
+
+(10) Exchange
+Input [2]: [ws_warehouse_sk#10, ws_order_number#11]
+Arguments: hashpartitioning(ws_order_number#11, 5), ENSURE_REQUIREMENTS, [id=#13]
+
+(11) Sort [codegen id : 4]
+Input [2]: [ws_warehouse_sk#10, ws_order_number#11]
+Arguments: [ws_order_number#11 ASC NULLS FIRST], false, 0
+
+(12) SortMergeJoin [codegen id : 5]
+Left keys [1]: [ws_order_number#5]
+Right keys [1]: [ws_order_number#11]
+Join condition: NOT (ws_warehouse_sk#4 = ws_warehouse_sk#10)
+
+(13) Project [codegen id : 5]
+Output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7]
+Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7]
+
+(14) Scan parquet default.web_returns
+Output [2]: [wr_order_number#14, wr_returned_date_sk#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_returns]
+ReadSchema: struct<wr_order_number:int>
+
+(15) ColumnarToRow [codegen id : 6]
+Input [2]: [wr_order_number#14, wr_returned_date_sk#15]
+
+(16) Project [codegen id : 6]
+Output [1]: [wr_order_number#14]
+Input [2]: [wr_order_number#14, wr_returned_date_sk#15]
+
+(17) Exchange
+Input [1]: [wr_order_number#14]
+Arguments: hashpartitioning(wr_order_number#14, 5), ENSURE_REQUIREMENTS, [id=#16]
+
+(18) Sort [codegen id : 7]
+Input [1]: [wr_order_number#14]
+Arguments: [wr_order_number#14 ASC NULLS FIRST], false, 0
+
+(19) SortMergeJoin
+Left keys [1]: [ws_order_number#5]
+Right keys [1]: [wr_order_number#14]
+Join condition: None
+
+(20) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#17, ca_state#18]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,IL), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(21) ColumnarToRow [codegen id : 8]
+Input [2]: [ca_address_sk#17, ca_state#18]
+
+(22) Filter [codegen id : 8]
+Input [2]: [ca_address_sk#17, ca_state#18]
+Condition : ((isnotnull(ca_state#18) AND (ca_state#18 = IL)) AND isnotnull(ca_address_sk#17))
+
+(23) Project [codegen id : 8]
+Output [1]: [ca_address_sk#17]
+Input [2]: [ca_address_sk#17, ca_state#18]
+
+(24) BroadcastExchange
+Input [1]: [ca_address_sk#17]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#19]
+
+(25) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ws_ship_addr_sk#2]
+Right keys [1]: [ca_address_sk#17]
+Join condition: None
+
+(26) Project [codegen id : 11]
+Output [5]: [ws_ship_date_sk#1, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7]
+Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ca_address_sk#17]
+
+(27) Scan parquet default.web_site
+Output [2]: [web_site_sk#20, web_company_name#21]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_site]
+PushedFilters: [IsNotNull(web_company_name), EqualTo(web_company_name,pri                                               ), IsNotNull(web_site_sk)]
+ReadSchema: struct<web_site_sk:int,web_company_name:string>
+
+(28) ColumnarToRow [codegen id : 9]
+Input [2]: [web_site_sk#20, web_company_name#21]
+
+(29) Filter [codegen id : 9]
+Input [2]: [web_site_sk#20, web_company_name#21]
+Condition : ((isnotnull(web_company_name#21) AND (web_company_name#21 = pri                                               )) AND isnotnull(web_site_sk#20))
+
+(30) Project [codegen id : 9]
+Output [1]: [web_site_sk#20]
+Input [2]: [web_site_sk#20, web_company_name#21]
+
+(31) BroadcastExchange
+Input [1]: [web_site_sk#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#22]
+
+(32) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ws_web_site_sk#3]
+Right keys [1]: [web_site_sk#20]
+Join condition: None
+
+(33) Project [codegen id : 11]
+Output [4]: [ws_ship_date_sk#1, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7]
+Input [6]: [ws_ship_date_sk#1, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, web_site_sk#20]
+
+(34) Scan parquet default.date_dim
+Output [2]: [d_date_sk#23, d_date#24]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-01), LessThanOrEqual(d_date,1999-04-02), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(35) ColumnarToRow [codegen id : 10]
+Input [2]: [d_date_sk#23, d_date#24]
+
+(36) Filter [codegen id : 10]
+Input [2]: [d_date_sk#23, d_date#24]
+Condition : (((isnotnull(d_date#24) AND (d_date#24 >= 1999-02-01)) AND (d_date#24 <= 1999-04-02)) AND isnotnull(d_date_sk#23))
+
+(37) Project [codegen id : 10]
+Output [1]: [d_date_sk#23]
+Input [2]: [d_date_sk#23, d_date#24]
+
+(38) BroadcastExchange
+Input [1]: [d_date_sk#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#25]
+
+(39) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ws_ship_date_sk#1]
+Right keys [1]: [d_date_sk#23]
+Join condition: None
+
+(40) Project [codegen id : 11]
+Output [3]: [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7]
+Input [5]: [ws_ship_date_sk#1, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, d_date_sk#23]
+
+(41) HashAggregate [codegen id : 11]
+Input [3]: [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7]
+Keys [1]: [ws_order_number#5]
+Functions [2]: [partial_sum(UnscaledValue(ws_ext_ship_cost#6)), partial_sum(UnscaledValue(ws_net_profit#7))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#6))#26, sum(UnscaledValue(ws_net_profit#7))#27]
+Results [3]: [ws_order_number#5, sum#28, sum#29]
+
+(42) HashAggregate [codegen id : 11]
+Input [3]: [ws_order_number#5, sum#28, sum#29]
+Keys [1]: [ws_order_number#5]
+Functions [2]: [merge_sum(UnscaledValue(ws_ext_ship_cost#6)), merge_sum(UnscaledValue(ws_net_profit#7))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#6))#26, sum(UnscaledValue(ws_net_profit#7))#27]
+Results [3]: [ws_order_number#5, sum#28, sum#29]
+
+(43) HashAggregate [codegen id : 11]
+Input [3]: [ws_order_number#5, sum#28, sum#29]
+Keys: []
+Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#6)), merge_sum(UnscaledValue(ws_net_profit#7)), partial_count(distinct ws_order_number#5)]
+Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#26, sum(UnscaledValue(ws_net_profit#7))#27, count(ws_order_number#5)#30]
+Results [3]: [sum#28, sum#29, count#31]
+
+(44) Exchange
+Input [3]: [sum#28, sum#29, count#31]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#32]
+
+(45) HashAggregate [codegen id : 12]
+Input [3]: [sum#28, sum#29, count#31]
+Keys: []
+Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#6)), sum(UnscaledValue(ws_net_profit#7)), count(distinct ws_order_number#5)]
+Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#26, sum(UnscaledValue(ws_net_profit#7))#27, count(ws_order_number#5)#30]
+Results [3]: [count(ws_order_number#5)#30 AS order count #33, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#6))#26,17,2) AS total shipping cost #34, MakeDecimal(sum(UnscaledValue(ws_net_profit#7))#27,17,2) AS total net profit #35]
+
+(46) Sort [codegen id : 12]
+Input [3]: [order count #33, total shipping cost #34, total net profit #35]
+Arguments: [order count #33 ASC NULLS FIRST], true, 0
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.sf100/simplified.txt
new file mode 100644
index 0000000000000..b3c313fb5ded6
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.sf100/simplified.txt
@@ -0,0 +1,74 @@
+WholeStageCodegen (12)
+  Sort [order count ]
+    HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count]
+      InputAdapter
+        Exchange #1
+          WholeStageCodegen (11)
+            HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count]
+              HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum]
+                HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum]
+                  Project [ws_order_number,ws_ext_ship_cost,ws_net_profit]
+                    BroadcastHashJoin [ws_ship_date_sk,d_date_sk]
+                      Project [ws_ship_date_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit]
+                        BroadcastHashJoin [ws_web_site_sk,web_site_sk]
+                          Project [ws_ship_date_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit]
+                            BroadcastHashJoin [ws_ship_addr_sk,ca_address_sk]
+                              InputAdapter
+                                SortMergeJoin [ws_order_number,wr_order_number]
+                                  WholeStageCodegen (5)
+                                    Project [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit]
+                                      SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk]
+                                        InputAdapter
+                                          WholeStageCodegen (2)
+                                            Sort [ws_order_number]
+                                              InputAdapter
+                                                Exchange [ws_order_number] #2
+                                                  WholeStageCodegen (1)
+                                                    Project [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit]
+                                                      Filter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk]
+                                        InputAdapter
+                                          WholeStageCodegen (4)
+                                            Sort [ws_order_number]
+                                              InputAdapter
+                                                Exchange [ws_order_number] #3
+                                                  WholeStageCodegen (3)
+                                                    Project [ws_warehouse_sk,ws_order_number]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk]
+                                  WholeStageCodegen (7)
+                                    Sort [wr_order_number]
+                                      InputAdapter
+                                        Exchange [wr_order_number] #4
+                                          WholeStageCodegen (6)
+                                            Project [wr_order_number]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.web_returns [wr_order_number,wr_returned_date_sk]
+                              InputAdapter
+                                BroadcastExchange #5
+                                  WholeStageCodegen (8)
+                                    Project [ca_address_sk]
+                                      Filter [ca_state,ca_address_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.customer_address [ca_address_sk,ca_state]
+                          InputAdapter
+                            BroadcastExchange #6
+                              WholeStageCodegen (9)
+                                Project [web_site_sk]
+                                  Filter [web_company_name,web_site_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.web_site [web_site_sk,web_company_name]
+                      InputAdapter
+                        BroadcastExchange #7
+                          WholeStageCodegen (10)
+                            Project [d_date_sk]
+                              Filter [d_date,d_date_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.date_dim [d_date_sk,d_date]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/explain.txt
new file mode 100644
index 0000000000000..2d8c00cc4c936
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/explain.txt
@@ -0,0 +1,260 @@
+== Physical Plan ==
+* Sort (46)
++- * HashAggregate (45)
+   +- Exchange (44)
+      +- * HashAggregate (43)
+         +- * HashAggregate (42)
+            +- * HashAggregate (41)
+               +- * Project (40)
+                  +- * BroadcastHashJoin Inner BuildRight (39)
+                     :- * Project (33)
+                     :  +- * BroadcastHashJoin Inner BuildRight (32)
+                     :     :- * Project (26)
+                     :     :  +- * BroadcastHashJoin Inner BuildRight (25)
+                     :     :     :- SortMergeJoin LeftAnti (19)
+                     :     :     :  :- * Project (13)
+                     :     :     :  :  +- * SortMergeJoin LeftSemi (12)
+                     :     :     :  :     :- * Sort (6)
+                     :     :     :  :     :  +- Exchange (5)
+                     :     :     :  :     :     +- * Project (4)
+                     :     :     :  :     :        +- * Filter (3)
+                     :     :     :  :     :           +- * ColumnarToRow (2)
+                     :     :     :  :     :              +- Scan parquet default.web_sales (1)
+                     :     :     :  :     +- * Sort (11)
+                     :     :     :  :        +- Exchange (10)
+                     :     :     :  :           +- * Project (9)
+                     :     :     :  :              +- * ColumnarToRow (8)
+                     :     :     :  :                 +- Scan parquet default.web_sales (7)
+                     :     :     :  +- * Sort (18)
+                     :     :     :     +- Exchange (17)
+                     :     :     :        +- * Project (16)
+                     :     :     :           +- * ColumnarToRow (15)
+                     :     :     :              +- Scan parquet default.web_returns (14)
+                     :     :     +- BroadcastExchange (24)
+                     :     :        +- * Project (23)
+                     :     :           +- * Filter (22)
+                     :     :              +- * ColumnarToRow (21)
+                     :     :                 +- Scan parquet default.date_dim (20)
+                     :     +- BroadcastExchange (31)
+                     :        +- * Project (30)
+                     :           +- * Filter (29)
+                     :              +- * ColumnarToRow (28)
+                     :                 +- Scan parquet default.customer_address (27)
+                     +- BroadcastExchange (38)
+                        +- * Project (37)
+                           +- * Filter (36)
+                              +- * ColumnarToRow (35)
+                                 +- Scan parquet default.web_site (34)
+
+
+(1) Scan parquet default.web_sales
+Output [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_sales]
+PushedFilters: [IsNotNull(ws_ship_date_sk), IsNotNull(ws_ship_addr_sk), IsNotNull(ws_web_site_sk)]
+ReadSchema: struct<ws_ship_date_sk:int,ws_ship_addr_sk:int,ws_web_site_sk:int,ws_warehouse_sk:int,ws_order_number:int,ws_ext_ship_cost:decimal(7,2),ws_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8]
+
+(3) Filter [codegen id : 1]
+Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8]
+Condition : ((isnotnull(ws_ship_date_sk#1) AND isnotnull(ws_ship_addr_sk#2)) AND isnotnull(ws_web_site_sk#3))
+
+(4) Project [codegen id : 1]
+Output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7]
+Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8]
+
+(5) Exchange
+Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7]
+Arguments: hashpartitioning(ws_order_number#5, 5), ENSURE_REQUIREMENTS, [id=#9]
+
+(6) Sort [codegen id : 2]
+Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7]
+Arguments: [ws_order_number#5 ASC NULLS FIRST], false, 0
+
+(7) Scan parquet default.web_sales
+Output [3]: [ws_warehouse_sk#10, ws_order_number#11, ws_sold_date_sk#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_sales]
+ReadSchema: struct<ws_warehouse_sk:int,ws_order_number:int>
+
+(8) ColumnarToRow [codegen id : 3]
+Input [3]: [ws_warehouse_sk#10, ws_order_number#11, ws_sold_date_sk#12]
+
+(9) Project [codegen id : 3]
+Output [2]: [ws_warehouse_sk#10, ws_order_number#11]
+Input [3]: [ws_warehouse_sk#10, ws_order_number#11, ws_sold_date_sk#12]
+
+(10) Exchange
+Input [2]: [ws_warehouse_sk#10, ws_order_number#11]
+Arguments: hashpartitioning(ws_order_number#11, 5), ENSURE_REQUIREMENTS, [id=#13]
+
+(11) Sort [codegen id : 4]
+Input [2]: [ws_warehouse_sk#10, ws_order_number#11]
+Arguments: [ws_order_number#11 ASC NULLS FIRST], false, 0
+
+(12) SortMergeJoin [codegen id : 5]
+Left keys [1]: [ws_order_number#5]
+Right keys [1]: [ws_order_number#11]
+Join condition: NOT (ws_warehouse_sk#4 = ws_warehouse_sk#10)
+
+(13) Project [codegen id : 5]
+Output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7]
+Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7]
+
+(14) Scan parquet default.web_returns
+Output [2]: [wr_order_number#14, wr_returned_date_sk#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_returns]
+ReadSchema: struct<wr_order_number:int>
+
+(15) ColumnarToRow [codegen id : 6]
+Input [2]: [wr_order_number#14, wr_returned_date_sk#15]
+
+(16) Project [codegen id : 6]
+Output [1]: [wr_order_number#14]
+Input [2]: [wr_order_number#14, wr_returned_date_sk#15]
+
+(17) Exchange
+Input [1]: [wr_order_number#14]
+Arguments: hashpartitioning(wr_order_number#14, 5), ENSURE_REQUIREMENTS, [id=#16]
+
+(18) Sort [codegen id : 7]
+Input [1]: [wr_order_number#14]
+Arguments: [wr_order_number#14 ASC NULLS FIRST], false, 0
+
+(19) SortMergeJoin
+Left keys [1]: [ws_order_number#5]
+Right keys [1]: [wr_order_number#14]
+Join condition: None
+
+(20) Scan parquet default.date_dim
+Output [2]: [d_date_sk#17, d_date#18]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-01), LessThanOrEqual(d_date,1999-04-02), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(21) ColumnarToRow [codegen id : 8]
+Input [2]: [d_date_sk#17, d_date#18]
+
+(22) Filter [codegen id : 8]
+Input [2]: [d_date_sk#17, d_date#18]
+Condition : (((isnotnull(d_date#18) AND (d_date#18 >= 1999-02-01)) AND (d_date#18 <= 1999-04-02)) AND isnotnull(d_date_sk#17))
+
+(23) Project [codegen id : 8]
+Output [1]: [d_date_sk#17]
+Input [2]: [d_date_sk#17, d_date#18]
+
+(24) BroadcastExchange
+Input [1]: [d_date_sk#17]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#19]
+
+(25) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ws_ship_date_sk#1]
+Right keys [1]: [d_date_sk#17]
+Join condition: None
+
+(26) Project [codegen id : 11]
+Output [5]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7]
+Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, d_date_sk#17]
+
+(27) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#20, ca_state#21]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,IL), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(28) ColumnarToRow [codegen id : 9]
+Input [2]: [ca_address_sk#20, ca_state#21]
+
+(29) Filter [codegen id : 9]
+Input [2]: [ca_address_sk#20, ca_state#21]
+Condition : ((isnotnull(ca_state#21) AND (ca_state#21 = IL)) AND isnotnull(ca_address_sk#20))
+
+(30) Project [codegen id : 9]
+Output [1]: [ca_address_sk#20]
+Input [2]: [ca_address_sk#20, ca_state#21]
+
+(31) BroadcastExchange
+Input [1]: [ca_address_sk#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#22]
+
+(32) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ws_ship_addr_sk#2]
+Right keys [1]: [ca_address_sk#20]
+Join condition: None
+
+(33) Project [codegen id : 11]
+Output [4]: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7]
+Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ca_address_sk#20]
+
+(34) Scan parquet default.web_site
+Output [2]: [web_site_sk#23, web_company_name#24]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_site]
+PushedFilters: [IsNotNull(web_company_name), EqualTo(web_company_name,pri                                               ), IsNotNull(web_site_sk)]
+ReadSchema: struct<web_site_sk:int,web_company_name:string>
+
+(35) ColumnarToRow [codegen id : 10]
+Input [2]: [web_site_sk#23, web_company_name#24]
+
+(36) Filter [codegen id : 10]
+Input [2]: [web_site_sk#23, web_company_name#24]
+Condition : ((isnotnull(web_company_name#24) AND (web_company_name#24 = pri                                               )) AND isnotnull(web_site_sk#23))
+
+(37) Project [codegen id : 10]
+Output [1]: [web_site_sk#23]
+Input [2]: [web_site_sk#23, web_company_name#24]
+
+(38) BroadcastExchange
+Input [1]: [web_site_sk#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#25]
+
+(39) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ws_web_site_sk#3]
+Right keys [1]: [web_site_sk#23]
+Join condition: None
+
+(40) Project [codegen id : 11]
+Output [3]: [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7]
+Input [5]: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, web_site_sk#23]
+
+(41) HashAggregate [codegen id : 11]
+Input [3]: [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7]
+Keys [1]: [ws_order_number#5]
+Functions [2]: [partial_sum(UnscaledValue(ws_ext_ship_cost#6)), partial_sum(UnscaledValue(ws_net_profit#7))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#6))#26, sum(UnscaledValue(ws_net_profit#7))#27]
+Results [3]: [ws_order_number#5, sum#28, sum#29]
+
+(42) HashAggregate [codegen id : 11]
+Input [3]: [ws_order_number#5, sum#28, sum#29]
+Keys [1]: [ws_order_number#5]
+Functions [2]: [merge_sum(UnscaledValue(ws_ext_ship_cost#6)), merge_sum(UnscaledValue(ws_net_profit#7))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#6))#26, sum(UnscaledValue(ws_net_profit#7))#27]
+Results [3]: [ws_order_number#5, sum#28, sum#29]
+
+(43) HashAggregate [codegen id : 11]
+Input [3]: [ws_order_number#5, sum#28, sum#29]
+Keys: []
+Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#6)), merge_sum(UnscaledValue(ws_net_profit#7)), partial_count(distinct ws_order_number#5)]
+Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#26, sum(UnscaledValue(ws_net_profit#7))#27, count(ws_order_number#5)#30]
+Results [3]: [sum#28, sum#29, count#31]
+
+(44) Exchange
+Input [3]: [sum#28, sum#29, count#31]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#32]
+
+(45) HashAggregate [codegen id : 12]
+Input [3]: [sum#28, sum#29, count#31]
+Keys: []
+Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#6)), sum(UnscaledValue(ws_net_profit#7)), count(distinct ws_order_number#5)]
+Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#26, sum(UnscaledValue(ws_net_profit#7))#27, count(ws_order_number#5)#30]
+Results [3]: [count(ws_order_number#5)#30 AS order count #33, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#6))#26,17,2) AS total shipping cost #34, MakeDecimal(sum(UnscaledValue(ws_net_profit#7))#27,17,2) AS total net profit #35]
+
+(46) Sort [codegen id : 12]
+Input [3]: [order count #33, total shipping cost #34, total net profit #35]
+Arguments: [order count #33 ASC NULLS FIRST], true, 0
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/simplified.txt
new file mode 100644
index 0000000000000..cecad61df0774
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/simplified.txt
@@ -0,0 +1,74 @@
+WholeStageCodegen (12)
+  Sort [order count ]
+    HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count]
+      InputAdapter
+        Exchange #1
+          WholeStageCodegen (11)
+            HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count]
+              HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum]
+                HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum]
+                  Project [ws_order_number,ws_ext_ship_cost,ws_net_profit]
+                    BroadcastHashJoin [ws_web_site_sk,web_site_sk]
+                      Project [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit]
+                        BroadcastHashJoin [ws_ship_addr_sk,ca_address_sk]
+                          Project [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit]
+                            BroadcastHashJoin [ws_ship_date_sk,d_date_sk]
+                              InputAdapter
+                                SortMergeJoin [ws_order_number,wr_order_number]
+                                  WholeStageCodegen (5)
+                                    Project [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit]
+                                      SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk]
+                                        InputAdapter
+                                          WholeStageCodegen (2)
+                                            Sort [ws_order_number]
+                                              InputAdapter
+                                                Exchange [ws_order_number] #2
+                                                  WholeStageCodegen (1)
+                                                    Project [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit]
+                                                      Filter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk]
+                                        InputAdapter
+                                          WholeStageCodegen (4)
+                                            Sort [ws_order_number]
+                                              InputAdapter
+                                                Exchange [ws_order_number] #3
+                                                  WholeStageCodegen (3)
+                                                    Project [ws_warehouse_sk,ws_order_number]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk]
+                                  WholeStageCodegen (7)
+                                    Sort [wr_order_number]
+                                      InputAdapter
+                                        Exchange [wr_order_number] #4
+                                          WholeStageCodegen (6)
+                                            Project [wr_order_number]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.web_returns [wr_order_number,wr_returned_date_sk]
+                              InputAdapter
+                                BroadcastExchange #5
+                                  WholeStageCodegen (8)
+                                    Project [d_date_sk]
+                                      Filter [d_date,d_date_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.date_dim [d_date_sk,d_date]
+                          InputAdapter
+                            BroadcastExchange #6
+                              WholeStageCodegen (9)
+                                Project [ca_address_sk]
+                                  Filter [ca_state,ca_address_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.customer_address [ca_address_sk,ca_state]
+                      InputAdapter
+                        BroadcastExchange #7
+                          WholeStageCodegen (10)
+                            Project [web_site_sk]
+                              Filter [web_company_name,web_site_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.web_site [web_site_sk,web_company_name]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.sf100/explain.txt
new file mode 100644
index 0000000000000..c9da298194fb8
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.sf100/explain.txt
@@ -0,0 +1,322 @@
+== Physical Plan ==
+* Sort (58)
++- * HashAggregate (57)
+   +- Exchange (56)
+      +- * HashAggregate (55)
+         +- * HashAggregate (54)
+            +- * HashAggregate (53)
+               +- * Project (52)
+                  +- * BroadcastHashJoin Inner BuildRight (51)
+                     :- * Project (45)
+                     :  +- * BroadcastHashJoin Inner BuildRight (44)
+                     :     :- * Project (38)
+                     :     :  +- * BroadcastHashJoin Inner BuildRight (37)
+                     :     :     :- * SortMergeJoin LeftSemi (31)
+                     :     :     :  :- * SortMergeJoin LeftSemi (17)
+                     :     :     :  :  :- * Sort (6)
+                     :     :     :  :  :  +- Exchange (5)
+                     :     :     :  :  :     +- * Project (4)
+                     :     :     :  :  :        +- * Filter (3)
+                     :     :     :  :  :           +- * ColumnarToRow (2)
+                     :     :     :  :  :              +- Scan parquet default.web_sales (1)
+                     :     :     :  :  +- * Project (16)
+                     :     :     :  :     +- * SortMergeJoin Inner (15)
+                     :     :     :  :        :- * Sort (12)
+                     :     :     :  :        :  +- Exchange (11)
+                     :     :     :  :        :     +- * Project (10)
+                     :     :     :  :        :        +- * Filter (9)
+                     :     :     :  :        :           +- * ColumnarToRow (8)
+                     :     :     :  :        :              +- Scan parquet default.web_sales (7)
+                     :     :     :  :        +- * Sort (14)
+                     :     :     :  :           +- ReusedExchange (13)
+                     :     :     :  +- * Project (30)
+                     :     :     :     +- * SortMergeJoin Inner (29)
+                     :     :     :        :- * SortMergeJoin Inner (26)
+                     :     :     :        :  :- * Sort (23)
+                     :     :     :        :  :  +- Exchange (22)
+                     :     :     :        :  :     +- * Project (21)
+                     :     :     :        :  :        +- * Filter (20)
+                     :     :     :        :  :           +- * ColumnarToRow (19)
+                     :     :     :        :  :              +- Scan parquet default.web_returns (18)
+                     :     :     :        :  +- * Sort (25)
+                     :     :     :        :     +- ReusedExchange (24)
+                     :     :     :        +- * Sort (28)
+                     :     :     :           +- ReusedExchange (27)
+                     :     :     +- BroadcastExchange (36)
+                     :     :        +- * Project (35)
+                     :     :           +- * Filter (34)
+                     :     :              +- * ColumnarToRow (33)
+                     :     :                 +- Scan parquet default.customer_address (32)
+                     :     +- BroadcastExchange (43)
+                     :        +- * Project (42)
+                     :           +- * Filter (41)
+                     :              +- * ColumnarToRow (40)
+                     :                 +- Scan parquet default.web_site (39)
+                     +- BroadcastExchange (50)
+                        +- * Project (49)
+                           +- * Filter (48)
+                              +- * ColumnarToRow (47)
+                                 +- Scan parquet default.date_dim (46)
+
+
+(1) Scan parquet default.web_sales
+Output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_sales]
+PushedFilters: [IsNotNull(ws_ship_date_sk), IsNotNull(ws_ship_addr_sk), IsNotNull(ws_web_site_sk)]
+ReadSchema: struct<ws_ship_date_sk:int,ws_ship_addr_sk:int,ws_web_site_sk:int,ws_order_number:int,ws_ext_ship_cost:decimal(7,2),ws_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7]
+
+(3) Filter [codegen id : 1]
+Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7]
+Condition : ((isnotnull(ws_ship_date_sk#1) AND isnotnull(ws_ship_addr_sk#2)) AND isnotnull(ws_web_site_sk#3))
+
+(4) Project [codegen id : 1]
+Output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6]
+Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7]
+
+(5) Exchange
+Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6]
+Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, [id=#8]
+
+(6) Sort [codegen id : 2]
+Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6]
+Arguments: [ws_order_number#4 ASC NULLS FIRST], false, 0
+
+(7) Scan parquet default.web_sales
+Output [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_sales]
+PushedFilters: [IsNotNull(ws_order_number), IsNotNull(ws_warehouse_sk)]
+ReadSchema: struct<ws_warehouse_sk:int,ws_order_number:int>
+
+(8) ColumnarToRow [codegen id : 3]
+Input [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11]
+
+(9) Filter [codegen id : 3]
+Input [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11]
+Condition : (isnotnull(ws_order_number#10) AND isnotnull(ws_warehouse_sk#9))
+
+(10) Project [codegen id : 3]
+Output [2]: [ws_warehouse_sk#9, ws_order_number#10]
+Input [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11]
+
+(11) Exchange
+Input [2]: [ws_warehouse_sk#9, ws_order_number#10]
+Arguments: hashpartitioning(ws_order_number#10, 5), ENSURE_REQUIREMENTS, [id=#12]
+
+(12) Sort [codegen id : 4]
+Input [2]: [ws_warehouse_sk#9, ws_order_number#10]
+Arguments: [ws_order_number#10 ASC NULLS FIRST], false, 0
+
+(13) ReusedExchange [Reuses operator id: 11]
+Output [2]: [ws_warehouse_sk#13, ws_order_number#14]
+
+(14) Sort [codegen id : 6]
+Input [2]: [ws_warehouse_sk#13, ws_order_number#14]
+Arguments: [ws_order_number#14 ASC NULLS FIRST], false, 0
+
+(15) SortMergeJoin [codegen id : 7]
+Left keys [1]: [ws_order_number#10]
+Right keys [1]: [ws_order_number#14]
+Join condition: NOT (ws_warehouse_sk#9 = ws_warehouse_sk#13)
+
+(16) Project [codegen id : 7]
+Output [1]: [ws_order_number#10]
+Input [4]: [ws_warehouse_sk#9, ws_order_number#10, ws_warehouse_sk#13, ws_order_number#14]
+
+(17) SortMergeJoin [codegen id : 8]
+Left keys [1]: [ws_order_number#4]
+Right keys [1]: [ws_order_number#10]
+Join condition: None
+
+(18) Scan parquet default.web_returns
+Output [2]: [wr_order_number#15, wr_returned_date_sk#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_returns]
+PushedFilters: [IsNotNull(wr_order_number)]
+ReadSchema: struct<wr_order_number:int>
+
+(19) ColumnarToRow [codegen id : 9]
+Input [2]: [wr_order_number#15, wr_returned_date_sk#16]
+
+(20) Filter [codegen id : 9]
+Input [2]: [wr_order_number#15, wr_returned_date_sk#16]
+Condition : isnotnull(wr_order_number#15)
+
+(21) Project [codegen id : 9]
+Output [1]: [wr_order_number#15]
+Input [2]: [wr_order_number#15, wr_returned_date_sk#16]
+
+(22) Exchange
+Input [1]: [wr_order_number#15]
+Arguments: hashpartitioning(wr_order_number#15, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(23) Sort [codegen id : 10]
+Input [1]: [wr_order_number#15]
+Arguments: [wr_order_number#15 ASC NULLS FIRST], false, 0
+
+(24) ReusedExchange [Reuses operator id: 11]
+Output [2]: [ws_warehouse_sk#18, ws_order_number#19]
+
+(25) Sort [codegen id : 12]
+Input [2]: [ws_warehouse_sk#18, ws_order_number#19]
+Arguments: [ws_order_number#19 ASC NULLS FIRST], false, 0
+
+(26) SortMergeJoin [codegen id : 13]
+Left keys [1]: [wr_order_number#15]
+Right keys [1]: [ws_order_number#19]
+Join condition: None
+
+(27) ReusedExchange [Reuses operator id: 11]
+Output [2]: [ws_warehouse_sk#20, ws_order_number#21]
+
+(28) Sort [codegen id : 15]
+Input [2]: [ws_warehouse_sk#20, ws_order_number#21]
+Arguments: [ws_order_number#21 ASC NULLS FIRST], false, 0
+
+(29) SortMergeJoin [codegen id : 16]
+Left keys [1]: [ws_order_number#19]
+Right keys [1]: [ws_order_number#21]
+Join condition: NOT (ws_warehouse_sk#18 = ws_warehouse_sk#20)
+
+(30) Project [codegen id : 16]
+Output [1]: [wr_order_number#15]
+Input [5]: [wr_order_number#15, ws_warehouse_sk#18, ws_order_number#19, ws_warehouse_sk#20, ws_order_number#21]
+
+(31) SortMergeJoin [codegen id : 20]
+Left keys [1]: [ws_order_number#4]
+Right keys [1]: [wr_order_number#15]
+Join condition: None
+
+(32) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#22, ca_state#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,IL), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(33) ColumnarToRow [codegen id : 17]
+Input [2]: [ca_address_sk#22, ca_state#23]
+
+(34) Filter [codegen id : 17]
+Input [2]: [ca_address_sk#22, ca_state#23]
+Condition : ((isnotnull(ca_state#23) AND (ca_state#23 = IL)) AND isnotnull(ca_address_sk#22))
+
+(35) Project [codegen id : 17]
+Output [1]: [ca_address_sk#22]
+Input [2]: [ca_address_sk#22, ca_state#23]
+
+(36) BroadcastExchange
+Input [1]: [ca_address_sk#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#24]
+
+(37) BroadcastHashJoin [codegen id : 20]
+Left keys [1]: [ws_ship_addr_sk#2]
+Right keys [1]: [ca_address_sk#22]
+Join condition: None
+
+(38) Project [codegen id : 20]
+Output [5]: [ws_ship_date_sk#1, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6]
+Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ca_address_sk#22]
+
+(39) Scan parquet default.web_site
+Output [2]: [web_site_sk#25, web_company_name#26]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_site]
+PushedFilters: [IsNotNull(web_company_name), EqualTo(web_company_name,pri                                               ), IsNotNull(web_site_sk)]
+ReadSchema: struct<web_site_sk:int,web_company_name:string>
+
+(40) ColumnarToRow [codegen id : 18]
+Input [2]: [web_site_sk#25, web_company_name#26]
+
+(41) Filter [codegen id : 18]
+Input [2]: [web_site_sk#25, web_company_name#26]
+Condition : ((isnotnull(web_company_name#26) AND (web_company_name#26 = pri                                               )) AND isnotnull(web_site_sk#25))
+
+(42) Project [codegen id : 18]
+Output [1]: [web_site_sk#25]
+Input [2]: [web_site_sk#25, web_company_name#26]
+
+(43) BroadcastExchange
+Input [1]: [web_site_sk#25]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#27]
+
+(44) BroadcastHashJoin [codegen id : 20]
+Left keys [1]: [ws_web_site_sk#3]
+Right keys [1]: [web_site_sk#25]
+Join condition: None
+
+(45) Project [codegen id : 20]
+Output [4]: [ws_ship_date_sk#1, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6]
+Input [6]: [ws_ship_date_sk#1, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, web_site_sk#25]
+
+(46) Scan parquet default.date_dim
+Output [2]: [d_date_sk#28, d_date#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-01), LessThanOrEqual(d_date,1999-04-02), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(47) ColumnarToRow [codegen id : 19]
+Input [2]: [d_date_sk#28, d_date#29]
+
+(48) Filter [codegen id : 19]
+Input [2]: [d_date_sk#28, d_date#29]
+Condition : (((isnotnull(d_date#29) AND (d_date#29 >= 1999-02-01)) AND (d_date#29 <= 1999-04-02)) AND isnotnull(d_date_sk#28))
+
+(49) Project [codegen id : 19]
+Output [1]: [d_date_sk#28]
+Input [2]: [d_date_sk#28, d_date#29]
+
+(50) BroadcastExchange
+Input [1]: [d_date_sk#28]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#30]
+
+(51) BroadcastHashJoin [codegen id : 20]
+Left keys [1]: [ws_ship_date_sk#1]
+Right keys [1]: [d_date_sk#28]
+Join condition: None
+
+(52) Project [codegen id : 20]
+Output [3]: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6]
+Input [5]: [ws_ship_date_sk#1, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, d_date_sk#28]
+
+(53) HashAggregate [codegen id : 20]
+Input [3]: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6]
+Keys [1]: [ws_order_number#4]
+Functions [2]: [partial_sum(UnscaledValue(ws_ext_ship_cost#5)), partial_sum(UnscaledValue(ws_net_profit#6))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#31, sum(UnscaledValue(ws_net_profit#6))#32]
+Results [3]: [ws_order_number#4, sum#33, sum#34]
+
+(54) HashAggregate [codegen id : 20]
+Input [3]: [ws_order_number#4, sum#33, sum#34]
+Keys [1]: [ws_order_number#4]
+Functions [2]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(UnscaledValue(ws_net_profit#6))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#31, sum(UnscaledValue(ws_net_profit#6))#32]
+Results [3]: [ws_order_number#4, sum#33, sum#34]
+
+(55) HashAggregate [codegen id : 20]
+Input [3]: [ws_order_number#4, sum#33, sum#34]
+Keys: []
+Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(UnscaledValue(ws_net_profit#6)), partial_count(distinct ws_order_number#4)]
+Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#31, sum(UnscaledValue(ws_net_profit#6))#32, count(ws_order_number#4)#35]
+Results [3]: [sum#33, sum#34, count#36]
+
+(56) Exchange
+Input [3]: [sum#33, sum#34, count#36]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#37]
+
+(57) HashAggregate [codegen id : 21]
+Input [3]: [sum#33, sum#34, count#36]
+Keys: []
+Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#5)), sum(UnscaledValue(ws_net_profit#6)), count(distinct ws_order_number#4)]
+Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#31, sum(UnscaledValue(ws_net_profit#6))#32, count(ws_order_number#4)#35]
+Results [3]: [count(ws_order_number#4)#35 AS order count #38, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#5))#31,17,2) AS total shipping cost #39, MakeDecimal(sum(UnscaledValue(ws_net_profit#6))#32,17,2) AS total net profit #40]
+
+(58) Sort [codegen id : 21]
+Input [3]: [order count #38, total shipping cost #39, total net profit #40]
+Arguments: [order count #38 ASC NULLS FIRST], true, 0
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.sf100/simplified.txt
new file mode 100644
index 0000000000000..28d164854629b
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.sf100/simplified.txt
@@ -0,0 +1,102 @@
+WholeStageCodegen (21)
+  Sort [order count ]
+    HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count]
+      InputAdapter
+        Exchange #1
+          WholeStageCodegen (20)
+            HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count]
+              HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum]
+                HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum]
+                  Project [ws_order_number,ws_ext_ship_cost,ws_net_profit]
+                    BroadcastHashJoin [ws_ship_date_sk,d_date_sk]
+                      Project [ws_ship_date_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit]
+                        BroadcastHashJoin [ws_web_site_sk,web_site_sk]
+                          Project [ws_ship_date_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit]
+                            BroadcastHashJoin [ws_ship_addr_sk,ca_address_sk]
+                              SortMergeJoin [ws_order_number,wr_order_number]
+                                InputAdapter
+                                  WholeStageCodegen (8)
+                                    SortMergeJoin [ws_order_number,ws_order_number]
+                                      InputAdapter
+                                        WholeStageCodegen (2)
+                                          Sort [ws_order_number]
+                                            InputAdapter
+                                              Exchange [ws_order_number] #2
+                                                WholeStageCodegen (1)
+                                                  Project [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit]
+                                                    Filter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk]
+                                      InputAdapter
+                                        WholeStageCodegen (7)
+                                          Project [ws_order_number]
+                                            SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk]
+                                              InputAdapter
+                                                WholeStageCodegen (4)
+                                                  Sort [ws_order_number]
+                                                    InputAdapter
+                                                      Exchange [ws_order_number] #3
+                                                        WholeStageCodegen (3)
+                                                          Project [ws_warehouse_sk,ws_order_number]
+                                                            Filter [ws_order_number,ws_warehouse_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk]
+                                              InputAdapter
+                                                WholeStageCodegen (6)
+                                                  Sort [ws_order_number]
+                                                    InputAdapter
+                                                      ReusedExchange [ws_warehouse_sk,ws_order_number] #3
+                                InputAdapter
+                                  WholeStageCodegen (16)
+                                    Project [wr_order_number]
+                                      SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk]
+                                        InputAdapter
+                                          WholeStageCodegen (13)
+                                            SortMergeJoin [wr_order_number,ws_order_number]
+                                              InputAdapter
+                                                WholeStageCodegen (10)
+                                                  Sort [wr_order_number]
+                                                    InputAdapter
+                                                      Exchange [wr_order_number] #4
+                                                        WholeStageCodegen (9)
+                                                          Project [wr_order_number]
+                                                            Filter [wr_order_number]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.web_returns [wr_order_number,wr_returned_date_sk]
+                                              InputAdapter
+                                                WholeStageCodegen (12)
+                                                  Sort [ws_order_number]
+                                                    InputAdapter
+                                                      ReusedExchange [ws_warehouse_sk,ws_order_number] #3
+                                        InputAdapter
+                                          WholeStageCodegen (15)
+                                            Sort [ws_order_number]
+                                              InputAdapter
+                                                ReusedExchange [ws_warehouse_sk,ws_order_number] #3
+                              InputAdapter
+                                BroadcastExchange #5
+                                  WholeStageCodegen (17)
+                                    Project [ca_address_sk]
+                                      Filter [ca_state,ca_address_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.customer_address [ca_address_sk,ca_state]
+                          InputAdapter
+                            BroadcastExchange #6
+                              WholeStageCodegen (18)
+                                Project [web_site_sk]
+                                  Filter [web_company_name,web_site_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.web_site [web_site_sk,web_company_name]
+                      InputAdapter
+                        BroadcastExchange #7
+                          WholeStageCodegen (19)
+                            Project [d_date_sk]
+                              Filter [d_date,d_date_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.date_dim [d_date_sk,d_date]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt
new file mode 100644
index 0000000000000..253132c88e2c5
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt
@@ -0,0 +1,327 @@
+== Physical Plan ==
+* Sort (59)
++- * HashAggregate (58)
+   +- Exchange (57)
+      +- * HashAggregate (56)
+         +- * HashAggregate (55)
+            +- * HashAggregate (54)
+               +- * Project (53)
+                  +- * BroadcastHashJoin Inner BuildRight (52)
+                     :- * Project (46)
+                     :  +- * BroadcastHashJoin Inner BuildRight (45)
+                     :     :- * Project (39)
+                     :     :  +- * BroadcastHashJoin Inner BuildRight (38)
+                     :     :     :- * SortMergeJoin LeftSemi (32)
+                     :     :     :  :- * SortMergeJoin LeftSemi (17)
+                     :     :     :  :  :- * Sort (6)
+                     :     :     :  :  :  +- Exchange (5)
+                     :     :     :  :  :     +- * Project (4)
+                     :     :     :  :  :        +- * Filter (3)
+                     :     :     :  :  :           +- * ColumnarToRow (2)
+                     :     :     :  :  :              +- Scan parquet default.web_sales (1)
+                     :     :     :  :  +- * Project (16)
+                     :     :     :  :     +- * SortMergeJoin Inner (15)
+                     :     :     :  :        :- * Sort (12)
+                     :     :     :  :        :  +- Exchange (11)
+                     :     :     :  :        :     +- * Project (10)
+                     :     :     :  :        :        +- * Filter (9)
+                     :     :     :  :        :           +- * ColumnarToRow (8)
+                     :     :     :  :        :              +- Scan parquet default.web_sales (7)
+                     :     :     :  :        +- * Sort (14)
+                     :     :     :  :           +- ReusedExchange (13)
+                     :     :     :  +- * Project (31)
+                     :     :     :     +- * SortMergeJoin Inner (30)
+                     :     :     :        :- * Sort (23)
+                     :     :     :        :  +- Exchange (22)
+                     :     :     :        :     +- * Project (21)
+                     :     :     :        :        +- * Filter (20)
+                     :     :     :        :           +- * ColumnarToRow (19)
+                     :     :     :        :              +- Scan parquet default.web_returns (18)
+                     :     :     :        +- * Project (29)
+                     :     :     :           +- * SortMergeJoin Inner (28)
+                     :     :     :              :- * Sort (25)
+                     :     :     :              :  +- ReusedExchange (24)
+                     :     :     :              +- * Sort (27)
+                     :     :     :                 +- ReusedExchange (26)
+                     :     :     +- BroadcastExchange (37)
+                     :     :        +- * Project (36)
+                     :     :           +- * Filter (35)
+                     :     :              +- * ColumnarToRow (34)
+                     :     :                 +- Scan parquet default.date_dim (33)
+                     :     +- BroadcastExchange (44)
+                     :        +- * Project (43)
+                     :           +- * Filter (42)
+                     :              +- * ColumnarToRow (41)
+                     :                 +- Scan parquet default.customer_address (40)
+                     +- BroadcastExchange (51)
+                        +- * Project (50)
+                           +- * Filter (49)
+                              +- * ColumnarToRow (48)
+                                 +- Scan parquet default.web_site (47)
+
+
+(1) Scan parquet default.web_sales
+Output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_sales]
+PushedFilters: [IsNotNull(ws_ship_date_sk), IsNotNull(ws_ship_addr_sk), IsNotNull(ws_web_site_sk)]
+ReadSchema: struct<ws_ship_date_sk:int,ws_ship_addr_sk:int,ws_web_site_sk:int,ws_order_number:int,ws_ext_ship_cost:decimal(7,2),ws_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7]
+
+(3) Filter [codegen id : 1]
+Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7]
+Condition : ((isnotnull(ws_ship_date_sk#1) AND isnotnull(ws_ship_addr_sk#2)) AND isnotnull(ws_web_site_sk#3))
+
+(4) Project [codegen id : 1]
+Output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6]
+Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7]
+
+(5) Exchange
+Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6]
+Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, [id=#8]
+
+(6) Sort [codegen id : 2]
+Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6]
+Arguments: [ws_order_number#4 ASC NULLS FIRST], false, 0
+
+(7) Scan parquet default.web_sales
+Output [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_sales]
+PushedFilters: [IsNotNull(ws_order_number), IsNotNull(ws_warehouse_sk)]
+ReadSchema: struct<ws_warehouse_sk:int,ws_order_number:int>
+
+(8) ColumnarToRow [codegen id : 3]
+Input [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11]
+
+(9) Filter [codegen id : 3]
+Input [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11]
+Condition : (isnotnull(ws_order_number#10) AND isnotnull(ws_warehouse_sk#9))
+
+(10) Project [codegen id : 3]
+Output [2]: [ws_warehouse_sk#9, ws_order_number#10]
+Input [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11]
+
+(11) Exchange
+Input [2]: [ws_warehouse_sk#9, ws_order_number#10]
+Arguments: hashpartitioning(ws_order_number#10, 5), ENSURE_REQUIREMENTS, [id=#12]
+
+(12) Sort [codegen id : 4]
+Input [2]: [ws_warehouse_sk#9, ws_order_number#10]
+Arguments: [ws_order_number#10 ASC NULLS FIRST], false, 0
+
+(13) ReusedExchange [Reuses operator id: 11]
+Output [2]: [ws_warehouse_sk#13, ws_order_number#14]
+
+(14) Sort [codegen id : 6]
+Input [2]: [ws_warehouse_sk#13, ws_order_number#14]
+Arguments: [ws_order_number#14 ASC NULLS FIRST], false, 0
+
+(15) SortMergeJoin [codegen id : 7]
+Left keys [1]: [ws_order_number#10]
+Right keys [1]: [ws_order_number#14]
+Join condition: NOT (ws_warehouse_sk#9 = ws_warehouse_sk#13)
+
+(16) Project [codegen id : 7]
+Output [1]: [ws_order_number#10]
+Input [4]: [ws_warehouse_sk#9, ws_order_number#10, ws_warehouse_sk#13, ws_order_number#14]
+
+(17) SortMergeJoin [codegen id : 8]
+Left keys [1]: [ws_order_number#4]
+Right keys [1]: [ws_order_number#10]
+Join condition: None
+
+(18) Scan parquet default.web_returns
+Output [2]: [wr_order_number#15, wr_returned_date_sk#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_returns]
+PushedFilters: [IsNotNull(wr_order_number)]
+ReadSchema: struct<wr_order_number:int>
+
+(19) ColumnarToRow [codegen id : 9]
+Input [2]: [wr_order_number#15, wr_returned_date_sk#16]
+
+(20) Filter [codegen id : 9]
+Input [2]: [wr_order_number#15, wr_returned_date_sk#16]
+Condition : isnotnull(wr_order_number#15)
+
+(21) Project [codegen id : 9]
+Output [1]: [wr_order_number#15]
+Input [2]: [wr_order_number#15, wr_returned_date_sk#16]
+
+(22) Exchange
+Input [1]: [wr_order_number#15]
+Arguments: hashpartitioning(wr_order_number#15, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(23) Sort [codegen id : 10]
+Input [1]: [wr_order_number#15]
+Arguments: [wr_order_number#15 ASC NULLS FIRST], false, 0
+
+(24) ReusedExchange [Reuses operator id: 11]
+Output [2]: [ws_warehouse_sk#18, ws_order_number#19]
+
+(25) Sort [codegen id : 12]
+Input [2]: [ws_warehouse_sk#18, ws_order_number#19]
+Arguments: [ws_order_number#19 ASC NULLS FIRST], false, 0
+
+(26) ReusedExchange [Reuses operator id: 11]
+Output [2]: [ws_warehouse_sk#20, ws_order_number#21]
+
+(27) Sort [codegen id : 14]
+Input [2]: [ws_warehouse_sk#20, ws_order_number#21]
+Arguments: [ws_order_number#21 ASC NULLS FIRST], false, 0
+
+(28) SortMergeJoin [codegen id : 15]
+Left keys [1]: [ws_order_number#19]
+Right keys [1]: [ws_order_number#21]
+Join condition: NOT (ws_warehouse_sk#18 = ws_warehouse_sk#20)
+
+(29) Project [codegen id : 15]
+Output [1]: [ws_order_number#19]
+Input [4]: [ws_warehouse_sk#18, ws_order_number#19, ws_warehouse_sk#20, ws_order_number#21]
+
+(30) SortMergeJoin [codegen id : 16]
+Left keys [1]: [wr_order_number#15]
+Right keys [1]: [ws_order_number#19]
+Join condition: None
+
+(31) Project [codegen id : 16]
+Output [1]: [wr_order_number#15]
+Input [2]: [wr_order_number#15, ws_order_number#19]
+
+(32) SortMergeJoin [codegen id : 20]
+Left keys [1]: [ws_order_number#4]
+Right keys [1]: [wr_order_number#15]
+Join condition: None
+
+(33) Scan parquet default.date_dim
+Output [2]: [d_date_sk#22, d_date#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-01), LessThanOrEqual(d_date,1999-04-02), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(34) ColumnarToRow [codegen id : 17]
+Input [2]: [d_date_sk#22, d_date#23]
+
+(35) Filter [codegen id : 17]
+Input [2]: [d_date_sk#22, d_date#23]
+Condition : (((isnotnull(d_date#23) AND (d_date#23 >= 1999-02-01)) AND (d_date#23 <= 1999-04-02)) AND isnotnull(d_date_sk#22))
+
+(36) Project [codegen id : 17]
+Output [1]: [d_date_sk#22]
+Input [2]: [d_date_sk#22, d_date#23]
+
+(37) BroadcastExchange
+Input [1]: [d_date_sk#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#24]
+
+(38) BroadcastHashJoin [codegen id : 20]
+Left keys [1]: [ws_ship_date_sk#1]
+Right keys [1]: [d_date_sk#22]
+Join condition: None
+
+(39) Project [codegen id : 20]
+Output [5]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6]
+Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, d_date_sk#22]
+
+(40) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#25, ca_state#26]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,IL), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(41) ColumnarToRow [codegen id : 18]
+Input [2]: [ca_address_sk#25, ca_state#26]
+
+(42) Filter [codegen id : 18]
+Input [2]: [ca_address_sk#25, ca_state#26]
+Condition : ((isnotnull(ca_state#26) AND (ca_state#26 = IL)) AND isnotnull(ca_address_sk#25))
+
+(43) Project [codegen id : 18]
+Output [1]: [ca_address_sk#25]
+Input [2]: [ca_address_sk#25, ca_state#26]
+
+(44) BroadcastExchange
+Input [1]: [ca_address_sk#25]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#27]
+
+(45) BroadcastHashJoin [codegen id : 20]
+Left keys [1]: [ws_ship_addr_sk#2]
+Right keys [1]: [ca_address_sk#25]
+Join condition: None
+
+(46) Project [codegen id : 20]
+Output [4]: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6]
+Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ca_address_sk#25]
+
+(47) Scan parquet default.web_site
+Output [2]: [web_site_sk#28, web_company_name#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_site]
+PushedFilters: [IsNotNull(web_company_name), EqualTo(web_company_name,pri                                               ), IsNotNull(web_site_sk)]
+ReadSchema: struct<web_site_sk:int,web_company_name:string>
+
+(48) ColumnarToRow [codegen id : 19]
+Input [2]: [web_site_sk#28, web_company_name#29]
+
+(49) Filter [codegen id : 19]
+Input [2]: [web_site_sk#28, web_company_name#29]
+Condition : ((isnotnull(web_company_name#29) AND (web_company_name#29 = pri                                               )) AND isnotnull(web_site_sk#28))
+
+(50) Project [codegen id : 19]
+Output [1]: [web_site_sk#28]
+Input [2]: [web_site_sk#28, web_company_name#29]
+
+(51) BroadcastExchange
+Input [1]: [web_site_sk#28]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#30]
+
+(52) BroadcastHashJoin [codegen id : 20]
+Left keys [1]: [ws_web_site_sk#3]
+Right keys [1]: [web_site_sk#28]
+Join condition: None
+
+(53) Project [codegen id : 20]
+Output [3]: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6]
+Input [5]: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, web_site_sk#28]
+
+(54) HashAggregate [codegen id : 20]
+Input [3]: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6]
+Keys [1]: [ws_order_number#4]
+Functions [2]: [partial_sum(UnscaledValue(ws_ext_ship_cost#5)), partial_sum(UnscaledValue(ws_net_profit#6))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#31, sum(UnscaledValue(ws_net_profit#6))#32]
+Results [3]: [ws_order_number#4, sum#33, sum#34]
+
+(55) HashAggregate [codegen id : 20]
+Input [3]: [ws_order_number#4, sum#33, sum#34]
+Keys [1]: [ws_order_number#4]
+Functions [2]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(UnscaledValue(ws_net_profit#6))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#31, sum(UnscaledValue(ws_net_profit#6))#32]
+Results [3]: [ws_order_number#4, sum#33, sum#34]
+
+(56) HashAggregate [codegen id : 20]
+Input [3]: [ws_order_number#4, sum#33, sum#34]
+Keys: []
+Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(UnscaledValue(ws_net_profit#6)), partial_count(distinct ws_order_number#4)]
+Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#31, sum(UnscaledValue(ws_net_profit#6))#32, count(ws_order_number#4)#35]
+Results [3]: [sum#33, sum#34, count#36]
+
+(57) Exchange
+Input [3]: [sum#33, sum#34, count#36]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#37]
+
+(58) HashAggregate [codegen id : 21]
+Input [3]: [sum#33, sum#34, count#36]
+Keys: []
+Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#5)), sum(UnscaledValue(ws_net_profit#6)), count(distinct ws_order_number#4)]
+Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#31, sum(UnscaledValue(ws_net_profit#6))#32, count(ws_order_number#4)#35]
+Results [3]: [count(ws_order_number#4)#35 AS order count #38, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#5))#31,17,2) AS total shipping cost #39, MakeDecimal(sum(UnscaledValue(ws_net_profit#6))#32,17,2) AS total net profit #40]
+
+(59) Sort [codegen id : 21]
+Input [3]: [order count #38, total shipping cost #39, total net profit #40]
+Arguments: [order count #38 ASC NULLS FIRST], true, 0
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/simplified.txt
new file mode 100644
index 0000000000000..4d40e31545a21
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/simplified.txt
@@ -0,0 +1,103 @@
+WholeStageCodegen (21)
+  Sort [order count ]
+    HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count]
+      InputAdapter
+        Exchange #1
+          WholeStageCodegen (20)
+            HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count]
+              HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum]
+                HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum]
+                  Project [ws_order_number,ws_ext_ship_cost,ws_net_profit]
+                    BroadcastHashJoin [ws_web_site_sk,web_site_sk]
+                      Project [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit]
+                        BroadcastHashJoin [ws_ship_addr_sk,ca_address_sk]
+                          Project [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit]
+                            BroadcastHashJoin [ws_ship_date_sk,d_date_sk]
+                              SortMergeJoin [ws_order_number,wr_order_number]
+                                InputAdapter
+                                  WholeStageCodegen (8)
+                                    SortMergeJoin [ws_order_number,ws_order_number]
+                                      InputAdapter
+                                        WholeStageCodegen (2)
+                                          Sort [ws_order_number]
+                                            InputAdapter
+                                              Exchange [ws_order_number] #2
+                                                WholeStageCodegen (1)
+                                                  Project [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit]
+                                                    Filter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk]
+                                      InputAdapter
+                                        WholeStageCodegen (7)
+                                          Project [ws_order_number]
+                                            SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk]
+                                              InputAdapter
+                                                WholeStageCodegen (4)
+                                                  Sort [ws_order_number]
+                                                    InputAdapter
+                                                      Exchange [ws_order_number] #3
+                                                        WholeStageCodegen (3)
+                                                          Project [ws_warehouse_sk,ws_order_number]
+                                                            Filter [ws_order_number,ws_warehouse_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk]
+                                              InputAdapter
+                                                WholeStageCodegen (6)
+                                                  Sort [ws_order_number]
+                                                    InputAdapter
+                                                      ReusedExchange [ws_warehouse_sk,ws_order_number] #3
+                                InputAdapter
+                                  WholeStageCodegen (16)
+                                    Project [wr_order_number]
+                                      SortMergeJoin [wr_order_number,ws_order_number]
+                                        InputAdapter
+                                          WholeStageCodegen (10)
+                                            Sort [wr_order_number]
+                                              InputAdapter
+                                                Exchange [wr_order_number] #4
+                                                  WholeStageCodegen (9)
+                                                    Project [wr_order_number]
+                                                      Filter [wr_order_number]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.web_returns [wr_order_number,wr_returned_date_sk]
+                                        InputAdapter
+                                          WholeStageCodegen (15)
+                                            Project [ws_order_number]
+                                              SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk]
+                                                InputAdapter
+                                                  WholeStageCodegen (12)
+                                                    Sort [ws_order_number]
+                                                      InputAdapter
+                                                        ReusedExchange [ws_warehouse_sk,ws_order_number] #3
+                                                InputAdapter
+                                                  WholeStageCodegen (14)
+                                                    Sort [ws_order_number]
+                                                      InputAdapter
+                                                        ReusedExchange [ws_warehouse_sk,ws_order_number] #3
+                              InputAdapter
+                                BroadcastExchange #5
+                                  WholeStageCodegen (17)
+                                    Project [d_date_sk]
+                                      Filter [d_date,d_date_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.date_dim [d_date_sk,d_date]
+                          InputAdapter
+                            BroadcastExchange #6
+                              WholeStageCodegen (18)
+                                Project [ca_address_sk]
+                                  Filter [ca_state,ca_address_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.customer_address [ca_address_sk,ca_state]
+                      InputAdapter
+                        BroadcastExchange #7
+                          WholeStageCodegen (19)
+                            Project [web_site_sk]
+                              Filter [web_company_name,web_site_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.web_site [web_site_sk,web_company_name]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.sf100/explain.txt
new file mode 100644
index 0000000000000..fdf792d409716
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.sf100/explain.txt
@@ -0,0 +1,165 @@
+== Physical Plan ==
+* Sort (29)
++- * HashAggregate (28)
+   +- Exchange (27)
+      +- * HashAggregate (26)
+         +- * Project (25)
+            +- * BroadcastHashJoin Inner BuildRight (24)
+               :- * Project (18)
+               :  +- * BroadcastHashJoin Inner BuildRight (17)
+               :     :- * Project (11)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (10)
+               :     :     :- * Project (4)
+               :     :     :  +- * Filter (3)
+               :     :     :     +- * ColumnarToRow (2)
+               :     :     :        +- Scan parquet default.store_sales (1)
+               :     :     +- BroadcastExchange (9)
+               :     :        +- * Project (8)
+               :     :           +- * Filter (7)
+               :     :              +- * ColumnarToRow (6)
+               :     :                 +- Scan parquet default.time_dim (5)
+               :     +- BroadcastExchange (16)
+               :        +- * Project (15)
+               :           +- * Filter (14)
+               :              +- * ColumnarToRow (13)
+               :                 +- Scan parquet default.store (12)
+               +- BroadcastExchange (23)
+                  +- * Project (22)
+                     +- * Filter (21)
+                        +- * ColumnarToRow (20)
+                           +- Scan parquet default.household_demographics (19)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_sold_time_sk:int,ss_hdemo_sk:int,ss_store_sk:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 4]
+Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4]
+Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isnotnull(ss_store_sk#3))
+
+(4) Project [codegen id : 4]
+Output [3]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3]
+Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4]
+
+(5) Scan parquet default.time_dim
+Output [3]: [t_time_sk#5, t_hour#6, t_minute#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,20), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_hour:int,t_minute:int>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [3]: [t_time_sk#5, t_hour#6, t_minute#7]
+
+(7) Filter [codegen id : 1]
+Input [3]: [t_time_sk#5, t_hour#6, t_minute#7]
+Condition : ((((isnotnull(t_hour#6) AND isnotnull(t_minute#7)) AND (t_hour#6 = 20)) AND (t_minute#7 >= 30)) AND isnotnull(t_time_sk#5))
+
+(8) Project [codegen id : 1]
+Output [1]: [t_time_sk#5]
+Input [3]: [t_time_sk#5, t_hour#6, t_minute#7]
+
+(9) BroadcastExchange
+Input [1]: [t_time_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#8]
+
+(10) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_time_sk#1]
+Right keys [1]: [t_time_sk#5]
+Join condition: None
+
+(11) Project [codegen id : 4]
+Output [2]: [ss_hdemo_sk#2, ss_store_sk#3]
+Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, t_time_sk#5]
+
+(12) Scan parquet default.store
+Output [2]: [s_store_sk#9, s_store_name#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_name), EqualTo(s_store_name,ese), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string>
+
+(13) ColumnarToRow [codegen id : 2]
+Input [2]: [s_store_sk#9, s_store_name#10]
+
+(14) Filter [codegen id : 2]
+Input [2]: [s_store_sk#9, s_store_name#10]
+Condition : ((isnotnull(s_store_name#10) AND (s_store_name#10 = ese)) AND isnotnull(s_store_sk#9))
+
+(15) Project [codegen id : 2]
+Output [1]: [s_store_sk#9]
+Input [2]: [s_store_sk#9, s_store_name#10]
+
+(16) BroadcastExchange
+Input [1]: [s_store_sk#9]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#11]
+
+(17) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#9]
+Join condition: None
+
+(18) Project [codegen id : 4]
+Output [1]: [ss_hdemo_sk#2]
+Input [3]: [ss_hdemo_sk#2, ss_store_sk#3, s_store_sk#9]
+
+(19) Scan parquet default.household_demographics
+Output [2]: [hd_demo_sk#12, hd_dep_count#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_dep_count), EqualTo(hd_dep_count,7), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_dep_count:int>
+
+(20) ColumnarToRow [codegen id : 3]
+Input [2]: [hd_demo_sk#12, hd_dep_count#13]
+
+(21) Filter [codegen id : 3]
+Input [2]: [hd_demo_sk#12, hd_dep_count#13]
+Condition : ((isnotnull(hd_dep_count#13) AND (hd_dep_count#13 = 7)) AND isnotnull(hd_demo_sk#12))
+
+(22) Project [codegen id : 3]
+Output [1]: [hd_demo_sk#12]
+Input [2]: [hd_demo_sk#12, hd_dep_count#13]
+
+(23) BroadcastExchange
+Input [1]: [hd_demo_sk#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(24) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#12]
+Join condition: None
+
+(25) Project [codegen id : 4]
+Output: []
+Input [2]: [ss_hdemo_sk#2, hd_demo_sk#12]
+
+(26) HashAggregate [codegen id : 4]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#15]
+Results [1]: [count#16]
+
+(27) Exchange
+Input [1]: [count#16]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#17]
+
+(28) HashAggregate [codegen id : 5]
+Input [1]: [count#16]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#18]
+Results [1]: [count(1)#18 AS count(1)#19]
+
+(29) Sort [codegen id : 5]
+Input [1]: [count(1)#19]
+Arguments: [count(1)#19 ASC NULLS FIRST], true, 0
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.sf100/simplified.txt
new file mode 100644
index 0000000000000..365c608815766
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.sf100/simplified.txt
@@ -0,0 +1,42 @@
+WholeStageCodegen (5)
+  Sort [count(1)]
+    HashAggregate [count] [count(1),count(1),count]
+      InputAdapter
+        Exchange #1
+          WholeStageCodegen (4)
+            HashAggregate [count,count]
+              Project
+                BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                  Project [ss_hdemo_sk]
+                    BroadcastHashJoin [ss_store_sk,s_store_sk]
+                      Project [ss_hdemo_sk,ss_store_sk]
+                        BroadcastHashJoin [ss_sold_time_sk,t_time_sk]
+                          Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk]
+                            Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk]
+                          InputAdapter
+                            BroadcastExchange #2
+                              WholeStageCodegen (1)
+                                Project [t_time_sk]
+                                  Filter [t_hour,t_minute,t_time_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.time_dim [t_time_sk,t_hour,t_minute]
+                      InputAdapter
+                        BroadcastExchange #3
+                          WholeStageCodegen (2)
+                            Project [s_store_sk]
+                              Filter [s_store_name,s_store_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.store [s_store_sk,s_store_name]
+                  InputAdapter
+                    BroadcastExchange #4
+                      WholeStageCodegen (3)
+                        Project [hd_demo_sk]
+                          Filter [hd_dep_count,hd_demo_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.household_demographics [hd_demo_sk,hd_dep_count]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/explain.txt
new file mode 100644
index 0000000000000..80ef7641eef1c
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/explain.txt
@@ -0,0 +1,165 @@
+== Physical Plan ==
+* Sort (29)
++- * HashAggregate (28)
+   +- Exchange (27)
+      +- * HashAggregate (26)
+         +- * Project (25)
+            +- * BroadcastHashJoin Inner BuildRight (24)
+               :- * Project (18)
+               :  +- * BroadcastHashJoin Inner BuildRight (17)
+               :     :- * Project (11)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (10)
+               :     :     :- * Project (4)
+               :     :     :  +- * Filter (3)
+               :     :     :     +- * ColumnarToRow (2)
+               :     :     :        +- Scan parquet default.store_sales (1)
+               :     :     +- BroadcastExchange (9)
+               :     :        +- * Project (8)
+               :     :           +- * Filter (7)
+               :     :              +- * ColumnarToRow (6)
+               :     :                 +- Scan parquet default.household_demographics (5)
+               :     +- BroadcastExchange (16)
+               :        +- * Project (15)
+               :           +- * Filter (14)
+               :              +- * ColumnarToRow (13)
+               :                 +- Scan parquet default.time_dim (12)
+               +- BroadcastExchange (23)
+                  +- * Project (22)
+                     +- * Filter (21)
+                        +- * ColumnarToRow (20)
+                           +- Scan parquet default.store (19)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_sold_time_sk:int,ss_hdemo_sk:int,ss_store_sk:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 4]
+Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4]
+Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isnotnull(ss_store_sk#3))
+
+(4) Project [codegen id : 4]
+Output [3]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3]
+Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4]
+
+(5) Scan parquet default.household_demographics
+Output [2]: [hd_demo_sk#5, hd_dep_count#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_dep_count), EqualTo(hd_dep_count,7), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_dep_count:int>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [2]: [hd_demo_sk#5, hd_dep_count#6]
+
+(7) Filter [codegen id : 1]
+Input [2]: [hd_demo_sk#5, hd_dep_count#6]
+Condition : ((isnotnull(hd_dep_count#6) AND (hd_dep_count#6 = 7)) AND isnotnull(hd_demo_sk#5))
+
+(8) Project [codegen id : 1]
+Output [1]: [hd_demo_sk#5]
+Input [2]: [hd_demo_sk#5, hd_dep_count#6]
+
+(9) BroadcastExchange
+Input [1]: [hd_demo_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7]
+
+(10) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#5]
+Join condition: None
+
+(11) Project [codegen id : 4]
+Output [2]: [ss_sold_time_sk#1, ss_store_sk#3]
+Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5]
+
+(12) Scan parquet default.time_dim
+Output [3]: [t_time_sk#8, t_hour#9, t_minute#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/time_dim]
+PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,20), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)]
+ReadSchema: struct<t_time_sk:int,t_hour:int,t_minute:int>
+
+(13) ColumnarToRow [codegen id : 2]
+Input [3]: [t_time_sk#8, t_hour#9, t_minute#10]
+
+(14) Filter [codegen id : 2]
+Input [3]: [t_time_sk#8, t_hour#9, t_minute#10]
+Condition : ((((isnotnull(t_hour#9) AND isnotnull(t_minute#10)) AND (t_hour#9 = 20)) AND (t_minute#10 >= 30)) AND isnotnull(t_time_sk#8))
+
+(15) Project [codegen id : 2]
+Output [1]: [t_time_sk#8]
+Input [3]: [t_time_sk#8, t_hour#9, t_minute#10]
+
+(16) BroadcastExchange
+Input [1]: [t_time_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#11]
+
+(17) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_time_sk#1]
+Right keys [1]: [t_time_sk#8]
+Join condition: None
+
+(18) Project [codegen id : 4]
+Output [1]: [ss_store_sk#3]
+Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#8]
+
+(19) Scan parquet default.store
+Output [2]: [s_store_sk#12, s_store_name#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_name), EqualTo(s_store_name,ese), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string>
+
+(20) ColumnarToRow [codegen id : 3]
+Input [2]: [s_store_sk#12, s_store_name#13]
+
+(21) Filter [codegen id : 3]
+Input [2]: [s_store_sk#12, s_store_name#13]
+Condition : ((isnotnull(s_store_name#13) AND (s_store_name#13 = ese)) AND isnotnull(s_store_sk#12))
+
+(22) Project [codegen id : 3]
+Output [1]: [s_store_sk#12]
+Input [2]: [s_store_sk#12, s_store_name#13]
+
+(23) BroadcastExchange
+Input [1]: [s_store_sk#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(24) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#12]
+Join condition: None
+
+(25) Project [codegen id : 4]
+Output: []
+Input [2]: [ss_store_sk#3, s_store_sk#12]
+
+(26) HashAggregate [codegen id : 4]
+Input: []
+Keys: []
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#15]
+Results [1]: [count#16]
+
+(27) Exchange
+Input [1]: [count#16]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#17]
+
+(28) HashAggregate [codegen id : 5]
+Input [1]: [count#16]
+Keys: []
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#18]
+Results [1]: [count(1)#18 AS count(1)#19]
+
+(29) Sort [codegen id : 5]
+Input [1]: [count(1)#19]
+Arguments: [count(1)#19 ASC NULLS FIRST], true, 0
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/simplified.txt
new file mode 100644
index 0000000000000..d783eb06cbc9c
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/simplified.txt
@@ -0,0 +1,42 @@
+WholeStageCodegen (5)
+  Sort [count(1)]
+    HashAggregate [count] [count(1),count(1),count]
+      InputAdapter
+        Exchange #1
+          WholeStageCodegen (4)
+            HashAggregate [count,count]
+              Project
+                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                  Project [ss_store_sk]
+                    BroadcastHashJoin [ss_sold_time_sk,t_time_sk]
+                      Project [ss_sold_time_sk,ss_store_sk]
+                        BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                          Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk]
+                            Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk]
+                          InputAdapter
+                            BroadcastExchange #2
+                              WholeStageCodegen (1)
+                                Project [hd_demo_sk]
+                                  Filter [hd_dep_count,hd_demo_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.household_demographics [hd_demo_sk,hd_dep_count]
+                      InputAdapter
+                        BroadcastExchange #3
+                          WholeStageCodegen (2)
+                            Project [t_time_sk]
+                              Filter [t_hour,t_minute,t_time_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.time_dim [t_time_sk,t_hour,t_minute]
+                  InputAdapter
+                    BroadcastExchange #4
+                      WholeStageCodegen (3)
+                        Project [s_store_sk]
+                          Filter [s_store_name,s_store_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.store [s_store_sk,s_store_name]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.sf100/explain.txt
new file mode 100644
index 0000000000000..c01f3465ed693
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.sf100/explain.txt
@@ -0,0 +1,176 @@
+== Physical Plan ==
+* HashAggregate (27)
++- Exchange (26)
+   +- * HashAggregate (25)
+      +- * Project (24)
+         +- SortMergeJoin FullOuter (23)
+            :- * Sort (13)
+            :  +- * HashAggregate (12)
+            :     +- Exchange (11)
+            :        +- * HashAggregate (10)
+            :           +- * Project (9)
+            :              +- * BroadcastHashJoin Inner BuildRight (8)
+            :                 :- * ColumnarToRow (2)
+            :                 :  +- Scan parquet default.store_sales (1)
+            :                 +- BroadcastExchange (7)
+            :                    +- * Project (6)
+            :                       +- * Filter (5)
+            :                          +- * ColumnarToRow (4)
+            :                             +- Scan parquet default.date_dim (3)
+            +- * Sort (22)
+               +- * HashAggregate (21)
+                  +- Exchange (20)
+                     +- * HashAggregate (19)
+                        +- * Project (18)
+                           +- * BroadcastHashJoin Inner BuildRight (17)
+                              :- * ColumnarToRow (15)
+                              :  +- Scan parquet default.catalog_sales (14)
+                              +- ReusedExchange (16)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3]
+
+(3) Scan parquet default.date_dim
+Output [2]: [d_date_sk#5, d_month_seq#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(4) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+
+(5) Filter [codegen id : 1]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5))
+
+(6) Project [codegen id : 1]
+Output [1]: [d_date_sk#5]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+
+(7) BroadcastExchange
+Input [1]: [d_date_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7]
+
+(8) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#5]
+Join condition: None
+
+(9) Project [codegen id : 2]
+Output [2]: [ss_item_sk#1, ss_customer_sk#2]
+Input [4]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3, d_date_sk#5]
+
+(10) HashAggregate [codegen id : 2]
+Input [2]: [ss_item_sk#1, ss_customer_sk#2]
+Keys [2]: [ss_customer_sk#2, ss_item_sk#1]
+Functions: []
+Aggregate Attributes: []
+Results [2]: [ss_customer_sk#2, ss_item_sk#1]
+
+(11) Exchange
+Input [2]: [ss_customer_sk#2, ss_item_sk#1]
+Arguments: hashpartitioning(ss_customer_sk#2, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#8]
+
+(12) HashAggregate [codegen id : 3]
+Input [2]: [ss_customer_sk#2, ss_item_sk#1]
+Keys [2]: [ss_customer_sk#2, ss_item_sk#1]
+Functions: []
+Aggregate Attributes: []
+Results [2]: [ss_customer_sk#2 AS customer_sk#9, ss_item_sk#1 AS item_sk#10]
+
+(13) Sort [codegen id : 3]
+Input [2]: [customer_sk#9, item_sk#10]
+Arguments: [customer_sk#9 ASC NULLS FIRST, item_sk#10 ASC NULLS FIRST], false, 0
+
+(14) Scan parquet default.catalog_sales
+Output [3]: [cs_bill_customer_sk#11, cs_item_sk#12, cs_sold_date_sk#13]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#13), dynamicpruningexpression(cs_sold_date_sk#13 IN dynamicpruning#4)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_item_sk:int>
+
+(15) ColumnarToRow [codegen id : 5]
+Input [3]: [cs_bill_customer_sk#11, cs_item_sk#12, cs_sold_date_sk#13]
+
+(16) ReusedExchange [Reuses operator id: 7]
+Output [1]: [d_date_sk#14]
+
+(17) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [cs_sold_date_sk#13]
+Right keys [1]: [d_date_sk#14]
+Join condition: None
+
+(18) Project [codegen id : 5]
+Output [2]: [cs_bill_customer_sk#11, cs_item_sk#12]
+Input [4]: [cs_bill_customer_sk#11, cs_item_sk#12, cs_sold_date_sk#13, d_date_sk#14]
+
+(19) HashAggregate [codegen id : 5]
+Input [2]: [cs_bill_customer_sk#11, cs_item_sk#12]
+Keys [2]: [cs_bill_customer_sk#11, cs_item_sk#12]
+Functions: []
+Aggregate Attributes: []
+Results [2]: [cs_bill_customer_sk#11, cs_item_sk#12]
+
+(20) Exchange
+Input [2]: [cs_bill_customer_sk#11, cs_item_sk#12]
+Arguments: hashpartitioning(cs_bill_customer_sk#11, cs_item_sk#12, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(21) HashAggregate [codegen id : 6]
+Input [2]: [cs_bill_customer_sk#11, cs_item_sk#12]
+Keys [2]: [cs_bill_customer_sk#11, cs_item_sk#12]
+Functions: []
+Aggregate Attributes: []
+Results [2]: [cs_bill_customer_sk#11 AS customer_sk#16, cs_item_sk#12 AS item_sk#17]
+
+(22) Sort [codegen id : 6]
+Input [2]: [customer_sk#16, item_sk#17]
+Arguments: [customer_sk#16 ASC NULLS FIRST, item_sk#17 ASC NULLS FIRST], false, 0
+
+(23) SortMergeJoin
+Left keys [2]: [customer_sk#9, item_sk#10]
+Right keys [2]: [customer_sk#16, item_sk#17]
+Join condition: None
+
+(24) Project [codegen id : 7]
+Output [2]: [customer_sk#9, customer_sk#16]
+Input [4]: [customer_sk#9, item_sk#10, customer_sk#16, item_sk#17]
+
+(25) HashAggregate [codegen id : 7]
+Input [2]: [customer_sk#9, customer_sk#16]
+Keys: []
+Functions [3]: [partial_sum(CASE WHEN (isnotnull(customer_sk#9) AND isnull(customer_sk#16)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnull(customer_sk#9) AND isnotnull(customer_sk#16)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnotnull(customer_sk#9) AND isnotnull(customer_sk#16)) THEN 1 ELSE 0 END)]
+Aggregate Attributes [3]: [sum#18, sum#19, sum#20]
+Results [3]: [sum#21, sum#22, sum#23]
+
+(26) Exchange
+Input [3]: [sum#21, sum#22, sum#23]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#24]
+
+(27) HashAggregate [codegen id : 8]
+Input [3]: [sum#21, sum#22, sum#23]
+Keys: []
+Functions [3]: [sum(CASE WHEN (isnotnull(customer_sk#9) AND isnull(customer_sk#16)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnull(customer_sk#9) AND isnotnull(customer_sk#16)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnotnull(customer_sk#9) AND isnotnull(customer_sk#16)) THEN 1 ELSE 0 END)]
+Aggregate Attributes [3]: [sum(CASE WHEN (isnotnull(customer_sk#9) AND isnull(customer_sk#16)) THEN 1 ELSE 0 END)#25, sum(CASE WHEN (isnull(customer_sk#9) AND isnotnull(customer_sk#16)) THEN 1 ELSE 0 END)#26, sum(CASE WHEN (isnotnull(customer_sk#9) AND isnotnull(customer_sk#16)) THEN 1 ELSE 0 END)#27]
+Results [3]: [sum(CASE WHEN (isnotnull(customer_sk#9) AND isnull(customer_sk#16)) THEN 1 ELSE 0 END)#25 AS store_only#28, sum(CASE WHEN (isnull(customer_sk#9) AND isnotnull(customer_sk#16)) THEN 1 ELSE 0 END)#26 AS catalog_only#29, sum(CASE WHEN (isnotnull(customer_sk#9) AND isnotnull(customer_sk#16)) THEN 1 ELSE 0 END)#27 AS store_and_catalog#30]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (28)
+
+
+(28) ReusedExchange [Reuses operator id: 7]
+Output [1]: [d_date_sk#5]
+
+Subquery:2 Hosting operator id = 14 Hosting Expression = cs_sold_date_sk#13 IN dynamicpruning#4
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.sf100/simplified.txt
new file mode 100644
index 0000000000000..5aba214f237d7
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.sf100/simplified.txt
@@ -0,0 +1,46 @@
+WholeStageCodegen (8)
+  HashAggregate [sum,sum,sum] [sum(CASE WHEN (isnotnull(customer_sk) AND isnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnotnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),store_only,catalog_only,store_and_catalog,sum,sum,sum]
+    InputAdapter
+      Exchange #1
+        WholeStageCodegen (7)
+          HashAggregate [customer_sk,customer_sk] [sum,sum,sum,sum,sum,sum]
+            Project [customer_sk,customer_sk]
+              InputAdapter
+                SortMergeJoin [customer_sk,item_sk,customer_sk,item_sk]
+                  WholeStageCodegen (3)
+                    Sort [customer_sk,item_sk]
+                      HashAggregate [ss_customer_sk,ss_item_sk] [customer_sk,item_sk]
+                        InputAdapter
+                          Exchange [ss_customer_sk,ss_item_sk] #2
+                            WholeStageCodegen (2)
+                              HashAggregate [ss_customer_sk,ss_item_sk]
+                                Project [ss_item_sk,ss_customer_sk]
+                                  BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk]
+                                          SubqueryBroadcast [d_date_sk] #1
+                                            ReusedExchange [d_date_sk] #3
+                                    InputAdapter
+                                      BroadcastExchange #3
+                                        WholeStageCodegen (1)
+                                          Project [d_date_sk]
+                                            Filter [d_month_seq,d_date_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.date_dim [d_date_sk,d_month_seq]
+                  WholeStageCodegen (6)
+                    Sort [customer_sk,item_sk]
+                      HashAggregate [cs_bill_customer_sk,cs_item_sk] [customer_sk,item_sk]
+                        InputAdapter
+                          Exchange [cs_bill_customer_sk,cs_item_sk] #4
+                            WholeStageCodegen (5)
+                              HashAggregate [cs_bill_customer_sk,cs_item_sk]
+                                Project [cs_bill_customer_sk,cs_item_sk]
+                                  BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk]
+                                          ReusedSubquery [d_date_sk] #1
+                                    InputAdapter
+                                      ReusedExchange [d_date_sk] #3
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/explain.txt
new file mode 100644
index 0000000000000..c01f3465ed693
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/explain.txt
@@ -0,0 +1,176 @@
+== Physical Plan ==
+* HashAggregate (27)
++- Exchange (26)
+   +- * HashAggregate (25)
+      +- * Project (24)
+         +- SortMergeJoin FullOuter (23)
+            :- * Sort (13)
+            :  +- * HashAggregate (12)
+            :     +- Exchange (11)
+            :        +- * HashAggregate (10)
+            :           +- * Project (9)
+            :              +- * BroadcastHashJoin Inner BuildRight (8)
+            :                 :- * ColumnarToRow (2)
+            :                 :  +- Scan parquet default.store_sales (1)
+            :                 +- BroadcastExchange (7)
+            :                    +- * Project (6)
+            :                       +- * Filter (5)
+            :                          +- * ColumnarToRow (4)
+            :                             +- Scan parquet default.date_dim (3)
+            +- * Sort (22)
+               +- * HashAggregate (21)
+                  +- Exchange (20)
+                     +- * HashAggregate (19)
+                        +- * Project (18)
+                           +- * BroadcastHashJoin Inner BuildRight (17)
+                              :- * ColumnarToRow (15)
+                              :  +- Scan parquet default.catalog_sales (14)
+                              +- ReusedExchange (16)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3]
+
+(3) Scan parquet default.date_dim
+Output [2]: [d_date_sk#5, d_month_seq#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(4) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+
+(5) Filter [codegen id : 1]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5))
+
+(6) Project [codegen id : 1]
+Output [1]: [d_date_sk#5]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+
+(7) BroadcastExchange
+Input [1]: [d_date_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7]
+
+(8) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#5]
+Join condition: None
+
+(9) Project [codegen id : 2]
+Output [2]: [ss_item_sk#1, ss_customer_sk#2]
+Input [4]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3, d_date_sk#5]
+
+(10) HashAggregate [codegen id : 2]
+Input [2]: [ss_item_sk#1, ss_customer_sk#2]
+Keys [2]: [ss_customer_sk#2, ss_item_sk#1]
+Functions: []
+Aggregate Attributes: []
+Results [2]: [ss_customer_sk#2, ss_item_sk#1]
+
+(11) Exchange
+Input [2]: [ss_customer_sk#2, ss_item_sk#1]
+Arguments: hashpartitioning(ss_customer_sk#2, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#8]
+
+(12) HashAggregate [codegen id : 3]
+Input [2]: [ss_customer_sk#2, ss_item_sk#1]
+Keys [2]: [ss_customer_sk#2, ss_item_sk#1]
+Functions: []
+Aggregate Attributes: []
+Results [2]: [ss_customer_sk#2 AS customer_sk#9, ss_item_sk#1 AS item_sk#10]
+
+(13) Sort [codegen id : 3]
+Input [2]: [customer_sk#9, item_sk#10]
+Arguments: [customer_sk#9 ASC NULLS FIRST, item_sk#10 ASC NULLS FIRST], false, 0
+
+(14) Scan parquet default.catalog_sales
+Output [3]: [cs_bill_customer_sk#11, cs_item_sk#12, cs_sold_date_sk#13]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#13), dynamicpruningexpression(cs_sold_date_sk#13 IN dynamicpruning#4)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_item_sk:int>
+
+(15) ColumnarToRow [codegen id : 5]
+Input [3]: [cs_bill_customer_sk#11, cs_item_sk#12, cs_sold_date_sk#13]
+
+(16) ReusedExchange [Reuses operator id: 7]
+Output [1]: [d_date_sk#14]
+
+(17) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [cs_sold_date_sk#13]
+Right keys [1]: [d_date_sk#14]
+Join condition: None
+
+(18) Project [codegen id : 5]
+Output [2]: [cs_bill_customer_sk#11, cs_item_sk#12]
+Input [4]: [cs_bill_customer_sk#11, cs_item_sk#12, cs_sold_date_sk#13, d_date_sk#14]
+
+(19) HashAggregate [codegen id : 5]
+Input [2]: [cs_bill_customer_sk#11, cs_item_sk#12]
+Keys [2]: [cs_bill_customer_sk#11, cs_item_sk#12]
+Functions: []
+Aggregate Attributes: []
+Results [2]: [cs_bill_customer_sk#11, cs_item_sk#12]
+
+(20) Exchange
+Input [2]: [cs_bill_customer_sk#11, cs_item_sk#12]
+Arguments: hashpartitioning(cs_bill_customer_sk#11, cs_item_sk#12, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(21) HashAggregate [codegen id : 6]
+Input [2]: [cs_bill_customer_sk#11, cs_item_sk#12]
+Keys [2]: [cs_bill_customer_sk#11, cs_item_sk#12]
+Functions: []
+Aggregate Attributes: []
+Results [2]: [cs_bill_customer_sk#11 AS customer_sk#16, cs_item_sk#12 AS item_sk#17]
+
+(22) Sort [codegen id : 6]
+Input [2]: [customer_sk#16, item_sk#17]
+Arguments: [customer_sk#16 ASC NULLS FIRST, item_sk#17 ASC NULLS FIRST], false, 0
+
+(23) SortMergeJoin
+Left keys [2]: [customer_sk#9, item_sk#10]
+Right keys [2]: [customer_sk#16, item_sk#17]
+Join condition: None
+
+(24) Project [codegen id : 7]
+Output [2]: [customer_sk#9, customer_sk#16]
+Input [4]: [customer_sk#9, item_sk#10, customer_sk#16, item_sk#17]
+
+(25) HashAggregate [codegen id : 7]
+Input [2]: [customer_sk#9, customer_sk#16]
+Keys: []
+Functions [3]: [partial_sum(CASE WHEN (isnotnull(customer_sk#9) AND isnull(customer_sk#16)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnull(customer_sk#9) AND isnotnull(customer_sk#16)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnotnull(customer_sk#9) AND isnotnull(customer_sk#16)) THEN 1 ELSE 0 END)]
+Aggregate Attributes [3]: [sum#18, sum#19, sum#20]
+Results [3]: [sum#21, sum#22, sum#23]
+
+(26) Exchange
+Input [3]: [sum#21, sum#22, sum#23]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#24]
+
+(27) HashAggregate [codegen id : 8]
+Input [3]: [sum#21, sum#22, sum#23]
+Keys: []
+Functions [3]: [sum(CASE WHEN (isnotnull(customer_sk#9) AND isnull(customer_sk#16)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnull(customer_sk#9) AND isnotnull(customer_sk#16)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnotnull(customer_sk#9) AND isnotnull(customer_sk#16)) THEN 1 ELSE 0 END)]
+Aggregate Attributes [3]: [sum(CASE WHEN (isnotnull(customer_sk#9) AND isnull(customer_sk#16)) THEN 1 ELSE 0 END)#25, sum(CASE WHEN (isnull(customer_sk#9) AND isnotnull(customer_sk#16)) THEN 1 ELSE 0 END)#26, sum(CASE WHEN (isnotnull(customer_sk#9) AND isnotnull(customer_sk#16)) THEN 1 ELSE 0 END)#27]
+Results [3]: [sum(CASE WHEN (isnotnull(customer_sk#9) AND isnull(customer_sk#16)) THEN 1 ELSE 0 END)#25 AS store_only#28, sum(CASE WHEN (isnull(customer_sk#9) AND isnotnull(customer_sk#16)) THEN 1 ELSE 0 END)#26 AS catalog_only#29, sum(CASE WHEN (isnotnull(customer_sk#9) AND isnotnull(customer_sk#16)) THEN 1 ELSE 0 END)#27 AS store_and_catalog#30]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (28)
+
+
+(28) ReusedExchange [Reuses operator id: 7]
+Output [1]: [d_date_sk#5]
+
+Subquery:2 Hosting operator id = 14 Hosting Expression = cs_sold_date_sk#13 IN dynamicpruning#4
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/simplified.txt
new file mode 100644
index 0000000000000..5aba214f237d7
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/simplified.txt
@@ -0,0 +1,46 @@
+WholeStageCodegen (8)
+  HashAggregate [sum,sum,sum] [sum(CASE WHEN (isnotnull(customer_sk) AND isnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnotnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),store_only,catalog_only,store_and_catalog,sum,sum,sum]
+    InputAdapter
+      Exchange #1
+        WholeStageCodegen (7)
+          HashAggregate [customer_sk,customer_sk] [sum,sum,sum,sum,sum,sum]
+            Project [customer_sk,customer_sk]
+              InputAdapter
+                SortMergeJoin [customer_sk,item_sk,customer_sk,item_sk]
+                  WholeStageCodegen (3)
+                    Sort [customer_sk,item_sk]
+                      HashAggregate [ss_customer_sk,ss_item_sk] [customer_sk,item_sk]
+                        InputAdapter
+                          Exchange [ss_customer_sk,ss_item_sk] #2
+                            WholeStageCodegen (2)
+                              HashAggregate [ss_customer_sk,ss_item_sk]
+                                Project [ss_item_sk,ss_customer_sk]
+                                  BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk]
+                                          SubqueryBroadcast [d_date_sk] #1
+                                            ReusedExchange [d_date_sk] #3
+                                    InputAdapter
+                                      BroadcastExchange #3
+                                        WholeStageCodegen (1)
+                                          Project [d_date_sk]
+                                            Filter [d_month_seq,d_date_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.date_dim [d_date_sk,d_month_seq]
+                  WholeStageCodegen (6)
+                    Sort [customer_sk,item_sk]
+                      HashAggregate [cs_bill_customer_sk,cs_item_sk] [customer_sk,item_sk]
+                        InputAdapter
+                          Exchange [cs_bill_customer_sk,cs_item_sk] #4
+                            WholeStageCodegen (5)
+                              HashAggregate [cs_bill_customer_sk,cs_item_sk]
+                                Project [cs_bill_customer_sk,cs_item_sk]
+                                  BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk]
+                                          ReusedSubquery [d_date_sk] #1
+                                    InputAdapter
+                                      ReusedExchange [d_date_sk] #3
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.sf100/explain.txt
new file mode 100644
index 0000000000000..91f246ce2aa12
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.sf100/explain.txt
@@ -0,0 +1,173 @@
+== Physical Plan ==
+* Project (29)
++- * Sort (28)
+   +- Exchange (27)
+      +- * Project (26)
+         +- Window (25)
+            +- * Sort (24)
+               +- Exchange (23)
+                  +- * HashAggregate (22)
+                     +- Exchange (21)
+                        +- * HashAggregate (20)
+                           +- * Project (19)
+                              +- * BroadcastHashJoin Inner BuildRight (18)
+                                 :- * Project (12)
+                                 :  +- * SortMergeJoin Inner (11)
+                                 :     :- * Sort (5)
+                                 :     :  +- Exchange (4)
+                                 :     :     +- * Filter (3)
+                                 :     :        +- * ColumnarToRow (2)
+                                 :     :           +- Scan parquet default.store_sales (1)
+                                 :     +- * Sort (10)
+                                 :        +- Exchange (9)
+                                 :           +- * Filter (8)
+                                 :              +- * ColumnarToRow (7)
+                                 :                 +- Scan parquet default.item (6)
+                                 +- BroadcastExchange (17)
+                                    +- * Project (16)
+                                       +- * Filter (15)
+                                          +- * ColumnarToRow (14)
+                                             +- Scan parquet default.date_dim (13)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 1]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_item_sk#1)
+
+(4) Exchange
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#5]
+
+(5) Sort [codegen id : 2]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Arguments: [ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.item
+Output [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [In(i_category, [Sports                                            ,Books                                             ,Home                                              ]), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_item_desc:string,i_current_price:decimal(7,2),i_class:string,i_category:string>
+
+(7) ColumnarToRow [codegen id : 3]
+Input [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+
+(8) Filter [codegen id : 3]
+Input [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Condition : (i_category#11 IN (Sports                                            ,Books                                             ,Home                                              ) AND isnotnull(i_item_sk#6))
+
+(9) Exchange
+Input [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Arguments: hashpartitioning(i_item_sk#6, 5), ENSURE_REQUIREMENTS, [id=#12]
+
+(10) Sort [codegen id : 4]
+Input [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Arguments: [i_item_sk#6 ASC NULLS FIRST], false, 0
+
+(11) SortMergeJoin [codegen id : 6]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#6]
+Join condition: None
+
+(12) Project [codegen id : 6]
+Output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+
+(13) Scan parquet default.date_dim
+Output [2]: [d_date_sk#13, d_date#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(14) ColumnarToRow [codegen id : 5]
+Input [2]: [d_date_sk#13, d_date#14]
+
+(15) Filter [codegen id : 5]
+Input [2]: [d_date_sk#13, d_date#14]
+Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13))
+
+(16) Project [codegen id : 5]
+Output [1]: [d_date_sk#13]
+Input [2]: [d_date_sk#13, d_date#14]
+
+(17) BroadcastExchange
+Input [1]: [d_date_sk#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#15]
+
+(18) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#13]
+Join condition: None
+
+(19) Project [codegen id : 6]
+Output [6]: [ss_ext_sales_price#2, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11, d_date_sk#13]
+
+(20) HashAggregate [codegen id : 6]
+Input [6]: [ss_ext_sales_price#2, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Keys [5]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum#16]
+Results [6]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, sum#17]
+
+(21) Exchange
+Input [6]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, sum#17]
+Arguments: hashpartitioning(i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(22) HashAggregate [codegen id : 7]
+Input [6]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, sum#17]
+Keys [5]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#19]
+Results [8]: [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#19,17,2) AS itemrevenue#20, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#19,17,2) AS _w0#21, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#19,17,2) AS _w1#22, i_item_id#7]
+
+(23) Exchange
+Input [8]: [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, _w0#21, _w1#22, i_item_id#7]
+Arguments: hashpartitioning(i_class#10, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(24) Sort [codegen id : 8]
+Input [8]: [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, _w0#21, _w1#22, i_item_id#7]
+Arguments: [i_class#10 ASC NULLS FIRST], false, 0
+
+(25) Window
+Input [8]: [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, _w0#21, _w1#22, i_item_id#7]
+Arguments: [sum(_w1#22) windowspecdefinition(i_class#10, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#24], [i_class#10]
+
+(26) Project [codegen id : 9]
+Output [7]: [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(_w0#21) * 100.00), DecimalType(21,2), true) as decimal(27,2))) / promote_precision(_we0#24)), DecimalType(38,17), true) AS revenueratio#25, i_item_id#7]
+Input [9]: [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, _w0#21, _w1#22, i_item_id#7, _we0#24]
+
+(27) Exchange
+Input [7]: [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, revenueratio#25, i_item_id#7]
+Arguments: rangepartitioning(i_category#11 ASC NULLS FIRST, i_class#10 ASC NULLS FIRST, i_item_id#7 ASC NULLS FIRST, i_item_desc#8 ASC NULLS FIRST, revenueratio#25 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [id=#26]
+
+(28) Sort [codegen id : 10]
+Input [7]: [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, revenueratio#25, i_item_id#7]
+Arguments: [i_category#11 ASC NULLS FIRST, i_class#10 ASC NULLS FIRST, i_item_id#7 ASC NULLS FIRST, i_item_desc#8 ASC NULLS FIRST, revenueratio#25 ASC NULLS FIRST], true, 0
+
+(29) Project [codegen id : 10]
+Output [6]: [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, revenueratio#25]
+Input [7]: [i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, revenueratio#25, i_item_id#7]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (30)
+
+
+(30) ReusedExchange [Reuses operator id: 17]
+Output [1]: [d_date_sk#13]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.sf100/simplified.txt
new file mode 100644
index 0000000000000..54ad1aead4c6d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.sf100/simplified.txt
@@ -0,0 +1,53 @@
+WholeStageCodegen (10)
+  Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio]
+    Sort [i_category,i_class,i_item_id,i_item_desc,revenueratio]
+      InputAdapter
+        Exchange [i_category,i_class,i_item_id,i_item_desc,revenueratio] #1
+          WholeStageCodegen (9)
+            Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id]
+              InputAdapter
+                Window [_w1,i_class]
+                  WholeStageCodegen (8)
+                    Sort [i_class]
+                      InputAdapter
+                        Exchange [i_class] #2
+                          WholeStageCodegen (7)
+                            HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ss_ext_sales_price)),itemrevenue,_w0,_w1,sum]
+                              InputAdapter
+                                Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3
+                                  WholeStageCodegen (6)
+                                    HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ss_ext_sales_price] [sum,sum]
+                                      Project [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                          Project [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                            SortMergeJoin [ss_item_sk,i_item_sk]
+                                              InputAdapter
+                                                WholeStageCodegen (2)
+                                                  Sort [ss_item_sk]
+                                                    InputAdapter
+                                                      Exchange [ss_item_sk] #4
+                                                        WholeStageCodegen (1)
+                                                          Filter [ss_item_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                                                  SubqueryBroadcast [d_date_sk] #1
+                                                                    ReusedExchange [d_date_sk] #5
+                                              InputAdapter
+                                                WholeStageCodegen (4)
+                                                  Sort [i_item_sk]
+                                                    InputAdapter
+                                                      Exchange [i_item_sk] #6
+                                                        WholeStageCodegen (3)
+                                                          Filter [i_category,i_item_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                          InputAdapter
+                                            BroadcastExchange #5
+                                              WholeStageCodegen (5)
+                                                Project [d_date_sk]
+                                                  Filter [d_date,d_date_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.date_dim [d_date_sk,d_date]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/explain.txt
new file mode 100644
index 0000000000000..1507b5ccbc0ae
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/explain.txt
@@ -0,0 +1,158 @@
+== Physical Plan ==
+* Project (26)
++- * Sort (25)
+   +- Exchange (24)
+      +- * Project (23)
+         +- Window (22)
+            +- * Sort (21)
+               +- Exchange (20)
+                  +- * HashAggregate (19)
+                     +- Exchange (18)
+                        +- * HashAggregate (17)
+                           +- * Project (16)
+                              +- * BroadcastHashJoin Inner BuildRight (15)
+                                 :- * Project (9)
+                                 :  +- * BroadcastHashJoin Inner BuildRight (8)
+                                 :     :- * Filter (3)
+                                 :     :  +- * ColumnarToRow (2)
+                                 :     :     +- Scan parquet default.store_sales (1)
+                                 :     +- BroadcastExchange (7)
+                                 :        +- * Filter (6)
+                                 :           +- * ColumnarToRow (5)
+                                 :              +- Scan parquet default.item (4)
+                                 +- BroadcastExchange (14)
+                                    +- * Project (13)
+                                       +- * Filter (12)
+                                          +- * ColumnarToRow (11)
+                                             +- Scan parquet default.date_dim (10)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_item_sk#1)
+
+(4) Scan parquet default.item
+Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [In(i_category, [Sports                                            ,Books                                             ,Home                                              ]), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_item_desc:string,i_current_price:decimal(7,2),i_class:string,i_category:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+
+(6) Filter [codegen id : 1]
+Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Condition : (i_category#10 IN (Sports                                            ,Books                                             ,Home                                              ) AND isnotnull(i_item_sk#5))
+
+(7) BroadcastExchange
+Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(8) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#5]
+Join condition: None
+
+(9) Project [codegen id : 3]
+Output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+
+(10) Scan parquet default.date_dim
+Output [2]: [d_date_sk#12, d_date#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#12, d_date#13]
+
+(12) Filter [codegen id : 2]
+Input [2]: [d_date_sk#12, d_date#13]
+Condition : (((isnotnull(d_date#13) AND (d_date#13 >= 1999-02-22)) AND (d_date#13 <= 1999-03-24)) AND isnotnull(d_date_sk#12))
+
+(13) Project [codegen id : 2]
+Output [1]: [d_date_sk#12]
+Input [2]: [d_date_sk#12, d_date#13]
+
+(14) BroadcastExchange
+Input [1]: [d_date_sk#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#12]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [6]: [ss_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10, d_date_sk#12]
+
+(17) HashAggregate [codegen id : 3]
+Input [6]: [ss_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum#15]
+Results [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#16]
+
+(18) Exchange
+Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#16]
+Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(19) HashAggregate [codegen id : 4]
+Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#16]
+Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#18]
+Results [8]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#18,17,2) AS itemrevenue#19, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#18,17,2) AS _w0#20, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#18,17,2) AS _w1#21, i_item_id#6]
+
+(20) Exchange
+Input [8]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21, i_item_id#6]
+Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(21) Sort [codegen id : 5]
+Input [8]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21, i_item_id#6]
+Arguments: [i_class#9 ASC NULLS FIRST], false, 0
+
+(22) Window
+Input [8]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21, i_item_id#6]
+Arguments: [sum(_w1#21) windowspecdefinition(i_class#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#23], [i_class#9]
+
+(23) Project [codegen id : 6]
+Output [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(_w0#20) * 100.00), DecimalType(21,2), true) as decimal(27,2))) / promote_precision(_we0#23)), DecimalType(38,17), true) AS revenueratio#24, i_item_id#6]
+Input [9]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21, i_item_id#6, _we0#23]
+
+(24) Exchange
+Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, revenueratio#24, i_item_id#6]
+Arguments: rangepartitioning(i_category#10 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_item_id#6 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#24 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(25) Sort [codegen id : 7]
+Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, revenueratio#24, i_item_id#6]
+Arguments: [i_category#10 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_item_id#6 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#24 ASC NULLS FIRST], true, 0
+
+(26) Project [codegen id : 7]
+Output [6]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, revenueratio#24]
+Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, revenueratio#24, i_item_id#6]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (27)
+
+
+(27) ReusedExchange [Reuses operator id: 14]
+Output [1]: [d_date_sk#12]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/simplified.txt
new file mode 100644
index 0000000000000..67205dff3ad35
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/simplified.txt
@@ -0,0 +1,44 @@
+WholeStageCodegen (7)
+  Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio]
+    Sort [i_category,i_class,i_item_id,i_item_desc,revenueratio]
+      InputAdapter
+        Exchange [i_category,i_class,i_item_id,i_item_desc,revenueratio] #1
+          WholeStageCodegen (6)
+            Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id]
+              InputAdapter
+                Window [_w1,i_class]
+                  WholeStageCodegen (5)
+                    Sort [i_class]
+                      InputAdapter
+                        Exchange [i_class] #2
+                          WholeStageCodegen (4)
+                            HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ss_ext_sales_price)),itemrevenue,_w0,_w1,sum]
+                              InputAdapter
+                                Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3
+                                  WholeStageCodegen (3)
+                                    HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ss_ext_sales_price] [sum,sum]
+                                      Project [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                          Project [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                            BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                              Filter [ss_item_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                                      SubqueryBroadcast [d_date_sk] #1
+                                                        ReusedExchange [d_date_sk] #4
+                                              InputAdapter
+                                                BroadcastExchange #5
+                                                  WholeStageCodegen (1)
+                                                    Filter [i_category,i_item_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                          InputAdapter
+                                            BroadcastExchange #4
+                                              WholeStageCodegen (2)
+                                                Project [d_date_sk]
+                                                  Filter [d_date,d_date_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.date_dim [d_date_sk,d_date]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.sf100/explain.txt
new file mode 100644
index 0000000000000..1b955ee3bd96c
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.sf100/explain.txt
@@ -0,0 +1,183 @@
+== Physical Plan ==
+TakeOrderedAndProject (32)
++- * HashAggregate (31)
+   +- Exchange (30)
+      +- * HashAggregate (29)
+         +- * Project (28)
+            +- * BroadcastHashJoin Inner BuildRight (27)
+               :- * Project (22)
+               :  +- * BroadcastHashJoin Inner BuildRight (21)
+               :     :- * Project (16)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (15)
+               :     :     :- * Project (10)
+               :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :     :     :- * Filter (3)
+               :     :     :     :  +- * ColumnarToRow (2)
+               :     :     :     :     +- Scan parquet default.catalog_sales (1)
+               :     :     :     +- BroadcastExchange (8)
+               :     :     :        +- * Project (7)
+               :     :     :           +- * Filter (6)
+               :     :     :              +- * ColumnarToRow (5)
+               :     :     :                 +- Scan parquet default.date_dim (4)
+               :     :     +- BroadcastExchange (14)
+               :     :        +- * Filter (13)
+               :     :           +- * ColumnarToRow (12)
+               :     :              +- Scan parquet default.ship_mode (11)
+               :     +- BroadcastExchange (20)
+               :        +- * Filter (19)
+               :           +- * ColumnarToRow (18)
+               :              +- Scan parquet default.call_center (17)
+               +- BroadcastExchange (26)
+                  +- * Filter (25)
+                     +- * ColumnarToRow (24)
+                        +- Scan parquet default.warehouse (23)
+
+
+(1) Scan parquet default.catalog_sales
+Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_sales]
+PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_ship_mode_sk), IsNotNull(cs_call_center_sk), IsNotNull(cs_ship_date_sk)]
+ReadSchema: struct<cs_ship_date_sk:int,cs_call_center_sk:int,cs_ship_mode_sk:int,cs_warehouse_sk:int>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5]
+
+(3) Filter [codegen id : 5]
+Input [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5]
+Condition : (((isnotnull(cs_warehouse_sk#4) AND isnotnull(cs_ship_mode_sk#3)) AND isnotnull(cs_call_center_sk#2)) AND isnotnull(cs_ship_date_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#6, d_month_seq#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#6, d_month_seq#7]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#6, d_month_seq#7]
+Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#6))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#6]
+Input [2]: [d_date_sk#6, d_month_seq#7]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#8]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [cs_ship_date_sk#1]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5]
+Input [6]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5, d_date_sk#6]
+
+(11) Scan parquet default.ship_mode
+Output [2]: [sm_ship_mode_sk#9, sm_type#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/ship_mode]
+PushedFilters: [IsNotNull(sm_ship_mode_sk)]
+ReadSchema: struct<sm_ship_mode_sk:int,sm_type:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [sm_ship_mode_sk#9, sm_type#10]
+
+(13) Filter [codegen id : 2]
+Input [2]: [sm_ship_mode_sk#9, sm_type#10]
+Condition : isnotnull(sm_ship_mode_sk#9)
+
+(14) BroadcastExchange
+Input [2]: [sm_ship_mode_sk#9, sm_type#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(15) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [cs_ship_mode_sk#3]
+Right keys [1]: [sm_ship_mode_sk#9]
+Join condition: None
+
+(16) Project [codegen id : 5]
+Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_warehouse_sk#4, cs_sold_date_sk#5, sm_type#10]
+Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5, sm_ship_mode_sk#9, sm_type#10]
+
+(17) Scan parquet default.call_center
+Output [2]: [cc_call_center_sk#12, cc_name#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/call_center]
+PushedFilters: [IsNotNull(cc_call_center_sk)]
+ReadSchema: struct<cc_call_center_sk:int,cc_name:string>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [2]: [cc_call_center_sk#12, cc_name#13]
+
+(19) Filter [codegen id : 3]
+Input [2]: [cc_call_center_sk#12, cc_name#13]
+Condition : isnotnull(cc_call_center_sk#12)
+
+(20) BroadcastExchange
+Input [2]: [cc_call_center_sk#12, cc_name#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#14]
+
+(21) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [cs_call_center_sk#2]
+Right keys [1]: [cc_call_center_sk#12]
+Join condition: None
+
+(22) Project [codegen id : 5]
+Output [5]: [cs_ship_date_sk#1, cs_warehouse_sk#4, cs_sold_date_sk#5, sm_type#10, cc_name#13]
+Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_warehouse_sk#4, cs_sold_date_sk#5, sm_type#10, cc_call_center_sk#12, cc_name#13]
+
+(23) Scan parquet default.warehouse
+Output [2]: [w_warehouse_sk#15, w_warehouse_name#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/warehouse]
+PushedFilters: [IsNotNull(w_warehouse_sk)]
+ReadSchema: struct<w_warehouse_sk:int,w_warehouse_name:string>
+
+(24) ColumnarToRow [codegen id : 4]
+Input [2]: [w_warehouse_sk#15, w_warehouse_name#16]
+
+(25) Filter [codegen id : 4]
+Input [2]: [w_warehouse_sk#15, w_warehouse_name#16]
+Condition : isnotnull(w_warehouse_sk#15)
+
+(26) BroadcastExchange
+Input [2]: [w_warehouse_sk#15, w_warehouse_name#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#17]
+
+(27) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [cs_warehouse_sk#4]
+Right keys [1]: [w_warehouse_sk#15]
+Join condition: None
+
+(28) Project [codegen id : 5]
+Output [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#10, cc_name#13, substr(w_warehouse_name#16, 1, 20) AS _groupingexpression#18]
+Input [7]: [cs_ship_date_sk#1, cs_warehouse_sk#4, cs_sold_date_sk#5, sm_type#10, cc_name#13, w_warehouse_sk#15, w_warehouse_name#16]
+
+(29) HashAggregate [codegen id : 5]
+Input [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#10, cc_name#13, _groupingexpression#18]
+Keys [3]: [_groupingexpression#18, sm_type#10, cc_name#13]
+Functions [5]: [partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)]
+Aggregate Attributes [5]: [sum#19, sum#20, sum#21, sum#22, sum#23]
+Results [8]: [_groupingexpression#18, sm_type#10, cc_name#13, sum#24, sum#25, sum#26, sum#27, sum#28]
+
+(30) Exchange
+Input [8]: [_groupingexpression#18, sm_type#10, cc_name#13, sum#24, sum#25, sum#26, sum#27, sum#28]
+Arguments: hashpartitioning(_groupingexpression#18, sm_type#10, cc_name#13, 5), ENSURE_REQUIREMENTS, [id=#29]
+
+(31) HashAggregate [codegen id : 6]
+Input [8]: [_groupingexpression#18, sm_type#10, cc_name#13, sum#24, sum#25, sum#26, sum#27, sum#28]
+Keys [3]: [_groupingexpression#18, sm_type#10, cc_name#13]
+Functions [5]: [sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)]
+Aggregate Attributes [5]: [sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#30, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#31, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#32, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#33, sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#34]
+Results [8]: [_groupingexpression#18 AS substr(w_warehouse_name, 1, 20)#35, sm_type#10, cc_name#13, sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#30 AS 30 days #36, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#31 AS 31 - 60 days #37, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#32 AS 61 - 90 days #38, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#33 AS 91 - 120 days #39, sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#34 AS >120 days #40]
+
+(32) TakeOrderedAndProject
+Input [8]: [substr(w_warehouse_name, 1, 20)#35, sm_type#10, cc_name#13, 30 days #36, 31 - 60 days #37, 61 - 90 days #38, 91 - 120 days #39, >120 days #40]
+Arguments: 100, [substr(w_warehouse_name, 1, 20)#35 ASC NULLS FIRST, sm_type#10 ASC NULLS FIRST, cc_name#13 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#35, sm_type#10, cc_name#13, 30 days #36, 31 - 60 days #37, 61 - 90 days #38, 91 - 120 days #39, >120 days #40]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.sf100/simplified.txt
new file mode 100644
index 0000000000000..f8abda81b72bc
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.sf100/simplified.txt
@@ -0,0 +1,48 @@
+TakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ]
+  WholeStageCodegen (6)
+    HashAggregate [_groupingexpression,sm_type,cc_name,sum,sum,sum,sum,sum] [sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 30) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 60) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 90) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) > 120) THEN 1 ELSE 0 END),substr(w_warehouse_name, 1, 20),30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum]
+      InputAdapter
+        Exchange [_groupingexpression,sm_type,cc_name] #1
+          WholeStageCodegen (5)
+            HashAggregate [_groupingexpression,sm_type,cc_name,cs_ship_date_sk,cs_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum]
+              Project [cs_ship_date_sk,cs_sold_date_sk,sm_type,cc_name,w_warehouse_name]
+                BroadcastHashJoin [cs_warehouse_sk,w_warehouse_sk]
+                  Project [cs_ship_date_sk,cs_warehouse_sk,cs_sold_date_sk,sm_type,cc_name]
+                    BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk]
+                      Project [cs_ship_date_sk,cs_call_center_sk,cs_warehouse_sk,cs_sold_date_sk,sm_type]
+                        BroadcastHashJoin [cs_ship_mode_sk,sm_ship_mode_sk]
+                          Project [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk]
+                            BroadcastHashJoin [cs_ship_date_sk,d_date_sk]
+                              Filter [cs_warehouse_sk,cs_ship_mode_sk,cs_call_center_sk,cs_ship_date_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.catalog_sales [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk]
+                              InputAdapter
+                                BroadcastExchange #2
+                                  WholeStageCodegen (1)
+                                    Project [d_date_sk]
+                                      Filter [d_month_seq,d_date_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.date_dim [d_date_sk,d_month_seq]
+                          InputAdapter
+                            BroadcastExchange #3
+                              WholeStageCodegen (2)
+                                Filter [sm_ship_mode_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.ship_mode [sm_ship_mode_sk,sm_type]
+                      InputAdapter
+                        BroadcastExchange #4
+                          WholeStageCodegen (3)
+                            Filter [cc_call_center_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.call_center [cc_call_center_sk,cc_name]
+                  InputAdapter
+                    BroadcastExchange #5
+                      WholeStageCodegen (4)
+                        Filter [w_warehouse_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.warehouse [w_warehouse_sk,w_warehouse_name]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99/explain.txt
new file mode 100644
index 0000000000000..1431623539828
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99/explain.txt
@@ -0,0 +1,183 @@
+== Physical Plan ==
+TakeOrderedAndProject (32)
++- * HashAggregate (31)
+   +- Exchange (30)
+      +- * HashAggregate (29)
+         +- * Project (28)
+            +- * BroadcastHashJoin Inner BuildRight (27)
+               :- * Project (21)
+               :  +- * BroadcastHashJoin Inner BuildRight (20)
+               :     :- * Project (15)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (14)
+               :     :     :- * Project (9)
+               :     :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+               :     :     :     :- * Filter (3)
+               :     :     :     :  +- * ColumnarToRow (2)
+               :     :     :     :     +- Scan parquet default.catalog_sales (1)
+               :     :     :     +- BroadcastExchange (7)
+               :     :     :        +- * Filter (6)
+               :     :     :           +- * ColumnarToRow (5)
+               :     :     :              +- Scan parquet default.warehouse (4)
+               :     :     +- BroadcastExchange (13)
+               :     :        +- * Filter (12)
+               :     :           +- * ColumnarToRow (11)
+               :     :              +- Scan parquet default.ship_mode (10)
+               :     +- BroadcastExchange (19)
+               :        +- * Filter (18)
+               :           +- * ColumnarToRow (17)
+               :              +- Scan parquet default.call_center (16)
+               +- BroadcastExchange (26)
+                  +- * Project (25)
+                     +- * Filter (24)
+                        +- * ColumnarToRow (23)
+                           +- Scan parquet default.date_dim (22)
+
+
+(1) Scan parquet default.catalog_sales
+Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_sales]
+PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_ship_mode_sk), IsNotNull(cs_call_center_sk), IsNotNull(cs_ship_date_sk)]
+ReadSchema: struct<cs_ship_date_sk:int,cs_call_center_sk:int,cs_ship_mode_sk:int,cs_warehouse_sk:int>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5]
+
+(3) Filter [codegen id : 5]
+Input [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5]
+Condition : (((isnotnull(cs_warehouse_sk#4) AND isnotnull(cs_ship_mode_sk#3)) AND isnotnull(cs_call_center_sk#2)) AND isnotnull(cs_ship_date_sk#1))
+
+(4) Scan parquet default.warehouse
+Output [2]: [w_warehouse_sk#6, w_warehouse_name#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/warehouse]
+PushedFilters: [IsNotNull(w_warehouse_sk)]
+ReadSchema: struct<w_warehouse_sk:int,w_warehouse_name:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [w_warehouse_sk#6, w_warehouse_name#7]
+
+(6) Filter [codegen id : 1]
+Input [2]: [w_warehouse_sk#6, w_warehouse_name#7]
+Condition : isnotnull(w_warehouse_sk#6)
+
+(7) BroadcastExchange
+Input [2]: [w_warehouse_sk#6, w_warehouse_name#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#8]
+
+(8) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [cs_warehouse_sk#4]
+Right keys [1]: [w_warehouse_sk#6]
+Join condition: None
+
+(9) Project [codegen id : 5]
+Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7]
+Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5, w_warehouse_sk#6, w_warehouse_name#7]
+
+(10) Scan parquet default.ship_mode
+Output [2]: [sm_ship_mode_sk#9, sm_type#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/ship_mode]
+PushedFilters: [IsNotNull(sm_ship_mode_sk)]
+ReadSchema: struct<sm_ship_mode_sk:int,sm_type:string>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [sm_ship_mode_sk#9, sm_type#10]
+
+(12) Filter [codegen id : 2]
+Input [2]: [sm_ship_mode_sk#9, sm_type#10]
+Condition : isnotnull(sm_ship_mode_sk#9)
+
+(13) BroadcastExchange
+Input [2]: [sm_ship_mode_sk#9, sm_type#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(14) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [cs_ship_mode_sk#3]
+Right keys [1]: [sm_ship_mode_sk#9]
+Join condition: None
+
+(15) Project [codegen id : 5]
+Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10]
+Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#9, sm_type#10]
+
+(16) Scan parquet default.call_center
+Output [2]: [cc_call_center_sk#12, cc_name#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/call_center]
+PushedFilters: [IsNotNull(cc_call_center_sk)]
+ReadSchema: struct<cc_call_center_sk:int,cc_name:string>
+
+(17) ColumnarToRow [codegen id : 3]
+Input [2]: [cc_call_center_sk#12, cc_name#13]
+
+(18) Filter [codegen id : 3]
+Input [2]: [cc_call_center_sk#12, cc_name#13]
+Condition : isnotnull(cc_call_center_sk#12)
+
+(19) BroadcastExchange
+Input [2]: [cc_call_center_sk#12, cc_name#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#14]
+
+(20) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [cs_call_center_sk#2]
+Right keys [1]: [cc_call_center_sk#12]
+Join condition: None
+
+(21) Project [codegen id : 5]
+Output [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#13]
+Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_call_center_sk#12, cc_name#13]
+
+(22) Scan parquet default.date_dim
+Output [2]: [d_date_sk#15, d_month_seq#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(23) ColumnarToRow [codegen id : 4]
+Input [2]: [d_date_sk#15, d_month_seq#16]
+
+(24) Filter [codegen id : 4]
+Input [2]: [d_date_sk#15, d_month_seq#16]
+Condition : (((isnotnull(d_month_seq#16) AND (d_month_seq#16 >= 1200)) AND (d_month_seq#16 <= 1211)) AND isnotnull(d_date_sk#15))
+
+(25) Project [codegen id : 4]
+Output [1]: [d_date_sk#15]
+Input [2]: [d_date_sk#15, d_month_seq#16]
+
+(26) BroadcastExchange
+Input [1]: [d_date_sk#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(27) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [cs_ship_date_sk#1]
+Right keys [1]: [d_date_sk#15]
+Join condition: None
+
+(28) Project [codegen id : 5]
+Output [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#10, cc_name#13, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#18]
+Input [6]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#13, d_date_sk#15]
+
+(29) HashAggregate [codegen id : 5]
+Input [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#10, cc_name#13, _groupingexpression#18]
+Keys [3]: [_groupingexpression#18, sm_type#10, cc_name#13]
+Functions [5]: [partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)]
+Aggregate Attributes [5]: [sum#19, sum#20, sum#21, sum#22, sum#23]
+Results [8]: [_groupingexpression#18, sm_type#10, cc_name#13, sum#24, sum#25, sum#26, sum#27, sum#28]
+
+(30) Exchange
+Input [8]: [_groupingexpression#18, sm_type#10, cc_name#13, sum#24, sum#25, sum#26, sum#27, sum#28]
+Arguments: hashpartitioning(_groupingexpression#18, sm_type#10, cc_name#13, 5), ENSURE_REQUIREMENTS, [id=#29]
+
+(31) HashAggregate [codegen id : 6]
+Input [8]: [_groupingexpression#18, sm_type#10, cc_name#13, sum#24, sum#25, sum#26, sum#27, sum#28]
+Keys [3]: [_groupingexpression#18, sm_type#10, cc_name#13]
+Functions [5]: [sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)]
+Aggregate Attributes [5]: [sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#30, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#31, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#32, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#33, sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#34]
+Results [8]: [_groupingexpression#18 AS substr(w_warehouse_name, 1, 20)#35, sm_type#10, cc_name#13, sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#30 AS 30 days #36, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#31 AS 31 - 60 days #37, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#32 AS 61 - 90 days #38, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#33 AS 91 - 120 days #39, sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#34 AS >120 days #40]
+
+(32) TakeOrderedAndProject
+Input [8]: [substr(w_warehouse_name, 1, 20)#35, sm_type#10, cc_name#13, 30 days #36, 31 - 60 days #37, 61 - 90 days #38, 91 - 120 days #39, >120 days #40]
+Arguments: 100, [substr(w_warehouse_name, 1, 20)#35 ASC NULLS FIRST, sm_type#10 ASC NULLS FIRST, cc_name#13 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#35, sm_type#10, cc_name#13, 30 days #36, 31 - 60 days #37, 61 - 90 days #38, 91 - 120 days #39, >120 days #40]
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99/simplified.txt
new file mode 100644
index 0000000000000..8c9e90042e5b3
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99/simplified.txt
@@ -0,0 +1,48 @@
+TakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ]
+  WholeStageCodegen (6)
+    HashAggregate [_groupingexpression,sm_type,cc_name,sum,sum,sum,sum,sum] [sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 30) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 60) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 90) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) > 120) THEN 1 ELSE 0 END),substr(w_warehouse_name, 1, 20),30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum]
+      InputAdapter
+        Exchange [_groupingexpression,sm_type,cc_name] #1
+          WholeStageCodegen (5)
+            HashAggregate [_groupingexpression,sm_type,cc_name,cs_ship_date_sk,cs_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum]
+              Project [cs_ship_date_sk,cs_sold_date_sk,sm_type,cc_name,w_warehouse_name]
+                BroadcastHashJoin [cs_ship_date_sk,d_date_sk]
+                  Project [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name]
+                    BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk]
+                      Project [cs_ship_date_sk,cs_call_center_sk,cs_sold_date_sk,w_warehouse_name,sm_type]
+                        BroadcastHashJoin [cs_ship_mode_sk,sm_ship_mode_sk]
+                          Project [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_sold_date_sk,w_warehouse_name]
+                            BroadcastHashJoin [cs_warehouse_sk,w_warehouse_sk]
+                              Filter [cs_warehouse_sk,cs_ship_mode_sk,cs_call_center_sk,cs_ship_date_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.catalog_sales [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk]
+                              InputAdapter
+                                BroadcastExchange #2
+                                  WholeStageCodegen (1)
+                                    Filter [w_warehouse_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.warehouse [w_warehouse_sk,w_warehouse_name]
+                          InputAdapter
+                            BroadcastExchange #3
+                              WholeStageCodegen (2)
+                                Filter [sm_ship_mode_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.ship_mode [sm_ship_mode_sk,sm_type]
+                      InputAdapter
+                        BroadcastExchange #4
+                          WholeStageCodegen (3)
+                            Filter [cc_call_center_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.call_center [cc_call_center_sk,cc_name]
+                  InputAdapter
+                    BroadcastExchange #5
+                      WholeStageCodegen (4)
+                        Project [d_date_sk]
+                          Filter [d_month_seq,d_date_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.date_dim [d_date_sk,d_month_seq]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.sf100/explain.txt
new file mode 100644
index 0000000000000..60b02bac254bb
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.sf100/explain.txt
@@ -0,0 +1,285 @@
+== Physical Plan ==
+TakeOrderedAndProject (49)
++- * HashAggregate (48)
+   +- Exchange (47)
+      +- * HashAggregate (46)
+         +- * Project (45)
+            +- * BroadcastHashJoin Inner BuildLeft (44)
+               :- BroadcastExchange (40)
+               :  +- * Project (39)
+               :     +- * BroadcastHashJoin Inner BuildRight (38)
+               :        :- * Project (32)
+               :        :  +- * SortMergeJoin LeftSemi (31)
+               :        :     :- * SortMergeJoin LeftSemi (17)
+               :        :     :  :- * Sort (5)
+               :        :     :  :  +- Exchange (4)
+               :        :     :  :     +- * Filter (3)
+               :        :     :  :        +- * ColumnarToRow (2)
+               :        :     :  :           +- Scan parquet default.customer (1)
+               :        :     :  +- * Sort (16)
+               :        :     :     +- Exchange (15)
+               :        :     :        +- * Project (14)
+               :        :     :           +- * BroadcastHashJoin Inner BuildRight (13)
+               :        :     :              :- * ColumnarToRow (7)
+               :        :     :              :  +- Scan parquet default.store_sales (6)
+               :        :     :              +- BroadcastExchange (12)
+               :        :     :                 +- * Project (11)
+               :        :     :                    +- * Filter (10)
+               :        :     :                       +- * ColumnarToRow (9)
+               :        :     :                          +- Scan parquet default.date_dim (8)
+               :        :     +- * Sort (30)
+               :        :        +- Exchange (29)
+               :        :           +- Union (28)
+               :        :              :- * Project (22)
+               :        :              :  +- * BroadcastHashJoin Inner BuildRight (21)
+               :        :              :     :- * ColumnarToRow (19)
+               :        :              :     :  +- Scan parquet default.web_sales (18)
+               :        :              :     +- ReusedExchange (20)
+               :        :              +- * Project (27)
+               :        :                 +- * BroadcastHashJoin Inner BuildRight (26)
+               :        :                    :- * ColumnarToRow (24)
+               :        :                    :  +- Scan parquet default.catalog_sales (23)
+               :        :                    +- ReusedExchange (25)
+               :        +- BroadcastExchange (37)
+               :           +- * Project (36)
+               :              +- * Filter (35)
+               :                 +- * ColumnarToRow (34)
+               :                    +- Scan parquet default.customer_address (33)
+               +- * Filter (43)
+                  +- * ColumnarToRow (42)
+                     +- Scan parquet default.customer_demographics (41)
+
+
+(1) Scan parquet default.customer
+Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_addr_sk:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+
+(3) Filter [codegen id : 1]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2))
+
+(4) Exchange
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+Arguments: hashpartitioning(c_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#4]
+
+(5) Sort [codegen id : 2]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+Arguments: [c_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.store_sales
+Output [2]: [ss_customer_sk#5, ss_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)]
+ReadSchema: struct<ss_customer_sk:int>
+
+(7) ColumnarToRow [codegen id : 4]
+Input [2]: [ss_customer_sk#5, ss_sold_date_sk#6]
+
+(8) Scan parquet default.date_dim
+Output [3]: [d_date_sk#8, d_year#9, d_moy#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,7), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(9) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#8, d_year#9, d_moy#10]
+
+(10) Filter [codegen id : 3]
+Input [3]: [d_date_sk#8, d_year#9, d_moy#10]
+Condition : (((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2002)) AND (d_moy#10 >= 4)) AND (d_moy#10 <= 7)) AND isnotnull(d_date_sk#8))
+
+(11) Project [codegen id : 3]
+Output [1]: [d_date_sk#8]
+Input [3]: [d_date_sk#8, d_year#9, d_moy#10]
+
+(12) BroadcastExchange
+Input [1]: [d_date_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#11]
+
+(13) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#6]
+Right keys [1]: [d_date_sk#8]
+Join condition: None
+
+(14) Project [codegen id : 4]
+Output [1]: [ss_customer_sk#5]
+Input [3]: [ss_customer_sk#5, ss_sold_date_sk#6, d_date_sk#8]
+
+(15) Exchange
+Input [1]: [ss_customer_sk#5]
+Arguments: hashpartitioning(ss_customer_sk#5, 5), ENSURE_REQUIREMENTS, [id=#12]
+
+(16) Sort [codegen id : 5]
+Input [1]: [ss_customer_sk#5]
+Arguments: [ss_customer_sk#5 ASC NULLS FIRST], false, 0
+
+(17) SortMergeJoin [codegen id : 6]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [ss_customer_sk#5]
+Join condition: None
+
+(18) Scan parquet default.web_sales
+Output [2]: [ws_bill_customer_sk#13, ws_sold_date_sk#14]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#14), dynamicpruningexpression(ws_sold_date_sk#14 IN dynamicpruning#7)]
+ReadSchema: struct<ws_bill_customer_sk:int>
+
+(19) ColumnarToRow [codegen id : 8]
+Input [2]: [ws_bill_customer_sk#13, ws_sold_date_sk#14]
+
+(20) ReusedExchange [Reuses operator id: 12]
+Output [1]: [d_date_sk#15]
+
+(21) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ws_sold_date_sk#14]
+Right keys [1]: [d_date_sk#15]
+Join condition: None
+
+(22) Project [codegen id : 8]
+Output [1]: [ws_bill_customer_sk#13 AS customer_sk#16]
+Input [3]: [ws_bill_customer_sk#13, ws_sold_date_sk#14, d_date_sk#15]
+
+(23) Scan parquet default.catalog_sales
+Output [2]: [cs_ship_customer_sk#17, cs_sold_date_sk#18]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_sold_date_sk#18 IN dynamicpruning#7)]
+ReadSchema: struct<cs_ship_customer_sk:int>
+
+(24) ColumnarToRow [codegen id : 10]
+Input [2]: [cs_ship_customer_sk#17, cs_sold_date_sk#18]
+
+(25) ReusedExchange [Reuses operator id: 12]
+Output [1]: [d_date_sk#19]
+
+(26) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_sold_date_sk#18]
+Right keys [1]: [d_date_sk#19]
+Join condition: None
+
+(27) Project [codegen id : 10]
+Output [1]: [cs_ship_customer_sk#17 AS customer_sk#20]
+Input [3]: [cs_ship_customer_sk#17, cs_sold_date_sk#18, d_date_sk#19]
+
+(28) Union
+
+(29) Exchange
+Input [1]: [customer_sk#16]
+Arguments: hashpartitioning(customer_sk#16, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(30) Sort [codegen id : 11]
+Input [1]: [customer_sk#16]
+Arguments: [customer_sk#16 ASC NULLS FIRST], false, 0
+
+(31) SortMergeJoin [codegen id : 13]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [customer_sk#16]
+Join condition: None
+
+(32) Project [codegen id : 13]
+Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+
+(33) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#22, ca_county#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [In(ca_county, [Walker County,Richland County,Gaines County,Douglas County,Dona Ana County]), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_county:string>
+
+(34) ColumnarToRow [codegen id : 12]
+Input [2]: [ca_address_sk#22, ca_county#23]
+
+(35) Filter [codegen id : 12]
+Input [2]: [ca_address_sk#22, ca_county#23]
+Condition : (ca_county#23 IN (Walker County,Richland County,Gaines County,Douglas County,Dona Ana County) AND isnotnull(ca_address_sk#22))
+
+(36) Project [codegen id : 12]
+Output [1]: [ca_address_sk#22]
+Input [2]: [ca_address_sk#22, ca_county#23]
+
+(37) BroadcastExchange
+Input [1]: [ca_address_sk#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#24]
+
+(38) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [c_current_addr_sk#3]
+Right keys [1]: [ca_address_sk#22]
+Join condition: None
+
+(39) Project [codegen id : 13]
+Output [1]: [c_current_cdemo_sk#2]
+Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#22]
+
+(40) BroadcastExchange
+Input [1]: [c_current_cdemo_sk#2]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#25]
+
+(41) Scan parquet default.customer_demographics
+Output [9]: [cd_demo_sk#26, cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#30, cd_credit_rating#31, cd_dep_count#32, cd_dep_employed_count#33, cd_dep_college_count#34]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_education_status:string,cd_purchase_estimate:int,cd_credit_rating:string,cd_dep_count:int,cd_dep_employed_count:int,cd_dep_college_count:int>
+
+(42) ColumnarToRow
+Input [9]: [cd_demo_sk#26, cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#30, cd_credit_rating#31, cd_dep_count#32, cd_dep_employed_count#33, cd_dep_college_count#34]
+
+(43) Filter
+Input [9]: [cd_demo_sk#26, cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#30, cd_credit_rating#31, cd_dep_count#32, cd_dep_employed_count#33, cd_dep_college_count#34]
+Condition : isnotnull(cd_demo_sk#26)
+
+(44) BroadcastHashJoin [codegen id : 14]
+Left keys [1]: [c_current_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#26]
+Join condition: None
+
+(45) Project [codegen id : 14]
+Output [8]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#30, cd_credit_rating#31, cd_dep_count#32, cd_dep_employed_count#33, cd_dep_college_count#34]
+Input [10]: [c_current_cdemo_sk#2, cd_demo_sk#26, cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#30, cd_credit_rating#31, cd_dep_count#32, cd_dep_employed_count#33, cd_dep_college_count#34]
+
+(46) HashAggregate [codegen id : 14]
+Input [8]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#30, cd_credit_rating#31, cd_dep_count#32, cd_dep_employed_count#33, cd_dep_college_count#34]
+Keys [8]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#30, cd_credit_rating#31, cd_dep_count#32, cd_dep_employed_count#33, cd_dep_college_count#34]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#35]
+Results [9]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#30, cd_credit_rating#31, cd_dep_count#32, cd_dep_employed_count#33, cd_dep_college_count#34, count#36]
+
+(47) Exchange
+Input [9]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#30, cd_credit_rating#31, cd_dep_count#32, cd_dep_employed_count#33, cd_dep_college_count#34, count#36]
+Arguments: hashpartitioning(cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#30, cd_credit_rating#31, cd_dep_count#32, cd_dep_employed_count#33, cd_dep_college_count#34, 5), ENSURE_REQUIREMENTS, [id=#37]
+
+(48) HashAggregate [codegen id : 15]
+Input [9]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#30, cd_credit_rating#31, cd_dep_count#32, cd_dep_employed_count#33, cd_dep_college_count#34, count#36]
+Keys [8]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#30, cd_credit_rating#31, cd_dep_count#32, cd_dep_employed_count#33, cd_dep_college_count#34]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#38]
+Results [14]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, count(1)#38 AS cnt1#39, cd_purchase_estimate#30, count(1)#38 AS cnt2#40, cd_credit_rating#31, count(1)#38 AS cnt3#41, cd_dep_count#32, count(1)#38 AS cnt4#42, cd_dep_employed_count#33, count(1)#38 AS cnt5#43, cd_dep_college_count#34, count(1)#38 AS cnt6#44]
+
+(49) TakeOrderedAndProject
+Input [14]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cnt1#39, cd_purchase_estimate#30, cnt2#40, cd_credit_rating#31, cnt3#41, cd_dep_count#32, cnt4#42, cd_dep_employed_count#33, cnt5#43, cd_dep_college_count#34, cnt6#44]
+Arguments: 100, [cd_gender#27 ASC NULLS FIRST, cd_marital_status#28 ASC NULLS FIRST, cd_education_status#29 ASC NULLS FIRST, cd_purchase_estimate#30 ASC NULLS FIRST, cd_credit_rating#31 ASC NULLS FIRST, cd_dep_count#32 ASC NULLS FIRST, cd_dep_employed_count#33 ASC NULLS FIRST, cd_dep_college_count#34 ASC NULLS FIRST], [cd_gender#27, cd_marital_status#28, cd_education_status#29, cnt1#39, cd_purchase_estimate#30, cnt2#40, cd_credit_rating#31, cnt3#41, cd_dep_count#32, cnt4#42, cd_dep_employed_count#33, cnt5#43, cd_dep_college_count#34, cnt6#44]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 6 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7
+ReusedExchange (50)
+
+
+(50) ReusedExchange [Reuses operator id: 12]
+Output [1]: [d_date_sk#8]
+
+Subquery:2 Hosting operator id = 18 Hosting Expression = ws_sold_date_sk#14 IN dynamicpruning#7
+
+Subquery:3 Hosting operator id = 23 Hosting Expression = cs_sold_date_sk#18 IN dynamicpruning#7
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.sf100/simplified.txt
new file mode 100644
index 0000000000000..433cb374a4c75
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.sf100/simplified.txt
@@ -0,0 +1,86 @@
+TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,cnt2,cnt3,cnt4,cnt5,cnt6]
+  WholeStageCodegen (15)
+    HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] [count(1),cnt1,cnt2,cnt3,cnt4,cnt5,cnt6,count]
+      InputAdapter
+        Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1
+          WholeStageCodegen (14)
+            HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count]
+              Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count]
+                BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk]
+                  InputAdapter
+                    BroadcastExchange #2
+                      WholeStageCodegen (13)
+                        Project [c_current_cdemo_sk]
+                          BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                            Project [c_current_cdemo_sk,c_current_addr_sk]
+                              SortMergeJoin [c_customer_sk,customer_sk]
+                                InputAdapter
+                                  WholeStageCodegen (6)
+                                    SortMergeJoin [c_customer_sk,ss_customer_sk]
+                                      InputAdapter
+                                        WholeStageCodegen (2)
+                                          Sort [c_customer_sk]
+                                            InputAdapter
+                                              Exchange [c_customer_sk] #3
+                                                WholeStageCodegen (1)
+                                                  Filter [c_current_addr_sk,c_current_cdemo_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk]
+                                      InputAdapter
+                                        WholeStageCodegen (5)
+                                          Sort [ss_customer_sk]
+                                            InputAdapter
+                                              Exchange [ss_customer_sk] #4
+                                                WholeStageCodegen (4)
+                                                  Project [ss_customer_sk]
+                                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.store_sales [ss_customer_sk,ss_sold_date_sk]
+                                                            SubqueryBroadcast [d_date_sk] #1
+                                                              ReusedExchange [d_date_sk] #5
+                                                      InputAdapter
+                                                        BroadcastExchange #5
+                                                          WholeStageCodegen (3)
+                                                            Project [d_date_sk]
+                                                              Filter [d_year,d_moy,d_date_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                InputAdapter
+                                  WholeStageCodegen (11)
+                                    Sort [customer_sk]
+                                      InputAdapter
+                                        Exchange [customer_sk] #6
+                                          Union
+                                            WholeStageCodegen (8)
+                                              Project [ws_bill_customer_sk]
+                                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.web_sales [ws_bill_customer_sk,ws_sold_date_sk]
+                                                        ReusedSubquery [d_date_sk] #1
+                                                  InputAdapter
+                                                    ReusedExchange [d_date_sk] #5
+                                            WholeStageCodegen (10)
+                                              Project [cs_ship_customer_sk]
+                                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk]
+                                                        ReusedSubquery [d_date_sk] #1
+                                                  InputAdapter
+                                                    ReusedExchange [d_date_sk] #5
+                            InputAdapter
+                              BroadcastExchange #7
+                                WholeStageCodegen (12)
+                                  Project [ca_address_sk]
+                                    Filter [ca_county,ca_address_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.customer_address [ca_address_sk,ca_county]
+                  Filter [cd_demo_sk]
+                    ColumnarToRow
+                      InputAdapter
+                        Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/explain.txt
new file mode 100644
index 0000000000000..62054fe96dd5c
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/explain.txt
@@ -0,0 +1,265 @@
+== Physical Plan ==
+TakeOrderedAndProject (45)
++- * HashAggregate (44)
+   +- Exchange (43)
+      +- * HashAggregate (42)
+         +- * Project (41)
+            +- * BroadcastHashJoin Inner BuildRight (40)
+               :- * Project (35)
+               :  +- * BroadcastHashJoin Inner BuildRight (34)
+               :     :- * Project (28)
+               :     :  +- * BroadcastHashJoin LeftSemi BuildRight (27)
+               :     :     :- * BroadcastHashJoin LeftSemi BuildRight (14)
+               :     :     :  :- * Filter (3)
+               :     :     :  :  +- * ColumnarToRow (2)
+               :     :     :  :     +- Scan parquet default.customer (1)
+               :     :     :  +- BroadcastExchange (13)
+               :     :     :     +- * Project (12)
+               :     :     :        +- * BroadcastHashJoin Inner BuildRight (11)
+               :     :     :           :- * ColumnarToRow (5)
+               :     :     :           :  +- Scan parquet default.store_sales (4)
+               :     :     :           +- BroadcastExchange (10)
+               :     :     :              +- * Project (9)
+               :     :     :                 +- * Filter (8)
+               :     :     :                    +- * ColumnarToRow (7)
+               :     :     :                       +- Scan parquet default.date_dim (6)
+               :     :     +- BroadcastExchange (26)
+               :     :        +- Union (25)
+               :     :           :- * Project (19)
+               :     :           :  +- * BroadcastHashJoin Inner BuildRight (18)
+               :     :           :     :- * ColumnarToRow (16)
+               :     :           :     :  +- Scan parquet default.web_sales (15)
+               :     :           :     +- ReusedExchange (17)
+               :     :           +- * Project (24)
+               :     :              +- * BroadcastHashJoin Inner BuildRight (23)
+               :     :                 :- * ColumnarToRow (21)
+               :     :                 :  +- Scan parquet default.catalog_sales (20)
+               :     :                 +- ReusedExchange (22)
+               :     +- BroadcastExchange (33)
+               :        +- * Project (32)
+               :           +- * Filter (31)
+               :              +- * ColumnarToRow (30)
+               :                 +- Scan parquet default.customer_address (29)
+               +- BroadcastExchange (39)
+                  +- * Filter (38)
+                     +- * ColumnarToRow (37)
+                        +- Scan parquet default.customer_demographics (36)
+
+
+(1) Scan parquet default.customer
+Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_addr_sk:int>
+
+(2) ColumnarToRow [codegen id : 9]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+
+(3) Filter [codegen id : 9]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2))
+
+(4) Scan parquet default.store_sales
+Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)]
+ReadSchema: struct<ss_customer_sk:int>
+
+(5) ColumnarToRow [codegen id : 2]
+Input [2]: [ss_customer_sk#4, ss_sold_date_sk#5]
+
+(6) Scan parquet default.date_dim
+Output [3]: [d_date_sk#7, d_year#8, d_moy#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,7), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(7) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_moy#9]
+
+(8) Filter [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_moy#9]
+Condition : (((((isnotnull(d_year#8) AND isnotnull(d_moy#9)) AND (d_year#8 = 2002)) AND (d_moy#9 >= 4)) AND (d_moy#9 <= 7)) AND isnotnull(d_date_sk#7))
+
+(9) Project [codegen id : 1]
+Output [1]: [d_date_sk#7]
+Input [3]: [d_date_sk#7, d_year#8, d_moy#9]
+
+(10) BroadcastExchange
+Input [1]: [d_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(11) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#5]
+Right keys [1]: [d_date_sk#7]
+Join condition: None
+
+(12) Project [codegen id : 2]
+Output [1]: [ss_customer_sk#4]
+Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#7]
+
+(13) BroadcastExchange
+Input [1]: [ss_customer_sk#4]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#11]
+
+(14) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [ss_customer_sk#4]
+Join condition: None
+
+(15) Scan parquet default.web_sales
+Output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#13), dynamicpruningexpression(ws_sold_date_sk#13 IN dynamicpruning#6)]
+ReadSchema: struct<ws_bill_customer_sk:int>
+
+(16) ColumnarToRow [codegen id : 4]
+Input [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13]
+
+(17) ReusedExchange [Reuses operator id: 10]
+Output [1]: [d_date_sk#14]
+
+(18) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ws_sold_date_sk#13]
+Right keys [1]: [d_date_sk#14]
+Join condition: None
+
+(19) Project [codegen id : 4]
+Output [1]: [ws_bill_customer_sk#12 AS customer_sk#15]
+Input [3]: [ws_bill_customer_sk#12, ws_sold_date_sk#13, d_date_sk#14]
+
+(20) Scan parquet default.catalog_sales
+Output [2]: [cs_ship_customer_sk#16, cs_sold_date_sk#17]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#17), dynamicpruningexpression(cs_sold_date_sk#17 IN dynamicpruning#6)]
+ReadSchema: struct<cs_ship_customer_sk:int>
+
+(21) ColumnarToRow [codegen id : 6]
+Input [2]: [cs_ship_customer_sk#16, cs_sold_date_sk#17]
+
+(22) ReusedExchange [Reuses operator id: 10]
+Output [1]: [d_date_sk#18]
+
+(23) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [cs_sold_date_sk#17]
+Right keys [1]: [d_date_sk#18]
+Join condition: None
+
+(24) Project [codegen id : 6]
+Output [1]: [cs_ship_customer_sk#16 AS customer_sk#19]
+Input [3]: [cs_ship_customer_sk#16, cs_sold_date_sk#17, d_date_sk#18]
+
+(25) Union
+
+(26) BroadcastExchange
+Input [1]: [customer_sk#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#20]
+
+(27) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [customer_sk#15]
+Join condition: None
+
+(28) Project [codegen id : 9]
+Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+
+(29) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#21, ca_county#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [In(ca_county, [Walker County,Richland County,Gaines County,Douglas County,Dona Ana County]), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_county:string>
+
+(30) ColumnarToRow [codegen id : 7]
+Input [2]: [ca_address_sk#21, ca_county#22]
+
+(31) Filter [codegen id : 7]
+Input [2]: [ca_address_sk#21, ca_county#22]
+Condition : (ca_county#22 IN (Walker County,Richland County,Gaines County,Douglas County,Dona Ana County) AND isnotnull(ca_address_sk#21))
+
+(32) Project [codegen id : 7]
+Output [1]: [ca_address_sk#21]
+Input [2]: [ca_address_sk#21, ca_county#22]
+
+(33) BroadcastExchange
+Input [1]: [ca_address_sk#21]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#23]
+
+(34) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_current_addr_sk#3]
+Right keys [1]: [ca_address_sk#21]
+Join condition: None
+
+(35) Project [codegen id : 9]
+Output [1]: [c_current_cdemo_sk#2]
+Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#21]
+
+(36) Scan parquet default.customer_demographics
+Output [9]: [cd_demo_sk#24, cd_gender#25, cd_marital_status#26, cd_education_status#27, cd_purchase_estimate#28, cd_credit_rating#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_education_status:string,cd_purchase_estimate:int,cd_credit_rating:string,cd_dep_count:int,cd_dep_employed_count:int,cd_dep_college_count:int>
+
+(37) ColumnarToRow [codegen id : 8]
+Input [9]: [cd_demo_sk#24, cd_gender#25, cd_marital_status#26, cd_education_status#27, cd_purchase_estimate#28, cd_credit_rating#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32]
+
+(38) Filter [codegen id : 8]
+Input [9]: [cd_demo_sk#24, cd_gender#25, cd_marital_status#26, cd_education_status#27, cd_purchase_estimate#28, cd_credit_rating#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32]
+Condition : isnotnull(cd_demo_sk#24)
+
+(39) BroadcastExchange
+Input [9]: [cd_demo_sk#24, cd_gender#25, cd_marital_status#26, cd_education_status#27, cd_purchase_estimate#28, cd_credit_rating#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#33]
+
+(40) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_current_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#24]
+Join condition: None
+
+(41) Project [codegen id : 9]
+Output [8]: [cd_gender#25, cd_marital_status#26, cd_education_status#27, cd_purchase_estimate#28, cd_credit_rating#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32]
+Input [10]: [c_current_cdemo_sk#2, cd_demo_sk#24, cd_gender#25, cd_marital_status#26, cd_education_status#27, cd_purchase_estimate#28, cd_credit_rating#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32]
+
+(42) HashAggregate [codegen id : 9]
+Input [8]: [cd_gender#25, cd_marital_status#26, cd_education_status#27, cd_purchase_estimate#28, cd_credit_rating#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32]
+Keys [8]: [cd_gender#25, cd_marital_status#26, cd_education_status#27, cd_purchase_estimate#28, cd_credit_rating#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#34]
+Results [9]: [cd_gender#25, cd_marital_status#26, cd_education_status#27, cd_purchase_estimate#28, cd_credit_rating#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32, count#35]
+
+(43) Exchange
+Input [9]: [cd_gender#25, cd_marital_status#26, cd_education_status#27, cd_purchase_estimate#28, cd_credit_rating#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32, count#35]
+Arguments: hashpartitioning(cd_gender#25, cd_marital_status#26, cd_education_status#27, cd_purchase_estimate#28, cd_credit_rating#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32, 5), ENSURE_REQUIREMENTS, [id=#36]
+
+(44) HashAggregate [codegen id : 10]
+Input [9]: [cd_gender#25, cd_marital_status#26, cd_education_status#27, cd_purchase_estimate#28, cd_credit_rating#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32, count#35]
+Keys [8]: [cd_gender#25, cd_marital_status#26, cd_education_status#27, cd_purchase_estimate#28, cd_credit_rating#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#37]
+Results [14]: [cd_gender#25, cd_marital_status#26, cd_education_status#27, count(1)#37 AS cnt1#38, cd_purchase_estimate#28, count(1)#37 AS cnt2#39, cd_credit_rating#29, count(1)#37 AS cnt3#40, cd_dep_count#30, count(1)#37 AS cnt4#41, cd_dep_employed_count#31, count(1)#37 AS cnt5#42, cd_dep_college_count#32, count(1)#37 AS cnt6#43]
+
+(45) TakeOrderedAndProject
+Input [14]: [cd_gender#25, cd_marital_status#26, cd_education_status#27, cnt1#38, cd_purchase_estimate#28, cnt2#39, cd_credit_rating#29, cnt3#40, cd_dep_count#30, cnt4#41, cd_dep_employed_count#31, cnt5#42, cd_dep_college_count#32, cnt6#43]
+Arguments: 100, [cd_gender#25 ASC NULLS FIRST, cd_marital_status#26 ASC NULLS FIRST, cd_education_status#27 ASC NULLS FIRST, cd_purchase_estimate#28 ASC NULLS FIRST, cd_credit_rating#29 ASC NULLS FIRST, cd_dep_count#30 ASC NULLS FIRST, cd_dep_employed_count#31 ASC NULLS FIRST, cd_dep_college_count#32 ASC NULLS FIRST], [cd_gender#25, cd_marital_status#26, cd_education_status#27, cnt1#38, cd_purchase_estimate#28, cnt2#39, cd_credit_rating#29, cnt3#40, cd_dep_count#30, cnt4#41, cd_dep_employed_count#31, cnt5#42, cd_dep_college_count#32, cnt6#43]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (46)
+
+
+(46) ReusedExchange [Reuses operator id: 10]
+Output [1]: [d_date_sk#7]
+
+Subquery:2 Hosting operator id = 15 Hosting Expression = ws_sold_date_sk#13 IN dynamicpruning#6
+
+Subquery:3 Hosting operator id = 20 Hosting Expression = cs_sold_date_sk#17 IN dynamicpruning#6
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/simplified.txt
new file mode 100644
index 0000000000000..5077b91e30c1b
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/simplified.txt
@@ -0,0 +1,72 @@
+TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,cnt2,cnt3,cnt4,cnt5,cnt6]
+  WholeStageCodegen (10)
+    HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] [count(1),cnt1,cnt2,cnt3,cnt4,cnt5,cnt6,count]
+      InputAdapter
+        Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1
+          WholeStageCodegen (9)
+            HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count]
+              Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count]
+                BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk]
+                  Project [c_current_cdemo_sk]
+                    BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                      Project [c_current_cdemo_sk,c_current_addr_sk]
+                        BroadcastHashJoin [c_customer_sk,customer_sk]
+                          BroadcastHashJoin [c_customer_sk,ss_customer_sk]
+                            Filter [c_current_addr_sk,c_current_cdemo_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk]
+                            InputAdapter
+                              BroadcastExchange #2
+                                WholeStageCodegen (2)
+                                  Project [ss_customer_sk]
+                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.store_sales [ss_customer_sk,ss_sold_date_sk]
+                                            SubqueryBroadcast [d_date_sk] #1
+                                              ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        BroadcastExchange #3
+                                          WholeStageCodegen (1)
+                                            Project [d_date_sk]
+                                              Filter [d_year,d_moy,d_date_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                          InputAdapter
+                            BroadcastExchange #4
+                              Union
+                                WholeStageCodegen (4)
+                                  Project [ws_bill_customer_sk]
+                                    BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.web_sales [ws_bill_customer_sk,ws_sold_date_sk]
+                                            ReusedSubquery [d_date_sk] #1
+                                      InputAdapter
+                                        ReusedExchange [d_date_sk] #3
+                                WholeStageCodegen (6)
+                                  Project [cs_ship_customer_sk]
+                                    BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk]
+                                            ReusedSubquery [d_date_sk] #1
+                                      InputAdapter
+                                        ReusedExchange [d_date_sk] #3
+                      InputAdapter
+                        BroadcastExchange #5
+                          WholeStageCodegen (7)
+                            Project [ca_address_sk]
+                              Filter [ca_county,ca_address_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.customer_address [ca_address_sk,ca_county]
+                  InputAdapter
+                    BroadcastExchange #6
+                      WholeStageCodegen (8)
+                        Filter [cd_demo_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.sf100/explain.txt
new file mode 100644
index 0000000000000..453a35d8d5175
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.sf100/explain.txt
@@ -0,0 +1,502 @@
+== Physical Plan ==
+TakeOrderedAndProject (86)
++- * Project (85)
+   +- * SortMergeJoin Inner (84)
+      :- * Project (66)
+      :  +- * SortMergeJoin Inner (65)
+      :     :- * SortMergeJoin Inner (45)
+      :     :  :- * Sort (24)
+      :     :  :  +- Exchange (23)
+      :     :  :     +- * Filter (22)
+      :     :  :        +- * HashAggregate (21)
+      :     :  :           +- Exchange (20)
+      :     :  :              +- * HashAggregate (19)
+      :     :  :                 +- * Project (18)
+      :     :  :                    +- * SortMergeJoin Inner (17)
+      :     :  :                       :- * Sort (11)
+      :     :  :                       :  +- Exchange (10)
+      :     :  :                       :     +- * Project (9)
+      :     :  :                       :        +- * BroadcastHashJoin Inner BuildRight (8)
+      :     :  :                       :           :- * Filter (3)
+      :     :  :                       :           :  +- * ColumnarToRow (2)
+      :     :  :                       :           :     +- Scan parquet default.store_sales (1)
+      :     :  :                       :           +- BroadcastExchange (7)
+      :     :  :                       :              +- * Filter (6)
+      :     :  :                       :                 +- * ColumnarToRow (5)
+      :     :  :                       :                    +- Scan parquet default.date_dim (4)
+      :     :  :                       +- * Sort (16)
+      :     :  :                          +- Exchange (15)
+      :     :  :                             +- * Filter (14)
+      :     :  :                                +- * ColumnarToRow (13)
+      :     :  :                                   +- Scan parquet default.customer (12)
+      :     :  +- * Sort (44)
+      :     :     +- Exchange (43)
+      :     :        +- * HashAggregate (42)
+      :     :           +- Exchange (41)
+      :     :              +- * HashAggregate (40)
+      :     :                 +- * Project (39)
+      :     :                    +- * SortMergeJoin Inner (38)
+      :     :                       :- * Sort (35)
+      :     :                       :  +- Exchange (34)
+      :     :                       :     +- * Project (33)
+      :     :                       :        +- * BroadcastHashJoin Inner BuildRight (32)
+      :     :                       :           :- * Filter (27)
+      :     :                       :           :  +- * ColumnarToRow (26)
+      :     :                       :           :     +- Scan parquet default.store_sales (25)
+      :     :                       :           +- BroadcastExchange (31)
+      :     :                       :              +- * Filter (30)
+      :     :                       :                 +- * ColumnarToRow (29)
+      :     :                       :                    +- Scan parquet default.date_dim (28)
+      :     :                       +- * Sort (37)
+      :     :                          +- ReusedExchange (36)
+      :     +- * Sort (64)
+      :        +- Exchange (63)
+      :           +- * Project (62)
+      :              +- * Filter (61)
+      :                 +- * HashAggregate (60)
+      :                    +- Exchange (59)
+      :                       +- * HashAggregate (58)
+      :                          +- * Project (57)
+      :                             +- * SortMergeJoin Inner (56)
+      :                                :- * Sort (53)
+      :                                :  +- Exchange (52)
+      :                                :     +- * Project (51)
+      :                                :        +- * BroadcastHashJoin Inner BuildRight (50)
+      :                                :           :- * Filter (48)
+      :                                :           :  +- * ColumnarToRow (47)
+      :                                :           :     +- Scan parquet default.web_sales (46)
+      :                                :           +- ReusedExchange (49)
+      :                                +- * Sort (55)
+      :                                   +- ReusedExchange (54)
+      +- * Sort (83)
+         +- Exchange (82)
+            +- * HashAggregate (81)
+               +- Exchange (80)
+                  +- * HashAggregate (79)
+                     +- * Project (78)
+                        +- * SortMergeJoin Inner (77)
+                           :- * Sort (74)
+                           :  +- Exchange (73)
+                           :     +- * Project (72)
+                           :        +- * BroadcastHashJoin Inner BuildRight (71)
+                           :           :- * Filter (69)
+                           :           :  +- * ColumnarToRow (68)
+                           :           :     +- Scan parquet default.web_sales (67)
+                           :           +- ReusedExchange (70)
+                           +- * Sort (76)
+                              +- ReusedExchange (75)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_customer_sk#1, ss_ext_discount_amt#2, ss_ext_list_price#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_ext_discount_amt:decimal(7,2),ss_ext_list_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [4]: [ss_customer_sk#1, ss_ext_discount_amt#2, ss_ext_list_price#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 2]
+Input [4]: [ss_customer_sk#1, ss_ext_discount_amt#2, ss_ext_list_price#3, ss_sold_date_sk#4]
+Condition : isnotnull(ss_customer_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#6, d_year#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#6, d_year#7]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#6, d_year#7]
+Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2001)) AND isnotnull(d_date_sk#6))
+
+(7) BroadcastExchange
+Input [2]: [d_date_sk#6, d_year#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#8]
+
+(8) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(9) Project [codegen id : 2]
+Output [4]: [ss_customer_sk#1, ss_ext_discount_amt#2, ss_ext_list_price#3, d_year#7]
+Input [6]: [ss_customer_sk#1, ss_ext_discount_amt#2, ss_ext_list_price#3, ss_sold_date_sk#4, d_date_sk#6, d_year#7]
+
+(10) Exchange
+Input [4]: [ss_customer_sk#1, ss_ext_discount_amt#2, ss_ext_list_price#3, d_year#7]
+Arguments: hashpartitioning(ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#9]
+
+(11) Sort [codegen id : 3]
+Input [4]: [ss_customer_sk#1, ss_ext_discount_amt#2, ss_ext_list_price#3, d_year#7]
+Arguments: [ss_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(12) Scan parquet default.customer
+Output [8]: [c_customer_sk#10, c_customer_id#11, c_first_name#12, c_last_name#13, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string,c_birth_country:string,c_login:string,c_email_address:string>
+
+(13) ColumnarToRow [codegen id : 4]
+Input [8]: [c_customer_sk#10, c_customer_id#11, c_first_name#12, c_last_name#13, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17]
+
+(14) Filter [codegen id : 4]
+Input [8]: [c_customer_sk#10, c_customer_id#11, c_first_name#12, c_last_name#13, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17]
+Condition : (isnotnull(c_customer_sk#10) AND isnotnull(c_customer_id#11))
+
+(15) Exchange
+Input [8]: [c_customer_sk#10, c_customer_id#11, c_first_name#12, c_last_name#13, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17]
+Arguments: hashpartitioning(c_customer_sk#10, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(16) Sort [codegen id : 5]
+Input [8]: [c_customer_sk#10, c_customer_id#11, c_first_name#12, c_last_name#13, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17]
+Arguments: [c_customer_sk#10 ASC NULLS FIRST], false, 0
+
+(17) SortMergeJoin [codegen id : 6]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#10]
+Join condition: None
+
+(18) Project [codegen id : 6]
+Output [10]: [c_customer_id#11, c_first_name#12, c_last_name#13, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17, ss_ext_discount_amt#2, ss_ext_list_price#3, d_year#7]
+Input [12]: [ss_customer_sk#1, ss_ext_discount_amt#2, ss_ext_list_price#3, d_year#7, c_customer_sk#10, c_customer_id#11, c_first_name#12, c_last_name#13, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17]
+
+(19) HashAggregate [codegen id : 6]
+Input [10]: [c_customer_id#11, c_first_name#12, c_last_name#13, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17, ss_ext_discount_amt#2, ss_ext_list_price#3, d_year#7]
+Keys [8]: [c_customer_id#11, c_first_name#12, c_last_name#13, d_year#7, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17]
+Functions [1]: [partial_sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#3 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#2 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum#19]
+Results [9]: [c_customer_id#11, c_first_name#12, c_last_name#13, d_year#7, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17, sum#20]
+
+(20) Exchange
+Input [9]: [c_customer_id#11, c_first_name#12, c_last_name#13, d_year#7, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17, sum#20]
+Arguments: hashpartitioning(c_customer_id#11, c_first_name#12, c_last_name#13, d_year#7, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(21) HashAggregate [codegen id : 7]
+Input [9]: [c_customer_id#11, c_first_name#12, c_last_name#13, d_year#7, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17, sum#20]
+Keys [8]: [c_customer_id#11, c_first_name#12, c_last_name#13, d_year#7, c_preferred_cust_flag#14, c_birth_country#15, c_login#16, c_email_address#17]
+Functions [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#3 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#2 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#3 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#2 as decimal(8,2)))), DecimalType(8,2), true)))#22]
+Results [2]: [c_customer_id#11 AS customer_id#23, MakeDecimal(sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#3 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#2 as decimal(8,2)))), DecimalType(8,2), true)))#22,18,2) AS year_total#24]
+
+(22) Filter [codegen id : 7]
+Input [2]: [customer_id#23, year_total#24]
+Condition : (isnotnull(year_total#24) AND (year_total#24 > 0.00))
+
+(23) Exchange
+Input [2]: [customer_id#23, year_total#24]
+Arguments: hashpartitioning(customer_id#23, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(24) Sort [codegen id : 8]
+Input [2]: [customer_id#23, year_total#24]
+Arguments: [customer_id#23 ASC NULLS FIRST], false, 0
+
+(25) Scan parquet default.store_sales
+Output [4]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#29), dynamicpruningexpression(ss_sold_date_sk#29 IN dynamicpruning#30)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_ext_discount_amt:decimal(7,2),ss_ext_list_price:decimal(7,2)>
+
+(26) ColumnarToRow [codegen id : 10]
+Input [4]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29]
+
+(27) Filter [codegen id : 10]
+Input [4]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29]
+Condition : isnotnull(ss_customer_sk#26)
+
+(28) Scan parquet default.date_dim
+Output [2]: [d_date_sk#31, d_year#32]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(29) ColumnarToRow [codegen id : 9]
+Input [2]: [d_date_sk#31, d_year#32]
+
+(30) Filter [codegen id : 9]
+Input [2]: [d_date_sk#31, d_year#32]
+Condition : ((isnotnull(d_year#32) AND (d_year#32 = 2002)) AND isnotnull(d_date_sk#31))
+
+(31) BroadcastExchange
+Input [2]: [d_date_sk#31, d_year#32]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#33]
+
+(32) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_sold_date_sk#29]
+Right keys [1]: [d_date_sk#31]
+Join condition: None
+
+(33) Project [codegen id : 10]
+Output [4]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#32]
+Input [6]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29, d_date_sk#31, d_year#32]
+
+(34) Exchange
+Input [4]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#32]
+Arguments: hashpartitioning(ss_customer_sk#26, 5), ENSURE_REQUIREMENTS, [id=#34]
+
+(35) Sort [codegen id : 11]
+Input [4]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#32]
+Arguments: [ss_customer_sk#26 ASC NULLS FIRST], false, 0
+
+(36) ReusedExchange [Reuses operator id: 15]
+Output [8]: [c_customer_sk#35, c_customer_id#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_country#40, c_login#41, c_email_address#42]
+
+(37) Sort [codegen id : 13]
+Input [8]: [c_customer_sk#35, c_customer_id#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_country#40, c_login#41, c_email_address#42]
+Arguments: [c_customer_sk#35 ASC NULLS FIRST], false, 0
+
+(38) SortMergeJoin [codegen id : 14]
+Left keys [1]: [ss_customer_sk#26]
+Right keys [1]: [c_customer_sk#35]
+Join condition: None
+
+(39) Project [codegen id : 14]
+Output [10]: [c_customer_id#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_country#40, c_login#41, c_email_address#42, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#32]
+Input [12]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#32, c_customer_sk#35, c_customer_id#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_country#40, c_login#41, c_email_address#42]
+
+(40) HashAggregate [codegen id : 14]
+Input [10]: [c_customer_id#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_country#40, c_login#41, c_email_address#42, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#32]
+Keys [8]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#32, c_preferred_cust_flag#39, c_birth_country#40, c_login#41, c_email_address#42]
+Functions [1]: [partial_sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#28 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#27 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum#43]
+Results [9]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#32, c_preferred_cust_flag#39, c_birth_country#40, c_login#41, c_email_address#42, sum#44]
+
+(41) Exchange
+Input [9]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#32, c_preferred_cust_flag#39, c_birth_country#40, c_login#41, c_email_address#42, sum#44]
+Arguments: hashpartitioning(c_customer_id#36, c_first_name#37, c_last_name#38, d_year#32, c_preferred_cust_flag#39, c_birth_country#40, c_login#41, c_email_address#42, 5), ENSURE_REQUIREMENTS, [id=#45]
+
+(42) HashAggregate [codegen id : 15]
+Input [9]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#32, c_preferred_cust_flag#39, c_birth_country#40, c_login#41, c_email_address#42, sum#44]
+Keys [8]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#32, c_preferred_cust_flag#39, c_birth_country#40, c_login#41, c_email_address#42]
+Functions [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#28 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#27 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#28 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#27 as decimal(8,2)))), DecimalType(8,2), true)))#46]
+Results [5]: [c_customer_id#36 AS customer_id#47, c_first_name#37 AS customer_first_name#48, c_last_name#38 AS customer_last_name#49, c_email_address#42 AS customer_email_address#50, MakeDecimal(sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#28 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#27 as decimal(8,2)))), DecimalType(8,2), true)))#46,18,2) AS year_total#51]
+
+(43) Exchange
+Input [5]: [customer_id#47, customer_first_name#48, customer_last_name#49, customer_email_address#50, year_total#51]
+Arguments: hashpartitioning(customer_id#47, 5), ENSURE_REQUIREMENTS, [id=#52]
+
+(44) Sort [codegen id : 16]
+Input [5]: [customer_id#47, customer_first_name#48, customer_last_name#49, customer_email_address#50, year_total#51]
+Arguments: [customer_id#47 ASC NULLS FIRST], false, 0
+
+(45) SortMergeJoin [codegen id : 17]
+Left keys [1]: [customer_id#23]
+Right keys [1]: [customer_id#47]
+Join condition: None
+
+(46) Scan parquet default.web_sales
+Output [4]: [ws_bill_customer_sk#53, ws_ext_discount_amt#54, ws_ext_list_price#55, ws_sold_date_sk#56]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#56), dynamicpruningexpression(ws_sold_date_sk#56 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int,ws_ext_discount_amt:decimal(7,2),ws_ext_list_price:decimal(7,2)>
+
+(47) ColumnarToRow [codegen id : 19]
+Input [4]: [ws_bill_customer_sk#53, ws_ext_discount_amt#54, ws_ext_list_price#55, ws_sold_date_sk#56]
+
+(48) Filter [codegen id : 19]
+Input [4]: [ws_bill_customer_sk#53, ws_ext_discount_amt#54, ws_ext_list_price#55, ws_sold_date_sk#56]
+Condition : isnotnull(ws_bill_customer_sk#53)
+
+(49) ReusedExchange [Reuses operator id: 7]
+Output [2]: [d_date_sk#57, d_year#58]
+
+(50) BroadcastHashJoin [codegen id : 19]
+Left keys [1]: [ws_sold_date_sk#56]
+Right keys [1]: [d_date_sk#57]
+Join condition: None
+
+(51) Project [codegen id : 19]
+Output [4]: [ws_bill_customer_sk#53, ws_ext_discount_amt#54, ws_ext_list_price#55, d_year#58]
+Input [6]: [ws_bill_customer_sk#53, ws_ext_discount_amt#54, ws_ext_list_price#55, ws_sold_date_sk#56, d_date_sk#57, d_year#58]
+
+(52) Exchange
+Input [4]: [ws_bill_customer_sk#53, ws_ext_discount_amt#54, ws_ext_list_price#55, d_year#58]
+Arguments: hashpartitioning(ws_bill_customer_sk#53, 5), ENSURE_REQUIREMENTS, [id=#59]
+
+(53) Sort [codegen id : 20]
+Input [4]: [ws_bill_customer_sk#53, ws_ext_discount_amt#54, ws_ext_list_price#55, d_year#58]
+Arguments: [ws_bill_customer_sk#53 ASC NULLS FIRST], false, 0
+
+(54) ReusedExchange [Reuses operator id: 15]
+Output [8]: [c_customer_sk#60, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67]
+
+(55) Sort [codegen id : 22]
+Input [8]: [c_customer_sk#60, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67]
+Arguments: [c_customer_sk#60 ASC NULLS FIRST], false, 0
+
+(56) SortMergeJoin [codegen id : 23]
+Left keys [1]: [ws_bill_customer_sk#53]
+Right keys [1]: [c_customer_sk#60]
+Join condition: None
+
+(57) Project [codegen id : 23]
+Output [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, ws_ext_discount_amt#54, ws_ext_list_price#55, d_year#58]
+Input [12]: [ws_bill_customer_sk#53, ws_ext_discount_amt#54, ws_ext_list_price#55, d_year#58, c_customer_sk#60, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67]
+
+(58) HashAggregate [codegen id : 23]
+Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, ws_ext_discount_amt#54, ws_ext_list_price#55, d_year#58]
+Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, d_year#58]
+Functions [1]: [partial_sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#55 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#54 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum#68]
+Results [9]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, d_year#58, sum#69]
+
+(59) Exchange
+Input [9]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, d_year#58, sum#69]
+Arguments: hashpartitioning(c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, d_year#58, 5), ENSURE_REQUIREMENTS, [id=#70]
+
+(60) HashAggregate [codegen id : 24]
+Input [9]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, d_year#58, sum#69]
+Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, d_year#58]
+Functions [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#55 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#54 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#55 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#54 as decimal(8,2)))), DecimalType(8,2), true)))#71]
+Results [2]: [c_customer_id#61 AS customer_id#72, MakeDecimal(sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#55 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#54 as decimal(8,2)))), DecimalType(8,2), true)))#71,18,2) AS year_total#73]
+
+(61) Filter [codegen id : 24]
+Input [2]: [customer_id#72, year_total#73]
+Condition : (isnotnull(year_total#73) AND (year_total#73 > 0.00))
+
+(62) Project [codegen id : 24]
+Output [2]: [customer_id#72 AS customer_id#74, year_total#73 AS year_total#75]
+Input [2]: [customer_id#72, year_total#73]
+
+(63) Exchange
+Input [2]: [customer_id#74, year_total#75]
+Arguments: hashpartitioning(customer_id#74, 5), ENSURE_REQUIREMENTS, [id=#76]
+
+(64) Sort [codegen id : 25]
+Input [2]: [customer_id#74, year_total#75]
+Arguments: [customer_id#74 ASC NULLS FIRST], false, 0
+
+(65) SortMergeJoin [codegen id : 26]
+Left keys [1]: [customer_id#23]
+Right keys [1]: [customer_id#74]
+Join condition: None
+
+(66) Project [codegen id : 26]
+Output [8]: [customer_id#23, year_total#24, customer_id#47, customer_first_name#48, customer_last_name#49, customer_email_address#50, year_total#51, year_total#75]
+Input [9]: [customer_id#23, year_total#24, customer_id#47, customer_first_name#48, customer_last_name#49, customer_email_address#50, year_total#51, customer_id#74, year_total#75]
+
+(67) Scan parquet default.web_sales
+Output [4]: [ws_bill_customer_sk#77, ws_ext_discount_amt#78, ws_ext_list_price#79, ws_sold_date_sk#80]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#80), dynamicpruningexpression(ws_sold_date_sk#80 IN dynamicpruning#30)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int,ws_ext_discount_amt:decimal(7,2),ws_ext_list_price:decimal(7,2)>
+
+(68) ColumnarToRow [codegen id : 28]
+Input [4]: [ws_bill_customer_sk#77, ws_ext_discount_amt#78, ws_ext_list_price#79, ws_sold_date_sk#80]
+
+(69) Filter [codegen id : 28]
+Input [4]: [ws_bill_customer_sk#77, ws_ext_discount_amt#78, ws_ext_list_price#79, ws_sold_date_sk#80]
+Condition : isnotnull(ws_bill_customer_sk#77)
+
+(70) ReusedExchange [Reuses operator id: 31]
+Output [2]: [d_date_sk#81, d_year#82]
+
+(71) BroadcastHashJoin [codegen id : 28]
+Left keys [1]: [ws_sold_date_sk#80]
+Right keys [1]: [d_date_sk#81]
+Join condition: None
+
+(72) Project [codegen id : 28]
+Output [4]: [ws_bill_customer_sk#77, ws_ext_discount_amt#78, ws_ext_list_price#79, d_year#82]
+Input [6]: [ws_bill_customer_sk#77, ws_ext_discount_amt#78, ws_ext_list_price#79, ws_sold_date_sk#80, d_date_sk#81, d_year#82]
+
+(73) Exchange
+Input [4]: [ws_bill_customer_sk#77, ws_ext_discount_amt#78, ws_ext_list_price#79, d_year#82]
+Arguments: hashpartitioning(ws_bill_customer_sk#77, 5), ENSURE_REQUIREMENTS, [id=#83]
+
+(74) Sort [codegen id : 29]
+Input [4]: [ws_bill_customer_sk#77, ws_ext_discount_amt#78, ws_ext_list_price#79, d_year#82]
+Arguments: [ws_bill_customer_sk#77 ASC NULLS FIRST], false, 0
+
+(75) ReusedExchange [Reuses operator id: 15]
+Output [8]: [c_customer_sk#84, c_customer_id#85, c_first_name#86, c_last_name#87, c_preferred_cust_flag#88, c_birth_country#89, c_login#90, c_email_address#91]
+
+(76) Sort [codegen id : 31]
+Input [8]: [c_customer_sk#84, c_customer_id#85, c_first_name#86, c_last_name#87, c_preferred_cust_flag#88, c_birth_country#89, c_login#90, c_email_address#91]
+Arguments: [c_customer_sk#84 ASC NULLS FIRST], false, 0
+
+(77) SortMergeJoin [codegen id : 32]
+Left keys [1]: [ws_bill_customer_sk#77]
+Right keys [1]: [c_customer_sk#84]
+Join condition: None
+
+(78) Project [codegen id : 32]
+Output [10]: [c_customer_id#85, c_first_name#86, c_last_name#87, c_preferred_cust_flag#88, c_birth_country#89, c_login#90, c_email_address#91, ws_ext_discount_amt#78, ws_ext_list_price#79, d_year#82]
+Input [12]: [ws_bill_customer_sk#77, ws_ext_discount_amt#78, ws_ext_list_price#79, d_year#82, c_customer_sk#84, c_customer_id#85, c_first_name#86, c_last_name#87, c_preferred_cust_flag#88, c_birth_country#89, c_login#90, c_email_address#91]
+
+(79) HashAggregate [codegen id : 32]
+Input [10]: [c_customer_id#85, c_first_name#86, c_last_name#87, c_preferred_cust_flag#88, c_birth_country#89, c_login#90, c_email_address#91, ws_ext_discount_amt#78, ws_ext_list_price#79, d_year#82]
+Keys [8]: [c_customer_id#85, c_first_name#86, c_last_name#87, c_preferred_cust_flag#88, c_birth_country#89, c_login#90, c_email_address#91, d_year#82]
+Functions [1]: [partial_sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#79 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#78 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum#92]
+Results [9]: [c_customer_id#85, c_first_name#86, c_last_name#87, c_preferred_cust_flag#88, c_birth_country#89, c_login#90, c_email_address#91, d_year#82, sum#93]
+
+(80) Exchange
+Input [9]: [c_customer_id#85, c_first_name#86, c_last_name#87, c_preferred_cust_flag#88, c_birth_country#89, c_login#90, c_email_address#91, d_year#82, sum#93]
+Arguments: hashpartitioning(c_customer_id#85, c_first_name#86, c_last_name#87, c_preferred_cust_flag#88, c_birth_country#89, c_login#90, c_email_address#91, d_year#82, 5), ENSURE_REQUIREMENTS, [id=#94]
+
+(81) HashAggregate [codegen id : 33]
+Input [9]: [c_customer_id#85, c_first_name#86, c_last_name#87, c_preferred_cust_flag#88, c_birth_country#89, c_login#90, c_email_address#91, d_year#82, sum#93]
+Keys [8]: [c_customer_id#85, c_first_name#86, c_last_name#87, c_preferred_cust_flag#88, c_birth_country#89, c_login#90, c_email_address#91, d_year#82]
+Functions [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#79 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#78 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#79 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#78 as decimal(8,2)))), DecimalType(8,2), true)))#95]
+Results [2]: [c_customer_id#85 AS customer_id#96, MakeDecimal(sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#79 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#78 as decimal(8,2)))), DecimalType(8,2), true)))#95,18,2) AS year_total#97]
+
+(82) Exchange
+Input [2]: [customer_id#96, year_total#97]
+Arguments: hashpartitioning(customer_id#96, 5), ENSURE_REQUIREMENTS, [id=#98]
+
+(83) Sort [codegen id : 34]
+Input [2]: [customer_id#96, year_total#97]
+Arguments: [customer_id#96 ASC NULLS FIRST], false, 0
+
+(84) SortMergeJoin [codegen id : 35]
+Left keys [1]: [customer_id#23]
+Right keys [1]: [customer_id#96]
+Join condition: (CASE WHEN (year_total#75 > 0.00) THEN CheckOverflow((promote_precision(year_total#97) / promote_precision(year_total#75)), DecimalType(38,20), true) ELSE 0E-20 END > CASE WHEN (year_total#24 > 0.00) THEN CheckOverflow((promote_precision(year_total#51) / promote_precision(year_total#24)), DecimalType(38,20), true) ELSE 0E-20 END)
+
+(85) Project [codegen id : 35]
+Output [4]: [customer_id#47, customer_first_name#48, customer_last_name#49, customer_email_address#50]
+Input [10]: [customer_id#23, year_total#24, customer_id#47, customer_first_name#48, customer_last_name#49, customer_email_address#50, year_total#51, year_total#75, customer_id#96, year_total#97]
+
+(86) TakeOrderedAndProject
+Input [4]: [customer_id#47, customer_first_name#48, customer_last_name#49, customer_email_address#50]
+Arguments: 100, [customer_id#47 ASC NULLS FIRST, customer_first_name#48 ASC NULLS FIRST, customer_last_name#49 ASC NULLS FIRST, customer_email_address#50 ASC NULLS FIRST], [customer_id#47, customer_first_name#48, customer_last_name#49, customer_email_address#50]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (87)
+
+
+(87) ReusedExchange [Reuses operator id: 7]
+Output [2]: [d_date_sk#6, d_year#7]
+
+Subquery:2 Hosting operator id = 25 Hosting Expression = ss_sold_date_sk#29 IN dynamicpruning#30
+ReusedExchange (88)
+
+
+(88) ReusedExchange [Reuses operator id: 31]
+Output [2]: [d_date_sk#31, d_year#32]
+
+Subquery:3 Hosting operator id = 46 Hosting Expression = ws_sold_date_sk#56 IN dynamicpruning#5
+
+Subquery:4 Hosting operator id = 67 Hosting Expression = ws_sold_date_sk#80 IN dynamicpruning#30
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.sf100/simplified.txt
new file mode 100644
index 0000000000000..67f6c90f67792
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.sf100/simplified.txt
@@ -0,0 +1,163 @@
+TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_email_address]
+  WholeStageCodegen (35)
+    Project [customer_id,customer_first_name,customer_last_name,customer_email_address]
+      SortMergeJoin [customer_id,customer_id,year_total,year_total,year_total,year_total]
+        InputAdapter
+          WholeStageCodegen (26)
+            Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,year_total]
+              SortMergeJoin [customer_id,customer_id]
+                InputAdapter
+                  WholeStageCodegen (17)
+                    SortMergeJoin [customer_id,customer_id]
+                      InputAdapter
+                        WholeStageCodegen (8)
+                          Sort [customer_id]
+                            InputAdapter
+                              Exchange [customer_id] #1
+                                WholeStageCodegen (7)
+                                  Filter [year_total]
+                                    HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt as decimal(8,2)))), DecimalType(8,2), true))),customer_id,year_total,sum]
+                                      InputAdapter
+                                        Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #2
+                                          WholeStageCodegen (6)
+                                            HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum]
+                                              Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year]
+                                                SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (3)
+                                                      Sort [ss_customer_sk]
+                                                        InputAdapter
+                                                          Exchange [ss_customer_sk] #3
+                                                            WholeStageCodegen (2)
+                                                              Project [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,d_year]
+                                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                  Filter [ss_customer_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk]
+                                                                          SubqueryBroadcast [d_date_sk] #1
+                                                                            ReusedExchange [d_date_sk,d_year] #4
+                                                                  InputAdapter
+                                                                    BroadcastExchange #4
+                                                                      WholeStageCodegen (1)
+                                                                        Filter [d_year,d_date_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.date_dim [d_date_sk,d_year]
+                                                  InputAdapter
+                                                    WholeStageCodegen (5)
+                                                      Sort [c_customer_sk]
+                                                        InputAdapter
+                                                          Exchange [c_customer_sk] #5
+                                                            WholeStageCodegen (4)
+                                                              Filter [c_customer_sk,c_customer_id]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address]
+                      InputAdapter
+                        WholeStageCodegen (16)
+                          Sort [customer_id]
+                            InputAdapter
+                              Exchange [customer_id] #6
+                                WholeStageCodegen (15)
+                                  HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt as decimal(8,2)))), DecimalType(8,2), true))),customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,sum]
+                                    InputAdapter
+                                      Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #7
+                                        WholeStageCodegen (14)
+                                          HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum]
+                                            Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year]
+                                              SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                                InputAdapter
+                                                  WholeStageCodegen (11)
+                                                    Sort [ss_customer_sk]
+                                                      InputAdapter
+                                                        Exchange [ss_customer_sk] #8
+                                                          WholeStageCodegen (10)
+                                                            Project [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,d_year]
+                                                              BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                Filter [ss_customer_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk]
+                                                                        SubqueryBroadcast [d_date_sk] #2
+                                                                          ReusedExchange [d_date_sk,d_year] #9
+                                                                InputAdapter
+                                                                  BroadcastExchange #9
+                                                                    WholeStageCodegen (9)
+                                                                      Filter [d_year,d_date_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.date_dim [d_date_sk,d_year]
+                                                InputAdapter
+                                                  WholeStageCodegen (13)
+                                                    Sort [c_customer_sk]
+                                                      InputAdapter
+                                                        ReusedExchange [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5
+                InputAdapter
+                  WholeStageCodegen (25)
+                    Sort [customer_id]
+                      InputAdapter
+                        Exchange [customer_id] #10
+                          WholeStageCodegen (24)
+                            Project [customer_id,year_total]
+                              Filter [year_total]
+                                HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt as decimal(8,2)))), DecimalType(8,2), true))),customer_id,year_total,sum]
+                                  InputAdapter
+                                    Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11
+                                      WholeStageCodegen (23)
+                                        HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum]
+                                          Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year]
+                                            SortMergeJoin [ws_bill_customer_sk,c_customer_sk]
+                                              InputAdapter
+                                                WholeStageCodegen (20)
+                                                  Sort [ws_bill_customer_sk]
+                                                    InputAdapter
+                                                      Exchange [ws_bill_customer_sk] #12
+                                                        WholeStageCodegen (19)
+                                                          Project [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,d_year]
+                                                            BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                              Filter [ws_bill_customer_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk]
+                                                                      ReusedSubquery [d_date_sk] #1
+                                                              InputAdapter
+                                                                ReusedExchange [d_date_sk,d_year] #4
+                                              InputAdapter
+                                                WholeStageCodegen (22)
+                                                  Sort [c_customer_sk]
+                                                    InputAdapter
+                                                      ReusedExchange [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5
+        InputAdapter
+          WholeStageCodegen (34)
+            Sort [customer_id]
+              InputAdapter
+                Exchange [customer_id] #13
+                  WholeStageCodegen (33)
+                    HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt as decimal(8,2)))), DecimalType(8,2), true))),customer_id,year_total,sum]
+                      InputAdapter
+                        Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14
+                          WholeStageCodegen (32)
+                            HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum]
+                              Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year]
+                                SortMergeJoin [ws_bill_customer_sk,c_customer_sk]
+                                  InputAdapter
+                                    WholeStageCodegen (29)
+                                      Sort [ws_bill_customer_sk]
+                                        InputAdapter
+                                          Exchange [ws_bill_customer_sk] #15
+                                            WholeStageCodegen (28)
+                                              Project [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,d_year]
+                                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                  Filter [ws_bill_customer_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk]
+                                                          ReusedSubquery [d_date_sk] #2
+                                                  InputAdapter
+                                                    ReusedExchange [d_date_sk,d_year] #9
+                                  InputAdapter
+                                    WholeStageCodegen (31)
+                                      Sort [c_customer_sk]
+                                        InputAdapter
+                                          ReusedExchange [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/explain.txt
new file mode 100644
index 0000000000000..17d16033bb5be
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/explain.txt
@@ -0,0 +1,471 @@
+== Physical Plan ==
+TakeOrderedAndProject (78)
++- * Project (77)
+   +- * BroadcastHashJoin Inner BuildRight (76)
+      :- * Project (59)
+      :  +- * BroadcastHashJoin Inner BuildRight (58)
+      :     :- * BroadcastHashJoin Inner BuildRight (39)
+      :     :  :- * Filter (19)
+      :     :  :  +- * HashAggregate (18)
+      :     :  :     +- Exchange (17)
+      :     :  :        +- * HashAggregate (16)
+      :     :  :           +- * Project (15)
+      :     :  :              +- * BroadcastHashJoin Inner BuildRight (14)
+      :     :  :                 :- * Project (9)
+      :     :  :                 :  +- * BroadcastHashJoin Inner BuildRight (8)
+      :     :  :                 :     :- * Filter (3)
+      :     :  :                 :     :  +- * ColumnarToRow (2)
+      :     :  :                 :     :     +- Scan parquet default.customer (1)
+      :     :  :                 :     +- BroadcastExchange (7)
+      :     :  :                 :        +- * Filter (6)
+      :     :  :                 :           +- * ColumnarToRow (5)
+      :     :  :                 :              +- Scan parquet default.store_sales (4)
+      :     :  :                 +- BroadcastExchange (13)
+      :     :  :                    +- * Filter (12)
+      :     :  :                       +- * ColumnarToRow (11)
+      :     :  :                          +- Scan parquet default.date_dim (10)
+      :     :  +- BroadcastExchange (38)
+      :     :     +- * HashAggregate (37)
+      :     :        +- Exchange (36)
+      :     :           +- * HashAggregate (35)
+      :     :              +- * Project (34)
+      :     :                 +- * BroadcastHashJoin Inner BuildRight (33)
+      :     :                    :- * Project (28)
+      :     :                    :  +- * BroadcastHashJoin Inner BuildRight (27)
+      :     :                    :     :- * Filter (22)
+      :     :                    :     :  +- * ColumnarToRow (21)
+      :     :                    :     :     +- Scan parquet default.customer (20)
+      :     :                    :     +- BroadcastExchange (26)
+      :     :                    :        +- * Filter (25)
+      :     :                    :           +- * ColumnarToRow (24)
+      :     :                    :              +- Scan parquet default.store_sales (23)
+      :     :                    +- BroadcastExchange (32)
+      :     :                       +- * Filter (31)
+      :     :                          +- * ColumnarToRow (30)
+      :     :                             +- Scan parquet default.date_dim (29)
+      :     +- BroadcastExchange (57)
+      :        +- * Project (56)
+      :           +- * Filter (55)
+      :              +- * HashAggregate (54)
+      :                 +- Exchange (53)
+      :                    +- * HashAggregate (52)
+      :                       +- * Project (51)
+      :                          +- * BroadcastHashJoin Inner BuildRight (50)
+      :                             :- * Project (48)
+      :                             :  +- * BroadcastHashJoin Inner BuildRight (47)
+      :                             :     :- * Filter (42)
+      :                             :     :  +- * ColumnarToRow (41)
+      :                             :     :     +- Scan parquet default.customer (40)
+      :                             :     +- BroadcastExchange (46)
+      :                             :        +- * Filter (45)
+      :                             :           +- * ColumnarToRow (44)
+      :                             :              +- Scan parquet default.web_sales (43)
+      :                             +- ReusedExchange (49)
+      +- BroadcastExchange (75)
+         +- * HashAggregate (74)
+            +- Exchange (73)
+               +- * HashAggregate (72)
+                  +- * Project (71)
+                     +- * BroadcastHashJoin Inner BuildRight (70)
+                        :- * Project (68)
+                        :  +- * BroadcastHashJoin Inner BuildRight (67)
+                        :     :- * Filter (62)
+                        :     :  +- * ColumnarToRow (61)
+                        :     :     +- Scan parquet default.customer (60)
+                        :     +- BroadcastExchange (66)
+                        :        +- * Filter (65)
+                        :           +- * ColumnarToRow (64)
+                        :              +- Scan parquet default.web_sales (63)
+                        +- ReusedExchange (69)
+
+
+(1) Scan parquet default.customer
+Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string,c_birth_country:string,c_login:string,c_email_address:string>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8]
+
+(3) Filter [codegen id : 3]
+Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8]
+Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2))
+
+(4) Scan parquet default.store_sales
+Output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_sold_date_sk#12 IN dynamicpruning#13)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_ext_discount_amt:decimal(7,2),ss_ext_list_price:decimal(7,2)>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12]
+
+(6) Filter [codegen id : 1]
+Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12]
+Condition : isnotnull(ss_customer_sk#9)
+
+(7) BroadcastExchange
+Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#14]
+
+(8) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [ss_customer_sk#9]
+Join condition: None
+
+(9) Project [codegen id : 3]
+Output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12]
+Input [12]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12]
+
+(10) Scan parquet default.date_dim
+Output [2]: [d_date_sk#15, d_year#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#15, d_year#16]
+
+(12) Filter [codegen id : 2]
+Input [2]: [d_date_sk#15, d_year#16]
+Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15))
+
+(13) BroadcastExchange
+Input [2]: [d_date_sk#15, d_year#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#17]
+
+(14) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#12]
+Right keys [1]: [d_date_sk#15]
+Join condition: None
+
+(15) Project [codegen id : 3]
+Output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#16]
+Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12, d_date_sk#15, d_year#16]
+
+(16) HashAggregate [codegen id : 3]
+Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#16]
+Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#16, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8]
+Functions [1]: [partial_sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#11 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#10 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum#18]
+Results [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#16, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#19]
+
+(17) Exchange
+Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#16, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#19]
+Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#16, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(18) HashAggregate [codegen id : 16]
+Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#16, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#19]
+Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#16, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8]
+Functions [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#11 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#10 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#11 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#10 as decimal(8,2)))), DecimalType(8,2), true)))#21]
+Results [2]: [c_customer_id#2 AS customer_id#22, MakeDecimal(sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#11 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#10 as decimal(8,2)))), DecimalType(8,2), true)))#21,18,2) AS year_total#23]
+
+(19) Filter [codegen id : 16]
+Input [2]: [customer_id#22, year_total#23]
+Condition : (isnotnull(year_total#23) AND (year_total#23 > 0.00))
+
+(20) Scan parquet default.customer
+Output [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string,c_birth_country:string,c_login:string,c_email_address:string>
+
+(21) ColumnarToRow [codegen id : 6]
+Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31]
+
+(22) Filter [codegen id : 6]
+Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31]
+Condition : (isnotnull(c_customer_sk#24) AND isnotnull(c_customer_id#25))
+
+(23) Scan parquet default.store_sales
+Output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#35), dynamicpruningexpression(ss_sold_date_sk#35 IN dynamicpruning#36)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_ext_discount_amt:decimal(7,2),ss_ext_list_price:decimal(7,2)>
+
+(24) ColumnarToRow [codegen id : 4]
+Input [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35]
+
+(25) Filter [codegen id : 4]
+Input [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35]
+Condition : isnotnull(ss_customer_sk#32)
+
+(26) BroadcastExchange
+Input [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#37]
+
+(27) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [c_customer_sk#24]
+Right keys [1]: [ss_customer_sk#32]
+Join condition: None
+
+(28) Project [codegen id : 6]
+Output [10]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35]
+Input [12]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35]
+
+(29) Scan parquet default.date_dim
+Output [2]: [d_date_sk#38, d_year#39]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(30) ColumnarToRow [codegen id : 5]
+Input [2]: [d_date_sk#38, d_year#39]
+
+(31) Filter [codegen id : 5]
+Input [2]: [d_date_sk#38, d_year#39]
+Condition : ((isnotnull(d_year#39) AND (d_year#39 = 2002)) AND isnotnull(d_date_sk#38))
+
+(32) BroadcastExchange
+Input [2]: [d_date_sk#38, d_year#39]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#40]
+
+(33) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_sold_date_sk#35]
+Right keys [1]: [d_date_sk#38]
+Join condition: None
+
+(34) Project [codegen id : 6]
+Output [10]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#39]
+Input [12]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35, d_date_sk#38, d_year#39]
+
+(35) HashAggregate [codegen id : 6]
+Input [10]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#39]
+Keys [8]: [c_customer_id#25, c_first_name#26, c_last_name#27, d_year#39, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31]
+Functions [1]: [partial_sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#34 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#33 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum#41]
+Results [9]: [c_customer_id#25, c_first_name#26, c_last_name#27, d_year#39, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, sum#42]
+
+(36) Exchange
+Input [9]: [c_customer_id#25, c_first_name#26, c_last_name#27, d_year#39, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, sum#42]
+Arguments: hashpartitioning(c_customer_id#25, c_first_name#26, c_last_name#27, d_year#39, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, 5), ENSURE_REQUIREMENTS, [id=#43]
+
+(37) HashAggregate [codegen id : 7]
+Input [9]: [c_customer_id#25, c_first_name#26, c_last_name#27, d_year#39, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, sum#42]
+Keys [8]: [c_customer_id#25, c_first_name#26, c_last_name#27, d_year#39, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31]
+Functions [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#34 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#33 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#34 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#33 as decimal(8,2)))), DecimalType(8,2), true)))#44]
+Results [5]: [c_customer_id#25 AS customer_id#45, c_first_name#26 AS customer_first_name#46, c_last_name#27 AS customer_last_name#47, c_email_address#31 AS customer_email_address#48, MakeDecimal(sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price#34 as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt#33 as decimal(8,2)))), DecimalType(8,2), true)))#44,18,2) AS year_total#49]
+
+(38) BroadcastExchange
+Input [5]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_email_address#48, year_total#49]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#50]
+
+(39) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [customer_id#22]
+Right keys [1]: [customer_id#45]
+Join condition: None
+
+(40) Scan parquet default.customer
+Output [8]: [c_customer_sk#51, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string,c_birth_country:string,c_login:string,c_email_address:string>
+
+(41) ColumnarToRow [codegen id : 10]
+Input [8]: [c_customer_sk#51, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58]
+
+(42) Filter [codegen id : 10]
+Input [8]: [c_customer_sk#51, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58]
+Condition : (isnotnull(c_customer_sk#51) AND isnotnull(c_customer_id#52))
+
+(43) Scan parquet default.web_sales
+Output [4]: [ws_bill_customer_sk#59, ws_ext_discount_amt#60, ws_ext_list_price#61, ws_sold_date_sk#62]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#62), dynamicpruningexpression(ws_sold_date_sk#62 IN dynamicpruning#13)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int,ws_ext_discount_amt:decimal(7,2),ws_ext_list_price:decimal(7,2)>
+
+(44) ColumnarToRow [codegen id : 8]
+Input [4]: [ws_bill_customer_sk#59, ws_ext_discount_amt#60, ws_ext_list_price#61, ws_sold_date_sk#62]
+
+(45) Filter [codegen id : 8]
+Input [4]: [ws_bill_customer_sk#59, ws_ext_discount_amt#60, ws_ext_list_price#61, ws_sold_date_sk#62]
+Condition : isnotnull(ws_bill_customer_sk#59)
+
+(46) BroadcastExchange
+Input [4]: [ws_bill_customer_sk#59, ws_ext_discount_amt#60, ws_ext_list_price#61, ws_sold_date_sk#62]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#63]
+
+(47) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [c_customer_sk#51]
+Right keys [1]: [ws_bill_customer_sk#59]
+Join condition: None
+
+(48) Project [codegen id : 10]
+Output [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58, ws_ext_discount_amt#60, ws_ext_list_price#61, ws_sold_date_sk#62]
+Input [12]: [c_customer_sk#51, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58, ws_bill_customer_sk#59, ws_ext_discount_amt#60, ws_ext_list_price#61, ws_sold_date_sk#62]
+
+(49) ReusedExchange [Reuses operator id: 13]
+Output [2]: [d_date_sk#64, d_year#65]
+
+(50) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ws_sold_date_sk#62]
+Right keys [1]: [d_date_sk#64]
+Join condition: None
+
+(51) Project [codegen id : 10]
+Output [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58, ws_ext_discount_amt#60, ws_ext_list_price#61, d_year#65]
+Input [12]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58, ws_ext_discount_amt#60, ws_ext_list_price#61, ws_sold_date_sk#62, d_date_sk#64, d_year#65]
+
+(52) HashAggregate [codegen id : 10]
+Input [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58, ws_ext_discount_amt#60, ws_ext_list_price#61, d_year#65]
+Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58, d_year#65]
+Functions [1]: [partial_sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#61 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#60 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum#66]
+Results [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58, d_year#65, sum#67]
+
+(53) Exchange
+Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58, d_year#65, sum#67]
+Arguments: hashpartitioning(c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58, d_year#65, 5), ENSURE_REQUIREMENTS, [id=#68]
+
+(54) HashAggregate [codegen id : 11]
+Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58, d_year#65, sum#67]
+Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58, d_year#65]
+Functions [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#61 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#60 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#61 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#60 as decimal(8,2)))), DecimalType(8,2), true)))#69]
+Results [2]: [c_customer_id#52 AS customer_id#70, MakeDecimal(sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#61 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#60 as decimal(8,2)))), DecimalType(8,2), true)))#69,18,2) AS year_total#71]
+
+(55) Filter [codegen id : 11]
+Input [2]: [customer_id#70, year_total#71]
+Condition : (isnotnull(year_total#71) AND (year_total#71 > 0.00))
+
+(56) Project [codegen id : 11]
+Output [2]: [customer_id#70 AS customer_id#72, year_total#71 AS year_total#73]
+Input [2]: [customer_id#70, year_total#71]
+
+(57) BroadcastExchange
+Input [2]: [customer_id#72, year_total#73]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#74]
+
+(58) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [customer_id#22]
+Right keys [1]: [customer_id#72]
+Join condition: None
+
+(59) Project [codegen id : 16]
+Output [8]: [customer_id#22, year_total#23, customer_id#45, customer_first_name#46, customer_last_name#47, customer_email_address#48, year_total#49, year_total#73]
+Input [9]: [customer_id#22, year_total#23, customer_id#45, customer_first_name#46, customer_last_name#47, customer_email_address#48, year_total#49, customer_id#72, year_total#73]
+
+(60) Scan parquet default.customer
+Output [8]: [c_customer_sk#75, c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string,c_birth_country:string,c_login:string,c_email_address:string>
+
+(61) ColumnarToRow [codegen id : 14]
+Input [8]: [c_customer_sk#75, c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82]
+
+(62) Filter [codegen id : 14]
+Input [8]: [c_customer_sk#75, c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82]
+Condition : (isnotnull(c_customer_sk#75) AND isnotnull(c_customer_id#76))
+
+(63) Scan parquet default.web_sales
+Output [4]: [ws_bill_customer_sk#83, ws_ext_discount_amt#84, ws_ext_list_price#85, ws_sold_date_sk#86]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#86), dynamicpruningexpression(ws_sold_date_sk#86 IN dynamicpruning#36)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int,ws_ext_discount_amt:decimal(7,2),ws_ext_list_price:decimal(7,2)>
+
+(64) ColumnarToRow [codegen id : 12]
+Input [4]: [ws_bill_customer_sk#83, ws_ext_discount_amt#84, ws_ext_list_price#85, ws_sold_date_sk#86]
+
+(65) Filter [codegen id : 12]
+Input [4]: [ws_bill_customer_sk#83, ws_ext_discount_amt#84, ws_ext_list_price#85, ws_sold_date_sk#86]
+Condition : isnotnull(ws_bill_customer_sk#83)
+
+(66) BroadcastExchange
+Input [4]: [ws_bill_customer_sk#83, ws_ext_discount_amt#84, ws_ext_list_price#85, ws_sold_date_sk#86]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#87]
+
+(67) BroadcastHashJoin [codegen id : 14]
+Left keys [1]: [c_customer_sk#75]
+Right keys [1]: [ws_bill_customer_sk#83]
+Join condition: None
+
+(68) Project [codegen id : 14]
+Output [10]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, ws_ext_discount_amt#84, ws_ext_list_price#85, ws_sold_date_sk#86]
+Input [12]: [c_customer_sk#75, c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, ws_bill_customer_sk#83, ws_ext_discount_amt#84, ws_ext_list_price#85, ws_sold_date_sk#86]
+
+(69) ReusedExchange [Reuses operator id: 32]
+Output [2]: [d_date_sk#88, d_year#89]
+
+(70) BroadcastHashJoin [codegen id : 14]
+Left keys [1]: [ws_sold_date_sk#86]
+Right keys [1]: [d_date_sk#88]
+Join condition: None
+
+(71) Project [codegen id : 14]
+Output [10]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, ws_ext_discount_amt#84, ws_ext_list_price#85, d_year#89]
+Input [12]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, ws_ext_discount_amt#84, ws_ext_list_price#85, ws_sold_date_sk#86, d_date_sk#88, d_year#89]
+
+(72) HashAggregate [codegen id : 14]
+Input [10]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, ws_ext_discount_amt#84, ws_ext_list_price#85, d_year#89]
+Keys [8]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, d_year#89]
+Functions [1]: [partial_sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#85 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#84 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum#90]
+Results [9]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, d_year#89, sum#91]
+
+(73) Exchange
+Input [9]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, d_year#89, sum#91]
+Arguments: hashpartitioning(c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, d_year#89, 5), ENSURE_REQUIREMENTS, [id=#92]
+
+(74) HashAggregate [codegen id : 15]
+Input [9]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, d_year#89, sum#91]
+Keys [8]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, d_year#89]
+Functions [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#85 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#84 as decimal(8,2)))), DecimalType(8,2), true)))]
+Aggregate Attributes [1]: [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#85 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#84 as decimal(8,2)))), DecimalType(8,2), true)))#93]
+Results [2]: [c_customer_id#76 AS customer_id#94, MakeDecimal(sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price#85 as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt#84 as decimal(8,2)))), DecimalType(8,2), true)))#93,18,2) AS year_total#95]
+
+(75) BroadcastExchange
+Input [2]: [customer_id#94, year_total#95]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#96]
+
+(76) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [customer_id#22]
+Right keys [1]: [customer_id#94]
+Join condition: (CASE WHEN (year_total#73 > 0.00) THEN CheckOverflow((promote_precision(year_total#95) / promote_precision(year_total#73)), DecimalType(38,20), true) ELSE 0E-20 END > CASE WHEN (year_total#23 > 0.00) THEN CheckOverflow((promote_precision(year_total#49) / promote_precision(year_total#23)), DecimalType(38,20), true) ELSE 0E-20 END)
+
+(77) Project [codegen id : 16]
+Output [4]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_email_address#48]
+Input [10]: [customer_id#22, year_total#23, customer_id#45, customer_first_name#46, customer_last_name#47, customer_email_address#48, year_total#49, year_total#73, customer_id#94, year_total#95]
+
+(78) TakeOrderedAndProject
+Input [4]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_email_address#48]
+Arguments: 100, [customer_id#45 ASC NULLS FIRST, customer_first_name#46 ASC NULLS FIRST, customer_last_name#47 ASC NULLS FIRST, customer_email_address#48 ASC NULLS FIRST], [customer_id#45, customer_first_name#46, customer_last_name#47, customer_email_address#48]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13
+ReusedExchange (79)
+
+
+(79) ReusedExchange [Reuses operator id: 13]
+Output [2]: [d_date_sk#15, d_year#16]
+
+Subquery:2 Hosting operator id = 23 Hosting Expression = ss_sold_date_sk#35 IN dynamicpruning#36
+ReusedExchange (80)
+
+
+(80) ReusedExchange [Reuses operator id: 32]
+Output [2]: [d_date_sk#38, d_year#39]
+
+Subquery:3 Hosting operator id = 43 Hosting Expression = ws_sold_date_sk#62 IN dynamicpruning#13
+
+Subquery:4 Hosting operator id = 63 Hosting Expression = ws_sold_date_sk#86 IN dynamicpruning#36
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/simplified.txt
new file mode 100644
index 0000000000000..310c68c06e5f0
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/simplified.txt
@@ -0,0 +1,123 @@
+TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_email_address]
+  WholeStageCodegen (16)
+    Project [customer_id,customer_first_name,customer_last_name,customer_email_address]
+      BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total]
+        Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,year_total]
+          BroadcastHashJoin [customer_id,customer_id]
+            BroadcastHashJoin [customer_id,customer_id]
+              Filter [year_total]
+                HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt as decimal(8,2)))), DecimalType(8,2), true))),customer_id,year_total,sum]
+                  InputAdapter
+                    Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1
+                      WholeStageCodegen (3)
+                        HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum]
+                          Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year]
+                            BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                              Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk]
+                                BroadcastHashJoin [c_customer_sk,ss_customer_sk]
+                                  Filter [c_customer_sk,c_customer_id]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address]
+                                  InputAdapter
+                                    BroadcastExchange #2
+                                      WholeStageCodegen (1)
+                                        Filter [ss_customer_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk]
+                                                SubqueryBroadcast [d_date_sk] #1
+                                                  ReusedExchange [d_date_sk,d_year] #3
+                              InputAdapter
+                                BroadcastExchange #3
+                                  WholeStageCodegen (2)
+                                    Filter [d_year,d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.date_dim [d_date_sk,d_year]
+              InputAdapter
+                BroadcastExchange #4
+                  WholeStageCodegen (7)
+                    HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ss_ext_list_price as decimal(8,2))) - promote_precision(cast(ss_ext_discount_amt as decimal(8,2)))), DecimalType(8,2), true))),customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,sum]
+                      InputAdapter
+                        Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5
+                          WholeStageCodegen (6)
+                            HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum]
+                              Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year]
+                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                  Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk]
+                                    BroadcastHashJoin [c_customer_sk,ss_customer_sk]
+                                      Filter [c_customer_sk,c_customer_id]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address]
+                                      InputAdapter
+                                        BroadcastExchange #6
+                                          WholeStageCodegen (4)
+                                            Filter [ss_customer_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #2
+                                                      ReusedExchange [d_date_sk,d_year] #7
+                                  InputAdapter
+                                    BroadcastExchange #7
+                                      WholeStageCodegen (5)
+                                        Filter [d_year,d_date_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.date_dim [d_date_sk,d_year]
+            InputAdapter
+              BroadcastExchange #8
+                WholeStageCodegen (11)
+                  Project [customer_id,year_total]
+                    Filter [year_total]
+                      HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt as decimal(8,2)))), DecimalType(8,2), true))),customer_id,year_total,sum]
+                        InputAdapter
+                          Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #9
+                            WholeStageCodegen (10)
+                              HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum]
+                                Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year]
+                                  BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                    Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk]
+                                      BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk]
+                                        Filter [c_customer_sk,c_customer_id]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address]
+                                        InputAdapter
+                                          BroadcastExchange #10
+                                            WholeStageCodegen (8)
+                                              Filter [ws_bill_customer_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk]
+                                                      ReusedSubquery [d_date_sk] #1
+                                    InputAdapter
+                                      ReusedExchange [d_date_sk,d_year] #3
+        InputAdapter
+          BroadcastExchange #11
+            WholeStageCodegen (15)
+              HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue(CheckOverflow((promote_precision(cast(ws_ext_list_price as decimal(8,2))) - promote_precision(cast(ws_ext_discount_amt as decimal(8,2)))), DecimalType(8,2), true))),customer_id,year_total,sum]
+                InputAdapter
+                  Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #12
+                    WholeStageCodegen (14)
+                      HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum]
+                        Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year]
+                          BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                            Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk]
+                              BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk]
+                                Filter [c_customer_sk,c_customer_id]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address]
+                                InputAdapter
+                                  BroadcastExchange #13
+                                    WholeStageCodegen (12)
+                                      Filter [ws_bill_customer_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk]
+                                              ReusedSubquery [d_date_sk] #2
+                            InputAdapter
+                              ReusedExchange [d_date_sk,d_year] #7
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.sf100/explain.txt
new file mode 100644
index 0000000000000..d4acaa2cec459
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.sf100/explain.txt
@@ -0,0 +1,163 @@
+== Physical Plan ==
+TakeOrderedAndProject (27)
++- * Project (26)
+   +- Window (25)
+      +- * Sort (24)
+         +- Exchange (23)
+            +- * HashAggregate (22)
+               +- Exchange (21)
+                  +- * HashAggregate (20)
+                     +- * Project (19)
+                        +- * BroadcastHashJoin Inner BuildRight (18)
+                           :- * Project (12)
+                           :  +- * SortMergeJoin Inner (11)
+                           :     :- * Sort (5)
+                           :     :  +- Exchange (4)
+                           :     :     +- * Filter (3)
+                           :     :        +- * ColumnarToRow (2)
+                           :     :           +- Scan parquet default.web_sales (1)
+                           :     +- * Sort (10)
+                           :        +- Exchange (9)
+                           :           +- * Filter (8)
+                           :              +- * ColumnarToRow (7)
+                           :                 +- Scan parquet default.item (6)
+                           +- BroadcastExchange (17)
+                              +- * Project (16)
+                                 +- * Filter (15)
+                                    +- * ColumnarToRow (14)
+                                       +- Scan parquet default.date_dim (13)
+
+
+(1) Scan parquet default.web_sales
+Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3]
+
+(3) Filter [codegen id : 1]
+Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3]
+Condition : isnotnull(ws_item_sk#1)
+
+(4) Exchange
+Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3]
+Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#5]
+
+(5) Sort [codegen id : 2]
+Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3]
+Arguments: [ws_item_sk#1 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.item
+Output [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [In(i_category, [Sports                                            ,Books                                             ,Home                                              ]), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_item_desc:string,i_current_price:decimal(7,2),i_class:string,i_category:string>
+
+(7) ColumnarToRow [codegen id : 3]
+Input [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+
+(8) Filter [codegen id : 3]
+Input [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Condition : (i_category#11 IN (Sports                                            ,Books                                             ,Home                                              ) AND isnotnull(i_item_sk#6))
+
+(9) Exchange
+Input [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Arguments: hashpartitioning(i_item_sk#6, 5), ENSURE_REQUIREMENTS, [id=#12]
+
+(10) Sort [codegen id : 4]
+Input [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Arguments: [i_item_sk#6 ASC NULLS FIRST], false, 0
+
+(11) SortMergeJoin [codegen id : 6]
+Left keys [1]: [ws_item_sk#1]
+Right keys [1]: [i_item_sk#6]
+Join condition: None
+
+(12) Project [codegen id : 6]
+Output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+
+(13) Scan parquet default.date_dim
+Output [2]: [d_date_sk#13, d_date#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(14) ColumnarToRow [codegen id : 5]
+Input [2]: [d_date_sk#13, d_date#14]
+
+(15) Filter [codegen id : 5]
+Input [2]: [d_date_sk#13, d_date#14]
+Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13))
+
+(16) Project [codegen id : 5]
+Output [1]: [d_date_sk#13]
+Input [2]: [d_date_sk#13, d_date#14]
+
+(17) BroadcastExchange
+Input [1]: [d_date_sk#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#15]
+
+(18) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ws_sold_date_sk#3]
+Right keys [1]: [d_date_sk#13]
+Join condition: None
+
+(19) Project [codegen id : 6]
+Output [6]: [ws_ext_sales_price#2, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11, d_date_sk#13]
+
+(20) HashAggregate [codegen id : 6]
+Input [6]: [ws_ext_sales_price#2, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Keys [5]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9]
+Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum#16]
+Results [6]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, sum#17]
+
+(21) Exchange
+Input [6]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, sum#17]
+Arguments: hashpartitioning(i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(22) HashAggregate [codegen id : 7]
+Input [6]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, sum#17]
+Keys [5]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9]
+Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#2))#19]
+Results [8]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#19,17,2) AS itemrevenue#20, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#19,17,2) AS _w0#21, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#19,17,2) AS _w1#22]
+
+(23) Exchange
+Input [8]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, _w0#21, _w1#22]
+Arguments: hashpartitioning(i_class#10, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(24) Sort [codegen id : 8]
+Input [8]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, _w0#21, _w1#22]
+Arguments: [i_class#10 ASC NULLS FIRST], false, 0
+
+(25) Window
+Input [8]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, _w0#21, _w1#22]
+Arguments: [sum(_w1#22) windowspecdefinition(i_class#10, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#24], [i_class#10]
+
+(26) Project [codegen id : 9]
+Output [7]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(_w0#21) * 100.00), DecimalType(21,2), true) as decimal(27,2))) / promote_precision(_we0#24)), DecimalType(38,17), true) AS revenueratio#25]
+Input [9]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, _w0#21, _w1#22, _we0#24]
+
+(27) TakeOrderedAndProject
+Input [7]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, revenueratio#25]
+Arguments: 100, [i_category#11 ASC NULLS FIRST, i_class#10 ASC NULLS FIRST, i_item_id#7 ASC NULLS FIRST, i_item_desc#8 ASC NULLS FIRST, revenueratio#25 ASC NULLS FIRST], [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, revenueratio#25]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (28)
+
+
+(28) ReusedExchange [Reuses operator id: 17]
+Output [1]: [d_date_sk#13]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.sf100/simplified.txt
new file mode 100644
index 0000000000000..801a70eda99ae
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.sf100/simplified.txt
@@ -0,0 +1,49 @@
+TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue]
+  WholeStageCodegen (9)
+    Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0]
+      InputAdapter
+        Window [_w1,i_class]
+          WholeStageCodegen (8)
+            Sort [i_class]
+              InputAdapter
+                Exchange [i_class] #1
+                  WholeStageCodegen (7)
+                    HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ws_ext_sales_price)),itemrevenue,_w0,_w1,sum]
+                      InputAdapter
+                        Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2
+                          WholeStageCodegen (6)
+                            HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ws_ext_sales_price] [sum,sum]
+                              Project [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                  Project [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                    SortMergeJoin [ws_item_sk,i_item_sk]
+                                      InputAdapter
+                                        WholeStageCodegen (2)
+                                          Sort [ws_item_sk]
+                                            InputAdapter
+                                              Exchange [ws_item_sk] #3
+                                                WholeStageCodegen (1)
+                                                  Filter [ws_item_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk]
+                                                          SubqueryBroadcast [d_date_sk] #1
+                                                            ReusedExchange [d_date_sk] #4
+                                      InputAdapter
+                                        WholeStageCodegen (4)
+                                          Sort [i_item_sk]
+                                            InputAdapter
+                                              Exchange [i_item_sk] #5
+                                                WholeStageCodegen (3)
+                                                  Filter [i_category,i_item_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                  InputAdapter
+                                    BroadcastExchange #4
+                                      WholeStageCodegen (5)
+                                        Project [d_date_sk]
+                                          Filter [d_date,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_date]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/explain.txt
new file mode 100644
index 0000000000000..f6dd412f5c4b1
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/explain.txt
@@ -0,0 +1,148 @@
+== Physical Plan ==
+TakeOrderedAndProject (24)
++- * Project (23)
+   +- Window (22)
+      +- * Sort (21)
+         +- Exchange (20)
+            +- * HashAggregate (19)
+               +- Exchange (18)
+                  +- * HashAggregate (17)
+                     +- * Project (16)
+                        +- * BroadcastHashJoin Inner BuildRight (15)
+                           :- * Project (9)
+                           :  +- * BroadcastHashJoin Inner BuildRight (8)
+                           :     :- * Filter (3)
+                           :     :  +- * ColumnarToRow (2)
+                           :     :     +- Scan parquet default.web_sales (1)
+                           :     +- BroadcastExchange (7)
+                           :        +- * Filter (6)
+                           :           +- * ColumnarToRow (5)
+                           :              +- Scan parquet default.item (4)
+                           +- BroadcastExchange (14)
+                              +- * Project (13)
+                                 +- * Filter (12)
+                                    +- * ColumnarToRow (11)
+                                       +- Scan parquet default.date_dim (10)
+
+
+(1) Scan parquet default.web_sales
+Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3]
+Condition : isnotnull(ws_item_sk#1)
+
+(4) Scan parquet default.item
+Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [In(i_category, [Sports                                            ,Books                                             ,Home                                              ]), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_item_desc:string,i_current_price:decimal(7,2),i_class:string,i_category:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+
+(6) Filter [codegen id : 1]
+Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Condition : (i_category#10 IN (Sports                                            ,Books                                             ,Home                                              ) AND isnotnull(i_item_sk#5))
+
+(7) BroadcastExchange
+Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(8) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ws_item_sk#1]
+Right keys [1]: [i_item_sk#5]
+Join condition: None
+
+(9) Project [codegen id : 3]
+Output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+
+(10) Scan parquet default.date_dim
+Output [2]: [d_date_sk#12, d_date#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#12, d_date#13]
+
+(12) Filter [codegen id : 2]
+Input [2]: [d_date_sk#12, d_date#13]
+Condition : (((isnotnull(d_date#13) AND (d_date#13 >= 1999-02-22)) AND (d_date#13 <= 1999-03-24)) AND isnotnull(d_date_sk#12))
+
+(13) Project [codegen id : 2]
+Output [1]: [d_date_sk#12]
+Input [2]: [d_date_sk#12, d_date#13]
+
+(14) BroadcastExchange
+Input [1]: [d_date_sk#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ws_sold_date_sk#3]
+Right keys [1]: [d_date_sk#12]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [6]: [ws_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10, d_date_sk#12]
+
+(17) HashAggregate [codegen id : 3]
+Input [6]: [ws_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8]
+Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum#15]
+Results [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#16]
+
+(18) Exchange
+Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#16]
+Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(19) HashAggregate [codegen id : 4]
+Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#16]
+Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8]
+Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#2))#18]
+Results [8]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#18,17,2) AS itemrevenue#19, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#18,17,2) AS _w0#20, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#18,17,2) AS _w1#21]
+
+(20) Exchange
+Input [8]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21]
+Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(21) Sort [codegen id : 5]
+Input [8]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21]
+Arguments: [i_class#9 ASC NULLS FIRST], false, 0
+
+(22) Window
+Input [8]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21]
+Arguments: [sum(_w1#21) windowspecdefinition(i_class#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#23], [i_class#9]
+
+(23) Project [codegen id : 6]
+Output [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(_w0#20) * 100.00), DecimalType(21,2), true) as decimal(27,2))) / promote_precision(_we0#23)), DecimalType(38,17), true) AS revenueratio#24]
+Input [9]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21, _we0#23]
+
+(24) TakeOrderedAndProject
+Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, revenueratio#24]
+Arguments: 100, [i_category#10 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_item_id#6 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#24 ASC NULLS FIRST], [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, revenueratio#24]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (25)
+
+
+(25) ReusedExchange [Reuses operator id: 14]
+Output [1]: [d_date_sk#12]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/simplified.txt
new file mode 100644
index 0000000000000..d4a9cb9eca9a9
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/simplified.txt
@@ -0,0 +1,40 @@
+TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue]
+  WholeStageCodegen (6)
+    Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0]
+      InputAdapter
+        Window [_w1,i_class]
+          WholeStageCodegen (5)
+            Sort [i_class]
+              InputAdapter
+                Exchange [i_class] #1
+                  WholeStageCodegen (4)
+                    HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ws_ext_sales_price)),itemrevenue,_w0,_w1,sum]
+                      InputAdapter
+                        Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2
+                          WholeStageCodegen (3)
+                            HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ws_ext_sales_price] [sum,sum]
+                              Project [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                  Project [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                    BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                      Filter [ws_item_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #1
+                                                ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        BroadcastExchange #4
+                                          WholeStageCodegen (1)
+                                            Filter [i_category,i_item_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                  InputAdapter
+                                    BroadcastExchange #3
+                                      WholeStageCodegen (2)
+                                        Project [d_date_sk]
+                                          Filter [d_date,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_date]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.sf100/explain.txt
new file mode 100644
index 0000000000000..e1ce480288a45
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.sf100/explain.txt
@@ -0,0 +1,869 @@
+== Physical Plan ==
+TakeOrderedAndProject (116)
++- * BroadcastHashJoin Inner BuildRight (115)
+   :- * Project (90)
+   :  +- * Filter (89)
+   :     +- * HashAggregate (88)
+   :        +- Exchange (87)
+   :           +- * HashAggregate (86)
+   :              +- * Project (85)
+   :                 +- * BroadcastHashJoin Inner BuildRight (84)
+   :                    :- * Project (74)
+   :                    :  +- * BroadcastHashJoin Inner BuildRight (73)
+   :                    :     :- * SortMergeJoin LeftSemi (67)
+   :                    :     :  :- * Sort (5)
+   :                    :     :  :  +- Exchange (4)
+   :                    :     :  :     +- * Filter (3)
+   :                    :     :  :        +- * ColumnarToRow (2)
+   :                    :     :  :           +- Scan parquet default.store_sales (1)
+   :                    :     :  +- * Sort (66)
+   :                    :     :     +- Exchange (65)
+   :                    :     :        +- * Project (64)
+   :                    :     :           +- * BroadcastHashJoin Inner BuildRight (63)
+   :                    :     :              :- * Filter (8)
+   :                    :     :              :  +- * ColumnarToRow (7)
+   :                    :     :              :     +- Scan parquet default.item (6)
+   :                    :     :              +- BroadcastExchange (62)
+   :                    :     :                 +- * HashAggregate (61)
+   :                    :     :                    +- Exchange (60)
+   :                    :     :                       +- * HashAggregate (59)
+   :                    :     :                          +- * SortMergeJoin LeftSemi (58)
+   :                    :     :                             :- * Sort (46)
+   :                    :     :                             :  +- Exchange (45)
+   :                    :     :                             :     +- * HashAggregate (44)
+   :                    :     :                             :        +- Exchange (43)
+   :                    :     :                             :           +- * HashAggregate (42)
+   :                    :     :                             :              +- * Project (41)
+   :                    :     :                             :                 +- * BroadcastHashJoin Inner BuildRight (40)
+   :                    :     :                             :                    :- * Project (18)
+   :                    :     :                             :                    :  +- * BroadcastHashJoin Inner BuildRight (17)
+   :                    :     :                             :                    :     :- * Filter (11)
+   :                    :     :                             :                    :     :  +- * ColumnarToRow (10)
+   :                    :     :                             :                    :     :     +- Scan parquet default.store_sales (9)
+   :                    :     :                             :                    :     +- BroadcastExchange (16)
+   :                    :     :                             :                    :        +- * Project (15)
+   :                    :     :                             :                    :           +- * Filter (14)
+   :                    :     :                             :                    :              +- * ColumnarToRow (13)
+   :                    :     :                             :                    :                 +- Scan parquet default.date_dim (12)
+   :                    :     :                             :                    +- BroadcastExchange (39)
+   :                    :     :                             :                       +- * SortMergeJoin LeftSemi (38)
+   :                    :     :                             :                          :- * Sort (23)
+   :                    :     :                             :                          :  +- Exchange (22)
+   :                    :     :                             :                          :     +- * Filter (21)
+   :                    :     :                             :                          :        +- * ColumnarToRow (20)
+   :                    :     :                             :                          :           +- Scan parquet default.item (19)
+   :                    :     :                             :                          +- * Sort (37)
+   :                    :     :                             :                             +- Exchange (36)
+   :                    :     :                             :                                +- * Project (35)
+   :                    :     :                             :                                   +- * BroadcastHashJoin Inner BuildRight (34)
+   :                    :     :                             :                                      :- * Project (29)
+   :                    :     :                             :                                      :  +- * BroadcastHashJoin Inner BuildRight (28)
+   :                    :     :                             :                                      :     :- * Filter (26)
+   :                    :     :                             :                                      :     :  +- * ColumnarToRow (25)
+   :                    :     :                             :                                      :     :     +- Scan parquet default.catalog_sales (24)
+   :                    :     :                             :                                      :     +- ReusedExchange (27)
+   :                    :     :                             :                                      +- BroadcastExchange (33)
+   :                    :     :                             :                                         +- * Filter (32)
+   :                    :     :                             :                                            +- * ColumnarToRow (31)
+   :                    :     :                             :                                               +- Scan parquet default.item (30)
+   :                    :     :                             +- * Sort (57)
+   :                    :     :                                +- Exchange (56)
+   :                    :     :                                   +- * Project (55)
+   :                    :     :                                      +- * BroadcastHashJoin Inner BuildRight (54)
+   :                    :     :                                         :- * Project (52)
+   :                    :     :                                         :  +- * BroadcastHashJoin Inner BuildRight (51)
+   :                    :     :                                         :     :- * Filter (49)
+   :                    :     :                                         :     :  +- * ColumnarToRow (48)
+   :                    :     :                                         :     :     +- Scan parquet default.web_sales (47)
+   :                    :     :                                         :     +- ReusedExchange (50)
+   :                    :     :                                         +- ReusedExchange (53)
+   :                    :     +- BroadcastExchange (72)
+   :                    :        +- * Project (71)
+   :                    :           +- * Filter (70)
+   :                    :              +- * ColumnarToRow (69)
+   :                    :                 +- Scan parquet default.date_dim (68)
+   :                    +- BroadcastExchange (83)
+   :                       +- * SortMergeJoin LeftSemi (82)
+   :                          :- * Sort (79)
+   :                          :  +- Exchange (78)
+   :                          :     +- * Filter (77)
+   :                          :        +- * ColumnarToRow (76)
+   :                          :           +- Scan parquet default.item (75)
+   :                          +- * Sort (81)
+   :                             +- ReusedExchange (80)
+   +- BroadcastExchange (114)
+      +- * Project (113)
+         +- * Filter (112)
+            +- * HashAggregate (111)
+               +- Exchange (110)
+                  +- * HashAggregate (109)
+                     +- * Project (108)
+                        +- * BroadcastHashJoin Inner BuildRight (107)
+                           :- * Project (105)
+                           :  +- * BroadcastHashJoin Inner BuildRight (104)
+                           :     :- * SortMergeJoin LeftSemi (98)
+                           :     :  :- * Sort (95)
+                           :     :  :  +- Exchange (94)
+                           :     :  :     +- * Filter (93)
+                           :     :  :        +- * ColumnarToRow (92)
+                           :     :  :           +- Scan parquet default.store_sales (91)
+                           :     :  +- * Sort (97)
+                           :     :     +- ReusedExchange (96)
+                           :     +- BroadcastExchange (103)
+                           :        +- * Project (102)
+                           :           +- * Filter (101)
+                           :              +- * ColumnarToRow (100)
+                           :                 +- Scan parquet default.date_dim (99)
+                           +- ReusedExchange (106)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_quantity:int,ss_list_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 1]
+Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+Condition : isnotnull(ss_item_sk#1)
+
+(4) Exchange
+Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#6]
+
+(5) Sort [codegen id : 2]
+Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+Arguments: [ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.item
+Output [4]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(7) ColumnarToRow [codegen id : 20]
+Input [4]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10]
+
+(8) Filter [codegen id : 20]
+Input [4]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10]
+Condition : ((isnotnull(i_brand_id#8) AND isnotnull(i_class_id#9)) AND isnotnull(i_category_id#10))
+
+(9) Scan parquet default.store_sales
+Output [2]: [ss_item_sk#11, ss_sold_date_sk#12]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_sold_date_sk#12 IN dynamicpruning#13)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int>
+
+(10) ColumnarToRow [codegen id : 11]
+Input [2]: [ss_item_sk#11, ss_sold_date_sk#12]
+
+(11) Filter [codegen id : 11]
+Input [2]: [ss_item_sk#11, ss_sold_date_sk#12]
+Condition : isnotnull(ss_item_sk#11)
+
+(12) Scan parquet default.date_dim
+Output [2]: [d_date_sk#14, d_year#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(13) ColumnarToRow [codegen id : 3]
+Input [2]: [d_date_sk#14, d_year#15]
+
+(14) Filter [codegen id : 3]
+Input [2]: [d_date_sk#14, d_year#15]
+Condition : (((isnotnull(d_year#15) AND (d_year#15 >= 1998)) AND (d_year#15 <= 2000)) AND isnotnull(d_date_sk#14))
+
+(15) Project [codegen id : 3]
+Output [1]: [d_date_sk#14]
+Input [2]: [d_date_sk#14, d_year#15]
+
+(16) BroadcastExchange
+Input [1]: [d_date_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#16]
+
+(17) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ss_sold_date_sk#12]
+Right keys [1]: [d_date_sk#14]
+Join condition: None
+
+(18) Project [codegen id : 11]
+Output [1]: [ss_item_sk#11]
+Input [3]: [ss_item_sk#11, ss_sold_date_sk#12, d_date_sk#14]
+
+(19) Scan parquet default.item
+Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(20) ColumnarToRow [codegen id : 4]
+Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+
+(21) Filter [codegen id : 4]
+Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+Condition : (((isnotnull(i_item_sk#17) AND isnotnull(i_brand_id#18)) AND isnotnull(i_class_id#19)) AND isnotnull(i_category_id#20))
+
+(22) Exchange
+Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+Arguments: hashpartitioning(coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20), 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(23) Sort [codegen id : 5]
+Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+Arguments: [coalesce(i_brand_id#18, 0) ASC NULLS FIRST, isnull(i_brand_id#18) ASC NULLS FIRST, coalesce(i_class_id#19, 0) ASC NULLS FIRST, isnull(i_class_id#19) ASC NULLS FIRST, coalesce(i_category_id#20, 0) ASC NULLS FIRST, isnull(i_category_id#20) ASC NULLS FIRST], false, 0
+
+(24) Scan parquet default.catalog_sales
+Output [2]: [cs_item_sk#22, cs_sold_date_sk#23]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#23), dynamicpruningexpression(cs_sold_date_sk#23 IN dynamicpruning#13)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int>
+
+(25) ColumnarToRow [codegen id : 8]
+Input [2]: [cs_item_sk#22, cs_sold_date_sk#23]
+
+(26) Filter [codegen id : 8]
+Input [2]: [cs_item_sk#22, cs_sold_date_sk#23]
+Condition : isnotnull(cs_item_sk#22)
+
+(27) ReusedExchange [Reuses operator id: 16]
+Output [1]: [d_date_sk#24]
+
+(28) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [cs_sold_date_sk#23]
+Right keys [1]: [d_date_sk#24]
+Join condition: None
+
+(29) Project [codegen id : 8]
+Output [1]: [cs_item_sk#22]
+Input [3]: [cs_item_sk#22, cs_sold_date_sk#23, d_date_sk#24]
+
+(30) Scan parquet default.item
+Output [4]: [i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(31) ColumnarToRow [codegen id : 7]
+Input [4]: [i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28]
+
+(32) Filter [codegen id : 7]
+Input [4]: [i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28]
+Condition : isnotnull(i_item_sk#25)
+
+(33) BroadcastExchange
+Input [4]: [i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#29]
+
+(34) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [cs_item_sk#22]
+Right keys [1]: [i_item_sk#25]
+Join condition: None
+
+(35) Project [codegen id : 8]
+Output [3]: [i_brand_id#26, i_class_id#27, i_category_id#28]
+Input [5]: [cs_item_sk#22, i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28]
+
+(36) Exchange
+Input [3]: [i_brand_id#26, i_class_id#27, i_category_id#28]
+Arguments: hashpartitioning(coalesce(i_brand_id#26, 0), isnull(i_brand_id#26), coalesce(i_class_id#27, 0), isnull(i_class_id#27), coalesce(i_category_id#28, 0), isnull(i_category_id#28), 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(37) Sort [codegen id : 9]
+Input [3]: [i_brand_id#26, i_class_id#27, i_category_id#28]
+Arguments: [coalesce(i_brand_id#26, 0) ASC NULLS FIRST, isnull(i_brand_id#26) ASC NULLS FIRST, coalesce(i_class_id#27, 0) ASC NULLS FIRST, isnull(i_class_id#27) ASC NULLS FIRST, coalesce(i_category_id#28, 0) ASC NULLS FIRST, isnull(i_category_id#28) ASC NULLS FIRST], false, 0
+
+(38) SortMergeJoin [codegen id : 10]
+Left keys [6]: [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)]
+Right keys [6]: [coalesce(i_brand_id#26, 0), isnull(i_brand_id#26), coalesce(i_class_id#27, 0), isnull(i_class_id#27), coalesce(i_category_id#28, 0), isnull(i_category_id#28)]
+Join condition: None
+
+(39) BroadcastExchange
+Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#31]
+
+(40) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ss_item_sk#11]
+Right keys [1]: [i_item_sk#17]
+Join condition: None
+
+(41) Project [codegen id : 11]
+Output [3]: [i_brand_id#18 AS brand_id#32, i_class_id#19 AS class_id#33, i_category_id#20 AS category_id#34]
+Input [5]: [ss_item_sk#11, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+
+(42) HashAggregate [codegen id : 11]
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Keys [3]: [brand_id#32, class_id#33, category_id#34]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#32, class_id#33, category_id#34]
+
+(43) Exchange
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Arguments: hashpartitioning(brand_id#32, class_id#33, category_id#34, 5), ENSURE_REQUIREMENTS, [id=#35]
+
+(44) HashAggregate [codegen id : 12]
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Keys [3]: [brand_id#32, class_id#33, category_id#34]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#32, class_id#33, category_id#34]
+
+(45) Exchange
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Arguments: hashpartitioning(coalesce(brand_id#32, 0), isnull(brand_id#32), coalesce(class_id#33, 0), isnull(class_id#33), coalesce(category_id#34, 0), isnull(category_id#34), 5), ENSURE_REQUIREMENTS, [id=#36]
+
+(46) Sort [codegen id : 13]
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Arguments: [coalesce(brand_id#32, 0) ASC NULLS FIRST, isnull(brand_id#32) ASC NULLS FIRST, coalesce(class_id#33, 0) ASC NULLS FIRST, isnull(class_id#33) ASC NULLS FIRST, coalesce(category_id#34, 0) ASC NULLS FIRST, isnull(category_id#34) ASC NULLS FIRST], false, 0
+
+(47) Scan parquet default.web_sales
+Output [2]: [ws_item_sk#37, ws_sold_date_sk#38]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#38), dynamicpruningexpression(ws_sold_date_sk#38 IN dynamicpruning#13)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int>
+
+(48) ColumnarToRow [codegen id : 16]
+Input [2]: [ws_item_sk#37, ws_sold_date_sk#38]
+
+(49) Filter [codegen id : 16]
+Input [2]: [ws_item_sk#37, ws_sold_date_sk#38]
+Condition : isnotnull(ws_item_sk#37)
+
+(50) ReusedExchange [Reuses operator id: 16]
+Output [1]: [d_date_sk#39]
+
+(51) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [ws_sold_date_sk#38]
+Right keys [1]: [d_date_sk#39]
+Join condition: None
+
+(52) Project [codegen id : 16]
+Output [1]: [ws_item_sk#37]
+Input [3]: [ws_item_sk#37, ws_sold_date_sk#38, d_date_sk#39]
+
+(53) ReusedExchange [Reuses operator id: 33]
+Output [4]: [i_item_sk#40, i_brand_id#41, i_class_id#42, i_category_id#43]
+
+(54) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [ws_item_sk#37]
+Right keys [1]: [i_item_sk#40]
+Join condition: None
+
+(55) Project [codegen id : 16]
+Output [3]: [i_brand_id#41, i_class_id#42, i_category_id#43]
+Input [5]: [ws_item_sk#37, i_item_sk#40, i_brand_id#41, i_class_id#42, i_category_id#43]
+
+(56) Exchange
+Input [3]: [i_brand_id#41, i_class_id#42, i_category_id#43]
+Arguments: hashpartitioning(coalesce(i_brand_id#41, 0), isnull(i_brand_id#41), coalesce(i_class_id#42, 0), isnull(i_class_id#42), coalesce(i_category_id#43, 0), isnull(i_category_id#43), 5), ENSURE_REQUIREMENTS, [id=#44]
+
+(57) Sort [codegen id : 17]
+Input [3]: [i_brand_id#41, i_class_id#42, i_category_id#43]
+Arguments: [coalesce(i_brand_id#41, 0) ASC NULLS FIRST, isnull(i_brand_id#41) ASC NULLS FIRST, coalesce(i_class_id#42, 0) ASC NULLS FIRST, isnull(i_class_id#42) ASC NULLS FIRST, coalesce(i_category_id#43, 0) ASC NULLS FIRST, isnull(i_category_id#43) ASC NULLS FIRST], false, 0
+
+(58) SortMergeJoin [codegen id : 18]
+Left keys [6]: [coalesce(brand_id#32, 0), isnull(brand_id#32), coalesce(class_id#33, 0), isnull(class_id#33), coalesce(category_id#34, 0), isnull(category_id#34)]
+Right keys [6]: [coalesce(i_brand_id#41, 0), isnull(i_brand_id#41), coalesce(i_class_id#42, 0), isnull(i_class_id#42), coalesce(i_category_id#43, 0), isnull(i_category_id#43)]
+Join condition: None
+
+(59) HashAggregate [codegen id : 18]
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Keys [3]: [brand_id#32, class_id#33, category_id#34]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#32, class_id#33, category_id#34]
+
+(60) Exchange
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Arguments: hashpartitioning(brand_id#32, class_id#33, category_id#34, 5), ENSURE_REQUIREMENTS, [id=#45]
+
+(61) HashAggregate [codegen id : 19]
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Keys [3]: [brand_id#32, class_id#33, category_id#34]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#32, class_id#33, category_id#34]
+
+(62) BroadcastExchange
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Arguments: HashedRelationBroadcastMode(List(input[0, int, true], input[1, int, true], input[2, int, true]),false), [id=#46]
+
+(63) BroadcastHashJoin [codegen id : 20]
+Left keys [3]: [i_brand_id#8, i_class_id#9, i_category_id#10]
+Right keys [3]: [brand_id#32, class_id#33, category_id#34]
+Join condition: None
+
+(64) Project [codegen id : 20]
+Output [1]: [i_item_sk#7 AS ss_item_sk#47]
+Input [7]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, brand_id#32, class_id#33, category_id#34]
+
+(65) Exchange
+Input [1]: [ss_item_sk#47]
+Arguments: hashpartitioning(ss_item_sk#47, 5), ENSURE_REQUIREMENTS, [id=#48]
+
+(66) Sort [codegen id : 21]
+Input [1]: [ss_item_sk#47]
+Arguments: [ss_item_sk#47 ASC NULLS FIRST], false, 0
+
+(67) SortMergeJoin [codegen id : 45]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [ss_item_sk#47]
+Join condition: None
+
+(68) Scan parquet default.date_dim
+Output [2]: [d_date_sk#49, d_week_seq#50]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_week_seq:int>
+
+(69) ColumnarToRow [codegen id : 22]
+Input [2]: [d_date_sk#49, d_week_seq#50]
+
+(70) Filter [codegen id : 22]
+Input [2]: [d_date_sk#49, d_week_seq#50]
+Condition : ((isnotnull(d_week_seq#50) AND (d_week_seq#50 = Subquery scalar-subquery#51, [id=#52])) AND isnotnull(d_date_sk#49))
+
+(71) Project [codegen id : 22]
+Output [1]: [d_date_sk#49]
+Input [2]: [d_date_sk#49, d_week_seq#50]
+
+(72) BroadcastExchange
+Input [1]: [d_date_sk#49]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#53]
+
+(73) BroadcastHashJoin [codegen id : 45]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#49]
+Join condition: None
+
+(74) Project [codegen id : 45]
+Output [3]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3]
+Input [5]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, d_date_sk#49]
+
+(75) Scan parquet default.item
+Output [4]: [i_item_sk#54, i_brand_id#55, i_class_id#56, i_category_id#57]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(76) ColumnarToRow [codegen id : 23]
+Input [4]: [i_item_sk#54, i_brand_id#55, i_class_id#56, i_category_id#57]
+
+(77) Filter [codegen id : 23]
+Input [4]: [i_item_sk#54, i_brand_id#55, i_class_id#56, i_category_id#57]
+Condition : (((isnotnull(i_item_sk#54) AND isnotnull(i_brand_id#55)) AND isnotnull(i_class_id#56)) AND isnotnull(i_category_id#57))
+
+(78) Exchange
+Input [4]: [i_item_sk#54, i_brand_id#55, i_class_id#56, i_category_id#57]
+Arguments: hashpartitioning(i_item_sk#54, 5), ENSURE_REQUIREMENTS, [id=#58]
+
+(79) Sort [codegen id : 24]
+Input [4]: [i_item_sk#54, i_brand_id#55, i_class_id#56, i_category_id#57]
+Arguments: [i_item_sk#54 ASC NULLS FIRST], false, 0
+
+(80) ReusedExchange [Reuses operator id: unknown]
+Output [1]: [ss_item_sk#47]
+
+(81) Sort [codegen id : 43]
+Input [1]: [ss_item_sk#47]
+Arguments: [ss_item_sk#47 ASC NULLS FIRST], false, 0
+
+(82) SortMergeJoin [codegen id : 44]
+Left keys [1]: [i_item_sk#54]
+Right keys [1]: [ss_item_sk#47]
+Join condition: None
+
+(83) BroadcastExchange
+Input [4]: [i_item_sk#54, i_brand_id#55, i_class_id#56, i_category_id#57]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#59]
+
+(84) BroadcastHashJoin [codegen id : 45]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#54]
+Join condition: None
+
+(85) Project [codegen id : 45]
+Output [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#55, i_class_id#56, i_category_id#57]
+Input [7]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, i_item_sk#54, i_brand_id#55, i_class_id#56, i_category_id#57]
+
+(86) HashAggregate [codegen id : 45]
+Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#55, i_class_id#56, i_category_id#57]
+Keys [3]: [i_brand_id#55, i_class_id#56, i_category_id#57]
+Functions [2]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true)), partial_count(1)]
+Aggregate Attributes [3]: [sum#60, isEmpty#61, count#62]
+Results [6]: [i_brand_id#55, i_class_id#56, i_category_id#57, sum#63, isEmpty#64, count#65]
+
+(87) Exchange
+Input [6]: [i_brand_id#55, i_class_id#56, i_category_id#57, sum#63, isEmpty#64, count#65]
+Arguments: hashpartitioning(i_brand_id#55, i_class_id#56, i_category_id#57, 5), ENSURE_REQUIREMENTS, [id=#66]
+
+(88) HashAggregate [codegen id : 92]
+Input [6]: [i_brand_id#55, i_class_id#56, i_category_id#57, sum#63, isEmpty#64, count#65]
+Keys [3]: [i_brand_id#55, i_class_id#56, i_category_id#57]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#67, count(1)#68]
+Results [6]: [i_brand_id#55, i_class_id#56, i_category_id#57, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#67 AS sales#69, count(1)#68 AS number_sales#70, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#67 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#71]
+
+(89) Filter [codegen id : 92]
+Input [6]: [i_brand_id#55, i_class_id#56, i_category_id#57, sales#69, number_sales#70, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#71]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#71) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#71 as decimal(32,6)) > cast(Subquery scalar-subquery#72, [id=#73] as decimal(32,6))))
+
+(90) Project [codegen id : 92]
+Output [6]: [store AS channel#74, i_brand_id#55, i_class_id#56, i_category_id#57, sales#69, number_sales#70]
+Input [6]: [i_brand_id#55, i_class_id#56, i_category_id#57, sales#69, number_sales#70, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#71]
+
+(91) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#75, ss_quantity#76, ss_list_price#77, ss_sold_date_sk#78]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#78), dynamicpruningexpression(ss_sold_date_sk#78 IN dynamicpruning#79)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_quantity:int,ss_list_price:decimal(7,2)>
+
+(92) ColumnarToRow [codegen id : 46]
+Input [4]: [ss_item_sk#75, ss_quantity#76, ss_list_price#77, ss_sold_date_sk#78]
+
+(93) Filter [codegen id : 46]
+Input [4]: [ss_item_sk#75, ss_quantity#76, ss_list_price#77, ss_sold_date_sk#78]
+Condition : isnotnull(ss_item_sk#75)
+
+(94) Exchange
+Input [4]: [ss_item_sk#75, ss_quantity#76, ss_list_price#77, ss_sold_date_sk#78]
+Arguments: hashpartitioning(ss_item_sk#75, 5), ENSURE_REQUIREMENTS, [id=#80]
+
+(95) Sort [codegen id : 47]
+Input [4]: [ss_item_sk#75, ss_quantity#76, ss_list_price#77, ss_sold_date_sk#78]
+Arguments: [ss_item_sk#75 ASC NULLS FIRST], false, 0
+
+(96) ReusedExchange [Reuses operator id: unknown]
+Output [1]: [ss_item_sk#47]
+
+(97) Sort [codegen id : 66]
+Input [1]: [ss_item_sk#47]
+Arguments: [ss_item_sk#47 ASC NULLS FIRST], false, 0
+
+(98) SortMergeJoin [codegen id : 90]
+Left keys [1]: [ss_item_sk#75]
+Right keys [1]: [ss_item_sk#47]
+Join condition: None
+
+(99) Scan parquet default.date_dim
+Output [2]: [d_date_sk#81, d_week_seq#82]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_week_seq:int>
+
+(100) ColumnarToRow [codegen id : 67]
+Input [2]: [d_date_sk#81, d_week_seq#82]
+
+(101) Filter [codegen id : 67]
+Input [2]: [d_date_sk#81, d_week_seq#82]
+Condition : ((isnotnull(d_week_seq#82) AND (d_week_seq#82 = Subquery scalar-subquery#83, [id=#84])) AND isnotnull(d_date_sk#81))
+
+(102) Project [codegen id : 67]
+Output [1]: [d_date_sk#81]
+Input [2]: [d_date_sk#81, d_week_seq#82]
+
+(103) BroadcastExchange
+Input [1]: [d_date_sk#81]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#85]
+
+(104) BroadcastHashJoin [codegen id : 90]
+Left keys [1]: [ss_sold_date_sk#78]
+Right keys [1]: [d_date_sk#81]
+Join condition: None
+
+(105) Project [codegen id : 90]
+Output [3]: [ss_item_sk#75, ss_quantity#76, ss_list_price#77]
+Input [5]: [ss_item_sk#75, ss_quantity#76, ss_list_price#77, ss_sold_date_sk#78, d_date_sk#81]
+
+(106) ReusedExchange [Reuses operator id: 83]
+Output [4]: [i_item_sk#86, i_brand_id#87, i_class_id#88, i_category_id#89]
+
+(107) BroadcastHashJoin [codegen id : 90]
+Left keys [1]: [ss_item_sk#75]
+Right keys [1]: [i_item_sk#86]
+Join condition: None
+
+(108) Project [codegen id : 90]
+Output [5]: [ss_quantity#76, ss_list_price#77, i_brand_id#87, i_class_id#88, i_category_id#89]
+Input [7]: [ss_item_sk#75, ss_quantity#76, ss_list_price#77, i_item_sk#86, i_brand_id#87, i_class_id#88, i_category_id#89]
+
+(109) HashAggregate [codegen id : 90]
+Input [5]: [ss_quantity#76, ss_list_price#77, i_brand_id#87, i_class_id#88, i_category_id#89]
+Keys [3]: [i_brand_id#87, i_class_id#88, i_category_id#89]
+Functions [2]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#76 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#77 as decimal(12,2)))), DecimalType(18,2), true)), partial_count(1)]
+Aggregate Attributes [3]: [sum#90, isEmpty#91, count#92]
+Results [6]: [i_brand_id#87, i_class_id#88, i_category_id#89, sum#93, isEmpty#94, count#95]
+
+(110) Exchange
+Input [6]: [i_brand_id#87, i_class_id#88, i_category_id#89, sum#93, isEmpty#94, count#95]
+Arguments: hashpartitioning(i_brand_id#87, i_class_id#88, i_category_id#89, 5), ENSURE_REQUIREMENTS, [id=#96]
+
+(111) HashAggregate [codegen id : 91]
+Input [6]: [i_brand_id#87, i_class_id#88, i_category_id#89, sum#93, isEmpty#94, count#95]
+Keys [3]: [i_brand_id#87, i_class_id#88, i_category_id#89]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#76 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#77 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#76 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#77 as decimal(12,2)))), DecimalType(18,2), true))#97, count(1)#98]
+Results [6]: [i_brand_id#87, i_class_id#88, i_category_id#89, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#76 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#77 as decimal(12,2)))), DecimalType(18,2), true))#97 AS sales#99, count(1)#98 AS number_sales#100, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#76 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#77 as decimal(12,2)))), DecimalType(18,2), true))#97 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#76 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#77 as decimal(12,2)))), DecimalType(18,2), true))#101]
+
+(112) Filter [codegen id : 91]
+Input [6]: [i_brand_id#87, i_class_id#88, i_category_id#89, sales#99, number_sales#100, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#76 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#77 as decimal(12,2)))), DecimalType(18,2), true))#101]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#76 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#77 as decimal(12,2)))), DecimalType(18,2), true))#101) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#76 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#77 as decimal(12,2)))), DecimalType(18,2), true))#101 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#72, [id=#73] as decimal(32,6))))
+
+(113) Project [codegen id : 91]
+Output [6]: [store AS channel#102, i_brand_id#87, i_class_id#88, i_category_id#89, sales#99, number_sales#100]
+Input [6]: [i_brand_id#87, i_class_id#88, i_category_id#89, sales#99, number_sales#100, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#76 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#77 as decimal(12,2)))), DecimalType(18,2), true))#101]
+
+(114) BroadcastExchange
+Input [6]: [channel#102, i_brand_id#87, i_class_id#88, i_category_id#89, sales#99, number_sales#100]
+Arguments: HashedRelationBroadcastMode(List(input[1, int, true], input[2, int, true], input[3, int, true]),false), [id=#103]
+
+(115) BroadcastHashJoin [codegen id : 92]
+Left keys [3]: [i_brand_id#55, i_class_id#56, i_category_id#57]
+Right keys [3]: [i_brand_id#87, i_class_id#88, i_category_id#89]
+Join condition: None
+
+(116) TakeOrderedAndProject
+Input [12]: [channel#74, i_brand_id#55, i_class_id#56, i_category_id#57, sales#69, number_sales#70, channel#102, i_brand_id#87, i_class_id#88, i_category_id#89, sales#99, number_sales#100]
+Arguments: 100, [i_brand_id#55 ASC NULLS FIRST, i_class_id#56 ASC NULLS FIRST, i_category_id#57 ASC NULLS FIRST], [channel#74, i_brand_id#55, i_class_id#56, i_category_id#57, sales#69, number_sales#70, channel#102, i_brand_id#87, i_class_id#88, i_category_id#89, sales#99, number_sales#100]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 89 Hosting Expression = Subquery scalar-subquery#72, [id=#73]
+* HashAggregate (139)
++- Exchange (138)
+   +- * HashAggregate (137)
+      +- Union (136)
+         :- * Project (125)
+         :  +- * BroadcastHashJoin Inner BuildRight (124)
+         :     :- * ColumnarToRow (118)
+         :     :  +- Scan parquet default.store_sales (117)
+         :     +- BroadcastExchange (123)
+         :        +- * Project (122)
+         :           +- * Filter (121)
+         :              +- * ColumnarToRow (120)
+         :                 +- Scan parquet default.date_dim (119)
+         :- * Project (130)
+         :  +- * BroadcastHashJoin Inner BuildRight (129)
+         :     :- * ColumnarToRow (127)
+         :     :  +- Scan parquet default.catalog_sales (126)
+         :     +- ReusedExchange (128)
+         +- * Project (135)
+            +- * BroadcastHashJoin Inner BuildRight (134)
+               :- * ColumnarToRow (132)
+               :  +- Scan parquet default.web_sales (131)
+               +- ReusedExchange (133)
+
+
+(117) Scan parquet default.store_sales
+Output [3]: [ss_quantity#104, ss_list_price#105, ss_sold_date_sk#106]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#106), dynamicpruningexpression(ss_sold_date_sk#106 IN dynamicpruning#107)]
+ReadSchema: struct<ss_quantity:int,ss_list_price:decimal(7,2)>
+
+(118) ColumnarToRow [codegen id : 2]
+Input [3]: [ss_quantity#104, ss_list_price#105, ss_sold_date_sk#106]
+
+(119) Scan parquet default.date_dim
+Output [2]: [d_date_sk#108, d_year#109]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(120) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#108, d_year#109]
+
+(121) Filter [codegen id : 1]
+Input [2]: [d_date_sk#108, d_year#109]
+Condition : (((isnotnull(d_year#109) AND (d_year#109 >= 1998)) AND (d_year#109 <= 2000)) AND isnotnull(d_date_sk#108))
+
+(122) Project [codegen id : 1]
+Output [1]: [d_date_sk#108]
+Input [2]: [d_date_sk#108, d_year#109]
+
+(123) BroadcastExchange
+Input [1]: [d_date_sk#108]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#110]
+
+(124) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#106]
+Right keys [1]: [d_date_sk#108]
+Join condition: None
+
+(125) Project [codegen id : 2]
+Output [2]: [ss_quantity#104 AS quantity#111, ss_list_price#105 AS list_price#112]
+Input [4]: [ss_quantity#104, ss_list_price#105, ss_sold_date_sk#106, d_date_sk#108]
+
+(126) Scan parquet default.catalog_sales
+Output [3]: [cs_quantity#113, cs_list_price#114, cs_sold_date_sk#115]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#115), dynamicpruningexpression(cs_sold_date_sk#115 IN dynamicpruning#107)]
+ReadSchema: struct<cs_quantity:int,cs_list_price:decimal(7,2)>
+
+(127) ColumnarToRow [codegen id : 4]
+Input [3]: [cs_quantity#113, cs_list_price#114, cs_sold_date_sk#115]
+
+(128) ReusedExchange [Reuses operator id: 123]
+Output [1]: [d_date_sk#116]
+
+(129) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_sold_date_sk#115]
+Right keys [1]: [d_date_sk#116]
+Join condition: None
+
+(130) Project [codegen id : 4]
+Output [2]: [cs_quantity#113 AS quantity#117, cs_list_price#114 AS list_price#118]
+Input [4]: [cs_quantity#113, cs_list_price#114, cs_sold_date_sk#115, d_date_sk#116]
+
+(131) Scan parquet default.web_sales
+Output [3]: [ws_quantity#119, ws_list_price#120, ws_sold_date_sk#121]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#121), dynamicpruningexpression(ws_sold_date_sk#121 IN dynamicpruning#107)]
+ReadSchema: struct<ws_quantity:int,ws_list_price:decimal(7,2)>
+
+(132) ColumnarToRow [codegen id : 6]
+Input [3]: [ws_quantity#119, ws_list_price#120, ws_sold_date_sk#121]
+
+(133) ReusedExchange [Reuses operator id: 123]
+Output [1]: [d_date_sk#122]
+
+(134) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ws_sold_date_sk#121]
+Right keys [1]: [d_date_sk#122]
+Join condition: None
+
+(135) Project [codegen id : 6]
+Output [2]: [ws_quantity#119 AS quantity#123, ws_list_price#120 AS list_price#124]
+Input [4]: [ws_quantity#119, ws_list_price#120, ws_sold_date_sk#121, d_date_sk#122]
+
+(136) Union
+
+(137) HashAggregate [codegen id : 7]
+Input [2]: [quantity#111, list_price#112]
+Keys: []
+Functions [1]: [partial_avg(CheckOverflow((promote_precision(cast(cast(quantity#111 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#112 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [2]: [sum#125, count#126]
+Results [2]: [sum#127, count#128]
+
+(138) Exchange
+Input [2]: [sum#127, count#128]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#129]
+
+(139) HashAggregate [codegen id : 8]
+Input [2]: [sum#127, count#128]
+Keys: []
+Functions [1]: [avg(CheckOverflow((promote_precision(cast(cast(quantity#111 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#112 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [avg(CheckOverflow((promote_precision(cast(cast(quantity#111 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#112 as decimal(12,2)))), DecimalType(18,2), true))#130]
+Results [1]: [avg(CheckOverflow((promote_precision(cast(cast(quantity#111 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#112 as decimal(12,2)))), DecimalType(18,2), true))#130 AS average_sales#131]
+
+Subquery:2 Hosting operator id = 117 Hosting Expression = ss_sold_date_sk#106 IN dynamicpruning#107
+ReusedExchange (140)
+
+
+(140) ReusedExchange [Reuses operator id: 123]
+Output [1]: [d_date_sk#108]
+
+Subquery:3 Hosting operator id = 126 Hosting Expression = cs_sold_date_sk#115 IN dynamicpruning#107
+
+Subquery:4 Hosting operator id = 131 Hosting Expression = ws_sold_date_sk#121 IN dynamicpruning#107
+
+Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (141)
+
+
+(141) ReusedExchange [Reuses operator id: 72]
+Output [1]: [d_date_sk#49]
+
+Subquery:6 Hosting operator id = 9 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13
+ReusedExchange (142)
+
+
+(142) ReusedExchange [Reuses operator id: 16]
+Output [1]: [d_date_sk#14]
+
+Subquery:7 Hosting operator id = 24 Hosting Expression = cs_sold_date_sk#23 IN dynamicpruning#13
+
+Subquery:8 Hosting operator id = 47 Hosting Expression = ws_sold_date_sk#38 IN dynamicpruning#13
+
+Subquery:9 Hosting operator id = 70 Hosting Expression = Subquery scalar-subquery#51, [id=#52]
+* Project (146)
++- * Filter (145)
+   +- * ColumnarToRow (144)
+      +- Scan parquet default.date_dim (143)
+
+
+(143) Scan parquet default.date_dim
+Output [4]: [d_week_seq#132, d_year#133, d_moy#134, d_dom#135]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,16)]
+ReadSchema: struct<d_week_seq:int,d_year:int,d_moy:int,d_dom:int>
+
+(144) ColumnarToRow [codegen id : 1]
+Input [4]: [d_week_seq#132, d_year#133, d_moy#134, d_dom#135]
+
+(145) Filter [codegen id : 1]
+Input [4]: [d_week_seq#132, d_year#133, d_moy#134, d_dom#135]
+Condition : (((((isnotnull(d_year#133) AND isnotnull(d_moy#134)) AND isnotnull(d_dom#135)) AND (d_year#133 = 1999)) AND (d_moy#134 = 12)) AND (d_dom#135 = 16))
+
+(146) Project [codegen id : 1]
+Output [1]: [d_week_seq#132]
+Input [4]: [d_week_seq#132, d_year#133, d_moy#134, d_dom#135]
+
+Subquery:10 Hosting operator id = 112 Hosting Expression = ReusedSubquery Subquery scalar-subquery#72, [id=#73]
+
+Subquery:11 Hosting operator id = 91 Hosting Expression = ss_sold_date_sk#78 IN dynamicpruning#79
+ReusedExchange (147)
+
+
+(147) ReusedExchange [Reuses operator id: 103]
+Output [1]: [d_date_sk#81]
+
+Subquery:12 Hosting operator id = 101 Hosting Expression = Subquery scalar-subquery#83, [id=#84]
+* Project (151)
++- * Filter (150)
+   +- * ColumnarToRow (149)
+      +- Scan parquet default.date_dim (148)
+
+
+(148) Scan parquet default.date_dim
+Output [4]: [d_week_seq#136, d_year#137, d_moy#138, d_dom#139]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1998), EqualTo(d_moy,12), EqualTo(d_dom,16)]
+ReadSchema: struct<d_week_seq:int,d_year:int,d_moy:int,d_dom:int>
+
+(149) ColumnarToRow [codegen id : 1]
+Input [4]: [d_week_seq#136, d_year#137, d_moy#138, d_dom#139]
+
+(150) Filter [codegen id : 1]
+Input [4]: [d_week_seq#136, d_year#137, d_moy#138, d_dom#139]
+Condition : (((((isnotnull(d_year#137) AND isnotnull(d_moy#138)) AND isnotnull(d_dom#139)) AND (d_year#137 = 1998)) AND (d_moy#138 = 12)) AND (d_dom#139 = 16))
+
+(151) Project [codegen id : 1]
+Output [1]: [d_week_seq#136]
+Input [4]: [d_week_seq#136, d_year#137, d_moy#138, d_dom#139]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.sf100/simplified.txt
new file mode 100644
index 0000000000000..6a8fcada7a61b
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.sf100/simplified.txt
@@ -0,0 +1,261 @@
+TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales]
+  WholeStageCodegen (92)
+    BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id]
+      Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+        Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+          Subquery #4
+            WholeStageCodegen (8)
+              HashAggregate [sum,count] [avg(CheckOverflow((promote_precision(cast(cast(quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price as decimal(12,2)))), DecimalType(18,2), true)),average_sales,sum,count]
+                InputAdapter
+                  Exchange #18
+                    WholeStageCodegen (7)
+                      HashAggregate [quantity,list_price] [sum,count,sum,count]
+                        InputAdapter
+                          Union
+                            WholeStageCodegen (2)
+                              Project [ss_quantity,ss_list_price]
+                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk]
+                                        SubqueryBroadcast [d_date_sk] #5
+                                          ReusedExchange [d_date_sk] #19
+                                  InputAdapter
+                                    BroadcastExchange #19
+                                      WholeStageCodegen (1)
+                                        Project [d_date_sk]
+                                          Filter [d_year,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_year]
+                            WholeStageCodegen (4)
+                              Project [cs_quantity,cs_list_price]
+                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk]
+                                        ReusedSubquery [d_date_sk] #5
+                                  InputAdapter
+                                    ReusedExchange [d_date_sk] #19
+                            WholeStageCodegen (6)
+                              Project [ws_quantity,ws_list_price]
+                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk]
+                                        ReusedSubquery [d_date_sk] #5
+                                  InputAdapter
+                                    ReusedExchange [d_date_sk] #19
+          HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+            InputAdapter
+              Exchange [i_brand_id,i_class_id,i_category_id] #1
+                WholeStageCodegen (45)
+                  HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count]
+                    Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id]
+                      BroadcastHashJoin [ss_item_sk,i_item_sk]
+                        Project [ss_item_sk,ss_quantity,ss_list_price]
+                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                            SortMergeJoin [ss_item_sk,ss_item_sk]
+                              InputAdapter
+                                WholeStageCodegen (2)
+                                  Sort [ss_item_sk]
+                                    InputAdapter
+                                      Exchange [ss_item_sk] #2
+                                        WholeStageCodegen (1)
+                                          Filter [ss_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk]
+                                                  SubqueryBroadcast [d_date_sk] #1
+                                                    ReusedExchange [d_date_sk] #3
+                              InputAdapter
+                                WholeStageCodegen (21)
+                                  Sort [ss_item_sk]
+                                    InputAdapter
+                                      Exchange [ss_item_sk] #4
+                                        WholeStageCodegen (20)
+                                          Project [i_item_sk]
+                                            BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id]
+                                              Filter [i_brand_id,i_class_id,i_category_id]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                              InputAdapter
+                                                BroadcastExchange #5
+                                                  WholeStageCodegen (19)
+                                                    HashAggregate [brand_id,class_id,category_id]
+                                                      InputAdapter
+                                                        Exchange [brand_id,class_id,category_id] #6
+                                                          WholeStageCodegen (18)
+                                                            HashAggregate [brand_id,class_id,category_id]
+                                                              SortMergeJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id]
+                                                                InputAdapter
+                                                                  WholeStageCodegen (13)
+                                                                    Sort [brand_id,class_id,category_id]
+                                                                      InputAdapter
+                                                                        Exchange [brand_id,class_id,category_id] #7
+                                                                          WholeStageCodegen (12)
+                                                                            HashAggregate [brand_id,class_id,category_id]
+                                                                              InputAdapter
+                                                                                Exchange [brand_id,class_id,category_id] #8
+                                                                                  WholeStageCodegen (11)
+                                                                                    HashAggregate [brand_id,class_id,category_id]
+                                                                                      Project [i_brand_id,i_class_id,i_category_id]
+                                                                                        BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                                                          Project [ss_item_sk]
+                                                                                            BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                              Filter [ss_item_sk]
+                                                                                                ColumnarToRow
+                                                                                                  InputAdapter
+                                                                                                    Scan parquet default.store_sales [ss_item_sk,ss_sold_date_sk]
+                                                                                                      SubqueryBroadcast [d_date_sk] #2
+                                                                                                        ReusedExchange [d_date_sk] #9
+                                                                                              InputAdapter
+                                                                                                BroadcastExchange #9
+                                                                                                  WholeStageCodegen (3)
+                                                                                                    Project [d_date_sk]
+                                                                                                      Filter [d_year,d_date_sk]
+                                                                                                        ColumnarToRow
+                                                                                                          InputAdapter
+                                                                                                            Scan parquet default.date_dim [d_date_sk,d_year]
+                                                                                          InputAdapter
+                                                                                            BroadcastExchange #10
+                                                                                              WholeStageCodegen (10)
+                                                                                                SortMergeJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id]
+                                                                                                  InputAdapter
+                                                                                                    WholeStageCodegen (5)
+                                                                                                      Sort [i_brand_id,i_class_id,i_category_id]
+                                                                                                        InputAdapter
+                                                                                                          Exchange [i_brand_id,i_class_id,i_category_id] #11
+                                                                                                            WholeStageCodegen (4)
+                                                                                                              Filter [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                                                ColumnarToRow
+                                                                                                                  InputAdapter
+                                                                                                                    Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                                  InputAdapter
+                                                                                                    WholeStageCodegen (9)
+                                                                                                      Sort [i_brand_id,i_class_id,i_category_id]
+                                                                                                        InputAdapter
+                                                                                                          Exchange [i_brand_id,i_class_id,i_category_id] #12
+                                                                                                            WholeStageCodegen (8)
+                                                                                                              Project [i_brand_id,i_class_id,i_category_id]
+                                                                                                                BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                                                                                                  Project [cs_item_sk]
+                                                                                                                    BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                                                                      Filter [cs_item_sk]
+                                                                                                                        ColumnarToRow
+                                                                                                                          InputAdapter
+                                                                                                                            Scan parquet default.catalog_sales [cs_item_sk,cs_sold_date_sk]
+                                                                                                                              ReusedSubquery [d_date_sk] #2
+                                                                                                                      InputAdapter
+                                                                                                                        ReusedExchange [d_date_sk] #9
+                                                                                                                  InputAdapter
+                                                                                                                    BroadcastExchange #13
+                                                                                                                      WholeStageCodegen (7)
+                                                                                                                        Filter [i_item_sk]
+                                                                                                                          ColumnarToRow
+                                                                                                                            InputAdapter
+                                                                                                                              Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                InputAdapter
+                                                                  WholeStageCodegen (17)
+                                                                    Sort [i_brand_id,i_class_id,i_category_id]
+                                                                      InputAdapter
+                                                                        Exchange [i_brand_id,i_class_id,i_category_id] #14
+                                                                          WholeStageCodegen (16)
+                                                                            Project [i_brand_id,i_class_id,i_category_id]
+                                                                              BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                                                                Project [ws_item_sk]
+                                                                                  BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                                    Filter [ws_item_sk]
+                                                                                      ColumnarToRow
+                                                                                        InputAdapter
+                                                                                          Scan parquet default.web_sales [ws_item_sk,ws_sold_date_sk]
+                                                                                            ReusedSubquery [d_date_sk] #2
+                                                                                    InputAdapter
+                                                                                      ReusedExchange [d_date_sk] #9
+                                                                                InputAdapter
+                                                                                  ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13
+                            InputAdapter
+                              BroadcastExchange #3
+                                WholeStageCodegen (22)
+                                  Project [d_date_sk]
+                                    Filter [d_week_seq,d_date_sk]
+                                      Subquery #3
+                                        WholeStageCodegen (1)
+                                          Project [d_week_seq]
+                                            Filter [d_year,d_moy,d_dom]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.date_dim [d_week_seq,d_year,d_moy,d_dom]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.date_dim [d_date_sk,d_week_seq]
+                        InputAdapter
+                          BroadcastExchange #15
+                            WholeStageCodegen (44)
+                              SortMergeJoin [i_item_sk,ss_item_sk]
+                                InputAdapter
+                                  WholeStageCodegen (24)
+                                    Sort [i_item_sk]
+                                      InputAdapter
+                                        Exchange [i_item_sk] #16
+                                          WholeStageCodegen (23)
+                                            Filter [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                InputAdapter
+                                  WholeStageCodegen (43)
+                                    Sort [ss_item_sk]
+                                      InputAdapter
+                                        ReusedExchange [ss_item_sk] #17
+      InputAdapter
+        BroadcastExchange #20
+          WholeStageCodegen (91)
+            Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+              Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                ReusedSubquery [average_sales] #4
+                HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                  InputAdapter
+                    Exchange [i_brand_id,i_class_id,i_category_id] #21
+                      WholeStageCodegen (90)
+                        HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count]
+                          Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id]
+                            BroadcastHashJoin [ss_item_sk,i_item_sk]
+                              Project [ss_item_sk,ss_quantity,ss_list_price]
+                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                  SortMergeJoin [ss_item_sk,ss_item_sk]
+                                    InputAdapter
+                                      WholeStageCodegen (47)
+                                        Sort [ss_item_sk]
+                                          InputAdapter
+                                            Exchange [ss_item_sk] #22
+                                              WholeStageCodegen (46)
+                                                Filter [ss_item_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk]
+                                                        SubqueryBroadcast [d_date_sk] #6
+                                                          ReusedExchange [d_date_sk] #23
+                                    InputAdapter
+                                      WholeStageCodegen (66)
+                                        Sort [ss_item_sk]
+                                          InputAdapter
+                                            ReusedExchange [ss_item_sk] #17
+                                  InputAdapter
+                                    BroadcastExchange #23
+                                      WholeStageCodegen (67)
+                                        Project [d_date_sk]
+                                          Filter [d_week_seq,d_date_sk]
+                                            Subquery #7
+                                              WholeStageCodegen (1)
+                                                Project [d_week_seq]
+                                                  Filter [d_year,d_moy,d_dom]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.date_dim [d_week_seq,d_year,d_moy,d_dom]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_week_seq]
+                              InputAdapter
+                                ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #15
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/explain.txt
new file mode 100644
index 0000000000000..bdafb17d69d45
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/explain.txt
@@ -0,0 +1,789 @@
+== Physical Plan ==
+TakeOrderedAndProject (100)
++- * BroadcastHashJoin Inner BuildRight (99)
+   :- * Project (77)
+   :  +- * Filter (76)
+   :     +- * HashAggregate (75)
+   :        +- Exchange (74)
+   :           +- * HashAggregate (73)
+   :              +- * Project (72)
+   :                 +- * BroadcastHashJoin Inner BuildRight (71)
+   :                    :- * Project (65)
+   :                    :  +- * BroadcastHashJoin Inner BuildRight (64)
+   :                    :     :- * BroadcastHashJoin LeftSemi BuildRight (57)
+   :                    :     :  :- * Filter (3)
+   :                    :     :  :  +- * ColumnarToRow (2)
+   :                    :     :  :     +- Scan parquet default.store_sales (1)
+   :                    :     :  +- BroadcastExchange (56)
+   :                    :     :     +- * Project (55)
+   :                    :     :        +- * BroadcastHashJoin Inner BuildRight (54)
+   :                    :     :           :- * Filter (6)
+   :                    :     :           :  +- * ColumnarToRow (5)
+   :                    :     :           :     +- Scan parquet default.item (4)
+   :                    :     :           +- BroadcastExchange (53)
+   :                    :     :              +- * HashAggregate (52)
+   :                    :     :                 +- * HashAggregate (51)
+   :                    :     :                    +- * BroadcastHashJoin LeftSemi BuildRight (50)
+   :                    :     :                       :- * HashAggregate (39)
+   :                    :     :                       :  +- Exchange (38)
+   :                    :     :                       :     +- * HashAggregate (37)
+   :                    :     :                       :        +- * Project (36)
+   :                    :     :                       :           +- * BroadcastHashJoin Inner BuildRight (35)
+   :                    :     :                       :              :- * Project (33)
+   :                    :     :                       :              :  +- * BroadcastHashJoin Inner BuildRight (32)
+   :                    :     :                       :              :     :- * Filter (9)
+   :                    :     :                       :              :     :  +- * ColumnarToRow (8)
+   :                    :     :                       :              :     :     +- Scan parquet default.store_sales (7)
+   :                    :     :                       :              :     +- BroadcastExchange (31)
+   :                    :     :                       :              :        +- * BroadcastHashJoin LeftSemi BuildRight (30)
+   :                    :     :                       :              :           :- * Filter (12)
+   :                    :     :                       :              :           :  +- * ColumnarToRow (11)
+   :                    :     :                       :              :           :     +- Scan parquet default.item (10)
+   :                    :     :                       :              :           +- BroadcastExchange (29)
+   :                    :     :                       :              :              +- * Project (28)
+   :                    :     :                       :              :                 +- * BroadcastHashJoin Inner BuildRight (27)
+   :                    :     :                       :              :                    :- * Project (21)
+   :                    :     :                       :              :                    :  +- * BroadcastHashJoin Inner BuildRight (20)
+   :                    :     :                       :              :                    :     :- * Filter (15)
+   :                    :     :                       :              :                    :     :  +- * ColumnarToRow (14)
+   :                    :     :                       :              :                    :     :     +- Scan parquet default.catalog_sales (13)
+   :                    :     :                       :              :                    :     +- BroadcastExchange (19)
+   :                    :     :                       :              :                    :        +- * Filter (18)
+   :                    :     :                       :              :                    :           +- * ColumnarToRow (17)
+   :                    :     :                       :              :                    :              +- Scan parquet default.item (16)
+   :                    :     :                       :              :                    +- BroadcastExchange (26)
+   :                    :     :                       :              :                       +- * Project (25)
+   :                    :     :                       :              :                          +- * Filter (24)
+   :                    :     :                       :              :                             +- * ColumnarToRow (23)
+   :                    :     :                       :              :                                +- Scan parquet default.date_dim (22)
+   :                    :     :                       :              +- ReusedExchange (34)
+   :                    :     :                       +- BroadcastExchange (49)
+   :                    :     :                          +- * Project (48)
+   :                    :     :                             +- * BroadcastHashJoin Inner BuildRight (47)
+   :                    :     :                                :- * Project (45)
+   :                    :     :                                :  +- * BroadcastHashJoin Inner BuildRight (44)
+   :                    :     :                                :     :- * Filter (42)
+   :                    :     :                                :     :  +- * ColumnarToRow (41)
+   :                    :     :                                :     :     +- Scan parquet default.web_sales (40)
+   :                    :     :                                :     +- ReusedExchange (43)
+   :                    :     :                                +- ReusedExchange (46)
+   :                    :     +- BroadcastExchange (63)
+   :                    :        +- * BroadcastHashJoin LeftSemi BuildRight (62)
+   :                    :           :- * Filter (60)
+   :                    :           :  +- * ColumnarToRow (59)
+   :                    :           :     +- Scan parquet default.item (58)
+   :                    :           +- ReusedExchange (61)
+   :                    +- BroadcastExchange (70)
+   :                       +- * Project (69)
+   :                          +- * Filter (68)
+   :                             +- * ColumnarToRow (67)
+   :                                +- Scan parquet default.date_dim (66)
+   +- BroadcastExchange (98)
+      +- * Project (97)
+         +- * Filter (96)
+            +- * HashAggregate (95)
+               +- Exchange (94)
+                  +- * HashAggregate (93)
+                     +- * Project (92)
+                        +- * BroadcastHashJoin Inner BuildRight (91)
+                           :- * Project (85)
+                           :  +- * BroadcastHashJoin Inner BuildRight (84)
+                           :     :- * BroadcastHashJoin LeftSemi BuildRight (82)
+                           :     :  :- * Filter (80)
+                           :     :  :  +- * ColumnarToRow (79)
+                           :     :  :     +- Scan parquet default.store_sales (78)
+                           :     :  +- ReusedExchange (81)
+                           :     +- ReusedExchange (83)
+                           +- BroadcastExchange (90)
+                              +- * Project (89)
+                                 +- * Filter (88)
+                                    +- * ColumnarToRow (87)
+                                       +- Scan parquet default.date_dim (86)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_quantity:int,ss_list_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 25]
+Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 25]
+Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+Condition : isnotnull(ss_item_sk#1)
+
+(4) Scan parquet default.item
+Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(5) ColumnarToRow [codegen id : 11]
+Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9]
+
+(6) Filter [codegen id : 11]
+Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9]
+Condition : ((isnotnull(i_brand_id#7) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9))
+
+(7) Scan parquet default.store_sales
+Output [2]: [ss_item_sk#10, ss_sold_date_sk#11]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#11), dynamicpruningexpression(ss_sold_date_sk#11 IN dynamicpruning#12)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int>
+
+(8) ColumnarToRow [codegen id : 6]
+Input [2]: [ss_item_sk#10, ss_sold_date_sk#11]
+
+(9) Filter [codegen id : 6]
+Input [2]: [ss_item_sk#10, ss_sold_date_sk#11]
+Condition : isnotnull(ss_item_sk#10)
+
+(10) Scan parquet default.item
+Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(11) ColumnarToRow [codegen id : 4]
+Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16]
+
+(12) Filter [codegen id : 4]
+Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16]
+Condition : (((isnotnull(i_item_sk#13) AND isnotnull(i_brand_id#14)) AND isnotnull(i_class_id#15)) AND isnotnull(i_category_id#16))
+
+(13) Scan parquet default.catalog_sales
+Output [2]: [cs_item_sk#17, cs_sold_date_sk#18]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_sold_date_sk#18 IN dynamicpruning#12)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int>
+
+(14) ColumnarToRow [codegen id : 3]
+Input [2]: [cs_item_sk#17, cs_sold_date_sk#18]
+
+(15) Filter [codegen id : 3]
+Input [2]: [cs_item_sk#17, cs_sold_date_sk#18]
+Condition : isnotnull(cs_item_sk#17)
+
+(16) Scan parquet default.item
+Output [4]: [i_item_sk#19, i_brand_id#20, i_class_id#21, i_category_id#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(17) ColumnarToRow [codegen id : 1]
+Input [4]: [i_item_sk#19, i_brand_id#20, i_class_id#21, i_category_id#22]
+
+(18) Filter [codegen id : 1]
+Input [4]: [i_item_sk#19, i_brand_id#20, i_class_id#21, i_category_id#22]
+Condition : isnotnull(i_item_sk#19)
+
+(19) BroadcastExchange
+Input [4]: [i_item_sk#19, i_brand_id#20, i_class_id#21, i_category_id#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#23]
+
+(20) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_item_sk#17]
+Right keys [1]: [i_item_sk#19]
+Join condition: None
+
+(21) Project [codegen id : 3]
+Output [4]: [cs_sold_date_sk#18, i_brand_id#20, i_class_id#21, i_category_id#22]
+Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#19, i_brand_id#20, i_class_id#21, i_category_id#22]
+
+(22) Scan parquet default.date_dim
+Output [2]: [d_date_sk#24, d_year#25]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(23) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#24, d_year#25]
+
+(24) Filter [codegen id : 2]
+Input [2]: [d_date_sk#24, d_year#25]
+Condition : (((isnotnull(d_year#25) AND (d_year#25 >= 1998)) AND (d_year#25 <= 2000)) AND isnotnull(d_date_sk#24))
+
+(25) Project [codegen id : 2]
+Output [1]: [d_date_sk#24]
+Input [2]: [d_date_sk#24, d_year#25]
+
+(26) BroadcastExchange
+Input [1]: [d_date_sk#24]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#26]
+
+(27) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_sold_date_sk#18]
+Right keys [1]: [d_date_sk#24]
+Join condition: None
+
+(28) Project [codegen id : 3]
+Output [3]: [i_brand_id#20, i_class_id#21, i_category_id#22]
+Input [5]: [cs_sold_date_sk#18, i_brand_id#20, i_class_id#21, i_category_id#22, d_date_sk#24]
+
+(29) BroadcastExchange
+Input [3]: [i_brand_id#20, i_class_id#21, i_category_id#22]
+Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [id=#27]
+
+(30) BroadcastHashJoin [codegen id : 4]
+Left keys [6]: [coalesce(i_brand_id#14, 0), isnull(i_brand_id#14), coalesce(i_class_id#15, 0), isnull(i_class_id#15), coalesce(i_category_id#16, 0), isnull(i_category_id#16)]
+Right keys [6]: [coalesce(i_brand_id#20, 0), isnull(i_brand_id#20), coalesce(i_class_id#21, 0), isnull(i_class_id#21), coalesce(i_category_id#22, 0), isnull(i_category_id#22)]
+Join condition: None
+
+(31) BroadcastExchange
+Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#28]
+
+(32) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_item_sk#10]
+Right keys [1]: [i_item_sk#13]
+Join condition: None
+
+(33) Project [codegen id : 6]
+Output [4]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16]
+Input [6]: [ss_item_sk#10, ss_sold_date_sk#11, i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16]
+
+(34) ReusedExchange [Reuses operator id: 26]
+Output [1]: [d_date_sk#29]
+
+(35) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_sold_date_sk#11]
+Right keys [1]: [d_date_sk#29]
+Join condition: None
+
+(36) Project [codegen id : 6]
+Output [3]: [i_brand_id#14 AS brand_id#30, i_class_id#15 AS class_id#31, i_category_id#16 AS category_id#32]
+Input [5]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16, d_date_sk#29]
+
+(37) HashAggregate [codegen id : 6]
+Input [3]: [brand_id#30, class_id#31, category_id#32]
+Keys [3]: [brand_id#30, class_id#31, category_id#32]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#30, class_id#31, category_id#32]
+
+(38) Exchange
+Input [3]: [brand_id#30, class_id#31, category_id#32]
+Arguments: hashpartitioning(brand_id#30, class_id#31, category_id#32, 5), ENSURE_REQUIREMENTS, [id=#33]
+
+(39) HashAggregate [codegen id : 10]
+Input [3]: [brand_id#30, class_id#31, category_id#32]
+Keys [3]: [brand_id#30, class_id#31, category_id#32]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#30, class_id#31, category_id#32]
+
+(40) Scan parquet default.web_sales
+Output [2]: [ws_item_sk#34, ws_sold_date_sk#35]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#35), dynamicpruningexpression(ws_sold_date_sk#35 IN dynamicpruning#12)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int>
+
+(41) ColumnarToRow [codegen id : 9]
+Input [2]: [ws_item_sk#34, ws_sold_date_sk#35]
+
+(42) Filter [codegen id : 9]
+Input [2]: [ws_item_sk#34, ws_sold_date_sk#35]
+Condition : isnotnull(ws_item_sk#34)
+
+(43) ReusedExchange [Reuses operator id: 19]
+Output [4]: [i_item_sk#36, i_brand_id#37, i_class_id#38, i_category_id#39]
+
+(44) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ws_item_sk#34]
+Right keys [1]: [i_item_sk#36]
+Join condition: None
+
+(45) Project [codegen id : 9]
+Output [4]: [ws_sold_date_sk#35, i_brand_id#37, i_class_id#38, i_category_id#39]
+Input [6]: [ws_item_sk#34, ws_sold_date_sk#35, i_item_sk#36, i_brand_id#37, i_class_id#38, i_category_id#39]
+
+(46) ReusedExchange [Reuses operator id: 26]
+Output [1]: [d_date_sk#40]
+
+(47) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ws_sold_date_sk#35]
+Right keys [1]: [d_date_sk#40]
+Join condition: None
+
+(48) Project [codegen id : 9]
+Output [3]: [i_brand_id#37, i_class_id#38, i_category_id#39]
+Input [5]: [ws_sold_date_sk#35, i_brand_id#37, i_class_id#38, i_category_id#39, d_date_sk#40]
+
+(49) BroadcastExchange
+Input [3]: [i_brand_id#37, i_class_id#38, i_category_id#39]
+Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [id=#41]
+
+(50) BroadcastHashJoin [codegen id : 10]
+Left keys [6]: [coalesce(brand_id#30, 0), isnull(brand_id#30), coalesce(class_id#31, 0), isnull(class_id#31), coalesce(category_id#32, 0), isnull(category_id#32)]
+Right keys [6]: [coalesce(i_brand_id#37, 0), isnull(i_brand_id#37), coalesce(i_class_id#38, 0), isnull(i_class_id#38), coalesce(i_category_id#39, 0), isnull(i_category_id#39)]
+Join condition: None
+
+(51) HashAggregate [codegen id : 10]
+Input [3]: [brand_id#30, class_id#31, category_id#32]
+Keys [3]: [brand_id#30, class_id#31, category_id#32]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#30, class_id#31, category_id#32]
+
+(52) HashAggregate [codegen id : 10]
+Input [3]: [brand_id#30, class_id#31, category_id#32]
+Keys [3]: [brand_id#30, class_id#31, category_id#32]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#30, class_id#31, category_id#32]
+
+(53) BroadcastExchange
+Input [3]: [brand_id#30, class_id#31, category_id#32]
+Arguments: HashedRelationBroadcastMode(List(input[0, int, true], input[1, int, true], input[2, int, true]),false), [id=#42]
+
+(54) BroadcastHashJoin [codegen id : 11]
+Left keys [3]: [i_brand_id#7, i_class_id#8, i_category_id#9]
+Right keys [3]: [brand_id#30, class_id#31, category_id#32]
+Join condition: None
+
+(55) Project [codegen id : 11]
+Output [1]: [i_item_sk#6 AS ss_item_sk#43]
+Input [7]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, brand_id#30, class_id#31, category_id#32]
+
+(56) BroadcastExchange
+Input [1]: [ss_item_sk#43]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#44]
+
+(57) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [ss_item_sk#43]
+Join condition: None
+
+(58) Scan parquet default.item
+Output [4]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(59) ColumnarToRow [codegen id : 23]
+Input [4]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48]
+
+(60) Filter [codegen id : 23]
+Input [4]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48]
+Condition : (((isnotnull(i_item_sk#45) AND isnotnull(i_brand_id#46)) AND isnotnull(i_class_id#47)) AND isnotnull(i_category_id#48))
+
+(61) ReusedExchange [Reuses operator id: unknown]
+Output [1]: [ss_item_sk#43]
+
+(62) BroadcastHashJoin [codegen id : 23]
+Left keys [1]: [i_item_sk#45]
+Right keys [1]: [ss_item_sk#43]
+Join condition: None
+
+(63) BroadcastExchange
+Input [4]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#49]
+
+(64) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#45]
+Join condition: None
+
+(65) Project [codegen id : 25]
+Output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#46, i_class_id#47, i_category_id#48]
+Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48]
+
+(66) Scan parquet default.date_dim
+Output [2]: [d_date_sk#50, d_week_seq#51]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_week_seq:int>
+
+(67) ColumnarToRow [codegen id : 24]
+Input [2]: [d_date_sk#50, d_week_seq#51]
+
+(68) Filter [codegen id : 24]
+Input [2]: [d_date_sk#50, d_week_seq#51]
+Condition : ((isnotnull(d_week_seq#51) AND (d_week_seq#51 = Subquery scalar-subquery#52, [id=#53])) AND isnotnull(d_date_sk#50))
+
+(69) Project [codegen id : 24]
+Output [1]: [d_date_sk#50]
+Input [2]: [d_date_sk#50, d_week_seq#51]
+
+(70) BroadcastExchange
+Input [1]: [d_date_sk#50]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#54]
+
+(71) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#50]
+Join condition: None
+
+(72) Project [codegen id : 25]
+Output [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#46, i_class_id#47, i_category_id#48]
+Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#46, i_class_id#47, i_category_id#48, d_date_sk#50]
+
+(73) HashAggregate [codegen id : 25]
+Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#46, i_class_id#47, i_category_id#48]
+Keys [3]: [i_brand_id#46, i_class_id#47, i_category_id#48]
+Functions [2]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true)), partial_count(1)]
+Aggregate Attributes [3]: [sum#55, isEmpty#56, count#57]
+Results [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sum#58, isEmpty#59, count#60]
+
+(74) Exchange
+Input [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sum#58, isEmpty#59, count#60]
+Arguments: hashpartitioning(i_brand_id#46, i_class_id#47, i_category_id#48, 5), ENSURE_REQUIREMENTS, [id=#61]
+
+(75) HashAggregate [codegen id : 52]
+Input [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sum#58, isEmpty#59, count#60]
+Keys [3]: [i_brand_id#46, i_class_id#47, i_category_id#48]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#62, count(1)#63]
+Results [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#62 AS sales#64, count(1)#63 AS number_sales#65, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#62 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#66]
+
+(76) Filter [codegen id : 52]
+Input [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sales#64, number_sales#65, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#66]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#66) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#66 as decimal(32,6)) > cast(Subquery scalar-subquery#67, [id=#68] as decimal(32,6))))
+
+(77) Project [codegen id : 52]
+Output [6]: [store AS channel#69, i_brand_id#46, i_class_id#47, i_category_id#48, sales#64, number_sales#65]
+Input [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sales#64, number_sales#65, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#66]
+
+(78) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#70, ss_quantity#71, ss_list_price#72, ss_sold_date_sk#73]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#73), dynamicpruningexpression(ss_sold_date_sk#73 IN dynamicpruning#74)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_quantity:int,ss_list_price:decimal(7,2)>
+
+(79) ColumnarToRow [codegen id : 50]
+Input [4]: [ss_item_sk#70, ss_quantity#71, ss_list_price#72, ss_sold_date_sk#73]
+
+(80) Filter [codegen id : 50]
+Input [4]: [ss_item_sk#70, ss_quantity#71, ss_list_price#72, ss_sold_date_sk#73]
+Condition : isnotnull(ss_item_sk#70)
+
+(81) ReusedExchange [Reuses operator id: unknown]
+Output [1]: [ss_item_sk#43]
+
+(82) BroadcastHashJoin [codegen id : 50]
+Left keys [1]: [ss_item_sk#70]
+Right keys [1]: [ss_item_sk#43]
+Join condition: None
+
+(83) ReusedExchange [Reuses operator id: 63]
+Output [4]: [i_item_sk#75, i_brand_id#76, i_class_id#77, i_category_id#78]
+
+(84) BroadcastHashJoin [codegen id : 50]
+Left keys [1]: [ss_item_sk#70]
+Right keys [1]: [i_item_sk#75]
+Join condition: None
+
+(85) Project [codegen id : 50]
+Output [6]: [ss_quantity#71, ss_list_price#72, ss_sold_date_sk#73, i_brand_id#76, i_class_id#77, i_category_id#78]
+Input [8]: [ss_item_sk#70, ss_quantity#71, ss_list_price#72, ss_sold_date_sk#73, i_item_sk#75, i_brand_id#76, i_class_id#77, i_category_id#78]
+
+(86) Scan parquet default.date_dim
+Output [2]: [d_date_sk#79, d_week_seq#80]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_week_seq:int>
+
+(87) ColumnarToRow [codegen id : 49]
+Input [2]: [d_date_sk#79, d_week_seq#80]
+
+(88) Filter [codegen id : 49]
+Input [2]: [d_date_sk#79, d_week_seq#80]
+Condition : ((isnotnull(d_week_seq#80) AND (d_week_seq#80 = Subquery scalar-subquery#81, [id=#82])) AND isnotnull(d_date_sk#79))
+
+(89) Project [codegen id : 49]
+Output [1]: [d_date_sk#79]
+Input [2]: [d_date_sk#79, d_week_seq#80]
+
+(90) BroadcastExchange
+Input [1]: [d_date_sk#79]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#83]
+
+(91) BroadcastHashJoin [codegen id : 50]
+Left keys [1]: [ss_sold_date_sk#73]
+Right keys [1]: [d_date_sk#79]
+Join condition: None
+
+(92) Project [codegen id : 50]
+Output [5]: [ss_quantity#71, ss_list_price#72, i_brand_id#76, i_class_id#77, i_category_id#78]
+Input [7]: [ss_quantity#71, ss_list_price#72, ss_sold_date_sk#73, i_brand_id#76, i_class_id#77, i_category_id#78, d_date_sk#79]
+
+(93) HashAggregate [codegen id : 50]
+Input [5]: [ss_quantity#71, ss_list_price#72, i_brand_id#76, i_class_id#77, i_category_id#78]
+Keys [3]: [i_brand_id#76, i_class_id#77, i_category_id#78]
+Functions [2]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#71 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#72 as decimal(12,2)))), DecimalType(18,2), true)), partial_count(1)]
+Aggregate Attributes [3]: [sum#84, isEmpty#85, count#86]
+Results [6]: [i_brand_id#76, i_class_id#77, i_category_id#78, sum#87, isEmpty#88, count#89]
+
+(94) Exchange
+Input [6]: [i_brand_id#76, i_class_id#77, i_category_id#78, sum#87, isEmpty#88, count#89]
+Arguments: hashpartitioning(i_brand_id#76, i_class_id#77, i_category_id#78, 5), ENSURE_REQUIREMENTS, [id=#90]
+
+(95) HashAggregate [codegen id : 51]
+Input [6]: [i_brand_id#76, i_class_id#77, i_category_id#78, sum#87, isEmpty#88, count#89]
+Keys [3]: [i_brand_id#76, i_class_id#77, i_category_id#78]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#71 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#72 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#71 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#72 as decimal(12,2)))), DecimalType(18,2), true))#91, count(1)#92]
+Results [6]: [i_brand_id#76, i_class_id#77, i_category_id#78, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#71 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#72 as decimal(12,2)))), DecimalType(18,2), true))#91 AS sales#93, count(1)#92 AS number_sales#94, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#71 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#72 as decimal(12,2)))), DecimalType(18,2), true))#91 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#71 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#72 as decimal(12,2)))), DecimalType(18,2), true))#95]
+
+(96) Filter [codegen id : 51]
+Input [6]: [i_brand_id#76, i_class_id#77, i_category_id#78, sales#93, number_sales#94, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#71 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#72 as decimal(12,2)))), DecimalType(18,2), true))#95]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#71 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#72 as decimal(12,2)))), DecimalType(18,2), true))#95) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#71 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#72 as decimal(12,2)))), DecimalType(18,2), true))#95 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#67, [id=#68] as decimal(32,6))))
+
+(97) Project [codegen id : 51]
+Output [6]: [store AS channel#96, i_brand_id#76, i_class_id#77, i_category_id#78, sales#93, number_sales#94]
+Input [6]: [i_brand_id#76, i_class_id#77, i_category_id#78, sales#93, number_sales#94, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#71 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#72 as decimal(12,2)))), DecimalType(18,2), true))#95]
+
+(98) BroadcastExchange
+Input [6]: [channel#96, i_brand_id#76, i_class_id#77, i_category_id#78, sales#93, number_sales#94]
+Arguments: HashedRelationBroadcastMode(List(input[1, int, true], input[2, int, true], input[3, int, true]),false), [id=#97]
+
+(99) BroadcastHashJoin [codegen id : 52]
+Left keys [3]: [i_brand_id#46, i_class_id#47, i_category_id#48]
+Right keys [3]: [i_brand_id#76, i_class_id#77, i_category_id#78]
+Join condition: None
+
+(100) TakeOrderedAndProject
+Input [12]: [channel#69, i_brand_id#46, i_class_id#47, i_category_id#48, sales#64, number_sales#65, channel#96, i_brand_id#76, i_class_id#77, i_category_id#78, sales#93, number_sales#94]
+Arguments: 100, [i_brand_id#46 ASC NULLS FIRST, i_class_id#47 ASC NULLS FIRST, i_category_id#48 ASC NULLS FIRST], [channel#69, i_brand_id#46, i_class_id#47, i_category_id#48, sales#64, number_sales#65, channel#96, i_brand_id#76, i_class_id#77, i_category_id#78, sales#93, number_sales#94]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 76 Hosting Expression = Subquery scalar-subquery#67, [id=#68]
+* HashAggregate (123)
++- Exchange (122)
+   +- * HashAggregate (121)
+      +- Union (120)
+         :- * Project (109)
+         :  +- * BroadcastHashJoin Inner BuildRight (108)
+         :     :- * ColumnarToRow (102)
+         :     :  +- Scan parquet default.store_sales (101)
+         :     +- BroadcastExchange (107)
+         :        +- * Project (106)
+         :           +- * Filter (105)
+         :              +- * ColumnarToRow (104)
+         :                 +- Scan parquet default.date_dim (103)
+         :- * Project (114)
+         :  +- * BroadcastHashJoin Inner BuildRight (113)
+         :     :- * ColumnarToRow (111)
+         :     :  +- Scan parquet default.catalog_sales (110)
+         :     +- ReusedExchange (112)
+         +- * Project (119)
+            +- * BroadcastHashJoin Inner BuildRight (118)
+               :- * ColumnarToRow (116)
+               :  +- Scan parquet default.web_sales (115)
+               +- ReusedExchange (117)
+
+
+(101) Scan parquet default.store_sales
+Output [3]: [ss_quantity#98, ss_list_price#99, ss_sold_date_sk#100]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#100), dynamicpruningexpression(ss_sold_date_sk#100 IN dynamicpruning#101)]
+ReadSchema: struct<ss_quantity:int,ss_list_price:decimal(7,2)>
+
+(102) ColumnarToRow [codegen id : 2]
+Input [3]: [ss_quantity#98, ss_list_price#99, ss_sold_date_sk#100]
+
+(103) Scan parquet default.date_dim
+Output [2]: [d_date_sk#102, d_year#103]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(104) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#102, d_year#103]
+
+(105) Filter [codegen id : 1]
+Input [2]: [d_date_sk#102, d_year#103]
+Condition : (((isnotnull(d_year#103) AND (d_year#103 >= 1998)) AND (d_year#103 <= 2000)) AND isnotnull(d_date_sk#102))
+
+(106) Project [codegen id : 1]
+Output [1]: [d_date_sk#102]
+Input [2]: [d_date_sk#102, d_year#103]
+
+(107) BroadcastExchange
+Input [1]: [d_date_sk#102]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#104]
+
+(108) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#100]
+Right keys [1]: [d_date_sk#102]
+Join condition: None
+
+(109) Project [codegen id : 2]
+Output [2]: [ss_quantity#98 AS quantity#105, ss_list_price#99 AS list_price#106]
+Input [4]: [ss_quantity#98, ss_list_price#99, ss_sold_date_sk#100, d_date_sk#102]
+
+(110) Scan parquet default.catalog_sales
+Output [3]: [cs_quantity#107, cs_list_price#108, cs_sold_date_sk#109]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#109), dynamicpruningexpression(cs_sold_date_sk#109 IN dynamicpruning#101)]
+ReadSchema: struct<cs_quantity:int,cs_list_price:decimal(7,2)>
+
+(111) ColumnarToRow [codegen id : 4]
+Input [3]: [cs_quantity#107, cs_list_price#108, cs_sold_date_sk#109]
+
+(112) ReusedExchange [Reuses operator id: 107]
+Output [1]: [d_date_sk#110]
+
+(113) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_sold_date_sk#109]
+Right keys [1]: [d_date_sk#110]
+Join condition: None
+
+(114) Project [codegen id : 4]
+Output [2]: [cs_quantity#107 AS quantity#111, cs_list_price#108 AS list_price#112]
+Input [4]: [cs_quantity#107, cs_list_price#108, cs_sold_date_sk#109, d_date_sk#110]
+
+(115) Scan parquet default.web_sales
+Output [3]: [ws_quantity#113, ws_list_price#114, ws_sold_date_sk#115]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#115), dynamicpruningexpression(ws_sold_date_sk#115 IN dynamicpruning#101)]
+ReadSchema: struct<ws_quantity:int,ws_list_price:decimal(7,2)>
+
+(116) ColumnarToRow [codegen id : 6]
+Input [3]: [ws_quantity#113, ws_list_price#114, ws_sold_date_sk#115]
+
+(117) ReusedExchange [Reuses operator id: 107]
+Output [1]: [d_date_sk#116]
+
+(118) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ws_sold_date_sk#115]
+Right keys [1]: [d_date_sk#116]
+Join condition: None
+
+(119) Project [codegen id : 6]
+Output [2]: [ws_quantity#113 AS quantity#117, ws_list_price#114 AS list_price#118]
+Input [4]: [ws_quantity#113, ws_list_price#114, ws_sold_date_sk#115, d_date_sk#116]
+
+(120) Union
+
+(121) HashAggregate [codegen id : 7]
+Input [2]: [quantity#105, list_price#106]
+Keys: []
+Functions [1]: [partial_avg(CheckOverflow((promote_precision(cast(cast(quantity#105 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#106 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [2]: [sum#119, count#120]
+Results [2]: [sum#121, count#122]
+
+(122) Exchange
+Input [2]: [sum#121, count#122]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#123]
+
+(123) HashAggregate [codegen id : 8]
+Input [2]: [sum#121, count#122]
+Keys: []
+Functions [1]: [avg(CheckOverflow((promote_precision(cast(cast(quantity#105 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#106 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [avg(CheckOverflow((promote_precision(cast(cast(quantity#105 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#106 as decimal(12,2)))), DecimalType(18,2), true))#124]
+Results [1]: [avg(CheckOverflow((promote_precision(cast(cast(quantity#105 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#106 as decimal(12,2)))), DecimalType(18,2), true))#124 AS average_sales#125]
+
+Subquery:2 Hosting operator id = 101 Hosting Expression = ss_sold_date_sk#100 IN dynamicpruning#101
+ReusedExchange (124)
+
+
+(124) ReusedExchange [Reuses operator id: 107]
+Output [1]: [d_date_sk#102]
+
+Subquery:3 Hosting operator id = 110 Hosting Expression = cs_sold_date_sk#109 IN dynamicpruning#101
+
+Subquery:4 Hosting operator id = 115 Hosting Expression = ws_sold_date_sk#115 IN dynamicpruning#101
+
+Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (125)
+
+
+(125) ReusedExchange [Reuses operator id: 70]
+Output [1]: [d_date_sk#50]
+
+Subquery:6 Hosting operator id = 7 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12
+ReusedExchange (126)
+
+
+(126) ReusedExchange [Reuses operator id: 26]
+Output [1]: [d_date_sk#29]
+
+Subquery:7 Hosting operator id = 13 Hosting Expression = cs_sold_date_sk#18 IN dynamicpruning#12
+
+Subquery:8 Hosting operator id = 40 Hosting Expression = ws_sold_date_sk#35 IN dynamicpruning#12
+
+Subquery:9 Hosting operator id = 68 Hosting Expression = Subquery scalar-subquery#52, [id=#53]
+* Project (130)
++- * Filter (129)
+   +- * ColumnarToRow (128)
+      +- Scan parquet default.date_dim (127)
+
+
+(127) Scan parquet default.date_dim
+Output [4]: [d_week_seq#126, d_year#127, d_moy#128, d_dom#129]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,16)]
+ReadSchema: struct<d_week_seq:int,d_year:int,d_moy:int,d_dom:int>
+
+(128) ColumnarToRow [codegen id : 1]
+Input [4]: [d_week_seq#126, d_year#127, d_moy#128, d_dom#129]
+
+(129) Filter [codegen id : 1]
+Input [4]: [d_week_seq#126, d_year#127, d_moy#128, d_dom#129]
+Condition : (((((isnotnull(d_year#127) AND isnotnull(d_moy#128)) AND isnotnull(d_dom#129)) AND (d_year#127 = 1999)) AND (d_moy#128 = 12)) AND (d_dom#129 = 16))
+
+(130) Project [codegen id : 1]
+Output [1]: [d_week_seq#126]
+Input [4]: [d_week_seq#126, d_year#127, d_moy#128, d_dom#129]
+
+Subquery:10 Hosting operator id = 96 Hosting Expression = ReusedSubquery Subquery scalar-subquery#67, [id=#68]
+
+Subquery:11 Hosting operator id = 78 Hosting Expression = ss_sold_date_sk#73 IN dynamicpruning#74
+ReusedExchange (131)
+
+
+(131) ReusedExchange [Reuses operator id: 90]
+Output [1]: [d_date_sk#79]
+
+Subquery:12 Hosting operator id = 88 Hosting Expression = Subquery scalar-subquery#81, [id=#82]
+* Project (135)
++- * Filter (134)
+   +- * ColumnarToRow (133)
+      +- Scan parquet default.date_dim (132)
+
+
+(132) Scan parquet default.date_dim
+Output [4]: [d_week_seq#130, d_year#131, d_moy#132, d_dom#133]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1998), EqualTo(d_moy,12), EqualTo(d_dom,16)]
+ReadSchema: struct<d_week_seq:int,d_year:int,d_moy:int,d_dom:int>
+
+(133) ColumnarToRow [codegen id : 1]
+Input [4]: [d_week_seq#130, d_year#131, d_moy#132, d_dom#133]
+
+(134) Filter [codegen id : 1]
+Input [4]: [d_week_seq#130, d_year#131, d_moy#132, d_dom#133]
+Condition : (((((isnotnull(d_year#131) AND isnotnull(d_moy#132)) AND isnotnull(d_dom#133)) AND (d_year#131 = 1998)) AND (d_moy#132 = 12)) AND (d_dom#133 = 16))
+
+(135) Project [codegen id : 1]
+Output [1]: [d_week_seq#130]
+Input [4]: [d_week_seq#130, d_year#131, d_moy#132, d_dom#133]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/simplified.txt
new file mode 100644
index 0000000000000..cb9b410ea8568
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/simplified.txt
@@ -0,0 +1,213 @@
+TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales]
+  WholeStageCodegen (52)
+    BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id]
+      Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+        Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+          Subquery #4
+            WholeStageCodegen (8)
+              HashAggregate [sum,count] [avg(CheckOverflow((promote_precision(cast(cast(quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price as decimal(12,2)))), DecimalType(18,2), true)),average_sales,sum,count]
+                InputAdapter
+                  Exchange #13
+                    WholeStageCodegen (7)
+                      HashAggregate [quantity,list_price] [sum,count,sum,count]
+                        InputAdapter
+                          Union
+                            WholeStageCodegen (2)
+                              Project [ss_quantity,ss_list_price]
+                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk]
+                                        SubqueryBroadcast [d_date_sk] #5
+                                          ReusedExchange [d_date_sk] #14
+                                  InputAdapter
+                                    BroadcastExchange #14
+                                      WholeStageCodegen (1)
+                                        Project [d_date_sk]
+                                          Filter [d_year,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_year]
+                            WholeStageCodegen (4)
+                              Project [cs_quantity,cs_list_price]
+                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk]
+                                        ReusedSubquery [d_date_sk] #5
+                                  InputAdapter
+                                    ReusedExchange [d_date_sk] #14
+                            WholeStageCodegen (6)
+                              Project [ws_quantity,ws_list_price]
+                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk]
+                                        ReusedSubquery [d_date_sk] #5
+                                  InputAdapter
+                                    ReusedExchange [d_date_sk] #14
+          HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+            InputAdapter
+              Exchange [i_brand_id,i_class_id,i_category_id] #1
+                WholeStageCodegen (25)
+                  HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count]
+                    Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id]
+                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                        Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id]
+                          BroadcastHashJoin [ss_item_sk,i_item_sk]
+                            BroadcastHashJoin [ss_item_sk,ss_item_sk]
+                              Filter [ss_item_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk]
+                                      SubqueryBroadcast [d_date_sk] #1
+                                        ReusedExchange [d_date_sk] #2
+                              InputAdapter
+                                BroadcastExchange #3
+                                  WholeStageCodegen (11)
+                                    Project [i_item_sk]
+                                      BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id]
+                                        Filter [i_brand_id,i_class_id,i_category_id]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                        InputAdapter
+                                          BroadcastExchange #4
+                                            WholeStageCodegen (10)
+                                              HashAggregate [brand_id,class_id,category_id]
+                                                HashAggregate [brand_id,class_id,category_id]
+                                                  BroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id]
+                                                    HashAggregate [brand_id,class_id,category_id]
+                                                      InputAdapter
+                                                        Exchange [brand_id,class_id,category_id] #5
+                                                          WholeStageCodegen (6)
+                                                            HashAggregate [brand_id,class_id,category_id]
+                                                              Project [i_brand_id,i_class_id,i_category_id]
+                                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                  Project [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id]
+                                                                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                                      Filter [ss_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.store_sales [ss_item_sk,ss_sold_date_sk]
+                                                                              SubqueryBroadcast [d_date_sk] #2
+                                                                                ReusedExchange [d_date_sk] #6
+                                                                      InputAdapter
+                                                                        BroadcastExchange #7
+                                                                          WholeStageCodegen (4)
+                                                                            BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id]
+                                                                              Filter [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                              InputAdapter
+                                                                                BroadcastExchange #8
+                                                                                  WholeStageCodegen (3)
+                                                                                    Project [i_brand_id,i_class_id,i_category_id]
+                                                                                      BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                                        Project [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                          BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                                                                            Filter [cs_item_sk]
+                                                                                              ColumnarToRow
+                                                                                                InputAdapter
+                                                                                                  Scan parquet default.catalog_sales [cs_item_sk,cs_sold_date_sk]
+                                                                                                    ReusedSubquery [d_date_sk] #2
+                                                                                            InputAdapter
+                                                                                              BroadcastExchange #9
+                                                                                                WholeStageCodegen (1)
+                                                                                                  Filter [i_item_sk]
+                                                                                                    ColumnarToRow
+                                                                                                      InputAdapter
+                                                                                                        Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                        InputAdapter
+                                                                                          BroadcastExchange #6
+                                                                                            WholeStageCodegen (2)
+                                                                                              Project [d_date_sk]
+                                                                                                Filter [d_year,d_date_sk]
+                                                                                                  ColumnarToRow
+                                                                                                    InputAdapter
+                                                                                                      Scan parquet default.date_dim [d_date_sk,d_year]
+                                                                  InputAdapter
+                                                                    ReusedExchange [d_date_sk] #6
+                                                    InputAdapter
+                                                      BroadcastExchange #10
+                                                        WholeStageCodegen (9)
+                                                          Project [i_brand_id,i_class_id,i_category_id]
+                                                            BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                              Project [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id]
+                                                                BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                                                  Filter [ws_item_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.web_sales [ws_item_sk,ws_sold_date_sk]
+                                                                          ReusedSubquery [d_date_sk] #2
+                                                                  InputAdapter
+                                                                    ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9
+                                                              InputAdapter
+                                                                ReusedExchange [d_date_sk] #6
+                            InputAdapter
+                              BroadcastExchange #11
+                                WholeStageCodegen (23)
+                                  BroadcastHashJoin [i_item_sk,ss_item_sk]
+                                    Filter [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                    InputAdapter
+                                      ReusedExchange [ss_item_sk] #12
+                        InputAdapter
+                          BroadcastExchange #2
+                            WholeStageCodegen (24)
+                              Project [d_date_sk]
+                                Filter [d_week_seq,d_date_sk]
+                                  Subquery #3
+                                    WholeStageCodegen (1)
+                                      Project [d_week_seq]
+                                        Filter [d_year,d_moy,d_dom]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.date_dim [d_week_seq,d_year,d_moy,d_dom]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.date_dim [d_date_sk,d_week_seq]
+      InputAdapter
+        BroadcastExchange #15
+          WholeStageCodegen (51)
+            Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+              Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                ReusedSubquery [average_sales] #4
+                HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                  InputAdapter
+                    Exchange [i_brand_id,i_class_id,i_category_id] #16
+                      WholeStageCodegen (50)
+                        HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count]
+                          Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id]
+                            BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                              Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id]
+                                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                  BroadcastHashJoin [ss_item_sk,ss_item_sk]
+                                    Filter [ss_item_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk]
+                                            SubqueryBroadcast [d_date_sk] #6
+                                              ReusedExchange [d_date_sk] #17
+                                    InputAdapter
+                                      ReusedExchange [ss_item_sk] #12
+                                  InputAdapter
+                                    ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11
+                              InputAdapter
+                                BroadcastExchange #17
+                                  WholeStageCodegen (49)
+                                    Project [d_date_sk]
+                                      Filter [d_week_seq,d_date_sk]
+                                        Subquery #7
+                                          WholeStageCodegen (1)
+                                            Project [d_week_seq]
+                                              Filter [d_year,d_moy,d_dom]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_week_seq,d_year,d_moy,d_dom]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.date_dim [d_date_sk,d_week_seq]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.sf100/explain.txt
new file mode 100644
index 0000000000000..ef5f915e1b17e
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.sf100/explain.txt
@@ -0,0 +1,1432 @@
+== Physical Plan ==
+TakeOrderedAndProject (213)
++- * HashAggregate (212)
+   +- Exchange (211)
+      +- * HashAggregate (210)
+         +- Union (209)
+            :- * HashAggregate (132)
+            :  +- Exchange (131)
+            :     +- * HashAggregate (130)
+            :        +- Union (129)
+            :           :- * Project (90)
+            :           :  +- * Filter (89)
+            :           :     +- * HashAggregate (88)
+            :           :        +- Exchange (87)
+            :           :           +- * HashAggregate (86)
+            :           :              +- * Project (85)
+            :           :                 +- * BroadcastHashJoin Inner BuildRight (84)
+            :           :                    :- * Project (74)
+            :           :                    :  +- * BroadcastHashJoin Inner BuildRight (73)
+            :           :                    :     :- * SortMergeJoin LeftSemi (67)
+            :           :                    :     :  :- * Sort (5)
+            :           :                    :     :  :  +- Exchange (4)
+            :           :                    :     :  :     +- * Filter (3)
+            :           :                    :     :  :        +- * ColumnarToRow (2)
+            :           :                    :     :  :           +- Scan parquet default.store_sales (1)
+            :           :                    :     :  +- * Sort (66)
+            :           :                    :     :     +- Exchange (65)
+            :           :                    :     :        +- * Project (64)
+            :           :                    :     :           +- * BroadcastHashJoin Inner BuildRight (63)
+            :           :                    :     :              :- * Filter (8)
+            :           :                    :     :              :  +- * ColumnarToRow (7)
+            :           :                    :     :              :     +- Scan parquet default.item (6)
+            :           :                    :     :              +- BroadcastExchange (62)
+            :           :                    :     :                 +- * HashAggregate (61)
+            :           :                    :     :                    +- Exchange (60)
+            :           :                    :     :                       +- * HashAggregate (59)
+            :           :                    :     :                          +- * SortMergeJoin LeftSemi (58)
+            :           :                    :     :                             :- * Sort (46)
+            :           :                    :     :                             :  +- Exchange (45)
+            :           :                    :     :                             :     +- * HashAggregate (44)
+            :           :                    :     :                             :        +- Exchange (43)
+            :           :                    :     :                             :           +- * HashAggregate (42)
+            :           :                    :     :                             :              +- * Project (41)
+            :           :                    :     :                             :                 +- * BroadcastHashJoin Inner BuildRight (40)
+            :           :                    :     :                             :                    :- * Project (18)
+            :           :                    :     :                             :                    :  +- * BroadcastHashJoin Inner BuildRight (17)
+            :           :                    :     :                             :                    :     :- * Filter (11)
+            :           :                    :     :                             :                    :     :  +- * ColumnarToRow (10)
+            :           :                    :     :                             :                    :     :     +- Scan parquet default.store_sales (9)
+            :           :                    :     :                             :                    :     +- BroadcastExchange (16)
+            :           :                    :     :                             :                    :        +- * Project (15)
+            :           :                    :     :                             :                    :           +- * Filter (14)
+            :           :                    :     :                             :                    :              +- * ColumnarToRow (13)
+            :           :                    :     :                             :                    :                 +- Scan parquet default.date_dim (12)
+            :           :                    :     :                             :                    +- BroadcastExchange (39)
+            :           :                    :     :                             :                       +- * SortMergeJoin LeftSemi (38)
+            :           :                    :     :                             :                          :- * Sort (23)
+            :           :                    :     :                             :                          :  +- Exchange (22)
+            :           :                    :     :                             :                          :     +- * Filter (21)
+            :           :                    :     :                             :                          :        +- * ColumnarToRow (20)
+            :           :                    :     :                             :                          :           +- Scan parquet default.item (19)
+            :           :                    :     :                             :                          +- * Sort (37)
+            :           :                    :     :                             :                             +- Exchange (36)
+            :           :                    :     :                             :                                +- * Project (35)
+            :           :                    :     :                             :                                   +- * BroadcastHashJoin Inner BuildRight (34)
+            :           :                    :     :                             :                                      :- * Project (29)
+            :           :                    :     :                             :                                      :  +- * BroadcastHashJoin Inner BuildRight (28)
+            :           :                    :     :                             :                                      :     :- * Filter (26)
+            :           :                    :     :                             :                                      :     :  +- * ColumnarToRow (25)
+            :           :                    :     :                             :                                      :     :     +- Scan parquet default.catalog_sales (24)
+            :           :                    :     :                             :                                      :     +- ReusedExchange (27)
+            :           :                    :     :                             :                                      +- BroadcastExchange (33)
+            :           :                    :     :                             :                                         +- * Filter (32)
+            :           :                    :     :                             :                                            +- * ColumnarToRow (31)
+            :           :                    :     :                             :                                               +- Scan parquet default.item (30)
+            :           :                    :     :                             +- * Sort (57)
+            :           :                    :     :                                +- Exchange (56)
+            :           :                    :     :                                   +- * Project (55)
+            :           :                    :     :                                      +- * BroadcastHashJoin Inner BuildRight (54)
+            :           :                    :     :                                         :- * Project (52)
+            :           :                    :     :                                         :  +- * BroadcastHashJoin Inner BuildRight (51)
+            :           :                    :     :                                         :     :- * Filter (49)
+            :           :                    :     :                                         :     :  +- * ColumnarToRow (48)
+            :           :                    :     :                                         :     :     +- Scan parquet default.web_sales (47)
+            :           :                    :     :                                         :     +- ReusedExchange (50)
+            :           :                    :     :                                         +- ReusedExchange (53)
+            :           :                    :     +- BroadcastExchange (72)
+            :           :                    :        +- * Project (71)
+            :           :                    :           +- * Filter (70)
+            :           :                    :              +- * ColumnarToRow (69)
+            :           :                    :                 +- Scan parquet default.date_dim (68)
+            :           :                    +- BroadcastExchange (83)
+            :           :                       +- * SortMergeJoin LeftSemi (82)
+            :           :                          :- * Sort (79)
+            :           :                          :  +- Exchange (78)
+            :           :                          :     +- * Filter (77)
+            :           :                          :        +- * ColumnarToRow (76)
+            :           :                          :           +- Scan parquet default.item (75)
+            :           :                          +- * Sort (81)
+            :           :                             +- ReusedExchange (80)
+            :           :- * Project (109)
+            :           :  +- * Filter (108)
+            :           :     +- * HashAggregate (107)
+            :           :        +- Exchange (106)
+            :           :           +- * HashAggregate (105)
+            :           :              +- * Project (104)
+            :           :                 +- * BroadcastHashJoin Inner BuildRight (103)
+            :           :                    :- * Project (101)
+            :           :                    :  +- * BroadcastHashJoin Inner BuildRight (100)
+            :           :                    :     :- * SortMergeJoin LeftSemi (98)
+            :           :                    :     :  :- * Sort (95)
+            :           :                    :     :  :  +- Exchange (94)
+            :           :                    :     :  :     +- * Filter (93)
+            :           :                    :     :  :        +- * ColumnarToRow (92)
+            :           :                    :     :  :           +- Scan parquet default.catalog_sales (91)
+            :           :                    :     :  +- * Sort (97)
+            :           :                    :     :     +- ReusedExchange (96)
+            :           :                    :     +- ReusedExchange (99)
+            :           :                    +- ReusedExchange (102)
+            :           +- * Project (128)
+            :              +- * Filter (127)
+            :                 +- * HashAggregate (126)
+            :                    +- Exchange (125)
+            :                       +- * HashAggregate (124)
+            :                          +- * Project (123)
+            :                             +- * BroadcastHashJoin Inner BuildRight (122)
+            :                                :- * Project (120)
+            :                                :  +- * BroadcastHashJoin Inner BuildRight (119)
+            :                                :     :- * SortMergeJoin LeftSemi (117)
+            :                                :     :  :- * Sort (114)
+            :                                :     :  :  +- Exchange (113)
+            :                                :     :  :     +- * Filter (112)
+            :                                :     :  :        +- * ColumnarToRow (111)
+            :                                :     :  :           +- Scan parquet default.web_sales (110)
+            :                                :     :  +- * Sort (116)
+            :                                :     :     +- ReusedExchange (115)
+            :                                :     +- ReusedExchange (118)
+            :                                +- ReusedExchange (121)
+            :- * HashAggregate (151)
+            :  +- Exchange (150)
+            :     +- * HashAggregate (149)
+            :        +- * HashAggregate (148)
+            :           +- Exchange (147)
+            :              +- * HashAggregate (146)
+            :                 +- Union (145)
+            :                    :- * Project (136)
+            :                    :  +- * Filter (135)
+            :                    :     +- * HashAggregate (134)
+            :                    :        +- ReusedExchange (133)
+            :                    :- * Project (140)
+            :                    :  +- * Filter (139)
+            :                    :     +- * HashAggregate (138)
+            :                    :        +- ReusedExchange (137)
+            :                    +- * Project (144)
+            :                       +- * Filter (143)
+            :                          +- * HashAggregate (142)
+            :                             +- ReusedExchange (141)
+            :- * HashAggregate (170)
+            :  +- Exchange (169)
+            :     +- * HashAggregate (168)
+            :        +- * HashAggregate (167)
+            :           +- Exchange (166)
+            :              +- * HashAggregate (165)
+            :                 +- Union (164)
+            :                    :- * Project (155)
+            :                    :  +- * Filter (154)
+            :                    :     +- * HashAggregate (153)
+            :                    :        +- ReusedExchange (152)
+            :                    :- * Project (159)
+            :                    :  +- * Filter (158)
+            :                    :     +- * HashAggregate (157)
+            :                    :        +- ReusedExchange (156)
+            :                    +- * Project (163)
+            :                       +- * Filter (162)
+            :                          +- * HashAggregate (161)
+            :                             +- ReusedExchange (160)
+            :- * HashAggregate (189)
+            :  +- Exchange (188)
+            :     +- * HashAggregate (187)
+            :        +- * HashAggregate (186)
+            :           +- Exchange (185)
+            :              +- * HashAggregate (184)
+            :                 +- Union (183)
+            :                    :- * Project (174)
+            :                    :  +- * Filter (173)
+            :                    :     +- * HashAggregate (172)
+            :                    :        +- ReusedExchange (171)
+            :                    :- * Project (178)
+            :                    :  +- * Filter (177)
+            :                    :     +- * HashAggregate (176)
+            :                    :        +- ReusedExchange (175)
+            :                    +- * Project (182)
+            :                       +- * Filter (181)
+            :                          +- * HashAggregate (180)
+            :                             +- ReusedExchange (179)
+            +- * HashAggregate (208)
+               +- Exchange (207)
+                  +- * HashAggregate (206)
+                     +- * HashAggregate (205)
+                        +- Exchange (204)
+                           +- * HashAggregate (203)
+                              +- Union (202)
+                                 :- * Project (193)
+                                 :  +- * Filter (192)
+                                 :     +- * HashAggregate (191)
+                                 :        +- ReusedExchange (190)
+                                 :- * Project (197)
+                                 :  +- * Filter (196)
+                                 :     +- * HashAggregate (195)
+                                 :        +- ReusedExchange (194)
+                                 +- * Project (201)
+                                    +- * Filter (200)
+                                       +- * HashAggregate (199)
+                                          +- ReusedExchange (198)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_quantity:int,ss_list_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 1]
+Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+Condition : isnotnull(ss_item_sk#1)
+
+(4) Exchange
+Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#6]
+
+(5) Sort [codegen id : 2]
+Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+Arguments: [ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.item
+Output [4]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(7) ColumnarToRow [codegen id : 20]
+Input [4]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10]
+
+(8) Filter [codegen id : 20]
+Input [4]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10]
+Condition : ((isnotnull(i_brand_id#8) AND isnotnull(i_class_id#9)) AND isnotnull(i_category_id#10))
+
+(9) Scan parquet default.store_sales
+Output [2]: [ss_item_sk#11, ss_sold_date_sk#12]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_sold_date_sk#12 IN dynamicpruning#13)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int>
+
+(10) ColumnarToRow [codegen id : 11]
+Input [2]: [ss_item_sk#11, ss_sold_date_sk#12]
+
+(11) Filter [codegen id : 11]
+Input [2]: [ss_item_sk#11, ss_sold_date_sk#12]
+Condition : isnotnull(ss_item_sk#11)
+
+(12) Scan parquet default.date_dim
+Output [2]: [d_date_sk#14, d_year#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(13) ColumnarToRow [codegen id : 3]
+Input [2]: [d_date_sk#14, d_year#15]
+
+(14) Filter [codegen id : 3]
+Input [2]: [d_date_sk#14, d_year#15]
+Condition : (((isnotnull(d_year#15) AND (d_year#15 >= 1999)) AND (d_year#15 <= 2001)) AND isnotnull(d_date_sk#14))
+
+(15) Project [codegen id : 3]
+Output [1]: [d_date_sk#14]
+Input [2]: [d_date_sk#14, d_year#15]
+
+(16) BroadcastExchange
+Input [1]: [d_date_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#16]
+
+(17) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ss_sold_date_sk#12]
+Right keys [1]: [d_date_sk#14]
+Join condition: None
+
+(18) Project [codegen id : 11]
+Output [1]: [ss_item_sk#11]
+Input [3]: [ss_item_sk#11, ss_sold_date_sk#12, d_date_sk#14]
+
+(19) Scan parquet default.item
+Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(20) ColumnarToRow [codegen id : 4]
+Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+
+(21) Filter [codegen id : 4]
+Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+Condition : (((isnotnull(i_item_sk#17) AND isnotnull(i_brand_id#18)) AND isnotnull(i_class_id#19)) AND isnotnull(i_category_id#20))
+
+(22) Exchange
+Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+Arguments: hashpartitioning(coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20), 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(23) Sort [codegen id : 5]
+Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+Arguments: [coalesce(i_brand_id#18, 0) ASC NULLS FIRST, isnull(i_brand_id#18) ASC NULLS FIRST, coalesce(i_class_id#19, 0) ASC NULLS FIRST, isnull(i_class_id#19) ASC NULLS FIRST, coalesce(i_category_id#20, 0) ASC NULLS FIRST, isnull(i_category_id#20) ASC NULLS FIRST], false, 0
+
+(24) Scan parquet default.catalog_sales
+Output [2]: [cs_item_sk#22, cs_sold_date_sk#23]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#23), dynamicpruningexpression(cs_sold_date_sk#23 IN dynamicpruning#13)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int>
+
+(25) ColumnarToRow [codegen id : 8]
+Input [2]: [cs_item_sk#22, cs_sold_date_sk#23]
+
+(26) Filter [codegen id : 8]
+Input [2]: [cs_item_sk#22, cs_sold_date_sk#23]
+Condition : isnotnull(cs_item_sk#22)
+
+(27) ReusedExchange [Reuses operator id: 16]
+Output [1]: [d_date_sk#24]
+
+(28) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [cs_sold_date_sk#23]
+Right keys [1]: [d_date_sk#24]
+Join condition: None
+
+(29) Project [codegen id : 8]
+Output [1]: [cs_item_sk#22]
+Input [3]: [cs_item_sk#22, cs_sold_date_sk#23, d_date_sk#24]
+
+(30) Scan parquet default.item
+Output [4]: [i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(31) ColumnarToRow [codegen id : 7]
+Input [4]: [i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28]
+
+(32) Filter [codegen id : 7]
+Input [4]: [i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28]
+Condition : isnotnull(i_item_sk#25)
+
+(33) BroadcastExchange
+Input [4]: [i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#29]
+
+(34) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [cs_item_sk#22]
+Right keys [1]: [i_item_sk#25]
+Join condition: None
+
+(35) Project [codegen id : 8]
+Output [3]: [i_brand_id#26, i_class_id#27, i_category_id#28]
+Input [5]: [cs_item_sk#22, i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28]
+
+(36) Exchange
+Input [3]: [i_brand_id#26, i_class_id#27, i_category_id#28]
+Arguments: hashpartitioning(coalesce(i_brand_id#26, 0), isnull(i_brand_id#26), coalesce(i_class_id#27, 0), isnull(i_class_id#27), coalesce(i_category_id#28, 0), isnull(i_category_id#28), 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(37) Sort [codegen id : 9]
+Input [3]: [i_brand_id#26, i_class_id#27, i_category_id#28]
+Arguments: [coalesce(i_brand_id#26, 0) ASC NULLS FIRST, isnull(i_brand_id#26) ASC NULLS FIRST, coalesce(i_class_id#27, 0) ASC NULLS FIRST, isnull(i_class_id#27) ASC NULLS FIRST, coalesce(i_category_id#28, 0) ASC NULLS FIRST, isnull(i_category_id#28) ASC NULLS FIRST], false, 0
+
+(38) SortMergeJoin [codegen id : 10]
+Left keys [6]: [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)]
+Right keys [6]: [coalesce(i_brand_id#26, 0), isnull(i_brand_id#26), coalesce(i_class_id#27, 0), isnull(i_class_id#27), coalesce(i_category_id#28, 0), isnull(i_category_id#28)]
+Join condition: None
+
+(39) BroadcastExchange
+Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#31]
+
+(40) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ss_item_sk#11]
+Right keys [1]: [i_item_sk#17]
+Join condition: None
+
+(41) Project [codegen id : 11]
+Output [3]: [i_brand_id#18 AS brand_id#32, i_class_id#19 AS class_id#33, i_category_id#20 AS category_id#34]
+Input [5]: [ss_item_sk#11, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20]
+
+(42) HashAggregate [codegen id : 11]
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Keys [3]: [brand_id#32, class_id#33, category_id#34]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#32, class_id#33, category_id#34]
+
+(43) Exchange
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Arguments: hashpartitioning(brand_id#32, class_id#33, category_id#34, 5), ENSURE_REQUIREMENTS, [id=#35]
+
+(44) HashAggregate [codegen id : 12]
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Keys [3]: [brand_id#32, class_id#33, category_id#34]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#32, class_id#33, category_id#34]
+
+(45) Exchange
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Arguments: hashpartitioning(coalesce(brand_id#32, 0), isnull(brand_id#32), coalesce(class_id#33, 0), isnull(class_id#33), coalesce(category_id#34, 0), isnull(category_id#34), 5), ENSURE_REQUIREMENTS, [id=#36]
+
+(46) Sort [codegen id : 13]
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Arguments: [coalesce(brand_id#32, 0) ASC NULLS FIRST, isnull(brand_id#32) ASC NULLS FIRST, coalesce(class_id#33, 0) ASC NULLS FIRST, isnull(class_id#33) ASC NULLS FIRST, coalesce(category_id#34, 0) ASC NULLS FIRST, isnull(category_id#34) ASC NULLS FIRST], false, 0
+
+(47) Scan parquet default.web_sales
+Output [2]: [ws_item_sk#37, ws_sold_date_sk#38]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#38), dynamicpruningexpression(ws_sold_date_sk#38 IN dynamicpruning#13)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int>
+
+(48) ColumnarToRow [codegen id : 16]
+Input [2]: [ws_item_sk#37, ws_sold_date_sk#38]
+
+(49) Filter [codegen id : 16]
+Input [2]: [ws_item_sk#37, ws_sold_date_sk#38]
+Condition : isnotnull(ws_item_sk#37)
+
+(50) ReusedExchange [Reuses operator id: 16]
+Output [1]: [d_date_sk#39]
+
+(51) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [ws_sold_date_sk#38]
+Right keys [1]: [d_date_sk#39]
+Join condition: None
+
+(52) Project [codegen id : 16]
+Output [1]: [ws_item_sk#37]
+Input [3]: [ws_item_sk#37, ws_sold_date_sk#38, d_date_sk#39]
+
+(53) ReusedExchange [Reuses operator id: 33]
+Output [4]: [i_item_sk#40, i_brand_id#41, i_class_id#42, i_category_id#43]
+
+(54) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [ws_item_sk#37]
+Right keys [1]: [i_item_sk#40]
+Join condition: None
+
+(55) Project [codegen id : 16]
+Output [3]: [i_brand_id#41, i_class_id#42, i_category_id#43]
+Input [5]: [ws_item_sk#37, i_item_sk#40, i_brand_id#41, i_class_id#42, i_category_id#43]
+
+(56) Exchange
+Input [3]: [i_brand_id#41, i_class_id#42, i_category_id#43]
+Arguments: hashpartitioning(coalesce(i_brand_id#41, 0), isnull(i_brand_id#41), coalesce(i_class_id#42, 0), isnull(i_class_id#42), coalesce(i_category_id#43, 0), isnull(i_category_id#43), 5), ENSURE_REQUIREMENTS, [id=#44]
+
+(57) Sort [codegen id : 17]
+Input [3]: [i_brand_id#41, i_class_id#42, i_category_id#43]
+Arguments: [coalesce(i_brand_id#41, 0) ASC NULLS FIRST, isnull(i_brand_id#41) ASC NULLS FIRST, coalesce(i_class_id#42, 0) ASC NULLS FIRST, isnull(i_class_id#42) ASC NULLS FIRST, coalesce(i_category_id#43, 0) ASC NULLS FIRST, isnull(i_category_id#43) ASC NULLS FIRST], false, 0
+
+(58) SortMergeJoin [codegen id : 18]
+Left keys [6]: [coalesce(brand_id#32, 0), isnull(brand_id#32), coalesce(class_id#33, 0), isnull(class_id#33), coalesce(category_id#34, 0), isnull(category_id#34)]
+Right keys [6]: [coalesce(i_brand_id#41, 0), isnull(i_brand_id#41), coalesce(i_class_id#42, 0), isnull(i_class_id#42), coalesce(i_category_id#43, 0), isnull(i_category_id#43)]
+Join condition: None
+
+(59) HashAggregate [codegen id : 18]
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Keys [3]: [brand_id#32, class_id#33, category_id#34]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#32, class_id#33, category_id#34]
+
+(60) Exchange
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Arguments: hashpartitioning(brand_id#32, class_id#33, category_id#34, 5), ENSURE_REQUIREMENTS, [id=#45]
+
+(61) HashAggregate [codegen id : 19]
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Keys [3]: [brand_id#32, class_id#33, category_id#34]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#32, class_id#33, category_id#34]
+
+(62) BroadcastExchange
+Input [3]: [brand_id#32, class_id#33, category_id#34]
+Arguments: HashedRelationBroadcastMode(List(input[0, int, true], input[1, int, true], input[2, int, true]),false), [id=#46]
+
+(63) BroadcastHashJoin [codegen id : 20]
+Left keys [3]: [i_brand_id#8, i_class_id#9, i_category_id#10]
+Right keys [3]: [brand_id#32, class_id#33, category_id#34]
+Join condition: None
+
+(64) Project [codegen id : 20]
+Output [1]: [i_item_sk#7 AS ss_item_sk#47]
+Input [7]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, brand_id#32, class_id#33, category_id#34]
+
+(65) Exchange
+Input [1]: [ss_item_sk#47]
+Arguments: hashpartitioning(ss_item_sk#47, 5), ENSURE_REQUIREMENTS, [id=#48]
+
+(66) Sort [codegen id : 21]
+Input [1]: [ss_item_sk#47]
+Arguments: [ss_item_sk#47 ASC NULLS FIRST], false, 0
+
+(67) SortMergeJoin [codegen id : 45]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [ss_item_sk#47]
+Join condition: None
+
+(68) Scan parquet default.date_dim
+Output [3]: [d_date_sk#49, d_year#50, d_moy#51]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,11), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(69) ColumnarToRow [codegen id : 22]
+Input [3]: [d_date_sk#49, d_year#50, d_moy#51]
+
+(70) Filter [codegen id : 22]
+Input [3]: [d_date_sk#49, d_year#50, d_moy#51]
+Condition : ((((isnotnull(d_year#50) AND isnotnull(d_moy#51)) AND (d_year#50 = 2000)) AND (d_moy#51 = 11)) AND isnotnull(d_date_sk#49))
+
+(71) Project [codegen id : 22]
+Output [1]: [d_date_sk#49]
+Input [3]: [d_date_sk#49, d_year#50, d_moy#51]
+
+(72) BroadcastExchange
+Input [1]: [d_date_sk#49]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#52]
+
+(73) BroadcastHashJoin [codegen id : 45]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#49]
+Join condition: None
+
+(74) Project [codegen id : 45]
+Output [3]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3]
+Input [5]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, d_date_sk#49]
+
+(75) Scan parquet default.item
+Output [4]: [i_item_sk#53, i_brand_id#54, i_class_id#55, i_category_id#56]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(76) ColumnarToRow [codegen id : 23]
+Input [4]: [i_item_sk#53, i_brand_id#54, i_class_id#55, i_category_id#56]
+
+(77) Filter [codegen id : 23]
+Input [4]: [i_item_sk#53, i_brand_id#54, i_class_id#55, i_category_id#56]
+Condition : isnotnull(i_item_sk#53)
+
+(78) Exchange
+Input [4]: [i_item_sk#53, i_brand_id#54, i_class_id#55, i_category_id#56]
+Arguments: hashpartitioning(i_item_sk#53, 5), ENSURE_REQUIREMENTS, [id=#57]
+
+(79) Sort [codegen id : 24]
+Input [4]: [i_item_sk#53, i_brand_id#54, i_class_id#55, i_category_id#56]
+Arguments: [i_item_sk#53 ASC NULLS FIRST], false, 0
+
+(80) ReusedExchange [Reuses operator id: unknown]
+Output [1]: [ss_item_sk#47]
+
+(81) Sort [codegen id : 43]
+Input [1]: [ss_item_sk#47]
+Arguments: [ss_item_sk#47 ASC NULLS FIRST], false, 0
+
+(82) SortMergeJoin [codegen id : 44]
+Left keys [1]: [i_item_sk#53]
+Right keys [1]: [ss_item_sk#47]
+Join condition: None
+
+(83) BroadcastExchange
+Input [4]: [i_item_sk#53, i_brand_id#54, i_class_id#55, i_category_id#56]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#58]
+
+(84) BroadcastHashJoin [codegen id : 45]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#53]
+Join condition: None
+
+(85) Project [codegen id : 45]
+Output [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#54, i_class_id#55, i_category_id#56]
+Input [7]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, i_item_sk#53, i_brand_id#54, i_class_id#55, i_category_id#56]
+
+(86) HashAggregate [codegen id : 45]
+Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#54, i_class_id#55, i_category_id#56]
+Keys [3]: [i_brand_id#54, i_class_id#55, i_category_id#56]
+Functions [2]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true)), partial_count(1)]
+Aggregate Attributes [3]: [sum#59, isEmpty#60, count#61]
+Results [6]: [i_brand_id#54, i_class_id#55, i_category_id#56, sum#62, isEmpty#63, count#64]
+
+(87) Exchange
+Input [6]: [i_brand_id#54, i_class_id#55, i_category_id#56, sum#62, isEmpty#63, count#64]
+Arguments: hashpartitioning(i_brand_id#54, i_class_id#55, i_category_id#56, 5), ENSURE_REQUIREMENTS, [id=#65]
+
+(88) HashAggregate [codegen id : 46]
+Input [6]: [i_brand_id#54, i_class_id#55, i_category_id#56, sum#62, isEmpty#63, count#64]
+Keys [3]: [i_brand_id#54, i_class_id#55, i_category_id#56]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#66, count(1)#67]
+Results [6]: [i_brand_id#54, i_class_id#55, i_category_id#56, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#66 AS sales#68, count(1)#67 AS number_sales#69, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#66 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#70]
+
+(89) Filter [codegen id : 46]
+Input [6]: [i_brand_id#54, i_class_id#55, i_category_id#56, sales#68, number_sales#69, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#70]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#70) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#70 as decimal(32,6)) > cast(Subquery scalar-subquery#71, [id=#72] as decimal(32,6))))
+
+(90) Project [codegen id : 46]
+Output [6]: [store AS channel#73, i_brand_id#54, i_class_id#55, i_category_id#56, sales#68, number_sales#69]
+Input [6]: [i_brand_id#54, i_class_id#55, i_category_id#56, sales#68, number_sales#69, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#70]
+
+(91) Scan parquet default.catalog_sales
+Output [4]: [cs_item_sk#74, cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#77), dynamicpruningexpression(cs_sold_date_sk#77 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int,cs_quantity:int,cs_list_price:decimal(7,2)>
+
+(92) ColumnarToRow [codegen id : 47]
+Input [4]: [cs_item_sk#74, cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77]
+
+(93) Filter [codegen id : 47]
+Input [4]: [cs_item_sk#74, cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77]
+Condition : isnotnull(cs_item_sk#74)
+
+(94) Exchange
+Input [4]: [cs_item_sk#74, cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77]
+Arguments: hashpartitioning(cs_item_sk#74, 5), ENSURE_REQUIREMENTS, [id=#78]
+
+(95) Sort [codegen id : 48]
+Input [4]: [cs_item_sk#74, cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77]
+Arguments: [cs_item_sk#74 ASC NULLS FIRST], false, 0
+
+(96) ReusedExchange [Reuses operator id: unknown]
+Output [1]: [ss_item_sk#47]
+
+(97) Sort [codegen id : 67]
+Input [1]: [ss_item_sk#47]
+Arguments: [ss_item_sk#47 ASC NULLS FIRST], false, 0
+
+(98) SortMergeJoin [codegen id : 91]
+Left keys [1]: [cs_item_sk#74]
+Right keys [1]: [ss_item_sk#47]
+Join condition: None
+
+(99) ReusedExchange [Reuses operator id: 72]
+Output [1]: [d_date_sk#79]
+
+(100) BroadcastHashJoin [codegen id : 91]
+Left keys [1]: [cs_sold_date_sk#77]
+Right keys [1]: [d_date_sk#79]
+Join condition: None
+
+(101) Project [codegen id : 91]
+Output [3]: [cs_item_sk#74, cs_quantity#75, cs_list_price#76]
+Input [5]: [cs_item_sk#74, cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77, d_date_sk#79]
+
+(102) ReusedExchange [Reuses operator id: 83]
+Output [4]: [i_item_sk#80, i_brand_id#81, i_class_id#82, i_category_id#83]
+
+(103) BroadcastHashJoin [codegen id : 91]
+Left keys [1]: [cs_item_sk#74]
+Right keys [1]: [i_item_sk#80]
+Join condition: None
+
+(104) Project [codegen id : 91]
+Output [5]: [cs_quantity#75, cs_list_price#76, i_brand_id#81, i_class_id#82, i_category_id#83]
+Input [7]: [cs_item_sk#74, cs_quantity#75, cs_list_price#76, i_item_sk#80, i_brand_id#81, i_class_id#82, i_category_id#83]
+
+(105) HashAggregate [codegen id : 91]
+Input [5]: [cs_quantity#75, cs_list_price#76, i_brand_id#81, i_class_id#82, i_category_id#83]
+Keys [3]: [i_brand_id#81, i_class_id#82, i_category_id#83]
+Functions [2]: [partial_sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#76 as decimal(12,2)))), DecimalType(18,2), true)), partial_count(1)]
+Aggregate Attributes [3]: [sum#84, isEmpty#85, count#86]
+Results [6]: [i_brand_id#81, i_class_id#82, i_category_id#83, sum#87, isEmpty#88, count#89]
+
+(106) Exchange
+Input [6]: [i_brand_id#81, i_class_id#82, i_category_id#83, sum#87, isEmpty#88, count#89]
+Arguments: hashpartitioning(i_brand_id#81, i_class_id#82, i_category_id#83, 5), ENSURE_REQUIREMENTS, [id=#90]
+
+(107) HashAggregate [codegen id : 92]
+Input [6]: [i_brand_id#81, i_class_id#82, i_category_id#83, sum#87, isEmpty#88, count#89]
+Keys [3]: [i_brand_id#81, i_class_id#82, i_category_id#83]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#76 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#76 as decimal(12,2)))), DecimalType(18,2), true))#91, count(1)#92]
+Results [6]: [i_brand_id#81, i_class_id#82, i_category_id#83, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#76 as decimal(12,2)))), DecimalType(18,2), true))#91 AS sales#93, count(1)#92 AS number_sales#94, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#76 as decimal(12,2)))), DecimalType(18,2), true))#91 AS sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#76 as decimal(12,2)))), DecimalType(18,2), true))#95]
+
+(108) Filter [codegen id : 92]
+Input [6]: [i_brand_id#81, i_class_id#82, i_category_id#83, sales#93, number_sales#94, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#76 as decimal(12,2)))), DecimalType(18,2), true))#95]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#76 as decimal(12,2)))), DecimalType(18,2), true))#95) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#76 as decimal(12,2)))), DecimalType(18,2), true))#95 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#71, [id=#72] as decimal(32,6))))
+
+(109) Project [codegen id : 92]
+Output [6]: [catalog AS channel#96, i_brand_id#81, i_class_id#82, i_category_id#83, sales#93, number_sales#94]
+Input [6]: [i_brand_id#81, i_class_id#82, i_category_id#83, sales#93, number_sales#94, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#75 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#76 as decimal(12,2)))), DecimalType(18,2), true))#95]
+
+(110) Scan parquet default.web_sales
+Output [4]: [ws_item_sk#97, ws_quantity#98, ws_list_price#99, ws_sold_date_sk#100]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#100), dynamicpruningexpression(ws_sold_date_sk#100 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_quantity:int,ws_list_price:decimal(7,2)>
+
+(111) ColumnarToRow [codegen id : 93]
+Input [4]: [ws_item_sk#97, ws_quantity#98, ws_list_price#99, ws_sold_date_sk#100]
+
+(112) Filter [codegen id : 93]
+Input [4]: [ws_item_sk#97, ws_quantity#98, ws_list_price#99, ws_sold_date_sk#100]
+Condition : isnotnull(ws_item_sk#97)
+
+(113) Exchange
+Input [4]: [ws_item_sk#97, ws_quantity#98, ws_list_price#99, ws_sold_date_sk#100]
+Arguments: hashpartitioning(ws_item_sk#97, 5), ENSURE_REQUIREMENTS, [id=#101]
+
+(114) Sort [codegen id : 94]
+Input [4]: [ws_item_sk#97, ws_quantity#98, ws_list_price#99, ws_sold_date_sk#100]
+Arguments: [ws_item_sk#97 ASC NULLS FIRST], false, 0
+
+(115) ReusedExchange [Reuses operator id: unknown]
+Output [1]: [ss_item_sk#47]
+
+(116) Sort [codegen id : 113]
+Input [1]: [ss_item_sk#47]
+Arguments: [ss_item_sk#47 ASC NULLS FIRST], false, 0
+
+(117) SortMergeJoin [codegen id : 137]
+Left keys [1]: [ws_item_sk#97]
+Right keys [1]: [ss_item_sk#47]
+Join condition: None
+
+(118) ReusedExchange [Reuses operator id: 72]
+Output [1]: [d_date_sk#102]
+
+(119) BroadcastHashJoin [codegen id : 137]
+Left keys [1]: [ws_sold_date_sk#100]
+Right keys [1]: [d_date_sk#102]
+Join condition: None
+
+(120) Project [codegen id : 137]
+Output [3]: [ws_item_sk#97, ws_quantity#98, ws_list_price#99]
+Input [5]: [ws_item_sk#97, ws_quantity#98, ws_list_price#99, ws_sold_date_sk#100, d_date_sk#102]
+
+(121) ReusedExchange [Reuses operator id: 83]
+Output [4]: [i_item_sk#103, i_brand_id#104, i_class_id#105, i_category_id#106]
+
+(122) BroadcastHashJoin [codegen id : 137]
+Left keys [1]: [ws_item_sk#97]
+Right keys [1]: [i_item_sk#103]
+Join condition: None
+
+(123) Project [codegen id : 137]
+Output [5]: [ws_quantity#98, ws_list_price#99, i_brand_id#104, i_class_id#105, i_category_id#106]
+Input [7]: [ws_item_sk#97, ws_quantity#98, ws_list_price#99, i_item_sk#103, i_brand_id#104, i_class_id#105, i_category_id#106]
+
+(124) HashAggregate [codegen id : 137]
+Input [5]: [ws_quantity#98, ws_list_price#99, i_brand_id#104, i_class_id#105, i_category_id#106]
+Keys [3]: [i_brand_id#104, i_class_id#105, i_category_id#106]
+Functions [2]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#98 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#99 as decimal(12,2)))), DecimalType(18,2), true)), partial_count(1)]
+Aggregate Attributes [3]: [sum#107, isEmpty#108, count#109]
+Results [6]: [i_brand_id#104, i_class_id#105, i_category_id#106, sum#110, isEmpty#111, count#112]
+
+(125) Exchange
+Input [6]: [i_brand_id#104, i_class_id#105, i_category_id#106, sum#110, isEmpty#111, count#112]
+Arguments: hashpartitioning(i_brand_id#104, i_class_id#105, i_category_id#106, 5), ENSURE_REQUIREMENTS, [id=#113]
+
+(126) HashAggregate [codegen id : 138]
+Input [6]: [i_brand_id#104, i_class_id#105, i_category_id#106, sum#110, isEmpty#111, count#112]
+Keys [3]: [i_brand_id#104, i_class_id#105, i_category_id#106]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#98 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#99 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#98 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#99 as decimal(12,2)))), DecimalType(18,2), true))#114, count(1)#115]
+Results [6]: [i_brand_id#104, i_class_id#105, i_category_id#106, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#98 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#99 as decimal(12,2)))), DecimalType(18,2), true))#114 AS sales#116, count(1)#115 AS number_sales#117, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#98 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#99 as decimal(12,2)))), DecimalType(18,2), true))#114 AS sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#98 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#99 as decimal(12,2)))), DecimalType(18,2), true))#118]
+
+(127) Filter [codegen id : 138]
+Input [6]: [i_brand_id#104, i_class_id#105, i_category_id#106, sales#116, number_sales#117, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#98 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#99 as decimal(12,2)))), DecimalType(18,2), true))#118]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#98 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#99 as decimal(12,2)))), DecimalType(18,2), true))#118) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#98 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#99 as decimal(12,2)))), DecimalType(18,2), true))#118 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#71, [id=#72] as decimal(32,6))))
+
+(128) Project [codegen id : 138]
+Output [6]: [web AS channel#119, i_brand_id#104, i_class_id#105, i_category_id#106, sales#116, number_sales#117]
+Input [6]: [i_brand_id#104, i_class_id#105, i_category_id#106, sales#116, number_sales#117, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#98 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#99 as decimal(12,2)))), DecimalType(18,2), true))#118]
+
+(129) Union
+
+(130) HashAggregate [codegen id : 139]
+Input [6]: [channel#73, i_brand_id#54, i_class_id#55, i_category_id#56, sales#68, number_sales#69]
+Keys [4]: [channel#73, i_brand_id#54, i_class_id#55, i_category_id#56]
+Functions [2]: [partial_sum(sales#68), partial_sum(number_sales#69)]
+Aggregate Attributes [3]: [sum#120, isEmpty#121, sum#122]
+Results [7]: [channel#73, i_brand_id#54, i_class_id#55, i_category_id#56, sum#123, isEmpty#124, sum#125]
+
+(131) Exchange
+Input [7]: [channel#73, i_brand_id#54, i_class_id#55, i_category_id#56, sum#123, isEmpty#124, sum#125]
+Arguments: hashpartitioning(channel#73, i_brand_id#54, i_class_id#55, i_category_id#56, 5), ENSURE_REQUIREMENTS, [id=#126]
+
+(132) HashAggregate [codegen id : 140]
+Input [7]: [channel#73, i_brand_id#54, i_class_id#55, i_category_id#56, sum#123, isEmpty#124, sum#125]
+Keys [4]: [channel#73, i_brand_id#54, i_class_id#55, i_category_id#56]
+Functions [2]: [sum(sales#68), sum(number_sales#69)]
+Aggregate Attributes [2]: [sum(sales#68)#127, sum(number_sales#69)#128]
+Results [6]: [channel#73, i_brand_id#54, i_class_id#55, i_category_id#56, sum(sales#68)#127 AS sum_sales#129, sum(number_sales#69)#128 AS number_sales#130]
+
+(133) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_brand_id#131, i_class_id#132, i_category_id#133, sum#134, isEmpty#135, count#136]
+
+(134) HashAggregate [codegen id : 186]
+Input [6]: [i_brand_id#131, i_class_id#132, i_category_id#133, sum#134, isEmpty#135, count#136]
+Keys [3]: [i_brand_id#131, i_class_id#132, i_category_id#133]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#137 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#138 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#137 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#138 as decimal(12,2)))), DecimalType(18,2), true))#139, count(1)#140]
+Results [6]: [i_brand_id#131, i_class_id#132, i_category_id#133, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#137 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#138 as decimal(12,2)))), DecimalType(18,2), true))#139 AS sales#68, count(1)#140 AS number_sales#69, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#137 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#138 as decimal(12,2)))), DecimalType(18,2), true))#139 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#137 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#138 as decimal(12,2)))), DecimalType(18,2), true))#141]
+
+(135) Filter [codegen id : 186]
+Input [6]: [i_brand_id#131, i_class_id#132, i_category_id#133, sales#68, number_sales#69, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#137 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#138 as decimal(12,2)))), DecimalType(18,2), true))#141]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#137 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#138 as decimal(12,2)))), DecimalType(18,2), true))#141) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#137 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#138 as decimal(12,2)))), DecimalType(18,2), true))#141 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#71, [id=#72] as decimal(32,6))))
+
+(136) Project [codegen id : 186]
+Output [6]: [store AS channel#73, i_brand_id#131, i_class_id#132, i_category_id#133, sales#68, number_sales#69]
+Input [6]: [i_brand_id#131, i_class_id#132, i_category_id#133, sales#68, number_sales#69, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#137 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#138 as decimal(12,2)))), DecimalType(18,2), true))#141]
+
+(137) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_brand_id#142, i_class_id#143, i_category_id#144, sum#145, isEmpty#146, count#147]
+
+(138) HashAggregate [codegen id : 232]
+Input [6]: [i_brand_id#142, i_class_id#143, i_category_id#144, sum#145, isEmpty#146, count#147]
+Keys [3]: [i_brand_id#142, i_class_id#143, i_category_id#144]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#148 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#149 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#148 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#149 as decimal(12,2)))), DecimalType(18,2), true))#150, count(1)#151]
+Results [6]: [i_brand_id#142, i_class_id#143, i_category_id#144, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#148 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#149 as decimal(12,2)))), DecimalType(18,2), true))#150 AS sales#93, count(1)#151 AS number_sales#94, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#148 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#149 as decimal(12,2)))), DecimalType(18,2), true))#150 AS sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#148 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#149 as decimal(12,2)))), DecimalType(18,2), true))#152]
+
+(139) Filter [codegen id : 232]
+Input [6]: [i_brand_id#142, i_class_id#143, i_category_id#144, sales#93, number_sales#94, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#148 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#149 as decimal(12,2)))), DecimalType(18,2), true))#152]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#148 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#149 as decimal(12,2)))), DecimalType(18,2), true))#152) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#148 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#149 as decimal(12,2)))), DecimalType(18,2), true))#152 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#71, [id=#72] as decimal(32,6))))
+
+(140) Project [codegen id : 232]
+Output [6]: [catalog AS channel#96, i_brand_id#142, i_class_id#143, i_category_id#144, sales#93, number_sales#94]
+Input [6]: [i_brand_id#142, i_class_id#143, i_category_id#144, sales#93, number_sales#94, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#148 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#149 as decimal(12,2)))), DecimalType(18,2), true))#152]
+
+(141) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_brand_id#153, i_class_id#154, i_category_id#155, sum#156, isEmpty#157, count#158]
+
+(142) HashAggregate [codegen id : 278]
+Input [6]: [i_brand_id#153, i_class_id#154, i_category_id#155, sum#156, isEmpty#157, count#158]
+Keys [3]: [i_brand_id#153, i_class_id#154, i_category_id#155]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#159 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#160 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#159 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#160 as decimal(12,2)))), DecimalType(18,2), true))#161, count(1)#162]
+Results [6]: [i_brand_id#153, i_class_id#154, i_category_id#155, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#159 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#160 as decimal(12,2)))), DecimalType(18,2), true))#161 AS sales#116, count(1)#162 AS number_sales#117, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#159 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#160 as decimal(12,2)))), DecimalType(18,2), true))#161 AS sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#159 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#160 as decimal(12,2)))), DecimalType(18,2), true))#163]
+
+(143) Filter [codegen id : 278]
+Input [6]: [i_brand_id#153, i_class_id#154, i_category_id#155, sales#116, number_sales#117, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#159 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#160 as decimal(12,2)))), DecimalType(18,2), true))#163]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#159 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#160 as decimal(12,2)))), DecimalType(18,2), true))#163) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#159 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#160 as decimal(12,2)))), DecimalType(18,2), true))#163 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#71, [id=#72] as decimal(32,6))))
+
+(144) Project [codegen id : 278]
+Output [6]: [web AS channel#119, i_brand_id#153, i_class_id#154, i_category_id#155, sales#116, number_sales#117]
+Input [6]: [i_brand_id#153, i_class_id#154, i_category_id#155, sales#116, number_sales#117, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#159 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#160 as decimal(12,2)))), DecimalType(18,2), true))#163]
+
+(145) Union
+
+(146) HashAggregate [codegen id : 279]
+Input [6]: [channel#73, i_brand_id#131, i_class_id#132, i_category_id#133, sales#68, number_sales#69]
+Keys [4]: [channel#73, i_brand_id#131, i_class_id#132, i_category_id#133]
+Functions [2]: [partial_sum(sales#68), partial_sum(number_sales#69)]
+Aggregate Attributes [3]: [sum#164, isEmpty#165, sum#166]
+Results [7]: [channel#73, i_brand_id#131, i_class_id#132, i_category_id#133, sum#167, isEmpty#168, sum#169]
+
+(147) Exchange
+Input [7]: [channel#73, i_brand_id#131, i_class_id#132, i_category_id#133, sum#167, isEmpty#168, sum#169]
+Arguments: hashpartitioning(channel#73, i_brand_id#131, i_class_id#132, i_category_id#133, 5), ENSURE_REQUIREMENTS, [id=#170]
+
+(148) HashAggregate [codegen id : 280]
+Input [7]: [channel#73, i_brand_id#131, i_class_id#132, i_category_id#133, sum#167, isEmpty#168, sum#169]
+Keys [4]: [channel#73, i_brand_id#131, i_class_id#132, i_category_id#133]
+Functions [2]: [sum(sales#68), sum(number_sales#69)]
+Aggregate Attributes [2]: [sum(sales#68)#171, sum(number_sales#69)#172]
+Results [5]: [channel#73, i_brand_id#131, i_class_id#132, sum(sales#68)#171 AS sum_sales#129, sum(number_sales#69)#172 AS number_sales#130]
+
+(149) HashAggregate [codegen id : 280]
+Input [5]: [channel#73, i_brand_id#131, i_class_id#132, sum_sales#129, number_sales#130]
+Keys [3]: [channel#73, i_brand_id#131, i_class_id#132]
+Functions [2]: [partial_sum(sum_sales#129), partial_sum(number_sales#130)]
+Aggregate Attributes [3]: [sum#173, isEmpty#174, sum#175]
+Results [6]: [channel#73, i_brand_id#131, i_class_id#132, sum#176, isEmpty#177, sum#178]
+
+(150) Exchange
+Input [6]: [channel#73, i_brand_id#131, i_class_id#132, sum#176, isEmpty#177, sum#178]
+Arguments: hashpartitioning(channel#73, i_brand_id#131, i_class_id#132, 5), ENSURE_REQUIREMENTS, [id=#179]
+
+(151) HashAggregate [codegen id : 281]
+Input [6]: [channel#73, i_brand_id#131, i_class_id#132, sum#176, isEmpty#177, sum#178]
+Keys [3]: [channel#73, i_brand_id#131, i_class_id#132]
+Functions [2]: [sum(sum_sales#129), sum(number_sales#130)]
+Aggregate Attributes [2]: [sum(sum_sales#129)#180, sum(number_sales#130)#181]
+Results [6]: [channel#73, i_brand_id#131, i_class_id#132, null AS i_category_id#182, sum(sum_sales#129)#180 AS sum(sum_sales)#183, sum(number_sales#130)#181 AS sum(number_sales)#184]
+
+(152) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_brand_id#185, i_class_id#186, i_category_id#187, sum#188, isEmpty#189, count#190]
+
+(153) HashAggregate [codegen id : 327]
+Input [6]: [i_brand_id#185, i_class_id#186, i_category_id#187, sum#188, isEmpty#189, count#190]
+Keys [3]: [i_brand_id#185, i_class_id#186, i_category_id#187]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#191 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#192 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#191 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#192 as decimal(12,2)))), DecimalType(18,2), true))#193, count(1)#194]
+Results [6]: [i_brand_id#185, i_class_id#186, i_category_id#187, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#191 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#192 as decimal(12,2)))), DecimalType(18,2), true))#193 AS sales#68, count(1)#194 AS number_sales#69, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#191 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#192 as decimal(12,2)))), DecimalType(18,2), true))#193 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#191 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#192 as decimal(12,2)))), DecimalType(18,2), true))#195]
+
+(154) Filter [codegen id : 327]
+Input [6]: [i_brand_id#185, i_class_id#186, i_category_id#187, sales#68, number_sales#69, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#191 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#192 as decimal(12,2)))), DecimalType(18,2), true))#195]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#191 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#192 as decimal(12,2)))), DecimalType(18,2), true))#195) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#191 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#192 as decimal(12,2)))), DecimalType(18,2), true))#195 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#71, [id=#72] as decimal(32,6))))
+
+(155) Project [codegen id : 327]
+Output [6]: [store AS channel#73, i_brand_id#185, i_class_id#186, i_category_id#187, sales#68, number_sales#69]
+Input [6]: [i_brand_id#185, i_class_id#186, i_category_id#187, sales#68, number_sales#69, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#191 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#192 as decimal(12,2)))), DecimalType(18,2), true))#195]
+
+(156) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_brand_id#196, i_class_id#197, i_category_id#198, sum#199, isEmpty#200, count#201]
+
+(157) HashAggregate [codegen id : 373]
+Input [6]: [i_brand_id#196, i_class_id#197, i_category_id#198, sum#199, isEmpty#200, count#201]
+Keys [3]: [i_brand_id#196, i_class_id#197, i_category_id#198]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#202 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#203 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#202 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#203 as decimal(12,2)))), DecimalType(18,2), true))#204, count(1)#205]
+Results [6]: [i_brand_id#196, i_class_id#197, i_category_id#198, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#202 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#203 as decimal(12,2)))), DecimalType(18,2), true))#204 AS sales#93, count(1)#205 AS number_sales#94, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#202 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#203 as decimal(12,2)))), DecimalType(18,2), true))#204 AS sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#202 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#203 as decimal(12,2)))), DecimalType(18,2), true))#206]
+
+(158) Filter [codegen id : 373]
+Input [6]: [i_brand_id#196, i_class_id#197, i_category_id#198, sales#93, number_sales#94, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#202 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#203 as decimal(12,2)))), DecimalType(18,2), true))#206]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#202 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#203 as decimal(12,2)))), DecimalType(18,2), true))#206) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#202 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#203 as decimal(12,2)))), DecimalType(18,2), true))#206 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#71, [id=#72] as decimal(32,6))))
+
+(159) Project [codegen id : 373]
+Output [6]: [catalog AS channel#96, i_brand_id#196, i_class_id#197, i_category_id#198, sales#93, number_sales#94]
+Input [6]: [i_brand_id#196, i_class_id#197, i_category_id#198, sales#93, number_sales#94, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#202 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#203 as decimal(12,2)))), DecimalType(18,2), true))#206]
+
+(160) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_brand_id#207, i_class_id#208, i_category_id#209, sum#210, isEmpty#211, count#212]
+
+(161) HashAggregate [codegen id : 419]
+Input [6]: [i_brand_id#207, i_class_id#208, i_category_id#209, sum#210, isEmpty#211, count#212]
+Keys [3]: [i_brand_id#207, i_class_id#208, i_category_id#209]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#213 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#214 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#213 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#214 as decimal(12,2)))), DecimalType(18,2), true))#215, count(1)#216]
+Results [6]: [i_brand_id#207, i_class_id#208, i_category_id#209, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#213 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#214 as decimal(12,2)))), DecimalType(18,2), true))#215 AS sales#116, count(1)#216 AS number_sales#117, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#213 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#214 as decimal(12,2)))), DecimalType(18,2), true))#215 AS sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#213 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#214 as decimal(12,2)))), DecimalType(18,2), true))#217]
+
+(162) Filter [codegen id : 419]
+Input [6]: [i_brand_id#207, i_class_id#208, i_category_id#209, sales#116, number_sales#117, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#213 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#214 as decimal(12,2)))), DecimalType(18,2), true))#217]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#213 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#214 as decimal(12,2)))), DecimalType(18,2), true))#217) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#213 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#214 as decimal(12,2)))), DecimalType(18,2), true))#217 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#71, [id=#72] as decimal(32,6))))
+
+(163) Project [codegen id : 419]
+Output [6]: [web AS channel#119, i_brand_id#207, i_class_id#208, i_category_id#209, sales#116, number_sales#117]
+Input [6]: [i_brand_id#207, i_class_id#208, i_category_id#209, sales#116, number_sales#117, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#213 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#214 as decimal(12,2)))), DecimalType(18,2), true))#217]
+
+(164) Union
+
+(165) HashAggregate [codegen id : 420]
+Input [6]: [channel#73, i_brand_id#185, i_class_id#186, i_category_id#187, sales#68, number_sales#69]
+Keys [4]: [channel#73, i_brand_id#185, i_class_id#186, i_category_id#187]
+Functions [2]: [partial_sum(sales#68), partial_sum(number_sales#69)]
+Aggregate Attributes [3]: [sum#218, isEmpty#219, sum#220]
+Results [7]: [channel#73, i_brand_id#185, i_class_id#186, i_category_id#187, sum#221, isEmpty#222, sum#223]
+
+(166) Exchange
+Input [7]: [channel#73, i_brand_id#185, i_class_id#186, i_category_id#187, sum#221, isEmpty#222, sum#223]
+Arguments: hashpartitioning(channel#73, i_brand_id#185, i_class_id#186, i_category_id#187, 5), ENSURE_REQUIREMENTS, [id=#224]
+
+(167) HashAggregate [codegen id : 421]
+Input [7]: [channel#73, i_brand_id#185, i_class_id#186, i_category_id#187, sum#221, isEmpty#222, sum#223]
+Keys [4]: [channel#73, i_brand_id#185, i_class_id#186, i_category_id#187]
+Functions [2]: [sum(sales#68), sum(number_sales#69)]
+Aggregate Attributes [2]: [sum(sales#68)#225, sum(number_sales#69)#226]
+Results [4]: [channel#73, i_brand_id#185, sum(sales#68)#225 AS sum_sales#129, sum(number_sales#69)#226 AS number_sales#130]
+
+(168) HashAggregate [codegen id : 421]
+Input [4]: [channel#73, i_brand_id#185, sum_sales#129, number_sales#130]
+Keys [2]: [channel#73, i_brand_id#185]
+Functions [2]: [partial_sum(sum_sales#129), partial_sum(number_sales#130)]
+Aggregate Attributes [3]: [sum#227, isEmpty#228, sum#229]
+Results [5]: [channel#73, i_brand_id#185, sum#230, isEmpty#231, sum#232]
+
+(169) Exchange
+Input [5]: [channel#73, i_brand_id#185, sum#230, isEmpty#231, sum#232]
+Arguments: hashpartitioning(channel#73, i_brand_id#185, 5), ENSURE_REQUIREMENTS, [id=#233]
+
+(170) HashAggregate [codegen id : 422]
+Input [5]: [channel#73, i_brand_id#185, sum#230, isEmpty#231, sum#232]
+Keys [2]: [channel#73, i_brand_id#185]
+Functions [2]: [sum(sum_sales#129), sum(number_sales#130)]
+Aggregate Attributes [2]: [sum(sum_sales#129)#234, sum(number_sales#130)#235]
+Results [6]: [channel#73, i_brand_id#185, null AS i_class_id#236, null AS i_category_id#237, sum(sum_sales#129)#234 AS sum(sum_sales)#238, sum(number_sales#130)#235 AS sum(number_sales)#239]
+
+(171) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_brand_id#240, i_class_id#241, i_category_id#242, sum#243, isEmpty#244, count#245]
+
+(172) HashAggregate [codegen id : 468]
+Input [6]: [i_brand_id#240, i_class_id#241, i_category_id#242, sum#243, isEmpty#244, count#245]
+Keys [3]: [i_brand_id#240, i_class_id#241, i_category_id#242]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#246 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#247 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#246 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#247 as decimal(12,2)))), DecimalType(18,2), true))#248, count(1)#249]
+Results [6]: [i_brand_id#240, i_class_id#241, i_category_id#242, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#246 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#247 as decimal(12,2)))), DecimalType(18,2), true))#248 AS sales#68, count(1)#249 AS number_sales#69, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#246 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#247 as decimal(12,2)))), DecimalType(18,2), true))#248 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#246 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#247 as decimal(12,2)))), DecimalType(18,2), true))#250]
+
+(173) Filter [codegen id : 468]
+Input [6]: [i_brand_id#240, i_class_id#241, i_category_id#242, sales#68, number_sales#69, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#246 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#247 as decimal(12,2)))), DecimalType(18,2), true))#250]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#246 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#247 as decimal(12,2)))), DecimalType(18,2), true))#250) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#246 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#247 as decimal(12,2)))), DecimalType(18,2), true))#250 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#71, [id=#72] as decimal(32,6))))
+
+(174) Project [codegen id : 468]
+Output [6]: [store AS channel#73, i_brand_id#240, i_class_id#241, i_category_id#242, sales#68, number_sales#69]
+Input [6]: [i_brand_id#240, i_class_id#241, i_category_id#242, sales#68, number_sales#69, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#246 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#247 as decimal(12,2)))), DecimalType(18,2), true))#250]
+
+(175) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_brand_id#251, i_class_id#252, i_category_id#253, sum#254, isEmpty#255, count#256]
+
+(176) HashAggregate [codegen id : 514]
+Input [6]: [i_brand_id#251, i_class_id#252, i_category_id#253, sum#254, isEmpty#255, count#256]
+Keys [3]: [i_brand_id#251, i_class_id#252, i_category_id#253]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#257 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#258 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#257 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#258 as decimal(12,2)))), DecimalType(18,2), true))#259, count(1)#260]
+Results [6]: [i_brand_id#251, i_class_id#252, i_category_id#253, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#257 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#258 as decimal(12,2)))), DecimalType(18,2), true))#259 AS sales#93, count(1)#260 AS number_sales#94, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#257 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#258 as decimal(12,2)))), DecimalType(18,2), true))#259 AS sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#257 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#258 as decimal(12,2)))), DecimalType(18,2), true))#261]
+
+(177) Filter [codegen id : 514]
+Input [6]: [i_brand_id#251, i_class_id#252, i_category_id#253, sales#93, number_sales#94, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#257 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#258 as decimal(12,2)))), DecimalType(18,2), true))#261]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#257 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#258 as decimal(12,2)))), DecimalType(18,2), true))#261) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#257 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#258 as decimal(12,2)))), DecimalType(18,2), true))#261 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#71, [id=#72] as decimal(32,6))))
+
+(178) Project [codegen id : 514]
+Output [6]: [catalog AS channel#96, i_brand_id#251, i_class_id#252, i_category_id#253, sales#93, number_sales#94]
+Input [6]: [i_brand_id#251, i_class_id#252, i_category_id#253, sales#93, number_sales#94, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#257 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#258 as decimal(12,2)))), DecimalType(18,2), true))#261]
+
+(179) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_brand_id#262, i_class_id#263, i_category_id#264, sum#265, isEmpty#266, count#267]
+
+(180) HashAggregate [codegen id : 560]
+Input [6]: [i_brand_id#262, i_class_id#263, i_category_id#264, sum#265, isEmpty#266, count#267]
+Keys [3]: [i_brand_id#262, i_class_id#263, i_category_id#264]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#268 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#269 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#268 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#269 as decimal(12,2)))), DecimalType(18,2), true))#270, count(1)#271]
+Results [6]: [i_brand_id#262, i_class_id#263, i_category_id#264, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#268 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#269 as decimal(12,2)))), DecimalType(18,2), true))#270 AS sales#116, count(1)#271 AS number_sales#117, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#268 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#269 as decimal(12,2)))), DecimalType(18,2), true))#270 AS sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#268 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#269 as decimal(12,2)))), DecimalType(18,2), true))#272]
+
+(181) Filter [codegen id : 560]
+Input [6]: [i_brand_id#262, i_class_id#263, i_category_id#264, sales#116, number_sales#117, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#268 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#269 as decimal(12,2)))), DecimalType(18,2), true))#272]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#268 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#269 as decimal(12,2)))), DecimalType(18,2), true))#272) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#268 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#269 as decimal(12,2)))), DecimalType(18,2), true))#272 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#71, [id=#72] as decimal(32,6))))
+
+(182) Project [codegen id : 560]
+Output [6]: [web AS channel#119, i_brand_id#262, i_class_id#263, i_category_id#264, sales#116, number_sales#117]
+Input [6]: [i_brand_id#262, i_class_id#263, i_category_id#264, sales#116, number_sales#117, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#268 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#269 as decimal(12,2)))), DecimalType(18,2), true))#272]
+
+(183) Union
+
+(184) HashAggregate [codegen id : 561]
+Input [6]: [channel#73, i_brand_id#240, i_class_id#241, i_category_id#242, sales#68, number_sales#69]
+Keys [4]: [channel#73, i_brand_id#240, i_class_id#241, i_category_id#242]
+Functions [2]: [partial_sum(sales#68), partial_sum(number_sales#69)]
+Aggregate Attributes [3]: [sum#273, isEmpty#274, sum#275]
+Results [7]: [channel#73, i_brand_id#240, i_class_id#241, i_category_id#242, sum#276, isEmpty#277, sum#278]
+
+(185) Exchange
+Input [7]: [channel#73, i_brand_id#240, i_class_id#241, i_category_id#242, sum#276, isEmpty#277, sum#278]
+Arguments: hashpartitioning(channel#73, i_brand_id#240, i_class_id#241, i_category_id#242, 5), ENSURE_REQUIREMENTS, [id=#279]
+
+(186) HashAggregate [codegen id : 562]
+Input [7]: [channel#73, i_brand_id#240, i_class_id#241, i_category_id#242, sum#276, isEmpty#277, sum#278]
+Keys [4]: [channel#73, i_brand_id#240, i_class_id#241, i_category_id#242]
+Functions [2]: [sum(sales#68), sum(number_sales#69)]
+Aggregate Attributes [2]: [sum(sales#68)#280, sum(number_sales#69)#281]
+Results [3]: [channel#73, sum(sales#68)#280 AS sum_sales#129, sum(number_sales#69)#281 AS number_sales#130]
+
+(187) HashAggregate [codegen id : 562]
+Input [3]: [channel#73, sum_sales#129, number_sales#130]
+Keys [1]: [channel#73]
+Functions [2]: [partial_sum(sum_sales#129), partial_sum(number_sales#130)]
+Aggregate Attributes [3]: [sum#282, isEmpty#283, sum#284]
+Results [4]: [channel#73, sum#285, isEmpty#286, sum#287]
+
+(188) Exchange
+Input [4]: [channel#73, sum#285, isEmpty#286, sum#287]
+Arguments: hashpartitioning(channel#73, 5), ENSURE_REQUIREMENTS, [id=#288]
+
+(189) HashAggregate [codegen id : 563]
+Input [4]: [channel#73, sum#285, isEmpty#286, sum#287]
+Keys [1]: [channel#73]
+Functions [2]: [sum(sum_sales#129), sum(number_sales#130)]
+Aggregate Attributes [2]: [sum(sum_sales#129)#289, sum(number_sales#130)#290]
+Results [6]: [channel#73, null AS i_brand_id#291, null AS i_class_id#292, null AS i_category_id#293, sum(sum_sales#129)#289 AS sum(sum_sales)#294, sum(number_sales#130)#290 AS sum(number_sales)#295]
+
+(190) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_brand_id#296, i_class_id#297, i_category_id#298, sum#299, isEmpty#300, count#301]
+
+(191) HashAggregate [codegen id : 609]
+Input [6]: [i_brand_id#296, i_class_id#297, i_category_id#298, sum#299, isEmpty#300, count#301]
+Keys [3]: [i_brand_id#296, i_class_id#297, i_category_id#298]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#302 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#303 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#302 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#303 as decimal(12,2)))), DecimalType(18,2), true))#304, count(1)#305]
+Results [6]: [i_brand_id#296, i_class_id#297, i_category_id#298, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#302 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#303 as decimal(12,2)))), DecimalType(18,2), true))#304 AS sales#68, count(1)#305 AS number_sales#69, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#302 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#303 as decimal(12,2)))), DecimalType(18,2), true))#304 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#302 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#303 as decimal(12,2)))), DecimalType(18,2), true))#306]
+
+(192) Filter [codegen id : 609]
+Input [6]: [i_brand_id#296, i_class_id#297, i_category_id#298, sales#68, number_sales#69, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#302 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#303 as decimal(12,2)))), DecimalType(18,2), true))#306]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#302 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#303 as decimal(12,2)))), DecimalType(18,2), true))#306) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#302 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#303 as decimal(12,2)))), DecimalType(18,2), true))#306 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#71, [id=#72] as decimal(32,6))))
+
+(193) Project [codegen id : 609]
+Output [6]: [store AS channel#73, i_brand_id#296, i_class_id#297, i_category_id#298, sales#68, number_sales#69]
+Input [6]: [i_brand_id#296, i_class_id#297, i_category_id#298, sales#68, number_sales#69, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#302 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#303 as decimal(12,2)))), DecimalType(18,2), true))#306]
+
+(194) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_brand_id#307, i_class_id#308, i_category_id#309, sum#310, isEmpty#311, count#312]
+
+(195) HashAggregate [codegen id : 655]
+Input [6]: [i_brand_id#307, i_class_id#308, i_category_id#309, sum#310, isEmpty#311, count#312]
+Keys [3]: [i_brand_id#307, i_class_id#308, i_category_id#309]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#313 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#314 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#313 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#314 as decimal(12,2)))), DecimalType(18,2), true))#315, count(1)#316]
+Results [6]: [i_brand_id#307, i_class_id#308, i_category_id#309, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#313 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#314 as decimal(12,2)))), DecimalType(18,2), true))#315 AS sales#93, count(1)#316 AS number_sales#94, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#313 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#314 as decimal(12,2)))), DecimalType(18,2), true))#315 AS sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#313 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#314 as decimal(12,2)))), DecimalType(18,2), true))#317]
+
+(196) Filter [codegen id : 655]
+Input [6]: [i_brand_id#307, i_class_id#308, i_category_id#309, sales#93, number_sales#94, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#313 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#314 as decimal(12,2)))), DecimalType(18,2), true))#317]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#313 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#314 as decimal(12,2)))), DecimalType(18,2), true))#317) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#313 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#314 as decimal(12,2)))), DecimalType(18,2), true))#317 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#71, [id=#72] as decimal(32,6))))
+
+(197) Project [codegen id : 655]
+Output [6]: [catalog AS channel#96, i_brand_id#307, i_class_id#308, i_category_id#309, sales#93, number_sales#94]
+Input [6]: [i_brand_id#307, i_class_id#308, i_category_id#309, sales#93, number_sales#94, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#313 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#314 as decimal(12,2)))), DecimalType(18,2), true))#317]
+
+(198) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_brand_id#318, i_class_id#319, i_category_id#320, sum#321, isEmpty#322, count#323]
+
+(199) HashAggregate [codegen id : 701]
+Input [6]: [i_brand_id#318, i_class_id#319, i_category_id#320, sum#321, isEmpty#322, count#323]
+Keys [3]: [i_brand_id#318, i_class_id#319, i_category_id#320]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#324 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#325 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#324 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#325 as decimal(12,2)))), DecimalType(18,2), true))#326, count(1)#327]
+Results [6]: [i_brand_id#318, i_class_id#319, i_category_id#320, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#324 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#325 as decimal(12,2)))), DecimalType(18,2), true))#326 AS sales#116, count(1)#327 AS number_sales#117, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#324 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#325 as decimal(12,2)))), DecimalType(18,2), true))#326 AS sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#324 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#325 as decimal(12,2)))), DecimalType(18,2), true))#328]
+
+(200) Filter [codegen id : 701]
+Input [6]: [i_brand_id#318, i_class_id#319, i_category_id#320, sales#116, number_sales#117, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#324 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#325 as decimal(12,2)))), DecimalType(18,2), true))#328]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#324 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#325 as decimal(12,2)))), DecimalType(18,2), true))#328) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#324 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#325 as decimal(12,2)))), DecimalType(18,2), true))#328 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#71, [id=#72] as decimal(32,6))))
+
+(201) Project [codegen id : 701]
+Output [6]: [web AS channel#119, i_brand_id#318, i_class_id#319, i_category_id#320, sales#116, number_sales#117]
+Input [6]: [i_brand_id#318, i_class_id#319, i_category_id#320, sales#116, number_sales#117, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#324 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#325 as decimal(12,2)))), DecimalType(18,2), true))#328]
+
+(202) Union
+
+(203) HashAggregate [codegen id : 702]
+Input [6]: [channel#73, i_brand_id#296, i_class_id#297, i_category_id#298, sales#68, number_sales#69]
+Keys [4]: [channel#73, i_brand_id#296, i_class_id#297, i_category_id#298]
+Functions [2]: [partial_sum(sales#68), partial_sum(number_sales#69)]
+Aggregate Attributes [3]: [sum#329, isEmpty#330, sum#331]
+Results [7]: [channel#73, i_brand_id#296, i_class_id#297, i_category_id#298, sum#332, isEmpty#333, sum#334]
+
+(204) Exchange
+Input [7]: [channel#73, i_brand_id#296, i_class_id#297, i_category_id#298, sum#332, isEmpty#333, sum#334]
+Arguments: hashpartitioning(channel#73, i_brand_id#296, i_class_id#297, i_category_id#298, 5), ENSURE_REQUIREMENTS, [id=#335]
+
+(205) HashAggregate [codegen id : 703]
+Input [7]: [channel#73, i_brand_id#296, i_class_id#297, i_category_id#298, sum#332, isEmpty#333, sum#334]
+Keys [4]: [channel#73, i_brand_id#296, i_class_id#297, i_category_id#298]
+Functions [2]: [sum(sales#68), sum(number_sales#69)]
+Aggregate Attributes [2]: [sum(sales#68)#336, sum(number_sales#69)#337]
+Results [2]: [sum(sales#68)#336 AS sum_sales#129, sum(number_sales#69)#337 AS number_sales#130]
+
+(206) HashAggregate [codegen id : 703]
+Input [2]: [sum_sales#129, number_sales#130]
+Keys: []
+Functions [2]: [partial_sum(sum_sales#129), partial_sum(number_sales#130)]
+Aggregate Attributes [3]: [sum#338, isEmpty#339, sum#340]
+Results [3]: [sum#341, isEmpty#342, sum#343]
+
+(207) Exchange
+Input [3]: [sum#341, isEmpty#342, sum#343]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#344]
+
+(208) HashAggregate [codegen id : 704]
+Input [3]: [sum#341, isEmpty#342, sum#343]
+Keys: []
+Functions [2]: [sum(sum_sales#129), sum(number_sales#130)]
+Aggregate Attributes [2]: [sum(sum_sales#129)#345, sum(number_sales#130)#346]
+Results [6]: [null AS channel#347, null AS i_brand_id#348, null AS i_class_id#349, null AS i_category_id#350, sum(sum_sales#129)#345 AS sum(sum_sales)#351, sum(number_sales#130)#346 AS sum(number_sales)#352]
+
+(209) Union
+
+(210) HashAggregate [codegen id : 705]
+Input [6]: [channel#73, i_brand_id#54, i_class_id#55, i_category_id#56, sum_sales#129, number_sales#130]
+Keys [6]: [channel#73, i_brand_id#54, i_class_id#55, i_category_id#56, sum_sales#129, number_sales#130]
+Functions: []
+Aggregate Attributes: []
+Results [6]: [channel#73, i_brand_id#54, i_class_id#55, i_category_id#56, sum_sales#129, number_sales#130]
+
+(211) Exchange
+Input [6]: [channel#73, i_brand_id#54, i_class_id#55, i_category_id#56, sum_sales#129, number_sales#130]
+Arguments: hashpartitioning(channel#73, i_brand_id#54, i_class_id#55, i_category_id#56, sum_sales#129, number_sales#130, 5), ENSURE_REQUIREMENTS, [id=#353]
+
+(212) HashAggregate [codegen id : 706]
+Input [6]: [channel#73, i_brand_id#54, i_class_id#55, i_category_id#56, sum_sales#129, number_sales#130]
+Keys [6]: [channel#73, i_brand_id#54, i_class_id#55, i_category_id#56, sum_sales#129, number_sales#130]
+Functions: []
+Aggregate Attributes: []
+Results [6]: [channel#73, i_brand_id#54, i_class_id#55, i_category_id#56, sum_sales#129, number_sales#130]
+
+(213) TakeOrderedAndProject
+Input [6]: [channel#73, i_brand_id#54, i_class_id#55, i_category_id#56, sum_sales#129, number_sales#130]
+Arguments: 100, [channel#73 ASC NULLS FIRST, i_brand_id#54 ASC NULLS FIRST, i_class_id#55 ASC NULLS FIRST, i_category_id#56 ASC NULLS FIRST], [channel#73, i_brand_id#54, i_class_id#55, i_category_id#56, sum_sales#129, number_sales#130]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 89 Hosting Expression = Subquery scalar-subquery#71, [id=#72]
+* HashAggregate (240)
++- Exchange (239)
+   +- * HashAggregate (238)
+      +- Union (237)
+         :- * Project (222)
+         :  +- * BroadcastHashJoin Inner BuildRight (221)
+         :     :- * ColumnarToRow (215)
+         :     :  +- Scan parquet default.store_sales (214)
+         :     +- BroadcastExchange (220)
+         :        +- * Project (219)
+         :           +- * Filter (218)
+         :              +- * ColumnarToRow (217)
+         :                 +- Scan parquet default.date_dim (216)
+         :- * Project (231)
+         :  +- * BroadcastHashJoin Inner BuildRight (230)
+         :     :- * ColumnarToRow (224)
+         :     :  +- Scan parquet default.catalog_sales (223)
+         :     +- BroadcastExchange (229)
+         :        +- * Project (228)
+         :           +- * Filter (227)
+         :              +- * ColumnarToRow (226)
+         :                 +- Scan parquet default.date_dim (225)
+         +- * Project (236)
+            +- * BroadcastHashJoin Inner BuildRight (235)
+               :- * ColumnarToRow (233)
+               :  +- Scan parquet default.web_sales (232)
+               +- ReusedExchange (234)
+
+
+(214) Scan parquet default.store_sales
+Output [3]: [ss_quantity#354, ss_list_price#355, ss_sold_date_sk#356]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#356), dynamicpruningexpression(ss_sold_date_sk#356 IN dynamicpruning#357)]
+ReadSchema: struct<ss_quantity:int,ss_list_price:decimal(7,2)>
+
+(215) ColumnarToRow [codegen id : 2]
+Input [3]: [ss_quantity#354, ss_list_price#355, ss_sold_date_sk#356]
+
+(216) Scan parquet default.date_dim
+Output [2]: [d_date_sk#358, d_year#359]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(217) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#358, d_year#359]
+
+(218) Filter [codegen id : 1]
+Input [2]: [d_date_sk#358, d_year#359]
+Condition : (((isnotnull(d_year#359) AND (d_year#359 >= 1999)) AND (d_year#359 <= 2001)) AND isnotnull(d_date_sk#358))
+
+(219) Project [codegen id : 1]
+Output [1]: [d_date_sk#358]
+Input [2]: [d_date_sk#358, d_year#359]
+
+(220) BroadcastExchange
+Input [1]: [d_date_sk#358]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#360]
+
+(221) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#356]
+Right keys [1]: [d_date_sk#358]
+Join condition: None
+
+(222) Project [codegen id : 2]
+Output [2]: [ss_quantity#354 AS quantity#361, ss_list_price#355 AS list_price#362]
+Input [4]: [ss_quantity#354, ss_list_price#355, ss_sold_date_sk#356, d_date_sk#358]
+
+(223) Scan parquet default.catalog_sales
+Output [3]: [cs_quantity#363, cs_list_price#364, cs_sold_date_sk#365]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#365), dynamicpruningexpression(cs_sold_date_sk#365 IN dynamicpruning#366)]
+ReadSchema: struct<cs_quantity:int,cs_list_price:decimal(7,2)>
+
+(224) ColumnarToRow [codegen id : 4]
+Input [3]: [cs_quantity#363, cs_list_price#364, cs_sold_date_sk#365]
+
+(225) Scan parquet default.date_dim
+Output [2]: [d_date_sk#367, d_year#368]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(226) ColumnarToRow [codegen id : 3]
+Input [2]: [d_date_sk#367, d_year#368]
+
+(227) Filter [codegen id : 3]
+Input [2]: [d_date_sk#367, d_year#368]
+Condition : (((isnotnull(d_year#368) AND (d_year#368 >= 1998)) AND (d_year#368 <= 2000)) AND isnotnull(d_date_sk#367))
+
+(228) Project [codegen id : 3]
+Output [1]: [d_date_sk#367]
+Input [2]: [d_date_sk#367, d_year#368]
+
+(229) BroadcastExchange
+Input [1]: [d_date_sk#367]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#369]
+
+(230) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_sold_date_sk#365]
+Right keys [1]: [d_date_sk#367]
+Join condition: None
+
+(231) Project [codegen id : 4]
+Output [2]: [cs_quantity#363 AS quantity#370, cs_list_price#364 AS list_price#371]
+Input [4]: [cs_quantity#363, cs_list_price#364, cs_sold_date_sk#365, d_date_sk#367]
+
+(232) Scan parquet default.web_sales
+Output [3]: [ws_quantity#372, ws_list_price#373, ws_sold_date_sk#374]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#374), dynamicpruningexpression(ws_sold_date_sk#374 IN dynamicpruning#366)]
+ReadSchema: struct<ws_quantity:int,ws_list_price:decimal(7,2)>
+
+(233) ColumnarToRow [codegen id : 6]
+Input [3]: [ws_quantity#372, ws_list_price#373, ws_sold_date_sk#374]
+
+(234) ReusedExchange [Reuses operator id: 229]
+Output [1]: [d_date_sk#375]
+
+(235) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ws_sold_date_sk#374]
+Right keys [1]: [d_date_sk#375]
+Join condition: None
+
+(236) Project [codegen id : 6]
+Output [2]: [ws_quantity#372 AS quantity#376, ws_list_price#373 AS list_price#377]
+Input [4]: [ws_quantity#372, ws_list_price#373, ws_sold_date_sk#374, d_date_sk#375]
+
+(237) Union
+
+(238) HashAggregate [codegen id : 7]
+Input [2]: [quantity#361, list_price#362]
+Keys: []
+Functions [1]: [partial_avg(CheckOverflow((promote_precision(cast(cast(quantity#361 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#362 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [2]: [sum#378, count#379]
+Results [2]: [sum#380, count#381]
+
+(239) Exchange
+Input [2]: [sum#380, count#381]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#382]
+
+(240) HashAggregate [codegen id : 8]
+Input [2]: [sum#380, count#381]
+Keys: []
+Functions [1]: [avg(CheckOverflow((promote_precision(cast(cast(quantity#361 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#362 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [avg(CheckOverflow((promote_precision(cast(cast(quantity#361 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#362 as decimal(12,2)))), DecimalType(18,2), true))#383]
+Results [1]: [avg(CheckOverflow((promote_precision(cast(cast(quantity#361 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#362 as decimal(12,2)))), DecimalType(18,2), true))#383 AS average_sales#384]
+
+Subquery:2 Hosting operator id = 214 Hosting Expression = ss_sold_date_sk#356 IN dynamicpruning#357
+ReusedExchange (241)
+
+
+(241) ReusedExchange [Reuses operator id: 220]
+Output [1]: [d_date_sk#358]
+
+Subquery:3 Hosting operator id = 223 Hosting Expression = cs_sold_date_sk#365 IN dynamicpruning#366
+ReusedExchange (242)
+
+
+(242) ReusedExchange [Reuses operator id: 229]
+Output [1]: [d_date_sk#367]
+
+Subquery:4 Hosting operator id = 232 Hosting Expression = ws_sold_date_sk#374 IN dynamicpruning#366
+
+Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (243)
+
+
+(243) ReusedExchange [Reuses operator id: 72]
+Output [1]: [d_date_sk#49]
+
+Subquery:6 Hosting operator id = 9 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13
+ReusedExchange (244)
+
+
+(244) ReusedExchange [Reuses operator id: 16]
+Output [1]: [d_date_sk#14]
+
+Subquery:7 Hosting operator id = 24 Hosting Expression = cs_sold_date_sk#23 IN dynamicpruning#13
+
+Subquery:8 Hosting operator id = 47 Hosting Expression = ws_sold_date_sk#38 IN dynamicpruning#13
+
+Subquery:9 Hosting operator id = 108 Hosting Expression = ReusedSubquery Subquery scalar-subquery#71, [id=#72]
+
+Subquery:10 Hosting operator id = 91 Hosting Expression = cs_sold_date_sk#77 IN dynamicpruning#5
+
+Subquery:11 Hosting operator id = 127 Hosting Expression = ReusedSubquery Subquery scalar-subquery#71, [id=#72]
+
+Subquery:12 Hosting operator id = 110 Hosting Expression = ws_sold_date_sk#100 IN dynamicpruning#5
+
+Subquery:13 Hosting operator id = 135 Hosting Expression = ReusedSubquery Subquery scalar-subquery#71, [id=#72]
+
+Subquery:14 Hosting operator id = 139 Hosting Expression = ReusedSubquery Subquery scalar-subquery#71, [id=#72]
+
+Subquery:15 Hosting operator id = 143 Hosting Expression = ReusedSubquery Subquery scalar-subquery#71, [id=#72]
+
+Subquery:16 Hosting operator id = 154 Hosting Expression = ReusedSubquery Subquery scalar-subquery#71, [id=#72]
+
+Subquery:17 Hosting operator id = 158 Hosting Expression = ReusedSubquery Subquery scalar-subquery#71, [id=#72]
+
+Subquery:18 Hosting operator id = 162 Hosting Expression = ReusedSubquery Subquery scalar-subquery#71, [id=#72]
+
+Subquery:19 Hosting operator id = 173 Hosting Expression = ReusedSubquery Subquery scalar-subquery#71, [id=#72]
+
+Subquery:20 Hosting operator id = 177 Hosting Expression = ReusedSubquery Subquery scalar-subquery#71, [id=#72]
+
+Subquery:21 Hosting operator id = 181 Hosting Expression = ReusedSubquery Subquery scalar-subquery#71, [id=#72]
+
+Subquery:22 Hosting operator id = 192 Hosting Expression = ReusedSubquery Subquery scalar-subquery#71, [id=#72]
+
+Subquery:23 Hosting operator id = 196 Hosting Expression = ReusedSubquery Subquery scalar-subquery#71, [id=#72]
+
+Subquery:24 Hosting operator id = 200 Hosting Expression = ReusedSubquery Subquery scalar-subquery#71, [id=#72]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.sf100/simplified.txt
new file mode 100644
index 0000000000000..0a9fa7cbf3498
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.sf100/simplified.txt
@@ -0,0 +1,430 @@
+TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales]
+  WholeStageCodegen (706)
+    HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales]
+      InputAdapter
+        Exchange [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] #1
+          WholeStageCodegen (705)
+            HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales]
+              InputAdapter
+                Union
+                  WholeStageCodegen (140)
+                    HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum]
+                      InputAdapter
+                        Exchange [channel,i_brand_id,i_class_id,i_category_id] #2
+                          WholeStageCodegen (139)
+                            HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum]
+                              InputAdapter
+                                Union
+                                  WholeStageCodegen (46)
+                                    Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                      Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                        Subquery #3
+                                          WholeStageCodegen (8)
+                                            HashAggregate [sum,count] [avg(CheckOverflow((promote_precision(cast(cast(quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price as decimal(12,2)))), DecimalType(18,2), true)),average_sales,sum,count]
+                                              InputAdapter
+                                                Exchange #20
+                                                  WholeStageCodegen (7)
+                                                    HashAggregate [quantity,list_price] [sum,count,sum,count]
+                                                      InputAdapter
+                                                        Union
+                                                          WholeStageCodegen (2)
+                                                            Project [ss_quantity,ss_list_price]
+                                                              BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk]
+                                                                      SubqueryBroadcast [d_date_sk] #4
+                                                                        ReusedExchange [d_date_sk] #21
+                                                                InputAdapter
+                                                                  BroadcastExchange #21
+                                                                    WholeStageCodegen (1)
+                                                                      Project [d_date_sk]
+                                                                        Filter [d_year,d_date_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.date_dim [d_date_sk,d_year]
+                                                          WholeStageCodegen (4)
+                                                            Project [cs_quantity,cs_list_price]
+                                                              BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk]
+                                                                      SubqueryBroadcast [d_date_sk] #5
+                                                                        ReusedExchange [d_date_sk] #22
+                                                                InputAdapter
+                                                                  BroadcastExchange #22
+                                                                    WholeStageCodegen (3)
+                                                                      Project [d_date_sk]
+                                                                        Filter [d_year,d_date_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.date_dim [d_date_sk,d_year]
+                                                          WholeStageCodegen (6)
+                                                            Project [ws_quantity,ws_list_price]
+                                                              BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk]
+                                                                      ReusedSubquery [d_date_sk] #5
+                                                                InputAdapter
+                                                                  ReusedExchange [d_date_sk] #22
+                                        HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                          InputAdapter
+                                            Exchange [i_brand_id,i_class_id,i_category_id] #3
+                                              WholeStageCodegen (45)
+                                                HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count]
+                                                  Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id]
+                                                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                      Project [ss_item_sk,ss_quantity,ss_list_price]
+                                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                          SortMergeJoin [ss_item_sk,ss_item_sk]
+                                                            InputAdapter
+                                                              WholeStageCodegen (2)
+                                                                Sort [ss_item_sk]
+                                                                  InputAdapter
+                                                                    Exchange [ss_item_sk] #4
+                                                                      WholeStageCodegen (1)
+                                                                        Filter [ss_item_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk]
+                                                                                SubqueryBroadcast [d_date_sk] #1
+                                                                                  ReusedExchange [d_date_sk] #5
+                                                            InputAdapter
+                                                              WholeStageCodegen (21)
+                                                                Sort [ss_item_sk]
+                                                                  InputAdapter
+                                                                    Exchange [ss_item_sk] #6
+                                                                      WholeStageCodegen (20)
+                                                                        Project [i_item_sk]
+                                                                          BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id]
+                                                                            Filter [i_brand_id,i_class_id,i_category_id]
+                                                                              ColumnarToRow
+                                                                                InputAdapter
+                                                                                  Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                            InputAdapter
+                                                                              BroadcastExchange #7
+                                                                                WholeStageCodegen (19)
+                                                                                  HashAggregate [brand_id,class_id,category_id]
+                                                                                    InputAdapter
+                                                                                      Exchange [brand_id,class_id,category_id] #8
+                                                                                        WholeStageCodegen (18)
+                                                                                          HashAggregate [brand_id,class_id,category_id]
+                                                                                            SortMergeJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id]
+                                                                                              InputAdapter
+                                                                                                WholeStageCodegen (13)
+                                                                                                  Sort [brand_id,class_id,category_id]
+                                                                                                    InputAdapter
+                                                                                                      Exchange [brand_id,class_id,category_id] #9
+                                                                                                        WholeStageCodegen (12)
+                                                                                                          HashAggregate [brand_id,class_id,category_id]
+                                                                                                            InputAdapter
+                                                                                                              Exchange [brand_id,class_id,category_id] #10
+                                                                                                                WholeStageCodegen (11)
+                                                                                                                  HashAggregate [brand_id,class_id,category_id]
+                                                                                                                    Project [i_brand_id,i_class_id,i_category_id]
+                                                                                                                      BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                                                                                        Project [ss_item_sk]
+                                                                                                                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                                                            Filter [ss_item_sk]
+                                                                                                                              ColumnarToRow
+                                                                                                                                InputAdapter
+                                                                                                                                  Scan parquet default.store_sales [ss_item_sk,ss_sold_date_sk]
+                                                                                                                                    SubqueryBroadcast [d_date_sk] #2
+                                                                                                                                      ReusedExchange [d_date_sk] #11
+                                                                                                                            InputAdapter
+                                                                                                                              BroadcastExchange #11
+                                                                                                                                WholeStageCodegen (3)
+                                                                                                                                  Project [d_date_sk]
+                                                                                                                                    Filter [d_year,d_date_sk]
+                                                                                                                                      ColumnarToRow
+                                                                                                                                        InputAdapter
+                                                                                                                                          Scan parquet default.date_dim [d_date_sk,d_year]
+                                                                                                                        InputAdapter
+                                                                                                                          BroadcastExchange #12
+                                                                                                                            WholeStageCodegen (10)
+                                                                                                                              SortMergeJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id]
+                                                                                                                                InputAdapter
+                                                                                                                                  WholeStageCodegen (5)
+                                                                                                                                    Sort [i_brand_id,i_class_id,i_category_id]
+                                                                                                                                      InputAdapter
+                                                                                                                                        Exchange [i_brand_id,i_class_id,i_category_id] #13
+                                                                                                                                          WholeStageCodegen (4)
+                                                                                                                                            Filter [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                                                                              ColumnarToRow
+                                                                                                                                                InputAdapter
+                                                                                                                                                  Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                                                                InputAdapter
+                                                                                                                                  WholeStageCodegen (9)
+                                                                                                                                    Sort [i_brand_id,i_class_id,i_category_id]
+                                                                                                                                      InputAdapter
+                                                                                                                                        Exchange [i_brand_id,i_class_id,i_category_id] #14
+                                                                                                                                          WholeStageCodegen (8)
+                                                                                                                                            Project [i_brand_id,i_class_id,i_category_id]
+                                                                                                                                              BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                                                                                                                                Project [cs_item_sk]
+                                                                                                                                                  BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                                                                                                    Filter [cs_item_sk]
+                                                                                                                                                      ColumnarToRow
+                                                                                                                                                        InputAdapter
+                                                                                                                                                          Scan parquet default.catalog_sales [cs_item_sk,cs_sold_date_sk]
+                                                                                                                                                            ReusedSubquery [d_date_sk] #2
+                                                                                                                                                    InputAdapter
+                                                                                                                                                      ReusedExchange [d_date_sk] #11
+                                                                                                                                                InputAdapter
+                                                                                                                                                  BroadcastExchange #15
+                                                                                                                                                    WholeStageCodegen (7)
+                                                                                                                                                      Filter [i_item_sk]
+                                                                                                                                                        ColumnarToRow
+                                                                                                                                                          InputAdapter
+                                                                                                                                                            Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                              InputAdapter
+                                                                                                WholeStageCodegen (17)
+                                                                                                  Sort [i_brand_id,i_class_id,i_category_id]
+                                                                                                    InputAdapter
+                                                                                                      Exchange [i_brand_id,i_class_id,i_category_id] #16
+                                                                                                        WholeStageCodegen (16)
+                                                                                                          Project [i_brand_id,i_class_id,i_category_id]
+                                                                                                            BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                                                                                              Project [ws_item_sk]
+                                                                                                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                                                                  Filter [ws_item_sk]
+                                                                                                                    ColumnarToRow
+                                                                                                                      InputAdapter
+                                                                                                                        Scan parquet default.web_sales [ws_item_sk,ws_sold_date_sk]
+                                                                                                                          ReusedSubquery [d_date_sk] #2
+                                                                                                                  InputAdapter
+                                                                                                                    ReusedExchange [d_date_sk] #11
+                                                                                                              InputAdapter
+                                                                                                                ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #15
+                                                          InputAdapter
+                                                            BroadcastExchange #5
+                                                              WholeStageCodegen (22)
+                                                                Project [d_date_sk]
+                                                                  Filter [d_year,d_moy,d_date_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                                      InputAdapter
+                                                        BroadcastExchange #17
+                                                          WholeStageCodegen (44)
+                                                            SortMergeJoin [i_item_sk,ss_item_sk]
+                                                              InputAdapter
+                                                                WholeStageCodegen (24)
+                                                                  Sort [i_item_sk]
+                                                                    InputAdapter
+                                                                      Exchange [i_item_sk] #18
+                                                                        WholeStageCodegen (23)
+                                                                          Filter [i_item_sk]
+                                                                            ColumnarToRow
+                                                                              InputAdapter
+                                                                                Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                              InputAdapter
+                                                                WholeStageCodegen (43)
+                                                                  Sort [ss_item_sk]
+                                                                    InputAdapter
+                                                                      ReusedExchange [ss_item_sk] #19
+                                  WholeStageCodegen (92)
+                                    Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                      Filter [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                        ReusedSubquery [average_sales] #3
+                                        HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                          InputAdapter
+                                            Exchange [i_brand_id,i_class_id,i_category_id] #23
+                                              WholeStageCodegen (91)
+                                                HashAggregate [i_brand_id,i_class_id,i_category_id,cs_quantity,cs_list_price] [sum,isEmpty,count,sum,isEmpty,count]
+                                                  Project [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id]
+                                                    BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                                      Project [cs_item_sk,cs_quantity,cs_list_price]
+                                                        BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                          SortMergeJoin [cs_item_sk,ss_item_sk]
+                                                            InputAdapter
+                                                              WholeStageCodegen (48)
+                                                                Sort [cs_item_sk]
+                                                                  InputAdapter
+                                                                    Exchange [cs_item_sk] #24
+                                                                      WholeStageCodegen (47)
+                                                                        Filter [cs_item_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk]
+                                                                                ReusedSubquery [d_date_sk] #1
+                                                            InputAdapter
+                                                              WholeStageCodegen (67)
+                                                                Sort [ss_item_sk]
+                                                                  InputAdapter
+                                                                    ReusedExchange [ss_item_sk] #19
+                                                          InputAdapter
+                                                            ReusedExchange [d_date_sk] #5
+                                                      InputAdapter
+                                                        ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #17
+                                  WholeStageCodegen (138)
+                                    Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                      Filter [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                        ReusedSubquery [average_sales] #3
+                                        HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                          InputAdapter
+                                            Exchange [i_brand_id,i_class_id,i_category_id] #25
+                                              WholeStageCodegen (137)
+                                                HashAggregate [i_brand_id,i_class_id,i_category_id,ws_quantity,ws_list_price] [sum,isEmpty,count,sum,isEmpty,count]
+                                                  Project [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id]
+                                                    BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                                      Project [ws_item_sk,ws_quantity,ws_list_price]
+                                                        BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                          SortMergeJoin [ws_item_sk,ss_item_sk]
+                                                            InputAdapter
+                                                              WholeStageCodegen (94)
+                                                                Sort [ws_item_sk]
+                                                                  InputAdapter
+                                                                    Exchange [ws_item_sk] #26
+                                                                      WholeStageCodegen (93)
+                                                                        Filter [ws_item_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk]
+                                                                                ReusedSubquery [d_date_sk] #1
+                                                            InputAdapter
+                                                              WholeStageCodegen (113)
+                                                                Sort [ss_item_sk]
+                                                                  InputAdapter
+                                                                    ReusedExchange [ss_item_sk] #19
+                                                          InputAdapter
+                                                            ReusedExchange [d_date_sk] #5
+                                                      InputAdapter
+                                                        ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #17
+                  WholeStageCodegen (281)
+                    HashAggregate [channel,i_brand_id,i_class_id,sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum]
+                      InputAdapter
+                        Exchange [channel,i_brand_id,i_class_id] #27
+                          WholeStageCodegen (280)
+                            HashAggregate [channel,i_brand_id,i_class_id,sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum]
+                              HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum]
+                                InputAdapter
+                                  Exchange [channel,i_brand_id,i_class_id,i_category_id] #28
+                                    WholeStageCodegen (279)
+                                      HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum]
+                                        InputAdapter
+                                          Union
+                                            WholeStageCodegen (186)
+                                              Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                                Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                                  ReusedSubquery [average_sales] #3
+                                                  HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                                    InputAdapter
+                                                      ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #29
+                                            WholeStageCodegen (232)
+                                              Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                                Filter [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                                  ReusedSubquery [average_sales] #3
+                                                  HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                                    InputAdapter
+                                                      ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #30
+                                            WholeStageCodegen (278)
+                                              Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                                Filter [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                                  ReusedSubquery [average_sales] #3
+                                                  HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                                    InputAdapter
+                                                      ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #31
+                  WholeStageCodegen (422)
+                    HashAggregate [channel,i_brand_id,sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum]
+                      InputAdapter
+                        Exchange [channel,i_brand_id] #32
+                          WholeStageCodegen (421)
+                            HashAggregate [channel,i_brand_id,sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum]
+                              HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum]
+                                InputAdapter
+                                  Exchange [channel,i_brand_id,i_class_id,i_category_id] #33
+                                    WholeStageCodegen (420)
+                                      HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum]
+                                        InputAdapter
+                                          Union
+                                            WholeStageCodegen (327)
+                                              Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                                Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                                  ReusedSubquery [average_sales] #3
+                                                  HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                                    InputAdapter
+                                                      ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #29
+                                            WholeStageCodegen (373)
+                                              Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                                Filter [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                                  ReusedSubquery [average_sales] #3
+                                                  HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                                    InputAdapter
+                                                      ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #30
+                                            WholeStageCodegen (419)
+                                              Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                                Filter [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                                  ReusedSubquery [average_sales] #3
+                                                  HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                                    InputAdapter
+                                                      ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #31
+                  WholeStageCodegen (563)
+                    HashAggregate [channel,sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum]
+                      InputAdapter
+                        Exchange [channel] #34
+                          WholeStageCodegen (562)
+                            HashAggregate [channel,sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum]
+                              HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum]
+                                InputAdapter
+                                  Exchange [channel,i_brand_id,i_class_id,i_category_id] #35
+                                    WholeStageCodegen (561)
+                                      HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum]
+                                        InputAdapter
+                                          Union
+                                            WholeStageCodegen (468)
+                                              Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                                Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                                  ReusedSubquery [average_sales] #3
+                                                  HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                                    InputAdapter
+                                                      ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #29
+                                            WholeStageCodegen (514)
+                                              Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                                Filter [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                                  ReusedSubquery [average_sales] #3
+                                                  HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                                    InputAdapter
+                                                      ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #30
+                                            WholeStageCodegen (560)
+                                              Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                                Filter [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                                  ReusedSubquery [average_sales] #3
+                                                  HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                                    InputAdapter
+                                                      ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #31
+                  WholeStageCodegen (704)
+                    HashAggregate [sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum]
+                      InputAdapter
+                        Exchange #36
+                          WholeStageCodegen (703)
+                            HashAggregate [sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum]
+                              HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum]
+                                InputAdapter
+                                  Exchange [channel,i_brand_id,i_class_id,i_category_id] #37
+                                    WholeStageCodegen (702)
+                                      HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum]
+                                        InputAdapter
+                                          Union
+                                            WholeStageCodegen (609)
+                                              Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                                Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                                  ReusedSubquery [average_sales] #3
+                                                  HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                                    InputAdapter
+                                                      ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #29
+                                            WholeStageCodegen (655)
+                                              Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                                Filter [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                                  ReusedSubquery [average_sales] #3
+                                                  HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                                    InputAdapter
+                                                      ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #30
+                                            WholeStageCodegen (701)
+                                              Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                                Filter [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                                  ReusedSubquery [average_sales] #3
+                                                  HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                                    InputAdapter
+                                                      ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #31
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/explain.txt
new file mode 100644
index 0000000000000..755973f79e493
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/explain.txt
@@ -0,0 +1,1337 @@
+== Physical Plan ==
+TakeOrderedAndProject (194)
++- * HashAggregate (193)
+   +- Exchange (192)
+      +- * HashAggregate (191)
+         +- Union (190)
+            :- * HashAggregate (113)
+            :  +- Exchange (112)
+            :     +- * HashAggregate (111)
+            :        +- Union (110)
+            :           :- * Project (77)
+            :           :  +- * Filter (76)
+            :           :     +- * HashAggregate (75)
+            :           :        +- Exchange (74)
+            :           :           +- * HashAggregate (73)
+            :           :              +- * Project (72)
+            :           :                 +- * BroadcastHashJoin Inner BuildRight (71)
+            :           :                    :- * Project (65)
+            :           :                    :  +- * BroadcastHashJoin Inner BuildRight (64)
+            :           :                    :     :- * BroadcastHashJoin LeftSemi BuildRight (57)
+            :           :                    :     :  :- * Filter (3)
+            :           :                    :     :  :  +- * ColumnarToRow (2)
+            :           :                    :     :  :     +- Scan parquet default.store_sales (1)
+            :           :                    :     :  +- BroadcastExchange (56)
+            :           :                    :     :     +- * Project (55)
+            :           :                    :     :        +- * BroadcastHashJoin Inner BuildRight (54)
+            :           :                    :     :           :- * Filter (6)
+            :           :                    :     :           :  +- * ColumnarToRow (5)
+            :           :                    :     :           :     +- Scan parquet default.item (4)
+            :           :                    :     :           +- BroadcastExchange (53)
+            :           :                    :     :              +- * HashAggregate (52)
+            :           :                    :     :                 +- * HashAggregate (51)
+            :           :                    :     :                    +- * BroadcastHashJoin LeftSemi BuildRight (50)
+            :           :                    :     :                       :- * HashAggregate (39)
+            :           :                    :     :                       :  +- Exchange (38)
+            :           :                    :     :                       :     +- * HashAggregate (37)
+            :           :                    :     :                       :        +- * Project (36)
+            :           :                    :     :                       :           +- * BroadcastHashJoin Inner BuildRight (35)
+            :           :                    :     :                       :              :- * Project (33)
+            :           :                    :     :                       :              :  +- * BroadcastHashJoin Inner BuildRight (32)
+            :           :                    :     :                       :              :     :- * Filter (9)
+            :           :                    :     :                       :              :     :  +- * ColumnarToRow (8)
+            :           :                    :     :                       :              :     :     +- Scan parquet default.store_sales (7)
+            :           :                    :     :                       :              :     +- BroadcastExchange (31)
+            :           :                    :     :                       :              :        +- * BroadcastHashJoin LeftSemi BuildRight (30)
+            :           :                    :     :                       :              :           :- * Filter (12)
+            :           :                    :     :                       :              :           :  +- * ColumnarToRow (11)
+            :           :                    :     :                       :              :           :     +- Scan parquet default.item (10)
+            :           :                    :     :                       :              :           +- BroadcastExchange (29)
+            :           :                    :     :                       :              :              +- * Project (28)
+            :           :                    :     :                       :              :                 +- * BroadcastHashJoin Inner BuildRight (27)
+            :           :                    :     :                       :              :                    :- * Project (21)
+            :           :                    :     :                       :              :                    :  +- * BroadcastHashJoin Inner BuildRight (20)
+            :           :                    :     :                       :              :                    :     :- * Filter (15)
+            :           :                    :     :                       :              :                    :     :  +- * ColumnarToRow (14)
+            :           :                    :     :                       :              :                    :     :     +- Scan parquet default.catalog_sales (13)
+            :           :                    :     :                       :              :                    :     +- BroadcastExchange (19)
+            :           :                    :     :                       :              :                    :        +- * Filter (18)
+            :           :                    :     :                       :              :                    :           +- * ColumnarToRow (17)
+            :           :                    :     :                       :              :                    :              +- Scan parquet default.item (16)
+            :           :                    :     :                       :              :                    +- BroadcastExchange (26)
+            :           :                    :     :                       :              :                       +- * Project (25)
+            :           :                    :     :                       :              :                          +- * Filter (24)
+            :           :                    :     :                       :              :                             +- * ColumnarToRow (23)
+            :           :                    :     :                       :              :                                +- Scan parquet default.date_dim (22)
+            :           :                    :     :                       :              +- ReusedExchange (34)
+            :           :                    :     :                       +- BroadcastExchange (49)
+            :           :                    :     :                          +- * Project (48)
+            :           :                    :     :                             +- * BroadcastHashJoin Inner BuildRight (47)
+            :           :                    :     :                                :- * Project (45)
+            :           :                    :     :                                :  +- * BroadcastHashJoin Inner BuildRight (44)
+            :           :                    :     :                                :     :- * Filter (42)
+            :           :                    :     :                                :     :  +- * ColumnarToRow (41)
+            :           :                    :     :                                :     :     +- Scan parquet default.web_sales (40)
+            :           :                    :     :                                :     +- ReusedExchange (43)
+            :           :                    :     :                                +- ReusedExchange (46)
+            :           :                    :     +- BroadcastExchange (63)
+            :           :                    :        +- * BroadcastHashJoin LeftSemi BuildRight (62)
+            :           :                    :           :- * Filter (60)
+            :           :                    :           :  +- * ColumnarToRow (59)
+            :           :                    :           :     +- Scan parquet default.item (58)
+            :           :                    :           +- ReusedExchange (61)
+            :           :                    +- BroadcastExchange (70)
+            :           :                       +- * Project (69)
+            :           :                          +- * Filter (68)
+            :           :                             +- * ColumnarToRow (67)
+            :           :                                +- Scan parquet default.date_dim (66)
+            :           :- * Project (93)
+            :           :  +- * Filter (92)
+            :           :     +- * HashAggregate (91)
+            :           :        +- Exchange (90)
+            :           :           +- * HashAggregate (89)
+            :           :              +- * Project (88)
+            :           :                 +- * BroadcastHashJoin Inner BuildRight (87)
+            :           :                    :- * Project (85)
+            :           :                    :  +- * BroadcastHashJoin Inner BuildRight (84)
+            :           :                    :     :- * BroadcastHashJoin LeftSemi BuildRight (82)
+            :           :                    :     :  :- * Filter (80)
+            :           :                    :     :  :  +- * ColumnarToRow (79)
+            :           :                    :     :  :     +- Scan parquet default.catalog_sales (78)
+            :           :                    :     :  +- ReusedExchange (81)
+            :           :                    :     +- ReusedExchange (83)
+            :           :                    +- ReusedExchange (86)
+            :           +- * Project (109)
+            :              +- * Filter (108)
+            :                 +- * HashAggregate (107)
+            :                    +- Exchange (106)
+            :                       +- * HashAggregate (105)
+            :                          +- * Project (104)
+            :                             +- * BroadcastHashJoin Inner BuildRight (103)
+            :                                :- * Project (101)
+            :                                :  +- * BroadcastHashJoin Inner BuildRight (100)
+            :                                :     :- * BroadcastHashJoin LeftSemi BuildRight (98)
+            :                                :     :  :- * Filter (96)
+            :                                :     :  :  +- * ColumnarToRow (95)
+            :                                :     :  :     +- Scan parquet default.web_sales (94)
+            :                                :     :  +- ReusedExchange (97)
+            :                                :     +- ReusedExchange (99)
+            :                                +- ReusedExchange (102)
+            :- * HashAggregate (132)
+            :  +- Exchange (131)
+            :     +- * HashAggregate (130)
+            :        +- * HashAggregate (129)
+            :           +- Exchange (128)
+            :              +- * HashAggregate (127)
+            :                 +- Union (126)
+            :                    :- * Project (117)
+            :                    :  +- * Filter (116)
+            :                    :     +- * HashAggregate (115)
+            :                    :        +- ReusedExchange (114)
+            :                    :- * Project (121)
+            :                    :  +- * Filter (120)
+            :                    :     +- * HashAggregate (119)
+            :                    :        +- ReusedExchange (118)
+            :                    +- * Project (125)
+            :                       +- * Filter (124)
+            :                          +- * HashAggregate (123)
+            :                             +- ReusedExchange (122)
+            :- * HashAggregate (151)
+            :  +- Exchange (150)
+            :     +- * HashAggregate (149)
+            :        +- * HashAggregate (148)
+            :           +- Exchange (147)
+            :              +- * HashAggregate (146)
+            :                 +- Union (145)
+            :                    :- * Project (136)
+            :                    :  +- * Filter (135)
+            :                    :     +- * HashAggregate (134)
+            :                    :        +- ReusedExchange (133)
+            :                    :- * Project (140)
+            :                    :  +- * Filter (139)
+            :                    :     +- * HashAggregate (138)
+            :                    :        +- ReusedExchange (137)
+            :                    +- * Project (144)
+            :                       +- * Filter (143)
+            :                          +- * HashAggregate (142)
+            :                             +- ReusedExchange (141)
+            :- * HashAggregate (170)
+            :  +- Exchange (169)
+            :     +- * HashAggregate (168)
+            :        +- * HashAggregate (167)
+            :           +- Exchange (166)
+            :              +- * HashAggregate (165)
+            :                 +- Union (164)
+            :                    :- * Project (155)
+            :                    :  +- * Filter (154)
+            :                    :     +- * HashAggregate (153)
+            :                    :        +- ReusedExchange (152)
+            :                    :- * Project (159)
+            :                    :  +- * Filter (158)
+            :                    :     +- * HashAggregate (157)
+            :                    :        +- ReusedExchange (156)
+            :                    +- * Project (163)
+            :                       +- * Filter (162)
+            :                          +- * HashAggregate (161)
+            :                             +- ReusedExchange (160)
+            +- * HashAggregate (189)
+               +- Exchange (188)
+                  +- * HashAggregate (187)
+                     +- * HashAggregate (186)
+                        +- Exchange (185)
+                           +- * HashAggregate (184)
+                              +- Union (183)
+                                 :- * Project (174)
+                                 :  +- * Filter (173)
+                                 :     +- * HashAggregate (172)
+                                 :        +- ReusedExchange (171)
+                                 :- * Project (178)
+                                 :  +- * Filter (177)
+                                 :     +- * HashAggregate (176)
+                                 :        +- ReusedExchange (175)
+                                 +- * Project (182)
+                                    +- * Filter (181)
+                                       +- * HashAggregate (180)
+                                          +- ReusedExchange (179)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_quantity:int,ss_list_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 25]
+Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 25]
+Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4]
+Condition : isnotnull(ss_item_sk#1)
+
+(4) Scan parquet default.item
+Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(5) ColumnarToRow [codegen id : 11]
+Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9]
+
+(6) Filter [codegen id : 11]
+Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9]
+Condition : ((isnotnull(i_brand_id#7) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9))
+
+(7) Scan parquet default.store_sales
+Output [2]: [ss_item_sk#10, ss_sold_date_sk#11]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#11), dynamicpruningexpression(ss_sold_date_sk#11 IN dynamicpruning#12)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int>
+
+(8) ColumnarToRow [codegen id : 6]
+Input [2]: [ss_item_sk#10, ss_sold_date_sk#11]
+
+(9) Filter [codegen id : 6]
+Input [2]: [ss_item_sk#10, ss_sold_date_sk#11]
+Condition : isnotnull(ss_item_sk#10)
+
+(10) Scan parquet default.item
+Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(11) ColumnarToRow [codegen id : 4]
+Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16]
+
+(12) Filter [codegen id : 4]
+Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16]
+Condition : (((isnotnull(i_item_sk#13) AND isnotnull(i_brand_id#14)) AND isnotnull(i_class_id#15)) AND isnotnull(i_category_id#16))
+
+(13) Scan parquet default.catalog_sales
+Output [2]: [cs_item_sk#17, cs_sold_date_sk#18]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_sold_date_sk#18 IN dynamicpruning#12)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int>
+
+(14) ColumnarToRow [codegen id : 3]
+Input [2]: [cs_item_sk#17, cs_sold_date_sk#18]
+
+(15) Filter [codegen id : 3]
+Input [2]: [cs_item_sk#17, cs_sold_date_sk#18]
+Condition : isnotnull(cs_item_sk#17)
+
+(16) Scan parquet default.item
+Output [4]: [i_item_sk#19, i_brand_id#20, i_class_id#21, i_category_id#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(17) ColumnarToRow [codegen id : 1]
+Input [4]: [i_item_sk#19, i_brand_id#20, i_class_id#21, i_category_id#22]
+
+(18) Filter [codegen id : 1]
+Input [4]: [i_item_sk#19, i_brand_id#20, i_class_id#21, i_category_id#22]
+Condition : isnotnull(i_item_sk#19)
+
+(19) BroadcastExchange
+Input [4]: [i_item_sk#19, i_brand_id#20, i_class_id#21, i_category_id#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#23]
+
+(20) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_item_sk#17]
+Right keys [1]: [i_item_sk#19]
+Join condition: None
+
+(21) Project [codegen id : 3]
+Output [4]: [cs_sold_date_sk#18, i_brand_id#20, i_class_id#21, i_category_id#22]
+Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#19, i_brand_id#20, i_class_id#21, i_category_id#22]
+
+(22) Scan parquet default.date_dim
+Output [2]: [d_date_sk#24, d_year#25]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(23) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#24, d_year#25]
+
+(24) Filter [codegen id : 2]
+Input [2]: [d_date_sk#24, d_year#25]
+Condition : (((isnotnull(d_year#25) AND (d_year#25 >= 1999)) AND (d_year#25 <= 2001)) AND isnotnull(d_date_sk#24))
+
+(25) Project [codegen id : 2]
+Output [1]: [d_date_sk#24]
+Input [2]: [d_date_sk#24, d_year#25]
+
+(26) BroadcastExchange
+Input [1]: [d_date_sk#24]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#26]
+
+(27) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_sold_date_sk#18]
+Right keys [1]: [d_date_sk#24]
+Join condition: None
+
+(28) Project [codegen id : 3]
+Output [3]: [i_brand_id#20, i_class_id#21, i_category_id#22]
+Input [5]: [cs_sold_date_sk#18, i_brand_id#20, i_class_id#21, i_category_id#22, d_date_sk#24]
+
+(29) BroadcastExchange
+Input [3]: [i_brand_id#20, i_class_id#21, i_category_id#22]
+Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [id=#27]
+
+(30) BroadcastHashJoin [codegen id : 4]
+Left keys [6]: [coalesce(i_brand_id#14, 0), isnull(i_brand_id#14), coalesce(i_class_id#15, 0), isnull(i_class_id#15), coalesce(i_category_id#16, 0), isnull(i_category_id#16)]
+Right keys [6]: [coalesce(i_brand_id#20, 0), isnull(i_brand_id#20), coalesce(i_class_id#21, 0), isnull(i_class_id#21), coalesce(i_category_id#22, 0), isnull(i_category_id#22)]
+Join condition: None
+
+(31) BroadcastExchange
+Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#28]
+
+(32) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_item_sk#10]
+Right keys [1]: [i_item_sk#13]
+Join condition: None
+
+(33) Project [codegen id : 6]
+Output [4]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16]
+Input [6]: [ss_item_sk#10, ss_sold_date_sk#11, i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16]
+
+(34) ReusedExchange [Reuses operator id: 26]
+Output [1]: [d_date_sk#29]
+
+(35) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_sold_date_sk#11]
+Right keys [1]: [d_date_sk#29]
+Join condition: None
+
+(36) Project [codegen id : 6]
+Output [3]: [i_brand_id#14 AS brand_id#30, i_class_id#15 AS class_id#31, i_category_id#16 AS category_id#32]
+Input [5]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16, d_date_sk#29]
+
+(37) HashAggregate [codegen id : 6]
+Input [3]: [brand_id#30, class_id#31, category_id#32]
+Keys [3]: [brand_id#30, class_id#31, category_id#32]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#30, class_id#31, category_id#32]
+
+(38) Exchange
+Input [3]: [brand_id#30, class_id#31, category_id#32]
+Arguments: hashpartitioning(brand_id#30, class_id#31, category_id#32, 5), ENSURE_REQUIREMENTS, [id=#33]
+
+(39) HashAggregate [codegen id : 10]
+Input [3]: [brand_id#30, class_id#31, category_id#32]
+Keys [3]: [brand_id#30, class_id#31, category_id#32]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#30, class_id#31, category_id#32]
+
+(40) Scan parquet default.web_sales
+Output [2]: [ws_item_sk#34, ws_sold_date_sk#35]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#35), dynamicpruningexpression(ws_sold_date_sk#35 IN dynamicpruning#12)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int>
+
+(41) ColumnarToRow [codegen id : 9]
+Input [2]: [ws_item_sk#34, ws_sold_date_sk#35]
+
+(42) Filter [codegen id : 9]
+Input [2]: [ws_item_sk#34, ws_sold_date_sk#35]
+Condition : isnotnull(ws_item_sk#34)
+
+(43) ReusedExchange [Reuses operator id: 19]
+Output [4]: [i_item_sk#36, i_brand_id#37, i_class_id#38, i_category_id#39]
+
+(44) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ws_item_sk#34]
+Right keys [1]: [i_item_sk#36]
+Join condition: None
+
+(45) Project [codegen id : 9]
+Output [4]: [ws_sold_date_sk#35, i_brand_id#37, i_class_id#38, i_category_id#39]
+Input [6]: [ws_item_sk#34, ws_sold_date_sk#35, i_item_sk#36, i_brand_id#37, i_class_id#38, i_category_id#39]
+
+(46) ReusedExchange [Reuses operator id: 26]
+Output [1]: [d_date_sk#40]
+
+(47) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ws_sold_date_sk#35]
+Right keys [1]: [d_date_sk#40]
+Join condition: None
+
+(48) Project [codegen id : 9]
+Output [3]: [i_brand_id#37, i_class_id#38, i_category_id#39]
+Input [5]: [ws_sold_date_sk#35, i_brand_id#37, i_class_id#38, i_category_id#39, d_date_sk#40]
+
+(49) BroadcastExchange
+Input [3]: [i_brand_id#37, i_class_id#38, i_category_id#39]
+Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [id=#41]
+
+(50) BroadcastHashJoin [codegen id : 10]
+Left keys [6]: [coalesce(brand_id#30, 0), isnull(brand_id#30), coalesce(class_id#31, 0), isnull(class_id#31), coalesce(category_id#32, 0), isnull(category_id#32)]
+Right keys [6]: [coalesce(i_brand_id#37, 0), isnull(i_brand_id#37), coalesce(i_class_id#38, 0), isnull(i_class_id#38), coalesce(i_category_id#39, 0), isnull(i_category_id#39)]
+Join condition: None
+
+(51) HashAggregate [codegen id : 10]
+Input [3]: [brand_id#30, class_id#31, category_id#32]
+Keys [3]: [brand_id#30, class_id#31, category_id#32]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#30, class_id#31, category_id#32]
+
+(52) HashAggregate [codegen id : 10]
+Input [3]: [brand_id#30, class_id#31, category_id#32]
+Keys [3]: [brand_id#30, class_id#31, category_id#32]
+Functions: []
+Aggregate Attributes: []
+Results [3]: [brand_id#30, class_id#31, category_id#32]
+
+(53) BroadcastExchange
+Input [3]: [brand_id#30, class_id#31, category_id#32]
+Arguments: HashedRelationBroadcastMode(List(input[0, int, true], input[1, int, true], input[2, int, true]),false), [id=#42]
+
+(54) BroadcastHashJoin [codegen id : 11]
+Left keys [3]: [i_brand_id#7, i_class_id#8, i_category_id#9]
+Right keys [3]: [brand_id#30, class_id#31, category_id#32]
+Join condition: None
+
+(55) Project [codegen id : 11]
+Output [1]: [i_item_sk#6 AS ss_item_sk#43]
+Input [7]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, brand_id#30, class_id#31, category_id#32]
+
+(56) BroadcastExchange
+Input [1]: [ss_item_sk#43]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#44]
+
+(57) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [ss_item_sk#43]
+Join condition: None
+
+(58) Scan parquet default.item
+Output [4]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int>
+
+(59) ColumnarToRow [codegen id : 23]
+Input [4]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48]
+
+(60) Filter [codegen id : 23]
+Input [4]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48]
+Condition : isnotnull(i_item_sk#45)
+
+(61) ReusedExchange [Reuses operator id: unknown]
+Output [1]: [ss_item_sk#43]
+
+(62) BroadcastHashJoin [codegen id : 23]
+Left keys [1]: [i_item_sk#45]
+Right keys [1]: [ss_item_sk#43]
+Join condition: None
+
+(63) BroadcastExchange
+Input [4]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#49]
+
+(64) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#45]
+Join condition: None
+
+(65) Project [codegen id : 25]
+Output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#46, i_class_id#47, i_category_id#48]
+Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48]
+
+(66) Scan parquet default.date_dim
+Output [3]: [d_date_sk#50, d_year#51, d_moy#52]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,11), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(67) ColumnarToRow [codegen id : 24]
+Input [3]: [d_date_sk#50, d_year#51, d_moy#52]
+
+(68) Filter [codegen id : 24]
+Input [3]: [d_date_sk#50, d_year#51, d_moy#52]
+Condition : ((((isnotnull(d_year#51) AND isnotnull(d_moy#52)) AND (d_year#51 = 2000)) AND (d_moy#52 = 11)) AND isnotnull(d_date_sk#50))
+
+(69) Project [codegen id : 24]
+Output [1]: [d_date_sk#50]
+Input [3]: [d_date_sk#50, d_year#51, d_moy#52]
+
+(70) BroadcastExchange
+Input [1]: [d_date_sk#50]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#53]
+
+(71) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#50]
+Join condition: None
+
+(72) Project [codegen id : 25]
+Output [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#46, i_class_id#47, i_category_id#48]
+Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#46, i_class_id#47, i_category_id#48, d_date_sk#50]
+
+(73) HashAggregate [codegen id : 25]
+Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#46, i_class_id#47, i_category_id#48]
+Keys [3]: [i_brand_id#46, i_class_id#47, i_category_id#48]
+Functions [2]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true)), partial_count(1)]
+Aggregate Attributes [3]: [sum#54, isEmpty#55, count#56]
+Results [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sum#57, isEmpty#58, count#59]
+
+(74) Exchange
+Input [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sum#57, isEmpty#58, count#59]
+Arguments: hashpartitioning(i_brand_id#46, i_class_id#47, i_category_id#48, 5), ENSURE_REQUIREMENTS, [id=#60]
+
+(75) HashAggregate [codegen id : 26]
+Input [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sum#57, isEmpty#58, count#59]
+Keys [3]: [i_brand_id#46, i_class_id#47, i_category_id#48]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#61, count(1)#62]
+Results [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#61 AS sales#63, count(1)#62 AS number_sales#64, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#61 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#65]
+
+(76) Filter [codegen id : 26]
+Input [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sales#63, number_sales#64, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#65]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#65) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#65 as decimal(32,6)) > cast(Subquery scalar-subquery#66, [id=#67] as decimal(32,6))))
+
+(77) Project [codegen id : 26]
+Output [6]: [store AS channel#68, i_brand_id#46, i_class_id#47, i_category_id#48, sales#63, number_sales#64]
+Input [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sales#63, number_sales#64, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#2 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#3 as decimal(12,2)))), DecimalType(18,2), true))#65]
+
+(78) Scan parquet default.catalog_sales
+Output [4]: [cs_item_sk#69, cs_quantity#70, cs_list_price#71, cs_sold_date_sk#72]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#72), dynamicpruningexpression(cs_sold_date_sk#72 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int,cs_quantity:int,cs_list_price:decimal(7,2)>
+
+(79) ColumnarToRow [codegen id : 51]
+Input [4]: [cs_item_sk#69, cs_quantity#70, cs_list_price#71, cs_sold_date_sk#72]
+
+(80) Filter [codegen id : 51]
+Input [4]: [cs_item_sk#69, cs_quantity#70, cs_list_price#71, cs_sold_date_sk#72]
+Condition : isnotnull(cs_item_sk#69)
+
+(81) ReusedExchange [Reuses operator id: unknown]
+Output [1]: [ss_item_sk#43]
+
+(82) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [cs_item_sk#69]
+Right keys [1]: [ss_item_sk#43]
+Join condition: None
+
+(83) ReusedExchange [Reuses operator id: 63]
+Output [4]: [i_item_sk#73, i_brand_id#74, i_class_id#75, i_category_id#76]
+
+(84) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [cs_item_sk#69]
+Right keys [1]: [i_item_sk#73]
+Join condition: None
+
+(85) Project [codegen id : 51]
+Output [6]: [cs_quantity#70, cs_list_price#71, cs_sold_date_sk#72, i_brand_id#74, i_class_id#75, i_category_id#76]
+Input [8]: [cs_item_sk#69, cs_quantity#70, cs_list_price#71, cs_sold_date_sk#72, i_item_sk#73, i_brand_id#74, i_class_id#75, i_category_id#76]
+
+(86) ReusedExchange [Reuses operator id: 70]
+Output [1]: [d_date_sk#77]
+
+(87) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [cs_sold_date_sk#72]
+Right keys [1]: [d_date_sk#77]
+Join condition: None
+
+(88) Project [codegen id : 51]
+Output [5]: [cs_quantity#70, cs_list_price#71, i_brand_id#74, i_class_id#75, i_category_id#76]
+Input [7]: [cs_quantity#70, cs_list_price#71, cs_sold_date_sk#72, i_brand_id#74, i_class_id#75, i_category_id#76, d_date_sk#77]
+
+(89) HashAggregate [codegen id : 51]
+Input [5]: [cs_quantity#70, cs_list_price#71, i_brand_id#74, i_class_id#75, i_category_id#76]
+Keys [3]: [i_brand_id#74, i_class_id#75, i_category_id#76]
+Functions [2]: [partial_sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#70 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#71 as decimal(12,2)))), DecimalType(18,2), true)), partial_count(1)]
+Aggregate Attributes [3]: [sum#78, isEmpty#79, count#80]
+Results [6]: [i_brand_id#74, i_class_id#75, i_category_id#76, sum#81, isEmpty#82, count#83]
+
+(90) Exchange
+Input [6]: [i_brand_id#74, i_class_id#75, i_category_id#76, sum#81, isEmpty#82, count#83]
+Arguments: hashpartitioning(i_brand_id#74, i_class_id#75, i_category_id#76, 5), ENSURE_REQUIREMENTS, [id=#84]
+
+(91) HashAggregate [codegen id : 52]
+Input [6]: [i_brand_id#74, i_class_id#75, i_category_id#76, sum#81, isEmpty#82, count#83]
+Keys [3]: [i_brand_id#74, i_class_id#75, i_category_id#76]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#70 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#71 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#70 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#71 as decimal(12,2)))), DecimalType(18,2), true))#85, count(1)#86]
+Results [6]: [i_brand_id#74, i_class_id#75, i_category_id#76, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#70 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#71 as decimal(12,2)))), DecimalType(18,2), true))#85 AS sales#87, count(1)#86 AS number_sales#88, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#70 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#71 as decimal(12,2)))), DecimalType(18,2), true))#85 AS sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#70 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#71 as decimal(12,2)))), DecimalType(18,2), true))#89]
+
+(92) Filter [codegen id : 52]
+Input [6]: [i_brand_id#74, i_class_id#75, i_category_id#76, sales#87, number_sales#88, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#70 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#71 as decimal(12,2)))), DecimalType(18,2), true))#89]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#70 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#71 as decimal(12,2)))), DecimalType(18,2), true))#89) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#70 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#71 as decimal(12,2)))), DecimalType(18,2), true))#89 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#66, [id=#67] as decimal(32,6))))
+
+(93) Project [codegen id : 52]
+Output [6]: [catalog AS channel#90, i_brand_id#74, i_class_id#75, i_category_id#76, sales#87, number_sales#88]
+Input [6]: [i_brand_id#74, i_class_id#75, i_category_id#76, sales#87, number_sales#88, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#70 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#71 as decimal(12,2)))), DecimalType(18,2), true))#89]
+
+(94) Scan parquet default.web_sales
+Output [4]: [ws_item_sk#91, ws_quantity#92, ws_list_price#93, ws_sold_date_sk#94]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#94), dynamicpruningexpression(ws_sold_date_sk#94 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_quantity:int,ws_list_price:decimal(7,2)>
+
+(95) ColumnarToRow [codegen id : 77]
+Input [4]: [ws_item_sk#91, ws_quantity#92, ws_list_price#93, ws_sold_date_sk#94]
+
+(96) Filter [codegen id : 77]
+Input [4]: [ws_item_sk#91, ws_quantity#92, ws_list_price#93, ws_sold_date_sk#94]
+Condition : isnotnull(ws_item_sk#91)
+
+(97) ReusedExchange [Reuses operator id: unknown]
+Output [1]: [ss_item_sk#43]
+
+(98) BroadcastHashJoin [codegen id : 77]
+Left keys [1]: [ws_item_sk#91]
+Right keys [1]: [ss_item_sk#43]
+Join condition: None
+
+(99) ReusedExchange [Reuses operator id: 63]
+Output [4]: [i_item_sk#95, i_brand_id#96, i_class_id#97, i_category_id#98]
+
+(100) BroadcastHashJoin [codegen id : 77]
+Left keys [1]: [ws_item_sk#91]
+Right keys [1]: [i_item_sk#95]
+Join condition: None
+
+(101) Project [codegen id : 77]
+Output [6]: [ws_quantity#92, ws_list_price#93, ws_sold_date_sk#94, i_brand_id#96, i_class_id#97, i_category_id#98]
+Input [8]: [ws_item_sk#91, ws_quantity#92, ws_list_price#93, ws_sold_date_sk#94, i_item_sk#95, i_brand_id#96, i_class_id#97, i_category_id#98]
+
+(102) ReusedExchange [Reuses operator id: 70]
+Output [1]: [d_date_sk#99]
+
+(103) BroadcastHashJoin [codegen id : 77]
+Left keys [1]: [ws_sold_date_sk#94]
+Right keys [1]: [d_date_sk#99]
+Join condition: None
+
+(104) Project [codegen id : 77]
+Output [5]: [ws_quantity#92, ws_list_price#93, i_brand_id#96, i_class_id#97, i_category_id#98]
+Input [7]: [ws_quantity#92, ws_list_price#93, ws_sold_date_sk#94, i_brand_id#96, i_class_id#97, i_category_id#98, d_date_sk#99]
+
+(105) HashAggregate [codegen id : 77]
+Input [5]: [ws_quantity#92, ws_list_price#93, i_brand_id#96, i_class_id#97, i_category_id#98]
+Keys [3]: [i_brand_id#96, i_class_id#97, i_category_id#98]
+Functions [2]: [partial_sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#92 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#93 as decimal(12,2)))), DecimalType(18,2), true)), partial_count(1)]
+Aggregate Attributes [3]: [sum#100, isEmpty#101, count#102]
+Results [6]: [i_brand_id#96, i_class_id#97, i_category_id#98, sum#103, isEmpty#104, count#105]
+
+(106) Exchange
+Input [6]: [i_brand_id#96, i_class_id#97, i_category_id#98, sum#103, isEmpty#104, count#105]
+Arguments: hashpartitioning(i_brand_id#96, i_class_id#97, i_category_id#98, 5), ENSURE_REQUIREMENTS, [id=#106]
+
+(107) HashAggregate [codegen id : 78]
+Input [6]: [i_brand_id#96, i_class_id#97, i_category_id#98, sum#103, isEmpty#104, count#105]
+Keys [3]: [i_brand_id#96, i_class_id#97, i_category_id#98]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#92 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#93 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#92 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#93 as decimal(12,2)))), DecimalType(18,2), true))#107, count(1)#108]
+Results [6]: [i_brand_id#96, i_class_id#97, i_category_id#98, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#92 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#93 as decimal(12,2)))), DecimalType(18,2), true))#107 AS sales#109, count(1)#108 AS number_sales#110, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#92 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#93 as decimal(12,2)))), DecimalType(18,2), true))#107 AS sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#92 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#93 as decimal(12,2)))), DecimalType(18,2), true))#111]
+
+(108) Filter [codegen id : 78]
+Input [6]: [i_brand_id#96, i_class_id#97, i_category_id#98, sales#109, number_sales#110, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#92 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#93 as decimal(12,2)))), DecimalType(18,2), true))#111]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#92 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#93 as decimal(12,2)))), DecimalType(18,2), true))#111) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#92 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#93 as decimal(12,2)))), DecimalType(18,2), true))#111 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#66, [id=#67] as decimal(32,6))))
+
+(109) Project [codegen id : 78]
+Output [6]: [web AS channel#112, i_brand_id#96, i_class_id#97, i_category_id#98, sales#109, number_sales#110]
+Input [6]: [i_brand_id#96, i_class_id#97, i_category_id#98, sales#109, number_sales#110, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#92 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#93 as decimal(12,2)))), DecimalType(18,2), true))#111]
+
+(110) Union
+
+(111) HashAggregate [codegen id : 79]
+Input [6]: [channel#68, i_brand_id#46, i_class_id#47, i_category_id#48, sales#63, number_sales#64]
+Keys [4]: [channel#68, i_brand_id#46, i_class_id#47, i_category_id#48]
+Functions [2]: [partial_sum(sales#63), partial_sum(number_sales#64)]
+Aggregate Attributes [3]: [sum#113, isEmpty#114, sum#115]
+Results [7]: [channel#68, i_brand_id#46, i_class_id#47, i_category_id#48, sum#116, isEmpty#117, sum#118]
+
+(112) Exchange
+Input [7]: [channel#68, i_brand_id#46, i_class_id#47, i_category_id#48, sum#116, isEmpty#117, sum#118]
+Arguments: hashpartitioning(channel#68, i_brand_id#46, i_class_id#47, i_category_id#48, 5), ENSURE_REQUIREMENTS, [id=#119]
+
+(113) HashAggregate [codegen id : 80]
+Input [7]: [channel#68, i_brand_id#46, i_class_id#47, i_category_id#48, sum#116, isEmpty#117, sum#118]
+Keys [4]: [channel#68, i_brand_id#46, i_class_id#47, i_category_id#48]
+Functions [2]: [sum(sales#63), sum(number_sales#64)]
+Aggregate Attributes [2]: [sum(sales#63)#120, sum(number_sales#64)#121]
+Results [6]: [channel#68, i_brand_id#46, i_class_id#47, i_category_id#48, sum(sales#63)#120 AS sum_sales#122, sum(number_sales#64)#121 AS number_sales#123]
+
+(114) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_brand_id#124, i_class_id#125, i_category_id#126, sum#127, isEmpty#128, count#129]
+
+(115) HashAggregate [codegen id : 106]
+Input [6]: [i_brand_id#124, i_class_id#125, i_category_id#126, sum#127, isEmpty#128, count#129]
+Keys [3]: [i_brand_id#124, i_class_id#125, i_category_id#126]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#130 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#131 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#130 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#131 as decimal(12,2)))), DecimalType(18,2), true))#132, count(1)#133]
+Results [6]: [i_brand_id#124, i_class_id#125, i_category_id#126, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#130 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#131 as decimal(12,2)))), DecimalType(18,2), true))#132 AS sales#63, count(1)#133 AS number_sales#64, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#130 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#131 as decimal(12,2)))), DecimalType(18,2), true))#132 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#130 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#131 as decimal(12,2)))), DecimalType(18,2), true))#134]
+
+(116) Filter [codegen id : 106]
+Input [6]: [i_brand_id#124, i_class_id#125, i_category_id#126, sales#63, number_sales#64, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#130 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#131 as decimal(12,2)))), DecimalType(18,2), true))#134]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#130 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#131 as decimal(12,2)))), DecimalType(18,2), true))#134) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#130 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#131 as decimal(12,2)))), DecimalType(18,2), true))#134 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#66, [id=#67] as decimal(32,6))))
+
+(117) Project [codegen id : 106]
+Output [6]: [store AS channel#68, i_brand_id#124, i_class_id#125, i_category_id#126, sales#63, number_sales#64]
+Input [6]: [i_brand_id#124, i_class_id#125, i_category_id#126, sales#63, number_sales#64, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#130 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#131 as decimal(12,2)))), DecimalType(18,2), true))#134]
+
+(118) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_brand_id#135, i_class_id#136, i_category_id#137, sum#138, isEmpty#139, count#140]
+
+(119) HashAggregate [codegen id : 132]
+Input [6]: [i_brand_id#135, i_class_id#136, i_category_id#137, sum#138, isEmpty#139, count#140]
+Keys [3]: [i_brand_id#135, i_class_id#136, i_category_id#137]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#141 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#142 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#141 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#142 as decimal(12,2)))), DecimalType(18,2), true))#143, count(1)#144]
+Results [6]: [i_brand_id#135, i_class_id#136, i_category_id#137, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#141 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#142 as decimal(12,2)))), DecimalType(18,2), true))#143 AS sales#87, count(1)#144 AS number_sales#88, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#141 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#142 as decimal(12,2)))), DecimalType(18,2), true))#143 AS sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#141 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#142 as decimal(12,2)))), DecimalType(18,2), true))#145]
+
+(120) Filter [codegen id : 132]
+Input [6]: [i_brand_id#135, i_class_id#136, i_category_id#137, sales#87, number_sales#88, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#141 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#142 as decimal(12,2)))), DecimalType(18,2), true))#145]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#141 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#142 as decimal(12,2)))), DecimalType(18,2), true))#145) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#141 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#142 as decimal(12,2)))), DecimalType(18,2), true))#145 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#66, [id=#67] as decimal(32,6))))
+
+(121) Project [codegen id : 132]
+Output [6]: [catalog AS channel#90, i_brand_id#135, i_class_id#136, i_category_id#137, sales#87, number_sales#88]
+Input [6]: [i_brand_id#135, i_class_id#136, i_category_id#137, sales#87, number_sales#88, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#141 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#142 as decimal(12,2)))), DecimalType(18,2), true))#145]
+
+(122) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_brand_id#146, i_class_id#147, i_category_id#148, sum#149, isEmpty#150, count#151]
+
+(123) HashAggregate [codegen id : 158]
+Input [6]: [i_brand_id#146, i_class_id#147, i_category_id#148, sum#149, isEmpty#150, count#151]
+Keys [3]: [i_brand_id#146, i_class_id#147, i_category_id#148]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#152 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#153 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#152 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#153 as decimal(12,2)))), DecimalType(18,2), true))#154, count(1)#155]
+Results [6]: [i_brand_id#146, i_class_id#147, i_category_id#148, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#152 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#153 as decimal(12,2)))), DecimalType(18,2), true))#154 AS sales#109, count(1)#155 AS number_sales#110, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#152 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#153 as decimal(12,2)))), DecimalType(18,2), true))#154 AS sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#152 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#153 as decimal(12,2)))), DecimalType(18,2), true))#156]
+
+(124) Filter [codegen id : 158]
+Input [6]: [i_brand_id#146, i_class_id#147, i_category_id#148, sales#109, number_sales#110, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#152 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#153 as decimal(12,2)))), DecimalType(18,2), true))#156]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#152 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#153 as decimal(12,2)))), DecimalType(18,2), true))#156) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#152 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#153 as decimal(12,2)))), DecimalType(18,2), true))#156 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#66, [id=#67] as decimal(32,6))))
+
+(125) Project [codegen id : 158]
+Output [6]: [web AS channel#112, i_brand_id#146, i_class_id#147, i_category_id#148, sales#109, number_sales#110]
+Input [6]: [i_brand_id#146, i_class_id#147, i_category_id#148, sales#109, number_sales#110, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#152 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#153 as decimal(12,2)))), DecimalType(18,2), true))#156]
+
+(126) Union
+
+(127) HashAggregate [codegen id : 159]
+Input [6]: [channel#68, i_brand_id#124, i_class_id#125, i_category_id#126, sales#63, number_sales#64]
+Keys [4]: [channel#68, i_brand_id#124, i_class_id#125, i_category_id#126]
+Functions [2]: [partial_sum(sales#63), partial_sum(number_sales#64)]
+Aggregate Attributes [3]: [sum#157, isEmpty#158, sum#159]
+Results [7]: [channel#68, i_brand_id#124, i_class_id#125, i_category_id#126, sum#160, isEmpty#161, sum#162]
+
+(128) Exchange
+Input [7]: [channel#68, i_brand_id#124, i_class_id#125, i_category_id#126, sum#160, isEmpty#161, sum#162]
+Arguments: hashpartitioning(channel#68, i_brand_id#124, i_class_id#125, i_category_id#126, 5), ENSURE_REQUIREMENTS, [id=#163]
+
+(129) HashAggregate [codegen id : 160]
+Input [7]: [channel#68, i_brand_id#124, i_class_id#125, i_category_id#126, sum#160, isEmpty#161, sum#162]
+Keys [4]: [channel#68, i_brand_id#124, i_class_id#125, i_category_id#126]
+Functions [2]: [sum(sales#63), sum(number_sales#64)]
+Aggregate Attributes [2]: [sum(sales#63)#164, sum(number_sales#64)#165]
+Results [5]: [channel#68, i_brand_id#124, i_class_id#125, sum(sales#63)#164 AS sum_sales#122, sum(number_sales#64)#165 AS number_sales#123]
+
+(130) HashAggregate [codegen id : 160]
+Input [5]: [channel#68, i_brand_id#124, i_class_id#125, sum_sales#122, number_sales#123]
+Keys [3]: [channel#68, i_brand_id#124, i_class_id#125]
+Functions [2]: [partial_sum(sum_sales#122), partial_sum(number_sales#123)]
+Aggregate Attributes [3]: [sum#166, isEmpty#167, sum#168]
+Results [6]: [channel#68, i_brand_id#124, i_class_id#125, sum#169, isEmpty#170, sum#171]
+
+(131) Exchange
+Input [6]: [channel#68, i_brand_id#124, i_class_id#125, sum#169, isEmpty#170, sum#171]
+Arguments: hashpartitioning(channel#68, i_brand_id#124, i_class_id#125, 5), ENSURE_REQUIREMENTS, [id=#172]
+
+(132) HashAggregate [codegen id : 161]
+Input [6]: [channel#68, i_brand_id#124, i_class_id#125, sum#169, isEmpty#170, sum#171]
+Keys [3]: [channel#68, i_brand_id#124, i_class_id#125]
+Functions [2]: [sum(sum_sales#122), sum(number_sales#123)]
+Aggregate Attributes [2]: [sum(sum_sales#122)#173, sum(number_sales#123)#174]
+Results [6]: [channel#68, i_brand_id#124, i_class_id#125, null AS i_category_id#175, sum(sum_sales#122)#173 AS sum(sum_sales)#176, sum(number_sales#123)#174 AS sum(number_sales)#177]
+
+(133) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_brand_id#178, i_class_id#179, i_category_id#180, sum#181, isEmpty#182, count#183]
+
+(134) HashAggregate [codegen id : 187]
+Input [6]: [i_brand_id#178, i_class_id#179, i_category_id#180, sum#181, isEmpty#182, count#183]
+Keys [3]: [i_brand_id#178, i_class_id#179, i_category_id#180]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#184 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#185 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#184 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#185 as decimal(12,2)))), DecimalType(18,2), true))#186, count(1)#187]
+Results [6]: [i_brand_id#178, i_class_id#179, i_category_id#180, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#184 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#185 as decimal(12,2)))), DecimalType(18,2), true))#186 AS sales#63, count(1)#187 AS number_sales#64, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#184 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#185 as decimal(12,2)))), DecimalType(18,2), true))#186 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#184 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#185 as decimal(12,2)))), DecimalType(18,2), true))#188]
+
+(135) Filter [codegen id : 187]
+Input [6]: [i_brand_id#178, i_class_id#179, i_category_id#180, sales#63, number_sales#64, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#184 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#185 as decimal(12,2)))), DecimalType(18,2), true))#188]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#184 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#185 as decimal(12,2)))), DecimalType(18,2), true))#188) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#184 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#185 as decimal(12,2)))), DecimalType(18,2), true))#188 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#66, [id=#67] as decimal(32,6))))
+
+(136) Project [codegen id : 187]
+Output [6]: [store AS channel#68, i_brand_id#178, i_class_id#179, i_category_id#180, sales#63, number_sales#64]
+Input [6]: [i_brand_id#178, i_class_id#179, i_category_id#180, sales#63, number_sales#64, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#184 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#185 as decimal(12,2)))), DecimalType(18,2), true))#188]
+
+(137) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_brand_id#189, i_class_id#190, i_category_id#191, sum#192, isEmpty#193, count#194]
+
+(138) HashAggregate [codegen id : 213]
+Input [6]: [i_brand_id#189, i_class_id#190, i_category_id#191, sum#192, isEmpty#193, count#194]
+Keys [3]: [i_brand_id#189, i_class_id#190, i_category_id#191]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#195 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#196 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#195 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#196 as decimal(12,2)))), DecimalType(18,2), true))#197, count(1)#198]
+Results [6]: [i_brand_id#189, i_class_id#190, i_category_id#191, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#195 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#196 as decimal(12,2)))), DecimalType(18,2), true))#197 AS sales#87, count(1)#198 AS number_sales#88, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#195 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#196 as decimal(12,2)))), DecimalType(18,2), true))#197 AS sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#195 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#196 as decimal(12,2)))), DecimalType(18,2), true))#199]
+
+(139) Filter [codegen id : 213]
+Input [6]: [i_brand_id#189, i_class_id#190, i_category_id#191, sales#87, number_sales#88, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#195 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#196 as decimal(12,2)))), DecimalType(18,2), true))#199]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#195 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#196 as decimal(12,2)))), DecimalType(18,2), true))#199) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#195 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#196 as decimal(12,2)))), DecimalType(18,2), true))#199 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#66, [id=#67] as decimal(32,6))))
+
+(140) Project [codegen id : 213]
+Output [6]: [catalog AS channel#90, i_brand_id#189, i_class_id#190, i_category_id#191, sales#87, number_sales#88]
+Input [6]: [i_brand_id#189, i_class_id#190, i_category_id#191, sales#87, number_sales#88, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#195 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#196 as decimal(12,2)))), DecimalType(18,2), true))#199]
+
+(141) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_brand_id#200, i_class_id#201, i_category_id#202, sum#203, isEmpty#204, count#205]
+
+(142) HashAggregate [codegen id : 239]
+Input [6]: [i_brand_id#200, i_class_id#201, i_category_id#202, sum#203, isEmpty#204, count#205]
+Keys [3]: [i_brand_id#200, i_class_id#201, i_category_id#202]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#206 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#207 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#206 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#207 as decimal(12,2)))), DecimalType(18,2), true))#208, count(1)#209]
+Results [6]: [i_brand_id#200, i_class_id#201, i_category_id#202, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#206 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#207 as decimal(12,2)))), DecimalType(18,2), true))#208 AS sales#109, count(1)#209 AS number_sales#110, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#206 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#207 as decimal(12,2)))), DecimalType(18,2), true))#208 AS sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#206 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#207 as decimal(12,2)))), DecimalType(18,2), true))#210]
+
+(143) Filter [codegen id : 239]
+Input [6]: [i_brand_id#200, i_class_id#201, i_category_id#202, sales#109, number_sales#110, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#206 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#207 as decimal(12,2)))), DecimalType(18,2), true))#210]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#206 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#207 as decimal(12,2)))), DecimalType(18,2), true))#210) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#206 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#207 as decimal(12,2)))), DecimalType(18,2), true))#210 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#66, [id=#67] as decimal(32,6))))
+
+(144) Project [codegen id : 239]
+Output [6]: [web AS channel#112, i_brand_id#200, i_class_id#201, i_category_id#202, sales#109, number_sales#110]
+Input [6]: [i_brand_id#200, i_class_id#201, i_category_id#202, sales#109, number_sales#110, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#206 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#207 as decimal(12,2)))), DecimalType(18,2), true))#210]
+
+(145) Union
+
+(146) HashAggregate [codegen id : 240]
+Input [6]: [channel#68, i_brand_id#178, i_class_id#179, i_category_id#180, sales#63, number_sales#64]
+Keys [4]: [channel#68, i_brand_id#178, i_class_id#179, i_category_id#180]
+Functions [2]: [partial_sum(sales#63), partial_sum(number_sales#64)]
+Aggregate Attributes [3]: [sum#211, isEmpty#212, sum#213]
+Results [7]: [channel#68, i_brand_id#178, i_class_id#179, i_category_id#180, sum#214, isEmpty#215, sum#216]
+
+(147) Exchange
+Input [7]: [channel#68, i_brand_id#178, i_class_id#179, i_category_id#180, sum#214, isEmpty#215, sum#216]
+Arguments: hashpartitioning(channel#68, i_brand_id#178, i_class_id#179, i_category_id#180, 5), ENSURE_REQUIREMENTS, [id=#217]
+
+(148) HashAggregate [codegen id : 241]
+Input [7]: [channel#68, i_brand_id#178, i_class_id#179, i_category_id#180, sum#214, isEmpty#215, sum#216]
+Keys [4]: [channel#68, i_brand_id#178, i_class_id#179, i_category_id#180]
+Functions [2]: [sum(sales#63), sum(number_sales#64)]
+Aggregate Attributes [2]: [sum(sales#63)#218, sum(number_sales#64)#219]
+Results [4]: [channel#68, i_brand_id#178, sum(sales#63)#218 AS sum_sales#122, sum(number_sales#64)#219 AS number_sales#123]
+
+(149) HashAggregate [codegen id : 241]
+Input [4]: [channel#68, i_brand_id#178, sum_sales#122, number_sales#123]
+Keys [2]: [channel#68, i_brand_id#178]
+Functions [2]: [partial_sum(sum_sales#122), partial_sum(number_sales#123)]
+Aggregate Attributes [3]: [sum#220, isEmpty#221, sum#222]
+Results [5]: [channel#68, i_brand_id#178, sum#223, isEmpty#224, sum#225]
+
+(150) Exchange
+Input [5]: [channel#68, i_brand_id#178, sum#223, isEmpty#224, sum#225]
+Arguments: hashpartitioning(channel#68, i_brand_id#178, 5), ENSURE_REQUIREMENTS, [id=#226]
+
+(151) HashAggregate [codegen id : 242]
+Input [5]: [channel#68, i_brand_id#178, sum#223, isEmpty#224, sum#225]
+Keys [2]: [channel#68, i_brand_id#178]
+Functions [2]: [sum(sum_sales#122), sum(number_sales#123)]
+Aggregate Attributes [2]: [sum(sum_sales#122)#227, sum(number_sales#123)#228]
+Results [6]: [channel#68, i_brand_id#178, null AS i_class_id#229, null AS i_category_id#230, sum(sum_sales#122)#227 AS sum(sum_sales)#231, sum(number_sales#123)#228 AS sum(number_sales)#232]
+
+(152) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_brand_id#233, i_class_id#234, i_category_id#235, sum#236, isEmpty#237, count#238]
+
+(153) HashAggregate [codegen id : 268]
+Input [6]: [i_brand_id#233, i_class_id#234, i_category_id#235, sum#236, isEmpty#237, count#238]
+Keys [3]: [i_brand_id#233, i_class_id#234, i_category_id#235]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#239 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#240 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#239 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#240 as decimal(12,2)))), DecimalType(18,2), true))#241, count(1)#242]
+Results [6]: [i_brand_id#233, i_class_id#234, i_category_id#235, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#239 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#240 as decimal(12,2)))), DecimalType(18,2), true))#241 AS sales#63, count(1)#242 AS number_sales#64, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#239 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#240 as decimal(12,2)))), DecimalType(18,2), true))#241 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#239 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#240 as decimal(12,2)))), DecimalType(18,2), true))#243]
+
+(154) Filter [codegen id : 268]
+Input [6]: [i_brand_id#233, i_class_id#234, i_category_id#235, sales#63, number_sales#64, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#239 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#240 as decimal(12,2)))), DecimalType(18,2), true))#243]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#239 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#240 as decimal(12,2)))), DecimalType(18,2), true))#243) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#239 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#240 as decimal(12,2)))), DecimalType(18,2), true))#243 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#66, [id=#67] as decimal(32,6))))
+
+(155) Project [codegen id : 268]
+Output [6]: [store AS channel#68, i_brand_id#233, i_class_id#234, i_category_id#235, sales#63, number_sales#64]
+Input [6]: [i_brand_id#233, i_class_id#234, i_category_id#235, sales#63, number_sales#64, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#239 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#240 as decimal(12,2)))), DecimalType(18,2), true))#243]
+
+(156) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_brand_id#244, i_class_id#245, i_category_id#246, sum#247, isEmpty#248, count#249]
+
+(157) HashAggregate [codegen id : 294]
+Input [6]: [i_brand_id#244, i_class_id#245, i_category_id#246, sum#247, isEmpty#248, count#249]
+Keys [3]: [i_brand_id#244, i_class_id#245, i_category_id#246]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#250 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#251 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#250 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#251 as decimal(12,2)))), DecimalType(18,2), true))#252, count(1)#253]
+Results [6]: [i_brand_id#244, i_class_id#245, i_category_id#246, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#250 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#251 as decimal(12,2)))), DecimalType(18,2), true))#252 AS sales#87, count(1)#253 AS number_sales#88, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#250 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#251 as decimal(12,2)))), DecimalType(18,2), true))#252 AS sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#250 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#251 as decimal(12,2)))), DecimalType(18,2), true))#254]
+
+(158) Filter [codegen id : 294]
+Input [6]: [i_brand_id#244, i_class_id#245, i_category_id#246, sales#87, number_sales#88, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#250 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#251 as decimal(12,2)))), DecimalType(18,2), true))#254]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#250 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#251 as decimal(12,2)))), DecimalType(18,2), true))#254) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#250 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#251 as decimal(12,2)))), DecimalType(18,2), true))#254 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#66, [id=#67] as decimal(32,6))))
+
+(159) Project [codegen id : 294]
+Output [6]: [catalog AS channel#90, i_brand_id#244, i_class_id#245, i_category_id#246, sales#87, number_sales#88]
+Input [6]: [i_brand_id#244, i_class_id#245, i_category_id#246, sales#87, number_sales#88, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#250 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#251 as decimal(12,2)))), DecimalType(18,2), true))#254]
+
+(160) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_brand_id#255, i_class_id#256, i_category_id#257, sum#258, isEmpty#259, count#260]
+
+(161) HashAggregate [codegen id : 320]
+Input [6]: [i_brand_id#255, i_class_id#256, i_category_id#257, sum#258, isEmpty#259, count#260]
+Keys [3]: [i_brand_id#255, i_class_id#256, i_category_id#257]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#261 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#262 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#261 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#262 as decimal(12,2)))), DecimalType(18,2), true))#263, count(1)#264]
+Results [6]: [i_brand_id#255, i_class_id#256, i_category_id#257, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#261 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#262 as decimal(12,2)))), DecimalType(18,2), true))#263 AS sales#109, count(1)#264 AS number_sales#110, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#261 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#262 as decimal(12,2)))), DecimalType(18,2), true))#263 AS sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#261 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#262 as decimal(12,2)))), DecimalType(18,2), true))#265]
+
+(162) Filter [codegen id : 320]
+Input [6]: [i_brand_id#255, i_class_id#256, i_category_id#257, sales#109, number_sales#110, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#261 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#262 as decimal(12,2)))), DecimalType(18,2), true))#265]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#261 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#262 as decimal(12,2)))), DecimalType(18,2), true))#265) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#261 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#262 as decimal(12,2)))), DecimalType(18,2), true))#265 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#66, [id=#67] as decimal(32,6))))
+
+(163) Project [codegen id : 320]
+Output [6]: [web AS channel#112, i_brand_id#255, i_class_id#256, i_category_id#257, sales#109, number_sales#110]
+Input [6]: [i_brand_id#255, i_class_id#256, i_category_id#257, sales#109, number_sales#110, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#261 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#262 as decimal(12,2)))), DecimalType(18,2), true))#265]
+
+(164) Union
+
+(165) HashAggregate [codegen id : 321]
+Input [6]: [channel#68, i_brand_id#233, i_class_id#234, i_category_id#235, sales#63, number_sales#64]
+Keys [4]: [channel#68, i_brand_id#233, i_class_id#234, i_category_id#235]
+Functions [2]: [partial_sum(sales#63), partial_sum(number_sales#64)]
+Aggregate Attributes [3]: [sum#266, isEmpty#267, sum#268]
+Results [7]: [channel#68, i_brand_id#233, i_class_id#234, i_category_id#235, sum#269, isEmpty#270, sum#271]
+
+(166) Exchange
+Input [7]: [channel#68, i_brand_id#233, i_class_id#234, i_category_id#235, sum#269, isEmpty#270, sum#271]
+Arguments: hashpartitioning(channel#68, i_brand_id#233, i_class_id#234, i_category_id#235, 5), ENSURE_REQUIREMENTS, [id=#272]
+
+(167) HashAggregate [codegen id : 322]
+Input [7]: [channel#68, i_brand_id#233, i_class_id#234, i_category_id#235, sum#269, isEmpty#270, sum#271]
+Keys [4]: [channel#68, i_brand_id#233, i_class_id#234, i_category_id#235]
+Functions [2]: [sum(sales#63), sum(number_sales#64)]
+Aggregate Attributes [2]: [sum(sales#63)#273, sum(number_sales#64)#274]
+Results [3]: [channel#68, sum(sales#63)#273 AS sum_sales#122, sum(number_sales#64)#274 AS number_sales#123]
+
+(168) HashAggregate [codegen id : 322]
+Input [3]: [channel#68, sum_sales#122, number_sales#123]
+Keys [1]: [channel#68]
+Functions [2]: [partial_sum(sum_sales#122), partial_sum(number_sales#123)]
+Aggregate Attributes [3]: [sum#275, isEmpty#276, sum#277]
+Results [4]: [channel#68, sum#278, isEmpty#279, sum#280]
+
+(169) Exchange
+Input [4]: [channel#68, sum#278, isEmpty#279, sum#280]
+Arguments: hashpartitioning(channel#68, 5), ENSURE_REQUIREMENTS, [id=#281]
+
+(170) HashAggregate [codegen id : 323]
+Input [4]: [channel#68, sum#278, isEmpty#279, sum#280]
+Keys [1]: [channel#68]
+Functions [2]: [sum(sum_sales#122), sum(number_sales#123)]
+Aggregate Attributes [2]: [sum(sum_sales#122)#282, sum(number_sales#123)#283]
+Results [6]: [channel#68, null AS i_brand_id#284, null AS i_class_id#285, null AS i_category_id#286, sum(sum_sales#122)#282 AS sum(sum_sales)#287, sum(number_sales#123)#283 AS sum(number_sales)#288]
+
+(171) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_brand_id#289, i_class_id#290, i_category_id#291, sum#292, isEmpty#293, count#294]
+
+(172) HashAggregate [codegen id : 349]
+Input [6]: [i_brand_id#289, i_class_id#290, i_category_id#291, sum#292, isEmpty#293, count#294]
+Keys [3]: [i_brand_id#289, i_class_id#290, i_category_id#291]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#295 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#296 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#295 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#296 as decimal(12,2)))), DecimalType(18,2), true))#297, count(1)#298]
+Results [6]: [i_brand_id#289, i_class_id#290, i_category_id#291, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#295 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#296 as decimal(12,2)))), DecimalType(18,2), true))#297 AS sales#63, count(1)#298 AS number_sales#64, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#295 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#296 as decimal(12,2)))), DecimalType(18,2), true))#297 AS sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#295 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#296 as decimal(12,2)))), DecimalType(18,2), true))#299]
+
+(173) Filter [codegen id : 349]
+Input [6]: [i_brand_id#289, i_class_id#290, i_category_id#291, sales#63, number_sales#64, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#295 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#296 as decimal(12,2)))), DecimalType(18,2), true))#299]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#295 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#296 as decimal(12,2)))), DecimalType(18,2), true))#299) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#295 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#296 as decimal(12,2)))), DecimalType(18,2), true))#299 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#66, [id=#67] as decimal(32,6))))
+
+(174) Project [codegen id : 349]
+Output [6]: [store AS channel#68, i_brand_id#289, i_class_id#290, i_category_id#291, sales#63, number_sales#64]
+Input [6]: [i_brand_id#289, i_class_id#290, i_category_id#291, sales#63, number_sales#64, sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#295 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price#296 as decimal(12,2)))), DecimalType(18,2), true))#299]
+
+(175) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_brand_id#300, i_class_id#301, i_category_id#302, sum#303, isEmpty#304, count#305]
+
+(176) HashAggregate [codegen id : 375]
+Input [6]: [i_brand_id#300, i_class_id#301, i_category_id#302, sum#303, isEmpty#304, count#305]
+Keys [3]: [i_brand_id#300, i_class_id#301, i_category_id#302]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#306 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#307 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#306 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#307 as decimal(12,2)))), DecimalType(18,2), true))#308, count(1)#309]
+Results [6]: [i_brand_id#300, i_class_id#301, i_category_id#302, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#306 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#307 as decimal(12,2)))), DecimalType(18,2), true))#308 AS sales#87, count(1)#309 AS number_sales#88, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#306 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#307 as decimal(12,2)))), DecimalType(18,2), true))#308 AS sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#306 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#307 as decimal(12,2)))), DecimalType(18,2), true))#310]
+
+(177) Filter [codegen id : 375]
+Input [6]: [i_brand_id#300, i_class_id#301, i_category_id#302, sales#87, number_sales#88, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#306 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#307 as decimal(12,2)))), DecimalType(18,2), true))#310]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#306 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#307 as decimal(12,2)))), DecimalType(18,2), true))#310) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#306 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#307 as decimal(12,2)))), DecimalType(18,2), true))#310 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#66, [id=#67] as decimal(32,6))))
+
+(178) Project [codegen id : 375]
+Output [6]: [catalog AS channel#90, i_brand_id#300, i_class_id#301, i_category_id#302, sales#87, number_sales#88]
+Input [6]: [i_brand_id#300, i_class_id#301, i_category_id#302, sales#87, number_sales#88, sum(CheckOverflow((promote_precision(cast(cast(cs_quantity#306 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price#307 as decimal(12,2)))), DecimalType(18,2), true))#310]
+
+(179) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_brand_id#311, i_class_id#312, i_category_id#313, sum#314, isEmpty#315, count#316]
+
+(180) HashAggregate [codegen id : 401]
+Input [6]: [i_brand_id#311, i_class_id#312, i_category_id#313, sum#314, isEmpty#315, count#316]
+Keys [3]: [i_brand_id#311, i_class_id#312, i_category_id#313]
+Functions [2]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#317 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#318 as decimal(12,2)))), DecimalType(18,2), true)), count(1)]
+Aggregate Attributes [2]: [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#317 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#318 as decimal(12,2)))), DecimalType(18,2), true))#319, count(1)#320]
+Results [6]: [i_brand_id#311, i_class_id#312, i_category_id#313, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#317 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#318 as decimal(12,2)))), DecimalType(18,2), true))#319 AS sales#109, count(1)#320 AS number_sales#110, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#317 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#318 as decimal(12,2)))), DecimalType(18,2), true))#319 AS sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#317 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#318 as decimal(12,2)))), DecimalType(18,2), true))#321]
+
+(181) Filter [codegen id : 401]
+Input [6]: [i_brand_id#311, i_class_id#312, i_category_id#313, sales#109, number_sales#110, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#317 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#318 as decimal(12,2)))), DecimalType(18,2), true))#321]
+Condition : (isnotnull(sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#317 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#318 as decimal(12,2)))), DecimalType(18,2), true))#321) AND (cast(sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#317 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#318 as decimal(12,2)))), DecimalType(18,2), true))#321 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#66, [id=#67] as decimal(32,6))))
+
+(182) Project [codegen id : 401]
+Output [6]: [web AS channel#112, i_brand_id#311, i_class_id#312, i_category_id#313, sales#109, number_sales#110]
+Input [6]: [i_brand_id#311, i_class_id#312, i_category_id#313, sales#109, number_sales#110, sum(CheckOverflow((promote_precision(cast(cast(ws_quantity#317 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price#318 as decimal(12,2)))), DecimalType(18,2), true))#321]
+
+(183) Union
+
+(184) HashAggregate [codegen id : 402]
+Input [6]: [channel#68, i_brand_id#289, i_class_id#290, i_category_id#291, sales#63, number_sales#64]
+Keys [4]: [channel#68, i_brand_id#289, i_class_id#290, i_category_id#291]
+Functions [2]: [partial_sum(sales#63), partial_sum(number_sales#64)]
+Aggregate Attributes [3]: [sum#322, isEmpty#323, sum#324]
+Results [7]: [channel#68, i_brand_id#289, i_class_id#290, i_category_id#291, sum#325, isEmpty#326, sum#327]
+
+(185) Exchange
+Input [7]: [channel#68, i_brand_id#289, i_class_id#290, i_category_id#291, sum#325, isEmpty#326, sum#327]
+Arguments: hashpartitioning(channel#68, i_brand_id#289, i_class_id#290, i_category_id#291, 5), ENSURE_REQUIREMENTS, [id=#328]
+
+(186) HashAggregate [codegen id : 403]
+Input [7]: [channel#68, i_brand_id#289, i_class_id#290, i_category_id#291, sum#325, isEmpty#326, sum#327]
+Keys [4]: [channel#68, i_brand_id#289, i_class_id#290, i_category_id#291]
+Functions [2]: [sum(sales#63), sum(number_sales#64)]
+Aggregate Attributes [2]: [sum(sales#63)#329, sum(number_sales#64)#330]
+Results [2]: [sum(sales#63)#329 AS sum_sales#122, sum(number_sales#64)#330 AS number_sales#123]
+
+(187) HashAggregate [codegen id : 403]
+Input [2]: [sum_sales#122, number_sales#123]
+Keys: []
+Functions [2]: [partial_sum(sum_sales#122), partial_sum(number_sales#123)]
+Aggregate Attributes [3]: [sum#331, isEmpty#332, sum#333]
+Results [3]: [sum#334, isEmpty#335, sum#336]
+
+(188) Exchange
+Input [3]: [sum#334, isEmpty#335, sum#336]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#337]
+
+(189) HashAggregate [codegen id : 404]
+Input [3]: [sum#334, isEmpty#335, sum#336]
+Keys: []
+Functions [2]: [sum(sum_sales#122), sum(number_sales#123)]
+Aggregate Attributes [2]: [sum(sum_sales#122)#338, sum(number_sales#123)#339]
+Results [6]: [null AS channel#340, null AS i_brand_id#341, null AS i_class_id#342, null AS i_category_id#343, sum(sum_sales#122)#338 AS sum(sum_sales)#344, sum(number_sales#123)#339 AS sum(number_sales)#345]
+
+(190) Union
+
+(191) HashAggregate [codegen id : 405]
+Input [6]: [channel#68, i_brand_id#46, i_class_id#47, i_category_id#48, sum_sales#122, number_sales#123]
+Keys [6]: [channel#68, i_brand_id#46, i_class_id#47, i_category_id#48, sum_sales#122, number_sales#123]
+Functions: []
+Aggregate Attributes: []
+Results [6]: [channel#68, i_brand_id#46, i_class_id#47, i_category_id#48, sum_sales#122, number_sales#123]
+
+(192) Exchange
+Input [6]: [channel#68, i_brand_id#46, i_class_id#47, i_category_id#48, sum_sales#122, number_sales#123]
+Arguments: hashpartitioning(channel#68, i_brand_id#46, i_class_id#47, i_category_id#48, sum_sales#122, number_sales#123, 5), ENSURE_REQUIREMENTS, [id=#346]
+
+(193) HashAggregate [codegen id : 406]
+Input [6]: [channel#68, i_brand_id#46, i_class_id#47, i_category_id#48, sum_sales#122, number_sales#123]
+Keys [6]: [channel#68, i_brand_id#46, i_class_id#47, i_category_id#48, sum_sales#122, number_sales#123]
+Functions: []
+Aggregate Attributes: []
+Results [6]: [channel#68, i_brand_id#46, i_class_id#47, i_category_id#48, sum_sales#122, number_sales#123]
+
+(194) TakeOrderedAndProject
+Input [6]: [channel#68, i_brand_id#46, i_class_id#47, i_category_id#48, sum_sales#122, number_sales#123]
+Arguments: 100, [channel#68 ASC NULLS FIRST, i_brand_id#46 ASC NULLS FIRST, i_class_id#47 ASC NULLS FIRST, i_category_id#48 ASC NULLS FIRST], [channel#68, i_brand_id#46, i_class_id#47, i_category_id#48, sum_sales#122, number_sales#123]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 76 Hosting Expression = Subquery scalar-subquery#66, [id=#67]
+* HashAggregate (221)
++- Exchange (220)
+   +- * HashAggregate (219)
+      +- Union (218)
+         :- * Project (203)
+         :  +- * BroadcastHashJoin Inner BuildRight (202)
+         :     :- * ColumnarToRow (196)
+         :     :  +- Scan parquet default.store_sales (195)
+         :     +- BroadcastExchange (201)
+         :        +- * Project (200)
+         :           +- * Filter (199)
+         :              +- * ColumnarToRow (198)
+         :                 +- Scan parquet default.date_dim (197)
+         :- * Project (212)
+         :  +- * BroadcastHashJoin Inner BuildRight (211)
+         :     :- * ColumnarToRow (205)
+         :     :  +- Scan parquet default.catalog_sales (204)
+         :     +- BroadcastExchange (210)
+         :        +- * Project (209)
+         :           +- * Filter (208)
+         :              +- * ColumnarToRow (207)
+         :                 +- Scan parquet default.date_dim (206)
+         +- * Project (217)
+            +- * BroadcastHashJoin Inner BuildRight (216)
+               :- * ColumnarToRow (214)
+               :  +- Scan parquet default.web_sales (213)
+               +- ReusedExchange (215)
+
+
+(195) Scan parquet default.store_sales
+Output [3]: [ss_quantity#347, ss_list_price#348, ss_sold_date_sk#349]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#349), dynamicpruningexpression(ss_sold_date_sk#349 IN dynamicpruning#350)]
+ReadSchema: struct<ss_quantity:int,ss_list_price:decimal(7,2)>
+
+(196) ColumnarToRow [codegen id : 2]
+Input [3]: [ss_quantity#347, ss_list_price#348, ss_sold_date_sk#349]
+
+(197) Scan parquet default.date_dim
+Output [2]: [d_date_sk#351, d_year#352]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(198) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#351, d_year#352]
+
+(199) Filter [codegen id : 1]
+Input [2]: [d_date_sk#351, d_year#352]
+Condition : (((isnotnull(d_year#352) AND (d_year#352 >= 1999)) AND (d_year#352 <= 2001)) AND isnotnull(d_date_sk#351))
+
+(200) Project [codegen id : 1]
+Output [1]: [d_date_sk#351]
+Input [2]: [d_date_sk#351, d_year#352]
+
+(201) BroadcastExchange
+Input [1]: [d_date_sk#351]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#353]
+
+(202) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#349]
+Right keys [1]: [d_date_sk#351]
+Join condition: None
+
+(203) Project [codegen id : 2]
+Output [2]: [ss_quantity#347 AS quantity#354, ss_list_price#348 AS list_price#355]
+Input [4]: [ss_quantity#347, ss_list_price#348, ss_sold_date_sk#349, d_date_sk#351]
+
+(204) Scan parquet default.catalog_sales
+Output [3]: [cs_quantity#356, cs_list_price#357, cs_sold_date_sk#358]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#358), dynamicpruningexpression(cs_sold_date_sk#358 IN dynamicpruning#359)]
+ReadSchema: struct<cs_quantity:int,cs_list_price:decimal(7,2)>
+
+(205) ColumnarToRow [codegen id : 4]
+Input [3]: [cs_quantity#356, cs_list_price#357, cs_sold_date_sk#358]
+
+(206) Scan parquet default.date_dim
+Output [2]: [d_date_sk#360, d_year#361]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(207) ColumnarToRow [codegen id : 3]
+Input [2]: [d_date_sk#360, d_year#361]
+
+(208) Filter [codegen id : 3]
+Input [2]: [d_date_sk#360, d_year#361]
+Condition : (((isnotnull(d_year#361) AND (d_year#361 >= 1998)) AND (d_year#361 <= 2000)) AND isnotnull(d_date_sk#360))
+
+(209) Project [codegen id : 3]
+Output [1]: [d_date_sk#360]
+Input [2]: [d_date_sk#360, d_year#361]
+
+(210) BroadcastExchange
+Input [1]: [d_date_sk#360]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#362]
+
+(211) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_sold_date_sk#358]
+Right keys [1]: [d_date_sk#360]
+Join condition: None
+
+(212) Project [codegen id : 4]
+Output [2]: [cs_quantity#356 AS quantity#363, cs_list_price#357 AS list_price#364]
+Input [4]: [cs_quantity#356, cs_list_price#357, cs_sold_date_sk#358, d_date_sk#360]
+
+(213) Scan parquet default.web_sales
+Output [3]: [ws_quantity#365, ws_list_price#366, ws_sold_date_sk#367]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#367), dynamicpruningexpression(ws_sold_date_sk#367 IN dynamicpruning#359)]
+ReadSchema: struct<ws_quantity:int,ws_list_price:decimal(7,2)>
+
+(214) ColumnarToRow [codegen id : 6]
+Input [3]: [ws_quantity#365, ws_list_price#366, ws_sold_date_sk#367]
+
+(215) ReusedExchange [Reuses operator id: 210]
+Output [1]: [d_date_sk#368]
+
+(216) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ws_sold_date_sk#367]
+Right keys [1]: [d_date_sk#368]
+Join condition: None
+
+(217) Project [codegen id : 6]
+Output [2]: [ws_quantity#365 AS quantity#369, ws_list_price#366 AS list_price#370]
+Input [4]: [ws_quantity#365, ws_list_price#366, ws_sold_date_sk#367, d_date_sk#368]
+
+(218) Union
+
+(219) HashAggregate [codegen id : 7]
+Input [2]: [quantity#354, list_price#355]
+Keys: []
+Functions [1]: [partial_avg(CheckOverflow((promote_precision(cast(cast(quantity#354 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#355 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [2]: [sum#371, count#372]
+Results [2]: [sum#373, count#374]
+
+(220) Exchange
+Input [2]: [sum#373, count#374]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#375]
+
+(221) HashAggregate [codegen id : 8]
+Input [2]: [sum#373, count#374]
+Keys: []
+Functions [1]: [avg(CheckOverflow((promote_precision(cast(cast(quantity#354 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#355 as decimal(12,2)))), DecimalType(18,2), true))]
+Aggregate Attributes [1]: [avg(CheckOverflow((promote_precision(cast(cast(quantity#354 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#355 as decimal(12,2)))), DecimalType(18,2), true))#376]
+Results [1]: [avg(CheckOverflow((promote_precision(cast(cast(quantity#354 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price#355 as decimal(12,2)))), DecimalType(18,2), true))#376 AS average_sales#377]
+
+Subquery:2 Hosting operator id = 195 Hosting Expression = ss_sold_date_sk#349 IN dynamicpruning#350
+ReusedExchange (222)
+
+
+(222) ReusedExchange [Reuses operator id: 201]
+Output [1]: [d_date_sk#351]
+
+Subquery:3 Hosting operator id = 204 Hosting Expression = cs_sold_date_sk#358 IN dynamicpruning#359
+ReusedExchange (223)
+
+
+(223) ReusedExchange [Reuses operator id: 210]
+Output [1]: [d_date_sk#360]
+
+Subquery:4 Hosting operator id = 213 Hosting Expression = ws_sold_date_sk#367 IN dynamicpruning#359
+
+Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (224)
+
+
+(224) ReusedExchange [Reuses operator id: 70]
+Output [1]: [d_date_sk#50]
+
+Subquery:6 Hosting operator id = 7 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12
+ReusedExchange (225)
+
+
+(225) ReusedExchange [Reuses operator id: 26]
+Output [1]: [d_date_sk#29]
+
+Subquery:7 Hosting operator id = 13 Hosting Expression = cs_sold_date_sk#18 IN dynamicpruning#12
+
+Subquery:8 Hosting operator id = 40 Hosting Expression = ws_sold_date_sk#35 IN dynamicpruning#12
+
+Subquery:9 Hosting operator id = 92 Hosting Expression = ReusedSubquery Subquery scalar-subquery#66, [id=#67]
+
+Subquery:10 Hosting operator id = 78 Hosting Expression = cs_sold_date_sk#72 IN dynamicpruning#5
+
+Subquery:11 Hosting operator id = 108 Hosting Expression = ReusedSubquery Subquery scalar-subquery#66, [id=#67]
+
+Subquery:12 Hosting operator id = 94 Hosting Expression = ws_sold_date_sk#94 IN dynamicpruning#5
+
+Subquery:13 Hosting operator id = 116 Hosting Expression = ReusedSubquery Subquery scalar-subquery#66, [id=#67]
+
+Subquery:14 Hosting operator id = 120 Hosting Expression = ReusedSubquery Subquery scalar-subquery#66, [id=#67]
+
+Subquery:15 Hosting operator id = 124 Hosting Expression = ReusedSubquery Subquery scalar-subquery#66, [id=#67]
+
+Subquery:16 Hosting operator id = 135 Hosting Expression = ReusedSubquery Subquery scalar-subquery#66, [id=#67]
+
+Subquery:17 Hosting operator id = 139 Hosting Expression = ReusedSubquery Subquery scalar-subquery#66, [id=#67]
+
+Subquery:18 Hosting operator id = 143 Hosting Expression = ReusedSubquery Subquery scalar-subquery#66, [id=#67]
+
+Subquery:19 Hosting operator id = 154 Hosting Expression = ReusedSubquery Subquery scalar-subquery#66, [id=#67]
+
+Subquery:20 Hosting operator id = 158 Hosting Expression = ReusedSubquery Subquery scalar-subquery#66, [id=#67]
+
+Subquery:21 Hosting operator id = 162 Hosting Expression = ReusedSubquery Subquery scalar-subquery#66, [id=#67]
+
+Subquery:22 Hosting operator id = 173 Hosting Expression = ReusedSubquery Subquery scalar-subquery#66, [id=#67]
+
+Subquery:23 Hosting operator id = 177 Hosting Expression = ReusedSubquery Subquery scalar-subquery#66, [id=#67]
+
+Subquery:24 Hosting operator id = 181 Hosting Expression = ReusedSubquery Subquery scalar-subquery#66, [id=#67]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/simplified.txt
new file mode 100644
index 0000000000000..e351f9e687027
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/simplified.txt
@@ -0,0 +1,373 @@
+TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales]
+  WholeStageCodegen (406)
+    HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales]
+      InputAdapter
+        Exchange [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] #1
+          WholeStageCodegen (405)
+            HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales]
+              InputAdapter
+                Union
+                  WholeStageCodegen (80)
+                    HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum]
+                      InputAdapter
+                        Exchange [channel,i_brand_id,i_class_id,i_category_id] #2
+                          WholeStageCodegen (79)
+                            HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum]
+                              InputAdapter
+                                Union
+                                  WholeStageCodegen (26)
+                                    Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                      Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                        Subquery #3
+                                          WholeStageCodegen (8)
+                                            HashAggregate [sum,count] [avg(CheckOverflow((promote_precision(cast(cast(quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(list_price as decimal(12,2)))), DecimalType(18,2), true)),average_sales,sum,count]
+                                              InputAdapter
+                                                Exchange #15
+                                                  WholeStageCodegen (7)
+                                                    HashAggregate [quantity,list_price] [sum,count,sum,count]
+                                                      InputAdapter
+                                                        Union
+                                                          WholeStageCodegen (2)
+                                                            Project [ss_quantity,ss_list_price]
+                                                              BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk]
+                                                                      SubqueryBroadcast [d_date_sk] #4
+                                                                        ReusedExchange [d_date_sk] #16
+                                                                InputAdapter
+                                                                  BroadcastExchange #16
+                                                                    WholeStageCodegen (1)
+                                                                      Project [d_date_sk]
+                                                                        Filter [d_year,d_date_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.date_dim [d_date_sk,d_year]
+                                                          WholeStageCodegen (4)
+                                                            Project [cs_quantity,cs_list_price]
+                                                              BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk]
+                                                                      SubqueryBroadcast [d_date_sk] #5
+                                                                        ReusedExchange [d_date_sk] #17
+                                                                InputAdapter
+                                                                  BroadcastExchange #17
+                                                                    WholeStageCodegen (3)
+                                                                      Project [d_date_sk]
+                                                                        Filter [d_year,d_date_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.date_dim [d_date_sk,d_year]
+                                                          WholeStageCodegen (6)
+                                                            Project [ws_quantity,ws_list_price]
+                                                              BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk]
+                                                                      ReusedSubquery [d_date_sk] #5
+                                                                InputAdapter
+                                                                  ReusedExchange [d_date_sk] #17
+                                        HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                          InputAdapter
+                                            Exchange [i_brand_id,i_class_id,i_category_id] #3
+                                              WholeStageCodegen (25)
+                                                HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count]
+                                                  Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id]
+                                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                      Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id]
+                                                        BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                          BroadcastHashJoin [ss_item_sk,ss_item_sk]
+                                                            Filter [ss_item_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk]
+                                                                    SubqueryBroadcast [d_date_sk] #1
+                                                                      ReusedExchange [d_date_sk] #4
+                                                            InputAdapter
+                                                              BroadcastExchange #5
+                                                                WholeStageCodegen (11)
+                                                                  Project [i_item_sk]
+                                                                    BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id]
+                                                                      Filter [i_brand_id,i_class_id,i_category_id]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                      InputAdapter
+                                                                        BroadcastExchange #6
+                                                                          WholeStageCodegen (10)
+                                                                            HashAggregate [brand_id,class_id,category_id]
+                                                                              HashAggregate [brand_id,class_id,category_id]
+                                                                                BroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id]
+                                                                                  HashAggregate [brand_id,class_id,category_id]
+                                                                                    InputAdapter
+                                                                                      Exchange [brand_id,class_id,category_id] #7
+                                                                                        WholeStageCodegen (6)
+                                                                                          HashAggregate [brand_id,class_id,category_id]
+                                                                                            Project [i_brand_id,i_class_id,i_category_id]
+                                                                                              BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                                Project [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                                  BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                                                                    Filter [ss_item_sk]
+                                                                                                      ColumnarToRow
+                                                                                                        InputAdapter
+                                                                                                          Scan parquet default.store_sales [ss_item_sk,ss_sold_date_sk]
+                                                                                                            SubqueryBroadcast [d_date_sk] #2
+                                                                                                              ReusedExchange [d_date_sk] #8
+                                                                                                    InputAdapter
+                                                                                                      BroadcastExchange #9
+                                                                                                        WholeStageCodegen (4)
+                                                                                                          BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id]
+                                                                                                            Filter [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                                              ColumnarToRow
+                                                                                                                InputAdapter
+                                                                                                                  Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                                            InputAdapter
+                                                                                                              BroadcastExchange #10
+                                                                                                                WholeStageCodegen (3)
+                                                                                                                  Project [i_brand_id,i_class_id,i_category_id]
+                                                                                                                    BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                                                                      Project [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                                                        BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                                                                                                          Filter [cs_item_sk]
+                                                                                                                            ColumnarToRow
+                                                                                                                              InputAdapter
+                                                                                                                                Scan parquet default.catalog_sales [cs_item_sk,cs_sold_date_sk]
+                                                                                                                                  ReusedSubquery [d_date_sk] #2
+                                                                                                                          InputAdapter
+                                                                                                                            BroadcastExchange #11
+                                                                                                                              WholeStageCodegen (1)
+                                                                                                                                Filter [i_item_sk]
+                                                                                                                                  ColumnarToRow
+                                                                                                                                    InputAdapter
+                                                                                                                                      Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                                                      InputAdapter
+                                                                                                                        BroadcastExchange #8
+                                                                                                                          WholeStageCodegen (2)
+                                                                                                                            Project [d_date_sk]
+                                                                                                                              Filter [d_year,d_date_sk]
+                                                                                                                                ColumnarToRow
+                                                                                                                                  InputAdapter
+                                                                                                                                    Scan parquet default.date_dim [d_date_sk,d_year]
+                                                                                                InputAdapter
+                                                                                                  ReusedExchange [d_date_sk] #8
+                                                                                  InputAdapter
+                                                                                    BroadcastExchange #12
+                                                                                      WholeStageCodegen (9)
+                                                                                        Project [i_brand_id,i_class_id,i_category_id]
+                                                                                          BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                                            Project [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id]
+                                                                                              BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                                                                                Filter [ws_item_sk]
+                                                                                                  ColumnarToRow
+                                                                                                    InputAdapter
+                                                                                                      Scan parquet default.web_sales [ws_item_sk,ws_sold_date_sk]
+                                                                                                        ReusedSubquery [d_date_sk] #2
+                                                                                                InputAdapter
+                                                                                                  ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11
+                                                                                            InputAdapter
+                                                                                              ReusedExchange [d_date_sk] #8
+                                                          InputAdapter
+                                                            BroadcastExchange #13
+                                                              WholeStageCodegen (23)
+                                                                BroadcastHashJoin [i_item_sk,ss_item_sk]
+                                                                  Filter [i_item_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id]
+                                                                  InputAdapter
+                                                                    ReusedExchange [ss_item_sk] #14
+                                                      InputAdapter
+                                                        BroadcastExchange #4
+                                                          WholeStageCodegen (24)
+                                                            Project [d_date_sk]
+                                                              Filter [d_year,d_moy,d_date_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                  WholeStageCodegen (52)
+                                    Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                      Filter [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                        ReusedSubquery [average_sales] #3
+                                        HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                          InputAdapter
+                                            Exchange [i_brand_id,i_class_id,i_category_id] #18
+                                              WholeStageCodegen (51)
+                                                HashAggregate [i_brand_id,i_class_id,i_category_id,cs_quantity,cs_list_price] [sum,isEmpty,count,sum,isEmpty,count]
+                                                  Project [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id]
+                                                    BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                      Project [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id]
+                                                        BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                                          BroadcastHashJoin [cs_item_sk,ss_item_sk]
+                                                            Filter [cs_item_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk]
+                                                                    ReusedSubquery [d_date_sk] #1
+                                                            InputAdapter
+                                                              ReusedExchange [ss_item_sk] #14
+                                                          InputAdapter
+                                                            ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13
+                                                      InputAdapter
+                                                        ReusedExchange [d_date_sk] #4
+                                  WholeStageCodegen (78)
+                                    Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                      Filter [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                        ReusedSubquery [average_sales] #3
+                                        HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                          InputAdapter
+                                            Exchange [i_brand_id,i_class_id,i_category_id] #19
+                                              WholeStageCodegen (77)
+                                                HashAggregate [i_brand_id,i_class_id,i_category_id,ws_quantity,ws_list_price] [sum,isEmpty,count,sum,isEmpty,count]
+                                                  Project [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id]
+                                                    BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                      Project [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id]
+                                                        BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                                          BroadcastHashJoin [ws_item_sk,ss_item_sk]
+                                                            Filter [ws_item_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk]
+                                                                    ReusedSubquery [d_date_sk] #1
+                                                            InputAdapter
+                                                              ReusedExchange [ss_item_sk] #14
+                                                          InputAdapter
+                                                            ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13
+                                                      InputAdapter
+                                                        ReusedExchange [d_date_sk] #4
+                  WholeStageCodegen (161)
+                    HashAggregate [channel,i_brand_id,i_class_id,sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum]
+                      InputAdapter
+                        Exchange [channel,i_brand_id,i_class_id] #20
+                          WholeStageCodegen (160)
+                            HashAggregate [channel,i_brand_id,i_class_id,sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum]
+                              HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum]
+                                InputAdapter
+                                  Exchange [channel,i_brand_id,i_class_id,i_category_id] #21
+                                    WholeStageCodegen (159)
+                                      HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum]
+                                        InputAdapter
+                                          Union
+                                            WholeStageCodegen (106)
+                                              Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                                Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                                  ReusedSubquery [average_sales] #3
+                                                  HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                                    InputAdapter
+                                                      ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #22
+                                            WholeStageCodegen (132)
+                                              Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                                Filter [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                                  ReusedSubquery [average_sales] #3
+                                                  HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                                    InputAdapter
+                                                      ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #23
+                                            WholeStageCodegen (158)
+                                              Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                                Filter [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                                  ReusedSubquery [average_sales] #3
+                                                  HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                                    InputAdapter
+                                                      ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #24
+                  WholeStageCodegen (242)
+                    HashAggregate [channel,i_brand_id,sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum]
+                      InputAdapter
+                        Exchange [channel,i_brand_id] #25
+                          WholeStageCodegen (241)
+                            HashAggregate [channel,i_brand_id,sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum]
+                              HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum]
+                                InputAdapter
+                                  Exchange [channel,i_brand_id,i_class_id,i_category_id] #26
+                                    WholeStageCodegen (240)
+                                      HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum]
+                                        InputAdapter
+                                          Union
+                                            WholeStageCodegen (187)
+                                              Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                                Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                                  ReusedSubquery [average_sales] #3
+                                                  HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                                    InputAdapter
+                                                      ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #22
+                                            WholeStageCodegen (213)
+                                              Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                                Filter [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                                  ReusedSubquery [average_sales] #3
+                                                  HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                                    InputAdapter
+                                                      ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #23
+                                            WholeStageCodegen (239)
+                                              Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                                Filter [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                                  ReusedSubquery [average_sales] #3
+                                                  HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                                    InputAdapter
+                                                      ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #24
+                  WholeStageCodegen (323)
+                    HashAggregate [channel,sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum]
+                      InputAdapter
+                        Exchange [channel] #27
+                          WholeStageCodegen (322)
+                            HashAggregate [channel,sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum]
+                              HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum]
+                                InputAdapter
+                                  Exchange [channel,i_brand_id,i_class_id,i_category_id] #28
+                                    WholeStageCodegen (321)
+                                      HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum]
+                                        InputAdapter
+                                          Union
+                                            WholeStageCodegen (268)
+                                              Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                                Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                                  ReusedSubquery [average_sales] #3
+                                                  HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                                    InputAdapter
+                                                      ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #22
+                                            WholeStageCodegen (294)
+                                              Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                                Filter [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                                  ReusedSubquery [average_sales] #3
+                                                  HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                                    InputAdapter
+                                                      ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #23
+                                            WholeStageCodegen (320)
+                                              Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                                Filter [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                                  ReusedSubquery [average_sales] #3
+                                                  HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                                    InputAdapter
+                                                      ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #24
+                  WholeStageCodegen (404)
+                    HashAggregate [sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum]
+                      InputAdapter
+                        Exchange #29
+                          WholeStageCodegen (403)
+                            HashAggregate [sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum]
+                              HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum]
+                                InputAdapter
+                                  Exchange [channel,i_brand_id,i_class_id,i_category_id] #30
+                                    WholeStageCodegen (402)
+                                      HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum]
+                                        InputAdapter
+                                          Union
+                                            WholeStageCodegen (349)
+                                              Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                                Filter [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                                  ReusedSubquery [average_sales] #3
+                                                  HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                                    InputAdapter
+                                                      ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #22
+                                            WholeStageCodegen (375)
+                                              Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                                Filter [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                                  ReusedSubquery [average_sales] #3
+                                                  HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(cs_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(cs_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                                    InputAdapter
+                                                      ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #23
+                                            WholeStageCodegen (401)
+                                              Project [i_brand_id,i_class_id,i_category_id,sales,number_sales]
+                                                Filter [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true))]
+                                                  ReusedSubquery [average_sales] #3
+                                                  HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),count(1),sales,number_sales,sum(CheckOverflow((promote_precision(cast(cast(ws_quantity as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ws_list_price as decimal(12,2)))), DecimalType(18,2), true)),sum,isEmpty,count]
+                                                    InputAdapter
+                                                      ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #24
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.sf100/explain.txt
new file mode 100644
index 0000000000000..35e3304de7082
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.sf100/explain.txt
@@ -0,0 +1,894 @@
+== Physical Plan ==
+TakeOrderedAndProject (160)
++- Union (159)
+   :- * HashAggregate (51)
+   :  +- Exchange (50)
+   :     +- * HashAggregate (49)
+   :        +- * Project (48)
+   :           +- * SortMergeJoin Inner (47)
+   :              :- * Sort (25)
+   :              :  +- Exchange (24)
+   :              :     +- * Project (23)
+   :              :        +- * BroadcastHashJoin Inner BuildRight (22)
+   :              :           :- * Project (17)
+   :              :           :  +- * BroadcastHashJoin Inner BuildRight (16)
+   :              :           :     :- * Project (10)
+   :              :           :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+   :              :           :     :     :- * Filter (3)
+   :              :           :     :     :  +- * ColumnarToRow (2)
+   :              :           :     :     :     +- Scan parquet default.catalog_sales (1)
+   :              :           :     :     +- BroadcastExchange (8)
+   :              :           :     :        +- * Project (7)
+   :              :           :     :           +- * Filter (6)
+   :              :           :     :              +- * ColumnarToRow (5)
+   :              :           :     :                 +- Scan parquet default.customer_demographics (4)
+   :              :           :     +- BroadcastExchange (15)
+   :              :           :        +- * Project (14)
+   :              :           :           +- * Filter (13)
+   :              :           :              +- * ColumnarToRow (12)
+   :              :           :                 +- Scan parquet default.date_dim (11)
+   :              :           +- BroadcastExchange (21)
+   :              :              +- * Filter (20)
+   :              :                 +- * ColumnarToRow (19)
+   :              :                    +- Scan parquet default.item (18)
+   :              +- * Sort (46)
+   :                 +- Exchange (45)
+   :                    +- * Project (44)
+   :                       +- * SortMergeJoin Inner (43)
+   :                          :- * Sort (37)
+   :                          :  +- Exchange (36)
+   :                          :     +- * Project (35)
+   :                          :        +- * BroadcastHashJoin Inner BuildRight (34)
+   :                          :           :- * Project (29)
+   :                          :           :  +- * Filter (28)
+   :                          :           :     +- * ColumnarToRow (27)
+   :                          :           :        +- Scan parquet default.customer (26)
+   :                          :           +- BroadcastExchange (33)
+   :                          :              +- * Filter (32)
+   :                          :                 +- * ColumnarToRow (31)
+   :                          :                    +- Scan parquet default.customer_address (30)
+   :                          +- * Sort (42)
+   :                             +- Exchange (41)
+   :                                +- * Filter (40)
+   :                                   +- * ColumnarToRow (39)
+   :                                      +- Scan parquet default.customer_demographics (38)
+   :- * HashAggregate (76)
+   :  +- Exchange (75)
+   :     +- * HashAggregate (74)
+   :        +- * Project (73)
+   :           +- * SortMergeJoin Inner (72)
+   :              :- * Sort (53)
+   :              :  +- ReusedExchange (52)
+   :              +- * Sort (71)
+   :                 +- Exchange (70)
+   :                    +- * Project (69)
+   :                       +- * SortMergeJoin Inner (68)
+   :                          :- * Sort (65)
+   :                          :  +- Exchange (64)
+   :                          :     +- * Project (63)
+   :                          :        +- * BroadcastHashJoin Inner BuildRight (62)
+   :                          :           :- * Project (57)
+   :                          :           :  +- * Filter (56)
+   :                          :           :     +- * ColumnarToRow (55)
+   :                          :           :        +- Scan parquet default.customer (54)
+   :                          :           +- BroadcastExchange (61)
+   :                          :              +- * Filter (60)
+   :                          :                 +- * ColumnarToRow (59)
+   :                          :                    +- Scan parquet default.customer_address (58)
+   :                          +- * Sort (67)
+   :                             +- ReusedExchange (66)
+   :- * HashAggregate (102)
+   :  +- Exchange (101)
+   :     +- * HashAggregate (100)
+   :        +- * Project (99)
+   :           +- * SortMergeJoin Inner (98)
+   :              :- * Sort (78)
+   :              :  +- ReusedExchange (77)
+   :              +- * Sort (97)
+   :                 +- Exchange (96)
+   :                    +- * Project (95)
+   :                       +- * SortMergeJoin Inner (94)
+   :                          :- * Sort (91)
+   :                          :  +- Exchange (90)
+   :                          :     +- * Project (89)
+   :                          :        +- * BroadcastHashJoin Inner BuildRight (88)
+   :                          :           :- * Project (82)
+   :                          :           :  +- * Filter (81)
+   :                          :           :     +- * ColumnarToRow (80)
+   :                          :           :        +- Scan parquet default.customer (79)
+   :                          :           +- BroadcastExchange (87)
+   :                          :              +- * Project (86)
+   :                          :                 +- * Filter (85)
+   :                          :                    +- * ColumnarToRow (84)
+   :                          :                       +- Scan parquet default.customer_address (83)
+   :                          +- * Sort (93)
+   :                             +- ReusedExchange (92)
+   :- * HashAggregate (137)
+   :  +- Exchange (136)
+   :     +- * HashAggregate (135)
+   :        +- * Project (134)
+   :           +- * BroadcastHashJoin Inner BuildRight (133)
+   :              :- * Project (131)
+   :              :  +- * BroadcastHashJoin Inner BuildRight (130)
+   :              :     :- * Project (111)
+   :              :     :  +- * BroadcastHashJoin Inner BuildRight (110)
+   :              :     :     :- * Project (108)
+   :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (107)
+   :              :     :     :     :- * Filter (105)
+   :              :     :     :     :  +- * ColumnarToRow (104)
+   :              :     :     :     :     +- Scan parquet default.catalog_sales (103)
+   :              :     :     :     +- ReusedExchange (106)
+   :              :     :     +- ReusedExchange (109)
+   :              :     +- BroadcastExchange (129)
+   :              :        +- * Project (128)
+   :              :           +- * BroadcastHashJoin Inner BuildLeft (127)
+   :              :              :- BroadcastExchange (123)
+   :              :              :  +- * Project (122)
+   :              :              :     +- * BroadcastHashJoin Inner BuildRight (121)
+   :              :              :        :- * Project (115)
+   :              :              :        :  +- * Filter (114)
+   :              :              :        :     +- * ColumnarToRow (113)
+   :              :              :        :        +- Scan parquet default.customer (112)
+   :              :              :        +- BroadcastExchange (120)
+   :              :              :           +- * Project (119)
+   :              :              :              +- * Filter (118)
+   :              :              :                 +- * ColumnarToRow (117)
+   :              :              :                    +- Scan parquet default.customer_address (116)
+   :              :              +- * Filter (126)
+   :              :                 +- * ColumnarToRow (125)
+   :              :                    +- Scan parquet default.customer_demographics (124)
+   :              +- ReusedExchange (132)
+   +- * HashAggregate (158)
+      +- Exchange (157)
+         +- * HashAggregate (156)
+            +- * Project (155)
+               +- * BroadcastHashJoin Inner BuildRight (154)
+                  :- * Project (152)
+                  :  +- * BroadcastHashJoin Inner BuildRight (151)
+                  :     :- * Project (146)
+                  :     :  +- * BroadcastHashJoin Inner BuildRight (145)
+                  :     :     :- * Project (143)
+                  :     :     :  +- * BroadcastHashJoin Inner BuildRight (142)
+                  :     :     :     :- * Filter (140)
+                  :     :     :     :  +- * ColumnarToRow (139)
+                  :     :     :     :     +- Scan parquet default.catalog_sales (138)
+                  :     :     :     +- ReusedExchange (141)
+                  :     :     +- ReusedExchange (144)
+                  :     +- BroadcastExchange (150)
+                  :        +- * Filter (149)
+                  :           +- * ColumnarToRow (148)
+                  :              +- Scan parquet default.item (147)
+                  +- ReusedExchange (153)
+
+
+(1) Scan parquet default.catalog_sales
+Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#9), dynamicpruningexpression(cs_sold_date_sk#9 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_bill_cdemo_sk:int,cs_item_sk:int,cs_quantity:int,cs_list_price:decimal(7,2),cs_sales_price:decimal(7,2),cs_coupon_amt:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9]
+
+(3) Filter [codegen id : 4]
+Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9]
+Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3))
+
+(4) Scan parquet default.customer_demographics
+Output [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_education_status,College             ), IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_education_status:string,cd_dep_count:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14]
+
+(6) Filter [codegen id : 1]
+Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14]
+Condition : ((((isnotnull(cd_gender#12) AND isnotnull(cd_education_status#13)) AND (cd_gender#12 = M)) AND (cd_education_status#13 = College             )) AND isnotnull(cd_demo_sk#11))
+
+(7) Project [codegen id : 1]
+Output [2]: [cd_demo_sk#11, cd_dep_count#14]
+Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14]
+
+(8) BroadcastExchange
+Input [2]: [cd_demo_sk#11, cd_dep_count#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#15]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_bill_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#11]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14]
+Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#11, cd_dep_count#14]
+
+(11) Scan parquet default.date_dim
+Output [2]: [d_date_sk#16, d_year#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#16, d_year#17]
+
+(13) Filter [codegen id : 2]
+Input [2]: [d_date_sk#16, d_year#17]
+Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16))
+
+(14) Project [codegen id : 2]
+Output [1]: [d_date_sk#16]
+Input [2]: [d_date_sk#16, d_year#17]
+
+(15) BroadcastExchange
+Input [1]: [d_date_sk#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_sold_date_sk#9]
+Right keys [1]: [d_date_sk#16]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [8]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14]
+Input [10]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, d_date_sk#16]
+
+(18) Scan parquet default.item
+Output [2]: [i_item_sk#19, i_item_id#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [2]: [i_item_sk#19, i_item_id#20]
+
+(20) Filter [codegen id : 3]
+Input [2]: [i_item_sk#19, i_item_id#20]
+Condition : isnotnull(i_item_sk#19)
+
+(21) BroadcastExchange
+Input [2]: [i_item_sk#19, i_item_id#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#21]
+
+(22) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_item_sk#3]
+Right keys [1]: [i_item_sk#19]
+Join condition: None
+
+(23) Project [codegen id : 4]
+Output [8]: [cs_bill_customer_sk#1, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, i_item_id#20]
+Input [10]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, i_item_sk#19, i_item_id#20]
+
+(24) Exchange
+Input [8]: [cs_bill_customer_sk#1, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, i_item_id#20]
+Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(25) Sort [codegen id : 5]
+Input [8]: [cs_bill_customer_sk#1, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, i_item_id#20]
+Arguments: [cs_bill_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(26) Scan parquet default.customer
+Output [5]: [c_customer_sk#23, c_current_cdemo_sk#24, c_current_addr_sk#25, c_birth_month#26, c_birth_year#27]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [In(c_birth_month, [9,5,12,4,1,10]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_addr_sk:int,c_birth_month:int,c_birth_year:int>
+
+(27) ColumnarToRow [codegen id : 7]
+Input [5]: [c_customer_sk#23, c_current_cdemo_sk#24, c_current_addr_sk#25, c_birth_month#26, c_birth_year#27]
+
+(28) Filter [codegen id : 7]
+Input [5]: [c_customer_sk#23, c_current_cdemo_sk#24, c_current_addr_sk#25, c_birth_month#26, c_birth_year#27]
+Condition : (((c_birth_month#26 IN (9,5,12,4,1,10) AND isnotnull(c_customer_sk#23)) AND isnotnull(c_current_cdemo_sk#24)) AND isnotnull(c_current_addr_sk#25))
+
+(29) Project [codegen id : 7]
+Output [4]: [c_customer_sk#23, c_current_cdemo_sk#24, c_current_addr_sk#25, c_birth_year#27]
+Input [5]: [c_customer_sk#23, c_current_cdemo_sk#24, c_current_addr_sk#25, c_birth_month#26, c_birth_year#27]
+
+(30) Scan parquet default.customer_address
+Output [4]: [ca_address_sk#28, ca_county#29, ca_state#30, ca_country#31]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [In(ca_state, [ND,WI,AL,NC,OK,MS,TN]), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_county:string,ca_state:string,ca_country:string>
+
+(31) ColumnarToRow [codegen id : 6]
+Input [4]: [ca_address_sk#28, ca_county#29, ca_state#30, ca_country#31]
+
+(32) Filter [codegen id : 6]
+Input [4]: [ca_address_sk#28, ca_county#29, ca_state#30, ca_country#31]
+Condition : (ca_state#30 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#28))
+
+(33) BroadcastExchange
+Input [4]: [ca_address_sk#28, ca_county#29, ca_state#30, ca_country#31]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#32]
+
+(34) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [c_current_addr_sk#25]
+Right keys [1]: [ca_address_sk#28]
+Join condition: None
+
+(35) Project [codegen id : 7]
+Output [6]: [c_customer_sk#23, c_current_cdemo_sk#24, c_birth_year#27, ca_county#29, ca_state#30, ca_country#31]
+Input [8]: [c_customer_sk#23, c_current_cdemo_sk#24, c_current_addr_sk#25, c_birth_year#27, ca_address_sk#28, ca_county#29, ca_state#30, ca_country#31]
+
+(36) Exchange
+Input [6]: [c_customer_sk#23, c_current_cdemo_sk#24, c_birth_year#27, ca_county#29, ca_state#30, ca_country#31]
+Arguments: hashpartitioning(c_current_cdemo_sk#24, 5), ENSURE_REQUIREMENTS, [id=#33]
+
+(37) Sort [codegen id : 8]
+Input [6]: [c_customer_sk#23, c_current_cdemo_sk#24, c_birth_year#27, ca_county#29, ca_state#30, ca_country#31]
+Arguments: [c_current_cdemo_sk#24 ASC NULLS FIRST], false, 0
+
+(38) Scan parquet default.customer_demographics
+Output [1]: [cd_demo_sk#34]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int>
+
+(39) ColumnarToRow [codegen id : 9]
+Input [1]: [cd_demo_sk#34]
+
+(40) Filter [codegen id : 9]
+Input [1]: [cd_demo_sk#34]
+Condition : isnotnull(cd_demo_sk#34)
+
+(41) Exchange
+Input [1]: [cd_demo_sk#34]
+Arguments: hashpartitioning(cd_demo_sk#34, 5), ENSURE_REQUIREMENTS, [id=#35]
+
+(42) Sort [codegen id : 10]
+Input [1]: [cd_demo_sk#34]
+Arguments: [cd_demo_sk#34 ASC NULLS FIRST], false, 0
+
+(43) SortMergeJoin [codegen id : 11]
+Left keys [1]: [c_current_cdemo_sk#24]
+Right keys [1]: [cd_demo_sk#34]
+Join condition: None
+
+(44) Project [codegen id : 11]
+Output [5]: [c_customer_sk#23, c_birth_year#27, ca_county#29, ca_state#30, ca_country#31]
+Input [7]: [c_customer_sk#23, c_current_cdemo_sk#24, c_birth_year#27, ca_county#29, ca_state#30, ca_country#31, cd_demo_sk#34]
+
+(45) Exchange
+Input [5]: [c_customer_sk#23, c_birth_year#27, ca_county#29, ca_state#30, ca_country#31]
+Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, [id=#36]
+
+(46) Sort [codegen id : 12]
+Input [5]: [c_customer_sk#23, c_birth_year#27, ca_county#29, ca_state#30, ca_country#31]
+Arguments: [c_customer_sk#23 ASC NULLS FIRST], false, 0
+
+(47) SortMergeJoin [codegen id : 13]
+Left keys [1]: [cs_bill_customer_sk#1]
+Right keys [1]: [c_customer_sk#23]
+Join condition: None
+
+(48) Project [codegen id : 13]
+Output [11]: [i_item_id#20, ca_country#31, ca_state#30, ca_county#29, cast(cs_quantity#4 as decimal(12,2)) AS agg1#37, cast(cs_list_price#5 as decimal(12,2)) AS agg2#38, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#39, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#40, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#41, cast(c_birth_year#27 as decimal(12,2)) AS agg6#42, cast(cd_dep_count#14 as decimal(12,2)) AS agg7#43]
+Input [13]: [cs_bill_customer_sk#1, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, i_item_id#20, c_customer_sk#23, c_birth_year#27, ca_county#29, ca_state#30, ca_country#31]
+
+(49) HashAggregate [codegen id : 13]
+Input [11]: [i_item_id#20, ca_country#31, ca_state#30, ca_county#29, agg1#37, agg2#38, agg3#39, agg4#40, agg5#41, agg6#42, agg7#43]
+Keys [4]: [i_item_id#20, ca_country#31, ca_state#30, ca_county#29]
+Functions [7]: [partial_avg(agg1#37), partial_avg(agg2#38), partial_avg(agg3#39), partial_avg(agg4#40), partial_avg(agg5#41), partial_avg(agg6#42), partial_avg(agg7#43)]
+Aggregate Attributes [14]: [sum#44, count#45, sum#46, count#47, sum#48, count#49, sum#50, count#51, sum#52, count#53, sum#54, count#55, sum#56, count#57]
+Results [18]: [i_item_id#20, ca_country#31, ca_state#30, ca_county#29, sum#58, count#59, sum#60, count#61, sum#62, count#63, sum#64, count#65, sum#66, count#67, sum#68, count#69, sum#70, count#71]
+
+(50) Exchange
+Input [18]: [i_item_id#20, ca_country#31, ca_state#30, ca_county#29, sum#58, count#59, sum#60, count#61, sum#62, count#63, sum#64, count#65, sum#66, count#67, sum#68, count#69, sum#70, count#71]
+Arguments: hashpartitioning(i_item_id#20, ca_country#31, ca_state#30, ca_county#29, 5), ENSURE_REQUIREMENTS, [id=#72]
+
+(51) HashAggregate [codegen id : 14]
+Input [18]: [i_item_id#20, ca_country#31, ca_state#30, ca_county#29, sum#58, count#59, sum#60, count#61, sum#62, count#63, sum#64, count#65, sum#66, count#67, sum#68, count#69, sum#70, count#71]
+Keys [4]: [i_item_id#20, ca_country#31, ca_state#30, ca_county#29]
+Functions [7]: [avg(agg1#37), avg(agg2#38), avg(agg3#39), avg(agg4#40), avg(agg5#41), avg(agg6#42), avg(agg7#43)]
+Aggregate Attributes [7]: [avg(agg1#37)#73, avg(agg2#38)#74, avg(agg3#39)#75, avg(agg4#40)#76, avg(agg5#41)#77, avg(agg6#42)#78, avg(agg7#43)#79]
+Results [11]: [i_item_id#20, ca_country#31, ca_state#30, ca_county#29, avg(agg1#37)#73 AS agg1#80, avg(agg2#38)#74 AS agg2#81, avg(agg3#39)#75 AS agg3#82, avg(agg4#40)#76 AS agg4#83, avg(agg5#41)#77 AS agg5#84, avg(agg6#42)#78 AS agg6#85, avg(agg7#43)#79 AS agg7#86]
+
+(52) ReusedExchange [Reuses operator id: unknown]
+Output [8]: [cs_bill_customer_sk#87, cs_quantity#88, cs_list_price#89, cs_sales_price#90, cs_coupon_amt#91, cs_net_profit#92, cd_dep_count#93, i_item_id#94]
+
+(53) Sort [codegen id : 19]
+Input [8]: [cs_bill_customer_sk#87, cs_quantity#88, cs_list_price#89, cs_sales_price#90, cs_coupon_amt#91, cs_net_profit#92, cd_dep_count#93, i_item_id#94]
+Arguments: [cs_bill_customer_sk#87 ASC NULLS FIRST], false, 0
+
+(54) Scan parquet default.customer
+Output [5]: [c_customer_sk#95, c_current_cdemo_sk#96, c_current_addr_sk#97, c_birth_month#98, c_birth_year#99]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [In(c_birth_month, [9,5,12,4,1,10]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_addr_sk:int,c_birth_month:int,c_birth_year:int>
+
+(55) ColumnarToRow [codegen id : 21]
+Input [5]: [c_customer_sk#95, c_current_cdemo_sk#96, c_current_addr_sk#97, c_birth_month#98, c_birth_year#99]
+
+(56) Filter [codegen id : 21]
+Input [5]: [c_customer_sk#95, c_current_cdemo_sk#96, c_current_addr_sk#97, c_birth_month#98, c_birth_year#99]
+Condition : (((c_birth_month#98 IN (9,5,12,4,1,10) AND isnotnull(c_customer_sk#95)) AND isnotnull(c_current_cdemo_sk#96)) AND isnotnull(c_current_addr_sk#97))
+
+(57) Project [codegen id : 21]
+Output [4]: [c_customer_sk#95, c_current_cdemo_sk#96, c_current_addr_sk#97, c_birth_year#99]
+Input [5]: [c_customer_sk#95, c_current_cdemo_sk#96, c_current_addr_sk#97, c_birth_month#98, c_birth_year#99]
+
+(58) Scan parquet default.customer_address
+Output [3]: [ca_address_sk#100, ca_state#101, ca_country#102]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [In(ca_state, [ND,WI,AL,NC,OK,MS,TN]), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string,ca_country:string>
+
+(59) ColumnarToRow [codegen id : 20]
+Input [3]: [ca_address_sk#100, ca_state#101, ca_country#102]
+
+(60) Filter [codegen id : 20]
+Input [3]: [ca_address_sk#100, ca_state#101, ca_country#102]
+Condition : (ca_state#101 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#100))
+
+(61) BroadcastExchange
+Input [3]: [ca_address_sk#100, ca_state#101, ca_country#102]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#103]
+
+(62) BroadcastHashJoin [codegen id : 21]
+Left keys [1]: [c_current_addr_sk#97]
+Right keys [1]: [ca_address_sk#100]
+Join condition: None
+
+(63) Project [codegen id : 21]
+Output [5]: [c_customer_sk#95, c_current_cdemo_sk#96, c_birth_year#99, ca_state#101, ca_country#102]
+Input [7]: [c_customer_sk#95, c_current_cdemo_sk#96, c_current_addr_sk#97, c_birth_year#99, ca_address_sk#100, ca_state#101, ca_country#102]
+
+(64) Exchange
+Input [5]: [c_customer_sk#95, c_current_cdemo_sk#96, c_birth_year#99, ca_state#101, ca_country#102]
+Arguments: hashpartitioning(c_current_cdemo_sk#96, 5), ENSURE_REQUIREMENTS, [id=#104]
+
+(65) Sort [codegen id : 22]
+Input [5]: [c_customer_sk#95, c_current_cdemo_sk#96, c_birth_year#99, ca_state#101, ca_country#102]
+Arguments: [c_current_cdemo_sk#96 ASC NULLS FIRST], false, 0
+
+(66) ReusedExchange [Reuses operator id: 41]
+Output [1]: [cd_demo_sk#105]
+
+(67) Sort [codegen id : 24]
+Input [1]: [cd_demo_sk#105]
+Arguments: [cd_demo_sk#105 ASC NULLS FIRST], false, 0
+
+(68) SortMergeJoin [codegen id : 25]
+Left keys [1]: [c_current_cdemo_sk#96]
+Right keys [1]: [cd_demo_sk#105]
+Join condition: None
+
+(69) Project [codegen id : 25]
+Output [4]: [c_customer_sk#95, c_birth_year#99, ca_state#101, ca_country#102]
+Input [6]: [c_customer_sk#95, c_current_cdemo_sk#96, c_birth_year#99, ca_state#101, ca_country#102, cd_demo_sk#105]
+
+(70) Exchange
+Input [4]: [c_customer_sk#95, c_birth_year#99, ca_state#101, ca_country#102]
+Arguments: hashpartitioning(c_customer_sk#95, 5), ENSURE_REQUIREMENTS, [id=#106]
+
+(71) Sort [codegen id : 26]
+Input [4]: [c_customer_sk#95, c_birth_year#99, ca_state#101, ca_country#102]
+Arguments: [c_customer_sk#95 ASC NULLS FIRST], false, 0
+
+(72) SortMergeJoin [codegen id : 27]
+Left keys [1]: [cs_bill_customer_sk#87]
+Right keys [1]: [c_customer_sk#95]
+Join condition: None
+
+(73) Project [codegen id : 27]
+Output [10]: [i_item_id#94, ca_country#102, ca_state#101, cast(cs_quantity#88 as decimal(12,2)) AS agg1#37, cast(cs_list_price#89 as decimal(12,2)) AS agg2#38, cast(cs_coupon_amt#91 as decimal(12,2)) AS agg3#39, cast(cs_sales_price#90 as decimal(12,2)) AS agg4#40, cast(cs_net_profit#92 as decimal(12,2)) AS agg5#41, cast(c_birth_year#99 as decimal(12,2)) AS agg6#42, cast(cd_dep_count#93 as decimal(12,2)) AS agg7#43]
+Input [12]: [cs_bill_customer_sk#87, cs_quantity#88, cs_list_price#89, cs_sales_price#90, cs_coupon_amt#91, cs_net_profit#92, cd_dep_count#93, i_item_id#94, c_customer_sk#95, c_birth_year#99, ca_state#101, ca_country#102]
+
+(74) HashAggregate [codegen id : 27]
+Input [10]: [i_item_id#94, ca_country#102, ca_state#101, agg1#37, agg2#38, agg3#39, agg4#40, agg5#41, agg6#42, agg7#43]
+Keys [3]: [i_item_id#94, ca_country#102, ca_state#101]
+Functions [7]: [partial_avg(agg1#37), partial_avg(agg2#38), partial_avg(agg3#39), partial_avg(agg4#40), partial_avg(agg5#41), partial_avg(agg6#42), partial_avg(agg7#43)]
+Aggregate Attributes [14]: [sum#107, count#108, sum#109, count#110, sum#111, count#112, sum#113, count#114, sum#115, count#116, sum#117, count#118, sum#119, count#120]
+Results [17]: [i_item_id#94, ca_country#102, ca_state#101, sum#121, count#122, sum#123, count#124, sum#125, count#126, sum#127, count#128, sum#129, count#130, sum#131, count#132, sum#133, count#134]
+
+(75) Exchange
+Input [17]: [i_item_id#94, ca_country#102, ca_state#101, sum#121, count#122, sum#123, count#124, sum#125, count#126, sum#127, count#128, sum#129, count#130, sum#131, count#132, sum#133, count#134]
+Arguments: hashpartitioning(i_item_id#94, ca_country#102, ca_state#101, 5), ENSURE_REQUIREMENTS, [id=#135]
+
+(76) HashAggregate [codegen id : 28]
+Input [17]: [i_item_id#94, ca_country#102, ca_state#101, sum#121, count#122, sum#123, count#124, sum#125, count#126, sum#127, count#128, sum#129, count#130, sum#131, count#132, sum#133, count#134]
+Keys [3]: [i_item_id#94, ca_country#102, ca_state#101]
+Functions [7]: [avg(agg1#37), avg(agg2#38), avg(agg3#39), avg(agg4#40), avg(agg5#41), avg(agg6#42), avg(agg7#43)]
+Aggregate Attributes [7]: [avg(agg1#37)#136, avg(agg2#38)#137, avg(agg3#39)#138, avg(agg4#40)#139, avg(agg5#41)#140, avg(agg6#42)#141, avg(agg7#43)#142]
+Results [11]: [i_item_id#94, ca_country#102, ca_state#101, null AS county#143, avg(agg1#37)#136 AS agg1#144, avg(agg2#38)#137 AS agg2#145, avg(agg3#39)#138 AS agg3#146, avg(agg4#40)#139 AS agg4#147, avg(agg5#41)#140 AS agg5#148, avg(agg6#42)#141 AS agg6#149, avg(agg7#43)#142 AS agg7#150]
+
+(77) ReusedExchange [Reuses operator id: unknown]
+Output [8]: [cs_bill_customer_sk#151, cs_quantity#152, cs_list_price#153, cs_sales_price#154, cs_coupon_amt#155, cs_net_profit#156, cd_dep_count#157, i_item_id#158]
+
+(78) Sort [codegen id : 33]
+Input [8]: [cs_bill_customer_sk#151, cs_quantity#152, cs_list_price#153, cs_sales_price#154, cs_coupon_amt#155, cs_net_profit#156, cd_dep_count#157, i_item_id#158]
+Arguments: [cs_bill_customer_sk#151 ASC NULLS FIRST], false, 0
+
+(79) Scan parquet default.customer
+Output [5]: [c_customer_sk#159, c_current_cdemo_sk#160, c_current_addr_sk#161, c_birth_month#162, c_birth_year#163]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [In(c_birth_month, [9,5,12,4,1,10]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_addr_sk:int,c_birth_month:int,c_birth_year:int>
+
+(80) ColumnarToRow [codegen id : 35]
+Input [5]: [c_customer_sk#159, c_current_cdemo_sk#160, c_current_addr_sk#161, c_birth_month#162, c_birth_year#163]
+
+(81) Filter [codegen id : 35]
+Input [5]: [c_customer_sk#159, c_current_cdemo_sk#160, c_current_addr_sk#161, c_birth_month#162, c_birth_year#163]
+Condition : (((c_birth_month#162 IN (9,5,12,4,1,10) AND isnotnull(c_customer_sk#159)) AND isnotnull(c_current_cdemo_sk#160)) AND isnotnull(c_current_addr_sk#161))
+
+(82) Project [codegen id : 35]
+Output [4]: [c_customer_sk#159, c_current_cdemo_sk#160, c_current_addr_sk#161, c_birth_year#163]
+Input [5]: [c_customer_sk#159, c_current_cdemo_sk#160, c_current_addr_sk#161, c_birth_month#162, c_birth_year#163]
+
+(83) Scan parquet default.customer_address
+Output [3]: [ca_address_sk#164, ca_state#165, ca_country#166]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [In(ca_state, [ND,WI,AL,NC,OK,MS,TN]), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string,ca_country:string>
+
+(84) ColumnarToRow [codegen id : 34]
+Input [3]: [ca_address_sk#164, ca_state#165, ca_country#166]
+
+(85) Filter [codegen id : 34]
+Input [3]: [ca_address_sk#164, ca_state#165, ca_country#166]
+Condition : (ca_state#165 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#164))
+
+(86) Project [codegen id : 34]
+Output [2]: [ca_address_sk#164, ca_country#166]
+Input [3]: [ca_address_sk#164, ca_state#165, ca_country#166]
+
+(87) BroadcastExchange
+Input [2]: [ca_address_sk#164, ca_country#166]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#167]
+
+(88) BroadcastHashJoin [codegen id : 35]
+Left keys [1]: [c_current_addr_sk#161]
+Right keys [1]: [ca_address_sk#164]
+Join condition: None
+
+(89) Project [codegen id : 35]
+Output [4]: [c_customer_sk#159, c_current_cdemo_sk#160, c_birth_year#163, ca_country#166]
+Input [6]: [c_customer_sk#159, c_current_cdemo_sk#160, c_current_addr_sk#161, c_birth_year#163, ca_address_sk#164, ca_country#166]
+
+(90) Exchange
+Input [4]: [c_customer_sk#159, c_current_cdemo_sk#160, c_birth_year#163, ca_country#166]
+Arguments: hashpartitioning(c_current_cdemo_sk#160, 5), ENSURE_REQUIREMENTS, [id=#168]
+
+(91) Sort [codegen id : 36]
+Input [4]: [c_customer_sk#159, c_current_cdemo_sk#160, c_birth_year#163, ca_country#166]
+Arguments: [c_current_cdemo_sk#160 ASC NULLS FIRST], false, 0
+
+(92) ReusedExchange [Reuses operator id: 41]
+Output [1]: [cd_demo_sk#169]
+
+(93) Sort [codegen id : 38]
+Input [1]: [cd_demo_sk#169]
+Arguments: [cd_demo_sk#169 ASC NULLS FIRST], false, 0
+
+(94) SortMergeJoin [codegen id : 39]
+Left keys [1]: [c_current_cdemo_sk#160]
+Right keys [1]: [cd_demo_sk#169]
+Join condition: None
+
+(95) Project [codegen id : 39]
+Output [3]: [c_customer_sk#159, c_birth_year#163, ca_country#166]
+Input [5]: [c_customer_sk#159, c_current_cdemo_sk#160, c_birth_year#163, ca_country#166, cd_demo_sk#169]
+
+(96) Exchange
+Input [3]: [c_customer_sk#159, c_birth_year#163, ca_country#166]
+Arguments: hashpartitioning(c_customer_sk#159, 5), ENSURE_REQUIREMENTS, [id=#170]
+
+(97) Sort [codegen id : 40]
+Input [3]: [c_customer_sk#159, c_birth_year#163, ca_country#166]
+Arguments: [c_customer_sk#159 ASC NULLS FIRST], false, 0
+
+(98) SortMergeJoin [codegen id : 41]
+Left keys [1]: [cs_bill_customer_sk#151]
+Right keys [1]: [c_customer_sk#159]
+Join condition: None
+
+(99) Project [codegen id : 41]
+Output [9]: [i_item_id#158, ca_country#166, cast(cs_quantity#152 as decimal(12,2)) AS agg1#37, cast(cs_list_price#153 as decimal(12,2)) AS agg2#38, cast(cs_coupon_amt#155 as decimal(12,2)) AS agg3#39, cast(cs_sales_price#154 as decimal(12,2)) AS agg4#40, cast(cs_net_profit#156 as decimal(12,2)) AS agg5#41, cast(c_birth_year#163 as decimal(12,2)) AS agg6#42, cast(cd_dep_count#157 as decimal(12,2)) AS agg7#43]
+Input [11]: [cs_bill_customer_sk#151, cs_quantity#152, cs_list_price#153, cs_sales_price#154, cs_coupon_amt#155, cs_net_profit#156, cd_dep_count#157, i_item_id#158, c_customer_sk#159, c_birth_year#163, ca_country#166]
+
+(100) HashAggregate [codegen id : 41]
+Input [9]: [i_item_id#158, ca_country#166, agg1#37, agg2#38, agg3#39, agg4#40, agg5#41, agg6#42, agg7#43]
+Keys [2]: [i_item_id#158, ca_country#166]
+Functions [7]: [partial_avg(agg1#37), partial_avg(agg2#38), partial_avg(agg3#39), partial_avg(agg4#40), partial_avg(agg5#41), partial_avg(agg6#42), partial_avg(agg7#43)]
+Aggregate Attributes [14]: [sum#171, count#172, sum#173, count#174, sum#175, count#176, sum#177, count#178, sum#179, count#180, sum#181, count#182, sum#183, count#184]
+Results [16]: [i_item_id#158, ca_country#166, sum#185, count#186, sum#187, count#188, sum#189, count#190, sum#191, count#192, sum#193, count#194, sum#195, count#196, sum#197, count#198]
+
+(101) Exchange
+Input [16]: [i_item_id#158, ca_country#166, sum#185, count#186, sum#187, count#188, sum#189, count#190, sum#191, count#192, sum#193, count#194, sum#195, count#196, sum#197, count#198]
+Arguments: hashpartitioning(i_item_id#158, ca_country#166, 5), ENSURE_REQUIREMENTS, [id=#199]
+
+(102) HashAggregate [codegen id : 42]
+Input [16]: [i_item_id#158, ca_country#166, sum#185, count#186, sum#187, count#188, sum#189, count#190, sum#191, count#192, sum#193, count#194, sum#195, count#196, sum#197, count#198]
+Keys [2]: [i_item_id#158, ca_country#166]
+Functions [7]: [avg(agg1#37), avg(agg2#38), avg(agg3#39), avg(agg4#40), avg(agg5#41), avg(agg6#42), avg(agg7#43)]
+Aggregate Attributes [7]: [avg(agg1#37)#200, avg(agg2#38)#201, avg(agg3#39)#202, avg(agg4#40)#203, avg(agg5#41)#204, avg(agg6#42)#205, avg(agg7#43)#206]
+Results [11]: [i_item_id#158, ca_country#166, null AS ca_state#207, null AS county#208, avg(agg1#37)#200 AS agg1#209, avg(agg2#38)#201 AS agg2#210, avg(agg3#39)#202 AS agg3#211, avg(agg4#40)#203 AS agg4#212, avg(agg5#41)#204 AS agg5#213, avg(agg6#42)#205 AS agg6#214, avg(agg7#43)#206 AS agg7#215]
+
+(103) Scan parquet default.catalog_sales
+Output [9]: [cs_bill_customer_sk#216, cs_bill_cdemo_sk#217, cs_item_sk#218, cs_quantity#219, cs_list_price#220, cs_sales_price#221, cs_coupon_amt#222, cs_net_profit#223, cs_sold_date_sk#224]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#224), dynamicpruningexpression(cs_sold_date_sk#224 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_bill_cdemo_sk:int,cs_item_sk:int,cs_quantity:int,cs_list_price:decimal(7,2),cs_sales_price:decimal(7,2),cs_coupon_amt:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(104) ColumnarToRow [codegen id : 49]
+Input [9]: [cs_bill_customer_sk#216, cs_bill_cdemo_sk#217, cs_item_sk#218, cs_quantity#219, cs_list_price#220, cs_sales_price#221, cs_coupon_amt#222, cs_net_profit#223, cs_sold_date_sk#224]
+
+(105) Filter [codegen id : 49]
+Input [9]: [cs_bill_customer_sk#216, cs_bill_cdemo_sk#217, cs_item_sk#218, cs_quantity#219, cs_list_price#220, cs_sales_price#221, cs_coupon_amt#222, cs_net_profit#223, cs_sold_date_sk#224]
+Condition : ((isnotnull(cs_bill_cdemo_sk#217) AND isnotnull(cs_bill_customer_sk#216)) AND isnotnull(cs_item_sk#218))
+
+(106) ReusedExchange [Reuses operator id: 8]
+Output [2]: [cd_demo_sk#225, cd_dep_count#226]
+
+(107) BroadcastHashJoin [codegen id : 49]
+Left keys [1]: [cs_bill_cdemo_sk#217]
+Right keys [1]: [cd_demo_sk#225]
+Join condition: None
+
+(108) Project [codegen id : 49]
+Output [9]: [cs_bill_customer_sk#216, cs_item_sk#218, cs_quantity#219, cs_list_price#220, cs_sales_price#221, cs_coupon_amt#222, cs_net_profit#223, cs_sold_date_sk#224, cd_dep_count#226]
+Input [11]: [cs_bill_customer_sk#216, cs_bill_cdemo_sk#217, cs_item_sk#218, cs_quantity#219, cs_list_price#220, cs_sales_price#221, cs_coupon_amt#222, cs_net_profit#223, cs_sold_date_sk#224, cd_demo_sk#225, cd_dep_count#226]
+
+(109) ReusedExchange [Reuses operator id: 15]
+Output [1]: [d_date_sk#227]
+
+(110) BroadcastHashJoin [codegen id : 49]
+Left keys [1]: [cs_sold_date_sk#224]
+Right keys [1]: [d_date_sk#227]
+Join condition: None
+
+(111) Project [codegen id : 49]
+Output [8]: [cs_bill_customer_sk#216, cs_item_sk#218, cs_quantity#219, cs_list_price#220, cs_sales_price#221, cs_coupon_amt#222, cs_net_profit#223, cd_dep_count#226]
+Input [10]: [cs_bill_customer_sk#216, cs_item_sk#218, cs_quantity#219, cs_list_price#220, cs_sales_price#221, cs_coupon_amt#222, cs_net_profit#223, cs_sold_date_sk#224, cd_dep_count#226, d_date_sk#227]
+
+(112) Scan parquet default.customer
+Output [5]: [c_customer_sk#228, c_current_cdemo_sk#229, c_current_addr_sk#230, c_birth_month#231, c_birth_year#232]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [In(c_birth_month, [9,5,12,4,1,10]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_addr_sk:int,c_birth_month:int,c_birth_year:int>
+
+(113) ColumnarToRow [codegen id : 46]
+Input [5]: [c_customer_sk#228, c_current_cdemo_sk#229, c_current_addr_sk#230, c_birth_month#231, c_birth_year#232]
+
+(114) Filter [codegen id : 46]
+Input [5]: [c_customer_sk#228, c_current_cdemo_sk#229, c_current_addr_sk#230, c_birth_month#231, c_birth_year#232]
+Condition : (((c_birth_month#231 IN (9,5,12,4,1,10) AND isnotnull(c_customer_sk#228)) AND isnotnull(c_current_cdemo_sk#229)) AND isnotnull(c_current_addr_sk#230))
+
+(115) Project [codegen id : 46]
+Output [4]: [c_customer_sk#228, c_current_cdemo_sk#229, c_current_addr_sk#230, c_birth_year#232]
+Input [5]: [c_customer_sk#228, c_current_cdemo_sk#229, c_current_addr_sk#230, c_birth_month#231, c_birth_year#232]
+
+(116) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#233, ca_state#234]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [In(ca_state, [ND,WI,AL,NC,OK,MS,TN]), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(117) ColumnarToRow [codegen id : 45]
+Input [2]: [ca_address_sk#233, ca_state#234]
+
+(118) Filter [codegen id : 45]
+Input [2]: [ca_address_sk#233, ca_state#234]
+Condition : (ca_state#234 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#233))
+
+(119) Project [codegen id : 45]
+Output [1]: [ca_address_sk#233]
+Input [2]: [ca_address_sk#233, ca_state#234]
+
+(120) BroadcastExchange
+Input [1]: [ca_address_sk#233]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#235]
+
+(121) BroadcastHashJoin [codegen id : 46]
+Left keys [1]: [c_current_addr_sk#230]
+Right keys [1]: [ca_address_sk#233]
+Join condition: None
+
+(122) Project [codegen id : 46]
+Output [3]: [c_customer_sk#228, c_current_cdemo_sk#229, c_birth_year#232]
+Input [5]: [c_customer_sk#228, c_current_cdemo_sk#229, c_current_addr_sk#230, c_birth_year#232, ca_address_sk#233]
+
+(123) BroadcastExchange
+Input [3]: [c_customer_sk#228, c_current_cdemo_sk#229, c_birth_year#232]
+Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [id=#236]
+
+(124) Scan parquet default.customer_demographics
+Output [1]: [cd_demo_sk#237]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int>
+
+(125) ColumnarToRow
+Input [1]: [cd_demo_sk#237]
+
+(126) Filter
+Input [1]: [cd_demo_sk#237]
+Condition : isnotnull(cd_demo_sk#237)
+
+(127) BroadcastHashJoin [codegen id : 47]
+Left keys [1]: [c_current_cdemo_sk#229]
+Right keys [1]: [cd_demo_sk#237]
+Join condition: None
+
+(128) Project [codegen id : 47]
+Output [2]: [c_customer_sk#228, c_birth_year#232]
+Input [4]: [c_customer_sk#228, c_current_cdemo_sk#229, c_birth_year#232, cd_demo_sk#237]
+
+(129) BroadcastExchange
+Input [2]: [c_customer_sk#228, c_birth_year#232]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#238]
+
+(130) BroadcastHashJoin [codegen id : 49]
+Left keys [1]: [cs_bill_customer_sk#216]
+Right keys [1]: [c_customer_sk#228]
+Join condition: None
+
+(131) Project [codegen id : 49]
+Output [8]: [cs_item_sk#218, cs_quantity#219, cs_list_price#220, cs_sales_price#221, cs_coupon_amt#222, cs_net_profit#223, cd_dep_count#226, c_birth_year#232]
+Input [10]: [cs_bill_customer_sk#216, cs_item_sk#218, cs_quantity#219, cs_list_price#220, cs_sales_price#221, cs_coupon_amt#222, cs_net_profit#223, cd_dep_count#226, c_customer_sk#228, c_birth_year#232]
+
+(132) ReusedExchange [Reuses operator id: 21]
+Output [2]: [i_item_sk#239, i_item_id#240]
+
+(133) BroadcastHashJoin [codegen id : 49]
+Left keys [1]: [cs_item_sk#218]
+Right keys [1]: [i_item_sk#239]
+Join condition: None
+
+(134) Project [codegen id : 49]
+Output [8]: [i_item_id#240, cast(cs_quantity#219 as decimal(12,2)) AS agg1#37, cast(cs_list_price#220 as decimal(12,2)) AS agg2#38, cast(cs_coupon_amt#222 as decimal(12,2)) AS agg3#39, cast(cs_sales_price#221 as decimal(12,2)) AS agg4#40, cast(cs_net_profit#223 as decimal(12,2)) AS agg5#41, cast(c_birth_year#232 as decimal(12,2)) AS agg6#42, cast(cd_dep_count#226 as decimal(12,2)) AS agg7#43]
+Input [10]: [cs_item_sk#218, cs_quantity#219, cs_list_price#220, cs_sales_price#221, cs_coupon_amt#222, cs_net_profit#223, cd_dep_count#226, c_birth_year#232, i_item_sk#239, i_item_id#240]
+
+(135) HashAggregate [codegen id : 49]
+Input [8]: [i_item_id#240, agg1#37, agg2#38, agg3#39, agg4#40, agg5#41, agg6#42, agg7#43]
+Keys [1]: [i_item_id#240]
+Functions [7]: [partial_avg(agg1#37), partial_avg(agg2#38), partial_avg(agg3#39), partial_avg(agg4#40), partial_avg(agg5#41), partial_avg(agg6#42), partial_avg(agg7#43)]
+Aggregate Attributes [14]: [sum#241, count#242, sum#243, count#244, sum#245, count#246, sum#247, count#248, sum#249, count#250, sum#251, count#252, sum#253, count#254]
+Results [15]: [i_item_id#240, sum#255, count#256, sum#257, count#258, sum#259, count#260, sum#261, count#262, sum#263, count#264, sum#265, count#266, sum#267, count#268]
+
+(136) Exchange
+Input [15]: [i_item_id#240, sum#255, count#256, sum#257, count#258, sum#259, count#260, sum#261, count#262, sum#263, count#264, sum#265, count#266, sum#267, count#268]
+Arguments: hashpartitioning(i_item_id#240, 5), ENSURE_REQUIREMENTS, [id=#269]
+
+(137) HashAggregate [codegen id : 50]
+Input [15]: [i_item_id#240, sum#255, count#256, sum#257, count#258, sum#259, count#260, sum#261, count#262, sum#263, count#264, sum#265, count#266, sum#267, count#268]
+Keys [1]: [i_item_id#240]
+Functions [7]: [avg(agg1#37), avg(agg2#38), avg(agg3#39), avg(agg4#40), avg(agg5#41), avg(agg6#42), avg(agg7#43)]
+Aggregate Attributes [7]: [avg(agg1#37)#270, avg(agg2#38)#271, avg(agg3#39)#272, avg(agg4#40)#273, avg(agg5#41)#274, avg(agg6#42)#275, avg(agg7#43)#276]
+Results [11]: [i_item_id#240, null AS ca_country#277, null AS ca_state#278, null AS county#279, avg(agg1#37)#270 AS agg1#280, avg(agg2#38)#271 AS agg2#281, avg(agg3#39)#272 AS agg3#282, avg(agg4#40)#273 AS agg4#283, avg(agg5#41)#274 AS agg5#284, avg(agg6#42)#275 AS agg6#285, avg(agg7#43)#276 AS agg7#286]
+
+(138) Scan parquet default.catalog_sales
+Output [9]: [cs_bill_customer_sk#287, cs_bill_cdemo_sk#288, cs_item_sk#289, cs_quantity#290, cs_list_price#291, cs_sales_price#292, cs_coupon_amt#293, cs_net_profit#294, cs_sold_date_sk#295]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#295), dynamicpruningexpression(cs_sold_date_sk#295 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_bill_cdemo_sk:int,cs_item_sk:int,cs_quantity:int,cs_list_price:decimal(7,2),cs_sales_price:decimal(7,2),cs_coupon_amt:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(139) ColumnarToRow [codegen id : 57]
+Input [9]: [cs_bill_customer_sk#287, cs_bill_cdemo_sk#288, cs_item_sk#289, cs_quantity#290, cs_list_price#291, cs_sales_price#292, cs_coupon_amt#293, cs_net_profit#294, cs_sold_date_sk#295]
+
+(140) Filter [codegen id : 57]
+Input [9]: [cs_bill_customer_sk#287, cs_bill_cdemo_sk#288, cs_item_sk#289, cs_quantity#290, cs_list_price#291, cs_sales_price#292, cs_coupon_amt#293, cs_net_profit#294, cs_sold_date_sk#295]
+Condition : ((isnotnull(cs_bill_cdemo_sk#288) AND isnotnull(cs_bill_customer_sk#287)) AND isnotnull(cs_item_sk#289))
+
+(141) ReusedExchange [Reuses operator id: 8]
+Output [2]: [cd_demo_sk#296, cd_dep_count#297]
+
+(142) BroadcastHashJoin [codegen id : 57]
+Left keys [1]: [cs_bill_cdemo_sk#288]
+Right keys [1]: [cd_demo_sk#296]
+Join condition: None
+
+(143) Project [codegen id : 57]
+Output [9]: [cs_bill_customer_sk#287, cs_item_sk#289, cs_quantity#290, cs_list_price#291, cs_sales_price#292, cs_coupon_amt#293, cs_net_profit#294, cs_sold_date_sk#295, cd_dep_count#297]
+Input [11]: [cs_bill_customer_sk#287, cs_bill_cdemo_sk#288, cs_item_sk#289, cs_quantity#290, cs_list_price#291, cs_sales_price#292, cs_coupon_amt#293, cs_net_profit#294, cs_sold_date_sk#295, cd_demo_sk#296, cd_dep_count#297]
+
+(144) ReusedExchange [Reuses operator id: 15]
+Output [1]: [d_date_sk#298]
+
+(145) BroadcastHashJoin [codegen id : 57]
+Left keys [1]: [cs_sold_date_sk#295]
+Right keys [1]: [d_date_sk#298]
+Join condition: None
+
+(146) Project [codegen id : 57]
+Output [8]: [cs_bill_customer_sk#287, cs_item_sk#289, cs_quantity#290, cs_list_price#291, cs_sales_price#292, cs_coupon_amt#293, cs_net_profit#294, cd_dep_count#297]
+Input [10]: [cs_bill_customer_sk#287, cs_item_sk#289, cs_quantity#290, cs_list_price#291, cs_sales_price#292, cs_coupon_amt#293, cs_net_profit#294, cs_sold_date_sk#295, cd_dep_count#297, d_date_sk#298]
+
+(147) Scan parquet default.item
+Output [1]: [i_item_sk#299]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int>
+
+(148) ColumnarToRow [codegen id : 53]
+Input [1]: [i_item_sk#299]
+
+(149) Filter [codegen id : 53]
+Input [1]: [i_item_sk#299]
+Condition : isnotnull(i_item_sk#299)
+
+(150) BroadcastExchange
+Input [1]: [i_item_sk#299]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#300]
+
+(151) BroadcastHashJoin [codegen id : 57]
+Left keys [1]: [cs_item_sk#289]
+Right keys [1]: [i_item_sk#299]
+Join condition: None
+
+(152) Project [codegen id : 57]
+Output [7]: [cs_bill_customer_sk#287, cs_quantity#290, cs_list_price#291, cs_sales_price#292, cs_coupon_amt#293, cs_net_profit#294, cd_dep_count#297]
+Input [9]: [cs_bill_customer_sk#287, cs_item_sk#289, cs_quantity#290, cs_list_price#291, cs_sales_price#292, cs_coupon_amt#293, cs_net_profit#294, cd_dep_count#297, i_item_sk#299]
+
+(153) ReusedExchange [Reuses operator id: 129]
+Output [2]: [c_customer_sk#301, c_birth_year#302]
+
+(154) BroadcastHashJoin [codegen id : 57]
+Left keys [1]: [cs_bill_customer_sk#287]
+Right keys [1]: [c_customer_sk#301]
+Join condition: None
+
+(155) Project [codegen id : 57]
+Output [7]: [cast(cs_quantity#290 as decimal(12,2)) AS agg1#37, cast(cs_list_price#291 as decimal(12,2)) AS agg2#38, cast(cs_coupon_amt#293 as decimal(12,2)) AS agg3#39, cast(cs_sales_price#292 as decimal(12,2)) AS agg4#40, cast(cs_net_profit#294 as decimal(12,2)) AS agg5#41, cast(c_birth_year#302 as decimal(12,2)) AS agg6#42, cast(cd_dep_count#297 as decimal(12,2)) AS agg7#43]
+Input [9]: [cs_bill_customer_sk#287, cs_quantity#290, cs_list_price#291, cs_sales_price#292, cs_coupon_amt#293, cs_net_profit#294, cd_dep_count#297, c_customer_sk#301, c_birth_year#302]
+
+(156) HashAggregate [codegen id : 57]
+Input [7]: [agg1#37, agg2#38, agg3#39, agg4#40, agg5#41, agg6#42, agg7#43]
+Keys: []
+Functions [7]: [partial_avg(agg1#37), partial_avg(agg2#38), partial_avg(agg3#39), partial_avg(agg4#40), partial_avg(agg5#41), partial_avg(agg6#42), partial_avg(agg7#43)]
+Aggregate Attributes [14]: [sum#303, count#304, sum#305, count#306, sum#307, count#308, sum#309, count#310, sum#311, count#312, sum#313, count#314, sum#315, count#316]
+Results [14]: [sum#317, count#318, sum#319, count#320, sum#321, count#322, sum#323, count#324, sum#325, count#326, sum#327, count#328, sum#329, count#330]
+
+(157) Exchange
+Input [14]: [sum#317, count#318, sum#319, count#320, sum#321, count#322, sum#323, count#324, sum#325, count#326, sum#327, count#328, sum#329, count#330]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#331]
+
+(158) HashAggregate [codegen id : 58]
+Input [14]: [sum#317, count#318, sum#319, count#320, sum#321, count#322, sum#323, count#324, sum#325, count#326, sum#327, count#328, sum#329, count#330]
+Keys: []
+Functions [7]: [avg(agg1#37), avg(agg2#38), avg(agg3#39), avg(agg4#40), avg(agg5#41), avg(agg6#42), avg(agg7#43)]
+Aggregate Attributes [7]: [avg(agg1#37)#332, avg(agg2#38)#333, avg(agg3#39)#334, avg(agg4#40)#335, avg(agg5#41)#336, avg(agg6#42)#337, avg(agg7#43)#338]
+Results [11]: [null AS i_item_id#339, null AS ca_country#340, null AS ca_state#341, null AS county#342, avg(agg1#37)#332 AS agg1#343, avg(agg2#38)#333 AS agg2#344, avg(agg3#39)#334 AS agg3#345, avg(agg4#40)#335 AS agg4#346, avg(agg5#41)#336 AS agg5#347, avg(agg6#42)#337 AS agg6#348, avg(agg7#43)#338 AS agg7#349]
+
+(159) Union
+
+(160) TakeOrderedAndProject
+Input [11]: [i_item_id#20, ca_country#31, ca_state#30, ca_county#29, agg1#80, agg2#81, agg3#82, agg4#83, agg5#84, agg6#85, agg7#86]
+Arguments: 100, [ca_country#31 ASC NULLS FIRST, ca_state#30 ASC NULLS FIRST, ca_county#29 ASC NULLS FIRST, i_item_id#20 ASC NULLS FIRST], [i_item_id#20, ca_country#31, ca_state#30, ca_county#29, agg1#80, agg2#81, agg3#82, agg4#83, agg5#84, agg6#85, agg7#86]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#9 IN dynamicpruning#10
+ReusedExchange (161)
+
+
+(161) ReusedExchange [Reuses operator id: 15]
+Output [1]: [d_date_sk#16]
+
+Subquery:2 Hosting operator id = 103 Hosting Expression = cs_sold_date_sk#224 IN dynamicpruning#10
+
+Subquery:3 Hosting operator id = 138 Hosting Expression = cs_sold_date_sk#295 IN dynamicpruning#10
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.sf100/simplified.txt
new file mode 100644
index 0000000000000..13aa887abaa2d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.sf100/simplified.txt
@@ -0,0 +1,266 @@
+TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7]
+  Union
+    WholeStageCodegen (14)
+      HashAggregate [i_item_id,ca_country,ca_state,ca_county,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+        InputAdapter
+          Exchange [i_item_id,ca_country,ca_state,ca_county] #1
+            WholeStageCodegen (13)
+              HashAggregate [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+                Project [i_item_id,ca_country,ca_state,ca_county,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count]
+                  SortMergeJoin [cs_bill_customer_sk,c_customer_sk]
+                    InputAdapter
+                      WholeStageCodegen (5)
+                        Sort [cs_bill_customer_sk]
+                          InputAdapter
+                            Exchange [cs_bill_customer_sk] #2
+                              WholeStageCodegen (4)
+                                Project [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,i_item_id]
+                                  BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                    Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count]
+                                      BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                        Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count]
+                                          BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk]
+                                            Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #1
+                                                      ReusedExchange [d_date_sk] #3
+                                            InputAdapter
+                                              BroadcastExchange #4
+                                                WholeStageCodegen (1)
+                                                  Project [cd_demo_sk,cd_dep_count]
+                                                    Filter [cd_gender,cd_education_status,cd_demo_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count]
+                                        InputAdapter
+                                          BroadcastExchange #3
+                                            WholeStageCodegen (2)
+                                              Project [d_date_sk]
+                                                Filter [d_year,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_year]
+                                    InputAdapter
+                                      BroadcastExchange #5
+                                        WholeStageCodegen (3)
+                                          Filter [i_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.item [i_item_sk,i_item_id]
+                    InputAdapter
+                      WholeStageCodegen (12)
+                        Sort [c_customer_sk]
+                          InputAdapter
+                            Exchange [c_customer_sk] #6
+                              WholeStageCodegen (11)
+                                Project [c_customer_sk,c_birth_year,ca_county,ca_state,ca_country]
+                                  SortMergeJoin [c_current_cdemo_sk,cd_demo_sk]
+                                    InputAdapter
+                                      WholeStageCodegen (8)
+                                        Sort [c_current_cdemo_sk]
+                                          InputAdapter
+                                            Exchange [c_current_cdemo_sk] #7
+                                              WholeStageCodegen (7)
+                                                Project [c_customer_sk,c_current_cdemo_sk,c_birth_year,ca_county,ca_state,ca_country]
+                                                  BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                                                    Project [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year]
+                                                      Filter [c_birth_month,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year]
+                                                    InputAdapter
+                                                      BroadcastExchange #8
+                                                        WholeStageCodegen (6)
+                                                          Filter [ca_state,ca_address_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.customer_address [ca_address_sk,ca_county,ca_state,ca_country]
+                                    InputAdapter
+                                      WholeStageCodegen (10)
+                                        Sort [cd_demo_sk]
+                                          InputAdapter
+                                            Exchange [cd_demo_sk] #9
+                                              WholeStageCodegen (9)
+                                                Filter [cd_demo_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.customer_demographics [cd_demo_sk]
+    WholeStageCodegen (28)
+      HashAggregate [i_item_id,ca_country,ca_state,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+        InputAdapter
+          Exchange [i_item_id,ca_country,ca_state] #10
+            WholeStageCodegen (27)
+              HashAggregate [i_item_id,ca_country,ca_state,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+                Project [i_item_id,ca_country,ca_state,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count]
+                  SortMergeJoin [cs_bill_customer_sk,c_customer_sk]
+                    InputAdapter
+                      WholeStageCodegen (19)
+                        Sort [cs_bill_customer_sk]
+                          InputAdapter
+                            ReusedExchange [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,i_item_id] #11
+                    InputAdapter
+                      WholeStageCodegen (26)
+                        Sort [c_customer_sk]
+                          InputAdapter
+                            Exchange [c_customer_sk] #12
+                              WholeStageCodegen (25)
+                                Project [c_customer_sk,c_birth_year,ca_state,ca_country]
+                                  SortMergeJoin [c_current_cdemo_sk,cd_demo_sk]
+                                    InputAdapter
+                                      WholeStageCodegen (22)
+                                        Sort [c_current_cdemo_sk]
+                                          InputAdapter
+                                            Exchange [c_current_cdemo_sk] #13
+                                              WholeStageCodegen (21)
+                                                Project [c_customer_sk,c_current_cdemo_sk,c_birth_year,ca_state,ca_country]
+                                                  BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                                                    Project [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year]
+                                                      Filter [c_birth_month,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year]
+                                                    InputAdapter
+                                                      BroadcastExchange #14
+                                                        WholeStageCodegen (20)
+                                                          Filter [ca_state,ca_address_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.customer_address [ca_address_sk,ca_state,ca_country]
+                                    InputAdapter
+                                      WholeStageCodegen (24)
+                                        Sort [cd_demo_sk]
+                                          InputAdapter
+                                            ReusedExchange [cd_demo_sk] #9
+    WholeStageCodegen (42)
+      HashAggregate [i_item_id,ca_country,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+        InputAdapter
+          Exchange [i_item_id,ca_country] #15
+            WholeStageCodegen (41)
+              HashAggregate [i_item_id,ca_country,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+                Project [i_item_id,ca_country,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count]
+                  SortMergeJoin [cs_bill_customer_sk,c_customer_sk]
+                    InputAdapter
+                      WholeStageCodegen (33)
+                        Sort [cs_bill_customer_sk]
+                          InputAdapter
+                            ReusedExchange [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,i_item_id] #11
+                    InputAdapter
+                      WholeStageCodegen (40)
+                        Sort [c_customer_sk]
+                          InputAdapter
+                            Exchange [c_customer_sk] #16
+                              WholeStageCodegen (39)
+                                Project [c_customer_sk,c_birth_year,ca_country]
+                                  SortMergeJoin [c_current_cdemo_sk,cd_demo_sk]
+                                    InputAdapter
+                                      WholeStageCodegen (36)
+                                        Sort [c_current_cdemo_sk]
+                                          InputAdapter
+                                            Exchange [c_current_cdemo_sk] #17
+                                              WholeStageCodegen (35)
+                                                Project [c_customer_sk,c_current_cdemo_sk,c_birth_year,ca_country]
+                                                  BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                                                    Project [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year]
+                                                      Filter [c_birth_month,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year]
+                                                    InputAdapter
+                                                      BroadcastExchange #18
+                                                        WholeStageCodegen (34)
+                                                          Project [ca_address_sk,ca_country]
+                                                            Filter [ca_state,ca_address_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.customer_address [ca_address_sk,ca_state,ca_country]
+                                    InputAdapter
+                                      WholeStageCodegen (38)
+                                        Sort [cd_demo_sk]
+                                          InputAdapter
+                                            ReusedExchange [cd_demo_sk] #9
+    WholeStageCodegen (50)
+      HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+        InputAdapter
+          Exchange [i_item_id] #19
+            WholeStageCodegen (49)
+              HashAggregate [i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+                Project [i_item_id,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count]
+                  BroadcastHashJoin [cs_item_sk,i_item_sk]
+                    Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year]
+                      BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk]
+                        Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count]
+                          BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                            Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count]
+                              BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk]
+                                Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk]
+                                        ReusedSubquery [d_date_sk] #1
+                                InputAdapter
+                                  ReusedExchange [cd_demo_sk,cd_dep_count] #4
+                            InputAdapter
+                              ReusedExchange [d_date_sk] #3
+                        InputAdapter
+                          BroadcastExchange #20
+                            WholeStageCodegen (47)
+                              Project [c_customer_sk,c_birth_year]
+                                BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk]
+                                  InputAdapter
+                                    BroadcastExchange #21
+                                      WholeStageCodegen (46)
+                                        Project [c_customer_sk,c_current_cdemo_sk,c_birth_year]
+                                          BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                                            Project [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year]
+                                              Filter [c_birth_month,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year]
+                                            InputAdapter
+                                              BroadcastExchange #22
+                                                WholeStageCodegen (45)
+                                                  Project [ca_address_sk]
+                                                    Filter [ca_state,ca_address_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.customer_address [ca_address_sk,ca_state]
+                                  Filter [cd_demo_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.customer_demographics [cd_demo_sk]
+                    InputAdapter
+                      ReusedExchange [i_item_sk,i_item_id] #5
+    WholeStageCodegen (58)
+      HashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+        InputAdapter
+          Exchange #23
+            WholeStageCodegen (57)
+              HashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+                Project [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count]
+                  BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk]
+                    Project [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count]
+                      BroadcastHashJoin [cs_item_sk,i_item_sk]
+                        Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count]
+                          BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                            Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count]
+                              BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk]
+                                Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk]
+                                        ReusedSubquery [d_date_sk] #1
+                                InputAdapter
+                                  ReusedExchange [cd_demo_sk,cd_dep_count] #4
+                            InputAdapter
+                              ReusedExchange [d_date_sk] #3
+                        InputAdapter
+                          BroadcastExchange #24
+                            WholeStageCodegen (53)
+                              Filter [i_item_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.item [i_item_sk]
+                    InputAdapter
+                      ReusedExchange [c_customer_sk,c_birth_year] #20
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/explain.txt
new file mode 100644
index 0000000000000..8e3c5958d2831
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/explain.txt
@@ -0,0 +1,879 @@
+== Physical Plan ==
+TakeOrderedAndProject (157)
++- Union (156)
+   :- * HashAggregate (45)
+   :  +- Exchange (44)
+   :     +- * HashAggregate (43)
+   :        +- * Project (42)
+   :           +- * BroadcastHashJoin Inner BuildRight (41)
+   :              :- * Project (36)
+   :              :  +- * BroadcastHashJoin Inner BuildRight (35)
+   :              :     :- * Project (29)
+   :              :     :  +- * BroadcastHashJoin Inner BuildRight (28)
+   :              :     :     :- * Project (23)
+   :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (22)
+   :              :     :     :     :- * Project (17)
+   :              :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (16)
+   :              :     :     :     :     :- * Project (10)
+   :              :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+   :              :     :     :     :     :     :- * Filter (3)
+   :              :     :     :     :     :     :  +- * ColumnarToRow (2)
+   :              :     :     :     :     :     :     +- Scan parquet default.catalog_sales (1)
+   :              :     :     :     :     :     +- BroadcastExchange (8)
+   :              :     :     :     :     :        +- * Project (7)
+   :              :     :     :     :     :           +- * Filter (6)
+   :              :     :     :     :     :              +- * ColumnarToRow (5)
+   :              :     :     :     :     :                 +- Scan parquet default.customer_demographics (4)
+   :              :     :     :     :     +- BroadcastExchange (15)
+   :              :     :     :     :        +- * Project (14)
+   :              :     :     :     :           +- * Filter (13)
+   :              :     :     :     :              +- * ColumnarToRow (12)
+   :              :     :     :     :                 +- Scan parquet default.customer (11)
+   :              :     :     :     +- BroadcastExchange (21)
+   :              :     :     :        +- * Filter (20)
+   :              :     :     :           +- * ColumnarToRow (19)
+   :              :     :     :              +- Scan parquet default.customer_demographics (18)
+   :              :     :     +- BroadcastExchange (27)
+   :              :     :        +- * Filter (26)
+   :              :     :           +- * ColumnarToRow (25)
+   :              :     :              +- Scan parquet default.customer_address (24)
+   :              :     +- BroadcastExchange (34)
+   :              :        +- * Project (33)
+   :              :           +- * Filter (32)
+   :              :              +- * ColumnarToRow (31)
+   :              :                 +- Scan parquet default.date_dim (30)
+   :              +- BroadcastExchange (40)
+   :                 +- * Filter (39)
+   :                    +- * ColumnarToRow (38)
+   :                       +- Scan parquet default.item (37)
+   :- * HashAggregate (72)
+   :  +- Exchange (71)
+   :     +- * HashAggregate (70)
+   :        +- * Project (69)
+   :           +- * BroadcastHashJoin Inner BuildRight (68)
+   :              :- * Project (66)
+   :              :  +- * BroadcastHashJoin Inner BuildRight (65)
+   :              :     :- * Project (63)
+   :              :     :  +- * BroadcastHashJoin Inner BuildRight (62)
+   :              :     :     :- * Project (57)
+   :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (56)
+   :              :     :     :     :- * Project (54)
+   :              :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (53)
+   :              :     :     :     :     :- * Project (51)
+   :              :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (50)
+   :              :     :     :     :     :     :- * Filter (48)
+   :              :     :     :     :     :     :  +- * ColumnarToRow (47)
+   :              :     :     :     :     :     :     +- Scan parquet default.catalog_sales (46)
+   :              :     :     :     :     :     +- ReusedExchange (49)
+   :              :     :     :     :     +- ReusedExchange (52)
+   :              :     :     :     +- ReusedExchange (55)
+   :              :     :     +- BroadcastExchange (61)
+   :              :     :        +- * Filter (60)
+   :              :     :           +- * ColumnarToRow (59)
+   :              :     :              +- Scan parquet default.customer_address (58)
+   :              :     +- ReusedExchange (64)
+   :              +- ReusedExchange (67)
+   :- * HashAggregate (100)
+   :  +- Exchange (99)
+   :     +- * HashAggregate (98)
+   :        +- * Project (97)
+   :           +- * BroadcastHashJoin Inner BuildRight (96)
+   :              :- * Project (94)
+   :              :  +- * BroadcastHashJoin Inner BuildRight (93)
+   :              :     :- * Project (91)
+   :              :     :  +- * BroadcastHashJoin Inner BuildRight (90)
+   :              :     :     :- * Project (84)
+   :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (83)
+   :              :     :     :     :- * Project (81)
+   :              :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (80)
+   :              :     :     :     :     :- * Project (78)
+   :              :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (77)
+   :              :     :     :     :     :     :- * Filter (75)
+   :              :     :     :     :     :     :  +- * ColumnarToRow (74)
+   :              :     :     :     :     :     :     +- Scan parquet default.catalog_sales (73)
+   :              :     :     :     :     :     +- ReusedExchange (76)
+   :              :     :     :     :     +- ReusedExchange (79)
+   :              :     :     :     +- ReusedExchange (82)
+   :              :     :     +- BroadcastExchange (89)
+   :              :     :        +- * Project (88)
+   :              :     :           +- * Filter (87)
+   :              :     :              +- * ColumnarToRow (86)
+   :              :     :                 +- Scan parquet default.customer_address (85)
+   :              :     +- ReusedExchange (92)
+   :              +- ReusedExchange (95)
+   :- * HashAggregate (128)
+   :  +- Exchange (127)
+   :     +- * HashAggregate (126)
+   :        +- * Project (125)
+   :           +- * BroadcastHashJoin Inner BuildRight (124)
+   :              :- * Project (122)
+   :              :  +- * BroadcastHashJoin Inner BuildRight (121)
+   :              :     :- * Project (119)
+   :              :     :  +- * BroadcastHashJoin Inner BuildRight (118)
+   :              :     :     :- * Project (112)
+   :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (111)
+   :              :     :     :     :- * Project (109)
+   :              :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (108)
+   :              :     :     :     :     :- * Project (106)
+   :              :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (105)
+   :              :     :     :     :     :     :- * Filter (103)
+   :              :     :     :     :     :     :  +- * ColumnarToRow (102)
+   :              :     :     :     :     :     :     +- Scan parquet default.catalog_sales (101)
+   :              :     :     :     :     :     +- ReusedExchange (104)
+   :              :     :     :     :     +- ReusedExchange (107)
+   :              :     :     :     +- ReusedExchange (110)
+   :              :     :     +- BroadcastExchange (117)
+   :              :     :        +- * Project (116)
+   :              :     :           +- * Filter (115)
+   :              :     :              +- * ColumnarToRow (114)
+   :              :     :                 +- Scan parquet default.customer_address (113)
+   :              :     +- ReusedExchange (120)
+   :              +- ReusedExchange (123)
+   +- * HashAggregate (155)
+      +- Exchange (154)
+         +- * HashAggregate (153)
+            +- * Project (152)
+               +- * BroadcastHashJoin Inner BuildRight (151)
+                  :- * Project (146)
+                  :  +- * BroadcastHashJoin Inner BuildRight (145)
+                  :     :- * Project (143)
+                  :     :  +- * BroadcastHashJoin Inner BuildRight (142)
+                  :     :     :- * Project (140)
+                  :     :     :  +- * BroadcastHashJoin Inner BuildRight (139)
+                  :     :     :     :- * Project (137)
+                  :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (136)
+                  :     :     :     :     :- * Project (134)
+                  :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (133)
+                  :     :     :     :     :     :- * Filter (131)
+                  :     :     :     :     :     :  +- * ColumnarToRow (130)
+                  :     :     :     :     :     :     +- Scan parquet default.catalog_sales (129)
+                  :     :     :     :     :     +- ReusedExchange (132)
+                  :     :     :     :     +- ReusedExchange (135)
+                  :     :     :     +- ReusedExchange (138)
+                  :     :     +- ReusedExchange (141)
+                  :     +- ReusedExchange (144)
+                  +- BroadcastExchange (150)
+                     +- * Filter (149)
+                        +- * ColumnarToRow (148)
+                           +- Scan parquet default.item (147)
+
+
+(1) Scan parquet default.catalog_sales
+Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#9), dynamicpruningexpression(cs_sold_date_sk#9 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_bill_cdemo_sk:int,cs_item_sk:int,cs_quantity:int,cs_list_price:decimal(7,2),cs_sales_price:decimal(7,2),cs_coupon_amt:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 7]
+Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9]
+
+(3) Filter [codegen id : 7]
+Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9]
+Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3))
+
+(4) Scan parquet default.customer_demographics
+Output [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_education_status,College             ), IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_education_status:string,cd_dep_count:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14]
+
+(6) Filter [codegen id : 1]
+Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14]
+Condition : ((((isnotnull(cd_gender#12) AND isnotnull(cd_education_status#13)) AND (cd_gender#12 = M)) AND (cd_education_status#13 = College             )) AND isnotnull(cd_demo_sk#11))
+
+(7) Project [codegen id : 1]
+Output [2]: [cd_demo_sk#11, cd_dep_count#14]
+Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14]
+
+(8) BroadcastExchange
+Input [2]: [cd_demo_sk#11, cd_dep_count#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#15]
+
+(9) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [cs_bill_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#11]
+Join condition: None
+
+(10) Project [codegen id : 7]
+Output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14]
+Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#11, cd_dep_count#14]
+
+(11) Scan parquet default.customer
+Output [5]: [c_customer_sk#16, c_current_cdemo_sk#17, c_current_addr_sk#18, c_birth_month#19, c_birth_year#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [In(c_birth_month, [9,5,12,4,1,10]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_addr_sk:int,c_birth_month:int,c_birth_year:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [5]: [c_customer_sk#16, c_current_cdemo_sk#17, c_current_addr_sk#18, c_birth_month#19, c_birth_year#20]
+
+(13) Filter [codegen id : 2]
+Input [5]: [c_customer_sk#16, c_current_cdemo_sk#17, c_current_addr_sk#18, c_birth_month#19, c_birth_year#20]
+Condition : (((c_birth_month#19 IN (9,5,12,4,1,10) AND isnotnull(c_customer_sk#16)) AND isnotnull(c_current_cdemo_sk#17)) AND isnotnull(c_current_addr_sk#18))
+
+(14) Project [codegen id : 2]
+Output [4]: [c_customer_sk#16, c_current_cdemo_sk#17, c_current_addr_sk#18, c_birth_year#20]
+Input [5]: [c_customer_sk#16, c_current_cdemo_sk#17, c_current_addr_sk#18, c_birth_month#19, c_birth_year#20]
+
+(15) BroadcastExchange
+Input [4]: [c_customer_sk#16, c_current_cdemo_sk#17, c_current_addr_sk#18, c_birth_year#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(16) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [cs_bill_customer_sk#1]
+Right keys [1]: [c_customer_sk#16]
+Join condition: None
+
+(17) Project [codegen id : 7]
+Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#17, c_current_addr_sk#18, c_birth_year#20]
+Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_customer_sk#16, c_current_cdemo_sk#17, c_current_addr_sk#18, c_birth_year#20]
+
+(18) Scan parquet default.customer_demographics
+Output [1]: [cd_demo_sk#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [1]: [cd_demo_sk#22]
+
+(20) Filter [codegen id : 3]
+Input [1]: [cd_demo_sk#22]
+Condition : isnotnull(cd_demo_sk#22)
+
+(21) BroadcastExchange
+Input [1]: [cd_demo_sk#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#23]
+
+(22) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [c_current_cdemo_sk#17]
+Right keys [1]: [cd_demo_sk#22]
+Join condition: None
+
+(23) Project [codegen id : 7]
+Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#18, c_birth_year#20]
+Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#17, c_current_addr_sk#18, c_birth_year#20, cd_demo_sk#22]
+
+(24) Scan parquet default.customer_address
+Output [4]: [ca_address_sk#24, ca_county#25, ca_state#26, ca_country#27]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [In(ca_state, [ND,WI,AL,NC,OK,MS,TN]), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_county:string,ca_state:string,ca_country:string>
+
+(25) ColumnarToRow [codegen id : 4]
+Input [4]: [ca_address_sk#24, ca_county#25, ca_state#26, ca_country#27]
+
+(26) Filter [codegen id : 4]
+Input [4]: [ca_address_sk#24, ca_county#25, ca_state#26, ca_country#27]
+Condition : (ca_state#26 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#24))
+
+(27) BroadcastExchange
+Input [4]: [ca_address_sk#24, ca_county#25, ca_state#26, ca_country#27]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#28]
+
+(28) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [c_current_addr_sk#18]
+Right keys [1]: [ca_address_sk#24]
+Join condition: None
+
+(29) Project [codegen id : 7]
+Output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#20, ca_county#25, ca_state#26, ca_country#27]
+Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#18, c_birth_year#20, ca_address_sk#24, ca_county#25, ca_state#26, ca_country#27]
+
+(30) Scan parquet default.date_dim
+Output [2]: [d_date_sk#29, d_year#30]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(31) ColumnarToRow [codegen id : 5]
+Input [2]: [d_date_sk#29, d_year#30]
+
+(32) Filter [codegen id : 5]
+Input [2]: [d_date_sk#29, d_year#30]
+Condition : ((isnotnull(d_year#30) AND (d_year#30 = 2001)) AND isnotnull(d_date_sk#29))
+
+(33) Project [codegen id : 5]
+Output [1]: [d_date_sk#29]
+Input [2]: [d_date_sk#29, d_year#30]
+
+(34) BroadcastExchange
+Input [1]: [d_date_sk#29]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#31]
+
+(35) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [cs_sold_date_sk#9]
+Right keys [1]: [d_date_sk#29]
+Join condition: None
+
+(36) Project [codegen id : 7]
+Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#20, ca_county#25, ca_state#26, ca_country#27]
+Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#20, ca_county#25, ca_state#26, ca_country#27, d_date_sk#29]
+
+(37) Scan parquet default.item
+Output [2]: [i_item_sk#32, i_item_id#33]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(38) ColumnarToRow [codegen id : 6]
+Input [2]: [i_item_sk#32, i_item_id#33]
+
+(39) Filter [codegen id : 6]
+Input [2]: [i_item_sk#32, i_item_id#33]
+Condition : isnotnull(i_item_sk#32)
+
+(40) BroadcastExchange
+Input [2]: [i_item_sk#32, i_item_id#33]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#34]
+
+(41) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [cs_item_sk#3]
+Right keys [1]: [i_item_sk#32]
+Join condition: None
+
+(42) Project [codegen id : 7]
+Output [11]: [i_item_id#33, ca_country#27, ca_state#26, ca_county#25, cast(cs_quantity#4 as decimal(12,2)) AS agg1#35, cast(cs_list_price#5 as decimal(12,2)) AS agg2#36, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#37, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#38, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#39, cast(c_birth_year#20 as decimal(12,2)) AS agg6#40, cast(cd_dep_count#14 as decimal(12,2)) AS agg7#41]
+Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#20, ca_county#25, ca_state#26, ca_country#27, i_item_sk#32, i_item_id#33]
+
+(43) HashAggregate [codegen id : 7]
+Input [11]: [i_item_id#33, ca_country#27, ca_state#26, ca_county#25, agg1#35, agg2#36, agg3#37, agg4#38, agg5#39, agg6#40, agg7#41]
+Keys [4]: [i_item_id#33, ca_country#27, ca_state#26, ca_county#25]
+Functions [7]: [partial_avg(agg1#35), partial_avg(agg2#36), partial_avg(agg3#37), partial_avg(agg4#38), partial_avg(agg5#39), partial_avg(agg6#40), partial_avg(agg7#41)]
+Aggregate Attributes [14]: [sum#42, count#43, sum#44, count#45, sum#46, count#47, sum#48, count#49, sum#50, count#51, sum#52, count#53, sum#54, count#55]
+Results [18]: [i_item_id#33, ca_country#27, ca_state#26, ca_county#25, sum#56, count#57, sum#58, count#59, sum#60, count#61, sum#62, count#63, sum#64, count#65, sum#66, count#67, sum#68, count#69]
+
+(44) Exchange
+Input [18]: [i_item_id#33, ca_country#27, ca_state#26, ca_county#25, sum#56, count#57, sum#58, count#59, sum#60, count#61, sum#62, count#63, sum#64, count#65, sum#66, count#67, sum#68, count#69]
+Arguments: hashpartitioning(i_item_id#33, ca_country#27, ca_state#26, ca_county#25, 5), ENSURE_REQUIREMENTS, [id=#70]
+
+(45) HashAggregate [codegen id : 8]
+Input [18]: [i_item_id#33, ca_country#27, ca_state#26, ca_county#25, sum#56, count#57, sum#58, count#59, sum#60, count#61, sum#62, count#63, sum#64, count#65, sum#66, count#67, sum#68, count#69]
+Keys [4]: [i_item_id#33, ca_country#27, ca_state#26, ca_county#25]
+Functions [7]: [avg(agg1#35), avg(agg2#36), avg(agg3#37), avg(agg4#38), avg(agg5#39), avg(agg6#40), avg(agg7#41)]
+Aggregate Attributes [7]: [avg(agg1#35)#71, avg(agg2#36)#72, avg(agg3#37)#73, avg(agg4#38)#74, avg(agg5#39)#75, avg(agg6#40)#76, avg(agg7#41)#77]
+Results [11]: [i_item_id#33, ca_country#27, ca_state#26, ca_county#25, avg(agg1#35)#71 AS agg1#78, avg(agg2#36)#72 AS agg2#79, avg(agg3#37)#73 AS agg3#80, avg(agg4#38)#74 AS agg4#81, avg(agg5#39)#75 AS agg5#82, avg(agg6#40)#76 AS agg6#83, avg(agg7#41)#77 AS agg7#84]
+
+(46) Scan parquet default.catalog_sales
+Output [9]: [cs_bill_customer_sk#85, cs_bill_cdemo_sk#86, cs_item_sk#87, cs_quantity#88, cs_list_price#89, cs_sales_price#90, cs_coupon_amt#91, cs_net_profit#92, cs_sold_date_sk#93]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#93), dynamicpruningexpression(cs_sold_date_sk#93 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_bill_cdemo_sk:int,cs_item_sk:int,cs_quantity:int,cs_list_price:decimal(7,2),cs_sales_price:decimal(7,2),cs_coupon_amt:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(47) ColumnarToRow [codegen id : 15]
+Input [9]: [cs_bill_customer_sk#85, cs_bill_cdemo_sk#86, cs_item_sk#87, cs_quantity#88, cs_list_price#89, cs_sales_price#90, cs_coupon_amt#91, cs_net_profit#92, cs_sold_date_sk#93]
+
+(48) Filter [codegen id : 15]
+Input [9]: [cs_bill_customer_sk#85, cs_bill_cdemo_sk#86, cs_item_sk#87, cs_quantity#88, cs_list_price#89, cs_sales_price#90, cs_coupon_amt#91, cs_net_profit#92, cs_sold_date_sk#93]
+Condition : ((isnotnull(cs_bill_cdemo_sk#86) AND isnotnull(cs_bill_customer_sk#85)) AND isnotnull(cs_item_sk#87))
+
+(49) ReusedExchange [Reuses operator id: 8]
+Output [2]: [cd_demo_sk#94, cd_dep_count#95]
+
+(50) BroadcastHashJoin [codegen id : 15]
+Left keys [1]: [cs_bill_cdemo_sk#86]
+Right keys [1]: [cd_demo_sk#94]
+Join condition: None
+
+(51) Project [codegen id : 15]
+Output [9]: [cs_bill_customer_sk#85, cs_item_sk#87, cs_quantity#88, cs_list_price#89, cs_sales_price#90, cs_coupon_amt#91, cs_net_profit#92, cs_sold_date_sk#93, cd_dep_count#95]
+Input [11]: [cs_bill_customer_sk#85, cs_bill_cdemo_sk#86, cs_item_sk#87, cs_quantity#88, cs_list_price#89, cs_sales_price#90, cs_coupon_amt#91, cs_net_profit#92, cs_sold_date_sk#93, cd_demo_sk#94, cd_dep_count#95]
+
+(52) ReusedExchange [Reuses operator id: 15]
+Output [4]: [c_customer_sk#96, c_current_cdemo_sk#97, c_current_addr_sk#98, c_birth_year#99]
+
+(53) BroadcastHashJoin [codegen id : 15]
+Left keys [1]: [cs_bill_customer_sk#85]
+Right keys [1]: [c_customer_sk#96]
+Join condition: None
+
+(54) Project [codegen id : 15]
+Output [11]: [cs_item_sk#87, cs_quantity#88, cs_list_price#89, cs_sales_price#90, cs_coupon_amt#91, cs_net_profit#92, cs_sold_date_sk#93, cd_dep_count#95, c_current_cdemo_sk#97, c_current_addr_sk#98, c_birth_year#99]
+Input [13]: [cs_bill_customer_sk#85, cs_item_sk#87, cs_quantity#88, cs_list_price#89, cs_sales_price#90, cs_coupon_amt#91, cs_net_profit#92, cs_sold_date_sk#93, cd_dep_count#95, c_customer_sk#96, c_current_cdemo_sk#97, c_current_addr_sk#98, c_birth_year#99]
+
+(55) ReusedExchange [Reuses operator id: 21]
+Output [1]: [cd_demo_sk#100]
+
+(56) BroadcastHashJoin [codegen id : 15]
+Left keys [1]: [c_current_cdemo_sk#97]
+Right keys [1]: [cd_demo_sk#100]
+Join condition: None
+
+(57) Project [codegen id : 15]
+Output [10]: [cs_item_sk#87, cs_quantity#88, cs_list_price#89, cs_sales_price#90, cs_coupon_amt#91, cs_net_profit#92, cs_sold_date_sk#93, cd_dep_count#95, c_current_addr_sk#98, c_birth_year#99]
+Input [12]: [cs_item_sk#87, cs_quantity#88, cs_list_price#89, cs_sales_price#90, cs_coupon_amt#91, cs_net_profit#92, cs_sold_date_sk#93, cd_dep_count#95, c_current_cdemo_sk#97, c_current_addr_sk#98, c_birth_year#99, cd_demo_sk#100]
+
+(58) Scan parquet default.customer_address
+Output [3]: [ca_address_sk#101, ca_state#102, ca_country#103]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [In(ca_state, [ND,WI,AL,NC,OK,MS,TN]), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string,ca_country:string>
+
+(59) ColumnarToRow [codegen id : 12]
+Input [3]: [ca_address_sk#101, ca_state#102, ca_country#103]
+
+(60) Filter [codegen id : 12]
+Input [3]: [ca_address_sk#101, ca_state#102, ca_country#103]
+Condition : (ca_state#102 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#101))
+
+(61) BroadcastExchange
+Input [3]: [ca_address_sk#101, ca_state#102, ca_country#103]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#104]
+
+(62) BroadcastHashJoin [codegen id : 15]
+Left keys [1]: [c_current_addr_sk#98]
+Right keys [1]: [ca_address_sk#101]
+Join condition: None
+
+(63) Project [codegen id : 15]
+Output [11]: [cs_item_sk#87, cs_quantity#88, cs_list_price#89, cs_sales_price#90, cs_coupon_amt#91, cs_net_profit#92, cs_sold_date_sk#93, cd_dep_count#95, c_birth_year#99, ca_state#102, ca_country#103]
+Input [13]: [cs_item_sk#87, cs_quantity#88, cs_list_price#89, cs_sales_price#90, cs_coupon_amt#91, cs_net_profit#92, cs_sold_date_sk#93, cd_dep_count#95, c_current_addr_sk#98, c_birth_year#99, ca_address_sk#101, ca_state#102, ca_country#103]
+
+(64) ReusedExchange [Reuses operator id: 34]
+Output [1]: [d_date_sk#105]
+
+(65) BroadcastHashJoin [codegen id : 15]
+Left keys [1]: [cs_sold_date_sk#93]
+Right keys [1]: [d_date_sk#105]
+Join condition: None
+
+(66) Project [codegen id : 15]
+Output [10]: [cs_item_sk#87, cs_quantity#88, cs_list_price#89, cs_sales_price#90, cs_coupon_amt#91, cs_net_profit#92, cd_dep_count#95, c_birth_year#99, ca_state#102, ca_country#103]
+Input [12]: [cs_item_sk#87, cs_quantity#88, cs_list_price#89, cs_sales_price#90, cs_coupon_amt#91, cs_net_profit#92, cs_sold_date_sk#93, cd_dep_count#95, c_birth_year#99, ca_state#102, ca_country#103, d_date_sk#105]
+
+(67) ReusedExchange [Reuses operator id: 40]
+Output [2]: [i_item_sk#106, i_item_id#107]
+
+(68) BroadcastHashJoin [codegen id : 15]
+Left keys [1]: [cs_item_sk#87]
+Right keys [1]: [i_item_sk#106]
+Join condition: None
+
+(69) Project [codegen id : 15]
+Output [10]: [i_item_id#107, ca_country#103, ca_state#102, cast(cs_quantity#88 as decimal(12,2)) AS agg1#35, cast(cs_list_price#89 as decimal(12,2)) AS agg2#36, cast(cs_coupon_amt#91 as decimal(12,2)) AS agg3#37, cast(cs_sales_price#90 as decimal(12,2)) AS agg4#38, cast(cs_net_profit#92 as decimal(12,2)) AS agg5#39, cast(c_birth_year#99 as decimal(12,2)) AS agg6#40, cast(cd_dep_count#95 as decimal(12,2)) AS agg7#41]
+Input [12]: [cs_item_sk#87, cs_quantity#88, cs_list_price#89, cs_sales_price#90, cs_coupon_amt#91, cs_net_profit#92, cd_dep_count#95, c_birth_year#99, ca_state#102, ca_country#103, i_item_sk#106, i_item_id#107]
+
+(70) HashAggregate [codegen id : 15]
+Input [10]: [i_item_id#107, ca_country#103, ca_state#102, agg1#35, agg2#36, agg3#37, agg4#38, agg5#39, agg6#40, agg7#41]
+Keys [3]: [i_item_id#107, ca_country#103, ca_state#102]
+Functions [7]: [partial_avg(agg1#35), partial_avg(agg2#36), partial_avg(agg3#37), partial_avg(agg4#38), partial_avg(agg5#39), partial_avg(agg6#40), partial_avg(agg7#41)]
+Aggregate Attributes [14]: [sum#108, count#109, sum#110, count#111, sum#112, count#113, sum#114, count#115, sum#116, count#117, sum#118, count#119, sum#120, count#121]
+Results [17]: [i_item_id#107, ca_country#103, ca_state#102, sum#122, count#123, sum#124, count#125, sum#126, count#127, sum#128, count#129, sum#130, count#131, sum#132, count#133, sum#134, count#135]
+
+(71) Exchange
+Input [17]: [i_item_id#107, ca_country#103, ca_state#102, sum#122, count#123, sum#124, count#125, sum#126, count#127, sum#128, count#129, sum#130, count#131, sum#132, count#133, sum#134, count#135]
+Arguments: hashpartitioning(i_item_id#107, ca_country#103, ca_state#102, 5), ENSURE_REQUIREMENTS, [id=#136]
+
+(72) HashAggregate [codegen id : 16]
+Input [17]: [i_item_id#107, ca_country#103, ca_state#102, sum#122, count#123, sum#124, count#125, sum#126, count#127, sum#128, count#129, sum#130, count#131, sum#132, count#133, sum#134, count#135]
+Keys [3]: [i_item_id#107, ca_country#103, ca_state#102]
+Functions [7]: [avg(agg1#35), avg(agg2#36), avg(agg3#37), avg(agg4#38), avg(agg5#39), avg(agg6#40), avg(agg7#41)]
+Aggregate Attributes [7]: [avg(agg1#35)#137, avg(agg2#36)#138, avg(agg3#37)#139, avg(agg4#38)#140, avg(agg5#39)#141, avg(agg6#40)#142, avg(agg7#41)#143]
+Results [11]: [i_item_id#107, ca_country#103, ca_state#102, null AS county#144, avg(agg1#35)#137 AS agg1#145, avg(agg2#36)#138 AS agg2#146, avg(agg3#37)#139 AS agg3#147, avg(agg4#38)#140 AS agg4#148, avg(agg5#39)#141 AS agg5#149, avg(agg6#40)#142 AS agg6#150, avg(agg7#41)#143 AS agg7#151]
+
+(73) Scan parquet default.catalog_sales
+Output [9]: [cs_bill_customer_sk#152, cs_bill_cdemo_sk#153, cs_item_sk#154, cs_quantity#155, cs_list_price#156, cs_sales_price#157, cs_coupon_amt#158, cs_net_profit#159, cs_sold_date_sk#160]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#160), dynamicpruningexpression(cs_sold_date_sk#160 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_bill_cdemo_sk:int,cs_item_sk:int,cs_quantity:int,cs_list_price:decimal(7,2),cs_sales_price:decimal(7,2),cs_coupon_amt:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(74) ColumnarToRow [codegen id : 23]
+Input [9]: [cs_bill_customer_sk#152, cs_bill_cdemo_sk#153, cs_item_sk#154, cs_quantity#155, cs_list_price#156, cs_sales_price#157, cs_coupon_amt#158, cs_net_profit#159, cs_sold_date_sk#160]
+
+(75) Filter [codegen id : 23]
+Input [9]: [cs_bill_customer_sk#152, cs_bill_cdemo_sk#153, cs_item_sk#154, cs_quantity#155, cs_list_price#156, cs_sales_price#157, cs_coupon_amt#158, cs_net_profit#159, cs_sold_date_sk#160]
+Condition : ((isnotnull(cs_bill_cdemo_sk#153) AND isnotnull(cs_bill_customer_sk#152)) AND isnotnull(cs_item_sk#154))
+
+(76) ReusedExchange [Reuses operator id: 8]
+Output [2]: [cd_demo_sk#161, cd_dep_count#162]
+
+(77) BroadcastHashJoin [codegen id : 23]
+Left keys [1]: [cs_bill_cdemo_sk#153]
+Right keys [1]: [cd_demo_sk#161]
+Join condition: None
+
+(78) Project [codegen id : 23]
+Output [9]: [cs_bill_customer_sk#152, cs_item_sk#154, cs_quantity#155, cs_list_price#156, cs_sales_price#157, cs_coupon_amt#158, cs_net_profit#159, cs_sold_date_sk#160, cd_dep_count#162]
+Input [11]: [cs_bill_customer_sk#152, cs_bill_cdemo_sk#153, cs_item_sk#154, cs_quantity#155, cs_list_price#156, cs_sales_price#157, cs_coupon_amt#158, cs_net_profit#159, cs_sold_date_sk#160, cd_demo_sk#161, cd_dep_count#162]
+
+(79) ReusedExchange [Reuses operator id: 15]
+Output [4]: [c_customer_sk#163, c_current_cdemo_sk#164, c_current_addr_sk#165, c_birth_year#166]
+
+(80) BroadcastHashJoin [codegen id : 23]
+Left keys [1]: [cs_bill_customer_sk#152]
+Right keys [1]: [c_customer_sk#163]
+Join condition: None
+
+(81) Project [codegen id : 23]
+Output [11]: [cs_item_sk#154, cs_quantity#155, cs_list_price#156, cs_sales_price#157, cs_coupon_amt#158, cs_net_profit#159, cs_sold_date_sk#160, cd_dep_count#162, c_current_cdemo_sk#164, c_current_addr_sk#165, c_birth_year#166]
+Input [13]: [cs_bill_customer_sk#152, cs_item_sk#154, cs_quantity#155, cs_list_price#156, cs_sales_price#157, cs_coupon_amt#158, cs_net_profit#159, cs_sold_date_sk#160, cd_dep_count#162, c_customer_sk#163, c_current_cdemo_sk#164, c_current_addr_sk#165, c_birth_year#166]
+
+(82) ReusedExchange [Reuses operator id: 21]
+Output [1]: [cd_demo_sk#167]
+
+(83) BroadcastHashJoin [codegen id : 23]
+Left keys [1]: [c_current_cdemo_sk#164]
+Right keys [1]: [cd_demo_sk#167]
+Join condition: None
+
+(84) Project [codegen id : 23]
+Output [10]: [cs_item_sk#154, cs_quantity#155, cs_list_price#156, cs_sales_price#157, cs_coupon_amt#158, cs_net_profit#159, cs_sold_date_sk#160, cd_dep_count#162, c_current_addr_sk#165, c_birth_year#166]
+Input [12]: [cs_item_sk#154, cs_quantity#155, cs_list_price#156, cs_sales_price#157, cs_coupon_amt#158, cs_net_profit#159, cs_sold_date_sk#160, cd_dep_count#162, c_current_cdemo_sk#164, c_current_addr_sk#165, c_birth_year#166, cd_demo_sk#167]
+
+(85) Scan parquet default.customer_address
+Output [3]: [ca_address_sk#168, ca_state#169, ca_country#170]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [In(ca_state, [ND,WI,AL,NC,OK,MS,TN]), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string,ca_country:string>
+
+(86) ColumnarToRow [codegen id : 20]
+Input [3]: [ca_address_sk#168, ca_state#169, ca_country#170]
+
+(87) Filter [codegen id : 20]
+Input [3]: [ca_address_sk#168, ca_state#169, ca_country#170]
+Condition : (ca_state#169 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#168))
+
+(88) Project [codegen id : 20]
+Output [2]: [ca_address_sk#168, ca_country#170]
+Input [3]: [ca_address_sk#168, ca_state#169, ca_country#170]
+
+(89) BroadcastExchange
+Input [2]: [ca_address_sk#168, ca_country#170]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#171]
+
+(90) BroadcastHashJoin [codegen id : 23]
+Left keys [1]: [c_current_addr_sk#165]
+Right keys [1]: [ca_address_sk#168]
+Join condition: None
+
+(91) Project [codegen id : 23]
+Output [10]: [cs_item_sk#154, cs_quantity#155, cs_list_price#156, cs_sales_price#157, cs_coupon_amt#158, cs_net_profit#159, cs_sold_date_sk#160, cd_dep_count#162, c_birth_year#166, ca_country#170]
+Input [12]: [cs_item_sk#154, cs_quantity#155, cs_list_price#156, cs_sales_price#157, cs_coupon_amt#158, cs_net_profit#159, cs_sold_date_sk#160, cd_dep_count#162, c_current_addr_sk#165, c_birth_year#166, ca_address_sk#168, ca_country#170]
+
+(92) ReusedExchange [Reuses operator id: 34]
+Output [1]: [d_date_sk#172]
+
+(93) BroadcastHashJoin [codegen id : 23]
+Left keys [1]: [cs_sold_date_sk#160]
+Right keys [1]: [d_date_sk#172]
+Join condition: None
+
+(94) Project [codegen id : 23]
+Output [9]: [cs_item_sk#154, cs_quantity#155, cs_list_price#156, cs_sales_price#157, cs_coupon_amt#158, cs_net_profit#159, cd_dep_count#162, c_birth_year#166, ca_country#170]
+Input [11]: [cs_item_sk#154, cs_quantity#155, cs_list_price#156, cs_sales_price#157, cs_coupon_amt#158, cs_net_profit#159, cs_sold_date_sk#160, cd_dep_count#162, c_birth_year#166, ca_country#170, d_date_sk#172]
+
+(95) ReusedExchange [Reuses operator id: 40]
+Output [2]: [i_item_sk#173, i_item_id#174]
+
+(96) BroadcastHashJoin [codegen id : 23]
+Left keys [1]: [cs_item_sk#154]
+Right keys [1]: [i_item_sk#173]
+Join condition: None
+
+(97) Project [codegen id : 23]
+Output [9]: [i_item_id#174, ca_country#170, cast(cs_quantity#155 as decimal(12,2)) AS agg1#35, cast(cs_list_price#156 as decimal(12,2)) AS agg2#36, cast(cs_coupon_amt#158 as decimal(12,2)) AS agg3#37, cast(cs_sales_price#157 as decimal(12,2)) AS agg4#38, cast(cs_net_profit#159 as decimal(12,2)) AS agg5#39, cast(c_birth_year#166 as decimal(12,2)) AS agg6#40, cast(cd_dep_count#162 as decimal(12,2)) AS agg7#41]
+Input [11]: [cs_item_sk#154, cs_quantity#155, cs_list_price#156, cs_sales_price#157, cs_coupon_amt#158, cs_net_profit#159, cd_dep_count#162, c_birth_year#166, ca_country#170, i_item_sk#173, i_item_id#174]
+
+(98) HashAggregate [codegen id : 23]
+Input [9]: [i_item_id#174, ca_country#170, agg1#35, agg2#36, agg3#37, agg4#38, agg5#39, agg6#40, agg7#41]
+Keys [2]: [i_item_id#174, ca_country#170]
+Functions [7]: [partial_avg(agg1#35), partial_avg(agg2#36), partial_avg(agg3#37), partial_avg(agg4#38), partial_avg(agg5#39), partial_avg(agg6#40), partial_avg(agg7#41)]
+Aggregate Attributes [14]: [sum#175, count#176, sum#177, count#178, sum#179, count#180, sum#181, count#182, sum#183, count#184, sum#185, count#186, sum#187, count#188]
+Results [16]: [i_item_id#174, ca_country#170, sum#189, count#190, sum#191, count#192, sum#193, count#194, sum#195, count#196, sum#197, count#198, sum#199, count#200, sum#201, count#202]
+
+(99) Exchange
+Input [16]: [i_item_id#174, ca_country#170, sum#189, count#190, sum#191, count#192, sum#193, count#194, sum#195, count#196, sum#197, count#198, sum#199, count#200, sum#201, count#202]
+Arguments: hashpartitioning(i_item_id#174, ca_country#170, 5), ENSURE_REQUIREMENTS, [id=#203]
+
+(100) HashAggregate [codegen id : 24]
+Input [16]: [i_item_id#174, ca_country#170, sum#189, count#190, sum#191, count#192, sum#193, count#194, sum#195, count#196, sum#197, count#198, sum#199, count#200, sum#201, count#202]
+Keys [2]: [i_item_id#174, ca_country#170]
+Functions [7]: [avg(agg1#35), avg(agg2#36), avg(agg3#37), avg(agg4#38), avg(agg5#39), avg(agg6#40), avg(agg7#41)]
+Aggregate Attributes [7]: [avg(agg1#35)#204, avg(agg2#36)#205, avg(agg3#37)#206, avg(agg4#38)#207, avg(agg5#39)#208, avg(agg6#40)#209, avg(agg7#41)#210]
+Results [11]: [i_item_id#174, ca_country#170, null AS ca_state#211, null AS county#212, avg(agg1#35)#204 AS agg1#213, avg(agg2#36)#205 AS agg2#214, avg(agg3#37)#206 AS agg3#215, avg(agg4#38)#207 AS agg4#216, avg(agg5#39)#208 AS agg5#217, avg(agg6#40)#209 AS agg6#218, avg(agg7#41)#210 AS agg7#219]
+
+(101) Scan parquet default.catalog_sales
+Output [9]: [cs_bill_customer_sk#220, cs_bill_cdemo_sk#221, cs_item_sk#222, cs_quantity#223, cs_list_price#224, cs_sales_price#225, cs_coupon_amt#226, cs_net_profit#227, cs_sold_date_sk#228]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#228), dynamicpruningexpression(cs_sold_date_sk#228 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_bill_cdemo_sk:int,cs_item_sk:int,cs_quantity:int,cs_list_price:decimal(7,2),cs_sales_price:decimal(7,2),cs_coupon_amt:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(102) ColumnarToRow [codegen id : 31]
+Input [9]: [cs_bill_customer_sk#220, cs_bill_cdemo_sk#221, cs_item_sk#222, cs_quantity#223, cs_list_price#224, cs_sales_price#225, cs_coupon_amt#226, cs_net_profit#227, cs_sold_date_sk#228]
+
+(103) Filter [codegen id : 31]
+Input [9]: [cs_bill_customer_sk#220, cs_bill_cdemo_sk#221, cs_item_sk#222, cs_quantity#223, cs_list_price#224, cs_sales_price#225, cs_coupon_amt#226, cs_net_profit#227, cs_sold_date_sk#228]
+Condition : ((isnotnull(cs_bill_cdemo_sk#221) AND isnotnull(cs_bill_customer_sk#220)) AND isnotnull(cs_item_sk#222))
+
+(104) ReusedExchange [Reuses operator id: 8]
+Output [2]: [cd_demo_sk#229, cd_dep_count#230]
+
+(105) BroadcastHashJoin [codegen id : 31]
+Left keys [1]: [cs_bill_cdemo_sk#221]
+Right keys [1]: [cd_demo_sk#229]
+Join condition: None
+
+(106) Project [codegen id : 31]
+Output [9]: [cs_bill_customer_sk#220, cs_item_sk#222, cs_quantity#223, cs_list_price#224, cs_sales_price#225, cs_coupon_amt#226, cs_net_profit#227, cs_sold_date_sk#228, cd_dep_count#230]
+Input [11]: [cs_bill_customer_sk#220, cs_bill_cdemo_sk#221, cs_item_sk#222, cs_quantity#223, cs_list_price#224, cs_sales_price#225, cs_coupon_amt#226, cs_net_profit#227, cs_sold_date_sk#228, cd_demo_sk#229, cd_dep_count#230]
+
+(107) ReusedExchange [Reuses operator id: 15]
+Output [4]: [c_customer_sk#231, c_current_cdemo_sk#232, c_current_addr_sk#233, c_birth_year#234]
+
+(108) BroadcastHashJoin [codegen id : 31]
+Left keys [1]: [cs_bill_customer_sk#220]
+Right keys [1]: [c_customer_sk#231]
+Join condition: None
+
+(109) Project [codegen id : 31]
+Output [11]: [cs_item_sk#222, cs_quantity#223, cs_list_price#224, cs_sales_price#225, cs_coupon_amt#226, cs_net_profit#227, cs_sold_date_sk#228, cd_dep_count#230, c_current_cdemo_sk#232, c_current_addr_sk#233, c_birth_year#234]
+Input [13]: [cs_bill_customer_sk#220, cs_item_sk#222, cs_quantity#223, cs_list_price#224, cs_sales_price#225, cs_coupon_amt#226, cs_net_profit#227, cs_sold_date_sk#228, cd_dep_count#230, c_customer_sk#231, c_current_cdemo_sk#232, c_current_addr_sk#233, c_birth_year#234]
+
+(110) ReusedExchange [Reuses operator id: 21]
+Output [1]: [cd_demo_sk#235]
+
+(111) BroadcastHashJoin [codegen id : 31]
+Left keys [1]: [c_current_cdemo_sk#232]
+Right keys [1]: [cd_demo_sk#235]
+Join condition: None
+
+(112) Project [codegen id : 31]
+Output [10]: [cs_item_sk#222, cs_quantity#223, cs_list_price#224, cs_sales_price#225, cs_coupon_amt#226, cs_net_profit#227, cs_sold_date_sk#228, cd_dep_count#230, c_current_addr_sk#233, c_birth_year#234]
+Input [12]: [cs_item_sk#222, cs_quantity#223, cs_list_price#224, cs_sales_price#225, cs_coupon_amt#226, cs_net_profit#227, cs_sold_date_sk#228, cd_dep_count#230, c_current_cdemo_sk#232, c_current_addr_sk#233, c_birth_year#234, cd_demo_sk#235]
+
+(113) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#236, ca_state#237]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [In(ca_state, [ND,WI,AL,NC,OK,MS,TN]), IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(114) ColumnarToRow [codegen id : 28]
+Input [2]: [ca_address_sk#236, ca_state#237]
+
+(115) Filter [codegen id : 28]
+Input [2]: [ca_address_sk#236, ca_state#237]
+Condition : (ca_state#237 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#236))
+
+(116) Project [codegen id : 28]
+Output [1]: [ca_address_sk#236]
+Input [2]: [ca_address_sk#236, ca_state#237]
+
+(117) BroadcastExchange
+Input [1]: [ca_address_sk#236]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#238]
+
+(118) BroadcastHashJoin [codegen id : 31]
+Left keys [1]: [c_current_addr_sk#233]
+Right keys [1]: [ca_address_sk#236]
+Join condition: None
+
+(119) Project [codegen id : 31]
+Output [9]: [cs_item_sk#222, cs_quantity#223, cs_list_price#224, cs_sales_price#225, cs_coupon_amt#226, cs_net_profit#227, cs_sold_date_sk#228, cd_dep_count#230, c_birth_year#234]
+Input [11]: [cs_item_sk#222, cs_quantity#223, cs_list_price#224, cs_sales_price#225, cs_coupon_amt#226, cs_net_profit#227, cs_sold_date_sk#228, cd_dep_count#230, c_current_addr_sk#233, c_birth_year#234, ca_address_sk#236]
+
+(120) ReusedExchange [Reuses operator id: 34]
+Output [1]: [d_date_sk#239]
+
+(121) BroadcastHashJoin [codegen id : 31]
+Left keys [1]: [cs_sold_date_sk#228]
+Right keys [1]: [d_date_sk#239]
+Join condition: None
+
+(122) Project [codegen id : 31]
+Output [8]: [cs_item_sk#222, cs_quantity#223, cs_list_price#224, cs_sales_price#225, cs_coupon_amt#226, cs_net_profit#227, cd_dep_count#230, c_birth_year#234]
+Input [10]: [cs_item_sk#222, cs_quantity#223, cs_list_price#224, cs_sales_price#225, cs_coupon_amt#226, cs_net_profit#227, cs_sold_date_sk#228, cd_dep_count#230, c_birth_year#234, d_date_sk#239]
+
+(123) ReusedExchange [Reuses operator id: 40]
+Output [2]: [i_item_sk#240, i_item_id#241]
+
+(124) BroadcastHashJoin [codegen id : 31]
+Left keys [1]: [cs_item_sk#222]
+Right keys [1]: [i_item_sk#240]
+Join condition: None
+
+(125) Project [codegen id : 31]
+Output [8]: [i_item_id#241, cast(cs_quantity#223 as decimal(12,2)) AS agg1#35, cast(cs_list_price#224 as decimal(12,2)) AS agg2#36, cast(cs_coupon_amt#226 as decimal(12,2)) AS agg3#37, cast(cs_sales_price#225 as decimal(12,2)) AS agg4#38, cast(cs_net_profit#227 as decimal(12,2)) AS agg5#39, cast(c_birth_year#234 as decimal(12,2)) AS agg6#40, cast(cd_dep_count#230 as decimal(12,2)) AS agg7#41]
+Input [10]: [cs_item_sk#222, cs_quantity#223, cs_list_price#224, cs_sales_price#225, cs_coupon_amt#226, cs_net_profit#227, cd_dep_count#230, c_birth_year#234, i_item_sk#240, i_item_id#241]
+
+(126) HashAggregate [codegen id : 31]
+Input [8]: [i_item_id#241, agg1#35, agg2#36, agg3#37, agg4#38, agg5#39, agg6#40, agg7#41]
+Keys [1]: [i_item_id#241]
+Functions [7]: [partial_avg(agg1#35), partial_avg(agg2#36), partial_avg(agg3#37), partial_avg(agg4#38), partial_avg(agg5#39), partial_avg(agg6#40), partial_avg(agg7#41)]
+Aggregate Attributes [14]: [sum#242, count#243, sum#244, count#245, sum#246, count#247, sum#248, count#249, sum#250, count#251, sum#252, count#253, sum#254, count#255]
+Results [15]: [i_item_id#241, sum#256, count#257, sum#258, count#259, sum#260, count#261, sum#262, count#263, sum#264, count#265, sum#266, count#267, sum#268, count#269]
+
+(127) Exchange
+Input [15]: [i_item_id#241, sum#256, count#257, sum#258, count#259, sum#260, count#261, sum#262, count#263, sum#264, count#265, sum#266, count#267, sum#268, count#269]
+Arguments: hashpartitioning(i_item_id#241, 5), ENSURE_REQUIREMENTS, [id=#270]
+
+(128) HashAggregate [codegen id : 32]
+Input [15]: [i_item_id#241, sum#256, count#257, sum#258, count#259, sum#260, count#261, sum#262, count#263, sum#264, count#265, sum#266, count#267, sum#268, count#269]
+Keys [1]: [i_item_id#241]
+Functions [7]: [avg(agg1#35), avg(agg2#36), avg(agg3#37), avg(agg4#38), avg(agg5#39), avg(agg6#40), avg(agg7#41)]
+Aggregate Attributes [7]: [avg(agg1#35)#271, avg(agg2#36)#272, avg(agg3#37)#273, avg(agg4#38)#274, avg(agg5#39)#275, avg(agg6#40)#276, avg(agg7#41)#277]
+Results [11]: [i_item_id#241, null AS ca_country#278, null AS ca_state#279, null AS county#280, avg(agg1#35)#271 AS agg1#281, avg(agg2#36)#272 AS agg2#282, avg(agg3#37)#273 AS agg3#283, avg(agg4#38)#274 AS agg4#284, avg(agg5#39)#275 AS agg5#285, avg(agg6#40)#276 AS agg6#286, avg(agg7#41)#277 AS agg7#287]
+
+(129) Scan parquet default.catalog_sales
+Output [9]: [cs_bill_customer_sk#288, cs_bill_cdemo_sk#289, cs_item_sk#290, cs_quantity#291, cs_list_price#292, cs_sales_price#293, cs_coupon_amt#294, cs_net_profit#295, cs_sold_date_sk#296]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#296), dynamicpruningexpression(cs_sold_date_sk#296 IN dynamicpruning#10)]
+PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_bill_cdemo_sk:int,cs_item_sk:int,cs_quantity:int,cs_list_price:decimal(7,2),cs_sales_price:decimal(7,2),cs_coupon_amt:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(130) ColumnarToRow [codegen id : 39]
+Input [9]: [cs_bill_customer_sk#288, cs_bill_cdemo_sk#289, cs_item_sk#290, cs_quantity#291, cs_list_price#292, cs_sales_price#293, cs_coupon_amt#294, cs_net_profit#295, cs_sold_date_sk#296]
+
+(131) Filter [codegen id : 39]
+Input [9]: [cs_bill_customer_sk#288, cs_bill_cdemo_sk#289, cs_item_sk#290, cs_quantity#291, cs_list_price#292, cs_sales_price#293, cs_coupon_amt#294, cs_net_profit#295, cs_sold_date_sk#296]
+Condition : ((isnotnull(cs_bill_cdemo_sk#289) AND isnotnull(cs_bill_customer_sk#288)) AND isnotnull(cs_item_sk#290))
+
+(132) ReusedExchange [Reuses operator id: 8]
+Output [2]: [cd_demo_sk#297, cd_dep_count#298]
+
+(133) BroadcastHashJoin [codegen id : 39]
+Left keys [1]: [cs_bill_cdemo_sk#289]
+Right keys [1]: [cd_demo_sk#297]
+Join condition: None
+
+(134) Project [codegen id : 39]
+Output [9]: [cs_bill_customer_sk#288, cs_item_sk#290, cs_quantity#291, cs_list_price#292, cs_sales_price#293, cs_coupon_amt#294, cs_net_profit#295, cs_sold_date_sk#296, cd_dep_count#298]
+Input [11]: [cs_bill_customer_sk#288, cs_bill_cdemo_sk#289, cs_item_sk#290, cs_quantity#291, cs_list_price#292, cs_sales_price#293, cs_coupon_amt#294, cs_net_profit#295, cs_sold_date_sk#296, cd_demo_sk#297, cd_dep_count#298]
+
+(135) ReusedExchange [Reuses operator id: 15]
+Output [4]: [c_customer_sk#299, c_current_cdemo_sk#300, c_current_addr_sk#301, c_birth_year#302]
+
+(136) BroadcastHashJoin [codegen id : 39]
+Left keys [1]: [cs_bill_customer_sk#288]
+Right keys [1]: [c_customer_sk#299]
+Join condition: None
+
+(137) Project [codegen id : 39]
+Output [11]: [cs_item_sk#290, cs_quantity#291, cs_list_price#292, cs_sales_price#293, cs_coupon_amt#294, cs_net_profit#295, cs_sold_date_sk#296, cd_dep_count#298, c_current_cdemo_sk#300, c_current_addr_sk#301, c_birth_year#302]
+Input [13]: [cs_bill_customer_sk#288, cs_item_sk#290, cs_quantity#291, cs_list_price#292, cs_sales_price#293, cs_coupon_amt#294, cs_net_profit#295, cs_sold_date_sk#296, cd_dep_count#298, c_customer_sk#299, c_current_cdemo_sk#300, c_current_addr_sk#301, c_birth_year#302]
+
+(138) ReusedExchange [Reuses operator id: 21]
+Output [1]: [cd_demo_sk#303]
+
+(139) BroadcastHashJoin [codegen id : 39]
+Left keys [1]: [c_current_cdemo_sk#300]
+Right keys [1]: [cd_demo_sk#303]
+Join condition: None
+
+(140) Project [codegen id : 39]
+Output [10]: [cs_item_sk#290, cs_quantity#291, cs_list_price#292, cs_sales_price#293, cs_coupon_amt#294, cs_net_profit#295, cs_sold_date_sk#296, cd_dep_count#298, c_current_addr_sk#301, c_birth_year#302]
+Input [12]: [cs_item_sk#290, cs_quantity#291, cs_list_price#292, cs_sales_price#293, cs_coupon_amt#294, cs_net_profit#295, cs_sold_date_sk#296, cd_dep_count#298, c_current_cdemo_sk#300, c_current_addr_sk#301, c_birth_year#302, cd_demo_sk#303]
+
+(141) ReusedExchange [Reuses operator id: 117]
+Output [1]: [ca_address_sk#304]
+
+(142) BroadcastHashJoin [codegen id : 39]
+Left keys [1]: [c_current_addr_sk#301]
+Right keys [1]: [ca_address_sk#304]
+Join condition: None
+
+(143) Project [codegen id : 39]
+Output [9]: [cs_item_sk#290, cs_quantity#291, cs_list_price#292, cs_sales_price#293, cs_coupon_amt#294, cs_net_profit#295, cs_sold_date_sk#296, cd_dep_count#298, c_birth_year#302]
+Input [11]: [cs_item_sk#290, cs_quantity#291, cs_list_price#292, cs_sales_price#293, cs_coupon_amt#294, cs_net_profit#295, cs_sold_date_sk#296, cd_dep_count#298, c_current_addr_sk#301, c_birth_year#302, ca_address_sk#304]
+
+(144) ReusedExchange [Reuses operator id: 34]
+Output [1]: [d_date_sk#305]
+
+(145) BroadcastHashJoin [codegen id : 39]
+Left keys [1]: [cs_sold_date_sk#296]
+Right keys [1]: [d_date_sk#305]
+Join condition: None
+
+(146) Project [codegen id : 39]
+Output [8]: [cs_item_sk#290, cs_quantity#291, cs_list_price#292, cs_sales_price#293, cs_coupon_amt#294, cs_net_profit#295, cd_dep_count#298, c_birth_year#302]
+Input [10]: [cs_item_sk#290, cs_quantity#291, cs_list_price#292, cs_sales_price#293, cs_coupon_amt#294, cs_net_profit#295, cs_sold_date_sk#296, cd_dep_count#298, c_birth_year#302, d_date_sk#305]
+
+(147) Scan parquet default.item
+Output [1]: [i_item_sk#306]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int>
+
+(148) ColumnarToRow [codegen id : 38]
+Input [1]: [i_item_sk#306]
+
+(149) Filter [codegen id : 38]
+Input [1]: [i_item_sk#306]
+Condition : isnotnull(i_item_sk#306)
+
+(150) BroadcastExchange
+Input [1]: [i_item_sk#306]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#307]
+
+(151) BroadcastHashJoin [codegen id : 39]
+Left keys [1]: [cs_item_sk#290]
+Right keys [1]: [i_item_sk#306]
+Join condition: None
+
+(152) Project [codegen id : 39]
+Output [7]: [cast(cs_quantity#291 as decimal(12,2)) AS agg1#35, cast(cs_list_price#292 as decimal(12,2)) AS agg2#36, cast(cs_coupon_amt#294 as decimal(12,2)) AS agg3#37, cast(cs_sales_price#293 as decimal(12,2)) AS agg4#38, cast(cs_net_profit#295 as decimal(12,2)) AS agg5#39, cast(c_birth_year#302 as decimal(12,2)) AS agg6#40, cast(cd_dep_count#298 as decimal(12,2)) AS agg7#41]
+Input [9]: [cs_item_sk#290, cs_quantity#291, cs_list_price#292, cs_sales_price#293, cs_coupon_amt#294, cs_net_profit#295, cd_dep_count#298, c_birth_year#302, i_item_sk#306]
+
+(153) HashAggregate [codegen id : 39]
+Input [7]: [agg1#35, agg2#36, agg3#37, agg4#38, agg5#39, agg6#40, agg7#41]
+Keys: []
+Functions [7]: [partial_avg(agg1#35), partial_avg(agg2#36), partial_avg(agg3#37), partial_avg(agg4#38), partial_avg(agg5#39), partial_avg(agg6#40), partial_avg(agg7#41)]
+Aggregate Attributes [14]: [sum#308, count#309, sum#310, count#311, sum#312, count#313, sum#314, count#315, sum#316, count#317, sum#318, count#319, sum#320, count#321]
+Results [14]: [sum#322, count#323, sum#324, count#325, sum#326, count#327, sum#328, count#329, sum#330, count#331, sum#332, count#333, sum#334, count#335]
+
+(154) Exchange
+Input [14]: [sum#322, count#323, sum#324, count#325, sum#326, count#327, sum#328, count#329, sum#330, count#331, sum#332, count#333, sum#334, count#335]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#336]
+
+(155) HashAggregate [codegen id : 40]
+Input [14]: [sum#322, count#323, sum#324, count#325, sum#326, count#327, sum#328, count#329, sum#330, count#331, sum#332, count#333, sum#334, count#335]
+Keys: []
+Functions [7]: [avg(agg1#35), avg(agg2#36), avg(agg3#37), avg(agg4#38), avg(agg5#39), avg(agg6#40), avg(agg7#41)]
+Aggregate Attributes [7]: [avg(agg1#35)#337, avg(agg2#36)#338, avg(agg3#37)#339, avg(agg4#38)#340, avg(agg5#39)#341, avg(agg6#40)#342, avg(agg7#41)#343]
+Results [11]: [null AS i_item_id#344, null AS ca_country#345, null AS ca_state#346, null AS county#347, avg(agg1#35)#337 AS agg1#348, avg(agg2#36)#338 AS agg2#349, avg(agg3#37)#339 AS agg3#350, avg(agg4#38)#340 AS agg4#351, avg(agg5#39)#341 AS agg5#352, avg(agg6#40)#342 AS agg6#353, avg(agg7#41)#343 AS agg7#354]
+
+(156) Union
+
+(157) TakeOrderedAndProject
+Input [11]: [i_item_id#33, ca_country#27, ca_state#26, ca_county#25, agg1#78, agg2#79, agg3#80, agg4#81, agg5#82, agg6#83, agg7#84]
+Arguments: 100, [ca_country#27 ASC NULLS FIRST, ca_state#26 ASC NULLS FIRST, ca_county#25 ASC NULLS FIRST, i_item_id#33 ASC NULLS FIRST], [i_item_id#33, ca_country#27, ca_state#26, ca_county#25, agg1#78, agg2#79, agg3#80, agg4#81, agg5#82, agg6#83, agg7#84]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#9 IN dynamicpruning#10
+ReusedExchange (158)
+
+
+(158) ReusedExchange [Reuses operator id: 34]
+Output [1]: [d_date_sk#29]
+
+Subquery:2 Hosting operator id = 46 Hosting Expression = cs_sold_date_sk#93 IN dynamicpruning#10
+
+Subquery:3 Hosting operator id = 73 Hosting Expression = cs_sold_date_sk#160 IN dynamicpruning#10
+
+Subquery:4 Hosting operator id = 101 Hosting Expression = cs_sold_date_sk#228 IN dynamicpruning#10
+
+Subquery:5 Hosting operator id = 129 Hosting Expression = cs_sold_date_sk#296 IN dynamicpruning#10
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/simplified.txt
new file mode 100644
index 0000000000000..abc243f55d639
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/simplified.txt
@@ -0,0 +1,233 @@
+TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7]
+  Union
+    WholeStageCodegen (8)
+      HashAggregate [i_item_id,ca_country,ca_state,ca_county,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+        InputAdapter
+          Exchange [i_item_id,ca_country,ca_state,ca_county] #1
+            WholeStageCodegen (7)
+              HashAggregate [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+                Project [i_item_id,ca_country,ca_state,ca_county,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count]
+                  BroadcastHashJoin [cs_item_sk,i_item_sk]
+                    Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country]
+                      BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                        Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country]
+                          BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                            Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year]
+                              BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk]
+                                Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year]
+                                  BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk]
+                                    Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count]
+                                      BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk]
+                                        Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk]
+                                                SubqueryBroadcast [d_date_sk] #1
+                                                  ReusedExchange [d_date_sk] #2
+                                        InputAdapter
+                                          BroadcastExchange #3
+                                            WholeStageCodegen (1)
+                                              Project [cd_demo_sk,cd_dep_count]
+                                                Filter [cd_gender,cd_education_status,cd_demo_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count]
+                                    InputAdapter
+                                      BroadcastExchange #4
+                                        WholeStageCodegen (2)
+                                          Project [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year]
+                                            Filter [c_birth_month,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year]
+                                InputAdapter
+                                  BroadcastExchange #5
+                                    WholeStageCodegen (3)
+                                      Filter [cd_demo_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.customer_demographics [cd_demo_sk]
+                            InputAdapter
+                              BroadcastExchange #6
+                                WholeStageCodegen (4)
+                                  Filter [ca_state,ca_address_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.customer_address [ca_address_sk,ca_county,ca_state,ca_country]
+                        InputAdapter
+                          BroadcastExchange #2
+                            WholeStageCodegen (5)
+                              Project [d_date_sk]
+                                Filter [d_year,d_date_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.date_dim [d_date_sk,d_year]
+                    InputAdapter
+                      BroadcastExchange #7
+                        WholeStageCodegen (6)
+                          Filter [i_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.item [i_item_sk,i_item_id]
+    WholeStageCodegen (16)
+      HashAggregate [i_item_id,ca_country,ca_state,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+        InputAdapter
+          Exchange [i_item_id,ca_country,ca_state] #8
+            WholeStageCodegen (15)
+              HashAggregate [i_item_id,ca_country,ca_state,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+                Project [i_item_id,ca_country,ca_state,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count]
+                  BroadcastHashJoin [cs_item_sk,i_item_sk]
+                    Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country]
+                      BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                        Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_state,ca_country]
+                          BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                            Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year]
+                              BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk]
+                                Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year]
+                                  BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk]
+                                    Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count]
+                                      BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk]
+                                        Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk]
+                                                ReusedSubquery [d_date_sk] #1
+                                        InputAdapter
+                                          ReusedExchange [cd_demo_sk,cd_dep_count] #3
+                                    InputAdapter
+                                      ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4
+                                InputAdapter
+                                  ReusedExchange [cd_demo_sk] #5
+                            InputAdapter
+                              BroadcastExchange #9
+                                WholeStageCodegen (12)
+                                  Filter [ca_state,ca_address_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.customer_address [ca_address_sk,ca_state,ca_country]
+                        InputAdapter
+                          ReusedExchange [d_date_sk] #2
+                    InputAdapter
+                      ReusedExchange [i_item_sk,i_item_id] #7
+    WholeStageCodegen (24)
+      HashAggregate [i_item_id,ca_country,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+        InputAdapter
+          Exchange [i_item_id,ca_country] #10
+            WholeStageCodegen (23)
+              HashAggregate [i_item_id,ca_country,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+                Project [i_item_id,ca_country,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count]
+                  BroadcastHashJoin [cs_item_sk,i_item_sk]
+                    Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country]
+                      BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                        Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_country]
+                          BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                            Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year]
+                              BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk]
+                                Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year]
+                                  BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk]
+                                    Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count]
+                                      BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk]
+                                        Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk]
+                                                ReusedSubquery [d_date_sk] #1
+                                        InputAdapter
+                                          ReusedExchange [cd_demo_sk,cd_dep_count] #3
+                                    InputAdapter
+                                      ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4
+                                InputAdapter
+                                  ReusedExchange [cd_demo_sk] #5
+                            InputAdapter
+                              BroadcastExchange #11
+                                WholeStageCodegen (20)
+                                  Project [ca_address_sk,ca_country]
+                                    Filter [ca_state,ca_address_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.customer_address [ca_address_sk,ca_state,ca_country]
+                        InputAdapter
+                          ReusedExchange [d_date_sk] #2
+                    InputAdapter
+                      ReusedExchange [i_item_sk,i_item_id] #7
+    WholeStageCodegen (32)
+      HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+        InputAdapter
+          Exchange [i_item_id] #12
+            WholeStageCodegen (31)
+              HashAggregate [i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+                Project [i_item_id,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count]
+                  BroadcastHashJoin [cs_item_sk,i_item_sk]
+                    Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year]
+                      BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                        Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year]
+                          BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                            Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year]
+                              BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk]
+                                Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year]
+                                  BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk]
+                                    Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count]
+                                      BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk]
+                                        Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk]
+                                                ReusedSubquery [d_date_sk] #1
+                                        InputAdapter
+                                          ReusedExchange [cd_demo_sk,cd_dep_count] #3
+                                    InputAdapter
+                                      ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4
+                                InputAdapter
+                                  ReusedExchange [cd_demo_sk] #5
+                            InputAdapter
+                              BroadcastExchange #13
+                                WholeStageCodegen (28)
+                                  Project [ca_address_sk]
+                                    Filter [ca_state,ca_address_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.customer_address [ca_address_sk,ca_state]
+                        InputAdapter
+                          ReusedExchange [d_date_sk] #2
+                    InputAdapter
+                      ReusedExchange [i_item_sk,i_item_id] #7
+    WholeStageCodegen (40)
+      HashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+        InputAdapter
+          Exchange #14
+            WholeStageCodegen (39)
+              HashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+                Project [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count]
+                  BroadcastHashJoin [cs_item_sk,i_item_sk]
+                    Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year]
+                      BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                        Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year]
+                          BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                            Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year]
+                              BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk]
+                                Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year]
+                                  BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk]
+                                    Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count]
+                                      BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk]
+                                        Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk]
+                                                ReusedSubquery [d_date_sk] #1
+                                        InputAdapter
+                                          ReusedExchange [cd_demo_sk,cd_dep_count] #3
+                                    InputAdapter
+                                      ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4
+                                InputAdapter
+                                  ReusedExchange [cd_demo_sk] #5
+                            InputAdapter
+                              ReusedExchange [ca_address_sk] #13
+                        InputAdapter
+                          ReusedExchange [d_date_sk] #2
+                    InputAdapter
+                      BroadcastExchange #15
+                        WholeStageCodegen (38)
+                          Filter [i_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.item [i_item_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.sf100/explain.txt
new file mode 100644
index 0000000000000..d9b14fcfb5bf9
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.sf100/explain.txt
@@ -0,0 +1,163 @@
+== Physical Plan ==
+TakeOrderedAndProject (27)
++- * Project (26)
+   +- Window (25)
+      +- * Sort (24)
+         +- Exchange (23)
+            +- * HashAggregate (22)
+               +- Exchange (21)
+                  +- * HashAggregate (20)
+                     +- * Project (19)
+                        +- * BroadcastHashJoin Inner BuildRight (18)
+                           :- * Project (12)
+                           :  +- * SortMergeJoin Inner (11)
+                           :     :- * Sort (5)
+                           :     :  +- Exchange (4)
+                           :     :     +- * Filter (3)
+                           :     :        +- * ColumnarToRow (2)
+                           :     :           +- Scan parquet default.catalog_sales (1)
+                           :     +- * Sort (10)
+                           :        +- Exchange (9)
+                           :           +- * Filter (8)
+                           :              +- * ColumnarToRow (7)
+                           :                 +- Scan parquet default.item (6)
+                           +- BroadcastExchange (17)
+                              +- * Project (16)
+                                 +- * Filter (15)
+                                    +- * ColumnarToRow (14)
+                                       +- Scan parquet default.date_dim (13)
+
+
+(1) Scan parquet default.catalog_sales
+Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int,cs_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3]
+
+(3) Filter [codegen id : 1]
+Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3]
+Condition : isnotnull(cs_item_sk#1)
+
+(4) Exchange
+Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3]
+Arguments: hashpartitioning(cs_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#5]
+
+(5) Sort [codegen id : 2]
+Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3]
+Arguments: [cs_item_sk#1 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.item
+Output [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [In(i_category, [Sports                                            ,Books                                             ,Home                                              ]), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_item_desc:string,i_current_price:decimal(7,2),i_class:string,i_category:string>
+
+(7) ColumnarToRow [codegen id : 3]
+Input [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+
+(8) Filter [codegen id : 3]
+Input [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Condition : (i_category#11 IN (Sports                                            ,Books                                             ,Home                                              ) AND isnotnull(i_item_sk#6))
+
+(9) Exchange
+Input [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Arguments: hashpartitioning(i_item_sk#6, 5), ENSURE_REQUIREMENTS, [id=#12]
+
+(10) Sort [codegen id : 4]
+Input [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Arguments: [i_item_sk#6 ASC NULLS FIRST], false, 0
+
+(11) SortMergeJoin [codegen id : 6]
+Left keys [1]: [cs_item_sk#1]
+Right keys [1]: [i_item_sk#6]
+Join condition: None
+
+(12) Project [codegen id : 6]
+Output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+
+(13) Scan parquet default.date_dim
+Output [2]: [d_date_sk#13, d_date#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(14) ColumnarToRow [codegen id : 5]
+Input [2]: [d_date_sk#13, d_date#14]
+
+(15) Filter [codegen id : 5]
+Input [2]: [d_date_sk#13, d_date#14]
+Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13))
+
+(16) Project [codegen id : 5]
+Output [1]: [d_date_sk#13]
+Input [2]: [d_date_sk#13, d_date#14]
+
+(17) BroadcastExchange
+Input [1]: [d_date_sk#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#15]
+
+(18) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [cs_sold_date_sk#3]
+Right keys [1]: [d_date_sk#13]
+Join condition: None
+
+(19) Project [codegen id : 6]
+Output [6]: [cs_ext_sales_price#2, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11, d_date_sk#13]
+
+(20) HashAggregate [codegen id : 6]
+Input [6]: [cs_ext_sales_price#2, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Keys [5]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9]
+Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum#16]
+Results [6]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, sum#17]
+
+(21) Exchange
+Input [6]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, sum#17]
+Arguments: hashpartitioning(i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(22) HashAggregate [codegen id : 7]
+Input [6]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, sum#17]
+Keys [5]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9]
+Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#2))#19]
+Results [8]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#19,17,2) AS itemrevenue#20, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#19,17,2) AS _w0#21, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#19,17,2) AS _w1#22]
+
+(23) Exchange
+Input [8]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, _w0#21, _w1#22]
+Arguments: hashpartitioning(i_class#10, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(24) Sort [codegen id : 8]
+Input [8]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, _w0#21, _w1#22]
+Arguments: [i_class#10 ASC NULLS FIRST], false, 0
+
+(25) Window
+Input [8]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, _w0#21, _w1#22]
+Arguments: [sum(_w1#22) windowspecdefinition(i_class#10, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#24], [i_class#10]
+
+(26) Project [codegen id : 9]
+Output [7]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(_w0#21) * 100.00), DecimalType(21,2), true) as decimal(27,2))) / promote_precision(_we0#24)), DecimalType(38,17), true) AS revenueratio#25]
+Input [9]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, _w0#21, _w1#22, _we0#24]
+
+(27) TakeOrderedAndProject
+Input [7]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, revenueratio#25]
+Arguments: 100, [i_category#11 ASC NULLS FIRST, i_class#10 ASC NULLS FIRST, i_item_id#7 ASC NULLS FIRST, i_item_desc#8 ASC NULLS FIRST, revenueratio#25 ASC NULLS FIRST], [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, revenueratio#25]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (28)
+
+
+(28) ReusedExchange [Reuses operator id: 17]
+Output [1]: [d_date_sk#13]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.sf100/simplified.txt
new file mode 100644
index 0000000000000..69e0754aed6d8
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.sf100/simplified.txt
@@ -0,0 +1,49 @@
+TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue]
+  WholeStageCodegen (9)
+    Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0]
+      InputAdapter
+        Window [_w1,i_class]
+          WholeStageCodegen (8)
+            Sort [i_class]
+              InputAdapter
+                Exchange [i_class] #1
+                  WholeStageCodegen (7)
+                    HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(cs_ext_sales_price)),itemrevenue,_w0,_w1,sum]
+                      InputAdapter
+                        Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2
+                          WholeStageCodegen (6)
+                            HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,cs_ext_sales_price] [sum,sum]
+                              Project [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                  Project [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                    SortMergeJoin [cs_item_sk,i_item_sk]
+                                      InputAdapter
+                                        WholeStageCodegen (2)
+                                          Sort [cs_item_sk]
+                                            InputAdapter
+                                              Exchange [cs_item_sk] #3
+                                                WholeStageCodegen (1)
+                                                  Filter [cs_item_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk]
+                                                          SubqueryBroadcast [d_date_sk] #1
+                                                            ReusedExchange [d_date_sk] #4
+                                      InputAdapter
+                                        WholeStageCodegen (4)
+                                          Sort [i_item_sk]
+                                            InputAdapter
+                                              Exchange [i_item_sk] #5
+                                                WholeStageCodegen (3)
+                                                  Filter [i_category,i_item_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                  InputAdapter
+                                    BroadcastExchange #4
+                                      WholeStageCodegen (5)
+                                        Project [d_date_sk]
+                                          Filter [d_date,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_date]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/explain.txt
new file mode 100644
index 0000000000000..c6a345be29c9d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/explain.txt
@@ -0,0 +1,148 @@
+== Physical Plan ==
+TakeOrderedAndProject (24)
++- * Project (23)
+   +- Window (22)
+      +- * Sort (21)
+         +- Exchange (20)
+            +- * HashAggregate (19)
+               +- Exchange (18)
+                  +- * HashAggregate (17)
+                     +- * Project (16)
+                        +- * BroadcastHashJoin Inner BuildRight (15)
+                           :- * Project (9)
+                           :  +- * BroadcastHashJoin Inner BuildRight (8)
+                           :     :- * Filter (3)
+                           :     :  +- * ColumnarToRow (2)
+                           :     :     +- Scan parquet default.catalog_sales (1)
+                           :     +- BroadcastExchange (7)
+                           :        +- * Filter (6)
+                           :           +- * ColumnarToRow (5)
+                           :              +- Scan parquet default.item (4)
+                           +- BroadcastExchange (14)
+                              +- * Project (13)
+                                 +- * Filter (12)
+                                    +- * ColumnarToRow (11)
+                                       +- Scan parquet default.date_dim (10)
+
+
+(1) Scan parquet default.catalog_sales
+Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int,cs_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3]
+Condition : isnotnull(cs_item_sk#1)
+
+(4) Scan parquet default.item
+Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [In(i_category, [Sports                                            ,Books                                             ,Home                                              ]), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_item_desc:string,i_current_price:decimal(7,2),i_class:string,i_category:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+
+(6) Filter [codegen id : 1]
+Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Condition : (i_category#10 IN (Sports                                            ,Books                                             ,Home                                              ) AND isnotnull(i_item_sk#5))
+
+(7) BroadcastExchange
+Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(8) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_item_sk#1]
+Right keys [1]: [i_item_sk#5]
+Join condition: None
+
+(9) Project [codegen id : 3]
+Output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+
+(10) Scan parquet default.date_dim
+Output [2]: [d_date_sk#12, d_date#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#12, d_date#13]
+
+(12) Filter [codegen id : 2]
+Input [2]: [d_date_sk#12, d_date#13]
+Condition : (((isnotnull(d_date#13) AND (d_date#13 >= 1999-02-22)) AND (d_date#13 <= 1999-03-24)) AND isnotnull(d_date_sk#12))
+
+(13) Project [codegen id : 2]
+Output [1]: [d_date_sk#12]
+Input [2]: [d_date_sk#12, d_date#13]
+
+(14) BroadcastExchange
+Input [1]: [d_date_sk#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_sold_date_sk#3]
+Right keys [1]: [d_date_sk#12]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [6]: [cs_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10, d_date_sk#12]
+
+(17) HashAggregate [codegen id : 3]
+Input [6]: [cs_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8]
+Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum#15]
+Results [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#16]
+
+(18) Exchange
+Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#16]
+Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(19) HashAggregate [codegen id : 4]
+Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#16]
+Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8]
+Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#2))#18]
+Results [8]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#18,17,2) AS itemrevenue#19, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#18,17,2) AS _w0#20, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#18,17,2) AS _w1#21]
+
+(20) Exchange
+Input [8]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21]
+Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(21) Sort [codegen id : 5]
+Input [8]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21]
+Arguments: [i_class#9 ASC NULLS FIRST], false, 0
+
+(22) Window
+Input [8]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21]
+Arguments: [sum(_w1#21) windowspecdefinition(i_class#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#23], [i_class#9]
+
+(23) Project [codegen id : 6]
+Output [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(_w0#20) * 100.00), DecimalType(21,2), true) as decimal(27,2))) / promote_precision(_we0#23)), DecimalType(38,17), true) AS revenueratio#24]
+Input [9]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21, _we0#23]
+
+(24) TakeOrderedAndProject
+Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, revenueratio#24]
+Arguments: 100, [i_category#10 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_item_id#6 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#24 ASC NULLS FIRST], [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, revenueratio#24]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (25)
+
+
+(25) ReusedExchange [Reuses operator id: 14]
+Output [1]: [d_date_sk#12]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/simplified.txt
new file mode 100644
index 0000000000000..2d49ed4224fc3
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/simplified.txt
@@ -0,0 +1,40 @@
+TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue]
+  WholeStageCodegen (6)
+    Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0]
+      InputAdapter
+        Window [_w1,i_class]
+          WholeStageCodegen (5)
+            Sort [i_class]
+              InputAdapter
+                Exchange [i_class] #1
+                  WholeStageCodegen (4)
+                    HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(cs_ext_sales_price)),itemrevenue,_w0,_w1,sum]
+                      InputAdapter
+                        Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2
+                          WholeStageCodegen (3)
+                            HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,cs_ext_sales_price] [sum,sum]
+                              Project [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                  Project [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                    BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                      Filter [cs_item_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #1
+                                                ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        BroadcastExchange #4
+                                          WholeStageCodegen (1)
+                                            Filter [i_category,i_item_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                  InputAdapter
+                                    BroadcastExchange #3
+                                      WholeStageCodegen (2)
+                                        Project [d_date_sk]
+                                          Filter [d_date,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_date]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.sf100/explain.txt
new file mode 100644
index 0000000000000..8f9ec9e9cd68a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.sf100/explain.txt
@@ -0,0 +1,168 @@
+== Physical Plan ==
+TakeOrderedAndProject (28)
++- * HashAggregate (27)
+   +- Exchange (26)
+      +- * HashAggregate (25)
+         +- * Expand (24)
+            +- * BroadcastNestedLoopJoin Inner BuildRight (23)
+               :- * Project (19)
+               :  +- * SortMergeJoin Inner (18)
+               :     :- * Sort (12)
+               :     :  +- Exchange (11)
+               :     :     +- * Project (10)
+               :     :        +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :           :- * Filter (3)
+               :     :           :  +- * ColumnarToRow (2)
+               :     :           :     +- Scan parquet default.inventory (1)
+               :     :           +- BroadcastExchange (8)
+               :     :              +- * Project (7)
+               :     :                 +- * Filter (6)
+               :     :                    +- * ColumnarToRow (5)
+               :     :                       +- Scan parquet default.date_dim (4)
+               :     +- * Sort (17)
+               :        +- Exchange (16)
+               :           +- * Filter (15)
+               :              +- * ColumnarToRow (14)
+               :                 +- Scan parquet default.item (13)
+               +- BroadcastExchange (22)
+                  +- * ColumnarToRow (21)
+                     +- Scan parquet default.warehouse (20)
+
+
+(1) Scan parquet default.inventory
+Output [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(inv_date_sk#3), dynamicpruningexpression(inv_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(inv_item_sk)]
+ReadSchema: struct<inv_item_sk:int,inv_quantity_on_hand:int>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3]
+
+(3) Filter [codegen id : 2]
+Input [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3]
+Condition : isnotnull(inv_item_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#5, d_month_seq#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#5]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [inv_date_sk#3]
+Right keys [1]: [d_date_sk#5]
+Join condition: None
+
+(10) Project [codegen id : 2]
+Output [2]: [inv_item_sk#1, inv_quantity_on_hand#2]
+Input [4]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3, d_date_sk#5]
+
+(11) Exchange
+Input [2]: [inv_item_sk#1, inv_quantity_on_hand#2]
+Arguments: hashpartitioning(inv_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#8]
+
+(12) Sort [codegen id : 3]
+Input [2]: [inv_item_sk#1, inv_quantity_on_hand#2]
+Arguments: [inv_item_sk#1 ASC NULLS FIRST], false, 0
+
+(13) Scan parquet default.item
+Output [5]: [i_item_sk#9, i_brand#10, i_class#11, i_category#12, i_product_name#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_class:string,i_category:string,i_product_name:string>
+
+(14) ColumnarToRow [codegen id : 4]
+Input [5]: [i_item_sk#9, i_brand#10, i_class#11, i_category#12, i_product_name#13]
+
+(15) Filter [codegen id : 4]
+Input [5]: [i_item_sk#9, i_brand#10, i_class#11, i_category#12, i_product_name#13]
+Condition : isnotnull(i_item_sk#9)
+
+(16) Exchange
+Input [5]: [i_item_sk#9, i_brand#10, i_class#11, i_category#12, i_product_name#13]
+Arguments: hashpartitioning(i_item_sk#9, 5), ENSURE_REQUIREMENTS, [id=#14]
+
+(17) Sort [codegen id : 5]
+Input [5]: [i_item_sk#9, i_brand#10, i_class#11, i_category#12, i_product_name#13]
+Arguments: [i_item_sk#9 ASC NULLS FIRST], false, 0
+
+(18) SortMergeJoin [codegen id : 7]
+Left keys [1]: [inv_item_sk#1]
+Right keys [1]: [i_item_sk#9]
+Join condition: None
+
+(19) Project [codegen id : 7]
+Output [5]: [inv_quantity_on_hand#2, i_brand#10, i_class#11, i_category#12, i_product_name#13]
+Input [7]: [inv_item_sk#1, inv_quantity_on_hand#2, i_item_sk#9, i_brand#10, i_class#11, i_category#12, i_product_name#13]
+
+(20) Scan parquet default.warehouse
+Output: []
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/warehouse]
+ReadSchema: struct<>
+
+(21) ColumnarToRow [codegen id : 6]
+Input: []
+
+(22) BroadcastExchange
+Input: []
+Arguments: IdentityBroadcastMode, [id=#15]
+
+(23) BroadcastNestedLoopJoin [codegen id : 7]
+Join condition: None
+
+(24) Expand [codegen id : 7]
+Input [5]: [inv_quantity_on_hand#2, i_brand#10, i_class#11, i_category#12, i_product_name#13]
+Arguments: [List(inv_quantity_on_hand#2, i_product_name#13, i_brand#10, i_class#11, i_category#12, 0), List(inv_quantity_on_hand#2, i_product_name#13, i_brand#10, i_class#11, null, 1), List(inv_quantity_on_hand#2, i_product_name#13, i_brand#10, null, null, 3), List(inv_quantity_on_hand#2, i_product_name#13, null, null, null, 7), List(inv_quantity_on_hand#2, null, null, null, null, 15)], [inv_quantity_on_hand#2, i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20]
+
+(25) HashAggregate [codegen id : 7]
+Input [6]: [inv_quantity_on_hand#2, i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20]
+Keys [5]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20]
+Functions [1]: [partial_avg(inv_quantity_on_hand#2)]
+Aggregate Attributes [2]: [sum#21, count#22]
+Results [7]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20, sum#23, count#24]
+
+(26) Exchange
+Input [7]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20, sum#23, count#24]
+Arguments: hashpartitioning(i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(27) HashAggregate [codegen id : 8]
+Input [7]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20, sum#23, count#24]
+Keys [5]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20]
+Functions [1]: [avg(inv_quantity_on_hand#2)]
+Aggregate Attributes [1]: [avg(inv_quantity_on_hand#2)#26]
+Results [5]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, avg(inv_quantity_on_hand#2)#26 AS qoh#27]
+
+(28) TakeOrderedAndProject
+Input [5]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, qoh#27]
+Arguments: 100, [qoh#27 ASC NULLS FIRST, i_product_name#16 ASC NULLS FIRST, i_brand#17 ASC NULLS FIRST, i_class#18 ASC NULLS FIRST, i_category#19 ASC NULLS FIRST], [i_product_name#16, i_brand#17, i_class#18, i_category#19, qoh#27]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#3 IN dynamicpruning#4
+ReusedExchange (29)
+
+
+(29) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#5]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.sf100/simplified.txt
new file mode 100644
index 0000000000000..ca9a5d0546027
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.sf100/simplified.txt
@@ -0,0 +1,49 @@
+TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category]
+  WholeStageCodegen (8)
+    HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count]
+      InputAdapter
+        Exchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1
+          WholeStageCodegen (7)
+            HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count]
+              Expand [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category]
+                BroadcastNestedLoopJoin
+                  Project [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name]
+                    SortMergeJoin [inv_item_sk,i_item_sk]
+                      InputAdapter
+                        WholeStageCodegen (3)
+                          Sort [inv_item_sk]
+                            InputAdapter
+                              Exchange [inv_item_sk] #2
+                                WholeStageCodegen (2)
+                                  Project [inv_item_sk,inv_quantity_on_hand]
+                                    BroadcastHashJoin [inv_date_sk,d_date_sk]
+                                      Filter [inv_item_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #1
+                                                ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        BroadcastExchange #3
+                                          WholeStageCodegen (1)
+                                            Project [d_date_sk]
+                                              Filter [d_month_seq,d_date_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_date_sk,d_month_seq]
+                      InputAdapter
+                        WholeStageCodegen (5)
+                          Sort [i_item_sk]
+                            InputAdapter
+                              Exchange [i_item_sk] #4
+                                WholeStageCodegen (4)
+                                  Filter [i_item_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.item [i_item_sk,i_brand,i_class,i_category,i_product_name]
+                  InputAdapter
+                    BroadcastExchange #5
+                      WholeStageCodegen (6)
+                        ColumnarToRow
+                          InputAdapter
+                            Scan parquet default.warehouse
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/explain.txt
new file mode 100644
index 0000000000000..7cd7526e9da63
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/explain.txt
@@ -0,0 +1,153 @@
+== Physical Plan ==
+TakeOrderedAndProject (25)
++- * HashAggregate (24)
+   +- Exchange (23)
+      +- * HashAggregate (22)
+         +- * Expand (21)
+            +- * BroadcastNestedLoopJoin Inner BuildRight (20)
+               :- * Project (16)
+               :  +- * BroadcastHashJoin Inner BuildRight (15)
+               :     :- * Project (10)
+               :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :     :     :- * Filter (3)
+               :     :     :  +- * ColumnarToRow (2)
+               :     :     :     +- Scan parquet default.inventory (1)
+               :     :     +- BroadcastExchange (8)
+               :     :        +- * Project (7)
+               :     :           +- * Filter (6)
+               :     :              +- * ColumnarToRow (5)
+               :     :                 +- Scan parquet default.date_dim (4)
+               :     +- BroadcastExchange (14)
+               :        +- * Filter (13)
+               :           +- * ColumnarToRow (12)
+               :              +- Scan parquet default.item (11)
+               +- BroadcastExchange (19)
+                  +- * ColumnarToRow (18)
+                     +- Scan parquet default.warehouse (17)
+
+
+(1) Scan parquet default.inventory
+Output [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(inv_date_sk#3), dynamicpruningexpression(inv_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(inv_item_sk)]
+ReadSchema: struct<inv_item_sk:int,inv_quantity_on_hand:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3]
+
+(3) Filter [codegen id : 4]
+Input [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3]
+Condition : isnotnull(inv_item_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#5, d_month_seq#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#5]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_date_sk#3]
+Right keys [1]: [d_date_sk#5]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [2]: [inv_item_sk#1, inv_quantity_on_hand#2]
+Input [4]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3, d_date_sk#5]
+
+(11) Scan parquet default.item
+Output [5]: [i_item_sk#8, i_brand#9, i_class#10, i_category#11, i_product_name#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_class:string,i_category:string,i_product_name:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [5]: [i_item_sk#8, i_brand#9, i_class#10, i_category#11, i_product_name#12]
+
+(13) Filter [codegen id : 2]
+Input [5]: [i_item_sk#8, i_brand#9, i_class#10, i_category#11, i_product_name#12]
+Condition : isnotnull(i_item_sk#8)
+
+(14) BroadcastExchange
+Input [5]: [i_item_sk#8, i_brand#9, i_class#10, i_category#11, i_product_name#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#13]
+
+(15) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_item_sk#1]
+Right keys [1]: [i_item_sk#8]
+Join condition: None
+
+(16) Project [codegen id : 4]
+Output [5]: [inv_quantity_on_hand#2, i_brand#9, i_class#10, i_category#11, i_product_name#12]
+Input [7]: [inv_item_sk#1, inv_quantity_on_hand#2, i_item_sk#8, i_brand#9, i_class#10, i_category#11, i_product_name#12]
+
+(17) Scan parquet default.warehouse
+Output: []
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/warehouse]
+ReadSchema: struct<>
+
+(18) ColumnarToRow [codegen id : 3]
+Input: []
+
+(19) BroadcastExchange
+Input: []
+Arguments: IdentityBroadcastMode, [id=#14]
+
+(20) BroadcastNestedLoopJoin [codegen id : 4]
+Join condition: None
+
+(21) Expand [codegen id : 4]
+Input [5]: [inv_quantity_on_hand#2, i_brand#9, i_class#10, i_category#11, i_product_name#12]
+Arguments: [List(inv_quantity_on_hand#2, i_product_name#12, i_brand#9, i_class#10, i_category#11, 0), List(inv_quantity_on_hand#2, i_product_name#12, i_brand#9, i_class#10, null, 1), List(inv_quantity_on_hand#2, i_product_name#12, i_brand#9, null, null, 3), List(inv_quantity_on_hand#2, i_product_name#12, null, null, null, 7), List(inv_quantity_on_hand#2, null, null, null, null, 15)], [inv_quantity_on_hand#2, i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19]
+
+(22) HashAggregate [codegen id : 4]
+Input [6]: [inv_quantity_on_hand#2, i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19]
+Keys [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19]
+Functions [1]: [partial_avg(inv_quantity_on_hand#2)]
+Aggregate Attributes [2]: [sum#20, count#21]
+Results [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23]
+
+(23) Exchange
+Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23]
+Arguments: hashpartitioning(i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(24) HashAggregate [codegen id : 5]
+Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23]
+Keys [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19]
+Functions [1]: [avg(inv_quantity_on_hand#2)]
+Aggregate Attributes [1]: [avg(inv_quantity_on_hand#2)#25]
+Results [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, avg(inv_quantity_on_hand#2)#25 AS qoh#26]
+
+(25) TakeOrderedAndProject
+Input [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, qoh#26]
+Arguments: 100, [qoh#26 ASC NULLS FIRST, i_product_name#15 ASC NULLS FIRST, i_brand#16 ASC NULLS FIRST, i_class#17 ASC NULLS FIRST, i_category#18 ASC NULLS FIRST], [i_product_name#15, i_brand#16, i_class#17, i_category#18, qoh#26]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#3 IN dynamicpruning#4
+ReusedExchange (26)
+
+
+(26) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#5]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/simplified.txt
new file mode 100644
index 0000000000000..c8ef4786f2a76
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/simplified.txt
@@ -0,0 +1,40 @@
+TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category]
+  WholeStageCodegen (5)
+    HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count]
+      InputAdapter
+        Exchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1
+          WholeStageCodegen (4)
+            HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count]
+              Expand [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category]
+                BroadcastNestedLoopJoin
+                  Project [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name]
+                    BroadcastHashJoin [inv_item_sk,i_item_sk]
+                      Project [inv_item_sk,inv_quantity_on_hand]
+                        BroadcastHashJoin [inv_date_sk,d_date_sk]
+                          Filter [inv_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk]
+                                  SubqueryBroadcast [d_date_sk] #1
+                                    ReusedExchange [d_date_sk] #2
+                          InputAdapter
+                            BroadcastExchange #2
+                              WholeStageCodegen (1)
+                                Project [d_date_sk]
+                                  Filter [d_month_seq,d_date_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.date_dim [d_date_sk,d_month_seq]
+                      InputAdapter
+                        BroadcastExchange #3
+                          WholeStageCodegen (2)
+                            Filter [i_item_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.item [i_item_sk,i_brand,i_class,i_category,i_product_name]
+                  InputAdapter
+                    BroadcastExchange #4
+                      WholeStageCodegen (3)
+                        ColumnarToRow
+                          InputAdapter
+                            Scan parquet default.warehouse
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.sf100/explain.txt
new file mode 100644
index 0000000000000..f5a7a9135cf29
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.sf100/explain.txt
@@ -0,0 +1,327 @@
+== Physical Plan ==
+TakeOrderedAndProject (52)
++- Union (51)
+   :- * HashAggregate (30)
+   :  +- * HashAggregate (29)
+   :     +- * HashAggregate (28)
+   :        +- Exchange (27)
+   :           +- * HashAggregate (26)
+   :              +- * Project (25)
+   :                 +- * SortMergeJoin Inner (24)
+   :                    :- * Sort (18)
+   :                    :  +- Exchange (17)
+   :                    :     +- * Project (16)
+   :                    :        +- * BroadcastHashJoin Inner BuildRight (15)
+   :                    :           :- * Project (9)
+   :                    :           :  +- * BroadcastHashJoin Inner BuildRight (8)
+   :                    :           :     :- * Filter (3)
+   :                    :           :     :  +- * ColumnarToRow (2)
+   :                    :           :     :     +- Scan parquet default.inventory (1)
+   :                    :           :     +- BroadcastExchange (7)
+   :                    :           :        +- * Filter (6)
+   :                    :           :           +- * ColumnarToRow (5)
+   :                    :           :              +- Scan parquet default.warehouse (4)
+   :                    :           +- BroadcastExchange (14)
+   :                    :              +- * Project (13)
+   :                    :                 +- * Filter (12)
+   :                    :                    +- * ColumnarToRow (11)
+   :                    :                       +- Scan parquet default.date_dim (10)
+   :                    +- * Sort (23)
+   :                       +- Exchange (22)
+   :                          +- * Filter (21)
+   :                             +- * ColumnarToRow (20)
+   :                                +- Scan parquet default.item (19)
+   :- * HashAggregate (35)
+   :  +- Exchange (34)
+   :     +- * HashAggregate (33)
+   :        +- * HashAggregate (32)
+   :           +- ReusedExchange (31)
+   :- * HashAggregate (40)
+   :  +- Exchange (39)
+   :     +- * HashAggregate (38)
+   :        +- * HashAggregate (37)
+   :           +- ReusedExchange (36)
+   :- * HashAggregate (45)
+   :  +- Exchange (44)
+   :     +- * HashAggregate (43)
+   :        +- * HashAggregate (42)
+   :           +- ReusedExchange (41)
+   +- * HashAggregate (50)
+      +- Exchange (49)
+         +- * HashAggregate (48)
+            +- * HashAggregate (47)
+               +- ReusedExchange (46)
+
+
+(1) Scan parquet default.inventory
+Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)]
+ReadSchema: struct<inv_item_sk:int,inv_warehouse_sk:int,inv_quantity_on_hand:int>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+
+(3) Filter [codegen id : 3]
+Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2))
+
+(4) Scan parquet default.warehouse
+Output [1]: [w_warehouse_sk#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/warehouse]
+PushedFilters: [IsNotNull(w_warehouse_sk)]
+ReadSchema: struct<w_warehouse_sk:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [1]: [w_warehouse_sk#6]
+
+(6) Filter [codegen id : 1]
+Input [1]: [w_warehouse_sk#6]
+Condition : isnotnull(w_warehouse_sk#6)
+
+(7) BroadcastExchange
+Input [1]: [w_warehouse_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#7]
+
+(8) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [inv_warehouse_sk#2]
+Right keys [1]: [w_warehouse_sk#6]
+Join condition: None
+
+(9) Project [codegen id : 3]
+Output [3]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4]
+Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_sk#6]
+
+(10) Scan parquet default.date_dim
+Output [2]: [d_date_sk#8, d_month_seq#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#8, d_month_seq#9]
+
+(12) Filter [codegen id : 2]
+Input [2]: [d_date_sk#8, d_month_seq#9]
+Condition : (((isnotnull(d_month_seq#9) AND (d_month_seq#9 >= 1212)) AND (d_month_seq#9 <= 1223)) AND isnotnull(d_date_sk#8))
+
+(13) Project [codegen id : 2]
+Output [1]: [d_date_sk#8]
+Input [2]: [d_date_sk#8, d_month_seq#9]
+
+(14) BroadcastExchange
+Input [1]: [d_date_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [inv_date_sk#4]
+Right keys [1]: [d_date_sk#8]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [2]: [inv_item_sk#1, inv_quantity_on_hand#3]
+Input [4]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#8]
+
+(17) Exchange
+Input [2]: [inv_item_sk#1, inv_quantity_on_hand#3]
+Arguments: hashpartitioning(inv_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#11]
+
+(18) Sort [codegen id : 4]
+Input [2]: [inv_item_sk#1, inv_quantity_on_hand#3]
+Arguments: [inv_item_sk#1 ASC NULLS FIRST], false, 0
+
+(19) Scan parquet default.item
+Output [5]: [i_item_sk#12, i_brand#13, i_class#14, i_category#15, i_product_name#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_class:string,i_category:string,i_product_name:string>
+
+(20) ColumnarToRow [codegen id : 5]
+Input [5]: [i_item_sk#12, i_brand#13, i_class#14, i_category#15, i_product_name#16]
+
+(21) Filter [codegen id : 5]
+Input [5]: [i_item_sk#12, i_brand#13, i_class#14, i_category#15, i_product_name#16]
+Condition : isnotnull(i_item_sk#12)
+
+(22) Exchange
+Input [5]: [i_item_sk#12, i_brand#13, i_class#14, i_category#15, i_product_name#16]
+Arguments: hashpartitioning(i_item_sk#12, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(23) Sort [codegen id : 6]
+Input [5]: [i_item_sk#12, i_brand#13, i_class#14, i_category#15, i_product_name#16]
+Arguments: [i_item_sk#12 ASC NULLS FIRST], false, 0
+
+(24) SortMergeJoin [codegen id : 7]
+Left keys [1]: [inv_item_sk#1]
+Right keys [1]: [i_item_sk#12]
+Join condition: None
+
+(25) Project [codegen id : 7]
+Output [5]: [inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16]
+Input [7]: [inv_item_sk#1, inv_quantity_on_hand#3, i_item_sk#12, i_brand#13, i_class#14, i_category#15, i_product_name#16]
+
+(26) HashAggregate [codegen id : 7]
+Input [5]: [inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16]
+Keys [4]: [i_product_name#16, i_brand#13, i_class#14, i_category#15]
+Functions [1]: [partial_avg(inv_quantity_on_hand#3)]
+Aggregate Attributes [2]: [sum#18, count#19]
+Results [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#20, count#21]
+
+(27) Exchange
+Input [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#20, count#21]
+Arguments: hashpartitioning(i_product_name#16, i_brand#13, i_class#14, i_category#15, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(28) HashAggregate [codegen id : 8]
+Input [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#20, count#21]
+Keys [4]: [i_product_name#16, i_brand#13, i_class#14, i_category#15]
+Functions [1]: [avg(inv_quantity_on_hand#3)]
+Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#23]
+Results [5]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, avg(inv_quantity_on_hand#3)#23 AS qoh#24]
+
+(29) HashAggregate [codegen id : 8]
+Input [5]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, qoh#24]
+Keys [4]: [i_product_name#16, i_brand#13, i_class#14, i_category#15]
+Functions [1]: [partial_avg(qoh#24)]
+Aggregate Attributes [2]: [sum#25, count#26]
+Results [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#27, count#28]
+
+(30) HashAggregate [codegen id : 8]
+Input [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#27, count#28]
+Keys [4]: [i_product_name#16, i_brand#13, i_class#14, i_category#15]
+Functions [1]: [avg(qoh#24)]
+Aggregate Attributes [1]: [avg(qoh#24)#29]
+Results [5]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, avg(qoh#24)#29 AS qoh#30]
+
+(31) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_product_name#31, i_brand#32, i_class#33, i_category#34, sum#35, count#36]
+
+(32) HashAggregate [codegen id : 16]
+Input [6]: [i_product_name#31, i_brand#32, i_class#33, i_category#34, sum#35, count#36]
+Keys [4]: [i_product_name#31, i_brand#32, i_class#33, i_category#34]
+Functions [1]: [avg(inv_quantity_on_hand#37)]
+Aggregate Attributes [1]: [avg(inv_quantity_on_hand#37)#38]
+Results [4]: [i_product_name#31, i_brand#32, i_class#33, avg(inv_quantity_on_hand#37)#38 AS qoh#24]
+
+(33) HashAggregate [codegen id : 16]
+Input [4]: [i_product_name#31, i_brand#32, i_class#33, qoh#24]
+Keys [3]: [i_product_name#31, i_brand#32, i_class#33]
+Functions [1]: [partial_avg(qoh#24)]
+Aggregate Attributes [2]: [sum#39, count#40]
+Results [5]: [i_product_name#31, i_brand#32, i_class#33, sum#41, count#42]
+
+(34) Exchange
+Input [5]: [i_product_name#31, i_brand#32, i_class#33, sum#41, count#42]
+Arguments: hashpartitioning(i_product_name#31, i_brand#32, i_class#33, 5), ENSURE_REQUIREMENTS, [id=#43]
+
+(35) HashAggregate [codegen id : 17]
+Input [5]: [i_product_name#31, i_brand#32, i_class#33, sum#41, count#42]
+Keys [3]: [i_product_name#31, i_brand#32, i_class#33]
+Functions [1]: [avg(qoh#24)]
+Aggregate Attributes [1]: [avg(qoh#24)#44]
+Results [5]: [i_product_name#31, i_brand#32, i_class#33, null AS i_category#45, avg(qoh#24)#44 AS qoh#46]
+
+(36) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_product_name#47, i_brand#48, i_class#49, i_category#50, sum#51, count#52]
+
+(37) HashAggregate [codegen id : 25]
+Input [6]: [i_product_name#47, i_brand#48, i_class#49, i_category#50, sum#51, count#52]
+Keys [4]: [i_product_name#47, i_brand#48, i_class#49, i_category#50]
+Functions [1]: [avg(inv_quantity_on_hand#53)]
+Aggregate Attributes [1]: [avg(inv_quantity_on_hand#53)#54]
+Results [3]: [i_product_name#47, i_brand#48, avg(inv_quantity_on_hand#53)#54 AS qoh#24]
+
+(38) HashAggregate [codegen id : 25]
+Input [3]: [i_product_name#47, i_brand#48, qoh#24]
+Keys [2]: [i_product_name#47, i_brand#48]
+Functions [1]: [partial_avg(qoh#24)]
+Aggregate Attributes [2]: [sum#55, count#56]
+Results [4]: [i_product_name#47, i_brand#48, sum#57, count#58]
+
+(39) Exchange
+Input [4]: [i_product_name#47, i_brand#48, sum#57, count#58]
+Arguments: hashpartitioning(i_product_name#47, i_brand#48, 5), ENSURE_REQUIREMENTS, [id=#59]
+
+(40) HashAggregate [codegen id : 26]
+Input [4]: [i_product_name#47, i_brand#48, sum#57, count#58]
+Keys [2]: [i_product_name#47, i_brand#48]
+Functions [1]: [avg(qoh#24)]
+Aggregate Attributes [1]: [avg(qoh#24)#60]
+Results [5]: [i_product_name#47, i_brand#48, null AS i_class#61, null AS i_category#62, avg(qoh#24)#60 AS qoh#63]
+
+(41) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_product_name#64, i_brand#65, i_class#66, i_category#67, sum#68, count#69]
+
+(42) HashAggregate [codegen id : 34]
+Input [6]: [i_product_name#64, i_brand#65, i_class#66, i_category#67, sum#68, count#69]
+Keys [4]: [i_product_name#64, i_brand#65, i_class#66, i_category#67]
+Functions [1]: [avg(inv_quantity_on_hand#70)]
+Aggregate Attributes [1]: [avg(inv_quantity_on_hand#70)#71]
+Results [2]: [i_product_name#64, avg(inv_quantity_on_hand#70)#71 AS qoh#24]
+
+(43) HashAggregate [codegen id : 34]
+Input [2]: [i_product_name#64, qoh#24]
+Keys [1]: [i_product_name#64]
+Functions [1]: [partial_avg(qoh#24)]
+Aggregate Attributes [2]: [sum#72, count#73]
+Results [3]: [i_product_name#64, sum#74, count#75]
+
+(44) Exchange
+Input [3]: [i_product_name#64, sum#74, count#75]
+Arguments: hashpartitioning(i_product_name#64, 5), ENSURE_REQUIREMENTS, [id=#76]
+
+(45) HashAggregate [codegen id : 35]
+Input [3]: [i_product_name#64, sum#74, count#75]
+Keys [1]: [i_product_name#64]
+Functions [1]: [avg(qoh#24)]
+Aggregate Attributes [1]: [avg(qoh#24)#77]
+Results [5]: [i_product_name#64, null AS i_brand#78, null AS i_class#79, null AS i_category#80, avg(qoh#24)#77 AS qoh#81]
+
+(46) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_product_name#82, i_brand#83, i_class#84, i_category#85, sum#86, count#87]
+
+(47) HashAggregate [codegen id : 43]
+Input [6]: [i_product_name#82, i_brand#83, i_class#84, i_category#85, sum#86, count#87]
+Keys [4]: [i_product_name#82, i_brand#83, i_class#84, i_category#85]
+Functions [1]: [avg(inv_quantity_on_hand#88)]
+Aggregate Attributes [1]: [avg(inv_quantity_on_hand#88)#89]
+Results [1]: [avg(inv_quantity_on_hand#88)#89 AS qoh#24]
+
+(48) HashAggregate [codegen id : 43]
+Input [1]: [qoh#24]
+Keys: []
+Functions [1]: [partial_avg(qoh#24)]
+Aggregate Attributes [2]: [sum#90, count#91]
+Results [2]: [sum#92, count#93]
+
+(49) Exchange
+Input [2]: [sum#92, count#93]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#94]
+
+(50) HashAggregate [codegen id : 44]
+Input [2]: [sum#92, count#93]
+Keys: []
+Functions [1]: [avg(qoh#24)]
+Aggregate Attributes [1]: [avg(qoh#24)#95]
+Results [5]: [null AS i_product_name#96, null AS i_brand#97, null AS i_class#98, null AS i_category#99, avg(qoh#24)#95 AS qoh#100]
+
+(51) Union
+
+(52) TakeOrderedAndProject
+Input [5]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, qoh#30]
+Arguments: 100, [qoh#30 ASC NULLS FIRST, i_product_name#16 ASC NULLS FIRST, i_brand#13 ASC NULLS FIRST, i_class#14 ASC NULLS FIRST, i_category#15 ASC NULLS FIRST], [i_product_name#16, i_brand#13, i_class#14, i_category#15, qoh#30]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5
+ReusedExchange (53)
+
+
+(53) ReusedExchange [Reuses operator id: 14]
+Output [1]: [d_date_sk#8]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.sf100/simplified.txt
new file mode 100644
index 0000000000000..492883dbb392b
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.sf100/simplified.txt
@@ -0,0 +1,89 @@
+TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category]
+  Union
+    WholeStageCodegen (8)
+      HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(qoh),qoh,sum,count]
+        HashAggregate [i_product_name,i_brand,i_class,i_category,qoh] [sum,count,sum,count]
+          HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count]
+            InputAdapter
+              Exchange [i_product_name,i_brand,i_class,i_category] #1
+                WholeStageCodegen (7)
+                  HashAggregate [i_product_name,i_brand,i_class,i_category,inv_quantity_on_hand] [sum,count,sum,count]
+                    Project [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name]
+                      SortMergeJoin [inv_item_sk,i_item_sk]
+                        InputAdapter
+                          WholeStageCodegen (4)
+                            Sort [inv_item_sk]
+                              InputAdapter
+                                Exchange [inv_item_sk] #2
+                                  WholeStageCodegen (3)
+                                    Project [inv_item_sk,inv_quantity_on_hand]
+                                      BroadcastHashJoin [inv_date_sk,d_date_sk]
+                                        Project [inv_item_sk,inv_quantity_on_hand,inv_date_sk]
+                                          BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk]
+                                            Filter [inv_item_sk,inv_warehouse_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #1
+                                                      ReusedExchange [d_date_sk] #3
+                                            InputAdapter
+                                              BroadcastExchange #4
+                                                WholeStageCodegen (1)
+                                                  Filter [w_warehouse_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.warehouse [w_warehouse_sk]
+                                        InputAdapter
+                                          BroadcastExchange #3
+                                            WholeStageCodegen (2)
+                                              Project [d_date_sk]
+                                                Filter [d_month_seq,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_month_seq]
+                        InputAdapter
+                          WholeStageCodegen (6)
+                            Sort [i_item_sk]
+                              InputAdapter
+                                Exchange [i_item_sk] #5
+                                  WholeStageCodegen (5)
+                                    Filter [i_item_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.item [i_item_sk,i_brand,i_class,i_category,i_product_name]
+    WholeStageCodegen (17)
+      HashAggregate [i_product_name,i_brand,i_class,sum,count] [avg(qoh),i_category,qoh,sum,count]
+        InputAdapter
+          Exchange [i_product_name,i_brand,i_class] #6
+            WholeStageCodegen (16)
+              HashAggregate [i_product_name,i_brand,i_class,qoh] [sum,count,sum,count]
+                HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count]
+                  InputAdapter
+                    ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #7
+    WholeStageCodegen (26)
+      HashAggregate [i_product_name,i_brand,sum,count] [avg(qoh),i_class,i_category,qoh,sum,count]
+        InputAdapter
+          Exchange [i_product_name,i_brand] #8
+            WholeStageCodegen (25)
+              HashAggregate [i_product_name,i_brand,qoh] [sum,count,sum,count]
+                HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count]
+                  InputAdapter
+                    ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #7
+    WholeStageCodegen (35)
+      HashAggregate [i_product_name,sum,count] [avg(qoh),i_brand,i_class,i_category,qoh,sum,count]
+        InputAdapter
+          Exchange [i_product_name] #9
+            WholeStageCodegen (34)
+              HashAggregate [i_product_name,qoh] [sum,count,sum,count]
+                HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count]
+                  InputAdapter
+                    ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #7
+    WholeStageCodegen (44)
+      HashAggregate [sum,count] [avg(qoh),i_product_name,i_brand,i_class,i_category,qoh,sum,count]
+        InputAdapter
+          Exchange #10
+            WholeStageCodegen (43)
+              HashAggregate [qoh] [sum,count,sum,count]
+                HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count]
+                  InputAdapter
+                    ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #7
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/explain.txt
new file mode 100644
index 0000000000000..527d2eb5e86a4
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/explain.txt
@@ -0,0 +1,312 @@
+== Physical Plan ==
+TakeOrderedAndProject (49)
++- Union (48)
+   :- * HashAggregate (27)
+   :  +- * HashAggregate (26)
+   :     +- * HashAggregate (25)
+   :        +- Exchange (24)
+   :           +- * HashAggregate (23)
+   :              +- * Project (22)
+   :                 +- * BroadcastHashJoin Inner BuildRight (21)
+   :                    :- * Project (16)
+   :                    :  +- * BroadcastHashJoin Inner BuildRight (15)
+   :                    :     :- * Project (10)
+   :                    :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+   :                    :     :     :- * Filter (3)
+   :                    :     :     :  +- * ColumnarToRow (2)
+   :                    :     :     :     +- Scan parquet default.inventory (1)
+   :                    :     :     +- BroadcastExchange (8)
+   :                    :     :        +- * Project (7)
+   :                    :     :           +- * Filter (6)
+   :                    :     :              +- * ColumnarToRow (5)
+   :                    :     :                 +- Scan parquet default.date_dim (4)
+   :                    :     +- BroadcastExchange (14)
+   :                    :        +- * Filter (13)
+   :                    :           +- * ColumnarToRow (12)
+   :                    :              +- Scan parquet default.item (11)
+   :                    +- BroadcastExchange (20)
+   :                       +- * Filter (19)
+   :                          +- * ColumnarToRow (18)
+   :                             +- Scan parquet default.warehouse (17)
+   :- * HashAggregate (32)
+   :  +- Exchange (31)
+   :     +- * HashAggregate (30)
+   :        +- * HashAggregate (29)
+   :           +- ReusedExchange (28)
+   :- * HashAggregate (37)
+   :  +- Exchange (36)
+   :     +- * HashAggregate (35)
+   :        +- * HashAggregate (34)
+   :           +- ReusedExchange (33)
+   :- * HashAggregate (42)
+   :  +- Exchange (41)
+   :     +- * HashAggregate (40)
+   :        +- * HashAggregate (39)
+   :           +- ReusedExchange (38)
+   +- * HashAggregate (47)
+      +- Exchange (46)
+         +- * HashAggregate (45)
+            +- * HashAggregate (44)
+               +- ReusedExchange (43)
+
+
+(1) Scan parquet default.inventory
+Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)]
+ReadSchema: struct<inv_item_sk:int,inv_warehouse_sk:int,inv_quantity_on_hand:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+
+(3) Filter [codegen id : 4]
+Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4]
+Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2))
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#6, d_month_seq#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#6, d_month_seq#7]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#6, d_month_seq#7]
+Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#6))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#6]
+Input [2]: [d_date_sk#6, d_month_seq#7]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#8]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3]
+Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#6]
+
+(11) Scan parquet default.item
+Output [5]: [i_item_sk#9, i_brand#10, i_class#11, i_category#12, i_product_name#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_class:string,i_category:string,i_product_name:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [5]: [i_item_sk#9, i_brand#10, i_class#11, i_category#12, i_product_name#13]
+
+(13) Filter [codegen id : 2]
+Input [5]: [i_item_sk#9, i_brand#10, i_class#11, i_category#12, i_product_name#13]
+Condition : isnotnull(i_item_sk#9)
+
+(14) BroadcastExchange
+Input [5]: [i_item_sk#9, i_brand#10, i_class#11, i_category#12, i_product_name#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#14]
+
+(15) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_item_sk#1]
+Right keys [1]: [i_item_sk#9]
+Join condition: None
+
+(16) Project [codegen id : 4]
+Output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#10, i_class#11, i_category#12, i_product_name#13]
+Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#9, i_brand#10, i_class#11, i_category#12, i_product_name#13]
+
+(17) Scan parquet default.warehouse
+Output [1]: [w_warehouse_sk#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/warehouse]
+PushedFilters: [IsNotNull(w_warehouse_sk)]
+ReadSchema: struct<w_warehouse_sk:int>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [1]: [w_warehouse_sk#15]
+
+(19) Filter [codegen id : 3]
+Input [1]: [w_warehouse_sk#15]
+Condition : isnotnull(w_warehouse_sk#15)
+
+(20) BroadcastExchange
+Input [1]: [w_warehouse_sk#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#16]
+
+(21) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [inv_warehouse_sk#2]
+Right keys [1]: [w_warehouse_sk#15]
+Join condition: None
+
+(22) Project [codegen id : 4]
+Output [5]: [inv_quantity_on_hand#3, i_brand#10, i_class#11, i_category#12, i_product_name#13]
+Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#10, i_class#11, i_category#12, i_product_name#13, w_warehouse_sk#15]
+
+(23) HashAggregate [codegen id : 4]
+Input [5]: [inv_quantity_on_hand#3, i_brand#10, i_class#11, i_category#12, i_product_name#13]
+Keys [4]: [i_product_name#13, i_brand#10, i_class#11, i_category#12]
+Functions [1]: [partial_avg(inv_quantity_on_hand#3)]
+Aggregate Attributes [2]: [sum#17, count#18]
+Results [6]: [i_product_name#13, i_brand#10, i_class#11, i_category#12, sum#19, count#20]
+
+(24) Exchange
+Input [6]: [i_product_name#13, i_brand#10, i_class#11, i_category#12, sum#19, count#20]
+Arguments: hashpartitioning(i_product_name#13, i_brand#10, i_class#11, i_category#12, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(25) HashAggregate [codegen id : 5]
+Input [6]: [i_product_name#13, i_brand#10, i_class#11, i_category#12, sum#19, count#20]
+Keys [4]: [i_product_name#13, i_brand#10, i_class#11, i_category#12]
+Functions [1]: [avg(inv_quantity_on_hand#3)]
+Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#22]
+Results [5]: [i_product_name#13, i_brand#10, i_class#11, i_category#12, avg(inv_quantity_on_hand#3)#22 AS qoh#23]
+
+(26) HashAggregate [codegen id : 5]
+Input [5]: [i_product_name#13, i_brand#10, i_class#11, i_category#12, qoh#23]
+Keys [4]: [i_product_name#13, i_brand#10, i_class#11, i_category#12]
+Functions [1]: [partial_avg(qoh#23)]
+Aggregate Attributes [2]: [sum#24, count#25]
+Results [6]: [i_product_name#13, i_brand#10, i_class#11, i_category#12, sum#26, count#27]
+
+(27) HashAggregate [codegen id : 5]
+Input [6]: [i_product_name#13, i_brand#10, i_class#11, i_category#12, sum#26, count#27]
+Keys [4]: [i_product_name#13, i_brand#10, i_class#11, i_category#12]
+Functions [1]: [avg(qoh#23)]
+Aggregate Attributes [1]: [avg(qoh#23)#28]
+Results [5]: [i_product_name#13, i_brand#10, i_class#11, i_category#12, avg(qoh#23)#28 AS qoh#29]
+
+(28) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_product_name#30, i_brand#31, i_class#32, i_category#33, sum#34, count#35]
+
+(29) HashAggregate [codegen id : 10]
+Input [6]: [i_product_name#30, i_brand#31, i_class#32, i_category#33, sum#34, count#35]
+Keys [4]: [i_product_name#30, i_brand#31, i_class#32, i_category#33]
+Functions [1]: [avg(inv_quantity_on_hand#36)]
+Aggregate Attributes [1]: [avg(inv_quantity_on_hand#36)#37]
+Results [4]: [i_product_name#30, i_brand#31, i_class#32, avg(inv_quantity_on_hand#36)#37 AS qoh#23]
+
+(30) HashAggregate [codegen id : 10]
+Input [4]: [i_product_name#30, i_brand#31, i_class#32, qoh#23]
+Keys [3]: [i_product_name#30, i_brand#31, i_class#32]
+Functions [1]: [partial_avg(qoh#23)]
+Aggregate Attributes [2]: [sum#38, count#39]
+Results [5]: [i_product_name#30, i_brand#31, i_class#32, sum#40, count#41]
+
+(31) Exchange
+Input [5]: [i_product_name#30, i_brand#31, i_class#32, sum#40, count#41]
+Arguments: hashpartitioning(i_product_name#30, i_brand#31, i_class#32, 5), ENSURE_REQUIREMENTS, [id=#42]
+
+(32) HashAggregate [codegen id : 11]
+Input [5]: [i_product_name#30, i_brand#31, i_class#32, sum#40, count#41]
+Keys [3]: [i_product_name#30, i_brand#31, i_class#32]
+Functions [1]: [avg(qoh#23)]
+Aggregate Attributes [1]: [avg(qoh#23)#43]
+Results [5]: [i_product_name#30, i_brand#31, i_class#32, null AS i_category#44, avg(qoh#23)#43 AS qoh#45]
+
+(33) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_product_name#46, i_brand#47, i_class#48, i_category#49, sum#50, count#51]
+
+(34) HashAggregate [codegen id : 16]
+Input [6]: [i_product_name#46, i_brand#47, i_class#48, i_category#49, sum#50, count#51]
+Keys [4]: [i_product_name#46, i_brand#47, i_class#48, i_category#49]
+Functions [1]: [avg(inv_quantity_on_hand#52)]
+Aggregate Attributes [1]: [avg(inv_quantity_on_hand#52)#53]
+Results [3]: [i_product_name#46, i_brand#47, avg(inv_quantity_on_hand#52)#53 AS qoh#23]
+
+(35) HashAggregate [codegen id : 16]
+Input [3]: [i_product_name#46, i_brand#47, qoh#23]
+Keys [2]: [i_product_name#46, i_brand#47]
+Functions [1]: [partial_avg(qoh#23)]
+Aggregate Attributes [2]: [sum#54, count#55]
+Results [4]: [i_product_name#46, i_brand#47, sum#56, count#57]
+
+(36) Exchange
+Input [4]: [i_product_name#46, i_brand#47, sum#56, count#57]
+Arguments: hashpartitioning(i_product_name#46, i_brand#47, 5), ENSURE_REQUIREMENTS, [id=#58]
+
+(37) HashAggregate [codegen id : 17]
+Input [4]: [i_product_name#46, i_brand#47, sum#56, count#57]
+Keys [2]: [i_product_name#46, i_brand#47]
+Functions [1]: [avg(qoh#23)]
+Aggregate Attributes [1]: [avg(qoh#23)#59]
+Results [5]: [i_product_name#46, i_brand#47, null AS i_class#60, null AS i_category#61, avg(qoh#23)#59 AS qoh#62]
+
+(38) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_product_name#63, i_brand#64, i_class#65, i_category#66, sum#67, count#68]
+
+(39) HashAggregate [codegen id : 22]
+Input [6]: [i_product_name#63, i_brand#64, i_class#65, i_category#66, sum#67, count#68]
+Keys [4]: [i_product_name#63, i_brand#64, i_class#65, i_category#66]
+Functions [1]: [avg(inv_quantity_on_hand#69)]
+Aggregate Attributes [1]: [avg(inv_quantity_on_hand#69)#70]
+Results [2]: [i_product_name#63, avg(inv_quantity_on_hand#69)#70 AS qoh#23]
+
+(40) HashAggregate [codegen id : 22]
+Input [2]: [i_product_name#63, qoh#23]
+Keys [1]: [i_product_name#63]
+Functions [1]: [partial_avg(qoh#23)]
+Aggregate Attributes [2]: [sum#71, count#72]
+Results [3]: [i_product_name#63, sum#73, count#74]
+
+(41) Exchange
+Input [3]: [i_product_name#63, sum#73, count#74]
+Arguments: hashpartitioning(i_product_name#63, 5), ENSURE_REQUIREMENTS, [id=#75]
+
+(42) HashAggregate [codegen id : 23]
+Input [3]: [i_product_name#63, sum#73, count#74]
+Keys [1]: [i_product_name#63]
+Functions [1]: [avg(qoh#23)]
+Aggregate Attributes [1]: [avg(qoh#23)#76]
+Results [5]: [i_product_name#63, null AS i_brand#77, null AS i_class#78, null AS i_category#79, avg(qoh#23)#76 AS qoh#80]
+
+(43) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_product_name#81, i_brand#82, i_class#83, i_category#84, sum#85, count#86]
+
+(44) HashAggregate [codegen id : 28]
+Input [6]: [i_product_name#81, i_brand#82, i_class#83, i_category#84, sum#85, count#86]
+Keys [4]: [i_product_name#81, i_brand#82, i_class#83, i_category#84]
+Functions [1]: [avg(inv_quantity_on_hand#87)]
+Aggregate Attributes [1]: [avg(inv_quantity_on_hand#87)#88]
+Results [1]: [avg(inv_quantity_on_hand#87)#88 AS qoh#23]
+
+(45) HashAggregate [codegen id : 28]
+Input [1]: [qoh#23]
+Keys: []
+Functions [1]: [partial_avg(qoh#23)]
+Aggregate Attributes [2]: [sum#89, count#90]
+Results [2]: [sum#91, count#92]
+
+(46) Exchange
+Input [2]: [sum#91, count#92]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#93]
+
+(47) HashAggregate [codegen id : 29]
+Input [2]: [sum#91, count#92]
+Keys: []
+Functions [1]: [avg(qoh#23)]
+Aggregate Attributes [1]: [avg(qoh#23)#94]
+Results [5]: [null AS i_product_name#95, null AS i_brand#96, null AS i_class#97, null AS i_category#98, avg(qoh#23)#94 AS qoh#99]
+
+(48) Union
+
+(49) TakeOrderedAndProject
+Input [5]: [i_product_name#13, i_brand#10, i_class#11, i_category#12, qoh#29]
+Arguments: 100, [qoh#29 ASC NULLS FIRST, i_product_name#13 ASC NULLS FIRST, i_brand#10 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_category#12 ASC NULLS FIRST], [i_product_name#13, i_brand#10, i_class#11, i_category#12, qoh#29]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5
+ReusedExchange (50)
+
+
+(50) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#6]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/simplified.txt
new file mode 100644
index 0000000000000..6b124d3ceed7a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/simplified.txt
@@ -0,0 +1,80 @@
+TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category]
+  Union
+    WholeStageCodegen (5)
+      HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(qoh),qoh,sum,count]
+        HashAggregate [i_product_name,i_brand,i_class,i_category,qoh] [sum,count,sum,count]
+          HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count]
+            InputAdapter
+              Exchange [i_product_name,i_brand,i_class,i_category] #1
+                WholeStageCodegen (4)
+                  HashAggregate [i_product_name,i_brand,i_class,i_category,inv_quantity_on_hand] [sum,count,sum,count]
+                    Project [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name]
+                      BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk]
+                        Project [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name]
+                          BroadcastHashJoin [inv_item_sk,i_item_sk]
+                            Project [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand]
+                              BroadcastHashJoin [inv_date_sk,d_date_sk]
+                                Filter [inv_item_sk,inv_warehouse_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk]
+                                        SubqueryBroadcast [d_date_sk] #1
+                                          ReusedExchange [d_date_sk] #2
+                                InputAdapter
+                                  BroadcastExchange #2
+                                    WholeStageCodegen (1)
+                                      Project [d_date_sk]
+                                        Filter [d_month_seq,d_date_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.date_dim [d_date_sk,d_month_seq]
+                            InputAdapter
+                              BroadcastExchange #3
+                                WholeStageCodegen (2)
+                                  Filter [i_item_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.item [i_item_sk,i_brand,i_class,i_category,i_product_name]
+                        InputAdapter
+                          BroadcastExchange #4
+                            WholeStageCodegen (3)
+                              Filter [w_warehouse_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.warehouse [w_warehouse_sk]
+    WholeStageCodegen (11)
+      HashAggregate [i_product_name,i_brand,i_class,sum,count] [avg(qoh),i_category,qoh,sum,count]
+        InputAdapter
+          Exchange [i_product_name,i_brand,i_class] #5
+            WholeStageCodegen (10)
+              HashAggregate [i_product_name,i_brand,i_class,qoh] [sum,count,sum,count]
+                HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count]
+                  InputAdapter
+                    ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #6
+    WholeStageCodegen (17)
+      HashAggregate [i_product_name,i_brand,sum,count] [avg(qoh),i_class,i_category,qoh,sum,count]
+        InputAdapter
+          Exchange [i_product_name,i_brand] #7
+            WholeStageCodegen (16)
+              HashAggregate [i_product_name,i_brand,qoh] [sum,count,sum,count]
+                HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count]
+                  InputAdapter
+                    ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #6
+    WholeStageCodegen (23)
+      HashAggregate [i_product_name,sum,count] [avg(qoh),i_brand,i_class,i_category,qoh,sum,count]
+        InputAdapter
+          Exchange [i_product_name] #8
+            WholeStageCodegen (22)
+              HashAggregate [i_product_name,qoh] [sum,count,sum,count]
+                HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count]
+                  InputAdapter
+                    ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #6
+    WholeStageCodegen (29)
+      HashAggregate [sum,count] [avg(qoh),i_product_name,i_brand,i_class,i_category,qoh,sum,count]
+        InputAdapter
+          Exchange #9
+            WholeStageCodegen (28)
+              HashAggregate [qoh] [sum,count,sum,count]
+                HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count]
+                  InputAdapter
+                    ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #6
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.sf100/explain.txt
new file mode 100644
index 0000000000000..7cdc8074e74e1
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.sf100/explain.txt
@@ -0,0 +1,552 @@
+== Physical Plan ==
+* Sort (49)
++- Exchange (48)
+   +- * Project (47)
+      +- * Filter (46)
+         +- * HashAggregate (45)
+            +- Exchange (44)
+               +- * HashAggregate (43)
+                  +- * HashAggregate (42)
+                     +- Exchange (41)
+                        +- * HashAggregate (40)
+                           +- * Project (39)
+                              +- * SortMergeJoin Inner (38)
+                                 :- * Sort (31)
+                                 :  +- Exchange (30)
+                                 :     +- * Project (29)
+                                 :        +- * BroadcastHashJoin Inner BuildRight (28)
+                                 :           :- * Project (23)
+                                 :           :  +- * BroadcastHashJoin Inner BuildLeft (22)
+                                 :           :     :- BroadcastExchange (17)
+                                 :           :     :  +- * Project (16)
+                                 :           :     :     +- * BroadcastHashJoin Inner BuildLeft (15)
+                                 :           :     :        :- BroadcastExchange (11)
+                                 :           :     :        :  +- * Project (10)
+                                 :           :     :        :     +- * BroadcastHashJoin Inner BuildLeft (9)
+                                 :           :     :        :        :- BroadcastExchange (5)
+                                 :           :     :        :        :  +- * Project (4)
+                                 :           :     :        :        :     +- * Filter (3)
+                                 :           :     :        :        :        +- * ColumnarToRow (2)
+                                 :           :     :        :        :           +- Scan parquet default.store (1)
+                                 :           :     :        :        +- * Filter (8)
+                                 :           :     :        :           +- * ColumnarToRow (7)
+                                 :           :     :        :              +- Scan parquet default.customer_address (6)
+                                 :           :     :        +- * Filter (14)
+                                 :           :     :           +- * ColumnarToRow (13)
+                                 :           :     :              +- Scan parquet default.customer (12)
+                                 :           :     +- * Project (21)
+                                 :           :        +- * Filter (20)
+                                 :           :           +- * ColumnarToRow (19)
+                                 :           :              +- Scan parquet default.store_sales (18)
+                                 :           +- BroadcastExchange (27)
+                                 :              +- * Filter (26)
+                                 :                 +- * ColumnarToRow (25)
+                                 :                    +- Scan parquet default.item (24)
+                                 +- * Sort (37)
+                                    +- Exchange (36)
+                                       +- * Project (35)
+                                          +- * Filter (34)
+                                             +- * ColumnarToRow (33)
+                                                +- Scan parquet default.store_returns (32)
+
+
+(1) Scan parquet default.store
+Output [5]: [s_store_sk#1, s_store_name#2, s_market_id#3, s_state#4, s_zip#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_market_id:int,s_state:string,s_zip:string>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [5]: [s_store_sk#1, s_store_name#2, s_market_id#3, s_state#4, s_zip#5]
+
+(3) Filter [codegen id : 1]
+Input [5]: [s_store_sk#1, s_store_name#2, s_market_id#3, s_state#4, s_zip#5]
+Condition : (((isnotnull(s_market_id#3) AND (s_market_id#3 = 8)) AND isnotnull(s_store_sk#1)) AND isnotnull(s_zip#5))
+
+(4) Project [codegen id : 1]
+Output [4]: [s_store_sk#1, s_store_name#2, s_state#4, s_zip#5]
+Input [5]: [s_store_sk#1, s_store_name#2, s_market_id#3, s_state#4, s_zip#5]
+
+(5) BroadcastExchange
+Input [4]: [s_store_sk#1, s_store_name#2, s_state#4, s_zip#5]
+Arguments: HashedRelationBroadcastMode(List(input[3, string, true]),false), [id=#6]
+
+(6) Scan parquet default.customer_address
+Output [4]: [ca_address_sk#7, ca_state#8, ca_zip#9, ca_country#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_country), IsNotNull(ca_zip)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string,ca_zip:string,ca_country:string>
+
+(7) ColumnarToRow
+Input [4]: [ca_address_sk#7, ca_state#8, ca_zip#9, ca_country#10]
+
+(8) Filter
+Input [4]: [ca_address_sk#7, ca_state#8, ca_zip#9, ca_country#10]
+Condition : ((isnotnull(ca_address_sk#7) AND isnotnull(ca_country#10)) AND isnotnull(ca_zip#9))
+
+(9) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [s_zip#5]
+Right keys [1]: [ca_zip#9]
+Join condition: None
+
+(10) Project [codegen id : 2]
+Output [6]: [s_store_sk#1, s_store_name#2, s_state#4, ca_address_sk#7, ca_state#8, ca_country#10]
+Input [8]: [s_store_sk#1, s_store_name#2, s_state#4, s_zip#5, ca_address_sk#7, ca_state#8, ca_zip#9, ca_country#10]
+
+(11) BroadcastExchange
+Input [6]: [s_store_sk#1, s_store_name#2, s_state#4, ca_address_sk#7, ca_state#8, ca_country#10]
+Arguments: HashedRelationBroadcastMode(List(input[3, int, true], upper(input[5, string, true])),false), [id=#11]
+
+(12) Scan parquet default.customer
+Output [5]: [c_customer_sk#12, c_current_addr_sk#13, c_first_name#14, c_last_name#15, c_birth_country#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_birth_country)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int,c_first_name:string,c_last_name:string,c_birth_country:string>
+
+(13) ColumnarToRow
+Input [5]: [c_customer_sk#12, c_current_addr_sk#13, c_first_name#14, c_last_name#15, c_birth_country#16]
+
+(14) Filter
+Input [5]: [c_customer_sk#12, c_current_addr_sk#13, c_first_name#14, c_last_name#15, c_birth_country#16]
+Condition : ((isnotnull(c_customer_sk#12) AND isnotnull(c_current_addr_sk#13)) AND isnotnull(c_birth_country#16))
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [2]: [ca_address_sk#7, upper(ca_country#10)]
+Right keys [2]: [c_current_addr_sk#13, c_birth_country#16]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [7]: [s_store_sk#1, s_store_name#2, s_state#4, ca_state#8, c_customer_sk#12, c_first_name#14, c_last_name#15]
+Input [11]: [s_store_sk#1, s_store_name#2, s_state#4, ca_address_sk#7, ca_state#8, ca_country#10, c_customer_sk#12, c_current_addr_sk#13, c_first_name#14, c_last_name#15, c_birth_country#16]
+
+(17) BroadcastExchange
+Input [7]: [s_store_sk#1, s_store_name#2, s_state#4, ca_state#8, c_customer_sk#12, c_first_name#14, c_last_name#15]
+Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, true] as bigint), 32) | (cast(input[4, int, true] as bigint) & 4294967295))),false), [id=#17]
+
+(18) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#18, ss_customer_sk#19, ss_store_sk#20, ss_ticket_number#21, ss_net_paid#22, ss_sold_date_sk#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_net_paid:decimal(7,2)>
+
+(19) ColumnarToRow
+Input [6]: [ss_item_sk#18, ss_customer_sk#19, ss_store_sk#20, ss_ticket_number#21, ss_net_paid#22, ss_sold_date_sk#23]
+
+(20) Filter
+Input [6]: [ss_item_sk#18, ss_customer_sk#19, ss_store_sk#20, ss_ticket_number#21, ss_net_paid#22, ss_sold_date_sk#23]
+Condition : (((isnotnull(ss_ticket_number#21) AND isnotnull(ss_item_sk#18)) AND isnotnull(ss_store_sk#20)) AND isnotnull(ss_customer_sk#19))
+
+(21) Project
+Output [5]: [ss_item_sk#18, ss_customer_sk#19, ss_store_sk#20, ss_ticket_number#21, ss_net_paid#22]
+Input [6]: [ss_item_sk#18, ss_customer_sk#19, ss_store_sk#20, ss_ticket_number#21, ss_net_paid#22, ss_sold_date_sk#23]
+
+(22) BroadcastHashJoin [codegen id : 5]
+Left keys [2]: [s_store_sk#1, c_customer_sk#12]
+Right keys [2]: [ss_store_sk#20, ss_customer_sk#19]
+Join condition: None
+
+(23) Project [codegen id : 5]
+Output [8]: [s_store_name#2, s_state#4, ca_state#8, c_first_name#14, c_last_name#15, ss_item_sk#18, ss_ticket_number#21, ss_net_paid#22]
+Input [12]: [s_store_sk#1, s_store_name#2, s_state#4, ca_state#8, c_customer_sk#12, c_first_name#14, c_last_name#15, ss_item_sk#18, ss_customer_sk#19, ss_store_sk#20, ss_ticket_number#21, ss_net_paid#22]
+
+(24) Scan parquet default.item
+Output [6]: [i_item_sk#24, i_current_price#25, i_size#26, i_color#27, i_units#28, i_manager_id#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_color), EqualTo(i_color,pale                ), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_current_price:decimal(7,2),i_size:string,i_color:string,i_units:string,i_manager_id:int>
+
+(25) ColumnarToRow [codegen id : 4]
+Input [6]: [i_item_sk#24, i_current_price#25, i_size#26, i_color#27, i_units#28, i_manager_id#29]
+
+(26) Filter [codegen id : 4]
+Input [6]: [i_item_sk#24, i_current_price#25, i_size#26, i_color#27, i_units#28, i_manager_id#29]
+Condition : ((isnotnull(i_color#27) AND (i_color#27 = pale                )) AND isnotnull(i_item_sk#24))
+
+(27) BroadcastExchange
+Input [6]: [i_item_sk#24, i_current_price#25, i_size#26, i_color#27, i_units#28, i_manager_id#29]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#30]
+
+(28) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_item_sk#18]
+Right keys [1]: [i_item_sk#24]
+Join condition: None
+
+(29) Project [codegen id : 5]
+Output [13]: [s_store_name#2, s_state#4, ca_state#8, c_first_name#14, c_last_name#15, ss_item_sk#18, ss_ticket_number#21, ss_net_paid#22, i_current_price#25, i_size#26, i_color#27, i_units#28, i_manager_id#29]
+Input [14]: [s_store_name#2, s_state#4, ca_state#8, c_first_name#14, c_last_name#15, ss_item_sk#18, ss_ticket_number#21, ss_net_paid#22, i_item_sk#24, i_current_price#25, i_size#26, i_color#27, i_units#28, i_manager_id#29]
+
+(30) Exchange
+Input [13]: [s_store_name#2, s_state#4, ca_state#8, c_first_name#14, c_last_name#15, ss_item_sk#18, ss_ticket_number#21, ss_net_paid#22, i_current_price#25, i_size#26, i_color#27, i_units#28, i_manager_id#29]
+Arguments: hashpartitioning(ss_ticket_number#21, ss_item_sk#18, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(31) Sort [codegen id : 6]
+Input [13]: [s_store_name#2, s_state#4, ca_state#8, c_first_name#14, c_last_name#15, ss_item_sk#18, ss_ticket_number#21, ss_net_paid#22, i_current_price#25, i_size#26, i_color#27, i_units#28, i_manager_id#29]
+Arguments: [ss_ticket_number#21 ASC NULLS FIRST, ss_item_sk#18 ASC NULLS FIRST], false, 0
+
+(32) Scan parquet default.store_returns
+Output [3]: [sr_item_sk#32, sr_ticket_number#33, sr_returned_date_sk#34]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int>
+
+(33) ColumnarToRow [codegen id : 7]
+Input [3]: [sr_item_sk#32, sr_ticket_number#33, sr_returned_date_sk#34]
+
+(34) Filter [codegen id : 7]
+Input [3]: [sr_item_sk#32, sr_ticket_number#33, sr_returned_date_sk#34]
+Condition : (isnotnull(sr_ticket_number#33) AND isnotnull(sr_item_sk#32))
+
+(35) Project [codegen id : 7]
+Output [2]: [sr_item_sk#32, sr_ticket_number#33]
+Input [3]: [sr_item_sk#32, sr_ticket_number#33, sr_returned_date_sk#34]
+
+(36) Exchange
+Input [2]: [sr_item_sk#32, sr_ticket_number#33]
+Arguments: hashpartitioning(sr_ticket_number#33, sr_item_sk#32, 5), ENSURE_REQUIREMENTS, [id=#35]
+
+(37) Sort [codegen id : 8]
+Input [2]: [sr_item_sk#32, sr_ticket_number#33]
+Arguments: [sr_ticket_number#33 ASC NULLS FIRST, sr_item_sk#32 ASC NULLS FIRST], false, 0
+
+(38) SortMergeJoin [codegen id : 9]
+Left keys [2]: [ss_ticket_number#21, ss_item_sk#18]
+Right keys [2]: [sr_ticket_number#33, sr_item_sk#32]
+Join condition: None
+
+(39) Project [codegen id : 9]
+Output [11]: [ss_net_paid#22, s_store_name#2, s_state#4, i_current_price#25, i_size#26, i_color#27, i_units#28, i_manager_id#29, c_first_name#14, c_last_name#15, ca_state#8]
+Input [15]: [s_store_name#2, s_state#4, ca_state#8, c_first_name#14, c_last_name#15, ss_item_sk#18, ss_ticket_number#21, ss_net_paid#22, i_current_price#25, i_size#26, i_color#27, i_units#28, i_manager_id#29, sr_item_sk#32, sr_ticket_number#33]
+
+(40) HashAggregate [codegen id : 9]
+Input [11]: [ss_net_paid#22, s_store_name#2, s_state#4, i_current_price#25, i_size#26, i_color#27, i_units#28, i_manager_id#29, c_first_name#14, c_last_name#15, ca_state#8]
+Keys [10]: [c_last_name#15, c_first_name#14, s_store_name#2, ca_state#8, s_state#4, i_color#27, i_current_price#25, i_manager_id#29, i_units#28, i_size#26]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#22))]
+Aggregate Attributes [1]: [sum#36]
+Results [11]: [c_last_name#15, c_first_name#14, s_store_name#2, ca_state#8, s_state#4, i_color#27, i_current_price#25, i_manager_id#29, i_units#28, i_size#26, sum#37]
+
+(41) Exchange
+Input [11]: [c_last_name#15, c_first_name#14, s_store_name#2, ca_state#8, s_state#4, i_color#27, i_current_price#25, i_manager_id#29, i_units#28, i_size#26, sum#37]
+Arguments: hashpartitioning(c_last_name#15, c_first_name#14, s_store_name#2, ca_state#8, s_state#4, i_color#27, i_current_price#25, i_manager_id#29, i_units#28, i_size#26, 5), ENSURE_REQUIREMENTS, [id=#38]
+
+(42) HashAggregate [codegen id : 10]
+Input [11]: [c_last_name#15, c_first_name#14, s_store_name#2, ca_state#8, s_state#4, i_color#27, i_current_price#25, i_manager_id#29, i_units#28, i_size#26, sum#37]
+Keys [10]: [c_last_name#15, c_first_name#14, s_store_name#2, ca_state#8, s_state#4, i_color#27, i_current_price#25, i_manager_id#29, i_units#28, i_size#26]
+Functions [1]: [sum(UnscaledValue(ss_net_paid#22))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#22))#39]
+Results [4]: [c_last_name#15, c_first_name#14, s_store_name#2, MakeDecimal(sum(UnscaledValue(ss_net_paid#22))#39,17,2) AS netpaid#40]
+
+(43) HashAggregate [codegen id : 10]
+Input [4]: [c_last_name#15, c_first_name#14, s_store_name#2, netpaid#40]
+Keys [3]: [c_last_name#15, c_first_name#14, s_store_name#2]
+Functions [1]: [partial_sum(netpaid#40)]
+Aggregate Attributes [2]: [sum#41, isEmpty#42]
+Results [5]: [c_last_name#15, c_first_name#14, s_store_name#2, sum#43, isEmpty#44]
+
+(44) Exchange
+Input [5]: [c_last_name#15, c_first_name#14, s_store_name#2, sum#43, isEmpty#44]
+Arguments: hashpartitioning(c_last_name#15, c_first_name#14, s_store_name#2, 5), ENSURE_REQUIREMENTS, [id=#45]
+
+(45) HashAggregate [codegen id : 11]
+Input [5]: [c_last_name#15, c_first_name#14, s_store_name#2, sum#43, isEmpty#44]
+Keys [3]: [c_last_name#15, c_first_name#14, s_store_name#2]
+Functions [1]: [sum(netpaid#40)]
+Aggregate Attributes [1]: [sum(netpaid#40)#46]
+Results [5]: [c_last_name#15, c_first_name#14, s_store_name#2, sum(netpaid#40)#46 AS paid#47, sum(netpaid#40)#46 AS sum(netpaid#40)#48]
+
+(46) Filter [codegen id : 11]
+Input [5]: [c_last_name#15, c_first_name#14, s_store_name#2, paid#47, sum(netpaid#40)#48]
+Condition : (isnotnull(sum(netpaid#40)#48) AND (cast(sum(netpaid#40)#48 as decimal(33,8)) > cast(Subquery scalar-subquery#49, [id=#50] as decimal(33,8))))
+
+(47) Project [codegen id : 11]
+Output [4]: [c_last_name#15, c_first_name#14, s_store_name#2, paid#47]
+Input [5]: [c_last_name#15, c_first_name#14, s_store_name#2, paid#47, sum(netpaid#40)#48]
+
+(48) Exchange
+Input [4]: [c_last_name#15, c_first_name#14, s_store_name#2, paid#47]
+Arguments: rangepartitioning(c_last_name#15 ASC NULLS FIRST, c_first_name#14 ASC NULLS FIRST, s_store_name#2 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [id=#51]
+
+(49) Sort [codegen id : 12]
+Input [4]: [c_last_name#15, c_first_name#14, s_store_name#2, paid#47]
+Arguments: [c_last_name#15 ASC NULLS FIRST, c_first_name#14 ASC NULLS FIRST, s_store_name#2 ASC NULLS FIRST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#49, [id=#50]
+* HashAggregate (97)
++- Exchange (96)
+   +- * HashAggregate (95)
+      +- * HashAggregate (94)
+         +- Exchange (93)
+            +- * HashAggregate (92)
+               +- * Project (91)
+                  +- * SortMergeJoin Inner (90)
+                     :- * Sort (83)
+                     :  +- Exchange (82)
+                     :     +- * Project (81)
+                     :        +- * SortMergeJoin Inner (80)
+                     :           :- * Sort (74)
+                     :           :  +- Exchange (73)
+                     :           :     +- * Project (72)
+                     :           :        +- * BroadcastHashJoin Inner BuildLeft (71)
+                     :           :           :- BroadcastExchange (66)
+                     :           :           :  +- * Project (65)
+                     :           :           :     +- * BroadcastHashJoin Inner BuildLeft (64)
+                     :           :           :        :- BroadcastExchange (60)
+                     :           :           :        :  +- * Project (59)
+                     :           :           :        :     +- * BroadcastHashJoin Inner BuildLeft (58)
+                     :           :           :        :        :- BroadcastExchange (54)
+                     :           :           :        :        :  +- * Project (53)
+                     :           :           :        :        :     +- * Filter (52)
+                     :           :           :        :        :        +- * ColumnarToRow (51)
+                     :           :           :        :        :           +- Scan parquet default.store (50)
+                     :           :           :        :        +- * Filter (57)
+                     :           :           :        :           +- * ColumnarToRow (56)
+                     :           :           :        :              +- Scan parquet default.customer_address (55)
+                     :           :           :        +- * Filter (63)
+                     :           :           :           +- * ColumnarToRow (62)
+                     :           :           :              +- Scan parquet default.customer (61)
+                     :           :           +- * Project (70)
+                     :           :              +- * Filter (69)
+                     :           :                 +- * ColumnarToRow (68)
+                     :           :                    +- Scan parquet default.store_sales (67)
+                     :           +- * Sort (79)
+                     :              +- Exchange (78)
+                     :                 +- * Filter (77)
+                     :                    +- * ColumnarToRow (76)
+                     :                       +- Scan parquet default.item (75)
+                     +- * Sort (89)
+                        +- Exchange (88)
+                           +- * Project (87)
+                              +- * Filter (86)
+                                 +- * ColumnarToRow (85)
+                                    +- Scan parquet default.store_returns (84)
+
+
+(50) Scan parquet default.store
+Output [5]: [s_store_sk#52, s_store_name#53, s_market_id#54, s_state#55, s_zip#56]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_market_id:int,s_state:string,s_zip:string>
+
+(51) ColumnarToRow [codegen id : 1]
+Input [5]: [s_store_sk#52, s_store_name#53, s_market_id#54, s_state#55, s_zip#56]
+
+(52) Filter [codegen id : 1]
+Input [5]: [s_store_sk#52, s_store_name#53, s_market_id#54, s_state#55, s_zip#56]
+Condition : (((isnotnull(s_market_id#54) AND (s_market_id#54 = 8)) AND isnotnull(s_store_sk#52)) AND isnotnull(s_zip#56))
+
+(53) Project [codegen id : 1]
+Output [4]: [s_store_sk#52, s_store_name#53, s_state#55, s_zip#56]
+Input [5]: [s_store_sk#52, s_store_name#53, s_market_id#54, s_state#55, s_zip#56]
+
+(54) BroadcastExchange
+Input [4]: [s_store_sk#52, s_store_name#53, s_state#55, s_zip#56]
+Arguments: HashedRelationBroadcastMode(List(input[3, string, true]),false), [id=#57]
+
+(55) Scan parquet default.customer_address
+Output [4]: [ca_address_sk#58, ca_state#59, ca_zip#60, ca_country#61]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_country), IsNotNull(ca_zip)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string,ca_zip:string,ca_country:string>
+
+(56) ColumnarToRow
+Input [4]: [ca_address_sk#58, ca_state#59, ca_zip#60, ca_country#61]
+
+(57) Filter
+Input [4]: [ca_address_sk#58, ca_state#59, ca_zip#60, ca_country#61]
+Condition : ((isnotnull(ca_address_sk#58) AND isnotnull(ca_country#61)) AND isnotnull(ca_zip#60))
+
+(58) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [s_zip#56]
+Right keys [1]: [ca_zip#60]
+Join condition: None
+
+(59) Project [codegen id : 2]
+Output [6]: [s_store_sk#52, s_store_name#53, s_state#55, ca_address_sk#58, ca_state#59, ca_country#61]
+Input [8]: [s_store_sk#52, s_store_name#53, s_state#55, s_zip#56, ca_address_sk#58, ca_state#59, ca_zip#60, ca_country#61]
+
+(60) BroadcastExchange
+Input [6]: [s_store_sk#52, s_store_name#53, s_state#55, ca_address_sk#58, ca_state#59, ca_country#61]
+Arguments: HashedRelationBroadcastMode(List(input[3, int, true], upper(input[5, string, true])),false), [id=#62]
+
+(61) Scan parquet default.customer
+Output [5]: [c_customer_sk#63, c_current_addr_sk#64, c_first_name#65, c_last_name#66, c_birth_country#67]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_birth_country)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int,c_first_name:string,c_last_name:string,c_birth_country:string>
+
+(62) ColumnarToRow
+Input [5]: [c_customer_sk#63, c_current_addr_sk#64, c_first_name#65, c_last_name#66, c_birth_country#67]
+
+(63) Filter
+Input [5]: [c_customer_sk#63, c_current_addr_sk#64, c_first_name#65, c_last_name#66, c_birth_country#67]
+Condition : ((isnotnull(c_customer_sk#63) AND isnotnull(c_current_addr_sk#64)) AND isnotnull(c_birth_country#67))
+
+(64) BroadcastHashJoin [codegen id : 3]
+Left keys [2]: [ca_address_sk#58, upper(ca_country#61)]
+Right keys [2]: [c_current_addr_sk#64, c_birth_country#67]
+Join condition: None
+
+(65) Project [codegen id : 3]
+Output [7]: [s_store_sk#52, s_store_name#53, s_state#55, ca_state#59, c_customer_sk#63, c_first_name#65, c_last_name#66]
+Input [11]: [s_store_sk#52, s_store_name#53, s_state#55, ca_address_sk#58, ca_state#59, ca_country#61, c_customer_sk#63, c_current_addr_sk#64, c_first_name#65, c_last_name#66, c_birth_country#67]
+
+(66) BroadcastExchange
+Input [7]: [s_store_sk#52, s_store_name#53, s_state#55, ca_state#59, c_customer_sk#63, c_first_name#65, c_last_name#66]
+Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, true] as bigint), 32) | (cast(input[4, int, true] as bigint) & 4294967295))),false), [id=#68]
+
+(67) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#69, ss_customer_sk#70, ss_store_sk#71, ss_ticket_number#72, ss_net_paid#73, ss_sold_date_sk#74]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_net_paid:decimal(7,2)>
+
+(68) ColumnarToRow
+Input [6]: [ss_item_sk#69, ss_customer_sk#70, ss_store_sk#71, ss_ticket_number#72, ss_net_paid#73, ss_sold_date_sk#74]
+
+(69) Filter
+Input [6]: [ss_item_sk#69, ss_customer_sk#70, ss_store_sk#71, ss_ticket_number#72, ss_net_paid#73, ss_sold_date_sk#74]
+Condition : (((isnotnull(ss_ticket_number#72) AND isnotnull(ss_item_sk#69)) AND isnotnull(ss_store_sk#71)) AND isnotnull(ss_customer_sk#70))
+
+(70) Project
+Output [5]: [ss_item_sk#69, ss_customer_sk#70, ss_store_sk#71, ss_ticket_number#72, ss_net_paid#73]
+Input [6]: [ss_item_sk#69, ss_customer_sk#70, ss_store_sk#71, ss_ticket_number#72, ss_net_paid#73, ss_sold_date_sk#74]
+
+(71) BroadcastHashJoin [codegen id : 4]
+Left keys [2]: [s_store_sk#52, c_customer_sk#63]
+Right keys [2]: [ss_store_sk#71, ss_customer_sk#70]
+Join condition: None
+
+(72) Project [codegen id : 4]
+Output [8]: [s_store_name#53, s_state#55, ca_state#59, c_first_name#65, c_last_name#66, ss_item_sk#69, ss_ticket_number#72, ss_net_paid#73]
+Input [12]: [s_store_sk#52, s_store_name#53, s_state#55, ca_state#59, c_customer_sk#63, c_first_name#65, c_last_name#66, ss_item_sk#69, ss_customer_sk#70, ss_store_sk#71, ss_ticket_number#72, ss_net_paid#73]
+
+(73) Exchange
+Input [8]: [s_store_name#53, s_state#55, ca_state#59, c_first_name#65, c_last_name#66, ss_item_sk#69, ss_ticket_number#72, ss_net_paid#73]
+Arguments: hashpartitioning(ss_item_sk#69, 5), ENSURE_REQUIREMENTS, [id=#75]
+
+(74) Sort [codegen id : 5]
+Input [8]: [s_store_name#53, s_state#55, ca_state#59, c_first_name#65, c_last_name#66, ss_item_sk#69, ss_ticket_number#72, ss_net_paid#73]
+Arguments: [ss_item_sk#69 ASC NULLS FIRST], false, 0
+
+(75) Scan parquet default.item
+Output [6]: [i_item_sk#76, i_current_price#77, i_size#78, i_color#79, i_units#80, i_manager_id#81]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_current_price:decimal(7,2),i_size:string,i_color:string,i_units:string,i_manager_id:int>
+
+(76) ColumnarToRow [codegen id : 6]
+Input [6]: [i_item_sk#76, i_current_price#77, i_size#78, i_color#79, i_units#80, i_manager_id#81]
+
+(77) Filter [codegen id : 6]
+Input [6]: [i_item_sk#76, i_current_price#77, i_size#78, i_color#79, i_units#80, i_manager_id#81]
+Condition : isnotnull(i_item_sk#76)
+
+(78) Exchange
+Input [6]: [i_item_sk#76, i_current_price#77, i_size#78, i_color#79, i_units#80, i_manager_id#81]
+Arguments: hashpartitioning(i_item_sk#76, 5), ENSURE_REQUIREMENTS, [id=#82]
+
+(79) Sort [codegen id : 7]
+Input [6]: [i_item_sk#76, i_current_price#77, i_size#78, i_color#79, i_units#80, i_manager_id#81]
+Arguments: [i_item_sk#76 ASC NULLS FIRST], false, 0
+
+(80) SortMergeJoin [codegen id : 8]
+Left keys [1]: [ss_item_sk#69]
+Right keys [1]: [i_item_sk#76]
+Join condition: None
+
+(81) Project [codegen id : 8]
+Output [13]: [s_store_name#53, s_state#55, ca_state#59, c_first_name#65, c_last_name#66, ss_item_sk#69, ss_ticket_number#72, ss_net_paid#73, i_current_price#77, i_size#78, i_color#79, i_units#80, i_manager_id#81]
+Input [14]: [s_store_name#53, s_state#55, ca_state#59, c_first_name#65, c_last_name#66, ss_item_sk#69, ss_ticket_number#72, ss_net_paid#73, i_item_sk#76, i_current_price#77, i_size#78, i_color#79, i_units#80, i_manager_id#81]
+
+(82) Exchange
+Input [13]: [s_store_name#53, s_state#55, ca_state#59, c_first_name#65, c_last_name#66, ss_item_sk#69, ss_ticket_number#72, ss_net_paid#73, i_current_price#77, i_size#78, i_color#79, i_units#80, i_manager_id#81]
+Arguments: hashpartitioning(ss_ticket_number#72, ss_item_sk#69, 5), ENSURE_REQUIREMENTS, [id=#83]
+
+(83) Sort [codegen id : 9]
+Input [13]: [s_store_name#53, s_state#55, ca_state#59, c_first_name#65, c_last_name#66, ss_item_sk#69, ss_ticket_number#72, ss_net_paid#73, i_current_price#77, i_size#78, i_color#79, i_units#80, i_manager_id#81]
+Arguments: [ss_ticket_number#72 ASC NULLS FIRST, ss_item_sk#69 ASC NULLS FIRST], false, 0
+
+(84) Scan parquet default.store_returns
+Output [3]: [sr_item_sk#84, sr_ticket_number#85, sr_returned_date_sk#86]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int>
+
+(85) ColumnarToRow [codegen id : 10]
+Input [3]: [sr_item_sk#84, sr_ticket_number#85, sr_returned_date_sk#86]
+
+(86) Filter [codegen id : 10]
+Input [3]: [sr_item_sk#84, sr_ticket_number#85, sr_returned_date_sk#86]
+Condition : (isnotnull(sr_ticket_number#85) AND isnotnull(sr_item_sk#84))
+
+(87) Project [codegen id : 10]
+Output [2]: [sr_item_sk#84, sr_ticket_number#85]
+Input [3]: [sr_item_sk#84, sr_ticket_number#85, sr_returned_date_sk#86]
+
+(88) Exchange
+Input [2]: [sr_item_sk#84, sr_ticket_number#85]
+Arguments: hashpartitioning(sr_ticket_number#85, sr_item_sk#84, 5), ENSURE_REQUIREMENTS, [id=#87]
+
+(89) Sort [codegen id : 11]
+Input [2]: [sr_item_sk#84, sr_ticket_number#85]
+Arguments: [sr_ticket_number#85 ASC NULLS FIRST, sr_item_sk#84 ASC NULLS FIRST], false, 0
+
+(90) SortMergeJoin [codegen id : 12]
+Left keys [2]: [ss_ticket_number#72, ss_item_sk#69]
+Right keys [2]: [sr_ticket_number#85, sr_item_sk#84]
+Join condition: None
+
+(91) Project [codegen id : 12]
+Output [11]: [ss_net_paid#73, s_store_name#53, s_state#55, i_current_price#77, i_size#78, i_color#79, i_units#80, i_manager_id#81, c_first_name#65, c_last_name#66, ca_state#59]
+Input [15]: [s_store_name#53, s_state#55, ca_state#59, c_first_name#65, c_last_name#66, ss_item_sk#69, ss_ticket_number#72, ss_net_paid#73, i_current_price#77, i_size#78, i_color#79, i_units#80, i_manager_id#81, sr_item_sk#84, sr_ticket_number#85]
+
+(92) HashAggregate [codegen id : 12]
+Input [11]: [ss_net_paid#73, s_store_name#53, s_state#55, i_current_price#77, i_size#78, i_color#79, i_units#80, i_manager_id#81, c_first_name#65, c_last_name#66, ca_state#59]
+Keys [10]: [c_last_name#66, c_first_name#65, s_store_name#53, ca_state#59, s_state#55, i_color#79, i_current_price#77, i_manager_id#81, i_units#80, i_size#78]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#73))]
+Aggregate Attributes [1]: [sum#88]
+Results [11]: [c_last_name#66, c_first_name#65, s_store_name#53, ca_state#59, s_state#55, i_color#79, i_current_price#77, i_manager_id#81, i_units#80, i_size#78, sum#89]
+
+(93) Exchange
+Input [11]: [c_last_name#66, c_first_name#65, s_store_name#53, ca_state#59, s_state#55, i_color#79, i_current_price#77, i_manager_id#81, i_units#80, i_size#78, sum#89]
+Arguments: hashpartitioning(c_last_name#66, c_first_name#65, s_store_name#53, ca_state#59, s_state#55, i_color#79, i_current_price#77, i_manager_id#81, i_units#80, i_size#78, 5), ENSURE_REQUIREMENTS, [id=#90]
+
+(94) HashAggregate [codegen id : 13]
+Input [11]: [c_last_name#66, c_first_name#65, s_store_name#53, ca_state#59, s_state#55, i_color#79, i_current_price#77, i_manager_id#81, i_units#80, i_size#78, sum#89]
+Keys [10]: [c_last_name#66, c_first_name#65, s_store_name#53, ca_state#59, s_state#55, i_color#79, i_current_price#77, i_manager_id#81, i_units#80, i_size#78]
+Functions [1]: [sum(UnscaledValue(ss_net_paid#73))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#73))#91]
+Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#73))#91,17,2) AS netpaid#40]
+
+(95) HashAggregate [codegen id : 13]
+Input [1]: [netpaid#40]
+Keys: []
+Functions [1]: [partial_avg(netpaid#40)]
+Aggregate Attributes [2]: [sum#92, count#93]
+Results [2]: [sum#94, count#95]
+
+(96) Exchange
+Input [2]: [sum#94, count#95]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#96]
+
+(97) HashAggregate [codegen id : 14]
+Input [2]: [sum#94, count#95]
+Keys: []
+Functions [1]: [avg(netpaid#40)]
+Aggregate Attributes [1]: [avg(netpaid#40)#97]
+Results [1]: [CheckOverflow((0.050000 * promote_precision(avg(netpaid#40)#97)), DecimalType(24,8), true) AS (0.05 * avg(netpaid))#98]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.sf100/simplified.txt
new file mode 100644
index 0000000000000..3feab0ac8cab5
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.sf100/simplified.txt
@@ -0,0 +1,160 @@
+WholeStageCodegen (12)
+  Sort [c_last_name,c_first_name,s_store_name]
+    InputAdapter
+      Exchange [c_last_name,c_first_name,s_store_name] #1
+        WholeStageCodegen (11)
+          Project [c_last_name,c_first_name,s_store_name,paid]
+            Filter [sum(netpaid)]
+              Subquery #1
+                WholeStageCodegen (14)
+                  HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count]
+                    InputAdapter
+                      Exchange #10
+                        WholeStageCodegen (13)
+                          HashAggregate [netpaid] [sum,count,sum,count]
+                            HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum]
+                              InputAdapter
+                                Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #11
+                                  WholeStageCodegen (12)
+                                    HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum]
+                                      Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state]
+                                        SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                          InputAdapter
+                                            WholeStageCodegen (9)
+                                              Sort [ss_ticket_number,ss_item_sk]
+                                                InputAdapter
+                                                  Exchange [ss_ticket_number,ss_item_sk] #12
+                                                    WholeStageCodegen (8)
+                                                      Project [s_store_name,s_state,ca_state,c_first_name,c_last_name,ss_item_sk,ss_ticket_number,ss_net_paid,i_current_price,i_size,i_color,i_units,i_manager_id]
+                                                        SortMergeJoin [ss_item_sk,i_item_sk]
+                                                          InputAdapter
+                                                            WholeStageCodegen (5)
+                                                              Sort [ss_item_sk]
+                                                                InputAdapter
+                                                                  Exchange [ss_item_sk] #13
+                                                                    WholeStageCodegen (4)
+                                                                      Project [s_store_name,s_state,ca_state,c_first_name,c_last_name,ss_item_sk,ss_ticket_number,ss_net_paid]
+                                                                        BroadcastHashJoin [s_store_sk,c_customer_sk,ss_store_sk,ss_customer_sk]
+                                                                          InputAdapter
+                                                                            BroadcastExchange #14
+                                                                              WholeStageCodegen (3)
+                                                                                Project [s_store_sk,s_store_name,s_state,ca_state,c_customer_sk,c_first_name,c_last_name]
+                                                                                  BroadcastHashJoin [ca_address_sk,ca_country,c_current_addr_sk,c_birth_country]
+                                                                                    InputAdapter
+                                                                                      BroadcastExchange #15
+                                                                                        WholeStageCodegen (2)
+                                                                                          Project [s_store_sk,s_store_name,s_state,ca_address_sk,ca_state,ca_country]
+                                                                                            BroadcastHashJoin [s_zip,ca_zip]
+                                                                                              InputAdapter
+                                                                                                BroadcastExchange #16
+                                                                                                  WholeStageCodegen (1)
+                                                                                                    Project [s_store_sk,s_store_name,s_state,s_zip]
+                                                                                                      Filter [s_market_id,s_store_sk,s_zip]
+                                                                                                        ColumnarToRow
+                                                                                                          InputAdapter
+                                                                                                            Scan parquet default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip]
+                                                                                              Filter [ca_address_sk,ca_country,ca_zip]
+                                                                                                ColumnarToRow
+                                                                                                  InputAdapter
+                                                                                                    Scan parquet default.customer_address [ca_address_sk,ca_state,ca_zip,ca_country]
+                                                                                    Filter [c_customer_sk,c_current_addr_sk,c_birth_country]
+                                                                                      ColumnarToRow
+                                                                                        InputAdapter
+                                                                                          Scan parquet default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country]
+                                                                          Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid]
+                                                                            Filter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk]
+                                                                              ColumnarToRow
+                                                                                InputAdapter
+                                                                                  Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk]
+                                                          InputAdapter
+                                                            WholeStageCodegen (7)
+                                                              Sort [i_item_sk]
+                                                                InputAdapter
+                                                                  Exchange [i_item_sk] #17
+                                                                    WholeStageCodegen (6)
+                                                                      Filter [i_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id]
+                                          InputAdapter
+                                            WholeStageCodegen (11)
+                                              Sort [sr_ticket_number,sr_item_sk]
+                                                InputAdapter
+                                                  Exchange [sr_ticket_number,sr_item_sk] #18
+                                                    WholeStageCodegen (10)
+                                                      Project [sr_item_sk,sr_ticket_number]
+                                                        Filter [sr_ticket_number,sr_item_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk]
+              HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum(netpaid),sum,isEmpty]
+                InputAdapter
+                  Exchange [c_last_name,c_first_name,s_store_name] #2
+                    WholeStageCodegen (10)
+                      HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty]
+                        HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum]
+                          InputAdapter
+                            Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #3
+                              WholeStageCodegen (9)
+                                HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum]
+                                  Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state]
+                                    SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                      InputAdapter
+                                        WholeStageCodegen (6)
+                                          Sort [ss_ticket_number,ss_item_sk]
+                                            InputAdapter
+                                              Exchange [ss_ticket_number,ss_item_sk] #4
+                                                WholeStageCodegen (5)
+                                                  Project [s_store_name,s_state,ca_state,c_first_name,c_last_name,ss_item_sk,ss_ticket_number,ss_net_paid,i_current_price,i_size,i_color,i_units,i_manager_id]
+                                                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                      Project [s_store_name,s_state,ca_state,c_first_name,c_last_name,ss_item_sk,ss_ticket_number,ss_net_paid]
+                                                        BroadcastHashJoin [s_store_sk,c_customer_sk,ss_store_sk,ss_customer_sk]
+                                                          InputAdapter
+                                                            BroadcastExchange #5
+                                                              WholeStageCodegen (3)
+                                                                Project [s_store_sk,s_store_name,s_state,ca_state,c_customer_sk,c_first_name,c_last_name]
+                                                                  BroadcastHashJoin [ca_address_sk,ca_country,c_current_addr_sk,c_birth_country]
+                                                                    InputAdapter
+                                                                      BroadcastExchange #6
+                                                                        WholeStageCodegen (2)
+                                                                          Project [s_store_sk,s_store_name,s_state,ca_address_sk,ca_state,ca_country]
+                                                                            BroadcastHashJoin [s_zip,ca_zip]
+                                                                              InputAdapter
+                                                                                BroadcastExchange #7
+                                                                                  WholeStageCodegen (1)
+                                                                                    Project [s_store_sk,s_store_name,s_state,s_zip]
+                                                                                      Filter [s_market_id,s_store_sk,s_zip]
+                                                                                        ColumnarToRow
+                                                                                          InputAdapter
+                                                                                            Scan parquet default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip]
+                                                                              Filter [ca_address_sk,ca_country,ca_zip]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.customer_address [ca_address_sk,ca_state,ca_zip,ca_country]
+                                                                    Filter [c_customer_sk,c_current_addr_sk,c_birth_country]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country]
+                                                          Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid]
+                                                            Filter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk]
+                                                      InputAdapter
+                                                        BroadcastExchange #8
+                                                          WholeStageCodegen (4)
+                                                            Filter [i_color,i_item_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id]
+                                      InputAdapter
+                                        WholeStageCodegen (8)
+                                          Sort [sr_ticket_number,sr_item_sk]
+                                            InputAdapter
+                                              Exchange [sr_ticket_number,sr_item_sk] #9
+                                                WholeStageCodegen (7)
+                                                  Project [sr_item_sk,sr_ticket_number]
+                                                    Filter [sr_ticket_number,sr_item_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/explain.txt
new file mode 100644
index 0000000000000..2686a277825d4
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/explain.txt
@@ -0,0 +1,537 @@
+== Physical Plan ==
+* Sort (49)
++- Exchange (48)
+   +- * Project (47)
+      +- * Filter (46)
+         +- * HashAggregate (45)
+            +- Exchange (44)
+               +- * HashAggregate (43)
+                  +- * HashAggregate (42)
+                     +- Exchange (41)
+                        +- * HashAggregate (40)
+                           +- * Project (39)
+                              +- * BroadcastHashJoin Inner BuildRight (38)
+                                 :- * Project (33)
+                                 :  +- * BroadcastHashJoin Inner BuildRight (32)
+                                 :     :- * Project (27)
+                                 :     :  +- * BroadcastHashJoin Inner BuildRight (26)
+                                 :     :     :- * Project (21)
+                                 :     :     :  +- * BroadcastHashJoin Inner BuildRight (20)
+                                 :     :     :     :- * Project (14)
+                                 :     :     :     :  +- * SortMergeJoin Inner (13)
+                                 :     :     :     :     :- * Sort (6)
+                                 :     :     :     :     :  +- Exchange (5)
+                                 :     :     :     :     :     +- * Project (4)
+                                 :     :     :     :     :        +- * Filter (3)
+                                 :     :     :     :     :           +- * ColumnarToRow (2)
+                                 :     :     :     :     :              +- Scan parquet default.store_sales (1)
+                                 :     :     :     :     +- * Sort (12)
+                                 :     :     :     :        +- Exchange (11)
+                                 :     :     :     :           +- * Project (10)
+                                 :     :     :     :              +- * Filter (9)
+                                 :     :     :     :                 +- * ColumnarToRow (8)
+                                 :     :     :     :                    +- Scan parquet default.store_returns (7)
+                                 :     :     :     +- BroadcastExchange (19)
+                                 :     :     :        +- * Project (18)
+                                 :     :     :           +- * Filter (17)
+                                 :     :     :              +- * ColumnarToRow (16)
+                                 :     :     :                 +- Scan parquet default.store (15)
+                                 :     :     +- BroadcastExchange (25)
+                                 :     :        +- * Filter (24)
+                                 :     :           +- * ColumnarToRow (23)
+                                 :     :              +- Scan parquet default.item (22)
+                                 :     +- BroadcastExchange (31)
+                                 :        +- * Filter (30)
+                                 :           +- * ColumnarToRow (29)
+                                 :              +- Scan parquet default.customer (28)
+                                 +- BroadcastExchange (37)
+                                    +- * Filter (36)
+                                       +- * ColumnarToRow (35)
+                                          +- Scan parquet default.customer_address (34)
+
+
+(1) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_net_paid:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6]
+
+(3) Filter [codegen id : 1]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6]
+Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2))
+
+(4) Project [codegen id : 1]
+Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6]
+
+(5) Exchange
+Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5]
+Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#7]
+
+(6) Sort [codegen id : 2]
+Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5]
+Arguments: [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(7) Scan parquet default.store_returns
+Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int>
+
+(8) ColumnarToRow [codegen id : 3]
+Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10]
+
+(9) Filter [codegen id : 3]
+Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10]
+Condition : (isnotnull(sr_ticket_number#9) AND isnotnull(sr_item_sk#8))
+
+(10) Project [codegen id : 3]
+Output [2]: [sr_item_sk#8, sr_ticket_number#9]
+Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10]
+
+(11) Exchange
+Input [2]: [sr_item_sk#8, sr_ticket_number#9]
+Arguments: hashpartitioning(sr_ticket_number#9, sr_item_sk#8, 5), ENSURE_REQUIREMENTS, [id=#11]
+
+(12) Sort [codegen id : 4]
+Input [2]: [sr_item_sk#8, sr_ticket_number#9]
+Arguments: [sr_ticket_number#9 ASC NULLS FIRST, sr_item_sk#8 ASC NULLS FIRST], false, 0
+
+(13) SortMergeJoin [codegen id : 9]
+Left keys [2]: [ss_ticket_number#4, ss_item_sk#1]
+Right keys [2]: [sr_ticket_number#9, sr_item_sk#8]
+Join condition: None
+
+(14) Project [codegen id : 9]
+Output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5]
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#8, sr_ticket_number#9]
+
+(15) Scan parquet default.store
+Output [5]: [s_store_sk#12, s_store_name#13, s_market_id#14, s_state#15, s_zip#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_market_id:int,s_state:string,s_zip:string>
+
+(16) ColumnarToRow [codegen id : 5]
+Input [5]: [s_store_sk#12, s_store_name#13, s_market_id#14, s_state#15, s_zip#16]
+
+(17) Filter [codegen id : 5]
+Input [5]: [s_store_sk#12, s_store_name#13, s_market_id#14, s_state#15, s_zip#16]
+Condition : (((isnotnull(s_market_id#14) AND (s_market_id#14 = 8)) AND isnotnull(s_store_sk#12)) AND isnotnull(s_zip#16))
+
+(18) Project [codegen id : 5]
+Output [4]: [s_store_sk#12, s_store_name#13, s_state#15, s_zip#16]
+Input [5]: [s_store_sk#12, s_store_name#13, s_market_id#14, s_state#15, s_zip#16]
+
+(19) BroadcastExchange
+Input [4]: [s_store_sk#12, s_store_name#13, s_state#15, s_zip#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(20) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#12]
+Join condition: None
+
+(21) Project [codegen id : 9]
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#13, s_state#15, s_zip#16]
+Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#12, s_store_name#13, s_state#15, s_zip#16]
+
+(22) Scan parquet default.item
+Output [6]: [i_item_sk#18, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_color), EqualTo(i_color,pale                ), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_current_price:decimal(7,2),i_size:string,i_color:string,i_units:string,i_manager_id:int>
+
+(23) ColumnarToRow [codegen id : 6]
+Input [6]: [i_item_sk#18, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23]
+
+(24) Filter [codegen id : 6]
+Input [6]: [i_item_sk#18, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23]
+Condition : ((isnotnull(i_color#21) AND (i_color#21 = pale                )) AND isnotnull(i_item_sk#18))
+
+(25) BroadcastExchange
+Input [6]: [i_item_sk#18, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#24]
+
+(26) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#18]
+Join condition: None
+
+(27) Project [codegen id : 9]
+Output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#13, s_state#15, s_zip#16, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23]
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#13, s_state#15, s_zip#16, i_item_sk#18, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23]
+
+(28) Scan parquet default.customer
+Output [5]: [c_customer_sk#25, c_current_addr_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_birth_country)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int,c_first_name:string,c_last_name:string,c_birth_country:string>
+
+(29) ColumnarToRow [codegen id : 7]
+Input [5]: [c_customer_sk#25, c_current_addr_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29]
+
+(30) Filter [codegen id : 7]
+Input [5]: [c_customer_sk#25, c_current_addr_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29]
+Condition : ((isnotnull(c_customer_sk#25) AND isnotnull(c_current_addr_sk#26)) AND isnotnull(c_birth_country#29))
+
+(31) BroadcastExchange
+Input [5]: [c_customer_sk#25, c_current_addr_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#30]
+
+(32) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_customer_sk#2]
+Right keys [1]: [c_customer_sk#25]
+Join condition: None
+
+(33) Project [codegen id : 9]
+Output [13]: [ss_net_paid#5, s_store_name#13, s_state#15, s_zip#16, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23, c_current_addr_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29]
+Input [15]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#13, s_state#15, s_zip#16, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23, c_customer_sk#25, c_current_addr_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29]
+
+(34) Scan parquet default.customer_address
+Output [4]: [ca_address_sk#31, ca_state#32, ca_zip#33, ca_country#34]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_country), IsNotNull(ca_zip)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string,ca_zip:string,ca_country:string>
+
+(35) ColumnarToRow [codegen id : 8]
+Input [4]: [ca_address_sk#31, ca_state#32, ca_zip#33, ca_country#34]
+
+(36) Filter [codegen id : 8]
+Input [4]: [ca_address_sk#31, ca_state#32, ca_zip#33, ca_country#34]
+Condition : ((isnotnull(ca_address_sk#31) AND isnotnull(ca_country#34)) AND isnotnull(ca_zip#33))
+
+(37) BroadcastExchange
+Input [4]: [ca_address_sk#31, ca_state#32, ca_zip#33, ca_country#34]
+Arguments: HashedRelationBroadcastMode(List(input[0, int, false], upper(input[3, string, false]), input[2, string, false]),false), [id=#35]
+
+(38) BroadcastHashJoin [codegen id : 9]
+Left keys [3]: [c_current_addr_sk#26, c_birth_country#29, s_zip#16]
+Right keys [3]: [ca_address_sk#31, upper(ca_country#34), ca_zip#33]
+Join condition: None
+
+(39) Project [codegen id : 9]
+Output [11]: [ss_net_paid#5, s_store_name#13, s_state#15, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23, c_first_name#27, c_last_name#28, ca_state#32]
+Input [17]: [ss_net_paid#5, s_store_name#13, s_state#15, s_zip#16, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23, c_current_addr_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29, ca_address_sk#31, ca_state#32, ca_zip#33, ca_country#34]
+
+(40) HashAggregate [codegen id : 9]
+Input [11]: [ss_net_paid#5, s_store_name#13, s_state#15, i_current_price#19, i_size#20, i_color#21, i_units#22, i_manager_id#23, c_first_name#27, c_last_name#28, ca_state#32]
+Keys [10]: [c_last_name#28, c_first_name#27, s_store_name#13, ca_state#32, s_state#15, i_color#21, i_current_price#19, i_manager_id#23, i_units#22, i_size#20]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))]
+Aggregate Attributes [1]: [sum#36]
+Results [11]: [c_last_name#28, c_first_name#27, s_store_name#13, ca_state#32, s_state#15, i_color#21, i_current_price#19, i_manager_id#23, i_units#22, i_size#20, sum#37]
+
+(41) Exchange
+Input [11]: [c_last_name#28, c_first_name#27, s_store_name#13, ca_state#32, s_state#15, i_color#21, i_current_price#19, i_manager_id#23, i_units#22, i_size#20, sum#37]
+Arguments: hashpartitioning(c_last_name#28, c_first_name#27, s_store_name#13, ca_state#32, s_state#15, i_color#21, i_current_price#19, i_manager_id#23, i_units#22, i_size#20, 5), ENSURE_REQUIREMENTS, [id=#38]
+
+(42) HashAggregate [codegen id : 10]
+Input [11]: [c_last_name#28, c_first_name#27, s_store_name#13, ca_state#32, s_state#15, i_color#21, i_current_price#19, i_manager_id#23, i_units#22, i_size#20, sum#37]
+Keys [10]: [c_last_name#28, c_first_name#27, s_store_name#13, ca_state#32, s_state#15, i_color#21, i_current_price#19, i_manager_id#23, i_units#22, i_size#20]
+Functions [1]: [sum(UnscaledValue(ss_net_paid#5))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#39]
+Results [4]: [c_last_name#28, c_first_name#27, s_store_name#13, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#39,17,2) AS netpaid#40]
+
+(43) HashAggregate [codegen id : 10]
+Input [4]: [c_last_name#28, c_first_name#27, s_store_name#13, netpaid#40]
+Keys [3]: [c_last_name#28, c_first_name#27, s_store_name#13]
+Functions [1]: [partial_sum(netpaid#40)]
+Aggregate Attributes [2]: [sum#41, isEmpty#42]
+Results [5]: [c_last_name#28, c_first_name#27, s_store_name#13, sum#43, isEmpty#44]
+
+(44) Exchange
+Input [5]: [c_last_name#28, c_first_name#27, s_store_name#13, sum#43, isEmpty#44]
+Arguments: hashpartitioning(c_last_name#28, c_first_name#27, s_store_name#13, 5), ENSURE_REQUIREMENTS, [id=#45]
+
+(45) HashAggregate [codegen id : 11]
+Input [5]: [c_last_name#28, c_first_name#27, s_store_name#13, sum#43, isEmpty#44]
+Keys [3]: [c_last_name#28, c_first_name#27, s_store_name#13]
+Functions [1]: [sum(netpaid#40)]
+Aggregate Attributes [1]: [sum(netpaid#40)#46]
+Results [5]: [c_last_name#28, c_first_name#27, s_store_name#13, sum(netpaid#40)#46 AS paid#47, sum(netpaid#40)#46 AS sum(netpaid#40)#48]
+
+(46) Filter [codegen id : 11]
+Input [5]: [c_last_name#28, c_first_name#27, s_store_name#13, paid#47, sum(netpaid#40)#48]
+Condition : (isnotnull(sum(netpaid#40)#48) AND (cast(sum(netpaid#40)#48 as decimal(33,8)) > cast(Subquery scalar-subquery#49, [id=#50] as decimal(33,8))))
+
+(47) Project [codegen id : 11]
+Output [4]: [c_last_name#28, c_first_name#27, s_store_name#13, paid#47]
+Input [5]: [c_last_name#28, c_first_name#27, s_store_name#13, paid#47, sum(netpaid#40)#48]
+
+(48) Exchange
+Input [4]: [c_last_name#28, c_first_name#27, s_store_name#13, paid#47]
+Arguments: rangepartitioning(c_last_name#28 ASC NULLS FIRST, c_first_name#27 ASC NULLS FIRST, s_store_name#13 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [id=#51]
+
+(49) Sort [codegen id : 12]
+Input [4]: [c_last_name#28, c_first_name#27, s_store_name#13, paid#47]
+Arguments: [c_last_name#28 ASC NULLS FIRST, c_first_name#27 ASC NULLS FIRST, s_store_name#13 ASC NULLS FIRST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#49, [id=#50]
+* HashAggregate (94)
++- Exchange (93)
+   +- * HashAggregate (92)
+      +- * HashAggregate (91)
+         +- Exchange (90)
+            +- * HashAggregate (89)
+               +- * Project (88)
+                  +- * BroadcastHashJoin Inner BuildRight (87)
+                     :- * Project (82)
+                     :  +- * BroadcastHashJoin Inner BuildRight (81)
+                     :     :- * Project (76)
+                     :     :  +- * BroadcastHashJoin Inner BuildRight (75)
+                     :     :     :- * Project (70)
+                     :     :     :  +- * BroadcastHashJoin Inner BuildRight (69)
+                     :     :     :     :- * Project (63)
+                     :     :     :     :  +- * SortMergeJoin Inner (62)
+                     :     :     :     :     :- * Sort (55)
+                     :     :     :     :     :  +- Exchange (54)
+                     :     :     :     :     :     +- * Project (53)
+                     :     :     :     :     :        +- * Filter (52)
+                     :     :     :     :     :           +- * ColumnarToRow (51)
+                     :     :     :     :     :              +- Scan parquet default.store_sales (50)
+                     :     :     :     :     +- * Sort (61)
+                     :     :     :     :        +- Exchange (60)
+                     :     :     :     :           +- * Project (59)
+                     :     :     :     :              +- * Filter (58)
+                     :     :     :     :                 +- * ColumnarToRow (57)
+                     :     :     :     :                    +- Scan parquet default.store_returns (56)
+                     :     :     :     +- BroadcastExchange (68)
+                     :     :     :        +- * Project (67)
+                     :     :     :           +- * Filter (66)
+                     :     :     :              +- * ColumnarToRow (65)
+                     :     :     :                 +- Scan parquet default.store (64)
+                     :     :     +- BroadcastExchange (74)
+                     :     :        +- * Filter (73)
+                     :     :           +- * ColumnarToRow (72)
+                     :     :              +- Scan parquet default.item (71)
+                     :     +- BroadcastExchange (80)
+                     :        +- * Filter (79)
+                     :           +- * ColumnarToRow (78)
+                     :              +- Scan parquet default.customer (77)
+                     +- BroadcastExchange (86)
+                        +- * Filter (85)
+                           +- * ColumnarToRow (84)
+                              +- Scan parquet default.customer_address (83)
+
+
+(50) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#52, ss_customer_sk#53, ss_store_sk#54, ss_ticket_number#55, ss_net_paid#56, ss_sold_date_sk#57]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_sales]
+PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_store_sk:int,ss_ticket_number:int,ss_net_paid:decimal(7,2)>
+
+(51) ColumnarToRow [codegen id : 1]
+Input [6]: [ss_item_sk#52, ss_customer_sk#53, ss_store_sk#54, ss_ticket_number#55, ss_net_paid#56, ss_sold_date_sk#57]
+
+(52) Filter [codegen id : 1]
+Input [6]: [ss_item_sk#52, ss_customer_sk#53, ss_store_sk#54, ss_ticket_number#55, ss_net_paid#56, ss_sold_date_sk#57]
+Condition : (((isnotnull(ss_ticket_number#55) AND isnotnull(ss_item_sk#52)) AND isnotnull(ss_store_sk#54)) AND isnotnull(ss_customer_sk#53))
+
+(53) Project [codegen id : 1]
+Output [5]: [ss_item_sk#52, ss_customer_sk#53, ss_store_sk#54, ss_ticket_number#55, ss_net_paid#56]
+Input [6]: [ss_item_sk#52, ss_customer_sk#53, ss_store_sk#54, ss_ticket_number#55, ss_net_paid#56, ss_sold_date_sk#57]
+
+(54) Exchange
+Input [5]: [ss_item_sk#52, ss_customer_sk#53, ss_store_sk#54, ss_ticket_number#55, ss_net_paid#56]
+Arguments: hashpartitioning(ss_ticket_number#55, ss_item_sk#52, 5), ENSURE_REQUIREMENTS, [id=#58]
+
+(55) Sort [codegen id : 2]
+Input [5]: [ss_item_sk#52, ss_customer_sk#53, ss_store_sk#54, ss_ticket_number#55, ss_net_paid#56]
+Arguments: [ss_ticket_number#55 ASC NULLS FIRST, ss_item_sk#52 ASC NULLS FIRST], false, 0
+
+(56) Scan parquet default.store_returns
+Output [3]: [sr_item_sk#59, sr_ticket_number#60, sr_returned_date_sk#61]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int>
+
+(57) ColumnarToRow [codegen id : 3]
+Input [3]: [sr_item_sk#59, sr_ticket_number#60, sr_returned_date_sk#61]
+
+(58) Filter [codegen id : 3]
+Input [3]: [sr_item_sk#59, sr_ticket_number#60, sr_returned_date_sk#61]
+Condition : (isnotnull(sr_ticket_number#60) AND isnotnull(sr_item_sk#59))
+
+(59) Project [codegen id : 3]
+Output [2]: [sr_item_sk#59, sr_ticket_number#60]
+Input [3]: [sr_item_sk#59, sr_ticket_number#60, sr_returned_date_sk#61]
+
+(60) Exchange
+Input [2]: [sr_item_sk#59, sr_ticket_number#60]
+Arguments: hashpartitioning(sr_ticket_number#60, sr_item_sk#59, 5), ENSURE_REQUIREMENTS, [id=#62]
+
+(61) Sort [codegen id : 4]
+Input [2]: [sr_item_sk#59, sr_ticket_number#60]
+Arguments: [sr_ticket_number#60 ASC NULLS FIRST, sr_item_sk#59 ASC NULLS FIRST], false, 0
+
+(62) SortMergeJoin [codegen id : 9]
+Left keys [2]: [ss_ticket_number#55, ss_item_sk#52]
+Right keys [2]: [sr_ticket_number#60, sr_item_sk#59]
+Join condition: None
+
+(63) Project [codegen id : 9]
+Output [4]: [ss_item_sk#52, ss_customer_sk#53, ss_store_sk#54, ss_net_paid#56]
+Input [7]: [ss_item_sk#52, ss_customer_sk#53, ss_store_sk#54, ss_ticket_number#55, ss_net_paid#56, sr_item_sk#59, sr_ticket_number#60]
+
+(64) Scan parquet default.store
+Output [5]: [s_store_sk#63, s_store_name#64, s_market_id#65, s_state#66, s_zip#67]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_market_id:int,s_state:string,s_zip:string>
+
+(65) ColumnarToRow [codegen id : 5]
+Input [5]: [s_store_sk#63, s_store_name#64, s_market_id#65, s_state#66, s_zip#67]
+
+(66) Filter [codegen id : 5]
+Input [5]: [s_store_sk#63, s_store_name#64, s_market_id#65, s_state#66, s_zip#67]
+Condition : (((isnotnull(s_market_id#65) AND (s_market_id#65 = 8)) AND isnotnull(s_store_sk#63)) AND isnotnull(s_zip#67))
+
+(67) Project [codegen id : 5]
+Output [4]: [s_store_sk#63, s_store_name#64, s_state#66, s_zip#67]
+Input [5]: [s_store_sk#63, s_store_name#64, s_market_id#65, s_state#66, s_zip#67]
+
+(68) BroadcastExchange
+Input [4]: [s_store_sk#63, s_store_name#64, s_state#66, s_zip#67]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#68]
+
+(69) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_store_sk#54]
+Right keys [1]: [s_store_sk#63]
+Join condition: None
+
+(70) Project [codegen id : 9]
+Output [6]: [ss_item_sk#52, ss_customer_sk#53, ss_net_paid#56, s_store_name#64, s_state#66, s_zip#67]
+Input [8]: [ss_item_sk#52, ss_customer_sk#53, ss_store_sk#54, ss_net_paid#56, s_store_sk#63, s_store_name#64, s_state#66, s_zip#67]
+
+(71) Scan parquet default.item
+Output [6]: [i_item_sk#69, i_current_price#70, i_size#71, i_color#72, i_units#73, i_manager_id#74]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_current_price:decimal(7,2),i_size:string,i_color:string,i_units:string,i_manager_id:int>
+
+(72) ColumnarToRow [codegen id : 6]
+Input [6]: [i_item_sk#69, i_current_price#70, i_size#71, i_color#72, i_units#73, i_manager_id#74]
+
+(73) Filter [codegen id : 6]
+Input [6]: [i_item_sk#69, i_current_price#70, i_size#71, i_color#72, i_units#73, i_manager_id#74]
+Condition : isnotnull(i_item_sk#69)
+
+(74) BroadcastExchange
+Input [6]: [i_item_sk#69, i_current_price#70, i_size#71, i_color#72, i_units#73, i_manager_id#74]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#75]
+
+(75) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_item_sk#52]
+Right keys [1]: [i_item_sk#69]
+Join condition: None
+
+(76) Project [codegen id : 9]
+Output [10]: [ss_customer_sk#53, ss_net_paid#56, s_store_name#64, s_state#66, s_zip#67, i_current_price#70, i_size#71, i_color#72, i_units#73, i_manager_id#74]
+Input [12]: [ss_item_sk#52, ss_customer_sk#53, ss_net_paid#56, s_store_name#64, s_state#66, s_zip#67, i_item_sk#69, i_current_price#70, i_size#71, i_color#72, i_units#73, i_manager_id#74]
+
+(77) Scan parquet default.customer
+Output [5]: [c_customer_sk#76, c_current_addr_sk#77, c_first_name#78, c_last_name#79, c_birth_country#80]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_birth_country)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int,c_first_name:string,c_last_name:string,c_birth_country:string>
+
+(78) ColumnarToRow [codegen id : 7]
+Input [5]: [c_customer_sk#76, c_current_addr_sk#77, c_first_name#78, c_last_name#79, c_birth_country#80]
+
+(79) Filter [codegen id : 7]
+Input [5]: [c_customer_sk#76, c_current_addr_sk#77, c_first_name#78, c_last_name#79, c_birth_country#80]
+Condition : ((isnotnull(c_customer_sk#76) AND isnotnull(c_current_addr_sk#77)) AND isnotnull(c_birth_country#80))
+
+(80) BroadcastExchange
+Input [5]: [c_customer_sk#76, c_current_addr_sk#77, c_first_name#78, c_last_name#79, c_birth_country#80]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#81]
+
+(81) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_customer_sk#53]
+Right keys [1]: [c_customer_sk#76]
+Join condition: None
+
+(82) Project [codegen id : 9]
+Output [13]: [ss_net_paid#56, s_store_name#64, s_state#66, s_zip#67, i_current_price#70, i_size#71, i_color#72, i_units#73, i_manager_id#74, c_current_addr_sk#77, c_first_name#78, c_last_name#79, c_birth_country#80]
+Input [15]: [ss_customer_sk#53, ss_net_paid#56, s_store_name#64, s_state#66, s_zip#67, i_current_price#70, i_size#71, i_color#72, i_units#73, i_manager_id#74, c_customer_sk#76, c_current_addr_sk#77, c_first_name#78, c_last_name#79, c_birth_country#80]
+
+(83) Scan parquet default.customer_address
+Output [4]: [ca_address_sk#82, ca_state#83, ca_zip#84, ca_country#85]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_country), IsNotNull(ca_zip)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string,ca_zip:string,ca_country:string>
+
+(84) ColumnarToRow [codegen id : 8]
+Input [4]: [ca_address_sk#82, ca_state#83, ca_zip#84, ca_country#85]
+
+(85) Filter [codegen id : 8]
+Input [4]: [ca_address_sk#82, ca_state#83, ca_zip#84, ca_country#85]
+Condition : ((isnotnull(ca_address_sk#82) AND isnotnull(ca_country#85)) AND isnotnull(ca_zip#84))
+
+(86) BroadcastExchange
+Input [4]: [ca_address_sk#82, ca_state#83, ca_zip#84, ca_country#85]
+Arguments: HashedRelationBroadcastMode(List(input[0, int, false], upper(input[3, string, false]), input[2, string, false]),false), [id=#86]
+
+(87) BroadcastHashJoin [codegen id : 9]
+Left keys [3]: [c_current_addr_sk#77, c_birth_country#80, s_zip#67]
+Right keys [3]: [ca_address_sk#82, upper(ca_country#85), ca_zip#84]
+Join condition: None
+
+(88) Project [codegen id : 9]
+Output [11]: [ss_net_paid#56, s_store_name#64, s_state#66, i_current_price#70, i_size#71, i_color#72, i_units#73, i_manager_id#74, c_first_name#78, c_last_name#79, ca_state#83]
+Input [17]: [ss_net_paid#56, s_store_name#64, s_state#66, s_zip#67, i_current_price#70, i_size#71, i_color#72, i_units#73, i_manager_id#74, c_current_addr_sk#77, c_first_name#78, c_last_name#79, c_birth_country#80, ca_address_sk#82, ca_state#83, ca_zip#84, ca_country#85]
+
+(89) HashAggregate [codegen id : 9]
+Input [11]: [ss_net_paid#56, s_store_name#64, s_state#66, i_current_price#70, i_size#71, i_color#72, i_units#73, i_manager_id#74, c_first_name#78, c_last_name#79, ca_state#83]
+Keys [10]: [c_last_name#79, c_first_name#78, s_store_name#64, ca_state#83, s_state#66, i_color#72, i_current_price#70, i_manager_id#74, i_units#73, i_size#71]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#56))]
+Aggregate Attributes [1]: [sum#87]
+Results [11]: [c_last_name#79, c_first_name#78, s_store_name#64, ca_state#83, s_state#66, i_color#72, i_current_price#70, i_manager_id#74, i_units#73, i_size#71, sum#88]
+
+(90) Exchange
+Input [11]: [c_last_name#79, c_first_name#78, s_store_name#64, ca_state#83, s_state#66, i_color#72, i_current_price#70, i_manager_id#74, i_units#73, i_size#71, sum#88]
+Arguments: hashpartitioning(c_last_name#79, c_first_name#78, s_store_name#64, ca_state#83, s_state#66, i_color#72, i_current_price#70, i_manager_id#74, i_units#73, i_size#71, 5), ENSURE_REQUIREMENTS, [id=#89]
+
+(91) HashAggregate [codegen id : 10]
+Input [11]: [c_last_name#79, c_first_name#78, s_store_name#64, ca_state#83, s_state#66, i_color#72, i_current_price#70, i_manager_id#74, i_units#73, i_size#71, sum#88]
+Keys [10]: [c_last_name#79, c_first_name#78, s_store_name#64, ca_state#83, s_state#66, i_color#72, i_current_price#70, i_manager_id#74, i_units#73, i_size#71]
+Functions [1]: [sum(UnscaledValue(ss_net_paid#56))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#56))#90]
+Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#56))#90,17,2) AS netpaid#40]
+
+(92) HashAggregate [codegen id : 10]
+Input [1]: [netpaid#40]
+Keys: []
+Functions [1]: [partial_avg(netpaid#40)]
+Aggregate Attributes [2]: [sum#91, count#92]
+Results [2]: [sum#93, count#94]
+
+(93) Exchange
+Input [2]: [sum#93, count#94]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#95]
+
+(94) HashAggregate [codegen id : 11]
+Input [2]: [sum#93, count#94]
+Keys: []
+Functions [1]: [avg(netpaid#40)]
+Aggregate Attributes [1]: [avg(netpaid#40)#96]
+Results [1]: [CheckOverflow((0.050000 * promote_precision(avg(netpaid#40)#96)), DecimalType(24,8), true) AS (0.05 * avg(netpaid))#97]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/simplified.txt
new file mode 100644
index 0000000000000..6309ec999ce77
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/simplified.txt
@@ -0,0 +1,151 @@
+WholeStageCodegen (12)
+  Sort [c_last_name,c_first_name,s_store_name]
+    InputAdapter
+      Exchange [c_last_name,c_first_name,s_store_name] #1
+        WholeStageCodegen (11)
+          Project [c_last_name,c_first_name,s_store_name,paid]
+            Filter [sum(netpaid)]
+              Subquery #1
+                WholeStageCodegen (11)
+                  HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count]
+                    InputAdapter
+                      Exchange #10
+                        WholeStageCodegen (10)
+                          HashAggregate [netpaid] [sum,count,sum,count]
+                            HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum]
+                              InputAdapter
+                                Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #11
+                                  WholeStageCodegen (9)
+                                    HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum]
+                                      Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state]
+                                        BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip]
+                                          Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country]
+                                            BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+                                              Project [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id]
+                                                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                  Project [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip]
+                                                    BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                      Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid]
+                                                        SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                                          InputAdapter
+                                                            WholeStageCodegen (2)
+                                                              Sort [ss_ticket_number,ss_item_sk]
+                                                                InputAdapter
+                                                                  Exchange [ss_ticket_number,ss_item_sk] #12
+                                                                    WholeStageCodegen (1)
+                                                                      Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid]
+                                                                        Filter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk]
+                                                          InputAdapter
+                                                            WholeStageCodegen (4)
+                                                              Sort [sr_ticket_number,sr_item_sk]
+                                                                InputAdapter
+                                                                  Exchange [sr_ticket_number,sr_item_sk] #13
+                                                                    WholeStageCodegen (3)
+                                                                      Project [sr_item_sk,sr_ticket_number]
+                                                                        Filter [sr_ticket_number,sr_item_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk]
+                                                      InputAdapter
+                                                        BroadcastExchange #14
+                                                          WholeStageCodegen (5)
+                                                            Project [s_store_sk,s_store_name,s_state,s_zip]
+                                                              Filter [s_market_id,s_store_sk,s_zip]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip]
+                                                  InputAdapter
+                                                    BroadcastExchange #15
+                                                      WholeStageCodegen (6)
+                                                        Filter [i_item_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id]
+                                              InputAdapter
+                                                BroadcastExchange #16
+                                                  WholeStageCodegen (7)
+                                                    Filter [c_customer_sk,c_current_addr_sk,c_birth_country]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country]
+                                          InputAdapter
+                                            BroadcastExchange #17
+                                              WholeStageCodegen (8)
+                                                Filter [ca_address_sk,ca_country,ca_zip]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.customer_address [ca_address_sk,ca_state,ca_zip,ca_country]
+              HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum(netpaid),sum,isEmpty]
+                InputAdapter
+                  Exchange [c_last_name,c_first_name,s_store_name] #2
+                    WholeStageCodegen (10)
+                      HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty]
+                        HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum]
+                          InputAdapter
+                            Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #3
+                              WholeStageCodegen (9)
+                                HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum]
+                                  Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state]
+                                    BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip]
+                                      Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country]
+                                        BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+                                          Project [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id]
+                                            BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                              Project [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip]
+                                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                  Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid]
+                                                    SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                                      InputAdapter
+                                                        WholeStageCodegen (2)
+                                                          Sort [ss_ticket_number,ss_item_sk]
+                                                            InputAdapter
+                                                              Exchange [ss_ticket_number,ss_item_sk] #4
+                                                                WholeStageCodegen (1)
+                                                                  Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid]
+                                                                    Filter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk]
+                                                      InputAdapter
+                                                        WholeStageCodegen (4)
+                                                          Sort [sr_ticket_number,sr_item_sk]
+                                                            InputAdapter
+                                                              Exchange [sr_ticket_number,sr_item_sk] #5
+                                                                WholeStageCodegen (3)
+                                                                  Project [sr_item_sk,sr_ticket_number]
+                                                                    Filter [sr_ticket_number,sr_item_sk]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk]
+                                                  InputAdapter
+                                                    BroadcastExchange #6
+                                                      WholeStageCodegen (5)
+                                                        Project [s_store_sk,s_store_name,s_state,s_zip]
+                                                          Filter [s_market_id,s_store_sk,s_zip]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip]
+                                              InputAdapter
+                                                BroadcastExchange #7
+                                                  WholeStageCodegen (6)
+                                                    Filter [i_color,i_item_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id]
+                                          InputAdapter
+                                            BroadcastExchange #8
+                                              WholeStageCodegen (7)
+                                                Filter [c_customer_sk,c_current_addr_sk,c_birth_country]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country]
+                                      InputAdapter
+                                        BroadcastExchange #9
+                                          WholeStageCodegen (8)
+                                            Filter [ca_address_sk,ca_country,ca_zip]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.customer_address [ca_address_sk,ca_state,ca_zip,ca_country]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.sf100/explain.txt
new file mode 100644
index 0000000000000..1fb9533024b2f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.sf100/explain.txt
@@ -0,0 +1,445 @@
+== Physical Plan ==
+TakeOrderedAndProject (77)
++- Union (76)
+   :- * HashAggregate (32)
+   :  +- Exchange (31)
+   :     +- * HashAggregate (30)
+   :        +- * Project (29)
+   :           +- * BroadcastHashJoin Inner BuildRight (28)
+   :              :- * Project (23)
+   :              :  +- * BroadcastHashJoin Inner BuildRight (22)
+   :              :     :- * Project (17)
+   :              :     :  +- * BroadcastHashJoin Inner BuildRight (16)
+   :              :     :     :- * Project (10)
+   :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+   :              :     :     :     :- * Filter (3)
+   :              :     :     :     :  +- * ColumnarToRow (2)
+   :              :     :     :     :     +- Scan parquet default.store_sales (1)
+   :              :     :     :     +- BroadcastExchange (8)
+   :              :     :     :        +- * Project (7)
+   :              :     :     :           +- * Filter (6)
+   :              :     :     :              +- * ColumnarToRow (5)
+   :              :     :     :                 +- Scan parquet default.customer_demographics (4)
+   :              :     :     +- BroadcastExchange (15)
+   :              :     :        +- * Project (14)
+   :              :     :           +- * Filter (13)
+   :              :     :              +- * ColumnarToRow (12)
+   :              :     :                 +- Scan parquet default.date_dim (11)
+   :              :     +- BroadcastExchange (21)
+   :              :        +- * Filter (20)
+   :              :           +- * ColumnarToRow (19)
+   :              :              +- Scan parquet default.store (18)
+   :              +- BroadcastExchange (27)
+   :                 +- * Filter (26)
+   :                    +- * ColumnarToRow (25)
+   :                       +- Scan parquet default.item (24)
+   :- * HashAggregate (54)
+   :  +- Exchange (53)
+   :     +- * HashAggregate (52)
+   :        +- * Project (51)
+   :           +- * BroadcastHashJoin Inner BuildRight (50)
+   :              :- * Project (48)
+   :              :  +- * BroadcastHashJoin Inner BuildRight (47)
+   :              :     :- * Project (45)
+   :              :     :  +- * BroadcastHashJoin Inner BuildRight (44)
+   :              :     :     :- * Project (38)
+   :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (37)
+   :              :     :     :     :- * Filter (35)
+   :              :     :     :     :  +- * ColumnarToRow (34)
+   :              :     :     :     :     +- Scan parquet default.store_sales (33)
+   :              :     :     :     +- ReusedExchange (36)
+   :              :     :     +- BroadcastExchange (43)
+   :              :     :        +- * Project (42)
+   :              :     :           +- * Filter (41)
+   :              :     :              +- * ColumnarToRow (40)
+   :              :     :                 +- Scan parquet default.store (39)
+   :              :     +- ReusedExchange (46)
+   :              +- ReusedExchange (49)
+   +- * HashAggregate (75)
+      +- Exchange (74)
+         +- * HashAggregate (73)
+            +- * Project (72)
+               +- * BroadcastHashJoin Inner BuildRight (71)
+                  :- * Project (66)
+                  :  +- * BroadcastHashJoin Inner BuildRight (65)
+                  :     :- * Project (63)
+                  :     :  +- * BroadcastHashJoin Inner BuildRight (62)
+                  :     :     :- * Project (60)
+                  :     :     :  +- * BroadcastHashJoin Inner BuildRight (59)
+                  :     :     :     :- * Filter (57)
+                  :     :     :     :  +- * ColumnarToRow (56)
+                  :     :     :     :     +- Scan parquet default.store_sales (55)
+                  :     :     :     +- ReusedExchange (58)
+                  :     :     +- ReusedExchange (61)
+                  :     +- ReusedExchange (64)
+                  +- BroadcastExchange (70)
+                     +- * Filter (69)
+                        +- * ColumnarToRow (68)
+                           +- Scan parquet default.item (67)
+
+
+(1) Scan parquet default.store_sales
+Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_cdemo_sk:int,ss_store_sk:int,ss_quantity:int,ss_list_price:decimal(7,2),ss_sales_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+
+(3) Filter [codegen id : 5]
+Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1))
+
+(4) Scan parquet default.customer_demographics
+Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,F), EqualTo(cd_marital_status,W), EqualTo(cd_education_status,Primary             ), IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_education_status:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+
+(6) Filter [codegen id : 1]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (cd_gender#11 = F)) AND (cd_marital_status#12 = W)) AND (cd_education_status#13 = Primary             )) AND isnotnull(cd_demo_sk#10))
+
+(7) Project [codegen id : 1]
+Output [1]: [cd_demo_sk#10]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+
+(8) BroadcastExchange
+Input [1]: [cd_demo_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#10]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#10]
+
+(11) Scan parquet default.date_dim
+Output [2]: [d_date_sk#15, d_year#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#15, d_year#16]
+
+(13) Filter [codegen id : 2]
+Input [2]: [d_date_sk#15, d_year#16]
+Condition : ((isnotnull(d_year#16) AND (d_year#16 = 1998)) AND isnotnull(d_date_sk#15))
+
+(14) Project [codegen id : 2]
+Output [1]: [d_date_sk#15]
+Input [2]: [d_date_sk#15, d_year#16]
+
+(15) BroadcastExchange
+Input [1]: [d_date_sk#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(16) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#15]
+Join condition: None
+
+(17) Project [codegen id : 5]
+Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7]
+Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#15]
+
+(18) Scan parquet default.store
+Output [2]: [s_store_sk#18, s_state#19]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_state:string>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [2]: [s_store_sk#18, s_state#19]
+
+(20) Filter [codegen id : 3]
+Input [2]: [s_store_sk#18, s_state#19]
+Condition : ((isnotnull(s_state#19) AND (s_state#19 = TN)) AND isnotnull(s_store_sk#18))
+
+(21) BroadcastExchange
+Input [2]: [s_store_sk#18, s_state#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#20]
+
+(22) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#18]
+Join condition: None
+
+(23) Project [codegen id : 5]
+Output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#19]
+Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#18, s_state#19]
+
+(24) Scan parquet default.item
+Output [2]: [i_item_sk#21, i_item_id#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(25) ColumnarToRow [codegen id : 4]
+Input [2]: [i_item_sk#21, i_item_id#22]
+
+(26) Filter [codegen id : 4]
+Input [2]: [i_item_sk#21, i_item_id#22]
+Condition : isnotnull(i_item_sk#21)
+
+(27) BroadcastExchange
+Input [2]: [i_item_sk#21, i_item_id#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#23]
+
+(28) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#21]
+Join condition: None
+
+(29) Project [codegen id : 5]
+Output [6]: [i_item_id#22, s_state#19, ss_quantity#4 AS agg1#24, ss_list_price#5 AS agg2#25, ss_coupon_amt#7 AS agg3#26, ss_sales_price#6 AS agg4#27]
+Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#19, i_item_sk#21, i_item_id#22]
+
+(30) HashAggregate [codegen id : 5]
+Input [6]: [i_item_id#22, s_state#19, agg1#24, agg2#25, agg3#26, agg4#27]
+Keys [2]: [i_item_id#22, s_state#19]
+Functions [4]: [partial_avg(agg1#24), partial_avg(UnscaledValue(agg2#25)), partial_avg(UnscaledValue(agg3#26)), partial_avg(UnscaledValue(agg4#27))]
+Aggregate Attributes [8]: [sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35]
+Results [10]: [i_item_id#22, s_state#19, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43]
+
+(31) Exchange
+Input [10]: [i_item_id#22, s_state#19, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43]
+Arguments: hashpartitioning(i_item_id#22, s_state#19, 5), ENSURE_REQUIREMENTS, [id=#44]
+
+(32) HashAggregate [codegen id : 6]
+Input [10]: [i_item_id#22, s_state#19, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43]
+Keys [2]: [i_item_id#22, s_state#19]
+Functions [4]: [avg(agg1#24), avg(UnscaledValue(agg2#25)), avg(UnscaledValue(agg3#26)), avg(UnscaledValue(agg4#27))]
+Aggregate Attributes [4]: [avg(agg1#24)#45, avg(UnscaledValue(agg2#25))#46, avg(UnscaledValue(agg3#26))#47, avg(UnscaledValue(agg4#27))#48]
+Results [7]: [i_item_id#22, s_state#19, 0 AS g_state#49, avg(agg1#24)#45 AS agg1#50, cast((avg(UnscaledValue(agg2#25))#46 / 100.0) as decimal(11,6)) AS agg2#51, cast((avg(UnscaledValue(agg3#26))#47 / 100.0) as decimal(11,6)) AS agg3#52, cast((avg(UnscaledValue(agg4#27))#48 / 100.0) as decimal(11,6)) AS agg4#53]
+
+(33) Scan parquet default.store_sales
+Output [8]: [ss_item_sk#54, ss_cdemo_sk#55, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, ss_sold_date_sk#61]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#61), dynamicpruningexpression(ss_sold_date_sk#61 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_cdemo_sk:int,ss_store_sk:int,ss_quantity:int,ss_list_price:decimal(7,2),ss_sales_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(34) ColumnarToRow [codegen id : 11]
+Input [8]: [ss_item_sk#54, ss_cdemo_sk#55, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, ss_sold_date_sk#61]
+
+(35) Filter [codegen id : 11]
+Input [8]: [ss_item_sk#54, ss_cdemo_sk#55, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, ss_sold_date_sk#61]
+Condition : ((isnotnull(ss_cdemo_sk#55) AND isnotnull(ss_store_sk#56)) AND isnotnull(ss_item_sk#54))
+
+(36) ReusedExchange [Reuses operator id: 8]
+Output [1]: [cd_demo_sk#62]
+
+(37) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ss_cdemo_sk#55]
+Right keys [1]: [cd_demo_sk#62]
+Join condition: None
+
+(38) Project [codegen id : 11]
+Output [7]: [ss_item_sk#54, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, ss_sold_date_sk#61]
+Input [9]: [ss_item_sk#54, ss_cdemo_sk#55, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, ss_sold_date_sk#61, cd_demo_sk#62]
+
+(39) Scan parquet default.store
+Output [2]: [s_store_sk#63, s_state#64]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_state:string>
+
+(40) ColumnarToRow [codegen id : 8]
+Input [2]: [s_store_sk#63, s_state#64]
+
+(41) Filter [codegen id : 8]
+Input [2]: [s_store_sk#63, s_state#64]
+Condition : ((isnotnull(s_state#64) AND (s_state#64 = TN)) AND isnotnull(s_store_sk#63))
+
+(42) Project [codegen id : 8]
+Output [1]: [s_store_sk#63]
+Input [2]: [s_store_sk#63, s_state#64]
+
+(43) BroadcastExchange
+Input [1]: [s_store_sk#63]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#65]
+
+(44) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ss_store_sk#56]
+Right keys [1]: [s_store_sk#63]
+Join condition: None
+
+(45) Project [codegen id : 11]
+Output [6]: [ss_item_sk#54, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, ss_sold_date_sk#61]
+Input [8]: [ss_item_sk#54, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, ss_sold_date_sk#61, s_store_sk#63]
+
+(46) ReusedExchange [Reuses operator id: 15]
+Output [1]: [d_date_sk#66]
+
+(47) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ss_sold_date_sk#61]
+Right keys [1]: [d_date_sk#66]
+Join condition: None
+
+(48) Project [codegen id : 11]
+Output [5]: [ss_item_sk#54, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60]
+Input [7]: [ss_item_sk#54, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, ss_sold_date_sk#61, d_date_sk#66]
+
+(49) ReusedExchange [Reuses operator id: 27]
+Output [2]: [i_item_sk#67, i_item_id#68]
+
+(50) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ss_item_sk#54]
+Right keys [1]: [i_item_sk#67]
+Join condition: None
+
+(51) Project [codegen id : 11]
+Output [5]: [i_item_id#68, ss_quantity#57 AS agg1#24, ss_list_price#58 AS agg2#25, ss_coupon_amt#60 AS agg3#26, ss_sales_price#59 AS agg4#27]
+Input [7]: [ss_item_sk#54, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, i_item_sk#67, i_item_id#68]
+
+(52) HashAggregate [codegen id : 11]
+Input [5]: [i_item_id#68, agg1#24, agg2#25, agg3#26, agg4#27]
+Keys [1]: [i_item_id#68]
+Functions [4]: [partial_avg(agg1#24), partial_avg(UnscaledValue(agg2#25)), partial_avg(UnscaledValue(agg3#26)), partial_avg(UnscaledValue(agg4#27))]
+Aggregate Attributes [8]: [sum#69, count#70, sum#71, count#72, sum#73, count#74, sum#75, count#76]
+Results [9]: [i_item_id#68, sum#77, count#78, sum#79, count#80, sum#81, count#82, sum#83, count#84]
+
+(53) Exchange
+Input [9]: [i_item_id#68, sum#77, count#78, sum#79, count#80, sum#81, count#82, sum#83, count#84]
+Arguments: hashpartitioning(i_item_id#68, 5), ENSURE_REQUIREMENTS, [id=#85]
+
+(54) HashAggregate [codegen id : 12]
+Input [9]: [i_item_id#68, sum#77, count#78, sum#79, count#80, sum#81, count#82, sum#83, count#84]
+Keys [1]: [i_item_id#68]
+Functions [4]: [avg(agg1#24), avg(UnscaledValue(agg2#25)), avg(UnscaledValue(agg3#26)), avg(UnscaledValue(agg4#27))]
+Aggregate Attributes [4]: [avg(agg1#24)#86, avg(UnscaledValue(agg2#25))#87, avg(UnscaledValue(agg3#26))#88, avg(UnscaledValue(agg4#27))#89]
+Results [7]: [i_item_id#68, null AS s_state#90, 1 AS g_state#91, avg(agg1#24)#86 AS agg1#92, cast((avg(UnscaledValue(agg2#25))#87 / 100.0) as decimal(11,6)) AS agg2#93, cast((avg(UnscaledValue(agg3#26))#88 / 100.0) as decimal(11,6)) AS agg3#94, cast((avg(UnscaledValue(agg4#27))#89 / 100.0) as decimal(11,6)) AS agg4#95]
+
+(55) Scan parquet default.store_sales
+Output [8]: [ss_item_sk#96, ss_cdemo_sk#97, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, ss_sold_date_sk#103]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#103), dynamicpruningexpression(ss_sold_date_sk#103 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_cdemo_sk:int,ss_store_sk:int,ss_quantity:int,ss_list_price:decimal(7,2),ss_sales_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(56) ColumnarToRow [codegen id : 17]
+Input [8]: [ss_item_sk#96, ss_cdemo_sk#97, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, ss_sold_date_sk#103]
+
+(57) Filter [codegen id : 17]
+Input [8]: [ss_item_sk#96, ss_cdemo_sk#97, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, ss_sold_date_sk#103]
+Condition : ((isnotnull(ss_cdemo_sk#97) AND isnotnull(ss_store_sk#98)) AND isnotnull(ss_item_sk#96))
+
+(58) ReusedExchange [Reuses operator id: 8]
+Output [1]: [cd_demo_sk#104]
+
+(59) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ss_cdemo_sk#97]
+Right keys [1]: [cd_demo_sk#104]
+Join condition: None
+
+(60) Project [codegen id : 17]
+Output [7]: [ss_item_sk#96, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, ss_sold_date_sk#103]
+Input [9]: [ss_item_sk#96, ss_cdemo_sk#97, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, ss_sold_date_sk#103, cd_demo_sk#104]
+
+(61) ReusedExchange [Reuses operator id: 43]
+Output [1]: [s_store_sk#105]
+
+(62) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ss_store_sk#98]
+Right keys [1]: [s_store_sk#105]
+Join condition: None
+
+(63) Project [codegen id : 17]
+Output [6]: [ss_item_sk#96, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, ss_sold_date_sk#103]
+Input [8]: [ss_item_sk#96, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, ss_sold_date_sk#103, s_store_sk#105]
+
+(64) ReusedExchange [Reuses operator id: 15]
+Output [1]: [d_date_sk#106]
+
+(65) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ss_sold_date_sk#103]
+Right keys [1]: [d_date_sk#106]
+Join condition: None
+
+(66) Project [codegen id : 17]
+Output [5]: [ss_item_sk#96, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102]
+Input [7]: [ss_item_sk#96, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, ss_sold_date_sk#103, d_date_sk#106]
+
+(67) Scan parquet default.item
+Output [1]: [i_item_sk#107]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int>
+
+(68) ColumnarToRow [codegen id : 16]
+Input [1]: [i_item_sk#107]
+
+(69) Filter [codegen id : 16]
+Input [1]: [i_item_sk#107]
+Condition : isnotnull(i_item_sk#107)
+
+(70) BroadcastExchange
+Input [1]: [i_item_sk#107]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#108]
+
+(71) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ss_item_sk#96]
+Right keys [1]: [i_item_sk#107]
+Join condition: None
+
+(72) Project [codegen id : 17]
+Output [4]: [ss_quantity#99 AS agg1#24, ss_list_price#100 AS agg2#25, ss_coupon_amt#102 AS agg3#26, ss_sales_price#101 AS agg4#27]
+Input [6]: [ss_item_sk#96, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, i_item_sk#107]
+
+(73) HashAggregate [codegen id : 17]
+Input [4]: [agg1#24, agg2#25, agg3#26, agg4#27]
+Keys: []
+Functions [4]: [partial_avg(agg1#24), partial_avg(UnscaledValue(agg2#25)), partial_avg(UnscaledValue(agg3#26)), partial_avg(UnscaledValue(agg4#27))]
+Aggregate Attributes [8]: [sum#109, count#110, sum#111, count#112, sum#113, count#114, sum#115, count#116]
+Results [8]: [sum#117, count#118, sum#119, count#120, sum#121, count#122, sum#123, count#124]
+
+(74) Exchange
+Input [8]: [sum#117, count#118, sum#119, count#120, sum#121, count#122, sum#123, count#124]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#125]
+
+(75) HashAggregate [codegen id : 18]
+Input [8]: [sum#117, count#118, sum#119, count#120, sum#121, count#122, sum#123, count#124]
+Keys: []
+Functions [4]: [avg(agg1#24), avg(UnscaledValue(agg2#25)), avg(UnscaledValue(agg3#26)), avg(UnscaledValue(agg4#27))]
+Aggregate Attributes [4]: [avg(agg1#24)#126, avg(UnscaledValue(agg2#25))#127, avg(UnscaledValue(agg3#26))#128, avg(UnscaledValue(agg4#27))#129]
+Results [7]: [null AS i_item_id#130, null AS s_state#131, 1 AS g_state#132, avg(agg1#24)#126 AS agg1#133, cast((avg(UnscaledValue(agg2#25))#127 / 100.0) as decimal(11,6)) AS agg2#134, cast((avg(UnscaledValue(agg3#26))#128 / 100.0) as decimal(11,6)) AS agg3#135, cast((avg(UnscaledValue(agg4#27))#129 / 100.0) as decimal(11,6)) AS agg4#136]
+
+(76) Union
+
+(77) TakeOrderedAndProject
+Input [7]: [i_item_id#22, s_state#19, g_state#49, agg1#50, agg2#51, agg3#52, agg4#53]
+Arguments: 100, [i_item_id#22 ASC NULLS FIRST, s_state#19 ASC NULLS FIRST], [i_item_id#22, s_state#19, g_state#49, agg1#50, agg2#51, agg3#52, agg4#53]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (78)
+
+
+(78) ReusedExchange [Reuses operator id: 15]
+Output [1]: [d_date_sk#15]
+
+Subquery:2 Hosting operator id = 33 Hosting Expression = ss_sold_date_sk#61 IN dynamicpruning#9
+
+Subquery:3 Hosting operator id = 55 Hosting Expression = ss_sold_date_sk#103 IN dynamicpruning#9
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.sf100/simplified.txt
new file mode 100644
index 0000000000000..e134318874eb8
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.sf100/simplified.txt
@@ -0,0 +1,117 @@
+TakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4]
+  Union
+    WholeStageCodegen (6)
+      HashAggregate [i_item_id,s_state,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4)),g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count]
+        InputAdapter
+          Exchange [i_item_id,s_state] #1
+            WholeStageCodegen (5)
+              HashAggregate [i_item_id,s_state,agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+                Project [i_item_id,s_state,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price]
+                  BroadcastHashJoin [ss_item_sk,i_item_sk]
+                    Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state]
+                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                        Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                            Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                              BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk]
+                                Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                                        SubqueryBroadcast [d_date_sk] #1
+                                          ReusedExchange [d_date_sk] #2
+                                InputAdapter
+                                  BroadcastExchange #3
+                                    WholeStageCodegen (1)
+                                      Project [cd_demo_sk]
+                                        Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status]
+                            InputAdapter
+                              BroadcastExchange #2
+                                WholeStageCodegen (2)
+                                  Project [d_date_sk]
+                                    Filter [d_year,d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.date_dim [d_date_sk,d_year]
+                        InputAdapter
+                          BroadcastExchange #4
+                            WholeStageCodegen (3)
+                              Filter [s_state,s_store_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.store [s_store_sk,s_state]
+                    InputAdapter
+                      BroadcastExchange #5
+                        WholeStageCodegen (4)
+                          Filter [i_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.item [i_item_sk,i_item_id]
+    WholeStageCodegen (12)
+      HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4)),s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count]
+        InputAdapter
+          Exchange [i_item_id] #6
+            WholeStageCodegen (11)
+              HashAggregate [i_item_id,agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+                Project [i_item_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price]
+                  BroadcastHashJoin [ss_item_sk,i_item_sk]
+                    Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                        Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                          BroadcastHashJoin [ss_store_sk,s_store_sk]
+                            Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                              BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk]
+                                Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                                        ReusedSubquery [d_date_sk] #1
+                                InputAdapter
+                                  ReusedExchange [cd_demo_sk] #3
+                            InputAdapter
+                              BroadcastExchange #7
+                                WholeStageCodegen (8)
+                                  Project [s_store_sk]
+                                    Filter [s_state,s_store_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.store [s_store_sk,s_state]
+                        InputAdapter
+                          ReusedExchange [d_date_sk] #2
+                    InputAdapter
+                      ReusedExchange [i_item_sk,i_item_id] #5
+    WholeStageCodegen (18)
+      HashAggregate [sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4)),i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count]
+        InputAdapter
+          Exchange #8
+            WholeStageCodegen (17)
+              HashAggregate [agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+                Project [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price]
+                  BroadcastHashJoin [ss_item_sk,i_item_sk]
+                    Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                        Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                          BroadcastHashJoin [ss_store_sk,s_store_sk]
+                            Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                              BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk]
+                                Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                                        ReusedSubquery [d_date_sk] #1
+                                InputAdapter
+                                  ReusedExchange [cd_demo_sk] #3
+                            InputAdapter
+                              ReusedExchange [s_store_sk] #7
+                        InputAdapter
+                          ReusedExchange [d_date_sk] #2
+                    InputAdapter
+                      BroadcastExchange #9
+                        WholeStageCodegen (16)
+                          Filter [i_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.item [i_item_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a/explain.txt
new file mode 100644
index 0000000000000..f05f4e9f2e43e
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a/explain.txt
@@ -0,0 +1,445 @@
+== Physical Plan ==
+TakeOrderedAndProject (77)
++- Union (76)
+   :- * HashAggregate (32)
+   :  +- Exchange (31)
+   :     +- * HashAggregate (30)
+   :        +- * Project (29)
+   :           +- * BroadcastHashJoin Inner BuildRight (28)
+   :              :- * Project (23)
+   :              :  +- * BroadcastHashJoin Inner BuildRight (22)
+   :              :     :- * Project (17)
+   :              :     :  +- * BroadcastHashJoin Inner BuildRight (16)
+   :              :     :     :- * Project (10)
+   :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+   :              :     :     :     :- * Filter (3)
+   :              :     :     :     :  +- * ColumnarToRow (2)
+   :              :     :     :     :     +- Scan parquet default.store_sales (1)
+   :              :     :     :     +- BroadcastExchange (8)
+   :              :     :     :        +- * Project (7)
+   :              :     :     :           +- * Filter (6)
+   :              :     :     :              +- * ColumnarToRow (5)
+   :              :     :     :                 +- Scan parquet default.customer_demographics (4)
+   :              :     :     +- BroadcastExchange (15)
+   :              :     :        +- * Project (14)
+   :              :     :           +- * Filter (13)
+   :              :     :              +- * ColumnarToRow (12)
+   :              :     :                 +- Scan parquet default.date_dim (11)
+   :              :     +- BroadcastExchange (21)
+   :              :        +- * Filter (20)
+   :              :           +- * ColumnarToRow (19)
+   :              :              +- Scan parquet default.store (18)
+   :              +- BroadcastExchange (27)
+   :                 +- * Filter (26)
+   :                    +- * ColumnarToRow (25)
+   :                       +- Scan parquet default.item (24)
+   :- * HashAggregate (54)
+   :  +- Exchange (53)
+   :     +- * HashAggregate (52)
+   :        +- * Project (51)
+   :           +- * BroadcastHashJoin Inner BuildRight (50)
+   :              :- * Project (48)
+   :              :  +- * BroadcastHashJoin Inner BuildRight (47)
+   :              :     :- * Project (41)
+   :              :     :  +- * BroadcastHashJoin Inner BuildRight (40)
+   :              :     :     :- * Project (38)
+   :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (37)
+   :              :     :     :     :- * Filter (35)
+   :              :     :     :     :  +- * ColumnarToRow (34)
+   :              :     :     :     :     +- Scan parquet default.store_sales (33)
+   :              :     :     :     +- ReusedExchange (36)
+   :              :     :     +- ReusedExchange (39)
+   :              :     +- BroadcastExchange (46)
+   :              :        +- * Project (45)
+   :              :           +- * Filter (44)
+   :              :              +- * ColumnarToRow (43)
+   :              :                 +- Scan parquet default.store (42)
+   :              +- ReusedExchange (49)
+   +- * HashAggregate (75)
+      +- Exchange (74)
+         +- * HashAggregate (73)
+            +- * Project (72)
+               +- * BroadcastHashJoin Inner BuildRight (71)
+                  :- * Project (66)
+                  :  +- * BroadcastHashJoin Inner BuildRight (65)
+                  :     :- * Project (63)
+                  :     :  +- * BroadcastHashJoin Inner BuildRight (62)
+                  :     :     :- * Project (60)
+                  :     :     :  +- * BroadcastHashJoin Inner BuildRight (59)
+                  :     :     :     :- * Filter (57)
+                  :     :     :     :  +- * ColumnarToRow (56)
+                  :     :     :     :     +- Scan parquet default.store_sales (55)
+                  :     :     :     +- ReusedExchange (58)
+                  :     :     +- ReusedExchange (61)
+                  :     +- ReusedExchange (64)
+                  +- BroadcastExchange (70)
+                     +- * Filter (69)
+                        +- * ColumnarToRow (68)
+                           +- Scan parquet default.item (67)
+
+
+(1) Scan parquet default.store_sales
+Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_cdemo_sk:int,ss_store_sk:int,ss_quantity:int,ss_list_price:decimal(7,2),ss_sales_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+
+(3) Filter [codegen id : 5]
+Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1))
+
+(4) Scan parquet default.customer_demographics
+Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,F), EqualTo(cd_marital_status,W), EqualTo(cd_education_status,Primary             ), IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_education_status:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+
+(6) Filter [codegen id : 1]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (cd_gender#11 = F)) AND (cd_marital_status#12 = W)) AND (cd_education_status#13 = Primary             )) AND isnotnull(cd_demo_sk#10))
+
+(7) Project [codegen id : 1]
+Output [1]: [cd_demo_sk#10]
+Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13]
+
+(8) BroadcastExchange
+Input [1]: [cd_demo_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(9) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#10]
+Join condition: None
+
+(10) Project [codegen id : 5]
+Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8]
+Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#10]
+
+(11) Scan parquet default.date_dim
+Output [2]: [d_date_sk#15, d_year#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#15, d_year#16]
+
+(13) Filter [codegen id : 2]
+Input [2]: [d_date_sk#15, d_year#16]
+Condition : ((isnotnull(d_year#16) AND (d_year#16 = 1998)) AND isnotnull(d_date_sk#15))
+
+(14) Project [codegen id : 2]
+Output [1]: [d_date_sk#15]
+Input [2]: [d_date_sk#15, d_year#16]
+
+(15) BroadcastExchange
+Input [1]: [d_date_sk#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(16) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#15]
+Join condition: None
+
+(17) Project [codegen id : 5]
+Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7]
+Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#15]
+
+(18) Scan parquet default.store
+Output [2]: [s_store_sk#18, s_state#19]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_state:string>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [2]: [s_store_sk#18, s_state#19]
+
+(20) Filter [codegen id : 3]
+Input [2]: [s_store_sk#18, s_state#19]
+Condition : ((isnotnull(s_state#19) AND (s_state#19 = TN)) AND isnotnull(s_store_sk#18))
+
+(21) BroadcastExchange
+Input [2]: [s_store_sk#18, s_state#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#20]
+
+(22) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#18]
+Join condition: None
+
+(23) Project [codegen id : 5]
+Output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#19]
+Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#18, s_state#19]
+
+(24) Scan parquet default.item
+Output [2]: [i_item_sk#21, i_item_id#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string>
+
+(25) ColumnarToRow [codegen id : 4]
+Input [2]: [i_item_sk#21, i_item_id#22]
+
+(26) Filter [codegen id : 4]
+Input [2]: [i_item_sk#21, i_item_id#22]
+Condition : isnotnull(i_item_sk#21)
+
+(27) BroadcastExchange
+Input [2]: [i_item_sk#21, i_item_id#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#23]
+
+(28) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#21]
+Join condition: None
+
+(29) Project [codegen id : 5]
+Output [6]: [i_item_id#22, s_state#19, ss_quantity#4 AS agg1#24, ss_list_price#5 AS agg2#25, ss_coupon_amt#7 AS agg3#26, ss_sales_price#6 AS agg4#27]
+Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#19, i_item_sk#21, i_item_id#22]
+
+(30) HashAggregate [codegen id : 5]
+Input [6]: [i_item_id#22, s_state#19, agg1#24, agg2#25, agg3#26, agg4#27]
+Keys [2]: [i_item_id#22, s_state#19]
+Functions [4]: [partial_avg(agg1#24), partial_avg(UnscaledValue(agg2#25)), partial_avg(UnscaledValue(agg3#26)), partial_avg(UnscaledValue(agg4#27))]
+Aggregate Attributes [8]: [sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35]
+Results [10]: [i_item_id#22, s_state#19, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43]
+
+(31) Exchange
+Input [10]: [i_item_id#22, s_state#19, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43]
+Arguments: hashpartitioning(i_item_id#22, s_state#19, 5), ENSURE_REQUIREMENTS, [id=#44]
+
+(32) HashAggregate [codegen id : 6]
+Input [10]: [i_item_id#22, s_state#19, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43]
+Keys [2]: [i_item_id#22, s_state#19]
+Functions [4]: [avg(agg1#24), avg(UnscaledValue(agg2#25)), avg(UnscaledValue(agg3#26)), avg(UnscaledValue(agg4#27))]
+Aggregate Attributes [4]: [avg(agg1#24)#45, avg(UnscaledValue(agg2#25))#46, avg(UnscaledValue(agg3#26))#47, avg(UnscaledValue(agg4#27))#48]
+Results [7]: [i_item_id#22, s_state#19, 0 AS g_state#49, avg(agg1#24)#45 AS agg1#50, cast((avg(UnscaledValue(agg2#25))#46 / 100.0) as decimal(11,6)) AS agg2#51, cast((avg(UnscaledValue(agg3#26))#47 / 100.0) as decimal(11,6)) AS agg3#52, cast((avg(UnscaledValue(agg4#27))#48 / 100.0) as decimal(11,6)) AS agg4#53]
+
+(33) Scan parquet default.store_sales
+Output [8]: [ss_item_sk#54, ss_cdemo_sk#55, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, ss_sold_date_sk#61]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#61), dynamicpruningexpression(ss_sold_date_sk#61 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_cdemo_sk:int,ss_store_sk:int,ss_quantity:int,ss_list_price:decimal(7,2),ss_sales_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(34) ColumnarToRow [codegen id : 11]
+Input [8]: [ss_item_sk#54, ss_cdemo_sk#55, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, ss_sold_date_sk#61]
+
+(35) Filter [codegen id : 11]
+Input [8]: [ss_item_sk#54, ss_cdemo_sk#55, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, ss_sold_date_sk#61]
+Condition : ((isnotnull(ss_cdemo_sk#55) AND isnotnull(ss_store_sk#56)) AND isnotnull(ss_item_sk#54))
+
+(36) ReusedExchange [Reuses operator id: 8]
+Output [1]: [cd_demo_sk#62]
+
+(37) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ss_cdemo_sk#55]
+Right keys [1]: [cd_demo_sk#62]
+Join condition: None
+
+(38) Project [codegen id : 11]
+Output [7]: [ss_item_sk#54, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, ss_sold_date_sk#61]
+Input [9]: [ss_item_sk#54, ss_cdemo_sk#55, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, ss_sold_date_sk#61, cd_demo_sk#62]
+
+(39) ReusedExchange [Reuses operator id: 15]
+Output [1]: [d_date_sk#63]
+
+(40) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ss_sold_date_sk#61]
+Right keys [1]: [d_date_sk#63]
+Join condition: None
+
+(41) Project [codegen id : 11]
+Output [6]: [ss_item_sk#54, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60]
+Input [8]: [ss_item_sk#54, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, ss_sold_date_sk#61, d_date_sk#63]
+
+(42) Scan parquet default.store
+Output [2]: [s_store_sk#64, s_state#65]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_state:string>
+
+(43) ColumnarToRow [codegen id : 9]
+Input [2]: [s_store_sk#64, s_state#65]
+
+(44) Filter [codegen id : 9]
+Input [2]: [s_store_sk#64, s_state#65]
+Condition : ((isnotnull(s_state#65) AND (s_state#65 = TN)) AND isnotnull(s_store_sk#64))
+
+(45) Project [codegen id : 9]
+Output [1]: [s_store_sk#64]
+Input [2]: [s_store_sk#64, s_state#65]
+
+(46) BroadcastExchange
+Input [1]: [s_store_sk#64]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#66]
+
+(47) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ss_store_sk#56]
+Right keys [1]: [s_store_sk#64]
+Join condition: None
+
+(48) Project [codegen id : 11]
+Output [5]: [ss_item_sk#54, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60]
+Input [7]: [ss_item_sk#54, ss_store_sk#56, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, s_store_sk#64]
+
+(49) ReusedExchange [Reuses operator id: 27]
+Output [2]: [i_item_sk#67, i_item_id#68]
+
+(50) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [ss_item_sk#54]
+Right keys [1]: [i_item_sk#67]
+Join condition: None
+
+(51) Project [codegen id : 11]
+Output [5]: [i_item_id#68, ss_quantity#57 AS agg1#24, ss_list_price#58 AS agg2#25, ss_coupon_amt#60 AS agg3#26, ss_sales_price#59 AS agg4#27]
+Input [7]: [ss_item_sk#54, ss_quantity#57, ss_list_price#58, ss_sales_price#59, ss_coupon_amt#60, i_item_sk#67, i_item_id#68]
+
+(52) HashAggregate [codegen id : 11]
+Input [5]: [i_item_id#68, agg1#24, agg2#25, agg3#26, agg4#27]
+Keys [1]: [i_item_id#68]
+Functions [4]: [partial_avg(agg1#24), partial_avg(UnscaledValue(agg2#25)), partial_avg(UnscaledValue(agg3#26)), partial_avg(UnscaledValue(agg4#27))]
+Aggregate Attributes [8]: [sum#69, count#70, sum#71, count#72, sum#73, count#74, sum#75, count#76]
+Results [9]: [i_item_id#68, sum#77, count#78, sum#79, count#80, sum#81, count#82, sum#83, count#84]
+
+(53) Exchange
+Input [9]: [i_item_id#68, sum#77, count#78, sum#79, count#80, sum#81, count#82, sum#83, count#84]
+Arguments: hashpartitioning(i_item_id#68, 5), ENSURE_REQUIREMENTS, [id=#85]
+
+(54) HashAggregate [codegen id : 12]
+Input [9]: [i_item_id#68, sum#77, count#78, sum#79, count#80, sum#81, count#82, sum#83, count#84]
+Keys [1]: [i_item_id#68]
+Functions [4]: [avg(agg1#24), avg(UnscaledValue(agg2#25)), avg(UnscaledValue(agg3#26)), avg(UnscaledValue(agg4#27))]
+Aggregate Attributes [4]: [avg(agg1#24)#86, avg(UnscaledValue(agg2#25))#87, avg(UnscaledValue(agg3#26))#88, avg(UnscaledValue(agg4#27))#89]
+Results [7]: [i_item_id#68, null AS s_state#90, 1 AS g_state#91, avg(agg1#24)#86 AS agg1#92, cast((avg(UnscaledValue(agg2#25))#87 / 100.0) as decimal(11,6)) AS agg2#93, cast((avg(UnscaledValue(agg3#26))#88 / 100.0) as decimal(11,6)) AS agg3#94, cast((avg(UnscaledValue(agg4#27))#89 / 100.0) as decimal(11,6)) AS agg4#95]
+
+(55) Scan parquet default.store_sales
+Output [8]: [ss_item_sk#96, ss_cdemo_sk#97, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, ss_sold_date_sk#103]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#103), dynamicpruningexpression(ss_sold_date_sk#103 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_cdemo_sk:int,ss_store_sk:int,ss_quantity:int,ss_list_price:decimal(7,2),ss_sales_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(56) ColumnarToRow [codegen id : 17]
+Input [8]: [ss_item_sk#96, ss_cdemo_sk#97, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, ss_sold_date_sk#103]
+
+(57) Filter [codegen id : 17]
+Input [8]: [ss_item_sk#96, ss_cdemo_sk#97, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, ss_sold_date_sk#103]
+Condition : ((isnotnull(ss_cdemo_sk#97) AND isnotnull(ss_store_sk#98)) AND isnotnull(ss_item_sk#96))
+
+(58) ReusedExchange [Reuses operator id: 8]
+Output [1]: [cd_demo_sk#104]
+
+(59) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ss_cdemo_sk#97]
+Right keys [1]: [cd_demo_sk#104]
+Join condition: None
+
+(60) Project [codegen id : 17]
+Output [7]: [ss_item_sk#96, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, ss_sold_date_sk#103]
+Input [9]: [ss_item_sk#96, ss_cdemo_sk#97, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, ss_sold_date_sk#103, cd_demo_sk#104]
+
+(61) ReusedExchange [Reuses operator id: 15]
+Output [1]: [d_date_sk#105]
+
+(62) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ss_sold_date_sk#103]
+Right keys [1]: [d_date_sk#105]
+Join condition: None
+
+(63) Project [codegen id : 17]
+Output [6]: [ss_item_sk#96, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102]
+Input [8]: [ss_item_sk#96, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, ss_sold_date_sk#103, d_date_sk#105]
+
+(64) ReusedExchange [Reuses operator id: 46]
+Output [1]: [s_store_sk#106]
+
+(65) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ss_store_sk#98]
+Right keys [1]: [s_store_sk#106]
+Join condition: None
+
+(66) Project [codegen id : 17]
+Output [5]: [ss_item_sk#96, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102]
+Input [7]: [ss_item_sk#96, ss_store_sk#98, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, s_store_sk#106]
+
+(67) Scan parquet default.item
+Output [1]: [i_item_sk#107]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int>
+
+(68) ColumnarToRow [codegen id : 16]
+Input [1]: [i_item_sk#107]
+
+(69) Filter [codegen id : 16]
+Input [1]: [i_item_sk#107]
+Condition : isnotnull(i_item_sk#107)
+
+(70) BroadcastExchange
+Input [1]: [i_item_sk#107]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#108]
+
+(71) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ss_item_sk#96]
+Right keys [1]: [i_item_sk#107]
+Join condition: None
+
+(72) Project [codegen id : 17]
+Output [4]: [ss_quantity#99 AS agg1#24, ss_list_price#100 AS agg2#25, ss_coupon_amt#102 AS agg3#26, ss_sales_price#101 AS agg4#27]
+Input [6]: [ss_item_sk#96, ss_quantity#99, ss_list_price#100, ss_sales_price#101, ss_coupon_amt#102, i_item_sk#107]
+
+(73) HashAggregate [codegen id : 17]
+Input [4]: [agg1#24, agg2#25, agg3#26, agg4#27]
+Keys: []
+Functions [4]: [partial_avg(agg1#24), partial_avg(UnscaledValue(agg2#25)), partial_avg(UnscaledValue(agg3#26)), partial_avg(UnscaledValue(agg4#27))]
+Aggregate Attributes [8]: [sum#109, count#110, sum#111, count#112, sum#113, count#114, sum#115, count#116]
+Results [8]: [sum#117, count#118, sum#119, count#120, sum#121, count#122, sum#123, count#124]
+
+(74) Exchange
+Input [8]: [sum#117, count#118, sum#119, count#120, sum#121, count#122, sum#123, count#124]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#125]
+
+(75) HashAggregate [codegen id : 18]
+Input [8]: [sum#117, count#118, sum#119, count#120, sum#121, count#122, sum#123, count#124]
+Keys: []
+Functions [4]: [avg(agg1#24), avg(UnscaledValue(agg2#25)), avg(UnscaledValue(agg3#26)), avg(UnscaledValue(agg4#27))]
+Aggregate Attributes [4]: [avg(agg1#24)#126, avg(UnscaledValue(agg2#25))#127, avg(UnscaledValue(agg3#26))#128, avg(UnscaledValue(agg4#27))#129]
+Results [7]: [null AS i_item_id#130, null AS s_state#131, 1 AS g_state#132, avg(agg1#24)#126 AS agg1#133, cast((avg(UnscaledValue(agg2#25))#127 / 100.0) as decimal(11,6)) AS agg2#134, cast((avg(UnscaledValue(agg3#26))#128 / 100.0) as decimal(11,6)) AS agg3#135, cast((avg(UnscaledValue(agg4#27))#129 / 100.0) as decimal(11,6)) AS agg4#136]
+
+(76) Union
+
+(77) TakeOrderedAndProject
+Input [7]: [i_item_id#22, s_state#19, g_state#49, agg1#50, agg2#51, agg3#52, agg4#53]
+Arguments: 100, [i_item_id#22 ASC NULLS FIRST, s_state#19 ASC NULLS FIRST], [i_item_id#22, s_state#19, g_state#49, agg1#50, agg2#51, agg3#52, agg4#53]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (78)
+
+
+(78) ReusedExchange [Reuses operator id: 15]
+Output [1]: [d_date_sk#15]
+
+Subquery:2 Hosting operator id = 33 Hosting Expression = ss_sold_date_sk#61 IN dynamicpruning#9
+
+Subquery:3 Hosting operator id = 55 Hosting Expression = ss_sold_date_sk#103 IN dynamicpruning#9
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a/simplified.txt
new file mode 100644
index 0000000000000..ac6635c260051
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a/simplified.txt
@@ -0,0 +1,117 @@
+TakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4]
+  Union
+    WholeStageCodegen (6)
+      HashAggregate [i_item_id,s_state,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4)),g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count]
+        InputAdapter
+          Exchange [i_item_id,s_state] #1
+            WholeStageCodegen (5)
+              HashAggregate [i_item_id,s_state,agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+                Project [i_item_id,s_state,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price]
+                  BroadcastHashJoin [ss_item_sk,i_item_sk]
+                    Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state]
+                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                        Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                            Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                              BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk]
+                                Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                                        SubqueryBroadcast [d_date_sk] #1
+                                          ReusedExchange [d_date_sk] #2
+                                InputAdapter
+                                  BroadcastExchange #3
+                                    WholeStageCodegen (1)
+                                      Project [cd_demo_sk]
+                                        Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status]
+                            InputAdapter
+                              BroadcastExchange #2
+                                WholeStageCodegen (2)
+                                  Project [d_date_sk]
+                                    Filter [d_year,d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.date_dim [d_date_sk,d_year]
+                        InputAdapter
+                          BroadcastExchange #4
+                            WholeStageCodegen (3)
+                              Filter [s_state,s_store_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.store [s_store_sk,s_state]
+                    InputAdapter
+                      BroadcastExchange #5
+                        WholeStageCodegen (4)
+                          Filter [i_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.item [i_item_sk,i_item_id]
+    WholeStageCodegen (12)
+      HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4)),s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count]
+        InputAdapter
+          Exchange [i_item_id] #6
+            WholeStageCodegen (11)
+              HashAggregate [i_item_id,agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+                Project [i_item_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price]
+                  BroadcastHashJoin [ss_item_sk,i_item_sk]
+                    Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                        Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                            Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                              BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk]
+                                Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                                        ReusedSubquery [d_date_sk] #1
+                                InputAdapter
+                                  ReusedExchange [cd_demo_sk] #3
+                            InputAdapter
+                              ReusedExchange [d_date_sk] #2
+                        InputAdapter
+                          BroadcastExchange #7
+                            WholeStageCodegen (9)
+                              Project [s_store_sk]
+                                Filter [s_state,s_store_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store [s_store_sk,s_state]
+                    InputAdapter
+                      ReusedExchange [i_item_sk,i_item_id] #5
+    WholeStageCodegen (18)
+      HashAggregate [sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4)),i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count]
+        InputAdapter
+          Exchange #8
+            WholeStageCodegen (17)
+              HashAggregate [agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count]
+                Project [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price]
+                  BroadcastHashJoin [ss_item_sk,i_item_sk]
+                    Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                        Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt]
+                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                            Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                              BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk]
+                                Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk]
+                                        ReusedSubquery [d_date_sk] #1
+                                InputAdapter
+                                  ReusedExchange [cd_demo_sk] #3
+                            InputAdapter
+                              ReusedExchange [d_date_sk] #2
+                        InputAdapter
+                          ReusedExchange [s_store_sk] #7
+                    InputAdapter
+                      BroadcastExchange #9
+                        WholeStageCodegen (16)
+                          Filter [i_item_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.item [i_item_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.sf100/explain.txt
new file mode 100644
index 0000000000000..f6151f4d79ec1
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.sf100/explain.txt
@@ -0,0 +1,229 @@
+== Physical Plan ==
+* Sort (39)
++- Exchange (38)
+   +- * Project (37)
+      +- * SortMergeJoin Inner (36)
+         :- * Sort (30)
+         :  +- Exchange (29)
+         :     +- * Filter (28)
+         :        +- * HashAggregate (27)
+         :           +- Exchange (26)
+         :              +- * HashAggregate (25)
+         :                 +- * Project (24)
+         :                    +- * BroadcastHashJoin Inner BuildRight (23)
+         :                       :- * Project (17)
+         :                       :  +- * BroadcastHashJoin Inner BuildRight (16)
+         :                       :     :- * Project (10)
+         :                       :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+         :                       :     :     :- * Filter (3)
+         :                       :     :     :  +- * ColumnarToRow (2)
+         :                       :     :     :     +- Scan parquet default.store_sales (1)
+         :                       :     :     +- BroadcastExchange (8)
+         :                       :     :        +- * Project (7)
+         :                       :     :           +- * Filter (6)
+         :                       :     :              +- * ColumnarToRow (5)
+         :                       :     :                 +- Scan parquet default.date_dim (4)
+         :                       :     +- BroadcastExchange (15)
+         :                       :        +- * Project (14)
+         :                       :           +- * Filter (13)
+         :                       :              +- * ColumnarToRow (12)
+         :                       :                 +- Scan parquet default.store (11)
+         :                       +- BroadcastExchange (22)
+         :                          +- * Project (21)
+         :                             +- * Filter (20)
+         :                                +- * ColumnarToRow (19)
+         :                                   +- Scan parquet default.household_demographics (18)
+         +- * Sort (35)
+            +- Exchange (34)
+               +- * Filter (33)
+                  +- * ColumnarToRow (32)
+                     +- Scan parquet default.customer (31)
+
+
+(1) Scan parquet default.store_sales
+Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_hdemo_sk:int,ss_store_sk:int,ss_ticket_number:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+
+(3) Filter [codegen id : 4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#7, d_year#8, d_dom#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_dom:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+Condition : (((((d_dom#9 >= 1) AND (d_dom#9 <= 3)) OR ((d_dom#9 >= 25) AND (d_dom#9 <= 28))) AND d_year#8 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#7]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#5]
+Right keys [1]: [d_date_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4]
+Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#7]
+
+(11) Scan parquet default.store
+Output [2]: [s_store_sk#11, s_county#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_county), EqualTo(s_county,Williamson County), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_county:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [s_store_sk#11, s_county#12]
+
+(13) Filter [codegen id : 2]
+Input [2]: [s_store_sk#11, s_county#12]
+Condition : ((isnotnull(s_county#12) AND (s_county#12 = Williamson County)) AND isnotnull(s_store_sk#11))
+
+(14) Project [codegen id : 2]
+Output [1]: [s_store_sk#11]
+Input [2]: [s_store_sk#11, s_county#12]
+
+(15) BroadcastExchange
+Input [1]: [s_store_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#11]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#11]
+
+(18) Scan parquet default.household_demographics
+Output [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_vehicle_count), IsNotNull(hd_dep_count), Or(EqualTo(hd_buy_potential,>10000         ),EqualTo(hd_buy_potential,unknown        )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_buy_potential:string,hd_dep_count:int,hd_vehicle_count:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+
+(20) Filter [codegen id : 3]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+Condition : (((((isnotnull(hd_vehicle_count#17) AND isnotnull(hd_dep_count#16)) AND ((hd_buy_potential#15 = >10000         ) OR (hd_buy_potential#15 = unknown        ))) AND (hd_vehicle_count#17 > 0)) AND ((cast(hd_dep_count#16 as double) / cast(hd_vehicle_count#17 as double)) > 1.2)) AND isnotnull(hd_demo_sk#14))
+
+(21) Project [codegen id : 3]
+Output [1]: [hd_demo_sk#14]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+
+(22) BroadcastExchange
+Input [1]: [hd_demo_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(23) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#14]
+Join condition: None
+
+(24) Project [codegen id : 4]
+Output [2]: [ss_customer_sk#1, ss_ticket_number#4]
+Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#14]
+
+(25) HashAggregate [codegen id : 4]
+Input [2]: [ss_customer_sk#1, ss_ticket_number#4]
+Keys [2]: [ss_ticket_number#4, ss_customer_sk#1]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#19]
+Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+
+(26) Exchange
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(27) HashAggregate [codegen id : 5]
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+Keys [2]: [ss_ticket_number#4, ss_customer_sk#1]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#22]
+Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count(1)#22 AS cnt#23]
+
+(28) Filter [codegen id : 5]
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23]
+Condition : ((cnt#23 >= 15) AND (cnt#23 <= 20))
+
+(29) Exchange
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23]
+Arguments: hashpartitioning(ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(30) Sort [codegen id : 6]
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23]
+Arguments: [ss_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(31) Scan parquet default.customer
+Output [5]: [c_customer_sk#25, c_salutation#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_salutation:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string>
+
+(32) ColumnarToRow [codegen id : 7]
+Input [5]: [c_customer_sk#25, c_salutation#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29]
+
+(33) Filter [codegen id : 7]
+Input [5]: [c_customer_sk#25, c_salutation#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29]
+Condition : isnotnull(c_customer_sk#25)
+
+(34) Exchange
+Input [5]: [c_customer_sk#25, c_salutation#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29]
+Arguments: hashpartitioning(c_customer_sk#25, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(35) Sort [codegen id : 8]
+Input [5]: [c_customer_sk#25, c_salutation#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29]
+Arguments: [c_customer_sk#25 ASC NULLS FIRST], false, 0
+
+(36) SortMergeJoin [codegen id : 9]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#25]
+Join condition: None
+
+(37) Project [codegen id : 9]
+Output [6]: [c_last_name#28, c_first_name#27, c_salutation#26, c_preferred_cust_flag#29, ss_ticket_number#4, cnt#23]
+Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23, c_customer_sk#25, c_salutation#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29]
+
+(38) Exchange
+Input [6]: [c_last_name#28, c_first_name#27, c_salutation#26, c_preferred_cust_flag#29, ss_ticket_number#4, cnt#23]
+Arguments: rangepartitioning(c_last_name#28 ASC NULLS FIRST, c_first_name#27 ASC NULLS FIRST, c_salutation#26 ASC NULLS FIRST, c_preferred_cust_flag#29 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(39) Sort [codegen id : 10]
+Input [6]: [c_last_name#28, c_first_name#27, c_salutation#26, c_preferred_cust_flag#29, ss_ticket_number#4, cnt#23]
+Arguments: [c_last_name#28 ASC NULLS FIRST, c_first_name#27 ASC NULLS FIRST, c_salutation#26 ASC NULLS FIRST, c_preferred_cust_flag#29 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (40)
+
+
+(40) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#7]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.sf100/simplified.txt
new file mode 100644
index 0000000000000..537cfdff2e1eb
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.sf100/simplified.txt
@@ -0,0 +1,65 @@
+WholeStageCodegen (10)
+  Sort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number]
+    InputAdapter
+      Exchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number] #1
+        WholeStageCodegen (9)
+          Project [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt]
+            SortMergeJoin [ss_customer_sk,c_customer_sk]
+              InputAdapter
+                WholeStageCodegen (6)
+                  Sort [ss_customer_sk]
+                    InputAdapter
+                      Exchange [ss_customer_sk] #2
+                        WholeStageCodegen (5)
+                          Filter [cnt]
+                            HashAggregate [ss_ticket_number,ss_customer_sk,count] [count(1),cnt,count]
+                              InputAdapter
+                                Exchange [ss_ticket_number,ss_customer_sk] #3
+                                  WholeStageCodegen (4)
+                                    HashAggregate [ss_ticket_number,ss_customer_sk] [count,count]
+                                      Project [ss_customer_sk,ss_ticket_number]
+                                        BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                          Project [ss_customer_sk,ss_hdemo_sk,ss_ticket_number]
+                                            BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                              Project [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number]
+                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                  Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk]
+                                                          SubqueryBroadcast [d_date_sk] #1
+                                                            ReusedExchange [d_date_sk] #4
+                                                  InputAdapter
+                                                    BroadcastExchange #4
+                                                      WholeStageCodegen (1)
+                                                        Project [d_date_sk]
+                                                          Filter [d_dom,d_year,d_date_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.date_dim [d_date_sk,d_year,d_dom]
+                                              InputAdapter
+                                                BroadcastExchange #5
+                                                  WholeStageCodegen (2)
+                                                    Project [s_store_sk]
+                                                      Filter [s_county,s_store_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.store [s_store_sk,s_county]
+                                          InputAdapter
+                                            BroadcastExchange #6
+                                              WholeStageCodegen (3)
+                                                Project [hd_demo_sk]
+                                                  Filter [hd_vehicle_count,hd_dep_count,hd_buy_potential,hd_demo_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count]
+              InputAdapter
+                WholeStageCodegen (8)
+                  Sort [c_customer_sk]
+                    InputAdapter
+                      Exchange [c_customer_sk] #7
+                        WholeStageCodegen (7)
+                          Filter [c_customer_sk]
+                            ColumnarToRow
+                              InputAdapter
+                                Scan parquet default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/explain.txt
new file mode 100644
index 0000000000000..e4d14f842f5a9
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/explain.txt
@@ -0,0 +1,214 @@
+== Physical Plan ==
+* Sort (36)
++- Exchange (35)
+   +- * Project (34)
+      +- * BroadcastHashJoin Inner BuildRight (33)
+         :- * Filter (28)
+         :  +- * HashAggregate (27)
+         :     +- Exchange (26)
+         :        +- * HashAggregate (25)
+         :           +- * Project (24)
+         :              +- * BroadcastHashJoin Inner BuildRight (23)
+         :                 :- * Project (17)
+         :                 :  +- * BroadcastHashJoin Inner BuildRight (16)
+         :                 :     :- * Project (10)
+         :                 :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+         :                 :     :     :- * Filter (3)
+         :                 :     :     :  +- * ColumnarToRow (2)
+         :                 :     :     :     +- Scan parquet default.store_sales (1)
+         :                 :     :     +- BroadcastExchange (8)
+         :                 :     :        +- * Project (7)
+         :                 :     :           +- * Filter (6)
+         :                 :     :              +- * ColumnarToRow (5)
+         :                 :     :                 +- Scan parquet default.date_dim (4)
+         :                 :     +- BroadcastExchange (15)
+         :                 :        +- * Project (14)
+         :                 :           +- * Filter (13)
+         :                 :              +- * ColumnarToRow (12)
+         :                 :                 +- Scan parquet default.store (11)
+         :                 +- BroadcastExchange (22)
+         :                    +- * Project (21)
+         :                       +- * Filter (20)
+         :                          +- * ColumnarToRow (19)
+         :                             +- Scan parquet default.household_demographics (18)
+         +- BroadcastExchange (32)
+            +- * Filter (31)
+               +- * ColumnarToRow (30)
+                  +- Scan parquet default.customer (29)
+
+
+(1) Scan parquet default.store_sales
+Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_hdemo_sk:int,ss_store_sk:int,ss_ticket_number:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+
+(3) Filter [codegen id : 4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5]
+Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#7, d_year#8, d_dom#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_dom:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+Condition : (((((d_dom#9 >= 1) AND (d_dom#9 <= 3)) OR ((d_dom#9 >= 25) AND (d_dom#9 <= 28))) AND d_year#8 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#7]
+Input [3]: [d_date_sk#7, d_year#8, d_dom#9]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#5]
+Right keys [1]: [d_date_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4]
+Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#7]
+
+(11) Scan parquet default.store
+Output [2]: [s_store_sk#11, s_county#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_county), EqualTo(s_county,Williamson County), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_county:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [s_store_sk#11, s_county#12]
+
+(13) Filter [codegen id : 2]
+Input [2]: [s_store_sk#11, s_county#12]
+Condition : ((isnotnull(s_county#12) AND (s_county#12 = Williamson County)) AND isnotnull(s_store_sk#11))
+
+(14) Project [codegen id : 2]
+Output [1]: [s_store_sk#11]
+Input [2]: [s_store_sk#11, s_county#12]
+
+(15) BroadcastExchange
+Input [1]: [s_store_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#3]
+Right keys [1]: [s_store_sk#11]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4]
+Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#11]
+
+(18) Scan parquet default.household_demographics
+Output [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_vehicle_count), IsNotNull(hd_dep_count), Or(EqualTo(hd_buy_potential,>10000         ),EqualTo(hd_buy_potential,unknown        )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_buy_potential:string,hd_dep_count:int,hd_vehicle_count:int>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+
+(20) Filter [codegen id : 3]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+Condition : (((((isnotnull(hd_vehicle_count#17) AND isnotnull(hd_dep_count#16)) AND ((hd_buy_potential#15 = >10000         ) OR (hd_buy_potential#15 = unknown        ))) AND (hd_vehicle_count#17 > 0)) AND ((cast(hd_dep_count#16 as double) / cast(hd_vehicle_count#17 as double)) > 1.2)) AND isnotnull(hd_demo_sk#14))
+
+(21) Project [codegen id : 3]
+Output [1]: [hd_demo_sk#14]
+Input [4]: [hd_demo_sk#14, hd_buy_potential#15, hd_dep_count#16, hd_vehicle_count#17]
+
+(22) BroadcastExchange
+Input [1]: [hd_demo_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(23) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_hdemo_sk#2]
+Right keys [1]: [hd_demo_sk#14]
+Join condition: None
+
+(24) Project [codegen id : 4]
+Output [2]: [ss_customer_sk#1, ss_ticket_number#4]
+Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#14]
+
+(25) HashAggregate [codegen id : 4]
+Input [2]: [ss_customer_sk#1, ss_ticket_number#4]
+Keys [2]: [ss_ticket_number#4, ss_customer_sk#1]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#19]
+Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+
+(26) Exchange
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(27) HashAggregate [codegen id : 6]
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#20]
+Keys [2]: [ss_ticket_number#4, ss_customer_sk#1]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#22]
+Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count(1)#22 AS cnt#23]
+
+(28) Filter [codegen id : 6]
+Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23]
+Condition : ((cnt#23 >= 15) AND (cnt#23 <= 20))
+
+(29) Scan parquet default.customer
+Output [5]: [c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_salutation:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string>
+
+(30) ColumnarToRow [codegen id : 5]
+Input [5]: [c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+
+(31) Filter [codegen id : 5]
+Input [5]: [c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+Condition : isnotnull(c_customer_sk#24)
+
+(32) BroadcastExchange
+Input [5]: [c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#29]
+
+(33) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#24]
+Join condition: None
+
+(34) Project [codegen id : 6]
+Output [6]: [c_last_name#27, c_first_name#26, c_salutation#25, c_preferred_cust_flag#28, ss_ticket_number#4, cnt#23]
+Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#23, c_customer_sk#24, c_salutation#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28]
+
+(35) Exchange
+Input [6]: [c_last_name#27, c_first_name#26, c_salutation#25, c_preferred_cust_flag#28, ss_ticket_number#4, cnt#23]
+Arguments: rangepartitioning(c_last_name#27 ASC NULLS FIRST, c_first_name#26 ASC NULLS FIRST, c_salutation#25 ASC NULLS FIRST, c_preferred_cust_flag#28 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(36) Sort [codegen id : 7]
+Input [6]: [c_last_name#27, c_first_name#26, c_salutation#25, c_preferred_cust_flag#28, ss_ticket_number#4, cnt#23]
+Arguments: [c_last_name#27 ASC NULLS FIRST, c_first_name#26 ASC NULLS FIRST, c_salutation#25 ASC NULLS FIRST, c_preferred_cust_flag#28 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (37)
+
+
+(37) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#7]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/simplified.txt
new file mode 100644
index 0000000000000..ab6398796def0
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/simplified.txt
@@ -0,0 +1,56 @@
+WholeStageCodegen (7)
+  Sort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number]
+    InputAdapter
+      Exchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number] #1
+        WholeStageCodegen (6)
+          Project [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt]
+            BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+              Filter [cnt]
+                HashAggregate [ss_ticket_number,ss_customer_sk,count] [count(1),cnt,count]
+                  InputAdapter
+                    Exchange [ss_ticket_number,ss_customer_sk] #2
+                      WholeStageCodegen (4)
+                        HashAggregate [ss_ticket_number,ss_customer_sk] [count,count]
+                          Project [ss_customer_sk,ss_ticket_number]
+                            BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                              Project [ss_customer_sk,ss_hdemo_sk,ss_ticket_number]
+                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                  Project [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number]
+                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                      Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk]
+                                              SubqueryBroadcast [d_date_sk] #1
+                                                ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        BroadcastExchange #3
+                                          WholeStageCodegen (1)
+                                            Project [d_date_sk]
+                                              Filter [d_dom,d_year,d_date_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_date_sk,d_year,d_dom]
+                                  InputAdapter
+                                    BroadcastExchange #4
+                                      WholeStageCodegen (2)
+                                        Project [s_store_sk]
+                                          Filter [s_county,s_store_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store [s_store_sk,s_county]
+                              InputAdapter
+                                BroadcastExchange #5
+                                  WholeStageCodegen (3)
+                                    Project [hd_demo_sk]
+                                      Filter [hd_vehicle_count,hd_dep_count,hd_buy_potential,hd_demo_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count]
+              InputAdapter
+                BroadcastExchange #6
+                  WholeStageCodegen (5)
+                    Filter [c_customer_sk]
+                      ColumnarToRow
+                        InputAdapter
+                          Scan parquet default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.sf100/explain.txt
new file mode 100644
index 0000000000000..7c032590fe404
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.sf100/explain.txt
@@ -0,0 +1,328 @@
+== Physical Plan ==
+TakeOrderedAndProject (57)
++- * HashAggregate (56)
+   +- Exchange (55)
+      +- * HashAggregate (54)
+         +- * Project (53)
+            +- * SortMergeJoin Inner (52)
+               :- * Sort (46)
+               :  +- Exchange (45)
+               :     +- * Project (44)
+               :        +- * SortMergeJoin Inner (43)
+               :           :- * Sort (37)
+               :           :  +- Exchange (36)
+               :           :     +- * Project (35)
+               :           :        +- * Filter (34)
+               :           :           +- SortMergeJoin ExistenceJoin(exists#1) (33)
+               :           :              :- SortMergeJoin ExistenceJoin(exists#2) (25)
+               :           :              :  :- * SortMergeJoin LeftSemi (17)
+               :           :              :  :  :- * Sort (5)
+               :           :              :  :  :  +- Exchange (4)
+               :           :              :  :  :     +- * Filter (3)
+               :           :              :  :  :        +- * ColumnarToRow (2)
+               :           :              :  :  :           +- Scan parquet default.customer (1)
+               :           :              :  :  +- * Sort (16)
+               :           :              :  :     +- Exchange (15)
+               :           :              :  :        +- * Project (14)
+               :           :              :  :           +- * BroadcastHashJoin Inner BuildRight (13)
+               :           :              :  :              :- * ColumnarToRow (7)
+               :           :              :  :              :  +- Scan parquet default.store_sales (6)
+               :           :              :  :              +- BroadcastExchange (12)
+               :           :              :  :                 +- * Project (11)
+               :           :              :  :                    +- * Filter (10)
+               :           :              :  :                       +- * ColumnarToRow (9)
+               :           :              :  :                          +- Scan parquet default.date_dim (8)
+               :           :              :  +- * Sort (24)
+               :           :              :     +- Exchange (23)
+               :           :              :        +- * Project (22)
+               :           :              :           +- * BroadcastHashJoin Inner BuildRight (21)
+               :           :              :              :- * ColumnarToRow (19)
+               :           :              :              :  +- Scan parquet default.web_sales (18)
+               :           :              :              +- ReusedExchange (20)
+               :           :              +- * Sort (32)
+               :           :                 +- Exchange (31)
+               :           :                    +- * Project (30)
+               :           :                       +- * BroadcastHashJoin Inner BuildRight (29)
+               :           :                          :- * ColumnarToRow (27)
+               :           :                          :  +- Scan parquet default.catalog_sales (26)
+               :           :                          +- ReusedExchange (28)
+               :           +- * Sort (42)
+               :              +- Exchange (41)
+               :                 +- * Filter (40)
+               :                    +- * ColumnarToRow (39)
+               :                       +- Scan parquet default.customer_address (38)
+               +- * Sort (51)
+                  +- Exchange (50)
+                     +- * Filter (49)
+                        +- * ColumnarToRow (48)
+                           +- Scan parquet default.customer_demographics (47)
+
+
+(1) Scan parquet default.customer
+Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_addr_sk:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5]
+
+(3) Filter [codegen id : 1]
+Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5]
+Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4))
+
+(4) Exchange
+Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5]
+Arguments: hashpartitioning(c_customer_sk#3, 5), ENSURE_REQUIREMENTS, [id=#6]
+
+(5) Sort [codegen id : 2]
+Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5]
+Arguments: [c_customer_sk#3 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.store_sales
+Output [2]: [ss_customer_sk#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+ReadSchema: struct<ss_customer_sk:int>
+
+(7) ColumnarToRow [codegen id : 4]
+Input [2]: [ss_customer_sk#7, ss_sold_date_sk#8]
+
+(8) Scan parquet default.date_dim
+Output [3]: [d_date_sk#10, d_year#11, d_qoy#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_qoy:int>
+
+(9) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#10, d_year#11, d_qoy#12]
+
+(10) Filter [codegen id : 3]
+Input [3]: [d_date_sk#10, d_year#11, d_qoy#12]
+Condition : ((((isnotnull(d_year#11) AND isnotnull(d_qoy#12)) AND (d_year#11 = 2002)) AND (d_qoy#12 < 4)) AND isnotnull(d_date_sk#10))
+
+(11) Project [codegen id : 3]
+Output [1]: [d_date_sk#10]
+Input [3]: [d_date_sk#10, d_year#11, d_qoy#12]
+
+(12) BroadcastExchange
+Input [1]: [d_date_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(13) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(14) Project [codegen id : 4]
+Output [1]: [ss_customer_sk#7]
+Input [3]: [ss_customer_sk#7, ss_sold_date_sk#8, d_date_sk#10]
+
+(15) Exchange
+Input [1]: [ss_customer_sk#7]
+Arguments: hashpartitioning(ss_customer_sk#7, 5), ENSURE_REQUIREMENTS, [id=#14]
+
+(16) Sort [codegen id : 5]
+Input [1]: [ss_customer_sk#7]
+Arguments: [ss_customer_sk#7 ASC NULLS FIRST], false, 0
+
+(17) SortMergeJoin [codegen id : 6]
+Left keys [1]: [c_customer_sk#3]
+Right keys [1]: [ss_customer_sk#7]
+Join condition: None
+
+(18) Scan parquet default.web_sales
+Output [2]: [ws_bill_customer_sk#15, ws_sold_date_sk#16]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#16), dynamicpruningexpression(ws_sold_date_sk#16 IN dynamicpruning#9)]
+ReadSchema: struct<ws_bill_customer_sk:int>
+
+(19) ColumnarToRow [codegen id : 8]
+Input [2]: [ws_bill_customer_sk#15, ws_sold_date_sk#16]
+
+(20) ReusedExchange [Reuses operator id: 12]
+Output [1]: [d_date_sk#17]
+
+(21) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ws_sold_date_sk#16]
+Right keys [1]: [d_date_sk#17]
+Join condition: None
+
+(22) Project [codegen id : 8]
+Output [1]: [ws_bill_customer_sk#15]
+Input [3]: [ws_bill_customer_sk#15, ws_sold_date_sk#16, d_date_sk#17]
+
+(23) Exchange
+Input [1]: [ws_bill_customer_sk#15]
+Arguments: hashpartitioning(ws_bill_customer_sk#15, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(24) Sort [codegen id : 9]
+Input [1]: [ws_bill_customer_sk#15]
+Arguments: [ws_bill_customer_sk#15 ASC NULLS FIRST], false, 0
+
+(25) SortMergeJoin
+Left keys [1]: [c_customer_sk#3]
+Right keys [1]: [ws_bill_customer_sk#15]
+Join condition: None
+
+(26) Scan parquet default.catalog_sales
+Output [2]: [cs_ship_customer_sk#19, cs_sold_date_sk#20]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#20), dynamicpruningexpression(cs_sold_date_sk#20 IN dynamicpruning#9)]
+ReadSchema: struct<cs_ship_customer_sk:int>
+
+(27) ColumnarToRow [codegen id : 11]
+Input [2]: [cs_ship_customer_sk#19, cs_sold_date_sk#20]
+
+(28) ReusedExchange [Reuses operator id: 12]
+Output [1]: [d_date_sk#21]
+
+(29) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [cs_sold_date_sk#20]
+Right keys [1]: [d_date_sk#21]
+Join condition: None
+
+(30) Project [codegen id : 11]
+Output [1]: [cs_ship_customer_sk#19]
+Input [3]: [cs_ship_customer_sk#19, cs_sold_date_sk#20, d_date_sk#21]
+
+(31) Exchange
+Input [1]: [cs_ship_customer_sk#19]
+Arguments: hashpartitioning(cs_ship_customer_sk#19, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(32) Sort [codegen id : 12]
+Input [1]: [cs_ship_customer_sk#19]
+Arguments: [cs_ship_customer_sk#19 ASC NULLS FIRST], false, 0
+
+(33) SortMergeJoin
+Left keys [1]: [c_customer_sk#3]
+Right keys [1]: [cs_ship_customer_sk#19]
+Join condition: None
+
+(34) Filter [codegen id : 13]
+Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1]
+Condition : (exists#2 OR exists#1)
+
+(35) Project [codegen id : 13]
+Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5]
+Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1]
+
+(36) Exchange
+Input [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5]
+Arguments: hashpartitioning(c_current_addr_sk#5, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(37) Sort [codegen id : 14]
+Input [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5]
+Arguments: [c_current_addr_sk#5 ASC NULLS FIRST], false, 0
+
+(38) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#24, ca_state#25]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(39) ColumnarToRow [codegen id : 15]
+Input [2]: [ca_address_sk#24, ca_state#25]
+
+(40) Filter [codegen id : 15]
+Input [2]: [ca_address_sk#24, ca_state#25]
+Condition : isnotnull(ca_address_sk#24)
+
+(41) Exchange
+Input [2]: [ca_address_sk#24, ca_state#25]
+Arguments: hashpartitioning(ca_address_sk#24, 5), ENSURE_REQUIREMENTS, [id=#26]
+
+(42) Sort [codegen id : 16]
+Input [2]: [ca_address_sk#24, ca_state#25]
+Arguments: [ca_address_sk#24 ASC NULLS FIRST], false, 0
+
+(43) SortMergeJoin [codegen id : 17]
+Left keys [1]: [c_current_addr_sk#5]
+Right keys [1]: [ca_address_sk#24]
+Join condition: None
+
+(44) Project [codegen id : 17]
+Output [2]: [c_current_cdemo_sk#4, ca_state#25]
+Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#24, ca_state#25]
+
+(45) Exchange
+Input [2]: [c_current_cdemo_sk#4, ca_state#25]
+Arguments: hashpartitioning(c_current_cdemo_sk#4, 5), ENSURE_REQUIREMENTS, [id=#27]
+
+(46) Sort [codegen id : 18]
+Input [2]: [c_current_cdemo_sk#4, ca_state#25]
+Arguments: [c_current_cdemo_sk#4 ASC NULLS FIRST], false, 0
+
+(47) Scan parquet default.customer_demographics
+Output [6]: [cd_demo_sk#28, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_dep_count:int,cd_dep_employed_count:int,cd_dep_college_count:int>
+
+(48) ColumnarToRow [codegen id : 19]
+Input [6]: [cd_demo_sk#28, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+
+(49) Filter [codegen id : 19]
+Input [6]: [cd_demo_sk#28, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Condition : isnotnull(cd_demo_sk#28)
+
+(50) Exchange
+Input [6]: [cd_demo_sk#28, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Arguments: hashpartitioning(cd_demo_sk#28, 5), ENSURE_REQUIREMENTS, [id=#34]
+
+(51) Sort [codegen id : 20]
+Input [6]: [cd_demo_sk#28, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Arguments: [cd_demo_sk#28 ASC NULLS FIRST], false, 0
+
+(52) SortMergeJoin [codegen id : 21]
+Left keys [1]: [c_current_cdemo_sk#4]
+Right keys [1]: [cd_demo_sk#28]
+Join condition: None
+
+(53) Project [codegen id : 21]
+Output [6]: [ca_state#25, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Input [8]: [c_current_cdemo_sk#4, ca_state#25, cd_demo_sk#28, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+
+(54) HashAggregate [codegen id : 21]
+Input [6]: [ca_state#25, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Keys [6]: [ca_state#25, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Functions [10]: [partial_count(1), partial_avg(cd_dep_count#31), partial_max(cd_dep_count#31), partial_sum(cd_dep_count#31), partial_avg(cd_dep_employed_count#32), partial_max(cd_dep_employed_count#32), partial_sum(cd_dep_employed_count#32), partial_avg(cd_dep_college_count#33), partial_max(cd_dep_college_count#33), partial_sum(cd_dep_college_count#33)]
+Aggregate Attributes [13]: [count#35, sum#36, count#37, max#38, sum#39, sum#40, count#41, max#42, sum#43, sum#44, count#45, max#46, sum#47]
+Results [19]: [ca_state#25, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33, count#48, sum#49, count#50, max#51, sum#52, sum#53, count#54, max#55, sum#56, sum#57, count#58, max#59, sum#60]
+
+(55) Exchange
+Input [19]: [ca_state#25, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33, count#48, sum#49, count#50, max#51, sum#52, sum#53, count#54, max#55, sum#56, sum#57, count#58, max#59, sum#60]
+Arguments: hashpartitioning(ca_state#25, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33, 5), ENSURE_REQUIREMENTS, [id=#61]
+
+(56) HashAggregate [codegen id : 22]
+Input [19]: [ca_state#25, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33, count#48, sum#49, count#50, max#51, sum#52, sum#53, count#54, max#55, sum#56, sum#57, count#58, max#59, sum#60]
+Keys [6]: [ca_state#25, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cd_dep_employed_count#32, cd_dep_college_count#33]
+Functions [10]: [count(1), avg(cd_dep_count#31), max(cd_dep_count#31), sum(cd_dep_count#31), avg(cd_dep_employed_count#32), max(cd_dep_employed_count#32), sum(cd_dep_employed_count#32), avg(cd_dep_college_count#33), max(cd_dep_college_count#33), sum(cd_dep_college_count#33)]
+Aggregate Attributes [10]: [count(1)#62, avg(cd_dep_count#31)#63, max(cd_dep_count#31)#64, sum(cd_dep_count#31)#65, avg(cd_dep_employed_count#32)#66, max(cd_dep_employed_count#32)#67, sum(cd_dep_employed_count#32)#68, avg(cd_dep_college_count#33)#69, max(cd_dep_college_count#33)#70, sum(cd_dep_college_count#33)#71]
+Results [18]: [ca_state#25, cd_gender#29, cd_marital_status#30, cd_dep_count#31, count(1)#62 AS cnt1#72, avg(cd_dep_count#31)#63 AS avg(cd_dep_count)#73, max(cd_dep_count#31)#64 AS max(cd_dep_count)#74, sum(cd_dep_count#31)#65 AS sum(cd_dep_count)#75, cd_dep_employed_count#32, count(1)#62 AS cnt2#76, avg(cd_dep_employed_count#32)#66 AS avg(cd_dep_employed_count)#77, max(cd_dep_employed_count#32)#67 AS max(cd_dep_employed_count)#78, sum(cd_dep_employed_count#32)#68 AS sum(cd_dep_employed_count)#79, cd_dep_college_count#33, count(1)#62 AS cnt3#80, avg(cd_dep_college_count#33)#69 AS avg(cd_dep_college_count)#81, max(cd_dep_college_count#33)#70 AS max(cd_dep_college_count)#82, sum(cd_dep_college_count#33)#71 AS sum(cd_dep_college_count)#83]
+
+(57) TakeOrderedAndProject
+Input [18]: [ca_state#25, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cnt1#72, avg(cd_dep_count)#73, max(cd_dep_count)#74, sum(cd_dep_count)#75, cd_dep_employed_count#32, cnt2#76, avg(cd_dep_employed_count)#77, max(cd_dep_employed_count)#78, sum(cd_dep_employed_count)#79, cd_dep_college_count#33, cnt3#80, avg(cd_dep_college_count)#81, max(cd_dep_college_count)#82, sum(cd_dep_college_count)#83]
+Arguments: 100, [ca_state#25 ASC NULLS FIRST, cd_gender#29 ASC NULLS FIRST, cd_marital_status#30 ASC NULLS FIRST, cd_dep_count#31 ASC NULLS FIRST, cd_dep_employed_count#32 ASC NULLS FIRST, cd_dep_college_count#33 ASC NULLS FIRST], [ca_state#25, cd_gender#29, cd_marital_status#30, cd_dep_count#31, cnt1#72, avg(cd_dep_count)#73, max(cd_dep_count)#74, sum(cd_dep_count)#75, cd_dep_employed_count#32, cnt2#76, avg(cd_dep_employed_count)#77, max(cd_dep_employed_count)#78, sum(cd_dep_employed_count)#79, cd_dep_college_count#33, cnt3#80, avg(cd_dep_college_count)#81, max(cd_dep_college_count)#82, sum(cd_dep_college_count)#83]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 6 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (58)
+
+
+(58) ReusedExchange [Reuses operator id: 12]
+Output [1]: [d_date_sk#10]
+
+Subquery:2 Hosting operator id = 18 Hosting Expression = ws_sold_date_sk#16 IN dynamicpruning#9
+
+Subquery:3 Hosting operator id = 26 Hosting Expression = cs_sold_date_sk#20 IN dynamicpruning#9
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.sf100/simplified.txt
new file mode 100644
index 0000000000000..6dc49f8dce3e5
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.sf100/simplified.txt
@@ -0,0 +1,107 @@
+TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)]
+  WholeStageCodegen (22)
+    HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] [count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum]
+      InputAdapter
+        Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1
+          WholeStageCodegen (21)
+            HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum]
+              Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count]
+                SortMergeJoin [c_current_cdemo_sk,cd_demo_sk]
+                  InputAdapter
+                    WholeStageCodegen (18)
+                      Sort [c_current_cdemo_sk]
+                        InputAdapter
+                          Exchange [c_current_cdemo_sk] #2
+                            WholeStageCodegen (17)
+                              Project [c_current_cdemo_sk,ca_state]
+                                SortMergeJoin [c_current_addr_sk,ca_address_sk]
+                                  InputAdapter
+                                    WholeStageCodegen (14)
+                                      Sort [c_current_addr_sk]
+                                        InputAdapter
+                                          Exchange [c_current_addr_sk] #3
+                                            WholeStageCodegen (13)
+                                              Project [c_current_cdemo_sk,c_current_addr_sk]
+                                                Filter [exists,exists]
+                                                  InputAdapter
+                                                    SortMergeJoin [c_customer_sk,cs_ship_customer_sk]
+                                                      SortMergeJoin [c_customer_sk,ws_bill_customer_sk]
+                                                        WholeStageCodegen (6)
+                                                          SortMergeJoin [c_customer_sk,ss_customer_sk]
+                                                            InputAdapter
+                                                              WholeStageCodegen (2)
+                                                                Sort [c_customer_sk]
+                                                                  InputAdapter
+                                                                    Exchange [c_customer_sk] #4
+                                                                      WholeStageCodegen (1)
+                                                                        Filter [c_current_addr_sk,c_current_cdemo_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk]
+                                                            InputAdapter
+                                                              WholeStageCodegen (5)
+                                                                Sort [ss_customer_sk]
+                                                                  InputAdapter
+                                                                    Exchange [ss_customer_sk] #5
+                                                                      WholeStageCodegen (4)
+                                                                        Project [ss_customer_sk]
+                                                                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                            ColumnarToRow
+                                                                              InputAdapter
+                                                                                Scan parquet default.store_sales [ss_customer_sk,ss_sold_date_sk]
+                                                                                  SubqueryBroadcast [d_date_sk] #1
+                                                                                    ReusedExchange [d_date_sk] #6
+                                                                            InputAdapter
+                                                                              BroadcastExchange #6
+                                                                                WholeStageCodegen (3)
+                                                                                  Project [d_date_sk]
+                                                                                    Filter [d_year,d_qoy,d_date_sk]
+                                                                                      ColumnarToRow
+                                                                                        InputAdapter
+                                                                                          Scan parquet default.date_dim [d_date_sk,d_year,d_qoy]
+                                                        WholeStageCodegen (9)
+                                                          Sort [ws_bill_customer_sk]
+                                                            InputAdapter
+                                                              Exchange [ws_bill_customer_sk] #7
+                                                                WholeStageCodegen (8)
+                                                                  Project [ws_bill_customer_sk]
+                                                                    BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.web_sales [ws_bill_customer_sk,ws_sold_date_sk]
+                                                                            ReusedSubquery [d_date_sk] #1
+                                                                      InputAdapter
+                                                                        ReusedExchange [d_date_sk] #6
+                                                      WholeStageCodegen (12)
+                                                        Sort [cs_ship_customer_sk]
+                                                          InputAdapter
+                                                            Exchange [cs_ship_customer_sk] #8
+                                                              WholeStageCodegen (11)
+                                                                Project [cs_ship_customer_sk]
+                                                                  BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk]
+                                                                          ReusedSubquery [d_date_sk] #1
+                                                                    InputAdapter
+                                                                      ReusedExchange [d_date_sk] #6
+                                  InputAdapter
+                                    WholeStageCodegen (16)
+                                      Sort [ca_address_sk]
+                                        InputAdapter
+                                          Exchange [ca_address_sk] #9
+                                            WholeStageCodegen (15)
+                                              Filter [ca_address_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.customer_address [ca_address_sk,ca_state]
+                  InputAdapter
+                    WholeStageCodegen (20)
+                      Sort [cd_demo_sk]
+                        InputAdapter
+                          Exchange [cd_demo_sk] #10
+                            WholeStageCodegen (19)
+                              Filter [cd_demo_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/explain.txt
new file mode 100644
index 0000000000000..642bf989159d3
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/explain.txt
@@ -0,0 +1,273 @@
+== Physical Plan ==
+TakeOrderedAndProject (46)
++- * HashAggregate (45)
+   +- Exchange (44)
+      +- * HashAggregate (43)
+         +- * Project (42)
+            +- * BroadcastHashJoin Inner BuildRight (41)
+               :- * Project (36)
+               :  +- * BroadcastHashJoin Inner BuildRight (35)
+               :     :- * Project (30)
+               :     :  +- * Filter (29)
+               :     :     +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (28)
+               :     :        :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (21)
+               :     :        :  :- * BroadcastHashJoin LeftSemi BuildRight (14)
+               :     :        :  :  :- * Filter (3)
+               :     :        :  :  :  +- * ColumnarToRow (2)
+               :     :        :  :  :     +- Scan parquet default.customer (1)
+               :     :        :  :  +- BroadcastExchange (13)
+               :     :        :  :     +- * Project (12)
+               :     :        :  :        +- * BroadcastHashJoin Inner BuildRight (11)
+               :     :        :  :           :- * ColumnarToRow (5)
+               :     :        :  :           :  +- Scan parquet default.store_sales (4)
+               :     :        :  :           +- BroadcastExchange (10)
+               :     :        :  :              +- * Project (9)
+               :     :        :  :                 +- * Filter (8)
+               :     :        :  :                    +- * ColumnarToRow (7)
+               :     :        :  :                       +- Scan parquet default.date_dim (6)
+               :     :        :  +- BroadcastExchange (20)
+               :     :        :     +- * Project (19)
+               :     :        :        +- * BroadcastHashJoin Inner BuildRight (18)
+               :     :        :           :- * ColumnarToRow (16)
+               :     :        :           :  +- Scan parquet default.web_sales (15)
+               :     :        :           +- ReusedExchange (17)
+               :     :        +- BroadcastExchange (27)
+               :     :           +- * Project (26)
+               :     :              +- * BroadcastHashJoin Inner BuildRight (25)
+               :     :                 :- * ColumnarToRow (23)
+               :     :                 :  +- Scan parquet default.catalog_sales (22)
+               :     :                 +- ReusedExchange (24)
+               :     +- BroadcastExchange (34)
+               :        +- * Filter (33)
+               :           +- * ColumnarToRow (32)
+               :              +- Scan parquet default.customer_address (31)
+               +- BroadcastExchange (40)
+                  +- * Filter (39)
+                     +- * ColumnarToRow (38)
+                        +- Scan parquet default.customer_demographics (37)
+
+
+(1) Scan parquet default.customer
+Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_addr_sk:int>
+
+(2) ColumnarToRow [codegen id : 9]
+Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5]
+
+(3) Filter [codegen id : 9]
+Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5]
+Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4))
+
+(4) Scan parquet default.store_sales
+Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)]
+ReadSchema: struct<ss_customer_sk:int>
+
+(5) ColumnarToRow [codegen id : 2]
+Input [2]: [ss_customer_sk#6, ss_sold_date_sk#7]
+
+(6) Scan parquet default.date_dim
+Output [3]: [d_date_sk#9, d_year#10, d_qoy#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_qoy:int>
+
+(7) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#9, d_year#10, d_qoy#11]
+
+(8) Filter [codegen id : 1]
+Input [3]: [d_date_sk#9, d_year#10, d_qoy#11]
+Condition : ((((isnotnull(d_year#10) AND isnotnull(d_qoy#11)) AND (d_year#10 = 2002)) AND (d_qoy#11 < 4)) AND isnotnull(d_date_sk#9))
+
+(9) Project [codegen id : 1]
+Output [1]: [d_date_sk#9]
+Input [3]: [d_date_sk#9, d_year#10, d_qoy#11]
+
+(10) BroadcastExchange
+Input [1]: [d_date_sk#9]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(11) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#7]
+Right keys [1]: [d_date_sk#9]
+Join condition: None
+
+(12) Project [codegen id : 2]
+Output [1]: [ss_customer_sk#6]
+Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#9]
+
+(13) BroadcastExchange
+Input [1]: [ss_customer_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(14) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_customer_sk#3]
+Right keys [1]: [ss_customer_sk#6]
+Join condition: None
+
+(15) Scan parquet default.web_sales
+Output [2]: [ws_bill_customer_sk#14, ws_sold_date_sk#15]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#15), dynamicpruningexpression(ws_sold_date_sk#15 IN dynamicpruning#8)]
+ReadSchema: struct<ws_bill_customer_sk:int>
+
+(16) ColumnarToRow [codegen id : 4]
+Input [2]: [ws_bill_customer_sk#14, ws_sold_date_sk#15]
+
+(17) ReusedExchange [Reuses operator id: 10]
+Output [1]: [d_date_sk#16]
+
+(18) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ws_sold_date_sk#15]
+Right keys [1]: [d_date_sk#16]
+Join condition: None
+
+(19) Project [codegen id : 4]
+Output [1]: [ws_bill_customer_sk#14]
+Input [3]: [ws_bill_customer_sk#14, ws_sold_date_sk#15, d_date_sk#16]
+
+(20) BroadcastExchange
+Input [1]: [ws_bill_customer_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(21) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_customer_sk#3]
+Right keys [1]: [ws_bill_customer_sk#14]
+Join condition: None
+
+(22) Scan parquet default.catalog_sales
+Output [2]: [cs_ship_customer_sk#18, cs_sold_date_sk#19]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#19), dynamicpruningexpression(cs_sold_date_sk#19 IN dynamicpruning#8)]
+ReadSchema: struct<cs_ship_customer_sk:int>
+
+(23) ColumnarToRow [codegen id : 6]
+Input [2]: [cs_ship_customer_sk#18, cs_sold_date_sk#19]
+
+(24) ReusedExchange [Reuses operator id: 10]
+Output [1]: [d_date_sk#20]
+
+(25) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [cs_sold_date_sk#19]
+Right keys [1]: [d_date_sk#20]
+Join condition: None
+
+(26) Project [codegen id : 6]
+Output [1]: [cs_ship_customer_sk#18]
+Input [3]: [cs_ship_customer_sk#18, cs_sold_date_sk#19, d_date_sk#20]
+
+(27) BroadcastExchange
+Input [1]: [cs_ship_customer_sk#18]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(28) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_customer_sk#3]
+Right keys [1]: [cs_ship_customer_sk#18]
+Join condition: None
+
+(29) Filter [codegen id : 9]
+Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1]
+Condition : (exists#2 OR exists#1)
+
+(30) Project [codegen id : 9]
+Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5]
+Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1]
+
+(31) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#22, ca_state#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(32) ColumnarToRow [codegen id : 7]
+Input [2]: [ca_address_sk#22, ca_state#23]
+
+(33) Filter [codegen id : 7]
+Input [2]: [ca_address_sk#22, ca_state#23]
+Condition : isnotnull(ca_address_sk#22)
+
+(34) BroadcastExchange
+Input [2]: [ca_address_sk#22, ca_state#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#24]
+
+(35) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_current_addr_sk#5]
+Right keys [1]: [ca_address_sk#22]
+Join condition: None
+
+(36) Project [codegen id : 9]
+Output [2]: [c_current_cdemo_sk#4, ca_state#23]
+Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#22, ca_state#23]
+
+(37) Scan parquet default.customer_demographics
+Output [6]: [cd_demo_sk#25, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_dep_count:int,cd_dep_employed_count:int,cd_dep_college_count:int>
+
+(38) ColumnarToRow [codegen id : 8]
+Input [6]: [cd_demo_sk#25, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30]
+
+(39) Filter [codegen id : 8]
+Input [6]: [cd_demo_sk#25, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30]
+Condition : isnotnull(cd_demo_sk#25)
+
+(40) BroadcastExchange
+Input [6]: [cd_demo_sk#25, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#31]
+
+(41) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_current_cdemo_sk#4]
+Right keys [1]: [cd_demo_sk#25]
+Join condition: None
+
+(42) Project [codegen id : 9]
+Output [6]: [ca_state#23, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30]
+Input [8]: [c_current_cdemo_sk#4, ca_state#23, cd_demo_sk#25, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30]
+
+(43) HashAggregate [codegen id : 9]
+Input [6]: [ca_state#23, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30]
+Keys [6]: [ca_state#23, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30]
+Functions [10]: [partial_count(1), partial_avg(cd_dep_count#28), partial_max(cd_dep_count#28), partial_sum(cd_dep_count#28), partial_avg(cd_dep_employed_count#29), partial_max(cd_dep_employed_count#29), partial_sum(cd_dep_employed_count#29), partial_avg(cd_dep_college_count#30), partial_max(cd_dep_college_count#30), partial_sum(cd_dep_college_count#30)]
+Aggregate Attributes [13]: [count#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40, sum#41, count#42, max#43, sum#44]
+Results [19]: [ca_state#23, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53, sum#54, count#55, max#56, sum#57]
+
+(44) Exchange
+Input [19]: [ca_state#23, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53, sum#54, count#55, max#56, sum#57]
+Arguments: hashpartitioning(ca_state#23, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, 5), ENSURE_REQUIREMENTS, [id=#58]
+
+(45) HashAggregate [codegen id : 10]
+Input [19]: [ca_state#23, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53, sum#54, count#55, max#56, sum#57]
+Keys [6]: [ca_state#23, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30]
+Functions [10]: [count(1), avg(cd_dep_count#28), max(cd_dep_count#28), sum(cd_dep_count#28), avg(cd_dep_employed_count#29), max(cd_dep_employed_count#29), sum(cd_dep_employed_count#29), avg(cd_dep_college_count#30), max(cd_dep_college_count#30), sum(cd_dep_college_count#30)]
+Aggregate Attributes [10]: [count(1)#59, avg(cd_dep_count#28)#60, max(cd_dep_count#28)#61, sum(cd_dep_count#28)#62, avg(cd_dep_employed_count#29)#63, max(cd_dep_employed_count#29)#64, sum(cd_dep_employed_count#29)#65, avg(cd_dep_college_count#30)#66, max(cd_dep_college_count#30)#67, sum(cd_dep_college_count#30)#68]
+Results [18]: [ca_state#23, cd_gender#26, cd_marital_status#27, cd_dep_count#28, count(1)#59 AS cnt1#69, avg(cd_dep_count#28)#60 AS avg(cd_dep_count)#70, max(cd_dep_count#28)#61 AS max(cd_dep_count)#71, sum(cd_dep_count#28)#62 AS sum(cd_dep_count)#72, cd_dep_employed_count#29, count(1)#59 AS cnt2#73, avg(cd_dep_employed_count#29)#63 AS avg(cd_dep_employed_count)#74, max(cd_dep_employed_count#29)#64 AS max(cd_dep_employed_count)#75, sum(cd_dep_employed_count#29)#65 AS sum(cd_dep_employed_count)#76, cd_dep_college_count#30, count(1)#59 AS cnt3#77, avg(cd_dep_college_count#30)#66 AS avg(cd_dep_college_count)#78, max(cd_dep_college_count#30)#67 AS max(cd_dep_college_count)#79, sum(cd_dep_college_count#30)#68 AS sum(cd_dep_college_count)#80]
+
+(46) TakeOrderedAndProject
+Input [18]: [ca_state#23, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cnt1#69, avg(cd_dep_count)#70, max(cd_dep_count)#71, sum(cd_dep_count)#72, cd_dep_employed_count#29, cnt2#73, avg(cd_dep_employed_count)#74, max(cd_dep_employed_count)#75, sum(cd_dep_employed_count)#76, cd_dep_college_count#30, cnt3#77, avg(cd_dep_college_count)#78, max(cd_dep_college_count)#79, sum(cd_dep_college_count)#80]
+Arguments: 100, [ca_state#23 ASC NULLS FIRST, cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_dep_count#28 ASC NULLS FIRST, cd_dep_employed_count#29 ASC NULLS FIRST, cd_dep_college_count#30 ASC NULLS FIRST], [ca_state#23, cd_gender#26, cd_marital_status#27, cd_dep_count#28, cnt1#69, avg(cd_dep_count)#70, max(cd_dep_count)#71, sum(cd_dep_count)#72, cd_dep_employed_count#29, cnt2#73, avg(cd_dep_employed_count)#74, max(cd_dep_employed_count)#75, sum(cd_dep_employed_count)#76, cd_dep_college_count#30, cnt3#77, avg(cd_dep_college_count)#78, max(cd_dep_college_count)#79, sum(cd_dep_college_count)#80]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (47)
+
+
+(47) ReusedExchange [Reuses operator id: 10]
+Output [1]: [d_date_sk#9]
+
+Subquery:2 Hosting operator id = 15 Hosting Expression = ws_sold_date_sk#15 IN dynamicpruning#8
+
+Subquery:3 Hosting operator id = 22 Hosting Expression = cs_sold_date_sk#19 IN dynamicpruning#8
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/simplified.txt
new file mode 100644
index 0000000000000..c6c9e1da58c26
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/simplified.txt
@@ -0,0 +1,74 @@
+TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)]
+  WholeStageCodegen (10)
+    HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] [count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum]
+      InputAdapter
+        Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1
+          WholeStageCodegen (9)
+            HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum]
+              Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count]
+                BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk]
+                  Project [c_current_cdemo_sk,ca_state]
+                    BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                      Project [c_current_cdemo_sk,c_current_addr_sk]
+                        Filter [exists,exists]
+                          BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk]
+                            BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk]
+                              BroadcastHashJoin [c_customer_sk,ss_customer_sk]
+                                Filter [c_current_addr_sk,c_current_cdemo_sk]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk]
+                                InputAdapter
+                                  BroadcastExchange #2
+                                    WholeStageCodegen (2)
+                                      Project [ss_customer_sk]
+                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.store_sales [ss_customer_sk,ss_sold_date_sk]
+                                                SubqueryBroadcast [d_date_sk] #1
+                                                  ReusedExchange [d_date_sk] #3
+                                          InputAdapter
+                                            BroadcastExchange #3
+                                              WholeStageCodegen (1)
+                                                Project [d_date_sk]
+                                                  Filter [d_year,d_qoy,d_date_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.date_dim [d_date_sk,d_year,d_qoy]
+                              InputAdapter
+                                BroadcastExchange #4
+                                  WholeStageCodegen (4)
+                                    Project [ws_bill_customer_sk]
+                                      BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.web_sales [ws_bill_customer_sk,ws_sold_date_sk]
+                                              ReusedSubquery [d_date_sk] #1
+                                        InputAdapter
+                                          ReusedExchange [d_date_sk] #3
+                            InputAdapter
+                              BroadcastExchange #5
+                                WholeStageCodegen (6)
+                                  Project [cs_ship_customer_sk]
+                                    BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk]
+                                            ReusedSubquery [d_date_sk] #1
+                                      InputAdapter
+                                        ReusedExchange [d_date_sk] #3
+                      InputAdapter
+                        BroadcastExchange #6
+                          WholeStageCodegen (7)
+                            Filter [ca_address_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.customer_address [ca_address_sk,ca_state]
+                  InputAdapter
+                    BroadcastExchange #7
+                      WholeStageCodegen (8)
+                        Filter [cd_demo_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.sf100/explain.txt
new file mode 100644
index 0000000000000..8e71d6c5c0965
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.sf100/explain.txt
@@ -0,0 +1,310 @@
+== Physical Plan ==
+TakeOrderedAndProject (54)
++- * HashAggregate (53)
+   +- Exchange (52)
+      +- * HashAggregate (51)
+         +- * Project (50)
+            +- * SortMergeJoin Inner (49)
+               :- * Sort (43)
+               :  +- Exchange (42)
+               :     +- * Project (41)
+               :        +- * SortMergeJoin Inner (40)
+               :           :- * Sort (34)
+               :           :  +- Exchange (33)
+               :           :     +- * Project (32)
+               :           :        +- * SortMergeJoin LeftSemi (31)
+               :           :           :- * SortMergeJoin LeftSemi (17)
+               :           :           :  :- * Sort (5)
+               :           :           :  :  +- Exchange (4)
+               :           :           :  :     +- * Filter (3)
+               :           :           :  :        +- * ColumnarToRow (2)
+               :           :           :  :           +- Scan parquet default.customer (1)
+               :           :           :  +- * Sort (16)
+               :           :           :     +- Exchange (15)
+               :           :           :        +- * Project (14)
+               :           :           :           +- * BroadcastHashJoin Inner BuildRight (13)
+               :           :           :              :- * ColumnarToRow (7)
+               :           :           :              :  +- Scan parquet default.store_sales (6)
+               :           :           :              +- BroadcastExchange (12)
+               :           :           :                 +- * Project (11)
+               :           :           :                    +- * Filter (10)
+               :           :           :                       +- * ColumnarToRow (9)
+               :           :           :                          +- Scan parquet default.date_dim (8)
+               :           :           +- * Sort (30)
+               :           :              +- Exchange (29)
+               :           :                 +- Union (28)
+               :           :                    :- * Project (22)
+               :           :                    :  +- * BroadcastHashJoin Inner BuildRight (21)
+               :           :                    :     :- * ColumnarToRow (19)
+               :           :                    :     :  +- Scan parquet default.web_sales (18)
+               :           :                    :     +- ReusedExchange (20)
+               :           :                    +- * Project (27)
+               :           :                       +- * BroadcastHashJoin Inner BuildRight (26)
+               :           :                          :- * ColumnarToRow (24)
+               :           :                          :  +- Scan parquet default.catalog_sales (23)
+               :           :                          +- ReusedExchange (25)
+               :           +- * Sort (39)
+               :              +- Exchange (38)
+               :                 +- * Filter (37)
+               :                    +- * ColumnarToRow (36)
+               :                       +- Scan parquet default.customer_address (35)
+               +- * Sort (48)
+                  +- Exchange (47)
+                     +- * Filter (46)
+                        +- * ColumnarToRow (45)
+                           +- Scan parquet default.customer_demographics (44)
+
+
+(1) Scan parquet default.customer
+Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_addr_sk:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+
+(3) Filter [codegen id : 1]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2))
+
+(4) Exchange
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+Arguments: hashpartitioning(c_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#4]
+
+(5) Sort [codegen id : 2]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+Arguments: [c_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.store_sales
+Output [2]: [ss_customer_sk#5, ss_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)]
+ReadSchema: struct<ss_customer_sk:int>
+
+(7) ColumnarToRow [codegen id : 4]
+Input [2]: [ss_customer_sk#5, ss_sold_date_sk#6]
+
+(8) Scan parquet default.date_dim
+Output [3]: [d_date_sk#8, d_year#9, d_qoy#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,1999), LessThan(d_qoy,4), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_qoy:int>
+
+(9) ColumnarToRow [codegen id : 3]
+Input [3]: [d_date_sk#8, d_year#9, d_qoy#10]
+
+(10) Filter [codegen id : 3]
+Input [3]: [d_date_sk#8, d_year#9, d_qoy#10]
+Condition : ((((isnotnull(d_year#9) AND isnotnull(d_qoy#10)) AND (d_year#9 = 1999)) AND (d_qoy#10 < 4)) AND isnotnull(d_date_sk#8))
+
+(11) Project [codegen id : 3]
+Output [1]: [d_date_sk#8]
+Input [3]: [d_date_sk#8, d_year#9, d_qoy#10]
+
+(12) BroadcastExchange
+Input [1]: [d_date_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#11]
+
+(13) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#6]
+Right keys [1]: [d_date_sk#8]
+Join condition: None
+
+(14) Project [codegen id : 4]
+Output [1]: [ss_customer_sk#5]
+Input [3]: [ss_customer_sk#5, ss_sold_date_sk#6, d_date_sk#8]
+
+(15) Exchange
+Input [1]: [ss_customer_sk#5]
+Arguments: hashpartitioning(ss_customer_sk#5, 5), ENSURE_REQUIREMENTS, [id=#12]
+
+(16) Sort [codegen id : 5]
+Input [1]: [ss_customer_sk#5]
+Arguments: [ss_customer_sk#5 ASC NULLS FIRST], false, 0
+
+(17) SortMergeJoin [codegen id : 6]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [ss_customer_sk#5]
+Join condition: None
+
+(18) Scan parquet default.web_sales
+Output [2]: [ws_bill_customer_sk#13, ws_sold_date_sk#14]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#14), dynamicpruningexpression(ws_sold_date_sk#14 IN dynamicpruning#7)]
+ReadSchema: struct<ws_bill_customer_sk:int>
+
+(19) ColumnarToRow [codegen id : 8]
+Input [2]: [ws_bill_customer_sk#13, ws_sold_date_sk#14]
+
+(20) ReusedExchange [Reuses operator id: 12]
+Output [1]: [d_date_sk#15]
+
+(21) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [ws_sold_date_sk#14]
+Right keys [1]: [d_date_sk#15]
+Join condition: None
+
+(22) Project [codegen id : 8]
+Output [1]: [ws_bill_customer_sk#13 AS customsk#16]
+Input [3]: [ws_bill_customer_sk#13, ws_sold_date_sk#14, d_date_sk#15]
+
+(23) Scan parquet default.catalog_sales
+Output [2]: [cs_ship_customer_sk#17, cs_sold_date_sk#18]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_sold_date_sk#18 IN dynamicpruning#7)]
+ReadSchema: struct<cs_ship_customer_sk:int>
+
+(24) ColumnarToRow [codegen id : 10]
+Input [2]: [cs_ship_customer_sk#17, cs_sold_date_sk#18]
+
+(25) ReusedExchange [Reuses operator id: 12]
+Output [1]: [d_date_sk#19]
+
+(26) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_sold_date_sk#18]
+Right keys [1]: [d_date_sk#19]
+Join condition: None
+
+(27) Project [codegen id : 10]
+Output [1]: [cs_ship_customer_sk#17 AS customsk#20]
+Input [3]: [cs_ship_customer_sk#17, cs_sold_date_sk#18, d_date_sk#19]
+
+(28) Union
+
+(29) Exchange
+Input [1]: [customsk#16]
+Arguments: hashpartitioning(customsk#16, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(30) Sort [codegen id : 11]
+Input [1]: [customsk#16]
+Arguments: [customsk#16 ASC NULLS FIRST], false, 0
+
+(31) SortMergeJoin [codegen id : 12]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [customsk#16]
+Join condition: None
+
+(32) Project [codegen id : 12]
+Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+
+(33) Exchange
+Input [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3]
+Arguments: hashpartitioning(c_current_addr_sk#3, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(34) Sort [codegen id : 13]
+Input [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3]
+Arguments: [c_current_addr_sk#3 ASC NULLS FIRST], false, 0
+
+(35) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#23, ca_state#24]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(36) ColumnarToRow [codegen id : 14]
+Input [2]: [ca_address_sk#23, ca_state#24]
+
+(37) Filter [codegen id : 14]
+Input [2]: [ca_address_sk#23, ca_state#24]
+Condition : isnotnull(ca_address_sk#23)
+
+(38) Exchange
+Input [2]: [ca_address_sk#23, ca_state#24]
+Arguments: hashpartitioning(ca_address_sk#23, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(39) Sort [codegen id : 15]
+Input [2]: [ca_address_sk#23, ca_state#24]
+Arguments: [ca_address_sk#23 ASC NULLS FIRST], false, 0
+
+(40) SortMergeJoin [codegen id : 16]
+Left keys [1]: [c_current_addr_sk#3]
+Right keys [1]: [ca_address_sk#23]
+Join condition: None
+
+(41) Project [codegen id : 16]
+Output [2]: [c_current_cdemo_sk#2, ca_state#24]
+Input [4]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#23, ca_state#24]
+
+(42) Exchange
+Input [2]: [c_current_cdemo_sk#2, ca_state#24]
+Arguments: hashpartitioning(c_current_cdemo_sk#2, 5), ENSURE_REQUIREMENTS, [id=#26]
+
+(43) Sort [codegen id : 17]
+Input [2]: [c_current_cdemo_sk#2, ca_state#24]
+Arguments: [c_current_cdemo_sk#2 ASC NULLS FIRST], false, 0
+
+(44) Scan parquet default.customer_demographics
+Output [6]: [cd_demo_sk#27, cd_gender#28, cd_marital_status#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_dep_count:int,cd_dep_employed_count:int,cd_dep_college_count:int>
+
+(45) ColumnarToRow [codegen id : 18]
+Input [6]: [cd_demo_sk#27, cd_gender#28, cd_marital_status#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32]
+
+(46) Filter [codegen id : 18]
+Input [6]: [cd_demo_sk#27, cd_gender#28, cd_marital_status#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32]
+Condition : isnotnull(cd_demo_sk#27)
+
+(47) Exchange
+Input [6]: [cd_demo_sk#27, cd_gender#28, cd_marital_status#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32]
+Arguments: hashpartitioning(cd_demo_sk#27, 5), ENSURE_REQUIREMENTS, [id=#33]
+
+(48) Sort [codegen id : 19]
+Input [6]: [cd_demo_sk#27, cd_gender#28, cd_marital_status#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32]
+Arguments: [cd_demo_sk#27 ASC NULLS FIRST], false, 0
+
+(49) SortMergeJoin [codegen id : 20]
+Left keys [1]: [c_current_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#27]
+Join condition: None
+
+(50) Project [codegen id : 20]
+Output [6]: [ca_state#24, cd_gender#28, cd_marital_status#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32]
+Input [8]: [c_current_cdemo_sk#2, ca_state#24, cd_demo_sk#27, cd_gender#28, cd_marital_status#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32]
+
+(51) HashAggregate [codegen id : 20]
+Input [6]: [ca_state#24, cd_gender#28, cd_marital_status#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32]
+Keys [6]: [ca_state#24, cd_gender#28, cd_marital_status#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32]
+Functions [10]: [partial_count(1), partial_avg(cd_dep_count#30), partial_max(cd_dep_count#30), partial_sum(cd_dep_count#30), partial_avg(cd_dep_employed_count#31), partial_max(cd_dep_employed_count#31), partial_sum(cd_dep_employed_count#31), partial_avg(cd_dep_college_count#32), partial_max(cd_dep_college_count#32), partial_sum(cd_dep_college_count#32)]
+Aggregate Attributes [13]: [count#34, sum#35, count#36, max#37, sum#38, sum#39, count#40, max#41, sum#42, sum#43, count#44, max#45, sum#46]
+Results [19]: [ca_state#24, cd_gender#28, cd_marital_status#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32, count#47, sum#48, count#49, max#50, sum#51, sum#52, count#53, max#54, sum#55, sum#56, count#57, max#58, sum#59]
+
+(52) Exchange
+Input [19]: [ca_state#24, cd_gender#28, cd_marital_status#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32, count#47, sum#48, count#49, max#50, sum#51, sum#52, count#53, max#54, sum#55, sum#56, count#57, max#58, sum#59]
+Arguments: hashpartitioning(ca_state#24, cd_gender#28, cd_marital_status#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32, 5), ENSURE_REQUIREMENTS, [id=#60]
+
+(53) HashAggregate [codegen id : 21]
+Input [19]: [ca_state#24, cd_gender#28, cd_marital_status#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32, count#47, sum#48, count#49, max#50, sum#51, sum#52, count#53, max#54, sum#55, sum#56, count#57, max#58, sum#59]
+Keys [6]: [ca_state#24, cd_gender#28, cd_marital_status#29, cd_dep_count#30, cd_dep_employed_count#31, cd_dep_college_count#32]
+Functions [10]: [count(1), avg(cd_dep_count#30), max(cd_dep_count#30), sum(cd_dep_count#30), avg(cd_dep_employed_count#31), max(cd_dep_employed_count#31), sum(cd_dep_employed_count#31), avg(cd_dep_college_count#32), max(cd_dep_college_count#32), sum(cd_dep_college_count#32)]
+Aggregate Attributes [10]: [count(1)#61, avg(cd_dep_count#30)#62, max(cd_dep_count#30)#63, sum(cd_dep_count#30)#64, avg(cd_dep_employed_count#31)#65, max(cd_dep_employed_count#31)#66, sum(cd_dep_employed_count#31)#67, avg(cd_dep_college_count#32)#68, max(cd_dep_college_count#32)#69, sum(cd_dep_college_count#32)#70]
+Results [18]: [ca_state#24, cd_gender#28, cd_marital_status#29, cd_dep_count#30, count(1)#61 AS cnt1#71, avg(cd_dep_count#30)#62 AS avg(cd_dep_count)#72, max(cd_dep_count#30)#63 AS max(cd_dep_count)#73, sum(cd_dep_count#30)#64 AS sum(cd_dep_count)#74, cd_dep_employed_count#31, count(1)#61 AS cnt2#75, avg(cd_dep_employed_count#31)#65 AS avg(cd_dep_employed_count)#76, max(cd_dep_employed_count#31)#66 AS max(cd_dep_employed_count)#77, sum(cd_dep_employed_count#31)#67 AS sum(cd_dep_employed_count)#78, cd_dep_college_count#32, count(1)#61 AS cnt3#79, avg(cd_dep_college_count#32)#68 AS avg(cd_dep_college_count)#80, max(cd_dep_college_count#32)#69 AS max(cd_dep_college_count)#81, sum(cd_dep_college_count#32)#70 AS sum(cd_dep_college_count)#82]
+
+(54) TakeOrderedAndProject
+Input [18]: [ca_state#24, cd_gender#28, cd_marital_status#29, cd_dep_count#30, cnt1#71, avg(cd_dep_count)#72, max(cd_dep_count)#73, sum(cd_dep_count)#74, cd_dep_employed_count#31, cnt2#75, avg(cd_dep_employed_count)#76, max(cd_dep_employed_count)#77, sum(cd_dep_employed_count)#78, cd_dep_college_count#32, cnt3#79, avg(cd_dep_college_count)#80, max(cd_dep_college_count)#81, sum(cd_dep_college_count)#82]
+Arguments: 100, [ca_state#24 ASC NULLS FIRST, cd_gender#28 ASC NULLS FIRST, cd_marital_status#29 ASC NULLS FIRST, cd_dep_count#30 ASC NULLS FIRST, cd_dep_employed_count#31 ASC NULLS FIRST, cd_dep_college_count#32 ASC NULLS FIRST], [ca_state#24, cd_gender#28, cd_marital_status#29, cd_dep_count#30, cnt1#71, avg(cd_dep_count)#72, max(cd_dep_count)#73, sum(cd_dep_count)#74, cd_dep_employed_count#31, cnt2#75, avg(cd_dep_employed_count)#76, max(cd_dep_employed_count)#77, sum(cd_dep_employed_count)#78, cd_dep_college_count#32, cnt3#79, avg(cd_dep_college_count)#80, max(cd_dep_college_count)#81, sum(cd_dep_college_count)#82]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 6 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7
+ReusedExchange (55)
+
+
+(55) ReusedExchange [Reuses operator id: 12]
+Output [1]: [d_date_sk#8]
+
+Subquery:2 Hosting operator id = 18 Hosting Expression = ws_sold_date_sk#14 IN dynamicpruning#7
+
+Subquery:3 Hosting operator id = 23 Hosting Expression = cs_sold_date_sk#18 IN dynamicpruning#7
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.sf100/simplified.txt
new file mode 100644
index 0000000000000..840b465af0747
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.sf100/simplified.txt
@@ -0,0 +1,103 @@
+TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)]
+  WholeStageCodegen (21)
+    HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] [count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum]
+      InputAdapter
+        Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1
+          WholeStageCodegen (20)
+            HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum]
+              Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count]
+                SortMergeJoin [c_current_cdemo_sk,cd_demo_sk]
+                  InputAdapter
+                    WholeStageCodegen (17)
+                      Sort [c_current_cdemo_sk]
+                        InputAdapter
+                          Exchange [c_current_cdemo_sk] #2
+                            WholeStageCodegen (16)
+                              Project [c_current_cdemo_sk,ca_state]
+                                SortMergeJoin [c_current_addr_sk,ca_address_sk]
+                                  InputAdapter
+                                    WholeStageCodegen (13)
+                                      Sort [c_current_addr_sk]
+                                        InputAdapter
+                                          Exchange [c_current_addr_sk] #3
+                                            WholeStageCodegen (12)
+                                              Project [c_current_cdemo_sk,c_current_addr_sk]
+                                                SortMergeJoin [c_customer_sk,customsk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (6)
+                                                      SortMergeJoin [c_customer_sk,ss_customer_sk]
+                                                        InputAdapter
+                                                          WholeStageCodegen (2)
+                                                            Sort [c_customer_sk]
+                                                              InputAdapter
+                                                                Exchange [c_customer_sk] #4
+                                                                  WholeStageCodegen (1)
+                                                                    Filter [c_current_addr_sk,c_current_cdemo_sk]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk]
+                                                        InputAdapter
+                                                          WholeStageCodegen (5)
+                                                            Sort [ss_customer_sk]
+                                                              InputAdapter
+                                                                Exchange [ss_customer_sk] #5
+                                                                  WholeStageCodegen (4)
+                                                                    Project [ss_customer_sk]
+                                                                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.store_sales [ss_customer_sk,ss_sold_date_sk]
+                                                                              SubqueryBroadcast [d_date_sk] #1
+                                                                                ReusedExchange [d_date_sk] #6
+                                                                        InputAdapter
+                                                                          BroadcastExchange #6
+                                                                            WholeStageCodegen (3)
+                                                                              Project [d_date_sk]
+                                                                                Filter [d_year,d_qoy,d_date_sk]
+                                                                                  ColumnarToRow
+                                                                                    InputAdapter
+                                                                                      Scan parquet default.date_dim [d_date_sk,d_year,d_qoy]
+                                                  InputAdapter
+                                                    WholeStageCodegen (11)
+                                                      Sort [customsk]
+                                                        InputAdapter
+                                                          Exchange [customsk] #7
+                                                            Union
+                                                              WholeStageCodegen (8)
+                                                                Project [ws_bill_customer_sk]
+                                                                  BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.web_sales [ws_bill_customer_sk,ws_sold_date_sk]
+                                                                          ReusedSubquery [d_date_sk] #1
+                                                                    InputAdapter
+                                                                      ReusedExchange [d_date_sk] #6
+                                                              WholeStageCodegen (10)
+                                                                Project [cs_ship_customer_sk]
+                                                                  BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk]
+                                                                          ReusedSubquery [d_date_sk] #1
+                                                                    InputAdapter
+                                                                      ReusedExchange [d_date_sk] #6
+                                  InputAdapter
+                                    WholeStageCodegen (15)
+                                      Sort [ca_address_sk]
+                                        InputAdapter
+                                          Exchange [ca_address_sk] #8
+                                            WholeStageCodegen (14)
+                                              Filter [ca_address_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.customer_address [ca_address_sk,ca_state]
+                  InputAdapter
+                    WholeStageCodegen (19)
+                      Sort [cd_demo_sk]
+                        InputAdapter
+                          Exchange [cd_demo_sk] #9
+                            WholeStageCodegen (18)
+                              Filter [cd_demo_sk]
+                                ColumnarToRow
+                                  InputAdapter
+                                    Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/explain.txt
new file mode 100644
index 0000000000000..57710f427f0f8
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/explain.txt
@@ -0,0 +1,260 @@
+== Physical Plan ==
+TakeOrderedAndProject (44)
++- * HashAggregate (43)
+   +- Exchange (42)
+      +- * HashAggregate (41)
+         +- * Project (40)
+            +- * BroadcastHashJoin Inner BuildRight (39)
+               :- * Project (34)
+               :  +- * BroadcastHashJoin Inner BuildRight (33)
+               :     :- * Project (28)
+               :     :  +- * BroadcastHashJoin LeftSemi BuildRight (27)
+               :     :     :- * BroadcastHashJoin LeftSemi BuildRight (14)
+               :     :     :  :- * Filter (3)
+               :     :     :  :  +- * ColumnarToRow (2)
+               :     :     :  :     +- Scan parquet default.customer (1)
+               :     :     :  +- BroadcastExchange (13)
+               :     :     :     +- * Project (12)
+               :     :     :        +- * BroadcastHashJoin Inner BuildRight (11)
+               :     :     :           :- * ColumnarToRow (5)
+               :     :     :           :  +- Scan parquet default.store_sales (4)
+               :     :     :           +- BroadcastExchange (10)
+               :     :     :              +- * Project (9)
+               :     :     :                 +- * Filter (8)
+               :     :     :                    +- * ColumnarToRow (7)
+               :     :     :                       +- Scan parquet default.date_dim (6)
+               :     :     +- BroadcastExchange (26)
+               :     :        +- Union (25)
+               :     :           :- * Project (19)
+               :     :           :  +- * BroadcastHashJoin Inner BuildRight (18)
+               :     :           :     :- * ColumnarToRow (16)
+               :     :           :     :  +- Scan parquet default.web_sales (15)
+               :     :           :     +- ReusedExchange (17)
+               :     :           +- * Project (24)
+               :     :              +- * BroadcastHashJoin Inner BuildRight (23)
+               :     :                 :- * ColumnarToRow (21)
+               :     :                 :  +- Scan parquet default.catalog_sales (20)
+               :     :                 +- ReusedExchange (22)
+               :     +- BroadcastExchange (32)
+               :        +- * Filter (31)
+               :           +- * ColumnarToRow (30)
+               :              +- Scan parquet default.customer_address (29)
+               +- BroadcastExchange (38)
+                  +- * Filter (37)
+                     +- * ColumnarToRow (36)
+                        +- Scan parquet default.customer_demographics (35)
+
+
+(1) Scan parquet default.customer
+Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_addr_sk:int>
+
+(2) ColumnarToRow [codegen id : 9]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+
+(3) Filter [codegen id : 9]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2))
+
+(4) Scan parquet default.store_sales
+Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)]
+ReadSchema: struct<ss_customer_sk:int>
+
+(5) ColumnarToRow [codegen id : 2]
+Input [2]: [ss_customer_sk#4, ss_sold_date_sk#5]
+
+(6) Scan parquet default.date_dim
+Output [3]: [d_date_sk#7, d_year#8, d_qoy#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,1999), LessThan(d_qoy,4), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_qoy:int>
+
+(7) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_qoy#9]
+
+(8) Filter [codegen id : 1]
+Input [3]: [d_date_sk#7, d_year#8, d_qoy#9]
+Condition : ((((isnotnull(d_year#8) AND isnotnull(d_qoy#9)) AND (d_year#8 = 1999)) AND (d_qoy#9 < 4)) AND isnotnull(d_date_sk#7))
+
+(9) Project [codegen id : 1]
+Output [1]: [d_date_sk#7]
+Input [3]: [d_date_sk#7, d_year#8, d_qoy#9]
+
+(10) BroadcastExchange
+Input [1]: [d_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#10]
+
+(11) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#5]
+Right keys [1]: [d_date_sk#7]
+Join condition: None
+
+(12) Project [codegen id : 2]
+Output [1]: [ss_customer_sk#4]
+Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#7]
+
+(13) BroadcastExchange
+Input [1]: [ss_customer_sk#4]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#11]
+
+(14) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [ss_customer_sk#4]
+Join condition: None
+
+(15) Scan parquet default.web_sales
+Output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#13), dynamicpruningexpression(ws_sold_date_sk#13 IN dynamicpruning#6)]
+ReadSchema: struct<ws_bill_customer_sk:int>
+
+(16) ColumnarToRow [codegen id : 4]
+Input [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13]
+
+(17) ReusedExchange [Reuses operator id: 10]
+Output [1]: [d_date_sk#14]
+
+(18) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ws_sold_date_sk#13]
+Right keys [1]: [d_date_sk#14]
+Join condition: None
+
+(19) Project [codegen id : 4]
+Output [1]: [ws_bill_customer_sk#12 AS customsk#15]
+Input [3]: [ws_bill_customer_sk#12, ws_sold_date_sk#13, d_date_sk#14]
+
+(20) Scan parquet default.catalog_sales
+Output [2]: [cs_ship_customer_sk#16, cs_sold_date_sk#17]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#17), dynamicpruningexpression(cs_sold_date_sk#17 IN dynamicpruning#6)]
+ReadSchema: struct<cs_ship_customer_sk:int>
+
+(21) ColumnarToRow [codegen id : 6]
+Input [2]: [cs_ship_customer_sk#16, cs_sold_date_sk#17]
+
+(22) ReusedExchange [Reuses operator id: 10]
+Output [1]: [d_date_sk#18]
+
+(23) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [cs_sold_date_sk#17]
+Right keys [1]: [d_date_sk#18]
+Join condition: None
+
+(24) Project [codegen id : 6]
+Output [1]: [cs_ship_customer_sk#16 AS customsk#19]
+Input [3]: [cs_ship_customer_sk#16, cs_sold_date_sk#17, d_date_sk#18]
+
+(25) Union
+
+(26) BroadcastExchange
+Input [1]: [customsk#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#20]
+
+(27) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [customsk#15]
+Join condition: None
+
+(28) Project [codegen id : 9]
+Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3]
+Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3]
+
+(29) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#21, ca_state#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(30) ColumnarToRow [codegen id : 7]
+Input [2]: [ca_address_sk#21, ca_state#22]
+
+(31) Filter [codegen id : 7]
+Input [2]: [ca_address_sk#21, ca_state#22]
+Condition : isnotnull(ca_address_sk#21)
+
+(32) BroadcastExchange
+Input [2]: [ca_address_sk#21, ca_state#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#23]
+
+(33) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_current_addr_sk#3]
+Right keys [1]: [ca_address_sk#21]
+Join condition: None
+
+(34) Project [codegen id : 9]
+Output [2]: [c_current_cdemo_sk#2, ca_state#22]
+Input [4]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#21, ca_state#22]
+
+(35) Scan parquet default.customer_demographics
+Output [6]: [cd_demo_sk#24, cd_gender#25, cd_marital_status#26, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_gender:string,cd_marital_status:string,cd_dep_count:int,cd_dep_employed_count:int,cd_dep_college_count:int>
+
+(36) ColumnarToRow [codegen id : 8]
+Input [6]: [cd_demo_sk#24, cd_gender#25, cd_marital_status#26, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29]
+
+(37) Filter [codegen id : 8]
+Input [6]: [cd_demo_sk#24, cd_gender#25, cd_marital_status#26, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29]
+Condition : isnotnull(cd_demo_sk#24)
+
+(38) BroadcastExchange
+Input [6]: [cd_demo_sk#24, cd_gender#25, cd_marital_status#26, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#30]
+
+(39) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [c_current_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#24]
+Join condition: None
+
+(40) Project [codegen id : 9]
+Output [6]: [ca_state#22, cd_gender#25, cd_marital_status#26, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29]
+Input [8]: [c_current_cdemo_sk#2, ca_state#22, cd_demo_sk#24, cd_gender#25, cd_marital_status#26, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29]
+
+(41) HashAggregate [codegen id : 9]
+Input [6]: [ca_state#22, cd_gender#25, cd_marital_status#26, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29]
+Keys [6]: [ca_state#22, cd_gender#25, cd_marital_status#26, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29]
+Functions [10]: [partial_count(1), partial_avg(cd_dep_count#27), partial_max(cd_dep_count#27), partial_sum(cd_dep_count#27), partial_avg(cd_dep_employed_count#28), partial_max(cd_dep_employed_count#28), partial_sum(cd_dep_employed_count#28), partial_avg(cd_dep_college_count#29), partial_max(cd_dep_college_count#29), partial_sum(cd_dep_college_count#29)]
+Aggregate Attributes [13]: [count#31, sum#32, count#33, max#34, sum#35, sum#36, count#37, max#38, sum#39, sum#40, count#41, max#42, sum#43]
+Results [19]: [ca_state#22, cd_gender#25, cd_marital_status#26, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29, count#44, sum#45, count#46, max#47, sum#48, sum#49, count#50, max#51, sum#52, sum#53, count#54, max#55, sum#56]
+
+(42) Exchange
+Input [19]: [ca_state#22, cd_gender#25, cd_marital_status#26, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29, count#44, sum#45, count#46, max#47, sum#48, sum#49, count#50, max#51, sum#52, sum#53, count#54, max#55, sum#56]
+Arguments: hashpartitioning(ca_state#22, cd_gender#25, cd_marital_status#26, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29, 5), ENSURE_REQUIREMENTS, [id=#57]
+
+(43) HashAggregate [codegen id : 10]
+Input [19]: [ca_state#22, cd_gender#25, cd_marital_status#26, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29, count#44, sum#45, count#46, max#47, sum#48, sum#49, count#50, max#51, sum#52, sum#53, count#54, max#55, sum#56]
+Keys [6]: [ca_state#22, cd_gender#25, cd_marital_status#26, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29]
+Functions [10]: [count(1), avg(cd_dep_count#27), max(cd_dep_count#27), sum(cd_dep_count#27), avg(cd_dep_employed_count#28), max(cd_dep_employed_count#28), sum(cd_dep_employed_count#28), avg(cd_dep_college_count#29), max(cd_dep_college_count#29), sum(cd_dep_college_count#29)]
+Aggregate Attributes [10]: [count(1)#58, avg(cd_dep_count#27)#59, max(cd_dep_count#27)#60, sum(cd_dep_count#27)#61, avg(cd_dep_employed_count#28)#62, max(cd_dep_employed_count#28)#63, sum(cd_dep_employed_count#28)#64, avg(cd_dep_college_count#29)#65, max(cd_dep_college_count#29)#66, sum(cd_dep_college_count#29)#67]
+Results [18]: [ca_state#22, cd_gender#25, cd_marital_status#26, cd_dep_count#27, count(1)#58 AS cnt1#68, avg(cd_dep_count#27)#59 AS avg(cd_dep_count)#69, max(cd_dep_count#27)#60 AS max(cd_dep_count)#70, sum(cd_dep_count#27)#61 AS sum(cd_dep_count)#71, cd_dep_employed_count#28, count(1)#58 AS cnt2#72, avg(cd_dep_employed_count#28)#62 AS avg(cd_dep_employed_count)#73, max(cd_dep_employed_count#28)#63 AS max(cd_dep_employed_count)#74, sum(cd_dep_employed_count#28)#64 AS sum(cd_dep_employed_count)#75, cd_dep_college_count#29, count(1)#58 AS cnt3#76, avg(cd_dep_college_count#29)#65 AS avg(cd_dep_college_count)#77, max(cd_dep_college_count#29)#66 AS max(cd_dep_college_count)#78, sum(cd_dep_college_count#29)#67 AS sum(cd_dep_college_count)#79]
+
+(44) TakeOrderedAndProject
+Input [18]: [ca_state#22, cd_gender#25, cd_marital_status#26, cd_dep_count#27, cnt1#68, avg(cd_dep_count)#69, max(cd_dep_count)#70, sum(cd_dep_count)#71, cd_dep_employed_count#28, cnt2#72, avg(cd_dep_employed_count)#73, max(cd_dep_employed_count)#74, sum(cd_dep_employed_count)#75, cd_dep_college_count#29, cnt3#76, avg(cd_dep_college_count)#77, max(cd_dep_college_count)#78, sum(cd_dep_college_count)#79]
+Arguments: 100, [ca_state#22 ASC NULLS FIRST, cd_gender#25 ASC NULLS FIRST, cd_marital_status#26 ASC NULLS FIRST, cd_dep_count#27 ASC NULLS FIRST, cd_dep_employed_count#28 ASC NULLS FIRST, cd_dep_college_count#29 ASC NULLS FIRST], [ca_state#22, cd_gender#25, cd_marital_status#26, cd_dep_count#27, cnt1#68, avg(cd_dep_count)#69, max(cd_dep_count)#70, sum(cd_dep_count)#71, cd_dep_employed_count#28, cnt2#72, avg(cd_dep_employed_count)#73, max(cd_dep_employed_count)#74, sum(cd_dep_employed_count)#75, cd_dep_college_count#29, cnt3#76, avg(cd_dep_college_count)#77, max(cd_dep_college_count)#78, sum(cd_dep_college_count)#79]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (45)
+
+
+(45) ReusedExchange [Reuses operator id: 10]
+Output [1]: [d_date_sk#7]
+
+Subquery:2 Hosting operator id = 15 Hosting Expression = ws_sold_date_sk#13 IN dynamicpruning#6
+
+Subquery:3 Hosting operator id = 20 Hosting Expression = cs_sold_date_sk#17 IN dynamicpruning#6
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/simplified.txt
new file mode 100644
index 0000000000000..1ab7d548e59dd
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/simplified.txt
@@ -0,0 +1,71 @@
+TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)]
+  WholeStageCodegen (10)
+    HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] [count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum]
+      InputAdapter
+        Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1
+          WholeStageCodegen (9)
+            HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum]
+              Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count]
+                BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk]
+                  Project [c_current_cdemo_sk,ca_state]
+                    BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                      Project [c_current_cdemo_sk,c_current_addr_sk]
+                        BroadcastHashJoin [c_customer_sk,customsk]
+                          BroadcastHashJoin [c_customer_sk,ss_customer_sk]
+                            Filter [c_current_addr_sk,c_current_cdemo_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk]
+                            InputAdapter
+                              BroadcastExchange #2
+                                WholeStageCodegen (2)
+                                  Project [ss_customer_sk]
+                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.store_sales [ss_customer_sk,ss_sold_date_sk]
+                                            SubqueryBroadcast [d_date_sk] #1
+                                              ReusedExchange [d_date_sk] #3
+                                      InputAdapter
+                                        BroadcastExchange #3
+                                          WholeStageCodegen (1)
+                                            Project [d_date_sk]
+                                              Filter [d_year,d_qoy,d_date_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_date_sk,d_year,d_qoy]
+                          InputAdapter
+                            BroadcastExchange #4
+                              Union
+                                WholeStageCodegen (4)
+                                  Project [ws_bill_customer_sk]
+                                    BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.web_sales [ws_bill_customer_sk,ws_sold_date_sk]
+                                            ReusedSubquery [d_date_sk] #1
+                                      InputAdapter
+                                        ReusedExchange [d_date_sk] #3
+                                WholeStageCodegen (6)
+                                  Project [cs_ship_customer_sk]
+                                    BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk]
+                                            ReusedSubquery [d_date_sk] #1
+                                      InputAdapter
+                                        ReusedExchange [d_date_sk] #3
+                      InputAdapter
+                        BroadcastExchange #5
+                          WholeStageCodegen (7)
+                            Filter [ca_address_sk]
+                              ColumnarToRow
+                                InputAdapter
+                                  Scan parquet default.customer_address [ca_address_sk,ca_state]
+                  InputAdapter
+                    BroadcastExchange #6
+                      WholeStageCodegen (8)
+                        Filter [cd_demo_sk]
+                          ColumnarToRow
+                            InputAdapter
+                              Scan parquet default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.sf100/explain.txt
new file mode 100644
index 0000000000000..79037befbd6b5
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.sf100/explain.txt
@@ -0,0 +1,276 @@
+== Physical Plan ==
+TakeOrderedAndProject (45)
++- * Project (44)
+   +- Window (43)
+      +- * Sort (42)
+         +- Exchange (41)
+            +- * HashAggregate (40)
+               +- Exchange (39)
+                  +- * HashAggregate (38)
+                     +- Union (37)
+                        :- * HashAggregate (26)
+                        :  +- Exchange (25)
+                        :     +- * HashAggregate (24)
+                        :        +- * Project (23)
+                        :           +- * BroadcastHashJoin Inner BuildRight (22)
+                        :              :- * Project (17)
+                        :              :  +- * BroadcastHashJoin Inner BuildRight (16)
+                        :              :     :- * Project (10)
+                        :              :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+                        :              :     :     :- * Filter (3)
+                        :              :     :     :  +- * ColumnarToRow (2)
+                        :              :     :     :     +- Scan parquet default.store_sales (1)
+                        :              :     :     +- BroadcastExchange (8)
+                        :              :     :        +- * Project (7)
+                        :              :     :           +- * Filter (6)
+                        :              :     :              +- * ColumnarToRow (5)
+                        :              :     :                 +- Scan parquet default.date_dim (4)
+                        :              :     +- BroadcastExchange (15)
+                        :              :        +- * Project (14)
+                        :              :           +- * Filter (13)
+                        :              :              +- * ColumnarToRow (12)
+                        :              :                 +- Scan parquet default.store (11)
+                        :              +- BroadcastExchange (21)
+                        :                 +- * Filter (20)
+                        :                    +- * ColumnarToRow (19)
+                        :                       +- Scan parquet default.item (18)
+                        :- * HashAggregate (31)
+                        :  +- Exchange (30)
+                        :     +- * HashAggregate (29)
+                        :        +- * HashAggregate (28)
+                        :           +- ReusedExchange (27)
+                        +- * HashAggregate (36)
+                           +- Exchange (35)
+                              +- * HashAggregate (34)
+                                 +- * HashAggregate (33)
+                                    +- ReusedExchange (32)
+
+
+(1) Scan parquet default.store_sales
+Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_ext_sales_price:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5]
+
+(3) Filter [codegen id : 4]
+Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5]
+Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2))
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#7, d_year#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#7, d_year#8]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#7, d_year#8]
+Condition : ((isnotnull(d_year#8) AND (d_year#8 = 2001)) AND isnotnull(d_date_sk#7))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#7]
+Input [2]: [d_date_sk#7, d_year#8]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#9]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#5]
+Right keys [1]: [d_date_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4]
+Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#7]
+
+(11) Scan parquet default.store
+Output [2]: [s_store_sk#10, s_state#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_state:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [s_store_sk#10, s_state#11]
+
+(13) Filter [codegen id : 2]
+Input [2]: [s_store_sk#10, s_state#11]
+Condition : ((isnotnull(s_state#11) AND (s_state#11 = TN)) AND isnotnull(s_store_sk#10))
+
+(14) Project [codegen id : 2]
+Output [1]: [s_store_sk#10]
+Input [2]: [s_store_sk#10, s_state#11]
+
+(15) BroadcastExchange
+Input [1]: [s_store_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#2]
+Right keys [1]: [s_store_sk#10]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [3]: [ss_item_sk#1, ss_ext_sales_price#3, ss_net_profit#4]
+Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, s_store_sk#10]
+
+(18) Scan parquet default.item
+Output [3]: [i_item_sk#13, i_class#14, i_category#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_class:string,i_category:string>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [3]: [i_item_sk#13, i_class#14, i_category#15]
+
+(20) Filter [codegen id : 3]
+Input [3]: [i_item_sk#13, i_class#14, i_category#15]
+Condition : isnotnull(i_item_sk#13)
+
+(21) BroadcastExchange
+Input [3]: [i_item_sk#13, i_class#14, i_category#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#16]
+
+(22) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#13]
+Join condition: None
+
+(23) Project [codegen id : 4]
+Output [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#14, i_category#15]
+Input [6]: [ss_item_sk#1, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#13, i_class#14, i_category#15]
+
+(24) HashAggregate [codegen id : 4]
+Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#14, i_category#15]
+Keys [2]: [i_category#15, i_class#14]
+Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))]
+Aggregate Attributes [2]: [sum#17, sum#18]
+Results [4]: [i_category#15, i_class#14, sum#19, sum#20]
+
+(25) Exchange
+Input [4]: [i_category#15, i_class#14, sum#19, sum#20]
+Arguments: hashpartitioning(i_category#15, i_class#14, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(26) HashAggregate [codegen id : 5]
+Input [4]: [i_category#15, i_class#14, sum#19, sum#20]
+Keys [2]: [i_category#15, i_class#14]
+Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#22, sum(UnscaledValue(ss_ext_sales_price#3))#23]
+Results [6]: [cast(CheckOverflow((promote_precision(MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#22,17,2)) / promote_precision(MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#23,17,2))), DecimalType(37,20), true) as decimal(38,20)) AS gross_margin#24, i_category#15, i_class#14, 0 AS t_category#25, 0 AS t_class#26, 0 AS lochierarchy#27]
+
+(27) ReusedExchange [Reuses operator id: unknown]
+Output [4]: [i_category#28, i_class#29, sum#30, sum#31]
+
+(28) HashAggregate [codegen id : 10]
+Input [4]: [i_category#28, i_class#29, sum#30, sum#31]
+Keys [2]: [i_category#28, i_class#29]
+Functions [2]: [sum(UnscaledValue(ss_net_profit#32)), sum(UnscaledValue(ss_ext_sales_price#33))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#32))#34, sum(UnscaledValue(ss_ext_sales_price#33))#35]
+Results [3]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#32))#34,17,2) AS ss_net_profit#36, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#33))#35,17,2) AS ss_ext_sales_price#37, i_category#28]
+
+(29) HashAggregate [codegen id : 10]
+Input [3]: [ss_net_profit#36, ss_ext_sales_price#37, i_category#28]
+Keys [1]: [i_category#28]
+Functions [2]: [partial_sum(ss_net_profit#36), partial_sum(ss_ext_sales_price#37)]
+Aggregate Attributes [4]: [sum#38, isEmpty#39, sum#40, isEmpty#41]
+Results [5]: [i_category#28, sum#42, isEmpty#43, sum#44, isEmpty#45]
+
+(30) Exchange
+Input [5]: [i_category#28, sum#42, isEmpty#43, sum#44, isEmpty#45]
+Arguments: hashpartitioning(i_category#28, 5), ENSURE_REQUIREMENTS, [id=#46]
+
+(31) HashAggregate [codegen id : 11]
+Input [5]: [i_category#28, sum#42, isEmpty#43, sum#44, isEmpty#45]
+Keys [1]: [i_category#28]
+Functions [2]: [sum(ss_net_profit#36), sum(ss_ext_sales_price#37)]
+Aggregate Attributes [2]: [sum(ss_net_profit#36)#47, sum(ss_ext_sales_price#37)#48]
+Results [6]: [cast(CheckOverflow((promote_precision(sum(ss_net_profit#36)#47) / promote_precision(sum(ss_ext_sales_price#37)#48)), DecimalType(38,11), true) as decimal(38,20)) AS gross_margin#49, i_category#28, null AS i_class#50, 0 AS t_category#51, 1 AS t_class#52, 1 AS lochierarchy#53]
+
+(32) ReusedExchange [Reuses operator id: unknown]
+Output [4]: [i_category#54, i_class#55, sum#56, sum#57]
+
+(33) HashAggregate [codegen id : 16]
+Input [4]: [i_category#54, i_class#55, sum#56, sum#57]
+Keys [2]: [i_category#54, i_class#55]
+Functions [2]: [sum(UnscaledValue(ss_net_profit#58)), sum(UnscaledValue(ss_ext_sales_price#59))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#58))#60, sum(UnscaledValue(ss_ext_sales_price#59))#61]
+Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#58))#60,17,2) AS ss_net_profit#36, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#59))#61,17,2) AS ss_ext_sales_price#37]
+
+(34) HashAggregate [codegen id : 16]
+Input [2]: [ss_net_profit#36, ss_ext_sales_price#37]
+Keys: []
+Functions [2]: [partial_sum(ss_net_profit#36), partial_sum(ss_ext_sales_price#37)]
+Aggregate Attributes [4]: [sum#62, isEmpty#63, sum#64, isEmpty#65]
+Results [4]: [sum#66, isEmpty#67, sum#68, isEmpty#69]
+
+(35) Exchange
+Input [4]: [sum#66, isEmpty#67, sum#68, isEmpty#69]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#70]
+
+(36) HashAggregate [codegen id : 17]
+Input [4]: [sum#66, isEmpty#67, sum#68, isEmpty#69]
+Keys: []
+Functions [2]: [sum(ss_net_profit#36), sum(ss_ext_sales_price#37)]
+Aggregate Attributes [2]: [sum(ss_net_profit#36)#71, sum(ss_ext_sales_price#37)#72]
+Results [6]: [cast(CheckOverflow((promote_precision(sum(ss_net_profit#36)#71) / promote_precision(sum(ss_ext_sales_price#37)#72)), DecimalType(38,11), true) as decimal(38,20)) AS gross_margin#73, null AS i_category#74, null AS i_class#75, 1 AS t_category#76, 1 AS t_class#77, 2 AS lochierarchy#78]
+
+(37) Union
+
+(38) HashAggregate [codegen id : 18]
+Input [6]: [gross_margin#24, i_category#15, i_class#14, t_category#25, t_class#26, lochierarchy#27]
+Keys [6]: [gross_margin#24, i_category#15, i_class#14, t_category#25, t_class#26, lochierarchy#27]
+Functions: []
+Aggregate Attributes: []
+Results [6]: [gross_margin#24, i_category#15, i_class#14, t_category#25, t_class#26, lochierarchy#27]
+
+(39) Exchange
+Input [6]: [gross_margin#24, i_category#15, i_class#14, t_category#25, t_class#26, lochierarchy#27]
+Arguments: hashpartitioning(gross_margin#24, i_category#15, i_class#14, t_category#25, t_class#26, lochierarchy#27, 5), ENSURE_REQUIREMENTS, [id=#79]
+
+(40) HashAggregate [codegen id : 19]
+Input [6]: [gross_margin#24, i_category#15, i_class#14, t_category#25, t_class#26, lochierarchy#27]
+Keys [6]: [gross_margin#24, i_category#15, i_class#14, t_category#25, t_class#26, lochierarchy#27]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [gross_margin#24, i_category#15, i_class#14, lochierarchy#27, CASE WHEN (t_class#26 = 0) THEN i_category#15 END AS _w0#80]
+
+(41) Exchange
+Input [5]: [gross_margin#24, i_category#15, i_class#14, lochierarchy#27, _w0#80]
+Arguments: hashpartitioning(lochierarchy#27, _w0#80, 5), ENSURE_REQUIREMENTS, [id=#81]
+
+(42) Sort [codegen id : 20]
+Input [5]: [gross_margin#24, i_category#15, i_class#14, lochierarchy#27, _w0#80]
+Arguments: [lochierarchy#27 ASC NULLS FIRST, _w0#80 ASC NULLS FIRST, gross_margin#24 ASC NULLS FIRST], false, 0
+
+(43) Window
+Input [5]: [gross_margin#24, i_category#15, i_class#14, lochierarchy#27, _w0#80]
+Arguments: [rank(gross_margin#24) windowspecdefinition(lochierarchy#27, _w0#80, gross_margin#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#82], [lochierarchy#27, _w0#80], [gross_margin#24 ASC NULLS FIRST]
+
+(44) Project [codegen id : 21]
+Output [5]: [gross_margin#24, i_category#15, i_class#14, lochierarchy#27, rank_within_parent#82]
+Input [6]: [gross_margin#24, i_category#15, i_class#14, lochierarchy#27, _w0#80, rank_within_parent#82]
+
+(45) TakeOrderedAndProject
+Input [5]: [gross_margin#24, i_category#15, i_class#14, lochierarchy#27, rank_within_parent#82]
+Arguments: 100, [lochierarchy#27 DESC NULLS LAST, CASE WHEN (lochierarchy#27 = 0) THEN i_category#15 END ASC NULLS FIRST, rank_within_parent#82 ASC NULLS FIRST], [gross_margin#24, i_category#15, i_class#14, lochierarchy#27, rank_within_parent#82]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (46)
+
+
+(46) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#7]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.sf100/simplified.txt
new file mode 100644
index 0000000000000..3a32a26cb0a75
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.sf100/simplified.txt
@@ -0,0 +1,76 @@
+TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i_class]
+  WholeStageCodegen (21)
+    Project [gross_margin,i_category,i_class,lochierarchy,rank_within_parent]
+      InputAdapter
+        Window [gross_margin,lochierarchy,_w0]
+          WholeStageCodegen (20)
+            Sort [lochierarchy,_w0,gross_margin]
+              InputAdapter
+                Exchange [lochierarchy,_w0] #1
+                  WholeStageCodegen (19)
+                    HashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] [_w0]
+                      InputAdapter
+                        Exchange [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] #2
+                          WholeStageCodegen (18)
+                            HashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy]
+                              InputAdapter
+                                Union
+                                  WholeStageCodegen (5)
+                                    HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),gross_margin,t_category,t_class,lochierarchy,sum,sum]
+                                      InputAdapter
+                                        Exchange [i_category,i_class] #3
+                                          WholeStageCodegen (4)
+                                            HashAggregate [i_category,i_class,ss_net_profit,ss_ext_sales_price] [sum,sum,sum,sum]
+                                              Project [ss_ext_sales_price,ss_net_profit,i_class,i_category]
+                                                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                  Project [ss_item_sk,ss_ext_sales_price,ss_net_profit]
+                                                    BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                      Project [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit]
+                                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                          Filter [ss_item_sk,ss_store_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk]
+                                                                  SubqueryBroadcast [d_date_sk] #1
+                                                                    ReusedExchange [d_date_sk] #4
+                                                          InputAdapter
+                                                            BroadcastExchange #4
+                                                              WholeStageCodegen (1)
+                                                                Project [d_date_sk]
+                                                                  Filter [d_year,d_date_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.date_dim [d_date_sk,d_year]
+                                                      InputAdapter
+                                                        BroadcastExchange #5
+                                                          WholeStageCodegen (2)
+                                                            Project [s_store_sk]
+                                                              Filter [s_state,s_store_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.store [s_store_sk,s_state]
+                                                  InputAdapter
+                                                    BroadcastExchange #6
+                                                      WholeStageCodegen (3)
+                                                        Filter [i_item_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.item [i_item_sk,i_class,i_category]
+                                  WholeStageCodegen (11)
+                                    HashAggregate [i_category,sum,isEmpty,sum,isEmpty] [sum(ss_net_profit),sum(ss_ext_sales_price),gross_margin,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty]
+                                      InputAdapter
+                                        Exchange [i_category] #7
+                                          WholeStageCodegen (10)
+                                            HashAggregate [i_category,ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                                              HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),ss_net_profit,ss_ext_sales_price,sum,sum]
+                                                InputAdapter
+                                                  ReusedExchange [i_category,i_class,sum,sum] #8
+                                  WholeStageCodegen (17)
+                                    HashAggregate [sum,isEmpty,sum,isEmpty] [sum(ss_net_profit),sum(ss_ext_sales_price),gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty]
+                                      InputAdapter
+                                        Exchange #9
+                                          WholeStageCodegen (16)
+                                            HashAggregate [ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                                              HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),ss_net_profit,ss_ext_sales_price,sum,sum]
+                                                InputAdapter
+                                                  ReusedExchange [i_category,i_class,sum,sum] #8
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/explain.txt
new file mode 100644
index 0000000000000..bc2931129868b
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/explain.txt
@@ -0,0 +1,276 @@
+== Physical Plan ==
+TakeOrderedAndProject (45)
++- * Project (44)
+   +- Window (43)
+      +- * Sort (42)
+         +- Exchange (41)
+            +- * HashAggregate (40)
+               +- Exchange (39)
+                  +- * HashAggregate (38)
+                     +- Union (37)
+                        :- * HashAggregate (26)
+                        :  +- Exchange (25)
+                        :     +- * HashAggregate (24)
+                        :        +- * Project (23)
+                        :           +- * BroadcastHashJoin Inner BuildRight (22)
+                        :              :- * Project (16)
+                        :              :  +- * BroadcastHashJoin Inner BuildRight (15)
+                        :              :     :- * Project (10)
+                        :              :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+                        :              :     :     :- * Filter (3)
+                        :              :     :     :  +- * ColumnarToRow (2)
+                        :              :     :     :     +- Scan parquet default.store_sales (1)
+                        :              :     :     +- BroadcastExchange (8)
+                        :              :     :        +- * Project (7)
+                        :              :     :           +- * Filter (6)
+                        :              :     :              +- * ColumnarToRow (5)
+                        :              :     :                 +- Scan parquet default.date_dim (4)
+                        :              :     +- BroadcastExchange (14)
+                        :              :        +- * Filter (13)
+                        :              :           +- * ColumnarToRow (12)
+                        :              :              +- Scan parquet default.item (11)
+                        :              +- BroadcastExchange (21)
+                        :                 +- * Project (20)
+                        :                    +- * Filter (19)
+                        :                       +- * ColumnarToRow (18)
+                        :                          +- Scan parquet default.store (17)
+                        :- * HashAggregate (31)
+                        :  +- Exchange (30)
+                        :     +- * HashAggregate (29)
+                        :        +- * HashAggregate (28)
+                        :           +- ReusedExchange (27)
+                        +- * HashAggregate (36)
+                           +- Exchange (35)
+                              +- * HashAggregate (34)
+                                 +- * HashAggregate (33)
+                                    +- ReusedExchange (32)
+
+
+(1) Scan parquet default.store_sales
+Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_ext_sales_price:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5]
+
+(3) Filter [codegen id : 4]
+Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5]
+Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2))
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#7, d_year#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#7, d_year#8]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#7, d_year#8]
+Condition : ((isnotnull(d_year#8) AND (d_year#8 = 2001)) AND isnotnull(d_date_sk#7))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#7]
+Input [2]: [d_date_sk#7, d_year#8]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#9]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#5]
+Right keys [1]: [d_date_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4]
+Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#7]
+
+(11) Scan parquet default.item
+Output [3]: [i_item_sk#10, i_class#11, i_category#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_class:string,i_category:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [i_item_sk#10, i_class#11, i_category#12]
+
+(13) Filter [codegen id : 2]
+Input [3]: [i_item_sk#10, i_class#11, i_category#12]
+Condition : isnotnull(i_item_sk#10)
+
+(14) BroadcastExchange
+Input [3]: [i_item_sk#10, i_class#11, i_category#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#13]
+
+(15) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#10]
+Join condition: None
+
+(16) Project [codegen id : 4]
+Output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12]
+Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#10, i_class#11, i_category#12]
+
+(17) Scan parquet default.store
+Output [2]: [s_store_sk#14, s_state#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_state:string>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [2]: [s_store_sk#14, s_state#15]
+
+(19) Filter [codegen id : 3]
+Input [2]: [s_store_sk#14, s_state#15]
+Condition : ((isnotnull(s_state#15) AND (s_state#15 = TN)) AND isnotnull(s_store_sk#14))
+
+(20) Project [codegen id : 3]
+Output [1]: [s_store_sk#14]
+Input [2]: [s_store_sk#14, s_state#15]
+
+(21) BroadcastExchange
+Input [1]: [s_store_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#16]
+
+(22) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#2]
+Right keys [1]: [s_store_sk#14]
+Join condition: None
+
+(23) Project [codegen id : 4]
+Output [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12]
+Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12, s_store_sk#14]
+
+(24) HashAggregate [codegen id : 4]
+Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12]
+Keys [2]: [i_category#12, i_class#11]
+Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))]
+Aggregate Attributes [2]: [sum#17, sum#18]
+Results [4]: [i_category#12, i_class#11, sum#19, sum#20]
+
+(25) Exchange
+Input [4]: [i_category#12, i_class#11, sum#19, sum#20]
+Arguments: hashpartitioning(i_category#12, i_class#11, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(26) HashAggregate [codegen id : 5]
+Input [4]: [i_category#12, i_class#11, sum#19, sum#20]
+Keys [2]: [i_category#12, i_class#11]
+Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#22, sum(UnscaledValue(ss_ext_sales_price#3))#23]
+Results [6]: [cast(CheckOverflow((promote_precision(MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#22,17,2)) / promote_precision(MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#23,17,2))), DecimalType(37,20), true) as decimal(38,20)) AS gross_margin#24, i_category#12, i_class#11, 0 AS t_category#25, 0 AS t_class#26, 0 AS lochierarchy#27]
+
+(27) ReusedExchange [Reuses operator id: unknown]
+Output [4]: [i_category#28, i_class#29, sum#30, sum#31]
+
+(28) HashAggregate [codegen id : 10]
+Input [4]: [i_category#28, i_class#29, sum#30, sum#31]
+Keys [2]: [i_category#28, i_class#29]
+Functions [2]: [sum(UnscaledValue(ss_net_profit#32)), sum(UnscaledValue(ss_ext_sales_price#33))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#32))#34, sum(UnscaledValue(ss_ext_sales_price#33))#35]
+Results [3]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#32))#34,17,2) AS ss_net_profit#36, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#33))#35,17,2) AS ss_ext_sales_price#37, i_category#28]
+
+(29) HashAggregate [codegen id : 10]
+Input [3]: [ss_net_profit#36, ss_ext_sales_price#37, i_category#28]
+Keys [1]: [i_category#28]
+Functions [2]: [partial_sum(ss_net_profit#36), partial_sum(ss_ext_sales_price#37)]
+Aggregate Attributes [4]: [sum#38, isEmpty#39, sum#40, isEmpty#41]
+Results [5]: [i_category#28, sum#42, isEmpty#43, sum#44, isEmpty#45]
+
+(30) Exchange
+Input [5]: [i_category#28, sum#42, isEmpty#43, sum#44, isEmpty#45]
+Arguments: hashpartitioning(i_category#28, 5), ENSURE_REQUIREMENTS, [id=#46]
+
+(31) HashAggregate [codegen id : 11]
+Input [5]: [i_category#28, sum#42, isEmpty#43, sum#44, isEmpty#45]
+Keys [1]: [i_category#28]
+Functions [2]: [sum(ss_net_profit#36), sum(ss_ext_sales_price#37)]
+Aggregate Attributes [2]: [sum(ss_net_profit#36)#47, sum(ss_ext_sales_price#37)#48]
+Results [6]: [cast(CheckOverflow((promote_precision(sum(ss_net_profit#36)#47) / promote_precision(sum(ss_ext_sales_price#37)#48)), DecimalType(38,11), true) as decimal(38,20)) AS gross_margin#49, i_category#28, null AS i_class#50, 0 AS t_category#51, 1 AS t_class#52, 1 AS lochierarchy#53]
+
+(32) ReusedExchange [Reuses operator id: unknown]
+Output [4]: [i_category#54, i_class#55, sum#56, sum#57]
+
+(33) HashAggregate [codegen id : 16]
+Input [4]: [i_category#54, i_class#55, sum#56, sum#57]
+Keys [2]: [i_category#54, i_class#55]
+Functions [2]: [sum(UnscaledValue(ss_net_profit#58)), sum(UnscaledValue(ss_ext_sales_price#59))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#58))#60, sum(UnscaledValue(ss_ext_sales_price#59))#61]
+Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#58))#60,17,2) AS ss_net_profit#36, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#59))#61,17,2) AS ss_ext_sales_price#37]
+
+(34) HashAggregate [codegen id : 16]
+Input [2]: [ss_net_profit#36, ss_ext_sales_price#37]
+Keys: []
+Functions [2]: [partial_sum(ss_net_profit#36), partial_sum(ss_ext_sales_price#37)]
+Aggregate Attributes [4]: [sum#62, isEmpty#63, sum#64, isEmpty#65]
+Results [4]: [sum#66, isEmpty#67, sum#68, isEmpty#69]
+
+(35) Exchange
+Input [4]: [sum#66, isEmpty#67, sum#68, isEmpty#69]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#70]
+
+(36) HashAggregate [codegen id : 17]
+Input [4]: [sum#66, isEmpty#67, sum#68, isEmpty#69]
+Keys: []
+Functions [2]: [sum(ss_net_profit#36), sum(ss_ext_sales_price#37)]
+Aggregate Attributes [2]: [sum(ss_net_profit#36)#71, sum(ss_ext_sales_price#37)#72]
+Results [6]: [cast(CheckOverflow((promote_precision(sum(ss_net_profit#36)#71) / promote_precision(sum(ss_ext_sales_price#37)#72)), DecimalType(38,11), true) as decimal(38,20)) AS gross_margin#73, null AS i_category#74, null AS i_class#75, 1 AS t_category#76, 1 AS t_class#77, 2 AS lochierarchy#78]
+
+(37) Union
+
+(38) HashAggregate [codegen id : 18]
+Input [6]: [gross_margin#24, i_category#12, i_class#11, t_category#25, t_class#26, lochierarchy#27]
+Keys [6]: [gross_margin#24, i_category#12, i_class#11, t_category#25, t_class#26, lochierarchy#27]
+Functions: []
+Aggregate Attributes: []
+Results [6]: [gross_margin#24, i_category#12, i_class#11, t_category#25, t_class#26, lochierarchy#27]
+
+(39) Exchange
+Input [6]: [gross_margin#24, i_category#12, i_class#11, t_category#25, t_class#26, lochierarchy#27]
+Arguments: hashpartitioning(gross_margin#24, i_category#12, i_class#11, t_category#25, t_class#26, lochierarchy#27, 5), ENSURE_REQUIREMENTS, [id=#79]
+
+(40) HashAggregate [codegen id : 19]
+Input [6]: [gross_margin#24, i_category#12, i_class#11, t_category#25, t_class#26, lochierarchy#27]
+Keys [6]: [gross_margin#24, i_category#12, i_class#11, t_category#25, t_class#26, lochierarchy#27]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [gross_margin#24, i_category#12, i_class#11, lochierarchy#27, CASE WHEN (t_class#26 = 0) THEN i_category#12 END AS _w0#80]
+
+(41) Exchange
+Input [5]: [gross_margin#24, i_category#12, i_class#11, lochierarchy#27, _w0#80]
+Arguments: hashpartitioning(lochierarchy#27, _w0#80, 5), ENSURE_REQUIREMENTS, [id=#81]
+
+(42) Sort [codegen id : 20]
+Input [5]: [gross_margin#24, i_category#12, i_class#11, lochierarchy#27, _w0#80]
+Arguments: [lochierarchy#27 ASC NULLS FIRST, _w0#80 ASC NULLS FIRST, gross_margin#24 ASC NULLS FIRST], false, 0
+
+(43) Window
+Input [5]: [gross_margin#24, i_category#12, i_class#11, lochierarchy#27, _w0#80]
+Arguments: [rank(gross_margin#24) windowspecdefinition(lochierarchy#27, _w0#80, gross_margin#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#82], [lochierarchy#27, _w0#80], [gross_margin#24 ASC NULLS FIRST]
+
+(44) Project [codegen id : 21]
+Output [5]: [gross_margin#24, i_category#12, i_class#11, lochierarchy#27, rank_within_parent#82]
+Input [6]: [gross_margin#24, i_category#12, i_class#11, lochierarchy#27, _w0#80, rank_within_parent#82]
+
+(45) TakeOrderedAndProject
+Input [5]: [gross_margin#24, i_category#12, i_class#11, lochierarchy#27, rank_within_parent#82]
+Arguments: 100, [lochierarchy#27 DESC NULLS LAST, CASE WHEN (lochierarchy#27 = 0) THEN i_category#12 END ASC NULLS FIRST, rank_within_parent#82 ASC NULLS FIRST], [gross_margin#24, i_category#12, i_class#11, lochierarchy#27, rank_within_parent#82]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (46)
+
+
+(46) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#7]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/simplified.txt
new file mode 100644
index 0000000000000..7cccdf955a9c4
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/simplified.txt
@@ -0,0 +1,76 @@
+TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i_class]
+  WholeStageCodegen (21)
+    Project [gross_margin,i_category,i_class,lochierarchy,rank_within_parent]
+      InputAdapter
+        Window [gross_margin,lochierarchy,_w0]
+          WholeStageCodegen (20)
+            Sort [lochierarchy,_w0,gross_margin]
+              InputAdapter
+                Exchange [lochierarchy,_w0] #1
+                  WholeStageCodegen (19)
+                    HashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] [_w0]
+                      InputAdapter
+                        Exchange [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] #2
+                          WholeStageCodegen (18)
+                            HashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy]
+                              InputAdapter
+                                Union
+                                  WholeStageCodegen (5)
+                                    HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),gross_margin,t_category,t_class,lochierarchy,sum,sum]
+                                      InputAdapter
+                                        Exchange [i_category,i_class] #3
+                                          WholeStageCodegen (4)
+                                            HashAggregate [i_category,i_class,ss_net_profit,ss_ext_sales_price] [sum,sum,sum,sum]
+                                              Project [ss_ext_sales_price,ss_net_profit,i_class,i_category]
+                                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                  Project [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category]
+                                                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                      Project [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit]
+                                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                          Filter [ss_item_sk,ss_store_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk]
+                                                                  SubqueryBroadcast [d_date_sk] #1
+                                                                    ReusedExchange [d_date_sk] #4
+                                                          InputAdapter
+                                                            BroadcastExchange #4
+                                                              WholeStageCodegen (1)
+                                                                Project [d_date_sk]
+                                                                  Filter [d_year,d_date_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.date_dim [d_date_sk,d_year]
+                                                      InputAdapter
+                                                        BroadcastExchange #5
+                                                          WholeStageCodegen (2)
+                                                            Filter [i_item_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.item [i_item_sk,i_class,i_category]
+                                                  InputAdapter
+                                                    BroadcastExchange #6
+                                                      WholeStageCodegen (3)
+                                                        Project [s_store_sk]
+                                                          Filter [s_state,s_store_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.store [s_store_sk,s_state]
+                                  WholeStageCodegen (11)
+                                    HashAggregate [i_category,sum,isEmpty,sum,isEmpty] [sum(ss_net_profit),sum(ss_ext_sales_price),gross_margin,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty]
+                                      InputAdapter
+                                        Exchange [i_category] #7
+                                          WholeStageCodegen (10)
+                                            HashAggregate [i_category,ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                                              HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),ss_net_profit,ss_ext_sales_price,sum,sum]
+                                                InputAdapter
+                                                  ReusedExchange [i_category,i_class,sum,sum] #8
+                                  WholeStageCodegen (17)
+                                    HashAggregate [sum,isEmpty,sum,isEmpty] [sum(ss_net_profit),sum(ss_ext_sales_price),gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty]
+                                      InputAdapter
+                                        Exchange #9
+                                          WholeStageCodegen (16)
+                                            HashAggregate [ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                                              HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),ss_net_profit,ss_ext_sales_price,sum,sum]
+                                                InputAdapter
+                                                  ReusedExchange [i_category,i_class,sum,sum] #8
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.sf100/explain.txt
new file mode 100644
index 0000000000000..636fa32f94212
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.sf100/explain.txt
@@ -0,0 +1,314 @@
+== Physical Plan ==
+TakeOrderedAndProject (56)
++- * Project (55)
+   +- * SortMergeJoin Inner (54)
+      :- * Project (47)
+      :  +- * SortMergeJoin Inner (46)
+      :     :- * Sort (37)
+      :     :  +- Exchange (36)
+      :     :     +- * Filter (35)
+      :     :        +- Window (34)
+      :     :           +- * Sort (33)
+      :     :              +- Exchange (32)
+      :     :                 +- * Project (31)
+      :     :                    +- Window (30)
+      :     :                       +- * Sort (29)
+      :     :                          +- Exchange (28)
+      :     :                             +- * HashAggregate (27)
+      :     :                                +- Exchange (26)
+      :     :                                   +- * HashAggregate (25)
+      :     :                                      +- * Project (24)
+      :     :                                         +- * SortMergeJoin Inner (23)
+      :     :                                            :- * Sort (17)
+      :     :                                            :  +- Exchange (16)
+      :     :                                            :     +- * Project (15)
+      :     :                                            :        +- * BroadcastHashJoin Inner BuildRight (14)
+      :     :                                            :           :- * Project (9)
+      :     :                                            :           :  +- * BroadcastHashJoin Inner BuildRight (8)
+      :     :                                            :           :     :- * Filter (3)
+      :     :                                            :           :     :  +- * ColumnarToRow (2)
+      :     :                                            :           :     :     +- Scan parquet default.store_sales (1)
+      :     :                                            :           :     +- BroadcastExchange (7)
+      :     :                                            :           :        +- * Filter (6)
+      :     :                                            :           :           +- * ColumnarToRow (5)
+      :     :                                            :           :              +- Scan parquet default.date_dim (4)
+      :     :                                            :           +- BroadcastExchange (13)
+      :     :                                            :              +- * Filter (12)
+      :     :                                            :                 +- * ColumnarToRow (11)
+      :     :                                            :                    +- Scan parquet default.store (10)
+      :     :                                            +- * Sort (22)
+      :     :                                               +- Exchange (21)
+      :     :                                                  +- * Filter (20)
+      :     :                                                     +- * ColumnarToRow (19)
+      :     :                                                        +- Scan parquet default.item (18)
+      :     +- * Sort (45)
+      :        +- Exchange (44)
+      :           +- * Project (43)
+      :              +- Window (42)
+      :                 +- * Sort (41)
+      :                    +- Exchange (40)
+      :                       +- * HashAggregate (39)
+      :                          +- ReusedExchange (38)
+      +- * Sort (53)
+         +- Exchange (52)
+            +- * Project (51)
+               +- Window (50)
+                  +- * Sort (49)
+                     +- ReusedExchange (48)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 3]
+Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, ss_sold_date_sk#4]
+Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Condition : ((((d_year#7 = 1999) OR ((d_year#7 = 1998) AND (d_moy#8 = 12))) OR ((d_year#7 = 2000) AND (d_moy#8 = 1))) AND isnotnull(d_date_sk#6))
+
+(7) BroadcastExchange
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#9]
+
+(8) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(9) Project [codegen id : 3]
+Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, d_year#7, d_moy#8]
+Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, ss_sold_date_sk#4, d_date_sk#6, d_year#7, d_moy#8]
+
+(10) Scan parquet default.store
+Output [3]: [s_store_sk#10, s_store_name#11, s_company_name#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_company_name:string>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [3]: [s_store_sk#10, s_store_name#11, s_company_name#12]
+
+(12) Filter [codegen id : 2]
+Input [3]: [s_store_sk#10, s_store_name#11, s_company_name#12]
+Condition : ((isnotnull(s_store_sk#10) AND isnotnull(s_store_name#11)) AND isnotnull(s_company_name#12))
+
+(13) BroadcastExchange
+Input [3]: [s_store_sk#10, s_store_name#11, s_company_name#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#13]
+
+(14) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_store_sk#2]
+Right keys [1]: [s_store_sk#10]
+Join condition: None
+
+(15) Project [codegen id : 3]
+Output [6]: [ss_item_sk#1, ss_sales_price#3, d_year#7, d_moy#8, s_store_name#11, s_company_name#12]
+Input [8]: [ss_item_sk#1, ss_store_sk#2, ss_sales_price#3, d_year#7, d_moy#8, s_store_sk#10, s_store_name#11, s_company_name#12]
+
+(16) Exchange
+Input [6]: [ss_item_sk#1, ss_sales_price#3, d_year#7, d_moy#8, s_store_name#11, s_company_name#12]
+Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#14]
+
+(17) Sort [codegen id : 4]
+Input [6]: [ss_item_sk#1, ss_sales_price#3, d_year#7, d_moy#8, s_store_name#11, s_company_name#12]
+Arguments: [ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(18) Scan parquet default.item
+Output [3]: [i_item_sk#15, i_brand#16, i_category#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_category:string>
+
+(19) ColumnarToRow [codegen id : 5]
+Input [3]: [i_item_sk#15, i_brand#16, i_category#17]
+
+(20) Filter [codegen id : 5]
+Input [3]: [i_item_sk#15, i_brand#16, i_category#17]
+Condition : ((isnotnull(i_item_sk#15) AND isnotnull(i_category#17)) AND isnotnull(i_brand#16))
+
+(21) Exchange
+Input [3]: [i_item_sk#15, i_brand#16, i_category#17]
+Arguments: hashpartitioning(i_item_sk#15, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(22) Sort [codegen id : 6]
+Input [3]: [i_item_sk#15, i_brand#16, i_category#17]
+Arguments: [i_item_sk#15 ASC NULLS FIRST], false, 0
+
+(23) SortMergeJoin [codegen id : 7]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#15]
+Join condition: None
+
+(24) Project [codegen id : 7]
+Output [7]: [i_brand#16, i_category#17, ss_sales_price#3, d_year#7, d_moy#8, s_store_name#11, s_company_name#12]
+Input [9]: [ss_item_sk#1, ss_sales_price#3, d_year#7, d_moy#8, s_store_name#11, s_company_name#12, i_item_sk#15, i_brand#16, i_category#17]
+
+(25) HashAggregate [codegen id : 7]
+Input [7]: [i_brand#16, i_category#17, ss_sales_price#3, d_year#7, d_moy#8, s_store_name#11, s_company_name#12]
+Keys [6]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#3))]
+Aggregate Attributes [1]: [sum#19]
+Results [7]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum#20]
+
+(26) Exchange
+Input [7]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum#20]
+Arguments: hashpartitioning(i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, 5), ENSURE_REQUIREMENTS, [id=#21]
+
+(27) HashAggregate [codegen id : 8]
+Input [7]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum#20]
+Keys [6]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#3))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#3))#22]
+Results [8]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, MakeDecimal(sum(UnscaledValue(ss_sales_price#3))#22,17,2) AS sum_sales#23, MakeDecimal(sum(UnscaledValue(ss_sales_price#3))#22,17,2) AS _w0#24]
+
+(28) Exchange
+Input [8]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, _w0#24]
+Arguments: hashpartitioning(i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(29) Sort [codegen id : 9]
+Input [8]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, _w0#24]
+Arguments: [i_category#17 ASC NULLS FIRST, i_brand#16 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, s_company_name#12 ASC NULLS FIRST, d_year#7 ASC NULLS FIRST], false, 0
+
+(30) Window
+Input [8]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, _w0#24]
+Arguments: [avg(_w0#24) windowspecdefinition(i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#26], [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7]
+
+(31) Project [codegen id : 10]
+Output [8]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, avg_monthly_sales#26]
+Input [9]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, _w0#24, avg_monthly_sales#26]
+
+(32) Exchange
+Input [8]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, avg_monthly_sales#26]
+Arguments: hashpartitioning(i_category#17, i_brand#16, s_store_name#11, s_company_name#12, 5), ENSURE_REQUIREMENTS, [id=#27]
+
+(33) Sort [codegen id : 11]
+Input [8]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, avg_monthly_sales#26]
+Arguments: [i_category#17 ASC NULLS FIRST, i_brand#16 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, s_company_name#12 ASC NULLS FIRST, d_year#7 ASC NULLS FIRST, d_moy#8 ASC NULLS FIRST], false, 0
+
+(34) Window
+Input [8]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, avg_monthly_sales#26]
+Arguments: [rank(d_year#7, d_moy#8) windowspecdefinition(i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7 ASC NULLS FIRST, d_moy#8 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#28], [i_category#17, i_brand#16, s_store_name#11, s_company_name#12], [d_year#7 ASC NULLS FIRST, d_moy#8 ASC NULLS FIRST]
+
+(35) Filter [codegen id : 12]
+Input [9]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, avg_monthly_sales#26, rn#28]
+Condition : ((((isnotnull(d_year#7) AND isnotnull(avg_monthly_sales#26)) AND (d_year#7 = 1999)) AND (avg_monthly_sales#26 > 0.000000)) AND (CheckOverflow((promote_precision(abs(CheckOverflow((promote_precision(cast(sum_sales#23 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#26 as decimal(22,6)))), DecimalType(22,6), true), false)) / promote_precision(cast(avg_monthly_sales#26 as decimal(22,6)))), DecimalType(38,16), true) > 0.1000000000000000))
+
+(36) Exchange
+Input [9]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, avg_monthly_sales#26, rn#28]
+Arguments: hashpartitioning(i_category#17, i_brand#16, s_store_name#11, s_company_name#12, rn#28, 5), ENSURE_REQUIREMENTS, [id=#29]
+
+(37) Sort [codegen id : 13]
+Input [9]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, avg_monthly_sales#26, rn#28]
+Arguments: [i_category#17 ASC NULLS FIRST, i_brand#16 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, s_company_name#12 ASC NULLS FIRST, rn#28 ASC NULLS FIRST], false, 0
+
+(38) ReusedExchange [Reuses operator id: unknown]
+Output [7]: [i_category#30, i_brand#31, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum#36]
+
+(39) HashAggregate [codegen id : 21]
+Input [7]: [i_category#30, i_brand#31, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum#36]
+Keys [6]: [i_category#30, i_brand#31, s_store_name#32, s_company_name#33, d_year#34, d_moy#35]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#37))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#37))#38]
+Results [7]: [i_category#30, i_brand#31, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, MakeDecimal(sum(UnscaledValue(ss_sales_price#37))#38,17,2) AS sum_sales#39]
+
+(40) Exchange
+Input [7]: [i_category#30, i_brand#31, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#39]
+Arguments: hashpartitioning(i_category#30, i_brand#31, s_store_name#32, s_company_name#33, 5), ENSURE_REQUIREMENTS, [id=#40]
+
+(41) Sort [codegen id : 22]
+Input [7]: [i_category#30, i_brand#31, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#39]
+Arguments: [i_category#30 ASC NULLS FIRST, i_brand#31 ASC NULLS FIRST, s_store_name#32 ASC NULLS FIRST, s_company_name#33 ASC NULLS FIRST, d_year#34 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST], false, 0
+
+(42) Window
+Input [7]: [i_category#30, i_brand#31, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#39]
+Arguments: [rank(d_year#34, d_moy#35) windowspecdefinition(i_category#30, i_brand#31, s_store_name#32, s_company_name#33, d_year#34 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#41], [i_category#30, i_brand#31, s_store_name#32, s_company_name#33], [d_year#34 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST]
+
+(43) Project [codegen id : 23]
+Output [6]: [i_category#30, i_brand#31, s_store_name#32, s_company_name#33, sum_sales#39, rn#41]
+Input [8]: [i_category#30, i_brand#31, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#39, rn#41]
+
+(44) Exchange
+Input [6]: [i_category#30, i_brand#31, s_store_name#32, s_company_name#33, sum_sales#39, rn#41]
+Arguments: hashpartitioning(i_category#30, i_brand#31, s_store_name#32, s_company_name#33, (rn#41 + 1), 5), ENSURE_REQUIREMENTS, [id=#42]
+
+(45) Sort [codegen id : 24]
+Input [6]: [i_category#30, i_brand#31, s_store_name#32, s_company_name#33, sum_sales#39, rn#41]
+Arguments: [i_category#30 ASC NULLS FIRST, i_brand#31 ASC NULLS FIRST, s_store_name#32 ASC NULLS FIRST, s_company_name#33 ASC NULLS FIRST, (rn#41 + 1) ASC NULLS FIRST], false, 0
+
+(46) SortMergeJoin [codegen id : 25]
+Left keys [5]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, rn#28]
+Right keys [5]: [i_category#30, i_brand#31, s_store_name#32, s_company_name#33, (rn#41 + 1)]
+Join condition: None
+
+(47) Project [codegen id : 25]
+Output [10]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, avg_monthly_sales#26, rn#28, sum_sales#39]
+Input [15]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, avg_monthly_sales#26, rn#28, i_category#30, i_brand#31, s_store_name#32, s_company_name#33, sum_sales#39, rn#41]
+
+(48) ReusedExchange [Reuses operator id: 40]
+Output [7]: [i_category#43, i_brand#44, s_store_name#45, s_company_name#46, d_year#47, d_moy#48, sum_sales#49]
+
+(49) Sort [codegen id : 34]
+Input [7]: [i_category#43, i_brand#44, s_store_name#45, s_company_name#46, d_year#47, d_moy#48, sum_sales#49]
+Arguments: [i_category#43 ASC NULLS FIRST, i_brand#44 ASC NULLS FIRST, s_store_name#45 ASC NULLS FIRST, s_company_name#46 ASC NULLS FIRST, d_year#47 ASC NULLS FIRST, d_moy#48 ASC NULLS FIRST], false, 0
+
+(50) Window
+Input [7]: [i_category#43, i_brand#44, s_store_name#45, s_company_name#46, d_year#47, d_moy#48, sum_sales#49]
+Arguments: [rank(d_year#47, d_moy#48) windowspecdefinition(i_category#43, i_brand#44, s_store_name#45, s_company_name#46, d_year#47 ASC NULLS FIRST, d_moy#48 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#50], [i_category#43, i_brand#44, s_store_name#45, s_company_name#46], [d_year#47 ASC NULLS FIRST, d_moy#48 ASC NULLS FIRST]
+
+(51) Project [codegen id : 35]
+Output [6]: [i_category#43, i_brand#44, s_store_name#45, s_company_name#46, sum_sales#49, rn#50]
+Input [8]: [i_category#43, i_brand#44, s_store_name#45, s_company_name#46, d_year#47, d_moy#48, sum_sales#49, rn#50]
+
+(52) Exchange
+Input [6]: [i_category#43, i_brand#44, s_store_name#45, s_company_name#46, sum_sales#49, rn#50]
+Arguments: hashpartitioning(i_category#43, i_brand#44, s_store_name#45, s_company_name#46, (rn#50 - 1), 5), ENSURE_REQUIREMENTS, [id=#51]
+
+(53) Sort [codegen id : 36]
+Input [6]: [i_category#43, i_brand#44, s_store_name#45, s_company_name#46, sum_sales#49, rn#50]
+Arguments: [i_category#43 ASC NULLS FIRST, i_brand#44 ASC NULLS FIRST, s_store_name#45 ASC NULLS FIRST, s_company_name#46 ASC NULLS FIRST, (rn#50 - 1) ASC NULLS FIRST], false, 0
+
+(54) SortMergeJoin [codegen id : 37]
+Left keys [5]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, rn#28]
+Right keys [5]: [i_category#43, i_brand#44, s_store_name#45, s_company_name#46, (rn#50 - 1)]
+Join condition: None
+
+(55) Project [codegen id : 37]
+Output [7]: [i_category#17, d_year#7, d_moy#8, avg_monthly_sales#26, sum_sales#23, sum_sales#39 AS psum#52, sum_sales#49 AS nsum#53]
+Input [16]: [i_category#17, i_brand#16, s_store_name#11, s_company_name#12, d_year#7, d_moy#8, sum_sales#23, avg_monthly_sales#26, rn#28, sum_sales#39, i_category#43, i_brand#44, s_store_name#45, s_company_name#46, sum_sales#49, rn#50]
+
+(56) TakeOrderedAndProject
+Input [7]: [i_category#17, d_year#7, d_moy#8, avg_monthly_sales#26, sum_sales#23, psum#52, nsum#53]
+Arguments: 100, [CheckOverflow((promote_precision(cast(sum_sales#23 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#26 as decimal(22,6)))), DecimalType(22,6), true) ASC NULLS FIRST, d_moy#8 ASC NULLS FIRST], [i_category#17, d_year#7, d_moy#8, avg_monthly_sales#26, sum_sales#23, psum#52, nsum#53]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (57)
+
+
+(57) ReusedExchange [Reuses operator id: 7]
+Output [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.sf100/simplified.txt
new file mode 100644
index 0000000000000..0ebb78eb49ff0
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.sf100/simplified.txt
@@ -0,0 +1,107 @@
+TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum,nsum]
+  WholeStageCodegen (37)
+    Project [i_category,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales]
+      SortMergeJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn]
+        InputAdapter
+          WholeStageCodegen (25)
+            Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales]
+              SortMergeJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn]
+                InputAdapter
+                  WholeStageCodegen (13)
+                    Sort [i_category,i_brand,s_store_name,s_company_name,rn]
+                      InputAdapter
+                        Exchange [i_category,i_brand,s_store_name,s_company_name,rn] #1
+                          WholeStageCodegen (12)
+                            Filter [d_year,avg_monthly_sales,sum_sales]
+                              InputAdapter
+                                Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name]
+                                  WholeStageCodegen (11)
+                                    Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy]
+                                      InputAdapter
+                                        Exchange [i_category,i_brand,s_store_name,s_company_name] #2
+                                          WholeStageCodegen (10)
+                                            Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales]
+                                              InputAdapter
+                                                Window [_w0,i_category,i_brand,s_store_name,s_company_name,d_year]
+                                                  WholeStageCodegen (9)
+                                                    Sort [i_category,i_brand,s_store_name,s_company_name,d_year]
+                                                      InputAdapter
+                                                        Exchange [i_category,i_brand,s_store_name,s_company_name,d_year] #3
+                                                          WholeStageCodegen (8)
+                                                            HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum]
+                                                              InputAdapter
+                                                                Exchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #4
+                                                                  WholeStageCodegen (7)
+                                                                    HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,ss_sales_price] [sum,sum]
+                                                                      Project [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name]
+                                                                        SortMergeJoin [ss_item_sk,i_item_sk]
+                                                                          InputAdapter
+                                                                            WholeStageCodegen (4)
+                                                                              Sort [ss_item_sk]
+                                                                                InputAdapter
+                                                                                  Exchange [ss_item_sk] #5
+                                                                                    WholeStageCodegen (3)
+                                                                                      Project [ss_item_sk,ss_sales_price,d_year,d_moy,s_store_name,s_company_name]
+                                                                                        BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                                                          Project [ss_item_sk,ss_store_sk,ss_sales_price,d_year,d_moy]
+                                                                                            BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                              Filter [ss_item_sk,ss_store_sk]
+                                                                                                ColumnarToRow
+                                                                                                  InputAdapter
+                                                                                                    Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                                                                                      SubqueryBroadcast [d_date_sk] #1
+                                                                                                        ReusedExchange [d_date_sk,d_year,d_moy] #6
+                                                                                              InputAdapter
+                                                                                                BroadcastExchange #6
+                                                                                                  WholeStageCodegen (1)
+                                                                                                    Filter [d_year,d_moy,d_date_sk]
+                                                                                                      ColumnarToRow
+                                                                                                        InputAdapter
+                                                                                                          Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                                                                          InputAdapter
+                                                                                            BroadcastExchange #7
+                                                                                              WholeStageCodegen (2)
+                                                                                                Filter [s_store_sk,s_store_name,s_company_name]
+                                                                                                  ColumnarToRow
+                                                                                                    InputAdapter
+                                                                                                      Scan parquet default.store [s_store_sk,s_store_name,s_company_name]
+                                                                          InputAdapter
+                                                                            WholeStageCodegen (6)
+                                                                              Sort [i_item_sk]
+                                                                                InputAdapter
+                                                                                  Exchange [i_item_sk] #8
+                                                                                    WholeStageCodegen (5)
+                                                                                      Filter [i_item_sk,i_category,i_brand]
+                                                                                        ColumnarToRow
+                                                                                          InputAdapter
+                                                                                            Scan parquet default.item [i_item_sk,i_brand,i_category]
+                InputAdapter
+                  WholeStageCodegen (24)
+                    Sort [i_category,i_brand,s_store_name,s_company_name,rn]
+                      InputAdapter
+                        Exchange [i_category,i_brand,s_store_name,s_company_name,rn] #9
+                          WholeStageCodegen (23)
+                            Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn]
+                              InputAdapter
+                                Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name]
+                                  WholeStageCodegen (22)
+                                    Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy]
+                                      InputAdapter
+                                        Exchange [i_category,i_brand,s_store_name,s_company_name] #10
+                                          WholeStageCodegen (21)
+                                            HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,sum]
+                                              InputAdapter
+                                                ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #11
+        InputAdapter
+          WholeStageCodegen (36)
+            Sort [i_category,i_brand,s_store_name,s_company_name,rn]
+              InputAdapter
+                Exchange [i_category,i_brand,s_store_name,s_company_name,rn] #12
+                  WholeStageCodegen (35)
+                    Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn]
+                      InputAdapter
+                        Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name]
+                          WholeStageCodegen (34)
+                            Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy]
+                              InputAdapter
+                                ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #10
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/explain.txt
new file mode 100644
index 0000000000000..734eec1be3565
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/explain.txt
@@ -0,0 +1,279 @@
+== Physical Plan ==
+TakeOrderedAndProject (49)
++- * Project (48)
+   +- * BroadcastHashJoin Inner BuildRight (47)
+      :- * Project (41)
+      :  +- * BroadcastHashJoin Inner BuildRight (40)
+      :     :- * Filter (32)
+      :     :  +- Window (31)
+      :     :     +- * Sort (30)
+      :     :        +- Exchange (29)
+      :     :           +- * Project (28)
+      :     :              +- Window (27)
+      :     :                 +- * Sort (26)
+      :     :                    +- Exchange (25)
+      :     :                       +- * HashAggregate (24)
+      :     :                          +- Exchange (23)
+      :     :                             +- * HashAggregate (22)
+      :     :                                +- * Project (21)
+      :     :                                   +- * BroadcastHashJoin Inner BuildRight (20)
+      :     :                                      :- * Project (15)
+      :     :                                      :  +- * BroadcastHashJoin Inner BuildRight (14)
+      :     :                                      :     :- * Project (9)
+      :     :                                      :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+      :     :                                      :     :     :- * Filter (3)
+      :     :                                      :     :     :  +- * ColumnarToRow (2)
+      :     :                                      :     :     :     +- Scan parquet default.item (1)
+      :     :                                      :     :     +- BroadcastExchange (7)
+      :     :                                      :     :        +- * Filter (6)
+      :     :                                      :     :           +- * ColumnarToRow (5)
+      :     :                                      :     :              +- Scan parquet default.store_sales (4)
+      :     :                                      :     +- BroadcastExchange (13)
+      :     :                                      :        +- * Filter (12)
+      :     :                                      :           +- * ColumnarToRow (11)
+      :     :                                      :              +- Scan parquet default.date_dim (10)
+      :     :                                      +- BroadcastExchange (19)
+      :     :                                         +- * Filter (18)
+      :     :                                            +- * ColumnarToRow (17)
+      :     :                                               +- Scan parquet default.store (16)
+      :     +- BroadcastExchange (39)
+      :        +- * Project (38)
+      :           +- Window (37)
+      :              +- * Sort (36)
+      :                 +- Exchange (35)
+      :                    +- * HashAggregate (34)
+      :                       +- ReusedExchange (33)
+      +- BroadcastExchange (46)
+         +- * Project (45)
+            +- Window (44)
+               +- * Sort (43)
+                  +- ReusedExchange (42)
+
+
+(1) Scan parquet default.item
+Output [3]: [i_item_sk#1, i_brand#2, i_category#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_category:string>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [3]: [i_item_sk#1, i_brand#2, i_category#3]
+
+(3) Filter [codegen id : 4]
+Input [3]: [i_item_sk#1, i_brand#2, i_category#3]
+Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2))
+
+(4) Scan parquet default.store_sales
+Output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_sales_price:decimal(7,2)>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7]
+
+(6) Filter [codegen id : 1]
+Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7]
+Condition : (isnotnull(ss_item_sk#4) AND isnotnull(ss_store_sk#5))
+
+(7) BroadcastExchange
+Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#9]
+
+(8) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [ss_item_sk#4]
+Join condition: None
+
+(9) Project [codegen id : 4]
+Output [5]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7]
+Input [7]: [i_item_sk#1, i_brand#2, i_category#3, ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7]
+
+(10) Scan parquet default.date_dim
+Output [3]: [d_date_sk#10, d_year#11, d_moy#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+
+(12) Filter [codegen id : 2]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10))
+
+(13) BroadcastExchange
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#13]
+
+(14) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#7]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(15) Project [codegen id : 4]
+Output [6]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#11, d_moy#12]
+Input [8]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#10, d_year#11, d_moy#12]
+
+(16) Scan parquet default.store
+Output [3]: [s_store_sk#14, s_store_name#15, s_company_name#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_company_name:string>
+
+(17) ColumnarToRow [codegen id : 3]
+Input [3]: [s_store_sk#14, s_store_name#15, s_company_name#16]
+
+(18) Filter [codegen id : 3]
+Input [3]: [s_store_sk#14, s_store_name#15, s_company_name#16]
+Condition : ((isnotnull(s_store_sk#14) AND isnotnull(s_store_name#15)) AND isnotnull(s_company_name#16))
+
+(19) BroadcastExchange
+Input [3]: [s_store_sk#14, s_store_name#15, s_company_name#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#17]
+
+(20) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#5]
+Right keys [1]: [s_store_sk#14]
+Join condition: None
+
+(21) Project [codegen id : 4]
+Output [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#11, d_moy#12, s_store_name#15, s_company_name#16]
+Input [9]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#11, d_moy#12, s_store_sk#14, s_store_name#15, s_company_name#16]
+
+(22) HashAggregate [codegen id : 4]
+Input [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#11, d_moy#12, s_store_name#15, s_company_name#16]
+Keys [6]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#6))]
+Aggregate Attributes [1]: [sum#18]
+Results [7]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum#19]
+
+(23) Exchange
+Input [7]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum#19]
+Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(24) HashAggregate [codegen id : 5]
+Input [7]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum#19]
+Keys [6]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#6))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#6))#21]
+Results [8]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#21,17,2) AS sum_sales#22, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#21,17,2) AS _w0#23]
+
+(25) Exchange
+Input [8]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum_sales#22, _w0#23]
+Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(26) Sort [codegen id : 6]
+Input [8]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum_sales#22, _w0#23]
+Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#15 ASC NULLS FIRST, s_company_name#16 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST], false, 0
+
+(27) Window
+Input [8]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum_sales#22, _w0#23]
+Arguments: [avg(_w0#23) windowspecdefinition(i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#25], [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11]
+
+(28) Project [codegen id : 7]
+Output [8]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum_sales#22, avg_monthly_sales#25]
+Input [9]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum_sales#22, _w0#23, avg_monthly_sales#25]
+
+(29) Exchange
+Input [8]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum_sales#22, avg_monthly_sales#25]
+Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#15, s_company_name#16, 5), ENSURE_REQUIREMENTS, [id=#26]
+
+(30) Sort [codegen id : 8]
+Input [8]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum_sales#22, avg_monthly_sales#25]
+Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#15 ASC NULLS FIRST, s_company_name#16 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST], false, 0
+
+(31) Window
+Input [8]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum_sales#22, avg_monthly_sales#25]
+Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#3, i_brand#2, s_store_name#15, s_company_name#16], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST]
+
+(32) Filter [codegen id : 23]
+Input [9]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum_sales#22, avg_monthly_sales#25, rn#27]
+Condition : ((((isnotnull(d_year#11) AND isnotnull(avg_monthly_sales#25)) AND (d_year#11 = 1999)) AND (avg_monthly_sales#25 > 0.000000)) AND (CheckOverflow((promote_precision(abs(CheckOverflow((promote_precision(cast(sum_sales#22 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#25 as decimal(22,6)))), DecimalType(22,6), true), false)) / promote_precision(cast(avg_monthly_sales#25 as decimal(22,6)))), DecimalType(38,16), true) > 0.1000000000000000))
+
+(33) ReusedExchange [Reuses operator id: unknown]
+Output [7]: [i_category#28, i_brand#29, s_store_name#30, s_company_name#31, d_year#32, d_moy#33, sum#34]
+
+(34) HashAggregate [codegen id : 13]
+Input [7]: [i_category#28, i_brand#29, s_store_name#30, s_company_name#31, d_year#32, d_moy#33, sum#34]
+Keys [6]: [i_category#28, i_brand#29, s_store_name#30, s_company_name#31, d_year#32, d_moy#33]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#35))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#35))#36]
+Results [7]: [i_category#28, i_brand#29, s_store_name#30, s_company_name#31, d_year#32, d_moy#33, MakeDecimal(sum(UnscaledValue(ss_sales_price#35))#36,17,2) AS sum_sales#37]
+
+(35) Exchange
+Input [7]: [i_category#28, i_brand#29, s_store_name#30, s_company_name#31, d_year#32, d_moy#33, sum_sales#37]
+Arguments: hashpartitioning(i_category#28, i_brand#29, s_store_name#30, s_company_name#31, 5), ENSURE_REQUIREMENTS, [id=#38]
+
+(36) Sort [codegen id : 14]
+Input [7]: [i_category#28, i_brand#29, s_store_name#30, s_company_name#31, d_year#32, d_moy#33, sum_sales#37]
+Arguments: [i_category#28 ASC NULLS FIRST, i_brand#29 ASC NULLS FIRST, s_store_name#30 ASC NULLS FIRST, s_company_name#31 ASC NULLS FIRST, d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST], false, 0
+
+(37) Window
+Input [7]: [i_category#28, i_brand#29, s_store_name#30, s_company_name#31, d_year#32, d_moy#33, sum_sales#37]
+Arguments: [rank(d_year#32, d_moy#33) windowspecdefinition(i_category#28, i_brand#29, s_store_name#30, s_company_name#31, d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#39], [i_category#28, i_brand#29, s_store_name#30, s_company_name#31], [d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST]
+
+(38) Project [codegen id : 15]
+Output [6]: [i_category#28, i_brand#29, s_store_name#30, s_company_name#31, sum_sales#37, rn#39]
+Input [8]: [i_category#28, i_brand#29, s_store_name#30, s_company_name#31, d_year#32, d_moy#33, sum_sales#37, rn#39]
+
+(39) BroadcastExchange
+Input [6]: [i_category#28, i_brand#29, s_store_name#30, s_company_name#31, sum_sales#37, rn#39]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [id=#40]
+
+(40) BroadcastHashJoin [codegen id : 23]
+Left keys [5]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, rn#27]
+Right keys [5]: [i_category#28, i_brand#29, s_store_name#30, s_company_name#31, (rn#39 + 1)]
+Join condition: None
+
+(41) Project [codegen id : 23]
+Output [10]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum_sales#22, avg_monthly_sales#25, rn#27, sum_sales#37]
+Input [15]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum_sales#22, avg_monthly_sales#25, rn#27, i_category#28, i_brand#29, s_store_name#30, s_company_name#31, sum_sales#37, rn#39]
+
+(42) ReusedExchange [Reuses operator id: 35]
+Output [7]: [i_category#41, i_brand#42, s_store_name#43, s_company_name#44, d_year#45, d_moy#46, sum_sales#47]
+
+(43) Sort [codegen id : 21]
+Input [7]: [i_category#41, i_brand#42, s_store_name#43, s_company_name#44, d_year#45, d_moy#46, sum_sales#47]
+Arguments: [i_category#41 ASC NULLS FIRST, i_brand#42 ASC NULLS FIRST, s_store_name#43 ASC NULLS FIRST, s_company_name#44 ASC NULLS FIRST, d_year#45 ASC NULLS FIRST, d_moy#46 ASC NULLS FIRST], false, 0
+
+(44) Window
+Input [7]: [i_category#41, i_brand#42, s_store_name#43, s_company_name#44, d_year#45, d_moy#46, sum_sales#47]
+Arguments: [rank(d_year#45, d_moy#46) windowspecdefinition(i_category#41, i_brand#42, s_store_name#43, s_company_name#44, d_year#45 ASC NULLS FIRST, d_moy#46 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#48], [i_category#41, i_brand#42, s_store_name#43, s_company_name#44], [d_year#45 ASC NULLS FIRST, d_moy#46 ASC NULLS FIRST]
+
+(45) Project [codegen id : 22]
+Output [6]: [i_category#41, i_brand#42, s_store_name#43, s_company_name#44, sum_sales#47, rn#48]
+Input [8]: [i_category#41, i_brand#42, s_store_name#43, s_company_name#44, d_year#45, d_moy#46, sum_sales#47, rn#48]
+
+(46) BroadcastExchange
+Input [6]: [i_category#41, i_brand#42, s_store_name#43, s_company_name#44, sum_sales#47, rn#48]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [id=#49]
+
+(47) BroadcastHashJoin [codegen id : 23]
+Left keys [5]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, rn#27]
+Right keys [5]: [i_category#41, i_brand#42, s_store_name#43, s_company_name#44, (rn#48 - 1)]
+Join condition: None
+
+(48) Project [codegen id : 23]
+Output [7]: [i_category#3, d_year#11, d_moy#12, avg_monthly_sales#25, sum_sales#22, sum_sales#37 AS psum#50, sum_sales#47 AS nsum#51]
+Input [16]: [i_category#3, i_brand#2, s_store_name#15, s_company_name#16, d_year#11, d_moy#12, sum_sales#22, avg_monthly_sales#25, rn#27, sum_sales#37, i_category#41, i_brand#42, s_store_name#43, s_company_name#44, sum_sales#47, rn#48]
+
+(49) TakeOrderedAndProject
+Input [7]: [i_category#3, d_year#11, d_moy#12, avg_monthly_sales#25, sum_sales#22, psum#50, nsum#51]
+Arguments: 100, [CheckOverflow((promote_precision(cast(sum_sales#22 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#25 as decimal(22,6)))), DecimalType(22,6), true) ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST], [i_category#3, d_year#11, d_moy#12, avg_monthly_sales#25, sum_sales#22, psum#50, nsum#51]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (50)
+
+
+(50) ReusedExchange [Reuses operator id: 13]
+Output [3]: [d_date_sk#10, d_year#11, d_moy#12]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/simplified.txt
new file mode 100644
index 0000000000000..d649a3b21237d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/simplified.txt
@@ -0,0 +1,84 @@
+TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum,nsum]
+  WholeStageCodegen (23)
+    Project [i_category,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales]
+      BroadcastHashJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn]
+        Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales]
+          BroadcastHashJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn]
+            Filter [d_year,avg_monthly_sales,sum_sales]
+              InputAdapter
+                Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name]
+                  WholeStageCodegen (8)
+                    Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy]
+                      InputAdapter
+                        Exchange [i_category,i_brand,s_store_name,s_company_name] #1
+                          WholeStageCodegen (7)
+                            Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales]
+                              InputAdapter
+                                Window [_w0,i_category,i_brand,s_store_name,s_company_name,d_year]
+                                  WholeStageCodegen (6)
+                                    Sort [i_category,i_brand,s_store_name,s_company_name,d_year]
+                                      InputAdapter
+                                        Exchange [i_category,i_brand,s_store_name,s_company_name,d_year] #2
+                                          WholeStageCodegen (5)
+                                            HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum]
+                                              InputAdapter
+                                                Exchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #3
+                                                  WholeStageCodegen (4)
+                                                    HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,ss_sales_price] [sum,sum]
+                                                      Project [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name]
+                                                        BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                          Project [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy]
+                                                            BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                              Project [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                                                BroadcastHashJoin [i_item_sk,ss_item_sk]
+                                                                  Filter [i_item_sk,i_category,i_brand]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.item [i_item_sk,i_brand,i_category]
+                                                                  InputAdapter
+                                                                    BroadcastExchange #4
+                                                                      WholeStageCodegen (1)
+                                                                        Filter [ss_item_sk,ss_store_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk]
+                                                                                SubqueryBroadcast [d_date_sk] #1
+                                                                                  ReusedExchange [d_date_sk,d_year,d_moy] #5
+                                                              InputAdapter
+                                                                BroadcastExchange #5
+                                                                  WholeStageCodegen (2)
+                                                                    Filter [d_year,d_moy,d_date_sk]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                                          InputAdapter
+                                                            BroadcastExchange #6
+                                                              WholeStageCodegen (3)
+                                                                Filter [s_store_sk,s_store_name,s_company_name]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.store [s_store_sk,s_store_name,s_company_name]
+            InputAdapter
+              BroadcastExchange #7
+                WholeStageCodegen (15)
+                  Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn]
+                    InputAdapter
+                      Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name]
+                        WholeStageCodegen (14)
+                          Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy]
+                            InputAdapter
+                              Exchange [i_category,i_brand,s_store_name,s_company_name] #8
+                                WholeStageCodegen (13)
+                                  HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,sum]
+                                    InputAdapter
+                                      ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #9
+        InputAdapter
+          BroadcastExchange #10
+            WholeStageCodegen (22)
+              Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn]
+                InputAdapter
+                  Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name]
+                    WholeStageCodegen (21)
+                      Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy]
+                        InputAdapter
+                          ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #8
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.sf100/explain.txt
new file mode 100644
index 0000000000000..b7c20b4826bb6
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.sf100/explain.txt
@@ -0,0 +1,510 @@
+== Physical Plan ==
+TakeOrderedAndProject (90)
++- * HashAggregate (89)
+   +- Exchange (88)
+      +- * HashAggregate (87)
+         +- Union (86)
+            :- * Project (31)
+            :  +- * Filter (30)
+            :     +- Window (29)
+            :        +- * Sort (28)
+            :           +- Window (27)
+            :              +- * Sort (26)
+            :                 +- Exchange (25)
+            :                    +- * HashAggregate (24)
+            :                       +- Exchange (23)
+            :                          +- * HashAggregate (22)
+            :                             +- * Project (21)
+            :                                +- * SortMergeJoin Inner (20)
+            :                                   :- * Sort (13)
+            :                                   :  +- Exchange (12)
+            :                                   :     +- * Project (11)
+            :                                   :        +- * BroadcastHashJoin Inner BuildRight (10)
+            :                                   :           :- * Project (4)
+            :                                   :           :  +- * Filter (3)
+            :                                   :           :     +- * ColumnarToRow (2)
+            :                                   :           :        +- Scan parquet default.web_sales (1)
+            :                                   :           +- BroadcastExchange (9)
+            :                                   :              +- * Project (8)
+            :                                   :                 +- * Filter (7)
+            :                                   :                    +- * ColumnarToRow (6)
+            :                                   :                       +- Scan parquet default.date_dim (5)
+            :                                   +- * Sort (19)
+            :                                      +- Exchange (18)
+            :                                         +- * Project (17)
+            :                                            +- * Filter (16)
+            :                                               +- * ColumnarToRow (15)
+            :                                                  +- Scan parquet default.web_returns (14)
+            :- * Project (58)
+            :  +- * Filter (57)
+            :     +- Window (56)
+            :        +- * Sort (55)
+            :           +- Window (54)
+            :              +- * Sort (53)
+            :                 +- Exchange (52)
+            :                    +- * HashAggregate (51)
+            :                       +- Exchange (50)
+            :                          +- * HashAggregate (49)
+            :                             +- * Project (48)
+            :                                +- * SortMergeJoin Inner (47)
+            :                                   :- * Sort (40)
+            :                                   :  +- Exchange (39)
+            :                                   :     +- * Project (38)
+            :                                   :        +- * BroadcastHashJoin Inner BuildRight (37)
+            :                                   :           :- * Project (35)
+            :                                   :           :  +- * Filter (34)
+            :                                   :           :     +- * ColumnarToRow (33)
+            :                                   :           :        +- Scan parquet default.catalog_sales (32)
+            :                                   :           +- ReusedExchange (36)
+            :                                   +- * Sort (46)
+            :                                      +- Exchange (45)
+            :                                         +- * Project (44)
+            :                                            +- * Filter (43)
+            :                                               +- * ColumnarToRow (42)
+            :                                                  +- Scan parquet default.catalog_returns (41)
+            +- * Project (85)
+               +- * Filter (84)
+                  +- Window (83)
+                     +- * Sort (82)
+                        +- Window (81)
+                           +- * Sort (80)
+                              +- Exchange (79)
+                                 +- * HashAggregate (78)
+                                    +- Exchange (77)
+                                       +- * HashAggregate (76)
+                                          +- * Project (75)
+                                             +- * SortMergeJoin Inner (74)
+                                                :- * Sort (67)
+                                                :  +- Exchange (66)
+                                                :     +- * Project (65)
+                                                :        +- * BroadcastHashJoin Inner BuildRight (64)
+                                                :           :- * Project (62)
+                                                :           :  +- * Filter (61)
+                                                :           :     +- * ColumnarToRow (60)
+                                                :           :        +- Scan parquet default.store_sales (59)
+                                                :           +- ReusedExchange (63)
+                                                +- * Sort (73)
+                                                   +- Exchange (72)
+                                                      +- * Project (71)
+                                                         +- * Filter (70)
+                                                            +- * ColumnarToRow (69)
+                                                               +- Scan parquet default.store_returns (68)
+
+
+(1) Scan parquet default.web_sales
+Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#6), dynamicpruningexpression(ws_sold_date_sk#6 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ws_net_profit), IsNotNull(ws_net_paid), IsNotNull(ws_quantity), GreaterThan(ws_net_profit,1.00), GreaterThan(ws_net_paid,0.00), GreaterThan(ws_quantity,0), IsNotNull(ws_order_number), IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_order_number:int,ws_quantity:int,ws_net_paid:decimal(7,2),ws_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6]
+
+(3) Filter [codegen id : 2]
+Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6]
+Condition : (((((((isnotnull(ws_net_profit#5) AND isnotnull(ws_net_paid#4)) AND isnotnull(ws_quantity#3)) AND (ws_net_profit#5 > 1.00)) AND (ws_net_paid#4 > 0.00)) AND (ws_quantity#3 > 0)) AND isnotnull(ws_order_number#2)) AND isnotnull(ws_item_sk#1))
+
+(4) Project [codegen id : 2]
+Output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6]
+Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6]
+
+(5) Scan parquet default.date_dim
+Output [3]: [d_date_sk#8, d_year#9, d_moy#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(6) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#8, d_year#9, d_moy#10]
+
+(7) Filter [codegen id : 1]
+Input [3]: [d_date_sk#8, d_year#9, d_moy#10]
+Condition : ((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2001)) AND (d_moy#10 = 12)) AND isnotnull(d_date_sk#8))
+
+(8) Project [codegen id : 1]
+Output [1]: [d_date_sk#8]
+Input [3]: [d_date_sk#8, d_year#9, d_moy#10]
+
+(9) BroadcastExchange
+Input [1]: [d_date_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#11]
+
+(10) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ws_sold_date_sk#6]
+Right keys [1]: [d_date_sk#8]
+Join condition: None
+
+(11) Project [codegen id : 2]
+Output [4]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4]
+Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, d_date_sk#8]
+
+(12) Exchange
+Input [4]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4]
+Arguments: hashpartitioning(ws_order_number#2, ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#12]
+
+(13) Sort [codegen id : 3]
+Input [4]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4]
+Arguments: [ws_order_number#2 ASC NULLS FIRST, ws_item_sk#1 ASC NULLS FIRST], false, 0
+
+(14) Scan parquet default.web_returns
+Output [5]: [wr_item_sk#13, wr_order_number#14, wr_return_quantity#15, wr_return_amt#16, wr_returned_date_sk#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_returns]
+PushedFilters: [IsNotNull(wr_return_amt), GreaterThan(wr_return_amt,10000.00), IsNotNull(wr_order_number), IsNotNull(wr_item_sk)]
+ReadSchema: struct<wr_item_sk:int,wr_order_number:int,wr_return_quantity:int,wr_return_amt:decimal(7,2)>
+
+(15) ColumnarToRow [codegen id : 4]
+Input [5]: [wr_item_sk#13, wr_order_number#14, wr_return_quantity#15, wr_return_amt#16, wr_returned_date_sk#17]
+
+(16) Filter [codegen id : 4]
+Input [5]: [wr_item_sk#13, wr_order_number#14, wr_return_quantity#15, wr_return_amt#16, wr_returned_date_sk#17]
+Condition : (((isnotnull(wr_return_amt#16) AND (wr_return_amt#16 > 10000.00)) AND isnotnull(wr_order_number#14)) AND isnotnull(wr_item_sk#13))
+
+(17) Project [codegen id : 4]
+Output [4]: [wr_item_sk#13, wr_order_number#14, wr_return_quantity#15, wr_return_amt#16]
+Input [5]: [wr_item_sk#13, wr_order_number#14, wr_return_quantity#15, wr_return_amt#16, wr_returned_date_sk#17]
+
+(18) Exchange
+Input [4]: [wr_item_sk#13, wr_order_number#14, wr_return_quantity#15, wr_return_amt#16]
+Arguments: hashpartitioning(wr_order_number#14, wr_item_sk#13, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(19) Sort [codegen id : 5]
+Input [4]: [wr_item_sk#13, wr_order_number#14, wr_return_quantity#15, wr_return_amt#16]
+Arguments: [wr_order_number#14 ASC NULLS FIRST, wr_item_sk#13 ASC NULLS FIRST], false, 0
+
+(20) SortMergeJoin [codegen id : 6]
+Left keys [2]: [ws_order_number#2, ws_item_sk#1]
+Right keys [2]: [wr_order_number#14, wr_item_sk#13]
+Join condition: None
+
+(21) Project [codegen id : 6]
+Output [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#15, wr_return_amt#16]
+Input [8]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, wr_item_sk#13, wr_order_number#14, wr_return_quantity#15, wr_return_amt#16]
+
+(22) HashAggregate [codegen id : 6]
+Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#15, wr_return_amt#16]
+Keys [1]: [ws_item_sk#1]
+Functions [4]: [partial_sum(coalesce(wr_return_quantity#15, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#16 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))]
+Aggregate Attributes [6]: [sum#19, sum#20, sum#21, isEmpty#22, sum#23, isEmpty#24]
+Results [7]: [ws_item_sk#1, sum#25, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30]
+
+(23) Exchange
+Input [7]: [ws_item_sk#1, sum#25, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30]
+Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(24) HashAggregate [codegen id : 7]
+Input [7]: [ws_item_sk#1, sum#25, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30]
+Keys [1]: [ws_item_sk#1]
+Functions [4]: [sum(coalesce(wr_return_quantity#15, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#16 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))]
+Aggregate Attributes [4]: [sum(coalesce(wr_return_quantity#15, 0))#32, sum(coalesce(ws_quantity#3, 0))#33, sum(coalesce(cast(wr_return_amt#16 as decimal(12,2)), 0.00))#34, sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#35]
+Results [3]: [ws_item_sk#1 AS item#36, CheckOverflow((promote_precision(cast(sum(coalesce(wr_return_quantity#15, 0))#32 as decimal(15,4))) / promote_precision(cast(sum(coalesce(ws_quantity#3, 0))#33 as decimal(15,4)))), DecimalType(35,20), true) AS return_ratio#37, CheckOverflow((promote_precision(cast(sum(coalesce(cast(wr_return_amt#16 as decimal(12,2)), 0.00))#34 as decimal(15,4))) / promote_precision(cast(sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#35 as decimal(15,4)))), DecimalType(35,20), true) AS currency_ratio#38]
+
+(25) Exchange
+Input [3]: [item#36, return_ratio#37, currency_ratio#38]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#39]
+
+(26) Sort [codegen id : 8]
+Input [3]: [item#36, return_ratio#37, currency_ratio#38]
+Arguments: [return_ratio#37 ASC NULLS FIRST], false, 0
+
+(27) Window
+Input [3]: [item#36, return_ratio#37, currency_ratio#38]
+Arguments: [rank(return_ratio#37) windowspecdefinition(return_ratio#37 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#40], [return_ratio#37 ASC NULLS FIRST]
+
+(28) Sort [codegen id : 9]
+Input [4]: [item#36, return_ratio#37, currency_ratio#38, return_rank#40]
+Arguments: [currency_ratio#38 ASC NULLS FIRST], false, 0
+
+(29) Window
+Input [4]: [item#36, return_ratio#37, currency_ratio#38, return_rank#40]
+Arguments: [rank(currency_ratio#38) windowspecdefinition(currency_ratio#38 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#41], [currency_ratio#38 ASC NULLS FIRST]
+
+(30) Filter [codegen id : 10]
+Input [5]: [item#36, return_ratio#37, currency_ratio#38, return_rank#40, currency_rank#41]
+Condition : ((return_rank#40 <= 10) OR (currency_rank#41 <= 10))
+
+(31) Project [codegen id : 10]
+Output [5]: [web AS channel#42, item#36, return_ratio#37, return_rank#40, currency_rank#41]
+Input [5]: [item#36, return_ratio#37, currency_ratio#38, return_rank#40, currency_rank#41]
+
+(32) Scan parquet default.catalog_sales
+Output [6]: [cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_net_paid#46, cs_net_profit#47, cs_sold_date_sk#48]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#48), dynamicpruningexpression(cs_sold_date_sk#48 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int,cs_order_number:int,cs_quantity:int,cs_net_paid:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(33) ColumnarToRow [codegen id : 12]
+Input [6]: [cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_net_paid#46, cs_net_profit#47, cs_sold_date_sk#48]
+
+(34) Filter [codegen id : 12]
+Input [6]: [cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_net_paid#46, cs_net_profit#47, cs_sold_date_sk#48]
+Condition : (((((((isnotnull(cs_net_profit#47) AND isnotnull(cs_net_paid#46)) AND isnotnull(cs_quantity#45)) AND (cs_net_profit#47 > 1.00)) AND (cs_net_paid#46 > 0.00)) AND (cs_quantity#45 > 0)) AND isnotnull(cs_order_number#44)) AND isnotnull(cs_item_sk#43))
+
+(35) Project [codegen id : 12]
+Output [5]: [cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_net_paid#46, cs_sold_date_sk#48]
+Input [6]: [cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_net_paid#46, cs_net_profit#47, cs_sold_date_sk#48]
+
+(36) ReusedExchange [Reuses operator id: 9]
+Output [1]: [d_date_sk#49]
+
+(37) BroadcastHashJoin [codegen id : 12]
+Left keys [1]: [cs_sold_date_sk#48]
+Right keys [1]: [d_date_sk#49]
+Join condition: None
+
+(38) Project [codegen id : 12]
+Output [4]: [cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_net_paid#46]
+Input [6]: [cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_net_paid#46, cs_sold_date_sk#48, d_date_sk#49]
+
+(39) Exchange
+Input [4]: [cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_net_paid#46]
+Arguments: hashpartitioning(cs_order_number#44, cs_item_sk#43, 5), ENSURE_REQUIREMENTS, [id=#50]
+
+(40) Sort [codegen id : 13]
+Input [4]: [cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_net_paid#46]
+Arguments: [cs_order_number#44 ASC NULLS FIRST, cs_item_sk#43 ASC NULLS FIRST], false, 0
+
+(41) Scan parquet default.catalog_returns
+Output [5]: [cr_item_sk#51, cr_order_number#52, cr_return_quantity#53, cr_return_amount#54, cr_returned_date_sk#55]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int,cr_return_quantity:int,cr_return_amount:decimal(7,2)>
+
+(42) ColumnarToRow [codegen id : 14]
+Input [5]: [cr_item_sk#51, cr_order_number#52, cr_return_quantity#53, cr_return_amount#54, cr_returned_date_sk#55]
+
+(43) Filter [codegen id : 14]
+Input [5]: [cr_item_sk#51, cr_order_number#52, cr_return_quantity#53, cr_return_amount#54, cr_returned_date_sk#55]
+Condition : (((isnotnull(cr_return_amount#54) AND (cr_return_amount#54 > 10000.00)) AND isnotnull(cr_order_number#52)) AND isnotnull(cr_item_sk#51))
+
+(44) Project [codegen id : 14]
+Output [4]: [cr_item_sk#51, cr_order_number#52, cr_return_quantity#53, cr_return_amount#54]
+Input [5]: [cr_item_sk#51, cr_order_number#52, cr_return_quantity#53, cr_return_amount#54, cr_returned_date_sk#55]
+
+(45) Exchange
+Input [4]: [cr_item_sk#51, cr_order_number#52, cr_return_quantity#53, cr_return_amount#54]
+Arguments: hashpartitioning(cr_order_number#52, cr_item_sk#51, 5), ENSURE_REQUIREMENTS, [id=#56]
+
+(46) Sort [codegen id : 15]
+Input [4]: [cr_item_sk#51, cr_order_number#52, cr_return_quantity#53, cr_return_amount#54]
+Arguments: [cr_order_number#52 ASC NULLS FIRST, cr_item_sk#51 ASC NULLS FIRST], false, 0
+
+(47) SortMergeJoin [codegen id : 16]
+Left keys [2]: [cs_order_number#44, cs_item_sk#43]
+Right keys [2]: [cr_order_number#52, cr_item_sk#51]
+Join condition: None
+
+(48) Project [codegen id : 16]
+Output [5]: [cs_item_sk#43, cs_quantity#45, cs_net_paid#46, cr_return_quantity#53, cr_return_amount#54]
+Input [8]: [cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_net_paid#46, cr_item_sk#51, cr_order_number#52, cr_return_quantity#53, cr_return_amount#54]
+
+(49) HashAggregate [codegen id : 16]
+Input [5]: [cs_item_sk#43, cs_quantity#45, cs_net_paid#46, cr_return_quantity#53, cr_return_amount#54]
+Keys [1]: [cs_item_sk#43]
+Functions [4]: [partial_sum(coalesce(cr_return_quantity#53, 0)), partial_sum(coalesce(cs_quantity#45, 0)), partial_sum(coalesce(cast(cr_return_amount#54 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#46 as decimal(12,2)), 0.00))]
+Aggregate Attributes [6]: [sum#57, sum#58, sum#59, isEmpty#60, sum#61, isEmpty#62]
+Results [7]: [cs_item_sk#43, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68]
+
+(50) Exchange
+Input [7]: [cs_item_sk#43, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68]
+Arguments: hashpartitioning(cs_item_sk#43, 5), ENSURE_REQUIREMENTS, [id=#69]
+
+(51) HashAggregate [codegen id : 17]
+Input [7]: [cs_item_sk#43, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68]
+Keys [1]: [cs_item_sk#43]
+Functions [4]: [sum(coalesce(cr_return_quantity#53, 0)), sum(coalesce(cs_quantity#45, 0)), sum(coalesce(cast(cr_return_amount#54 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#46 as decimal(12,2)), 0.00))]
+Aggregate Attributes [4]: [sum(coalesce(cr_return_quantity#53, 0))#70, sum(coalesce(cs_quantity#45, 0))#71, sum(coalesce(cast(cr_return_amount#54 as decimal(12,2)), 0.00))#72, sum(coalesce(cast(cs_net_paid#46 as decimal(12,2)), 0.00))#73]
+Results [3]: [cs_item_sk#43 AS item#74, CheckOverflow((promote_precision(cast(sum(coalesce(cr_return_quantity#53, 0))#70 as decimal(15,4))) / promote_precision(cast(sum(coalesce(cs_quantity#45, 0))#71 as decimal(15,4)))), DecimalType(35,20), true) AS return_ratio#75, CheckOverflow((promote_precision(cast(sum(coalesce(cast(cr_return_amount#54 as decimal(12,2)), 0.00))#72 as decimal(15,4))) / promote_precision(cast(sum(coalesce(cast(cs_net_paid#46 as decimal(12,2)), 0.00))#73 as decimal(15,4)))), DecimalType(35,20), true) AS currency_ratio#76]
+
+(52) Exchange
+Input [3]: [item#74, return_ratio#75, currency_ratio#76]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#77]
+
+(53) Sort [codegen id : 18]
+Input [3]: [item#74, return_ratio#75, currency_ratio#76]
+Arguments: [return_ratio#75 ASC NULLS FIRST], false, 0
+
+(54) Window
+Input [3]: [item#74, return_ratio#75, currency_ratio#76]
+Arguments: [rank(return_ratio#75) windowspecdefinition(return_ratio#75 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#78], [return_ratio#75 ASC NULLS FIRST]
+
+(55) Sort [codegen id : 19]
+Input [4]: [item#74, return_ratio#75, currency_ratio#76, return_rank#78]
+Arguments: [currency_ratio#76 ASC NULLS FIRST], false, 0
+
+(56) Window
+Input [4]: [item#74, return_ratio#75, currency_ratio#76, return_rank#78]
+Arguments: [rank(currency_ratio#76) windowspecdefinition(currency_ratio#76 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#79], [currency_ratio#76 ASC NULLS FIRST]
+
+(57) Filter [codegen id : 20]
+Input [5]: [item#74, return_ratio#75, currency_ratio#76, return_rank#78, currency_rank#79]
+Condition : ((return_rank#78 <= 10) OR (currency_rank#79 <= 10))
+
+(58) Project [codegen id : 20]
+Output [5]: [catalog AS channel#80, item#74, return_ratio#75, return_rank#78, currency_rank#79]
+Input [5]: [item#74, return_ratio#75, currency_ratio#76, return_rank#78, currency_rank#79]
+
+(59) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_net_paid#84, ss_net_profit#85, ss_sold_date_sk#86]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#86), dynamicpruningexpression(ss_sold_date_sk#86 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ticket_number:int,ss_quantity:int,ss_net_paid:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(60) ColumnarToRow [codegen id : 22]
+Input [6]: [ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_net_paid#84, ss_net_profit#85, ss_sold_date_sk#86]
+
+(61) Filter [codegen id : 22]
+Input [6]: [ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_net_paid#84, ss_net_profit#85, ss_sold_date_sk#86]
+Condition : (((((((isnotnull(ss_net_profit#85) AND isnotnull(ss_net_paid#84)) AND isnotnull(ss_quantity#83)) AND (ss_net_profit#85 > 1.00)) AND (ss_net_paid#84 > 0.00)) AND (ss_quantity#83 > 0)) AND isnotnull(ss_ticket_number#82)) AND isnotnull(ss_item_sk#81))
+
+(62) Project [codegen id : 22]
+Output [5]: [ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_net_paid#84, ss_sold_date_sk#86]
+Input [6]: [ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_net_paid#84, ss_net_profit#85, ss_sold_date_sk#86]
+
+(63) ReusedExchange [Reuses operator id: 9]
+Output [1]: [d_date_sk#87]
+
+(64) BroadcastHashJoin [codegen id : 22]
+Left keys [1]: [ss_sold_date_sk#86]
+Right keys [1]: [d_date_sk#87]
+Join condition: None
+
+(65) Project [codegen id : 22]
+Output [4]: [ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_net_paid#84]
+Input [6]: [ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_net_paid#84, ss_sold_date_sk#86, d_date_sk#87]
+
+(66) Exchange
+Input [4]: [ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_net_paid#84]
+Arguments: hashpartitioning(ss_ticket_number#82, ss_item_sk#81, 5), ENSURE_REQUIREMENTS, [id=#88]
+
+(67) Sort [codegen id : 23]
+Input [4]: [ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_net_paid#84]
+Arguments: [ss_ticket_number#82 ASC NULLS FIRST, ss_item_sk#81 ASC NULLS FIRST], false, 0
+
+(68) Scan parquet default.store_returns
+Output [5]: [sr_item_sk#89, sr_ticket_number#90, sr_return_quantity#91, sr_return_amt#92, sr_returned_date_sk#93]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int,sr_return_quantity:int,sr_return_amt:decimal(7,2)>
+
+(69) ColumnarToRow [codegen id : 24]
+Input [5]: [sr_item_sk#89, sr_ticket_number#90, sr_return_quantity#91, sr_return_amt#92, sr_returned_date_sk#93]
+
+(70) Filter [codegen id : 24]
+Input [5]: [sr_item_sk#89, sr_ticket_number#90, sr_return_quantity#91, sr_return_amt#92, sr_returned_date_sk#93]
+Condition : (((isnotnull(sr_return_amt#92) AND (sr_return_amt#92 > 10000.00)) AND isnotnull(sr_ticket_number#90)) AND isnotnull(sr_item_sk#89))
+
+(71) Project [codegen id : 24]
+Output [4]: [sr_item_sk#89, sr_ticket_number#90, sr_return_quantity#91, sr_return_amt#92]
+Input [5]: [sr_item_sk#89, sr_ticket_number#90, sr_return_quantity#91, sr_return_amt#92, sr_returned_date_sk#93]
+
+(72) Exchange
+Input [4]: [sr_item_sk#89, sr_ticket_number#90, sr_return_quantity#91, sr_return_amt#92]
+Arguments: hashpartitioning(sr_ticket_number#90, sr_item_sk#89, 5), ENSURE_REQUIREMENTS, [id=#94]
+
+(73) Sort [codegen id : 25]
+Input [4]: [sr_item_sk#89, sr_ticket_number#90, sr_return_quantity#91, sr_return_amt#92]
+Arguments: [sr_ticket_number#90 ASC NULLS FIRST, sr_item_sk#89 ASC NULLS FIRST], false, 0
+
+(74) SortMergeJoin [codegen id : 26]
+Left keys [2]: [ss_ticket_number#82, ss_item_sk#81]
+Right keys [2]: [sr_ticket_number#90, sr_item_sk#89]
+Join condition: None
+
+(75) Project [codegen id : 26]
+Output [5]: [ss_item_sk#81, ss_quantity#83, ss_net_paid#84, sr_return_quantity#91, sr_return_amt#92]
+Input [8]: [ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_net_paid#84, sr_item_sk#89, sr_ticket_number#90, sr_return_quantity#91, sr_return_amt#92]
+
+(76) HashAggregate [codegen id : 26]
+Input [5]: [ss_item_sk#81, ss_quantity#83, ss_net_paid#84, sr_return_quantity#91, sr_return_amt#92]
+Keys [1]: [ss_item_sk#81]
+Functions [4]: [partial_sum(coalesce(sr_return_quantity#91, 0)), partial_sum(coalesce(ss_quantity#83, 0)), partial_sum(coalesce(cast(sr_return_amt#92 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#84 as decimal(12,2)), 0.00))]
+Aggregate Attributes [6]: [sum#95, sum#96, sum#97, isEmpty#98, sum#99, isEmpty#100]
+Results [7]: [ss_item_sk#81, sum#101, sum#102, sum#103, isEmpty#104, sum#105, isEmpty#106]
+
+(77) Exchange
+Input [7]: [ss_item_sk#81, sum#101, sum#102, sum#103, isEmpty#104, sum#105, isEmpty#106]
+Arguments: hashpartitioning(ss_item_sk#81, 5), ENSURE_REQUIREMENTS, [id=#107]
+
+(78) HashAggregate [codegen id : 27]
+Input [7]: [ss_item_sk#81, sum#101, sum#102, sum#103, isEmpty#104, sum#105, isEmpty#106]
+Keys [1]: [ss_item_sk#81]
+Functions [4]: [sum(coalesce(sr_return_quantity#91, 0)), sum(coalesce(ss_quantity#83, 0)), sum(coalesce(cast(sr_return_amt#92 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#84 as decimal(12,2)), 0.00))]
+Aggregate Attributes [4]: [sum(coalesce(sr_return_quantity#91, 0))#108, sum(coalesce(ss_quantity#83, 0))#109, sum(coalesce(cast(sr_return_amt#92 as decimal(12,2)), 0.00))#110, sum(coalesce(cast(ss_net_paid#84 as decimal(12,2)), 0.00))#111]
+Results [3]: [ss_item_sk#81 AS item#112, CheckOverflow((promote_precision(cast(sum(coalesce(sr_return_quantity#91, 0))#108 as decimal(15,4))) / promote_precision(cast(sum(coalesce(ss_quantity#83, 0))#109 as decimal(15,4)))), DecimalType(35,20), true) AS return_ratio#113, CheckOverflow((promote_precision(cast(sum(coalesce(cast(sr_return_amt#92 as decimal(12,2)), 0.00))#110 as decimal(15,4))) / promote_precision(cast(sum(coalesce(cast(ss_net_paid#84 as decimal(12,2)), 0.00))#111 as decimal(15,4)))), DecimalType(35,20), true) AS currency_ratio#114]
+
+(79) Exchange
+Input [3]: [item#112, return_ratio#113, currency_ratio#114]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#115]
+
+(80) Sort [codegen id : 28]
+Input [3]: [item#112, return_ratio#113, currency_ratio#114]
+Arguments: [return_ratio#113 ASC NULLS FIRST], false, 0
+
+(81) Window
+Input [3]: [item#112, return_ratio#113, currency_ratio#114]
+Arguments: [rank(return_ratio#113) windowspecdefinition(return_ratio#113 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#116], [return_ratio#113 ASC NULLS FIRST]
+
+(82) Sort [codegen id : 29]
+Input [4]: [item#112, return_ratio#113, currency_ratio#114, return_rank#116]
+Arguments: [currency_ratio#114 ASC NULLS FIRST], false, 0
+
+(83) Window
+Input [4]: [item#112, return_ratio#113, currency_ratio#114, return_rank#116]
+Arguments: [rank(currency_ratio#114) windowspecdefinition(currency_ratio#114 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#117], [currency_ratio#114 ASC NULLS FIRST]
+
+(84) Filter [codegen id : 30]
+Input [5]: [item#112, return_ratio#113, currency_ratio#114, return_rank#116, currency_rank#117]
+Condition : ((return_rank#116 <= 10) OR (currency_rank#117 <= 10))
+
+(85) Project [codegen id : 30]
+Output [5]: [store AS channel#118, item#112, return_ratio#113, return_rank#116, currency_rank#117]
+Input [5]: [item#112, return_ratio#113, currency_ratio#114, return_rank#116, currency_rank#117]
+
+(86) Union
+
+(87) HashAggregate [codegen id : 31]
+Input [5]: [channel#42, item#36, return_ratio#37, return_rank#40, currency_rank#41]
+Keys [5]: [channel#42, item#36, return_ratio#37, return_rank#40, currency_rank#41]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [channel#42, item#36, return_ratio#37, return_rank#40, currency_rank#41]
+
+(88) Exchange
+Input [5]: [channel#42, item#36, return_ratio#37, return_rank#40, currency_rank#41]
+Arguments: hashpartitioning(channel#42, item#36, return_ratio#37, return_rank#40, currency_rank#41, 5), ENSURE_REQUIREMENTS, [id=#119]
+
+(89) HashAggregate [codegen id : 32]
+Input [5]: [channel#42, item#36, return_ratio#37, return_rank#40, currency_rank#41]
+Keys [5]: [channel#42, item#36, return_ratio#37, return_rank#40, currency_rank#41]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [channel#42, item#36, return_ratio#37, return_rank#40, currency_rank#41]
+
+(90) TakeOrderedAndProject
+Input [5]: [channel#42, item#36, return_ratio#37, return_rank#40, currency_rank#41]
+Arguments: 100, [channel#42 ASC NULLS FIRST, return_rank#40 ASC NULLS FIRST, currency_rank#41 ASC NULLS FIRST, item#36 ASC NULLS FIRST], [channel#42, item#36, return_ratio#37, return_rank#40, currency_rank#41]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#6 IN dynamicpruning#7
+ReusedExchange (91)
+
+
+(91) ReusedExchange [Reuses operator id: 9]
+Output [1]: [d_date_sk#8]
+
+Subquery:2 Hosting operator id = 32 Hosting Expression = cs_sold_date_sk#48 IN dynamicpruning#7
+
+Subquery:3 Hosting operator id = 59 Hosting Expression = ss_sold_date_sk#86 IN dynamicpruning#7
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.sf100/simplified.txt
new file mode 100644
index 0000000000000..60d16f8b606af
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.sf100/simplified.txt
@@ -0,0 +1,160 @@
+TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio]
+  WholeStageCodegen (32)
+    HashAggregate [channel,item,return_ratio,return_rank,currency_rank]
+      InputAdapter
+        Exchange [channel,item,return_ratio,return_rank,currency_rank] #1
+          WholeStageCodegen (31)
+            HashAggregate [channel,item,return_ratio,return_rank,currency_rank]
+              InputAdapter
+                Union
+                  WholeStageCodegen (10)
+                    Project [item,return_ratio,return_rank,currency_rank]
+                      Filter [return_rank,currency_rank]
+                        InputAdapter
+                          Window [currency_ratio]
+                            WholeStageCodegen (9)
+                              Sort [currency_ratio]
+                                InputAdapter
+                                  Window [return_ratio]
+                                    WholeStageCodegen (8)
+                                      Sort [return_ratio]
+                                        InputAdapter
+                                          Exchange #2
+                                            WholeStageCodegen (7)
+                                              HashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty]
+                                                InputAdapter
+                                                  Exchange [ws_item_sk] #3
+                                                    WholeStageCodegen (6)
+                                                      HashAggregate [ws_item_sk,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty]
+                                                        Project [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt]
+                                                          SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk]
+                                                            InputAdapter
+                                                              WholeStageCodegen (3)
+                                                                Sort [ws_order_number,ws_item_sk]
+                                                                  InputAdapter
+                                                                    Exchange [ws_order_number,ws_item_sk] #4
+                                                                      WholeStageCodegen (2)
+                                                                        Project [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid]
+                                                                          BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                            Project [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk]
+                                                                              Filter [ws_net_profit,ws_net_paid,ws_quantity,ws_order_number,ws_item_sk]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk]
+                                                                                      SubqueryBroadcast [d_date_sk] #1
+                                                                                        ReusedExchange [d_date_sk] #5
+                                                                            InputAdapter
+                                                                              BroadcastExchange #5
+                                                                                WholeStageCodegen (1)
+                                                                                  Project [d_date_sk]
+                                                                                    Filter [d_year,d_moy,d_date_sk]
+                                                                                      ColumnarToRow
+                                                                                        InputAdapter
+                                                                                          Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                                            InputAdapter
+                                                              WholeStageCodegen (5)
+                                                                Sort [wr_order_number,wr_item_sk]
+                                                                  InputAdapter
+                                                                    Exchange [wr_order_number,wr_item_sk] #6
+                                                                      WholeStageCodegen (4)
+                                                                        Project [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt]
+                                                                          Filter [wr_return_amt,wr_order_number,wr_item_sk]
+                                                                            ColumnarToRow
+                                                                              InputAdapter
+                                                                                Scan parquet default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk]
+                  WholeStageCodegen (20)
+                    Project [item,return_ratio,return_rank,currency_rank]
+                      Filter [return_rank,currency_rank]
+                        InputAdapter
+                          Window [currency_ratio]
+                            WholeStageCodegen (19)
+                              Sort [currency_ratio]
+                                InputAdapter
+                                  Window [return_ratio]
+                                    WholeStageCodegen (18)
+                                      Sort [return_ratio]
+                                        InputAdapter
+                                          Exchange #7
+                                            WholeStageCodegen (17)
+                                              HashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty]
+                                                InputAdapter
+                                                  Exchange [cs_item_sk] #8
+                                                    WholeStageCodegen (16)
+                                                      HashAggregate [cs_item_sk,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty]
+                                                        Project [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount]
+                                                          SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk]
+                                                            InputAdapter
+                                                              WholeStageCodegen (13)
+                                                                Sort [cs_order_number,cs_item_sk]
+                                                                  InputAdapter
+                                                                    Exchange [cs_order_number,cs_item_sk] #9
+                                                                      WholeStageCodegen (12)
+                                                                        Project [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid]
+                                                                          BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                            Project [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk]
+                                                                              Filter [cs_net_profit,cs_net_paid,cs_quantity,cs_order_number,cs_item_sk]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk]
+                                                                                      ReusedSubquery [d_date_sk] #1
+                                                                            InputAdapter
+                                                                              ReusedExchange [d_date_sk] #5
+                                                            InputAdapter
+                                                              WholeStageCodegen (15)
+                                                                Sort [cr_order_number,cr_item_sk]
+                                                                  InputAdapter
+                                                                    Exchange [cr_order_number,cr_item_sk] #10
+                                                                      WholeStageCodegen (14)
+                                                                        Project [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount]
+                                                                          Filter [cr_return_amount,cr_order_number,cr_item_sk]
+                                                                            ColumnarToRow
+                                                                              InputAdapter
+                                                                                Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk]
+                  WholeStageCodegen (30)
+                    Project [item,return_ratio,return_rank,currency_rank]
+                      Filter [return_rank,currency_rank]
+                        InputAdapter
+                          Window [currency_ratio]
+                            WholeStageCodegen (29)
+                              Sort [currency_ratio]
+                                InputAdapter
+                                  Window [return_ratio]
+                                    WholeStageCodegen (28)
+                                      Sort [return_ratio]
+                                        InputAdapter
+                                          Exchange #11
+                                            WholeStageCodegen (27)
+                                              HashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty]
+                                                InputAdapter
+                                                  Exchange [ss_item_sk] #12
+                                                    WholeStageCodegen (26)
+                                                      HashAggregate [ss_item_sk,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty]
+                                                        Project [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt]
+                                                          SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                                            InputAdapter
+                                                              WholeStageCodegen (23)
+                                                                Sort [ss_ticket_number,ss_item_sk]
+                                                                  InputAdapter
+                                                                    Exchange [ss_ticket_number,ss_item_sk] #13
+                                                                      WholeStageCodegen (22)
+                                                                        Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid]
+                                                                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                            Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk]
+                                                                              Filter [ss_net_profit,ss_net_paid,ss_quantity,ss_ticket_number,ss_item_sk]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk]
+                                                                                      ReusedSubquery [d_date_sk] #1
+                                                                            InputAdapter
+                                                                              ReusedExchange [d_date_sk] #5
+                                                            InputAdapter
+                                                              WholeStageCodegen (25)
+                                                                Sort [sr_ticket_number,sr_item_sk]
+                                                                  InputAdapter
+                                                                    Exchange [sr_ticket_number,sr_item_sk] #14
+                                                                      WholeStageCodegen (24)
+                                                                        Project [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt]
+                                                                          Filter [sr_return_amt,sr_ticket_number,sr_item_sk]
+                                                                            ColumnarToRow
+                                                                              InputAdapter
+                                                                                Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/explain.txt
new file mode 100644
index 0000000000000..1665244594850
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/explain.txt
@@ -0,0 +1,465 @@
+== Physical Plan ==
+TakeOrderedAndProject (81)
++- * HashAggregate (80)
+   +- Exchange (79)
+      +- * HashAggregate (78)
+         +- Union (77)
+            :- * Project (28)
+            :  +- * Filter (27)
+            :     +- Window (26)
+            :        +- * Sort (25)
+            :           +- Window (24)
+            :              +- * Sort (23)
+            :                 +- Exchange (22)
+            :                    +- * HashAggregate (21)
+            :                       +- Exchange (20)
+            :                          +- * HashAggregate (19)
+            :                             +- * Project (18)
+            :                                +- * BroadcastHashJoin Inner BuildRight (17)
+            :                                   :- * Project (11)
+            :                                   :  +- * BroadcastHashJoin Inner BuildLeft (10)
+            :                                   :     :- BroadcastExchange (5)
+            :                                   :     :  +- * Project (4)
+            :                                   :     :     +- * Filter (3)
+            :                                   :     :        +- * ColumnarToRow (2)
+            :                                   :     :           +- Scan parquet default.web_sales (1)
+            :                                   :     +- * Project (9)
+            :                                   :        +- * Filter (8)
+            :                                   :           +- * ColumnarToRow (7)
+            :                                   :              +- Scan parquet default.web_returns (6)
+            :                                   +- BroadcastExchange (16)
+            :                                      +- * Project (15)
+            :                                         +- * Filter (14)
+            :                                            +- * ColumnarToRow (13)
+            :                                               +- Scan parquet default.date_dim (12)
+            :- * Project (52)
+            :  +- * Filter (51)
+            :     +- Window (50)
+            :        +- * Sort (49)
+            :           +- Window (48)
+            :              +- * Sort (47)
+            :                 +- Exchange (46)
+            :                    +- * HashAggregate (45)
+            :                       +- Exchange (44)
+            :                          +- * HashAggregate (43)
+            :                             +- * Project (42)
+            :                                +- * BroadcastHashJoin Inner BuildRight (41)
+            :                                   :- * Project (39)
+            :                                   :  +- * BroadcastHashJoin Inner BuildLeft (38)
+            :                                   :     :- BroadcastExchange (33)
+            :                                   :     :  +- * Project (32)
+            :                                   :     :     +- * Filter (31)
+            :                                   :     :        +- * ColumnarToRow (30)
+            :                                   :     :           +- Scan parquet default.catalog_sales (29)
+            :                                   :     +- * Project (37)
+            :                                   :        +- * Filter (36)
+            :                                   :           +- * ColumnarToRow (35)
+            :                                   :              +- Scan parquet default.catalog_returns (34)
+            :                                   +- ReusedExchange (40)
+            +- * Project (76)
+               +- * Filter (75)
+                  +- Window (74)
+                     +- * Sort (73)
+                        +- Window (72)
+                           +- * Sort (71)
+                              +- Exchange (70)
+                                 +- * HashAggregate (69)
+                                    +- Exchange (68)
+                                       +- * HashAggregate (67)
+                                          +- * Project (66)
+                                             +- * BroadcastHashJoin Inner BuildRight (65)
+                                                :- * Project (63)
+                                                :  +- * BroadcastHashJoin Inner BuildLeft (62)
+                                                :     :- BroadcastExchange (57)
+                                                :     :  +- * Project (56)
+                                                :     :     +- * Filter (55)
+                                                :     :        +- * ColumnarToRow (54)
+                                                :     :           +- Scan parquet default.store_sales (53)
+                                                :     +- * Project (61)
+                                                :        +- * Filter (60)
+                                                :           +- * ColumnarToRow (59)
+                                                :              +- Scan parquet default.store_returns (58)
+                                                +- ReusedExchange (64)
+
+
+(1) Scan parquet default.web_sales
+Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#6), dynamicpruningexpression(ws_sold_date_sk#6 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ws_net_profit), IsNotNull(ws_net_paid), IsNotNull(ws_quantity), GreaterThan(ws_net_profit,1.00), GreaterThan(ws_net_paid,0.00), GreaterThan(ws_quantity,0), IsNotNull(ws_order_number), IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_order_number:int,ws_quantity:int,ws_net_paid:decimal(7,2),ws_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6]
+
+(3) Filter [codegen id : 1]
+Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6]
+Condition : (((((((isnotnull(ws_net_profit#5) AND isnotnull(ws_net_paid#4)) AND isnotnull(ws_quantity#3)) AND (ws_net_profit#5 > 1.00)) AND (ws_net_paid#4 > 0.00)) AND (ws_quantity#3 > 0)) AND isnotnull(ws_order_number#2)) AND isnotnull(ws_item_sk#1))
+
+(4) Project [codegen id : 1]
+Output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6]
+Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6]
+
+(5) BroadcastExchange
+Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [id=#8]
+
+(6) Scan parquet default.web_returns
+Output [5]: [wr_item_sk#9, wr_order_number#10, wr_return_quantity#11, wr_return_amt#12, wr_returned_date_sk#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_returns]
+PushedFilters: [IsNotNull(wr_return_amt), GreaterThan(wr_return_amt,10000.00), IsNotNull(wr_order_number), IsNotNull(wr_item_sk)]
+ReadSchema: struct<wr_item_sk:int,wr_order_number:int,wr_return_quantity:int,wr_return_amt:decimal(7,2)>
+
+(7) ColumnarToRow
+Input [5]: [wr_item_sk#9, wr_order_number#10, wr_return_quantity#11, wr_return_amt#12, wr_returned_date_sk#13]
+
+(8) Filter
+Input [5]: [wr_item_sk#9, wr_order_number#10, wr_return_quantity#11, wr_return_amt#12, wr_returned_date_sk#13]
+Condition : (((isnotnull(wr_return_amt#12) AND (wr_return_amt#12 > 10000.00)) AND isnotnull(wr_order_number#10)) AND isnotnull(wr_item_sk#9))
+
+(9) Project
+Output [4]: [wr_item_sk#9, wr_order_number#10, wr_return_quantity#11, wr_return_amt#12]
+Input [5]: [wr_item_sk#9, wr_order_number#10, wr_return_quantity#11, wr_return_amt#12, wr_returned_date_sk#13]
+
+(10) BroadcastHashJoin [codegen id : 3]
+Left keys [2]: [ws_order_number#2, ws_item_sk#1]
+Right keys [2]: [wr_order_number#10, wr_item_sk#9]
+Join condition: None
+
+(11) Project [codegen id : 3]
+Output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#11, wr_return_amt#12]
+Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#9, wr_order_number#10, wr_return_quantity#11, wr_return_amt#12]
+
+(12) Scan parquet default.date_dim
+Output [3]: [d_date_sk#14, d_year#15, d_moy#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(13) ColumnarToRow [codegen id : 2]
+Input [3]: [d_date_sk#14, d_year#15, d_moy#16]
+
+(14) Filter [codegen id : 2]
+Input [3]: [d_date_sk#14, d_year#15, d_moy#16]
+Condition : ((((isnotnull(d_year#15) AND isnotnull(d_moy#16)) AND (d_year#15 = 2001)) AND (d_moy#16 = 12)) AND isnotnull(d_date_sk#14))
+
+(15) Project [codegen id : 2]
+Output [1]: [d_date_sk#14]
+Input [3]: [d_date_sk#14, d_year#15, d_moy#16]
+
+(16) BroadcastExchange
+Input [1]: [d_date_sk#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#17]
+
+(17) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ws_sold_date_sk#6]
+Right keys [1]: [d_date_sk#14]
+Join condition: None
+
+(18) Project [codegen id : 3]
+Output [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#11, wr_return_amt#12]
+Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#11, wr_return_amt#12, d_date_sk#14]
+
+(19) HashAggregate [codegen id : 3]
+Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#11, wr_return_amt#12]
+Keys [1]: [ws_item_sk#1]
+Functions [4]: [partial_sum(coalesce(wr_return_quantity#11, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#12 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))]
+Aggregate Attributes [6]: [sum#18, sum#19, sum#20, isEmpty#21, sum#22, isEmpty#23]
+Results [7]: [ws_item_sk#1, sum#24, sum#25, sum#26, isEmpty#27, sum#28, isEmpty#29]
+
+(20) Exchange
+Input [7]: [ws_item_sk#1, sum#24, sum#25, sum#26, isEmpty#27, sum#28, isEmpty#29]
+Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(21) HashAggregate [codegen id : 4]
+Input [7]: [ws_item_sk#1, sum#24, sum#25, sum#26, isEmpty#27, sum#28, isEmpty#29]
+Keys [1]: [ws_item_sk#1]
+Functions [4]: [sum(coalesce(wr_return_quantity#11, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#12 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))]
+Aggregate Attributes [4]: [sum(coalesce(wr_return_quantity#11, 0))#31, sum(coalesce(ws_quantity#3, 0))#32, sum(coalesce(cast(wr_return_amt#12 as decimal(12,2)), 0.00))#33, sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#34]
+Results [3]: [ws_item_sk#1 AS item#35, CheckOverflow((promote_precision(cast(sum(coalesce(wr_return_quantity#11, 0))#31 as decimal(15,4))) / promote_precision(cast(sum(coalesce(ws_quantity#3, 0))#32 as decimal(15,4)))), DecimalType(35,20), true) AS return_ratio#36, CheckOverflow((promote_precision(cast(sum(coalesce(cast(wr_return_amt#12 as decimal(12,2)), 0.00))#33 as decimal(15,4))) / promote_precision(cast(sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#34 as decimal(15,4)))), DecimalType(35,20), true) AS currency_ratio#37]
+
+(22) Exchange
+Input [3]: [item#35, return_ratio#36, currency_ratio#37]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#38]
+
+(23) Sort [codegen id : 5]
+Input [3]: [item#35, return_ratio#36, currency_ratio#37]
+Arguments: [return_ratio#36 ASC NULLS FIRST], false, 0
+
+(24) Window
+Input [3]: [item#35, return_ratio#36, currency_ratio#37]
+Arguments: [rank(return_ratio#36) windowspecdefinition(return_ratio#36 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#39], [return_ratio#36 ASC NULLS FIRST]
+
+(25) Sort [codegen id : 6]
+Input [4]: [item#35, return_ratio#36, currency_ratio#37, return_rank#39]
+Arguments: [currency_ratio#37 ASC NULLS FIRST], false, 0
+
+(26) Window
+Input [4]: [item#35, return_ratio#36, currency_ratio#37, return_rank#39]
+Arguments: [rank(currency_ratio#37) windowspecdefinition(currency_ratio#37 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#40], [currency_ratio#37 ASC NULLS FIRST]
+
+(27) Filter [codegen id : 7]
+Input [5]: [item#35, return_ratio#36, currency_ratio#37, return_rank#39, currency_rank#40]
+Condition : ((return_rank#39 <= 10) OR (currency_rank#40 <= 10))
+
+(28) Project [codegen id : 7]
+Output [5]: [web AS channel#41, item#35, return_ratio#36, return_rank#39, currency_rank#40]
+Input [5]: [item#35, return_ratio#36, currency_ratio#37, return_rank#39, currency_rank#40]
+
+(29) Scan parquet default.catalog_sales
+Output [6]: [cs_item_sk#42, cs_order_number#43, cs_quantity#44, cs_net_paid#45, cs_net_profit#46, cs_sold_date_sk#47]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#47), dynamicpruningexpression(cs_sold_date_sk#47 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int,cs_order_number:int,cs_quantity:int,cs_net_paid:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(30) ColumnarToRow [codegen id : 8]
+Input [6]: [cs_item_sk#42, cs_order_number#43, cs_quantity#44, cs_net_paid#45, cs_net_profit#46, cs_sold_date_sk#47]
+
+(31) Filter [codegen id : 8]
+Input [6]: [cs_item_sk#42, cs_order_number#43, cs_quantity#44, cs_net_paid#45, cs_net_profit#46, cs_sold_date_sk#47]
+Condition : (((((((isnotnull(cs_net_profit#46) AND isnotnull(cs_net_paid#45)) AND isnotnull(cs_quantity#44)) AND (cs_net_profit#46 > 1.00)) AND (cs_net_paid#45 > 0.00)) AND (cs_quantity#44 > 0)) AND isnotnull(cs_order_number#43)) AND isnotnull(cs_item_sk#42))
+
+(32) Project [codegen id : 8]
+Output [5]: [cs_item_sk#42, cs_order_number#43, cs_quantity#44, cs_net_paid#45, cs_sold_date_sk#47]
+Input [6]: [cs_item_sk#42, cs_order_number#43, cs_quantity#44, cs_net_paid#45, cs_net_profit#46, cs_sold_date_sk#47]
+
+(33) BroadcastExchange
+Input [5]: [cs_item_sk#42, cs_order_number#43, cs_quantity#44, cs_net_paid#45, cs_sold_date_sk#47]
+Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [id=#48]
+
+(34) Scan parquet default.catalog_returns
+Output [5]: [cr_item_sk#49, cr_order_number#50, cr_return_quantity#51, cr_return_amount#52, cr_returned_date_sk#53]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int,cr_return_quantity:int,cr_return_amount:decimal(7,2)>
+
+(35) ColumnarToRow
+Input [5]: [cr_item_sk#49, cr_order_number#50, cr_return_quantity#51, cr_return_amount#52, cr_returned_date_sk#53]
+
+(36) Filter
+Input [5]: [cr_item_sk#49, cr_order_number#50, cr_return_quantity#51, cr_return_amount#52, cr_returned_date_sk#53]
+Condition : (((isnotnull(cr_return_amount#52) AND (cr_return_amount#52 > 10000.00)) AND isnotnull(cr_order_number#50)) AND isnotnull(cr_item_sk#49))
+
+(37) Project
+Output [4]: [cr_item_sk#49, cr_order_number#50, cr_return_quantity#51, cr_return_amount#52]
+Input [5]: [cr_item_sk#49, cr_order_number#50, cr_return_quantity#51, cr_return_amount#52, cr_returned_date_sk#53]
+
+(38) BroadcastHashJoin [codegen id : 10]
+Left keys [2]: [cs_order_number#43, cs_item_sk#42]
+Right keys [2]: [cr_order_number#50, cr_item_sk#49]
+Join condition: None
+
+(39) Project [codegen id : 10]
+Output [6]: [cs_item_sk#42, cs_quantity#44, cs_net_paid#45, cs_sold_date_sk#47, cr_return_quantity#51, cr_return_amount#52]
+Input [9]: [cs_item_sk#42, cs_order_number#43, cs_quantity#44, cs_net_paid#45, cs_sold_date_sk#47, cr_item_sk#49, cr_order_number#50, cr_return_quantity#51, cr_return_amount#52]
+
+(40) ReusedExchange [Reuses operator id: 16]
+Output [1]: [d_date_sk#54]
+
+(41) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_sold_date_sk#47]
+Right keys [1]: [d_date_sk#54]
+Join condition: None
+
+(42) Project [codegen id : 10]
+Output [5]: [cs_item_sk#42, cs_quantity#44, cs_net_paid#45, cr_return_quantity#51, cr_return_amount#52]
+Input [7]: [cs_item_sk#42, cs_quantity#44, cs_net_paid#45, cs_sold_date_sk#47, cr_return_quantity#51, cr_return_amount#52, d_date_sk#54]
+
+(43) HashAggregate [codegen id : 10]
+Input [5]: [cs_item_sk#42, cs_quantity#44, cs_net_paid#45, cr_return_quantity#51, cr_return_amount#52]
+Keys [1]: [cs_item_sk#42]
+Functions [4]: [partial_sum(coalesce(cr_return_quantity#51, 0)), partial_sum(coalesce(cs_quantity#44, 0)), partial_sum(coalesce(cast(cr_return_amount#52 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#45 as decimal(12,2)), 0.00))]
+Aggregate Attributes [6]: [sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60]
+Results [7]: [cs_item_sk#42, sum#61, sum#62, sum#63, isEmpty#64, sum#65, isEmpty#66]
+
+(44) Exchange
+Input [7]: [cs_item_sk#42, sum#61, sum#62, sum#63, isEmpty#64, sum#65, isEmpty#66]
+Arguments: hashpartitioning(cs_item_sk#42, 5), ENSURE_REQUIREMENTS, [id=#67]
+
+(45) HashAggregate [codegen id : 11]
+Input [7]: [cs_item_sk#42, sum#61, sum#62, sum#63, isEmpty#64, sum#65, isEmpty#66]
+Keys [1]: [cs_item_sk#42]
+Functions [4]: [sum(coalesce(cr_return_quantity#51, 0)), sum(coalesce(cs_quantity#44, 0)), sum(coalesce(cast(cr_return_amount#52 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#45 as decimal(12,2)), 0.00))]
+Aggregate Attributes [4]: [sum(coalesce(cr_return_quantity#51, 0))#68, sum(coalesce(cs_quantity#44, 0))#69, sum(coalesce(cast(cr_return_amount#52 as decimal(12,2)), 0.00))#70, sum(coalesce(cast(cs_net_paid#45 as decimal(12,2)), 0.00))#71]
+Results [3]: [cs_item_sk#42 AS item#72, CheckOverflow((promote_precision(cast(sum(coalesce(cr_return_quantity#51, 0))#68 as decimal(15,4))) / promote_precision(cast(sum(coalesce(cs_quantity#44, 0))#69 as decimal(15,4)))), DecimalType(35,20), true) AS return_ratio#73, CheckOverflow((promote_precision(cast(sum(coalesce(cast(cr_return_amount#52 as decimal(12,2)), 0.00))#70 as decimal(15,4))) / promote_precision(cast(sum(coalesce(cast(cs_net_paid#45 as decimal(12,2)), 0.00))#71 as decimal(15,4)))), DecimalType(35,20), true) AS currency_ratio#74]
+
+(46) Exchange
+Input [3]: [item#72, return_ratio#73, currency_ratio#74]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#75]
+
+(47) Sort [codegen id : 12]
+Input [3]: [item#72, return_ratio#73, currency_ratio#74]
+Arguments: [return_ratio#73 ASC NULLS FIRST], false, 0
+
+(48) Window
+Input [3]: [item#72, return_ratio#73, currency_ratio#74]
+Arguments: [rank(return_ratio#73) windowspecdefinition(return_ratio#73 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#76], [return_ratio#73 ASC NULLS FIRST]
+
+(49) Sort [codegen id : 13]
+Input [4]: [item#72, return_ratio#73, currency_ratio#74, return_rank#76]
+Arguments: [currency_ratio#74 ASC NULLS FIRST], false, 0
+
+(50) Window
+Input [4]: [item#72, return_ratio#73, currency_ratio#74, return_rank#76]
+Arguments: [rank(currency_ratio#74) windowspecdefinition(currency_ratio#74 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#77], [currency_ratio#74 ASC NULLS FIRST]
+
+(51) Filter [codegen id : 14]
+Input [5]: [item#72, return_ratio#73, currency_ratio#74, return_rank#76, currency_rank#77]
+Condition : ((return_rank#76 <= 10) OR (currency_rank#77 <= 10))
+
+(52) Project [codegen id : 14]
+Output [5]: [catalog AS channel#78, item#72, return_ratio#73, return_rank#76, currency_rank#77]
+Input [5]: [item#72, return_ratio#73, currency_ratio#74, return_rank#76, currency_rank#77]
+
+(53) Scan parquet default.store_sales
+Output [6]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_net_paid#82, ss_net_profit#83, ss_sold_date_sk#84]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#84), dynamicpruningexpression(ss_sold_date_sk#84 IN dynamicpruning#7)]
+PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ticket_number:int,ss_quantity:int,ss_net_paid:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(54) ColumnarToRow [codegen id : 15]
+Input [6]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_net_paid#82, ss_net_profit#83, ss_sold_date_sk#84]
+
+(55) Filter [codegen id : 15]
+Input [6]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_net_paid#82, ss_net_profit#83, ss_sold_date_sk#84]
+Condition : (((((((isnotnull(ss_net_profit#83) AND isnotnull(ss_net_paid#82)) AND isnotnull(ss_quantity#81)) AND (ss_net_profit#83 > 1.00)) AND (ss_net_paid#82 > 0.00)) AND (ss_quantity#81 > 0)) AND isnotnull(ss_ticket_number#80)) AND isnotnull(ss_item_sk#79))
+
+(56) Project [codegen id : 15]
+Output [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_net_paid#82, ss_sold_date_sk#84]
+Input [6]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_net_paid#82, ss_net_profit#83, ss_sold_date_sk#84]
+
+(57) BroadcastExchange
+Input [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_net_paid#82, ss_sold_date_sk#84]
+Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [id=#85]
+
+(58) Scan parquet default.store_returns
+Output [5]: [sr_item_sk#86, sr_ticket_number#87, sr_return_quantity#88, sr_return_amt#89, sr_returned_date_sk#90]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int,sr_return_quantity:int,sr_return_amt:decimal(7,2)>
+
+(59) ColumnarToRow
+Input [5]: [sr_item_sk#86, sr_ticket_number#87, sr_return_quantity#88, sr_return_amt#89, sr_returned_date_sk#90]
+
+(60) Filter
+Input [5]: [sr_item_sk#86, sr_ticket_number#87, sr_return_quantity#88, sr_return_amt#89, sr_returned_date_sk#90]
+Condition : (((isnotnull(sr_return_amt#89) AND (sr_return_amt#89 > 10000.00)) AND isnotnull(sr_ticket_number#87)) AND isnotnull(sr_item_sk#86))
+
+(61) Project
+Output [4]: [sr_item_sk#86, sr_ticket_number#87, sr_return_quantity#88, sr_return_amt#89]
+Input [5]: [sr_item_sk#86, sr_ticket_number#87, sr_return_quantity#88, sr_return_amt#89, sr_returned_date_sk#90]
+
+(62) BroadcastHashJoin [codegen id : 17]
+Left keys [2]: [ss_ticket_number#80, ss_item_sk#79]
+Right keys [2]: [sr_ticket_number#87, sr_item_sk#86]
+Join condition: None
+
+(63) Project [codegen id : 17]
+Output [6]: [ss_item_sk#79, ss_quantity#81, ss_net_paid#82, ss_sold_date_sk#84, sr_return_quantity#88, sr_return_amt#89]
+Input [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_net_paid#82, ss_sold_date_sk#84, sr_item_sk#86, sr_ticket_number#87, sr_return_quantity#88, sr_return_amt#89]
+
+(64) ReusedExchange [Reuses operator id: 16]
+Output [1]: [d_date_sk#91]
+
+(65) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ss_sold_date_sk#84]
+Right keys [1]: [d_date_sk#91]
+Join condition: None
+
+(66) Project [codegen id : 17]
+Output [5]: [ss_item_sk#79, ss_quantity#81, ss_net_paid#82, sr_return_quantity#88, sr_return_amt#89]
+Input [7]: [ss_item_sk#79, ss_quantity#81, ss_net_paid#82, ss_sold_date_sk#84, sr_return_quantity#88, sr_return_amt#89, d_date_sk#91]
+
+(67) HashAggregate [codegen id : 17]
+Input [5]: [ss_item_sk#79, ss_quantity#81, ss_net_paid#82, sr_return_quantity#88, sr_return_amt#89]
+Keys [1]: [ss_item_sk#79]
+Functions [4]: [partial_sum(coalesce(sr_return_quantity#88, 0)), partial_sum(coalesce(ss_quantity#81, 0)), partial_sum(coalesce(cast(sr_return_amt#89 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#82 as decimal(12,2)), 0.00))]
+Aggregate Attributes [6]: [sum#92, sum#93, sum#94, isEmpty#95, sum#96, isEmpty#97]
+Results [7]: [ss_item_sk#79, sum#98, sum#99, sum#100, isEmpty#101, sum#102, isEmpty#103]
+
+(68) Exchange
+Input [7]: [ss_item_sk#79, sum#98, sum#99, sum#100, isEmpty#101, sum#102, isEmpty#103]
+Arguments: hashpartitioning(ss_item_sk#79, 5), ENSURE_REQUIREMENTS, [id=#104]
+
+(69) HashAggregate [codegen id : 18]
+Input [7]: [ss_item_sk#79, sum#98, sum#99, sum#100, isEmpty#101, sum#102, isEmpty#103]
+Keys [1]: [ss_item_sk#79]
+Functions [4]: [sum(coalesce(sr_return_quantity#88, 0)), sum(coalesce(ss_quantity#81, 0)), sum(coalesce(cast(sr_return_amt#89 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#82 as decimal(12,2)), 0.00))]
+Aggregate Attributes [4]: [sum(coalesce(sr_return_quantity#88, 0))#105, sum(coalesce(ss_quantity#81, 0))#106, sum(coalesce(cast(sr_return_amt#89 as decimal(12,2)), 0.00))#107, sum(coalesce(cast(ss_net_paid#82 as decimal(12,2)), 0.00))#108]
+Results [3]: [ss_item_sk#79 AS item#109, CheckOverflow((promote_precision(cast(sum(coalesce(sr_return_quantity#88, 0))#105 as decimal(15,4))) / promote_precision(cast(sum(coalesce(ss_quantity#81, 0))#106 as decimal(15,4)))), DecimalType(35,20), true) AS return_ratio#110, CheckOverflow((promote_precision(cast(sum(coalesce(cast(sr_return_amt#89 as decimal(12,2)), 0.00))#107 as decimal(15,4))) / promote_precision(cast(sum(coalesce(cast(ss_net_paid#82 as decimal(12,2)), 0.00))#108 as decimal(15,4)))), DecimalType(35,20), true) AS currency_ratio#111]
+
+(70) Exchange
+Input [3]: [item#109, return_ratio#110, currency_ratio#111]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#112]
+
+(71) Sort [codegen id : 19]
+Input [3]: [item#109, return_ratio#110, currency_ratio#111]
+Arguments: [return_ratio#110 ASC NULLS FIRST], false, 0
+
+(72) Window
+Input [3]: [item#109, return_ratio#110, currency_ratio#111]
+Arguments: [rank(return_ratio#110) windowspecdefinition(return_ratio#110 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#113], [return_ratio#110 ASC NULLS FIRST]
+
+(73) Sort [codegen id : 20]
+Input [4]: [item#109, return_ratio#110, currency_ratio#111, return_rank#113]
+Arguments: [currency_ratio#111 ASC NULLS FIRST], false, 0
+
+(74) Window
+Input [4]: [item#109, return_ratio#110, currency_ratio#111, return_rank#113]
+Arguments: [rank(currency_ratio#111) windowspecdefinition(currency_ratio#111 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#114], [currency_ratio#111 ASC NULLS FIRST]
+
+(75) Filter [codegen id : 21]
+Input [5]: [item#109, return_ratio#110, currency_ratio#111, return_rank#113, currency_rank#114]
+Condition : ((return_rank#113 <= 10) OR (currency_rank#114 <= 10))
+
+(76) Project [codegen id : 21]
+Output [5]: [store AS channel#115, item#109, return_ratio#110, return_rank#113, currency_rank#114]
+Input [5]: [item#109, return_ratio#110, currency_ratio#111, return_rank#113, currency_rank#114]
+
+(77) Union
+
+(78) HashAggregate [codegen id : 22]
+Input [5]: [channel#41, item#35, return_ratio#36, return_rank#39, currency_rank#40]
+Keys [5]: [channel#41, item#35, return_ratio#36, return_rank#39, currency_rank#40]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [channel#41, item#35, return_ratio#36, return_rank#39, currency_rank#40]
+
+(79) Exchange
+Input [5]: [channel#41, item#35, return_ratio#36, return_rank#39, currency_rank#40]
+Arguments: hashpartitioning(channel#41, item#35, return_ratio#36, return_rank#39, currency_rank#40, 5), ENSURE_REQUIREMENTS, [id=#116]
+
+(80) HashAggregate [codegen id : 23]
+Input [5]: [channel#41, item#35, return_ratio#36, return_rank#39, currency_rank#40]
+Keys [5]: [channel#41, item#35, return_ratio#36, return_rank#39, currency_rank#40]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [channel#41, item#35, return_ratio#36, return_rank#39, currency_rank#40]
+
+(81) TakeOrderedAndProject
+Input [5]: [channel#41, item#35, return_ratio#36, return_rank#39, currency_rank#40]
+Arguments: 100, [channel#41 ASC NULLS FIRST, return_rank#39 ASC NULLS FIRST, currency_rank#40 ASC NULLS FIRST, item#35 ASC NULLS FIRST], [channel#41, item#35, return_ratio#36, return_rank#39, currency_rank#40]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#6 IN dynamicpruning#7
+ReusedExchange (82)
+
+
+(82) ReusedExchange [Reuses operator id: 16]
+Output [1]: [d_date_sk#14]
+
+Subquery:2 Hosting operator id = 29 Hosting Expression = cs_sold_date_sk#47 IN dynamicpruning#7
+
+Subquery:3 Hosting operator id = 53 Hosting Expression = ss_sold_date_sk#84 IN dynamicpruning#7
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/simplified.txt
new file mode 100644
index 0000000000000..0b548432d6fed
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/simplified.txt
@@ -0,0 +1,133 @@
+TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio]
+  WholeStageCodegen (23)
+    HashAggregate [channel,item,return_ratio,return_rank,currency_rank]
+      InputAdapter
+        Exchange [channel,item,return_ratio,return_rank,currency_rank] #1
+          WholeStageCodegen (22)
+            HashAggregate [channel,item,return_ratio,return_rank,currency_rank]
+              InputAdapter
+                Union
+                  WholeStageCodegen (7)
+                    Project [item,return_ratio,return_rank,currency_rank]
+                      Filter [return_rank,currency_rank]
+                        InputAdapter
+                          Window [currency_ratio]
+                            WholeStageCodegen (6)
+                              Sort [currency_ratio]
+                                InputAdapter
+                                  Window [return_ratio]
+                                    WholeStageCodegen (5)
+                                      Sort [return_ratio]
+                                        InputAdapter
+                                          Exchange #2
+                                            WholeStageCodegen (4)
+                                              HashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty]
+                                                InputAdapter
+                                                  Exchange [ws_item_sk] #3
+                                                    WholeStageCodegen (3)
+                                                      HashAggregate [ws_item_sk,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty]
+                                                        Project [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt]
+                                                          BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                            Project [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt]
+                                                              BroadcastHashJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk]
+                                                                InputAdapter
+                                                                  BroadcastExchange #4
+                                                                    WholeStageCodegen (1)
+                                                                      Project [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk]
+                                                                        Filter [ws_net_profit,ws_net_paid,ws_quantity,ws_order_number,ws_item_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk]
+                                                                                SubqueryBroadcast [d_date_sk] #1
+                                                                                  ReusedExchange [d_date_sk] #5
+                                                                Project [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt]
+                                                                  Filter [wr_return_amt,wr_order_number,wr_item_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk]
+                                                            InputAdapter
+                                                              BroadcastExchange #5
+                                                                WholeStageCodegen (2)
+                                                                  Project [d_date_sk]
+                                                                    Filter [d_year,d_moy,d_date_sk]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                  WholeStageCodegen (14)
+                    Project [item,return_ratio,return_rank,currency_rank]
+                      Filter [return_rank,currency_rank]
+                        InputAdapter
+                          Window [currency_ratio]
+                            WholeStageCodegen (13)
+                              Sort [currency_ratio]
+                                InputAdapter
+                                  Window [return_ratio]
+                                    WholeStageCodegen (12)
+                                      Sort [return_ratio]
+                                        InputAdapter
+                                          Exchange #6
+                                            WholeStageCodegen (11)
+                                              HashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty]
+                                                InputAdapter
+                                                  Exchange [cs_item_sk] #7
+                                                    WholeStageCodegen (10)
+                                                      HashAggregate [cs_item_sk,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty]
+                                                        Project [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount]
+                                                          BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                            Project [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount]
+                                                              BroadcastHashJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk]
+                                                                InputAdapter
+                                                                  BroadcastExchange #8
+                                                                    WholeStageCodegen (8)
+                                                                      Project [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk]
+                                                                        Filter [cs_net_profit,cs_net_paid,cs_quantity,cs_order_number,cs_item_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk]
+                                                                                ReusedSubquery [d_date_sk] #1
+                                                                Project [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount]
+                                                                  Filter [cr_return_amount,cr_order_number,cr_item_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk]
+                                                            InputAdapter
+                                                              ReusedExchange [d_date_sk] #5
+                  WholeStageCodegen (21)
+                    Project [item,return_ratio,return_rank,currency_rank]
+                      Filter [return_rank,currency_rank]
+                        InputAdapter
+                          Window [currency_ratio]
+                            WholeStageCodegen (20)
+                              Sort [currency_ratio]
+                                InputAdapter
+                                  Window [return_ratio]
+                                    WholeStageCodegen (19)
+                                      Sort [return_ratio]
+                                        InputAdapter
+                                          Exchange #9
+                                            WholeStageCodegen (18)
+                                              HashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty]
+                                                InputAdapter
+                                                  Exchange [ss_item_sk] #10
+                                                    WholeStageCodegen (17)
+                                                      HashAggregate [ss_item_sk,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty]
+                                                        Project [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt]
+                                                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                            Project [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt]
+                                                              BroadcastHashJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                                                InputAdapter
+                                                                  BroadcastExchange #11
+                                                                    WholeStageCodegen (15)
+                                                                      Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk]
+                                                                        Filter [ss_net_profit,ss_net_paid,ss_quantity,ss_ticket_number,ss_item_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk]
+                                                                                ReusedSubquery [d_date_sk] #1
+                                                                Project [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt]
+                                                                  Filter [sr_return_amt,sr_ticket_number,sr_item_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk]
+                                                            InputAdapter
+                                                              ReusedExchange [d_date_sk] #5
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.sf100/explain.txt
new file mode 100644
index 0000000000000..0a8fdb66edb77
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.sf100/explain.txt
@@ -0,0 +1,425 @@
+== Physical Plan ==
+TakeOrderedAndProject (74)
++- * Filter (73)
+   +- * HashAggregate (72)
+      +- * HashAggregate (71)
+         +- * Project (70)
+            +- * SortMergeJoin Inner (69)
+               :- Window (64)
+               :  +- * Sort (63)
+               :     +- Exchange (62)
+               :        +- * Project (61)
+               :           +- * Filter (60)
+               :              +- SortMergeJoin FullOuter (59)
+               :                 :- * Sort (31)
+               :                 :  +- Exchange (30)
+               :                 :     +- * HashAggregate (29)
+               :                 :        +- * HashAggregate (28)
+               :                 :           +- * Project (27)
+               :                 :              +- * SortMergeJoin Inner (26)
+               :                 :                 :- * Sort (19)
+               :                 :                 :  +- Exchange (18)
+               :                 :                 :     +- * Project (17)
+               :                 :                 :        +- Window (16)
+               :                 :                 :           +- * Sort (15)
+               :                 :                 :              +- Exchange (14)
+               :                 :                 :                 +- * HashAggregate (13)
+               :                 :                 :                    +- Exchange (12)
+               :                 :                 :                       +- * HashAggregate (11)
+               :                 :                 :                          +- * Project (10)
+               :                 :                 :                             +- * BroadcastHashJoin Inner BuildRight (9)
+               :                 :                 :                                :- * Filter (3)
+               :                 :                 :                                :  +- * ColumnarToRow (2)
+               :                 :                 :                                :     +- Scan parquet default.web_sales (1)
+               :                 :                 :                                +- BroadcastExchange (8)
+               :                 :                 :                                   +- * Project (7)
+               :                 :                 :                                      +- * Filter (6)
+               :                 :                 :                                         +- * ColumnarToRow (5)
+               :                 :                 :                                            +- Scan parquet default.date_dim (4)
+               :                 :                 +- * Sort (25)
+               :                 :                    +- Exchange (24)
+               :                 :                       +- * Project (23)
+               :                 :                          +- Window (22)
+               :                 :                             +- * Sort (21)
+               :                 :                                +- ReusedExchange (20)
+               :                 +- * Sort (58)
+               :                    +- Exchange (57)
+               :                       +- * HashAggregate (56)
+               :                          +- * HashAggregate (55)
+               :                             +- * Project (54)
+               :                                +- * SortMergeJoin Inner (53)
+               :                                   :- * Sort (46)
+               :                                   :  +- Exchange (45)
+               :                                   :     +- * Project (44)
+               :                                   :        +- Window (43)
+               :                                   :           +- * Sort (42)
+               :                                   :              +- Exchange (41)
+               :                                   :                 +- * HashAggregate (40)
+               :                                   :                    +- Exchange (39)
+               :                                   :                       +- * HashAggregate (38)
+               :                                   :                          +- * Project (37)
+               :                                   :                             +- * BroadcastHashJoin Inner BuildRight (36)
+               :                                   :                                :- * Filter (34)
+               :                                   :                                :  +- * ColumnarToRow (33)
+               :                                   :                                :     +- Scan parquet default.store_sales (32)
+               :                                   :                                +- ReusedExchange (35)
+               :                                   +- * Sort (52)
+               :                                      +- Exchange (51)
+               :                                         +- * Project (50)
+               :                                            +- Window (49)
+               :                                               +- * Sort (48)
+               :                                                  +- ReusedExchange (47)
+               +- * Project (68)
+                  +- Window (67)
+                     +- * Sort (66)
+                        +- ReusedExchange (65)
+
+
+(1) Scan parquet default.web_sales
+Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3]
+
+(3) Filter [codegen id : 2]
+Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3]
+Condition : isnotnull(ws_item_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date,d_month_seq:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7]
+Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#5))
+
+(7) Project [codegen id : 1]
+Output [2]: [d_date_sk#5, d_date#6]
+Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7]
+
+(8) BroadcastExchange
+Input [2]: [d_date_sk#5, d_date#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#8]
+
+(9) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ws_sold_date_sk#3]
+Right keys [1]: [d_date_sk#5]
+Join condition: None
+
+(10) Project [codegen id : 2]
+Output [3]: [ws_item_sk#1, ws_sales_price#2, d_date#6]
+Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#5, d_date#6]
+
+(11) HashAggregate [codegen id : 2]
+Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#6]
+Keys [2]: [ws_item_sk#1, d_date#6]
+Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))]
+Aggregate Attributes [1]: [sum#9]
+Results [3]: [ws_item_sk#1, d_date#6, sum#10]
+
+(12) Exchange
+Input [3]: [ws_item_sk#1, d_date#6, sum#10]
+Arguments: hashpartitioning(ws_item_sk#1, d_date#6, 5), ENSURE_REQUIREMENTS, [id=#11]
+
+(13) HashAggregate [codegen id : 3]
+Input [3]: [ws_item_sk#1, d_date#6, sum#10]
+Keys [2]: [ws_item_sk#1, d_date#6]
+Functions [1]: [sum(UnscaledValue(ws_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#2))#12]
+Results [4]: [ws_item_sk#1 AS item_sk#13, d_date#6, MakeDecimal(sum(UnscaledValue(ws_sales_price#2))#12,17,2) AS sumws#14, ws_item_sk#1]
+
+(14) Exchange
+Input [4]: [item_sk#13, d_date#6, sumws#14, ws_item_sk#1]
+Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(15) Sort [codegen id : 4]
+Input [4]: [item_sk#13, d_date#6, sumws#14, ws_item_sk#1]
+Arguments: [ws_item_sk#1 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST], false, 0
+
+(16) Window
+Input [4]: [item_sk#13, d_date#6, sumws#14, ws_item_sk#1]
+Arguments: [row_number() windowspecdefinition(ws_item_sk#1, d_date#6 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#16], [ws_item_sk#1], [d_date#6 ASC NULLS FIRST]
+
+(17) Project [codegen id : 5]
+Output [4]: [item_sk#13, d_date#6, sumws#14, rk#16]
+Input [5]: [item_sk#13, d_date#6, sumws#14, ws_item_sk#1, rk#16]
+
+(18) Exchange
+Input [4]: [item_sk#13, d_date#6, sumws#14, rk#16]
+Arguments: hashpartitioning(item_sk#13, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(19) Sort [codegen id : 6]
+Input [4]: [item_sk#13, d_date#6, sumws#14, rk#16]
+Arguments: [item_sk#13 ASC NULLS FIRST], false, 0
+
+(20) ReusedExchange [Reuses operator id: unknown]
+Output [4]: [item_sk#18, d_date#19, sumws#20, ws_item_sk#21]
+
+(21) Sort [codegen id : 10]
+Input [4]: [item_sk#18, d_date#19, sumws#20, ws_item_sk#21]
+Arguments: [ws_item_sk#21 ASC NULLS FIRST, d_date#19 ASC NULLS FIRST], false, 0
+
+(22) Window
+Input [4]: [item_sk#18, d_date#19, sumws#20, ws_item_sk#21]
+Arguments: [row_number() windowspecdefinition(ws_item_sk#21, d_date#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#22], [ws_item_sk#21], [d_date#19 ASC NULLS FIRST]
+
+(23) Project [codegen id : 11]
+Output [3]: [item_sk#18, sumws#20, rk#22]
+Input [5]: [item_sk#18, d_date#19, sumws#20, ws_item_sk#21, rk#22]
+
+(24) Exchange
+Input [3]: [item_sk#18, sumws#20, rk#22]
+Arguments: hashpartitioning(item_sk#18, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(25) Sort [codegen id : 12]
+Input [3]: [item_sk#18, sumws#20, rk#22]
+Arguments: [item_sk#18 ASC NULLS FIRST], false, 0
+
+(26) SortMergeJoin [codegen id : 13]
+Left keys [1]: [item_sk#13]
+Right keys [1]: [item_sk#18]
+Join condition: (rk#16 >= rk#22)
+
+(27) Project [codegen id : 13]
+Output [4]: [item_sk#13, d_date#6, sumws#14, sumws#20]
+Input [7]: [item_sk#13, d_date#6, sumws#14, rk#16, item_sk#18, sumws#20, rk#22]
+
+(28) HashAggregate [codegen id : 13]
+Input [4]: [item_sk#13, d_date#6, sumws#14, sumws#20]
+Keys [3]: [item_sk#13, d_date#6, sumws#14]
+Functions [1]: [partial_sum(sumws#20)]
+Aggregate Attributes [2]: [sum#24, isEmpty#25]
+Results [5]: [item_sk#13, d_date#6, sumws#14, sum#26, isEmpty#27]
+
+(29) HashAggregate [codegen id : 13]
+Input [5]: [item_sk#13, d_date#6, sumws#14, sum#26, isEmpty#27]
+Keys [3]: [item_sk#13, d_date#6, sumws#14]
+Functions [1]: [sum(sumws#20)]
+Aggregate Attributes [1]: [sum(sumws#20)#28]
+Results [3]: [item_sk#13, d_date#6, sum(sumws#20)#28 AS cume_sales#29]
+
+(30) Exchange
+Input [3]: [item_sk#13, d_date#6, cume_sales#29]
+Arguments: hashpartitioning(item_sk#13, d_date#6, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(31) Sort [codegen id : 14]
+Input [3]: [item_sk#13, d_date#6, cume_sales#29]
+Arguments: [item_sk#13 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST], false, 0
+
+(32) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#31, ss_sales_price#32, ss_sold_date_sk#33]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#33), dynamicpruningexpression(ss_sold_date_sk#33 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_sales_price:decimal(7,2)>
+
+(33) ColumnarToRow [codegen id : 16]
+Input [3]: [ss_item_sk#31, ss_sales_price#32, ss_sold_date_sk#33]
+
+(34) Filter [codegen id : 16]
+Input [3]: [ss_item_sk#31, ss_sales_price#32, ss_sold_date_sk#33]
+Condition : isnotnull(ss_item_sk#31)
+
+(35) ReusedExchange [Reuses operator id: 8]
+Output [2]: [d_date_sk#34, d_date#35]
+
+(36) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [ss_sold_date_sk#33]
+Right keys [1]: [d_date_sk#34]
+Join condition: None
+
+(37) Project [codegen id : 16]
+Output [3]: [ss_item_sk#31, ss_sales_price#32, d_date#35]
+Input [5]: [ss_item_sk#31, ss_sales_price#32, ss_sold_date_sk#33, d_date_sk#34, d_date#35]
+
+(38) HashAggregate [codegen id : 16]
+Input [3]: [ss_item_sk#31, ss_sales_price#32, d_date#35]
+Keys [2]: [ss_item_sk#31, d_date#35]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#32))]
+Aggregate Attributes [1]: [sum#36]
+Results [3]: [ss_item_sk#31, d_date#35, sum#37]
+
+(39) Exchange
+Input [3]: [ss_item_sk#31, d_date#35, sum#37]
+Arguments: hashpartitioning(ss_item_sk#31, d_date#35, 5), ENSURE_REQUIREMENTS, [id=#38]
+
+(40) HashAggregate [codegen id : 17]
+Input [3]: [ss_item_sk#31, d_date#35, sum#37]
+Keys [2]: [ss_item_sk#31, d_date#35]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#32))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#32))#39]
+Results [4]: [ss_item_sk#31 AS item_sk#40, d_date#35, MakeDecimal(sum(UnscaledValue(ss_sales_price#32))#39,17,2) AS sumss#41, ss_item_sk#31]
+
+(41) Exchange
+Input [4]: [item_sk#40, d_date#35, sumss#41, ss_item_sk#31]
+Arguments: hashpartitioning(ss_item_sk#31, 5), ENSURE_REQUIREMENTS, [id=#42]
+
+(42) Sort [codegen id : 18]
+Input [4]: [item_sk#40, d_date#35, sumss#41, ss_item_sk#31]
+Arguments: [ss_item_sk#31 ASC NULLS FIRST, d_date#35 ASC NULLS FIRST], false, 0
+
+(43) Window
+Input [4]: [item_sk#40, d_date#35, sumss#41, ss_item_sk#31]
+Arguments: [row_number() windowspecdefinition(ss_item_sk#31, d_date#35 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#43], [ss_item_sk#31], [d_date#35 ASC NULLS FIRST]
+
+(44) Project [codegen id : 19]
+Output [4]: [item_sk#40, d_date#35, sumss#41, rk#43]
+Input [5]: [item_sk#40, d_date#35, sumss#41, ss_item_sk#31, rk#43]
+
+(45) Exchange
+Input [4]: [item_sk#40, d_date#35, sumss#41, rk#43]
+Arguments: hashpartitioning(item_sk#40, 5), ENSURE_REQUIREMENTS, [id=#44]
+
+(46) Sort [codegen id : 20]
+Input [4]: [item_sk#40, d_date#35, sumss#41, rk#43]
+Arguments: [item_sk#40 ASC NULLS FIRST], false, 0
+
+(47) ReusedExchange [Reuses operator id: unknown]
+Output [4]: [item_sk#45, d_date#46, sumss#47, ss_item_sk#48]
+
+(48) Sort [codegen id : 24]
+Input [4]: [item_sk#45, d_date#46, sumss#47, ss_item_sk#48]
+Arguments: [ss_item_sk#48 ASC NULLS FIRST, d_date#46 ASC NULLS FIRST], false, 0
+
+(49) Window
+Input [4]: [item_sk#45, d_date#46, sumss#47, ss_item_sk#48]
+Arguments: [row_number() windowspecdefinition(ss_item_sk#48, d_date#46 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#49], [ss_item_sk#48], [d_date#46 ASC NULLS FIRST]
+
+(50) Project [codegen id : 25]
+Output [3]: [item_sk#45, sumss#47, rk#49]
+Input [5]: [item_sk#45, d_date#46, sumss#47, ss_item_sk#48, rk#49]
+
+(51) Exchange
+Input [3]: [item_sk#45, sumss#47, rk#49]
+Arguments: hashpartitioning(item_sk#45, 5), ENSURE_REQUIREMENTS, [id=#50]
+
+(52) Sort [codegen id : 26]
+Input [3]: [item_sk#45, sumss#47, rk#49]
+Arguments: [item_sk#45 ASC NULLS FIRST], false, 0
+
+(53) SortMergeJoin [codegen id : 27]
+Left keys [1]: [item_sk#40]
+Right keys [1]: [item_sk#45]
+Join condition: (rk#43 >= rk#49)
+
+(54) Project [codegen id : 27]
+Output [4]: [item_sk#40, d_date#35, sumss#41, sumss#47]
+Input [7]: [item_sk#40, d_date#35, sumss#41, rk#43, item_sk#45, sumss#47, rk#49]
+
+(55) HashAggregate [codegen id : 27]
+Input [4]: [item_sk#40, d_date#35, sumss#41, sumss#47]
+Keys [3]: [item_sk#40, d_date#35, sumss#41]
+Functions [1]: [partial_sum(sumss#47)]
+Aggregate Attributes [2]: [sum#51, isEmpty#52]
+Results [5]: [item_sk#40, d_date#35, sumss#41, sum#53, isEmpty#54]
+
+(56) HashAggregate [codegen id : 27]
+Input [5]: [item_sk#40, d_date#35, sumss#41, sum#53, isEmpty#54]
+Keys [3]: [item_sk#40, d_date#35, sumss#41]
+Functions [1]: [sum(sumss#47)]
+Aggregate Attributes [1]: [sum(sumss#47)#55]
+Results [3]: [item_sk#40, d_date#35, sum(sumss#47)#55 AS cume_sales#56]
+
+(57) Exchange
+Input [3]: [item_sk#40, d_date#35, cume_sales#56]
+Arguments: hashpartitioning(item_sk#40, d_date#35, 5), ENSURE_REQUIREMENTS, [id=#57]
+
+(58) Sort [codegen id : 28]
+Input [3]: [item_sk#40, d_date#35, cume_sales#56]
+Arguments: [item_sk#40 ASC NULLS FIRST, d_date#35 ASC NULLS FIRST], false, 0
+
+(59) SortMergeJoin
+Left keys [2]: [item_sk#13, d_date#6]
+Right keys [2]: [item_sk#40, d_date#35]
+Join condition: None
+
+(60) Filter [codegen id : 29]
+Input [6]: [item_sk#13, d_date#6, cume_sales#29, item_sk#40, d_date#35, cume_sales#56]
+Condition : isnotnull(CASE WHEN isnotnull(item_sk#13) THEN item_sk#13 ELSE item_sk#40 END)
+
+(61) Project [codegen id : 29]
+Output [4]: [CASE WHEN isnotnull(item_sk#13) THEN item_sk#13 ELSE item_sk#40 END AS item_sk#58, CASE WHEN isnotnull(d_date#6) THEN d_date#6 ELSE d_date#35 END AS d_date#59, cume_sales#29 AS web_sales#60, cume_sales#56 AS store_sales#61]
+Input [6]: [item_sk#13, d_date#6, cume_sales#29, item_sk#40, d_date#35, cume_sales#56]
+
+(62) Exchange
+Input [4]: [item_sk#58, d_date#59, web_sales#60, store_sales#61]
+Arguments: hashpartitioning(item_sk#58, 5), ENSURE_REQUIREMENTS, [id=#62]
+
+(63) Sort [codegen id : 30]
+Input [4]: [item_sk#58, d_date#59, web_sales#60, store_sales#61]
+Arguments: [item_sk#58 ASC NULLS FIRST, d_date#59 ASC NULLS FIRST], false, 0
+
+(64) Window
+Input [4]: [item_sk#58, d_date#59, web_sales#60, store_sales#61]
+Arguments: [row_number() windowspecdefinition(item_sk#58, d_date#59 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#63], [item_sk#58], [d_date#59 ASC NULLS FIRST]
+
+(65) ReusedExchange [Reuses operator id: unknown]
+Output [4]: [item_sk#64, d_date#65, web_sales#66, store_sales#67]
+
+(66) Sort [codegen id : 60]
+Input [4]: [item_sk#64, d_date#65, web_sales#66, store_sales#67]
+Arguments: [item_sk#64 ASC NULLS FIRST, d_date#65 ASC NULLS FIRST], false, 0
+
+(67) Window
+Input [4]: [item_sk#64, d_date#65, web_sales#66, store_sales#67]
+Arguments: [row_number() windowspecdefinition(item_sk#64, d_date#65 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#68], [item_sk#64], [d_date#65 ASC NULLS FIRST]
+
+(68) Project [codegen id : 61]
+Output [4]: [item_sk#64, web_sales#66, store_sales#67, rk#68]
+Input [5]: [item_sk#64, d_date#65, web_sales#66, store_sales#67, rk#68]
+
+(69) SortMergeJoin [codegen id : 62]
+Left keys [1]: [item_sk#58]
+Right keys [1]: [item_sk#64]
+Join condition: (rk#63 >= rk#68)
+
+(70) Project [codegen id : 62]
+Output [6]: [item_sk#58, d_date#59, web_sales#60, store_sales#61, web_sales#66, store_sales#67]
+Input [9]: [item_sk#58, d_date#59, web_sales#60, store_sales#61, rk#63, item_sk#64, web_sales#66, store_sales#67, rk#68]
+
+(71) HashAggregate [codegen id : 62]
+Input [6]: [item_sk#58, d_date#59, web_sales#60, store_sales#61, web_sales#66, store_sales#67]
+Keys [4]: [item_sk#58, d_date#59, web_sales#60, store_sales#61]
+Functions [2]: [partial_max(web_sales#66), partial_max(store_sales#67)]
+Aggregate Attributes [2]: [max#69, max#70]
+Results [6]: [item_sk#58, d_date#59, web_sales#60, store_sales#61, max#71, max#72]
+
+(72) HashAggregate [codegen id : 62]
+Input [6]: [item_sk#58, d_date#59, web_sales#60, store_sales#61, max#71, max#72]
+Keys [4]: [item_sk#58, d_date#59, web_sales#60, store_sales#61]
+Functions [2]: [max(web_sales#66), max(store_sales#67)]
+Aggregate Attributes [2]: [max(web_sales#66)#73, max(store_sales#67)#74]
+Results [6]: [item_sk#58, d_date#59, web_sales#60, store_sales#61, max(web_sales#66)#73 AS web_cumulative#75, max(store_sales#67)#74 AS store_cumulative#76]
+
+(73) Filter [codegen id : 62]
+Input [6]: [item_sk#58, d_date#59, web_sales#60, store_sales#61, web_cumulative#75, store_cumulative#76]
+Condition : ((isnotnull(web_cumulative#75) AND isnotnull(store_cumulative#76)) AND (web_cumulative#75 > store_cumulative#76))
+
+(74) TakeOrderedAndProject
+Input [6]: [item_sk#58, d_date#59, web_sales#60, store_sales#61, web_cumulative#75, store_cumulative#76]
+Arguments: 100, [item_sk#58 ASC NULLS FIRST, d_date#59 ASC NULLS FIRST], [item_sk#58, d_date#59, web_sales#60, store_sales#61, web_cumulative#75, store_cumulative#76]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (75)
+
+
+(75) ReusedExchange [Reuses operator id: 8]
+Output [2]: [d_date_sk#5, d_date#6]
+
+Subquery:2 Hosting operator id = 32 Hosting Expression = ss_sold_date_sk#33 IN dynamicpruning#4
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.sf100/simplified.txt
new file mode 100644
index 0000000000000..fe7a795bd98e2
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.sf100/simplified.txt
@@ -0,0 +1,134 @@
+TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative]
+  WholeStageCodegen (62)
+    Filter [web_cumulative,store_cumulative]
+      HashAggregate [item_sk,d_date,web_sales,store_sales,max,max] [max(web_sales),max(store_sales),web_cumulative,store_cumulative,max,max]
+        HashAggregate [item_sk,d_date,web_sales,store_sales,web_sales,store_sales] [max,max,max,max]
+          Project [item_sk,d_date,web_sales,store_sales,web_sales,store_sales]
+            SortMergeJoin [item_sk,item_sk,rk,rk]
+              InputAdapter
+                Window [item_sk,d_date]
+                  WholeStageCodegen (30)
+                    Sort [item_sk,d_date]
+                      InputAdapter
+                        Exchange [item_sk] #1
+                          WholeStageCodegen (29)
+                            Project [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales]
+                              Filter [item_sk,item_sk]
+                                InputAdapter
+                                  SortMergeJoin [item_sk,d_date,item_sk,d_date]
+                                    WholeStageCodegen (14)
+                                      Sort [item_sk,d_date]
+                                        InputAdapter
+                                          Exchange [item_sk,d_date] #2
+                                            WholeStageCodegen (13)
+                                              HashAggregate [item_sk,d_date,sumws,sum,isEmpty] [sum(sumws),cume_sales,sum,isEmpty]
+                                                HashAggregate [item_sk,d_date,sumws,sumws] [sum,isEmpty,sum,isEmpty]
+                                                  Project [item_sk,d_date,sumws,sumws]
+                                                    SortMergeJoin [item_sk,item_sk,rk,rk]
+                                                      InputAdapter
+                                                        WholeStageCodegen (6)
+                                                          Sort [item_sk]
+                                                            InputAdapter
+                                                              Exchange [item_sk] #3
+                                                                WholeStageCodegen (5)
+                                                                  Project [item_sk,d_date,sumws,rk]
+                                                                    InputAdapter
+                                                                      Window [ws_item_sk,d_date]
+                                                                        WholeStageCodegen (4)
+                                                                          Sort [ws_item_sk,d_date]
+                                                                            InputAdapter
+                                                                              Exchange [ws_item_sk] #4
+                                                                                WholeStageCodegen (3)
+                                                                                  HashAggregate [ws_item_sk,d_date,sum] [sum(UnscaledValue(ws_sales_price)),item_sk,sumws,sum]
+                                                                                    InputAdapter
+                                                                                      Exchange [ws_item_sk,d_date] #5
+                                                                                        WholeStageCodegen (2)
+                                                                                          HashAggregate [ws_item_sk,d_date,ws_sales_price] [sum,sum]
+                                                                                            Project [ws_item_sk,ws_sales_price,d_date]
+                                                                                              BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                                                Filter [ws_item_sk]
+                                                                                                  ColumnarToRow
+                                                                                                    InputAdapter
+                                                                                                      Scan parquet default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk]
+                                                                                                        SubqueryBroadcast [d_date_sk] #1
+                                                                                                          ReusedExchange [d_date_sk,d_date] #6
+                                                                                                InputAdapter
+                                                                                                  BroadcastExchange #6
+                                                                                                    WholeStageCodegen (1)
+                                                                                                      Project [d_date_sk,d_date]
+                                                                                                        Filter [d_month_seq,d_date_sk]
+                                                                                                          ColumnarToRow
+                                                                                                            InputAdapter
+                                                                                                              Scan parquet default.date_dim [d_date_sk,d_date,d_month_seq]
+                                                      InputAdapter
+                                                        WholeStageCodegen (12)
+                                                          Sort [item_sk]
+                                                            InputAdapter
+                                                              Exchange [item_sk] #7
+                                                                WholeStageCodegen (11)
+                                                                  Project [item_sk,sumws,rk]
+                                                                    InputAdapter
+                                                                      Window [ws_item_sk,d_date]
+                                                                        WholeStageCodegen (10)
+                                                                          Sort [ws_item_sk,d_date]
+                                                                            InputAdapter
+                                                                              ReusedExchange [item_sk,d_date,sumws,ws_item_sk] #8
+                                    WholeStageCodegen (28)
+                                      Sort [item_sk,d_date]
+                                        InputAdapter
+                                          Exchange [item_sk,d_date] #9
+                                            WholeStageCodegen (27)
+                                              HashAggregate [item_sk,d_date,sumss,sum,isEmpty] [sum(sumss),cume_sales,sum,isEmpty]
+                                                HashAggregate [item_sk,d_date,sumss,sumss] [sum,isEmpty,sum,isEmpty]
+                                                  Project [item_sk,d_date,sumss,sumss]
+                                                    SortMergeJoin [item_sk,item_sk,rk,rk]
+                                                      InputAdapter
+                                                        WholeStageCodegen (20)
+                                                          Sort [item_sk]
+                                                            InputAdapter
+                                                              Exchange [item_sk] #10
+                                                                WholeStageCodegen (19)
+                                                                  Project [item_sk,d_date,sumss,rk]
+                                                                    InputAdapter
+                                                                      Window [ss_item_sk,d_date]
+                                                                        WholeStageCodegen (18)
+                                                                          Sort [ss_item_sk,d_date]
+                                                                            InputAdapter
+                                                                              Exchange [ss_item_sk] #11
+                                                                                WholeStageCodegen (17)
+                                                                                  HashAggregate [ss_item_sk,d_date,sum] [sum(UnscaledValue(ss_sales_price)),item_sk,sumss,sum]
+                                                                                    InputAdapter
+                                                                                      Exchange [ss_item_sk,d_date] #12
+                                                                                        WholeStageCodegen (16)
+                                                                                          HashAggregate [ss_item_sk,d_date,ss_sales_price] [sum,sum]
+                                                                                            Project [ss_item_sk,ss_sales_price,d_date]
+                                                                                              BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                                Filter [ss_item_sk]
+                                                                                                  ColumnarToRow
+                                                                                                    InputAdapter
+                                                                                                      Scan parquet default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk]
+                                                                                                        ReusedSubquery [d_date_sk] #1
+                                                                                                InputAdapter
+                                                                                                  ReusedExchange [d_date_sk,d_date] #6
+                                                      InputAdapter
+                                                        WholeStageCodegen (26)
+                                                          Sort [item_sk]
+                                                            InputAdapter
+                                                              Exchange [item_sk] #13
+                                                                WholeStageCodegen (25)
+                                                                  Project [item_sk,sumss,rk]
+                                                                    InputAdapter
+                                                                      Window [ss_item_sk,d_date]
+                                                                        WholeStageCodegen (24)
+                                                                          Sort [ss_item_sk,d_date]
+                                                                            InputAdapter
+                                                                              ReusedExchange [item_sk,d_date,sumss,ss_item_sk] #14
+              InputAdapter
+                WholeStageCodegen (61)
+                  Project [item_sk,web_sales,store_sales,rk]
+                    InputAdapter
+                      Window [item_sk,d_date]
+                        WholeStageCodegen (60)
+                          Sort [item_sk,d_date]
+                            InputAdapter
+                              ReusedExchange [item_sk,d_date,web_sales,store_sales] #15
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/explain.txt
new file mode 100644
index 0000000000000..b7d3461ce34ea
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/explain.txt
@@ -0,0 +1,410 @@
+== Physical Plan ==
+TakeOrderedAndProject (71)
++- * Filter (70)
+   +- * HashAggregate (69)
+      +- * HashAggregate (68)
+         +- * Project (67)
+            +- * BroadcastHashJoin Inner BuildRight (66)
+               :- Window (60)
+               :  +- * Sort (59)
+               :     +- Exchange (58)
+               :        +- * Project (57)
+               :           +- * Filter (56)
+               :              +- SortMergeJoin FullOuter (55)
+               :                 :- * Sort (29)
+               :                 :  +- Exchange (28)
+               :                 :     +- * HashAggregate (27)
+               :                 :        +- Exchange (26)
+               :                 :           +- * HashAggregate (25)
+               :                 :              +- * Project (24)
+               :                 :                 +- * BroadcastHashJoin Inner BuildRight (23)
+               :                 :                    :- * Project (17)
+               :                 :                    :  +- Window (16)
+               :                 :                    :     +- * Sort (15)
+               :                 :                    :        +- Exchange (14)
+               :                 :                    :           +- * HashAggregate (13)
+               :                 :                    :              +- Exchange (12)
+               :                 :                    :                 +- * HashAggregate (11)
+               :                 :                    :                    +- * Project (10)
+               :                 :                    :                       +- * BroadcastHashJoin Inner BuildRight (9)
+               :                 :                    :                          :- * Filter (3)
+               :                 :                    :                          :  +- * ColumnarToRow (2)
+               :                 :                    :                          :     +- Scan parquet default.web_sales (1)
+               :                 :                    :                          +- BroadcastExchange (8)
+               :                 :                    :                             +- * Project (7)
+               :                 :                    :                                +- * Filter (6)
+               :                 :                    :                                   +- * ColumnarToRow (5)
+               :                 :                    :                                      +- Scan parquet default.date_dim (4)
+               :                 :                    +- BroadcastExchange (22)
+               :                 :                       +- * Project (21)
+               :                 :                          +- Window (20)
+               :                 :                             +- * Sort (19)
+               :                 :                                +- ReusedExchange (18)
+               :                 +- * Sort (54)
+               :                    +- Exchange (53)
+               :                       +- * HashAggregate (52)
+               :                          +- Exchange (51)
+               :                             +- * HashAggregate (50)
+               :                                +- * Project (49)
+               :                                   +- * BroadcastHashJoin Inner BuildRight (48)
+               :                                      :- * Project (42)
+               :                                      :  +- Window (41)
+               :                                      :     +- * Sort (40)
+               :                                      :        +- Exchange (39)
+               :                                      :           +- * HashAggregate (38)
+               :                                      :              +- Exchange (37)
+               :                                      :                 +- * HashAggregate (36)
+               :                                      :                    +- * Project (35)
+               :                                      :                       +- * BroadcastHashJoin Inner BuildRight (34)
+               :                                      :                          :- * Filter (32)
+               :                                      :                          :  +- * ColumnarToRow (31)
+               :                                      :                          :     +- Scan parquet default.store_sales (30)
+               :                                      :                          +- ReusedExchange (33)
+               :                                      +- BroadcastExchange (47)
+               :                                         +- * Project (46)
+               :                                            +- Window (45)
+               :                                               +- * Sort (44)
+               :                                                  +- ReusedExchange (43)
+               +- BroadcastExchange (65)
+                  +- * Project (64)
+                     +- Window (63)
+                        +- * Sort (62)
+                           +- ReusedExchange (61)
+
+
+(1) Scan parquet default.web_sales
+Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3]
+
+(3) Filter [codegen id : 2]
+Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3]
+Condition : isnotnull(ws_item_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date,d_month_seq:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7]
+Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#5))
+
+(7) Project [codegen id : 1]
+Output [2]: [d_date_sk#5, d_date#6]
+Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7]
+
+(8) BroadcastExchange
+Input [2]: [d_date_sk#5, d_date#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#8]
+
+(9) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ws_sold_date_sk#3]
+Right keys [1]: [d_date_sk#5]
+Join condition: None
+
+(10) Project [codegen id : 2]
+Output [3]: [ws_item_sk#1, ws_sales_price#2, d_date#6]
+Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#5, d_date#6]
+
+(11) HashAggregate [codegen id : 2]
+Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#6]
+Keys [2]: [ws_item_sk#1, d_date#6]
+Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))]
+Aggregate Attributes [1]: [sum#9]
+Results [3]: [ws_item_sk#1, d_date#6, sum#10]
+
+(12) Exchange
+Input [3]: [ws_item_sk#1, d_date#6, sum#10]
+Arguments: hashpartitioning(ws_item_sk#1, d_date#6, 5), ENSURE_REQUIREMENTS, [id=#11]
+
+(13) HashAggregate [codegen id : 3]
+Input [3]: [ws_item_sk#1, d_date#6, sum#10]
+Keys [2]: [ws_item_sk#1, d_date#6]
+Functions [1]: [sum(UnscaledValue(ws_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#2))#12]
+Results [4]: [ws_item_sk#1 AS item_sk#13, d_date#6, MakeDecimal(sum(UnscaledValue(ws_sales_price#2))#12,17,2) AS sumws#14, ws_item_sk#1]
+
+(14) Exchange
+Input [4]: [item_sk#13, d_date#6, sumws#14, ws_item_sk#1]
+Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(15) Sort [codegen id : 4]
+Input [4]: [item_sk#13, d_date#6, sumws#14, ws_item_sk#1]
+Arguments: [ws_item_sk#1 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST], false, 0
+
+(16) Window
+Input [4]: [item_sk#13, d_date#6, sumws#14, ws_item_sk#1]
+Arguments: [row_number() windowspecdefinition(ws_item_sk#1, d_date#6 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#16], [ws_item_sk#1], [d_date#6 ASC NULLS FIRST]
+
+(17) Project [codegen id : 10]
+Output [4]: [item_sk#13, d_date#6, sumws#14, rk#16]
+Input [5]: [item_sk#13, d_date#6, sumws#14, ws_item_sk#1, rk#16]
+
+(18) ReusedExchange [Reuses operator id: unknown]
+Output [4]: [item_sk#17, d_date#18, sumws#19, ws_item_sk#20]
+
+(19) Sort [codegen id : 8]
+Input [4]: [item_sk#17, d_date#18, sumws#19, ws_item_sk#20]
+Arguments: [ws_item_sk#20 ASC NULLS FIRST, d_date#18 ASC NULLS FIRST], false, 0
+
+(20) Window
+Input [4]: [item_sk#17, d_date#18, sumws#19, ws_item_sk#20]
+Arguments: [row_number() windowspecdefinition(ws_item_sk#20, d_date#18 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#21], [ws_item_sk#20], [d_date#18 ASC NULLS FIRST]
+
+(21) Project [codegen id : 9]
+Output [3]: [item_sk#17, sumws#19, rk#21]
+Input [5]: [item_sk#17, d_date#18, sumws#19, ws_item_sk#20, rk#21]
+
+(22) BroadcastExchange
+Input [3]: [item_sk#17, sumws#19, rk#21]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#22]
+
+(23) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [item_sk#13]
+Right keys [1]: [item_sk#17]
+Join condition: (rk#16 >= rk#21)
+
+(24) Project [codegen id : 10]
+Output [4]: [item_sk#13, d_date#6, sumws#14, sumws#19]
+Input [7]: [item_sk#13, d_date#6, sumws#14, rk#16, item_sk#17, sumws#19, rk#21]
+
+(25) HashAggregate [codegen id : 10]
+Input [4]: [item_sk#13, d_date#6, sumws#14, sumws#19]
+Keys [3]: [item_sk#13, d_date#6, sumws#14]
+Functions [1]: [partial_sum(sumws#19)]
+Aggregate Attributes [2]: [sum#23, isEmpty#24]
+Results [5]: [item_sk#13, d_date#6, sumws#14, sum#25, isEmpty#26]
+
+(26) Exchange
+Input [5]: [item_sk#13, d_date#6, sumws#14, sum#25, isEmpty#26]
+Arguments: hashpartitioning(item_sk#13, d_date#6, sumws#14, 5), ENSURE_REQUIREMENTS, [id=#27]
+
+(27) HashAggregate [codegen id : 11]
+Input [5]: [item_sk#13, d_date#6, sumws#14, sum#25, isEmpty#26]
+Keys [3]: [item_sk#13, d_date#6, sumws#14]
+Functions [1]: [sum(sumws#19)]
+Aggregate Attributes [1]: [sum(sumws#19)#28]
+Results [3]: [item_sk#13, d_date#6, sum(sumws#19)#28 AS cume_sales#29]
+
+(28) Exchange
+Input [3]: [item_sk#13, d_date#6, cume_sales#29]
+Arguments: hashpartitioning(item_sk#13, d_date#6, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(29) Sort [codegen id : 12]
+Input [3]: [item_sk#13, d_date#6, cume_sales#29]
+Arguments: [item_sk#13 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST], false, 0
+
+(30) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#31, ss_sales_price#32, ss_sold_date_sk#33]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#33), dynamicpruningexpression(ss_sold_date_sk#33 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_sales_price:decimal(7,2)>
+
+(31) ColumnarToRow [codegen id : 14]
+Input [3]: [ss_item_sk#31, ss_sales_price#32, ss_sold_date_sk#33]
+
+(32) Filter [codegen id : 14]
+Input [3]: [ss_item_sk#31, ss_sales_price#32, ss_sold_date_sk#33]
+Condition : isnotnull(ss_item_sk#31)
+
+(33) ReusedExchange [Reuses operator id: 8]
+Output [2]: [d_date_sk#34, d_date#35]
+
+(34) BroadcastHashJoin [codegen id : 14]
+Left keys [1]: [ss_sold_date_sk#33]
+Right keys [1]: [d_date_sk#34]
+Join condition: None
+
+(35) Project [codegen id : 14]
+Output [3]: [ss_item_sk#31, ss_sales_price#32, d_date#35]
+Input [5]: [ss_item_sk#31, ss_sales_price#32, ss_sold_date_sk#33, d_date_sk#34, d_date#35]
+
+(36) HashAggregate [codegen id : 14]
+Input [3]: [ss_item_sk#31, ss_sales_price#32, d_date#35]
+Keys [2]: [ss_item_sk#31, d_date#35]
+Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#32))]
+Aggregate Attributes [1]: [sum#36]
+Results [3]: [ss_item_sk#31, d_date#35, sum#37]
+
+(37) Exchange
+Input [3]: [ss_item_sk#31, d_date#35, sum#37]
+Arguments: hashpartitioning(ss_item_sk#31, d_date#35, 5), ENSURE_REQUIREMENTS, [id=#38]
+
+(38) HashAggregate [codegen id : 15]
+Input [3]: [ss_item_sk#31, d_date#35, sum#37]
+Keys [2]: [ss_item_sk#31, d_date#35]
+Functions [1]: [sum(UnscaledValue(ss_sales_price#32))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#32))#39]
+Results [4]: [ss_item_sk#31 AS item_sk#40, d_date#35, MakeDecimal(sum(UnscaledValue(ss_sales_price#32))#39,17,2) AS sumss#41, ss_item_sk#31]
+
+(39) Exchange
+Input [4]: [item_sk#40, d_date#35, sumss#41, ss_item_sk#31]
+Arguments: hashpartitioning(ss_item_sk#31, 5), ENSURE_REQUIREMENTS, [id=#42]
+
+(40) Sort [codegen id : 16]
+Input [4]: [item_sk#40, d_date#35, sumss#41, ss_item_sk#31]
+Arguments: [ss_item_sk#31 ASC NULLS FIRST, d_date#35 ASC NULLS FIRST], false, 0
+
+(41) Window
+Input [4]: [item_sk#40, d_date#35, sumss#41, ss_item_sk#31]
+Arguments: [row_number() windowspecdefinition(ss_item_sk#31, d_date#35 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#43], [ss_item_sk#31], [d_date#35 ASC NULLS FIRST]
+
+(42) Project [codegen id : 22]
+Output [4]: [item_sk#40, d_date#35, sumss#41, rk#43]
+Input [5]: [item_sk#40, d_date#35, sumss#41, ss_item_sk#31, rk#43]
+
+(43) ReusedExchange [Reuses operator id: unknown]
+Output [4]: [item_sk#44, d_date#45, sumss#46, ss_item_sk#47]
+
+(44) Sort [codegen id : 20]
+Input [4]: [item_sk#44, d_date#45, sumss#46, ss_item_sk#47]
+Arguments: [ss_item_sk#47 ASC NULLS FIRST, d_date#45 ASC NULLS FIRST], false, 0
+
+(45) Window
+Input [4]: [item_sk#44, d_date#45, sumss#46, ss_item_sk#47]
+Arguments: [row_number() windowspecdefinition(ss_item_sk#47, d_date#45 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#48], [ss_item_sk#47], [d_date#45 ASC NULLS FIRST]
+
+(46) Project [codegen id : 21]
+Output [3]: [item_sk#44, sumss#46, rk#48]
+Input [5]: [item_sk#44, d_date#45, sumss#46, ss_item_sk#47, rk#48]
+
+(47) BroadcastExchange
+Input [3]: [item_sk#44, sumss#46, rk#48]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#49]
+
+(48) BroadcastHashJoin [codegen id : 22]
+Left keys [1]: [item_sk#40]
+Right keys [1]: [item_sk#44]
+Join condition: (rk#43 >= rk#48)
+
+(49) Project [codegen id : 22]
+Output [4]: [item_sk#40, d_date#35, sumss#41, sumss#46]
+Input [7]: [item_sk#40, d_date#35, sumss#41, rk#43, item_sk#44, sumss#46, rk#48]
+
+(50) HashAggregate [codegen id : 22]
+Input [4]: [item_sk#40, d_date#35, sumss#41, sumss#46]
+Keys [3]: [item_sk#40, d_date#35, sumss#41]
+Functions [1]: [partial_sum(sumss#46)]
+Aggregate Attributes [2]: [sum#50, isEmpty#51]
+Results [5]: [item_sk#40, d_date#35, sumss#41, sum#52, isEmpty#53]
+
+(51) Exchange
+Input [5]: [item_sk#40, d_date#35, sumss#41, sum#52, isEmpty#53]
+Arguments: hashpartitioning(item_sk#40, d_date#35, sumss#41, 5), ENSURE_REQUIREMENTS, [id=#54]
+
+(52) HashAggregate [codegen id : 23]
+Input [5]: [item_sk#40, d_date#35, sumss#41, sum#52, isEmpty#53]
+Keys [3]: [item_sk#40, d_date#35, sumss#41]
+Functions [1]: [sum(sumss#46)]
+Aggregate Attributes [1]: [sum(sumss#46)#55]
+Results [3]: [item_sk#40, d_date#35, sum(sumss#46)#55 AS cume_sales#56]
+
+(53) Exchange
+Input [3]: [item_sk#40, d_date#35, cume_sales#56]
+Arguments: hashpartitioning(item_sk#40, d_date#35, 5), ENSURE_REQUIREMENTS, [id=#57]
+
+(54) Sort [codegen id : 24]
+Input [3]: [item_sk#40, d_date#35, cume_sales#56]
+Arguments: [item_sk#40 ASC NULLS FIRST, d_date#35 ASC NULLS FIRST], false, 0
+
+(55) SortMergeJoin
+Left keys [2]: [item_sk#13, d_date#6]
+Right keys [2]: [item_sk#40, d_date#35]
+Join condition: None
+
+(56) Filter [codegen id : 25]
+Input [6]: [item_sk#13, d_date#6, cume_sales#29, item_sk#40, d_date#35, cume_sales#56]
+Condition : isnotnull(CASE WHEN isnotnull(item_sk#13) THEN item_sk#13 ELSE item_sk#40 END)
+
+(57) Project [codegen id : 25]
+Output [4]: [CASE WHEN isnotnull(item_sk#13) THEN item_sk#13 ELSE item_sk#40 END AS item_sk#58, CASE WHEN isnotnull(d_date#6) THEN d_date#6 ELSE d_date#35 END AS d_date#59, cume_sales#29 AS web_sales#60, cume_sales#56 AS store_sales#61]
+Input [6]: [item_sk#13, d_date#6, cume_sales#29, item_sk#40, d_date#35, cume_sales#56]
+
+(58) Exchange
+Input [4]: [item_sk#58, d_date#59, web_sales#60, store_sales#61]
+Arguments: hashpartitioning(item_sk#58, 5), ENSURE_REQUIREMENTS, [id=#62]
+
+(59) Sort [codegen id : 26]
+Input [4]: [item_sk#58, d_date#59, web_sales#60, store_sales#61]
+Arguments: [item_sk#58 ASC NULLS FIRST, d_date#59 ASC NULLS FIRST], false, 0
+
+(60) Window
+Input [4]: [item_sk#58, d_date#59, web_sales#60, store_sales#61]
+Arguments: [row_number() windowspecdefinition(item_sk#58, d_date#59 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#63], [item_sk#58], [d_date#59 ASC NULLS FIRST]
+
+(61) ReusedExchange [Reuses operator id: unknown]
+Output [4]: [item_sk#64, d_date#65, web_sales#66, store_sales#67]
+
+(62) Sort [codegen id : 52]
+Input [4]: [item_sk#64, d_date#65, web_sales#66, store_sales#67]
+Arguments: [item_sk#64 ASC NULLS FIRST, d_date#65 ASC NULLS FIRST], false, 0
+
+(63) Window
+Input [4]: [item_sk#64, d_date#65, web_sales#66, store_sales#67]
+Arguments: [row_number() windowspecdefinition(item_sk#64, d_date#65 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#68], [item_sk#64], [d_date#65 ASC NULLS FIRST]
+
+(64) Project [codegen id : 53]
+Output [4]: [item_sk#64, web_sales#66, store_sales#67, rk#68]
+Input [5]: [item_sk#64, d_date#65, web_sales#66, store_sales#67, rk#68]
+
+(65) BroadcastExchange
+Input [4]: [item_sk#64, web_sales#66, store_sales#67, rk#68]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#69]
+
+(66) BroadcastHashJoin [codegen id : 54]
+Left keys [1]: [item_sk#58]
+Right keys [1]: [item_sk#64]
+Join condition: (rk#63 >= rk#68)
+
+(67) Project [codegen id : 54]
+Output [6]: [item_sk#58, d_date#59, web_sales#60, store_sales#61, web_sales#66, store_sales#67]
+Input [9]: [item_sk#58, d_date#59, web_sales#60, store_sales#61, rk#63, item_sk#64, web_sales#66, store_sales#67, rk#68]
+
+(68) HashAggregate [codegen id : 54]
+Input [6]: [item_sk#58, d_date#59, web_sales#60, store_sales#61, web_sales#66, store_sales#67]
+Keys [4]: [item_sk#58, d_date#59, web_sales#60, store_sales#61]
+Functions [2]: [partial_max(web_sales#66), partial_max(store_sales#67)]
+Aggregate Attributes [2]: [max#70, max#71]
+Results [6]: [item_sk#58, d_date#59, web_sales#60, store_sales#61, max#72, max#73]
+
+(69) HashAggregate [codegen id : 54]
+Input [6]: [item_sk#58, d_date#59, web_sales#60, store_sales#61, max#72, max#73]
+Keys [4]: [item_sk#58, d_date#59, web_sales#60, store_sales#61]
+Functions [2]: [max(web_sales#66), max(store_sales#67)]
+Aggregate Attributes [2]: [max(web_sales#66)#74, max(store_sales#67)#75]
+Results [6]: [item_sk#58, d_date#59, web_sales#60, store_sales#61, max(web_sales#66)#74 AS web_cumulative#76, max(store_sales#67)#75 AS store_cumulative#77]
+
+(70) Filter [codegen id : 54]
+Input [6]: [item_sk#58, d_date#59, web_sales#60, store_sales#61, web_cumulative#76, store_cumulative#77]
+Condition : ((isnotnull(web_cumulative#76) AND isnotnull(store_cumulative#77)) AND (web_cumulative#76 > store_cumulative#77))
+
+(71) TakeOrderedAndProject
+Input [6]: [item_sk#58, d_date#59, web_sales#60, store_sales#61, web_cumulative#76, store_cumulative#77]
+Arguments: 100, [item_sk#58 ASC NULLS FIRST, d_date#59 ASC NULLS FIRST], [item_sk#58, d_date#59, web_sales#60, store_sales#61, web_cumulative#76, store_cumulative#77]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (72)
+
+
+(72) ReusedExchange [Reuses operator id: 8]
+Output [2]: [d_date_sk#5, d_date#6]
+
+Subquery:2 Hosting operator id = 30 Hosting Expression = ss_sold_date_sk#33 IN dynamicpruning#4
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/simplified.txt
new file mode 100644
index 0000000000000..0359376372a8b
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/simplified.txt
@@ -0,0 +1,123 @@
+TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative]
+  WholeStageCodegen (54)
+    Filter [web_cumulative,store_cumulative]
+      HashAggregate [item_sk,d_date,web_sales,store_sales,max,max] [max(web_sales),max(store_sales),web_cumulative,store_cumulative,max,max]
+        HashAggregate [item_sk,d_date,web_sales,store_sales,web_sales,store_sales] [max,max,max,max]
+          Project [item_sk,d_date,web_sales,store_sales,web_sales,store_sales]
+            BroadcastHashJoin [item_sk,item_sk,rk,rk]
+              InputAdapter
+                Window [item_sk,d_date]
+                  WholeStageCodegen (26)
+                    Sort [item_sk,d_date]
+                      InputAdapter
+                        Exchange [item_sk] #1
+                          WholeStageCodegen (25)
+                            Project [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales]
+                              Filter [item_sk,item_sk]
+                                InputAdapter
+                                  SortMergeJoin [item_sk,d_date,item_sk,d_date]
+                                    WholeStageCodegen (12)
+                                      Sort [item_sk,d_date]
+                                        InputAdapter
+                                          Exchange [item_sk,d_date] #2
+                                            WholeStageCodegen (11)
+                                              HashAggregate [item_sk,d_date,sumws,sum,isEmpty] [sum(sumws),cume_sales,sum,isEmpty]
+                                                InputAdapter
+                                                  Exchange [item_sk,d_date,sumws] #3
+                                                    WholeStageCodegen (10)
+                                                      HashAggregate [item_sk,d_date,sumws,sumws] [sum,isEmpty,sum,isEmpty]
+                                                        Project [item_sk,d_date,sumws,sumws]
+                                                          BroadcastHashJoin [item_sk,item_sk,rk,rk]
+                                                            Project [item_sk,d_date,sumws,rk]
+                                                              InputAdapter
+                                                                Window [ws_item_sk,d_date]
+                                                                  WholeStageCodegen (4)
+                                                                    Sort [ws_item_sk,d_date]
+                                                                      InputAdapter
+                                                                        Exchange [ws_item_sk] #4
+                                                                          WholeStageCodegen (3)
+                                                                            HashAggregate [ws_item_sk,d_date,sum] [sum(UnscaledValue(ws_sales_price)),item_sk,sumws,sum]
+                                                                              InputAdapter
+                                                                                Exchange [ws_item_sk,d_date] #5
+                                                                                  WholeStageCodegen (2)
+                                                                                    HashAggregate [ws_item_sk,d_date,ws_sales_price] [sum,sum]
+                                                                                      Project [ws_item_sk,ws_sales_price,d_date]
+                                                                                        BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                                          Filter [ws_item_sk]
+                                                                                            ColumnarToRow
+                                                                                              InputAdapter
+                                                                                                Scan parquet default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk]
+                                                                                                  SubqueryBroadcast [d_date_sk] #1
+                                                                                                    ReusedExchange [d_date_sk,d_date] #6
+                                                                                          InputAdapter
+                                                                                            BroadcastExchange #6
+                                                                                              WholeStageCodegen (1)
+                                                                                                Project [d_date_sk,d_date]
+                                                                                                  Filter [d_month_seq,d_date_sk]
+                                                                                                    ColumnarToRow
+                                                                                                      InputAdapter
+                                                                                                        Scan parquet default.date_dim [d_date_sk,d_date,d_month_seq]
+                                                            InputAdapter
+                                                              BroadcastExchange #7
+                                                                WholeStageCodegen (9)
+                                                                  Project [item_sk,sumws,rk]
+                                                                    InputAdapter
+                                                                      Window [ws_item_sk,d_date]
+                                                                        WholeStageCodegen (8)
+                                                                          Sort [ws_item_sk,d_date]
+                                                                            InputAdapter
+                                                                              ReusedExchange [item_sk,d_date,sumws,ws_item_sk] #8
+                                    WholeStageCodegen (24)
+                                      Sort [item_sk,d_date]
+                                        InputAdapter
+                                          Exchange [item_sk,d_date] #9
+                                            WholeStageCodegen (23)
+                                              HashAggregate [item_sk,d_date,sumss,sum,isEmpty] [sum(sumss),cume_sales,sum,isEmpty]
+                                                InputAdapter
+                                                  Exchange [item_sk,d_date,sumss] #10
+                                                    WholeStageCodegen (22)
+                                                      HashAggregate [item_sk,d_date,sumss,sumss] [sum,isEmpty,sum,isEmpty]
+                                                        Project [item_sk,d_date,sumss,sumss]
+                                                          BroadcastHashJoin [item_sk,item_sk,rk,rk]
+                                                            Project [item_sk,d_date,sumss,rk]
+                                                              InputAdapter
+                                                                Window [ss_item_sk,d_date]
+                                                                  WholeStageCodegen (16)
+                                                                    Sort [ss_item_sk,d_date]
+                                                                      InputAdapter
+                                                                        Exchange [ss_item_sk] #11
+                                                                          WholeStageCodegen (15)
+                                                                            HashAggregate [ss_item_sk,d_date,sum] [sum(UnscaledValue(ss_sales_price)),item_sk,sumss,sum]
+                                                                              InputAdapter
+                                                                                Exchange [ss_item_sk,d_date] #12
+                                                                                  WholeStageCodegen (14)
+                                                                                    HashAggregate [ss_item_sk,d_date,ss_sales_price] [sum,sum]
+                                                                                      Project [ss_item_sk,ss_sales_price,d_date]
+                                                                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                          Filter [ss_item_sk]
+                                                                                            ColumnarToRow
+                                                                                              InputAdapter
+                                                                                                Scan parquet default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk]
+                                                                                                  ReusedSubquery [d_date_sk] #1
+                                                                                          InputAdapter
+                                                                                            ReusedExchange [d_date_sk,d_date] #6
+                                                            InputAdapter
+                                                              BroadcastExchange #13
+                                                                WholeStageCodegen (21)
+                                                                  Project [item_sk,sumss,rk]
+                                                                    InputAdapter
+                                                                      Window [ss_item_sk,d_date]
+                                                                        WholeStageCodegen (20)
+                                                                          Sort [ss_item_sk,d_date]
+                                                                            InputAdapter
+                                                                              ReusedExchange [item_sk,d_date,sumss,ss_item_sk] #14
+              InputAdapter
+                BroadcastExchange #15
+                  WholeStageCodegen (53)
+                    Project [item_sk,web_sales,store_sales,rk]
+                      InputAdapter
+                        Window [item_sk,d_date]
+                          WholeStageCodegen (52)
+                            Sort [item_sk,d_date]
+                              InputAdapter
+                                ReusedExchange [item_sk,d_date,web_sales,store_sales] #16
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.sf100/explain.txt
new file mode 100644
index 0000000000000..e3de4a7a053b1
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.sf100/explain.txt
@@ -0,0 +1,314 @@
+== Physical Plan ==
+TakeOrderedAndProject (56)
++- * Project (55)
+   +- * SortMergeJoin Inner (54)
+      :- * Project (47)
+      :  +- * SortMergeJoin Inner (46)
+      :     :- * Sort (37)
+      :     :  +- Exchange (36)
+      :     :     +- * Filter (35)
+      :     :        +- Window (34)
+      :     :           +- * Sort (33)
+      :     :              +- Exchange (32)
+      :     :                 +- * Project (31)
+      :     :                    +- Window (30)
+      :     :                       +- * Sort (29)
+      :     :                          +- Exchange (28)
+      :     :                             +- * HashAggregate (27)
+      :     :                                +- Exchange (26)
+      :     :                                   +- * HashAggregate (25)
+      :     :                                      +- * Project (24)
+      :     :                                         +- * SortMergeJoin Inner (23)
+      :     :                                            :- * Sort (17)
+      :     :                                            :  +- Exchange (16)
+      :     :                                            :     +- * Project (15)
+      :     :                                            :        +- * BroadcastHashJoin Inner BuildRight (14)
+      :     :                                            :           :- * Project (9)
+      :     :                                            :           :  +- * BroadcastHashJoin Inner BuildRight (8)
+      :     :                                            :           :     :- * Filter (3)
+      :     :                                            :           :     :  +- * ColumnarToRow (2)
+      :     :                                            :           :     :     +- Scan parquet default.catalog_sales (1)
+      :     :                                            :           :     +- BroadcastExchange (7)
+      :     :                                            :           :        +- * Filter (6)
+      :     :                                            :           :           +- * ColumnarToRow (5)
+      :     :                                            :           :              +- Scan parquet default.date_dim (4)
+      :     :                                            :           +- BroadcastExchange (13)
+      :     :                                            :              +- * Filter (12)
+      :     :                                            :                 +- * ColumnarToRow (11)
+      :     :                                            :                    +- Scan parquet default.call_center (10)
+      :     :                                            +- * Sort (22)
+      :     :                                               +- Exchange (21)
+      :     :                                                  +- * Filter (20)
+      :     :                                                     +- * ColumnarToRow (19)
+      :     :                                                        +- Scan parquet default.item (18)
+      :     +- * Sort (45)
+      :        +- Exchange (44)
+      :           +- * Project (43)
+      :              +- Window (42)
+      :                 +- * Sort (41)
+      :                    +- Exchange (40)
+      :                       +- * HashAggregate (39)
+      :                          +- ReusedExchange (38)
+      +- * Sort (53)
+         +- Exchange (52)
+            +- * Project (51)
+               +- Window (50)
+                  +- * Sort (49)
+                     +- ReusedExchange (48)
+
+
+(1) Scan parquet default.catalog_sales
+Output [4]: [cs_call_center_sk#1, cs_item_sk#2, cs_sales_price#3, cs_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#4), dynamicpruningexpression(cs_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)]
+ReadSchema: struct<cs_call_center_sk:int,cs_item_sk:int,cs_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [4]: [cs_call_center_sk#1, cs_item_sk#2, cs_sales_price#3, cs_sold_date_sk#4]
+
+(3) Filter [codegen id : 3]
+Input [4]: [cs_call_center_sk#1, cs_item_sk#2, cs_sales_price#3, cs_sold_date_sk#4]
+Condition : (isnotnull(cs_item_sk#2) AND isnotnull(cs_call_center_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+(6) Filter [codegen id : 1]
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Condition : ((((d_year#7 = 1999) OR ((d_year#7 = 1998) AND (d_moy#8 = 12))) OR ((d_year#7 = 2000) AND (d_moy#8 = 1))) AND isnotnull(d_date_sk#6))
+
+(7) BroadcastExchange
+Input [3]: [d_date_sk#6, d_year#7, d_moy#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#9]
+
+(8) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_sold_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(9) Project [codegen id : 3]
+Output [5]: [cs_call_center_sk#1, cs_item_sk#2, cs_sales_price#3, d_year#7, d_moy#8]
+Input [7]: [cs_call_center_sk#1, cs_item_sk#2, cs_sales_price#3, cs_sold_date_sk#4, d_date_sk#6, d_year#7, d_moy#8]
+
+(10) Scan parquet default.call_center
+Output [2]: [cc_call_center_sk#10, cc_name#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/call_center]
+PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)]
+ReadSchema: struct<cc_call_center_sk:int,cc_name:string>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [cc_call_center_sk#10, cc_name#11]
+
+(12) Filter [codegen id : 2]
+Input [2]: [cc_call_center_sk#10, cc_name#11]
+Condition : (isnotnull(cc_call_center_sk#10) AND isnotnull(cc_name#11))
+
+(13) BroadcastExchange
+Input [2]: [cc_call_center_sk#10, cc_name#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#12]
+
+(14) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_call_center_sk#1]
+Right keys [1]: [cc_call_center_sk#10]
+Join condition: None
+
+(15) Project [codegen id : 3]
+Output [5]: [cs_item_sk#2, cs_sales_price#3, d_year#7, d_moy#8, cc_name#11]
+Input [7]: [cs_call_center_sk#1, cs_item_sk#2, cs_sales_price#3, d_year#7, d_moy#8, cc_call_center_sk#10, cc_name#11]
+
+(16) Exchange
+Input [5]: [cs_item_sk#2, cs_sales_price#3, d_year#7, d_moy#8, cc_name#11]
+Arguments: hashpartitioning(cs_item_sk#2, 5), ENSURE_REQUIREMENTS, [id=#13]
+
+(17) Sort [codegen id : 4]
+Input [5]: [cs_item_sk#2, cs_sales_price#3, d_year#7, d_moy#8, cc_name#11]
+Arguments: [cs_item_sk#2 ASC NULLS FIRST], false, 0
+
+(18) Scan parquet default.item
+Output [3]: [i_item_sk#14, i_brand#15, i_category#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_category:string>
+
+(19) ColumnarToRow [codegen id : 5]
+Input [3]: [i_item_sk#14, i_brand#15, i_category#16]
+
+(20) Filter [codegen id : 5]
+Input [3]: [i_item_sk#14, i_brand#15, i_category#16]
+Condition : ((isnotnull(i_item_sk#14) AND isnotnull(i_category#16)) AND isnotnull(i_brand#15))
+
+(21) Exchange
+Input [3]: [i_item_sk#14, i_brand#15, i_category#16]
+Arguments: hashpartitioning(i_item_sk#14, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(22) Sort [codegen id : 6]
+Input [3]: [i_item_sk#14, i_brand#15, i_category#16]
+Arguments: [i_item_sk#14 ASC NULLS FIRST], false, 0
+
+(23) SortMergeJoin [codegen id : 7]
+Left keys [1]: [cs_item_sk#2]
+Right keys [1]: [i_item_sk#14]
+Join condition: None
+
+(24) Project [codegen id : 7]
+Output [6]: [i_brand#15, i_category#16, cs_sales_price#3, d_year#7, d_moy#8, cc_name#11]
+Input [8]: [cs_item_sk#2, cs_sales_price#3, d_year#7, d_moy#8, cc_name#11, i_item_sk#14, i_brand#15, i_category#16]
+
+(25) HashAggregate [codegen id : 7]
+Input [6]: [i_brand#15, i_category#16, cs_sales_price#3, d_year#7, d_moy#8, cc_name#11]
+Keys [5]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8]
+Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#3))]
+Aggregate Attributes [1]: [sum#18]
+Results [6]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum#19]
+
+(26) Exchange
+Input [6]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum#19]
+Arguments: hashpartitioning(i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(27) HashAggregate [codegen id : 8]
+Input [6]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum#19]
+Keys [5]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8]
+Functions [1]: [sum(UnscaledValue(cs_sales_price#3))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#3))#21]
+Results [7]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, MakeDecimal(sum(UnscaledValue(cs_sales_price#3))#21,17,2) AS sum_sales#22, MakeDecimal(sum(UnscaledValue(cs_sales_price#3))#21,17,2) AS _w0#23]
+
+(28) Exchange
+Input [7]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, _w0#23]
+Arguments: hashpartitioning(i_category#16, i_brand#15, cc_name#11, d_year#7, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(29) Sort [codegen id : 9]
+Input [7]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, _w0#23]
+Arguments: [i_category#16 ASC NULLS FIRST, i_brand#15 ASC NULLS FIRST, cc_name#11 ASC NULLS FIRST, d_year#7 ASC NULLS FIRST], false, 0
+
+(30) Window
+Input [7]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, _w0#23]
+Arguments: [avg(_w0#23) windowspecdefinition(i_category#16, i_brand#15, cc_name#11, d_year#7, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#25], [i_category#16, i_brand#15, cc_name#11, d_year#7]
+
+(31) Project [codegen id : 10]
+Output [7]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, avg_monthly_sales#25]
+Input [8]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, _w0#23, avg_monthly_sales#25]
+
+(32) Exchange
+Input [7]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, avg_monthly_sales#25]
+Arguments: hashpartitioning(i_category#16, i_brand#15, cc_name#11, 5), ENSURE_REQUIREMENTS, [id=#26]
+
+(33) Sort [codegen id : 11]
+Input [7]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, avg_monthly_sales#25]
+Arguments: [i_category#16 ASC NULLS FIRST, i_brand#15 ASC NULLS FIRST, cc_name#11 ASC NULLS FIRST, d_year#7 ASC NULLS FIRST, d_moy#8 ASC NULLS FIRST], false, 0
+
+(34) Window
+Input [7]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, avg_monthly_sales#25]
+Arguments: [rank(d_year#7, d_moy#8) windowspecdefinition(i_category#16, i_brand#15, cc_name#11, d_year#7 ASC NULLS FIRST, d_moy#8 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#16, i_brand#15, cc_name#11], [d_year#7 ASC NULLS FIRST, d_moy#8 ASC NULLS FIRST]
+
+(35) Filter [codegen id : 12]
+Input [8]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, avg_monthly_sales#25, rn#27]
+Condition : ((((isnotnull(d_year#7) AND isnotnull(avg_monthly_sales#25)) AND (d_year#7 = 1999)) AND (avg_monthly_sales#25 > 0.000000)) AND (CheckOverflow((promote_precision(abs(CheckOverflow((promote_precision(cast(sum_sales#22 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#25 as decimal(22,6)))), DecimalType(22,6), true), false)) / promote_precision(cast(avg_monthly_sales#25 as decimal(22,6)))), DecimalType(38,16), true) > 0.1000000000000000))
+
+(36) Exchange
+Input [8]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, avg_monthly_sales#25, rn#27]
+Arguments: hashpartitioning(i_category#16, i_brand#15, cc_name#11, rn#27, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(37) Sort [codegen id : 13]
+Input [8]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, avg_monthly_sales#25, rn#27]
+Arguments: [i_category#16 ASC NULLS FIRST, i_brand#15 ASC NULLS FIRST, cc_name#11 ASC NULLS FIRST, rn#27 ASC NULLS FIRST], false, 0
+
+(38) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum#34]
+
+(39) HashAggregate [codegen id : 21]
+Input [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum#34]
+Keys [5]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33]
+Functions [1]: [sum(UnscaledValue(cs_sales_price#35))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#35))#36]
+Results [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, MakeDecimal(sum(UnscaledValue(cs_sales_price#35))#36,17,2) AS sum_sales#37]
+
+(40) Exchange
+Input [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#37]
+Arguments: hashpartitioning(i_category#29, i_brand#30, cc_name#31, 5), ENSURE_REQUIREMENTS, [id=#38]
+
+(41) Sort [codegen id : 22]
+Input [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#37]
+Arguments: [i_category#29 ASC NULLS FIRST, i_brand#30 ASC NULLS FIRST, cc_name#31 ASC NULLS FIRST, d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST], false, 0
+
+(42) Window
+Input [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#37]
+Arguments: [rank(d_year#32, d_moy#33) windowspecdefinition(i_category#29, i_brand#30, cc_name#31, d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#39], [i_category#29, i_brand#30, cc_name#31], [d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST]
+
+(43) Project [codegen id : 23]
+Output [5]: [i_category#29, i_brand#30, cc_name#31, sum_sales#37, rn#39]
+Input [7]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#37, rn#39]
+
+(44) Exchange
+Input [5]: [i_category#29, i_brand#30, cc_name#31, sum_sales#37, rn#39]
+Arguments: hashpartitioning(i_category#29, i_brand#30, cc_name#31, (rn#39 + 1), 5), ENSURE_REQUIREMENTS, [id=#40]
+
+(45) Sort [codegen id : 24]
+Input [5]: [i_category#29, i_brand#30, cc_name#31, sum_sales#37, rn#39]
+Arguments: [i_category#29 ASC NULLS FIRST, i_brand#30 ASC NULLS FIRST, cc_name#31 ASC NULLS FIRST, (rn#39 + 1) ASC NULLS FIRST], false, 0
+
+(46) SortMergeJoin [codegen id : 25]
+Left keys [4]: [i_category#16, i_brand#15, cc_name#11, rn#27]
+Right keys [4]: [i_category#29, i_brand#30, cc_name#31, (rn#39 + 1)]
+Join condition: None
+
+(47) Project [codegen id : 25]
+Output [9]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, avg_monthly_sales#25, rn#27, sum_sales#37]
+Input [13]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, avg_monthly_sales#25, rn#27, i_category#29, i_brand#30, cc_name#31, sum_sales#37, rn#39]
+
+(48) ReusedExchange [Reuses operator id: 40]
+Output [6]: [i_category#41, i_brand#42, cc_name#43, d_year#44, d_moy#45, sum_sales#46]
+
+(49) Sort [codegen id : 34]
+Input [6]: [i_category#41, i_brand#42, cc_name#43, d_year#44, d_moy#45, sum_sales#46]
+Arguments: [i_category#41 ASC NULLS FIRST, i_brand#42 ASC NULLS FIRST, cc_name#43 ASC NULLS FIRST, d_year#44 ASC NULLS FIRST, d_moy#45 ASC NULLS FIRST], false, 0
+
+(50) Window
+Input [6]: [i_category#41, i_brand#42, cc_name#43, d_year#44, d_moy#45, sum_sales#46]
+Arguments: [rank(d_year#44, d_moy#45) windowspecdefinition(i_category#41, i_brand#42, cc_name#43, d_year#44 ASC NULLS FIRST, d_moy#45 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#47], [i_category#41, i_brand#42, cc_name#43], [d_year#44 ASC NULLS FIRST, d_moy#45 ASC NULLS FIRST]
+
+(51) Project [codegen id : 35]
+Output [5]: [i_category#41, i_brand#42, cc_name#43, sum_sales#46, rn#47]
+Input [7]: [i_category#41, i_brand#42, cc_name#43, d_year#44, d_moy#45, sum_sales#46, rn#47]
+
+(52) Exchange
+Input [5]: [i_category#41, i_brand#42, cc_name#43, sum_sales#46, rn#47]
+Arguments: hashpartitioning(i_category#41, i_brand#42, cc_name#43, (rn#47 - 1), 5), ENSURE_REQUIREMENTS, [id=#48]
+
+(53) Sort [codegen id : 36]
+Input [5]: [i_category#41, i_brand#42, cc_name#43, sum_sales#46, rn#47]
+Arguments: [i_category#41 ASC NULLS FIRST, i_brand#42 ASC NULLS FIRST, cc_name#43 ASC NULLS FIRST, (rn#47 - 1) ASC NULLS FIRST], false, 0
+
+(54) SortMergeJoin [codegen id : 37]
+Left keys [4]: [i_category#16, i_brand#15, cc_name#11, rn#27]
+Right keys [4]: [i_category#41, i_brand#42, cc_name#43, (rn#47 - 1)]
+Join condition: None
+
+(55) Project [codegen id : 37]
+Output [8]: [i_category#16, i_brand#15, d_year#7, d_moy#8, avg_monthly_sales#25, sum_sales#22, sum_sales#37 AS psum#49, sum_sales#46 AS nsum#50]
+Input [14]: [i_category#16, i_brand#15, cc_name#11, d_year#7, d_moy#8, sum_sales#22, avg_monthly_sales#25, rn#27, sum_sales#37, i_category#41, i_brand#42, cc_name#43, sum_sales#46, rn#47]
+
+(56) TakeOrderedAndProject
+Input [8]: [i_category#16, i_brand#15, d_year#7, d_moy#8, avg_monthly_sales#25, sum_sales#22, psum#49, nsum#50]
+Arguments: 100, [CheckOverflow((promote_precision(cast(sum_sales#22 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#25 as decimal(22,6)))), DecimalType(22,6), true) ASC NULLS FIRST, d_year#7 ASC NULLS FIRST], [i_category#16, i_brand#15, d_year#7, d_moy#8, avg_monthly_sales#25, sum_sales#22, psum#49, nsum#50]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (57)
+
+
+(57) ReusedExchange [Reuses operator id: 7]
+Output [3]: [d_date_sk#6, d_year#7, d_moy#8]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.sf100/simplified.txt
new file mode 100644
index 0000000000000..1a4bc755694d0
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.sf100/simplified.txt
@@ -0,0 +1,107 @@
+TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_moy,psum,nsum]
+  WholeStageCodegen (37)
+    Project [i_category,i_brand,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales]
+      SortMergeJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn]
+        InputAdapter
+          WholeStageCodegen (25)
+            Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales]
+              SortMergeJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn]
+                InputAdapter
+                  WholeStageCodegen (13)
+                    Sort [i_category,i_brand,cc_name,rn]
+                      InputAdapter
+                        Exchange [i_category,i_brand,cc_name,rn] #1
+                          WholeStageCodegen (12)
+                            Filter [d_year,avg_monthly_sales,sum_sales]
+                              InputAdapter
+                                Window [d_year,d_moy,i_category,i_brand,cc_name]
+                                  WholeStageCodegen (11)
+                                    Sort [i_category,i_brand,cc_name,d_year,d_moy]
+                                      InputAdapter
+                                        Exchange [i_category,i_brand,cc_name] #2
+                                          WholeStageCodegen (10)
+                                            Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales]
+                                              InputAdapter
+                                                Window [_w0,i_category,i_brand,cc_name,d_year]
+                                                  WholeStageCodegen (9)
+                                                    Sort [i_category,i_brand,cc_name,d_year]
+                                                      InputAdapter
+                                                        Exchange [i_category,i_brand,cc_name,d_year] #3
+                                                          WholeStageCodegen (8)
+                                                            HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,_w0,sum]
+                                                              InputAdapter
+                                                                Exchange [i_category,i_brand,cc_name,d_year,d_moy] #4
+                                                                  WholeStageCodegen (7)
+                                                                    HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,cs_sales_price] [sum,sum]
+                                                                      Project [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name]
+                                                                        SortMergeJoin [cs_item_sk,i_item_sk]
+                                                                          InputAdapter
+                                                                            WholeStageCodegen (4)
+                                                                              Sort [cs_item_sk]
+                                                                                InputAdapter
+                                                                                  Exchange [cs_item_sk] #5
+                                                                                    WholeStageCodegen (3)
+                                                                                      Project [cs_item_sk,cs_sales_price,d_year,d_moy,cc_name]
+                                                                                        BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk]
+                                                                                          Project [cs_call_center_sk,cs_item_sk,cs_sales_price,d_year,d_moy]
+                                                                                            BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                                              Filter [cs_item_sk,cs_call_center_sk]
+                                                                                                ColumnarToRow
+                                                                                                  InputAdapter
+                                                                                                    Scan parquet default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk]
+                                                                                                      SubqueryBroadcast [d_date_sk] #1
+                                                                                                        ReusedExchange [d_date_sk,d_year,d_moy] #6
+                                                                                              InputAdapter
+                                                                                                BroadcastExchange #6
+                                                                                                  WholeStageCodegen (1)
+                                                                                                    Filter [d_year,d_moy,d_date_sk]
+                                                                                                      ColumnarToRow
+                                                                                                        InputAdapter
+                                                                                                          Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                                                                          InputAdapter
+                                                                                            BroadcastExchange #7
+                                                                                              WholeStageCodegen (2)
+                                                                                                Filter [cc_call_center_sk,cc_name]
+                                                                                                  ColumnarToRow
+                                                                                                    InputAdapter
+                                                                                                      Scan parquet default.call_center [cc_call_center_sk,cc_name]
+                                                                          InputAdapter
+                                                                            WholeStageCodegen (6)
+                                                                              Sort [i_item_sk]
+                                                                                InputAdapter
+                                                                                  Exchange [i_item_sk] #8
+                                                                                    WholeStageCodegen (5)
+                                                                                      Filter [i_item_sk,i_category,i_brand]
+                                                                                        ColumnarToRow
+                                                                                          InputAdapter
+                                                                                            Scan parquet default.item [i_item_sk,i_brand,i_category]
+                InputAdapter
+                  WholeStageCodegen (24)
+                    Sort [i_category,i_brand,cc_name,rn]
+                      InputAdapter
+                        Exchange [i_category,i_brand,cc_name,rn] #9
+                          WholeStageCodegen (23)
+                            Project [i_category,i_brand,cc_name,sum_sales,rn]
+                              InputAdapter
+                                Window [d_year,d_moy,i_category,i_brand,cc_name]
+                                  WholeStageCodegen (22)
+                                    Sort [i_category,i_brand,cc_name,d_year,d_moy]
+                                      InputAdapter
+                                        Exchange [i_category,i_brand,cc_name] #10
+                                          WholeStageCodegen (21)
+                                            HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,sum]
+                                              InputAdapter
+                                                ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #11
+        InputAdapter
+          WholeStageCodegen (36)
+            Sort [i_category,i_brand,cc_name,rn]
+              InputAdapter
+                Exchange [i_category,i_brand,cc_name,rn] #12
+                  WholeStageCodegen (35)
+                    Project [i_category,i_brand,cc_name,sum_sales,rn]
+                      InputAdapter
+                        Window [d_year,d_moy,i_category,i_brand,cc_name]
+                          WholeStageCodegen (34)
+                            Sort [i_category,i_brand,cc_name,d_year,d_moy]
+                              InputAdapter
+                                ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #10
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/explain.txt
new file mode 100644
index 0000000000000..a6742cf4ab1cf
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/explain.txt
@@ -0,0 +1,279 @@
+== Physical Plan ==
+TakeOrderedAndProject (49)
++- * Project (48)
+   +- * BroadcastHashJoin Inner BuildRight (47)
+      :- * Project (41)
+      :  +- * BroadcastHashJoin Inner BuildRight (40)
+      :     :- * Filter (32)
+      :     :  +- Window (31)
+      :     :     +- * Sort (30)
+      :     :        +- Exchange (29)
+      :     :           +- * Project (28)
+      :     :              +- Window (27)
+      :     :                 +- * Sort (26)
+      :     :                    +- Exchange (25)
+      :     :                       +- * HashAggregate (24)
+      :     :                          +- Exchange (23)
+      :     :                             +- * HashAggregate (22)
+      :     :                                +- * Project (21)
+      :     :                                   +- * BroadcastHashJoin Inner BuildRight (20)
+      :     :                                      :- * Project (15)
+      :     :                                      :  +- * BroadcastHashJoin Inner BuildRight (14)
+      :     :                                      :     :- * Project (9)
+      :     :                                      :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+      :     :                                      :     :     :- * Filter (3)
+      :     :                                      :     :     :  +- * ColumnarToRow (2)
+      :     :                                      :     :     :     +- Scan parquet default.item (1)
+      :     :                                      :     :     +- BroadcastExchange (7)
+      :     :                                      :     :        +- * Filter (6)
+      :     :                                      :     :           +- * ColumnarToRow (5)
+      :     :                                      :     :              +- Scan parquet default.catalog_sales (4)
+      :     :                                      :     +- BroadcastExchange (13)
+      :     :                                      :        +- * Filter (12)
+      :     :                                      :           +- * ColumnarToRow (11)
+      :     :                                      :              +- Scan parquet default.date_dim (10)
+      :     :                                      +- BroadcastExchange (19)
+      :     :                                         +- * Filter (18)
+      :     :                                            +- * ColumnarToRow (17)
+      :     :                                               +- Scan parquet default.call_center (16)
+      :     +- BroadcastExchange (39)
+      :        +- * Project (38)
+      :           +- Window (37)
+      :              +- * Sort (36)
+      :                 +- Exchange (35)
+      :                    +- * HashAggregate (34)
+      :                       +- ReusedExchange (33)
+      +- BroadcastExchange (46)
+         +- * Project (45)
+            +- Window (44)
+               +- * Sort (43)
+                  +- ReusedExchange (42)
+
+
+(1) Scan parquet default.item
+Output [3]: [i_item_sk#1, i_brand#2, i_category#3]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_category:string>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [3]: [i_item_sk#1, i_brand#2, i_category#3]
+
+(3) Filter [codegen id : 4]
+Input [3]: [i_item_sk#1, i_brand#2, i_category#3]
+Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2))
+
+(4) Scan parquet default.catalog_sales
+Output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#7), dynamicpruningexpression(cs_sold_date_sk#7 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)]
+ReadSchema: struct<cs_call_center_sk:int,cs_item_sk:int,cs_sales_price:decimal(7,2)>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7]
+
+(6) Filter [codegen id : 1]
+Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7]
+Condition : (isnotnull(cs_item_sk#5) AND isnotnull(cs_call_center_sk#4))
+
+(7) BroadcastExchange
+Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [id=#9]
+
+(8) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [i_item_sk#1]
+Right keys [1]: [cs_item_sk#5]
+Join condition: None
+
+(9) Project [codegen id : 4]
+Output [5]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7]
+Input [7]: [i_item_sk#1, i_brand#2, i_category#3, cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7]
+
+(10) Scan parquet default.date_dim
+Output [3]: [d_date_sk#10, d_year#11, d_moy#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int,d_moy:int>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+
+(12) Filter [codegen id : 2]
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10))
+
+(13) BroadcastExchange
+Input [3]: [d_date_sk#10, d_year#11, d_moy#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#13]
+
+(14) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_sold_date_sk#7]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(15) Project [codegen id : 4]
+Output [6]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#11, d_moy#12]
+Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7, d_date_sk#10, d_year#11, d_moy#12]
+
+(16) Scan parquet default.call_center
+Output [2]: [cc_call_center_sk#14, cc_name#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/call_center]
+PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)]
+ReadSchema: struct<cc_call_center_sk:int,cc_name:string>
+
+(17) ColumnarToRow [codegen id : 3]
+Input [2]: [cc_call_center_sk#14, cc_name#15]
+
+(18) Filter [codegen id : 3]
+Input [2]: [cc_call_center_sk#14, cc_name#15]
+Condition : (isnotnull(cc_call_center_sk#14) AND isnotnull(cc_name#15))
+
+(19) BroadcastExchange
+Input [2]: [cc_call_center_sk#14, cc_name#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#16]
+
+(20) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_call_center_sk#4]
+Right keys [1]: [cc_call_center_sk#14]
+Join condition: None
+
+(21) Project [codegen id : 4]
+Output [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#11, d_moy#12, cc_name#15]
+Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#11, d_moy#12, cc_call_center_sk#14, cc_name#15]
+
+(22) HashAggregate [codegen id : 4]
+Input [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#11, d_moy#12, cc_name#15]
+Keys [5]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12]
+Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#6))]
+Aggregate Attributes [1]: [sum#17]
+Results [6]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum#18]
+
+(23) Exchange
+Input [6]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum#18]
+Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, [id=#19]
+
+(24) HashAggregate [codegen id : 5]
+Input [6]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum#18]
+Keys [5]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12]
+Functions [1]: [sum(UnscaledValue(cs_sales_price#6))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#6))#20]
+Results [7]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#20,17,2) AS sum_sales#21, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#20,17,2) AS _w0#22]
+
+(25) Exchange
+Input [7]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum_sales#21, _w0#22]
+Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#15, d_year#11, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(26) Sort [codegen id : 6]
+Input [7]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum_sales#21, _w0#22]
+Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, cc_name#15 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST], false, 0
+
+(27) Window
+Input [7]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum_sales#21, _w0#22]
+Arguments: [avg(_w0#22) windowspecdefinition(i_category#3, i_brand#2, cc_name#15, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#24], [i_category#3, i_brand#2, cc_name#15, d_year#11]
+
+(28) Project [codegen id : 7]
+Output [7]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum_sales#21, avg_monthly_sales#24]
+Input [8]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum_sales#21, _w0#22, avg_monthly_sales#24]
+
+(29) Exchange
+Input [7]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum_sales#21, avg_monthly_sales#24]
+Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#15, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(30) Sort [codegen id : 8]
+Input [7]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum_sales#21, avg_monthly_sales#24]
+Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, cc_name#15 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST], false, 0
+
+(31) Window
+Input [7]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum_sales#21, avg_monthly_sales#24]
+Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#3, i_brand#2, cc_name#15, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#26], [i_category#3, i_brand#2, cc_name#15], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST]
+
+(32) Filter [codegen id : 23]
+Input [8]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum_sales#21, avg_monthly_sales#24, rn#26]
+Condition : ((((isnotnull(d_year#11) AND isnotnull(avg_monthly_sales#24)) AND (d_year#11 = 1999)) AND (avg_monthly_sales#24 > 0.000000)) AND (CheckOverflow((promote_precision(abs(CheckOverflow((promote_precision(cast(sum_sales#21 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#24 as decimal(22,6)))), DecimalType(22,6), true), false)) / promote_precision(cast(avg_monthly_sales#24 as decimal(22,6)))), DecimalType(38,16), true) > 0.1000000000000000))
+
+(33) ReusedExchange [Reuses operator id: unknown]
+Output [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum#32]
+
+(34) HashAggregate [codegen id : 13]
+Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum#32]
+Keys [5]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31]
+Functions [1]: [sum(UnscaledValue(cs_sales_price#33))]
+Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#33))#34]
+Results [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, MakeDecimal(sum(UnscaledValue(cs_sales_price#33))#34,17,2) AS sum_sales#35]
+
+(35) Exchange
+Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#35]
+Arguments: hashpartitioning(i_category#27, i_brand#28, cc_name#29, 5), ENSURE_REQUIREMENTS, [id=#36]
+
+(36) Sort [codegen id : 14]
+Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#35]
+Arguments: [i_category#27 ASC NULLS FIRST, i_brand#28 ASC NULLS FIRST, cc_name#29 ASC NULLS FIRST, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST], false, 0
+
+(37) Window
+Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#35]
+Arguments: [rank(d_year#30, d_moy#31) windowspecdefinition(i_category#27, i_brand#28, cc_name#29, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#37], [i_category#27, i_brand#28, cc_name#29], [d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST]
+
+(38) Project [codegen id : 15]
+Output [5]: [i_category#27, i_brand#28, cc_name#29, sum_sales#35, rn#37]
+Input [7]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#35, rn#37]
+
+(39) BroadcastExchange
+Input [5]: [i_category#27, i_brand#28, cc_name#29, sum_sales#35, rn#37]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [id=#38]
+
+(40) BroadcastHashJoin [codegen id : 23]
+Left keys [4]: [i_category#3, i_brand#2, cc_name#15, rn#26]
+Right keys [4]: [i_category#27, i_brand#28, cc_name#29, (rn#37 + 1)]
+Join condition: None
+
+(41) Project [codegen id : 23]
+Output [9]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum_sales#21, avg_monthly_sales#24, rn#26, sum_sales#35]
+Input [13]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum_sales#21, avg_monthly_sales#24, rn#26, i_category#27, i_brand#28, cc_name#29, sum_sales#35, rn#37]
+
+(42) ReusedExchange [Reuses operator id: 35]
+Output [6]: [i_category#39, i_brand#40, cc_name#41, d_year#42, d_moy#43, sum_sales#44]
+
+(43) Sort [codegen id : 21]
+Input [6]: [i_category#39, i_brand#40, cc_name#41, d_year#42, d_moy#43, sum_sales#44]
+Arguments: [i_category#39 ASC NULLS FIRST, i_brand#40 ASC NULLS FIRST, cc_name#41 ASC NULLS FIRST, d_year#42 ASC NULLS FIRST, d_moy#43 ASC NULLS FIRST], false, 0
+
+(44) Window
+Input [6]: [i_category#39, i_brand#40, cc_name#41, d_year#42, d_moy#43, sum_sales#44]
+Arguments: [rank(d_year#42, d_moy#43) windowspecdefinition(i_category#39, i_brand#40, cc_name#41, d_year#42 ASC NULLS FIRST, d_moy#43 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#45], [i_category#39, i_brand#40, cc_name#41], [d_year#42 ASC NULLS FIRST, d_moy#43 ASC NULLS FIRST]
+
+(45) Project [codegen id : 22]
+Output [5]: [i_category#39, i_brand#40, cc_name#41, sum_sales#44, rn#45]
+Input [7]: [i_category#39, i_brand#40, cc_name#41, d_year#42, d_moy#43, sum_sales#44, rn#45]
+
+(46) BroadcastExchange
+Input [5]: [i_category#39, i_brand#40, cc_name#41, sum_sales#44, rn#45]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [id=#46]
+
+(47) BroadcastHashJoin [codegen id : 23]
+Left keys [4]: [i_category#3, i_brand#2, cc_name#15, rn#26]
+Right keys [4]: [i_category#39, i_brand#40, cc_name#41, (rn#45 - 1)]
+Join condition: None
+
+(48) Project [codegen id : 23]
+Output [8]: [i_category#3, i_brand#2, d_year#11, d_moy#12, avg_monthly_sales#24, sum_sales#21, sum_sales#35 AS psum#47, sum_sales#44 AS nsum#48]
+Input [14]: [i_category#3, i_brand#2, cc_name#15, d_year#11, d_moy#12, sum_sales#21, avg_monthly_sales#24, rn#26, sum_sales#35, i_category#39, i_brand#40, cc_name#41, sum_sales#44, rn#45]
+
+(49) TakeOrderedAndProject
+Input [8]: [i_category#3, i_brand#2, d_year#11, d_moy#12, avg_monthly_sales#24, sum_sales#21, psum#47, nsum#48]
+Arguments: 100, [CheckOverflow((promote_precision(cast(sum_sales#21 as decimal(22,6))) - promote_precision(cast(avg_monthly_sales#24 as decimal(22,6)))), DecimalType(22,6), true) ASC NULLS FIRST, d_year#11 ASC NULLS FIRST], [i_category#3, i_brand#2, d_year#11, d_moy#12, avg_monthly_sales#24, sum_sales#21, psum#47, nsum#48]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 4 Hosting Expression = cs_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (50)
+
+
+(50) ReusedExchange [Reuses operator id: 13]
+Output [3]: [d_date_sk#10, d_year#11, d_moy#12]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/simplified.txt
new file mode 100644
index 0000000000000..501fb09522772
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/simplified.txt
@@ -0,0 +1,84 @@
+TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_moy,psum,nsum]
+  WholeStageCodegen (23)
+    Project [i_category,i_brand,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales]
+      BroadcastHashJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn]
+        Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales]
+          BroadcastHashJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn]
+            Filter [d_year,avg_monthly_sales,sum_sales]
+              InputAdapter
+                Window [d_year,d_moy,i_category,i_brand,cc_name]
+                  WholeStageCodegen (8)
+                    Sort [i_category,i_brand,cc_name,d_year,d_moy]
+                      InputAdapter
+                        Exchange [i_category,i_brand,cc_name] #1
+                          WholeStageCodegen (7)
+                            Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales]
+                              InputAdapter
+                                Window [_w0,i_category,i_brand,cc_name,d_year]
+                                  WholeStageCodegen (6)
+                                    Sort [i_category,i_brand,cc_name,d_year]
+                                      InputAdapter
+                                        Exchange [i_category,i_brand,cc_name,d_year] #2
+                                          WholeStageCodegen (5)
+                                            HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,_w0,sum]
+                                              InputAdapter
+                                                Exchange [i_category,i_brand,cc_name,d_year,d_moy] #3
+                                                  WholeStageCodegen (4)
+                                                    HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,cs_sales_price] [sum,sum]
+                                                      Project [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name]
+                                                        BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk]
+                                                          Project [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy]
+                                                            BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                              Project [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk]
+                                                                BroadcastHashJoin [i_item_sk,cs_item_sk]
+                                                                  Filter [i_item_sk,i_category,i_brand]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.item [i_item_sk,i_brand,i_category]
+                                                                  InputAdapter
+                                                                    BroadcastExchange #4
+                                                                      WholeStageCodegen (1)
+                                                                        Filter [cs_item_sk,cs_call_center_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk]
+                                                                                SubqueryBroadcast [d_date_sk] #1
+                                                                                  ReusedExchange [d_date_sk,d_year,d_moy] #5
+                                                              InputAdapter
+                                                                BroadcastExchange #5
+                                                                  WholeStageCodegen (2)
+                                                                    Filter [d_year,d_moy,d_date_sk]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.date_dim [d_date_sk,d_year,d_moy]
+                                                          InputAdapter
+                                                            BroadcastExchange #6
+                                                              WholeStageCodegen (3)
+                                                                Filter [cc_call_center_sk,cc_name]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.call_center [cc_call_center_sk,cc_name]
+            InputAdapter
+              BroadcastExchange #7
+                WholeStageCodegen (15)
+                  Project [i_category,i_brand,cc_name,sum_sales,rn]
+                    InputAdapter
+                      Window [d_year,d_moy,i_category,i_brand,cc_name]
+                        WholeStageCodegen (14)
+                          Sort [i_category,i_brand,cc_name,d_year,d_moy]
+                            InputAdapter
+                              Exchange [i_category,i_brand,cc_name] #8
+                                WholeStageCodegen (13)
+                                  HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,sum]
+                                    InputAdapter
+                                      ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #9
+        InputAdapter
+          BroadcastExchange #10
+            WholeStageCodegen (22)
+              Project [i_category,i_brand,cc_name,sum_sales,rn]
+                InputAdapter
+                  Window [d_year,d_moy,i_category,i_brand,cc_name]
+                    WholeStageCodegen (21)
+                      Sort [i_category,i_brand,cc_name,d_year,d_moy]
+                        InputAdapter
+                          ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #8
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.sf100/explain.txt
new file mode 100644
index 0000000000000..e660800796abe
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.sf100/explain.txt
@@ -0,0 +1,561 @@
+== Physical Plan ==
+TakeOrderedAndProject (94)
++- * HashAggregate (93)
+   +- Exchange (92)
+      +- * HashAggregate (91)
+         +- Union (90)
+            :- * HashAggregate (79)
+            :  +- Exchange (78)
+            :     +- * HashAggregate (77)
+            :        +- Union (76)
+            :           :- * HashAggregate (25)
+            :           :  +- Exchange (24)
+            :           :     +- * HashAggregate (23)
+            :           :        +- * Project (22)
+            :           :           +- * BroadcastHashJoin Inner BuildRight (21)
+            :           :              :- * Project (15)
+            :           :              :  +- * BroadcastHashJoin Inner BuildRight (14)
+            :           :              :     :- Union (9)
+            :           :              :     :  :- * Project (4)
+            :           :              :     :  :  +- * Filter (3)
+            :           :              :     :  :     +- * ColumnarToRow (2)
+            :           :              :     :  :        +- Scan parquet default.store_sales (1)
+            :           :              :     :  +- * Project (8)
+            :           :              :     :     +- * Filter (7)
+            :           :              :     :        +- * ColumnarToRow (6)
+            :           :              :     :           +- Scan parquet default.store_returns (5)
+            :           :              :     +- BroadcastExchange (13)
+            :           :              :        +- * Filter (12)
+            :           :              :           +- * ColumnarToRow (11)
+            :           :              :              +- Scan parquet default.store (10)
+            :           :              +- BroadcastExchange (20)
+            :           :                 +- * Project (19)
+            :           :                    +- * Filter (18)
+            :           :                       +- * ColumnarToRow (17)
+            :           :                          +- Scan parquet default.date_dim (16)
+            :           :- * HashAggregate (46)
+            :           :  +- Exchange (45)
+            :           :     +- * HashAggregate (44)
+            :           :        +- * Project (43)
+            :           :           +- * BroadcastHashJoin Inner BuildRight (42)
+            :           :              :- * Project (40)
+            :           :              :  +- * BroadcastHashJoin Inner BuildRight (39)
+            :           :              :     :- Union (34)
+            :           :              :     :  :- * Project (29)
+            :           :              :     :  :  +- * Filter (28)
+            :           :              :     :  :     +- * ColumnarToRow (27)
+            :           :              :     :  :        +- Scan parquet default.catalog_sales (26)
+            :           :              :     :  +- * Project (33)
+            :           :              :     :     +- * Filter (32)
+            :           :              :     :        +- * ColumnarToRow (31)
+            :           :              :     :           +- Scan parquet default.catalog_returns (30)
+            :           :              :     +- BroadcastExchange (38)
+            :           :              :        +- * Filter (37)
+            :           :              :           +- * ColumnarToRow (36)
+            :           :              :              +- Scan parquet default.catalog_page (35)
+            :           :              +- ReusedExchange (41)
+            :           +- * HashAggregate (75)
+            :              +- Exchange (74)
+            :                 +- * HashAggregate (73)
+            :                    +- * Project (72)
+            :                       +- * BroadcastHashJoin Inner BuildRight (71)
+            :                          :- * Project (69)
+            :                          :  +- * BroadcastHashJoin Inner BuildRight (68)
+            :                          :     :- Union (63)
+            :                          :     :  :- * Project (50)
+            :                          :     :  :  +- * Filter (49)
+            :                          :     :  :     +- * ColumnarToRow (48)
+            :                          :     :  :        +- Scan parquet default.web_sales (47)
+            :                          :     :  +- * Project (62)
+            :                          :     :     +- * SortMergeJoin Inner (61)
+            :                          :     :        :- * Sort (54)
+            :                          :     :        :  +- Exchange (53)
+            :                          :     :        :     +- * ColumnarToRow (52)
+            :                          :     :        :        +- Scan parquet default.web_returns (51)
+            :                          :     :        +- * Sort (60)
+            :                          :     :           +- Exchange (59)
+            :                          :     :              +- * Project (58)
+            :                          :     :                 +- * Filter (57)
+            :                          :     :                    +- * ColumnarToRow (56)
+            :                          :     :                       +- Scan parquet default.web_sales (55)
+            :                          :     +- BroadcastExchange (67)
+            :                          :        +- * Filter (66)
+            :                          :           +- * ColumnarToRow (65)
+            :                          :              +- Scan parquet default.web_site (64)
+            :                          +- ReusedExchange (70)
+            :- * HashAggregate (84)
+            :  +- Exchange (83)
+            :     +- * HashAggregate (82)
+            :        +- * HashAggregate (81)
+            :           +- ReusedExchange (80)
+            +- * HashAggregate (89)
+               +- Exchange (88)
+                  +- * HashAggregate (87)
+                     +- * HashAggregate (86)
+                        +- ReusedExchange (85)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_ext_sales_price:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 1]
+Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+Condition : isnotnull(ss_store_sk#1)
+
+(4) Project [codegen id : 1]
+Output [6]: [ss_store_sk#1 AS store_sk#6, ss_sold_date_sk#4 AS date_sk#7, ss_ext_sales_price#2 AS sales_price#8, ss_net_profit#3 AS profit#9, 0.00 AS return_amt#10, 0.00 AS net_loss#11]
+Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+
+(5) Scan parquet default.store_returns
+Output [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(sr_returned_date_sk#15), dynamicpruningexpression(sr_returned_date_sk#15 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(sr_store_sk)]
+ReadSchema: struct<sr_store_sk:int,sr_return_amt:decimal(7,2),sr_net_loss:decimal(7,2)>
+
+(6) ColumnarToRow [codegen id : 2]
+Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15]
+
+(7) Filter [codegen id : 2]
+Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15]
+Condition : isnotnull(sr_store_sk#12)
+
+(8) Project [codegen id : 2]
+Output [6]: [sr_store_sk#12 AS store_sk#16, sr_returned_date_sk#15 AS date_sk#17, 0.00 AS sales_price#18, 0.00 AS profit#19, sr_return_amt#13 AS return_amt#20, sr_net_loss#14 AS net_loss#21]
+Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15]
+
+(9) Union
+
+(10) Scan parquet default.store
+Output [2]: [s_store_sk#22, s_store_id#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string>
+
+(11) ColumnarToRow [codegen id : 3]
+Input [2]: [s_store_sk#22, s_store_id#23]
+
+(12) Filter [codegen id : 3]
+Input [2]: [s_store_sk#22, s_store_id#23]
+Condition : isnotnull(s_store_sk#22)
+
+(13) BroadcastExchange
+Input [2]: [s_store_sk#22, s_store_id#23]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#24]
+
+(14) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [store_sk#6]
+Right keys [1]: [s_store_sk#22]
+Join condition: None
+
+(15) Project [codegen id : 5]
+Output [6]: [date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#23]
+Input [8]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_sk#22, s_store_id#23]
+
+(16) Scan parquet default.date_dim
+Output [2]: [d_date_sk#25, d_date#26]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-08-18), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(17) ColumnarToRow [codegen id : 4]
+Input [2]: [d_date_sk#25, d_date#26]
+
+(18) Filter [codegen id : 4]
+Input [2]: [d_date_sk#25, d_date#26]
+Condition : (((isnotnull(d_date#26) AND (d_date#26 >= 1998-08-04)) AND (d_date#26 <= 1998-08-18)) AND isnotnull(d_date_sk#25))
+
+(19) Project [codegen id : 4]
+Output [1]: [d_date_sk#25]
+Input [2]: [d_date_sk#25, d_date#26]
+
+(20) BroadcastExchange
+Input [1]: [d_date_sk#25]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#27]
+
+(21) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [date_sk#7]
+Right keys [1]: [d_date_sk#25]
+Join condition: None
+
+(22) Project [codegen id : 5]
+Output [5]: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#23]
+Input [7]: [date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#23, d_date_sk#25]
+
+(23) HashAggregate [codegen id : 5]
+Input [5]: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#23]
+Keys [1]: [s_store_id#23]
+Functions [4]: [partial_sum(UnscaledValue(sales_price#8)), partial_sum(UnscaledValue(return_amt#10)), partial_sum(UnscaledValue(profit#9)), partial_sum(UnscaledValue(net_loss#11))]
+Aggregate Attributes [4]: [sum#28, sum#29, sum#30, sum#31]
+Results [5]: [s_store_id#23, sum#32, sum#33, sum#34, sum#35]
+
+(24) Exchange
+Input [5]: [s_store_id#23, sum#32, sum#33, sum#34, sum#35]
+Arguments: hashpartitioning(s_store_id#23, 5), ENSURE_REQUIREMENTS, [id=#36]
+
+(25) HashAggregate [codegen id : 6]
+Input [5]: [s_store_id#23, sum#32, sum#33, sum#34, sum#35]
+Keys [1]: [s_store_id#23]
+Functions [4]: [sum(UnscaledValue(sales_price#8)), sum(UnscaledValue(return_amt#10)), sum(UnscaledValue(profit#9)), sum(UnscaledValue(net_loss#11))]
+Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#8))#37, sum(UnscaledValue(return_amt#10))#38, sum(UnscaledValue(profit#9))#39, sum(UnscaledValue(net_loss#11))#40]
+Results [5]: [store channel AS channel#41, concat(store, s_store_id#23) AS id#42, MakeDecimal(sum(UnscaledValue(sales_price#8))#37,17,2) AS sales#43, MakeDecimal(sum(UnscaledValue(return_amt#10))#38,17,2) AS returns#44, CheckOverflow((promote_precision(cast(MakeDecimal(sum(UnscaledValue(profit#9))#39,17,2) as decimal(18,2))) - promote_precision(cast(MakeDecimal(sum(UnscaledValue(net_loss#11))#40,17,2) as decimal(18,2)))), DecimalType(18,2), true) AS profit#45]
+
+(26) Scan parquet default.catalog_sales
+Output [4]: [cs_catalog_page_sk#46, cs_ext_sales_price#47, cs_net_profit#48, cs_sold_date_sk#49]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#49), dynamicpruningexpression(cs_sold_date_sk#49 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(cs_catalog_page_sk)]
+ReadSchema: struct<cs_catalog_page_sk:int,cs_ext_sales_price:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(27) ColumnarToRow [codegen id : 7]
+Input [4]: [cs_catalog_page_sk#46, cs_ext_sales_price#47, cs_net_profit#48, cs_sold_date_sk#49]
+
+(28) Filter [codegen id : 7]
+Input [4]: [cs_catalog_page_sk#46, cs_ext_sales_price#47, cs_net_profit#48, cs_sold_date_sk#49]
+Condition : isnotnull(cs_catalog_page_sk#46)
+
+(29) Project [codegen id : 7]
+Output [6]: [cs_catalog_page_sk#46 AS page_sk#50, cs_sold_date_sk#49 AS date_sk#51, cs_ext_sales_price#47 AS sales_price#52, cs_net_profit#48 AS profit#53, 0.00 AS return_amt#54, 0.00 AS net_loss#55]
+Input [4]: [cs_catalog_page_sk#46, cs_ext_sales_price#47, cs_net_profit#48, cs_sold_date_sk#49]
+
+(30) Scan parquet default.catalog_returns
+Output [4]: [cr_catalog_page_sk#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cr_returned_date_sk#59), dynamicpruningexpression(cr_returned_date_sk#59 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(cr_catalog_page_sk)]
+ReadSchema: struct<cr_catalog_page_sk:int,cr_return_amount:decimal(7,2),cr_net_loss:decimal(7,2)>
+
+(31) ColumnarToRow [codegen id : 8]
+Input [4]: [cr_catalog_page_sk#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+
+(32) Filter [codegen id : 8]
+Input [4]: [cr_catalog_page_sk#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+Condition : isnotnull(cr_catalog_page_sk#56)
+
+(33) Project [codegen id : 8]
+Output [6]: [cr_catalog_page_sk#56 AS page_sk#60, cr_returned_date_sk#59 AS date_sk#61, 0.00 AS sales_price#62, 0.00 AS profit#63, cr_return_amount#57 AS return_amt#64, cr_net_loss#58 AS net_loss#65]
+Input [4]: [cr_catalog_page_sk#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+
+(34) Union
+
+(35) Scan parquet default.catalog_page
+Output [2]: [cp_catalog_page_sk#66, cp_catalog_page_id#67]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_page]
+PushedFilters: [IsNotNull(cp_catalog_page_sk)]
+ReadSchema: struct<cp_catalog_page_sk:int,cp_catalog_page_id:string>
+
+(36) ColumnarToRow [codegen id : 9]
+Input [2]: [cp_catalog_page_sk#66, cp_catalog_page_id#67]
+
+(37) Filter [codegen id : 9]
+Input [2]: [cp_catalog_page_sk#66, cp_catalog_page_id#67]
+Condition : isnotnull(cp_catalog_page_sk#66)
+
+(38) BroadcastExchange
+Input [2]: [cp_catalog_page_sk#66, cp_catalog_page_id#67]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#68]
+
+(39) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [page_sk#50]
+Right keys [1]: [cp_catalog_page_sk#66]
+Join condition: None
+
+(40) Project [codegen id : 11]
+Output [6]: [date_sk#51, sales_price#52, profit#53, return_amt#54, net_loss#55, cp_catalog_page_id#67]
+Input [8]: [page_sk#50, date_sk#51, sales_price#52, profit#53, return_amt#54, net_loss#55, cp_catalog_page_sk#66, cp_catalog_page_id#67]
+
+(41) ReusedExchange [Reuses operator id: 20]
+Output [1]: [d_date_sk#69]
+
+(42) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [date_sk#51]
+Right keys [1]: [d_date_sk#69]
+Join condition: None
+
+(43) Project [codegen id : 11]
+Output [5]: [sales_price#52, profit#53, return_amt#54, net_loss#55, cp_catalog_page_id#67]
+Input [7]: [date_sk#51, sales_price#52, profit#53, return_amt#54, net_loss#55, cp_catalog_page_id#67, d_date_sk#69]
+
+(44) HashAggregate [codegen id : 11]
+Input [5]: [sales_price#52, profit#53, return_amt#54, net_loss#55, cp_catalog_page_id#67]
+Keys [1]: [cp_catalog_page_id#67]
+Functions [4]: [partial_sum(UnscaledValue(sales_price#52)), partial_sum(UnscaledValue(return_amt#54)), partial_sum(UnscaledValue(profit#53)), partial_sum(UnscaledValue(net_loss#55))]
+Aggregate Attributes [4]: [sum#70, sum#71, sum#72, sum#73]
+Results [5]: [cp_catalog_page_id#67, sum#74, sum#75, sum#76, sum#77]
+
+(45) Exchange
+Input [5]: [cp_catalog_page_id#67, sum#74, sum#75, sum#76, sum#77]
+Arguments: hashpartitioning(cp_catalog_page_id#67, 5), ENSURE_REQUIREMENTS, [id=#78]
+
+(46) HashAggregate [codegen id : 12]
+Input [5]: [cp_catalog_page_id#67, sum#74, sum#75, sum#76, sum#77]
+Keys [1]: [cp_catalog_page_id#67]
+Functions [4]: [sum(UnscaledValue(sales_price#52)), sum(UnscaledValue(return_amt#54)), sum(UnscaledValue(profit#53)), sum(UnscaledValue(net_loss#55))]
+Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#52))#79, sum(UnscaledValue(return_amt#54))#80, sum(UnscaledValue(profit#53))#81, sum(UnscaledValue(net_loss#55))#82]
+Results [5]: [catalog channel AS channel#83, concat(catalog_page, cp_catalog_page_id#67) AS id#84, MakeDecimal(sum(UnscaledValue(sales_price#52))#79,17,2) AS sales#85, MakeDecimal(sum(UnscaledValue(return_amt#54))#80,17,2) AS returns#86, CheckOverflow((promote_precision(cast(MakeDecimal(sum(UnscaledValue(profit#53))#81,17,2) as decimal(18,2))) - promote_precision(cast(MakeDecimal(sum(UnscaledValue(net_loss#55))#82,17,2) as decimal(18,2)))), DecimalType(18,2), true) AS profit#87]
+
+(47) Scan parquet default.web_sales
+Output [4]: [ws_web_site_sk#88, ws_ext_sales_price#89, ws_net_profit#90, ws_sold_date_sk#91]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#91), dynamicpruningexpression(ws_sold_date_sk#91 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ws_web_site_sk)]
+ReadSchema: struct<ws_web_site_sk:int,ws_ext_sales_price:decimal(7,2),ws_net_profit:decimal(7,2)>
+
+(48) ColumnarToRow [codegen id : 13]
+Input [4]: [ws_web_site_sk#88, ws_ext_sales_price#89, ws_net_profit#90, ws_sold_date_sk#91]
+
+(49) Filter [codegen id : 13]
+Input [4]: [ws_web_site_sk#88, ws_ext_sales_price#89, ws_net_profit#90, ws_sold_date_sk#91]
+Condition : isnotnull(ws_web_site_sk#88)
+
+(50) Project [codegen id : 13]
+Output [6]: [ws_web_site_sk#88 AS wsr_web_site_sk#92, ws_sold_date_sk#91 AS date_sk#93, ws_ext_sales_price#89 AS sales_price#94, ws_net_profit#90 AS profit#95, 0.00 AS return_amt#96, 0.00 AS net_loss#97]
+Input [4]: [ws_web_site_sk#88, ws_ext_sales_price#89, ws_net_profit#90, ws_sold_date_sk#91]
+
+(51) Scan parquet default.web_returns
+Output [5]: [wr_item_sk#98, wr_order_number#99, wr_return_amt#100, wr_net_loss#101, wr_returned_date_sk#102]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(wr_returned_date_sk#102), dynamicpruningexpression(wr_returned_date_sk#102 IN dynamicpruning#5)]
+ReadSchema: struct<wr_item_sk:int,wr_order_number:int,wr_return_amt:decimal(7,2),wr_net_loss:decimal(7,2)>
+
+(52) ColumnarToRow [codegen id : 14]
+Input [5]: [wr_item_sk#98, wr_order_number#99, wr_return_amt#100, wr_net_loss#101, wr_returned_date_sk#102]
+
+(53) Exchange
+Input [5]: [wr_item_sk#98, wr_order_number#99, wr_return_amt#100, wr_net_loss#101, wr_returned_date_sk#102]
+Arguments: hashpartitioning(wr_item_sk#98, wr_order_number#99, 5), ENSURE_REQUIREMENTS, [id=#103]
+
+(54) Sort [codegen id : 15]
+Input [5]: [wr_item_sk#98, wr_order_number#99, wr_return_amt#100, wr_net_loss#101, wr_returned_date_sk#102]
+Arguments: [wr_item_sk#98 ASC NULLS FIRST, wr_order_number#99 ASC NULLS FIRST], false, 0
+
+(55) Scan parquet default.web_sales
+Output [4]: [ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106, ws_sold_date_sk#107]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_sales]
+PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_web_site_sk:int,ws_order_number:int>
+
+(56) ColumnarToRow [codegen id : 16]
+Input [4]: [ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106, ws_sold_date_sk#107]
+
+(57) Filter [codegen id : 16]
+Input [4]: [ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106, ws_sold_date_sk#107]
+Condition : ((isnotnull(ws_item_sk#104) AND isnotnull(ws_order_number#106)) AND isnotnull(ws_web_site_sk#105))
+
+(58) Project [codegen id : 16]
+Output [3]: [ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106]
+Input [4]: [ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106, ws_sold_date_sk#107]
+
+(59) Exchange
+Input [3]: [ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106]
+Arguments: hashpartitioning(ws_item_sk#104, ws_order_number#106, 5), ENSURE_REQUIREMENTS, [id=#108]
+
+(60) Sort [codegen id : 17]
+Input [3]: [ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106]
+Arguments: [ws_item_sk#104 ASC NULLS FIRST, ws_order_number#106 ASC NULLS FIRST], false, 0
+
+(61) SortMergeJoin [codegen id : 18]
+Left keys [2]: [wr_item_sk#98, wr_order_number#99]
+Right keys [2]: [ws_item_sk#104, ws_order_number#106]
+Join condition: None
+
+(62) Project [codegen id : 18]
+Output [6]: [ws_web_site_sk#105 AS wsr_web_site_sk#109, wr_returned_date_sk#102 AS date_sk#110, 0.00 AS sales_price#111, 0.00 AS profit#112, wr_return_amt#100 AS return_amt#113, wr_net_loss#101 AS net_loss#114]
+Input [8]: [wr_item_sk#98, wr_order_number#99, wr_return_amt#100, wr_net_loss#101, wr_returned_date_sk#102, ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106]
+
+(63) Union
+
+(64) Scan parquet default.web_site
+Output [2]: [web_site_sk#115, web_site_id#116]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_site]
+PushedFilters: [IsNotNull(web_site_sk)]
+ReadSchema: struct<web_site_sk:int,web_site_id:string>
+
+(65) ColumnarToRow [codegen id : 19]
+Input [2]: [web_site_sk#115, web_site_id#116]
+
+(66) Filter [codegen id : 19]
+Input [2]: [web_site_sk#115, web_site_id#116]
+Condition : isnotnull(web_site_sk#115)
+
+(67) BroadcastExchange
+Input [2]: [web_site_sk#115, web_site_id#116]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#117]
+
+(68) BroadcastHashJoin [codegen id : 21]
+Left keys [1]: [wsr_web_site_sk#92]
+Right keys [1]: [web_site_sk#115]
+Join condition: None
+
+(69) Project [codegen id : 21]
+Output [6]: [date_sk#93, sales_price#94, profit#95, return_amt#96, net_loss#97, web_site_id#116]
+Input [8]: [wsr_web_site_sk#92, date_sk#93, sales_price#94, profit#95, return_amt#96, net_loss#97, web_site_sk#115, web_site_id#116]
+
+(70) ReusedExchange [Reuses operator id: 20]
+Output [1]: [d_date_sk#118]
+
+(71) BroadcastHashJoin [codegen id : 21]
+Left keys [1]: [date_sk#93]
+Right keys [1]: [d_date_sk#118]
+Join condition: None
+
+(72) Project [codegen id : 21]
+Output [5]: [sales_price#94, profit#95, return_amt#96, net_loss#97, web_site_id#116]
+Input [7]: [date_sk#93, sales_price#94, profit#95, return_amt#96, net_loss#97, web_site_id#116, d_date_sk#118]
+
+(73) HashAggregate [codegen id : 21]
+Input [5]: [sales_price#94, profit#95, return_amt#96, net_loss#97, web_site_id#116]
+Keys [1]: [web_site_id#116]
+Functions [4]: [partial_sum(UnscaledValue(sales_price#94)), partial_sum(UnscaledValue(return_amt#96)), partial_sum(UnscaledValue(profit#95)), partial_sum(UnscaledValue(net_loss#97))]
+Aggregate Attributes [4]: [sum#119, sum#120, sum#121, sum#122]
+Results [5]: [web_site_id#116, sum#123, sum#124, sum#125, sum#126]
+
+(74) Exchange
+Input [5]: [web_site_id#116, sum#123, sum#124, sum#125, sum#126]
+Arguments: hashpartitioning(web_site_id#116, 5), ENSURE_REQUIREMENTS, [id=#127]
+
+(75) HashAggregate [codegen id : 22]
+Input [5]: [web_site_id#116, sum#123, sum#124, sum#125, sum#126]
+Keys [1]: [web_site_id#116]
+Functions [4]: [sum(UnscaledValue(sales_price#94)), sum(UnscaledValue(return_amt#96)), sum(UnscaledValue(profit#95)), sum(UnscaledValue(net_loss#97))]
+Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#94))#128, sum(UnscaledValue(return_amt#96))#129, sum(UnscaledValue(profit#95))#130, sum(UnscaledValue(net_loss#97))#131]
+Results [5]: [web channel AS channel#132, concat(web_site, web_site_id#116) AS id#133, MakeDecimal(sum(UnscaledValue(sales_price#94))#128,17,2) AS sales#134, MakeDecimal(sum(UnscaledValue(return_amt#96))#129,17,2) AS returns#135, CheckOverflow((promote_precision(cast(MakeDecimal(sum(UnscaledValue(profit#95))#130,17,2) as decimal(18,2))) - promote_precision(cast(MakeDecimal(sum(UnscaledValue(net_loss#97))#131,17,2) as decimal(18,2)))), DecimalType(18,2), true) AS profit#136]
+
+(76) Union
+
+(77) HashAggregate [codegen id : 23]
+Input [5]: [channel#41, id#42, sales#43, returns#44, profit#45]
+Keys [2]: [channel#41, id#42]
+Functions [3]: [partial_sum(sales#43), partial_sum(returns#44), partial_sum(profit#45)]
+Aggregate Attributes [6]: [sum#137, isEmpty#138, sum#139, isEmpty#140, sum#141, isEmpty#142]
+Results [8]: [channel#41, id#42, sum#143, isEmpty#144, sum#145, isEmpty#146, sum#147, isEmpty#148]
+
+(78) Exchange
+Input [8]: [channel#41, id#42, sum#143, isEmpty#144, sum#145, isEmpty#146, sum#147, isEmpty#148]
+Arguments: hashpartitioning(channel#41, id#42, 5), ENSURE_REQUIREMENTS, [id=#149]
+
+(79) HashAggregate [codegen id : 24]
+Input [8]: [channel#41, id#42, sum#143, isEmpty#144, sum#145, isEmpty#146, sum#147, isEmpty#148]
+Keys [2]: [channel#41, id#42]
+Functions [3]: [sum(sales#43), sum(returns#44), sum(profit#45)]
+Aggregate Attributes [3]: [sum(sales#43)#150, sum(returns#44)#151, sum(profit#45)#152]
+Results [5]: [channel#41, id#42, cast(sum(sales#43)#150 as decimal(37,2)) AS sales#153, cast(sum(returns#44)#151 as decimal(37,2)) AS returns#154, cast(sum(profit#45)#152 as decimal(38,2)) AS profit#155]
+
+(80) ReusedExchange [Reuses operator id: unknown]
+Output [8]: [channel#41, id#42, sum#156, isEmpty#157, sum#158, isEmpty#159, sum#160, isEmpty#161]
+
+(81) HashAggregate [codegen id : 48]
+Input [8]: [channel#41, id#42, sum#156, isEmpty#157, sum#158, isEmpty#159, sum#160, isEmpty#161]
+Keys [2]: [channel#41, id#42]
+Functions [3]: [sum(sales#43), sum(returns#44), sum(profit#45)]
+Aggregate Attributes [3]: [sum(sales#43)#162, sum(returns#44)#163, sum(profit#45)#164]
+Results [4]: [channel#41, sum(sales#43)#162 AS sales#165, sum(returns#44)#163 AS returns#166, sum(profit#45)#164 AS profit#167]
+
+(82) HashAggregate [codegen id : 48]
+Input [4]: [channel#41, sales#165, returns#166, profit#167]
+Keys [1]: [channel#41]
+Functions [3]: [partial_sum(sales#165), partial_sum(returns#166), partial_sum(profit#167)]
+Aggregate Attributes [6]: [sum#168, isEmpty#169, sum#170, isEmpty#171, sum#172, isEmpty#173]
+Results [7]: [channel#41, sum#174, isEmpty#175, sum#176, isEmpty#177, sum#178, isEmpty#179]
+
+(83) Exchange
+Input [7]: [channel#41, sum#174, isEmpty#175, sum#176, isEmpty#177, sum#178, isEmpty#179]
+Arguments: hashpartitioning(channel#41, 5), ENSURE_REQUIREMENTS, [id=#180]
+
+(84) HashAggregate [codegen id : 49]
+Input [7]: [channel#41, sum#174, isEmpty#175, sum#176, isEmpty#177, sum#178, isEmpty#179]
+Keys [1]: [channel#41]
+Functions [3]: [sum(sales#165), sum(returns#166), sum(profit#167)]
+Aggregate Attributes [3]: [sum(sales#165)#181, sum(returns#166)#182, sum(profit#167)#183]
+Results [5]: [channel#41, null AS id#184, sum(sales#165)#181 AS sum(sales)#185, sum(returns#166)#182 AS sum(returns)#186, sum(profit#167)#183 AS sum(profit)#187]
+
+(85) ReusedExchange [Reuses operator id: unknown]
+Output [8]: [channel#41, id#42, sum#188, isEmpty#189, sum#190, isEmpty#191, sum#192, isEmpty#193]
+
+(86) HashAggregate [codegen id : 73]
+Input [8]: [channel#41, id#42, sum#188, isEmpty#189, sum#190, isEmpty#191, sum#192, isEmpty#193]
+Keys [2]: [channel#41, id#42]
+Functions [3]: [sum(sales#43), sum(returns#44), sum(profit#45)]
+Aggregate Attributes [3]: [sum(sales#43)#194, sum(returns#44)#195, sum(profit#45)#196]
+Results [3]: [sum(sales#43)#194 AS sales#165, sum(returns#44)#195 AS returns#166, sum(profit#45)#196 AS profit#167]
+
+(87) HashAggregate [codegen id : 73]
+Input [3]: [sales#165, returns#166, profit#167]
+Keys: []
+Functions [3]: [partial_sum(sales#165), partial_sum(returns#166), partial_sum(profit#167)]
+Aggregate Attributes [6]: [sum#197, isEmpty#198, sum#199, isEmpty#200, sum#201, isEmpty#202]
+Results [6]: [sum#203, isEmpty#204, sum#205, isEmpty#206, sum#207, isEmpty#208]
+
+(88) Exchange
+Input [6]: [sum#203, isEmpty#204, sum#205, isEmpty#206, sum#207, isEmpty#208]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#209]
+
+(89) HashAggregate [codegen id : 74]
+Input [6]: [sum#203, isEmpty#204, sum#205, isEmpty#206, sum#207, isEmpty#208]
+Keys: []
+Functions [3]: [sum(sales#165), sum(returns#166), sum(profit#167)]
+Aggregate Attributes [3]: [sum(sales#165)#210, sum(returns#166)#211, sum(profit#167)#212]
+Results [5]: [null AS channel#213, null AS id#214, sum(sales#165)#210 AS sum(sales)#215, sum(returns#166)#211 AS sum(returns)#216, sum(profit#167)#212 AS sum(profit)#217]
+
+(90) Union
+
+(91) HashAggregate [codegen id : 75]
+Input [5]: [channel#41, id#42, sales#153, returns#154, profit#155]
+Keys [5]: [channel#41, id#42, sales#153, returns#154, profit#155]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [channel#41, id#42, sales#153, returns#154, profit#155]
+
+(92) Exchange
+Input [5]: [channel#41, id#42, sales#153, returns#154, profit#155]
+Arguments: hashpartitioning(channel#41, id#42, sales#153, returns#154, profit#155, 5), ENSURE_REQUIREMENTS, [id=#218]
+
+(93) HashAggregate [codegen id : 76]
+Input [5]: [channel#41, id#42, sales#153, returns#154, profit#155]
+Keys [5]: [channel#41, id#42, sales#153, returns#154, profit#155]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [channel#41, id#42, sales#153, returns#154, profit#155]
+
+(94) TakeOrderedAndProject
+Input [5]: [channel#41, id#42, sales#153, returns#154, profit#155]
+Arguments: 100, [channel#41 ASC NULLS FIRST, id#42 ASC NULLS FIRST], [channel#41, id#42, sales#153, returns#154, profit#155]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (95)
+
+
+(95) ReusedExchange [Reuses operator id: 20]
+Output [1]: [d_date_sk#25]
+
+Subquery:2 Hosting operator id = 5 Hosting Expression = sr_returned_date_sk#15 IN dynamicpruning#5
+
+Subquery:3 Hosting operator id = 26 Hosting Expression = cs_sold_date_sk#49 IN dynamicpruning#5
+
+Subquery:4 Hosting operator id = 30 Hosting Expression = cr_returned_date_sk#59 IN dynamicpruning#5
+
+Subquery:5 Hosting operator id = 47 Hosting Expression = ws_sold_date_sk#91 IN dynamicpruning#5
+
+Subquery:6 Hosting operator id = 51 Hosting Expression = wr_returned_date_sk#102 IN dynamicpruning#5
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.sf100/simplified.txt
new file mode 100644
index 0000000000000..558e8bcc406bc
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.sf100/simplified.txt
@@ -0,0 +1,164 @@
+TakeOrderedAndProject [channel,id,sales,returns,profit]
+  WholeStageCodegen (76)
+    HashAggregate [channel,id,sales,returns,profit]
+      InputAdapter
+        Exchange [channel,id,sales,returns,profit] #1
+          WholeStageCodegen (75)
+            HashAggregate [channel,id,sales,returns,profit]
+              InputAdapter
+                Union
+                  WholeStageCodegen (24)
+                    HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                      InputAdapter
+                        Exchange [channel,id] #2
+                          WholeStageCodegen (23)
+                            HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                              InputAdapter
+                                Union
+                                  WholeStageCodegen (6)
+                                    HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum]
+                                      InputAdapter
+                                        Exchange [s_store_id] #3
+                                          WholeStageCodegen (5)
+                                            HashAggregate [s_store_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum]
+                                              Project [sales_price,profit,return_amt,net_loss,s_store_id]
+                                                BroadcastHashJoin [date_sk,d_date_sk]
+                                                  Project [date_sk,sales_price,profit,return_amt,net_loss,s_store_id]
+                                                    BroadcastHashJoin [store_sk,s_store_sk]
+                                                      InputAdapter
+                                                        Union
+                                                          WholeStageCodegen (1)
+                                                            Project [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit]
+                                                              Filter [ss_store_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk]
+                                                                      SubqueryBroadcast [d_date_sk] #1
+                                                                        ReusedExchange [d_date_sk] #4
+                                                          WholeStageCodegen (2)
+                                                            Project [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss]
+                                                              Filter [sr_store_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk]
+                                                                      ReusedSubquery [d_date_sk] #1
+                                                      InputAdapter
+                                                        BroadcastExchange #5
+                                                          WholeStageCodegen (3)
+                                                            Filter [s_store_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.store [s_store_sk,s_store_id]
+                                                  InputAdapter
+                                                    BroadcastExchange #4
+                                                      WholeStageCodegen (4)
+                                                        Project [d_date_sk]
+                                                          Filter [d_date,d_date_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.date_dim [d_date_sk,d_date]
+                                  WholeStageCodegen (12)
+                                    HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum]
+                                      InputAdapter
+                                        Exchange [cp_catalog_page_id] #6
+                                          WholeStageCodegen (11)
+                                            HashAggregate [cp_catalog_page_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum]
+                                              Project [sales_price,profit,return_amt,net_loss,cp_catalog_page_id]
+                                                BroadcastHashJoin [date_sk,d_date_sk]
+                                                  Project [date_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_id]
+                                                    BroadcastHashJoin [page_sk,cp_catalog_page_sk]
+                                                      InputAdapter
+                                                        Union
+                                                          WholeStageCodegen (7)
+                                                            Project [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit]
+                                                              Filter [cs_catalog_page_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk]
+                                                                      ReusedSubquery [d_date_sk] #1
+                                                          WholeStageCodegen (8)
+                                                            Project [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss]
+                                                              Filter [cr_catalog_page_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk]
+                                                                      ReusedSubquery [d_date_sk] #1
+                                                      InputAdapter
+                                                        BroadcastExchange #7
+                                                          WholeStageCodegen (9)
+                                                            Filter [cp_catalog_page_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id]
+                                                  InputAdapter
+                                                    ReusedExchange [d_date_sk] #4
+                                  WholeStageCodegen (22)
+                                    HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum]
+                                      InputAdapter
+                                        Exchange [web_site_id] #8
+                                          WholeStageCodegen (21)
+                                            HashAggregate [web_site_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum]
+                                              Project [sales_price,profit,return_amt,net_loss,web_site_id]
+                                                BroadcastHashJoin [date_sk,d_date_sk]
+                                                  Project [date_sk,sales_price,profit,return_amt,net_loss,web_site_id]
+                                                    BroadcastHashJoin [wsr_web_site_sk,web_site_sk]
+                                                      InputAdapter
+                                                        Union
+                                                          WholeStageCodegen (13)
+                                                            Project [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit]
+                                                              Filter [ws_web_site_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk]
+                                                                      ReusedSubquery [d_date_sk] #1
+                                                          WholeStageCodegen (18)
+                                                            Project [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss]
+                                                              SortMergeJoin [wr_item_sk,wr_order_number,ws_item_sk,ws_order_number]
+                                                                InputAdapter
+                                                                  WholeStageCodegen (15)
+                                                                    Sort [wr_item_sk,wr_order_number]
+                                                                      InputAdapter
+                                                                        Exchange [wr_item_sk,wr_order_number] #9
+                                                                          WholeStageCodegen (14)
+                                                                            ColumnarToRow
+                                                                              InputAdapter
+                                                                                Scan parquet default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk]
+                                                                                  ReusedSubquery [d_date_sk] #1
+                                                                InputAdapter
+                                                                  WholeStageCodegen (17)
+                                                                    Sort [ws_item_sk,ws_order_number]
+                                                                      InputAdapter
+                                                                        Exchange [ws_item_sk,ws_order_number] #10
+                                                                          WholeStageCodegen (16)
+                                                                            Project [ws_item_sk,ws_web_site_sk,ws_order_number]
+                                                                              Filter [ws_item_sk,ws_order_number,ws_web_site_sk]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk]
+                                                      InputAdapter
+                                                        BroadcastExchange #11
+                                                          WholeStageCodegen (19)
+                                                            Filter [web_site_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.web_site [web_site_sk,web_site_id]
+                                                  InputAdapter
+                                                    ReusedExchange [d_date_sk] #4
+                  WholeStageCodegen (49)
+                    HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                      InputAdapter
+                        Exchange [channel] #12
+                          WholeStageCodegen (48)
+                            HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                              HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                                InputAdapter
+                                  ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #13
+                  WholeStageCodegen (74)
+                    HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                      InputAdapter
+                        Exchange #14
+                          WholeStageCodegen (73)
+                            HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                              HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                                InputAdapter
+                                  ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #13
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/explain.txt
new file mode 100644
index 0000000000000..036b3cdcb22fe
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/explain.txt
@@ -0,0 +1,546 @@
+== Physical Plan ==
+TakeOrderedAndProject (91)
++- * HashAggregate (90)
+   +- Exchange (89)
+      +- * HashAggregate (88)
+         +- Union (87)
+            :- * HashAggregate (76)
+            :  +- Exchange (75)
+            :     +- * HashAggregate (74)
+            :        +- Union (73)
+            :           :- * HashAggregate (25)
+            :           :  +- Exchange (24)
+            :           :     +- * HashAggregate (23)
+            :           :        +- * Project (22)
+            :           :           +- * BroadcastHashJoin Inner BuildRight (21)
+            :           :              :- * Project (16)
+            :           :              :  +- * BroadcastHashJoin Inner BuildRight (15)
+            :           :              :     :- Union (9)
+            :           :              :     :  :- * Project (4)
+            :           :              :     :  :  +- * Filter (3)
+            :           :              :     :  :     +- * ColumnarToRow (2)
+            :           :              :     :  :        +- Scan parquet default.store_sales (1)
+            :           :              :     :  +- * Project (8)
+            :           :              :     :     +- * Filter (7)
+            :           :              :     :        +- * ColumnarToRow (6)
+            :           :              :     :           +- Scan parquet default.store_returns (5)
+            :           :              :     +- BroadcastExchange (14)
+            :           :              :        +- * Project (13)
+            :           :              :           +- * Filter (12)
+            :           :              :              +- * ColumnarToRow (11)
+            :           :              :                 +- Scan parquet default.date_dim (10)
+            :           :              +- BroadcastExchange (20)
+            :           :                 +- * Filter (19)
+            :           :                    +- * ColumnarToRow (18)
+            :           :                       +- Scan parquet default.store (17)
+            :           :- * HashAggregate (46)
+            :           :  +- Exchange (45)
+            :           :     +- * HashAggregate (44)
+            :           :        +- * Project (43)
+            :           :           +- * BroadcastHashJoin Inner BuildRight (42)
+            :           :              :- * Project (37)
+            :           :              :  +- * BroadcastHashJoin Inner BuildRight (36)
+            :           :              :     :- Union (34)
+            :           :              :     :  :- * Project (29)
+            :           :              :     :  :  +- * Filter (28)
+            :           :              :     :  :     +- * ColumnarToRow (27)
+            :           :              :     :  :        +- Scan parquet default.catalog_sales (26)
+            :           :              :     :  +- * Project (33)
+            :           :              :     :     +- * Filter (32)
+            :           :              :     :        +- * ColumnarToRow (31)
+            :           :              :     :           +- Scan parquet default.catalog_returns (30)
+            :           :              :     +- ReusedExchange (35)
+            :           :              +- BroadcastExchange (41)
+            :           :                 +- * Filter (40)
+            :           :                    +- * ColumnarToRow (39)
+            :           :                       +- Scan parquet default.catalog_page (38)
+            :           +- * HashAggregate (72)
+            :              +- Exchange (71)
+            :                 +- * HashAggregate (70)
+            :                    +- * Project (69)
+            :                       +- * BroadcastHashJoin Inner BuildRight (68)
+            :                          :- * Project (63)
+            :                          :  +- * BroadcastHashJoin Inner BuildRight (62)
+            :                          :     :- Union (60)
+            :                          :     :  :- * Project (50)
+            :                          :     :  :  +- * Filter (49)
+            :                          :     :  :     +- * ColumnarToRow (48)
+            :                          :     :  :        +- Scan parquet default.web_sales (47)
+            :                          :     :  +- * Project (59)
+            :                          :     :     +- * BroadcastHashJoin Inner BuildLeft (58)
+            :                          :     :        :- BroadcastExchange (53)
+            :                          :     :        :  +- * ColumnarToRow (52)
+            :                          :     :        :     +- Scan parquet default.web_returns (51)
+            :                          :     :        +- * Project (57)
+            :                          :     :           +- * Filter (56)
+            :                          :     :              +- * ColumnarToRow (55)
+            :                          :     :                 +- Scan parquet default.web_sales (54)
+            :                          :     +- ReusedExchange (61)
+            :                          +- BroadcastExchange (67)
+            :                             +- * Filter (66)
+            :                                +- * ColumnarToRow (65)
+            :                                   +- Scan parquet default.web_site (64)
+            :- * HashAggregate (81)
+            :  +- Exchange (80)
+            :     +- * HashAggregate (79)
+            :        +- * HashAggregate (78)
+            :           +- ReusedExchange (77)
+            +- * HashAggregate (86)
+               +- Exchange (85)
+                  +- * HashAggregate (84)
+                     +- * HashAggregate (83)
+                        +- ReusedExchange (82)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_ext_sales_price:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 1]
+Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+Condition : isnotnull(ss_store_sk#1)
+
+(4) Project [codegen id : 1]
+Output [6]: [ss_store_sk#1 AS store_sk#6, ss_sold_date_sk#4 AS date_sk#7, ss_ext_sales_price#2 AS sales_price#8, ss_net_profit#3 AS profit#9, 0.00 AS return_amt#10, 0.00 AS net_loss#11]
+Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+
+(5) Scan parquet default.store_returns
+Output [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(sr_returned_date_sk#15), dynamicpruningexpression(sr_returned_date_sk#15 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(sr_store_sk)]
+ReadSchema: struct<sr_store_sk:int,sr_return_amt:decimal(7,2),sr_net_loss:decimal(7,2)>
+
+(6) ColumnarToRow [codegen id : 2]
+Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15]
+
+(7) Filter [codegen id : 2]
+Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15]
+Condition : isnotnull(sr_store_sk#12)
+
+(8) Project [codegen id : 2]
+Output [6]: [sr_store_sk#12 AS store_sk#16, sr_returned_date_sk#15 AS date_sk#17, 0.00 AS sales_price#18, 0.00 AS profit#19, sr_return_amt#13 AS return_amt#20, sr_net_loss#14 AS net_loss#21]
+Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15]
+
+(9) Union
+
+(10) Scan parquet default.date_dim
+Output [2]: [d_date_sk#22, d_date#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-08-18), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(11) ColumnarToRow [codegen id : 3]
+Input [2]: [d_date_sk#22, d_date#23]
+
+(12) Filter [codegen id : 3]
+Input [2]: [d_date_sk#22, d_date#23]
+Condition : (((isnotnull(d_date#23) AND (d_date#23 >= 1998-08-04)) AND (d_date#23 <= 1998-08-18)) AND isnotnull(d_date_sk#22))
+
+(13) Project [codegen id : 3]
+Output [1]: [d_date_sk#22]
+Input [2]: [d_date_sk#22, d_date#23]
+
+(14) BroadcastExchange
+Input [1]: [d_date_sk#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#24]
+
+(15) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [date_sk#7]
+Right keys [1]: [d_date_sk#22]
+Join condition: None
+
+(16) Project [codegen id : 5]
+Output [5]: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11]
+Input [7]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, d_date_sk#22]
+
+(17) Scan parquet default.store
+Output [2]: [s_store_sk#25, s_store_id#26]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string>
+
+(18) ColumnarToRow [codegen id : 4]
+Input [2]: [s_store_sk#25, s_store_id#26]
+
+(19) Filter [codegen id : 4]
+Input [2]: [s_store_sk#25, s_store_id#26]
+Condition : isnotnull(s_store_sk#25)
+
+(20) BroadcastExchange
+Input [2]: [s_store_sk#25, s_store_id#26]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#27]
+
+(21) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [store_sk#6]
+Right keys [1]: [s_store_sk#25]
+Join condition: None
+
+(22) Project [codegen id : 5]
+Output [5]: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#26]
+Input [7]: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_sk#25, s_store_id#26]
+
+(23) HashAggregate [codegen id : 5]
+Input [5]: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#26]
+Keys [1]: [s_store_id#26]
+Functions [4]: [partial_sum(UnscaledValue(sales_price#8)), partial_sum(UnscaledValue(return_amt#10)), partial_sum(UnscaledValue(profit#9)), partial_sum(UnscaledValue(net_loss#11))]
+Aggregate Attributes [4]: [sum#28, sum#29, sum#30, sum#31]
+Results [5]: [s_store_id#26, sum#32, sum#33, sum#34, sum#35]
+
+(24) Exchange
+Input [5]: [s_store_id#26, sum#32, sum#33, sum#34, sum#35]
+Arguments: hashpartitioning(s_store_id#26, 5), ENSURE_REQUIREMENTS, [id=#36]
+
+(25) HashAggregate [codegen id : 6]
+Input [5]: [s_store_id#26, sum#32, sum#33, sum#34, sum#35]
+Keys [1]: [s_store_id#26]
+Functions [4]: [sum(UnscaledValue(sales_price#8)), sum(UnscaledValue(return_amt#10)), sum(UnscaledValue(profit#9)), sum(UnscaledValue(net_loss#11))]
+Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#8))#37, sum(UnscaledValue(return_amt#10))#38, sum(UnscaledValue(profit#9))#39, sum(UnscaledValue(net_loss#11))#40]
+Results [5]: [store channel AS channel#41, concat(store, s_store_id#26) AS id#42, MakeDecimal(sum(UnscaledValue(sales_price#8))#37,17,2) AS sales#43, MakeDecimal(sum(UnscaledValue(return_amt#10))#38,17,2) AS returns#44, CheckOverflow((promote_precision(cast(MakeDecimal(sum(UnscaledValue(profit#9))#39,17,2) as decimal(18,2))) - promote_precision(cast(MakeDecimal(sum(UnscaledValue(net_loss#11))#40,17,2) as decimal(18,2)))), DecimalType(18,2), true) AS profit#45]
+
+(26) Scan parquet default.catalog_sales
+Output [4]: [cs_catalog_page_sk#46, cs_ext_sales_price#47, cs_net_profit#48, cs_sold_date_sk#49]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#49), dynamicpruningexpression(cs_sold_date_sk#49 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(cs_catalog_page_sk)]
+ReadSchema: struct<cs_catalog_page_sk:int,cs_ext_sales_price:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(27) ColumnarToRow [codegen id : 7]
+Input [4]: [cs_catalog_page_sk#46, cs_ext_sales_price#47, cs_net_profit#48, cs_sold_date_sk#49]
+
+(28) Filter [codegen id : 7]
+Input [4]: [cs_catalog_page_sk#46, cs_ext_sales_price#47, cs_net_profit#48, cs_sold_date_sk#49]
+Condition : isnotnull(cs_catalog_page_sk#46)
+
+(29) Project [codegen id : 7]
+Output [6]: [cs_catalog_page_sk#46 AS page_sk#50, cs_sold_date_sk#49 AS date_sk#51, cs_ext_sales_price#47 AS sales_price#52, cs_net_profit#48 AS profit#53, 0.00 AS return_amt#54, 0.00 AS net_loss#55]
+Input [4]: [cs_catalog_page_sk#46, cs_ext_sales_price#47, cs_net_profit#48, cs_sold_date_sk#49]
+
+(30) Scan parquet default.catalog_returns
+Output [4]: [cr_catalog_page_sk#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cr_returned_date_sk#59), dynamicpruningexpression(cr_returned_date_sk#59 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(cr_catalog_page_sk)]
+ReadSchema: struct<cr_catalog_page_sk:int,cr_return_amount:decimal(7,2),cr_net_loss:decimal(7,2)>
+
+(31) ColumnarToRow [codegen id : 8]
+Input [4]: [cr_catalog_page_sk#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+
+(32) Filter [codegen id : 8]
+Input [4]: [cr_catalog_page_sk#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+Condition : isnotnull(cr_catalog_page_sk#56)
+
+(33) Project [codegen id : 8]
+Output [6]: [cr_catalog_page_sk#56 AS page_sk#60, cr_returned_date_sk#59 AS date_sk#61, 0.00 AS sales_price#62, 0.00 AS profit#63, cr_return_amount#57 AS return_amt#64, cr_net_loss#58 AS net_loss#65]
+Input [4]: [cr_catalog_page_sk#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+
+(34) Union
+
+(35) ReusedExchange [Reuses operator id: 14]
+Output [1]: [d_date_sk#66]
+
+(36) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [date_sk#51]
+Right keys [1]: [d_date_sk#66]
+Join condition: None
+
+(37) Project [codegen id : 11]
+Output [5]: [page_sk#50, sales_price#52, profit#53, return_amt#54, net_loss#55]
+Input [7]: [page_sk#50, date_sk#51, sales_price#52, profit#53, return_amt#54, net_loss#55, d_date_sk#66]
+
+(38) Scan parquet default.catalog_page
+Output [2]: [cp_catalog_page_sk#67, cp_catalog_page_id#68]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_page]
+PushedFilters: [IsNotNull(cp_catalog_page_sk)]
+ReadSchema: struct<cp_catalog_page_sk:int,cp_catalog_page_id:string>
+
+(39) ColumnarToRow [codegen id : 10]
+Input [2]: [cp_catalog_page_sk#67, cp_catalog_page_id#68]
+
+(40) Filter [codegen id : 10]
+Input [2]: [cp_catalog_page_sk#67, cp_catalog_page_id#68]
+Condition : isnotnull(cp_catalog_page_sk#67)
+
+(41) BroadcastExchange
+Input [2]: [cp_catalog_page_sk#67, cp_catalog_page_id#68]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#69]
+
+(42) BroadcastHashJoin [codegen id : 11]
+Left keys [1]: [page_sk#50]
+Right keys [1]: [cp_catalog_page_sk#67]
+Join condition: None
+
+(43) Project [codegen id : 11]
+Output [5]: [sales_price#52, profit#53, return_amt#54, net_loss#55, cp_catalog_page_id#68]
+Input [7]: [page_sk#50, sales_price#52, profit#53, return_amt#54, net_loss#55, cp_catalog_page_sk#67, cp_catalog_page_id#68]
+
+(44) HashAggregate [codegen id : 11]
+Input [5]: [sales_price#52, profit#53, return_amt#54, net_loss#55, cp_catalog_page_id#68]
+Keys [1]: [cp_catalog_page_id#68]
+Functions [4]: [partial_sum(UnscaledValue(sales_price#52)), partial_sum(UnscaledValue(return_amt#54)), partial_sum(UnscaledValue(profit#53)), partial_sum(UnscaledValue(net_loss#55))]
+Aggregate Attributes [4]: [sum#70, sum#71, sum#72, sum#73]
+Results [5]: [cp_catalog_page_id#68, sum#74, sum#75, sum#76, sum#77]
+
+(45) Exchange
+Input [5]: [cp_catalog_page_id#68, sum#74, sum#75, sum#76, sum#77]
+Arguments: hashpartitioning(cp_catalog_page_id#68, 5), ENSURE_REQUIREMENTS, [id=#78]
+
+(46) HashAggregate [codegen id : 12]
+Input [5]: [cp_catalog_page_id#68, sum#74, sum#75, sum#76, sum#77]
+Keys [1]: [cp_catalog_page_id#68]
+Functions [4]: [sum(UnscaledValue(sales_price#52)), sum(UnscaledValue(return_amt#54)), sum(UnscaledValue(profit#53)), sum(UnscaledValue(net_loss#55))]
+Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#52))#79, sum(UnscaledValue(return_amt#54))#80, sum(UnscaledValue(profit#53))#81, sum(UnscaledValue(net_loss#55))#82]
+Results [5]: [catalog channel AS channel#83, concat(catalog_page, cp_catalog_page_id#68) AS id#84, MakeDecimal(sum(UnscaledValue(sales_price#52))#79,17,2) AS sales#85, MakeDecimal(sum(UnscaledValue(return_amt#54))#80,17,2) AS returns#86, CheckOverflow((promote_precision(cast(MakeDecimal(sum(UnscaledValue(profit#53))#81,17,2) as decimal(18,2))) - promote_precision(cast(MakeDecimal(sum(UnscaledValue(net_loss#55))#82,17,2) as decimal(18,2)))), DecimalType(18,2), true) AS profit#87]
+
+(47) Scan parquet default.web_sales
+Output [4]: [ws_web_site_sk#88, ws_ext_sales_price#89, ws_net_profit#90, ws_sold_date_sk#91]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#91), dynamicpruningexpression(ws_sold_date_sk#91 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ws_web_site_sk)]
+ReadSchema: struct<ws_web_site_sk:int,ws_ext_sales_price:decimal(7,2),ws_net_profit:decimal(7,2)>
+
+(48) ColumnarToRow [codegen id : 13]
+Input [4]: [ws_web_site_sk#88, ws_ext_sales_price#89, ws_net_profit#90, ws_sold_date_sk#91]
+
+(49) Filter [codegen id : 13]
+Input [4]: [ws_web_site_sk#88, ws_ext_sales_price#89, ws_net_profit#90, ws_sold_date_sk#91]
+Condition : isnotnull(ws_web_site_sk#88)
+
+(50) Project [codegen id : 13]
+Output [6]: [ws_web_site_sk#88 AS wsr_web_site_sk#92, ws_sold_date_sk#91 AS date_sk#93, ws_ext_sales_price#89 AS sales_price#94, ws_net_profit#90 AS profit#95, 0.00 AS return_amt#96, 0.00 AS net_loss#97]
+Input [4]: [ws_web_site_sk#88, ws_ext_sales_price#89, ws_net_profit#90, ws_sold_date_sk#91]
+
+(51) Scan parquet default.web_returns
+Output [5]: [wr_item_sk#98, wr_order_number#99, wr_return_amt#100, wr_net_loss#101, wr_returned_date_sk#102]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(wr_returned_date_sk#102), dynamicpruningexpression(wr_returned_date_sk#102 IN dynamicpruning#5)]
+ReadSchema: struct<wr_item_sk:int,wr_order_number:int,wr_return_amt:decimal(7,2),wr_net_loss:decimal(7,2)>
+
+(52) ColumnarToRow [codegen id : 14]
+Input [5]: [wr_item_sk#98, wr_order_number#99, wr_return_amt#100, wr_net_loss#101, wr_returned_date_sk#102]
+
+(53) BroadcastExchange
+Input [5]: [wr_item_sk#98, wr_order_number#99, wr_return_amt#100, wr_net_loss#101, wr_returned_date_sk#102]
+Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, true] as bigint), 32) | (cast(input[1, int, true] as bigint) & 4294967295))),false), [id=#103]
+
+(54) Scan parquet default.web_sales
+Output [4]: [ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106, ws_sold_date_sk#107]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_sales]
+PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_web_site_sk:int,ws_order_number:int>
+
+(55) ColumnarToRow
+Input [4]: [ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106, ws_sold_date_sk#107]
+
+(56) Filter
+Input [4]: [ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106, ws_sold_date_sk#107]
+Condition : ((isnotnull(ws_item_sk#104) AND isnotnull(ws_order_number#106)) AND isnotnull(ws_web_site_sk#105))
+
+(57) Project
+Output [3]: [ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106]
+Input [4]: [ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106, ws_sold_date_sk#107]
+
+(58) BroadcastHashJoin [codegen id : 15]
+Left keys [2]: [wr_item_sk#98, wr_order_number#99]
+Right keys [2]: [ws_item_sk#104, ws_order_number#106]
+Join condition: None
+
+(59) Project [codegen id : 15]
+Output [6]: [ws_web_site_sk#105 AS wsr_web_site_sk#108, wr_returned_date_sk#102 AS date_sk#109, 0.00 AS sales_price#110, 0.00 AS profit#111, wr_return_amt#100 AS return_amt#112, wr_net_loss#101 AS net_loss#113]
+Input [8]: [wr_item_sk#98, wr_order_number#99, wr_return_amt#100, wr_net_loss#101, wr_returned_date_sk#102, ws_item_sk#104, ws_web_site_sk#105, ws_order_number#106]
+
+(60) Union
+
+(61) ReusedExchange [Reuses operator id: 14]
+Output [1]: [d_date_sk#114]
+
+(62) BroadcastHashJoin [codegen id : 18]
+Left keys [1]: [date_sk#93]
+Right keys [1]: [d_date_sk#114]
+Join condition: None
+
+(63) Project [codegen id : 18]
+Output [5]: [wsr_web_site_sk#92, sales_price#94, profit#95, return_amt#96, net_loss#97]
+Input [7]: [wsr_web_site_sk#92, date_sk#93, sales_price#94, profit#95, return_amt#96, net_loss#97, d_date_sk#114]
+
+(64) Scan parquet default.web_site
+Output [2]: [web_site_sk#115, web_site_id#116]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_site]
+PushedFilters: [IsNotNull(web_site_sk)]
+ReadSchema: struct<web_site_sk:int,web_site_id:string>
+
+(65) ColumnarToRow [codegen id : 17]
+Input [2]: [web_site_sk#115, web_site_id#116]
+
+(66) Filter [codegen id : 17]
+Input [2]: [web_site_sk#115, web_site_id#116]
+Condition : isnotnull(web_site_sk#115)
+
+(67) BroadcastExchange
+Input [2]: [web_site_sk#115, web_site_id#116]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#117]
+
+(68) BroadcastHashJoin [codegen id : 18]
+Left keys [1]: [wsr_web_site_sk#92]
+Right keys [1]: [web_site_sk#115]
+Join condition: None
+
+(69) Project [codegen id : 18]
+Output [5]: [sales_price#94, profit#95, return_amt#96, net_loss#97, web_site_id#116]
+Input [7]: [wsr_web_site_sk#92, sales_price#94, profit#95, return_amt#96, net_loss#97, web_site_sk#115, web_site_id#116]
+
+(70) HashAggregate [codegen id : 18]
+Input [5]: [sales_price#94, profit#95, return_amt#96, net_loss#97, web_site_id#116]
+Keys [1]: [web_site_id#116]
+Functions [4]: [partial_sum(UnscaledValue(sales_price#94)), partial_sum(UnscaledValue(return_amt#96)), partial_sum(UnscaledValue(profit#95)), partial_sum(UnscaledValue(net_loss#97))]
+Aggregate Attributes [4]: [sum#118, sum#119, sum#120, sum#121]
+Results [5]: [web_site_id#116, sum#122, sum#123, sum#124, sum#125]
+
+(71) Exchange
+Input [5]: [web_site_id#116, sum#122, sum#123, sum#124, sum#125]
+Arguments: hashpartitioning(web_site_id#116, 5), ENSURE_REQUIREMENTS, [id=#126]
+
+(72) HashAggregate [codegen id : 19]
+Input [5]: [web_site_id#116, sum#122, sum#123, sum#124, sum#125]
+Keys [1]: [web_site_id#116]
+Functions [4]: [sum(UnscaledValue(sales_price#94)), sum(UnscaledValue(return_amt#96)), sum(UnscaledValue(profit#95)), sum(UnscaledValue(net_loss#97))]
+Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#94))#127, sum(UnscaledValue(return_amt#96))#128, sum(UnscaledValue(profit#95))#129, sum(UnscaledValue(net_loss#97))#130]
+Results [5]: [web channel AS channel#131, concat(web_site, web_site_id#116) AS id#132, MakeDecimal(sum(UnscaledValue(sales_price#94))#127,17,2) AS sales#133, MakeDecimal(sum(UnscaledValue(return_amt#96))#128,17,2) AS returns#134, CheckOverflow((promote_precision(cast(MakeDecimal(sum(UnscaledValue(profit#95))#129,17,2) as decimal(18,2))) - promote_precision(cast(MakeDecimal(sum(UnscaledValue(net_loss#97))#130,17,2) as decimal(18,2)))), DecimalType(18,2), true) AS profit#135]
+
+(73) Union
+
+(74) HashAggregate [codegen id : 20]
+Input [5]: [channel#41, id#42, sales#43, returns#44, profit#45]
+Keys [2]: [channel#41, id#42]
+Functions [3]: [partial_sum(sales#43), partial_sum(returns#44), partial_sum(profit#45)]
+Aggregate Attributes [6]: [sum#136, isEmpty#137, sum#138, isEmpty#139, sum#140, isEmpty#141]
+Results [8]: [channel#41, id#42, sum#142, isEmpty#143, sum#144, isEmpty#145, sum#146, isEmpty#147]
+
+(75) Exchange
+Input [8]: [channel#41, id#42, sum#142, isEmpty#143, sum#144, isEmpty#145, sum#146, isEmpty#147]
+Arguments: hashpartitioning(channel#41, id#42, 5), ENSURE_REQUIREMENTS, [id=#148]
+
+(76) HashAggregate [codegen id : 21]
+Input [8]: [channel#41, id#42, sum#142, isEmpty#143, sum#144, isEmpty#145, sum#146, isEmpty#147]
+Keys [2]: [channel#41, id#42]
+Functions [3]: [sum(sales#43), sum(returns#44), sum(profit#45)]
+Aggregate Attributes [3]: [sum(sales#43)#149, sum(returns#44)#150, sum(profit#45)#151]
+Results [5]: [channel#41, id#42, cast(sum(sales#43)#149 as decimal(37,2)) AS sales#152, cast(sum(returns#44)#150 as decimal(37,2)) AS returns#153, cast(sum(profit#45)#151 as decimal(38,2)) AS profit#154]
+
+(77) ReusedExchange [Reuses operator id: unknown]
+Output [8]: [channel#41, id#42, sum#155, isEmpty#156, sum#157, isEmpty#158, sum#159, isEmpty#160]
+
+(78) HashAggregate [codegen id : 42]
+Input [8]: [channel#41, id#42, sum#155, isEmpty#156, sum#157, isEmpty#158, sum#159, isEmpty#160]
+Keys [2]: [channel#41, id#42]
+Functions [3]: [sum(sales#43), sum(returns#44), sum(profit#45)]
+Aggregate Attributes [3]: [sum(sales#43)#161, sum(returns#44)#162, sum(profit#45)#163]
+Results [4]: [channel#41, sum(sales#43)#161 AS sales#164, sum(returns#44)#162 AS returns#165, sum(profit#45)#163 AS profit#166]
+
+(79) HashAggregate [codegen id : 42]
+Input [4]: [channel#41, sales#164, returns#165, profit#166]
+Keys [1]: [channel#41]
+Functions [3]: [partial_sum(sales#164), partial_sum(returns#165), partial_sum(profit#166)]
+Aggregate Attributes [6]: [sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172]
+Results [7]: [channel#41, sum#173, isEmpty#174, sum#175, isEmpty#176, sum#177, isEmpty#178]
+
+(80) Exchange
+Input [7]: [channel#41, sum#173, isEmpty#174, sum#175, isEmpty#176, sum#177, isEmpty#178]
+Arguments: hashpartitioning(channel#41, 5), ENSURE_REQUIREMENTS, [id=#179]
+
+(81) HashAggregate [codegen id : 43]
+Input [7]: [channel#41, sum#173, isEmpty#174, sum#175, isEmpty#176, sum#177, isEmpty#178]
+Keys [1]: [channel#41]
+Functions [3]: [sum(sales#164), sum(returns#165), sum(profit#166)]
+Aggregate Attributes [3]: [sum(sales#164)#180, sum(returns#165)#181, sum(profit#166)#182]
+Results [5]: [channel#41, null AS id#183, sum(sales#164)#180 AS sum(sales)#184, sum(returns#165)#181 AS sum(returns)#185, sum(profit#166)#182 AS sum(profit)#186]
+
+(82) ReusedExchange [Reuses operator id: unknown]
+Output [8]: [channel#41, id#42, sum#187, isEmpty#188, sum#189, isEmpty#190, sum#191, isEmpty#192]
+
+(83) HashAggregate [codegen id : 64]
+Input [8]: [channel#41, id#42, sum#187, isEmpty#188, sum#189, isEmpty#190, sum#191, isEmpty#192]
+Keys [2]: [channel#41, id#42]
+Functions [3]: [sum(sales#43), sum(returns#44), sum(profit#45)]
+Aggregate Attributes [3]: [sum(sales#43)#193, sum(returns#44)#194, sum(profit#45)#195]
+Results [3]: [sum(sales#43)#193 AS sales#164, sum(returns#44)#194 AS returns#165, sum(profit#45)#195 AS profit#166]
+
+(84) HashAggregate [codegen id : 64]
+Input [3]: [sales#164, returns#165, profit#166]
+Keys: []
+Functions [3]: [partial_sum(sales#164), partial_sum(returns#165), partial_sum(profit#166)]
+Aggregate Attributes [6]: [sum#196, isEmpty#197, sum#198, isEmpty#199, sum#200, isEmpty#201]
+Results [6]: [sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207]
+
+(85) Exchange
+Input [6]: [sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#208]
+
+(86) HashAggregate [codegen id : 65]
+Input [6]: [sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207]
+Keys: []
+Functions [3]: [sum(sales#164), sum(returns#165), sum(profit#166)]
+Aggregate Attributes [3]: [sum(sales#164)#209, sum(returns#165)#210, sum(profit#166)#211]
+Results [5]: [null AS channel#212, null AS id#213, sum(sales#164)#209 AS sum(sales)#214, sum(returns#165)#210 AS sum(returns)#215, sum(profit#166)#211 AS sum(profit)#216]
+
+(87) Union
+
+(88) HashAggregate [codegen id : 66]
+Input [5]: [channel#41, id#42, sales#152, returns#153, profit#154]
+Keys [5]: [channel#41, id#42, sales#152, returns#153, profit#154]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [channel#41, id#42, sales#152, returns#153, profit#154]
+
+(89) Exchange
+Input [5]: [channel#41, id#42, sales#152, returns#153, profit#154]
+Arguments: hashpartitioning(channel#41, id#42, sales#152, returns#153, profit#154, 5), ENSURE_REQUIREMENTS, [id=#217]
+
+(90) HashAggregate [codegen id : 67]
+Input [5]: [channel#41, id#42, sales#152, returns#153, profit#154]
+Keys [5]: [channel#41, id#42, sales#152, returns#153, profit#154]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [channel#41, id#42, sales#152, returns#153, profit#154]
+
+(91) TakeOrderedAndProject
+Input [5]: [channel#41, id#42, sales#152, returns#153, profit#154]
+Arguments: 100, [channel#41 ASC NULLS FIRST, id#42 ASC NULLS FIRST], [channel#41, id#42, sales#152, returns#153, profit#154]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (92)
+
+
+(92) ReusedExchange [Reuses operator id: 14]
+Output [1]: [d_date_sk#22]
+
+Subquery:2 Hosting operator id = 5 Hosting Expression = sr_returned_date_sk#15 IN dynamicpruning#5
+
+Subquery:3 Hosting operator id = 26 Hosting Expression = cs_sold_date_sk#49 IN dynamicpruning#5
+
+Subquery:4 Hosting operator id = 30 Hosting Expression = cr_returned_date_sk#59 IN dynamicpruning#5
+
+Subquery:5 Hosting operator id = 47 Hosting Expression = ws_sold_date_sk#91 IN dynamicpruning#5
+
+Subquery:6 Hosting operator id = 51 Hosting Expression = wr_returned_date_sk#102 IN dynamicpruning#5
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/simplified.txt
new file mode 100644
index 0000000000000..0e98998fb31d5
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/simplified.txt
@@ -0,0 +1,155 @@
+TakeOrderedAndProject [channel,id,sales,returns,profit]
+  WholeStageCodegen (67)
+    HashAggregate [channel,id,sales,returns,profit]
+      InputAdapter
+        Exchange [channel,id,sales,returns,profit] #1
+          WholeStageCodegen (66)
+            HashAggregate [channel,id,sales,returns,profit]
+              InputAdapter
+                Union
+                  WholeStageCodegen (21)
+                    HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                      InputAdapter
+                        Exchange [channel,id] #2
+                          WholeStageCodegen (20)
+                            HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                              InputAdapter
+                                Union
+                                  WholeStageCodegen (6)
+                                    HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum]
+                                      InputAdapter
+                                        Exchange [s_store_id] #3
+                                          WholeStageCodegen (5)
+                                            HashAggregate [s_store_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum]
+                                              Project [sales_price,profit,return_amt,net_loss,s_store_id]
+                                                BroadcastHashJoin [store_sk,s_store_sk]
+                                                  Project [store_sk,sales_price,profit,return_amt,net_loss]
+                                                    BroadcastHashJoin [date_sk,d_date_sk]
+                                                      InputAdapter
+                                                        Union
+                                                          WholeStageCodegen (1)
+                                                            Project [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit]
+                                                              Filter [ss_store_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk]
+                                                                      SubqueryBroadcast [d_date_sk] #1
+                                                                        ReusedExchange [d_date_sk] #4
+                                                          WholeStageCodegen (2)
+                                                            Project [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss]
+                                                              Filter [sr_store_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk]
+                                                                      ReusedSubquery [d_date_sk] #1
+                                                      InputAdapter
+                                                        BroadcastExchange #4
+                                                          WholeStageCodegen (3)
+                                                            Project [d_date_sk]
+                                                              Filter [d_date,d_date_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.date_dim [d_date_sk,d_date]
+                                                  InputAdapter
+                                                    BroadcastExchange #5
+                                                      WholeStageCodegen (4)
+                                                        Filter [s_store_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.store [s_store_sk,s_store_id]
+                                  WholeStageCodegen (12)
+                                    HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum]
+                                      InputAdapter
+                                        Exchange [cp_catalog_page_id] #6
+                                          WholeStageCodegen (11)
+                                            HashAggregate [cp_catalog_page_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum]
+                                              Project [sales_price,profit,return_amt,net_loss,cp_catalog_page_id]
+                                                BroadcastHashJoin [page_sk,cp_catalog_page_sk]
+                                                  Project [page_sk,sales_price,profit,return_amt,net_loss]
+                                                    BroadcastHashJoin [date_sk,d_date_sk]
+                                                      InputAdapter
+                                                        Union
+                                                          WholeStageCodegen (7)
+                                                            Project [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit]
+                                                              Filter [cs_catalog_page_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk]
+                                                                      ReusedSubquery [d_date_sk] #1
+                                                          WholeStageCodegen (8)
+                                                            Project [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss]
+                                                              Filter [cr_catalog_page_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk]
+                                                                      ReusedSubquery [d_date_sk] #1
+                                                      InputAdapter
+                                                        ReusedExchange [d_date_sk] #4
+                                                  InputAdapter
+                                                    BroadcastExchange #7
+                                                      WholeStageCodegen (10)
+                                                        Filter [cp_catalog_page_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id]
+                                  WholeStageCodegen (19)
+                                    HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum]
+                                      InputAdapter
+                                        Exchange [web_site_id] #8
+                                          WholeStageCodegen (18)
+                                            HashAggregate [web_site_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum]
+                                              Project [sales_price,profit,return_amt,net_loss,web_site_id]
+                                                BroadcastHashJoin [wsr_web_site_sk,web_site_sk]
+                                                  Project [wsr_web_site_sk,sales_price,profit,return_amt,net_loss]
+                                                    BroadcastHashJoin [date_sk,d_date_sk]
+                                                      InputAdapter
+                                                        Union
+                                                          WholeStageCodegen (13)
+                                                            Project [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit]
+                                                              Filter [ws_web_site_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk]
+                                                                      ReusedSubquery [d_date_sk] #1
+                                                          WholeStageCodegen (15)
+                                                            Project [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss]
+                                                              BroadcastHashJoin [wr_item_sk,wr_order_number,ws_item_sk,ws_order_number]
+                                                                InputAdapter
+                                                                  BroadcastExchange #9
+                                                                    WholeStageCodegen (14)
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk]
+                                                                            ReusedSubquery [d_date_sk] #1
+                                                                Project [ws_item_sk,ws_web_site_sk,ws_order_number]
+                                                                  Filter [ws_item_sk,ws_order_number,ws_web_site_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk]
+                                                      InputAdapter
+                                                        ReusedExchange [d_date_sk] #4
+                                                  InputAdapter
+                                                    BroadcastExchange #10
+                                                      WholeStageCodegen (17)
+                                                        Filter [web_site_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.web_site [web_site_sk,web_site_id]
+                  WholeStageCodegen (43)
+                    HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                      InputAdapter
+                        Exchange [channel] #11
+                          WholeStageCodegen (42)
+                            HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                              HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                                InputAdapter
+                                  ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #12
+                  WholeStageCodegen (65)
+                    HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                      InputAdapter
+                        Exchange #13
+                          WholeStageCodegen (64)
+                            HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                              HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                                InputAdapter
+                                  ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #12
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.sf100/explain.txt
new file mode 100644
index 0000000000000..3f2b62a7b8fa4
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.sf100/explain.txt
@@ -0,0 +1,339 @@
+== Physical Plan ==
+TakeOrderedAndProject (50)
++- * Project (49)
+   +- * Filter (48)
+      +- * HashAggregate (47)
+         +- Exchange (46)
+            +- * HashAggregate (45)
+               +- * Project (44)
+                  +- * SortMergeJoin Inner (43)
+                     :- * Sort (28)
+                     :  +- Exchange (27)
+                     :     +- * Project (26)
+                     :        +- * BroadcastHashJoin Inner BuildRight (25)
+                     :           :- * Project (19)
+                     :           :  +- * BroadcastHashJoin Inner BuildRight (18)
+                     :           :     :- * Filter (3)
+                     :           :     :  +- * ColumnarToRow (2)
+                     :           :     :     +- Scan parquet default.store_sales (1)
+                     :           :     +- BroadcastExchange (17)
+                     :           :        +- * Project (16)
+                     :           :           +- * Filter (15)
+                     :           :              +- * BroadcastHashJoin LeftOuter BuildRight (14)
+                     :           :                 :- * Filter (6)
+                     :           :                 :  +- * ColumnarToRow (5)
+                     :           :                 :     +- Scan parquet default.item (4)
+                     :           :                 +- BroadcastExchange (13)
+                     :           :                    +- * HashAggregate (12)
+                     :           :                       +- Exchange (11)
+                     :           :                          +- * HashAggregate (10)
+                     :           :                             +- * Filter (9)
+                     :           :                                +- * ColumnarToRow (8)
+                     :           :                                   +- Scan parquet default.item (7)
+                     :           +- BroadcastExchange (24)
+                     :              +- * Project (23)
+                     :                 +- * Filter (22)
+                     :                    +- * ColumnarToRow (21)
+                     :                       +- Scan parquet default.date_dim (20)
+                     +- * Sort (42)
+                        +- Exchange (41)
+                           +- * Project (40)
+                              +- * SortMergeJoin Inner (39)
+                                 :- * Sort (33)
+                                 :  +- Exchange (32)
+                                 :     +- * Filter (31)
+                                 :        +- * ColumnarToRow (30)
+                                 :           +- Scan parquet default.customer_address (29)
+                                 +- * Sort (38)
+                                    +- Exchange (37)
+                                       +- * Filter (36)
+                                          +- * ColumnarToRow (35)
+                                             +- Scan parquet default.customer (34)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int>
+
+(2) ColumnarToRow [codegen id : 5]
+Input [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 5]
+Input [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3]
+Condition : (isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1))
+
+(4) Scan parquet default.item
+Output [3]: [i_item_sk#5, i_current_price#6, i_category#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_current_price:decimal(7,2),i_category:string>
+
+(5) ColumnarToRow [codegen id : 3]
+Input [3]: [i_item_sk#5, i_current_price#6, i_category#7]
+
+(6) Filter [codegen id : 3]
+Input [3]: [i_item_sk#5, i_current_price#6, i_category#7]
+Condition : (isnotnull(i_current_price#6) AND isnotnull(i_item_sk#5))
+
+(7) Scan parquet default.item
+Output [2]: [i_current_price#8, i_category#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_category)]
+ReadSchema: struct<i_current_price:decimal(7,2),i_category:string>
+
+(8) ColumnarToRow [codegen id : 1]
+Input [2]: [i_current_price#8, i_category#9]
+
+(9) Filter [codegen id : 1]
+Input [2]: [i_current_price#8, i_category#9]
+Condition : isnotnull(i_category#9)
+
+(10) HashAggregate [codegen id : 1]
+Input [2]: [i_current_price#8, i_category#9]
+Keys [1]: [i_category#9]
+Functions [1]: [partial_avg(UnscaledValue(i_current_price#8))]
+Aggregate Attributes [2]: [sum#10, count#11]
+Results [3]: [i_category#9, sum#12, count#13]
+
+(11) Exchange
+Input [3]: [i_category#9, sum#12, count#13]
+Arguments: hashpartitioning(i_category#9, 5), ENSURE_REQUIREMENTS, [id=#14]
+
+(12) HashAggregate [codegen id : 2]
+Input [3]: [i_category#9, sum#12, count#13]
+Keys [1]: [i_category#9]
+Functions [1]: [avg(UnscaledValue(i_current_price#8))]
+Aggregate Attributes [1]: [avg(UnscaledValue(i_current_price#8))#15]
+Results [2]: [cast((avg(UnscaledValue(i_current_price#8))#15 / 100.0) as decimal(11,6)) AS avg(i_current_price)#16, i_category#9]
+
+(13) BroadcastExchange
+Input [2]: [avg(i_current_price)#16, i_category#9]
+Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [id=#17]
+
+(14) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [i_category#7]
+Right keys [1]: [i_category#9]
+Join condition: None
+
+(15) Filter [codegen id : 3]
+Input [5]: [i_item_sk#5, i_current_price#6, i_category#7, avg(i_current_price)#16, i_category#9]
+Condition : (cast(i_current_price#6 as decimal(14,7)) > CheckOverflow((1.200000 * promote_precision(avg(i_current_price)#16)), DecimalType(14,7), true))
+
+(16) Project [codegen id : 3]
+Output [1]: [i_item_sk#5]
+Input [5]: [i_item_sk#5, i_current_price#6, i_category#7, avg(i_current_price)#16, i_category#9]
+
+(17) BroadcastExchange
+Input [1]: [i_item_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(18) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#5]
+Join condition: None
+
+(19) Project [codegen id : 5]
+Output [2]: [ss_customer_sk#2, ss_sold_date_sk#3]
+Input [4]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3, i_item_sk#5]
+
+(20) Scan parquet default.date_dim
+Output [2]: [d_date_sk#19, d_month_seq#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(21) ColumnarToRow [codegen id : 4]
+Input [2]: [d_date_sk#19, d_month_seq#20]
+
+(22) Filter [codegen id : 4]
+Input [2]: [d_date_sk#19, d_month_seq#20]
+Condition : ((isnotnull(d_month_seq#20) AND (d_month_seq#20 = Subquery scalar-subquery#21, [id=#22])) AND isnotnull(d_date_sk#19))
+
+(23) Project [codegen id : 4]
+Output [1]: [d_date_sk#19]
+Input [2]: [d_date_sk#19, d_month_seq#20]
+
+(24) BroadcastExchange
+Input [1]: [d_date_sk#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#23]
+
+(25) BroadcastHashJoin [codegen id : 5]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#19]
+Join condition: None
+
+(26) Project [codegen id : 5]
+Output [1]: [ss_customer_sk#2]
+Input [3]: [ss_customer_sk#2, ss_sold_date_sk#3, d_date_sk#19]
+
+(27) Exchange
+Input [1]: [ss_customer_sk#2]
+Arguments: hashpartitioning(ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(28) Sort [codegen id : 6]
+Input [1]: [ss_customer_sk#2]
+Arguments: [ss_customer_sk#2 ASC NULLS FIRST], false, 0
+
+(29) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#25, ca_state#26]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(30) ColumnarToRow [codegen id : 7]
+Input [2]: [ca_address_sk#25, ca_state#26]
+
+(31) Filter [codegen id : 7]
+Input [2]: [ca_address_sk#25, ca_state#26]
+Condition : isnotnull(ca_address_sk#25)
+
+(32) Exchange
+Input [2]: [ca_address_sk#25, ca_state#26]
+Arguments: hashpartitioning(ca_address_sk#25, 5), ENSURE_REQUIREMENTS, [id=#27]
+
+(33) Sort [codegen id : 8]
+Input [2]: [ca_address_sk#25, ca_state#26]
+Arguments: [ca_address_sk#25 ASC NULLS FIRST], false, 0
+
+(34) Scan parquet default.customer
+Output [2]: [c_customer_sk#28, c_current_addr_sk#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int>
+
+(35) ColumnarToRow [codegen id : 9]
+Input [2]: [c_customer_sk#28, c_current_addr_sk#29]
+
+(36) Filter [codegen id : 9]
+Input [2]: [c_customer_sk#28, c_current_addr_sk#29]
+Condition : (isnotnull(c_current_addr_sk#29) AND isnotnull(c_customer_sk#28))
+
+(37) Exchange
+Input [2]: [c_customer_sk#28, c_current_addr_sk#29]
+Arguments: hashpartitioning(c_current_addr_sk#29, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(38) Sort [codegen id : 10]
+Input [2]: [c_customer_sk#28, c_current_addr_sk#29]
+Arguments: [c_current_addr_sk#29 ASC NULLS FIRST], false, 0
+
+(39) SortMergeJoin [codegen id : 11]
+Left keys [1]: [ca_address_sk#25]
+Right keys [1]: [c_current_addr_sk#29]
+Join condition: None
+
+(40) Project [codegen id : 11]
+Output [2]: [ca_state#26, c_customer_sk#28]
+Input [4]: [ca_address_sk#25, ca_state#26, c_customer_sk#28, c_current_addr_sk#29]
+
+(41) Exchange
+Input [2]: [ca_state#26, c_customer_sk#28]
+Arguments: hashpartitioning(c_customer_sk#28, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(42) Sort [codegen id : 12]
+Input [2]: [ca_state#26, c_customer_sk#28]
+Arguments: [c_customer_sk#28 ASC NULLS FIRST], false, 0
+
+(43) SortMergeJoin [codegen id : 13]
+Left keys [1]: [ss_customer_sk#2]
+Right keys [1]: [c_customer_sk#28]
+Join condition: None
+
+(44) Project [codegen id : 13]
+Output [1]: [ca_state#26]
+Input [3]: [ss_customer_sk#2, ca_state#26, c_customer_sk#28]
+
+(45) HashAggregate [codegen id : 13]
+Input [1]: [ca_state#26]
+Keys [1]: [ca_state#26]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#32]
+Results [2]: [ca_state#26, count#33]
+
+(46) Exchange
+Input [2]: [ca_state#26, count#33]
+Arguments: hashpartitioning(ca_state#26, 5), ENSURE_REQUIREMENTS, [id=#34]
+
+(47) HashAggregate [codegen id : 14]
+Input [2]: [ca_state#26, count#33]
+Keys [1]: [ca_state#26]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#35]
+Results [4]: [ca_state#26 AS state#36, count(1)#35 AS cnt#37, count(1)#35 AS count(1)#38, ca_state#26]
+
+(48) Filter [codegen id : 14]
+Input [4]: [state#36, cnt#37, count(1)#38, ca_state#26]
+Condition : (count(1)#38 >= 10)
+
+(49) Project [codegen id : 14]
+Output [3]: [state#36, cnt#37, ca_state#26]
+Input [4]: [state#36, cnt#37, count(1)#38, ca_state#26]
+
+(50) TakeOrderedAndProject
+Input [3]: [state#36, cnt#37, ca_state#26]
+Arguments: 100, [cnt#37 ASC NULLS FIRST, ca_state#26 ASC NULLS FIRST], [state#36, cnt#37]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (51)
+
+
+(51) ReusedExchange [Reuses operator id: 24]
+Output [1]: [d_date_sk#19]
+
+Subquery:2 Hosting operator id = 22 Hosting Expression = Subquery scalar-subquery#21, [id=#22]
+* HashAggregate (58)
++- Exchange (57)
+   +- * HashAggregate (56)
+      +- * Project (55)
+         +- * Filter (54)
+            +- * ColumnarToRow (53)
+               +- Scan parquet default.date_dim (52)
+
+
+(52) Scan parquet default.date_dim
+Output [3]: [d_month_seq#39, d_year#40, d_moy#41]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)]
+ReadSchema: struct<d_month_seq:int,d_year:int,d_moy:int>
+
+(53) ColumnarToRow [codegen id : 1]
+Input [3]: [d_month_seq#39, d_year#40, d_moy#41]
+
+(54) Filter [codegen id : 1]
+Input [3]: [d_month_seq#39, d_year#40, d_moy#41]
+Condition : (((isnotnull(d_year#40) AND isnotnull(d_moy#41)) AND (d_year#40 = 2000)) AND (d_moy#41 = 1))
+
+(55) Project [codegen id : 1]
+Output [1]: [d_month_seq#39]
+Input [3]: [d_month_seq#39, d_year#40, d_moy#41]
+
+(56) HashAggregate [codegen id : 1]
+Input [1]: [d_month_seq#39]
+Keys [1]: [d_month_seq#39]
+Functions: []
+Aggregate Attributes: []
+Results [1]: [d_month_seq#39]
+
+(57) Exchange
+Input [1]: [d_month_seq#39]
+Arguments: hashpartitioning(d_month_seq#39, 5), ENSURE_REQUIREMENTS, [id=#42]
+
+(58) HashAggregate [codegen id : 2]
+Input [1]: [d_month_seq#39]
+Keys [1]: [d_month_seq#39]
+Functions: []
+Aggregate Attributes: []
+Results [1]: [d_month_seq#39]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.sf100/simplified.txt
new file mode 100644
index 0000000000000..f3badf6efe6b6
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.sf100/simplified.txt
@@ -0,0 +1,97 @@
+TakeOrderedAndProject [cnt,ca_state,state]
+  WholeStageCodegen (14)
+    Project [state,cnt,ca_state]
+      Filter [count(1)]
+        HashAggregate [ca_state,count] [count(1),state,cnt,count(1),count]
+          InputAdapter
+            Exchange [ca_state] #1
+              WholeStageCodegen (13)
+                HashAggregate [ca_state] [count,count]
+                  Project [ca_state]
+                    SortMergeJoin [ss_customer_sk,c_customer_sk]
+                      InputAdapter
+                        WholeStageCodegen (6)
+                          Sort [ss_customer_sk]
+                            InputAdapter
+                              Exchange [ss_customer_sk] #2
+                                WholeStageCodegen (5)
+                                  Project [ss_customer_sk]
+                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                      Project [ss_customer_sk,ss_sold_date_sk]
+                                        BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                          Filter [ss_customer_sk,ss_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk]
+                                                  SubqueryBroadcast [d_date_sk] #1
+                                                    ReusedExchange [d_date_sk] #3
+                                          InputAdapter
+                                            BroadcastExchange #4
+                                              WholeStageCodegen (3)
+                                                Project [i_item_sk]
+                                                  Filter [i_current_price,avg(i_current_price)]
+                                                    BroadcastHashJoin [i_category,i_category]
+                                                      Filter [i_current_price,i_item_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.item [i_item_sk,i_current_price,i_category]
+                                                      InputAdapter
+                                                        BroadcastExchange #5
+                                                          WholeStageCodegen (2)
+                                                            HashAggregate [i_category,sum,count] [avg(UnscaledValue(i_current_price)),avg(i_current_price),sum,count]
+                                                              InputAdapter
+                                                                Exchange [i_category] #6
+                                                                  WholeStageCodegen (1)
+                                                                    HashAggregate [i_category,i_current_price] [sum,count,sum,count]
+                                                                      Filter [i_category]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.item [i_current_price,i_category]
+                                      InputAdapter
+                                        BroadcastExchange #3
+                                          WholeStageCodegen (4)
+                                            Project [d_date_sk]
+                                              Filter [d_month_seq,d_date_sk]
+                                                Subquery #2
+                                                  WholeStageCodegen (2)
+                                                    HashAggregate [d_month_seq]
+                                                      InputAdapter
+                                                        Exchange [d_month_seq] #7
+                                                          WholeStageCodegen (1)
+                                                            HashAggregate [d_month_seq]
+                                                              Project [d_month_seq]
+                                                                Filter [d_year,d_moy]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.date_dim [d_month_seq,d_year,d_moy]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.date_dim [d_date_sk,d_month_seq]
+                      InputAdapter
+                        WholeStageCodegen (12)
+                          Sort [c_customer_sk]
+                            InputAdapter
+                              Exchange [c_customer_sk] #8
+                                WholeStageCodegen (11)
+                                  Project [ca_state,c_customer_sk]
+                                    SortMergeJoin [ca_address_sk,c_current_addr_sk]
+                                      InputAdapter
+                                        WholeStageCodegen (8)
+                                          Sort [ca_address_sk]
+                                            InputAdapter
+                                              Exchange [ca_address_sk] #9
+                                                WholeStageCodegen (7)
+                                                  Filter [ca_address_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.customer_address [ca_address_sk,ca_state]
+                                      InputAdapter
+                                        WholeStageCodegen (10)
+                                          Sort [c_current_addr_sk]
+                                            InputAdapter
+                                              Exchange [c_current_addr_sk] #10
+                                                WholeStageCodegen (9)
+                                                  Filter [c_current_addr_sk,c_customer_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.customer [c_customer_sk,c_current_addr_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/explain.txt
new file mode 100644
index 0000000000000..b37db85388e0f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/explain.txt
@@ -0,0 +1,309 @@
+== Physical Plan ==
+TakeOrderedAndProject (44)
++- * Project (43)
+   +- * Filter (42)
+      +- * HashAggregate (41)
+         +- Exchange (40)
+            +- * HashAggregate (39)
+               +- * Project (38)
+                  +- * BroadcastHashJoin Inner BuildRight (37)
+                     :- * Project (22)
+                     :  +- * BroadcastHashJoin Inner BuildRight (21)
+                     :     :- * Project (15)
+                     :     :  +- * BroadcastHashJoin Inner BuildRight (14)
+                     :     :     :- * Project (9)
+                     :     :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+                     :     :     :     :- * Filter (3)
+                     :     :     :     :  +- * ColumnarToRow (2)
+                     :     :     :     :     +- Scan parquet default.customer_address (1)
+                     :     :     :     +- BroadcastExchange (7)
+                     :     :     :        +- * Filter (6)
+                     :     :     :           +- * ColumnarToRow (5)
+                     :     :     :              +- Scan parquet default.customer (4)
+                     :     :     +- BroadcastExchange (13)
+                     :     :        +- * Filter (12)
+                     :     :           +- * ColumnarToRow (11)
+                     :     :              +- Scan parquet default.store_sales (10)
+                     :     +- BroadcastExchange (20)
+                     :        +- * Project (19)
+                     :           +- * Filter (18)
+                     :              +- * ColumnarToRow (17)
+                     :                 +- Scan parquet default.date_dim (16)
+                     +- BroadcastExchange (36)
+                        +- * Project (35)
+                           +- * Filter (34)
+                              +- * BroadcastHashJoin LeftOuter BuildRight (33)
+                                 :- * Filter (25)
+                                 :  +- * ColumnarToRow (24)
+                                 :     +- Scan parquet default.item (23)
+                                 +- BroadcastExchange (32)
+                                    +- * HashAggregate (31)
+                                       +- Exchange (30)
+                                          +- * HashAggregate (29)
+                                             +- * Filter (28)
+                                                +- * ColumnarToRow (27)
+                                                   +- Scan parquet default.item (26)
+
+
+(1) Scan parquet default.customer_address
+Output [2]: [ca_address_sk#1, ca_state#2]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_state:string>
+
+(2) ColumnarToRow [codegen id : 7]
+Input [2]: [ca_address_sk#1, ca_state#2]
+
+(3) Filter [codegen id : 7]
+Input [2]: [ca_address_sk#1, ca_state#2]
+Condition : isnotnull(ca_address_sk#1)
+
+(4) Scan parquet default.customer
+Output [2]: [c_customer_sk#3, c_current_addr_sk#4]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_customer_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [c_customer_sk#3, c_current_addr_sk#4]
+
+(6) Filter [codegen id : 1]
+Input [2]: [c_customer_sk#3, c_current_addr_sk#4]
+Condition : (isnotnull(c_current_addr_sk#4) AND isnotnull(c_customer_sk#3))
+
+(7) BroadcastExchange
+Input [2]: [c_customer_sk#3, c_current_addr_sk#4]
+Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [id=#5]
+
+(8) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [ca_address_sk#1]
+Right keys [1]: [c_current_addr_sk#4]
+Join condition: None
+
+(9) Project [codegen id : 7]
+Output [2]: [ca_state#2, c_customer_sk#3]
+Input [4]: [ca_address_sk#1, ca_state#2, c_customer_sk#3, c_current_addr_sk#4]
+
+(10) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8]
+
+(12) Filter [codegen id : 2]
+Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8]
+Condition : (isnotnull(ss_customer_sk#7) AND isnotnull(ss_item_sk#6))
+
+(13) BroadcastExchange
+Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8]
+Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [id=#10]
+
+(14) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [c_customer_sk#3]
+Right keys [1]: [ss_customer_sk#7]
+Join condition: None
+
+(15) Project [codegen id : 7]
+Output [3]: [ca_state#2, ss_item_sk#6, ss_sold_date_sk#8]
+Input [5]: [ca_state#2, c_customer_sk#3, ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8]
+
+(16) Scan parquet default.date_dim
+Output [2]: [d_date_sk#11, d_month_seq#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(17) ColumnarToRow [codegen id : 3]
+Input [2]: [d_date_sk#11, d_month_seq#12]
+
+(18) Filter [codegen id : 3]
+Input [2]: [d_date_sk#11, d_month_seq#12]
+Condition : ((isnotnull(d_month_seq#12) AND (d_month_seq#12 = Subquery scalar-subquery#13, [id=#14])) AND isnotnull(d_date_sk#11))
+
+(19) Project [codegen id : 3]
+Output [1]: [d_date_sk#11]
+Input [2]: [d_date_sk#11, d_month_seq#12]
+
+(20) BroadcastExchange
+Input [1]: [d_date_sk#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#15]
+
+(21) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [ss_sold_date_sk#8]
+Right keys [1]: [d_date_sk#11]
+Join condition: None
+
+(22) Project [codegen id : 7]
+Output [2]: [ca_state#2, ss_item_sk#6]
+Input [4]: [ca_state#2, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#11]
+
+(23) Scan parquet default.item
+Output [3]: [i_item_sk#16, i_current_price#17, i_category#18]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_current_price:decimal(7,2),i_category:string>
+
+(24) ColumnarToRow [codegen id : 6]
+Input [3]: [i_item_sk#16, i_current_price#17, i_category#18]
+
+(25) Filter [codegen id : 6]
+Input [3]: [i_item_sk#16, i_current_price#17, i_category#18]
+Condition : (isnotnull(i_current_price#17) AND isnotnull(i_item_sk#16))
+
+(26) Scan parquet default.item
+Output [2]: [i_current_price#19, i_category#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_category)]
+ReadSchema: struct<i_current_price:decimal(7,2),i_category:string>
+
+(27) ColumnarToRow [codegen id : 4]
+Input [2]: [i_current_price#19, i_category#20]
+
+(28) Filter [codegen id : 4]
+Input [2]: [i_current_price#19, i_category#20]
+Condition : isnotnull(i_category#20)
+
+(29) HashAggregate [codegen id : 4]
+Input [2]: [i_current_price#19, i_category#20]
+Keys [1]: [i_category#20]
+Functions [1]: [partial_avg(UnscaledValue(i_current_price#19))]
+Aggregate Attributes [2]: [sum#21, count#22]
+Results [3]: [i_category#20, sum#23, count#24]
+
+(30) Exchange
+Input [3]: [i_category#20, sum#23, count#24]
+Arguments: hashpartitioning(i_category#20, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(31) HashAggregate [codegen id : 5]
+Input [3]: [i_category#20, sum#23, count#24]
+Keys [1]: [i_category#20]
+Functions [1]: [avg(UnscaledValue(i_current_price#19))]
+Aggregate Attributes [1]: [avg(UnscaledValue(i_current_price#19))#26]
+Results [2]: [cast((avg(UnscaledValue(i_current_price#19))#26 / 100.0) as decimal(11,6)) AS avg(i_current_price)#27, i_category#20]
+
+(32) BroadcastExchange
+Input [2]: [avg(i_current_price)#27, i_category#20]
+Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [id=#28]
+
+(33) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [i_category#18]
+Right keys [1]: [i_category#20]
+Join condition: None
+
+(34) Filter [codegen id : 6]
+Input [5]: [i_item_sk#16, i_current_price#17, i_category#18, avg(i_current_price)#27, i_category#20]
+Condition : (cast(i_current_price#17 as decimal(14,7)) > CheckOverflow((1.200000 * promote_precision(avg(i_current_price)#27)), DecimalType(14,7), true))
+
+(35) Project [codegen id : 6]
+Output [1]: [i_item_sk#16]
+Input [5]: [i_item_sk#16, i_current_price#17, i_category#18, avg(i_current_price)#27, i_category#20]
+
+(36) BroadcastExchange
+Input [1]: [i_item_sk#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#29]
+
+(37) BroadcastHashJoin [codegen id : 7]
+Left keys [1]: [ss_item_sk#6]
+Right keys [1]: [i_item_sk#16]
+Join condition: None
+
+(38) Project [codegen id : 7]
+Output [1]: [ca_state#2]
+Input [3]: [ca_state#2, ss_item_sk#6, i_item_sk#16]
+
+(39) HashAggregate [codegen id : 7]
+Input [1]: [ca_state#2]
+Keys [1]: [ca_state#2]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#30]
+Results [2]: [ca_state#2, count#31]
+
+(40) Exchange
+Input [2]: [ca_state#2, count#31]
+Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, [id=#32]
+
+(41) HashAggregate [codegen id : 8]
+Input [2]: [ca_state#2, count#31]
+Keys [1]: [ca_state#2]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#33]
+Results [4]: [ca_state#2 AS state#34, count(1)#33 AS cnt#35, count(1)#33 AS count(1)#36, ca_state#2]
+
+(42) Filter [codegen id : 8]
+Input [4]: [state#34, cnt#35, count(1)#36, ca_state#2]
+Condition : (count(1)#36 >= 10)
+
+(43) Project [codegen id : 8]
+Output [3]: [state#34, cnt#35, ca_state#2]
+Input [4]: [state#34, cnt#35, count(1)#36, ca_state#2]
+
+(44) TakeOrderedAndProject
+Input [3]: [state#34, cnt#35, ca_state#2]
+Arguments: 100, [cnt#35 ASC NULLS FIRST, ca_state#2 ASC NULLS FIRST], [state#34, cnt#35]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 10 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (45)
+
+
+(45) ReusedExchange [Reuses operator id: 20]
+Output [1]: [d_date_sk#11]
+
+Subquery:2 Hosting operator id = 18 Hosting Expression = Subquery scalar-subquery#13, [id=#14]
+* HashAggregate (52)
++- Exchange (51)
+   +- * HashAggregate (50)
+      +- * Project (49)
+         +- * Filter (48)
+            +- * ColumnarToRow (47)
+               +- Scan parquet default.date_dim (46)
+
+
+(46) Scan parquet default.date_dim
+Output [3]: [d_month_seq#37, d_year#38, d_moy#39]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)]
+ReadSchema: struct<d_month_seq:int,d_year:int,d_moy:int>
+
+(47) ColumnarToRow [codegen id : 1]
+Input [3]: [d_month_seq#37, d_year#38, d_moy#39]
+
+(48) Filter [codegen id : 1]
+Input [3]: [d_month_seq#37, d_year#38, d_moy#39]
+Condition : (((isnotnull(d_year#38) AND isnotnull(d_moy#39)) AND (d_year#38 = 2000)) AND (d_moy#39 = 1))
+
+(49) Project [codegen id : 1]
+Output [1]: [d_month_seq#37]
+Input [3]: [d_month_seq#37, d_year#38, d_moy#39]
+
+(50) HashAggregate [codegen id : 1]
+Input [1]: [d_month_seq#37]
+Keys [1]: [d_month_seq#37]
+Functions: []
+Aggregate Attributes: []
+Results [1]: [d_month_seq#37]
+
+(51) Exchange
+Input [1]: [d_month_seq#37]
+Arguments: hashpartitioning(d_month_seq#37, 5), ENSURE_REQUIREMENTS, [id=#40]
+
+(52) HashAggregate [codegen id : 2]
+Input [1]: [d_month_seq#37]
+Keys [1]: [d_month_seq#37]
+Functions: []
+Aggregate Attributes: []
+Results [1]: [d_month_seq#37]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/simplified.txt
new file mode 100644
index 0000000000000..4ba09283e73cd
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/simplified.txt
@@ -0,0 +1,79 @@
+TakeOrderedAndProject [cnt,ca_state,state]
+  WholeStageCodegen (8)
+    Project [state,cnt,ca_state]
+      Filter [count(1)]
+        HashAggregate [ca_state,count] [count(1),state,cnt,count(1),count]
+          InputAdapter
+            Exchange [ca_state] #1
+              WholeStageCodegen (7)
+                HashAggregate [ca_state] [count,count]
+                  Project [ca_state]
+                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                      Project [ca_state,ss_item_sk]
+                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                          Project [ca_state,ss_item_sk,ss_sold_date_sk]
+                            BroadcastHashJoin [c_customer_sk,ss_customer_sk]
+                              Project [ca_state,c_customer_sk]
+                                BroadcastHashJoin [ca_address_sk,c_current_addr_sk]
+                                  Filter [ca_address_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.customer_address [ca_address_sk,ca_state]
+                                  InputAdapter
+                                    BroadcastExchange #2
+                                      WholeStageCodegen (1)
+                                        Filter [c_current_addr_sk,c_customer_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer [c_customer_sk,c_current_addr_sk]
+                              InputAdapter
+                                BroadcastExchange #3
+                                  WholeStageCodegen (2)
+                                    Filter [ss_customer_sk,ss_item_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk]
+                                            SubqueryBroadcast [d_date_sk] #1
+                                              ReusedExchange [d_date_sk] #4
+                          InputAdapter
+                            BroadcastExchange #4
+                              WholeStageCodegen (3)
+                                Project [d_date_sk]
+                                  Filter [d_month_seq,d_date_sk]
+                                    Subquery #2
+                                      WholeStageCodegen (2)
+                                        HashAggregate [d_month_seq]
+                                          InputAdapter
+                                            Exchange [d_month_seq] #5
+                                              WholeStageCodegen (1)
+                                                HashAggregate [d_month_seq]
+                                                  Project [d_month_seq]
+                                                    Filter [d_year,d_moy]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.date_dim [d_month_seq,d_year,d_moy]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.date_dim [d_date_sk,d_month_seq]
+                      InputAdapter
+                        BroadcastExchange #6
+                          WholeStageCodegen (6)
+                            Project [i_item_sk]
+                              Filter [i_current_price,avg(i_current_price)]
+                                BroadcastHashJoin [i_category,i_category]
+                                  Filter [i_current_price,i_item_sk]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.item [i_item_sk,i_current_price,i_category]
+                                  InputAdapter
+                                    BroadcastExchange #7
+                                      WholeStageCodegen (5)
+                                        HashAggregate [i_category,sum,count] [avg(UnscaledValue(i_current_price)),avg(i_current_price),sum,count]
+                                          InputAdapter
+                                            Exchange [i_category] #8
+                                              WholeStageCodegen (4)
+                                                HashAggregate [i_category,i_current_price] [sum,count,sum,count]
+                                                  Filter [i_category]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.item [i_current_price,i_category]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.sf100/explain.txt
new file mode 100644
index 0000000000000..ce584592dd831
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.sf100/explain.txt
@@ -0,0 +1,1162 @@
+== Physical Plan ==
+* Sort (215)
++- Exchange (214)
+   +- * Project (213)
+      +- * SortMergeJoin Inner (212)
+         :- * Sort (131)
+         :  +- Exchange (130)
+         :     +- * HashAggregate (129)
+         :        +- Exchange (128)
+         :           +- * HashAggregate (127)
+         :              +- * Project (126)
+         :                 +- * BroadcastHashJoin Inner BuildRight (125)
+         :                    :- * Project (119)
+         :                    :  +- * BroadcastHashJoin Inner BuildRight (118)
+         :                    :     :- * Project (116)
+         :                    :     :  +- * BroadcastHashJoin Inner BuildRight (115)
+         :                    :     :     :- * Project (110)
+         :                    :     :     :  +- * SortMergeJoin Inner (109)
+         :                    :     :     :     :- * Sort (106)
+         :                    :     :     :     :  +- Exchange (105)
+         :                    :     :     :     :     +- * Project (104)
+         :                    :     :     :     :        +- * SortMergeJoin Inner (103)
+         :                    :     :     :     :           :- * Sort (97)
+         :                    :     :     :     :           :  +- Exchange (96)
+         :                    :     :     :     :           :     +- * Project (95)
+         :                    :     :     :     :           :        +- * BroadcastHashJoin Inner BuildRight (94)
+         :                    :     :     :     :           :           :- * Project (92)
+         :                    :     :     :     :           :           :  +- * BroadcastHashJoin Inner BuildRight (91)
+         :                    :     :     :     :           :           :     :- * Project (86)
+         :                    :     :     :     :           :           :     :  +- * BroadcastHashJoin Inner BuildRight (85)
+         :                    :     :     :     :           :           :     :     :- * Project (80)
+         :                    :     :     :     :           :           :     :     :  +- * SortMergeJoin Inner (79)
+         :                    :     :     :     :           :           :     :     :     :- * Sort (76)
+         :                    :     :     :     :           :           :     :     :     :  +- Exchange (75)
+         :                    :     :     :     :           :           :     :     :     :     +- * Project (74)
+         :                    :     :     :     :           :           :     :     :     :        +- * SortMergeJoin Inner (73)
+         :                    :     :     :     :           :           :     :     :     :           :- * Sort (67)
+         :                    :     :     :     :           :           :     :     :     :           :  +- Exchange (66)
+         :                    :     :     :     :           :           :     :     :     :           :     +- * Project (65)
+         :                    :     :     :     :           :           :     :     :     :           :        +- * BroadcastHashJoin Inner BuildRight (64)
+         :                    :     :     :     :           :           :     :     :     :           :           :- * Project (62)
+         :                    :     :     :     :           :           :     :     :     :           :           :  +- * BroadcastHashJoin Inner BuildRight (61)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :- * Project (56)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :  +- * SortMergeJoin Inner (55)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :- * Sort (49)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :  +- Exchange (48)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :     +- * Project (47)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :        +- * BroadcastHashJoin Inner BuildRight (46)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :- * Project (41)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :  +- * BroadcastHashJoin Inner BuildRight (40)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :- * Project (35)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :  +- * BroadcastHashJoin Inner BuildRight (34)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :- * Project (13)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :  +- * SortMergeJoin Inner (12)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :     :- * Sort (5)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :     :  +- Exchange (4)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :     :     +- * Filter (3)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :     :        +- * ColumnarToRow (2)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :     :           +- Scan parquet default.store_sales (1)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :     +- * Sort (11)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :        +- Exchange (10)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :           +- * Project (9)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :              +- * Filter (8)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :                 +- * ColumnarToRow (7)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :                    +- Scan parquet default.store_returns (6)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     +- BroadcastExchange (33)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :        +- * Project (32)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :           +- * Filter (31)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :              +- * HashAggregate (30)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                 +- Exchange (29)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                    +- * HashAggregate (28)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                       +- * Project (27)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                          +- * SortMergeJoin Inner (26)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                             :- * Sort (19)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                             :  +- Exchange (18)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                             :     +- * Project (17)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                             :        +- * Filter (16)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                             :           +- * ColumnarToRow (15)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                             :              +- Scan parquet default.catalog_sales (14)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                             +- * Sort (25)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                                +- Exchange (24)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                                   +- * Project (23)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                                      +- * Filter (22)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                                         +- * ColumnarToRow (21)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     :                                            +- Scan parquet default.catalog_returns (20)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :     +- BroadcastExchange (39)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :        +- * Filter (38)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :           +- * ColumnarToRow (37)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           :              +- Scan parquet default.date_dim (36)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :           +- BroadcastExchange (45)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :              +- * Filter (44)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :                 +- * ColumnarToRow (43)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     :                    +- Scan parquet default.store (42)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :     +- * Sort (54)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :        +- Exchange (53)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :           +- * Filter (52)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :              +- * ColumnarToRow (51)
+         :                    :     :     :     :           :           :     :     :     :           :           :     :                 +- Scan parquet default.customer (50)
+         :                    :     :     :     :           :           :     :     :     :           :           :     +- BroadcastExchange (60)
+         :                    :     :     :     :           :           :     :     :     :           :           :        +- * Filter (59)
+         :                    :     :     :     :           :           :     :     :     :           :           :           +- * ColumnarToRow (58)
+         :                    :     :     :     :           :           :     :     :     :           :           :              +- Scan parquet default.date_dim (57)
+         :                    :     :     :     :           :           :     :     :     :           :           +- ReusedExchange (63)
+         :                    :     :     :     :           :           :     :     :     :           +- * Sort (72)
+         :                    :     :     :     :           :           :     :     :     :              +- Exchange (71)
+         :                    :     :     :     :           :           :     :     :     :                 +- * Filter (70)
+         :                    :     :     :     :           :           :     :     :     :                    +- * ColumnarToRow (69)
+         :                    :     :     :     :           :           :     :     :     :                       +- Scan parquet default.customer_demographics (68)
+         :                    :     :     :     :           :           :     :     :     +- * Sort (78)
+         :                    :     :     :     :           :           :     :     :        +- ReusedExchange (77)
+         :                    :     :     :     :           :           :     :     +- BroadcastExchange (84)
+         :                    :     :     :     :           :           :     :        +- * Filter (83)
+         :                    :     :     :     :           :           :     :           +- * ColumnarToRow (82)
+         :                    :     :     :     :           :           :     :              +- Scan parquet default.promotion (81)
+         :                    :     :     :     :           :           :     +- BroadcastExchange (90)
+         :                    :     :     :     :           :           :        +- * Filter (89)
+         :                    :     :     :     :           :           :           +- * ColumnarToRow (88)
+         :                    :     :     :     :           :           :              +- Scan parquet default.household_demographics (87)
+         :                    :     :     :     :           :           +- ReusedExchange (93)
+         :                    :     :     :     :           +- * Sort (102)
+         :                    :     :     :     :              +- Exchange (101)
+         :                    :     :     :     :                 +- * Filter (100)
+         :                    :     :     :     :                    +- * ColumnarToRow (99)
+         :                    :     :     :     :                       +- Scan parquet default.customer_address (98)
+         :                    :     :     :     +- * Sort (108)
+         :                    :     :     :        +- ReusedExchange (107)
+         :                    :     :     +- BroadcastExchange (114)
+         :                    :     :        +- * Filter (113)
+         :                    :     :           +- * ColumnarToRow (112)
+         :                    :     :              +- Scan parquet default.income_band (111)
+         :                    :     +- ReusedExchange (117)
+         :                    +- BroadcastExchange (124)
+         :                       +- * Project (123)
+         :                          +- * Filter (122)
+         :                             +- * ColumnarToRow (121)
+         :                                +- Scan parquet default.item (120)
+         +- * Sort (211)
+            +- Exchange (210)
+               +- * HashAggregate (209)
+                  +- Exchange (208)
+                     +- * HashAggregate (207)
+                        +- * Project (206)
+                           +- * BroadcastHashJoin Inner BuildRight (205)
+                              :- * Project (203)
+                              :  +- * BroadcastHashJoin Inner BuildRight (202)
+                              :     :- * Project (200)
+                              :     :  +- * BroadcastHashJoin Inner BuildRight (199)
+                              :     :     :- * Project (197)
+                              :     :     :  +- * SortMergeJoin Inner (196)
+                              :     :     :     :- * Sort (193)
+                              :     :     :     :  +- Exchange (192)
+                              :     :     :     :     +- * Project (191)
+                              :     :     :     :        +- * SortMergeJoin Inner (190)
+                              :     :     :     :           :- * Sort (187)
+                              :     :     :     :           :  +- Exchange (186)
+                              :     :     :     :           :     +- * Project (185)
+                              :     :     :     :           :        +- * BroadcastHashJoin Inner BuildRight (184)
+                              :     :     :     :           :           :- * Project (182)
+                              :     :     :     :           :           :  +- * BroadcastHashJoin Inner BuildRight (181)
+                              :     :     :     :           :           :     :- * Project (179)
+                              :     :     :     :           :           :     :  +- * BroadcastHashJoin Inner BuildRight (178)
+                              :     :     :     :           :           :     :     :- * Project (176)
+                              :     :     :     :           :           :     :     :  +- * SortMergeJoin Inner (175)
+                              :     :     :     :           :           :     :     :     :- * Sort (172)
+                              :     :     :     :           :           :     :     :     :  +- Exchange (171)
+                              :     :     :     :           :           :     :     :     :     +- * Project (170)
+                              :     :     :     :           :           :     :     :     :        +- * SortMergeJoin Inner (169)
+                              :     :     :     :           :           :     :     :     :           :- * Sort (166)
+                              :     :     :     :           :           :     :     :     :           :  +- Exchange (165)
+                              :     :     :     :           :           :     :     :     :           :     +- * Project (164)
+                              :     :     :     :           :           :     :     :     :           :        +- * BroadcastHashJoin Inner BuildRight (163)
+                              :     :     :     :           :           :     :     :     :           :           :- * Project (161)
+                              :     :     :     :           :           :     :     :     :           :           :  +- * BroadcastHashJoin Inner BuildRight (160)
+                              :     :     :     :           :           :     :     :     :           :           :     :- * Project (158)
+                              :     :     :     :           :           :     :     :     :           :           :     :  +- * SortMergeJoin Inner (157)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :- * Sort (154)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :  +- Exchange (153)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :     +- * Project (152)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :        +- * BroadcastHashJoin Inner BuildRight (151)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :- * Project (149)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :  +- * BroadcastHashJoin Inner BuildRight (148)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     :- * Project (143)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     :  +- * BroadcastHashJoin Inner BuildRight (142)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :- * Project (140)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :  +- * SortMergeJoin Inner (139)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :     :- * Sort (136)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :     :  +- Exchange (135)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :     :     +- * Filter (134)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :     :        +- * ColumnarToRow (133)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :     :           +- Scan parquet default.store_sales (132)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :     +- * Sort (138)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     :        +- ReusedExchange (137)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     :     +- ReusedExchange (141)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :     +- BroadcastExchange (147)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :        +- * Filter (146)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :           +- * ColumnarToRow (145)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           :              +- Scan parquet default.date_dim (144)
+                              :     :     :     :           :           :     :     :     :           :           :     :     :           +- ReusedExchange (150)
+                              :     :     :     :           :           :     :     :     :           :           :     :     +- * Sort (156)
+                              :     :     :     :           :           :     :     :     :           :           :     :        +- ReusedExchange (155)
+                              :     :     :     :           :           :     :     :     :           :           :     +- ReusedExchange (159)
+                              :     :     :     :           :           :     :     :     :           :           +- ReusedExchange (162)
+                              :     :     :     :           :           :     :     :     :           +- * Sort (168)
+                              :     :     :     :           :           :     :     :     :              +- ReusedExchange (167)
+                              :     :     :     :           :           :     :     :     +- * Sort (174)
+                              :     :     :     :           :           :     :     :        +- ReusedExchange (173)
+                              :     :     :     :           :           :     :     +- ReusedExchange (177)
+                              :     :     :     :           :           :     +- ReusedExchange (180)
+                              :     :     :     :           :           +- ReusedExchange (183)
+                              :     :     :     :           +- * Sort (189)
+                              :     :     :     :              +- ReusedExchange (188)
+                              :     :     :     +- * Sort (195)
+                              :     :     :        +- ReusedExchange (194)
+                              :     :     +- ReusedExchange (198)
+                              :     +- ReusedExchange (201)
+                              +- ReusedExchange (204)
+
+
+(1) Scan parquet default.store_sales
+Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_sold_date_sk#12 IN dynamicpruning#13)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_cdemo_sk:int,ss_hdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_promo_sk:int,ss_ticket_number:int,ss_wholesale_cost:decimal(7,2),ss_list_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+
+(3) Filter [codegen id : 1]
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AND isnotnull(ss_store_sk#6)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_cdemo_sk#3)) AND isnotnull(ss_promo_sk#7)) AND isnotnull(ss_hdemo_sk#4)) AND isnotnull(ss_addr_sk#5))
+
+(4) Exchange
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#8, 5), ENSURE_REQUIREMENTS, [id=#14]
+
+(5) Sort [codegen id : 2]
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Arguments: [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#8 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.store_returns
+Output [3]: [sr_item_sk#15, sr_ticket_number#16, sr_returned_date_sk#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int>
+
+(7) ColumnarToRow [codegen id : 3]
+Input [3]: [sr_item_sk#15, sr_ticket_number#16, sr_returned_date_sk#17]
+
+(8) Filter [codegen id : 3]
+Input [3]: [sr_item_sk#15, sr_ticket_number#16, sr_returned_date_sk#17]
+Condition : (isnotnull(sr_item_sk#15) AND isnotnull(sr_ticket_number#16))
+
+(9) Project [codegen id : 3]
+Output [2]: [sr_item_sk#15, sr_ticket_number#16]
+Input [3]: [sr_item_sk#15, sr_ticket_number#16, sr_returned_date_sk#17]
+
+(10) Exchange
+Input [2]: [sr_item_sk#15, sr_ticket_number#16]
+Arguments: hashpartitioning(sr_item_sk#15, sr_ticket_number#16, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(11) Sort [codegen id : 4]
+Input [2]: [sr_item_sk#15, sr_ticket_number#16]
+Arguments: [sr_item_sk#15 ASC NULLS FIRST, sr_ticket_number#16 ASC NULLS FIRST], false, 0
+
+(12) SortMergeJoin [codegen id : 13]
+Left keys [2]: [ss_item_sk#1, ss_ticket_number#8]
+Right keys [2]: [sr_item_sk#15, sr_ticket_number#16]
+Join condition: None
+
+(13) Project [codegen id : 13]
+Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#15, sr_ticket_number#16]
+
+(14) Scan parquet default.catalog_sales
+Output [4]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21, cs_sold_date_sk#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_sales]
+PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)]
+ReadSchema: struct<cs_item_sk:int,cs_order_number:int,cs_ext_list_price:decimal(7,2)>
+
+(15) ColumnarToRow [codegen id : 5]
+Input [4]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21, cs_sold_date_sk#22]
+
+(16) Filter [codegen id : 5]
+Input [4]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21, cs_sold_date_sk#22]
+Condition : (isnotnull(cs_item_sk#19) AND isnotnull(cs_order_number#20))
+
+(17) Project [codegen id : 5]
+Output [3]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21]
+Input [4]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21, cs_sold_date_sk#22]
+
+(18) Exchange
+Input [3]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21]
+Arguments: hashpartitioning(cs_item_sk#19, cs_order_number#20, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(19) Sort [codegen id : 6]
+Input [3]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21]
+Arguments: [cs_item_sk#19 ASC NULLS FIRST, cs_order_number#20 ASC NULLS FIRST], false, 0
+
+(20) Scan parquet default.catalog_returns
+Output [6]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28, cr_returned_date_sk#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int,cr_refunded_cash:decimal(7,2),cr_reversed_charge:decimal(7,2),cr_store_credit:decimal(7,2)>
+
+(21) ColumnarToRow [codegen id : 7]
+Input [6]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28, cr_returned_date_sk#29]
+
+(22) Filter [codegen id : 7]
+Input [6]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28, cr_returned_date_sk#29]
+Condition : (isnotnull(cr_item_sk#24) AND isnotnull(cr_order_number#25))
+
+(23) Project [codegen id : 7]
+Output [5]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28]
+Input [6]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28, cr_returned_date_sk#29]
+
+(24) Exchange
+Input [5]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28]
+Arguments: hashpartitioning(cr_item_sk#24, cr_order_number#25, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(25) Sort [codegen id : 8]
+Input [5]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28]
+Arguments: [cr_item_sk#24 ASC NULLS FIRST, cr_order_number#25 ASC NULLS FIRST], false, 0
+
+(26) SortMergeJoin [codegen id : 9]
+Left keys [2]: [cs_item_sk#19, cs_order_number#20]
+Right keys [2]: [cr_item_sk#24, cr_order_number#25]
+Join condition: None
+
+(27) Project [codegen id : 9]
+Output [5]: [cs_item_sk#19, cs_ext_list_price#21, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28]
+Input [8]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21, cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28]
+
+(28) HashAggregate [codegen id : 9]
+Input [5]: [cs_item_sk#19, cs_ext_list_price#21, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28]
+Keys [1]: [cs_item_sk#19]
+Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#21)), partial_sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))]
+Aggregate Attributes [3]: [sum#31, sum#32, isEmpty#33]
+Results [4]: [cs_item_sk#19, sum#34, sum#35, isEmpty#36]
+
+(29) Exchange
+Input [4]: [cs_item_sk#19, sum#34, sum#35, isEmpty#36]
+Arguments: hashpartitioning(cs_item_sk#19, 5), ENSURE_REQUIREMENTS, [id=#37]
+
+(30) HashAggregate [codegen id : 10]
+Input [4]: [cs_item_sk#19, sum#34, sum#35, isEmpty#36]
+Keys [1]: [cs_item_sk#19]
+Functions [2]: [sum(UnscaledValue(cs_ext_list_price#21)), sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))]
+Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#21))#38, sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))#39]
+Results [3]: [cs_item_sk#19, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#21))#38,17,2) AS sum(cs_ext_list_price#21)#40, sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))#39 AS sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))#41]
+
+(31) Filter [codegen id : 10]
+Input [3]: [cs_item_sk#19, sum(cs_ext_list_price#21)#40, sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))#41]
+Condition : (isnotnull(sum(cs_ext_list_price#21)#40) AND (cast(sum(cs_ext_list_price#21)#40 as decimal(21,2)) > CheckOverflow((2.00 * promote_precision(sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))#41)), DecimalType(21,2), true)))
+
+(32) Project [codegen id : 10]
+Output [1]: [cs_item_sk#19]
+Input [3]: [cs_item_sk#19, sum(cs_ext_list_price#21)#40, sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))#41]
+
+(33) BroadcastExchange
+Input [1]: [cs_item_sk#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#42]
+
+(34) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [cs_item_sk#19]
+Join condition: None
+
+(35) Project [codegen id : 13]
+Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#19]
+
+(36) Scan parquet default.date_dim
+Output [2]: [d_date_sk#43, d_year#44]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(37) ColumnarToRow [codegen id : 11]
+Input [2]: [d_date_sk#43, d_year#44]
+
+(38) Filter [codegen id : 11]
+Input [2]: [d_date_sk#43, d_year#44]
+Condition : ((isnotnull(d_year#44) AND (d_year#44 = 1999)) AND isnotnull(d_date_sk#43))
+
+(39) BroadcastExchange
+Input [2]: [d_date_sk#43, d_year#44]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#45]
+
+(40) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ss_sold_date_sk#12]
+Right keys [1]: [d_date_sk#43]
+Join condition: None
+
+(41) Project [codegen id : 13]
+Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44]
+Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#43, d_year#44]
+
+(42) Scan parquet default.store
+Output [3]: [s_store_sk#46, s_store_name#47, s_zip#48]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_zip)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_zip:string>
+
+(43) ColumnarToRow [codegen id : 12]
+Input [3]: [s_store_sk#46, s_store_name#47, s_zip#48]
+
+(44) Filter [codegen id : 12]
+Input [3]: [s_store_sk#46, s_store_name#47, s_zip#48]
+Condition : ((isnotnull(s_store_sk#46) AND isnotnull(s_store_name#47)) AND isnotnull(s_zip#48))
+
+(45) BroadcastExchange
+Input [3]: [s_store_sk#46, s_store_name#47, s_zip#48]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#49]
+
+(46) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ss_store_sk#6]
+Right keys [1]: [s_store_sk#46]
+Join condition: None
+
+(47) Project [codegen id : 13]
+Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48]
+Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_sk#46, s_store_name#47, s_zip#48]
+
+(48) Exchange
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48]
+Arguments: hashpartitioning(ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [id=#50]
+
+(49) Sort [codegen id : 14]
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48]
+Arguments: [ss_customer_sk#2 ASC NULLS FIRST], false, 0
+
+(50) Scan parquet default.customer
+Output [6]: [c_customer_sk#51, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, c_first_shipto_date_sk#55, c_first_sales_date_sk#56]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_hdemo_sk:int,c_current_addr_sk:int,c_first_shipto_date_sk:int,c_first_sales_date_sk:int>
+
+(51) ColumnarToRow [codegen id : 15]
+Input [6]: [c_customer_sk#51, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, c_first_shipto_date_sk#55, c_first_sales_date_sk#56]
+
+(52) Filter [codegen id : 15]
+Input [6]: [c_customer_sk#51, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, c_first_shipto_date_sk#55, c_first_sales_date_sk#56]
+Condition : (((((isnotnull(c_customer_sk#51) AND isnotnull(c_first_sales_date_sk#56)) AND isnotnull(c_first_shipto_date_sk#55)) AND isnotnull(c_current_cdemo_sk#52)) AND isnotnull(c_current_hdemo_sk#53)) AND isnotnull(c_current_addr_sk#54))
+
+(53) Exchange
+Input [6]: [c_customer_sk#51, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, c_first_shipto_date_sk#55, c_first_sales_date_sk#56]
+Arguments: hashpartitioning(c_customer_sk#51, 5), ENSURE_REQUIREMENTS, [id=#57]
+
+(54) Sort [codegen id : 16]
+Input [6]: [c_customer_sk#51, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, c_first_shipto_date_sk#55, c_first_sales_date_sk#56]
+Arguments: [c_customer_sk#51 ASC NULLS FIRST], false, 0
+
+(55) SortMergeJoin [codegen id : 19]
+Left keys [1]: [ss_customer_sk#2]
+Right keys [1]: [c_customer_sk#51]
+Join condition: None
+
+(56) Project [codegen id : 19]
+Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, c_first_shipto_date_sk#55, c_first_sales_date_sk#56]
+Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_customer_sk#51, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, c_first_shipto_date_sk#55, c_first_sales_date_sk#56]
+
+(57) Scan parquet default.date_dim
+Output [2]: [d_date_sk#58, d_year#59]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(58) ColumnarToRow [codegen id : 17]
+Input [2]: [d_date_sk#58, d_year#59]
+
+(59) Filter [codegen id : 17]
+Input [2]: [d_date_sk#58, d_year#59]
+Condition : isnotnull(d_date_sk#58)
+
+(60) BroadcastExchange
+Input [2]: [d_date_sk#58, d_year#59]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#60]
+
+(61) BroadcastHashJoin [codegen id : 19]
+Left keys [1]: [c_first_sales_date_sk#56]
+Right keys [1]: [d_date_sk#58]
+Join condition: None
+
+(62) Project [codegen id : 19]
+Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, c_first_shipto_date_sk#55, d_year#59]
+Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, c_first_shipto_date_sk#55, c_first_sales_date_sk#56, d_date_sk#58, d_year#59]
+
+(63) ReusedExchange [Reuses operator id: 60]
+Output [2]: [d_date_sk#61, d_year#62]
+
+(64) BroadcastHashJoin [codegen id : 19]
+Left keys [1]: [c_first_shipto_date_sk#55]
+Right keys [1]: [d_date_sk#61]
+Join condition: None
+
+(65) Project [codegen id : 19]
+Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62]
+Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, c_first_shipto_date_sk#55, d_year#59, d_date_sk#61, d_year#62]
+
+(66) Exchange
+Input [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62]
+Arguments: hashpartitioning(ss_cdemo_sk#3, 5), ENSURE_REQUIREMENTS, [id=#63]
+
+(67) Sort [codegen id : 20]
+Input [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62]
+Arguments: [ss_cdemo_sk#3 ASC NULLS FIRST], false, 0
+
+(68) Scan parquet default.customer_demographics
+Output [2]: [cd_demo_sk#64, cd_marital_status#65]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status)]
+ReadSchema: struct<cd_demo_sk:int,cd_marital_status:string>
+
+(69) ColumnarToRow [codegen id : 21]
+Input [2]: [cd_demo_sk#64, cd_marital_status#65]
+
+(70) Filter [codegen id : 21]
+Input [2]: [cd_demo_sk#64, cd_marital_status#65]
+Condition : (isnotnull(cd_demo_sk#64) AND isnotnull(cd_marital_status#65))
+
+(71) Exchange
+Input [2]: [cd_demo_sk#64, cd_marital_status#65]
+Arguments: hashpartitioning(cd_demo_sk#64, 5), ENSURE_REQUIREMENTS, [id=#66]
+
+(72) Sort [codegen id : 22]
+Input [2]: [cd_demo_sk#64, cd_marital_status#65]
+Arguments: [cd_demo_sk#64 ASC NULLS FIRST], false, 0
+
+(73) SortMergeJoin [codegen id : 23]
+Left keys [1]: [ss_cdemo_sk#3]
+Right keys [1]: [cd_demo_sk#64]
+Join condition: None
+
+(74) Project [codegen id : 23]
+Output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62, cd_marital_status#65]
+Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62, cd_demo_sk#64, cd_marital_status#65]
+
+(75) Exchange
+Input [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62, cd_marital_status#65]
+Arguments: hashpartitioning(c_current_cdemo_sk#52, 5), ENSURE_REQUIREMENTS, [id=#67]
+
+(76) Sort [codegen id : 24]
+Input [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62, cd_marital_status#65]
+Arguments: [c_current_cdemo_sk#52 ASC NULLS FIRST], false, 0
+
+(77) ReusedExchange [Reuses operator id: 71]
+Output [2]: [cd_demo_sk#68, cd_marital_status#69]
+
+(78) Sort [codegen id : 26]
+Input [2]: [cd_demo_sk#68, cd_marital_status#69]
+Arguments: [cd_demo_sk#68 ASC NULLS FIRST], false, 0
+
+(79) SortMergeJoin [codegen id : 30]
+Left keys [1]: [c_current_cdemo_sk#52]
+Right keys [1]: [cd_demo_sk#68]
+Join condition: NOT (cd_marital_status#65 = cd_marital_status#69)
+
+(80) Project [codegen id : 30]
+Output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62]
+Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_cdemo_sk#52, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62, cd_marital_status#65, cd_demo_sk#68, cd_marital_status#69]
+
+(81) Scan parquet default.promotion
+Output [1]: [p_promo_sk#70]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/promotion]
+PushedFilters: [IsNotNull(p_promo_sk)]
+ReadSchema: struct<p_promo_sk:int>
+
+(82) ColumnarToRow [codegen id : 27]
+Input [1]: [p_promo_sk#70]
+
+(83) Filter [codegen id : 27]
+Input [1]: [p_promo_sk#70]
+Condition : isnotnull(p_promo_sk#70)
+
+(84) BroadcastExchange
+Input [1]: [p_promo_sk#70]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#71]
+
+(85) BroadcastHashJoin [codegen id : 30]
+Left keys [1]: [ss_promo_sk#7]
+Right keys [1]: [p_promo_sk#70]
+Join condition: None
+
+(86) Project [codegen id : 30]
+Output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62]
+Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62, p_promo_sk#70]
+
+(87) Scan parquet default.household_demographics
+Output [2]: [hd_demo_sk#72, hd_income_band_sk#73]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_income_band_sk:int>
+
+(88) ColumnarToRow [codegen id : 28]
+Input [2]: [hd_demo_sk#72, hd_income_band_sk#73]
+
+(89) Filter [codegen id : 28]
+Input [2]: [hd_demo_sk#72, hd_income_band_sk#73]
+Condition : (isnotnull(hd_demo_sk#72) AND isnotnull(hd_income_band_sk#73))
+
+(90) BroadcastExchange
+Input [2]: [hd_demo_sk#72, hd_income_band_sk#73]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#74]
+
+(91) BroadcastHashJoin [codegen id : 30]
+Left keys [1]: [ss_hdemo_sk#4]
+Right keys [1]: [hd_demo_sk#72]
+Join condition: None
+
+(92) Project [codegen id : 30]
+Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62, hd_income_band_sk#73]
+Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62, hd_demo_sk#72, hd_income_band_sk#73]
+
+(93) ReusedExchange [Reuses operator id: 90]
+Output [2]: [hd_demo_sk#75, hd_income_band_sk#76]
+
+(94) BroadcastHashJoin [codegen id : 30]
+Left keys [1]: [c_current_hdemo_sk#53]
+Right keys [1]: [hd_demo_sk#75]
+Join condition: None
+
+(95) Project [codegen id : 30]
+Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_addr_sk#54, d_year#59, d_year#62, hd_income_band_sk#73, hd_income_band_sk#76]
+Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_hdemo_sk#53, c_current_addr_sk#54, d_year#59, d_year#62, hd_income_band_sk#73, hd_demo_sk#75, hd_income_band_sk#76]
+
+(96) Exchange
+Input [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_addr_sk#54, d_year#59, d_year#62, hd_income_band_sk#73, hd_income_band_sk#76]
+Arguments: hashpartitioning(ss_addr_sk#5, 5), ENSURE_REQUIREMENTS, [id=#77]
+
+(97) Sort [codegen id : 31]
+Input [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_addr_sk#54, d_year#59, d_year#62, hd_income_band_sk#73, hd_income_band_sk#76]
+Arguments: [ss_addr_sk#5 ASC NULLS FIRST], false, 0
+
+(98) Scan parquet default.customer_address
+Output [5]: [ca_address_sk#78, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_street_number:string,ca_street_name:string,ca_city:string,ca_zip:string>
+
+(99) ColumnarToRow [codegen id : 32]
+Input [5]: [ca_address_sk#78, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82]
+
+(100) Filter [codegen id : 32]
+Input [5]: [ca_address_sk#78, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82]
+Condition : isnotnull(ca_address_sk#78)
+
+(101) Exchange
+Input [5]: [ca_address_sk#78, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82]
+Arguments: hashpartitioning(ca_address_sk#78, 5), ENSURE_REQUIREMENTS, [id=#83]
+
+(102) Sort [codegen id : 33]
+Input [5]: [ca_address_sk#78, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82]
+Arguments: [ca_address_sk#78 ASC NULLS FIRST], false, 0
+
+(103) SortMergeJoin [codegen id : 34]
+Left keys [1]: [ss_addr_sk#5]
+Right keys [1]: [ca_address_sk#78]
+Join condition: None
+
+(104) Project [codegen id : 34]
+Output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_addr_sk#54, d_year#59, d_year#62, hd_income_band_sk#73, hd_income_band_sk#76, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82]
+Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_addr_sk#54, d_year#59, d_year#62, hd_income_band_sk#73, hd_income_band_sk#76, ca_address_sk#78, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82]
+
+(105) Exchange
+Input [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_addr_sk#54, d_year#59, d_year#62, hd_income_band_sk#73, hd_income_band_sk#76, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82]
+Arguments: hashpartitioning(c_current_addr_sk#54, 5), ENSURE_REQUIREMENTS, [id=#84]
+
+(106) Sort [codegen id : 35]
+Input [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_addr_sk#54, d_year#59, d_year#62, hd_income_band_sk#73, hd_income_band_sk#76, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82]
+Arguments: [c_current_addr_sk#54 ASC NULLS FIRST], false, 0
+
+(107) ReusedExchange [Reuses operator id: 101]
+Output [5]: [ca_address_sk#85, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89]
+
+(108) Sort [codegen id : 37]
+Input [5]: [ca_address_sk#85, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89]
+Arguments: [ca_address_sk#85 ASC NULLS FIRST], false, 0
+
+(109) SortMergeJoin [codegen id : 41]
+Left keys [1]: [c_current_addr_sk#54]
+Right keys [1]: [ca_address_sk#85]
+Join condition: None
+
+(110) Project [codegen id : 41]
+Output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, d_year#59, d_year#62, hd_income_band_sk#73, hd_income_band_sk#76, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89]
+Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, c_current_addr_sk#54, d_year#59, d_year#62, hd_income_band_sk#73, hd_income_band_sk#76, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_address_sk#85, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89]
+
+(111) Scan parquet default.income_band
+Output [1]: [ib_income_band_sk#90]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/income_band]
+PushedFilters: [IsNotNull(ib_income_band_sk)]
+ReadSchema: struct<ib_income_band_sk:int>
+
+(112) ColumnarToRow [codegen id : 38]
+Input [1]: [ib_income_band_sk#90]
+
+(113) Filter [codegen id : 38]
+Input [1]: [ib_income_band_sk#90]
+Condition : isnotnull(ib_income_band_sk#90)
+
+(114) BroadcastExchange
+Input [1]: [ib_income_band_sk#90]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#91]
+
+(115) BroadcastHashJoin [codegen id : 41]
+Left keys [1]: [hd_income_band_sk#73]
+Right keys [1]: [ib_income_band_sk#90]
+Join condition: None
+
+(116) Project [codegen id : 41]
+Output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, d_year#59, d_year#62, hd_income_band_sk#76, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89]
+Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, d_year#59, d_year#62, hd_income_band_sk#73, hd_income_band_sk#76, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89, ib_income_band_sk#90]
+
+(117) ReusedExchange [Reuses operator id: 114]
+Output [1]: [ib_income_band_sk#92]
+
+(118) BroadcastHashJoin [codegen id : 41]
+Left keys [1]: [hd_income_band_sk#76]
+Right keys [1]: [ib_income_band_sk#92]
+Join condition: None
+
+(119) Project [codegen id : 41]
+Output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, d_year#59, d_year#62, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89]
+Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, d_year#59, d_year#62, hd_income_band_sk#76, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89, ib_income_band_sk#92]
+
+(120) Scan parquet default.item
+Output [4]: [i_item_sk#93, i_current_price#94, i_color#95, i_product_name#96]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_current_price), In(i_color, [purple              ,burlywood           ,indian              ,spring              ,floral              ,medium              ]), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_current_price:decimal(7,2),i_color:string,i_product_name:string>
+
+(121) ColumnarToRow [codegen id : 40]
+Input [4]: [i_item_sk#93, i_current_price#94, i_color#95, i_product_name#96]
+
+(122) Filter [codegen id : 40]
+Input [4]: [i_item_sk#93, i_current_price#94, i_color#95, i_product_name#96]
+Condition : ((((((isnotnull(i_current_price#94) AND i_color#95 IN (purple              ,burlywood           ,indian              ,spring              ,floral              ,medium              )) AND (i_current_price#94 >= 64.00)) AND (i_current_price#94 <= 74.00)) AND (i_current_price#94 >= 65.00)) AND (i_current_price#94 <= 79.00)) AND isnotnull(i_item_sk#93))
+
+(123) Project [codegen id : 40]
+Output [2]: [i_item_sk#93, i_product_name#96]
+Input [4]: [i_item_sk#93, i_current_price#94, i_color#95, i_product_name#96]
+
+(124) BroadcastExchange
+Input [2]: [i_item_sk#93, i_product_name#96]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#97]
+
+(125) BroadcastHashJoin [codegen id : 41]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#93]
+Join condition: None
+
+(126) Project [codegen id : 41]
+Output [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, d_year#59, d_year#62, s_store_name#47, s_zip#48, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89, i_item_sk#93, i_product_name#96]
+Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, s_store_name#47, s_zip#48, d_year#59, d_year#62, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89, i_item_sk#93, i_product_name#96]
+
+(127) HashAggregate [codegen id : 41]
+Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#44, d_year#59, d_year#62, s_store_name#47, s_zip#48, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89, i_item_sk#93, i_product_name#96]
+Keys [15]: [i_product_name#96, i_item_sk#93, s_store_name#47, s_zip#48, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89, d_year#44, d_year#59, d_year#62]
+Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))]
+Aggregate Attributes [4]: [count#98, sum#99, sum#100, sum#101]
+Results [19]: [i_product_name#96, i_item_sk#93, s_store_name#47, s_zip#48, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89, d_year#44, d_year#59, d_year#62, count#102, sum#103, sum#104, sum#105]
+
+(128) Exchange
+Input [19]: [i_product_name#96, i_item_sk#93, s_store_name#47, s_zip#48, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89, d_year#44, d_year#59, d_year#62, count#102, sum#103, sum#104, sum#105]
+Arguments: hashpartitioning(i_product_name#96, i_item_sk#93, s_store_name#47, s_zip#48, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89, d_year#44, d_year#59, d_year#62, 5), ENSURE_REQUIREMENTS, [id=#106]
+
+(129) HashAggregate [codegen id : 42]
+Input [19]: [i_product_name#96, i_item_sk#93, s_store_name#47, s_zip#48, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89, d_year#44, d_year#59, d_year#62, count#102, sum#103, sum#104, sum#105]
+Keys [15]: [i_product_name#96, i_item_sk#93, s_store_name#47, s_zip#48, ca_street_number#79, ca_street_name#80, ca_city#81, ca_zip#82, ca_street_number#86, ca_street_name#87, ca_city#88, ca_zip#89, d_year#44, d_year#59, d_year#62]
+Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))]
+Aggregate Attributes [4]: [count(1)#107, sum(UnscaledValue(ss_wholesale_cost#9))#108, sum(UnscaledValue(ss_list_price#10))#109, sum(UnscaledValue(ss_coupon_amt#11))#110]
+Results [17]: [i_product_name#96 AS product_name#111, i_item_sk#93 AS item_sk#112, s_store_name#47 AS store_name#113, s_zip#48 AS store_zip#114, ca_street_number#79 AS b_street_number#115, ca_street_name#80 AS b_streen_name#116, ca_city#81 AS b_city#117, ca_zip#82 AS b_zip#118, ca_street_number#86 AS c_street_number#119, ca_street_name#87 AS c_street_name#120, ca_city#88 AS c_city#121, ca_zip#89 AS c_zip#122, d_year#44 AS syear#123, count(1)#107 AS cnt#124, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#9))#108,17,2) AS s1#125, MakeDecimal(sum(UnscaledValue(ss_list_price#10))#109,17,2) AS s2#126, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#11))#110,17,2) AS s3#127]
+
+(130) Exchange
+Input [17]: [product_name#111, item_sk#112, store_name#113, store_zip#114, b_street_number#115, b_streen_name#116, b_city#117, b_zip#118, c_street_number#119, c_street_name#120, c_city#121, c_zip#122, syear#123, cnt#124, s1#125, s2#126, s3#127]
+Arguments: hashpartitioning(item_sk#112, store_name#113, store_zip#114, 5), ENSURE_REQUIREMENTS, [id=#128]
+
+(131) Sort [codegen id : 43]
+Input [17]: [product_name#111, item_sk#112, store_name#113, store_zip#114, b_street_number#115, b_streen_name#116, b_city#117, b_zip#118, c_street_number#119, c_street_name#120, c_city#121, c_zip#122, syear#123, cnt#124, s1#125, s2#126, s3#127]
+Arguments: [item_sk#112 ASC NULLS FIRST, store_name#113 ASC NULLS FIRST, store_zip#114 ASC NULLS FIRST], false, 0
+
+(132) Scan parquet default.store_sales
+Output [12]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_store_sk#134, ss_promo_sk#135, ss_ticket_number#136, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, ss_sold_date_sk#140]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#140), dynamicpruningexpression(ss_sold_date_sk#140 IN dynamicpruning#141)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_cdemo_sk:int,ss_hdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_promo_sk:int,ss_ticket_number:int,ss_wholesale_cost:decimal(7,2),ss_list_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(133) ColumnarToRow [codegen id : 44]
+Input [12]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_store_sk#134, ss_promo_sk#135, ss_ticket_number#136, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, ss_sold_date_sk#140]
+
+(134) Filter [codegen id : 44]
+Input [12]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_store_sk#134, ss_promo_sk#135, ss_ticket_number#136, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, ss_sold_date_sk#140]
+Condition : (((((((isnotnull(ss_item_sk#129) AND isnotnull(ss_ticket_number#136)) AND isnotnull(ss_store_sk#134)) AND isnotnull(ss_customer_sk#130)) AND isnotnull(ss_cdemo_sk#131)) AND isnotnull(ss_promo_sk#135)) AND isnotnull(ss_hdemo_sk#132)) AND isnotnull(ss_addr_sk#133))
+
+(135) Exchange
+Input [12]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_store_sk#134, ss_promo_sk#135, ss_ticket_number#136, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, ss_sold_date_sk#140]
+Arguments: hashpartitioning(ss_item_sk#129, ss_ticket_number#136, 5), ENSURE_REQUIREMENTS, [id=#142]
+
+(136) Sort [codegen id : 45]
+Input [12]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_store_sk#134, ss_promo_sk#135, ss_ticket_number#136, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, ss_sold_date_sk#140]
+Arguments: [ss_item_sk#129 ASC NULLS FIRST, ss_ticket_number#136 ASC NULLS FIRST], false, 0
+
+(137) ReusedExchange [Reuses operator id: 10]
+Output [2]: [sr_item_sk#143, sr_ticket_number#144]
+
+(138) Sort [codegen id : 47]
+Input [2]: [sr_item_sk#143, sr_ticket_number#144]
+Arguments: [sr_item_sk#143 ASC NULLS FIRST, sr_ticket_number#144 ASC NULLS FIRST], false, 0
+
+(139) SortMergeJoin [codegen id : 56]
+Left keys [2]: [ss_item_sk#129, ss_ticket_number#136]
+Right keys [2]: [sr_item_sk#143, sr_ticket_number#144]
+Join condition: None
+
+(140) Project [codegen id : 56]
+Output [11]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_store_sk#134, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, ss_sold_date_sk#140]
+Input [14]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_store_sk#134, ss_promo_sk#135, ss_ticket_number#136, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, ss_sold_date_sk#140, sr_item_sk#143, sr_ticket_number#144]
+
+(141) ReusedExchange [Reuses operator id: 33]
+Output [1]: [cs_item_sk#145]
+
+(142) BroadcastHashJoin [codegen id : 56]
+Left keys [1]: [ss_item_sk#129]
+Right keys [1]: [cs_item_sk#145]
+Join condition: None
+
+(143) Project [codegen id : 56]
+Output [11]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_store_sk#134, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, ss_sold_date_sk#140]
+Input [12]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_store_sk#134, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, ss_sold_date_sk#140, cs_item_sk#145]
+
+(144) Scan parquet default.date_dim
+Output [2]: [d_date_sk#146, d_year#147]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(145) ColumnarToRow [codegen id : 54]
+Input [2]: [d_date_sk#146, d_year#147]
+
+(146) Filter [codegen id : 54]
+Input [2]: [d_date_sk#146, d_year#147]
+Condition : ((isnotnull(d_year#147) AND (d_year#147 = 2000)) AND isnotnull(d_date_sk#146))
+
+(147) BroadcastExchange
+Input [2]: [d_date_sk#146, d_year#147]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#148]
+
+(148) BroadcastHashJoin [codegen id : 56]
+Left keys [1]: [ss_sold_date_sk#140]
+Right keys [1]: [d_date_sk#146]
+Join condition: None
+
+(149) Project [codegen id : 56]
+Output [11]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_store_sk#134, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147]
+Input [13]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_store_sk#134, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, ss_sold_date_sk#140, d_date_sk#146, d_year#147]
+
+(150) ReusedExchange [Reuses operator id: 45]
+Output [3]: [s_store_sk#149, s_store_name#150, s_zip#151]
+
+(151) BroadcastHashJoin [codegen id : 56]
+Left keys [1]: [ss_store_sk#134]
+Right keys [1]: [s_store_sk#149]
+Join condition: None
+
+(152) Project [codegen id : 56]
+Output [12]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151]
+Input [14]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_store_sk#134, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_sk#149, s_store_name#150, s_zip#151]
+
+(153) Exchange
+Input [12]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151]
+Arguments: hashpartitioning(ss_customer_sk#130, 5), ENSURE_REQUIREMENTS, [id=#152]
+
+(154) Sort [codegen id : 57]
+Input [12]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151]
+Arguments: [ss_customer_sk#130 ASC NULLS FIRST], false, 0
+
+(155) ReusedExchange [Reuses operator id: 53]
+Output [6]: [c_customer_sk#153, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, c_first_shipto_date_sk#157, c_first_sales_date_sk#158]
+
+(156) Sort [codegen id : 59]
+Input [6]: [c_customer_sk#153, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, c_first_shipto_date_sk#157, c_first_sales_date_sk#158]
+Arguments: [c_customer_sk#153 ASC NULLS FIRST], false, 0
+
+(157) SortMergeJoin [codegen id : 62]
+Left keys [1]: [ss_customer_sk#130]
+Right keys [1]: [c_customer_sk#153]
+Join condition: None
+
+(158) Project [codegen id : 62]
+Output [16]: [ss_item_sk#129, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, c_first_shipto_date_sk#157, c_first_sales_date_sk#158]
+Input [18]: [ss_item_sk#129, ss_customer_sk#130, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_customer_sk#153, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, c_first_shipto_date_sk#157, c_first_sales_date_sk#158]
+
+(159) ReusedExchange [Reuses operator id: 60]
+Output [2]: [d_date_sk#159, d_year#160]
+
+(160) BroadcastHashJoin [codegen id : 62]
+Left keys [1]: [c_first_sales_date_sk#158]
+Right keys [1]: [d_date_sk#159]
+Join condition: None
+
+(161) Project [codegen id : 62]
+Output [16]: [ss_item_sk#129, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, c_first_shipto_date_sk#157, d_year#160]
+Input [18]: [ss_item_sk#129, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, c_first_shipto_date_sk#157, c_first_sales_date_sk#158, d_date_sk#159, d_year#160]
+
+(162) ReusedExchange [Reuses operator id: 60]
+Output [2]: [d_date_sk#161, d_year#162]
+
+(163) BroadcastHashJoin [codegen id : 62]
+Left keys [1]: [c_first_shipto_date_sk#157]
+Right keys [1]: [d_date_sk#161]
+Join condition: None
+
+(164) Project [codegen id : 62]
+Output [16]: [ss_item_sk#129, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162]
+Input [18]: [ss_item_sk#129, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, c_first_shipto_date_sk#157, d_year#160, d_date_sk#161, d_year#162]
+
+(165) Exchange
+Input [16]: [ss_item_sk#129, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162]
+Arguments: hashpartitioning(ss_cdemo_sk#131, 5), ENSURE_REQUIREMENTS, [id=#163]
+
+(166) Sort [codegen id : 63]
+Input [16]: [ss_item_sk#129, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162]
+Arguments: [ss_cdemo_sk#131 ASC NULLS FIRST], false, 0
+
+(167) ReusedExchange [Reuses operator id: 71]
+Output [2]: [cd_demo_sk#164, cd_marital_status#165]
+
+(168) Sort [codegen id : 65]
+Input [2]: [cd_demo_sk#164, cd_marital_status#165]
+Arguments: [cd_demo_sk#164 ASC NULLS FIRST], false, 0
+
+(169) SortMergeJoin [codegen id : 66]
+Left keys [1]: [ss_cdemo_sk#131]
+Right keys [1]: [cd_demo_sk#164]
+Join condition: None
+
+(170) Project [codegen id : 66]
+Output [16]: [ss_item_sk#129, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162, cd_marital_status#165]
+Input [18]: [ss_item_sk#129, ss_cdemo_sk#131, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162, cd_demo_sk#164, cd_marital_status#165]
+
+(171) Exchange
+Input [16]: [ss_item_sk#129, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162, cd_marital_status#165]
+Arguments: hashpartitioning(c_current_cdemo_sk#154, 5), ENSURE_REQUIREMENTS, [id=#166]
+
+(172) Sort [codegen id : 67]
+Input [16]: [ss_item_sk#129, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162, cd_marital_status#165]
+Arguments: [c_current_cdemo_sk#154 ASC NULLS FIRST], false, 0
+
+(173) ReusedExchange [Reuses operator id: 71]
+Output [2]: [cd_demo_sk#167, cd_marital_status#168]
+
+(174) Sort [codegen id : 69]
+Input [2]: [cd_demo_sk#167, cd_marital_status#168]
+Arguments: [cd_demo_sk#167 ASC NULLS FIRST], false, 0
+
+(175) SortMergeJoin [codegen id : 73]
+Left keys [1]: [c_current_cdemo_sk#154]
+Right keys [1]: [cd_demo_sk#167]
+Join condition: NOT (cd_marital_status#165 = cd_marital_status#168)
+
+(176) Project [codegen id : 73]
+Output [14]: [ss_item_sk#129, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162]
+Input [18]: [ss_item_sk#129, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_cdemo_sk#154, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162, cd_marital_status#165, cd_demo_sk#167, cd_marital_status#168]
+
+(177) ReusedExchange [Reuses operator id: 84]
+Output [1]: [p_promo_sk#169]
+
+(178) BroadcastHashJoin [codegen id : 73]
+Left keys [1]: [ss_promo_sk#135]
+Right keys [1]: [p_promo_sk#169]
+Join condition: None
+
+(179) Project [codegen id : 73]
+Output [13]: [ss_item_sk#129, ss_hdemo_sk#132, ss_addr_sk#133, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162]
+Input [15]: [ss_item_sk#129, ss_hdemo_sk#132, ss_addr_sk#133, ss_promo_sk#135, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162, p_promo_sk#169]
+
+(180) ReusedExchange [Reuses operator id: 90]
+Output [2]: [hd_demo_sk#170, hd_income_band_sk#171]
+
+(181) BroadcastHashJoin [codegen id : 73]
+Left keys [1]: [ss_hdemo_sk#132]
+Right keys [1]: [hd_demo_sk#170]
+Join condition: None
+
+(182) Project [codegen id : 73]
+Output [13]: [ss_item_sk#129, ss_addr_sk#133, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162, hd_income_band_sk#171]
+Input [15]: [ss_item_sk#129, ss_hdemo_sk#132, ss_addr_sk#133, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162, hd_demo_sk#170, hd_income_band_sk#171]
+
+(183) ReusedExchange [Reuses operator id: 90]
+Output [2]: [hd_demo_sk#172, hd_income_band_sk#173]
+
+(184) BroadcastHashJoin [codegen id : 73]
+Left keys [1]: [c_current_hdemo_sk#155]
+Right keys [1]: [hd_demo_sk#172]
+Join condition: None
+
+(185) Project [codegen id : 73]
+Output [13]: [ss_item_sk#129, ss_addr_sk#133, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_addr_sk#156, d_year#160, d_year#162, hd_income_band_sk#171, hd_income_band_sk#173]
+Input [15]: [ss_item_sk#129, ss_addr_sk#133, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_hdemo_sk#155, c_current_addr_sk#156, d_year#160, d_year#162, hd_income_band_sk#171, hd_demo_sk#172, hd_income_band_sk#173]
+
+(186) Exchange
+Input [13]: [ss_item_sk#129, ss_addr_sk#133, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_addr_sk#156, d_year#160, d_year#162, hd_income_band_sk#171, hd_income_band_sk#173]
+Arguments: hashpartitioning(ss_addr_sk#133, 5), ENSURE_REQUIREMENTS, [id=#174]
+
+(187) Sort [codegen id : 74]
+Input [13]: [ss_item_sk#129, ss_addr_sk#133, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_addr_sk#156, d_year#160, d_year#162, hd_income_band_sk#171, hd_income_band_sk#173]
+Arguments: [ss_addr_sk#133 ASC NULLS FIRST], false, 0
+
+(188) ReusedExchange [Reuses operator id: 101]
+Output [5]: [ca_address_sk#175, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179]
+
+(189) Sort [codegen id : 76]
+Input [5]: [ca_address_sk#175, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179]
+Arguments: [ca_address_sk#175 ASC NULLS FIRST], false, 0
+
+(190) SortMergeJoin [codegen id : 77]
+Left keys [1]: [ss_addr_sk#133]
+Right keys [1]: [ca_address_sk#175]
+Join condition: None
+
+(191) Project [codegen id : 77]
+Output [16]: [ss_item_sk#129, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_addr_sk#156, d_year#160, d_year#162, hd_income_band_sk#171, hd_income_band_sk#173, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179]
+Input [18]: [ss_item_sk#129, ss_addr_sk#133, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_addr_sk#156, d_year#160, d_year#162, hd_income_band_sk#171, hd_income_band_sk#173, ca_address_sk#175, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179]
+
+(192) Exchange
+Input [16]: [ss_item_sk#129, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_addr_sk#156, d_year#160, d_year#162, hd_income_band_sk#171, hd_income_band_sk#173, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179]
+Arguments: hashpartitioning(c_current_addr_sk#156, 5), ENSURE_REQUIREMENTS, [id=#180]
+
+(193) Sort [codegen id : 78]
+Input [16]: [ss_item_sk#129, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_addr_sk#156, d_year#160, d_year#162, hd_income_band_sk#171, hd_income_band_sk#173, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179]
+Arguments: [c_current_addr_sk#156 ASC NULLS FIRST], false, 0
+
+(194) ReusedExchange [Reuses operator id: 101]
+Output [5]: [ca_address_sk#181, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185]
+
+(195) Sort [codegen id : 80]
+Input [5]: [ca_address_sk#181, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185]
+Arguments: [ca_address_sk#181 ASC NULLS FIRST], false, 0
+
+(196) SortMergeJoin [codegen id : 84]
+Left keys [1]: [c_current_addr_sk#156]
+Right keys [1]: [ca_address_sk#181]
+Join condition: None
+
+(197) Project [codegen id : 84]
+Output [19]: [ss_item_sk#129, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, d_year#160, d_year#162, hd_income_band_sk#171, hd_income_band_sk#173, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185]
+Input [21]: [ss_item_sk#129, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, c_current_addr_sk#156, d_year#160, d_year#162, hd_income_band_sk#171, hd_income_band_sk#173, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_address_sk#181, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185]
+
+(198) ReusedExchange [Reuses operator id: 114]
+Output [1]: [ib_income_band_sk#186]
+
+(199) BroadcastHashJoin [codegen id : 84]
+Left keys [1]: [hd_income_band_sk#171]
+Right keys [1]: [ib_income_band_sk#186]
+Join condition: None
+
+(200) Project [codegen id : 84]
+Output [18]: [ss_item_sk#129, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, d_year#160, d_year#162, hd_income_band_sk#173, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185]
+Input [20]: [ss_item_sk#129, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, d_year#160, d_year#162, hd_income_band_sk#171, hd_income_band_sk#173, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185, ib_income_band_sk#186]
+
+(201) ReusedExchange [Reuses operator id: 114]
+Output [1]: [ib_income_band_sk#187]
+
+(202) BroadcastHashJoin [codegen id : 84]
+Left keys [1]: [hd_income_band_sk#173]
+Right keys [1]: [ib_income_band_sk#187]
+Join condition: None
+
+(203) Project [codegen id : 84]
+Output [17]: [ss_item_sk#129, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, d_year#160, d_year#162, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185]
+Input [19]: [ss_item_sk#129, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, d_year#160, d_year#162, hd_income_band_sk#173, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185, ib_income_band_sk#187]
+
+(204) ReusedExchange [Reuses operator id: 124]
+Output [2]: [i_item_sk#188, i_product_name#189]
+
+(205) BroadcastHashJoin [codegen id : 84]
+Left keys [1]: [ss_item_sk#129]
+Right keys [1]: [i_item_sk#188]
+Join condition: None
+
+(206) Project [codegen id : 84]
+Output [18]: [ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, d_year#160, d_year#162, s_store_name#150, s_zip#151, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185, i_item_sk#188, i_product_name#189]
+Input [19]: [ss_item_sk#129, ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, s_store_name#150, s_zip#151, d_year#160, d_year#162, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185, i_item_sk#188, i_product_name#189]
+
+(207) HashAggregate [codegen id : 84]
+Input [18]: [ss_wholesale_cost#137, ss_list_price#138, ss_coupon_amt#139, d_year#147, d_year#160, d_year#162, s_store_name#150, s_zip#151, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185, i_item_sk#188, i_product_name#189]
+Keys [15]: [i_product_name#189, i_item_sk#188, s_store_name#150, s_zip#151, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185, d_year#147, d_year#160, d_year#162]
+Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#137)), partial_sum(UnscaledValue(ss_list_price#138)), partial_sum(UnscaledValue(ss_coupon_amt#139))]
+Aggregate Attributes [4]: [count#190, sum#191, sum#192, sum#193]
+Results [19]: [i_product_name#189, i_item_sk#188, s_store_name#150, s_zip#151, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185, d_year#147, d_year#160, d_year#162, count#194, sum#195, sum#196, sum#197]
+
+(208) Exchange
+Input [19]: [i_product_name#189, i_item_sk#188, s_store_name#150, s_zip#151, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185, d_year#147, d_year#160, d_year#162, count#194, sum#195, sum#196, sum#197]
+Arguments: hashpartitioning(i_product_name#189, i_item_sk#188, s_store_name#150, s_zip#151, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185, d_year#147, d_year#160, d_year#162, 5), ENSURE_REQUIREMENTS, [id=#198]
+
+(209) HashAggregate [codegen id : 85]
+Input [19]: [i_product_name#189, i_item_sk#188, s_store_name#150, s_zip#151, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185, d_year#147, d_year#160, d_year#162, count#194, sum#195, sum#196, sum#197]
+Keys [15]: [i_product_name#189, i_item_sk#188, s_store_name#150, s_zip#151, ca_street_number#176, ca_street_name#177, ca_city#178, ca_zip#179, ca_street_number#182, ca_street_name#183, ca_city#184, ca_zip#185, d_year#147, d_year#160, d_year#162]
+Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#137)), sum(UnscaledValue(ss_list_price#138)), sum(UnscaledValue(ss_coupon_amt#139))]
+Aggregate Attributes [4]: [count(1)#199, sum(UnscaledValue(ss_wholesale_cost#137))#200, sum(UnscaledValue(ss_list_price#138))#201, sum(UnscaledValue(ss_coupon_amt#139))#202]
+Results [8]: [i_item_sk#188 AS item_sk#203, s_store_name#150 AS store_name#204, s_zip#151 AS store_zip#205, d_year#147 AS syear#206, count(1)#199 AS cnt#207, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#137))#200,17,2) AS s1#208, MakeDecimal(sum(UnscaledValue(ss_list_price#138))#201,17,2) AS s2#209, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#139))#202,17,2) AS s3#210]
+
+(210) Exchange
+Input [8]: [item_sk#203, store_name#204, store_zip#205, syear#206, cnt#207, s1#208, s2#209, s3#210]
+Arguments: hashpartitioning(item_sk#203, store_name#204, store_zip#205, 5), ENSURE_REQUIREMENTS, [id=#211]
+
+(211) Sort [codegen id : 86]
+Input [8]: [item_sk#203, store_name#204, store_zip#205, syear#206, cnt#207, s1#208, s2#209, s3#210]
+Arguments: [item_sk#203 ASC NULLS FIRST, store_name#204 ASC NULLS FIRST, store_zip#205 ASC NULLS FIRST], false, 0
+
+(212) SortMergeJoin [codegen id : 87]
+Left keys [3]: [item_sk#112, store_name#113, store_zip#114]
+Right keys [3]: [item_sk#203, store_name#204, store_zip#205]
+Join condition: (cnt#207 <= cnt#124)
+
+(213) Project [codegen id : 87]
+Output [21]: [product_name#111, store_name#113, store_zip#114, b_street_number#115, b_streen_name#116, b_city#117, b_zip#118, c_street_number#119, c_street_name#120, c_city#121, c_zip#122, syear#123, cnt#124, s1#125, s2#126, s3#127, s1#208, s2#209, s3#210, syear#206, cnt#207]
+Input [25]: [product_name#111, item_sk#112, store_name#113, store_zip#114, b_street_number#115, b_streen_name#116, b_city#117, b_zip#118, c_street_number#119, c_street_name#120, c_city#121, c_zip#122, syear#123, cnt#124, s1#125, s2#126, s3#127, item_sk#203, store_name#204, store_zip#205, syear#206, cnt#207, s1#208, s2#209, s3#210]
+
+(214) Exchange
+Input [21]: [product_name#111, store_name#113, store_zip#114, b_street_number#115, b_streen_name#116, b_city#117, b_zip#118, c_street_number#119, c_street_name#120, c_city#121, c_zip#122, syear#123, cnt#124, s1#125, s2#126, s3#127, s1#208, s2#209, s3#210, syear#206, cnt#207]
+Arguments: rangepartitioning(product_name#111 ASC NULLS FIRST, store_name#113 ASC NULLS FIRST, cnt#207 ASC NULLS FIRST, s1#125 ASC NULLS FIRST, s1#208 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [id=#212]
+
+(215) Sort [codegen id : 88]
+Input [21]: [product_name#111, store_name#113, store_zip#114, b_street_number#115, b_streen_name#116, b_city#117, b_zip#118, c_street_number#119, c_street_name#120, c_city#121, c_zip#122, syear#123, cnt#124, s1#125, s2#126, s3#127, s1#208, s2#209, s3#210, syear#206, cnt#207]
+Arguments: [product_name#111 ASC NULLS FIRST, store_name#113 ASC NULLS FIRST, cnt#207 ASC NULLS FIRST, s1#125 ASC NULLS FIRST, s1#208 ASC NULLS FIRST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13
+ReusedExchange (216)
+
+
+(216) ReusedExchange [Reuses operator id: 39]
+Output [2]: [d_date_sk#43, d_year#44]
+
+Subquery:2 Hosting operator id = 132 Hosting Expression = ss_sold_date_sk#140 IN dynamicpruning#141
+ReusedExchange (217)
+
+
+(217) ReusedExchange [Reuses operator id: 147]
+Output [2]: [d_date_sk#146, d_year#147]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.sf100/simplified.txt
new file mode 100644
index 0000000000000..22228c7657d26
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.sf100/simplified.txt
@@ -0,0 +1,379 @@
+WholeStageCodegen (88)
+  Sort [product_name,store_name,cnt,s1,s1]
+    InputAdapter
+      Exchange [product_name,store_name,cnt,s1,s1] #1
+        WholeStageCodegen (87)
+          Project [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt]
+            SortMergeJoin [item_sk,store_name,store_zip,item_sk,store_name,store_zip,cnt,cnt]
+              InputAdapter
+                WholeStageCodegen (43)
+                  Sort [item_sk,store_name,store_zip]
+                    InputAdapter
+                      Exchange [item_sk,store_name,store_zip] #2
+                        WholeStageCodegen (42)
+                          HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum]
+                            InputAdapter
+                              Exchange [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year] #3
+                                WholeStageCodegen (41)
+                                  HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum]
+                                    Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name]
+                                      BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                        Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                          BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk]
+                                            Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                              BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk]
+                                                Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                                  SortMergeJoin [c_current_addr_sk,ca_address_sk]
+                                                    InputAdapter
+                                                      WholeStageCodegen (35)
+                                                        Sort [c_current_addr_sk]
+                                                          InputAdapter
+                                                            Exchange [c_current_addr_sk] #4
+                                                              WholeStageCodegen (34)
+                                                                Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                                                  SortMergeJoin [ss_addr_sk,ca_address_sk]
+                                                                    InputAdapter
+                                                                      WholeStageCodegen (31)
+                                                                        Sort [ss_addr_sk]
+                                                                          InputAdapter
+                                                                            Exchange [ss_addr_sk] #5
+                                                                              WholeStageCodegen (30)
+                                                                                Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk]
+                                                                                  BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk]
+                                                                                    Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk]
+                                                                                      BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                                                                        Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year]
+                                                                                          BroadcastHashJoin [ss_promo_sk,p_promo_sk]
+                                                                                            Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year]
+                                                                                              SortMergeJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status]
+                                                                                                InputAdapter
+                                                                                                  WholeStageCodegen (24)
+                                                                                                    Sort [c_current_cdemo_sk]
+                                                                                                      InputAdapter
+                                                                                                        Exchange [c_current_cdemo_sk] #6
+                                                                                                          WholeStageCodegen (23)
+                                                                                                            Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status]
+                                                                                                              SortMergeJoin [ss_cdemo_sk,cd_demo_sk]
+                                                                                                                InputAdapter
+                                                                                                                  WholeStageCodegen (20)
+                                                                                                                    Sort [ss_cdemo_sk]
+                                                                                                                      InputAdapter
+                                                                                                                        Exchange [ss_cdemo_sk] #7
+                                                                                                                          WholeStageCodegen (19)
+                                                                                                                            Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year]
+                                                                                                                              BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk]
+                                                                                                                                Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year]
+                                                                                                                                  BroadcastHashJoin [c_first_sales_date_sk,d_date_sk]
+                                                                                                                                    Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk]
+                                                                                                                                      SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                                                                                                                        InputAdapter
+                                                                                                                                          WholeStageCodegen (14)
+                                                                                                                                            Sort [ss_customer_sk]
+                                                                                                                                              InputAdapter
+                                                                                                                                                Exchange [ss_customer_sk] #8
+                                                                                                                                                  WholeStageCodegen (13)
+                                                                                                                                                    Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip]
+                                                                                                                                                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                                                                                                                        Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year]
+                                                                                                                                                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                                                                                            Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+                                                                                                                                                              BroadcastHashJoin [ss_item_sk,cs_item_sk]
+                                                                                                                                                                Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+                                                                                                                                                                  SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number]
+                                                                                                                                                                    InputAdapter
+                                                                                                                                                                      WholeStageCodegen (2)
+                                                                                                                                                                        Sort [ss_item_sk,ss_ticket_number]
+                                                                                                                                                                          InputAdapter
+                                                                                                                                                                            Exchange [ss_item_sk,ss_ticket_number] #9
+                                                                                                                                                                              WholeStageCodegen (1)
+                                                                                                                                                                                Filter [ss_item_sk,ss_ticket_number,ss_store_sk,ss_customer_sk,ss_cdemo_sk,ss_promo_sk,ss_hdemo_sk,ss_addr_sk]
+                                                                                                                                                                                  ColumnarToRow
+                                                                                                                                                                                    InputAdapter
+                                                                                                                                                                                      Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+                                                                                                                                                                                        SubqueryBroadcast [d_date_sk] #1
+                                                                                                                                                                                          ReusedExchange [d_date_sk,d_year] #10
+                                                                                                                                                                    InputAdapter
+                                                                                                                                                                      WholeStageCodegen (4)
+                                                                                                                                                                        Sort [sr_item_sk,sr_ticket_number]
+                                                                                                                                                                          InputAdapter
+                                                                                                                                                                            Exchange [sr_item_sk,sr_ticket_number] #11
+                                                                                                                                                                              WholeStageCodegen (3)
+                                                                                                                                                                                Project [sr_item_sk,sr_ticket_number]
+                                                                                                                                                                                  Filter [sr_item_sk,sr_ticket_number]
+                                                                                                                                                                                    ColumnarToRow
+                                                                                                                                                                                      InputAdapter
+                                                                                                                                                                                        Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk]
+                                                                                                                                                                InputAdapter
+                                                                                                                                                                  BroadcastExchange #12
+                                                                                                                                                                    WholeStageCodegen (10)
+                                                                                                                                                                      Project [cs_item_sk]
+                                                                                                                                                                        Filter [sum(cs_ext_list_price),sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash as decimal(8,2))) + promote_precision(cast(cr_reversed_charge as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit as decimal(9,2)))), DecimalType(9,2), true))]
+                                                                                                                                                                          HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash as decimal(8,2))) + promote_precision(cast(cr_reversed_charge as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit as decimal(9,2)))), DecimalType(9,2), true)),sum(cs_ext_list_price),sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash as decimal(8,2))) + promote_precision(cast(cr_reversed_charge as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit as decimal(9,2)))), DecimalType(9,2), true)),sum,sum,isEmpty]
+                                                                                                                                                                            InputAdapter
+                                                                                                                                                                              Exchange [cs_item_sk] #13
+                                                                                                                                                                                WholeStageCodegen (9)
+                                                                                                                                                                                  HashAggregate [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [sum,sum,isEmpty,sum,sum,isEmpty]
+                                                                                                                                                                                    Project [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit]
+                                                                                                                                                                                      SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number]
+                                                                                                                                                                                        InputAdapter
+                                                                                                                                                                                          WholeStageCodegen (6)
+                                                                                                                                                                                            Sort [cs_item_sk,cs_order_number]
+                                                                                                                                                                                              InputAdapter
+                                                                                                                                                                                                Exchange [cs_item_sk,cs_order_number] #14
+                                                                                                                                                                                                  WholeStageCodegen (5)
+                                                                                                                                                                                                    Project [cs_item_sk,cs_order_number,cs_ext_list_price]
+                                                                                                                                                                                                      Filter [cs_item_sk,cs_order_number]
+                                                                                                                                                                                                        ColumnarToRow
+                                                                                                                                                                                                          InputAdapter
+                                                                                                                                                                                                            Scan parquet default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk]
+                                                                                                                                                                                        InputAdapter
+                                                                                                                                                                                          WholeStageCodegen (8)
+                                                                                                                                                                                            Sort [cr_item_sk,cr_order_number]
+                                                                                                                                                                                              InputAdapter
+                                                                                                                                                                                                Exchange [cr_item_sk,cr_order_number] #15
+                                                                                                                                                                                                  WholeStageCodegen (7)
+                                                                                                                                                                                                    Project [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit]
+                                                                                                                                                                                                      Filter [cr_item_sk,cr_order_number]
+                                                                                                                                                                                                        ColumnarToRow
+                                                                                                                                                                                                          InputAdapter
+                                                                                                                                                                                                            Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk]
+                                                                                                                                                            InputAdapter
+                                                                                                                                                              BroadcastExchange #10
+                                                                                                                                                                WholeStageCodegen (11)
+                                                                                                                                                                  Filter [d_year,d_date_sk]
+                                                                                                                                                                    ColumnarToRow
+                                                                                                                                                                      InputAdapter
+                                                                                                                                                                        Scan parquet default.date_dim [d_date_sk,d_year]
+                                                                                                                                                        InputAdapter
+                                                                                                                                                          BroadcastExchange #16
+                                                                                                                                                            WholeStageCodegen (12)
+                                                                                                                                                              Filter [s_store_sk,s_store_name,s_zip]
+                                                                                                                                                                ColumnarToRow
+                                                                                                                                                                  InputAdapter
+                                                                                                                                                                    Scan parquet default.store [s_store_sk,s_store_name,s_zip]
+                                                                                                                                        InputAdapter
+                                                                                                                                          WholeStageCodegen (16)
+                                                                                                                                            Sort [c_customer_sk]
+                                                                                                                                              InputAdapter
+                                                                                                                                                Exchange [c_customer_sk] #17
+                                                                                                                                                  WholeStageCodegen (15)
+                                                                                                                                                    Filter [c_customer_sk,c_first_sales_date_sk,c_first_shipto_date_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk]
+                                                                                                                                                      ColumnarToRow
+                                                                                                                                                        InputAdapter
+                                                                                                                                                          Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk]
+                                                                                                                                    InputAdapter
+                                                                                                                                      BroadcastExchange #18
+                                                                                                                                        WholeStageCodegen (17)
+                                                                                                                                          Filter [d_date_sk]
+                                                                                                                                            ColumnarToRow
+                                                                                                                                              InputAdapter
+                                                                                                                                                Scan parquet default.date_dim [d_date_sk,d_year]
+                                                                                                                                InputAdapter
+                                                                                                                                  ReusedExchange [d_date_sk,d_year] #18
+                                                                                                                InputAdapter
+                                                                                                                  WholeStageCodegen (22)
+                                                                                                                    Sort [cd_demo_sk]
+                                                                                                                      InputAdapter
+                                                                                                                        Exchange [cd_demo_sk] #19
+                                                                                                                          WholeStageCodegen (21)
+                                                                                                                            Filter [cd_demo_sk,cd_marital_status]
+                                                                                                                              ColumnarToRow
+                                                                                                                                InputAdapter
+                                                                                                                                  Scan parquet default.customer_demographics [cd_demo_sk,cd_marital_status]
+                                                                                                InputAdapter
+                                                                                                  WholeStageCodegen (26)
+                                                                                                    Sort [cd_demo_sk]
+                                                                                                      InputAdapter
+                                                                                                        ReusedExchange [cd_demo_sk,cd_marital_status] #19
+                                                                                            InputAdapter
+                                                                                              BroadcastExchange #20
+                                                                                                WholeStageCodegen (27)
+                                                                                                  Filter [p_promo_sk]
+                                                                                                    ColumnarToRow
+                                                                                                      InputAdapter
+                                                                                                        Scan parquet default.promotion [p_promo_sk]
+                                                                                        InputAdapter
+                                                                                          BroadcastExchange #21
+                                                                                            WholeStageCodegen (28)
+                                                                                              Filter [hd_demo_sk,hd_income_band_sk]
+                                                                                                ColumnarToRow
+                                                                                                  InputAdapter
+                                                                                                    Scan parquet default.household_demographics [hd_demo_sk,hd_income_band_sk]
+                                                                                    InputAdapter
+                                                                                      ReusedExchange [hd_demo_sk,hd_income_band_sk] #21
+                                                                    InputAdapter
+                                                                      WholeStageCodegen (33)
+                                                                        Sort [ca_address_sk]
+                                                                          InputAdapter
+                                                                            Exchange [ca_address_sk] #22
+                                                                              WholeStageCodegen (32)
+                                                                                Filter [ca_address_sk]
+                                                                                  ColumnarToRow
+                                                                                    InputAdapter
+                                                                                      Scan parquet default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                                    InputAdapter
+                                                      WholeStageCodegen (37)
+                                                        Sort [ca_address_sk]
+                                                          InputAdapter
+                                                            ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #22
+                                                InputAdapter
+                                                  BroadcastExchange #23
+                                                    WholeStageCodegen (38)
+                                                      Filter [ib_income_band_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.income_band [ib_income_band_sk]
+                                            InputAdapter
+                                              ReusedExchange [ib_income_band_sk] #23
+                                        InputAdapter
+                                          BroadcastExchange #24
+                                            WholeStageCodegen (40)
+                                              Project [i_item_sk,i_product_name]
+                                                Filter [i_current_price,i_color,i_item_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.item [i_item_sk,i_current_price,i_color,i_product_name]
+              InputAdapter
+                WholeStageCodegen (86)
+                  Sort [item_sk,store_name,store_zip]
+                    InputAdapter
+                      Exchange [item_sk,store_name,store_zip] #25
+                        WholeStageCodegen (85)
+                          HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum]
+                            InputAdapter
+                              Exchange [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year] #26
+                                WholeStageCodegen (84)
+                                  HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum]
+                                    Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name]
+                                      BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                        Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                          BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk]
+                                            Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                              BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk]
+                                                Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                                  SortMergeJoin [c_current_addr_sk,ca_address_sk]
+                                                    InputAdapter
+                                                      WholeStageCodegen (78)
+                                                        Sort [c_current_addr_sk]
+                                                          InputAdapter
+                                                            Exchange [c_current_addr_sk] #27
+                                                              WholeStageCodegen (77)
+                                                                Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                                                  SortMergeJoin [ss_addr_sk,ca_address_sk]
+                                                                    InputAdapter
+                                                                      WholeStageCodegen (74)
+                                                                        Sort [ss_addr_sk]
+                                                                          InputAdapter
+                                                                            Exchange [ss_addr_sk] #28
+                                                                              WholeStageCodegen (73)
+                                                                                Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk]
+                                                                                  BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk]
+                                                                                    Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk]
+                                                                                      BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                                                                        Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year]
+                                                                                          BroadcastHashJoin [ss_promo_sk,p_promo_sk]
+                                                                                            Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year]
+                                                                                              SortMergeJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status]
+                                                                                                InputAdapter
+                                                                                                  WholeStageCodegen (67)
+                                                                                                    Sort [c_current_cdemo_sk]
+                                                                                                      InputAdapter
+                                                                                                        Exchange [c_current_cdemo_sk] #29
+                                                                                                          WholeStageCodegen (66)
+                                                                                                            Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status]
+                                                                                                              SortMergeJoin [ss_cdemo_sk,cd_demo_sk]
+                                                                                                                InputAdapter
+                                                                                                                  WholeStageCodegen (63)
+                                                                                                                    Sort [ss_cdemo_sk]
+                                                                                                                      InputAdapter
+                                                                                                                        Exchange [ss_cdemo_sk] #30
+                                                                                                                          WholeStageCodegen (62)
+                                                                                                                            Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year]
+                                                                                                                              BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk]
+                                                                                                                                Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year]
+                                                                                                                                  BroadcastHashJoin [c_first_sales_date_sk,d_date_sk]
+                                                                                                                                    Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk]
+                                                                                                                                      SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                                                                                                                        InputAdapter
+                                                                                                                                          WholeStageCodegen (57)
+                                                                                                                                            Sort [ss_customer_sk]
+                                                                                                                                              InputAdapter
+                                                                                                                                                Exchange [ss_customer_sk] #31
+                                                                                                                                                  WholeStageCodegen (56)
+                                                                                                                                                    Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip]
+                                                                                                                                                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                                                                                                                        Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year]
+                                                                                                                                                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                                                                                            Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+                                                                                                                                                              BroadcastHashJoin [ss_item_sk,cs_item_sk]
+                                                                                                                                                                Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+                                                                                                                                                                  SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number]
+                                                                                                                                                                    InputAdapter
+                                                                                                                                                                      WholeStageCodegen (45)
+                                                                                                                                                                        Sort [ss_item_sk,ss_ticket_number]
+                                                                                                                                                                          InputAdapter
+                                                                                                                                                                            Exchange [ss_item_sk,ss_ticket_number] #32
+                                                                                                                                                                              WholeStageCodegen (44)
+                                                                                                                                                                                Filter [ss_item_sk,ss_ticket_number,ss_store_sk,ss_customer_sk,ss_cdemo_sk,ss_promo_sk,ss_hdemo_sk,ss_addr_sk]
+                                                                                                                                                                                  ColumnarToRow
+                                                                                                                                                                                    InputAdapter
+                                                                                                                                                                                      Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+                                                                                                                                                                                        SubqueryBroadcast [d_date_sk] #2
+                                                                                                                                                                                          ReusedExchange [d_date_sk,d_year] #33
+                                                                                                                                                                    InputAdapter
+                                                                                                                                                                      WholeStageCodegen (47)
+                                                                                                                                                                        Sort [sr_item_sk,sr_ticket_number]
+                                                                                                                                                                          InputAdapter
+                                                                                                                                                                            ReusedExchange [sr_item_sk,sr_ticket_number] #11
+                                                                                                                                                                InputAdapter
+                                                                                                                                                                  ReusedExchange [cs_item_sk] #12
+                                                                                                                                                            InputAdapter
+                                                                                                                                                              BroadcastExchange #33
+                                                                                                                                                                WholeStageCodegen (54)
+                                                                                                                                                                  Filter [d_year,d_date_sk]
+                                                                                                                                                                    ColumnarToRow
+                                                                                                                                                                      InputAdapter
+                                                                                                                                                                        Scan parquet default.date_dim [d_date_sk,d_year]
+                                                                                                                                                        InputAdapter
+                                                                                                                                                          ReusedExchange [s_store_sk,s_store_name,s_zip] #16
+                                                                                                                                        InputAdapter
+                                                                                                                                          WholeStageCodegen (59)
+                                                                                                                                            Sort [c_customer_sk]
+                                                                                                                                              InputAdapter
+                                                                                                                                                ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #17
+                                                                                                                                    InputAdapter
+                                                                                                                                      ReusedExchange [d_date_sk,d_year] #18
+                                                                                                                                InputAdapter
+                                                                                                                                  ReusedExchange [d_date_sk,d_year] #18
+                                                                                                                InputAdapter
+                                                                                                                  WholeStageCodegen (65)
+                                                                                                                    Sort [cd_demo_sk]
+                                                                                                                      InputAdapter
+                                                                                                                        ReusedExchange [cd_demo_sk,cd_marital_status] #19
+                                                                                                InputAdapter
+                                                                                                  WholeStageCodegen (69)
+                                                                                                    Sort [cd_demo_sk]
+                                                                                                      InputAdapter
+                                                                                                        ReusedExchange [cd_demo_sk,cd_marital_status] #19
+                                                                                            InputAdapter
+                                                                                              ReusedExchange [p_promo_sk] #20
+                                                                                        InputAdapter
+                                                                                          ReusedExchange [hd_demo_sk,hd_income_band_sk] #21
+                                                                                    InputAdapter
+                                                                                      ReusedExchange [hd_demo_sk,hd_income_band_sk] #21
+                                                                    InputAdapter
+                                                                      WholeStageCodegen (76)
+                                                                        Sort [ca_address_sk]
+                                                                          InputAdapter
+                                                                            ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #22
+                                                    InputAdapter
+                                                      WholeStageCodegen (80)
+                                                        Sort [ca_address_sk]
+                                                          InputAdapter
+                                                            ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #22
+                                                InputAdapter
+                                                  ReusedExchange [ib_income_band_sk] #23
+                                            InputAdapter
+                                              ReusedExchange [ib_income_band_sk] #23
+                                        InputAdapter
+                                          ReusedExchange [i_item_sk,i_product_name] #24
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt
new file mode 100644
index 0000000000000..8a0e66f7f44ed
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt
@@ -0,0 +1,1038 @@
+== Physical Plan ==
+* Sort (189)
++- Exchange (188)
+   +- * Project (187)
+      +- * SortMergeJoin Inner (186)
+         :- * Sort (114)
+         :  +- Exchange (113)
+         :     +- * HashAggregate (112)
+         :        +- * HashAggregate (111)
+         :           +- * Project (110)
+         :              +- * BroadcastHashJoin Inner BuildRight (109)
+         :                 :- * Project (103)
+         :                 :  +- * BroadcastHashJoin Inner BuildRight (102)
+         :                 :     :- * Project (100)
+         :                 :     :  +- * BroadcastHashJoin Inner BuildRight (99)
+         :                 :     :     :- * Project (94)
+         :                 :     :     :  +- * BroadcastHashJoin Inner BuildRight (93)
+         :                 :     :     :     :- * Project (91)
+         :                 :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (90)
+         :                 :     :     :     :     :- * Project (85)
+         :                 :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (84)
+         :                 :     :     :     :     :     :- * Project (82)
+         :                 :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (81)
+         :                 :     :     :     :     :     :     :- * Project (76)
+         :                 :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (75)
+         :                 :     :     :     :     :     :     :     :- * Project (70)
+         :                 :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (69)
+         :                 :     :     :     :     :     :     :     :     :- * Project (67)
+         :                 :     :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (66)
+         :                 :     :     :     :     :     :     :     :     :     :- * Project (61)
+         :                 :     :     :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (60)
+         :                 :     :     :     :     :     :     :     :     :     :     :- * Project (58)
+         :                 :     :     :     :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (57)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :- * Project (52)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (51)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :- * Project (46)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (45)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :- * Project (40)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (39)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :- * Project (34)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :  +- * SortMergeJoin Inner (33)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :- * Sort (12)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :  +- Exchange (11)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     +- * Project (10)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :        +- * BroadcastHashJoin Inner BuildLeft (9)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :           :- BroadcastExchange (4)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :           :  +- * Filter (3)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :           :     +- * ColumnarToRow (2)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :           :        +- Scan parquet default.store_sales (1)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :           +- * Project (8)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :              +- * Filter (7)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                 +- * ColumnarToRow (6)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                    +- Scan parquet default.store_returns (5)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     +- * Sort (32)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :        +- * Project (31)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :           +- * Filter (30)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :              +- * HashAggregate (29)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                 +- Exchange (28)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                    +- * HashAggregate (27)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                       +- * Project (26)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                          +- * SortMergeJoin Inner (25)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                             :- * Sort (18)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                             :  +- Exchange (17)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                             :     +- * Project (16)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                             :        +- * Filter (15)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                             :           +- * ColumnarToRow (14)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                             :              +- Scan parquet default.catalog_sales (13)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                             +- * Sort (24)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                                +- Exchange (23)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                                   +- * Project (22)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                                      +- * Filter (21)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                                         +- * ColumnarToRow (20)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                                            +- Scan parquet default.catalog_returns (19)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :     +- BroadcastExchange (38)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :        +- * Filter (37)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :           +- * ColumnarToRow (36)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     :              +- Scan parquet default.date_dim (35)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :     +- BroadcastExchange (44)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :        +- * Filter (43)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :           +- * ColumnarToRow (42)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     :              +- Scan parquet default.store (41)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :     +- BroadcastExchange (50)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :        +- * Filter (49)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :           +- * ColumnarToRow (48)
+         :                 :     :     :     :     :     :     :     :     :     :     :     :              +- Scan parquet default.customer (47)
+         :                 :     :     :     :     :     :     :     :     :     :     :     +- BroadcastExchange (56)
+         :                 :     :     :     :     :     :     :     :     :     :     :        +- * Filter (55)
+         :                 :     :     :     :     :     :     :     :     :     :     :           +- * ColumnarToRow (54)
+         :                 :     :     :     :     :     :     :     :     :     :     :              +- Scan parquet default.date_dim (53)
+         :                 :     :     :     :     :     :     :     :     :     :     +- ReusedExchange (59)
+         :                 :     :     :     :     :     :     :     :     :     +- BroadcastExchange (65)
+         :                 :     :     :     :     :     :     :     :     :        +- * Filter (64)
+         :                 :     :     :     :     :     :     :     :     :           +- * ColumnarToRow (63)
+         :                 :     :     :     :     :     :     :     :     :              +- Scan parquet default.customer_demographics (62)
+         :                 :     :     :     :     :     :     :     :     +- ReusedExchange (68)
+         :                 :     :     :     :     :     :     :     +- BroadcastExchange (74)
+         :                 :     :     :     :     :     :     :        +- * Filter (73)
+         :                 :     :     :     :     :     :     :           +- * ColumnarToRow (72)
+         :                 :     :     :     :     :     :     :              +- Scan parquet default.promotion (71)
+         :                 :     :     :     :     :     :     +- BroadcastExchange (80)
+         :                 :     :     :     :     :     :        +- * Filter (79)
+         :                 :     :     :     :     :     :           +- * ColumnarToRow (78)
+         :                 :     :     :     :     :     :              +- Scan parquet default.household_demographics (77)
+         :                 :     :     :     :     :     +- ReusedExchange (83)
+         :                 :     :     :     :     +- BroadcastExchange (89)
+         :                 :     :     :     :        +- * Filter (88)
+         :                 :     :     :     :           +- * ColumnarToRow (87)
+         :                 :     :     :     :              +- Scan parquet default.customer_address (86)
+         :                 :     :     :     +- ReusedExchange (92)
+         :                 :     :     +- BroadcastExchange (98)
+         :                 :     :        +- * Filter (97)
+         :                 :     :           +- * ColumnarToRow (96)
+         :                 :     :              +- Scan parquet default.income_band (95)
+         :                 :     +- ReusedExchange (101)
+         :                 +- BroadcastExchange (108)
+         :                    +- * Project (107)
+         :                       +- * Filter (106)
+         :                          +- * ColumnarToRow (105)
+         :                             +- Scan parquet default.item (104)
+         +- * Sort (185)
+            +- Exchange (184)
+               +- * HashAggregate (183)
+                  +- * HashAggregate (182)
+                     +- * Project (181)
+                        +- * BroadcastHashJoin Inner BuildRight (180)
+                           :- * Project (178)
+                           :  +- * BroadcastHashJoin Inner BuildRight (177)
+                           :     :- * Project (175)
+                           :     :  +- * BroadcastHashJoin Inner BuildRight (174)
+                           :     :     :- * Project (172)
+                           :     :     :  +- * BroadcastHashJoin Inner BuildRight (171)
+                           :     :     :     :- * Project (169)
+                           :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (168)
+                           :     :     :     :     :- * Project (166)
+                           :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (165)
+                           :     :     :     :     :     :- * Project (163)
+                           :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (162)
+                           :     :     :     :     :     :     :- * Project (160)
+                           :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (159)
+                           :     :     :     :     :     :     :     :- * Project (157)
+                           :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (156)
+                           :     :     :     :     :     :     :     :     :- * Project (154)
+                           :     :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (153)
+                           :     :     :     :     :     :     :     :     :     :- * Project (151)
+                           :     :     :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (150)
+                           :     :     :     :     :     :     :     :     :     :     :- * Project (148)
+                           :     :     :     :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (147)
+                           :     :     :     :     :     :     :     :     :     :     :     :- * Project (145)
+                           :     :     :     :     :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (144)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :- * Project (142)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (141)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :- * Project (139)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (138)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :- * Project (133)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :  +- * SortMergeJoin Inner (132)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :- * Sort (126)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :  +- Exchange (125)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     +- * Project (124)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :        +- * BroadcastHashJoin Inner BuildLeft (123)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :           :- BroadcastExchange (118)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :           :  +- * Filter (117)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :           :     +- * ColumnarToRow (116)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :           :        +- Scan parquet default.store_sales (115)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :           +- * Project (122)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :              +- * Filter (121)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                 +- * ColumnarToRow (120)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                    +- Scan parquet default.store_returns (119)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :     +- * Sort (131)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :        +- * Project (130)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :           +- * Filter (129)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :              +- * HashAggregate (128)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     :                 +- ReusedExchange (127)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :     +- BroadcastExchange (137)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :        +- * Filter (136)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :           +- * ColumnarToRow (135)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     :              +- Scan parquet default.date_dim (134)
+                           :     :     :     :     :     :     :     :     :     :     :     :     :     +- ReusedExchange (140)
+                           :     :     :     :     :     :     :     :     :     :     :     :     +- ReusedExchange (143)
+                           :     :     :     :     :     :     :     :     :     :     :     +- ReusedExchange (146)
+                           :     :     :     :     :     :     :     :     :     :     +- ReusedExchange (149)
+                           :     :     :     :     :     :     :     :     :     +- ReusedExchange (152)
+                           :     :     :     :     :     :     :     :     +- ReusedExchange (155)
+                           :     :     :     :     :     :     :     +- ReusedExchange (158)
+                           :     :     :     :     :     :     +- ReusedExchange (161)
+                           :     :     :     :     :     +- ReusedExchange (164)
+                           :     :     :     :     +- ReusedExchange (167)
+                           :     :     :     +- ReusedExchange (170)
+                           :     :     +- ReusedExchange (173)
+                           :     +- ReusedExchange (176)
+                           +- ReusedExchange (179)
+
+
+(1) Scan parquet default.store_sales
+Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_sold_date_sk#12 IN dynamicpruning#13)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_cdemo_sk:int,ss_hdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_promo_sk:int,ss_ticket_number:int,ss_wholesale_cost:decimal(7,2),ss_list_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+
+(3) Filter [codegen id : 1]
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AND isnotnull(ss_store_sk#6)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_cdemo_sk#3)) AND isnotnull(ss_promo_sk#7)) AND isnotnull(ss_hdemo_sk#4)) AND isnotnull(ss_addr_sk#5))
+
+(4) BroadcastExchange
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[7, int, false] as bigint) & 4294967295))),false), [id=#14]
+
+(5) Scan parquet default.store_returns
+Output [3]: [sr_item_sk#15, sr_ticket_number#16, sr_returned_date_sk#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int>
+
+(6) ColumnarToRow
+Input [3]: [sr_item_sk#15, sr_ticket_number#16, sr_returned_date_sk#17]
+
+(7) Filter
+Input [3]: [sr_item_sk#15, sr_ticket_number#16, sr_returned_date_sk#17]
+Condition : (isnotnull(sr_item_sk#15) AND isnotnull(sr_ticket_number#16))
+
+(8) Project
+Output [2]: [sr_item_sk#15, sr_ticket_number#16]
+Input [3]: [sr_item_sk#15, sr_ticket_number#16, sr_returned_date_sk#17]
+
+(9) BroadcastHashJoin [codegen id : 2]
+Left keys [2]: [ss_item_sk#1, ss_ticket_number#8]
+Right keys [2]: [sr_item_sk#15, sr_ticket_number#16]
+Join condition: None
+
+(10) Project [codegen id : 2]
+Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#15, sr_ticket_number#16]
+
+(11) Exchange
+Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(12) Sort [codegen id : 3]
+Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Arguments: [ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(13) Scan parquet default.catalog_sales
+Output [4]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21, cs_sold_date_sk#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_sales]
+PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)]
+ReadSchema: struct<cs_item_sk:int,cs_order_number:int,cs_ext_list_price:decimal(7,2)>
+
+(14) ColumnarToRow [codegen id : 4]
+Input [4]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21, cs_sold_date_sk#22]
+
+(15) Filter [codegen id : 4]
+Input [4]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21, cs_sold_date_sk#22]
+Condition : (isnotnull(cs_item_sk#19) AND isnotnull(cs_order_number#20))
+
+(16) Project [codegen id : 4]
+Output [3]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21]
+Input [4]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21, cs_sold_date_sk#22]
+
+(17) Exchange
+Input [3]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21]
+Arguments: hashpartitioning(cs_item_sk#19, cs_order_number#20, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(18) Sort [codegen id : 5]
+Input [3]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21]
+Arguments: [cs_item_sk#19 ASC NULLS FIRST, cs_order_number#20 ASC NULLS FIRST], false, 0
+
+(19) Scan parquet default.catalog_returns
+Output [6]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28, cr_returned_date_sk#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int,cr_refunded_cash:decimal(7,2),cr_reversed_charge:decimal(7,2),cr_store_credit:decimal(7,2)>
+
+(20) ColumnarToRow [codegen id : 6]
+Input [6]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28, cr_returned_date_sk#29]
+
+(21) Filter [codegen id : 6]
+Input [6]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28, cr_returned_date_sk#29]
+Condition : (isnotnull(cr_item_sk#24) AND isnotnull(cr_order_number#25))
+
+(22) Project [codegen id : 6]
+Output [5]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28]
+Input [6]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28, cr_returned_date_sk#29]
+
+(23) Exchange
+Input [5]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28]
+Arguments: hashpartitioning(cr_item_sk#24, cr_order_number#25, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(24) Sort [codegen id : 7]
+Input [5]: [cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28]
+Arguments: [cr_item_sk#24 ASC NULLS FIRST, cr_order_number#25 ASC NULLS FIRST], false, 0
+
+(25) SortMergeJoin [codegen id : 8]
+Left keys [2]: [cs_item_sk#19, cs_order_number#20]
+Right keys [2]: [cr_item_sk#24, cr_order_number#25]
+Join condition: None
+
+(26) Project [codegen id : 8]
+Output [5]: [cs_item_sk#19, cs_ext_list_price#21, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28]
+Input [8]: [cs_item_sk#19, cs_order_number#20, cs_ext_list_price#21, cr_item_sk#24, cr_order_number#25, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28]
+
+(27) HashAggregate [codegen id : 8]
+Input [5]: [cs_item_sk#19, cs_ext_list_price#21, cr_refunded_cash#26, cr_reversed_charge#27, cr_store_credit#28]
+Keys [1]: [cs_item_sk#19]
+Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#21)), partial_sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))]
+Aggregate Attributes [3]: [sum#31, sum#32, isEmpty#33]
+Results [4]: [cs_item_sk#19, sum#34, sum#35, isEmpty#36]
+
+(28) Exchange
+Input [4]: [cs_item_sk#19, sum#34, sum#35, isEmpty#36]
+Arguments: hashpartitioning(cs_item_sk#19, 5), ENSURE_REQUIREMENTS, [id=#37]
+
+(29) HashAggregate [codegen id : 9]
+Input [4]: [cs_item_sk#19, sum#34, sum#35, isEmpty#36]
+Keys [1]: [cs_item_sk#19]
+Functions [2]: [sum(UnscaledValue(cs_ext_list_price#21)), sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))]
+Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#21))#38, sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))#39]
+Results [3]: [cs_item_sk#19, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#21))#38,17,2) AS sum(cs_ext_list_price#21)#40, sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))#39 AS sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))#41]
+
+(30) Filter [codegen id : 9]
+Input [3]: [cs_item_sk#19, sum(cs_ext_list_price#21)#40, sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))#41]
+Condition : (isnotnull(sum(cs_ext_list_price#21)#40) AND (cast(sum(cs_ext_list_price#21)#40 as decimal(21,2)) > CheckOverflow((2.00 * promote_precision(sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))#41)), DecimalType(21,2), true)))
+
+(31) Project [codegen id : 9]
+Output [1]: [cs_item_sk#19]
+Input [3]: [cs_item_sk#19, sum(cs_ext_list_price#21)#40, sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#26 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#27 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#28 as decimal(9,2)))), DecimalType(9,2), true))#41]
+
+(32) Sort [codegen id : 9]
+Input [1]: [cs_item_sk#19]
+Arguments: [cs_item_sk#19 ASC NULLS FIRST], false, 0
+
+(33) SortMergeJoin [codegen id : 25]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [cs_item_sk#19]
+Join condition: None
+
+(34) Project [codegen id : 25]
+Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12]
+Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#19]
+
+(35) Scan parquet default.date_dim
+Output [2]: [d_date_sk#42, d_year#43]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(36) ColumnarToRow [codegen id : 10]
+Input [2]: [d_date_sk#42, d_year#43]
+
+(37) Filter [codegen id : 10]
+Input [2]: [d_date_sk#42, d_year#43]
+Condition : ((isnotnull(d_year#43) AND (d_year#43 = 1999)) AND isnotnull(d_date_sk#42))
+
+(38) BroadcastExchange
+Input [2]: [d_date_sk#42, d_year#43]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#44]
+
+(39) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_sold_date_sk#12]
+Right keys [1]: [d_date_sk#42]
+Join condition: None
+
+(40) Project [codegen id : 25]
+Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43]
+Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#42, d_year#43]
+
+(41) Scan parquet default.store
+Output [3]: [s_store_sk#45, s_store_name#46, s_zip#47]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_zip)]
+ReadSchema: struct<s_store_sk:int,s_store_name:string,s_zip:string>
+
+(42) ColumnarToRow [codegen id : 11]
+Input [3]: [s_store_sk#45, s_store_name#46, s_zip#47]
+
+(43) Filter [codegen id : 11]
+Input [3]: [s_store_sk#45, s_store_name#46, s_zip#47]
+Condition : ((isnotnull(s_store_sk#45) AND isnotnull(s_store_name#46)) AND isnotnull(s_zip#47))
+
+(44) BroadcastExchange
+Input [3]: [s_store_sk#45, s_store_name#46, s_zip#47]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#48]
+
+(45) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_store_sk#6]
+Right keys [1]: [s_store_sk#45]
+Join condition: None
+
+(46) Project [codegen id : 25]
+Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47]
+Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_sk#45, s_store_name#46, s_zip#47]
+
+(47) Scan parquet default.customer
+Output [6]: [c_customer_sk#49, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, c_first_shipto_date_sk#53, c_first_sales_date_sk#54]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)]
+ReadSchema: struct<c_customer_sk:int,c_current_cdemo_sk:int,c_current_hdemo_sk:int,c_current_addr_sk:int,c_first_shipto_date_sk:int,c_first_sales_date_sk:int>
+
+(48) ColumnarToRow [codegen id : 12]
+Input [6]: [c_customer_sk#49, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, c_first_shipto_date_sk#53, c_first_sales_date_sk#54]
+
+(49) Filter [codegen id : 12]
+Input [6]: [c_customer_sk#49, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, c_first_shipto_date_sk#53, c_first_sales_date_sk#54]
+Condition : (((((isnotnull(c_customer_sk#49) AND isnotnull(c_first_sales_date_sk#54)) AND isnotnull(c_first_shipto_date_sk#53)) AND isnotnull(c_current_cdemo_sk#50)) AND isnotnull(c_current_hdemo_sk#51)) AND isnotnull(c_current_addr_sk#52))
+
+(50) BroadcastExchange
+Input [6]: [c_customer_sk#49, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, c_first_shipto_date_sk#53, c_first_sales_date_sk#54]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#55]
+
+(51) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_customer_sk#2]
+Right keys [1]: [c_customer_sk#49]
+Join condition: None
+
+(52) Project [codegen id : 25]
+Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, c_first_shipto_date_sk#53, c_first_sales_date_sk#54]
+Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_customer_sk#49, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, c_first_shipto_date_sk#53, c_first_sales_date_sk#54]
+
+(53) Scan parquet default.date_dim
+Output [2]: [d_date_sk#56, d_year#57]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(54) ColumnarToRow [codegen id : 13]
+Input [2]: [d_date_sk#56, d_year#57]
+
+(55) Filter [codegen id : 13]
+Input [2]: [d_date_sk#56, d_year#57]
+Condition : isnotnull(d_date_sk#56)
+
+(56) BroadcastExchange
+Input [2]: [d_date_sk#56, d_year#57]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#58]
+
+(57) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [c_first_sales_date_sk#54]
+Right keys [1]: [d_date_sk#56]
+Join condition: None
+
+(58) Project [codegen id : 25]
+Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, c_first_shipto_date_sk#53, d_year#57]
+Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, c_first_shipto_date_sk#53, c_first_sales_date_sk#54, d_date_sk#56, d_year#57]
+
+(59) ReusedExchange [Reuses operator id: 56]
+Output [2]: [d_date_sk#59, d_year#60]
+
+(60) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [c_first_shipto_date_sk#53]
+Right keys [1]: [d_date_sk#59]
+Join condition: None
+
+(61) Project [codegen id : 25]
+Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, d_year#57, d_year#60]
+Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, c_first_shipto_date_sk#53, d_year#57, d_date_sk#59, d_year#60]
+
+(62) Scan parquet default.customer_demographics
+Output [2]: [cd_demo_sk#61, cd_marital_status#62]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status)]
+ReadSchema: struct<cd_demo_sk:int,cd_marital_status:string>
+
+(63) ColumnarToRow [codegen id : 15]
+Input [2]: [cd_demo_sk#61, cd_marital_status#62]
+
+(64) Filter [codegen id : 15]
+Input [2]: [cd_demo_sk#61, cd_marital_status#62]
+Condition : (isnotnull(cd_demo_sk#61) AND isnotnull(cd_marital_status#62))
+
+(65) BroadcastExchange
+Input [2]: [cd_demo_sk#61, cd_marital_status#62]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#63]
+
+(66) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_cdemo_sk#3]
+Right keys [1]: [cd_demo_sk#61]
+Join condition: None
+
+(67) Project [codegen id : 25]
+Output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, d_year#57, d_year#60, cd_marital_status#62]
+Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, d_year#57, d_year#60, cd_demo_sk#61, cd_marital_status#62]
+
+(68) ReusedExchange [Reuses operator id: 65]
+Output [2]: [cd_demo_sk#64, cd_marital_status#65]
+
+(69) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [c_current_cdemo_sk#50]
+Right keys [1]: [cd_demo_sk#64]
+Join condition: NOT (cd_marital_status#62 = cd_marital_status#65)
+
+(70) Project [codegen id : 25]
+Output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_hdemo_sk#51, c_current_addr_sk#52, d_year#57, d_year#60]
+Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_cdemo_sk#50, c_current_hdemo_sk#51, c_current_addr_sk#52, d_year#57, d_year#60, cd_marital_status#62, cd_demo_sk#64, cd_marital_status#65]
+
+(71) Scan parquet default.promotion
+Output [1]: [p_promo_sk#66]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/promotion]
+PushedFilters: [IsNotNull(p_promo_sk)]
+ReadSchema: struct<p_promo_sk:int>
+
+(72) ColumnarToRow [codegen id : 17]
+Input [1]: [p_promo_sk#66]
+
+(73) Filter [codegen id : 17]
+Input [1]: [p_promo_sk#66]
+Condition : isnotnull(p_promo_sk#66)
+
+(74) BroadcastExchange
+Input [1]: [p_promo_sk#66]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#67]
+
+(75) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_promo_sk#7]
+Right keys [1]: [p_promo_sk#66]
+Join condition: None
+
+(76) Project [codegen id : 25]
+Output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_hdemo_sk#51, c_current_addr_sk#52, d_year#57, d_year#60]
+Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_hdemo_sk#51, c_current_addr_sk#52, d_year#57, d_year#60, p_promo_sk#66]
+
+(77) Scan parquet default.household_demographics
+Output [2]: [hd_demo_sk#68, hd_income_band_sk#69]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_income_band_sk:int>
+
+(78) ColumnarToRow [codegen id : 18]
+Input [2]: [hd_demo_sk#68, hd_income_band_sk#69]
+
+(79) Filter [codegen id : 18]
+Input [2]: [hd_demo_sk#68, hd_income_band_sk#69]
+Condition : (isnotnull(hd_demo_sk#68) AND isnotnull(hd_income_band_sk#69))
+
+(80) BroadcastExchange
+Input [2]: [hd_demo_sk#68, hd_income_band_sk#69]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#70]
+
+(81) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_hdemo_sk#4]
+Right keys [1]: [hd_demo_sk#68]
+Join condition: None
+
+(82) Project [codegen id : 25]
+Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_hdemo_sk#51, c_current_addr_sk#52, d_year#57, d_year#60, hd_income_band_sk#69]
+Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_hdemo_sk#51, c_current_addr_sk#52, d_year#57, d_year#60, hd_demo_sk#68, hd_income_band_sk#69]
+
+(83) ReusedExchange [Reuses operator id: 80]
+Output [2]: [hd_demo_sk#71, hd_income_band_sk#72]
+
+(84) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [c_current_hdemo_sk#51]
+Right keys [1]: [hd_demo_sk#71]
+Join condition: None
+
+(85) Project [codegen id : 25]
+Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_addr_sk#52, d_year#57, d_year#60, hd_income_band_sk#69, hd_income_band_sk#72]
+Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_hdemo_sk#51, c_current_addr_sk#52, d_year#57, d_year#60, hd_income_band_sk#69, hd_demo_sk#71, hd_income_band_sk#72]
+
+(86) Scan parquet default.customer_address
+Output [5]: [ca_address_sk#73, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_address]
+PushedFilters: [IsNotNull(ca_address_sk)]
+ReadSchema: struct<ca_address_sk:int,ca_street_number:string,ca_street_name:string,ca_city:string,ca_zip:string>
+
+(87) ColumnarToRow [codegen id : 20]
+Input [5]: [ca_address_sk#73, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77]
+
+(88) Filter [codegen id : 20]
+Input [5]: [ca_address_sk#73, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77]
+Condition : isnotnull(ca_address_sk#73)
+
+(89) BroadcastExchange
+Input [5]: [ca_address_sk#73, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#78]
+
+(90) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_addr_sk#5]
+Right keys [1]: [ca_address_sk#73]
+Join condition: None
+
+(91) Project [codegen id : 25]
+Output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_addr_sk#52, d_year#57, d_year#60, hd_income_band_sk#69, hd_income_band_sk#72, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77]
+Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_addr_sk#52, d_year#57, d_year#60, hd_income_band_sk#69, hd_income_band_sk#72, ca_address_sk#73, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77]
+
+(92) ReusedExchange [Reuses operator id: 89]
+Output [5]: [ca_address_sk#79, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83]
+
+(93) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [c_current_addr_sk#52]
+Right keys [1]: [ca_address_sk#79]
+Join condition: None
+
+(94) Project [codegen id : 25]
+Output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, d_year#57, d_year#60, hd_income_band_sk#69, hd_income_band_sk#72, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83]
+Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, c_current_addr_sk#52, d_year#57, d_year#60, hd_income_band_sk#69, hd_income_band_sk#72, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_address_sk#79, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83]
+
+(95) Scan parquet default.income_band
+Output [1]: [ib_income_band_sk#84]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/income_band]
+PushedFilters: [IsNotNull(ib_income_band_sk)]
+ReadSchema: struct<ib_income_band_sk:int>
+
+(96) ColumnarToRow [codegen id : 22]
+Input [1]: [ib_income_band_sk#84]
+
+(97) Filter [codegen id : 22]
+Input [1]: [ib_income_band_sk#84]
+Condition : isnotnull(ib_income_band_sk#84)
+
+(98) BroadcastExchange
+Input [1]: [ib_income_band_sk#84]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#85]
+
+(99) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [hd_income_band_sk#69]
+Right keys [1]: [ib_income_band_sk#84]
+Join condition: None
+
+(100) Project [codegen id : 25]
+Output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, d_year#57, d_year#60, hd_income_band_sk#72, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83]
+Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, d_year#57, d_year#60, hd_income_band_sk#69, hd_income_band_sk#72, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83, ib_income_band_sk#84]
+
+(101) ReusedExchange [Reuses operator id: 98]
+Output [1]: [ib_income_band_sk#86]
+
+(102) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [hd_income_band_sk#72]
+Right keys [1]: [ib_income_band_sk#86]
+Join condition: None
+
+(103) Project [codegen id : 25]
+Output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, d_year#57, d_year#60, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83]
+Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, d_year#57, d_year#60, hd_income_band_sk#72, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83, ib_income_band_sk#86]
+
+(104) Scan parquet default.item
+Output [4]: [i_item_sk#87, i_current_price#88, i_color#89, i_product_name#90]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_current_price), In(i_color, [purple              ,burlywood           ,indian              ,spring              ,floral              ,medium              ]), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_current_price:decimal(7,2),i_color:string,i_product_name:string>
+
+(105) ColumnarToRow [codegen id : 24]
+Input [4]: [i_item_sk#87, i_current_price#88, i_color#89, i_product_name#90]
+
+(106) Filter [codegen id : 24]
+Input [4]: [i_item_sk#87, i_current_price#88, i_color#89, i_product_name#90]
+Condition : ((((((isnotnull(i_current_price#88) AND i_color#89 IN (purple              ,burlywood           ,indian              ,spring              ,floral              ,medium              )) AND (i_current_price#88 >= 64.00)) AND (i_current_price#88 <= 74.00)) AND (i_current_price#88 >= 65.00)) AND (i_current_price#88 <= 79.00)) AND isnotnull(i_item_sk#87))
+
+(107) Project [codegen id : 24]
+Output [2]: [i_item_sk#87, i_product_name#90]
+Input [4]: [i_item_sk#87, i_current_price#88, i_color#89, i_product_name#90]
+
+(108) BroadcastExchange
+Input [2]: [i_item_sk#87, i_product_name#90]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#91]
+
+(109) BroadcastHashJoin [codegen id : 25]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#87]
+Join condition: None
+
+(110) Project [codegen id : 25]
+Output [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, d_year#57, d_year#60, s_store_name#46, s_zip#47, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83, i_item_sk#87, i_product_name#90]
+Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, s_store_name#46, s_zip#47, d_year#57, d_year#60, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83, i_item_sk#87, i_product_name#90]
+
+(111) HashAggregate [codegen id : 25]
+Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#43, d_year#57, d_year#60, s_store_name#46, s_zip#47, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83, i_item_sk#87, i_product_name#90]
+Keys [15]: [i_product_name#90, i_item_sk#87, s_store_name#46, s_zip#47, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83, d_year#43, d_year#57, d_year#60]
+Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))]
+Aggregate Attributes [4]: [count#92, sum#93, sum#94, sum#95]
+Results [19]: [i_product_name#90, i_item_sk#87, s_store_name#46, s_zip#47, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83, d_year#43, d_year#57, d_year#60, count#96, sum#97, sum#98, sum#99]
+
+(112) HashAggregate [codegen id : 25]
+Input [19]: [i_product_name#90, i_item_sk#87, s_store_name#46, s_zip#47, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83, d_year#43, d_year#57, d_year#60, count#96, sum#97, sum#98, sum#99]
+Keys [15]: [i_product_name#90, i_item_sk#87, s_store_name#46, s_zip#47, ca_street_number#74, ca_street_name#75, ca_city#76, ca_zip#77, ca_street_number#80, ca_street_name#81, ca_city#82, ca_zip#83, d_year#43, d_year#57, d_year#60]
+Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))]
+Aggregate Attributes [4]: [count(1)#100, sum(UnscaledValue(ss_wholesale_cost#9))#101, sum(UnscaledValue(ss_list_price#10))#102, sum(UnscaledValue(ss_coupon_amt#11))#103]
+Results [17]: [i_product_name#90 AS product_name#104, i_item_sk#87 AS item_sk#105, s_store_name#46 AS store_name#106, s_zip#47 AS store_zip#107, ca_street_number#74 AS b_street_number#108, ca_street_name#75 AS b_streen_name#109, ca_city#76 AS b_city#110, ca_zip#77 AS b_zip#111, ca_street_number#80 AS c_street_number#112, ca_street_name#81 AS c_street_name#113, ca_city#82 AS c_city#114, ca_zip#83 AS c_zip#115, d_year#43 AS syear#116, count(1)#100 AS cnt#117, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#9))#101,17,2) AS s1#118, MakeDecimal(sum(UnscaledValue(ss_list_price#10))#102,17,2) AS s2#119, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#11))#103,17,2) AS s3#120]
+
+(113) Exchange
+Input [17]: [product_name#104, item_sk#105, store_name#106, store_zip#107, b_street_number#108, b_streen_name#109, b_city#110, b_zip#111, c_street_number#112, c_street_name#113, c_city#114, c_zip#115, syear#116, cnt#117, s1#118, s2#119, s3#120]
+Arguments: hashpartitioning(item_sk#105, store_name#106, store_zip#107, 5), ENSURE_REQUIREMENTS, [id=#121]
+
+(114) Sort [codegen id : 26]
+Input [17]: [product_name#104, item_sk#105, store_name#106, store_zip#107, b_street_number#108, b_streen_name#109, b_city#110, b_zip#111, c_street_number#112, c_street_name#113, c_city#114, c_zip#115, syear#116, cnt#117, s1#118, s2#119, s3#120]
+Arguments: [item_sk#105 ASC NULLS FIRST, store_name#106 ASC NULLS FIRST, store_zip#107 ASC NULLS FIRST], false, 0
+
+(115) Scan parquet default.store_sales
+Output [12]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_ticket_number#129, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, ss_sold_date_sk#133]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#133), dynamicpruningexpression(ss_sold_date_sk#133 IN dynamicpruning#134)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_cdemo_sk:int,ss_hdemo_sk:int,ss_addr_sk:int,ss_store_sk:int,ss_promo_sk:int,ss_ticket_number:int,ss_wholesale_cost:decimal(7,2),ss_list_price:decimal(7,2),ss_coupon_amt:decimal(7,2)>
+
+(116) ColumnarToRow [codegen id : 27]
+Input [12]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_ticket_number#129, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, ss_sold_date_sk#133]
+
+(117) Filter [codegen id : 27]
+Input [12]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_ticket_number#129, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, ss_sold_date_sk#133]
+Condition : (((((((isnotnull(ss_item_sk#122) AND isnotnull(ss_ticket_number#129)) AND isnotnull(ss_store_sk#127)) AND isnotnull(ss_customer_sk#123)) AND isnotnull(ss_cdemo_sk#124)) AND isnotnull(ss_promo_sk#128)) AND isnotnull(ss_hdemo_sk#125)) AND isnotnull(ss_addr_sk#126))
+
+(118) BroadcastExchange
+Input [12]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_ticket_number#129, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, ss_sold_date_sk#133]
+Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[7, int, false] as bigint) & 4294967295))),false), [id=#135]
+
+(119) Scan parquet default.store_returns
+Output [3]: [sr_item_sk#136, sr_ticket_number#137, sr_returned_date_sk#138]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int>
+
+(120) ColumnarToRow
+Input [3]: [sr_item_sk#136, sr_ticket_number#137, sr_returned_date_sk#138]
+
+(121) Filter
+Input [3]: [sr_item_sk#136, sr_ticket_number#137, sr_returned_date_sk#138]
+Condition : (isnotnull(sr_item_sk#136) AND isnotnull(sr_ticket_number#137))
+
+(122) Project
+Output [2]: [sr_item_sk#136, sr_ticket_number#137]
+Input [3]: [sr_item_sk#136, sr_ticket_number#137, sr_returned_date_sk#138]
+
+(123) BroadcastHashJoin [codegen id : 28]
+Left keys [2]: [ss_item_sk#122, ss_ticket_number#129]
+Right keys [2]: [sr_item_sk#136, sr_ticket_number#137]
+Join condition: None
+
+(124) Project [codegen id : 28]
+Output [11]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, ss_sold_date_sk#133]
+Input [14]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_ticket_number#129, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, ss_sold_date_sk#133, sr_item_sk#136, sr_ticket_number#137]
+
+(125) Exchange
+Input [11]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, ss_sold_date_sk#133]
+Arguments: hashpartitioning(ss_item_sk#122, 5), ENSURE_REQUIREMENTS, [id=#139]
+
+(126) Sort [codegen id : 29]
+Input [11]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, ss_sold_date_sk#133]
+Arguments: [ss_item_sk#122 ASC NULLS FIRST], false, 0
+
+(127) ReusedExchange [Reuses operator id: 28]
+Output [4]: [cs_item_sk#140, sum#141, sum#142, isEmpty#143]
+
+(128) HashAggregate [codegen id : 35]
+Input [4]: [cs_item_sk#140, sum#141, sum#142, isEmpty#143]
+Keys [1]: [cs_item_sk#140]
+Functions [2]: [sum(UnscaledValue(cs_ext_list_price#144)), sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#145 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#146 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#147 as decimal(9,2)))), DecimalType(9,2), true))]
+Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#144))#148, sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#145 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#146 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#147 as decimal(9,2)))), DecimalType(9,2), true))#149]
+Results [3]: [cs_item_sk#140, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#144))#148,17,2) AS sum(cs_ext_list_price#144)#150, sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#145 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#146 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#147 as decimal(9,2)))), DecimalType(9,2), true))#149 AS sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#145 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#146 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#147 as decimal(9,2)))), DecimalType(9,2), true))#151]
+
+(129) Filter [codegen id : 35]
+Input [3]: [cs_item_sk#140, sum(cs_ext_list_price#144)#150, sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#145 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#146 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#147 as decimal(9,2)))), DecimalType(9,2), true))#151]
+Condition : (isnotnull(sum(cs_ext_list_price#144)#150) AND (cast(sum(cs_ext_list_price#144)#150 as decimal(21,2)) > CheckOverflow((2.00 * promote_precision(sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#145 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#146 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#147 as decimal(9,2)))), DecimalType(9,2), true))#151)), DecimalType(21,2), true)))
+
+(130) Project [codegen id : 35]
+Output [1]: [cs_item_sk#140]
+Input [3]: [cs_item_sk#140, sum(cs_ext_list_price#144)#150, sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash#145 as decimal(8,2))) + promote_precision(cast(cr_reversed_charge#146 as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit#147 as decimal(9,2)))), DecimalType(9,2), true))#151]
+
+(131) Sort [codegen id : 35]
+Input [1]: [cs_item_sk#140]
+Arguments: [cs_item_sk#140 ASC NULLS FIRST], false, 0
+
+(132) SortMergeJoin [codegen id : 51]
+Left keys [1]: [ss_item_sk#122]
+Right keys [1]: [cs_item_sk#140]
+Join condition: None
+
+(133) Project [codegen id : 51]
+Output [11]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, ss_sold_date_sk#133]
+Input [12]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, ss_sold_date_sk#133, cs_item_sk#140]
+
+(134) Scan parquet default.date_dim
+Output [2]: [d_date_sk#152, d_year#153]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(135) ColumnarToRow [codegen id : 36]
+Input [2]: [d_date_sk#152, d_year#153]
+
+(136) Filter [codegen id : 36]
+Input [2]: [d_date_sk#152, d_year#153]
+Condition : ((isnotnull(d_year#153) AND (d_year#153 = 2000)) AND isnotnull(d_date_sk#152))
+
+(137) BroadcastExchange
+Input [2]: [d_date_sk#152, d_year#153]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#154]
+
+(138) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [ss_sold_date_sk#133]
+Right keys [1]: [d_date_sk#152]
+Join condition: None
+
+(139) Project [codegen id : 51]
+Output [11]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153]
+Input [13]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, ss_sold_date_sk#133, d_date_sk#152, d_year#153]
+
+(140) ReusedExchange [Reuses operator id: 44]
+Output [3]: [s_store_sk#155, s_store_name#156, s_zip#157]
+
+(141) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [ss_store_sk#127]
+Right keys [1]: [s_store_sk#155]
+Join condition: None
+
+(142) Project [codegen id : 51]
+Output [12]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157]
+Input [14]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_store_sk#127, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_sk#155, s_store_name#156, s_zip#157]
+
+(143) ReusedExchange [Reuses operator id: 50]
+Output [6]: [c_customer_sk#158, c_current_cdemo_sk#159, c_current_hdemo_sk#160, c_current_addr_sk#161, c_first_shipto_date_sk#162, c_first_sales_date_sk#163]
+
+(144) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [ss_customer_sk#123]
+Right keys [1]: [c_customer_sk#158]
+Join condition: None
+
+(145) Project [codegen id : 51]
+Output [16]: [ss_item_sk#122, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_cdemo_sk#159, c_current_hdemo_sk#160, c_current_addr_sk#161, c_first_shipto_date_sk#162, c_first_sales_date_sk#163]
+Input [18]: [ss_item_sk#122, ss_customer_sk#123, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_customer_sk#158, c_current_cdemo_sk#159, c_current_hdemo_sk#160, c_current_addr_sk#161, c_first_shipto_date_sk#162, c_first_sales_date_sk#163]
+
+(146) ReusedExchange [Reuses operator id: 56]
+Output [2]: [d_date_sk#164, d_year#165]
+
+(147) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [c_first_sales_date_sk#163]
+Right keys [1]: [d_date_sk#164]
+Join condition: None
+
+(148) Project [codegen id : 51]
+Output [16]: [ss_item_sk#122, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_cdemo_sk#159, c_current_hdemo_sk#160, c_current_addr_sk#161, c_first_shipto_date_sk#162, d_year#165]
+Input [18]: [ss_item_sk#122, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_cdemo_sk#159, c_current_hdemo_sk#160, c_current_addr_sk#161, c_first_shipto_date_sk#162, c_first_sales_date_sk#163, d_date_sk#164, d_year#165]
+
+(149) ReusedExchange [Reuses operator id: 56]
+Output [2]: [d_date_sk#166, d_year#167]
+
+(150) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [c_first_shipto_date_sk#162]
+Right keys [1]: [d_date_sk#166]
+Join condition: None
+
+(151) Project [codegen id : 51]
+Output [16]: [ss_item_sk#122, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_cdemo_sk#159, c_current_hdemo_sk#160, c_current_addr_sk#161, d_year#165, d_year#167]
+Input [18]: [ss_item_sk#122, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_cdemo_sk#159, c_current_hdemo_sk#160, c_current_addr_sk#161, c_first_shipto_date_sk#162, d_year#165, d_date_sk#166, d_year#167]
+
+(152) ReusedExchange [Reuses operator id: 65]
+Output [2]: [cd_demo_sk#168, cd_marital_status#169]
+
+(153) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [ss_cdemo_sk#124]
+Right keys [1]: [cd_demo_sk#168]
+Join condition: None
+
+(154) Project [codegen id : 51]
+Output [16]: [ss_item_sk#122, ss_hdemo_sk#125, ss_addr_sk#126, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_cdemo_sk#159, c_current_hdemo_sk#160, c_current_addr_sk#161, d_year#165, d_year#167, cd_marital_status#169]
+Input [18]: [ss_item_sk#122, ss_cdemo_sk#124, ss_hdemo_sk#125, ss_addr_sk#126, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_cdemo_sk#159, c_current_hdemo_sk#160, c_current_addr_sk#161, d_year#165, d_year#167, cd_demo_sk#168, cd_marital_status#169]
+
+(155) ReusedExchange [Reuses operator id: 65]
+Output [2]: [cd_demo_sk#170, cd_marital_status#171]
+
+(156) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [c_current_cdemo_sk#159]
+Right keys [1]: [cd_demo_sk#170]
+Join condition: NOT (cd_marital_status#169 = cd_marital_status#171)
+
+(157) Project [codegen id : 51]
+Output [14]: [ss_item_sk#122, ss_hdemo_sk#125, ss_addr_sk#126, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_hdemo_sk#160, c_current_addr_sk#161, d_year#165, d_year#167]
+Input [18]: [ss_item_sk#122, ss_hdemo_sk#125, ss_addr_sk#126, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_cdemo_sk#159, c_current_hdemo_sk#160, c_current_addr_sk#161, d_year#165, d_year#167, cd_marital_status#169, cd_demo_sk#170, cd_marital_status#171]
+
+(158) ReusedExchange [Reuses operator id: 74]
+Output [1]: [p_promo_sk#172]
+
+(159) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [ss_promo_sk#128]
+Right keys [1]: [p_promo_sk#172]
+Join condition: None
+
+(160) Project [codegen id : 51]
+Output [13]: [ss_item_sk#122, ss_hdemo_sk#125, ss_addr_sk#126, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_hdemo_sk#160, c_current_addr_sk#161, d_year#165, d_year#167]
+Input [15]: [ss_item_sk#122, ss_hdemo_sk#125, ss_addr_sk#126, ss_promo_sk#128, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_hdemo_sk#160, c_current_addr_sk#161, d_year#165, d_year#167, p_promo_sk#172]
+
+(161) ReusedExchange [Reuses operator id: 80]
+Output [2]: [hd_demo_sk#173, hd_income_band_sk#174]
+
+(162) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [ss_hdemo_sk#125]
+Right keys [1]: [hd_demo_sk#173]
+Join condition: None
+
+(163) Project [codegen id : 51]
+Output [13]: [ss_item_sk#122, ss_addr_sk#126, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_hdemo_sk#160, c_current_addr_sk#161, d_year#165, d_year#167, hd_income_band_sk#174]
+Input [15]: [ss_item_sk#122, ss_hdemo_sk#125, ss_addr_sk#126, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_hdemo_sk#160, c_current_addr_sk#161, d_year#165, d_year#167, hd_demo_sk#173, hd_income_band_sk#174]
+
+(164) ReusedExchange [Reuses operator id: 80]
+Output [2]: [hd_demo_sk#175, hd_income_band_sk#176]
+
+(165) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [c_current_hdemo_sk#160]
+Right keys [1]: [hd_demo_sk#175]
+Join condition: None
+
+(166) Project [codegen id : 51]
+Output [13]: [ss_item_sk#122, ss_addr_sk#126, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_addr_sk#161, d_year#165, d_year#167, hd_income_band_sk#174, hd_income_band_sk#176]
+Input [15]: [ss_item_sk#122, ss_addr_sk#126, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_hdemo_sk#160, c_current_addr_sk#161, d_year#165, d_year#167, hd_income_band_sk#174, hd_demo_sk#175, hd_income_band_sk#176]
+
+(167) ReusedExchange [Reuses operator id: 89]
+Output [5]: [ca_address_sk#177, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181]
+
+(168) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [ss_addr_sk#126]
+Right keys [1]: [ca_address_sk#177]
+Join condition: None
+
+(169) Project [codegen id : 51]
+Output [16]: [ss_item_sk#122, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_addr_sk#161, d_year#165, d_year#167, hd_income_band_sk#174, hd_income_band_sk#176, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181]
+Input [18]: [ss_item_sk#122, ss_addr_sk#126, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_addr_sk#161, d_year#165, d_year#167, hd_income_band_sk#174, hd_income_band_sk#176, ca_address_sk#177, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181]
+
+(170) ReusedExchange [Reuses operator id: 89]
+Output [5]: [ca_address_sk#182, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186]
+
+(171) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [c_current_addr_sk#161]
+Right keys [1]: [ca_address_sk#182]
+Join condition: None
+
+(172) Project [codegen id : 51]
+Output [19]: [ss_item_sk#122, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, d_year#165, d_year#167, hd_income_band_sk#174, hd_income_band_sk#176, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186]
+Input [21]: [ss_item_sk#122, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, c_current_addr_sk#161, d_year#165, d_year#167, hd_income_band_sk#174, hd_income_band_sk#176, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_address_sk#182, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186]
+
+(173) ReusedExchange [Reuses operator id: 98]
+Output [1]: [ib_income_band_sk#187]
+
+(174) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [hd_income_band_sk#174]
+Right keys [1]: [ib_income_band_sk#187]
+Join condition: None
+
+(175) Project [codegen id : 51]
+Output [18]: [ss_item_sk#122, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, d_year#165, d_year#167, hd_income_band_sk#176, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186]
+Input [20]: [ss_item_sk#122, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, d_year#165, d_year#167, hd_income_band_sk#174, hd_income_band_sk#176, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186, ib_income_band_sk#187]
+
+(176) ReusedExchange [Reuses operator id: 98]
+Output [1]: [ib_income_band_sk#188]
+
+(177) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [hd_income_band_sk#176]
+Right keys [1]: [ib_income_band_sk#188]
+Join condition: None
+
+(178) Project [codegen id : 51]
+Output [17]: [ss_item_sk#122, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, d_year#165, d_year#167, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186]
+Input [19]: [ss_item_sk#122, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, d_year#165, d_year#167, hd_income_band_sk#176, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186, ib_income_band_sk#188]
+
+(179) ReusedExchange [Reuses operator id: 108]
+Output [2]: [i_item_sk#189, i_product_name#190]
+
+(180) BroadcastHashJoin [codegen id : 51]
+Left keys [1]: [ss_item_sk#122]
+Right keys [1]: [i_item_sk#189]
+Join condition: None
+
+(181) Project [codegen id : 51]
+Output [18]: [ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, d_year#165, d_year#167, s_store_name#156, s_zip#157, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186, i_item_sk#189, i_product_name#190]
+Input [19]: [ss_item_sk#122, ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, s_store_name#156, s_zip#157, d_year#165, d_year#167, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186, i_item_sk#189, i_product_name#190]
+
+(182) HashAggregate [codegen id : 51]
+Input [18]: [ss_wholesale_cost#130, ss_list_price#131, ss_coupon_amt#132, d_year#153, d_year#165, d_year#167, s_store_name#156, s_zip#157, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186, i_item_sk#189, i_product_name#190]
+Keys [15]: [i_product_name#190, i_item_sk#189, s_store_name#156, s_zip#157, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186, d_year#153, d_year#165, d_year#167]
+Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#130)), partial_sum(UnscaledValue(ss_list_price#131)), partial_sum(UnscaledValue(ss_coupon_amt#132))]
+Aggregate Attributes [4]: [count#191, sum#192, sum#193, sum#194]
+Results [19]: [i_product_name#190, i_item_sk#189, s_store_name#156, s_zip#157, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186, d_year#153, d_year#165, d_year#167, count#195, sum#196, sum#197, sum#198]
+
+(183) HashAggregate [codegen id : 51]
+Input [19]: [i_product_name#190, i_item_sk#189, s_store_name#156, s_zip#157, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186, d_year#153, d_year#165, d_year#167, count#195, sum#196, sum#197, sum#198]
+Keys [15]: [i_product_name#190, i_item_sk#189, s_store_name#156, s_zip#157, ca_street_number#178, ca_street_name#179, ca_city#180, ca_zip#181, ca_street_number#183, ca_street_name#184, ca_city#185, ca_zip#186, d_year#153, d_year#165, d_year#167]
+Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#130)), sum(UnscaledValue(ss_list_price#131)), sum(UnscaledValue(ss_coupon_amt#132))]
+Aggregate Attributes [4]: [count(1)#199, sum(UnscaledValue(ss_wholesale_cost#130))#200, sum(UnscaledValue(ss_list_price#131))#201, sum(UnscaledValue(ss_coupon_amt#132))#202]
+Results [8]: [i_item_sk#189 AS item_sk#203, s_store_name#156 AS store_name#204, s_zip#157 AS store_zip#205, d_year#153 AS syear#206, count(1)#199 AS cnt#207, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#130))#200,17,2) AS s1#208, MakeDecimal(sum(UnscaledValue(ss_list_price#131))#201,17,2) AS s2#209, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#132))#202,17,2) AS s3#210]
+
+(184) Exchange
+Input [8]: [item_sk#203, store_name#204, store_zip#205, syear#206, cnt#207, s1#208, s2#209, s3#210]
+Arguments: hashpartitioning(item_sk#203, store_name#204, store_zip#205, 5), ENSURE_REQUIREMENTS, [id=#211]
+
+(185) Sort [codegen id : 52]
+Input [8]: [item_sk#203, store_name#204, store_zip#205, syear#206, cnt#207, s1#208, s2#209, s3#210]
+Arguments: [item_sk#203 ASC NULLS FIRST, store_name#204 ASC NULLS FIRST, store_zip#205 ASC NULLS FIRST], false, 0
+
+(186) SortMergeJoin [codegen id : 53]
+Left keys [3]: [item_sk#105, store_name#106, store_zip#107]
+Right keys [3]: [item_sk#203, store_name#204, store_zip#205]
+Join condition: (cnt#207 <= cnt#117)
+
+(187) Project [codegen id : 53]
+Output [21]: [product_name#104, store_name#106, store_zip#107, b_street_number#108, b_streen_name#109, b_city#110, b_zip#111, c_street_number#112, c_street_name#113, c_city#114, c_zip#115, syear#116, cnt#117, s1#118, s2#119, s3#120, s1#208, s2#209, s3#210, syear#206, cnt#207]
+Input [25]: [product_name#104, item_sk#105, store_name#106, store_zip#107, b_street_number#108, b_streen_name#109, b_city#110, b_zip#111, c_street_number#112, c_street_name#113, c_city#114, c_zip#115, syear#116, cnt#117, s1#118, s2#119, s3#120, item_sk#203, store_name#204, store_zip#205, syear#206, cnt#207, s1#208, s2#209, s3#210]
+
+(188) Exchange
+Input [21]: [product_name#104, store_name#106, store_zip#107, b_street_number#108, b_streen_name#109, b_city#110, b_zip#111, c_street_number#112, c_street_name#113, c_city#114, c_zip#115, syear#116, cnt#117, s1#118, s2#119, s3#120, s1#208, s2#209, s3#210, syear#206, cnt#207]
+Arguments: rangepartitioning(product_name#104 ASC NULLS FIRST, store_name#106 ASC NULLS FIRST, cnt#207 ASC NULLS FIRST, s1#118 ASC NULLS FIRST, s1#208 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [id=#212]
+
+(189) Sort [codegen id : 54]
+Input [21]: [product_name#104, store_name#106, store_zip#107, b_street_number#108, b_streen_name#109, b_city#110, b_zip#111, c_street_number#112, c_street_name#113, c_city#114, c_zip#115, syear#116, cnt#117, s1#118, s2#119, s3#120, s1#208, s2#209, s3#210, syear#206, cnt#207]
+Arguments: [product_name#104 ASC NULLS FIRST, store_name#106 ASC NULLS FIRST, cnt#207 ASC NULLS FIRST, s1#118 ASC NULLS FIRST, s1#208 ASC NULLS FIRST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13
+ReusedExchange (190)
+
+
+(190) ReusedExchange [Reuses operator id: 38]
+Output [2]: [d_date_sk#42, d_year#43]
+
+Subquery:2 Hosting operator id = 115 Hosting Expression = ss_sold_date_sk#133 IN dynamicpruning#134
+ReusedExchange (191)
+
+
+(191) ReusedExchange [Reuses operator id: 137]
+Output [2]: [d_date_sk#152, d_year#153]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt
new file mode 100644
index 0000000000000..2b44e5ed9d4de
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt
@@ -0,0 +1,289 @@
+WholeStageCodegen (54)
+  Sort [product_name,store_name,cnt,s1,s1]
+    InputAdapter
+      Exchange [product_name,store_name,cnt,s1,s1] #1
+        WholeStageCodegen (53)
+          Project [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt]
+            SortMergeJoin [item_sk,store_name,store_zip,item_sk,store_name,store_zip,cnt,cnt]
+              InputAdapter
+                WholeStageCodegen (26)
+                  Sort [item_sk,store_name,store_zip]
+                    InputAdapter
+                      Exchange [item_sk,store_name,store_zip] #2
+                        WholeStageCodegen (25)
+                          HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum]
+                            HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum]
+                              Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name]
+                                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                  Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                    BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk]
+                                      Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                        BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk]
+                                          Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                            BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                                              Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                                BroadcastHashJoin [ss_addr_sk,ca_address_sk]
+                                                  Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk]
+                                                    BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk]
+                                                      Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk]
+                                                        BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                                          Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year]
+                                                            BroadcastHashJoin [ss_promo_sk,p_promo_sk]
+                                                              Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year]
+                                                                BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status]
+                                                                  Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status]
+                                                                    BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk]
+                                                                      Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year]
+                                                                        BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk]
+                                                                          Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year]
+                                                                            BroadcastHashJoin [c_first_sales_date_sk,d_date_sk]
+                                                                              Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk]
+                                                                                BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+                                                                                  Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip]
+                                                                                    BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                                                      Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year]
+                                                                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                          Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+                                                                                            SortMergeJoin [ss_item_sk,cs_item_sk]
+                                                                                              InputAdapter
+                                                                                                WholeStageCodegen (3)
+                                                                                                  Sort [ss_item_sk]
+                                                                                                    InputAdapter
+                                                                                                      Exchange [ss_item_sk] #3
+                                                                                                        WholeStageCodegen (2)
+                                                                                                          Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+                                                                                                            BroadcastHashJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number]
+                                                                                                              InputAdapter
+                                                                                                                BroadcastExchange #4
+                                                                                                                  WholeStageCodegen (1)
+                                                                                                                    Filter [ss_item_sk,ss_ticket_number,ss_store_sk,ss_customer_sk,ss_cdemo_sk,ss_promo_sk,ss_hdemo_sk,ss_addr_sk]
+                                                                                                                      ColumnarToRow
+                                                                                                                        InputAdapter
+                                                                                                                          Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+                                                                                                                            SubqueryBroadcast [d_date_sk] #1
+                                                                                                                              ReusedExchange [d_date_sk,d_year] #5
+                                                                                                              Project [sr_item_sk,sr_ticket_number]
+                                                                                                                Filter [sr_item_sk,sr_ticket_number]
+                                                                                                                  ColumnarToRow
+                                                                                                                    InputAdapter
+                                                                                                                      Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk]
+                                                                                              InputAdapter
+                                                                                                WholeStageCodegen (9)
+                                                                                                  Sort [cs_item_sk]
+                                                                                                    Project [cs_item_sk]
+                                                                                                      Filter [sum(cs_ext_list_price),sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash as decimal(8,2))) + promote_precision(cast(cr_reversed_charge as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit as decimal(9,2)))), DecimalType(9,2), true))]
+                                                                                                        HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash as decimal(8,2))) + promote_precision(cast(cr_reversed_charge as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit as decimal(9,2)))), DecimalType(9,2), true)),sum(cs_ext_list_price),sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash as decimal(8,2))) + promote_precision(cast(cr_reversed_charge as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit as decimal(9,2)))), DecimalType(9,2), true)),sum,sum,isEmpty]
+                                                                                                          InputAdapter
+                                                                                                            Exchange [cs_item_sk] #6
+                                                                                                              WholeStageCodegen (8)
+                                                                                                                HashAggregate [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [sum,sum,isEmpty,sum,sum,isEmpty]
+                                                                                                                  Project [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit]
+                                                                                                                    SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number]
+                                                                                                                      InputAdapter
+                                                                                                                        WholeStageCodegen (5)
+                                                                                                                          Sort [cs_item_sk,cs_order_number]
+                                                                                                                            InputAdapter
+                                                                                                                              Exchange [cs_item_sk,cs_order_number] #7
+                                                                                                                                WholeStageCodegen (4)
+                                                                                                                                  Project [cs_item_sk,cs_order_number,cs_ext_list_price]
+                                                                                                                                    Filter [cs_item_sk,cs_order_number]
+                                                                                                                                      ColumnarToRow
+                                                                                                                                        InputAdapter
+                                                                                                                                          Scan parquet default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk]
+                                                                                                                      InputAdapter
+                                                                                                                        WholeStageCodegen (7)
+                                                                                                                          Sort [cr_item_sk,cr_order_number]
+                                                                                                                            InputAdapter
+                                                                                                                              Exchange [cr_item_sk,cr_order_number] #8
+                                                                                                                                WholeStageCodegen (6)
+                                                                                                                                  Project [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit]
+                                                                                                                                    Filter [cr_item_sk,cr_order_number]
+                                                                                                                                      ColumnarToRow
+                                                                                                                                        InputAdapter
+                                                                                                                                          Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk]
+                                                                                          InputAdapter
+                                                                                            BroadcastExchange #5
+                                                                                              WholeStageCodegen (10)
+                                                                                                Filter [d_year,d_date_sk]
+                                                                                                  ColumnarToRow
+                                                                                                    InputAdapter
+                                                                                                      Scan parquet default.date_dim [d_date_sk,d_year]
+                                                                                      InputAdapter
+                                                                                        BroadcastExchange #9
+                                                                                          WholeStageCodegen (11)
+                                                                                            Filter [s_store_sk,s_store_name,s_zip]
+                                                                                              ColumnarToRow
+                                                                                                InputAdapter
+                                                                                                  Scan parquet default.store [s_store_sk,s_store_name,s_zip]
+                                                                                  InputAdapter
+                                                                                    BroadcastExchange #10
+                                                                                      WholeStageCodegen (12)
+                                                                                        Filter [c_customer_sk,c_first_sales_date_sk,c_first_shipto_date_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk]
+                                                                                          ColumnarToRow
+                                                                                            InputAdapter
+                                                                                              Scan parquet default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk]
+                                                                              InputAdapter
+                                                                                BroadcastExchange #11
+                                                                                  WholeStageCodegen (13)
+                                                                                    Filter [d_date_sk]
+                                                                                      ColumnarToRow
+                                                                                        InputAdapter
+                                                                                          Scan parquet default.date_dim [d_date_sk,d_year]
+                                                                          InputAdapter
+                                                                            ReusedExchange [d_date_sk,d_year] #11
+                                                                      InputAdapter
+                                                                        BroadcastExchange #12
+                                                                          WholeStageCodegen (15)
+                                                                            Filter [cd_demo_sk,cd_marital_status]
+                                                                              ColumnarToRow
+                                                                                InputAdapter
+                                                                                  Scan parquet default.customer_demographics [cd_demo_sk,cd_marital_status]
+                                                                  InputAdapter
+                                                                    ReusedExchange [cd_demo_sk,cd_marital_status] #12
+                                                              InputAdapter
+                                                                BroadcastExchange #13
+                                                                  WholeStageCodegen (17)
+                                                                    Filter [p_promo_sk]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.promotion [p_promo_sk]
+                                                          InputAdapter
+                                                            BroadcastExchange #14
+                                                              WholeStageCodegen (18)
+                                                                Filter [hd_demo_sk,hd_income_band_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.household_demographics [hd_demo_sk,hd_income_band_sk]
+                                                      InputAdapter
+                                                        ReusedExchange [hd_demo_sk,hd_income_band_sk] #14
+                                                  InputAdapter
+                                                    BroadcastExchange #15
+                                                      WholeStageCodegen (20)
+                                                        Filter [ca_address_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                              InputAdapter
+                                                ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15
+                                          InputAdapter
+                                            BroadcastExchange #16
+                                              WholeStageCodegen (22)
+                                                Filter [ib_income_band_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.income_band [ib_income_band_sk]
+                                      InputAdapter
+                                        ReusedExchange [ib_income_band_sk] #16
+                                  InputAdapter
+                                    BroadcastExchange #17
+                                      WholeStageCodegen (24)
+                                        Project [i_item_sk,i_product_name]
+                                          Filter [i_current_price,i_color,i_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.item [i_item_sk,i_current_price,i_color,i_product_name]
+              InputAdapter
+                WholeStageCodegen (52)
+                  Sort [item_sk,store_name,store_zip]
+                    InputAdapter
+                      Exchange [item_sk,store_name,store_zip] #18
+                        WholeStageCodegen (51)
+                          HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum]
+                            HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum]
+                              Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name]
+                                BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                  Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                    BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk]
+                                      Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                        BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk]
+                                          Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                            BroadcastHashJoin [c_current_addr_sk,ca_address_sk]
+                                              Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip]
+                                                BroadcastHashJoin [ss_addr_sk,ca_address_sk]
+                                                  Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk]
+                                                    BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk]
+                                                      Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk]
+                                                        BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk]
+                                                          Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year]
+                                                            BroadcastHashJoin [ss_promo_sk,p_promo_sk]
+                                                              Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year]
+                                                                BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status]
+                                                                  Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status]
+                                                                    BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk]
+                                                                      Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year]
+                                                                        BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk]
+                                                                          Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year]
+                                                                            BroadcastHashJoin [c_first_sales_date_sk,d_date_sk]
+                                                                              Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk]
+                                                                                BroadcastHashJoin [ss_customer_sk,c_customer_sk]
+                                                                                  Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip]
+                                                                                    BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                                                      Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year]
+                                                                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                          Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+                                                                                            SortMergeJoin [ss_item_sk,cs_item_sk]
+                                                                                              InputAdapter
+                                                                                                WholeStageCodegen (29)
+                                                                                                  Sort [ss_item_sk]
+                                                                                                    InputAdapter
+                                                                                                      Exchange [ss_item_sk] #19
+                                                                                                        WholeStageCodegen (28)
+                                                                                                          Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+                                                                                                            BroadcastHashJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number]
+                                                                                                              InputAdapter
+                                                                                                                BroadcastExchange #20
+                                                                                                                  WholeStageCodegen (27)
+                                                                                                                    Filter [ss_item_sk,ss_ticket_number,ss_store_sk,ss_customer_sk,ss_cdemo_sk,ss_promo_sk,ss_hdemo_sk,ss_addr_sk]
+                                                                                                                      ColumnarToRow
+                                                                                                                        InputAdapter
+                                                                                                                          Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
+                                                                                                                            SubqueryBroadcast [d_date_sk] #2
+                                                                                                                              ReusedExchange [d_date_sk,d_year] #21
+                                                                                                              Project [sr_item_sk,sr_ticket_number]
+                                                                                                                Filter [sr_item_sk,sr_ticket_number]
+                                                                                                                  ColumnarToRow
+                                                                                                                    InputAdapter
+                                                                                                                      Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk]
+                                                                                              InputAdapter
+                                                                                                WholeStageCodegen (35)
+                                                                                                  Sort [cs_item_sk]
+                                                                                                    Project [cs_item_sk]
+                                                                                                      Filter [sum(cs_ext_list_price),sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash as decimal(8,2))) + promote_precision(cast(cr_reversed_charge as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit as decimal(9,2)))), DecimalType(9,2), true))]
+                                                                                                        HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash as decimal(8,2))) + promote_precision(cast(cr_reversed_charge as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit as decimal(9,2)))), DecimalType(9,2), true)),sum(cs_ext_list_price),sum(CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(cr_refunded_cash as decimal(8,2))) + promote_precision(cast(cr_reversed_charge as decimal(8,2)))), DecimalType(8,2), true) as decimal(9,2))) + promote_precision(cast(cr_store_credit as decimal(9,2)))), DecimalType(9,2), true)),sum,sum,isEmpty]
+                                                                                                          InputAdapter
+                                                                                                            ReusedExchange [cs_item_sk,sum,sum,isEmpty] #6
+                                                                                          InputAdapter
+                                                                                            BroadcastExchange #21
+                                                                                              WholeStageCodegen (36)
+                                                                                                Filter [d_year,d_date_sk]
+                                                                                                  ColumnarToRow
+                                                                                                    InputAdapter
+                                                                                                      Scan parquet default.date_dim [d_date_sk,d_year]
+                                                                                      InputAdapter
+                                                                                        ReusedExchange [s_store_sk,s_store_name,s_zip] #9
+                                                                                  InputAdapter
+                                                                                    ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10
+                                                                              InputAdapter
+                                                                                ReusedExchange [d_date_sk,d_year] #11
+                                                                          InputAdapter
+                                                                            ReusedExchange [d_date_sk,d_year] #11
+                                                                      InputAdapter
+                                                                        ReusedExchange [cd_demo_sk,cd_marital_status] #12
+                                                                  InputAdapter
+                                                                    ReusedExchange [cd_demo_sk,cd_marital_status] #12
+                                                              InputAdapter
+                                                                ReusedExchange [p_promo_sk] #13
+                                                          InputAdapter
+                                                            ReusedExchange [hd_demo_sk,hd_income_band_sk] #14
+                                                      InputAdapter
+                                                        ReusedExchange [hd_demo_sk,hd_income_band_sk] #14
+                                                  InputAdapter
+                                                    ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15
+                                              InputAdapter
+                                                ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15
+                                          InputAdapter
+                                            ReusedExchange [ib_income_band_sk] #16
+                                      InputAdapter
+                                        ReusedExchange [ib_income_band_sk] #16
+                                  InputAdapter
+                                    ReusedExchange [i_item_sk,i_product_name] #17
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.sf100/explain.txt
new file mode 100644
index 0000000000000..af8035e67b061
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.sf100/explain.txt
@@ -0,0 +1,463 @@
+== Physical Plan ==
+TakeOrderedAndProject (74)
++- * Filter (73)
+   +- Window (72)
+      +- * Sort (71)
+         +- Exchange (70)
+            +- Union (69)
+               :- * HashAggregate (28)
+               :  +- Exchange (27)
+               :     +- * HashAggregate (26)
+               :        +- * Project (25)
+               :           +- * SortMergeJoin Inner (24)
+               :              :- * Sort (18)
+               :              :  +- Exchange (17)
+               :              :     +- * Project (16)
+               :              :        +- * BroadcastHashJoin Inner BuildRight (15)
+               :              :           :- * Project (10)
+               :              :           :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :              :           :     :- * Filter (3)
+               :              :           :     :  +- * ColumnarToRow (2)
+               :              :           :     :     +- Scan parquet default.store_sales (1)
+               :              :           :     +- BroadcastExchange (8)
+               :              :           :        +- * Project (7)
+               :              :           :           +- * Filter (6)
+               :              :           :              +- * ColumnarToRow (5)
+               :              :           :                 +- Scan parquet default.date_dim (4)
+               :              :           +- BroadcastExchange (14)
+               :              :              +- * Filter (13)
+               :              :                 +- * ColumnarToRow (12)
+               :              :                    +- Scan parquet default.store (11)
+               :              +- * Sort (23)
+               :                 +- Exchange (22)
+               :                    +- * Filter (21)
+               :                       +- * ColumnarToRow (20)
+               :                          +- Scan parquet default.item (19)
+               :- * HashAggregate (33)
+               :  +- Exchange (32)
+               :     +- * HashAggregate (31)
+               :        +- * HashAggregate (30)
+               :           +- ReusedExchange (29)
+               :- * HashAggregate (38)
+               :  +- Exchange (37)
+               :     +- * HashAggregate (36)
+               :        +- * HashAggregate (35)
+               :           +- ReusedExchange (34)
+               :- * HashAggregate (43)
+               :  +- Exchange (42)
+               :     +- * HashAggregate (41)
+               :        +- * HashAggregate (40)
+               :           +- ReusedExchange (39)
+               :- * HashAggregate (48)
+               :  +- Exchange (47)
+               :     +- * HashAggregate (46)
+               :        +- * HashAggregate (45)
+               :           +- ReusedExchange (44)
+               :- * HashAggregate (53)
+               :  +- Exchange (52)
+               :     +- * HashAggregate (51)
+               :        +- * HashAggregate (50)
+               :           +- ReusedExchange (49)
+               :- * HashAggregate (58)
+               :  +- Exchange (57)
+               :     +- * HashAggregate (56)
+               :        +- * HashAggregate (55)
+               :           +- ReusedExchange (54)
+               :- * HashAggregate (63)
+               :  +- Exchange (62)
+               :     +- * HashAggregate (61)
+               :        +- * HashAggregate (60)
+               :           +- ReusedExchange (59)
+               +- * HashAggregate (68)
+                  +- Exchange (67)
+                     +- * HashAggregate (66)
+                        +- * HashAggregate (65)
+                           +- ReusedExchange (64)
+
+
+(1) Scan parquet default.store_sales
+Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_quantity:int,ss_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5]
+
+(3) Filter [codegen id : 3]
+Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5]
+Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int,d_year:int,d_moy:int,d_qoy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11]
+
+(6) Filter [codegen id : 1]
+Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11]
+Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1212)) AND (d_month_seq#8 <= 1223)) AND isnotnull(d_date_sk#7))
+
+(7) Project [codegen id : 1]
+Output [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11]
+Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11]
+
+(8) BroadcastExchange
+Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#5]
+Right keys [1]: [d_date_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11]
+Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#7, d_year#9, d_moy#10, d_qoy#11]
+
+(11) Scan parquet default.store
+Output [2]: [s_store_sk#13, s_store_id#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [s_store_sk#13, s_store_id#14]
+
+(13) Filter [codegen id : 2]
+Input [2]: [s_store_sk#13, s_store_id#14]
+Condition : isnotnull(s_store_sk#13)
+
+(14) BroadcastExchange
+Input [2]: [s_store_sk#13, s_store_id#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#15]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_store_sk#2]
+Right keys [1]: [s_store_sk#13]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14]
+Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_sk#13, s_store_id#14]
+
+(17) Exchange
+Input [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14]
+Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#16]
+
+(18) Sort [codegen id : 4]
+Input [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14]
+Arguments: [ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(19) Scan parquet default.item
+Output [5]: [i_item_sk#17, i_brand#18, i_class#19, i_category#20, i_product_name#21]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_class:string,i_category:string,i_product_name:string>
+
+(20) ColumnarToRow [codegen id : 5]
+Input [5]: [i_item_sk#17, i_brand#18, i_class#19, i_category#20, i_product_name#21]
+
+(21) Filter [codegen id : 5]
+Input [5]: [i_item_sk#17, i_brand#18, i_class#19, i_category#20, i_product_name#21]
+Condition : isnotnull(i_item_sk#17)
+
+(22) Exchange
+Input [5]: [i_item_sk#17, i_brand#18, i_class#19, i_category#20, i_product_name#21]
+Arguments: hashpartitioning(i_item_sk#17, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(23) Sort [codegen id : 6]
+Input [5]: [i_item_sk#17, i_brand#18, i_class#19, i_category#20, i_product_name#21]
+Arguments: [i_item_sk#17 ASC NULLS FIRST], false, 0
+
+(24) SortMergeJoin [codegen id : 7]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#17]
+Join condition: None
+
+(25) Project [codegen id : 7]
+Output [10]: [ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14, i_brand#18, i_class#19, i_category#20, i_product_name#21]
+Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14, i_item_sk#17, i_brand#18, i_class#19, i_category#20, i_product_name#21]
+
+(26) HashAggregate [codegen id : 7]
+Input [10]: [ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14, i_brand#18, i_class#19, i_category#20, i_product_name#21]
+Keys [8]: [i_category#20, i_class#19, i_brand#18, i_product_name#21, d_year#9, d_qoy#11, d_moy#10, s_store_id#14]
+Functions [1]: [partial_sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#4 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#3 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))]
+Aggregate Attributes [2]: [sum#23, isEmpty#24]
+Results [10]: [i_category#20, i_class#19, i_brand#18, i_product_name#21, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, sum#25, isEmpty#26]
+
+(27) Exchange
+Input [10]: [i_category#20, i_class#19, i_brand#18, i_product_name#21, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, sum#25, isEmpty#26]
+Arguments: hashpartitioning(i_category#20, i_class#19, i_brand#18, i_product_name#21, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, 5), ENSURE_REQUIREMENTS, [id=#27]
+
+(28) HashAggregate [codegen id : 8]
+Input [10]: [i_category#20, i_class#19, i_brand#18, i_product_name#21, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, sum#25, isEmpty#26]
+Keys [8]: [i_category#20, i_class#19, i_brand#18, i_product_name#21, d_year#9, d_qoy#11, d_moy#10, s_store_id#14]
+Functions [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#4 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#3 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))]
+Aggregate Attributes [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#4 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#3 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#28]
+Results [9]: [i_category#20, i_class#19, i_brand#18, i_product_name#21, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, cast(sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#4 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#3 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#28 as decimal(38,2)) AS sumsales#29]
+
+(29) ReusedExchange [Reuses operator id: unknown]
+Output [10]: [i_category#30, i_class#31, i_brand#32, i_product_name#33, d_year#34, d_qoy#35, d_moy#36, s_store_id#37, sum#38, isEmpty#39]
+
+(30) HashAggregate [codegen id : 16]
+Input [10]: [i_category#30, i_class#31, i_brand#32, i_product_name#33, d_year#34, d_qoy#35, d_moy#36, s_store_id#37, sum#38, isEmpty#39]
+Keys [8]: [i_category#30, i_class#31, i_brand#32, i_product_name#33, d_year#34, d_qoy#35, d_moy#36, s_store_id#37]
+Functions [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#40 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#41 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))]
+Aggregate Attributes [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#40 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#41 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#42]
+Results [8]: [i_category#30, i_class#31, i_brand#32, i_product_name#33, d_year#34, d_qoy#35, d_moy#36, sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#40 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#41 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#42 AS sumsales#43]
+
+(31) HashAggregate [codegen id : 16]
+Input [8]: [i_category#30, i_class#31, i_brand#32, i_product_name#33, d_year#34, d_qoy#35, d_moy#36, sumsales#43]
+Keys [7]: [i_category#30, i_class#31, i_brand#32, i_product_name#33, d_year#34, d_qoy#35, d_moy#36]
+Functions [1]: [partial_sum(sumsales#43)]
+Aggregate Attributes [2]: [sum#44, isEmpty#45]
+Results [9]: [i_category#30, i_class#31, i_brand#32, i_product_name#33, d_year#34, d_qoy#35, d_moy#36, sum#46, isEmpty#47]
+
+(32) Exchange
+Input [9]: [i_category#30, i_class#31, i_brand#32, i_product_name#33, d_year#34, d_qoy#35, d_moy#36, sum#46, isEmpty#47]
+Arguments: hashpartitioning(i_category#30, i_class#31, i_brand#32, i_product_name#33, d_year#34, d_qoy#35, d_moy#36, 5), ENSURE_REQUIREMENTS, [id=#48]
+
+(33) HashAggregate [codegen id : 17]
+Input [9]: [i_category#30, i_class#31, i_brand#32, i_product_name#33, d_year#34, d_qoy#35, d_moy#36, sum#46, isEmpty#47]
+Keys [7]: [i_category#30, i_class#31, i_brand#32, i_product_name#33, d_year#34, d_qoy#35, d_moy#36]
+Functions [1]: [sum(sumsales#43)]
+Aggregate Attributes [1]: [sum(sumsales#43)#49]
+Results [9]: [i_category#30, i_class#31, i_brand#32, i_product_name#33, d_year#34, d_qoy#35, d_moy#36, null AS s_store_id#50, sum(sumsales#43)#49 AS sumsales#51]
+
+(34) ReusedExchange [Reuses operator id: unknown]
+Output [10]: [i_category#52, i_class#53, i_brand#54, i_product_name#55, d_year#56, d_qoy#57, d_moy#58, s_store_id#59, sum#60, isEmpty#61]
+
+(35) HashAggregate [codegen id : 25]
+Input [10]: [i_category#52, i_class#53, i_brand#54, i_product_name#55, d_year#56, d_qoy#57, d_moy#58, s_store_id#59, sum#60, isEmpty#61]
+Keys [8]: [i_category#52, i_class#53, i_brand#54, i_product_name#55, d_year#56, d_qoy#57, d_moy#58, s_store_id#59]
+Functions [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#62 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#63 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))]
+Aggregate Attributes [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#62 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#63 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#64]
+Results [7]: [i_category#52, i_class#53, i_brand#54, i_product_name#55, d_year#56, d_qoy#57, sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#62 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#63 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#64 AS sumsales#43]
+
+(36) HashAggregate [codegen id : 25]
+Input [7]: [i_category#52, i_class#53, i_brand#54, i_product_name#55, d_year#56, d_qoy#57, sumsales#43]
+Keys [6]: [i_category#52, i_class#53, i_brand#54, i_product_name#55, d_year#56, d_qoy#57]
+Functions [1]: [partial_sum(sumsales#43)]
+Aggregate Attributes [2]: [sum#65, isEmpty#66]
+Results [8]: [i_category#52, i_class#53, i_brand#54, i_product_name#55, d_year#56, d_qoy#57, sum#67, isEmpty#68]
+
+(37) Exchange
+Input [8]: [i_category#52, i_class#53, i_brand#54, i_product_name#55, d_year#56, d_qoy#57, sum#67, isEmpty#68]
+Arguments: hashpartitioning(i_category#52, i_class#53, i_brand#54, i_product_name#55, d_year#56, d_qoy#57, 5), ENSURE_REQUIREMENTS, [id=#69]
+
+(38) HashAggregate [codegen id : 26]
+Input [8]: [i_category#52, i_class#53, i_brand#54, i_product_name#55, d_year#56, d_qoy#57, sum#67, isEmpty#68]
+Keys [6]: [i_category#52, i_class#53, i_brand#54, i_product_name#55, d_year#56, d_qoy#57]
+Functions [1]: [sum(sumsales#43)]
+Aggregate Attributes [1]: [sum(sumsales#43)#70]
+Results [9]: [i_category#52, i_class#53, i_brand#54, i_product_name#55, d_year#56, d_qoy#57, null AS d_moy#71, null AS s_store_id#72, sum(sumsales#43)#70 AS sumsales#73]
+
+(39) ReusedExchange [Reuses operator id: unknown]
+Output [10]: [i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78, d_qoy#79, d_moy#80, s_store_id#81, sum#82, isEmpty#83]
+
+(40) HashAggregate [codegen id : 34]
+Input [10]: [i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78, d_qoy#79, d_moy#80, s_store_id#81, sum#82, isEmpty#83]
+Keys [8]: [i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78, d_qoy#79, d_moy#80, s_store_id#81]
+Functions [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#84 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#85 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))]
+Aggregate Attributes [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#84 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#85 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#86]
+Results [6]: [i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78, sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#84 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#85 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#86 AS sumsales#43]
+
+(41) HashAggregate [codegen id : 34]
+Input [6]: [i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78, sumsales#43]
+Keys [5]: [i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78]
+Functions [1]: [partial_sum(sumsales#43)]
+Aggregate Attributes [2]: [sum#87, isEmpty#88]
+Results [7]: [i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78, sum#89, isEmpty#90]
+
+(42) Exchange
+Input [7]: [i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78, sum#89, isEmpty#90]
+Arguments: hashpartitioning(i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78, 5), ENSURE_REQUIREMENTS, [id=#91]
+
+(43) HashAggregate [codegen id : 35]
+Input [7]: [i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78, sum#89, isEmpty#90]
+Keys [5]: [i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78]
+Functions [1]: [sum(sumsales#43)]
+Aggregate Attributes [1]: [sum(sumsales#43)#92]
+Results [9]: [i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78, null AS d_qoy#93, null AS d_moy#94, null AS s_store_id#95, sum(sumsales#43)#92 AS sumsales#96]
+
+(44) ReusedExchange [Reuses operator id: unknown]
+Output [10]: [i_category#97, i_class#98, i_brand#99, i_product_name#100, d_year#101, d_qoy#102, d_moy#103, s_store_id#104, sum#105, isEmpty#106]
+
+(45) HashAggregate [codegen id : 43]
+Input [10]: [i_category#97, i_class#98, i_brand#99, i_product_name#100, d_year#101, d_qoy#102, d_moy#103, s_store_id#104, sum#105, isEmpty#106]
+Keys [8]: [i_category#97, i_class#98, i_brand#99, i_product_name#100, d_year#101, d_qoy#102, d_moy#103, s_store_id#104]
+Functions [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#107 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#108 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))]
+Aggregate Attributes [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#107 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#108 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#109]
+Results [5]: [i_category#97, i_class#98, i_brand#99, i_product_name#100, sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#107 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#108 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#109 AS sumsales#43]
+
+(46) HashAggregate [codegen id : 43]
+Input [5]: [i_category#97, i_class#98, i_brand#99, i_product_name#100, sumsales#43]
+Keys [4]: [i_category#97, i_class#98, i_brand#99, i_product_name#100]
+Functions [1]: [partial_sum(sumsales#43)]
+Aggregate Attributes [2]: [sum#110, isEmpty#111]
+Results [6]: [i_category#97, i_class#98, i_brand#99, i_product_name#100, sum#112, isEmpty#113]
+
+(47) Exchange
+Input [6]: [i_category#97, i_class#98, i_brand#99, i_product_name#100, sum#112, isEmpty#113]
+Arguments: hashpartitioning(i_category#97, i_class#98, i_brand#99, i_product_name#100, 5), ENSURE_REQUIREMENTS, [id=#114]
+
+(48) HashAggregate [codegen id : 44]
+Input [6]: [i_category#97, i_class#98, i_brand#99, i_product_name#100, sum#112, isEmpty#113]
+Keys [4]: [i_category#97, i_class#98, i_brand#99, i_product_name#100]
+Functions [1]: [sum(sumsales#43)]
+Aggregate Attributes [1]: [sum(sumsales#43)#115]
+Results [9]: [i_category#97, i_class#98, i_brand#99, i_product_name#100, null AS d_year#116, null AS d_qoy#117, null AS d_moy#118, null AS s_store_id#119, sum(sumsales#43)#115 AS sumsales#120]
+
+(49) ReusedExchange [Reuses operator id: unknown]
+Output [10]: [i_category#121, i_class#122, i_brand#123, i_product_name#124, d_year#125, d_qoy#126, d_moy#127, s_store_id#128, sum#129, isEmpty#130]
+
+(50) HashAggregate [codegen id : 52]
+Input [10]: [i_category#121, i_class#122, i_brand#123, i_product_name#124, d_year#125, d_qoy#126, d_moy#127, s_store_id#128, sum#129, isEmpty#130]
+Keys [8]: [i_category#121, i_class#122, i_brand#123, i_product_name#124, d_year#125, d_qoy#126, d_moy#127, s_store_id#128]
+Functions [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#131 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#132 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))]
+Aggregate Attributes [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#131 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#132 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#133]
+Results [4]: [i_category#121, i_class#122, i_brand#123, sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#131 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#132 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#133 AS sumsales#43]
+
+(51) HashAggregate [codegen id : 52]
+Input [4]: [i_category#121, i_class#122, i_brand#123, sumsales#43]
+Keys [3]: [i_category#121, i_class#122, i_brand#123]
+Functions [1]: [partial_sum(sumsales#43)]
+Aggregate Attributes [2]: [sum#134, isEmpty#135]
+Results [5]: [i_category#121, i_class#122, i_brand#123, sum#136, isEmpty#137]
+
+(52) Exchange
+Input [5]: [i_category#121, i_class#122, i_brand#123, sum#136, isEmpty#137]
+Arguments: hashpartitioning(i_category#121, i_class#122, i_brand#123, 5), ENSURE_REQUIREMENTS, [id=#138]
+
+(53) HashAggregate [codegen id : 53]
+Input [5]: [i_category#121, i_class#122, i_brand#123, sum#136, isEmpty#137]
+Keys [3]: [i_category#121, i_class#122, i_brand#123]
+Functions [1]: [sum(sumsales#43)]
+Aggregate Attributes [1]: [sum(sumsales#43)#139]
+Results [9]: [i_category#121, i_class#122, i_brand#123, null AS i_product_name#140, null AS d_year#141, null AS d_qoy#142, null AS d_moy#143, null AS s_store_id#144, sum(sumsales#43)#139 AS sumsales#145]
+
+(54) ReusedExchange [Reuses operator id: unknown]
+Output [10]: [i_category#146, i_class#147, i_brand#148, i_product_name#149, d_year#150, d_qoy#151, d_moy#152, s_store_id#153, sum#154, isEmpty#155]
+
+(55) HashAggregate [codegen id : 61]
+Input [10]: [i_category#146, i_class#147, i_brand#148, i_product_name#149, d_year#150, d_qoy#151, d_moy#152, s_store_id#153, sum#154, isEmpty#155]
+Keys [8]: [i_category#146, i_class#147, i_brand#148, i_product_name#149, d_year#150, d_qoy#151, d_moy#152, s_store_id#153]
+Functions [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#156 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#157 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))]
+Aggregate Attributes [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#156 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#157 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#158]
+Results [3]: [i_category#146, i_class#147, sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#156 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#157 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#158 AS sumsales#43]
+
+(56) HashAggregate [codegen id : 61]
+Input [3]: [i_category#146, i_class#147, sumsales#43]
+Keys [2]: [i_category#146, i_class#147]
+Functions [1]: [partial_sum(sumsales#43)]
+Aggregate Attributes [2]: [sum#159, isEmpty#160]
+Results [4]: [i_category#146, i_class#147, sum#161, isEmpty#162]
+
+(57) Exchange
+Input [4]: [i_category#146, i_class#147, sum#161, isEmpty#162]
+Arguments: hashpartitioning(i_category#146, i_class#147, 5), ENSURE_REQUIREMENTS, [id=#163]
+
+(58) HashAggregate [codegen id : 62]
+Input [4]: [i_category#146, i_class#147, sum#161, isEmpty#162]
+Keys [2]: [i_category#146, i_class#147]
+Functions [1]: [sum(sumsales#43)]
+Aggregate Attributes [1]: [sum(sumsales#43)#164]
+Results [9]: [i_category#146, i_class#147, null AS i_brand#165, null AS i_product_name#166, null AS d_year#167, null AS d_qoy#168, null AS d_moy#169, null AS s_store_id#170, sum(sumsales#43)#164 AS sumsales#171]
+
+(59) ReusedExchange [Reuses operator id: unknown]
+Output [10]: [i_category#172, i_class#173, i_brand#174, i_product_name#175, d_year#176, d_qoy#177, d_moy#178, s_store_id#179, sum#180, isEmpty#181]
+
+(60) HashAggregate [codegen id : 70]
+Input [10]: [i_category#172, i_class#173, i_brand#174, i_product_name#175, d_year#176, d_qoy#177, d_moy#178, s_store_id#179, sum#180, isEmpty#181]
+Keys [8]: [i_category#172, i_class#173, i_brand#174, i_product_name#175, d_year#176, d_qoy#177, d_moy#178, s_store_id#179]
+Functions [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#182 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#183 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))]
+Aggregate Attributes [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#182 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#183 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#184]
+Results [2]: [i_category#172, sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#182 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#183 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#184 AS sumsales#43]
+
+(61) HashAggregate [codegen id : 70]
+Input [2]: [i_category#172, sumsales#43]
+Keys [1]: [i_category#172]
+Functions [1]: [partial_sum(sumsales#43)]
+Aggregate Attributes [2]: [sum#185, isEmpty#186]
+Results [3]: [i_category#172, sum#187, isEmpty#188]
+
+(62) Exchange
+Input [3]: [i_category#172, sum#187, isEmpty#188]
+Arguments: hashpartitioning(i_category#172, 5), ENSURE_REQUIREMENTS, [id=#189]
+
+(63) HashAggregate [codegen id : 71]
+Input [3]: [i_category#172, sum#187, isEmpty#188]
+Keys [1]: [i_category#172]
+Functions [1]: [sum(sumsales#43)]
+Aggregate Attributes [1]: [sum(sumsales#43)#190]
+Results [9]: [i_category#172, null AS i_class#191, null AS i_brand#192, null AS i_product_name#193, null AS d_year#194, null AS d_qoy#195, null AS d_moy#196, null AS s_store_id#197, sum(sumsales#43)#190 AS sumsales#198]
+
+(64) ReusedExchange [Reuses operator id: unknown]
+Output [10]: [i_category#199, i_class#200, i_brand#201, i_product_name#202, d_year#203, d_qoy#204, d_moy#205, s_store_id#206, sum#207, isEmpty#208]
+
+(65) HashAggregate [codegen id : 79]
+Input [10]: [i_category#199, i_class#200, i_brand#201, i_product_name#202, d_year#203, d_qoy#204, d_moy#205, s_store_id#206, sum#207, isEmpty#208]
+Keys [8]: [i_category#199, i_class#200, i_brand#201, i_product_name#202, d_year#203, d_qoy#204, d_moy#205, s_store_id#206]
+Functions [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#209 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#210 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))]
+Aggregate Attributes [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#209 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#210 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#211]
+Results [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#209 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#210 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#211 AS sumsales#43]
+
+(66) HashAggregate [codegen id : 79]
+Input [1]: [sumsales#43]
+Keys: []
+Functions [1]: [partial_sum(sumsales#43)]
+Aggregate Attributes [2]: [sum#212, isEmpty#213]
+Results [2]: [sum#214, isEmpty#215]
+
+(67) Exchange
+Input [2]: [sum#214, isEmpty#215]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#216]
+
+(68) HashAggregate [codegen id : 80]
+Input [2]: [sum#214, isEmpty#215]
+Keys: []
+Functions [1]: [sum(sumsales#43)]
+Aggregate Attributes [1]: [sum(sumsales#43)#217]
+Results [9]: [null AS i_category#218, null AS i_class#219, null AS i_brand#220, null AS i_product_name#221, null AS d_year#222, null AS d_qoy#223, null AS d_moy#224, null AS s_store_id#225, sum(sumsales#43)#217 AS sumsales#226]
+
+(69) Union
+
+(70) Exchange
+Input [9]: [i_category#20, i_class#19, i_brand#18, i_product_name#21, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, sumsales#29]
+Arguments: hashpartitioning(i_category#20, 5), ENSURE_REQUIREMENTS, [id=#227]
+
+(71) Sort [codegen id : 81]
+Input [9]: [i_category#20, i_class#19, i_brand#18, i_product_name#21, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, sumsales#29]
+Arguments: [i_category#20 ASC NULLS FIRST, sumsales#29 DESC NULLS LAST], false, 0
+
+(72) Window
+Input [9]: [i_category#20, i_class#19, i_brand#18, i_product_name#21, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, sumsales#29]
+Arguments: [rank(sumsales#29) windowspecdefinition(i_category#20, sumsales#29 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#228], [i_category#20], [sumsales#29 DESC NULLS LAST]
+
+(73) Filter [codegen id : 82]
+Input [10]: [i_category#20, i_class#19, i_brand#18, i_product_name#21, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, sumsales#29, rk#228]
+Condition : (rk#228 <= 100)
+
+(74) TakeOrderedAndProject
+Input [10]: [i_category#20, i_class#19, i_brand#18, i_product_name#21, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, sumsales#29, rk#228]
+Arguments: 100, [i_category#20 ASC NULLS FIRST, i_class#19 ASC NULLS FIRST, i_brand#18 ASC NULLS FIRST, i_product_name#21 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_qoy#11 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, s_store_id#14 ASC NULLS FIRST, sumsales#29 ASC NULLS FIRST, rk#228 ASC NULLS FIRST], [i_category#20, i_class#19, i_brand#18, i_product_name#21, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, sumsales#29, rk#228]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (75)
+
+
+(75) ReusedExchange [Reuses operator id: 8]
+Output [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.sf100/simplified.txt
new file mode 100644
index 0000000000000..a3354d7b32d61
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.sf100/simplified.txt
@@ -0,0 +1,131 @@
+TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,rk]
+  WholeStageCodegen (82)
+    Filter [rk]
+      InputAdapter
+        Window [sumsales,i_category]
+          WholeStageCodegen (81)
+            Sort [i_category,sumsales]
+              InputAdapter
+                Exchange [i_category] #1
+                  Union
+                    WholeStageCodegen (8)
+                      HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price as decimal(12,2))) * promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00)),sumsales,sum,isEmpty]
+                        InputAdapter
+                          Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] #2
+                            WholeStageCodegen (7)
+                              HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,ss_sales_price,ss_quantity] [sum,isEmpty,sum,isEmpty]
+                                Project [ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_brand,i_class,i_category,i_product_name]
+                                  SortMergeJoin [ss_item_sk,i_item_sk]
+                                    InputAdapter
+                                      WholeStageCodegen (4)
+                                        Sort [ss_item_sk]
+                                          InputAdapter
+                                            Exchange [ss_item_sk] #3
+                                              WholeStageCodegen (3)
+                                                Project [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id]
+                                                  BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                    Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy]
+                                                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                        Filter [ss_store_sk,ss_item_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk]
+                                                                SubqueryBroadcast [d_date_sk] #1
+                                                                  ReusedExchange [d_date_sk,d_year,d_moy,d_qoy] #4
+                                                        InputAdapter
+                                                          BroadcastExchange #4
+                                                            WholeStageCodegen (1)
+                                                              Project [d_date_sk,d_year,d_moy,d_qoy]
+                                                                Filter [d_month_seq,d_date_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy]
+                                                    InputAdapter
+                                                      BroadcastExchange #5
+                                                        WholeStageCodegen (2)
+                                                          Filter [s_store_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.store [s_store_sk,s_store_id]
+                                    InputAdapter
+                                      WholeStageCodegen (6)
+                                        Sort [i_item_sk]
+                                          InputAdapter
+                                            Exchange [i_item_sk] #6
+                                              WholeStageCodegen (5)
+                                                Filter [i_item_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.item [i_item_sk,i_brand,i_class,i_category,i_product_name]
+                    WholeStageCodegen (17)
+                      HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sum,isEmpty] [sum(sumsales),s_store_id,sumsales,sum,isEmpty]
+                        InputAdapter
+                          Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #7
+                            WholeStageCodegen (16)
+                              HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sumsales] [sum,isEmpty,sum,isEmpty]
+                                HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price as decimal(12,2))) * promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00)),sumsales,sum,isEmpty]
+                                  InputAdapter
+                                    ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #8
+                    WholeStageCodegen (26)
+                      HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sum,isEmpty] [sum(sumsales),d_moy,s_store_id,sumsales,sum,isEmpty]
+                        InputAdapter
+                          Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #9
+                            WholeStageCodegen (25)
+                              HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sumsales] [sum,isEmpty,sum,isEmpty]
+                                HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price as decimal(12,2))) * promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00)),sumsales,sum,isEmpty]
+                                  InputAdapter
+                                    ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #8
+                    WholeStageCodegen (35)
+                      HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sum,isEmpty] [sum(sumsales),d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty]
+                        InputAdapter
+                          Exchange [i_category,i_class,i_brand,i_product_name,d_year] #10
+                            WholeStageCodegen (34)
+                              HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sumsales] [sum,isEmpty,sum,isEmpty]
+                                HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price as decimal(12,2))) * promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00)),sumsales,sum,isEmpty]
+                                  InputAdapter
+                                    ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #8
+                    WholeStageCodegen (44)
+                      HashAggregate [i_category,i_class,i_brand,i_product_name,sum,isEmpty] [sum(sumsales),d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty]
+                        InputAdapter
+                          Exchange [i_category,i_class,i_brand,i_product_name] #11
+                            WholeStageCodegen (43)
+                              HashAggregate [i_category,i_class,i_brand,i_product_name,sumsales] [sum,isEmpty,sum,isEmpty]
+                                HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price as decimal(12,2))) * promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00)),sumsales,sum,isEmpty]
+                                  InputAdapter
+                                    ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #8
+                    WholeStageCodegen (53)
+                      HashAggregate [i_category,i_class,i_brand,sum,isEmpty] [sum(sumsales),i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty]
+                        InputAdapter
+                          Exchange [i_category,i_class,i_brand] #12
+                            WholeStageCodegen (52)
+                              HashAggregate [i_category,i_class,i_brand,sumsales] [sum,isEmpty,sum,isEmpty]
+                                HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price as decimal(12,2))) * promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00)),sumsales,sum,isEmpty]
+                                  InputAdapter
+                                    ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #8
+                    WholeStageCodegen (62)
+                      HashAggregate [i_category,i_class,sum,isEmpty] [sum(sumsales),i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty]
+                        InputAdapter
+                          Exchange [i_category,i_class] #13
+                            WholeStageCodegen (61)
+                              HashAggregate [i_category,i_class,sumsales] [sum,isEmpty,sum,isEmpty]
+                                HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price as decimal(12,2))) * promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00)),sumsales,sum,isEmpty]
+                                  InputAdapter
+                                    ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #8
+                    WholeStageCodegen (71)
+                      HashAggregate [i_category,sum,isEmpty] [sum(sumsales),i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty]
+                        InputAdapter
+                          Exchange [i_category] #14
+                            WholeStageCodegen (70)
+                              HashAggregate [i_category,sumsales] [sum,isEmpty,sum,isEmpty]
+                                HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price as decimal(12,2))) * promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00)),sumsales,sum,isEmpty]
+                                  InputAdapter
+                                    ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #8
+                    WholeStageCodegen (80)
+                      HashAggregate [sum,isEmpty] [sum(sumsales),i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty]
+                        InputAdapter
+                          Exchange #15
+                            WholeStageCodegen (79)
+                              HashAggregate [sumsales] [sum,isEmpty,sum,isEmpty]
+                                HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price as decimal(12,2))) * promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00)),sumsales,sum,isEmpty]
+                                  InputAdapter
+                                    ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #8
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/explain.txt
new file mode 100644
index 0000000000000..e10a5ea0aff0b
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/explain.txt
@@ -0,0 +1,448 @@
+== Physical Plan ==
+TakeOrderedAndProject (71)
++- * Filter (70)
+   +- Window (69)
+      +- * Sort (68)
+         +- Exchange (67)
+            +- Union (66)
+               :- * HashAggregate (25)
+               :  +- Exchange (24)
+               :     +- * HashAggregate (23)
+               :        +- * Project (22)
+               :           +- * BroadcastHashJoin Inner BuildRight (21)
+               :              :- * Project (16)
+               :              :  +- * BroadcastHashJoin Inner BuildRight (15)
+               :              :     :- * Project (10)
+               :              :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :              :     :     :- * Filter (3)
+               :              :     :     :  +- * ColumnarToRow (2)
+               :              :     :     :     +- Scan parquet default.store_sales (1)
+               :              :     :     +- BroadcastExchange (8)
+               :              :     :        +- * Project (7)
+               :              :     :           +- * Filter (6)
+               :              :     :              +- * ColumnarToRow (5)
+               :              :     :                 +- Scan parquet default.date_dim (4)
+               :              :     +- BroadcastExchange (14)
+               :              :        +- * Filter (13)
+               :              :           +- * ColumnarToRow (12)
+               :              :              +- Scan parquet default.store (11)
+               :              +- BroadcastExchange (20)
+               :                 +- * Filter (19)
+               :                    +- * ColumnarToRow (18)
+               :                       +- Scan parquet default.item (17)
+               :- * HashAggregate (30)
+               :  +- Exchange (29)
+               :     +- * HashAggregate (28)
+               :        +- * HashAggregate (27)
+               :           +- ReusedExchange (26)
+               :- * HashAggregate (35)
+               :  +- Exchange (34)
+               :     +- * HashAggregate (33)
+               :        +- * HashAggregate (32)
+               :           +- ReusedExchange (31)
+               :- * HashAggregate (40)
+               :  +- Exchange (39)
+               :     +- * HashAggregate (38)
+               :        +- * HashAggregate (37)
+               :           +- ReusedExchange (36)
+               :- * HashAggregate (45)
+               :  +- Exchange (44)
+               :     +- * HashAggregate (43)
+               :        +- * HashAggregate (42)
+               :           +- ReusedExchange (41)
+               :- * HashAggregate (50)
+               :  +- Exchange (49)
+               :     +- * HashAggregate (48)
+               :        +- * HashAggregate (47)
+               :           +- ReusedExchange (46)
+               :- * HashAggregate (55)
+               :  +- Exchange (54)
+               :     +- * HashAggregate (53)
+               :        +- * HashAggregate (52)
+               :           +- ReusedExchange (51)
+               :- * HashAggregate (60)
+               :  +- Exchange (59)
+               :     +- * HashAggregate (58)
+               :        +- * HashAggregate (57)
+               :           +- ReusedExchange (56)
+               +- * HashAggregate (65)
+                  +- Exchange (64)
+                     +- * HashAggregate (63)
+                        +- * HashAggregate (62)
+                           +- ReusedExchange (61)
+
+
+(1) Scan parquet default.store_sales
+Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_quantity:int,ss_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5]
+
+(3) Filter [codegen id : 4]
+Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5]
+Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1))
+
+(4) Scan parquet default.date_dim
+Output [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int,d_year:int,d_moy:int,d_qoy:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11]
+
+(6) Filter [codegen id : 1]
+Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11]
+Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1212)) AND (d_month_seq#8 <= 1223)) AND isnotnull(d_date_sk#7))
+
+(7) Project [codegen id : 1]
+Output [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11]
+Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11]
+
+(8) BroadcastExchange
+Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#5]
+Right keys [1]: [d_date_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11]
+Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#7, d_year#9, d_moy#10, d_qoy#11]
+
+(11) Scan parquet default.store
+Output [2]: [s_store_sk#13, s_store_id#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [s_store_sk#13, s_store_id#14]
+
+(13) Filter [codegen id : 2]
+Input [2]: [s_store_sk#13, s_store_id#14]
+Condition : isnotnull(s_store_sk#13)
+
+(14) BroadcastExchange
+Input [2]: [s_store_sk#13, s_store_id#14]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#15]
+
+(15) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#2]
+Right keys [1]: [s_store_sk#13]
+Join condition: None
+
+(16) Project [codegen id : 4]
+Output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14]
+Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_sk#13, s_store_id#14]
+
+(17) Scan parquet default.item
+Output [5]: [i_item_sk#16, i_brand#17, i_class#18, i_category#19, i_product_name#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_brand:string,i_class:string,i_category:string,i_product_name:string>
+
+(18) ColumnarToRow [codegen id : 3]
+Input [5]: [i_item_sk#16, i_brand#17, i_class#18, i_category#19, i_product_name#20]
+
+(19) Filter [codegen id : 3]
+Input [5]: [i_item_sk#16, i_brand#17, i_class#18, i_category#19, i_product_name#20]
+Condition : isnotnull(i_item_sk#16)
+
+(20) BroadcastExchange
+Input [5]: [i_item_sk#16, i_brand#17, i_class#18, i_category#19, i_product_name#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#21]
+
+(21) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#16]
+Join condition: None
+
+(22) Project [codegen id : 4]
+Output [10]: [ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14, i_brand#17, i_class#18, i_category#19, i_product_name#20]
+Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14, i_item_sk#16, i_brand#17, i_class#18, i_category#19, i_product_name#20]
+
+(23) HashAggregate [codegen id : 4]
+Input [10]: [ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14, i_brand#17, i_class#18, i_category#19, i_product_name#20]
+Keys [8]: [i_category#19, i_class#18, i_brand#17, i_product_name#20, d_year#9, d_qoy#11, d_moy#10, s_store_id#14]
+Functions [1]: [partial_sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#4 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#3 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))]
+Aggregate Attributes [2]: [sum#22, isEmpty#23]
+Results [10]: [i_category#19, i_class#18, i_brand#17, i_product_name#20, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, sum#24, isEmpty#25]
+
+(24) Exchange
+Input [10]: [i_category#19, i_class#18, i_brand#17, i_product_name#20, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, sum#24, isEmpty#25]
+Arguments: hashpartitioning(i_category#19, i_class#18, i_brand#17, i_product_name#20, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, 5), ENSURE_REQUIREMENTS, [id=#26]
+
+(25) HashAggregate [codegen id : 5]
+Input [10]: [i_category#19, i_class#18, i_brand#17, i_product_name#20, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, sum#24, isEmpty#25]
+Keys [8]: [i_category#19, i_class#18, i_brand#17, i_product_name#20, d_year#9, d_qoy#11, d_moy#10, s_store_id#14]
+Functions [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#4 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#3 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))]
+Aggregate Attributes [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#4 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#3 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#27]
+Results [9]: [i_category#19, i_class#18, i_brand#17, i_product_name#20, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, cast(sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#4 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#3 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#27 as decimal(38,2)) AS sumsales#28]
+
+(26) ReusedExchange [Reuses operator id: unknown]
+Output [10]: [i_category#29, i_class#30, i_brand#31, i_product_name#32, d_year#33, d_qoy#34, d_moy#35, s_store_id#36, sum#37, isEmpty#38]
+
+(27) HashAggregate [codegen id : 10]
+Input [10]: [i_category#29, i_class#30, i_brand#31, i_product_name#32, d_year#33, d_qoy#34, d_moy#35, s_store_id#36, sum#37, isEmpty#38]
+Keys [8]: [i_category#29, i_class#30, i_brand#31, i_product_name#32, d_year#33, d_qoy#34, d_moy#35, s_store_id#36]
+Functions [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#39 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#40 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))]
+Aggregate Attributes [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#39 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#40 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#41]
+Results [8]: [i_category#29, i_class#30, i_brand#31, i_product_name#32, d_year#33, d_qoy#34, d_moy#35, sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#39 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#40 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#41 AS sumsales#42]
+
+(28) HashAggregate [codegen id : 10]
+Input [8]: [i_category#29, i_class#30, i_brand#31, i_product_name#32, d_year#33, d_qoy#34, d_moy#35, sumsales#42]
+Keys [7]: [i_category#29, i_class#30, i_brand#31, i_product_name#32, d_year#33, d_qoy#34, d_moy#35]
+Functions [1]: [partial_sum(sumsales#42)]
+Aggregate Attributes [2]: [sum#43, isEmpty#44]
+Results [9]: [i_category#29, i_class#30, i_brand#31, i_product_name#32, d_year#33, d_qoy#34, d_moy#35, sum#45, isEmpty#46]
+
+(29) Exchange
+Input [9]: [i_category#29, i_class#30, i_brand#31, i_product_name#32, d_year#33, d_qoy#34, d_moy#35, sum#45, isEmpty#46]
+Arguments: hashpartitioning(i_category#29, i_class#30, i_brand#31, i_product_name#32, d_year#33, d_qoy#34, d_moy#35, 5), ENSURE_REQUIREMENTS, [id=#47]
+
+(30) HashAggregate [codegen id : 11]
+Input [9]: [i_category#29, i_class#30, i_brand#31, i_product_name#32, d_year#33, d_qoy#34, d_moy#35, sum#45, isEmpty#46]
+Keys [7]: [i_category#29, i_class#30, i_brand#31, i_product_name#32, d_year#33, d_qoy#34, d_moy#35]
+Functions [1]: [sum(sumsales#42)]
+Aggregate Attributes [1]: [sum(sumsales#42)#48]
+Results [9]: [i_category#29, i_class#30, i_brand#31, i_product_name#32, d_year#33, d_qoy#34, d_moy#35, null AS s_store_id#49, sum(sumsales#42)#48 AS sumsales#50]
+
+(31) ReusedExchange [Reuses operator id: unknown]
+Output [10]: [i_category#51, i_class#52, i_brand#53, i_product_name#54, d_year#55, d_qoy#56, d_moy#57, s_store_id#58, sum#59, isEmpty#60]
+
+(32) HashAggregate [codegen id : 16]
+Input [10]: [i_category#51, i_class#52, i_brand#53, i_product_name#54, d_year#55, d_qoy#56, d_moy#57, s_store_id#58, sum#59, isEmpty#60]
+Keys [8]: [i_category#51, i_class#52, i_brand#53, i_product_name#54, d_year#55, d_qoy#56, d_moy#57, s_store_id#58]
+Functions [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#61 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#62 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))]
+Aggregate Attributes [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#61 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#62 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#63]
+Results [7]: [i_category#51, i_class#52, i_brand#53, i_product_name#54, d_year#55, d_qoy#56, sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#61 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#62 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#63 AS sumsales#42]
+
+(33) HashAggregate [codegen id : 16]
+Input [7]: [i_category#51, i_class#52, i_brand#53, i_product_name#54, d_year#55, d_qoy#56, sumsales#42]
+Keys [6]: [i_category#51, i_class#52, i_brand#53, i_product_name#54, d_year#55, d_qoy#56]
+Functions [1]: [partial_sum(sumsales#42)]
+Aggregate Attributes [2]: [sum#64, isEmpty#65]
+Results [8]: [i_category#51, i_class#52, i_brand#53, i_product_name#54, d_year#55, d_qoy#56, sum#66, isEmpty#67]
+
+(34) Exchange
+Input [8]: [i_category#51, i_class#52, i_brand#53, i_product_name#54, d_year#55, d_qoy#56, sum#66, isEmpty#67]
+Arguments: hashpartitioning(i_category#51, i_class#52, i_brand#53, i_product_name#54, d_year#55, d_qoy#56, 5), ENSURE_REQUIREMENTS, [id=#68]
+
+(35) HashAggregate [codegen id : 17]
+Input [8]: [i_category#51, i_class#52, i_brand#53, i_product_name#54, d_year#55, d_qoy#56, sum#66, isEmpty#67]
+Keys [6]: [i_category#51, i_class#52, i_brand#53, i_product_name#54, d_year#55, d_qoy#56]
+Functions [1]: [sum(sumsales#42)]
+Aggregate Attributes [1]: [sum(sumsales#42)#69]
+Results [9]: [i_category#51, i_class#52, i_brand#53, i_product_name#54, d_year#55, d_qoy#56, null AS d_moy#70, null AS s_store_id#71, sum(sumsales#42)#69 AS sumsales#72]
+
+(36) ReusedExchange [Reuses operator id: unknown]
+Output [10]: [i_category#73, i_class#74, i_brand#75, i_product_name#76, d_year#77, d_qoy#78, d_moy#79, s_store_id#80, sum#81, isEmpty#82]
+
+(37) HashAggregate [codegen id : 22]
+Input [10]: [i_category#73, i_class#74, i_brand#75, i_product_name#76, d_year#77, d_qoy#78, d_moy#79, s_store_id#80, sum#81, isEmpty#82]
+Keys [8]: [i_category#73, i_class#74, i_brand#75, i_product_name#76, d_year#77, d_qoy#78, d_moy#79, s_store_id#80]
+Functions [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#83 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#84 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))]
+Aggregate Attributes [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#83 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#84 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#85]
+Results [6]: [i_category#73, i_class#74, i_brand#75, i_product_name#76, d_year#77, sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#83 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#84 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#85 AS sumsales#42]
+
+(38) HashAggregate [codegen id : 22]
+Input [6]: [i_category#73, i_class#74, i_brand#75, i_product_name#76, d_year#77, sumsales#42]
+Keys [5]: [i_category#73, i_class#74, i_brand#75, i_product_name#76, d_year#77]
+Functions [1]: [partial_sum(sumsales#42)]
+Aggregate Attributes [2]: [sum#86, isEmpty#87]
+Results [7]: [i_category#73, i_class#74, i_brand#75, i_product_name#76, d_year#77, sum#88, isEmpty#89]
+
+(39) Exchange
+Input [7]: [i_category#73, i_class#74, i_brand#75, i_product_name#76, d_year#77, sum#88, isEmpty#89]
+Arguments: hashpartitioning(i_category#73, i_class#74, i_brand#75, i_product_name#76, d_year#77, 5), ENSURE_REQUIREMENTS, [id=#90]
+
+(40) HashAggregate [codegen id : 23]
+Input [7]: [i_category#73, i_class#74, i_brand#75, i_product_name#76, d_year#77, sum#88, isEmpty#89]
+Keys [5]: [i_category#73, i_class#74, i_brand#75, i_product_name#76, d_year#77]
+Functions [1]: [sum(sumsales#42)]
+Aggregate Attributes [1]: [sum(sumsales#42)#91]
+Results [9]: [i_category#73, i_class#74, i_brand#75, i_product_name#76, d_year#77, null AS d_qoy#92, null AS d_moy#93, null AS s_store_id#94, sum(sumsales#42)#91 AS sumsales#95]
+
+(41) ReusedExchange [Reuses operator id: unknown]
+Output [10]: [i_category#96, i_class#97, i_brand#98, i_product_name#99, d_year#100, d_qoy#101, d_moy#102, s_store_id#103, sum#104, isEmpty#105]
+
+(42) HashAggregate [codegen id : 28]
+Input [10]: [i_category#96, i_class#97, i_brand#98, i_product_name#99, d_year#100, d_qoy#101, d_moy#102, s_store_id#103, sum#104, isEmpty#105]
+Keys [8]: [i_category#96, i_class#97, i_brand#98, i_product_name#99, d_year#100, d_qoy#101, d_moy#102, s_store_id#103]
+Functions [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#106 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#107 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))]
+Aggregate Attributes [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#106 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#107 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#108]
+Results [5]: [i_category#96, i_class#97, i_brand#98, i_product_name#99, sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#106 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#107 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#108 AS sumsales#42]
+
+(43) HashAggregate [codegen id : 28]
+Input [5]: [i_category#96, i_class#97, i_brand#98, i_product_name#99, sumsales#42]
+Keys [4]: [i_category#96, i_class#97, i_brand#98, i_product_name#99]
+Functions [1]: [partial_sum(sumsales#42)]
+Aggregate Attributes [2]: [sum#109, isEmpty#110]
+Results [6]: [i_category#96, i_class#97, i_brand#98, i_product_name#99, sum#111, isEmpty#112]
+
+(44) Exchange
+Input [6]: [i_category#96, i_class#97, i_brand#98, i_product_name#99, sum#111, isEmpty#112]
+Arguments: hashpartitioning(i_category#96, i_class#97, i_brand#98, i_product_name#99, 5), ENSURE_REQUIREMENTS, [id=#113]
+
+(45) HashAggregate [codegen id : 29]
+Input [6]: [i_category#96, i_class#97, i_brand#98, i_product_name#99, sum#111, isEmpty#112]
+Keys [4]: [i_category#96, i_class#97, i_brand#98, i_product_name#99]
+Functions [1]: [sum(sumsales#42)]
+Aggregate Attributes [1]: [sum(sumsales#42)#114]
+Results [9]: [i_category#96, i_class#97, i_brand#98, i_product_name#99, null AS d_year#115, null AS d_qoy#116, null AS d_moy#117, null AS s_store_id#118, sum(sumsales#42)#114 AS sumsales#119]
+
+(46) ReusedExchange [Reuses operator id: unknown]
+Output [10]: [i_category#120, i_class#121, i_brand#122, i_product_name#123, d_year#124, d_qoy#125, d_moy#126, s_store_id#127, sum#128, isEmpty#129]
+
+(47) HashAggregate [codegen id : 34]
+Input [10]: [i_category#120, i_class#121, i_brand#122, i_product_name#123, d_year#124, d_qoy#125, d_moy#126, s_store_id#127, sum#128, isEmpty#129]
+Keys [8]: [i_category#120, i_class#121, i_brand#122, i_product_name#123, d_year#124, d_qoy#125, d_moy#126, s_store_id#127]
+Functions [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#130 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#131 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))]
+Aggregate Attributes [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#130 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#131 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#132]
+Results [4]: [i_category#120, i_class#121, i_brand#122, sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#130 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#131 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#132 AS sumsales#42]
+
+(48) HashAggregate [codegen id : 34]
+Input [4]: [i_category#120, i_class#121, i_brand#122, sumsales#42]
+Keys [3]: [i_category#120, i_class#121, i_brand#122]
+Functions [1]: [partial_sum(sumsales#42)]
+Aggregate Attributes [2]: [sum#133, isEmpty#134]
+Results [5]: [i_category#120, i_class#121, i_brand#122, sum#135, isEmpty#136]
+
+(49) Exchange
+Input [5]: [i_category#120, i_class#121, i_brand#122, sum#135, isEmpty#136]
+Arguments: hashpartitioning(i_category#120, i_class#121, i_brand#122, 5), ENSURE_REQUIREMENTS, [id=#137]
+
+(50) HashAggregate [codegen id : 35]
+Input [5]: [i_category#120, i_class#121, i_brand#122, sum#135, isEmpty#136]
+Keys [3]: [i_category#120, i_class#121, i_brand#122]
+Functions [1]: [sum(sumsales#42)]
+Aggregate Attributes [1]: [sum(sumsales#42)#138]
+Results [9]: [i_category#120, i_class#121, i_brand#122, null AS i_product_name#139, null AS d_year#140, null AS d_qoy#141, null AS d_moy#142, null AS s_store_id#143, sum(sumsales#42)#138 AS sumsales#144]
+
+(51) ReusedExchange [Reuses operator id: unknown]
+Output [10]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sum#153, isEmpty#154]
+
+(52) HashAggregate [codegen id : 40]
+Input [10]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sum#153, isEmpty#154]
+Keys [8]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152]
+Functions [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#155 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#156 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))]
+Aggregate Attributes [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#155 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#156 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#157]
+Results [3]: [i_category#145, i_class#146, sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#155 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#156 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#157 AS sumsales#42]
+
+(53) HashAggregate [codegen id : 40]
+Input [3]: [i_category#145, i_class#146, sumsales#42]
+Keys [2]: [i_category#145, i_class#146]
+Functions [1]: [partial_sum(sumsales#42)]
+Aggregate Attributes [2]: [sum#158, isEmpty#159]
+Results [4]: [i_category#145, i_class#146, sum#160, isEmpty#161]
+
+(54) Exchange
+Input [4]: [i_category#145, i_class#146, sum#160, isEmpty#161]
+Arguments: hashpartitioning(i_category#145, i_class#146, 5), ENSURE_REQUIREMENTS, [id=#162]
+
+(55) HashAggregate [codegen id : 41]
+Input [4]: [i_category#145, i_class#146, sum#160, isEmpty#161]
+Keys [2]: [i_category#145, i_class#146]
+Functions [1]: [sum(sumsales#42)]
+Aggregate Attributes [1]: [sum(sumsales#42)#163]
+Results [9]: [i_category#145, i_class#146, null AS i_brand#164, null AS i_product_name#165, null AS d_year#166, null AS d_qoy#167, null AS d_moy#168, null AS s_store_id#169, sum(sumsales#42)#163 AS sumsales#170]
+
+(56) ReusedExchange [Reuses operator id: unknown]
+Output [10]: [i_category#171, i_class#172, i_brand#173, i_product_name#174, d_year#175, d_qoy#176, d_moy#177, s_store_id#178, sum#179, isEmpty#180]
+
+(57) HashAggregate [codegen id : 46]
+Input [10]: [i_category#171, i_class#172, i_brand#173, i_product_name#174, d_year#175, d_qoy#176, d_moy#177, s_store_id#178, sum#179, isEmpty#180]
+Keys [8]: [i_category#171, i_class#172, i_brand#173, i_product_name#174, d_year#175, d_qoy#176, d_moy#177, s_store_id#178]
+Functions [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#181 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#182 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))]
+Aggregate Attributes [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#181 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#182 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#183]
+Results [2]: [i_category#171, sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#181 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#182 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#183 AS sumsales#42]
+
+(58) HashAggregate [codegen id : 46]
+Input [2]: [i_category#171, sumsales#42]
+Keys [1]: [i_category#171]
+Functions [1]: [partial_sum(sumsales#42)]
+Aggregate Attributes [2]: [sum#184, isEmpty#185]
+Results [3]: [i_category#171, sum#186, isEmpty#187]
+
+(59) Exchange
+Input [3]: [i_category#171, sum#186, isEmpty#187]
+Arguments: hashpartitioning(i_category#171, 5), ENSURE_REQUIREMENTS, [id=#188]
+
+(60) HashAggregate [codegen id : 47]
+Input [3]: [i_category#171, sum#186, isEmpty#187]
+Keys [1]: [i_category#171]
+Functions [1]: [sum(sumsales#42)]
+Aggregate Attributes [1]: [sum(sumsales#42)#189]
+Results [9]: [i_category#171, null AS i_class#190, null AS i_brand#191, null AS i_product_name#192, null AS d_year#193, null AS d_qoy#194, null AS d_moy#195, null AS s_store_id#196, sum(sumsales#42)#189 AS sumsales#197]
+
+(61) ReusedExchange [Reuses operator id: unknown]
+Output [10]: [i_category#198, i_class#199, i_brand#200, i_product_name#201, d_year#202, d_qoy#203, d_moy#204, s_store_id#205, sum#206, isEmpty#207]
+
+(62) HashAggregate [codegen id : 52]
+Input [10]: [i_category#198, i_class#199, i_brand#200, i_product_name#201, d_year#202, d_qoy#203, d_moy#204, s_store_id#205, sum#206, isEmpty#207]
+Keys [8]: [i_category#198, i_class#199, i_brand#200, i_product_name#201, d_year#202, d_qoy#203, d_moy#204, s_store_id#205]
+Functions [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#208 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#209 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))]
+Aggregate Attributes [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#208 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#209 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#210]
+Results [1]: [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price#208 as decimal(12,2))) * promote_precision(cast(cast(ss_quantity#209 as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00))#210 AS sumsales#42]
+
+(63) HashAggregate [codegen id : 52]
+Input [1]: [sumsales#42]
+Keys: []
+Functions [1]: [partial_sum(sumsales#42)]
+Aggregate Attributes [2]: [sum#211, isEmpty#212]
+Results [2]: [sum#213, isEmpty#214]
+
+(64) Exchange
+Input [2]: [sum#213, isEmpty#214]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#215]
+
+(65) HashAggregate [codegen id : 53]
+Input [2]: [sum#213, isEmpty#214]
+Keys: []
+Functions [1]: [sum(sumsales#42)]
+Aggregate Attributes [1]: [sum(sumsales#42)#216]
+Results [9]: [null AS i_category#217, null AS i_class#218, null AS i_brand#219, null AS i_product_name#220, null AS d_year#221, null AS d_qoy#222, null AS d_moy#223, null AS s_store_id#224, sum(sumsales#42)#216 AS sumsales#225]
+
+(66) Union
+
+(67) Exchange
+Input [9]: [i_category#19, i_class#18, i_brand#17, i_product_name#20, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, sumsales#28]
+Arguments: hashpartitioning(i_category#19, 5), ENSURE_REQUIREMENTS, [id=#226]
+
+(68) Sort [codegen id : 54]
+Input [9]: [i_category#19, i_class#18, i_brand#17, i_product_name#20, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, sumsales#28]
+Arguments: [i_category#19 ASC NULLS FIRST, sumsales#28 DESC NULLS LAST], false, 0
+
+(69) Window
+Input [9]: [i_category#19, i_class#18, i_brand#17, i_product_name#20, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, sumsales#28]
+Arguments: [rank(sumsales#28) windowspecdefinition(i_category#19, sumsales#28 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#227], [i_category#19], [sumsales#28 DESC NULLS LAST]
+
+(70) Filter [codegen id : 55]
+Input [10]: [i_category#19, i_class#18, i_brand#17, i_product_name#20, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, sumsales#28, rk#227]
+Condition : (rk#227 <= 100)
+
+(71) TakeOrderedAndProject
+Input [10]: [i_category#19, i_class#18, i_brand#17, i_product_name#20, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, sumsales#28, rk#227]
+Arguments: 100, [i_category#19 ASC NULLS FIRST, i_class#18 ASC NULLS FIRST, i_brand#17 ASC NULLS FIRST, i_product_name#20 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_qoy#11 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, s_store_id#14 ASC NULLS FIRST, sumsales#28 ASC NULLS FIRST, rk#227 ASC NULLS FIRST], [i_category#19, i_class#18, i_brand#17, i_product_name#20, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, sumsales#28, rk#227]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (72)
+
+
+(72) ReusedExchange [Reuses operator id: 8]
+Output [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/simplified.txt
new file mode 100644
index 0000000000000..c5735c20728c5
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/simplified.txt
@@ -0,0 +1,122 @@
+TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,rk]
+  WholeStageCodegen (55)
+    Filter [rk]
+      InputAdapter
+        Window [sumsales,i_category]
+          WholeStageCodegen (54)
+            Sort [i_category,sumsales]
+              InputAdapter
+                Exchange [i_category] #1
+                  Union
+                    WholeStageCodegen (5)
+                      HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price as decimal(12,2))) * promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00)),sumsales,sum,isEmpty]
+                        InputAdapter
+                          Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] #2
+                            WholeStageCodegen (4)
+                              HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,ss_sales_price,ss_quantity] [sum,isEmpty,sum,isEmpty]
+                                Project [ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_brand,i_class,i_category,i_product_name]
+                                  BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                    Project [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id]
+                                      BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                        Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy]
+                                          BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                            Filter [ss_store_sk,ss_item_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #1
+                                                      ReusedExchange [d_date_sk,d_year,d_moy,d_qoy] #3
+                                            InputAdapter
+                                              BroadcastExchange #3
+                                                WholeStageCodegen (1)
+                                                  Project [d_date_sk,d_year,d_moy,d_qoy]
+                                                    Filter [d_month_seq,d_date_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy]
+                                        InputAdapter
+                                          BroadcastExchange #4
+                                            WholeStageCodegen (2)
+                                              Filter [s_store_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.store [s_store_sk,s_store_id]
+                                    InputAdapter
+                                      BroadcastExchange #5
+                                        WholeStageCodegen (3)
+                                          Filter [i_item_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.item [i_item_sk,i_brand,i_class,i_category,i_product_name]
+                    WholeStageCodegen (11)
+                      HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sum,isEmpty] [sum(sumsales),s_store_id,sumsales,sum,isEmpty]
+                        InputAdapter
+                          Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #6
+                            WholeStageCodegen (10)
+                              HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sumsales] [sum,isEmpty,sum,isEmpty]
+                                HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price as decimal(12,2))) * promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00)),sumsales,sum,isEmpty]
+                                  InputAdapter
+                                    ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #7
+                    WholeStageCodegen (17)
+                      HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sum,isEmpty] [sum(sumsales),d_moy,s_store_id,sumsales,sum,isEmpty]
+                        InputAdapter
+                          Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #8
+                            WholeStageCodegen (16)
+                              HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sumsales] [sum,isEmpty,sum,isEmpty]
+                                HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price as decimal(12,2))) * promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00)),sumsales,sum,isEmpty]
+                                  InputAdapter
+                                    ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #7
+                    WholeStageCodegen (23)
+                      HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sum,isEmpty] [sum(sumsales),d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty]
+                        InputAdapter
+                          Exchange [i_category,i_class,i_brand,i_product_name,d_year] #9
+                            WholeStageCodegen (22)
+                              HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sumsales] [sum,isEmpty,sum,isEmpty]
+                                HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price as decimal(12,2))) * promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00)),sumsales,sum,isEmpty]
+                                  InputAdapter
+                                    ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #7
+                    WholeStageCodegen (29)
+                      HashAggregate [i_category,i_class,i_brand,i_product_name,sum,isEmpty] [sum(sumsales),d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty]
+                        InputAdapter
+                          Exchange [i_category,i_class,i_brand,i_product_name] #10
+                            WholeStageCodegen (28)
+                              HashAggregate [i_category,i_class,i_brand,i_product_name,sumsales] [sum,isEmpty,sum,isEmpty]
+                                HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price as decimal(12,2))) * promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00)),sumsales,sum,isEmpty]
+                                  InputAdapter
+                                    ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #7
+                    WholeStageCodegen (35)
+                      HashAggregate [i_category,i_class,i_brand,sum,isEmpty] [sum(sumsales),i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty]
+                        InputAdapter
+                          Exchange [i_category,i_class,i_brand] #11
+                            WholeStageCodegen (34)
+                              HashAggregate [i_category,i_class,i_brand,sumsales] [sum,isEmpty,sum,isEmpty]
+                                HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price as decimal(12,2))) * promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00)),sumsales,sum,isEmpty]
+                                  InputAdapter
+                                    ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #7
+                    WholeStageCodegen (41)
+                      HashAggregate [i_category,i_class,sum,isEmpty] [sum(sumsales),i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty]
+                        InputAdapter
+                          Exchange [i_category,i_class] #12
+                            WholeStageCodegen (40)
+                              HashAggregate [i_category,i_class,sumsales] [sum,isEmpty,sum,isEmpty]
+                                HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price as decimal(12,2))) * promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00)),sumsales,sum,isEmpty]
+                                  InputAdapter
+                                    ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #7
+                    WholeStageCodegen (47)
+                      HashAggregate [i_category,sum,isEmpty] [sum(sumsales),i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty]
+                        InputAdapter
+                          Exchange [i_category] #13
+                            WholeStageCodegen (46)
+                              HashAggregate [i_category,sumsales] [sum,isEmpty,sum,isEmpty]
+                                HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price as decimal(12,2))) * promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00)),sumsales,sum,isEmpty]
+                                  InputAdapter
+                                    ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #7
+                    WholeStageCodegen (53)
+                      HashAggregate [sum,isEmpty] [sum(sumsales),i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty]
+                        InputAdapter
+                          Exchange #14
+                            WholeStageCodegen (52)
+                              HashAggregate [sumsales] [sum,isEmpty,sum,isEmpty]
+                                HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce(CheckOverflow((promote_precision(cast(ss_sales_price as decimal(12,2))) * promote_precision(cast(cast(ss_quantity as decimal(10,0)) as decimal(12,2)))), DecimalType(18,2), true), 0.00)),sumsales,sum,isEmpty]
+                                  InputAdapter
+                                    ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #7
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.sf100/explain.txt
new file mode 100644
index 0000000000000..35ed13e84c742
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.sf100/explain.txt
@@ -0,0 +1,363 @@
+== Physical Plan ==
+TakeOrderedAndProject (60)
++- * Project (59)
+   +- Window (58)
+      +- * Sort (57)
+         +- Exchange (56)
+            +- * HashAggregate (55)
+               +- Exchange (54)
+                  +- * HashAggregate (53)
+                     +- Union (52)
+                        :- * HashAggregate (41)
+                        :  +- Exchange (40)
+                        :     +- * HashAggregate (39)
+                        :        +- * Project (38)
+                        :           +- * BroadcastHashJoin Inner BuildRight (37)
+                        :              :- * Project (10)
+                        :              :  +- * BroadcastHashJoin Inner BuildRight (9)
+                        :              :     :- * Filter (3)
+                        :              :     :  +- * ColumnarToRow (2)
+                        :              :     :     +- Scan parquet default.store_sales (1)
+                        :              :     +- BroadcastExchange (8)
+                        :              :        +- * Project (7)
+                        :              :           +- * Filter (6)
+                        :              :              +- * ColumnarToRow (5)
+                        :              :                 +- Scan parquet default.date_dim (4)
+                        :              +- BroadcastExchange (36)
+                        :                 +- * BroadcastHashJoin LeftSemi BuildRight (35)
+                        :                    :- * Filter (13)
+                        :                    :  +- * ColumnarToRow (12)
+                        :                    :     +- Scan parquet default.store (11)
+                        :                    +- BroadcastExchange (34)
+                        :                       +- * Project (33)
+                        :                          +- * Filter (32)
+                        :                             +- Window (31)
+                        :                                +- * Sort (30)
+                        :                                   +- Exchange (29)
+                        :                                      +- * HashAggregate (28)
+                        :                                         +- Exchange (27)
+                        :                                            +- * HashAggregate (26)
+                        :                                               +- * Project (25)
+                        :                                                  +- * BroadcastHashJoin Inner BuildRight (24)
+                        :                                                     :- * Project (19)
+                        :                                                     :  +- * BroadcastHashJoin Inner BuildRight (18)
+                        :                                                     :     :- * Filter (16)
+                        :                                                     :     :  +- * ColumnarToRow (15)
+                        :                                                     :     :     +- Scan parquet default.store_sales (14)
+                        :                                                     :     +- ReusedExchange (17)
+                        :                                                     +- BroadcastExchange (23)
+                        :                                                        +- * Filter (22)
+                        :                                                           +- * ColumnarToRow (21)
+                        :                                                              +- Scan parquet default.store (20)
+                        :- * HashAggregate (46)
+                        :  +- Exchange (45)
+                        :     +- * HashAggregate (44)
+                        :        +- * HashAggregate (43)
+                        :           +- ReusedExchange (42)
+                        +- * HashAggregate (51)
+                           +- Exchange (50)
+                              +- * HashAggregate (49)
+                                 +- * HashAggregate (48)
+                                    +- ReusedExchange (47)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 9]
+Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 9]
+Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_store_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#5, d_month_seq#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#5]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#5]
+Join condition: None
+
+(10) Project [codegen id : 9]
+Output [2]: [ss_store_sk#1, ss_net_profit#2]
+Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#5]
+
+(11) Scan parquet default.store
+Output [3]: [s_store_sk#8, s_county#9, s_state#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_county:string,s_state:string>
+
+(12) ColumnarToRow [codegen id : 8]
+Input [3]: [s_store_sk#8, s_county#9, s_state#10]
+
+(13) Filter [codegen id : 8]
+Input [3]: [s_store_sk#8, s_county#9, s_state#10]
+Condition : isnotnull(s_store_sk#8)
+
+(14) Scan parquet default.store_sales
+Output [3]: [ss_store_sk#11, ss_net_profit#12, ss_sold_date_sk#13]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#13), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_net_profit:decimal(7,2)>
+
+(15) ColumnarToRow [codegen id : 4]
+Input [3]: [ss_store_sk#11, ss_net_profit#12, ss_sold_date_sk#13]
+
+(16) Filter [codegen id : 4]
+Input [3]: [ss_store_sk#11, ss_net_profit#12, ss_sold_date_sk#13]
+Condition : isnotnull(ss_store_sk#11)
+
+(17) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#14]
+
+(18) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#13]
+Right keys [1]: [d_date_sk#14]
+Join condition: None
+
+(19) Project [codegen id : 4]
+Output [2]: [ss_store_sk#11, ss_net_profit#12]
+Input [4]: [ss_store_sk#11, ss_net_profit#12, ss_sold_date_sk#13, d_date_sk#14]
+
+(20) Scan parquet default.store
+Output [2]: [s_store_sk#15, s_state#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_state:string>
+
+(21) ColumnarToRow [codegen id : 3]
+Input [2]: [s_store_sk#15, s_state#16]
+
+(22) Filter [codegen id : 3]
+Input [2]: [s_store_sk#15, s_state#16]
+Condition : isnotnull(s_store_sk#15)
+
+(23) BroadcastExchange
+Input [2]: [s_store_sk#15, s_state#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#17]
+
+(24) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#11]
+Right keys [1]: [s_store_sk#15]
+Join condition: None
+
+(25) Project [codegen id : 4]
+Output [2]: [ss_net_profit#12, s_state#16]
+Input [4]: [ss_store_sk#11, ss_net_profit#12, s_store_sk#15, s_state#16]
+
+(26) HashAggregate [codegen id : 4]
+Input [2]: [ss_net_profit#12, s_state#16]
+Keys [1]: [s_state#16]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#12))]
+Aggregate Attributes [1]: [sum#18]
+Results [2]: [s_state#16, sum#19]
+
+(27) Exchange
+Input [2]: [s_state#16, sum#19]
+Arguments: hashpartitioning(s_state#16, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(28) HashAggregate [codegen id : 5]
+Input [2]: [s_state#16, sum#19]
+Keys [1]: [s_state#16]
+Functions [1]: [sum(UnscaledValue(ss_net_profit#12))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#12))#21]
+Results [3]: [s_state#16 AS s_state#22, s_state#16, MakeDecimal(sum(UnscaledValue(ss_net_profit#12))#21,17,2) AS _w2#23]
+
+(29) Exchange
+Input [3]: [s_state#22, s_state#16, _w2#23]
+Arguments: hashpartitioning(s_state#16, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(30) Sort [codegen id : 6]
+Input [3]: [s_state#22, s_state#16, _w2#23]
+Arguments: [s_state#16 ASC NULLS FIRST, _w2#23 DESC NULLS LAST], false, 0
+
+(31) Window
+Input [3]: [s_state#22, s_state#16, _w2#23]
+Arguments: [rank(_w2#23) windowspecdefinition(s_state#16, _w2#23 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#25], [s_state#16], [_w2#23 DESC NULLS LAST]
+
+(32) Filter [codegen id : 7]
+Input [4]: [s_state#22, s_state#16, _w2#23, ranking#25]
+Condition : (ranking#25 <= 5)
+
+(33) Project [codegen id : 7]
+Output [1]: [s_state#22]
+Input [4]: [s_state#22, s_state#16, _w2#23, ranking#25]
+
+(34) BroadcastExchange
+Input [1]: [s_state#22]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#26]
+
+(35) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [s_state#10]
+Right keys [1]: [s_state#22]
+Join condition: None
+
+(36) BroadcastExchange
+Input [3]: [s_store_sk#8, s_county#9, s_state#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#27]
+
+(37) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_store_sk#1]
+Right keys [1]: [s_store_sk#8]
+Join condition: None
+
+(38) Project [codegen id : 9]
+Output [3]: [ss_net_profit#2, s_county#9, s_state#10]
+Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#8, s_county#9, s_state#10]
+
+(39) HashAggregate [codegen id : 9]
+Input [3]: [ss_net_profit#2, s_county#9, s_state#10]
+Keys [2]: [s_state#10, s_county#9]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))]
+Aggregate Attributes [1]: [sum#28]
+Results [3]: [s_state#10, s_county#9, sum#29]
+
+(40) Exchange
+Input [3]: [s_state#10, s_county#9, sum#29]
+Arguments: hashpartitioning(s_state#10, s_county#9, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(41) HashAggregate [codegen id : 10]
+Input [3]: [s_state#10, s_county#9, sum#29]
+Keys [2]: [s_state#10, s_county#9]
+Functions [1]: [sum(UnscaledValue(ss_net_profit#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#31]
+Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#31,17,2) as decimal(27,2)) AS total_sum#32, s_state#10, s_county#9, 0 AS g_state#33, 0 AS g_county#34, 0 AS lochierarchy#35]
+
+(42) ReusedExchange [Reuses operator id: unknown]
+Output [3]: [s_state#36, s_county#37, sum#38]
+
+(43) HashAggregate [codegen id : 20]
+Input [3]: [s_state#36, s_county#37, sum#38]
+Keys [2]: [s_state#36, s_county#37]
+Functions [1]: [sum(UnscaledValue(ss_net_profit#39))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#39))#40]
+Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#39))#40,17,2) AS total_sum#41, s_state#36]
+
+(44) HashAggregate [codegen id : 20]
+Input [2]: [total_sum#41, s_state#36]
+Keys [1]: [s_state#36]
+Functions [1]: [partial_sum(total_sum#41)]
+Aggregate Attributes [2]: [sum#42, isEmpty#43]
+Results [3]: [s_state#36, sum#44, isEmpty#45]
+
+(45) Exchange
+Input [3]: [s_state#36, sum#44, isEmpty#45]
+Arguments: hashpartitioning(s_state#36, 5), ENSURE_REQUIREMENTS, [id=#46]
+
+(46) HashAggregate [codegen id : 21]
+Input [3]: [s_state#36, sum#44, isEmpty#45]
+Keys [1]: [s_state#36]
+Functions [1]: [sum(total_sum#41)]
+Aggregate Attributes [1]: [sum(total_sum#41)#47]
+Results [6]: [sum(total_sum#41)#47 AS total_sum#48, s_state#36, null AS s_county#49, 0 AS g_state#50, 1 AS g_county#51, 1 AS lochierarchy#52]
+
+(47) ReusedExchange [Reuses operator id: unknown]
+Output [3]: [s_state#53, s_county#54, sum#55]
+
+(48) HashAggregate [codegen id : 31]
+Input [3]: [s_state#53, s_county#54, sum#55]
+Keys [2]: [s_state#53, s_county#54]
+Functions [1]: [sum(UnscaledValue(ss_net_profit#56))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#56))#57]
+Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#56))#57,17,2) AS total_sum#41]
+
+(49) HashAggregate [codegen id : 31]
+Input [1]: [total_sum#41]
+Keys: []
+Functions [1]: [partial_sum(total_sum#41)]
+Aggregate Attributes [2]: [sum#58, isEmpty#59]
+Results [2]: [sum#60, isEmpty#61]
+
+(50) Exchange
+Input [2]: [sum#60, isEmpty#61]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#62]
+
+(51) HashAggregate [codegen id : 32]
+Input [2]: [sum#60, isEmpty#61]
+Keys: []
+Functions [1]: [sum(total_sum#41)]
+Aggregate Attributes [1]: [sum(total_sum#41)#63]
+Results [6]: [sum(total_sum#41)#63 AS total_sum#64, null AS s_state#65, null AS s_county#66, 1 AS g_state#67, 1 AS g_county#68, 2 AS lochierarchy#69]
+
+(52) Union
+
+(53) HashAggregate [codegen id : 33]
+Input [6]: [total_sum#32, s_state#10, s_county#9, g_state#33, g_county#34, lochierarchy#35]
+Keys [6]: [total_sum#32, s_state#10, s_county#9, g_state#33, g_county#34, lochierarchy#35]
+Functions: []
+Aggregate Attributes: []
+Results [6]: [total_sum#32, s_state#10, s_county#9, g_state#33, g_county#34, lochierarchy#35]
+
+(54) Exchange
+Input [6]: [total_sum#32, s_state#10, s_county#9, g_state#33, g_county#34, lochierarchy#35]
+Arguments: hashpartitioning(total_sum#32, s_state#10, s_county#9, g_state#33, g_county#34, lochierarchy#35, 5), ENSURE_REQUIREMENTS, [id=#70]
+
+(55) HashAggregate [codegen id : 34]
+Input [6]: [total_sum#32, s_state#10, s_county#9, g_state#33, g_county#34, lochierarchy#35]
+Keys [6]: [total_sum#32, s_state#10, s_county#9, g_state#33, g_county#34, lochierarchy#35]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [total_sum#32, s_state#10, s_county#9, lochierarchy#35, CASE WHEN (g_county#34 = 0) THEN s_state#10 END AS _w0#71]
+
+(56) Exchange
+Input [5]: [total_sum#32, s_state#10, s_county#9, lochierarchy#35, _w0#71]
+Arguments: hashpartitioning(lochierarchy#35, _w0#71, 5), ENSURE_REQUIREMENTS, [id=#72]
+
+(57) Sort [codegen id : 35]
+Input [5]: [total_sum#32, s_state#10, s_county#9, lochierarchy#35, _w0#71]
+Arguments: [lochierarchy#35 ASC NULLS FIRST, _w0#71 ASC NULLS FIRST, total_sum#32 DESC NULLS LAST], false, 0
+
+(58) Window
+Input [5]: [total_sum#32, s_state#10, s_county#9, lochierarchy#35, _w0#71]
+Arguments: [rank(total_sum#32) windowspecdefinition(lochierarchy#35, _w0#71, total_sum#32 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#73], [lochierarchy#35, _w0#71], [total_sum#32 DESC NULLS LAST]
+
+(59) Project [codegen id : 36]
+Output [5]: [total_sum#32, s_state#10, s_county#9, lochierarchy#35, rank_within_parent#73]
+Input [6]: [total_sum#32, s_state#10, s_county#9, lochierarchy#35, _w0#71, rank_within_parent#73]
+
+(60) TakeOrderedAndProject
+Input [5]: [total_sum#32, s_state#10, s_county#9, lochierarchy#35, rank_within_parent#73]
+Arguments: 100, [lochierarchy#35 DESC NULLS LAST, CASE WHEN (lochierarchy#35 = 0) THEN s_state#10 END ASC NULLS FIRST, rank_within_parent#73 ASC NULLS FIRST], [total_sum#32, s_state#10, s_county#9, lochierarchy#35, rank_within_parent#73]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (61)
+
+
+(61) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#5]
+
+Subquery:2 Hosting operator id = 14 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#4
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.sf100/simplified.txt
new file mode 100644
index 0000000000000..c1b7cfef5638c
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.sf100/simplified.txt
@@ -0,0 +1,102 @@
+TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_county]
+  WholeStageCodegen (36)
+    Project [total_sum,s_state,s_county,lochierarchy,rank_within_parent]
+      InputAdapter
+        Window [total_sum,lochierarchy,_w0]
+          WholeStageCodegen (35)
+            Sort [lochierarchy,_w0,total_sum]
+              InputAdapter
+                Exchange [lochierarchy,_w0] #1
+                  WholeStageCodegen (34)
+                    HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] [_w0]
+                      InputAdapter
+                        Exchange [total_sum,s_state,s_county,g_state,g_county,lochierarchy] #2
+                          WholeStageCodegen (33)
+                            HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy]
+                              InputAdapter
+                                Union
+                                  WholeStageCodegen (10)
+                                    HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,g_state,g_county,lochierarchy,sum]
+                                      InputAdapter
+                                        Exchange [s_state,s_county] #3
+                                          WholeStageCodegen (9)
+                                            HashAggregate [s_state,s_county,ss_net_profit] [sum,sum]
+                                              Project [ss_net_profit,s_county,s_state]
+                                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                  Project [ss_store_sk,ss_net_profit]
+                                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                      Filter [ss_store_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk]
+                                                              SubqueryBroadcast [d_date_sk] #1
+                                                                ReusedExchange [d_date_sk] #4
+                                                      InputAdapter
+                                                        BroadcastExchange #4
+                                                          WholeStageCodegen (1)
+                                                            Project [d_date_sk]
+                                                              Filter [d_month_seq,d_date_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.date_dim [d_date_sk,d_month_seq]
+                                                  InputAdapter
+                                                    BroadcastExchange #5
+                                                      WholeStageCodegen (8)
+                                                        BroadcastHashJoin [s_state,s_state]
+                                                          Filter [s_store_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.store [s_store_sk,s_county,s_state]
+                                                          InputAdapter
+                                                            BroadcastExchange #6
+                                                              WholeStageCodegen (7)
+                                                                Project [s_state]
+                                                                  Filter [ranking]
+                                                                    InputAdapter
+                                                                      Window [_w2,s_state]
+                                                                        WholeStageCodegen (6)
+                                                                          Sort [s_state,_w2]
+                                                                            InputAdapter
+                                                                              Exchange [s_state] #7
+                                                                                WholeStageCodegen (5)
+                                                                                  HashAggregate [s_state,sum] [sum(UnscaledValue(ss_net_profit)),s_state,_w2,sum]
+                                                                                    InputAdapter
+                                                                                      Exchange [s_state] #8
+                                                                                        WholeStageCodegen (4)
+                                                                                          HashAggregate [s_state,ss_net_profit] [sum,sum]
+                                                                                            Project [ss_net_profit,s_state]
+                                                                                              BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                                                                Project [ss_store_sk,ss_net_profit]
+                                                                                                  BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                                    Filter [ss_store_sk]
+                                                                                                      ColumnarToRow
+                                                                                                        InputAdapter
+                                                                                                          Scan parquet default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk]
+                                                                                                            ReusedSubquery [d_date_sk] #1
+                                                                                                    InputAdapter
+                                                                                                      ReusedExchange [d_date_sk] #4
+                                                                                                InputAdapter
+                                                                                                  BroadcastExchange #9
+                                                                                                    WholeStageCodegen (3)
+                                                                                                      Filter [s_store_sk]
+                                                                                                        ColumnarToRow
+                                                                                                          InputAdapter
+                                                                                                            Scan parquet default.store [s_store_sk,s_state]
+                                  WholeStageCodegen (21)
+                                    HashAggregate [s_state,sum,isEmpty] [sum(total_sum),total_sum,s_county,g_state,g_county,lochierarchy,sum,isEmpty]
+                                      InputAdapter
+                                        Exchange [s_state] #10
+                                          WholeStageCodegen (20)
+                                            HashAggregate [s_state,total_sum] [sum,isEmpty,sum,isEmpty]
+                                              HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum]
+                                                InputAdapter
+                                                  ReusedExchange [s_state,s_county,sum] #11
+                                  WholeStageCodegen (32)
+                                    HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum,isEmpty]
+                                      InputAdapter
+                                        Exchange #12
+                                          WholeStageCodegen (31)
+                                            HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty]
+                                              HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum]
+                                                InputAdapter
+                                                  ReusedExchange [s_state,s_county,sum] #11
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/explain.txt
new file mode 100644
index 0000000000000..df0ee68806062
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/explain.txt
@@ -0,0 +1,363 @@
+== Physical Plan ==
+TakeOrderedAndProject (60)
++- * Project (59)
+   +- Window (58)
+      +- * Sort (57)
+         +- Exchange (56)
+            +- * HashAggregate (55)
+               +- Exchange (54)
+                  +- * HashAggregate (53)
+                     +- Union (52)
+                        :- * HashAggregate (41)
+                        :  +- Exchange (40)
+                        :     +- * HashAggregate (39)
+                        :        +- * Project (38)
+                        :           +- * BroadcastHashJoin Inner BuildRight (37)
+                        :              :- * Project (10)
+                        :              :  +- * BroadcastHashJoin Inner BuildRight (9)
+                        :              :     :- * Filter (3)
+                        :              :     :  +- * ColumnarToRow (2)
+                        :              :     :     +- Scan parquet default.store_sales (1)
+                        :              :     +- BroadcastExchange (8)
+                        :              :        +- * Project (7)
+                        :              :           +- * Filter (6)
+                        :              :              +- * ColumnarToRow (5)
+                        :              :                 +- Scan parquet default.date_dim (4)
+                        :              +- BroadcastExchange (36)
+                        :                 +- * BroadcastHashJoin LeftSemi BuildRight (35)
+                        :                    :- * Filter (13)
+                        :                    :  +- * ColumnarToRow (12)
+                        :                    :     +- Scan parquet default.store (11)
+                        :                    +- BroadcastExchange (34)
+                        :                       +- * Project (33)
+                        :                          +- * Filter (32)
+                        :                             +- Window (31)
+                        :                                +- * Sort (30)
+                        :                                   +- Exchange (29)
+                        :                                      +- * HashAggregate (28)
+                        :                                         +- Exchange (27)
+                        :                                            +- * HashAggregate (26)
+                        :                                               +- * Project (25)
+                        :                                                  +- * BroadcastHashJoin Inner BuildRight (24)
+                        :                                                     :- * Project (22)
+                        :                                                     :  +- * BroadcastHashJoin Inner BuildRight (21)
+                        :                                                     :     :- * Filter (16)
+                        :                                                     :     :  +- * ColumnarToRow (15)
+                        :                                                     :     :     +- Scan parquet default.store_sales (14)
+                        :                                                     :     +- BroadcastExchange (20)
+                        :                                                     :        +- * Filter (19)
+                        :                                                     :           +- * ColumnarToRow (18)
+                        :                                                     :              +- Scan parquet default.store (17)
+                        :                                                     +- ReusedExchange (23)
+                        :- * HashAggregate (46)
+                        :  +- Exchange (45)
+                        :     +- * HashAggregate (44)
+                        :        +- * HashAggregate (43)
+                        :           +- ReusedExchange (42)
+                        +- * HashAggregate (51)
+                           +- Exchange (50)
+                              +- * HashAggregate (49)
+                                 +- * HashAggregate (48)
+                                    +- ReusedExchange (47)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 9]
+Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 9]
+Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_store_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#5, d_month_seq#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#5]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#5]
+Join condition: None
+
+(10) Project [codegen id : 9]
+Output [2]: [ss_store_sk#1, ss_net_profit#2]
+Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#5]
+
+(11) Scan parquet default.store
+Output [3]: [s_store_sk#8, s_county#9, s_state#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_county:string,s_state:string>
+
+(12) ColumnarToRow [codegen id : 8]
+Input [3]: [s_store_sk#8, s_county#9, s_state#10]
+
+(13) Filter [codegen id : 8]
+Input [3]: [s_store_sk#8, s_county#9, s_state#10]
+Condition : isnotnull(s_store_sk#8)
+
+(14) Scan parquet default.store_sales
+Output [3]: [ss_store_sk#11, ss_net_profit#12, ss_sold_date_sk#13]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#13), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_net_profit:decimal(7,2)>
+
+(15) ColumnarToRow [codegen id : 4]
+Input [3]: [ss_store_sk#11, ss_net_profit#12, ss_sold_date_sk#13]
+
+(16) Filter [codegen id : 4]
+Input [3]: [ss_store_sk#11, ss_net_profit#12, ss_sold_date_sk#13]
+Condition : isnotnull(ss_store_sk#11)
+
+(17) Scan parquet default.store
+Output [2]: [s_store_sk#14, s_state#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_state:string>
+
+(18) ColumnarToRow [codegen id : 2]
+Input [2]: [s_store_sk#14, s_state#15]
+
+(19) Filter [codegen id : 2]
+Input [2]: [s_store_sk#14, s_state#15]
+Condition : isnotnull(s_store_sk#14)
+
+(20) BroadcastExchange
+Input [2]: [s_store_sk#14, s_state#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#16]
+
+(21) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_store_sk#11]
+Right keys [1]: [s_store_sk#14]
+Join condition: None
+
+(22) Project [codegen id : 4]
+Output [3]: [ss_net_profit#12, ss_sold_date_sk#13, s_state#15]
+Input [5]: [ss_store_sk#11, ss_net_profit#12, ss_sold_date_sk#13, s_store_sk#14, s_state#15]
+
+(23) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#17]
+
+(24) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [ss_sold_date_sk#13]
+Right keys [1]: [d_date_sk#17]
+Join condition: None
+
+(25) Project [codegen id : 4]
+Output [2]: [ss_net_profit#12, s_state#15]
+Input [4]: [ss_net_profit#12, ss_sold_date_sk#13, s_state#15, d_date_sk#17]
+
+(26) HashAggregate [codegen id : 4]
+Input [2]: [ss_net_profit#12, s_state#15]
+Keys [1]: [s_state#15]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#12))]
+Aggregate Attributes [1]: [sum#18]
+Results [2]: [s_state#15, sum#19]
+
+(27) Exchange
+Input [2]: [s_state#15, sum#19]
+Arguments: hashpartitioning(s_state#15, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(28) HashAggregate [codegen id : 5]
+Input [2]: [s_state#15, sum#19]
+Keys [1]: [s_state#15]
+Functions [1]: [sum(UnscaledValue(ss_net_profit#12))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#12))#21]
+Results [3]: [s_state#15 AS s_state#22, s_state#15, MakeDecimal(sum(UnscaledValue(ss_net_profit#12))#21,17,2) AS _w2#23]
+
+(29) Exchange
+Input [3]: [s_state#22, s_state#15, _w2#23]
+Arguments: hashpartitioning(s_state#15, 5), ENSURE_REQUIREMENTS, [id=#24]
+
+(30) Sort [codegen id : 6]
+Input [3]: [s_state#22, s_state#15, _w2#23]
+Arguments: [s_state#15 ASC NULLS FIRST, _w2#23 DESC NULLS LAST], false, 0
+
+(31) Window
+Input [3]: [s_state#22, s_state#15, _w2#23]
+Arguments: [rank(_w2#23) windowspecdefinition(s_state#15, _w2#23 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#25], [s_state#15], [_w2#23 DESC NULLS LAST]
+
+(32) Filter [codegen id : 7]
+Input [4]: [s_state#22, s_state#15, _w2#23, ranking#25]
+Condition : (ranking#25 <= 5)
+
+(33) Project [codegen id : 7]
+Output [1]: [s_state#22]
+Input [4]: [s_state#22, s_state#15, _w2#23, ranking#25]
+
+(34) BroadcastExchange
+Input [1]: [s_state#22]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#26]
+
+(35) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [s_state#10]
+Right keys [1]: [s_state#22]
+Join condition: None
+
+(36) BroadcastExchange
+Input [3]: [s_store_sk#8, s_county#9, s_state#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#27]
+
+(37) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_store_sk#1]
+Right keys [1]: [s_store_sk#8]
+Join condition: None
+
+(38) Project [codegen id : 9]
+Output [3]: [ss_net_profit#2, s_county#9, s_state#10]
+Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#8, s_county#9, s_state#10]
+
+(39) HashAggregate [codegen id : 9]
+Input [3]: [ss_net_profit#2, s_county#9, s_state#10]
+Keys [2]: [s_state#10, s_county#9]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))]
+Aggregate Attributes [1]: [sum#28]
+Results [3]: [s_state#10, s_county#9, sum#29]
+
+(40) Exchange
+Input [3]: [s_state#10, s_county#9, sum#29]
+Arguments: hashpartitioning(s_state#10, s_county#9, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(41) HashAggregate [codegen id : 10]
+Input [3]: [s_state#10, s_county#9, sum#29]
+Keys [2]: [s_state#10, s_county#9]
+Functions [1]: [sum(UnscaledValue(ss_net_profit#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#31]
+Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#31,17,2) as decimal(27,2)) AS total_sum#32, s_state#10, s_county#9, 0 AS g_state#33, 0 AS g_county#34, 0 AS lochierarchy#35]
+
+(42) ReusedExchange [Reuses operator id: unknown]
+Output [3]: [s_state#36, s_county#37, sum#38]
+
+(43) HashAggregate [codegen id : 20]
+Input [3]: [s_state#36, s_county#37, sum#38]
+Keys [2]: [s_state#36, s_county#37]
+Functions [1]: [sum(UnscaledValue(ss_net_profit#39))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#39))#40]
+Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#39))#40,17,2) AS total_sum#41, s_state#36]
+
+(44) HashAggregate [codegen id : 20]
+Input [2]: [total_sum#41, s_state#36]
+Keys [1]: [s_state#36]
+Functions [1]: [partial_sum(total_sum#41)]
+Aggregate Attributes [2]: [sum#42, isEmpty#43]
+Results [3]: [s_state#36, sum#44, isEmpty#45]
+
+(45) Exchange
+Input [3]: [s_state#36, sum#44, isEmpty#45]
+Arguments: hashpartitioning(s_state#36, 5), ENSURE_REQUIREMENTS, [id=#46]
+
+(46) HashAggregate [codegen id : 21]
+Input [3]: [s_state#36, sum#44, isEmpty#45]
+Keys [1]: [s_state#36]
+Functions [1]: [sum(total_sum#41)]
+Aggregate Attributes [1]: [sum(total_sum#41)#47]
+Results [6]: [sum(total_sum#41)#47 AS total_sum#48, s_state#36, null AS s_county#49, 0 AS g_state#50, 1 AS g_county#51, 1 AS lochierarchy#52]
+
+(47) ReusedExchange [Reuses operator id: unknown]
+Output [3]: [s_state#53, s_county#54, sum#55]
+
+(48) HashAggregate [codegen id : 31]
+Input [3]: [s_state#53, s_county#54, sum#55]
+Keys [2]: [s_state#53, s_county#54]
+Functions [1]: [sum(UnscaledValue(ss_net_profit#56))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#56))#57]
+Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#56))#57,17,2) AS total_sum#41]
+
+(49) HashAggregate [codegen id : 31]
+Input [1]: [total_sum#41]
+Keys: []
+Functions [1]: [partial_sum(total_sum#41)]
+Aggregate Attributes [2]: [sum#58, isEmpty#59]
+Results [2]: [sum#60, isEmpty#61]
+
+(50) Exchange
+Input [2]: [sum#60, isEmpty#61]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#62]
+
+(51) HashAggregate [codegen id : 32]
+Input [2]: [sum#60, isEmpty#61]
+Keys: []
+Functions [1]: [sum(total_sum#41)]
+Aggregate Attributes [1]: [sum(total_sum#41)#63]
+Results [6]: [sum(total_sum#41)#63 AS total_sum#64, null AS s_state#65, null AS s_county#66, 1 AS g_state#67, 1 AS g_county#68, 2 AS lochierarchy#69]
+
+(52) Union
+
+(53) HashAggregate [codegen id : 33]
+Input [6]: [total_sum#32, s_state#10, s_county#9, g_state#33, g_county#34, lochierarchy#35]
+Keys [6]: [total_sum#32, s_state#10, s_county#9, g_state#33, g_county#34, lochierarchy#35]
+Functions: []
+Aggregate Attributes: []
+Results [6]: [total_sum#32, s_state#10, s_county#9, g_state#33, g_county#34, lochierarchy#35]
+
+(54) Exchange
+Input [6]: [total_sum#32, s_state#10, s_county#9, g_state#33, g_county#34, lochierarchy#35]
+Arguments: hashpartitioning(total_sum#32, s_state#10, s_county#9, g_state#33, g_county#34, lochierarchy#35, 5), ENSURE_REQUIREMENTS, [id=#70]
+
+(55) HashAggregate [codegen id : 34]
+Input [6]: [total_sum#32, s_state#10, s_county#9, g_state#33, g_county#34, lochierarchy#35]
+Keys [6]: [total_sum#32, s_state#10, s_county#9, g_state#33, g_county#34, lochierarchy#35]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [total_sum#32, s_state#10, s_county#9, lochierarchy#35, CASE WHEN (g_county#34 = 0) THEN s_state#10 END AS _w0#71]
+
+(56) Exchange
+Input [5]: [total_sum#32, s_state#10, s_county#9, lochierarchy#35, _w0#71]
+Arguments: hashpartitioning(lochierarchy#35, _w0#71, 5), ENSURE_REQUIREMENTS, [id=#72]
+
+(57) Sort [codegen id : 35]
+Input [5]: [total_sum#32, s_state#10, s_county#9, lochierarchy#35, _w0#71]
+Arguments: [lochierarchy#35 ASC NULLS FIRST, _w0#71 ASC NULLS FIRST, total_sum#32 DESC NULLS LAST], false, 0
+
+(58) Window
+Input [5]: [total_sum#32, s_state#10, s_county#9, lochierarchy#35, _w0#71]
+Arguments: [rank(total_sum#32) windowspecdefinition(lochierarchy#35, _w0#71, total_sum#32 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#73], [lochierarchy#35, _w0#71], [total_sum#32 DESC NULLS LAST]
+
+(59) Project [codegen id : 36]
+Output [5]: [total_sum#32, s_state#10, s_county#9, lochierarchy#35, rank_within_parent#73]
+Input [6]: [total_sum#32, s_state#10, s_county#9, lochierarchy#35, _w0#71, rank_within_parent#73]
+
+(60) TakeOrderedAndProject
+Input [5]: [total_sum#32, s_state#10, s_county#9, lochierarchy#35, rank_within_parent#73]
+Arguments: 100, [lochierarchy#35 DESC NULLS LAST, CASE WHEN (lochierarchy#35 = 0) THEN s_state#10 END ASC NULLS FIRST, rank_within_parent#73 ASC NULLS FIRST], [total_sum#32, s_state#10, s_county#9, lochierarchy#35, rank_within_parent#73]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (61)
+
+
+(61) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#5]
+
+Subquery:2 Hosting operator id = 14 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#4
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/simplified.txt
new file mode 100644
index 0000000000000..cd1e7e69e65d0
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/simplified.txt
@@ -0,0 +1,102 @@
+TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_county]
+  WholeStageCodegen (36)
+    Project [total_sum,s_state,s_county,lochierarchy,rank_within_parent]
+      InputAdapter
+        Window [total_sum,lochierarchy,_w0]
+          WholeStageCodegen (35)
+            Sort [lochierarchy,_w0,total_sum]
+              InputAdapter
+                Exchange [lochierarchy,_w0] #1
+                  WholeStageCodegen (34)
+                    HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] [_w0]
+                      InputAdapter
+                        Exchange [total_sum,s_state,s_county,g_state,g_county,lochierarchy] #2
+                          WholeStageCodegen (33)
+                            HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy]
+                              InputAdapter
+                                Union
+                                  WholeStageCodegen (10)
+                                    HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,g_state,g_county,lochierarchy,sum]
+                                      InputAdapter
+                                        Exchange [s_state,s_county] #3
+                                          WholeStageCodegen (9)
+                                            HashAggregate [s_state,s_county,ss_net_profit] [sum,sum]
+                                              Project [ss_net_profit,s_county,s_state]
+                                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                  Project [ss_store_sk,ss_net_profit]
+                                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                      Filter [ss_store_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk]
+                                                              SubqueryBroadcast [d_date_sk] #1
+                                                                ReusedExchange [d_date_sk] #4
+                                                      InputAdapter
+                                                        BroadcastExchange #4
+                                                          WholeStageCodegen (1)
+                                                            Project [d_date_sk]
+                                                              Filter [d_month_seq,d_date_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.date_dim [d_date_sk,d_month_seq]
+                                                  InputAdapter
+                                                    BroadcastExchange #5
+                                                      WholeStageCodegen (8)
+                                                        BroadcastHashJoin [s_state,s_state]
+                                                          Filter [s_store_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.store [s_store_sk,s_county,s_state]
+                                                          InputAdapter
+                                                            BroadcastExchange #6
+                                                              WholeStageCodegen (7)
+                                                                Project [s_state]
+                                                                  Filter [ranking]
+                                                                    InputAdapter
+                                                                      Window [_w2,s_state]
+                                                                        WholeStageCodegen (6)
+                                                                          Sort [s_state,_w2]
+                                                                            InputAdapter
+                                                                              Exchange [s_state] #7
+                                                                                WholeStageCodegen (5)
+                                                                                  HashAggregate [s_state,sum] [sum(UnscaledValue(ss_net_profit)),s_state,_w2,sum]
+                                                                                    InputAdapter
+                                                                                      Exchange [s_state] #8
+                                                                                        WholeStageCodegen (4)
+                                                                                          HashAggregate [s_state,ss_net_profit] [sum,sum]
+                                                                                            Project [ss_net_profit,s_state]
+                                                                                              BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                                                Project [ss_net_profit,ss_sold_date_sk,s_state]
+                                                                                                  BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                                                                    Filter [ss_store_sk]
+                                                                                                      ColumnarToRow
+                                                                                                        InputAdapter
+                                                                                                          Scan parquet default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk]
+                                                                                                            ReusedSubquery [d_date_sk] #1
+                                                                                                    InputAdapter
+                                                                                                      BroadcastExchange #9
+                                                                                                        WholeStageCodegen (2)
+                                                                                                          Filter [s_store_sk]
+                                                                                                            ColumnarToRow
+                                                                                                              InputAdapter
+                                                                                                                Scan parquet default.store [s_store_sk,s_state]
+                                                                                                InputAdapter
+                                                                                                  ReusedExchange [d_date_sk] #4
+                                  WholeStageCodegen (21)
+                                    HashAggregate [s_state,sum,isEmpty] [sum(total_sum),total_sum,s_county,g_state,g_county,lochierarchy,sum,isEmpty]
+                                      InputAdapter
+                                        Exchange [s_state] #10
+                                          WholeStageCodegen (20)
+                                            HashAggregate [s_state,total_sum] [sum,isEmpty,sum,isEmpty]
+                                              HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum]
+                                                InputAdapter
+                                                  ReusedExchange [s_state,s_county,sum] #11
+                                  WholeStageCodegen (32)
+                                    HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum,isEmpty]
+                                      InputAdapter
+                                        Exchange #12
+                                          WholeStageCodegen (31)
+                                            HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty]
+                                              HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum]
+                                                InputAdapter
+                                                  ReusedExchange [s_state,s_county,sum] #11
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.sf100/explain.txt
new file mode 100644
index 0000000000000..c22dec073af41
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.sf100/explain.txt
@@ -0,0 +1,453 @@
+== Physical Plan ==
+TakeOrderedAndProject (80)
++- * HashAggregate (79)
+   +- Exchange (78)
+      +- * HashAggregate (77)
+         +- * Project (76)
+            +- * SortMergeJoin LeftOuter (75)
+               :- * Sort (68)
+               :  +- Exchange (67)
+               :     +- * Project (66)
+               :        +- * BroadcastHashJoin LeftOuter BuildRight (65)
+               :           :- * Project (60)
+               :           :  +- * SortMergeJoin Inner (59)
+               :           :     :- * Sort (47)
+               :           :     :  +- Exchange (46)
+               :           :     :     +- * Project (45)
+               :           :     :        +- * BroadcastHashJoin Inner BuildRight (44)
+               :           :     :           :- * Project (32)
+               :           :     :           :  +- * SortMergeJoin Inner (31)
+               :           :     :           :     :- * Sort (25)
+               :           :     :           :     :  +- Exchange (24)
+               :           :     :           :     :     +- * Project (23)
+               :           :     :           :     :        +- * BroadcastHashJoin Inner BuildRight (22)
+               :           :     :           :     :           :- * Project (17)
+               :           :     :           :     :           :  +- * BroadcastHashJoin Inner BuildRight (16)
+               :           :     :           :     :           :     :- * Project (10)
+               :           :     :           :     :           :     :  +- * BroadcastHashJoin Inner BuildRight (9)
+               :           :     :           :     :           :     :     :- * Filter (3)
+               :           :     :           :     :           :     :     :  +- * ColumnarToRow (2)
+               :           :     :           :     :           :     :     :     +- Scan parquet default.catalog_sales (1)
+               :           :     :           :     :           :     :     +- BroadcastExchange (8)
+               :           :     :           :     :           :     :        +- * Project (7)
+               :           :     :           :     :           :     :           +- * Filter (6)
+               :           :     :           :     :           :     :              +- * ColumnarToRow (5)
+               :           :     :           :     :           :     :                 +- Scan parquet default.household_demographics (4)
+               :           :     :           :     :           :     +- BroadcastExchange (15)
+               :           :     :           :     :           :        +- * Project (14)
+               :           :     :           :     :           :           +- * Filter (13)
+               :           :     :           :     :           :              +- * ColumnarToRow (12)
+               :           :     :           :     :           :                 +- Scan parquet default.customer_demographics (11)
+               :           :     :           :     :           +- BroadcastExchange (21)
+               :           :     :           :     :              +- * Filter (20)
+               :           :     :           :     :                 +- * ColumnarToRow (19)
+               :           :     :           :     :                    +- Scan parquet default.date_dim (18)
+               :           :     :           :     +- * Sort (30)
+               :           :     :           :        +- Exchange (29)
+               :           :     :           :           +- * Filter (28)
+               :           :     :           :              +- * ColumnarToRow (27)
+               :           :     :           :                 +- Scan parquet default.item (26)
+               :           :     :           +- BroadcastExchange (43)
+               :           :     :              +- * Project (42)
+               :           :     :                 +- * BroadcastHashJoin Inner BuildLeft (41)
+               :           :     :                    :- BroadcastExchange (37)
+               :           :     :                    :  +- * Project (36)
+               :           :     :                    :     +- * Filter (35)
+               :           :     :                    :        +- * ColumnarToRow (34)
+               :           :     :                    :           +- Scan parquet default.date_dim (33)
+               :           :     :                    +- * Filter (40)
+               :           :     :                       +- * ColumnarToRow (39)
+               :           :     :                          +- Scan parquet default.date_dim (38)
+               :           :     +- * Sort (58)
+               :           :        +- Exchange (57)
+               :           :           +- * Project (56)
+               :           :              +- * BroadcastHashJoin Inner BuildRight (55)
+               :           :                 :- * Filter (50)
+               :           :                 :  +- * ColumnarToRow (49)
+               :           :                 :     +- Scan parquet default.inventory (48)
+               :           :                 +- BroadcastExchange (54)
+               :           :                    +- * Filter (53)
+               :           :                       +- * ColumnarToRow (52)
+               :           :                          +- Scan parquet default.warehouse (51)
+               :           +- BroadcastExchange (64)
+               :              +- * Filter (63)
+               :                 +- * ColumnarToRow (62)
+               :                    +- Scan parquet default.promotion (61)
+               +- * Sort (74)
+                  +- Exchange (73)
+                     +- * Project (72)
+                        +- * Filter (71)
+                           +- * ColumnarToRow (70)
+                              +- Scan parquet default.catalog_returns (69)
+
+
+(1) Scan parquet default.catalog_sales
+Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#8), dynamicpruningexpression(cs_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(cs_quantity), IsNotNull(cs_item_sk), IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_hdemo_sk), IsNotNull(cs_ship_date_sk)]
+ReadSchema: struct<cs_ship_date_sk:int,cs_bill_cdemo_sk:int,cs_bill_hdemo_sk:int,cs_item_sk:int,cs_promo_sk:int,cs_order_number:int,cs_quantity:int>
+
+(2) ColumnarToRow [codegen id : 4]
+Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8]
+
+(3) Filter [codegen id : 4]
+Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8]
+Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1))
+
+(4) Scan parquet default.household_demographics
+Output [2]: [hd_demo_sk#10, hd_buy_potential#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_buy_potential), EqualTo(hd_buy_potential,1001-5000      ), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_buy_potential:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [hd_demo_sk#10, hd_buy_potential#11]
+
+(6) Filter [codegen id : 1]
+Input [2]: [hd_demo_sk#10, hd_buy_potential#11]
+Condition : ((isnotnull(hd_buy_potential#11) AND (hd_buy_potential#11 = 1001-5000      )) AND isnotnull(hd_demo_sk#10))
+
+(7) Project [codegen id : 1]
+Output [1]: [hd_demo_sk#10]
+Input [2]: [hd_demo_sk#10, hd_buy_potential#11]
+
+(8) BroadcastExchange
+Input [1]: [hd_demo_sk#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#12]
+
+(9) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_bill_hdemo_sk#3]
+Right keys [1]: [hd_demo_sk#10]
+Join condition: None
+
+(10) Project [codegen id : 4]
+Output [7]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8]
+Input [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, hd_demo_sk#10]
+
+(11) Scan parquet default.customer_demographics
+Output [2]: [cd_demo_sk#13, cd_marital_status#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_marital_status), EqualTo(cd_marital_status,M), IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_marital_status:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [cd_demo_sk#13, cd_marital_status#14]
+
+(13) Filter [codegen id : 2]
+Input [2]: [cd_demo_sk#13, cd_marital_status#14]
+Condition : ((isnotnull(cd_marital_status#14) AND (cd_marital_status#14 = M)) AND isnotnull(cd_demo_sk#13))
+
+(14) Project [codegen id : 2]
+Output [1]: [cd_demo_sk#13]
+Input [2]: [cd_demo_sk#13, cd_marital_status#14]
+
+(15) BroadcastExchange
+Input [1]: [cd_demo_sk#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#15]
+
+(16) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_bill_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#13]
+Join condition: None
+
+(17) Project [codegen id : 4]
+Output [6]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8]
+Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, cd_demo_sk#13]
+
+(18) Scan parquet default.date_dim
+Output [2]: [d_date_sk#16, d_date#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(19) ColumnarToRow [codegen id : 3]
+Input [2]: [d_date_sk#16, d_date#17]
+
+(20) Filter [codegen id : 3]
+Input [2]: [d_date_sk#16, d_date#17]
+Condition : (isnotnull(d_date#17) AND isnotnull(d_date_sk#16))
+
+(21) BroadcastExchange
+Input [2]: [d_date_sk#16, d_date#17]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#18]
+
+(22) BroadcastHashJoin [codegen id : 4]
+Left keys [1]: [cs_ship_date_sk#1]
+Right keys [1]: [d_date_sk#16]
+Join condition: None
+
+(23) Project [codegen id : 4]
+Output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, d_date#17]
+Input [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, d_date_sk#16, d_date#17]
+
+(24) Exchange
+Input [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, d_date#17]
+Arguments: hashpartitioning(cs_item_sk#4, 5), ENSURE_REQUIREMENTS, [id=#19]
+
+(25) Sort [codegen id : 5]
+Input [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, d_date#17]
+Arguments: [cs_item_sk#4 ASC NULLS FIRST], false, 0
+
+(26) Scan parquet default.item
+Output [2]: [i_item_sk#20, i_item_desc#21]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_desc:string>
+
+(27) ColumnarToRow [codegen id : 6]
+Input [2]: [i_item_sk#20, i_item_desc#21]
+
+(28) Filter [codegen id : 6]
+Input [2]: [i_item_sk#20, i_item_desc#21]
+Condition : isnotnull(i_item_sk#20)
+
+(29) Exchange
+Input [2]: [i_item_sk#20, i_item_desc#21]
+Arguments: hashpartitioning(i_item_sk#20, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(30) Sort [codegen id : 7]
+Input [2]: [i_item_sk#20, i_item_desc#21]
+Arguments: [i_item_sk#20 ASC NULLS FIRST], false, 0
+
+(31) SortMergeJoin [codegen id : 10]
+Left keys [1]: [cs_item_sk#4]
+Right keys [1]: [i_item_sk#20]
+Join condition: None
+
+(32) Project [codegen id : 10]
+Output [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, d_date#17, i_item_desc#21]
+Input [8]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, d_date#17, i_item_sk#20, i_item_desc#21]
+
+(33) Scan parquet default.date_dim
+Output [4]: [d_date_sk#23, d_date#24, d_week_seq#25, d_year#26]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)]
+ReadSchema: struct<d_date_sk:int,d_date:date,d_week_seq:int,d_year:int>
+
+(34) ColumnarToRow [codegen id : 8]
+Input [4]: [d_date_sk#23, d_date#24, d_week_seq#25, d_year#26]
+
+(35) Filter [codegen id : 8]
+Input [4]: [d_date_sk#23, d_date#24, d_week_seq#25, d_year#26]
+Condition : ((((isnotnull(d_year#26) AND (d_year#26 = 2001)) AND isnotnull(d_date_sk#23)) AND isnotnull(d_week_seq#25)) AND isnotnull(d_date#24))
+
+(36) Project [codegen id : 8]
+Output [3]: [d_date_sk#23, d_date#24, d_week_seq#25]
+Input [4]: [d_date_sk#23, d_date#24, d_week_seq#25, d_year#26]
+
+(37) BroadcastExchange
+Input [3]: [d_date_sk#23, d_date#24, d_week_seq#25]
+Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [id=#27]
+
+(38) Scan parquet default.date_dim
+Output [2]: [d_date_sk#28, d_week_seq#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_week_seq:int>
+
+(39) ColumnarToRow
+Input [2]: [d_date_sk#28, d_week_seq#29]
+
+(40) Filter
+Input [2]: [d_date_sk#28, d_week_seq#29]
+Condition : (isnotnull(d_week_seq#29) AND isnotnull(d_date_sk#28))
+
+(41) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [d_week_seq#25]
+Right keys [1]: [d_week_seq#29]
+Join condition: None
+
+(42) Project [codegen id : 9]
+Output [4]: [d_date_sk#23, d_date#24, d_week_seq#25, d_date_sk#28]
+Input [5]: [d_date_sk#23, d_date#24, d_week_seq#25, d_date_sk#28, d_week_seq#29]
+
+(43) BroadcastExchange
+Input [4]: [d_date_sk#23, d_date#24, d_week_seq#25, d_date_sk#28]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#30]
+
+(44) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_sold_date_sk#8]
+Right keys [1]: [d_date_sk#23]
+Join condition: (d_date#17 > d_date#24 + 5 days)
+
+(45) Project [codegen id : 10]
+Output [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, i_item_desc#21, d_week_seq#25, d_date_sk#28]
+Input [11]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, d_date#17, i_item_desc#21, d_date_sk#23, d_date#24, d_week_seq#25, d_date_sk#28]
+
+(46) Exchange
+Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, i_item_desc#21, d_week_seq#25, d_date_sk#28]
+Arguments: hashpartitioning(cs_item_sk#4, d_date_sk#28, 5), ENSURE_REQUIREMENTS, [id=#31]
+
+(47) Sort [codegen id : 11]
+Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, i_item_desc#21, d_week_seq#25, d_date_sk#28]
+Arguments: [cs_item_sk#4 ASC NULLS FIRST, d_date_sk#28 ASC NULLS FIRST], false, 0
+
+(48) Scan parquet default.inventory
+Output [4]: [inv_item_sk#32, inv_warehouse_sk#33, inv_quantity_on_hand#34, inv_date_sk#35]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(inv_date_sk#35), dynamicpruningexpression(true)]
+PushedFilters: [IsNotNull(inv_quantity_on_hand), IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)]
+ReadSchema: struct<inv_item_sk:int,inv_warehouse_sk:int,inv_quantity_on_hand:int>
+
+(49) ColumnarToRow [codegen id : 13]
+Input [4]: [inv_item_sk#32, inv_warehouse_sk#33, inv_quantity_on_hand#34, inv_date_sk#35]
+
+(50) Filter [codegen id : 13]
+Input [4]: [inv_item_sk#32, inv_warehouse_sk#33, inv_quantity_on_hand#34, inv_date_sk#35]
+Condition : ((isnotnull(inv_quantity_on_hand#34) AND isnotnull(inv_item_sk#32)) AND isnotnull(inv_warehouse_sk#33))
+
+(51) Scan parquet default.warehouse
+Output [2]: [w_warehouse_sk#36, w_warehouse_name#37]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/warehouse]
+PushedFilters: [IsNotNull(w_warehouse_sk)]
+ReadSchema: struct<w_warehouse_sk:int,w_warehouse_name:string>
+
+(52) ColumnarToRow [codegen id : 12]
+Input [2]: [w_warehouse_sk#36, w_warehouse_name#37]
+
+(53) Filter [codegen id : 12]
+Input [2]: [w_warehouse_sk#36, w_warehouse_name#37]
+Condition : isnotnull(w_warehouse_sk#36)
+
+(54) BroadcastExchange
+Input [2]: [w_warehouse_sk#36, w_warehouse_name#37]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#38]
+
+(55) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [inv_warehouse_sk#33]
+Right keys [1]: [w_warehouse_sk#36]
+Join condition: None
+
+(56) Project [codegen id : 13]
+Output [4]: [inv_item_sk#32, inv_quantity_on_hand#34, inv_date_sk#35, w_warehouse_name#37]
+Input [6]: [inv_item_sk#32, inv_warehouse_sk#33, inv_quantity_on_hand#34, inv_date_sk#35, w_warehouse_sk#36, w_warehouse_name#37]
+
+(57) Exchange
+Input [4]: [inv_item_sk#32, inv_quantity_on_hand#34, inv_date_sk#35, w_warehouse_name#37]
+Arguments: hashpartitioning(inv_item_sk#32, inv_date_sk#35, 5), ENSURE_REQUIREMENTS, [id=#39]
+
+(58) Sort [codegen id : 14]
+Input [4]: [inv_item_sk#32, inv_quantity_on_hand#34, inv_date_sk#35, w_warehouse_name#37]
+Arguments: [inv_item_sk#32 ASC NULLS FIRST, inv_date_sk#35 ASC NULLS FIRST], false, 0
+
+(59) SortMergeJoin [codegen id : 16]
+Left keys [2]: [cs_item_sk#4, d_date_sk#28]
+Right keys [2]: [inv_item_sk#32, inv_date_sk#35]
+Join condition: (inv_quantity_on_hand#34 < cs_quantity#7)
+
+(60) Project [codegen id : 16]
+Output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#37, i_item_desc#21, d_week_seq#25]
+Input [11]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, i_item_desc#21, d_week_seq#25, d_date_sk#28, inv_item_sk#32, inv_quantity_on_hand#34, inv_date_sk#35, w_warehouse_name#37]
+
+(61) Scan parquet default.promotion
+Output [1]: [p_promo_sk#40]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/promotion]
+PushedFilters: [IsNotNull(p_promo_sk)]
+ReadSchema: struct<p_promo_sk:int>
+
+(62) ColumnarToRow [codegen id : 15]
+Input [1]: [p_promo_sk#40]
+
+(63) Filter [codegen id : 15]
+Input [1]: [p_promo_sk#40]
+Condition : isnotnull(p_promo_sk#40)
+
+(64) BroadcastExchange
+Input [1]: [p_promo_sk#40]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#41]
+
+(65) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [cs_promo_sk#5]
+Right keys [1]: [p_promo_sk#40]
+Join condition: None
+
+(66) Project [codegen id : 16]
+Output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#37, i_item_desc#21, d_week_seq#25]
+Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#37, i_item_desc#21, d_week_seq#25, p_promo_sk#40]
+
+(67) Exchange
+Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#37, i_item_desc#21, d_week_seq#25]
+Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, [id=#42]
+
+(68) Sort [codegen id : 17]
+Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#37, i_item_desc#21, d_week_seq#25]
+Arguments: [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST], false, 0
+
+(69) Scan parquet default.catalog_returns
+Output [3]: [cr_item_sk#43, cr_order_number#44, cr_returned_date_sk#45]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int>
+
+(70) ColumnarToRow [codegen id : 18]
+Input [3]: [cr_item_sk#43, cr_order_number#44, cr_returned_date_sk#45]
+
+(71) Filter [codegen id : 18]
+Input [3]: [cr_item_sk#43, cr_order_number#44, cr_returned_date_sk#45]
+Condition : (isnotnull(cr_item_sk#43) AND isnotnull(cr_order_number#44))
+
+(72) Project [codegen id : 18]
+Output [2]: [cr_item_sk#43, cr_order_number#44]
+Input [3]: [cr_item_sk#43, cr_order_number#44, cr_returned_date_sk#45]
+
+(73) Exchange
+Input [2]: [cr_item_sk#43, cr_order_number#44]
+Arguments: hashpartitioning(cr_item_sk#43, cr_order_number#44, 5), ENSURE_REQUIREMENTS, [id=#46]
+
+(74) Sort [codegen id : 19]
+Input [2]: [cr_item_sk#43, cr_order_number#44]
+Arguments: [cr_item_sk#43 ASC NULLS FIRST, cr_order_number#44 ASC NULLS FIRST], false, 0
+
+(75) SortMergeJoin [codegen id : 20]
+Left keys [2]: [cs_item_sk#4, cs_order_number#6]
+Right keys [2]: [cr_item_sk#43, cr_order_number#44]
+Join condition: None
+
+(76) Project [codegen id : 20]
+Output [3]: [w_warehouse_name#37, i_item_desc#21, d_week_seq#25]
+Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#37, i_item_desc#21, d_week_seq#25, cr_item_sk#43, cr_order_number#44]
+
+(77) HashAggregate [codegen id : 20]
+Input [3]: [w_warehouse_name#37, i_item_desc#21, d_week_seq#25]
+Keys [3]: [i_item_desc#21, w_warehouse_name#37, d_week_seq#25]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#47]
+Results [4]: [i_item_desc#21, w_warehouse_name#37, d_week_seq#25, count#48]
+
+(78) Exchange
+Input [4]: [i_item_desc#21, w_warehouse_name#37, d_week_seq#25, count#48]
+Arguments: hashpartitioning(i_item_desc#21, w_warehouse_name#37, d_week_seq#25, 5), ENSURE_REQUIREMENTS, [id=#49]
+
+(79) HashAggregate [codegen id : 21]
+Input [4]: [i_item_desc#21, w_warehouse_name#37, d_week_seq#25, count#48]
+Keys [3]: [i_item_desc#21, w_warehouse_name#37, d_week_seq#25]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#50]
+Results [6]: [i_item_desc#21, w_warehouse_name#37, d_week_seq#25, count(1)#50 AS no_promo#51, count(1)#50 AS promo#52, count(1)#50 AS total_cnt#53]
+
+(80) TakeOrderedAndProject
+Input [6]: [i_item_desc#21, w_warehouse_name#37, d_week_seq#25, no_promo#51, promo#52, total_cnt#53]
+Arguments: 100, [total_cnt#53 DESC NULLS LAST, i_item_desc#21 ASC NULLS FIRST, w_warehouse_name#37 ASC NULLS FIRST, d_week_seq#25 ASC NULLS FIRST], [i_item_desc#21, w_warehouse_name#37, d_week_seq#25, no_promo#51, promo#52, total_cnt#53]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (81)
+
+
+(81) ReusedExchange [Reuses operator id: 43]
+Output [4]: [d_date_sk#23, d_date#24, d_week_seq#25, d_date_sk#28]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.sf100/simplified.txt
new file mode 100644
index 0000000000000..da6b79ca3c79d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.sf100/simplified.txt
@@ -0,0 +1,134 @@
+TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo]
+  WholeStageCodegen (21)
+    HashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] [count(1),no_promo,promo,total_cnt,count]
+      InputAdapter
+        Exchange [i_item_desc,w_warehouse_name,d_week_seq] #1
+          WholeStageCodegen (20)
+            HashAggregate [i_item_desc,w_warehouse_name,d_week_seq] [count,count]
+              Project [w_warehouse_name,i_item_desc,d_week_seq]
+                SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number]
+                  InputAdapter
+                    WholeStageCodegen (17)
+                      Sort [cs_item_sk,cs_order_number]
+                        InputAdapter
+                          Exchange [cs_item_sk,cs_order_number] #2
+                            WholeStageCodegen (16)
+                              Project [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq]
+                                BroadcastHashJoin [cs_promo_sk,p_promo_sk]
+                                  Project [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq]
+                                    SortMergeJoin [cs_item_sk,d_date_sk,inv_item_sk,inv_date_sk,inv_quantity_on_hand,cs_quantity]
+                                      InputAdapter
+                                        WholeStageCodegen (11)
+                                          Sort [cs_item_sk,d_date_sk]
+                                            InputAdapter
+                                              Exchange [cs_item_sk,d_date_sk] #3
+                                                WholeStageCodegen (10)
+                                                  Project [cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,i_item_desc,d_week_seq,d_date_sk]
+                                                    BroadcastHashJoin [cs_sold_date_sk,d_date_sk,d_date,d_date]
+                                                      Project [cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,d_date,i_item_desc]
+                                                        SortMergeJoin [cs_item_sk,i_item_sk]
+                                                          InputAdapter
+                                                            WholeStageCodegen (5)
+                                                              Sort [cs_item_sk]
+                                                                InputAdapter
+                                                                  Exchange [cs_item_sk] #4
+                                                                    WholeStageCodegen (4)
+                                                                      Project [cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,d_date]
+                                                                        BroadcastHashJoin [cs_ship_date_sk,d_date_sk]
+                                                                          Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk]
+                                                                            BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk]
+                                                                              Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk]
+                                                                                BroadcastHashJoin [cs_bill_hdemo_sk,hd_demo_sk]
+                                                                                  Filter [cs_quantity,cs_item_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_ship_date_sk]
+                                                                                    ColumnarToRow
+                                                                                      InputAdapter
+                                                                                        Scan parquet default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk]
+                                                                                          SubqueryBroadcast [d_date_sk] #1
+                                                                                            ReusedExchange [d_date_sk,d_date,d_week_seq,d_date_sk] #5
+                                                                                  InputAdapter
+                                                                                    BroadcastExchange #6
+                                                                                      WholeStageCodegen (1)
+                                                                                        Project [hd_demo_sk]
+                                                                                          Filter [hd_buy_potential,hd_demo_sk]
+                                                                                            ColumnarToRow
+                                                                                              InputAdapter
+                                                                                                Scan parquet default.household_demographics [hd_demo_sk,hd_buy_potential]
+                                                                              InputAdapter
+                                                                                BroadcastExchange #7
+                                                                                  WholeStageCodegen (2)
+                                                                                    Project [cd_demo_sk]
+                                                                                      Filter [cd_marital_status,cd_demo_sk]
+                                                                                        ColumnarToRow
+                                                                                          InputAdapter
+                                                                                            Scan parquet default.customer_demographics [cd_demo_sk,cd_marital_status]
+                                                                          InputAdapter
+                                                                            BroadcastExchange #8
+                                                                              WholeStageCodegen (3)
+                                                                                Filter [d_date,d_date_sk]
+                                                                                  ColumnarToRow
+                                                                                    InputAdapter
+                                                                                      Scan parquet default.date_dim [d_date_sk,d_date]
+                                                          InputAdapter
+                                                            WholeStageCodegen (7)
+                                                              Sort [i_item_sk]
+                                                                InputAdapter
+                                                                  Exchange [i_item_sk] #9
+                                                                    WholeStageCodegen (6)
+                                                                      Filter [i_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.item [i_item_sk,i_item_desc]
+                                                      InputAdapter
+                                                        BroadcastExchange #5
+                                                          WholeStageCodegen (9)
+                                                            Project [d_date_sk,d_date,d_week_seq,d_date_sk]
+                                                              BroadcastHashJoin [d_week_seq,d_week_seq]
+                                                                InputAdapter
+                                                                  BroadcastExchange #10
+                                                                    WholeStageCodegen (8)
+                                                                      Project [d_date_sk,d_date,d_week_seq]
+                                                                        Filter [d_year,d_date_sk,d_week_seq,d_date]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.date_dim [d_date_sk,d_date,d_week_seq,d_year]
+                                                                Filter [d_week_seq,d_date_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.date_dim [d_date_sk,d_week_seq]
+                                      InputAdapter
+                                        WholeStageCodegen (14)
+                                          Sort [inv_item_sk,inv_date_sk]
+                                            InputAdapter
+                                              Exchange [inv_item_sk,inv_date_sk] #11
+                                                WholeStageCodegen (13)
+                                                  Project [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name]
+                                                    BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk]
+                                                      Filter [inv_quantity_on_hand,inv_item_sk,inv_warehouse_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk]
+                                                      InputAdapter
+                                                        BroadcastExchange #12
+                                                          WholeStageCodegen (12)
+                                                            Filter [w_warehouse_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.warehouse [w_warehouse_sk,w_warehouse_name]
+                                  InputAdapter
+                                    BroadcastExchange #13
+                                      WholeStageCodegen (15)
+                                        Filter [p_promo_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.promotion [p_promo_sk]
+                  InputAdapter
+                    WholeStageCodegen (19)
+                      Sort [cr_item_sk,cr_order_number]
+                        InputAdapter
+                          Exchange [cr_item_sk,cr_order_number] #14
+                            WholeStageCodegen (18)
+                              Project [cr_item_sk,cr_order_number]
+                                Filter [cr_item_sk,cr_order_number]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt
new file mode 100644
index 0000000000000..ee861169d77a4
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt
@@ -0,0 +1,423 @@
+== Physical Plan ==
+TakeOrderedAndProject (74)
++- * HashAggregate (73)
+   +- Exchange (72)
+      +- * HashAggregate (71)
+         +- * Project (70)
+            +- * SortMergeJoin LeftOuter (69)
+               :- * Sort (62)
+               :  +- Exchange (61)
+               :     +- * Project (60)
+               :        +- * BroadcastHashJoin LeftOuter BuildRight (59)
+               :           :- * Project (54)
+               :           :  +- * BroadcastHashJoin Inner BuildRight (53)
+               :           :     :- * Project (48)
+               :           :     :  +- * BroadcastHashJoin Inner BuildRight (47)
+               :           :     :     :- * Project (42)
+               :           :     :     :  +- * BroadcastHashJoin Inner BuildRight (41)
+               :           :     :     :     :- * Project (35)
+               :           :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (34)
+               :           :     :     :     :     :- * Project (28)
+               :           :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (27)
+               :           :     :     :     :     :     :- * Project (21)
+               :           :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (20)
+               :           :     :     :     :     :     :     :- * Project (15)
+               :           :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (14)
+               :           :     :     :     :     :     :     :     :- * Project (9)
+               :           :     :     :     :     :     :     :     :  +- * BroadcastHashJoin Inner BuildRight (8)
+               :           :     :     :     :     :     :     :     :     :- * Filter (3)
+               :           :     :     :     :     :     :     :     :     :  +- * ColumnarToRow (2)
+               :           :     :     :     :     :     :     :     :     :     +- Scan parquet default.catalog_sales (1)
+               :           :     :     :     :     :     :     :     :     +- BroadcastExchange (7)
+               :           :     :     :     :     :     :     :     :        +- * Filter (6)
+               :           :     :     :     :     :     :     :     :           +- * ColumnarToRow (5)
+               :           :     :     :     :     :     :     :     :              +- Scan parquet default.inventory (4)
+               :           :     :     :     :     :     :     :     +- BroadcastExchange (13)
+               :           :     :     :     :     :     :     :        +- * Filter (12)
+               :           :     :     :     :     :     :     :           +- * ColumnarToRow (11)
+               :           :     :     :     :     :     :     :              +- Scan parquet default.warehouse (10)
+               :           :     :     :     :     :     :     +- BroadcastExchange (19)
+               :           :     :     :     :     :     :        +- * Filter (18)
+               :           :     :     :     :     :     :           +- * ColumnarToRow (17)
+               :           :     :     :     :     :     :              +- Scan parquet default.item (16)
+               :           :     :     :     :     :     +- BroadcastExchange (26)
+               :           :     :     :     :     :        +- * Project (25)
+               :           :     :     :     :     :           +- * Filter (24)
+               :           :     :     :     :     :              +- * ColumnarToRow (23)
+               :           :     :     :     :     :                 +- Scan parquet default.customer_demographics (22)
+               :           :     :     :     :     +- BroadcastExchange (33)
+               :           :     :     :     :        +- * Project (32)
+               :           :     :     :     :           +- * Filter (31)
+               :           :     :     :     :              +- * ColumnarToRow (30)
+               :           :     :     :     :                 +- Scan parquet default.household_demographics (29)
+               :           :     :     :     +- BroadcastExchange (40)
+               :           :     :     :        +- * Project (39)
+               :           :     :     :           +- * Filter (38)
+               :           :     :     :              +- * ColumnarToRow (37)
+               :           :     :     :                 +- Scan parquet default.date_dim (36)
+               :           :     :     +- BroadcastExchange (46)
+               :           :     :        +- * Filter (45)
+               :           :     :           +- * ColumnarToRow (44)
+               :           :     :              +- Scan parquet default.date_dim (43)
+               :           :     +- BroadcastExchange (52)
+               :           :        +- * Filter (51)
+               :           :           +- * ColumnarToRow (50)
+               :           :              +- Scan parquet default.date_dim (49)
+               :           +- BroadcastExchange (58)
+               :              +- * Filter (57)
+               :                 +- * ColumnarToRow (56)
+               :                    +- Scan parquet default.promotion (55)
+               +- * Sort (68)
+                  +- Exchange (67)
+                     +- * Project (66)
+                        +- * Filter (65)
+                           +- * ColumnarToRow (64)
+                              +- Scan parquet default.catalog_returns (63)
+
+
+(1) Scan parquet default.catalog_sales
+Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#8), dynamicpruningexpression(cs_sold_date_sk#8 IN dynamicpruning#9)]
+PushedFilters: [IsNotNull(cs_quantity), IsNotNull(cs_item_sk), IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_hdemo_sk), IsNotNull(cs_ship_date_sk)]
+ReadSchema: struct<cs_ship_date_sk:int,cs_bill_cdemo_sk:int,cs_bill_hdemo_sk:int,cs_item_sk:int,cs_promo_sk:int,cs_order_number:int,cs_quantity:int>
+
+(2) ColumnarToRow [codegen id : 10]
+Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8]
+
+(3) Filter [codegen id : 10]
+Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8]
+Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1))
+
+(4) Scan parquet default.inventory
+Output [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(inv_date_sk#13)]
+PushedFilters: [IsNotNull(inv_quantity_on_hand), IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)]
+ReadSchema: struct<inv_item_sk:int,inv_warehouse_sk:int,inv_quantity_on_hand:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13]
+
+(6) Filter [codegen id : 1]
+Input [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13]
+Condition : ((isnotnull(inv_quantity_on_hand#12) AND isnotnull(inv_item_sk#10)) AND isnotnull(inv_warehouse_sk#11))
+
+(7) BroadcastExchange
+Input [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#14]
+
+(8) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_item_sk#4]
+Right keys [1]: [inv_item_sk#10]
+Join condition: (inv_quantity_on_hand#12 < cs_quantity#7)
+
+(9) Project [codegen id : 10]
+Output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13]
+Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13]
+
+(10) Scan parquet default.warehouse
+Output [2]: [w_warehouse_sk#15, w_warehouse_name#16]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/warehouse]
+PushedFilters: [IsNotNull(w_warehouse_sk)]
+ReadSchema: struct<w_warehouse_sk:int,w_warehouse_name:string>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [w_warehouse_sk#15, w_warehouse_name#16]
+
+(12) Filter [codegen id : 2]
+Input [2]: [w_warehouse_sk#15, w_warehouse_name#16]
+Condition : isnotnull(w_warehouse_sk#15)
+
+(13) BroadcastExchange
+Input [2]: [w_warehouse_sk#15, w_warehouse_name#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#17]
+
+(14) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [inv_warehouse_sk#11]
+Right keys [1]: [w_warehouse_sk#15]
+Join condition: None
+
+(15) Project [codegen id : 10]
+Output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#16]
+Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13, w_warehouse_sk#15, w_warehouse_name#16]
+
+(16) Scan parquet default.item
+Output [2]: [i_item_sk#18, i_item_desc#19]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_desc:string>
+
+(17) ColumnarToRow [codegen id : 3]
+Input [2]: [i_item_sk#18, i_item_desc#19]
+
+(18) Filter [codegen id : 3]
+Input [2]: [i_item_sk#18, i_item_desc#19]
+Condition : isnotnull(i_item_sk#18)
+
+(19) BroadcastExchange
+Input [2]: [i_item_sk#18, i_item_desc#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#20]
+
+(20) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_item_sk#4]
+Right keys [1]: [i_item_sk#18]
+Join condition: None
+
+(21) Project [codegen id : 10]
+Output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#16, i_item_desc#19]
+Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#16, i_item_sk#18, i_item_desc#19]
+
+(22) Scan parquet default.customer_demographics
+Output [2]: [cd_demo_sk#21, cd_marital_status#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer_demographics]
+PushedFilters: [IsNotNull(cd_marital_status), EqualTo(cd_marital_status,M), IsNotNull(cd_demo_sk)]
+ReadSchema: struct<cd_demo_sk:int,cd_marital_status:string>
+
+(23) ColumnarToRow [codegen id : 4]
+Input [2]: [cd_demo_sk#21, cd_marital_status#22]
+
+(24) Filter [codegen id : 4]
+Input [2]: [cd_demo_sk#21, cd_marital_status#22]
+Condition : ((isnotnull(cd_marital_status#22) AND (cd_marital_status#22 = M)) AND isnotnull(cd_demo_sk#21))
+
+(25) Project [codegen id : 4]
+Output [1]: [cd_demo_sk#21]
+Input [2]: [cd_demo_sk#21, cd_marital_status#22]
+
+(26) BroadcastExchange
+Input [1]: [cd_demo_sk#21]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#23]
+
+(27) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_bill_cdemo_sk#2]
+Right keys [1]: [cd_demo_sk#21]
+Join condition: None
+
+(28) Project [codegen id : 10]
+Output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#16, i_item_desc#19]
+Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#16, i_item_desc#19, cd_demo_sk#21]
+
+(29) Scan parquet default.household_demographics
+Output [2]: [hd_demo_sk#24, hd_buy_potential#25]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/household_demographics]
+PushedFilters: [IsNotNull(hd_buy_potential), EqualTo(hd_buy_potential,1001-5000      ), IsNotNull(hd_demo_sk)]
+ReadSchema: struct<hd_demo_sk:int,hd_buy_potential:string>
+
+(30) ColumnarToRow [codegen id : 5]
+Input [2]: [hd_demo_sk#24, hd_buy_potential#25]
+
+(31) Filter [codegen id : 5]
+Input [2]: [hd_demo_sk#24, hd_buy_potential#25]
+Condition : ((isnotnull(hd_buy_potential#25) AND (hd_buy_potential#25 = 1001-5000      )) AND isnotnull(hd_demo_sk#24))
+
+(32) Project [codegen id : 5]
+Output [1]: [hd_demo_sk#24]
+Input [2]: [hd_demo_sk#24, hd_buy_potential#25]
+
+(33) BroadcastExchange
+Input [1]: [hd_demo_sk#24]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#26]
+
+(34) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_bill_hdemo_sk#3]
+Right keys [1]: [hd_demo_sk#24]
+Join condition: None
+
+(35) Project [codegen id : 10]
+Output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#16, i_item_desc#19]
+Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#16, i_item_desc#19, hd_demo_sk#24]
+
+(36) Scan parquet default.date_dim
+Output [4]: [d_date_sk#27, d_date#28, d_week_seq#29, d_year#30]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)]
+ReadSchema: struct<d_date_sk:int,d_date:date,d_week_seq:int,d_year:int>
+
+(37) ColumnarToRow [codegen id : 6]
+Input [4]: [d_date_sk#27, d_date#28, d_week_seq#29, d_year#30]
+
+(38) Filter [codegen id : 6]
+Input [4]: [d_date_sk#27, d_date#28, d_week_seq#29, d_year#30]
+Condition : ((((isnotnull(d_year#30) AND (d_year#30 = 2001)) AND isnotnull(d_date_sk#27)) AND isnotnull(d_week_seq#29)) AND isnotnull(d_date#28))
+
+(39) Project [codegen id : 6]
+Output [3]: [d_date_sk#27, d_date#28, d_week_seq#29]
+Input [4]: [d_date_sk#27, d_date#28, d_week_seq#29, d_year#30]
+
+(40) BroadcastExchange
+Input [3]: [d_date_sk#27, d_date#28, d_week_seq#29]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#31]
+
+(41) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_sold_date_sk#8]
+Right keys [1]: [d_date_sk#27]
+Join condition: None
+
+(42) Project [codegen id : 10]
+Output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#16, i_item_desc#19, d_date#28, d_week_seq#29]
+Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#16, i_item_desc#19, d_date_sk#27, d_date#28, d_week_seq#29]
+
+(43) Scan parquet default.date_dim
+Output [2]: [d_date_sk#32, d_week_seq#33]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_week_seq:int>
+
+(44) ColumnarToRow [codegen id : 7]
+Input [2]: [d_date_sk#32, d_week_seq#33]
+
+(45) Filter [codegen id : 7]
+Input [2]: [d_date_sk#32, d_week_seq#33]
+Condition : (isnotnull(d_week_seq#33) AND isnotnull(d_date_sk#32))
+
+(46) BroadcastExchange
+Input [2]: [d_date_sk#32, d_week_seq#33]
+Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, false] as bigint), 32) | (cast(input[0, int, false] as bigint) & 4294967295))),false), [id=#34]
+
+(47) BroadcastHashJoin [codegen id : 10]
+Left keys [2]: [d_week_seq#29, inv_date_sk#13]
+Right keys [2]: [d_week_seq#33, d_date_sk#32]
+Join condition: None
+
+(48) Project [codegen id : 10]
+Output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#16, i_item_desc#19, d_date#28, d_week_seq#29]
+Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#16, i_item_desc#19, d_date#28, d_week_seq#29, d_date_sk#32, d_week_seq#33]
+
+(49) Scan parquet default.date_dim
+Output [2]: [d_date_sk#35, d_date#36]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(50) ColumnarToRow [codegen id : 8]
+Input [2]: [d_date_sk#35, d_date#36]
+
+(51) Filter [codegen id : 8]
+Input [2]: [d_date_sk#35, d_date#36]
+Condition : (isnotnull(d_date#36) AND isnotnull(d_date_sk#35))
+
+(52) BroadcastExchange
+Input [2]: [d_date_sk#35, d_date#36]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#37]
+
+(53) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_ship_date_sk#1]
+Right keys [1]: [d_date_sk#35]
+Join condition: (d_date#36 > d_date#28 + 5 days)
+
+(54) Project [codegen id : 10]
+Output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#16, i_item_desc#19, d_week_seq#29]
+Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#16, i_item_desc#19, d_date#28, d_week_seq#29, d_date_sk#35, d_date#36]
+
+(55) Scan parquet default.promotion
+Output [1]: [p_promo_sk#38]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/promotion]
+PushedFilters: [IsNotNull(p_promo_sk)]
+ReadSchema: struct<p_promo_sk:int>
+
+(56) ColumnarToRow [codegen id : 9]
+Input [1]: [p_promo_sk#38]
+
+(57) Filter [codegen id : 9]
+Input [1]: [p_promo_sk#38]
+Condition : isnotnull(p_promo_sk#38)
+
+(58) BroadcastExchange
+Input [1]: [p_promo_sk#38]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#39]
+
+(59) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_promo_sk#5]
+Right keys [1]: [p_promo_sk#38]
+Join condition: None
+
+(60) Project [codegen id : 10]
+Output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#16, i_item_desc#19, d_week_seq#29]
+Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#16, i_item_desc#19, d_week_seq#29, p_promo_sk#38]
+
+(61) Exchange
+Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#16, i_item_desc#19, d_week_seq#29]
+Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, [id=#40]
+
+(62) Sort [codegen id : 11]
+Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#16, i_item_desc#19, d_week_seq#29]
+Arguments: [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST], false, 0
+
+(63) Scan parquet default.catalog_returns
+Output [3]: [cr_item_sk#41, cr_order_number#42, cr_returned_date_sk#43]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int>
+
+(64) ColumnarToRow [codegen id : 12]
+Input [3]: [cr_item_sk#41, cr_order_number#42, cr_returned_date_sk#43]
+
+(65) Filter [codegen id : 12]
+Input [3]: [cr_item_sk#41, cr_order_number#42, cr_returned_date_sk#43]
+Condition : (isnotnull(cr_item_sk#41) AND isnotnull(cr_order_number#42))
+
+(66) Project [codegen id : 12]
+Output [2]: [cr_item_sk#41, cr_order_number#42]
+Input [3]: [cr_item_sk#41, cr_order_number#42, cr_returned_date_sk#43]
+
+(67) Exchange
+Input [2]: [cr_item_sk#41, cr_order_number#42]
+Arguments: hashpartitioning(cr_item_sk#41, cr_order_number#42, 5), ENSURE_REQUIREMENTS, [id=#44]
+
+(68) Sort [codegen id : 13]
+Input [2]: [cr_item_sk#41, cr_order_number#42]
+Arguments: [cr_item_sk#41 ASC NULLS FIRST, cr_order_number#42 ASC NULLS FIRST], false, 0
+
+(69) SortMergeJoin [codegen id : 14]
+Left keys [2]: [cs_item_sk#4, cs_order_number#6]
+Right keys [2]: [cr_item_sk#41, cr_order_number#42]
+Join condition: None
+
+(70) Project [codegen id : 14]
+Output [3]: [w_warehouse_name#16, i_item_desc#19, d_week_seq#29]
+Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#16, i_item_desc#19, d_week_seq#29, cr_item_sk#41, cr_order_number#42]
+
+(71) HashAggregate [codegen id : 14]
+Input [3]: [w_warehouse_name#16, i_item_desc#19, d_week_seq#29]
+Keys [3]: [i_item_desc#19, w_warehouse_name#16, d_week_seq#29]
+Functions [1]: [partial_count(1)]
+Aggregate Attributes [1]: [count#45]
+Results [4]: [i_item_desc#19, w_warehouse_name#16, d_week_seq#29, count#46]
+
+(72) Exchange
+Input [4]: [i_item_desc#19, w_warehouse_name#16, d_week_seq#29, count#46]
+Arguments: hashpartitioning(i_item_desc#19, w_warehouse_name#16, d_week_seq#29, 5), ENSURE_REQUIREMENTS, [id=#47]
+
+(73) HashAggregate [codegen id : 15]
+Input [4]: [i_item_desc#19, w_warehouse_name#16, d_week_seq#29, count#46]
+Keys [3]: [i_item_desc#19, w_warehouse_name#16, d_week_seq#29]
+Functions [1]: [count(1)]
+Aggregate Attributes [1]: [count(1)#48]
+Results [6]: [i_item_desc#19, w_warehouse_name#16, d_week_seq#29, count(1)#48 AS no_promo#49, count(1)#48 AS promo#50, count(1)#48 AS total_cnt#51]
+
+(74) TakeOrderedAndProject
+Input [6]: [i_item_desc#19, w_warehouse_name#16, d_week_seq#29, no_promo#49, promo#50, total_cnt#51]
+Arguments: 100, [total_cnt#51 DESC NULLS LAST, i_item_desc#19 ASC NULLS FIRST, w_warehouse_name#16 ASC NULLS FIRST, d_week_seq#29 ASC NULLS FIRST], [i_item_desc#19, w_warehouse_name#16, d_week_seq#29, no_promo#49, promo#50, total_cnt#51]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#8 IN dynamicpruning#9
+ReusedExchange (75)
+
+
+(75) ReusedExchange [Reuses operator id: 40]
+Output [3]: [d_date_sk#27, d_date#28, d_week_seq#29]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/simplified.txt
new file mode 100644
index 0000000000000..92800d31b82f1
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/simplified.txt
@@ -0,0 +1,116 @@
+TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo]
+  WholeStageCodegen (15)
+    HashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] [count(1),no_promo,promo,total_cnt,count]
+      InputAdapter
+        Exchange [i_item_desc,w_warehouse_name,d_week_seq] #1
+          WholeStageCodegen (14)
+            HashAggregate [i_item_desc,w_warehouse_name,d_week_seq] [count,count]
+              Project [w_warehouse_name,i_item_desc,d_week_seq]
+                SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number]
+                  InputAdapter
+                    WholeStageCodegen (11)
+                      Sort [cs_item_sk,cs_order_number]
+                        InputAdapter
+                          Exchange [cs_item_sk,cs_order_number] #2
+                            WholeStageCodegen (10)
+                              Project [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq]
+                                BroadcastHashJoin [cs_promo_sk,p_promo_sk]
+                                  Project [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq]
+                                    BroadcastHashJoin [cs_ship_date_sk,d_date_sk,d_date,d_date]
+                                      Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq]
+                                        BroadcastHashJoin [d_week_seq,inv_date_sk,d_week_seq,d_date_sk]
+                                          Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq]
+                                            BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                              Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc]
+                                                BroadcastHashJoin [cs_bill_hdemo_sk,hd_demo_sk]
+                                                  Project [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc]
+                                                    BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk]
+                                                      Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc]
+                                                        BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                                          Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name]
+                                                            BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk]
+                                                              Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk]
+                                                                BroadcastHashJoin [cs_item_sk,inv_item_sk,inv_quantity_on_hand,cs_quantity]
+                                                                  Filter [cs_quantity,cs_item_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_ship_date_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk]
+                                                                          SubqueryBroadcast [d_date_sk] #1
+                                                                            ReusedExchange [d_date_sk,d_date,d_week_seq] #3
+                                                                  InputAdapter
+                                                                    BroadcastExchange #4
+                                                                      WholeStageCodegen (1)
+                                                                        Filter [inv_quantity_on_hand,inv_item_sk,inv_warehouse_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk]
+                                                              InputAdapter
+                                                                BroadcastExchange #5
+                                                                  WholeStageCodegen (2)
+                                                                    Filter [w_warehouse_sk]
+                                                                      ColumnarToRow
+                                                                        InputAdapter
+                                                                          Scan parquet default.warehouse [w_warehouse_sk,w_warehouse_name]
+                                                          InputAdapter
+                                                            BroadcastExchange #6
+                                                              WholeStageCodegen (3)
+                                                                Filter [i_item_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.item [i_item_sk,i_item_desc]
+                                                      InputAdapter
+                                                        BroadcastExchange #7
+                                                          WholeStageCodegen (4)
+                                                            Project [cd_demo_sk]
+                                                              Filter [cd_marital_status,cd_demo_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.customer_demographics [cd_demo_sk,cd_marital_status]
+                                                  InputAdapter
+                                                    BroadcastExchange #8
+                                                      WholeStageCodegen (5)
+                                                        Project [hd_demo_sk]
+                                                          Filter [hd_buy_potential,hd_demo_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.household_demographics [hd_demo_sk,hd_buy_potential]
+                                              InputAdapter
+                                                BroadcastExchange #3
+                                                  WholeStageCodegen (6)
+                                                    Project [d_date_sk,d_date,d_week_seq]
+                                                      Filter [d_year,d_date_sk,d_week_seq,d_date]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.date_dim [d_date_sk,d_date,d_week_seq,d_year]
+                                          InputAdapter
+                                            BroadcastExchange #9
+                                              WholeStageCodegen (7)
+                                                Filter [d_week_seq,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_week_seq]
+                                      InputAdapter
+                                        BroadcastExchange #10
+                                          WholeStageCodegen (8)
+                                            Filter [d_date,d_date_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.date_dim [d_date_sk,d_date]
+                                  InputAdapter
+                                    BroadcastExchange #11
+                                      WholeStageCodegen (9)
+                                        Filter [p_promo_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.promotion [p_promo_sk]
+                  InputAdapter
+                    WholeStageCodegen (13)
+                      Sort [cr_item_sk,cr_order_number]
+                        InputAdapter
+                          Exchange [cr_item_sk,cr_order_number] #12
+                            WholeStageCodegen (12)
+                              Project [cr_item_sk,cr_order_number]
+                                Filter [cr_item_sk,cr_order_number]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.sf100/explain.txt
new file mode 100644
index 0000000000000..7eea8040043d6
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.sf100/explain.txt
@@ -0,0 +1,502 @@
+== Physical Plan ==
+TakeOrderedAndProject (86)
++- * Project (85)
+   +- * SortMergeJoin Inner (84)
+      :- * Project (66)
+      :  +- * SortMergeJoin Inner (65)
+      :     :- * SortMergeJoin Inner (45)
+      :     :  :- * Sort (24)
+      :     :  :  +- Exchange (23)
+      :     :  :     +- * Filter (22)
+      :     :  :        +- * HashAggregate (21)
+      :     :  :           +- Exchange (20)
+      :     :  :              +- * HashAggregate (19)
+      :     :  :                 +- * Project (18)
+      :     :  :                    +- * SortMergeJoin Inner (17)
+      :     :  :                       :- * Sort (11)
+      :     :  :                       :  +- Exchange (10)
+      :     :  :                       :     +- * Project (9)
+      :     :  :                       :        +- * BroadcastHashJoin Inner BuildRight (8)
+      :     :  :                       :           :- * Filter (3)
+      :     :  :                       :           :  +- * ColumnarToRow (2)
+      :     :  :                       :           :     +- Scan parquet default.store_sales (1)
+      :     :  :                       :           +- BroadcastExchange (7)
+      :     :  :                       :              +- * Filter (6)
+      :     :  :                       :                 +- * ColumnarToRow (5)
+      :     :  :                       :                    +- Scan parquet default.date_dim (4)
+      :     :  :                       +- * Sort (16)
+      :     :  :                          +- Exchange (15)
+      :     :  :                             +- * Filter (14)
+      :     :  :                                +- * ColumnarToRow (13)
+      :     :  :                                   +- Scan parquet default.customer (12)
+      :     :  +- * Sort (44)
+      :     :     +- Exchange (43)
+      :     :        +- * HashAggregate (42)
+      :     :           +- Exchange (41)
+      :     :              +- * HashAggregate (40)
+      :     :                 +- * Project (39)
+      :     :                    +- * SortMergeJoin Inner (38)
+      :     :                       :- * Sort (35)
+      :     :                       :  +- Exchange (34)
+      :     :                       :     +- * Project (33)
+      :     :                       :        +- * BroadcastHashJoin Inner BuildRight (32)
+      :     :                       :           :- * Filter (27)
+      :     :                       :           :  +- * ColumnarToRow (26)
+      :     :                       :           :     +- Scan parquet default.store_sales (25)
+      :     :                       :           +- BroadcastExchange (31)
+      :     :                       :              +- * Filter (30)
+      :     :                       :                 +- * ColumnarToRow (29)
+      :     :                       :                    +- Scan parquet default.date_dim (28)
+      :     :                       +- * Sort (37)
+      :     :                          +- ReusedExchange (36)
+      :     +- * Sort (64)
+      :        +- Exchange (63)
+      :           +- * Project (62)
+      :              +- * Filter (61)
+      :                 +- * HashAggregate (60)
+      :                    +- Exchange (59)
+      :                       +- * HashAggregate (58)
+      :                          +- * Project (57)
+      :                             +- * SortMergeJoin Inner (56)
+      :                                :- * Sort (53)
+      :                                :  +- Exchange (52)
+      :                                :     +- * Project (51)
+      :                                :        +- * BroadcastHashJoin Inner BuildRight (50)
+      :                                :           :- * Filter (48)
+      :                                :           :  +- * ColumnarToRow (47)
+      :                                :           :     +- Scan parquet default.web_sales (46)
+      :                                :           +- ReusedExchange (49)
+      :                                +- * Sort (55)
+      :                                   +- ReusedExchange (54)
+      +- * Sort (83)
+         +- Exchange (82)
+            +- * HashAggregate (81)
+               +- Exchange (80)
+                  +- * HashAggregate (79)
+                     +- * Project (78)
+                        +- * SortMergeJoin Inner (77)
+                           :- * Sort (74)
+                           :  +- Exchange (73)
+                           :     +- * Project (72)
+                           :        +- * BroadcastHashJoin Inner BuildRight (71)
+                           :           :- * Filter (69)
+                           :           :  +- * ColumnarToRow (68)
+                           :           :     +- Scan parquet default.web_sales (67)
+                           :           +- ReusedExchange (70)
+                           +- * Sort (76)
+                              +- ReusedExchange (75)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_customer_sk#1, ss_net_paid#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_net_paid:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 2]
+Input [3]: [ss_customer_sk#1, ss_net_paid#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 2]
+Input [3]: [ss_customer_sk#1, ss_net_paid#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_customer_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#5, d_year#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#5, d_year#6]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#5, d_year#6]
+Condition : (((isnotnull(d_year#6) AND (d_year#6 = 2001)) AND d_year#6 IN (2001,2002)) AND isnotnull(d_date_sk#5))
+
+(7) BroadcastExchange
+Input [2]: [d_date_sk#5, d_year#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#7]
+
+(8) BroadcastHashJoin [codegen id : 2]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#5]
+Join condition: None
+
+(9) Project [codegen id : 2]
+Output [3]: [ss_customer_sk#1, ss_net_paid#2, d_year#6]
+Input [5]: [ss_customer_sk#1, ss_net_paid#2, ss_sold_date_sk#3, d_date_sk#5, d_year#6]
+
+(10) Exchange
+Input [3]: [ss_customer_sk#1, ss_net_paid#2, d_year#6]
+Arguments: hashpartitioning(ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [id=#8]
+
+(11) Sort [codegen id : 3]
+Input [3]: [ss_customer_sk#1, ss_net_paid#2, d_year#6]
+Arguments: [ss_customer_sk#1 ASC NULLS FIRST], false, 0
+
+(12) Scan parquet default.customer
+Output [4]: [c_customer_sk#9, c_customer_id#10, c_first_name#11, c_last_name#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string>
+
+(13) ColumnarToRow [codegen id : 4]
+Input [4]: [c_customer_sk#9, c_customer_id#10, c_first_name#11, c_last_name#12]
+
+(14) Filter [codegen id : 4]
+Input [4]: [c_customer_sk#9, c_customer_id#10, c_first_name#11, c_last_name#12]
+Condition : (isnotnull(c_customer_sk#9) AND isnotnull(c_customer_id#10))
+
+(15) Exchange
+Input [4]: [c_customer_sk#9, c_customer_id#10, c_first_name#11, c_last_name#12]
+Arguments: hashpartitioning(c_customer_sk#9, 5), ENSURE_REQUIREMENTS, [id=#13]
+
+(16) Sort [codegen id : 5]
+Input [4]: [c_customer_sk#9, c_customer_id#10, c_first_name#11, c_last_name#12]
+Arguments: [c_customer_sk#9 ASC NULLS FIRST], false, 0
+
+(17) SortMergeJoin [codegen id : 6]
+Left keys [1]: [ss_customer_sk#1]
+Right keys [1]: [c_customer_sk#9]
+Join condition: None
+
+(18) Project [codegen id : 6]
+Output [5]: [c_customer_id#10, c_first_name#11, c_last_name#12, ss_net_paid#2, d_year#6]
+Input [7]: [ss_customer_sk#1, ss_net_paid#2, d_year#6, c_customer_sk#9, c_customer_id#10, c_first_name#11, c_last_name#12]
+
+(19) HashAggregate [codegen id : 6]
+Input [5]: [c_customer_id#10, c_first_name#11, c_last_name#12, ss_net_paid#2, d_year#6]
+Keys [4]: [c_customer_id#10, c_first_name#11, c_last_name#12, d_year#6]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#2))]
+Aggregate Attributes [1]: [sum#14]
+Results [5]: [c_customer_id#10, c_first_name#11, c_last_name#12, d_year#6, sum#15]
+
+(20) Exchange
+Input [5]: [c_customer_id#10, c_first_name#11, c_last_name#12, d_year#6, sum#15]
+Arguments: hashpartitioning(c_customer_id#10, c_first_name#11, c_last_name#12, d_year#6, 5), ENSURE_REQUIREMENTS, [id=#16]
+
+(21) HashAggregate [codegen id : 7]
+Input [5]: [c_customer_id#10, c_first_name#11, c_last_name#12, d_year#6, sum#15]
+Keys [4]: [c_customer_id#10, c_first_name#11, c_last_name#12, d_year#6]
+Functions [1]: [sum(UnscaledValue(ss_net_paid#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#2))#17]
+Results [2]: [c_customer_id#10 AS customer_id#18, MakeDecimal(sum(UnscaledValue(ss_net_paid#2))#17,17,2) AS year_total#19]
+
+(22) Filter [codegen id : 7]
+Input [2]: [customer_id#18, year_total#19]
+Condition : (isnotnull(year_total#19) AND (year_total#19 > 0.00))
+
+(23) Exchange
+Input [2]: [customer_id#18, year_total#19]
+Arguments: hashpartitioning(customer_id#18, 5), ENSURE_REQUIREMENTS, [id=#20]
+
+(24) Sort [codegen id : 8]
+Input [2]: [customer_id#18, year_total#19]
+Arguments: [customer_id#18 ASC NULLS FIRST], false, 0
+
+(25) Scan parquet default.store_sales
+Output [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#23), dynamicpruningexpression(ss_sold_date_sk#23 IN dynamicpruning#24)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_net_paid:decimal(7,2)>
+
+(26) ColumnarToRow [codegen id : 10]
+Input [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23]
+
+(27) Filter [codegen id : 10]
+Input [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23]
+Condition : isnotnull(ss_customer_sk#21)
+
+(28) Scan parquet default.date_dim
+Output [2]: [d_date_sk#25, d_year#26]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(29) ColumnarToRow [codegen id : 9]
+Input [2]: [d_date_sk#25, d_year#26]
+
+(30) Filter [codegen id : 9]
+Input [2]: [d_date_sk#25, d_year#26]
+Condition : (((isnotnull(d_year#26) AND (d_year#26 = 2002)) AND d_year#26 IN (2001,2002)) AND isnotnull(d_date_sk#25))
+
+(31) BroadcastExchange
+Input [2]: [d_date_sk#25, d_year#26]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#27]
+
+(32) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_sold_date_sk#23]
+Right keys [1]: [d_date_sk#25]
+Join condition: None
+
+(33) Project [codegen id : 10]
+Output [3]: [ss_customer_sk#21, ss_net_paid#22, d_year#26]
+Input [5]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23, d_date_sk#25, d_year#26]
+
+(34) Exchange
+Input [3]: [ss_customer_sk#21, ss_net_paid#22, d_year#26]
+Arguments: hashpartitioning(ss_customer_sk#21, 5), ENSURE_REQUIREMENTS, [id=#28]
+
+(35) Sort [codegen id : 11]
+Input [3]: [ss_customer_sk#21, ss_net_paid#22, d_year#26]
+Arguments: [ss_customer_sk#21 ASC NULLS FIRST], false, 0
+
+(36) ReusedExchange [Reuses operator id: 15]
+Output [4]: [c_customer_sk#29, c_customer_id#30, c_first_name#31, c_last_name#32]
+
+(37) Sort [codegen id : 13]
+Input [4]: [c_customer_sk#29, c_customer_id#30, c_first_name#31, c_last_name#32]
+Arguments: [c_customer_sk#29 ASC NULLS FIRST], false, 0
+
+(38) SortMergeJoin [codegen id : 14]
+Left keys [1]: [ss_customer_sk#21]
+Right keys [1]: [c_customer_sk#29]
+Join condition: None
+
+(39) Project [codegen id : 14]
+Output [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, ss_net_paid#22, d_year#26]
+Input [7]: [ss_customer_sk#21, ss_net_paid#22, d_year#26, c_customer_sk#29, c_customer_id#30, c_first_name#31, c_last_name#32]
+
+(40) HashAggregate [codegen id : 14]
+Input [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, ss_net_paid#22, d_year#26]
+Keys [4]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#26]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#22))]
+Aggregate Attributes [1]: [sum#33]
+Results [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#26, sum#34]
+
+(41) Exchange
+Input [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#26, sum#34]
+Arguments: hashpartitioning(c_customer_id#30, c_first_name#31, c_last_name#32, d_year#26, 5), ENSURE_REQUIREMENTS, [id=#35]
+
+(42) HashAggregate [codegen id : 15]
+Input [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#26, sum#34]
+Keys [4]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#26]
+Functions [1]: [sum(UnscaledValue(ss_net_paid#22))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#22))#36]
+Results [4]: [c_customer_id#30 AS customer_id#37, c_first_name#31 AS customer_first_name#38, c_last_name#32 AS customer_last_name#39, MakeDecimal(sum(UnscaledValue(ss_net_paid#22))#36,17,2) AS year_total#40]
+
+(43) Exchange
+Input [4]: [customer_id#37, customer_first_name#38, customer_last_name#39, year_total#40]
+Arguments: hashpartitioning(customer_id#37, 5), ENSURE_REQUIREMENTS, [id=#41]
+
+(44) Sort [codegen id : 16]
+Input [4]: [customer_id#37, customer_first_name#38, customer_last_name#39, year_total#40]
+Arguments: [customer_id#37 ASC NULLS FIRST], false, 0
+
+(45) SortMergeJoin [codegen id : 17]
+Left keys [1]: [customer_id#18]
+Right keys [1]: [customer_id#37]
+Join condition: None
+
+(46) Scan parquet default.web_sales
+Output [3]: [ws_bill_customer_sk#42, ws_net_paid#43, ws_sold_date_sk#44]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#44), dynamicpruningexpression(ws_sold_date_sk#44 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int,ws_net_paid:decimal(7,2)>
+
+(47) ColumnarToRow [codegen id : 19]
+Input [3]: [ws_bill_customer_sk#42, ws_net_paid#43, ws_sold_date_sk#44]
+
+(48) Filter [codegen id : 19]
+Input [3]: [ws_bill_customer_sk#42, ws_net_paid#43, ws_sold_date_sk#44]
+Condition : isnotnull(ws_bill_customer_sk#42)
+
+(49) ReusedExchange [Reuses operator id: 7]
+Output [2]: [d_date_sk#45, d_year#46]
+
+(50) BroadcastHashJoin [codegen id : 19]
+Left keys [1]: [ws_sold_date_sk#44]
+Right keys [1]: [d_date_sk#45]
+Join condition: None
+
+(51) Project [codegen id : 19]
+Output [3]: [ws_bill_customer_sk#42, ws_net_paid#43, d_year#46]
+Input [5]: [ws_bill_customer_sk#42, ws_net_paid#43, ws_sold_date_sk#44, d_date_sk#45, d_year#46]
+
+(52) Exchange
+Input [3]: [ws_bill_customer_sk#42, ws_net_paid#43, d_year#46]
+Arguments: hashpartitioning(ws_bill_customer_sk#42, 5), ENSURE_REQUIREMENTS, [id=#47]
+
+(53) Sort [codegen id : 20]
+Input [3]: [ws_bill_customer_sk#42, ws_net_paid#43, d_year#46]
+Arguments: [ws_bill_customer_sk#42 ASC NULLS FIRST], false, 0
+
+(54) ReusedExchange [Reuses operator id: 15]
+Output [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51]
+
+(55) Sort [codegen id : 22]
+Input [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51]
+Arguments: [c_customer_sk#48 ASC NULLS FIRST], false, 0
+
+(56) SortMergeJoin [codegen id : 23]
+Left keys [1]: [ws_bill_customer_sk#42]
+Right keys [1]: [c_customer_sk#48]
+Join condition: None
+
+(57) Project [codegen id : 23]
+Output [5]: [c_customer_id#49, c_first_name#50, c_last_name#51, ws_net_paid#43, d_year#46]
+Input [7]: [ws_bill_customer_sk#42, ws_net_paid#43, d_year#46, c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51]
+
+(58) HashAggregate [codegen id : 23]
+Input [5]: [c_customer_id#49, c_first_name#50, c_last_name#51, ws_net_paid#43, d_year#46]
+Keys [4]: [c_customer_id#49, c_first_name#50, c_last_name#51, d_year#46]
+Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#43))]
+Aggregate Attributes [1]: [sum#52]
+Results [5]: [c_customer_id#49, c_first_name#50, c_last_name#51, d_year#46, sum#53]
+
+(59) Exchange
+Input [5]: [c_customer_id#49, c_first_name#50, c_last_name#51, d_year#46, sum#53]
+Arguments: hashpartitioning(c_customer_id#49, c_first_name#50, c_last_name#51, d_year#46, 5), ENSURE_REQUIREMENTS, [id=#54]
+
+(60) HashAggregate [codegen id : 24]
+Input [5]: [c_customer_id#49, c_first_name#50, c_last_name#51, d_year#46, sum#53]
+Keys [4]: [c_customer_id#49, c_first_name#50, c_last_name#51, d_year#46]
+Functions [1]: [sum(UnscaledValue(ws_net_paid#43))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#43))#55]
+Results [2]: [c_customer_id#49 AS customer_id#56, MakeDecimal(sum(UnscaledValue(ws_net_paid#43))#55,17,2) AS year_total#57]
+
+(61) Filter [codegen id : 24]
+Input [2]: [customer_id#56, year_total#57]
+Condition : (isnotnull(year_total#57) AND (year_total#57 > 0.00))
+
+(62) Project [codegen id : 24]
+Output [2]: [customer_id#56 AS customer_id#58, year_total#57 AS year_total#59]
+Input [2]: [customer_id#56, year_total#57]
+
+(63) Exchange
+Input [2]: [customer_id#58, year_total#59]
+Arguments: hashpartitioning(customer_id#58, 5), ENSURE_REQUIREMENTS, [id=#60]
+
+(64) Sort [codegen id : 25]
+Input [2]: [customer_id#58, year_total#59]
+Arguments: [customer_id#58 ASC NULLS FIRST], false, 0
+
+(65) SortMergeJoin [codegen id : 26]
+Left keys [1]: [customer_id#18]
+Right keys [1]: [customer_id#58]
+Join condition: None
+
+(66) Project [codegen id : 26]
+Output [7]: [customer_id#18, year_total#19, customer_id#37, customer_first_name#38, customer_last_name#39, year_total#40, year_total#59]
+Input [8]: [customer_id#18, year_total#19, customer_id#37, customer_first_name#38, customer_last_name#39, year_total#40, customer_id#58, year_total#59]
+
+(67) Scan parquet default.web_sales
+Output [3]: [ws_bill_customer_sk#61, ws_net_paid#62, ws_sold_date_sk#63]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#63), dynamicpruningexpression(ws_sold_date_sk#63 IN dynamicpruning#24)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int,ws_net_paid:decimal(7,2)>
+
+(68) ColumnarToRow [codegen id : 28]
+Input [3]: [ws_bill_customer_sk#61, ws_net_paid#62, ws_sold_date_sk#63]
+
+(69) Filter [codegen id : 28]
+Input [3]: [ws_bill_customer_sk#61, ws_net_paid#62, ws_sold_date_sk#63]
+Condition : isnotnull(ws_bill_customer_sk#61)
+
+(70) ReusedExchange [Reuses operator id: 31]
+Output [2]: [d_date_sk#64, d_year#65]
+
+(71) BroadcastHashJoin [codegen id : 28]
+Left keys [1]: [ws_sold_date_sk#63]
+Right keys [1]: [d_date_sk#64]
+Join condition: None
+
+(72) Project [codegen id : 28]
+Output [3]: [ws_bill_customer_sk#61, ws_net_paid#62, d_year#65]
+Input [5]: [ws_bill_customer_sk#61, ws_net_paid#62, ws_sold_date_sk#63, d_date_sk#64, d_year#65]
+
+(73) Exchange
+Input [3]: [ws_bill_customer_sk#61, ws_net_paid#62, d_year#65]
+Arguments: hashpartitioning(ws_bill_customer_sk#61, 5), ENSURE_REQUIREMENTS, [id=#66]
+
+(74) Sort [codegen id : 29]
+Input [3]: [ws_bill_customer_sk#61, ws_net_paid#62, d_year#65]
+Arguments: [ws_bill_customer_sk#61 ASC NULLS FIRST], false, 0
+
+(75) ReusedExchange [Reuses operator id: 15]
+Output [4]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70]
+
+(76) Sort [codegen id : 31]
+Input [4]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70]
+Arguments: [c_customer_sk#67 ASC NULLS FIRST], false, 0
+
+(77) SortMergeJoin [codegen id : 32]
+Left keys [1]: [ws_bill_customer_sk#61]
+Right keys [1]: [c_customer_sk#67]
+Join condition: None
+
+(78) Project [codegen id : 32]
+Output [5]: [c_customer_id#68, c_first_name#69, c_last_name#70, ws_net_paid#62, d_year#65]
+Input [7]: [ws_bill_customer_sk#61, ws_net_paid#62, d_year#65, c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70]
+
+(79) HashAggregate [codegen id : 32]
+Input [5]: [c_customer_id#68, c_first_name#69, c_last_name#70, ws_net_paid#62, d_year#65]
+Keys [4]: [c_customer_id#68, c_first_name#69, c_last_name#70, d_year#65]
+Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#62))]
+Aggregate Attributes [1]: [sum#71]
+Results [5]: [c_customer_id#68, c_first_name#69, c_last_name#70, d_year#65, sum#72]
+
+(80) Exchange
+Input [5]: [c_customer_id#68, c_first_name#69, c_last_name#70, d_year#65, sum#72]
+Arguments: hashpartitioning(c_customer_id#68, c_first_name#69, c_last_name#70, d_year#65, 5), ENSURE_REQUIREMENTS, [id=#73]
+
+(81) HashAggregate [codegen id : 33]
+Input [5]: [c_customer_id#68, c_first_name#69, c_last_name#70, d_year#65, sum#72]
+Keys [4]: [c_customer_id#68, c_first_name#69, c_last_name#70, d_year#65]
+Functions [1]: [sum(UnscaledValue(ws_net_paid#62))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#62))#74]
+Results [2]: [c_customer_id#68 AS customer_id#75, MakeDecimal(sum(UnscaledValue(ws_net_paid#62))#74,17,2) AS year_total#76]
+
+(82) Exchange
+Input [2]: [customer_id#75, year_total#76]
+Arguments: hashpartitioning(customer_id#75, 5), ENSURE_REQUIREMENTS, [id=#77]
+
+(83) Sort [codegen id : 34]
+Input [2]: [customer_id#75, year_total#76]
+Arguments: [customer_id#75 ASC NULLS FIRST], false, 0
+
+(84) SortMergeJoin [codegen id : 35]
+Left keys [1]: [customer_id#18]
+Right keys [1]: [customer_id#75]
+Join condition: (CASE WHEN (year_total#59 > 0.00) THEN CheckOverflow((promote_precision(year_total#76) / promote_precision(year_total#59)), DecimalType(37,20), true) ELSE null END > CASE WHEN (year_total#19 > 0.00) THEN CheckOverflow((promote_precision(year_total#40) / promote_precision(year_total#19)), DecimalType(37,20), true) ELSE null END)
+
+(85) Project [codegen id : 35]
+Output [3]: [customer_id#37, customer_first_name#38, customer_last_name#39]
+Input [9]: [customer_id#18, year_total#19, customer_id#37, customer_first_name#38, customer_last_name#39, year_total#40, year_total#59, customer_id#75, year_total#76]
+
+(86) TakeOrderedAndProject
+Input [3]: [customer_id#37, customer_first_name#38, customer_last_name#39]
+Arguments: 100, [customer_first_name#38 ASC NULLS FIRST, customer_id#37 ASC NULLS FIRST, customer_last_name#39 ASC NULLS FIRST], [customer_id#37, customer_first_name#38, customer_last_name#39]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (87)
+
+
+(87) ReusedExchange [Reuses operator id: 7]
+Output [2]: [d_date_sk#5, d_year#6]
+
+Subquery:2 Hosting operator id = 25 Hosting Expression = ss_sold_date_sk#23 IN dynamicpruning#24
+ReusedExchange (88)
+
+
+(88) ReusedExchange [Reuses operator id: 31]
+Output [2]: [d_date_sk#25, d_year#26]
+
+Subquery:3 Hosting operator id = 46 Hosting Expression = ws_sold_date_sk#44 IN dynamicpruning#4
+
+Subquery:4 Hosting operator id = 67 Hosting Expression = ws_sold_date_sk#63 IN dynamicpruning#24
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.sf100/simplified.txt
new file mode 100644
index 0000000000000..7152af691bd03
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.sf100/simplified.txt
@@ -0,0 +1,163 @@
+TakeOrderedAndProject [customer_first_name,customer_id,customer_last_name]
+  WholeStageCodegen (35)
+    Project [customer_id,customer_first_name,customer_last_name]
+      SortMergeJoin [customer_id,customer_id,year_total,year_total,year_total,year_total]
+        InputAdapter
+          WholeStageCodegen (26)
+            Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total]
+              SortMergeJoin [customer_id,customer_id]
+                InputAdapter
+                  WholeStageCodegen (17)
+                    SortMergeJoin [customer_id,customer_id]
+                      InputAdapter
+                        WholeStageCodegen (8)
+                          Sort [customer_id]
+                            InputAdapter
+                              Exchange [customer_id] #1
+                                WholeStageCodegen (7)
+                                  Filter [year_total]
+                                    HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,year_total,sum]
+                                      InputAdapter
+                                        Exchange [c_customer_id,c_first_name,c_last_name,d_year] #2
+                                          WholeStageCodegen (6)
+                                            HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum]
+                                              Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year]
+                                                SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (3)
+                                                      Sort [ss_customer_sk]
+                                                        InputAdapter
+                                                          Exchange [ss_customer_sk] #3
+                                                            WholeStageCodegen (2)
+                                                              Project [ss_customer_sk,ss_net_paid,d_year]
+                                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                  Filter [ss_customer_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk]
+                                                                          SubqueryBroadcast [d_date_sk] #1
+                                                                            ReusedExchange [d_date_sk,d_year] #4
+                                                                  InputAdapter
+                                                                    BroadcastExchange #4
+                                                                      WholeStageCodegen (1)
+                                                                        Filter [d_year,d_date_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.date_dim [d_date_sk,d_year]
+                                                  InputAdapter
+                                                    WholeStageCodegen (5)
+                                                      Sort [c_customer_sk]
+                                                        InputAdapter
+                                                          Exchange [c_customer_sk] #5
+                                                            WholeStageCodegen (4)
+                                                              Filter [c_customer_sk,c_customer_id]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name]
+                      InputAdapter
+                        WholeStageCodegen (16)
+                          Sort [customer_id]
+                            InputAdapter
+                              Exchange [customer_id] #6
+                                WholeStageCodegen (15)
+                                  HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,customer_first_name,customer_last_name,year_total,sum]
+                                    InputAdapter
+                                      Exchange [c_customer_id,c_first_name,c_last_name,d_year] #7
+                                        WholeStageCodegen (14)
+                                          HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum]
+                                            Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year]
+                                              SortMergeJoin [ss_customer_sk,c_customer_sk]
+                                                InputAdapter
+                                                  WholeStageCodegen (11)
+                                                    Sort [ss_customer_sk]
+                                                      InputAdapter
+                                                        Exchange [ss_customer_sk] #8
+                                                          WholeStageCodegen (10)
+                                                            Project [ss_customer_sk,ss_net_paid,d_year]
+                                                              BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                Filter [ss_customer_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk]
+                                                                        SubqueryBroadcast [d_date_sk] #2
+                                                                          ReusedExchange [d_date_sk,d_year] #9
+                                                                InputAdapter
+                                                                  BroadcastExchange #9
+                                                                    WholeStageCodegen (9)
+                                                                      Filter [d_year,d_date_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.date_dim [d_date_sk,d_year]
+                                                InputAdapter
+                                                  WholeStageCodegen (13)
+                                                    Sort [c_customer_sk]
+                                                      InputAdapter
+                                                        ReusedExchange [c_customer_sk,c_customer_id,c_first_name,c_last_name] #5
+                InputAdapter
+                  WholeStageCodegen (25)
+                    Sort [customer_id]
+                      InputAdapter
+                        Exchange [customer_id] #10
+                          WholeStageCodegen (24)
+                            Project [customer_id,year_total]
+                              Filter [year_total]
+                                HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum]
+                                  InputAdapter
+                                    Exchange [c_customer_id,c_first_name,c_last_name,d_year] #11
+                                      WholeStageCodegen (23)
+                                        HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum]
+                                          Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year]
+                                            SortMergeJoin [ws_bill_customer_sk,c_customer_sk]
+                                              InputAdapter
+                                                WholeStageCodegen (20)
+                                                  Sort [ws_bill_customer_sk]
+                                                    InputAdapter
+                                                      Exchange [ws_bill_customer_sk] #12
+                                                        WholeStageCodegen (19)
+                                                          Project [ws_bill_customer_sk,ws_net_paid,d_year]
+                                                            BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                              Filter [ws_bill_customer_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk]
+                                                                      ReusedSubquery [d_date_sk] #1
+                                                              InputAdapter
+                                                                ReusedExchange [d_date_sk,d_year] #4
+                                              InputAdapter
+                                                WholeStageCodegen (22)
+                                                  Sort [c_customer_sk]
+                                                    InputAdapter
+                                                      ReusedExchange [c_customer_sk,c_customer_id,c_first_name,c_last_name] #5
+        InputAdapter
+          WholeStageCodegen (34)
+            Sort [customer_id]
+              InputAdapter
+                Exchange [customer_id] #13
+                  WholeStageCodegen (33)
+                    HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum]
+                      InputAdapter
+                        Exchange [c_customer_id,c_first_name,c_last_name,d_year] #14
+                          WholeStageCodegen (32)
+                            HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum]
+                              Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year]
+                                SortMergeJoin [ws_bill_customer_sk,c_customer_sk]
+                                  InputAdapter
+                                    WholeStageCodegen (29)
+                                      Sort [ws_bill_customer_sk]
+                                        InputAdapter
+                                          Exchange [ws_bill_customer_sk] #15
+                                            WholeStageCodegen (28)
+                                              Project [ws_bill_customer_sk,ws_net_paid,d_year]
+                                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                  Filter [ws_bill_customer_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk]
+                                                          ReusedSubquery [d_date_sk] #2
+                                                  InputAdapter
+                                                    ReusedExchange [d_date_sk,d_year] #9
+                                  InputAdapter
+                                    WholeStageCodegen (31)
+                                      Sort [c_customer_sk]
+                                        InputAdapter
+                                          ReusedExchange [c_customer_sk,c_customer_id,c_first_name,c_last_name] #5
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/explain.txt
new file mode 100644
index 0000000000000..3ee6b56189888
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/explain.txt
@@ -0,0 +1,471 @@
+== Physical Plan ==
+TakeOrderedAndProject (78)
++- * Project (77)
+   +- * BroadcastHashJoin Inner BuildRight (76)
+      :- * Project (59)
+      :  +- * BroadcastHashJoin Inner BuildRight (58)
+      :     :- * BroadcastHashJoin Inner BuildRight (39)
+      :     :  :- * Filter (19)
+      :     :  :  +- * HashAggregate (18)
+      :     :  :     +- Exchange (17)
+      :     :  :        +- * HashAggregate (16)
+      :     :  :           +- * Project (15)
+      :     :  :              +- * BroadcastHashJoin Inner BuildRight (14)
+      :     :  :                 :- * Project (9)
+      :     :  :                 :  +- * BroadcastHashJoin Inner BuildRight (8)
+      :     :  :                 :     :- * Filter (3)
+      :     :  :                 :     :  +- * ColumnarToRow (2)
+      :     :  :                 :     :     +- Scan parquet default.customer (1)
+      :     :  :                 :     +- BroadcastExchange (7)
+      :     :  :                 :        +- * Filter (6)
+      :     :  :                 :           +- * ColumnarToRow (5)
+      :     :  :                 :              +- Scan parquet default.store_sales (4)
+      :     :  :                 +- BroadcastExchange (13)
+      :     :  :                    +- * Filter (12)
+      :     :  :                       +- * ColumnarToRow (11)
+      :     :  :                          +- Scan parquet default.date_dim (10)
+      :     :  +- BroadcastExchange (38)
+      :     :     +- * HashAggregate (37)
+      :     :        +- Exchange (36)
+      :     :           +- * HashAggregate (35)
+      :     :              +- * Project (34)
+      :     :                 +- * BroadcastHashJoin Inner BuildRight (33)
+      :     :                    :- * Project (28)
+      :     :                    :  +- * BroadcastHashJoin Inner BuildRight (27)
+      :     :                    :     :- * Filter (22)
+      :     :                    :     :  +- * ColumnarToRow (21)
+      :     :                    :     :     +- Scan parquet default.customer (20)
+      :     :                    :     +- BroadcastExchange (26)
+      :     :                    :        +- * Filter (25)
+      :     :                    :           +- * ColumnarToRow (24)
+      :     :                    :              +- Scan parquet default.store_sales (23)
+      :     :                    +- BroadcastExchange (32)
+      :     :                       +- * Filter (31)
+      :     :                          +- * ColumnarToRow (30)
+      :     :                             +- Scan parquet default.date_dim (29)
+      :     +- BroadcastExchange (57)
+      :        +- * Project (56)
+      :           +- * Filter (55)
+      :              +- * HashAggregate (54)
+      :                 +- Exchange (53)
+      :                    +- * HashAggregate (52)
+      :                       +- * Project (51)
+      :                          +- * BroadcastHashJoin Inner BuildRight (50)
+      :                             :- * Project (48)
+      :                             :  +- * BroadcastHashJoin Inner BuildRight (47)
+      :                             :     :- * Filter (42)
+      :                             :     :  +- * ColumnarToRow (41)
+      :                             :     :     +- Scan parquet default.customer (40)
+      :                             :     +- BroadcastExchange (46)
+      :                             :        +- * Filter (45)
+      :                             :           +- * ColumnarToRow (44)
+      :                             :              +- Scan parquet default.web_sales (43)
+      :                             +- ReusedExchange (49)
+      +- BroadcastExchange (75)
+         +- * HashAggregate (74)
+            +- Exchange (73)
+               +- * HashAggregate (72)
+                  +- * Project (71)
+                     +- * BroadcastHashJoin Inner BuildRight (70)
+                        :- * Project (68)
+                        :  +- * BroadcastHashJoin Inner BuildRight (67)
+                        :     :- * Filter (62)
+                        :     :  +- * ColumnarToRow (61)
+                        :     :     +- Scan parquet default.customer (60)
+                        :     +- BroadcastExchange (66)
+                        :        +- * Filter (65)
+                        :           +- * ColumnarToRow (64)
+                        :              +- Scan parquet default.web_sales (63)
+                        +- ReusedExchange (69)
+
+
+(1) Scan parquet default.customer
+Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4]
+
+(3) Filter [codegen id : 3]
+Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4]
+Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2))
+
+(4) Scan parquet default.store_sales
+Output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_net_paid:decimal(7,2)>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7]
+
+(6) Filter [codegen id : 1]
+Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7]
+Condition : isnotnull(ss_customer_sk#5)
+
+(7) BroadcastExchange
+Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#9]
+
+(8) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [c_customer_sk#1]
+Right keys [1]: [ss_customer_sk#5]
+Join condition: None
+
+(9) Project [codegen id : 3]
+Output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7]
+Input [7]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7]
+
+(10) Scan parquet default.date_dim
+Output [2]: [d_date_sk#10, d_year#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#10, d_year#11]
+
+(12) Filter [codegen id : 2]
+Input [2]: [d_date_sk#10, d_year#11]
+Condition : (((isnotnull(d_year#11) AND (d_year#11 = 2001)) AND d_year#11 IN (2001,2002)) AND isnotnull(d_date_sk#10))
+
+(13) BroadcastExchange
+Input [2]: [d_date_sk#10, d_year#11]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#12]
+
+(14) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#7]
+Right keys [1]: [d_date_sk#10]
+Join condition: None
+
+(15) Project [codegen id : 3]
+Output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#11]
+Input [7]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7, d_date_sk#10, d_year#11]
+
+(16) HashAggregate [codegen id : 3]
+Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#11]
+Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#11]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#6))]
+Aggregate Attributes [1]: [sum#13]
+Results [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#11, sum#14]
+
+(17) Exchange
+Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#11, sum#14]
+Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#11, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(18) HashAggregate [codegen id : 16]
+Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#11, sum#14]
+Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#11]
+Functions [1]: [sum(UnscaledValue(ss_net_paid#6))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#6))#16]
+Results [2]: [c_customer_id#2 AS customer_id#17, MakeDecimal(sum(UnscaledValue(ss_net_paid#6))#16,17,2) AS year_total#18]
+
+(19) Filter [codegen id : 16]
+Input [2]: [customer_id#17, year_total#18]
+Condition : (isnotnull(year_total#18) AND (year_total#18 > 0.00))
+
+(20) Scan parquet default.customer
+Output [4]: [c_customer_sk#19, c_customer_id#20, c_first_name#21, c_last_name#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string>
+
+(21) ColumnarToRow [codegen id : 6]
+Input [4]: [c_customer_sk#19, c_customer_id#20, c_first_name#21, c_last_name#22]
+
+(22) Filter [codegen id : 6]
+Input [4]: [c_customer_sk#19, c_customer_id#20, c_first_name#21, c_last_name#22]
+Condition : (isnotnull(c_customer_sk#19) AND isnotnull(c_customer_id#20))
+
+(23) Scan parquet default.store_sales
+Output [3]: [ss_customer_sk#23, ss_net_paid#24, ss_sold_date_sk#25]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#25), dynamicpruningexpression(ss_sold_date_sk#25 IN dynamicpruning#26)]
+PushedFilters: [IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_customer_sk:int,ss_net_paid:decimal(7,2)>
+
+(24) ColumnarToRow [codegen id : 4]
+Input [3]: [ss_customer_sk#23, ss_net_paid#24, ss_sold_date_sk#25]
+
+(25) Filter [codegen id : 4]
+Input [3]: [ss_customer_sk#23, ss_net_paid#24, ss_sold_date_sk#25]
+Condition : isnotnull(ss_customer_sk#23)
+
+(26) BroadcastExchange
+Input [3]: [ss_customer_sk#23, ss_net_paid#24, ss_sold_date_sk#25]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#27]
+
+(27) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [c_customer_sk#19]
+Right keys [1]: [ss_customer_sk#23]
+Join condition: None
+
+(28) Project [codegen id : 6]
+Output [5]: [c_customer_id#20, c_first_name#21, c_last_name#22, ss_net_paid#24, ss_sold_date_sk#25]
+Input [7]: [c_customer_sk#19, c_customer_id#20, c_first_name#21, c_last_name#22, ss_customer_sk#23, ss_net_paid#24, ss_sold_date_sk#25]
+
+(29) Scan parquet default.date_dim
+Output [2]: [d_date_sk#28, d_year#29]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(30) ColumnarToRow [codegen id : 5]
+Input [2]: [d_date_sk#28, d_year#29]
+
+(31) Filter [codegen id : 5]
+Input [2]: [d_date_sk#28, d_year#29]
+Condition : (((isnotnull(d_year#29) AND (d_year#29 = 2002)) AND d_year#29 IN (2001,2002)) AND isnotnull(d_date_sk#28))
+
+(32) BroadcastExchange
+Input [2]: [d_date_sk#28, d_year#29]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#30]
+
+(33) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_sold_date_sk#25]
+Right keys [1]: [d_date_sk#28]
+Join condition: None
+
+(34) Project [codegen id : 6]
+Output [5]: [c_customer_id#20, c_first_name#21, c_last_name#22, ss_net_paid#24, d_year#29]
+Input [7]: [c_customer_id#20, c_first_name#21, c_last_name#22, ss_net_paid#24, ss_sold_date_sk#25, d_date_sk#28, d_year#29]
+
+(35) HashAggregate [codegen id : 6]
+Input [5]: [c_customer_id#20, c_first_name#21, c_last_name#22, ss_net_paid#24, d_year#29]
+Keys [4]: [c_customer_id#20, c_first_name#21, c_last_name#22, d_year#29]
+Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#24))]
+Aggregate Attributes [1]: [sum#31]
+Results [5]: [c_customer_id#20, c_first_name#21, c_last_name#22, d_year#29, sum#32]
+
+(36) Exchange
+Input [5]: [c_customer_id#20, c_first_name#21, c_last_name#22, d_year#29, sum#32]
+Arguments: hashpartitioning(c_customer_id#20, c_first_name#21, c_last_name#22, d_year#29, 5), ENSURE_REQUIREMENTS, [id=#33]
+
+(37) HashAggregate [codegen id : 7]
+Input [5]: [c_customer_id#20, c_first_name#21, c_last_name#22, d_year#29, sum#32]
+Keys [4]: [c_customer_id#20, c_first_name#21, c_last_name#22, d_year#29]
+Functions [1]: [sum(UnscaledValue(ss_net_paid#24))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#24))#34]
+Results [4]: [c_customer_id#20 AS customer_id#35, c_first_name#21 AS customer_first_name#36, c_last_name#22 AS customer_last_name#37, MakeDecimal(sum(UnscaledValue(ss_net_paid#24))#34,17,2) AS year_total#38]
+
+(38) BroadcastExchange
+Input [4]: [customer_id#35, customer_first_name#36, customer_last_name#37, year_total#38]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#39]
+
+(39) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [customer_id#17]
+Right keys [1]: [customer_id#35]
+Join condition: None
+
+(40) Scan parquet default.customer
+Output [4]: [c_customer_sk#40, c_customer_id#41, c_first_name#42, c_last_name#43]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string>
+
+(41) ColumnarToRow [codegen id : 10]
+Input [4]: [c_customer_sk#40, c_customer_id#41, c_first_name#42, c_last_name#43]
+
+(42) Filter [codegen id : 10]
+Input [4]: [c_customer_sk#40, c_customer_id#41, c_first_name#42, c_last_name#43]
+Condition : (isnotnull(c_customer_sk#40) AND isnotnull(c_customer_id#41))
+
+(43) Scan parquet default.web_sales
+Output [3]: [ws_bill_customer_sk#44, ws_net_paid#45, ws_sold_date_sk#46]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#46), dynamicpruningexpression(ws_sold_date_sk#46 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int,ws_net_paid:decimal(7,2)>
+
+(44) ColumnarToRow [codegen id : 8]
+Input [3]: [ws_bill_customer_sk#44, ws_net_paid#45, ws_sold_date_sk#46]
+
+(45) Filter [codegen id : 8]
+Input [3]: [ws_bill_customer_sk#44, ws_net_paid#45, ws_sold_date_sk#46]
+Condition : isnotnull(ws_bill_customer_sk#44)
+
+(46) BroadcastExchange
+Input [3]: [ws_bill_customer_sk#44, ws_net_paid#45, ws_sold_date_sk#46]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#47]
+
+(47) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [c_customer_sk#40]
+Right keys [1]: [ws_bill_customer_sk#44]
+Join condition: None
+
+(48) Project [codegen id : 10]
+Output [5]: [c_customer_id#41, c_first_name#42, c_last_name#43, ws_net_paid#45, ws_sold_date_sk#46]
+Input [7]: [c_customer_sk#40, c_customer_id#41, c_first_name#42, c_last_name#43, ws_bill_customer_sk#44, ws_net_paid#45, ws_sold_date_sk#46]
+
+(49) ReusedExchange [Reuses operator id: 13]
+Output [2]: [d_date_sk#48, d_year#49]
+
+(50) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ws_sold_date_sk#46]
+Right keys [1]: [d_date_sk#48]
+Join condition: None
+
+(51) Project [codegen id : 10]
+Output [5]: [c_customer_id#41, c_first_name#42, c_last_name#43, ws_net_paid#45, d_year#49]
+Input [7]: [c_customer_id#41, c_first_name#42, c_last_name#43, ws_net_paid#45, ws_sold_date_sk#46, d_date_sk#48, d_year#49]
+
+(52) HashAggregate [codegen id : 10]
+Input [5]: [c_customer_id#41, c_first_name#42, c_last_name#43, ws_net_paid#45, d_year#49]
+Keys [4]: [c_customer_id#41, c_first_name#42, c_last_name#43, d_year#49]
+Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#45))]
+Aggregate Attributes [1]: [sum#50]
+Results [5]: [c_customer_id#41, c_first_name#42, c_last_name#43, d_year#49, sum#51]
+
+(53) Exchange
+Input [5]: [c_customer_id#41, c_first_name#42, c_last_name#43, d_year#49, sum#51]
+Arguments: hashpartitioning(c_customer_id#41, c_first_name#42, c_last_name#43, d_year#49, 5), ENSURE_REQUIREMENTS, [id=#52]
+
+(54) HashAggregate [codegen id : 11]
+Input [5]: [c_customer_id#41, c_first_name#42, c_last_name#43, d_year#49, sum#51]
+Keys [4]: [c_customer_id#41, c_first_name#42, c_last_name#43, d_year#49]
+Functions [1]: [sum(UnscaledValue(ws_net_paid#45))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#45))#53]
+Results [2]: [c_customer_id#41 AS customer_id#54, MakeDecimal(sum(UnscaledValue(ws_net_paid#45))#53,17,2) AS year_total#55]
+
+(55) Filter [codegen id : 11]
+Input [2]: [customer_id#54, year_total#55]
+Condition : (isnotnull(year_total#55) AND (year_total#55 > 0.00))
+
+(56) Project [codegen id : 11]
+Output [2]: [customer_id#54 AS customer_id#56, year_total#55 AS year_total#57]
+Input [2]: [customer_id#54, year_total#55]
+
+(57) BroadcastExchange
+Input [2]: [customer_id#56, year_total#57]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#58]
+
+(58) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [customer_id#17]
+Right keys [1]: [customer_id#56]
+Join condition: None
+
+(59) Project [codegen id : 16]
+Output [7]: [customer_id#17, year_total#18, customer_id#35, customer_first_name#36, customer_last_name#37, year_total#38, year_total#57]
+Input [8]: [customer_id#17, year_total#18, customer_id#35, customer_first_name#36, customer_last_name#37, year_total#38, customer_id#56, year_total#57]
+
+(60) Scan parquet default.customer
+Output [4]: [c_customer_sk#59, c_customer_id#60, c_first_name#61, c_last_name#62]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/customer]
+PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)]
+ReadSchema: struct<c_customer_sk:int,c_customer_id:string,c_first_name:string,c_last_name:string>
+
+(61) ColumnarToRow [codegen id : 14]
+Input [4]: [c_customer_sk#59, c_customer_id#60, c_first_name#61, c_last_name#62]
+
+(62) Filter [codegen id : 14]
+Input [4]: [c_customer_sk#59, c_customer_id#60, c_first_name#61, c_last_name#62]
+Condition : (isnotnull(c_customer_sk#59) AND isnotnull(c_customer_id#60))
+
+(63) Scan parquet default.web_sales
+Output [3]: [ws_bill_customer_sk#63, ws_net_paid#64, ws_sold_date_sk#65]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#65), dynamicpruningexpression(ws_sold_date_sk#65 IN dynamicpruning#26)]
+PushedFilters: [IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_bill_customer_sk:int,ws_net_paid:decimal(7,2)>
+
+(64) ColumnarToRow [codegen id : 12]
+Input [3]: [ws_bill_customer_sk#63, ws_net_paid#64, ws_sold_date_sk#65]
+
+(65) Filter [codegen id : 12]
+Input [3]: [ws_bill_customer_sk#63, ws_net_paid#64, ws_sold_date_sk#65]
+Condition : isnotnull(ws_bill_customer_sk#63)
+
+(66) BroadcastExchange
+Input [3]: [ws_bill_customer_sk#63, ws_net_paid#64, ws_sold_date_sk#65]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#66]
+
+(67) BroadcastHashJoin [codegen id : 14]
+Left keys [1]: [c_customer_sk#59]
+Right keys [1]: [ws_bill_customer_sk#63]
+Join condition: None
+
+(68) Project [codegen id : 14]
+Output [5]: [c_customer_id#60, c_first_name#61, c_last_name#62, ws_net_paid#64, ws_sold_date_sk#65]
+Input [7]: [c_customer_sk#59, c_customer_id#60, c_first_name#61, c_last_name#62, ws_bill_customer_sk#63, ws_net_paid#64, ws_sold_date_sk#65]
+
+(69) ReusedExchange [Reuses operator id: 32]
+Output [2]: [d_date_sk#67, d_year#68]
+
+(70) BroadcastHashJoin [codegen id : 14]
+Left keys [1]: [ws_sold_date_sk#65]
+Right keys [1]: [d_date_sk#67]
+Join condition: None
+
+(71) Project [codegen id : 14]
+Output [5]: [c_customer_id#60, c_first_name#61, c_last_name#62, ws_net_paid#64, d_year#68]
+Input [7]: [c_customer_id#60, c_first_name#61, c_last_name#62, ws_net_paid#64, ws_sold_date_sk#65, d_date_sk#67, d_year#68]
+
+(72) HashAggregate [codegen id : 14]
+Input [5]: [c_customer_id#60, c_first_name#61, c_last_name#62, ws_net_paid#64, d_year#68]
+Keys [4]: [c_customer_id#60, c_first_name#61, c_last_name#62, d_year#68]
+Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#64))]
+Aggregate Attributes [1]: [sum#69]
+Results [5]: [c_customer_id#60, c_first_name#61, c_last_name#62, d_year#68, sum#70]
+
+(73) Exchange
+Input [5]: [c_customer_id#60, c_first_name#61, c_last_name#62, d_year#68, sum#70]
+Arguments: hashpartitioning(c_customer_id#60, c_first_name#61, c_last_name#62, d_year#68, 5), ENSURE_REQUIREMENTS, [id=#71]
+
+(74) HashAggregate [codegen id : 15]
+Input [5]: [c_customer_id#60, c_first_name#61, c_last_name#62, d_year#68, sum#70]
+Keys [4]: [c_customer_id#60, c_first_name#61, c_last_name#62, d_year#68]
+Functions [1]: [sum(UnscaledValue(ws_net_paid#64))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#64))#72]
+Results [2]: [c_customer_id#60 AS customer_id#73, MakeDecimal(sum(UnscaledValue(ws_net_paid#64))#72,17,2) AS year_total#74]
+
+(75) BroadcastExchange
+Input [2]: [customer_id#73, year_total#74]
+Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [id=#75]
+
+(76) BroadcastHashJoin [codegen id : 16]
+Left keys [1]: [customer_id#17]
+Right keys [1]: [customer_id#73]
+Join condition: (CASE WHEN (year_total#57 > 0.00) THEN CheckOverflow((promote_precision(year_total#74) / promote_precision(year_total#57)), DecimalType(37,20), true) ELSE null END > CASE WHEN (year_total#18 > 0.00) THEN CheckOverflow((promote_precision(year_total#38) / promote_precision(year_total#18)), DecimalType(37,20), true) ELSE null END)
+
+(77) Project [codegen id : 16]
+Output [3]: [customer_id#35, customer_first_name#36, customer_last_name#37]
+Input [9]: [customer_id#17, year_total#18, customer_id#35, customer_first_name#36, customer_last_name#37, year_total#38, year_total#57, customer_id#73, year_total#74]
+
+(78) TakeOrderedAndProject
+Input [3]: [customer_id#35, customer_first_name#36, customer_last_name#37]
+Arguments: 100, [customer_first_name#36 ASC NULLS FIRST, customer_id#35 ASC NULLS FIRST, customer_last_name#37 ASC NULLS FIRST], [customer_id#35, customer_first_name#36, customer_last_name#37]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (79)
+
+
+(79) ReusedExchange [Reuses operator id: 13]
+Output [2]: [d_date_sk#10, d_year#11]
+
+Subquery:2 Hosting operator id = 23 Hosting Expression = ss_sold_date_sk#25 IN dynamicpruning#26
+ReusedExchange (80)
+
+
+(80) ReusedExchange [Reuses operator id: 32]
+Output [2]: [d_date_sk#28, d_year#29]
+
+Subquery:3 Hosting operator id = 43 Hosting Expression = ws_sold_date_sk#46 IN dynamicpruning#8
+
+Subquery:4 Hosting operator id = 63 Hosting Expression = ws_sold_date_sk#65 IN dynamicpruning#26
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/simplified.txt
new file mode 100644
index 0000000000000..1304bdd334213
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/simplified.txt
@@ -0,0 +1,123 @@
+TakeOrderedAndProject [customer_first_name,customer_id,customer_last_name]
+  WholeStageCodegen (16)
+    Project [customer_id,customer_first_name,customer_last_name]
+      BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total]
+        Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total]
+          BroadcastHashJoin [customer_id,customer_id]
+            BroadcastHashJoin [customer_id,customer_id]
+              Filter [year_total]
+                HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,year_total,sum]
+                  InputAdapter
+                    Exchange [c_customer_id,c_first_name,c_last_name,d_year] #1
+                      WholeStageCodegen (3)
+                        HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum]
+                          Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year]
+                            BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                              Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk]
+                                BroadcastHashJoin [c_customer_sk,ss_customer_sk]
+                                  Filter [c_customer_sk,c_customer_id]
+                                    ColumnarToRow
+                                      InputAdapter
+                                        Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name]
+                                  InputAdapter
+                                    BroadcastExchange #2
+                                      WholeStageCodegen (1)
+                                        Filter [ss_customer_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk]
+                                                SubqueryBroadcast [d_date_sk] #1
+                                                  ReusedExchange [d_date_sk,d_year] #3
+                              InputAdapter
+                                BroadcastExchange #3
+                                  WholeStageCodegen (2)
+                                    Filter [d_year,d_date_sk]
+                                      ColumnarToRow
+                                        InputAdapter
+                                          Scan parquet default.date_dim [d_date_sk,d_year]
+              InputAdapter
+                BroadcastExchange #4
+                  WholeStageCodegen (7)
+                    HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,customer_first_name,customer_last_name,year_total,sum]
+                      InputAdapter
+                        Exchange [c_customer_id,c_first_name,c_last_name,d_year] #5
+                          WholeStageCodegen (6)
+                            HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum]
+                              Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year]
+                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                  Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk]
+                                    BroadcastHashJoin [c_customer_sk,ss_customer_sk]
+                                      Filter [c_customer_sk,c_customer_id]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name]
+                                      InputAdapter
+                                        BroadcastExchange #6
+                                          WholeStageCodegen (4)
+                                            Filter [ss_customer_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #2
+                                                      ReusedExchange [d_date_sk,d_year] #7
+                                  InputAdapter
+                                    BroadcastExchange #7
+                                      WholeStageCodegen (5)
+                                        Filter [d_year,d_date_sk]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.date_dim [d_date_sk,d_year]
+            InputAdapter
+              BroadcastExchange #8
+                WholeStageCodegen (11)
+                  Project [customer_id,year_total]
+                    Filter [year_total]
+                      HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum]
+                        InputAdapter
+                          Exchange [c_customer_id,c_first_name,c_last_name,d_year] #9
+                            WholeStageCodegen (10)
+                              HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum]
+                                Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year]
+                                  BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                    Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk]
+                                      BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk]
+                                        Filter [c_customer_sk,c_customer_id]
+                                          ColumnarToRow
+                                            InputAdapter
+                                              Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name]
+                                        InputAdapter
+                                          BroadcastExchange #10
+                                            WholeStageCodegen (8)
+                                              Filter [ws_bill_customer_sk]
+                                                ColumnarToRow
+                                                  InputAdapter
+                                                    Scan parquet default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk]
+                                                      ReusedSubquery [d_date_sk] #1
+                                    InputAdapter
+                                      ReusedExchange [d_date_sk,d_year] #3
+        InputAdapter
+          BroadcastExchange #11
+            WholeStageCodegen (15)
+              HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum]
+                InputAdapter
+                  Exchange [c_customer_id,c_first_name,c_last_name,d_year] #12
+                    WholeStageCodegen (14)
+                      HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum]
+                        Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year]
+                          BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                            Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk]
+                              BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk]
+                                Filter [c_customer_sk,c_customer_id]
+                                  ColumnarToRow
+                                    InputAdapter
+                                      Scan parquet default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name]
+                                InputAdapter
+                                  BroadcastExchange #13
+                                    WholeStageCodegen (12)
+                                      Filter [ws_bill_customer_sk]
+                                        ColumnarToRow
+                                          InputAdapter
+                                            Scan parquet default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk]
+                                              ReusedSubquery [d_date_sk] #2
+                            InputAdapter
+                              ReusedExchange [d_date_sk,d_year] #7
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.sf100/explain.txt
new file mode 100644
index 0000000000000..eb5287bfa8850
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.sf100/explain.txt
@@ -0,0 +1,750 @@
+== Physical Plan ==
+TakeOrderedAndProject (133)
++- * Project (132)
+   +- * SortMergeJoin Inner (131)
+      :- * Sort (73)
+      :  +- Exchange (72)
+      :     +- * HashAggregate (71)
+      :        +- Exchange (70)
+      :           +- * HashAggregate (69)
+      :              +- * HashAggregate (68)
+      :                 +- Exchange (67)
+      :                    +- * HashAggregate (66)
+      :                       +- Union (65)
+      :                          :- * Project (26)
+      :                          :  +- * SortMergeJoin LeftOuter (25)
+      :                          :     :- * Sort (18)
+      :                          :     :  +- Exchange (17)
+      :                          :     :     +- * Project (16)
+      :                          :     :        +- * BroadcastHashJoin Inner BuildRight (15)
+      :                          :     :           :- * Project (10)
+      :                          :     :           :  +- * BroadcastHashJoin Inner BuildRight (9)
+      :                          :     :           :     :- * Filter (3)
+      :                          :     :           :     :  +- * ColumnarToRow (2)
+      :                          :     :           :     :     +- Scan parquet default.catalog_sales (1)
+      :                          :     :           :     +- BroadcastExchange (8)
+      :                          :     :           :        +- * Project (7)
+      :                          :     :           :           +- * Filter (6)
+      :                          :     :           :              +- * ColumnarToRow (5)
+      :                          :     :           :                 +- Scan parquet default.item (4)
+      :                          :     :           +- BroadcastExchange (14)
+      :                          :     :              +- * Filter (13)
+      :                          :     :                 +- * ColumnarToRow (12)
+      :                          :     :                    +- Scan parquet default.date_dim (11)
+      :                          :     +- * Sort (24)
+      :                          :        +- Exchange (23)
+      :                          :           +- * Project (22)
+      :                          :              +- * Filter (21)
+      :                          :                 +- * ColumnarToRow (20)
+      :                          :                    +- Scan parquet default.catalog_returns (19)
+      :                          :- * Project (45)
+      :                          :  +- * SortMergeJoin LeftOuter (44)
+      :                          :     :- * Sort (37)
+      :                          :     :  +- Exchange (36)
+      :                          :     :     +- * Project (35)
+      :                          :     :        +- * BroadcastHashJoin Inner BuildRight (34)
+      :                          :     :           :- * Project (32)
+      :                          :     :           :  +- * BroadcastHashJoin Inner BuildRight (31)
+      :                          :     :           :     :- * Filter (29)
+      :                          :     :           :     :  +- * ColumnarToRow (28)
+      :                          :     :           :     :     +- Scan parquet default.store_sales (27)
+      :                          :     :           :     +- ReusedExchange (30)
+      :                          :     :           +- ReusedExchange (33)
+      :                          :     +- * Sort (43)
+      :                          :        +- Exchange (42)
+      :                          :           +- * Project (41)
+      :                          :              +- * Filter (40)
+      :                          :                 +- * ColumnarToRow (39)
+      :                          :                    +- Scan parquet default.store_returns (38)
+      :                          +- * Project (64)
+      :                             +- * SortMergeJoin LeftOuter (63)
+      :                                :- * Sort (56)
+      :                                :  +- Exchange (55)
+      :                                :     +- * Project (54)
+      :                                :        +- * BroadcastHashJoin Inner BuildRight (53)
+      :                                :           :- * Project (51)
+      :                                :           :  +- * BroadcastHashJoin Inner BuildRight (50)
+      :                                :           :     :- * Filter (48)
+      :                                :           :     :  +- * ColumnarToRow (47)
+      :                                :           :     :     +- Scan parquet default.web_sales (46)
+      :                                :           :     +- ReusedExchange (49)
+      :                                :           +- ReusedExchange (52)
+      :                                +- * Sort (62)
+      :                                   +- Exchange (61)
+      :                                      +- * Project (60)
+      :                                         +- * Filter (59)
+      :                                            +- * ColumnarToRow (58)
+      :                                               +- Scan parquet default.web_returns (57)
+      +- * Sort (130)
+         +- Exchange (129)
+            +- * HashAggregate (128)
+               +- Exchange (127)
+                  +- * HashAggregate (126)
+                     +- * HashAggregate (125)
+                        +- Exchange (124)
+                           +- * HashAggregate (123)
+                              +- Union (122)
+                                 :- * Project (91)
+                                 :  +- * SortMergeJoin LeftOuter (90)
+                                 :     :- * Sort (87)
+                                 :     :  +- Exchange (86)
+                                 :     :     +- * Project (85)
+                                 :     :        +- * BroadcastHashJoin Inner BuildRight (84)
+                                 :     :           :- * Project (79)
+                                 :     :           :  +- * BroadcastHashJoin Inner BuildRight (78)
+                                 :     :           :     :- * Filter (76)
+                                 :     :           :     :  +- * ColumnarToRow (75)
+                                 :     :           :     :     +- Scan parquet default.catalog_sales (74)
+                                 :     :           :     +- ReusedExchange (77)
+                                 :     :           +- BroadcastExchange (83)
+                                 :     :              +- * Filter (82)
+                                 :     :                 +- * ColumnarToRow (81)
+                                 :     :                    +- Scan parquet default.date_dim (80)
+                                 :     +- * Sort (89)
+                                 :        +- ReusedExchange (88)
+                                 :- * Project (106)
+                                 :  +- * SortMergeJoin LeftOuter (105)
+                                 :     :- * Sort (102)
+                                 :     :  +- Exchange (101)
+                                 :     :     +- * Project (100)
+                                 :     :        +- * BroadcastHashJoin Inner BuildRight (99)
+                                 :     :           :- * Project (97)
+                                 :     :           :  +- * BroadcastHashJoin Inner BuildRight (96)
+                                 :     :           :     :- * Filter (94)
+                                 :     :           :     :  +- * ColumnarToRow (93)
+                                 :     :           :     :     +- Scan parquet default.store_sales (92)
+                                 :     :           :     +- ReusedExchange (95)
+                                 :     :           +- ReusedExchange (98)
+                                 :     +- * Sort (104)
+                                 :        +- ReusedExchange (103)
+                                 +- * Project (121)
+                                    +- * SortMergeJoin LeftOuter (120)
+                                       :- * Sort (117)
+                                       :  +- Exchange (116)
+                                       :     +- * Project (115)
+                                       :        +- * BroadcastHashJoin Inner BuildRight (114)
+                                       :           :- * Project (112)
+                                       :           :  +- * BroadcastHashJoin Inner BuildRight (111)
+                                       :           :     :- * Filter (109)
+                                       :           :     :  +- * ColumnarToRow (108)
+                                       :           :     :     +- Scan parquet default.web_sales (107)
+                                       :           :     +- ReusedExchange (110)
+                                       :           +- ReusedExchange (113)
+                                       +- * Sort (119)
+                                          +- ReusedExchange (118)
+
+
+(1) Scan parquet default.catalog_sales
+Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int,cs_order_number:int,cs_quantity:int,cs_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5]
+
+(3) Filter [codegen id : 3]
+Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5]
+Condition : isnotnull(cs_item_sk#1)
+
+(4) Scan parquet default.item
+Output [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Books                                             ), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int,i_category:string,i_manufact_id:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12]
+
+(6) Filter [codegen id : 1]
+Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12]
+Condition : ((((((isnotnull(i_category#11) AND (i_category#11 = Books                                             )) AND isnotnull(i_item_sk#7)) AND isnotnull(i_brand_id#8)) AND isnotnull(i_class_id#9)) AND isnotnull(i_category_id#10)) AND isnotnull(i_manufact_id#12))
+
+(7) Project [codegen id : 1]
+Output [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12]
+
+(8) BroadcastExchange
+Input [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_item_sk#1]
+Right keys [1]: [i_item_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+
+(11) Scan parquet default.date_dim
+Output [2]: [d_date_sk#14, d_year#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#14, d_year#15]
+
+(13) Filter [codegen id : 2]
+Input [2]: [d_date_sk#14, d_year#15]
+Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2002)) AND isnotnull(d_date_sk#14))
+
+(14) BroadcastExchange
+Input [2]: [d_date_sk#14, d_year#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#16]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_sold_date_sk#5]
+Right keys [1]: [d_date_sk#14]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#15]
+Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_date_sk#14, d_year#15]
+
+(17) Exchange
+Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#15]
+Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(18) Sort [codegen id : 4]
+Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#15]
+Arguments: [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST], false, 0
+
+(19) Scan parquet default.catalog_returns
+Output [5]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21, cr_returned_date_sk#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int,cr_return_quantity:int,cr_return_amount:decimal(7,2)>
+
+(20) ColumnarToRow [codegen id : 5]
+Input [5]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21, cr_returned_date_sk#22]
+
+(21) Filter [codegen id : 5]
+Input [5]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21, cr_returned_date_sk#22]
+Condition : (isnotnull(cr_order_number#19) AND isnotnull(cr_item_sk#18))
+
+(22) Project [codegen id : 5]
+Output [4]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21]
+Input [5]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21, cr_returned_date_sk#22]
+
+(23) Exchange
+Input [4]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21]
+Arguments: hashpartitioning(cr_order_number#19, cr_item_sk#18, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(24) Sort [codegen id : 6]
+Input [4]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21]
+Arguments: [cr_order_number#19 ASC NULLS FIRST, cr_item_sk#18 ASC NULLS FIRST], false, 0
+
+(25) SortMergeJoin [codegen id : 7]
+Left keys [2]: [cs_order_number#2, cs_item_sk#1]
+Right keys [2]: [cr_order_number#19, cr_item_sk#18]
+Join condition: None
+
+(26) Project [codegen id : 7]
+Output [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, (cs_quantity#3 - coalesce(cr_return_quantity#20, 0)) AS sales_cnt#24, CheckOverflow((promote_precision(cast(cs_ext_sales_price#4 as decimal(8,2))) - promote_precision(cast(coalesce(cr_return_amount#21, 0.00) as decimal(8,2)))), DecimalType(8,2), true) AS sales_amt#25]
+Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#15, cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21]
+
+(27) Scan parquet default.store_sales
+Output [5]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, ss_sold_date_sk#30]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#30), dynamicpruningexpression(ss_sold_date_sk#30 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ticket_number:int,ss_quantity:int,ss_ext_sales_price:decimal(7,2)>
+
+(28) ColumnarToRow [codegen id : 10]
+Input [5]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, ss_sold_date_sk#30]
+
+(29) Filter [codegen id : 10]
+Input [5]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, ss_sold_date_sk#30]
+Condition : isnotnull(ss_item_sk#26)
+
+(30) ReusedExchange [Reuses operator id: 8]
+Output [5]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35]
+
+(31) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_item_sk#26]
+Right keys [1]: [i_item_sk#31]
+Join condition: None
+
+(32) Project [codegen id : 10]
+Output [9]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, ss_sold_date_sk#30, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35]
+Input [10]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, ss_sold_date_sk#30, i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35]
+
+(33) ReusedExchange [Reuses operator id: 14]
+Output [2]: [d_date_sk#36, d_year#37]
+
+(34) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_sold_date_sk#30]
+Right keys [1]: [d_date_sk#36]
+Join condition: None
+
+(35) Project [codegen id : 10]
+Output [9]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35, d_year#37]
+Input [11]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, ss_sold_date_sk#30, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35, d_date_sk#36, d_year#37]
+
+(36) Exchange
+Input [9]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35, d_year#37]
+Arguments: hashpartitioning(ss_ticket_number#27, ss_item_sk#26, 5), ENSURE_REQUIREMENTS, [id=#38]
+
+(37) Sort [codegen id : 11]
+Input [9]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35, d_year#37]
+Arguments: [ss_ticket_number#27 ASC NULLS FIRST, ss_item_sk#26 ASC NULLS FIRST], false, 0
+
+(38) Scan parquet default.store_returns
+Output [5]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42, sr_returned_date_sk#43]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int,sr_return_quantity:int,sr_return_amt:decimal(7,2)>
+
+(39) ColumnarToRow [codegen id : 12]
+Input [5]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42, sr_returned_date_sk#43]
+
+(40) Filter [codegen id : 12]
+Input [5]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42, sr_returned_date_sk#43]
+Condition : (isnotnull(sr_ticket_number#40) AND isnotnull(sr_item_sk#39))
+
+(41) Project [codegen id : 12]
+Output [4]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42]
+Input [5]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42, sr_returned_date_sk#43]
+
+(42) Exchange
+Input [4]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42]
+Arguments: hashpartitioning(sr_ticket_number#40, sr_item_sk#39, 5), ENSURE_REQUIREMENTS, [id=#44]
+
+(43) Sort [codegen id : 13]
+Input [4]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42]
+Arguments: [sr_ticket_number#40 ASC NULLS FIRST, sr_item_sk#39 ASC NULLS FIRST], false, 0
+
+(44) SortMergeJoin [codegen id : 14]
+Left keys [2]: [ss_ticket_number#27, ss_item_sk#26]
+Right keys [2]: [sr_ticket_number#40, sr_item_sk#39]
+Join condition: None
+
+(45) Project [codegen id : 14]
+Output [7]: [d_year#37, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35, (ss_quantity#28 - coalesce(sr_return_quantity#41, 0)) AS sales_cnt#45, CheckOverflow((promote_precision(cast(ss_ext_sales_price#29 as decimal(8,2))) - promote_precision(cast(coalesce(sr_return_amt#42, 0.00) as decimal(8,2)))), DecimalType(8,2), true) AS sales_amt#46]
+Input [13]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35, d_year#37, sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42]
+
+(46) Scan parquet default.web_sales
+Output [5]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, ws_sold_date_sk#51]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#51), dynamicpruningexpression(ws_sold_date_sk#51 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_order_number:int,ws_quantity:int,ws_ext_sales_price:decimal(7,2)>
+
+(47) ColumnarToRow [codegen id : 17]
+Input [5]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, ws_sold_date_sk#51]
+
+(48) Filter [codegen id : 17]
+Input [5]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, ws_sold_date_sk#51]
+Condition : isnotnull(ws_item_sk#47)
+
+(49) ReusedExchange [Reuses operator id: 8]
+Output [5]: [i_item_sk#52, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56]
+
+(50) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_item_sk#47]
+Right keys [1]: [i_item_sk#52]
+Join condition: None
+
+(51) Project [codegen id : 17]
+Output [9]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, ws_sold_date_sk#51, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56]
+Input [10]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, ws_sold_date_sk#51, i_item_sk#52, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56]
+
+(52) ReusedExchange [Reuses operator id: 14]
+Output [2]: [d_date_sk#57, d_year#58]
+
+(53) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_sold_date_sk#51]
+Right keys [1]: [d_date_sk#57]
+Join condition: None
+
+(54) Project [codegen id : 17]
+Output [9]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56, d_year#58]
+Input [11]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, ws_sold_date_sk#51, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56, d_date_sk#57, d_year#58]
+
+(55) Exchange
+Input [9]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56, d_year#58]
+Arguments: hashpartitioning(ws_order_number#48, ws_item_sk#47, 5), ENSURE_REQUIREMENTS, [id=#59]
+
+(56) Sort [codegen id : 18]
+Input [9]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56, d_year#58]
+Arguments: [ws_order_number#48 ASC NULLS FIRST, ws_item_sk#47 ASC NULLS FIRST], false, 0
+
+(57) Scan parquet default.web_returns
+Output [5]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63, wr_returned_date_sk#64]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_returns]
+PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)]
+ReadSchema: struct<wr_item_sk:int,wr_order_number:int,wr_return_quantity:int,wr_return_amt:decimal(7,2)>
+
+(58) ColumnarToRow [codegen id : 19]
+Input [5]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63, wr_returned_date_sk#64]
+
+(59) Filter [codegen id : 19]
+Input [5]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63, wr_returned_date_sk#64]
+Condition : (isnotnull(wr_order_number#61) AND isnotnull(wr_item_sk#60))
+
+(60) Project [codegen id : 19]
+Output [4]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63]
+Input [5]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63, wr_returned_date_sk#64]
+
+(61) Exchange
+Input [4]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63]
+Arguments: hashpartitioning(wr_order_number#61, wr_item_sk#60, 5), ENSURE_REQUIREMENTS, [id=#65]
+
+(62) Sort [codegen id : 20]
+Input [4]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63]
+Arguments: [wr_order_number#61 ASC NULLS FIRST, wr_item_sk#60 ASC NULLS FIRST], false, 0
+
+(63) SortMergeJoin [codegen id : 21]
+Left keys [2]: [ws_order_number#48, ws_item_sk#47]
+Right keys [2]: [wr_order_number#61, wr_item_sk#60]
+Join condition: None
+
+(64) Project [codegen id : 21]
+Output [7]: [d_year#58, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56, (ws_quantity#49 - coalesce(wr_return_quantity#62, 0)) AS sales_cnt#66, CheckOverflow((promote_precision(cast(ws_ext_sales_price#50 as decimal(8,2))) - promote_precision(cast(coalesce(wr_return_amt#63, 0.00) as decimal(8,2)))), DecimalType(8,2), true) AS sales_amt#67]
+Input [13]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56, d_year#58, wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63]
+
+(65) Union
+
+(66) HashAggregate [codegen id : 22]
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+Keys [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+Functions: []
+Aggregate Attributes: []
+Results [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+
+(67) Exchange
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+Arguments: hashpartitioning(d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25, 5), ENSURE_REQUIREMENTS, [id=#68]
+
+(68) HashAggregate [codegen id : 23]
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+Keys [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+Functions: []
+Aggregate Attributes: []
+Results [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+
+(69) HashAggregate [codegen id : 23]
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+Keys [5]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+Functions [2]: [partial_sum(sales_cnt#24), partial_sum(UnscaledValue(sales_amt#25))]
+Aggregate Attributes [2]: [sum#69, sum#70]
+Results [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#71, sum#72]
+
+(70) Exchange
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#71, sum#72]
+Arguments: hashpartitioning(d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [id=#73]
+
+(71) HashAggregate [codegen id : 24]
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#71, sum#72]
+Keys [5]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+Functions [2]: [sum(sales_cnt#24), sum(UnscaledValue(sales_amt#25))]
+Aggregate Attributes [2]: [sum(sales_cnt#24)#74, sum(UnscaledValue(sales_amt#25))#75]
+Results [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum(sales_cnt#24)#74 AS sales_cnt#76, MakeDecimal(sum(UnscaledValue(sales_amt#25))#75,18,2) AS sales_amt#77]
+
+(72) Exchange
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#76, sales_amt#77]
+Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [id=#78]
+
+(73) Sort [codegen id : 25]
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#76, sales_amt#77]
+Arguments: [i_brand_id#8 ASC NULLS FIRST, i_class_id#9 ASC NULLS FIRST, i_category_id#10 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST], false, 0
+
+(74) Scan parquet default.catalog_sales
+Output [5]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, cs_sold_date_sk#83]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#83), dynamicpruningexpression(cs_sold_date_sk#83 IN dynamicpruning#84)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int,cs_order_number:int,cs_quantity:int,cs_ext_sales_price:decimal(7,2)>
+
+(75) ColumnarToRow [codegen id : 28]
+Input [5]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, cs_sold_date_sk#83]
+
+(76) Filter [codegen id : 28]
+Input [5]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, cs_sold_date_sk#83]
+Condition : isnotnull(cs_item_sk#79)
+
+(77) ReusedExchange [Reuses operator id: 8]
+Output [5]: [i_item_sk#85, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89]
+
+(78) BroadcastHashJoin [codegen id : 28]
+Left keys [1]: [cs_item_sk#79]
+Right keys [1]: [i_item_sk#85]
+Join condition: None
+
+(79) Project [codegen id : 28]
+Output [9]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, cs_sold_date_sk#83, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89]
+Input [10]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, cs_sold_date_sk#83, i_item_sk#85, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89]
+
+(80) Scan parquet default.date_dim
+Output [2]: [d_date_sk#90, d_year#91]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(81) ColumnarToRow [codegen id : 27]
+Input [2]: [d_date_sk#90, d_year#91]
+
+(82) Filter [codegen id : 27]
+Input [2]: [d_date_sk#90, d_year#91]
+Condition : ((isnotnull(d_year#91) AND (d_year#91 = 2001)) AND isnotnull(d_date_sk#90))
+
+(83) BroadcastExchange
+Input [2]: [d_date_sk#90, d_year#91]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#92]
+
+(84) BroadcastHashJoin [codegen id : 28]
+Left keys [1]: [cs_sold_date_sk#83]
+Right keys [1]: [d_date_sk#90]
+Join condition: None
+
+(85) Project [codegen id : 28]
+Output [9]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, d_year#91]
+Input [11]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, cs_sold_date_sk#83, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, d_date_sk#90, d_year#91]
+
+(86) Exchange
+Input [9]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, d_year#91]
+Arguments: hashpartitioning(cs_order_number#80, cs_item_sk#79, 5), ENSURE_REQUIREMENTS, [id=#93]
+
+(87) Sort [codegen id : 29]
+Input [9]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, d_year#91]
+Arguments: [cs_order_number#80 ASC NULLS FIRST, cs_item_sk#79 ASC NULLS FIRST], false, 0
+
+(88) ReusedExchange [Reuses operator id: 23]
+Output [4]: [cr_item_sk#94, cr_order_number#95, cr_return_quantity#96, cr_return_amount#97]
+
+(89) Sort [codegen id : 31]
+Input [4]: [cr_item_sk#94, cr_order_number#95, cr_return_quantity#96, cr_return_amount#97]
+Arguments: [cr_order_number#95 ASC NULLS FIRST, cr_item_sk#94 ASC NULLS FIRST], false, 0
+
+(90) SortMergeJoin [codegen id : 32]
+Left keys [2]: [cs_order_number#80, cs_item_sk#79]
+Right keys [2]: [cr_order_number#95, cr_item_sk#94]
+Join condition: None
+
+(91) Project [codegen id : 32]
+Output [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, (cs_quantity#81 - coalesce(cr_return_quantity#96, 0)) AS sales_cnt#24, CheckOverflow((promote_precision(cast(cs_ext_sales_price#82 as decimal(8,2))) - promote_precision(cast(coalesce(cr_return_amount#97, 0.00) as decimal(8,2)))), DecimalType(8,2), true) AS sales_amt#25]
+Input [13]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, d_year#91, cr_item_sk#94, cr_order_number#95, cr_return_quantity#96, cr_return_amount#97]
+
+(92) Scan parquet default.store_sales
+Output [5]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, ss_sold_date_sk#102]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#102), dynamicpruningexpression(ss_sold_date_sk#102 IN dynamicpruning#84)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ticket_number:int,ss_quantity:int,ss_ext_sales_price:decimal(7,2)>
+
+(93) ColumnarToRow [codegen id : 35]
+Input [5]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, ss_sold_date_sk#102]
+
+(94) Filter [codegen id : 35]
+Input [5]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, ss_sold_date_sk#102]
+Condition : isnotnull(ss_item_sk#98)
+
+(95) ReusedExchange [Reuses operator id: 8]
+Output [5]: [i_item_sk#103, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107]
+
+(96) BroadcastHashJoin [codegen id : 35]
+Left keys [1]: [ss_item_sk#98]
+Right keys [1]: [i_item_sk#103]
+Join condition: None
+
+(97) Project [codegen id : 35]
+Output [9]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, ss_sold_date_sk#102, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107]
+Input [10]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, ss_sold_date_sk#102, i_item_sk#103, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107]
+
+(98) ReusedExchange [Reuses operator id: 83]
+Output [2]: [d_date_sk#108, d_year#109]
+
+(99) BroadcastHashJoin [codegen id : 35]
+Left keys [1]: [ss_sold_date_sk#102]
+Right keys [1]: [d_date_sk#108]
+Join condition: None
+
+(100) Project [codegen id : 35]
+Output [9]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107, d_year#109]
+Input [11]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, ss_sold_date_sk#102, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107, d_date_sk#108, d_year#109]
+
+(101) Exchange
+Input [9]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107, d_year#109]
+Arguments: hashpartitioning(ss_ticket_number#99, ss_item_sk#98, 5), ENSURE_REQUIREMENTS, [id=#110]
+
+(102) Sort [codegen id : 36]
+Input [9]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107, d_year#109]
+Arguments: [ss_ticket_number#99 ASC NULLS FIRST, ss_item_sk#98 ASC NULLS FIRST], false, 0
+
+(103) ReusedExchange [Reuses operator id: 42]
+Output [4]: [sr_item_sk#111, sr_ticket_number#112, sr_return_quantity#113, sr_return_amt#114]
+
+(104) Sort [codegen id : 38]
+Input [4]: [sr_item_sk#111, sr_ticket_number#112, sr_return_quantity#113, sr_return_amt#114]
+Arguments: [sr_ticket_number#112 ASC NULLS FIRST, sr_item_sk#111 ASC NULLS FIRST], false, 0
+
+(105) SortMergeJoin [codegen id : 39]
+Left keys [2]: [ss_ticket_number#99, ss_item_sk#98]
+Right keys [2]: [sr_ticket_number#112, sr_item_sk#111]
+Join condition: None
+
+(106) Project [codegen id : 39]
+Output [7]: [d_year#109, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107, (ss_quantity#100 - coalesce(sr_return_quantity#113, 0)) AS sales_cnt#45, CheckOverflow((promote_precision(cast(ss_ext_sales_price#101 as decimal(8,2))) - promote_precision(cast(coalesce(sr_return_amt#114, 0.00) as decimal(8,2)))), DecimalType(8,2), true) AS sales_amt#46]
+Input [13]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107, d_year#109, sr_item_sk#111, sr_ticket_number#112, sr_return_quantity#113, sr_return_amt#114]
+
+(107) Scan parquet default.web_sales
+Output [5]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, ws_sold_date_sk#119]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#119), dynamicpruningexpression(ws_sold_date_sk#119 IN dynamicpruning#84)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_order_number:int,ws_quantity:int,ws_ext_sales_price:decimal(7,2)>
+
+(108) ColumnarToRow [codegen id : 42]
+Input [5]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, ws_sold_date_sk#119]
+
+(109) Filter [codegen id : 42]
+Input [5]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, ws_sold_date_sk#119]
+Condition : isnotnull(ws_item_sk#115)
+
+(110) ReusedExchange [Reuses operator id: 8]
+Output [5]: [i_item_sk#120, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124]
+
+(111) BroadcastHashJoin [codegen id : 42]
+Left keys [1]: [ws_item_sk#115]
+Right keys [1]: [i_item_sk#120]
+Join condition: None
+
+(112) Project [codegen id : 42]
+Output [9]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, ws_sold_date_sk#119, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124]
+Input [10]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, ws_sold_date_sk#119, i_item_sk#120, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124]
+
+(113) ReusedExchange [Reuses operator id: 83]
+Output [2]: [d_date_sk#125, d_year#126]
+
+(114) BroadcastHashJoin [codegen id : 42]
+Left keys [1]: [ws_sold_date_sk#119]
+Right keys [1]: [d_date_sk#125]
+Join condition: None
+
+(115) Project [codegen id : 42]
+Output [9]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124, d_year#126]
+Input [11]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, ws_sold_date_sk#119, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124, d_date_sk#125, d_year#126]
+
+(116) Exchange
+Input [9]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124, d_year#126]
+Arguments: hashpartitioning(ws_order_number#116, ws_item_sk#115, 5), ENSURE_REQUIREMENTS, [id=#127]
+
+(117) Sort [codegen id : 43]
+Input [9]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124, d_year#126]
+Arguments: [ws_order_number#116 ASC NULLS FIRST, ws_item_sk#115 ASC NULLS FIRST], false, 0
+
+(118) ReusedExchange [Reuses operator id: 61]
+Output [4]: [wr_item_sk#128, wr_order_number#129, wr_return_quantity#130, wr_return_amt#131]
+
+(119) Sort [codegen id : 45]
+Input [4]: [wr_item_sk#128, wr_order_number#129, wr_return_quantity#130, wr_return_amt#131]
+Arguments: [wr_order_number#129 ASC NULLS FIRST, wr_item_sk#128 ASC NULLS FIRST], false, 0
+
+(120) SortMergeJoin [codegen id : 46]
+Left keys [2]: [ws_order_number#116, ws_item_sk#115]
+Right keys [2]: [wr_order_number#129, wr_item_sk#128]
+Join condition: None
+
+(121) Project [codegen id : 46]
+Output [7]: [d_year#126, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124, (ws_quantity#117 - coalesce(wr_return_quantity#130, 0)) AS sales_cnt#66, CheckOverflow((promote_precision(cast(ws_ext_sales_price#118 as decimal(8,2))) - promote_precision(cast(coalesce(wr_return_amt#131, 0.00) as decimal(8,2)))), DecimalType(8,2), true) AS sales_amt#67]
+Input [13]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124, d_year#126, wr_item_sk#128, wr_order_number#129, wr_return_quantity#130, wr_return_amt#131]
+
+(122) Union
+
+(123) HashAggregate [codegen id : 47]
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+Keys [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+Functions: []
+Aggregate Attributes: []
+Results [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+
+(124) Exchange
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+Arguments: hashpartitioning(d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25, 5), ENSURE_REQUIREMENTS, [id=#132]
+
+(125) HashAggregate [codegen id : 48]
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+Keys [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+Functions: []
+Aggregate Attributes: []
+Results [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+
+(126) HashAggregate [codegen id : 48]
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+Keys [5]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89]
+Functions [2]: [partial_sum(sales_cnt#24), partial_sum(UnscaledValue(sales_amt#25))]
+Aggregate Attributes [2]: [sum#133, sum#134]
+Results [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sum#135, sum#136]
+
+(127) Exchange
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sum#135, sum#136]
+Arguments: hashpartitioning(d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, 5), ENSURE_REQUIREMENTS, [id=#137]
+
+(128) HashAggregate [codegen id : 49]
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sum#135, sum#136]
+Keys [5]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89]
+Functions [2]: [sum(sales_cnt#24), sum(UnscaledValue(sales_amt#25))]
+Aggregate Attributes [2]: [sum(sales_cnt#24)#138, sum(UnscaledValue(sales_amt#25))#139]
+Results [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sum(sales_cnt#24)#138 AS sales_cnt#140, MakeDecimal(sum(UnscaledValue(sales_amt#25))#139,18,2) AS sales_amt#141]
+
+(129) Exchange
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#140, sales_amt#141]
+Arguments: hashpartitioning(i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, 5), ENSURE_REQUIREMENTS, [id=#142]
+
+(130) Sort [codegen id : 50]
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#140, sales_amt#141]
+Arguments: [i_brand_id#86 ASC NULLS FIRST, i_class_id#87 ASC NULLS FIRST, i_category_id#88 ASC NULLS FIRST, i_manufact_id#89 ASC NULLS FIRST], false, 0
+
+(131) SortMergeJoin [codegen id : 51]
+Left keys [4]: [i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+Right keys [4]: [i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89]
+Join condition: (CheckOverflow((promote_precision(cast(sales_cnt#76 as decimal(17,2))) / promote_precision(cast(sales_cnt#140 as decimal(17,2)))), DecimalType(37,20), true) < 0.90000000000000000000)
+
+(132) Project [codegen id : 51]
+Output [10]: [d_year#91 AS prev_year#143, d_year#15 AS year#144, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#140 AS prev_yr_cnt#145, sales_cnt#76 AS curr_yr_cnt#146, (sales_cnt#76 - sales_cnt#140) AS sales_cnt_diff#147, CheckOverflow((promote_precision(cast(sales_amt#77 as decimal(19,2))) - promote_precision(cast(sales_amt#141 as decimal(19,2)))), DecimalType(19,2), true) AS sales_amt_diff#148]
+Input [14]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#76, sales_amt#77, d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#140, sales_amt#141]
+
+(133) TakeOrderedAndProject
+Input [10]: [prev_year#143, year#144, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#145, curr_yr_cnt#146, sales_cnt_diff#147, sales_amt_diff#148]
+Arguments: 100, [sales_cnt_diff#147 ASC NULLS FIRST, sales_amt_diff#148 ASC NULLS FIRST], [prev_year#143, year#144, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#145, curr_yr_cnt#146, sales_cnt_diff#147, sales_amt_diff#148]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (134)
+
+
+(134) ReusedExchange [Reuses operator id: 14]
+Output [2]: [d_date_sk#14, d_year#15]
+
+Subquery:2 Hosting operator id = 27 Hosting Expression = ss_sold_date_sk#30 IN dynamicpruning#6
+
+Subquery:3 Hosting operator id = 46 Hosting Expression = ws_sold_date_sk#51 IN dynamicpruning#6
+
+Subquery:4 Hosting operator id = 74 Hosting Expression = cs_sold_date_sk#83 IN dynamicpruning#84
+ReusedExchange (135)
+
+
+(135) ReusedExchange [Reuses operator id: 83]
+Output [2]: [d_date_sk#90, d_year#91]
+
+Subquery:5 Hosting operator id = 92 Hosting Expression = ss_sold_date_sk#102 IN dynamicpruning#84
+
+Subquery:6 Hosting operator id = 107 Hosting Expression = ws_sold_date_sk#119 IN dynamicpruning#84
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.sf100/simplified.txt
new file mode 100644
index 0000000000000..1e5555b76b4e5
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.sf100/simplified.txt
@@ -0,0 +1,238 @@
+TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt]
+  WholeStageCodegen (51)
+    Project [d_year,d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt,sales_amt,sales_amt]
+      SortMergeJoin [i_brand_id,i_class_id,i_category_id,i_manufact_id,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt]
+        InputAdapter
+          WholeStageCodegen (25)
+            Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id]
+              InputAdapter
+                Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1
+                  WholeStageCodegen (24)
+                    HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum]
+                      InputAdapter
+                        Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2
+                          WholeStageCodegen (23)
+                            HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum]
+                              HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt]
+                                InputAdapter
+                                  Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3
+                                    WholeStageCodegen (22)
+                                      HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt]
+                                        InputAdapter
+                                          Union
+                                            WholeStageCodegen (7)
+                                              Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount]
+                                                SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (4)
+                                                      Sort [cs_order_number,cs_item_sk]
+                                                        InputAdapter
+                                                          Exchange [cs_order_number,cs_item_sk] #4
+                                                            WholeStageCodegen (3)
+                                                              Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year]
+                                                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                  Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                    BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                                                      Filter [cs_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk]
+                                                                              SubqueryBroadcast [d_date_sk] #1
+                                                                                ReusedExchange [d_date_sk,d_year] #5
+                                                                      InputAdapter
+                                                                        BroadcastExchange #6
+                                                                          WholeStageCodegen (1)
+                                                                            Project [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                              Filter [i_category,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id]
+                                                                  InputAdapter
+                                                                    BroadcastExchange #5
+                                                                      WholeStageCodegen (2)
+                                                                        Filter [d_year,d_date_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.date_dim [d_date_sk,d_year]
+                                                  InputAdapter
+                                                    WholeStageCodegen (6)
+                                                      Sort [cr_order_number,cr_item_sk]
+                                                        InputAdapter
+                                                          Exchange [cr_order_number,cr_item_sk] #7
+                                                            WholeStageCodegen (5)
+                                                              Project [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount]
+                                                                Filter [cr_order_number,cr_item_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk]
+                                            WholeStageCodegen (14)
+                                              Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt]
+                                                SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (11)
+                                                      Sort [ss_ticket_number,ss_item_sk]
+                                                        InputAdapter
+                                                          Exchange [ss_ticket_number,ss_item_sk] #8
+                                                            WholeStageCodegen (10)
+                                                              Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year]
+                                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                  Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                                      Filter [ss_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk]
+                                                                              ReusedSubquery [d_date_sk] #1
+                                                                      InputAdapter
+                                                                        ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6
+                                                                  InputAdapter
+                                                                    ReusedExchange [d_date_sk,d_year] #5
+                                                  InputAdapter
+                                                    WholeStageCodegen (13)
+                                                      Sort [sr_ticket_number,sr_item_sk]
+                                                        InputAdapter
+                                                          Exchange [sr_ticket_number,sr_item_sk] #9
+                                                            WholeStageCodegen (12)
+                                                              Project [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt]
+                                                                Filter [sr_ticket_number,sr_item_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk]
+                                            WholeStageCodegen (21)
+                                              Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt]
+                                                SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (18)
+                                                      Sort [ws_order_number,ws_item_sk]
+                                                        InputAdapter
+                                                          Exchange [ws_order_number,ws_item_sk] #10
+                                                            WholeStageCodegen (17)
+                                                              Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year]
+                                                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                  Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                    BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                                                      Filter [ws_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk]
+                                                                              ReusedSubquery [d_date_sk] #1
+                                                                      InputAdapter
+                                                                        ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6
+                                                                  InputAdapter
+                                                                    ReusedExchange [d_date_sk,d_year] #5
+                                                  InputAdapter
+                                                    WholeStageCodegen (20)
+                                                      Sort [wr_order_number,wr_item_sk]
+                                                        InputAdapter
+                                                          Exchange [wr_order_number,wr_item_sk] #11
+                                                            WholeStageCodegen (19)
+                                                              Project [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt]
+                                                                Filter [wr_order_number,wr_item_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk]
+        InputAdapter
+          WholeStageCodegen (50)
+            Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id]
+              InputAdapter
+                Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12
+                  WholeStageCodegen (49)
+                    HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum]
+                      InputAdapter
+                        Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13
+                          WholeStageCodegen (48)
+                            HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum]
+                              HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt]
+                                InputAdapter
+                                  Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14
+                                    WholeStageCodegen (47)
+                                      HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt]
+                                        InputAdapter
+                                          Union
+                                            WholeStageCodegen (32)
+                                              Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount]
+                                                SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (29)
+                                                      Sort [cs_order_number,cs_item_sk]
+                                                        InputAdapter
+                                                          Exchange [cs_order_number,cs_item_sk] #15
+                                                            WholeStageCodegen (28)
+                                                              Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year]
+                                                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                  Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                    BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                                                      Filter [cs_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk]
+                                                                              SubqueryBroadcast [d_date_sk] #2
+                                                                                ReusedExchange [d_date_sk,d_year] #16
+                                                                      InputAdapter
+                                                                        ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6
+                                                                  InputAdapter
+                                                                    BroadcastExchange #16
+                                                                      WholeStageCodegen (27)
+                                                                        Filter [d_year,d_date_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.date_dim [d_date_sk,d_year]
+                                                  InputAdapter
+                                                    WholeStageCodegen (31)
+                                                      Sort [cr_order_number,cr_item_sk]
+                                                        InputAdapter
+                                                          ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7
+                                            WholeStageCodegen (39)
+                                              Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt]
+                                                SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (36)
+                                                      Sort [ss_ticket_number,ss_item_sk]
+                                                        InputAdapter
+                                                          Exchange [ss_ticket_number,ss_item_sk] #17
+                                                            WholeStageCodegen (35)
+                                                              Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year]
+                                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                  Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                                      Filter [ss_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk]
+                                                                              ReusedSubquery [d_date_sk] #2
+                                                                      InputAdapter
+                                                                        ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6
+                                                                  InputAdapter
+                                                                    ReusedExchange [d_date_sk,d_year] #16
+                                                  InputAdapter
+                                                    WholeStageCodegen (38)
+                                                      Sort [sr_ticket_number,sr_item_sk]
+                                                        InputAdapter
+                                                          ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9
+                                            WholeStageCodegen (46)
+                                              Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt]
+                                                SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (43)
+                                                      Sort [ws_order_number,ws_item_sk]
+                                                        InputAdapter
+                                                          Exchange [ws_order_number,ws_item_sk] #18
+                                                            WholeStageCodegen (42)
+                                                              Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year]
+                                                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                  Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                    BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                                                      Filter [ws_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk]
+                                                                              ReusedSubquery [d_date_sk] #2
+                                                                      InputAdapter
+                                                                        ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6
+                                                                  InputAdapter
+                                                                    ReusedExchange [d_date_sk,d_year] #16
+                                                  InputAdapter
+                                                    WholeStageCodegen (45)
+                                                      Sort [wr_order_number,wr_item_sk]
+                                                        InputAdapter
+                                                          ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt
new file mode 100644
index 0000000000000..eb5287bfa8850
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt
@@ -0,0 +1,750 @@
+== Physical Plan ==
+TakeOrderedAndProject (133)
++- * Project (132)
+   +- * SortMergeJoin Inner (131)
+      :- * Sort (73)
+      :  +- Exchange (72)
+      :     +- * HashAggregate (71)
+      :        +- Exchange (70)
+      :           +- * HashAggregate (69)
+      :              +- * HashAggregate (68)
+      :                 +- Exchange (67)
+      :                    +- * HashAggregate (66)
+      :                       +- Union (65)
+      :                          :- * Project (26)
+      :                          :  +- * SortMergeJoin LeftOuter (25)
+      :                          :     :- * Sort (18)
+      :                          :     :  +- Exchange (17)
+      :                          :     :     +- * Project (16)
+      :                          :     :        +- * BroadcastHashJoin Inner BuildRight (15)
+      :                          :     :           :- * Project (10)
+      :                          :     :           :  +- * BroadcastHashJoin Inner BuildRight (9)
+      :                          :     :           :     :- * Filter (3)
+      :                          :     :           :     :  +- * ColumnarToRow (2)
+      :                          :     :           :     :     +- Scan parquet default.catalog_sales (1)
+      :                          :     :           :     +- BroadcastExchange (8)
+      :                          :     :           :        +- * Project (7)
+      :                          :     :           :           +- * Filter (6)
+      :                          :     :           :              +- * ColumnarToRow (5)
+      :                          :     :           :                 +- Scan parquet default.item (4)
+      :                          :     :           +- BroadcastExchange (14)
+      :                          :     :              +- * Filter (13)
+      :                          :     :                 +- * ColumnarToRow (12)
+      :                          :     :                    +- Scan parquet default.date_dim (11)
+      :                          :     +- * Sort (24)
+      :                          :        +- Exchange (23)
+      :                          :           +- * Project (22)
+      :                          :              +- * Filter (21)
+      :                          :                 +- * ColumnarToRow (20)
+      :                          :                    +- Scan parquet default.catalog_returns (19)
+      :                          :- * Project (45)
+      :                          :  +- * SortMergeJoin LeftOuter (44)
+      :                          :     :- * Sort (37)
+      :                          :     :  +- Exchange (36)
+      :                          :     :     +- * Project (35)
+      :                          :     :        +- * BroadcastHashJoin Inner BuildRight (34)
+      :                          :     :           :- * Project (32)
+      :                          :     :           :  +- * BroadcastHashJoin Inner BuildRight (31)
+      :                          :     :           :     :- * Filter (29)
+      :                          :     :           :     :  +- * ColumnarToRow (28)
+      :                          :     :           :     :     +- Scan parquet default.store_sales (27)
+      :                          :     :           :     +- ReusedExchange (30)
+      :                          :     :           +- ReusedExchange (33)
+      :                          :     +- * Sort (43)
+      :                          :        +- Exchange (42)
+      :                          :           +- * Project (41)
+      :                          :              +- * Filter (40)
+      :                          :                 +- * ColumnarToRow (39)
+      :                          :                    +- Scan parquet default.store_returns (38)
+      :                          +- * Project (64)
+      :                             +- * SortMergeJoin LeftOuter (63)
+      :                                :- * Sort (56)
+      :                                :  +- Exchange (55)
+      :                                :     +- * Project (54)
+      :                                :        +- * BroadcastHashJoin Inner BuildRight (53)
+      :                                :           :- * Project (51)
+      :                                :           :  +- * BroadcastHashJoin Inner BuildRight (50)
+      :                                :           :     :- * Filter (48)
+      :                                :           :     :  +- * ColumnarToRow (47)
+      :                                :           :     :     +- Scan parquet default.web_sales (46)
+      :                                :           :     +- ReusedExchange (49)
+      :                                :           +- ReusedExchange (52)
+      :                                +- * Sort (62)
+      :                                   +- Exchange (61)
+      :                                      +- * Project (60)
+      :                                         +- * Filter (59)
+      :                                            +- * ColumnarToRow (58)
+      :                                               +- Scan parquet default.web_returns (57)
+      +- * Sort (130)
+         +- Exchange (129)
+            +- * HashAggregate (128)
+               +- Exchange (127)
+                  +- * HashAggregate (126)
+                     +- * HashAggregate (125)
+                        +- Exchange (124)
+                           +- * HashAggregate (123)
+                              +- Union (122)
+                                 :- * Project (91)
+                                 :  +- * SortMergeJoin LeftOuter (90)
+                                 :     :- * Sort (87)
+                                 :     :  +- Exchange (86)
+                                 :     :     +- * Project (85)
+                                 :     :        +- * BroadcastHashJoin Inner BuildRight (84)
+                                 :     :           :- * Project (79)
+                                 :     :           :  +- * BroadcastHashJoin Inner BuildRight (78)
+                                 :     :           :     :- * Filter (76)
+                                 :     :           :     :  +- * ColumnarToRow (75)
+                                 :     :           :     :     +- Scan parquet default.catalog_sales (74)
+                                 :     :           :     +- ReusedExchange (77)
+                                 :     :           +- BroadcastExchange (83)
+                                 :     :              +- * Filter (82)
+                                 :     :                 +- * ColumnarToRow (81)
+                                 :     :                    +- Scan parquet default.date_dim (80)
+                                 :     +- * Sort (89)
+                                 :        +- ReusedExchange (88)
+                                 :- * Project (106)
+                                 :  +- * SortMergeJoin LeftOuter (105)
+                                 :     :- * Sort (102)
+                                 :     :  +- Exchange (101)
+                                 :     :     +- * Project (100)
+                                 :     :        +- * BroadcastHashJoin Inner BuildRight (99)
+                                 :     :           :- * Project (97)
+                                 :     :           :  +- * BroadcastHashJoin Inner BuildRight (96)
+                                 :     :           :     :- * Filter (94)
+                                 :     :           :     :  +- * ColumnarToRow (93)
+                                 :     :           :     :     +- Scan parquet default.store_sales (92)
+                                 :     :           :     +- ReusedExchange (95)
+                                 :     :           +- ReusedExchange (98)
+                                 :     +- * Sort (104)
+                                 :        +- ReusedExchange (103)
+                                 +- * Project (121)
+                                    +- * SortMergeJoin LeftOuter (120)
+                                       :- * Sort (117)
+                                       :  +- Exchange (116)
+                                       :     +- * Project (115)
+                                       :        +- * BroadcastHashJoin Inner BuildRight (114)
+                                       :           :- * Project (112)
+                                       :           :  +- * BroadcastHashJoin Inner BuildRight (111)
+                                       :           :     :- * Filter (109)
+                                       :           :     :  +- * ColumnarToRow (108)
+                                       :           :     :     +- Scan parquet default.web_sales (107)
+                                       :           :     +- ReusedExchange (110)
+                                       :           +- ReusedExchange (113)
+                                       +- * Sort (119)
+                                          +- ReusedExchange (118)
+
+
+(1) Scan parquet default.catalog_sales
+Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sold_date_sk#5 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int,cs_order_number:int,cs_quantity:int,cs_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5]
+
+(3) Filter [codegen id : 3]
+Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5]
+Condition : isnotnull(cs_item_sk#1)
+
+(4) Scan parquet default.item
+Output [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Books                                             ), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)]
+ReadSchema: struct<i_item_sk:int,i_brand_id:int,i_class_id:int,i_category_id:int,i_category:string,i_manufact_id:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12]
+
+(6) Filter [codegen id : 1]
+Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12]
+Condition : ((((((isnotnull(i_category#11) AND (i_category#11 = Books                                             )) AND isnotnull(i_item_sk#7)) AND isnotnull(i_brand_id#8)) AND isnotnull(i_class_id#9)) AND isnotnull(i_category_id#10)) AND isnotnull(i_manufact_id#12))
+
+(7) Project [codegen id : 1]
+Output [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12]
+
+(8) BroadcastExchange
+Input [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#13]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_item_sk#1]
+Right keys [1]: [i_item_sk#7]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+
+(11) Scan parquet default.date_dim
+Output [2]: [d_date_sk#14, d_year#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#14, d_year#15]
+
+(13) Filter [codegen id : 2]
+Input [2]: [d_date_sk#14, d_year#15]
+Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2002)) AND isnotnull(d_date_sk#14))
+
+(14) BroadcastExchange
+Input [2]: [d_date_sk#14, d_year#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#16]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [cs_sold_date_sk#5]
+Right keys [1]: [d_date_sk#14]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#15]
+Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_date_sk#14, d_year#15]
+
+(17) Exchange
+Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#15]
+Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(18) Sort [codegen id : 4]
+Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#15]
+Arguments: [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST], false, 0
+
+(19) Scan parquet default.catalog_returns
+Output [5]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21, cr_returned_date_sk#22]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int,cr_return_quantity:int,cr_return_amount:decimal(7,2)>
+
+(20) ColumnarToRow [codegen id : 5]
+Input [5]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21, cr_returned_date_sk#22]
+
+(21) Filter [codegen id : 5]
+Input [5]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21, cr_returned_date_sk#22]
+Condition : (isnotnull(cr_order_number#19) AND isnotnull(cr_item_sk#18))
+
+(22) Project [codegen id : 5]
+Output [4]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21]
+Input [5]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21, cr_returned_date_sk#22]
+
+(23) Exchange
+Input [4]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21]
+Arguments: hashpartitioning(cr_order_number#19, cr_item_sk#18, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(24) Sort [codegen id : 6]
+Input [4]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21]
+Arguments: [cr_order_number#19 ASC NULLS FIRST, cr_item_sk#18 ASC NULLS FIRST], false, 0
+
+(25) SortMergeJoin [codegen id : 7]
+Left keys [2]: [cs_order_number#2, cs_item_sk#1]
+Right keys [2]: [cr_order_number#19, cr_item_sk#18]
+Join condition: None
+
+(26) Project [codegen id : 7]
+Output [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, (cs_quantity#3 - coalesce(cr_return_quantity#20, 0)) AS sales_cnt#24, CheckOverflow((promote_precision(cast(cs_ext_sales_price#4 as decimal(8,2))) - promote_precision(cast(coalesce(cr_return_amount#21, 0.00) as decimal(8,2)))), DecimalType(8,2), true) AS sales_amt#25]
+Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#15, cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21]
+
+(27) Scan parquet default.store_sales
+Output [5]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, ss_sold_date_sk#30]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#30), dynamicpruningexpression(ss_sold_date_sk#30 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ticket_number:int,ss_quantity:int,ss_ext_sales_price:decimal(7,2)>
+
+(28) ColumnarToRow [codegen id : 10]
+Input [5]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, ss_sold_date_sk#30]
+
+(29) Filter [codegen id : 10]
+Input [5]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, ss_sold_date_sk#30]
+Condition : isnotnull(ss_item_sk#26)
+
+(30) ReusedExchange [Reuses operator id: 8]
+Output [5]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35]
+
+(31) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_item_sk#26]
+Right keys [1]: [i_item_sk#31]
+Join condition: None
+
+(32) Project [codegen id : 10]
+Output [9]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, ss_sold_date_sk#30, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35]
+Input [10]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, ss_sold_date_sk#30, i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35]
+
+(33) ReusedExchange [Reuses operator id: 14]
+Output [2]: [d_date_sk#36, d_year#37]
+
+(34) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [ss_sold_date_sk#30]
+Right keys [1]: [d_date_sk#36]
+Join condition: None
+
+(35) Project [codegen id : 10]
+Output [9]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35, d_year#37]
+Input [11]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, ss_sold_date_sk#30, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35, d_date_sk#36, d_year#37]
+
+(36) Exchange
+Input [9]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35, d_year#37]
+Arguments: hashpartitioning(ss_ticket_number#27, ss_item_sk#26, 5), ENSURE_REQUIREMENTS, [id=#38]
+
+(37) Sort [codegen id : 11]
+Input [9]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35, d_year#37]
+Arguments: [ss_ticket_number#27 ASC NULLS FIRST, ss_item_sk#26 ASC NULLS FIRST], false, 0
+
+(38) Scan parquet default.store_returns
+Output [5]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42, sr_returned_date_sk#43]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int,sr_return_quantity:int,sr_return_amt:decimal(7,2)>
+
+(39) ColumnarToRow [codegen id : 12]
+Input [5]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42, sr_returned_date_sk#43]
+
+(40) Filter [codegen id : 12]
+Input [5]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42, sr_returned_date_sk#43]
+Condition : (isnotnull(sr_ticket_number#40) AND isnotnull(sr_item_sk#39))
+
+(41) Project [codegen id : 12]
+Output [4]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42]
+Input [5]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42, sr_returned_date_sk#43]
+
+(42) Exchange
+Input [4]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42]
+Arguments: hashpartitioning(sr_ticket_number#40, sr_item_sk#39, 5), ENSURE_REQUIREMENTS, [id=#44]
+
+(43) Sort [codegen id : 13]
+Input [4]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42]
+Arguments: [sr_ticket_number#40 ASC NULLS FIRST, sr_item_sk#39 ASC NULLS FIRST], false, 0
+
+(44) SortMergeJoin [codegen id : 14]
+Left keys [2]: [ss_ticket_number#27, ss_item_sk#26]
+Right keys [2]: [sr_ticket_number#40, sr_item_sk#39]
+Join condition: None
+
+(45) Project [codegen id : 14]
+Output [7]: [d_year#37, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35, (ss_quantity#28 - coalesce(sr_return_quantity#41, 0)) AS sales_cnt#45, CheckOverflow((promote_precision(cast(ss_ext_sales_price#29 as decimal(8,2))) - promote_precision(cast(coalesce(sr_return_amt#42, 0.00) as decimal(8,2)))), DecimalType(8,2), true) AS sales_amt#46]
+Input [13]: [ss_item_sk#26, ss_ticket_number#27, ss_quantity#28, ss_ext_sales_price#29, i_brand_id#32, i_class_id#33, i_category_id#34, i_manufact_id#35, d_year#37, sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42]
+
+(46) Scan parquet default.web_sales
+Output [5]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, ws_sold_date_sk#51]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#51), dynamicpruningexpression(ws_sold_date_sk#51 IN dynamicpruning#6)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_order_number:int,ws_quantity:int,ws_ext_sales_price:decimal(7,2)>
+
+(47) ColumnarToRow [codegen id : 17]
+Input [5]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, ws_sold_date_sk#51]
+
+(48) Filter [codegen id : 17]
+Input [5]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, ws_sold_date_sk#51]
+Condition : isnotnull(ws_item_sk#47)
+
+(49) ReusedExchange [Reuses operator id: 8]
+Output [5]: [i_item_sk#52, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56]
+
+(50) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_item_sk#47]
+Right keys [1]: [i_item_sk#52]
+Join condition: None
+
+(51) Project [codegen id : 17]
+Output [9]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, ws_sold_date_sk#51, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56]
+Input [10]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, ws_sold_date_sk#51, i_item_sk#52, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56]
+
+(52) ReusedExchange [Reuses operator id: 14]
+Output [2]: [d_date_sk#57, d_year#58]
+
+(53) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_sold_date_sk#51]
+Right keys [1]: [d_date_sk#57]
+Join condition: None
+
+(54) Project [codegen id : 17]
+Output [9]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56, d_year#58]
+Input [11]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, ws_sold_date_sk#51, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56, d_date_sk#57, d_year#58]
+
+(55) Exchange
+Input [9]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56, d_year#58]
+Arguments: hashpartitioning(ws_order_number#48, ws_item_sk#47, 5), ENSURE_REQUIREMENTS, [id=#59]
+
+(56) Sort [codegen id : 18]
+Input [9]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56, d_year#58]
+Arguments: [ws_order_number#48 ASC NULLS FIRST, ws_item_sk#47 ASC NULLS FIRST], false, 0
+
+(57) Scan parquet default.web_returns
+Output [5]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63, wr_returned_date_sk#64]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_returns]
+PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)]
+ReadSchema: struct<wr_item_sk:int,wr_order_number:int,wr_return_quantity:int,wr_return_amt:decimal(7,2)>
+
+(58) ColumnarToRow [codegen id : 19]
+Input [5]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63, wr_returned_date_sk#64]
+
+(59) Filter [codegen id : 19]
+Input [5]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63, wr_returned_date_sk#64]
+Condition : (isnotnull(wr_order_number#61) AND isnotnull(wr_item_sk#60))
+
+(60) Project [codegen id : 19]
+Output [4]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63]
+Input [5]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63, wr_returned_date_sk#64]
+
+(61) Exchange
+Input [4]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63]
+Arguments: hashpartitioning(wr_order_number#61, wr_item_sk#60, 5), ENSURE_REQUIREMENTS, [id=#65]
+
+(62) Sort [codegen id : 20]
+Input [4]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63]
+Arguments: [wr_order_number#61 ASC NULLS FIRST, wr_item_sk#60 ASC NULLS FIRST], false, 0
+
+(63) SortMergeJoin [codegen id : 21]
+Left keys [2]: [ws_order_number#48, ws_item_sk#47]
+Right keys [2]: [wr_order_number#61, wr_item_sk#60]
+Join condition: None
+
+(64) Project [codegen id : 21]
+Output [7]: [d_year#58, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56, (ws_quantity#49 - coalesce(wr_return_quantity#62, 0)) AS sales_cnt#66, CheckOverflow((promote_precision(cast(ws_ext_sales_price#50 as decimal(8,2))) - promote_precision(cast(coalesce(wr_return_amt#63, 0.00) as decimal(8,2)))), DecimalType(8,2), true) AS sales_amt#67]
+Input [13]: [ws_item_sk#47, ws_order_number#48, ws_quantity#49, ws_ext_sales_price#50, i_brand_id#53, i_class_id#54, i_category_id#55, i_manufact_id#56, d_year#58, wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63]
+
+(65) Union
+
+(66) HashAggregate [codegen id : 22]
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+Keys [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+Functions: []
+Aggregate Attributes: []
+Results [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+
+(67) Exchange
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+Arguments: hashpartitioning(d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25, 5), ENSURE_REQUIREMENTS, [id=#68]
+
+(68) HashAggregate [codegen id : 23]
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+Keys [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+Functions: []
+Aggregate Attributes: []
+Results [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+
+(69) HashAggregate [codegen id : 23]
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#24, sales_amt#25]
+Keys [5]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+Functions [2]: [partial_sum(sales_cnt#24), partial_sum(UnscaledValue(sales_amt#25))]
+Aggregate Attributes [2]: [sum#69, sum#70]
+Results [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#71, sum#72]
+
+(70) Exchange
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#71, sum#72]
+Arguments: hashpartitioning(d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [id=#73]
+
+(71) HashAggregate [codegen id : 24]
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#71, sum#72]
+Keys [5]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+Functions [2]: [sum(sales_cnt#24), sum(UnscaledValue(sales_amt#25))]
+Aggregate Attributes [2]: [sum(sales_cnt#24)#74, sum(UnscaledValue(sales_amt#25))#75]
+Results [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum(sales_cnt#24)#74 AS sales_cnt#76, MakeDecimal(sum(UnscaledValue(sales_amt#25))#75,18,2) AS sales_amt#77]
+
+(72) Exchange
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#76, sales_amt#77]
+Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [id=#78]
+
+(73) Sort [codegen id : 25]
+Input [7]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#76, sales_amt#77]
+Arguments: [i_brand_id#8 ASC NULLS FIRST, i_class_id#9 ASC NULLS FIRST, i_category_id#10 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST], false, 0
+
+(74) Scan parquet default.catalog_sales
+Output [5]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, cs_sold_date_sk#83]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#83), dynamicpruningexpression(cs_sold_date_sk#83 IN dynamicpruning#84)]
+PushedFilters: [IsNotNull(cs_item_sk)]
+ReadSchema: struct<cs_item_sk:int,cs_order_number:int,cs_quantity:int,cs_ext_sales_price:decimal(7,2)>
+
+(75) ColumnarToRow [codegen id : 28]
+Input [5]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, cs_sold_date_sk#83]
+
+(76) Filter [codegen id : 28]
+Input [5]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, cs_sold_date_sk#83]
+Condition : isnotnull(cs_item_sk#79)
+
+(77) ReusedExchange [Reuses operator id: 8]
+Output [5]: [i_item_sk#85, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89]
+
+(78) BroadcastHashJoin [codegen id : 28]
+Left keys [1]: [cs_item_sk#79]
+Right keys [1]: [i_item_sk#85]
+Join condition: None
+
+(79) Project [codegen id : 28]
+Output [9]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, cs_sold_date_sk#83, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89]
+Input [10]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, cs_sold_date_sk#83, i_item_sk#85, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89]
+
+(80) Scan parquet default.date_dim
+Output [2]: [d_date_sk#90, d_year#91]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(81) ColumnarToRow [codegen id : 27]
+Input [2]: [d_date_sk#90, d_year#91]
+
+(82) Filter [codegen id : 27]
+Input [2]: [d_date_sk#90, d_year#91]
+Condition : ((isnotnull(d_year#91) AND (d_year#91 = 2001)) AND isnotnull(d_date_sk#90))
+
+(83) BroadcastExchange
+Input [2]: [d_date_sk#90, d_year#91]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#92]
+
+(84) BroadcastHashJoin [codegen id : 28]
+Left keys [1]: [cs_sold_date_sk#83]
+Right keys [1]: [d_date_sk#90]
+Join condition: None
+
+(85) Project [codegen id : 28]
+Output [9]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, d_year#91]
+Input [11]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, cs_sold_date_sk#83, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, d_date_sk#90, d_year#91]
+
+(86) Exchange
+Input [9]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, d_year#91]
+Arguments: hashpartitioning(cs_order_number#80, cs_item_sk#79, 5), ENSURE_REQUIREMENTS, [id=#93]
+
+(87) Sort [codegen id : 29]
+Input [9]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, d_year#91]
+Arguments: [cs_order_number#80 ASC NULLS FIRST, cs_item_sk#79 ASC NULLS FIRST], false, 0
+
+(88) ReusedExchange [Reuses operator id: 23]
+Output [4]: [cr_item_sk#94, cr_order_number#95, cr_return_quantity#96, cr_return_amount#97]
+
+(89) Sort [codegen id : 31]
+Input [4]: [cr_item_sk#94, cr_order_number#95, cr_return_quantity#96, cr_return_amount#97]
+Arguments: [cr_order_number#95 ASC NULLS FIRST, cr_item_sk#94 ASC NULLS FIRST], false, 0
+
+(90) SortMergeJoin [codegen id : 32]
+Left keys [2]: [cs_order_number#80, cs_item_sk#79]
+Right keys [2]: [cr_order_number#95, cr_item_sk#94]
+Join condition: None
+
+(91) Project [codegen id : 32]
+Output [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, (cs_quantity#81 - coalesce(cr_return_quantity#96, 0)) AS sales_cnt#24, CheckOverflow((promote_precision(cast(cs_ext_sales_price#82 as decimal(8,2))) - promote_precision(cast(coalesce(cr_return_amount#97, 0.00) as decimal(8,2)))), DecimalType(8,2), true) AS sales_amt#25]
+Input [13]: [cs_item_sk#79, cs_order_number#80, cs_quantity#81, cs_ext_sales_price#82, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, d_year#91, cr_item_sk#94, cr_order_number#95, cr_return_quantity#96, cr_return_amount#97]
+
+(92) Scan parquet default.store_sales
+Output [5]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, ss_sold_date_sk#102]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#102), dynamicpruningexpression(ss_sold_date_sk#102 IN dynamicpruning#84)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ticket_number:int,ss_quantity:int,ss_ext_sales_price:decimal(7,2)>
+
+(93) ColumnarToRow [codegen id : 35]
+Input [5]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, ss_sold_date_sk#102]
+
+(94) Filter [codegen id : 35]
+Input [5]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, ss_sold_date_sk#102]
+Condition : isnotnull(ss_item_sk#98)
+
+(95) ReusedExchange [Reuses operator id: 8]
+Output [5]: [i_item_sk#103, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107]
+
+(96) BroadcastHashJoin [codegen id : 35]
+Left keys [1]: [ss_item_sk#98]
+Right keys [1]: [i_item_sk#103]
+Join condition: None
+
+(97) Project [codegen id : 35]
+Output [9]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, ss_sold_date_sk#102, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107]
+Input [10]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, ss_sold_date_sk#102, i_item_sk#103, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107]
+
+(98) ReusedExchange [Reuses operator id: 83]
+Output [2]: [d_date_sk#108, d_year#109]
+
+(99) BroadcastHashJoin [codegen id : 35]
+Left keys [1]: [ss_sold_date_sk#102]
+Right keys [1]: [d_date_sk#108]
+Join condition: None
+
+(100) Project [codegen id : 35]
+Output [9]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107, d_year#109]
+Input [11]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, ss_sold_date_sk#102, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107, d_date_sk#108, d_year#109]
+
+(101) Exchange
+Input [9]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107, d_year#109]
+Arguments: hashpartitioning(ss_ticket_number#99, ss_item_sk#98, 5), ENSURE_REQUIREMENTS, [id=#110]
+
+(102) Sort [codegen id : 36]
+Input [9]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107, d_year#109]
+Arguments: [ss_ticket_number#99 ASC NULLS FIRST, ss_item_sk#98 ASC NULLS FIRST], false, 0
+
+(103) ReusedExchange [Reuses operator id: 42]
+Output [4]: [sr_item_sk#111, sr_ticket_number#112, sr_return_quantity#113, sr_return_amt#114]
+
+(104) Sort [codegen id : 38]
+Input [4]: [sr_item_sk#111, sr_ticket_number#112, sr_return_quantity#113, sr_return_amt#114]
+Arguments: [sr_ticket_number#112 ASC NULLS FIRST, sr_item_sk#111 ASC NULLS FIRST], false, 0
+
+(105) SortMergeJoin [codegen id : 39]
+Left keys [2]: [ss_ticket_number#99, ss_item_sk#98]
+Right keys [2]: [sr_ticket_number#112, sr_item_sk#111]
+Join condition: None
+
+(106) Project [codegen id : 39]
+Output [7]: [d_year#109, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107, (ss_quantity#100 - coalesce(sr_return_quantity#113, 0)) AS sales_cnt#45, CheckOverflow((promote_precision(cast(ss_ext_sales_price#101 as decimal(8,2))) - promote_precision(cast(coalesce(sr_return_amt#114, 0.00) as decimal(8,2)))), DecimalType(8,2), true) AS sales_amt#46]
+Input [13]: [ss_item_sk#98, ss_ticket_number#99, ss_quantity#100, ss_ext_sales_price#101, i_brand_id#104, i_class_id#105, i_category_id#106, i_manufact_id#107, d_year#109, sr_item_sk#111, sr_ticket_number#112, sr_return_quantity#113, sr_return_amt#114]
+
+(107) Scan parquet default.web_sales
+Output [5]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, ws_sold_date_sk#119]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#119), dynamicpruningexpression(ws_sold_date_sk#119 IN dynamicpruning#84)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_order_number:int,ws_quantity:int,ws_ext_sales_price:decimal(7,2)>
+
+(108) ColumnarToRow [codegen id : 42]
+Input [5]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, ws_sold_date_sk#119]
+
+(109) Filter [codegen id : 42]
+Input [5]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, ws_sold_date_sk#119]
+Condition : isnotnull(ws_item_sk#115)
+
+(110) ReusedExchange [Reuses operator id: 8]
+Output [5]: [i_item_sk#120, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124]
+
+(111) BroadcastHashJoin [codegen id : 42]
+Left keys [1]: [ws_item_sk#115]
+Right keys [1]: [i_item_sk#120]
+Join condition: None
+
+(112) Project [codegen id : 42]
+Output [9]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, ws_sold_date_sk#119, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124]
+Input [10]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, ws_sold_date_sk#119, i_item_sk#120, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124]
+
+(113) ReusedExchange [Reuses operator id: 83]
+Output [2]: [d_date_sk#125, d_year#126]
+
+(114) BroadcastHashJoin [codegen id : 42]
+Left keys [1]: [ws_sold_date_sk#119]
+Right keys [1]: [d_date_sk#125]
+Join condition: None
+
+(115) Project [codegen id : 42]
+Output [9]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124, d_year#126]
+Input [11]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, ws_sold_date_sk#119, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124, d_date_sk#125, d_year#126]
+
+(116) Exchange
+Input [9]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124, d_year#126]
+Arguments: hashpartitioning(ws_order_number#116, ws_item_sk#115, 5), ENSURE_REQUIREMENTS, [id=#127]
+
+(117) Sort [codegen id : 43]
+Input [9]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124, d_year#126]
+Arguments: [ws_order_number#116 ASC NULLS FIRST, ws_item_sk#115 ASC NULLS FIRST], false, 0
+
+(118) ReusedExchange [Reuses operator id: 61]
+Output [4]: [wr_item_sk#128, wr_order_number#129, wr_return_quantity#130, wr_return_amt#131]
+
+(119) Sort [codegen id : 45]
+Input [4]: [wr_item_sk#128, wr_order_number#129, wr_return_quantity#130, wr_return_amt#131]
+Arguments: [wr_order_number#129 ASC NULLS FIRST, wr_item_sk#128 ASC NULLS FIRST], false, 0
+
+(120) SortMergeJoin [codegen id : 46]
+Left keys [2]: [ws_order_number#116, ws_item_sk#115]
+Right keys [2]: [wr_order_number#129, wr_item_sk#128]
+Join condition: None
+
+(121) Project [codegen id : 46]
+Output [7]: [d_year#126, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124, (ws_quantity#117 - coalesce(wr_return_quantity#130, 0)) AS sales_cnt#66, CheckOverflow((promote_precision(cast(ws_ext_sales_price#118 as decimal(8,2))) - promote_precision(cast(coalesce(wr_return_amt#131, 0.00) as decimal(8,2)))), DecimalType(8,2), true) AS sales_amt#67]
+Input [13]: [ws_item_sk#115, ws_order_number#116, ws_quantity#117, ws_ext_sales_price#118, i_brand_id#121, i_class_id#122, i_category_id#123, i_manufact_id#124, d_year#126, wr_item_sk#128, wr_order_number#129, wr_return_quantity#130, wr_return_amt#131]
+
+(122) Union
+
+(123) HashAggregate [codegen id : 47]
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+Keys [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+Functions: []
+Aggregate Attributes: []
+Results [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+
+(124) Exchange
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+Arguments: hashpartitioning(d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25, 5), ENSURE_REQUIREMENTS, [id=#132]
+
+(125) HashAggregate [codegen id : 48]
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+Keys [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+Functions: []
+Aggregate Attributes: []
+Results [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+
+(126) HashAggregate [codegen id : 48]
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#24, sales_amt#25]
+Keys [5]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89]
+Functions [2]: [partial_sum(sales_cnt#24), partial_sum(UnscaledValue(sales_amt#25))]
+Aggregate Attributes [2]: [sum#133, sum#134]
+Results [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sum#135, sum#136]
+
+(127) Exchange
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sum#135, sum#136]
+Arguments: hashpartitioning(d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, 5), ENSURE_REQUIREMENTS, [id=#137]
+
+(128) HashAggregate [codegen id : 49]
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sum#135, sum#136]
+Keys [5]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89]
+Functions [2]: [sum(sales_cnt#24), sum(UnscaledValue(sales_amt#25))]
+Aggregate Attributes [2]: [sum(sales_cnt#24)#138, sum(UnscaledValue(sales_amt#25))#139]
+Results [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sum(sales_cnt#24)#138 AS sales_cnt#140, MakeDecimal(sum(UnscaledValue(sales_amt#25))#139,18,2) AS sales_amt#141]
+
+(129) Exchange
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#140, sales_amt#141]
+Arguments: hashpartitioning(i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, 5), ENSURE_REQUIREMENTS, [id=#142]
+
+(130) Sort [codegen id : 50]
+Input [7]: [d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#140, sales_amt#141]
+Arguments: [i_brand_id#86 ASC NULLS FIRST, i_class_id#87 ASC NULLS FIRST, i_category_id#88 ASC NULLS FIRST, i_manufact_id#89 ASC NULLS FIRST], false, 0
+
+(131) SortMergeJoin [codegen id : 51]
+Left keys [4]: [i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12]
+Right keys [4]: [i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89]
+Join condition: (CheckOverflow((promote_precision(cast(sales_cnt#76 as decimal(17,2))) / promote_precision(cast(sales_cnt#140 as decimal(17,2)))), DecimalType(37,20), true) < 0.90000000000000000000)
+
+(132) Project [codegen id : 51]
+Output [10]: [d_year#91 AS prev_year#143, d_year#15 AS year#144, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#140 AS prev_yr_cnt#145, sales_cnt#76 AS curr_yr_cnt#146, (sales_cnt#76 - sales_cnt#140) AS sales_cnt_diff#147, CheckOverflow((promote_precision(cast(sales_amt#77 as decimal(19,2))) - promote_precision(cast(sales_amt#141 as decimal(19,2)))), DecimalType(19,2), true) AS sales_amt_diff#148]
+Input [14]: [d_year#15, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#76, sales_amt#77, d_year#91, i_brand_id#86, i_class_id#87, i_category_id#88, i_manufact_id#89, sales_cnt#140, sales_amt#141]
+
+(133) TakeOrderedAndProject
+Input [10]: [prev_year#143, year#144, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#145, curr_yr_cnt#146, sales_cnt_diff#147, sales_amt_diff#148]
+Arguments: 100, [sales_cnt_diff#147 ASC NULLS FIRST, sales_amt_diff#148 ASC NULLS FIRST], [prev_year#143, year#144, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#145, curr_yr_cnt#146, sales_cnt_diff#147, sales_amt_diff#148]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6
+ReusedExchange (134)
+
+
+(134) ReusedExchange [Reuses operator id: 14]
+Output [2]: [d_date_sk#14, d_year#15]
+
+Subquery:2 Hosting operator id = 27 Hosting Expression = ss_sold_date_sk#30 IN dynamicpruning#6
+
+Subquery:3 Hosting operator id = 46 Hosting Expression = ws_sold_date_sk#51 IN dynamicpruning#6
+
+Subquery:4 Hosting operator id = 74 Hosting Expression = cs_sold_date_sk#83 IN dynamicpruning#84
+ReusedExchange (135)
+
+
+(135) ReusedExchange [Reuses operator id: 83]
+Output [2]: [d_date_sk#90, d_year#91]
+
+Subquery:5 Hosting operator id = 92 Hosting Expression = ss_sold_date_sk#102 IN dynamicpruning#84
+
+Subquery:6 Hosting operator id = 107 Hosting Expression = ws_sold_date_sk#119 IN dynamicpruning#84
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/simplified.txt
new file mode 100644
index 0000000000000..1e5555b76b4e5
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/simplified.txt
@@ -0,0 +1,238 @@
+TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt]
+  WholeStageCodegen (51)
+    Project [d_year,d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt,sales_amt,sales_amt]
+      SortMergeJoin [i_brand_id,i_class_id,i_category_id,i_manufact_id,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt]
+        InputAdapter
+          WholeStageCodegen (25)
+            Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id]
+              InputAdapter
+                Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1
+                  WholeStageCodegen (24)
+                    HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum]
+                      InputAdapter
+                        Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2
+                          WholeStageCodegen (23)
+                            HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum]
+                              HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt]
+                                InputAdapter
+                                  Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3
+                                    WholeStageCodegen (22)
+                                      HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt]
+                                        InputAdapter
+                                          Union
+                                            WholeStageCodegen (7)
+                                              Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount]
+                                                SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (4)
+                                                      Sort [cs_order_number,cs_item_sk]
+                                                        InputAdapter
+                                                          Exchange [cs_order_number,cs_item_sk] #4
+                                                            WholeStageCodegen (3)
+                                                              Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year]
+                                                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                  Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                    BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                                                      Filter [cs_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk]
+                                                                              SubqueryBroadcast [d_date_sk] #1
+                                                                                ReusedExchange [d_date_sk,d_year] #5
+                                                                      InputAdapter
+                                                                        BroadcastExchange #6
+                                                                          WholeStageCodegen (1)
+                                                                            Project [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                              Filter [i_category,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id]
+                                                                  InputAdapter
+                                                                    BroadcastExchange #5
+                                                                      WholeStageCodegen (2)
+                                                                        Filter [d_year,d_date_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.date_dim [d_date_sk,d_year]
+                                                  InputAdapter
+                                                    WholeStageCodegen (6)
+                                                      Sort [cr_order_number,cr_item_sk]
+                                                        InputAdapter
+                                                          Exchange [cr_order_number,cr_item_sk] #7
+                                                            WholeStageCodegen (5)
+                                                              Project [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount]
+                                                                Filter [cr_order_number,cr_item_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk]
+                                            WholeStageCodegen (14)
+                                              Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt]
+                                                SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (11)
+                                                      Sort [ss_ticket_number,ss_item_sk]
+                                                        InputAdapter
+                                                          Exchange [ss_ticket_number,ss_item_sk] #8
+                                                            WholeStageCodegen (10)
+                                                              Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year]
+                                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                  Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                                      Filter [ss_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk]
+                                                                              ReusedSubquery [d_date_sk] #1
+                                                                      InputAdapter
+                                                                        ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6
+                                                                  InputAdapter
+                                                                    ReusedExchange [d_date_sk,d_year] #5
+                                                  InputAdapter
+                                                    WholeStageCodegen (13)
+                                                      Sort [sr_ticket_number,sr_item_sk]
+                                                        InputAdapter
+                                                          Exchange [sr_ticket_number,sr_item_sk] #9
+                                                            WholeStageCodegen (12)
+                                                              Project [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt]
+                                                                Filter [sr_ticket_number,sr_item_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk]
+                                            WholeStageCodegen (21)
+                                              Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt]
+                                                SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (18)
+                                                      Sort [ws_order_number,ws_item_sk]
+                                                        InputAdapter
+                                                          Exchange [ws_order_number,ws_item_sk] #10
+                                                            WholeStageCodegen (17)
+                                                              Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year]
+                                                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                  Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                    BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                                                      Filter [ws_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk]
+                                                                              ReusedSubquery [d_date_sk] #1
+                                                                      InputAdapter
+                                                                        ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6
+                                                                  InputAdapter
+                                                                    ReusedExchange [d_date_sk,d_year] #5
+                                                  InputAdapter
+                                                    WholeStageCodegen (20)
+                                                      Sort [wr_order_number,wr_item_sk]
+                                                        InputAdapter
+                                                          Exchange [wr_order_number,wr_item_sk] #11
+                                                            WholeStageCodegen (19)
+                                                              Project [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt]
+                                                                Filter [wr_order_number,wr_item_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk]
+        InputAdapter
+          WholeStageCodegen (50)
+            Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id]
+              InputAdapter
+                Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12
+                  WholeStageCodegen (49)
+                    HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum]
+                      InputAdapter
+                        Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13
+                          WholeStageCodegen (48)
+                            HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum]
+                              HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt]
+                                InputAdapter
+                                  Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14
+                                    WholeStageCodegen (47)
+                                      HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt]
+                                        InputAdapter
+                                          Union
+                                            WholeStageCodegen (32)
+                                              Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount]
+                                                SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (29)
+                                                      Sort [cs_order_number,cs_item_sk]
+                                                        InputAdapter
+                                                          Exchange [cs_order_number,cs_item_sk] #15
+                                                            WholeStageCodegen (28)
+                                                              Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year]
+                                                                BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                                  Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                    BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                                                      Filter [cs_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk]
+                                                                              SubqueryBroadcast [d_date_sk] #2
+                                                                                ReusedExchange [d_date_sk,d_year] #16
+                                                                      InputAdapter
+                                                                        ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6
+                                                                  InputAdapter
+                                                                    BroadcastExchange #16
+                                                                      WholeStageCodegen (27)
+                                                                        Filter [d_year,d_date_sk]
+                                                                          ColumnarToRow
+                                                                            InputAdapter
+                                                                              Scan parquet default.date_dim [d_date_sk,d_year]
+                                                  InputAdapter
+                                                    WholeStageCodegen (31)
+                                                      Sort [cr_order_number,cr_item_sk]
+                                                        InputAdapter
+                                                          ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7
+                                            WholeStageCodegen (39)
+                                              Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt]
+                                                SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (36)
+                                                      Sort [ss_ticket_number,ss_item_sk]
+                                                        InputAdapter
+                                                          Exchange [ss_ticket_number,ss_item_sk] #17
+                                                            WholeStageCodegen (35)
+                                                              Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year]
+                                                                BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                                  Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                                      Filter [ss_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk]
+                                                                              ReusedSubquery [d_date_sk] #2
+                                                                      InputAdapter
+                                                                        ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6
+                                                                  InputAdapter
+                                                                    ReusedExchange [d_date_sk,d_year] #16
+                                                  InputAdapter
+                                                    WholeStageCodegen (38)
+                                                      Sort [sr_ticket_number,sr_item_sk]
+                                                        InputAdapter
+                                                          ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9
+                                            WholeStageCodegen (46)
+                                              Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt]
+                                                SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk]
+                                                  InputAdapter
+                                                    WholeStageCodegen (43)
+                                                      Sort [ws_order_number,ws_item_sk]
+                                                        InputAdapter
+                                                          Exchange [ws_order_number,ws_item_sk] #18
+                                                            WholeStageCodegen (42)
+                                                              Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year]
+                                                                BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                                  Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id]
+                                                                    BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                                                      Filter [ws_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk]
+                                                                              ReusedSubquery [d_date_sk] #2
+                                                                      InputAdapter
+                                                                        ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6
+                                                                  InputAdapter
+                                                                    ReusedExchange [d_date_sk,d_year] #16
+                                                  InputAdapter
+                                                    WholeStageCodegen (45)
+                                                      Sort [wr_order_number,wr_item_sk]
+                                                        InputAdapter
+                                                          ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.sf100/explain.txt
new file mode 100644
index 0000000000000..4c884e185904f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.sf100/explain.txt
@@ -0,0 +1,619 @@
+== Physical Plan ==
+TakeOrderedAndProject (102)
++- * HashAggregate (101)
+   +- Exchange (100)
+      +- * HashAggregate (99)
+         +- Union (98)
+            :- * HashAggregate (87)
+            :  +- Exchange (86)
+            :     +- * HashAggregate (85)
+            :        +- Union (84)
+            :           :- * Project (34)
+            :           :  +- * BroadcastHashJoin LeftOuter BuildRight (33)
+            :           :     :- * HashAggregate (19)
+            :           :     :  +- Exchange (18)
+            :           :     :     +- * HashAggregate (17)
+            :           :     :        +- * Project (16)
+            :           :     :           +- * BroadcastHashJoin Inner BuildRight (15)
+            :           :     :              :- * Project (10)
+            :           :     :              :  +- * BroadcastHashJoin Inner BuildRight (9)
+            :           :     :              :     :- * Filter (3)
+            :           :     :              :     :  +- * ColumnarToRow (2)
+            :           :     :              :     :     +- Scan parquet default.store_sales (1)
+            :           :     :              :     +- BroadcastExchange (8)
+            :           :     :              :        +- * Project (7)
+            :           :     :              :           +- * Filter (6)
+            :           :     :              :              +- * ColumnarToRow (5)
+            :           :     :              :                 +- Scan parquet default.date_dim (4)
+            :           :     :              +- BroadcastExchange (14)
+            :           :     :                 +- * Filter (13)
+            :           :     :                    +- * ColumnarToRow (12)
+            :           :     :                       +- Scan parquet default.store (11)
+            :           :     +- BroadcastExchange (32)
+            :           :        +- * HashAggregate (31)
+            :           :           +- Exchange (30)
+            :           :              +- * HashAggregate (29)
+            :           :                 +- * Project (28)
+            :           :                    +- * BroadcastHashJoin Inner BuildRight (27)
+            :           :                       :- * Project (25)
+            :           :                       :  +- * BroadcastHashJoin Inner BuildRight (24)
+            :           :                       :     :- * Filter (22)
+            :           :                       :     :  +- * ColumnarToRow (21)
+            :           :                       :     :     +- Scan parquet default.store_returns (20)
+            :           :                       :     +- ReusedExchange (23)
+            :           :                       +- ReusedExchange (26)
+            :           :- * Project (53)
+            :           :  +- * BroadcastNestedLoopJoin Inner BuildRight (52)
+            :           :     :- * HashAggregate (42)
+            :           :     :  +- Exchange (41)
+            :           :     :     +- * HashAggregate (40)
+            :           :     :        +- * Project (39)
+            :           :     :           +- * BroadcastHashJoin Inner BuildRight (38)
+            :           :     :              :- * ColumnarToRow (36)
+            :           :     :              :  +- Scan parquet default.catalog_sales (35)
+            :           :     :              +- ReusedExchange (37)
+            :           :     +- BroadcastExchange (51)
+            :           :        +- * HashAggregate (50)
+            :           :           +- Exchange (49)
+            :           :              +- * HashAggregate (48)
+            :           :                 +- * Project (47)
+            :           :                    +- * BroadcastHashJoin Inner BuildRight (46)
+            :           :                       :- * ColumnarToRow (44)
+            :           :                       :  +- Scan parquet default.catalog_returns (43)
+            :           :                       +- ReusedExchange (45)
+            :           +- * Project (83)
+            :              +- * BroadcastHashJoin LeftOuter BuildRight (82)
+            :                 :- * HashAggregate (68)
+            :                 :  +- Exchange (67)
+            :                 :     +- * HashAggregate (66)
+            :                 :        +- * Project (65)
+            :                 :           +- * BroadcastHashJoin Inner BuildRight (64)
+            :                 :              :- * Project (59)
+            :                 :              :  +- * BroadcastHashJoin Inner BuildRight (58)
+            :                 :              :     :- * Filter (56)
+            :                 :              :     :  +- * ColumnarToRow (55)
+            :                 :              :     :     +- Scan parquet default.web_sales (54)
+            :                 :              :     +- ReusedExchange (57)
+            :                 :              +- BroadcastExchange (63)
+            :                 :                 +- * Filter (62)
+            :                 :                    +- * ColumnarToRow (61)
+            :                 :                       +- Scan parquet default.web_page (60)
+            :                 +- BroadcastExchange (81)
+            :                    +- * HashAggregate (80)
+            :                       +- Exchange (79)
+            :                          +- * HashAggregate (78)
+            :                             +- * Project (77)
+            :                                +- * BroadcastHashJoin Inner BuildRight (76)
+            :                                   :- * Project (74)
+            :                                   :  +- * BroadcastHashJoin Inner BuildRight (73)
+            :                                   :     :- * Filter (71)
+            :                                   :     :  +- * ColumnarToRow (70)
+            :                                   :     :     +- Scan parquet default.web_returns (69)
+            :                                   :     +- ReusedExchange (72)
+            :                                   +- ReusedExchange (75)
+            :- * HashAggregate (92)
+            :  +- Exchange (91)
+            :     +- * HashAggregate (90)
+            :        +- * HashAggregate (89)
+            :           +- ReusedExchange (88)
+            +- * HashAggregate (97)
+               +- Exchange (96)
+                  +- * HashAggregate (95)
+                     +- * HashAggregate (94)
+                        +- ReusedExchange (93)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_ext_sales_price:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 3]
+Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+Condition : isnotnull(ss_store_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#6, d_date#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#6, d_date#7]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#6, d_date#7]
+Condition : (((isnotnull(d_date#7) AND (d_date#7 >= 1998-08-04)) AND (d_date#7 <= 1998-09-03)) AND isnotnull(d_date_sk#6))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#6]
+Input [2]: [d_date_sk#6, d_date#7]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#8]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3]
+Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#6]
+
+(11) Scan parquet default.store
+Output [1]: [s_store_sk#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [1]: [s_store_sk#9]
+
+(13) Filter [codegen id : 2]
+Input [1]: [s_store_sk#9]
+Condition : isnotnull(s_store_sk#9)
+
+(14) BroadcastExchange
+Input [1]: [s_store_sk#9]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#10]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_store_sk#1]
+Right keys [1]: [s_store_sk#9]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#9]
+Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#9]
+
+(17) HashAggregate [codegen id : 3]
+Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#9]
+Keys [1]: [s_store_sk#9]
+Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))]
+Aggregate Attributes [2]: [sum#11, sum#12]
+Results [3]: [s_store_sk#9, sum#13, sum#14]
+
+(18) Exchange
+Input [3]: [s_store_sk#9, sum#13, sum#14]
+Arguments: hashpartitioning(s_store_sk#9, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(19) HashAggregate [codegen id : 8]
+Input [3]: [s_store_sk#9, sum#13, sum#14]
+Keys [1]: [s_store_sk#9]
+Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ss_ext_sales_price#2))#16, sum(UnscaledValue(ss_net_profit#3))#17]
+Results [3]: [s_store_sk#9, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#16,17,2) AS sales#18, MakeDecimal(sum(UnscaledValue(ss_net_profit#3))#17,17,2) AS profit#19]
+
+(20) Scan parquet default.store_returns
+Output [4]: [sr_store_sk#20, sr_return_amt#21, sr_net_loss#22, sr_returned_date_sk#23]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(sr_returned_date_sk#23), dynamicpruningexpression(sr_returned_date_sk#23 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(sr_store_sk)]
+ReadSchema: struct<sr_store_sk:int,sr_return_amt:decimal(7,2),sr_net_loss:decimal(7,2)>
+
+(21) ColumnarToRow [codegen id : 6]
+Input [4]: [sr_store_sk#20, sr_return_amt#21, sr_net_loss#22, sr_returned_date_sk#23]
+
+(22) Filter [codegen id : 6]
+Input [4]: [sr_store_sk#20, sr_return_amt#21, sr_net_loss#22, sr_returned_date_sk#23]
+Condition : isnotnull(sr_store_sk#20)
+
+(23) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#24]
+
+(24) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [sr_returned_date_sk#23]
+Right keys [1]: [d_date_sk#24]
+Join condition: None
+
+(25) Project [codegen id : 6]
+Output [3]: [sr_store_sk#20, sr_return_amt#21, sr_net_loss#22]
+Input [5]: [sr_store_sk#20, sr_return_amt#21, sr_net_loss#22, sr_returned_date_sk#23, d_date_sk#24]
+
+(26) ReusedExchange [Reuses operator id: 14]
+Output [1]: [s_store_sk#25]
+
+(27) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [sr_store_sk#20]
+Right keys [1]: [s_store_sk#25]
+Join condition: None
+
+(28) Project [codegen id : 6]
+Output [3]: [sr_return_amt#21, sr_net_loss#22, s_store_sk#25]
+Input [4]: [sr_store_sk#20, sr_return_amt#21, sr_net_loss#22, s_store_sk#25]
+
+(29) HashAggregate [codegen id : 6]
+Input [3]: [sr_return_amt#21, sr_net_loss#22, s_store_sk#25]
+Keys [1]: [s_store_sk#25]
+Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#21)), partial_sum(UnscaledValue(sr_net_loss#22))]
+Aggregate Attributes [2]: [sum#26, sum#27]
+Results [3]: [s_store_sk#25, sum#28, sum#29]
+
+(30) Exchange
+Input [3]: [s_store_sk#25, sum#28, sum#29]
+Arguments: hashpartitioning(s_store_sk#25, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(31) HashAggregate [codegen id : 7]
+Input [3]: [s_store_sk#25, sum#28, sum#29]
+Keys [1]: [s_store_sk#25]
+Functions [2]: [sum(UnscaledValue(sr_return_amt#21)), sum(UnscaledValue(sr_net_loss#22))]
+Aggregate Attributes [2]: [sum(UnscaledValue(sr_return_amt#21))#31, sum(UnscaledValue(sr_net_loss#22))#32]
+Results [3]: [s_store_sk#25, MakeDecimal(sum(UnscaledValue(sr_return_amt#21))#31,17,2) AS returns#33, MakeDecimal(sum(UnscaledValue(sr_net_loss#22))#32,17,2) AS profit_loss#34]
+
+(32) BroadcastExchange
+Input [3]: [s_store_sk#25, returns#33, profit_loss#34]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#35]
+
+(33) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [s_store_sk#9]
+Right keys [1]: [s_store_sk#25]
+Join condition: None
+
+(34) Project [codegen id : 8]
+Output [5]: [store channel AS channel#36, s_store_sk#9 AS id#37, sales#18, coalesce(returns#33, 0.00) AS returns#38, CheckOverflow((promote_precision(cast(profit#19 as decimal(18,2))) - promote_precision(cast(coalesce(profit_loss#34, 0.00) as decimal(18,2)))), DecimalType(18,2), true) AS profit#39]
+Input [6]: [s_store_sk#9, sales#18, profit#19, s_store_sk#25, returns#33, profit_loss#34]
+
+(35) Scan parquet default.catalog_sales
+Output [4]: [cs_call_center_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#43), dynamicpruningexpression(cs_sold_date_sk#43 IN dynamicpruning#5)]
+ReadSchema: struct<cs_call_center_sk:int,cs_ext_sales_price:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(36) ColumnarToRow [codegen id : 10]
+Input [4]: [cs_call_center_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43]
+
+(37) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#44]
+
+(38) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_sold_date_sk#43]
+Right keys [1]: [d_date_sk#44]
+Join condition: None
+
+(39) Project [codegen id : 10]
+Output [3]: [cs_call_center_sk#40, cs_ext_sales_price#41, cs_net_profit#42]
+Input [5]: [cs_call_center_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43, d_date_sk#44]
+
+(40) HashAggregate [codegen id : 10]
+Input [3]: [cs_call_center_sk#40, cs_ext_sales_price#41, cs_net_profit#42]
+Keys [1]: [cs_call_center_sk#40]
+Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#41)), partial_sum(UnscaledValue(cs_net_profit#42))]
+Aggregate Attributes [2]: [sum#45, sum#46]
+Results [3]: [cs_call_center_sk#40, sum#47, sum#48]
+
+(41) Exchange
+Input [3]: [cs_call_center_sk#40, sum#47, sum#48]
+Arguments: hashpartitioning(cs_call_center_sk#40, 5), ENSURE_REQUIREMENTS, [id=#49]
+
+(42) HashAggregate [codegen id : 14]
+Input [3]: [cs_call_center_sk#40, sum#47, sum#48]
+Keys [1]: [cs_call_center_sk#40]
+Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#41)), sum(UnscaledValue(cs_net_profit#42))]
+Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_sales_price#41))#50, sum(UnscaledValue(cs_net_profit#42))#51]
+Results [3]: [cs_call_center_sk#40, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#41))#50,17,2) AS sales#52, MakeDecimal(sum(UnscaledValue(cs_net_profit#42))#51,17,2) AS profit#53]
+
+(43) Scan parquet default.catalog_returns
+Output [3]: [cr_return_amount#54, cr_net_loss#55, cr_returned_date_sk#56]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cr_returned_date_sk#56), dynamicpruningexpression(cr_returned_date_sk#56 IN dynamicpruning#5)]
+ReadSchema: struct<cr_return_amount:decimal(7,2),cr_net_loss:decimal(7,2)>
+
+(44) ColumnarToRow [codegen id : 12]
+Input [3]: [cr_return_amount#54, cr_net_loss#55, cr_returned_date_sk#56]
+
+(45) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#57]
+
+(46) BroadcastHashJoin [codegen id : 12]
+Left keys [1]: [cr_returned_date_sk#56]
+Right keys [1]: [d_date_sk#57]
+Join condition: None
+
+(47) Project [codegen id : 12]
+Output [2]: [cr_return_amount#54, cr_net_loss#55]
+Input [4]: [cr_return_amount#54, cr_net_loss#55, cr_returned_date_sk#56, d_date_sk#57]
+
+(48) HashAggregate [codegen id : 12]
+Input [2]: [cr_return_amount#54, cr_net_loss#55]
+Keys: []
+Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#54)), partial_sum(UnscaledValue(cr_net_loss#55))]
+Aggregate Attributes [2]: [sum#58, sum#59]
+Results [2]: [sum#60, sum#61]
+
+(49) Exchange
+Input [2]: [sum#60, sum#61]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#62]
+
+(50) HashAggregate [codegen id : 13]
+Input [2]: [sum#60, sum#61]
+Keys: []
+Functions [2]: [sum(UnscaledValue(cr_return_amount#54)), sum(UnscaledValue(cr_net_loss#55))]
+Aggregate Attributes [2]: [sum(UnscaledValue(cr_return_amount#54))#63, sum(UnscaledValue(cr_net_loss#55))#64]
+Results [2]: [MakeDecimal(sum(UnscaledValue(cr_return_amount#54))#63,17,2) AS returns#65, MakeDecimal(sum(UnscaledValue(cr_net_loss#55))#64,17,2) AS profit_loss#66]
+
+(51) BroadcastExchange
+Input [2]: [returns#65, profit_loss#66]
+Arguments: IdentityBroadcastMode, [id=#67]
+
+(52) BroadcastNestedLoopJoin [codegen id : 14]
+Join condition: None
+
+(53) Project [codegen id : 14]
+Output [5]: [catalog channel AS channel#68, cs_call_center_sk#40 AS id#69, sales#52, returns#65, CheckOverflow((promote_precision(cast(profit#53 as decimal(18,2))) - promote_precision(cast(profit_loss#66 as decimal(18,2)))), DecimalType(18,2), true) AS profit#70]
+Input [5]: [cs_call_center_sk#40, sales#52, profit#53, returns#65, profit_loss#66]
+
+(54) Scan parquet default.web_sales
+Output [4]: [ws_web_page_sk#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#74), dynamicpruningexpression(ws_sold_date_sk#74 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ws_web_page_sk)]
+ReadSchema: struct<ws_web_page_sk:int,ws_ext_sales_price:decimal(7,2),ws_net_profit:decimal(7,2)>
+
+(55) ColumnarToRow [codegen id : 17]
+Input [4]: [ws_web_page_sk#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74]
+
+(56) Filter [codegen id : 17]
+Input [4]: [ws_web_page_sk#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74]
+Condition : isnotnull(ws_web_page_sk#71)
+
+(57) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#75]
+
+(58) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_sold_date_sk#74]
+Right keys [1]: [d_date_sk#75]
+Join condition: None
+
+(59) Project [codegen id : 17]
+Output [3]: [ws_web_page_sk#71, ws_ext_sales_price#72, ws_net_profit#73]
+Input [5]: [ws_web_page_sk#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, d_date_sk#75]
+
+(60) Scan parquet default.web_page
+Output [1]: [wp_web_page_sk#76]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_page]
+PushedFilters: [IsNotNull(wp_web_page_sk)]
+ReadSchema: struct<wp_web_page_sk:int>
+
+(61) ColumnarToRow [codegen id : 16]
+Input [1]: [wp_web_page_sk#76]
+
+(62) Filter [codegen id : 16]
+Input [1]: [wp_web_page_sk#76]
+Condition : isnotnull(wp_web_page_sk#76)
+
+(63) BroadcastExchange
+Input [1]: [wp_web_page_sk#76]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#77]
+
+(64) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_web_page_sk#71]
+Right keys [1]: [wp_web_page_sk#76]
+Join condition: None
+
+(65) Project [codegen id : 17]
+Output [3]: [ws_ext_sales_price#72, ws_net_profit#73, wp_web_page_sk#76]
+Input [4]: [ws_web_page_sk#71, ws_ext_sales_price#72, ws_net_profit#73, wp_web_page_sk#76]
+
+(66) HashAggregate [codegen id : 17]
+Input [3]: [ws_ext_sales_price#72, ws_net_profit#73, wp_web_page_sk#76]
+Keys [1]: [wp_web_page_sk#76]
+Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#72)), partial_sum(UnscaledValue(ws_net_profit#73))]
+Aggregate Attributes [2]: [sum#78, sum#79]
+Results [3]: [wp_web_page_sk#76, sum#80, sum#81]
+
+(67) Exchange
+Input [3]: [wp_web_page_sk#76, sum#80, sum#81]
+Arguments: hashpartitioning(wp_web_page_sk#76, 5), ENSURE_REQUIREMENTS, [id=#82]
+
+(68) HashAggregate [codegen id : 22]
+Input [3]: [wp_web_page_sk#76, sum#80, sum#81]
+Keys [1]: [wp_web_page_sk#76]
+Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#72)), sum(UnscaledValue(ws_net_profit#73))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_sales_price#72))#83, sum(UnscaledValue(ws_net_profit#73))#84]
+Results [3]: [wp_web_page_sk#76, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#72))#83,17,2) AS sales#85, MakeDecimal(sum(UnscaledValue(ws_net_profit#73))#84,17,2) AS profit#86]
+
+(69) Scan parquet default.web_returns
+Output [4]: [wr_web_page_sk#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(wr_returned_date_sk#90), dynamicpruningexpression(wr_returned_date_sk#90 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(wr_web_page_sk)]
+ReadSchema: struct<wr_web_page_sk:int,wr_return_amt:decimal(7,2),wr_net_loss:decimal(7,2)>
+
+(70) ColumnarToRow [codegen id : 20]
+Input [4]: [wr_web_page_sk#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90]
+
+(71) Filter [codegen id : 20]
+Input [4]: [wr_web_page_sk#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90]
+Condition : isnotnull(wr_web_page_sk#87)
+
+(72) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#91]
+
+(73) BroadcastHashJoin [codegen id : 20]
+Left keys [1]: [wr_returned_date_sk#90]
+Right keys [1]: [d_date_sk#91]
+Join condition: None
+
+(74) Project [codegen id : 20]
+Output [3]: [wr_web_page_sk#87, wr_return_amt#88, wr_net_loss#89]
+Input [5]: [wr_web_page_sk#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90, d_date_sk#91]
+
+(75) ReusedExchange [Reuses operator id: 63]
+Output [1]: [wp_web_page_sk#92]
+
+(76) BroadcastHashJoin [codegen id : 20]
+Left keys [1]: [wr_web_page_sk#87]
+Right keys [1]: [wp_web_page_sk#92]
+Join condition: None
+
+(77) Project [codegen id : 20]
+Output [3]: [wr_return_amt#88, wr_net_loss#89, wp_web_page_sk#92]
+Input [4]: [wr_web_page_sk#87, wr_return_amt#88, wr_net_loss#89, wp_web_page_sk#92]
+
+(78) HashAggregate [codegen id : 20]
+Input [3]: [wr_return_amt#88, wr_net_loss#89, wp_web_page_sk#92]
+Keys [1]: [wp_web_page_sk#92]
+Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#88)), partial_sum(UnscaledValue(wr_net_loss#89))]
+Aggregate Attributes [2]: [sum#93, sum#94]
+Results [3]: [wp_web_page_sk#92, sum#95, sum#96]
+
+(79) Exchange
+Input [3]: [wp_web_page_sk#92, sum#95, sum#96]
+Arguments: hashpartitioning(wp_web_page_sk#92, 5), ENSURE_REQUIREMENTS, [id=#97]
+
+(80) HashAggregate [codegen id : 21]
+Input [3]: [wp_web_page_sk#92, sum#95, sum#96]
+Keys [1]: [wp_web_page_sk#92]
+Functions [2]: [sum(UnscaledValue(wr_return_amt#88)), sum(UnscaledValue(wr_net_loss#89))]
+Aggregate Attributes [2]: [sum(UnscaledValue(wr_return_amt#88))#98, sum(UnscaledValue(wr_net_loss#89))#99]
+Results [3]: [wp_web_page_sk#92, MakeDecimal(sum(UnscaledValue(wr_return_amt#88))#98,17,2) AS returns#100, MakeDecimal(sum(UnscaledValue(wr_net_loss#89))#99,17,2) AS profit_loss#101]
+
+(81) BroadcastExchange
+Input [3]: [wp_web_page_sk#92, returns#100, profit_loss#101]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#102]
+
+(82) BroadcastHashJoin [codegen id : 22]
+Left keys [1]: [wp_web_page_sk#76]
+Right keys [1]: [wp_web_page_sk#92]
+Join condition: None
+
+(83) Project [codegen id : 22]
+Output [5]: [web channel AS channel#103, wp_web_page_sk#76 AS id#104, sales#85, coalesce(returns#100, 0.00) AS returns#105, CheckOverflow((promote_precision(cast(profit#86 as decimal(18,2))) - promote_precision(cast(coalesce(profit_loss#101, 0.00) as decimal(18,2)))), DecimalType(18,2), true) AS profit#106]
+Input [6]: [wp_web_page_sk#76, sales#85, profit#86, wp_web_page_sk#92, returns#100, profit_loss#101]
+
+(84) Union
+
+(85) HashAggregate [codegen id : 23]
+Input [5]: [channel#36, id#37, sales#18, returns#38, profit#39]
+Keys [2]: [channel#36, id#37]
+Functions [3]: [partial_sum(sales#18), partial_sum(returns#38), partial_sum(profit#39)]
+Aggregate Attributes [6]: [sum#107, isEmpty#108, sum#109, isEmpty#110, sum#111, isEmpty#112]
+Results [8]: [channel#36, id#37, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118]
+
+(86) Exchange
+Input [8]: [channel#36, id#37, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118]
+Arguments: hashpartitioning(channel#36, id#37, 5), ENSURE_REQUIREMENTS, [id=#119]
+
+(87) HashAggregate [codegen id : 24]
+Input [8]: [channel#36, id#37, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118]
+Keys [2]: [channel#36, id#37]
+Functions [3]: [sum(sales#18), sum(returns#38), sum(profit#39)]
+Aggregate Attributes [3]: [sum(sales#18)#120, sum(returns#38)#121, sum(profit#39)#122]
+Results [5]: [channel#36, id#37, cast(sum(sales#18)#120 as decimal(37,2)) AS sales#123, cast(sum(returns#38)#121 as decimal(37,2)) AS returns#124, cast(sum(profit#39)#122 as decimal(38,2)) AS profit#125]
+
+(88) ReusedExchange [Reuses operator id: unknown]
+Output [8]: [channel#36, id#37, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131]
+
+(89) HashAggregate [codegen id : 48]
+Input [8]: [channel#36, id#37, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131]
+Keys [2]: [channel#36, id#37]
+Functions [3]: [sum(sales#18), sum(returns#38), sum(profit#39)]
+Aggregate Attributes [3]: [sum(sales#18)#132, sum(returns#38)#133, sum(profit#39)#134]
+Results [4]: [channel#36, sum(sales#18)#132 AS sales#135, sum(returns#38)#133 AS returns#136, sum(profit#39)#134 AS profit#137]
+
+(90) HashAggregate [codegen id : 48]
+Input [4]: [channel#36, sales#135, returns#136, profit#137]
+Keys [1]: [channel#36]
+Functions [3]: [partial_sum(sales#135), partial_sum(returns#136), partial_sum(profit#137)]
+Aggregate Attributes [6]: [sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143]
+Results [7]: [channel#36, sum#144, isEmpty#145, sum#146, isEmpty#147, sum#148, isEmpty#149]
+
+(91) Exchange
+Input [7]: [channel#36, sum#144, isEmpty#145, sum#146, isEmpty#147, sum#148, isEmpty#149]
+Arguments: hashpartitioning(channel#36, 5), ENSURE_REQUIREMENTS, [id=#150]
+
+(92) HashAggregate [codegen id : 49]
+Input [7]: [channel#36, sum#144, isEmpty#145, sum#146, isEmpty#147, sum#148, isEmpty#149]
+Keys [1]: [channel#36]
+Functions [3]: [sum(sales#135), sum(returns#136), sum(profit#137)]
+Aggregate Attributes [3]: [sum(sales#135)#151, sum(returns#136)#152, sum(profit#137)#153]
+Results [5]: [channel#36, null AS id#154, sum(sales#135)#151 AS sales#155, sum(returns#136)#152 AS returns#156, sum(profit#137)#153 AS profit#157]
+
+(93) ReusedExchange [Reuses operator id: unknown]
+Output [8]: [channel#36, id#37, sum#158, isEmpty#159, sum#160, isEmpty#161, sum#162, isEmpty#163]
+
+(94) HashAggregate [codegen id : 73]
+Input [8]: [channel#36, id#37, sum#158, isEmpty#159, sum#160, isEmpty#161, sum#162, isEmpty#163]
+Keys [2]: [channel#36, id#37]
+Functions [3]: [sum(sales#18), sum(returns#38), sum(profit#39)]
+Aggregate Attributes [3]: [sum(sales#18)#164, sum(returns#38)#165, sum(profit#39)#166]
+Results [3]: [sum(sales#18)#164 AS sales#135, sum(returns#38)#165 AS returns#136, sum(profit#39)#166 AS profit#137]
+
+(95) HashAggregate [codegen id : 73]
+Input [3]: [sales#135, returns#136, profit#137]
+Keys: []
+Functions [3]: [partial_sum(sales#135), partial_sum(returns#136), partial_sum(profit#137)]
+Aggregate Attributes [6]: [sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172]
+Results [6]: [sum#173, isEmpty#174, sum#175, isEmpty#176, sum#177, isEmpty#178]
+
+(96) Exchange
+Input [6]: [sum#173, isEmpty#174, sum#175, isEmpty#176, sum#177, isEmpty#178]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#179]
+
+(97) HashAggregate [codegen id : 74]
+Input [6]: [sum#173, isEmpty#174, sum#175, isEmpty#176, sum#177, isEmpty#178]
+Keys: []
+Functions [3]: [sum(sales#135), sum(returns#136), sum(profit#137)]
+Aggregate Attributes [3]: [sum(sales#135)#180, sum(returns#136)#181, sum(profit#137)#182]
+Results [5]: [null AS channel#183, null AS id#184, sum(sales#135)#180 AS sales#185, sum(returns#136)#181 AS returns#186, sum(profit#137)#182 AS profit#187]
+
+(98) Union
+
+(99) HashAggregate [codegen id : 75]
+Input [5]: [channel#36, id#37, sales#123, returns#124, profit#125]
+Keys [5]: [channel#36, id#37, sales#123, returns#124, profit#125]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [channel#36, id#37, sales#123, returns#124, profit#125]
+
+(100) Exchange
+Input [5]: [channel#36, id#37, sales#123, returns#124, profit#125]
+Arguments: hashpartitioning(channel#36, id#37, sales#123, returns#124, profit#125, 5), ENSURE_REQUIREMENTS, [id=#188]
+
+(101) HashAggregate [codegen id : 76]
+Input [5]: [channel#36, id#37, sales#123, returns#124, profit#125]
+Keys [5]: [channel#36, id#37, sales#123, returns#124, profit#125]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [channel#36, id#37, sales#123, returns#124, profit#125]
+
+(102) TakeOrderedAndProject
+Input [5]: [channel#36, id#37, sales#123, returns#124, profit#125]
+Arguments: 100, [channel#36 ASC NULLS FIRST, id#37 ASC NULLS FIRST], [channel#36, id#37, sales#123, returns#124, profit#125]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (103)
+
+
+(103) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#6]
+
+Subquery:2 Hosting operator id = 20 Hosting Expression = sr_returned_date_sk#23 IN dynamicpruning#5
+
+Subquery:3 Hosting operator id = 35 Hosting Expression = cs_sold_date_sk#43 IN dynamicpruning#5
+
+Subquery:4 Hosting operator id = 43 Hosting Expression = cr_returned_date_sk#56 IN dynamicpruning#5
+
+Subquery:5 Hosting operator id = 54 Hosting Expression = ws_sold_date_sk#74 IN dynamicpruning#5
+
+Subquery:6 Hosting operator id = 69 Hosting Expression = wr_returned_date_sk#90 IN dynamicpruning#5
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.sf100/simplified.txt
new file mode 100644
index 0000000000000..4cadc7fb960a2
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.sf100/simplified.txt
@@ -0,0 +1,168 @@
+TakeOrderedAndProject [channel,id,sales,returns,profit]
+  WholeStageCodegen (76)
+    HashAggregate [channel,id,sales,returns,profit]
+      InputAdapter
+        Exchange [channel,id,sales,returns,profit] #1
+          WholeStageCodegen (75)
+            HashAggregate [channel,id,sales,returns,profit]
+              InputAdapter
+                Union
+                  WholeStageCodegen (24)
+                    HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                      InputAdapter
+                        Exchange [channel,id] #2
+                          WholeStageCodegen (23)
+                            HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                              InputAdapter
+                                Union
+                                  WholeStageCodegen (8)
+                                    Project [s_store_sk,sales,returns,profit,profit_loss]
+                                      BroadcastHashJoin [s_store_sk,s_store_sk]
+                                        HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit)),sales,profit,sum,sum]
+                                          InputAdapter
+                                            Exchange [s_store_sk] #3
+                                              WholeStageCodegen (3)
+                                                HashAggregate [s_store_sk,ss_ext_sales_price,ss_net_profit] [sum,sum,sum,sum]
+                                                  Project [ss_ext_sales_price,ss_net_profit,s_store_sk]
+                                                    BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                      Project [ss_store_sk,ss_ext_sales_price,ss_net_profit]
+                                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                          Filter [ss_store_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk]
+                                                                  SubqueryBroadcast [d_date_sk] #1
+                                                                    ReusedExchange [d_date_sk] #4
+                                                          InputAdapter
+                                                            BroadcastExchange #4
+                                                              WholeStageCodegen (1)
+                                                                Project [d_date_sk]
+                                                                  Filter [d_date,d_date_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.date_dim [d_date_sk,d_date]
+                                                      InputAdapter
+                                                        BroadcastExchange #5
+                                                          WholeStageCodegen (2)
+                                                            Filter [s_store_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.store [s_store_sk]
+                                        InputAdapter
+                                          BroadcastExchange #6
+                                            WholeStageCodegen (7)
+                                              HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss)),returns,profit_loss,sum,sum]
+                                                InputAdapter
+                                                  Exchange [s_store_sk] #7
+                                                    WholeStageCodegen (6)
+                                                      HashAggregate [s_store_sk,sr_return_amt,sr_net_loss] [sum,sum,sum,sum]
+                                                        Project [sr_return_amt,sr_net_loss,s_store_sk]
+                                                          BroadcastHashJoin [sr_store_sk,s_store_sk]
+                                                            Project [sr_store_sk,sr_return_amt,sr_net_loss]
+                                                              BroadcastHashJoin [sr_returned_date_sk,d_date_sk]
+                                                                Filter [sr_store_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk]
+                                                                        ReusedSubquery [d_date_sk] #1
+                                                                InputAdapter
+                                                                  ReusedExchange [d_date_sk] #4
+                                                            InputAdapter
+                                                              ReusedExchange [s_store_sk] #5
+                                  WholeStageCodegen (14)
+                                    Project [cs_call_center_sk,sales,returns,profit,profit_loss]
+                                      BroadcastNestedLoopJoin
+                                        HashAggregate [cs_call_center_sk,sum,sum] [sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit)),sales,profit,sum,sum]
+                                          InputAdapter
+                                            Exchange [cs_call_center_sk] #8
+                                              WholeStageCodegen (10)
+                                                HashAggregate [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] [sum,sum,sum,sum]
+                                                  Project [cs_call_center_sk,cs_ext_sales_price,cs_net_profit]
+                                                    BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk]
+                                                            ReusedSubquery [d_date_sk] #1
+                                                      InputAdapter
+                                                        ReusedExchange [d_date_sk] #4
+                                        InputAdapter
+                                          BroadcastExchange #9
+                                            WholeStageCodegen (13)
+                                              HashAggregate [sum,sum] [sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss)),returns,profit_loss,sum,sum]
+                                                InputAdapter
+                                                  Exchange #10
+                                                    WholeStageCodegen (12)
+                                                      HashAggregate [cr_return_amount,cr_net_loss] [sum,sum,sum,sum]
+                                                        Project [cr_return_amount,cr_net_loss]
+                                                          BroadcastHashJoin [cr_returned_date_sk,d_date_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk]
+                                                                  ReusedSubquery [d_date_sk] #1
+                                                            InputAdapter
+                                                              ReusedExchange [d_date_sk] #4
+                                  WholeStageCodegen (22)
+                                    Project [wp_web_page_sk,sales,returns,profit,profit_loss]
+                                      BroadcastHashJoin [wp_web_page_sk,wp_web_page_sk]
+                                        HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit)),sales,profit,sum,sum]
+                                          InputAdapter
+                                            Exchange [wp_web_page_sk] #11
+                                              WholeStageCodegen (17)
+                                                HashAggregate [wp_web_page_sk,ws_ext_sales_price,ws_net_profit] [sum,sum,sum,sum]
+                                                  Project [ws_ext_sales_price,ws_net_profit,wp_web_page_sk]
+                                                    BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk]
+                                                      Project [ws_web_page_sk,ws_ext_sales_price,ws_net_profit]
+                                                        BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                          Filter [ws_web_page_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk]
+                                                                  ReusedSubquery [d_date_sk] #1
+                                                          InputAdapter
+                                                            ReusedExchange [d_date_sk] #4
+                                                      InputAdapter
+                                                        BroadcastExchange #12
+                                                          WholeStageCodegen (16)
+                                                            Filter [wp_web_page_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.web_page [wp_web_page_sk]
+                                        InputAdapter
+                                          BroadcastExchange #13
+                                            WholeStageCodegen (21)
+                                              HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss)),returns,profit_loss,sum,sum]
+                                                InputAdapter
+                                                  Exchange [wp_web_page_sk] #14
+                                                    WholeStageCodegen (20)
+                                                      HashAggregate [wp_web_page_sk,wr_return_amt,wr_net_loss] [sum,sum,sum,sum]
+                                                        Project [wr_return_amt,wr_net_loss,wp_web_page_sk]
+                                                          BroadcastHashJoin [wr_web_page_sk,wp_web_page_sk]
+                                                            Project [wr_web_page_sk,wr_return_amt,wr_net_loss]
+                                                              BroadcastHashJoin [wr_returned_date_sk,d_date_sk]
+                                                                Filter [wr_web_page_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk]
+                                                                        ReusedSubquery [d_date_sk] #1
+                                                                InputAdapter
+                                                                  ReusedExchange [d_date_sk] #4
+                                                            InputAdapter
+                                                              ReusedExchange [wp_web_page_sk] #12
+                  WholeStageCodegen (49)
+                    HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                      InputAdapter
+                        Exchange [channel] #15
+                          WholeStageCodegen (48)
+                            HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                              HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                                InputAdapter
+                                  ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #16
+                  WholeStageCodegen (74)
+                    HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                      InputAdapter
+                        Exchange #17
+                          WholeStageCodegen (73)
+                            HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                              HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                                InputAdapter
+                                  ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #16
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a/explain.txt
new file mode 100644
index 0000000000000..c74b44df70a65
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a/explain.txt
@@ -0,0 +1,619 @@
+== Physical Plan ==
+TakeOrderedAndProject (102)
++- * HashAggregate (101)
+   +- Exchange (100)
+      +- * HashAggregate (99)
+         +- Union (98)
+            :- * HashAggregate (87)
+            :  +- Exchange (86)
+            :     +- * HashAggregate (85)
+            :        +- Union (84)
+            :           :- * Project (34)
+            :           :  +- * BroadcastHashJoin LeftOuter BuildRight (33)
+            :           :     :- * HashAggregate (19)
+            :           :     :  +- Exchange (18)
+            :           :     :     +- * HashAggregate (17)
+            :           :     :        +- * Project (16)
+            :           :     :           +- * BroadcastHashJoin Inner BuildRight (15)
+            :           :     :              :- * Project (10)
+            :           :     :              :  +- * BroadcastHashJoin Inner BuildRight (9)
+            :           :     :              :     :- * Filter (3)
+            :           :     :              :     :  +- * ColumnarToRow (2)
+            :           :     :              :     :     +- Scan parquet default.store_sales (1)
+            :           :     :              :     +- BroadcastExchange (8)
+            :           :     :              :        +- * Project (7)
+            :           :     :              :           +- * Filter (6)
+            :           :     :              :              +- * ColumnarToRow (5)
+            :           :     :              :                 +- Scan parquet default.date_dim (4)
+            :           :     :              +- BroadcastExchange (14)
+            :           :     :                 +- * Filter (13)
+            :           :     :                    +- * ColumnarToRow (12)
+            :           :     :                       +- Scan parquet default.store (11)
+            :           :     +- BroadcastExchange (32)
+            :           :        +- * HashAggregate (31)
+            :           :           +- Exchange (30)
+            :           :              +- * HashAggregate (29)
+            :           :                 +- * Project (28)
+            :           :                    +- * BroadcastHashJoin Inner BuildRight (27)
+            :           :                       :- * Project (25)
+            :           :                       :  +- * BroadcastHashJoin Inner BuildRight (24)
+            :           :                       :     :- * Filter (22)
+            :           :                       :     :  +- * ColumnarToRow (21)
+            :           :                       :     :     +- Scan parquet default.store_returns (20)
+            :           :                       :     +- ReusedExchange (23)
+            :           :                       +- ReusedExchange (26)
+            :           :- * Project (53)
+            :           :  +- * BroadcastNestedLoopJoin Inner BuildLeft (52)
+            :           :     :- BroadcastExchange (43)
+            :           :     :  +- * HashAggregate (42)
+            :           :     :     +- Exchange (41)
+            :           :     :        +- * HashAggregate (40)
+            :           :     :           +- * Project (39)
+            :           :     :              +- * BroadcastHashJoin Inner BuildRight (38)
+            :           :     :                 :- * ColumnarToRow (36)
+            :           :     :                 :  +- Scan parquet default.catalog_sales (35)
+            :           :     :                 +- ReusedExchange (37)
+            :           :     +- * HashAggregate (51)
+            :           :        +- Exchange (50)
+            :           :           +- * HashAggregate (49)
+            :           :              +- * Project (48)
+            :           :                 +- * BroadcastHashJoin Inner BuildRight (47)
+            :           :                    :- * ColumnarToRow (45)
+            :           :                    :  +- Scan parquet default.catalog_returns (44)
+            :           :                    +- ReusedExchange (46)
+            :           +- * Project (83)
+            :              +- * BroadcastHashJoin LeftOuter BuildRight (82)
+            :                 :- * HashAggregate (68)
+            :                 :  +- Exchange (67)
+            :                 :     +- * HashAggregate (66)
+            :                 :        +- * Project (65)
+            :                 :           +- * BroadcastHashJoin Inner BuildRight (64)
+            :                 :              :- * Project (59)
+            :                 :              :  +- * BroadcastHashJoin Inner BuildRight (58)
+            :                 :              :     :- * Filter (56)
+            :                 :              :     :  +- * ColumnarToRow (55)
+            :                 :              :     :     +- Scan parquet default.web_sales (54)
+            :                 :              :     +- ReusedExchange (57)
+            :                 :              +- BroadcastExchange (63)
+            :                 :                 +- * Filter (62)
+            :                 :                    +- * ColumnarToRow (61)
+            :                 :                       +- Scan parquet default.web_page (60)
+            :                 +- BroadcastExchange (81)
+            :                    +- * HashAggregate (80)
+            :                       +- Exchange (79)
+            :                          +- * HashAggregate (78)
+            :                             +- * Project (77)
+            :                                +- * BroadcastHashJoin Inner BuildRight (76)
+            :                                   :- * Project (74)
+            :                                   :  +- * BroadcastHashJoin Inner BuildRight (73)
+            :                                   :     :- * Filter (71)
+            :                                   :     :  +- * ColumnarToRow (70)
+            :                                   :     :     +- Scan parquet default.web_returns (69)
+            :                                   :     +- ReusedExchange (72)
+            :                                   +- ReusedExchange (75)
+            :- * HashAggregate (92)
+            :  +- Exchange (91)
+            :     +- * HashAggregate (90)
+            :        +- * HashAggregate (89)
+            :           +- ReusedExchange (88)
+            +- * HashAggregate (97)
+               +- Exchange (96)
+                  +- * HashAggregate (95)
+                     +- * HashAggregate (94)
+                        +- ReusedExchange (93)
+
+
+(1) Scan parquet default.store_sales
+Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ss_store_sk)]
+ReadSchema: struct<ss_store_sk:int,ss_ext_sales_price:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+
+(3) Filter [codegen id : 3]
+Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4]
+Condition : isnotnull(ss_store_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#6, d_date#7]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#6, d_date#7]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#6, d_date#7]
+Condition : (((isnotnull(d_date#7) AND (d_date#7 >= 1998-08-04)) AND (d_date#7 <= 1998-09-03)) AND isnotnull(d_date_sk#6))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#6]
+Input [2]: [d_date_sk#6, d_date#7]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#6]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#8]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#4]
+Right keys [1]: [d_date_sk#6]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3]
+Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#6]
+
+(11) Scan parquet default.store
+Output [1]: [s_store_sk#9]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [1]: [s_store_sk#9]
+
+(13) Filter [codegen id : 2]
+Input [1]: [s_store_sk#9]
+Condition : isnotnull(s_store_sk#9)
+
+(14) BroadcastExchange
+Input [1]: [s_store_sk#9]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#10]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_store_sk#1]
+Right keys [1]: [s_store_sk#9]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#9]
+Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#9]
+
+(17) HashAggregate [codegen id : 3]
+Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#9]
+Keys [1]: [s_store_sk#9]
+Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))]
+Aggregate Attributes [2]: [sum#11, sum#12]
+Results [3]: [s_store_sk#9, sum#13, sum#14]
+
+(18) Exchange
+Input [3]: [s_store_sk#9, sum#13, sum#14]
+Arguments: hashpartitioning(s_store_sk#9, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(19) HashAggregate [codegen id : 8]
+Input [3]: [s_store_sk#9, sum#13, sum#14]
+Keys [1]: [s_store_sk#9]
+Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ss_ext_sales_price#2))#16, sum(UnscaledValue(ss_net_profit#3))#17]
+Results [3]: [s_store_sk#9, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#16,17,2) AS sales#18, MakeDecimal(sum(UnscaledValue(ss_net_profit#3))#17,17,2) AS profit#19]
+
+(20) Scan parquet default.store_returns
+Output [4]: [sr_store_sk#20, sr_return_amt#21, sr_net_loss#22, sr_returned_date_sk#23]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(sr_returned_date_sk#23), dynamicpruningexpression(sr_returned_date_sk#23 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(sr_store_sk)]
+ReadSchema: struct<sr_store_sk:int,sr_return_amt:decimal(7,2),sr_net_loss:decimal(7,2)>
+
+(21) ColumnarToRow [codegen id : 6]
+Input [4]: [sr_store_sk#20, sr_return_amt#21, sr_net_loss#22, sr_returned_date_sk#23]
+
+(22) Filter [codegen id : 6]
+Input [4]: [sr_store_sk#20, sr_return_amt#21, sr_net_loss#22, sr_returned_date_sk#23]
+Condition : isnotnull(sr_store_sk#20)
+
+(23) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#24]
+
+(24) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [sr_returned_date_sk#23]
+Right keys [1]: [d_date_sk#24]
+Join condition: None
+
+(25) Project [codegen id : 6]
+Output [3]: [sr_store_sk#20, sr_return_amt#21, sr_net_loss#22]
+Input [5]: [sr_store_sk#20, sr_return_amt#21, sr_net_loss#22, sr_returned_date_sk#23, d_date_sk#24]
+
+(26) ReusedExchange [Reuses operator id: 14]
+Output [1]: [s_store_sk#25]
+
+(27) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [sr_store_sk#20]
+Right keys [1]: [s_store_sk#25]
+Join condition: None
+
+(28) Project [codegen id : 6]
+Output [3]: [sr_return_amt#21, sr_net_loss#22, s_store_sk#25]
+Input [4]: [sr_store_sk#20, sr_return_amt#21, sr_net_loss#22, s_store_sk#25]
+
+(29) HashAggregate [codegen id : 6]
+Input [3]: [sr_return_amt#21, sr_net_loss#22, s_store_sk#25]
+Keys [1]: [s_store_sk#25]
+Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#21)), partial_sum(UnscaledValue(sr_net_loss#22))]
+Aggregate Attributes [2]: [sum#26, sum#27]
+Results [3]: [s_store_sk#25, sum#28, sum#29]
+
+(30) Exchange
+Input [3]: [s_store_sk#25, sum#28, sum#29]
+Arguments: hashpartitioning(s_store_sk#25, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(31) HashAggregate [codegen id : 7]
+Input [3]: [s_store_sk#25, sum#28, sum#29]
+Keys [1]: [s_store_sk#25]
+Functions [2]: [sum(UnscaledValue(sr_return_amt#21)), sum(UnscaledValue(sr_net_loss#22))]
+Aggregate Attributes [2]: [sum(UnscaledValue(sr_return_amt#21))#31, sum(UnscaledValue(sr_net_loss#22))#32]
+Results [3]: [s_store_sk#25, MakeDecimal(sum(UnscaledValue(sr_return_amt#21))#31,17,2) AS returns#33, MakeDecimal(sum(UnscaledValue(sr_net_loss#22))#32,17,2) AS profit_loss#34]
+
+(32) BroadcastExchange
+Input [3]: [s_store_sk#25, returns#33, profit_loss#34]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#35]
+
+(33) BroadcastHashJoin [codegen id : 8]
+Left keys [1]: [s_store_sk#9]
+Right keys [1]: [s_store_sk#25]
+Join condition: None
+
+(34) Project [codegen id : 8]
+Output [5]: [store channel AS channel#36, s_store_sk#9 AS id#37, sales#18, coalesce(returns#33, 0.00) AS returns#38, CheckOverflow((promote_precision(cast(profit#19 as decimal(18,2))) - promote_precision(cast(coalesce(profit_loss#34, 0.00) as decimal(18,2)))), DecimalType(18,2), true) AS profit#39]
+Input [6]: [s_store_sk#9, sales#18, profit#19, s_store_sk#25, returns#33, profit_loss#34]
+
+(35) Scan parquet default.catalog_sales
+Output [4]: [cs_call_center_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#43), dynamicpruningexpression(cs_sold_date_sk#43 IN dynamicpruning#5)]
+ReadSchema: struct<cs_call_center_sk:int,cs_ext_sales_price:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(36) ColumnarToRow [codegen id : 10]
+Input [4]: [cs_call_center_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43]
+
+(37) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#44]
+
+(38) BroadcastHashJoin [codegen id : 10]
+Left keys [1]: [cs_sold_date_sk#43]
+Right keys [1]: [d_date_sk#44]
+Join condition: None
+
+(39) Project [codegen id : 10]
+Output [3]: [cs_call_center_sk#40, cs_ext_sales_price#41, cs_net_profit#42]
+Input [5]: [cs_call_center_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43, d_date_sk#44]
+
+(40) HashAggregate [codegen id : 10]
+Input [3]: [cs_call_center_sk#40, cs_ext_sales_price#41, cs_net_profit#42]
+Keys [1]: [cs_call_center_sk#40]
+Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#41)), partial_sum(UnscaledValue(cs_net_profit#42))]
+Aggregate Attributes [2]: [sum#45, sum#46]
+Results [3]: [cs_call_center_sk#40, sum#47, sum#48]
+
+(41) Exchange
+Input [3]: [cs_call_center_sk#40, sum#47, sum#48]
+Arguments: hashpartitioning(cs_call_center_sk#40, 5), ENSURE_REQUIREMENTS, [id=#49]
+
+(42) HashAggregate [codegen id : 11]
+Input [3]: [cs_call_center_sk#40, sum#47, sum#48]
+Keys [1]: [cs_call_center_sk#40]
+Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#41)), sum(UnscaledValue(cs_net_profit#42))]
+Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_sales_price#41))#50, sum(UnscaledValue(cs_net_profit#42))#51]
+Results [3]: [cs_call_center_sk#40, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#41))#50,17,2) AS sales#52, MakeDecimal(sum(UnscaledValue(cs_net_profit#42))#51,17,2) AS profit#53]
+
+(43) BroadcastExchange
+Input [3]: [cs_call_center_sk#40, sales#52, profit#53]
+Arguments: IdentityBroadcastMode, [id=#54]
+
+(44) Scan parquet default.catalog_returns
+Output [3]: [cr_return_amount#55, cr_net_loss#56, cr_returned_date_sk#57]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cr_returned_date_sk#57), dynamicpruningexpression(cr_returned_date_sk#57 IN dynamicpruning#5)]
+ReadSchema: struct<cr_return_amount:decimal(7,2),cr_net_loss:decimal(7,2)>
+
+(45) ColumnarToRow [codegen id : 13]
+Input [3]: [cr_return_amount#55, cr_net_loss#56, cr_returned_date_sk#57]
+
+(46) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#58]
+
+(47) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [cr_returned_date_sk#57]
+Right keys [1]: [d_date_sk#58]
+Join condition: None
+
+(48) Project [codegen id : 13]
+Output [2]: [cr_return_amount#55, cr_net_loss#56]
+Input [4]: [cr_return_amount#55, cr_net_loss#56, cr_returned_date_sk#57, d_date_sk#58]
+
+(49) HashAggregate [codegen id : 13]
+Input [2]: [cr_return_amount#55, cr_net_loss#56]
+Keys: []
+Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#55)), partial_sum(UnscaledValue(cr_net_loss#56))]
+Aggregate Attributes [2]: [sum#59, sum#60]
+Results [2]: [sum#61, sum#62]
+
+(50) Exchange
+Input [2]: [sum#61, sum#62]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#63]
+
+(51) HashAggregate
+Input [2]: [sum#61, sum#62]
+Keys: []
+Functions [2]: [sum(UnscaledValue(cr_return_amount#55)), sum(UnscaledValue(cr_net_loss#56))]
+Aggregate Attributes [2]: [sum(UnscaledValue(cr_return_amount#55))#64, sum(UnscaledValue(cr_net_loss#56))#65]
+Results [2]: [MakeDecimal(sum(UnscaledValue(cr_return_amount#55))#64,17,2) AS returns#66, MakeDecimal(sum(UnscaledValue(cr_net_loss#56))#65,17,2) AS profit_loss#67]
+
+(52) BroadcastNestedLoopJoin [codegen id : 14]
+Join condition: None
+
+(53) Project [codegen id : 14]
+Output [5]: [catalog channel AS channel#68, cs_call_center_sk#40 AS id#69, sales#52, returns#66, CheckOverflow((promote_precision(cast(profit#53 as decimal(18,2))) - promote_precision(cast(profit_loss#67 as decimal(18,2)))), DecimalType(18,2), true) AS profit#70]
+Input [5]: [cs_call_center_sk#40, sales#52, profit#53, returns#66, profit_loss#67]
+
+(54) Scan parquet default.web_sales
+Output [4]: [ws_web_page_sk#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#74), dynamicpruningexpression(ws_sold_date_sk#74 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(ws_web_page_sk)]
+ReadSchema: struct<ws_web_page_sk:int,ws_ext_sales_price:decimal(7,2),ws_net_profit:decimal(7,2)>
+
+(55) ColumnarToRow [codegen id : 17]
+Input [4]: [ws_web_page_sk#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74]
+
+(56) Filter [codegen id : 17]
+Input [4]: [ws_web_page_sk#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74]
+Condition : isnotnull(ws_web_page_sk#71)
+
+(57) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#75]
+
+(58) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_sold_date_sk#74]
+Right keys [1]: [d_date_sk#75]
+Join condition: None
+
+(59) Project [codegen id : 17]
+Output [3]: [ws_web_page_sk#71, ws_ext_sales_price#72, ws_net_profit#73]
+Input [5]: [ws_web_page_sk#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, d_date_sk#75]
+
+(60) Scan parquet default.web_page
+Output [1]: [wp_web_page_sk#76]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_page]
+PushedFilters: [IsNotNull(wp_web_page_sk)]
+ReadSchema: struct<wp_web_page_sk:int>
+
+(61) ColumnarToRow [codegen id : 16]
+Input [1]: [wp_web_page_sk#76]
+
+(62) Filter [codegen id : 16]
+Input [1]: [wp_web_page_sk#76]
+Condition : isnotnull(wp_web_page_sk#76)
+
+(63) BroadcastExchange
+Input [1]: [wp_web_page_sk#76]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#77]
+
+(64) BroadcastHashJoin [codegen id : 17]
+Left keys [1]: [ws_web_page_sk#71]
+Right keys [1]: [wp_web_page_sk#76]
+Join condition: None
+
+(65) Project [codegen id : 17]
+Output [3]: [ws_ext_sales_price#72, ws_net_profit#73, wp_web_page_sk#76]
+Input [4]: [ws_web_page_sk#71, ws_ext_sales_price#72, ws_net_profit#73, wp_web_page_sk#76]
+
+(66) HashAggregate [codegen id : 17]
+Input [3]: [ws_ext_sales_price#72, ws_net_profit#73, wp_web_page_sk#76]
+Keys [1]: [wp_web_page_sk#76]
+Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#72)), partial_sum(UnscaledValue(ws_net_profit#73))]
+Aggregate Attributes [2]: [sum#78, sum#79]
+Results [3]: [wp_web_page_sk#76, sum#80, sum#81]
+
+(67) Exchange
+Input [3]: [wp_web_page_sk#76, sum#80, sum#81]
+Arguments: hashpartitioning(wp_web_page_sk#76, 5), ENSURE_REQUIREMENTS, [id=#82]
+
+(68) HashAggregate [codegen id : 22]
+Input [3]: [wp_web_page_sk#76, sum#80, sum#81]
+Keys [1]: [wp_web_page_sk#76]
+Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#72)), sum(UnscaledValue(ws_net_profit#73))]
+Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_sales_price#72))#83, sum(UnscaledValue(ws_net_profit#73))#84]
+Results [3]: [wp_web_page_sk#76, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#72))#83,17,2) AS sales#85, MakeDecimal(sum(UnscaledValue(ws_net_profit#73))#84,17,2) AS profit#86]
+
+(69) Scan parquet default.web_returns
+Output [4]: [wr_web_page_sk#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(wr_returned_date_sk#90), dynamicpruningexpression(wr_returned_date_sk#90 IN dynamicpruning#5)]
+PushedFilters: [IsNotNull(wr_web_page_sk)]
+ReadSchema: struct<wr_web_page_sk:int,wr_return_amt:decimal(7,2),wr_net_loss:decimal(7,2)>
+
+(70) ColumnarToRow [codegen id : 20]
+Input [4]: [wr_web_page_sk#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90]
+
+(71) Filter [codegen id : 20]
+Input [4]: [wr_web_page_sk#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90]
+Condition : isnotnull(wr_web_page_sk#87)
+
+(72) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#91]
+
+(73) BroadcastHashJoin [codegen id : 20]
+Left keys [1]: [wr_returned_date_sk#90]
+Right keys [1]: [d_date_sk#91]
+Join condition: None
+
+(74) Project [codegen id : 20]
+Output [3]: [wr_web_page_sk#87, wr_return_amt#88, wr_net_loss#89]
+Input [5]: [wr_web_page_sk#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90, d_date_sk#91]
+
+(75) ReusedExchange [Reuses operator id: 63]
+Output [1]: [wp_web_page_sk#92]
+
+(76) BroadcastHashJoin [codegen id : 20]
+Left keys [1]: [wr_web_page_sk#87]
+Right keys [1]: [wp_web_page_sk#92]
+Join condition: None
+
+(77) Project [codegen id : 20]
+Output [3]: [wr_return_amt#88, wr_net_loss#89, wp_web_page_sk#92]
+Input [4]: [wr_web_page_sk#87, wr_return_amt#88, wr_net_loss#89, wp_web_page_sk#92]
+
+(78) HashAggregate [codegen id : 20]
+Input [3]: [wr_return_amt#88, wr_net_loss#89, wp_web_page_sk#92]
+Keys [1]: [wp_web_page_sk#92]
+Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#88)), partial_sum(UnscaledValue(wr_net_loss#89))]
+Aggregate Attributes [2]: [sum#93, sum#94]
+Results [3]: [wp_web_page_sk#92, sum#95, sum#96]
+
+(79) Exchange
+Input [3]: [wp_web_page_sk#92, sum#95, sum#96]
+Arguments: hashpartitioning(wp_web_page_sk#92, 5), ENSURE_REQUIREMENTS, [id=#97]
+
+(80) HashAggregate [codegen id : 21]
+Input [3]: [wp_web_page_sk#92, sum#95, sum#96]
+Keys [1]: [wp_web_page_sk#92]
+Functions [2]: [sum(UnscaledValue(wr_return_amt#88)), sum(UnscaledValue(wr_net_loss#89))]
+Aggregate Attributes [2]: [sum(UnscaledValue(wr_return_amt#88))#98, sum(UnscaledValue(wr_net_loss#89))#99]
+Results [3]: [wp_web_page_sk#92, MakeDecimal(sum(UnscaledValue(wr_return_amt#88))#98,17,2) AS returns#100, MakeDecimal(sum(UnscaledValue(wr_net_loss#89))#99,17,2) AS profit_loss#101]
+
+(81) BroadcastExchange
+Input [3]: [wp_web_page_sk#92, returns#100, profit_loss#101]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#102]
+
+(82) BroadcastHashJoin [codegen id : 22]
+Left keys [1]: [wp_web_page_sk#76]
+Right keys [1]: [wp_web_page_sk#92]
+Join condition: None
+
+(83) Project [codegen id : 22]
+Output [5]: [web channel AS channel#103, wp_web_page_sk#76 AS id#104, sales#85, coalesce(returns#100, 0.00) AS returns#105, CheckOverflow((promote_precision(cast(profit#86 as decimal(18,2))) - promote_precision(cast(coalesce(profit_loss#101, 0.00) as decimal(18,2)))), DecimalType(18,2), true) AS profit#106]
+Input [6]: [wp_web_page_sk#76, sales#85, profit#86, wp_web_page_sk#92, returns#100, profit_loss#101]
+
+(84) Union
+
+(85) HashAggregate [codegen id : 23]
+Input [5]: [channel#36, id#37, sales#18, returns#38, profit#39]
+Keys [2]: [channel#36, id#37]
+Functions [3]: [partial_sum(sales#18), partial_sum(returns#38), partial_sum(profit#39)]
+Aggregate Attributes [6]: [sum#107, isEmpty#108, sum#109, isEmpty#110, sum#111, isEmpty#112]
+Results [8]: [channel#36, id#37, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118]
+
+(86) Exchange
+Input [8]: [channel#36, id#37, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118]
+Arguments: hashpartitioning(channel#36, id#37, 5), ENSURE_REQUIREMENTS, [id=#119]
+
+(87) HashAggregate [codegen id : 24]
+Input [8]: [channel#36, id#37, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118]
+Keys [2]: [channel#36, id#37]
+Functions [3]: [sum(sales#18), sum(returns#38), sum(profit#39)]
+Aggregate Attributes [3]: [sum(sales#18)#120, sum(returns#38)#121, sum(profit#39)#122]
+Results [5]: [channel#36, id#37, cast(sum(sales#18)#120 as decimal(37,2)) AS sales#123, cast(sum(returns#38)#121 as decimal(37,2)) AS returns#124, cast(sum(profit#39)#122 as decimal(38,2)) AS profit#125]
+
+(88) ReusedExchange [Reuses operator id: unknown]
+Output [8]: [channel#36, id#37, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131]
+
+(89) HashAggregate [codegen id : 48]
+Input [8]: [channel#36, id#37, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131]
+Keys [2]: [channel#36, id#37]
+Functions [3]: [sum(sales#18), sum(returns#38), sum(profit#39)]
+Aggregate Attributes [3]: [sum(sales#18)#132, sum(returns#38)#133, sum(profit#39)#134]
+Results [4]: [channel#36, sum(sales#18)#132 AS sales#135, sum(returns#38)#133 AS returns#136, sum(profit#39)#134 AS profit#137]
+
+(90) HashAggregate [codegen id : 48]
+Input [4]: [channel#36, sales#135, returns#136, profit#137]
+Keys [1]: [channel#36]
+Functions [3]: [partial_sum(sales#135), partial_sum(returns#136), partial_sum(profit#137)]
+Aggregate Attributes [6]: [sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143]
+Results [7]: [channel#36, sum#144, isEmpty#145, sum#146, isEmpty#147, sum#148, isEmpty#149]
+
+(91) Exchange
+Input [7]: [channel#36, sum#144, isEmpty#145, sum#146, isEmpty#147, sum#148, isEmpty#149]
+Arguments: hashpartitioning(channel#36, 5), ENSURE_REQUIREMENTS, [id=#150]
+
+(92) HashAggregate [codegen id : 49]
+Input [7]: [channel#36, sum#144, isEmpty#145, sum#146, isEmpty#147, sum#148, isEmpty#149]
+Keys [1]: [channel#36]
+Functions [3]: [sum(sales#135), sum(returns#136), sum(profit#137)]
+Aggregate Attributes [3]: [sum(sales#135)#151, sum(returns#136)#152, sum(profit#137)#153]
+Results [5]: [channel#36, null AS id#154, sum(sales#135)#151 AS sales#155, sum(returns#136)#152 AS returns#156, sum(profit#137)#153 AS profit#157]
+
+(93) ReusedExchange [Reuses operator id: unknown]
+Output [8]: [channel#36, id#37, sum#158, isEmpty#159, sum#160, isEmpty#161, sum#162, isEmpty#163]
+
+(94) HashAggregate [codegen id : 73]
+Input [8]: [channel#36, id#37, sum#158, isEmpty#159, sum#160, isEmpty#161, sum#162, isEmpty#163]
+Keys [2]: [channel#36, id#37]
+Functions [3]: [sum(sales#18), sum(returns#38), sum(profit#39)]
+Aggregate Attributes [3]: [sum(sales#18)#164, sum(returns#38)#165, sum(profit#39)#166]
+Results [3]: [sum(sales#18)#164 AS sales#135, sum(returns#38)#165 AS returns#136, sum(profit#39)#166 AS profit#137]
+
+(95) HashAggregate [codegen id : 73]
+Input [3]: [sales#135, returns#136, profit#137]
+Keys: []
+Functions [3]: [partial_sum(sales#135), partial_sum(returns#136), partial_sum(profit#137)]
+Aggregate Attributes [6]: [sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172]
+Results [6]: [sum#173, isEmpty#174, sum#175, isEmpty#176, sum#177, isEmpty#178]
+
+(96) Exchange
+Input [6]: [sum#173, isEmpty#174, sum#175, isEmpty#176, sum#177, isEmpty#178]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#179]
+
+(97) HashAggregate [codegen id : 74]
+Input [6]: [sum#173, isEmpty#174, sum#175, isEmpty#176, sum#177, isEmpty#178]
+Keys: []
+Functions [3]: [sum(sales#135), sum(returns#136), sum(profit#137)]
+Aggregate Attributes [3]: [sum(sales#135)#180, sum(returns#136)#181, sum(profit#137)#182]
+Results [5]: [null AS channel#183, null AS id#184, sum(sales#135)#180 AS sales#185, sum(returns#136)#181 AS returns#186, sum(profit#137)#182 AS profit#187]
+
+(98) Union
+
+(99) HashAggregate [codegen id : 75]
+Input [5]: [channel#36, id#37, sales#123, returns#124, profit#125]
+Keys [5]: [channel#36, id#37, sales#123, returns#124, profit#125]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [channel#36, id#37, sales#123, returns#124, profit#125]
+
+(100) Exchange
+Input [5]: [channel#36, id#37, sales#123, returns#124, profit#125]
+Arguments: hashpartitioning(channel#36, id#37, sales#123, returns#124, profit#125, 5), ENSURE_REQUIREMENTS, [id=#188]
+
+(101) HashAggregate [codegen id : 76]
+Input [5]: [channel#36, id#37, sales#123, returns#124, profit#125]
+Keys [5]: [channel#36, id#37, sales#123, returns#124, profit#125]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [channel#36, id#37, sales#123, returns#124, profit#125]
+
+(102) TakeOrderedAndProject
+Input [5]: [channel#36, id#37, sales#123, returns#124, profit#125]
+Arguments: 100, [channel#36 ASC NULLS FIRST, id#37 ASC NULLS FIRST], [channel#36, id#37, sales#123, returns#124, profit#125]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5
+ReusedExchange (103)
+
+
+(103) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#6]
+
+Subquery:2 Hosting operator id = 20 Hosting Expression = sr_returned_date_sk#23 IN dynamicpruning#5
+
+Subquery:3 Hosting operator id = 35 Hosting Expression = cs_sold_date_sk#43 IN dynamicpruning#5
+
+Subquery:4 Hosting operator id = 44 Hosting Expression = cr_returned_date_sk#57 IN dynamicpruning#5
+
+Subquery:5 Hosting operator id = 54 Hosting Expression = ws_sold_date_sk#74 IN dynamicpruning#5
+
+Subquery:6 Hosting operator id = 69 Hosting Expression = wr_returned_date_sk#90 IN dynamicpruning#5
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a/simplified.txt
new file mode 100644
index 0000000000000..6248fbf0129bc
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a/simplified.txt
@@ -0,0 +1,168 @@
+TakeOrderedAndProject [channel,id,sales,returns,profit]
+  WholeStageCodegen (76)
+    HashAggregate [channel,id,sales,returns,profit]
+      InputAdapter
+        Exchange [channel,id,sales,returns,profit] #1
+          WholeStageCodegen (75)
+            HashAggregate [channel,id,sales,returns,profit]
+              InputAdapter
+                Union
+                  WholeStageCodegen (24)
+                    HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                      InputAdapter
+                        Exchange [channel,id] #2
+                          WholeStageCodegen (23)
+                            HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                              InputAdapter
+                                Union
+                                  WholeStageCodegen (8)
+                                    Project [s_store_sk,sales,returns,profit,profit_loss]
+                                      BroadcastHashJoin [s_store_sk,s_store_sk]
+                                        HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit)),sales,profit,sum,sum]
+                                          InputAdapter
+                                            Exchange [s_store_sk] #3
+                                              WholeStageCodegen (3)
+                                                HashAggregate [s_store_sk,ss_ext_sales_price,ss_net_profit] [sum,sum,sum,sum]
+                                                  Project [ss_ext_sales_price,ss_net_profit,s_store_sk]
+                                                    BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                      Project [ss_store_sk,ss_ext_sales_price,ss_net_profit]
+                                                        BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                          Filter [ss_store_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk]
+                                                                  SubqueryBroadcast [d_date_sk] #1
+                                                                    ReusedExchange [d_date_sk] #4
+                                                          InputAdapter
+                                                            BroadcastExchange #4
+                                                              WholeStageCodegen (1)
+                                                                Project [d_date_sk]
+                                                                  Filter [d_date,d_date_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.date_dim [d_date_sk,d_date]
+                                                      InputAdapter
+                                                        BroadcastExchange #5
+                                                          WholeStageCodegen (2)
+                                                            Filter [s_store_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.store [s_store_sk]
+                                        InputAdapter
+                                          BroadcastExchange #6
+                                            WholeStageCodegen (7)
+                                              HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss)),returns,profit_loss,sum,sum]
+                                                InputAdapter
+                                                  Exchange [s_store_sk] #7
+                                                    WholeStageCodegen (6)
+                                                      HashAggregate [s_store_sk,sr_return_amt,sr_net_loss] [sum,sum,sum,sum]
+                                                        Project [sr_return_amt,sr_net_loss,s_store_sk]
+                                                          BroadcastHashJoin [sr_store_sk,s_store_sk]
+                                                            Project [sr_store_sk,sr_return_amt,sr_net_loss]
+                                                              BroadcastHashJoin [sr_returned_date_sk,d_date_sk]
+                                                                Filter [sr_store_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk]
+                                                                        ReusedSubquery [d_date_sk] #1
+                                                                InputAdapter
+                                                                  ReusedExchange [d_date_sk] #4
+                                                            InputAdapter
+                                                              ReusedExchange [s_store_sk] #5
+                                  WholeStageCodegen (14)
+                                    Project [cs_call_center_sk,sales,returns,profit,profit_loss]
+                                      BroadcastNestedLoopJoin
+                                        InputAdapter
+                                          BroadcastExchange #8
+                                            WholeStageCodegen (11)
+                                              HashAggregate [cs_call_center_sk,sum,sum] [sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit)),sales,profit,sum,sum]
+                                                InputAdapter
+                                                  Exchange [cs_call_center_sk] #9
+                                                    WholeStageCodegen (10)
+                                                      HashAggregate [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] [sum,sum,sum,sum]
+                                                        Project [cs_call_center_sk,cs_ext_sales_price,cs_net_profit]
+                                                          BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk]
+                                                                  ReusedSubquery [d_date_sk] #1
+                                                            InputAdapter
+                                                              ReusedExchange [d_date_sk] #4
+                                        HashAggregate [sum,sum] [sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss)),returns,profit_loss,sum,sum]
+                                          InputAdapter
+                                            Exchange #10
+                                              WholeStageCodegen (13)
+                                                HashAggregate [cr_return_amount,cr_net_loss] [sum,sum,sum,sum]
+                                                  Project [cr_return_amount,cr_net_loss]
+                                                    BroadcastHashJoin [cr_returned_date_sk,d_date_sk]
+                                                      ColumnarToRow
+                                                        InputAdapter
+                                                          Scan parquet default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk]
+                                                            ReusedSubquery [d_date_sk] #1
+                                                      InputAdapter
+                                                        ReusedExchange [d_date_sk] #4
+                                  WholeStageCodegen (22)
+                                    Project [wp_web_page_sk,sales,returns,profit,profit_loss]
+                                      BroadcastHashJoin [wp_web_page_sk,wp_web_page_sk]
+                                        HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit)),sales,profit,sum,sum]
+                                          InputAdapter
+                                            Exchange [wp_web_page_sk] #11
+                                              WholeStageCodegen (17)
+                                                HashAggregate [wp_web_page_sk,ws_ext_sales_price,ws_net_profit] [sum,sum,sum,sum]
+                                                  Project [ws_ext_sales_price,ws_net_profit,wp_web_page_sk]
+                                                    BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk]
+                                                      Project [ws_web_page_sk,ws_ext_sales_price,ws_net_profit]
+                                                        BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                          Filter [ws_web_page_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk]
+                                                                  ReusedSubquery [d_date_sk] #1
+                                                          InputAdapter
+                                                            ReusedExchange [d_date_sk] #4
+                                                      InputAdapter
+                                                        BroadcastExchange #12
+                                                          WholeStageCodegen (16)
+                                                            Filter [wp_web_page_sk]
+                                                              ColumnarToRow
+                                                                InputAdapter
+                                                                  Scan parquet default.web_page [wp_web_page_sk]
+                                        InputAdapter
+                                          BroadcastExchange #13
+                                            WholeStageCodegen (21)
+                                              HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss)),returns,profit_loss,sum,sum]
+                                                InputAdapter
+                                                  Exchange [wp_web_page_sk] #14
+                                                    WholeStageCodegen (20)
+                                                      HashAggregate [wp_web_page_sk,wr_return_amt,wr_net_loss] [sum,sum,sum,sum]
+                                                        Project [wr_return_amt,wr_net_loss,wp_web_page_sk]
+                                                          BroadcastHashJoin [wr_web_page_sk,wp_web_page_sk]
+                                                            Project [wr_web_page_sk,wr_return_amt,wr_net_loss]
+                                                              BroadcastHashJoin [wr_returned_date_sk,d_date_sk]
+                                                                Filter [wr_web_page_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk]
+                                                                        ReusedSubquery [d_date_sk] #1
+                                                                InputAdapter
+                                                                  ReusedExchange [d_date_sk] #4
+                                                            InputAdapter
+                                                              ReusedExchange [wp_web_page_sk] #12
+                  WholeStageCodegen (49)
+                    HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                      InputAdapter
+                        Exchange [channel] #15
+                          WholeStageCodegen (48)
+                            HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                              HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                                InputAdapter
+                                  ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #16
+                  WholeStageCodegen (74)
+                    HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                      InputAdapter
+                        Exchange #17
+                          WholeStageCodegen (73)
+                            HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                              HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                                InputAdapter
+                                  ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #16
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.sf100/explain.txt
new file mode 100644
index 0000000000000..5694b16800d8e
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.sf100/explain.txt
@@ -0,0 +1,423 @@
+== Physical Plan ==
+TakeOrderedAndProject (73)
++- * Project (72)
+   +- * SortMergeJoin Inner (71)
+      :- * Project (48)
+      :  +- * SortMergeJoin Inner (47)
+      :     :- * Sort (24)
+      :     :  +- * HashAggregate (23)
+      :     :     +- Exchange (22)
+      :     :        +- * HashAggregate (21)
+      :     :           +- * Project (20)
+      :     :              +- * BroadcastHashJoin Inner BuildRight (19)
+      :     :                 :- * Project (14)
+      :     :                 :  +- * Filter (13)
+      :     :                 :     +- * SortMergeJoin LeftOuter (12)
+      :     :                 :        :- * Sort (5)
+      :     :                 :        :  +- Exchange (4)
+      :     :                 :        :     +- * Filter (3)
+      :     :                 :        :        +- * ColumnarToRow (2)
+      :     :                 :        :           +- Scan parquet default.store_sales (1)
+      :     :                 :        +- * Sort (11)
+      :     :                 :           +- Exchange (10)
+      :     :                 :              +- * Project (9)
+      :     :                 :                 +- * Filter (8)
+      :     :                 :                    +- * ColumnarToRow (7)
+      :     :                 :                       +- Scan parquet default.store_returns (6)
+      :     :                 +- BroadcastExchange (18)
+      :     :                    +- * Filter (17)
+      :     :                       +- * ColumnarToRow (16)
+      :     :                          +- Scan parquet default.date_dim (15)
+      :     +- * Sort (46)
+      :        +- * Filter (45)
+      :           +- * HashAggregate (44)
+      :              +- Exchange (43)
+      :                 +- * HashAggregate (42)
+      :                    +- * Project (41)
+      :                       +- * BroadcastHashJoin Inner BuildRight (40)
+      :                          :- * Project (38)
+      :                          :  +- * Filter (37)
+      :                          :     +- * SortMergeJoin LeftOuter (36)
+      :                          :        :- * Sort (29)
+      :                          :        :  +- Exchange (28)
+      :                          :        :     +- * Filter (27)
+      :                          :        :        +- * ColumnarToRow (26)
+      :                          :        :           +- Scan parquet default.web_sales (25)
+      :                          :        +- * Sort (35)
+      :                          :           +- Exchange (34)
+      :                          :              +- * Project (33)
+      :                          :                 +- * Filter (32)
+      :                          :                    +- * ColumnarToRow (31)
+      :                          :                       +- Scan parquet default.web_returns (30)
+      :                          +- ReusedExchange (39)
+      +- * Sort (70)
+         +- * Filter (69)
+            +- * HashAggregate (68)
+               +- Exchange (67)
+                  +- * HashAggregate (66)
+                     +- * Project (65)
+                        +- * BroadcastHashJoin Inner BuildRight (64)
+                           :- * Project (62)
+                           :  +- * Filter (61)
+                           :     +- * SortMergeJoin LeftOuter (60)
+                           :        :- * Sort (53)
+                           :        :  +- Exchange (52)
+                           :        :     +- * Filter (51)
+                           :        :        +- * ColumnarToRow (50)
+                           :        :           +- Scan parquet default.catalog_sales (49)
+                           :        +- * Sort (59)
+                           :           +- Exchange (58)
+                           :              +- * Project (57)
+                           :                 +- * Filter (56)
+                           :                    +- * ColumnarToRow (55)
+                           :                       +- Scan parquet default.catalog_returns (54)
+                           +- ReusedExchange (63)
+
+
+(1) Scan parquet default.store_sales
+Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_ticket_number:int,ss_quantity:int,ss_wholesale_cost:decimal(7,2),ss_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7]
+
+(3) Filter [codegen id : 1]
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7]
+Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_customer_sk#2))
+
+(4) Exchange
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7]
+Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#9]
+
+(5) Sort [codegen id : 2]
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7]
+Arguments: [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.store_returns
+Output [3]: [sr_item_sk#10, sr_ticket_number#11, sr_returned_date_sk#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int>
+
+(7) ColumnarToRow [codegen id : 3]
+Input [3]: [sr_item_sk#10, sr_ticket_number#11, sr_returned_date_sk#12]
+
+(8) Filter [codegen id : 3]
+Input [3]: [sr_item_sk#10, sr_ticket_number#11, sr_returned_date_sk#12]
+Condition : (isnotnull(sr_ticket_number#11) AND isnotnull(sr_item_sk#10))
+
+(9) Project [codegen id : 3]
+Output [2]: [sr_item_sk#10, sr_ticket_number#11]
+Input [3]: [sr_item_sk#10, sr_ticket_number#11, sr_returned_date_sk#12]
+
+(10) Exchange
+Input [2]: [sr_item_sk#10, sr_ticket_number#11]
+Arguments: hashpartitioning(sr_ticket_number#11, sr_item_sk#10, 5), ENSURE_REQUIREMENTS, [id=#13]
+
+(11) Sort [codegen id : 4]
+Input [2]: [sr_item_sk#10, sr_ticket_number#11]
+Arguments: [sr_ticket_number#11 ASC NULLS FIRST, sr_item_sk#10 ASC NULLS FIRST], false, 0
+
+(12) SortMergeJoin [codegen id : 6]
+Left keys [2]: [ss_ticket_number#3, ss_item_sk#1]
+Right keys [2]: [sr_ticket_number#11, sr_item_sk#10]
+Join condition: None
+
+(13) Filter [codegen id : 6]
+Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#10, sr_ticket_number#11]
+Condition : isnull(sr_ticket_number#11)
+
+(14) Project [codegen id : 6]
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7]
+Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#10, sr_ticket_number#11]
+
+(15) Scan parquet default.date_dim
+Output [2]: [d_date_sk#14, d_year#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(16) ColumnarToRow [codegen id : 5]
+Input [2]: [d_date_sk#14, d_year#15]
+
+(17) Filter [codegen id : 5]
+Input [2]: [d_date_sk#14, d_year#15]
+Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2000)) AND isnotnull(d_date_sk#14))
+
+(18) BroadcastExchange
+Input [2]: [d_date_sk#14, d_year#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#16]
+
+(19) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_sold_date_sk#7]
+Right keys [1]: [d_date_sk#14]
+Join condition: None
+
+(20) Project [codegen id : 6]
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#15]
+Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#14, d_year#15]
+
+(21) HashAggregate [codegen id : 6]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#15]
+Keys [3]: [d_year#15, ss_item_sk#1, ss_customer_sk#2]
+Functions [3]: [partial_sum(ss_quantity#4), partial_sum(UnscaledValue(ss_wholesale_cost#5)), partial_sum(UnscaledValue(ss_sales_price#6))]
+Aggregate Attributes [3]: [sum#17, sum#18, sum#19]
+Results [6]: [d_year#15, ss_item_sk#1, ss_customer_sk#2, sum#20, sum#21, sum#22]
+
+(22) Exchange
+Input [6]: [d_year#15, ss_item_sk#1, ss_customer_sk#2, sum#20, sum#21, sum#22]
+Arguments: hashpartitioning(d_year#15, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(23) HashAggregate [codegen id : 7]
+Input [6]: [d_year#15, ss_item_sk#1, ss_customer_sk#2, sum#20, sum#21, sum#22]
+Keys [3]: [d_year#15, ss_item_sk#1, ss_customer_sk#2]
+Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum(UnscaledValue(ss_sales_price#6))]
+Aggregate Attributes [3]: [sum(ss_quantity#4)#24, sum(UnscaledValue(ss_wholesale_cost#5))#25, sum(UnscaledValue(ss_sales_price#6))#26]
+Results [6]: [d_year#15 AS ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2, sum(ss_quantity#4)#24 AS ss_qty#28, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#5))#25,17,2) AS ss_wc#29, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#26,17,2) AS ss_sp#30]
+
+(24) Sort [codegen id : 7]
+Input [6]: [ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2, ss_qty#28, ss_wc#29, ss_sp#30]
+Arguments: [ss_sold_year#27 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], false, 0
+
+(25) Scan parquet default.web_sales
+Output [7]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#37), dynamicpruningexpression(ws_sold_date_sk#37 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_bill_customer_sk:int,ws_order_number:int,ws_quantity:int,ws_wholesale_cost:decimal(7,2),ws_sales_price:decimal(7,2)>
+
+(26) ColumnarToRow [codegen id : 8]
+Input [7]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37]
+
+(27) Filter [codegen id : 8]
+Input [7]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37]
+Condition : (isnotnull(ws_item_sk#31) AND isnotnull(ws_bill_customer_sk#32))
+
+(28) Exchange
+Input [7]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37]
+Arguments: hashpartitioning(ws_order_number#33, ws_item_sk#31, 5), ENSURE_REQUIREMENTS, [id=#38]
+
+(29) Sort [codegen id : 9]
+Input [7]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37]
+Arguments: [ws_order_number#33 ASC NULLS FIRST, ws_item_sk#31 ASC NULLS FIRST], false, 0
+
+(30) Scan parquet default.web_returns
+Output [3]: [wr_item_sk#39, wr_order_number#40, wr_returned_date_sk#41]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_returns]
+PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)]
+ReadSchema: struct<wr_item_sk:int,wr_order_number:int>
+
+(31) ColumnarToRow [codegen id : 10]
+Input [3]: [wr_item_sk#39, wr_order_number#40, wr_returned_date_sk#41]
+
+(32) Filter [codegen id : 10]
+Input [3]: [wr_item_sk#39, wr_order_number#40, wr_returned_date_sk#41]
+Condition : (isnotnull(wr_order_number#40) AND isnotnull(wr_item_sk#39))
+
+(33) Project [codegen id : 10]
+Output [2]: [wr_item_sk#39, wr_order_number#40]
+Input [3]: [wr_item_sk#39, wr_order_number#40, wr_returned_date_sk#41]
+
+(34) Exchange
+Input [2]: [wr_item_sk#39, wr_order_number#40]
+Arguments: hashpartitioning(wr_order_number#40, wr_item_sk#39, 5), ENSURE_REQUIREMENTS, [id=#42]
+
+(35) Sort [codegen id : 11]
+Input [2]: [wr_item_sk#39, wr_order_number#40]
+Arguments: [wr_order_number#40 ASC NULLS FIRST, wr_item_sk#39 ASC NULLS FIRST], false, 0
+
+(36) SortMergeJoin [codegen id : 13]
+Left keys [2]: [ws_order_number#33, ws_item_sk#31]
+Right keys [2]: [wr_order_number#40, wr_item_sk#39]
+Join condition: None
+
+(37) Filter [codegen id : 13]
+Input [9]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37, wr_item_sk#39, wr_order_number#40]
+Condition : isnull(wr_order_number#40)
+
+(38) Project [codegen id : 13]
+Output [6]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37]
+Input [9]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37, wr_item_sk#39, wr_order_number#40]
+
+(39) ReusedExchange [Reuses operator id: 18]
+Output [2]: [d_date_sk#43, d_year#44]
+
+(40) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ws_sold_date_sk#37]
+Right keys [1]: [d_date_sk#43]
+Join condition: None
+
+(41) Project [codegen id : 13]
+Output [6]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, d_year#44]
+Input [8]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37, d_date_sk#43, d_year#44]
+
+(42) HashAggregate [codegen id : 13]
+Input [6]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, d_year#44]
+Keys [3]: [d_year#44, ws_item_sk#31, ws_bill_customer_sk#32]
+Functions [3]: [partial_sum(ws_quantity#34), partial_sum(UnscaledValue(ws_wholesale_cost#35)), partial_sum(UnscaledValue(ws_sales_price#36))]
+Aggregate Attributes [3]: [sum#45, sum#46, sum#47]
+Results [6]: [d_year#44, ws_item_sk#31, ws_bill_customer_sk#32, sum#48, sum#49, sum#50]
+
+(43) Exchange
+Input [6]: [d_year#44, ws_item_sk#31, ws_bill_customer_sk#32, sum#48, sum#49, sum#50]
+Arguments: hashpartitioning(d_year#44, ws_item_sk#31, ws_bill_customer_sk#32, 5), ENSURE_REQUIREMENTS, [id=#51]
+
+(44) HashAggregate [codegen id : 14]
+Input [6]: [d_year#44, ws_item_sk#31, ws_bill_customer_sk#32, sum#48, sum#49, sum#50]
+Keys [3]: [d_year#44, ws_item_sk#31, ws_bill_customer_sk#32]
+Functions [3]: [sum(ws_quantity#34), sum(UnscaledValue(ws_wholesale_cost#35)), sum(UnscaledValue(ws_sales_price#36))]
+Aggregate Attributes [3]: [sum(ws_quantity#34)#52, sum(UnscaledValue(ws_wholesale_cost#35))#53, sum(UnscaledValue(ws_sales_price#36))#54]
+Results [6]: [d_year#44 AS ws_sold_year#55, ws_item_sk#31, ws_bill_customer_sk#32 AS ws_customer_sk#56, sum(ws_quantity#34)#52 AS ws_qty#57, MakeDecimal(sum(UnscaledValue(ws_wholesale_cost#35))#53,17,2) AS ws_wc#58, MakeDecimal(sum(UnscaledValue(ws_sales_price#36))#54,17,2) AS ws_sp#59]
+
+(45) Filter [codegen id : 14]
+Input [6]: [ws_sold_year#55, ws_item_sk#31, ws_customer_sk#56, ws_qty#57, ws_wc#58, ws_sp#59]
+Condition : (coalesce(ws_qty#57, 0) > 0)
+
+(46) Sort [codegen id : 14]
+Input [6]: [ws_sold_year#55, ws_item_sk#31, ws_customer_sk#56, ws_qty#57, ws_wc#58, ws_sp#59]
+Arguments: [ws_sold_year#55 ASC NULLS FIRST, ws_item_sk#31 ASC NULLS FIRST, ws_customer_sk#56 ASC NULLS FIRST], false, 0
+
+(47) SortMergeJoin [codegen id : 15]
+Left keys [3]: [ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2]
+Right keys [3]: [ws_sold_year#55, ws_item_sk#31, ws_customer_sk#56]
+Join condition: None
+
+(48) Project [codegen id : 15]
+Output [9]: [ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2, ss_qty#28, ss_wc#29, ss_sp#30, ws_qty#57, ws_wc#58, ws_sp#59]
+Input [12]: [ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2, ss_qty#28, ss_wc#29, ss_sp#30, ws_sold_year#55, ws_item_sk#31, ws_customer_sk#56, ws_qty#57, ws_wc#58, ws_sp#59]
+
+(49) Scan parquet default.catalog_sales
+Output [7]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#66), dynamicpruningexpression(cs_sold_date_sk#66 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_item_sk:int,cs_order_number:int,cs_quantity:int,cs_wholesale_cost:decimal(7,2),cs_sales_price:decimal(7,2)>
+
+(50) ColumnarToRow [codegen id : 16]
+Input [7]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66]
+
+(51) Filter [codegen id : 16]
+Input [7]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66]
+Condition : (isnotnull(cs_item_sk#61) AND isnotnull(cs_bill_customer_sk#60))
+
+(52) Exchange
+Input [7]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66]
+Arguments: hashpartitioning(cs_order_number#62, cs_item_sk#61, 5), ENSURE_REQUIREMENTS, [id=#67]
+
+(53) Sort [codegen id : 17]
+Input [7]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66]
+Arguments: [cs_order_number#62 ASC NULLS FIRST, cs_item_sk#61 ASC NULLS FIRST], false, 0
+
+(54) Scan parquet default.catalog_returns
+Output [3]: [cr_item_sk#68, cr_order_number#69, cr_returned_date_sk#70]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int>
+
+(55) ColumnarToRow [codegen id : 18]
+Input [3]: [cr_item_sk#68, cr_order_number#69, cr_returned_date_sk#70]
+
+(56) Filter [codegen id : 18]
+Input [3]: [cr_item_sk#68, cr_order_number#69, cr_returned_date_sk#70]
+Condition : (isnotnull(cr_order_number#69) AND isnotnull(cr_item_sk#68))
+
+(57) Project [codegen id : 18]
+Output [2]: [cr_item_sk#68, cr_order_number#69]
+Input [3]: [cr_item_sk#68, cr_order_number#69, cr_returned_date_sk#70]
+
+(58) Exchange
+Input [2]: [cr_item_sk#68, cr_order_number#69]
+Arguments: hashpartitioning(cr_order_number#69, cr_item_sk#68, 5), ENSURE_REQUIREMENTS, [id=#71]
+
+(59) Sort [codegen id : 19]
+Input [2]: [cr_item_sk#68, cr_order_number#69]
+Arguments: [cr_order_number#69 ASC NULLS FIRST, cr_item_sk#68 ASC NULLS FIRST], false, 0
+
+(60) SortMergeJoin [codegen id : 21]
+Left keys [2]: [cs_order_number#62, cs_item_sk#61]
+Right keys [2]: [cr_order_number#69, cr_item_sk#68]
+Join condition: None
+
+(61) Filter [codegen id : 21]
+Input [9]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66, cr_item_sk#68, cr_order_number#69]
+Condition : isnull(cr_order_number#69)
+
+(62) Project [codegen id : 21]
+Output [6]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66]
+Input [9]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66, cr_item_sk#68, cr_order_number#69]
+
+(63) ReusedExchange [Reuses operator id: 18]
+Output [2]: [d_date_sk#72, d_year#73]
+
+(64) BroadcastHashJoin [codegen id : 21]
+Left keys [1]: [cs_sold_date_sk#66]
+Right keys [1]: [d_date_sk#72]
+Join condition: None
+
+(65) Project [codegen id : 21]
+Output [6]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, d_year#73]
+Input [8]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66, d_date_sk#72, d_year#73]
+
+(66) HashAggregate [codegen id : 21]
+Input [6]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, d_year#73]
+Keys [3]: [d_year#73, cs_item_sk#61, cs_bill_customer_sk#60]
+Functions [3]: [partial_sum(cs_quantity#63), partial_sum(UnscaledValue(cs_wholesale_cost#64)), partial_sum(UnscaledValue(cs_sales_price#65))]
+Aggregate Attributes [3]: [sum#74, sum#75, sum#76]
+Results [6]: [d_year#73, cs_item_sk#61, cs_bill_customer_sk#60, sum#77, sum#78, sum#79]
+
+(67) Exchange
+Input [6]: [d_year#73, cs_item_sk#61, cs_bill_customer_sk#60, sum#77, sum#78, sum#79]
+Arguments: hashpartitioning(d_year#73, cs_item_sk#61, cs_bill_customer_sk#60, 5), ENSURE_REQUIREMENTS, [id=#80]
+
+(68) HashAggregate [codegen id : 22]
+Input [6]: [d_year#73, cs_item_sk#61, cs_bill_customer_sk#60, sum#77, sum#78, sum#79]
+Keys [3]: [d_year#73, cs_item_sk#61, cs_bill_customer_sk#60]
+Functions [3]: [sum(cs_quantity#63), sum(UnscaledValue(cs_wholesale_cost#64)), sum(UnscaledValue(cs_sales_price#65))]
+Aggregate Attributes [3]: [sum(cs_quantity#63)#81, sum(UnscaledValue(cs_wholesale_cost#64))#82, sum(UnscaledValue(cs_sales_price#65))#83]
+Results [6]: [d_year#73 AS cs_sold_year#84, cs_item_sk#61, cs_bill_customer_sk#60 AS cs_customer_sk#85, sum(cs_quantity#63)#81 AS cs_qty#86, MakeDecimal(sum(UnscaledValue(cs_wholesale_cost#64))#82,17,2) AS cs_wc#87, MakeDecimal(sum(UnscaledValue(cs_sales_price#65))#83,17,2) AS cs_sp#88]
+
+(69) Filter [codegen id : 22]
+Input [6]: [cs_sold_year#84, cs_item_sk#61, cs_customer_sk#85, cs_qty#86, cs_wc#87, cs_sp#88]
+Condition : (coalesce(cs_qty#86, 0) > 0)
+
+(70) Sort [codegen id : 22]
+Input [6]: [cs_sold_year#84, cs_item_sk#61, cs_customer_sk#85, cs_qty#86, cs_wc#87, cs_sp#88]
+Arguments: [cs_sold_year#84 ASC NULLS FIRST, cs_item_sk#61 ASC NULLS FIRST, cs_customer_sk#85 ASC NULLS FIRST], false, 0
+
+(71) SortMergeJoin [codegen id : 23]
+Left keys [3]: [ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2]
+Right keys [3]: [cs_sold_year#84, cs_item_sk#61, cs_customer_sk#85]
+Join condition: None
+
+(72) Project [codegen id : 23]
+Output [13]: [round((cast(ss_qty#28 as double) / cast(coalesce((ws_qty#57 + cs_qty#86), 1) as double)), 2) AS ratio#89, ss_qty#28 AS store_qty#90, ss_wc#29 AS store_wholesale_cost#91, ss_sp#30 AS store_sales_price#92, (coalesce(ws_qty#57, 0) + coalesce(cs_qty#86, 0)) AS other_chan_qty#93, CheckOverflow((promote_precision(cast(coalesce(ws_wc#58, 0.00) as decimal(18,2))) + promote_precision(cast(coalesce(cs_wc#87, 0.00) as decimal(18,2)))), DecimalType(18,2), true) AS other_chan_wholesale_cost#94, CheckOverflow((promote_precision(cast(coalesce(ws_sp#59, 0.00) as decimal(18,2))) + promote_precision(cast(coalesce(cs_sp#88, 0.00) as decimal(18,2)))), DecimalType(18,2), true) AS other_chan_sales_price#95, ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2, ss_qty#28, ss_wc#29, ss_sp#30]
+Input [15]: [ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2, ss_qty#28, ss_wc#29, ss_sp#30, ws_qty#57, ws_wc#58, ws_sp#59, cs_sold_year#84, cs_item_sk#61, cs_customer_sk#85, cs_qty#86, cs_wc#87, cs_sp#88]
+
+(73) TakeOrderedAndProject
+Input [13]: [ratio#89, store_qty#90, store_wholesale_cost#91, store_sales_price#92, other_chan_qty#93, other_chan_wholesale_cost#94, other_chan_sales_price#95, ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2, ss_qty#28, ss_wc#29, ss_sp#30]
+Arguments: 100, [ss_sold_year#27 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST, ss_qty#28 DESC NULLS LAST, ss_wc#29 DESC NULLS LAST, ss_sp#30 DESC NULLS LAST, other_chan_qty#93 ASC NULLS FIRST, other_chan_wholesale_cost#94 ASC NULLS FIRST, other_chan_sales_price#95 ASC NULLS FIRST, ratio#89 ASC NULLS FIRST], [ratio#89, store_qty#90, store_wholesale_cost#91, store_sales_price#92, other_chan_qty#93, other_chan_wholesale_cost#94, other_chan_sales_price#95]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (74)
+
+
+(74) ReusedExchange [Reuses operator id: 18]
+Output [2]: [d_date_sk#14, d_year#15]
+
+Subquery:2 Hosting operator id = 25 Hosting Expression = ws_sold_date_sk#37 IN dynamicpruning#8
+
+Subquery:3 Hosting operator id = 49 Hosting Expression = cs_sold_date_sk#66 IN dynamicpruning#8
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.sf100/simplified.txt
new file mode 100644
index 0000000000000..cb9ca4f3d3d32
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.sf100/simplified.txt
@@ -0,0 +1,127 @@
+TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholesale_cost,other_chan_sales_price,ratio,store_qty,store_wholesale_cost,store_sales_price]
+  WholeStageCodegen (23)
+    Project [ss_qty,ws_qty,cs_qty,ss_wc,ss_sp,ws_wc,cs_wc,ws_sp,cs_sp,ss_sold_year,ss_item_sk,ss_customer_sk]
+      SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,cs_sold_year,cs_item_sk,cs_customer_sk]
+        InputAdapter
+          WholeStageCodegen (15)
+            Project [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp]
+              SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ws_sold_year,ws_item_sk,ws_customer_sk]
+                InputAdapter
+                  WholeStageCodegen (7)
+                    Sort [ss_sold_year,ss_item_sk,ss_customer_sk]
+                      HashAggregate [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] [sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price)),ss_sold_year,ss_qty,ss_wc,ss_sp,sum,sum,sum]
+                        InputAdapter
+                          Exchange [d_year,ss_item_sk,ss_customer_sk] #1
+                            WholeStageCodegen (6)
+                              HashAggregate [d_year,ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price] [sum,sum,sum,sum,sum,sum]
+                                Project [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year]
+                                  BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                    Project [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk]
+                                      Filter [sr_ticket_number]
+                                        SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                          InputAdapter
+                                            WholeStageCodegen (2)
+                                              Sort [ss_ticket_number,ss_item_sk]
+                                                InputAdapter
+                                                  Exchange [ss_ticket_number,ss_item_sk] #2
+                                                    WholeStageCodegen (1)
+                                                      Filter [ss_item_sk,ss_customer_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk]
+                                                              SubqueryBroadcast [d_date_sk] #1
+                                                                ReusedExchange [d_date_sk,d_year] #3
+                                          InputAdapter
+                                            WholeStageCodegen (4)
+                                              Sort [sr_ticket_number,sr_item_sk]
+                                                InputAdapter
+                                                  Exchange [sr_ticket_number,sr_item_sk] #4
+                                                    WholeStageCodegen (3)
+                                                      Project [sr_item_sk,sr_ticket_number]
+                                                        Filter [sr_ticket_number,sr_item_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk]
+                                    InputAdapter
+                                      BroadcastExchange #3
+                                        WholeStageCodegen (5)
+                                          Filter [d_year,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_year]
+                InputAdapter
+                  WholeStageCodegen (14)
+                    Sort [ws_sold_year,ws_item_sk,ws_customer_sk]
+                      Filter [ws_qty]
+                        HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] [sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price)),ws_sold_year,ws_customer_sk,ws_qty,ws_wc,ws_sp,sum,sum,sum]
+                          InputAdapter
+                            Exchange [d_year,ws_item_sk,ws_bill_customer_sk] #5
+                              WholeStageCodegen (13)
+                                HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price] [sum,sum,sum,sum,sum,sum]
+                                  Project [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year]
+                                    BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                      Project [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk]
+                                        Filter [wr_order_number]
+                                          SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk]
+                                            InputAdapter
+                                              WholeStageCodegen (9)
+                                                Sort [ws_order_number,ws_item_sk]
+                                                  InputAdapter
+                                                    Exchange [ws_order_number,ws_item_sk] #6
+                                                      WholeStageCodegen (8)
+                                                        Filter [ws_item_sk,ws_bill_customer_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk]
+                                                                ReusedSubquery [d_date_sk] #1
+                                            InputAdapter
+                                              WholeStageCodegen (11)
+                                                Sort [wr_order_number,wr_item_sk]
+                                                  InputAdapter
+                                                    Exchange [wr_order_number,wr_item_sk] #7
+                                                      WholeStageCodegen (10)
+                                                        Project [wr_item_sk,wr_order_number]
+                                                          Filter [wr_order_number,wr_item_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk]
+                                      InputAdapter
+                                        ReusedExchange [d_date_sk,d_year] #3
+        InputAdapter
+          WholeStageCodegen (22)
+            Sort [cs_sold_year,cs_item_sk,cs_customer_sk]
+              Filter [cs_qty]
+                HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] [sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price)),cs_sold_year,cs_customer_sk,cs_qty,cs_wc,cs_sp,sum,sum,sum]
+                  InputAdapter
+                    Exchange [d_year,cs_item_sk,cs_bill_customer_sk] #8
+                      WholeStageCodegen (21)
+                        HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,cs_quantity,cs_wholesale_cost,cs_sales_price] [sum,sum,sum,sum,sum,sum]
+                          Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year]
+                            BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                              Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk]
+                                Filter [cr_order_number]
+                                  SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk]
+                                    InputAdapter
+                                      WholeStageCodegen (17)
+                                        Sort [cs_order_number,cs_item_sk]
+                                          InputAdapter
+                                            Exchange [cs_order_number,cs_item_sk] #9
+                                              WholeStageCodegen (16)
+                                                Filter [cs_item_sk,cs_bill_customer_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk]
+                                                        ReusedSubquery [d_date_sk] #1
+                                    InputAdapter
+                                      WholeStageCodegen (19)
+                                        Sort [cr_order_number,cr_item_sk]
+                                          InputAdapter
+                                            Exchange [cr_order_number,cr_item_sk] #10
+                                              WholeStageCodegen (18)
+                                                Project [cr_item_sk,cr_order_number]
+                                                  Filter [cr_order_number,cr_item_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk]
+                              InputAdapter
+                                ReusedExchange [d_date_sk,d_year] #3
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/explain.txt
new file mode 100644
index 0000000000000..5694b16800d8e
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/explain.txt
@@ -0,0 +1,423 @@
+== Physical Plan ==
+TakeOrderedAndProject (73)
++- * Project (72)
+   +- * SortMergeJoin Inner (71)
+      :- * Project (48)
+      :  +- * SortMergeJoin Inner (47)
+      :     :- * Sort (24)
+      :     :  +- * HashAggregate (23)
+      :     :     +- Exchange (22)
+      :     :        +- * HashAggregate (21)
+      :     :           +- * Project (20)
+      :     :              +- * BroadcastHashJoin Inner BuildRight (19)
+      :     :                 :- * Project (14)
+      :     :                 :  +- * Filter (13)
+      :     :                 :     +- * SortMergeJoin LeftOuter (12)
+      :     :                 :        :- * Sort (5)
+      :     :                 :        :  +- Exchange (4)
+      :     :                 :        :     +- * Filter (3)
+      :     :                 :        :        +- * ColumnarToRow (2)
+      :     :                 :        :           +- Scan parquet default.store_sales (1)
+      :     :                 :        +- * Sort (11)
+      :     :                 :           +- Exchange (10)
+      :     :                 :              +- * Project (9)
+      :     :                 :                 +- * Filter (8)
+      :     :                 :                    +- * ColumnarToRow (7)
+      :     :                 :                       +- Scan parquet default.store_returns (6)
+      :     :                 +- BroadcastExchange (18)
+      :     :                    +- * Filter (17)
+      :     :                       +- * ColumnarToRow (16)
+      :     :                          +- Scan parquet default.date_dim (15)
+      :     +- * Sort (46)
+      :        +- * Filter (45)
+      :           +- * HashAggregate (44)
+      :              +- Exchange (43)
+      :                 +- * HashAggregate (42)
+      :                    +- * Project (41)
+      :                       +- * BroadcastHashJoin Inner BuildRight (40)
+      :                          :- * Project (38)
+      :                          :  +- * Filter (37)
+      :                          :     +- * SortMergeJoin LeftOuter (36)
+      :                          :        :- * Sort (29)
+      :                          :        :  +- Exchange (28)
+      :                          :        :     +- * Filter (27)
+      :                          :        :        +- * ColumnarToRow (26)
+      :                          :        :           +- Scan parquet default.web_sales (25)
+      :                          :        +- * Sort (35)
+      :                          :           +- Exchange (34)
+      :                          :              +- * Project (33)
+      :                          :                 +- * Filter (32)
+      :                          :                    +- * ColumnarToRow (31)
+      :                          :                       +- Scan parquet default.web_returns (30)
+      :                          +- ReusedExchange (39)
+      +- * Sort (70)
+         +- * Filter (69)
+            +- * HashAggregate (68)
+               +- Exchange (67)
+                  +- * HashAggregate (66)
+                     +- * Project (65)
+                        +- * BroadcastHashJoin Inner BuildRight (64)
+                           :- * Project (62)
+                           :  +- * Filter (61)
+                           :     +- * SortMergeJoin LeftOuter (60)
+                           :        :- * Sort (53)
+                           :        :  +- Exchange (52)
+                           :        :     +- * Filter (51)
+                           :        :        +- * ColumnarToRow (50)
+                           :        :           +- Scan parquet default.catalog_sales (49)
+                           :        +- * Sort (59)
+                           :           +- Exchange (58)
+                           :              +- * Project (57)
+                           :                 +- * Filter (56)
+                           :                    +- * ColumnarToRow (55)
+                           :                       +- Scan parquet default.catalog_returns (54)
+                           +- ReusedExchange (63)
+
+
+(1) Scan parquet default.store_sales
+Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_customer_sk:int,ss_ticket_number:int,ss_quantity:int,ss_wholesale_cost:decimal(7,2),ss_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7]
+
+(3) Filter [codegen id : 1]
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7]
+Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_customer_sk#2))
+
+(4) Exchange
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7]
+Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#9]
+
+(5) Sort [codegen id : 2]
+Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7]
+Arguments: [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.store_returns
+Output [3]: [sr_item_sk#10, sr_ticket_number#11, sr_returned_date_sk#12]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int>
+
+(7) ColumnarToRow [codegen id : 3]
+Input [3]: [sr_item_sk#10, sr_ticket_number#11, sr_returned_date_sk#12]
+
+(8) Filter [codegen id : 3]
+Input [3]: [sr_item_sk#10, sr_ticket_number#11, sr_returned_date_sk#12]
+Condition : (isnotnull(sr_ticket_number#11) AND isnotnull(sr_item_sk#10))
+
+(9) Project [codegen id : 3]
+Output [2]: [sr_item_sk#10, sr_ticket_number#11]
+Input [3]: [sr_item_sk#10, sr_ticket_number#11, sr_returned_date_sk#12]
+
+(10) Exchange
+Input [2]: [sr_item_sk#10, sr_ticket_number#11]
+Arguments: hashpartitioning(sr_ticket_number#11, sr_item_sk#10, 5), ENSURE_REQUIREMENTS, [id=#13]
+
+(11) Sort [codegen id : 4]
+Input [2]: [sr_item_sk#10, sr_ticket_number#11]
+Arguments: [sr_ticket_number#11 ASC NULLS FIRST, sr_item_sk#10 ASC NULLS FIRST], false, 0
+
+(12) SortMergeJoin [codegen id : 6]
+Left keys [2]: [ss_ticket_number#3, ss_item_sk#1]
+Right keys [2]: [sr_ticket_number#11, sr_item_sk#10]
+Join condition: None
+
+(13) Filter [codegen id : 6]
+Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#10, sr_ticket_number#11]
+Condition : isnull(sr_ticket_number#11)
+
+(14) Project [codegen id : 6]
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7]
+Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#10, sr_ticket_number#11]
+
+(15) Scan parquet default.date_dim
+Output [2]: [d_date_sk#14, d_year#15]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_year:int>
+
+(16) ColumnarToRow [codegen id : 5]
+Input [2]: [d_date_sk#14, d_year#15]
+
+(17) Filter [codegen id : 5]
+Input [2]: [d_date_sk#14, d_year#15]
+Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2000)) AND isnotnull(d_date_sk#14))
+
+(18) BroadcastExchange
+Input [2]: [d_date_sk#14, d_year#15]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#16]
+
+(19) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_sold_date_sk#7]
+Right keys [1]: [d_date_sk#14]
+Join condition: None
+
+(20) Project [codegen id : 6]
+Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#15]
+Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#14, d_year#15]
+
+(21) HashAggregate [codegen id : 6]
+Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#15]
+Keys [3]: [d_year#15, ss_item_sk#1, ss_customer_sk#2]
+Functions [3]: [partial_sum(ss_quantity#4), partial_sum(UnscaledValue(ss_wholesale_cost#5)), partial_sum(UnscaledValue(ss_sales_price#6))]
+Aggregate Attributes [3]: [sum#17, sum#18, sum#19]
+Results [6]: [d_year#15, ss_item_sk#1, ss_customer_sk#2, sum#20, sum#21, sum#22]
+
+(22) Exchange
+Input [6]: [d_year#15, ss_item_sk#1, ss_customer_sk#2, sum#20, sum#21, sum#22]
+Arguments: hashpartitioning(d_year#15, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(23) HashAggregate [codegen id : 7]
+Input [6]: [d_year#15, ss_item_sk#1, ss_customer_sk#2, sum#20, sum#21, sum#22]
+Keys [3]: [d_year#15, ss_item_sk#1, ss_customer_sk#2]
+Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum(UnscaledValue(ss_sales_price#6))]
+Aggregate Attributes [3]: [sum(ss_quantity#4)#24, sum(UnscaledValue(ss_wholesale_cost#5))#25, sum(UnscaledValue(ss_sales_price#6))#26]
+Results [6]: [d_year#15 AS ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2, sum(ss_quantity#4)#24 AS ss_qty#28, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#5))#25,17,2) AS ss_wc#29, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#26,17,2) AS ss_sp#30]
+
+(24) Sort [codegen id : 7]
+Input [6]: [ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2, ss_qty#28, ss_wc#29, ss_sp#30]
+Arguments: [ss_sold_year#27 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], false, 0
+
+(25) Scan parquet default.web_sales
+Output [7]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#37), dynamicpruningexpression(ws_sold_date_sk#37 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_bill_customer_sk:int,ws_order_number:int,ws_quantity:int,ws_wholesale_cost:decimal(7,2),ws_sales_price:decimal(7,2)>
+
+(26) ColumnarToRow [codegen id : 8]
+Input [7]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37]
+
+(27) Filter [codegen id : 8]
+Input [7]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37]
+Condition : (isnotnull(ws_item_sk#31) AND isnotnull(ws_bill_customer_sk#32))
+
+(28) Exchange
+Input [7]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37]
+Arguments: hashpartitioning(ws_order_number#33, ws_item_sk#31, 5), ENSURE_REQUIREMENTS, [id=#38]
+
+(29) Sort [codegen id : 9]
+Input [7]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37]
+Arguments: [ws_order_number#33 ASC NULLS FIRST, ws_item_sk#31 ASC NULLS FIRST], false, 0
+
+(30) Scan parquet default.web_returns
+Output [3]: [wr_item_sk#39, wr_order_number#40, wr_returned_date_sk#41]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_returns]
+PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)]
+ReadSchema: struct<wr_item_sk:int,wr_order_number:int>
+
+(31) ColumnarToRow [codegen id : 10]
+Input [3]: [wr_item_sk#39, wr_order_number#40, wr_returned_date_sk#41]
+
+(32) Filter [codegen id : 10]
+Input [3]: [wr_item_sk#39, wr_order_number#40, wr_returned_date_sk#41]
+Condition : (isnotnull(wr_order_number#40) AND isnotnull(wr_item_sk#39))
+
+(33) Project [codegen id : 10]
+Output [2]: [wr_item_sk#39, wr_order_number#40]
+Input [3]: [wr_item_sk#39, wr_order_number#40, wr_returned_date_sk#41]
+
+(34) Exchange
+Input [2]: [wr_item_sk#39, wr_order_number#40]
+Arguments: hashpartitioning(wr_order_number#40, wr_item_sk#39, 5), ENSURE_REQUIREMENTS, [id=#42]
+
+(35) Sort [codegen id : 11]
+Input [2]: [wr_item_sk#39, wr_order_number#40]
+Arguments: [wr_order_number#40 ASC NULLS FIRST, wr_item_sk#39 ASC NULLS FIRST], false, 0
+
+(36) SortMergeJoin [codegen id : 13]
+Left keys [2]: [ws_order_number#33, ws_item_sk#31]
+Right keys [2]: [wr_order_number#40, wr_item_sk#39]
+Join condition: None
+
+(37) Filter [codegen id : 13]
+Input [9]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37, wr_item_sk#39, wr_order_number#40]
+Condition : isnull(wr_order_number#40)
+
+(38) Project [codegen id : 13]
+Output [6]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37]
+Input [9]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_order_number#33, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37, wr_item_sk#39, wr_order_number#40]
+
+(39) ReusedExchange [Reuses operator id: 18]
+Output [2]: [d_date_sk#43, d_year#44]
+
+(40) BroadcastHashJoin [codegen id : 13]
+Left keys [1]: [ws_sold_date_sk#37]
+Right keys [1]: [d_date_sk#43]
+Join condition: None
+
+(41) Project [codegen id : 13]
+Output [6]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, d_year#44]
+Input [8]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, ws_sold_date_sk#37, d_date_sk#43, d_year#44]
+
+(42) HashAggregate [codegen id : 13]
+Input [6]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#34, ws_wholesale_cost#35, ws_sales_price#36, d_year#44]
+Keys [3]: [d_year#44, ws_item_sk#31, ws_bill_customer_sk#32]
+Functions [3]: [partial_sum(ws_quantity#34), partial_sum(UnscaledValue(ws_wholesale_cost#35)), partial_sum(UnscaledValue(ws_sales_price#36))]
+Aggregate Attributes [3]: [sum#45, sum#46, sum#47]
+Results [6]: [d_year#44, ws_item_sk#31, ws_bill_customer_sk#32, sum#48, sum#49, sum#50]
+
+(43) Exchange
+Input [6]: [d_year#44, ws_item_sk#31, ws_bill_customer_sk#32, sum#48, sum#49, sum#50]
+Arguments: hashpartitioning(d_year#44, ws_item_sk#31, ws_bill_customer_sk#32, 5), ENSURE_REQUIREMENTS, [id=#51]
+
+(44) HashAggregate [codegen id : 14]
+Input [6]: [d_year#44, ws_item_sk#31, ws_bill_customer_sk#32, sum#48, sum#49, sum#50]
+Keys [3]: [d_year#44, ws_item_sk#31, ws_bill_customer_sk#32]
+Functions [3]: [sum(ws_quantity#34), sum(UnscaledValue(ws_wholesale_cost#35)), sum(UnscaledValue(ws_sales_price#36))]
+Aggregate Attributes [3]: [sum(ws_quantity#34)#52, sum(UnscaledValue(ws_wholesale_cost#35))#53, sum(UnscaledValue(ws_sales_price#36))#54]
+Results [6]: [d_year#44 AS ws_sold_year#55, ws_item_sk#31, ws_bill_customer_sk#32 AS ws_customer_sk#56, sum(ws_quantity#34)#52 AS ws_qty#57, MakeDecimal(sum(UnscaledValue(ws_wholesale_cost#35))#53,17,2) AS ws_wc#58, MakeDecimal(sum(UnscaledValue(ws_sales_price#36))#54,17,2) AS ws_sp#59]
+
+(45) Filter [codegen id : 14]
+Input [6]: [ws_sold_year#55, ws_item_sk#31, ws_customer_sk#56, ws_qty#57, ws_wc#58, ws_sp#59]
+Condition : (coalesce(ws_qty#57, 0) > 0)
+
+(46) Sort [codegen id : 14]
+Input [6]: [ws_sold_year#55, ws_item_sk#31, ws_customer_sk#56, ws_qty#57, ws_wc#58, ws_sp#59]
+Arguments: [ws_sold_year#55 ASC NULLS FIRST, ws_item_sk#31 ASC NULLS FIRST, ws_customer_sk#56 ASC NULLS FIRST], false, 0
+
+(47) SortMergeJoin [codegen id : 15]
+Left keys [3]: [ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2]
+Right keys [3]: [ws_sold_year#55, ws_item_sk#31, ws_customer_sk#56]
+Join condition: None
+
+(48) Project [codegen id : 15]
+Output [9]: [ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2, ss_qty#28, ss_wc#29, ss_sp#30, ws_qty#57, ws_wc#58, ws_sp#59]
+Input [12]: [ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2, ss_qty#28, ss_wc#29, ss_sp#30, ws_sold_year#55, ws_item_sk#31, ws_customer_sk#56, ws_qty#57, ws_wc#58, ws_sp#59]
+
+(49) Scan parquet default.catalog_sales
+Output [7]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#66), dynamicpruningexpression(cs_sold_date_sk#66 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)]
+ReadSchema: struct<cs_bill_customer_sk:int,cs_item_sk:int,cs_order_number:int,cs_quantity:int,cs_wholesale_cost:decimal(7,2),cs_sales_price:decimal(7,2)>
+
+(50) ColumnarToRow [codegen id : 16]
+Input [7]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66]
+
+(51) Filter [codegen id : 16]
+Input [7]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66]
+Condition : (isnotnull(cs_item_sk#61) AND isnotnull(cs_bill_customer_sk#60))
+
+(52) Exchange
+Input [7]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66]
+Arguments: hashpartitioning(cs_order_number#62, cs_item_sk#61, 5), ENSURE_REQUIREMENTS, [id=#67]
+
+(53) Sort [codegen id : 17]
+Input [7]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66]
+Arguments: [cs_order_number#62 ASC NULLS FIRST, cs_item_sk#61 ASC NULLS FIRST], false, 0
+
+(54) Scan parquet default.catalog_returns
+Output [3]: [cr_item_sk#68, cr_order_number#69, cr_returned_date_sk#70]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int>
+
+(55) ColumnarToRow [codegen id : 18]
+Input [3]: [cr_item_sk#68, cr_order_number#69, cr_returned_date_sk#70]
+
+(56) Filter [codegen id : 18]
+Input [3]: [cr_item_sk#68, cr_order_number#69, cr_returned_date_sk#70]
+Condition : (isnotnull(cr_order_number#69) AND isnotnull(cr_item_sk#68))
+
+(57) Project [codegen id : 18]
+Output [2]: [cr_item_sk#68, cr_order_number#69]
+Input [3]: [cr_item_sk#68, cr_order_number#69, cr_returned_date_sk#70]
+
+(58) Exchange
+Input [2]: [cr_item_sk#68, cr_order_number#69]
+Arguments: hashpartitioning(cr_order_number#69, cr_item_sk#68, 5), ENSURE_REQUIREMENTS, [id=#71]
+
+(59) Sort [codegen id : 19]
+Input [2]: [cr_item_sk#68, cr_order_number#69]
+Arguments: [cr_order_number#69 ASC NULLS FIRST, cr_item_sk#68 ASC NULLS FIRST], false, 0
+
+(60) SortMergeJoin [codegen id : 21]
+Left keys [2]: [cs_order_number#62, cs_item_sk#61]
+Right keys [2]: [cr_order_number#69, cr_item_sk#68]
+Join condition: None
+
+(61) Filter [codegen id : 21]
+Input [9]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66, cr_item_sk#68, cr_order_number#69]
+Condition : isnull(cr_order_number#69)
+
+(62) Project [codegen id : 21]
+Output [6]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66]
+Input [9]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_order_number#62, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66, cr_item_sk#68, cr_order_number#69]
+
+(63) ReusedExchange [Reuses operator id: 18]
+Output [2]: [d_date_sk#72, d_year#73]
+
+(64) BroadcastHashJoin [codegen id : 21]
+Left keys [1]: [cs_sold_date_sk#66]
+Right keys [1]: [d_date_sk#72]
+Join condition: None
+
+(65) Project [codegen id : 21]
+Output [6]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, d_year#73]
+Input [8]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, cs_sold_date_sk#66, d_date_sk#72, d_year#73]
+
+(66) HashAggregate [codegen id : 21]
+Input [6]: [cs_bill_customer_sk#60, cs_item_sk#61, cs_quantity#63, cs_wholesale_cost#64, cs_sales_price#65, d_year#73]
+Keys [3]: [d_year#73, cs_item_sk#61, cs_bill_customer_sk#60]
+Functions [3]: [partial_sum(cs_quantity#63), partial_sum(UnscaledValue(cs_wholesale_cost#64)), partial_sum(UnscaledValue(cs_sales_price#65))]
+Aggregate Attributes [3]: [sum#74, sum#75, sum#76]
+Results [6]: [d_year#73, cs_item_sk#61, cs_bill_customer_sk#60, sum#77, sum#78, sum#79]
+
+(67) Exchange
+Input [6]: [d_year#73, cs_item_sk#61, cs_bill_customer_sk#60, sum#77, sum#78, sum#79]
+Arguments: hashpartitioning(d_year#73, cs_item_sk#61, cs_bill_customer_sk#60, 5), ENSURE_REQUIREMENTS, [id=#80]
+
+(68) HashAggregate [codegen id : 22]
+Input [6]: [d_year#73, cs_item_sk#61, cs_bill_customer_sk#60, sum#77, sum#78, sum#79]
+Keys [3]: [d_year#73, cs_item_sk#61, cs_bill_customer_sk#60]
+Functions [3]: [sum(cs_quantity#63), sum(UnscaledValue(cs_wholesale_cost#64)), sum(UnscaledValue(cs_sales_price#65))]
+Aggregate Attributes [3]: [sum(cs_quantity#63)#81, sum(UnscaledValue(cs_wholesale_cost#64))#82, sum(UnscaledValue(cs_sales_price#65))#83]
+Results [6]: [d_year#73 AS cs_sold_year#84, cs_item_sk#61, cs_bill_customer_sk#60 AS cs_customer_sk#85, sum(cs_quantity#63)#81 AS cs_qty#86, MakeDecimal(sum(UnscaledValue(cs_wholesale_cost#64))#82,17,2) AS cs_wc#87, MakeDecimal(sum(UnscaledValue(cs_sales_price#65))#83,17,2) AS cs_sp#88]
+
+(69) Filter [codegen id : 22]
+Input [6]: [cs_sold_year#84, cs_item_sk#61, cs_customer_sk#85, cs_qty#86, cs_wc#87, cs_sp#88]
+Condition : (coalesce(cs_qty#86, 0) > 0)
+
+(70) Sort [codegen id : 22]
+Input [6]: [cs_sold_year#84, cs_item_sk#61, cs_customer_sk#85, cs_qty#86, cs_wc#87, cs_sp#88]
+Arguments: [cs_sold_year#84 ASC NULLS FIRST, cs_item_sk#61 ASC NULLS FIRST, cs_customer_sk#85 ASC NULLS FIRST], false, 0
+
+(71) SortMergeJoin [codegen id : 23]
+Left keys [3]: [ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2]
+Right keys [3]: [cs_sold_year#84, cs_item_sk#61, cs_customer_sk#85]
+Join condition: None
+
+(72) Project [codegen id : 23]
+Output [13]: [round((cast(ss_qty#28 as double) / cast(coalesce((ws_qty#57 + cs_qty#86), 1) as double)), 2) AS ratio#89, ss_qty#28 AS store_qty#90, ss_wc#29 AS store_wholesale_cost#91, ss_sp#30 AS store_sales_price#92, (coalesce(ws_qty#57, 0) + coalesce(cs_qty#86, 0)) AS other_chan_qty#93, CheckOverflow((promote_precision(cast(coalesce(ws_wc#58, 0.00) as decimal(18,2))) + promote_precision(cast(coalesce(cs_wc#87, 0.00) as decimal(18,2)))), DecimalType(18,2), true) AS other_chan_wholesale_cost#94, CheckOverflow((promote_precision(cast(coalesce(ws_sp#59, 0.00) as decimal(18,2))) + promote_precision(cast(coalesce(cs_sp#88, 0.00) as decimal(18,2)))), DecimalType(18,2), true) AS other_chan_sales_price#95, ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2, ss_qty#28, ss_wc#29, ss_sp#30]
+Input [15]: [ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2, ss_qty#28, ss_wc#29, ss_sp#30, ws_qty#57, ws_wc#58, ws_sp#59, cs_sold_year#84, cs_item_sk#61, cs_customer_sk#85, cs_qty#86, cs_wc#87, cs_sp#88]
+
+(73) TakeOrderedAndProject
+Input [13]: [ratio#89, store_qty#90, store_wholesale_cost#91, store_sales_price#92, other_chan_qty#93, other_chan_wholesale_cost#94, other_chan_sales_price#95, ss_sold_year#27, ss_item_sk#1, ss_customer_sk#2, ss_qty#28, ss_wc#29, ss_sp#30]
+Arguments: 100, [ss_sold_year#27 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST, ss_qty#28 DESC NULLS LAST, ss_wc#29 DESC NULLS LAST, ss_sp#30 DESC NULLS LAST, other_chan_qty#93 ASC NULLS FIRST, other_chan_wholesale_cost#94 ASC NULLS FIRST, other_chan_sales_price#95 ASC NULLS FIRST, ratio#89 ASC NULLS FIRST], [ratio#89, store_qty#90, store_wholesale_cost#91, store_sales_price#92, other_chan_qty#93, other_chan_wholesale_cost#94, other_chan_sales_price#95]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (74)
+
+
+(74) ReusedExchange [Reuses operator id: 18]
+Output [2]: [d_date_sk#14, d_year#15]
+
+Subquery:2 Hosting operator id = 25 Hosting Expression = ws_sold_date_sk#37 IN dynamicpruning#8
+
+Subquery:3 Hosting operator id = 49 Hosting Expression = cs_sold_date_sk#66 IN dynamicpruning#8
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/simplified.txt
new file mode 100644
index 0000000000000..cb9ca4f3d3d32
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/simplified.txt
@@ -0,0 +1,127 @@
+TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholesale_cost,other_chan_sales_price,ratio,store_qty,store_wholesale_cost,store_sales_price]
+  WholeStageCodegen (23)
+    Project [ss_qty,ws_qty,cs_qty,ss_wc,ss_sp,ws_wc,cs_wc,ws_sp,cs_sp,ss_sold_year,ss_item_sk,ss_customer_sk]
+      SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,cs_sold_year,cs_item_sk,cs_customer_sk]
+        InputAdapter
+          WholeStageCodegen (15)
+            Project [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp]
+              SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ws_sold_year,ws_item_sk,ws_customer_sk]
+                InputAdapter
+                  WholeStageCodegen (7)
+                    Sort [ss_sold_year,ss_item_sk,ss_customer_sk]
+                      HashAggregate [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] [sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price)),ss_sold_year,ss_qty,ss_wc,ss_sp,sum,sum,sum]
+                        InputAdapter
+                          Exchange [d_year,ss_item_sk,ss_customer_sk] #1
+                            WholeStageCodegen (6)
+                              HashAggregate [d_year,ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price] [sum,sum,sum,sum,sum,sum]
+                                Project [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year]
+                                  BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                    Project [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk]
+                                      Filter [sr_ticket_number]
+                                        SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk]
+                                          InputAdapter
+                                            WholeStageCodegen (2)
+                                              Sort [ss_ticket_number,ss_item_sk]
+                                                InputAdapter
+                                                  Exchange [ss_ticket_number,ss_item_sk] #2
+                                                    WholeStageCodegen (1)
+                                                      Filter [ss_item_sk,ss_customer_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk]
+                                                              SubqueryBroadcast [d_date_sk] #1
+                                                                ReusedExchange [d_date_sk,d_year] #3
+                                          InputAdapter
+                                            WholeStageCodegen (4)
+                                              Sort [sr_ticket_number,sr_item_sk]
+                                                InputAdapter
+                                                  Exchange [sr_ticket_number,sr_item_sk] #4
+                                                    WholeStageCodegen (3)
+                                                      Project [sr_item_sk,sr_ticket_number]
+                                                        Filter [sr_ticket_number,sr_item_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk]
+                                    InputAdapter
+                                      BroadcastExchange #3
+                                        WholeStageCodegen (5)
+                                          Filter [d_year,d_date_sk]
+                                            ColumnarToRow
+                                              InputAdapter
+                                                Scan parquet default.date_dim [d_date_sk,d_year]
+                InputAdapter
+                  WholeStageCodegen (14)
+                    Sort [ws_sold_year,ws_item_sk,ws_customer_sk]
+                      Filter [ws_qty]
+                        HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] [sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price)),ws_sold_year,ws_customer_sk,ws_qty,ws_wc,ws_sp,sum,sum,sum]
+                          InputAdapter
+                            Exchange [d_year,ws_item_sk,ws_bill_customer_sk] #5
+                              WholeStageCodegen (13)
+                                HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price] [sum,sum,sum,sum,sum,sum]
+                                  Project [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year]
+                                    BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                      Project [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk]
+                                        Filter [wr_order_number]
+                                          SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk]
+                                            InputAdapter
+                                              WholeStageCodegen (9)
+                                                Sort [ws_order_number,ws_item_sk]
+                                                  InputAdapter
+                                                    Exchange [ws_order_number,ws_item_sk] #6
+                                                      WholeStageCodegen (8)
+                                                        Filter [ws_item_sk,ws_bill_customer_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk]
+                                                                ReusedSubquery [d_date_sk] #1
+                                            InputAdapter
+                                              WholeStageCodegen (11)
+                                                Sort [wr_order_number,wr_item_sk]
+                                                  InputAdapter
+                                                    Exchange [wr_order_number,wr_item_sk] #7
+                                                      WholeStageCodegen (10)
+                                                        Project [wr_item_sk,wr_order_number]
+                                                          Filter [wr_order_number,wr_item_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk]
+                                      InputAdapter
+                                        ReusedExchange [d_date_sk,d_year] #3
+        InputAdapter
+          WholeStageCodegen (22)
+            Sort [cs_sold_year,cs_item_sk,cs_customer_sk]
+              Filter [cs_qty]
+                HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] [sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price)),cs_sold_year,cs_customer_sk,cs_qty,cs_wc,cs_sp,sum,sum,sum]
+                  InputAdapter
+                    Exchange [d_year,cs_item_sk,cs_bill_customer_sk] #8
+                      WholeStageCodegen (21)
+                        HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,cs_quantity,cs_wholesale_cost,cs_sales_price] [sum,sum,sum,sum,sum,sum]
+                          Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year]
+                            BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                              Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk]
+                                Filter [cr_order_number]
+                                  SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk]
+                                    InputAdapter
+                                      WholeStageCodegen (17)
+                                        Sort [cs_order_number,cs_item_sk]
+                                          InputAdapter
+                                            Exchange [cs_order_number,cs_item_sk] #9
+                                              WholeStageCodegen (16)
+                                                Filter [cs_item_sk,cs_bill_customer_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk]
+                                                        ReusedSubquery [d_date_sk] #1
+                                    InputAdapter
+                                      WholeStageCodegen (19)
+                                        Sort [cr_order_number,cr_item_sk]
+                                          InputAdapter
+                                            Exchange [cr_order_number,cr_item_sk] #10
+                                              WholeStageCodegen (18)
+                                                Project [cr_item_sk,cr_order_number]
+                                                  Filter [cr_order_number,cr_item_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk]
+                              InputAdapter
+                                ReusedExchange [d_date_sk,d_year] #3
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.sf100/explain.txt
new file mode 100644
index 0000000000000..61cae9d5bf5be
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.sf100/explain.txt
@@ -0,0 +1,715 @@
+== Physical Plan ==
+TakeOrderedAndProject (124)
++- * HashAggregate (123)
+   +- Exchange (122)
+      +- * HashAggregate (121)
+         +- Union (120)
+            :- * HashAggregate (109)
+            :  +- Exchange (108)
+            :     +- * HashAggregate (107)
+            :        +- Union (106)
+            :           :- * HashAggregate (43)
+            :           :  +- Exchange (42)
+            :           :     +- * HashAggregate (41)
+            :           :        +- * Project (40)
+            :           :           +- * BroadcastHashJoin Inner BuildRight (39)
+            :           :              :- * Project (34)
+            :           :              :  +- * BroadcastHashJoin Inner BuildRight (33)
+            :           :              :     :- * Project (27)
+            :           :              :     :  +- * BroadcastHashJoin Inner BuildRight (26)
+            :           :              :     :     :- * Project (20)
+            :           :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (19)
+            :           :              :     :     :     :- * Project (13)
+            :           :              :     :     :     :  +- * SortMergeJoin LeftOuter (12)
+            :           :              :     :     :     :     :- * Sort (5)
+            :           :              :     :     :     :     :  +- Exchange (4)
+            :           :              :     :     :     :     :     +- * Filter (3)
+            :           :              :     :     :     :     :        +- * ColumnarToRow (2)
+            :           :              :     :     :     :     :           +- Scan parquet default.store_sales (1)
+            :           :              :     :     :     :     +- * Sort (11)
+            :           :              :     :     :     :        +- Exchange (10)
+            :           :              :     :     :     :           +- * Project (9)
+            :           :              :     :     :     :              +- * Filter (8)
+            :           :              :     :     :     :                 +- * ColumnarToRow (7)
+            :           :              :     :     :     :                    +- Scan parquet default.store_returns (6)
+            :           :              :     :     :     +- BroadcastExchange (18)
+            :           :              :     :     :        +- * Project (17)
+            :           :              :     :     :           +- * Filter (16)
+            :           :              :     :     :              +- * ColumnarToRow (15)
+            :           :              :     :     :                 +- Scan parquet default.item (14)
+            :           :              :     :     +- BroadcastExchange (25)
+            :           :              :     :        +- * Project (24)
+            :           :              :     :           +- * Filter (23)
+            :           :              :     :              +- * ColumnarToRow (22)
+            :           :              :     :                 +- Scan parquet default.promotion (21)
+            :           :              :     +- BroadcastExchange (32)
+            :           :              :        +- * Project (31)
+            :           :              :           +- * Filter (30)
+            :           :              :              +- * ColumnarToRow (29)
+            :           :              :                 +- Scan parquet default.date_dim (28)
+            :           :              +- BroadcastExchange (38)
+            :           :                 +- * Filter (37)
+            :           :                    +- * ColumnarToRow (36)
+            :           :                       +- Scan parquet default.store (35)
+            :           :- * HashAggregate (74)
+            :           :  +- Exchange (73)
+            :           :     +- * HashAggregate (72)
+            :           :        +- * Project (71)
+            :           :           +- * BroadcastHashJoin Inner BuildRight (70)
+            :           :              :- * Project (65)
+            :           :              :  +- * BroadcastHashJoin Inner BuildRight (64)
+            :           :              :     :- * Project (62)
+            :           :              :     :  +- * BroadcastHashJoin Inner BuildRight (61)
+            :           :              :     :     :- * Project (59)
+            :           :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (58)
+            :           :              :     :     :     :- * Project (56)
+            :           :              :     :     :     :  +- * SortMergeJoin LeftOuter (55)
+            :           :              :     :     :     :     :- * Sort (48)
+            :           :              :     :     :     :     :  +- Exchange (47)
+            :           :              :     :     :     :     :     +- * Filter (46)
+            :           :              :     :     :     :     :        +- * ColumnarToRow (45)
+            :           :              :     :     :     :     :           +- Scan parquet default.catalog_sales (44)
+            :           :              :     :     :     :     +- * Sort (54)
+            :           :              :     :     :     :        +- Exchange (53)
+            :           :              :     :     :     :           +- * Project (52)
+            :           :              :     :     :     :              +- * Filter (51)
+            :           :              :     :     :     :                 +- * ColumnarToRow (50)
+            :           :              :     :     :     :                    +- Scan parquet default.catalog_returns (49)
+            :           :              :     :     :     +- ReusedExchange (57)
+            :           :              :     :     +- ReusedExchange (60)
+            :           :              :     +- ReusedExchange (63)
+            :           :              +- BroadcastExchange (69)
+            :           :                 +- * Filter (68)
+            :           :                    +- * ColumnarToRow (67)
+            :           :                       +- Scan parquet default.catalog_page (66)
+            :           +- * HashAggregate (105)
+            :              +- Exchange (104)
+            :                 +- * HashAggregate (103)
+            :                    +- * Project (102)
+            :                       +- * BroadcastHashJoin Inner BuildRight (101)
+            :                          :- * Project (96)
+            :                          :  +- * BroadcastHashJoin Inner BuildRight (95)
+            :                          :     :- * Project (93)
+            :                          :     :  +- * BroadcastHashJoin Inner BuildRight (92)
+            :                          :     :     :- * Project (90)
+            :                          :     :     :  +- * BroadcastHashJoin Inner BuildRight (89)
+            :                          :     :     :     :- * Project (87)
+            :                          :     :     :     :  +- * SortMergeJoin LeftOuter (86)
+            :                          :     :     :     :     :- * Sort (79)
+            :                          :     :     :     :     :  +- Exchange (78)
+            :                          :     :     :     :     :     +- * Filter (77)
+            :                          :     :     :     :     :        +- * ColumnarToRow (76)
+            :                          :     :     :     :     :           +- Scan parquet default.web_sales (75)
+            :                          :     :     :     :     +- * Sort (85)
+            :                          :     :     :     :        +- Exchange (84)
+            :                          :     :     :     :           +- * Project (83)
+            :                          :     :     :     :              +- * Filter (82)
+            :                          :     :     :     :                 +- * ColumnarToRow (81)
+            :                          :     :     :     :                    +- Scan parquet default.web_returns (80)
+            :                          :     :     :     +- ReusedExchange (88)
+            :                          :     :     +- ReusedExchange (91)
+            :                          :     +- ReusedExchange (94)
+            :                          +- BroadcastExchange (100)
+            :                             +- * Filter (99)
+            :                                +- * ColumnarToRow (98)
+            :                                   +- Scan parquet default.web_site (97)
+            :- * HashAggregate (114)
+            :  +- Exchange (113)
+            :     +- * HashAggregate (112)
+            :        +- * HashAggregate (111)
+            :           +- ReusedExchange (110)
+            +- * HashAggregate (119)
+               +- Exchange (118)
+                  +- * HashAggregate (117)
+                     +- * HashAggregate (116)
+                        +- ReusedExchange (115)
+
+
+(1) Scan parquet default.store_sales
+Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_promo_sk:int,ss_ticket_number:int,ss_ext_sales_price:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+
+(3) Filter [codegen id : 1]
+Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3))
+
+(4) Exchange
+Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, [id=#9]
+
+(5) Sort [codegen id : 2]
+Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+Arguments: [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.store_returns
+Output [5]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int,sr_return_amt:decimal(7,2),sr_net_loss:decimal(7,2)>
+
+(7) ColumnarToRow [codegen id : 3]
+Input [5]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14]
+
+(8) Filter [codegen id : 3]
+Input [5]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14]
+Condition : (isnotnull(sr_item_sk#10) AND isnotnull(sr_ticket_number#11))
+
+(9) Project [codegen id : 3]
+Output [4]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13]
+Input [5]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14]
+
+(10) Exchange
+Input [4]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13]
+Arguments: hashpartitioning(sr_item_sk#10, sr_ticket_number#11, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(11) Sort [codegen id : 4]
+Input [4]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13]
+Arguments: [sr_item_sk#10 ASC NULLS FIRST, sr_ticket_number#11 ASC NULLS FIRST], false, 0
+
+(12) SortMergeJoin [codegen id : 9]
+Left keys [2]: [ss_item_sk#1, ss_ticket_number#4]
+Right keys [2]: [sr_item_sk#10, sr_ticket_number#11]
+Join condition: None
+
+(13) Project [codegen id : 9]
+Output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#12, sr_net_loss#13]
+Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13]
+
+(14) Scan parquet default.item
+Output [2]: [i_item_sk#16, i_current_price#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_current_price:decimal(7,2)>
+
+(15) ColumnarToRow [codegen id : 5]
+Input [2]: [i_item_sk#16, i_current_price#17]
+
+(16) Filter [codegen id : 5]
+Input [2]: [i_item_sk#16, i_current_price#17]
+Condition : ((isnotnull(i_current_price#17) AND (i_current_price#17 > 50.00)) AND isnotnull(i_item_sk#16))
+
+(17) Project [codegen id : 5]
+Output [1]: [i_item_sk#16]
+Input [2]: [i_item_sk#16, i_current_price#17]
+
+(18) BroadcastExchange
+Input [1]: [i_item_sk#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(19) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#16]
+Join condition: None
+
+(20) Project [codegen id : 9]
+Output [7]: [ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#12, sr_net_loss#13]
+Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#12, sr_net_loss#13, i_item_sk#16]
+
+(21) Scan parquet default.promotion
+Output [2]: [p_promo_sk#19, p_channel_tv#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/promotion]
+PushedFilters: [IsNotNull(p_channel_tv), EqualTo(p_channel_tv,N), IsNotNull(p_promo_sk)]
+ReadSchema: struct<p_promo_sk:int,p_channel_tv:string>
+
+(22) ColumnarToRow [codegen id : 6]
+Input [2]: [p_promo_sk#19, p_channel_tv#20]
+
+(23) Filter [codegen id : 6]
+Input [2]: [p_promo_sk#19, p_channel_tv#20]
+Condition : ((isnotnull(p_channel_tv#20) AND (p_channel_tv#20 = N)) AND isnotnull(p_promo_sk#19))
+
+(24) Project [codegen id : 6]
+Output [1]: [p_promo_sk#19]
+Input [2]: [p_promo_sk#19, p_channel_tv#20]
+
+(25) BroadcastExchange
+Input [1]: [p_promo_sk#19]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#21]
+
+(26) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_promo_sk#3]
+Right keys [1]: [p_promo_sk#19]
+Join condition: None
+
+(27) Project [codegen id : 9]
+Output [6]: [ss_store_sk#2, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#12, sr_net_loss#13]
+Input [8]: [ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#12, sr_net_loss#13, p_promo_sk#19]
+
+(28) Scan parquet default.date_dim
+Output [2]: [d_date_sk#22, d_date#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(29) ColumnarToRow [codegen id : 7]
+Input [2]: [d_date_sk#22, d_date#23]
+
+(30) Filter [codegen id : 7]
+Input [2]: [d_date_sk#22, d_date#23]
+Condition : (((isnotnull(d_date#23) AND (d_date#23 >= 1998-08-04)) AND (d_date#23 <= 1998-09-03)) AND isnotnull(d_date_sk#22))
+
+(31) Project [codegen id : 7]
+Output [1]: [d_date_sk#22]
+Input [2]: [d_date_sk#22, d_date#23]
+
+(32) BroadcastExchange
+Input [1]: [d_date_sk#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#24]
+
+(33) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_sold_date_sk#7]
+Right keys [1]: [d_date_sk#22]
+Join condition: None
+
+(34) Project [codegen id : 9]
+Output [5]: [ss_store_sk#2, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#12, sr_net_loss#13]
+Input [7]: [ss_store_sk#2, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#12, sr_net_loss#13, d_date_sk#22]
+
+(35) Scan parquet default.store
+Output [2]: [s_store_sk#25, s_store_id#26]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string>
+
+(36) ColumnarToRow [codegen id : 8]
+Input [2]: [s_store_sk#25, s_store_id#26]
+
+(37) Filter [codegen id : 8]
+Input [2]: [s_store_sk#25, s_store_id#26]
+Condition : isnotnull(s_store_sk#25)
+
+(38) BroadcastExchange
+Input [2]: [s_store_sk#25, s_store_id#26]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#27]
+
+(39) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_store_sk#2]
+Right keys [1]: [s_store_sk#25]
+Join condition: None
+
+(40) Project [codegen id : 9]
+Output [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#12, sr_net_loss#13, s_store_id#26]
+Input [7]: [ss_store_sk#2, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#12, sr_net_loss#13, s_store_sk#25, s_store_id#26]
+
+(41) HashAggregate [codegen id : 9]
+Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#12, sr_net_loss#13, s_store_id#26]
+Keys [1]: [s_store_id#26]
+Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#12 as decimal(12,2)), 0.00)), partial_sum(CheckOverflow((promote_precision(cast(ss_net_profit#6 as decimal(13,2))) - promote_precision(cast(coalesce(cast(sr_net_loss#13 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))]
+Aggregate Attributes [5]: [sum#28, sum#29, isEmpty#30, sum#31, isEmpty#32]
+Results [6]: [s_store_id#26, sum#33, sum#34, isEmpty#35, sum#36, isEmpty#37]
+
+(42) Exchange
+Input [6]: [s_store_id#26, sum#33, sum#34, isEmpty#35, sum#36, isEmpty#37]
+Arguments: hashpartitioning(s_store_id#26, 5), ENSURE_REQUIREMENTS, [id=#38]
+
+(43) HashAggregate [codegen id : 10]
+Input [6]: [s_store_id#26, sum#33, sum#34, isEmpty#35, sum#36, isEmpty#37]
+Keys [1]: [s_store_id#26]
+Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#12 as decimal(12,2)), 0.00)), sum(CheckOverflow((promote_precision(cast(ss_net_profit#6 as decimal(13,2))) - promote_precision(cast(coalesce(cast(sr_net_loss#13 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))]
+Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#39, sum(coalesce(cast(sr_return_amt#12 as decimal(12,2)), 0.00))#40, sum(CheckOverflow((promote_precision(cast(ss_net_profit#6 as decimal(13,2))) - promote_precision(cast(coalesce(cast(sr_net_loss#13 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))#41]
+Results [5]: [store channel AS channel#42, concat(store, s_store_id#26) AS id#43, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#39,17,2) AS sales#44, sum(coalesce(cast(sr_return_amt#12 as decimal(12,2)), 0.00))#40 AS returns#45, sum(CheckOverflow((promote_precision(cast(ss_net_profit#6 as decimal(13,2))) - promote_precision(cast(coalesce(cast(sr_net_loss#13 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))#41 AS profit#46]
+
+(44) Scan parquet default.catalog_sales
+Output [7]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_order_number#50, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#53), dynamicpruningexpression(cs_sold_date_sk#53 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)]
+ReadSchema: struct<cs_catalog_page_sk:int,cs_item_sk:int,cs_promo_sk:int,cs_order_number:int,cs_ext_sales_price:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(45) ColumnarToRow [codegen id : 11]
+Input [7]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_order_number#50, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53]
+
+(46) Filter [codegen id : 11]
+Input [7]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_order_number#50, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53]
+Condition : ((isnotnull(cs_catalog_page_sk#47) AND isnotnull(cs_item_sk#48)) AND isnotnull(cs_promo_sk#49))
+
+(47) Exchange
+Input [7]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_order_number#50, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53]
+Arguments: hashpartitioning(cs_item_sk#48, cs_order_number#50, 5), ENSURE_REQUIREMENTS, [id=#54]
+
+(48) Sort [codegen id : 12]
+Input [7]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_order_number#50, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53]
+Arguments: [cs_item_sk#48 ASC NULLS FIRST, cs_order_number#50 ASC NULLS FIRST], false, 0
+
+(49) Scan parquet default.catalog_returns
+Output [5]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int,cr_return_amount:decimal(7,2),cr_net_loss:decimal(7,2)>
+
+(50) ColumnarToRow [codegen id : 13]
+Input [5]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+
+(51) Filter [codegen id : 13]
+Input [5]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+Condition : (isnotnull(cr_item_sk#55) AND isnotnull(cr_order_number#56))
+
+(52) Project [codegen id : 13]
+Output [4]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58]
+Input [5]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+
+(53) Exchange
+Input [4]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58]
+Arguments: hashpartitioning(cr_item_sk#55, cr_order_number#56, 5), ENSURE_REQUIREMENTS, [id=#60]
+
+(54) Sort [codegen id : 14]
+Input [4]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58]
+Arguments: [cr_item_sk#55 ASC NULLS FIRST, cr_order_number#56 ASC NULLS FIRST], false, 0
+
+(55) SortMergeJoin [codegen id : 19]
+Left keys [2]: [cs_item_sk#48, cs_order_number#50]
+Right keys [2]: [cr_item_sk#55, cr_order_number#56]
+Join condition: None
+
+(56) Project [codegen id : 19]
+Output [8]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53, cr_return_amount#57, cr_net_loss#58]
+Input [11]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_order_number#50, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53, cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58]
+
+(57) ReusedExchange [Reuses operator id: 18]
+Output [1]: [i_item_sk#61]
+
+(58) BroadcastHashJoin [codegen id : 19]
+Left keys [1]: [cs_item_sk#48]
+Right keys [1]: [i_item_sk#61]
+Join condition: None
+
+(59) Project [codegen id : 19]
+Output [7]: [cs_catalog_page_sk#47, cs_promo_sk#49, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53, cr_return_amount#57, cr_net_loss#58]
+Input [9]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53, cr_return_amount#57, cr_net_loss#58, i_item_sk#61]
+
+(60) ReusedExchange [Reuses operator id: 25]
+Output [1]: [p_promo_sk#62]
+
+(61) BroadcastHashJoin [codegen id : 19]
+Left keys [1]: [cs_promo_sk#49]
+Right keys [1]: [p_promo_sk#62]
+Join condition: None
+
+(62) Project [codegen id : 19]
+Output [6]: [cs_catalog_page_sk#47, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53, cr_return_amount#57, cr_net_loss#58]
+Input [8]: [cs_catalog_page_sk#47, cs_promo_sk#49, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53, cr_return_amount#57, cr_net_loss#58, p_promo_sk#62]
+
+(63) ReusedExchange [Reuses operator id: 32]
+Output [1]: [d_date_sk#63]
+
+(64) BroadcastHashJoin [codegen id : 19]
+Left keys [1]: [cs_sold_date_sk#53]
+Right keys [1]: [d_date_sk#63]
+Join condition: None
+
+(65) Project [codegen id : 19]
+Output [5]: [cs_catalog_page_sk#47, cs_ext_sales_price#51, cs_net_profit#52, cr_return_amount#57, cr_net_loss#58]
+Input [7]: [cs_catalog_page_sk#47, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53, cr_return_amount#57, cr_net_loss#58, d_date_sk#63]
+
+(66) Scan parquet default.catalog_page
+Output [2]: [cp_catalog_page_sk#64, cp_catalog_page_id#65]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_page]
+PushedFilters: [IsNotNull(cp_catalog_page_sk)]
+ReadSchema: struct<cp_catalog_page_sk:int,cp_catalog_page_id:string>
+
+(67) ColumnarToRow [codegen id : 18]
+Input [2]: [cp_catalog_page_sk#64, cp_catalog_page_id#65]
+
+(68) Filter [codegen id : 18]
+Input [2]: [cp_catalog_page_sk#64, cp_catalog_page_id#65]
+Condition : isnotnull(cp_catalog_page_sk#64)
+
+(69) BroadcastExchange
+Input [2]: [cp_catalog_page_sk#64, cp_catalog_page_id#65]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#66]
+
+(70) BroadcastHashJoin [codegen id : 19]
+Left keys [1]: [cs_catalog_page_sk#47]
+Right keys [1]: [cp_catalog_page_sk#64]
+Join condition: None
+
+(71) Project [codegen id : 19]
+Output [5]: [cs_ext_sales_price#51, cs_net_profit#52, cr_return_amount#57, cr_net_loss#58, cp_catalog_page_id#65]
+Input [7]: [cs_catalog_page_sk#47, cs_ext_sales_price#51, cs_net_profit#52, cr_return_amount#57, cr_net_loss#58, cp_catalog_page_sk#64, cp_catalog_page_id#65]
+
+(72) HashAggregate [codegen id : 19]
+Input [5]: [cs_ext_sales_price#51, cs_net_profit#52, cr_return_amount#57, cr_net_loss#58, cp_catalog_page_id#65]
+Keys [1]: [cp_catalog_page_id#65]
+Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#51)), partial_sum(coalesce(cast(cr_return_amount#57 as decimal(12,2)), 0.00)), partial_sum(CheckOverflow((promote_precision(cast(cs_net_profit#52 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_net_loss#58 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))]
+Aggregate Attributes [5]: [sum#67, sum#68, isEmpty#69, sum#70, isEmpty#71]
+Results [6]: [cp_catalog_page_id#65, sum#72, sum#73, isEmpty#74, sum#75, isEmpty#76]
+
+(73) Exchange
+Input [6]: [cp_catalog_page_id#65, sum#72, sum#73, isEmpty#74, sum#75, isEmpty#76]
+Arguments: hashpartitioning(cp_catalog_page_id#65, 5), ENSURE_REQUIREMENTS, [id=#77]
+
+(74) HashAggregate [codegen id : 20]
+Input [6]: [cp_catalog_page_id#65, sum#72, sum#73, isEmpty#74, sum#75, isEmpty#76]
+Keys [1]: [cp_catalog_page_id#65]
+Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#51)), sum(coalesce(cast(cr_return_amount#57 as decimal(12,2)), 0.00)), sum(CheckOverflow((promote_precision(cast(cs_net_profit#52 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_net_loss#58 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))]
+Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#51))#78, sum(coalesce(cast(cr_return_amount#57 as decimal(12,2)), 0.00))#79, sum(CheckOverflow((promote_precision(cast(cs_net_profit#52 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_net_loss#58 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))#80]
+Results [5]: [catalog channel AS channel#81, concat(catalog_page, cp_catalog_page_id#65) AS id#82, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#51))#78,17,2) AS sales#83, sum(coalesce(cast(cr_return_amount#57 as decimal(12,2)), 0.00))#79 AS returns#84, sum(CheckOverflow((promote_precision(cast(cs_net_profit#52 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_net_loss#58 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))#80 AS profit#85]
+
+(75) Scan parquet default.web_sales
+Output [7]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_order_number#89, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#92), dynamicpruningexpression(ws_sold_date_sk#92 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_web_site_sk:int,ws_promo_sk:int,ws_order_number:int,ws_ext_sales_price:decimal(7,2),ws_net_profit:decimal(7,2)>
+
+(76) ColumnarToRow [codegen id : 21]
+Input [7]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_order_number#89, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92]
+
+(77) Filter [codegen id : 21]
+Input [7]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_order_number#89, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92]
+Condition : ((isnotnull(ws_web_site_sk#87) AND isnotnull(ws_item_sk#86)) AND isnotnull(ws_promo_sk#88))
+
+(78) Exchange
+Input [7]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_order_number#89, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92]
+Arguments: hashpartitioning(ws_item_sk#86, ws_order_number#89, 5), ENSURE_REQUIREMENTS, [id=#93]
+
+(79) Sort [codegen id : 22]
+Input [7]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_order_number#89, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92]
+Arguments: [ws_item_sk#86 ASC NULLS FIRST, ws_order_number#89 ASC NULLS FIRST], false, 0
+
+(80) Scan parquet default.web_returns
+Output [5]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_returns]
+PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)]
+ReadSchema: struct<wr_item_sk:int,wr_order_number:int,wr_return_amt:decimal(7,2),wr_net_loss:decimal(7,2)>
+
+(81) ColumnarToRow [codegen id : 23]
+Input [5]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98]
+
+(82) Filter [codegen id : 23]
+Input [5]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98]
+Condition : (isnotnull(wr_item_sk#94) AND isnotnull(wr_order_number#95))
+
+(83) Project [codegen id : 23]
+Output [4]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97]
+Input [5]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98]
+
+(84) Exchange
+Input [4]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97]
+Arguments: hashpartitioning(wr_item_sk#94, wr_order_number#95, 5), ENSURE_REQUIREMENTS, [id=#99]
+
+(85) Sort [codegen id : 24]
+Input [4]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97]
+Arguments: [wr_item_sk#94 ASC NULLS FIRST, wr_order_number#95 ASC NULLS FIRST], false, 0
+
+(86) SortMergeJoin [codegen id : 29]
+Left keys [2]: [ws_item_sk#86, ws_order_number#89]
+Right keys [2]: [wr_item_sk#94, wr_order_number#95]
+Join condition: None
+
+(87) Project [codegen id : 29]
+Output [8]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92, wr_return_amt#96, wr_net_loss#97]
+Input [11]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_order_number#89, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92, wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97]
+
+(88) ReusedExchange [Reuses operator id: 18]
+Output [1]: [i_item_sk#100]
+
+(89) BroadcastHashJoin [codegen id : 29]
+Left keys [1]: [ws_item_sk#86]
+Right keys [1]: [i_item_sk#100]
+Join condition: None
+
+(90) Project [codegen id : 29]
+Output [7]: [ws_web_site_sk#87, ws_promo_sk#88, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92, wr_return_amt#96, wr_net_loss#97]
+Input [9]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92, wr_return_amt#96, wr_net_loss#97, i_item_sk#100]
+
+(91) ReusedExchange [Reuses operator id: 25]
+Output [1]: [p_promo_sk#101]
+
+(92) BroadcastHashJoin [codegen id : 29]
+Left keys [1]: [ws_promo_sk#88]
+Right keys [1]: [p_promo_sk#101]
+Join condition: None
+
+(93) Project [codegen id : 29]
+Output [6]: [ws_web_site_sk#87, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92, wr_return_amt#96, wr_net_loss#97]
+Input [8]: [ws_web_site_sk#87, ws_promo_sk#88, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92, wr_return_amt#96, wr_net_loss#97, p_promo_sk#101]
+
+(94) ReusedExchange [Reuses operator id: 32]
+Output [1]: [d_date_sk#102]
+
+(95) BroadcastHashJoin [codegen id : 29]
+Left keys [1]: [ws_sold_date_sk#92]
+Right keys [1]: [d_date_sk#102]
+Join condition: None
+
+(96) Project [codegen id : 29]
+Output [5]: [ws_web_site_sk#87, ws_ext_sales_price#90, ws_net_profit#91, wr_return_amt#96, wr_net_loss#97]
+Input [7]: [ws_web_site_sk#87, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92, wr_return_amt#96, wr_net_loss#97, d_date_sk#102]
+
+(97) Scan parquet default.web_site
+Output [2]: [web_site_sk#103, web_site_id#104]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_site]
+PushedFilters: [IsNotNull(web_site_sk)]
+ReadSchema: struct<web_site_sk:int,web_site_id:string>
+
+(98) ColumnarToRow [codegen id : 28]
+Input [2]: [web_site_sk#103, web_site_id#104]
+
+(99) Filter [codegen id : 28]
+Input [2]: [web_site_sk#103, web_site_id#104]
+Condition : isnotnull(web_site_sk#103)
+
+(100) BroadcastExchange
+Input [2]: [web_site_sk#103, web_site_id#104]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#105]
+
+(101) BroadcastHashJoin [codegen id : 29]
+Left keys [1]: [ws_web_site_sk#87]
+Right keys [1]: [web_site_sk#103]
+Join condition: None
+
+(102) Project [codegen id : 29]
+Output [5]: [ws_ext_sales_price#90, ws_net_profit#91, wr_return_amt#96, wr_net_loss#97, web_site_id#104]
+Input [7]: [ws_web_site_sk#87, ws_ext_sales_price#90, ws_net_profit#91, wr_return_amt#96, wr_net_loss#97, web_site_sk#103, web_site_id#104]
+
+(103) HashAggregate [codegen id : 29]
+Input [5]: [ws_ext_sales_price#90, ws_net_profit#91, wr_return_amt#96, wr_net_loss#97, web_site_id#104]
+Keys [1]: [web_site_id#104]
+Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#90)), partial_sum(coalesce(cast(wr_return_amt#96 as decimal(12,2)), 0.00)), partial_sum(CheckOverflow((promote_precision(cast(ws_net_profit#91 as decimal(13,2))) - promote_precision(cast(coalesce(cast(wr_net_loss#97 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))]
+Aggregate Attributes [5]: [sum#106, sum#107, isEmpty#108, sum#109, isEmpty#110]
+Results [6]: [web_site_id#104, sum#111, sum#112, isEmpty#113, sum#114, isEmpty#115]
+
+(104) Exchange
+Input [6]: [web_site_id#104, sum#111, sum#112, isEmpty#113, sum#114, isEmpty#115]
+Arguments: hashpartitioning(web_site_id#104, 5), ENSURE_REQUIREMENTS, [id=#116]
+
+(105) HashAggregate [codegen id : 30]
+Input [6]: [web_site_id#104, sum#111, sum#112, isEmpty#113, sum#114, isEmpty#115]
+Keys [1]: [web_site_id#104]
+Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#90)), sum(coalesce(cast(wr_return_amt#96 as decimal(12,2)), 0.00)), sum(CheckOverflow((promote_precision(cast(ws_net_profit#91 as decimal(13,2))) - promote_precision(cast(coalesce(cast(wr_net_loss#97 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))]
+Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#90))#117, sum(coalesce(cast(wr_return_amt#96 as decimal(12,2)), 0.00))#118, sum(CheckOverflow((promote_precision(cast(ws_net_profit#91 as decimal(13,2))) - promote_precision(cast(coalesce(cast(wr_net_loss#97 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))#119]
+Results [5]: [web channel AS channel#120, concat(web_site, web_site_id#104) AS id#121, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#90))#117,17,2) AS sales#122, sum(coalesce(cast(wr_return_amt#96 as decimal(12,2)), 0.00))#118 AS returns#123, sum(CheckOverflow((promote_precision(cast(ws_net_profit#91 as decimal(13,2))) - promote_precision(cast(coalesce(cast(wr_net_loss#97 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))#119 AS profit#124]
+
+(106) Union
+
+(107) HashAggregate [codegen id : 31]
+Input [5]: [channel#42, id#43, sales#44, returns#45, profit#46]
+Keys [2]: [channel#42, id#43]
+Functions [3]: [partial_sum(sales#44), partial_sum(returns#45), partial_sum(profit#46)]
+Aggregate Attributes [6]: [sum#125, isEmpty#126, sum#127, isEmpty#128, sum#129, isEmpty#130]
+Results [8]: [channel#42, id#43, sum#131, isEmpty#132, sum#133, isEmpty#134, sum#135, isEmpty#136]
+
+(108) Exchange
+Input [8]: [channel#42, id#43, sum#131, isEmpty#132, sum#133, isEmpty#134, sum#135, isEmpty#136]
+Arguments: hashpartitioning(channel#42, id#43, 5), ENSURE_REQUIREMENTS, [id=#137]
+
+(109) HashAggregate [codegen id : 32]
+Input [8]: [channel#42, id#43, sum#131, isEmpty#132, sum#133, isEmpty#134, sum#135, isEmpty#136]
+Keys [2]: [channel#42, id#43]
+Functions [3]: [sum(sales#44), sum(returns#45), sum(profit#46)]
+Aggregate Attributes [3]: [sum(sales#44)#138, sum(returns#45)#139, sum(profit#46)#140]
+Results [5]: [channel#42, id#43, cast(sum(sales#44)#138 as decimal(37,2)) AS sales#141, cast(sum(returns#45)#139 as decimal(38,2)) AS returns#142, cast(sum(profit#46)#140 as decimal(38,2)) AS profit#143]
+
+(110) ReusedExchange [Reuses operator id: unknown]
+Output [8]: [channel#42, id#43, sum#144, isEmpty#145, sum#146, isEmpty#147, sum#148, isEmpty#149]
+
+(111) HashAggregate [codegen id : 64]
+Input [8]: [channel#42, id#43, sum#144, isEmpty#145, sum#146, isEmpty#147, sum#148, isEmpty#149]
+Keys [2]: [channel#42, id#43]
+Functions [3]: [sum(sales#44), sum(returns#45), sum(profit#46)]
+Aggregate Attributes [3]: [sum(sales#44)#150, sum(returns#45)#151, sum(profit#46)#152]
+Results [4]: [channel#42, sum(sales#44)#150 AS sales#153, sum(returns#45)#151 AS returns#154, sum(profit#46)#152 AS profit#155]
+
+(112) HashAggregate [codegen id : 64]
+Input [4]: [channel#42, sales#153, returns#154, profit#155]
+Keys [1]: [channel#42]
+Functions [3]: [partial_sum(sales#153), partial_sum(returns#154), partial_sum(profit#155)]
+Aggregate Attributes [6]: [sum#156, isEmpty#157, sum#158, isEmpty#159, sum#160, isEmpty#161]
+Results [7]: [channel#42, sum#162, isEmpty#163, sum#164, isEmpty#165, sum#166, isEmpty#167]
+
+(113) Exchange
+Input [7]: [channel#42, sum#162, isEmpty#163, sum#164, isEmpty#165, sum#166, isEmpty#167]
+Arguments: hashpartitioning(channel#42, 5), ENSURE_REQUIREMENTS, [id=#168]
+
+(114) HashAggregate [codegen id : 65]
+Input [7]: [channel#42, sum#162, isEmpty#163, sum#164, isEmpty#165, sum#166, isEmpty#167]
+Keys [1]: [channel#42]
+Functions [3]: [sum(sales#153), sum(returns#154), sum(profit#155)]
+Aggregate Attributes [3]: [sum(sales#153)#169, sum(returns#154)#170, sum(profit#155)#171]
+Results [5]: [channel#42, null AS id#172, sum(sales#153)#169 AS sales#173, sum(returns#154)#170 AS returns#174, sum(profit#155)#171 AS profit#175]
+
+(115) ReusedExchange [Reuses operator id: unknown]
+Output [8]: [channel#42, id#43, sum#176, isEmpty#177, sum#178, isEmpty#179, sum#180, isEmpty#181]
+
+(116) HashAggregate [codegen id : 97]
+Input [8]: [channel#42, id#43, sum#176, isEmpty#177, sum#178, isEmpty#179, sum#180, isEmpty#181]
+Keys [2]: [channel#42, id#43]
+Functions [3]: [sum(sales#44), sum(returns#45), sum(profit#46)]
+Aggregate Attributes [3]: [sum(sales#44)#182, sum(returns#45)#183, sum(profit#46)#184]
+Results [3]: [sum(sales#44)#182 AS sales#153, sum(returns#45)#183 AS returns#154, sum(profit#46)#184 AS profit#155]
+
+(117) HashAggregate [codegen id : 97]
+Input [3]: [sales#153, returns#154, profit#155]
+Keys: []
+Functions [3]: [partial_sum(sales#153), partial_sum(returns#154), partial_sum(profit#155)]
+Aggregate Attributes [6]: [sum#185, isEmpty#186, sum#187, isEmpty#188, sum#189, isEmpty#190]
+Results [6]: [sum#191, isEmpty#192, sum#193, isEmpty#194, sum#195, isEmpty#196]
+
+(118) Exchange
+Input [6]: [sum#191, isEmpty#192, sum#193, isEmpty#194, sum#195, isEmpty#196]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#197]
+
+(119) HashAggregate [codegen id : 98]
+Input [6]: [sum#191, isEmpty#192, sum#193, isEmpty#194, sum#195, isEmpty#196]
+Keys: []
+Functions [3]: [sum(sales#153), sum(returns#154), sum(profit#155)]
+Aggregate Attributes [3]: [sum(sales#153)#198, sum(returns#154)#199, sum(profit#155)#200]
+Results [5]: [null AS channel#201, null AS id#202, sum(sales#153)#198 AS sales#203, sum(returns#154)#199 AS returns#204, sum(profit#155)#200 AS profit#205]
+
+(120) Union
+
+(121) HashAggregate [codegen id : 99]
+Input [5]: [channel#42, id#43, sales#141, returns#142, profit#143]
+Keys [5]: [channel#42, id#43, sales#141, returns#142, profit#143]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [channel#42, id#43, sales#141, returns#142, profit#143]
+
+(122) Exchange
+Input [5]: [channel#42, id#43, sales#141, returns#142, profit#143]
+Arguments: hashpartitioning(channel#42, id#43, sales#141, returns#142, profit#143, 5), ENSURE_REQUIREMENTS, [id=#206]
+
+(123) HashAggregate [codegen id : 100]
+Input [5]: [channel#42, id#43, sales#141, returns#142, profit#143]
+Keys [5]: [channel#42, id#43, sales#141, returns#142, profit#143]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [channel#42, id#43, sales#141, returns#142, profit#143]
+
+(124) TakeOrderedAndProject
+Input [5]: [channel#42, id#43, sales#141, returns#142, profit#143]
+Arguments: 100, [channel#42 ASC NULLS FIRST, id#43 ASC NULLS FIRST], [channel#42, id#43, sales#141, returns#142, profit#143]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (125)
+
+
+(125) ReusedExchange [Reuses operator id: 32]
+Output [1]: [d_date_sk#22]
+
+Subquery:2 Hosting operator id = 44 Hosting Expression = cs_sold_date_sk#53 IN dynamicpruning#8
+
+Subquery:3 Hosting operator id = 75 Hosting Expression = ws_sold_date_sk#92 IN dynamicpruning#8
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.sf100/simplified.txt
new file mode 100644
index 0000000000000..e73825fd5ad7c
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.sf100/simplified.txt
@@ -0,0 +1,207 @@
+TakeOrderedAndProject [channel,id,sales,returns,profit]
+  WholeStageCodegen (100)
+    HashAggregate [channel,id,sales,returns,profit]
+      InputAdapter
+        Exchange [channel,id,sales,returns,profit] #1
+          WholeStageCodegen (99)
+            HashAggregate [channel,id,sales,returns,profit]
+              InputAdapter
+                Union
+                  WholeStageCodegen (32)
+                    HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                      InputAdapter
+                        Exchange [channel,id] #2
+                          WholeStageCodegen (31)
+                            HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                              InputAdapter
+                                Union
+                                  WholeStageCodegen (10)
+                                    HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(CheckOverflow((promote_precision(cast(ss_net_profit as decimal(13,2))) - promote_precision(cast(coalesce(cast(sr_net_loss as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true)),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty]
+                                      InputAdapter
+                                        Exchange [s_store_id] #3
+                                          WholeStageCodegen (9)
+                                            HashAggregate [s_store_id,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty]
+                                              Project [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id]
+                                                BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                  Project [ss_store_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss]
+                                                    BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                      Project [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss]
+                                                        BroadcastHashJoin [ss_promo_sk,p_promo_sk]
+                                                          Project [ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss]
+                                                            BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                              Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss]
+                                                                SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number]
+                                                                  InputAdapter
+                                                                    WholeStageCodegen (2)
+                                                                      Sort [ss_item_sk,ss_ticket_number]
+                                                                        InputAdapter
+                                                                          Exchange [ss_item_sk,ss_ticket_number] #4
+                                                                            WholeStageCodegen (1)
+                                                                              Filter [ss_store_sk,ss_item_sk,ss_promo_sk]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk]
+                                                                                      SubqueryBroadcast [d_date_sk] #1
+                                                                                        ReusedExchange [d_date_sk] #5
+                                                                  InputAdapter
+                                                                    WholeStageCodegen (4)
+                                                                      Sort [sr_item_sk,sr_ticket_number]
+                                                                        InputAdapter
+                                                                          Exchange [sr_item_sk,sr_ticket_number] #6
+                                                                            WholeStageCodegen (3)
+                                                                              Project [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss]
+                                                                                Filter [sr_item_sk,sr_ticket_number]
+                                                                                  ColumnarToRow
+                                                                                    InputAdapter
+                                                                                      Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk]
+                                                              InputAdapter
+                                                                BroadcastExchange #7
+                                                                  WholeStageCodegen (5)
+                                                                    Project [i_item_sk]
+                                                                      Filter [i_current_price,i_item_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.item [i_item_sk,i_current_price]
+                                                          InputAdapter
+                                                            BroadcastExchange #8
+                                                              WholeStageCodegen (6)
+                                                                Project [p_promo_sk]
+                                                                  Filter [p_channel_tv,p_promo_sk]
+                                                                    ColumnarToRow
+                                                                      InputAdapter
+                                                                        Scan parquet default.promotion [p_promo_sk,p_channel_tv]
+                                                      InputAdapter
+                                                        BroadcastExchange #5
+                                                          WholeStageCodegen (7)
+                                                            Project [d_date_sk]
+                                                              Filter [d_date,d_date_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.date_dim [d_date_sk,d_date]
+                                                  InputAdapter
+                                                    BroadcastExchange #9
+                                                      WholeStageCodegen (8)
+                                                        Filter [s_store_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.store [s_store_sk,s_store_id]
+                                  WholeStageCodegen (20)
+                                    HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(CheckOverflow((promote_precision(cast(cs_net_profit as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_net_loss as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true)),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty]
+                                      InputAdapter
+                                        Exchange [cp_catalog_page_id] #10
+                                          WholeStageCodegen (19)
+                                            HashAggregate [cp_catalog_page_id,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty]
+                                              Project [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id]
+                                                BroadcastHashJoin [cs_catalog_page_sk,cp_catalog_page_sk]
+                                                  Project [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss]
+                                                    BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                      Project [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss]
+                                                        BroadcastHashJoin [cs_promo_sk,p_promo_sk]
+                                                          Project [cs_catalog_page_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss]
+                                                            BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                                              Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss]
+                                                                SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number]
+                                                                  InputAdapter
+                                                                    WholeStageCodegen (12)
+                                                                      Sort [cs_item_sk,cs_order_number]
+                                                                        InputAdapter
+                                                                          Exchange [cs_item_sk,cs_order_number] #11
+                                                                            WholeStageCodegen (11)
+                                                                              Filter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk]
+                                                                                      ReusedSubquery [d_date_sk] #1
+                                                                  InputAdapter
+                                                                    WholeStageCodegen (14)
+                                                                      Sort [cr_item_sk,cr_order_number]
+                                                                        InputAdapter
+                                                                          Exchange [cr_item_sk,cr_order_number] #12
+                                                                            WholeStageCodegen (13)
+                                                                              Project [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss]
+                                                                                Filter [cr_item_sk,cr_order_number]
+                                                                                  ColumnarToRow
+                                                                                    InputAdapter
+                                                                                      Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk]
+                                                              InputAdapter
+                                                                ReusedExchange [i_item_sk] #7
+                                                          InputAdapter
+                                                            ReusedExchange [p_promo_sk] #8
+                                                      InputAdapter
+                                                        ReusedExchange [d_date_sk] #5
+                                                  InputAdapter
+                                                    BroadcastExchange #13
+                                                      WholeStageCodegen (18)
+                                                        Filter [cp_catalog_page_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id]
+                                  WholeStageCodegen (30)
+                                    HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(CheckOverflow((promote_precision(cast(ws_net_profit as decimal(13,2))) - promote_precision(cast(coalesce(cast(wr_net_loss as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true)),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty]
+                                      InputAdapter
+                                        Exchange [web_site_id] #14
+                                          WholeStageCodegen (29)
+                                            HashAggregate [web_site_id,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty]
+                                              Project [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id]
+                                                BroadcastHashJoin [ws_web_site_sk,web_site_sk]
+                                                  Project [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss]
+                                                    BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                      Project [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss]
+                                                        BroadcastHashJoin [ws_promo_sk,p_promo_sk]
+                                                          Project [ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss]
+                                                            BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                                              Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss]
+                                                                SortMergeJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number]
+                                                                  InputAdapter
+                                                                    WholeStageCodegen (22)
+                                                                      Sort [ws_item_sk,ws_order_number]
+                                                                        InputAdapter
+                                                                          Exchange [ws_item_sk,ws_order_number] #15
+                                                                            WholeStageCodegen (21)
+                                                                              Filter [ws_web_site_sk,ws_item_sk,ws_promo_sk]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk]
+                                                                                      ReusedSubquery [d_date_sk] #1
+                                                                  InputAdapter
+                                                                    WholeStageCodegen (24)
+                                                                      Sort [wr_item_sk,wr_order_number]
+                                                                        InputAdapter
+                                                                          Exchange [wr_item_sk,wr_order_number] #16
+                                                                            WholeStageCodegen (23)
+                                                                              Project [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss]
+                                                                                Filter [wr_item_sk,wr_order_number]
+                                                                                  ColumnarToRow
+                                                                                    InputAdapter
+                                                                                      Scan parquet default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk]
+                                                              InputAdapter
+                                                                ReusedExchange [i_item_sk] #7
+                                                          InputAdapter
+                                                            ReusedExchange [p_promo_sk] #8
+                                                      InputAdapter
+                                                        ReusedExchange [d_date_sk] #5
+                                                  InputAdapter
+                                                    BroadcastExchange #17
+                                                      WholeStageCodegen (28)
+                                                        Filter [web_site_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.web_site [web_site_sk,web_site_id]
+                  WholeStageCodegen (65)
+                    HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                      InputAdapter
+                        Exchange [channel] #18
+                          WholeStageCodegen (64)
+                            HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                              HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                                InputAdapter
+                                  ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #19
+                  WholeStageCodegen (98)
+                    HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                      InputAdapter
+                        Exchange #20
+                          WholeStageCodegen (97)
+                            HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                              HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                                InputAdapter
+                                  ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #19
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/explain.txt
new file mode 100644
index 0000000000000..845d05e05b9bf
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/explain.txt
@@ -0,0 +1,715 @@
+== Physical Plan ==
+TakeOrderedAndProject (124)
++- * HashAggregate (123)
+   +- Exchange (122)
+      +- * HashAggregate (121)
+         +- Union (120)
+            :- * HashAggregate (109)
+            :  +- Exchange (108)
+            :     +- * HashAggregate (107)
+            :        +- Union (106)
+            :           :- * HashAggregate (43)
+            :           :  +- Exchange (42)
+            :           :     +- * HashAggregate (41)
+            :           :        +- * Project (40)
+            :           :           +- * BroadcastHashJoin Inner BuildRight (39)
+            :           :              :- * Project (33)
+            :           :              :  +- * BroadcastHashJoin Inner BuildRight (32)
+            :           :              :     :- * Project (26)
+            :           :              :     :  +- * BroadcastHashJoin Inner BuildRight (25)
+            :           :              :     :     :- * Project (20)
+            :           :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (19)
+            :           :              :     :     :     :- * Project (13)
+            :           :              :     :     :     :  +- * SortMergeJoin LeftOuter (12)
+            :           :              :     :     :     :     :- * Sort (5)
+            :           :              :     :     :     :     :  +- Exchange (4)
+            :           :              :     :     :     :     :     +- * Filter (3)
+            :           :              :     :     :     :     :        +- * ColumnarToRow (2)
+            :           :              :     :     :     :     :           +- Scan parquet default.store_sales (1)
+            :           :              :     :     :     :     +- * Sort (11)
+            :           :              :     :     :     :        +- Exchange (10)
+            :           :              :     :     :     :           +- * Project (9)
+            :           :              :     :     :     :              +- * Filter (8)
+            :           :              :     :     :     :                 +- * ColumnarToRow (7)
+            :           :              :     :     :     :                    +- Scan parquet default.store_returns (6)
+            :           :              :     :     :     +- BroadcastExchange (18)
+            :           :              :     :     :        +- * Project (17)
+            :           :              :     :     :           +- * Filter (16)
+            :           :              :     :     :              +- * ColumnarToRow (15)
+            :           :              :     :     :                 +- Scan parquet default.date_dim (14)
+            :           :              :     :     +- BroadcastExchange (24)
+            :           :              :     :        +- * Filter (23)
+            :           :              :     :           +- * ColumnarToRow (22)
+            :           :              :     :              +- Scan parquet default.store (21)
+            :           :              :     +- BroadcastExchange (31)
+            :           :              :        +- * Project (30)
+            :           :              :           +- * Filter (29)
+            :           :              :              +- * ColumnarToRow (28)
+            :           :              :                 +- Scan parquet default.item (27)
+            :           :              +- BroadcastExchange (38)
+            :           :                 +- * Project (37)
+            :           :                    +- * Filter (36)
+            :           :                       +- * ColumnarToRow (35)
+            :           :                          +- Scan parquet default.promotion (34)
+            :           :- * HashAggregate (74)
+            :           :  +- Exchange (73)
+            :           :     +- * HashAggregate (72)
+            :           :        +- * Project (71)
+            :           :           +- * BroadcastHashJoin Inner BuildRight (70)
+            :           :              :- * Project (68)
+            :           :              :  +- * BroadcastHashJoin Inner BuildRight (67)
+            :           :              :     :- * Project (65)
+            :           :              :     :  +- * BroadcastHashJoin Inner BuildRight (64)
+            :           :              :     :     :- * Project (59)
+            :           :              :     :     :  +- * BroadcastHashJoin Inner BuildRight (58)
+            :           :              :     :     :     :- * Project (56)
+            :           :              :     :     :     :  +- * SortMergeJoin LeftOuter (55)
+            :           :              :     :     :     :     :- * Sort (48)
+            :           :              :     :     :     :     :  +- Exchange (47)
+            :           :              :     :     :     :     :     +- * Filter (46)
+            :           :              :     :     :     :     :        +- * ColumnarToRow (45)
+            :           :              :     :     :     :     :           +- Scan parquet default.catalog_sales (44)
+            :           :              :     :     :     :     +- * Sort (54)
+            :           :              :     :     :     :        +- Exchange (53)
+            :           :              :     :     :     :           +- * Project (52)
+            :           :              :     :     :     :              +- * Filter (51)
+            :           :              :     :     :     :                 +- * ColumnarToRow (50)
+            :           :              :     :     :     :                    +- Scan parquet default.catalog_returns (49)
+            :           :              :     :     :     +- ReusedExchange (57)
+            :           :              :     :     +- BroadcastExchange (63)
+            :           :              :     :        +- * Filter (62)
+            :           :              :     :           +- * ColumnarToRow (61)
+            :           :              :     :              +- Scan parquet default.catalog_page (60)
+            :           :              :     +- ReusedExchange (66)
+            :           :              +- ReusedExchange (69)
+            :           +- * HashAggregate (105)
+            :              +- Exchange (104)
+            :                 +- * HashAggregate (103)
+            :                    +- * Project (102)
+            :                       +- * BroadcastHashJoin Inner BuildRight (101)
+            :                          :- * Project (99)
+            :                          :  +- * BroadcastHashJoin Inner BuildRight (98)
+            :                          :     :- * Project (96)
+            :                          :     :  +- * BroadcastHashJoin Inner BuildRight (95)
+            :                          :     :     :- * Project (90)
+            :                          :     :     :  +- * BroadcastHashJoin Inner BuildRight (89)
+            :                          :     :     :     :- * Project (87)
+            :                          :     :     :     :  +- * SortMergeJoin LeftOuter (86)
+            :                          :     :     :     :     :- * Sort (79)
+            :                          :     :     :     :     :  +- Exchange (78)
+            :                          :     :     :     :     :     +- * Filter (77)
+            :                          :     :     :     :     :        +- * ColumnarToRow (76)
+            :                          :     :     :     :     :           +- Scan parquet default.web_sales (75)
+            :                          :     :     :     :     +- * Sort (85)
+            :                          :     :     :     :        +- Exchange (84)
+            :                          :     :     :     :           +- * Project (83)
+            :                          :     :     :     :              +- * Filter (82)
+            :                          :     :     :     :                 +- * ColumnarToRow (81)
+            :                          :     :     :     :                    +- Scan parquet default.web_returns (80)
+            :                          :     :     :     +- ReusedExchange (88)
+            :                          :     :     +- BroadcastExchange (94)
+            :                          :     :        +- * Filter (93)
+            :                          :     :           +- * ColumnarToRow (92)
+            :                          :     :              +- Scan parquet default.web_site (91)
+            :                          :     +- ReusedExchange (97)
+            :                          +- ReusedExchange (100)
+            :- * HashAggregate (114)
+            :  +- Exchange (113)
+            :     +- * HashAggregate (112)
+            :        +- * HashAggregate (111)
+            :           +- ReusedExchange (110)
+            +- * HashAggregate (119)
+               +- Exchange (118)
+                  +- * HashAggregate (117)
+                     +- * HashAggregate (116)
+                        +- ReusedExchange (115)
+
+
+(1) Scan parquet default.store_sales
+Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_store_sk:int,ss_promo_sk:int,ss_ticket_number:int,ss_ext_sales_price:decimal(7,2),ss_net_profit:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+
+(3) Filter [codegen id : 1]
+Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3))
+
+(4) Exchange
+Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, [id=#9]
+
+(5) Sort [codegen id : 2]
+Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7]
+Arguments: [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.store_returns
+Output [5]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store_returns]
+PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)]
+ReadSchema: struct<sr_item_sk:int,sr_ticket_number:int,sr_return_amt:decimal(7,2),sr_net_loss:decimal(7,2)>
+
+(7) ColumnarToRow [codegen id : 3]
+Input [5]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14]
+
+(8) Filter [codegen id : 3]
+Input [5]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14]
+Condition : (isnotnull(sr_item_sk#10) AND isnotnull(sr_ticket_number#11))
+
+(9) Project [codegen id : 3]
+Output [4]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13]
+Input [5]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14]
+
+(10) Exchange
+Input [4]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13]
+Arguments: hashpartitioning(sr_item_sk#10, sr_ticket_number#11, 5), ENSURE_REQUIREMENTS, [id=#15]
+
+(11) Sort [codegen id : 4]
+Input [4]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13]
+Arguments: [sr_item_sk#10 ASC NULLS FIRST, sr_ticket_number#11 ASC NULLS FIRST], false, 0
+
+(12) SortMergeJoin [codegen id : 9]
+Left keys [2]: [ss_item_sk#1, ss_ticket_number#4]
+Right keys [2]: [sr_item_sk#10, sr_ticket_number#11]
+Join condition: None
+
+(13) Project [codegen id : 9]
+Output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#12, sr_net_loss#13]
+Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13]
+
+(14) Scan parquet default.date_dim
+Output [2]: [d_date_sk#16, d_date#17]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(15) ColumnarToRow [codegen id : 5]
+Input [2]: [d_date_sk#16, d_date#17]
+
+(16) Filter [codegen id : 5]
+Input [2]: [d_date_sk#16, d_date#17]
+Condition : (((isnotnull(d_date#17) AND (d_date#17 >= 1998-08-04)) AND (d_date#17 <= 1998-09-03)) AND isnotnull(d_date_sk#16))
+
+(17) Project [codegen id : 5]
+Output [1]: [d_date_sk#16]
+Input [2]: [d_date_sk#16, d_date#17]
+
+(18) BroadcastExchange
+Input [1]: [d_date_sk#16]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#18]
+
+(19) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_sold_date_sk#7]
+Right keys [1]: [d_date_sk#16]
+Join condition: None
+
+(20) Project [codegen id : 9]
+Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#12, sr_net_loss#13]
+Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#12, sr_net_loss#13, d_date_sk#16]
+
+(21) Scan parquet default.store
+Output [2]: [s_store_sk#19, s_store_id#20]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/store]
+PushedFilters: [IsNotNull(s_store_sk)]
+ReadSchema: struct<s_store_sk:int,s_store_id:string>
+
+(22) ColumnarToRow [codegen id : 6]
+Input [2]: [s_store_sk#19, s_store_id#20]
+
+(23) Filter [codegen id : 6]
+Input [2]: [s_store_sk#19, s_store_id#20]
+Condition : isnotnull(s_store_sk#19)
+
+(24) BroadcastExchange
+Input [2]: [s_store_sk#19, s_store_id#20]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#21]
+
+(25) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_store_sk#2]
+Right keys [1]: [s_store_sk#19]
+Join condition: None
+
+(26) Project [codegen id : 9]
+Output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#12, sr_net_loss#13, s_store_id#20]
+Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#12, sr_net_loss#13, s_store_sk#19, s_store_id#20]
+
+(27) Scan parquet default.item
+Output [2]: [i_item_sk#22, i_current_price#23]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_current_price:decimal(7,2)>
+
+(28) ColumnarToRow [codegen id : 7]
+Input [2]: [i_item_sk#22, i_current_price#23]
+
+(29) Filter [codegen id : 7]
+Input [2]: [i_item_sk#22, i_current_price#23]
+Condition : ((isnotnull(i_current_price#23) AND (i_current_price#23 > 50.00)) AND isnotnull(i_item_sk#22))
+
+(30) Project [codegen id : 7]
+Output [1]: [i_item_sk#22]
+Input [2]: [i_item_sk#22, i_current_price#23]
+
+(31) BroadcastExchange
+Input [1]: [i_item_sk#22]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#24]
+
+(32) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#22]
+Join condition: None
+
+(33) Project [codegen id : 9]
+Output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#12, sr_net_loss#13, s_store_id#20]
+Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#12, sr_net_loss#13, s_store_id#20, i_item_sk#22]
+
+(34) Scan parquet default.promotion
+Output [2]: [p_promo_sk#25, p_channel_tv#26]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/promotion]
+PushedFilters: [IsNotNull(p_channel_tv), EqualTo(p_channel_tv,N), IsNotNull(p_promo_sk)]
+ReadSchema: struct<p_promo_sk:int,p_channel_tv:string>
+
+(35) ColumnarToRow [codegen id : 8]
+Input [2]: [p_promo_sk#25, p_channel_tv#26]
+
+(36) Filter [codegen id : 8]
+Input [2]: [p_promo_sk#25, p_channel_tv#26]
+Condition : ((isnotnull(p_channel_tv#26) AND (p_channel_tv#26 = N)) AND isnotnull(p_promo_sk#25))
+
+(37) Project [codegen id : 8]
+Output [1]: [p_promo_sk#25]
+Input [2]: [p_promo_sk#25, p_channel_tv#26]
+
+(38) BroadcastExchange
+Input [1]: [p_promo_sk#25]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#27]
+
+(39) BroadcastHashJoin [codegen id : 9]
+Left keys [1]: [ss_promo_sk#3]
+Right keys [1]: [p_promo_sk#25]
+Join condition: None
+
+(40) Project [codegen id : 9]
+Output [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#12, sr_net_loss#13, s_store_id#20]
+Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#12, sr_net_loss#13, s_store_id#20, p_promo_sk#25]
+
+(41) HashAggregate [codegen id : 9]
+Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#12, sr_net_loss#13, s_store_id#20]
+Keys [1]: [s_store_id#20]
+Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#12 as decimal(12,2)), 0.00)), partial_sum(CheckOverflow((promote_precision(cast(ss_net_profit#6 as decimal(13,2))) - promote_precision(cast(coalesce(cast(sr_net_loss#13 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))]
+Aggregate Attributes [5]: [sum#28, sum#29, isEmpty#30, sum#31, isEmpty#32]
+Results [6]: [s_store_id#20, sum#33, sum#34, isEmpty#35, sum#36, isEmpty#37]
+
+(42) Exchange
+Input [6]: [s_store_id#20, sum#33, sum#34, isEmpty#35, sum#36, isEmpty#37]
+Arguments: hashpartitioning(s_store_id#20, 5), ENSURE_REQUIREMENTS, [id=#38]
+
+(43) HashAggregate [codegen id : 10]
+Input [6]: [s_store_id#20, sum#33, sum#34, isEmpty#35, sum#36, isEmpty#37]
+Keys [1]: [s_store_id#20]
+Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#12 as decimal(12,2)), 0.00)), sum(CheckOverflow((promote_precision(cast(ss_net_profit#6 as decimal(13,2))) - promote_precision(cast(coalesce(cast(sr_net_loss#13 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))]
+Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#39, sum(coalesce(cast(sr_return_amt#12 as decimal(12,2)), 0.00))#40, sum(CheckOverflow((promote_precision(cast(ss_net_profit#6 as decimal(13,2))) - promote_precision(cast(coalesce(cast(sr_net_loss#13 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))#41]
+Results [5]: [store channel AS channel#42, concat(store, s_store_id#20) AS id#43, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#39,17,2) AS sales#44, sum(coalesce(cast(sr_return_amt#12 as decimal(12,2)), 0.00))#40 AS returns#45, sum(CheckOverflow((promote_precision(cast(ss_net_profit#6 as decimal(13,2))) - promote_precision(cast(coalesce(cast(sr_net_loss#13 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))#41 AS profit#46]
+
+(44) Scan parquet default.catalog_sales
+Output [7]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_order_number#50, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(cs_sold_date_sk#53), dynamicpruningexpression(cs_sold_date_sk#53 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)]
+ReadSchema: struct<cs_catalog_page_sk:int,cs_item_sk:int,cs_promo_sk:int,cs_order_number:int,cs_ext_sales_price:decimal(7,2),cs_net_profit:decimal(7,2)>
+
+(45) ColumnarToRow [codegen id : 11]
+Input [7]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_order_number#50, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53]
+
+(46) Filter [codegen id : 11]
+Input [7]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_order_number#50, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53]
+Condition : ((isnotnull(cs_catalog_page_sk#47) AND isnotnull(cs_item_sk#48)) AND isnotnull(cs_promo_sk#49))
+
+(47) Exchange
+Input [7]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_order_number#50, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53]
+Arguments: hashpartitioning(cs_item_sk#48, cs_order_number#50, 5), ENSURE_REQUIREMENTS, [id=#54]
+
+(48) Sort [codegen id : 12]
+Input [7]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_order_number#50, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53]
+Arguments: [cs_item_sk#48 ASC NULLS FIRST, cs_order_number#50 ASC NULLS FIRST], false, 0
+
+(49) Scan parquet default.catalog_returns
+Output [5]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_returns]
+PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)]
+ReadSchema: struct<cr_item_sk:int,cr_order_number:int,cr_return_amount:decimal(7,2),cr_net_loss:decimal(7,2)>
+
+(50) ColumnarToRow [codegen id : 13]
+Input [5]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+
+(51) Filter [codegen id : 13]
+Input [5]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+Condition : (isnotnull(cr_item_sk#55) AND isnotnull(cr_order_number#56))
+
+(52) Project [codegen id : 13]
+Output [4]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58]
+Input [5]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58, cr_returned_date_sk#59]
+
+(53) Exchange
+Input [4]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58]
+Arguments: hashpartitioning(cr_item_sk#55, cr_order_number#56, 5), ENSURE_REQUIREMENTS, [id=#60]
+
+(54) Sort [codegen id : 14]
+Input [4]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58]
+Arguments: [cr_item_sk#55 ASC NULLS FIRST, cr_order_number#56 ASC NULLS FIRST], false, 0
+
+(55) SortMergeJoin [codegen id : 19]
+Left keys [2]: [cs_item_sk#48, cs_order_number#50]
+Right keys [2]: [cr_item_sk#55, cr_order_number#56]
+Join condition: None
+
+(56) Project [codegen id : 19]
+Output [8]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53, cr_return_amount#57, cr_net_loss#58]
+Input [11]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_order_number#50, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53, cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58]
+
+(57) ReusedExchange [Reuses operator id: 18]
+Output [1]: [d_date_sk#61]
+
+(58) BroadcastHashJoin [codegen id : 19]
+Left keys [1]: [cs_sold_date_sk#53]
+Right keys [1]: [d_date_sk#61]
+Join condition: None
+
+(59) Project [codegen id : 19]
+Output [7]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_ext_sales_price#51, cs_net_profit#52, cr_return_amount#57, cr_net_loss#58]
+Input [9]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_ext_sales_price#51, cs_net_profit#52, cs_sold_date_sk#53, cr_return_amount#57, cr_net_loss#58, d_date_sk#61]
+
+(60) Scan parquet default.catalog_page
+Output [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/catalog_page]
+PushedFilters: [IsNotNull(cp_catalog_page_sk)]
+ReadSchema: struct<cp_catalog_page_sk:int,cp_catalog_page_id:string>
+
+(61) ColumnarToRow [codegen id : 16]
+Input [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63]
+
+(62) Filter [codegen id : 16]
+Input [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63]
+Condition : isnotnull(cp_catalog_page_sk#62)
+
+(63) BroadcastExchange
+Input [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#64]
+
+(64) BroadcastHashJoin [codegen id : 19]
+Left keys [1]: [cs_catalog_page_sk#47]
+Right keys [1]: [cp_catalog_page_sk#62]
+Join condition: None
+
+(65) Project [codegen id : 19]
+Output [7]: [cs_item_sk#48, cs_promo_sk#49, cs_ext_sales_price#51, cs_net_profit#52, cr_return_amount#57, cr_net_loss#58, cp_catalog_page_id#63]
+Input [9]: [cs_catalog_page_sk#47, cs_item_sk#48, cs_promo_sk#49, cs_ext_sales_price#51, cs_net_profit#52, cr_return_amount#57, cr_net_loss#58, cp_catalog_page_sk#62, cp_catalog_page_id#63]
+
+(66) ReusedExchange [Reuses operator id: 31]
+Output [1]: [i_item_sk#65]
+
+(67) BroadcastHashJoin [codegen id : 19]
+Left keys [1]: [cs_item_sk#48]
+Right keys [1]: [i_item_sk#65]
+Join condition: None
+
+(68) Project [codegen id : 19]
+Output [6]: [cs_promo_sk#49, cs_ext_sales_price#51, cs_net_profit#52, cr_return_amount#57, cr_net_loss#58, cp_catalog_page_id#63]
+Input [8]: [cs_item_sk#48, cs_promo_sk#49, cs_ext_sales_price#51, cs_net_profit#52, cr_return_amount#57, cr_net_loss#58, cp_catalog_page_id#63, i_item_sk#65]
+
+(69) ReusedExchange [Reuses operator id: 38]
+Output [1]: [p_promo_sk#66]
+
+(70) BroadcastHashJoin [codegen id : 19]
+Left keys [1]: [cs_promo_sk#49]
+Right keys [1]: [p_promo_sk#66]
+Join condition: None
+
+(71) Project [codegen id : 19]
+Output [5]: [cs_ext_sales_price#51, cs_net_profit#52, cr_return_amount#57, cr_net_loss#58, cp_catalog_page_id#63]
+Input [7]: [cs_promo_sk#49, cs_ext_sales_price#51, cs_net_profit#52, cr_return_amount#57, cr_net_loss#58, cp_catalog_page_id#63, p_promo_sk#66]
+
+(72) HashAggregate [codegen id : 19]
+Input [5]: [cs_ext_sales_price#51, cs_net_profit#52, cr_return_amount#57, cr_net_loss#58, cp_catalog_page_id#63]
+Keys [1]: [cp_catalog_page_id#63]
+Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#51)), partial_sum(coalesce(cast(cr_return_amount#57 as decimal(12,2)), 0.00)), partial_sum(CheckOverflow((promote_precision(cast(cs_net_profit#52 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_net_loss#58 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))]
+Aggregate Attributes [5]: [sum#67, sum#68, isEmpty#69, sum#70, isEmpty#71]
+Results [6]: [cp_catalog_page_id#63, sum#72, sum#73, isEmpty#74, sum#75, isEmpty#76]
+
+(73) Exchange
+Input [6]: [cp_catalog_page_id#63, sum#72, sum#73, isEmpty#74, sum#75, isEmpty#76]
+Arguments: hashpartitioning(cp_catalog_page_id#63, 5), ENSURE_REQUIREMENTS, [id=#77]
+
+(74) HashAggregate [codegen id : 20]
+Input [6]: [cp_catalog_page_id#63, sum#72, sum#73, isEmpty#74, sum#75, isEmpty#76]
+Keys [1]: [cp_catalog_page_id#63]
+Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#51)), sum(coalesce(cast(cr_return_amount#57 as decimal(12,2)), 0.00)), sum(CheckOverflow((promote_precision(cast(cs_net_profit#52 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_net_loss#58 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))]
+Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#51))#78, sum(coalesce(cast(cr_return_amount#57 as decimal(12,2)), 0.00))#79, sum(CheckOverflow((promote_precision(cast(cs_net_profit#52 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_net_loss#58 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))#80]
+Results [5]: [catalog channel AS channel#81, concat(catalog_page, cp_catalog_page_id#63) AS id#82, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#51))#78,17,2) AS sales#83, sum(coalesce(cast(cr_return_amount#57 as decimal(12,2)), 0.00))#79 AS returns#84, sum(CheckOverflow((promote_precision(cast(cs_net_profit#52 as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_net_loss#58 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))#80 AS profit#85]
+
+(75) Scan parquet default.web_sales
+Output [7]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_order_number#89, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#92), dynamicpruningexpression(ws_sold_date_sk#92 IN dynamicpruning#8)]
+PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_web_site_sk:int,ws_promo_sk:int,ws_order_number:int,ws_ext_sales_price:decimal(7,2),ws_net_profit:decimal(7,2)>
+
+(76) ColumnarToRow [codegen id : 21]
+Input [7]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_order_number#89, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92]
+
+(77) Filter [codegen id : 21]
+Input [7]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_order_number#89, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92]
+Condition : ((isnotnull(ws_web_site_sk#87) AND isnotnull(ws_item_sk#86)) AND isnotnull(ws_promo_sk#88))
+
+(78) Exchange
+Input [7]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_order_number#89, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92]
+Arguments: hashpartitioning(ws_item_sk#86, ws_order_number#89, 5), ENSURE_REQUIREMENTS, [id=#93]
+
+(79) Sort [codegen id : 22]
+Input [7]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_order_number#89, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92]
+Arguments: [ws_item_sk#86 ASC NULLS FIRST, ws_order_number#89 ASC NULLS FIRST], false, 0
+
+(80) Scan parquet default.web_returns
+Output [5]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_returns]
+PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)]
+ReadSchema: struct<wr_item_sk:int,wr_order_number:int,wr_return_amt:decimal(7,2),wr_net_loss:decimal(7,2)>
+
+(81) ColumnarToRow [codegen id : 23]
+Input [5]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98]
+
+(82) Filter [codegen id : 23]
+Input [5]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98]
+Condition : (isnotnull(wr_item_sk#94) AND isnotnull(wr_order_number#95))
+
+(83) Project [codegen id : 23]
+Output [4]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97]
+Input [5]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98]
+
+(84) Exchange
+Input [4]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97]
+Arguments: hashpartitioning(wr_item_sk#94, wr_order_number#95, 5), ENSURE_REQUIREMENTS, [id=#99]
+
+(85) Sort [codegen id : 24]
+Input [4]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97]
+Arguments: [wr_item_sk#94 ASC NULLS FIRST, wr_order_number#95 ASC NULLS FIRST], false, 0
+
+(86) SortMergeJoin [codegen id : 29]
+Left keys [2]: [ws_item_sk#86, ws_order_number#89]
+Right keys [2]: [wr_item_sk#94, wr_order_number#95]
+Join condition: None
+
+(87) Project [codegen id : 29]
+Output [8]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92, wr_return_amt#96, wr_net_loss#97]
+Input [11]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_order_number#89, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92, wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97]
+
+(88) ReusedExchange [Reuses operator id: 18]
+Output [1]: [d_date_sk#100]
+
+(89) BroadcastHashJoin [codegen id : 29]
+Left keys [1]: [ws_sold_date_sk#92]
+Right keys [1]: [d_date_sk#100]
+Join condition: None
+
+(90) Project [codegen id : 29]
+Output [7]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_ext_sales_price#90, ws_net_profit#91, wr_return_amt#96, wr_net_loss#97]
+Input [9]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_ext_sales_price#90, ws_net_profit#91, ws_sold_date_sk#92, wr_return_amt#96, wr_net_loss#97, d_date_sk#100]
+
+(91) Scan parquet default.web_site
+Output [2]: [web_site_sk#101, web_site_id#102]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/web_site]
+PushedFilters: [IsNotNull(web_site_sk)]
+ReadSchema: struct<web_site_sk:int,web_site_id:string>
+
+(92) ColumnarToRow [codegen id : 26]
+Input [2]: [web_site_sk#101, web_site_id#102]
+
+(93) Filter [codegen id : 26]
+Input [2]: [web_site_sk#101, web_site_id#102]
+Condition : isnotnull(web_site_sk#101)
+
+(94) BroadcastExchange
+Input [2]: [web_site_sk#101, web_site_id#102]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#103]
+
+(95) BroadcastHashJoin [codegen id : 29]
+Left keys [1]: [ws_web_site_sk#87]
+Right keys [1]: [web_site_sk#101]
+Join condition: None
+
+(96) Project [codegen id : 29]
+Output [7]: [ws_item_sk#86, ws_promo_sk#88, ws_ext_sales_price#90, ws_net_profit#91, wr_return_amt#96, wr_net_loss#97, web_site_id#102]
+Input [9]: [ws_item_sk#86, ws_web_site_sk#87, ws_promo_sk#88, ws_ext_sales_price#90, ws_net_profit#91, wr_return_amt#96, wr_net_loss#97, web_site_sk#101, web_site_id#102]
+
+(97) ReusedExchange [Reuses operator id: 31]
+Output [1]: [i_item_sk#104]
+
+(98) BroadcastHashJoin [codegen id : 29]
+Left keys [1]: [ws_item_sk#86]
+Right keys [1]: [i_item_sk#104]
+Join condition: None
+
+(99) Project [codegen id : 29]
+Output [6]: [ws_promo_sk#88, ws_ext_sales_price#90, ws_net_profit#91, wr_return_amt#96, wr_net_loss#97, web_site_id#102]
+Input [8]: [ws_item_sk#86, ws_promo_sk#88, ws_ext_sales_price#90, ws_net_profit#91, wr_return_amt#96, wr_net_loss#97, web_site_id#102, i_item_sk#104]
+
+(100) ReusedExchange [Reuses operator id: 38]
+Output [1]: [p_promo_sk#105]
+
+(101) BroadcastHashJoin [codegen id : 29]
+Left keys [1]: [ws_promo_sk#88]
+Right keys [1]: [p_promo_sk#105]
+Join condition: None
+
+(102) Project [codegen id : 29]
+Output [5]: [ws_ext_sales_price#90, ws_net_profit#91, wr_return_amt#96, wr_net_loss#97, web_site_id#102]
+Input [7]: [ws_promo_sk#88, ws_ext_sales_price#90, ws_net_profit#91, wr_return_amt#96, wr_net_loss#97, web_site_id#102, p_promo_sk#105]
+
+(103) HashAggregate [codegen id : 29]
+Input [5]: [ws_ext_sales_price#90, ws_net_profit#91, wr_return_amt#96, wr_net_loss#97, web_site_id#102]
+Keys [1]: [web_site_id#102]
+Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#90)), partial_sum(coalesce(cast(wr_return_amt#96 as decimal(12,2)), 0.00)), partial_sum(CheckOverflow((promote_precision(cast(ws_net_profit#91 as decimal(13,2))) - promote_precision(cast(coalesce(cast(wr_net_loss#97 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))]
+Aggregate Attributes [5]: [sum#106, sum#107, isEmpty#108, sum#109, isEmpty#110]
+Results [6]: [web_site_id#102, sum#111, sum#112, isEmpty#113, sum#114, isEmpty#115]
+
+(104) Exchange
+Input [6]: [web_site_id#102, sum#111, sum#112, isEmpty#113, sum#114, isEmpty#115]
+Arguments: hashpartitioning(web_site_id#102, 5), ENSURE_REQUIREMENTS, [id=#116]
+
+(105) HashAggregate [codegen id : 30]
+Input [6]: [web_site_id#102, sum#111, sum#112, isEmpty#113, sum#114, isEmpty#115]
+Keys [1]: [web_site_id#102]
+Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#90)), sum(coalesce(cast(wr_return_amt#96 as decimal(12,2)), 0.00)), sum(CheckOverflow((promote_precision(cast(ws_net_profit#91 as decimal(13,2))) - promote_precision(cast(coalesce(cast(wr_net_loss#97 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))]
+Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#90))#117, sum(coalesce(cast(wr_return_amt#96 as decimal(12,2)), 0.00))#118, sum(CheckOverflow((promote_precision(cast(ws_net_profit#91 as decimal(13,2))) - promote_precision(cast(coalesce(cast(wr_net_loss#97 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))#119]
+Results [5]: [web channel AS channel#120, concat(web_site, web_site_id#102) AS id#121, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#90))#117,17,2) AS sales#122, sum(coalesce(cast(wr_return_amt#96 as decimal(12,2)), 0.00))#118 AS returns#123, sum(CheckOverflow((promote_precision(cast(ws_net_profit#91 as decimal(13,2))) - promote_precision(cast(coalesce(cast(wr_net_loss#97 as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true))#119 AS profit#124]
+
+(106) Union
+
+(107) HashAggregate [codegen id : 31]
+Input [5]: [channel#42, id#43, sales#44, returns#45, profit#46]
+Keys [2]: [channel#42, id#43]
+Functions [3]: [partial_sum(sales#44), partial_sum(returns#45), partial_sum(profit#46)]
+Aggregate Attributes [6]: [sum#125, isEmpty#126, sum#127, isEmpty#128, sum#129, isEmpty#130]
+Results [8]: [channel#42, id#43, sum#131, isEmpty#132, sum#133, isEmpty#134, sum#135, isEmpty#136]
+
+(108) Exchange
+Input [8]: [channel#42, id#43, sum#131, isEmpty#132, sum#133, isEmpty#134, sum#135, isEmpty#136]
+Arguments: hashpartitioning(channel#42, id#43, 5), ENSURE_REQUIREMENTS, [id=#137]
+
+(109) HashAggregate [codegen id : 32]
+Input [8]: [channel#42, id#43, sum#131, isEmpty#132, sum#133, isEmpty#134, sum#135, isEmpty#136]
+Keys [2]: [channel#42, id#43]
+Functions [3]: [sum(sales#44), sum(returns#45), sum(profit#46)]
+Aggregate Attributes [3]: [sum(sales#44)#138, sum(returns#45)#139, sum(profit#46)#140]
+Results [5]: [channel#42, id#43, cast(sum(sales#44)#138 as decimal(37,2)) AS sales#141, cast(sum(returns#45)#139 as decimal(38,2)) AS returns#142, cast(sum(profit#46)#140 as decimal(38,2)) AS profit#143]
+
+(110) ReusedExchange [Reuses operator id: unknown]
+Output [8]: [channel#42, id#43, sum#144, isEmpty#145, sum#146, isEmpty#147, sum#148, isEmpty#149]
+
+(111) HashAggregate [codegen id : 64]
+Input [8]: [channel#42, id#43, sum#144, isEmpty#145, sum#146, isEmpty#147, sum#148, isEmpty#149]
+Keys [2]: [channel#42, id#43]
+Functions [3]: [sum(sales#44), sum(returns#45), sum(profit#46)]
+Aggregate Attributes [3]: [sum(sales#44)#150, sum(returns#45)#151, sum(profit#46)#152]
+Results [4]: [channel#42, sum(sales#44)#150 AS sales#153, sum(returns#45)#151 AS returns#154, sum(profit#46)#152 AS profit#155]
+
+(112) HashAggregate [codegen id : 64]
+Input [4]: [channel#42, sales#153, returns#154, profit#155]
+Keys [1]: [channel#42]
+Functions [3]: [partial_sum(sales#153), partial_sum(returns#154), partial_sum(profit#155)]
+Aggregate Attributes [6]: [sum#156, isEmpty#157, sum#158, isEmpty#159, sum#160, isEmpty#161]
+Results [7]: [channel#42, sum#162, isEmpty#163, sum#164, isEmpty#165, sum#166, isEmpty#167]
+
+(113) Exchange
+Input [7]: [channel#42, sum#162, isEmpty#163, sum#164, isEmpty#165, sum#166, isEmpty#167]
+Arguments: hashpartitioning(channel#42, 5), ENSURE_REQUIREMENTS, [id=#168]
+
+(114) HashAggregate [codegen id : 65]
+Input [7]: [channel#42, sum#162, isEmpty#163, sum#164, isEmpty#165, sum#166, isEmpty#167]
+Keys [1]: [channel#42]
+Functions [3]: [sum(sales#153), sum(returns#154), sum(profit#155)]
+Aggregate Attributes [3]: [sum(sales#153)#169, sum(returns#154)#170, sum(profit#155)#171]
+Results [5]: [channel#42, null AS id#172, sum(sales#153)#169 AS sales#173, sum(returns#154)#170 AS returns#174, sum(profit#155)#171 AS profit#175]
+
+(115) ReusedExchange [Reuses operator id: unknown]
+Output [8]: [channel#42, id#43, sum#176, isEmpty#177, sum#178, isEmpty#179, sum#180, isEmpty#181]
+
+(116) HashAggregate [codegen id : 97]
+Input [8]: [channel#42, id#43, sum#176, isEmpty#177, sum#178, isEmpty#179, sum#180, isEmpty#181]
+Keys [2]: [channel#42, id#43]
+Functions [3]: [sum(sales#44), sum(returns#45), sum(profit#46)]
+Aggregate Attributes [3]: [sum(sales#44)#182, sum(returns#45)#183, sum(profit#46)#184]
+Results [3]: [sum(sales#44)#182 AS sales#153, sum(returns#45)#183 AS returns#154, sum(profit#46)#184 AS profit#155]
+
+(117) HashAggregate [codegen id : 97]
+Input [3]: [sales#153, returns#154, profit#155]
+Keys: []
+Functions [3]: [partial_sum(sales#153), partial_sum(returns#154), partial_sum(profit#155)]
+Aggregate Attributes [6]: [sum#185, isEmpty#186, sum#187, isEmpty#188, sum#189, isEmpty#190]
+Results [6]: [sum#191, isEmpty#192, sum#193, isEmpty#194, sum#195, isEmpty#196]
+
+(118) Exchange
+Input [6]: [sum#191, isEmpty#192, sum#193, isEmpty#194, sum#195, isEmpty#196]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#197]
+
+(119) HashAggregate [codegen id : 98]
+Input [6]: [sum#191, isEmpty#192, sum#193, isEmpty#194, sum#195, isEmpty#196]
+Keys: []
+Functions [3]: [sum(sales#153), sum(returns#154), sum(profit#155)]
+Aggregate Attributes [3]: [sum(sales#153)#198, sum(returns#154)#199, sum(profit#155)#200]
+Results [5]: [null AS channel#201, null AS id#202, sum(sales#153)#198 AS sales#203, sum(returns#154)#199 AS returns#204, sum(profit#155)#200 AS profit#205]
+
+(120) Union
+
+(121) HashAggregate [codegen id : 99]
+Input [5]: [channel#42, id#43, sales#141, returns#142, profit#143]
+Keys [5]: [channel#42, id#43, sales#141, returns#142, profit#143]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [channel#42, id#43, sales#141, returns#142, profit#143]
+
+(122) Exchange
+Input [5]: [channel#42, id#43, sales#141, returns#142, profit#143]
+Arguments: hashpartitioning(channel#42, id#43, sales#141, returns#142, profit#143, 5), ENSURE_REQUIREMENTS, [id=#206]
+
+(123) HashAggregate [codegen id : 100]
+Input [5]: [channel#42, id#43, sales#141, returns#142, profit#143]
+Keys [5]: [channel#42, id#43, sales#141, returns#142, profit#143]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [channel#42, id#43, sales#141, returns#142, profit#143]
+
+(124) TakeOrderedAndProject
+Input [5]: [channel#42, id#43, sales#141, returns#142, profit#143]
+Arguments: 100, [channel#42 ASC NULLS FIRST, id#43 ASC NULLS FIRST], [channel#42, id#43, sales#141, returns#142, profit#143]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8
+ReusedExchange (125)
+
+
+(125) ReusedExchange [Reuses operator id: 18]
+Output [1]: [d_date_sk#16]
+
+Subquery:2 Hosting operator id = 44 Hosting Expression = cs_sold_date_sk#53 IN dynamicpruning#8
+
+Subquery:3 Hosting operator id = 75 Hosting Expression = ws_sold_date_sk#92 IN dynamicpruning#8
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/simplified.txt
new file mode 100644
index 0000000000000..30b609e21c467
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/simplified.txt
@@ -0,0 +1,207 @@
+TakeOrderedAndProject [channel,id,sales,returns,profit]
+  WholeStageCodegen (100)
+    HashAggregate [channel,id,sales,returns,profit]
+      InputAdapter
+        Exchange [channel,id,sales,returns,profit] #1
+          WholeStageCodegen (99)
+            HashAggregate [channel,id,sales,returns,profit]
+              InputAdapter
+                Union
+                  WholeStageCodegen (32)
+                    HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                      InputAdapter
+                        Exchange [channel,id] #2
+                          WholeStageCodegen (31)
+                            HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                              InputAdapter
+                                Union
+                                  WholeStageCodegen (10)
+                                    HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(CheckOverflow((promote_precision(cast(ss_net_profit as decimal(13,2))) - promote_precision(cast(coalesce(cast(sr_net_loss as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true)),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty]
+                                      InputAdapter
+                                        Exchange [s_store_id] #3
+                                          WholeStageCodegen (9)
+                                            HashAggregate [s_store_id,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty]
+                                              Project [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id]
+                                                BroadcastHashJoin [ss_promo_sk,p_promo_sk]
+                                                  Project [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id]
+                                                    BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                                      Project [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id]
+                                                        BroadcastHashJoin [ss_store_sk,s_store_sk]
+                                                          Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss]
+                                                            BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                                              Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss]
+                                                                SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number]
+                                                                  InputAdapter
+                                                                    WholeStageCodegen (2)
+                                                                      Sort [ss_item_sk,ss_ticket_number]
+                                                                        InputAdapter
+                                                                          Exchange [ss_item_sk,ss_ticket_number] #4
+                                                                            WholeStageCodegen (1)
+                                                                              Filter [ss_store_sk,ss_item_sk,ss_promo_sk]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk]
+                                                                                      SubqueryBroadcast [d_date_sk] #1
+                                                                                        ReusedExchange [d_date_sk] #5
+                                                                  InputAdapter
+                                                                    WholeStageCodegen (4)
+                                                                      Sort [sr_item_sk,sr_ticket_number]
+                                                                        InputAdapter
+                                                                          Exchange [sr_item_sk,sr_ticket_number] #6
+                                                                            WholeStageCodegen (3)
+                                                                              Project [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss]
+                                                                                Filter [sr_item_sk,sr_ticket_number]
+                                                                                  ColumnarToRow
+                                                                                    InputAdapter
+                                                                                      Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk]
+                                                              InputAdapter
+                                                                BroadcastExchange #5
+                                                                  WholeStageCodegen (5)
+                                                                    Project [d_date_sk]
+                                                                      Filter [d_date,d_date_sk]
+                                                                        ColumnarToRow
+                                                                          InputAdapter
+                                                                            Scan parquet default.date_dim [d_date_sk,d_date]
+                                                          InputAdapter
+                                                            BroadcastExchange #7
+                                                              WholeStageCodegen (6)
+                                                                Filter [s_store_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.store [s_store_sk,s_store_id]
+                                                      InputAdapter
+                                                        BroadcastExchange #8
+                                                          WholeStageCodegen (7)
+                                                            Project [i_item_sk]
+                                                              Filter [i_current_price,i_item_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.item [i_item_sk,i_current_price]
+                                                  InputAdapter
+                                                    BroadcastExchange #9
+                                                      WholeStageCodegen (8)
+                                                        Project [p_promo_sk]
+                                                          Filter [p_channel_tv,p_promo_sk]
+                                                            ColumnarToRow
+                                                              InputAdapter
+                                                                Scan parquet default.promotion [p_promo_sk,p_channel_tv]
+                                  WholeStageCodegen (20)
+                                    HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(CheckOverflow((promote_precision(cast(cs_net_profit as decimal(13,2))) - promote_precision(cast(coalesce(cast(cr_net_loss as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true)),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty]
+                                      InputAdapter
+                                        Exchange [cp_catalog_page_id] #10
+                                          WholeStageCodegen (19)
+                                            HashAggregate [cp_catalog_page_id,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty]
+                                              Project [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id]
+                                                BroadcastHashJoin [cs_promo_sk,p_promo_sk]
+                                                  Project [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id]
+                                                    BroadcastHashJoin [cs_item_sk,i_item_sk]
+                                                      Project [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id]
+                                                        BroadcastHashJoin [cs_catalog_page_sk,cp_catalog_page_sk]
+                                                          Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss]
+                                                            BroadcastHashJoin [cs_sold_date_sk,d_date_sk]
+                                                              Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss]
+                                                                SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number]
+                                                                  InputAdapter
+                                                                    WholeStageCodegen (12)
+                                                                      Sort [cs_item_sk,cs_order_number]
+                                                                        InputAdapter
+                                                                          Exchange [cs_item_sk,cs_order_number] #11
+                                                                            WholeStageCodegen (11)
+                                                                              Filter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk]
+                                                                                      ReusedSubquery [d_date_sk] #1
+                                                                  InputAdapter
+                                                                    WholeStageCodegen (14)
+                                                                      Sort [cr_item_sk,cr_order_number]
+                                                                        InputAdapter
+                                                                          Exchange [cr_item_sk,cr_order_number] #12
+                                                                            WholeStageCodegen (13)
+                                                                              Project [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss]
+                                                                                Filter [cr_item_sk,cr_order_number]
+                                                                                  ColumnarToRow
+                                                                                    InputAdapter
+                                                                                      Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk]
+                                                              InputAdapter
+                                                                ReusedExchange [d_date_sk] #5
+                                                          InputAdapter
+                                                            BroadcastExchange #13
+                                                              WholeStageCodegen (16)
+                                                                Filter [cp_catalog_page_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id]
+                                                      InputAdapter
+                                                        ReusedExchange [i_item_sk] #8
+                                                  InputAdapter
+                                                    ReusedExchange [p_promo_sk] #9
+                                  WholeStageCodegen (30)
+                                    HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(CheckOverflow((promote_precision(cast(ws_net_profit as decimal(13,2))) - promote_precision(cast(coalesce(cast(wr_net_loss as decimal(12,2)), 0.00) as decimal(13,2)))), DecimalType(13,2), true)),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty]
+                                      InputAdapter
+                                        Exchange [web_site_id] #14
+                                          WholeStageCodegen (29)
+                                            HashAggregate [web_site_id,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty]
+                                              Project [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id]
+                                                BroadcastHashJoin [ws_promo_sk,p_promo_sk]
+                                                  Project [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id]
+                                                    BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                                      Project [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id]
+                                                        BroadcastHashJoin [ws_web_site_sk,web_site_sk]
+                                                          Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss]
+                                                            BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                              Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss]
+                                                                SortMergeJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number]
+                                                                  InputAdapter
+                                                                    WholeStageCodegen (22)
+                                                                      Sort [ws_item_sk,ws_order_number]
+                                                                        InputAdapter
+                                                                          Exchange [ws_item_sk,ws_order_number] #15
+                                                                            WholeStageCodegen (21)
+                                                                              Filter [ws_web_site_sk,ws_item_sk,ws_promo_sk]
+                                                                                ColumnarToRow
+                                                                                  InputAdapter
+                                                                                    Scan parquet default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk]
+                                                                                      ReusedSubquery [d_date_sk] #1
+                                                                  InputAdapter
+                                                                    WholeStageCodegen (24)
+                                                                      Sort [wr_item_sk,wr_order_number]
+                                                                        InputAdapter
+                                                                          Exchange [wr_item_sk,wr_order_number] #16
+                                                                            WholeStageCodegen (23)
+                                                                              Project [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss]
+                                                                                Filter [wr_item_sk,wr_order_number]
+                                                                                  ColumnarToRow
+                                                                                    InputAdapter
+                                                                                      Scan parquet default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk]
+                                                              InputAdapter
+                                                                ReusedExchange [d_date_sk] #5
+                                                          InputAdapter
+                                                            BroadcastExchange #17
+                                                              WholeStageCodegen (26)
+                                                                Filter [web_site_sk]
+                                                                  ColumnarToRow
+                                                                    InputAdapter
+                                                                      Scan parquet default.web_site [web_site_sk,web_site_id]
+                                                      InputAdapter
+                                                        ReusedExchange [i_item_sk] #8
+                                                  InputAdapter
+                                                    ReusedExchange [p_promo_sk] #9
+                  WholeStageCodegen (65)
+                    HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                      InputAdapter
+                        Exchange [channel] #18
+                          WholeStageCodegen (64)
+                            HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                              HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                                InputAdapter
+                                  ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #19
+                  WholeStageCodegen (98)
+                    HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                      InputAdapter
+                        Exchange #20
+                          WholeStageCodegen (97)
+                            HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                              HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty]
+                                InputAdapter
+                                  ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #19
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.sf100/explain.txt
new file mode 100644
index 0000000000000..64f6270e81183
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.sf100/explain.txt
@@ -0,0 +1,238 @@
+== Physical Plan ==
+TakeOrderedAndProject (38)
++- * Project (37)
+   +- Window (36)
+      +- * Sort (35)
+         +- Exchange (34)
+            +- * HashAggregate (33)
+               +- Exchange (32)
+                  +- * HashAggregate (31)
+                     +- Union (30)
+                        :- * HashAggregate (19)
+                        :  +- Exchange (18)
+                        :     +- * HashAggregate (17)
+                        :        +- * Project (16)
+                        :           +- * BroadcastHashJoin Inner BuildRight (15)
+                        :              :- * Project (10)
+                        :              :  +- * BroadcastHashJoin Inner BuildRight (9)
+                        :              :     :- * Filter (3)
+                        :              :     :  +- * ColumnarToRow (2)
+                        :              :     :     +- Scan parquet default.web_sales (1)
+                        :              :     +- BroadcastExchange (8)
+                        :              :        +- * Project (7)
+                        :              :           +- * Filter (6)
+                        :              :              +- * ColumnarToRow (5)
+                        :              :                 +- Scan parquet default.date_dim (4)
+                        :              +- BroadcastExchange (14)
+                        :                 +- * Filter (13)
+                        :                    +- * ColumnarToRow (12)
+                        :                       +- Scan parquet default.item (11)
+                        :- * HashAggregate (24)
+                        :  +- Exchange (23)
+                        :     +- * HashAggregate (22)
+                        :        +- * HashAggregate (21)
+                        :           +- ReusedExchange (20)
+                        +- * HashAggregate (29)
+                           +- Exchange (28)
+                              +- * HashAggregate (27)
+                                 +- * HashAggregate (26)
+                                    +- ReusedExchange (25)
+
+
+(1) Scan parquet default.web_sales
+Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_net_paid:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3]
+Condition : isnotnull(ws_item_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#5, d_month_seq#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#5]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ws_sold_date_sk#3]
+Right keys [1]: [d_date_sk#5]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [2]: [ws_item_sk#1, ws_net_paid#2]
+Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#5]
+
+(11) Scan parquet default.item
+Output [3]: [i_item_sk#8, i_class#9, i_category#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_class:string,i_category:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [i_item_sk#8, i_class#9, i_category#10]
+
+(13) Filter [codegen id : 2]
+Input [3]: [i_item_sk#8, i_class#9, i_category#10]
+Condition : isnotnull(i_item_sk#8)
+
+(14) BroadcastExchange
+Input [3]: [i_item_sk#8, i_class#9, i_category#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ws_item_sk#1]
+Right keys [1]: [i_item_sk#8]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [3]: [ws_net_paid#2, i_class#9, i_category#10]
+Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#8, i_class#9, i_category#10]
+
+(17) HashAggregate [codegen id : 3]
+Input [3]: [ws_net_paid#2, i_class#9, i_category#10]
+Keys [2]: [i_category#10, i_class#9]
+Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))]
+Aggregate Attributes [1]: [sum#12]
+Results [3]: [i_category#10, i_class#9, sum#13]
+
+(18) Exchange
+Input [3]: [i_category#10, i_class#9, sum#13]
+Arguments: hashpartitioning(i_category#10, i_class#9, 5), ENSURE_REQUIREMENTS, [id=#14]
+
+(19) HashAggregate [codegen id : 4]
+Input [3]: [i_category#10, i_class#9, sum#13]
+Keys [2]: [i_category#10, i_class#9]
+Functions [1]: [sum(UnscaledValue(ws_net_paid#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#15]
+Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#15,17,2) as decimal(27,2)) AS total_sum#16, i_category#10, i_class#9, 0 AS g_category#17, 0 AS g_class#18, 0 AS lochierarchy#19]
+
+(20) ReusedExchange [Reuses operator id: unknown]
+Output [3]: [i_category#20, i_class#21, sum#22]
+
+(21) HashAggregate [codegen id : 8]
+Input [3]: [i_category#20, i_class#21, sum#22]
+Keys [2]: [i_category#20, i_class#21]
+Functions [1]: [sum(UnscaledValue(ws_net_paid#23))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#23))#24]
+Results [2]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#23))#24,17,2) AS total_sum#25, i_category#20]
+
+(22) HashAggregate [codegen id : 8]
+Input [2]: [total_sum#25, i_category#20]
+Keys [1]: [i_category#20]
+Functions [1]: [partial_sum(total_sum#25)]
+Aggregate Attributes [2]: [sum#26, isEmpty#27]
+Results [3]: [i_category#20, sum#28, isEmpty#29]
+
+(23) Exchange
+Input [3]: [i_category#20, sum#28, isEmpty#29]
+Arguments: hashpartitioning(i_category#20, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(24) HashAggregate [codegen id : 9]
+Input [3]: [i_category#20, sum#28, isEmpty#29]
+Keys [1]: [i_category#20]
+Functions [1]: [sum(total_sum#25)]
+Aggregate Attributes [1]: [sum(total_sum#25)#31]
+Results [6]: [sum(total_sum#25)#31 AS total_sum#32, i_category#20, null AS i_class#33, 0 AS g_category#34, 1 AS g_class#35, 1 AS lochierarchy#36]
+
+(25) ReusedExchange [Reuses operator id: unknown]
+Output [3]: [i_category#37, i_class#38, sum#39]
+
+(26) HashAggregate [codegen id : 13]
+Input [3]: [i_category#37, i_class#38, sum#39]
+Keys [2]: [i_category#37, i_class#38]
+Functions [1]: [sum(UnscaledValue(ws_net_paid#40))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#40))#41]
+Results [1]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#40))#41,17,2) AS total_sum#25]
+
+(27) HashAggregate [codegen id : 13]
+Input [1]: [total_sum#25]
+Keys: []
+Functions [1]: [partial_sum(total_sum#25)]
+Aggregate Attributes [2]: [sum#42, isEmpty#43]
+Results [2]: [sum#44, isEmpty#45]
+
+(28) Exchange
+Input [2]: [sum#44, isEmpty#45]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#46]
+
+(29) HashAggregate [codegen id : 14]
+Input [2]: [sum#44, isEmpty#45]
+Keys: []
+Functions [1]: [sum(total_sum#25)]
+Aggregate Attributes [1]: [sum(total_sum#25)#47]
+Results [6]: [sum(total_sum#25)#47 AS total_sum#48, null AS i_category#49, null AS i_class#50, 1 AS g_category#51, 1 AS g_class#52, 2 AS lochierarchy#53]
+
+(30) Union
+
+(31) HashAggregate [codegen id : 15]
+Input [6]: [total_sum#16, i_category#10, i_class#9, g_category#17, g_class#18, lochierarchy#19]
+Keys [6]: [total_sum#16, i_category#10, i_class#9, g_category#17, g_class#18, lochierarchy#19]
+Functions: []
+Aggregate Attributes: []
+Results [6]: [total_sum#16, i_category#10, i_class#9, g_category#17, g_class#18, lochierarchy#19]
+
+(32) Exchange
+Input [6]: [total_sum#16, i_category#10, i_class#9, g_category#17, g_class#18, lochierarchy#19]
+Arguments: hashpartitioning(total_sum#16, i_category#10, i_class#9, g_category#17, g_class#18, lochierarchy#19, 5), ENSURE_REQUIREMENTS, [id=#54]
+
+(33) HashAggregate [codegen id : 16]
+Input [6]: [total_sum#16, i_category#10, i_class#9, g_category#17, g_class#18, lochierarchy#19]
+Keys [6]: [total_sum#16, i_category#10, i_class#9, g_category#17, g_class#18, lochierarchy#19]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [total_sum#16, i_category#10, i_class#9, lochierarchy#19, CASE WHEN (g_class#18 = 0) THEN i_category#10 END AS _w0#55]
+
+(34) Exchange
+Input [5]: [total_sum#16, i_category#10, i_class#9, lochierarchy#19, _w0#55]
+Arguments: hashpartitioning(lochierarchy#19, _w0#55, 5), ENSURE_REQUIREMENTS, [id=#56]
+
+(35) Sort [codegen id : 17]
+Input [5]: [total_sum#16, i_category#10, i_class#9, lochierarchy#19, _w0#55]
+Arguments: [lochierarchy#19 ASC NULLS FIRST, _w0#55 ASC NULLS FIRST, total_sum#16 DESC NULLS LAST], false, 0
+
+(36) Window
+Input [5]: [total_sum#16, i_category#10, i_class#9, lochierarchy#19, _w0#55]
+Arguments: [rank(total_sum#16) windowspecdefinition(lochierarchy#19, _w0#55, total_sum#16 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#57], [lochierarchy#19, _w0#55], [total_sum#16 DESC NULLS LAST]
+
+(37) Project [codegen id : 18]
+Output [5]: [total_sum#16, i_category#10, i_class#9, lochierarchy#19, rank_within_parent#57]
+Input [6]: [total_sum#16, i_category#10, i_class#9, lochierarchy#19, _w0#55, rank_within_parent#57]
+
+(38) TakeOrderedAndProject
+Input [5]: [total_sum#16, i_category#10, i_class#9, lochierarchy#19, rank_within_parent#57]
+Arguments: 100, [lochierarchy#19 DESC NULLS LAST, CASE WHEN (lochierarchy#19 = 0) THEN i_category#10 END ASC NULLS FIRST, rank_within_parent#57 ASC NULLS FIRST], [total_sum#16, i_category#10, i_class#9, lochierarchy#19, rank_within_parent#57]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (39)
+
+
+(39) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#5]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.sf100/simplified.txt
new file mode 100644
index 0000000000000..6070dbc29fa6f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.sf100/simplified.txt
@@ -0,0 +1,66 @@
+TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_class]
+  WholeStageCodegen (18)
+    Project [total_sum,i_category,i_class,lochierarchy,rank_within_parent]
+      InputAdapter
+        Window [total_sum,lochierarchy,_w0]
+          WholeStageCodegen (17)
+            Sort [lochierarchy,_w0,total_sum]
+              InputAdapter
+                Exchange [lochierarchy,_w0] #1
+                  WholeStageCodegen (16)
+                    HashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy] [_w0]
+                      InputAdapter
+                        Exchange [total_sum,i_category,i_class,g_category,g_class,lochierarchy] #2
+                          WholeStageCodegen (15)
+                            HashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy]
+                              InputAdapter
+                                Union
+                                  WholeStageCodegen (4)
+                                    HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,g_category,g_class,lochierarchy,sum]
+                                      InputAdapter
+                                        Exchange [i_category,i_class] #3
+                                          WholeStageCodegen (3)
+                                            HashAggregate [i_category,i_class,ws_net_paid] [sum,sum]
+                                              Project [ws_net_paid,i_class,i_category]
+                                                BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                                  Project [ws_item_sk,ws_net_paid]
+                                                    BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                      Filter [ws_item_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk]
+                                                              SubqueryBroadcast [d_date_sk] #1
+                                                                ReusedExchange [d_date_sk] #4
+                                                      InputAdapter
+                                                        BroadcastExchange #4
+                                                          WholeStageCodegen (1)
+                                                            Project [d_date_sk]
+                                                              Filter [d_month_seq,d_date_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.date_dim [d_date_sk,d_month_seq]
+                                                  InputAdapter
+                                                    BroadcastExchange #5
+                                                      WholeStageCodegen (2)
+                                                        Filter [i_item_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.item [i_item_sk,i_class,i_category]
+                                  WholeStageCodegen (9)
+                                    HashAggregate [i_category,sum,isEmpty] [sum(total_sum),total_sum,i_class,g_category,g_class,lochierarchy,sum,isEmpty]
+                                      InputAdapter
+                                        Exchange [i_category] #6
+                                          WholeStageCodegen (8)
+                                            HashAggregate [i_category,total_sum] [sum,isEmpty,sum,isEmpty]
+                                              HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,sum]
+                                                InputAdapter
+                                                  ReusedExchange [i_category,i_class,sum] #7
+                                  WholeStageCodegen (14)
+                                    HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,isEmpty]
+                                      InputAdapter
+                                        Exchange #8
+                                          WholeStageCodegen (13)
+                                            HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty]
+                                              HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,sum]
+                                                InputAdapter
+                                                  ReusedExchange [i_category,i_class,sum] #7
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/explain.txt
new file mode 100644
index 0000000000000..64f6270e81183
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/explain.txt
@@ -0,0 +1,238 @@
+== Physical Plan ==
+TakeOrderedAndProject (38)
++- * Project (37)
+   +- Window (36)
+      +- * Sort (35)
+         +- Exchange (34)
+            +- * HashAggregate (33)
+               +- Exchange (32)
+                  +- * HashAggregate (31)
+                     +- Union (30)
+                        :- * HashAggregate (19)
+                        :  +- Exchange (18)
+                        :     +- * HashAggregate (17)
+                        :        +- * Project (16)
+                        :           +- * BroadcastHashJoin Inner BuildRight (15)
+                        :              :- * Project (10)
+                        :              :  +- * BroadcastHashJoin Inner BuildRight (9)
+                        :              :     :- * Filter (3)
+                        :              :     :  +- * ColumnarToRow (2)
+                        :              :     :     +- Scan parquet default.web_sales (1)
+                        :              :     +- BroadcastExchange (8)
+                        :              :        +- * Project (7)
+                        :              :           +- * Filter (6)
+                        :              :              +- * ColumnarToRow (5)
+                        :              :                 +- Scan parquet default.date_dim (4)
+                        :              +- BroadcastExchange (14)
+                        :                 +- * Filter (13)
+                        :                    +- * ColumnarToRow (12)
+                        :                       +- Scan parquet default.item (11)
+                        :- * HashAggregate (24)
+                        :  +- Exchange (23)
+                        :     +- * HashAggregate (22)
+                        :        +- * HashAggregate (21)
+                        :           +- ReusedExchange (20)
+                        +- * HashAggregate (29)
+                           +- Exchange (28)
+                              +- * HashAggregate (27)
+                                 +- * HashAggregate (26)
+                                    +- ReusedExchange (25)
+
+
+(1) Scan parquet default.web_sales
+Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ws_item_sk)]
+ReadSchema: struct<ws_item_sk:int,ws_net_paid:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3]
+Condition : isnotnull(ws_item_sk#1)
+
+(4) Scan parquet default.date_dim
+Output [2]: [d_date_sk#5, d_month_seq#6]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_month_seq:int>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+
+(6) Filter [codegen id : 1]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5))
+
+(7) Project [codegen id : 1]
+Output [1]: [d_date_sk#5]
+Input [2]: [d_date_sk#5, d_month_seq#6]
+
+(8) BroadcastExchange
+Input [1]: [d_date_sk#5]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7]
+
+(9) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ws_sold_date_sk#3]
+Right keys [1]: [d_date_sk#5]
+Join condition: None
+
+(10) Project [codegen id : 3]
+Output [2]: [ws_item_sk#1, ws_net_paid#2]
+Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#5]
+
+(11) Scan parquet default.item
+Output [3]: [i_item_sk#8, i_class#9, i_category#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_class:string,i_category:string>
+
+(12) ColumnarToRow [codegen id : 2]
+Input [3]: [i_item_sk#8, i_class#9, i_category#10]
+
+(13) Filter [codegen id : 2]
+Input [3]: [i_item_sk#8, i_class#9, i_category#10]
+Condition : isnotnull(i_item_sk#8)
+
+(14) BroadcastExchange
+Input [3]: [i_item_sk#8, i_class#9, i_category#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ws_item_sk#1]
+Right keys [1]: [i_item_sk#8]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [3]: [ws_net_paid#2, i_class#9, i_category#10]
+Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#8, i_class#9, i_category#10]
+
+(17) HashAggregate [codegen id : 3]
+Input [3]: [ws_net_paid#2, i_class#9, i_category#10]
+Keys [2]: [i_category#10, i_class#9]
+Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))]
+Aggregate Attributes [1]: [sum#12]
+Results [3]: [i_category#10, i_class#9, sum#13]
+
+(18) Exchange
+Input [3]: [i_category#10, i_class#9, sum#13]
+Arguments: hashpartitioning(i_category#10, i_class#9, 5), ENSURE_REQUIREMENTS, [id=#14]
+
+(19) HashAggregate [codegen id : 4]
+Input [3]: [i_category#10, i_class#9, sum#13]
+Keys [2]: [i_category#10, i_class#9]
+Functions [1]: [sum(UnscaledValue(ws_net_paid#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#15]
+Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#15,17,2) as decimal(27,2)) AS total_sum#16, i_category#10, i_class#9, 0 AS g_category#17, 0 AS g_class#18, 0 AS lochierarchy#19]
+
+(20) ReusedExchange [Reuses operator id: unknown]
+Output [3]: [i_category#20, i_class#21, sum#22]
+
+(21) HashAggregate [codegen id : 8]
+Input [3]: [i_category#20, i_class#21, sum#22]
+Keys [2]: [i_category#20, i_class#21]
+Functions [1]: [sum(UnscaledValue(ws_net_paid#23))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#23))#24]
+Results [2]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#23))#24,17,2) AS total_sum#25, i_category#20]
+
+(22) HashAggregate [codegen id : 8]
+Input [2]: [total_sum#25, i_category#20]
+Keys [1]: [i_category#20]
+Functions [1]: [partial_sum(total_sum#25)]
+Aggregate Attributes [2]: [sum#26, isEmpty#27]
+Results [3]: [i_category#20, sum#28, isEmpty#29]
+
+(23) Exchange
+Input [3]: [i_category#20, sum#28, isEmpty#29]
+Arguments: hashpartitioning(i_category#20, 5), ENSURE_REQUIREMENTS, [id=#30]
+
+(24) HashAggregate [codegen id : 9]
+Input [3]: [i_category#20, sum#28, isEmpty#29]
+Keys [1]: [i_category#20]
+Functions [1]: [sum(total_sum#25)]
+Aggregate Attributes [1]: [sum(total_sum#25)#31]
+Results [6]: [sum(total_sum#25)#31 AS total_sum#32, i_category#20, null AS i_class#33, 0 AS g_category#34, 1 AS g_class#35, 1 AS lochierarchy#36]
+
+(25) ReusedExchange [Reuses operator id: unknown]
+Output [3]: [i_category#37, i_class#38, sum#39]
+
+(26) HashAggregate [codegen id : 13]
+Input [3]: [i_category#37, i_class#38, sum#39]
+Keys [2]: [i_category#37, i_class#38]
+Functions [1]: [sum(UnscaledValue(ws_net_paid#40))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#40))#41]
+Results [1]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#40))#41,17,2) AS total_sum#25]
+
+(27) HashAggregate [codegen id : 13]
+Input [1]: [total_sum#25]
+Keys: []
+Functions [1]: [partial_sum(total_sum#25)]
+Aggregate Attributes [2]: [sum#42, isEmpty#43]
+Results [2]: [sum#44, isEmpty#45]
+
+(28) Exchange
+Input [2]: [sum#44, isEmpty#45]
+Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#46]
+
+(29) HashAggregate [codegen id : 14]
+Input [2]: [sum#44, isEmpty#45]
+Keys: []
+Functions [1]: [sum(total_sum#25)]
+Aggregate Attributes [1]: [sum(total_sum#25)#47]
+Results [6]: [sum(total_sum#25)#47 AS total_sum#48, null AS i_category#49, null AS i_class#50, 1 AS g_category#51, 1 AS g_class#52, 2 AS lochierarchy#53]
+
+(30) Union
+
+(31) HashAggregate [codegen id : 15]
+Input [6]: [total_sum#16, i_category#10, i_class#9, g_category#17, g_class#18, lochierarchy#19]
+Keys [6]: [total_sum#16, i_category#10, i_class#9, g_category#17, g_class#18, lochierarchy#19]
+Functions: []
+Aggregate Attributes: []
+Results [6]: [total_sum#16, i_category#10, i_class#9, g_category#17, g_class#18, lochierarchy#19]
+
+(32) Exchange
+Input [6]: [total_sum#16, i_category#10, i_class#9, g_category#17, g_class#18, lochierarchy#19]
+Arguments: hashpartitioning(total_sum#16, i_category#10, i_class#9, g_category#17, g_class#18, lochierarchy#19, 5), ENSURE_REQUIREMENTS, [id=#54]
+
+(33) HashAggregate [codegen id : 16]
+Input [6]: [total_sum#16, i_category#10, i_class#9, g_category#17, g_class#18, lochierarchy#19]
+Keys [6]: [total_sum#16, i_category#10, i_class#9, g_category#17, g_class#18, lochierarchy#19]
+Functions: []
+Aggregate Attributes: []
+Results [5]: [total_sum#16, i_category#10, i_class#9, lochierarchy#19, CASE WHEN (g_class#18 = 0) THEN i_category#10 END AS _w0#55]
+
+(34) Exchange
+Input [5]: [total_sum#16, i_category#10, i_class#9, lochierarchy#19, _w0#55]
+Arguments: hashpartitioning(lochierarchy#19, _w0#55, 5), ENSURE_REQUIREMENTS, [id=#56]
+
+(35) Sort [codegen id : 17]
+Input [5]: [total_sum#16, i_category#10, i_class#9, lochierarchy#19, _w0#55]
+Arguments: [lochierarchy#19 ASC NULLS FIRST, _w0#55 ASC NULLS FIRST, total_sum#16 DESC NULLS LAST], false, 0
+
+(36) Window
+Input [5]: [total_sum#16, i_category#10, i_class#9, lochierarchy#19, _w0#55]
+Arguments: [rank(total_sum#16) windowspecdefinition(lochierarchy#19, _w0#55, total_sum#16 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#57], [lochierarchy#19, _w0#55], [total_sum#16 DESC NULLS LAST]
+
+(37) Project [codegen id : 18]
+Output [5]: [total_sum#16, i_category#10, i_class#9, lochierarchy#19, rank_within_parent#57]
+Input [6]: [total_sum#16, i_category#10, i_class#9, lochierarchy#19, _w0#55, rank_within_parent#57]
+
+(38) TakeOrderedAndProject
+Input [5]: [total_sum#16, i_category#10, i_class#9, lochierarchy#19, rank_within_parent#57]
+Arguments: 100, [lochierarchy#19 DESC NULLS LAST, CASE WHEN (lochierarchy#19 = 0) THEN i_category#10 END ASC NULLS FIRST, rank_within_parent#57 ASC NULLS FIRST], [total_sum#16, i_category#10, i_class#9, lochierarchy#19, rank_within_parent#57]
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (39)
+
+
+(39) ReusedExchange [Reuses operator id: 8]
+Output [1]: [d_date_sk#5]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/simplified.txt
new file mode 100644
index 0000000000000..6070dbc29fa6f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/simplified.txt
@@ -0,0 +1,66 @@
+TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_class]
+  WholeStageCodegen (18)
+    Project [total_sum,i_category,i_class,lochierarchy,rank_within_parent]
+      InputAdapter
+        Window [total_sum,lochierarchy,_w0]
+          WholeStageCodegen (17)
+            Sort [lochierarchy,_w0,total_sum]
+              InputAdapter
+                Exchange [lochierarchy,_w0] #1
+                  WholeStageCodegen (16)
+                    HashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy] [_w0]
+                      InputAdapter
+                        Exchange [total_sum,i_category,i_class,g_category,g_class,lochierarchy] #2
+                          WholeStageCodegen (15)
+                            HashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy]
+                              InputAdapter
+                                Union
+                                  WholeStageCodegen (4)
+                                    HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,g_category,g_class,lochierarchy,sum]
+                                      InputAdapter
+                                        Exchange [i_category,i_class] #3
+                                          WholeStageCodegen (3)
+                                            HashAggregate [i_category,i_class,ws_net_paid] [sum,sum]
+                                              Project [ws_net_paid,i_class,i_category]
+                                                BroadcastHashJoin [ws_item_sk,i_item_sk]
+                                                  Project [ws_item_sk,ws_net_paid]
+                                                    BroadcastHashJoin [ws_sold_date_sk,d_date_sk]
+                                                      Filter [ws_item_sk]
+                                                        ColumnarToRow
+                                                          InputAdapter
+                                                            Scan parquet default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk]
+                                                              SubqueryBroadcast [d_date_sk] #1
+                                                                ReusedExchange [d_date_sk] #4
+                                                      InputAdapter
+                                                        BroadcastExchange #4
+                                                          WholeStageCodegen (1)
+                                                            Project [d_date_sk]
+                                                              Filter [d_month_seq,d_date_sk]
+                                                                ColumnarToRow
+                                                                  InputAdapter
+                                                                    Scan parquet default.date_dim [d_date_sk,d_month_seq]
+                                                  InputAdapter
+                                                    BroadcastExchange #5
+                                                      WholeStageCodegen (2)
+                                                        Filter [i_item_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.item [i_item_sk,i_class,i_category]
+                                  WholeStageCodegen (9)
+                                    HashAggregate [i_category,sum,isEmpty] [sum(total_sum),total_sum,i_class,g_category,g_class,lochierarchy,sum,isEmpty]
+                                      InputAdapter
+                                        Exchange [i_category] #6
+                                          WholeStageCodegen (8)
+                                            HashAggregate [i_category,total_sum] [sum,isEmpty,sum,isEmpty]
+                                              HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,sum]
+                                                InputAdapter
+                                                  ReusedExchange [i_category,i_class,sum] #7
+                                  WholeStageCodegen (14)
+                                    HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,isEmpty]
+                                      InputAdapter
+                                        Exchange #8
+                                          WholeStageCodegen (13)
+                                            HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty]
+                                              HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,sum]
+                                                InputAdapter
+                                                  ReusedExchange [i_category,i_class,sum] #7
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.sf100/explain.txt
new file mode 100644
index 0000000000000..33d4b7d49e1d0
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.sf100/explain.txt
@@ -0,0 +1,168 @@
+== Physical Plan ==
+* Sort (28)
++- Exchange (27)
+   +- * Project (26)
+      +- Window (25)
+         +- * Sort (24)
+            +- Exchange (23)
+               +- * HashAggregate (22)
+                  +- Exchange (21)
+                     +- * HashAggregate (20)
+                        +- * Project (19)
+                           +- * BroadcastHashJoin Inner BuildRight (18)
+                              :- * Project (12)
+                              :  +- * SortMergeJoin Inner (11)
+                              :     :- * Sort (5)
+                              :     :  +- Exchange (4)
+                              :     :     +- * Filter (3)
+                              :     :        +- * ColumnarToRow (2)
+                              :     :           +- Scan parquet default.store_sales (1)
+                              :     +- * Sort (10)
+                              :        +- Exchange (9)
+                              :           +- * Filter (8)
+                              :              +- * ColumnarToRow (7)
+                              :                 +- Scan parquet default.item (6)
+                              +- BroadcastExchange (17)
+                                 +- * Project (16)
+                                    +- * Filter (15)
+                                       +- * ColumnarToRow (14)
+                                          +- Scan parquet default.date_dim (13)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 1]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_item_sk#1)
+
+(4) Exchange
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [id=#5]
+
+(5) Sort [codegen id : 2]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Arguments: [ss_item_sk#1 ASC NULLS FIRST], false, 0
+
+(6) Scan parquet default.item
+Output [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [In(i_category, [Sports                                            ,Books                                             ,Home                                              ]), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_item_desc:string,i_current_price:decimal(7,2),i_class:string,i_category:string>
+
+(7) ColumnarToRow [codegen id : 3]
+Input [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+
+(8) Filter [codegen id : 3]
+Input [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Condition : (i_category#11 IN (Sports                                            ,Books                                             ,Home                                              ) AND isnotnull(i_item_sk#6))
+
+(9) Exchange
+Input [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Arguments: hashpartitioning(i_item_sk#6, 5), ENSURE_REQUIREMENTS, [id=#12]
+
+(10) Sort [codegen id : 4]
+Input [6]: [i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Arguments: [i_item_sk#6 ASC NULLS FIRST], false, 0
+
+(11) SortMergeJoin [codegen id : 6]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#6]
+Join condition: None
+
+(12) Project [codegen id : 6]
+Output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#6, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+
+(13) Scan parquet default.date_dim
+Output [2]: [d_date_sk#13, d_date#14]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(14) ColumnarToRow [codegen id : 5]
+Input [2]: [d_date_sk#13, d_date#14]
+
+(15) Filter [codegen id : 5]
+Input [2]: [d_date_sk#13, d_date#14]
+Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13))
+
+(16) Project [codegen id : 5]
+Output [1]: [d_date_sk#13]
+Input [2]: [d_date_sk#13, d_date#14]
+
+(17) BroadcastExchange
+Input [1]: [d_date_sk#13]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#15]
+
+(18) BroadcastHashJoin [codegen id : 6]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#13]
+Join condition: None
+
+(19) Project [codegen id : 6]
+Output [6]: [ss_ext_sales_price#2, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11, d_date_sk#13]
+
+(20) HashAggregate [codegen id : 6]
+Input [6]: [ss_ext_sales_price#2, i_item_id#7, i_item_desc#8, i_current_price#9, i_class#10, i_category#11]
+Keys [5]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum#16]
+Results [6]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, sum#17]
+
+(21) Exchange
+Input [6]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, sum#17]
+Arguments: hashpartitioning(i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, 5), ENSURE_REQUIREMENTS, [id=#18]
+
+(22) HashAggregate [codegen id : 7]
+Input [6]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, sum#17]
+Keys [5]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#19]
+Results [8]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#19,17,2) AS itemrevenue#20, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#19,17,2) AS _w0#21, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#19,17,2) AS _w1#22]
+
+(23) Exchange
+Input [8]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, _w0#21, _w1#22]
+Arguments: hashpartitioning(i_class#10, 5), ENSURE_REQUIREMENTS, [id=#23]
+
+(24) Sort [codegen id : 8]
+Input [8]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, _w0#21, _w1#22]
+Arguments: [i_class#10 ASC NULLS FIRST], false, 0
+
+(25) Window
+Input [8]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, _w0#21, _w1#22]
+Arguments: [sum(_w1#22) windowspecdefinition(i_class#10, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#24], [i_class#10]
+
+(26) Project [codegen id : 9]
+Output [7]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(_w0#21) * 100.00), DecimalType(21,2), true) as decimal(27,2))) / promote_precision(_we0#24)), DecimalType(38,17), true) AS revenueratio#25]
+Input [9]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, _w0#21, _w1#22, _we0#24]
+
+(27) Exchange
+Input [7]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, revenueratio#25]
+Arguments: rangepartitioning(i_category#11 ASC NULLS FIRST, i_class#10 ASC NULLS FIRST, i_item_id#7 ASC NULLS FIRST, i_item_desc#8 ASC NULLS FIRST, revenueratio#25 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [id=#26]
+
+(28) Sort [codegen id : 10]
+Input [7]: [i_item_id#7, i_item_desc#8, i_category#11, i_class#10, i_current_price#9, itemrevenue#20, revenueratio#25]
+Arguments: [i_category#11 ASC NULLS FIRST, i_class#10 ASC NULLS FIRST, i_item_id#7 ASC NULLS FIRST, i_item_desc#8 ASC NULLS FIRST, revenueratio#25 ASC NULLS FIRST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (29)
+
+
+(29) ReusedExchange [Reuses operator id: 17]
+Output [1]: [d_date_sk#13]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.sf100/simplified.txt
new file mode 100644
index 0000000000000..45e9e64099cb7
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.sf100/simplified.txt
@@ -0,0 +1,52 @@
+WholeStageCodegen (10)
+  Sort [i_category,i_class,i_item_id,i_item_desc,revenueratio]
+    InputAdapter
+      Exchange [i_category,i_class,i_item_id,i_item_desc,revenueratio] #1
+        WholeStageCodegen (9)
+          Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0]
+            InputAdapter
+              Window [_w1,i_class]
+                WholeStageCodegen (8)
+                  Sort [i_class]
+                    InputAdapter
+                      Exchange [i_class] #2
+                        WholeStageCodegen (7)
+                          HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ss_ext_sales_price)),itemrevenue,_w0,_w1,sum]
+                            InputAdapter
+                              Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3
+                                WholeStageCodegen (6)
+                                  HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ss_ext_sales_price] [sum,sum]
+                                    Project [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                        Project [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                          SortMergeJoin [ss_item_sk,i_item_sk]
+                                            InputAdapter
+                                              WholeStageCodegen (2)
+                                                Sort [ss_item_sk]
+                                                  InputAdapter
+                                                    Exchange [ss_item_sk] #4
+                                                      WholeStageCodegen (1)
+                                                        Filter [ss_item_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                                                SubqueryBroadcast [d_date_sk] #1
+                                                                  ReusedExchange [d_date_sk] #5
+                                            InputAdapter
+                                              WholeStageCodegen (4)
+                                                Sort [i_item_sk]
+                                                  InputAdapter
+                                                    Exchange [i_item_sk] #6
+                                                      WholeStageCodegen (3)
+                                                        Filter [i_category,i_item_sk]
+                                                          ColumnarToRow
+                                                            InputAdapter
+                                                              Scan parquet default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                        InputAdapter
+                                          BroadcastExchange #5
+                                            WholeStageCodegen (5)
+                                              Project [d_date_sk]
+                                                Filter [d_date,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_date]
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/explain.txt
new file mode 100644
index 0000000000000..b7c5f4081a1f3
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/explain.txt
@@ -0,0 +1,153 @@
+== Physical Plan ==
+* Sort (25)
++- Exchange (24)
+   +- * Project (23)
+      +- Window (22)
+         +- * Sort (21)
+            +- Exchange (20)
+               +- * HashAggregate (19)
+                  +- Exchange (18)
+                     +- * HashAggregate (17)
+                        +- * Project (16)
+                           +- * BroadcastHashJoin Inner BuildRight (15)
+                              :- * Project (9)
+                              :  +- * BroadcastHashJoin Inner BuildRight (8)
+                              :     :- * Filter (3)
+                              :     :  +- * ColumnarToRow (2)
+                              :     :     +- Scan parquet default.store_sales (1)
+                              :     +- BroadcastExchange (7)
+                              :        +- * Filter (6)
+                              :           +- * ColumnarToRow (5)
+                              :              +- Scan parquet default.item (4)
+                              +- BroadcastExchange (14)
+                                 +- * Project (13)
+                                    +- * Filter (12)
+                                       +- * ColumnarToRow (11)
+                                          +- Scan parquet default.date_dim (10)
+
+
+(1) Scan parquet default.store_sales
+Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Batched: true
+Location: InMemoryFileIndex []
+PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)]
+PushedFilters: [IsNotNull(ss_item_sk)]
+ReadSchema: struct<ss_item_sk:int,ss_ext_sales_price:decimal(7,2)>
+
+(2) ColumnarToRow [codegen id : 3]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+
+(3) Filter [codegen id : 3]
+Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3]
+Condition : isnotnull(ss_item_sk#1)
+
+(4) Scan parquet default.item
+Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/item]
+PushedFilters: [In(i_category, [Sports                                            ,Books                                             ,Home                                              ]), IsNotNull(i_item_sk)]
+ReadSchema: struct<i_item_sk:int,i_item_id:string,i_item_desc:string,i_current_price:decimal(7,2),i_class:string,i_category:string>
+
+(5) ColumnarToRow [codegen id : 1]
+Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+
+(6) Filter [codegen id : 1]
+Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Condition : (i_category#10 IN (Sports                                            ,Books                                             ,Home                                              ) AND isnotnull(i_item_sk#5))
+
+(7) BroadcastExchange
+Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#11]
+
+(8) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_item_sk#1]
+Right keys [1]: [i_item_sk#5]
+Join condition: None
+
+(9) Project [codegen id : 3]
+Output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+
+(10) Scan parquet default.date_dim
+Output [2]: [d_date_sk#12, d_date#13]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/date_dim]
+PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)]
+ReadSchema: struct<d_date_sk:int,d_date:date>
+
+(11) ColumnarToRow [codegen id : 2]
+Input [2]: [d_date_sk#12, d_date#13]
+
+(12) Filter [codegen id : 2]
+Input [2]: [d_date_sk#12, d_date#13]
+Condition : (((isnotnull(d_date#13) AND (d_date#13 >= 1999-02-22)) AND (d_date#13 <= 1999-03-24)) AND isnotnull(d_date_sk#12))
+
+(13) Project [codegen id : 2]
+Output [1]: [d_date_sk#12]
+Input [2]: [d_date_sk#12, d_date#13]
+
+(14) BroadcastExchange
+Input [1]: [d_date_sk#12]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#14]
+
+(15) BroadcastHashJoin [codegen id : 3]
+Left keys [1]: [ss_sold_date_sk#3]
+Right keys [1]: [d_date_sk#12]
+Join condition: None
+
+(16) Project [codegen id : 3]
+Output [6]: [ss_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10, d_date_sk#12]
+
+(17) HashAggregate [codegen id : 3]
+Input [6]: [ss_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10]
+Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8]
+Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum#15]
+Results [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#16]
+
+(18) Exchange
+Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#16]
+Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, [id=#17]
+
+(19) HashAggregate [codegen id : 4]
+Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#16]
+Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8]
+Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))]
+Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#18]
+Results [8]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#18,17,2) AS itemrevenue#19, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#18,17,2) AS _w0#20, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#18,17,2) AS _w1#21]
+
+(20) Exchange
+Input [8]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21]
+Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, [id=#22]
+
+(21) Sort [codegen id : 5]
+Input [8]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21]
+Arguments: [i_class#9 ASC NULLS FIRST], false, 0
+
+(22) Window
+Input [8]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21]
+Arguments: [sum(_w1#21) windowspecdefinition(i_class#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#23], [i_class#9]
+
+(23) Project [codegen id : 6]
+Output [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(_w0#20) * 100.00), DecimalType(21,2), true) as decimal(27,2))) / promote_precision(_we0#23)), DecimalType(38,17), true) AS revenueratio#24]
+Input [9]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, _w0#20, _w1#21, _we0#23]
+
+(24) Exchange
+Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, revenueratio#24]
+Arguments: rangepartitioning(i_category#10 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_item_id#6 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#24 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [id=#25]
+
+(25) Sort [codegen id : 7]
+Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#19, revenueratio#24]
+Arguments: [i_category#10 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_item_id#6 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#24 ASC NULLS FIRST], true, 0
+
+===== Subqueries =====
+
+Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4
+ReusedExchange (26)
+
+
+(26) ReusedExchange [Reuses operator id: 14]
+Output [1]: [d_date_sk#12]
+
+
diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/simplified.txt
new file mode 100644
index 0000000000000..68157d791c3fc
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/simplified.txt
@@ -0,0 +1,43 @@
+WholeStageCodegen (7)
+  Sort [i_category,i_class,i_item_id,i_item_desc,revenueratio]
+    InputAdapter
+      Exchange [i_category,i_class,i_item_id,i_item_desc,revenueratio] #1
+        WholeStageCodegen (6)
+          Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0]
+            InputAdapter
+              Window [_w1,i_class]
+                WholeStageCodegen (5)
+                  Sort [i_class]
+                    InputAdapter
+                      Exchange [i_class] #2
+                        WholeStageCodegen (4)
+                          HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ss_ext_sales_price)),itemrevenue,_w0,_w1,sum]
+                            InputAdapter
+                              Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3
+                                WholeStageCodegen (3)
+                                  HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ss_ext_sales_price] [sum,sum]
+                                    Project [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                      BroadcastHashJoin [ss_sold_date_sk,d_date_sk]
+                                        Project [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                          BroadcastHashJoin [ss_item_sk,i_item_sk]
+                                            Filter [ss_item_sk]
+                                              ColumnarToRow
+                                                InputAdapter
+                                                  Scan parquet default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk]
+                                                    SubqueryBroadcast [d_date_sk] #1
+                                                      ReusedExchange [d_date_sk] #4
+                                            InputAdapter
+                                              BroadcastExchange #5
+                                                WholeStageCodegen (1)
+                                                  Filter [i_category,i_item_sk]
+                                                    ColumnarToRow
+                                                      InputAdapter
+                                                        Scan parquet default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category]
+                                        InputAdapter
+                                          BroadcastExchange #4
+                                            WholeStageCodegen (2)
+                                              Project [d_date_sk]
+                                                Filter [d_date,d_date_sk]
+                                                  ColumnarToRow
+                                                    InputAdapter
+                                                      Scan parquet default.date_dim [d_date_sk,d_date]
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q1.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q1.sql.out
new file mode 100644
index 0000000000000..d3b808c48dc2d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q1.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<c_customer_id:string>
+-- !query output
+AAAAAAAAAAABAAAA
+AAAAAAAAAAAHBAAA
+AAAAAAAAAAAMAAAA
+AAAAAAAAAAAOAAAA
+AAAAAAAAAABCBAAA
+AAAAAAAAAABEAAAA
+AAAAAAAAAABFAAAA
+AAAAAAAAAACFBAAA
+AAAAAAAAAACFBAAA
+AAAAAAAAAADBBAAA
+AAAAAAAAAADOAAAA
+AAAAAAAAAADPAAAA
+AAAAAAAAAAEABAAA
+AAAAAAAAAAEEAAAA
+AAAAAAAAAAEGBAAA
+AAAAAAAAAAENAAAA
+AAAAAAAAAAFCBAAA
+AAAAAAAAAAFEBAAA
+AAAAAAAAAAFGAAAA
+AAAAAAAAAAFLAAAA
+AAAAAAAAAAFPAAAA
+AAAAAAAAAAGCAAAA
+AAAAAAAAAAGEAAAA
+AAAAAAAAAAGIBAAA
+AAAAAAAAAAGOAAAA
+AAAAAAAAAAHABAAA
+AAAAAAAAAAHGBAAA
+AAAAAAAAAAHHAAAA
+AAAAAAAAAAHMAAAA
+AAAAAAAAAAHPAAAA
+AAAAAAAAAAHPAAAA
+AAAAAAAAAAHPAAAA
+AAAAAAAAAAJAAAAA
+AAAAAAAAAAJEBAAA
+AAAAAAAAAAJMAAAA
+AAAAAAAAAAJPAAAA
+AAAAAAAAAAKBBAAA
+AAAAAAAAAAKGBAAA
+AAAAAAAAAAKHBAAA
+AAAAAAAAAAKLAAAA
+AAAAAAAAAALCAAAA
+AAAAAAAAAALJAAAA
+AAAAAAAAAALJAAAA
+AAAAAAAAAAMABAAA
+AAAAAAAAAAMGAAAA
+AAAAAAAAAAMLAAAA
+AAAAAAAAAAMMAAAA
+AAAAAAAAAANHBAAA
+AAAAAAAAAANMAAAA
+AAAAAAAAAAOBBAAA
+AAAAAAAAAAPDAAAA
+AAAAAAAAAAPKAAAA
+AAAAAAAAAAPLAAAA
+AAAAAAAAABANAAAA
+AAAAAAAAABCCBAAA
+AAAAAAAAABCGAAAA
+AAAAAAAAABDABAAA
+AAAAAAAAABDBAAAA
+AAAAAAAAABDEAAAA
+AAAAAAAAABDEBAAA
+AAAAAAAAABDEBAAA
+AAAAAAAAABDFBAAA
+AAAAAAAAABDOAAAA
+AAAAAAAAABDOAAAA
+AAAAAAAAABEBBAAA
+AAAAAAAAABEDAAAA
+AAAAAAAAABEEAAAA
+AAAAAAAAABEEBAAA
+AAAAAAAAABEIBAAA
+AAAAAAAAABEOAAAA
+AAAAAAAAABFFBAAA
+AAAAAAAAABFHAAAA
+AAAAAAAAABFNAAAA
+AAAAAAAAABFOAAAA
+AAAAAAAAABGAAAAA
+AAAAAAAAABHDBAAA
+AAAAAAAAABHGAAAA
+AAAAAAAAABHGBAAA
+AAAAAAAAABHLAAAA
+AAAAAAAAABIAAAAA
+AAAAAAAAABIBAAAA
+AAAAAAAAABIDBAAA
+AAAAAAAAABIEBAAA
+AAAAAAAAABKLAAAA
+AAAAAAAAABKNAAAA
+AAAAAAAAABKNAAAA
+AAAAAAAAABLJAAAA
+AAAAAAAAABLNAAAA
+AAAAAAAAABMAAAAA
+AAAAAAAAABMEBAAA
+AAAAAAAAABMPAAAA
+AAAAAAAAABNABAAA
+AAAAAAAAABNBAAAA
+AAAAAAAAABNEAAAA
+AAAAAAAAABNEAAAA
+AAAAAAAAABNGAAAA
+AAAAAAAAABNNAAAA
+AAAAAAAAABOEAAAA
+AAAAAAAAABOGBAAA
+AAAAAAAAABPABAAA
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q10.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q10.sql.out
new file mode 100644
index 0000000000000..195c969028573
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q10.sql.out
@@ -0,0 +1,11 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<cd_gender:string,cd_marital_status:string,cd_education_status:string,cnt1:bigint,cd_purchase_estimate:int,cnt2:bigint,cd_credit_rating:string,cnt3:bigint,cd_dep_count:int,cnt4:bigint,cd_dep_employed_count:int,cnt5:bigint,cd_dep_college_count:int,cnt6:bigint>
+-- !query output
+F	W	4 yr Degree         	1	4000	1	Low Risk  	1	4	1	6	1	4	1
+M	D	4 yr Degree         	1	1500	1	Low Risk  	1	3	1	4	1	2	1
+M	S	College             	1	4500	1	High Risk 	1	3	1	4	1	3	1
+M	S	Primary             	1	9500	1	Low Risk  	1	3	1	0	1	6	1
+M	S	Secondary           	1	3000	1	High Risk 	1	1	1	1	1	4	1
+M	U	4 yr Degree         	1	2000	1	Low Risk  	1	3	1	1	1	3	1
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q11.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q11.sql.out
new file mode 100644
index 0000000000000..8e02562680cf9
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q11.sql.out
@@ -0,0 +1,99 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<customer_preferred_cust_flag:string>
+-- !query output
+NULL
+NULL
+NULL
+NULL
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+N
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
+Y
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q12.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q12.sql.out
new file mode 100644
index 0000000000000..92e6e202b7a33
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q12.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_item_desc:string,i_category:string,i_class:string,i_current_price:decimal(7,2),itemrevenue:decimal(17,2),revenueratio:decimal(38,17)>
+-- !query output
+Precisely elderly bodies	Books                                             	arts                                              	1.40	11.21	0.01417562243122168
+Great, contemporary workers would not remove of course cultural values. Then due children might see positive seconds. Significant problems w	Books                                             	arts                                              	0.55	515.52	0.65190159462474560
+Forward psychological plants establish closely yet eastern changes. Likewise necessary techniques might drop. Pleasant operations like lonely things; dogs let regions. Forces might not result clearl	Books                                             	arts                                              	2.43	11462.46	14.49487110552909973
+Black, following services justify by a investors; dirty, different charts will fly however prizes. Temporary, l	Books                                             	arts                                              	5.56	3400.60	4.30023386615632740
+United, important objectives put similarly large, previous phenomena; old, present days receive. Happy detectives assi	Books                                             	arts                                              	1.26	784.30	0.99178774958137022
+Naturally new years put serious, negative vehicles. Fin	Books                                             	arts                                              	3.34	3319.96	4.19826043236027781
+Hard different differences would not paint even. Together suitable schemes marry directly only open women. Social ca	Books                                             	arts                                              	2.65	229.68	0.29044219090124839
+Anonymous, useful women provoke slightly present persons. Ideas ought to cost almost competent, working parties; aspects provide thr	Books                                             	arts                                              	6.73	5752.44	7.27425669029944833
+Powerful walls will find; there scottish decades must not	Books                                             	arts                                              	4.16	434.76	0.54977641464745189
+Careful privileges ought to live rather to a boards. Possible, broad p	Books                                             	arts                                              	3.93	969.48	1.22595739827125692
+Aside legitimate decisions may not stand probably sexual g	Books                                             	arts                                              	3.88	349.20	0.44158138742039332
+Specially interesting crews continue current, foreign directions; only social men would not call at least political children; circumstances could not understand now in a assessme	Books                                             	arts                                              	2.13	3343.99	4.22864760515441312
+Unlikely states take later in general extra inf	Books                                             	arts                                              	0.32	20046.98	25.35043883731064290
+Inches may lose from a problems. Firm, other corporations shall protect ashamed, important practices. Materials shall not make then by a police. Weeks used	Books                                             	arts                                              	0.84	11869.78	15.00994822673206253
+Relevant lips take so sure, manufacturing 	Books                                             	arts                                              	8.80	5995.28	7.58134037907713537
+Extra, primitive weeks look obviou	Books                                             	arts                                              	1.18	425.89	0.53855984275049058
+More than key reasons should remain. Words used to offer slowly british	Books                                             	arts                                              	0.28	7814.52	9.88186306879843074
+Children may turn also above, historical aspects. Surveys migh	Books                                             	arts                                              	7.22	544.72	0.68882649872748182
+Trustees know operations. Now past issues cut today german governments. British lines go critical, individual structures. Tonight adequate problems should no	Books                                             	arts                                              	4.05	152.67	0.19305907908783347
+Floors could not go only for a years. Special reasons shape consequently black, concerned instances. Mutual depths encourage both simple teachers. Cards favour massive 	Books                                             	arts                                              	1.83	503.10	0.63619586486597904
+Certain customers think exactly already necessary factories. Awkward doubts shall not forget fine	Books                                             	arts                                              	0.30	922.40	1.16642231316314662
+Deep, big areas take for a facilities. Words could replace certainly cases; lights test. Nevertheless practical arts cross. Fa	Books                                             	arts                                              	7.37	230.48	0.29145383210954253
+New, reluctant associations see more different, physical symptoms; useful pounds ought to give. Subjects 	Books                                             	business                                          	9.02	306.85	0.37352072221391094
+Natural plans might not like n	Books                                             	business                                          	4.29	2484.54	3.02436752540117416
+Years shall want free objects. Old residents use absolutely so residential steps. Letters will share variables. Sure fres	Books                                             	business                                          	40.76	90.28	0.10989555418436330
+Simple, great shops glance from a years. Lessons deepen here previous clients. Increased, silent flights open more great rocks. Brill	Books                                             	business                                          	8.92	393.75	0.47930188812686144
+Groups must not put new, civil moves. Correct men laugh slightly total novels. Relatively public girls set even scott	Books                                             	business                                          	3.36	344.10	0.41886420242400767
+Just young degrees stop posts. More than tight artists buy to a arts. European, essential techniques ought to sell to a offences. Sentences be	Books                                             	business                                          	2.58	184.08	0.22407591508925118
+Junior, severe restrictions ought to want principles. Sure,	Books                                             	business                                          	9.77	1549.80	1.88653223166732663
+Remaining subjects handle even only certain ladies; eagerly literary days could not provide. Very different articles cut then. Boys see out of a houses. Governme	Books                                             	business                                          	9.03	6463.45	7.86779374936777799
+Russian windows should see in a weapons. New, considerable branches walk. English regions apply neither alone police; very new years w	Books                                             	business                                          	2.79	1635.60	1.99097439548011320
+Long groups used to create more tiny feet; tools used to dare still	Books                                             	business                                          	57.04	10558.62	12.85274032257534413
+Drugs must compensate dark, modest houses. Small pubs claim naturally accessible relationships. Distinguished	Books                                             	business                                          	1.66	31.78	0.03868498794837246
+Small, capable centres	Books                                             	business                                          	2.98	3219.72	3.91928349267255446
+Popular, different parameters might take open, used modules. Prisoners use pretty alternative lovers. Annual, professional others spend once true men. Other, small subsidies seem politically	Books                                             	business                                          	7.25	3862.88	4.70218584789203943
+Supreme, free uses handle even in the customers. Other minutes might not make of course social neighbours. So environmental rights come other, able sales	Books                                             	business                                          	8.08	10904.74	13.27406341976510738
+Sound, original activities consider quite to a attitudes. In order weak improvements marry available, hard studie	Books                                             	business                                          	71.27	385.84	0.46967324575204627
+Classic issues will draw as european, engl	Books                                             	business                                          	75.64	92.64	0.11276832232653319
+Again british shareholders see shares. American lives ought to establish horses. Then ideal conservatives might charge even nec	Books                                             	business                                          	2.44	5353.50	6.51667976657054660
+Departments could seek now for a commu	Books                                             	business                                          	5.93	6535.44	7.95542535045032467
+Paintings must not know primary, royal stands; similar, available others ough	Books                                             	business                                          	0.39	303.68	0.36966196161616580
+Most present eyes restore fat, central relationships; again considerable habits must face in a discussions. Engineers help at all direct occasions. Curiously del	Books                                             	business                                          	80.10	2096.55	2.55207713918062566
+Children would not mean in favour of a parts. Heavy, whole others shall mean on	Books                                             	business                                          	3.13	6646.96	8.09117581791421695
+White fees might combine reports. Tr	Books                                             	business                                          	2.09	500.56	0.60931899205277908
+Most new weeks go yet members. Also encouraging delegates make publications. Different competitors run resources; somehow common views m	Books                                             	business                                          	1.07	974.26	1.18594198736882801
+Only new systems might join late speeches. Materials could stay on a benefits. Corporate regulations must crawl definitely practical deaths. Windows might soothe despite a organisations. Old	Books                                             	business                                          	0.67	9075.35	11.04719337247520503
+Professional managers form later initial grounds. Conscious, big risks restore. American, full rises say for a systems. Already	Books                                             	business                                          	5.27	890.13	1.08353267219901759
+Memories can earn particularly over quick contexts; alone differences make separate years; irish men mea	Books                                             	business                                          	4.23	2059.92	2.50748836924516678
+Only, gothic	Books                                             	business                                          	1.68	4777.17	5.81512787530920297
+Silver, critical operations could help howev	Books                                             	business                                          	5.56	428.54	0.52165087273113702
+Else substantial problems slip months. Just unique corporations put vast areas. Supporters like far perfect chapters. Now young reports become wrong trials. Available ears shall	Books                                             	computers                                         	51.46	2456.28	1.26602601850774711
+At least remaining results shall keep cuts. Clients should meet policies. Glorious, local times could use enough; clever styles will live political parents. Single, gradual contracts will describe ho	Books                                             	computers                                         	9.51	10252.90	5.28459221471415324
+Years learn here. Days make too. Only moving systems avoid old groups; short movements cannot see respectiv	Books                                             	computers                                         	0.60	1761.68	0.90801240749618444
+Ga	Books                                             	computers                                         	5.53	7541.48	3.88706087988983530
+S	Books                                             	computers                                         	65.78	4566.02	2.35343695385979752
+Boxes batt	Books                                             	computers                                         	0.83	7844.04	4.04300760915510798
+Artists make times. Rather ready functions must pre	Books                                             	computers                                         	5.71	3694.01	1.90398194531071494
+Limited, capable cities shall try during the bodies. Specially economic services ought to prevent old area	Books                                             	computers                                         	2.93	96.18	0.04957349425150028
+Legs throw then. Old-fashioned develo	Books                                             	computers                                         	2.66	718.55	0.37035801928067716
+Important, educational variables used to appear months. A	Books                                             	computers                                         	2.47	9922.02	5.11404867366677942
+Men should not turn shadows. Different, single concessions guarantee only therefore alone products.	Books                                             	computers                                         	8.38	4194.24	2.16181256528813215
+Educational, white teachers should not fix. Considerable, other services might not cover today on a forms. Successful genes fall otherwise so	Books                                             	computers                                         	1.65	14569.68	7.50956485471198434
+Present 	Books                                             	computers                                         	2.84	12393.53	6.38792460190056468
+Multiple, dark feet mean more complex girls; schools may not answer frequently blue assets. Spiritual, dry patients may reply personnel	Books                                             	computers                                         	2.04	371.40	0.19142852739662305
+Private teachers ap	Books                                             	computers                                         	5.27	4911.39	2.53144899076602182
+Daily numbers sense interesting players. General advantages would speak here. Shelves shall know with the reductions. Again wrong mothers provide ways; as hot pr	Books                                             	computers                                         	7.56	689.26	0.35526124607807325
+Inc, corporate ships slow evident degrees. Chosen, acute prices throw always. Budgets spend points. Commonly large events may mean. Bottles c	Books                                             	computers                                         	68.38	4.17	0.00214931868401701
+However old hours ma	Books                                             	computers                                         	8.84	451.53	0.23272946412330966
+Indeed other actions should provide after a ideas; exhibitio	Books                                             	computers                                         	6.95	8062.32	4.15551439149257400
+Perfect days find at all. Crimes might develop hopes. Much socialist grants drive current, useful walls. Emissions open naturally. Combinations shall not know. Tragic things shall not receive just	Books                                             	computers                                         	6.71	526.49	0.27136565802113105
+Human windows take right, variable steps. Years should buy often. Indeed thin figures may beat even up to a cars. Details may tell enough. Impossible, sufficient differences ought to return 	Books                                             	computers                                         	4.47	1542.60	0.79509328584283986
+Left diff	Books                                             	computers                                         	0.74	5248.81	2.70536340572070289
+Friendly, hot computers tax elsewhere units. New, real officials should l	Books                                             	computers                                         	3.19	12378.72	6.38029117031536278
+Kinds relieve really major practices. Then capable reserves could not approve foundations. Pos	Books                                             	computers                                         	7.23	1786.41	0.92075884659828053
+Visible, average words shall not want quite only public boundaries. Cars must telephone proposals. German things ask abilities. Windows cut again favorite offi	Books                                             	computers                                         	6.75	25255.89	13.01749550563031296
+Only increased errors must submit as rich, main 	Books                                             	computers                                         	6.94	2429.79	1.25237243291071818
+Meals ought to test. Round days might need most urban years. Political, english pages must see on a eyes. Only subsequent women may come better methods; difficult, social childr	Books                                             	computers                                         	7.23	6457.72	3.32846480866914548
+Systems cannot see fairly practitioners. Little ca	Books                                             	computers                                         	1.73	6197.59	3.19438752586978211
+Past beautiful others might not like more than legislative, small products. Close, wh	Books                                             	computers                                         	3.02	10232.30	5.27397447733028024
+Main problems wait properly. Everyday, foreign offenders can worry activities. Social, important shoes will afford okay physical parts. Very	Books                                             	computers                                         	1.40	2034.30	1.04852733786470188
+Schools offer quickly others. Further main buildings satisfy sadly great, productive figures. Years contribute acti	Books                                             	computers                                         	4.11	3975.90	2.04927485750197523
+Tiny, rare leaders mention old, precious areas; students will improve much multiple stars. Even confident solutions will include clearly single women. Please little rights will not mention harder com	Books                                             	computers                                         	1.45	11902.91	6.13504720795513872
+Guidelines should investigate so. Usual personnel look now old, modern aspects. Discussions could appear once br	Books                                             	computers                                         	2.44	640.06	0.32990237815154161
+Flat pleasant groups would go private, redundant eyes. Main devic	Books                                             	computers                                         	2.83	8359.41	4.30864175068552700
+Fine users go for a networks. Sympathetic, effective industries could not alter particularly other concepts; wooden women used to feel however 	Books                                             	computers                                         	5.30	247.79	0.12771694885193653
+Real, domestic facilities turn often guilty symptoms. Winds get naturally intense islands. Products shall not travel a little clear shares; improved children may not apply wrong c	Books                                             	computers                                         	5.28	297.60	0.15339022550682558
+Then irish champions must look now forward good women. Future, big models sign. Then different o	Books                                             	cooking                                           	85.81	6496.48	10.66582432143788856
+Valuable studies should persist so concerned parties. Always polite songs include then from the holes. There conventional areas might not explain theore	Books                                             	cooking                                           	1.58	2088.03	3.42809662430915734
+Meanings occur in a things. Also essential features may not satisfy by the potatoes; happy words change childre	Books                                             	cooking                                           	3.46	1496.40	2.45676728237440221
+Then dominant goods should combine probably american items. Important artists guess only sill	Books                                             	cooking                                           	6.67	5638.06	9.25648312220250073
+Individual, remarkable services take by the interest	Books                                             	cooking                                           	6.05	0.00	0.00000000000000000
+Ultimately senior elections marry at l	Books                                             	cooking                                           	5.06	2060.48	3.38286544372280691
+Hence young effects shall not solve however months. In order small activities must not return almost national foods. International decades take contributions. Sessions must see 	Books                                             	cooking                                           	1.43	242.28	0.39777170353760369
+Points trace so simple eyes. Short advisers shall not say limitations. Keys stretch in full now blue wings. Immediately strategic students would not make strangely for the players.	Books                                             	cooking                                           	1.69	12518.00	20.55186637313737424
+Great pp. will not r	Books                                             	cooking                                           	1.91	7268.22	11.93285558480304571
+Products may not resist further specif	Books                                             	cooking                                           	5.37	2.72	0.00446565557876128
+Somet	Books                                             	cooking                                           	7.34	580.58	0.95318761614603744
+Genetic properties might describe therefore leaves; right other organisers must not talk even lives; methods carry thus wrong minutes. Proud worke	Books                                             	cooking                                           	1.08	1445.15	2.37262579398781566
+Urgent agencies mean over as a plants; then	Books                                             	cooking                                           	6.47	1312.79	2.15531911295662354
+As light hundreds must establish on a services. Sometimes special results 	Books                                             	cooking                                           	44.82	3513.30	5.76808372972867366
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q13.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q13.sql.out
new file mode 100644
index 0000000000000..518f598f30aa6
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q13.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<avg(ss_quantity):double,avg(ss_ext_sales_price):decimal(11,6),avg(ss_ext_wholesale_cost):decimal(11,6),sum(ss_ext_wholesale_cost):decimal(17,2)>
+-- !query output
+31.875	2306.480000	2168.643750	17349.15
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q14a.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q14a.sql.out
new file mode 100644
index 0000000000000..ae4502a8cd93c
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q14a.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<channel:string,i_brand_id:int,i_class_id:int,i_category_id:int,sum(sales):decimal(38,2),sum(number_sales):bigint>
+-- !query output
+NULL	NULL	NULL	NULL	674681657.38	155667
+catalog	NULL	NULL	NULL	235666845.52	46085
+catalog	1001001	NULL	NULL	1698135.15	341
+catalog	1001001	1	NULL	1050210.86	214
+catalog	1001001	1	1	42507.62	8
+catalog	1001001	1	2	106083.96	19
+catalog	1001001	1	3	101297.49	22
+catalog	1001001	1	4	68779.51	25
+catalog	1001001	1	5	201671.76	31
+catalog	1001001	1	6	165473.64	25
+catalog	1001001	1	7	57452.17	9
+catalog	1001001	1	8	107124.01	25
+catalog	1001001	1	9	61121.32	18
+catalog	1001001	1	10	138699.38	32
+catalog	1001001	2	NULL	34937.77	4
+catalog	1001001	2	3	34937.77	4
+catalog	1001001	3	NULL	208126.74	43
+catalog	1001001	3	1	65982.89	13
+catalog	1001001	3	4	108943.96	20
+catalog	1001001	3	6	23652.37	7
+catalog	1001001	3	7	9547.52	3
+catalog	1001001	4	NULL	20486.85	5
+catalog	1001001	4	7	20486.85	5
+catalog	1001001	5	NULL	41532.27	8
+catalog	1001001	5	7	11217.24	4
+catalog	1001001	5	9	30315.03	4
+catalog	1001001	8	NULL	139099.45	25
+catalog	1001001	8	9	77814.57	15
+catalog	1001001	8	10	61284.88	10
+catalog	1001001	10	NULL	24174.96	5
+catalog	1001001	10	7	24174.96	5
+catalog	1001001	13	NULL	20891.29	3
+catalog	1001001	13	9	20891.29	3
+catalog	1001001	14	NULL	66038.13	14
+catalog	1001001	14	9	26363.83	7
+catalog	1001001	14	10	39674.30	7
+catalog	1001001	15	NULL	63699.54	12
+catalog	1001001	15	10	63699.54	12
+catalog	1001001	16	NULL	28937.29	8
+catalog	1001001	16	9	28937.29	8
+catalog	1001002	NULL	NULL	3137545.73	657
+catalog	1001002	1	NULL	2597194.10	539
+catalog	1001002	1	1	2597194.10	539
+catalog	1001002	2	NULL	120152.76	28
+catalog	1001002	2	1	120152.76	28
+catalog	1001002	3	NULL	43520.49	11
+catalog	1001002	3	1	43520.49	11
+catalog	1001002	4	NULL	138778.24	30
+catalog	1001002	4	1	138778.24	30
+catalog	1001002	7	NULL	26096.94	4
+catalog	1001002	7	1	26096.94	4
+catalog	1001002	10	NULL	15192.85	6
+catalog	1001002	10	1	15192.85	6
+catalog	1001002	12	NULL	79002.90	17
+catalog	1001002	12	1	79002.90	17
+catalog	1001002	13	NULL	28858.71	5
+catalog	1001002	13	1	28858.71	5
+catalog	1001002	14	NULL	88748.74	17
+catalog	1001002	14	1	88748.74	17
+catalog	1002001	NULL	NULL	1812816.55	357
+catalog	1002001	1	NULL	158520.06	30
+catalog	1002001	1	1	25188.03	6
+catalog	1002001	1	2	77805.61	11
+catalog	1002001	1	4	15320.55	6
+catalog	1002001	1	5	40205.87	7
+catalog	1002001	2	NULL	944972.06	193
+catalog	1002001	2	1	69392.20	14
+catalog	1002001	2	2	143259.76	24
+catalog	1002001	2	3	190639.48	38
+catalog	1002001	2	4	60154.92	11
+catalog	1002001	2	6	72905.53	16
+catalog	1002001	2	7	17237.62	3
+catalog	1002001	2	8	141091.32	36
+catalog	1002001	2	9	141603.60	25
+catalog	1002001	2	10	108687.63	26
+catalog	1002001	3	NULL	61348.83	9
+catalog	1002001	3	1	61348.83	9
+catalog	1002001	4	NULL	126215.12	28
+catalog	1002001	4	1	59437.19	12
+catalog	1002001	4	3	24792.24	9
+catalog	1002001	4	5	41985.69	7
+catalog	1002001	5	NULL	60039.55	7
+catalog	1002001	5	9	60039.55	7
+catalog	1002001	6	NULL	160520.59	27
+catalog	1002001	6	7	10403.70	5
+catalog	1002001	6	9	100458.19	15
+catalog	1002001	6	10	49658.70	7
+catalog	1002001	8	NULL	18830.90	5
+catalog	1002001	8	8	18830.90	5
+catalog	1002001	9	NULL	31370.49	5
+catalog	1002001	9	9	31370.49	5
+catalog	1002001	10	NULL	40164.96	9
+catalog	1002001	10	6	7759.96	5
+catalog	1002001	10	8	32405.00	4
+catalog	1002001	11	NULL	42852.27	11
+catalog	1002001	11	8	25836.23	7
+catalog	1002001	11	9	17016.04	4
+catalog	1002001	12	NULL	12275.20	4
+catalog	1002001	12	7	12275.20	4
+catalog	1002001	13	NULL	27702.27	5
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q14b.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q14b.sql.out
new file mode 100644
index 0000000000000..cf254c3d32fdf
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q14b.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<channel:string,i_brand_id:int,i_class_id:int,i_category_id:int,sales:decimal(28,2),number_sales:bigint,channel:string,i_brand_id:int,i_class_id:int,i_category_id:int,sales:decimal(28,2),number_sales:bigint>
+-- !query output
+store	1001001	1	1	618811.76	171	store	1001001	1	1	1120886.77	328
+store	1001002	1	1	679257.16	207	store	1001002	1	1	597317.93	174
+store	1002001	2	1	590902.05	166	store	1002001	2	1	1382457.19	367
+store	1002002	2	1	712644.78	166	store	1002002	2	1	686649.72	155
+store	1003001	3	1	567349.84	161	store	1003001	3	1	1120432.56	309
+store	1003002	3	1	775782.67	214	store	1003002	3	1	561917.26	173
+store	1004001	4	1	602670.66	187	store	1004001	4	1	1284658.68	355
+store	1004002	4	1	680693.19	185	store	1004002	4	1	572656.37	181
+store	2001001	1	2	721151.24	193	store	2001001	1	2	1206198.64	359
+store	2001002	1	2	815659.11	223	store	2001002	1	2	621816.55	166
+store	2002001	2	2	657826.11	158	store	2002001	2	2	1229089.24	366
+store	2002002	2	2	738888.97	213	store	2002002	2	2	701020.25	187
+store	2003001	3	2	856103.36	229	store	2003001	3	2	1517919.93	408
+store	2003002	3	2	1035024.58	255	store	2003002	3	2	693873.11	187
+store	2004001	4	2	818535.48	210	store	2004001	4	2	1584158.70	423
+store	2004002	4	2	902066.26	231	store	2004002	4	2	652868.55	184
+store	3001001	1	3	690099.22	187	store	3001001	1	3	1540182.88	363
+store	3001002	1	3	796955.01	238	store	3001002	1	3	690610.81	178
+store	3002001	2	3	601571.31	163	store	3002001	2	3	1332847.53	353
+store	3002002	2	3	889691.45	239	store	3002002	2	3	711790.52	187
+store	3003001	3	3	599641.77	168	store	3003001	3	3	1218300.45	329
+store	3003002	3	3	785846.50	222	store	3003002	3	3	727813.84	205
+store	3004001	4	3	636017.18	169	store	3004001	4	3	1300058.21	347
+store	3004002	4	3	850298.30	240	store	3004002	4	3	605722.91	192
+store	4001001	1	4	709294.08	189	store	4001001	1	4	1317772.16	332
+store	4001002	1	4	833503.02	203	store	4001002	1	4	651619.28	161
+store	4002001	2	4	633353.59	188	store	4002001	2	4	1235025.47	341
+store	4002002	2	4	928549.29	239	store	4002002	2	4	724613.68	193
+store	4003001	3	4	645858.10	173	store	4003001	3	4	1417068.83	390
+store	4003002	3	4	872089.82	236	store	4003002	3	4	707566.93	201
+store	4004001	4	4	657080.07	179	store	4004001	4	4	1279798.81	334
+store	4004002	4	4	836177.44	216	store	4004002	4	4	821961.41	198
+store	5001001	1	5	719353.50	186	store	5001001	1	5	1189108.84	310
+store	5001002	1	5	842595.01	234	store	5001002	1	5	788955.51	198
+store	5002001	2	5	773460.67	218	store	5002001	2	5	1274443.41	357
+store	5002002	2	5	910604.00	253	store	5002002	2	5	649585.02	178
+store	5003001	3	5	699927.22	177	store	5003001	3	5	1388232.27	389
+store	5003002	3	5	795797.08	221	store	5003002	3	5	640802.27	168
+store	5004001	4	5	746611.79	186	store	5004001	4	5	1149863.23	334
+store	5004002	4	5	661403.11	175	store	5004002	4	5	522457.04	139
+store	6001001	1	6	32952.14	10	store	6001001	1	6	100674.40	25
+store	6001002	1	6	32363.63	11	store	6001002	1	6	21192.44	6
+store	6001003	1	6	32105.72	12	store	6001003	1	6	49799.29	16
+store	6001004	1	6	69366.44	18	store	6001004	1	6	33159.67	12
+store	6001005	1	6	39316.28	13	store	6001005	1	6	176957.80	40
+store	6001006	1	6	33851.16	8	store	6001006	1	6	59928.72	18
+store	6001007	1	6	52934.26	10	store	6001007	1	6	68730.18	15
+store	6001008	1	6	61470.12	17	store	6001008	1	6	21110.81	7
+store	6002001	2	6	58059.46	12	store	6002001	2	6	118560.37	25
+store	6002002	2	6	45696.16	15	store	6002002	2	6	98188.35	25
+store	6002003	2	6	27662.18	10	store	6002003	2	6	52033.85	19
+store	6002004	2	6	49212.09	11	store	6002004	2	6	23087.83	7
+store	6002005	2	6	54618.54	15	store	6002005	2	6	94438.29	22
+store	6002006	2	6	4768.74	6	store	6002006	2	6	41696.39	7
+store	6002007	2	6	23765.55	6	store	6002007	2	6	100119.20	29
+store	6002008	2	6	26747.76	5	store	6002008	2	6	19965.51	3
+store	6003001	3	6	17874.97	4	store	6003001	3	6	39734.27	10
+store	6003004	3	6	36845.45	13	store	6003004	3	6	28332.73	5
+store	6003005	3	6	33876.00	12	store	6003005	3	6	149594.43	34
+store	6003006	3	6	78594.03	17	store	6003006	3	6	80057.92	16
+store	6003007	3	6	90969.04	23	store	6003007	3	6	130880.43	32
+store	6003008	3	6	51335.82	14	store	6003008	3	6	45068.89	16
+store	6004001	4	6	14673.04	4	store	6004001	4	6	84488.05	22
+store	6004002	4	6	41801.05	15	store	6004002	4	6	31527.39	12
+store	6004003	4	6	15735.99	8	store	6004003	4	6	102005.55	23
+store	6004004	4	6	70293.74	15	store	6004004	4	6	16694.72	7
+store	6004006	4	6	98506.28	26	store	6004006	4	6	40380.88	13
+store	6004007	4	6	7974.20	5	store	6004007	4	6	56390.64	19
+store	6004008	4	6	93726.06	22	store	6004008	4	6	49454.40	15
+store	6005001	5	6	25935.02	8	store	6005001	5	6	76459.38	18
+store	6005002	5	6	27560.91	8	store	6005002	5	6	29874.66	8
+store	6005003	5	6	76424.40	23	store	6005003	5	6	120380.82	37
+store	6005004	5	6	61026.67	15	store	6005004	5	6	32867.59	7
+store	6005005	5	6	49398.50	13	store	6005005	5	6	125117.31	33
+store	6005006	5	6	40539.82	9	store	6005006	5	6	24310.11	8
+store	6005007	5	6	18454.21	5	store	6005007	5	6	52454.65	14
+store	6005008	5	6	71116.83	19	store	6005008	5	6	40388.90	6
+store	6006001	6	6	26990.02	8	store	6006001	6	6	19931.96	5
+store	6006003	6	6	39807.40	6	store	6006003	6	6	100194.33	22
+store	6006004	6	6	126180.26	29	store	6006004	6	6	44060.41	16
+store	6006005	6	6	24214.85	8	store	6006005	6	6	79947.56	24
+store	6006006	6	6	59581.58	21	store	6006006	6	6	45979.99	11
+store	6006008	6	6	81635.42	14	store	6006008	6	6	26221.85	8
+store	6007001	7	6	15407.87	3	store	6007001	7	6	14185.50	7
+store	6007002	7	6	77223.28	17	store	6007002	7	6	21012.93	7
+store	6007003	7	6	30617.75	9	store	6007003	7	6	21503.75	7
+store	6007004	7	6	35752.37	11	store	6007004	7	6	39378.54	9
+store	6007005	7	6	49598.38	16	store	6007005	7	6	102247.83	29
+store	6007006	7	6	43099.21	12	store	6007006	7	6	17599.96	11
+store	6007007	7	6	36750.53	10	store	6007007	7	6	47280.71	19
+store	6007008	7	6	63116.82	21	store	6007008	7	6	16366.95	10
+store	6008001	8	6	78930.46	18	store	6008001	8	6	123392.71	36
+store	6008002	8	6	20148.09	5	store	6008002	8	6	98780.67	17
+store	6008004	8	6	41158.44	15	store	6008004	8	6	7371.54	4
+store	6008005	8	6	52094.82	11	store	6008005	8	6	153697.84	42
+store	6008006	8	6	40340.12	8	store	6008006	8	6	32564.87	11
+store	6008007	8	6	8554.42	2	store	6008007	8	6	40138.79	9
+store	6008008	8	6	26928.47	7	store	6008008	8	6	18485.16	4
+store	6009001	9	6	69811.04	16	store	6009001	9	6	136769.36	32
+store	6009002	9	6	31689.61	14	store	6009002	9	6	63253.41	19
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q15.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q15.sql.out
new file mode 100644
index 0000000000000..7ec445691906f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q15.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<ca_zip:string,sum(cs_sales_price):decimal(17,2)>
+-- !query output
+NULL	2519.19
+30059     	812.16
+30069     	2248.42
+30150     	157.03
+30162     	2039.97
+30169     	1670.46
+30191     	544.76
+30399     	638.05
+30411     	771.09
+30499     	1826.69
+30587     	3142.51
+30945     	159.37
+30965     	331.74
+31087     	1264.39
+31218     	1221.13
+31289     	366.49
+31387     	1650.86
+31521     	490.31
+31675     	1909.58
+31711     	458.28
+31749     	1229.27
+31757     	1207.80
+31904     	1584.89
+31933     	829.70
+31952     	794.71
+32293     	1025.43
+32477     	95.37
+32808     	1306.54
+32812     	209.90
+32898     	448.94
+33003     	236.71
+33199     	385.03
+33372     	170.95
+33394     	842.14
+33445     	1084.78
+33511     	447.67
+33604     	616.64
+33951     	2103.68
+33957     	319.38
+34098     	288.86
+34107     	2903.74
+34167     	717.59
+34244     	694.54
+34289     	185.70
+34338     	411.46
+34466     	1736.55
+34536     	2259.62
+34593     	346.48
+34694     	592.38
+34843     	461.27
+34854     	448.63
+35038     	1972.67
+35124     	415.20
+35281     	538.84
+35709     	1008.27
+35752     	1067.71
+35804     	1004.19
+35817     	418.39
+36060     	659.72
+36074     	509.22
+36098     	2175.73
+36192     	1679.14
+36277     	554.20
+36534     	982.64
+36557     	437.46
+36787     	2030.37
+36788     	357.97
+36867     	649.86
+36871     	551.07
+36971     	473.15
+36997     	953.02
+37057     	832.78
+37411     	447.31
+37683     	1675.77
+37745     	689.08
+37752     	871.12
+37838     	1238.05
+37940     	257.25
+38014     	1047.08
+38059     	374.46
+38075     	1052.41
+38095     	935.02
+38354     	310.51
+38370     	2677.66
+38371     	1890.06
+38482     	1035.92
+38579     	1957.82
+38605     	652.30
+38721     	1593.98
+38784     	739.06
+38828     	832.91
+38877     	669.68
+38883     	1743.88
+39003     	400.20
+39101     	481.37
+39145     	633.33
+39231     	576.23
+39237     	895.30
+39275     	622.01
+39303     	825.38
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q16.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q16.sql.out
new file mode 100644
index 0000000000000..6de87c375ca54
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q16.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<order count :bigint,total shipping cost :decimal(17,2),total net profit :decimal(17,2)>
+-- !query output
+280	1300478.25	-285761.26
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q17.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q17.sql.out
new file mode 100644
index 0000000000000..a3facf3ecb9ec
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q17.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_item_id:string,i_item_desc:string,s_state:string,store_sales_quantitycount:bigint,store_sales_quantityave:double,store_sales_quantitystdev:double,store_sales_quantitycov:double,as_store_returns_quantitycount:bigint,as_store_returns_quantityave:double,as_store_returns_quantitystdev:double,store_returns_quantitycov:double,catalog_sales_quantitycount:bigint,catalog_sales_quantityave:double,catalog_sales_quantitystdev:double,catalog_sales_quantitycov:double>
+-- !query output
+
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q18.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q18.sql.out
new file mode 100644
index 0000000000000..cac227c379b81
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q18.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_item_id:string,ca_country:string,ca_state:string,ca_county:string,agg1:decimal(16,6),agg2:decimal(16,6),agg3:decimal(16,6),agg4:decimal(16,6),agg5:decimal(16,6),agg6:decimal(16,6),agg7:decimal(16,6)>
+-- !query output
+NULL	NULL	NULL	NULL	50.804533	99.251888	256.393484	49.082603	-338.277102	1957.826868	3.274912
+AAAAAAAAAACDAAAA	NULL	NULL	NULL	8.000000	204.490000	0.000000	47.030000	-299.760000	1948.000000	2.000000
+AAAAAAAAAADCAAAA	NULL	NULL	NULL	46.000000	178.690000	0.000000	146.520000	2502.860000	1962.000000	0.000000
+AAAAAAAAAAGBAAAA	NULL	NULL	NULL	86.000000	1.520000	40.290000	0.660000	-97.910000	1945.000000	2.000000
+AAAAAAAAAAGCAAAA	NULL	NULL	NULL	84.000000	41.920000	0.000000	8.380000	-640.080000	1945.000000	2.000000
+AAAAAAAAAAGEAAAA	NULL	NULL	NULL	99.000000	107.550000	0.000000	65.600000	738.540000	1973.000000	3.000000
+AAAAAAAAAAJBAAAA	NULL	NULL	NULL	97.000000	48.580000	0.000000	44.690000	534.470000	1984.000000	1.000000
+AAAAAAAAAAKAAAAA	NULL	NULL	NULL	36.000000	164.023333	0.000000	57.763333	-1073.803333	1962.666667	2.333333
+AAAAAAAAAAOCAAAA	NULL	NULL	NULL	82.000000	148.640000	0.000000	107.020000	4483.760000	1948.000000	6.000000
+AAAAAAAAAAODAAAA	NULL	NULL	NULL	100.000000	80.700000	0.000000	64.560000	1505.000000	1981.000000	4.000000
+AAAAAAAAABADAAAA	NULL	NULL	NULL	55.000000	134.120000	3189.310000	63.030000	-2198.210000	1945.000000	2.000000
+AAAAAAAAABCEAAAA	NULL	NULL	NULL	25.000000	150.360000	85.690000	9.020000	-1702.940000	1948.000000	6.000000
+AAAAAAAAABDEAAAA	NULL	NULL	NULL	24.000000	99.490000	0.000000	94.510000	562.560000	1958.000000	0.000000
+AAAAAAAAABFBAAAA	NULL	NULL	NULL	2.000000	60.810000	0.000000	45.600000	-26.880000	1931.000000	6.000000
+AAAAAAAAABFCAAAA	NULL	NULL	NULL	65.000000	45.000000	0.000000	23.310000	-1071.285000	1937.500000	3.000000
+AAAAAAAAABGBAAAA	NULL	NULL	NULL	65.000000	20.450000	0.000000	0.810000	-401.050000	1983.000000	2.000000
+AAAAAAAAABJBAAAA	NULL	NULL	NULL	42.000000	69.930000	0.000000	49.650000	260.820000	1943.000000	2.000000
+AAAAAAAAABNCAAAA	NULL	NULL	NULL	81.000000	140.170000	0.000000	86.900000	1733.400000	1953.000000	4.000000
+AAAAAAAAACCBAAAA	NULL	NULL	NULL	50.000000	99.750000	0.000000	75.810000	1109.000000	1945.000000	1.000000
+AAAAAAAAACCDAAAA	NULL	NULL	NULL	71.000000	92.520000	0.000000	25.900000	-4026.410000	1928.000000	0.000000
+AAAAAAAAACDDAAAA	NULL	NULL	NULL	29.000000	147.490000	0.000000	35.390000	-741.240000	1933.000000	5.000000
+AAAAAAAAACIBAAAA	NULL	NULL	NULL	4.000000	38.470000	0.000000	35.770000	41.160000	1952.000000	2.000000
+AAAAAAAAACPDAAAA	NULL	NULL	NULL	91.000000	86.510000	0.000000	20.760000	-1989.260000	1981.000000	3.000000
+AAAAAAAAADACAAAA	NULL	NULL	NULL	53.000000	227.940000	0.000000	113.970000	1089.150000	1954.000000	6.000000
+AAAAAAAAADBDAAAA	NULL	NULL	NULL	70.000000	35.050000	0.000000	29.440000	970.200000	1990.000000	2.000000
+AAAAAAAAADEEAAAA	NULL	NULL	NULL	66.000000	85.760000	2913.510000	44.590000	-3553.050000	1926.000000	3.000000
+AAAAAAAAADJBAAAA	NULL	NULL	NULL	28.000000	90.830000	0.000000	15.155000	-466.175000	1986.000000	1.000000
+AAAAAAAAADPBAAAA	NULL	NULL	NULL	55.000000	47.460000	0.000000	18.500000	43.450000	1989.000000	2.000000
+AAAAAAAAAEABAAAA	NULL	NULL	NULL	62.000000	89.450000	0.000000	61.720000	225.060000	1958.000000	0.000000
+AAAAAAAAAEBAAAAA	NULL	NULL	NULL	93.000000	175.980000	0.000000	156.620000	6503.490000	1952.000000	6.000000
+AAAAAAAAAECCAAAA	NULL	NULL	NULL	97.000000	179.380000	0.000000	127.350000	4183.610000	1952.000000	1.000000
+AAAAAAAAAEDDAAAA	NULL	NULL	NULL	81.000000	104.250000	0.000000	53.160000	-92.340000	1982.000000	0.000000
+AAAAAAAAAEGDAAAA	NULL	NULL	NULL	6.000000	170.770000	61.470000	102.460000	103.890000	1985.000000	0.000000
+AAAAAAAAAEGEAAAA	NULL	NULL	NULL	97.000000	34.300000	0.000000	24.350000	392.850000	1976.000000	6.000000
+AAAAAAAAAEJDAAAA	NULL	NULL	NULL	74.000000	168.330000	0.000000	143.080000	3926.440000	1930.000000	1.000000
+AAAAAAAAAENAAAAA	NULL	NULL	NULL	15.000000	73.420000	0.000000	23.490000	-108.450000	1963.000000	2.000000
+AAAAAAAAAEPAAAAA	NULL	NULL	NULL	34.000000	4.380000	0.000000	3.460000	67.320000	1952.000000	1.000000
+AAAAAAAAAEPDAAAA	NULL	NULL	NULL	54.000000	99.220000	1013.550000	43.650000	-3909.570000	1958.000000	0.000000
+AAAAAAAAAFGAAAAA	NULL	NULL	NULL	74.000000	90.960000	0.000000	0.900000	-4919.520000	1964.000000	6.000000
+AAAAAAAAAFGCAAAA	NULL	NULL	NULL	79.000000	51.830000	0.000000	36.280000	787.630000	1975.000000	5.000000
+AAAAAAAAAFGDAAAA	NULL	NULL	NULL	80.000000	38.080000	0.000000	0.000000	-1116.000000	1926.000000	0.000000
+AAAAAAAAAFHBAAAA	NULL	NULL	NULL	57.000000	55.990000	0.000000	31.910000	320.340000	1970.000000	6.000000
+AAAAAAAAAFLBAAAA	NULL	NULL	NULL	92.000000	44.330000	0.000000	41.220000	836.280000	1943.000000	3.000000
+AAAAAAAAAGBEAAAA	NULL	NULL	NULL	6.000000	260.570000	0.000000	93.800000	14.220000	1970.000000	2.000000
+AAAAAAAAAGCAAAAA	NULL	NULL	NULL	54.000000	231.920000	0.000000	48.700000	-2187.000000	1969.000000	5.000000
+AAAAAAAAAGCCAAAA	NULL	NULL	NULL	80.000000	183.680000	0.000000	99.180000	475.200000	1946.000000	0.000000
+AAAAAAAAAGCDAAAA	NULL	NULL	NULL	69.000000	143.220000	2478.280000	54.420000	-4676.620000	1970.000000	6.000000
+AAAAAAAAAGEDAAAA	NULL	NULL	NULL	93.000000	60.830000	0.000000	52.310000	2769.540000	1942.000000	6.000000
+AAAAAAAAAGIDAAAA	NULL	NULL	NULL	28.000000	75.550000	0.000000	9.060000	-1013.040000	1942.000000	6.000000
+AAAAAAAAAGJBAAAA	NULL	NULL	NULL	36.000000	7.170000	0.000000	5.230000	65.880000	1984.000000	6.000000
+AAAAAAAAAGKAAAAA	NULL	NULL	NULL	33.000000	95.840000	0.000000	4.790000	-1480.710000	1948.000000	6.000000
+AAAAAAAAAGLAAAAA	NULL	NULL	NULL	85.000000	143.410000	570.460000	74.570000	276.990000	1948.000000	6.000000
+AAAAAAAAAGLCAAAA	NULL	NULL	NULL	65.000000	13.730000	0.000000	3.980000	-176.800000	1976.000000	6.000000
+AAAAAAAAAHADAAAA	NULL	NULL	NULL	52.000000	33.050000	0.000000	10.900000	-1054.560000	1936.000000	1.000000
+AAAAAAAAAHBCAAAA	NULL	NULL	NULL	20.000000	187.740000	0.000000	31.910000	-994.400000	1947.000000	5.000000
+AAAAAAAAAHBDAAAA	NULL	NULL	NULL	61.000000	83.490000	0.000000	59.270000	1201.700000	1945.000000	2.000000
+AAAAAAAAAHCBAAAA	NULL	NULL	NULL	2.000000	136.960000	0.000000	0.000000	-95.780000	1970.000000	5.000000
+AAAAAAAAAHDEAAAA	NULL	NULL	NULL	15.000000	31.950000	232.130000	16.290000	-164.630000	1934.000000	2.000000
+AAAAAAAAAHGAAAAA	NULL	NULL	NULL	99.000000	100.770000	12.870000	1.000000	-9600.030000	1932.000000	4.000000
+AAAAAAAAAHHAAAAA	NULL	NULL	NULL	18.000000	109.980000	0.000000	24.190000	-634.680000	1933.000000	2.000000
+AAAAAAAAAHIBAAAA	NULL	NULL	NULL	86.000000	20.920000	0.000000	5.230000	-1332.140000	1990.000000	1.000000
+AAAAAAAAAHJDAAAA	NULL	NULL	NULL	53.000000	88.370000	0.000000	37.990000	-1308.570000	1948.000000	2.000000
+AAAAAAAAAHMDAAAA	NULL	NULL	NULL	68.000000	200.900000	546.440000	20.090000	-4993.640000	NULL	6.000000
+AAAAAAAAAHPDAAAA	NULL	NULL	NULL	6.000000	73.210000	60.840000	13.170000	-276.660000	1936.000000	6.000000
+AAAAAAAAAIACAAAA	NULL	NULL	NULL	1.000000	17.830000	0.000000	10.160000	1.940000	1935.000000	1.000000
+AAAAAAAAAIADAAAA	NULL	NULL	NULL	100.000000	22.020000	0.000000	15.630000	-131.000000	1982.000000	0.000000
+AAAAAAAAAIEBAAAA	NULL	NULL	NULL	4.000000	51.860000	0.000000	42.520000	-8.760000	1967.000000	4.000000
+AAAAAAAAAIEEAAAA	NULL	NULL	NULL	64.000000	18.810000	0.000000	15.610000	468.480000	1930.000000	1.000000
+AAAAAAAAAINBAAAA	NULL	NULL	NULL	6.000000	102.610000	2.950000	24.620000	-110.710000	1930.000000	2.000000
+AAAAAAAAAIODAAAA	NULL	NULL	NULL	81.000000	42.490000	0.000000	12.740000	-339.390000	1990.000000	1.000000
+AAAAAAAAAJBEAAAA	NULL	NULL	NULL	95.000000	118.220000	0.000000	15.360000	-3863.650000	1962.000000	0.000000
+AAAAAAAAAJDCAAAA	NULL	NULL	NULL	85.000000	190.480000	0.000000	125.710000	4732.800000	1989.000000	2.000000
+AAAAAAAAAJEAAAAA	NULL	NULL	NULL	36.000000	183.430000	0.000000	0.000000	-3318.480000	1942.000000	6.000000
+AAAAAAAAAJGCAAAA	NULL	NULL	NULL	43.500000	39.415000	619.885000	28.940000	-340.325000	1960.000000	5.000000
+AAAAAAAAAJIAAAAA	NULL	NULL	NULL	69.000000	118.330000	0.000000	61.530000	1117.110000	1970.000000	2.000000
+AAAAAAAAAJICAAAA	NULL	NULL	NULL	39.333333	102.063333	0.000000	24.780000	-848.446667	1962.666667	2.333333
+AAAAAAAAAJIDAAAA	NULL	NULL	NULL	75.000000	222.870000	6498.720000	180.520000	-42.720000	1935.000000	5.000000
+AAAAAAAAAJJBAAAA	NULL	NULL	NULL	42.000000	103.240000	0.000000	43.360000	-149.940000	1932.000000	4.000000
+AAAAAAAAAJNAAAAA	NULL	NULL	NULL	93.000000	14.470000	0.000000	13.740000	136.710000	1954.000000	0.000000
+AAAAAAAAAJOCAAAA	NULL	NULL	NULL	24.000000	88.660000	0.000000	2.650000	-1000.320000	1927.000000	2.000000
+AAAAAAAAAJPBAAAA	NULL	NULL	NULL	16.000000	159.510000	0.000000	121.220000	1085.920000	1983.000000	4.000000
+AAAAAAAAAKAEAAAA	NULL	NULL	NULL	5.000000	178.480000	0.000000	57.110000	-160.650000	1990.000000	2.000000
+AAAAAAAAAKFCAAAA	NULL	NULL	NULL	84.000000	284.260000	0.000000	247.300000	12760.440000	1984.000000	1.000000
+AAAAAAAAAKGDAAAA	NULL	NULL	NULL	26.000000	132.060000	0.000000	64.700000	-291.200000	1930.000000	2.000000
+AAAAAAAAAKHCAAAA	NULL	NULL	NULL	99.000000	97.000000	0.000000	19.400000	-3327.390000	1935.000000	1.000000
+AAAAAAAAALBEAAAA	NULL	NULL	NULL	61.000000	7.760000	0.000000	2.630000	-125.050000	1981.000000	0.000000
+AAAAAAAAALFBAAAA	NULL	NULL	NULL	58.000000	92.090000	0.000000	35.910000	-97.440000	1970.000000	3.000000
+AAAAAAAAALGCAAAA	NULL	NULL	NULL	77.000000	36.240000	0.000000	18.840000	-66.220000	1983.000000	4.000000
+AAAAAAAAALLBAAAA	NULL	NULL	NULL	82.000000	53.000000	0.000000	8.480000	-840.500000	1981.000000	4.000000
+AAAAAAAAALMDAAAA	NULL	NULL	NULL	86.000000	32.020000	802.660000	17.610000	-1020.240000	1963.000000	6.000000
+AAAAAAAAALOAAAAA	NULL	NULL	NULL	78.000000	91.520000	0.000000	73.210000	44.460000	1964.000000	6.000000
+AAAAAAAAAMBAAAAA	NULL	NULL	NULL	79.000000	93.970000	0.000000	93.970000	4924.070000	1948.000000	2.000000
+AAAAAAAAAMDBAAAA	NULL	NULL	NULL	15.000000	79.140000	0.000000	18.990000	-421.800000	1969.000000	2.000000
+AAAAAAAAAMEAAAAA	NULL	NULL	NULL	10.000000	164.570000	0.000000	80.630000	-123.500000	1934.000000	5.000000
+AAAAAAAAAMFDAAAA	NULL	NULL	NULL	2.000000	41.000000	0.000000	17.220000	-4.800000	1948.000000	6.000000
+AAAAAAAAAMGBAAAA	NULL	NULL	NULL	66.000000	12.410000	0.000000	3.590000	-140.580000	1990.000000	0.000000
+AAAAAAAAAMLDAAAA	NULL	NULL	NULL	70.000000	147.430000	297.020000	8.840000	-3444.920000	1943.000000	4.000000
+AAAAAAAAAMNAAAAA	NULL	NULL	NULL	41.000000	53.225000	0.000000	37.950000	121.915000	1960.000000	5.000000
+AAAAAAAAANABAAAA	NULL	NULL	NULL	94.000000	8.250000	30.830000	0.410000	-293.090000	1986.000000	6.000000
+AAAAAAAAANAEAAAA	NULL	NULL	NULL	5.000000	47.390000	0.000000	9.470000	-151.800000	1931.000000	2.000000
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q19.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q19.sql.out
new file mode 100644
index 0000000000000..29347b2d590bc
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q19.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<brand_id:int,brand:string,i_manufact_id:int,i_manufact:string,ext_price:decimal(17,2)>
+-- !query output
+10012006	importoamalgamalg #x                              	221	oughtableable                                     	56586.14
+6015008	scholarbrand #x                                   	66	callycally                                        	46201.70
+4001001	amalgedu pack #x                                  	950	barantin st                                       	43315.50
+3002002	importoexporti #x                                 	752	ableantiation                                     	42749.12
+2003001	exportiimporto #x                                 	586	callyeinganti                                     	41753.03
+6009007	maxicorp #x                                       	120	barableought                                      	41183.43
+8010005	univmaxi #x                                       	520	barableanti                                       	40133.14
+7005005	scholarbrand #x                                   	187	ationeingought                                    	37940.95
+10004015	edu packunivamalg #x                             	439	n stpriese                                        	36037.04
+9015005	scholarunivamalg #x                               	297	ationn stable                                     	34881.18
+5003001	exportischolar #x                                 	227	ationableable                                     	34528.86
+2002001	importoimporto #x                                 	536	callyprianti                                      	34412.64
+7007003	brandbrand #x                                     	759	n stantiation                                     	33533.84
+7016003	corpnameless #x                                   	271	oughtationable                                    	32722.68
+10003006	exportiunivamalg #x                               	105	antibarought                                      	31509.32
+3003001	exportiexporti #x                                 	178	eingationought                                    	30944.05
+2003001	exportiimporto #x                                 	269	n stcallyable                                     	30743.96
+7012007	importonameless #x                                	578	eingationanti                                     	30629.71
+9010011	univunivamalg #x                                 	27	ationable                                         	30165.17
+1001001	amalgamalg #x                                     	390	barn stpri                                        	29510.42
+1004002	edu packamalg #x                                  	86	callyeing                                         	28798.78
+8016010	corpmaxi #x                                      	745	antieseation                                      	28399.44
+3003002	exportiexporti #x                                 	52	ableanti                                          	28330.70
+7002007	importobrand #x                                   	261	oughtcallyable                                    	28076.07
+2002001	importoimporto #x                                 	364	esecallypri                                       	27831.19
+5001002	amalgscholar #x                                   	743	prieseation                                       	27622.14
+8006009	corpnameless #x                                   	148	eingeseought                                      	26685.21
+2002001	importoimporto #x                                 	68	eingcally                                         	26391.94
+8004005	edu packnameless #x                               	192	ablen stought                                     	26231.73
+1004001	edu packamalg #x                                  	583	prieinganti                                       	26107.88
+1001001	amalgamalg #x                                     	282	ableeingable                                      	26016.70
+8006009	corpnameless #x                                   	319	n stoughtpri                                      	25529.26
+6016003	corpbrand #x                                      	110	baroughtought                                     	25233.47
+9007003	brandmaxi #x                                      	34	esepri                                            	25164.92
+2004001	edu packimporto #x                                	5	anti                                              	25083.59
+5002001	importoscholar #x                                 	582	ableeinganti                                      	24752.75
+3001001	amalgexporti #x                                   	296	callyn stable                                     	24732.94
+7007007	brandbrand #x                                     	529	n stableanti                                      	24268.11
+8010009	univmaxi #x                                       	777	ationationation                                   	24160.84
+8007002	brandnameless #x                                  	192	ablen stought                                     	23590.40
+8014006	edu packmaxi #x                                   	4	ese                                               	23430.31
+6005005	scholarcorp #x                                    	129	n stableought                                     	23382.47
+8015001	scholarmaxi #x                                    	78	eingation                                         	23235.50
+6004007	edu packcorp #x                                   	158	eingantiought                                     	23188.37
+9016003	corpunivamalg #x                                  	304	esebarpri                                         	23156.77
+10010013	univamalgamalg #x                                	591	oughtn stanti                                     	23127.55
+2004001	edu packimporto #x                                	563	pricallyanti                                      	22985.88
+2003002	exportiimporto #x                                 	490	barn stese                                        	22045.36
+1003002	exportiamalg #x                                   	376	callyationpri                                     	21890.84
+1004001	edu packamalg #x                                  	39	n stpri                                           	21878.31
+1001001	amalgamalg #x                                     	760	barcallyation                                     	21856.16
+9014002	edu packunivamalg #x                              	78	eingation                                         	21725.66
+1004001	edu packamalg #x                                  	366	callycallypri                                     	21166.78
+1004001	edu packamalg #x                                  	513	prioughtanti                                      	21125.73
+1002001	importoamalg #x                                   	942	ableesen st                                       	21119.54
+5002002	importoscholar #x                                 	102	ablebarought                                      	21049.49
+5002001	importoscholar #x                                 	75	antiation                                         	20510.26
+8002004	importonameless #x                                	2	able                                              	20401.46
+8011002	amalgmaxi #x                                      	29	n stable                                          	20270.33
+5001001	amalgscholar #x                                   	127	ationableought                                    	19976.77
+1001001	amalgamalg #x                                     	522	ableableanti                                      	19773.14
+7003005	exportibrand #x                                   	45	antiese                                           	19662.73
+1002001	importoamalg #x                                   	361	oughtcallypri                                     	19619.15
+7007002	brandbrand #x                                     	410	baroughtese                                       	18996.04
+NULL	brandcorp #x                                      	NULL	NULL	18842.73
+9008008	namelessmaxi #x                                   	607	ationbarcally                                     	18650.79
+8015010	scholarmaxi #x                                   	410	baroughtese                                       	18465.35
+10008005	namelessunivamalg #x                              	349	n stesepri                                        	18229.28
+9001008	amalgmaxi #x                                      	840	bareseeing                                        	17824.97
+2004002	edu packimporto #x                                	314	eseoughtpri                                       	17523.17
+4003001	exportiedu pack #x                                	873	priationeing                                      	17462.90
+9007005	brandmaxi #x                                      	546	callyeseanti                                      	17405.98
+10013015	exportiamalgamalg #x                             	427	ationableese                                      	17058.19
+3001001	amalgexporti #x                                   	139	n stpriought                                      	16994.07
+8015002	scholarmaxi #x                                    	15	antiought                                         	16963.84
+4003002	exportiedu pack #x                                	664	esecallycally                                     	16936.98
+6005001	scholarcorp #x                                    	404	esebarese                                         	16500.58
+2003002	exportiimporto #x                                 	333	pripripri                                         	16133.68
+9006003	corpmaxi #x                                       	494	esen stese                                        	16104.95
+6009007	maxicorp #x                                       	468	eingcallyese                                      	16083.46
+4002002	importoedu pack #x                                	408	eingbarese                                        	15980.04
+8012009	importomaxi #x                                    	351	oughtantipri                                      	15946.21
+4001002	amalgedu pack #x                                  	102	ablebarought                                      	15911.13
+6011007	amalgbrand #x                                     	822	ableableeing                                      	15556.11
+4004002	edu packedu pack #x                               	51	oughtanti                                         	15506.23
+4003002	exportiedu pack #x                                	12	ableought                                         	15452.21
+10010010	univamalgamalg #x                                	721	oughtableation                                    	15210.55
+3001001	amalgexporti #x                                   	574	eseationanti                                      	14927.08
+3002002	importoexporti #x                                 	382	ableeingpri                                       	14658.43
+3001001	amalgexporti #x                                   	142	ableeseought                                      	14633.69
+10002014	importounivamalg #x                              	153	priantiought                                      	14599.02
+1004001	edu packamalg #x                                  	148	eingeseought                                      	14327.50
+5003002	exportischolar #x                                 	164	esecallyought                                     	14289.22
+7008009	namelessbrand #x                                  	532	ableprianti                                       	14197.70
+4003001	exportiedu pack #x                                	258	eingantiable                                      	14148.81
+7007001	brandbrand #x                                     	281	oughteingable                                     	13920.21
+2002001	importoimporto #x                                 	258	eingantiable                                      	13819.06
+8007007	brandnameless #x                                  	423	priableese                                        	13589.86
+4004001	edu packedu pack #x                               	456	callyantiese                                      	13461.44
+8013009	exportimaxi #x                                    	599	n stn stanti                                      	13459.83
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q2.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q2.sql.out
new file mode 100644
index 0000000000000..44e1f7bfef7fa
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q2.sql.out
@@ -0,0 +1,2518 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<d_week_seq1:int,round((sun_sales1 / sun_sales2), 2):decimal(37,2),round((mon_sales1 / mon_sales2), 2):decimal(37,2),round((tue_sales1 / tue_sales2), 2):decimal(37,2),round((wed_sales1 / wed_sales2), 2):decimal(37,2),round((thu_sales1 / thu_sales2), 2):decimal(37,2),round((fri_sales1 / fri_sales2), 2):decimal(37,2),round((sat_sales1 / sat_sales2), 2):decimal(37,2)>
+-- !query output
+5270	3.18	1.63	2.25	1.64	3.41	3.62	3.72
+5270	3.18	1.63	2.25	1.64	3.41	3.62	3.72
+5270	3.18	1.63	2.25	1.64	3.41	3.62	3.72
+5270	3.18	1.63	2.25	1.64	3.41	3.62	3.72
+5270	3.18	1.63	2.25	1.64	3.41	3.62	3.72
+5270	3.18	1.63	2.25	1.64	3.41	3.62	3.72
+5270	3.18	1.63	2.25	1.64	3.41	3.62	3.72
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5271	1.00	1.15	1.23	0.82	1.06	0.85	0.95
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5272	1.22	0.86	1.29	0.95	0.98	1.00	0.84
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5273	1.20	0.99	1.15	0.86	0.91	1.19	0.90
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5274	0.97	0.95	1.08	1.19	0.97	0.89	0.96
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5275	0.99	0.89	1.42	1.00	0.84	1.11	0.98
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5276	0.97	1.06	1.16	0.86	0.98	1.25	1.13
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5277	0.91	0.91	1.02	0.95	1.00	1.05	0.90
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5278	0.96	0.94	1.29	1.04	0.95	2.06	1.46
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5279	0.87	0.86	0.88	0.99	0.38	0.83	0.77
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5280	0.96	0.89	0.73	1.09	1.09	0.91	1.21
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5281	0.99	1.03	1.09	0.86	0.92	1.13	1.01
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5282	0.97	0.89	1.08	1.10	1.15	0.86	1.33
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5283	1.27	0.85	1.07	0.88	1.23	0.93	1.27
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5284	1.06	0.92	0.95	1.04	0.95	1.29	0.89
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5285	1.10	1.20	0.97	1.14	1.11	1.12	0.95
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5286	1.09	1.17	1.17	0.86	1.12	1.39	1.00
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5287	0.94	1.00	0.99	0.76	1.10	0.90	0.96
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5288	0.95	1.07	1.40	1.05	0.94	0.94	1.01
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5289	0.99	1.20	0.90	0.95	1.01	0.82	0.76
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5290	1.05	1.10	1.04	1.22	0.97	0.98	0.98
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5291	0.78	1.28	0.90	0.99	0.97	1.09	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5292	1.13	0.79	1.17	1.07	0.88	0.85	0.91
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5293	1.29	0.96	0.89	0.87	0.97	0.89	1.06
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5294	1.18	1.05	1.25	0.98	0.88	1.02	0.69
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5295	0.87	1.08	1.03	1.01	0.88	1.27	1.12
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5296	0.74	1.11	0.96	1.12	1.00	1.25	1.02
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5297	1.04	0.97	0.98	0.79	0.90	0.90	1.03
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5298	0.80	0.78	0.94	1.04	1.26	0.91	0.90
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5299	0.87	1.00	1.14	1.13	1.07	1.20	1.01
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5300	0.53	0.41	0.98	0.67	0.80	0.56	0.41
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5301	0.95	1.05	0.38	0.50	0.76	1.10	1.14
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5302	1.03	0.80	0.85	1.12	1.00	0.96	1.01
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5303	0.77	1.23	0.94	1.09	0.98	0.96	0.98
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5304	0.96	0.82	1.13	1.04	1.07	1.24	1.00
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5305	1.01	0.96	0.91	0.91	0.90	1.15	1.09
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5306	0.83	0.99	0.97	1.04	0.90	1.00	1.03
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5307	0.97	0.99	0.89	0.88	1.07	1.02	0.94
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5308	0.93	0.95	1.10	0.97	1.10	0.89	1.04
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5309	1.20	0.90	1.05	1.11	0.96	1.05	1.06
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5310	0.97	1.10	1.12	1.06	0.96	1.03	0.87
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5311	1.09	0.88	1.06	1.01	0.91	1.10	1.03
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5312	1.06	0.93	0.78	0.99	1.03	0.87	1.07
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5313	0.62	0.78	1.13	1.03	1.13	1.04	0.85
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5314	0.98	1.16	0.71	0.57	0.70	1.07	1.06
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5315	1.10	1.09	0.99	1.02	1.11	1.00	1.02
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5316	1.01	1.07	1.09	1.01	1.08	1.03	0.87
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5317	0.96	1.04	0.94	0.99	1.09	0.95	0.88
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5318	0.94	1.00	1.18	0.94	0.87	1.02	1.11
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5319	0.97	0.96	0.96	1.12	1.00	0.93	0.88
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5320	1.06	0.94	0.97	1.14	0.96	0.90	1.04
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5321	0.96	1.01	0.96	1.03	0.97	0.95	0.89
+5322	4.54	4.72	0.99	1.74	1.43	4.36	5.11
+5322	4.54	4.72	0.99	1.74	1.43	4.36	5.11
+5322	4.54	4.72	0.99	1.74	1.43	4.36	5.11
+5322	4.54	4.72	0.99	1.74	1.43	4.36	5.11
+5322	4.54	4.72	0.99	1.74	1.43	4.36	5.11
+5322	4.54	4.72	0.99	1.74	1.43	4.36	5.11
+5322	4.54	4.72	0.99	1.74	1.43	4.36	5.11
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q20.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q20.sql.out
new file mode 100644
index 0000000000000..3c2757f0645f7
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q20.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_item_desc:string,i_category:string,i_class:string,i_current_price:decimal(7,2),itemrevenue:decimal(17,2),revenueratio:decimal(38,17)>
+-- !query output
+NULL	Books                                             	NULL	NULL	9010.49	72.07596281370536693
+NULL	Books                                             	NULL	6.35	1491.96	11.93436244638591899
+Precisely elderly bodies	Books                                             	arts                                              	1.40	4094.31	1.52075020571659240
+Close, precise teeth should go for a qualities. Political groups shall not become just important occasions. Trials mean ne	Books                                             	arts                                              	2.53	332.38	0.12345595555199313
+Abilities could affect cruel parts. Predominantly other events telephone strong signs. Accurate mate	Books                                             	arts                                              	25.69	2626.56	0.97558359291967949
+Average parents require also times. Children would not describe lightly purposes; large miles love now correct relations. Usual, german goals proceed literary, wooden visitors. Initial councils wil	Books                                             	arts                                              	1.24	12327.20	4.57869383019594946
+Great, contemporary workers would not remove of course cultural values. Then due children might see positive seconds. Significant problems w	Books                                             	arts                                              	0.55	343.80	0.12769768794384511
+Small objects stop etc mediterranean patterns; liberal, free initiatives would not leave less clear british attitudes; good, blue relationships find softly very	Books                                             	arts                                              	58.41	886.92	0.32942883476194038
+Newly national rights head curiously all electrical cells. Chinese, long values might not pull bad lines. High fun clothes ough	Books                                             	arts                                              	3.28	2219.85	0.82451923380495801
+Quick, easy studies must make always necessary systems. Upper, new persons should buy much physical technologies. English sciences hear solicitors. S	Books                                             	arts                                              	0.99	2050.16	0.76149125047979491
+Early, short v	Books                                             	arts                                              	75.57	5429.86	2.01681375177070042
+Black, following services justify by a investors; dirty, different charts will fly however prizes. Temporary, l	Books                                             	arts                                              	5.56	13539.35	5.02892289488801418
+Scientific, difficult polls would not achieve. Countries reach of course. Bad, new churches realize most english	Books                                             	arts                                              	3.98	143.88	0.05344137097545211
+United, important objectives put similarly large, previous phenomena; old, present days receive. Happy detectives assi	Books                                             	arts                                              	1.26	12297.15	4.56753235398096242
+Naturally new years put serious, negative vehicles. Fin	Books                                             	arts                                              	3.34	4587.47	1.70392470189572752
+Ago correct profits must not handle else. Healthy children may not go only ancient words. Later just characters ought to drink about. British parts must watch soon ago other clients. So vital d	Books                                             	arts                                              	4.03	5359.20	1.99056849688381241
+Much new waters 	Books                                             	arts                                              	1.85	6718.63	2.49550179508480530
+Hard different differences would not paint even. Together suitable schemes marry directly only open women. Social ca	Books                                             	arts                                              	2.65	3208.60	1.19177080144450674
+Tall, following actions keep widely willing, secondary groups. Heads could afford however; agricultural, square pri	Books                                             	arts                                              	9.99	4780.52	1.77562929368618505
+Anonymous, useful women provoke slightly present persons. Ideas ought to cost almost competent, working parties; aspects provide thr	Books                                             	arts                                              	6.73	5622.46	2.08835119999055082
+Powerful walls will find; there scottish decades must not	Books                                             	arts                                              	4.16	7914.41	2.93965054810833964
+Too executive doors progress mainly seemingly possible parts; hundreds stay virtually simple workers. Sola	Books                                             	arts                                              	34.32	3029.48	1.12524023173973205
+Careful privileges ought to live rather to a boards. Possible, broad p	Books                                             	arts                                              	3.93	1450.99	0.53894144336718969
+Aside legitimate decisions may not stand probably sexual g	Books                                             	arts                                              	3.88	9619.83	3.57309496629679899
+Specially interesting crews continue current, foreign directions; only social men would not call at least political children; circumstances could not understand now in a assessme	Books                                             	arts                                              	2.13	13616.57	5.05760473160419719
+Unlikely states take later in general extra inf	Books                                             	arts                                              	0.32	11879.56	4.41242683475911751
+Sometimes careful things state probably so	Books                                             	arts                                              	5.08	25457.85	9.45581321995700176
+Circumstances would not use. Principles seem writers. Times go from a hands. Members find grounds. Central, only teachers pursue properly into a p	Books                                             	arts                                              	5.95	2567.54	0.95366178505916251
+Inches may lose from a problems. Firm, other corporations shall protect ashamed, important practices. Materials shall not make then by a police. Weeks used	Books                                             	arts                                              	0.84	1811.85	0.67297572978782944
+Systems cannot await regions. Home appropr	Books                                             	arts                                              	7.30	1730.16	0.64263360027028230
+Extra, primitive weeks look obviou	Books                                             	arts                                              	1.18	22.77	0.00845746467272063
+More than key reasons should remain. Words used to offer slowly british	Books                                             	arts                                              	0.28	10311.18	3.82988320527288194
+Children may turn also above, historical aspects. Surveys migh	Books                                             	arts                                              	7.22	11872.32	4.40973768042312729
+Trustees know operations. Now past issues cut today german governments. British lines go critical, individual structures. Tonight adequate problems should no	Books                                             	arts                                              	4.05	8348.99	3.10106666569599586
+Useful observers start often white colleagues; simple pro	Books                                             	arts                                              	3.47	7565.51	2.81005856636428042
+Members should say earnings. Detailed departments would not move just at the hopes. Figures can take. Actually open houses want. Good teachers combine the	Books                                             	arts                                              	3.09	4363.97	1.62091006182752106
+Major, senior words afford economic libraries; successful seconds need outside. Clinical, new ideas put now red c	Books                                             	arts                                              	5.87	9661.08	3.58841646026911898
+Likely states feel astonishingly working roads. Parents put so somewhere able policies. Others may rely shortly instead interesting bodies; bri	Books                                             	arts                                              	7.50	132.66	0.04927392461498107
+Floors could not go only for a years. Special reasons shape consequently black, concerned instances. Mutual depths encourage both simple teachers. Cards favour massive 	Books                                             	arts                                              	1.83	20114.53	7.47114303396483641
+Accurate years want then other organisations. Simple lines mean as well so red results. Orthodox, central scales will not in	Books                                             	arts                                              	7.69	2153.04	0.79970398502215321
+Certain customers think exactly already necessary factories. Awkward doubts shall not forget fine	Books                                             	arts                                              	0.30	231.71	0.08606408165639427
+Visitors could not allow glad wages. Communist, real figures used to apply factors. Aggressive, optimistic days must mean about trees. Detailed courts consider really large pro	Books                                             	arts                                              	9.08	24425.09	9.07221501111207600
+Deep, big areas take for a facilities. Words could replace certainly cases; lights test. Nevertheless practical arts cross. Fa	Books                                             	arts                                              	7.37	4380.23	1.62694951617879192
+New, reluctant associations see more different, physical symptoms; useful pounds ought to give. Subjects 	Books                                             	business                                          	9.02	3044.02	1.58609001939612781
+Imports involve most now indian women. Developments announce intimately in a copies. Projects 	Books                                             	business                                          	3.26	472.29	0.24608723177265498
+Years shall want free objects. Old residents use absolutely so residential steps. Letters will share variables. Sure fres	Books                                             	business                                          	40.76	30227.05	15.74983814849696292
+Whole, important problems make. Indeed industrial members go skills. Soft	Books                                             	business                                          	3.22	137.92	0.07186336997625310
+Other, black houses flow. New soldiers put only eastern hours. Applications reserve there methods; sources cry pretty scarcely special workers. Never british opportunities 	Books                                             	business                                          	8.20	736.96	0.38399383075478162
+Rows could not	Books                                             	business                                          	1.65	1290.88	0.67261446516056841
+Remaining subjects handle even only certain ladies; eagerly literary days could not provide. Very different articles cut then. Boys see out of a houses. Governme	Books                                             	business                                          	9.03	1065.30	0.55507575431918810
+White members see highly on a negotiations. Evident, passive colours can refer familiar, ugly factors; away small examinations shall prove 	Books                                             	business                                          	17.97	1446.00	0.75343991433919646
+Manufacturing, ready concerns see already then new pupils. Both stable types used to manage otherw	Books                                             	business                                          	1.18	2635.71	1.37333963805184198
+Small, capable centres	Books                                             	business                                          	2.98	5029.45	2.62060053746422658
+Popular, different parameters might take open, used modules. Prisoners use pretty alternative lovers. Annual, professional others spend once true men. Other, small subsidies seem politically	Books                                             	business                                          	7.25	621.26	0.32370821658531756
+Supreme, free uses handle even in the customers. Other minutes might not make of course social neighbours. So environmental rights come other, able sales	Books                                             	business                                          	8.08	3950.22	2.05826654109334761
+Always other hours used to use. Women should jump then. Civil samples take therefore other offices. Concrete, major demands	Books                                             	business                                          	1.42	2013.79	1.04928752772968910
+Visual fragments 	Books                                             	business                                          	6.77	930.13	0.48464527491308216
+Classic issues will draw as european, engl	Books                                             	business                                          	75.64	556.83	0.29013689315456070
+Again british shareholders see shares. American lives ought to establish horses. Then ideal conservatives might charge even nec	Books                                             	business                                          	2.44	1898.13	0.98902275560488173
+Confident, video-tape	Books                                             	business                                          	3.17	1131.00	0.58930881266779474
+Of course fundamental children will not deal still including a suppliers. More crucial powers will not keep enough. As good comments used to devote even convenient electric problems. Publi	Books                                             	business                                          	8.85	414.75	0.21610595053401226
+Departments could seek now for a commu	Books                                             	business                                          	5.93	9895.85	5.15624369039663714
+Paintings must not know primary, royal stands; similar, available others ough	Books                                             	business                                          	0.39	13809.44	7.19542412909562460
+Most present eyes restore fat, central relationships; again considerable habits must face in a discussions. Engineers help at all direct occasions. Curiously del	Books                                             	business                                          	80.10	9267.25	4.82871095861681771
+So white countries could secure more angry items. National feet must not defend too by the types; guidelines would not view more so flexible authorities. Critics will handle closely lig	Books                                             	business                                          	2.50	2542.50	1.32477246349059959
+Simple changes ought to vote almost sudden techniques. Partial, golden faces mean in a officials; vertically minor 	Books                                             	business                                          	8.73	22710.22	11.83318548507904997
+Christian lines stand once deep formal aspirations. National, fine islands play together with a patterns. New journals lose etc positive armie	Books                                             	business                                          	4.89	11560.78	6.02375732565303988
+Children would not mean in favour of a parts. Heavy, whole others shall mean on	Books                                             	business                                          	3.13	9065.09	4.72337526492192700
+Lips will n	Books                                             	business                                          	8.48	541.62	0.28221170567385587
+White fees might combine reports. Tr	Books                                             	business                                          	2.09	37.60	0.01959152197728478
+Asleep children invite more. Wealthy forms could expect as. Indeed statistical examinations could la	Books                                             	business                                          	3.71	2082.24	1.08495347664844290
+Most new weeks go yet members. Also encouraging delegates make publications. Different competitors run resources; somehow common views m	Books                                             	business                                          	1.07	13412.42	6.98855641485568838
+Local, bloody names 	Books                                             	business                                          	4.40	1997.44	1.04076834197626873
+Large, larg	Books                                             	business                                          	3.50	12097.82	6.30358261721370521
+Only, gothic	Books                                             	business                                          	1.68	5708.95	2.97465477106967886
+Low, large clouds will not visit for example as the notions. Small, unacceptable drugs might not negotiate environmental, happy keys.	Books                                             	business                                          	3.11	3020.85	1.57401726502874248
+Silver, critical operations could help howev	Books                                             	business                                          	5.56	2286.24	1.19124790439754116
+Terrible, psychiatric bones will destroy also used studies; solely usual windows should not make shares. Advances continue sufficiently. As key days might not use far artists. Offici	Books                                             	business                                          	5.83	6672.40	3.47666146918178041
+Too white addresses end by the talks. Hands get only companies. Statements know. Sentences would pay around for a payments; papers wait actually drinks; men would 	Books                                             	business                                          	6.06	7609.35	3.96486031270882752
+New, big arguments may not win since by a tenant	Books                                             	computers                                         	1.00	904.16	0.32327741862037314
+Else substantial problems slip months. Just unique corporations put vast areas. Supporters like far perfect chapters. Now young reports become wrong trials. Available ears shall	Books                                             	computers                                         	51.46	18752.88	6.70498876094676063
+Cheap, desirable members take immediate, estimated debts; months must track typica	Books                                             	computers                                         	3.26	10027.86	3.58540600677589698
+Expert, scottish terms will ask quiet demands; poor bits attempt northern, dangerous si	Books                                             	computers                                         	2.66	7330.68	2.62104418148557444
+Gradually serious visitors bear no doubt technical hearts. Critics continue earlier soviet, standard minute	Books                                             	computers                                         	6.40	1711.84	0.61205894564136830
+Clear, general goods must know never women. Communications meet about. Other rewards spot wide in a skills. Relative, empty drawings facilitate too rooms. Still asian police end speedily comp	Books                                             	computers                                         	7.64	1292.04	0.46196177220211789
+At least remaining results shall keep cuts. Clients should meet policies. Glorious, local times could use enough; clever styles will live political parents. Single, gradual contracts will describe ho	Books                                             	computers                                         	9.51	3033.10	1.08446816760026298
+Environmental, new women pay again fingers. Different, uncomfortable records miss far russian, dependent members. Enough double men will go here immediatel	Books                                             	computers                                         	89.89	8553.39	3.05821739476786568
+Years learn here. Days make too. Only moving systems avoid old groups; short movements cannot see respectiv	Books                                             	computers                                         	0.60	3411.40	1.21972724504682903
+Magnetic	Books                                             	computers                                         	57.19	3569.09	1.27610843437421206
+Ga	Books                                             	computers                                         	5.53	2687.70	0.96097230360331899
+S	Books                                             	computers                                         	65.78	1613.04	0.57673355084432699
+Simple year	Books                                             	computers                                         	3.01	1262.79	0.45150359611088856
+Agricultural players shall smoke. So full reasons undertake 	Books                                             	computers                                         	0.70	4408.27	1.57615261257037727
+Then basic years can encourage later traditions. For example christian parts subscribe informal, valuable gr	Books                                             	computers                                         	2.75	844.19	0.30183547604973987
+Boxes batt	Books                                             	computers                                         	0.83	15300.82	5.47072375727191844
+Separate, dead buildings think possibly english, net policies. Big divisions can use almost	Books                                             	computers                                         	9.46	12403.71	4.43487806374503246
+Artists make times. Rather ready functions must pre	Books                                             	computers                                         	5.71	1533.00	0.54811569052494252
+Advantages emerge moves; special, expected operations pass etc natural preferences; very posit	Books                                             	computers                                         	0.15	5241.45	1.87405152387603389
+Since other birds shall blame sudden	Books                                             	computers                                         	6.74	2098.16	0.75018552983158082
+Legs throw then. Old-fashioned develo	Books                                             	computers                                         	2.66	163.26	0.05837271209073850
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q21.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q21.sql.out
new file mode 100644
index 0000000000000..9d7ddb1678a57
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q21.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<w_warehouse_name:string,i_item_id:string,inv_before:bigint,inv_after:bigint>
+-- !query output
+Just good amou	AAAAAAAAAAEAAAAA	2473	2677
+Just good amou	AAAAAAAAAAPBAAAA	2415	2071
+Just good amou	AAAAAAAAACACAAAA	2792	2465
+Just good amou	AAAAAAAAACGCAAAA	1396	2084
+Just good amou	AAAAAAAAACKCAAAA	1974	1585
+Just good amou	AAAAAAAAACPCAAAA	2131	1690
+Just good amou	AAAAAAAAADACAAAA	2432	2093
+Just good amou	AAAAAAAAADBBAAAA	1747	2529
+Just good amou	AAAAAAAAADNAAAAA	2763	1862
+Just good amou	AAAAAAAAAELBAAAA	2984	2371
+Just good amou	AAAAAAAAAFFDAAAA	2858	2415
+Just good amou	AAAAAAAAAFJDAAAA	2479	2176
+Just good amou	AAAAAAAAAFLDAAAA	2892	2303
+Just good amou	AAAAAAAAAGLDAAAA	2083	1892
+Just good amou	AAAAAAAAAGPCAAAA	1596	1191
+Just good amou	AAAAAAAAAHBAAAAA	2398	2956
+Just good amou	AAAAAAAAAHPBAAAA	1775	2655
+Just good amou	AAAAAAAAAINCAAAA	2296	2458
+Just good amou	AAAAAAAAAJIDAAAA	2528	2552
+Just good amou	AAAAAAAAAKABAAAA	1881	1465
+Just good amou	AAAAAAAAALFEAAAA	2952	2933
+Just good amou	AAAAAAAAALIDAAAA	1918	2438
+Just good amou	AAAAAAAAALOAAAAA	1904	1396
+Just good amou	AAAAAAAAALOBAAAA	1340	1570
+Just good amou	AAAAAAAAALPAAAAA	2299	1692
+Just good amou	AAAAAAAAANLCAAAA	3000	3113
+Just good amou	AAAAAAAAAOMDAAAA	2294	1928
+Just good amou	AAAAAAAABABCAAAA	2308	1807
+Just good amou	AAAAAAAABAGAAAAA	2358	2308
+Just good amou	AAAAAAAABCCCAAAA	1694	1863
+Just good amou	AAAAAAAABECDAAAA	2361	1636
+Just good amou	AAAAAAAABEMCAAAA	2644	2018
+Just good amou	AAAAAAAABHACAAAA	1471	1982
+Just good amou	AAAAAAAABHPCAAAA	1729	1481
+Just good amou	AAAAAAAABKEBAAAA	2284	1940
+Just good amou	AAAAAAAABLBDAAAA	1910	2399
+Just good amou	AAAAAAAABMCBAAAA	2460	2407
+Just good amou	AAAAAAAABMOBAAAA	2405	1669
+Just good amou	AAAAAAAABNEBAAAA	2733	2113
+Just good amou	AAAAAAAABOKDAAAA	3609	2690
+Just good amou	AAAAAAAABONAAAAA	2096	2325
+Just good amou	AAAAAAAABPPAAAAA	1788	1540
+Just good amou	AAAAAAAACACEAAAA	1763	1240
+Just good amou	AAAAAAAACAHCAAAA	1877	2693
+Just good amou	AAAAAAAACALAAAAA	3249	2868
+Just good amou	AAAAAAAACALBAAAA	1663	1528
+Just good amou	AAAAAAAACBDBAAAA	2334	2813
+Just good amou	AAAAAAAACBHBAAAA	1791	2015
+Just good amou	AAAAAAAACCABAAAA	2715	2959
+Just good amou	AAAAAAAACCJBAAAA	2461	1696
+Just good amou	AAAAAAAACCPDAAAA	3129	2313
+Just good amou	AAAAAAAACDGCAAAA	2114	1790
+Just good amou	AAAAAAAACDIBAAAA	2874	3081
+Just good amou	AAAAAAAACEHDAAAA	1616	1765
+Just good amou	AAAAAAAACEICAAAA	2037	2461
+Just good amou	AAAAAAAACFKDAAAA	2373	1698
+Just good amou	AAAAAAAACGJDAAAA	2578	1814
+Just good amou	AAAAAAAACGMAAAAA	2285	1553
+Just good amou	AAAAAAAACHCDAAAA	2620	2504
+Just good amou	AAAAAAAACIICAAAA	2800	2293
+Just good amou	AAAAAAAACJBEAAAA	2718	2070
+Just good amou	AAAAAAAACJGDAAAA	2153	1778
+Just good amou	AAAAAAAACJNCAAAA	1482	1582
+Just good amou	AAAAAAAACKBAAAAA	3122	2281
+Just good amou	AAAAAAAACKCAAAAA	1664	1982
+Just good amou	AAAAAAAACKHBAAAA	2222	1546
+Just good amou	AAAAAAAACKHDAAAA	2312	1798
+Just good amou	AAAAAAAACKKBAAAA	2463	1829
+Just good amou	AAAAAAAACLDAAAAA	2523	2348
+Just good amou	AAAAAAAACOLBAAAA	1630	2360
+Just good amou	AAAAAAAACPFDAAAA	1863	1405
+Just good amou	AAAAAAAACPKBAAAA	2088	2537
+Just good amou	AAAAAAAACPLDAAAA	1982	1599
+Just good amou	AAAAAAAADBECAAAA	3507	2356
+Just good amou	AAAAAAAADBGAAAAA	1819	1772
+Just good amou	AAAAAAAADCEEAAAA	2655	1843
+Just good amou	AAAAAAAADDEAAAAA	2210	2733
+Just good amou	AAAAAAAADDOCAAAA	2198	2067
+Just good amou	AAAAAAAADEHCAAAA	3190	2928
+Just good amou	AAAAAAAADFLDAAAA	2603	1991
+Just good amou	AAAAAAAADHDAAAAA	1649	1777
+Just good amou	AAAAAAAADIIAAAAA	1914	1818
+Just good amou	AAAAAAAADIOAAAAA	2058	2133
+Just good amou	AAAAAAAADJPBAAAA	2840	1910
+Just good amou	AAAAAAAADLBEAAAA	2293	1578
+Just good amou	AAAAAAAADLHBAAAA	1817	1316
+Just good amou	AAAAAAAADMCCAAAA	1944	2028
+Just good amou	AAAAAAAADPDBAAAA	1993	1430
+Just good amou	AAAAAAAAECEDAAAA	1968	2076
+Just good amou	AAAAAAAAEEOBAAAA	1992	1737
+Just good amou	AAAAAAAAEEODAAAA	2938	2820
+Just good amou	AAAAAAAAEFACAAAA	2213	2877
+Just good amou	AAAAAAAAEGCCAAAA	2262	3212
+Just good amou	AAAAAAAAEGCEAAAA	3052	2175
+Just good amou	AAAAAAAAEIEDAAAA	1786	2175
+Just good amou	AAAAAAAAEIHDAAAA	1938	1944
+Just good amou	AAAAAAAAEINDAAAA	2402	2113
+Just good amou	AAAAAAAAEKADAAAA	1327	1683
+Just good amou	AAAAAAAAELKAAAAA	1817	2630
+Just good amou	AAAAAAAAEMHCAAAA	2260	2878
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q22.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q22.sql.out
new file mode 100644
index 0000000000000..e8a553dd5094e
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q22.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_product_name:string,i_brand:string,i_class:string,i_category:string,qoh:double>
+-- !query output
+esepriableanti                                    	NULL	NULL	NULL	429.7808764940239
+esepriableanti                                    	importoamalg #x                                   	NULL	NULL	429.7808764940239
+esepriableanti                                    	importoamalg #x                                   	fragrances                                        	NULL	429.7808764940239
+esepriableanti                                    	importoamalg #x                                   	fragrances                                        	Women                                             	429.7808764940239
+n stbarn stbarought                               	NULL	NULL	NULL	430.0122448979592
+n stbarn stbarought                               	amalgimporto #x                                   	NULL	NULL	430.0122448979592
+n stbarn stbarought                               	amalgimporto #x                                   	accessories                                       	NULL	430.0122448979592
+n stbarn stbarought                               	amalgimporto #x                                   	accessories                                       	Men                                               	430.0122448979592
+antiationeing                                     	NULL	NULL	NULL	437.03614457831327
+antiationeing                                     	amalgexporti #x                                   	NULL	NULL	437.03614457831327
+antiationeing                                     	amalgexporti #x                                   	newborn                                           	NULL	437.03614457831327
+antiationeing                                     	amalgexporti #x                                   	newborn                                           	Children                                          	437.03614457831327
+n stpriantin st                                   	NULL	NULL	NULL	438.77868852459017
+n stpriantin st                                   	exportiexporti #x                                 	NULL	NULL	438.77868852459017
+n stpriantin st                                   	exportiexporti #x                                 	toddlers                                          	NULL	438.77868852459017
+n stpriantin st                                   	exportiexporti #x                                 	toddlers                                          	Children                                          	438.77868852459017
+eingprically                                      	NULL	NULL	NULL	439.97975708502025
+eingprically                                      	amalgbrand #x                                     	NULL	NULL	439.97975708502025
+eingprically                                      	amalgbrand #x                                     	semi-precious                                     	NULL	439.97975708502025
+eingprically                                      	amalgbrand #x                                     	semi-precious                                     	Jewelry                                           	439.97975708502025
+prieingable                                       	NULL	NULL	NULL	440.096
+prieingable                                       	exportiunivamalg #x                               	NULL	NULL	440.096
+prieingable                                       	exportiunivamalg #x                               	self-help                                         	NULL	440.096
+prieingable                                       	exportiunivamalg #x                               	self-help                                         	Books                                             	440.096
+oughteingn stationought                           	NULL	NULL	NULL	440.1497975708502
+oughteingn stationought                           	amalgscholar #x                                   	NULL	NULL	440.1497975708502
+oughteingn stationought                           	amalgscholar #x                                   	rock                                              	NULL	440.1497975708502
+oughteingn stationought                           	amalgscholar #x                                   	rock                                              	Music                                             	440.1497975708502
+eingationbaroughtought                            	NULL	NULL	NULL	440.9721115537849
+eingationbaroughtought                            	maxicorp #x                                       	NULL	NULL	440.9721115537849
+eingationbaroughtought                            	maxicorp #x                                       	womens watch                                      	NULL	440.9721115537849
+eingationbaroughtought                            	maxicorp #x                                       	womens watch                                      	Jewelry                                           	440.9721115537849
+priantibarpri                                     	NULL	NULL	NULL	443.45849802371544
+priantibarpri                                     	exportiimporto #x                                 	NULL	NULL	443.45849802371544
+priantibarpri                                     	exportiimporto #x                                 	pants                                             	NULL	443.45849802371544
+priantibarpri                                     	exportiimporto #x                                 	pants                                             	Men                                               	443.45849802371544
+prioughtantiation                                 	NULL	NULL	NULL	443.8825910931174
+prioughtantiation                                 	corpmaxi #x                                       	NULL	NULL	443.8825910931174
+prioughtantiation                                 	corpmaxi #x                                       	parenting                                         	NULL	443.8825910931174
+prioughtantiation                                 	corpmaxi #x                                       	parenting                                         	Books                                             	443.8825910931174
+eseprieingoughtought                              	NULL	NULL	NULL	445.2016129032258
+eseprieingoughtought                              	importonameless #x                                	NULL	NULL	445.2016129032258
+eseprieingoughtought                              	importonameless #x                                	baseball                                          	NULL	445.2016129032258
+eseprieingoughtought                              	importonameless #x                                	baseball                                          	Sports                                            	445.2016129032258
+eingpriationanti                                  	NULL	NULL	NULL	445.4920634920635
+eingpriationanti                                  	scholarunivamalg #x                               	NULL	NULL	445.4920634920635
+eingpriationanti                                  	scholarunivamalg #x                               	fiction                                           	NULL	445.4920634920635
+eingpriationanti                                  	scholarunivamalg #x                               	fiction                                           	Books                                             	445.4920634920635
+antin stablecallyought                            	NULL	NULL	NULL	445.54918032786884
+antin stablecallyought                            	importoedu pack #x                                	NULL	NULL	445.54918032786884
+antin stablecallyought                            	importoedu pack #x                                	mens                                              	NULL	445.54918032786884
+antin stablecallyought                            	importoedu pack #x                                	mens                                              	Shoes                                             	445.54918032786884
+callycallyn steing                                	NULL	NULL	NULL	445.9012345679012
+callycallyn steing                                	corpunivamalg #x                                  	NULL	NULL	445.9012345679012
+callycallyn steing                                	corpunivamalg #x                                  	mystery                                           	NULL	445.9012345679012
+callycallyn steing                                	corpunivamalg #x                                  	mystery                                           	Books                                             	445.9012345679012
+oughtpribarought                                  	NULL	NULL	NULL	446.125
+oughtpribarought                                  	exportinameless #x                                	NULL	NULL	446.125
+oughtpribarought                                  	exportinameless #x                                	wallpaper                                         	NULL	446.125
+oughtpribarought                                  	exportinameless #x                                	wallpaper                                         	Home                                              	446.125
+oughtantioughtbarought                            	NULL	NULL	NULL	446.1847389558233
+oughtantioughtbarought                            	edu packmaxi #x                                  	NULL	NULL	446.1847389558233
+oughtantioughtbarought                            	edu packmaxi #x                                  	entertainments                                    	NULL	446.1847389558233
+oughtantioughtbarought                            	edu packmaxi #x                                  	entertainments                                    	Books                                             	446.1847389558233
+callyoughtn stcallyought                          	NULL	NULL	NULL	446.43650793650795
+callyoughtn stcallyought                          	exportischolar #x                                 	NULL	NULL	446.43650793650795
+callyoughtn stcallyought                          	exportischolar #x                                 	pop                                               	NULL	446.43650793650795
+callyoughtn stcallyought                          	exportischolar #x                                 	pop                                               	Music                                             	446.43650793650795
+ationeingationableought                           	NULL	NULL	NULL	446.48192771084337
+ationeingationableought                           	namelessnameless #x                               	NULL	NULL	446.48192771084337
+ationeingationableought                           	namelessnameless #x                               	outdoor                                           	NULL	446.48192771084337
+ationeingationableought                           	namelessnameless #x                               	outdoor                                           	Sports                                            	446.48192771084337
+priantiableese                                    	NULL	NULL	NULL	446.85483870967744
+priantiableese                                    	exportiedu pack #x                                	NULL	NULL	446.85483870967744
+priantiableese                                    	exportiedu pack #x                                	kids                                              	NULL	446.85483870967744
+priantiableese                                    	exportiedu pack #x                                	kids                                              	Shoes                                             	446.85483870967744
+prieseeseableought                                	NULL	NULL	NULL	446.9186991869919
+prieseeseableought                                	amalgscholar #x                                   	NULL	NULL	446.9186991869919
+prieseeseableought                                	amalgscholar #x                                   	rock                                              	NULL	446.9186991869919
+prieseeseableought                                	amalgscholar #x                                   	rock                                              	Music                                             	446.9186991869919
+ationableoughtcallyought                          	NULL	NULL	NULL	447.165991902834
+ationableoughtcallyought                          	exportischolar #x                                 	NULL	NULL	447.165991902834
+ationableoughtcallyought                          	exportischolar #x                                 	pop                                               	NULL	447.165991902834
+ationableoughtcallyought                          	exportischolar #x                                 	pop                                               	Music                                             	447.165991902834
+pripricallyese                                    	NULL	NULL	NULL	447.2550607287449
+pripricallyese                                    	edu packimporto #x                                	NULL	NULL	447.2550607287449
+pripricallyese                                    	edu packimporto #x                                	sports-apparel                                    	NULL	447.2550607287449
+pripricallyese                                    	edu packimporto #x                                	sports-apparel                                    	Men                                               	447.2550607287449
+eingableationn st                                 	NULL	NULL	NULL	447.3541666666667
+eingableationn st                                 	namelessmaxi #x                                   	NULL	NULL	447.3541666666667
+eingableationn st                                 	namelessmaxi #x                                   	romance                                           	NULL	447.3541666666667
+eingableationn st                                 	namelessmaxi #x                                   	romance                                           	Books                                             	447.3541666666667
+n stantin stoughtought                            	NULL	NULL	NULL	448.2396694214876
+n stantin stoughtought                            	importoscholar #x                                 	NULL	NULL	448.2396694214876
+n stantin stoughtought                            	importoscholar #x                                 	country                                           	NULL	448.2396694214876
+n stantin stoughtought                            	importoscholar #x                                 	country                                           	Music                                             	448.2396694214876
+n steingbaranti                                   	NULL	NULL	NULL	448.702479338843
+n steingbaranti                                   	amalgamalg #x                                     	NULL	NULL	448.702479338843
+n steingbaranti                                   	amalgamalg #x                                     	dresses                                           	NULL	448.702479338843
+n steingbaranti                                   	amalgamalg #x                                     	dresses                                           	Women                                             	448.702479338843
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q23a.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q23a.sql.out
new file mode 100644
index 0000000000000..abc7f25b4469f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q23a.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<sum(sales):decimal(28,2)>
+-- !query output
+NULL
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q23b.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q23b.sql.out
new file mode 100644
index 0000000000000..04ba480fbec45
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q23b.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<c_last_name:string,c_first_name:string,sales:decimal(28,2)>
+-- !query output
+
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q24a.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q24a.sql.out
new file mode 100644
index 0000000000000..e50b231441a5d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q24a.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<c_last_name:string,c_first_name:string,s_store_name:string,paid:decimal(27,2)>
+-- !query output
+
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q24b.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q24b.sql.out
new file mode 100644
index 0000000000000..e50b231441a5d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q24b.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<c_last_name:string,c_first_name:string,s_store_name:string,paid:decimal(27,2)>
+-- !query output
+
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q25.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q25.sql.out
new file mode 100644
index 0000000000000..24b42a129ba4d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q25.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_item_id:string,i_item_desc:string,s_store_id:string,s_store_name:string,store_sales_profit:decimal(17,2),store_returns_loss:decimal(17,2),catalog_sales_profit:decimal(17,2)>
+-- !query output
+
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q26.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q26.sql.out
new file mode 100644
index 0000000000000..c6760949c1f40
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q26.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_item_id:string,agg1:double,agg2:decimal(11,6),agg3:decimal(11,6),agg4:decimal(11,6)>
+-- !query output
+AAAAAAAAAABDAAAA	84.0	18.790000	0.000000	17.280000
+AAAAAAAAAABEAAAA	76.0	145.970000	4732.320000	115.310000
+AAAAAAAAAACAAAAA	67.0	55.950000	1532.610000	48.670000
+AAAAAAAAAADBAAAA	34.333333333333336	70.963333	0.000000	20.753333
+AAAAAAAAAAEDAAAA	45.0	93.180000	0.000000	16.770000
+AAAAAAAAAAEEAAAA	46.666666666666664	73.350000	10.526667	15.433333
+AAAAAAAAAAFAAAAA	31.0	42.450000	0.000000	22.920000
+AAAAAAAAAAHAAAAA	42.0	9.700000	0.000000	0.480000
+AAAAAAAAAAHDAAAA	44.5	89.835000	0.000000	75.130000
+AAAAAAAAAAICAAAA	2.0	117.590000	68.420000	35.270000
+AAAAAAAAAAIDAAAA	11.0	6.310000	13.950000	1.510000
+AAAAAAAAAAKAAAAA	87.0	142.525000	0.000000	32.775000
+AAAAAAAAAALCAAAA	65.5	114.655000	0.000000	12.395000
+AAAAAAAAAAOAAAAA	91.0	42.190000	0.000000	30.790000
+AAAAAAAAAAPBAAAA	56.0	37.590000	0.000000	22.335000
+AAAAAAAAAAPCAAAA	35.0	49.890000	0.000000	30.430000
+AAAAAAAAABAAAAAA	87.0	77.640000	0.000000	29.500000
+AAAAAAAAABAEAAAA	79.0	34.720000	0.000000	30.900000
+AAAAAAAAABBAAAAA	16.0	53.500000	0.000000	26.515000
+AAAAAAAAABBDAAAA	79.0	91.570000	0.000000	0.910000
+AAAAAAAAABCEAAAA	100.0	106.260000	0.000000	26.560000
+AAAAAAAAABDAAAAA	58.0	3.020000	0.000000	2.080000
+AAAAAAAAABDBAAAA	21.0	72.850000	0.000000	57.550000
+AAAAAAAAABEAAAAA	71.0	70.970000	0.000000	48.250000
+AAAAAAAAABFCAAAA	33.0	284.590000	0.000000	36.990000
+AAAAAAAAABGAAAAA	54.5	77.980000	0.000000	51.590000
+AAAAAAAAABGBAAAA	84.0	18.370000	300.835000	14.035000
+AAAAAAAAABJDAAAA	42.0	41.960000	0.000000	8.810000
+AAAAAAAAABKCAAAA	27.0	62.695000	0.000000	24.770000
+AAAAAAAAABLBAAAA	62.0	112.810000	0.000000	54.795000
+AAAAAAAAABLCAAAA	42.0	206.490000	0.000000	61.940000
+AAAAAAAAABMAAAAA	45.5	158.505000	1467.465000	50.865000
+AAAAAAAAABMBAAAA	54.0	38.760000	0.000000	20.930000
+AAAAAAAAABNDAAAA	2.0	227.200000	0.000000	34.080000
+AAAAAAAAACBCAAAA	82.0	200.340000	0.000000	154.260000
+AAAAAAAAACCDAAAA	8.0	96.950000	0.000000	89.190000
+AAAAAAAAACDDAAAA	99.0	215.550000	5622.900000	66.820000
+AAAAAAAAACECAAAA	35.0	117.300000	0.000000	32.840000
+AAAAAAAAACEEAAAA	67.0	221.660000	2747.330000	164.020000
+AAAAAAAAACFAAAAA	81.0	16.420000	148.390000	2.290000
+AAAAAAAAACFBAAAA	23.0	148.690000	0.000000	96.640000
+AAAAAAAAACGAAAAA	71.5	182.975000	0.000000	89.260000
+AAAAAAAAACGDAAAA	38.0	111.060000	0.000000	43.310000
+AAAAAAAAACHBAAAA	62.0	199.820000	0.000000	67.930000
+AAAAAAAAACIDAAAA	73.0	184.700000	1925.030000	38.780000
+AAAAAAAAACJDAAAA	12.0	58.740000	319.710000	42.290000
+AAAAAAAAACLBAAAA	57.5	78.230000	391.295000	11.920000
+AAAAAAAAACLDAAAA	6.0	127.370000	0.000000	118.450000
+AAAAAAAAACMAAAAA	84.0	118.000000	0.000000	20.060000
+AAAAAAAAACNCAAAA	8.0	25.910000	99.840000	18.910000
+AAAAAAAAACODAAAA	49.0	106.120000	0.000000	6.360000
+AAAAAAAAADBBAAAA	77.0	166.050000	3605.600000	99.630000
+AAAAAAAAADCDAAAA	3.0	191.710000	0.000000	70.930000
+AAAAAAAAADDBAAAA	46.0	68.830000	0.000000	28.900000
+AAAAAAAAADEAAAAA	78.0	20.720000	0.000000	5.800000
+AAAAAAAAADEBAAAA	57.0	54.900000	0.000000	8.230000
+AAAAAAAAADFAAAAA	70.0	75.673333	777.880000	21.056667
+AAAAAAAAADFCAAAA	44.0	98.340000	182.190000	82.775000
+AAAAAAAAADFDAAAA	5.0	40.030000	0.000000	28.020000
+AAAAAAAAADGCAAAA	22.0	42.730000	0.000000	8.540000
+AAAAAAAAADKAAAAA	35.0	62.020000	0.000000	36.590000
+AAAAAAAAADMBAAAA	10.0	46.770000	0.000000	37.410000
+AAAAAAAAADNAAAAA	14.0	258.660000	0.000000	178.470000
+AAAAAAAAADNBAAAA	53.0	94.195000	0.000000	27.755000
+AAAAAAAAADNDAAAA	9.0	150.480000	0.000000	75.240000
+AAAAAAAAADOAAAAA	76.5	187.970000	310.730000	36.320000
+AAAAAAAAAEBDAAAA	7.0	68.300000	0.000000	15.700000
+AAAAAAAAAECEAAAA	81.0	241.650000	0.000000	9.660000
+AAAAAAAAAEDEAAAA	18.0	184.510000	979.740000	108.860000
+AAAAAAAAAEEDAAAA	81.0	72.050000	0.000000	56.910000
+AAAAAAAAAEGAAAAA	44.0	192.830000	0.000000	30.850000
+AAAAAAAAAEGBAAAA	39.0	6.230000	91.590000	3.050000
+AAAAAAAAAEGDAAAA	52.0	74.130000	0.000000	18.530000
+AAAAAAAAAEHAAAAA	68.0	72.870000	602.000000	59.020000
+AAAAAAAAAEHCAAAA	50.0	52.560000	0.000000	29.430000
+AAAAAAAAAEJBAAAA	66.0	66.110000	0.000000	5.940000
+AAAAAAAAAEKAAAAA	17.0	186.350000	339.590000	124.850000
+AAAAAAAAAEKCAAAA	93.0	57.210000	549.870000	6.290000
+AAAAAAAAAEKDAAAA	55.0	143.730000	0.000000	10.060000
+AAAAAAAAAELBAAAA	12.0	137.550000	0.000000	126.540000
+AAAAAAAAAEMBAAAA	100.0	52.750000	1092.980000	14.770000
+AAAAAAAAAENAAAAA	81.0	43.060000	2510.870000	32.290000
+AAAAAAAAAEPBAAAA	13.5	80.590000	0.000000	16.975000
+AAAAAAAAAFBEAAAA	93.0	115.340000	0.000000	42.670000
+AAAAAAAAAFCDAAAA	47.0	170.770000	0.000000	163.930000
+AAAAAAAAAFCEAAAA	73.0	91.970000	0.000000	59.780000
+AAAAAAAAAFDAAAAA	100.0	229.510000	2616.300000	68.850000
+AAAAAAAAAFDCAAAA	82.0	93.600000	0.000000	24.330000
+AAAAAAAAAFEEAAAA	61.0	245.950000	0.000000	199.210000
+AAAAAAAAAFFAAAAA	59.0	54.550000	555.140000	40.910000
+AAAAAAAAAFFBAAAA	65.0	142.570000	0.000000	69.850000
+AAAAAAAAAFFEAAAA	39.0	237.790000	681.580000	116.510000
+AAAAAAAAAFGCAAAA	45.0	205.590000	0.000000	47.280000
+AAAAAAAAAFHBAAAA	48.5	68.835000	0.000000	23.875000
+AAAAAAAAAFIAAAAA	72.0	84.430000	0.000000	0.000000
+AAAAAAAAAFIDAAAA	40.5	45.650000	1212.090000	17.425000
+AAAAAAAAAFJCAAAA	13.0	133.270000	0.000000	6.660000
+AAAAAAAAAFKCAAAA	5.0	178.640000	0.000000	105.390000
+AAAAAAAAAFNBAAAA	16.0	32.220000	0.000000	31.570000
+AAAAAAAAAFODAAAA	59.0	88.455000	138.365000	14.180000
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q27.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q27.sql.out
new file mode 100644
index 0000000000000..4e402d0379f2e
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q27.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_item_id:string,s_state:string,g_state:tinyint,agg1:double,agg2:decimal(11,6),agg3:decimal(11,6),agg4:decimal(11,6)>
+-- !query output
+NULL	NULL	1	50.20319480167863	76.358588	197.423228	38.217862
+AAAAAAAAAAABAAAA	NULL	1	45.0	20.970000	0.000000	10.900000
+AAAAAAAAAAABAAAA	TN	0	45.0	20.970000	0.000000	10.900000
+AAAAAAAAAAACAAAA	NULL	1	4.0	60.970000	0.000000	35.970000
+AAAAAAAAAAACAAAA	TN	0	4.0	60.970000	0.000000	35.970000
+AAAAAAAAAACDAAAA	NULL	1	86.0	56.830000	0.000000	38.070000
+AAAAAAAAAACDAAAA	TN	0	86.0	56.830000	0.000000	38.070000
+AAAAAAAAAADBAAAA	NULL	1	61.0	40.060000	0.000000	7.210000
+AAAAAAAAAADBAAAA	TN	0	61.0	40.060000	0.000000	7.210000
+AAAAAAAAAADCAAAA	NULL	1	37.0	96.405000	0.000000	82.010000
+AAAAAAAAAADCAAAA	TN	0	37.0	96.405000	0.000000	82.010000
+AAAAAAAAAADEAAAA	NULL	1	23.0	96.010000	0.000000	57.600000
+AAAAAAAAAADEAAAA	TN	0	23.0	96.010000	0.000000	57.600000
+AAAAAAAAAAEDAAAA	NULL	1	64.0	10.940000	0.000000	10.390000
+AAAAAAAAAAEDAAAA	TN	0	64.0	10.940000	0.000000	10.390000
+AAAAAAAAAAEEAAAA	NULL	1	65.0	97.825000	0.000000	24.320000
+AAAAAAAAAAEEAAAA	TN	0	65.0	97.825000	0.000000	24.320000
+AAAAAAAAAAFAAAAA	NULL	1	71.0	88.080000	0.000000	10.560000
+AAAAAAAAAAFAAAAA	TN	0	71.0	88.080000	0.000000	10.560000
+AAAAAAAAAAFCAAAA	NULL	1	21.0	72.140000	0.000000	54.725000
+AAAAAAAAAAFCAAAA	TN	0	21.0	72.140000	0.000000	54.725000
+AAAAAAAAAAGBAAAA	NULL	1	23.0	97.980000	0.000000	45.070000
+AAAAAAAAAAGBAAAA	TN	0	23.0	97.980000	0.000000	45.070000
+AAAAAAAAAAGCAAAA	NULL	1	30.0	62.280000	0.000000	9.340000
+AAAAAAAAAAGCAAAA	TN	0	30.0	62.280000	0.000000	9.340000
+AAAAAAAAAAHAAAAA	NULL	1	30.0	91.910000	0.000000	52.380000
+AAAAAAAAAAHAAAAA	TN	0	30.0	91.910000	0.000000	52.380000
+AAAAAAAAAAHBAAAA	NULL	1	76.0	30.060000	345.610000	26.750000
+AAAAAAAAAAHBAAAA	TN	0	76.0	30.060000	345.610000	26.750000
+AAAAAAAAAAHDAAAA	NULL	1	60.0	49.390000	0.000000	19.155000
+AAAAAAAAAAHDAAAA	TN	0	60.0	49.390000	0.000000	19.155000
+AAAAAAAAAAIAAAAA	NULL	1	65.0	115.230000	0.000000	79.555000
+AAAAAAAAAAIAAAAA	TN	0	65.0	115.230000	0.000000	79.555000
+AAAAAAAAAAICAAAA	NULL	1	NULL	NULL	262.480000	NULL
+AAAAAAAAAAICAAAA	TN	0	NULL	NULL	262.480000	NULL
+AAAAAAAAAAJCAAAA	NULL	1	7.0	111.745000	0.000000	82.515000
+AAAAAAAAAAJCAAAA	TN	0	7.0	111.745000	0.000000	82.515000
+AAAAAAAAAAKAAAAA	NULL	1	31.5	51.350000	0.000000	36.555000
+AAAAAAAAAAKAAAAA	TN	0	31.5	51.350000	0.000000	36.555000
+AAAAAAAAAAKBAAAA	NULL	1	3.0	146.600000	0.000000	105.550000
+AAAAAAAAAAKBAAAA	TN	0	3.0	146.600000	0.000000	105.550000
+AAAAAAAAAAKDAAAA	NULL	1	69.0	34.660000	0.000000	11.090000
+AAAAAAAAAAKDAAAA	TN	0	69.0	34.660000	0.000000	11.090000
+AAAAAAAAAALAAAAA	NULL	1	97.0	14.270000	0.000000	12.410000
+AAAAAAAAAALAAAAA	TN	0	97.0	14.270000	0.000000	12.410000
+AAAAAAAAAAMBAAAA	NULL	1	68.5	70.250000	0.000000	34.085000
+AAAAAAAAAAMBAAAA	TN	0	68.5	70.250000	0.000000	34.085000
+AAAAAAAAAAMCAAAA	NULL	1	51.5	73.135000	0.000000	25.570000
+AAAAAAAAAAMCAAAA	TN	0	51.5	73.135000	0.000000	25.570000
+AAAAAAAAAANAAAAA	NULL	1	50.5	29.315000	9.580000	15.805000
+AAAAAAAAAANAAAAA	TN	0	50.5	29.315000	9.580000	15.805000
+AAAAAAAAAAOCAAAA	NULL	1	1.0	74.630000	0.000000	19.400000
+AAAAAAAAAAOCAAAA	TN	0	1.0	74.630000	0.000000	19.400000
+AAAAAAAAAAODAAAA	NULL	1	66.33333333333333	52.823333	1793.560000	28.406667
+AAAAAAAAAAODAAAA	TN	0	66.33333333333333	52.823333	1793.560000	28.406667
+AAAAAAAAAAPBAAAA	NULL	1	17.0	167.070000	0.000000	40.090000
+AAAAAAAAAAPBAAAA	TN	0	17.0	167.070000	0.000000	40.090000
+AAAAAAAAABAAAAAA	NULL	1	79.0	48.110000	0.000000	14.430000
+AAAAAAAAABAAAAAA	TN	0	79.0	48.110000	0.000000	14.430000
+AAAAAAAAABAEAAAA	NULL	1	16.5	26.370000	0.000000	9.325000
+AAAAAAAAABAEAAAA	TN	0	16.5	26.370000	0.000000	9.325000
+AAAAAAAAABCBAAAA	NULL	1	32.0	98.600000	0.000000	46.340000
+AAAAAAAAABCBAAAA	TN	0	32.0	98.600000	0.000000	46.340000
+AAAAAAAAABFCAAAA	NULL	1	24.0	101.420000	0.000000	3.040000
+AAAAAAAAABFCAAAA	TN	0	24.0	101.420000	0.000000	3.040000
+AAAAAAAAABFEAAAA	NULL	1	34.5	42.490000	131.055000	15.505000
+AAAAAAAAABFEAAAA	TN	0	34.5	42.490000	131.055000	15.505000
+AAAAAAAAABGAAAAA	NULL	1	57.666666666666664	94.343333	569.626667	56.296667
+AAAAAAAAABGAAAAA	TN	0	57.666666666666664	94.343333	569.626667	56.296667
+AAAAAAAAABHCAAAA	NULL	1	58.0	38.370000	0.000000	16.880000
+AAAAAAAAABHCAAAA	TN	0	58.0	38.370000	0.000000	16.880000
+AAAAAAAAABHDAAAA	NULL	1	43.0	10.290000	0.000000	3.800000
+AAAAAAAAABHDAAAA	TN	0	43.0	10.290000	0.000000	3.800000
+AAAAAAAAABIBAAAA	NULL	1	60.25	74.462500	0.000000	58.247500
+AAAAAAAAABIBAAAA	TN	0	60.25	74.462500	0.000000	58.247500
+AAAAAAAAABJDAAAA	NULL	1	64.5	28.405000	0.000000	22.080000
+AAAAAAAAABJDAAAA	TN	0	64.5	28.405000	0.000000	22.080000
+AAAAAAAAABKCAAAA	NULL	1	69.0	44.880000	518.585000	18.855000
+AAAAAAAAABKCAAAA	TN	0	69.0	44.880000	518.585000	18.855000
+AAAAAAAAABLBAAAA	NULL	1	18.0	43.495000	0.000000	24.660000
+AAAAAAAAABLBAAAA	TN	0	18.0	43.495000	0.000000	24.660000
+AAAAAAAAABLCAAAA	NULL	1	6.0	102.740000	0.000000	90.410000
+AAAAAAAAABLCAAAA	TN	0	6.0	102.740000	0.000000	90.410000
+AAAAAAAAABMDAAAA	NULL	1	50.5	32.745000	0.000000	12.005000
+AAAAAAAAABMDAAAA	TN	0	50.5	32.745000	0.000000	12.005000
+AAAAAAAAABNAAAAA	NULL	1	85.0	31.880000	0.000000	5.100000
+AAAAAAAAABNAAAAA	TN	0	85.0	31.880000	0.000000	5.100000
+AAAAAAAAABOCAAAA	NULL	1	60.0	113.590000	51.520000	1.130000
+AAAAAAAAABOCAAAA	TN	0	60.0	113.590000	51.520000	1.130000
+AAAAAAAAABPAAAAA	NULL	1	55.0	89.150000	3442.050000	76.440000
+AAAAAAAAABPAAAAA	TN	0	55.0	89.150000	3442.050000	76.440000
+AAAAAAAAABPBAAAA	NULL	1	80.0	16.010000	0.000000	9.440000
+AAAAAAAAABPBAAAA	TN	0	80.0	16.010000	0.000000	9.440000
+AAAAAAAAABPDAAAA	NULL	1	73.0	112.940000	2248.960000	99.380000
+AAAAAAAAABPDAAAA	TN	0	73.0	112.940000	2248.960000	99.380000
+AAAAAAAAACAAAAAA	NULL	1	61.0	101.820000	0.000000	90.610000
+AAAAAAAAACAAAAAA	TN	0	61.0	101.820000	0.000000	90.610000
+AAAAAAAAACACAAAA	NULL	1	86.0	101.500000	0.000000	57.850000
+AAAAAAAAACACAAAA	TN	0	86.0	101.500000	0.000000	57.850000
+AAAAAAAAACADAAAA	NULL	1	65.0	97.210000	0.000000	83.595000
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q28.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q28.sql.out
new file mode 100644
index 0000000000000..5d3f3fa3c8602
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q28.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<B1_LP:decimal(11,6),B1_CNT:bigint,B1_CNTD:bigint,B2_LP:decimal(11,6),B2_CNT:bigint,B2_CNTD:bigint,B3_LP:decimal(11,6),B3_CNT:bigint,B3_CNTD:bigint,B4_LP:decimal(11,6),B4_CNT:bigint,B4_CNTD:bigint,B5_LP:decimal(11,6),B5_CNT:bigint,B5_CNTD:bigint,B6_LP:decimal(11,6),B6_CNT:bigint,B6_CNTD:bigint>
+-- !query output
+78.045281	36383	9236	69.528580	35193	6542	133.847037	28274	9714	81.911887	31756	7687	61.160300	36338	8603	39.282627	29915	5210
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q29.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q29.sql.out
new file mode 100644
index 0000000000000..467b5c3ba3eaf
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q29.sql.out
@@ -0,0 +1,7 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_item_id:string,i_item_desc:string,s_store_id:string,s_store_name:string,store_sales_quantity:bigint,store_returns_quantity:bigint,catalog_sales_quantity:bigint>
+-- !query output
+AAAAAAAADIFDAAAA	Now christian papers believe very major, new branches. Annual wars include harshly so-called sites. 	AAAAAAAAHAAAAAAA	ation	11	10	13
+AAAAAAAANNBEAAAA	Old forces shall not think more than foreign earnings. Controls could carry almos	AAAAAAAACAAAAAAA	able	56	25	10
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q3.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q3.sql.out
new file mode 100644
index 0000000000000..65e15769d5b0f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q3.sql.out
@@ -0,0 +1,88 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<d_year:int,brand_id:int,brand:string,sum_agg:decimal(17,2)>
+-- !query output
+1998	1004001	edu packamalg #x                                  	65716.37
+1998	10001008	amalgunivamalg #x                                 	34140.78
+1998	8012001	importomaxi #x                                    	32669.34
+1998	2004001	edu packimporto #x                                	25130.97
+1998	10003014	exportiunivamalg #x                              	23720.25
+1998	5004001	edu packscholar #x                                	23713.55
+1998	9002008	importomaxi #x                                    	22002.12
+1998	3003001	exportiexporti #x                                 	21596.96
+1998	8014004	edu packmaxi #x                                   	20442.12
+1998	9009005	maximaxi #x                                       	19866.63
+1998	3002001	importoexporti #x                                 	17347.94
+1998	3001001	amalgexporti #x                                   	16882.10
+1998	2003001	exportiimporto #x                                 	13095.00
+1998	8010001	univmaxi #x                                       	12408.22
+1998	8007001	brandnameless #x                                  	12021.05
+1998	6005005	scholarcorp #x                                    	10366.33
+1998	8013008	exportimaxi #x                                    	10008.30
+1998	10008008	namelessunivamalg #x                              	7909.24
+1998	1003002	exportiamalg #x                                   	5046.65
+1999	1004001	edu packamalg #x                                  	72111.20
+1999	8012001	importomaxi #x                                    	45932.95
+1999	9009005	maximaxi #x                                       	32382.90
+1999	1003002	exportiamalg #x                                   	28586.32
+1999	8007001	brandnameless #x                                  	27105.57
+1999	9002008	importomaxi #x                                    	26746.44
+1999	5004001	edu packscholar #x                                	25906.67
+1999	8010001	univmaxi #x                                       	23297.75
+1999	8013008	exportimaxi #x                                    	20896.22
+1999	2004001	edu packimporto #x                                	18025.93
+1999	10003014	exportiunivamalg #x                              	16211.31
+1999	8014004	edu packmaxi #x                                   	15207.49
+1999	3001001	amalgexporti #x                                   	13536.73
+1999	10001008	amalgunivamalg #x                                 	12980.62
+1999	3003001	exportiexporti #x                                 	12753.61
+1999	10008008	namelessunivamalg #x                              	12446.57
+1999	2003001	exportiimporto #x                                 	11284.94
+1999	3002001	importoexporti #x                                 	9820.44
+1999	6005005	scholarcorp #x                                    	5487.63
+2000	3002001	importoexporti #x                                 	49567.04
+2000	1004001	edu packamalg #x                                  	35173.68
+2000	6006004	corpcorp #x                                       	31206.02
+2000	1003002	exportiamalg #x                                   	30500.97
+2000	6005005	scholarcorp #x                                    	30270.09
+2000	9001012	amalgmaxi #x                                     	29107.66
+2000	8012001	importomaxi #x                                    	26610.58
+2000	5004001	edu packscholar #x                                	26200.25
+2000	10003014	exportiunivamalg #x                              	23018.57
+2000	3001001	amalgexporti #x                                   	20499.18
+2000	2004001	edu packimporto #x                                	19982.58
+2000	10015010	importoedu pack #x                                	18697.08
+2000	8007001	brandnameless #x                                  	17315.93
+2000	8014004	edu packmaxi #x                                   	16235.88
+2000	3003001	exportiexporti #x                                 	14272.86
+2000	8013008	exportimaxi #x                                    	10570.32
+2000	9002008	importomaxi #x                                    	10262.91
+2001	1003002	exportiamalg #x                                   	36559.22
+2001	9002008	importomaxi #x                                    	25301.41
+2001	8013008	exportimaxi #x                                    	21069.43
+2001	6005003	importoamalg #x                                   	20705.88
+2001	9001012	amalgmaxi #x                                     	18795.96
+2001	8012001	exportiedu pack #x                                	18477.27
+2001	6006004	corpcorp #x                                       	18283.57
+2001	9003010	exportimaxi #x                                   	17793.12
+2001	3002001	scholarmaxi #x                                    	17206.42
+2001	10003014	exportiunivamalg #x                              	15781.84
+2001	10001003	edu packamalg #x                                  	15683.42
+2001	8014004	edu packmaxi #x                                   	13108.20
+2001	10015010	importoedu pack #x                                	11683.34
+2001	2002002	importoimporto #x                                 	8532.67
+2002	1003002	exportiamalg #x                                   	53079.32
+2002	9002008	importomaxi #x                                    	39552.73
+2002	3002001	scholarmaxi #x                                    	38802.50
+2002	9003010	exportimaxi #x                                   	36257.87
+2002	5002001	importoscholar #x                                 	36116.22
+2002	10015010	amalgunivamalg #x                                	28522.27
+2002	8012001	exportiedu pack #x                                	27777.47
+2002	8013008	exportimaxi #x                                    	23919.27
+2002	2002002	importoimporto #x                                 	21849.78
+2002	8014004	edu packmaxi #x                                   	19276.30
+2002	10001003	edu packamalg #x                                  	13837.38
+2002	10003014	exportiunivamalg #x                              	13106.68
+2002	9015001	scholarunivamalg #x                               	11700.36
+2002	6005003	importoamalg #x                                   	6367.68
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q30.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q30.sql.out
new file mode 100644
index 0000000000000..2a298679ad558
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q30.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<c_customer_id:string,c_salutation:string,c_first_name:string,c_last_name:string,c_preferred_cust_flag:string,c_birth_day:int,c_birth_month:int,c_birth_year:int,c_birth_country:string,c_login:string,c_email_address:string,c_last_review_date:int,ctr_total_return:decimal(17,2)>
+-- !query output
+AAAAAAAAABEBBAAA	Sir       	Carlton             	Miles                         	N	23	9	1957	MALI	NULL	Carlton.Miles@4DbIUoznbr.org                      	2452588	12916.86
+AAAAAAAAABKGBAAA	Miss      	Sharee              	Stevens                       	Y	15	8	1992	SLOVAKIA	NULL	Sharee.Stevens@Z.edu                              	2452492	2480.80
+AAAAAAAAAEIPAAAA	Mr.       	Chris               	Ryan                          	Y	23	4	1931	TURKMENISTAN	NULL	Chris.Ryan@QlH9G0fkAR5.org                        	2452443	3469.95
+AAAAAAAAAHNKAAAA	Sir       	Scott               	Garcia                        	N	19	10	1952	WALLIS AND FUTUNA	NULL	Scott.Garcia@3pd6mnJYbKxb.org                     	2452611	2944.24
+AAAAAAAAAJDABAAA	Sir       	Gerald              	Monroe                        	Y	19	5	1985	LIECHTENSTEIN	NULL	Gerald.Monroe@opYT.org                            	2452632	4030.41
+AAAAAAAAAJJBAAAA	Sir       	Jose                	Garcia                        	Y	11	6	1977	AMERICAN SAMOA	NULL	Jose.Garcia@n1VrEIOg4f.com                        	2452406	4131.05
+AAAAAAAAALNKAAAA	Dr.       	Wanda               	Devries                       	N	6	2	1953	YEMEN	NULL	Wanda.Devries@lzbOovP.com                         	2452411	3197.25
+AAAAAAAAANPAAAAA	NULL	NULL	Sanders                       	NULL	12	10	NULL	NULL	NULL	NULL	2452646	2091.90
+AAAAAAAAAOKPAAAA	Mr.       	Brandon             	Foster                        	N	28	4	1992	FRENCH POLYNESIA	NULL	Brandon.Foster@vKi8eFToOZHK.org                   	2452527	3706.08
+AAAAAAAABBEDAAAA	Mr.       	James               	Pantoja                       	Y	5	12	1982	NEPAL	NULL	James.Pantoja@LTZLj3ddIvIG0.edu                   	2452503	5668.25
+AAAAAAAABBIHAAAA	Ms.       	Sherry              	Kennedy                       	Y	3	8	1984	MOZAMBIQUE	NULL	Sherry.Kennedy@9hhBNI6.edu                        	2452481	1561.14
+AAAAAAAABBJMAAAA	NULL	Paul                	Lindsey                       	N	NULL	10	NULL	NULL	NULL	Paul.Lindsey@bQo8Zt9XStF.edu                      	NULL	3139.50
+AAAAAAAABBNFBAAA	Mrs.      	Nellie              	Johnson                       	Y	14	6	1962	BAHRAIN	NULL	Nellie.Johnson@U.com                              	2452411	1609.90
+AAAAAAAABDKOAAAA	Dr.       	Vernita             	Bennett                       	Y	12	3	1955	NICARAGUA	NULL	Vernita.Bennett@u2QKJHt.com                       	2452352	11315.85
+AAAAAAAABEJLAAAA	Mr.       	Abel                	Lucero                        	N	15	9	1930	CAPE VERDE	NULL	Abel.Lucero@txdlc3sekSyj7mLokv.org                	2452461	4524.56
+AAAAAAAABFFHBAAA	Mrs.      	Arlene              	Parrish                       	N	18	9	1957	MALAWI	NULL	Arlene.Parrish@m5lX3mTVhbxAEz.org                 	2452433	3184.72
+AAAAAAAABGCFBAAA	Sir       	Alberto             	Whitney                       	Y	20	5	1946	MALAYSIA	NULL	Alberto.Whitney@En6lzc8TlV7vF.org                 	2452496	3806.56
+AAAAAAAABHAGBAAA	Mrs.      	Minnie              	Thompson                      	N	1	8	1976	NEW ZEALAND	NULL	Minnie.Thompson@7u.org                            	2452558	1871.31
+AAAAAAAABLMCAAAA	Dr.       	Frank               	Price                         	N	13	2	1962	SYRIAN ARAB REPUBLIC	NULL	Frank.Price@KO.org                                	2452372	1779.84
+AAAAAAAABNEGAAAA	Ms.       	Laurel              	Olson                         	Y	24	6	1960	BANGLADESH	NULL	Laurel.Olson@92YpV3hofMbNz.edu                    	2452508	4912.49
+AAAAAAAABNMIAAAA	Mrs.      	Judi                	Lopes                         	Y	21	2	1943	NICARAGUA	NULL	Judi.Lopes@DYAyBUb1E.org                          	2452297	8335.86
+AAAAAAAABOGDAAAA	Ms.       	Laura               	Mcfadden                      	N	29	3	1964	GREECE	NULL	Laura.Mcfadden@Ks.com                             	2452364	3942.35
+AAAAAAAABPECAAAA	Mr.       	William             	Hoffman                       	N	21	4	1991	PERU	NULL	William.Hoffman@m8E1N4.com                        	2452321	4550.73
+AAAAAAAACGHDAAAA	Mr.       	Ricky               	Marshall                      	N	11	7	1979	NAMIBIA	NULL	Ricky.Marshall@tJKkX.edu                          	2452610	10400.46
+AAAAAAAACIKBAAAA	Ms.       	Virginia            	Lawson                        	N	26	11	1929	PHILIPPINES	NULL	Virginia.Lawson@bdtmjhsnqQZx.edu                  	2452490	2164.20
+AAAAAAAACILFBAAA	NULL	Frederick           	NULL	N	23	11	1954	NULL	NULL	Frederick.Lambert@BNHNh23xF7qIUJ30.edu            	NULL	3849.92
+AAAAAAAACLNHBAAA	Dr.       	Gary                	Perez                         	N	21	4	1992	MACAO	NULL	Gary.Perez@5dGey4.com                             	2452300	9647.56
+AAAAAAAACOEAAAAA	Dr.       	David               	Saunders                      	Y	6	12	1991	HAITI	NULL	David.Saunders@APTXJyYNgVa.edu                    	2452482	8357.76
+AAAAAAAADBIHAAAA	Mr.       	John                	Sturm                         	Y	5	8	1973	GUAM	NULL	John.Sturm@PnUSI7.com                             	2452457	7161.74
+AAAAAAAADCCCBAAA	Ms.       	Antonia             	Miller                        	N	1	5	1992	PALAU	NULL	Antonia.Miller@CsPe3fE38F.com                     	2452635	2688.66
+AAAAAAAADGIGBAAA	Miss      	Margarita           	Park                          	Y	20	1	1947	LITHUANIA	NULL	Margarita.Park@qb0FrD2Pk.org                      	2452462	1557.37
+AAAAAAAADGMJAAAA	Mrs.      	Karen               	Olson                         	Y	18	11	1974	ARGENTINA	NULL	Karen.Olson@Qe3bBGr.com                           	2452596	11849.40
+AAAAAAAADGOIAAAA	NULL	NULL	NULL	NULL	2	NULL	1977	NULL	NULL	Kathleen.Bell@B0Ze.com                            	2452322	3979.64
+AAAAAAAADHFBAAAA	Mr.       	William             	Lopez                         	N	12	1	1967	MALAYSIA	NULL	William.Lopez@9JMTYo5O22ZFskMMybp.org             	2452435	2302.41
+AAAAAAAADJJMAAAA	Mr.       	James               	Cooper                        	Y	26	4	1982	TUNISIA	NULL	James.Cooper@KH.com                               	2452365	3388.66
+AAAAAAAADJPEBAAA	Ms.       	Lizzie              	Neal                          	Y	16	2	1939	MAURITIUS	NULL	Lizzie.Neal@ru9qksDH5rgLsRn.edu                   	2452433	1915.72
+AAAAAAAADKJCAAAA	Dr.       	Daniel              	Green                         	N	29	8	1970	AMERICAN SAMOA	NULL	Daniel.Green@GPktIG5Y.org                         	2452421	5675.02
+AAAAAAAADMFKAAAA	Mr.       	Ray                 	Gonzales                      	N	22	2	1930	TURKMENISTAN	NULL	Ray.Gonzales@kH8kp.com                            	2452378	2294.25
+AAAAAAAADMHDAAAA	Mr.       	Lee                 	Stanley                       	Y	9	3	1959	CZECH REPUBLIC	NULL	Lee.Stanley@rONmrEDug8q.com                       	2452451	1912.92
+AAAAAAAADMONAAAA	Sir       	Joshua              	West                          	Y	15	3	1987	ALAND ISLANDS	NULL	Joshua.West@5q4JZHUuo9h0e1ol.org                  	2452545	2870.00
+AAAAAAAADOBAAAAA	Dr.       	Paul                	Yeager                        	N	17	11	1946	EGYPT	NULL	Paul.Yeager@PKzHg.edu                             	2452571	3050.46
+AAAAAAAAEADIAAAA	Dr.       	Gregory             	Carey                         	Y	6	1	1937	SUDAN	NULL	Gregory.Carey@495UmqFpU0My0GB8.edu                	2452330	5133.18
+AAAAAAAAECBEAAAA	Ms.       	Megan               	Wilson                        	N	1	12	1935	COMOROS	NULL	Megan.Wilson@mi.com                               	2452501	1622.24
+AAAAAAAAECIEAAAA	Miss      	Danille             	Sanders                       	Y	15	3	1942	OMAN	NULL	Danille.Sanders@dnjPPYBZQ.com                     	2452566	5474.66
+AAAAAAAAEEBEBAAA	Dr.       	Jamie               	Jackson                       	Y	21	11	1933	POLAND	NULL	Jamie.Jackson@dLUBOuU.com                         	2452365	1761.68
+AAAAAAAAEEGMAAAA	Dr.       	NULL	NULL	N	5	NULL	NULL	NULL	NULL	NULL	2452438	2046.20
+AAAAAAAAEEJAAAAA	Dr.       	Rhonda              	Anderson                      	N	27	12	1942	ISLE OF MAN	NULL	Rhonda.Anderson@P2VN7VQxGjj.com                   	2452394	2888.50
+AAAAAAAAEENFAAAA	Dr.       	James               	Brown                         	N	27	9	1971	NEW CALEDONIA	NULL	James.Brown@dzkgBG43cs.org                        	2452466	2474.24
+AAAAAAAAEGDDBAAA	Mr.       	Timothy             	Barajas                       	N	8	10	1976	MAURITANIA	NULL	Timothy.Barajas@FFhEdu3MO4n.com                   	2452594	2209.06
+AAAAAAAAEGEGAAAA	Mrs.      	Arthur              	Kirk                          	Y	24	8	1972	SLOVAKIA	NULL	Arthur.Kirk@3eFiyMXiuRCqt1ofD.com                 	2452286	3189.74
+AAAAAAAAEGGKAAAA	Miss      	Johanna             	Moses                         	Y	3	5	1971	R�UNION	NULL	Johanna.Moses@AhPc5dGr7FubqU1Lyj.com              	2452443	2842.06
+AAAAAAAAEGPPAAAA	Mr.       	Michael             	Pringle                       	Y	1	10	1990	JERSEY	NULL	Michael.Pringle@noEgytx7nOED.edu                  	2452398	1870.56
+AAAAAAAAEHFCBAAA	Mrs.      	Lillian             	Yazzie                        	N	14	1	1936	GEORGIA	NULL	Lillian.Yazzie@Hzg3QQh.com                        	2452555	1758.12
+AAAAAAAAEHLDAAAA	Ms.       	NULL	Brown                         	NULL	NULL	3	NULL	NULL	NULL	NULL	NULL	4483.20
+AAAAAAAAEHLNAAAA	Mr.       	Gregory             	Wester                        	Y	9	3	1946	SPAIN	NULL	Gregory.Wester@ko.com                             	2452535	2303.28
+AAAAAAAAEKCJAAAA	Mrs.      	Keri                	Lawrence                      	Y	31	10	1955	GUERNSEY	NULL	Keri.Lawrence@yIBxLUPgozYINi.com                  	2452523	1523.98
+AAAAAAAAEMELAAAA	Dr.       	Steven              	Parker                        	N	11	9	1936	MADAGASCAR	NULL	Steven.Parker@E6kIVz3.org                         	2452372	2106.30
+AAAAAAAAEMHBAAAA	Mr.       	Brandon             	Ray                           	Y	13	8	1976	ANTARCTICA	NULL	Brandon.Ray@OP6YgS6SQnuykF.org                    	2452290	2258.53
+AAAAAAAAFCPEBAAA	Dr.       	Kristi              	Brennan                       	Y	NULL	11	NULL	NULL	NULL	NULL	2452349	6815.00
+AAAAAAAAFDBAAAAA	Mrs.      	Virginia            	Sims                          	Y	16	4	1969	BULGARIA	NULL	Virginia.Sims@3qndx2y.edu                         	2452302	6723.52
+AAAAAAAAFGICAAAA	Dr.       	Will                	Isbell                        	N	1	10	1975	ERITREA	NULL	Will.Isbell@100I71HVxMaaTVZ5MH2m.org              	2452482	1765.89
+AAAAAAAAFKGEBAAA	Dr.       	Dorothy             	Hendricks                     	N	14	9	1947	NAMIBIA	NULL	Dorothy.Hendricks@7X7OXy1xAs7hVN.org              	2452562	2333.76
+AAAAAAAAFKNEAAAA	Ms.       	Ruth                	Catron                        	Y	2	3	1983	SAINT LUCIA	NULL	Ruth.Catron@g4krLkSRUFX60t4P.edu                  	2452401	1873.40
+AAAAAAAAFKOHBAAA	Ms.       	Annie               	Stevens                       	N	12	2	1933	KENYA	NULL	Annie.Stevens@e3vFdRXUEQ.org                      	2452504	4229.82
+AAAAAAAAFMDEAAAA	Sir       	Dennis              	Mayfield                      	N	5	1	1933	SOMALIA	NULL	Dennis.Mayfield@9DPHTIUxRlxd.edu                  	2452535	2849.00
+AAAAAAAAFMIPAAAA	Dr.       	Michael             	Moran                         	N	1	2	1940	SAINT HELENA	NULL	Michael.Moran@3.edu                               	2452411	3869.60
+AAAAAAAAFPKCBAAA	Dr.       	William             	Terry                         	Y	10	3	1991	GUATEMALA	NULL	William.Terry@jKyKRKfrxH.org                      	2452325	3550.05
+AAAAAAAAFPPGAAAA	Ms.       	Margie              	Lee                           	Y	22	10	1933	GUINEA	NULL	Margie.Lee@Tg2pE7.com                             	2452419	3288.60
+AAAAAAAAGBOAAAAA	Mr.       	Micheal             	NULL	NULL	NULL	NULL	1991	NULL	NULL	Micheal.Holland@4rhkIkNJy6fSU.edu                 	2452530	2676.00
+AAAAAAAAGIEDAAAA	Miss      	Adriana             	Maxfield                      	Y	6	1	1948	MONGOLIA	NULL	Adriana.Maxfield@Na4ize7RHB.com                   	2452415	4872.96
+AAAAAAAAGJJEAAAA	Miss      	NULL	Barnard                       	N	NULL	NULL	1987	NULL	NULL	NULL	NULL	1628.77
+AAAAAAAAGMJPAAAA	Miss      	Deena               	Ferguson                      	Y	8	9	1986	PHILIPPINES	NULL	Deena.Ferguson@19lazfjoSTXBorQ.edu                	2452301	5713.74
+AAAAAAAAHANIAAAA	Mr.       	Robert              	Rogers                        	Y	5	7	1990	SOUTH AFRICA	NULL	Robert.Rogers@m77qeFKJfIO3rugyC.org               	2452456	1638.00
+AAAAAAAAHCHKAAAA	Dr.       	Harry               	Bearden                       	Y	10	2	1957	CHINA	NULL	Harry.Bearden@kNxzQ3SYEmp.com                     	2452533	6455.11
+AAAAAAAAHHAKAAAA	Dr.       	Jeremy              	Richter                       	Y	8	8	1951	SOUTH AFRICA	NULL	Jeremy.Richter@hCAQGRgTCrU.edu                    	2452496	3764.21
+AAAAAAAAHIMLAAAA	Dr.       	Kathleen            	Floyd                         	N	16	12	1983	TURKMENISTAN	NULL	Kathleen.Floyd@ACpkOV2nlHDL.edu                   	2452644	5796.90
+AAAAAAAAHLMCAAAA	Miss      	Adriana             	Case                          	N	16	3	1959	GREECE	NULL	Adriana.Case@aRv.com                              	2452408	1740.52
+AAAAAAAAHMFBBAAA	Dr.       	Katheryn            	White                         	Y	21	7	1983	CAPE VERDE	NULL	Katheryn.White@dqdzeup0a7TYhnOB.edu               	2452417	20518.95
+AAAAAAAAHMNFAAAA	Dr.       	Stephen             	Belanger                      	Y	10	11	1951	GUADELOUPE	NULL	Stephen.Belanger@emUp.edu                         	2452340	5734.80
+AAAAAAAAHNAOAAAA	Mrs.      	Anita               	Gardner                       	Y	27	6	1979	GEORGIA	NULL	Anita.Gardner@g.org                               	2452557	2921.42
+AAAAAAAAHOIBAAAA	Miss      	Vera                	Forte                         	N	20	9	1941	ZAMBIA	NULL	Vera.Forte@XxMnMUGRS.edu                          	2452549	2533.00
+AAAAAAAAHOOGAAAA	Sir       	Jerry               	Rader                         	Y	31	1	1948	PAPUA NEW GUINEA	NULL	Jerry.Rader@FAhBv4pGRIGB.com                      	2452327	3590.16
+AAAAAAAAHPGNAAAA	Dr.       	Renae               	Richardson                    	Y	4	8	1927	SYRIAN ARAB REPUBLIC	NULL	Renae.Richardson@eGnpquq6hFfm20r.com              	2452426	6468.92
+AAAAAAAAIAGBBAAA	Ms.       	Alexis              	Evans                         	N	3	12	1966	LITHUANIA	NULL	Alexis.Evans@2cqJZnt7z8.edu                       	2452596	2043.99
+AAAAAAAAICGBAAAA	Miss      	Gertrude            	Rodriguez                     	Y	10	4	1932	SAN MARINO	NULL	Gertrude.Rodriguez@lj4qAeL9afqB9jS.com            	2452447	2865.39
+AAAAAAAAIDFJAAAA	Dr.       	Dixie               	Brown                         	N	1	5	1983	FRENCH POLYNESIA	NULL	Dixie.Brown@PrD49KMoXd8SEMuCS.com                 	2452595	2135.70
+AAAAAAAAIJCHBAAA	Dr.       	Karen               	Cosby                         	Y	17	9	1963	IRAQ	NULL	Karen.Cosby@H17xf1rPlPtkdKTr.com                  	2452368	3731.50
+AAAAAAAAIJDEBAAA	Mrs.      	Elisa               	Baldwin                       	Y	14	5	1983	SWEDEN	NULL	Elisa.Baldwin@S.edu                               	2452579	3168.75
+AAAAAAAAILKLAAAA	Ms.       	Janice              	Cannon                        	N	11	12	1983	ARGENTINA	NULL	Janice.Cannon@lmR.org                             	2452353	2810.73
+AAAAAAAAINLKAAAA	Ms.       	Mandy               	Anderson                      	N	30	6	1958	MOZAMBIQUE	NULL	Mandy.Anderson@ONxvienlYHJpCe.org                 	2452539	9893.12
+AAAAAAAAIOGPAAAA	Ms.       	Kathryn             	Cooper                        	N	27	1	1982	ARGENTINA	NULL	Kathryn.Cooper@VBO8.com                           	2452457	1870.56
+AAAAAAAAJDKCAAAA	Dr.       	Curtis              	Mcguire                       	Y	10	5	1974	ETHIOPIA	NULL	Curtis.Mcguire@Vex25USDKvv.org                    	2452411	4900.15
+AAAAAAAAJFEFBAAA	Ms.       	Nancy               	Hampton                       	N	24	3	1946	WESTERN SAHARA	NULL	Nancy.Hampton@K8CudFMgtgyGyzS.com                 	2452309	6277.08
+AAAAAAAAJGNBAAAA	Sir       	Robert              	Johnston                      	N	14	5	1956	FAROE ISLANDS	NULL	Robert.Johnston@ccQCj7j.edu                       	2452631	2383.74
+AAAAAAAAJHHEBAAA	Miss      	Else                	Carter                        	Y	19	8	1963	NORFOLK ISLAND	NULL	Else.Carter@e.com                                 	2452580	4210.60
+AAAAAAAAJHIBAAAA	Mr.       	Edwin               	Christensen                   	N	23	6	1947	URUGUAY	NULL	Edwin.Christensen@3fBqkiUX.com                    	2452485	5984.00
+AAAAAAAAJJFBBAAA	Sir       	Michael             	Toney                         	Y	2	6	1981	COSTA RICA	NULL	Michael.Toney@Oe1SH.edu                           	2452306	3453.63
+AAAAAAAAJKDCBAAA	Dr.       	Robert              	James                         	Y	8	11	1929	LEBANON	NULL	Robert.James@DSVGYuMQLZNo9oga1.edu                	2452306	1922.62
+AAAAAAAAJLAGBAAA	Mrs.      	Isabel              	Barber                        	N	24	4	1964	ROMANIA	NULL	Isabel.Barber@0RSzJgpalSYmAoYJgnL.org             	2452435	2562.91
+AAAAAAAAJLICBAAA	Ms.       	Fumiko              	Ebert                         	N	5	12	1956	NAURU	NULL	Fumiko.Ebert@8kz.com                              	2452309	2408.38
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q31.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q31.sql.out
new file mode 100644
index 0000000000000..7ef2fb4948563
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q31.sql.out
@@ -0,0 +1,66 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<ca_county:string,d_year:int,web_q1_q2_increase:decimal(37,20),store_q1_q2_increase:decimal(37,20),web_q2_q3_increase:decimal(37,20),store_q2_q3_increase:decimal(37,20)>
+-- !query output
+Bedford city	2000	1.41895245428186244343	1.28647520475423955859	1.98397166014536480387	1.22052485500235683695
+Blaine County	2000	0.90103169326915114550	0.83066560812940517400	5.91832927336153981754	2.20534708400177561487
+Boone County	2000	1.10318140625440703802	1.01388129938545215917	2.93583104434541244580	0.81595222675278264090
+Bristol County	2000	2.03252306215265583168	0.84389707668347883239	1.88436946252627433938	0.50855932416819944462
+Buffalo County	2000	0.79851677786134640592	0.66608776395688494889	4.12963020014993273566	1.41417145222467738551
+Carlisle County	2000	1.69492598618078795489	1.25168931735787673049	1.50718182376907682203	0.88181897085363449075
+Carroll County	2000	1.35087769364534219248	1.29257051856006991407	2.20994529474585418644	1.48849785082428358420
+Cedar County	2000	1.73903789422605330870	0.46305877715234029287	8.77007422523370529780	3.61252572439703538764
+Clinton County	2000	1.74067151347304310575	1.41218893174550937759	1.44667681199587337982	1.25445563958487852628
+Colbert County	2000	1.18024333271178132777	0.83788572603247302022	0.54236640028023008306	0.42663333520116081271
+Crawford County	2000	1.31661755312891088929	0.94956543151027392277	2.05054841119621802578	1.32014748612874091613
+Cumberland County	2000	1.08524453978148728453	0.80144708440874338843	1.60419019384387788968	1.54733087343509046916
+Dillon County	2000	0.32644424794620832608	0.12281769967769250465	2.54843886762550432202	2.36794563665865531250
+Drew County	2000	1.33752592432768148542	0.42051684931974493150	4.42925583278116865936	3.59005466753091100322
+El Paso County	2000	1.33106683366793877022	0.52026046839940548659	6.85897871779069724650	3.52274370756107989319
+Escambia County	2000	1.29140013472373800933	1.22312591570259586531	4.94949857002487942979	1.90881493105514558681
+Essex County	2000	0.91140422255526176760	0.80397981840092167933	2.40581838121906819055	1.51850239999231343611
+Forsyth County	2000	0.68611318852553022140	0.65587408738915750878	3.37787340847752340465	1.32466135376522054649
+Franklin County	2000	1.23346105169430839663	1.16004116355148303419	3.11011280618208475383	1.72852883588875015022
+Fremont County	2000	2.08342397478237427534	0.99231216361555377660	4.64450632140050612837	1.66543397250235949142
+Gaines County	2000	0.18699408635518783605	0.15864149496085374898	4.38012199409197995027	3.26392730974685542866
+Gallatin County	2000	0.57267047178378396314	0.42960913482855774228	5.40426025792003986274	2.60129747557845045914
+Hale County	2000	1.13181285759861396698	0.37084472158598998345	2.76039662764995099764	2.04880905907673376536
+Howard County	2000	0.75689303418215066506	0.40416603686332285826	5.20017335230756289550	3.93113017208635232323
+Jennings County	2000	2.78984076175313855515	2.07599697431575122264	3.44972481162691656332	2.44378922187060355318
+Johnson County	2000	2.57795641894299816468	0.91616273582603658596	1.74313979276860348710	1.18351065602569093409
+Langlade County	2000	1.68676813359812642201	0.96428449439984805852	2.54522036089486624332	1.23454212891942526190
+Lee County	2000	1.30443690529253126871	0.72035286226320500421	2.23150088973480218462	1.84186089836441132231
+Limestone County	2000	3.32976919500999168869	0.75905050855357115484	1.82617549463146988019	1.71235192999491983271
+Lonoke County	2000	2.08982990182342723842	1.10340085014357551300	1.15539868818698913544	0.99289022395794532472
+Milam County	2000	2.28508195011379140646	0.46652927795640169049	0.78408081688146536737	0.47493776386092038861
+Mineral County	2000	3.34300539922925594005	2.45993211849872338822	2.20409211845792015572	1.08747970913885456325
+Mitchell County	2000	1.67453892386598503677	1.34365298077586851819	1.23346415770637208954	0.95073923197301166934
+Montgomery County	2000	1.56847414391333207258	0.83177598227578637405	2.13678203835469561935	2.10257648921068871721
+Moore County	2000	2.15694071746428807771	0.40796143296369785394	1.74966386964488096764	1.16471835770020727271
+Ottawa County	2000	1.75323872261604668329	0.83468698542567460304	1.61973935146544593444	1.42351390679742385404
+Pamlico County	2000	1.94782724762452970276	0.35933779785623271693	1.56217364489823888166	1.36537077933199287260
+Pawnee County	2000	1.78950195911489308028	1.16132303881758630677	3.42292675284956044317	0.55048211219399261479
+Pierce County	2000	2.54716933553443590360	1.41791053358023116142	1.22686753138250342480	1.16331445295301712751
+Pike County	2000	0.87756608639678817034	0.86487524757082952225	1.66450846493571420317	1.60752153195890666018
+Plymouth County	2000	0.42751024431730323179	0.24869467440858413355	1.77347026156010104138	0.95743334493327210261
+Pocahontas County	2000	1.81898673318791661987	1.68668175433656035469	2.78840493521175989923	2.31448886854618583141
+Pope County	2000	1.42325802767704214927	0.71289882499386886358	5.32400461162588232396	4.36612471915760924134
+Quitman County	2000	0.36883113368425186793	0.36314047030366149345	5.16370853507835506706	3.21226900707602706150
+Richland County	2000	3.25294641523656020681	1.07950401261765093476	2.63822689761623055166	1.41044850738739382893
+Rockland County	2000	0.42416114343500113082	0.20590865976918589352	3.77761891177163143690	3.03503154543555890597
+Salem city	2000	0.70139973333391227808	0.35839062570476603239	2.05906098316466157534	1.08354507133942051245
+Seminole County	2000	1.44053424523671633073	0.43123781775978624959	5.98509759915132505814	3.90425489309553190978
+Sherman County	2000	3.80549228460960365296	0.85018052505371507262	3.28429802752271011291	2.95060760395590827092
+Sioux County	2000	4.05794434246552571650	1.42665905155276116003	1.56803884104955881507	1.13419970938634791615
+Stafford County	2000	2.47320669600627229039	0.89946337873032839807	1.93845899918871104768	0.90136132522419968048
+Suffolk County	2000	2.42331649558212058212	1.82204858730891230166	6.53566355839665770211	1.95538998836281647012
+Sully County	2000	0.93205458749496189963	0.25090187329447210579	2.29778378271979174484	1.08802927271862880423
+Thomas County	2000	2.83931404059757943504	1.88036403728663893895	1.98634825955250608974	0.52141190626809911920
+Todd County	2000	2.82280525212475972969	0.89465756573513943151	3.16893938472753630323	1.86121311814899148488
+Valdez-Cordova Census Area	2000	1.25070886925917779038	0.59811263426251699657	1.22504083956972939990	0.84771422379395948262
+Waushara County	2000	7.21438509071107960599	3.30949368270859706271	0.60525565982701327843	0.60022615175317538544
+Wayne County	2000	0.94936213717458544629	0.78911846142834762231	2.95594340748058137863	1.96105182147450904414
+Williamson County	2000	6.40051753949925199369	0.73796090630650528678	2.74186195207121934591	2.27659084754941781749
+Woodford County	2000	1.66114001659859092820	0.55664623460469568767	1.82209706086687377022	1.17870749798601085737
+Wyoming County	2000	0.71452417457513699221	0.46814038549181972360	9.85342118017399816692	3.19644465036087064672
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q32.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q32.sql.out
new file mode 100644
index 0000000000000..de81b93b81cc1
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q32.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<excess discount amount :int>
+-- !query output
+1
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q33.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q33.sql.out
new file mode 100644
index 0000000000000..668a1c44a6907
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q33.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_manufact_id:int,total_sales:decimal(27,2)>
+-- !query output
+796	319.61
+820	445.38
+945	550.19
+753	655.92
+455	711.84
+564	739.49
+951	850.52
+776	1039.28
+676	1386.92
+911	1536.42
+832	1643.04
+925	1744.74
+701	1859.69
+707	1912.37
+780	2339.10
+891	2412.45
+890	3020.90
+870	3142.82
+656	3227.47
+681	3427.68
+985	3441.42
+881	3677.27
+717	3733.60
+854	3734.59
+449	4015.00
+965	4331.20
+960	4344.29
+621	4411.86
+756	4863.42
+918	5140.55
+795	5145.78
+833	5186.15
+976	5317.66
+696	5726.05
+853	5799.18
+770	5958.32
+722	6049.15
+362	6348.46
+399	6401.19
+973	6606.87
+715	6673.32
+708	6760.71
+537	7094.17
+724	7333.62
+375	7385.50
+444	7791.92
+716	7793.63
+887	7844.78
+464	7874.66
+709	7898.12
+978	7952.34
+893	8530.99
+677	8770.76
+595	8787.96
+869	9002.04
+338	9032.32
+738	9136.32
+429	9177.98
+650	9182.94
+901	9212.11
+921	9223.79
+946	9353.49
+858	9443.98
+905	9494.47
+568	9614.09
+389	9939.44
+836	9951.19
+821	9963.08
+728	10017.99
+679	10066.80
+721	10348.99
+910	10453.73
+698	10543.49
+777	10545.94
+675	10556.87
+783	10559.82
+624	10836.32
+763	10885.70
+730	10897.97
+463	10912.55
+534	10925.55
+913	10942.78
+970	11004.86
+846	11218.39
+742	11301.99
+794	11444.72
+442	11482.50
+393	11644.19
+706	11654.35
+577	11722.62
+929	11744.05
+733	11781.60
+896	11875.86
+807	11893.86
+386	12070.02
+865	12380.13
+561	12458.46
+552	12588.43
+683	12592.26
+997	12921.36
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q34.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q34.sql.out
new file mode 100644
index 0000000000000..71b7844e8e93a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q34.sql.out
@@ -0,0 +1,223 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<c_last_name:string,c_first_name:string,c_salutation:string,c_preferred_cust_flag:string,ss_ticket_number:int,cnt:bigint>
+-- !query output
+NULL	NULL	NULL	Y	47915	15
+NULL	NULL	NULL	NULL	126143	15
+NULL	NULL	NULL	NULL	215293	15
+NULL	NULL	Mrs.      	N	120593	15
+NULL	Rubin               	Sir       	NULL	30056	15
+Adler                         	Justin              	Sir       	Y	226187	15
+Allen                         	Rose                	Mrs.      	N	179476	16
+Anderson                      	Marvin              	Dr.       	N	211012	16
+Andrews                       	Jacob               	Mr.       	N	67111	16
+Andrews                       	Samuel              	Dr.       	Y	139993	16
+Angel                         	Kevin               	Mr.       	Y	106628	15
+Ashley                        	Linda               	Mrs.      	Y	82173	15
+Baca                          	Dorothy             	Mrs.      	N	64890	15
+Baker                         	Jamie               	Dr.       	Y	9916	15
+Banks                         	Leroy               	Sir       	N	206730	15
+Barber                        	Dianna              	Mrs.      	Y	119959	16
+Barksdale                     	Joann               	Miss      	Y	138994	15
+Barnes                        	Ruth                	Dr.       	N	84038	15
+Barney                        	Samuel              	Sir       	N	15288	15
+Barnhart                      	Charley             	Mr.       	Y	166576	15
+Barone                        	Seth                	Mr.       	Y	162374	15
+Barrett                       	David               	Sir       	N	189879	15
+Bartels                       	Elmer               	Sir       	Y	114760	16
+Bear                          	Scott               	Sir       	Y	82291	15
+Beers                         	Kendra              	Dr.       	NULL	137960	15
+Belcher                       	James               	Sir       	Y	239470	16
+Bell                          	Carrie              	Miss      	N	5527	15
+Bell                          	Matthew             	Dr.       	N	20400	15
+Benjamin                      	Consuelo            	Ms.       	Y	201086	15
+Bergman                       	Joann               	Miss      	N	177052	15
+Brooks                        	Robert              	Sir       	N	155576	16
+Byrd                          	Kelly               	Sir       	N	165115	16
+Cagle                         	Jennifer            	Miss      	N	163129	15
+Campbell                      	Robert              	Mr.       	N	8964	15
+Cardona                       	Robert              	Mr.       	N	200501	15
+Carter                        	Wendy               	Ms.       	N	96439	15
+Carver                        	Bernard             	Mr.       	Y	194943	16
+Chen                          	Wanita              	Miss      	N	137713	16
+Christensen                   	Larry               	Dr.       	Y	58094	15
+Cochrane                      	Anne                	Mrs.      	N	208347	16
+Coleman                       	Inez                	Dr.       	Y	88249	16
+Coleman                       	John                	Mr.       	N	49444	15
+Colon                         	Anna                	Dr.       	Y	143694	15
+Conley                        	Roxie               	Dr.       	N	196663	15
+Cook                          	Adam                	Ms.       	Y	167339	15
+Cote                          	Justin              	Dr.       	N	93466	15
+Council                       	Donald              	Sir       	Y	102958	15
+Cramer                        	Linda               	Ms.       	N	126628	15
+Crittenden                    	Amie                	Ms.       	N	138787	15
+Cruz                          	James               	Mr.       	Y	201430	15
+Cuellar                       	Oscar               	Mr.       	Y	86781	16
+Cullen                        	Larry               	Mr.       	Y	221242	16
+Cushing                       	Antonia             	Mrs.      	Y	118927	15
+Davis                         	Gordon              	Dr.       	N	227822	15
+Davis                         	Myrtle              	Dr.       	Y	37430	15
+Decker                        	Vera                	Miss      	Y	75737	16
+Diamond                       	Fernando            	Dr.       	N	216391	15
+Diaz                          	Walton              	Mr.       	N	131135	16
+Dickinson                     	Steven              	Mr.       	N	8057	16
+Douglas                       	Lester              	Sir       	N	26043	15
+Dove                          	Garry               	Dr.       	N	152171	16
+Drake                         	Rosetta             	Dr.       	Y	238040	15
+Dumas                         	Travis              	Mr.       	Y	94154	15
+Duncan                        	Olivia              	Dr.       	Y	102032	15
+Durham                        	Andrea              	Dr.       	Y	144734	15
+Dutton                        	Gay                 	Miss      	Y	110886	15
+Ellis                         	Karen               	Miss      	N	229706	16
+Ely                           	Cesar               	Dr.       	Y	36054	16
+Etheridge                     	Mike                	Dr.       	N	19648	15
+Farmer                        	Eugenia             	Miss      	Y	98187	16
+Farrow                        	Kathy               	Miss      	Y	200078	15
+Faulkner                      	Lakeisha            	Dr.       	Y	178393	16
+Faulkner                      	Robert              	Dr.       	N	109423	15
+Felton                        	David               	Mr.       	N	97807	16
+Ferreira                      	Christine           	Mrs.      	Y	155822	15
+Finn                          	Robert              	Mr.       	N	38057	15
+Finney                        	Crystal             	Miss      	Y	158304	15
+Fischer                       	Tamara              	Mrs.      	N	66790	15
+Foote                         	Roy                 	Sir       	N	68086	15
+Foreman                       	Autumn              	Mrs.      	Y	164060	15
+Funk                          	Marvin              	Sir       	Y	61516	15
+Garcia                        	Christopher         	Sir       	Y	181616	16
+Garcia                        	Karen               	Miss      	N	236987	15
+Garcia                        	Robert              	Dr.       	N	172185	16
+Garland                       	Michael             	Mr.       	N	234421	15
+Gaylord                       	Keith               	Mr.       	Y	123333	16
+Gifford                       	Mark                	Mr.       	N	225973	16
+Gilbert                       	NULL	Sir       	N	16844	15
+Gilmore                       	Austin              	Dr.       	Y	239871	15
+Goldsmith                     	Bernice             	Ms.       	Y	2347	15
+Good                          	Nancy               	Dr.       	N	132655	15
+Goodman                       	NULL	NULL	N	71903	15
+Gower                         	Nettie              	Miss      	N	10576	15
+Gray                          	Evelyn              	Miss      	N	157486	15
+Hammond                       	Roger               	Sir       	Y	54884	16
+Hardin                        	Kimberly            	Dr.       	N	192424	16
+Harp                          	Vance               	Mr.       	N	199017	15
+Harper                        	Madeline            	Dr.       	N	173835	16
+Harris                        	Tammy               	Dr.       	N	217761	16
+Hartmann                      	Joey                	Dr.       	N	230915	15
+Hayes                         	David               	Sir       	N	82274	15
+Haynes                        	Sara                	Miss      	Y	139168	16
+Heath                         	Matthew             	Dr.       	N	30710	15
+Hennessey                     	Debbie              	Dr.       	Y	79256	15
+Herman                        	Stella              	Ms.       	Y	33801	16
+Hernandez                     	Max                 	Mr.       	N	16015	15
+Hernandez                     	Ruth                	Ms.       	Y	97000	15
+Hess                          	Joseph              	Sir       	N	151336	15
+Hodges                        	Lucas               	Dr.       	Y	163325	15
+Holland                       	Jeremiah            	Dr.       	N	95938	16
+Jackson                       	William             	Mr.       	Y	16425	16
+Jameson                       	Miguel              	Dr.       	N	9181	16
+Jarrell                       	Thomas              	Mr.       	Y	85787	16
+Johnson                       	Julia               	Dr.       	N	27560	15
+Jones                         	Theresa             	Ms.       	N	219765	16
+Kelly                         	Mark                	Mr.       	Y	17039	16
+Khan                          	Hank                	Mr.       	N	177803	15
+Kim                           	Charlotte           	Dr.       	Y	7208	16
+Kunz                          	Sarah               	Dr.       	N	74568	15
+Lake                          	Robert              	Sir       	N	13264	15
+Landry                        	Rudolph             	Sir       	N	117581	15
+Lane                          	Luis                	Sir       	N	232302	16
+Langford                      	Darlene             	Mrs.      	N	214891	15
+Larson                        	Kevin               	Mr.       	Y	35053	15
+Larson                        	Thomas              	Mr.       	N	114265	15
+Lee                           	Malik               	Dr.       	N	20122	16
+Leonard                       	Orlando             	Dr.       	Y	133168	15
+Lincoln                       	Anthony             	Miss      	Y	1407	16
+Lindsey                       	Linda               	Dr.       	N	62687	16
+Lopez                         	Karen               	Dr.       	Y	136008	15
+Lunsford                      	Kevin               	Dr.       	N	159120	16
+Lynch                         	Sylvia              	Ms.       	Y	115438	15
+Lyon                          	Michael             	Mr.       	N	140323	15
+Maestas                       	Mabel               	Mrs.      	N	184265	15
+Magana                        	Diann               	Mrs.      	Y	19139	15
+Manning                       	Annamarie           	Ms.       	N	4984	16
+Marshall                      	Felipe              	Sir       	N	138890	15
+Martin                        	Paul                	Dr.       	N	26115	16
+Martinez                      	Earl                	Sir       	N	108982	15
+Martinez                      	Robert              	Sir       	Y	157672	16
+Masterson                     	Barbara             	Mrs.      	N	231070	15
+Mata                          	Deborah             	Miss      	Y	4323	15
+Mccoy                         	Debbie              	Dr.       	N	91552	15
+Mcgill                        	Tony                	Sir       	N	110030	15
+Mckeon                        	Christina           	Dr.       	N	26190	15
+Mcnamara                      	Linda               	Dr.       	Y	7957	15
+Means                         	Michael             	Mr.       	Y	226164	16
+Medina                        	Joseph              	Sir       	Y	110246	15
+Meyers                        	Zachary             	Mr.       	Y	59549	15
+Montgomery                    	John                	Mr.       	Y	103718	15
+Moody                         	Miranda             	Ms.       	Y	171671	15
+Moore                         	Mark                	Dr.       	N	191471	15
+Moran                         	Celia               	Ms.       	Y	200691	15
+Morgan                        	Cecelia             	Mrs.      	N	200742	15
+Morrell                       	Chad                	Mr.       	N	93790	15
+Morse                         	Robert              	Mr.       	N	68627	16
+Neel                          	Audrey              	Ms.       	Y	193308	15
+Neff                          	Sheri               	Mrs.      	Y	52556	15
+Nelson                        	Katherine           	Mrs.      	N	110232	15
+New                           	Suzanne             	Miss      	N	5120	16
+Nielsen                       	Veronica            	Mrs.      	N	23905	15
+Oakley                        	George              	Mr.       	Y	177890	15
+Parker                        	Barbar              	Dr.       	N	57241	15
+Parker                        	Jeff                	Sir       	N	213566	16
+Pemberton                     	Jennifer            	Mrs.      	Y	49875	16
+Perry                         	Robert              	Mr.       	Y	153147	16
+Phillips                      	David               	Dr.       	N	148883	15
+Phillips                      	Georgia             	NULL	Y	26878	15
+Phillips                      	Stanley             	Sir       	N	31989	15
+Pinkston                      	Brenda              	Dr.       	N	126440	15
+Pryor                         	Dorothy             	Mrs.      	N	213779	16
+Reed                          	William             	Dr.       	N	145002	15
+Reynolds                      	Amelia              	Ms.       	Y	68440	15
+Rice                          	David               	Dr.       	Y	70484	16
+Robertson                     	Daniel              	Mr.       	N	40407	16
+Rosales                       	NULL	NULL	Y	156406	16
+Russo                         	Cheryl              	Miss      	N	81123	15
+Sanchez                       	Bruce               	Sir       	Y	124479	15
+Schmitz                       	Kaitlyn             	Miss      	N	105162	15
+Sebastian                     	Homer               	Dr.       	Y	64994	15
+Sexton                        	Jerry               	Sir       	N	91446	15
+Sierra                        	David               	Sir       	Y	61810	15
+Simmons                       	Joseph              	Dr.       	N	54185	15
+Simpson                       	Michael             	Sir       	Y	186613	16
+Simpson                       	Shalanda            	Dr.       	Y	181123	15
+Singleton                     	Andrew              	Ms.       	N	45464	15
+Smith                         	Danny               	Dr.       	Y	143297	15
+Smith                         	Edward              	Sir       	Y	81178	16
+Smith                         	Hung                	Sir       	N	44710	15
+Smith                         	Kimberly            	Mrs.      	Y	174638	15
+Smith                         	Vern                	Sir       	N	50960	15
+Sosa                          	Leah                	Ms.       	Y	77106	16
+Sparks                        	Erick               	Dr.       	N	220337	15
+Taylor                        	Kenneth             	Dr.       	Y	194337	15
+Todd                          	Linda               	Ms.       	Y	235816	15
+Trout                         	Harley              	Mr.       	Y	214547	15
+Urban                         	NULL	NULL	NULL	214898	15
+Varner                        	Elsie               	Ms.       	N	199602	16
+Vazquez                       	Bill                	Dr.       	Y	62049	15
+Velazquez                     	William             	Dr.       	N	46239	15
+Wagner                        	Barbara             	Ms.       	Y	233595	15
+Ward                          	Anna                	Miss      	N	52941	16
+Watkins                       	Rosa                	Miss      	Y	152190	16
+Welch                         	James               	Dr.       	Y	51441	16
+West                          	Teresa              	Ms.       	N	233179	16
+White                         	Maurice             	Mr.       	N	10107	15
+Williams                      	John                	Mr.       	Y	84783	15
+Williams                      	Robert              	Mr.       	Y	41233	15
+Williamson                    	Ruth                	Mrs.      	Y	86369	15
+Wilson                        	Joel                	Sir       	Y	91826	16
+Wilson                        	John                	Sir       	Y	26543	15
+Wilson                        	Mariano             	Mr.       	Y	67472	16
+Winkler                       	Jose                	Dr.       	Y	78400	15
+Winter                        	Cora                	Mrs.      	N	8978	16
+Wood                          	Marcia              	Ms.       	Y	219276	16
+Wood                          	Michelle            	Mrs.      	N	39560	15
+Wright                        	Richie              	Sir       	Y	106818	15
+Young                         	William             	Mr.       	Y	51127	15
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q35.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q35.sql.out
new file mode 100644
index 0000000000000..3307163872566
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q35.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<ca_state:string,cd_gender:string,cd_marital_status:string,cnt1:bigint,min(cd_dep_count):int,max(cd_dep_count):int,avg(cd_dep_count):double,cd_dep_employed_count:int,cnt2:bigint,min(cd_dep_employed_count):int,max(cd_dep_employed_count):int,avg(cd_dep_employed_count):double,cd_dep_college_count:int,cnt3:bigint,min(cd_dep_college_count):int,max(cd_dep_college_count):int,avg(cd_dep_college_count):double>
+-- !query output
+NULL	F	D	1	0	0	0.0	2	1	2	2	2.0	2	1	2	2	2.0
+NULL	F	D	1	0	0	0.0	3	1	3	3	3.0	4	1	4	4	4.0
+NULL	F	D	1	0	0	0.0	5	1	5	5	5.0	2	1	2	2	2.0
+NULL	F	D	1	0	0	0.0	6	1	6	6	6.0	4	1	4	4	4.0
+NULL	F	D	1	1	1	1.0	1	1	1	1	1.0	1	1	1	1	1.0
+NULL	F	D	1	1	1	1.0	4	1	4	4	4.0	4	1	4	4	4.0
+NULL	F	D	1	1	1	1.0	4	1	4	4	4.0	5	1	5	5	5.0
+NULL	F	D	1	2	2	2.0	0	1	0	0	0.0	4	1	4	4	4.0
+NULL	F	D	1	2	2	2.0	1	1	1	1	1.0	3	1	3	3	3.0
+NULL	F	D	1	2	2	2.0	6	1	6	6	6.0	1	1	1	1	1.0
+NULL	F	D	1	3	3	3.0	3	1	3	3	3.0	2	1	2	2	2.0
+NULL	F	D	1	3	3	3.0	3	1	3	3	3.0	6	1	6	6	6.0
+NULL	F	D	1	3	3	3.0	4	1	4	4	4.0	1	1	1	1	1.0
+NULL	F	D	1	4	4	4.0	0	1	0	0	0.0	3	1	3	3	3.0
+NULL	F	D	1	4	4	4.0	1	1	1	1	1.0	1	1	1	1	1.0
+NULL	F	D	1	4	4	4.0	1	1	1	1	1.0	4	1	4	4	4.0
+NULL	F	D	1	4	4	4.0	5	1	5	5	5.0	6	1	6	6	6.0
+NULL	F	D	1	5	5	5.0	4	1	4	4	4.0	3	1	3	3	3.0
+NULL	F	D	1	5	5	5.0	5	1	5	5	5.0	2	1	2	2	2.0
+NULL	F	D	1	6	6	6.0	1	1	1	1	1.0	3	1	3	3	3.0
+NULL	F	D	1	6	6	6.0	2	1	2	2	2.0	2	1	2	2	2.0
+NULL	F	D	1	6	6	6.0	4	1	4	4	4.0	1	1	1	1	1.0
+NULL	F	M	1	0	0	0.0	5	1	5	5	5.0	5	1	5	5	5.0
+NULL	F	M	1	1	1	1.0	3	1	3	3	3.0	0	1	0	0	0.0
+NULL	F	M	1	1	1	1.0	6	1	6	6	6.0	0	1	0	0	0.0
+NULL	F	M	1	1	1	1.0	6	1	6	6	6.0	1	1	1	1	1.0
+NULL	F	M	1	2	2	2.0	2	1	2	2	2.0	6	1	6	6	6.0
+NULL	F	M	1	2	2	2.0	4	1	4	4	4.0	4	1	4	4	4.0
+NULL	F	M	1	3	3	3.0	2	1	2	2	2.0	1	1	1	1	1.0
+NULL	F	M	1	3	3	3.0	5	1	5	5	5.0	0	1	0	0	0.0
+NULL	F	M	1	3	3	3.0	5	1	5	5	5.0	1	1	1	1	1.0
+NULL	F	M	1	4	4	4.0	1	1	1	1	1.0	4	1	4	4	4.0
+NULL	F	M	1	4	4	4.0	2	1	2	2	2.0	1	1	1	1	1.0
+NULL	F	M	1	4	4	4.0	3	1	3	3	3.0	3	1	3	3	3.0
+NULL	F	M	1	5	5	5.0	2	1	2	2	2.0	2	1	2	2	2.0
+NULL	F	M	1	6	6	6.0	1	1	1	1	1.0	1	1	1	1	1.0
+NULL	F	M	1	6	6	6.0	5	1	5	5	5.0	6	1	6	6	6.0
+NULL	F	S	1	0	0	0.0	3	1	3	3	3.0	6	1	6	6	6.0
+NULL	F	S	1	1	1	1.0	0	1	0	0	0.0	4	1	4	4	4.0
+NULL	F	S	1	1	1	1.0	1	1	1	1	1.0	2	1	2	2	2.0
+NULL	F	S	1	1	1	1.0	2	1	2	2	2.0	6	1	6	6	6.0
+NULL	F	S	1	1	1	1.0	5	1	5	5	5.0	5	1	5	5	5.0
+NULL	F	S	1	2	2	2.0	0	1	0	0	0.0	3	1	3	3	3.0
+NULL	F	S	2	2	2	2.0	5	2	5	5	5.0	6	2	6	6	6.0
+NULL	F	S	1	3	3	3.0	0	1	0	0	0.0	4	1	4	4	4.0
+NULL	F	S	1	3	3	3.0	2	1	2	2	2.0	1	1	1	1	1.0
+NULL	F	S	1	3	3	3.0	2	1	2	2	2.0	5	1	5	5	5.0
+NULL	F	S	1	3	3	3.0	3	1	3	3	3.0	3	1	3	3	3.0
+NULL	F	S	1	4	4	4.0	1	1	1	1	1.0	4	1	4	4	4.0
+NULL	F	S	1	4	4	4.0	2	1	2	2	2.0	4	1	4	4	4.0
+NULL	F	S	1	5	5	5.0	6	1	6	6	6.0	0	1	0	0	0.0
+NULL	F	U	1	0	0	0.0	1	1	1	1	1.0	3	1	3	3	3.0
+NULL	F	U	1	0	0	0.0	3	1	3	3	3.0	0	1	0	0	0.0
+NULL	F	U	1	1	1	1.0	3	1	3	3	3.0	2	1	2	2	2.0
+NULL	F	U	1	1	1	1.0	5	1	5	5	5.0	6	1	6	6	6.0
+NULL	F	U	1	2	2	2.0	0	1	0	0	0.0	1	1	1	1	1.0
+NULL	F	U	1	2	2	2.0	4	1	4	4	4.0	4	1	4	4	4.0
+NULL	F	U	2	3	3	3.0	1	2	1	1	1.0	6	2	6	6	6.0
+NULL	F	U	1	4	4	4.0	0	1	0	0	0.0	4	1	4	4	4.0
+NULL	F	U	1	5	5	5.0	3	1	3	3	3.0	6	1	6	6	6.0
+NULL	F	U	1	6	6	6.0	2	1	2	2	2.0	2	1	2	2	2.0
+NULL	F	U	1	6	6	6.0	4	1	4	4	4.0	4	1	4	4	4.0
+NULL	F	U	1	6	6	6.0	5	1	5	5	5.0	0	1	0	0	0.0
+NULL	F	U	1	6	6	6.0	5	1	5	5	5.0	6	1	6	6	6.0
+NULL	F	W	1	0	0	0.0	0	1	0	0	0.0	4	1	4	4	4.0
+NULL	F	W	1	0	0	0.0	5	1	5	5	5.0	5	1	5	5	5.0
+NULL	F	W	1	1	1	1.0	3	1	3	3	3.0	4	1	4	4	4.0
+NULL	F	W	1	2	2	2.0	0	1	0	0	0.0	5	1	5	5	5.0
+NULL	F	W	1	3	3	3.0	3	1	3	3	3.0	6	1	6	6	6.0
+NULL	F	W	1	3	3	3.0	6	1	6	6	6.0	6	1	6	6	6.0
+NULL	F	W	1	4	4	4.0	3	1	3	3	3.0	1	1	1	1	1.0
+NULL	F	W	1	5	5	5.0	1	1	1	1	1.0	1	1	1	1	1.0
+NULL	F	W	1	5	5	5.0	1	1	1	1	1.0	4	1	4	4	4.0
+NULL	F	W	1	5	5	5.0	3	1	3	3	3.0	6	1	6	6	6.0
+NULL	F	W	1	5	5	5.0	4	1	4	4	4.0	6	1	6	6	6.0
+NULL	F	W	1	6	6	6.0	0	1	0	0	0.0	5	1	5	5	5.0
+NULL	F	W	1	6	6	6.0	2	1	2	2	2.0	3	1	3	3	3.0
+NULL	F	W	1	6	6	6.0	5	1	5	5	5.0	5	1	5	5	5.0
+NULL	M	D	1	0	0	0.0	3	1	3	3	3.0	0	1	0	0	0.0
+NULL	M	D	1	1	1	1.0	3	1	3	3	3.0	0	1	0	0	0.0
+NULL	M	D	1	1	1	1.0	3	1	3	3	3.0	2	1	2	2	2.0
+NULL	M	D	1	2	2	2.0	0	1	0	0	0.0	6	1	6	6	6.0
+NULL	M	D	1	2	2	2.0	4	1	4	4	4.0	4	1	4	4	4.0
+NULL	M	D	1	2	2	2.0	5	1	5	5	5.0	3	1	3	3	3.0
+NULL	M	D	1	3	3	3.0	1	1	1	1	1.0	5	1	5	5	5.0
+NULL	M	D	1	3	3	3.0	2	1	2	2	2.0	3	1	3	3	3.0
+NULL	M	D	1	4	4	4.0	5	1	5	5	5.0	2	1	2	2	2.0
+NULL	M	D	1	6	6	6.0	1	1	1	1	1.0	6	1	6	6	6.0
+NULL	M	D	1	6	6	6.0	3	1	3	3	3.0	1	1	1	1	1.0
+NULL	M	M	1	0	0	0.0	0	1	0	0	0.0	1	1	1	1	1.0
+NULL	M	M	2	0	0	0.0	1	2	1	1	1.0	2	2	2	2	2.0
+NULL	M	M	1	0	0	0.0	2	1	2	2	2.0	1	1	1	1	1.0
+NULL	M	M	1	0	0	0.0	3	1	3	3	3.0	5	1	5	5	5.0
+NULL	M	M	1	0	0	0.0	5	1	5	5	5.0	0	1	0	0	0.0
+NULL	M	M	1	1	1	1.0	0	1	0	0	0.0	1	1	1	1	1.0
+NULL	M	M	1	1	1	1.0	0	1	0	0	0.0	2	1	2	2	2.0
+NULL	M	M	1	2	2	2.0	6	1	6	6	6.0	5	1	5	5	5.0
+NULL	M	M	1	3	3	3.0	5	1	5	5	5.0	1	1	1	1	1.0
+NULL	M	M	1	3	3	3.0	6	1	6	6	6.0	4	1	4	4	4.0
+NULL	M	M	1	4	4	4.0	1	1	1	1	1.0	3	1	3	3	3.0
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q36.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q36.sql.out
new file mode 100644
index 0000000000000..9fd97cbed90ed
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q36.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<gross_margin:decimal(37,20),i_category:string,i_class:string,lochierarchy:tinyint,rank_within_parent:int>
+-- !query output
+-0.43310777864678831165	NULL	NULL	2	1
+-0.44057752675240550259	Home                                              	NULL	1	1
+-0.43759152110176221048	Music                                             	NULL	1	2
+-0.43708103961494058652	NULL	NULL	1	3
+-0.43616253138693450880	Shoes                                             	NULL	1	4
+-0.43567118609322457134	Children                                          	NULL	1	5
+-0.43423932351647837678	Sports                                            	NULL	1	6
+-0.43342977299642408093	Electronics                                       	NULL	1	7
+-0.43243283120699560700	Women                                             	NULL	1	8
+-0.43164166899823508408	Men                                               	NULL	1	9
+-0.42516187689954540402	Books                                             	NULL	1	10
+-0.42448713380832790884	Jewelry                                           	NULL	1	11
+-0.73902664238792748962	NULL	shirts                                            	0	1
+-0.61125804873635587486	NULL	country                                           	0	2
+-0.53129803597069255822	NULL	dresses                                           	0	3
+-0.51266635289382758517	NULL	athletic                                          	0	4
+-0.45290387783638603924	NULL	mens                                              	0	5
+-0.41288056661656330013	NULL	accessories                                       	0	6
+-0.40784754677005682440	NULL	NULL	0	7
+-0.34254844860867375832	NULL	baseball                                          	0	8
+-0.32511461675631534897	NULL	infants                                           	0	9
+-0.44733955704648003493	Books                                             	computers                                         	0	1
+-0.44221358112622373783	Books                                             	home repair                                       	0	2
+-0.44131129175272951442	Books                                             	romance                                           	0	3
+-0.43954111564375046074	Books                                             	history                                           	0	4
+-0.43921337505389731821	Books                                             	mystery                                           	0	5
+-0.43904020269360481109	Books                                             	sports                                            	0	6
+-0.42821476999837619396	Books                                             	travel                                            	0	7
+-0.42609067296303848297	Books                                             	cooking                                           	0	8
+-0.42538995145338568328	Books                                             	fiction                                           	0	9
+-0.42446563616188232944	Books                                             	arts                                              	0	10
+-0.42424821311884350413	Books                                             	parenting                                         	0	11
+-0.41822014479424203008	Books                                             	reference                                         	0	12
+-0.41350839325516811781	Books                                             	business                                          	0	13
+-0.40935208137315013129	Books                                             	science                                           	0	14
+-0.40159380735731858928	Books                                             	self-help                                         	0	15
+-0.36957884843305744526	Books                                             	entertainments                                    	0	16
+-0.44602461556731552282	Children                                          	school-uniforms                                   	0	1
+-0.44141106040000560852	Children                                          	toddlers                                          	0	2
+-0.43479886701046623711	Children                                          	infants                                           	0	3
+-0.41900662971936329442	Children                                          	newborn                                           	0	4
+-0.41526603781609697786	Children                                          	NULL	0	5
+-0.45347482218635333366	Electronics                                       	personal                                          	0	1
+-0.44349670349829474271	Electronics                                       	stereo                                            	0	2
+-0.44262427232850112058	Electronics                                       	automotive                                        	0	3
+-0.44115886172705231970	Electronics                                       	portable                                          	0	4
+-0.43972786651639318010	Electronics                                       	memory                                            	0	5
+-0.43889275271590953040	Electronics                                       	scanners                                          	0	6
+-0.43879181695132886061	Electronics                                       	karoke                                            	0	7
+-0.43743655149948399284	Electronics                                       	dvd/vcr players                                   	0	8
+-0.43737666390514154910	Electronics                                       	cameras                                           	0	9
+-0.43390499017233926812	Electronics                                       	wireless                                          	0	10
+-0.43163869754114299547	Electronics                                       	audio                                             	0	11
+-0.42958938669780912634	Electronics                                       	camcorders                                        	0	12
+-0.42872845803629855724	Electronics                                       	musical                                           	0	13
+-0.42228240153396399656	Electronics                                       	televisions                                       	0	14
+-0.41893847772039275795	Electronics                                       	monitors                                          	0	15
+-0.39793878022746331540	Electronics                                       	disk drives                                       	0	16
+-0.49051156860507320113	Home                                              	NULL	0	1
+-0.48431476750686752965	Home                                              	blinds/shades                                     	0	2
+-0.47545837941951440918	Home                                              	bathroom                                          	0	3
+-0.45726228921216284093	Home                                              	rugs                                              	0	4
+-0.45540507568891021759	Home                                              	furniture                                         	0	5
+-0.45303572267019508501	Home                                              	flatware                                          	0	6
+-0.44755542058111800358	Home                                              	tables                                            	0	7
+-0.44419847780930149402	Home                                              	wallpaper                                         	0	8
+-0.44092345226680695671	Home                                              	glassware                                         	0	9
+-0.43877591834074789745	Home                                              	decor                                             	0	10
+-0.43765482553654514822	Home                                              	accent                                            	0	11
+-0.43188199218974854630	Home                                              	bedding                                           	0	12
+-0.43107417904272222899	Home                                              	kids                                              	0	13
+-0.42474436355625900935	Home                                              	lighting                                          	0	14
+-0.41783311109052416746	Home                                              	curtains/drapes                                   	0	15
+-0.41767111806961188479	Home                                              	mattresses                                        	0	16
+-0.40562188698541221499	Home                                              	paint                                             	0	17
+-0.45165056505480816921	Jewelry                                           	jewelry boxes                                     	0	1
+-0.44372227804836590137	Jewelry                                           	estate                                            	0	2
+-0.44251815032563188894	Jewelry                                           	gold                                              	0	3
+-0.43978127753996883542	Jewelry                                           	consignment                                       	0	4
+-0.43821750044359339153	Jewelry                                           	custom                                            	0	5
+-0.43439645036479672989	Jewelry                                           	bracelets                                         	0	6
+-0.43208398325687772942	Jewelry                                           	loose stones                                      	0	7
+-0.43060897375114375156	Jewelry                                           	diamonds                                          	0	8
+-0.42847505748860847066	Jewelry                                           	costume                                           	0	9
+-0.42667449062277843561	Jewelry                                           	rings                                             	0	10
+-0.41987969011585456826	Jewelry                                           	mens watch                                        	0	11
+-0.41624621972944533035	Jewelry                                           	semi-precious                                     	0	12
+-0.41148949162100715771	Jewelry                                           	womens watch                                      	0	13
+-0.39725668174847694299	Jewelry                                           	birdal                                            	0	14
+-0.39665274051903254057	Jewelry                                           	pendants                                          	0	15
+-0.38423525233438861010	Jewelry                                           	earings                                           	0	16
+-0.44464388887858793403	Men                                               	shirts                                            	0	1
+-0.43719860800637369827	Men                                               	accessories                                       	0	2
+-0.43164606665359630905	Men                                               	sports-apparel                                    	0	3
+-0.41530906677293519754	Men                                               	pants                                             	0	4
+-0.38332708894803499123	Men                                               	NULL	0	5
+-0.47339698705534020269	Music                                             	NULL	0	1
+-0.44193214675249008923	Music                                             	rock                                              	0	2
+-0.44008174913565459246	Music                                             	country                                           	0	3
+-0.43863444992223641373	Music                                             	pop                                               	0	4
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q37.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q37.sql.out
new file mode 100644
index 0000000000000..1782f9025fa1e
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q37.sql.out
@@ -0,0 +1,7 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_item_id:string,i_item_desc:string,i_current_price:decimal(7,2)>
+-- !query output
+AAAAAAAADHGDAAAA	Necessary times believe probably. Cruel traders know ho	92.95
+AAAAAAAAFMLDAAAA	Given groups please unfortu	84.79
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q38.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q38.sql.out
new file mode 100644
index 0000000000000..7bcc92dc20802
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q38.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<count(1):bigint>
+-- !query output
+104
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q39a.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q39a.sql.out
new file mode 100644
index 0000000000000..f2b8dc07af08d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q39a.sql.out
@@ -0,0 +1,211 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<w_warehouse_sk:int,i_item_sk:int,d_moy:int,mean:double,cov:double,w_warehouse_sk:int,i_item_sk:int,d_moy:int,mean:double,cov:double>
+-- !query output
+1	823	1	301.5	1.1271370519097714	1	823	2	399.25	1.0208768007427147
+1	1015	1	344.25	1.2511428113709673	1	1015	2	385.5	1.0470314051933909
+1	1555	1	264.25	1.2283186159550554	1	1555	2	308.25	1.058767914170086
+1	1691	1	116.33333333333333	1.0248470977646387	1	1691	2	304.75	1.1191291694661885
+1	1859	1	434.0	1.0498116619056204	1	1859	2	314.0	1.1900690532464868
+1	2239	1	322.5	1.0978977568088866	1	2239	2	382.75	1.0474268515036576
+1	3859	1	352.25	1.1782246058681007	1	3859	2	437.5	1.145917046540028
+1	4975	1	388.5	1.0001018617093298	1	4975	2	363.3333333333333	1.0038038922115327
+1	5109	1	370.75	1.1395542642353356	1	5109	2	322.25	1.1495609463470746
+1	5177	1	292.0	1.3461430973363442	1	5177	2	216.25	1.05761161489175
+1	6619	1	409.5	1.0040570532236324	1	6619	2	322.3333333333333	1.0619035677903699
+1	7535	1	428.25	1.0544682695099616	1	7535	2	391.5	1.222946431626478
+1	8283	1	194.25	1.2003971691624762	1	8283	2	400.6666666666667	1.1209666759436547
+1	8401	1	328.25	1.0063843177755347	1	8401	2	297.75	1.4183634329309731
+1	8547	1	328.25	1.1425055398027557	1	8547	2	264.5	1.0416250678095451
+1	8717	1	228.5	1.2085987127693683	1	8717	2	288.25	1.2149351569032296
+1	8933	1	361.0	1.1736874031675129	1	8933	2	433.0	1.0089048763341881
+1	8997	1	250.0	1.035266149354841	1	8997	2	430.6666666666667	1.1313205669952624
+1	9245	1	392.3333333333333	1.1119320768290193	1	9245	2	343.0	1.1904615139612167
+1	9621	1	277.6666666666667	1.2243644452469618	1	9621	2	337.25	1.2482331965937552
+1	10299	1	380.5	1.1949268339105126	1	10299	2	144.5	1.184591012875645
+1	10745	1	287.3333333333333	1.084129539111935	1	10745	2	335.6666666666667	1.1651939735017574
+1	11125	1	472.75	1.0530701492045889	1	11125	2	264.5	1.0730737223380644
+1	11859	1	256.5	1.4406552725835113	1	11859	2	370.75	1.1082243282754298
+1	12101	1	334.75	1.1423000747353222	1	12101	2	396.5	1.0099457859523537
+1	12259	1	326.5	1.219693210219279	1	12259	2	292.6666666666667	1.2808898286830026
+1	12641	1	321.25	1.1286221893301993	1	12641	2	279.25	1.129134558577743
+1	13043	1	260.5	1.355894484625015	1	13043	2	295.0	1.056210118409035
+1	13157	1	260.5	1.5242630430075292	1	13157	2	413.5	1.0422561797285326
+1	13293	1	325.25	1.1599721810918615	1	13293	2	345.75	1.0626233629994524
+1	13729	1	486.0	1.0680776434770018	1	13729	2	389.6666666666667	1.3522269473359647
+1	14137	1	427.0	1.0418229612154228	1	14137	2	387.5	1.0294855239302605
+1	14159	1	398.6666666666667	1.0001328196713188	1	14159	2	186.5	1.01269532733355
+1	14161	1	84.66666666666667	1.4291501268026987	1	14161	2	450.5	1.0059037693687187
+1	14911	1	397.6666666666667	1.1801953152598252	1	14911	2	195.0	1.0294118715204632
+1	14983	1	286.6666666666667	1.335077674103522	1	14983	2	138.25	1.0223829110804588
+1	15181	1	270.0	1.0247553155705627	1	15181	2	431.25	1.0014337967301454
+1	15425	1	311.5	1.0406385606636968	1	15425	2	348.25	1.1725274574998379
+1	15441	1	295.0	1.597573089984185	1	15441	2	296.0	1.534664092337063
+1	16601	1	428.5	1.0395250642903893	1	16601	2	397.75	1.108690797111687
+1	16895	1	197.25	1.6216484458744376	1	16895	2	201.33333333333334	1.001838963026182
+1	17285	1	291.5	1.221338355118313	1	17285	2	384.3333333333333	1.3166511804104957
+1	17399	1	335.25	1.18017157377048	1	17399	2	208.0	1.2199892235676928
+1	17521	1	248.0	1.1446287380504054	1	17521	2	321.25	1.1065301391790767
+2	39	1	306.75	1.3140265341053214	2	39	2	207.75	1.0612481315699458
+2	575	1	139.75	1.0651946548671536	2	575	2	390.0	1.2985907498394962
+2	577	1	398.3333333333333	1.0782049277126313	2	577	2	125.25	1.015309703853557
+2	795	1	445.0	1.008215213597613	2	795	2	226.0	1.0374761691860421
+2	977	1	383.0	1.0133755425339792	2	977	2	365.25	1.013000783116057
+2	993	1	361.0	1.0341824955539196	2	993	2	228.0	1.2304019216861168
+2	1367	1	402.0	1.143084675277277	2	1367	2	204.25	1.0427544150441603
+2	2863	1	366.5	1.0309651709854288	2	2863	2	216.75	1.213127273469588
+2	3041	1	330.75	1.0432879984065397	2	3041	2	397.25	1.0315273164376213
+2	3323	1	427.25	1.0090121310722835	2	3323	2	338.5	1.0459307196329675
+2	3999	1	422.0	1.0082188847601554	2	3999	2	403.25	1.0051474075339424
+2	4023	1	410.3333333333333	1.0275376199691826	2	4023	2	247.75	1.1609399487145424
+2	4331	1	384.75	1.064007972514113	2	4331	2	385.75	1.011321832432027
+2	5915	1	231.25	1.8705235468767831	2	5915	2	407.0	1.0885509926353207
+2	6275	1	420.6666666666667	1.008294829045718	2	6275	2	297.6666666666667	1.4615952762209015
+2	7087	1	494.5	1.0102112511298167	2	7087	2	148.5	1.7921780145107034
+2	7333	1	150.33333333333334	1.433050233356339	2	7333	2	214.75	1.0557216871980746
+2	7497	1	360.25	1.0793771564436658	2	7497	2	200.0	1.2799283834131763
+2	8825	1	381.5	1.097428368281076	2	8825	2	414.25	1.0153496363346763
+2	10795	1	393.3333333333333	1.0666650582315749	2	10795	2	322.75	1.0370108734399346
+2	10863	1	330.75	1.0627419399194518	2	10863	2	276.0	1.6689178243621974
+2	11073	1	274.0	1.6770433716259476	2	11073	2	359.0	1.0133649606830526
+2	11465	1	312.25	1.1867998857912492	2	11465	2	154.0	1.1105857906239676
+2	11853	1	292.25	1.0772136418065505	2	11853	2	401.5	1.0787107704829733
+2	12389	1	366.25	1.067334394007718	2	12389	2	278.5	1.1044576295014297
+2	13247	1	356.0	1.0660645893691494	2	13247	2	342.5	1.0634325916808434
+2	13661	1	293.0	1.2174529148923212	2	13661	2	412.25	1.048653193049242
+2	13923	1	335.75	1.2358541998052608	2	13923	2	100.75	1.0814570294681372
+2	14671	1	262.75	1.0250871002782607	2	14671	2	338.75	1.0628054504205149
+2	15397	1	297.6666666666667	1.2809438534554334	2	15397	2	492.75	1.0943752797356943
+2	15477	1	251.0	1.124353329693781	2	15477	2	307.6666666666667	1.3795297219800364
+2	15795	1	304.5	1.0607926164522463	2	15795	2	397.25	1.1176167493994051
+2	16603	1	293.3333333333333	1.387199822342635	2	16603	2	433.3333333333333	1.1660592106922516
+2	16969	1	364.6666666666667	1.1611696936141274	2	16969	2	375.3333333333333	1.0294034440006494
+2	17393	1	179.25	1.1070920857377156	2	17393	2	294.25	1.1481110921426008
+3	29	1	438.25	1.0131261466664097	3	29	2	344.0	1.1151530577310618
+3	247	1	321.0	1.0042014719826915	3	247	2	423.0	1.180182949214427
+3	953	1	338.75	1.1838346915880587	3	953	2	321.5	1.2363739805879619
+3	1541	1	110.0	1.1791578448793427	3	1541	2	273.75	1.197487924282276
+3	1649	1	360.6666666666667	1.1853733590339803	3	1649	2	334.25	1.1482623798952447
+3	2459	1	313.75	1.0048197369511642	3	2459	2	352.75	1.21947122536524
+3	2619	1	241.0	1.1159485992928209	3	2619	2	261.0	1.1099544779211474
+3	2707	1	375.25	1.1207806068743988	3	2707	2	290.75	1.0006820492941273
+3	2975	1	304.0	1.0591594463002163	3	2975	2	190.0	1.2046769431661426
+3	3315	1	271.75	1.555976998814345	3	3315	2	393.75	1.0196319345405949
+3	3393	1	260.0	1.5009563026568116	3	3393	2	470.25	1.129275872154205
+3	3597	1	304.0	1.2471400801439207	3	3597	2	364.0	1.057917059038131
+3	3661	1	331.25	1.2138186201312904	3	3661	2	398.25	1.0134502284121254
+3	3951	1	328.3333333333333	1.3920958631929026	3	3951	2	378.0	1.057830622993178
+3	4793	1	439.5	1.3208979917045633	3	4793	2	298.6666666666667	1.2536383791454593
+3	5221	1	395.25	1.012020609314844	3	5221	2	423.6666666666667	1.0742618083358388
+3	5857	1	331.5	1.1548423818657882	3	5857	2	394.3333333333333	1.101836576034495
+3	6045	1	313.5	1.1971443861134845	3	6045	2	67.25	1.2083633449201445
+3	6615	1	366.0	1.4103495908912012	3	6615	2	228.0	1.0322683130436006
+3	7071	1	182.75	1.402155194063468	3	7071	2	438.25	1.0176436798626307
+3	7211	1	355.25	1.2455338321801286	3	7211	2	462.0	1.0449517641148738
+3	8761	1	253.75	1.1207897246865177	3	8761	2	212.5	1.1557740307473354
+3	9305	1	350.6666666666667	1.3141448475357504	3	9305	2	387.3333333333333	1.043391324490137
+3	9373	1	179.75	1.3318949893741667	3	9373	2	321.25	1.1314604181366261
+3	9669	1	315.75	1.093783081996044	3	9669	2	321.0	1.1239703852823903
+3	9699	1	362.25	1.0269679854596525	3	9699	2	358.0	1.5025258842887776
+3	10301	1	348.5	1.2820855632941448	3	10301	2	318.5	1.289483896046129
+3	10427	1	241.33333333333334	1.5634035191786233	3	10427	2	381.25	1.0623056061004696
+3	11103	1	260.25	1.0537747255764836	3	11103	2	334.0	1.2702517027303248
+3	11141	1	251.0	1.0896833134701018	3	11141	2	272.0	1.1910327315841194
+3	12019	1	362.25	1.0966647561341047	3	12019	2	282.25	1.0983756663144604
+3	12743	1	276.5	1.005648259467935	3	12743	2	352.5	1.08876682930328
+3	12753	1	250.25	1.3386846981823803	3	12753	2	468.0	1.0383135087299893
+3	12931	1	322.75	1.1146291380437745	3	12931	2	320.0	1.17009448069376
+3	13487	1	308.25	1.2961991776642086	3	13487	2	293.25	1.0585525936033
+3	13555	1	373.25	1.0745070114317623	3	13555	2	152.75	1.412684197862033
+3	13581	1	292.75	1.1902035296028353	3	13581	2	253.0	1.155002663143799
+3	13829	1	233.5	1.1312399620732085	3	13829	2	444.25	1.0391188483200453
+3	13847	1	417.5	1.039525557170396	3	13847	2	263.5	1.4436108729741235
+3	14073	1	355.5	1.0476440697241391	3	14073	2	437.25	1.0172135605078851
+3	14767	1	311.5	1.0034195608338836	3	14767	2	339.0	1.2032144276415566
+3	14981	1	193.25	1.0060336654947306	3	14981	2	441.5	1.3661655364714043
+3	16331	1	272.0	1.1467170493846688	3	16331	2	339.25	1.2786638701439956
+3	16847	1	273.6666666666667	1.3346016934186173	3	16847	2	398.0	1.2041547394959626
+3	16987	1	358.0	1.101510614957325	3	16987	2	420.0	1.0848663494738469
+3	17613	1	424.3333333333333	1.0320925947787334	3	17613	2	390.75	1.0761214357356987
+3	17987	1	387.5	1.1128327233395303	3	17987	2	131.66666666666666	1.1227241574530091
+4	225	1	180.0	1.2847074573726138	4	225	2	366.5	1.112494070167504
+4	299	1	293.5	1.135267940218844	4	299	2	380.0	1.0485028679413595
+4	825	1	223.25	1.2488574716961685	4	825	2	254.0	1.5182802586094637
+4	1393	1	418.75	1.0408989038120988	4	1393	2	413.3333333333333	1.1020163503416796
+4	1523	1	363.25	1.0130673543588669	4	1523	2	253.5	1.2817761298828965
+4	1729	1	313.25	1.3148930771572687	4	1729	2	296.5	1.210664179669432
+4	2989	1	424.6666666666667	1.03767453099966	4	2989	2	123.75	1.4454541925191389
+4	3183	1	190.0	1.2520196057807818	4	3183	2	245.0	1.0300119488354766
+4	4175	1	395.0	1.042998032908585	4	4175	2	485.0	1.0145126110736231
+4	4293	1	285.0	1.042264740588342	4	4293	2	331.25	1.0702681575369872
+4	4573	1	243.75	1.4457774863358526	4	4573	2	431.25	1.0010829394909448
+4	4875	1	401.0	1.0066599946104444	4	4875	2	410.5	1.051550593497737
+4	5009	1	386.5	1.0301582587751055	4	5009	2	473.75	1.055073121585445
+4	5947	1	291.5	1.046282184237671	4	5947	2	320.5	1.1280002765664996
+4	6359	1	193.33333333333334	1.2483139639831744	4	6359	2	371.75	1.0993680760045068
+4	6517	1	289.0	1.0911931716633327	4	6517	2	148.0	1.0471156482980475
+4	8309	1	371.0	1.2845214196617782	4	8309	2	371.5	1.0748626938819539
+4	8339	1	392.75	1.0058445869354098	4	8339	2	345.75	1.2872431560206488
+4	9685	1	288.75	1.0017436994234579	4	9685	2	440.75	1.0083448738924952
+4	10255	1	373.5	1.1222827247788254	4	10255	2	352.0	1.1003307048901103
+4	10925	1	199.5	1.3875238422301213	4	10925	2	261.75	1.283642511996497
+4	11213	1	226.66666666666666	1.09984270658979	4	11213	2	413.5	1.0174813417315496
+4	11305	1	351.75	1.1922401157939606	4	11305	2	365.25	1.1258535465411879
+4	11473	1	394.6666666666667	1.0178948794541924	4	11473	2	212.66666666666666	1.195359710715888
+4	12353	1	340.25	1.164721531085477	4	12353	2	432.0	1.0523203480868901
+4	12783	1	329.5	1.0329266474827115	4	12783	2	187.0	1.2621302720196819
+4	12971	1	370.3333333333333	1.097620185659271	4	12971	2	278.0	1.4524982093215804
+4	13665	1	363.0	1.04089223995917	4	13665	2	332.6666666666667	1.1900176061910035
+4	13913	1	297.3333333333333	1.071040936419414	4	13913	2	316.25	1.3567449933143143
+4	15161	1	305.75	1.3571548565863678	4	15161	2	262.3333333333333	1.2292106140967536
+4	15401	1	341.0	1.0164918336889106	4	15401	2	337.25	1.0178529534898602
+4	15467	1	355.3333333333333	1.27670099607062	4	15467	2	416.6666666666667	1.1678517714162187
+4	16211	1	257.6666666666667	1.6381074811154002	4	16211	2	352.25	1.055236934125639
+4	16367	1	344.25	1.1865617643407205	4	16367	2	330.5	1.001436680208246
+4	16623	1	174.75	1.1547312605990323	4	16623	2	261.75	1.4692073123565808
+4	16753	1	283.6666666666667	1.4179905875607177	4	16753	2	331.0	1.0757450815976775
+4	16791	1	229.75	1.0415889892942085	4	16791	2	348.75	1.2365182061688882
+5	507	1	360.5	1.0016609878348282	5	507	2	397.25	1.1165805580468837
+5	1379	1	418.3333333333333	1.1593756930293735	5	1379	2	362.25	1.1381161323894302
+5	1451	1	259.75	1.0166115859746467	5	1451	2	186.5	1.3009837449067687
+5	1761	1	245.25	1.0674277258886877	5	1761	2	356.3333333333333	1.0319105846046546
+5	1919	1	558.0	1.051789656603646	5	1919	2	280.75	1.435982403616447
+5	2153	1	398.25	1.038369445511033	5	2153	2	322.0	1.2495167076207327
+5	2583	1	357.25	1.0689747703230787	5	2583	2	321.5	1.174109700061395
+5	2725	1	306.25	1.0685532393228003	5	2725	2	193.5	1.4095901314659105
+5	3547	1	357.0	1.1544864737016736	5	3547	2	343.75	1.2077817108886129
+5	3785	1	215.5	1.231057632809026	5	3785	2	460.0	1.048938011267006
+5	4445	1	327.3333333333333	1.0177488158574015	5	4445	2	414.75	1.046288264177383
+5	4601	1	327.25	1.15815714609041	5	4601	2	142.66666666666666	1.2197537262761011
+5	5019	1	341.6666666666667	1.2014886661438384	5	5019	2	363.5	1.1056740335885162
+5	5635	1	275.75	1.003161317494043	5	5635	2	195.33333333333334	1.49437494371756
+5	5725	1	308.6666666666667	1.2494665767967896	5	5725	2	315.5	1.4329959977644893
+5	5787	1	335.75	1.3581868619406905	5	5787	2	453.5	1.031825110180606
+5	8665	1	257.75	1.8119629759287612	5	8665	2	368.0	1.0243808356311228
+5	9037	1	189.75	1.0334701022027994	5	9037	2	326.0	1.1188906978754734
+5	9241	1	342.75	1.037524616861255	5	9241	2	174.5	1.1953290244295067
+5	9245	1	339.5	1.0092696575112496	5	9245	2	303.0	1.2214283206597227
+5	9789	1	391.25	1.0458503093728178	5	9789	2	343.75	1.070040394695916
+5	10775	1	439.5	1.2565424257262654	5	10775	2	330.75	1.3194508007529422
+5	10851	1	296.25	1.4450973535233087	5	10851	2	185.75	1.0920078306591938
+5	11409	1	337.5	1.2675445661798022	5	11409	2	267.5	1.2735562175240271
+5	11543	1	373.75	1.1069130009236565	5	11543	2	347.0	1.0384881272212296
+5	11907	1	312.0	1.1200627653353177	5	11907	2	130.0	1.0970913669588425
+5	12315	1	255.75	1.118436212304132	5	12315	2	329.25	1.1943065884369533
+5	12589	1	372.75	1.1109412437666168	5	12589	2	355.75	1.0853990131935114
+5	12853	1	306.75	1.4054585232279222	5	12853	2	174.75	1.0143495332647992
+5	13001	1	420.3333333333333	1.0350551797504086	5	13001	2	214.0	1.115488847819715
+5	13399	1	304.75	1.1656588906680398	5	13399	2	359.5	1.0278593838157082
+5	13809	1	338.0	1.3356560129512516	5	13809	2	254.25	1.3229081483155771
+5	13997	1	372.75	1.0067987273536196	5	13997	2	488.5	1.0136437168469898
+5	14683	1	313.75	1.0038487229336792	5	14683	2	344.5	1.1175817890990485
+5	14721	1	420.0	1.0064164891633403	5	14721	2	280.5	1.0511776533295065
+5	14891	1	445.25	1.0918502542585706	5	14891	2	420.3333333333333	1.1234936159879627
+5	15261	1	514.0	1.0718025790789742	5	15261	2	367.25	1.015824493979522
+5	15477	1	225.0	1.1250854562879289	5	15477	2	235.75	1.0790388329037188
+5	15655	1	367.5	1.100326204582237	5	15655	2	306.3333333333333	1.0511442359018315
+5	15673	1	351.25	1.0192896453224356	5	15673	2	389.6666666666667	1.0824061270803669
+5	15959	1	414.0	1.0773325961138016	5	15959	2	237.25	1.4024256583845858
+5	17339	1	399.25	1.0240754930004161	5	17339	2	265.5	1.1851526004436805
+5	17581	1	426.0	1.2083890532953205	5	17581	2	233.25	1.1436871765942431
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q39b.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q39b.sql.out
new file mode 100644
index 0000000000000..a250c0afd3d44
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q39b.sql.out
@@ -0,0 +1,15 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<w_warehouse_sk:int,i_item_sk:int,d_moy:int,mean:double,cov:double,w_warehouse_sk:int,i_item_sk:int,d_moy:int,mean:double,cov:double>
+-- !query output
+1	13157	1	260.5	1.5242630430075292	1	13157	2	413.5	1.0422561797285326
+1	15441	1	295.0	1.597573089984185	1	15441	2	296.0	1.534664092337063
+1	16895	1	197.25	1.6216484458744376	1	16895	2	201.33333333333334	1.001838963026182
+2	5915	1	231.25	1.8705235468767831	2	5915	2	407.0	1.0885509926353207
+2	11073	1	274.0	1.6770433716259476	2	11073	2	359.0	1.0133649606830526
+3	3315	1	271.75	1.555976998814345	3	3315	2	393.75	1.0196319345405949
+3	3393	1	260.0	1.5009563026568116	3	3393	2	470.25	1.129275872154205
+3	10427	1	241.33333333333334	1.5634035191786233	3	10427	2	381.25	1.0623056061004696
+4	16211	1	257.6666666666667	1.6381074811154002	4	16211	2	352.25	1.055236934125639
+5	8665	1	257.75	1.8119629759287612	5	8665	2	368.0	1.0243808356311228
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q4.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q4.sql.out
new file mode 100644
index 0000000000000..a9706ef782fde
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q4.sql.out
@@ -0,0 +1,7 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<customer_id:string,customer_first_name:string,customer_last_name:string,customer_preferred_cust_flag:string,customer_birth_country:string,customer_login:string,customer_email_address:string>
+-- !query output
+AAAAAAAAMHOLAAAA	Terri               	Cook                          	N	COSTA RICA	NULL	Terri.Cook@Vz02fJPUlPO.edu                        
+AAAAAAAANBECBAAA	Michael             	Lombardi                      	Y	ZIMBABWE	NULL	Michael.Lombardi@J.com
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q40.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q40.sql.out
new file mode 100644
index 0000000000000..db2d76e5d019e
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q40.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<w_state:string,i_item_id:string,sales_before:decimal(23,2),sales_after:decimal(23,2)>
+-- !query output
+TN	AAAAAAAAAAEAAAAA	51.75	-61.82
+TN	AAAAAAAAAANDAAAA	131.09	324.87
+TN	AAAAAAAAAAPBAAAA	50.90	146.62
+TN	AAAAAAAAACACAAAA	113.63	89.13
+TN	AAAAAAAAACGCAAAA	53.39	37.02
+TN	AAAAAAAAACKCAAAA	65.49	99.60
+TN	AAAAAAAAACPCAAAA	0.00	73.18
+TN	AAAAAAAAADACAAAA	132.24	0.00
+TN	AAAAAAAAADBBAAAA	211.20	0.00
+TN	AAAAAAAAADNAAAAA	202.46	0.00
+TN	AAAAAAAAAEFCAAAA	45.57	38.18
+TN	AAAAAAAAAEGEAAAA	-19.57	109.62
+TN	AAAAAAAAAELBAAAA	34.98	144.32
+TN	AAAAAAAAAFBEAAAA	130.26	3.53
+TN	AAAAAAAAAFFDAAAA	27.12	104.61
+TN	AAAAAAAAAFJDAAAA	-41.08	0.00
+TN	AAAAAAAAAFLDAAAA	89.24	0.00
+TN	AAAAAAAAAFODAAAA	16.39	0.00
+TN	AAAAAAAAAGEBAAAA	146.51	20.38
+TN	AAAAAAAAAGIDAAAA	2.84	0.00
+TN	AAAAAAAAAGLDAAAA	-40.82	75.88
+TN	AAAAAAAAAGPCAAAA	136.76	190.61
+TN	AAAAAAAAAHBAAAAA	66.92	219.76
+TN	AAAAAAAAAHPBAAAA	148.73	-316.99
+TN	AAAAAAAAAIADAAAA	230.28	132.32
+TN	AAAAAAAAAIGDAAAA	190.52	0.00
+TN	AAAAAAAAAINCAAAA	207.61	208.83
+TN	AAAAAAAAAJHDAAAA	-39.87	215.04
+TN	AAAAAAAAAJIDAAAA	-351.64	142.38
+TN	AAAAAAAAAJOAAAAA	58.56	0.00
+TN	AAAAAAAAAKABAAAA	110.04	0.00
+TN	AAAAAAAAAKFEAAAA	-284.41	91.42
+TN	AAAAAAAAALAAAAAA	-4.78	133.34
+TN	AAAAAAAAALDCAAAA	39.16	0.00
+TN	AAAAAAAAALFEAAAA	27.83	212.44
+TN	AAAAAAAAALIDAAAA	0.00	118.88
+TN	AAAAAAAAALOAAAAA	4.18	0.00
+TN	AAAAAAAAALOBAAAA	0.00	0.00
+TN	AAAAAAAAALPAAAAA	-30.21	10.95
+TN	AAAAAAAAAMJBAAAA	-24.54	85.92
+TN	AAAAAAAAANLCAAAA	84.38	67.38
+TN	AAAAAAAAANPAAAAA	74.98	3.48
+TN	AAAAAAAAANPDAAAA	484.43	90.58
+TN	AAAAAAAAAOFAAAAA	0.00	91.90
+TN	AAAAAAAAAOMDAAAA	188.82	29.34
+TN	AAAAAAAAAPPBAAAA	0.00	146.30
+TN	AAAAAAAABABCAAAA	252.04	27.49
+TN	AAAAAAAABADAAAAA	27.80	35.00
+TN	AAAAAAAABAGAAAAA	17.30	2.53
+TN	AAAAAAAABCCCAAAA	-254.31	0.00
+TN	AAAAAAAABDCBAAAA	38.34	101.58
+TN	AAAAAAAABECDAAAA	223.68	112.52
+TN	AAAAAAAABEMCAAAA	0.00	160.01
+TN	AAAAAAAABGECAAAA	0.00	224.53
+TN	AAAAAAAABGIBAAAA	105.83	0.00
+TN	AAAAAAAABGNCAAAA	0.00	72.03
+TN	AAAAAAAABHACAAAA	0.00	0.00
+TN	AAAAAAAABHPCAAAA	97.28	190.95
+TN	AAAAAAAABJFEAAAA	70.68	38.07
+TN	AAAAAAAABJMAAAAA	31.23	140.56
+TN	AAAAAAAABKEBAAAA	133.61	92.51
+TN	AAAAAAAABLBDAAAA	50.89	37.68
+TN	AAAAAAAABLEDAAAA	-119.59	0.00
+TN	AAAAAAAABLOCAAAA	112.88	62.95
+TN	AAAAAAAABMCBAAAA	16.30	70.83
+TN	AAAAAAAABMOBAAAA	54.02	130.38
+TN	AAAAAAAABNEBAAAA	48.98	-803.52
+TN	AAAAAAAABOKDAAAA	0.00	135.88
+TN	AAAAAAAABONAAAAA	30.07	213.69
+TN	AAAAAAAABPPAAAAA	4.77	91.54
+TN	AAAAAAAACACEAAAA	244.10	0.00
+TN	AAAAAAAACALAAAAA	-832.53	0.00
+TN	AAAAAAAACALBAAAA	0.00	-641.98
+TN	AAAAAAAACBDBAAAA	45.72	145.13
+TN	AAAAAAAACBHBAAAA	102.42	0.00
+TN	AAAAAAAACBICAAAA	0.95	85.93
+TN	AAAAAAAACBJCAAAA	89.33	143.00
+TN	AAAAAAAACBNAAAAA	198.92	133.15
+TN	AAAAAAAACCABAAAA	61.26	65.34
+TN	AAAAAAAACCJBAAAA	4.84	0.00
+TN	AAAAAAAACCPDAAAA	280.66	283.09
+TN	AAAAAAAACDECAAAA	-667.40	-693.66
+TN	AAAAAAAACDIBAAAA	3.73	148.26
+TN	AAAAAAAACEACAAAA	0.00	-978.31
+TN	AAAAAAAACEHDAAAA	66.23	123.74
+TN	AAAAAAAACEICAAAA	30.14	117.63
+TN	AAAAAAAACEODAAAA	0.00	1.28
+TN	AAAAAAAACFKDAAAA	0.00	325.59
+TN	AAAAAAAACFLCAAAA	13.34	76.65
+TN	AAAAAAAACGCBAAAA	-1636.68	167.66
+TN	AAAAAAAACGCDAAAA	0.00	8.58
+TN	AAAAAAAACGJDAAAA	-422.98	490.30
+TN	AAAAAAAACGMAAAAA	142.04	196.75
+TN	AAAAAAAACHCDAAAA	-936.02	96.99
+TN	AAAAAAAACIICAAAA	-175.56	40.39
+TN	AAAAAAAACIJBAAAA	0.00	16.71
+TN	AAAAAAAACIMDAAAA	266.31	0.00
+TN	AAAAAAAACJBEAAAA	-5.33	22.57
+TN	AAAAAAAACJDDAAAA	72.37	0.00
+TN	AAAAAAAACJGCAAAA	4.59	237.09
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q41.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q41.sql.out
new file mode 100644
index 0000000000000..aaaa39f72f03a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q41.sql.out
@@ -0,0 +1,10 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_product_name:string>
+-- !query output
+antieingeseableought                              
+eingeseeseese                                     
+eseeingeseableought                               
+oughtoughtablecally                               
+priantin stoughtought
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q42.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q42.sql.out
new file mode 100644
index 0000000000000..1b17f237b3d06
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q42.sql.out
@@ -0,0 +1,16 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<d_year:int,i_category_id:int,i_category:string,sum(ss_ext_sales_price):decimal(17,2)>
+-- !query output
+2000	7	Home                                              	494603.64
+2000	2	Men                                               	390852.57
+2000	4	Shoes                                             	378230.23
+2000	3	Children                                          	359411.01
+2000	9	Books                                             	319480.51
+2000	10	Electronics                                       	317086.16
+2000	8	Sports                                            	287853.86
+2000	6	Jewelry                                           	278786.18
+2000	1	Women                                             	245897.86
+2000	5	Music                                             	189405.76
+2000	NULL	NULL	39507.19
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q43.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q43.sql.out
new file mode 100644
index 0000000000000..38e7f9316f5c0
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q43.sql.out
@@ -0,0 +1,11 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<s_store_name:string,s_store_id:string,sun_sales:decimal(17,2),mon_sales:decimal(17,2),tue_sales:decimal(17,2),wed_sales:decimal(17,2),thu_sales:decimal(17,2),fri_sales:decimal(17,2),sat_sales:decimal(17,2)>
+-- !query output
+able	AAAAAAAACAAAAAAA	495537.85	454457.37	480383.13	468495.25	496050.12	471996.47	510311.54
+ation	AAAAAAAAHAAAAAAA	485290.49	465106.17	462449.17	478799.15	521647.09	474980.87	484757.85
+bar	AAAAAAAAKAAAAAAA	510374.60	458247.16	464054.12	473015.97	487045.12	495531.52	502011.00
+eing	AAAAAAAAIAAAAAAA	513799.80	464451.67	440681.32	501857.36	476201.82	452754.56	481703.98
+ese	AAAAAAAAEAAAAAAA	529524.71	460191.13	492178.33	458067.77	488508.48	477658.66	490178.23
+ought	AAAAAAAABAAAAAAA	464514.26	460448.28	465249.85	482655.67	474754.90	479860.76	474064.44
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q44.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q44.sql.out
new file mode 100644
index 0000000000000..959a3fe521152
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q44.sql.out
@@ -0,0 +1,15 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<rnk:int,best_performing:string,worst_performing:string>
+-- !query output
+1	oughtantiought                                    	eingbarantiationought                             
+2	barprin steing                                    	callyableationpri                                 
+3	ableoughtantipriought                             	ablebarcallypriought                              
+4	oughtcallyoughtation                              	callyn steseable                                  
+5	oughteseesecally                                  	ableoughtbarn st                                  
+6	eingationationanti                                	callyableoughtn st                                
+7	ablecallyought                                    	barbarcallyableought                              
+8	n steingprieseought                               	eseeingationeseought                              
+9	priprieinganti                                    	bareseprieseought                                 
+10	oughtcallybaroughtought                           	eingcallyantipri
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q45.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q45.sql.out
new file mode 100644
index 0000000000000..c807bbb5771ed
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q45.sql.out
@@ -0,0 +1,25 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<ca_zip:string,ca_city:string,sum(ws_sales_price):decimal(17,2)>
+-- !query output
+12150     	Montezuma	11.23
+20525     	Ryan	57.27
+21087     	Macedonia	23.20
+21933     	Mount Pleasant	51.15
+22924     	Belleville	25.42
+24289     	NULL	17.89
+26098     	Five Points	27.24
+45281     	Bethel	113.11
+48014     	Clifton	20.69
+48828     	Greenwood	58.48
+50411     	Cedar Grove	70.45
+51933     	Mount Pleasant	23.43
+54536     	Friendship	123.37
+58605     	Antioch	95.49
+65817     	Bridgeport	47.98
+65867     	Riceville	11.06
+69843     	Oakland	11.82
+71944     	Gravel Hill	23.40
+76971     	Wilson	111.77
+78048     	Salem	188.36
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q46.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q46.sql.out
new file mode 100644
index 0000000000000..b9f5ef0471ae9
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q46.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<c_last_name:string,c_first_name:string,ca_city:string,bought_city:string,ss_ticket_number:int,amt:decimal(17,2),profit:decimal(17,2)>
+-- !query output
+NULL	NULL	Antioch	Florence	19942	6675.56	-20224.05
+NULL	NULL	Antioch	Jamestown	189909	188.04	4440.84
+NULL	NULL	Antioch	Sulphur Springs	211628	0.00	1690.47
+NULL	NULL	Arlington	Leesburg	112878	1749.37	-4137.65
+NULL	NULL	Arlington	Mount Olive	186130	3625.05	-9174.03
+NULL	NULL	Arthur	Bridgeport	89194	1207.33	-3747.04
+NULL	NULL	Ashland	Antioch	25749	1184.60	-22285.21
+NULL	NULL	Ashland	Cherry Valley	92172	2534.66	-5553.97
+NULL	NULL	Ashland	Superior	91601	7305.88	-10487.66
+NULL	NULL	Belmont	Hamilton	1029	935.93	-1026.72
+NULL	NULL	Bethel	Oakwood	179326	2593.05	-13662.26
+NULL	NULL	Bethel	Woodland	95034	407.35	-15767.89
+NULL	NULL	Bridgeport	Riverside	23264	89.15	-6827.82
+NULL	NULL	Bridgeport	Wildwood	90648	4796.33	-6445.14
+NULL	NULL	Brownsville	Oak Grove	114024	2720.05	-18657.62
+NULL	NULL	Brunswick	Valley View	153419	0.00	-13840.52
+NULL	NULL	Buena Vista	Clearview	109316	257.04	-11539.44
+NULL	NULL	Buena Vista	Deerfield	17940	0.00	-1431.61
+NULL	NULL	Bunker Hill	Colfax	45044	2803.00	2310.74
+NULL	NULL	Cedar Grove	Brownsville	17159	865.78	-9843.40
+NULL	NULL	Centerville	Fairfield	11688	604.66	-7431.13
+NULL	NULL	Centerville	Fisher	24700	330.27	-8639.65
+NULL	NULL	Centerville	Glenwood	27267	0.00	-7024.69
+NULL	NULL	Clifton	Springhill	190087	2987.19	-13925.26
+NULL	NULL	Clinton	Edgewood	84531	8561.97	-28472.50
+NULL	NULL	Clinton	Oakland	179459	879.23	-875.68
+NULL	NULL	Concord	Lakewood	81197	1244.92	866.30
+NULL	NULL	Crossroads	Mount Olive	229868	223.72	139.81
+NULL	NULL	Ellsworth	Shiloh	103474	1412.84	-8861.07
+NULL	NULL	Enterprise	Five Forks	234468	2248.50	-7218.60
+NULL	NULL	Enterprise	Hillcrest	216432	529.37	-2245.35
+NULL	NULL	Fairfield	Arlington	75875	565.09	-14322.52
+NULL	NULL	Fairview	Friendship	172926	1897.88	-11040.49
+NULL	NULL	Fairview	Mount Olive	127246	7793.58	-12492.10
+NULL	NULL	Farmington	Springfield	222902	1213.28	-1240.73
+NULL	NULL	Fayette	Kingston	32825	0.00	-6315.77
+NULL	NULL	Ferguson	Belmont	238393	11864.37	-26448.01
+NULL	NULL	Five Forks	Florence	224673	1507.52	-1549.30
+NULL	NULL	Five Forks	Newport	131275	3957.94	-8960.42
+NULL	NULL	Five Points	Brownsville	122401	166.11	-12890.14
+NULL	NULL	Five Points	Shady Grove	127068	41.49	-3127.13
+NULL	NULL	Five Points	Union	233626	243.82	-13369.20
+NULL	NULL	Florence	Deerfield	181713	2673.95	-11875.68
+NULL	NULL	Forest Hills	Shiloh	76517	2206.26	-20795.18
+NULL	NULL	Friendship	Hamilton	49257	2542.47	14.28
+NULL	NULL	Friendship	Lebanon	230509	7322.26	-3897.95
+NULL	NULL	Friendship	Springdale	55155	187.12	-8555.50
+NULL	NULL	Glendale	Liberty	127860	179.92	-9645.97
+NULL	NULL	Glendale	Springfield	107410	236.04	-11521.38
+NULL	NULL	Glendale	Stringtown	31383	0.00	-5195.83
+NULL	NULL	Glendale	Union	3484	793.55	5358.57
+NULL	NULL	Glenwood	Arlington	227159	236.19	-10529.52
+NULL	NULL	Glenwood	Belmont	72581	1808.15	-7585.34
+NULL	NULL	Gravel Hill	Farmington	62500	0.00	-1479.92
+NULL	NULL	Greenville	Bethel	84058	225.62	-1544.42
+NULL	NULL	Greenville	Marion	221320	669.39	-7120.33
+NULL	NULL	Greenville	Spring Hill	47915	11384.61	-24018.75
+NULL	NULL	Greenwood	Pleasant Hill	4709	822.69	-17295.72
+NULL	NULL	Greenwood	Springdale	204972	263.33	3302.02
+NULL	NULL	Hamilton	Cedar Grove	130494	0.00	-10426.22
+NULL	NULL	Hamilton	Jackson	170132	5140.01	-20602.97
+NULL	NULL	Hamilton	Jamestown	153509	2156.76	-9529.97
+NULL	NULL	Hardy	Harmony	94856	6219.17	-11811.92
+NULL	NULL	Harmony	Plainview	226229	9483.18	-21260.39
+NULL	NULL	Highland Park	Clinton	75671	818.52	-17086.12
+NULL	NULL	Hillcrest	Arlington	131958	608.34	-10931.94
+NULL	NULL	Hillcrest	Berea	60399	764.60	-10177.35
+NULL	NULL	Hillcrest	Harmony	196512	1.70	-9338.87
+NULL	NULL	Hillcrest	Mount Olive	129867	7792.52	-10929.36
+NULL	NULL	Hillcrest	Wilson	33115	4120.87	-17407.74
+NULL	NULL	Indian Village	Golden	98530	284.90	-11785.95
+NULL	NULL	Jackson	Florence	167217	254.54	-18883.41
+NULL	NULL	Jackson	Pleasant Grove	172765	5569.64	-7776.84
+NULL	NULL	Jamestown	Spring Valley	125125	4067.22	-8344.61
+NULL	NULL	Kingston	Farmington	196244	0.00	-4449.03
+NULL	NULL	Kingston	Highland	45785	5257.12	-21282.22
+NULL	NULL	Lakeside	Ashley	233189	1043.83	-3335.36
+NULL	NULL	Lakeside	Highland Park	194552	677.94	-11488.01
+NULL	NULL	Lakeside	Jenkins	71469	434.92	-9825.80
+NULL	NULL	Lakeside	Lakewood	181131	42.20	3148.24
+NULL	NULL	Lakeside	Riverview	183085	116.06	-1892.62
+NULL	NULL	Lakeside	Summit	200887	554.47	-8140.32
+NULL	NULL	Lakeview	Greenfield	230719	0.00	-3529.16
+NULL	NULL	Lancaster	Waterloo	197637	976.52	-11255.13
+NULL	NULL	Langdon	Buena Vista	12247	1626.56	-12689.74
+NULL	NULL	Lebanon	Mount Zion	2701	11616.00	-11748.63
+NULL	NULL	Lincoln	Greenville	126303	0.00	-18688.05
+NULL	NULL	Louisville	Kingston	215207	986.85	-9348.55
+NULL	NULL	Macedonia	Ashland	81673	142.17	-56.02
+NULL	NULL	Macedonia	Guthrie	132070	1459.41	-9746.92
+NULL	NULL	Maple Grove	Five Points	40712	4051.76	-4016.52
+NULL	NULL	Marion	Lakewood	203438	8203.21	-18846.98
+NULL	NULL	Midway	Summerfield	117504	1027.47	-21021.42
+NULL	NULL	Mount Olive	Salem	34593	910.10	-6410.67
+NULL	NULL	Mount Olive	Springdale	73957	2276.08	-18160.48
+NULL	NULL	Mount Vernon	Crossroads	216545	0.00	-10080.14
+NULL	NULL	Mount Vernon	White Oak	25937	2221.06	-12575.90
+NULL	NULL	Murphy	Buena Vista	17149	0.00	-39.90
+NULL	NULL	New Hope	Farmington	38760	6931.68	-10392.42
+NULL	NULL	New Hope	Hillcrest	119378	625.62	-15891.66
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q47.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q47.sql.out
new file mode 100644
index 0000000000000..5cd532e0008c8
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q47.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_category:string,i_brand:string,s_store_name:string,s_company_name:string,d_year:int,d_moy:int,avg_monthly_sales:decimal(21,6),sum_sales:decimal(17,2),psum:decimal(17,2),nsum:decimal(17,2)>
+-- !query output
+Shoes                                             	exportiedu pack #x                                	ation	Unknown	1999	3	5607.487500	2197.48	3271.66	2831.67
+Shoes                                             	exportiedu pack #x                                	ought	Unknown	1999	2	5643.938333	2393.31	4463.11	2652.44
+Shoes                                             	exportiedu pack #x                                	able	Unknown	1999	4	5640.362500	2416.57	3348.90	2987.78
+Men                                               	amalgimporto #x                                   	ought	Unknown	1999	6	4702.116667	1534.95	2666.37	2514.13
+Men                                               	edu packimporto #x                                	eing	Unknown	1999	7	5330.618333	2218.63	3182.74	7436.83
+Shoes                                             	exportiedu pack #x                                	ese	Unknown	1999	4	5338.852500	2233.60	3470.43	2832.41
+Music                                             	exportischolar #x                                 	ese	Unknown	1999	5	5139.465000	2034.96	3149.72	3648.17
+Men                                               	exportiimporto #x                                 	eing	Unknown	1999	7	5748.707500	2645.88	3432.67	7646.91
+Men                                               	importoimporto #x                                 	bar	Unknown	1999	3	4915.190000	1815.92	2884.81	2956.00
+Women                                             	amalgamalg #x                                     	ation	Unknown	1999	6	4586.300000	1508.07	2992.12	3059.37
+Shoes                                             	exportiedu pack #x                                	eing	Unknown	1999	4	5374.032500	2322.80	2484.06	3313.69
+Men                                               	importoimporto #x                                 	ese	Unknown	1999	6	4596.057500	1577.69	2457.43	2439.68
+Men                                               	edu packimporto #x                                	ation	Unknown	1999	3	5839.670833	2825.07	3157.46	3531.36
+Men                                               	edu packimporto #x                                	able	Unknown	1999	4	5342.149167	2347.82	2787.31	3588.16
+Shoes                                             	exportiedu pack #x                                	ought	Unknown	1999	3	5643.938333	2652.44	2393.31	3008.88
+Men                                               	exportiimporto #x                                 	ought	Unknown	1999	3	5475.719167	2515.22	2709.11	2702.85
+Shoes                                             	exportiedu pack #x                                	ese	Unknown	1999	6	5338.852500	2388.17	2832.41	4216.10
+Shoes                                             	exportiedu pack #x                                	bar	Unknown	1999	2	5065.706667	2119.85	3439.05	2640.59
+Shoes                                             	amalgedu pack #x                                  	ation	Unknown	1999	2	4713.009167	1774.19	3366.21	1940.05
+Shoes                                             	exportiedu pack #x                                	eing	Unknown	1999	3	5374.032500	2484.06	2994.92	2322.80
+Men                                               	edu packimporto #x                                	able	Unknown	1999	7	5342.149167	2454.72	2766.42	7665.06
+Men                                               	amalgimporto #x                                   	ese	Unknown	1999	4	4741.993333	1879.47	3419.59	2634.29
+Music                                             	exportischolar #x                                 	able	Unknown	1999	4	4723.742500	1866.30	2384.82	2931.55
+Shoes                                             	importoedu pack #x                                	ation	Unknown	1999	4	4732.205000	1875.08	2686.46	3422.03
+Children                                          	importoexporti #x                                 	ese	Unknown	1999	6	4849.967500	2002.59	2590.23	3380.67
+Men                                               	importoimporto #x                                 	ation	Unknown	1999	6	4920.419167	2077.36	3402.55	3347.44
+Men                                               	importoimporto #x                                 	ese	Unknown	1999	4	4596.057500	1762.17	2728.14	2457.43
+Children                                          	edu packexporti #x                                	eing	Unknown	1999	4	4739.001667	1923.04	2309.91	2849.64
+Music                                             	exportischolar #x                                 	ought	Unknown	1999	3	4816.848333	2010.47	2539.57	2940.38
+Shoes                                             	edu packedu pack #x                               	ese	Unknown	1999	2	4707.697500	1903.70	2693.49	3474.34
+Shoes                                             	importoedu pack #x                                	ought	Unknown	1999	2	4443.995833	1642.32	3972.50	2319.04
+Men                                               	edu packimporto #x                                	ation	Unknown	1999	6	5839.670833	3053.78	3151.27	3622.65
+Shoes                                             	exportiedu pack #x                                	ation	Unknown	1999	4	5607.487500	2831.67	2197.48	4187.54
+Men                                               	edu packimporto #x                                	ought	Unknown	1999	3	5598.894167	2824.13	3154.80	3135.60
+Shoes                                             	amalgedu pack #x                                  	ation	Unknown	1999	3	4713.009167	1940.05	1774.19	2496.18
+Men                                               	exportiimporto #x                                 	ought	Unknown	1999	4	5475.719167	2702.85	2515.22	4364.56
+Shoes                                             	amalgedu pack #x                                  	ese	Unknown	1999	4	4596.537500	1825.00	2777.56	3234.34
+Music                                             	importoscholar #x                                 	able	Unknown	1999	6	4332.550833	1563.26	2484.37	2460.11
+Men                                               	edu packimporto #x                                	ese	Unknown	1999	2	5102.436667	2333.32	3417.85	2536.68
+Men                                               	exportiimporto #x                                 	ought	Unknown	1999	2	5475.719167	2709.11	4740.72	2515.22
+Men                                               	exportiimporto #x                                 	ought	Unknown	1999	6	5475.719167	2723.08	4364.56	4000.42
+Shoes                                             	exportiedu pack #x                                	ese	Unknown	1999	2	5338.852500	2587.72	3837.18	3470.43
+Children                                          	exportiexporti #x                                 	eing	Unknown	1999	6	4426.704167	1683.86	2186.50	1970.76
+Shoes                                             	exportiedu pack #x                                	ought	Unknown	1999	6	5643.938333	2902.72	3046.01	3994.18
+Shoes                                             	exportiedu pack #x                                	eing	Unknown	1999	6	5374.032500	2637.31	3313.69	3158.65
+Men                                               	importoimporto #x                                 	ation	Unknown	1999	3	4920.419167	2183.89	3598.63	2841.78
+Men                                               	edu packimporto #x                                	eing	Unknown	1999	3	5330.618333	2602.65	2613.45	2624.00
+Women                                             	edu packamalg #x                                  	eing	Unknown	1999	6	4366.627500	1640.10	3159.43	2882.91
+Music                                             	exportischolar #x                                 	ese	Unknown	1999	2	5139.465000	2413.63	4627.89	2773.28
+Men                                               	exportiimporto #x                                 	ese	Unknown	1999	2	4929.611667	2207.89	3940.06	2246.76
+Women                                             	edu packamalg #x                                  	ese	Unknown	1999	2	4551.444167	1833.08	3729.30	2940.96
+Men                                               	edu packimporto #x                                	eing	Unknown	1999	2	5330.618333	2613.45	3859.71	2602.65
+Children                                          	edu packexporti #x                                	ese	Unknown	1999	2	4640.935833	1927.95	3524.16	2584.03
+Children                                          	exportiexporti #x                                 	ese	Unknown	1999	7	4565.828333	1853.57	2846.40	6447.36
+Women                                             	edu packamalg #x                                  	ese	Unknown	1999	6	4551.444167	1840.98	2478.75	3174.74
+Men                                               	edu packimporto #x                                	eing	Unknown	1999	4	5330.618333	2624.00	2602.65	3722.18
+Children                                          	importoexporti #x                                 	able	Unknown	1999	3	5024.325000	2319.29	2691.31	2350.53
+Children                                          	edu packexporti #x                                	ought	Unknown	1999	2	4836.831667	2132.80	3984.13	2182.12
+Children                                          	edu packexporti #x                                	ese	Unknown	1999	4	4640.935833	1950.03	2584.03	2919.51
+Shoes                                             	importoedu pack #x                                	bar	Unknown	1999	3	4791.740833	2100.87	2211.40	3278.75
+Men                                               	edu packimporto #x                                	ation	Unknown	1999	5	5839.670833	3151.27	3531.36	3053.78
+Men                                               	exportiimporto #x                                 	ese	Unknown	1999	3	4929.611667	2246.76	2207.89	3208.14
+Men                                               	edu packimporto #x                                	ation	Unknown	1999	2	5839.670833	3157.46	3425.68	2825.07
+Men                                               	exportiimporto #x                                 	bar	Unknown	1999	5	5311.965833	2631.11	2819.12	3022.54
+Music                                             	importoscholar #x                                 	ation	Unknown	1999	2	4388.770833	1714.25	2228.69	2407.93
+Shoes                                             	importoedu pack #x                                	ese	Unknown	1999	2	4989.784167	2315.46	4430.70	2976.26
+Children                                          	importoexporti #x                                 	able	Unknown	1999	4	5024.325000	2350.53	2319.29	2984.92
+Women                                             	amalgamalg #x                                     	ation	Unknown	1999	3	4586.300000	1917.37	3282.12	2557.46
+Children                                          	edu packexporti #x                                	ought	Unknown	1999	3	4836.831667	2182.12	2132.80	3466.25
+Shoes                                             	exportiedu pack #x                                	able	Unknown	1999	5	5640.362500	2987.78	2416.57	3437.87
+Men                                               	importoimporto #x                                 	ought	Unknown	1999	4	5071.261667	2425.64	2696.60	2641.17
+Shoes                                             	importoedu pack #x                                	ought	Unknown	1999	6	4443.995833	1801.87	2297.76	3078.40
+Children                                          	importoexporti #x                                 	ought	Unknown	1999	2	4480.353333	1838.88	3250.53	2375.04
+Men                                               	edu packimporto #x                                	ese	Unknown	1999	7	5102.436667	2464.06	2798.80	6978.09
+Shoes                                             	exportiedu pack #x                                	ought	Unknown	1999	4	5643.938333	3008.88	2652.44	3046.01
+Men                                               	importoimporto #x                                 	ought	Unknown	1999	2	5071.261667	2438.50	4282.24	2696.60
+Children                                          	edu packexporti #x                                	ation	Unknown	1999	6	4525.318333	1896.34	2883.95	2727.36
+Children                                          	amalgexporti #x                                   	ese	Unknown	1999	4	4212.889167	1588.72	2296.49	3077.12
+Shoes                                             	importoedu pack #x                                	eing	Unknown	1999	4	4884.260000	2261.01	2502.36	3210.47
+Men                                               	exportiimporto #x                                 	eing	Unknown	1999	3	5748.707500	3132.90	3747.34	3768.23
+Shoes                                             	exportiedu pack #x                                	able	Unknown	1999	2	5640.362500	3027.85	4285.15	3348.90
+Music                                             	edu packscholar #x                                	ought	Unknown	1999	3	4490.221667	1883.06	2321.31	2211.82
+Children                                          	exportiexporti #x                                 	able	Unknown	1999	2	4423.506667	1816.43	3192.19	3334.11
+Music                                             	amalgscholar #x                                   	eing	Unknown	1999	6	4894.035000	2288.21	3156.77	2845.45
+Music                                             	amalgscholar #x                                   	able	Unknown	1999	5	4829.768333	2225.46	3380.94	2782.42
+Women                                             	edu packamalg #x                                  	ation	Unknown	1999	3	4268.562500	1665.68	1758.65	2082.99
+Children                                          	exportiexporti #x                                 	bar	Unknown	1999	3	4267.374167	1667.03	2579.77	2802.05
+Men                                               	exportiimporto #x                                 	able	Unknown	1999	3	4987.821667	2387.78	2962.40	2928.83
+Women                                             	importoamalg #x                                   	bar	Unknown	1999	3	4309.575833	1710.42	2004.22	2742.41
+Shoes                                             	exportiedu pack #x                                	ought	Unknown	1999	5	5643.938333	3046.01	3008.88	2902.72
+Music                                             	exportischolar #x                                 	bar	Unknown	1999	4	4518.543333	1922.23	2683.26	2227.02
+Children                                          	amalgexporti #x                                   	bar	Unknown	1999	3	4441.555000	1846.53	3827.47	3623.30
+Men                                               	exportiimporto #x                                 	eing	Unknown	1999	5	5748.707500	3156.57	3768.23	3432.67
+Music                                             	importoscholar #x                                 	ese	Unknown	1999	2	4302.831667	1712.70	3561.46	2414.41
+Women                                             	amalgamalg #x                                     	eing	Unknown	1999	7	4494.382500	1904.98	2308.58	5603.14
+Women                                             	amalgamalg #x                                     	able	Unknown	1999	4	4582.713333	1994.71	2408.40	2321.48
+Music                                             	edu packscholar #x                                	able	Unknown	1999	3	4426.623333	1840.51	2707.80	3147.10
+Shoes                                             	importoedu pack #x                                	bar	Unknown	1999	2	4791.740833	2211.40	3912.90	2100.87
+Shoes                                             	exportiedu pack #x                                	able	Unknown	1999	7	5640.362500	3062.31	3437.87	6376.30
+Children                                          	amalgexporti #x                                   	eing	Unknown	1999	3	4733.152500	2155.79	2710.50	2685.74
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q48.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q48.sql.out
new file mode 100644
index 0000000000000..c7fa3497c93f8
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q48.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<sum(ss_quantity):bigint>
+-- !query output
+28312
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q49.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q49.sql.out
new file mode 100644
index 0000000000000..a1933e05565ca
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q49.sql.out
@@ -0,0 +1,37 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<channel:string,item:int,return_ratio:decimal(35,20),return_rank:int,currency_rank:int>
+-- !query output
+catalog	16735	0.50505050505050505051	1	1
+catalog	12633	0.69662921348314606742	2	2
+catalog	13967	0.70000000000000000000	3	3
+catalog	12819	0.70129870129870129870	4	8
+catalog	16155	0.72043010752688172043	5	4
+catalog	17681	0.75268817204301075269	6	5
+catalog	5975	0.76404494382022471910	7	6
+catalog	11451	0.76744186046511627907	8	7
+catalog	1689	0.80219780219780219780	9	9
+catalog	10311	0.81818181818181818182	10	10
+store	5111	0.78947368421052631579	1	1
+store	11073	0.83505154639175257732	2	3
+store	14429	0.84782608695652173913	3	2
+store	15927	0.86419753086419753086	4	4
+store	10171	0.86868686868686868687	5	5
+store	12783	0.88775510204081632653	6	6
+store	11075	0.89743589743589743590	7	7
+store	12889	0.95652173913043478261	8	8
+store	1939	0.99000000000000000000	9	9
+store	12975	1.00000000000000000000	10	10
+store	10455	1.00000000000000000000	10	10
+store	4333	1.00000000000000000000	10	10
+web	10485	0.48863636363636363636	1	1
+web	4483	0.52688172043010752688	2	2
+web	8833	0.58241758241758241758	3	3
+web	1165	0.61458333333333333333	4	4
+web	17197	0.73076923076923076923	5	5
+web	10319	0.73469387755102040816	6	6
+web	13159	0.75257731958762886598	7	7
+web	9629	0.77894736842105263158	8	8
+web	5909	0.78378378378378378378	9	9
+web	7057	0.86746987951807228916	10	10
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q5.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q5.sql.out
new file mode 100644
index 0000000000000..0f752c2136fa6
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q5.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<channel:string,id:string,sales:decimal(27,2),returns:decimal(27,2),profit:decimal(28,2)>
+-- !query output
+NULL	NULL	113832751.85	3036754.81	-31846972.27
+catalog channel	NULL	39549896.80	890805.91	-4700833.96
+catalog channel	catalog_pageAAAAAAAAAAABAAAA	217002.51	0.00	13655.90
+catalog channel	catalog_pageAAAAAAAAABABAAAA	79307.07	0.00	-3323.53
+catalog channel	catalog_pageAAAAAAAAACABAAAA	109295.76	3505.20	8275.32
+catalog channel	catalog_pageAAAAAAAAACJAAAAA	0.00	14.82	-113.03
+catalog channel	catalog_pageAAAAAAAAADABAAAA	112264.38	0.00	-11845.75
+catalog channel	catalog_pageAAAAAAAAADCBAAAA	111462.44	0.00	2684.17
+catalog channel	catalog_pageAAAAAAAAAEABAAAA	60875.92	753.55	991.75
+catalog channel	catalog_pageAAAAAAAAAECBAAAA	89060.07	0.00	7186.17
+catalog channel	catalog_pageAAAAAAAAAEPAAAAA	0.00	2466.17	-1023.59
+catalog channel	catalog_pageAAAAAAAAAFABAAAA	42313.54	0.00	-25310.58
+catalog channel	catalog_pageAAAAAAAAAFCBAAAA	74908.61	0.00	-10384.86
+catalog channel	catalog_pageAAAAAAAAAFPAAAAA	0.00	523.37	-422.16
+catalog channel	catalog_pageAAAAAAAAAGABAAAA	120028.56	0.00	4694.80
+catalog channel	catalog_pageAAAAAAAAAGBBAAAA	0.00	541.44	-170.28
+catalog channel	catalog_pageAAAAAAAAAGCBAAAA	97228.89	0.00	-7579.73
+catalog channel	catalog_pageAAAAAAAAAGPAAAAA	0.00	1814.55	-667.34
+catalog channel	catalog_pageAAAAAAAAAHABAAAA	69941.58	0.00	-4444.41
+catalog channel	catalog_pageAAAAAAAAAHBBAAAA	0.00	138.66	-128.30
+catalog channel	catalog_pageAAAAAAAAAHCBAAAA	73967.17	0.00	-5904.39
+catalog channel	catalog_pageAAAAAAAAAHPAAAAA	0.00	10125.01	-3291.39
+catalog channel	catalog_pageAAAAAAAAAICBAAAA	95341.98	0.00	-10935.38
+catalog channel	catalog_pageAAAAAAAAAIMAAAAA	0.00	247.30	-261.67
+catalog channel	catalog_pageAAAAAAAAAJCBAAAA	91289.86	0.00	-38131.57
+catalog channel	catalog_pageAAAAAAAAAJPAAAAA	0.00	19.98	-102.55
+catalog channel	catalog_pageAAAAAAAAAKABAAAA	0.00	3460.18	-1790.30
+catalog channel	catalog_pageAAAAAAAAAKCBAAAA	1409.22	0.00	-2367.32
+catalog channel	catalog_pageAAAAAAAAAKPAAAAA	176800.33	2203.74	17358.83
+catalog channel	catalog_pageAAAAAAAAALBBAAAA	0.00	1719.45	-223.86
+catalog channel	catalog_pageAAAAAAAAALCBAAAA	9622.11	0.00	-995.37
+catalog channel	catalog_pageAAAAAAAAALPAAAAA	226923.74	0.00	-17678.34
+catalog channel	catalog_pageAAAAAAAAAMCBAAAA	1044.59	0.00	-371.42
+catalog channel	catalog_pageAAAAAAAAAMPAAAAA	210026.33	0.00	-4598.43
+catalog channel	catalog_pageAAAAAAAAANABAAAA	0.00	4859.01	-620.39
+catalog channel	catalog_pageAAAAAAAAANPAAAAA	204079.30	6617.69	-43707.08
+catalog channel	catalog_pageAAAAAAAAAOCBAAAA	3093.75	0.00	-3029.40
+catalog channel	catalog_pageAAAAAAAAAOPAAAAA	205664.00	5.35	-7249.23
+catalog channel	catalog_pageAAAAAAAAAPCBAAAA	11986.70	0.00	5352.16
+catalog channel	catalog_pageAAAAAAAAAPPAAAAA	259979.15	0.00	15500.93
+catalog channel	catalog_pageAAAAAAAABAABAAAA	205054.92	0.00	-13900.72
+catalog channel	catalog_pageAAAAAAAABBABAAAA	123870.49	817.57	-35169.40
+catalog channel	catalog_pageAAAAAAAABCABAAAA	110311.76	6.60	-2807.92
+catalog channel	catalog_pageAAAAAAAABDABAAAA	54375.62	0.00	-16298.08
+catalog channel	catalog_pageAAAAAAAABDCBAAAA	77225.26	0.00	-9404.24
+catalog channel	catalog_pageAAAAAAAABDIAAAAA	0.00	2247.56	-536.43
+catalog channel	catalog_pageAAAAAAAABDPAAAAA	0.00	13366.54	-4827.89
+catalog channel	catalog_pageAAAAAAAABEABAAAA	83653.23	301.44	-14544.63
+catalog channel	catalog_pageAAAAAAAABECBAAAA	102256.79	0.00	2795.53
+catalog channel	catalog_pageAAAAAAAABEPAAAAA	0.00	4374.58	-3395.35
+catalog channel	catalog_pageAAAAAAAABFABAAAA	77015.36	0.00	-4370.15
+catalog channel	catalog_pageAAAAAAAABFBBAAAA	0.00	2435.93	-3450.68
+catalog channel	catalog_pageAAAAAAAABFCBAAAA	105831.18	0.00	12908.55
+catalog channel	catalog_pageAAAAAAAABFPAAAAA	0.00	203.00	-162.37
+catalog channel	catalog_pageAAAAAAAABGABAAAA	72193.84	0.00	-10412.85
+catalog channel	catalog_pageAAAAAAAABGCBAAAA	98074.44	0.00	-1396.21
+catalog channel	catalog_pageAAAAAAAABGIAAAAA	0.00	500.56	-315.89
+catalog channel	catalog_pageAAAAAAAABGPAAAAA	0.00	257.80	-125.00
+catalog channel	catalog_pageAAAAAAAABHABAAAA	109670.37	0.00	17680.12
+catalog channel	catalog_pageAAAAAAAABHBBAAAA	0.00	332.44	-218.41
+catalog channel	catalog_pageAAAAAAAABHCBAAAA	73731.28	0.00	-14352.31
+catalog channel	catalog_pageAAAAAAAABHPAAAAA	0.00	1366.24	-3867.70
+catalog channel	catalog_pageAAAAAAAABICBAAAA	71039.47	0.00	2742.64
+catalog channel	catalog_pageAAAAAAAABIPAAAAA	0.00	14223.07	-7591.98
+catalog channel	catalog_pageAAAAAAAABJBBAAAA	0.00	96.66	-140.68
+catalog channel	catalog_pageAAAAAAAABJCBAAAA	5619.24	0.00	-2531.23
+catalog channel	catalog_pageAAAAAAAABJPAAAAA	0.00	3881.66	-2212.48
+catalog channel	catalog_pageAAAAAAAABKCBAAAA	5420.99	0.00	-5691.26
+catalog channel	catalog_pageAAAAAAAABKPAAAAA	161336.55	0.00	-513.02
+catalog channel	catalog_pageAAAAAAAABLBBAAAA	0.00	303.27	-173.59
+catalog channel	catalog_pageAAAAAAAABLCBAAAA	201.70	0.00	-393.37
+catalog channel	catalog_pageAAAAAAAABLPAAAAA	162028.76	54.34	-54551.60
+catalog channel	catalog_pageAAAAAAAABMABAAAA	0.00	163.32	-383.25
+catalog channel	catalog_pageAAAAAAAABMBBAAAA	0.00	483.00	-4825.77
+catalog channel	catalog_pageAAAAAAAABMCBAAAA	2213.59	0.00	-212.78
+catalog channel	catalog_pageAAAAAAAABMPAAAAA	140634.77	138.84	-39261.21
+catalog channel	catalog_pageAAAAAAAABNABAAAA	0.00	NULL	NULL
+catalog channel	catalog_pageAAAAAAAABNBBAAAA	0.00	2152.88	-910.95
+catalog channel	catalog_pageAAAAAAAABNCBAAAA	21066.23	0.00	5619.74
+catalog channel	catalog_pageAAAAAAAABNPAAAAA	168937.50	0.00	-7428.35
+catalog channel	catalog_pageAAAAAAAABOABAAAA	0.00	756.28	-521.22
+catalog channel	catalog_pageAAAAAAAABOBBAAAA	0.00	289.38	-262.27
+catalog channel	catalog_pageAAAAAAAABOCBAAAA	3380.37	0.00	337.45
+catalog channel	catalog_pageAAAAAAAABOIAAAAA	0.00	553.08	-124.33
+catalog channel	catalog_pageAAAAAAAABOPAAAAA	178060.89	453.33	-39647.59
+catalog channel	catalog_pageAAAAAAAABPCBAAAA	308.43	0.00	-6294.87
+catalog channel	catalog_pageAAAAAAAABPPAAAAA	230886.45	423.60	-7979.91
+catalog channel	catalog_pageAAAAAAAACAABAAAA	143217.90	4739.60	-14933.90
+catalog channel	catalog_pageAAAAAAAACABBAAAA	0.00	2902.35	-421.56
+catalog channel	catalog_pageAAAAAAAACACBAAAA	0.00	275.93	-770.61
+catalog channel	catalog_pageAAAAAAAACBABAAAA	82856.21	0.00	-33285.95
+catalog channel	catalog_pageAAAAAAAACCABAAAA	87477.08	0.00	818.09
+catalog channel	catalog_pageAAAAAAAACDABAAAA	83602.86	0.00	-33313.03
+catalog channel	catalog_pageAAAAAAAACDCBAAAA	58195.74	0.00	-16518.32
+catalog channel	catalog_pageAAAAAAAACDPAAAAA	0.00	317.65	-658.75
+catalog channel	catalog_pageAAAAAAAACEABAAAA	51882.61	0.00	-18201.87
+catalog channel	catalog_pageAAAAAAAACECBAAAA	87686.10	0.00	4983.58
+catalog channel	catalog_pageAAAAAAAACEPAAAAA	0.00	9419.81	-3490.18
+catalog channel	catalog_pageAAAAAAAACFABAAAA	100092.31	0.00	2640.22
+catalog channel	catalog_pageAAAAAAAACFCBAAAA	82085.45	0.00	-6014.66
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q50.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q50.sql.out
new file mode 100644
index 0000000000000..b1f112afefd8a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q50.sql.out
@@ -0,0 +1,11 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<s_store_name:string,s_company_id:int,s_street_number:string,s_street_name:string,s_street_type:string,s_suite_number:string,s_city:string,s_county:string,s_state:string,s_zip:string,30 days :bigint,31 - 60 days :bigint,61 - 90 days :bigint,91 - 120 days :bigint,>120 days :bigint>
+-- !query output
+able	1	666	Cedar Spruce	Avenue         	Suite 10  	Midway	Williamson County	TN	31904     	78	53	68	63	103
+ation	1	405	3rd 	Wy             	Suite 220 	Fairview	Williamson County	TN	35709     	71	65	63	43	97
+bar	1	71	Cedar 	Blvd           	Suite B   	Midway	Williamson County	TN	31904     	79	54	43	67	115
+eing	1	914	Lake 11th	Road           	Suite T   	Midway	Williamson County	TN	31904     	82	45	56	49	105
+ese	1	120	6th 	Lane           	Suite B   	Midway	Williamson County	TN	31904     	63	52	68	43	108
+ought	1	32	3rd 	Street         	Suite 220 	Midway	Williamson County	TN	31904     	75	63	51	51	99
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q51.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q51.sql.out
new file mode 100644
index 0000000000000..a342736812723
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q51.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<item_sk:int,d_date:date,web_sales:decimal(27,2),store_sales:decimal(27,2),web_cumulative:decimal(27,2),store_cumulative:decimal(27,2)>
+-- !query output
+8	2000-01-16	NULL	20.83	35.36	20.83
+8	2000-01-18	83.21	NULL	83.21	20.83
+8	2000-02-11	NULL	69.88	83.21	69.88
+14	2000-01-14	NULL	31.65	183.58	31.65
+14	2000-01-29	NULL	63.92	183.58	63.92
+14	2000-02-10	NULL	103.12	183.58	103.12
+14	2000-02-15	NULL	157.03	183.58	157.03
+14	2000-02-16	NULL	183.40	183.58	183.40
+17	2000-05-24	439.75	NULL	439.75	398.92
+17	2000-06-09	NULL	407.77	439.75	407.77
+17	2000-06-13	NULL	408.53	439.75	408.53
+26	2000-01-14	124.68	NULL	124.68	61.34
+26	2000-01-29	NULL	72.77	124.68	72.77
+26	2000-02-06	126.41	NULL	126.41	72.77
+29	2000-02-05	160.79	NULL	160.79	141.46
+29	2000-02-06	167.35	NULL	167.35	141.46
+29	2000-02-07	214.40	NULL	214.40	141.46
+29	2000-02-15	NULL	164.35	214.40	164.35
+29	2000-02-18	NULL	172.82	214.40	172.82
+32	2000-01-21	59.48	NULL	59.48	6.20
+32	2000-01-25	NULL	10.50	59.48	10.50
+32	2000-02-03	NULL	38.01	59.48	38.01
+32	2000-02-09	109.67	NULL	109.67	38.01
+32	2000-02-23	NULL	43.04	109.67	43.04
+32	2000-02-26	NULL	81.08	109.67	81.08
+32	2000-03-05	NULL	99.43	109.67	99.43
+32	2000-03-06	NULL	99.43	109.67	99.43
+43	2000-04-09	172.09	NULL	172.09	169.55
+43	2000-04-23	257.19	NULL	257.19	207.76
+43	2000-04-24	272.83	NULL	272.83	207.76
+43	2000-05-15	NULL	221.70	272.83	221.70
+44	2000-01-02	75.87	26.58	75.87	26.58
+44	2000-01-05	125.14	NULL	125.14	26.58
+47	2000-02-14	148.00	NULL	148.00	74.22
+61	2000-01-19	140.35	75.64	140.35	75.64
+61	2000-01-28	231.75	NULL	231.75	75.64
+61	2000-02-04	NULL	156.79	231.75	156.79
+61	2000-02-27	NULL	224.70	231.75	224.70
+67	2000-01-01	88.63	46.38	88.63	46.38
+67	2000-01-02	NULL	53.21	88.63	53.21
+67	2000-01-04	185.14	100.85	185.14	100.85
+67	2000-01-29	309.98	NULL	309.98	100.85
+67	2000-02-14	NULL	148.29	309.98	148.29
+67	2000-02-20	NULL	174.01	309.98	174.01
+67	2000-03-05	NULL	199.85	309.98	199.85
+67	2000-03-07	NULL	284.98	309.98	284.98
+74	2000-01-06	71.82	NULL	71.82	9.01
+80	2000-03-16	NULL	67.38	169.88	67.38
+80	2000-04-10	NULL	137.63	169.88	137.63
+80	2000-07-01	309.57	NULL	309.57	273.52
+80	2000-07-02	455.94	NULL	455.94	273.52
+80	2000-07-25	NULL	443.48	455.94	443.48
+80	2000-08-04	NULL	443.48	455.94	443.48
+82	2000-01-01	85.75	64.27	85.75	64.27
+83	2000-01-08	134.96	NULL	134.96	8.10
+83	2000-02-01	NULL	52.54	134.96	52.54
+83	2000-02-27	NULL	76.22	134.96	76.22
+83	2000-03-01	NULL	112.53	134.96	112.53
+83	2000-03-02	248.36	NULL	248.36	112.53
+83	2000-03-05	NULL	118.07	248.36	118.07
+83	2000-03-16	250.29	NULL	250.29	118.07
+83	2000-03-19	NULL	163.65	250.29	163.65
+83	2000-04-07	289.65	NULL	289.65	163.65
+83	2000-04-08	NULL	201.68	289.65	201.68
+83	2000-04-10	357.89	NULL	357.89	201.68
+83	2000-04-20	364.69	NULL	364.69	201.68
+83	2000-04-26	371.15	NULL	371.15	201.68
+83	2000-05-06	NULL	206.29	371.15	206.29
+83	2000-05-27	NULL	228.07	371.15	228.07
+83	2000-06-06	393.25	NULL	393.25	228.07
+83	2000-06-09	NULL	235.88	393.25	235.88
+83	2000-06-10	NULL	299.68	393.25	299.68
+85	2000-01-16	48.36	NULL	48.36	42.45
+85	2000-01-18	NULL	47.47	48.36	47.47
+85	2000-02-02	126.73	NULL	126.73	47.47
+85	2000-04-01	NULL	47.47	126.73	47.47
+85	2000-04-08	NULL	58.91	126.73	58.91
+85	2000-04-15	NULL	69.74	126.73	69.74
+85	2000-05-01	NULL	93.80	126.73	93.80
+86	2000-01-16	NULL	19.92	38.25	19.92
+86	2000-01-17	NULL	34.32	38.25	34.32
+86	2000-05-06	257.78	NULL	257.78	204.60
+86	2000-05-08	NULL	222.06	257.78	222.06
+86	2000-05-12	NULL	235.00	257.78	235.00
+89	2000-02-01	143.30	NULL	143.30	125.19
+89	2000-02-11	273.46	NULL	273.46	125.19
+89	2000-03-01	NULL	131.53	273.46	131.53
+89	2000-03-24	NULL	200.30	273.46	200.30
+89	2000-04-04	NULL	212.25	273.46	212.25
+89	2000-04-05	NULL	265.57	273.46	265.57
+97	2000-01-22	114.93	NULL	114.93	55.81
+97	2000-01-31	NULL	60.35	114.93	60.35
+97	2000-03-07	263.86	NULL	263.86	220.68
+98	2000-02-10	NULL	1.04	42.10	1.04
+98	2000-02-23	44.27	NULL	44.27	1.04
+98	2000-02-24	274.58	NULL	274.58	1.04
+98	2000-02-29	301.60	NULL	301.60	1.04
+98	2000-03-13	309.78	NULL	309.78	1.04
+98	2000-03-17	335.79	NULL	335.79	1.04
+98	2000-03-27	NULL	37.67	335.79	37.67
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q52.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q52.sql.out
new file mode 100644
index 0000000000000..568469bf9afa6
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q52.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<d_year:int,brand_id:int,brand:string,ext_price:decimal(17,2)>
+-- !query output
+2000	1002002	importoamalg #x                                   	98247.74
+2000	4001001	amalgedu pack #x                                  	97944.74
+2000	2004002	edu packimporto #x                                	90901.39
+2000	2004001	edu packimporto #x                                	89064.36
+2000	9003003	exportimaxi #x                                    	67483.21
+2000	4004002	edu packedu pack #x                               	64713.03
+2000	2002002	importoimporto #x                                 	61594.80
+2000	2001001	amalgimporto #x                                   	57219.92
+2000	10016008	corpamalgamalg #x                                 	56274.63
+2000	3002001	importoexporti #x                                 	52756.72
+2000	3004001	edu packexporti #x                                	49336.05
+2000	4003001	exportiedu pack #x                                	47889.60
+2000	1004001	edu packbrand #x                                  	46738.09
+2000	3003002	exportiexporti #x                                 	46024.13
+2000	3004002	edu packexporti #x                                	42712.09
+2000	6003005	edu packmaxi #x                                   	42381.36
+2000	9015011	exportibrand #x                                   	41249.43
+2000	5001002	amalgscholar #x                                   	40525.36
+2000	9003006	exportimaxi #x                                    	40252.87
+2000	NULL	NULL	39507.19
+2000	5002001	importoscholar #x                                 	38829.75
+2000	3003001	exportiexporti #x                                 	36506.92
+2000	4002001	importoedu pack #x                                	35994.46
+2000	9013003	exportiunivamalg #x                               	35549.32
+2000	6003004	exporticorp #x                                    	34314.00
+2000	8010002	univmaxi #x                                       	32664.31
+2000	6005004	scholarcorp #x                                    	32175.90
+2000	7011006	amalgnameless #x                                  	30979.02
+2000	10008016	namelessunivamalg #x                             	30424.25
+2000	10014001	edu packamalgamalg #x                             	30406.55
+2000	5003001	exportischolar #x                                 	30278.44
+2000	10003008	exportiunivamalg #x                               	30127.51
+2000	7002002	importobrand #x                                   	29602.59
+2000	7005006	scholarbrand #x                                   	29319.83
+2000	9012009	importounivamalg #x                               	29204.86
+2000	1001001	importoexporti #x                                 	28800.60
+2000	7011010	amalgnameless #x                                 	28614.32
+2000	10007004	exportiamalg #x                                   	28600.67
+2000	4001002	amalgedu pack #x                                  	27963.59
+2000	4002001	edu packscholar #x                                	27934.37
+2000	10016002	corpamalgamalg #x                                 	27695.20
+2000	7015008	scholarnameless #x                                	27490.44
+2000	8006010	corpnameless #x                                  	27390.35
+2000	6008004	namelesscorp #x                                   	27296.01
+2000	5002002	importoscholar #x                                 	27037.76
+2000	8003006	exportinameless #x                                	26842.25
+2000	1003001	exportiamalg #x                                   	26808.25
+2000	10001007	amalgunivamalg #x                                 	26031.20
+2000	6011001	amalgbrand #x                                     	25567.26
+2000	1001001	amalgamalg #x                                     	25210.15
+2000	8002008	importonameless #x                                	24937.40
+2000	5004002	edu packscholar #x                                	24800.08
+2000	9007003	brandmaxi #x                                      	24694.06
+2000	4002002	importoedu pack #x                                	23924.37
+2000	9008003	namelessmaxi #x                                   	23871.65
+2000	3001001	amalgexporti #x                                   	23023.19
+2000	10009002	maxiunivamalg #x                                  	22932.13
+2000	8014006	edu packmaxi #x                                   	22164.19
+2000	8008006	namelessnameless #x                               	22159.19
+2000	1002001	importoamalg #x                                   	21952.87
+2000	10005002	edu packedu pack #x                               	21623.64
+2000	9016008	corpunivamalg #x                                  	21503.39
+2000	8002007	importonameless #x                                	21370.99
+2000	7013002	exportinameless #x                                	21315.92
+2000	5003001	importonameless #x                                	21140.49
+2000	1003002	exportiamalg #x                                   	20558.37
+2000	7014009	edu packedu pack #x                               	20518.93
+2000	4003001	corpnameless #x                                   	20345.42
+2000	3002001	edu packedu pack #x                               	19948.39
+2000	6015004	scholarbrand #x                                   	19806.27
+2000	7015004	scholarnameless #x                                	19805.86
+2000	10005002	exportiimporto #x                                 	19553.70
+2000	6005003	amalgbrand #x                                     	18383.14
+2000	10013010	exportiamalgamalg #x                             	18349.28
+2000	10010002	exportiexporti #x                                 	18338.53
+2000	2002001	amalgimporto #x                                   	18298.41
+2000	10009016	maxiunivamalg #x                                 	17942.21
+2000	2002001	exportiexporti #x                                 	17817.83
+2000	4003002	exportiedu pack #x                                	17709.48
+2000	9004008	edu packmaxi #x                                   	17660.51
+2000	7006005	corpbrand #x                                      	17628.79
+2000	10013006	exportiamalgamalg #x                              	17579.59
+2000	9002011	brandbrand #x                                     	17177.90
+2000	2001001	exportiexporti #x                                 	16270.97
+2000	6011003	amalgbrand #x                                     	16222.01
+2000	8004002	edu packnameless #x                               	16054.52
+2000	10008008	importoexporti #x                                 	15563.77
+2000	6002008	importocorp #x                                    	15447.93
+2000	4003001	edu packimporto #x                                	15313.51
+2000	8015004	scholarmaxi #x                                    	15299.50
+2000	8010001	univmaxi #x                                       	15188.28
+2000	7016008	corpnameless #x                                   	15123.33
+2000	6008001	namelesscorp #x                                   	15096.05
+2000	7009003	maxibrand #x                                      	15073.42
+2000	7015007	scholarnameless #x                                	14934.32
+2000	1001002	amalgamalg #x                                     	14264.80
+2000	7010005	univnameless #x                                   	14247.64
+2000	9005002	scholarmaxi #x                                    	14062.39
+2000	7012008	importonameless #x                                	14043.20
+2000	9010002	univunivamalg #x                                  	13922.13
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q53.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q53.sql.out
new file mode 100644
index 0000000000000..8528e2da8bfe0
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q53.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_manufact_id:int,sum_sales:decimal(17,2),avg_quarterly_sales:decimal(21,6)>
+-- !query output
+132	119.11	258.580000
+132	222.31	258.580000
+132	342.09	258.580000
+132	350.81	258.580000
+612	77.21	301.970000
+612	174.17	301.970000
+612	248.75	301.970000
+612	707.75	301.970000
+650	240.94	329.890000
+650	451.30	329.890000
+483	175.17	341.280000
+483	284.63	341.280000
+483	379.24	341.280000
+483	526.08	341.280000
+39	83.05	347.250000
+39	201.75	347.250000
+39	249.20	347.250000
+39	855.00	347.250000
+659	270.05	357.907500
+659	301.12	357.907500
+659	521.64	357.907500
+487	194.22	363.632500
+487	213.65	363.632500
+487	489.98	363.632500
+487	556.68	363.632500
+423	156.29	372.700000
+423	323.98	372.700000
+423	607.23	372.700000
+872	117.67	375.220000
+872	118.10	375.220000
+872	880.58	375.220000
+556	245.39	386.777500
+556	311.12	386.777500
+556	446.09	386.777500
+556	544.51	386.777500
+179	308.69	388.540000
+179	438.91	388.540000
+274	40.68	390.370000
+274	140.57	390.370000
+274	283.26	390.370000
+274	1096.97	390.370000
+551	154.35	397.355000
+551	327.31	397.355000
+551	696.52	397.355000
+67	271.97	397.382500
+67	305.28	397.382500
+67	621.24	397.382500
+53	110.01	397.517500
+53	250.03	397.517500
+53	468.76	397.517500
+53	761.27	397.517500
+172	154.34	398.030000
+172	347.87	398.030000
+172	439.68	398.030000
+172	650.23	398.030000
+915	210.94	403.120000
+915	359.81	403.120000
+915	678.57	403.120000
+350	30.44	407.737500
+350	328.11	407.737500
+350	339.56	407.737500
+350	932.84	407.737500
+203	164.68	408.595000
+203	305.41	408.595000
+203	735.85	408.595000
+2	239.50	408.602500
+2	289.78	408.602500
+2	696.25	408.602500
+687	70.08	410.057500
+687	80.72	410.057500
+687	368.29	410.057500
+687	1121.14	410.057500
+113	76.68	414.170000
+113	268.47	414.170000
+113	531.42	414.170000
+113	780.11	414.170000
+161	224.11	416.682500
+161	298.25	416.682500
+161	480.46	416.682500
+161	663.91	416.682500
+233	284.33	421.205000
+233	320.60	421.205000
+233	322.01	421.205000
+233	757.88	421.205000
+743	173.26	430.360000
+743	198.59	430.360000
+743	565.04	430.360000
+743	784.55	430.360000
+581	51.95	432.642500
+581	247.90	432.642500
+581	509.07	432.642500
+581	921.65	432.642500
+107	254.35	434.100000
+107	348.41	434.100000
+107	388.70	434.100000
+107	744.94	434.100000
+359	163.98	436.225000
+359	215.17	436.225000
+359	379.98	436.225000
+359	985.77	436.225000
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q54.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q54.sql.out
new file mode 100644
index 0000000000000..dd5f10b74372c
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q54.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<segment:int,num_customers:bigint,segment_base:int>
+-- !query output
+
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q55.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q55.sql.out
new file mode 100644
index 0000000000000..9592803da4665
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q55.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<brand_id:int,brand:string,ext_price:decimal(17,2)>
+-- !query output
+4002001	importoedu pack #x                                	141955.03
+2001001	amalgimporto #x                                   	115066.00
+2003001	exportiimporto #x                                 	96597.01
+5003002	exportischolar #x                                 	93316.15
+1002001	importoamalg #x                                   	85318.86
+2002001	importoimporto #x                                 	85035.18
+3003001	exportiexporti #x                                 	80017.81
+2004001	edu packimporto #x                                	68604.08
+9001008	amalgmaxi #x                                      	60900.34
+4004001	edu packedu pack #x                               	58815.34
+4003002	exportiedu pack #x                                	56587.37
+6012003	importobrand #x                                   	55151.38
+1001002	amalgamalg #x                                     	54471.07
+3004001	edu packexporti #x                                	54057.61
+3001001	amalgexporti #x                                   	52690.20
+4003001	exportiedu pack #x                                	51021.71
+5002002	importoscholar #x                                 	46601.52
+8002007	importonameless #x                                	45983.92
+6016003	corpbrand #x                                      	44911.30
+1003001	exportiamalg #x                                   	44127.51
+6014003	edu packbrand #x                                  	41310.31
+7014009	edu packnameless #x                               	40891.96
+7008004	namelessbrand #x                                  	36589.35
+10001016	amalgunivamalg #x                                	36270.01
+6016004	corpbrand #x                                      	36208.64
+9013011	exportiunivamalg #x                              	35794.13
+1003002	exportiamalg #x                                   	35113.55
+1004002	edu packamalg #x                                  	34667.61
+7016005	corpnameless #x                                   	33397.41
+7014005	edu packnameless #x                               	33313.92
+7012010	importonameless #x                               	32862.88
+7012001	importonameless #x                                	32815.12
+6001007	amalgcorp #x                                      	31948.28
+3002001	importoexporti #x                                 	31518.29
+8015001	scholarmaxi #x                                    	31235.19
+10012006	importoamalgamalg #x                              	31123.11
+6004004	edu packcorp #x                                   	29839.27
+10009008	maxiunivamalg #x                                  	29180.04
+5004001	edu packscholar #x                                	28912.61
+9006011	corpmaxi #x                                      	26493.45
+8014004	edu packmaxi #x                                   	26228.69
+9012008	importounivamalg #x                               	25354.49
+8015003	scholarmaxi #x                                    	23994.34
+7013009	exportinameless #x                                	23887.30
+2004002	edu packimporto #x                                	23778.49
+6005005	scholarcorp #x                                    	23243.88
+5001001	amalgscholar #x                                   	23043.20
+8009007	maxinameless #x                                   	22910.03
+8004009	edu packnameless #x                               	22679.38
+9016002	corpunivamalg #x                                  	21881.94
+8003001	exportinameless #x                                	21655.96
+8004004	edu packnameless #x                               	20920.43
+10010007	univamalgamalg #x                                 	20530.11
+8007004	brandnameless #x                                  	19973.79
+9015005	scholarunivamalg #x                               	19929.31
+6004002	edu packcorp #x                                   	19890.20
+10004005	edu packunivamalg #x                              	19819.04
+9014008	edu packunivamalg #x                              	19700.99
+5003001	exportischolar #x                                 	19595.76
+7007006	brandbrand #x                                     	19541.72
+8009001	maxinameless #x                                   	19474.02
+2002002	importoimporto #x                                 	19249.25
+8003002	exportinameless #x                                	18931.96
+9016011	corpunivamalg #x                                 	18840.83
+10012011	importoamalgamalg #x                             	18632.17
+9004005	edu packmaxi #x                                   	17997.32
+7003003	exportibrand #x                                   	16699.26
+9011011	amalgunivamalg #x                                	16656.62
+10008002	namelessunivamalg #x                              	16609.49
+10016015	corpamalgamalg #x                                	16397.21
+8008001	namelessnameless #x                               	15538.37
+6004007	edu packcorp #x                                   	15474.56
+7006001	corpbrand #x                                      	15365.31
+6010003	univbrand #x                                      	15337.21
+1001001	amalgamalg #x                                     	14813.15
+9012003	importounivamalg #x                               	14033.76
+8015008	scholarmaxi #x                                    	13408.30
+6002005	importocorp #x                                    	13393.21
+7010009	univnameless #x                                   	13359.47
+10010002	univamalgamalg #x                                 	13288.24
+8014003	edu packmaxi #x                                   	13137.78
+10001009	amalgunivamalg #x                                 	12326.55
+5001002	amalgscholar #x                                   	12319.21
+5002001	importoscholar #x                                 	12023.80
+8005010	scholarnameless #x                               	12017.53
+6008005	namelesscorp #x                                   	11908.45
+3001002	amalgexporti #x                                   	11679.83
+10008006	namelessunivamalg #x                              	11503.15
+10014009	edu packamalgamalg #x                             	11462.45
+10007017	brandunivamalg #x                                	11188.38
+7002004	importobrand #x                                   	10699.51
+7001006	amalgbrand #x                                     	10166.47
+10004009	edu packunivamalg #x                              	9913.46
+10002011	importounivamalg #x                              	9828.64
+3003002	exportiexporti #x                                 	9521.40
+10014017	edu packamalgamalg #x                            	9150.09
+6002003	importocorp #x                                    	8810.78
+6001005	amalgcorp #x                                      	8582.71
+4001001	amalgedu pack #x                                  	8362.72
+8004002	edu packnameless #x                               	8010.27
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q56.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q56.sql.out
new file mode 100644
index 0000000000000..f6d3a7470781c
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q56.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_item_id:string,total_sales:decimal(27,2)>
+-- !query output
+AAAAAAAANMMDAAAA	NULL
+AAAAAAAAIMPDAAAA	0.00
+AAAAAAAAEGJBAAAA	0.59
+AAAAAAAAOGIDAAAA	3.92
+AAAAAAAAFLGAAAAA	17.01
+AAAAAAAAGPEEAAAA	21.60
+AAAAAAAACJCDAAAA	27.30
+AAAAAAAAAAEDAAAA	38.88
+AAAAAAAACPBEAAAA	46.20
+AAAAAAAAMGKBAAAA	50.79
+AAAAAAAACLHAAAAA	58.52
+AAAAAAAAKJOBAAAA	74.48
+AAAAAAAAGMEDAAAA	78.90
+AAAAAAAADGCCAAAA	84.81
+AAAAAAAALLDAAAAA	95.60
+AAAAAAAAONEDAAAA	100.91
+AAAAAAAAAKMAAAAA	102.75
+AAAAAAAAIJIAAAAA	103.53
+AAAAAAAAMLFDAAAA	112.10
+AAAAAAAAHDDAAAAA	113.62
+AAAAAAAAPBMDAAAA	117.80
+AAAAAAAAPFCDAAAA	119.56
+AAAAAAAALICBAAAA	119.68
+AAAAAAAAKAJAAAAA	126.21
+AAAAAAAAOIAAAAAA	127.82
+AAAAAAAAABDEAAAA	145.52
+AAAAAAAAOCKDAAAA	149.00
+AAAAAAAAEMECAAAA	154.20
+AAAAAAAAGOLDAAAA	172.12
+AAAAAAAAAGLDAAAA	173.46
+AAAAAAAAGBGEAAAA	178.20
+AAAAAAAAKOBDAAAA	183.60
+AAAAAAAAEGGDAAAA	187.32
+AAAAAAAAHDKCAAAA	227.76
+AAAAAAAAFDBEAAAA	236.91
+AAAAAAAAEICCAAAA	241.28
+AAAAAAAAAKAAAAAA	250.16
+AAAAAAAAOMBEAAAA	310.90
+AAAAAAAAANHCAAAA	320.32
+AAAAAAAAIJGAAAAA	336.00
+AAAAAAAAAFAAAAAA	336.28
+AAAAAAAAKFGBAAAA	380.37
+AAAAAAAAEACCAAAA	389.75
+AAAAAAAAIOMAAAAA	418.92
+AAAAAAAAEKODAAAA	448.26
+AAAAAAAAENDAAAAA	456.30
+AAAAAAAAAJGBAAAA	457.68
+AAAAAAAAIBEAAAAA	466.74
+AAAAAAAAOMEEAAAA	469.80
+AAAAAAAAMDIDAAAA	501.28
+AAAAAAAAPANDAAAA	519.27
+AAAAAAAAKKECAAAA	523.27
+AAAAAAAAKPABAAAA	564.00
+AAAAAAAABHMCAAAA	566.58
+AAAAAAAAGMGBAAAA	584.56
+AAAAAAAAGKNCAAAA	614.00
+AAAAAAAAGELBAAAA	625.04
+AAAAAAAAKGGDAAAA	642.63
+AAAAAAAAFLPAAAAA	680.48
+AAAAAAAAGJODAAAA	698.32
+AAAAAAAAIBDBAAAA	702.93
+AAAAAAAAGAFAAAAA	733.20
+AAAAAAAAMPEEAAAA	760.16
+AAAAAAAAMDFCAAAA	776.72
+AAAAAAAANPFEAAAA	778.00
+AAAAAAAAMHEAAAAA	778.27
+AAAAAAAAOHLDAAAA	801.88
+AAAAAAAAGIJDAAAA	815.74
+AAAAAAAAICOCAAAA	817.74
+AAAAAAAAAICAAAAA	823.35
+AAAAAAAAKCNAAAAA	833.31
+AAAAAAAACDGAAAAA	836.90
+AAAAAAAAAAKDAAAA	850.07
+AAAAAAAAFOJAAAAA	859.28
+AAAAAAAAMNLCAAAA	865.29
+AAAAAAAAOINCAAAA	865.60
+AAAAAAAAKGOBAAAA	875.20
+AAAAAAAAOFKAAAAA	882.60
+AAAAAAAAIGMCAAAA	886.89
+AAAAAAAAIIOBAAAA	892.05
+AAAAAAAAGOBBAAAA	909.74
+AAAAAAAAPNNCAAAA	981.72
+AAAAAAAAKPAAAAAA	1006.11
+AAAAAAAANDBCAAAA	1028.34
+AAAAAAAAABCCAAAA	1051.08
+AAAAAAAACJCEAAAA	1055.78
+AAAAAAAACEPCAAAA	1064.00
+AAAAAAAACGFEAAAA	1067.78
+AAAAAAAAMOIAAAAA	1073.80
+AAAAAAAAMIFDAAAA	1104.17
+AAAAAAAAAOHBAAAA	1104.40
+AAAAAAAAMGJBAAAA	1124.12
+AAAAAAAAAEFCAAAA	1134.68
+AAAAAAAAAJIDAAAA	1153.92
+AAAAAAAAFEKDAAAA	1156.54
+AAAAAAAAGIBEAAAA	1165.02
+AAAAAAAAKLBBAAAA	1181.93
+AAAAAAAAOBBAAAAA	1183.08
+AAAAAAAABEEEAAAA	1190.91
+AAAAAAAADHJAAAAA	1202.11
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q57.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q57.sql.out
new file mode 100644
index 0000000000000..dc712720d3ac5
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q57.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_category:string,i_brand:string,cc_name:string,d_year:int,d_moy:int,avg_monthly_sales:decimal(21,6),sum_sales:decimal(17,2),psum:decimal(17,2),nsum:decimal(17,2)>
+-- !query output
+Music                                             	amalgscholar #x                                   	Mid Atlantic	1999	2	6662.669167	1961.57	4348.07	3386.25
+Shoes                                             	amalgedu pack #x                                  	North Midwest	1999	2	6493.071667	2044.05	4348.88	3443.20
+Shoes                                             	exportiedu pack #x                                	Mid Atlantic	1999	3	7416.141667	2980.15	4654.22	5157.83
+Children                                          	importoexporti #x                                 	North Midwest	1999	4	6577.143333	2152.15	3291.07	3659.32
+Shoes                                             	importoedu pack #x                                	NY Metro	1999	6	6926.960833	2523.33	4014.93	4254.99
+Men                                               	importoimporto #x                                 	North Midwest	1999	2	6707.315833	2449.22	4311.11	3583.31
+Men                                               	amalgimporto #x                                   	North Midwest	1999	4	7098.680833	2965.42	3526.45	4923.53
+Men                                               	exportiimporto #x                                 	NY Metro	1999	2	7146.240000	3013.99	6183.83	5091.17
+Children                                          	amalgexporti #x                                   	North Midwest	1999	4	6364.467500	2270.79	3330.83	3817.50
+Men                                               	edu packimporto #x                                	North Midwest	1999	4	7386.333333	3329.74	3488.01	4860.20
+Men                                               	edu packimporto #x                                	North Midwest	1999	2	7386.333333	3347.65	4007.40	3488.01
+Music                                             	edu packscholar #x                                	NY Metro	1999	7	6639.040000	2653.55	4219.52	10071.22
+Music                                             	amalgscholar #x                                   	North Midwest	1999	4	6719.304167	2739.33	3690.54	3872.98
+Men                                               	importoimporto #x                                 	NY Metro	1999	3	6610.034167	2645.24	3661.14	4282.01
+Music                                             	exportischolar #x                                 	NY Metro	1999	2	7043.051667	3115.83	4457.95	5258.95
+Men                                               	edu packimporto #x                                	North Midwest	1999	3	7386.333333	3488.01	3347.65	3329.74
+Shoes                                             	exportiedu pack #x                                	North Midwest	1999	3	7255.790000	3411.07	4194.64	3624.85
+Women                                             	exportiamalg #x                                   	Mid Atlantic	1999	2	5646.671667	1809.52	4198.70	2172.85
+Music                                             	importoscholar #x                                 	Mid Atlantic	1999	6	6279.081667	2456.98	4361.44	4256.24
+Children                                          	importoexporti #x                                 	Mid Atlantic	1999	7	6786.750000	2978.82	3942.59	7809.22
+Music                                             	exportischolar #x                                 	North Midwest	1999	2	7041.705833	3245.77	3608.31	4127.40
+Shoes                                             	importoedu pack #x                                	Mid Atlantic	1999	2	6864.320833	3104.48	3135.52	3606.28
+Shoes                                             	importoedu pack #x                                	Mid Atlantic	1999	1	6864.320833	3135.52	14580.30	3104.48
+Children                                          	edu packexporti #x                                	Mid Atlantic	1999	5	6511.800833	2785.92	3956.90	3906.63
+Women                                             	amalgamalg #x                                     	Mid Atlantic	1999	2	6480.683333	2769.69	3353.67	3876.51
+Men                                               	exportiimporto #x                                 	NY Metro	1999	5	7146.240000	3440.57	3561.11	3971.13
+Women                                             	importoamalg #x                                   	NY Metro	1999	3	6512.794167	2808.28	3789.51	4335.27
+Men                                               	amalgimporto #x                                   	NY Metro	1999	2	6720.550000	3018.90	4328.03	3810.74
+Shoes                                             	exportiedu pack #x                                	North Midwest	1999	7	7255.790000	3557.87	4937.98	9496.49
+Music                                             	exportischolar #x                                 	Mid Atlantic	1999	5	6791.260833	3096.02	3918.04	3801.90
+Men                                               	exportiimporto #x                                 	North Midwest	1999	1	7343.719167	3652.72	13689.13	3984.13
+Men                                               	importoimporto #x                                 	North Midwest	1999	5	6707.315833	3030.18	4977.33	4620.75
+Children                                          	exportiexporti #x                                 	NY Metro	1999	2	6386.880833	2717.07	4809.11	3355.48
+Men                                               	amalgimporto #x                                   	North Midwest	1999	2	7098.680833	3440.27	5293.69	3526.45
+Men                                               	importoimporto #x                                 	NY Metro	1999	5	6610.034167	2954.71	4282.01	3166.43
+Shoes                                             	exportiedu pack #x                                	North Midwest	1999	4	7255.790000	3624.85	3411.07	5169.09
+Men                                               	exportiimporto #x                                 	North Midwest	1999	5	7343.719167	3727.75	3729.62	4580.93
+Men                                               	exportiimporto #x                                 	North Midwest	1999	4	7343.719167	3729.62	4033.37	3727.75
+Music                                             	edu packscholar #x                                	North Midwest	1999	2	6489.175000	2875.98	4299.82	4028.97
+Men                                               	edu packimporto #x                                	NY Metro	1999	1	7202.242500	3614.07	15582.63	4234.79
+Music                                             	importoscholar #x                                 	North Midwest	1999	2	5816.271667	2229.79	2919.29	4298.41
+Men                                               	exportiimporto #x                                 	NY Metro	1999	4	7146.240000	3561.11	5091.17	3440.57
+Shoes                                             	exportiedu pack #x                                	NY Metro	1999	7	7073.462500	3493.41	4534.31	8701.59
+Music                                             	exportischolar #x                                 	Mid Atlantic	1999	3	6791.260833	3218.13	3847.87	3918.04
+Shoes                                             	edu packedu pack #x                               	NY Metro	1999	4	6203.331667	2631.02	4424.18	4186.85
+Men                                               	amalgimporto #x                                   	North Midwest	1999	3	7098.680833	3526.45	3440.27	2965.42
+Men                                               	edu packimporto #x                                	NY Metro	1999	3	7202.242500	3639.93	4234.79	4016.25
+Children                                          	amalgexporti #x                                   	North Midwest	1999	2	6364.467500	2825.09	4111.08	3330.83
+Shoes                                             	edu packedu pack #x                               	North Midwest	1999	2	6464.239167	2928.99	4233.86	3840.97
+Shoes                                             	amalgedu pack #x                                  	North Midwest	1999	4	6493.071667	2962.20	3443.20	4212.60
+Music                                             	importoscholar #x                                 	NY Metro	1999	4	5707.844167	2179.41	3789.16	4317.53
+Shoes                                             	exportiedu pack #x                                	Mid Atlantic	1999	1	7416.141667	3892.54	14170.68	4654.22
+Women                                             	importoamalg #x                                   	NY Metro	1999	5	6512.794167	2991.07	4335.27	4624.86
+Music                                             	exportischolar #x                                 	NY Metro	1999	4	7043.051667	3521.99	5258.95	4135.21
+Women                                             	edu packamalg #x                                  	Mid Atlantic	1999	2	6354.045833	2836.23	3719.67	3527.07
+Music                                             	amalgscholar #x                                   	NY Metro	1999	3	6123.475000	2617.39	3080.43	4919.93
+Shoes                                             	amalgedu pack #x                                  	Mid Atlantic	1999	7	6674.896667	3178.12	3342.98	8050.81
+Men                                               	amalgimporto #x                                   	Mid Atlantic	1999	5	6618.534167	3127.28	4291.66	4669.62
+Women                                             	amalgamalg #x                                     	North Midwest	1999	6	6874.250000	3387.53	4798.69	4329.48
+Women                                             	exportiamalg #x                                   	Mid Atlantic	1999	3	5646.671667	2172.85	1809.52	4461.31
+Children                                          	edu packexporti #x                                	NY Metro	1999	2	6112.954167	2641.54	3567.58	3196.45
+Children                                          	amalgexporti #x                                   	NY Metro	1999	5	6294.100833	2834.48	3317.60	3803.79
+Women                                             	edu packamalg #x                                  	NY Metro	1999	5	6027.880000	2575.81	2750.96	4459.01
+Music                                             	exportischolar #x                                 	North Midwest	1999	6	7041.705833	3589.97	4134.03	4892.26
+Music                                             	exportischolar #x                                 	North Midwest	1999	4	7041.705833	3593.86	4127.40	4134.03
+Men                                               	importoimporto #x                                 	NY Metro	1999	6	6610.034167	3166.43	2954.71	3673.39
+Music                                             	exportischolar #x                                 	North Midwest	1999	1	7041.705833	3608.31	15046.54	3245.77
+Music                                             	edu packscholar #x                                	Mid Atlantic	1999	2	6602.385000	3173.44	3434.91	3929.40
+Music                                             	amalgscholar #x                                   	NY Metro	1999	6	6123.475000	2699.75	4038.47	3330.87
+Children                                          	importoexporti #x                                 	Mid Atlantic	1999	4	6786.750000	3366.25	3847.60	4259.57
+Men                                               	edu packimporto #x                                	Mid Atlantic	1999	1	7230.493333	3811.64	14668.93	4497.31
+Shoes                                             	importoedu pack #x                                	Mid Atlantic	1999	5	6864.320833	3449.62	3869.15	3531.93
+Children                                          	edu packexporti #x                                	North Midwest	1999	2	6739.498333	3328.01	4986.50	3623.32
+Children                                          	importoexporti #x                                 	Mid Atlantic	1999	1	6786.750000	3376.55	12504.93	5018.22
+Children                                          	edu packexporti #x                                	NY Metro	1999	7	6112.954167	2711.04	3254.04	9465.10
+Shoes                                             	importoedu pack #x                                	North Midwest	1999	3	6588.741667	3187.25	4283.21	3212.76
+Men                                               	importoimporto #x                                 	Mid Atlantic	1999	3	6702.415000	3310.55	3981.06	4901.56
+Men                                               	edu packimporto #x                                	North Midwest	1999	1	7386.333333	4007.40	14005.45	3347.65
+Shoes                                             	importoedu pack #x                                	North Midwest	1999	4	6588.741667	3212.76	3187.25	3974.78
+Shoes                                             	edu packedu pack #x                               	NY Metro	1999	6	6203.331667	2835.78	4186.85	3192.53
+Men                                               	exportiimporto #x                                 	North Midwest	1999	2	7343.719167	3984.13	3652.72	4033.37
+Men                                               	amalgimporto #x                                   	NY Metro	1999	4	6720.550000	3364.32	3810.74	4333.58
+Children                                          	edu packexporti #x                                	North Midwest	1999	4	6739.498333	3389.03	3623.32	3605.25
+Shoes                                             	importoedu pack #x                                	Mid Atlantic	1999	6	6864.320833	3531.93	3449.62	4414.17
+Shoes                                             	amalgedu pack #x                                  	Mid Atlantic	1999	6	6674.896667	3342.98	4458.26	3178.12
+Children                                          	edu packexporti #x                                	Mid Atlantic	1999	2	6511.800833	3185.28	3581.75	3410.75
+Children                                          	amalgexporti #x                                   	Mid Atlantic	1999	4	6854.405833	3541.62	3854.33	3938.42
+Men                                               	exportiimporto #x                                 	North Midwest	1999	3	7343.719167	4033.37	3984.13	3729.62
+Men                                               	amalgimporto #x                                   	Mid Atlantic	1999	3	6618.534167	3313.62	4044.36	4291.66
+Shoes                                             	exportiedu pack #x                                	Mid Atlantic	1999	7	7416.141667	4121.24	4239.08	8658.42
+Women                                             	importoamalg #x                                   	Mid Atlantic	1999	6	6395.326667	3102.55	4234.22	3650.03
+Children                                          	importoexporti #x                                 	North Midwest	1999	3	6577.143333	3291.07	3773.61	2152.15
+Women                                             	edu packamalg #x                                  	NY Metro	1999	4	6027.880000	2750.96	3199.35	2575.81
+Music                                             	amalgscholar #x                                   	Mid Atlantic	1999	3	6662.669167	3386.25	1961.57	4799.18
+Men                                               	amalgimporto #x                                   	North Midwest	1999	6	7098.680833	3834.90	4923.53	4115.57
+Shoes                                             	importoedu pack #x                                	Mid Atlantic	1999	3	6864.320833	3606.28	3104.48	3869.15
+Music                                             	exportischolar #x                                 	NY Metro	1999	6	7043.051667	3793.48	4135.21	5006.69
+Shoes                                             	edu packedu pack #x                               	Mid Atlantic	1999	1	6711.753333	3473.10	15060.83	4085.86
+Men                                               	exportiimporto #x                                 	Mid Atlantic	1999	1	7419.459167	4188.88	16358.86	4366.77
+Women                                             	amalgamalg #x                                     	NY Metro	1999	2	6362.709167	3137.03	4180.91	3181.26
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q58.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q58.sql.out
new file mode 100644
index 0000000000000..7e282de310b78
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q58.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<item_id:string,ss_item_rev:decimal(17,2),ss_dev:decimal(38,17),cs_item_rev:decimal(17,2),cs_dev:decimal(38,17),ws_item_rev:decimal(17,2),ws_dev:decimal(38,17),average:decimal(23,6)>
+-- !query output
+AAAAAAAAOEHBAAAA	4202.05	11.64402731132023188	3952.65	10.95293120074485419	3874.50	10.73637482126824727	4009.733333
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q59.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q59.sql.out
new file mode 100644
index 0000000000000..020860f8baefd
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q59.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<s_store_name1:string,s_store_id1:string,d_week_seq1:int,(sun_sales1 / sun_sales2):decimal(37,20),(mon_sales1 / mon_sales2):decimal(37,20),(tue_sales1 / tue_sales2):decimal(37,20),(wed_sales1 / wed_sales2):decimal(37,20),(thu_sales1 / thu_sales2):decimal(37,20),(fri_sales1 / fri_sales2):decimal(37,20),(sat_sales1 / sat_sales2):decimal(37,20)>
+-- !query output
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5271	0.74419775623719778195	0.74258389025810779226	0.18455097389041153754	0.11049123944133143765	2.01369701972914228390	0.74322255494522317957	0.71433787179231334416
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5272	0.84505990801516636507	1.64762915953700921988	0.38486496991736952802	0.68701158883293751293	1.56192545028377515216	1.16497445041784212487	0.46921185549145987093
+able	AAAAAAAACAAAAAAA	5273	0.35130200745800939483	0.85485871077793239664	0.66980552712384851586	0.66252919917140464542	0.75558600483388711996	0.52111936828952912290	0.63215140490620731712
+able	AAAAAAAACAAAAAAA	5273	0.35130200745800939483	0.85485871077793239664	0.66980552712384851586	0.66252919917140464542	0.75558600483388711996	0.52111936828952912290	0.63215140490620731712
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q60.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q60.sql.out
new file mode 100644
index 0000000000000..632d983192f6f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q60.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_item_id:string,total_sales:decimal(27,2)>
+-- !query output
+AAAAAAAAAABAAAAA	6334.66
+AAAAAAAAAACCAAAA	5389.16
+AAAAAAAAAADBAAAA	11655.98
+AAAAAAAAAAHAAAAA	5388.00
+AAAAAAAAAAHDAAAA	12685.87
+AAAAAAAAAAIAAAAA	8916.30
+AAAAAAAAAAKBAAAA	2140.04
+AAAAAAAAAAMBAAAA	22048.17
+AAAAAAAAAAPBAAAA	13195.87
+AAAAAAAAABAEAAAA	1071.67
+AAAAAAAAABBCAAAA	7064.02
+AAAAAAAAABEAAAAA	13239.06
+AAAAAAAAABFBAAAA	1877.72
+AAAAAAAAABFEAAAA	699.95
+AAAAAAAAABGAAAAA	8169.00
+AAAAAAAAABJAAAAA	10752.87
+AAAAAAAAABLBAAAA	5962.43
+AAAAAAAAABLCAAAA	18131.46
+AAAAAAAAABMBAAAA	12049.00
+AAAAAAAAABMDAAAA	18947.93
+AAAAAAAAABNDAAAA	8479.29
+AAAAAAAAACDCAAAA	13466.87
+AAAAAAAAACEEAAAA	7634.65
+AAAAAAAAACJCAAAA	51774.63
+AAAAAAAAACKBAAAA	18766.24
+AAAAAAAAACLBAAAA	16002.89
+AAAAAAAAACLDAAAA	12302.68
+AAAAAAAAACOBAAAA	2707.13
+AAAAAAAAADABAAAA	6234.38
+AAAAAAAAADBDAAAA	27724.30
+AAAAAAAAADEAAAAA	2867.87
+AAAAAAAAADFAAAAA	10667.94
+AAAAAAAAADFDAAAA	9051.70
+AAAAAAAAADGBAAAA	6502.68
+AAAAAAAAADGCAAAA	1174.33
+AAAAAAAAADHDAAAA	18882.96
+AAAAAAAAADNBAAAA	11526.41
+AAAAAAAAADPBAAAA	1026.20
+AAAAAAAAAEADAAAA	3582.21
+AAAAAAAAAECBAAAA	12360.25
+AAAAAAAAAECCAAAA	20954.85
+AAAAAAAAAEDAAAAA	8537.71
+AAAAAAAAAEECAAAA	10513.50
+AAAAAAAAAEEDAAAA	2561.17
+AAAAAAAAAEGAAAAA	7836.96
+AAAAAAAAAEGDAAAA	638.36
+AAAAAAAAAEJBAAAA	11429.14
+AAAAAAAAAELBAAAA	6543.86
+AAAAAAAAAEMDAAAA	13797.32
+AAAAAAAAAEOBAAAA	6699.66
+AAAAAAAAAFAAAAAA	1979.04
+AAAAAAAAAFCBAAAA	6157.90
+AAAAAAAAAFDAAAAA	14494.27
+AAAAAAAAAFEBAAAA	11552.54
+AAAAAAAAAFGAAAAA	13218.28
+AAAAAAAAAFGCAAAA	10293.39
+AAAAAAAAAFGDAAAA	6078.36
+AAAAAAAAAFJDAAAA	21440.62
+AAAAAAAAAFLBAAAA	11714.63
+AAAAAAAAAFNCAAAA	17910.15
+AAAAAAAAAGACAAAA	12212.53
+AAAAAAAAAGAEAAAA	17187.86
+AAAAAAAAAGBAAAAA	2753.50
+AAAAAAAAAGDBAAAA	5244.60
+AAAAAAAAAGDEAAAA	466.32
+AAAAAAAAAGEAAAAA	2030.78
+AAAAAAAAAGEBAAAA	673.70
+AAAAAAAAAGEDAAAA	4861.80
+AAAAAAAAAGIAAAAA	20717.88
+AAAAAAAAAGKAAAAA	4481.54
+AAAAAAAAAGLAAAAA	259.22
+AAAAAAAAAGNDAAAA	9371.85
+AAAAAAAAAGOAAAAA	12624.61
+AAAAAAAAAGODAAAA	15071.99
+AAAAAAAAAHBDAAAA	6674.73
+AAAAAAAAAHCCAAAA	4327.67
+AAAAAAAAAHECAAAA	13309.93
+AAAAAAAAAHGAAAAA	8277.89
+AAAAAAAAAHIBAAAA	6574.61
+AAAAAAAAAHICAAAA	4810.18
+AAAAAAAAAHJDAAAA	9151.90
+AAAAAAAAAHKAAAAA	424.72
+AAAAAAAAAHKCAAAA	23702.30
+AAAAAAAAAHKDAAAA	8747.48
+AAAAAAAAAHMBAAAA	17707.03
+AAAAAAAAAHMDAAAA	5608.89
+AAAAAAAAAHNAAAAA	9929.30
+AAAAAAAAAHOCAAAA	21257.55
+AAAAAAAAAIAAAAAA	36659.16
+AAAAAAAAAICEAAAA	23564.63
+AAAAAAAAAIDCAAAA	7648.02
+AAAAAAAAAIEBAAAA	6151.13
+AAAAAAAAAIEEAAAA	1406.98
+AAAAAAAAAIFBAAAA	2088.56
+AAAAAAAAAIGAAAAA	2468.08
+AAAAAAAAAIGCAAAA	1834.76
+AAAAAAAAAIGDAAAA	558.00
+AAAAAAAAAIIBAAAA	5422.03
+AAAAAAAAAILDAAAA	4833.43
+AAAAAAAAAIOAAAAA	8317.81
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q61.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q61.sql.out
new file mode 100644
index 0000000000000..0a8af60ec1265
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q61.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<promotions:decimal(17,2),total:decimal(17,2),((CAST(promotions AS DECIMAL(15,4)) / CAST(total AS DECIMAL(15,4))) * 100):decimal(38,19)>
+-- !query output
+2574995.94	5792384.50	44.4548517108972997220
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q62.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q62.sql.out
new file mode 100644
index 0000000000000..879eb4c6355ac
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q62.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<substr(w_warehouse_name, 1, 20):string,sm_type:string,web_name:string,30 days :bigint,31 - 60 days :bigint,61 - 90 days :bigint,91 - 120 days :bigint,>120 days :bigint>
+-- !query output
+Just good amou	EXPRESS                       	site_0	270	248	280	287	0
+Just good amou	EXPRESS                       	site_1	290	327	288	262	0
+Just good amou	EXPRESS                       	site_2	314	295	311	299	0
+Just good amou	EXPRESS                       	site_3	309	300	299	267	0
+Just good amou	EXPRESS                       	site_4	224	250	297	273	0
+Just good amou	LIBRARY                       	site_0	217	192	210	225	0
+Just good amou	LIBRARY                       	site_1	163	207	172	205	0
+Just good amou	LIBRARY                       	site_2	176	204	208	190	0
+Just good amou	LIBRARY                       	site_3	222	220	240	240	0
+Just good amou	LIBRARY                       	site_4	254	230	241	263	0
+Just good amou	NEXT DAY                      	site_0	311	275	303	291	0
+Just good amou	NEXT DAY                      	site_1	291	307	284	289	0
+Just good amou	NEXT DAY                      	site_2	275	299	323	279	0
+Just good amou	NEXT DAY                      	site_3	260	290	242	248	0
+Just good amou	NEXT DAY                      	site_4	233	285	271	281	0
+Just good amou	OVERNIGHT                     	site_0	242	262	221	248	0
+Just good amou	OVERNIGHT                     	site_1	251	245	249	233	0
+Just good amou	OVERNIGHT                     	site_2	198	226	209	204	0
+Just good amou	OVERNIGHT                     	site_3	168	199	184	175	0
+Just good amou	OVERNIGHT                     	site_4	237	231	193	207	0
+Just good amou	REGULAR                       	site_0	140	236	154	190	0
+Just good amou	REGULAR                       	site_1	189	190	205	194	0
+Just good amou	REGULAR                       	site_2	234	251	264	235	0
+Just good amou	REGULAR                       	site_3	229	251	244	243	0
+Just good amou	REGULAR                       	site_4	198	209	188	216	0
+Just good amou	TWO DAY                       	site_0	256	258	218	235	0
+Just good amou	TWO DAY                       	site_1	217	226	195	198	0
+Just good amou	TWO DAY                       	site_2	176	151	186	202	0
+Just good amou	TWO DAY                       	site_3	221	225	236	190	0
+Just good amou	TWO DAY                       	site_4	239	240	219	252	0
+Matches produce	EXPRESS                       	site_0	330	277	239	314	0
+Matches produce	EXPRESS                       	site_1	296	336	288	307	0
+Matches produce	EXPRESS                       	site_2	262	307	273	305	0
+Matches produce	EXPRESS                       	site_3	284	253	253	256	0
+Matches produce	EXPRESS                       	site_4	260	309	252	296	0
+Matches produce	LIBRARY                       	site_0	188	181	179	170	0
+Matches produce	LIBRARY                       	site_1	189	211	193	181	0
+Matches produce	LIBRARY                       	site_2	253	230	207	248	0
+Matches produce	LIBRARY                       	site_3	225	230	217	214	0
+Matches produce	LIBRARY                       	site_4	220	218	208	213	0
+Matches produce	NEXT DAY                      	site_0	296	286	328	316	0
+Matches produce	NEXT DAY                      	site_1	303	303	307	281	0
+Matches produce	NEXT DAY                      	site_2	271	258	253	256	0
+Matches produce	NEXT DAY                      	site_3	265	263	290	296	0
+Matches produce	NEXT DAY                      	site_4	312	312	291	311	0
+Matches produce	OVERNIGHT                     	site_0	217	254	216	258	0
+Matches produce	OVERNIGHT                     	site_1	214	187	229	230	0
+Matches produce	OVERNIGHT                     	site_2	171	176	188	174	0
+Matches produce	OVERNIGHT                     	site_3	181	218	193	219	0
+Matches produce	OVERNIGHT                     	site_4	244	216	237	234	0
+Matches produce	REGULAR                       	site_0	205	216	204	229	0
+Matches produce	REGULAR                       	site_1	230	232	229	244	0
+Matches produce	REGULAR                       	site_2	225	250	209	254	0
+Matches produce	REGULAR                       	site_3	230	212	212	195	0
+Matches produce	REGULAR                       	site_4	207	167	180	172	0
+Matches produce	TWO DAY                       	site_0	228	195	222	198	0
+Matches produce	TWO DAY                       	site_1	169	154	170	146	0
+Matches produce	TWO DAY                       	site_2	197	215	201	210	0
+Matches produce	TWO DAY                       	site_3	247	236	234	237	0
+Matches produce	TWO DAY                       	site_4	242	259	245	247	0
+Operations	EXPRESS                       	site_0	270	284	268	286	0
+Operations	EXPRESS                       	site_1	274	251	245	275	0
+Operations	EXPRESS                       	site_2	287	337	296	310	0
+Operations	EXPRESS                       	site_3	349	307	322	286	0
+Operations	EXPRESS                       	site_4	284	287	285	271	0
+Operations	LIBRARY                       	site_0	266	227	229	237	0
+Operations	LIBRARY                       	site_1	201	228	190	200	0
+Operations	LIBRARY                       	site_2	177	170	182	171	0
+Operations	LIBRARY                       	site_3	227	217	214	226	0
+Operations	LIBRARY                       	site_4	268	258	242	242	0
+Operations	NEXT DAY                      	site_0	269	255	273	257	0
+Operations	NEXT DAY                      	site_1	292	295	258	291	0
+Operations	NEXT DAY                      	site_2	296	327	305	294	0
+Operations	NEXT DAY                      	site_3	274	286	322	289	0
+Operations	NEXT DAY                      	site_4	311	270	243	255	0
+Operations	OVERNIGHT                     	site_0	217	205	193	227	0
+Operations	OVERNIGHT                     	site_1	221	243	232	237	0
+Operations	OVERNIGHT                     	site_2	247	243	235	238	0
+Operations	OVERNIGHT                     	site_3	242	218	205	223	0
+Operations	OVERNIGHT                     	site_4	181	172	192	186	0
+Operations	REGULAR                       	site_0	190	211	206	195	0
+Operations	REGULAR                       	site_1	166	173	173	174	0
+Operations	REGULAR                       	site_2	198	219	215	226	0
+Operations	REGULAR                       	site_3	255	234	239	215	0
+Operations	REGULAR                       	site_4	246	231	266	208	0
+Operations	TWO DAY                       	site_0	247	232	246	240	0
+Operations	TWO DAY                       	site_1	244	240	259	234	0
+Operations	TWO DAY                       	site_2	210	214	200	190	0
+Operations	TWO DAY                       	site_3	166	181	174	184	0
+Operations	TWO DAY                       	site_4	194	197	224	211	0
+Selective,	EXPRESS                       	site_0	290	275	273	332	0
+Selective,	EXPRESS                       	site_1	303	319	311	299	0
+Selective,	EXPRESS                       	site_2	299	238	268	261	0
+Selective,	EXPRESS                       	site_3	262	273	281	300	0
+Selective,	EXPRESS                       	site_4	285	298	291	300	0
+Selective,	LIBRARY                       	site_0	215	204	202	211	0
+Selective,	LIBRARY                       	site_1	221	253	243	240	0
+Selective,	LIBRARY                       	site_2	218	261	226	234	0
+Selective,	LIBRARY                       	site_3	178	189	233	217	0
+Selective,	LIBRARY                       	site_4	159	186	194	199	0
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q63.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q63.sql.out
new file mode 100644
index 0000000000000..4c5123f2cc90b
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q63.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_manager_id:int,sum_sales:decimal(17,2),avg_monthly_sales:decimal(21,6)>
+-- !query output
+1	175.21	726.555833
+1	231.64	726.555833
+1	400.27	726.555833
+1	409.69	726.555833
+1	468.02	726.555833
+1	530.63	726.555833
+1	880.08	726.555833
+1	972.96	726.555833
+1	1028.83	726.555833
+1	1038.59	726.555833
+1	1820.16	726.555833
+2	392.91	1124.957500
+2	469.26	1124.957500
+2	654.97	1124.957500
+2	703.18	1124.957500
+2	726.27	1124.957500
+2	813.75	1124.957500
+2	819.59	1124.957500
+2	1248.04	1124.957500
+2	1253.65	1124.957500
+2	1690.71	1124.957500
+2	1961.13	1124.957500
+2	2766.03	1124.957500
+3	259.59	1110.663333
+3	530.95	1110.663333
+3	639.03	1110.663333
+3	727.70	1110.663333
+3	822.22	1110.663333
+3	853.58	1110.663333
+3	910.78	1110.663333
+3	1455.11	1110.663333
+3	1615.68	1110.663333
+3	1741.42	1110.663333
+3	2618.11	1110.663333
+4	465.88	1333.585833
+4	472.55	1333.585833
+4	685.93	1333.585833
+4	703.90	1333.585833
+4	773.55	1333.585833
+4	1535.08	1333.585833
+4	1900.09	1333.585833
+4	2557.73	1333.585833
+4	2842.07	1333.585833
+5	507.40	1378.418333
+5	604.37	1378.418333
+5	633.93	1378.418333
+5	830.17	1378.418333
+5	1013.51	1378.418333
+5	1093.85	1378.418333
+5	1223.10	1378.418333
+5	1735.77	1378.418333
+5	2024.74	1378.418333
+5	2065.17	1378.418333
+5	2319.65	1378.418333
+5	2489.36	1378.418333
+6	852.04	1879.384167
+6	975.86	1879.384167
+6	1082.92	1879.384167
+6	1121.29	1879.384167
+6	1133.12	1879.384167
+6	1558.44	1879.384167
+6	1581.26	1879.384167
+6	1654.62	1879.384167
+6	2408.41	1879.384167
+6	3334.37	1879.384167
+6	4855.76	1879.384167
+7	655.33	1433.785833
+7	691.77	1433.785833
+7	750.55	1433.785833
+7	862.94	1433.785833
+7	868.32	1433.785833
+7	992.39	1433.785833
+7	1209.78	1433.785833
+7	1694.72	1433.785833
+7	2081.47	1433.785833
+7	2257.98	1433.785833
+7	2383.35	1433.785833
+7	2756.83	1433.785833
+8	54.86	410.067500
+8	183.63	410.067500
+8	192.44	410.067500
+8	271.04	410.067500
+8	472.18	410.067500
+8	492.97	410.067500
+8	644.56	410.067500
+8	669.81	410.067500
+8	676.94	410.067500
+9	479.27	1306.207500
+9	572.00	1306.207500
+9	742.40	1306.207500
+9	759.92	1306.207500
+9	805.90	1306.207500
+9	993.12	1306.207500
+9	1030.85	1306.207500
+9	1764.83	1306.207500
+9	1826.50	1306.207500
+9	2065.07	1306.207500
+9	2158.76	1306.207500
+9	2475.87	1306.207500
+10	475.18	1442.284167
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q64.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q64.sql.out
new file mode 100644
index 0000000000000..a803af3a52c21
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q64.sql.out
@@ -0,0 +1,14 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<product_name:string,store_name:string,store_zip:string,b_street_number:string,b_streen_name:string,b_city:string,b_zip:string,c_street_number:string,c_street_name:string,c_city:string,c_zip:string,syear:int,cnt:bigint,s1:decimal(17,2),s2:decimal(17,2),s3:decimal(17,2),s1:decimal(17,2),s2:decimal(17,2),s3:decimal(17,2),syear:int,cnt:bigint>
+-- !query output
+ablepricallyantiought                             	ation	35709     	996       	NULL	Bridgeport	65817     	752       	Lakeview Lincoln	Friendship	74536     	1999	1	15.78	24.93	0.00	17.09	26.31	0.00	2000	1
+ablepricallyantiought                             	ation	35709     	71        	River River	Friendship	34536     	NULL	NULL	Newport	NULL	1999	1	22.60	38.87	0.00	17.09	26.31	0.00	2000	1
+ablepricallyantiought                             	bar	31904     	128       	East 	Franklin	19101     	990       	2nd 	Friendship	94536     	1999	1	54.76	78.30	0.00	15.80	23.54	0.00	2000	1
+ationbarpri                                       	ation	35709     	362       	Central Ridge	Oakland	69843     	666       	13th Ridge	Shiloh	29275     	1999	1	74.00	95.46	0.00	11.32	20.94	0.00	2000	1
+ationbarpri                                       	ese	31904     	759       	Elm Pine	Belmont	20191     	35        	Madison 	Waterloo	31675     	1999	1	12.92	22.22	0.00	24.15	36.70	0.00	2000	1
+ationbarpri                                       	ese	31904     	759       	Elm Pine	Belmont	20191     	35        	Madison 	Waterloo	31675     	1999	1	12.92	22.22	0.00	83.87	147.61	0.00	2000	1
+ationbarpri                                       	ought	31904     	754       	NULL	NULL	65804     	897       	8th 	Ashland	54244     	1999	1	74.70	90.38	0.00	12.02	12.74	0.00	2000	1
+ationbarpri                                       	ought	31904     	754       	NULL	NULL	65804     	897       	8th 	Ashland	54244     	1999	1	74.70	90.38	0.00	28.08	38.18	0.00	2000	1
+ationbarpri                                       	ought	31904     	754       	NULL	NULL	65804     	897       	8th 	Ashland	54244     	1999	1	74.70	90.38	0.00	56.60	63.39	0.00	2000	1
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q65.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q65.sql.out
new file mode 100644
index 0000000000000..16a7d8785e484
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q65.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<s_store_name:string,i_item_desc:string,revenue:decimal(17,2),i_current_price:decimal(7,2),i_wholesale_cost:decimal(7,2),i_brand:string>
+-- !query output
+able	Academic concerns help during a courses. Hard, working-class types concentrate in a costs. Commercial shares work directly additional, ordinary orders. Profi	29.50	3.57	2.92	amalgamalg #x                                     
+able	Available ministers s	20.59	7.69	3.99	exportiimporto #x                                 
+able	Before main women cover so rive	35.39	4.63	2.36	importoamalg #x                                   
+able	Days will lift for a months. Public differences could like	30.44	1.78	0.99	importoexporti #x                                 
+able	Fields expire perhaps available schools. Common machines allow later employees; lawyers will pull worldwide objectives. Hidden orders may turn well fo	19.20	88.16	54.65	importoimporto #x                                 
+able	General, sure features drop for the time being integrated proposals. Occasionally eastern generations might not arrive today ever vast opportunities. Soft attacks used to sell 	31.34	1.16	0.90	exportiexporti #x                                 
+able	Lost, individual parents rally. Then various differences shall fill never private strong courts. Successfully global directors clean medical	2.94	8.89	5.95	edu packimporto #x                                
+able	Members say then authorities. Various wome	32.82	1.34	1.17	amalgamalgamalg #x                                
+able	National, round fields would not accomp	32.77	0.17	0.05	amalgnameless #x                                 
+able	New measures shall pay under a agencies; comparatively heavy police shall beat similarly concepts. However japanese times cannot check like a police. Long, long-term auth	21.49	1.87	1.19	amalgnameless #x                                  
+able	Only concerned times used to know however in the trees. Developers might not wear in the times. Studies see far variations. Calculations must not transport hardl	32.50	0.15	0.12	corpmaxi #x                                       
+able	Only, foreign problems make in a women. Naturally very countries will beli	13.72	7.58	5.15	scholarunivamalg #x                              
+able	Perfect grants fight highly as great minutes. Severe, available millions like counties. Young legs cook however from a years. Early armed services reject yet with 	23.87	4.31	2.84	scholarnameless #x                                
+able	Popular, ethical types must see human, central steps. Other options spare products; large, odd machines must fill clear, public users. Away local teachers make l	25.26	8.08	5.73	amalgexporti #x                                   
+able	Privileges cut perhaps reasons. Ideas finish times. Women envy general programmes. Hands shall unveil never to a facilities; official proposals conform. Scot	18.45	7.52	4.73	univmaxi #x                                       
+able	Probably suitable compan	21.47	2.39	1.29	importobrand #x                                   
+able	Royal speeches take evil, front margins. For example hard events ought to go angles. Possible, foreign lakes shall not reconsider. Other honours hear momen	7.23	8.13	2.84	scholarnameless #x                                
+able	Rules offer. Important, italian goo	9.19	4.06	2.51	scholarmaxi #x                                   
+able	Ships could not introduce as. Complete equations take different, european names. Respondents would help fine styles; really formal workers 	32.96	2.55	2.04	edu packunivamalg #x                             
+able	Stores visit values. Others cannot hang around rather civil brothers. Direct systems go then free, other instructions. Difficult, top feet will al	33.93	13.91	6.39	exportinameless #x                                
+able	Systematic 	12.54	2.98	2.17	amalgbrand #x                                     
+able	Times concentrate religious forms. Soon social agents understand again on a officials. Miles w	25.36	5.16	2.58	importoexporti #x                                 
+ation	Areas establish in 	28.85	93.54	69.21	amalgunivamalg #x                                
+ation	Arrangements	26.15	0.27	0.20	edu packunivamalg #x                             
+ation	Boring areas used to print; companies delegate lines. Clients shall amount also then	18.54	3.84	1.84	importoexporti #x                                 
+ation	Deliberately ordinary procedures will not pay by a months. Feet reach very s	34.44	9.43	5.46	univunivamalg #x                                  
+ation	Ears might remind far charges. Pleased years discharge oth	24.58	2.73	1.22	edu packbrand #x                                  
+ation	However old figures ask only good, large sources. Yet naked researchers shall deal to a women. Right, common miles describe there also prime bags. Readily significant shares	21.02	7.78	4.97	brandbrand #x                                     
+ation	Just possible women say. Reasonably strong employers light almost degrees. Palestinian, smart rules help really visual	33.67	3.71	2.26	scholarbrand #x                                  
+ation	Local years may not seek formally hard, interesting properties. Local values serve. Nevertheless private tales c	28.58	8.36	5.68	corpunivamalg #x                                  
+ation	Much present elements regard previously glorious homes. Important, royal judges can ad	12.98	8.12	6.41	edu packscholar #x                                
+ation	Police succeed schools; supplies calculate far countries; new words move shares; officers must complete years. Asian things may bear warm things. Aw	29.31	6.66	2.26	univnameless #x                                   
+ation	Respondents see also. Sure, american horses must go actually election	34.94	0.85	0.50	exportiamalg #x                                   
+ation	Small, japanese rights will not think enough on the cars. Very fond hospitals may choose originally. Right, other businesses relish. As large decades include federal tears. Usual, important quar	21.23	3.63	1.81	amalgscholar #x                                   
+ation	Technical, open seats used to become accordingly. Real, actual qualifications may not carry highly interesting others. Wide, sexual knees may stay expenses. Labour, american	30.78	0.61	0.51	exportiedu pack #x                                
+ation	Ultimately good sets could go short, early examinations. Things ought to know relatively. Linguistic, applicable children establish curiou	35.23	6.22	3.54	importoimporto #x                                 
+ation	Unacceptable flowers should not give reasonable, ethnic governments. Employees shall complain 	35.82	8.39	5.87	corpmaxi #x                                       
+bar	Also coming years need with a faces; fresh poems judge for example already thick choices. Hands help individual, relevant samples. Together british fingers would not spe	23.68	2.15	0.98	edu packexporti #x                                
+bar	Bad, recent right	31.34	0.96	0.57	importoimporto #x                                 
+bar	Cards should strike largely in a concessions. Still true signs might talk; essentially ro	29.21	55.23	48.05	importoamalg #x                                   
+bar	Cases must not spend on	14.60	1.30	1.17	importoimporto #x                                 
+bar	Cold, old days stem thereby difficult, nuclear men; likely contents shall threaten often outer years. All real or	3.53	9.08	3.35	edu packnameless #x                               
+bar	Conscious, solar ambitions support outside countries; warm facilities rise occupations. Appropriate columns grow. Availabl	26.55	3.35	2.84	importonameless #x                                
+bar	General, legal businesses should use completely expensive teachers. Linguistic friends vote problem	0.21	0.53	0.32	scholarbrand #x                                   
+bar	High, official employees shall not start too left circumstances. Patients used to touch obviously popular, senior members. British, impossible theories make only. Young, international wo	28.68	4.85	3.63	edu packmaxi #x                                   
+bar	However old hours ma	22.18	8.84	5.65	exportimaxi #x                                    
+bar	Illegal technologies might distinguish that on a change	12.14	2.73	1.28	namelessmaxi #x                                  
+bar	Never various elements play other rights	30.37	0.37	0.28	edu packscholar #x                                
+bar	Newspapers would ensure certainly short inadequate problems. Bedrooms would argue however halfway bad coun	29.16	2.78	1.69	scholarcorp #x                                    
+bar	Only groups will not prove united, furious results. Papers shall think in a opportunities; techniques decide that. American, initial forces might mean previous books. That delighted arts give so dimen	28.98	2.78	1.64	exportiimporto #x                                 
+bar	Open prod	11.05	2.74	1.53	edu packunivamalg #x                              
+bar	Presidential, mild tests justify yesterday unusual points. Notable individuals can set only external trousers. Here french letters may photogra	27.05	2.44	2.17	importoimporto #x                                 
+bar	Prices acquire more out of a christians. Efficiently local prices 	20.48	2.11	0.78	exportimaxi #x                                    
+bar	Rare, full workers stay plant	8.61	0.55	0.34	exportiimporto #x                                 
+bar	Social pieces become; reservations rescue probably old hopes. Different, high records buy just general centuries. Recently industrial relationships cannot	16.63	9.15	5.03	importoedu pack #x                                
+bar	Though private depths accomplis	12.68	0.58	0.23	amalgamalgamalg #x                               
+bar	Thus certain stars appear totally even local guests. Urban friends might not take properly various vehicles	35.61	4.55	2.09	amalgunivamalg #x                                 
+bar	Traditional legs pull at least better difficult circumstances; other, inner clients step burning arms; able, numerous weapons keep li	3.50	45.72	20.11	amalgscholar #x                                   
+bar	Unknown topics ought to answer far present pictures. Estimated considerations might meet 	32.53	2.55	1.32	exportiamalg #x                                   
+bar	Widely splendid others deprive only. Different, main soldiers discover then other periods. Too main birds must change public, terrible houses. Different, armed females may foster; science	22.86	4.26	3.36	amalgunivamalg #x                                 
+eing	Formerly central designers must not save. Scottish, small horses elicit men. British, fine companies pay little taxes; here pure lakes should benefit however small top countr	33.81	0.51	0.31	exportiamalg #x                                   
+eing	Forms shall involve then normal bodies. Left words may find	11.97	2.02	0.64	edu packexporti #x                                
+eing	Front elections ensure to a adults; valuable moments decide in a aspects. Marked books stand rooms. Expectations ought	37.36	59.51	43.44	amalgimporto #x                                   
+eing	Gradual volunteers keep bc months. Calls get pleasantly questions. Repre	12.82	1.61	1.07	edu packexporti #x                                
+eing	Gradually new flowers support suddenly. Left, light errors ought to steal other memories. Periods should not say never to a nurses;	31.57	2.05	1.12	edu packamalg #x                                  
+eing	Great, possible children used to	11.87	4.00	2.60	namelessnameless #x                               
+eing	Neat, desirable words make especially gradu	25.33	7.11	3.76	amalgnameless #x                                  
+eing	Never top observations spend appropriate, common states. Homes make. There available hospitals will appreciate away upon a years. Roots hang 	31.15	2.07	0.84	edu packnameless #x                               
+eing	Often	21.89	7.85	4.16	amalgscholar #x                                   
+eing	Poor years produce questions. Marine events ensure inner systems. Individuals could kill to a managers. Drugs should not authorise thankfully traditional, strong holders. Just amazing injuri	19.66	4.11	3.16	importoimporto #x                                 
+eing	Sensibly foreign parties must not suffer well indian personal students. About private cattle handle th	4.79	2.42	1.01	amalgscholar #x                                   
+eing	Silver, political interviews might know in common families. Far possible houses shall insist in a places. Whole, political gardens would adopt eggs. Others might live even offi	33.77	6.13	2.88	namelessmaxi #x                                   
+eing	Social areas undergo actually within a plants. More communist days would play on a books. Later educational policies shoul	30.97	5.52	2.92	exportiunivamalg #x                               
+eing	Soon artificial notions think no longer lights; clearly late members could not trace good countries. Cultures can proceed away wealthy 	0.98	2.38	1.68	exportibrand #x                                   
+ese	Additional, interior police provide words. Different, long qualities answer really concerns; then other words state dry, political services. Awfully di	15.35	9.78	7.53	corpbrand #x                                      
+ese	Concer	32.51	9.66	8.11	amalgamalg #x                                     
+ese	Constitutional, good pupils might not begin below level devices. External savings fit hardly. Parents shall dry. Actually literary companies improve a	17.22	4.22	2.40	scholarunivamalg #x                               
+ese	Earnings feel possibilities. Single, poor problems make full, sho	29.18	2.75	1.62	maximaxi #x                                       
+ese	Financial, likely artists assume now c	5.10	5.63	2.42	importoamalgamalg #x                             
+ese	Forces enter efficient things. S	23.42	2.64	0.81	amalgamalgamalg #x                               
+ese	Forward new lights should not meet once yesterday national buildings. Natural, australian eyes may not fetch progressively unfair	29.40	7.87	5.66	amalgimporto #x                                   
+ese	Fresh respondents would not encourage in a years.	26.81	3.90	1.67	importoimporto #x                                 
+ese	Goals ought to strengthen. Early industries would take. Early men could hear then. Difficult, new machines endorse 	9.97	1.24	0.57	importoamalg #x                                   
+ese	Identical solicitors must maintain sources. Factors take already unusual minutes. Just various sales sell agricultural, long states. 	27.99	3.77	1.16	maxinameless #x                                   
+ese	In order turkish meanings should involve nevertheless to a inches. Common, free colleagues may know other, safe services. So sure plates might go hidden, formidable powers. Domes	33.33	1.12	0.35	namelesscorp #x                                   
+ese	Just associated missiles could	20.43	1.23	0.46	edu packedu pack #x                               
+ese	Meanwhile single companies shall go either in a statements. Vo	9.56	7.96	3.42	corpbrand #x                                      
+ese	Models feel figures. Homes get still at the positions. Political, other makers will make there in the servants; necessary, technical markets should not cope also; warm, 	32.66	6.16	4.49	univbrand #x                                      
+ese	New interests must turn largely. High, essential females mark. Gradual police would not exercise old, national	23.00	6.87	2.33	edu packcorp #x                                   
+ese	Parliamentary pieces shine never tragic patterns. Great human eyes would not get groups. Plant	9.58	6.03	4.82	scholarmaxi #x                                    
+ese	Photographs get everywhere merely 	13.94	3.75	2.96	exportiamalg #x                                   
+ese	Plates shall think; new, economic pupils collect entirely. Really powerful books develop yet girls. Best unlik	33.09	3.44	2.99	namelessbrand #x                                  
+ese	Proposals should involve more soviet, young islands. Little resources try even books. Fundamental systems end recent, total provisions. Working-class matte	30.46	5.15	3.60	exportischolar #x                                 
+ese	Public operations need wonderfully improved routes; days may not admit for a circles. Able, wise girls lay later. Authorities know really reasons. Scottish accountants take customs. 	2.20	1.40	0.99	amalgimporto #x                                   
+ese	Sharply bright systems used to want. Other projects should benefit. Common parts use	35.01	1.16	0.39	importoexporti #x                                 
+ese	Short, known programmes reject quite documents. Really interna	7.78	36.94	22.90	univamalgamalg #x                                 
+ese	Small years could spend soon	31.85	0.55	0.39	importoscholar #x                                 
+ese	Special gates decide mutually. Current, appropriate terms feel thus better royal arms. Children starve; likely girls make yesterday local workers. 	26.14	2.93	1.11	exportischolar #x                                 
+ese	There round authorities will show in a seasons. Other, far firms miss very mad, private lips. Powerful, m	16.77	5.93	4.44	importoamalg #x                                   
+ese	Women get also chairs. Full, integrated paintings sit 	1.47	6.34	2.15	scholarnameless #x
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q66.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q66.sql.out
new file mode 100644
index 0000000000000..3287cb908761f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q66.sql.out
@@ -0,0 +1,10 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<w_warehouse_name:string,w_warehouse_sq_ft:int,w_city:string,w_county:string,w_state:string,w_country:string,ship_carriers:string,year:int,jan_sales:decimal(38,2),feb_sales:decimal(38,2),mar_sales:decimal(38,2),apr_sales:decimal(38,2),may_sales:decimal(38,2),jun_sales:decimal(38,2),jul_sales:decimal(38,2),aug_sales:decimal(38,2),sep_sales:decimal(38,2),oct_sales:decimal(38,2),nov_sales:decimal(38,2),dec_sales:decimal(38,2),jan_sales_per_sq_foot:decimal(38,12),feb_sales_per_sq_foot:decimal(38,12),mar_sales_per_sq_foot:decimal(38,12),apr_sales_per_sq_foot:decimal(38,12),may_sales_per_sq_foot:decimal(38,12),jun_sales_per_sq_foot:decimal(38,12),jul_sales_per_sq_foot:decimal(38,12),aug_sales_per_sq_foot:decimal(38,12),sep_sales_per_sq_foot:decimal(38,12),oct_sales_per_sq_foot:decimal(38,12),nov_sales_per_sq_foot:decimal(38,12),dec_sales_per_sq_foot:decimal(38,12),jan_net:decimal(38,2),feb_net:decimal(38,2),mar_net:decimal(38,2),apr_net:decimal(38,2),may_net:decimal(38,2),jun_net:decimal(38,2),jul_net:decimal(38,2),aug_net:decimal(38,2),sep_net:decimal(38,2),oct_net:decimal(38,2),nov_net:decimal(38,2),dec_net:decimal(38,2)>
+-- !query output
+Just good amou	933435	Midway	Williamson County	TN	United States	DHL,BARIAN	2001	9246702.65	4973596.11	8243830.24	11536095.04	8848441.04	6589907.46	14946163.85	24260830.90	23156311.42	21897174.69	33046376.63	24591398.12	9.906102353136	5.328272573880	8.831713231238	12.358755606978	9.479439961004	7.059846116762	16.012002817550	25.990916239481	24.807631404437	23.458703273394	35.402975708003	26.345056827738	29452942.94	19052763.82	30303577.65	28563648.00	19179558.75	25111623.08	26955891.11	62345171.47	55967232.63	60120132.61	94483100.18	85104727.47
+Matches produce	198821	Midway	Williamson County	TN	United States	DHL,BARIAN	2001	8171037.69	7444707.30	8904945.91	8019953.83	13081755.25	8229030.48	11805829.83	25145588.96	15707862.93	18731722.90	27837144.77	33176280.56	41.097457964702	37.444270474447	44.788759285990	40.337559060662	65.796647486935	41.389141388486	59.379189471937	126.473506118569	79.005049416309	94.214006065758	140.011089220958	166.865072401809	25824877.19	17086383.78	24620145.44	28759520.46	24988081.00	21709799.17	25386565.52	54025360.94	54325725.26	59965431.57	75829271.22	85511883.03
+Operations	500020	Fairview	Williamson County	TN	United States	DHL,BARIAN	2001	10409399.91	16141530.81	8148742.71	6680962.19	11728095.96	9767093.38	11458408.56	20106135.67	25314120.67	22893962.36	35876332.21	28738513.96	20.817967101315	32.281770349186	16.296833546658	13.361389924403	23.455253709851	19.533405423783	22.915900483980	40.210662913483	50.626216291349	45.786093276269	71.749794428223	57.474728930843	18813981.10	29928354.06	20237634.01	22392433.57	25938775.04	28927292.16	25641922.14	60075246.63	49668486.97	68646750.13	88585464.12	75478344.38
+Selective,	720621	Fairview	Williamson County	TN	United States	DHL,BARIAN	2001	10540885.44	8098669.29	9370217.14	9753408.96	9006503.21	7725824.55	12918857.72	27286339.36	17315063.55	22573845.28	39002921.92	33502638.23	14.627502445808	11.238458621106	13.002975405935	13.534727630752	12.498252493335	10.721064956475	17.927395565769	37.865034962900	24.027975246350	31.325544606665	54.124042901886	46.491343202599	27419902.20	22882381.71	25735954.25	18682264.05	24373316.64	27192350.35	33019111.44	54462656.98	50994470.37	58509681.21	88329849.81	100833156.77
+Significantly	200313	Fairview	Williamson County	TN	United States	DHL,BARIAN	2001	9802846.67	12625248.69	6602477.32	11033269.92	6082390.20	15618748.55	5368995.93	19287503.44	21876336.16	23132474.86	35201297.49	21425392.36	48.937645934113	63.027605247787	32.960802943393	55.080149166554	30.364430666008	77.971717012875	26.803032903506	96.286828313689	109.210765951287	115.481645524754	175.731467703044	106.959570072836	23574229.26	27163084.88	24006453.37	23356164.02	19405795.41	35795815.91	25006450.52	58626312.30	57258316.63	60471338.55	83474267.08	73080652.01
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q67.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q67.sql.out
new file mode 100644
index 0000000000000..dbb955dcae502
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q67.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_category:string,i_class:string,i_brand:string,i_product_name:string,d_year:int,d_qoy:int,d_moy:int,s_store_id:string,sumsales:decimal(28,2),rk:int>
+-- !query output
+NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	672624.93	5
+NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	1328761.89	4
+NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	1867833.60	3
+NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	2951322.99	2
+NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	1012102066.67	1
+NULL	NULL	NULL	NULL	2000	NULL	NULL	NULL	672624.93	5
+NULL	NULL	NULL	NULL	2000	1	NULL	NULL	109354.01	39
+NULL	NULL	NULL	NULL	2000	1	1	NULL	56696.62	88
+NULL	NULL	NULL	NULL	2000	2	NULL	NULL	117554.43	24
+NULL	NULL	NULL	NULL	2000	2	6	NULL	50119.54	93
+NULL	NULL	NULL	NULL	2000	3	NULL	NULL	147622.32	12
+NULL	NULL	NULL	NULL	2000	3	8	NULL	64629.56	85
+NULL	NULL	NULL	NULL	2000	3	9	NULL	59755.91	86
+NULL	NULL	NULL	NULL	2000	4	NULL	NULL	298094.17	7
+NULL	NULL	NULL	NULL	2000	4	10	NULL	96651.33	58
+NULL	NULL	NULL	NULL	2000	4	10	AAAAAAAAKAAAAAAA	41992.37	100
+NULL	NULL	NULL	NULL	2000	4	11	NULL	110100.01	35
+NULL	NULL	NULL	NULL	2000	4	12	NULL	91342.83	68
+NULL	NULL	NULL	ationesebareing                                   	NULL	NULL	NULL	NULL	136570.22	16
+NULL	NULL	NULL	ationesebareing                                   	2000	NULL	NULL	NULL	136570.22	16
+NULL	NULL	NULL	ationesebareing                                   	2000	4	NULL	NULL	54262.81	90
+NULL	NULL	NULL	eseationablebarought                              	NULL	NULL	NULL	NULL	106226.70	44
+NULL	NULL	NULL	eseationablebarought                              	2000	NULL	NULL	NULL	106226.70	44
+NULL	NULL	NULL	eseationablebarought                              	2000	4	NULL	NULL	46143.77	96
+NULL	NULL	NULL	n stought                                         	NULL	NULL	NULL	NULL	90518.72	69
+NULL	NULL	NULL	n stought                                         	2000	NULL	NULL	NULL	90518.72	69
+NULL	NULL	NULL	oughtableantiable                                 	NULL	NULL	NULL	NULL	96782.30	52
+NULL	NULL	NULL	oughtableantiable                                 	2000	NULL	NULL	NULL	96782.30	52
+NULL	NULL	NULL	oughtableantiable                                 	2000	4	NULL	NULL	49543.67	94
+NULL	NULL	NULL	oughtablen stationought                           	NULL	NULL	NULL	NULL	105616.28	46
+NULL	NULL	NULL	oughtablen stationought                           	2000	NULL	NULL	NULL	105616.28	46
+NULL	NULL	NULL	oughtablen stationought                           	2000	4	NULL	NULL	66351.81	84
+NULL	NULL	NULL	oughteseoughtation                                	NULL	NULL	NULL	NULL	120422.74	22
+NULL	NULL	NULL	oughteseoughtation                                	2000	NULL	NULL	NULL	120422.74	22
+NULL	NULL	NULL	oughteseoughtation                                	2000	3	NULL	NULL	51575.65	92
+NULL	NULL	NULL	oughteseoughtation                                	2000	4	NULL	NULL	53356.26	91
+NULL	NULL	brandcorp #x                                      	NULL	NULL	NULL	NULL	NULL	110558.31	32
+NULL	NULL	brandcorp #x                                      	esepriantieing                                    	NULL	NULL	NULL	NULL	110558.31	32
+NULL	NULL	brandcorp #x                                      	esepriantieing                                    	2000	NULL	NULL	NULL	110558.31	32
+NULL	NULL	brandcorp #x                                      	esepriantieing                                    	2000	4	NULL	NULL	57539.32	87
+NULL	NULL	corpamalgamalg #x                                 	NULL	NULL	NULL	NULL	NULL	178041.47	9
+NULL	NULL	corpamalgamalg #x                                 	callyationationantiought                          	NULL	NULL	NULL	NULL	178041.47	9
+NULL	NULL	corpamalgamalg #x                                 	callyationationantiought                          	2000	NULL	NULL	NULL	178041.47	9
+NULL	NULL	corpamalgamalg #x                                 	callyationationantiought                          	2000	4	NULL	NULL	92734.16	59
+NULL	NULL	corpmaxi #x                                       	NULL	NULL	NULL	NULL	NULL	140599.56	13
+NULL	NULL	corpmaxi #x                                       	callyeingpri                                      	NULL	NULL	NULL	NULL	140599.56	13
+NULL	NULL	corpmaxi #x                                       	callyeingpri                                      	2000	NULL	NULL	NULL	140599.56	13
+NULL	NULL	corpmaxi #x                                       	callyeingpri                                      	2000	4	NULL	NULL	68417.63	83
+NULL	NULL	corpmaxi #x                                       	callyeingpri                                      	2000	4	11	NULL	44322.49	99
+NULL	NULL	importoamalg #x                                   	NULL	NULL	NULL	NULL	NULL	109872.37	36
+NULL	NULL	importoamalg #x                                   	NULL	NULL	NULL	NULL	NULL	109872.37	36
+NULL	NULL	importoamalg #x                                   	NULL	2000	NULL	NULL	NULL	109872.37	36
+NULL	accessories                                       	NULL	NULL	NULL	NULL	NULL	NULL	102003.04	48
+NULL	accessories                                       	NULL	NULL	NULL	NULL	NULL	NULL	102003.04	48
+NULL	accessories                                       	NULL	prin stn stoughtought                             	NULL	NULL	NULL	NULL	102003.04	48
+NULL	accessories                                       	NULL	prin stn stoughtought                             	2000	NULL	NULL	NULL	102003.04	48
+NULL	accessories                                       	NULL	prin stn stoughtought                             	2000	4	NULL	NULL	56122.71	89
+NULL	athletic                                          	NULL	NULL	NULL	NULL	NULL	NULL	72722.46	79
+NULL	athletic                                          	NULL	NULL	NULL	NULL	NULL	NULL	72722.46	79
+NULL	athletic                                          	NULL	NULL	NULL	NULL	NULL	NULL	72722.46	79
+NULL	athletic                                          	NULL	NULL	2000	NULL	NULL	NULL	72722.46	79
+NULL	baseball                                          	NULL	NULL	NULL	NULL	NULL	NULL	96654.08	54
+NULL	baseball                                          	NULL	NULL	NULL	NULL	NULL	NULL	96654.08	54
+NULL	baseball                                          	NULL	NULL	NULL	NULL	NULL	NULL	96654.08	54
+NULL	baseball                                          	NULL	NULL	2000	NULL	NULL	NULL	96654.08	54
+NULL	baseball                                          	NULL	NULL	2000	3	NULL	NULL	44473.92	98
+NULL	country                                           	NULL	NULL	NULL	NULL	NULL	NULL	114628.36	25
+NULL	country                                           	importoscholar #x                                 	NULL	NULL	NULL	NULL	NULL	114628.36	25
+NULL	country                                           	importoscholar #x                                 	NULL	NULL	NULL	NULL	NULL	114628.36	25
+NULL	country                                           	importoscholar #x                                 	NULL	2000	NULL	NULL	NULL	114628.36	25
+NULL	country                                           	importoscholar #x                                 	NULL	2000	4	NULL	NULL	44753.51	97
+NULL	dresses                                           	NULL	NULL	NULL	NULL	NULL	NULL	77687.53	72
+NULL	dresses                                           	NULL	NULL	NULL	NULL	NULL	NULL	192205.73	8
+NULL	dresses                                           	NULL	oughteingantieing                                 	NULL	NULL	NULL	NULL	77687.53	72
+NULL	dresses                                           	NULL	oughteingantieing                                 	2000	NULL	NULL	NULL	77687.53	72
+NULL	dresses                                           	amalgamalg #x                                     	NULL	NULL	NULL	NULL	NULL	114518.20	29
+NULL	dresses                                           	amalgamalg #x                                     	NULL	NULL	NULL	NULL	NULL	114518.20	29
+NULL	dresses                                           	amalgamalg #x                                     	NULL	2000	NULL	NULL	NULL	114518.20	29
+NULL	dresses                                           	amalgamalg #x                                     	NULL	2000	4	NULL	NULL	49313.59	95
+NULL	guns                                              	NULL	NULL	NULL	NULL	NULL	NULL	92681.88	60
+NULL	guns                                              	NULL	NULL	NULL	NULL	NULL	NULL	92681.88	60
+NULL	guns                                              	NULL	NULL	NULL	NULL	NULL	NULL	92681.88	60
+NULL	guns                                              	NULL	NULL	2000	NULL	NULL	NULL	92681.88	60
+NULL	infants                                           	NULL	NULL	NULL	NULL	NULL	NULL	129135.94	18
+NULL	infants                                           	importoexporti #x                                 	NULL	NULL	NULL	NULL	NULL	129135.94	18
+NULL	infants                                           	importoexporti #x                                 	antiationeseese                                   	NULL	NULL	NULL	NULL	129135.94	18
+NULL	infants                                           	importoexporti #x                                 	antiationeseese                                   	2000	NULL	NULL	NULL	129135.94	18
+NULL	infants                                           	importoexporti #x                                 	antiationeseese                                   	2000	4	NULL	NULL	80824.80	71
+NULL	lighting                                          	NULL	NULL	NULL	NULL	NULL	NULL	107477.64	40
+NULL	lighting                                          	NULL	NULL	NULL	NULL	NULL	NULL	107477.64	40
+NULL	lighting                                          	NULL	NULL	NULL	NULL	NULL	NULL	107477.64	40
+NULL	lighting                                          	NULL	NULL	2000	NULL	NULL	NULL	107477.64	40
+NULL	mens                                              	NULL	NULL	NULL	NULL	NULL	NULL	91415.81	64
+NULL	mens                                              	importoedu pack #x                                	NULL	NULL	NULL	NULL	NULL	91415.81	64
+NULL	mens                                              	importoedu pack #x                                	oughtablen steseought                             	NULL	NULL	NULL	NULL	91415.81	64
+NULL	mens                                              	importoedu pack #x                                	oughtablen steseought                             	2000	NULL	NULL	NULL	91415.81	64
+NULL	shirts                                            	NULL	NULL	NULL	NULL	NULL	NULL	72876.08	75
+NULL	shirts                                            	NULL	NULL	NULL	NULL	NULL	NULL	72876.08	75
+NULL	shirts                                            	NULL	oughtcallyeseantiought                            	NULL	NULL	NULL	NULL	72876.08	75
+NULL	shirts                                            	NULL	oughtcallyeseantiought                            	2000	NULL	NULL	NULL	72876.08	75
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q68.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q68.sql.out
new file mode 100644
index 0000000000000..11a564346ec52
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q68.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<c_last_name:string,c_first_name:string,ca_city:string,bought_city:string,ss_ticket_number:int,extended_price:decimal(17,2),extended_tax:decimal(17,2),list_price:decimal(17,2)>
+-- !query output
+NULL	NULL	Salem	Lakeside	2203	23464.72	675.04	46100.94
+NULL	Ira                 	Riverdale	Oak Ridge	10508	32911.54	938.79	59812.32
+NULL	Rebecca             	Belmont	Marion	13206	14180.52	1072.25	23887.85
+NULL	NULL	Pleasant Valley	Lincoln	20829	16340.86	289.50	34809.84
+NULL	Joel                	Oak Hill	Clifton	21390	19561.57	939.54	46319.37
+NULL	Max                 	Stringtown	Pine Grove	22049	20059.77	974.94	38268.07
+NULL	NULL	Bridgeport	Riverside	23264	10699.95	496.36	29285.70
+NULL	NULL	White Oak	Pleasant Hill	24860	16258.45	733.62	52186.60
+NULL	Rebecca             	Union Hill	Farmington	26031	7636.54	515.78	13443.39
+NULL	Alecia              	Jamestown	Oakdale	30174	29125.41	1118.58	69857.70
+NULL	Raymond             	Bunker Hill	Clifton	30655	27237.08	1237.59	51322.22
+NULL	NULL	Pleasant Valley	Riley	30915	25079.14	998.46	35200.27
+NULL	David               	Glendale	Greenville	33716	15434.87	1012.60	30661.71
+NULL	NULL	New Hope	Farmington	38760	29150.85	647.52	50433.60
+NULL	Norma               	Hamilton	Macedonia	41929	35397.41	1268.87	55360.80
+NULL	NULL	Wildwood	Allison	45786	10448.67	492.27	32424.46
+NULL	NULL	Greenville	Spring Hill	47915	26747.77	512.05	57910.19
+NULL	Jerome              	Galena	Georgetown	50559	33831.95	890.50	57970.13
+NULL	NULL	Woodlawn	Springfield	59625	22470.48	828.96	54458.90
+NULL	NULL	Hillcrest	Cordova	64306	10534.53	295.14	15380.96
+NULL	John                	Stringtown	Arlington	64522	21495.01	783.81	45067.59
+NULL	NULL	Gladstone	Summit	66488	20881.36	605.92	32362.22
+NULL	NULL	Hopewell	Riverdale	67278	24471.59	1549.17	48406.81
+NULL	Russel              	Red Hill	Woodland	82061	17750.10	1025.09	40565.68
+NULL	NULL	Mount Olive	Maple Grove	82183	16184.99	573.72	44448.96
+NULL	NULL	Shady Grove	Springdale	91775	17103.31	911.07	35127.63
+NULL	NULL	Indian Village	Golden	98530	18842.65	1005.52	40742.92
+NULL	NULL	Greenfield	Farmington	98985	13264.63	589.99	20576.04
+NULL	Virginia            	Five Forks	Oakland	106543	24859.75	1006.88	39693.34
+NULL	NULL	Scottsville	Greenfield	107236	14638.50	802.00	35356.18
+NULL	Virginia            	Oak Hill	Amherst	109641	7667.92	271.98	30755.88
+NULL	NULL	Midway	New Hope	113272	50295.57	2963.67	78595.37
+NULL	William             	Valley View	Providence	115559	29947.61	673.81	50883.54
+NULL	NULL	Newport	Indian Village	120966	15268.90	721.76	27022.81
+NULL	Robert              	Cedar Grove	Riverside	122552	20398.86	1049.35	44761.40
+NULL	NULL	Jamestown	Spring Valley	125125	31795.01	1364.75	52740.75
+NULL	NULL	Liberty	Oakwood	128427	41495.05	2399.87	77105.28
+NULL	Brandon             	Arlington	Woodland	135601	43789.02	1493.45	66479.43
+NULL	Juan                	Mount Pleasant	Greenwood	138846	12326.46	472.96	25735.00
+NULL	NULL	Shirley	Fairview	143290	19808.13	596.98	34256.82
+NULL	NULL	White Oak	Ferguson	146537	5235.24	246.72	11343.46
+NULL	Luther              	Greenville	Hillcrest	149654	27079.37	939.22	41067.50
+NULL	Michelle            	Fairview	Hopewell	149783	29018.58	1426.60	48040.63
+NULL	Angelica            	Barnes	Plainview	151659	21082.45	306.14	40857.65
+NULL	Donna               	Centerville	Friendship	152588	20862.64	969.20	46304.16
+NULL	Brian               	Oak Ridge	Woodville	152862	22493.16	1611.93	43153.08
+NULL	NULL	Stringtown	Liberty	158006	16267.97	450.16	25629.75
+NULL	Daniel              	Globe	Spring Valley	161922	15092.01	847.21	42715.60
+NULL	Marlene             	Georgetown	Oakland	163799	42790.16	1130.51	77449.21
+NULL	Dean                	Lakeview	New Hope	167847	24431.20	1333.52	38658.46
+NULL	NULL	Greenfield	Hopewell	168370	13878.80	617.56	31230.84
+NULL	Robert              	Four Points	Farmington	176911	14828.21	674.85	32517.24
+NULL	Betty               	Lakewood	Sulphur Springs	178057	15008.49	693.61	25864.62
+NULL	NULL	Harmony	Spring Valley	178332	10288.94	631.05	35516.38
+NULL	NULL	Enterprise	Bunker Hill	182211	14072.57	405.42	33771.00
+NULL	NULL	Salem	Valley View	187904	15702.34	439.59	41237.94
+NULL	Faye                	Walnut Grove	Oakland	189770	18527.90	1113.75	39987.40
+NULL	Debra               	Green Acres	Woodlawn	192040	9053.50	169.31	13181.96
+NULL	NULL	Salem	Antioch	192673	5412.31	119.61	12746.52
+NULL	NULL	Mount Olive	Mount Pleasant	197746	32038.93	1017.39	87608.72
+NULL	Nancy               	Shady Grove	Hillcrest	201415	14985.28	328.13	21957.26
+NULL	Scott               	Hillcrest	Louisville	203119	8629.88	329.70	14712.89
+NULL	NULL	Plainview	Oak Grove	203425	33448.32	1632.67	62674.71
+NULL	NULL	Rocky Hill	Providence	211665	15060.41	598.75	29385.31
+NULL	Tonya               	Buena Vista	Bunker Hill	213228	46912.33	2039.49	81223.03
+NULL	Troy                	Plainview	Ashland	216821	45620.34	2295.90	74013.12
+NULL	Sandra              	Unionville	Hopewell	221077	8341.60	527.66	17817.11
+NULL	Robert              	Summit	Five Points	224508	15607.69	500.96	37984.93
+NULL	Rebecca             	Union Hill	Mountain View	225489	35322.48	1859.03	58568.59
+NULL	Jack                	Sunnyside	Lakeview	231229	16186.51	577.86	30596.37
+NULL	NULL	Bethel	Sunnyside	233081	13271.69	468.80	32129.60
+NULL	Alice               	Oak Grove	Marion	233656	30278.85	1490.14	65326.68
+NULL	Shayne              	Riverview	Crossroads	234750	20554.66	737.10	52975.61
+NULL	Kevin               	Buena Vista	Clifton	239837	28589.10	999.94	48962.74
+Aaron                         	Nick                	Harmony	Edgewood	73734	29649.48	959.10	56974.10
+Abney                         	Janice              	Mount Zion	Enterprise	27585	16154.82	636.94	37751.16
+Abraham                       	Gerald              	Pleasant Grove	Midway	1779	8661.58	330.60	14811.87
+Abrams                        	Alma                	Fairfield	Macedonia	33078	28262.54	808.09	48263.03
+Adame                         	Brian               	Marion	Forest Hills	142280	15398.91	717.67	52277.17
+Adams                         	Adam                	The Meadows	Glenwood	35054	33420.92	1477.44	37326.99
+Adams                         	Paulette            	Tremont	Lakeside	82644	25822.05	365.13	46273.62
+Adams                         	Edwin               	Plainview	Greenwood	108138	23483.91	982.40	35181.82
+Adams                         	Nichole             	Edgewood	Friendship	171894	20821.96	1333.70	48375.06
+Adams                         	NULL	Oakland	Oak Ridge	173206	18700.18	1085.29	29361.60
+Adams                         	Nichole             	Edgewood	Green Acres	180862	17519.04	657.52	33555.87
+Adams                         	Donna               	Plainview	Liberty	219849	18150.78	448.49	22444.41
+Aguilar                       	Jeannine            	Unionville	Jackson	152737	34732.21	1299.63	56844.65
+Ahmed                         	Jeffrey             	Stewart	Edgewood	202936	27333.88	948.83	39990.68
+Albert                        	Sally               	Nottingham	Mount Zion	7000	11496.97	305.77	28524.27
+Albrecht                      	Bob                 	Marion	Pleasant Hill	60194	23528.46	1101.20	43551.97
+Alcorn                        	Jeffery             	Cedar Grove	Salem	46467	17634.79	785.53	57718.43
+Alderman                      	Melanie             	Edgewood	Mount Zion	14230	28157.47	270.85	48818.21
+Aldridge                      	Daniel              	Berlin	Wilson	114445	15440.33	552.85	22637.40
+Alger                         	Beverly             	Allentown	Riverview	173449	24541.75	797.49	48511.28
+Allen                         	Brittany            	Lincoln	Hillcrest	10696	17579.89	595.90	46637.81
+Allen                         	Yvette              	Providence	Flatwoods	73587	8938.13	561.96	27604.45
+Allen                         	Lori                	Hamilton	Greenwood	138690	16085.88	579.59	35840.88
+Allison                       	Anya                	Union	Ashland	79895	9453.18	499.78	22112.21
+Alvarez                       	Terrence            	Concord	Spring Valley	32211	13206.58	470.82	33690.82
+Alvarez                       	Marie               	Walnut Grove	Red Hill	206011	25535.02	1344.58	63224.42
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q69.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q69.sql.out
new file mode 100644
index 0000000000000..bd90d440c523c
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q69.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<cd_gender:string,cd_marital_status:string,cd_education_status:string,cnt1:bigint,cd_purchase_estimate:int,cnt2:bigint,cd_credit_rating:string,cnt3:bigint>
+-- !query output
+F	D	2 yr Degree         	1	500	1	Low Risk  	1
+F	D	2 yr Degree         	1	3000	1	High Risk 	1
+F	D	2 yr Degree         	1	4500	1	Good      	1
+F	D	2 yr Degree         	1	5000	1	Low Risk  	1
+F	D	2 yr Degree         	2	6000	2	High Risk 	2
+F	D	2 yr Degree         	1	6500	1	Good      	1
+F	D	2 yr Degree         	1	7000	1	High Risk 	1
+F	D	2 yr Degree         	1	8500	1	High Risk 	1
+F	D	2 yr Degree         	1	8500	1	Low Risk  	1
+F	D	4 yr Degree         	1	1000	1	High Risk 	1
+F	D	4 yr Degree         	1	2500	1	Good      	1
+F	D	4 yr Degree         	1	3000	1	Low Risk  	1
+F	D	4 yr Degree         	1	3500	1	High Risk 	1
+F	D	4 yr Degree         	1	3500	1	Unknown   	1
+F	D	4 yr Degree         	1	4000	1	Unknown   	1
+F	D	4 yr Degree         	1	4500	1	Unknown   	1
+F	D	4 yr Degree         	1	6500	1	High Risk 	1
+F	D	4 yr Degree         	1	7500	1	High Risk 	1
+F	D	4 yr Degree         	1	8500	1	Low Risk  	1
+F	D	4 yr Degree         	1	9000	1	Low Risk  	1
+F	D	4 yr Degree         	1	9000	1	Unknown   	1
+F	D	4 yr Degree         	1	10000	1	High Risk 	1
+F	D	Advanced Degree     	2	1000	2	Unknown   	2
+F	D	Advanced Degree     	1	1500	1	High Risk 	1
+F	D	Advanced Degree     	1	1500	1	Unknown   	1
+F	D	Advanced Degree     	1	3000	1	Good      	1
+F	D	Advanced Degree     	1	3000	1	Unknown   	1
+F	D	Advanced Degree     	1	6000	1	Low Risk  	1
+F	D	Advanced Degree     	1	8000	1	Unknown   	1
+F	D	College             	1	1000	1	Low Risk  	1
+F	D	College             	1	2000	1	High Risk 	1
+F	D	College             	3	2500	3	High Risk 	3
+F	D	College             	1	4000	1	High Risk 	1
+F	D	College             	1	5500	1	High Risk 	1
+F	D	College             	2	7500	2	Unknown   	2
+F	D	College             	1	8000	1	Good      	1
+F	D	College             	1	9000	1	Unknown   	1
+F	D	Primary             	1	500	1	Good      	1
+F	D	Primary             	1	1000	1	Unknown   	1
+F	D	Primary             	1	1500	1	Good      	1
+F	D	Primary             	1	2000	1	High Risk 	1
+F	D	Primary             	2	2000	2	Unknown   	2
+F	D	Primary             	1	2500	1	Unknown   	1
+F	D	Primary             	1	4000	1	Low Risk  	1
+F	D	Primary             	1	5000	1	High Risk 	1
+F	D	Primary             	1	6000	1	Unknown   	1
+F	D	Primary             	1	6500	1	High Risk 	1
+F	D	Primary             	1	7500	1	High Risk 	1
+F	D	Primary             	1	7500	1	Unknown   	1
+F	D	Primary             	1	8000	1	Low Risk  	1
+F	D	Primary             	2	9500	2	Low Risk  	2
+F	D	Secondary           	1	1500	1	High Risk 	1
+F	D	Secondary           	1	2000	1	Unknown   	1
+F	D	Secondary           	1	2500	1	High Risk 	1
+F	D	Secondary           	2	4000	2	High Risk 	2
+F	D	Secondary           	1	5000	1	Low Risk  	1
+F	D	Secondary           	1	6000	1	High Risk 	1
+F	D	Secondary           	1	10000	1	Low Risk  	1
+F	D	Unknown             	1	2000	1	High Risk 	1
+F	D	Unknown             	1	5000	1	Low Risk  	1
+F	D	Unknown             	1	6500	1	Good      	1
+F	D	Unknown             	1	8000	1	Unknown   	1
+F	D	Unknown             	1	9000	1	High Risk 	1
+F	M	2 yr Degree         	1	500	1	High Risk 	1
+F	M	2 yr Degree         	1	4000	1	Unknown   	1
+F	M	2 yr Degree         	1	4500	1	Low Risk  	1
+F	M	2 yr Degree         	1	5000	1	Unknown   	1
+F	M	2 yr Degree         	1	5500	1	Low Risk  	1
+F	M	2 yr Degree         	1	9000	1	Low Risk  	1
+F	M	2 yr Degree         	1	10000	1	Low Risk  	1
+F	M	4 yr Degree         	1	500	1	High Risk 	1
+F	M	4 yr Degree         	1	1000	1	Good      	1
+F	M	4 yr Degree         	1	2500	1	Good      	1
+F	M	4 yr Degree         	1	3500	1	Low Risk  	1
+F	M	4 yr Degree         	1	4500	1	Good      	1
+F	M	4 yr Degree         	1	7000	1	Good      	1
+F	M	4 yr Degree         	1	7500	1	Good      	1
+F	M	4 yr Degree         	1	7500	1	High Risk 	1
+F	M	4 yr Degree         	1	8000	1	Unknown   	1
+F	M	4 yr Degree         	1	8500	1	High Risk 	1
+F	M	4 yr Degree         	1	9000	1	Unknown   	1
+F	M	4 yr Degree         	1	10000	1	Unknown   	1
+F	M	Advanced Degree     	1	3000	1	Unknown   	1
+F	M	Advanced Degree     	1	4000	1	Low Risk  	1
+F	M	Advanced Degree     	1	4500	1	Unknown   	1
+F	M	Advanced Degree     	1	6000	1	Unknown   	1
+F	M	Advanced Degree     	1	6500	1	High Risk 	1
+F	M	Advanced Degree     	1	8500	1	Good      	1
+F	M	Advanced Degree     	1	8500	1	High Risk 	1
+F	M	College             	1	1500	1	Unknown   	1
+F	M	College             	1	4500	1	Low Risk  	1
+F	M	College             	1	7000	1	Unknown   	1
+F	M	College             	1	8500	1	High Risk 	1
+F	M	College             	1	8500	1	Low Risk  	1
+F	M	College             	1	8500	1	Unknown   	1
+F	M	College             	1	9500	1	Good      	1
+F	M	Primary             	1	500	1	Unknown   	1
+F	M	Primary             	1	1000	1	Good      	1
+F	M	Primary             	1	4500	1	Low Risk  	1
+F	M	Primary             	1	5000	1	High Risk 	1
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q7.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q7.sql.out
new file mode 100644
index 0000000000000..6c4ec28f8ddc3
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q7.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_item_id:string,agg1:double,agg2:decimal(11,6),agg3:decimal(11,6),agg4:decimal(11,6)>
+-- !query output
+AAAAAAAAAAACAAAA	60.0	52.450000	0.000000	13.630000
+AAAAAAAAAAAEAAAA	96.0	51.560000	0.000000	46.400000
+AAAAAAAAAABEAAAA	8.0	135.430000	0.000000	105.630000
+AAAAAAAAAACAAAAA	43.0	90.780000	0.000000	17.240000
+AAAAAAAAAACCAAAA	31.0	80.140000	0.000000	74.530000
+AAAAAAAAAADBAAAA	79.0	99.440000	2573.530000	90.490000
+AAAAAAAAAADCAAAA	68.0	115.045000	0.000000	72.910000
+AAAAAAAAAAEAAAAA	42.0	16.800000	0.000000	13.770000
+AAAAAAAAAAEDAAAA	42.5	52.175000	0.000000	21.300000
+AAAAAAAAAAFAAAAA	32.0	24.750000	0.000000	23.510000
+AAAAAAAAAAGBAAAA	48.0	119.350000	0.000000	59.670000
+AAAAAAAAAAGCAAAA	54.0	39.420000	964.260000	30.305000
+AAAAAAAAAAHBAAAA	59.333333333333336	86.790000	198.126667	47.576667
+AAAAAAAAAAHDAAAA	82.0	26.030000	981.850000	26.030000
+AAAAAAAAAAKAAAAA	51.666666666666664	47.246667	0.000000	5.333333
+AAAAAAAAAAKBAAAA	11.0	114.560000	0.000000	90.500000
+AAAAAAAAAAKDAAAA	27.0	23.140000	0.000000	20.820000
+AAAAAAAAAALAAAAA	38.0	83.580000	0.000000	51.810000
+AAAAAAAAAALCAAAA	51.0	26.610000	0.000000	19.950000
+AAAAAAAAAAMBAAAA	79.0	41.660000	0.000000	38.740000
+AAAAAAAAAAMCAAAA	70.0	83.210000	0.000000	49.090000
+AAAAAAAAAANAAAAA	2.5	97.780000	0.000000	77.765000
+AAAAAAAAAAOAAAAA	60.0	101.410000	1674.240000	43.600000
+AAAAAAAAAAOCAAAA	7.0	185.510000	0.000000	109.450000
+AAAAAAAAAAPBAAAA	51.5	57.890000	839.850000	52.465000
+AAAAAAAAABABAAAA	20.0	19.300000	0.000000	0.770000
+AAAAAAAAABADAAAA	95.0	100.800000	0.000000	35.280000
+AAAAAAAAABAEAAAA	55.0	2.220000	0.000000	2.080000
+AAAAAAAAABBAAAAA	46.0	61.386667	0.000000	27.306667
+AAAAAAAAABBDAAAA	35.0	166.510000	166.080000	158.180000
+AAAAAAAAABCBAAAA	32.0	33.820000	175.495000	13.025000
+AAAAAAAAABDDAAAA	38.666666666666664	48.240000	1.143333	27.163333
+AAAAAAAAABDEAAAA	8.0	135.240000	38.095000	18.845000
+AAAAAAAAABECAAAA	13.0	73.870000	0.000000	64.260000
+AAAAAAAAABEDAAAA	98.0	76.895000	0.000000	25.375000
+AAAAAAAAABGAAAAA	81.0	46.320000	0.000000	26.580000
+AAAAAAAAABGDAAAA	88.0	81.150000	0.000000	43.820000
+AAAAAAAAABGEAAAA	81.0	18.760000	0.000000	7.310000
+AAAAAAAAABHAAAAA	61.5	101.695000	224.313333	33.520000
+AAAAAAAAABHDAAAA	51.0	128.445000	0.000000	83.725000
+AAAAAAAAABIBAAAA	51.0	20.940000	0.000000	6.490000
+AAAAAAAAABICAAAA	82.5	62.635000	0.860000	27.015000
+AAAAAAAAABJAAAAA	18.333333333333332	76.346667	167.953333	44.930000
+AAAAAAAAABKAAAAA	77.0	122.740000	0.000000	98.190000
+AAAAAAAAABKDAAAA	42.0	125.690000	1504.230000	37.700000
+AAAAAAAAABLBAAAA	59.0	118.240000	72.480000	9.450000
+AAAAAAAAABMBAAAA	97.5	91.465000	0.000000	60.660000
+AAAAAAAAABMDAAAA	46.0	171.290000	0.000000	92.490000
+AAAAAAAAABNCAAAA	20.0	109.530000	0.000000	97.480000
+AAAAAAAAABPAAAAA	59.25	47.130000	62.827500	29.995000
+AAAAAAAAABPBAAAA	66.0	53.950000	30.970000	46.930000
+AAAAAAAAABPDAAAA	66.5	52.420000	0.000000	30.925000
+AAAAAAAAACAAAAAA	45.0	75.860000	0.000000	22.750000
+AAAAAAAAACBBAAAA	64.5	27.585000	0.000000	16.820000
+AAAAAAAAACBCAAAA	71.0	47.330000	0.000000	18.450000
+AAAAAAAAACCDAAAA	68.0	92.350000	0.000000	41.550000
+AAAAAAAAACDAAAAA	21.0	64.390000	59.620000	13.520000
+AAAAAAAAACDDAAAA	65.66666666666667	34.473333	23.316667	14.916667
+AAAAAAAAACEBAAAA	48.0	111.330000	0.000000	12.240000
+AAAAAAAAACECAAAA	67.5	36.365000	373.635000	26.500000
+AAAAAAAAACFBAAAA	9.0	19.600000	0.000000	1.370000
+AAAAAAAAACFDAAAA	26.0	18.350000	0.000000	13.390000
+AAAAAAAAACGCAAAA	60.5	100.270000	0.000000	45.862500
+AAAAAAAAACGDAAAA	58.5	32.805000	396.170000	12.550000
+AAAAAAAAACHBAAAA	97.0	5.380000	27.470000	0.480000
+AAAAAAAAACIAAAAA	57.0	56.880000	0.000000	52.750000
+AAAAAAAAACIBAAAA	58.0	26.060000	152.250000	12.500000
+AAAAAAAAACIDAAAA	71.0	4.020000	0.000000	2.210000
+AAAAAAAAACJAAAAA	30.0	46.910000	0.000000	8.910000
+AAAAAAAAACJDAAAA	63.5	112.305000	0.000000	30.610000
+AAAAAAAAACKCAAAA	50.666666666666664	63.280000	0.000000	26.126667
+AAAAAAAAACLAAAAA	95.0	22.310000	0.000000	0.000000
+AAAAAAAAACLBAAAA	88.0	27.450000	0.000000	23.880000
+AAAAAAAAACMAAAAA	86.0	90.600000	1511.530000	87.880000
+AAAAAAAAACNBAAAA	44.666666666666664	73.076667	52.400000	54.546667
+AAAAAAAAACNCAAAA	62.0	105.380000	0.000000	70.600000
+AAAAAAAAACOAAAAA	41.0	42.810000	412.370000	10.700000
+AAAAAAAAACOBAAAA	36.5	66.735000	14.770000	33.725000
+AAAAAAAAACODAAAA	25.0	65.280000	238.920000	39.820000
+AAAAAAAAACPAAAAA	88.0	164.900000	104.470000	14.840000
+AAAAAAAAACPDAAAA	10.0	78.550000	0.000000	10.990000
+AAAAAAAAADABAAAA	81.0	60.225000	0.000000	43.775000
+AAAAAAAAADBDAAAA	2.0	94.600000	0.000000	59.590000
+AAAAAAAAADCAAAAA	52.0	72.480000	98.720000	15.350000
+AAAAAAAAADCCAAAA	89.0	61.690000	0.000000	59.220000
+AAAAAAAAADCDAAAA	27.5	126.585000	0.000000	61.340000
+AAAAAAAAADDBAAAA	64.0	19.570000	934.160000	16.040000
+AAAAAAAAADDEAAAA	65.0	78.255000	348.165000	49.595000
+AAAAAAAAADEAAAAA	65.0	108.640000	0.000000	91.250000
+AAAAAAAAADEBAAAA	70.0	104.400000	1189.580000	45.930000
+AAAAAAAAADEDAAAA	31.0	130.200000	0.000000	76.810000
+AAAAAAAAADFAAAAA	80.0	5.240000	0.000000	1.310000
+AAAAAAAAADFCAAAA	14.0	9.110000	0.000000	6.650000
+AAAAAAAAADGCAAAA	10.0	84.260000	0.000000	9.260000
+AAAAAAAAADGEAAAA	37.333333333333336	34.403333	101.936667	17.150000
+AAAAAAAAADHBAAAA	32.333333333333336	65.606667	751.803333	54.530000
+AAAAAAAAADJBAAAA	57.0	63.110000	0.000000	47.960000
+AAAAAAAAADKBAAAA	44.0	44.980000	0.000000	3.590000
+AAAAAAAAADLAAAAA	21.0	17.740000	0.000000	9.930000
+AAAAAAAAADMBAAAA	29.0	102.770000	2052.360000	72.960000
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q70.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q70.sql.out
new file mode 100644
index 0000000000000..a28271a34dcb1
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q70.sql.out
@@ -0,0 +1,8 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<total_sum:decimal(17,2),s_state:string,s_county:string,lochierarchy:tinyint,rank_within_parent:int>
+-- !query output
+-439591881.24	NULL	NULL	2	1
+-439591881.24	TN	NULL	1	1
+-439591881.24	TN	Williamson County	0	1
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q71.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q71.sql.out
new file mode 100644
index 0000000000000..502ee6a80b7ad
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q71.sql.out
@@ -0,0 +1,1195 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<brand_id:int,brand:string,t_hour:int,t_minute:int,ext_price:decimal(17,2)>
+-- !query output
+10016002	corpamalgamalg #x                                 	19	7	23183.08
+10005002	scholarunivamalg #x                               	8	8	17109.63
+8014006	edu packmaxi #x                                   	9	53	16420.75
+3003002	exportiexporti #x                                 	17	44	16337.23
+3001001	amalgexporti #x                                   	17	46	16088.53
+4001001	amalgedu pack #x                                  	9	28	15557.83
+6015001	scholarbrand #x                                   	17	50	15411.90
+6007007	brandcorp #x                                      	19	25	15067.50
+6007007	brandcorp #x                                      	18	59	14974.32
+8014006	edu packmaxi #x                                   	19	40	14889.35
+4004002	edu packedu pack #x                               	18	4	14418.00
+3002002	importoexporti #x                                 	19	40	14063.50
+3003001	exportiexporti #x                                 	19	58	13900.80
+7014010	edu packnameless #x                              	9	52	13795.60
+2004002	edu packimporto #x                                	19	17	13663.76
+9013003	exportiunivamalg #x                               	19	24	13511.26
+5001002	amalgscholar #x                                   	9	41	13318.05
+7014003	edu packnameless #x                               	19	52	12754.50
+5002002	importoscholar #x                                 	7	16	12584.65
+10005002	scholarunivamalg #x                               	17	19	12325.56
+4003001	exportiedu pack #x                                	17	0	12313.71
+1004001	edu packamalg #x                                  	9	6	12311.28
+4001001	amalgedu pack #x                                  	18	21	12299.94
+8004002	edu packnameless #x                               	9	3	12191.36
+10008006	namelessunivamalg #x                              	19	47	11971.67
+8001007	amalgnameless #x                                  	19	36	11926.98
+9003003	exportimaxi #x                                    	17	10	11905.95
+1003001	exportiamalg #x                                   	17	48	11845.76
+6009008	maxicorp #x                                       	17	1	11772.54
+6015003	scholarbrand #x                                   	9	40	11755.64
+9003003	exportimaxi #x                                    	19	3	11623.24
+9014011	edu packunivamalg #x                             	19	12	11407.38
+1001001	amalgamalg #x                                     	17	38	11250.06
+7013007	exportinameless #x                                	7	27	11169.57
+5001002	amalgscholar #x                                   	18	35	11152.35
+9016003	corpunivamalg #x                                  	17	11	10903.08
+2002002	importoimporto #x                                 	8	38	10861.98
+7009003	maxibrand #x                                      	9	44	10687.08
+3002001	importoexporti #x                                 	17	37	10580.80
+9008003	namelessmaxi #x                                   	9	49	10517.90
+8001007	amalgnameless #x                                  	9	22	10416.24
+7008005	namelessbrand #x                                  	18	31	10196.64
+4004002	edu packedu pack #x                               	18	29	10166.76
+4003001	exportiedu pack #x                                	8	41	10118.45
+8012007	importomaxi #x                                    	17	47	10079.52
+5001002	amalgscholar #x                                   	9	11	9922.06
+4003002	exportiedu pack #x                                	9	8	9888.56
+7004005	edu packbrand #x                                  	17	8	9600.00
+9008003	namelessmaxi #x                                   	18	46	9599.75
+9007003	brandmaxi #x                                      	18	23	9492.33
+7014003	edu packnameless #x                               	19	39	9431.61
+3004001	edu packexporti #x                                	18	53	9398.02
+3002001	importoexporti #x                                 	17	51	9370.58
+3002001	importoexporti #x                                 	9	58	9365.40
+1002001	importoamalg #x                                   	8	59	9292.90
+7013002	exportinameless #x                                	6	23	9269.32
+7002009	importobrand #x                                   	17	31	9114.30
+5003001	exportischolar #x                                 	19	22	9074.52
+10007016	brandunivamalg #x                                	18	38	9003.20
+10005012	scholarunivamalg #x                              	18	6	8947.68
+9003003	exportimaxi #x                                    	9	0	8849.28
+8002008	importonameless #x                                	18	59	8721.28
+8002007	importonameless #x                                	8	51	8694.00
+3004002	edu packexporti #x                                	8	18	8480.98
+5002002	importoscholar #x                                 	9	33	8431.24
+3004001	edu packexporti #x                                	19	5	8427.36
+9003003	exportimaxi #x                                    	8	35	8400.48
+4001001	amalgedu pack #x                                  	17	37	8391.68
+9014011	edu packunivamalg #x                             	7	33	8367.30
+8002007	importonameless #x                                	19	52	8361.36
+10014005	edu packamalgamalg #x                             	9	4	8332.00
+10001003	amalgunivamalg #x                                 	17	56	8299.50
+10011008	amalgamalgamalg #x                                	18	25	8223.96
+9016005	corpunivamalg #x                                  	17	2	8104.92
+8001007	amalgnameless #x                                  	19	42	8077.98
+10014001	edu packamalgamalg #x                             	19	52	8077.98
+2002002	importoimporto #x                                 	17	52	8045.25
+8005009	scholarnameless #x                                	8	17	8033.28
+9013003	exportiunivamalg #x                               	19	38	7994.24
+5001002	amalgscholar #x                                   	19	18	7980.48
+4003001	exportiedu pack #x                                	8	57	7973.82
+8006010	corpnameless #x                                  	19	41	7864.08
+10005002	scholarunivamalg #x                               	6	44	7846.24
+4003001	exportiedu pack #x                                	19	19	7784.72
+9014011	edu packunivamalg #x                             	17	48	7677.20
+7014010	edu packnameless #x                              	9	29	7671.96
+4003001	exportiedu pack #x                                	19	29	7630.71
+4001001	amalgedu pack #x                                  	9	33	7626.44
+2001001	amalgimporto #x                                   	17	46	7444.64
+9004008	edu packmaxi #x                                   	19	53	7429.73
+9010002	univunivamalg #x                                  	17	25	7428.08
+2001001	amalgimporto #x                                   	19	10	7387.20
+8010001	univmaxi #x                                       	17	38	7327.00
+9003003	exportimaxi #x                                    	17	49	7325.40
+4002001	importoedu pack #x                                	19	29	7273.80
+5001002	amalgscholar #x                                   	19	31	7253.56
+10001007	amalgunivamalg #x                                 	17	35	7188.16
+3002001	importoexporti #x                                 	19	36	7184.41
+1004001	edu packamalg #x                                  	9	39	7168.20
+8006010	corpnameless #x                                  	6	45	7122.15
+9014008	edu packunivamalg #x                              	19	35	7048.58
+6015001	scholarbrand #x                                   	19	45	7043.40
+7014001	edu packnameless #x                               	9	9	6921.20
+3001001	amalgexporti #x                                   	17	58	6919.84
+7016008	corpnameless #x                                   	18	20	6866.37
+4004002	edu packedu pack #x                               	9	23	6819.60
+7013002	exportinameless #x                                	8	4	6757.74
+4001001	amalgedu pack #x                                  	18	20	6744.87
+7014010	edu packnameless #x                              	17	23	6714.90
+2002002	importoimporto #x                                 	9	49	6696.46
+3003002	exportiexporti #x                                 	17	19	6690.42
+4004002	edu packedu pack #x                               	18	45	6669.03
+8003006	exportinameless #x                                	9	46	6630.78
+9003003	exportimaxi #x                                    	9	48	6603.20
+9003003	exportimaxi #x                                    	8	28	6580.50
+7012007	importonameless #x                                	18	6	6548.40
+7015007	scholarnameless #x                                	17	32	6544.80
+3004001	edu packexporti #x                                	19	27	6510.42
+9001003	amalgmaxi #x                                      	6	57	6506.50
+2001001	amalgimporto #x                                   	19	8	6500.76
+9003003	exportimaxi #x                                    	18	46	6443.99
+9016003	corpunivamalg #x                                  	8	47	6398.37
+2004002	edu packimporto #x                                	9	25	6339.69
+9010002	univunivamalg #x                                  	8	21	6302.19
+2004002	edu packimporto #x                                	19	48	6257.70
+5003001	exportischolar #x                                 	17	44	6225.62
+10016002	corpamalgamalg #x                                 	17	45	6222.80
+9004008	edu packmaxi #x                                   	8	46	6201.78
+7015007	scholarnameless #x                                	8	18	6162.84
+4002001	importoedu pack #x                                	18	3	6101.83
+1002001	importoamalg #x                                   	9	1	6058.89
+6008001	namelesscorp #x                                   	17	19	6056.16
+3003002	exportiexporti #x                                 	18	48	6054.30
+7010009	univnameless #x                                   	9	8	6013.24
+9016005	corpunivamalg #x                                  	19	14	5985.00
+7015004	scholarnameless #x                                	19	49	5981.96
+1004001	edu packamalg #x                                  	19	55	5921.48
+7013007	exportinameless #x                                	7	0	5905.55
+1003001	exportiamalg #x                                   	8	27	5891.34
+5004001	edu packscholar #x                                	18	15	5875.10
+8010002	univmaxi #x                                       	17	24	5865.60
+8001007	amalgnameless #x                                  	9	35	5839.47
+3004001	edu packexporti #x                                	8	45	5819.52
+2001001	amalgimporto #x                                   	19	27	5797.79
+8002007	importonameless #x                                	19	30	5762.88
+9014008	edu packunivamalg #x                              	17	53	5754.24
+2003001	exportiimporto #x                                 	8	49	5747.50
+7012007	importonameless #x                                	17	5	5701.60
+3002001	importoexporti #x                                 	19	17	5699.90
+3003001	exportiexporti #x                                 	8	39	5681.76
+7011010	amalgnameless #x                                 	17	56	5639.04
+6008004	namelesscorp #x                                   	8	42	5623.09
+1003001	exportiamalg #x                                   	18	12	5598.45
+8001009	amalgnameless #x                                  	18	33	5576.00
+7009003	maxibrand #x                                      	17	30	5541.85
+10013006	exportiamalgamalg #x                              	9	48	5522.14
+10014005	edu packamalgamalg #x                             	7	45	5490.66
+7013002	exportinameless #x                                	19	13	5482.24
+9003003	exportimaxi #x                                    	19	15	5480.13
+1004001	edu packamalg #x                                  	18	0	5476.76
+3004001	edu packexporti #x                                	8	31	5466.72
+4002001	importoedu pack #x                                	18	42	5460.21
+7010005	univnameless #x                                   	18	48	5435.52
+8008006	namelessnameless #x                               	8	40	5425.28
+1004001	edu packamalg #x                                  	19	56	5418.16
+1003002	exportiamalg #x                                   	18	52	5397.60
+4004002	edu packedu pack #x                               	17	21	5377.26
+2002002	importoimporto #x                                 	8	18	5372.40
+1004001	edu packamalg #x                                  	17	16	5341.47
+10015013	scholaramalgamalg #x                             	6	21	5340.06
+2001001	amalgimporto #x                                   	18	33	5337.60
+6002008	importocorp #x                                    	19	36	5328.99
+3004002	edu packexporti #x                                	18	33	5301.04
+9003003	exportimaxi #x                                    	9	43	5298.43
+7009003	maxibrand #x                                      	9	26	5193.32
+8010001	univmaxi #x                                       	17	48	5172.16
+8001009	amalgnameless #x                                  	17	1	5166.56
+7015007	scholarnameless #x                                	17	22	5156.10
+10005012	scholarunivamalg #x                              	9	16	5114.80
+3003001	exportiexporti #x                                 	18	5	5092.80
+3003001	exportiexporti #x                                 	17	41	5084.64
+9007003	brandmaxi #x                                      	17	23	5045.62
+3002001	importoexporti #x                                 	7	13	5010.32
+3002001	importoexporti #x                                 	17	1	4992.44
+3004001	edu packexporti #x                                	17	11	4980.35
+4004002	edu packedu pack #x                               	8	43	4979.04
+10001007	amalgunivamalg #x                                 	19	43	4955.52
+6007007	brandcorp #x                                      	19	20	4942.00
+9014008	edu packunivamalg #x                              	8	24	4912.02
+5001001	amalgscholar #x                                   	18	54	4879.18
+2001001	amalgimporto #x                                   	18	49	4878.45
+7014010	edu packnameless #x                              	18	34	4873.60
+8001007	amalgnameless #x                                  	19	41	4830.00
+3002001	importoexporti #x                                 	9	44	4827.15
+4004001	edu packedu pack #x                               	7	3	4804.34
+4001001	amalgedu pack #x                                  	19	28	4773.53
+4004002	edu packedu pack #x                               	17	26	4734.60
+6015003	scholarbrand #x                                   	18	34	4665.65
+2004001	edu packimporto #x                                	17	12	4651.86
+5003001	exportischolar #x                                 	18	4	4631.00
+NULL	brandmaxi #x                                     	17	37	4616.78
+4001001	amalgedu pack #x                                  	7	32	4597.00
+4001001	amalgedu pack #x                                  	17	13	4588.56
+6012005	importobrand #x                                   	18	50	4575.60
+3002002	importoexporti #x                                 	19	57	4570.02
+3002001	importoexporti #x                                 	19	39	4528.04
+9016005	corpunivamalg #x                                  	17	16	4526.48
+10008016	namelessunivamalg #x                             	8	28	4506.81
+2003002	exportiimporto #x                                 	17	42	4484.11
+7015007	scholarnameless #x                                	18	19	4469.40
+3003002	exportiexporti #x                                 	6	39	4465.80
+10005012	scholarunivamalg #x                              	18	59	4437.00
+1004001	edu packamalg #x                                  	19	23	4432.40
+2003001	exportiimporto #x                                 	17	19	4421.20
+3004002	edu packexporti #x                                	18	46	4403.07
+10011008	amalgamalgamalg #x                                	7	42	4402.30
+4003001	exportiedu pack #x                                	8	13	4392.08
+8005009	scholarnameless #x                                	18	0	4379.70
+4003001	exportiedu pack #x                                	19	50	4349.40
+7002009	importobrand #x                                   	17	33	4341.12
+7006007	corpbrand #x                                      	19	42	4310.33
+1003001	exportiamalg #x                                   	17	21	4301.91
+10008007	namelessunivamalg #x                              	17	34	4292.47
+8015004	scholarmaxi #x                                    	18	58	4180.80
+2002001	importoimporto #x                                 	18	40	4176.56
+2002002	importoimporto #x                                 	9	54	4173.54
+3001001	amalgexporti #x                                   	9	4	4165.80
+2004001	edu packimporto #x                                	17	43	4164.04
+3002001	importoexporti #x                                 	17	30	4151.64
+10011008	amalgamalgamalg #x                                	18	39	4149.63
+4004002	edu packedu pack #x                               	17	38	4133.66
+8001009	amalgnameless #x                                  	6	43	4122.69
+3002001	importoexporti #x                                 	19	22	4115.04
+3002001	importoexporti #x                                 	19	18	4097.52
+4004002	edu packedu pack #x                               	19	45	4076.16
+1003001	exportiamalg #x                                   	8	31	4075.59
+7015004	scholarnameless #x                                	17	5	4068.78
+6009008	maxicorp #x                                       	17	18	4024.77
+7012007	importonameless #x                                	17	51	4005.99
+7009003	maxibrand #x                                      	18	39	4005.48
+9013005	exportiunivamalg #x                               	8	42	3946.02
+1004001	edu packamalg #x                                  	9	54	3944.02
+1002002	importoamalg #x                                   	19	45	3925.44
+6005004	scholarcorp #x                                    	9	37	3921.38
+8002007	importonameless #x                                	18	45	3842.84
+1002001	importoamalg #x                                   	9	30	3842.56
+2002002	importoimporto #x                                 	8	41	3837.18
+8013007	exportimaxi #x                                    	17	41	3815.46
+10005012	scholarunivamalg #x                              	6	34	3806.53
+6005003	scholarcorp #x                                    	19	28	3796.65
+4004002	edu packedu pack #x                               	18	28	3780.70
+4004001	edu packedu pack #x                               	19	7	3777.74
+3003001	exportiexporti #x                                 	9	41	3754.92
+5001002	amalgscholar #x                                   	8	15	3754.40
+1003001	exportiamalg #x                                   	17	53	3748.70
+6007007	brandcorp #x                                      	18	15	3697.20
+7015004	scholarnameless #x                                	7	52	3666.06
+9014008	edu packunivamalg #x                              	18	52	3645.18
+4003001	exportiedu pack #x                                	18	8	3619.35
+3001001	amalgexporti #x                                   	19	54	3606.90
+6009008	maxicorp #x                                       	18	19	3556.14
+5001002	amalgscholar #x                                   	9	4	3522.24
+2004001	edu packimporto #x                                	17	55	3502.84
+6011001	amalgbrand #x                                     	8	10	3499.45
+3003001	exportiexporti #x                                 	19	55	3482.00
+2003001	exportiimporto #x                                 	17	53	3472.00
+2001001	amalgimporto #x                                   	19	53	3459.52
+2001001	amalgimporto #x                                   	9	49	3457.48
+4001001	amalgedu pack #x                                  	17	39	3433.56
+1004001	edu packamalg #x                                  	19	50	3426.37
+6002008	importocorp #x                                    	18	6	3412.64
+7014010	edu packnameless #x                              	9	18	3412.08
+7012008	importonameless #x                                	18	4	3410.74
+3002001	importoexporti #x                                 	17	55	3405.60
+5002001	importoscholar #x                                 	19	17	3400.38
+2001001	amalgimporto #x                                   	17	57	3397.19
+10014001	edu packamalgamalg #x                             	19	31	3392.50
+7006007	corpbrand #x                                      	17	10	3376.96
+10001003	amalgunivamalg #x                                 	19	41	3352.25
+5001002	amalgscholar #x                                   	19	44	3340.22
+8009005	maxinameless #x                                   	9	24	3338.52
+4002002	importoedu pack #x                                	19	17	3331.64
+6011003	amalgbrand #x                                     	9	50	3319.68
+4003002	exportiedu pack #x                                	19	38	3314.22
+1002001	importoamalg #x                                   	19	46	3308.76
+NULL	brandmaxi #x                                     	17	53	3303.10
+3004001	edu packexporti #x                                	19	53	3300.96
+4001001	amalgedu pack #x                                  	17	45	3295.24
+5002001	importoscholar #x                                 	8	11	3271.52
+9012009	importounivamalg #x                               	17	7	3268.00
+8010001	univmaxi #x                                       	17	31	3267.20
+8010001	univmaxi #x                                       	17	39	3263.40
+9013003	exportiunivamalg #x                               	17	52	3260.32
+10011008	amalgamalgamalg #x                                	17	41	3255.54
+6003004	exporticorp #x                                    	19	25	3249.90
+7013002	exportinameless #x                                	19	39	3219.00
+4003001	exportiedu pack #x                                	7	6	3211.20
+4001001	amalgedu pack #x                                  	17	49	3193.61
+9009009	maximaxi #x                                       	18	41	3176.58
+4002001	importoedu pack #x                                	19	11	3175.90
+2001001	amalgimporto #x                                   	17	47	3157.44
+9014011	edu packunivamalg #x                             	8	58	3137.94
+3002001	importoexporti #x                                 	19	8	3127.98
+7002009	importobrand #x                                   	17	51	3104.96
+10001003	amalgunivamalg #x                                 	17	0	3099.87
+6007007	brandcorp #x                                      	17	2	3096.72
+7014010	edu packnameless #x                              	19	17	3081.42
+10005012	scholarunivamalg #x                              	8	39	3081.11
+5001002	amalgscholar #x                                   	9	43	3079.14
+4002001	importoedu pack #x                                	6	35	3071.00
+4001001	amalgedu pack #x                                  	18	56	3070.71
+7008005	namelessbrand #x                                  	8	5	3070.54
+4003001	exportiedu pack #x                                	17	14	3060.92
+4004002	edu packedu pack #x                               	18	30	3026.87
+6015003	scholarbrand #x                                   	19	16	3021.75
+1002001	importoamalg #x                                   	18	23	3018.06
+1003001	exportiamalg #x                                   	17	12	3016.20
+8008006	namelessnameless #x                               	9	19	3006.85
+8005004	scholarnameless #x                                	8	7	3004.80
+9008003	namelessmaxi #x                                   	8	13	3004.80
+2004002	edu packimporto #x                                	17	25	2991.85
+8005009	scholarnameless #x                                	6	44	2991.08
+7016008	corpnameless #x                                   	8	20	2987.85
+1002001	importoamalg #x                                   	18	38	2983.83
+9016011	corpunivamalg #x                                 	7	43	2923.80
+9016011	corpunivamalg #x                                 	17	56	2910.60
+3003002	exportiexporti #x                                 	17	37	2908.44
+1004001	edu packamalg #x                                  	9	40	2907.30
+7011010	amalgnameless #x                                 	9	39	2899.84
+8012007	importomaxi #x                                    	8	37	2858.82
+4002001	importoedu pack #x                                	18	7	2846.40
+7010005	univnameless #x                                   	19	20	2838.58
+7002002	importobrand #x                                   	9	28	2829.06
+6011001	amalgbrand #x                                     	19	23	2824.58
+10005012	scholarunivamalg #x                              	18	51	2800.24
+3001001	amalgexporti #x                                   	18	58	2767.91
+9003003	exportimaxi #x                                    	8	46	2762.53
+4003001	exportiedu pack #x                                	19	55	2748.48
+7014001	edu packnameless #x                               	17	15	2742.96
+5002001	importoscholar #x                                 	19	14	2740.79
+7006005	corpbrand #x                                      	17	5	2734.45
+5001002	amalgscholar #x                                   	19	24	2708.16
+7015007	scholarnameless #x                                	18	6	2707.96
+3004001	edu packexporti #x                                	19	0	2705.12
+4001001	amalgedu pack #x                                  	18	45	2693.82
+3003001	exportiexporti #x                                 	19	15	2676.38
+7015007	scholarnameless #x                                	17	51	2675.55
+8010001	univmaxi #x                                       	18	15	2671.00
+3004001	edu packexporti #x                                	18	37	2668.55
+9015011	scholarunivamalg #x                              	9	27	2666.22
+8002008	importonameless #x                                	18	53	2666.00
+6009008	maxicorp #x                                       	19	14	2663.54
+3002001	importoexporti #x                                 	19	47	2647.33
+5002001	importoscholar #x                                 	19	37	2639.14
+3003001	exportiexporti #x                                 	9	1	2634.17
+3004001	edu packexporti #x                                	18	51	2628.00
+3002001	importoexporti #x                                 	19	30	2612.00
+2001001	amalgimporto #x                                   	18	53	2610.58
+1003001	exportiamalg #x                                   	17	44	2592.59
+3002001	importoexporti #x                                 	9	47	2580.48
+10009010	maxiunivamalg #x                                 	18	53	2579.82
+6008001	namelesscorp #x                                   	17	29	2579.57
+8012007	importomaxi #x                                    	17	52	2569.00
+2002002	importoimporto #x                                 	18	55	2555.57
+8013007	exportimaxi #x                                    	17	23	2545.20
+7006001	corpbrand #x                                      	17	36	2531.43
+10014001	edu packamalgamalg #x                             	9	26	2527.24
+1003001	exportiamalg #x                                   	17	2	2517.48
+9008003	namelessmaxi #x                                   	8	35	2485.56
+8008006	namelessnameless #x                               	19	13	2475.20
+10003008	exportiunivamalg #x                               	8	6	2453.95
+1001001	amalgamalg #x                                     	17	26	2448.16
+7010009	univnameless #x                                   	18	53	2437.00
+9010002	univunivamalg #x                                  	19	27	2435.80
+4001001	amalgedu pack #x                                  	18	38	2431.24
+4004001	edu packedu pack #x                               	17	33	2426.48
+6012005	importobrand #x                                   	17	41	2420.71
+8013007	exportimaxi #x                                    	19	6	2420.00
+9016005	corpunivamalg #x                                  	18	54	2415.70
+2003001	exportiimporto #x                                 	6	25	2413.20
+2002002	importoimporto #x                                 	18	6	2390.73
+9016008	corpunivamalg #x                                  	17	9	2390.70
+10015013	scholaramalgamalg #x                             	8	28	2368.85
+4001001	amalgedu pack #x                                  	17	20	2367.30
+8008006	namelessnameless #x                               	8	20	2363.24
+9012009	importounivamalg #x                               	8	3	2348.10
+1002002	importoamalg #x                                   	9	44	2336.62
+6007007	brandcorp #x                                      	9	23	2334.24
+1003001	exportiamalg #x                                   	9	54	2325.44
+3004002	edu packexporti #x                                	9	16	2313.30
+4004001	edu packedu pack #x                               	19	17	2307.75
+3004001	edu packexporti #x                                	8	15	2299.77
+7004005	edu packbrand #x                                  	9	34	2294.12
+4003002	exportiedu pack #x                                	18	55	2280.73
+6005004	scholarcorp #x                                    	9	40	2279.25
+8015004	scholarmaxi #x                                    	9	47	2276.40
+4004002	edu packedu pack #x                               	19	3	2271.28
+9015011	scholarunivamalg #x                              	17	54	2269.76
+1002001	importoamalg #x                                   	19	39	2258.08
+6002008	importocorp #x                                    	17	53	2247.20
+9013005	exportiunivamalg #x                               	7	55	2234.00
+1003002	exportiamalg #x                                   	6	59	2231.46
+10016008	corpamalgamalg #x                                 	17	17	2230.20
+4002001	importoedu pack #x                                	19	24	2220.00
+4004002	edu packedu pack #x                               	19	5	2215.50
+9004008	edu packmaxi #x                                   	17	15	2206.40
+4004002	edu packedu pack #x                               	17	52	2204.16
+2002002	importoimporto #x                                 	17	22	2199.69
+2003002	exportiimporto #x                                 	17	14	2197.84
+8004002	edu packnameless #x                               	18	33	2188.04
+3003001	exportiexporti #x                                 	17	58	2187.64
+6005003	scholarcorp #x                                    	17	15	2180.85
+8001009	amalgnameless #x                                  	18	41	2178.21
+2002001	importoimporto #x                                 	17	34	2177.28
+6011003	amalgbrand #x                                     	18	48	2176.72
+6008004	namelesscorp #x                                   	17	17	2162.08
+5004002	edu packscholar #x                                	9	20	2158.40
+6005003	scholarcorp #x                                    	18	7	2152.57
+8010001	univmaxi #x                                       	18	21	2147.42
+9016003	corpunivamalg #x                                  	9	4	2146.05
+9012009	importounivamalg #x                               	9	6	2138.08
+1003002	exportiamalg #x                                   	9	22	2133.04
+4004002	edu packedu pack #x                               	18	15	2126.52
+8006010	corpnameless #x                                  	18	41	2126.29
+7013007	exportinameless #x                                	9	51	2114.10
+10009016	maxiunivamalg #x                                 	9	14	2113.20
+8004002	edu packnameless #x                               	8	32	2111.54
+3003002	exportiexporti #x                                 	17	45	2106.00
+9008003	namelessmaxi #x                                   	17	55	2100.24
+3004001	edu packexporti #x                                	8	7	2099.20
+9016011	corpunivamalg #x                                 	19	3	2094.48
+9007003	brandmaxi #x                                      	17	38	2081.01
+7013007	exportinameless #x                                	8	47	2079.36
+2004001	edu packimporto #x                                	8	45	2069.88
+2002002	importoimporto #x                                 	8	39	2067.52
+7012007	importonameless #x                                	7	30	2056.54
+9001003	amalgmaxi #x                                      	19	35	2046.96
+4002001	importoedu pack #x                                	7	52	2027.30
+9013003	exportiunivamalg #x                               	17	7	2015.02
+4003002	exportiedu pack #x                                	9	26	2007.44
+10009016	maxiunivamalg #x                                 	9	57	1997.28
+10008006	namelessunivamalg #x                              	18	6	1989.49
+5002001	importoscholar #x                                 	19	39	1969.54
+2004001	edu packimporto #x                                	17	47	1968.80
+9003003	exportimaxi #x                                    	9	22	1964.48
+10009016	maxiunivamalg #x                                 	9	47	1942.08
+2003001	exportiimporto #x                                 	9	48	1919.01
+3003001	exportiexporti #x                                 	18	44	1914.96
+10008016	namelessunivamalg #x                             	18	1	1902.04
+4003001	exportiedu pack #x                                	8	27	1899.78
+1003001	exportiamalg #x                                   	7	16	1889.55
+1002001	importoamalg #x                                   	17	40	1889.16
+3001001	amalgexporti #x                                   	8	45	1888.80
+10003008	exportiunivamalg #x                               	17	42	1858.50
+4003002	exportiedu pack #x                                	17	55	1851.30
+3002001	importoexporti #x                                 	18	39	1848.96
+8003006	exportinameless #x                                	8	57	1847.78
+4003001	exportiedu pack #x                                	18	10	1835.50
+9014011	edu packunivamalg #x                             	19	9	1816.08
+6012005	importobrand #x                                   	19	32	1815.98
+7010005	univnameless #x                                   	19	23	1815.60
+6011003	amalgbrand #x                                     	8	50	1808.80
+4001002	amalgedu pack #x                                  	17	10	1800.90
+2002002	importoimporto #x                                 	19	0	1798.16
+7010005	univnameless #x                                   	18	31	1789.30
+7002002	importobrand #x                                   	19	33	1786.47
+4004002	edu packedu pack #x                               	18	14	1786.36
+1002001	importoamalg #x                                   	17	53	1781.00
+8002008	importonameless #x                                	18	21	1770.21
+10014001	edu packamalgamalg #x                             	19	7	1760.54
+8003006	exportinameless #x                                	19	11	1749.55
+9004008	edu packmaxi #x                                   	19	37	1748.25
+4001001	amalgedu pack #x                                  	9	2	1743.00
+3004001	edu packexporti #x                                	18	21	1736.00
+7006001	corpbrand #x                                      	9	59	1722.93
+7014001	edu packnameless #x                               	8	14	1720.33
+1002001	importoamalg #x                                   	19	19	1708.92
+1003001	exportiamalg #x                                   	17	55	1703.00
+1003001	exportiamalg #x                                   	8	17	1697.28
+6005003	scholarcorp #x                                    	19	13	1695.72
+2004002	edu packimporto #x                                	19	26	1693.89
+5003001	exportischolar #x                                 	9	49	1690.08
+7011010	amalgnameless #x                                 	19	13	1688.23
+10014005	edu packamalgamalg #x                             	17	18	1679.70
+1001001	amalgamalg #x                                     	19	26	1678.27
+2003002	exportiimporto #x                                 	18	10	1677.90
+9007003	brandmaxi #x                                      	18	29	1676.16
+7013002	exportinameless #x                                	9	21	1674.33
+9004008	edu packmaxi #x                                   	17	45	1667.47
+7012007	importonameless #x                                	9	9	1662.44
+9010002	univunivamalg #x                                  	8	32	1636.20
+1004001	edu packamalg #x                                  	19	48	1629.36
+6005004	scholarcorp #x                                    	19	28	1623.60
+10014001	edu packamalgamalg #x                             	19	42	1622.88
+7002002	importobrand #x                                   	19	43	1611.50
+6009008	maxicorp #x                                       	8	40	1607.76
+10008016	namelessunivamalg #x                             	8	35	1606.32
+2004002	edu packimporto #x                                	19	41	1600.06
+8013007	exportimaxi #x                                    	19	12	1597.52
+3002001	importoexporti #x                                 	19	5	1597.32
+8013009	exportimaxi #x                                    	8	38	1588.47
+6012005	importobrand #x                                   	18	13	1569.12
+10013006	exportiamalgamalg #x                              	17	4	1565.76
+2001001	amalgimporto #x                                   	18	22	1560.28
+5002001	importoscholar #x                                 	18	23	1558.80
+1001001	amalgamalg #x                                     	19	13	1542.42
+NULL	brandmaxi #x                                     	18	10	1538.24
+10011008	amalgamalgamalg #x                                	9	49	1536.50
+8014006	edu packmaxi #x                                   	9	49	1534.26
+9013003	exportiunivamalg #x                               	7	55	1520.45
+7006007	corpbrand #x                                      	8	43	1517.67
+2003001	exportiimporto #x                                 	9	16	1512.00
+10011008	amalgamalgamalg #x                                	17	46	1506.54
+9016005	corpunivamalg #x                                  	17	54	1505.28
+6012005	importobrand #x                                   	9	47	1503.81
+1002002	importoamalg #x                                   	17	3	1499.84
+9013005	exportiunivamalg #x                               	18	8	1499.40
+2002002	importoimporto #x                                 	17	14	1483.50
+3001001	amalgexporti #x                                   	17	17	1459.44
+9001003	amalgmaxi #x                                      	19	55	1455.30
+9016011	corpunivamalg #x                                 	18	49	1455.08
+9013003	exportiunivamalg #x                               	17	2	1443.52
+9003003	exportimaxi #x                                    	9	20	1442.40
+7004005	edu packbrand #x                                  	17	20	1432.67
+8002008	importonameless #x                                	17	42	1428.68
+9005002	scholarmaxi #x                                    	7	32	1420.70
+10008007	namelessunivamalg #x                              	9	59	1406.40
+7011010	amalgnameless #x                                 	17	19	1404.86
+7015007	scholarnameless #x                                	18	46	1402.40
+2001001	amalgimporto #x                                   	17	23	1397.76
+10001007	amalgunivamalg #x                                 	17	42	1388.49
+1003001	exportiamalg #x                                   	17	57	1388.03
+7006001	corpbrand #x                                      	19	58	1387.75
+10005002	scholarunivamalg #x                               	19	11	1382.68
+9001003	amalgmaxi #x                                      	17	17	1382.16
+3002001	importoexporti #x                                 	6	8	1379.81
+7004005	edu packbrand #x                                  	17	33	1365.52
+3002001	importoexporti #x                                 	18	0	1350.00
+7014003	edu packnameless #x                               	8	21	1349.76
+9016003	corpunivamalg #x                                  	18	38	1340.64
+2004002	edu packimporto #x                                	9	4	1334.41
+4003001	exportiedu pack #x                                	17	55	1326.42
+1002001	importoamalg #x                                   	18	51	1308.82
+6003004	exporticorp #x                                    	17	39	1305.00
+1004001	edu packamalg #x                                  	9	44	1304.91
+10005012	scholarunivamalg #x                              	6	30	1298.70
+10016008	corpamalgamalg #x                                 	19	32	1293.44
+10008006	namelessunivamalg #x                              	8	20	1288.16
+3002001	importoexporti #x                                 	19	25	1284.20
+9013005	exportiunivamalg #x                               	17	50	1269.05
+4003001	exportiedu pack #x                                	9	12	1266.16
+5001002	amalgscholar #x                                   	17	40	1262.40
+1004001	edu packamalg #x                                  	17	34	1257.90
+10008007	namelessunivamalg #x                              	8	13	1246.77
+4001001	amalgedu pack #x                                  	9	56	1233.28
+7015007	scholarnameless #x                                	19	52	1218.28
+9003003	exportimaxi #x                                    	8	39	1211.34
+4003001	exportiedu pack #x                                	18	3	1208.70
+5004001	edu packscholar #x                                	18	16	1197.76
+6003004	exporticorp #x                                    	6	50	1195.92
+6002008	importocorp #x                                    	9	14	1194.08
+3003001	exportiexporti #x                                 	17	21	1192.50
+8001009	amalgnameless #x                                  	19	49	1191.50
+6011003	amalgbrand #x                                     	7	32	1189.59
+6009008	maxicorp #x                                       	18	34	1179.80
+2004002	edu packimporto #x                                	7	19	1178.10
+9013005	exportiunivamalg #x                               	17	40	1173.15
+7006007	corpbrand #x                                      	8	2	1166.66
+2002002	importoimporto #x                                 	19	17	1162.28
+3001001	amalgexporti #x                                   	18	25	1161.54
+2002001	importoimporto #x                                 	9	59	1161.00
+8010001	univmaxi #x                                       	19	55	1160.64
+4001001	amalgedu pack #x                                  	17	51	1153.88
+9014008	edu packunivamalg #x                              	18	4	1153.00
+4001001	amalgedu pack #x                                  	19	48	1151.94
+7011010	amalgnameless #x                                 	8	40	1151.64
+5004001	edu packscholar #x                                	6	57	1148.68
+9004008	edu packmaxi #x                                   	19	3	1140.21
+8001009	amalgnameless #x                                  	17	27	1139.00
+8001007	amalgnameless #x                                  	19	3	1135.20
+9015011	scholarunivamalg #x                              	9	36	1134.00
+9015011	scholarunivamalg #x                              	9	41	1133.76
+7010005	univnameless #x                                   	19	8	1130.29
+1001001	amalgamalg #x                                     	18	54	1129.80
+4001001	amalgedu pack #x                                  	8	44	1129.80
+7016008	corpnameless #x                                   	18	27	1128.50
+4002002	importoedu pack #x                                	17	24	1124.37
+5002002	importoscholar #x                                 	19	20	1124.23
+4002001	importoedu pack #x                                	9	42	1120.98
+9013005	exportiunivamalg #x                               	17	16	1113.20
+7015007	scholarnameless #x                                	17	7	1111.10
+8001009	amalgnameless #x                                  	19	40	1110.42
+2001001	amalgimporto #x                                   	17	19	1108.80
+3004001	edu packexporti #x                                	18	56	1100.47
+4003001	exportiedu pack #x                                	9	44	1100.32
+8003006	exportinameless #x                                	17	54	1098.20
+8001007	amalgnameless #x                                  	19	51	1097.28
+7002002	importobrand #x                                   	17	27	1092.25
+2001001	amalgimporto #x                                   	17	41	1090.98
+2004001	edu packimporto #x                                	17	14	1088.64
+3003001	exportiexporti #x                                 	17	54	1080.64
+9007003	brandmaxi #x                                      	17	2	1079.52
+1003001	exportiamalg #x                                   	9	36	1079.10
+10008007	namelessunivamalg #x                              	17	12	1073.16
+6011001	amalgbrand #x                                     	8	53	1071.84
+7008005	namelessbrand #x                                  	19	55	1071.84
+3003001	exportiexporti #x                                 	8	20	1064.85
+7015004	scholarnameless #x                                	19	59	1059.50
+4004002	edu packedu pack #x                               	18	19	1052.10
+2001001	amalgimporto #x                                   	9	24	1050.07
+9013005	exportiunivamalg #x                               	17	19	1049.20
+4001001	amalgedu pack #x                                  	9	3	1047.60
+1004001	edu packamalg #x                                  	17	29	1041.70
+7009003	maxibrand #x                                      	18	17	1040.98
+10008006	namelessunivamalg #x                              	8	21	1040.04
+9013005	exportiunivamalg #x                               	9	58	1032.46
+5001002	amalgscholar #x                                   	19	7	1029.76
+9016011	corpunivamalg #x                                 	17	51	1029.34
+1002001	importoamalg #x                                   	18	53	1024.10
+5001001	amalgscholar #x                                   	6	19	1012.69
+2002002	importoimporto #x                                 	17	49	1001.90
+6012005	importobrand #x                                   	17	38	1001.60
+2001001	amalgimporto #x                                   	17	36	992.25
+7008005	namelessbrand #x                                  	19	19	991.36
+5004001	edu packscholar #x                                	9	9	989.86
+2004001	edu packimporto #x                                	17	28	984.62
+2003002	exportiimporto #x                                 	19	14	981.54
+3001001	amalgexporti #x                                   	19	24	979.20
+8015004	scholarmaxi #x                                    	9	32	973.50
+9010002	univunivamalg #x                                  	17	40	969.15
+4001001	amalgedu pack #x                                  	9	38	966.92
+3004002	edu packexporti #x                                	17	39	965.25
+2003001	exportiimporto #x                                 	8	10	960.33
+2002002	importoimporto #x                                 	18	51	958.86
+9016008	corpunivamalg #x                                  	17	52	957.18
+7010005	univnameless #x                                   	17	54	948.42
+9009009	maximaxi #x                                       	8	10	939.60
+4003001	exportiedu pack #x                                	17	45	937.30
+2004001	edu packimporto #x                                	17	54	932.94
+9003003	exportimaxi #x                                    	19	36	928.32
+4003002	exportiedu pack #x                                	9	48	924.80
+8002008	importonameless #x                                	19	54	915.38
+6002008	importocorp #x                                    	9	0	914.64
+7006007	corpbrand #x                                      	18	2	914.39
+7014010	edu packnameless #x                              	18	8	910.30
+3004002	edu packexporti #x                                	18	8	907.12
+5001002	amalgscholar #x                                   	6	8	906.84
+2004001	edu packimporto #x                                	9	57	906.78
+8013007	exportimaxi #x                                    	17	54	904.68
+7008005	namelessbrand #x                                  	19	29	901.32
+1002001	importoamalg #x                                   	17	45	898.82
+10013006	exportiamalgamalg #x                              	9	14	896.26
+1003001	exportiamalg #x                                   	18	51	891.87
+9008003	namelessmaxi #x                                   	17	24	890.40
+10008006	namelessunivamalg #x                              	18	37	889.76
+7014010	edu packnameless #x                              	8	15	888.12
+8008006	namelessnameless #x                               	19	3	884.00
+7011010	amalgnameless #x                                 	8	20	883.96
+9013005	exportiunivamalg #x                               	18	52	883.35
+6015001	scholarbrand #x                                   	9	55	881.45
+3004001	edu packexporti #x                                	18	0	876.68
+1003001	exportiamalg #x                                   	19	56	874.50
+9014008	edu packunivamalg #x                              	8	10	873.30
+8003006	exportinameless #x                                	19	44	863.04
+2001001	amalgimporto #x                                   	8	47	862.92
+3003001	exportiexporti #x                                 	17	48	860.52
+7004005	edu packbrand #x                                  	17	0	857.55
+8004002	edu packnameless #x                               	6	59	855.84
+8002008	importonameless #x                                	8	23	855.21
+5001002	amalgscholar #x                                   	7	57	854.70
+6011001	amalgbrand #x                                     	8	7	850.86
+8002007	importonameless #x                                	17	0	843.50
+2001001	amalgimporto #x                                   	19	21	840.63
+3004001	edu packexporti #x                                	17	42	836.43
+2004001	edu packimporto #x                                	17	27	833.91
+4003001	exportiedu pack #x                                	6	35	825.24
+2001001	amalgimporto #x                                   	9	45	822.00
+2004001	edu packimporto #x                                	9	15	817.02
+7012007	importonameless #x                                	19	21	814.45
+6012005	importobrand #x                                   	17	53	805.65
+7011010	amalgnameless #x                                 	18	34	802.80
+4001001	amalgedu pack #x                                  	9	19	799.77
+7014010	edu packnameless #x                              	9	5	796.86
+4003002	exportiedu pack #x                                	18	59	794.58
+1001001	amalgamalg #x                                     	17	49	793.50
+2001002	amalgimporto #x                                   	7	46	789.96
+6005003	scholarcorp #x                                    	17	22	788.72
+9013005	exportiunivamalg #x                               	19	43	788.67
+3004002	edu packexporti #x                                	19	6	788.12
+3002001	importoexporti #x                                 	18	46	787.76
+9016011	corpunivamalg #x                                 	8	49	787.71
+10014001	edu packamalgamalg #x                             	19	37	783.36
+1002002	importoamalg #x                                   	18	18	781.60
+5002001	importoscholar #x                                 	17	40	781.55
+5002002	importoscholar #x                                 	18	34	779.22
+7002002	importobrand #x                                   	18	17	773.22
+8013007	exportimaxi #x                                    	7	56	772.80
+8014006	edu packmaxi #x                                   	9	32	771.12
+8008006	namelessnameless #x                               	9	2	767.36
+5001001	amalgscholar #x                                   	8	51	767.23
+2001001	amalgimporto #x                                   	17	20	765.15
+2001002	amalgimporto #x                                   	18	17	764.40
+9010002	univunivamalg #x                                  	19	2	764.16
+2002002	importoimporto #x                                 	18	46	757.80
+7012008	importonameless #x                                	17	57	751.74
+9003003	exportimaxi #x                                    	19	24	749.84
+1002002	importoamalg #x                                   	9	12	748.60
+1004001	edu packamalg #x                                  	18	24	746.88
+4001001	amalgedu pack #x                                  	19	46	743.38
+2004001	edu packimporto #x                                	17	18	742.14
+9013005	exportiunivamalg #x                               	18	49	739.25
+5004001	edu packscholar #x                                	17	50	736.65
+7008005	namelessbrand #x                                  	18	7	734.16
+9013005	exportiunivamalg #x                               	6	58	733.04
+7004005	edu packbrand #x                                  	19	38	728.64
+2004001	edu packimporto #x                                	19	35	723.36
+5004002	edu packscholar #x                                	19	57	723.18
+4003001	exportiedu pack #x                                	8	5	720.16
+8005004	scholarnameless #x                                	19	1	707.37
+5004002	edu packscholar #x                                	8	20	704.84
+4001002	amalgedu pack #x                                  	9	46	701.68
+5002001	importoscholar #x                                 	9	42	700.00
+2001001	amalgimporto #x                                   	18	45	698.10
+7002009	importobrand #x                                   	17	55	697.62
+1004001	edu packamalg #x                                  	18	8	694.96
+4003001	exportiedu pack #x                                	9	11	693.24
+5004001	edu packscholar #x                                	18	18	693.00
+2004001	edu packimporto #x                                	9	38	689.92
+6008001	namelesscorp #x                                   	18	21	689.22
+8001007	amalgnameless #x                                  	17	23	685.44
+3002001	importoexporti #x                                 	18	53	677.28
+5003001	exportischolar #x                                 	8	32	675.90
+7009003	maxibrand #x                                      	19	5	672.80
+9013003	exportiunivamalg #x                               	18	10	672.29
+9014011	edu packunivamalg #x                             	19	27	671.46
+4004002	edu packedu pack #x                               	7	46	670.32
+10005002	scholarunivamalg #x                               	17	42	669.28
+9003003	exportimaxi #x                                    	9	44	666.39
+6005003	scholarcorp #x                                    	9	26	665.28
+8005004	scholarnameless #x                                	9	14	664.74
+5001002	amalgscholar #x                                   	18	32	657.69
+6011003	amalgbrand #x                                     	9	53	657.57
+7006001	corpbrand #x                                      	8	34	655.68
+6007007	brandcorp #x                                      	17	39	653.90
+8008006	namelessnameless #x                               	6	9	648.45
+2003001	exportiimporto #x                                 	17	54	645.00
+10008016	namelessunivamalg #x                             	9	53	638.52
+4003001	exportiedu pack #x                                	19	4	638.02
+8005004	scholarnameless #x                                	19	2	633.36
+4001002	amalgedu pack #x                                  	19	11	630.36
+1003001	exportiamalg #x                                   	17	4	629.68
+4004002	edu packedu pack #x                               	17	15	617.28
+4004002	edu packedu pack #x                               	18	8	615.36
+9016003	corpunivamalg #x                                  	18	12	609.96
+6007007	brandcorp #x                                      	18	14	607.62
+9016005	corpunivamalg #x                                  	19	54	603.95
+8005009	scholarnameless #x                                	19	1	601.68
+9009009	maximaxi #x                                       	18	47	598.50
+1002001	importoamalg #x                                   	8	7	593.56
+2003002	exportiimporto #x                                 	17	0	591.43
+3002001	importoexporti #x                                 	17	11	587.94
+4003001	exportiedu pack #x                                	8	26	587.36
+3002001	importoexporti #x                                 	19	13	586.08
+10013006	exportiamalgamalg #x                              	18	6	585.12
+9003003	exportimaxi #x                                    	8	4	581.28
+3003001	exportiexporti #x                                 	19	0	577.68
+9007003	brandmaxi #x                                      	17	0	574.52
+3002002	importoexporti #x                                 	9	53	570.78
+10009016	maxiunivamalg #x                                 	19	4	570.57
+1003001	exportiamalg #x                                   	17	59	560.34
+7002002	importobrand #x                                   	8	58	560.28
+7006001	corpbrand #x                                      	7	48	558.90
+NULL	brandmaxi #x                                     	18	7	555.45
+6015003	scholarbrand #x                                   	18	2	555.30
+9003003	exportimaxi #x                                    	19	54	546.96
+9003003	exportimaxi #x                                    	9	51	541.80
+1002001	importoamalg #x                                   	19	29	540.00
+6009008	maxicorp #x                                       	17	8	537.37
+1002001	importoamalg #x                                   	17	50	536.58
+7011010	amalgnameless #x                                 	17	34	536.50
+8009005	maxinameless #x                                   	18	44	535.78
+8009005	maxinameless #x                                   	17	9	535.16
+3002001	importoexporti #x                                 	17	44	530.42
+9007003	brandmaxi #x                                      	8	40	529.72
+2001001	amalgimporto #x                                   	9	54	529.55
+2003002	exportiimporto #x                                 	8	41	527.80
+2002002	importoimporto #x                                 	6	9	527.40
+9001003	amalgmaxi #x                                      	17	22	517.16
+2002002	importoimporto #x                                 	19	32	516.40
+3002002	importoexporti #x                                 	9	58	515.78
+7006001	corpbrand #x                                      	8	37	515.08
+4002001	importoedu pack #x                                	9	0	512.46
+4001001	amalgedu pack #x                                  	18	10	511.88
+9003003	exportimaxi #x                                    	18	59	510.95
+1002002	importoamalg #x                                   	8	10	504.75
+4001001	amalgedu pack #x                                  	9	57	500.32
+2003001	exportiimporto #x                                 	19	55	499.92
+6011003	amalgbrand #x                                     	9	24	498.96
+2001001	amalgimporto #x                                   	19	0	498.18
+2004001	edu packimporto #x                                	9	35	497.09
+3004002	edu packexporti #x                                	18	40	495.95
+4001001	amalgedu pack #x                                  	9	34	494.52
+5004002	edu packscholar #x                                	9	15	491.25
+4002001	importoedu pack #x                                	8	5	490.56
+10009016	maxiunivamalg #x                                 	17	20	486.96
+7014003	edu packnameless #x                               	19	23	486.08
+4001001	amalgedu pack #x                                  	9	32	484.72
+5004001	edu packscholar #x                                	17	24	484.47
+4002001	importoedu pack #x                                	9	9	477.28
+1004001	edu packamalg #x                                  	8	5	476.13
+6011003	amalgbrand #x                                     	19	9	474.96
+8001009	amalgnameless #x                                  	7	52	473.55
+5001002	amalgscholar #x                                   	17	3	470.64
+3003001	exportiexporti #x                                 	18	10	470.25
+6011001	amalgbrand #x                                     	19	2	467.52
+8009005	maxinameless #x                                   	18	37	467.00
+7015004	scholarnameless #x                                	17	24	466.86
+1003001	exportiamalg #x                                   	18	7	462.00
+5001002	amalgscholar #x                                   	18	26	461.89
+2001001	amalgimporto #x                                   	18	39	461.70
+9016005	corpunivamalg #x                                  	19	30	458.64
+3002002	importoexporti #x                                 	17	53	454.08
+7014003	edu packnameless #x                               	8	47	453.84
+2004001	edu packimporto #x                                	8	27	450.78
+8013007	exportimaxi #x                                    	17	50	449.35
+9013005	exportiunivamalg #x                               	9	3	448.74
+3004001	edu packexporti #x                                	8	20	447.76
+3004001	edu packexporti #x                                	9	57	445.60
+2001002	amalgimporto #x                                   	8	16	444.62
+5003001	exportischolar #x                                 	18	23	443.56
+7010009	univnameless #x                                   	17	24	440.88
+7013007	exportinameless #x                                	9	47	438.55
+5002001	importoscholar #x                                 	9	45	432.48
+10016008	corpamalgamalg #x                                 	18	41	430.55
+2004002	edu packimporto #x                                	18	10	430.08
+10009010	maxiunivamalg #x                                 	8	26	429.00
+4003001	exportiedu pack #x                                	18	42	424.02
+NULL	brandmaxi #x                                     	17	16	422.31
+10001007	amalgunivamalg #x                                 	9	36	419.76
+7008005	namelessbrand #x                                  	17	44	419.12
+2004001	edu packimporto #x                                	8	54	417.34
+4004002	edu packedu pack #x                               	17	0	415.48
+4003001	exportiedu pack #x                                	17	9	415.32
+4003002	exportiedu pack #x                                	18	24	415.16
+4001001	amalgedu pack #x                                  	19	54	414.72
+9014011	edu packunivamalg #x                             	8	32	413.76
+8001007	amalgnameless #x                                  	19	43	409.08
+3002001	importoexporti #x                                 	17	28	408.90
+4003001	exportiedu pack #x                                	19	48	406.90
+9013005	exportiunivamalg #x                               	17	9	400.20
+6003004	exporticorp #x                                    	9	11	399.04
+3003002	exportiexporti #x                                 	17	7	396.00
+10007016	brandunivamalg #x                                	9	16	394.80
+3002001	importoexporti #x                                 	18	38	393.38
+4001002	amalgedu pack #x                                  	8	57	392.60
+10015013	scholaramalgamalg #x                             	17	22	390.39
+3002001	importoexporti #x                                 	17	5	390.18
+5004001	edu packscholar #x                                	9	47	384.09
+3001001	amalgexporti #x                                   	19	19	380.60
+4004002	edu packedu pack #x                               	18	48	377.60
+7005006	scholarbrand #x                                   	18	52	377.28
+10003008	exportiunivamalg #x                               	17	8	372.52
+8004002	edu packnameless #x                               	8	44	372.33
+2003001	exportiimporto #x                                 	18	12	371.25
+3002002	importoexporti #x                                 	19	35	370.59
+1002002	importoamalg #x                                   	18	2	370.08
+6002008	importocorp #x                                    	17	4	369.55
+4003001	exportiedu pack #x                                	18	51	362.70
+10009010	maxiunivamalg #x                                 	17	40	360.36
+1002002	importoamalg #x                                   	17	35	354.96
+3004001	edu packexporti #x                                	9	58	354.48
+10003008	exportiunivamalg #x                               	17	59	352.32
+4001001	amalgedu pack #x                                  	18	48	350.46
+4001001	amalgedu pack #x                                  	17	54	349.44
+4001001	amalgedu pack #x                                  	17	14	347.51
+5001001	amalgscholar #x                                   	19	20	346.40
+4003001	exportiedu pack #x                                	9	9	345.80
+4001001	amalgedu pack #x                                  	17	21	344.75
+10014005	edu packamalgamalg #x                             	18	25	344.00
+8008006	namelessnameless #x                               	17	34	342.72
+9010002	univunivamalg #x                                  	18	53	342.35
+8002007	importonameless #x                                	8	17	340.80
+1002001	importoamalg #x                                   	7	36	333.32
+4001001	amalgedu pack #x                                  	19	31	330.45
+4004002	edu packedu pack #x                               	18	44	329.04
+1003001	exportiamalg #x                                   	18	41	326.83
+6008001	namelesscorp #x                                   	19	54	325.65
+7013002	exportinameless #x                                	17	24	324.24
+4004002	edu packedu pack #x                               	8	0	323.50
+3002001	importoexporti #x                                 	18	51	323.46
+10016008	corpamalgamalg #x                                 	18	35	322.00
+10014005	edu packamalgamalg #x                             	17	51	320.76
+1003001	exportiamalg #x                                   	9	56	320.40
+5002002	importoscholar #x                                 	17	54	318.20
+3004001	edu packexporti #x                                	17	37	317.52
+10007016	brandunivamalg #x                                	9	6	315.75
+3003001	exportiexporti #x                                 	17	32	314.72
+8010001	univmaxi #x                                       	19	13	314.72
+4001001	amalgedu pack #x                                  	8	0	312.13
+5004002	edu packscholar #x                                	18	11	310.80
+1002001	importoamalg #x                                   	19	37	309.33
+1002001	importoamalg #x                                   	6	21	308.34
+7006005	corpbrand #x                                      	17	40	307.97
+4001001	amalgedu pack #x                                  	17	18	307.05
+1002001	importoamalg #x                                   	17	24	306.93
+2004001	edu packimporto #x                                	7	51	305.40
+2004002	edu packimporto #x                                	17	39	302.90
+9003003	exportimaxi #x                                    	8	20	298.80
+4001001	amalgedu pack #x                                  	19	32	296.38
+5003001	exportischolar #x                                 	19	27	295.95
+2001001	amalgimporto #x                                   	18	4	294.80
+10013006	exportiamalgamalg #x                              	9	41	290.03
+1002001	importoamalg #x                                   	19	12	281.76
+7013002	exportinameless #x                                	17	0	279.00
+10005012	scholarunivamalg #x                              	9	50	275.52
+8010002	univmaxi #x                                       	7	31	272.25
+10015013	scholaramalgamalg #x                             	7	13	270.28
+1003001	exportiamalg #x                                   	8	47	269.50
+10014005	edu packamalgamalg #x                             	9	30	267.54
+7011010	amalgnameless #x                                 	19	3	265.47
+9003003	exportimaxi #x                                    	18	55	264.77
+9016003	corpunivamalg #x                                  	9	6	264.64
+2002002	importoimporto #x                                 	18	1	264.30
+1004001	edu packamalg #x                                  	8	13	263.55
+4004001	edu packedu pack #x                               	6	8	261.08
+8004002	edu packnameless #x                               	9	56	258.30
+2002002	importoimporto #x                                 	18	36	257.56
+3002001	importoexporti #x                                 	19	20	256.14
+2003002	exportiimporto #x                                 	17	57	255.50
+9010002	univunivamalg #x                                  	19	30	254.52
+6007007	brandcorp #x                                      	9	39	253.23
+9013005	exportiunivamalg #x                               	7	35	251.93
+3003001	exportiexporti #x                                 	9	42	250.80
+10016002	corpamalgamalg #x                                 	17	47	250.80
+7015007	scholarnameless #x                                	18	44	249.20
+3004002	edu packexporti #x                                	18	35	247.28
+8001007	amalgnameless #x                                  	19	32	246.54
+9010002	univunivamalg #x                                  	19	47	246.43
+3002001	importoexporti #x                                 	9	5	244.62
+10016008	corpamalgamalg #x                                 	19	0	243.09
+9009009	maximaxi #x                                       	18	26	242.13
+9008003	namelessmaxi #x                                   	18	9	241.86
+8001009	amalgnameless #x                                  	17	46	241.83
+7013007	exportinameless #x                                	19	19	241.20
+1002002	importoamalg #x                                   	18	49	241.08
+8005004	scholarnameless #x                                	8	16	239.98
+4002001	importoedu pack #x                                	8	45	239.44
+3004001	edu packexporti #x                                	9	26	237.60
+6012005	importobrand #x                                   	18	8	235.32
+10016008	corpamalgamalg #x                                 	17	27	232.70
+4003002	exportiedu pack #x                                	8	8	220.08
+1002002	importoamalg #x                                   	18	29	219.36
+2003001	exportiimporto #x                                 	18	31	218.00
+7014010	edu packnameless #x                              	17	7	217.80
+9005002	scholarmaxi #x                                    	18	41	217.40
+9016003	corpunivamalg #x                                  	17	3	217.00
+NULL	brandmaxi #x                                     	18	27	211.40
+2002002	importoimporto #x                                 	17	42	208.98
+4001002	amalgedu pack #x                                  	18	29	206.64
+3002001	importoexporti #x                                 	18	1	205.55
+6009008	maxicorp #x                                       	18	13	204.48
+4001001	amalgedu pack #x                                  	19	35	203.76
+4003002	exportiedu pack #x                                	9	9	201.41
+1002002	importoamalg #x                                   	6	7	200.80
+9014011	edu packunivamalg #x                             	17	27	200.72
+8006010	corpnameless #x                                  	9	27	199.75
+3004001	edu packexporti #x                                	17	18	199.64
+8005009	scholarnameless #x                                	19	46	198.38
+3004001	edu packexporti #x                                	9	12	197.62
+6002008	importocorp #x                                    	18	24	196.94
+8013009	exportimaxi #x                                    	17	17	196.50
+10007016	brandunivamalg #x                                	9	7	188.63
+4002002	importoedu pack #x                                	17	54	188.25
+4002001	importoedu pack #x                                	19	55	187.29
+8009005	maxinameless #x                                   	17	0	187.09
+4001001	amalgedu pack #x                                  	17	33	186.10
+1003002	exportiamalg #x                                   	7	56	179.82
+4001001	amalgedu pack #x                                  	9	8	178.40
+3002002	importoexporti #x                                 	17	16	177.30
+8003006	exportinameless #x                                	9	15	174.05
+9003003	exportimaxi #x                                    	17	52	172.62
+9014011	edu packunivamalg #x                             	6	12	172.05
+2004001	edu packimporto #x                                	17	37	170.66
+10001003	amalgunivamalg #x                                 	6	57	169.92
+7005006	scholarbrand #x                                   	7	32	168.08
+6005003	scholarcorp #x                                    	18	59	168.00
+6007007	brandcorp #x                                      	17	22	166.74
+2004001	edu packimporto #x                                	17	58	165.69
+9003003	exportimaxi #x                                    	8	13	164.78
+6011003	amalgbrand #x                                     	9	43	163.40
+4004002	edu packedu pack #x                               	7	14	162.92
+2001001	amalgimporto #x                                   	17	0	162.47
+2001002	amalgimporto #x                                   	19	30	161.44
+7013002	exportinameless #x                                	19	29	160.95
+10011008	amalgamalgamalg #x                                	17	13	160.56
+9009009	maximaxi #x                                       	17	35	160.40
+3002001	importoexporti #x                                 	17	57	159.60
+2003001	exportiimporto #x                                 	19	37	159.08
+7002009	importobrand #x                                   	19	5	158.40
+7006007	corpbrand #x                                      	8	38	158.16
+4003001	exportiedu pack #x                                	19	52	157.78
+10009016	maxiunivamalg #x                                 	9	41	157.68
+10015013	scholaramalgamalg #x                             	17	32	157.62
+6015001	scholarbrand #x                                   	17	0	157.28
+6008004	namelesscorp #x                                   	17	42	156.80
+7013002	exportinameless #x                                	9	42	156.75
+10014001	edu packamalgamalg #x                             	18	27	156.18
+2001001	amalgimporto #x                                   	8	5	155.75
+7010005	univnameless #x                                   	9	33	154.64
+7013002	exportinameless #x                                	8	59	154.38
+5001002	amalgscholar #x                                   	18	31	154.35
+10003008	exportiunivamalg #x                               	17	27	154.08
+6012005	importobrand #x                                   	9	26	153.51
+10016002	corpamalgamalg #x                                 	17	46	152.10
+5001002	amalgscholar #x                                   	18	41	151.14
+8001009	amalgnameless #x                                  	17	8	150.66
+6008004	namelesscorp #x                                   	19	44	150.40
+6002008	importocorp #x                                    	17	37	150.22
+8004002	edu packnameless #x                               	19	36	148.23
+8013007	exportimaxi #x                                    	18	33	147.84
+6015003	scholarbrand #x                                   	18	10	147.66
+4002001	importoedu pack #x                                	19	41	146.60
+4003001	exportiedu pack #x                                	17	42	146.32
+5003001	exportischolar #x                                 	17	57	142.56
+4004002	edu packedu pack #x                               	17	4	141.24
+9015011	scholarunivamalg #x                              	7	56	139.16
+3003001	exportiexporti #x                                 	17	25	135.24
+3003001	exportiexporti #x                                 	9	36	133.92
+8004002	edu packnameless #x                               	19	42	133.70
+6008004	namelesscorp #x                                   	19	16	132.81
+7014010	edu packnameless #x                              	17	30	132.46
+2001001	amalgimporto #x                                   	18	6	132.25
+7004005	edu packbrand #x                                  	17	37	132.22
+1003001	exportiamalg #x                                   	19	29	131.58
+8002008	importonameless #x                                	17	2	131.22
+7012008	importonameless #x                                	17	53	130.18
+4001001	amalgedu pack #x                                  	7	51	128.50
+8010001	univmaxi #x                                       	17	17	127.28
+9014008	edu packunivamalg #x                              	9	53	126.70
+8002007	importonameless #x                                	17	21	126.00
+9004008	edu packmaxi #x                                   	9	19	126.00
+7008005	namelessbrand #x                                  	17	52	125.23
+9003003	exportimaxi #x                                    	17	23	122.74
+6015001	scholarbrand #x                                   	17	36	121.92
+3002002	importoexporti #x                                 	9	40	119.60
+NULL	brandmaxi #x                                     	18	20	119.52
+4001001	amalgedu pack #x                                  	19	29	119.38
+4003001	exportiedu pack #x                                	19	14	119.36
+7015004	scholarnameless #x                                	17	15	118.36
+2001001	amalgimporto #x                                   	6	35	116.13
+9016005	corpunivamalg #x                                  	9	58	116.09
+4002001	importoedu pack #x                                	19	19	114.52
+3001001	amalgexporti #x                                   	19	20	114.00
+5001002	amalgscholar #x                                   	19	42	113.04
+3002001	importoexporti #x                                 	6	19	111.52
+3004001	edu packexporti #x                                	17	45	110.88
+1004001	edu packamalg #x                                  	18	22	110.64
+1002001	importoamalg #x                                   	7	52	110.25
+9003003	exportimaxi #x                                    	18	58	109.34
+6005003	scholarcorp #x                                    	9	27	106.75
+4003002	exportiedu pack #x                                	9	40	106.44
+8008006	namelessnameless #x                               	9	39	105.60
+6015003	scholarbrand #x                                   	18	5	104.70
+4002001	importoedu pack #x                                	18	18	104.00
+7009003	maxibrand #x                                      	18	33	102.00
+10014005	edu packamalgamalg #x                             	18	19	99.36
+8005004	scholarnameless #x                                	18	47	98.98
+6008004	namelesscorp #x                                   	17	54	97.40
+2003001	exportiimporto #x                                 	19	0	92.79
+10001007	amalgunivamalg #x                                 	17	1	90.90
+1003001	exportiamalg #x                                   	18	16	90.45
+4003001	exportiedu pack #x                                	17	4	88.83
+10016002	corpamalgamalg #x                                 	9	14	87.36
+10008016	namelessunivamalg #x                             	18	7	87.06
+3002002	importoexporti #x                                 	8	24	86.73
+8006010	corpnameless #x                                  	19	44	85.60
+1003001	exportiamalg #x                                   	8	29	84.15
+2001001	amalgimporto #x                                   	18	25	84.04
+9008003	namelessmaxi #x                                   	9	43	83.94
+4003002	exportiedu pack #x                                	18	2	82.94
+9003003	exportimaxi #x                                    	18	29	81.88
+10003008	exportiunivamalg #x                               	18	51	81.80
+8006010	corpnameless #x                                  	8	15	80.00
+5004001	edu packscholar #x                                	8	28	78.75
+10016002	corpamalgamalg #x                                 	19	8	78.75
+8002007	importonameless #x                                	9	7	78.00
+8001007	amalgnameless #x                                  	9	23	77.04
+3002001	importoexporti #x                                 	6	57	76.38
+10001007	amalgunivamalg #x                                 	8	36	76.20
+8015004	scholarmaxi #x                                    	9	53	74.88
+3003001	exportiexporti #x                                 	9	27	74.52
+9013005	exportiunivamalg #x                               	8	30	74.20
+9016011	corpunivamalg #x                                 	9	14	72.80
+2002002	importoimporto #x                                 	19	35	70.85
+8005004	scholarnameless #x                                	9	18	69.60
+9007003	brandmaxi #x                                      	9	18	68.76
+9016008	corpunivamalg #x                                  	9	0	68.64
+9005002	scholarmaxi #x                                    	9	24	67.89
+2001001	amalgimporto #x                                   	18	23	66.65
+5003001	exportischolar #x                                 	17	1	66.36
+7008005	namelessbrand #x                                  	18	3	66.30
+8003006	exportinameless #x                                	9	8	63.84
+7010005	univnameless #x                                   	19	10	61.88
+4001001	amalgedu pack #x                                  	19	20	60.90
+5002002	importoscholar #x                                 	9	8	58.83
+4001001	amalgedu pack #x                                  	18	27	58.66
+4003002	exportiedu pack #x                                	9	24	58.56
+3002001	importoexporti #x                                 	9	50	58.32
+8005009	scholarnameless #x                                	19	10	58.29
+3003001	exportiexporti #x                                 	9	4	57.89
+7014010	edu packnameless #x                              	9	59	56.76
+7012007	importonameless #x                                	8	53	55.98
+NULL	brandmaxi #x                                     	17	54	55.45
+10007016	brandunivamalg #x                                	17	11	53.72
+7002002	importobrand #x                                   	7	35	53.52
+7010009	univnameless #x                                   	17	40	53.16
+4002001	importoedu pack #x                                	8	49	52.87
+4004002	edu packedu pack #x                               	19	55	50.31
+4001001	amalgedu pack #x                                  	17	46	50.12
+5002001	importoscholar #x                                 	8	10	49.80
+9016005	corpunivamalg #x                                  	19	28	49.50
+4002001	importoedu pack #x                                	9	22	49.44
+5002002	importoscholar #x                                 	9	38	47.88
+2003001	exportiimporto #x                                 	9	52	47.46
+7016008	corpnameless #x                                   	19	40	47.32
+4004002	edu packedu pack #x                               	18	34	47.11
+10008006	namelessunivamalg #x                              	9	37	45.05
+10001003	amalgunivamalg #x                                 	9	14	44.55
+2002002	importoimporto #x                                 	8	9	44.32
+2002002	importoimporto #x                                 	18	10	41.16
+7006007	corpbrand #x                                      	18	6	40.32
+1003001	exportiamalg #x                                   	8	39	38.22
+10001003	amalgunivamalg #x                                 	17	11	37.84
+7012008	importonameless #x                                	17	1	37.44
+1002001	importoamalg #x                                   	19	15	36.74
+3003001	exportiexporti #x                                 	18	22	34.76
+4001001	amalgedu pack #x                                  	8	32	32.23
+9013005	exportiunivamalg #x                               	19	33	29.85
+1002001	importoamalg #x                                   	19	57	29.52
+10016002	corpamalgamalg #x                                 	9	18	25.47
+6007007	brandcorp #x                                      	9	11	24.77
+1003002	exportiamalg #x                                   	19	43	24.43
+9004008	edu packmaxi #x                                   	18	57	24.32
+10009016	maxiunivamalg #x                                 	19	19	22.66
+10008006	namelessunivamalg #x                              	9	3	22.14
+3003001	exportiexporti #x                                 	19	25	21.40
+2001001	amalgimporto #x                                   	8	58	19.88
+3004001	edu packexporti #x                                	17	29	17.76
+6008001	namelesscorp #x                                   	17	58	17.00
+9009009	maximaxi #x                                       	9	38	16.80
+3003001	exportiexporti #x                                 	17	52	15.96
+6002008	importocorp #x                                    	9	41	15.78
+2004002	edu packimporto #x                                	8	35	15.21
+6011001	amalgbrand #x                                     	19	40	14.55
+10008016	namelessunivamalg #x                             	8	54	14.49
+5004001	edu packscholar #x                                	6	4	13.64
+2003002	exportiimporto #x                                 	8	47	13.04
+2001001	amalgimporto #x                                   	19	59	12.84
+7013002	exportinameless #x                                	17	1	12.54
+10014001	edu packamalgamalg #x                             	18	49	12.50
+8015004	scholarmaxi #x                                    	9	0	10.06
+9004008	edu packmaxi #x                                   	9	31	9.80
+5004002	edu packscholar #x                                	9	46	8.19
+4002002	importoedu pack #x                                	8	58	7.79
+4001001	amalgedu pack #x                                  	19	59	7.38
+4004002	edu packedu pack #x                               	9	18	7.33
+2002002	importoimporto #x                                 	19	14	6.54
+5004002	edu packscholar #x                                	19	19	6.50
+6008001	namelesscorp #x                                   	19	32	5.75
+7010009	univnameless #x                                   	17	29	4.25
+6015003	scholarbrand #x                                   	18	44	3.94
+7014001	edu packnameless #x                               	18	31	3.08
+6015003	scholarbrand #x                                   	17	18	3.04
+8004002	edu packnameless #x                               	9	57	0.24
+2001001	amalgimporto #x                                   	17	34	0.00
+2004002	edu packimporto #x                                	19	40	0.00
+4004002	edu packedu pack #x                               	17	50	0.00
+6009008	maxicorp #x                                       	17	19	0.00
+6015003	scholarbrand #x                                   	9	51	0.00
+8002007	importonameless #x                                	19	40	0.00
+9003003	exportimaxi #x                                    	19	33	0.00
+9015011	scholarunivamalg #x                              	18	10	0.00
+1001001	amalgamalg #x                                     	19	20	NULL
+4003001	exportiedu pack #x                                	9	42	NULL
+5002001	importoscholar #x                                 	9	8	NULL
+5003001	exportischolar #x                                 	17	43	NULL
+8005009	scholarnameless #x                                	9	58	NULL
+8010002	univmaxi #x                                       	9	53	NULL
+9013003	exportiunivamalg #x                               	9	35	NULL
+10003008	exportiunivamalg #x                               	19	30	NULL
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q72.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q72.sql.out
new file mode 100644
index 0000000000000..1c0b50b785491
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q72.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_item_desc:string,w_warehouse_name:string,d_week_seq:int,no_promo:bigint,promo:bigint,total_cnt:bigint>
+-- !query output
+NULL	Just good amou	5166	2	2	2
+Actions see of course informal phrases. Markedly right men buy honest, additional stations. In order imaginative factors used to move human thanks. Centres shall catch altogether succe	Significantly	5189	2	2	2
+Children shall write only systems. Again outdoor c	Operations	5215	2	2	2
+Clouds may compensate about religious man	Matches produce	5216	2	2	2
+Critical hours might stress above married, sufficient thousands. Poets shall die medical, ameri	Operations	5217	2	2	2
+Customers find for a dogs. Main, following members must live closely because of the interests. Children could imagine more. Innocent, social forces will welcome. I	Significantly	5210	2	2	2
+Delicate readers gain too able officers. Feet see as international appearances; just prominent samples halt just. Substantia	Just good amou	5197	2	2	2
+Fields might need only however new lengths; explicit, impossible parents cut early able items. Details specify particularly 	Significantly	5208	2	2	2
+Guests agree around trying, young costs. Here annual banks appeas	Just good amou	5204	2	2	2
+Guests agree around trying, young costs. Here annual banks appeas	Significantly	5204	2	2	2
+Here extra efforts ensure eyes; merely little periods will not loosen home past a boys. Just local aspects must reclaim. Standard qualities might not roll today. Military, national clothes must go wid	Matches produce	5215	2	2	2
+More than familiar lives survive independent reports. Sites must find clearly regulations. Together honest savings refuse so other fingers; british tables	Selective,	5215	2	2	2
+Mysterious p	Matches produce	5218	2	2	2
+Please separate charges point spiritual, new areas. Angry, able units should try certainly in a accounts. Years retain alternatively. Certain, constant women spend really vital rights. Medical, round 	Matches produce	5210	2	2	2
+Scottish, forward years could interrupt yesterday pure scienc	Just good amou	5214	2	2	2
+Sectors might not know properly. Large, electric workers used to drop even as ca	Matches produce	5171	2	2	2
+Sma	Matches produce	5204	2	2	2
+Social universities get. Easier yellow results question above basic, direct roots; individual, respective 	Just good amou	5200	2	2	2
+Upper, usual agencies would not evaluate able, simple faces. Poor lights g	Selective,	5197	2	2	2
+NULL	Matches produce	5177	1	1	1
+NULL	Operations	5203	1	1	1
+NULL	Significantly	5171	1	1	1
+NULL	Significantly	5203	1	1	1
+NULL	Significantly	5213	1	1	1
+A bit liable flowers change also writings. Currently soviet ministers come. Hotels telephone before aggressive, economic eyes. Blue changes improve. Overal	Just good amou	5201	1	1	1
+A bit liable flowers change also writings. Currently soviet ministers come. Hotels telephone before aggressive, economic eyes. Blue changes improve. Overal	Matches produce	5201	1	1	1
+A lot young materials remain below from a rises. 	Matches produce	5190	1	1	1
+A lot young materials remain below from a rises. 	Operations	5190	1	1	1
+A lot young materials remain below from a rises. 	Selective,	5190	1	1	1
+Aback british songs meet. Chief jobs k	Selective,	5178	1	1	1
+Able differences may not spoil particularly then civil ways. Less unusual plants swallow for example in a pp.. Eastern, typical children start to a councils. Exciting cells must meet new, huge me	Matches produce	5178	1	1	1
+Able differences may not spoil particularly then civil ways. Less unusual plants swallow for example in a pp.. Eastern, typical children start to a councils. Exciting cells must meet new, huge me	Significantly	5167	1	1	1
+Able issues bother however political services. French teachers will act voices. Pregnant, existing cases make by th	Just good amou	5208	1	1	1
+Able, available problems apply even to a bodies. Patients so	Selective,	5216	1	1	1
+Able, continuous figures see with a patients. Men go more open notes. Different engineers can display. Even strong fortunes cannot put at least general fans; reliable talk	Operations	5216	1	1	1
+Able, potential products should	Just good amou	5208	1	1	1
+Able, potential products should	Operations	5208	1	1	1
+About careful activities hear level cases. However satisfactory reports feel as words. More bad things preserve now poor tories; only strong tools intervene canadian waters. Blin	Just good amou	5193	1	1	1
+About right clothes must get thoughtfully to a cases. Eastern improvements 	Just good amou	5197	1	1	1
+About statistical blocks shall point so brothers. Even new affairs spend hopefully even old contexts. Possible officers wait absolutely with	Operations	5168	1	1	1
+Above, new groups will not like much local bodies. However traditional sessions can walk slowly big, young aspects. Quite close companies ought to take in a rules. Leaders must not like of cou	Matches produce	5187	1	1	1
+Above, new groups will not like much local bodies. However traditional sessions can walk slowly big, young aspects. Quite close companies ought to take in a rules. Leaders must not like of cou	Operations	5187	1	1	1
+Absolutely b	Matches produce	5211	1	1	1
+Accidentally wrong communities look more goods. Rural matters recognize. Large, new days go hap	Matches produce	5190	1	1	1
+Accidents can include below other, marginal plans. Comparable, welsh exceptions argue most as usual physical claims. Certain months may smell far from in a cases. Active seconds used to restore t	Significantly	5213	1	1	1
+Accounts return into a colleagues	Operations	5218	1	1	1
+Accurate others could not enable raw goods. Usually early drawings choose originally into a boys. So capable students place 	Selective,	5188	1	1	1
+Actions see of course informal phrases. Markedly right men buy honest, additional stations. In order imaginative factors used to move human thanks. Centres shall catch altogether succe	Just good amou	5209	1	1	1
+Active studies state away already large shelves. Extremely international appli	Significantly	5208	1	1	1
+Active, mi	Selective,	5192	1	1	1
+Actual things should prevent there responsible schemes. Others go all undoubtedly increasing things. Small, full samples analys	Operations	5190	1	1	1
+Added activities leave hands. Nevertheless individual moments could repre	Operations	5198	1	1	1
+Added activities leave hands. Nevertheless individual moments could repre	Selective,	5174	1	1	1
+Added activities leave hands. Nevertheless individual moments could repre	Selective,	5198	1	1	1
+Additional officers shall not apply so poin	Just good amou	5217	1	1	1
+Additional, secondary movements will hurt theoretical, major seconds. Families hear well possible, magnetic minutes. Earlier financial women would s	Operations	5194	1	1	1
+Addresses retain once more applicable events. Following blocks follow for a develo	Operations	5197	1	1	1
+Adequate, true insects clear similar payments. Front rela	Just good amou	5177	1	1	1
+Adults might not surrender doubtful, upper industries; earnings insist m	Operations	5188	1	1	1
+Afraid questions ta	Selective,	5187	1	1	1
+Afraid women shall correct so only women. Red, severe friends repay suddenly out of a elements. Very rigid complaints want however great, little years. Black, itali	Selective,	5218	1	1	1
+African, good purposes would determine quite visitors. Sources can make then royal jobs; still full sciences should concentrate no longer elections. Fair applicants talk there under a c	Just good amou	5214	1	1	1
+African, good purposes would determine quite visitors. Sources can make then royal jobs; still full sciences should concentrate no longer elections. Fair applicants talk there under a c	Selective,	5168	1	1	1
+African, good purposes would determine quite visitors. Sources can make then royal jobs; still full sciences should concentrate no longer elections. Fair applicants talk there under a c	Selective,	5214	1	1	1
+Again british shareholders see shares. American lives ought to establish horses. Then ideal conservatives might charge even nec	Significantly	5186	1	1	1
+Again economic objections spend suddenly urgently worried boats. Pupils talk yet nonethele	Just good amou	5193	1	1	1
+Again major troubles create new, other children. Fair interactions may telephone	Selective,	5199	1	1	1
+Again remote branches should help; processes may s	Just good amou	5191	1	1	1
+Again scottish accidents would destroy italian places; please careful services pick sometimes overall men; immensely extra sets move optimistic, substantial actors. Human, likely reports	Just good amou	5193	1	1	1
+Again specialist words transform still as perfect forces; expensive, like businesses might want u	Just good amou	5191	1	1	1
+Agencies should need likely recommendations. Active, fresh stars shall get just young fragments. Personal 	Significantly	5216	1	1	1
+Ago natural taxes could protect rats. More local days shall tend closely. Proteins may intervene very perfect men. Procedures make expens	Just good amou	5214	1	1	1
+Ago sexual courts may attract. Important, alone observations expect. New, available ways represent years. Excell	Operations	5203	1	1	1
+Agricultural, important boys know willingly after the interests. S	Significantly	5213	1	1	1
+Agricultural, social tiles would come tragic, various buildings. Good employees shall result high wet plants. Only single contacts support already. Priests would not say unreasonably. Upstairs good 	Matches produce	5191	1	1	1
+Ahead new columns s	Operations	5187	1	1	1
+Ahead young classes should take more central late conservatives. Formal, common details used to think	Significantly	5180	1	1	1
+All capital bacteria make jobs. Again appropriate eyes may not leave others. There fixed ways	Matches produce	5202	1	1	1
+All direct guns would look cool sure sophisticated bonds; irish sequences look just also local years. Almost close things can look. Build	Matches produce	5178	1	1	1
+Allowances might lay at best children. Academic sections burst hot times. Short-term, warm goods	Selective,	5177	1	1	1
+Almost busy threats go together recent sides; still tired wines shall not admit on a	Matches produce	5209	1	1	1
+Almost good hours should not make. Fully appropriate cases may stop for a terms. Legal compl	Matches produce	5188	1	1	1
+Alone bills protect adults. Demands make in a gains. Students train harshly. Ashamed periods choose just general, free places. Senses would finish quite slow, gla	Just good amou	5213	1	1	1
+Alone friends would not know else armies. Services recover too extreme, fiscal machines	Operations	5205	1	1	1
+Alone new copies discuss to a dates; all black machines get just just royal years. For example free weeks underestimate accurately individual mountains. National, delicious	Significantly	5185	1	1	1
+Alone responsibilities used to argue all. Eventual, past reasons lead electrical, absent years. Again big sessions embrace about later familiar hundreds. Certain parts cannot assist desperately good m	Just good amou	5208	1	1	1
+Alone responsibilities used to argue all. Eventual, past reasons lead electrical, absent years. Again big sessions embrace about later familiar hundreds. Certain parts cannot assist desperately good m	Matches produce	5208	1	1	1
+Alone sole services keep only; stairs shall eliminate for the woods. Methods must need yet. Other students can	Just good amou	5203	1	1	1
+Alone, fiscal attitudes will see subsequently. Arrangements used to prefe	Selective,	5207	1	1	1
+Alone, fortunate minutes can put particularly out of a consequences. Darling costs run already in a laws. Molecules discover. Temporary, political ty	Matches produce	5200	1	1	1
+Already bright poems evaluate somewhere problems; regulations will not conceal now delighted objects; false thoughts follow then. Months should not work only outside times. Fingers prove worker	Operations	5170	1	1	1
+Already bright poems evaluate somewhere problems; regulations will not conceal now delighted objects; false thoughts follow then. Months should not work only outside times. Fingers prove worker	Significantly	5215	1	1	1
+Also alone patients seem also for the connections. Significant flowers prove finally in a opportunities. Closely international women might avoid tomorrow hidden, following 	Operations	5209	1	1	1
+Also difficult acres researc	Matches produce	5214	1	1	1
+Also dominant elections call only more conventional films. Magazines shall not hand really soon opening hundreds. Equally particula	Significantly	5216	1	1	1
+Also eastern matters should not enable now irish, 	Matches produce	5215	1	1	1
+Also good forms	Just good amou	5215	1	1	1
+Also interesting sides acknowledge basically. Tonight low employees run thus. More sympathetic results watch rarely. Even severe arrangements study very	Matches produce	5212	1	1	1
+Also invisible shoes sell whit	Just good amou	5201	1	1	1
+Also invisible shoes sell whit	Selective,	5193	1	1	1
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q73.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q73.sql.out
new file mode 100644
index 0000000000000..615ac241c0e66
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q73.sql.out
@@ -0,0 +1,9 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<c_last_name:string,c_first_name:string,c_salutation:string,c_preferred_cust_flag:string,ss_ticket_number:int,cnt:bigint>
+-- !query output
+Ransom                        	Thomas              	Sir       	N	872	5
+Valle                         	Chandra             	Dr.       	N	45338	5
+Richardson                    	Harry               	Mr.       	Y	85055	5
+Sauer                         	Larry               	Mr.       	N	215795	5
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q74.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q74.sql.out
new file mode 100644
index 0000000000000..2e3972217b820
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q74.sql.out
@@ -0,0 +1,94 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<customer_id:string,customer_first_name:string,customer_last_name:string>
+-- !query output
+AAAAAAAAAAECBAAA	Frank               	Wenzel                        
+AAAAAAAAABGKAAAA	Jonna               	King                          
+AAAAAAAAAFAGAAAA	Robert              	Chang                         
+AAAAAAAAAFBNAAAA	Robert              	Baines                        
+AAAAAAAAAGLPAAAA	Charlene            	Marcus                        
+AAAAAAAAAHKEAAAA	William             	Stafford                      
+AAAAAAAABAAGAAAA	Luis                	James                         
+AAAAAAAABBEAAAAA	Jason               	Gallegos                      
+AAAAAAAABGMHBAAA	Michael             	Gillespie                     
+AAAAAAAABIABAAAA	Letha               	Stone                         
+AAAAAAAABIIHAAAA	Charles             	Quarles                       
+AAAAAAAABILCAAAA	Theresa             	Mullins                       
+AAAAAAAABJEDBAAA	Arthur              	Bryan                         
+AAAAAAAACEMIAAAA	James               	Hernandez                     
+AAAAAAAACGLDAAAA	Angelo              	Sloan                         
+AAAAAAAACOEHBAAA	Christine           	Gonzalez                      
+AAAAAAAACPDFBAAA	Cheryl              	Barry                         
+AAAAAAAADFJBBAAA	Patrick             	Jones                         
+AAAAAAAADHCBAAAA	Therese             	Perez                         
+AAAAAAAADHNHBAAA	Patrick             	Cooper                        
+AAAAAAAADKMBAAAA	Donald              	Nelson                        
+AAAAAAAAEBFHAAAA	Esther              	Merrill                       
+AAAAAAAAEFCEBAAA	Cornelius           	Martino                       
+AAAAAAAAEIAHAAAA	Henry               	Desantis                      
+AAAAAAAAEIPIAAAA	Luke                	Rios                          
+AAAAAAAAFAIEAAAA	Betty               	Gipson                        
+AAAAAAAAFDIMAAAA	Stephanie           	Cowan                         
+AAAAAAAAFGMHBAAA	Donald              	Coleman                       
+AAAAAAAAFGNEAAAA	Andrew              	Silva                         
+AAAAAAAAFHNDAAAA	Virgil              	Mcdonald                      
+AAAAAAAAFMOKAAAA	Harry               	Brown                         
+AAAAAAAAFMPPAAAA	Manuel              	Bryant                        
+AAAAAAAAFOEDAAAA	Lori                	Erwin                         
+AAAAAAAAGCGIAAAA	Mae                 	Williams                      
+AAAAAAAAGEKLAAAA	Jerilyn             	Walker                        
+AAAAAAAAGGMHAAAA	Julia               	Fisher                        
+AAAAAAAAGLDMAAAA	Alex                	Norris                        
+AAAAAAAAGMFHAAAA	Bruce               	Howe                          
+AAAAAAAAGMGEBAAA	Tamika              	Potts                         
+AAAAAAAAHEIFBAAA	NULL	Jones                         
+AAAAAAAAHEPFBAAA	Kathryn             	Kinney                        
+AAAAAAAAHGKLAAAA	Arthur              	Christensen                   
+AAAAAAAAHIEIAAAA	William             	Roberts                       
+AAAAAAAAHLEAAAAA	Geneva              	Sims                          
+AAAAAAAAHLJCAAAA	Marlene             	Grover                        
+AAAAAAAAIANDAAAA	Elva                	Wade                          
+AAAAAAAAIBBFBAAA	James               	Compton                       
+AAAAAAAAIBHHBAAA	Vennie              	Loya                          
+AAAAAAAAIBJDBAAA	Dean                	Velez                         
+AAAAAAAAILLJAAAA	Billy               	Ortiz                         
+AAAAAAAAIODCBAAA	Jennifer            	Crane                         
+AAAAAAAAIPGJAAAA	Michael             	NULL
+AAAAAAAAIPKJAAAA	Charles             	Jones                         
+AAAAAAAAJADIAAAA	Margaret            	Roberts                       
+AAAAAAAAJBELAAAA	Sean                	Busby                         
+AAAAAAAAJCNBBAAA	Johnnie             	Cox                           
+AAAAAAAAJDEFAAAA	Loretta             	Serrano                       
+AAAAAAAAJGDLAAAA	Fredrick            	Davis                         
+AAAAAAAAJHGFAAAA	Pamela              	Gannon                        
+AAAAAAAAJIAHAAAA	Shawna              	Delgado                       
+AAAAAAAAJINGAAAA	Ella                	Moore                         
+AAAAAAAAJMIDAAAA	Sally               	Thurman                       
+AAAAAAAAKAKPAAAA	Carolann            	Royer                         
+AAAAAAAAKMHPAAAA	Robert              	Jones                         
+AAAAAAAAKNMEBAAA	Amber               	Gonzalez                      
+AAAAAAAALEAHBAAA	Eddie               	Pena                          
+AAAAAAAALMAJAAAA	Ileen               	Linn                          
+AAAAAAAALMGGBAAA	Dedra               	Rainey                        
+AAAAAAAALNLABAAA	Janie               	Garcia                        
+AAAAAAAALPHGBAAA	Dorothy             	Heller                        
+AAAAAAAAMFMKAAAA	John                	Sanders                       
+AAAAAAAAMHOLAAAA	Terri               	Cook                          
+AAAAAAAAMJFAAAAA	Marcus              	Espinal                       
+AAAAAAAAMLOEAAAA	Miguel              	Jackson                       
+AAAAAAAANBECBAAA	Michael             	Lombardi                      
+AAAAAAAANKBBAAAA	Diann               	Saunders                      
+AAAAAAAAOCDCAAAA	Armando             	Jackson                       
+AAAAAAAAOCFLAAAA	Bill                	Freeman                       
+AAAAAAAAOEDIAAAA	Alexander           	Rich                          
+AAAAAAAAOJBPAAAA	Jonathan            	Mcbride                       
+AAAAAAAAOMOKAAAA	Laurette            	Gary                          
+AAAAAAAAOOKKAAAA	Deborah             	Early                         
+AAAAAAAAOPMDAAAA	Peggy               	Smith                         
+AAAAAAAAPAEEBAAA	Audria              	Mattson                       
+AAAAAAAAPBIGBAAA	Susie               	Zavala                        
+AAAAAAAAPEFLAAAA	David               	Martinez                      
+AAAAAAAAPFKDAAAA	Linda               	Simmons                       
+AAAAAAAAPIGBBAAA	Charles             	Welch                         
+AAAAAAAAPNMGAAAA	Christine           	Olds
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q76.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q76.sql.out
new file mode 100644
index 0000000000000..c2b51267a1973
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q76.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<channel:string,col_name:int,d_year:int,d_qoy:int,i_category:string,sales_cnt:bigint,sales_amt:decimal(17,2)>
+-- !query output
+catalog	NULL	1998	1	Books                                             	8	9808.66
+catalog	NULL	1998	1	Children                                          	11	14843.15
+catalog	NULL	1998	1	Electronics                                       	12	34659.85
+catalog	NULL	1998	1	Home                                              	4	1149.06
+catalog	NULL	1998	1	Jewelry                                           	7	16037.86
+catalog	NULL	1998	1	Men                                               	17	30340.73
+catalog	NULL	1998	1	Music                                             	14	20075.63
+catalog	NULL	1998	1	Shoes                                             	10	15432.10
+catalog	NULL	1998	1	Sports                                            	4	2660.73
+catalog	NULL	1998	1	Women                                             	14	32287.07
+catalog	NULL	1998	2	Books                                             	8	27523.09
+catalog	NULL	1998	2	Children                                          	11	33106.16
+catalog	NULL	1998	2	Electronics                                       	16	24105.87
+catalog	NULL	1998	2	Home                                              	14	14282.44
+catalog	NULL	1998	2	Jewelry                                           	10	6120.92
+catalog	NULL	1998	2	Men                                               	13	25776.26
+catalog	NULL	1998	2	Music                                             	11	11113.26
+catalog	NULL	1998	2	Shoes                                             	10	5887.74
+catalog	NULL	1998	2	Sports                                            	9	15135.88
+catalog	NULL	1998	2	Women                                             	7	2373.08
+catalog	NULL	1998	3	Books                                             	13	5452.76
+catalog	NULL	1998	3	Children                                          	18	32298.89
+catalog	NULL	1998	3	Electronics                                       	21	49769.09
+catalog	NULL	1998	3	Home                                              	17	14007.10
+catalog	NULL	1998	3	Jewelry                                           	23	25680.05
+catalog	NULL	1998	3	Men                                               	20	45022.03
+catalog	NULL	1998	3	Music                                             	14	20882.65
+catalog	NULL	1998	3	Shoes                                             	19	11602.79
+catalog	NULL	1998	3	Sports                                            	18	17526.27
+catalog	NULL	1998	3	Women                                             	17	20129.34
+catalog	NULL	1998	4	NULL	2	607.00
+catalog	NULL	1998	4	Books                                             	33	57441.77
+catalog	NULL	1998	4	Children                                          	35	53915.78
+catalog	NULL	1998	4	Electronics                                       	39	40769.87
+catalog	NULL	1998	4	Home                                              	28	40240.88
+catalog	NULL	1998	4	Jewelry                                           	38	58972.54
+catalog	NULL	1998	4	Men                                               	30	39765.00
+catalog	NULL	1998	4	Music                                             	32	70931.52
+catalog	NULL	1998	4	Shoes                                             	33	65217.73
+catalog	NULL	1998	4	Sports                                            	31	45867.98
+catalog	NULL	1998	4	Women                                             	32	33059.29
+catalog	NULL	1999	1	Books                                             	11	13083.10
+catalog	NULL	1999	1	Children                                          	10	6952.44
+catalog	NULL	1999	1	Electronics                                       	10	3112.96
+catalog	NULL	1999	1	Home                                              	9	4768.55
+catalog	NULL	1999	1	Jewelry                                           	6	7143.30
+catalog	NULL	1999	1	Men                                               	10	21187.82
+catalog	NULL	1999	1	Music                                             	6	265.20
+catalog	NULL	1999	1	Shoes                                             	19	24625.11
+catalog	NULL	1999	1	Sports                                            	12	15481.90
+catalog	NULL	1999	1	Women                                             	15	25291.30
+catalog	NULL	1999	2	Books                                             	17	14284.73
+catalog	NULL	1999	2	Children                                          	8	4453.70
+catalog	NULL	1999	2	Electronics                                       	14	7808.53
+catalog	NULL	1999	2	Home                                              	7	7418.42
+catalog	NULL	1999	2	Jewelry                                           	13	11931.85
+catalog	NULL	1999	2	Men                                               	8	1730.92
+catalog	NULL	1999	2	Music                                             	11	14585.34
+catalog	NULL	1999	2	Shoes                                             	10	16267.47
+catalog	NULL	1999	2	Sports                                            	7	1058.62
+catalog	NULL	1999	2	Women                                             	17	17451.00
+catalog	NULL	1999	3	Books                                             	21	34033.74
+catalog	NULL	1999	3	Children                                          	15	28755.03
+catalog	NULL	1999	3	Electronics                                       	24	41518.93
+catalog	NULL	1999	3	Home                                              	20	39919.72
+catalog	NULL	1999	3	Jewelry                                           	22	15372.42
+catalog	NULL	1999	3	Men                                               	26	49692.31
+catalog	NULL	1999	3	Music                                             	23	6840.77
+catalog	NULL	1999	3	Shoes                                             	19	21542.78
+catalog	NULL	1999	3	Sports                                            	17	15957.19
+catalog	NULL	1999	3	Women                                             	27	30416.44
+catalog	NULL	1999	4	NULL	1	9077.75
+catalog	NULL	1999	4	Books                                             	36	60721.76
+catalog	NULL	1999	4	Children                                          	22	21641.02
+catalog	NULL	1999	4	Electronics                                       	37	30157.36
+catalog	NULL	1999	4	Home                                              	34	42467.56
+catalog	NULL	1999	4	Jewelry                                           	35	38566.86
+catalog	NULL	1999	4	Men                                               	26	28008.47
+catalog	NULL	1999	4	Music                                             	26	31237.65
+catalog	NULL	1999	4	Shoes                                             	27	45175.99
+catalog	NULL	1999	4	Sports                                            	39	68801.24
+catalog	NULL	1999	4	Women                                             	38	55600.11
+catalog	NULL	2000	1	Books                                             	12	14558.48
+catalog	NULL	2000	1	Children                                          	11	10218.22
+catalog	NULL	2000	1	Electronics                                       	6	13621.14
+catalog	NULL	2000	1	Home                                              	11	13460.77
+catalog	NULL	2000	1	Jewelry                                           	10	8071.78
+catalog	NULL	2000	1	Men                                               	10	14579.76
+catalog	NULL	2000	1	Music                                             	9	23840.61
+catalog	NULL	2000	1	Shoes                                             	15	38230.31
+catalog	NULL	2000	1	Sports                                            	13	5259.09
+catalog	NULL	2000	1	Women                                             	10	6568.10
+catalog	NULL	2000	2	NULL	1	NULL
+catalog	NULL	2000	2	Books                                             	15	27517.21
+catalog	NULL	2000	2	Children                                          	9	4430.92
+catalog	NULL	2000	2	Electronics                                       	9	7435.41
+catalog	NULL	2000	2	Home                                              	10	12542.28
+catalog	NULL	2000	2	Jewelry                                           	8	4325.38
+catalog	NULL	2000	2	Men                                               	12	5896.08
+catalog	NULL	2000	2	Music                                             	11	2962.75
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q77.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q77.sql.out
new file mode 100644
index 0000000000000..aef644cbf749a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q77.sql.out
@@ -0,0 +1,49 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<channel:string,id:int,sales:decimal(27,2),returns:decimal(27,2),profit:decimal(28,2)>
+-- !query output
+NULL	NULL	238379361.39	11949589.80	-69066318.65
+catalog channel	NULL	116209.49	1989207.49	-1103184.43
+catalog channel	NULL	81893158.01	7956829.96	-13266843.17
+catalog channel	1	26819348.55	1989207.49	-4169636.96
+catalog channel	2	27454600.50	1989207.49	-3825432.73
+catalog channel	5	27502999.47	1989207.49	-4168589.05
+store channel	NULL	114945147.06	2805428.32	-51038302.63
+store channel	1	19743223.74	437906.57	-8831106.92
+store channel	2	18272722.14	522196.16	-8183951.59
+store channel	4	19720603.73	449683.19	-8686183.94
+store channel	7	19275817.79	456008.74	-8633897.32
+store channel	8	19342554.44	467014.66	-8767463.34
+store channel	10	18590225.22	472619.00	-7935699.52
+web channel	NULL	41541056.32	1187331.52	-4761172.85
+web channel	1	1228578.03	47675.10	-188274.21
+web channel	2	1343477.51	44041.96	-110450.27
+web channel	5	1355045.09	11417.70	-99307.08
+web channel	7	1439105.65	44505.99	-131740.09
+web channel	8	1222672.59	64741.73	-137947.41
+web channel	11	1380233.43	12631.13	-89907.09
+web channel	13	1380503.10	66341.09	-190038.15
+web channel	14	1322143.80	66841.97	-172945.60
+web channel	17	1640923.76	6485.63	-15639.73
+web channel	19	1258408.09	43554.94	-95271.34
+web channel	20	1413076.47	25716.55	-176644.94
+web channel	23	1522205.18	26405.14	-102673.55
+web channel	25	1435872.54	58108.85	-141661.85
+web channel	26	1364954.66	41289.03	-226994.92
+web channel	29	1456398.83	23148.07	-127320.23
+web channel	31	1331158.19	58914.20	-259972.34
+web channel	32	1430016.29	68634.13	-216355.06
+web channel	35	1275017.69	9253.48	-86568.79
+web channel	37	1527487.12	34512.52	-145699.04
+web channel	38	1459100.39	77580.83	-145709.91
+web channel	41	1542741.42	22802.83	-126554.81
+web channel	43	1556206.93	43143.17	-170251.07
+web channel	44	1197577.68	47675.73	-232746.02
+web channel	47	1311689.48	14426.63	-151021.48
+web channel	49	1487129.22	38844.09	-129263.73
+web channel	50	1404935.78	52795.27	-203592.12
+web channel	53	1308281.20	25132.82	-234014.24
+web channel	55	1348261.69	34116.67	-208865.51
+web channel	56	1223987.10	55188.28	-252201.97
+web channel	59	1373867.41	21405.99	-191540.30
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q78.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q78.sql.out
new file mode 100644
index 0000000000000..df28fcb255ae8
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q78.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<ratio:double,store_qty:bigint,store_wholesale_cost:decimal(17,2),store_sales_price:decimal(17,2),other_chan_qty:bigint,other_chan_wholesale_cost:decimal(18,2),other_chan_sales_price:decimal(18,2)>
+-- !query output
+
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q79.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q79.sql.out
new file mode 100644
index 0000000000000..0a8f10fa2ce4a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q79.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<c_last_name:string,c_first_name:string,substr(s_city, 1, 30):string,ss_ticket_number:int,amt:decimal(17,2),profit:decimal(17,2)>
+-- !query output
+NULL	NULL	Fairview	198070	3414.76	-28275.11
+NULL	NULL	Fairview	178356	4636.16	-24754.68
+NULL	NULL	Fairview	4561	38.97	-17858.20
+NULL	NULL	Fairview	227170	4972.68	-17709.46
+NULL	NULL	Fairview	78433	689.97	-16192.40
+NULL	NULL	Fairview	150646	3348.61	-15819.67
+NULL	NULL	Fairview	15261	2836.87	-15688.87
+NULL	NULL	Fairview	226126	651.62	-14052.62
+NULL	NULL	Fairview	36397	2615.22	-13738.93
+NULL	NULL	Fairview	40	532.59	-12157.64
+NULL	NULL	Fairview	237656	9409.20	-12036.60
+NULL	NULL	Fairview	178664	0.00	-11859.96
+NULL	NULL	Fairview	60434	7214.21	-11468.85
+NULL	NULL	Fairview	23163	1270.90	-11127.55
+NULL	NULL	Fairview	36041	7796.51	-11067.35
+NULL	NULL	Fairview	35679	5305.03	-10515.13
+NULL	NULL	Fairview	233081	361.69	-9831.11
+NULL	NULL	Fairview	65032	3442.33	-9620.20
+NULL	NULL	Fairview	162505	7179.34	-8572.34
+NULL	NULL	Fairview	142857	8377.25	-8392.77
+NULL	NULL	Fairview	193575	1828.27	-6805.77
+NULL	NULL	Fairview	49934	228.47	-5430.28
+NULL	NULL	Fairview	67999	621.15	-5031.30
+NULL	NULL	Fairview	223653	3570.27	-4327.27
+NULL	NULL	Fairview	6384	0.00	-3987.67
+NULL	NULL	Fairview	168833	1894.07	-3677.23
+NULL	NULL	Fairview	93344	0.00	-1599.51
+NULL	NULL	Fairview	53781	16.91	764.24
+NULL	NULL	Fairview	175529	1252.60	1017.06
+NULL	NULL	Fairview	197495	373.09	2365.53
+NULL	NULL	Fairview	4047	3121.19	2715.67
+NULL	NULL	Midway	48072	11113.87	-29510.66
+NULL	NULL	Midway	11394	5172.08	-25597.50
+NULL	NULL	Midway	207040	2008.73	-25394.40
+NULL	NULL	Midway	222765	7629.39	-25247.34
+NULL	NULL	Midway	217712	4349.03	-24697.82
+NULL	NULL	Midway	45131	5788.09	-24158.43
+NULL	NULL	Midway	164632	1394.75	-23255.21
+NULL	NULL	Midway	127656	5021.65	-22989.97
+NULL	NULL	Midway	83311	7875.67	-22914.32
+NULL	NULL	Midway	56977	3159.77	-20447.95
+NULL	NULL	Midway	215293	0.00	-19925.34
+NULL	NULL	Midway	140002	905.08	-18610.09
+NULL	NULL	Midway	173634	9179.91	-17739.01
+NULL	NULL	Midway	23863	152.72	-17463.88
+NULL	NULL	Midway	160686	5769.52	-17219.86
+NULL	NULL	Midway	82279	1835.82	-17174.21
+NULL	NULL	Midway	112924	10990.60	-16901.50
+NULL	NULL	Midway	166267	653.18	-16156.26
+NULL	NULL	Midway	213647	1924.30	-16123.38
+NULL	NULL	Midway	204741	2259.86	-15336.56
+NULL	NULL	Midway	122738	12543.65	-15281.12
+NULL	NULL	Midway	173203	4314.45	-14126.67
+NULL	NULL	Midway	147660	3460.04	-14088.14
+NULL	NULL	Midway	204175	1442.30	-13954.82
+NULL	NULL	Midway	124093	3122.91	-13873.66
+NULL	NULL	Midway	230300	1956.43	-13769.94
+NULL	NULL	Midway	217344	0.00	-13534.82
+NULL	NULL	Midway	91656	2427.25	-13507.46
+NULL	NULL	Midway	148611	5844.43	-13212.15
+NULL	NULL	Midway	234650	2836.85	-12426.80
+NULL	NULL	Midway	170455	3175.00	-11962.07
+NULL	NULL	Midway	26663	264.59	-11479.46
+NULL	NULL	Midway	75387	5918.43	-11406.83
+NULL	NULL	Midway	77944	788.27	-11329.12
+NULL	NULL	Midway	135318	805.73	-11284.26
+NULL	NULL	Midway	2896	12623.23	-11061.86
+NULL	NULL	Midway	158461	29.32	-10931.38
+NULL	NULL	Midway	1333	1390.53	-10865.55
+NULL	NULL	Midway	116866	3621.69	-10822.24
+NULL	NULL	Midway	194993	3401.82	-10654.47
+NULL	NULL	Midway	187290	6137.11	-10592.41
+NULL	NULL	Midway	151959	8548.17	-10204.65
+NULL	NULL	Midway	149775	15.18	-10113.87
+NULL	NULL	Midway	157401	1094.24	-10070.65
+NULL	NULL	Midway	210606	2098.02	-9961.51
+NULL	NULL	Midway	118130	182.41	-9901.81
+NULL	NULL	Midway	7439	2125.08	-9595.87
+NULL	NULL	Midway	19796	1532.38	-9553.82
+NULL	NULL	Midway	222335	0.00	-9523.98
+NULL	NULL	Midway	208688	1211.17	-9220.71
+NULL	NULL	Midway	32396	2191.49	-9201.97
+NULL	NULL	Midway	111809	4648.48	-9170.68
+NULL	NULL	Midway	12549	1306.17	-8842.14
+NULL	NULL	Midway	6762	2278.78	-8696.20
+NULL	NULL	Midway	12892	2172.51	-8615.80
+NULL	NULL	Midway	133874	0.00	-8601.78
+NULL	NULL	Midway	86109	161.95	-8348.77
+NULL	NULL	Midway	58015	5671.25	-8231.58
+NULL	NULL	Midway	173045	0.00	-8004.30
+NULL	NULL	Midway	221481	315.96	-7893.04
+NULL	NULL	Midway	4682	0.00	-7769.63
+NULL	NULL	Midway	155285	0.00	-7757.04
+NULL	NULL	Midway	194931	1315.23	-7705.13
+NULL	NULL	Midway	208424	878.43	-7702.00
+NULL	NULL	Midway	199504	2052.21	-7662.62
+NULL	NULL	Midway	73227	2869.94	-7484.05
+NULL	NULL	Midway	12578	3486.23	-7447.58
+NULL	NULL	Midway	160835	3133.37	-7258.48
+NULL	NULL	Midway	164697	6050.09	-7208.84
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q8.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q8.sql.out
new file mode 100644
index 0000000000000..17b390542941c
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q8.sql.out
@@ -0,0 +1,10 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<s_store_name:string,sum(ss_net_profit):decimal(17,2)>
+-- !query output
+able	-9960913.46
+bar	-10200043.73
+eing	-11104757.88
+ese	-11009853.93
+ought	-10574796.20
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q80.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q80.sql.out
new file mode 100644
index 0000000000000..bdadd9e04e13c
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q80.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<channel:string,id:string,sales:decimal(27,2),returns:decimal(32,2),profit:decimal(33,2)>
+-- !query output
+NULL	NULL	13553636.71	630819.07	-3876883.14
+catalog channel	NULL	4818809.23	250952.51	-519182.04
+catalog channel	catalog_pageAAAAAAAAAAABAAAA	18407.14	67.62	4548.90
+catalog channel	catalog_pageAAAAAAAAABABAAAA	1837.52	0.00	-3388.65
+catalog channel	catalog_pageAAAAAAAAACABAAAA	18390.60	0.00	9578.02
+catalog channel	catalog_pageAAAAAAAAADABAAAA	1404.55	27.58	-3703.26
+catalog channel	catalog_pageAAAAAAAAADCBAAAA	2868.85	0.00	826.15
+catalog channel	catalog_pageAAAAAAAAAEABAAAA	5039.78	0.00	-3701.62
+catalog channel	catalog_pageAAAAAAAAAECBAAAA	3971.05	0.00	1757.41
+catalog channel	catalog_pageAAAAAAAAAFABAAAA	2316.00	0.00	-1002.65
+catalog channel	catalog_pageAAAAAAAAAFCBAAAA	7015.40	0.00	915.49
+catalog channel	catalog_pageAAAAAAAAAGABAAAA	23067.38	0.00	10478.19
+catalog channel	catalog_pageAAAAAAAAAGCBAAAA	27364.22	0.00	8607.28
+catalog channel	catalog_pageAAAAAAAAAHABAAAA	12794.77	0.00	-7812.73
+catalog channel	catalog_pageAAAAAAAAAHCBAAAA	2640.87	63.18	-2334.42
+catalog channel	catalog_pageAAAAAAAAAICBAAAA	1845.15	0.00	-4629.81
+catalog channel	catalog_pageAAAAAAAAAJCBAAAA	1555.72	0.00	-5039.18
+catalog channel	catalog_pageAAAAAAAAAKPAAAAA	42070.02	0.00	6567.19
+catalog channel	catalog_pageAAAAAAAAALCBAAAA	8772.47	0.00	479.78
+catalog channel	catalog_pageAAAAAAAAALPAAAAA	20380.13	0.00	2437.83
+catalog channel	catalog_pageAAAAAAAAAMCBAAAA	15549.59	5438.72	-7150.88
+catalog channel	catalog_pageAAAAAAAAAMPAAAAA	11607.07	0.00	-7275.33
+catalog channel	catalog_pageAAAAAAAAANCBAAAA	21588.37	0.00	4221.36
+catalog channel	catalog_pageAAAAAAAAANPAAAAA	21638.94	0.00	4905.57
+catalog channel	catalog_pageAAAAAAAAAOCBAAAA	521.86	0.00	160.52
+catalog channel	catalog_pageAAAAAAAAAOPAAAAA	38265.71	0.00	-15751.13
+catalog channel	catalog_pageAAAAAAAAAPCBAAAA	4872.72	0.00	2322.60
+catalog channel	catalog_pageAAAAAAAAAPPAAAAA	16581.43	5140.00	-3778.70
+catalog channel	catalog_pageAAAAAAAABAABAAAA	6642.83	0.00	768.98
+catalog channel	catalog_pageAAAAAAAABBABAAAA	280.96	0.00	-50.56
+catalog channel	catalog_pageAAAAAAAABCABAAAA	1048.56	0.00	-600.26
+catalog channel	catalog_pageAAAAAAAABDABAAAA	17545.67	0.00	-9838.60
+catalog channel	catalog_pageAAAAAAAABDCBAAAA	9090.78	0.00	-572.50
+catalog channel	catalog_pageAAAAAAAABEABAAAA	17525.88	232.32	-2410.45
+catalog channel	catalog_pageAAAAAAAABECBAAAA	18693.14	0.00	-3538.78
+catalog channel	catalog_pageAAAAAAAABFABAAAA	17175.44	61.36	2821.53
+catalog channel	catalog_pageAAAAAAAABFCBAAAA	140.82	0.00	-569.12
+catalog channel	catalog_pageAAAAAAAABGABAAAA	384.55	0.00	-619.85
+catalog channel	catalog_pageAAAAAAAABGCBAAAA	3166.40	1590.00	-9892.69
+catalog channel	catalog_pageAAAAAAAABHABAAAA	14266.65	0.00	-4959.21
+catalog channel	catalog_pageAAAAAAAABHCBAAAA	1741.48	0.00	403.78
+catalog channel	catalog_pageAAAAAAAABICBAAAA	1413.03	0.00	-789.20
+catalog channel	catalog_pageAAAAAAAABJCBAAAA	17473.29	18.29	2514.85
+catalog channel	catalog_pageAAAAAAAABKCBAAAA	415.15	0.00	-1.90
+catalog channel	catalog_pageAAAAAAAABKPAAAAA	2724.27	0.00	-8394.13
+catalog channel	catalog_pageAAAAAAAABLCBAAAA	15422.53	0.00	7523.39
+catalog channel	catalog_pageAAAAAAAABLPAAAAA	20440.65	0.00	-6095.43
+catalog channel	catalog_pageAAAAAAAABMCBAAAA	2324.92	0.00	1256.32
+catalog channel	catalog_pageAAAAAAAABMPAAAAA	2322.14	0.00	-4465.41
+catalog channel	catalog_pageAAAAAAAABNCBAAAA	10606.89	0.00	-4039.08
+catalog channel	catalog_pageAAAAAAAABNPAAAAA	5671.18	0.00	-6038.20
+catalog channel	catalog_pageAAAAAAAABOCBAAAA	7494.52	0.00	2578.36
+catalog channel	catalog_pageAAAAAAAABOPAAAAA	19163.74	0.00	2677.78
+catalog channel	catalog_pageAAAAAAAABPCBAAAA	48.42	0.00	-92.80
+catalog channel	catalog_pageAAAAAAAABPPAAAAA	65484.56	8188.50	16714.73
+catalog channel	catalog_pageAAAAAAAACAABAAAA	34936.59	0.00	6574.55
+catalog channel	catalog_pageAAAAAAAACBABAAAA	8178.17	0.00	1370.92
+catalog channel	catalog_pageAAAAAAAACCABAAAA	24428.99	0.00	7616.93
+catalog channel	catalog_pageAAAAAAAACDABAAAA	17628.32	1503.67	-2086.58
+catalog channel	catalog_pageAAAAAAAACDCBAAAA	222.09	0.00	108.57
+catalog channel	catalog_pageAAAAAAAACEABAAAA	9898.50	491.66	-4581.91
+catalog channel	catalog_pageAAAAAAAACECBAAAA	11493.96	0.00	-7027.70
+catalog channel	catalog_pageAAAAAAAACFABAAAA	25027.83	11283.47	4064.93
+catalog channel	catalog_pageAAAAAAAACFCBAAAA	618.24	0.00	-1316.28
+catalog channel	catalog_pageAAAAAAAACGABAAAA	1830.05	0.00	-3073.99
+catalog channel	catalog_pageAAAAAAAACHABAAAA	1121.15	275.52	-5850.52
+catalog channel	catalog_pageAAAAAAAACHCBAAAA	13108.45	0.00	-786.42
+catalog channel	catalog_pageAAAAAAAACICBAAAA	1755.49	0.00	146.98
+catalog channel	catalog_pageAAAAAAAACJCBAAAA	2982.05	0.00	-196.21
+catalog channel	catalog_pageAAAAAAAACKCBAAAA	13658.88	0.00	5924.52
+catalog channel	catalog_pageAAAAAAAACKPAAAAA	26319.77	162.81	-8423.49
+catalog channel	catalog_pageAAAAAAAACLCBAAAA	6469.97	0.00	-5067.48
+catalog channel	catalog_pageAAAAAAAACLPAAAAA	43938.15	7021.98	-4638.29
+catalog channel	catalog_pageAAAAAAAACMCBAAAA	9332.13	0.00	-5421.54
+catalog channel	catalog_pageAAAAAAAACMPAAAAA	13306.48	0.00	-6449.30
+catalog channel	catalog_pageAAAAAAAACNPAAAAA	18488.66	0.00	186.37
+catalog channel	catalog_pageAAAAAAAACOCBAAAA	1220.46	451.26	-6770.17
+catalog channel	catalog_pageAAAAAAAACOPAAAAA	27469.76	1091.48	7443.78
+catalog channel	catalog_pageAAAAAAAACPCBAAAA	117.48	0.00	30.19
+catalog channel	catalog_pageAAAAAAAACPPAAAAA	19041.61	172.22	380.68
+catalog channel	catalog_pageAAAAAAAADAABAAAA	12391.42	953.40	-7032.35
+catalog channel	catalog_pageAAAAAAAADBABAAAA	1495.00	0.00	-9772.65
+catalog channel	catalog_pageAAAAAAAADCABAAAA	0.00	0.00	-3388.10
+catalog channel	catalog_pageAAAAAAAADDABAAAA	15917.59	0.00	4459.43
+catalog channel	catalog_pageAAAAAAAADDCBAAAA	4409.76	733.12	912.46
+catalog channel	catalog_pageAAAAAAAADEABAAAA	14545.59	10419.31	-2030.01
+catalog channel	catalog_pageAAAAAAAADFABAAAA	5222.68	0.00	-3306.65
+catalog channel	catalog_pageAAAAAAAADFCBAAAA	2442.96	0.00	-758.68
+catalog channel	catalog_pageAAAAAAAADGABAAAA	14604.59	0.00	-3980.39
+catalog channel	catalog_pageAAAAAAAADHABAAAA	19078.94	12277.75	6448.20
+catalog channel	catalog_pageAAAAAAAADHCBAAAA	12207.03	0.00	1895.71
+catalog channel	catalog_pageAAAAAAAADICBAAAA	12548.28	0.00	407.59
+catalog channel	catalog_pageAAAAAAAADJCBAAAA	3526.74	0.00	2014.47
+catalog channel	catalog_pageAAAAAAAADKCBAAAA	3071.53	0.00	-5979.53
+catalog channel	catalog_pageAAAAAAAADKPAAAAA	52608.41	1959.75	20557.42
+catalog channel	catalog_pageAAAAAAAADLCBAAAA	386.12	0.00	-1094.08
+catalog channel	catalog_pageAAAAAAAADLPAAAAA	22380.47	230.88	611.66
+catalog channel	catalog_pageAAAAAAAADMCBAAAA	4943.75	0.00	2158.10
+catalog channel	catalog_pageAAAAAAAADMPAAAAA	17113.94	0.00	4231.57
+catalog channel	catalog_pageAAAAAAAADNCBAAAA	982.53	0.00	-4098.99
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q81.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q81.sql.out
new file mode 100644
index 0000000000000..0a7e038efaabf
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q81.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<c_customer_id:string,c_salutation:string,c_first_name:string,c_last_name:string,ca_street_number:string,ca_street_name:string,ca_street_type:string,ca_suite_number:string,ca_city:string,ca_county:string,ca_state:string,ca_zip:string,ca_country:string,ca_gmt_offset:decimal(5,2),ca_location_type:string,ctr_total_return:decimal(17,2)>
+-- !query output
+AAAAAAAAAAGOAAAA	Mrs.      	Melissa             	Barton                        	761       	Lincoln 	Drive          	Suite 440 	Marion	Cobb County	GA	30399     	United States	-5.00	condo               	2053.35
+AAAAAAAAACCMAAAA	Mr.       	Craig               	Thompson                      	607       	Lakeview Laurel	Lane           	Suite 140 	Hillcrest	Floyd County	GA	33003     	United States	-5.00	apartment           	2314.62
+AAAAAAAAADBBAAAA	Dr.       	Robert              	Willis                        	48        	Ninth West	Lane           	Suite 370 	Oak Grove	Cook County	GA	38370     	United States	-5.00	single family       	2232.39
+AAAAAAAAAELEAAAA	Miss      	NULL	Werner                        	800       	11th 	Avenue         	Suite M   	Harmony	Echols County	GA	35804     	United States	-5.00	condo               	2312.04
+AAAAAAAAAFIHBAAA	Sir       	Chadwick            	Stevens                       	679       	Forest Central	Avenue         	Suite 60  	Sulphur Springs	Gordon County	GA	38354     	United States	-5.00	condo               	2282.39
+AAAAAAAAAHDGAAAA	Mr.       	Justin              	Garcia                        	281       	Spring 	Way            	Suite 320 	Shiloh	Fannin County	GA	39275     	United States	-5.00	single family       	4185.32
+AAAAAAAAAICMAAAA	Sir       	Willie              	Maldonado                     	492       	Park 	Street         	Suite C   	Shady Grove	Chatham County	GA	32812     	United States	-5.00	apartment           	2428.92
+AAAAAAAAALPABAAA	Dr.       	John                	Wang                          	724       	Oak 	Ave            	Suite 70  	Maple Grove	Newton County	GA	38252     	United States	-5.00	single family       	1727.64
+AAAAAAAAAMJAAAAA	Sir       	Roy                 	Mark                          	92        	Cedar Sycamore	Way            	Suite 30  	Union Hill	Jenkins County	GA	37746     	United States	-5.00	single family       	1825.38
+AAAAAAAAAMKFBAAA	Dr.       	Lamar               	Jones                         	194       	Maple 	Cir.           	Suite 80  	Oakwood	Jefferson County	GA	30169     	United States	-5.00	single family       	3977.04
+AAAAAAAAAMLOAAAA	Ms.       	Brooke              	Garcia                        	914       	Hillcrest Cedar	Drive          	Suite 210 	Stringtown	Jasper County	GA	30162     	United States	-5.00	condo               	4133.35
+AAAAAAAAAMODAAAA	Mrs.      	Lori                	Campos                        	728       	Pine Walnut	Cir.           	Suite M   	Woodlawn	Worth County	GA	34098     	United States	-5.00	condo               	4182.98
+AAAAAAAAANJOAAAA	Ms.       	Allen               	Rodriguez                     	156       	Ash Seventh	Way            	Suite 100 	Pleasant Valley	Sumter County	GA	32477     	United States	-5.00	apartment           	2710.46
+AAAAAAAAAODABAAA	Dr.       	Oscar               	Miller                        	651       	Smith West	Dr.            	Suite E   	Antioch	Evans County	GA	38605     	United States	-5.00	condo               	7234.29
+AAAAAAAAAODIAAAA	Mr.       	Julio               	Gamble                        	211       	View 	Cir.           	Suite Y   	Summit	Bulloch County	GA	30499     	United States	-5.00	single family       	3291.69
+AAAAAAAABBAGAAAA	Dr.       	Charles             	Butts                         	NULL	Park Hickory	RD             	NULL	NULL	NULL	GA	38605     	NULL	-5.00	NULL	3291.71
+AAAAAAAABBCJAAAA	Sir       	Doyle               	Shaffer                       	401       	Johnson Main	Boulevard      	Suite 150 	Shiloh	Worth County	GA	39275     	United States	-5.00	single family       	4027.79
+AAAAAAAABBJKAAAA	Dr.       	Mildred             	Hogan                         	174       	Birch 	Pkwy           	Suite Q   	Enterprise	Webster County	GA	31757     	United States	-5.00	condo               	4006.97
+AAAAAAAABCCEAAAA	Mrs.      	Linda               	Worth                         	283       	2nd 	Pkwy           	Suite 230 	Fairfield	Chattooga County	GA	36192     	United States	-5.00	condo               	5047.24
+AAAAAAAABCEFBAAA	Dr.       	William             	Parks                         	19        	Meadow 	Circle         	Suite P   	Providence	Gwinnett County	GA	36614     	United States	-5.00	apartment           	4262.30
+AAAAAAAABCFKAAAA	NULL	Shawn               	Peterson                      	220       	Johnson First	Ct.            	Suite F   	Kingston	Randolph County	GA	34975     	United States	-5.00	condo               	2786.80
+AAAAAAAABCGEAAAA	Miss      	Margaret            	Wright                        	314       	5th Washington	Street         	Suite S   	Forest Hills	Dougherty County	GA	39237     	United States	-5.00	single family       	2049.52
+AAAAAAAABDKOAAAA	Dr.       	Vernita             	Bennett                       	763       	13th 	Court          	Suite 60  	Maple Hill	Rabun County	GA	38095     	United States	-5.00	single family       	18570.61
+AAAAAAAABEIKAAAA	Miss      	Nena                	August                        	297       	Lee 	Lane           	Suite 240 	Cedar	Henry County	GA	31229     	United States	-5.00	condo               	2094.88
+AAAAAAAABEJLAAAA	Mr.       	Abel                	Lucero                        	382       	Ash North	Parkway        	Suite J   	Plainview	Catoosa County	GA	33683     	United States	-5.00	single family       	3530.52
+AAAAAAAABIDEAAAA	Dr.       	Marcus              	Holder                        	724       	Oak 	Ave            	Suite 70  	Maple Grove	Newton County	GA	38252     	United States	-5.00	single family       	1927.32
+AAAAAAAABIGIAAAA	Mrs.      	Caprice             	Espinoza                      	527       	Church Second	Court          	Suite M   	Derby	Taylor County	GA	37702     	United States	-5.00	condo               	6808.21
+AAAAAAAABJEEAAAA	Dr.       	Clarence            	Swanson                       	NULL	NULL	Ct.            	Suite 110 	NULL	Fannin County	GA	NULL	United States	-5.00	condo               	1676.20
+AAAAAAAABJHJAAAA	Sir       	James               	Culpepper                     	16        	Park 	Way            	Suite 460 	Freeman	Charlton County	GA	32297     	United States	-5.00	single family       	2096.68
+AAAAAAAABKKAAAAA	Sir       	Jeffrey             	Young                         	379       	7th 	Lane           	Suite 90  	Plainville	Brantley County	GA	36115     	United States	-5.00	apartment           	2161.20
+AAAAAAAABLIBAAAA	Dr.       	Charles             	Hawkins                       	939       	Jackson Forest	Boulevard      	Suite Y   	Bunker Hill	Candler County	GA	30150     	United States	-5.00	condo               	4396.68
+AAAAAAAABLPJAAAA	NULL	NULL	Gibson                        	458       	Second 	Boulevard      	Suite 480 	Plainview	Liberty County	GA	33683     	United States	-5.00	single family       	1513.94
+AAAAAAAABLPNAAAA	Mr.       	Eric                	Durham                        	917       	Second Park	Wy             	Suite 180 	Friendship	Marion County	GA	34536     	United States	-5.00	condo               	2166.25
+AAAAAAAABMCOAAAA	Sir       	Warren              	Skinner                       	934       	Walnut Hickory	Court          	Suite 310 	Riverview	Long County	GA	39003     	United States	-5.00	condo               	5699.34
+AAAAAAAABNJBAAAA	Miss      	Santa               	Richmond                      	859       	4th Maple	ST             	Suite 60  	Antioch	Toombs County	GA	38605     	United States	-5.00	single family       	2136.92
+AAAAAAAABPGJAAAA	Dr.       	June                	Hill                          	480       	Second 15th	ST             	Suite T   	Woodlawn	Atkinson County	GA	34098     	United States	-5.00	condo               	3081.42
+AAAAAAAABPKOAAAA	Mr.       	Robert              	Tanner                        	676       	Sixth Walnut	Road           	Suite L   	Cumberland	Oglethorpe County	GA	38971     	United States	-5.00	condo               	3966.42
+AAAAAAAACAINAAAA	Dr.       	Donald              	Beebe                         	960       	Railroad Davis	Ln             	Suite Y   	Silver Springs	Clay County	GA	34843     	United States	-5.00	apartment           	4925.75
+AAAAAAAACBEPAAAA	Sir       	Otis                	Hartman                       	939       	Third 	Lane           	Suite R   	Forest Hills	Polk County	GA	39237     	United States	-5.00	single family       	4904.39
+AAAAAAAACCBGBAAA	Ms.       	Judy                	Williams                      	283       	Center 	Cir.           	Suite Q   	Farmington	Dade County	GA	39145     	United States	-5.00	single family       	3655.41
+AAAAAAAACCOFAAAA	Sir       	John                	Bower                         	970       	Johnson Franklin	ST             	Suite 20  	Walnut Grove	Union County	GA	37752     	United States	-5.00	single family       	5456.48
+AAAAAAAACDNAAAAA	Miss      	Beatriz             	Rome                          	947       	Elm 	Ln             	Suite 210 	Wilson	Lanier County	GA	36971     	United States	-5.00	apartment           	3994.62
+AAAAAAAACEKEBAAA	Mr.       	Gregory             	Maynard                       	214       	Park Forest	Wy             	Suite 90  	Lakewood	Jones County	GA	38877     	United States	-5.00	apartment           	3840.02
+AAAAAAAACGIBBAAA	Dr.       	Pamela              	Taylor                        	364       	Cherry 	Pkwy           	Suite 430 	Rolling Hills	Glascock County	GA	37272     	United States	-5.00	apartment           	2025.02
+AAAAAAAACIDCBAAA	Dr.       	Andre               	Worth                         	307       	10th 6th	Parkway        	Suite I   	Five Forks	Haralson County	GA	32293     	United States	-5.00	single family       	1947.99
+AAAAAAAACJLAAAAA	Sir       	Kevin               	Ellis                         	291       	7th Valley	Circle         	Suite H   	Hartland	Marion County	GA	36594     	United States	-5.00	apartment           	5655.03
+AAAAAAAACKDGAAAA	Ms.       	Catherine           	Anderson                      	875       	Walnut Lake	Ct.            	Suite 410 	Taft	Telfair County	GA	30589     	United States	-5.00	condo               	1701.58
+AAAAAAAACKIOAAAA	Mr.       	Francisco           	Davis                         	394       	Spruce 	Dr.            	Suite 250 	Smith	Warren County	GA	37317     	United States	-5.00	condo               	4471.40
+AAAAAAAACNIEBAAA	Dr.       	Andrea              	Lyon                          	700       	Fifth Center	Cir.           	Suite P   	Franklin	Putnam County	GA	39101     	United States	-5.00	condo               	1725.43
+AAAAAAAACNOHAAAA	Ms.       	Jessica             	Pitman                        	11        	Spring East	Street         	Suite 470 	Glenwood	Columbia County	GA	33511     	United States	-5.00	condo               	3469.20
+AAAAAAAACOOKAAAA	Miss      	Rebecca             	Morris                        	NULL	Park Pine	NULL	NULL	NULL	NULL	GA	NULL	United States	-5.00	NULL	1834.52
+AAAAAAAADAFIBAAA	Dr.       	Todd                	Alexander                     	371       	East 	Pkwy           	Suite Y   	Jackson	Long County	GA	39583     	United States	-5.00	single family       	3081.99
+AAAAAAAADBAABAAA	Dr.       	Martha              	Valenti                       	605       	13th Lake	Ave            	Suite 100 	Mount Pleasant	Pickens County	GA	31933     	United States	-5.00	single family       	2721.38
+AAAAAAAADBNCAAAA	Mrs.      	Katie               	Novak                         	352       	Jackson Hill	Street         	Suite 360 	Liberty	Jeff Davis County	GA	33451     	United States	-5.00	apartment           	2970.22
+AAAAAAAADCCHBAAA	Dr.       	Harley              	Kohl                          	116       	Center Park	Cir.           	Suite 300 	White Oak	Cook County	GA	36668     	United States	-5.00	single family       	5818.25
+AAAAAAAADCICAAAA	Mrs.      	Bonnie              	Harden                        	375       	Railroad 3rd	Road           	Suite A   	Greenwood	Glynn County	GA	38828     	United States	-5.00	single family       	2770.41
+AAAAAAAADDJPAAAA	Sir       	Rick                	Lindstrom                     	994       	Jefferson View	Road           	Suite R   	Spring Valley	Barrow County	GA	36060     	United States	-5.00	apartment           	8135.54
+AAAAAAAADDKOAAAA	Sir       	Derek               	Watkins                       	344       	View Cedar	Boulevard      	Suite 280 	Fairfield	Gordon County	GA	36192     	United States	-5.00	single family       	1699.26
+AAAAAAAADDLMAAAA	Mr.       	Alan                	Meredith                      	994       	Jefferson View	Road           	Suite R   	Spring Valley	Barrow County	GA	36060     	United States	-5.00	apartment           	2784.48
+AAAAAAAADGDBBAAA	Sir       	Walter              	Barron                        	728       	River 	Court          	Suite 110 	Walnut Grove	Paulding County	GA	37752     	United States	-5.00	condo               	4128.19
+AAAAAAAADGGCBAAA	Ms.       	Anna                	Maynard                       	956       	Hill 	Blvd           	Suite N   	Macedonia	Emanuel County	GA	31087     	United States	-5.00	condo               	6425.51
+AAAAAAAADGPLAAAA	Mrs.      	Colleen             	Rogers                        	633       	6th 	Dr.            	Suite I   	Bridgeport	Harris County	GA	35817     	United States	-5.00	apartment           	3184.68
+AAAAAAAADHBGAAAA	Miss      	NULL	NULL	862       	Locust 	Circle         	Suite 430 	Greenfield	Liberty County	GA	35038     	United States	-5.00	single family       	14758.14
+AAAAAAAADJICBAAA	Dr.       	Julian              	Olivares                      	56        	1st Washington	Parkway        	Suite 190 	Friendship	Jeff Davis County	GA	34536     	United States	-5.00	single family       	3078.43
+AAAAAAAADKOLAAAA	Dr.       	Matthew             	Goodman                       	385       	Oak 	RD             	Suite D   	New Hope	Jefferson County	GA	39431     	United States	-5.00	apartment           	4047.82
+AAAAAAAADLEHAAAA	Dr.       	Robert              	Lilley                        	926       	Davis First	Road           	Suite 360 	Salem	Chattooga County	GA	38048     	United States	-5.00	apartment           	2256.31
+AAAAAAAADLEHAAAA	Dr.       	Robert              	Lilley                        	926       	Davis First	Road           	Suite 360 	Salem	Chattooga County	GA	38048     	United States	-5.00	apartment           	2298.55
+AAAAAAAADMOOAAAA	Sir       	Eddie               	Craig                         	788       	Lake 	Dr.            	Suite F   	Buena Vista	Marion County	GA	35752     	United States	-5.00	single family       	5585.06
+AAAAAAAADNCKAAAA	Dr.       	Scott               	Whitehurst                    	481       	4th 	Wy             	Suite U   	Sugar Hill	Effingham County	GA	35114     	United States	-5.00	condo               	3355.64
+AAAAAAAADNHFBAAA	Dr.       	Gerald              	Hayden                        	884       	1st Smith	Court          	Suite 390 	Highland	Towns County	GA	39454     	United States	-5.00	single family       	3481.60
+AAAAAAAADONFBAAA	Dr.       	Jane                	Williams                      	527       	Cedar South	Street         	Suite 310 	Fairview	Twiggs County	GA	35709     	United States	-5.00	condo               	2304.16
+AAAAAAAADPLEAAAA	Mr.       	Michael             	Powell                        	616       	6th 	Pkwy           	Suite G   	Glendale	Dougherty County	GA	33951     	United States	-5.00	single family       	2061.25
+AAAAAAAAECBEAAAA	Ms.       	Megan               	Wilson                        	93        	13th 	Drive          	Suite X   	Riverdale	Clarke County	GA	39391     	United States	-5.00	single family       	5256.96
+AAAAAAAAECBMAAAA	Mrs.      	Paula               	Clark                         	145       	Locust 	Drive          	Suite M   	Mount Vernon	Wilcox County	GA	38482     	United States	-5.00	condo               	3445.05
+AAAAAAAAECIEAAAA	Miss      	Danille             	Sanders                       	825       	Jackson South	Boulevard      	Suite 480 	Riverdale	Clarke County	GA	39391     	United States	-5.00	single family       	2897.72
+AAAAAAAAEDDFAAAA	Dr.       	Michelle            	Louis                         	375       	Highland East	Avenue         	Suite L   	Oakdale	Haralson County	GA	39584     	United States	-5.00	apartment           	2301.53
+AAAAAAAAEGJBAAAA	Mrs.      	Manuela             	Roche                         	935       	10th Davis	Way            	Suite O   	Oakwood	Worth County	GA	30169     	United States	-5.00	apartment           	6416.92
+AAAAAAAAEGMCBAAA	Mr.       	Christopher         	Saunders                      	476       	West 	Lane           	Suite J   	Plainview	Gilmer County	GA	33683     	United States	-5.00	single family       	3623.76
+AAAAAAAAEGOCBAAA	Mrs.      	Elise               	Huff                          	40        	Central 	Court          	Suite P   	Farmington	Coffee County	GA	39145     	United States	-5.00	condo               	2518.63
+AAAAAAAAEGOMAAAA	Sir       	James               	Handy                         	316       	Washington 	Lane           	Suite 480 	Oakland	Richmond County	GA	39843     	United States	-5.00	single family       	3035.76
+AAAAAAAAEHIJAAAA	NULL	NULL	Boggs                         	721       	2nd 5th	Ave            	Suite C   	Greenwood	Wilkes County	GA	38828     	United States	-5.00	apartment           	2623.44
+AAAAAAAAEICDBAAA	Ms.       	Alice               	Garrett                       	157       	West Smith	Drive          	Suite 20  	River Oaks	Hancock County	GA	38075     	United States	-5.00	single family       	16853.44
+AAAAAAAAEIINAAAA	Mrs.      	Marie               	Coffey                        	539       	Hill Walnut	Road           	Suite F   	Marion	Dade County	GA	30399     	United States	-5.00	apartment           	3608.02
+AAAAAAAAEIINAAAA	Mrs.      	Marie               	Coffey                        	539       	Hill Walnut	Road           	Suite F   	Marion	Dade County	GA	30399     	United States	-5.00	apartment           	7928.23
+AAAAAAAAEJCKAAAA	Mr.       	Bryan               	Simpson                       	313       	Seventh 7th	Ln             	Suite B   	Belmont	Washington County	GA	30191     	United States	-5.00	condo               	2094.14
+AAAAAAAAELCCBAAA	Ms.       	Robyn               	Martinez                      	274       	Willow Birch	Parkway        	Suite O   	Pleasant Valley	Burke County	GA	32477     	United States	-5.00	condo               	3081.88
+AAAAAAAAEMFEAAAA	Sir       	Dan                 	Porter                        	110       	North West	Ave            	Suite 270 	Farmington	Calhoun County	GA	39145     	United States	-5.00	single family       	5717.10
+AAAAAAAAENCEBAAA	Mrs.      	Latonya             	Doyle                         	634       	Railroad Park	Ln             	Suite S   	Oak Hill	Dade County	GA	37838     	United States	-5.00	single family       	4450.56
+AAAAAAAAEPHFBAAA	Mrs.      	Charlotte           	Matthews                      	368       	Center 4th	ST             	Suite L   	Martinsville	Baldwin County	GA	30419     	United States	-5.00	apartment           	2301.95
+AAAAAAAAFACDAAAA	Mr.       	William             	Marquez                       	724       	1st Walnut	Dr.            	Suite X   	Oak Hill	Appling County	GA	37838     	United States	-5.00	apartment           	5728.95
+AAAAAAAAFADGAAAA	Ms.       	Edna                	Brown                         	871       	11th Third	Cir.           	Suite 380 	Waterloo	Quitman County	GA	31675     	United States	-5.00	condo               	3552.69
+AAAAAAAAFANIAAAA	Dr.       	Jeanette            	Mccormick                     	89        	Jackson Birch	RD             	Suite 440 	Lakeview	Coweta County	GA	38579     	United States	-5.00	condo               	9022.24
+AAAAAAAAFCPCAAAA	Dr.       	Christine           	Smith                         	820       	Park Lincoln	Cir.           	Suite 60  	Springfield	Irwin County	GA	39303     	United States	-5.00	apartment           	1916.99
+AAAAAAAAFFAPAAAA	Sir       	Donald              	Wright                        	35        	Jefferson 	Parkway        	Suite A   	Buena Vista	Marion County	GA	35752     	United States	-5.00	single family       	3434.76
+AAAAAAAAFFFDAAAA	Mr.       	Nathan              	Pratt                         	317       	Main Second	Dr.            	Suite J   	Shaw	Madison County	GA	30618     	United States	-5.00	condo               	3953.10
+AAAAAAAAFGJHAAAA	NULL	NULL	Carey                         	605       	13th Lake	Ave            	Suite 100 	Mount Pleasant	Pickens County	GA	31933     	United States	-5.00	single family       	3349.63
+AAAAAAAAFGMKAAAA	Mrs.      	Albert              	Bradshaw                      	774       	15th 9th	Blvd           	Suite I   	White Oak	Peach County	GA	36668     	United States	-5.00	single family       	1695.86
+AAAAAAAAFKDJAAAA	NULL	NULL	NULL	186       	Spring 	Road           	Suite 290 	Buena Vista	Johnson County	GA	35752     	United States	-5.00	condo               	1919.82
+AAAAAAAAFLFNAAAA	Mrs.      	Scarlet             	Cook                          	127       	2nd 	Blvd           	Suite U   	Lebanon	Columbia County	GA	32898     	United States	-5.00	condo               	2722.78
+AAAAAAAAFLHCAAAA	Sir       	Patrick             	Taylor                        	635       	Third 	Ave            	Suite H   	Kingston	Wayne County	GA	34975     	United States	-5.00	apartment           	3413.14
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q82.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q82.sql.out
new file mode 100644
index 0000000000000..9f526a8be98ca
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q82.sql.out
@@ -0,0 +1,9 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_item_id:string,i_item_desc:string,i_current_price:decimal(7,2)>
+-- !query output
+AAAAAAAAEIJBAAAA	New games used to suggest. Annual, legal expenses see male pubs; almost early offences must come from the heads. En route small conditions underestimate e	64.55
+AAAAAAAAELDEAAAA	Methods get hours. American, great schools l	89.24
+AAAAAAAAMOCDAAAA	Colours think. Partial, rich things would not appeal extremely open students. New, working magis	66.03
+AAAAAAAAOIOBAAAA	Just open variables used to mind well also new 	65.54
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q83.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q83.sql.out
new file mode 100644
index 0000000000000..2ccae595d52f2
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q83.sql.out
@@ -0,0 +1,32 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<item_id:string,sr_item_qty:bigint,sr_dev:double,cr_item_qty:bigint,cr_dev:double,wr_item_qty:bigint,wr_dev:double,average:decimal(27,6)>
+-- !query output
+AAAAAAAAAEFEAAAA	35	9.04392764857881	4	1.03359173126615	90	23.25581395348837	43.000000
+AAAAAAAAAFDDAAAA	11	8.94308943089431	21	17.073170731707318	9	7.317073170731707	13.666667
+AAAAAAAAAFKBAAAA	2	1.5151515151515151	3	2.2727272727272725	39	29.545454545454547	14.666667
+AAAAAAAABGOBAAAA	37	17.129629629629626	31	14.351851851851851	4	1.8518518518518516	24.000000
+AAAAAAAACIEAAAAA	34	26.35658914728682	6	4.651162790697675	3	2.3255813953488373	14.333333
+AAAAAAAACLLBAAAA	63	23.863636363636363	16	6.0606060606060606	9	3.4090909090909096	29.333333
+AAAAAAAADHDAAAAA	4	8.88888888888889	2	4.444444444444445	9	20.0	5.000000
+AAAAAAAADKNCAAAA	6	NULL	41	NULL	NULL	NULL	NULL
+AAAAAAAAEIABAAAA	56	13.82716049382716	27	6.666666666666667	52	12.839506172839506	45.000000
+AAAAAAAAFKICAAAA	73	15.904139433551197	29	6.318082788671024	51	11.11111111111111	51.000000
+AAAAAAAAGDHAAAAA	79	21.236559139784948	3	0.8064516129032258	42	11.29032258064516	41.333333
+AAAAAAAAGJEAAAAA	94	23.55889724310777	4	1.0025062656641603	35	8.771929824561402	44.333333
+AAAAAAAAHKEBAAAA	12	7.4074074074074066	10	6.172839506172839	32	19.753086419753085	18.000000
+AAAAAAAAICPBAAAA	19	13.768115942028986	20	14.492753623188406	7	5.072463768115942	15.333333
+AAAAAAAAJICAAAAA	15	5.319148936170213	49	17.375886524822697	30	10.638297872340425	31.333333
+AAAAAAAAKDMBAAAA	7	11.666666666666666	9	15.0	4	6.666666666666667	6.666667
+AAAAAAAAKEEBAAAA	18	6.382978723404255	29	10.28368794326241	47	16.666666666666664	31.333333
+AAAAAAAALJIAAAAA	1	0.32362459546925565	9	2.912621359223301	93	30.09708737864078	34.333333
+AAAAAAAALLJAAAAA	53	16.358024691358025	25	7.716049382716049	30	9.25925925925926	36.000000
+AAAAAAAALMDCAAAA	103	21.458333333333336	46	9.583333333333332	11	2.291666666666667	53.333333
+AAAAAAAALNDBAAAA	31	10.99290780141844	29	10.28368794326241	34	12.056737588652483	31.333333
+AAAAAAAAMLDDAAAA	1	0.32051282051282054	22	7.051282051282051	81	25.961538461538463	34.666667
+AAAAAAAANAFEAAAA	12	10.0	21	17.5	7	5.833333333333333	13.333333
+AAAAAAAANFNAAAAA	10	2.849002849002849	82	23.36182336182336	25	7.122507122507122	39.000000
+AAAAAAAANIDEAAAA	18	5.172413793103448	14	4.022988505747127	84	24.137931034482758	38.666667
+AAAAAAAAPDAEAAAA	61	29.04761904761905	7	3.3333333333333335	2	0.9523809523809523	23.333333
+AAAAAAAAPDFCAAAA	7	4.861111111111112	20	13.88888888888889	21	14.583333333333334	16.000000
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q84.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q84.sql.out
new file mode 100644
index 0000000000000..4f956713021d5
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q84.sql.out
@@ -0,0 +1,21 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<customer_id:string,customername:string>
+-- !query output
+AAAAAAAAAAAKAAAA	Perry                         , Lori                
+AAAAAAAAANFABAAA	Harris                        , Alexander           
+AAAAAAAACHLDAAAA	Phipps                        , Charles             
+AAAAAAAACIHPAAAA	Nelson                        , Timothy             
+AAAAAAAACJOJAAAA	Mata                          , Eva                 
+AAAAAAAADEANAAAA	Gaines                        , David               
+AAAAAAAAEJPOAAAA	Wesley                        , Paul                
+AAAAAAAAFFKBAAAA	Duncan                        , Diana               
+AAAAAAAAFOBCAAAA	Null                          , Thomas              
+AAAAAAAAJFBBAAAA	Jacob                         , Ross                
+AAAAAAAAJGFBBAAA	Johnson                       , Windy               
+AAAAAAAALBGPAAAA	Porter                        , Veronica            
+AAAAAAAANJBHAAAA	Thompson                      , Kenneth             
+AAAAAAAANOLKAAAA	Thomas                        , Billy               
+AAAAAAAAONJOAAAA	Floyd                         , Janette             
+AAAAAAAAONLJAAAA	Howard                        , Tommy
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q85.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q85.sql.out
new file mode 100644
index 0000000000000..d46c5afed2b00
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q85.sql.out
@@ -0,0 +1,8 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<substr(r_reason_desc, 1, 20):string,avg(ws_quantity):double,avg(wr_refunded_cash):decimal(11,6),avg(wr_fee):decimal(11,6)>
+-- !query output
+Not the product that	10.0	177.330000	86.330000
+Not working any more	38.0	892.470000	14.940000
+reason 25           	79.0	4640.200000	47.240000
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q86.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q86.sql.out
new file mode 100644
index 0000000000000..4714aa00b0bfc
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q86.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<total_sum:decimal(17,2),i_category:string,i_class:string,lochierarchy:tinyint,rank_within_parent:int>
+-- !query output
+329088997.80	NULL	NULL	2	1
+35430664.91	Books                                             	NULL	1	1
+34918777.34	Men                                               	NULL	1	2
+33760378.41	Children                                          	NULL	1	3
+33243530.17	Electronics                                       	NULL	1	4
+33145164.93	Home                                              	NULL	1	5
+32756827.42	Shoes                                             	NULL	1	6
+32285995.12	Music                                             	NULL	1	7
+31712254.78	Sports                                            	NULL	1	8
+30795989.18	Women                                             	NULL	1	9
+30076030.73	Jewelry                                           	NULL	1	10
+963384.81	NULL	NULL	1	11
+615989.10	NULL	NULL	0	1
+65752.25	NULL	dresses                                           	0	2
+54445.48	NULL	lighting                                          	0	3
+40391.44	NULL	infants                                           	0	4
+35471.06	NULL	mens                                              	0	5
+34067.03	NULL	country                                           	0	6
+34046.70	NULL	accessories                                       	0	7
+27537.73	NULL	baseball                                          	0	8
+26174.29	NULL	guns                                              	0	9
+17743.12	NULL	athletic                                          	0	10
+11766.61	NULL	shirts                                            	0	11
+2662546.58	Books                                             	computers                                         	0	1
+2495626.03	Books                                             	parenting                                         	0	2
+2425647.20	Books                                             	fiction                                           	0	3
+2401196.81	Books                                             	sports                                            	0	4
+2346704.09	Books                                             	history                                           	0	5
+2302719.16	Books                                             	science                                           	0	6
+2276270.78	Books                                             	self-help                                         	0	7
+2257444.51	Books                                             	business                                          	0	8
+2216374.08	Books                                             	home repair                                       	0	9
+2206720.90	Books                                             	reference                                         	0	10
+2149203.74	Books                                             	mystery                                           	0	11
+2140640.32	Books                                             	travel                                            	0	12
+2048647.89	Books                                             	romance                                           	0	13
+1956865.76	Books                                             	cooking                                           	0	14
+1859356.14	Books                                             	arts                                              	0	15
+1617536.06	Books                                             	entertainments                                    	0	16
+67164.86	Books                                             	NULL	0	17
+8836884.43	Children                                          	school-uniforms                                   	0	1
+8760984.73	Children                                          	infants                                           	0	2
+8142524.86	Children                                          	newborn                                           	0	3
+7925776.25	Children                                          	toddlers                                          	0	4
+94208.14	Children                                          	NULL	0	5
+2489469.99	Electronics                                       	disk drives                                       	0	1
+2323648.66	Electronics                                       	personal                                          	0	2
+2305415.91	Electronics                                       	camcorders                                        	0	3
+2240199.44	Electronics                                       	karoke                                            	0	4
+2235691.33	Electronics                                       	stereo                                            	0	5
+2233058.51	Electronics                                       	televisions                                       	0	6
+2113792.92	Electronics                                       	portable                                          	0	7
+2092500.41	Electronics                                       	musical                                           	0	8
+2088701.69	Electronics                                       	dvd/vcr players                                   	0	9
+2081097.01	Electronics                                       	cameras                                           	0	10
+2055973.50	Electronics                                       	automotive                                        	0	11
+2021900.56	Electronics                                       	audio                                             	0	12
+2006817.42	Electronics                                       	memory                                            	0	13
+1729410.62	Electronics                                       	monitors                                          	0	14
+1621759.97	Electronics                                       	wireless                                          	0	15
+1604092.23	Electronics                                       	scanners                                          	0	16
+2580794.04	Home                                              	mattresses                                        	0	1
+2552960.47	Home                                              	glassware                                         	0	2
+2479089.66	Home                                              	lighting                                          	0	3
+2241963.26	Home                                              	bathroom                                          	0	4
+2227778.30	Home                                              	bedding                                           	0	5
+2201961.72	Home                                              	wallpaper                                         	0	6
+2125093.38	Home                                              	rugs                                              	0	7
+2029622.97	Home                                              	accent                                            	0	8
+2005244.71	Home                                              	paint                                             	0	9
+2004693.90	Home                                              	tables                                            	0	10
+1882095.92	Home                                              	blinds/shades                                     	0	11
+1833537.38	Home                                              	decor                                             	0	12
+1805505.20	Home                                              	kids                                              	0	13
+1761357.98	Home                                              	curtains/drapes                                   	0	14
+1684930.79	Home                                              	flatware                                          	0	15
+1596468.10	Home                                              	furniture                                         	0	16
+132067.15	Home                                              	NULL	0	17
+2311781.76	Jewelry                                           	custom                                            	0	1
+2143571.63	Jewelry                                           	birdal                                            	0	2
+2116270.85	Jewelry                                           	loose stones                                      	0	3
+2091585.73	Jewelry                                           	diamonds                                          	0	4
+1929363.68	Jewelry                                           	bracelets                                         	0	5
+1896051.77	Jewelry                                           	gold                                              	0	6
+1858969.13	Jewelry                                           	costume                                           	0	7
+1849246.62	Jewelry                                           	mens watch                                        	0	8
+1845580.36	Jewelry                                           	estate                                            	0	9
+1834863.97	Jewelry                                           	earings                                           	0	10
+1809049.82	Jewelry                                           	womens watch                                      	0	11
+1738077.18	Jewelry                                           	semi-precious                                     	0	12
+1736010.32	Jewelry                                           	rings                                             	0	13
+1703934.55	Jewelry                                           	consignment                                       	0	14
+1608464.69	Jewelry                                           	pendants                                          	0	15
+1603208.67	Jewelry                                           	jewelry boxes                                     	0	16
+9509484.62	Men                                               	sports-apparel                                    	0	1
+8897383.77	Men                                               	pants                                             	0	2
+8508441.20	Men                                               	accessories                                       	0	3
+7952307.23	Men                                               	shirts                                            	0	4
+51160.52	Men                                               	NULL	0	5
+8668502.54	Music                                             	rock                                              	0	1
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q87.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q87.sql.out
new file mode 100644
index 0000000000000..cc6cf7d448318
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q87.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<count(1):bigint>
+-- !query output
+47170
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q88.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q88.sql.out
new file mode 100644
index 0000000000000..f2f9cd0812e84
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q88.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<h8_30_to_9:bigint,h9_to_9_30:bigint,h9_30_to_10:bigint,h10_to_10_30:bigint,h10_30_to_11:bigint,h11_to_11_30:bigint,h11_30_to_12:bigint,h12_to_12_30:bigint>
+-- !query output
+2358	4664	4828	7447	6997	3886	4073	4923
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q89.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q89.sql.out
new file mode 100644
index 0000000000000..7a48b3b50f536
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q89.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_category:string,i_class:string,i_brand:string,s_store_name:string,s_company_name:string,d_moy:int,sum_sales:decimal(17,2),avg_monthly_sales:decimal(21,6)>
+-- !query output
+Men                                               	shirts                                            	importoimporto #x                                 	bar	Unknown	3	1815.92	4915.190000
+Women                                             	dresses                                           	amalgamalg #x                                     	ation	Unknown	6	1508.07	4586.300000
+Men                                               	shirts                                            	importoimporto #x                                 	ese	Unknown	6	1577.69	4596.057500
+Men                                               	shirts                                            	importoimporto #x                                 	ation	Unknown	6	2077.36	4920.419167
+Men                                               	shirts                                            	importoimporto #x                                 	ese	Unknown	4	1762.17	4596.057500
+Men                                               	shirts                                            	importoimporto #x                                 	ation	Unknown	3	2183.89	4920.419167
+Women                                             	dresses                                           	amalgamalg #x                                     	ation	Unknown	3	1917.37	4586.300000
+Men                                               	shirts                                            	importoimporto #x                                 	ought	Unknown	4	2425.64	5071.261667
+Men                                               	shirts                                            	importoimporto #x                                 	ought	Unknown	2	2438.50	5071.261667
+Women                                             	dresses                                           	amalgamalg #x                                     	eing	Unknown	7	1904.98	4494.382500
+Women                                             	dresses                                           	amalgamalg #x                                     	able	Unknown	4	1994.71	4582.713333
+Men                                               	shirts                                            	importoimporto #x                                 	eing	Unknown	3	2458.87	5035.925833
+Women                                             	dresses                                           	amalgamalg #x                                     	ought	Unknown	3	2138.31	4707.378333
+Men                                               	shirts                                            	importoimporto #x                                 	ought	Unknown	7	2511.79	5071.261667
+Men                                               	shirts                                            	importoimporto #x                                 	eing	Unknown	4	2528.19	5035.925833
+Men                                               	shirts                                            	importoimporto #x                                 	ought	Unknown	5	2641.17	5071.261667
+Men                                               	shirts                                            	importoimporto #x                                 	able	Unknown	7	2360.38	4779.735000
+Men                                               	shirts                                            	importoimporto #x                                 	bar	Unknown	6	2521.43	4915.190000
+Men                                               	shirts                                            	importoimporto #x                                 	ought	Unknown	3	2696.60	5071.261667
+Men                                               	shirts                                            	importoimporto #x                                 	eing	Unknown	7	2689.88	5035.925833
+Women                                             	dresses                                           	amalgamalg #x                                     	ought	Unknown	4	2396.08	4707.378333
+Men                                               	shirts                                            	importoimporto #x                                 	eing	Unknown	6	2725.29	5035.925833
+Women                                             	dresses                                           	amalgamalg #x                                     	able	Unknown	7	2299.82	4582.713333
+Women                                             	dresses                                           	amalgamalg #x                                     	able	Unknown	5	2321.48	4582.713333
+Women                                             	dresses                                           	amalgamalg #x                                     	ought	Unknown	7	2449.19	4707.378333
+Men                                               	shirts                                            	importoimporto #x                                 	eing	Unknown	2	2784.04	5035.925833
+Women                                             	dresses                                           	amalgamalg #x                                     	ese	Unknown	4	2254.90	4490.582500
+Women                                             	dresses                                           	amalgamalg #x                                     	ese	Unknown	3	2257.13	4490.582500
+Women                                             	dresses                                           	amalgamalg #x                                     	ese	Unknown	6	2257.18	4490.582500
+Women                                             	dresses                                           	amalgamalg #x                                     	bar	Unknown	4	1998.31	4205.080833
+Men                                               	shirts                                            	importoimporto #x                                 	ese	Unknown	2	2393.90	4596.057500
+Women                                             	dresses                                           	amalgamalg #x                                     	eing	Unknown	6	2308.58	4494.382500
+Men                                               	shirts                                            	importoimporto #x                                 	eing	Unknown	1	2856.92	5035.925833
+Women                                             	dresses                                           	amalgamalg #x                                     	able	Unknown	3	2408.40	4582.713333
+Women                                             	dresses                                           	amalgamalg #x                                     	ese	Unknown	5	2317.37	4490.582500
+Men                                               	shirts                                            	importoimporto #x                                 	ese	Unknown	7	2439.68	4596.057500
+Women                                             	dresses                                           	amalgamalg #x                                     	ese	Unknown	7	2341.06	4490.582500
+Men                                               	shirts                                            	importoimporto #x                                 	ese	Unknown	5	2457.43	4596.057500
+Women                                             	dresses                                           	amalgamalg #x                                     	ought	Unknown	6	2577.66	4707.378333
+Women                                             	dresses                                           	amalgamalg #x                                     	bar	Unknown	3	2097.37	4205.080833
+Women                                             	dresses                                           	amalgamalg #x                                     	eing	Unknown	5	2398.63	4494.382500
+Men                                               	shirts                                            	importoimporto #x                                 	ation	Unknown	4	2841.78	4920.419167
+Men                                               	shirts                                            	importoimporto #x                                 	bar	Unknown	2	2884.81	4915.190000
+Women                                             	dresses                                           	amalgamalg #x                                     	ation	Unknown	4	2557.46	4586.300000
+Men                                               	shirts                                            	importoimporto #x                                 	bar	Unknown	1	2926.69	4915.190000
+Women                                             	dresses                                           	amalgamalg #x                                     	eing	Unknown	3	2528.38	4494.382500
+Women                                             	dresses                                           	amalgamalg #x                                     	able	Unknown	2	2616.82	4582.713333
+Men                                               	shirts                                            	importoimporto #x                                 	bar	Unknown	4	2956.00	4915.190000
+Men                                               	shirts                                            	importoimporto #x                                 	able	Unknown	5	2829.72	4779.735000
+Women                                             	dresses                                           	amalgamalg #x                                     	ese	Unknown	2	2586.26	4490.582500
+Men                                               	shirts                                            	importoimporto #x                                 	ese	Unknown	3	2728.14	4596.057500
+Men                                               	shirts                                            	importoimporto #x                                 	able	Unknown	2	2913.44	4779.735000
+Men                                               	shirts                                            	importoimporto #x                                 	ought	Unknown	6	3210.96	5071.261667
+Women                                             	dresses                                           	amalgamalg #x                                     	eing	Unknown	4	2647.58	4494.382500
+Women                                             	dresses                                           	amalgamalg #x                                     	bar	Unknown	7	2407.85	4205.080833
+Women                                             	dresses                                           	amalgamalg #x                                     	bar	Unknown	6	2427.96	4205.080833
+Women                                             	dresses                                           	amalgamalg #x                                     	ought	Unknown	2	2933.33	4707.378333
+Women                                             	dresses                                           	amalgamalg #x                                     	ought	Unknown	5	2946.43	4707.378333
+Women                                             	dresses                                           	amalgamalg #x                                     	bar	Unknown	2	2465.92	4205.080833
+Men                                               	shirts                                            	importoimporto #x                                 	able	Unknown	4	3120.21	4779.735000
+Men                                               	shirts                                            	importoimporto #x                                 	able	Unknown	7	868.21	2511.135833
+Men                                               	shirts                                            	importoimporto #x                                 	ese	Unknown	6	985.62	2603.736667
+Women                                             	dresses                                           	amalgamalg #x                                     	bar	Unknown	5	2594.86	4205.080833
+Women                                             	dresses                                           	amalgamalg #x                                     	ation	Unknown	5	2992.12	4586.300000
+Men                                               	shirts                                            	importoimporto #x                                 	able	Unknown	3	3197.55	4779.735000
+Men                                               	shirts                                            	importoimporto #x                                 	ation	Unknown	7	3347.44	4920.419167
+Women                                             	dresses                                           	amalgamalg #x                                     	ation	Unknown	7	3059.37	4586.300000
+Men                                               	shirts                                            	importoimporto #x                                 	ation	Unknown	5	3402.55	4920.419167
+Women                                             	dresses                                           	amalgamalg #x                                     	able	Unknown	6	3085.16	4582.713333
+Men                                               	shirts                                            	importoimporto #x                                 	bar	Unknown	7	3426.04	4915.190000
+Women                                             	dresses                                           	amalgamalg #x                                     	eing	Unknown	2	3013.27	4494.382500
+Women                                             	dresses                                           	amalgamalg #x                                     	bar	Unknown	2	512.15	1992.243333
+Men                                               	shirts                                            	importoimporto #x                                 	ese	Unknown	2	1143.39	2603.736667
+Women                                             	dresses                                           	amalgamalg #x                                     	able	Unknown	2	652.58	2089.189167
+Women                                             	dresses                                           	amalgamalg #x                                     	able	Unknown	3	656.84	2089.189167
+Women                                             	dresses                                           	amalgamalg #x                                     	bar	Unknown	3	564.90	1992.243333
+Men                                               	shirts                                            	importoimporto #x                                 	bar	Unknown	5	3491.77	4915.190000
+Men                                               	shirts                                            	importoimporto #x                                 	ation	Unknown	3	1075.39	2478.654167
+Women                                             	dresses                                           	amalgamalg #x                                     	ese	Unknown	1	3107.05	4490.582500
+Men                                               	shirts                                            	importoimporto #x                                 	ought	Unknown	2	1061.74	2425.509167
+Men                                               	shirts                                            	importoimporto #x                                 	eing	Unknown	7	1040.11	2396.908333
+Men                                               	shirts                                            	importoimporto #x                                 	ese	Unknown	1	3253.77	4596.057500
+Men                                               	shirts                                            	importoimporto #x                                 	able	Unknown	2	1181.12	2511.135833
+Men                                               	shirts                                            	importoimporto #x                                 	ation	Unknown	2	3598.63	4920.419167
+Women                                             	dresses                                           	amalgamalg #x                                     	eing	Unknown	3	758.25	2074.660833
+Women                                             	dresses                                           	amalgamalg #x                                     	ese	Unknown	5	782.25	2094.761667
+Men                                               	shirts                                            	importoimporto #x                                 	ese	Unknown	4	1293.95	2603.736667
+Women                                             	dresses                                           	amalgamalg #x                                     	ation	Unknown	2	3282.12	4586.300000
+Men                                               	shirts                                            	importoimporto #x                                 	able	Unknown	3	1216.42	2511.135833
+Men                                               	shirts                                            	importoimporto #x                                 	able	Unknown	6	3503.61	4779.735000
+Men                                               	shirts                                            	importoimporto #x                                 	ese	Unknown	1	1333.77	2603.736667
+Men                                               	shirts                                            	importoimporto #x                                 	able	Unknown	6	1243.07	2511.135833
+Women                                             	dresses                                           	amalgamalg #x                                     	ought	Unknown	2	760.60	2026.577500
+Men                                               	shirts                                            	importoimporto #x                                 	bar	Unknown	4	970.79	2234.079167
+Women                                             	dresses                                           	amalgamalg #x                                     	ese	Unknown	4	832.51	2094.761667
+Men                                               	shirts                                            	importoimporto #x                                 	ation	Unknown	6	1226.01	2478.654167
+Men                                               	shirts                                            	importoimporto #x                                 	ought	Unknown	6	1205.46	2425.509167
+Women                                             	dresses                                           	amalgamalg #x                                     	ation	Unknown	4	595.85	1797.562500
+Men                                               	shirts                                            	importoimporto #x                                 	bar	Unknown	2	1036.00	2234.079167
+Women                                             	dresses                                           	amalgamalg #x                                     	eing	Unknown	4	878.88	2074.660833
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q9.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q9.sql.out
new file mode 100644
index 0000000000000..3331c24a2af4c
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q9.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<bucket1:decimal(11,6),bucket2:decimal(11,6),bucket3:decimal(11,6),bucket4:decimal(11,6),bucket5:decimal(11,6)>
+-- !query output
+358.920754	1040.839473	1718.531826	2401.883865	3085.520204
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q90.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q90.sql.out
new file mode 100644
index 0000000000000..96a51e152051f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q90.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<am_pm_ratio:decimal(35,20)>
+-- !query output
+0.58933333333333333333
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q91.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q91.sql.out
new file mode 100644
index 0000000000000..4f59414d82c51
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q91.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<Call_Center:string,Call_Center_Name:string,Manager:string,Returns_Loss:decimal(17,2)>
+-- !query output
+
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q92.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q92.sql.out
new file mode 100644
index 0000000000000..e01ed10078cd6
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q92.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<Excess Discount Amount :decimal(17,2)>
+-- !query output
+39037.04
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q93.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q93.sql.out
new file mode 100644
index 0000000000000..c311fa1002618
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q93.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<ss_customer_sk:int,sumsales:decimal(28,2)>
+-- !query output
+108	NULL
+331	NULL
+533	NULL
+564	NULL
+1391	NULL
+1680	NULL
+2096	NULL
+2534	NULL
+2766	NULL
+2857	NULL
+3400	NULL
+3565	NULL
+4075	NULL
+4516	NULL
+4888	NULL
+4980	NULL
+5111	NULL
+5626	NULL
+5743	NULL
+6078	NULL
+6760	NULL
+6910	NULL
+7034	NULL
+7770	NULL
+7791	NULL
+8140	NULL
+8364	NULL
+8725	NULL
+8913	NULL
+8938	NULL
+9203	NULL
+9220	NULL
+9860	NULL
+10269	NULL
+11218	NULL
+11625	NULL
+12405	NULL
+12935	NULL
+12954	NULL
+13050	NULL
+13298	NULL
+13626	NULL
+13730	NULL
+14105	NULL
+14422	NULL
+14587	NULL
+14627	NULL
+14734	NULL
+14958	NULL
+15599	NULL
+16142	NULL
+16759	NULL
+16918	NULL
+17479	NULL
+18540	NULL
+18710	NULL
+19327	NULL
+19514	NULL
+19917	NULL
+20571	NULL
+21532	NULL
+22571	NULL
+23925	NULL
+25168	NULL
+25250	NULL
+25414	NULL
+26175	NULL
+26407	NULL
+27166	NULL
+27240	NULL
+28852	NULL
+29635	NULL
+30207	NULL
+30278	NULL
+30438	NULL
+30445	NULL
+31204	NULL
+31911	NULL
+31987	NULL
+32147	NULL
+32154	NULL
+32169	NULL
+32291	NULL
+32639	NULL
+32646	NULL
+33232	NULL
+33313	NULL
+33872	NULL
+33951	NULL
+34860	NULL
+35502	NULL
+35684	NULL
+35774	NULL
+35779	NULL
+35957	NULL
+36221	NULL
+36333	NULL
+36556	NULL
+37254	NULL
+37411	NULL
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q94.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q94.sql.out
new file mode 100644
index 0000000000000..0c727d0b36846
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q94.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<order count :bigint,total shipping cost :decimal(17,2),total net profit :decimal(17,2)>
+-- !query output
+38	84308.56	21690.18
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q95.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q95.sql.out
new file mode 100644
index 0000000000000..9558536825f1b
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q95.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<order count :bigint,total shipping cost :decimal(17,2),total net profit :decimal(17,2)>
+-- !query output
+101	237942.25	-27171.32
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q96.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q96.sql.out
new file mode 100644
index 0000000000000..574180c4d661f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q96.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<count(1):bigint>
+-- !query output
+888
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q97.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q97.sql.out
new file mode 100644
index 0000000000000..34652db5ee36f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q97.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<store_only:bigint,catalog_only:bigint,store_and_catalog:bigint>
+-- !query output
+537833	285408	200
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q98.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q98.sql.out
new file mode 100644
index 0000000000000..03160c0d31d44
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q98.sql.out
@@ -0,0 +1,2534 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_item_desc:string,i_category:string,i_class:string,i_current_price:decimal(7,2),itemrevenue:decimal(17,2),revenueratio:decimal(38,17)>
+-- !query output
+NULL	Books                                             	NULL	NULL	557.55	2.41577919115267179
+NULL	Books                                             	NULL	6.35	361.02	1.56424464817493959
+Precisely elderly bodies	Books                                             	arts                                              	1.40	4303.08	1.75015577438987686
+Abilities could affect cruel parts. Predominantly other events telephone strong signs. Accurate mate	Books                                             	arts                                              	25.69	9236.96	3.75687156218529913
+Germa	Books                                             	arts                                              	5.82	3191.92	1.29822295179047002
+Great, contemporary workers would not remove of course cultural values. Then due children might see positive seconds. Significant problems w	Books                                             	arts                                              	0.55	2096.07	0.85251703756969175
+Small objects stop etc mediterranean patterns; liberal, free initiatives would not leave less clear british attitudes; good, blue relationships find softly very	Books                                             	arts                                              	58.41	5760.88	2.34307458786895754
+Newly national rights head curiously all electrical cells. Chinese, long values might not pull bad lines. High fun clothes ough	Books                                             	arts                                              	3.28	571.10	0.23227873122369528
+Forward psychological plants establish closely yet eastern changes. Likewise necessary techniques might drop. Pleasant operations like lonely things; dogs let regions. Forces might not result clearl	Books                                             	arts                                              	2.43	3457.82	1.40637023708618106
+Early, short v	Books                                             	arts                                              	75.57	850.08	0.34574593563060564
+Black, following services justify by a investors; dirty, different charts will fly however prizes. Temporary, l	Books                                             	arts                                              	5.56	4798.20	1.95153179505784395
+Scientific, difficult polls would not achieve. Countries reach of course. Bad, new churches realize most english	Books                                             	arts                                              	3.98	17.50	0.00711762878027433
+United, important objectives put similarly large, previous phenomena; old, present days receive. Happy detectives assi	Books                                             	arts                                              	1.26	11646.43	4.73685516316858938
+Naturally new years put serious, negative vehicles. Fin	Books                                             	arts                                              	3.34	20302.86	8.25761260902173683
+Ago correct profits must not handle else. Healthy children may not go only ancient words. Later just characters ought to drink about. British parts must watch soon ago other clients. So vital d	Books                                             	arts                                              	4.03	9410.73	3.82754758236520025
+Much new waters 	Books                                             	arts                                              	1.85	2864.61	1.16509889030066491
+Tall, following actions keep widely willing, secondary groups. Heads could afford however; agricultural, square pri	Books                                             	arts                                              	9.99	1903.68	0.77426786036757875
+Anonymous, useful women provoke slightly present persons. Ideas ought to cost almost competent, working parties; aspects provide thr	Books                                             	arts                                              	6.73	7841.73	3.18940132200803357
+Powerful walls will find; there scottish decades must not	Books                                             	arts                                              	4.16	5934.47	2.41367739815283298
+Too executive doors progress mainly seemingly possible parts; hundreds stay virtually simple workers. Sola	Books                                             	arts                                              	34.32	10139.18	4.12382396436467639
+Careful privileges ought to live rather to a boards. Possible, broad p	Books                                             	arts                                              	3.93	10008.95	4.07085660459009779
+Aside legitimate decisions may not stand probably sexual g	Books                                             	arts                                              	3.88	874.84	0.35581636355058234
+Specially interesting crews continue current, foreign directions; only social men would not call at least political children; circumstances could not understand now in a assessme	Books                                             	arts                                              	2.13	15380.51	6.25558632178840388
+Unlikely states take later in general extra inf	Books                                             	arts                                              	0.32	6478.12	2.63479162023261224
+Sometimes careful things state probably so	Books                                             	arts                                              	5.08	17118.92	6.96263529595507190
+Circumstances would not use. Principles seem writers. Times go from a hands. Members find grounds. Central, only teachers pursue properly into a p	Books                                             	arts                                              	5.95	7863.28	3.19816617344888566
+Inches may lose from a problems. Firm, other corporations shall protect ashamed, important practices. Materials shall not make then by a police. Weeks used	Books                                             	arts                                              	0.84	6519.57	2.65165023240074772
+Systems cannot await regions. Home appropr	Books                                             	arts                                              	7.30	889.75	0.36188058327137607
+Relevant lips take so sure, manufacturing 	Books                                             	arts                                              	8.80	932.33	0.37919879089789497
+Extra, primitive weeks look obviou	Books                                             	arts                                              	1.18	10006.38	4.06981132996350893
+More than key reasons should remain. Words used to offer slowly british	Books                                             	arts                                              	0.28	1075.86	0.43757554854548205
+Children may turn also above, historical aspects. Surveys migh	Books                                             	arts                                              	7.22	3864.60	1.57181646767132336
+Trustees know operations. Now past issues cut today german governments. British lines go critical, individual structures. Tonight adequate problems should no	Books                                             	arts                                              	4.05	3061.36	1.24512137387317768
+Useful observers start often white colleagues; simple pro	Books                                             	arts                                              	3.47	724.40	0.29462915933889837
+Members should say earnings. Detailed departments would not move just at the hopes. Figures can take. Actually open houses want. Good teachers combine the	Books                                             	arts                                              	3.09	7805.88	3.17482035104958588
+Major, senior words afford economic libraries; successful seconds need outside. Clinical, new ideas put now red c	Books                                             	arts                                              	5.87	616.62	0.25079270048530027
+Likely states feel astonishingly working roads. Parents put so somewhere able policies. Others may rely shortly instead interesting bodies; bri	Books                                             	arts                                              	7.50	8829.29	3.59106334933647431
+Floors could not go only for a years. Special reasons shape consequently black, concerned instances. Mutual depths encourage both simple teachers. Cards favour massive 	Books                                             	arts                                              	1.83	6344.62	2.58049428068023382
+Accurate years want then other organisations. Simple lines mean as well so red results. Orthodox, central scales will not in	Books                                             	arts                                              	7.69	3640.38	1.48062134052200283
+Certain customers think exactly already necessary factories. Awkward doubts shall not forget fine	Books                                             	arts                                              	0.30	12867.38	5.23344195512721440
+Weak effects set far in the effects. Positive, true classes sell frankly ever open studies. Unique problems must mean as yet new genes. Essential businesses agree deep current stages. Not	Books                                             	arts                                              	4.40	4471.87	1.81880632077973420
+Visitors could not allow glad wages. Communist, real figures used to apply factors. Aggressive, optimistic days must mean about trees. Detailed courts consider really large pro	Books                                             	arts                                              	9.08	1737.00	0.70647549664780021
+Deep, big areas take for a facilities. Words could replace certainly cases; lights test. Nevertheless practical arts cross. Fa	Books                                             	arts                                              	7.37	8016.08	3.26031324074179520
+Similar situations come separate programmes. National, large others could not ask opportunities. Severe, large findings accept; twins go more. Tiny rights may see specifi	Books                                             	arts                                              	1.27	2413.11	0.98146406776958731
+Natural plans might not like n	Books                                             	business                                          	4.29	8813.76	2.98246237973343018
+Years shall want free objects. Old residents use absolutely so residential steps. Letters will share variables. Sure fres	Books                                             	business                                          	40.76	1861.77	0.62999888636816844
+Whole, important problems make. Indeed industrial members go skills. Soft	Books                                             	business                                          	3.22	6877.01	2.32709123121693768
+Cheap depths calm as in a traditi	Books                                             	business                                          	7.92	2554.82	0.86451804190159048
+Simple, great shops glance from a years. Lessons deepen here previous clients. Increased, silent flights open more great rocks. Brill	Books                                             	business                                          	8.92	8014.35	2.71195237594586375
+Just sudden ideas ought to serve full sources; uncertain, open qualifications shout questions; chronic, informal	Books                                             	business                                          	4.62	4172.49	1.41191664565564981
+Groups must not put new, civil moves. Correct men laugh slightly total novels. Relatively public girls set even scott	Books                                             	business                                          	3.36	414.96	0.14041709657333354
+Just young degrees stop posts. More than tight artists buy to a arts. European, essential techniques ought to sell to a offences. Sentences be	Books                                             	business                                          	2.58	6286.70	2.12733796276165399
+Other, black houses flow. New soldiers put only eastern hours. Applications reserve there methods; sources cry pretty scarcely special workers. Never british opportunities 	Books                                             	business                                          	8.20	3426.05	1.15933100471146462
+Junior, severe restrictions ought to want principles. Sure,	Books                                             	business                                          	9.77	3899.69	1.31960465427044307
+Rows could not	Books                                             	business                                          	1.65	15875.48	5.37205708576254358
+Remaining subjects handle even only certain ladies; eagerly literary days could not provide. Very different articles cut then. Boys see out of a houses. Governme	Books                                             	business                                          	9.03	916.17	0.31002007751973922
+Manufacturing, ready concerns see already then new pupils. Both stable types used to manage otherw	Books                                             	business                                          	1.18	8723.00	2.95175036969632840
+Russian windows should see in a weapons. New, considerable branches walk. English regions apply neither alone police; very new years w	Books                                             	business                                          	2.79	8434.64	2.85417307557668685
+Long groups used to create more tiny feet; tools used to dare still	Books                                             	business                                          	57.04	795.66	0.26924105229308502
+Drugs must compensate dark, modest houses. Small pubs claim naturally accessible relationships. Distinguished	Books                                             	business                                          	1.66	11559.25	3.91150068335575881
+Small, capable centres	Books                                             	business                                          	2.98	19190.81	6.49392187461561344
+Popular, different parameters might take open, used modules. Prisoners use pretty alternative lovers. Annual, professional others spend once true men. Other, small subsidies seem politically	Books                                             	business                                          	7.25	1761.61	0.59610603791823330
+Supreme, free uses handle even in the customers. Other minutes might not make of course social neighbours. So environmental rights come other, able sales	Books                                             	business                                          	8.08	3500.18	1.18441563785437289
+Sound, original activities consider quite to a attitudes. In order weak improvements marry available, hard studie	Books                                             	business                                          	71.27	11431.86	3.86839355512056274
+Always other hours used to use. Women should jump then. Civil samples take therefore other offices. Concrete, major demands	Books                                             	business                                          	1.42	2038.40	0.68976819369356825
+Changes ensure different clients. Distinct, alone attacks think directly previous feelings. White children tell so medieval, responsible yea	Books                                             	business                                          	5.89	5116.38	1.73131681262259552
+Visual fragments 	Books                                             	business                                          	6.77	2739.02	0.92684893931051673
+Classic issues will draw as european, engl	Books                                             	business                                          	75.64	14735.99	4.98646840884344817
+Again british shareholders see shares. American lives ought to establish horses. Then ideal conservatives might charge even nec	Books                                             	business                                          	2.44	9396.38	3.17961345165736401
+Critical cases tell anywhere to the circumstances. Dependent, new numbers must not	Books                                             	business                                          	3.72	726.75	0.24592279963049486
+Confident, video-tape	Books                                             	business                                          	3.17	6173.95	2.08918482116091330
+Of course fundamental children will not deal still including a suppliers. More crucial powers will not keep enough. As good comments used to devote even convenient electric problems. Publi	Books                                             	business                                          	8.85	2672.80	0.90444094785330122
+Departments could seek now for a commu	Books                                             	business                                          	5.93	3205.71	1.08477079876638965
+Paintings must not know primary, royal stands; similar, available others ough	Books                                             	business                                          	0.39	12939.97	4.37871847201185356
+Most present eyes restore fat, central relationships; again considerable habits must face in a discussions. Engineers help at all direct occasions. Curiously del	Books                                             	business                                          	80.10	6877.89	2.32738901183430931
+So white countries could secure more angry items. National feet must not defend too by the types; guidelines would not view more so flexible authorities. Critics will handle closely lig	Books                                             	business                                          	2.50	2135.27	0.72254774869901171
+Simple changes ought to vote almost sudden techniques. Partial, golden faces mean in a officials; vertically minor 	Books                                             	business                                          	8.73	5996.87	2.02926323965617573
+Christian lines stand once deep formal aspirations. National, fine islands play together with a patterns. New journals lose etc positive armie	Books                                             	business                                          	4.89	6106.50	2.06636061361350790
+Children would not mean in favour of a parts. Heavy, whole others shall mean on	Books                                             	business                                          	3.13	5521.91	1.86854291917113983
+Lips will n	Books                                             	business                                          	8.48	7806.43	2.64159493735051117
+White fees might combine reports. Tr	Books                                             	business                                          	2.09	6566.98	2.22218108939451963
+Asleep children invite more. Wealthy forms could expect as. Indeed statistical examinations could la	Books                                             	business                                          	3.71	11535.83	3.90357565828889099
+Most new weeks go yet members. Also encouraging delegates make publications. Different competitors run resources; somehow common views m	Books                                             	business                                          	1.07	9334.32	3.15861315039135987
+Local, bloody names 	Books                                             	business                                          	4.40	2927.75	0.99071273012477651
+Large, larg	Books                                             	business                                          	3.50	5826.76	1.97170021599584758
+Only new systems might join late speeches. Materials could stay on a benefits. Corporate regulations must crawl definitely practical deaths. Windows might soothe despite a organisations. Old	Books                                             	business                                          	0.67	123.41	0.04176034771571981
+Professional managers form later initial grounds. Conscious, big risks restore. American, full rises say for a systems. Already	Books                                             	business                                          	5.27	1616.40	0.54696885218126163
+Memories can earn particularly over quick contexts; alone differences make separate years; irish men mea	Books                                             	business                                          	4.23	6855.84	2.31992757704675870
+Only, gothic	Books                                             	business                                          	1.68	7807.37	2.64191302119179451
+Low, large clouds will not visit for example as the notions. Small, unacceptable drugs might not negotiate environmental, happy keys.	Books                                             	business                                          	3.11	3933.56	1.33106582416859905
+Silver, critical operations could help howev	Books                                             	business                                          	5.56	9087.69	3.07515674850230731
+Terrible, psychiatric bones will destroy also used studies; solely usual windows should not make shares. Advances continue sufficiently. As key days might not use far artists. Offici	Books                                             	business                                          	5.83	2024.32	0.68500370381562209
+Too white addresses end by the talks. Hands get only companies. Statements know. Sentences would pay around for a payments; papers wait actually drinks; men would 	Books                                             	business                                          	6.06	5178.86	1.75245923645598158
+New, big arguments may not win since by a tenant	Books                                             	computers                                         	1.00	1686.72	0.45326198032962534
+Else substantial problems slip months. Just unique corporations put vast areas. Supporters like far perfect chapters. Now young reports become wrong trials. Available ears shall	Books                                             	computers                                         	51.46	471.00	0.12656895793922734
+Cheap, desirable members take immediate, estimated debts; months must track typica	Books                                             	computers                                         	3.26	7226.83	1.94202195818247621
+Expert, scottish terms will ask quiet demands; poor bits attempt northern, dangerous si	Books                                             	computers                                         	2.66	4463.91	1.19955931429829364
+Gradually serious visitors bear no doubt technical hearts. Critics continue earlier soviet, standard minute	Books                                             	computers                                         	6.40	3244.45	0.87186126451364360
+Clear, general goods must know never women. Communications meet about. Other rewards spot wide in a skills. Relative, empty drawings facilitate too rooms. Still asian police end speedily comp	Books                                             	computers                                         	7.64	5385.04	1.44708896233770017
+Wide, essential activities make steadily procedures. Modules	Books                                             	computers                                         	35.95	7285.54	1.95779873848101557
+At least remaining results shall keep cuts. Clients should meet policies. Glorious, local times could use enough; clever styles will live political parents. Single, gradual contracts will describe ho	Books                                             	computers                                         	9.51	14393.77	3.86795004186180949
+Environmental, new women pay again fingers. Different, uncomfortable records miss far russian, dependent members. Enough double men will go here immediatel	Books                                             	computers                                         	89.89	9487.37	2.54948308807619376
+Years learn here. Days make too. Only moving systems avoid old groups; short movements cannot see respectiv	Books                                             	computers                                         	0.60	1706.66	0.45862033493962150
+Magnetic	Books                                             	computers                                         	57.19	7638.33	2.05260184394042112
+Ga	Books                                             	computers                                         	5.53	7904.13	2.12402865714688954
+S	Books                                             	computers                                         	65.78	578.19	0.15537347301673430
+Simple year	Books                                             	computers                                         	3.01	5038.44	1.35394925783295241
+Agricultural players shall smoke. So full reasons undertake 	Books                                             	computers                                         	0.70	5739.18	1.54225484506508439
+Then basic years can encourage later traditions. For example christian parts subscribe informal, valuable gr	Books                                             	computers                                         	2.75	11359.94	3.05268740563088364
+Boxes batt	Books                                             	computers                                         	0.83	6659.54	1.78957757569979198
+Blocks extend ev	Books                                             	computers                                         	9.29	11249.90	3.02311702743208836
+Separate, dead buildings think possibly english, net policies. Big divisions can use almost	Books                                             	computers                                         	9.46	3529.22	0.94838577014496795
+Artists make times. Rather ready functions must pre	Books                                             	computers                                         	5.71	7805.93	2.09763996995021836
+Limited, capable cities shall try during the bodies. Specially economic services ought to prevent old area	Books                                             	computers                                         	2.93	6458.26	1.73548882866368225
+Since other birds shall blame sudden	Books                                             	computers                                         	6.74	2404.97	0.64627292308939187
+Legs throw then. Old-fashioned develo	Books                                             	computers                                         	2.66	12518.22	3.36394492707854445
+Only careful men define judicial, special lawyers. Now able funds will not put too black, economic terms. Objectives know both points. Teeth pay.	Books                                             	computers                                         	9.85	911.50	0.24494183686115864
+Men should not turn shadows. Different, single concessions guarantee only therefore alone products.	Books                                             	computers                                         	8.38	11864.77	3.18834729318175442
+Educational, white teachers should not fix. Considerable, other services might not cover today on a forms. Successful genes fall otherwise so	Books                                             	computers                                         	1.65	7042.80	1.89256869845942737
+Women note days. Other, efficient qualificati	Books                                             	computers                                         	7.64	8012.26	2.15308577269247054
+Present 	Books                                             	computers                                         	2.84	4786.32	1.28619858760866792
+Multiple, dark feet mean more complex girls; schools may not answer frequently blue assets. Spiritual, dry patients may reply personnel	Books                                             	computers                                         	2.04	2973.19	0.79896721880112808
+Private teachers ap	Books                                             	computers                                         	5.27	8109.35	2.17917617635769258
+Daily numbers sense interesting players. General advantages would speak here. Shelves shall know with the reductions. Again wrong mothers provide ways; as hot pr	Books                                             	computers                                         	7.56	13142.36	3.53166626340166409
+Inc, corporate ships slow evident degrees. Chosen, acute prices throw always. Budgets spend points. Commonly large events may mean. Bottles c	Books                                             	computers                                         	68.38	12405.10	3.33354687926095337
+European, possible problems ought to restore then unfair interests. States would sleep in respect of the questions. Ideal stages affect only pressures. About spanish employees might kno	Books                                             	computers                                         	3.42	6760.19	1.81662463645686890
+Upper others narrow deaths. Situations could e	Books                                             	computers                                         	5.42	10932.74	2.93788855460829783
+However old hours ma	Books                                             	computers                                         	8.84	5208.75	1.39971562561772907
+Indeed other actions should provide after a ideas; exhibitio	Books                                             	computers                                         	6.95	3949.76	1.06139491997885895
+Effective females must answer too english projects. Firm, political experiments see in terms of 	Books                                             	computers                                         	0.76	246.87	0.06633986973770075
+Of course responsible fears tell. Now clear substances might develop at least independent civil tourists.	Books                                             	computers                                         	4.95	619.44	0.16645833398274943
+Perfect days find at all. Crimes might develop hopes. Much socialist grants drive current, useful walls. Emissions open naturally. Combinations shall not know. Tragic things shall not receive just	Books                                             	computers                                         	6.71	8038.78	2.16021233057898500
+Advantages apply almost on a services; materials defeat today individual ideas. Domestic divisions used to win smoothly irish talks. Subsequent quantities make only, automatic pounds. Flower	Books                                             	computers                                         	7.87	442.26	0.11884583298981461
+Close, historic tactics lead ago large, typical stars. Generally significant facilities check leading years; yesterday general years 	Books                                             	computers                                         	3.87	8448.38	2.27028164092273769
+Internal seats used to sell dark words. Universal items show now in the roles. Most wonderf	Books                                             	computers                                         	2.57	870.24	0.23385428865612144
+Likely, separate attacks prefer seats. Informally equal women could use easy prime, big forces. Long technical women save conditions; fast alone rooms sell. Ne	Books                                             	computers                                         	3.77	344.40	0.09254851191989362
+Economic customs should not put unpleasant shops. Colonial, middle goods used to see. Closely explicit legs continue	Books                                             	computers                                         	3.32	8481.54	2.27919252551990282
+Human windows take right, variable steps. Years should buy often. Indeed thin figures may beat even up to a cars. Details may tell enough. Impossible, sufficient differences ought to return 	Books                                             	computers                                         	4.47	5466.16	1.46888784528468556
+Left diff	Books                                             	computers                                         	0.74	3269.32	0.87854442796151745
+Male levels shall reduce else high, local conditions; further personal agencies control. Successful days wake eve	Books                                             	computers                                         	6.55	2376.38	0.63859010672531010
+Wide governments conform widely in proportion to a friends. So living rooms wear too clothes; most essential measures should not bring previously pains. Real accounts become also gue	Books                                             	computers                                         	9.35	11110.42	2.98563541755233586
+Places transform 	Books                                             	computers                                         	3.10	5805.20	1.55999599708875273
+Appropriate effects beg passages. Running contracts must keep only upper sons. Safely available reports intend perhaps w	Books                                             	computers                                         	5.81	8969.60	2.41034591323077181
+Friendly, hot computers tax elsewhere units. New, real officials should l	Books                                             	computers                                         	3.19	2999.57	0.80605615534133364
+Perfect members state democratic schools. Genuine, enormous knees must afford around the implications. Matters will indicate with a months. Still regular machines would not l	Books                                             	computers                                         	4.07	2110.95	0.56726272136265806
+Kinds relieve really major practices. Then capable reserves could not approve foundations. Pos	Books                                             	computers                                         	7.23	1522.62	0.40916438797755059
+Only increased errors must submit as rich, main 	Books                                             	computers                                         	6.94	8287.27	2.22698753303826016
+Meals ought to test. Round days might need most urban years. Political, english pages must see on a eyes. Only subsequent women may come better methods; difficult, social childr	Books                                             	computers                                         	7.23	15325.54	4.11833891222069241
+Systems cannot see fairly practitioners. Little ca	Books                                             	computers                                         	1.73	2428.71	0.65265242852777245
+Past beautiful others might not like more than legislative, small products. Close, wh	Books                                             	computers                                         	3.02	4174.86	1.12188467036552578
+Main problems wait properly. Everyday, foreign offenders can worry activities. Social, important shoes will afford okay physical parts. Very	Books                                             	computers                                         	1.40	939.26	0.25240161238640906
+Schools offer quickly others. Further main buildings satisfy sadly great, productive figures. Years contribute acti	Books                                             	computers                                         	4.11	885.92	0.23806787944271822
+Chief me	Books                                             	computers                                         	2.62	9675.59	2.60006230094948754
+Tiny, rare leaders mention old, precious areas; students will improve much multiple stars. Even confident solutions will include clearly single women. Please little rights will not mention harder com	Books                                             	computers                                         	1.45	3092.13	0.83092923972956056
+Guidelines should investigate so. Usual personnel look now old, modern aspects. Discussions could appear once br	Books                                             	computers                                         	2.44	9923.72	2.66674076280396839
+Flat pleasant groups would go private, redundant eyes. Main devic	Books                                             	computers                                         	2.83	2445.21	0.65708637291417851
+Popular, obvious copies should believe still difficult parts. Forms ought to soften characteristic	Books                                             	computers                                         	1.05	2156.19	0.57941979069847684
+Real, domestic facilities turn often guilty symptoms. Winds get naturally intense islands. Products shall not travel a little clear shares; improved children may not apply wrong c	Books                                             	computers                                         	5.28	1338.00	0.35955258115219995
+Directions would ask yet profits. Forthcoming, specified discussions ought	Books                                             	cooking                                           	0.58	5750.02	2.05632295473197783
+Then irish champions must look now forward good women. Future, big models sign. Then different o	Books                                             	cooking                                           	85.81	2279.71	0.81527020830049933
+Black ears see sensibly glad months. Equal members must afford approximately o	Books                                             	cooking                                           	8.37	10363.44	3.70617485886789408
+Considerable benefits should govern. Well experienced years provide please in an towns. Exc	Books                                             	cooking                                           	4.18	0.00	0.00000000000000000
+Valuable studies should persist so concerned parties. Always polite songs include then from the holes. There conventional areas might not explain theore	Books                                             	cooking                                           	1.58	1326.45	0.47436523408687831
+Meanings occur in a things. Also essential features may not satisfy by the potatoes; happy words change childre	Books                                             	cooking                                           	3.46	1262.55	0.45151330717055917
+Then dominant goods should combine probably american items. Important artists guess only sill	Books                                             	cooking                                           	6.67	5569.20	1.99165808110116677
+Libraries shall note still. Children would not concentrate. Local, public modes must mean low children. Outer, good years should vis	Books                                             	cooking                                           	1.42	2178.99	0.77925070784648269
+Children ought to miss historical effects. Honest girls may not think activities. Woo	Books                                             	cooking                                           	4.42	348.88	0.12476651428114901
+Single, past rates mark blue, evident discussions. Only literary children ought to publish exactly really recent themes; conscious, ready conditions would adopt advanced, ideal provisions. A	Books                                             	cooking                                           	4.99	9499.97	3.39738059698316657
+Standards could lead no longer ago great tactics; difficult lives might feel french, easy costs. Students drop certainly unabl	Books                                             	cooking                                           	3.05	16321.01	5.83672187356046718
+Individual, remarkable services take by the interest	Books                                             	cooking                                           	6.05	1054.65	0.37716408016112647
+Positions shall	Books                                             	cooking                                           	4.21	2629.53	0.94037288551281172
+Ultimately senior elections marry at l	Books                                             	cooking                                           	5.06	7756.87	2.77401293175881769
+Hence young effects shall not solve however months. In order small activities must not return almost national foods. International decades take contributions. Sessions must see 	Books                                             	cooking                                           	1.43	19276.07	6.89351084309627374
+Members need for a regions. Leading needs go at least under the others; old police could play on a drinks. Very similar machines must consider fully effec	Books                                             	cooking                                           	9.86	10833.86	3.87440652490818908
+Mainly catholic activities could assume just fat, o	Books                                             	cooking                                           	2.68	2262.61	0.80915490391444210
+Points trace so simple eyes. Short advisers shall not say limitations. Keys stretch in full now blue wings. Immediately strategic students would not make strangely for the players.	Books                                             	cooking                                           	1.69	5132.94	1.83564271902740482
+Projects become more from a pupils. Details may precede generally; good, marvellous birds could suffer fair	Books                                             	cooking                                           	9.88	628.36	0.22471419087853357
+Great pp. will not r	Books                                             	cooking                                           	1.91	2941.23	1.05184308300603044
+New, general students raise therefore to a women; united letters would start black positio	Books                                             	cooking                                           	4.03	3747.49	1.34017789670793138
+Products may not resist further specif	Books                                             	cooking                                           	5.37	8721.33	3.11892325153523644
+Dramatic months deal broadly in a films. Almost new occasions may get together sources. Under dry orders wor	Books                                             	cooking                                           	3.92	1412.78	0.50523858073297895
+Thus certain stars appear totally even local guests. Urban friends might not take properly various vehicles	Books                                             	cooking                                           	4.55	1446.44	0.51727607462974425
+Somet	Books                                             	cooking                                           	7.34	6593.72	2.35804706645808830
+Genetic properties might describe therefore leaves; right other organisers must not talk even lives; methods carry thus wrong minutes. Proud worke	Books                                             	cooking                                           	1.08	119.92	0.04288580713309846
+Urgent agencies mean over as a plants; then	Books                                             	cooking                                           	6.47	9566.59	3.42120525067902230
+Men could require evolutionary falls; taxes teach dead parents; only financial servants might not buy eastern things. Different payments develop. New inhabitants might not eat w	Books                                             	cooking                                           	80.50	3855.42	1.37877583836799906
+Hours ought to cope thus into the eyes. Dark states reduce most for the feelings. National, tragic children shall establish enough typical boats. In order secret hours must mean; sin	Books                                             	cooking                                           	2.30	12966.63	4.63712802990534045
+Guests agree around trying, young costs. Here annual banks appeas	Books                                             	cooking                                           	58.88	8031.52	2.87223330308224573
+Wonderful qualities suffer of course light leaders. True clubs used to see early living operat	Books                                             	cooking                                           	9.91	4482.62	1.60307518988467144
+High big appeals may	Books                                             	cooking                                           	36.23	675.62	0.24161531867298181
+Final women should establish on a pupils. Full, northern years might not avoid full	Books                                             	cooking                                           	60.38	5877.02	2.10174071245298770
+Little part	Books                                             	cooking                                           	9.90	4729.36	1.69131438311366337
+Here other affairs afford directly effective leads. Plants cannot undertake as coming, huge photographs; d	Books                                             	cooking                                           	0.87	20785.39	7.43327407210001090
+Stairs might bring early others. Large forms rel	Books                                             	cooking                                           	1.88	2350.18	0.84047169953356678
+Now available m	Books                                             	cooking                                           	3.55	1102.96	0.39444070910208700
+Major instructions put flatly british, other democrats. Operations represent well upon a stores. Thousands will not appear surely 	Books                                             	cooking                                           	1.29	582.88	0.20844962693245854
+New, single products raise too extreme, efficient minutes; hands support leaders. Additional, english things prefer halfway private, slow churches. High white things could f	Books                                             	cooking                                           	4.13	2472.08	0.88406559454294555
+Golden, sure days fill of course. Early free minutes must not express only, cap	Books                                             	cooking                                           	9.44	4521.21	1.61687575106934680
+Purposes hide tears. Small laws award good eyes. 	Books                                             	cooking                                           	55.11	5382.78	1.92499053468895684
+Yet religious animals ensure also. Rough, real heads resist dead. Civil, evolutionary votes dissuade rapidly left cars. Buyers 	Books                                             	cooking                                           	2.20	11624.81	4.15726617427380135
+Here comprehensive years should tend sensibly particular front sales. Official, coherent tears regulate animals. Rewards cannot w	Books                                             	cooking                                           	2.50	2499.59	0.89390372458156745
+Widely splendid others deprive only. Different, main soldiers discover then other periods. Too main birds must change public, terrible houses. Different, armed females may foster; science	Books                                             	cooking                                           	4.26	6853.89	2.45108909816104214
+New women add however. Scottish managers place mostly. Normal, financial purposes should lea	Books                                             	cooking                                           	4.74	319.20	0.11415234853973505
+Extra theories drop. Other resources shall know eventually anyway open students. Long-term, liable di	Books                                             	cooking                                           	6.96	5834.64	2.08658477093947276
+Special, public skills agree at a parent	Books                                             	cooking                                           	5.87	4713.66	1.68569974692295585
+Gentle fans cannot pay else can	Books                                             	cooking                                           	2.45	7576.48	2.70950183478800689
+Sound, new offices might equip hot, new reports; calculations fight great scientists. Professional, little issues learn of c	Books                                             	cooking                                           	66.16	6628.48	2.37047794250834265
+Well angry rebels drop in a methods. Studies argue most sometimes residential organisations. Rural, different children know o	Books                                             	cooking                                           	4.42	453.06	0.16202338041795852
+Half general features used to defend as ready medical pounds. Turkish, trying rooms secure with a ci	Books                                             	cooking                                           	7.08	683.53	0.24444409397670770
+African, elected carers would examine proba	Books                                             	cooking                                           	6.20	15598.69	5.57840569437117702
+Already accessible clubs match all enough o	Books                                             	entertainments                                    	5.00	1196.30	0.46493128593083651
+Likely, various days develop no longer. Officials say before agricultural, rare ho	Books                                             	entertainments                                    	2.67	23516.84	9.13960934734576042
+Less progressive experiences would silence as economic, soviet specialists. Alone legal brothers fight only ears. Methods could not return records. E	Books                                             	entertainments                                    	8.36	5931.28	2.30513887621487248
+Strict heads discuss as categories. Alone, specific markets wait single, human numbers. External, various changes want very relatively nuclear orders. Old, pre	Books                                             	entertainments                                    	2.32	4525.09	1.75863572068274594
+Instances used to lower out of a costs. Irish supporters sign in a types. Bad things shall participate clear 	Books                                             	entertainments                                    	1.58	3570.57	1.38767006737947580
+Trustees may encourage today necessary, political tears; inner, foreign times pay in the historians. Areas may belie	Books                                             	entertainments                                    	1.79	17322.75	6.73233171726021741
+Rare, radical beds say over readers; han	Books                                             	entertainments                                    	7.10	7808.46	3.03468807902658165
+L	Books                                             	entertainments                                    	1.63	4264.23	1.65725481685601518
+Always constitutional advertisements go for a spaces. Cars spend bad difficulties. Rights encourage further great qualities. Blue, high homes would produce u	Books                                             	entertainments                                    	2.63	3974.52	1.54466161878945775
+Companies ought to record now detailed, good roads. Muscles shall not argue significantly territorial months. Clearly new periods could write in a committees. Figures will not find more there	Books                                             	entertainments                                    	3.07	7276.45	2.82792715498740725
+Falsely large trees shall reflect against a 	Books                                             	entertainments                                    	0.70	957.09	0.37196446079707792
+Deep patterns shall find british, american expectations. Sufficient patients must see. English, large assets could not meet for the proceedings. White, chinese matches shal	Books                                             	entertainments                                    	0.56	1499.01	0.58257681762365897
+Particular, deliberate things rain however original ways. Virtually old deaths consider women. Notably w	Books                                             	entertainments                                    	9.71	1611.84	0.62642718708915783
+New, previous police outline right in a persons. Wealthy quest	Books                                             	entertainments                                    	2.07	5037.58	1.95781037146155928
+Doors cannot happen here severe, old rates. Inevitable, int	Books                                             	entertainments                                    	2.29	1047.84	0.40723363591888968
+Limitations respond. Bare rivers will not create yesterday. Well local persons may unders	Books                                             	entertainments                                    	8.95	2096.28	0.81470045646668390
+So perfect changes would twist again; legal standards like improvements; rights used to tell working stations. Official, immediate loans listen much possible pictures. Always d	Books                                             	entertainments                                    	6.32	1017.52	0.39545003933824690
+Prisons take angry, logical sums. Now old grounds cannot help so increased problems. Blue, negative designs would freeze. Small payments ask alike to a hundreds. Exte	Books                                             	entertainments                                    	2.62	11202.91	4.35391068500161131
+High, official employees shall not start too left circumstances. Patients used to touch obviously popular, senior members. British, impossible theories make only. Young, international wo	Books                                             	entertainments                                    	4.85	1041.70	0.40484737988309988
+Now old tears give. Other kids coincide up a animals; specific procedures remove future, french levels. Coming, strong values a	Books                                             	entertainments                                    	5.08	24460.84	9.50648649682223761
+Large women establish today polite, easy horses. Details sha	Books                                             	entertainments                                    	5.06	1748.58	0.67956996401650263
+Plans would not claim; most particular horses will not tell simply cases; more british enquiries could not smile blue men. Old, dangerous play	Books                                             	entertainments                                    	0.95	6942.27	2.69805108950854163
+Pieces threaten	Books                                             	entertainments                                    	0.69	1273.35	0.49487607869266126
+Cases can accept gmt sudden services; tools show all also capable meals; important, spatial days would not happen human, cold backs. Red, economic effects must s	Books                                             	entertainments                                    	9.58	1334.73	0.51873086622959576
+Financial gods might presume divine, tiny 	Books                                             	entertainments                                    	8.42	731.84	0.28442306469583164
+Marginal, available teeth pay recently little services. Then british times could require more scottish fair tea	Books                                             	entertainments                                    	95.74	3018.65	1.17317130007115240
+Now complete others shall pass. Just front advantages could exercise more establish	Books                                             	entertainments                                    	6.51	5281.66	2.05266987849992639
+Young reasons could not say impossible experiences. Prisoners cancel particularly; forms might k	Books                                             	entertainments                                    	3.77	3626.88	1.40955444480216694
+Just particular actions seem very; necessarily poor eleme	Books                                             	entertainments                                    	0.26	6872.96	2.67111437845958545
+Japanese, efficient sports withdraw recently severe days; factors mean originally impossible items. Quiet workers become from a officers. Pieces explore. For example o	Books                                             	entertainments                                    	3.74	16796.75	6.52790652592057016
+Never able feet go on the provisions. Services play brown studies. Cruel,	Books                                             	entertainments                                    	9.79	12846.63	4.99272774870656373
+Internal claims speculate perhaps through a expectations. Immediate courts appeal to a councils; transactions materialise entirely; fine, serious conditions may not use to a types. Short, large lip	Books                                             	entertainments                                    	3.11	5231.34	2.03311346095579892
+Front, possible foundations hear well. Old, close inches change pointedly for the employees; odd, financial members work under on the arrangements; st	Books                                             	entertainments                                    	92.23	225.66	0.08770073893099771
+Local words co	Books                                             	entertainments                                    	2.95	9381.26	3.64594271959501737
+Hardly local women should tell easily tall, able issues. Important, available conditions could no	Books                                             	entertainments                                    	2.21	15740.54	6.11741996442846214
+General, raw tests would not buy heavy, considerable blues. High, regional modules meet often young, responsible calculations. Things hesitat	Books                                             	entertainments                                    	2.00	5567.90	2.16391449212931922
+H	Books                                             	entertainments                                    	4.80	2493.52	0.96908422644341674
+New hours borrow new poets. Youngsters mind especially. Laws must add there in a ends. Factors must not take strategic, royal tr	Books                                             	entertainments                                    	2.30	4109.90	1.59727584389128560
+Clear materials will ship evidently literally scottish targets. Residential heads make prominent times. Internal, open subjects feel subsequent 	Books                                             	entertainments                                    	0.75	263.40	0.10236805208909332
+Other practices get feet. Numbers will not increase now large, simple foreigners. Flowers cover	Books                                             	entertainments                                    	1.00	315.51	0.12262013710945267
+Heavy, formal factors could want then truly serious players. Be	Books                                             	entertainments                                    	4.31	8757.62	3.40357061631163789
+Men call tonight particularly mental lines. Recent markets must dress children. Multiple relations should seem relatively about a arts. Funny, real proteins shall keep citie	Books                                             	entertainments                                    	5.20	3090.94	1.20126616144366780
+Dirty trials should get. Balls shall win later national programmes. Elements ought to explain apart poss	Books                                             	entertainments                                    	1.62	290.34	0.11283804192690719
+Subsequent, 	Books                                             	entertainments                                    	1.29	9603.95	3.73248919461293761
+Countries turn more actually scientific patients. Good writers could not drag perhaps. Suddenly left months cannot announce more overall loans; beds transform far 	Books                                             	entertainments                                    	1.32	2401.56	0.93334479565331415
+Royal, blue men used to convey jobs. Other, technical things would say as mere children; ab	Books                                             	fiction                                           	0.62	555.50	0.18274906106295868
+Exclusively ready fields invest right in a courts. Quite glad facts would not imitate usually by a types. More large managers can continue both small matters. Additional, basic scholars s	Books                                             	fiction                                           	1.11	3969.44	1.30587116641899316
+Dollars get on a years; separate economies can say. Firms know even sons. Simple, definite members will say most cold, big policies; main, true agents might repeat too. Elements know goods. Great 	Books                                             	fiction                                           	5.03	149.04	0.04903135924540659
+Wild officials will not watch onl	Books                                             	fiction                                           	0.47	6954.51	2.28790310108543073
+Just minor eyes exc	Books                                             	fiction                                           	7.11	16681.12	5.48777500896227056
+Married circumstances face human, compulsory hours. Years make sometimes national problems. Difficulties should invest far running, medical centuries; perf	Books                                             	fiction                                           	2.71	10221.52	3.36268799754501063
+Other horses apply able schools; possible enquiries would not describe easily r	Books                                             	fiction                                           	3.83	10067.63	3.31206107944063852
+Firm, local examinations may not sponsor most rural charges. Countries shall help beautiful, different terms	Books                                             	fiction                                           	7.72	5090.34	1.67462620250444840
+As joint men would so	Books                                             	fiction                                           	2.13	2773.11	0.91230107781152357
+Pictures get with a conditions; still gross eyes go that. Personal beings contact thereafter in a systems. New, medium goals might not tell; as official years mu	Books                                             	fiction                                           	5.52	2061.58	0.67822107885899974
+Essential, alternative fans let unlikel	Books                                             	fiction                                           	1.52	2460.17	0.80934969856932323
+Basic changes may not see; afraid names seek in excess of a characteristics. Awful scientists shall not want now right eyes. Here used workers will not pray in part	Books                                             	fiction                                           	2.27	6034.24	1.98515156476786280
+Local companies would restrict yet most imaginative days. Married, str	Books                                             	fiction                                           	99.71	7003.69	2.30408239689654919
+Different stations may smell; weapons disguise cons	Books                                             	fiction                                           	1.47	1671.19	0.54979010505455611
+Private, quiet populations shall receive more somewhat proposed machines. Heads protect abroad parent	Books                                             	fiction                                           	74.86	3243.16	1.06693869464796593
+Circumstances should include parties. Good investigations fall as deposits. Characters might force at all convenient, special years; 	Books                                             	fiction                                           	5.18	12.59	0.00414187340914968
+Old, official cases look enough; actual emotions go statistical, wild limits. Mental cities hear above mod	Books                                             	fiction                                           	2.55	769.44	0.25313130070978025
+Times should not get on a lists; different students undermine suddenly groups. Even actual modules may stay for a 	Books                                             	fiction                                           	8.31	638.38	0.21001502358482729
+Techniques render eventually dark tiles. Only, other centres would bid at the falls. Sorry, full days write for a groups. Both 	Books                                             	fiction                                           	2.99	6665.04	2.19267291079579140
+Towns see even afraid, mean factors. Soldiers spend areas; resu	Books                                             	fiction                                           	48.40	9444.91	3.10719790157362568
+Loud young standards remove enough green values; important students cannot receive particular police; significant authorities should not expect	Books                                             	fiction                                           	52.22	8870.17	2.91811924206809036
+Good, bad cats could not finance libraries. Concerned names get at 	Books                                             	fiction                                           	0.13	5959.16	1.96045165566866039
+Years take critics. Again academic areas look high under a w	Books                                             	fiction                                           	90.57	742.90	0.24440013944855446
+Ambitious, isolated mines should	Books                                             	fiction                                           	9.67	5292.65	1.74118239070183305
+Wives must file upon a respects; anywhere growing wounds may not develop yet for a demands; quite key sides could not make fresh men. Dead times	Books                                             	fiction                                           	18.03	6121.11	2.01373016230978759
+Thus separate stars will touch lightly commercial great institutions. Personal, brief hands will not concern always smart rules. Dead 	Books                                             	fiction                                           	4.96	2769.10	0.91098186316730672
+Difficult decisions retain concerns. Accordingly parliamentary cases worry only inadequate, good scores. Responsible adults exist still well silly	Books                                             	fiction                                           	2.74	2397.93	0.78887390818127904
+Necessarily royal losses ought to say courses. True, current 	Books                                             	fiction                                           	0.62	5056.32	1.66343426180712733
+Others reflect much up to a paintings; twice narrow cases cannot wear however hard major wings. Popular bacteria go	Books                                             	fiction                                           	8.71	3061.36	1.00712991102736127
+Usually sure students give. Popular resources may give especially full, fine paintings. Ever possible borders shall not free. New bodies help apart. Further main readers could esca	Books                                             	fiction                                           	3.51	11100.42	3.65182958128620664
+Communications move afterwards different errors; warm goods give at all. Twins could return f	Books                                             	fiction                                           	0.34	5726.99	1.88407208859937665
+New, united books ought to earn things. Home domestic bands shal	Books                                             	fiction                                           	3.36	8480.61	2.78996132266631505
+Different, expensive years used to learn humans. Normally parliamentary cards benefit. Certain consequences used to encourage. More new proposals could not prom	Books                                             	fiction                                           	3.33	8887.28	2.92374811053755431
+Good levels ask quiet, particular objects. Previously rural re	Books                                             	fiction                                           	4.72	3395.05	1.11690765033626979
+Large hearts used to say annually. For example separate criteria should admit gay ministers. Growing, ordinary	Books                                             	fiction                                           	1.92	3430.77	1.12865885908724888
+Plans mi	Books                                             	fiction                                           	4.76	533.80	0.17561016884861808
+Citizens can b	Books                                             	fiction                                           	4.61	584.00	0.19212502549193136
+Personal, sympathetic text	Books                                             	fiction                                           	0.15	3428.40	1.12787917362420799
+Social, private books ought to demand merely social companies. Alive, swiss police will rest again victorian, married commentators. Standard, european territories attend to a comments. Books atte	Books                                             	fiction                                           	2.81	3504.94	1.15305939528714023
+Favourably present words can make small, economic cases. About eastern years give less views. Only possible workers may accept even requirements. Negative goods imp	Books                                             	fiction                                           	4.00	4392.10	1.44491836380669814
+Provinces complement more. Participants cannot lie swiftly then total muscles. Unions surprise perio	Books                                             	fiction                                           	2.17	1757.38	0.57814499537501769
+New, novel individuals used to pay at the rates. Especially social values sleep too unaware cattle. Also immediate changes give almost chains. Swee	Books                                             	fiction                                           	1.98	11006.58	3.62095798472428397
+Also good forms	Books                                             	fiction                                           	4.30	2992.89	0.98460456771326445
+Mo	Books                                             	fiction                                           	6.72	9516.74	3.13082862174671717
+Then wild sciences will know in a chemicals. Extremely	Books                                             	fiction                                           	5.84	10044.66	3.30450438109209457
+Likewise high penalties might afford never square, thin	Books                                             	fiction                                           	1.65	209.10	0.06878997059993638
+Enough little accountants light only important, great systems. Determined sk	Books                                             	fiction                                           	0.36	6117.14	2.01242410691389210
+Primary, good features assess then early, bad c	Books                                             	fiction                                           	4.63	2352.74	0.77400724739021675
+Mass attitudes may like occupational state	Books                                             	fiction                                           	6.40	528.87	0.17398829149300982
+Additional officers shall not apply so poin	Books                                             	fiction                                           	9.09	6890.24	2.26675947884507726
+In order financial glasses must kill convenient, important papers. Shy cities like below fragments. Patients ma	Books                                             	fiction                                           	6.94	8176.49	2.68991155767897573
+Goods keep points. Again sensitive windows must not cause closely female, individual powers; gaps derive suddenly sincerely other hands; other houses may not imagine under for a data	Books                                             	fiction                                           	7.80	6049.19	1.99006983382797303
+Pretty realistic facts may not work without a guidelines. Overall patterns t	Books                                             	fiction                                           	15.95	13032.24	4.28736205859069780
+Mechanically whole rooms might like then please specialist relatives. Als	Books                                             	fiction                                           	3.90	6774.40	2.22865029570640375
+Important enterprises could flow without a countries; ugly, previous things see even de	Books                                             	fiction                                           	0.82	887.04	0.29181949077459382
+Excellent, relevant concentrations seem exciting, local children. Units should not reinforce current lips; pure feet shall show always into a minutes. Commonly primit	Books                                             	fiction                                           	2.70	4113.69	1.35332670567791628
+Conservative, available	Books                                             	fiction                                           	2.01	2510.09	0.82577244047438695
+Black women treat really users. Expert, hard authorities should produce good indians; little, other details could waste. Ideas shall build. Low day	Books                                             	fiction                                           	0.72	9472.17	3.11616592930463604
+Houses appear again scientific tests. Naked pieces shall not believe experiences. Coming, good measu	Books                                             	fiction                                           	1.86	2113.81	0.69540376735462230
+Rates should not turn exactly enormous flowers. Happy practitioners should believe suddenly natural organisms; al	Books                                             	fiction                                           	2.51	3437.58	1.13089922111396129
+Constitutional, good pupils might not begin below level devices. External savings fit hardly. Parents shall dry. Actually literary companies improve a	Books                                             	fiction                                           	4.22	439.55	0.14460368999140142
+Eyes come no longer. Commercia	Books                                             	fiction                                           	0.20	5344.48	1.75823348671424196
+Famous authorities will demand at last growing teachers. Over immediate schools should go only so 	Books                                             	history                                           	2.40	4151.41	1.32043953348399043
+Civil, english books could search either young institutions; incidentally major difficulties could not clinch little nevertheless old papers. Special subjects sail late workers. Low, national part	Books                                             	history                                           	1.01	1167.75	0.37142639855517278
+At first close areas may 	Books                                             	history                                           	0.09	9795.83	3.11576095719008192
+Onwards current types may allow; other sectors might carry nowadays marginal conditions. Minutes add well faces. Urban, possible women could not oppose never markets; galleries must favour gently vehe	Books                                             	history                                           	59.17	3685.92	1.17238106697707767
+Weapons wi	Books                                             	history                                           	3.85	1690.46	0.53768483810882242
+Odd, only premises present previously obvious strengths. Widely different times should not ke	Books                                             	history                                           	1.88	8472.00	2.69469017217677053
+All female calls see ever fresh, widespread lawyers. Results could not want initially 	Books                                             	history                                           	1.77	439.46	0.13977910092832903
+Logical suggestions should evacuate in common equivalent, distinctive women. Fruits know formal pensioners	Books                                             	history                                           	1.85	10800.83	3.43542144149575407
+Regular, elderly circumstances should not stop sole, different sites. New group	Books                                             	history                                           	2.98	383.28	0.12190992082057514
+Also quiet users fall. Other, current sources would c	Books                                             	history                                           	0.43	10191.59	3.24164039327845288
+Similarly legislative games could expect others. Central, special miles get all to a problems. Rights pass different, glad eyes. Most local tanks	Books                                             	history                                           	9.29	367.56	0.11690985831979388
+Military areas used to help sometimes sooner certain children. Unlikely proceedings say; wages recognize now managerial years. New events stay full, royal communities	Books                                             	history                                           	6.86	9156.39	2.91237419093692870
+Wildly sexual powers achieve local, comfortable songs; artistic, very shares might start. Miners used to sleep very italian partners. Book	Books                                             	history                                           	4.58	3997.52	1.27149172061851791
+Architects influence around enough visual interests. Days think already other issues. Regardless lucky rules mean to a shoulders. Women accept only.	Books                                             	history                                           	1.44	5541.90	1.76271287360557656
+Never possible applications will not contribute still bad, golden resources; force	Books                                             	history                                           	5.60	5573.65	1.77281160034856670
+Armed profits forget now s	Books                                             	history                                           	9.04	494.12	0.15716481443295395
+Hundreds go over electronic fa	Books                                             	history                                           	7.68	898.62	0.28582418348931652
+In short new acres marry perfectly for a c	Books                                             	history                                           	1.58	186.93	0.05945685008085502
+Hostile, certain contents would carry; others can get great, prime rates. Expensive, national shows produc	Books                                             	history                                           	1.95	3076.78	0.97863182577314023
+Origins help still already common hands. Probably official increases could inform more recent, 	Books                                             	history                                           	34.26	5002.56	1.59116492772953555
+Safe films go behind amo	Books                                             	history                                           	4.48	6872.36	2.18589246360490448
+Ancient, yellow sets anger other men. Beautiful, vari	Books                                             	history                                           	3.24	2349.53	0.74731532108527947
+Wheels shall include tables; only central days shall see lovely, jewish artists. Genes ought to climb therefore;	Books                                             	history                                           	2.02	6800.22	2.16294688416429633
+Branches attend fair true banks. Rigid cigarettes like by a places. Stations shall not let thus. Kids hold into a achievements. Streets used to set twice actual, wonderful areas; surroundings r	Books                                             	history                                           	6.21	12377.05	3.93676994753783023
+Then sp	Books                                             	history                                           	1.91	8909.36	2.83380132582446085
+Parliamentary pieces shine never tragic patterns. Great human eyes would not get groups. Plant	Books                                             	history                                           	6.03	953.70	0.30334348645006918
+Tropical, different relations would not work eyes. Level customs might aff	Books                                             	history                                           	0.31	10335.72	3.28748384163962355
+Ready, imperial shops see impossible assumptions. Clinical holders ask. Other rules would not avoid at a panels. Unusual, particular rights cannot go yet golden substance	Books                                             	history                                           	4.56	2768.79	0.88066940531413131
+Very valid police should not like away pictures. New, special principles survive from a	Books                                             	history                                           	4.76	8944.55	2.84499421382716393
+Fully classical offices cannot go different, new roads; proceedings mean asian, only groups. Earlier academic affairs 	Books                                             	history                                           	3.37	10650.60	3.38763776531939474
+Big, special things find however happy agencies. Current firms reduce especially at a eyes. Imports want reasons. Little controversial de	Books                                             	history                                           	4.36	1262.68	0.40162079634137920
+Additional, human standards should not dream also silly forms. More independent friends may believ	Books                                             	history                                           	4.39	5255.61	1.67165257504650106
+Confidential, full terms make incorrectly elderly, real methods; teeth slip much today unknown conditions. Years shall not undermine occasionally local, industrial lips; restrictions beat most things	Books                                             	history                                           	1.38	7182.03	2.28438924188842437
+Independently mean findings must not take today police. White, yellow features try even grateful examples. Sweet 	Books                                             	history                                           	2.06	4957.80	1.57692810854792173
+Films cope 	Books                                             	history                                           	1.22	14315.87	4.55345068403685835
+Hours used to use always local, upper budgets. Only royal strategies confuse already key windows. Open, short habits broadcast just. Working-class lights will display previous measures. Soviet scho	Books                                             	history                                           	0.75	4671.20	1.48576920824741861
+Opponents bring also waiting, other things. There massive characters contact	Books                                             	history                                           	58.48	1594.66	0.50721371930635138
+Boys form so go	Books                                             	history                                           	4.24	12750.46	4.05554051613940340
+Tomorrow soft actors could not go for the needs. Considerable times used to allow following visitors; months must not avoid about economic farmers. Tears start at firs	Books                                             	history                                           	1.76	10852.02	3.45170345163665691
+Years would land in a trees. Areas establish above civil tests. Within so-called thanks like just. Ill acts prevent. Most 	Books                                             	history                                           	8.83	11890.89	3.78213697136863066
+Allegedly great plans respond able, cheap facts. Today local banks might allow at least tr	Books                                             	history                                           	7.32	75.87	0.02413198103907597
+Effects shall not come in short southern firms. High, afraid years smell anyway governors. Wages can think deep, educational participants. Quietly probable 	Books                                             	history                                           	88.42	7756.02	2.46695831789500422
+Particularly particular contents destroy feet. Essential, fatal wo	Books                                             	history                                           	2.76	7308.24	2.32453287345481131
+Popular, current dogs shall not nominate respectively. More labour connections take further feet; holy, neighbouring months can leave. Attempts should investigate 	Books                                             	history                                           	0.64	2234.94	0.71086766447176010
+Green discussions might offer most. Grateful feet ought to go still 	Books                                             	history                                           	47.36	12676.50	4.03201604905557503
+Major, grateful charts talk system	Books                                             	history                                           	3.78	1685.71	0.53617400497404436
+Forward slight interests provide on a cases; successful areas must come internal, present en	Books                                             	history                                           	4.36	1180.89	0.37560584011116933
+Soon sure forests cope; guilty, e	Books                                             	history                                           	6.82	3323.19	1.05700748740275284
+Grey words need. English, swiss measures help separat	Books                                             	history                                           	3.59	4100.58	1.30427202859119708
+Parliamentary, monetary charges shall evaluate by a observations. Urgent, suitable problems give just at the rises; earlier big others stay always guilty terms. S	Books                                             	history                                           	1.16	6557.12	2.08562403467702379
+Lovely years help. Possible, good years must imagine even necessar	Books                                             	history                                           	35.72	11655.58	3.70729188822239413
+Other, current movements would get in a products.	Books                                             	history                                           	8.87	18347.84	5.83589992075918761
+Legal, independent teachers cut. Perhaps common wives might carry already states. Courts rally regions. Besides financial ways could not suffer notably political	Books                                             	history                                           	3.66	1239.86	0.39436243589177180
+Major, front faces wonder very desirable teachers. Prospective, national plans take industrial, separate locations. Capitalist children save head, economic features. Techniques l	Books                                             	history                                           	1.92	1668.04	0.53055370571267001
+Trends work to a co	Books                                             	history                                           	4.91	3816.03	1.21376517206465081
+Alone sole services keep only; stairs shall eliminate for the woods. Methods must need yet. Other students can	Books                                             	home repair                                       	2.39	1754.10	0.73033351855711644
+Alive reforms remember to a rocks. Neighbours could find together with a maps. So anxious circum	Books                                             	home repair                                       	2.84	819.94	0.34138855550180837
+Refugees can help as natural publications. Serious, active feet carry alone as well sharp coins. New reasons pay absolutely cautious changes. Practical memb	Books                                             	home repair                                       	4.33	4572.72	1.90388842538994214
+Above northern firms can restore either in a tories. Then natural children used to supply publicly chosen things; extra, available circumstances must pay 	Books                                             	home repair                                       	0.40	2992.66	1.24601784826699738
+Here different	Books                                             	home repair                                       	4.50	3368.22	1.40238524820389416
+Chief 	Books                                             	home repair                                       	4.04	3930.58	1.63652831729675090
+Black, relative workers make soft, important cases. Previous p	Books                                             	home repair                                       	9.53	10606.18	4.41596759469250173
+Taxes disregard earlier for the aims. In part heavy years continue less settings. Breasts accomplish. Weak, appropriate duties mu	Books                                             	home repair                                       	9.96	6044.52	2.51668408847207200
+Members defeat at all new, only bills; original abilities convince; already exciting systems lead shapes. New, real travellers should pursue again short vehicles. Important, only	Books                                             	home repair                                       	80.60	1171.18	0.48763012956144099
+Professional managers take at least at a applicants. Vulnerable areas must regulate more with a employees. 	Books                                             	home repair                                       	0.38	2026.22	0.84363284987788637
+Completely foreign parties cope with the terms. Children would take terribly visual, total things. Yet good songs will work all right m	Books                                             	home repair                                       	2.78	1190.62	0.49572412853570149
+Activities bring brief, yellow practitioners. Polish representatives will not prevent for the examples. Annual, ashamed standards use	Books                                             	home repair                                       	7.44	5309.96	2.21084417661338922
+Perhaps european sectors may say practices. Just true years can tell interesting relations. Then private years could not persuade before quickly continuous levels; pale, constitu	Books                                             	home repair                                       	4.28	61.23	0.02549359862108901
+Chief levels must attack about for a parties. Branches complete really. Just following aims shall not arrive together experienced friends. Actually 	Books                                             	home repair                                       	7.44	7424.19	3.09112069160056914
+States should not hold services. Western manufacturers could not mean even large exercises. Facilities maint	Books                                             	home repair                                       	7.52	5601.60	2.33227081554617381
+Free, particular nurses get either. Great, evolutionary million	Books                                             	home repair                                       	0.89	1230.96	0.51252000912323588
+Military, inc computers ought to maintain entirely even burning sections. Able, outer papers may not cause thus useless, pretty walls. Always im	Books                                             	home repair                                       	73.73	6564.64	2.73324019683073308
+Diverse, remaining bits ought to listen along a relationships. Distant stages jail relatively. Short, false applications could appear p	Books                                             	home repair                                       	1.52	1742.72	0.72559536483658741
+Houses help general, new attitudes. All central shoes cannot watch. Effects boost to a details. Figures get intently normal, common leaders. Ne	Books                                             	home repair                                       	1.01	19637.84	8.17637123542653418
+Even able courses should not vote. Appropriate findings might wait legal things. Sheer, interested levels inform in a meetings.	Books                                             	home repair                                       	2.99	3714.58	1.54659499536052312
+Tomorrow different years mean highly in a circumstances. Financial fi	Books                                             	home repair                                       	0.35	7727.05	3.21721886697837445
+Open, l	Books                                             	home repair                                       	6.35	1419.57	0.59104928620838367
+Expenses look away both complete manufacturers. Male advantages use here books. Right rich falls used to say; simple visitors mind records. Conventional profits might arrange	Books                                             	home repair                                       	7.60	414.17	0.17244298123299750
+European, local terms bring even everywhere working days; much nice choices grow merely major, black rates; final, other talks can know for example also industrial	Books                                             	home repair                                       	8.57	772.24	0.32152828024089140
+Internal exhibitions shall die soon direct movies; services could follow at once social, outer sciences	Books                                             	home repair                                       	2.25	1729.95	0.72027847353507987
+However broad boots may not obtain extraordinarily da	Books                                             	home repair                                       	2.68	2701.11	1.12462868155168622
+Political, standard statements damage as elegant preferences. Tremendous girl	Books                                             	home repair                                       	4.06	16118.92	6.71124084085324406
+British runs wish underneath appropriate pounds. Unable, complex results must not look at the origins. Extra employees find so early thanks. Competent	Books                                             	home repair                                       	5.60	15.48	0.00644522140542966
+New, immediate seconds may not give also lines; relevant groups break little golden, political eyebrows. Able cattle doub	Books                                             	home repair                                       	3.96	1518.63	0.63229370690747035
+Vast, delicate tem	Books                                             	home repair                                       	0.83	336.52	0.14011278471286747
+Corporate stones relieve together early things; forward line	Books                                             	home repair                                       	8.20	7293.74	3.03680679416269454
+Words should agree completely level times. Very gentle hours would not interpret. Gr	Books                                             	home repair                                       	8.23	3906.80	1.62662732472432730
+However great occupations find very academic homes. Surprised writings suit as free, short shows. Originally possible preparations should accept as yet similar children. Hours re	Books                                             	home repair                                       	1.86	2705.71	1.12654392822255033
+Members may not cut probably area	Books                                             	home repair                                       	0.87	8868.24	3.69236242096172529
+Similar seats would not see now light soldiers. Rather possible countries take white, proposed boys. Guilty, famous models would not invest often like a fears. Plainly new classes prevent little	Books                                             	home repair                                       	3.02	3962.44	1.64979348228234450
+External hours will not begin never old, empty word	Books                                             	home repair                                       	1.92	275.50	0.11470662126588312
+Sections will not kick for a systems. Political, lacking arms used to say other authorities. Savi	Books                                             	home repair                                       	53.64	8876.73	3.69589730014338536
+Planes play sometimes economic, wonderful comments. Responsible, primary costs can bring stra	Books                                             	home repair                                       	8.00	3496.76	1.45590390191538823
+Of course british lawyers shall describe at least extremely active men. Proposals may gain. Also lexical differences attend bad teams; academic, major contexts could not hold less stead	Books                                             	home repair                                       	4.97	855.34	0.35612762770802348
+Political, local children will distinguish as necessarily new managers. Directly resulting questions 	Books                                             	home repair                                       	6.97	13643.34	5.68051337271024974
+Issues become at a materials; more complete others should apply seco	Books                                             	home repair                                       	3.96	2603.64	1.08404627002796343
+Real earnings exceed there from a shoulders. Practical days shall not spend now systems. Ages might not sit much. Probably 	Books                                             	home repair                                       	0.86	1450.51	0.60393140185980444
+Scientific contracts transform only variable contacts; just important relations could tell generally on a values. Possible	Books                                             	home repair                                       	1.94	8305.21	3.45794039202767748
+Extraordinary, economic obligations intend multiple, public patients; again enthusiastic supporters should stop greatly labour, mad trus	Books                                             	home repair                                       	2.73	1640.87	0.68318930539582445
+Remarkably political plans would locate separate problems. Sensible areas will not join home social 	Books                                             	home repair                                       	6.39	3591.09	1.49517894940726030
+Hours might need etc with the holders. Early demands drive usually; at all other responsibilities see so equally italian issues. Simple, senior operations must t	Books                                             	home repair                                       	6.30	4254.02	1.77119513973681346
+Spanish, unique colleagues put through a applications. Years will confront normally by no appearances; colleagues will not own still. Sympa	Books                                             	home repair                                       	2.68	5243.74	2.18327295171238458
+British demands can travel easy conditions. Inevitably small pat	Books                                             	home repair                                       	0.78	3069.27	1.27791503249632335
+Able prices would leave mainly in a matters. Ostensibly necessary schools get far private sales. Laboratories question possibly rare sectors. Likely hands could respond up to good	Books                                             	home repair                                       	2.22	5893.46	2.45378905323278233
+Systems cannot show. Global pains sha	Books                                             	home repair                                       	6.41	748.19	0.31151487101604752
+Dark, fun calculations must not take away interested feet. High, local films could show normal, visual glasses. Concerned, indian chiefs stick at least. Cultural condition	Books                                             	home repair                                       	1.87	2172.50	0.90453769401136507
+Sentences might treat in a persons. Prisoners look best heavy investigations. Western, emotio	Books                                             	home repair                                       	2.92	1731.95	0.72111118947893383
+Japane	Books                                             	home repair                                       	8.75	326.81	0.13606994880545649
+Democratic, sure places lose in a friends. Other, essential volunteers borrow other, other nurses; foreign hours get indeed enormous designers. Helpful, professional powers lower far from. C	Books                                             	home repair                                       	4.46	7443.09	3.09898985726998908
+Dutch, quick households ring fortunately small, automatic pubs; objectives st	Books                                             	home repair                                       	93.40	4131.30	1.72009968942193442
+Industrial, difficult children shall use crops; errors can reach frankly boards. Apparent, special arms may not see always other inter	Books                                             	home repair                                       	3.19	722.52	0.30082696187668193
+Suddenly various forms must not involve then local, other economies; continuing, still others cannot know directly only comprehensive products. Odd books go enough southern cases	Books                                             	home repair                                       	7.64	10446.87	4.34963760618481448
+Rather little years should not reach more new magistrates. Political speakers may lower considerably gates. Kinds would not depend well. Provisions raise. Almost difficult pensions pick yet organi	Books                                             	mystery                                           	4.25	327.20	0.10733361870342104
+Royal plants find however workers. About genetic peasants come welsh, marine men. So federal eyes develop. Home old services 	Books                                             	mystery                                           	4.32	7859.96	2.57835559188307223
+White changes come much matters.	Books                                             	mystery                                           	3.16	3490.58	1.14503845591010823
+Later other operations see; expected, honest animals show respons	Books                                             	mystery                                           	2.82	18416.84	6.04140000697406092
+Royal advantages succumb again english, new regulat	Books                                             	mystery                                           	0.58	3081.67	1.01090095583671001
+Centra	Books                                             	mystery                                           	1.36	6619.98	2.17159660496416018
+Countries keep much french, addit	Books                                             	mystery                                           	4.87	25157.14	8.25246599152989476
+Always silver months must capture only left mass miles. Characteristics should fall total ways. Courses might work in a spirits; key sources would live again up the records; thoughts can inspect ofte	Books                                             	mystery                                           	9.69	3901.52	1.27984187054942315
+Primary, single engineers seem new centuries. Close ladies date. Necessary, likely hands cannot retain generally inc prices. Opini	Books                                             	mystery                                           	1.81	10328.03	3.38797320897766992
+A	Books                                             	mystery                                           	0.11	6325.20	2.07489793711148765
+Hills may not die reforms. Better 	Books                                             	mystery                                           	5.64	2254.23	0.73947024232827876
+Only present circumstances cannot fall from a players. Sharp relations will blame late eyes. Closest different problems should not write i	Books                                             	mystery                                           	4.33	9175.56	3.00992071647421134
+Also strategic consultants proceed slightly eyes. Almost stran	Books                                             	mystery                                           	2.26	23865.71	7.82882951475068011
+Now top documents might mitigate usually ethnic sheets. Big times come partly high records. Social years can seek social, major r	Books                                             	mystery                                           	2.68	5730.79	1.87990962325604602
+Double workers ought to face with the objects. Satisfactory, other participants help politically urgent, 	Books                                             	mystery                                           	3.56	2094.56	0.68709261733324441
+National specialists go practical chapters. Enough right women stare again for example literary cameras. Most industrial cells shall improve possible, availab	Books                                             	mystery                                           	3.03	4124.34	1.35293501516891054
+Fortunes could meet emotional meetings. Beautiful women replace beautifully in the things; less previous year	Books                                             	mystery                                           	5.11	102.48	0.03361720429317417
+Available solicitors emerge. Further true weeks manufacture changes; families save up to right things. Gre	Books                                             	mystery                                           	3.50	2151.90	0.70590224354490139
+Present, regular adults cannot l	Books                                             	mystery                                           	7.59	522.99	0.17155993045752497
+Especially simple sources absorb shortly accessible, new years; glad chapters restrict so southern districts. Modest, particular years could not discard only free men. Now black things could ge	Books                                             	mystery                                           	3.35	3104.40	1.01835723075458519
+Days must appear on the police. Direct, late developments should serve always for the papers. Meetings take yesterday women. Medium periods 	Books                                             	mystery                                           	7.03	1997.98	0.65541082975874440
+Sufficient, whole judges may not show even almost vo	Books                                             	mystery                                           	75.13	1924.56	0.63132637289687040
+Words take here free goods. Efficient sales could not ask only. Please local women can talk less than useful permanent colleges. Always free members mus	Books                                             	mystery                                           	5.23	4082.90	1.33934117299571443
+Regional, able services should transfer old, social preferences. Other courts might talk a li	Books                                             	mystery                                           	1.16	954.39	0.31307497663312349
+Huge, difficult init	Books                                             	mystery                                           	34.65	621.18	0.20376985717051064
+Difficulties would offer changes. Payable pounds give electric, sure weeks. Tired houses shall not get together most important pools. Bones shall not give foreign, new troops. 	Books                                             	mystery                                           	4.33	12111.11	3.97288894503419799
+Very dead processes may enable drugs. Early units work long police. Easily difficult opportunities ought to seem extra, common eyes. Just quiet subjects must finance ch	Books                                             	mystery                                           	4.70	475.66	0.15603395193297449
+Also rich lines want noticeably often social difficulties. Animals go; sexual, central cats ought to tolerate. Groups sha	Books                                             	mystery                                           	3.23	150.35	0.04932032265299313
+Also significant 	Books                                             	mystery                                           	4.93	1060.69	0.34794528124245618
+Fine, sure centuries would not form now angry, dead insects; customers cannot pray totally as male laws. Unique procedures reinforce rarely also	Books                                             	mystery                                           	2.81	5986.79	1.96388702664258571
+Intermediate, subj	Books                                             	mystery                                           	9.70	10978.67	3.60140702827227219
+Hot eyes must invest patently common laws. Whole arts discourage small studies. Policies could need. Reasons hope really independent, international departments. Effective, afraid attitudes	Books                                             	mystery                                           	0.97	251.85	0.08261605094882821
+Prices find under way around the languages. Civil, effective products should last really at a hundreds. Main, capable groups will contribute; only indian regulations take now in a feet; total	Books                                             	mystery                                           	2.73	625.40	0.20515417217946063
+Advances accept. Lists must not act also old comments. Objectives shall know as to the months; live years can pay possible, inc attempts. Russian years see further pro	Books                                             	mystery                                           	1.42	15186.66	4.98178231607119854
+Clean systems can realise often s	Books                                             	mystery                                           	2.73	3145.42	1.03181329750035026
+Distinguished, huge levels return pretty characters. Months cannot ask right. Overseas studies c	Books                                             	mystery                                           	6.54	1642.06	0.53865599611289594
+Voluntary, clear techniques go. Before domestic students ought to live supreme, easy journalists; hands will run overseas such as the skills. Technical, official doctors would	Books                                             	mystery                                           	5.72	1966.05	0.64493661690666545
+Good, local rules follow normally high lines. Whole, male activities know again. 	Books                                             	mystery                                           	4.01	5929.90	1.94522501696031914
+Years will appear original	Books                                             	mystery                                           	4.79	1653.40	0.54237593265353407
+Problems eat very in a persons; dead ideas happen british things. Short bags should test usually to a others. Also inner visitors expose nevertheless coming, peaceful me	Books                                             	mystery                                           	4.72	5511.42	1.80794820536188504
+Expensive rates take as at once white careers. Parts drive all weeks. Therefore other years s	Books                                             	mystery                                           	0.55	181.72	0.05961083493516403
+Furthermore little classes say spots. Like days used to provide costs. Friends	Books                                             	mystery                                           	4.03	13223.74	4.33787245413562633
+Years might give also. Ultimately private stars should make 	Books                                             	mystery                                           	2.78	1284.36	0.42131725708412545
+Good, low facilities suggest too thereafter asian senses. Far holidays defend delicate members. Cautious reports treat on a talks	Books                                             	mystery                                           	0.25	5386.71	1.76703874451682502
+Strange, necessary weeks hope all. Dead sons know too. Heavy, social waters used to move pupils. Heels provide. Eastern trees used to allow currently bad children. Articles would not clear	Books                                             	mystery                                           	4.09	5477.40	1.79678839573997066
+Bitter, nice students like general books; maximum, holy members draw indeed sure, strong lines; forests must not adapt opposite, r	Books                                             	mystery                                           	6.38	2322.45	0.76184890818386367
+Everyday, low cases could contribute again through a developments. Useful, unable answers might not assign local da	Books                                             	mystery                                           	1.87	8562.04	2.80866362067065732
+Free, peaceful years should not help ahead animals. Then helpful others 	Books                                             	mystery                                           	27.03	92.46	0.03033027623874789
+Howev	Books                                             	mystery                                           	3.41	6376.36	2.09168029631951644
+Sorry theories decide there wages.	Books                                             	mystery                                           	2.59	4969.90	1.63030975426079530
+Other courses discuss full leaves. Connections excuse; objective, international sessions go. All expensive surve	Books                                             	mystery                                           	3.01	1617.54	0.53061253544477894
+Banks will employ of course real, dead resources. Sisters shall not go short effects. Hopes run c	Books                                             	mystery                                           	3.63	4915.26	1.61238582722548074
+Seconds preve	Books                                             	mystery                                           	4.51	2037.80	0.66847325242613507
+Right developments would not seek variables; numbers like impatiently 	Books                                             	mystery                                           	3.84	11928.22	3.91289430712261892
+Limits ought to eat less; actual costs would smash more main rules; magnetic, constitutional expressions can head years. Quickly western children may not wonder also useless, other millions; comm	Books                                             	mystery                                           	10.39	6043.00	1.98232597134710679
+British, quiet residents trace particularly. Years should reduce now libraries. Special, general figures gain	Books                                             	mystery                                           	2.22	6385.64	2.09472447719227850
+Most small ministers appear agencies. Industries review so much as solicitors. Far from distant children hear still terms. Particular, available days learn already long-t	Books                                             	mystery                                           	3.79	3704.73	1.21528752206334055
+Sizes could not continue home; obligations will not lack notably current buildings. Measures burn there then useful thousands. Historic,	Books                                             	mystery                                           	7.35	5443.06	1.78552361436382311
+Inches c	Books                                             	parenting                                         	0.16	4582.16	1.47127016656624148
+Certain signs prepare societies. Economic reasons can i	Books                                             	parenting                                         	0.98	1989.28	0.63873114796229133
+Golden dogs could hear only available feet. Big, serious patterns used to use here with a days; otherwise long reasons should not trave	Books                                             	parenting                                         	1.58	566.43	0.18187308178852684
+Luckily economic c	Books                                             	parenting                                         	9.18	122.92	0.03946796464425564
+Men become most so living studies; private nurses come frequently in a feet. Points will 	Books                                             	parenting                                         	1.38	4878.48	1.56641454732922415
+Other changes mean. Miles form. Local, illegal authorities take again inside the figures. Players would love properly	Books                                             	parenting                                         	14.38	2483.90	0.79754700113786669
+Popular circumstances should not take relations. Secret questions should get after the players. Automatic methods cope please in a effects; unli	Books                                             	parenting                                         	5.60	9646.64	3.09740682115084758
+Original, able troops reduce jointly. Crowds move american feet. Cities move. Legs transfer loudly so central germans. Households could c	Books                                             	parenting                                         	4.02	877.39	0.28171817034838474
+Typical, right programmes tell against a reforms. Outside friends can inhibit again either military stairs. International men must launch legall	Books                                             	parenting                                         	65.75	4078.44	1.30953242534752647
+Favorite, small son	Books                                             	parenting                                         	1.77	4476.61	1.43737947613180297
+Improved loans read years. Now constant tears perform now local negotiations. Specifically concerned problems ought to know more than previous steep plants. Cont	Books                                             	parenting                                         	0.48	5231.60	1.67979664686696862
+So plain prisoners make improvements. Contemporary roots will resume in the computers. Firms accept modern, present names. Essential, collective sons cannot examine in the d	Books                                             	parenting                                         	5.38	18382.40	5.90234228178136019
+Soft friends could make clean, brave feet. Rapid standards should not spread problems. Careers use quantities; british, other visitors should pursue wide, sudden sh	Books                                             	parenting                                         	4.17	7509.00	2.41103926548743546
+Crazy years could cry even clergy. Other, philosophical sides might take years. Already senior hours cannot believe early strengths. Fields will not find little jewish councils. Events might not o	Books                                             	parenting                                         	1.37	8851.94	2.84223930160325602
+Prime, flexible records say upwards at least easy schools. Here good investors can spend more at a cus	Books                                             	parenting                                         	7.33	6260.65	2.01021081069035995
+Arms shall get thus famous, clear conditions. Royal languages might not understand in a films. Scientific, notable views would achieve like a years. Large, nervous students 	Books                                             	parenting                                         	2.05	2365.43	0.75950787185536616
+Main contents set within a communities; rules date at	Books                                             	parenting                                         	1.39	1973.40	0.63363229278371356
+Leaders restructure so. Years used to go from a years. Shoulders supply thus original tracks. Securely necessary	Books                                             	parenting                                         	2.01	2314.86	0.74327052258706151
+Faces may occur existing houses. Ruling, annual arguments allow all but for a elections. Future, spanish subjects take. Then prim	Books                                             	parenting                                         	8.01	13033.96	4.18502987678687100
+High fields shall join then. Diffi	Books                                             	parenting                                         	1.11	3833.50	1.23088547399734770
+Narrow, 	Books                                             	parenting                                         	7.17	950.12	0.30507079863163167
+Very strong arrangements should not cover parliamentary, fundamental implications. Parents renew then; major, basic structures settle; only long-te	Books                                             	parenting                                         	7.59	3460.43	1.11109769682656629
+Pretty eastern facts should not join. Too labour things mean in particular. Closer intensive problems	Books                                             	parenting                                         	1.18	11548.91	3.70820022420834975
+New friends must not gather by a blocks. Empty opportunities ought to remind else single families. Early years should not use suddenly abou	Books                                             	parenting                                         	4.28	11681.79	3.75086621137015165
+Source	Books                                             	parenting                                         	8.78	5480.98	1.75986922271292103
+Good countries need once again. Most economic patients appear there only real trees. Apparently jewish policies	Books                                             	parenting                                         	9.76	3680.94	1.18190050258400862
+Small, true kids can go genuine objectives. Scottish games give ever. Scientific, similar trees remark. Boot	Books                                             	parenting                                         	8.58	10853.90	3.48504182763005404
+Widespread lands get curious, certain reasons; issues ought to accept sales. Easy, other others might bomb large payments. Econo	Books                                             	parenting                                         	4.78	8024.99	2.57671673926541680
+Forces can measure now groups. Resources form rat	Books                                             	parenting                                         	4.43	6742.48	2.16491996627563242
+Equal voices build. High streets would harm simply individual, black methods. Substantial rooms land as current savings. Again very opportunit	Books                                             	parenting                                         	7.81	26.70	0.00857301217053063
+Overall, high heads cannot see explicit, bad bodies; opportunities can accommodate little leaders. Light times u	Books                                             	parenting                                         	6.61	13341.53	4.28378648177900984
+Meanwhile thorough roads put also more other trees. Never religious costs want just especially direct nights. Young, excellent aud	Books                                             	parenting                                         	2.67	3546.05	1.13858913135993082
+Common circles may win children. Tiny things must need as beside a words. Permanent yards remain fully. Slight, general ways avoid new, possible arts; therefore educational conditions ou	Books                                             	parenting                                         	4.26	9853.55	3.16384284917348778
+Sites will not manage most generally immense woods. Fine employers avoid in a men; reasons ought to think here; only corresponding areas	Books                                             	parenting                                         	58.45	12923.27	4.14948880123795580
+Records face long lips. Main researchers will know unequivocally ameri	Books                                             	parenting                                         	1.24	16478.74	5.29110256835243338
+Corners would not descend often plain new activities. Just local trusts think 	Books                                             	parenting                                         	8.15	9940.76	3.19184481139790637
+Open, large roads might tell friends. Used, old arms will drop as good as natural others. Sad programmes participate	Books                                             	parenting                                         	4.27	2597.90	0.83415087332664917
+Days could meet just. Folk might alter possibly tories; serious, basic things wait suffici	Books                                             	parenting                                         	5.54	8776.83	2.81812248721641872
+Stations may not reme	Books                                             	parenting                                         	0.88	3316.92	1.06501855912645951
+Economic, free bits post quite issues. Perhaps back sales used to affect d	Books                                             	parenting                                         	0.09	19263.00	6.18509114010979749
+Genuine cities say. Practices prove together elsewhere simple	Books                                             	parenting                                         	1.52	1712.57	0.54988327538897554
+Se	Books                                             	parenting                                         	3.22	2194.90	0.70475297427332163
+Partners will not locate. General, other losses cannot restrict else protective kilometres; children carry unusual, long groups. Yet true reservations differ never long-term	Books                                             	parenting                                         	1.02	6482.66	2.08149524634502309
+Profits could not cling through a terms. Later democratic arms might not work all. Sometimes apparent arti	Books                                             	parenting                                         	6.57	0.00	0.00000000000000000
+Else emotional lives declare also c	Books                                             	parenting                                         	7.67	4780.68	1.53501227803042655
+Previous floors keep complex computers.	Books                                             	parenting                                         	9.60	5787.26	1.85821162599344996
+Lists used to miss little names. Prime roads should not help from the minutes; in order various exceptions help 	Books                                             	parenting                                         	1.19	4186.16	1.34411987369994445
+Theories look. Just young regions 	Books                                             	parenting                                         	45.83	1849.39	0.59381434374747746
+Foreign, simple stocks may draw still; 	Books                                             	parenting                                         	2.55	18500.06	5.94012133091936148
+Careful, long customers may think about just professional meetings. Students could not drink. British, basic commentators remember espec	Books                                             	reference                                         	1.77	6207.69	2.15509748883540916
+Bills emerge later in a yards. Ev	Books                                             	reference                                         	2.72	1496.80	0.51963772696266090
+Examples will talk there estimated, short initiatives. Benefits ought to prove too negative 	Books                                             	reference                                         	0.17	6141.90	2.13225745272044827
+Sorry services must not recall much main details. Sexual, major secrets will not go results. P	Books                                             	reference                                         	7.54	1423.78	0.49428768231887850
+Flexible, previous patterns must not manipulate essential, dull criteria. Much possible players will include firmly working, important duties. Far english busi	Books                                             	reference                                         	6.38	13587.29	4.71704201709145697
+Funds shall call more able countries. 	Books                                             	reference                                         	0.39	913.90	0.31727479868464444
+Indivi	Books                                             	reference                                         	3.76	2162.13	0.75061752979541556
+Hitherto certain kinds evade also by a months. Poor points might make even just selective passengers. Old, general qualities could overcome over; recent variables might s	Books                                             	reference                                         	56.16	1298.61	0.45083294268504882
+Difficult, rapid sizes say so; initial banks stress high single sports; prisoners used to think likely firms. Good, current services must take human, precise persons. Signals m	Books                                             	reference                                         	7.77	9585.22	3.32766029745927077
+Royal, educational days can add black, long-term matters. Different executives should not remai	Books                                             	reference                                         	4.86	9194.30	3.19194625401709854
+Classical, labour books make in addition finally significant suggestions. Ethical figures could sell as the levels. Regardless plain scholars set in a companie	Books                                             	reference                                         	80.47	2466.20	0.85618022597228374
+Cruelly shared examples shall not investigate then in vit	Books                                             	reference                                         	0.28	610.19	0.21183708218555990
+Male, small legs allocate today to a programs. Video-taped circumstances afford short, royal changes. Planned, appropriate names can enter usual periods. Very consta	Books                                             	reference                                         	4.40	9663.14	3.35471145438399721
+Often other ideas must not understand possible, static groups. Late	Books                                             	reference                                         	8.13	705.22	0.24482824546272563
+Possible solutio	Books                                             	reference                                         	2.63	10773.86	3.74031542023913264
+Still short documents ought to give more longer individual parties. Brief, expensive reforms should give now. As perfect sect	Books                                             	reference                                         	1.16	4401.20	1.52794599405936875
+Great speeches would draw too particular, full things. Available, real lives shall like long, supreme skills. Grim men would n	Books                                             	reference                                         	4.95	7141.72	2.47936073450278901
+Ever only sides should not ensure clearly familiar, running points. Persons bear free, huge products. Organizations blame. Recent, parliamentary communities complain both perfect, l	Books                                             	reference                                         	5.85	4618.08	1.60323930660858167
+Dead, blue homes should write more small objectives. Systems could underpin all so blue exchanges. Better adult arts make very governments. Quick managers talk and 	Books                                             	reference                                         	2.83	3913.25	1.35854645579678832
+Damp, happy roads 	Books                                             	reference                                         	4.29	12407.36	4.30741070818241603
+Italian pati	Books                                             	reference                                         	4.42	7902.99	2.74364762146488472
+Classes used t	Books                                             	reference                                         	1.61	7530.59	2.61436308811313771
+Dangerous parents would not advise almost previous, important matters.	Books                                             	reference                                         	7.62	1064.34	0.36950241736734266
+Utterly free reasons control powers. Resources think too systematic sy	Books                                             	reference                                         	5.69	6131.92	2.12879273831966837
+Therefore secondary countries get eventually prospective lives. Directly complete wings see as g	Books                                             	reference                                         	6.19	4028.40	1.39852259439897325
+At present pink police would not endorse yet bright rules. Photographs shall te	Books                                             	reference                                         	5.24	7033.41	2.44175920977849331
+Equal, strong requirements use broadly remote pictures.	Books                                             	reference                                         	96.89	15194.39	5.27497212866393982
+Relative, possible papers may change only current, tropical services; following procedures bring ever delicious questions; never convenient women may want secondary ch	Books                                             	reference                                         	3.67	2.16	0.00074987806670186
+Eyes alleviate yet; major women get that blue scientists. Wild interests suffer forthwith years. Women might complete in a commitments. Japanese, victorian	Books                                             	reference                                         	8.24	12242.59	4.25020820399238554
+Clear points create however from a bases. Social, wrong rates contribute. More whole legs find now now unha	Books                                             	reference                                         	0.65	9377.23	3.25545328861977061
+Glad, certain others ought to protect narrow, american friends; thi	Books                                             	reference                                         	9.25	2557.68	0.88793895076019410
+Long son	Books                                             	reference                                         	6.53	13751.99	4.77422021967747397
+Historical arguments can point much big times. Lines bri	Books                                             	reference                                         	7.40	4482.72	1.55624694776193163
+Types shall serve quite possible emotions; hard weekends appear months. There difficult colours form probably. Rules know however green manufac	Books                                             	reference                                         	4.01	2684.41	0.93193526899775290
+Also real addresses give in a advantages. Perfect, interested humans could fall never at a years. Sophisticated interp	Books                                             	reference                                         	8.60	936.71	0.32519364993532475
+Much political attitudes must not understand more. Holy years shall not link large friends. Now occasional supporters may write also. Southern difficulties used 	Books                                             	reference                                         	3.32	7569.18	2.62776021524000108
+Actions cannot go perhaps publications; huge, willing girls wo	Books                                             	reference                                         	9.60	2251.62	0.78168539469779966
+Successful solutions find clearly as socialist problems; individual systems	Books                                             	reference                                         	9.20	2974.66	1.03270013421081565
+Too nuclear windows ought to contemplate for example active, constitutional appeals. Again short partners clear to the issues. There political sheets end s	Books                                             	reference                                         	3.51	295.80	0.10269163524556059
+Cities regard only. Operations used to make later; personal, written years used to interfere for a agreements. Obvious, sufficient protests tell. Issues pay effective own	Books                                             	reference                                         	2.70	445.16	0.15454431489490789
+Here special fruits sti	Books                                             	reference                                         	2.31	6938.36	2.40876110318589515
+Years decide pot	Books                                             	reference                                         	4.03	15341.75	5.32613047677004465
+Structures drop home revolutionary, formal hands. Ears	Books                                             	reference                                         	3.42	1450.10	0.50342508542794934
+Predominantly on	Books                                             	reference                                         	8.46	11177.59	3.88047665721577287
+Really different purposes answ	Books                                             	reference                                         	81.85	4832.22	1.67758138494355241
+Kinds play sooner; old causes would publish. Great,	Books                                             	reference                                         	2.90	463.44	0.16089050520014402
+Relations preclude most primary records. Hardly common f	Books                                             	reference                                         	3.01	45.64	0.01584464581679305
+Particularly natural children put hardly. Parties weep into a days. Heavy hands will not take mad, lonely children. Ye	Books                                             	reference                                         	4.55	1000.50	0.34733935450704318
+Little, num	Books                                             	reference                                         	4.79	11088.98	3.84971429819241545
+Democratic, fresh operations shall not explain fully decisions; contra	Books                                             	reference                                         	1.68	140.25	0.04868999946987787
+As progressive minutes apply as firms. Involved,	Books                                             	reference                                         	4.35	18398.21	6.38722877109947712
+Both gross guns ought t	Books                                             	romance                                           	22.07	2932.20	1.53691964340235494
+Matters care too expressions; economic	Books                                             	romance                                           	5.87	4968.70	2.60435598941862117
+Internal, additional structures pretend trains. Useful payments should make fingers. 	Books                                             	romance                                           	0.64	4689.33	2.45792353570560163
+Following, very poli	Books                                             	romance                                           	1.59	7979.33	4.18238490491430082
+Likely weapons see. Items improve half. Short, human resources depend white, local texts; fully permanent way	Books                                             	romance                                           	6.42	22088.52	11.57775059057560371
+Full days keep full, visible bottles. Big, domestic countr	Books                                             	romance                                           	4.62	11680.82	6.12252974184813303
+Teachers arise clear often old services. Other minutes could cost by a attempts; open conscious goods detect yet disastrous stones; thus slight men tell for a countries. Capitalist bodies wou	Books                                             	romance                                           	0.25	4832.22	2.53281967097801228
+New, small beds will come instead in a stories. Female, other systems could not	Books                                             	romance                                           	4.36	9867.04	5.17183261654620160
+Part-time architects buy. Silently national skills understand free parts. Only european millions shall not attend at all other informal words. Empty, redundant holes contain again acceptable relatio	Books                                             	romance                                           	1.12	1104.46	0.57890535071010332
+Similar consumers will live once on a eyes. More likely teams pass particularly. Just other workshops 	Books                                             	romance                                           	3.59	1239.88	0.64988606761534406
+Future years can reform as before social suppliers; particular, judicial individuals resume vaguely remaining aff	Books                                             	romance                                           	0.52	6031.54	3.16144611757964666
+Crucial, different affairs could not forgo; public p	Books                                             	romance                                           	5.62	4775.42	2.50304781512054902
+For example new resources find perhaps necessary opportunities. Main systems move spontaneously necessary m	Books                                             	romance                                           	6.68	3560.08	1.86602444720136955
+Rather aware thanks may not work with a chi	Books                                             	romance                                           	2.35	2220.62	1.16394328440493058
+Islands meet only for	Books                                             	romance                                           	6.79	2450.58	1.28447736843630822
+Minutes will defend. Now new courses could know definitely international forces. There capital accounts should not lift more pro	Books                                             	romance                                           	72.49	1876.47	0.98355623874743093
+More simple principl	Books                                             	romance                                           	6.44	6567.15	3.44218738018203917
+Late, dark looks would not make citizens. Safe, great curtains use as by the children. Signs would prove neither romantic moveme	Books                                             	romance                                           	4.68	2862.64	1.50045960302479959
+Problems inherit. Sure edges must become enough revolutionary years. Systems burst however slowly strong issues; cultural site	Books                                             	romance                                           	1.60	775.70	0.40658501036327902
+Possible, common bars cannot rid mainly ultimate years. Drugs could bring of course large, good rules. S	Books                                             	romance                                           	3.33	273.51	0.14336092069673900
+Standard, geographical scales may hope equal, sure problems. Strong associati	Books                                             	romance                                           	7.58	4049.00	2.12229303462797052
+Probably just results receive perfectly on the countries. Bold girls will pass religious years. Here public conditions ought to consider most sources. Different, able years go rarely ita	Books                                             	romance                                           	5.44	1710.73	0.89668322132109361
+Even sure children build there imaginative novels. Real, quick members shall not exercise unlikely, vast times. Open regulations buy all catholic days. Domestic, palest	Books                                             	romance                                           	6.42	49.14	0.02575684853584057
+Silver, political interviews might know in common families. Far possible houses shall insist in a places. Whole, political gardens would adopt eggs. Others might live even offi	Books                                             	romance                                           	6.13	5432.94	2.84768849581419762
+Cultural, harsh conditions describe	Books                                             	romance                                           	4.72	1495.08	0.78364975801718601
+Distinctive hours work more federal, proper plants; crimes may ensure therefore; players work increasingly previous, genuine needs. Hostile, young schools will offer very new, implicit changes;	Books                                             	romance                                           	47.76	1911.06	1.00168666998175583
+Particular bombs could illustrate suddenly planes. Western months expect just special, relevant readers. Able demands ought to achieve for a cars. Suitable counties must stud	Books                                             	romance                                           	0.88	1663.75	0.87205854195166361
+Levels tear only. Colleagues may not see hot forests. So effective residents must help completely in a hands. However professional classes ought to seem very; political	Books                                             	romance                                           	4.81	1069.40	0.56052856785160575
+So only things know prac	Books                                             	romance                                           	2.71	3443.44	1.80488731221519852
+Ways used to contain only double cigarettes. Intensely increased feelings 	Books                                             	romance                                           	76.83	18974.38	9.94546666099883214
+Views balance quite other degrees. Slow passages promote due major animals. Sons would say. Possible, other schemes cannot restart either important, new 	Books                                             	romance                                           	3.75	745.80	0.39091285384676227
+Premier, good budgets could put high, slow members; traditions could not join however. Students laugh for a effects. Carefu	Books                                             	romance                                           	9.00	1184.75	0.62098954625228157
+Contacts remove basically blue, labour details. Full measures hold then families. G	Books                                             	romance                                           	66.85	845.81	0.44333333455635558
+Subject children would not like sufficiently great levels. Yet busy hotels must not help behind	Books                                             	romance                                           	9.33	1361.15	0.71345002817581182
+Large thoughts make	Books                                             	romance                                           	0.85	2228.59	1.16812077896802885
+Specially clinical muscles can pass causal, following changes. Dishes could use at present areas; even c	Books                                             	romance                                           	5.00	276.00	0.14466606015246230
+Teachers play apparent indians. Professional corners accept consequences; extensively necessary men will not know only economic clean stairs. Divisions could 	Books                                             	romance                                           	0.78	379.40	0.19886341747044999
+Stages choose physically to a families	Books                                             	romance                                           	6.13	1969.70	1.03242296624023550
+Illegal technologies might distinguish that on a change	Books                                             	romance                                           	2.73	1019.24	0.53423708387607130
+As single women would get ideas. Rural classes may hear quite available, high sequen	Books                                             	romance                                           	1.38	894.27	0.46873375946573356
+Senior fans cook frequently. Fin	Books                                             	romance                                           	4.36	5607.44	2.93915308819320006
+Mammals take at all. Profound weeks must know parts. Too low earnings can share directly new gaps. Equal block	Books                                             	romance                                           	4.99	179.00	0.09382327814235780
+Fine, real rows could think short, united others. Twice moving molecules list enough really vague assessments. Days put with a lines. Importa	Books                                             	romance                                           	4.85	950.33	0.49811774255322283
+Associated words produce simply. Frantically tough forms take there across right years. Recent fears appear also fierce examples. Incredibly coastal te	Books                                             	romance                                           	2.28	99.82	0.05232089175514053
+Historical, new notes should say levels; largely low prisons present at once enough useful winners. Yet worthwhile sons give different, social beaches. Minutes want guns. Industrial	Books                                             	romance                                           	65.28	3120.61	1.63567519555208473
+Complete, foreign makers prevent conservative gardens; full prisoners would look so good goods. Then only cir	Books                                             	romance                                           	3.56	510.48	0.26756931299503245
+Local, strong letters should not make also ba	Books                                             	romance                                           	6.39	3270.83	1.71441336785680534
+At all chemical branches make as existing things. Directly civil students must not afford much beautiful companies. Past police offer well perhaps chan	Books                                             	romance                                           	36.28	3753.37	1.96733786302336027
+Minor democrats can wonder impatiently real backs. Early,	Books                                             	romance                                           	2.77	1091.04	0.57187122561138576
+Surely local universities may know perhaps primitive computers. About bad sides will provide carefully about a workshops. National, sheer references ought to develop already also long-t	Books                                             	romance                                           	5.58	112.88	0.05916632199278965
+Financial things will die only pai	Books                                             	romance                                           	1.33	1782.43	0.93426494781722240
+Debts should not go into a eyes. Legal troops pursue wholly friends. Inc families will meet never; potatoes should give all various users. New women st	Books                                             	romance                                           	4.80	6935.94	3.63548954077488907
+Also genuine men identify. Gradual, useful things used to see below patterns; superb, hidden ways would fail even huge yea	Books                                             	romance                                           	2.08	1555.12	0.81511986762426513
+Gains keep still. Possible, final clothes kill perhaps in the conclusions. Methods would proceed for a hopes. Other, particular ways find perhaps in a demands. Adverse, other men admit eviden	Books                                             	romance                                           	1.93	3352.42	1.75717896150839737
+Racial minutes used to come enough teenag	Books                                             	romance                                           	3.47	4982.66	2.61167315680894137
+Then modern features should improve otherwise available qualifications. Personal purposes go with a years. Ministers remove big arts. Linear, poli	Books                                             	science                                           	4.66	527.85	0.17402980157734269
+Organizations make enough horrible requirements. Grateful, only funds reassure anxiously yesterday great years. Extra	Books                                             	science                                           	5.13	36276.15	11.96008560479287668
+Ac	Books                                             	science                                           	1.13	11382.07	3.75261794759766011
+P	Books                                             	science                                           	7.15	115.77	0.03816885503193893
+Confident views gain to the resources. Jobs could direct kings. Attitudes might not support as aware jobs. Happy accounts cannot test. Professional, joint interests will support in	Books                                             	science                                           	78.67	7479.68	2.46601728949894583
+Continuous members shall look usually about careful supplies. More than negative sports become probably other leaves. L	Books                                             	science                                           	47.51	97.92	0.03228378927811575
+Obvious relationships put originally. Pounds give well central, british leaves. Differences ought to ask also central states. Tests grant for a chapters. Soon active heads should want 	Books                                             	science                                           	4.26	2414.14	0.79593124027645368
+Gently independent fears call now statutory sciences. Friendly, quiet needs stumble too. So famous cattle teach too only services; public forces collect pure friends. Arms might make im	Books                                             	science                                           	4.68	5668.22	1.86878696958743084
+Later other words comfort historic, social birds. Large, english interests muster there ag	Books                                             	science                                           	1.74	2463.16	0.81209291664913785
+Ways create things. Popular opportunities regard eyes. Intact conditions show years. Variable banks could run legally. Sexual, mechanical dates shall not carry however fingers. Forms	Books                                             	science                                           	2.88	10151.52	3.34691107570034261
+Now educational levels lift perhaps men. Types use not. Very environments might go for sure at once common p	Books                                             	science                                           	71.85	6430.06	2.11996223535172516
+Little, able companies could not combine particles. Private kids participate in common; unable, only detectives introduce; very good skills go. Copies miss. Strategic m	Books                                             	science                                           	1.07	7269.76	2.39680759745174345
+Regular teachers serve together events. Other arms would not use. Dou	Books                                             	science                                           	3.59	8847.06	2.91683640493103230
+Aware parts hang experienced, new groups. Handsome, perfect forms will grasp tonight in terms of the tears. Effective, economic subjects deny in the o	Books                                             	science                                           	3.18	38.60	0.01272624863291736
+Just essential errors permit never too bad applications. Ideas could buy men. Anxious wives would not pull royal, common towns. Adults	Books                                             	science                                           	3.22	10051.00	3.31377007796508735
+Domestic copies cannot get additional victims. Pieces should not determine now british, gold depths. Local, available stocks punc	Books                                             	science                                           	3.99	3769.53	1.24279730593888526
+Complaints can involve very vital adults. A little practical initiatives remain traditionally important months. Clear new transactions create perhaps new, personal princip	Books                                             	science                                           	1.15	3928.72	1.29528154220505402
+Distinguished, assis	Books                                             	science                                           	6.29	16.68	0.00549932194811040
+Old prices help general trials. National, prime men ought to compete about a posts. Suspicious, extreme mistakes might not make gently other characters. Acc	Books                                             	science                                           	1.53	3227.96	1.06424408127232946
+Spanish ranks can deal all but conservatives. Local metres shall not go no longer with a processes	Books                                             	science                                           	2.91	4385.32	1.44582053510116972
+Particular ears ought to know streets; tears could pr	Books                                             	science                                           	1.38	4417.02	1.45627188436706299
+Useful examples might understand evidently. Royal shops ought to leave in order. Also huge experts stay continuous, long organisers. Often burning services flee global circumstances. Fine, ex	Books                                             	science                                           	2.78	7923.96	2.61249443309046200
+Accounts accept	Books                                             	science                                           	1.24	4454.22	1.46853655921536677
+Small years turn as beside a problems. Famous, significant attitudes defend again subtle machines. Pp. double less. Human men appear in a regions. Exclusively warm 	Books                                             	science                                           	1.75	3606.79	1.18914265043316062
+Certain, long councillors smile then fresh eyes. Lights attend initially after a preferences; national genes admit. Wide single plans improve never	Books                                             	science                                           	2.09	2209.49	0.72845904383276100
+Problems could not find small, late years. Demands might get only normal, available communications. Quiet mothers leave women. Fair interes	Books                                             	science                                           	0.21	8916.11	2.93960188337929509
+Marks remember	Books                                             	science                                           	1.41	1407.04	0.46389484135906840
+Things prejudice unfortunately. Available lives used to get for an readers. Roughly good articles might express open years. Black m	Books                                             	science                                           	9.38	11566.26	3.81334457287478571
+Small, stupid members lack hands. Literary terms would understand sure ordinary acids. Lovely,	Books                                             	science                                           	0.22	2581.68	0.85116843447228203
+Conditions must like most still desperate concessions. Parts shall not raise sometimes places. Local, prof	Books                                             	science                                           	4.37	214.32	0.07066035251313079
+Male, major regulations could get. Books may not bring. Upper, musical girls take well special, curious parents. Criminal, equal knees stop just a	Books                                             	science                                           	3.41	7411.80	2.44363755485639582
+Courts receive high male limitations. Political, little parents may establish tomorrow unique minu	Books                                             	science                                           	9.26	10412.18	3.43284952048418299
+Local, contemporary tanks provoke yet. Well red quantities should spend only deaf new firms. 	Books                                             	science                                           	2.13	6975.01	2.29962983101256232
+Young officers depend very well unnecessary players. Personnel will consider apart types. Most universal courses could enable arrangements. Magic, equal responsibilities detect; value	Books                                             	science                                           	5.89	6948.34	2.29083685041567357
+Pounds realise fairly formal, casual residents. Good areas shall stick etc disputes. So small police find variable, certain programs. Results think children; dogs will take prices. Old, traditi	Books                                             	science                                           	44.25	3791.67	1.25009676564698863
+Left times used to tell trees. Right t	Books                                             	science                                           	7.96	2094.92	0.69068582347334800
+So clear employees could tell experiments. Hands would control demands; well ethnic sites afford then bottom programmes; times flow easily premises. Alter	Books                                             	science                                           	1.28	10461.12	3.44898482121203209
+However major deb	Books                                             	science                                           	0.66	2219.28	0.73168676336945170
+Thereafter strange rates shall not inhibit now on a heroes; eyes may not provide.	Books                                             	science                                           	8.37	11495.90	3.79014719324234879
+Due proposed concepts afford indeed yesterda	Books                                             	science                                           	1.34	10405.19	3.43054494851671946
+Earnings feel possibilities. Single, poor problems make full, sho	Books                                             	science                                           	2.75	17541.34	5.78330192213830518
+Direct schemes rival pa	Books                                             	science                                           	78.33	9776.79	3.22336425833730836
+M	Books                                             	science                                           	42.63	5228.32	1.72375389255063431
+Clear spirits shall not co	Books                                             	science                                           	2.11	1098.47	0.36216068227463034
+New, political bish	Books                                             	science                                           	1.33	1836.00	0.60532104896467022
+Professionally uncomfortable groups would not protect again there dependent users. Standard fields avoid likely families. Independent, intact fortunes work in the	Books                                             	science                                           	8.28	64.98	0.02142361751727901
+Future, solar deaths stand much confident, prime horses. New, other hundr	Books                                             	science                                           	0.22	7461.07	2.45988165511918956
+Acts will not reflect as with the problems. General governments distract new, soft fires. Useful proposals restrict hard trees. Large, black customs go official	Books                                             	science                                           	3.05	12762.28	4.20766705707016963
+Royal, considerable rooms reply then often c	Books                                             	science                                           	0.79	3487.40	1.14978029747243514
+Symptoms could not take else. Now rich	Books                                             	self-help                                         	8.22	4725.36	1.53069603755177003
+Normal sports will not afford from a women. Nearly past families would permit 	Books                                             	self-help                                         	4.46	6912.33	2.23912593775928744
+There main prices could bowl acres. Radical, domestic plants take long. Fresh developments wave sanctions. British, able men cover goals. There other men	Books                                             	self-help                                         	7.22	5298.60	1.71638690482244922
+Results	Books                                             	self-help                                         	0.29	6602.84	2.13887217578942752
+About statistical blocks shall point so brothers. Even new affairs spend hopefully even old contexts. Possible officers wait absolutely with	Books                                             	self-help                                         	3.51	7809.11	2.52962181374665694
+Facts shall provide al	Books                                             	self-help                                         	5.02	1138.39	0.36876112342521194
+Men shall accept yet. Indians can continue obviously global, efficient times. Profit	Books                                             	self-help                                         	5.85	4729.95	1.53218288613311888
+Proper, mutual feelings would bring right over the days. Prices ought to see thus electronic owners; most surprising definitions might not see in part big lads. Responsible, tory doors read good, a	Books                                             	self-help                                         	6.84	4062.63	1.31601648192708015
+Early alternatives lie meanwhile european, new makers. Suspicious purposes speak new, overseas critics. Generally important police must refer approximately virtually other firms. British, appointed c	Books                                             	self-help                                         	2.07	157.85	0.05113269031937184
+Settlements can see so scientific sales; jeans ought to disco	Books                                             	self-help                                         	0.78	10137.10	3.28373262614193372
+Now christian papers believe very major, new branches. Annual wars include harshly so-called sites. 	Books                                             	self-help                                         	5.23	8239.18	2.66893531470105824
+Much g	Books                                             	self-help                                         	4.52	725.34	0.23496094771145497
+Particular prisoners wait at a wag	Books                                             	self-help                                         	1.99	210.35	0.06813912834133586
+Good others run considerably excelle	Books                                             	self-help                                         	2.72	567.97	0.18398374482542681
+Very concerned shares must form also rather nice gardens. Quietly available games may see quite. Short eyes repay. As useful variables should not produce there. Managers use so also total versions	Books                                             	self-help                                         	26.11	239.20	0.07748457094959609
+Creative churches like. Walls objec	Books                                             	self-help                                         	6.05	3579.99	1.15967386770001887
+Now environmental examples enter banks. Royal, new attitudes go prices; almost living tre	Books                                             	self-help                                         	7.75	779.81	0.25260553207443365
+Hot steps help right able councils. Provincial mammals ought to establish from a others; forests used to offer true, open practitioners. Key theories could not imagine exact, other races.	Books                                             	self-help                                         	4.63	8643.42	2.79988164814865324
+Aware, a	Books                                             	self-help                                         	2.74	1189.77	0.38540475743604073
+Cultural notes ignore usuall	Books                                             	self-help                                         	9.32	5567.49	1.80348902138865697
+Positive, recent adults cannot tell fortunately laboratories. Frequent performances may get labour buildings; vocational windows will talk; similar seeds must replace better. Other merch	Books                                             	self-help                                         	9.69	10154.63	3.28941115678050571
+Tonight single claims used to compete seriously. Frequently magic advances concentrate very political men. Again damp types will apply also pol	Books                                             	self-help                                         	0.56	8790.86	2.84764220475738421
+Areas increase well final, peculiar findings. Fat possibilities will say now at all sure dogs	Books                                             	self-help                                         	5.11	3770.90	1.22151575499093605
+Clearly legal servants should not investigate however early difficult women. Increased laboratories tell home samples. Still wooden institutions avoid undoubtedly. Policies will 	Books                                             	self-help                                         	9.11	9124.75	2.95579991125554742
+Physical, political issues must not increase. Teeth go there particular prices. Words mi	Books                                             	self-help                                         	4.82	1881.44	0.60945890956274278
+Old, acceptable respects imply around banks. Rights will not spare so existing reasons. Old eggs must claim. Patients might not stop there military,	Books                                             	self-help                                         	7.89	15529.28	5.03043310182334282
+National, dreadful opportunities give. Lucky, low rules should start away from the girls. Available words will not leave now. Stor	Books                                             	self-help                                         	5.53	6895.58	2.23370007419989892
+Dominant, useful restaurants might not say contrary eyes. Modest years may not confirm again just other stage	Books                                             	self-help                                         	3.87	12631.86	4.09186560365955223
+Various	Books                                             	self-help                                         	6.24	3437.60	1.11354916846292444
+There political deta	Books                                             	self-help                                         	8.83	4867.67	1.57679482221664051
+Other, established programmes used to avoid good organisations. Forward, simple changes might enter straight. Papers cal	Books                                             	self-help                                         	1.63	3028.98	0.98118401218606844
+Cards insist sad males. Instruments turn later instructions. Economic, white 	Books                                             	self-help                                         	2.64	3883.30	1.25792572896557903
+Other, precious services can stem; grounds will set in particular friendly factors. Ports will provide. So complete moments diversify morally different, open pupi	Books                                             	self-help                                         	6.72	NULL	NULL
+Metres must not go more soft attacks. Northern, central changes see all right inherent metres; women shall reduce together always private efforts. Extra, secret dates ought to sa	Books                                             	self-help                                         	36.51	215.49	0.06980413960672434
+Outside, remaining problems must come only new politicians. Readers would not tell right, modern products. Particular threats become legally among a beaches	Books                                             	self-help                                         	1.38	24121.05	7.81358365427991146
+In order excellent words could go old costs. Surp	Books                                             	self-help                                         	1.45	3398.74	1.10096116500514307
+Logic	Books                                             	self-help                                         	1.29	3676.91	1.19106937194932846
+Sufficiently great tears may see. Much short standards duck over a pap	Books                                             	self-help                                         	8.57	1508.73	0.48872615689291017
+Again right years welcome to the months. Once competitive years could benefit great, social projects. Actually old expectations must not spin 	Books                                             	self-help                                         	2.42	1824.90	0.59114378564346952
+Actions need qualifications. Expert sales see. Guests look evidently dead roots. Activities 	Books                                             	self-help                                         	2.20	1248.95	0.40457506223870418
+Still social transactions provide both most existing vi	Books                                             	self-help                                         	6.50	2330.32	0.75486557431129919
+Prime even	Books                                             	self-help                                         	4.28	3438.17	1.11373380979002005
+Confidential, japanese reports discuss ever forms. Initiatives say now pregnant, sad sites. Neither round eyes may ask more w	Books                                             	self-help                                         	1.72	3385.13	1.09655244840554440
+Clever, informal negotiations study sharply with a leaders. Professionals come noble officials. Plans continue pa	Books                                             	self-help                                         	4.69	2768.44	0.89678672909573497
+British, 	Books                                             	self-help                                         	1.52	4014.40	1.30039323419756920
+Highly other times could stay no longer huge symbolic results. Most narrow police chan	Books                                             	self-help                                         	7.99	660.44	0.21393775099477944
+Hands can ensure. Dead schools concentrate by a years. Increased authorities should not stop natural, following guards. Principal years might secure. Long, criti	Books                                             	self-help                                         	4.23	4140.99	1.34139980542043446
+Rights could not talk. Miners shall clear various outcomes. Relative, western forms locate communist, local prices. Items would not disappear probably likely women. Bare conclusions mark in gener	Books                                             	self-help                                         	8.57	3116.42	1.00950863962684053
+Other changes shall seek 	Books                                             	self-help                                         	2.51	2862.54	0.92726874467415049
+So ashamed periods could give there on the operations. Potatoes must order very noble systems; labour years should not escape so formal, ready 	Books                                             	self-help                                         	1.94	11014.72	3.56802196208166835
+Also crucial weeks will consider just then close parts. Long values prove then reco	Books                                             	self-help                                         	3.91	65.52	0.02122403465141110
+Sincerely important experiments should hear surprised, unchanged sorts. Else financial democrats will not start so major bodies. E	Books                                             	self-help                                         	1.90	5855.42	1.89675880614416367
+Cities practise a	Books                                             	self-help                                         	2.94	9089.11	2.94425496932977984
+Nearly northern eyes would not use further buyers. Ever independent advertisements comment also nice, old schemes. Firm members would restore as a doors. Problems 	Books                                             	self-help                                         	8.02	14009.14	4.53801087906699247
+Essential, modern goods help friendly roads. Cultures	Books                                             	self-help                                         	1.13	8764.28	2.83903208813597843
+Gentlemen construct. Inevitable proposals tell more subject troops. Feelings used to come thus a	Books                                             	self-help                                         	1.73	8962.10	2.90311234660273887
+Miles kiss silently difficult streets. Less social rules see never 	Books                                             	self-help                                         	7.03	283.44	0.09181532938943778
+Yards shall build gradually steep, possible players. Foreign, wild lines used to understand vital layers. Problems shall go likely, parliamentary rats. Suspicious, wrong thousands 	Books                                             	self-help                                         	7.63	7823.86	2.53439981300044683
+Results	Books                                             	self-help                                         	9.21	3280.19	1.06255900829078431
+Smooth, othe	Books                                             	self-help                                         	8.62	11533.69	3.73613303141992873
+Available, other responsibilities ban common, english authorities. Participants save little for a years. Well local plans look. As entir	Books                                             	sports                                            	2.98	624.68	0.24146901355107034
+Now beautiful results scream just official payments. Carefully 	Books                                             	sports                                            	4.89	12518.36	4.83895120778186737
+Agricultural elections go users. Popular customers could threaten upside down hard, able pages. European, interesting bases spend at a fingers. 	Books                                             	sports                                            	2.47	7461.50	2.88423039734153702
+Levels should rethink really typically other women. Elections respond long numbers. Firms might sum nearly present, personal homes. Again clear	Books                                             	sports                                            	3.91	6886.83	2.66209266599525798
+Very social engineers ask facilities. Numerous, stupid 	Books                                             	sports                                            	7.36	4152.23	1.60503759066587821
+Green levels provide. Students would agree. Very upper states get finally for a	Books                                             	sports                                            	1.29	4251.46	1.64339478189126194
+In order	Books                                             	sports                                            	9.54	5723.96	2.21258720433787633
+As specific characteristics contain for the hours. Free, double men avoid in the meals. Trying, potential institutions share above from the months. Contemporary problems could cheer only heav	Books                                             	sports                                            	1.58	1246.89	0.48198325271610120
+Grounds ought 	Books                                             	sports                                            	1.69	6467.35	2.49994337066900616
+Completely particular voices shall not say straight. Used ideas must recall current colonies. New techniques could not make naturally old, great versions; great adults test	Books                                             	sports                                            	2.88	6653.24	2.57179884055600185
+Procedures order here shops. Late static sciences shall not see cultures. Polite implications cover underway. That is right communications might not say cool principles. Strange keys	Books                                             	sports                                            	1.34	2498.12	0.96564412520362400
+More big results develop again on a politicians. Characteristics live flowers. Children wipe perhaps appropriate roles. Wrong, external shows want somewhat little ways. Then difficult	Books                                             	sports                                            	3.64	4362.77	1.68642147699654727
+Basic, functional circumstances must 	Books                                             	sports                                            	7.87	2947.46	1.13933575379592397
+Neighbours shall not represent overall dramatic trees. Random chiefs could not interfere basic, special fruits. A little poli	Books                                             	sports                                            	5.46	3974.85	1.53647164710487281
+Immediately impossible teachers cut kindly busy, national products. Important, principal communities could die all very video-taped words. Short children doubt windows. Sometimes russian developm	Books                                             	sports                                            	96.08	4160.79	1.60834644440858994
+Twice detailed customers know women; economic, intense values listen wide industr	Books                                             	sports                                            	0.74	6802.45	2.62947571753614401
+Sad, very sales could gather hence on a pounds. Issues see just within a things. Eastern directors put very in a initiatives. 	Books                                             	sports                                            	3.99	5533.59	2.13899999791263899
+Sick organizations cannot cause to the situations. Direct nations seek to a genes. Able, invisible polls c	Books                                             	sports                                            	52.92	10879.04	4.20527479218581719
+Letters help; international directions should hu	Books                                             	sports                                            	37.74	460.35	0.17794752575436260
+Appointments might not hold to a tickets. Proper, private areas describe and so on prime, natural calls. Miners shall receive typically safe units. Little times will develop pointedly very mus	Books                                             	sports                                            	6.13	3351.79	1.29562884185557735
+Ministers prove perhaps social processes. Aggressive characters could get open signals. Products try at all public, loyal councils; wholly historical respondents see there from a statements. C	Books                                             	sports                                            	7.24	13142.40	5.08017283039890319
+Likely days shall get. Great users would love even. However acceptable walls	Books                                             	sports                                            	8.23	2406.70	0.93030587647013029
+Just average men might make so faintly free parents. J	Books                                             	sports                                            	1.41	9937.58	3.84135499725434718
+Papers conceive over immediate victims. Local, expert members add ill over immediate tiles. Profits pay even. Tall classes begin for instance grand fields; ru	Books                                             	sports                                            	0.25	3880.85	1.50013610366855243
+Great, reliable children see french, proper dates. Public passages like closely traditionally academic books. Values used to distinguish leaders. Much key oper	Books                                             	sports                                            	31.97	1293.62	0.50004665638396557
+Dual months should survive only large, political eyes; new, new merchants pass fairly conseque	Books                                             	sports                                            	6.26	4192.74	1.62069666369359458
+Conversely good eggs would not call too. Police happen present courses. Fine procedures finish well forward private	Books                                             	sports                                            	6.31	6912.27	2.67192645562313022
+Real, japanese systems would need downstairs for the phrases; level waters might not go about existing, little friends. Nation	Books                                             	sports                                            	5.90	2794.92	1.08037167086213344
+Devices take truly afraid, great men. Both true parties hurt even with a proposals. All internal candidates prevent more. Distinctive, prime women would say. Little, english departme	Books                                             	sports                                            	0.63	1050.56	0.40609221821766738
+Parents prevent alone little children. Cases might dispose again lives; very strange windows violate officially. Improved, cheap critics should alert plates. Expressions build c	Books                                             	sports                                            	5.56	4342.45	1.67856681484095121
+Wrong others miss less to the respects. Especially other standards start in order regula	Books                                             	sports                                            	7.53	11059.22	4.27492307108322362
+Adults will foresee most left, social children. Different eyes make personal counties. Readers would not admit more musical proceedings; titles take here away fast institutions; bird	Books                                             	sports                                            	3.83	10985.10	4.24627210853535058
+International, coloured contexts think. Relevant, british conservatives ought to happen ago. Perhaps human shops must see animals; rights must h	Books                                             	sports                                            	44.83	10933.78	4.22643444801245737
+Years should comment then central, internal implications; directly collective feet may find around extra, victorian crimes. Short	Books                                             	sports                                            	2.75	1868.42	0.72223463901372038
+So single phrases could not sum; desirable friends see times. French efforts think	Books                                             	sports                                            	4.59	4611.30	1.78249033455217177
+Central, visible moments 	Books                                             	sports                                            	57.64	1362.54	0.52668756759280813
+Old, straight enemies obtain however years. Largely social questions disrupt never. Measures rule fine, extensive trees. Already economic friends would not show more beautiful engines. Systems ret	Books                                             	sports                                            	9.99	4644.12	1.79517685088812959
+Freely proud children cannot continue countries. Rates shall not look applications. Compl	Books                                             	sports                                            	4.13	886.97	0.34285677618843706
+Already secondary year	Books                                             	sports                                            	72.51	8152.72	3.15142033706550904
+Developers ought to recognize again. British, fast artists shall experi	Books                                             	sports                                            	79.00	2317.17	0.89569820408870728
+Paths check still international attitudes. Immediate	Books                                             	sports                                            	0.37	2211.39	0.85480912127281399
+All capital bacteria make jobs. Again appropriate eyes may not leave others. There fixed ways	Books                                             	sports                                            	0.32	7910.07	3.05762438371632671
+Papers occur critically relatively happy numbers; related, soviet genes experiment governments; voluntary devices	Books                                             	sports                                            	2.52	3864.91	1.49397452321775512
+Indeed similar changes might drink too national careful areas. Wise, good rooms give large opportunities. Various patients shall research directly plants. International hands can get pieces	Books                                             	sports                                            	9.31	3710.53	1.43429919134861534
+Here familiar rooms would not believe particularly new, fresh rights. Levels allow then wives; temporary, big ears may sound always others. Lovely, essentia	Books                                             	sports                                            	9.23	1808.93	0.69923887859854273
+Lines might clear too high eyes. Great women balance as the things. Natural requirements	Books                                             	sports                                            	8.76	5395.16	2.08549011197764081
+General, local thanks must soar actually about p	Books                                             	sports                                            	22.08	7752.94	2.99688604392750734
+Inc others look in the varieties. Cold methods write values. Partners will make often times. Democratic, dramatic personnel shall not see	Books                                             	sports                                            	3.64	473.00	0.18283736218488870
+Others used to coincide there so as historical sites; syste	Books                                             	sports                                            	4.08	4391.31	1.69745356646114923
+Poor, major pairs affect complex, redundant results. Different animals could find so great, honest designs. Dull, linguistic studies might not get more cons	Books                                             	sports                                            	33.21	1010.07	0.39044087615663959
+Open prod	Books                                             	sports                                            	2.74	12438.41	4.80804666844427361
+Bloody masters pull only women; shops take aggressively also legal cells. Continually underlying grounds would interfere. Entries shall not separate. Senior techniques see in 	Books                                             	travel                                            	2.25	4171.41	1.64665291182793628
+Active, mi	Books                                             	travel                                            	1.40	12936.19	5.10652631399441219
+Voluntary others will imply again international, important birds; ill old publishers can get dark powers. Features stretch now progressive procedures. Tough n	Books                                             	travel                                            	1.83	3612.43	1.42599705573765030
+Cold terms shall comply only early claims; head, different politicians shall not commend good, foreign organizations; criminal, po	Books                                             	travel                                            	1.03	5504.86	2.17302872367020583
+Operations s	Books                                             	travel                                            	1.00	193.62	0.07643097580629212
+Applications might gather rather out of a problems. Scales could observe presumably for a directors; totally empty questions will forget. Just, symbolic question	Books                                             	travel                                            	21.48	5351.75	2.11258896173599765
+For example influential subjects shall work for example. Modules should persuade aside overall preliminary relatives. American, available reasons may use to the weekends; streets used t	Books                                             	travel                                            	2.18	6997.28	2.76215751673304277
+Similar sides assess more yet complete improvements. Bacteria would stay; general, curious trends used to reac	Books                                             	travel                                            	1.61	221.43	0.08740889873353613
+Communist, small cases may not turn other rules. Little, forward men should assist quite available technique	Books                                             	travel                                            	2.29	16204.92	6.39684871636659094
+Conflicts could give really sole institutions. Then advanced proceedings could not receive. Black experiences shall 	Books                                             	travel                                            	1.91	5880.48	2.32130371144192077
+Leading players will sa	Books                                             	travel                                            	4.51	262.65	0.10368038320174892
+There european members turn; industrial, honest leaders cut exactly happy, consistent reasons. Incidentally european millions worry at first aware 	Books                                             	travel                                            	3.81	2395.24	0.94551456714318326
+Deliberately ordinary procedures will not pay by a months. Feet reach very s	Books                                             	travel                                            	9.43	1776.74	0.70136335065629308
+Good, national parts remove animals; 	Books                                             	travel                                            	2.57	3370.45	1.33047609960911726
+Odd, artistic databases open now; female, left days use all obligations. Simple, early sites may not hesitate statements. Left, free s	Books                                             	travel                                            	2.31	9717.76	3.83605970174234756
+However solid hours visit painfully things. Clubs must take most other words; officials will follow necessary developers. Alternative, great decisio	Books                                             	travel                                            	2.68	1892.11	0.74690534879063830
+Finally surprising cells cannot look better points. Elections might choo	Books                                             	travel                                            	1.98	3145.02	1.24148821160161580
+Right schools go now; average, invisible hands should get also good persons. Usually good ministers will make. Notes ought to stem average words. Heavy, certain suggestions summaris	Books                                             	travel                                            	4.55	337.50	0.13322721999082528
+Thanks could talk well individually national records; just simple officials go then encouraging, remarkable needs. Signals assess now. Upper, cheap pp. would not see. Hard trousers shall send whol	Books                                             	travel                                            	4.23	6920.66	2.73191197719023675
+Reports used to think characteristics. True types break extremely deliberately white tasks. Courses must cost. Economic, nervous resou	Books                                             	travel                                            	0.74	1273.19	0.50258833842998175
+Dear signals know finally. Positions answer payable payments. Superior babies can exis	Books                                             	travel                                            	1.78	16390.16	6.46997170964392568
+Horizontal problems continue members; modern, other interactio	Books                                             	travel                                            	8.51	2371.88	0.93629326978322569
+Open conditio	Books                                             	travel                                            	8.17	9456.37	3.73287670016189772
+Practical writers used to succeed recent arms. 	Books                                             	travel                                            	9.48	10115.82	3.99319281934100804
+Members show yards. Economic stones get newspapers. Only magic views lea	Books                                             	travel                                            	9.23	1653.26	0.65261995176898313
+Investments ought to use still also professional developments. Only fresh visitors know steadily never main occ	Books                                             	travel                                            	1.37	4036.41	1.59336202383160616
+Conclusions might take on a ch	Books                                             	travel                                            	4.48	4341.46	1.71377969333738765
+Small, original things announce in addition at last other functions. Best political women make even old materials. Downstairs wet arr	Books                                             	travel                                            	0.34	8289.45	3.27223815926799005
+Again english deals cut for the cases. Yet normal systems reach biological, original reasons. So other remains spread steadily. Much inadequate members consider	Books                                             	travel                                            	1.92	7192.94	2.83939377706905721
+Later severe rules would	Books                                             	travel                                            	1.57	3713.31	1.46581916522705609
+Movements may describe quite southern, nervous views. Young notes imagine. Sensitive women might excuse then sales. Proportions may not exist only from a controls. Are	Books                                             	travel                                            	2.49	6651.86	2.62580389797976612
+That is fine terms know to the goods; useful colleagues us	Books                                             	travel                                            	6.31	6202.60	2.44845971767434933
+Yesterday long babies may not include as else able companies. Large, true d	Books                                             	travel                                            	4.19	1813.84	0.71600847617232157
+Words see low courts. National, democratic plants avoid. Days should go stupid, apparent days. Dependent hours should not want police. Also urban wages shall not define so great, typic	Books                                             	travel                                            	8.88	8312.77	3.28144366673520796
+Masses can contain as. Military men retain in a earnings; british, related instructions shall know different, precise needs; favorite	Books                                             	travel                                            	5.09	959.36	0.37870478746784635
+Behind relevant areas find then necessary papers. Copies might come envi	Books                                             	travel                                            	7.07	7437.38	2.93588581160107894
+Remarkably good bishops would deprive transactions. I	Books                                             	travel                                            	0.59	7014.30	2.76887611609376528
+Running businesses find emotions; 	Books                                             	travel                                            	4.40	2300.61	0.90815962839434831
+Pink, central countries shall defend rapidly 	Books                                             	travel                                            	6.87	6536.14	2.58012373828394893
+Local, conservati	Books                                             	travel                                            	1.68	8121.86	3.20608245616202735
+Strong women know also also obvious votes. Private, natural areas should play strongly for 	Books                                             	travel                                            	2.11	184.12	0.07268087628062445
+Colours meet certainly hours; aw	Books                                             	travel                                            	1.63	5441.98	2.14820701228347073
+Too full weeks might obtain most today vital cities. Police shall take for example full sto	Books                                             	travel                                            	3.82	5904.69	2.33086054402259597
+Exceptional hundreds compare else then previous scientists. Rapid, popular differences get exactly now social persons. Naturally fundamental dreams hold on a changes. Brilliant birds pursue te	Books                                             	travel                                            	5.39	3124.51	1.23339194409935853
+British leaders can focus. Different workers cannot breathe only in an objectives; arrangements might enter predictably hours; reduced, effective phases operate ready men. Others say o	Books                                             	travel                                            	4.95	1624.50	0.64126701888917236
+Yesterday public notes work at least students; accidents might not apply today rural, subject premises. National, particular organisations could not endorse simply under a respondents. Sti	Books                                             	travel                                            	9.83	531.86	0.20995030881280099
+Maybe gastric variations will see as. However physical plants would not choose for example wi	Books                                             	travel                                            	6.36	1691.34	0.66765192965713314
+Little arts can grow directly rights. Full, slim argum	Books                                             	travel                                            	4.77	16542.31	6.53003251415238218
+About right clothes must get thoughtfully to a cases. Eastern improvements 	Books                                             	travel                                            	98.75	2730.37	1.07780623598918408
+Countries want incorr	Books                                             	travel                                            	63.33	473.46	0.18689706541290708
+Fields would die clear horses. However new problems go nasty, smooth ways. Interested others go great societies. Familiar patients shall seem trends. Yellow, r	Home                                              	NULL	NULL	7995.48	34.64319649767261090
+NULL	Home                                              	NULL	0.87	14048.70	60.87087637475838958
+NULL	Home                                              	NULL	NULL	116.76	0.50590328824138814
+Neat, desirable words make especially gradu	Home                                              	accent                                            	7.11	1583.88	0.73384072874422647
+Common males protest probably statements. Subsequent, main ways begin then titles. Rights come therefore interesting, ordinary thin	Home                                              	accent                                            	8.82	1429.40	0.66226730413099308
+Offers go odds. Black, certain readers prove again in a cases. Public, black things watch as else modern forces. Difficult, new crops comp	Home                                              	accent                                            	3.59	4707.69	2.18115934307012370
+National, round fields would not accomp	Home                                              	accent                                            	0.17	1970.93	0.91316811090730250
+More general applications work also moves. Final, equal instruction	Home                                              	accent                                            	33.79	1466.94	0.67966027642501678
+Severe plants filter fair with the days. Both great hills bring still. Military standards ask now for a conditions. Ago new proposals may like particularly men. Then alone a	Home                                              	accent                                            	5.54	6369.32	2.95102307649896240
+Present, good grounds fall students. Big, long nerves remain events. Important, black years must not use principles. Fatal mines cannot order hospitals. Forces apply elsewhere; now final members	Home                                              	accent                                            	5.37	187.59	0.08691389644741359
+Terms must work slow signs. Just american movements make surprisingly	Home                                              	accent                                            	0.26	481.20	0.22294880841460324
+Discussions could inform; legitimately potential miles remember again from the factors. Then administrative changes may	Home                                              	accent                                            	2.20	1475.60	0.68367261366705848
+Ago light fingers blame enough green, british years. Children go general stands. Economic, great numbers affect deputies. Purposes urge annually. Always electrical ways vote judicial, regular ac	Home                                              	accent                                            	6.86	11873.28	5.50110895256222018
+Days shall want later romantic, american changes. Reasons read; great reasons may occupy economically. Strong, new films go then objects. English relations would resolve over. New, crazy feelin	Home                                              	accent                                            	1.78	715.86	0.33167110139583931
+New, large words stop more strong cars. Back views leave other, young shoes. White conte	Home                                              	accent                                            	2.81	9585.07	4.44093918343840622
+Decades try then. Different leaders stray examples. Things would not participate too good, good messages. Exactly new thanks can forget; companies u	Home                                              	accent                                            	3.51	4955.85	2.29613643429241784
+Very afraid concepts will not disentangle with a days. Long-term, civil points c	Home                                              	accent                                            	8.15	3501.80	1.62244833189164095
+New measures shall pay under a agencies; comparatively heavy police shall beat similarly concepts. However japanese times cannot check like a police. Long, long-term auth	Home                                              	accent                                            	1.87	5547.93	2.57045798559357804
+Useful, n	Home                                              	accent                                            	9.44	3014.70	1.39676594498650122
+Days give briefly vulnerable months. Sexual feelings create just animals. Charts study; changes knock rapidly aware sites. Schemes include sufficiently. For example speci	Home                                              	accent                                            	7.15	303.87	0.14078855863039378
+Connections must not come right finally certain parties. Wild parties fi	Home                                              	accent                                            	2.55	1293.30	0.59920967149336320
+Little powers reach by a subjects; traditional insects make also others. Numbers shall make. Products take serious, military rules. Curiously economic methods approac	Home                                              	accent                                            	3.52	99.03	0.04588241998607265
+Old buildings must proceed;	Home                                              	accent                                            	9.33	595.01	0.27567907417866391
+Additional eyes give nationally. Territorial groups should talk previously strange differences. Small discus	Home                                              	accent                                            	6.07	18159.55	8.41365343691896978
+Almost busy pounds lose at last for an factors. Good mothers would	Home                                              	accent                                            	1.45	2292.51	1.06216203819318802
+Benefits might choose only by a directors. Continued eggs must not make much black, back arrangements. Living,	Home                                              	accent                                            	1.62	9494.68	4.39905983432661074
+Holes may avoid of course genuine	Home                                              	accent                                            	3.27	409.64	0.18979374455311320
+Supporters will laugh well indirect, old reductions. Men can increase critical words. Eyes ought to drift better parties. Other, social goods avoid costs; similar, substantial days learn;	Home                                              	accent                                            	63.79	5475.88	2.53707589572185700
+Main, powerful kilometres should like certainly political directors. Left families go tall, clear organizatio	Home                                              	accent                                            	0.18	11613.93	5.38094732857567124
+Promptly soviet faces could confirm now consistent new procedure	Home                                              	accent                                            	1.85	5675.68	2.62964690968951645
+Old events can try far natural genes. Primary months explain at all par	Home                                              	accent                                            	0.15	20335.22	9.42168135463177076
+Women should hear among a pages. Everywhere main techniques go just unlikely principles. Broad, willing differences can make also short, modern roots. Together sorry thoug	Home                                              	accent                                            	8.25	1632.64	0.75643213335415177
+Attractive, pale rights stop in a delegates. Answers go as; variable, alone roles ought to relax quickly concerned, detailed parents. Poor, physical matches would send as for a details; cent	Home                                              	accent                                            	1.45	989.82	0.45860180703437776
+Ancient periods will not see in a affairs. Fun	Home                                              	accent                                            	4.09	8014.62	3.71332082064806196
+Perhaps material e	Home                                              	accent                                            	6.64	2552.44	1.18259238684490834
+Here german thanks trust further remarkable towns. Other years	Home                                              	accent                                            	2.04	7200.88	3.33630011541261051
+Supreme others can decide. Unfair, short presents give. Activities give simply police. Dark, impossible 	Home                                              	accent                                            	0.13	2033.98	0.94238033528498482
+Still different holes ought to enjoy early problems. Mammals see usually. Powerful, public 	Home                                              	accent                                            	6.84	1085.87	0.50310353822353537
+Always potential wages shall not restart sometimes at the efforts. Mere, high weapons would not go there physical pr	Home                                              	accent                                            	66.58	7246.44	3.35740890118021093
+Boys ought to answer. International citizens call areas. All quick cuts might back most white, central amounts. Strong mice make on a lines. Cultures would dismiss changes. Left chil	Home                                              	accent                                            	5.45	18131.76	8.40077781891015469
+Most main firms would know highly for an companies. D	Home                                              	accent                                            	1.31	5733.85	2.65659814033265334
+New investors think especially secondary parties. Farmers detect adequately. Hum	Home                                              	accent                                            	38.04	1460.72	0.67677843605024781
+International, nice forces will turn modest ways. Trees might not deal eastern others. Responsibilities ought t	Home                                              	accent                                            	2.75	6806.25	3.15346077986677743
+Quite political women like home seriously formal chains. Certainly male lips 	Home                                              	accent                                            	4.86	1551.13	0.71866705152980782
+Rules meet as; authorities shall not kill moreover near a 	Home                                              	accent                                            	3.55	651.58	0.30188899540063836
+Also possible systems could go forward. Local, british babies d	Home                                              	accent                                            	2.53	2797.54	1.29615172379922932
+British results cou	Home                                              	accent                                            	4.30	118.60	0.05494956084366572
+Simply perfect shareholders come others. Other, tired eyes contact therefore educational jobs. Over cathol	Home                                              	accent                                            	7.12	11929.65	5.52722621010654933
+Enough labour losses demonstrate also quickly happy women; near available things might surrender also ge	Home                                              	accent                                            	1.26	1093.19	0.50649502882535352
+Royal children 	Home                                              	accent                                            	3.70	188.00	0.08710385698658647
+Future, real fears mean far interests; ill, mean payments speak far so labour lights. Already other applicants might not go so powerful lengths; japanese, central modes boil. Old homes ough	Home                                              	bathroom                                          	1.70	19546.11	7.34362930968507144
+Also eastern matters should not enable now irish, 	Home                                              	bathroom                                          	3.46	2574.19	0.96714369931910820
+Quite public shoulders help even ministers. Short, tall groups cannot overcome too other notes. Thus surprising reasons find	Home                                              	bathroom                                          	1.77	11046.40	4.15022051991445731
+In	Home                                              	bathroom                                          	0.42	1225.60	0.46046768804381146
+Necessary, p	Home                                              	bathroom                                          	8.13	5680.58	2.13423918027734537
+Letter	Home                                              	bathroom                                          	9.54	6366.89	2.39209131717465953
+Modern companies shall not become also old, grateful agents. Enough joint programs approve titles. Jeans will not fall already wrong teachers. High, silver children manage a	Home                                              	bathroom                                          	2.28	16790.19	6.30820820097611185
+Detailed, unhappy groups play old, human others. Well anxious councils will study whole, democratic employees. Educational, english customers get more. Explicitly cold deci	Home                                              	bathroom                                          	79.37	2249.42	0.84512502189907830
+Pp. may not record also human rocks. Extraordinary, industrial measures may not operate only out of a officials. Ready subjects show clearly new things. Projects should enable	Home                                              	bathroom                                          	3.56	11356.89	4.26687408752274959
+Here economic areas develop too sole processes; grateful, new children pass shares; fat, proposed aspects affect gmt on the terms. Years remind e	Home                                              	bathroom                                          	6.16	5399.13	2.02849617211813296
+Appropriate, active areas change alternative books. Clients will not look now only, other rates. Usually effecti	Home                                              	bathroom                                          	2.89	2344.36	0.88079473657179327
+Employees watch never at the imports. Cases resist actually reliable prices. Alive, var	Home                                              	bathroom                                          	7.17	2759.95	1.03693521182809843
+Very oral hands ought to smoke military, independent issues. Moving sons play. Patients contradict to a measures. Other cattle enable significant goods. Initial, possible groups let soci	Home                                              	bathroom                                          	7.17	3821.04	1.43559518172562445
+New sports will give now students. Scarcely free countries damage there prime, necessary members. Big units should not fill probably mental child	Home                                              	bathroom                                          	4.29	1777.37	0.66777207465602902
+Unions last moving pur	Home                                              	bathroom                                          	2.72	3881.21	1.45820153028110433
+Indeed political miles imagine. Urgent, able males can explain companies. Accor	Home                                              	bathroom                                          	5.47	2914.22	1.09489568036148517
+Almost other bodies call cars. So international benefits ought to suppose in a points. Officers can ensure also for a books. Carefully different police sleep. Irish, u	Home                                              	bathroom                                          	9.17	4471.44	1.67995564541989254
+Labour, japanese economies care more minor, great gardens; events may m	Home                                              	bathroom                                          	5.15	5956.38	2.23785943840600333
+Smal	Home                                              	bathroom                                          	3.40	1261.44	0.47393306168895686
+Free, sad bits might not speed then. Troubles	Home                                              	bathroom                                          	5.76	175.15	0.06580525094718797
+Hard players show empty troops. Expectations used to know even; alternative organs could not consume historical, direct practices. Material restrictions could count deep. Gifts could s	Home                                              	bathroom                                          	4.64	8640.19	3.24618824539756797
+Mere, alternativ	Home                                              	bathroom                                          	6.84	4069.67	1.52900745430912057
+Strong taxes represent nece	Home                                              	bathroom                                          	3.36	2436.99	0.91559656583378597
+Simply costly processes should not believe therefore by the weeks. Instead earl	Home                                              	bathroom                                          	7.28	419.52	0.15761700757844303
+Joint lovers can mention tomorrow minor techniques. Major markets may no	Home                                              	bathroom                                          	17.20	2682.86	1.00797188442005549
+Pretty figures ought to join that things. Extra authorities find dramatic items. Over mutual cases give for the time being as successful lines; permanent arms return publi	Home                                              	bathroom                                          	0.31	15228.27	5.72138240845865918
+Both long tories will not get together; problems seem by now special,	Home                                              	bathroom                                          	5.62	8655.20	3.25182762202741263
+Sanctions will know black quarters. Cent	Home                                              	bathroom                                          	4.35	2089.84	0.78516954404494038
+Comfortable clothes ought to carry violently. New, united services must look always. Common, recent workers could prevent. New, local languages need very often young kinds. Structures might	Home                                              	bathroom                                          	1.84	4089.18	1.53633751680400859
+Drivers might put 	Home                                              	bathroom                                          	7.91	1583.75	0.59502749750276305
+Financial forces may bring yet. Unknown, expensive assets offer enough securities; female movements ought to grow great, aware modules. Normal contacts mus	Home                                              	bathroom                                          	2.10	4156.11	1.56148365123675362
+By now developing masses used to flourish subtle methods. Much 	Home                                              	bathroom                                          	9.84	4755.08	1.78652145403342606
+Thereby social children should report to a days. Times meet anyway as a whole liable reasons. Physical, region	Home                                              	bathroom                                          	5.82	12047.28	4.52625911293770307
+So present rises l	Home                                              	bathroom                                          	5.86	3137.27	1.17869734307213477
+Philosophical,	Home                                              	bathroom                                          	6.72	3878.46	1.45716833336357782
+Single p	Home                                              	bathroom                                          	3.92	6593.22	2.47712530202694074
+Areas ride perhaps even leading women. High sides cannot get then throughout the officers. Long signs may not embrace to the friends. Very, tory	Home                                              	bathroom                                          	9.18	6130.98	2.30345804996968600
+Hi	Home                                              	bathroom                                          	2.13	440.85	0.16563085857874860
+Force	Home                                              	bathroom                                          	0.20	6396.38	2.40317094521024374
+Hard programmes make as other goods. Rational, similar computers could go to the streets. Options mi	Home                                              	bathroom                                          	7.10	4799.14	1.80307514719205068
+So straightforwar	Home                                              	bathroom                                          	1.16	1899.26	0.71356711912050371
+Properties go industrial troops; sweet companies would start more constant negotiations. Groups will protect. Public so	Home                                              	bathroom                                          	5.64	10621.64	3.99063480257316377
+Especially linguistic games cover to a officials. Minor, main days know completely variations	Home                                              	bathroom                                          	1.60	3572.22	1.34211152462782650
+From time to time successful books decide important, active elements. Parts will hear on a clubs. Firstly following supplies take barely upon a years. Other cases may find	Home                                              	bathroom                                          	3.90	218.22	0.08198699321550305
+Important kinds can catch again slim areas. Good, past men must 	Home                                              	bathroom                                          	5.17	6013.16	2.25919213694315054
+Formal, positive soldiers co-operate long along a offices. Great, able details must overtake responsible, remaining papers. Lives would think acute, labour shapes. Representative	Home                                              	bathroom                                          	10.92	3002.22	1.12795798172233325
+Social	Home                                              	bathroom                                          	5.38	4680.62	1.75854623858650847
+Main forms matter constitutional, popular animals; ministers might not allow hardly. Officials will think so. Soon brief relations interfere for example old terms. Co	Home                                              	bathroom                                          	8.37	867.00	0.32573880999835553
+Probably awful sales require massively as annual notes. A little national devices arrest sharply short, grateful legs. Trees may protect immediately in a courses. Indians will not get i	Home                                              	bathroom                                          	4.33	1138.62	0.42778860881237321
+Military characters would	Home                                              	bathroom                                          	2.10	8317.61	3.12499236843185918
+In particular acute origins could like thousands; impatiently small stones might give away female, crucial models. Colleagues might accompany bes	Home                                              	bathroom                                          	3.25	4807.80	1.80632877821233414
+Afterwards oth	Home                                              	bathroom                                          	0.24	7197.60	2.70419568494136532
+Material officials tackle employers. Clear shareholders go very products. Areas imagine systems; superior, precise tonnes will make much minutes. Milita	Home                                              	bedding                                           	18.44	3038.10	1.25620354127751860
+Large tests complain dark, pales	Home                                              	bedding                                           	37.80	10472.58	4.33023668816435133
+Great servants deal primarily certainly possible gates. Problems ca	Home                                              	bedding                                           	4.62	4172.20	1.72513492476154936
+Usually large paintings might not go beautifully local appeals. Clothes bring partially different, very orders. Fruits provide except a schools. R	Home                                              	bedding                                           	33.55	1050.47	0.43435177709943549
+Well healthy	Home                                              	bedding                                           	7.46	10368.46	4.28718480945140073
+Conditions know both popular	Home                                              	bedding                                           	2.48	18121.95	7.49312325626349635
+Payable, mutual pictures will not help new women; mole	Home                                              	bedding                                           	49.59	591.36	0.24451747018527152
+Increasingly sexual	Home                                              	bedding                                           	0.50	233.74	0.09664758096777828
+Thus angry stations would not demonstrate forward; single, political winds must not accept then dark profits. Patterns used to know obviously. Wars use particular met	Home                                              	bedding                                           	64.50	744.66	0.30790445641937955
+Notes shall say slightly to a files. Important suggestions stay today acts. New, true powers make in particular; awkwardly left prices g	Home                                              	bedding                                           	0.79	546.70	0.22605130707232133
+About political men	Home                                              	bedding                                           	3.09	589.74	0.24384762727790521
+Yet personal children answer; sp	Home                                              	bedding                                           	4.17	1458.28	0.60297439194699971
+Sacred, other police run competent, poor solutions. Just subsequent lips allow far all small sentences; programmes used to develop with a conditions. Properties m	Home                                              	bedding                                           	1.39	2951.80	1.22051993454559739
+Attractive, dead situations shall enter also great, forward groups; thus compatible sections give still troubles. Cold, known waters can ho	Home                                              	bedding                                           	5.95	634.78	0.26247091403579318
+New, hard children say needs. Particular, horrible sports can clean. Corporate, adminis	Home                                              	bedding                                           	8.14	2691.36	1.11283235010455958
+Female abilities remove hard, happy customs. Really current shoulders lead to a heads. Vast advantages ought to explai	Home                                              	bedding                                           	2.45	2906.03	1.20159480499611843
+Clearly profitable ages cancel above evolutionary lessons. Steps would live better; labour women can bounce inst	Home                                              	bedding                                           	3.09	4184.78	1.73033654437554205
+Usefully clinical hours laugh almost attractive instruments. Responsible, obvious results follow even powers. Away big cups should d	Home                                              	bedding                                           	9.21	12113.91	5.00889919381098232
+Of course political others should turn social, low charges. Thoughts must not expand. Prime letters will not correspond alone 	Home                                              	bedding                                           	3.60	3509.07	1.45094175984684579
+Immediately legitimate details may not laugh over bad, great publications. Pale conditions cost high, commercial arms; new problems should gai	Home                                              	bedding                                           	1.16	272.24	0.11256668709963190
+Criminal faces can exercise always to a members. And so on likely lines can know patients. New premises used to top also yesterday physical relatives. Organisational, alone operations 	Home                                              	bedding                                           	93.25	255.70	0.10572767371207712
+Expensive parents could become very over the implications; prominent reasons bring 	Home                                              	bedding                                           	92.94	4461.34	1.84468947922815077
+Just joint transactions might take now still national tests. Cells vary less so orange texts	Home                                              	bedding                                           	6.63	7559.57	3.12575576990069165
+Important, local transactions set overhead single prices. Available, white particles shall develop concerned, remote comments. Whole efforts m	Home                                              	bedding                                           	1.47	361.08	0.14930054135297930
+Eager, low years shall report clearly. Others should operate since a meanings. Directors would know holes. Poor boundaries hear early hours. Important countries make of course small, rec	Home                                              	bedding                                           	2.90	15764.84	6.51849769121275679
+Goods want special children. Personal plans remain. Payable, royal things go always concessions. Free, academic dogs raise still ra	Home                                              	bedding                                           	2.19	10328.90	4.27082741104682595
+Public applications will include less elderly, double businessmen. Federal cards impose partners. Places pay completely. Quite old ways deny ac	Home                                              	bedding                                           	6.98	7984.50	3.30145721843597883
+Good benefits pretend completely; s	Home                                              	bedding                                           	1.31	2239.67	0.92606608909944376
+Ways become just together close centuries; shots account also perhaps lengthy profits. Both eastern efforts might grab together tight countries. Police will express today for	Home                                              	bedding                                           	1.95	405.51	0.16767160331241453
+Electronic, long-term theories would give especially; elderly forms know yet later old risks. Different m	Home                                              	bedding                                           	82.96	15743.55	6.50969463226347981
+Double services precipitate finally demands. Authorities write early with a things. Full changes may not see in the doll	Home                                              	bedding                                           	4.48	1865.76	0.77146055731343376
+Critical, whole men forget in a industries. Alone lips look soon for a natio	Home                                              	bedding                                           	5.35	3628.30	1.50024137086245375
+Total, unlikely images get either open measures. Politicians visualise economically children. Able, ready states could not go in addition small 	Home                                              	bedding                                           	1.42	334.80	0.13843420085570364
+Firm managers will not walk at a g	Home                                              	bedding                                           	3.23	1994.75	0.82479576510428565
+There controversial beings upset sure at a arms. Broad circumstances see pale memb	Home                                              	bedding                                           	0.56	8534.56	3.52889782931617102
+Difficulties will not feel most. Like things used to avoid both favor	Home                                              	bedding                                           	0.82	2845.65	1.17662868478205813
+Special, true decades cannot convert cool normal, old-fashioned books. Old ministers become. Substantial, economic recordings see particularly patients. Mass, absolute thanks could not su	Home                                              	bedding                                           	3.58	8483.58	3.50781845189793992
+Areas cannot get just. Horses achieve finally sad fans; tough examinations will not love also concrete mines. Experts shall l	Home                                              	bedding                                           	6.67	1746.36	0.72209065414087995
+Questions will encourage finally final, small institutions. Additional holes enjoy alread	Home                                              	bedding                                           	4.45	7157.46	2.95949000972719407
+Able, small executives would complete ne	Home                                              	bedding                                           	5.70	11277.99	4.66326025360996743
+Shortly official associations find however weeks. Empty subjects draw much linguistic, whole powers. Typical, payable feet shall sink also narrow boys. Permanent, i	Home                                              	bedding                                           	4.13	10215.08	4.22376474455520053
+Nevertheless left things must appear for instance again h	Home                                              	bedding                                           	6.76	6935.76	2.86782076740428637
+Enough lost problems manage before excellent champions	Home                                              	bedding                                           	0.97	425.46	0.17592059467164776
+Crude dates should convin	Home                                              	bedding                                           	9.48	2442.81	1.01006108181696956
+Personal, major issues say palestinian, german gods; angry styles keep surprising, pleased years. Authori	Home                                              	bedding                                           	8.78	375.34	0.15519681287090742
+Final off	Home                                              	bedding                                           	4.48	10411.01	4.30477852285167011
+Children used to solve all right required, military a	Home                                              	bedding                                           	4.08	5342.86	2.20918325682169878
+Able, red references might hire so direct children. Experiments ban too different, labour met	Home                                              	bedding                                           	4.41	1941.93	0.80295557845793480
+Then distant children plot. Previous roads w	Home                                              	bedding                                           	8.48	514.40	0.21269579725261037
+Powerful, happy companies seem also very national implications; children scan natural charts; really single subjects used to preserve. New re	Home                                              	bedding                                           	1.99	9617.02	3.97647693641971033
+Slight, royal projects will ask audiences. Elabora	Home                                              	blinds/shades                                     	5.27	7981.68	2.95699007390289399
+Years say much at a eyes; surely different theories may hear well powerful, free wars. Well little conservatives weave physical, fundamental servants; c	Home                                              	blinds/shades                                     	4.42	1284.84	0.47599742492224623
+States must not harm maybe late changes. Good, original steps must abandon incredible, useful neighbours. Sure annual shareholders could analyse absolutely patently dark 	Home                                              	blinds/shades                                     	7.32	10474.36	3.88045856893354741
+Very able governments m	Home                                              	blinds/shades                                     	2.20	7440.10	2.75634977208368684
+Companies want as from the reports. Often different purposes will not work cases; principal towns guess 	Home                                              	blinds/shades                                     	9.34	5385.32	1.99511102735147651
+Cells cannot give. Indeed english trees shall talk ever. In particular foreign things may catch too soviet, rich situations. N	Home                                              	blinds/shades                                     	0.28	8695.50	3.22144049719139513
+Tiny, exi	Home                                              	blinds/shades                                     	7.04	7025.12	2.60261124324411636
+Women must sleep in a scales. Agents can get generally extraordinary, general studies. Central systems profit; either comprehensive rivers use in the cars; cases shall ke	Home                                              	blinds/shades                                     	0.63	5940.92	2.20094534857964501
+Theories employ more specific offenders. Modes must see preferences. Certainly main studies see from the varieties. Pleasant elements 	Home                                              	blinds/shades                                     	97.19	4156.26	1.53977853842294381
+Young opinions make fine weeks; copies would reply soon from the accountants. Interesting losses would go only slightly old families. Most famous patterns ex	Home                                              	blinds/shades                                     	2.76	8530.68	3.16037927900416200
+Industrial losses take letters; organic, likely yards could know all possible questions. Old studies retrie	Home                                              	blinds/shades                                     	9.59	8586.88	3.18119981329686010
+New, light associations feel on a officials. Potential, enormous customers find in the me	Home                                              	blinds/shades                                     	4.62	4568.78	1.69260570579703321
+Certainly tory systems read now in a prisons; evenings may seduce anywhere other months; new customers talk with the cells. Police lead more other exports. Young provisions 	Home                                              	blinds/shades                                     	7.50	11150.34	4.13089032642781908
+Common, interesting figures would not see high; naked studies would get both. Changes might face over prayers. Tremendous, intact considerations shall choose just.	Home                                              	blinds/shades                                     	1.19	3490.71	1.29321080535345580
+True, impossible trees could get no longer exclusive cel	Home                                              	blinds/shades                                     	7.65	13982.16	5.18000074316711372
+Less whole balls should purchase often difficult specialists. Impossible, international intentions will not counter completely during a trees. Important sciences look initia	Home                                              	blinds/shades                                     	0.25	4673.99	1.73158307969266965
+National, electric sections must market in the decisions; b	Home                                              	blinds/shades                                     	3.94	13578.70	5.03053005338540591
+Thin, financial others can mobilize from a stories. Anywhere related others should remain following patients. Equations sh	Home                                              	blinds/shades                                     	5.47	1070.00	0.39640519027023090
+Steep, slow terms get. Affairs will decide upwards dominant courts. Familiar, serious years add	Home                                              	blinds/shades                                     	2.80	2331.69	0.86382618514130345
+Available laws get worldwide waste, new policies; then other societies understand by a interests; often local problems can help whole hours. Certain, m	Home                                              	blinds/shades                                     	8.96	9879.49	3.66007580675032100
+Clear accounts will not play even spectacular offices. Christian, impossible changes say for ins	Home                                              	blinds/shades                                     	0.25	7864.42	2.91354851071496196
+Rural, top years must accept again unusual shelves. Directors used to move later known, form	Home                                              	blinds/shades                                     	4.05	3163.86	1.17212198625081564
+Healthy directors understand at least young conditions. Excellent members prevent well meetings. Obvious	Home                                              	blinds/shades                                     	4.77	821.24	0.30424654061450881
+Thoughts must not achieve forward from the eyes. Powers seem recent	Home                                              	blinds/shades                                     	1.53	8071.29	2.99018808240767473
+Services must move amongst a bedrooms. Small markets used to advance in a courses. New levels could say from a centres. In particular present buyers must not transfer again. Indian, net courses s	Home                                              	blinds/shades                                     	0.19	3825.58	1.41727081102242049
+Different, upper days receive thorough, personal couples. Social, new girls must not prove strangely in a words; feet shall help however now full th	Home                                              	blinds/shades                                     	4.79	7716.79	2.85885570862188328
+Scarcely crucial groups may bring especially effective, important losses. Now new drugs wan	Home                                              	blinds/shades                                     	3.48	2706.56	1.00270507642784686
+Short candidates shed women. Involved, wooden needs might violate then long-term times. Students must not	Home                                              	blinds/shades                                     	5.18	NULL	NULL
+Only normal subjects might create over in the teachers. Main hours used t	Home                                              	blinds/shades                                     	4.63	2891.18	1.07110164299578147
+Bars like full, traditional politicians. Things used to show properly at the holidays; less specific relations may say possibly. Forces could 	Home                                              	blinds/shades                                     	6.30	144.44	0.05351099596507678
+Prime, international results test ever conditions. Territorial users should love never barely emotional detectives. Firms resi	Home                                              	blinds/shades                                     	3.79	5465.05	2.02464877110871531
+Conditions make patients. New, various eggs will not watch appropri	Home                                              	blinds/shades                                     	2.22	360.68	0.13362189161370737
+Already early meetings cannot go animals. As comprehensive evenings w	Home                                              	blinds/shades                                     	4.11	511.70	0.18957059426287584
+Serious, free symptoms used to remember certainly able runs. Feelings shall pro	Home                                              	blinds/shades                                     	5.48	2291.60	0.84897395703108517
+Also long lines make further near a dogs. Rather foreign jobs can sit in the trends. Chronic police shall experience apparently diverse, proper years. Only notable companies migrate also years. Free,	Home                                              	blinds/shades                                     	73.55	6931.61	2.56796839339162169
+Complete costs become again industrial theories. Populations vary trustees. Countr	Home                                              	blinds/shades                                     	3.42	4143.26	1.53496240059723073
+P	Home                                              	blinds/shades                                     	2.11	8507.90	3.15193992364495091
+Minutes might not reply polish, main days. Main beans make properly agencies. As new years de	Home                                              	blinds/shades                                     	9.78	8403.34	3.11320335664060012
+Lives would look. Things exist for a patterns. Local, palestinian members should get problems; statements may not make yet nasty, specific men; numbers find clear women. Groups shall seem m	Home                                              	blinds/shades                                     	3.38	2112.47	0.78261128251416324
+Appropriate, extensive scenes stem openly now financial personnel. More concerned signs stay now members; also full days could prepare subtle horses. Ancient achievements say america	Home                                              	blinds/shades                                     	2.98	14371.92	5.32439596462480082
+Primary, occupational regions set particularly here prime ideas. Clinical, sophisticated minutes allocate just. Needs want interested let	Home                                              	blinds/shades                                     	4.77	5863.19	2.17214854910328515
+Large colours must win over; months assess extreme days. Blacks might signify then fully concerned points; here political potatoes might not die 	Home                                              	blinds/shades                                     	0.55	3969.07	1.47042985845407977
+Sad increases ought to mean too levels. Organs used to present; other, sympathetic controls like always new interests. Other, small women deal in a edges. Outcomes run 	Home                                              	blinds/shades                                     	8.43	7535.76	2.79178913703812636
+New parts come also old, tiny chains; responsible seats involve now properly small secrets; eligible chains get complete communications. Talks beat about married, liable books. Big,	Home                                              	blinds/shades                                     	7.11	1861.92	0.68978948772705450
+Social, central lights warn police. 	Home                                              	blinds/shades                                     	7.78	6660.62	2.46757414805393022
+Subjects sha	Home                                              	blinds/shades                                     	0.26	360.45	0.13353668302140629
+Free, educational times ensure practically. So linguistic officers need. N	Home                                              	blinds/shades                                     	9.32	4744.02	1.75752724368764560
+Just possible women say. Reasonably strong employers light almost degrees. Palestinian, smart rules help really visual	Home                                              	blinds/shades                                     	3.71	8398.39	3.11136951954542476
+Labour taxes could get even lab	Home                                              	curtains/drapes                                   	4.54	24984.53	7.47827965622433549
+All real copies loosen more definite doors.	Home                                              	curtains/drapes                                   	9.49	736.67	0.22049741477429358
+Very, various goods should turn local arran	Home                                              	curtains/drapes                                   	3.04	3989.59	1.19414972919947050
+Unlikely sides sell sometimes friends; mutual floors used to say i	Home                                              	curtains/drapes                                   	3.70	11830.01	3.54091604348492652
+Roads help less functions. Relevant, increased procedures may not respond. All labour children ought to say workers. Given findings could decide thus royal shareholders	Home                                              	curtains/drapes                                   	4.28	5979.42	1.78973848785712263
+Weak girls swim; provinces may introduce. Nervous, green tracks say better british, public rebels. Houses must not s	Home                                              	curtains/drapes                                   	8.21	9746.45	2.91727235835165499
+Mainly alternative politicians will not maintain from a matters. Principles should not tell always details; suddenly democratic years formulate far. Western, wise years ge	Home                                              	curtains/drapes                                   	2.73	3116.99	0.93296623573285915
+Public metres want; characteristics shoul	Home                                              	curtains/drapes                                   	0.82	6428.18	1.92405971697478996
+Services decide only easy, single bases. Now british solicitors ought to transfer now over a drawings. Thorough elections run still religious, tough parameters. Complete, sole consequences ac	Home                                              	curtains/drapes                                   	4.49	6448.14	1.93003407238344634
+New, intimate hours go unfortunately forms. Subsequently experienced advisers must feed n	Home                                              	curtains/drapes                                   	0.70	188.16	0.05631937443350629
+Words might correct long old, major relations. Visible, desperate policemen may become extra agreements. General, other students include so	Home                                              	curtains/drapes                                   	3.90	10122.80	3.02992008671076475
+Centres look nevertheless with a advertisements. Naked users address to a reports. Im	Home                                              	curtains/drapes                                   	3.82	6381.83	1.91018640168464850
+Clear partners ought to take effective, black books. Circumstances become hospitals. Forces answer gradua	Home                                              	curtains/drapes                                   	1.32	1013.02	0.30321350280947356
+Certain, conservativ	Home                                              	curtains/drapes                                   	0.28	11983.75	3.58693294731893617
+Private years forgive then in the computers; more exclusive differences get sources. Minutes meet insects. Small circumstances will contact sudd	Home                                              	curtains/drapes                                   	1.69	2179.00	0.65221044265843012
+Known, possible years may approve. Forth wrong aspects see again local girls. Excellent peasants can run usually with a exchanges;	Home                                              	curtains/drapes                                   	3.79	4760.53	1.42490471711277482
+Prime, national features claim different, great views. Versions might not sign european 	Home                                              	curtains/drapes                                   	0.67	9131.87	2.73331848324884729
+Free funds cause still new,	Home                                              	curtains/drapes                                   	4.69	8170.69	2.44562154278329893
+Years must not enable existing others; other, political ties like then short products. Quite	Home                                              	curtains/drapes                                   	4.35	696.96	0.20861156040166106
+Private parents carry really british dreams; writings look probab	Home                                              	curtains/drapes                                   	9.60	2216.28	0.66336895817119114
+Responses used to bring of course video-taped loans. Hot, positive systems would remember. New, personal words may not answer on	Home                                              	curtains/drapes                                   	6.31	2854.74	0.85447050898335328
+Germans will throw perhaps with a	Home                                              	curtains/drapes                                   	6.68	11036.19	3.30331269626550706
+Generally left questions bri	Home                                              	curtains/drapes                                   	93.18	2295.48	0.68707481730774354
+Particular, british wa	Home                                              	curtains/drapes                                   	3.20	6421.72	1.92212613300986409
+Democratic, likely appearances might expand both good, certain pounds; american values can pick. Only previous figures will not repa	Home                                              	curtains/drapes                                   	6.11	15070.04	4.51071016947234888
+Different, local measures say there political doors. Open assets progress minus th	Home                                              	curtains/drapes                                   	9.40	2024.63	0.60600496949037970
+Statements might not test financial, concerned authorities. United scenes back just bare publishers. More simple things could cope 	Home                                              	curtains/drapes                                   	0.37	4710.47	1.40992093796661557
+Accountants look equally marvellous, british schemes. Things shall study tiny events. Both normal courses could appeal faintly. Then black practices used to die hardly. Advisor	Home                                              	curtains/drapes                                   	2.23	9441.66	2.82604371180834938
+Valid resources ought to say still tears. M	Home                                              	curtains/drapes                                   	1.25	8697.98	2.60344808904734832
+Electronic reports try in comparison with the problems. Germans might not go as. Common, social cups come sure about intact	Home                                              	curtains/drapes                                   	3.25	817.84	0.24479292722522739
+Outside mammals can ignore eyes. Amounts stand always that is ready notes. Structures remember most attractive issues. Subjective difficulties cause very. Adequate, di	Home                                              	curtains/drapes                                   	1.51	3062.90	0.91677621148164553
+Small females would allow topics; local, local tears find	Home                                              	curtains/drapes                                   	0.60	123.41	0.03693863732376175
+Problems must not hate there in a stars. Fully forward teams may work yet white, concerned personnel. Merely common years stem methods; measures could introduce more on a areas. L	Home                                              	curtains/drapes                                   	3.73	15982.27	4.78375557199933360
+Here other years may like later. Terms call yesterday also	Home                                              	curtains/drapes                                   	1.50	1201.77	0.35970947392089103
+Free, competitive aspects get even specific, medical times. Other, free days 	Home                                              	curtains/drapes                                   	4.40	3406.63	1.01966023876708940
+National features sing then really magnificent values. Light, shallow advertisements should acknowledge. Possible, good forms should move anyway political, irish estates. Simply	Home                                              	curtains/drapes                                   	2.02	2017.71	0.60393369997996376
+Linguistic, appropriate degrees shout. Educational poles will study now in a names. Full arms look in a ways. Minute, modest systems deal unique experiments; automatically regular 	Home                                              	curtains/drapes                                   	2.54	6407.34	1.91782196313128299
+Active books find; important, remarkable personnel may turn alone prices; public eyes administer different, financial waters. Obvious, weekly managers cannot make so. Proble	Home                                              	curtains/drapes                                   	8.93	25.68	0.00768644523518517
+Socially extra interpretations continue other men. Also odd initiatives must need now by a hills. So gross rules can divide. Significant, impossible parent	Home                                              	curtains/drapes                                   	4.37	100.62	0.03011721649393815
+Effects might tolerate reasonably. Comparisons take other, clear others. French, christian 	Home                                              	curtains/drapes                                   	1.91	6527.01	1.95364115710692977
+New, different elections kill arms. As good as new yards would calcula	Home                                              	curtains/drapes                                   	0.59	4150.32	1.24225885469212285
+Events explore away. Unusual rights should affect so in a posts. New journalists might not find wrong scientists. For example tall authorities shall not con	Home                                              	curtains/drapes                                   	6.84	1245.00	0.37264892203292588
+Tall, whole women would not create. Still national hands bear around flat, poor attacks. Fiel	Home                                              	curtains/drapes                                   	6.19	2226.86	0.66653572571746292
+Months shall not find also intact forces; super ju	Home                                              	curtains/drapes                                   	0.99	6731.10	2.01472864184403808
+Superbly loyal police would contemplate twice sure nights. Even succ	Home                                              	curtains/drapes                                   	0.44	49.08	0.01469044907098474
+Legs solve by a women. Early, early weekends neglect again loans; proposals	Home                                              	curtains/drapes                                   	57.92	10980.48	3.28663777944104577
+Likely, normal policies believe very children. Twice old knees should suggest with a details. Lives take students; questions will not look as deeply ready areas; valuable members wor	Home                                              	curtains/drapes                                   	5.17	249.22	0.07459563401529782
+Budgets keep so lesser women. Stairs determine 	Home                                              	curtains/drapes                                   	1.55	4402.52	1.31774645158907378
+Di	Home                                              	curtains/drapes                                   	6.03	5657.98	1.69352622319988272
+Particularly old assumptions might learn repeatedly fine sessions; payments compete more bad times. Days will plan formerly; all right simple jeans reject weeks. Today national representati	Home                                              	curtains/drapes                                   	24.89	14029.64	4.19930138354218335
+Goals commit then obvious tracks. Excellent days k	Home                                              	curtains/drapes                                   	6.14	1920.32	0.57478327546848854
+Human drinks	Home                                              	curtains/drapes                                   	0.71	1522.69	0.45576609404844651
+Dead, obvious terms would serve more through a forces; worthy, possible arms decide for the falls. Rules 	Home                                              	curtains/drapes                                   	2.34	14312.02	4.28382234948889629
+Small branches cause smoothly right duties. Outstanding ministers give real policies. Increased, japanese settlements used to protect electoral, large offices; clouds 	Home                                              	curtains/drapes                                   	3.90	15202.77	4.55043843567430089
+Specific, small functions make about a children. Other, hot notes request however new things. Very slight eyes should want always serious, normal	Home                                              	curtains/drapes                                   	6.32	1409.34	0.42183857974127210
+Somehow surprising officials eat important cells. Mature police operate close, permanent flights. Old, fine engineers will pay away fingers. Hardly cultural activities watch gay, new 	Home                                              	curtains/drapes                                   	0.25	6118.86	1.83147516712481033
+New, perfect clothes let. High centuries could go months. Part-time, legal things think even there new systems. Aware losses come yet that wide functions. Big, british ears send please economic hee	Home                                              	curtains/drapes                                   	7.09	4208.63	1.25971199416500631
+Less than dark patients teach however national senses; as positive problems can take instead liberal collective sectors; urgent resources raise so southern motives. Private p	Home                                              	curtains/drapes                                   	0.67	7346.83	2.19902673081057097
+Still available arguments	Home                                              	decor                                             	6.57	7479.82	2.46562464048976131
+Then adequate experiments ought to need pp.. Able unions could need please on a countries. Women continue previously british ways.	Home                                              	decor                                             	0.96	3319.93	1.09437141705297364
+Now imaginative boys shall look. Experiments tell main confl	Home                                              	decor                                             	3.59	1502.18	0.49517395103771343
+Independent, limited numbers claim nonetheless to a firms; never managerial sources would want only special terms. Changing, present homes shall suffer 	Home                                              	decor                                             	6.24	1843.18	0.60758013225691504
+Fre	Home                                              	decor                                             	2.65	4396.90	1.44938046393755886
+Wonderful, brief ships continue; less vital o	Home                                              	decor                                             	9.80	3685.64	1.21492292594937898
+Perhaps spanish	Home                                              	decor                                             	7.44	2152.90	0.70967527139829663
+Regional circumstances see really matters. Again sexual years secure adjacent trials. Old animals will solve new, necessary eyes. Level views migh	Home                                              	decor                                             	7.80	157.04	0.05176617800194552
+Old fruits tak	Home                                              	decor                                             	2.26	7882.54	2.59837601087274335
+Parliamentary, favorite months escape almost necessary, environmental beliefs; closely high doctors used to run far exact contributions. Kinds accept never european trades. Sorry, great tho	Home                                              	decor                                             	2.64	8778.45	2.89370100153577829
+Much red years would not repeat by the others. Particularly environ	Home                                              	decor                                             	1.45	2736.60	0.90208432705122327
+Sol	Home                                              	decor                                             	1.01	9042.00	2.98057680523173309
+Schemes wield usually other 	Home                                              	decor                                             	1.43	5016.00	1.65345866567599792
+Helpful, very colleagues shall provide members. Concessions go other, tired eyes. Accurate windows ride slowly different hours. Speciali	Home                                              	decor                                             	1.48	2381.42	0.78500389465991526
+Frequently small crimes spend as primary regions; exactly small students simplify very. Early workers make interpretations. Late direct pensioners ca	Home                                              	decor                                             	2.82	6192.37	2.04123361993063780
+Maps form houses. Whole assumptions used to know for a premises; black titles 	Home                                              	decor                                             	5.19	6005.87	1.97975633899990144
+Contacts choose to the governments. Over angry contracts could sell as yet national technical tables; violent, toxic patterns cannot express solid crops. Feet shall use	Home                                              	decor                                             	9.88	1269.31	0.41841140728253607
+Formerly prime men influence incentives; new bars support children. Machines end certainly so economic drawings; other, christian eff	Home                                              	decor                                             	2.26	5503.23	1.81406765006142784
+As 	Home                                              	decor                                             	2.03	7855.62	2.58950218565743277
+Long-term st	Home                                              	decor                                             	8.22	2874.12	0.94741599286138340
+Contemporary feet used to go still political, late lives. Statutory, scottish genes must smell. Good lips establish quite. Old women must avoid with the places. Too wet l	Home                                              	decor                                             	4.58	710.24	0.23412130835520749
+Citizens can keep for the most part at the things. Branches visit terms. Available, slight problems may avoid. Problems help more. Social years feel inherent acres. Individuals use	Home                                              	decor                                             	49.10	5668.87	1.86866870536098372
+Workers shall not control never on a studies. Sophisticated activities go separately according to a bodies; co	Home                                              	decor                                             	40.34	2145.78	0.70732825670539131
+Prematurely other systems assume nearly under w	Home                                              	decor                                             	0.88	9056.13	2.98523457455908593
+Always cool temperatures meet there social grounds. Threats h	Home                                              	decor                                             	5.44	3350.86	1.10456708621751882
+Too complete events try environmental, national topi	Home                                              	decor                                             	3.31	7994.82	2.63538764145131214
+Fresh, beautiful functions give empty, fast origins. Sons get other companies. Lights say delightful, native services. Small, soviet things could go already also dead systems. Medical, comm	Home                                              	decor                                             	34.78	11689.03	3.85313555559144935
+Resulting, distinct clients shall tell intellectually difficult gardens. Villages turn then by a things; fresh, supreme powers succeed here. Historical hands st	Home                                              	decor                                             	4.30	269.93	0.08897888708650760
+Possible shoes render undoubt	Home                                              	decor                                             	8.28	13638.47	4.49574290431860593
+However old figures ask only good, large sources. Yet naked researchers shall deal to a women. Right, common miles describe there also prime bags. Readily significant shares	Home                                              	decor                                             	7.78	NULL	NULL
+Relatively regional months wish then needs. Eyes follo	Home                                              	decor                                             	66.29	7883.31	2.59862983128194800
+Deposits shall leave more skills. Close ce	Home                                              	decor                                             	5.30	5555.19	1.83119558312931557
+Regular findings put. Little, national cattle should say most mothers. Asleep eyes stay over thoughts. Western, golden walls might not move distinct, small boxes. Swiss, go	Home                                              	decor                                             	3.83	3030.40	0.99893164682307498
+Gentlemen work always. Religious, spiritual variations think fairly so electronic resources. Diplomatic, civil others split both mathematical, new contacts. Ultimate	Home                                              	decor                                             	9.53	6205.11	2.04543319397384199
+There final techniques wear so old winners. Old, particular prices will return especially motives. Around early members shall pay systems. Unions call rather. Else old ter	Home                                              	decor                                             	2.10	13195.83	4.34983242908439067
+Similar, ready forces play often arms. Marrie	Home                                              	decor                                             	7.68	7302.41	2.40714375893522009
+Nearly delighted services know then eventually political p	Home                                              	decor                                             	0.48	4915.69	1.62039278873142867
+Top modules ought to go. Funds shall offer in	Home                                              	decor                                             	4.71	13454.30	4.43503367735338493
+Important rights justify now still e	Home                                              	decor                                             	53.89	3370.57	1.11106422941936768
+Fields divorce hardl	Home                                              	decor                                             	1.25	14250.34	4.69743783130568185
+Able, assistant positions should die	Home                                              	decor                                             	4.24	3308.46	1.09059048186650958
+British, electric ye	Home                                              	decor                                             	4.13	6855.95	2.25997407076183372
+Immediate designs reward more speedily expected things. Good, happy feet create interesting, political signals. Still general stations help. Remote, flat ideas ma	Home                                              	decor                                             	0.10	6799.02	2.24120784232544325
+Sa	Home                                              	decor                                             	2.03	474.81	0.15651489414864844
+Minutes must not reduce in addition conditions. Australian, likely methods miss on a grou	Home                                              	decor                                             	25.40	111.84	0.03686659034473756
+Quickl	Home                                              	decor                                             	9.23	2919.06	0.96222987492587290
+Abroad great methods can call all labour main clubs. Minerals may make often countries. Apparently good pairs used to write terrible accounts; able funds close again with the times; earlier average 	Home                                              	decor                                             	4.93	5327.91	1.75627570961758494
+Minor, usual members come more before good waters. Circumstances cannot take interests	Home                                              	decor                                             	0.15	15519.10	5.11566793829592889
+Present, responsible rates contribute at all records. Eyes ought to wait political, national awards. Politically int	Home                                              	decor                                             	0.18	20899.05	6.88909795193300723
+Nations realize on a shadows. Managerial, disabled systems stay between the councils. Capitalist girls might live 	Home                                              	decor                                             	4.02	1089.18	0.35903391337340180
+Military issues face rather once previous thanks. Then famous sources ought to transport boats; readily impossible requirements trust again with 	Home                                              	decor                                             	5.27	7325.56	2.41477485305611310
+Private, direct rates increase furious meals. Italian values buy for instance random members. Available reforms work financial, impossible adults. Immediate, good experimen	Home                                              	decor                                             	6.40	7796.60	2.57004701611034397
+So far conditions may r	Home                                              	decor                                             	8.95	1175.16	0.38737609361160401
+Suspiciou	Home                                              	flatware                                          	8.91	11913.78	5.12961692885790892
+Material, rough relations think cities. As popular studies should not ask at a boo	Home                                              	flatware                                          	0.28	1925.64	0.82910676064909237
+Real times could cultivate honours. Great carers enter like a drugs. Sufficient years read o	Home                                              	flatware                                          	3.21	32.10	0.01382102938079593
+Long, other grounds give now clinical, essential areas. Possible languages make. So similar costs would say. More similar propos	Home                                              	flatware                                          	3.20	180.81	0.07784985427855798
+Present variables shall raise royal, american structures. 	Home                                              	flatware                                          	1.03	26390.07	11.36255242464987910
+Remarkable m	Home                                              	flatware                                          	20.08	15671.25	6.74743946055445923
+Changes like old, perfect streets. Thousands say. Whole factors work particular	Home                                              	flatware                                          	1.83	3396.31	1.46232088150439278
+Police succeed schools; supplies calculate far countries; new words move shares; officers must complete years. Asian things may bear warm things. Aw	Home                                              	flatware                                          	6.66	2788.28	1.20052647357899259
+Suppo	Home                                              	flatware                                          	2.16	18092.16	7.78979049601435527
+Streets will marry. Agencies tell regularly students. Years study here colonial, economic transactions. Cards shall not hide of course inside technical sons; else environmental 	Home                                              	flatware                                          	58.71	3036.50	1.30740048955722201
+Early, particular conditions fulfil just women. All new sales might not feel large, active books; current children should take. Generally di	Home                                              	flatware                                          	14.12	22.62	0.00973930481600012
+Foreign parties could not keep ston	Home                                              	flatware                                          	1.70	4789.08	2.06199424881564327
+Patient 	Home                                              	flatware                                          	1.87	9772.43	4.20763371189319384
+Years know more medical citizens. Then comprehensive observers come finally by a processes. Small voters must waste others. Statistical levels study. Ex	Home                                              	flatware                                          	0.33	741.75	0.31936911349549462
+Arrangements keep simply close large terms. Projects might not live true, easy others. So new years take labour members. Original towns travel away away americ	Home                                              	flatware                                          	9.19	2252.25	0.96973250538621876
+Possible, thick acids shall not go in a c	Home                                              	flatware                                          	3.98	5764.14	2.48181770389473594
+Random influences can force low for a subjects; young days will not travel historic hills. Unlikely, huge guards arrest now by th	Home                                              	flatware                                          	3.46	5434.00	2.33967207648738495
+Domestic, new tasks show here very various farms. Internal, old homes used to impose long traditional, high 	Home                                              	flatware                                          	1.93	627.94	0.27036689063479730
+More special scots ought to see just on a pupils. Grounds might shut complex writers. Empty, actual eyes may get little wrong, odd words; social, full tact	Home                                              	flatware                                          	3.31	2123.58	0.91433213621403771
+Legal ci	Home                                              	flatware                                          	4.71	5052.16	2.17526641110535642
+Hom	Home                                              	flatware                                          	8.19	3362.38	1.44771192428039261
+Leaves cannot lose colours; european, dynamic sentences will 	Home                                              	flatware                                          	96.77	1428.58	0.61509178046160258
+Further o	Home                                              	flatware                                          	5.51	11480.35	4.94299858728412768
+Thus internal planes would not apply never rather than a	Home                                              	flatware                                          	2.06	4826.77	2.07822211789234727
+European seconds wou	Home                                              	flatware                                          	5.97	12128.66	5.22213601899328053
+Labour, likely area	Home                                              	flatware                                          	84.74	7106.28	3.05969173421066874
+Particular, healthy talks get written, apparent months; then great attacks used to secure characteristically to a agencies. Accounts answer prod	Home                                              	flatware                                          	3.87	179.28	0.07719109493423967
+Yesterday angry obligations defi	Home                                              	flatware                                          	3.77	1418.04	0.61055366053407644
+European, rigid voters believe in common including a meetings. Complete trends broadcast directly;	Home                                              	flatware                                          	2.19	10595.74	4.56211943461914690
+Likely, odd offences shall ease enough true, chinese eyes. Other indi	Home                                              	flatware                                          	4.09	3818.90	1.64427193465176194
+Left, white ways might intervene es	Home                                              	flatware                                          	9.19	416.05	0.17913517987165560
+Later substantial changes give wisely. Minor taxes would shed forward reasons; yet long shareholders will live close small constitutional bags; supplies rea	Home                                              	flatware                                          	3.08	1033.24	0.44487353262970659
+Rather inc researchers might not answer sure. Most actual lives 	Home                                              	flatware                                          	4.89	317.32	0.13662582688829168
+Forces used to adapt in a musicians. Rather political t	Home                                              	flatware                                          	89.07	4073.22	1.75377237677400555
+Other, white years get meanwhile plans; more royal sciences would not contain triumphantly splendid specific concepts; free months 	Home                                              	flatware                                          	1.62	21553.63	9.28016677547677492
+Financial, black securities may support vague, late offices. So marginal incomes make on the men. Hotly close occupation	Home                                              	flatware                                          	6.87	280.44	0.12074671275857973
+Actively fierce lines should not feel quite confident new rules. Arms pay long settings. Wide, black women should pick real talks. Important friends make today between the revenues. Noisily expe	Home                                              	flatware                                          	4.53	8713.76	3.75181099617458879
+Brief regions ought to inclu	Home                                              	flatware                                          	4.98	5812.86	2.50279466811381312
+Forward general regulations can begin forward women; galleries consist typic	Home                                              	flatware                                          	8.74	2672.21	1.15055118136002115
+Uncertain, statistical jobs walk there; agreements show to a rights. Useless years may not resist locally only marginal experts. Concerned,	Home                                              	flatware                                          	0.14	7564.70	3.25706981174164905
+Beneficial, moving years ought to see difficult, political stocks; attitudes can say british questions. Upper, educational chapters should end then back lives. Workers talk there in a boundaries; pro	Home                                              	flatware                                          	2.02	609.71	0.26251775151916148
+Busy, new things go satisfactory services. Now old years must take. Scottish procedure	Home                                              	flatware                                          	0.85	2855.80	1.22959799706158888
+Mislea	Home                                              	furniture                                         	1.06	2910.97	1.06321050660037366
+Papers check other, industrial boards. Violent, social things give cars. Local councillors give ther	Home                                              	furniture                                         	3.38	3631.97	1.32655048442868154
+Dutch, busy firms must not return thereof full, naval plants. Parts shall get ashore early politicians. Good organisms try rather also close boys. Positive, big ingredients foster greatly local grou	Home                                              	furniture                                         	1.71	1113.86	0.40682922011628158
+Arrangements will trade however in every negotia	Home                                              	furniture                                         	3.24	15049.37	5.49667234692094570
+Black, perfect visitors should test more low english interests; about major wives believe examples. Other, available gro	Home                                              	furniture                                         	0.66	10969.33	4.00646757141663321
+Marine, new services shall reach more more significant elements. Late, solid rights would like also. Notes complete elements; continually personal armies will compare clearly curre	Home                                              	furniture                                         	3.59	965.34	0.35258337613977633
+Ways become worldwide specially common gene	Home                                              	furniture                                         	8.57	791.04	0.28892157567448637
+Very likely areas should k	Home                                              	furniture                                         	2.37	3579.84	1.30751038311912580
+Arms fail other faces; leaders could arise good characteristics; gol	Home                                              	furniture                                         	8.75	2288.09	0.83570814128872814
+Stones tell. Still brown relationships put initially long r	Home                                              	furniture                                         	9.54	5599.90	2.04532252682488396
+Private, young standards find even so in the women. Sheer, expert classes cannot present men. Small, sure enquiries must support mildly p	Home                                              	furniture                                         	4.99	2942.39	1.07468643184775984
+Authorities used to consider; general weapons seek particularly economic papers; much american walls	Home                                              	furniture                                         	1.27	2216.17	0.80943988718968251
+Severe, likely areas make on board formal, new conditions. Democratic, individual numbers should not fight workers. Poor options think. Independent feelings squeeze only ideas. Thin prob	Home                                              	furniture                                         	8.47	3094.07	1.13008644271738222
+Adults might not surrender doubtful, upper industries; earnings insist m	Home                                              	furniture                                         	1.61	6969.96	2.54572692352870019
+Shareholders mean; more very teams believe necessary, charming words. Courses would not suggest as popular, similar assets. Subjects must make on the things. Liabilities used to get very to a lines; 	Home                                              	furniture                                         	8.45	3751.07	1.37005088853319121
+Directly high lines move calmly also international files. Pounds cannot ensure creditors. Similar, favorable colleagues could gather written police. Free days might provide so. Probably other rock	Home                                              	furniture                                         	6.83	5386.33	1.96731764601379975
+Streets know half. National, 	Home                                              	furniture                                         	0.39	9772.83	3.56945469558921243
+Soviet, evident ways change able, huge woods. Smart sales ask sales. Thus possible transactions can want below effective, available families. Also external 	Home                                              	furniture                                         	4.84	145.90	0.05328890813474358
+Usual tools happen little young children. Dramatic,	Home                                              	furniture                                         	1.68	11143.74	4.07016954857756966
+Judicial operations cannot kick currently h	Home                                              	furniture                                         	6.22	9022.42	3.29537293031578591
+Too young things leave individually skills. Contexts suffer enormously so romantic	Home                                              	furniture                                         	29.66	20545.03	7.50392197598047208
+Superb lights occur with a standards. Bright services specify at the sides. Then urgent versions get earlier prisoners. Available heroes would not believe civil sides. Banks could t	Home                                              	furniture                                         	0.12	16046.32	5.86080104441877032
+Royal, military notions will not find very very wet acids. Funny actions take western, remaining homes. Great patients will replace simply. Signs can think equivalent reasons. Campaigns 	Home                                              	furniture                                         	7.54	1334.66	0.48747480555940278
+Yet huge priests think today unlikely, absolute things. Whole, modern changes might not manipulate most only, desirable companies; accused, particular girls may take serious, central hours	Home                                              	furniture                                         	0.52	10920.86	3.98876425834404225
+Local blocks shall not get natural things; already post-war patients may exploit british, sexual grounds. Easy centuries would not	Home                                              	furniture                                         	3.75	2996.52	1.09445701853270617
+Ago new arguments accept previously european parents; fo	Home                                              	furniture                                         	3.03	6882.58	2.51381201747788529
+Walls s	Home                                              	furniture                                         	4.80	1253.04	0.45766369738971278
+Late general supporters see more informal, blank employees; very similar methods shall help complex, likely schemes. More than new groups reconsider unanimously. Physical incenti	Home                                              	furniture                                         	37.53	2259.23	0.82516723732184192
+Mountains ought to join pressures. Bright countries used to pay there owners. Imperial issues might establish thus calmly senior members. Just regular 	Home                                              	furniture                                         	7.01	10713.70	3.91310058316108488
+Contacts open considerable, suprem	Home                                              	furniture                                         	7.01	1997.51	0.72957592109822925
+Effects must quit about small values; full paths must get. Problem	Home                                              	furniture                                         	1.87	4806.19	1.75542575317425115
+Political girls used to ask hands. Large-scale, chief areas can produce including the children. Sufficiently new areas will	Home                                              	furniture                                         	2.26	3164.50	1.15581048521176187
+Now late makers used to	Home                                              	furniture                                         	0.85	7607.78	2.77868601459451341
+Greatly commercial animals used to live now as wide personnel. Enough hot wars keep. Min	Home                                              	furniture                                         	4.37	894.54	0.32672419385094943
+Better high children	Home                                              	furniture                                         	4.48	4768.72	1.74174010966630844
+Thus light firms expect anyway in a toys. Laws used to ab	Home                                              	furniture                                         	2.06	12227.85	4.46613279873491621
+Widespread others hold quickly new teachers. Societies wou	Home                                              	furniture                                         	3.01	1696.19	0.61952099444188288
+Hot, small levels ought to arrive only there other features. Often irish columns used to spend now new natural months. Once british flowers shall penetrate funds. 	Home                                              	furniture                                         	5.70	20519.61	7.49463750685925767
+Electronic organizations transfer still natural, whole posts. Plants ought to curl just animals; already huge women can dream eventua	Home                                              	furniture                                         	3.59	6214.52	2.26980798753616633
+Increasingly other policies happen previously under a targets. Efficient, experienced points will see mostly even english machines. Fine states must remedy also good thoughts; normally clear years i	Home                                              	furniture                                         	5.85	9156.23	3.34424605435629337
+Natural costs assist during the types. Sometimes possible concerns make as real, right forms. 	Home                                              	furniture                                         	6.28	1707.15	0.62352405429902331
+Therefore early eyes stay recent, expert studies; varieties halt in a parts. Unable i	Home                                              	furniture                                         	7.52	742.08	0.27103929368492471
+Funds drink much common months. Royal, long trees will expect sometimes front coins. Old ears can allow very similar, short members. Even public rules act common, open 	Home                                              	furniture                                         	17.29	6237.51	2.27820491692628117
+Intensive minutes might see like a boys. Questions might know more young communications. Ready, southern others may result. Lonely, trying seeds love probably good farms. 	Home                                              	furniture                                         	9.12	11445.81	4.18049840724968750
+At least competitive notions may not convince white, familiar principles. Valuable, fat books convince further cases. Yet ordinary cities cannot need so as. Ri	Home                                              	furniture                                         	8.51	1332.65	0.48674066775713524
+Women should not knock doubtless details. Sure northern products must go very cruel, other tickets. Poor, physical objectives highlight only by the discussions; now slow crowds must 	Home                                              	furniture                                         	0.77	87.87	0.03209387496778559
+Little, evil parties would not act subject	Home                                              	furniture                                         	7.63	1108.98	0.40504683580032854
+Easy, philosophical levels must 	Home                                              	furniture                                         	2.32	3778.34	1.38001105662664191
+Now additional reasons hate. Original, use	Home                                              	glassware                                         	4.41	6349.14	1.56441659290736902
+Jobs notify about future boxes. Now main policies will think above offers. Criminal men used to think soon national women. Sure talks ought to appreciate there companies. So appropri	Home                                              	glassware                                         	1.19	7756.30	1.91113826747676477
+Seats will cope similarly new shares; massive deals explore semantic, important thi	Home                                              	glassware                                         	1.53	4412.81	1.08730838906490754
+Powerful hours take worth a authorities. Respondents must generate aside c	Home                                              	glassware                                         	31.97	10526.17	2.59362921714811148
+Unfair, possible hands will not arrive surely tight russian employers. Really necessary walls should decide varieties. Talks would raise probably moral meetings. Bright, necessary 	Home                                              	glassware                                         	1.54	3919.44	0.96574291493097623
+Old	Home                                              	glassware                                         	1.47	1351.66	0.33304657512185499
+Conditions criticise enough more particular shops. Be	Home                                              	glassware                                         	6.38	1038.40	0.25585987867254652
+Countries ensure in a christians. Expected ends used to run high holes. Broad, unlike women specify therefore. Lit	Home                                              	glassware                                         	2.94	153.37	0.03779009013097887
+Onc	Home                                              	glassware                                         	4.53	1345.23	0.33146223477144621
+Western, complete meetings follow also educational shareho	Home                                              	glassware                                         	7.67	2508.40	0.61806521539119384
+Similar, low sites remember peaceful days. Faster permanent views give then churches. Others make well public processes. Eventually other schemes can trus	Home                                              	glassware                                         	0.29	105.75	0.02605660840680065
+Statistical bedrooms analyse there good, little residents. 	Home                                              	glassware                                         	8.08	5239.63	1.29103533906879324
+Less than outside students go more. Military views should not let more different, big steps. Average, black animals ought to begin automatically with a notes. Needs	Home                                              	glassware                                         	3.76	13328.83	3.28419956341197821
+Wide, great premises mean ever severe courses. Used ministers face there about a things.	Home                                              	glassware                                         	0.83	1275.20	0.31420696964872045
+Faintly actual prices may not wait dramatic terms. Others shall see shortly priests. Very na	Home                                              	glassware                                         	27.85	6812.75	1.67864925695915955
+Agents invest often things. French cars ought to get locally distinctive, local powers. More american entries compensate only	Home                                              	glassware                                         	6.43	10473.16	2.58056764918929822
+Again other wheels ought to find on a employees. Developments make really together new groups. Drinks would not assess bright women; special, australian t	Home                                              	glassware                                         	3.25	516.63	0.12729669599248624
+Words visit authorities. American occasions must need available, pure c	Home                                              	glassware                                         	5.43	5888.06	1.45080731627183575
+Purposes look events. Words convert over complete sites. New notes tell up a 	Home                                              	glassware                                         	9.93	9702.28	2.39062421383578063
+Free kids would become only 	Home                                              	glassware                                         	1.05	8484.78	2.09063441964873770
+Interested, square savings change off	Home                                              	glassware                                         	2.10	8572.37	2.11221643695702771
+Exactly single cities used to deserve ago false services. Suddenly psychological managers could sustain far together big changes. Parents should r	Home                                              	glassware                                         	0.64	2997.09	0.73847754600414333
+Heavy, desperate standards could produce still fine, important weeks. Accordingly 	Home                                              	glassware                                         	9.90	11317.37	2.78857946368674669
+Long, surprised sections keep positive sports. Strategies go northern, precious forms; readers emerge about reports. Large, unusual legs might show affairs; as usual ac	Home                                              	glassware                                         	4.43	12838.25	3.16332154022324760
+Red rooms could not apply	Home                                              	glassware                                         	4.96	1551.75	0.38234838860759250
+Present materials would say real, rare relationships. Particular conclusions contribute well to a hand	Home                                              	glassware                                         	4.07	8454.05	2.08306260332400026
+Separate moments come months. Avail	Home                                              	glassware                                         	0.58	5564.41	1.37106054264667234
+Professional, local chemicals can feel eyes. Familiar shops bear early in a accounts. Western arrangements ride reserves. Sorry, scottish ministers might not keep constantly w	Home                                              	glassware                                         	6.13	5921.40	1.45902223186788996
+Rows come 	Home                                              	glassware                                         	0.29	840.56	0.20711246111035795
+White, local attitudes ca	Home                                              	glassware                                         	1.74	1012.36	0.24944366985067333
+Seconds may make ahead quite little lips. Young, criminal consumers s	Home                                              	glassware                                         	7.17	1471.96	0.36268827716760552
+Recently nice particles hear above in a candidates. Human errors register. American, old days live. 	Home                                              	glassware                                         	8.16	528.66	0.13026086619706129
+Traditional, old-fashioned men show too final,	Home                                              	glassware                                         	4.84	6698.16	1.65041448856828214
+Years must share new, white loans. Able 	Home                                              	glassware                                         	1.64	1410.40	0.34752000469930625
+Single, roman facts may hear by a rights; different, able preferences must produce as internal surveys. Similar heads might stabilize direct na	Home                                              	glassware                                         	6.70	8825.39	2.17456010654651897
+Stones should send perhaps at the groups. Perhaps individual facts 	Home                                              	glassware                                         	4.18	26041.20	6.41650449969907389
+More black members would run more central poor phases. Personal responsibiliti	Home                                              	glassware                                         	8.30	423.06	0.10424121751849724
+Safe, distinct millions must not deliver at the men. Indeed old claims might put exercises; particular, wooden households should learn clear, lucky votes. Mean, level terms might write bot	Home                                              	glassware                                         	9.86	7952.69	1.95952840766599957
+Significant difficulties could observe numbers. Very alone centuries affect forwards by a matters. Glad fields ought to spread hardly british str	Home                                              	glassware                                         	3.06	501.96	0.12368203457094708
+Novel, small attitudes may warn now however good terms. Aware earnings must eat much; lat	Home                                              	glassware                                         	2.84	5534.76	1.36375483636523840
+Cold, old days stem thereby difficult, nuclear men; likely contents shall threaten often outer years. All real or	Home                                              	glassware                                         	9.08	11902.21	2.93268298009935465
+Now strong fields may not feel. Again	Home                                              	glassware                                         	3.96	9805.52	2.41606236279008890
+Even sexual men can clear thereby always male members. Shoulders extract. Negotiations used to alter else 	Home                                              	glassware                                         	3.47	1371.15	0.33784887581073012
+Conditions could not estimate following problems. Theories get sure; extremely complete scholars ought to thrive only strong, european businesses. Important, social p	Home                                              	glassware                                         	1.56	6751.07	1.66345141670827100
+Holes buy then markets. Practical themes ought to escape above australian children.	Home                                              	glassware                                         	1.43	3401.20	0.83804951785541719
+Willing, due values will chat hardly gmt central records. Necessary, adult stairs make fast in terms of a years. Views would not dig	Home                                              	glassware                                         	0.24	2373.76	0.58489016332602467
+Moments used to contract really boats. A	Home                                              	glassware                                         	68.94	1997.56	0.49219516490864023
+Insects indicate germans. Other, particular properties might 	Home                                              	glassware                                         	4.52	2374.24	0.58500843445638178
+Persons might live here doctors. Chil	Home                                              	glassware                                         	2.86	15578.10	3.83841561628351009
+Materials make apart colonies. Rates make naturally poor, appropriate companies; c	Home                                              	glassware                                         	0.80	1956.16	0.48199427991533955
+Used groups ought to fail high from the districts. Immediate, main walls could exploit rights. Therefore late friends ought to try away. In short widespread lakes sh	Home                                              	glassware                                         	80.17	9287.91	2.28852419657312357
+Too only affairs put nonetheless big numbers. Rapid students appeal for the 	Home                                              	glassware                                         	9.29	13621.22	3.35624392967263487
+Good windows say widely actions. Simple, imaginative findings see to a recommendations. Environmental, l	Home                                              	glassware                                         	4.66	12892.65	3.17672560166371999
+Japanese emotions speak disabled, new techniques. Experts should not tell only refugees. Years cannot afford well head quarters. Offices make conscious, primary stories	Home                                              	glassware                                         	7.31	4129.01	1.01738058324126665
+Full goods should find then. Only able years exploit completely mode	Home                                              	glassware                                         	2.13	3040.36	0.74913919560946025
+Sexual, due tensions take quite lucky circumstances. For ever formal districts respond ways. Poor relations should not come correctly in an facilities; important times could look away common 	Home                                              	glassware                                         	42.90	1247.40	0.30735710001553787
+Bad boys might claim shortly italian, good lines. Times learn additional, sick cards; measures work sometimes pleasant male	Home                                              	glassware                                         	2.10	3225.77	0.79482388369177617
+Children want on a paintings. Over nice teachers must not sell. Richly accurate pp. hate as african, fiscal days. Claims eat part	Home                                              	glassware                                         	7.95	6793.78	1.67397508332817129
+Always sad weeks would not put close to a masses. Fresh, atomic sides will not help together previous 	Home                                              	glassware                                         	0.83	6893.14	1.69845720731209292
+As other 	Home                                              	glassware                                         	4.88	2352.12	0.57955810653242499
+Serious branches use. Rich, english bombs keep much vulnerable consequences. Little, furious sales can keep to a gentlemen. As gold customers overlap betwee	Home                                              	glassware                                         	2.54	3062.18	0.75451560407694385
+Really different shares ought to help clearly p	Home                                              	glassware                                         	2.82	6640.72	1.63626137663554805
+There possible newspapers experiment. Annual accounts might visit possible, prime groups; competitive, universal pr	Home                                              	glassware                                         	1.12	63.36	0.01561178920713843
+Recent, labour complaints must read in a units. Softly old courts rely even. Actual	Home                                              	glassware                                         	8.70	2861.55	0.70508073556955459
+Well new carers shall give together with a samples. Individual, central birds find there weapons. Kind details proceed ultimate miles. Unlike, independent months mus	Home                                              	glassware                                         	0.46	6486.44	1.59824706415326716
+Overseas businesses conceal gmt in a farmers. Level functions could support all right dreadful processes. Walls buy furth	Home                                              	glassware                                         	3.81	10274.91	2.53171920836992962
+Mental techniques might prohibit by a chiefs; other, waiting defendants vary else. Now old skills would see. Common jobs will no	Home                                              	glassware                                         	89.76	2200.15	0.54211297386498769
+Dogs will cover never. Bitter children restore cheaply upper, short views; other teams shall exist too high customs. Yards must not help now present, coming mines. However federal method	Home                                              	glassware                                         	3.22	2352.77	0.57971826535478358
+More than divine areas will control together from 	Home                                              	glassware                                         	4.90	563.56	0.13886016296677611
+Surely national arguments address working, soviet effects. Again central parents say english rules; carefully military chang	Home                                              	glassware                                         	8.61	13637.98	3.36037356330760394
+Classical, attractive employers want only prices. Financial approaches used to hear considerable votes. Bo	Home                                              	glassware                                         	2.50	13555.23	3.33998411323041478
+Other patients see normal colleagues	Home                                              	glassware                                         	4.62	1970.54	0.48553748586228795
+Newspapers ought to pursue. Well rare criticisms used to tell so. Powerful, new matters touch. Home magic brothers can read now rather supreme rats. As evolu	Home                                              	glassware                                         	4.99	1537.58	0.37885692628017534
+Surely additional years work never remote, great bits; women deal in a judges. Far ethnic hands might help afterwards already dead awards. Rich, social experts target social children. National	Home                                              	kids                                              	0.50	361.08	0.11815869948988022
+Yet black costs must not judge here lively variables. Full, po	Home                                              	kids                                              	1.68	3938.44	1.28880289248621866
+Proud investors may not visit extremely. Alone, everyday houses move widely global countries. Only single gardens come further shadows. Scottish, wo	Home                                              	kids                                              	2.68	31.68	0.01036686496022877
+Total, new savings would make short, popular consultants. Short, other contracts might discuss for a	Home                                              	kids                                              	9.91	1600.56	0.52376229105883094
+Effective, free arrangements will build social, possible agreemen	Home                                              	kids                                              	4.30	2319.90	0.75915688198341950
+Enterprises shall not influence perhaps delighted, big police. Novels keep early temporary bacteria; rates will not cope men	Home                                              	kids                                              	3.57	6583.08	2.15422668504996302
+Agricultural sites will not provide skills. Again	Home                                              	kids                                              	0.55	5015.40	1.64122394323015739
+Conservatives tell effectively in a parties. Dir	Home                                              	kids                                              	6.35	8063.47	2.63866491795631001
+Too old	Home                                              	kids                                              	0.95	114.66	0.03752098283900982
+Following occasions see then only real lovers	Home                                              	kids                                              	5.63	6310.36	2.06498263795546836
+Permanent details would help also off a owners. External children used to listen like a years	Home                                              	kids                                              	30.73	6001.32	1.96385334668939829
+Farmers might not assume now to the tanks. For	Home                                              	kids                                              	3.80	11826.88	3.87019153601106270
+Local farmers skip also shoulders; things ought to seem so only applications. Foreign, voluntary voices may not find new	Home                                              	kids                                              	3.96	2251.62	0.73681314651989612
+Now close items become already against a groups. Authorities would work as well natural, dependent parties. Operators should not fall l	Home                                              	kids                                              	5.59	7257.25	2.37483998524685165
+Appropriate items take mediterranean centuries. High, very days see ways. Careful, technical minds remai	Home                                              	kids                                              	4.98	10259.21	3.35719206656024705
+Dire	Home                                              	kids                                              	4.41	1733.90	0.56739605917110697
+Short areas would not improve below to the measurements. Vo	Home                                              	kids                                              	0.36	18342.34	6.00229046195084044
+As beautiful children strike really natural services. Too assistant pow	Home                                              	kids                                              	3.30	2799.11	0.91597207635182954
+Even growing seats may build for a times. Obvious, different systems require american settlements. Evil yards support worldwide possible members. Courses could build also users. Alm	Home                                              	kids                                              	4.28	2619.47	0.85718723981598684
+Gold, young schools shall not go far human hands. Aware terms brush almost. Real years treat early. Edges cannot stop still british assessments. Very royal skills shall say already other	Home                                              	kids                                              	5.63	4448.98	1.45587041890020849
+Dogs hang perhaps chief, visual brothers. Minimum, small families shall work strong mountains. Small, defensive factors make by	Home                                              	kids                                              	5.44	2978.61	0.97471109972181264
+So dependent things distinguish again new subjects. Critical, firm centuries increase then institutions. Effects allo	Home                                              	kids                                              	1.59	10537.48	3.44825227844417572
+Turkish, old women must improve far from full, new changes. Days keep again exactly secondary visitors. Things used to make great, other notes. General, hig	Home                                              	kids                                              	1.38	355.77	0.11642107155620551
+Examinations reduce other, late things. Police should help very strong boxes. Annual, sole reports might benefit fortunate, total seats. Never rural shapes shall cease pictures. Physical periods wi	Home                                              	kids                                              	3.60	1189.98	0.38940536506859327
+Likely products ought to work other, considerable arrangements. Also other funds kill possible, royal patterns. Old, good files know palestinian colours. Northern	Home                                              	kids                                              	1.60	3252.96	1.06448854296167261
+Minds could not decide later avail	Home                                              	kids                                              	2.36	7178.10	2.34893918469122957
+Teams make never features. Now russian individuals may reproduce indeed other visual lakes. International legs drive also married views. Catholic populat	Home                                              	kids                                              	8.74	5328.40	1.74364909261625606
+Healthy, delighted conclusions may offer experienced condi	Home                                              	kids                                              	4.30	1952.10	0.63879915053227863
+Reasonable pictures could not try features. Unexpected politicians remember always. Serious buildings pay thereafter aged a offers. Large, material products go tomorrow interesting, individual re	Home                                              	kids                                              	44.54	107.20	0.03507979557249130
+Equal supplies could get easily still new years. Equivalent, national policemen shall appeal. Tables would 	Home                                              	kids                                              	7.14	13784.20	4.51069886315610630
+Hours get skills; foreign, positive events disguise currently apparent prices; other programmes may sink honours. For instance var	Home                                              	kids                                              	7.04	2430.74	0.79542781986826031
+Apparently effective deals could stand 	Home                                              	kids                                              	0.92	1924.93	0.62990812398652687
+Funny times go actually much old details. Military parameters tell so studies. Values u	Home                                              	kids                                              	4.41	1907.42	0.62417820588508729
+Levels contact in a sides. Companies must not count with an boxes; yet physical days happen never from a opera	Home                                              	kids                                              	8.77	13024.65	4.26214607652284354
+Questions seem strongly. Political years establish guilty centres. Necessary, pale eyes used to generate social, particular assets. Conditions help as firm directors. Persona	Home                                              	kids                                              	9.37	8639.50	2.82716318888562125
+Subsequent qualities say broadly good objectives. Odd workers ought to make commonly therefore intact times. Objectives will not hold just with the types. B	Home                                              	kids                                              	0.64	3035.53	0.99333742401272873
+Soon artificial notions think no longer lights; clearly late members could not trace good countries. Cultures can proceed away wealthy 	Home                                              	kids                                              	2.38	3035.43	0.99330470032282902
+Appropriate, new ad	Home                                              	kids                                              	3.99	396.54	0.12976251992831810
+Ruthlessly empty times shall not focus to a lectures. Skills involve even; boring periods re	Home                                              	kids                                              	0.63	1007.86	0.32980898102323771
+Lists could play round, new roads. Soon national calculations think usually at first similar benefits. Skilfully specific practitioners will believe that is bars. More immediate	Home                                              	kids                                              	8.24	3098.01	1.01378318546206881
+Suggestions must see much large assessments. Disabled charges might claim besides wide, white passengers. Democratic, wide relationships test little years. Working, bri	Home                                              	kids                                              	0.50	934.46	0.30578979263684908
+Strong settlements should close here. Forms may seem quickly other unions. Places employ difficult banks. Women must not accept too areas. Vast possibilities know; never healthy subjects cancel most j	Home                                              	kids                                              	1.95	10592.00	3.46609323417749873
+English requests serve also intervals. More late cards might make only other companies. Tragic lights learn more royal, attractive studies. Businessmen ought to defend close po	Home                                              	kids                                              	1.59	17495.72	5.72524515852189842
+Goals help still human plates. Practical groups t	Home                                              	kids                                              	4.79	16455.90	5.38497768620671273
+Full, good fans might not pose of course parts. Daily	Home                                              	kids                                              	85.83	7041.80	2.30433679535792207
+Due years show just ashamed homes. Large, australian parties suit there automatic grounds. Sexual steps might not mean today technical molecules. Al	Home                                              	kids                                              	6.52	4853.82	1.58834900509020269
+Then dark tactics should not follow then. Ashamed, g	Home                                              	kids                                              	1.43	11882.09	3.88825828520469372
+Vocational, political styles run incorrectly indeed only hands. Complete, confident employers expect big owners. Inc times should stop more; consi	Home                                              	kids                                              	8.09	3606.10	1.18004898147351569
+Formal matters must admire much. Capable rules rise however. Harder only studies would show more. Old stones oppose common, secure police. Opinions come grey, appropriate systems. Eye	Home                                              	kids                                              	6.02	261.24	0.08548736749400772
+Soft, good estates must not join most likely, accused pieces. Coming, historical pictures arrange; best old loans cannot	Home                                              	kids                                              	6.24	6536.61	2.13901998635356684
+About american members provide certainly increased, special experienc	Home                                              	kids                                              	0.99	5029.15	1.64572345059136780
+Trying, ti	Home                                              	kids                                              	3.34	16043.89	5.25015281145090918
+New, other companies could take always political years. Important charges wait sure other aspects. Legal grounds may not worry to	Home                                              	kids                                              	6.49	5131.46	1.67920305772776318
+Windows recommend never internal cells. Mutual, other moments should not see levels. Necessary, national costs shall not walk light, high types; more digital days might continue.	Home                                              	kids                                              	2.75	8373.49	2.74011490138339726
+Fresh, f	Home                                              	kids                                              	1.45	4190.94	1.37143020948299155
+Quickly wrong facilities prepare as. Similar surveys look hopelessl	Home                                              	kids                                              	3.16	116.22	0.03803147240144533
+Remote, left figures used to feed on a records. Over economic depths must understand in particular at the ranks; degrees can think go	Home                                              	lighting                                          	2.60	5654.38	2.08346575200781715
+Lovely letters would require now questions; communities will add years. Emotional, traditional times make for a patterns. Perhap	Home                                              	lighting                                          	8.69	2656.29	0.97876146321981272
+Moving, powerful drugs use so blind honours. Efficient, other seconds look just rare, planned homes. German, specified sons reside further red weeks. Available lists undergo young, milit	Home                                              	lighting                                          	0.67	10412.96	3.83685665573012774
+Different men may not inform by now between a eyes. Members can cause new minds. Strong, chief rooms will carry high lessons; natural molecules accept here because of a talks. Eyes may disc	Home                                              	lighting                                          	0.56	7704.59	2.83890530849746709
+Incidentally immediate flames earn. Friends influence certain, potential men. Early, opening conventions should see per a agencies. Economic, senior practitioner	Home                                              	lighting                                          	1.62	616.89	0.22730506045863602
+Original others get industrial yar	Home                                              	lighting                                          	1.48	6297.95	2.32060157486013180
+So soviet years get. Good things must appreciate well real churches. Overseas, constant boxes complete for no months. Subjects may not suffer widel	Home                                              	lighting                                          	5.50	178.36	0.06572019417303299
+Important, toxic commun	Home                                              	lighting                                          	0.33	431.67	0.15905716650971714
+Prisoners give fundamental months. Opportunities grasp capital actions. British iss	Home                                              	lighting                                          	5.72	5860.48	2.15940728609091930
+Under way long interpretations might take yesterday. Little little shares get quickly families. Measures occur. Forward daily hands m	Home                                              	lighting                                          	2.56	2458.11	0.90573820642898698
+New, future communities should make yesterday particular, primary relations. Significant students mea	Home                                              	lighting                                          	83.07	7959.15	2.93270286752800800
+Opportunities drop cars. Officials change as for a inches. Other, american societies take straight leading, total posts. Agreements get 	Home                                              	lighting                                          	65.24	13670.55	5.03717874216279499
+Vital problems may lig	Home                                              	lighting                                          	60.33	6799.66	2.50546633500003077
+Rather american gentlemen might generate rather in a studies. Enough current negotiations used to co-operate nearly rough main rivals. Dramatic, overall weeks used to provide too other, great meal	Home                                              	lighting                                          	7.69	3528.80	1.30025466022538018
+Also new colonies go unhappily eggs; typically modern centres would provide then	Home                                              	lighting                                          	0.51	5329.54	1.96377216670187391
+Prayers increase ever depths. International, official member	Home                                              	lighting                                          	7.88	4324.07	1.59328728424415089
+Sick, old-fashioned birds might think there imports. Grant	Home                                              	lighting                                          	7.01	5314.03	1.95805720700449927
+Common contracts will undergo for the goods. Generous, long laws shall not reach less traditional men. All pla	Home                                              	lighting                                          	3.29	973.56	0.35872702533694772
+Front shelves produce more at a principles; previously everyday birds avoid on a matters. Up	Home                                              	lighting                                          	18.01	4993.08	1.83979696748983826
+Problems should prevent finally in a effects. Now economic men sign. Royal, permanent teeth can start colonies. Geographical eyes wi	Home                                              	lighting                                          	9.41	5689.57	2.09643218861327258
+Essentially everyday lines sing s	Home                                              	lighting                                          	6.37	2165.33	0.79785774864708186
+Famous, attractive arms shall go publicly just democratic men. Importantly private ministers ought to write. Levels bring most true, adjacent days. Successful, particular constraints may pl	Home                                              	lighting                                          	3.16	2680.48	0.98767473691932868
+Just familiar police work virtually rare fruits; blind police might not succeed possible, stable churches. Senior communications light old, economic activities; almost direct characters ca	Home                                              	lighting                                          	2.42	14392.73	5.30327994101837339
+New kinds will go wholly great, occasional models; efforts may seem then too local homes. However religious co	Home                                              	lighting                                          	4.11	408.39	0.15047919992332890
+More possible newspapers	Home                                              	lighting                                          	9.78	3183.02	1.17284532662394854
+Of course high 	Home                                              	lighting                                          	4.02	405.11	0.14927062043864877
+Further high men can give stairs. Controversial, great fingers hate sometimes generally ancient books. Other dogs woul	Home                                              	lighting                                          	6.69	1549.44	0.57092115754353125
+Visual, sensible rates know instead excellent honours. Other, inc christians fill plans. Girls may not make to a institutions. Days could build appropriate, small statements. Left, runnin	Home                                              	lighting                                          	1.12	8531.28	3.14351523965302125
+Propos	Home                                              	lighting                                          	1.14	5525.76	2.03607322355673225
+Significantly severe hundreds let right. Domestic, good approaches like of course later main records. General firms will preve	Home                                              	lighting                                          	17.01	2134.46	0.78648309965559538
+More great societies press. Years make still other, lively standards. Decisions may strike to	Home                                              	lighting                                          	0.43	2644.48	0.97440984013625407
+Unusual, fierce imports can press fine rural contents. Perhaps public	Home                                              	lighting                                          	4.21	7474.73	2.75420894253753570
+Middle-class years record also recent problems; certain, mild others can show. Matters will influence solely books. Loca	Home                                              	lighting                                          	6.43	2611.80	0.96236826161206301
+Able, double cells monitor quickly tomorrow direct men. Different weeks used to become n	Home                                              	lighting                                          	7.19	187.35	0.06903273367525079
+Legal conventions ought to work in accordance with a cases. Together left books may not come sure subsequent things. Short, real products deal excessive, im	Home                                              	lighting                                          	5.79	9924.55	3.65689253801286467
+International, final writers must learn political eyes. Immediate times reach also also wrong requests. Isolated years may not plan yesterday minutes. Long, old researc	Home                                              	lighting                                          	0.62	4542.45	1.67375362200770182
+Alone new conditions will recognise personal, hot others. Sooner scottish eyes permit probably only advanced cases. Never impossible services use again direct	Home                                              	lighting                                          	4.82	8731.18	3.21717226373459388
+Usually severe kinds cost incidentally conclusions; normally continuing concentrations ought to tell amer	Home                                              	lighting                                          	0.90	8588.69	3.16466906532847440
+Empty, splendid pounds make relatively skills; public, simple exchanges might exploit simply. Basically quiet perceptions shall not sleep. Old, alone individuals get permanent, new minerals. Fo	Home                                              	lighting                                          	2.10	4427.11	1.63125436659215111
+White, fair artists take. Simply silent years could create general, alternative issues. Deliberately natural moves take so n	Home                                              	lighting                                          	5.13	1353.00	0.49853903743055412
+Regular villages will raise finally small, rich terms; working-class, smooth states may violate european interests; discussions should not tell particularly times. Delightful, previous obje	Home                                              	lighting                                          	2.57	1509.56	0.55622659966272526
+Happy sorts should care. Definite, sensitive pages should happen else smooth clouds. Local, legal years might not represent easy unfair clothes. Poor, other powers change only fo	Home                                              	lighting                                          	8.25	6600.48	2.43207460885411963
+Plates shall think; new, economic pupils collect entirely. Really powerful books develop yet girls. Best unlik	Home                                              	lighting                                          	3.44	2151.42	0.79273233991784386
+Writers say. Spanish, local targets find able weapons. Figures would win most into the effects; as steady workers shall understand. Social point	Home                                              	lighting                                          	5.26	5754.60	2.12039375077447653
+Fiscal, occasional subjects ought to provide ill altogether royal stocks. Individual students save within a students.	Home                                              	lighting                                          	2.33	6565.32	2.41911922632931676
+Villages	Home                                              	lighting                                          	3.15	5303.78	1.95428039611487386
+Rich, logical 	Home                                              	lighting                                          	7.93	2820.76	1.03936361805070942
+Residents a	Home                                              	lighting                                          	4.83	13929.25	5.13250176432338949
+Ridicu	Home                                              	lighting                                          	4.71	6980.98	2.57227719846411656
+Behind aware variables cannot bring into a contents. Different, electronic mistakes measure; however additional students should like. Interesting sales wo	Home                                              	lighting                                          	1.37	1624.72	0.59865953059436060
+Common feet cannot send at a engines. Orders should think prime, conservative cell	Home                                              	lighting                                          	2.52	2080.16	0.76647521367445784
+Emotional areas make then new targets. Difficulties should halt much. Military circumstances might mount very much white systems. Other holidays drag further through a	Home                                              	lighting                                          	6.63	10785.78	3.97422940069306875
+Yet young minutes could not walk here; enough pale others may not	Home                                              	lighting                                          	1.89	7242.84	2.66876458378678093
+Difficulties apply just initially high surroundings. Enough usual requirements assist repeatedly for a students. Directions make too through the flowers. More national historia	Home                                              	lighting                                          	9.68	372.50	0.13725483476931368
+Always top problems change almost expensive women. Supreme, industrial discussions 	Home                                              	lighting                                          	4.16	1004.00	0.36994323250574748
+Discussions emerge so annual lessons. Good, early faces play really legislative products. Cold, private societies understand clearly ahead fat manufacturers. Abstract causes become so as executi	Home                                              	lighting                                          	9.11	4351.81	1.60350862415422005
+Approximately senior colours cannot encomp	Home                                              	mattresses                                        	4.73	2262.11	0.80877478687478841
+Facilities shall look just much quiet clients. Specific prices should explain on a ways. Aspects ought to establish ill high chains. Suitable, enormous areas c	Home                                              	mattresses                                        	0.21	4913.00	1.75655053375646430
+Sufficient, united companies earn either for no months. Comfortable, big tears come spiritual, old bir	Home                                              	mattresses                                        	6.95	6514.82	2.32925107843014222
+Complex, social miles cannot tie faces; probably future universities get objectives. Given settlements cannot g	Home                                              	mattresses                                        	4.30	100.50	0.03593188044830545
+Even widespread figures help also new, coloured trees. American, potential chapters may get actually years. Genes alter sides. Fingers develop par	Home                                              	mattresses                                        	4.87	NULL	NULL
+Dark companies stem in a offices. However multiple hours will preserve most long copies. Over mil	Home                                              	mattresses                                        	4.19	265.00	0.09474575441592979
+Early children shall not burst environmental	Home                                              	mattresses                                        	29.32	1972.12	0.70509432905186207
+Strong t	Home                                              	mattresses                                        	3.26	972.30	0.34762753591927748
+Also unknown books want very structural eyes. Well existing years could not buy much constant simple clients. Clouds find; ordinary, magic years prevent surely. Pensioners	Home                                              	mattresses                                        	0.47	5228.57	1.86937663836414340
+Central, new children agree strangely legitimate, full values. Underneath adequate rights avoid just rough museums; dead, local shareholders spare various forces. Small letters force finally women.	Home                                              	mattresses                                        	2.58	4991.57	1.78464175611291563
+Terms connect too all personal doctors. Current, new hours used to revive for the schools; practical, willing leaders discuss cases. Ago new structures must answer. More willing minutes claim more. F	Home                                              	mattresses                                        	5.91	5652.60	2.02098057136409324
+Physically useless findings continue other critics; perhaps young forms substitute coins; arms command 	Home                                              	mattresses                                        	0.77	13274.08	4.74589707085813303
+Groups make in t	Home                                              	mattresses                                        	4.98	5572.29	1.99226724480883542
+Skills should spend twins. Certain, industrial homes will get to a rights. Decisions could buy politically so difficult differences. Running magistrates cannot respect thickl	Home                                              	mattresses                                        	7.20	4964.20	1.77485612857191941
+Here extra efforts ensure eyes; merely little periods will not loosen home past a boys. Just local aspects must reclaim. Standard qualities might not roll today. Military, national clothes must go wid	Home                                              	mattresses                                        	3.34	4129.43	1.47639985153876580
+Possible, rich d	Home                                              	mattresses                                        	4.63	10156.22	3.63116500344963929
+Japanese years escape so good objects. Tiny features see then proud heads; abroad full secrets might not re	Home                                              	mattresses                                        	0.95	2753.98	0.98463363300521627
+Past, interior years fetch accidents. Away internal feet would not organise so square collective rocks. M	Home                                              	mattresses                                        	6.31	3321.81	1.18765054519388575
+National, difficult pain	Home                                              	mattresses                                        	0.37	987.66	0.35311921436391401
+British differences discuss almost in the advantages; in particular international operations go then in a architects. Regional, fair costs commit merely political items. Then difficult travel	Home                                              	mattresses                                        	3.06	430.92	0.15406732261476401
+Never arab policies follow only. Valuable employees might shed. Recently relative costs order just with a areas; sessions may come somewh	Home                                              	mattresses                                        	6.84	7661.12	2.73908903423006793
+Perhaps blank models work certain	Home                                              	mattresses                                        	4.17	1990.47	0.71165502563122929
+Keys must not read political, italian farmers. Red, single years should play however at the dates. Authors disturb no longer for a purposes. Ever essential agencies will answer as fundame	Home                                              	mattresses                                        	42.14	5401.80	1.93131175926026233
+Payments forget. Doubts make respects. Considerable, available states should go here. Only public pages might differ. In	Home                                              	mattresses                                        	3.45	2289.13	0.81843527851372585
+Well able areas examine respectively previous services. Surprised computers ought to love british, sole appeals. Common, similar inhabitants finish from a seco	Home                                              	mattresses                                        	7.94	3465.86	1.23915290716979022
+Social councils used to determine yet at the boats. Persons ask alive months. Individual, considerable rooms note cases. Then only policies may look to a 	Home                                              	mattresses                                        	4.91	4363.94	1.56024448122963257
+Films must ta	Home                                              	mattresses                                        	6.04	6064.00	2.16806888595546499
+Educational hopes appear more high others; black thoughts might close always in a officials; close years base top workers. Regulations ask over high widespread	Home                                              	mattresses                                        	3.52	15000.77	5.36324253007489455
+Vital arms generate slow, neat judges. Specially simi	Home                                              	mattresses                                        	4.42	10296.27	3.68123724083058633
+Closely blind winners might come similar, local crops. Very difficult evenings can stretch only ago naked hands. Sufficient, similar 	Home                                              	mattresses                                        	6.05	13831.69	4.94526001470668627
+Natural beans look often bacteria. Square, small items must negotiate for the forces. Hence chief ha	Home                                              	mattresses                                        	6.40	161.10	0.05759826806191052
+Large, very materials like otherwise long, rough concepts. Sources give as local children. Rapid customers remove gently downwards short expressions. Behind national crimes confess n	Home                                              	mattresses                                        	7.74	1076.05	0.38472139260098583
+Growing, social objectiv	Home                                              	mattresses                                        	7.70	8.96	0.00320347909270464
+Ago new studies shall not apply of course small forces. Dead parts used to point on a students. Then other students should pay only	Home                                              	mattresses                                        	8.92	16657.18	5.95546070015825401
+Good, ethical interests stress now because of the eyes; patients used to give so hills. Social operations will pronounce basic ideas. British friends store too p	Home                                              	mattresses                                        	0.68	2433.04	0.86988758612880682
+Following, combined cells must ease of course continued changes. German te	Home                                              	mattresses                                        	5.91	785.92	0.28099088041723599
+Old words cannot force. Equal, capital problems would not produce; great, competitive things congratulate only times. Vice versa unemployed complaints will say previous gardens. Difficult, uncomfort	Home                                              	mattresses                                        	1.57	1412.84	0.50513430818491411
+Now comfortable grounds bowl much only double groups. Good talks must not support somewhat; used, linear	Home                                              	mattresses                                        	5.00	5416.79	1.93667115117986530
+Respectively excellent things speak reliable, historical movements. Masters respond. Cheap ideas should featu	Home                                              	mattresses                                        	3.37	5563.35	1.98907091633910557
+Prisoners ought to leave. Main items should not strengthen ago allowances. Ideas provide together between a patients. Regional, english conditions argue also in a minutes; ordinary trials become lon	Home                                              	mattresses                                        	36.96	6326.30	2.26184930626979851
+Cases move so very natural tories. Therefore political cells win there already eastern events. Extra questions encourage skilled efforts. Serious, physical clothes would 	Home                                              	mattresses                                        	80.68	751.60	0.26872041139250123
+Individuals recognise. Really elegant relations should extend totally types; attitudes would relate muc	Home                                              	mattresses                                        	7.09	1139.56	0.40742819585742244
+Evidently super tales may risk just; others match maybe. Lovers describe anywhere 	Home                                              	mattresses                                        	2.32	9619.86	3.43939959651179740
+Minimum words	Home                                              	mattresses                                        	6.86	4696.59	1.67917721785777990
+Other, extra notes alter so social ways. Different, preliminary parts take so diffic	Home                                              	mattresses                                        	3.40	10150.42	3.62909132278695101
+So social decisions fulfil again comparative times. Academic governments ought to arise then on a decades. 	Home                                              	mattresses                                        	1.81	1346.52	0.48142284240051991
+Often presidential councillors used to take in the friends. Exact, rich visits used to want sophi	Home                                              	mattresses                                        	0.41	8719.30	3.11742134520308145
+Constant, domestic things might worry like a minutes. Literary, kind sales tell however emotional branches. Too specific troops may not bring most fair unknown owners. Issues look official	Home                                              	mattresses                                        	0.51	148.32	0.05302901998102153
+Even successful questions continue indian areas; good, good jobs get nice, famous interests. Labour, generous circumstances help good changes. Strict, vulnera	Home                                              	mattresses                                        	2.55	2079.26	0.74340021632779686
+Good, full governments consider elsewhere genuinely	Home                                              	mattresses                                        	0.33	11909.49	4.25801364059989293
+Days should conti	Home                                              	mattresses                                        	3.57	1697.42	0.60688052249316052
+Personal, national arts ought to rely still strategic, dead instruments. Finally federal spots remember. Laws 	Home                                              	mattresses                                        	3.72	13796.99	4.93285368384543056
+New products should not see. Much separate subjects give at least existing implications. Similar corporations might turn years; local	Home                                              	mattresses                                        	3.84	1888.50	0.67519757439427698
+Other parties will add regional, special owners. Little administrative horses may indicate; 	Home                                              	mattresses                                        	1.41	23082.32	8.25264838516945064
+Often local men ought to suppress trousers. Angry studies might cool seeming	Home                                              	paint                                             	0.70	4572.36	1.91646328201692969
+Worthy, rich types force both shy years. Tropical, personal views might work. Other eyes ought to administer neve	Home                                              	paint                                             	0.28	12758.19	5.34747978724238078
+Rural others come as through a estimates. Publications should worry really powerful	Home                                              	paint                                             	3.24	4960.42	2.07911511634744823
+Early, dangerous weeks live still to a changes. Vari	Home                                              	paint                                             	2.74	12614.97	5.28745042138963413
+Perhaps de	Home                                              	paint                                             	1.44	1475.69	0.61852209813740890
+Clinical, national residents might cry; objects ought to justify only relatives	Home                                              	paint                                             	7.77	2688.57	1.12688976505180184
+Equal forces tell together feet. Never new police cannot place hardly big, independent years. Then old choices ought to afford especially; parties accept	Home                                              	paint                                             	6.51	6336.50	2.65588658515520978
+Always huge concessions express directly ameri	Home                                              	paint                                             	4.52	9357.30	3.92202754569128769
+Average decisions shall see. Lovely, wide temperatures prepare in a regulations. Right arms ought to make now applic	Home                                              	paint                                             	57.27	3310.24	1.38745711507049343
+Vast, separate feet wear financially other, dangerous workers. Other, old genes spin for instance ordinary purposes. Events could focus anywhere under fresh countries	Home                                              	paint                                             	7.37	10616.13	4.44965473893533925
+Quickly far walls shall see gold, true patients. Above bad pensions will insist as round detailed degrees. Free,	Home                                              	paint                                             	0.70	809.31	0.33921495655834654
+Probably political hands may park easily. Little, suitable officials apply today men; women ought to take to the provi	Home                                              	paint                                             	6.55	2700.80	1.13201585878437474
+Special words should tell by a follower	Home                                              	paint                                             	1.68	592.00	0.24813143824065086
+Both usual effects include repeatedly low, possible practices. Professional, past countries retain yesterday ways. Equally old	Home                                              	paint                                             	0.84	1006.06	0.42168093708849528
+Capital areas judge almost active, numerous f	Home                                              	paint                                             	9.32	661.29	0.27717371417932434
+Pale, original yards agree social, little generations. Weeks used to include now oral shows; 	Home                                              	paint                                             	2.40	5882.28	2.46550438603752661
+Appropriate, like v	Home                                              	paint                                             	4.82	372.76	0.15623897790301523
+Attitudes must build ge	Home                                              	paint                                             	45.77	9930.33	4.16220788024372040
+Very difficult parts ought to know else areas. Members could not comment of course male, popular girls. Primary, worried actions might send indirectly elsewhere hard children. New resou	Home                                              	paint                                             	3.98	770.04	0.32275529172775471
+Careful universities may find cultural methods; artificial, apparent sections ought to tell highly reforms. Medical, glorious studies shall not agree straight almost actual states. Enough n	Home                                              	paint                                             	4.20	103.50	0.04338108759781649
+Players shall mean with a rights. Occasionally popular enemies worry. In general basic patients get perhaps parts. Other varieties enjoy thousands; classes shall not spend as for the families. New f	Home                                              	paint                                             	2.13	5837.14	2.44658436387167698
+Students wi	Home                                              	paint                                             	2.79	4724.08	1.98005534588495595
+For example clear witnesses used to enjoy yet international, environmental computers. Ill	Home                                              	paint                                             	9.67	59.46	0.02492212046923835
+Opposite youngsters see altogether. Plans may not say to the problems. Popular, new lands might create cha	Home                                              	paint                                             	4.08	7043.01	2.95201385277582167
+Objects sell so yellow ru	Home                                              	paint                                             	1.47	1136.47	0.47634110746174406
+Only horses can forget meanwhile animals. Rich exception	Home                                              	paint                                             	67.74	386.10	0.16183031808228935
+Responsible, useless sources explore there. Serious, conventional fields could defend once again famous efforts. Officials call as notions. Big, ap	Home                                              	paint                                             	3.14	8952.05	3.75217067855104485
+Aware groups could finish services. Companies make also glad, top ways; t	Home                                              	paint                                             	3.27	1574.90	0.66010507108986663
+Ever insufficien	Home                                              	paint                                             	2.77	3898.21	1.63389941531095878
+Se	Home                                              	paint                                             	7.48	13291.94	5.57119626555479193
+Windows avoid. Always noble funds should lead nowhere able initiatives. Under new groups wait plans. High enterprises could know inadvertently different, main	Home                                              	paint                                             	8.31	804.05	0.33701027519830292
+Human honours tell requests. Effective, late crimes know on a courses. Adequate, typical men should not tend already in a nerves. 	Home                                              	paint                                             	1.35	7526.60	3.15470622138865334
+Patterns might not maintain. Great, vast eyes say still different views. Easily national plants develop together with the cities. Able g	Home                                              	paint                                             	21.04	8770.96	3.67626844518787008
+Possible guests want only; organisations weigh however explicitly c	Home                                              	paint                                             	4.69	2761.50	1.15745771402290094
+Letters state on a chains. General, criminal cases shall look unknown months. Special, poor nights give as ever	Home                                              	paint                                             	7.47	3235.66	1.35619758354348711
+Also inc goods could not lay 	Home                                              	paint                                             	2.41	2540.30	1.06474373743703612
+Additional companies visit. Grey opportunities may not look numbers. Entire, british models assist also great quarters. Little males show	Home                                              	paint                                             	51.57	13562.60	5.68464095318015436
+Communist, different demands die perhaps kinds; likely, public forests should make moral, nice faces. Efficient, central services can p	Home                                              	paint                                             	0.27	668.17	0.28005740386698596
+Effectively initial representatives amount dark areas; comprehensive, christian words will not want hearts. There judicial men explain r	Home                                              	paint                                             	4.54	5116.69	2.14461427150600652
+Reasons will look probably key students. Now very bones us	Home                                              	paint                                             	3.58	54.00	0.02263361092059991
+Features need stages; french cells come hours. Still small beliefs look scarcely electric, good producers. Churches receive for the seats; businesses get appropriate, high ways. Purpo	Home                                              	paint                                             	2.89	7559.52	3.16850434123135981
+Managers ought to express so new faces. Universities should not appear at a stories. Accidents dismiss only single times. Other, current companies could not meet effectively that is to say perfe	Home                                              	paint                                             	0.74	6272.75	2.62916635004061266
+There blue items see in a conditions; lives ask silent countries. Here necessary months may encourage free 	Home                                              	paint                                             	7.02	4828.00	2.02361247267882156
+New dollars might end old relationships. Other, gentle groups 	Home                                              	paint                                             	8.34	2369.97	0.99335146062026237
+International years collect respectively affairs. Exter	Home                                              	paint                                             	69.84	5908.06	2.47630983954739820
+Colleagues attach th	Home                                              	paint                                             	9.80	2499.83	1.04778110347487541
+Furthermore additional 	Home                                              	paint                                             	8.18	1563.59	0.65536458702482987
+Months find there costly foreigners. White, particular changes used to share away in a subjects. Muscles make fully less gold fingers. Norm	Home                                              	paint                                             	4.97	14512.01	6.08257755584916844
+English persons last there golden units. Special services help far vital fingers. Very complicated birds sho	Home                                              	paint                                             	0.74	1043.89	0.43753703896120444
+Hands might contact enough growing things. Criteria used to make convincing forms. Particular organizations sha	Home                                              	paint                                             	48.89	8562.98	3.58909551186812250
+New, american owners might not appear. Parties move heavily also high variations. Unable, american terms might create indeed years. Nations absorb over normal experienc	Home                                              	rugs                                              	0.89	2701.48	0.99827241978850362
+Concepts shall allow cautiously there	Home                                              	rugs                                              	4.82	8082.19	2.98659526203801105
+Awards might mention better real, video-taped fires. Familiar patients must yield finally never net rules. Courses should attend; black ac	Home                                              	rugs                                              	0.79	120.11	0.04438400444970800
+Smoothly main organisations yield here pensioners; subtle, british rights say public books. Only, social pairs take up to the police. Important, other men could go mor	Home                                              	rugs                                              	6.67	21599.16	7.98149374365127852
+For example brief children must change almost. Fierce manufacturers ought to throw comfortably alone, subsequent loans; other boots switch. Very main men k	Home                                              	rugs                                              	7.88	1113.44	0.41144722266657961
+Forms carry here american negotiations. Partly subject drivers should tell only stiffly local orders. Quite clean forces will enhance intentionally full ministers; stories mus	Home                                              	rugs                                              	7.64	9195.42	3.39796488383093785
+Royal, comprehensive reports cost loyal, critical minutes. Exciting, short areas ought to pay for a appearances. Public, large institutions can	Home                                              	rugs                                              	4.30	2726.74	1.00760669630502701
+Of course institutional forces occur violently from a governments. Patient, western teams count 	Home                                              	rugs                                              	1.97	500.94	0.18511134117922509
+Great images may not pay only, certain plans. Internationally new years command so in the days. Stairs tell teams; else unlike customers see elected, different numbe	Home                                              	rugs                                              	2.11	8294.23	3.06494997274915987
+Organizations understand also instead accurate settlements. Costs become co	Home                                              	rugs                                              	7.44	12898.01	4.76617544944116470
+Broad, political premises must not continue even. Short local levels stay in a germans. Encouraging, poor priorities i	Home                                              	rugs                                              	9.98	13098.17	4.84014016787138328
+Consumers must light now human schools; systems take 	Home                                              	rugs                                              	37.18	2295.76	0.84834753189127999
+Hardly happy reforms may not try quickly along a pp.; sure sources use then now different	Home                                              	rugs                                              	3.58	2396.96	0.88574376243253759
+However magic things should not take for a firms. Estimates supply; able, doubtful children must maintain left, lacking banks; simple sons c	Home                                              	rugs                                              	1.73	113.88	0.04208184519800805
+Ideological members get sometimes modest abilities. Used, certain services would make all victorian, angry regulations. Even voluntary directions must sail however equations. Other, specific others ge	Home                                              	rugs                                              	8.46	4771.52	1.76321009834210907
+Turkish members shall know to a subjects. No doubt decisive millions might know virtually public industries. Good, artificial 	Home                                              	rugs                                              	1.62	4557.68	1.68419023728536476
+Softly social men get only with a miles. Only afraid difficulties should emerge t	Home                                              	rugs                                              	4.09	5355.01	1.97882597342628292
+Others could withdraw buildings. Clothes know partly. Inner prese	Home                                              	rugs                                              	4.44	7946.40	2.93641705902222677
+Parallel dead relations check further international men. Types improve apart half way steady ways; back metres shall not support at leas	Home                                              	rugs                                              	1.00	9684.36	3.57864188937285967
+Good, alone centuries might not see gently subjective ships. Less ambitious 	Home                                              	rugs                                              	6.42	3762.17	1.39022704204943760
+Also other republics could not prescribe almost permanent mental p	Home                                              	rugs                                              	3.56	1252.71	0.46291138301718183
+Coastal agencies encourage. Obviously other events understand local students. Western subjects cannot set in a e	Home                                              	rugs                                              	6.19	3558.04	1.31479529757921118
+Existing services make talks. Concerned, running	Home                                              	rugs                                              	30.02	2214.66	0.81837881354250538
+However major months execute either elements. Enough left provisions used to prove so away gastric police. Animals shall add faintly things. Well modern principles might pay suddenly other, soc	Home                                              	rugs                                              	1.32	16957.77	6.26637032001602569
+Mental horses could grab 	Home                                              	rugs                                              	1.74	1044.31	0.38590175411601501
+Other, initial companies could know definitely mere funds. Italian years get already thereafte	Home                                              	rugs                                              	8.14	4357.37	1.61017008965967989
+Additional, interior police provide words. Different, long qualities answer really concerns; then other words state dry, political services. Awfully di	Home                                              	rugs                                              	9.78	7977.70	2.94798328447619281
+Firm, main skills can measure already electoral, white activities. Fairly disciplinary men protest there new changes. Strong, good reactions might prompt arbitrarily wild product	Home                                              	rugs                                              	6.42	9423.50	3.48224682317728204
+Origins used to play very on a matters. Long, important shows tackle more. Further vast fingers succeed only. Much dead values must rem	Home                                              	rugs                                              	4.71	7612.23	2.81293189736242391
+Possibly southern complaints would not produce to a years; months take. Services give; always professional days might develop quickly major urba	Home                                              	rugs                                              	36.03	10189.52	3.76531263858453641
+British stories ought to read furt	Home                                              	rugs                                              	2.05	1296.18	0.47897476386331293
+Better silent colleges protect never concessions. Certainly material words 	Home                                              	rugs                                              	2.45	7108.50	2.62678957314752580
+Still global systems would find real forces. Facts get rivals. Ahead british features must not rest nearly. Flats will restrict always subsequent miles. Then new children can allay only ordi	Home                                              	rugs                                              	8.72	430.95	0.15924807857465376
+Possible	Home                                              	rugs                                              	0.41	9833.88	3.63389371141365844
+True 	Home                                              	rugs                                              	55.56	1867.47	0.69008239771622846
+Difficult writings improve full charges. Western incidents run in a options. Parts happen possible, forw	Home                                              	rugs                                              	4.45	2413.98	0.89203312847811273
+Past losses will feel nowhere options. Political, free situations must produce selectively classes. Difficult ways believe sometimes enormous scientists. Interesting, simple rights ought to flush ago	Home                                              	rugs                                              	4.83	1972.51	0.72889761566142310
+Minds apply reluctantly dirty goods; therefore extended unions make secret, working men. Followin	Home                                              	rugs                                              	0.63	215.67	0.07969609724143306
+Possible, false publications produce toda	Home                                              	rugs                                              	62.90	1868.41	0.69042975400781722
+Wonderful, scottish unions go nearby for a teams. Gladly current systems cannot look so major, emotional p	Home                                              	rugs                                              	7.31	5243.52	1.93762730007603777
+Dead names spend as a schools. Polit	Home                                              	rugs                                              	1.98	718.90	0.26565365747144353
+Standard, foreign hospitals say later adult difficulties. Things ask very into a metals. Enough public persons will not give however ago sweet c	Home                                              	rugs                                              	0.57	5940.00	2.19499614046511968
+Single institutions place also local ideas; variations used to appear yesterday domestic, corresponding attempts. Unlike, possible amounts open locally. National, main cig	Home                                              	rugs                                              	7.07	11038.74	4.07912318107709347
+Also noble characteristics might sound about a miles. Again social funds would stretch en	Home                                              	rugs                                              	7.90	2544.16	0.94013827958345773
+International metres minimise originally small allowances. Eminently favorite lines compare just never bottom things. British poets take countries; individual, in	Home                                              	rugs                                              	1.63	3135.51	1.15865864451006522
+Colourful bones may adjust so. Pupils might catch so. Final,	Home                                              	rugs                                              	86.39	282.42	0.10436208922393251
+Able armies bring certain, pretty requirements. Dogs pay weeks. Simi	Home                                              	rugs                                              	46.20	4674.82	1.72747674366484020
+Foreign, absolute bills sh	Home                                              	rugs                                              	0.23	4232.41	1.56399387455656182
+Levels look only steep, cold results. Examples used to ensure together only expensi	Home                                              	rugs                                              	5.36	2875.57	1.06260354404668084
+African days incorporate economic, similar cells; vast, automatic stations ought to plan previously in a judges. Blank times would pay into the workers. Gradually ultima	Home                                              	rugs                                              	2.42	1831.70	0.67686438223736696
+Hands order. Pl	Home                                              	rugs                                              	91.05	5998.14	2.21648049662785404
+Magic facilities should not fight only likely months. Later present members absorb once more	Home                                              	rugs                                              	8.11	1193.91	0.44118313839439580
+As active accounts talk slowly. Big implications make as a children. Rounds should not check. Likely, military	Home                                              	rugs                                              	5.59	2607.00	0.96335941720413586
+Prime members must need so regulations. Only injuries might run adequately to a shares; inevitably orthodox poets think yesterday protests. Thinking, full changes could put more. Months 	Home                                              	rugs                                              	9.27	2740.60	1.01272835396611229
+Clinical photographs look also popular, common men. Loose, concerned earnings must go maybe only able enquiries; black unions observe exactly by a	Home                                              	rugs                                              	24.08	2749.12	1.01587673226859761
+Directly green hours will maintain also	Home                                              	tables                                            	1.10	1433.48	0.74353680625971805
+Then legal services may bother circumstances; obvious, original years worry scottish, static areas; much fresh journals mean exactly routes. I	Home                                              	tables                                            	4.46	15267.45	7.91912758652365733
+Small motives shall use large, patient payments. Answers refer here odd, average officers. Always powerful sections might yield into a 	Home                                              	tables                                            	4.41	5271.29	2.73418403568155059
+Odd, poor times could recycle suddenly eyes. Fa	Home                                              	tables                                            	0.33	2225.20	1.15419685052398680
+Perfect grants fight highly as great minutes. Severe, available millions like counties. Young legs cook however from a years. Early armed services reject yet with 	Home                                              	tables                                            	4.31	7602.83	3.94353875654740364
+True, particular parties drop for a times. Too mad 	Home                                              	tables                                            	56.61	2020.10	1.04781280682343418
+Usually complete artists will give from the weeks. Units swallow political minutes; books might not arrest continually lips. Modest, royal problems must behave consequently genera	Home                                              	tables                                            	4.25	4496.26	2.33218098648974514
+Particularly popular detectives avoid rather free, major relations. Financial servants may know also widely surprising children. Delegates cannot get. However separate thousands discuss alway	Home                                              	tables                                            	4.93	7737.75	4.01352088807387150
+Nuclear needs can want. Overwhelmingly clo	Home                                              	tables                                            	0.43	930.32	0.48255096799365244
+Enough bad rounds arrange later well black places. Courses reduce then in a experts. Also poor systems offer wonderful performances. Economic, unlikel	Home                                              	tables                                            	21.49	7678.11	3.98258600574183368
+Actions see of course informal phrases. Markedly right men buy honest, additional stations. In order imaginative factors used to move human thanks. Centres shall catch altogether succe	Home                                              	tables                                            	1.61	33.06	0.01714800821423827
+Federal, clear months please. New lips take slightly interesting 	Home                                              	tables                                            	3.47	361.20	0.18735210426445445
+Roots should not lend overnight in a feet; fine children retire once usually evident forests. Sometimes novel effects might not go tons. Casualties involve more. Correct, perfect deleg	Home                                              	tables                                            	3.13	10251.08	5.31716890637669900
+Provincial, important tr	Home                                              	tables                                            	3.22	2399.31	1.24450658162444130
+Western, complex eyes can tell only regular acts. Perhaps high processes could put. Changes stay in the prisoners. Ages give now fascinating methods. British, quick words shall not expect new 	Home                                              	tables                                            	4.27	9672.26	5.01693871537351095
+Now professional schools will not visit useful lists. Beautiful plans can recommen	Home                                              	tables                                            	2.52	408.50	0.21188630839432348
+Personal dimensions can dissolve final variations. Gradual sounds migh	Home                                              	tables                                            	1.19	5519.07	2.86270591938765946
+Hard sheets share so books. Permanent	Home                                              	tables                                            	31.00	443.40	0.22998871271001966
+Current degrees see in particular be	Home                                              	tables                                            	2.99	2250.99	1.16757395675039954
+Vast girls call benefits. Good, difficult makers deliver local things. High, formal hours play for a payments; well new men increase all equal newspapers. Top, total rights	Home                                              	tables                                            	2.62	10786.92	5.59510564931431049
+Just responsible poems ask only just joint patients. Solid, equal books prevent. Never universal fields must ignore black, main cameras	Home                                              	tables                                            	0.32	6835.22	3.54538441337343388
+Most official languages might not feel anywhere careful points; good, post-war prices refer originally ruling varieties. Increased lands would not get we	Home                                              	tables                                            	0.35	13164.59	6.82838770287595335
+Important, small girls should realise only high numbers. Previous, statutory products can give rather scientific methods. Isolated, living estates move now old trees; univ	Home                                              	tables                                            	2.85	3966.40	2.05734603088187185
+More german bags might not give always about a words. Recently new guests ought to 	Home                                              	tables                                            	8.63	4805.11	2.49237948428065532
+Too labour operators tell more	Home                                              	tables                                            	3.43	9131.41	4.73640331783356027
+Families must not hear more great, english feelings. Proper faces justify extremely skills. Immediate discussions undertake often pa	Home                                              	tables                                            	0.18	2677.96	1.38904053470664016
+Experts should not offer; low easy cities flourish particularly integrated, decisive 	Home                                              	tables                                            	9.66	3549.82	1.84126867873766800
+Simply different statements complete always social, international speakers. Early serious buildings shall overcome just by a husbands; complex, common criteria will work little, fair countr	Home                                              	tables                                            	2.23	2835.45	1.47072957928196943
+Only long brothers detect in a years; commitments can imagine near little great fields. Civil, soviet patients profit already just long arrangements. Often indi	Home                                              	tables                                            	8.94	690.05	0.35792447272338536
+Central houses increase actually essential payments. Minor organizations take subsequently careful players; good, molecular righ	Home                                              	tables                                            	7.94	13582.01	7.04490075758821408
+Women get also chairs. Full, integrated paintings sit 	Home                                              	tables                                            	6.34	1123.11	0.58254989429803830
+Wild volunteers expand approximately sales. Specific, close versions must stress longer able powers. Far me	Home                                              	tables                                            	3.86	2363.26	1.22580767974533392
+Bold parties could revert newly equal plans. Also other products cry as at least lovely discussions. Manufacturing, french letters lay economically ready duties; serious, stron	Home                                              	tables                                            	1.02	2741.71	1.42210724746095625
+Areas ought to calculate slowly charges. Difficult, national participants might not write away bus	Home                                              	tables                                            	4.21	5457.26	2.83064547208814138
+Closely young offic	Home                                              	tables                                            	8.10	25.92	0.01344453638575487
+Wide, new changes reduce highly on a notes. Nurses re	Home                                              	tables                                            	0.25	1860.34	0.96494632792728456
+Critical, neighbouring feelings should achieve unusual, hungry types; po	Home                                              	tables                                            	5.93	619.20	0.32117503588192191
+A	Home                                              	tables                                            	4.83	2031.72	1.05384002568155423
+New situations seem. National missiles will cater departments. Women come astonishingly. Spanish mont	Home                                              	tables                                            	5.87	8171.71	4.23861313382858538
+Highly tory votes could no	Home                                              	tables                                            	8.80	3686.85	1.91234525361961205
+Slight, present techniques run writers. Schemes make. Grand boys could help fine, past re	Home                                              	tables                                            	1.51	332.04	0.17222700083048022
+Dead, big talks will rest old offers. Dead, competitive authorities occupy alone	Home                                              	tables                                            	0.38	2425.28	1.25797705268686622
+Almost working things shall not see import	Home                                              	tables                                            	3.78	3316.68	1.72034046836055031
+Police know more families. Atlantic birds might keep there far presen	Home                                              	tables                                            	40.62	0.00	0.00000000000000000
+Obviously elaborate members would not retu	Home                                              	tables                                            	3.94	610.39	0.31660534585265877
+Quiet levels must achieve. Local, national metres fill to a businessmen. Real, key boots could not determine at best. Young groups may know ever happy, magnetic difficulties	Home                                              	tables                                            	2.15	NULL	NULL
+Labour, middle children might produce useful signals. Surprising farmers kill on the costs. Trees return recent, single animals. Original governments read over there. Previous	Home                                              	wallpaper                                         	3.08	5699.40	1.39109945794862842
+Once again only measures shall destroy independent, normal prisons. Present, industrial ambitions can prevent as employers. Large, previous origins say inside 	Home                                              	wallpaper                                         	3.32	262.60	0.06409494291632625
+Reports can say. Constant, other keys will analyse here white months. Dreams would not change to a neighbours; visual, financial wages set in a girls. Fingers 	Home                                              	wallpaper                                         	4.24	9127.17	2.22774348871898495
+Nearer regular men like in a ministers; children come therefore female views. Only financial events must not allow old miles. Very british forces get. 	Home                                              	wallpaper                                         	9.72	5545.66	1.35357487103333520
+Great, strategic live	Home                                              	wallpaper                                         	2.35	12111.89	2.95624866016307208
+Groups can consent close. Awful, soft friends pursue comfortable departments. C	Home                                              	wallpaper                                         	6.57	1777.90	0.43394668320996359
+Empty, additional russians should ensure commonly in a books. Sure, close difficulties follow always on a weeks. Royal y	Home                                              	wallpaper                                         	0.85	328.29	0.08012844177456491
+Educational, reasonable rooms mi	Home                                              	wallpaper                                         	2.73	737.08	0.17990518097778275
+Then french ministers aid	Home                                              	wallpaper                                         	3.16	7027.37	1.71522802361730232
+Old eyes would not develop to a parents; nice, red games come now to a molecules. Sheer centuries could follow as usually late powers; backs affect police. Almost tiny trees shall buy fro	Home                                              	wallpaper                                         	1.22	20810.71	5.07944123952101991
+American, long organisations terminate for a agents. Facilities determine open. Now general students rebuild even particular pounds. Good teachers might not press names. Guidelines evaluate clear	Home                                              	wallpaper                                         	4.09	293.44	0.07162231549644621
+Public 	Home                                              	wallpaper                                         	0.64	1015.94	0.24796883589646797
+Initial unions agree still otherwise individual councillors. Leading minutes bring mathematical conditions. Full, huge banks must not feel exclusively special lines. Ago other cases will hold	Home                                              	wallpaper                                         	8.36	1699.28	0.41475725285169409
+Fresh, othe	Home                                              	wallpaper                                         	8.40	501.78	0.12247357371117359
+Ahead national cir	Home                                              	wallpaper                                         	14.29	13998.80	3.41680231110840781
+Still fortun	Home                                              	wallpaper                                         	4.83	4391.94	1.07197693675525478
+Minor, single things could cry too profits. Examples focus material, young observations. Existing tensions would stop away. Facilities reply most thoroughly small	Home                                              	wallpaper                                         	3.85	6735.50	1.64398891094027208
+Wooden, clear considerations will not become now proceedings. A bit institutional firms will 	Home                                              	wallpaper                                         	4.94	9408.96	2.29652229284842735
+Thick, other ways come completely. Careful men would find later there valid children. Interesting owners allow a bit best wide polls. Miles behave other, considerable heads; inte	Home                                              	wallpaper                                         	0.96	3860.39	0.94223715416891351
+Marked, free flowers carry restrict	Home                                              	wallpaper                                         	0.67	4918.41	1.20047680193864503
+Less western books give physically only 	Home                                              	wallpaper                                         	4.22	5084.28	1.24096205777082719
+NULL	Home                                              	wallpaper                                         	NULL	15833.49	3.86461019693915650
+Liable, other others provide also in a resources. Months get briefly long sheets. Windows talk activities. American	Home                                              	wallpaper                                         	5.42	151.36	0.03694368073044608
+New citiz	Home                                              	wallpaper                                         	3.50	6508.22	1.58851481106966039
+Main elements write generally however secondary periods. Documents persuade empty, labour margins. Over other friends contend afterwards friendly, labour buildings. Canadian birds 	Home                                              	wallpaper                                         	4.07	2883.10	0.70370194182048822
+Shortly economic records cause nevertheless by a requirements. Privately silent forms take. Pink leaves aba	Home                                              	wallpaper                                         	8.70	0.00	0.00000000000000000
+Stores visit values. Others cannot hang around rather civil brothers. Direct systems go then free, other instructions. Difficult, top feet will al	Home                                              	wallpaper                                         	13.91	2088.96	0.50986965710010998
+Small, social patterns design deeply without a judges. Moving feet arrange in the developments; sports say	Home                                              	wallpaper                                         	0.63	13980.62	3.41236496890650830
+Tests should allow finally times. Thus other differences act already important weapons. So ridiculous spor	Home                                              	wallpaper                                         	3.26	12082.76	2.94913866135441792
+Courts must not understand ideas. British figures would isolate sure young preparations; able, short governments should express more private properties. Countries de	Home                                              	wallpaper                                         	0.28	15297.35	3.73375009528203862
+Military, poor questions challenge that with a costs. Appropriate, main patients will not see concerned, industrial findings; terrible, concerned eyes decl	Home                                              	wallpaper                                         	3.37	3242.71	0.79147491372505823
+Green, european terms privatise new arms; also local duties need damp, successful professionals. Fresh, furious odds will undertake too only probable players.	Home                                              	wallpaper                                         	2.81	227.73	0.05558393507362900
+Impossible, other patients provide somewhat. Initially helpful ref	Home                                              	wallpaper                                         	2.44	10361.84	2.52909955562873563
+Always western women run soon in the solutions; left members should allow national, innocent products. Of course left tonnes thank unduly especially interested customers. Elderly pen	Home                                              	wallpaper                                         	0.99	7449.54	1.81827052952356833
+Artificial, careful years behave even specialist volumes. Assistant che	Home                                              	wallpaper                                         	7.43	6528.95	1.59357455275532468
+Short things come from a activities. Losses should not work ro	Home                                              	wallpaper                                         	9.19	3438.64	0.83929716111879700
+Courts can pu	Home                                              	wallpaper                                         	9.63	7132.45	1.74087576391298992
+Representative, keen problems might exam	Home                                              	wallpaper                                         	6.78	17424.37	4.25290936977512414
+Useful developments might control effective, unknown homes. Other, right marks cannot become by the moments. Natural, christian bars used to enable also new	Home                                              	wallpaper                                         	75.10	6730.56	1.64278316448937089
+Perhaps different figures hang new women. Dynamic goods finance now; birds keep already proposals. Schemes guess animal	Home                                              	wallpaper                                         	4.93	11316.14	2.76202340949412078
+S	Home                                              	wallpaper                                         	2.23	2663.69	0.65014873761153490
+Difficulties should 	Home                                              	wallpaper                                         	3.85	3734.34	0.91147109341261905
+New, poor adults used to fear; new offers may make undoubtedly cells. Clinical dogs decide. Then poor models know then entirely rea	Home                                              	wallpaper                                         	0.20	10778.60	2.63082159831650459
+Significantly poor employees will not attend over interactions. Other babies used to choose departments. Young members repair. Easy patients find ever pers	Home                                              	wallpaper                                         	6.87	6138.42	1.49825468201232053
+Perfectly tall bodies think there a	Home                                              	wallpaper                                         	6.25	2518.24	0.61464755921404955
+Areas would stop also logical, local initiatives. Existing, increasing words should take open concerns. Objectives protect jointly at t	Home                                              	wallpaper                                         	6.48	7065.22	1.72446638458220312
+Human, back b	Home                                              	wallpaper                                         	4.28	8161.86	1.99213233355310951
+Measures should make rec	Home                                              	wallpaper                                         	2.45	3471.50	0.84731757172135024
+Familiar thanks should see proposals; more single lakes shall not announce employees. Specified lawyers canno	Home                                              	wallpaper                                         	7.89	509.65	0.12439446937283196
+Basic moves mig	Home                                              	wallpaper                                         	0.30	11860.26	2.89483125541807904
+Components could identify hopef	Home                                              	wallpaper                                         	1.39	1204.56	0.29400687143674770
+Social dealers shall emerge even figures. Clear prayers could not send. 	Home                                              	wallpaper                                         	6.93	6706.36	1.63687647134932864
+Actual, urban police learn quickly low, british years; ethnic, common months should fail then overall markets. Years get. Criminal statio	Home                                              	wallpaper                                         	7.74	1379.50	0.33670591680530107
+Particularly tight problems cannot lead special, simple sales. Warm bodies get. New, primary attempts wo	Home                                              	wallpaper                                         	5.23	15517.89	3.78757910788967986
+Chief, other others speak fairly; established years may reduce political museums. Vulnerable, male features sug	Home                                              	wallpaper                                         	4.79	7653.42	1.86803319883727966
+Much following charges cannot complete difficult, effective jews. Poor, commercial pro	Home                                              	wallpaper                                         	1.85	5730.05	1.39858045566525218
+Special, long-term cases may not like sharply favorite arms. Insufficient papers bring. Legal cheeks could not apply with a sales. Terms give. Judicial, natural sets see at the cells.	Home                                              	wallpaper                                         	2.40	15153.09	3.69853936997697683
+Sensitive, labour areas would not suffer general, successful seconds; golden, substantial methods pay then available beliefs. Afterwards round years will defeat 	Home                                              	wallpaper                                         	1.96	4949.14	1.20797732591358298
+That positive banks ought to fall perhaps eligible, white proceedings. Voluntary, political bodies suggest united, unlikely women; soviet, long comm	Home                                              	wallpaper                                         	5.69	NULL	NULL
+Later recent years could take further; opening intervals weaken; personal years say often. Main pairs generalize articles; functions know quite other varieties. Pounds include to the hands. Claims h	Home                                              	wallpaper                                         	1.19	7033.67	1.71676571645954473
+Long potential cards make previous subjects. Continued, firm rounds might support. Royal, powerful vessels exist employees	Home                                              	wallpaper                                         	1.91	7286.37	1.77844428490949006
+Societies could make now below a lev	Home                                              	wallpaper                                         	6.61	5369.24	1.31051458988596934
+Boxes would not let further level groups. Different priests get chapters. Languages may stop still legs. Blocks must make good, important securities. Complete diffe	Home                                              	wallpaper                                         	4.83	1053.00	0.25701437506051615
+Protective, absolute fingers could hear usually daily, rapid schemes. Normal y	Home                                              	wallpaper                                         	6.16	437.24	0.10672076481620141
+Brown, natural periods might avoid in a changes; standard, military improvements should seem enough. Things commit easily from a hopes. General courts could close part	Home                                              	wallpaper                                         	2.54	1591.79	0.38852128402429154
+Times used to remember to the trains. Evidently chief tests will not look often apparent foreign holidays. Images will not meet earlier rows. Today happy months cannot like as against th	Home                                              	wallpaper                                         	5.03	5511.22	1.34516881682907673
+Proteins must remember above beneath available rights; good pl	Home                                              	wallpaper                                         	0.82	8210.81	2.00407996285910406
+No equal occasions care poor materials. Students could not operate briefly shares. Very determined effects go already heavy factors; full possibilities make certainly by the posi	Sports                                            	archery                                           	6.40	8728.20	2.57886262177629984
+Appointments will not go inc, temporary factors. Static, specific proport	Sports                                            	archery                                           	1.85	1021.30	0.30175665035404036
+Lives shall mean servants. Short inner balls shall take policies.	Sports                                            	archery                                           	0.82	20373.51	6.01962413938563079
+Eyes can go extremely hard numbers. Early, real others would discuss also. Good members 	Sports                                            	archery                                           	4.61	3215.40	0.95003263835149453
+Days can establish routine members. Associations replace both simple, crucial areas. Parties transmit variables. Statistical foreigners should not play 	Sports                                            	archery                                           	2.48	2613.03	0.77205442090925102
+Players will come just about senior matters; external hours may become natural principles. Smooth, national sentences can support public women. Protests tell too in a leaders. Labour studi	Sports                                            	archery                                           	1.36	426.80	0.12610372894458477
+Just silver assets move quite both statistical difficulties. Mainly national hours must prevent. Electronic	Sports                                            	archery                                           	9.78	10843.65	3.20390042260999677
+Entirely social buildings step all including the standards. Massive months read children; irish years come for a words.	Sports                                            	archery                                           	5.76	12915.10	3.81593783901641692
+Religious, subsequent views cannot meet around important min	Sports                                            	archery                                           	5.76	23175.78	6.84759203186346949
+Shares take. Consequences warn liberal, fresh workshops; illustrations ought to measure sports. White, universal organizations assist young provisions	Sports                                            	archery                                           	5.83	3736.19	1.10390696121243713
+Long, immediate cars	Sports                                            	archery                                           	0.47	7961.21	2.35224523877909490
+Holy days like new years. Excellent, standard standards regain more simply friendly others. Easily previous texts can	Sports                                            	archery                                           	1.24	2736.34	0.80848799826669420
+Low days go photographs; attacks may not tear probably similar, mathematical police. Likely, small name	Sports                                            	archery                                           	2.59	11492.70	3.39567086607645118
+Now public weapons used to specialise always limited	Sports                                            	archery                                           	6.16	609.03	0.17994600290328131
+Materials go furt	Sports                                            	archery                                           	3.67	48.41	0.01430337750282884
+Previously white patients should set sometimes level theoretical studies. Federal, european trends keep. Social, other hills can leave opportunities. Organisers lower experiences. Recent criteri	Sports                                            	archery                                           	2.18	4063.94	1.20074505203152723
+Scientific, elegant blues must eliminate. Basically local musicians might slow never now spiritual bedrooms. Wrong studies ought to impose relations. S	Sports                                            	archery                                           	1.70	4653.68	1.37499156821657742
+Constant, olympic languages could not bow other	Sports                                            	archery                                           	2.01	7616.46	2.25038427215855694
+Strong, essential rates could	Sports                                            	archery                                           	8.43	4002.55	1.18260656112265174
+Critical, secondary cars will extend social parts; together serious voices see personally a	Sports                                            	archery                                           	42.19	29.70	0.00877525948841183
+Women aim entirely reasonable, whole surfaces. Young drawings meet then sure, executive projects. Public, new offers used to sweep too light, old ar	Sports                                            	archery                                           	65.59	3949.47	1.16692337009083694
+Marginal, bright boats re-open subsequent figures. Most anxious positions produce nearly together with a causes. Invariably necessary hands must not le	Sports                                            	archery                                           	8.66	312.64	0.09237364062145029
+So blue parents will get at a conferences. Toxic methods 	Sports                                            	archery                                           	1.14	2037.09	0.60188529802184673
+Differences give financial, current reasons. Working, legal memories cannot launch into a activities; small, difficult parties coul	Sports                                            	archery                                           	1.62	7284.54	2.15231409945169992
+Competitive holidays should not keep all democratic, o	Sports                                            	archery                                           	61.08	8753.34	2.58629056869679390
+Crude, silly estates walk. Specific eyes mus	Sports                                            	archery                                           	3.16	11104.29	3.28090997254466541
+Normally eastern men protect also only explicit quantities. Royal, modest miles build by a opportunities. Shoulders judge more slightl	Sports                                            	archery                                           	5.58	12487.62	3.68963319503977423
+Nowhere other groups say home chief members. Contemporary letters practi	Sports                                            	archery                                           	8.43	2359.96	0.69728152802263887
+Current children take additional workers; far waiting arguments like bad, little days. Comp	Sports                                            	archery                                           	2.50	7478.91	2.20974329765919510
+Aware families give men. All social winners pose confident, new increases; most glad years wonder in genera	Sports                                            	archery                                           	1.55	2973.81	0.87865166394727186
+Welcome, united enemies fall. Nationally late profits complete mili	Sports                                            	archery                                           	7.03	11118.64	3.28514987064765227
+French photographs shall not advise clearly for an demands. Important, statutory cases rate well times. Other, local doctors assess terms. Normally white considerati	Sports                                            	archery                                           	7.09	408.72	0.12076175279810376
+Designs would throw especially linear, horizontal characters. Fundament	Sports                                            	archery                                           	3.73	8691.82	2.56811366756120145
+Changes set shortly. Mental, different jobs need more with a solicitors. Other, federal pieces thank then to a chang	Sports                                            	archery                                           	1.50	15462.27	4.56853304814429410
+Other consequences may shape quite. Personal, particular lawyers take brown, large men. Skills would gather as busy fears. Days will 	Sports                                            	archery                                           	3.96	12677.69	3.74579190113278554
+Political troops forget plates. Emotional lists must intervene virtually in the children. Ready, only 	Sports                                            	archery                                           	7.31	402.50	0.11892397118133873
+Months could not change curiously public contexts. Confident hotels would motivate in a studies. Workers sing fully again due positions. Irrelevant hands might create otherwise here strategic po	Sports                                            	archery                                           	0.40	1385.73	0.40943233437296029
+In short major reasons ought to sell already professional local institutions; corporate, able jobs will insure so su	Sports                                            	archery                                           	9.22	989.95	0.29249387644960565
+Privileges face mostly solicitors. Different soldiers suggest home. Deep stations make right parents. Safe, central things would tackle just. As famil	Sports                                            	archery                                           	37.12	16530.14	4.88404942356147718
+Goods go only. Accountants may unite. Almost agricultural muscles go just regional police. Real samples used to build auditors; following women can believe. Very concerned tonnes would fit there	Sports                                            	archery                                           	7.66	2295.32	0.67818278144583953
+Young countries should restore increasingly others. Combined, large activities match in a cases. Positions can 	Sports                                            	archery                                           	4.34	2791.69	0.82484189094964351
+Local, main troops cannot support never diffe	Sports                                            	archery                                           	3.65	463.60	0.13697677773830717
+Earlier controversial auditors s	Sports                                            	archery                                           	2.90	258.93	0.07650430772169947
+Old relationships in	Sports                                            	archery                                           	0.71	2104.62	0.62183793348489221
+Individual, grand relatives must provide much areas. Italian, respectable experts might revise nationally public standards. Comfortable forces record forward importan	Sports                                            	archery                                           	3.59	7433.10	2.19620812469070534
+Patient teachers shall stop already serious weeks	Sports                                            	archery                                           	2.66	11143.58	3.29251872491165869
+Schools will get financial, small years. Chronic, real societies 	Sports                                            	archery                                           	93.67	840.45	0.24832211572510841
+More leading requirements cross; elderly, able structures know obviously only houses. Enough light populations postpone blank payment	Sports                                            	archery                                           	2.76	5506.32	1.62691538135460637
+Real pupils could adopt fine years. Big neighbours must understand for a visitors. Duties would not give almost at last blue priests. Previous, small miles make finally	Sports                                            	archery                                           	7.47	1309.14	0.38680280157102555
+Domestic, chief devices can capture provincial lin	Sports                                            	archery                                           	3.78	18988.01	5.61025976156763126
+Strings ought to include even. Difficult, medical 	Sports                                            	archery                                           	64.26	5845.14	1.72702425071028634
+Big affa	Sports                                            	archery                                           	7.86	4365.75	1.28991882530417280
+There aware passengers allow all after a reservations. Simply environmental feet may close hardly labour members. Influential, old shareholders must	Sports                                            	archery                                           	2.48	5434.29	1.60563316112058941
+Bad publications improve by the years. Regular movements might give at least profits. Hard tests might not meet 	Sports                                            	archery                                           	9.45	12999.48	3.84086903078854452
+Sources make visual representatives. European regions will not run unacceptable loans. Right, natural firms get merely moral buildings. Virtually various sa	Sports                                            	athletic shoes                                    	2.23	3212.86	1.46013319558188889
+Distinguished powers upset very at a makers; animals shall see afterwards complete, working institutions. 	Sports                                            	athletic shoes                                    	4.30	909.15	0.41317707424639551
+Seriously social measures might give. Less technical travellers contradict entirely for a possibilities. Major, young police give only; more than important findings be	Sports                                            	athletic shoes                                    	35.35	15716.62	7.14265750276894310
+Priorities jump now important drawings. Both still movements will determine early massive, right patients. As huge goods might include at least chi	Sports                                            	athletic shoes                                    	1.75	11184.41	5.08292559090593238
+Degrees know as after a heads; new, complex ma	Sports                                            	athletic shoes                                    	1.41	3007.89	1.36698145504591167
+Real, comparative methods insta	Sports                                            	athletic shoes                                    	1.70	11493.02	5.22317810906375025
+Develop	Sports                                            	athletic shoes                                    	6.28	2742.72	1.24647090697582786
+However local things might not feel regional, responsible roots. Local, suitable nations set strong days. Patients might seem to a rooms. Sure othe	Sports                                            	athletic shoes                                    	2.00	303.48	0.13792111146928022
+Enormous, pure beaches lie highly financial periods. So united ships used to stay. Simply famous tons shall ensure separately extensive needs. In order educational statements must not pa	Sports                                            	athletic shoes                                    	3.52	3499.90	1.59058289848205428
+Grey problems must not acquire detailed times.	Sports                                            	athletic shoes                                    	16.36	1039.15	0.47225755563233998
+Current, political advantages will g	Sports                                            	athletic shoes                                    	3.15	125.13	0.05686723566017871
+Prices ought to go yesterday. Interests might rest here funds. Letters damage also rich agreements. Central, i	Sports                                            	athletic shoes                                    	1.72	128.63	0.05845786400518490
+Generally top practices can reduce most links. Earnings will tell as techniques. Flat direct measures would not go far material whole sentences. Simply defensive services evaluate nat	Sports                                            	athletic shoes                                    	6.06	794.64	0.36113625945020704
+Sentences will retire always from the marks. Modern activities may perform lon	Sports                                            	athletic shoes                                    	4.66	1180.16	0.53634169932643252
+Almost uncomfortable shares may believe wrongly constant levels. Red, other words used to resist more frien	Sports                                            	athletic shoes                                    	0.12	23738.70	10.78841402674246177
+Items used to thin	Sports                                            	athletic shoes                                    	4.26	23.25	0.01056631686325545
+Eyes may not give children. Good great beans shall cook. Visible,	Sports                                            	athletic shoes                                    	36.86	5204.23	2.36514164340902922
+Religious, alone results go all investigations. Banks ma	Sports                                            	athletic shoes                                    	1.04	3489.00	1.58562922735046355
+Homes cannot inform almost fresh hotels. Plans could kill today hi	Sports                                            	athletic shoes                                    	3.62	7136.25	3.24317757915727874
+Woods wear indeed from a numbers. Counties must not receive as greatly public windows. Above hostile groups breed of course usually true members. Sources introduce similarly words. Largel	Sports                                            	athletic shoes                                    	8.59	4113.45	1.86942004736164067
+Military, considerable sizes wash social consultants. Equal ways stand detailed writings. Tough, potential directions interpret then. Free wives would restore still. Better fresh men carry others. St	Sports                                            	athletic shoes                                    	8.09	4091.45	1.85942181205017314
+As usual religious variables may believe heavy, available sister	Sports                                            	athletic shoes                                    	6.51	590.67	0.26843898415566016
+Objectives shall get with a years. Huma	Sports                                            	athletic shoes                                    	6.42	6968.96	3.16715008891839681
+Existing theories wait supplies. Proper partners measure things. Areas must not thank a little. Hard white rules formulate then major, institutional differences.	Sports                                            	athletic shoes                                    	1.47	16050.71	7.29448979527840609
+Absolute companies might not raise in order powerful, recent waves. Major chil	Sports                                            	athletic shoes                                    	0.18	14627.31	6.64760397062645716
+NULL	Sports                                            	athletic shoes                                    	0.74	2201.76	1.00062338997167000
+Clean, large conditions would understand labour dates. Large clergy should give high jobs. Patients might escape. As national polic	Sports                                            	athletic shoes                                    	5.50	257.64	0.11708842480211334
+Particular, financial years shape then twice visual friends. Limited, future women ought to come casual scots. Relations concentrate charges. Shares shall establish in a plants. Then double	Sports                                            	athletic shoes                                    	4.22	164.92	0.07495040761669202
+Presumably yo	Sports                                            	athletic shoes                                    	4.44	163.80	0.07444140654629003
+In particular financial studies can gain less than huge, model consequences. Really other activities walk o	Sports                                            	athletic shoes                                    	47.58	1719.85	0.78161204547397384
+Now political women could	Sports                                            	athletic shoes                                    	8.57	57.80	0.02626809095467377
+Chronic lines shall take enough by the sales; international, welsh angles used to rule now front powers. Standard othe	Sports                                            	athletic shoes                                    	3.00	16781.46	7.62659027045362857
+Skills use rather than a principles. Easy employe	Sports                                            	athletic shoes                                    	6.29	9250.24	4.20391255488860762
+Accounts could think aspects. Industrial, large	Sports                                            	athletic shoes                                    	1.92	6322.30	2.87326559589505180
+Cells call no doubt pilots. Arms should pay rather good duties. Thus long s	Sports                                            	athletic shoes                                    	9.73	857.50	0.38970394452651834
+Friends cry easily sure varieties. Appropriate proposals provide recently between a books. New, considerable forces seem like the elections. Right big clothes fr	Sports                                            	athletic shoes                                    	9.64	2708.86	1.23108271390099647
+Words live only anxious countries. British, northern substances criticise most extra,	Sports                                            	athletic shoes                                    	3.18	2390.50	1.08639915963923277
+New rules continue wet cuts. German, following procedures shall see findings. As good charges cannot pay notably routine, short plates. Problems used to alleviate a	Sports                                            	athletic shoes                                    	30.73	3030.00	1.37702968153393653
+Supposedly parental instructions see. Broken, raw habits should not issue at all friendly beliefs. Certain constraints know 	Sports                                            	athletic shoes                                    	0.59	5983.42	2.71925641487913747
+Also other measurements pay at least around the artists. Perfect, good cul	Sports                                            	athletic shoes                                    	2.83	4854.06	2.20600154981736633
+Democratic forests use on a communities. Potatoes could not include still easy movies. Direct leads could sh	Sports                                            	athletic shoes                                    	3.61	1739.94	0.79074225217430942
+Levels may not go involved issues. Miles will beat good institutions. Tiny, c	Sports                                            	athletic shoes                                    	9.51	9805.35	4.45619075505900481
+Never national communities could turn so. National, whole styles buy far really high leaders. Indeed beautiful others liv	Sports                                            	athletic shoes                                    	5.39	306.50	0.13929359649839985
+More than hot women govern only full polic	Sports                                            	athletic shoes                                    	1.64	3354.48	1.52449456307325393
+Notably international minutes write too national, important visits. Human, clean patients	Sports                                            	athletic shoes                                    	1.21	6716.71	3.05251123176759302
+Major missiles may reply british dogs. Other, c	Sports                                            	baseball                                          	1.15	12361.94	4.21788969172030922
+Also other adults ought to uphold usually in a hills; carefully good signs would ensure with an benefits. Continuous, nuclear days shall feel just in the politicia	Sports                                            	baseball                                          	0.75	3265.70	1.11425572088612417
+Therefore unexp	Sports                                            	baseball                                          	3.99	3063.58	1.04529244615007878
+Often unnec	Sports                                            	baseball                                          	6.08	2524.58	0.86138583085852692
+Eggs shall not encourage as. Economic classes act other girls. Technical features wash even. Social goods can monitor probably 	Sports                                            	baseball                                          	2.18	3658.98	1.24844272211406762
+Managers shall put even. Physically additional guests help therefore high times; here specialist successes tend old plan	Sports                                            	baseball                                          	9.08	251.02	0.08564793797863701
+Dreams cannot need further at a securities. Modern societies devote once again for a businesses; ways used to say to a	Sports                                            	baseball                                          	1.06	4758.65	1.62364974927113782
+Fun activities cost white camps. Bare, solar databases go especially days. More subject sites deal certainly; partly equal occasions hear subs	Sports                                            	baseball                                          	6.89	1014.60	0.34618117230947778
+Most other delegates enhance natural, successful shows. American, similar times can derive easy, small departments. Artificial, other manager	Sports                                            	baseball                                          	4.91	1022.10	0.34874016973932312
+Fully silent bishops ought to seek only. Just new forms change immediately deeply raw cells. White corners shall lighten really reportedly glad games; teachers think at pre	Sports                                            	baseball                                          	3.06	14501.24	4.94781811860939439
+Winds owe urgently military managers. Internal conditions used to satisfy now as disable	Sports                                            	baseball                                          	7.10	7772.75	2.65205963637738361
+Organisations restore far. Far notes might not ask very places. Innocent requirements would not change to a children. Cer	Sports                                            	baseball                                          	1.20	8146.44	2.77956253631857102
+Also international soldiers use shortly decisive parties. Major, above advertisements expect about already loyal stairs. Lucky, small towns appear. Then english children corresp	Sports                                            	baseball                                          	1.92	3722.51	1.27011913634314422
+Guilty, oth	Sports                                            	baseball                                          	3.01	5530.46	1.88699105678166221
+Rather american exercises might remember times. Below western accidents give	Sports                                            	baseball                                          	0.71	7321.35	2.49804211106642533
+Later federal objectives	Sports                                            	baseball                                          	5.97	7447.00	2.54091384800776761
+Feet used to make import	Sports                                            	baseball                                          	2.92	798.30	0.27237968643273813
+Parents induce free deaths. Empty, red rec	Sports                                            	baseball                                          	39.45	15343.37	5.23515258602214870
+Symbols could enable too wrong problems. Real, old	Sports                                            	baseball                                          	0.29	5569.42	1.90028419543056548
+Elements shall arrange more. Coins would constitute however. Departments subscribe only in a children. And so on significant areas protect within	Sports                                            	baseball                                          	1.17	1171.52	0.39972222253498857
+Residents will happen here police. Owners might not match lines. Temporary, good symptoms used to achieve about in a issues. Troops can arrange. Even true comments shall not get ba	Sports                                            	baseball                                          	3.86	3886.24	1.32598375623495459
+Relevant numbers happen by the variables. Basic, italian fingers l	Sports                                            	baseball                                          	8.19	5295.33	1.80676478135772420
+Fascinating companies could tell partly about a 	Sports                                            	baseball                                          	8.54	2203.05	0.75167990504277057
+Rig	Sports                                            	baseball                                          	4.47	7838.81	2.67459928573946137
+Easily natural relatives used to walk thorough, real rocks. Front implications tell either. Members achieve in a words. So black ages help far 	Sports                                            	baseball                                          	90.17	13337.28	4.55067536548368992
+Teachers might not send unusual arrangements. Complex steps ought to hold all but statistical, recent pr	Sports                                            	baseball                                          	7.75	1162.44	0.39662412964658915
+Kids live so other goods. Colleagues ought to gain at first burning guidelines. Electronic, public figures give. Little leaves interfere. Stages could not determine yet environm	Sports                                            	baseball                                          	3.90	6580.60	2.24529846491203378
+Only solid days cannot cope ever suitable recordings. Inches go ever chro	Sports                                            	baseball                                          	9.36	11126.11	3.79622491922354013
+Cities ought to assess to the parties. Likely organs help domestic, passive stages. Therefore private obligati	Sports                                            	baseball                                          	1.03	7447.72	2.54115951176103277
+Hundreds would give seldom national, philosophical words. Obvious things li	Sports                                            	baseball                                          	2.21	83.50	0.02849017138561147
+Most local companies shall see already. Politicia	Sports                                            	baseball                                          	18.00	3997.41	1.36391492213840880
+Surprising applications could not explore. Tonight expensive layers meet then between a statements. Days de	Sports                                            	baseball                                          	0.95	4521.40	1.54270013057369686
+Offices obtain surprisingly according to the cups. Separate, only children work also social rates. Public conflicts force at least. Gradually australian storie	Sports                                            	baseball                                          	1.45	8302.97	2.83297051867772986
+Conscious, solar ambitions support outside countries; warm facilities rise occupations. Appropriate columns grow. Availabl	Sports                                            	baseball                                          	3.35	2187.71	0.74644590229959357
+Certain places kn	Sports                                            	baseball                                          	4.63	546.48	0.18645878872825095
+Single, wonderful departments will appea	Sports                                            	baseball                                          	3.19	5797.68	1.97816642920876516
+Statutory	Sports                                            	baseball                                          	4.72	3059.64	1.04394811950026670
+No scottish accidents will rely chan	Sports                                            	baseball                                          	4.35	25561.00	8.72140444057023607
+Properly common methods remember thus successful levels. Statistical families exist; trees will not go simply. Bottom, full things could see in the feet. Used, de	Sports                                            	baseball                                          	2.57	12848.83	4.38401639286929566
+Good effe	Sports                                            	baseball                                          	9.77	8394.54	2.86421417129785492
+Central standards ease written eyes. Simple, brief groups send in the ideas. Technical, possible islands try on a parties; activities must change adul	Sports                                            	baseball                                          	5.06	9693.92	3.30756218201684687
+Legal, other houses compete well new companies. Young, able layers would find orders. Rather good beaches die finally historical applications. Comments	Sports                                            	baseball                                          	89.48	2008.38	0.68525856775370489
+Clubs may take major changes. Procedures need. Lawyers shall not say pretty	Sports                                            	baseball                                          	1.61	8727.74	2.97790189711445061
+Clear practices might not own as. External	Sports                                            	baseball                                          	1.32	525.24	0.17921170800692895
+As simple views visit only japanese, direct differences. Hours assist locally too severe products. Else lesser dangers telephone 	Sports                                            	baseball                                          	7.20	316.92	0.10813299539554474
+Anxious, just years must come various schools; rarely surprising students ought to talk complex hundreds. Thin, other makers shall look actually american, ta	Sports                                            	baseball                                          	7.88	11407.21	3.89213614289414352
+Too particular pages used to give here by a markets; capital, different researchers gain specialist, small directors. Required patie	Sports                                            	baseball                                          	60.56	503.66	0.17184861940212062
+New friends would leave long motives. Dogs shall face occasionally increased schools. New, green parents decide also probably beautiful men. Real tanks shall 	Sports                                            	baseball                                          	0.54	928.53	0.31681411780457264
+Important, private results identify sh	Sports                                            	baseball                                          	1.25	4287.60	1.46292765069398475
+Other, significant materials could not mention economic, current races. Animals go straight living, young groups; masters may date. Top, able computers avoid less hours; questions recommend 	Sports                                            	baseball                                          	0.56	225.54	0.07695417071030911
+Only warm clouds ought to hold really 	Sports                                            	baseball                                          	4.99	1216.60	0.41510350308664564
+Books change slightly. Radical, distinguished characteristics imagine always as a ministers. Red strings deal late, sexual states. Peculiar, strong patterns live always. N	Sports                                            	baseball                                          	1.51	2123.42	0.72451017633095930
+Real, social cigarettes wou	Sports                                            	baseball                                          	0.29	5316.32	1.81392656216471802
+At least middle departments arrange international, environmental sites. More key kids might take up to the relations. Policie	Sports                                            	baseball                                          	4.87	2378.20	0.81144102502109211
+Young workers ac	Sports                                            	basketball                                        	7.78	1526.51	0.57071382054190651
+Inter	Sports                                            	basketball                                        	85.58	1184.67	0.44291065357015702
+Levels evaluate old arms. Attractive, dangerous men isolate very poor things; solid, sorry others shall leave now	Sports                                            	basketball                                        	1.44	153.89	0.05753460497683867
+Others ought to ensure still buildings; new patients keep notably in a drivers. Relative, good im	Sports                                            	basketball                                        	1.20	625.50	0.23385467160317491
+Favorite, pure features see green decisions. Imp	Sports                                            	basketball                                        	8.03	5094.18	1.90455282332128144
+Also federal cells shou	Sports                                            	basketball                                        	6.62	8298.39	3.10250562475630792
+Considerable ears cross during a members; very southern politicians allow numbers. Patients deprive earlier shares. Men used to press beautiful tactics. Eyes might develop on a co	Sports                                            	basketball                                        	4.97	937.69	0.35057264111204009
+Youn	Sports                                            	basketball                                        	3.28	1166.47	0.43610624905668334
+Always front rumours ought to improve. Hours use about a centuries. As uncomfortable links learn neither about real reasons. Dark days could deal much big, sole 	Sports                                            	basketball                                        	6.68	10473.18	3.91559083859462726
+About national assets raise much. Other inhabitants may like thick annual characteri	Sports                                            	basketball                                        	6.72	1181.36	0.44167314923281648
+Early types tell links. Local reasons succeed probably properties. Friends carry low fruits. Able, old tensions get. Recently other vegetables	Sports                                            	basketball                                        	3.00	11903.67	4.45040581730226223
+Cases should soften courses; complex letters use experimentally far whole parties. Great, liberal decisions confirm. Households know very reasonable users. New, short feature	Sports                                            	basketball                                        	2.58	5361.15	2.00436446469282357
+At all attractive answers would beat. Trousers might take of course fine constant lives. Ladies shall not challen	Sports                                            	basketball                                        	8.87	19675.51	7.35605104664266008
+Whole councils would see again white	Sports                                            	basketball                                        	4.23	4485.02	1.67680716104503839
+So early systems would place only to a m	Sports                                            	basketball                                        	2.69	249.12	0.09313809079101988
+Different plans may make so in a trials. Provincial, perfect items must wear together. Simple aspects must not prefer then in the sections; alone, good rights can put psycho	Sports                                            	basketball                                        	4.46	9055.60	3.38560250067100027
+S	Sports                                            	basketball                                        	1.06	458.00	0.17123171797642543
+Often final groups participate with the characters. Superior, in	Sports                                            	basketball                                        	62.36	9883.09	3.69497484632233713
+Decisions bring young farmers; easy other minerals credit preliminary, basic offices.	Sports                                            	basketball                                        	0.22	9644.13	3.60563525827070695
+Properly large others say briefly other police. Results used to prefer worried, old opportunities. Very big contents create forces. Possible, famous clu	Sports                                            	basketball                                        	4.35	9926.05	3.71103623192117389
+Succ	Sports                                            	basketball                                        	9.92	8445.05	3.15733716134675017
+Similar, new events may need sometimes combined prisons. Communications pay from a relat	Sports                                            	basketball                                        	20.67	3976.01	1.48650441701189361
+Charming, general guns would look superficially; big heads can set essentially straight voluntary refuge	Sports                                            	basketball                                        	0.21	5246.26	1.96141072653057135
+Authorities might destroy however to the profits. S	Sports                                            	basketball                                        	2.28	2179.53	0.81485734995886139
+Favourably major feelings used to turn new, necessary years. Labour products go pr	Sports                                            	basketball                                        	7.28	256.36	0.09584489786121490
+Different organizations shall split; emotional, com	Sports                                            	basketball                                        	2.22	12749.88	4.76677697902460058
+Smooth years help more british, curious arms. Inter alia acute members must improve also in a years. Now regional	Sports                                            	basketball                                        	3.91	2159.38	0.80732390210465840
+Women may not represent very common muscles. More late stones smile again on the surveys. Topics must not find as variations. Economic boots	Sports                                            	basketball                                        	60.56	202.95	0.07587658769282869
+Heavy paintings s	Sports                                            	basketball                                        	4.08	4622.30	1.72813181223238268
+Huge, helpful heads think low policies. Absolute tons restore generally. Tradit	Sports                                            	basketball                                        	5.01	24011.93	8.97730136644032550
+White interests might	Sports                                            	basketball                                        	53.99	3630.36	1.35727681151287298
+Outstanding friends must reduce figures. Travellers 	Sports                                            	basketball                                        	0.95	3994.52	1.49342472072312426
+Redundant, new writers draw sharp	Sports                                            	basketball                                        	4.80	9195.80	3.43801884752753924
+Clear members work national, personal operations. He	Sports                                            	basketball                                        	4.17	4072.64	1.52263131855788049
+Times remove other effects. Almost english conservatives can measure however new, normal r	Sports                                            	basketball                                        	7.65	1107.60	0.41409661753425504
+Now due eyes keep about. Then annual progr	Sports                                            	basketball                                        	0.83	3016.20	1.12766180733732398
+Addresses retain once more applicable events. Following blocks follow for a develo	Sports                                            	basketball                                        	70.89	268.59	0.10041730814691726
+Other, g	Sports                                            	basketball                                        	0.70	15012.84	5.61282616791528113
+Political aspects ought to say months. Of course 	Sports                                            	basketball                                        	3.77	123.24	0.04607553913409317
+Fortunately favorite decisio	Sports                                            	basketball                                        	2.86	9079.28	3.39445570390611328
+Ancient, similar ways equip immediately. Never european leader	Sports                                            	basketball                                        	0.67	5371.94	2.00839850451152582
+No doubt established kinds ensure both comparative buildings. Threats attract almost traditional students; questions must not fight widely clean, minor relations. National, famous assets go commer	Sports                                            	basketball                                        	9.10	1401.61	0.52401765989724377
+Only social changes could achieve again soon go	Sports                                            	basketball                                        	9.05	4303.38	1.60889770852705168
+Early favorite contexts will save quite as empty pages. Unusual languages suffer soon actual cars; corporate businesses ought 	Sports                                            	basketball                                        	54.80	7564.49	2.82812362077617992
+Recently free woods know 	Sports                                            	basketball                                        	2.84	3637.05	1.35977799097414435
+Confidential members cannot modify either dirty organisations. Men might think increasingly failures. Internationa	Sports                                            	basketball                                        	1.70	6383.10	2.38643925549196761
+Old, poor pp. form seconds; bags know much; 	Sports                                            	basketball                                        	9.50	5416.98	2.02523753634047386
+Comparatively unable miles show already; interesting drugs will not run parts. Yet political priests will run strangely left, d	Sports                                            	basketball                                        	4.52	1863.76	0.69680093165009314
+However comprehensive times ought to level even for a blacks. New employers see; far presidenti	Sports                                            	basketball                                        	4.48	4373.10	1.63496381197097391
+Areas expect. Organic, democratic resources would last previously. Cheap, residential fields cannot achieve seriously about 	Sports                                            	basketball                                        	0.77	2524.50	0.94383072495957642
+Automatically competitive deaths jump wooden friends. Average, legal events know. Losses ought to cross. Conventional toys st	Sports                                            	camping                                           	4.38	8168.10	2.37504813353538829
+Only far tests take to a others. Appropriate comparisons will say fully musical personnel. Beautiful, administrative aspects get standards. Huge, sin	Sports                                            	camping                                           	1.74	11263.88	3.27521175920551774
+Cells give only serious walls; arrangemen	Sports                                            	camping                                           	0.18	151.45	0.04403729628970441
+Sorry eyes could shake obviously across a commentators; more other numbers may control schools. Children maintain. Powerful elements gather very then active opportun	Sports                                            	camping                                           	3.69	5210.19	1.51497313143383954
+A bit important 	Sports                                            	camping                                           	3.97	2060.79	0.59921835471020101
+Straightforward deal	Sports                                            	camping                                           	4.48	14808.62	4.30592001704617007
+Whole services live since the wheels. 	Sports                                            	camping                                           	2.26	8417.24	2.44749086709509087
+So-called, classical travellers contain capital, new paintings. Japanese stories 	Sports                                            	camping                                           	6.17	18270.48	5.31252915889810863
+Financial, massive ideas might boil also leading companies. Even long 	Sports                                            	camping                                           	9.92	4367.79	1.27002748340183563
+Groups should display of course possibly productive areas. Gro	Sports                                            	camping                                           	2.04	12234.96	3.55757384359644646
+However general jobs tell basic results. Issues lose critical husbands. Back,	Sports                                            	camping                                           	21.20	4822.68	1.40229638871199501
+Equal, different facts emphasise necessary inhabitants. Complex, active moves might put in a reports. Commercial groups can restrict curiously to a players; identical purposes cou	Sports                                            	camping                                           	8.94	13999.26	4.07058144903669396
+Always opposite skills take well in the prices. Colonial, weak issues shall deny more just respective funds; mental, creative patients would not play even in	Sports                                            	camping                                           	16.73	5674.31	1.64992585480113970
+Procedures find groups. Possible	Sports                                            	camping                                           	4.18	5862.76	1.70472168501437704
+Wild changes shall delay soon representatives; other countries die also legal, superb boys. Never video-taped sounds think substantially previous approa	Sports                                            	camping                                           	75.50	1678.45	0.48804489902577986
+Dear officers communicate much long interested relationships. Casualties position normal, good issues. Aspirations remind now quick words. Financial, l	Sports                                            	camping                                           	3.38	1526.49	0.44385930943064297
+Exceptions say richly normal losses; british, old individuals used to win. Childr	Sports                                            	camping                                           	4.27	4862.61	1.41390688221379690
+Then bad merch	Sports                                            	camping                                           	0.84	409.38	0.11903590858421386
+More fine pressures free there into the records; rights turn seconds; great areas ought to drain allegedly especially gothic dealers; programs speak even european, o	Sports                                            	camping                                           	2.25	4430.31	1.28820649802073507
+National systems must believe old issues. Long police would make able towns. Slow years earn exactly nearer the terms. Social, old comparisons shall survive wildly previous children	Sports                                            	camping                                           	2.12	4781.18	1.39022938444641077
+Well main goods share probably traditional times. Enorm	Sports                                            	camping                                           	5.17	3862.11	1.12299030949772389
+Terms reduce standards. Free things put	Sports                                            	camping                                           	2.60	1759.84	0.51171076594568109
+Players must argue away significantly national sides. Elections might	Sports                                            	camping                                           	3.53	14678.84	4.26818373238141050
+Labour, bright taxes could not shock still with a reasons. Dominant weapons will cause home; women say therefore bloody, complete areas; dem	Sports                                            	camping                                           	30.04	3575.90	1.03976868803138980
+Unable school	Sports                                            	camping                                           	2.63	9178.29	2.66878227905467845
+Still royal companies reach years. Complex, british plants must tell however such as a detectives. Ite	Sports                                            	camping                                           	6.35	8374.50	2.43506330655747472
+Just capitalist exceptions communicate 	Sports                                            	camping                                           	7.91	397.64	0.11562225484739558
+Available tests forgive also policies. Almost local rights used to argue there new only men. Chi	Sports                                            	camping                                           	2.78	316.16	0.09193021852065332
+Never top observations spend appropriate, common states. Homes make. There available hospitals will appreciate away upon a years. Roots hang 	Sports                                            	camping                                           	2.07	4784.91	1.39131396097437774
+Residents will l	Sports                                            	camping                                           	7.50	7103.96	2.06562688247083863
+Bold campaigns get with a numbers. Public, medical emotions recognize sources. Very single countries shall fit enough along with	Sports                                            	camping                                           	4.72	5615.05	1.63269475425225965
+Democrats may say again. There private services can think about fa	Sports                                            	camping                                           	1.65	18235.67	5.30240741387437400
+Different, ltd. students may not try scottish sheets. Almost likely schools may not order. Partly effective c	Sports                                            	camping                                           	3.91	11958.94	3.47731518052689077
+Certain, official generations might allow polish letters. Months provide equally product	Sports                                            	camping                                           	8.26	3715.04	1.08022659100761608
+Central, clear fingers must 	Sports                                            	camping                                           	5.58	104.95	0.03051643608850761
+Always clinical doors	Sports                                            	camping                                           	33.45	2954.82	0.85917651913334019
+Available implications try only; magistrates must reduce quite black, ugly girls. Animals read. Chief pupils will manipulate easy more real seconds. Men might throw only british policies. Aspects ex	Sports                                            	camping                                           	6.42	12904.54	3.75226841506993789
+Affectionately sad chains answer sideways small, concerned documents. Interested minutes notice as a yards. Difficult c	Sports                                            	camping                                           	0.18	7683.32	2.23408807744213704
+Crucial sources make to a police. Great farmers make recent limitations. Yet indian colleagues should get. Mea	Sports                                            	camping                                           	7.95	1656.32	0.48161013265475868
+Good, white statements de	Sports                                            	camping                                           	8.79	4572.10	1.32943494464283601
+Conventional workers shall not take numbers. French, premier things could remember as to a gardens. Red districts ought to implement flowers. Fiscal, curious terms study much explicit words. Third	Sports                                            	camping                                           	3.61	5559.40	1.61651333768889187
+Fresh, electoral doors get at a teachers; children become more ministers; comfortable places shall not lift much safe, genuine procedures; official, extra beliefs break. Openly new days find ther	Sports                                            	camping                                           	1.27	4702.53	1.36736023057922522
+Much basic birds can light apparently. Normal, close teeth cannot expect as civil ends. Long principal conditions could not cover less more new officers. Efficient words get to a years. Real, able 	Sports                                            	camping                                           	1.68	3665.26	1.06575200131265745
+Far specific clothes learn indeed times. Gastric, steady criteria imagine again in n	Sports                                            	camping                                           	50.85	6713.37	1.95205456449265676
+Grounds will take then by the boards. Historical candidates feel suitable numbers. Normally inevitable savings return systems. Psychological rooms would turn almost	Sports                                            	camping                                           	2.39	16931.42	4.92316909306983803
+Accounts listen firmly incredible trends. Votes must not exert away natural fears. Able terms reflect well golden others. British feet could not re	Sports                                            	camping                                           	8.64	12203.84	3.54852504425319390
+Labour patients shall	Sports                                            	camping                                           	2.75	7756.62	2.25540160545821715
+Powerful populations can produce honest lines; soviet, working-class feet w	Sports                                            	camping                                           	2.14	2940.02	0.85487310556392702
+Minutes can compete much mathematical areas; pages put for example more good passengers. Differences undertake to a parts. About conscious situations know light studies; mad, l	Sports                                            	camping                                           	1.46	2184.90	0.63530596674397594
+Visual, surprising parties earn resources. Particular, just situations can lose currently to a others. Social actors want loudly prime years. Fresh, other responsibilities obtain offices. Afraid t	Sports                                            	camping                                           	9.02	6215.95	1.80741916059417696
+Great explanations would not fill; sure, political powers let eventually horses. Continually public examples ask yet wrong, dependent officials. Early, g	Sports                                            	camping                                           	1.82	3966.35	1.15330029804337451
+Trustees could respond further precise surveys. Conditions would weigh. White areas secure particularly living costs. Strong, bare provisions can keep so useful, physical feet. Demanding, supreme	Sports                                            	camping                                           	4.48	9027.65	2.62498050742654327
+Just available years undertake social units. Alone long-term years communicate very huge workers. Relevant, false farmers start hardly bottom windows. Associations shall 	Sports                                            	camping                                           	7.57	5611.89	1.63177591730095251
+Steps would make repeatedly short pairs. As good stages protect skills. Plants could not sweep observations. C	Sports                                            	fishing                                           	8.71	4424.59	1.05964726402462346
+Christ	Sports                                            	fishing                                           	9.05	1582.84	0.37907514038334286
+Almost personal matters may deal; major, australian offences happen prime, usual hours. Functions might visit at the followers. Championships shall smile observations; compani	Sports                                            	fishing                                           	2.61	1554.46	0.37227840004061759
+Accidentally wrong communities look more goods. Rural matters recognize. Large, new days go hap	Sports                                            	fishing                                           	1.32	4303.95	1.03075513030558269
+Problems ought to remove rapidly then new authorities. Half way exotic months bar systems. Front, new models cause too; difficult, full others comprehend eve	Sports                                            	fishing                                           	2.89	2105.84	0.50432867101214193
+Delightful, married guns should go much tremendous, clear networks. Again just hours shall know there. Large, whole years cannot want 	Sports                                            	fishing                                           	9.33	2187.51	0.52388786001109799
+Very modern weeks must prevent hotly to a situations. Points look strongly regulations. Times take good groups. As af	Sports                                            	fishing                                           	68.83	2026.90	0.48542329107363830
+Members support general, mysterious programmes. Front times want with the services. Now new details should impose never cheap live activiti	Sports                                            	fishing                                           	4.96	11202.69	2.68293781078382606
+Tests shall see famous, good words; sexual, significant theo	Sports                                            	fishing                                           	8.63	11684.99	2.79844407813042221
+Personal, lacking artists cut pieces. Prices make quickly for a rooms. High, overall types ought to use together supposed women; reductions shall give prices. Lengthy, fundamental meas	Sports                                            	fishing                                           	9.23	13101.80	3.13775661107533389
+Other offices shall embark blindly resources. Spectacular copies may look also old, other offices. Properties fill better important others. Very wrong supplies will not own both aspects. Certainly	Sports                                            	fishing                                           	7.25	386.95	0.09267084833042791
+Sheets identify in a persons. Successful studies cannot solve for instance impressive governments; public buildings can move to a women. Substances sweep even on a tales; however great spac	Sports                                            	fishing                                           	4.50	5339.33	1.27871880247087137
+Inherent, public girls run. Opposite, similar players might adjust though central ties. Entirely small generations achieve rats. At all western boxes prosecute almost suspicious, ordinary v	Sports                                            	fishing                                           	0.46	2861.92	0.68540264699268189
+Difficult skills can tell specifically significant applicants. Irish women find si	Sports                                            	fishing                                           	8.65	0.00	0.00000000000000000
+Usually english commentators will indicate still dangerous, spiritu	Sports                                            	fishing                                           	9.90	13087.32	3.13428878865945433
+Early, associated parents continue stories. Alive, key costs will not supply. For example excellent wi	Sports                                            	fishing                                           	0.65	9375.15	2.24525934545809862
+Just left grounds would not shoot other, accessible readers. Long, true winners shall vary; male conditions must hear never local, clean studies. Major, generous pp. must not get always gre	Sports                                            	fishing                                           	3.62	8.19	0.00196142718135729
+Groups deserve also only members. Inevitable, rare dreams worry much old enquiries. Please clear nerves turn necessar	Sports                                            	fishing                                           	2.58	3603.80	0.86307585789687587
+Foreign advances expand never new, colonial players. Colours confess lines. Urgent, massive items sit then men. Different countries cut however. Effectively old ideas suggest only actually particul	Sports                                            	fishing                                           	4.19	20.28	0.00485686730621806
+Sole, public skills require long opportunities. Parents destroy how	Sports                                            	fishing                                           	4.84	1396.88	0.33453948731311060
+Courses try military parents. Fast, w	Sports                                            	fishing                                           	1.64	6454.18	1.54571478453878082
+New parties strengthen please at all current things. Similar teams must lead most real firms. Simply tiny planes will set moving advantages. Concerned, average memories use	Sports                                            	fishing                                           	2.13	5552.34	1.32973267352104439
+International, new heads succeed altogether. Inc men see about accord	Sports                                            	fishing                                           	4.11	4917.54	1.17770410517847910
+Illegal campaigns ought to become here western, certain abilities. Indirect teachers would not tend no longer long, main agreements. Twice sweet patients ought to enjoy	Sports                                            	fishing                                           	0.33	2469.18	0.59134514867689882
+Common, preliminary children will not maintain early further international 	Sports                                            	fishing                                           	3.67	4265.38	1.02151798178483168
+Northern, de	Sports                                            	fishing                                           	15.22	1489.04	0.35661093163959266
+Unable occasions command more effective, other birds. Proper songs know in a ports. Later wealthy details look now hours. Aware, black issues 	Sports                                            	fishing                                           	0.59	4257.58	1.01964995589782473
+Points can appoint even pregnant ideas. Other, basic bodies shall frighten too modern laws; features accompa	Sports                                            	fishing                                           	1.97	15202.78	3.64092135826557149
+Home available features need with a questions. Hard waters can operate still more content bands. Organic, large ideas contribute points. Difficult, right produc	Sports                                            	fishing                                           	2.47	7589.73	1.81766821992220870
+Collective, full signals will assume only services. Political villages think children. So old	Sports                                            	fishing                                           	2.56	2552.33	0.61125878361338953
+Industrial, slight needs would disturb too for a folk. Now known buildings ought to suggest so. Papers create colours. Good levels tell into a r	Sports                                            	fishing                                           	2.72	5261.10	1.25998346078618504
+Norma	Sports                                            	fishing                                           	1.01	8662.39	2.07456009786539724
+Onwards horizontal sports find. Normal, powerful eyes come american, commercial situations. Major, enormo	Sports                                            	fishing                                           	1.89	13071.78	3.13056710631534049
+Shoes give more now annual ch	Sports                                            	fishing                                           	1.18	6235.99	1.49346035270723652
+As modern women may find only with a bones. However simple minutes end changes. Catholic hands provide hard able rights. Weeks used to affect then tiny c	Sports                                            	fishing                                           	2.55	3728.50	0.89294032303915358
+Strong, southern weeks use to a exceptions. Shoulders write natural, particular courses. Cold, labour things will hang. New authorities may bu	Sports                                            	fishing                                           	1.08	5888.16	1.41015837267164344
+Automatically private stands go always easy terms. Well distinctive depar	Sports                                            	fishing                                           	1.17	5365.88	1.28507727520164501
+Internatio	Sports                                            	fishing                                           	1.86	8437.51	2.02070347459999698
+Apparent,	Sports                                            	fishing                                           	7.13	2649.10	0.63443427913719237
+Special, easy things invest here hot 	Sports                                            	fishing                                           	4.61	8905.67	2.13282334630014721
+Leaves could not help accounts; maximum, supreme expenses may not build in a officers; r	Sports                                            	fishing                                           	0.44	13341.40	3.19513853447621392
+Still original workers solve merely villages. Only long years punish already. Scottish features should not take from th	Sports                                            	fishing                                           	4.81	3.50	0.00083821674416978
+Settlements must make significa	Sports                                            	fishing                                           	7.42	7154.29	1.71338447732755427
+Shortly new terms would recover yet satisfactory, previou	Sports                                            	fishing                                           	2.86	3393.96	0.81282117172642234
+Public, certain lives could not choose indeed in a tools. Then bad things gain women.	Sports                                            	fishing                                           	2.62	392.55	0.09401199512109957
+Circumstances cannot take lines. Modern goods would make corresponding tools. Subsequently toxic practices see annually alm	Sports                                            	fishing                                           	3.56	12846.92	3.07671527285990692
+Also normal groups must not keep possibly others. Rates will not depend centuries. Fields could indicate already in a months; important arti	Sports                                            	fishing                                           	64.57	16106.48	3.85734892161020958
+Crops shall argue already for the responses. Easy committees like as with a figures. Easy current groups should not meet nevertheless; evident, international forces sen	Sports                                            	fishing                                           	6.00	1274.25	0.30517076750238473
+Elements take further vital, typical 	Sports                                            	fishing                                           	1.73	6796.42	1.62767801268868558
+Good, silent examples close so literary, financial years. Often foreign interests discourage best suddenly whi	Sports                                            	fishing                                           	4.19	4776.06	1.14382098947415311
+Projects support indeed in a departments. Populations involve even with a terms; fine, classical miles visit continuously crucial, great days. Steady, sc	Sports                                            	fishing                                           	0.68	7528.93	1.80310719762348789
+Directions use none the less. Military, new recordings pass yellow tasks. Frequently wor	Sports                                            	fishing                                           	1.49	1880.44	0.45034751268760788
+Poor networks explain personally to a funds. Already federal words accelerate companie	Sports                                            	fishing                                           	2.01	7024.79	1.68237045779327228
+Sectors might not know properly. Large, electric workers used to drop even as ca	Sports                                            	fishing                                           	6.89	1774.46	0.42496630967414683
+Old others will cut other activities. Sharp passages avoid allegedly orthodox, additional firms. High officers must form.	Sports                                            	fishing                                           	0.25	2612.13	0.62558031541377612
+Very acids should depend much a little christian tons. Warm rules defeat at large details. Banks should not seek then. Times can back stiffly ordinary, chemical	Sports                                            	fishing                                           	6.07	10778.84	2.58142976306486528
+Factors might assist now absent, voluntary demands; political companies might know no longer concerned things; autonomous, possible events can dry at 	Sports                                            	fishing                                           	6.68	6637.53	1.58962536740836076
+Pale, other animals assist in a words. Is	Sports                                            	fishing                                           	3.40	1226.34	0.29369677772719206
+Necessary women know little international troops. Immediate, possible drugs can try effectively too gentle spots. Northern, german ideas tell to a areas. False appropriat	Sports                                            	fishing                                           	2.18	505.79	0.12113189915246708
+Western, social things will go in order. Warm, male cards used to describe. High, briti	Sports                                            	fishing                                           	0.51	2346.30	0.56191655624158939
+Different, common buildings could not comply even. Impossible transactions build always qualities. Police move tiles. Options must use just different stages; words	Sports                                            	fishing                                           	8.87	4167.10	0.99798085560854416
+Members like very. Then interested principles could remember yet important, new agents. Necessarily due assets generate across. Areas give anyway as social projects. Main, 	Sports                                            	fishing                                           	1.79	7991.56	1.91390268686784986
+Blind sessions hurt traditionally public, various clothes. High, southern schools might not tal	Sports                                            	fishing                                           	1.43	1122.60	0.26885203342999968
+Practical roads mean either dishes. Necessary issues determine simply fund	Sports                                            	fishing                                           	3.40	4810.52	1.15207383204675046
+Just formal teams ask still, certain interests. Well l	Sports                                            	fishing                                           	9.79	2218.77	0.53137433298902583
+Books must work major, able forces. Clearly future teachers would measure certain, direct measures. Hard tears go main nurses. Cruel patients used to leave much only days. Yet social defence	Sports                                            	fishing                                           	8.56	1810.80	0.43366939438361253
+Comprehensive, able acts must not resign. British, red forces convict perhaps; years want as well problems	Sports                                            	fishing                                           	54.91	119.66	0.02865743303067322
+New companies must benefit always to a companies; adults might get yet international, comfortable indicators. Dual bones shall find ever like parents. Wars need new, heavy purposes	Sports                                            	fishing                                           	3.43	7734.29	1.85228896636140409
+Backs think now back, british wines. Very fine shows get often serious, fatal prisoners. Good terms ought to come far easy, obvious shoulders. Machines play more ac	Sports                                            	fishing                                           	2.94	7583.99	1.81629354446177025
+Tiny values allow equations. 	Sports                                            	fishing                                           	4.39	7729.84	1.85122323364381680
+Ill, simple objects shall bear solid trees. Ears should use there minimum, inappropriate personnel. Available practices should not apply increasingly pr	Sports                                            	fishing                                           	7.87	15557.69	3.72591893102937088
+Sure reliable suppliers show upright other women. Maybe	Sports                                            	fishing                                           	1.11	12392.70	2.96793389870653577
+Much common times achieve existing, continuing positions. Los	Sports                                            	fishing                                           	8.20	9965.46	2.38663298152977430
+Good, whole facilities maintain for a points. More worthwhile directors battle annual hours. Yes	Sports                                            	fishing                                           	8.90	603.00	0.14441277049553697
+Rules offer. Important, italian goo	Sports                                            	fishing                                           	4.06	3150.39	0.75448847104715544
+Vital, similar activities change thickly. Seats would sit essentially brilliant words. Hig	Sports                                            	fishing                                           	68.38	6302.32	1.50934575746174558
+Even useless times make old, old studies. Early public employees must open together warm consequences. Sufficient, evident men would operate stars. Various, other sections control l	Sports                                            	fishing                                           	89.62	2679.48	0.64171000047658609
+A	Sports                                            	fitness                                           	7.12	10468.61	4.22441966945963305
+Fast bizarre situations fulfil all as political plans. Thus labour conventions run more part-time experiments. Early considerable own	Sports                                            	fitness                                           	0.81	5713.17	2.30544721056249987
+Other, cultural differences might take. Musical branches take only new defences. 	Sports                                            	fitness                                           	3.76	18567.33	7.49251276543379958
+Increased machines claim 	Sports                                            	fitness                                           	1.76	2327.22	0.93910786084875139
+New parties survive	Sports                                            	fitness                                           	1.06	5055.94	2.04023384036732070
+Abruptly real years cope together; significant accounts provide at a others. Twice competent languages cannot impose most protests. Identical leaders 	Sports                                            	fitness                                           	3.76	11311.78	4.56466578930728034
+Clinical, real figures figure effects. Full, pleased bacteria used to fit immediately more main things. Windows will not present perhaps	Sports                                            	fitness                                           	4.25	1715.39	0.69221484579083182
+Concerned clothes comment less small probl	Sports                                            	fitness                                           	0.73	1855.00	0.74855195549816254
+Large, working matters oppose etc far remote aspects; today amer	Sports                                            	fitness                                           	3.52	11563.15	4.66610164108818229
+Physical questions confirm much to the marks. Irish, pleased eyes would know in an subsi	Sports                                            	fitness                                           	2.86	8639.15	3.48617392255630775
+Little, national services will buy young molecules. In part video-taped activities join now	Sports                                            	fitness                                           	5.91	408.38	0.16479441918401058
+Intelligent trends used to bother open. Bedrooms will not hit all senior, economic boys; objects would sum. Often blue times should deal in a	Sports                                            	fitness                                           	3.84	1925.10	0.77683955230701493
+Absolutely wild standards impose only so scottish schools. New, complex incomes can establish children. Certainly free groups will rest. Impressive teeth must go front s	Sports                                            	fitness                                           	4.00	2927.91	1.18150552885316716
+Policies think races. Loc	Sports                                            	fitness                                           	40.32	1793.89	0.72389211183212873
+Shares could release barely months. Aware writings used to use so very impossible authorities. 	Sports                                            	fitness                                           	6.66	3449.47	1.39197170562385268
+Boys might not fill in a problems. Military, young ways will encourage somehow inner, large matters. Ways will begin today whole firm	Sports                                            	fitness                                           	3.62	2731.00	1.10204603259594711
+Corporate heroes examine forth technical, formal shares; buildings may not emphasize abo	Sports                                            	fitness                                           	68.11	4428.60	1.78708204319092324
+Below old resources could cover lo	Sports                                            	fitness                                           	2.86	2908.84	1.17381017263141516
+Running children may continue common, small wives; great, subtle teams shall change bad, good lines; others may want; parties used to like near a sty	Sports                                            	fitness                                           	2.32	2591.76	1.04585822974766455
+Labour, dominant dreams come. Please various symptoms cannot persuade so owners. Primary colours would argue once small posts. Live, asia	Sports                                            	fitness                                           	48.03	4332.46	1.74828647176149287
+Deep, light measures could ask around experimental sections. Days attend social, wise cases. Children should find; as	Sports                                            	fitness                                           	3.91	12590.50	5.08067029417769025
+Times force also years. Emotional solutions ought to allow elderly differences. Too urban parents shall accommodate so. Traditional, effective im	Sports                                            	fitness                                           	3.60	8417.45	3.39671086674286159
+Principal eyes should pay frequently relevant areas. Light police m	Sports                                            	fitness                                           	3.17	451.78	0.18230771021830721
+Original hands know as. So prime things might not identify. Less than little journals let very hard things; nurses see; large bodies name once political, national c	Sports                                            	fitness                                           	6.83	1540.63	0.62169358447392677
+Methods develop warm males. Governments depend please with the hospitals. At random tory weaknesses enter approximately simply young me	Sports                                            	fitness                                           	6.01	24.98	0.01008023064600760
+Also new activities would not drop immediately fina	Sports                                            	fitness                                           	6.42	9171.55	3.70101438676505262
+Beings should affect close projects. In common labour metres might call directly 	Sports                                            	fitness                                           	2.85	837.90	0.33811950593633983
+Men could not escape so old victims. Tiny horses give together effective teeth; little, beneficial bones used to forget again days. Of course	Sports                                            	fitness                                           	71.90	2421.19	0.97702776772646693
+Regions see in the cop	Sports                                            	fitness                                           	1.90	8595.06	3.46838219440648889
+Asleep, fat topics pick into a rul	Sports                                            	fitness                                           	2.70	3452.62	1.39324283158601937
+Conscious, central results play above about the hands. Stages stay so available universities. Tomorrow professional birds decide; enthusiastically big views appear new window	Sports                                            	fitness                                           	9.62	412.47	0.16644486527456987
+Please positive sys	Sports                                            	fitness                                           	0.31	4494.44	1.81365059346046449
+Simply necessary girls could not take supreme hospitals. Issues ought to 	Sports                                            	fitness                                           	93.50	342.93	0.13838324641454710
+Overseas campaigns must finance just. Researchers believe sure, positive days. Workers appear from a values. Periods can lift ago related, extens	Sports                                            	fitness                                           	8.92	691.02	0.27884871821473869
+Regular, gold effects take gently for a terms. Good, strong difficulties attract articles. Ultimate farmers develop 	Sports                                            	fitness                                           	1.12	3313.24	1.33699853425052940
+Round prisoners go at all into a lives. Streets find again places. Kindly liable men offer plainly on a contents. Early accurate regions should no	Sports                                            	fitness                                           	4.49	3281.89	1.32434780443658472
+More 	Sports                                            	fitness                                           	0.82	1089.45	0.43962799348650845
+Solid, romantic feet would come so equations. Only economic feet will n	Sports                                            	fitness                                           	0.36	6592.06	2.66010749528906595
+Only subjects think for a goods. Windows wo	Sports                                            	fitness                                           	3.66	9334.78	3.76688292352837611
+Special miles must ease under across a conditions. Points might continue australian, australian places. Entirely 	Sports                                            	fitness                                           	3.17	0.00	0.00000000000000000
+Men mean also weapons. Individual proposals ought to mean farmers. Sometimes valuable eyes might take rights. Rough, different rewards cost real, alone ministers. Requirements may no	Sports                                            	fitness                                           	64.89	3913.00	1.57902091744706739
+Together working cases used to buy in a structures. Millions must 	Sports                                            	fitness                                           	1.88	3472.20	1.40114398915402693
+Sure, coming sessions could not pass very. Concerned children pick on a individuals. Easy pairs shall return. Reports consider subsequently rough sites. Vital, normal w	Sports                                            	fitness                                           	2.27	5967.84	2.40821471811329074
+Girls move ways. Other, human actors should participate serious families. New di	Sports                                            	fitness                                           	4.79	10717.00	4.32465299572712017
+Quick reasons could set only distant a	Sports                                            	fitness                                           	1.29	968.12	0.39066744968025936
+So close miles would seem american, emotional horses. Other, alive operations ought to want further red layers. Parameters might faint bad, significant stations. So prime newspapers wou	Sports                                            	fitness                                           	2.97	9281.14	3.74523746428690903
+Royal speeches take evil, front margins. For example hard events ought to go angles. Possible, foreign lakes shall not reconsider. Other honours hear momen	Sports                                            	fitness                                           	8.13	0.00	0.00000000000000000
+Points force into the symptoms. Local, strong negotiations get examples. For the time being fat result	Sports                                            	fitness                                           	5.61	19543.75	7.88652953114135530
+Subject, dead qualifications benefit more real nurses. Up to special writers give most responses; social circumstances de	Sports                                            	fitness                                           	2.69	12178.65	4.91447561877503891
+Just ready clothes try live skills. Girls investigate up 	Sports                                            	football                                          	1.80	3028.92	1.26615656780156976
+Mostly furious applications cut in a workers; successful, substantia	Sports                                            	football                                          	3.20	4690.04	1.96054202463322710
+Dynamic, technical problems cannot go important, general sources. Overall inevitable subjects may take. Recent ends would n	Sports                                            	football                                          	2.51	10300.92	4.30601584472305176
+Allowances might lay at best children. Academic sections burst hot times. Short-term, warm goods	Sports                                            	football                                          	4.96	652.80	0.27288505720219244
+Sophisticated, unfair questions may remove separate premises. Typical patterns intervene typically walls. Naked areas ought to return now military, necessary children; young met	Sports                                            	football                                          	33.19	7921.58	3.31139830182558766
+Only available cars could not allow during a films. Cuts might not grow also unfortunately poor names. Windows go at first so key effects. Leading, possible relationships used to rec	Sports                                            	football                                          	1.80	5455.78	2.28063853765713464
+Pupils talk tonight even expected rights. However federal costs may not borrow large decisions. Social, american soldiers repair legal, economi	Sports                                            	football                                          	11.06	1681.47	0.70289221374658476
+British components must go. Wrong, overseas jobs explain with a towns. Quite ideological habits may	Sports                                            	football                                          	0.63	8173.32	3.41663127409899441
+Girls would face criminal, special offenders. Healthy principles get very greek, ade	Sports                                            	football                                          	1.47	435.76	0.18215746404170861
+Delicate readers gain too able officers. Feet see as international appearances; just prominent samples halt just. Substantia	Sports                                            	football                                          	94.83	12471.06	5.21318309049015641
+Daily, level areas fetch known, other 	Sports                                            	football                                          	69.68	818.79	0.34227260414611390
+More reasonable opp	Sports                                            	football                                          	3.70	3418.34	1.42894287137950754
+Awful eyes get now like a gentlemen. Final countries may become french, turkish sciences. French lives repeat great, big standards. Large, able roads cl	Sports                                            	football                                          	6.18	5009.22	2.09396643112494858
+Thanks may add suddenly strong weeks. Times abandon as files. Systems feel cheap targets. Green, formal events understand french, rea	Sports                                            	football                                          	0.97	2280.64	0.95335872680393409
+Miserable officers introduce clearly. Much mathematical eyes could change so before prominent plans. Prices i	Sports                                            	football                                          	4.67	20055.07	8.38346955291662626
+Else social offenders will not support mines. Gently intelligent expressions speed days. Sometimes old houses offer really important, local month	Sports                                            	football                                          	2.19	15388.53	6.43275105592471583
+Critics can cover only str	Sports                                            	football                                          	1.79	10295.54	4.30376688392686948
+Sources negotiate never books.	Sports                                            	football                                          	12.71	1473.07	0.61577633457848288
+Young, previous metals keep here due, equal churches. Strong temperatures avoid. Established, average children could help also technical aspects. Feelings navigate now weekl	Sports                                            	football                                          	1.45	8988.48	3.75738645674136449
+White, vital departments should become aga	Sports                                            	football                                          	2.88	4166.35	1.74162784631488126
+Daily, marked years may not save players. Then hot families please universally always parental opportunities. Closely medic	Sports                                            	football                                          	3.21	1605.80	0.67126045474154508
+Popular, strong farms worry certainly followers. New documents will argue considerably under a men. Catholic, exist	Sports                                            	football                                          	0.10	1110.81	0.46434352081919024
+Clearly great options cannot believe. Responsible products ought to condemn at a systems. Dull types assure; real ser	Sports                                            	football                                          	3.03	8226.16	3.43871958050610814
+Succ	Sports                                            	football                                          	4.47	9246.93	3.86542435967320677
+Almost busy threats go together recent sides; still tired wines shall not admit on a	Sports                                            	football                                          	3.88	7510.88	3.13971648045159802
+Economic, crude hands put available payments; irish months pay main, tropical members. Neither soft syste	Sports                                            	football                                          	4.23	2877.00	1.20265059676885365
+International, profitable schools sit rather di	Sports                                            	football                                          	81.85	205.56	0.08592869540208744
+Young features may seem actually for the plans. Unduly	Sports                                            	football                                          	9.86	3012.65	1.25935534249415605
+Standards must pa	Sports                                            	football                                          	3.63	836.01	0.34947095078370849
+Very aspects use then. Popular, weste	Sports                                            	football                                          	6.30	1501.17	0.62752276550278069
+Models may register still digital, professional birds. There necessary things can fail never irish forces. All corporate readers identify more	Sports                                            	football                                          	68.59	9190.37	3.84178100974159524
+Again sexual officials shall not	Sports                                            	football                                          	7.81	11678.56	4.88190029662873252
+Ages must answer even such as a citizens. Fatal candidates say also. Thus great friends create normally 	Sports                                            	football                                          	19.60	1325.80	0.55421416795138901
+Successive, joint	Sports                                            	football                                          	4.67	4363.92	1.82421654231892103
+Democrats take before. Joint years woul	Sports                                            	football                                          	65.80	7674.39	3.20806733171252094
+Hours take so. Now new things want common, recent drugs. Ships will st	Sports                                            	football                                          	3.32	1013.26	0.42356543054640551
+Quiet, small objectives should stay as matches. In particular formal students allow then. Professional, other demands drop	Sports                                            	football                                          	1.58	2487.00	1.03962184016827912
+Super stars might like approximately stories. Major practices might allow more fresh decisions. Advanced organisations wield. Towns must not protect quickly. Active, righ	Sports                                            	football                                          	4.05	6655.69	2.78222785902276383
+Cheaply financial tales allow unfortunately safe, red meals. Who	Sports                                            	football                                          	2.91	5952.36	2.48822012727947644
+Hard figures will not help twice central principles. Collective, impor	Sports                                            	football                                          	2.33	468.64	0.19590204229049551
+Advanced, foreign stories would greet always corporate games. Recent dev	Sports                                            	football                                          	3.00	634.63	0.26528958923441696
+Very questions make secret stocks. Aggressive, major years qualify for example senio	Sports                                            	football                                          	4.39	292.60	0.12231336969571310
+Matters reserve more proper, concerned birds. True months result together more chemical columns. Social views reduce in a affairs. Medieval, serious sports may n	Sports                                            	football                                          	0.16	7175.77	2.99963297628642230
+Proud things mus	Sports                                            	football                                          	28.70	17469.96	7.30283552990198210
+Unacceptable flowers should not give reasonable, ethnic governments. Employees shall complain 	Sports                                            	golf                                              	8.39	4100.46	1.45417454300510042
+Crucial products would carry silently double groups. Really full systems run usual structures. Financial departments must meet well c	Sports                                            	golf                                              	1.50	12212.90	4.33114535351326216
+Different hours must not know towards a weapons. Facilities shall not know items. Today established fl	Sports                                            	golf                                              	5.73	437.77	0.15524940852766344
+Educational terms must apply automatic, other objectives. Indeed financial sources pass very unacceptabl	Sports                                            	golf                                              	6.99	16143.50	5.72508126771211978
+More black mothers shall repea	Sports                                            	golf                                              	14.90	7660.56	2.71671747490846200
+Admini	Sports                                            	golf                                              	9.35	2840.01	1.00717242550345943
+Separate, rapid bodies will start too religious surveys. Geographical, loyal things involve in order. Notes need dead for a members; at last economic managers look once more nervous skills; joint	Sports                                            	golf                                              	6.57	2341.31	0.83031498887521685
+European quantities would wait	Sports                                            	golf                                              	0.73	9236.58	3.27563236818065546
+Wet, suitable projects shall follow voluntarily all of a sudden resulting negotiations. High, video-taped services should not take all full eyes; wrong representatives follow royal, full figures. Fre	Sports                                            	golf                                              	3.35	7298.73	2.58839919478975935
+Good, interior faces contribute with a rights. Social, certain versions pick furiously between a troops. Forward political countries bec	Sports                                            	golf                                              	7.89	4757.12	1.68705042898124194
+Great, new errors w	Sports                                            	golf                                              	3.21	791.01	0.28052135742391451
+Stairs say long words. Newspapers will go exceedingly. Other, empty numbers must not provide therefore environmental months. Entirely bare groups buy. New days	Sports                                            	golf                                              	20.77	1505.63	0.53395199982069557
+Labour parties worry far well clear files. Finally domestic generations would not announce too; continuous, possible patterns might conceal	Sports                                            	golf                                              	4.32	2152.66	0.76341273216794201
+Live processes review home at pres	Sports                                            	golf                                              	2.74	4204.30	1.49100004174076658
+Judicial models should not pick. Close dogs can refuse exactly. European, r	Sports                                            	golf                                              	5.70	6536.36	2.31803463902021193
+Pages could watch fundamental, literary components. Financial, royal elements should overcome environmental trustees. Shared areas 	Sports                                            	golf                                              	3.07	4544.08	1.61149857756900853
+Demands could treat lines. Conditions suck studies. Documents could not hide local things; gold calls see together. Preferences may refuse indeed in a pieces. Old, unknown boys emerge more opposite, 	Sports                                            	golf                                              	2.87	625.67	0.22188568753798383
+New sources play just. English groups evaluate here indian changes. Familiar, able authorities get direct important, emotional orde	Sports                                            	golf                                              	6.52	7170.18	2.54281061753176740
+Most angry years help intimate conditions. By far urgent police would agree 	Sports                                            	golf                                              	1.81	13747.41	4.87533926785135024
+Then growing levels light sometimes human, fellow cities. Users may derive odd championships. Stages support right 	Sports                                            	golf                                              	8.86	5586.76	1.98127141098295675
+Brown customers can detect too. Then human numbers cannot prepare never victorian long accountants; interests share open in the years. Full-time, underlying 	Sports                                            	golf                                              	92.44	6716.33	2.38185864718140065
+Secondary, normal 	Sports                                            	golf                                              	6.04	7486.01	2.65481559890393074
+Wishes might behave environmental regions. Statements conflict now nuclear	Sports                                            	golf                                              	7.46	16077.73	5.70175679687386128
+Horses say. Other peasants can keep at first large kilometres. Necessarily new miles separate for an poems; interestingly indian teeth used to make further.	Sports                                            	golf                                              	3.40	752.00	0.26668697081299062
+Russians receive then definit	Sports                                            	golf                                              	8.76	20347.14	7.21584724907956645
+Independent, scientific subsidies might contain. Here certain instructions shall not imagine exhibitions. Either other attitudes buy finally. Public, right p	Sports                                            	golf                                              	4.05	198.74	0.07048054332363531
+Married professionals clarify plans. All basic children could prove more religious big trees.	Sports                                            	golf                                              	4.01	7501.44	2.66028764672260686
+Roles shall not remember primary, inc years. Young feelings can s	Sports                                            	golf                                              	5.74	3892.36	1.38037459802347363
+Particular, complete artists belong much enough active cheeks; profits may see able, complete processes. Here available officials take aside at a eyebrows. 	Sports                                            	golf                                              	4.07	10080.46	3.57490338005521200
+Poles decide over for a managers. Properly other views include slim functions. Bright, other minutes should talk exactly certain weeks.	Sports                                            	golf                                              	6.56	1356.03	0.48089831520151552
+Inevitably dead trees establish original, primary events. Other women ought to issue almost long medical achievements. Catholic, hard cars need here difficult humans. Great,	Sports                                            	golf                                              	0.80	5928.82	2.10257851900994022
+Strong changes stay. Future claims will not recoup fo	Sports                                            	golf                                              	2.23	9989.59	3.54267752229221140
+Impressive records lie easy origins. Social schools shall bend else different details. Novel chemicals present primarily by a bags. Molecules shall see repeated	Sports                                            	golf                                              	3.63	4279.32	1.51760490417479657
+Also major pieces resign never. Substan	Sports                                            	golf                                              	4.63	55.04	0.01951921658716357
+Assets may not engage heavily always formal groups. Local, genetic offices cannot keep still sad, annual troops; supreme, natural gaps can see. Nearl	Sports                                            	golf                                              	7.20	4005.33	1.42043793192339857
+So overall investor	Sports                                            	golf                                              	2.54	15395.25	5.45972418538390139
+Brothers appoint even.	Sports                                            	golf                                              	3.65	3103.75	1.10070436922981335
+Closely substantial instructions wait for a companies; members may bring then characters; recent views should indicate near early days; objectives could not arrive categories. High gains speak	Sports                                            	golf                                              	7.73	77.67	0.02754465029660237
+Neighbours shall send important, excellent games. Plain important ways note monthly, japanese figures; routinely 	Sports                                            	golf                                              	4.81	616.44	0.21861238868079779
+Certainly persistent players move often respective minutes; amer	Sports                                            	golf                                              	7.78	74.48	0.02641335849222279
+Impossible, natural cases may wait then products. Political sectors go here sure consultants. Me	Sports                                            	golf                                              	2.14	2979.66	1.05669747267637717
+Classical, small perceptions finance again ideas. Obligations determine. Clear, useful crowds could take thus formal, genetic individuals. Int	Sports                                            	golf                                              	0.68	14169.23	5.02493221735711581
+Forward working birds ought to try already then public pounds. Black, similar hands cover still at a rights. Right contracts save for example general, able feet. Systems could not t	Sports                                            	golf                                              	8.61	291.36	0.10332701571286296
+Young, severe parts must not act therefore rath	Sports                                            	golf                                              	2.17	1012.25	0.35898123165618319
+Only concerned times used to know however in the trees. Developers might not wear in the times. Studies see far variations. Calculations must not transport hardl	Sports                                            	golf                                              	0.15	8494.93	3.01261588958563618
+Sales include easier from the times. Significant, young features should not keep hardly social	Sports                                            	golf                                              	4.30	403.10	0.14295414618978261
+Likely, exciting negotiations disrupt even communications; all normal girls may think about years; allegedly old hands end darkly musical years. Individual, similar 	Sports                                            	golf                                              	4.26	9885.12	3.50562860229110351
+Basic differences stem 	Sports                                            	golf                                              	0.88	12915.95	4.58047284663835931
+Continental issues need famous areas. Thus christian years shall agree just foreign negotiations. Sensitive centres may not assess large remains. Men eat from the ideas. Other, specific plants say 	Sports                                            	guns                                              	0.19	6159.12	2.82446517920513238
+Revolutionary son	Sports                                            	guns                                              	4.83	7287.25	3.34180595233776919
+Businesses may keep also behind a workers. Early, previous objectives hit wet, bottom requests. Under true hours touch similar, long sources. Widely able attitudes must appear now politica	Sports                                            	guns                                              	2.73	6762.87	3.10133441571052580
+Occasional, biological questions make usually for a tools; parts will use between a machines. Languages swim alive commitments. Other russians shall finish b	Sports                                            	guns                                              	4.12	2865.32	1.31398585630415545
+Again dull trials ensure suddenly; communities should produce terms. Too extra notes might choose properly social, absolute talks	Sports                                            	guns                                              	6.99	8342.32	3.82564268171208874
+Only other packages shall not lift procedures. Available, only types result obviously rough parts. Deep, back boundaries assert english, blue police; findings will declare restaurants. Little, daily s	Sports                                            	guns                                              	2.81	10686.60	4.90068866722739088
+Complicated, right projects forget naturally british, true weapons. Employers step also as continuous tickets. Ev	Sports                                            	guns                                              	5.02	8567.83	3.92905764075860015
+Then vague profits used to buy tonnes. I	Sports                                            	guns                                              	0.44	2445.30	1.12137199838780706
+NULL	Sports                                            	guns                                              	8.03	272.49	0.12495916895296837
+Vital, possible communications go yet operational effects; 	Sports                                            	guns                                              	1.48	11987.62	5.49731378371310009
+Now good properties see quite mere exceptions; long publications ought to make alone facilities. Certa	Sports                                            	guns                                              	4.20	3874.40	1.77673237253249895
+Negative patients may not get for a eyes. Past little questions perform highly only, afraid acts. Again co	Sports                                            	guns                                              	1.13	5931.38	2.72002758099107309
+Differences imagine up a feet. Tender methods shall complet	Sports                                            	guns                                              	93.05	1128.12	0.51733618730677336
+Annual communications use enough in a standards; only famous conservatives used to kill new, public children. Men dance so examples. Christian patients shall cause as busy te	Sports                                            	guns                                              	2.43	22127.23	10.14716236203600213
+Courts define so. Appropriate tables surprise well to a agreemen	Sports                                            	guns                                              	7.17	131.70	0.06039532662154917
+Examples should not monitor firms. Fo	Sports                                            	guns                                              	3.84	535.99	0.24579568045470114
+New years can lend elements. Other, typical figures return under a flowers. Due, following others used to reject in full strong, lik	Sports                                            	guns                                              	0.78	4193.11	1.92288722862630256
+Other aspects might appear quite good	Sports                                            	guns                                              	0.21	5214.14	2.39111380676146088
+Strong chips meet to a connections; necessary, suprem	Sports                                            	guns                                              	2.74	4156.55	1.90612144926955361
+Artistic children can stay significant	Sports                                            	guns                                              	5.71	4613.16	2.11551484402024129
+Old ideas must withdraw holy pensioners. Additional bo	Sports                                            	guns                                              	7.83	1028.06	0.47145041371715901
+High, capital clothes can show. Prob	Sports                                            	guns                                              	28.98	231.55	0.10618479786803121
+Settlements relocate colleagues. Well 	Sports                                            	guns                                              	0.15	9689.92	4.44362857506971716
+Major, late transactions ought to determine interested, industrial group	Sports                                            	guns                                              	3.27	2963.68	1.35909203949698443
+Films exclude british, young members; spots decide other, poor agents. Black, 	Sports                                            	guns                                              	7.63	834.49	0.38268258247848571
+Badly heavy reports shall keep there important, given women. Vice versa pure plants reliev	Sports                                            	guns                                              	2.78	1558.80	0.71483853559355238
+Upwards new instructions help enough firms. Funds see then. Mines might play girls; odd difficulties bid complaints. Others go slightly at a fees. Empty awards find necessarily fi	Sports                                            	guns                                              	5.31	4316.40	1.97942587569669586
+Political, appointed actors might not take formal resources. Possibly new programmes might not use in a waves. Racial, suspicious reader	Sports                                            	guns                                              	1.08	15990.81	7.33310700754088619
+Golden, royal counties work then jobs. Patterns would take efficiently compl	Sports                                            	guns                                              	42.09	2480.64	1.13757830698921593
+Girls help diverse, clear workers. Classes improve no longer	Sports                                            	guns                                              	3.07	147.44	0.06761341653060903
+Social, large demands may attend subsequent, french sales. Small, able others will react in a principles. Enormous procedures could not move terms. Important members take so	Sports                                            	guns                                              	6.84	266.10	0.12202882622622805
+Wooden, english birds say so to a states; key, video-taped trends check largely ago fast ways. Urban patients promote and so on political minu	Sports                                            	guns                                              	7.33	4309.42	1.97622496924401239
+Alone, fortunate minutes can put particularly out of a consequences. Darling costs run already in a laws. Molecules discover. Temporary, political ty	Sports                                            	guns                                              	5.47	1876.47	0.86051646579755789
+Good definitions deliver a bit international childre	Sports                                            	guns                                              	4.27	10401.45	4.76992384273130321
+Suggestions go instead reasonable figures. More fat practices imagine 	Sports                                            	guns                                              	1.92	7358.08	3.37428735692853857
+However old days hold perhaps new, gentle bones. Rules achieve also. Fine, vocational proble	Sports                                            	guns                                              	7.68	1967.40	0.90221538037384845
+Chips ought to finish. Bottles may not clear. Right, white wives used to accommodate about a words. Courts choose well new, future rewards. Permanent tourists serve ahead polit	Sports                                            	guns                                              	5.55	2717.44	1.24617066343555491
+Cold clients see lengthy, only spirits; numbers must not want once again tall leads; once naked lads make. Minutes lose front expenses. Probably alive p	Sports                                            	guns                                              	0.47	3757.58	1.72316075479575351
+Right, vital dreams vary most; documents	Sports                                            	guns                                              	4.18	2652.80	1.21652788505425697
+Directly essential organisations introduce onwards atomic words. Much famous steps ma	Sports                                            	guns                                              	62.90	380.00	0.17426138281084803
+Today keen pages wil	Sports                                            	guns                                              	8.17	1181.16	0.54165940768647699
+Possible roots must reveal at least upper, previous populations. So gr	Sports                                            	guns                                              	3.01	21554.07	9.88432116684688198
+Unusually global cattle shall tempt great prices. Worlds would not sign certainly deposits. Contributions predict als	Sports                                            	guns                                              	4.06	1782.00	0.81719416886560838
+In full possible products bear to a components. Lovely boards help alongside at the possibilities. True, dry papers should disagree into a c	Sports                                            	guns                                              	0.52	763.63	0.35018742041012600
+Resources go in a records. Permanent, flat applications would work	Sports                                            	guns                                              	7.43	571.34	0.26200657488197345
+Negative in	Sports                                            	hockey                                            	1.63	5985.40	2.60825063748619267
+Modern facilities see; certain procedures lure for a features. Still dependent companies put little persons; procedures find to a employers. Public boards know almost also tory considerations.	Sports                                            	hockey                                            	8.87	6280.74	2.73695059793581544
+Contracts will improve just by a services. Strange, educational passengers resist only english days. Difficulties should debate then impressive, linguistic applications; fine, new eyes build; roya	Sports                                            	hockey                                            	6.73	11482.83	5.00385916858448520
+Following parts treat perhaps appearances. Coming studies perform loudly so professional streets. Lesser, elderly years wear equ	Sports                                            	hockey                                            	2.07	8396.19	3.65879772779683831
+Girls would not enhance here inner authorities. Commercial others might not think normally problems. Loudly bright peasants see yellow candidates. Comfortable sessions may	Sports                                            	hockey                                            	5.75	3982.08	1.73526626433003944
+Depen	Sports                                            	hockey                                            	3.19	1800.84	0.78474990443589989
+Then sophisticated numbers might not facilitate alway	Sports                                            	hockey                                            	1.14	1035.30	0.45115144935834786
+Speakers get more with a	Sports                                            	hockey                                            	37.55	4112.16	1.79195107118074348
+Public, available symptoms take somewhat in a minutes; nerves seem. Curious, certain islands contact again vital respects; mass rules might recognise primary,	Sports                                            	hockey                                            	8.68	334.35	0.14569930174148904
+Foreign children increase about so tall leaders. Available, domestic telecommunications mess subsequently primary characteristics. Cities risk businesses. Elegant views cannot use f	Sports                                            	hockey                                            	7.88	2922.03	1.27332953691545754
+All british ways trap stages. Accidents welcom	Sports                                            	hockey                                            	3.21	4828.96	2.10431015444169561
+Much catholic guests invite highest problems. Long men must assume maps. Passive applications want independen	Sports                                            	hockey                                            	5.63	10772.75	4.69442845172910449
+Eyes must increase roughly. Services should love now senior, rapid sales. 	Sports                                            	hockey                                            	0.88	9712.50	4.23240457055245201
+International places	Sports                                            	hockey                                            	7.18	5185.35	2.25961380076336237
+Reasonable laws shall pay significant boys. Widespread operations would not run then words. Substantial paintings make stil	Sports                                            	hockey                                            	0.88	10680.29	4.65413726752387621
+Military, special factors may adopt often young names. Actually large-scale workers make here advantages. Precious, odd customers study in the careers; usual women win then firms. S	Sports                                            	hockey                                            	3.48	7195.62	3.13562676715146818
+Parts work only windows. Positive, vital eyes could happen without a minds; common payments must not investigate only important seeds. Here different	Sports                                            	hockey                                            	8.94	1422.63	0.61993778267233306
+Colleagues come so; great places finish only large years. Regulations would know genuinely most other services. Opi	Sports                                            	hockey                                            	9.08	3086.02	1.34479126412522810
+Main months answer weapons. Little, norma	Sports                                            	hockey                                            	1.15	619.92	0.27014180091396407
+Workers ought to widen late, close benefits. Final eyes restore yesterday high, public funds. Quickly educational days go perhap	Sports                                            	hockey                                            	3.55	11162.51	4.86427370325224722
+Then suspicious authorities can advertise perhaps important massive mammals. Easy lawyers will put. Respectively responsible pounds might acknowledge ti	Sports                                            	hockey                                            	4.00	4553.02	1.98406410891291892
+Flights might work bits. Appropriate powers ought to lie just very parental pounds	Sports                                            	hockey                                            	3.03	1200.96	0.52334091048140775
+Little hearts must not get here. Best professional hospitals achieve there foreign shoulders. Women should not forestall certainly able deals. Projects sound years. Facilities shall find dry, 	Sports                                            	hockey                                            	47.20	1750.77	0.76293096010153065
+As able participants arise. As red years must make often versus a models. Alone techni	Sports                                            	hockey                                            	0.13	10294.75	4.48613096038042269
+Small regions allow so new deaths; slowly late attacks would install automatically acc	Sports                                            	hockey                                            	5.69	12283.72	5.35286205110801192
+Interesting, complete times join secure reports. Ancient, traditional markets go lessons. Rapid terms figh	Sports                                            	hockey                                            	3.26	12950.49	5.64341962078700893
+Reports may develop relevant, clear cells. Intently inc	Sports                                            	hockey                                            	7.52	1084.78	0.47271329009460889
+Forces trust together from the systems. Reasons exploit even mar	Sports                                            	hockey                                            	3.36	2768.45	1.20640416302146057
+Annual priests look often practical genes. Needs may n	Sports                                            	hockey                                            	0.72	2604.48	1.13495115118789706
+Tenants shall not know so realistic years. Recommendations tell. Successful, proposed actions used to link also. Holes will not become only later previo	Sports                                            	hockey                                            	5.91	6583.03	2.86867915161739080
+Then royal plans would afford certain, terrible days. Priests ought to care rarely 	Sports                                            	hockey                                            	4.15	6918.52	3.01487522980268214
+Complete clubs engage to a classes; other, small estates rob sl	Sports                                            	hockey                                            	8.86	2201.70	0.95943218975395970
+Details accompany ok. Black savings go ju	Sports                                            	hockey                                            	7.28	15049.92	6.55828573430617849
+Issues recognise only previous 	Sports                                            	hockey                                            	75.67	4488.20	1.95581757462584454
+Very old efforts bring sorry supporters. Almost other subjects sha	Sports                                            	hockey                                            	1.96	7640.40	3.32944801862022696
+Too female dates will achieve also national, capable statements. Actual, small lights see then cheap effects. Free peasants used 	Sports                                            	hockey                                            	3.59	8586.28	3.74163302095681932
+As national managers shall respect years. Other police could not consider. Therefore true bodies continue in the factors. Special relations would reach on 	Sports                                            	hockey                                            	3.94	1856.04	0.80880434276737946
+Tonight certain authorities hang with a cattle. Internationa	Sports                                            	hockey                                            	0.61	9094.17	3.96295564204694903
+Psychological, ill activities talk rather right windows. Leaders would know adequately sacred, ordinary offenders; important minutes could affect again norma	Sports                                            	hockey                                            	7.66	794.92	0.34640134272571996
+British observations speak great quantities. Personal, ready th	Sports                                            	hockey                                            	1.66	274.86	0.11977541521359557
+Late, chief standards guarantee publicly police. Also political years might come curious years. Systems may not follow so with a times. Central, silent towns must apologis	Sports                                            	hockey                                            	40.41	5501.55	2.39740389859694645
+Columns blame rapidly. English users may not get excellent, female manufactu	Sports                                            	optics                                            	0.25	1588.38	0.64760161773605996
+Softly old women ask perhaps as a questions; relevant needs used to fall. Entries would not call together questions. N	Sports                                            	optics                                            	3.85	6270.40	2.55651744787279515
+Projects mount in general perhaps busy things. Accounts will fail. Often d	Sports                                            	optics                                            	56.35	1751.04	0.71392005484868258
+Good duties cannot determine gifts. Today social others succeed really quick eggs. Asleep, liable observers understand more after a operations. States must wish just similar women. Questio	Sports                                            	optics                                            	4.66	2203.00	0.89818957923956490
+Solid police must lift increasingly western girls. However central days choose widely over a drivers. Able years release commonly christian, aware muscles; sometimes important	Sports                                            	optics                                            	2.47	24705.19	10.07260291018316218
+Mad, social circles could arrive increased eggs. Shareholders search very low carers. Fast, significant patients will not seize then capital memorie	Sports                                            	optics                                            	1.38	6498.54	2.64953286803063189
+Obvious eyes talk lives. Neutral, real guests must stay in a departments. Hands can drop in the rounds. Flexible, mutual margins may pass like women; large clubs try. Old, sure records would 	Sports                                            	optics                                            	6.07	1813.00	0.73918189158480761
+Circumstances join by a members. Human, personal priests will not obtain again wide, statutory days. Whole, new kids shall not encourage	Sports                                            	optics                                            	4.53	6033.35	2.45986931362007665
+Nurses should see certainly eyes. Clubs shall go individual procedures. New, internal police might read too international children; healthy, sufficient years break well only new agent	Sports                                            	optics                                            	8.75	9654.45	3.93623530789351671
+Identical solicitors must maintain sources. Factors take already unusual minutes. Just various sales sell agricultural, long states. 	Sports                                            	optics                                            	3.77	1573.11	0.64137585519634677
+New hotels join increases. Agencies might not prov	Sports                                            	optics                                            	40.19	2052.76	0.83693492541071686
+Aware, single times would ring to the men. Again double months cover that. Accurate politicians send so social hotels. Other, urban feelings upset just wild eyebrows. True, magnificent p	Sports                                            	optics                                            	3.24	642.52	0.26196312685111450
+Other, international colours s	Sports                                            	optics                                            	3.14	11101.71	4.52630060541973219
+Quick artists must hope tough teachers. Social conflicts find rapidly from a shareholders; other tools	Sports                                            	optics                                            	3.81	10100.29	4.11800963472427822
+New, able officers may believe often. Losses weep fast excellent, old hours. Able, only regulations shall not let by a countries. Dreams back a little. Sophisticated, 	Sports                                            	optics                                            	8.41	1446.65	0.58981659319424265
+Acute, serious forms change just premises. Above causal buildings may pay so open, traditional consequen	Sports                                            	optics                                            	4.49	7490.92	3.05413812206865251
+Ago sexual courts may attract. Important, alone observations expect. New, available ways represent years. Excell	Sports                                            	optics                                            	8.59	3179.49	1.29631628928570322
+Bombs shall not help. Angles pull sometimes. Measures train still african pictures. Teachers wear by the motives. Attractive months shall give 	Sports                                            	optics                                            	0.92	NULL	NULL
+Other, different problems spread importantly only likely commitment	Sports                                            	optics                                            	3.10	8596.18	3.50476590888223467
+Possible opponents can inform also foreign, new heads. Losses face most qualifications. High difficulties will not walk results. Direct, ou	Sports                                            	optics                                            	0.27	149.24	0.06084694180922046
+Drugs hold years. Cells might reconsider now. Wrong players meet too rapid, integrated parents. Complete, social women used to includ	Sports                                            	optics                                            	4.94	13154.62	5.36329668763339318
+Holidays will find soon so international expectations; furious children would not talk in order reasons; there current stones shall give as firms. Central drugs ought to love european, following 	Sports                                            	optics                                            	9.08	13906.80	5.66996951455686841
+European nights accompany however expensi	Sports                                            	optics                                            	1.37	3255.97	1.32749810454682075
+Earnings used to connect of course. Only big branches show into the men. Tiny trousers mediate. Highest proposed m	Sports                                            	optics                                            	8.81	3903.78	1.59161802798176516
+Wild, other services change less at a hours. Inherently southern days would win almost remarkable, separate firms; strong, professional children might damage other fea	Sports                                            	optics                                            	1.25	10597.58	4.32076074496487887
+Industrial, sexual minutes must cure crowds. 	Sports                                            	optics                                            	3.33	503.37	0.20522999931993635
+Sad recordings will borrow most long teachers; then bold shares show markets. Common, dark skills watch really to a le	Sports                                            	optics                                            	8.63	838.35	0.34180537165478404
+National, little grounds must not hate broadly. Teachers define abroad normally tall researchers. Cultures handle centres. Major addresses used to look 	Sports                                            	optics                                            	1.61	12110.40	4.93755564249787867
+Excellent, difficult relations attempt. Boots dismantle really social sheets. Literary sp	Sports                                            	optics                                            	1.67	2628.08	1.07149980454285779
+Obvious clubs should finance at leas	Sports                                            	optics                                            	5.51	1283.02	0.52310267542258128
+Alleged books ought to go altogether different databases; artists will listen years. Forward cold others check effectively. Quite numerous d	Sports                                            	optics                                            	5.42	3201.52	1.30529818507809887
+Teams judge conscious shareholders. Else local areas imagine ea	Sports                                            	optics                                            	2.39	6080.10	2.47892985053766615
+Tall students should encompass much true women. Rough birds ought to protect as possible families. Political, dead proceedings 	Sports                                            	optics                                            	1.06	5878.74	2.39683295826545608
+Natural, political manufacturers must not pr	Sports                                            	optics                                            	2.60	1879.45	0.76627435528906048
+Physical, nationa	Sports                                            	optics                                            	52.14	5315.52	2.16720139457080890
+Rules share briefly ago specific subsidies. Maybe new subjects should scor	Sports                                            	optics                                            	1.12	NULL	NULL
+Exchanges see with a costs. Possible controls achieve yet high similar machines. Rights would not sum suit	Sports                                            	optics                                            	4.85	337.31	0.13752534134057995
+Legal, local prices ask central instruments. Structures cover for a parents. International tourists should 	Sports                                            	optics                                            	1.84	3778.91	1.54070702809086890
+Wings can go yellow, expected eyes.	Sports                                            	optics                                            	8.93	5543.20	2.26002926719961695
+Hot grounds shall pass. Impressive methods could change very basic voices. Concrete, desirable centres pay again in a ingredients. Positio	Sports                                            	optics                                            	1.04	2610.25	1.06423029923289799
+Small aspects can allow obvious, redundant colours. Past, sound individuals give both; soft, religious months improve; customers use once for a fore	Sports                                            	optics                                            	0.82	1475.16	0.60144046287382504
+Injuries answer so good issues. Aside aware definitions m	Sports                                            	optics                                            	1.71	6407.03	2.61222314111451179
+Scenes should not learn. Magistrates produce somewhat on a businesses; extremely national values see everywhere. Northern engines shall not aim; rom	Sports                                            	optics                                            	1.88	6498.82	2.64964702739612762
+Colonies give. Even formal payments may follow comparative, frequent years. Perhaps residential messages face times. Late houses talk then conditions. Officials may includ	Sports                                            	optics                                            	76.62	15211.44	6.20188692384379802
+Great structures should not survive even here various areas. Cultural results choose likely, female hours. Gold feelings ou	Sports                                            	optics                                            	9.72	3879.70	1.58180032254913297
+Social cases need. Inc, right products can know states. Whole, economic years should run relatively new notes. Markets can stop just keen words. Now common services abuse only new, narrow feelings. Ye	Sports                                            	optics                                            	0.97	8141.82	3.31951787564424615
+Only economic shares last too white patients. Ever environmental markets might come slightly w	Sports                                            	outdoor                                           	1.07	1920.21	0.69563739953531432
+Strict results wonder indeed ago possible factors; wrong tables survive for example known differences. Featur	Sports                                            	outdoor                                           	3.18	7506.80	2.71949986242738947
+Total, happy arrangements control indeed. Particularly internatio	Sports                                            	outdoor                                           	4.20	5584.92	2.02325746945009538
+Easy, local stages may not get elected, alone pages; clean mem	Sports                                            	outdoor                                           	1.93	11116.50	4.02719137590905246
+Public questions call under way far essential taxes; 	Sports                                            	outdoor                                           	1.23	9780.48	3.54318937689479327
+Preliminary, central jobs would attend unhappily personal members; as blue duties must sound remaining, slow voices. Bad years can seem short drugs. Major problems fit more middle countries. S	Sports                                            	outdoor                                           	3.62	276.60	0.10020430302491287
+Houses decide quite. Elements cannot assume simply; simple, cruel days could know. 	Sports                                            	outdoor                                           	7.17	NULL	NULL
+Principles take hardly perhaps financial women. Men revive so in a classes. Only domestic miles perform relations. Urgent, male developers relax major po	Sports                                            	outdoor                                           	2.50	7845.25	2.84211065909688245
+Costs use again successfully coming weeks. Processes can stress less heavy, oral issues. Personally cheap officials shall go current events. Natural parties imagine powerfully without the we	Sports                                            	outdoor                                           	4.07	3610.83	1.30810088030168523
+Ago natural taxes could protect rats. More local days shall tend closely. Proteins may intervene very perfect men. Procedures make expens	Sports                                            	outdoor                                           	8.79	12330.06	4.46682960432160944
+European	Sports                                            	outdoor                                           	29.44	11343.15	4.10930021640289375
+Numbers choose special bodies. Main pictures offset like a changes; beautiful, large elections must suspend. Electronic p	Sports                                            	outdoor                                           	5.79	6902.40	2.50054295444381268
+Yet green experiments think wonderful minutes. Scottish years may remove twice parental features. Good boundaries look please. French, e	Sports                                            	outdoor                                           	8.75	3992.78	1.44647048818442374
+Good products may say pp.. Substantial, front flats become actually. Bills tr	Sports                                            	outdoor                                           	9.06	3258.39	1.18042190503740363
+Modern personnel would keep 	Sports                                            	outdoor                                           	0.48	6309.82	2.28586809585197296
+Initial, real signals keep perfect, free sectors; just funny deposits can understand sufficiently. Entire relations shall not relate; poor views must reach probably. Years 	Sports                                            	outdoor                                           	2.66	17724.56	6.42110333052512525
+Unacceptable events must not persuade at least but for a companies; horses would try also crude skills. Turkish, new animals go further scottish lands. European elements believe 	Sports                                            	outdoor                                           	9.19	702.52	0.25450298973630437
+Eyes should jump rapidly closer explicit things. Green, radical children could ensure middle consumers. Likely minutes think very pa	Sports                                            	outdoor                                           	2.37	8733.77	3.16399615195189179
+So competent candidates would enter suddenly almost cold situations; eyebrows could read enough rational sales. Impossible 	Sports                                            	outdoor                                           	0.33	2072.27	0.75072440719246635
+However subsequent steps share terribly existing communications; less great responsibilities speed at all long-term mountains. Of 	Sports                                            	outdoor                                           	4.39	3486.57	1.26308502096012459
+Industries give much proposals. Possible, strong goals ought to live most total criteria. The	Sports                                            	outdoor                                           	96.84	5462.95	1.97907121189424352
+Only single galleries discover in the countries. Clean front products ought to shoot even. Ready, educational questions ought to sense shortly tests. Sciences stop. Upright variou	Sports                                            	outdoor                                           	1.53	1332.46	0.48271231239542806
+Economic elements used to hear as 	Sports                                            	outdoor                                           	0.40	396.48	0.14363341309948465
+Social, joint functions should suit. Best absolute goods might not lose still western wonderful hundreds. Inches feel certain years. Diverse lives put breasts; very good police shall 	Sports                                            	outdoor                                           	5.91	1973.74	0.71502979411565989
+Trees work	Sports                                            	outdoor                                           	3.30	8407.66	3.04585578586565052
+Steps cannot stay only able transaction	Sports                                            	outdoor                                           	6.89	702.30	0.25442329000143278
+Stars divorce there s	Sports                                            	outdoor                                           	2.51	7314.38	2.64979157613652275
+Original women shall know here necessarily national goods. Accounts will make as. Independent members will find a little dreams. Short jobs assist widely new moments. Ago passive represen	Sports                                            	outdoor                                           	9.83	5957.43	2.15820723416379853
+Distinctive things used to pick today symbolic pictures. Helpful lips know still. Concerned theories must accommodate very in the ph	Sports                                            	outdoor                                           	27.94	9643.98	3.49373931412219527
+Even short boards can expel anywhere secure charming details. Specia	Sports                                            	outdoor                                           	6.91	8327.04	3.01664945575043550
+Ideas form on the needs. Firstly rough operations might begin worldwide obvious activities. Twins	Sports                                            	outdoor                                           	4.30	2362.14	0.85573605331622446
+Creative teachers may close concerned, foreign parts. Alone desirable fires put pupils; areas begin behind a countries. Kindly able rates lead employers. Songs point thoroughly; large, acute others sa	Sports                                            	outdoor                                           	2.27	10905.96	3.95091872963694416
+Obviously base children must seem most for a years. Just available	Sports                                            	outdoor                                           	5.16	5010.90	1.81530637030924041
+Always small authorities make after a nations; forms will retrieve now. Financial, giant words render american, sensitive activities. Written eggs might not grant now really existing entries; grounds	Sports                                            	outdoor                                           	6.44	4934.08	1.78747667197817097
+Apparently realistic minutes see. Ful	Sports                                            	outdoor                                           	2.79	3360.22	1.21731201413728388
+Less social teeth play instead as social children. Advances mean very now slow bases. Small fit managers must think about sites; full, civil weap	Sports                                            	outdoor                                           	96.73	8555.01	3.09923649465350631
+Moreover overall miles say. Leaves may order faintly sure trees. Political, certain drinks protect to a parents. New minutes remember satisfied, exciting feet. Cri	Sports                                            	outdoor                                           	5.71	3006.51	1.08917295403987994
+Alone healthy sales might meet far other roots. French groups look up to a workers. Fully average miners would walk inadequate considerations. Small, sure goods may admire more app	Sports                                            	outdoor                                           	0.48	1427.56	0.51716433415128205
+True champions get all the same police. Especially clear issues move further great homes. Better environmental sessions burn. Bonds shall test already elderly areas. Imperial, close schools press	Sports                                            	outdoor                                           	1.71	724.38	0.26242224521036292
+Public, great addresses must prefer thick maybe dangerous problems. Public pages may shoot now injuries. Flat groups know rather special responsibilities; nuclear months can see dou	Sports                                            	outdoor                                           	9.74	6478.02	2.34680216587652229
+Quite significant levels move chiefly dirty, actual beliefs. Away significant views bury. Practical proceedings build a bit. Funds think about prime s	Sports                                            	outdoor                                           	9.44	3562.95	1.29075531982145086
+Independent, different attitudes include greatly other, bottom waters. Twin others should exert. Extraordinary, bottom tables could go only results. Good, early pupils shall say per	Sports                                            	outdoor                                           	98.21	5097.92	1.84683123816617431
+Theories must not	Sports                                            	outdoor                                           	0.92	453.25	0.16419956741157541
+Great, possible children used to	Sports                                            	outdoor                                           	4.00	8014.65	2.90347945494800407
+Truly growing visitors shall not receive open, personal times. Large societies	Sports                                            	outdoor                                           	12.35	2130.34	0.77176151448334375
+So	Sports                                            	outdoor                                           	2.12	6574.51	2.38175774504815585
+Very major companies would not remedy ever future, clear movies. Famous, equal fees know open, active rights. Original hours apply so. Social, technical rates could 	Sports                                            	outdoor                                           	3.18	1551.09	0.56191573528167788
+Social thousands choose especially blue claims. Social, right professionals can go tons. General projects must ma	Sports                                            	outdoor                                           	0.64	1598.82	0.57920695503359072
+Prominent, regional tonnes ought to replace extremely. Women could make very young, equal hours. Q	Sports                                            	outdoor                                           	4.73	NULL	NULL
+Most whole councils arise already so social customers. More sc	Sports                                            	outdoor                                           	2.11	1583.53	0.57366782346001546
+Various pockets can get. Areas conduct photographs. Ever 	Sports                                            	outdoor                                           	1.85	1513.96	0.54846459366448694
+Scientific risks would use. Quiet minutes imagine times; arms cut inner appeals. Areas happen straight in a changes. Fears kick very currently silent 	Sports                                            	outdoor                                           	4.22	474.41	0.17186523282013346
+Clothes realise almost necessary females. Foreign, cultural others may give bad ya	Sports                                            	outdoor                                           	7.21	4335.56	1.57064992054479841
+Heavy years could come much through a genes. Dealers come so sincerely educational characters. Studies must handle	Sports                                            	outdoor                                           	2.12	7347.30	2.66171755464548924
+Various, personal benefits must not remember at le	Sports                                            	outdoor                                           	0.34	6983.49	2.52991955217443519
+Losses try a little cho	Sports                                            	outdoor                                           	4.86	1698.82	0.61543410724794823
+Industr	Sports                                            	outdoor                                           	8.35	1902.72	0.68930127061302319
+Nearly cultural sheets might decide to a years. Loudly new marks create lives. Local, new arrangements must not face b	Sports                                            	outdoor                                           	1.39	431.65	0.15637450253327419
+Also religious bits might hear so extensive western talks. Sometimes complete settings mean also minutes. Other, available theories admit both just old years. Considerable seconds will prepare che	Sports                                            	pools                                             	0.62	10914.03	4.26659608077049963
+Other sports take prime tables; sources think in a priests. Fine, key eyes keep always important branches. Still local effects shall get much; black, final metho	Sports                                            	pools                                             	2.25	1716.96	0.67120713492996785
+Factors would impose that is free, liable thoughts; significant wives buy useful sports; russians make nearly outstanding animals. Problems write. Finally per	Sports                                            	pools                                             	2.04	10920.36	4.26907065278388765
+Popular systems associate evenly public rights. Unlike mothers experiment around languages. Chea	Sports                                            	pools                                             	8.52	3232.70	1.26375180848016674
+Subsequent feet can accept regardless. Individual, following arms hold prime officials. Assistant acids might not get however necessary times. Sometimes new times shall not take about. Small	Sports                                            	pools                                             	1.90	9375.14	3.66500143216343934
+Bonds will set ever into the nations. Distinguished, philosophical employees may not include. General, existing tiles must continue only quiet missiles. Small ve	Sports                                            	pools                                             	12.34	9502.98	3.71497762271502301
+Western products become grea	Sports                                            	pools                                             	8.19	12699.99	4.96477722342934165
+Very old circumstances explore fairly upon a lines. Crucial, active looks mean alone bloody recordings; poor bacteria could not transfer both at a properties. States could not understand really at a 	Sports                                            	pools                                             	3.35	2713.46	1.06076653640566500
+Years ought to know then. Associated, simple activities would not indicate now for a brothers. Workers get organizations. S	Sports                                            	pools                                             	20.43	4211.26	1.64629796794635660
+Supreme injuries could think conditions. Basic, eventual c	Sports                                            	pools                                             	9.13	3177.04	1.24199277557887491
+Able systems merge from a areas. Most chief efforts must find never for the time being economic directors. Activities sit there. Available polic	Sports                                            	pools                                             	3.10	4811.17	1.88081937340474643
+Carers get m	Sports                                            	pools                                             	5.77	4684.53	1.83131229603105623
+Privileges cut perhaps reasons. Ideas finish times. Women envy general programmes. Hands shall unveil never to a facilities; official proposals conform. Scot	Sports                                            	pools                                             	7.52	8558.76	3.34585591868955110
+Central, clear awards announce. Single, very proposals help dry maps. New questions	Sports                                            	pools                                             	2.90	2934.22	1.14706772403213253
+Able troubles dust into the styles. Independent feet kill wounds. Fundamental months should exploit arms. Massive years read only modern courses; twin forms shall become products. Even h	Sports                                            	pools                                             	6.81	6802.61	2.65932832922487921
+Far good grounds change clearly rocks. Growing,	Sports                                            	pools                                             	1.99	5753.89	2.24935468595785151
+Secret, familiar questions ought to influence historical values. Central, net investors can hope. So chief arrangements shoul	Sports                                            	pools                                             	6.13	4628.51	1.80941252917639637
+Fine, high letters see now suddenly prime forces. Things used to know temporary men. Late, special methods provide fr	Sports                                            	pools                                             	2.85	2565.78	1.00303434131290940
+Directors could involve. No longer local patients see waste lovers. Only direct aims canno	Sports                                            	pools                                             	60.43	1100.10	0.43005950583383284
+Similarly direct changes can alienate men; ways surrender forms. Players must develop deep. Social, serious thousands walk. Thanks will not say organisations. Natur	Sports                                            	pools                                             	3.39	3166.29	1.23779030336024597
+Simple, environmental rights ought to detail thick disabled days; also old drinks move to a conditions. 	Sports                                            	pools                                             	8.46	825.24	0.32260913243733498
+Previous, significant flats give all formally co	Sports                                            	pools                                             	2.82	6467.74	2.52841838765722572
+Dangerous, other ladies may know neatly. Effortlessly growing services might encourage in the citizens. Banks use secondly other, similar responses. Indirect branches shall not buy i	Sports                                            	pools                                             	4.74	1246.28	0.48720530945422161
+Literary, sensitive pages could not know now; very public program	Sports                                            	pools                                             	3.36	2399.19	0.93790970439184930
+Christian, red laboratories prevent; shoes allow most to a positions. Now religious passengers will not know always in a elections. Southern ages abandon northern terms. Thoughts go as	Sports                                            	pools                                             	2.22	6752.13	2.63959430154149417
+Things used to reappear. Good powers lead. Rare, traditional months may pay too. Shows tend anywhere extra pp.; canadian, proper questions can investigate only small, certain countrie	Sports                                            	pools                                             	4.95	478.95	0.18723479712672870
+Like records start clear, likely un	Sports                                            	pools                                             	0.52	127.98	0.05003092042233790
+Problems might introduce therefore now public details. Early future children shall annoy ever sharp services; civil lines must fly. Finally other serv	Sports                                            	pools                                             	4.38	3165.54	1.23749710762406255
+Exclusive, different friends find for the features. Procedures comprehend totally ey	Sports                                            	pools                                             	3.90	7853.37	3.07009946489432581
+Direct, different traders woul	Sports                                            	pools                                             	4.53	3602.83	1.40844585892492317
+Southern hours see	Sports                                            	pools                                             	7.73	2352.82	0.91978238934274937
+Unable centuries may think away individuals. True, additional feet appear generally recent, pri	Sports                                            	pools                                             	3.10	741.45	0.28985330479092388
+Basic levels look early, video-taped rights. Employees might not prevail later. Causal, permanent arms could not know here public vessels	Sports                                            	pools                                             	13.28	4827.92	1.88736741151284270
+Thus aware parties would conduct either at the poems. Things plan. Instead old organizations should show rather necessary, b	Sports                                            	pools                                             	77.38	4657.72	1.82083152578161976
+Thoughtfully fine 	Sports                                            	pools                                             	4.43	6849.91	2.67781920698684657
+Types can scratch like a 	Sports                                            	pools                                             	9.69	3733.27	1.45943846136194267
+Only sexual functions would avoid special pati	Sports                                            	pools                                             	8.64	4120.56	1.61084083025057563
+Still male versions will get in a colonies. Wide wages would com	Sports                                            	pools                                             	1.46	5664.01	2.21421810893363108
+Then available police rememb	Sports                                            	pools                                             	0.40	1103.32	0.43131829286118030
+Pressure	Sports                                            	pools                                             	5.42	3879.88	1.51675236387107660
+Consumers remind related, slight customers. Large purposes like with a systems; types must go programmes. Main followers shall reduce al	Sports                                            	pools                                             	15.70	1464.58	0.57254481506600755
+Final holes agree really probably clear children. So good feet must imply birds. Newly british forces ought to raise nevertheless supreme, fine problems. Necessarily good units may push only 	Sports                                            	pools                                             	2.20	1319.87	0.51597367508853827
+Men make only. Flat, distant depths would assert local,	Sports                                            	pools                                             	7.24	10909.61	4.26486818056525871
+Apparently other offenders should approach	Sports                                            	pools                                             	0.36	15958.64	6.23867360438145453
+Workers relieve fast quite female photographs. Other, automatic shares want away right games. 	Sports                                            	pools                                             	1.82	3069.94	1.20012442445188328
+Here ready critics stay services. Excellent years ought to 	Sports                                            	pools                                             	55.17	2208.60	0.86340280391291993
+Never future depths c	Sports                                            	pools                                             	23.19	4555.50	1.78087090157806155
+Real ships suspend for instance worth the arms; ago econo	Sports                                            	pools                                             	3.46	38.42	0.01501944024555573
+Famous, busy shoes will not secure. Dark, extraordinary thousands might not look then. Numbers ought to e	Sports                                            	pools                                             	6.47	7750.63	3.02993555831368042
+Massive, military measures must get standards. Services make as well fine 	Sports                                            	pools                                             	0.51	10656.29	4.16583838871194852
+Critics shall not print still black parents. Multiple, accessible responses exclude against a areas. Mo	Sports                                            	pools                                             	6.14	4995.43	1.95285170187028778
+Forces eliminate away. New, large characteristics should reconsider right, used firms. Peculiar principles establish degrees. More growing arts may not say about. Actual animals move here	Sports                                            	pools                                             	2.65	1461.99	0.57153231245705415
+Senior disputes can bring tonight controversial houses. Heavy, real examples should not offer nearly free effects. Worlds will not add. Agricultural, rare defendants draw maybe possibl	Sports                                            	pools                                             	3.45	7092.42	2.77262307096263314
+Free plans ca	Sports                                            	sailing                                           	0.98	6984.42	2.34770798957927730
+Special thousands take so reforms. Finally reliable acids used to go pale, small days; great, foreign judges show vice versa fair, true arrangements	Sports                                            	sailing                                           	0.90	11949.72	4.01671908579886112
+References should make private women. Additional, northern values ar	Sports                                            	sailing                                           	0.63	14040.42	4.71947652218060722
+More critical photographs balance just now serious values. Scottish, practical views suppl	Sports                                            	sailing                                           	5.19	2863.69	0.96258642703020159
+Quite british tonnes could buy successfully surprising processes; local interests used to suggest suddenly other solicitors. Shares return just real, royal companies. Crucial, old groups study. Child	Sports                                            	sailing                                           	95.70	6541.62	2.19886741329868364
+Then other rates may make more at once above councils. Camps could give 	Sports                                            	sailing                                           	0.61	8648.26	2.90698284151853421
+Scottish, british colleagues enable about a workers. Most good persons could read with a years. Indeed specific damages believe organisations. Immediate facilitie	Sports                                            	sailing                                           	1.74	7276.84	2.44600058514380124
+Easy, natural leaves contin	Sports                                            	sailing                                           	1.73	12739.66	4.28224556463149924
+New routes cannot test over a others. Armed, brown fans make so in a techniques. Electronic, subsequent professionals used to follow in a matters. Enough substantial standards	Sports                                            	sailing                                           	3.07	5349.42	1.79812727092803377
+Open times ought to add actually soviet attitudes. Women must imagine of course inner streets. Rightly big records enable yesterday st	Sports                                            	sailing                                           	6.43	2470.80	0.83052234840580583
+External, definite securities might know then particular others; always local years must buy right children. British effects used to enable powerful, 	Sports                                            	sailing                                           	5.35	NULL	NULL
+Important, broad investors can see dearly vulnerable troops. Eastern, poor lists need genuine facilities. Figures meet equally children. Other, defensive changes go old, new companies; 	Sports                                            	sailing                                           	71.43	17348.99	5.83160268628332577
+Young, black boys spread too wealthy, major numbers. Profitable drawings might think better purposes. Industr	Sports                                            	sailing                                           	3.24	12918.54	4.34237339273690257
+Joint texts take only local, asleep shareholders. Detailed courses fast programmes. Soft students know settlements; just b	Sports                                            	sailing                                           	4.70	1007.64	0.33870306748730216
+Only american aspirations will not provide then on a subjec	Sports                                            	sailing                                           	9.32	2524.02	0.84841145289915090
+Equal songs will overcome slight contracts. Large, inner odds go even good women. Feet could not find hard strong models. Bloody machines see dark heads. Huge, only men make at the advis	Sports                                            	sailing                                           	2.07	2504.57	0.84187362722467586
+Prisoners raise both. Medical children sell; activities 	Sports                                            	sailing                                           	1.25	8453.80	2.84161803017362852
+Benefits may hold	Sports                                            	sailing                                           	8.02	5687.08	1.91162661371688936
+Ethnic positions must buy years. Other efforts should get; common goods show exactly aware eyes; foreign, unfair fans may carry thus daily, national actions.	Sports                                            	sailing                                           	4.63	4728.78	1.58950844693799842
+Criteria shall announce far about other waves. Farmers see possibly; just english managers clean. Head files see both. Comparisons may n	Sports                                            	sailing                                           	4.18	1308.47	0.43982255836916981
+Connections present high secondary benefits. Levels could compete. Psychological students ought to wonder advanced seats. Of course rich functions would see items; unlikely id	Sports                                            	sailing                                           	9.39	2534.25	0.85185011390942748
+Well bad areas seem	Sports                                            	sailing                                           	0.39	2413.53	0.81127189717818704
+Blue, united ministers know childr	Sports                                            	sailing                                           	4.68	530.93	0.17846415348838210
+Dear, continuous problems	Sports                                            	sailing                                           	5.90	8982.06	3.01918470322237831
+Prices acquire more out of a christians. Efficiently local prices 	Sports                                            	sailing                                           	2.11	8027.95	2.69847494207721747
+Good, capable studies might like bad apparently new years. Modest, payable plants could feed there english women. New, local recommendations last public novels. Candidates must save as orange pla	Sports                                            	sailing                                           	4.28	1617.69	0.54376222186845881
+Mothers may not obtain p	Sports                                            	sailing                                           	9.99	205.80	0.06917658220087212
+British figures can tell much white methods. New, french men could think marginally nuclear relatives. Electronic, differ	Sports                                            	sailing                                           	7.39	13316.13	4.47601730584304808
+Real appearances could join miles. A	Sports                                            	sailing                                           	2.44	1182.16	0.39736534700963551
+At present financial areas used to link very purposes. Likely members can retaliate true, blac	Sports                                            	sailing                                           	1.69	7800.18	2.62191347401165555
+Special birds will not answer especially then public walls. Most human areas could require major groups. Particularly diverse children could continue to the readers	Sports                                            	sailing                                           	4.71	7976.59	2.68121104867664997
+Students would rise broad obligations. Good, statistical children would not see. Gradually elegant cases can look w	Sports                                            	sailing                                           	4.63	391.82	0.13170441417855061
+Reliable stages cannot see similarly. Feelings repeat together significant, available notes. Rich, basic roots provide instinctively before the talks. Parties arrest there other investigations. Bom	Sports                                            	sailing                                           	7.89	7983.29	2.68346315315063365
+Demands can imagine also purely fresh eyebrows. Busy skills become almost; complete pa	Sports                                            	sailing                                           	4.98	12443.47	4.18268574013161433
+Proper applications stand now very limited arms. Angrily slow boys shall aid too previous old masses. Mechanical contents think through the times. Sequences may not agree. Old, working stren	Sports                                            	sailing                                           	0.63	679.89	0.22853482250996573
+Successful, able hearts cite then contents. Urban rights will use long important, suspicious ideas; police speak for a methods. Plans seek no longer good gardens	Sports                                            	sailing                                           	4.39	8675.35	2.91608873856334289
+Scientific packages make banks. Then important parents must get front, little bact	Sports                                            	sailing                                           	4.23	6135.42	2.06232937787597103
+Also long ways should not give only now good resources. Previous, economic units s	Sports                                            	sailing                                           	4.65	389.74	0.13100525338662731
+Social years attend. Bloody wee	Sports                                            	sailing                                           	1.94	3178.08	1.06826390845941533
+Capital, foreign problems 	Sports                                            	sailing                                           	3.60	1277.78	0.42950657533834004
+Original, major nations should come once more now permanent feet. Prizes revise thus with the spots. Aside ordinary studies can learn l	Sports                                            	sailing                                           	1.46	7468.82	2.51053178169833686
+Industrial, open sites would throw before a men. Also p	Sports                                            	sailing                                           	7.20	1089.19	0.36611487642064095
+Loose patients used to look at all companies. Old, low centres may illustr	Sports                                            	sailing                                           	6.35	7701.71	2.58881426094401766
+Especially moral students used to keep guilty, bizarre things. Unknown trends reduce later terms; general mothers can find as right n	Sports                                            	sailing                                           	3.35	12086.74	4.06277630296680815
+Origins would come sales. Educational eyes could invite actually stupid, forei	Sports                                            	sailing                                           	3.77	9292.44	3.12351428331716300
+Legal, secondary sales elect. Big years appeal low with a doubts. Military videos might describe; comparable, long companies would not extend now industrial tools. Even ne	Sports                                            	sailing                                           	5.45	1828.50	0.61462284039987695
+Additional organisations will adopt usually schemes. Conventional problems should not create attacks. Generally european powers win very human, busy months; fai	Sports                                            	sailing                                           	0.87	6498.29	2.18430268391693540
+Wrong, local indians train excellent, comprehensive holidays. Meals c	Sports                                            	sailing                                           	60.65	1510.40	0.50769829813506926
+National shareholders learn. Effective proceedings will develop now other, informal days; new, big waves last americans. Solicitors ought to sue flames; interested conservatives might understand just	Sports                                            	sailing                                           	0.24	5784.43	1.94434935558887624
+Ambitious exceptions appoint. V	Sports                                            	sailing                                           	7.35	9044.55	3.04018977912972767
+Proceedings mi	Sports                                            	sailing                                           	7.11	4105.60	1.38003584005782598
+Again standard families change literally. Narrow lips work certainly carefully vast stages. Drugs see also right factors. Financial, western examples ought to let desperately ago sudden	Sports                                            	tennis                                            	9.39	6556.29	1.81601129267527792
+Late global concepts shall understand very quiet, upper heads. Already english buildings make women. Others try. Please minimal agreements conflict largely forthcoming police. 	Sports                                            	tennis                                            	4.33	7426.08	2.05693237186122454
+Seriously complete characteristics make forward in a projects. Industries should rise then also new departments. Physical babies encourage under to a workers. Personal, beautiful ministers cont	Sports                                            	tennis                                            	0.82	14172.38	3.92557408596710262
+Whole, new meetings may last; free plans broaden there mammals. Public, honest areas may risk on a profits. Good, normal generations ought to walk almost over a reductions. Otherwise basic s	Sports                                            	tennis                                            	4.88	8723.48	2.41629613568450044
+Economic, content activit	Sports                                            	tennis                                            	5.07	16087.57	4.45605804375706699
+Women would come fair unaware, current bars. Villages may go then on a neighbours. Early numbers should not change however cr	Sports                                            	tennis                                            	2.92	13912.86	3.85369025369685708
+Women should leave also annual, marginal techniques; intellectual, appropriate factors could think profits. Neverthe	Sports                                            	tennis                                            	8.24	23633.13	6.54608489881669218
+Of course equal nee	Sports                                            	tennis                                            	3.49	11949.65	3.30990534944566741
+Free representatives can fall much prime, useful banks. Recent, secondary practitioners can talk times; libraries take from now on young prices. Bodies appear only yellow rates. Second	Sports                                            	tennis                                            	6.85	7304.83	2.02334762054045053
+Costly offices collect officially for a debts; readers greet. Women get by a write	Sports                                            	tennis                                            	3.22	2864.47	0.79342278446035080
+Rapidly main banks shall not bring extremely decades. For example main clothes might not see less. Certainly co	Sports                                            	tennis                                            	3.15	5004.38	1.38615140465694887
+Just able pounds should join then successful modern pieces. Associated, sorry clubs pay close issues. Resources will e	Sports                                            	tennis                                            	7.67	7567.71	2.09616213128028617
+Necessary times believe probably. Cruel traders know ho	Sports                                            	tennis                                            	92.95	7731.85	2.14162688247032202
+Funny, armed savings go yet thin	Sports                                            	tennis                                            	3.97	3362.82	0.93145957473422897
+Elected, marvellous advisers may not pass all in a programmes. Directly soviet studies could not stress more than; convenient, public	Sports                                            	tennis                                            	4.67	18.70	0.00517966886349257
+Men could remove only; economic, clear children raise public, extensive poli	Sports                                            	tennis                                            	5.04	2721.49	0.75381909172761457
+Able, common villages read. Only social grounds remember e	Sports                                            	tennis                                            	2.08	2677.23	0.74155961879188295
+Successful parties see once on a ideas. Scottish, natural men would not examine regulatory, multiple payments. Steadily loc	Sports                                            	tennis                                            	2.55	8031.03	2.22449604453340795
+Current, 	Sports                                            	tennis                                            	0.47	18310.05	5.07165753336856247
+Years may speak to a	Sports                                            	tennis                                            	2.02	3056.11	0.84650469574375807
+Separate, comfortable consumers get. Tests work even high, different faces. Hars	Sports                                            	tennis                                            	8.09	11878.41	3.29017274998923903
+Much critical possibilities might ensure; hence northern ways may persuade much japanese, running notes. Small, ed	Sports                                            	tennis                                            	8.53	8171.42	2.26338233927916847
+As specific ears worry also labour components. Duly proper articles would attend more easy shapes; years wait head convention	Sports                                            	tennis                                            	0.85	11273.32	3.12257029904748936
+Early, experimental factors mean usually suitable detectives; just black assets must not store only. So british employers must see elaborate, complete pages. Mental years should t	Sports                                            	tennis                                            	88.56	15092.59	4.18046088194969605
+Social, substantial orders would not offset however to a colleagues. Small students give for sure husbands. Subjects shall not make generations; acceptable lights g	Sports                                            	tennis                                            	56.30	5682.58	1.57400442194147617
+I	Sports                                            	tennis                                            	1.04	4973.48	1.37759248658839698
+Automatic amounts may find more in a regulations. Boys can give available, current seasons; here complicated camps may spot even generous open individuals. Channels remain currently protest	Sports                                            	tennis                                            	8.43	3330.22	0.92242977767808685
+Points used to find cool typical managers. However military horses understand indeed inc periods. Coloured developments could make very roots. 	Sports                                            	tennis                                            	8.52	11481.61	3.18026405453288334
+Sides express even new women. Also joint markets should switch authorities. Trees would play always about a skills. Teams deprive future pubs; ways recover national, old days. Rea	Sports                                            	tennis                                            	90.25	3634.02	1.00657862263685918
+Secret children will start in short familie	Sports                                            	tennis                                            	38.21	13612.04	3.77036683190456646
+Other, general countries keep successfully teachers. Major, traditional relationships could not become in a subjects. Constant observers wil	Sports                                            	tennis                                            	99.16	7979.51	2.21022564133302628
+Upper, industrial years shall opera	Sports                                            	tennis                                            	1.58	369.36	0.10230815462136981
+Afraid, spanish matt	Sports                                            	tennis                                            	3.06	141.37	0.03915774263272431
+Light, social animals resist instead then female societies. Also informal minutes shall not implement. Servants win. Hands will a	Sports                                            	tennis                                            	8.30	3341.21	0.92547387183903783
+Modest, educational principles would 	Sports                                            	tennis                                            	6.42	18707.39	5.18171580215038800
+Far little eyes can happen pp.. Related margins will suffer low below active children; times feel just similar, nervous birds. Vegetabl	Sports                                            	tennis                                            	9.01	813.78	0.22540700148304722
+Then various shoes date good, bad shops. Here open rats match badly well dual games. No doubt small kids answer much points. Completely free services shall understand. Following patients	Sports                                            	tennis                                            	5.46	1154.69	0.31983485775327459
+Widely free parties would find in a problems. Men like parties; straight a	Sports                                            	tennis                                            	8.95	10297.10	2.85216942536199653
+Tired rights free. Paintings sell	Sports                                            	tennis                                            	8.06	5429.22	1.50382683353214583
+Meetings improve early women. Even likely variables might want approxi	Sports                                            	tennis                                            	2.56	7342.79	2.03386207134570068
+Growing jews see only grey tactics. Also indian parts ought to provide pretty other, canadian ways. Countries shall correspond really to a doubts. Star sounds ought to mean further at a steps. 	Sports                                            	tennis                                            	8.04	4423.03	1.22512464028307694
+Else single arrangements will not keep approximately from a teachers. Large levels tolerate daily financial, critical others. Properties make a	Sports                                            	tennis                                            	0.30	5475.18	1.51655718545546767
+Equivalent, important points would not reject foreign, high mountains. Always alive cups mark near the games. Sons will not stay extremely. Unfortunatel	Sports                                            	tennis                                            	0.19	5314.97	1.47218099568968454
+Confidential companies can write highly; potentially new children mix sympathetically military, economic gains. Various, traditional designers make in a measurements. Individuals tell only se	Sports                                            	tennis                                            	7.12	1837.86	0.50906450360740392
+Examples show waves. Currently representative farmers should put like a customers. Both full rights practise with a police. Legal re	Sports                                            	tennis                                            	4.24	735.27	0.20366070188557120
+Part	Sports                                            	tennis                                            	6.53	4928.46	1.36512250304644856
+Great, big arts will not let brilliant pp.. Real, new or	Sports                                            	tennis                                            	0.88	13772.83	3.81490367450140978
+Inc presents cannot break often subjects. Of course capital services would pay. Systems cannot	Sports                                            	tennis                                            	9.67	3395.45	0.94049768141956387
+Parts may refuse primarily old holidays. Scottish, good tests handle however for the households; low measurements will remember into a calls; inc, genuine events used to think again r	Sports                                            	tennis                                            	6.88	733.87	0.20327291918990865
+Literary pai	Sports                                            	tennis                                            	2.68	3317.04	0.91877908058606374
+Themes would not reflect on the jeans. Traditional relations would not force mildly smal	Sports                                            	tennis                                            	9.89	1274.76	0.35309276365913303
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q99.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q99.sql.out
new file mode 100644
index 0000000000000..2a7da81704380
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q99.sql.out
@@ -0,0 +1,95 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<substr(w_warehouse_name, 1, 20):string,sm_type:string,cc_name:string,30 days :bigint,31 - 60 days :bigint,61 - 90 days :bigint,91 - 120 days :bigint,>120 days :bigint>
+-- !query output
+Just good amou	EXPRESS                       	Mid Atlantic	1260	1337	1352	0	0
+Just good amou	EXPRESS                       	NY Metro	1297	1268	1203	0	0
+Just good amou	EXPRESS                       	North Midwest	1291	1266	1327	0	0
+Just good amou	LIBRARY                       	Mid Atlantic	932	1025	970	0	0
+Just good amou	LIBRARY                       	NY Metro	918	973	922	0	0
+Just good amou	LIBRARY                       	North Midwest	899	945	970	0	0
+Just good amou	NEXT DAY                      	Mid Atlantic	1273	1356	1306	0	0
+Just good amou	NEXT DAY                      	NY Metro	1301	1304	1226	0	0
+Just good amou	NEXT DAY                      	North Midwest	1158	1250	1247	0	0
+Just good amou	OVERNIGHT                     	Mid Atlantic	1021	961	1055	0	0
+Just good amou	OVERNIGHT                     	NY Metro	852	992	977	0	0
+Just good amou	OVERNIGHT                     	North Midwest	858	993	982	0	0
+Just good amou	REGULAR                       	Mid Atlantic	948	982	993	0	0
+Just good amou	REGULAR                       	NY Metro	910	941	951	0	0
+Just good amou	REGULAR                       	North Midwest	880	948	993	0	0
+Just good amou	TWO DAY                       	Mid Atlantic	958	1030	981	0	0
+Just good amou	TWO DAY                       	NY Metro	891	907	907	0	0
+Just good amou	TWO DAY                       	North Midwest	924	886	955	0	0
+Matches produce	EXPRESS                       	Mid Atlantic	1216	1329	1314	0	0
+Matches produce	EXPRESS                       	NY Metro	1164	1301	1225	0	0
+Matches produce	EXPRESS                       	North Midwest	1246	1265	1264	0	0
+Matches produce	LIBRARY                       	Mid Atlantic	890	962	963	0	0
+Matches produce	LIBRARY                       	NY Metro	891	970	963	0	0
+Matches produce	LIBRARY                       	North Midwest	956	902	964	0	0
+Matches produce	NEXT DAY                      	Mid Atlantic	1339	1308	1245	0	0
+Matches produce	NEXT DAY                      	NY Metro	1218	1280	1177	0	0
+Matches produce	NEXT DAY                      	North Midwest	1265	1249	1309	0	0
+Matches produce	OVERNIGHT                     	Mid Atlantic	928	916	959	0	0
+Matches produce	OVERNIGHT                     	NY Metro	928	937	959	0	0
+Matches produce	OVERNIGHT                     	North Midwest	924	985	923	0	0
+Matches produce	REGULAR                       	Mid Atlantic	937	919	956	0	0
+Matches produce	REGULAR                       	NY Metro	920	970	942	0	0
+Matches produce	REGULAR                       	North Midwest	920	978	1033	0	0
+Matches produce	TWO DAY                       	Mid Atlantic	947	961	1010	0	0
+Matches produce	TWO DAY                       	NY Metro	870	950	1004	0	0
+Matches produce	TWO DAY                       	North Midwest	896	989	883	0	0
+Operations	EXPRESS                       	Mid Atlantic	1282	1274	1361	0	0
+Operations	EXPRESS                       	NY Metro	1183	1267	1206	0	0
+Operations	EXPRESS                       	North Midwest	1182	1297	1234	0	0
+Operations	LIBRARY                       	Mid Atlantic	955	1001	1015	0	0
+Operations	LIBRARY                       	NY Metro	917	948	930	0	0
+Operations	LIBRARY                       	North Midwest	890	926	977	0	0
+Operations	NEXT DAY                      	Mid Atlantic	1197	1322	1291	0	0
+Operations	NEXT DAY                      	NY Metro	1221	1238	1294	0	0
+Operations	NEXT DAY                      	North Midwest	1277	1295	1273	0	0
+Operations	OVERNIGHT                     	Mid Atlantic	904	1021	953	0	0
+Operations	OVERNIGHT                     	NY Metro	923	915	975	0	0
+Operations	OVERNIGHT                     	North Midwest	932	1010	987	0	0
+Operations	REGULAR                       	Mid Atlantic	953	1024	974	0	0
+Operations	REGULAR                       	NY Metro	902	892	901	0	0
+Operations	REGULAR                       	North Midwest	938	942	990	0	0
+Operations	TWO DAY                       	Mid Atlantic	964	990	1011	0	0
+Operations	TWO DAY                       	NY Metro	917	946	886	0	0
+Operations	TWO DAY                       	North Midwest	926	973	980	0	0
+Selective,	EXPRESS                       	Mid Atlantic	1307	1294	1263	0	0
+Selective,	EXPRESS                       	NY Metro	1186	1296	1230	0	0
+Selective,	EXPRESS                       	North Midwest	1203	1289	1271	0	0
+Selective,	LIBRARY                       	Mid Atlantic	1004	959	972	0	0
+Selective,	LIBRARY                       	NY Metro	932	940	931	0	0
+Selective,	LIBRARY                       	North Midwest	925	912	940	0	0
+Selective,	NEXT DAY                      	Mid Atlantic	1267	1309	1327	0	0
+Selective,	NEXT DAY                      	NY Metro	1244	1244	1261	0	0
+Selective,	NEXT DAY                      	North Midwest	1234	1268	1270	0	0
+Selective,	OVERNIGHT                     	Mid Atlantic	978	945	1068	0	0
+Selective,	OVERNIGHT                     	NY Metro	938	963	947	0	0
+Selective,	OVERNIGHT                     	North Midwest	882	936	932	0	0
+Selective,	REGULAR                       	Mid Atlantic	989	948	970	0	0
+Selective,	REGULAR                       	NY Metro	917	972	980	0	0
+Selective,	REGULAR                       	North Midwest	876	937	1001	0	0
+Selective,	TWO DAY                       	Mid Atlantic	951	974	972	0	0
+Selective,	TWO DAY                       	NY Metro	928	1007	934	0	0
+Selective,	TWO DAY                       	North Midwest	968	942	996	0	0
+Significantly	EXPRESS                       	Mid Atlantic	1260	1340	1298	0	0
+Significantly	EXPRESS                       	NY Metro	1231	1326	1236	0	0
+Significantly	EXPRESS                       	North Midwest	1200	1222	1233	0	0
+Significantly	LIBRARY                       	Mid Atlantic	949	1048	965	0	0
+Significantly	LIBRARY                       	NY Metro	908	963	915	0	0
+Significantly	LIBRARY                       	North Midwest	970	984	920	0	0
+Significantly	NEXT DAY                      	Mid Atlantic	1312	1347	1268	0	0
+Significantly	NEXT DAY                      	NY Metro	1198	1251	1190	0	0
+Significantly	NEXT DAY                      	North Midwest	1231	1232	1307	0	0
+Significantly	OVERNIGHT                     	Mid Atlantic	990	973	990	0	0
+Significantly	OVERNIGHT                     	NY Metro	891	925	954	0	0
+Significantly	OVERNIGHT                     	North Midwest	876	971	958	0	0
+Significantly	REGULAR                       	Mid Atlantic	942	1006	913	0	0
+Significantly	REGULAR                       	NY Metro	955	956	957	0	0
+Significantly	REGULAR                       	North Midwest	910	937	1001	0	0
+Significantly	TWO DAY                       	Mid Atlantic	957	1027	1018	0	0
+Significantly	TWO DAY                       	NY Metro	971	972	958	0	0
+Significantly	TWO DAY                       	North Midwest	885	977	919	0	0
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q10a.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q10a.sql.out
new file mode 100644
index 0000000000000..683a462c2ecc6
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q10a.sql.out
@@ -0,0 +1,9 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<cd_gender:string,cd_marital_status:string,cd_education_status:string,cnt1:bigint,cd_purchase_estimate:int,cnt2:bigint,cd_credit_rating:string,cnt3:bigint,cd_dep_count:int,cnt4:bigint,cd_dep_employed_count:int,cnt5:bigint,cd_dep_college_count:int,cnt6:bigint>
+-- !query output
+F	S	Advanced Degree     	1	4500	1	High Risk 	1	2	1	0	1	0	1
+M	D	Unknown             	1	5000	1	Unknown   	1	5	1	4	1	6	1
+M	M	2 yr Degree         	1	2500	1	Unknown   	1	2	1	0	1	4	1
+M	W	Primary             	1	2000	1	Low Risk  	1	2	1	3	1	4	1
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q11.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q11.sql.out
new file mode 100644
index 0000000000000..f8a1d6edcd267
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q11.sql.out
@@ -0,0 +1,99 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<customer_id:string,customer_first_name:string,customer_last_name:string,customer_email_address:string>
+-- !query output
+AAAAAAAAAAECBAAA	Frank               	Wenzel                        	Frank.Wenzel@zhXN6.com                            
+AAAAAAAAABGKAAAA	Jonna               	King                          	Jonna.King@aFu3Mu88XNgLh7lzUbBd.com               
+AAAAAAAAAFAGAAAA	Robert              	Chang                         	Robert.Chang@q5SPPnTKPgA2siE.org                  
+AAAAAAAAAFBNAAAA	Robert              	Baines                        	Robert.Baines@FI8euotCCfA0dfsoy.com               
+AAAAAAAAAGLPAAAA	Charlene            	Marcus                        	Charlene.Marcus@XYRXjq9m6.com                     
+AAAAAAAABAAGAAAA	Luis                	James                         	Luis.James@oLxkv69Mc9.edu                         
+AAAAAAAABBEAAAAA	Jason               	Gallegos                      	Jason.Gallegos@sg0JhLIArOU5lOS.org                
+AAAAAAAABGMHBAAA	Michael             	Gillespie                     	Michael.Gillespie@J63SDK8lTkTx.edu                
+AAAAAAAABIABAAAA	Letha               	Stone                         	Letha.Stone@BkqMc.com                             
+AAAAAAAABILCAAAA	Theresa             	Mullins                       	Theresa.Mullins@96UTbTai7sO.org                   
+AAAAAAAABJEDBAAA	Arthur              	Bryan                         	Arthur.Bryan@ZvCpRQMEbZYcg.org                    
+AAAAAAAABKDKAAAA	Gerald              	Ruiz                          	Gerald.Ruiz@kHcL2q.com                            
+AAAAAAAACEMIAAAA	James               	Hernandez                     	James.Hernandez@gj0dkjapodlS.com                  
+AAAAAAAACGLDAAAA	Angelo              	Sloan                         	Angelo.Sloan@dabad6klflJ.edu                      
+AAAAAAAACKKIAAAA	NULL	NULL	Lorraine.Miller@31f.edu                           
+AAAAAAAACOEHBAAA	Christine           	Gonzalez                      	Christine.Gonzalez@oHMdrqfEDX.org                 
+AAAAAAAACPDFBAAA	Cheryl              	Barry                         	Cheryl.Barry@b4id7Q6XJNsB.edu                     
+AAAAAAAADFJBBAAA	Patrick             	Jones                         	Patrick.Jones@L0aUVXsdRxldn.com                   
+AAAAAAAADHNHBAAA	Patrick             	Cooper                        	Patrick.Cooper@2kXFgCYx14V.edu                    
+AAAAAAAADKMBAAAA	Donald              	Nelson                        	Donald.Nelson@b6TdhXbAelMn8oF.edu                 
+AAAAAAAAEBFHAAAA	Esther              	Merrill                       	Esther.Merrill@hGu.edu                            
+AAAAAAAAEBJNAAAA	Alfred              	Glenn                         	Alfred.Glenn@xCi0.edu                             
+AAAAAAAAEFCEBAAA	Cornelius           	Martino                       	Cornelius.Martino@In2KFInUjUY.com                 
+AAAAAAAAEIAHAAAA	Henry               	Desantis                      	Henry.Desantis@znTqdvjJGag4.edu                   
+AAAAAAAAEIPIAAAA	Luke                	Rios                          	Luke.Rios@NgqF4xn2Qxgm00FR0.com                   
+AAAAAAAAFAIEAAAA	Betty               	Gipson                        	Betty.Gipson@13Lp7iesLn.com                       
+AAAAAAAAFDIMAAAA	Stephanie           	Cowan                         	Stephanie.Cowan@R80Njmu1D1n0d.com                 
+AAAAAAAAFGMHBAAA	Donald              	Coleman                       	Donald.Coleman@S4KL45.org                         
+AAAAAAAAFGNEAAAA	Andrew              	Silva                         	Andrew.Silva@hx4.edu                              
+AAAAAAAAFHNDAAAA	Virgil              	Mcdonald                      	Virgil.Mcdonald@dUD.org                           
+AAAAAAAAFMOKAAAA	Harry               	Brown                         	Harry.Brown@Clj2rtJAo.com                         
+AAAAAAAAFMPPAAAA	Manuel              	Bryant                        	Manuel.Bryant@1LtMa1H0t8B5.edu                    
+AAAAAAAAFOEDAAAA	Lori                	Erwin                         	Lori.Erwin@SkmpHUaEnhHBkQ.com                     
+AAAAAAAAGCGIAAAA	Mae                 	Williams                      	Mae.Williams@mfBvsN8VAQOX21Yh.org                 
+AAAAAAAAGEKLAAAA	Jerilyn             	Walker                        	Jerilyn.Walker@hOIXjGj8unTzQ5J3Um.edu             
+AAAAAAAAGGMHAAAA	Julia               	Fisher                        	Julia.Fisher@eyrOB7M7abp.org                      
+AAAAAAAAGHFDAAAA	Laura               	Roy                           	Laura.Roy@xb1d3mQ2.org                            
+AAAAAAAAGLDMAAAA	Alex                	Norris                        	Alex.Norris@GABnCVFfjXxUV2Q.edu                   
+AAAAAAAAGMFHAAAA	Bruce               	Howe                          	Bruce.Howe@yNj94o0DBJ.com                         
+AAAAAAAAGMGEBAAA	Tamika              	Potts                         	Tamika.Potts@yzUu.edu                             
+AAAAAAAAHBEABAAA	Bonnie              	Cochran                       	Bonnie.Cochran@D3oggm81Joopv.com                  
+AAAAAAAAHEIFBAAA	NULL	Jones                         	Ann.Jones@E1eNB.edu                               
+AAAAAAAAHEPFBAAA	Kathryn             	Kinney                        	Kathryn.Kinney@Stq02g.com                         
+AAAAAAAAHGKLAAAA	Arthur              	Christensen                   	Arthur.Christensen@VFNBhqKt1TAdrr.edu             
+AAAAAAAAHIEIAAAA	William             	Roberts                       	William.Roberts@ObeXEfeXMMgm.org                  
+AAAAAAAAHLEAAAAA	Geneva              	Sims                          	Geneva.Sims@1E0ayoK5qFo.edu                       
+AAAAAAAAHLJCAAAA	Marlene             	Grover                        	Marlene.Grover@F9DZzXQsJNYJ.org                   
+AAAAAAAAHPMLAAAA	Elizabeth           	Kennedy                       	Elizabeth.Kennedy@A0YBfGbsbmc3om.edu              
+AAAAAAAAIANDAAAA	Elva                	Wade                          	Elva.Wade@4xZ1agk4PU9.edu                         
+AAAAAAAAIBBFBAAA	James               	Compton                       	James.Compton@KzmxhGNiTqrp.com                    
+AAAAAAAAIBJDBAAA	Dean                	Velez                         	Dean.Velez@ycRqZT5hVfX8ZZk.org                    
+AAAAAAAAILLJAAAA	Billy               	Ortiz                         	Billy.Ortiz@AEcBAd1rTF.org                        
+AAAAAAAAIODCBAAA	Jennifer            	Crane                         	Jennifer.Crane@Sbzbbg2f7tIl32aDBj.org             
+AAAAAAAAIPGJAAAA	Michael             	NULL	Michael.Connelly@K.edu                            
+AAAAAAAAIPKJAAAA	Charles             	Jones                         	Charles.Jones@aZTRs91tA.org                       
+AAAAAAAAJADIAAAA	Margaret            	Roberts                       	Margaret.Roberts@jp.edu                           
+AAAAAAAAJBELAAAA	Sean                	Busby                         	Sean.Busby@HlbL26U77.edu                          
+AAAAAAAAJCNBBAAA	Johnnie             	Cox                           	Johnnie.Cox@nNTlnRXjr5.edu                        
+AAAAAAAAJDEFAAAA	Loretta             	Serrano                       	Loretta.Serrano@GYZpg38p40VgqS7L9.edu             
+AAAAAAAAJDKKAAAA	Sharon              	Reynolds                      	Sharon.Reynolds@tk5.org                           
+AAAAAAAAJEDJAAAA	David               	Taylor                        	David.Taylor@kxn8ngym6u9XoC.org                   
+AAAAAAAAJGDLAAAA	Fredrick            	Davis                         	Fredrick.Davis@fBIx4ZgRJ2.org                     
+AAAAAAAAJHGFAAAA	Pamela              	Gannon                        	Pamela.Gannon@dx1Vy6KLG.org                       
+AAAAAAAAJIAHAAAA	Shawna              	Delgado                       	Shawna.Delgado@Mu5QaTkI2N4tdINV.org               
+AAAAAAAAJILDBAAA	Erica               	Reynolds                      	Erica.Reynolds@NjAGMPr5SynCgvs.org                
+AAAAAAAAJMIDAAAA	Sally               	Thurman                       	Sally.Thurman@xTyyZ3qRIlqa8oBLYTNm.org            
+AAAAAAAAKAKPAAAA	Carolann            	Royer                         	Carolann.Royer@yVh8tzAJmV.com                     
+AAAAAAAAKLHDAAAA	Brittany            	Knox                          	Brittany.Knox@ldm93PY1oSCUpHZQfy.org              
+AAAAAAAAKMHPAAAA	Robert              	Jones                         	Robert.Jones@rYXHiKMN4A.org                       
+AAAAAAAAKNMEBAAA	Amber               	Gonzalez                      	Amber.Gonzalez@Va5m6mBBm.edu                      
+AAAAAAAALEAHBAAA	Eddie               	Pena                          	Eddie.Pena@2L00HyEmYqdOy.org                      
+AAAAAAAALMAJAAAA	Ileen               	Linn                          	Ileen.Linn@5FUTo7S.org                            
+AAAAAAAALMGGBAAA	Dedra               	Rainey                        	Dedra.Rainey@ik6CKcRSdO6GBi.edu                   
+AAAAAAAALNLABAAA	Janie               	Garcia                        	Janie.Garcia@IpXCI4cANG0F1M.org                   
+AAAAAAAALPHGBAAA	Dorothy             	Heller                        	Dorothy.Heller@dXV5.edu                           
+AAAAAAAAMFMKAAAA	John                	Sanders                       	John.Sanders@g.com                                
+AAAAAAAAMHOLAAAA	Terri               	Cook                          	Terri.Cook@Vz02fJPUlPO.edu                        
+AAAAAAAAMJFAAAAA	Marcus              	Espinal                       	Marcus.Espinal@zoIoG4RpC.com                      
+AAAAAAAAMLOEAAAA	Miguel              	Jackson                       	Miguel.Jackson@i8Me4xM79.org                      
+AAAAAAAANBECBAAA	Michael             	Lombardi                      	Michael.Lombardi@J.com                            
+AAAAAAAANKBBAAAA	Diann               	Saunders                      	Diann.Saunders@g6OYMRl4DEBFz.org                  
+AAAAAAAAOCDCAAAA	Armando             	Jackson                       	Armando.Jackson@IoY0Kf.edu                        
+AAAAAAAAOEDIAAAA	Alexander           	Rich                          	Alexander.Rich@YT4HorlXiEXVj.org                  
+AAAAAAAAOFFIAAAA	Frank               	Milton                        	Frank.Milton@satakl9QHE.edu                       
+AAAAAAAAOJBPAAAA	Jonathan            	Mcbride                       	Jonathan.Mcbride@SjQzPb47cUO.com                  
+AAAAAAAAOMOKAAAA	Laurette            	Gary                          	Laurette.Gary@bSgl2F0kEo2tf.org                   
+AAAAAAAAOOKKAAAA	Deborah             	Early                         	Deborah.Early@Zpi3TmsGBi.edu                      
+AAAAAAAAOPMDAAAA	Peggy               	Smith                         	Peggy.Smith@CfqGXuI6hH.org                        
+AAAAAAAAOPPKAAAA	Tina                	Johnson                       	Tina.Johnson@LlITreSC2jD7.com                     
+AAAAAAAAPAEEBAAA	Audria              	Mattson                       	Audria.Mattson@V6zU0l0A.com                       
+AAAAAAAAPBIGBAAA	Susie               	Zavala                        	Susie.Zavala@C0UOUuL65F7kV.com                    
+AAAAAAAAPEFLAAAA	David               	Martinez                      	David.Martinez@ghefIHRjR1N.com                    
+AAAAAAAAPFKDAAAA	Linda               	Simmons                       	Linda.Simmons@P3L.com                             
+AAAAAAAAPNMGAAAA	Christine           	Olds                          	Christine.Olds@acdIL3Bsp4QnMIc.org
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q12.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q12.sql.out
new file mode 100644
index 0000000000000..e60df322b6489
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q12.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_item_id:string,i_item_desc:string,i_category:string,i_class:string,i_current_price:decimal(7,2),itemrevenue:decimal(17,2),revenueratio:decimal(38,17)>
+-- !query output
+AAAAAAAAAELBAAAA	Precisely elderly bodies	Books                                             	arts                                              	1.40	11.21	0.01417562243122168
+AAAAAAAAALNCAAAA	Great, contemporary workers would not remove of course cultural values. Then due children might see positive seconds. Significant problems w	Books                                             	arts                                              	0.55	515.52	0.65190159462474560
+AAAAAAAABKLDAAAA	Forward psychological plants establish closely yet eastern changes. Likewise necessary techniques might drop. Pleasant operations like lonely things; dogs let regions. Forces might not result clearl	Books                                             	arts                                              	2.43	11462.46	14.49487110552909973
+AAAAAAAADLLDAAAA	Black, following services justify by a investors; dirty, different charts will fly however prizes. Temporary, l	Books                                             	arts                                              	5.56	3400.60	4.30023386615632740
+AAAAAAAAFCIBAAAA	United, important objectives put similarly large, previous phenomena; old, present days receive. Happy detectives assi	Books                                             	arts                                              	1.26	784.30	0.99178774958137022
+AAAAAAAAFFIBAAAA	Naturally new years put serious, negative vehicles. Fin	Books                                             	arts                                              	3.34	3319.96	4.19826043236027781
+AAAAAAAAGHBAAAAA	Hard different differences would not paint even. Together suitable schemes marry directly only open women. Social ca	Books                                             	arts                                              	2.65	229.68	0.29044219090124839
+AAAAAAAAGMFAAAAA	Anonymous, useful women provoke slightly present persons. Ideas ought to cost almost competent, working parties; aspects provide thr	Books                                             	arts                                              	6.73	5752.44	7.27425669029944833
+AAAAAAAAHHEBAAAA	Powerful walls will find; there scottish decades must not	Books                                             	arts                                              	4.16	434.76	0.54977641464745189
+AAAAAAAAIBOCAAAA	Careful privileges ought to live rather to a boards. Possible, broad p	Books                                             	arts                                              	3.93	969.48	1.22595739827125692
+AAAAAAAAICMBAAAA	Aside legitimate decisions may not stand probably sexual g	Books                                             	arts                                              	3.88	349.20	0.44158138742039332
+AAAAAAAAIFPBAAAA	Specially interesting crews continue current, foreign directions; only social men would not call at least political children; circumstances could not understand now in a assessme	Books                                             	arts                                              	2.13	3343.99	4.22864760515441312
+AAAAAAAAIHNAAAAA	Unlikely states take later in general extra inf	Books                                             	arts                                              	0.32	20046.98	25.35043883731064290
+AAAAAAAAJLHBAAAA	Inches may lose from a problems. Firm, other corporations shall protect ashamed, important practices. Materials shall not make then by a police. Weeks used	Books                                             	arts                                              	0.84	11869.78	15.00994822673206253
+AAAAAAAAKHJDAAAA	Relevant lips take so sure, manufacturing 	Books                                             	arts                                              	8.80	5995.28	7.58134037907713537
+AAAAAAAAKHLBAAAA	Extra, primitive weeks look obviou	Books                                             	arts                                              	1.18	425.89	0.53855984275049058
+AAAAAAAALCFBAAAA	More than key reasons should remain. Words used to offer slowly british	Books                                             	arts                                              	0.28	7814.52	9.88186306879843074
+AAAAAAAALGEEAAAA	Children may turn also above, historical aspects. Surveys migh	Books                                             	arts                                              	7.22	544.72	0.68882649872748182
+AAAAAAAALOKCAAAA	Trustees know operations. Now past issues cut today german governments. British lines go critical, individual structures. Tonight adequate problems should no	Books                                             	arts                                              	4.05	152.67	0.19305907908783347
+AAAAAAAANMECAAAA	Floors could not go only for a years. Special reasons shape consequently black, concerned instances. Mutual depths encourage both simple teachers. Cards favour massive 	Books                                             	arts                                              	1.83	503.10	0.63619586486597904
+AAAAAAAAODBEAAAA	Certain customers think exactly already necessary factories. Awkward doubts shall not forget fine	Books                                             	arts                                              	0.30	922.40	1.16642231316314662
+AAAAAAAAOODBAAAA	Deep, big areas take for a facilities. Words could replace certainly cases; lights test. Nevertheless practical arts cross. Fa	Books                                             	arts                                              	7.37	230.48	0.29145383210954253
+AAAAAAAAAJJBAAAA	New, reluctant associations see more different, physical symptoms; useful pounds ought to give. Subjects 	Books                                             	business                                          	9.02	306.85	0.37352072221391094
+AAAAAAAABBLDAAAA	Natural plans might not like n	Books                                             	business                                          	4.29	2484.54	3.02436752540117416
+AAAAAAAABINDAAAA	Years shall want free objects. Old residents use absolutely so residential steps. Letters will share variables. Sure fres	Books                                             	business                                          	40.76	90.28	0.10989555418436330
+AAAAAAAACDDCAAAA	Simple, great shops glance from a years. Lessons deepen here previous clients. Increased, silent flights open more great rocks. Brill	Books                                             	business                                          	8.92	393.75	0.47930188812686144
+AAAAAAAACGIDAAAA	Groups must not put new, civil moves. Correct men laugh slightly total novels. Relatively public girls set even scott	Books                                             	business                                          	3.36	344.10	0.41886420242400767
+AAAAAAAACNEDAAAA	Just young degrees stop posts. More than tight artists buy to a arts. European, essential techniques ought to sell to a offences. Sentences be	Books                                             	business                                          	2.58	184.08	0.22407591508925118
+AAAAAAAADEDAAAAA	Junior, severe restrictions ought to want principles. Sure,	Books                                             	business                                          	9.77	1549.80	1.88653223166732663
+AAAAAAAAEEFDAAAA	Remaining subjects handle even only certain ladies; eagerly literary days could not provide. Very different articles cut then. Boys see out of a houses. Governme	Books                                             	business                                          	9.03	6463.45	7.86779374936777799
+AAAAAAAAEGKCAAAA	Russian windows should see in a weapons. New, considerable branches walk. English regions apply neither alone police; very new years w	Books                                             	business                                          	2.79	1635.60	1.99097439548011320
+AAAAAAAAEKDAAAAA	Long groups used to create more tiny feet; tools used to dare still	Books                                             	business                                          	57.04	10558.62	12.85274032257534413
+AAAAAAAAEPLBAAAA	Drugs must compensate dark, modest houses. Small pubs claim naturally accessible relationships. Distinguished	Books                                             	business                                          	1.66	31.78	0.03868498794837246
+AAAAAAAAFCGDAAAA	Small, capable centres	Books                                             	business                                          	2.98	3219.72	3.91928349267255446
+AAAAAAAAFDLAAAAA	Popular, different parameters might take open, used modules. Prisoners use pretty alternative lovers. Annual, professional others spend once true men. Other, small subsidies seem politically	Books                                             	business                                          	7.25	3862.88	4.70218584789203943
+AAAAAAAAFEGEAAAA	Supreme, free uses handle even in the customers. Other minutes might not make of course social neighbours. So environmental rights come other, able sales	Books                                             	business                                          	8.08	10904.74	13.27406341976510738
+AAAAAAAAFHFCAAAA	Sound, original activities consider quite to a attitudes. In order weak improvements marry available, hard studie	Books                                             	business                                          	71.27	385.84	0.46967324575204627
+AAAAAAAAHDLBAAAA	Classic issues will draw as european, engl	Books                                             	business                                          	75.64	92.64	0.11276832232653319
+AAAAAAAAHJAAAAAA	Again british shareholders see shares. American lives ought to establish horses. Then ideal conservatives might charge even nec	Books                                             	business                                          	2.44	5353.50	6.51667976657054660
+AAAAAAAAIMJAAAAA	Departments could seek now for a commu	Books                                             	business                                          	5.93	6535.44	7.95542535045032467
+AAAAAAAAJFBEAAAA	Paintings must not know primary, royal stands; similar, available others ough	Books                                             	business                                          	0.39	303.68	0.36966196161616580
+AAAAAAAAJJGBAAAA	Most present eyes restore fat, central relationships; again considerable habits must face in a discussions. Engineers help at all direct occasions. Curiously del	Books                                             	business                                          	80.10	2096.55	2.55207713918062566
+AAAAAAAAKKDAAAAA	Children would not mean in favour of a parts. Heavy, whole others shall mean on	Books                                             	business                                          	3.13	6646.96	8.09117581791421695
+AAAAAAAAKNJCAAAA	White fees might combine reports. Tr	Books                                             	business                                          	2.09	500.56	0.60931899205277908
+AAAAAAAALDHBAAAA	Most new weeks go yet members. Also encouraging delegates make publications. Different competitors run resources; somehow common views m	Books                                             	business                                          	1.07	974.26	1.18594198736882801
+AAAAAAAALOMDAAAA	Only new systems might join late speeches. Materials could stay on a benefits. Corporate regulations must crawl definitely practical deaths. Windows might soothe despite a organisations. Old	Books                                             	business                                          	0.67	9075.35	11.04719337247520503
+AAAAAAAAMBECAAAA	Professional managers form later initial grounds. Conscious, big risks restore. American, full rises say for a systems. Already	Books                                             	business                                          	5.27	890.13	1.08353267219901759
+AAAAAAAAMKGDAAAA	Memories can earn particularly over quick contexts; alone differences make separate years; irish men mea	Books                                             	business                                          	4.23	2059.92	2.50748836924516678
+AAAAAAAANJLBAAAA	Only, gothic	Books                                             	business                                          	1.68	4777.17	5.81512787530920297
+AAAAAAAAOAPAAAAA	Silver, critical operations could help howev	Books                                             	business                                          	5.56	428.54	0.52165087273113702
+AAAAAAAAALNBAAAA	Else substantial problems slip months. Just unique corporations put vast areas. Supporters like far perfect chapters. Now young reports become wrong trials. Available ears shall	Books                                             	computers                                         	51.46	2456.28	1.26602601850774711
+AAAAAAAACFMBAAAA	At least remaining results shall keep cuts. Clients should meet policies. Glorious, local times could use enough; clever styles will live political parents. Single, gradual contracts will describe ho	Books                                             	computers                                         	9.51	10252.90	5.28459221471415324
+AAAAAAAACOFCAAAA	Years learn here. Days make too. Only moving systems avoid old groups; short movements cannot see respectiv	Books                                             	computers                                         	0.60	1761.68	0.90801240749618444
+AAAAAAAADAHAAAAA	Ga	Books                                             	computers                                         	5.53	7541.48	3.88706087988983530
+AAAAAAAADDBAAAAA	S	Books                                             	computers                                         	65.78	4566.02	2.35343695385979752
+AAAAAAAAECHAAAAA	Boxes batt	Books                                             	computers                                         	0.83	7844.04	4.04300760915510798
+AAAAAAAAEJECAAAA	Artists make times. Rather ready functions must pre	Books                                             	computers                                         	5.71	3694.01	1.90398194531071494
+AAAAAAAAFDPCAAAA	Limited, capable cities shall try during the bodies. Specially economic services ought to prevent old area	Books                                             	computers                                         	2.93	96.18	0.04957349425150028
+AAAAAAAAFHNAAAAA	Legs throw then. Old-fashioned develo	Books                                             	computers                                         	2.66	718.55	0.37035801928067716
+AAAAAAAAFOCEAAAA	Important, educational variables used to appear months. A	Books                                             	computers                                         	2.47	9922.02	5.11404867366677942
+AAAAAAAAGHEAAAAA	Men should not turn shadows. Different, single concessions guarantee only therefore alone products.	Books                                             	computers                                         	8.38	4194.24	2.16181256528813215
+AAAAAAAAGIFEAAAA	Educational, white teachers should not fix. Considerable, other services might not cover today on a forms. Successful genes fall otherwise so	Books                                             	computers                                         	1.65	14569.68	7.50956485471198434
+AAAAAAAAHGCEAAAA	Present 	Books                                             	computers                                         	2.84	12393.53	6.38792460190056468
+AAAAAAAAHHFDAAAA	Multiple, dark feet mean more complex girls; schools may not answer frequently blue assets. Spiritual, dry patients may reply personnel	Books                                             	computers                                         	2.04	371.40	0.19142852739662305
+AAAAAAAAIBDEAAAA	Private teachers ap	Books                                             	computers                                         	5.27	4911.39	2.53144899076602182
+AAAAAAAAIDCDAAAA	Daily numbers sense interesting players. General advantages would speak here. Shelves shall know with the reductions. Again wrong mothers provide ways; as hot pr	Books                                             	computers                                         	7.56	689.26	0.35526124607807325
+AAAAAAAAIECAAAAA	Inc, corporate ships slow evident degrees. Chosen, acute prices throw always. Budgets spend points. Commonly large events may mean. Bottles c	Books                                             	computers                                         	68.38	4.17	0.00214931868401701
+AAAAAAAAIOKCAAAA	However old hours ma	Books                                             	computers                                         	8.84	451.53	0.23272946412330966
+AAAAAAAAJDOCAAAA	Indeed other actions should provide after a ideas; exhibitio	Books                                             	computers                                         	6.95	8062.32	4.15551439149257400
+AAAAAAAAKDGEAAAA	Perfect days find at all. Crimes might develop hopes. Much socialist grants drive current, useful walls. Emissions open naturally. Combinations shall not know. Tragic things shall not receive just	Books                                             	computers                                         	6.71	526.49	0.27136565802113105
+AAAAAAAAKOKBAAAA	Human windows take right, variable steps. Years should buy often. Indeed thin figures may beat even up to a cars. Details may tell enough. Impossible, sufficient differences ought to return 	Books                                             	computers                                         	4.47	1542.60	0.79509328584283986
+AAAAAAAAKPNDAAAA	Left diff	Books                                             	computers                                         	0.74	5248.81	2.70536340572070289
+AAAAAAAAMDKBAAAA	Friendly, hot computers tax elsewhere units. New, real officials should l	Books                                             	computers                                         	3.19	12378.72	6.38029117031536278
+AAAAAAAAMENDAAAA	Kinds relieve really major practices. Then capable reserves could not approve foundations. Pos	Books                                             	computers                                         	7.23	1786.41	0.92075884659828053
+AAAAAAAAMJJCAAAA	Visible, average words shall not want quite only public boundaries. Cars must telephone proposals. German things ask abilities. Windows cut again favorite offi	Books                                             	computers                                         	6.75	25255.89	13.01749550563031296
+AAAAAAAANANCAAAA	Only increased errors must submit as rich, main 	Books                                             	computers                                         	6.94	2429.79	1.25237243291071818
+AAAAAAAANFHDAAAA	Meals ought to test. Round days might need most urban years. Political, english pages must see on a eyes. Only subsequent women may come better methods; difficult, social childr	Books                                             	computers                                         	7.23	6457.72	3.32846480866914548
+AAAAAAAANHFDAAAA	Systems cannot see fairly practitioners. Little ca	Books                                             	computers                                         	1.73	6197.59	3.19438752586978211
+AAAAAAAANKLDAAAA	Past beautiful others might not like more than legislative, small products. Close, wh	Books                                             	computers                                         	3.02	10232.30	5.27397447733028024
+AAAAAAAAOGDDAAAA	Main problems wait properly. Everyday, foreign offenders can worry activities. Social, important shoes will afford okay physical parts. Very	Books                                             	computers                                         	1.40	2034.30	1.04852733786470188
+AAAAAAAAOGMDAAAA	Schools offer quickly others. Further main buildings satisfy sadly great, productive figures. Years contribute acti	Books                                             	computers                                         	4.11	3975.90	2.04927485750197523
+AAAAAAAAOMDAAAAA	Tiny, rare leaders mention old, precious areas; students will improve much multiple stars. Even confident solutions will include clearly single women. Please little rights will not mention harder com	Books                                             	computers                                         	1.45	11902.91	6.13504720795513872
+AAAAAAAAONDCAAAA	Guidelines should investigate so. Usual personnel look now old, modern aspects. Discussions could appear once br	Books                                             	computers                                         	2.44	640.06	0.32990237815154161
+AAAAAAAAONHDAAAA	Flat pleasant groups would go private, redundant eyes. Main devic	Books                                             	computers                                         	2.83	8359.41	4.30864175068552700
+AAAAAAAAOPNBAAAA	Fine users go for a networks. Sympathetic, effective industries could not alter particularly other concepts; wooden women used to feel however 	Books                                             	computers                                         	5.30	247.79	0.12771694885193653
+AAAAAAAAPAKAAAAA	Real, domestic facilities turn often guilty symptoms. Winds get naturally intense islands. Products shall not travel a little clear shares; improved children may not apply wrong c	Books                                             	computers                                         	5.28	297.60	0.15339022550682558
+AAAAAAAAABIBAAAA	Then irish champions must look now forward good women. Future, big models sign. Then different o	Books                                             	cooking                                           	85.81	6496.48	10.66582432143788856
+AAAAAAAAAGHBAAAA	Valuable studies should persist so concerned parties. Always polite songs include then from the holes. There conventional areas might not explain theore	Books                                             	cooking                                           	1.58	2088.03	3.42809662430915734
+AAAAAAAAAIJCAAAA	Meanings occur in a things. Also essential features may not satisfy by the potatoes; happy words change childre	Books                                             	cooking                                           	3.46	1496.40	2.45676728237440221
+AAAAAAAAAJDBAAAA	Then dominant goods should combine probably american items. Important artists guess only sill	Books                                             	cooking                                           	6.67	5638.06	9.25648312220250073
+AAAAAAAADDNAAAAA	Individual, remarkable services take by the interest	Books                                             	cooking                                           	6.05	0.00	0.00000000000000000
+AAAAAAAAEGFEAAAA	Ultimately senior elections marry at l	Books                                             	cooking                                           	5.06	2060.48	3.38286544372280691
+AAAAAAAAEINDAAAA	Hence young effects shall not solve however months. In order small activities must not return almost national foods. International decades take contributions. Sessions must see 	Books                                             	cooking                                           	1.43	242.28	0.39777170353760369
+AAAAAAAAENDCAAAA	Points trace so simple eyes. Short advisers shall not say limitations. Keys stretch in full now blue wings. Immediately strategic students would not make strangely for the players.	Books                                             	cooking                                           	1.69	12518.00	20.55186637313737424
+AAAAAAAAFDIAAAAA	Great pp. will not r	Books                                             	cooking                                           	1.91	7268.22	11.93285558480304571
+AAAAAAAAGFGCAAAA	Products may not resist further specif	Books                                             	cooking                                           	5.37	2.72	0.00446565557876128
+AAAAAAAAICAAAAAA	Somet	Books                                             	cooking                                           	7.34	580.58	0.95318761614603744
+AAAAAAAAIFHDAAAA	Genetic properties might describe therefore leaves; right other organisers must not talk even lives; methods carry thus wrong minutes. Proud worke	Books                                             	cooking                                           	1.08	1445.15	2.37262579398781566
+AAAAAAAAIHHDAAAA	Urgent agencies mean over as a plants; then	Books                                             	cooking                                           	6.47	1312.79	2.15531911295662354
+AAAAAAAAILAEAAAA	As light hundreds must establish on a services. Sometimes special results 	Books                                             	cooking                                           	44.82	3513.30	5.76808372972867366
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q14.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q14.sql.out
new file mode 100644
index 0000000000000..46b0ab1a3018d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q14.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<channel:string,i_brand_id:int,i_class_id:int,i_category_id:int,sales:decimal(28,2),number_sales:bigint,channel:string,i_brand_id:int,i_class_id:int,i_category_id:int,sales:decimal(28,2),number_sales:bigint>
+-- !query output
+store	1001001	1	1	1217789.46	331	store	1001001	1	1	1234065.50	342
+store	1001002	1	1	553751.85	151	store	1001002	1	1	559183.12	158
+store	1002001	2	1	1249466.39	353	store	1002001	2	1	1576146.08	368
+store	1002002	2	1	547670.08	152	store	1002002	2	1	491380.52	130
+store	1003001	3	1	1167282.59	309	store	1003001	3	1	1045547.02	287
+store	1003002	3	1	789483.23	193	store	1003002	3	1	601149.89	165
+store	1004001	4	1	1522903.35	398	store	1004001	4	1	1062756.03	278
+store	1004002	4	1	541183.58	151	store	1004002	4	1	607217.67	151
+store	2001001	1	2	1145363.89	364	store	2001001	1	2	1303027.45	345
+store	2001002	1	2	627833.76	169	store	2001002	1	2	536535.24	161
+store	2002001	2	2	1440545.64	383	store	2002001	2	2	1329963.82	368
+store	2002002	2	2	747435.75	207	store	2002002	2	2	816248.81	208
+store	2003001	3	2	1388229.40	395	store	2003001	3	2	1469176.64	414
+store	2003002	3	2	716344.82	190	store	2003002	3	2	729626.91	181
+store	2004001	4	2	1613653.10	440	store	2004001	4	2	1488785.35	406
+store	2004002	4	2	657357.03	182	store	2004002	4	2	593298.47	162
+store	3001001	1	3	1282986.35	374	store	3001001	1	3	1519875.76	390
+store	3001002	1	3	673633.15	169	store	3001002	1	3	656845.65	177
+store	3002001	2	3	1333021.08	362	store	3002001	2	3	1241035.46	351
+store	3002002	2	3	748022.41	210	store	3002002	2	3	699267.97	190
+store	3003001	3	3	1134792.91	326	store	3003001	3	3	1020180.77	305
+store	3003002	3	3	802127.48	197	store	3003002	3	3	703596.57	195
+store	3004001	4	3	1385106.21	399	store	3004001	4	3	1338923.03	363
+store	3004002	4	3	732944.48	203	store	3004002	4	3	722668.95	194
+store	4001001	1	4	1157990.42	340	store	4001001	1	4	1349185.21	367
+store	4001002	1	4	621674.47	152	store	4001002	1	4	549243.01	154
+store	4002001	2	4	1568059.38	402	store	4002001	2	4	1452439.59	374
+store	4002002	2	4	694485.13	177	store	4002002	2	4	802206.88	214
+store	4003001	3	4	1593249.87	451	store	4003001	3	4	1456053.57	403
+store	4003002	3	4	662121.36	163	store	4003002	3	4	717341.48	189
+store	4004001	4	4	1248825.41	348	store	4004001	4	4	1389020.06	347
+store	4004002	4	4	718297.35	201	store	4004002	4	4	708937.79	199
+store	5001001	1	5	1273579.39	349	store	5001001	1	5	1253638.00	334
+store	5001002	1	5	825892.06	217	store	5001002	1	5	702168.77	206
+store	5002001	2	5	1486176.43	365	store	5002001	2	5	1243594.81	313
+store	5002002	2	5	633123.05	172	store	5002002	2	5	765400.35	202
+store	5003001	3	5	1477157.87	390	store	5003001	3	5	1314942.13	338
+store	5003002	3	5	697211.48	193	store	5003002	3	5	611056.63	180
+store	5004001	4	5	1287387.74	353	store	5004001	4	5	1459943.10	376
+store	5004002	4	5	587935.16	160	store	5004002	4	5	554584.12	154
+store	6001001	1	6	83017.27	25	store	6001001	1	6	53839.44	19
+store	6001002	1	6	38797.04	9	store	6001002	1	6	40326.43	9
+store	6001003	1	6	50457.02	12	store	6001003	1	6	53585.59	10
+store	6001004	1	6	85147.15	18	store	6001004	1	6	88513.17	27
+store	6001005	1	6	174523.58	45	store	6001005	1	6	96635.15	31
+store	6001006	1	6	61231.20	15	store	6001006	1	6	52077.07	12
+store	6001007	1	6	69263.80	19	store	6001007	1	6	51940.50	18
+store	6001008	1	6	7774.49	5	store	6001008	1	6	34176.00	11
+store	6002001	2	6	137288.11	36	store	6002001	2	6	158530.44	40
+store	6002002	2	6	32548.73	11	store	6002002	2	6	87976.59	21
+store	6002003	2	6	73606.51	23	store	6002003	2	6	75815.97	30
+store	6002004	2	6	53750.06	10	store	6002004	2	6	47235.26	13
+store	6002005	2	6	102178.51	28	store	6002005	2	6	65676.11	26
+store	6002006	2	6	54942.99	11	store	6002006	2	6	19627.43	5
+store	6002007	2	6	90084.17	30	store	6002007	2	6	92767.58	27
+store	6002008	2	6	39639.50	9	store	6002008	2	6	22406.00	6
+store	6003001	3	6	51483.85	13	store	6003001	3	6	54481.52	8
+store	6003003	3	6	47337.51	12	store	6003003	3	6	45051.71	13
+store	6003004	3	6	10107.64	5	store	6003004	3	6	32499.05	7
+store	6003005	3	6	66634.50	24	store	6003005	3	6	108128.10	28
+store	6003006	3	6	48367.44	18	store	6003006	3	6	40436.80	10
+store	6003007	3	6	81724.63	24	store	6003007	3	6	53676.25	15
+store	6003008	3	6	38023.42	10	store	6003008	3	6	86371.33	18
+store	6004001	4	6	46759.68	16	store	6004001	4	6	98037.79	25
+store	6004002	4	6	32304.25	8	store	6004002	4	6	31274.82	9
+store	6004003	4	6	69089.26	18	store	6004003	4	6	70840.13	21
+store	6004004	4	6	40891.73	9	store	6004004	4	6	37496.52	13
+store	6004005	4	6	47341.21	14	store	6004005	4	6	11576.16	4
+store	6004006	4	6	37823.00	15	store	6004006	4	6	95562.26	20
+store	6004007	4	6	107423.33	25	store	6004007	4	6	49625.35	14
+store	6004008	4	6	29725.16	11	store	6004008	4	6	80015.81	19
+store	6005001	5	6	57198.07	20	store	6005001	5	6	61313.19	22
+store	6005002	5	6	26742.25	7	store	6005002	5	6	8143.14	3
+store	6005003	5	6	220162.11	49	store	6005003	5	6	156718.19	37
+store	6005004	5	6	11571.97	5	store	6005004	5	6	47370.39	12
+store	6005005	5	6	98809.34	31	store	6005005	5	6	139898.64	34
+store	6005006	5	6	18108.22	5	store	6005006	5	6	23139.57	5
+store	6005007	5	6	74967.30	18	store	6005007	5	6	49412.83	17
+store	6005008	5	6	71048.66	14	store	6005008	5	6	35086.91	13
+store	6006001	6	6	33781.32	9	store	6006001	6	6	33793.98	13
+store	6006003	6	6	97264.81	19	store	6006003	6	6	68109.24	28
+store	6006004	6	6	68406.20	16	store	6006004	6	6	45670.46	17
+store	6006005	6	6	56934.37	15	store	6006005	6	6	80912.54	19
+store	6006006	6	6	9360.35	4	store	6006006	6	6	63735.59	18
+store	6006007	6	6	15367.23	7	store	6006007	6	6	35007.03	7
+store	6006008	6	6	32684.13	6	store	6006008	6	6	17541.68	5
+store	6007001	7	6	20878.30	5	store	6007001	7	6	31745.91	6
+store	6007002	7	6	22897.56	7	store	6007002	7	6	49259.06	10
+store	6007003	7	6	23239.07	10	store	6007003	7	6	58808.25	19
+store	6007004	7	6	47157.82	10	store	6007004	7	6	35948.03	11
+store	6007005	7	6	108100.32	27	store	6007005	7	6	61513.57	21
+store	6007006	7	6	49256.53	10	store	6007006	7	6	16532.02	5
+store	6007007	7	6	65032.83	21	store	6007007	7	6	110874.54	26
+store	6007008	7	6	56626.18	12	store	6007008	7	6	88630.08	17
+store	6008001	8	6	138363.66	40	store	6008001	8	6	118280.46	34
+store	6008002	8	6	59363.03	20	store	6008002	8	6	35498.23	9
+store	6008004	8	6	48538.79	12	store	6008004	8	6	15279.50	6
+store	6008005	8	6	107128.07	41	store	6008005	8	6	174087.69	43
+store	6008006	8	6	18420.16	7	store	6008006	8	6	19669.85	8
+store	6008007	8	6	33281.27	10	store	6008007	8	6	50246.87	12
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q14a.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q14a.sql.out
new file mode 100644
index 0000000000000..027c177ab7710
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q14a.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<channel:string,i_brand_id:int,i_class_id:int,i_category_id:int,sum_sales:decimal(38,2),number_sales:bigint>
+-- !query output
+NULL	NULL	NULL	NULL	677178449.86	157050
+catalog	NULL	NULL	NULL	235662013.22	46193
+catalog	1001001	NULL	NULL	2553656.44	520
+catalog	1001001	1	NULL	2222983.82	453
+catalog	1001001	1	1	1860468.47	369
+catalog	1001001	1	2	33739.75	3
+catalog	1001001	1	3	49979.66	13
+catalog	1001001	1	4	49610.76	10
+catalog	1001001	1	5	61333.54	13
+catalog	1001001	1	6	31767.56	6
+catalog	1001001	1	7	19752.34	6
+catalog	1001001	1	8	43708.18	12
+catalog	1001001	1	9	36388.85	11
+catalog	1001001	1	10	36234.71	10
+catalog	1001001	2	NULL	41208.06	6
+catalog	1001001	2	3	41208.06	6
+catalog	1001001	3	NULL	85518.68	19
+catalog	1001001	3	1	11946.51	5
+catalog	1001001	3	4	32093.25	7
+catalog	1001001	3	6	6905.04	2
+catalog	1001001	3	7	34573.88	5
+catalog	1001001	4	NULL	22702.37	6
+catalog	1001001	4	7	22702.37	6
+catalog	1001001	5	NULL	15433.39	3
+catalog	1001001	5	9	15433.39	3
+catalog	1001001	8	NULL	22550.38	4
+catalog	1001001	8	9	22550.38	4
+catalog	1001001	10	NULL	29206.10	5
+catalog	1001001	10	7	29206.10	5
+catalog	1001001	13	NULL	9727.21	3
+catalog	1001001	13	9	9727.21	3
+catalog	1001001	14	NULL	80584.37	15
+catalog	1001001	14	9	17298.44	3
+catalog	1001001	14	10	63285.93	12
+catalog	1001001	16	NULL	23742.06	6
+catalog	1001001	16	9	23742.06	6
+catalog	1001002	NULL	NULL	2674718.24	515
+catalog	1001002	1	NULL	2413496.48	456
+catalog	1001002	1	1	2413496.48	456
+catalog	1001002	2	NULL	40653.02	10
+catalog	1001002	2	1	40653.02	10
+catalog	1001002	3	NULL	55083.13	11
+catalog	1001002	3	1	55083.13	11
+catalog	1001002	4	NULL	59222.69	15
+catalog	1001002	4	1	59222.69	15
+catalog	1001002	7	NULL	23174.76	3
+catalog	1001002	7	1	23174.76	3
+catalog	1001002	10	NULL	40160.58	11
+catalog	1001002	10	1	40160.58	11
+catalog	1001002	12	NULL	25963.72	6
+catalog	1001002	12	1	25963.72	6
+catalog	1001002	13	NULL	11354.62	2
+catalog	1001002	13	1	11354.62	2
+catalog	1001002	14	NULL	5609.24	1
+catalog	1001002	14	1	5609.24	1
+catalog	1002001	NULL	NULL	2907063.96	511
+catalog	1002001	1	NULL	88041.35	10
+catalog	1002001	1	1	38209.40	6
+catalog	1002001	1	5	49831.95	4
+catalog	1002001	2	NULL	2574507.86	463
+catalog	1002001	2	1	2132551.78	377
+catalog	1002001	2	3	34961.36	11
+catalog	1002001	2	4	50761.02	9
+catalog	1002001	2	6	99382.39	16
+catalog	1002001	2	7	43547.86	4
+catalog	1002001	2	8	104272.59	23
+catalog	1002001	2	9	62741.20	11
+catalog	1002001	2	10	46289.66	12
+catalog	1002001	3	NULL	44220.16	5
+catalog	1002001	3	1	44220.16	5
+catalog	1002001	4	NULL	86121.32	15
+catalog	1002001	4	1	28141.23	5
+catalog	1002001	4	3	29119.15	5
+catalog	1002001	4	5	28860.94	5
+catalog	1002001	5	NULL	26371.70	2
+catalog	1002001	5	9	26371.70	2
+catalog	1002001	6	NULL	32428.43	4
+catalog	1002001	6	10	32428.43	4
+catalog	1002001	8	NULL	19630.88	3
+catalog	1002001	8	8	19630.88	3
+catalog	1002001	10	NULL	20942.49	5
+catalog	1002001	10	8	20942.49	5
+catalog	1002001	11	NULL	6069.54	2
+catalog	1002001	11	8	6069.54	2
+catalog	1002001	12	NULL	8730.23	2
+catalog	1002001	12	7	8730.23	2
+catalog	1002002	NULL	NULL	2695862.91	567
+catalog	1002002	1	NULL	79482.49	21
+catalog	1002002	1	1	79482.49	21
+catalog	1002002	2	NULL	2153480.10	444
+catalog	1002002	2	1	2153480.10	444
+catalog	1002002	3	NULL	64329.01	20
+catalog	1002002	3	1	64329.01	20
+catalog	1002002	4	NULL	98180.19	26
+catalog	1002002	4	1	98180.19	26
+catalog	1002002	7	NULL	45311.61	6
+catalog	1002002	7	1	45311.61	6
+catalog	1002002	8	NULL	16754.12	4
+catalog	1002002	8	1	16754.12	4
+catalog	1002002	9	NULL	30603.32	7
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q18a.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q18a.sql.out
new file mode 100644
index 0000000000000..40e6ac10e6a33
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q18a.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_item_id:string,ca_country:string,ca_state:string,ca_county:string,agg1:decimal(16,6),agg2:decimal(16,6),agg3:decimal(16,6),agg4:decimal(16,6),agg5:decimal(16,6),agg6:decimal(16,6),agg7:decimal(16,6)>
+-- !query output
+NULL	NULL	NULL	NULL	49.988822	101.021588	257.990037	51.117224	-216.279125	1957.948730	3.062849
+AAAAAAAAAAAEAAAA	NULL	NULL	NULL	83.000000	14.310000	0.000000	7.580000	-377.650000	1980.000000	5.000000
+AAAAAAAAAABEAAAA	NULL	NULL	NULL	45.000000	57.980000	0.000000	44.640000	-95.400000	1951.000000	1.000000
+AAAAAAAAAADBAAAA	NULL	NULL	NULL	19.000000	232.670000	0.000000	62.820000	-679.630000	1933.000000	6.000000
+AAAAAAAAAAEDAAAA	NULL	NULL	NULL	57.000000	119.920000	0.000000	119.920000	3329.940000	1946.000000	6.000000
+AAAAAAAAAAFAAAAA	NULL	NULL	NULL	83.000000	141.350000	0.000000	12.720000	-6142.000000	1991.000000	4.000000
+AAAAAAAAAAHBAAAA	NULL	NULL	NULL	97.000000	83.210000	0.000000	71.560000	431.650000	1924.000000	3.000000
+AAAAAAAAAAHDAAAA	NULL	NULL	NULL	53.500000	97.390000	0.000000	52.640000	-6.425000	1974.000000	2.500000
+AAAAAAAAAAKAAAAA	NULL	NULL	NULL	69.000000	127.490000	0.000000	101.990000	2660.640000	1948.000000	2.000000
+AAAAAAAAAALAAAAA	NULL	NULL	NULL	88.000000	143.850000	0.000000	28.770000	-2337.280000	1944.000000	5.000000
+AAAAAAAAAALDAAAA	NULL	NULL	NULL	19.000000	116.590000	0.000000	5.820000	-1366.290000	1949.000000	2.000000
+AAAAAAAAAAMBAAAA	NULL	NULL	NULL	1.000000	37.170000	0.000000	30.100000	13.430000	1932.000000	2.000000
+AAAAAAAAAAOAAAAA	NULL	NULL	NULL	87.000000	26.050000	0.000000	1.820000	-963.960000	1985.000000	4.000000
+AAAAAAAAABAEAAAA	NULL	NULL	NULL	91.000000	67.860000	56.180000	8.820000	-4041.070000	1970.000000	0.000000
+AAAAAAAAABBDAAAA	NULL	NULL	NULL	71.000000	31.880000	0.000000	15.300000	-23.430000	1979.000000	6.000000
+AAAAAAAAABCEAAAA	NULL	NULL	NULL	65.000000	115.550000	0.000000	25.420000	-1641.900000	1973.000000	6.000000
+AAAAAAAAABDAAAAA	NULL	NULL	NULL	71.000000	40.670000	0.000000	13.820000	-269.090000	1968.000000	0.000000
+AAAAAAAAABFCAAAA	NULL	NULL	NULL	2.000000	237.540000	0.000000	35.630000	-101.500000	1932.000000	1.000000
+AAAAAAAAABGBAAAA	NULL	NULL	NULL	25.500000	149.375000	0.000000	87.680000	-85.130000	1955.000000	4.000000
+AAAAAAAAABGEAAAA	NULL	NULL	NULL	32.500000	237.875000	0.000000	107.305000	615.725000	1969.000000	3.000000
+AAAAAAAAABJDAAAA	NULL	NULL	NULL	12.000000	118.140000	0.000000	54.340000	-405.960000	1971.000000	4.000000
+AAAAAAAAACAAAAAA	NULL	NULL	NULL	40.000000	218.900000	0.000000	168.550000	3400.000000	1984.000000	2.000000
+AAAAAAAAACDCAAAA	NULL	NULL	NULL	9.000000	27.400000	0.000000	18.630000	-11.070000	1982.000000	0.000000
+AAAAAAAAACDDAAAA	NULL	NULL	NULL	9.000000	72.980000	0.000000	36.490000	85.140000	1925.000000	4.000000
+AAAAAAAAACGDAAAA	NULL	NULL	NULL	100.000000	240.000000	0.000000	52.800000	-3171.000000	1984.000000	2.000000
+AAAAAAAAACJAAAAA	NULL	NULL	NULL	89.000000	141.410000	0.000000	91.910000	3394.460000	1939.000000	5.000000
+AAAAAAAAACJCAAAA	NULL	NULL	NULL	97.000000	68.650000	0.000000	60.410000	2160.190000	1979.000000	5.000000
+AAAAAAAAACMAAAAA	NULL	NULL	NULL	36.000000	5.030000	0.000000	1.200000	-23.400000	1973.000000	6.000000
+AAAAAAAAACPDAAAA	NULL	NULL	NULL	89.000000	165.880000	0.000000	48.100000	-1248.670000	1988.000000	5.000000
+AAAAAAAAADDEAAAA	NULL	NULL	NULL	44.000000	152.720000	0.000000	138.970000	3326.400000	1930.000000	6.000000
+AAAAAAAAADFAAAAA	NULL	NULL	NULL	52.000000	99.460000	465.420000	59.670000	242.820000	1945.000000	2.000000
+AAAAAAAAADGBAAAA	NULL	NULL	NULL	51.000000	48.560000	0.000000	12.620000	-1473.390000	1935.000000	3.000000
+AAAAAAAAADIAAAAA	NULL	NULL	NULL	27.000000	175.020000	0.000000	47.250000	-388.260000	1924.000000	3.000000
+AAAAAAAAADMCAAAA	NULL	NULL	NULL	89.000000	161.440000	0.000000	35.510000	-4648.470000	1924.000000	6.000000
+AAAAAAAAADNAAAAA	NULL	NULL	NULL	57.000000	107.880000	975.770000	24.810000	-2503.940000	1975.000000	2.000000
+AAAAAAAAADNBAAAA	NULL	NULL	NULL	32.000000	52.410000	0.000000	33.540000	274.560000	1975.000000	5.000000
+AAAAAAAAADPBAAAA	NULL	NULL	NULL	72.000000	65.940000	3.770000	1.310000	-3562.010000	1932.000000	5.000000
+AAAAAAAAAEABAAAA	NULL	NULL	NULL	57.000000	200.610000	0.000000	24.070000	-4047.570000	1987.000000	5.000000
+AAAAAAAAAEADAAAA	NULL	NULL	NULL	61.000000	28.010000	0.000000	18.480000	85.400000	1970.000000	2.000000
+AAAAAAAAAEAEAAAA	NULL	NULL	NULL	95.000000	155.800000	0.000000	4.670000	-5184.150000	1966.000000	4.000000
+AAAAAAAAAEBDAAAA	NULL	NULL	NULL	57.000000	257.970000	0.000000	36.110000	-2994.780000	1990.000000	4.000000
+AAAAAAAAAECCAAAA	NULL	NULL	NULL	40.000000	81.220000	0.000000	51.160000	746.800000	1975.000000	1.000000
+AAAAAAAAAEDAAAAA	NULL	NULL	NULL	63.000000	31.950000	564.530000	14.690000	-809.600000	1966.000000	6.000000
+AAAAAAAAAEJAAAAA	NULL	NULL	NULL	83.000000	48.220000	202.630000	6.260000	-2670.220000	1944.000000	3.000000
+AAAAAAAAAEJDAAAA	NULL	NULL	NULL	4.000000	39.860000	0.000000	0.790000	-66.480000	1958.000000	3.000000
+AAAAAAAAAELCAAAA	NULL	NULL	NULL	34.000000	106.700000	0.000000	17.070000	-1481.040000	1925.000000	1.000000
+AAAAAAAAAENAAAAA	NULL	NULL	NULL	49.000000	205.770000	0.000000	141.980000	3018.400000	1926.000000	2.000000
+AAAAAAAAAENCAAAA	NULL	NULL	NULL	67.000000	121.280000	0.000000	115.210000	4025.360000	1947.000000	3.000000
+AAAAAAAAAEPAAAAA	NULL	NULL	NULL	73.000000	85.250000	0.000000	3.410000	-4651.560000	1975.000000	1.000000
+AAAAAAAAAEPDAAAA	NULL	NULL	NULL	28.000000	91.090000	0.000000	43.720000	-177.240000	1987.000000	5.000000
+AAAAAAAAAFADAAAA	NULL	NULL	NULL	15.000000	109.790000	0.000000	62.580000	139.200000	1963.000000	4.000000
+AAAAAAAAAFCAAAAA	NULL	NULL	NULL	23.000000	8.880000	0.000000	1.950000	-67.390000	1988.000000	0.000000
+AAAAAAAAAFCBAAAA	NULL	NULL	NULL	74.000000	14.690000	0.000000	10.720000	-69.560000	1954.000000	3.000000
+AAAAAAAAAFHCAAAA	NULL	NULL	NULL	53.000000	48.210000	350.520000	47.240000	5.640000	1960.000000	1.000000
+AAAAAAAAAFMCAAAA	NULL	NULL	NULL	42.000000	56.730000	828.590000	53.320000	-380.870000	1944.000000	5.000000
+AAAAAAAAAFODAAAA	NULL	NULL	NULL	15.000000	23.690000	0.000000	17.530000	139.950000	1973.000000	5.000000
+AAAAAAAAAFPAAAAA	NULL	NULL	NULL	89.000000	126.010000	0.000000	75.600000	-507.300000	1938.000000	6.000000
+AAAAAAAAAGEBAAAA	NULL	NULL	NULL	55.000000	78.640000	0.000000	7.070000	-1731.400000	1957.000000	0.000000
+AAAAAAAAAGEDAAAA	NULL	NULL	NULL	19.000000	205.240000	0.000000	22.570000	-1223.600000	1935.000000	0.000000
+AAAAAAAAAGEEAAAA	NULL	NULL	NULL	40.000000	128.550000	488.400000	48.840000	-882.800000	1934.000000	6.000000
+AAAAAAAAAGIDAAAA	NULL	NULL	NULL	11.000000	125.030000	0.000000	13.750000	-753.610000	1935.000000	0.000000
+AAAAAAAAAGJCAAAA	NULL	NULL	NULL	8.000000	190.590000	0.000000	142.940000	369.520000	1941.000000	4.000000
+AAAAAAAAAGKDAAAA	NULL	NULL	NULL	80.000000	257.180000	0.000000	100.300000	953.600000	1960.000000	1.000000
+AAAAAAAAAGLAAAAA	NULL	NULL	NULL	44.000000	58.460000	411.400000	9.350000	-1811.480000	1973.000000	6.000000
+AAAAAAAAAGMBAAAA	NULL	NULL	NULL	86.000000	110.960000	0.000000	4.430000	-3732.400000	1946.000000	0.000000
+AAAAAAAAAGNBAAAA	NULL	NULL	NULL	41.000000	133.900000	0.000000	1.330000	-3118.870000	1948.000000	5.000000
+AAAAAAAAAGPBAAAA	NULL	NULL	NULL	45.000000	57.940000	876.960000	33.600000	-529.110000	1965.000000	6.000000
+AAAAAAAAAHAAAAAA	NULL	NULL	NULL	20.000000	51.470000	0.000000	26.240000	-205.400000	1942.000000	5.000000
+AAAAAAAAAHBAAAAA	NULL	NULL	NULL	50.500000	70.085000	0.000000	55.760000	724.565000	1957.000000	2.000000
+AAAAAAAAAHDEAAAA	NULL	NULL	NULL	79.000000	21.510000	0.000000	7.950000	-631.210000	1943.000000	6.000000
+AAAAAAAAAHEDAAAA	NULL	NULL	NULL	93.000000	23.200000	0.000000	2.320000	-1272.240000	1948.000000	5.000000
+AAAAAAAAAHFEAAAA	NULL	NULL	NULL	75.000000	57.750000	508.650000	14.430000	-870.150000	1965.000000	6.000000
+AAAAAAAAAHGDAAAA	NULL	NULL	NULL	71.000000	3.020000	0.000000	1.540000	-80.940000	1984.000000	2.000000
+AAAAAAAAAHIBAAAA	NULL	NULL	NULL	30.500000	59.105000	0.000000	41.310000	452.755000	1957.500000	3.000000
+AAAAAAAAAHKAAAAA	NULL	NULL	NULL	38.000000	89.720000	0.000000	20.630000	-1257.800000	1939.000000	5.000000
+AAAAAAAAAHKCAAAA	NULL	NULL	NULL	71.000000	141.010000	0.000000	90.240000	2269.870000	1924.000000	0.000000
+AAAAAAAAAHNDAAAA	NULL	NULL	NULL	51.000000	63.060000	0.000000	5.040000	-2107.830000	1975.000000	1.000000
+AAAAAAAAAIDAAAAA	NULL	NULL	NULL	77.000000	36.770000	0.000000	26.100000	-72.380000	1986.000000	2.000000
+AAAAAAAAAIEEAAAA	NULL	NULL	NULL	1.000000	145.160000	0.000000	133.540000	45.560000	1958.000000	3.000000
+AAAAAAAAAIIAAAAA	NULL	NULL	NULL	53.000000	232.610000	0.000000	0.000000	-4466.840000	1971.000000	4.000000
+AAAAAAAAAIJAAAAA	NULL	NULL	NULL	91.000000	62.950000	0.000000	26.430000	-107.380000	1935.000000	6.000000
+AAAAAAAAAIKBAAAA	NULL	NULL	NULL	73.000000	6.890000	124.470000	2.890000	-280.690000	1935.000000	3.000000
+AAAAAAAAAILBAAAA	NULL	NULL	NULL	7.000000	15.340000	0.000000	1.070000	-73.290000	1966.000000	4.000000
+AAAAAAAAAILDAAAA	NULL	NULL	NULL	27.500000	69.010000	200.210000	38.180000	-143.600000	1974.000000	2.500000
+AAAAAAAAAIODAAAA	NULL	NULL	NULL	99.000000	84.610000	663.300000	60.910000	-931.590000	1986.000000	1.000000
+AAAAAAAAAIPCAAAA	NULL	NULL	NULL	96.000000	183.920000	0.000000	22.070000	-3887.040000	1924.000000	3.000000
+AAAAAAAAAJACAAAA	NULL	NULL	NULL	62.000000	128.680000	0.000000	104.230000	2698.860000	1975.000000	4.000000
+AAAAAAAAAJAEAAAA	NULL	NULL	NULL	81.000000	131.730000	0.000000	23.710000	-4963.680000	1968.000000	5.000000
+AAAAAAAAAJBAAAAA	NULL	NULL	NULL	19.000000	113.950000	0.000000	79.760000	613.320000	1930.000000	2.000000
+AAAAAAAAAJBBAAAA	NULL	NULL	NULL	60.500000	140.315000	1093.800000	51.610000	-1455.575000	1976.000000	2.500000
+AAAAAAAAAJBDAAAA	NULL	NULL	NULL	4.000000	85.990000	82.520000	20.630000	-252.920000	1988.000000	5.000000
+AAAAAAAAAJDEAAAA	NULL	NULL	NULL	21.000000	7.750000	38.220000	7.280000	42.000000	1948.000000	0.000000
+AAAAAAAAAJEAAAAA	NULL	NULL	NULL	47.000000	95.750000	0.000000	44.040000	-1744.170000	1935.000000	0.000000
+AAAAAAAAAJEDAAAA	NULL	NULL	NULL	27.000000	38.270000	0.000000	21.040000	-300.240000	1976.000000	6.000000
+AAAAAAAAAJFDAAAA	NULL	NULL	NULL	82.000000	22.310000	0.000000	6.690000	-80.360000	1973.000000	5.000000
+AAAAAAAAAJGBAAAA	NULL	NULL	NULL	64.000000	155.160000	0.000000	34.130000	-3623.040000	1975.000000	5.000000
+AAAAAAAAAJGCAAAA	NULL	NULL	NULL	8.000000	2.080000	0.000000	1.200000	0.720000	1932.000000	0.000000
+AAAAAAAAAJJCAAAA	NULL	NULL	NULL	61.000000	30.960000	0.000000	0.610000	-875.350000	1932.000000	2.000000
+AAAAAAAAAJKBAAAA	NULL	NULL	NULL	13.000000	30.910000	0.000000	14.520000	43.680000	1928.000000	6.000000
+AAAAAAAAAJLCAAAA	NULL	NULL	NULL	25.000000	7.330000	0.000000	0.580000	-116.500000	1970.000000	0.000000
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q20.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q20.sql.out
new file mode 100644
index 0000000000000..49c6cf986ae4c
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q20.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_item_id:string,i_item_desc:string,i_category:string,i_class:string,i_current_price:decimal(7,2),itemrevenue:decimal(17,2),revenueratio:decimal(38,17)>
+-- !query output
+AAAAAAAAMJJBAAAA	NULL	Books                                             	NULL	NULL	9010.49	72.07596281370536693
+AAAAAAAAMLGDAAAA	NULL	Books                                             	NULL	6.35	1491.96	11.93436244638591899
+AAAAAAAAAELBAAAA	Precisely elderly bodies	Books                                             	arts                                              	1.40	4094.31	1.52075020571659240
+AAAAAAAAAFODAAAA	Close, precise teeth should go for a qualities. Political groups shall not become just important occasions. Trials mean ne	Books                                             	arts                                              	2.53	332.38	0.12345595555199313
+AAAAAAAAAHMAAAAA	Abilities could affect cruel parts. Predominantly other events telephone strong signs. Accurate mate	Books                                             	arts                                              	25.69	2626.56	0.97558359291967949
+AAAAAAAAAJOAAAAA	Average parents require also times. Children would not describe lightly purposes; large miles love now correct relations. Usual, german goals proceed literary, wooden visitors. Initial councils wil	Books                                             	arts                                              	1.24	12327.20	4.57869383019594946
+AAAAAAAAALNCAAAA	Great, contemporary workers would not remove of course cultural values. Then due children might see positive seconds. Significant problems w	Books                                             	arts                                              	0.55	343.80	0.12769768794384511
+AAAAAAAAANKAAAAA	Small objects stop etc mediterranean patterns; liberal, free initiatives would not leave less clear british attitudes; good, blue relationships find softly very	Books                                             	arts                                              	58.41	886.92	0.32942883476194038
+AAAAAAAABGDAAAAA	Newly national rights head curiously all electrical cells. Chinese, long values might not pull bad lines. High fun clothes ough	Books                                             	arts                                              	3.28	2219.85	0.82451923380495801
+AAAAAAAACBBAAAAA	Quick, easy studies must make always necessary systems. Upper, new persons should buy much physical technologies. English sciences hear solicitors. S	Books                                             	arts                                              	0.99	2050.16	0.76149125047979491
+AAAAAAAACMFDAAAA	Early, short v	Books                                             	arts                                              	75.57	5429.86	2.01681375177070042
+AAAAAAAADLLDAAAA	Black, following services justify by a investors; dirty, different charts will fly however prizes. Temporary, l	Books                                             	arts                                              	5.56	13539.35	5.02892289488801418
+AAAAAAAAEIPCAAAA	Scientific, difficult polls would not achieve. Countries reach of course. Bad, new churches realize most english	Books                                             	arts                                              	3.98	143.88	0.05344137097545211
+AAAAAAAAFCIBAAAA	United, important objectives put similarly large, previous phenomena; old, present days receive. Happy detectives assi	Books                                             	arts                                              	1.26	12297.15	4.56753235398096242
+AAAAAAAAFFIBAAAA	Naturally new years put serious, negative vehicles. Fin	Books                                             	arts                                              	3.34	4587.47	1.70392470189572752
+AAAAAAAAFJGCAAAA	Ago correct profits must not handle else. Healthy children may not go only ancient words. Later just characters ought to drink about. British parts must watch soon ago other clients. So vital d	Books                                             	arts                                              	4.03	5359.20	1.99056849688381241
+AAAAAAAAFLNCAAAA	Much new waters 	Books                                             	arts                                              	1.85	6718.63	2.49550179508480530
+AAAAAAAAGHBAAAAA	Hard different differences would not paint even. Together suitable schemes marry directly only open women. Social ca	Books                                             	arts                                              	2.65	3208.60	1.19177080144450674
+AAAAAAAAGLEBAAAA	Tall, following actions keep widely willing, secondary groups. Heads could afford however; agricultural, square pri	Books                                             	arts                                              	9.99	4780.52	1.77562929368618505
+AAAAAAAAGMFAAAAA	Anonymous, useful women provoke slightly present persons. Ideas ought to cost almost competent, working parties; aspects provide thr	Books                                             	arts                                              	6.73	5622.46	2.08835119999055082
+AAAAAAAAHHEBAAAA	Powerful walls will find; there scottish decades must not	Books                                             	arts                                              	4.16	7914.41	2.93965054810833964
+AAAAAAAAHMCEAAAA	Too executive doors progress mainly seemingly possible parts; hundreds stay virtually simple workers. Sola	Books                                             	arts                                              	34.32	3029.48	1.12524023173973205
+AAAAAAAAIBOCAAAA	Careful privileges ought to live rather to a boards. Possible, broad p	Books                                             	arts                                              	3.93	1450.99	0.53894144336718969
+AAAAAAAAICMBAAAA	Aside legitimate decisions may not stand probably sexual g	Books                                             	arts                                              	3.88	9619.83	3.57309496629679899
+AAAAAAAAIFPBAAAA	Specially interesting crews continue current, foreign directions; only social men would not call at least political children; circumstances could not understand now in a assessme	Books                                             	arts                                              	2.13	13616.57	5.05760473160419719
+AAAAAAAAIHNAAAAA	Unlikely states take later in general extra inf	Books                                             	arts                                              	0.32	11879.56	4.41242683475911751
+AAAAAAAAINFDAAAA	Sometimes careful things state probably so	Books                                             	arts                                              	5.08	25457.85	9.45581321995700176
+AAAAAAAAJGHDAAAA	Circumstances would not use. Principles seem writers. Times go from a hands. Members find grounds. Central, only teachers pursue properly into a p	Books                                             	arts                                              	5.95	2567.54	0.95366178505916251
+AAAAAAAAJLHBAAAA	Inches may lose from a problems. Firm, other corporations shall protect ashamed, important practices. Materials shall not make then by a police. Weeks used	Books                                             	arts                                              	0.84	1811.85	0.67297572978782944
+AAAAAAAAKFGBAAAA	Systems cannot await regions. Home appropr	Books                                             	arts                                              	7.30	1730.16	0.64263360027028230
+AAAAAAAAKHLBAAAA	Extra, primitive weeks look obviou	Books                                             	arts                                              	1.18	22.77	0.00845746467272063
+AAAAAAAALCFBAAAA	More than key reasons should remain. Words used to offer slowly british	Books                                             	arts                                              	0.28	10311.18	3.82988320527288194
+AAAAAAAALGEEAAAA	Children may turn also above, historical aspects. Surveys migh	Books                                             	arts                                              	7.22	11872.32	4.40973768042312729
+AAAAAAAALOKCAAAA	Trustees know operations. Now past issues cut today german governments. British lines go critical, individual structures. Tonight adequate problems should no	Books                                             	arts                                              	4.05	8348.99	3.10106666569599586
+AAAAAAAAMACDAAAA	Useful observers start often white colleagues; simple pro	Books                                             	arts                                              	3.47	7565.51	2.81005856636428042
+AAAAAAAAMNPAAAAA	Members should say earnings. Detailed departments would not move just at the hopes. Figures can take. Actually open houses want. Good teachers combine the	Books                                             	arts                                              	3.09	4363.97	1.62091006182752106
+AAAAAAAAMPFCAAAA	Major, senior words afford economic libraries; successful seconds need outside. Clinical, new ideas put now red c	Books                                             	arts                                              	5.87	9661.08	3.58841646026911898
+AAAAAAAANABCAAAA	Likely states feel astonishingly working roads. Parents put so somewhere able policies. Others may rely shortly instead interesting bodies; bri	Books                                             	arts                                              	7.50	132.66	0.04927392461498107
+AAAAAAAANMECAAAA	Floors could not go only for a years. Special reasons shape consequently black, concerned instances. Mutual depths encourage both simple teachers. Cards favour massive 	Books                                             	arts                                              	1.83	20114.53	7.47114303396483641
+AAAAAAAAOAHCAAAA	Accurate years want then other organisations. Simple lines mean as well so red results. Orthodox, central scales will not in	Books                                             	arts                                              	7.69	2153.04	0.79970398502215321
+AAAAAAAAODBEAAAA	Certain customers think exactly already necessary factories. Awkward doubts shall not forget fine	Books                                             	arts                                              	0.30	231.71	0.08606408165639427
+AAAAAAAAOKEDAAAA	Visitors could not allow glad wages. Communist, real figures used to apply factors. Aggressive, optimistic days must mean about trees. Detailed courts consider really large pro	Books                                             	arts                                              	9.08	24425.09	9.07221501111207600
+AAAAAAAAOODBAAAA	Deep, big areas take for a facilities. Words could replace certainly cases; lights test. Nevertheless practical arts cross. Fa	Books                                             	arts                                              	7.37	4380.23	1.62694951617879192
+AAAAAAAAAJJBAAAA	New, reluctant associations see more different, physical symptoms; useful pounds ought to give. Subjects 	Books                                             	business                                          	9.02	3044.02	1.58609001939612781
+AAAAAAAABDMAAAAA	Imports involve most now indian women. Developments announce intimately in a copies. Projects 	Books                                             	business                                          	3.26	472.29	0.24608723177265498
+AAAAAAAABINDAAAA	Years shall want free objects. Old residents use absolutely so residential steps. Letters will share variables. Sure fres	Books                                             	business                                          	40.76	30227.05	15.74983814849696292
+AAAAAAAACAADAAAA	Whole, important problems make. Indeed industrial members go skills. Soft	Books                                             	business                                          	3.22	137.92	0.07186336997625310
+AAAAAAAACPBBAAAA	Other, black houses flow. New soldiers put only eastern hours. Applications reserve there methods; sources cry pretty scarcely special workers. Never british opportunities 	Books                                             	business                                          	8.20	736.96	0.38399383075478162
+AAAAAAAAEBPAAAAA	Rows could not	Books                                             	business                                          	1.65	1290.88	0.67261446516056841
+AAAAAAAAEEFDAAAA	Remaining subjects handle even only certain ladies; eagerly literary days could not provide. Very different articles cut then. Boys see out of a houses. Governme	Books                                             	business                                          	9.03	1065.30	0.55507575431918810
+AAAAAAAAEFEEAAAA	White members see highly on a negotiations. Evident, passive colours can refer familiar, ugly factors; away small examinations shall prove 	Books                                             	business                                          	17.97	1446.00	0.75343991433919646
+AAAAAAAAEGCCAAAA	Manufacturing, ready concerns see already then new pupils. Both stable types used to manage otherw	Books                                             	business                                          	1.18	2635.71	1.37333963805184198
+AAAAAAAAFCGDAAAA	Small, capable centres	Books                                             	business                                          	2.98	5029.45	2.62060053746422658
+AAAAAAAAFDLAAAAA	Popular, different parameters might take open, used modules. Prisoners use pretty alternative lovers. Annual, professional others spend once true men. Other, small subsidies seem politically	Books                                             	business                                          	7.25	621.26	0.32370821658531756
+AAAAAAAAFEGEAAAA	Supreme, free uses handle even in the customers. Other minutes might not make of course social neighbours. So environmental rights come other, able sales	Books                                             	business                                          	8.08	3950.22	2.05826654109334761
+AAAAAAAAGIJAAAAA	Always other hours used to use. Women should jump then. Civil samples take therefore other offices. Concrete, major demands	Books                                             	business                                          	1.42	2013.79	1.04928752772968910
+AAAAAAAAHDKCAAAA	Visual fragments 	Books                                             	business                                          	6.77	930.13	0.48464527491308216
+AAAAAAAAHDLBAAAA	Classic issues will draw as european, engl	Books                                             	business                                          	75.64	556.83	0.29013689315456070
+AAAAAAAAHJAAAAAA	Again british shareholders see shares. American lives ought to establish horses. Then ideal conservatives might charge even nec	Books                                             	business                                          	2.44	1898.13	0.98902275560488173
+AAAAAAAAHLKAAAAA	Confident, video-tape	Books                                             	business                                          	3.17	1131.00	0.58930881266779474
+AAAAAAAAIHNDAAAA	Of course fundamental children will not deal still including a suppliers. More crucial powers will not keep enough. As good comments used to devote even convenient electric problems. Publi	Books                                             	business                                          	8.85	414.75	0.21610595053401226
+AAAAAAAAIMJAAAAA	Departments could seek now for a commu	Books                                             	business                                          	5.93	9895.85	5.15624369039663714
+AAAAAAAAJFBEAAAA	Paintings must not know primary, royal stands; similar, available others ough	Books                                             	business                                          	0.39	13809.44	7.19542412909562460
+AAAAAAAAJJGBAAAA	Most present eyes restore fat, central relationships; again considerable habits must face in a discussions. Engineers help at all direct occasions. Curiously del	Books                                             	business                                          	80.10	9267.25	4.82871095861681771
+AAAAAAAAKBMDAAAA	So white countries could secure more angry items. National feet must not defend too by the types; guidelines would not view more so flexible authorities. Critics will handle closely lig	Books                                             	business                                          	2.50	2542.50	1.32477246349059959
+AAAAAAAAKJHDAAAA	Simple changes ought to vote almost sudden techniques. Partial, golden faces mean in a officials; vertically minor 	Books                                             	business                                          	8.73	22710.22	11.83318548507904997
+AAAAAAAAKJOBAAAA	Christian lines stand once deep formal aspirations. National, fine islands play together with a patterns. New journals lose etc positive armie	Books                                             	business                                          	4.89	11560.78	6.02375732565303988
+AAAAAAAAKKDAAAAA	Children would not mean in favour of a parts. Heavy, whole others shall mean on	Books                                             	business                                          	3.13	9065.09	4.72337526492192700
+AAAAAAAAKLCCAAAA	Lips will n	Books                                             	business                                          	8.48	541.62	0.28221170567385587
+AAAAAAAAKNJCAAAA	White fees might combine reports. Tr	Books                                             	business                                          	2.09	37.60	0.01959152197728478
+AAAAAAAALAJCAAAA	Asleep children invite more. Wealthy forms could expect as. Indeed statistical examinations could la	Books                                             	business                                          	3.71	2082.24	1.08495347664844290
+AAAAAAAALDHBAAAA	Most new weeks go yet members. Also encouraging delegates make publications. Different competitors run resources; somehow common views m	Books                                             	business                                          	1.07	13412.42	6.98855641485568838
+AAAAAAAALHMBAAAA	Local, bloody names 	Books                                             	business                                          	4.40	1997.44	1.04076834197626873
+AAAAAAAALJJCAAAA	Large, larg	Books                                             	business                                          	3.50	12097.82	6.30358261721370521
+AAAAAAAANJLBAAAA	Only, gothic	Books                                             	business                                          	1.68	5708.95	2.97465477106967886
+AAAAAAAANKCAAAAA	Low, large clouds will not visit for example as the notions. Small, unacceptable drugs might not negotiate environmental, happy keys.	Books                                             	business                                          	3.11	3020.85	1.57401726502874248
+AAAAAAAAOAPAAAAA	Silver, critical operations could help howev	Books                                             	business                                          	5.56	2286.24	1.19124790439754116
+AAAAAAAAOBAEAAAA	Terrible, psychiatric bones will destroy also used studies; solely usual windows should not make shares. Advances continue sufficiently. As key days might not use far artists. Offici	Books                                             	business                                          	5.83	6672.40	3.47666146918178041
+AAAAAAAAOCHCAAAA	Too white addresses end by the talks. Hands get only companies. Statements know. Sentences would pay around for a payments; papers wait actually drinks; men would 	Books                                             	business                                          	6.06	7609.35	3.96486031270882752
+AAAAAAAAAGLDAAAA	New, big arguments may not win since by a tenant	Books                                             	computers                                         	1.00	904.16	0.32327741862037314
+AAAAAAAAALNBAAAA	Else substantial problems slip months. Just unique corporations put vast areas. Supporters like far perfect chapters. Now young reports become wrong trials. Available ears shall	Books                                             	computers                                         	51.46	18752.88	6.70498876094676063
+AAAAAAAABEBEAAAA	Cheap, desirable members take immediate, estimated debts; months must track typica	Books                                             	computers                                         	3.26	10027.86	3.58540600677589698
+AAAAAAAABHOAAAAA	Expert, scottish terms will ask quiet demands; poor bits attempt northern, dangerous si	Books                                             	computers                                         	2.66	7330.68	2.62104418148557444
+AAAAAAAACCDBAAAA	Gradually serious visitors bear no doubt technical hearts. Critics continue earlier soviet, standard minute	Books                                             	computers                                         	6.40	1711.84	0.61205894564136830
+AAAAAAAACCPBAAAA	Clear, general goods must know never women. Communications meet about. Other rewards spot wide in a skills. Relative, empty drawings facilitate too rooms. Still asian police end speedily comp	Books                                             	computers                                         	7.64	1292.04	0.46196177220211789
+AAAAAAAACFMBAAAA	At least remaining results shall keep cuts. Clients should meet policies. Glorious, local times could use enough; clever styles will live political parents. Single, gradual contracts will describe ho	Books                                             	computers                                         	9.51	3033.10	1.08446816760026298
+AAAAAAAACLPDAAAA	Environmental, new women pay again fingers. Different, uncomfortable records miss far russian, dependent members. Enough double men will go here immediatel	Books                                             	computers                                         	89.89	8553.39	3.05821739476786568
+AAAAAAAACOFCAAAA	Years learn here. Days make too. Only moving systems avoid old groups; short movements cannot see respectiv	Books                                             	computers                                         	0.60	3411.40	1.21972724504682903
+AAAAAAAACONDAAAA	Magnetic	Books                                             	computers                                         	57.19	3569.09	1.27610843437421206
+AAAAAAAADAHAAAAA	Ga	Books                                             	computers                                         	5.53	2687.70	0.96097230360331899
+AAAAAAAADDBAAAAA	S	Books                                             	computers                                         	65.78	1613.04	0.57673355084432699
+AAAAAAAAEAHCAAAA	Simple year	Books                                             	computers                                         	3.01	1262.79	0.45150359611088856
+AAAAAAAAECEEAAAA	Agricultural players shall smoke. So full reasons undertake 	Books                                             	computers                                         	0.70	4408.27	1.57615261257037727
+AAAAAAAAECGEAAAA	Then basic years can encourage later traditions. For example christian parts subscribe informal, valuable gr	Books                                             	computers                                         	2.75	844.19	0.30183547604973987
+AAAAAAAAECHAAAAA	Boxes batt	Books                                             	computers                                         	0.83	15300.82	5.47072375727191844
+AAAAAAAAEIGCAAAA	Separate, dead buildings think possibly english, net policies. Big divisions can use almost	Books                                             	computers                                         	9.46	12403.71	4.43487806374503246
+AAAAAAAAEJECAAAA	Artists make times. Rather ready functions must pre	Books                                             	computers                                         	5.71	1533.00	0.54811569052494252
+AAAAAAAAEMKDAAAA	Advantages emerge moves; special, expected operations pass etc natural preferences; very posit	Books                                             	computers                                         	0.15	5241.45	1.87405152387603389
+AAAAAAAAFGLAAAAA	Since other birds shall blame sudden	Books                                             	computers                                         	6.74	2098.16	0.75018552983158082
+AAAAAAAAFHNAAAAA	Legs throw then. Old-fashioned develo	Books                                             	computers                                         	2.66	163.26	0.05837271209073850
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q22.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q22.sql.out
new file mode 100644
index 0000000000000..e8a553dd5094e
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q22.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_product_name:string,i_brand:string,i_class:string,i_category:string,qoh:double>
+-- !query output
+esepriableanti                                    	NULL	NULL	NULL	429.7808764940239
+esepriableanti                                    	importoamalg #x                                   	NULL	NULL	429.7808764940239
+esepriableanti                                    	importoamalg #x                                   	fragrances                                        	NULL	429.7808764940239
+esepriableanti                                    	importoamalg #x                                   	fragrances                                        	Women                                             	429.7808764940239
+n stbarn stbarought                               	NULL	NULL	NULL	430.0122448979592
+n stbarn stbarought                               	amalgimporto #x                                   	NULL	NULL	430.0122448979592
+n stbarn stbarought                               	amalgimporto #x                                   	accessories                                       	NULL	430.0122448979592
+n stbarn stbarought                               	amalgimporto #x                                   	accessories                                       	Men                                               	430.0122448979592
+antiationeing                                     	NULL	NULL	NULL	437.03614457831327
+antiationeing                                     	amalgexporti #x                                   	NULL	NULL	437.03614457831327
+antiationeing                                     	amalgexporti #x                                   	newborn                                           	NULL	437.03614457831327
+antiationeing                                     	amalgexporti #x                                   	newborn                                           	Children                                          	437.03614457831327
+n stpriantin st                                   	NULL	NULL	NULL	438.77868852459017
+n stpriantin st                                   	exportiexporti #x                                 	NULL	NULL	438.77868852459017
+n stpriantin st                                   	exportiexporti #x                                 	toddlers                                          	NULL	438.77868852459017
+n stpriantin st                                   	exportiexporti #x                                 	toddlers                                          	Children                                          	438.77868852459017
+eingprically                                      	NULL	NULL	NULL	439.97975708502025
+eingprically                                      	amalgbrand #x                                     	NULL	NULL	439.97975708502025
+eingprically                                      	amalgbrand #x                                     	semi-precious                                     	NULL	439.97975708502025
+eingprically                                      	amalgbrand #x                                     	semi-precious                                     	Jewelry                                           	439.97975708502025
+prieingable                                       	NULL	NULL	NULL	440.096
+prieingable                                       	exportiunivamalg #x                               	NULL	NULL	440.096
+prieingable                                       	exportiunivamalg #x                               	self-help                                         	NULL	440.096
+prieingable                                       	exportiunivamalg #x                               	self-help                                         	Books                                             	440.096
+oughteingn stationought                           	NULL	NULL	NULL	440.1497975708502
+oughteingn stationought                           	amalgscholar #x                                   	NULL	NULL	440.1497975708502
+oughteingn stationought                           	amalgscholar #x                                   	rock                                              	NULL	440.1497975708502
+oughteingn stationought                           	amalgscholar #x                                   	rock                                              	Music                                             	440.1497975708502
+eingationbaroughtought                            	NULL	NULL	NULL	440.9721115537849
+eingationbaroughtought                            	maxicorp #x                                       	NULL	NULL	440.9721115537849
+eingationbaroughtought                            	maxicorp #x                                       	womens watch                                      	NULL	440.9721115537849
+eingationbaroughtought                            	maxicorp #x                                       	womens watch                                      	Jewelry                                           	440.9721115537849
+priantibarpri                                     	NULL	NULL	NULL	443.45849802371544
+priantibarpri                                     	exportiimporto #x                                 	NULL	NULL	443.45849802371544
+priantibarpri                                     	exportiimporto #x                                 	pants                                             	NULL	443.45849802371544
+priantibarpri                                     	exportiimporto #x                                 	pants                                             	Men                                               	443.45849802371544
+prioughtantiation                                 	NULL	NULL	NULL	443.8825910931174
+prioughtantiation                                 	corpmaxi #x                                       	NULL	NULL	443.8825910931174
+prioughtantiation                                 	corpmaxi #x                                       	parenting                                         	NULL	443.8825910931174
+prioughtantiation                                 	corpmaxi #x                                       	parenting                                         	Books                                             	443.8825910931174
+eseprieingoughtought                              	NULL	NULL	NULL	445.2016129032258
+eseprieingoughtought                              	importonameless #x                                	NULL	NULL	445.2016129032258
+eseprieingoughtought                              	importonameless #x                                	baseball                                          	NULL	445.2016129032258
+eseprieingoughtought                              	importonameless #x                                	baseball                                          	Sports                                            	445.2016129032258
+eingpriationanti                                  	NULL	NULL	NULL	445.4920634920635
+eingpriationanti                                  	scholarunivamalg #x                               	NULL	NULL	445.4920634920635
+eingpriationanti                                  	scholarunivamalg #x                               	fiction                                           	NULL	445.4920634920635
+eingpriationanti                                  	scholarunivamalg #x                               	fiction                                           	Books                                             	445.4920634920635
+antin stablecallyought                            	NULL	NULL	NULL	445.54918032786884
+antin stablecallyought                            	importoedu pack #x                                	NULL	NULL	445.54918032786884
+antin stablecallyought                            	importoedu pack #x                                	mens                                              	NULL	445.54918032786884
+antin stablecallyought                            	importoedu pack #x                                	mens                                              	Shoes                                             	445.54918032786884
+callycallyn steing                                	NULL	NULL	NULL	445.9012345679012
+callycallyn steing                                	corpunivamalg #x                                  	NULL	NULL	445.9012345679012
+callycallyn steing                                	corpunivamalg #x                                  	mystery                                           	NULL	445.9012345679012
+callycallyn steing                                	corpunivamalg #x                                  	mystery                                           	Books                                             	445.9012345679012
+oughtpribarought                                  	NULL	NULL	NULL	446.125
+oughtpribarought                                  	exportinameless #x                                	NULL	NULL	446.125
+oughtpribarought                                  	exportinameless #x                                	wallpaper                                         	NULL	446.125
+oughtpribarought                                  	exportinameless #x                                	wallpaper                                         	Home                                              	446.125
+oughtantioughtbarought                            	NULL	NULL	NULL	446.1847389558233
+oughtantioughtbarought                            	edu packmaxi #x                                  	NULL	NULL	446.1847389558233
+oughtantioughtbarought                            	edu packmaxi #x                                  	entertainments                                    	NULL	446.1847389558233
+oughtantioughtbarought                            	edu packmaxi #x                                  	entertainments                                    	Books                                             	446.1847389558233
+callyoughtn stcallyought                          	NULL	NULL	NULL	446.43650793650795
+callyoughtn stcallyought                          	exportischolar #x                                 	NULL	NULL	446.43650793650795
+callyoughtn stcallyought                          	exportischolar #x                                 	pop                                               	NULL	446.43650793650795
+callyoughtn stcallyought                          	exportischolar #x                                 	pop                                               	Music                                             	446.43650793650795
+ationeingationableought                           	NULL	NULL	NULL	446.48192771084337
+ationeingationableought                           	namelessnameless #x                               	NULL	NULL	446.48192771084337
+ationeingationableought                           	namelessnameless #x                               	outdoor                                           	NULL	446.48192771084337
+ationeingationableought                           	namelessnameless #x                               	outdoor                                           	Sports                                            	446.48192771084337
+priantiableese                                    	NULL	NULL	NULL	446.85483870967744
+priantiableese                                    	exportiedu pack #x                                	NULL	NULL	446.85483870967744
+priantiableese                                    	exportiedu pack #x                                	kids                                              	NULL	446.85483870967744
+priantiableese                                    	exportiedu pack #x                                	kids                                              	Shoes                                             	446.85483870967744
+prieseeseableought                                	NULL	NULL	NULL	446.9186991869919
+prieseeseableought                                	amalgscholar #x                                   	NULL	NULL	446.9186991869919
+prieseeseableought                                	amalgscholar #x                                   	rock                                              	NULL	446.9186991869919
+prieseeseableought                                	amalgscholar #x                                   	rock                                              	Music                                             	446.9186991869919
+ationableoughtcallyought                          	NULL	NULL	NULL	447.165991902834
+ationableoughtcallyought                          	exportischolar #x                                 	NULL	NULL	447.165991902834
+ationableoughtcallyought                          	exportischolar #x                                 	pop                                               	NULL	447.165991902834
+ationableoughtcallyought                          	exportischolar #x                                 	pop                                               	Music                                             	447.165991902834
+pripricallyese                                    	NULL	NULL	NULL	447.2550607287449
+pripricallyese                                    	edu packimporto #x                                	NULL	NULL	447.2550607287449
+pripricallyese                                    	edu packimporto #x                                	sports-apparel                                    	NULL	447.2550607287449
+pripricallyese                                    	edu packimporto #x                                	sports-apparel                                    	Men                                               	447.2550607287449
+eingableationn st                                 	NULL	NULL	NULL	447.3541666666667
+eingableationn st                                 	namelessmaxi #x                                   	NULL	NULL	447.3541666666667
+eingableationn st                                 	namelessmaxi #x                                   	romance                                           	NULL	447.3541666666667
+eingableationn st                                 	namelessmaxi #x                                   	romance                                           	Books                                             	447.3541666666667
+n stantin stoughtought                            	NULL	NULL	NULL	448.2396694214876
+n stantin stoughtought                            	importoscholar #x                                 	NULL	NULL	448.2396694214876
+n stantin stoughtought                            	importoscholar #x                                 	country                                           	NULL	448.2396694214876
+n stantin stoughtought                            	importoscholar #x                                 	country                                           	Music                                             	448.2396694214876
+n steingbaranti                                   	NULL	NULL	NULL	448.702479338843
+n steingbaranti                                   	amalgamalg #x                                     	NULL	NULL	448.702479338843
+n steingbaranti                                   	amalgamalg #x                                     	dresses                                           	NULL	448.702479338843
+n steingbaranti                                   	amalgamalg #x                                     	dresses                                           	Women                                             	448.702479338843
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q22a.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q22a.sql.out
new file mode 100644
index 0000000000000..42f0ab7f6e1f0
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q22a.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_product_name:string,i_brand:string,i_class:string,i_category:string,qoh:double>
+-- !query output
+oughtn steingcally                                	NULL	NULL	NULL	429.75206611570246
+oughtn steingcally                                	exportiexporti #x                                 	NULL	NULL	429.75206611570246
+oughtn steingcally                                	exportiexporti #x                                 	toddlers                                          	NULL	429.75206611570246
+oughtn steingcally                                	exportiexporti #x                                 	toddlers                                          	Children                                          	429.75206611570246
+oughtcallypripriought                             	NULL	NULL	NULL	433.04918032786884
+oughtcallypripriought                             	corpunivamalg #x                                 	NULL	NULL	433.04918032786884
+oughtcallypripriought                             	corpunivamalg #x                                 	musical                                           	NULL	433.04918032786884
+oughtcallypripriought                             	corpunivamalg #x                                 	musical                                           	Electronics                                       	433.04918032786884
+oughtesecallyeseought                             	NULL	NULL	NULL	434.8091286307054
+oughtesecallyeseought                             	scholarbrand #x                                   	NULL	NULL	434.8091286307054
+oughtesecallyeseought                             	scholarbrand #x                                   	blinds/shades                                     	NULL	434.8091286307054
+oughtesecallyeseought                             	scholarbrand #x                                   	blinds/shades                                     	Home                                              	434.8091286307054
+antiesen stese                                    	NULL	NULL	NULL	436.39676113360326
+antiesen stese                                    	edu packexporti #x                                	NULL	NULL	436.39676113360326
+antiesen stese                                    	edu packexporti #x                                	school-uniforms                                   	NULL	436.39676113360326
+antiesen stese                                    	edu packexporti #x                                	school-uniforms                                   	Children                                          	436.39676113360326
+priesecallyantiought                              	NULL	NULL	NULL	440.51612903225805
+priesecallyantiought                              	importounivamalg #x                               	NULL	NULL	440.51612903225805
+priesecallyantiought                              	importounivamalg #x                               	home repair                                       	NULL	440.51612903225805
+priesecallyantiought                              	importounivamalg #x                               	home repair                                       	Books                                             	440.51612903225805
+prioughtableoughtought                            	NULL	NULL	NULL	440.97478991596637
+prioughtableoughtought                            	exportiunivamalg #x                              	NULL	NULL	440.97478991596637
+prioughtableoughtought                            	exportiunivamalg #x                              	dvd/vcr players                                   	NULL	440.97478991596637
+prioughtableoughtought                            	exportiunivamalg #x                              	dvd/vcr players                                   	Electronics                                       	440.97478991596637
+antiablen stought                                 	NULL	NULL	NULL	441.520325203252
+antiablen stought                                 	amalgamalg #x                                     	NULL	NULL	441.520325203252
+antiablen stought                                 	amalgamalg #x                                     	dresses                                           	NULL	441.520325203252
+antiablen stought                                 	amalgamalg #x                                     	dresses                                           	Women                                             	441.520325203252
+n steingantianti                                  	NULL	NULL	NULL	442.4404761904762
+n steingantianti                                  	corpbrand #x                                     	NULL	NULL	442.4404761904762
+n steingantianti                                  	corpbrand #x                                     	rugs                                              	NULL	442.4404761904762
+n steingantianti                                  	corpbrand #x                                     	rugs                                              	Home                                              	442.4404761904762
+prieingation                                      	NULL	NULL	NULL	442.68595041322317
+prieingation                                      	importoamalg #x                                   	NULL	NULL	442.68595041322317
+prieingation                                      	importoamalg #x                                   	fragrances                                        	NULL	442.68595041322317
+prieingation                                      	importoamalg #x                                   	fragrances                                        	Women                                             	442.68595041322317
+n stesecallypri                                   	NULL	NULL	NULL	442.84
+n stesecallypri                                   	amalgimporto #x                                   	NULL	NULL	442.84
+n stesecallypri                                   	amalgimporto #x                                   	accessories                                       	NULL	442.84
+n stesecallypri                                   	amalgimporto #x                                   	accessories                                       	Men                                               	442.84
+n stn stcallyoughtought                           	NULL	NULL	NULL	443.20883534136544
+n stn stcallyoughtought                           	corpmaxi #x                                      	NULL	NULL	443.20883534136544
+n stn stcallyoughtought                           	corpmaxi #x                                      	golf                                              	NULL	443.20883534136544
+n stn stcallyoughtought                           	corpmaxi #x                                      	golf                                              	Sports                                            	443.20883534136544
+ationableprieing                                  	NULL	NULL	NULL	443.349593495935
+ationableprieing                                  	exportiamalg #x                                   	NULL	NULL	443.349593495935
+ationableprieing                                  	exportiamalg #x                                   	maternity                                         	NULL	443.349593495935
+ationableprieing                                  	exportiamalg #x                                   	maternity                                         	Women                                             	443.349593495935
+ationoughtesepri                                  	NULL	NULL	NULL	443.8292682926829
+ationoughtesepri                                  	edu packunivamalg #x                             	NULL	NULL	443.8292682926829
+ationoughtesepri                                  	edu packunivamalg #x                             	sports                                            	NULL	443.8292682926829
+ationoughtesepri                                  	edu packunivamalg #x                             	sports                                            	Books                                             	443.8292682926829
+oughtbarcallycallyought                           	NULL	NULL	NULL	444.5889328063241
+oughtbarcallycallyought                           	corpmaxi #x                                       	NULL	NULL	444.5889328063241
+oughtbarcallycallyought                           	corpmaxi #x                                       	golf                                              	NULL	444.5889328063241
+oughtbarcallycallyought                           	corpmaxi #x                                       	golf                                              	Sports                                            	444.5889328063241
+n steingcallycally                                	NULL	NULL	NULL	445.0833333333333
+n steingcallycally                                	importoscholar #x                                 	NULL	NULL	445.0833333333333
+n steingcallycally                                	importoscholar #x                                 	country                                           	NULL	445.0833333333333
+n steingcallycally                                	importoscholar #x                                 	country                                           	Music                                             	445.0833333333333
+ationcallyoughteing                               	NULL	NULL	NULL	445.83534136546183
+ationcallyoughteing                               	amalgedu pack #x                                  	NULL	NULL	445.83534136546183
+ationcallyoughteing                               	amalgedu pack #x                                  	womens                                            	NULL	445.83534136546183
+ationcallyoughteing                               	amalgedu pack #x                                  	womens                                            	Shoes                                             	445.83534136546183
+antiablebarantiought                              	NULL	NULL	NULL	446.05555555555554
+antiablebarantiought                              	scholarbrand #x                                   	NULL	NULL	446.05555555555554
+antiablebarantiought                              	scholarbrand #x                                   	custom                                            	NULL	446.05555555555554
+antiablebarantiought                              	scholarbrand #x                                   	custom                                            	Jewelry                                           	446.05555555555554
+ationantiationeseought                            	NULL	NULL	NULL	446.33870967741933
+ationantiationeseought                            	amalgimporto #x                                   	NULL	NULL	446.33870967741933
+ationantiationeseought                            	amalgimporto #x                                   	accessories                                       	NULL	446.33870967741933
+ationantiationeseought                            	amalgimporto #x                                   	accessories                                       	Men                                               	446.33870967741933
+pricallypriationought                             	NULL	NULL	NULL	446.96
+pricallypriationought                             	maxibrand #x                                      	NULL	NULL	446.96
+pricallypriationought                             	maxibrand #x                                      	mattresses                                        	NULL	446.96
+pricallypriationought                             	maxibrand #x                                      	mattresses                                        	Home                                              	446.96
+ationcallyn stbarought                            	NULL	NULL	NULL	446.9918032786885
+ationcallyn stbarought                            	corpbrand #x                                      	NULL	NULL	446.9918032786885
+ationcallyn stbarought                            	corpbrand #x                                      	rugs                                              	NULL	446.9918032786885
+ationcallyn stbarought                            	corpbrand #x                                      	rugs                                              	Home                                              	446.9918032786885
+ationeseoughtpri                                  	NULL	NULL	NULL	447.0769230769231
+ationeseoughtpri                                  	edu packamalg #x                                  	NULL	NULL	447.0769230769231
+ationeseoughtpri                                  	edu packamalg #x                                  	swimwear                                          	NULL	447.0769230769231
+ationeseoughtpri                                  	edu packamalg #x                                  	swimwear                                          	Women                                             	447.0769230769231
+anticallyese                                      	NULL	NULL	NULL	447.4313725490196
+anticallyese                                      	amalgcorp #x                                      	NULL	NULL	447.4313725490196
+anticallyese                                      	amalgcorp #x                                      	birdal                                            	NULL	447.4313725490196
+anticallyese                                      	amalgcorp #x                                      	birdal                                            	Jewelry                                           	447.4313725490196
+n stpriprioughtought                              	NULL	NULL	NULL	447.6375
+n stpriprioughtought                              	scholarbrand #x                                  	NULL	NULL	447.6375
+n stpriprioughtought                              	scholarbrand #x                                  	blinds/shades                                     	NULL	447.6375
+n stpriprioughtought                              	scholarbrand #x                                  	blinds/shades                                     	Home                                              	447.6375
+oughtantioughteing                                	NULL	NULL	NULL	448.08097165991904
+oughtantioughteing                                	univmaxi #x                                       	NULL	NULL	448.08097165991904
+oughtantioughteing                                	univmaxi #x                                       	pools                                             	NULL	448.08097165991904
+oughtantioughteing                                	univmaxi #x                                       	pools                                             	Sports                                            	448.08097165991904
+n steseeseationought                              	NULL	NULL	NULL	448.1769547325103
+n steseeseationought                              	exportiunivamalg #x                               	NULL	NULL	448.1769547325103
+n steseeseationought                              	exportiunivamalg #x                               	self-help                                         	NULL	448.1769547325103
+n steseeseationought                              	exportiunivamalg #x                               	self-help                                         	Books                                             	448.1769547325103
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q24.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q24.sql.out
new file mode 100644
index 0000000000000..e50b231441a5d
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q24.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<c_last_name:string,c_first_name:string,s_store_name:string,paid:decimal(27,2)>
+-- !query output
+
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q27a.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q27a.sql.out
new file mode 100644
index 0000000000000..5032ce4890dce
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q27a.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_item_id:string,s_state:string,g_state:int,agg1:double,agg2:decimal(11,6),agg3:decimal(11,6),agg4:decimal(11,6)>
+-- !query output
+NULL	NULL	1	49.916138404648706	75.425222	191.109448	37.896850
+AAAAAAAAAAABAAAA	NULL	1	67.0	37.520000	0.000000	24.010000
+AAAAAAAAAAABAAAA	TN	0	67.0	37.520000	0.000000	24.010000
+AAAAAAAAAAACAAAA	NULL	1	34.0	89.610000	186.400000	32.250000
+AAAAAAAAAAACAAAA	TN	0	34.0	89.610000	186.400000	32.250000
+AAAAAAAAAACCAAAA	NULL	1	69.0	31.085000	0.000000	7.825000
+AAAAAAAAAACCAAAA	TN	0	69.0	31.085000	0.000000	7.825000
+AAAAAAAAAACDAAAA	NULL	1	36.0	107.280000	0.000000	54.063333
+AAAAAAAAAACDAAAA	TN	0	36.0	107.280000	0.000000	54.063333
+AAAAAAAAAADBAAAA	NULL	1	21.0	74.470000	0.000000	44.680000
+AAAAAAAAAADBAAAA	TN	0	21.0	74.470000	0.000000	44.680000
+AAAAAAAAAAEBAAAA	NULL	1	47.0	30.540000	125.010000	20.460000
+AAAAAAAAAAEBAAAA	TN	0	47.0	30.540000	125.010000	20.460000
+AAAAAAAAAAEEAAAA	NULL	1	26.5	100.840000	362.865000	44.030000
+AAAAAAAAAAEEAAAA	TN	0	26.5	100.840000	362.865000	44.030000
+AAAAAAAAAAFCAAAA	NULL	1	50.0	37.055000	0.000000	7.555000
+AAAAAAAAAAFCAAAA	TN	0	50.0	37.055000	0.000000	7.555000
+AAAAAAAAAAGBAAAA	NULL	1	14.0	55.130000	0.000000	50.160000
+AAAAAAAAAAGBAAAA	TN	0	14.0	55.130000	0.000000	50.160000
+AAAAAAAAAAHBAAAA	NULL	1	50.0	106.065000	0.000000	29.695000
+AAAAAAAAAAHBAAAA	TN	0	50.0	106.065000	0.000000	29.695000
+AAAAAAAAAAIAAAAA	NULL	1	67.0	68.610000	0.000000	26.405000
+AAAAAAAAAAIAAAAA	TN	0	67.0	68.610000	0.000000	26.405000
+AAAAAAAAAAIDAAAA	NULL	1	37.333333333333336	78.440000	0.000000	51.863333
+AAAAAAAAAAIDAAAA	TN	0	37.333333333333336	78.440000	0.000000	51.863333
+AAAAAAAAAAJBAAAA	NULL	1	55.0	131.290000	0.000000	36.760000
+AAAAAAAAAAJBAAAA	TN	0	55.0	131.290000	0.000000	36.760000
+AAAAAAAAAAKBAAAA	NULL	1	70.0	66.120000	0.000000	21.150000
+AAAAAAAAAAKBAAAA	TN	0	70.0	66.120000	0.000000	21.150000
+AAAAAAAAAALCAAAA	NULL	1	93.0	104.200000	0.000000	6.250000
+AAAAAAAAAALCAAAA	TN	0	93.0	104.200000	0.000000	6.250000
+AAAAAAAAAALDAAAA	NULL	1	61.5	150.645000	0.000000	57.335000
+AAAAAAAAAALDAAAA	TN	0	61.5	150.645000	0.000000	57.335000
+AAAAAAAAAANAAAAA	NULL	1	52.0	2.510000	0.000000	2.280000
+AAAAAAAAAANAAAAA	TN	0	52.0	2.510000	0.000000	2.280000
+AAAAAAAAAAOAAAAA	NULL	1	23.5	17.590000	0.000000	12.130000
+AAAAAAAAAAOAAAAA	TN	0	23.5	17.590000	0.000000	12.130000
+AAAAAAAAAAOCAAAA	NULL	1	30.0	22.380000	0.000000	4.690000
+AAAAAAAAAAOCAAAA	TN	0	30.0	22.380000	0.000000	4.690000
+AAAAAAAAAAPCAAAA	NULL	1	92.0	75.630000	0.000000	64.280000
+AAAAAAAAAAPCAAAA	TN	0	92.0	75.630000	0.000000	64.280000
+AAAAAAAAABAAAAAA	NULL	1	10.0	45.790000	0.000000	43.950000
+AAAAAAAAABAAAAAA	TN	0	10.0	45.790000	0.000000	43.950000
+AAAAAAAAABABAAAA	NULL	1	63.0	21.890000	0.000000	19.700000
+AAAAAAAAABABAAAA	TN	0	63.0	21.890000	0.000000	19.700000
+AAAAAAAAABBAAAAA	NULL	1	26.5	78.785000	0.000000	35.615000
+AAAAAAAAABBAAAAA	TN	0	26.5	78.785000	0.000000	35.615000
+AAAAAAAAABBDAAAA	NULL	1	14.0	67.910000	0.000000	36.670000
+AAAAAAAAABBDAAAA	TN	0	14.0	67.910000	0.000000	36.670000
+AAAAAAAAABCBAAAA	NULL	1	42.5	72.900000	0.000000	44.685000
+AAAAAAAAABCBAAAA	TN	0	42.5	72.900000	0.000000	44.685000
+AAAAAAAAABCCAAAA	NULL	1	50.0	90.835000	0.000000	42.605000
+AAAAAAAAABCCAAAA	TN	0	50.0	90.835000	0.000000	42.605000
+AAAAAAAAABDBAAAA	NULL	1	40.0	10.100000	0.000000	4.240000
+AAAAAAAAABDBAAAA	TN	0	40.0	10.100000	0.000000	4.240000
+AAAAAAAAABDEAAAA	NULL	1	67.0	85.420000	0.000000	31.600000
+AAAAAAAAABDEAAAA	TN	0	67.0	85.420000	0.000000	31.600000
+AAAAAAAAABECAAAA	NULL	1	78.5	36.280000	1673.320000	25.170000
+AAAAAAAAABECAAAA	TN	0	78.5	36.280000	1673.320000	25.170000
+AAAAAAAAABEDAAAA	NULL	1	60.0	92.410000	2270.190000	58.210000
+AAAAAAAAABEDAAAA	TN	0	60.0	92.410000	2270.190000	58.210000
+AAAAAAAAABGAAAAA	NULL	1	57.0	11.390000	201.900000	3.980000
+AAAAAAAAABGAAAAA	TN	0	57.0	11.390000	201.900000	3.980000
+AAAAAAAAABGDAAAA	NULL	1	74.0	174.750000	232.650000	5.240000
+AAAAAAAAABGDAAAA	TN	0	74.0	174.750000	232.650000	5.240000
+AAAAAAAAABHDAAAA	NULL	1	34.5	120.730000	0.000000	61.460000
+AAAAAAAAABHDAAAA	TN	0	34.5	120.730000	0.000000	61.460000
+AAAAAAAAABIBAAAA	NULL	1	4.0	12.680000	5.040000	10.520000
+AAAAAAAAABIBAAAA	TN	0	4.0	12.680000	5.040000	10.520000
+AAAAAAAAABICAAAA	NULL	1	20.5	22.420000	0.000000	20.300000
+AAAAAAAAABICAAAA	TN	0	20.5	22.420000	0.000000	20.300000
+AAAAAAAAABJBAAAA	NULL	1	38.0	2.850000	0.000000	0.880000
+AAAAAAAAABJBAAAA	TN	0	38.0	2.850000	0.000000	0.880000
+AAAAAAAAABKCAAAA	NULL	1	8.0	73.980000	0.000000	51.780000
+AAAAAAAAABKCAAAA	TN	0	8.0	73.980000	0.000000	51.780000
+AAAAAAAAABLCAAAA	NULL	1	49.0	96.580000	0.000000	28.970000
+AAAAAAAAABLCAAAA	TN	0	49.0	96.580000	0.000000	28.970000
+AAAAAAAAABMAAAAA	NULL	1	50.0	136.990000	838.320000	69.860000
+AAAAAAAAABMAAAAA	TN	0	50.0	136.990000	838.320000	69.860000
+AAAAAAAAABMBAAAA	NULL	1	76.0	120.520000	0.000000	83.150000
+AAAAAAAAABMBAAAA	TN	0	76.0	120.520000	0.000000	83.150000
+AAAAAAAAABNAAAAA	NULL	1	17.0	91.190000	0.000000	16.410000
+AAAAAAAAABNAAAAA	TN	0	17.0	91.190000	0.000000	16.410000
+AAAAAAAAABNCAAAA	NULL	1	61.0	13.080000	16.650000	0.650000
+AAAAAAAAABNCAAAA	TN	0	61.0	13.080000	16.650000	0.650000
+AAAAAAAAABPAAAAA	NULL	1	30.0	118.380000	623.160000	46.160000
+AAAAAAAAABPAAAAA	TN	0	30.0	118.380000	623.160000	46.160000
+AAAAAAAAABPBAAAA	NULL	1	30.0	NULL	NULL	17.350000
+AAAAAAAAABPBAAAA	TN	0	30.0	NULL	NULL	17.350000
+AAAAAAAAACAAAAAA	NULL	1	20.0	3.710000	0.000000	0.250000
+AAAAAAAAACAAAAAA	TN	0	20.0	3.710000	0.000000	0.250000
+AAAAAAAAACACAAAA	NULL	1	66.0	117.590000	0.000000	34.100000
+AAAAAAAAACACAAAA	TN	0	66.0	117.590000	0.000000	34.100000
+AAAAAAAAACADAAAA	NULL	1	79.0	171.850000	0.000000	127.160000
+AAAAAAAAACADAAAA	TN	0	79.0	171.850000	0.000000	127.160000
+AAAAAAAAACBBAAAA	NULL	1	81.0	88.700000	0.000000	13.300000
+AAAAAAAAACBBAAAA	TN	0	81.0	88.700000	0.000000	13.300000
+AAAAAAAAACBCAAAA	NULL	1	26.0	62.780000	1311.600000	61.520000
+AAAAAAAAACBCAAAA	TN	0	26.0	62.780000	1311.600000	61.520000
+AAAAAAAAACBEAAAA	NULL	1	60.0	84.015000	0.000000	33.495000
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q34.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q34.sql.out
new file mode 100644
index 0000000000000..71b7844e8e93a
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q34.sql.out
@@ -0,0 +1,223 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<c_last_name:string,c_first_name:string,c_salutation:string,c_preferred_cust_flag:string,ss_ticket_number:int,cnt:bigint>
+-- !query output
+NULL	NULL	NULL	Y	47915	15
+NULL	NULL	NULL	NULL	126143	15
+NULL	NULL	NULL	NULL	215293	15
+NULL	NULL	Mrs.      	N	120593	15
+NULL	Rubin               	Sir       	NULL	30056	15
+Adler                         	Justin              	Sir       	Y	226187	15
+Allen                         	Rose                	Mrs.      	N	179476	16
+Anderson                      	Marvin              	Dr.       	N	211012	16
+Andrews                       	Jacob               	Mr.       	N	67111	16
+Andrews                       	Samuel              	Dr.       	Y	139993	16
+Angel                         	Kevin               	Mr.       	Y	106628	15
+Ashley                        	Linda               	Mrs.      	Y	82173	15
+Baca                          	Dorothy             	Mrs.      	N	64890	15
+Baker                         	Jamie               	Dr.       	Y	9916	15
+Banks                         	Leroy               	Sir       	N	206730	15
+Barber                        	Dianna              	Mrs.      	Y	119959	16
+Barksdale                     	Joann               	Miss      	Y	138994	15
+Barnes                        	Ruth                	Dr.       	N	84038	15
+Barney                        	Samuel              	Sir       	N	15288	15
+Barnhart                      	Charley             	Mr.       	Y	166576	15
+Barone                        	Seth                	Mr.       	Y	162374	15
+Barrett                       	David               	Sir       	N	189879	15
+Bartels                       	Elmer               	Sir       	Y	114760	16
+Bear                          	Scott               	Sir       	Y	82291	15
+Beers                         	Kendra              	Dr.       	NULL	137960	15
+Belcher                       	James               	Sir       	Y	239470	16
+Bell                          	Carrie              	Miss      	N	5527	15
+Bell                          	Matthew             	Dr.       	N	20400	15
+Benjamin                      	Consuelo            	Ms.       	Y	201086	15
+Bergman                       	Joann               	Miss      	N	177052	15
+Brooks                        	Robert              	Sir       	N	155576	16
+Byrd                          	Kelly               	Sir       	N	165115	16
+Cagle                         	Jennifer            	Miss      	N	163129	15
+Campbell                      	Robert              	Mr.       	N	8964	15
+Cardona                       	Robert              	Mr.       	N	200501	15
+Carter                        	Wendy               	Ms.       	N	96439	15
+Carver                        	Bernard             	Mr.       	Y	194943	16
+Chen                          	Wanita              	Miss      	N	137713	16
+Christensen                   	Larry               	Dr.       	Y	58094	15
+Cochrane                      	Anne                	Mrs.      	N	208347	16
+Coleman                       	Inez                	Dr.       	Y	88249	16
+Coleman                       	John                	Mr.       	N	49444	15
+Colon                         	Anna                	Dr.       	Y	143694	15
+Conley                        	Roxie               	Dr.       	N	196663	15
+Cook                          	Adam                	Ms.       	Y	167339	15
+Cote                          	Justin              	Dr.       	N	93466	15
+Council                       	Donald              	Sir       	Y	102958	15
+Cramer                        	Linda               	Ms.       	N	126628	15
+Crittenden                    	Amie                	Ms.       	N	138787	15
+Cruz                          	James               	Mr.       	Y	201430	15
+Cuellar                       	Oscar               	Mr.       	Y	86781	16
+Cullen                        	Larry               	Mr.       	Y	221242	16
+Cushing                       	Antonia             	Mrs.      	Y	118927	15
+Davis                         	Gordon              	Dr.       	N	227822	15
+Davis                         	Myrtle              	Dr.       	Y	37430	15
+Decker                        	Vera                	Miss      	Y	75737	16
+Diamond                       	Fernando            	Dr.       	N	216391	15
+Diaz                          	Walton              	Mr.       	N	131135	16
+Dickinson                     	Steven              	Mr.       	N	8057	16
+Douglas                       	Lester              	Sir       	N	26043	15
+Dove                          	Garry               	Dr.       	N	152171	16
+Drake                         	Rosetta             	Dr.       	Y	238040	15
+Dumas                         	Travis              	Mr.       	Y	94154	15
+Duncan                        	Olivia              	Dr.       	Y	102032	15
+Durham                        	Andrea              	Dr.       	Y	144734	15
+Dutton                        	Gay                 	Miss      	Y	110886	15
+Ellis                         	Karen               	Miss      	N	229706	16
+Ely                           	Cesar               	Dr.       	Y	36054	16
+Etheridge                     	Mike                	Dr.       	N	19648	15
+Farmer                        	Eugenia             	Miss      	Y	98187	16
+Farrow                        	Kathy               	Miss      	Y	200078	15
+Faulkner                      	Lakeisha            	Dr.       	Y	178393	16
+Faulkner                      	Robert              	Dr.       	N	109423	15
+Felton                        	David               	Mr.       	N	97807	16
+Ferreira                      	Christine           	Mrs.      	Y	155822	15
+Finn                          	Robert              	Mr.       	N	38057	15
+Finney                        	Crystal             	Miss      	Y	158304	15
+Fischer                       	Tamara              	Mrs.      	N	66790	15
+Foote                         	Roy                 	Sir       	N	68086	15
+Foreman                       	Autumn              	Mrs.      	Y	164060	15
+Funk                          	Marvin              	Sir       	Y	61516	15
+Garcia                        	Christopher         	Sir       	Y	181616	16
+Garcia                        	Karen               	Miss      	N	236987	15
+Garcia                        	Robert              	Dr.       	N	172185	16
+Garland                       	Michael             	Mr.       	N	234421	15
+Gaylord                       	Keith               	Mr.       	Y	123333	16
+Gifford                       	Mark                	Mr.       	N	225973	16
+Gilbert                       	NULL	Sir       	N	16844	15
+Gilmore                       	Austin              	Dr.       	Y	239871	15
+Goldsmith                     	Bernice             	Ms.       	Y	2347	15
+Good                          	Nancy               	Dr.       	N	132655	15
+Goodman                       	NULL	NULL	N	71903	15
+Gower                         	Nettie              	Miss      	N	10576	15
+Gray                          	Evelyn              	Miss      	N	157486	15
+Hammond                       	Roger               	Sir       	Y	54884	16
+Hardin                        	Kimberly            	Dr.       	N	192424	16
+Harp                          	Vance               	Mr.       	N	199017	15
+Harper                        	Madeline            	Dr.       	N	173835	16
+Harris                        	Tammy               	Dr.       	N	217761	16
+Hartmann                      	Joey                	Dr.       	N	230915	15
+Hayes                         	David               	Sir       	N	82274	15
+Haynes                        	Sara                	Miss      	Y	139168	16
+Heath                         	Matthew             	Dr.       	N	30710	15
+Hennessey                     	Debbie              	Dr.       	Y	79256	15
+Herman                        	Stella              	Ms.       	Y	33801	16
+Hernandez                     	Max                 	Mr.       	N	16015	15
+Hernandez                     	Ruth                	Ms.       	Y	97000	15
+Hess                          	Joseph              	Sir       	N	151336	15
+Hodges                        	Lucas               	Dr.       	Y	163325	15
+Holland                       	Jeremiah            	Dr.       	N	95938	16
+Jackson                       	William             	Mr.       	Y	16425	16
+Jameson                       	Miguel              	Dr.       	N	9181	16
+Jarrell                       	Thomas              	Mr.       	Y	85787	16
+Johnson                       	Julia               	Dr.       	N	27560	15
+Jones                         	Theresa             	Ms.       	N	219765	16
+Kelly                         	Mark                	Mr.       	Y	17039	16
+Khan                          	Hank                	Mr.       	N	177803	15
+Kim                           	Charlotte           	Dr.       	Y	7208	16
+Kunz                          	Sarah               	Dr.       	N	74568	15
+Lake                          	Robert              	Sir       	N	13264	15
+Landry                        	Rudolph             	Sir       	N	117581	15
+Lane                          	Luis                	Sir       	N	232302	16
+Langford                      	Darlene             	Mrs.      	N	214891	15
+Larson                        	Kevin               	Mr.       	Y	35053	15
+Larson                        	Thomas              	Mr.       	N	114265	15
+Lee                           	Malik               	Dr.       	N	20122	16
+Leonard                       	Orlando             	Dr.       	Y	133168	15
+Lincoln                       	Anthony             	Miss      	Y	1407	16
+Lindsey                       	Linda               	Dr.       	N	62687	16
+Lopez                         	Karen               	Dr.       	Y	136008	15
+Lunsford                      	Kevin               	Dr.       	N	159120	16
+Lynch                         	Sylvia              	Ms.       	Y	115438	15
+Lyon                          	Michael             	Mr.       	N	140323	15
+Maestas                       	Mabel               	Mrs.      	N	184265	15
+Magana                        	Diann               	Mrs.      	Y	19139	15
+Manning                       	Annamarie           	Ms.       	N	4984	16
+Marshall                      	Felipe              	Sir       	N	138890	15
+Martin                        	Paul                	Dr.       	N	26115	16
+Martinez                      	Earl                	Sir       	N	108982	15
+Martinez                      	Robert              	Sir       	Y	157672	16
+Masterson                     	Barbara             	Mrs.      	N	231070	15
+Mata                          	Deborah             	Miss      	Y	4323	15
+Mccoy                         	Debbie              	Dr.       	N	91552	15
+Mcgill                        	Tony                	Sir       	N	110030	15
+Mckeon                        	Christina           	Dr.       	N	26190	15
+Mcnamara                      	Linda               	Dr.       	Y	7957	15
+Means                         	Michael             	Mr.       	Y	226164	16
+Medina                        	Joseph              	Sir       	Y	110246	15
+Meyers                        	Zachary             	Mr.       	Y	59549	15
+Montgomery                    	John                	Mr.       	Y	103718	15
+Moody                         	Miranda             	Ms.       	Y	171671	15
+Moore                         	Mark                	Dr.       	N	191471	15
+Moran                         	Celia               	Ms.       	Y	200691	15
+Morgan                        	Cecelia             	Mrs.      	N	200742	15
+Morrell                       	Chad                	Mr.       	N	93790	15
+Morse                         	Robert              	Mr.       	N	68627	16
+Neel                          	Audrey              	Ms.       	Y	193308	15
+Neff                          	Sheri               	Mrs.      	Y	52556	15
+Nelson                        	Katherine           	Mrs.      	N	110232	15
+New                           	Suzanne             	Miss      	N	5120	16
+Nielsen                       	Veronica            	Mrs.      	N	23905	15
+Oakley                        	George              	Mr.       	Y	177890	15
+Parker                        	Barbar              	Dr.       	N	57241	15
+Parker                        	Jeff                	Sir       	N	213566	16
+Pemberton                     	Jennifer            	Mrs.      	Y	49875	16
+Perry                         	Robert              	Mr.       	Y	153147	16
+Phillips                      	David               	Dr.       	N	148883	15
+Phillips                      	Georgia             	NULL	Y	26878	15
+Phillips                      	Stanley             	Sir       	N	31989	15
+Pinkston                      	Brenda              	Dr.       	N	126440	15
+Pryor                         	Dorothy             	Mrs.      	N	213779	16
+Reed                          	William             	Dr.       	N	145002	15
+Reynolds                      	Amelia              	Ms.       	Y	68440	15
+Rice                          	David               	Dr.       	Y	70484	16
+Robertson                     	Daniel              	Mr.       	N	40407	16
+Rosales                       	NULL	NULL	Y	156406	16
+Russo                         	Cheryl              	Miss      	N	81123	15
+Sanchez                       	Bruce               	Sir       	Y	124479	15
+Schmitz                       	Kaitlyn             	Miss      	N	105162	15
+Sebastian                     	Homer               	Dr.       	Y	64994	15
+Sexton                        	Jerry               	Sir       	N	91446	15
+Sierra                        	David               	Sir       	Y	61810	15
+Simmons                       	Joseph              	Dr.       	N	54185	15
+Simpson                       	Michael             	Sir       	Y	186613	16
+Simpson                       	Shalanda            	Dr.       	Y	181123	15
+Singleton                     	Andrew              	Ms.       	N	45464	15
+Smith                         	Danny               	Dr.       	Y	143297	15
+Smith                         	Edward              	Sir       	Y	81178	16
+Smith                         	Hung                	Sir       	N	44710	15
+Smith                         	Kimberly            	Mrs.      	Y	174638	15
+Smith                         	Vern                	Sir       	N	50960	15
+Sosa                          	Leah                	Ms.       	Y	77106	16
+Sparks                        	Erick               	Dr.       	N	220337	15
+Taylor                        	Kenneth             	Dr.       	Y	194337	15
+Todd                          	Linda               	Ms.       	Y	235816	15
+Trout                         	Harley              	Mr.       	Y	214547	15
+Urban                         	NULL	NULL	NULL	214898	15
+Varner                        	Elsie               	Ms.       	N	199602	16
+Vazquez                       	Bill                	Dr.       	Y	62049	15
+Velazquez                     	William             	Dr.       	N	46239	15
+Wagner                        	Barbara             	Ms.       	Y	233595	15
+Ward                          	Anna                	Miss      	N	52941	16
+Watkins                       	Rosa                	Miss      	Y	152190	16
+Welch                         	James               	Dr.       	Y	51441	16
+West                          	Teresa              	Ms.       	N	233179	16
+White                         	Maurice             	Mr.       	N	10107	15
+Williams                      	John                	Mr.       	Y	84783	15
+Williams                      	Robert              	Mr.       	Y	41233	15
+Williamson                    	Ruth                	Mrs.      	Y	86369	15
+Wilson                        	Joel                	Sir       	Y	91826	16
+Wilson                        	John                	Sir       	Y	26543	15
+Wilson                        	Mariano             	Mr.       	Y	67472	16
+Winkler                       	Jose                	Dr.       	Y	78400	15
+Winter                        	Cora                	Mrs.      	N	8978	16
+Wood                          	Marcia              	Ms.       	Y	219276	16
+Wood                          	Michelle            	Mrs.      	N	39560	15
+Wright                        	Richie              	Sir       	Y	106818	15
+Young                         	William             	Mr.       	Y	51127	15
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q35.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q35.sql.out
new file mode 100644
index 0000000000000..0c6741654b7ee
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q35.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<ca_state:string,cd_gender:string,cd_marital_status:string,cd_dep_count:int,cnt1:bigint,avg(cd_dep_count):double,max(cd_dep_count):int,sum(cd_dep_count):bigint,cd_dep_employed_count:int,cnt2:bigint,avg(cd_dep_employed_count):double,max(cd_dep_employed_count):int,sum(cd_dep_employed_count):bigint,cd_dep_college_count:int,cnt3:bigint,avg(cd_dep_college_count):double,max(cd_dep_college_count):int,sum(cd_dep_college_count):bigint>
+-- !query output
+NULL	F	D	0	1	0.0	0	0	2	1	2.0	2	2	2	1	2.0	2	2
+NULL	F	D	0	1	0.0	0	0	3	1	3.0	3	3	4	1	4.0	4	4
+NULL	F	D	0	1	0.0	0	0	5	1	5.0	5	5	2	1	2.0	2	2
+NULL	F	D	0	1	0.0	0	0	6	1	6.0	6	6	4	1	4.0	4	4
+NULL	F	D	1	1	1.0	1	1	1	1	1.0	1	1	1	1	1.0	1	1
+NULL	F	D	1	1	1.0	1	1	4	1	4.0	4	4	4	1	4.0	4	4
+NULL	F	D	1	1	1.0	1	1	4	1	4.0	4	4	5	1	5.0	5	5
+NULL	F	D	2	1	2.0	2	2	0	1	0.0	0	0	4	1	4.0	4	4
+NULL	F	D	2	1	2.0	2	2	1	1	1.0	1	1	3	1	3.0	3	3
+NULL	F	D	2	1	2.0	2	2	6	1	6.0	6	6	1	1	1.0	1	1
+NULL	F	D	3	1	3.0	3	3	3	1	3.0	3	3	2	1	2.0	2	2
+NULL	F	D	3	1	3.0	3	3	3	1	3.0	3	3	6	1	6.0	6	6
+NULL	F	D	3	1	3.0	3	3	4	1	4.0	4	4	1	1	1.0	1	1
+NULL	F	D	4	1	4.0	4	4	0	1	0.0	0	0	3	1	3.0	3	3
+NULL	F	D	4	1	4.0	4	4	1	1	1.0	1	1	1	1	1.0	1	1
+NULL	F	D	4	1	4.0	4	4	1	1	1.0	1	1	4	1	4.0	4	4
+NULL	F	D	4	1	4.0	4	4	5	1	5.0	5	5	6	1	6.0	6	6
+NULL	F	D	5	1	5.0	5	5	4	1	4.0	4	4	3	1	3.0	3	3
+NULL	F	D	5	1	5.0	5	5	5	1	5.0	5	5	2	1	2.0	2	2
+NULL	F	D	6	1	6.0	6	6	1	1	1.0	1	1	3	1	3.0	3	3
+NULL	F	D	6	1	6.0	6	6	2	1	2.0	2	2	2	1	2.0	2	2
+NULL	F	D	6	1	6.0	6	6	4	1	4.0	4	4	1	1	1.0	1	1
+NULL	F	M	0	1	0.0	0	0	5	1	5.0	5	5	5	1	5.0	5	5
+NULL	F	M	1	1	1.0	1	1	3	1	3.0	3	3	0	1	0.0	0	0
+NULL	F	M	1	1	1.0	1	1	6	1	6.0	6	6	0	1	0.0	0	0
+NULL	F	M	1	1	1.0	1	1	6	1	6.0	6	6	1	1	1.0	1	1
+NULL	F	M	2	1	2.0	2	2	2	1	2.0	2	2	6	1	6.0	6	6
+NULL	F	M	2	1	2.0	2	2	4	1	4.0	4	4	4	1	4.0	4	4
+NULL	F	M	3	1	3.0	3	3	2	1	2.0	2	2	1	1	1.0	1	1
+NULL	F	M	3	1	3.0	3	3	5	1	5.0	5	5	0	1	0.0	0	0
+NULL	F	M	3	1	3.0	3	3	5	1	5.0	5	5	1	1	1.0	1	1
+NULL	F	M	4	1	4.0	4	4	1	1	1.0	1	1	4	1	4.0	4	4
+NULL	F	M	4	1	4.0	4	4	2	1	2.0	2	2	1	1	1.0	1	1
+NULL	F	M	4	1	4.0	4	4	3	1	3.0	3	3	3	1	3.0	3	3
+NULL	F	M	5	1	5.0	5	5	2	1	2.0	2	2	2	1	2.0	2	2
+NULL	F	M	6	1	6.0	6	6	1	1	1.0	1	1	1	1	1.0	1	1
+NULL	F	M	6	1	6.0	6	6	5	1	5.0	5	5	6	1	6.0	6	6
+NULL	F	S	0	1	0.0	0	0	3	1	3.0	3	3	6	1	6.0	6	6
+NULL	F	S	1	1	1.0	1	1	0	1	0.0	0	0	4	1	4.0	4	4
+NULL	F	S	1	1	1.0	1	1	1	1	1.0	1	1	2	1	2.0	2	2
+NULL	F	S	1	1	1.0	1	1	2	1	2.0	2	2	6	1	6.0	6	6
+NULL	F	S	1	1	1.0	1	1	5	1	5.0	5	5	5	1	5.0	5	5
+NULL	F	S	2	1	2.0	2	2	0	1	0.0	0	0	3	1	3.0	3	3
+NULL	F	S	2	2	2.0	2	4	5	2	5.0	5	10	6	2	6.0	6	12
+NULL	F	S	3	1	3.0	3	3	0	1	0.0	0	0	4	1	4.0	4	4
+NULL	F	S	3	1	3.0	3	3	2	1	2.0	2	2	1	1	1.0	1	1
+NULL	F	S	3	1	3.0	3	3	2	1	2.0	2	2	5	1	5.0	5	5
+NULL	F	S	3	1	3.0	3	3	3	1	3.0	3	3	3	1	3.0	3	3
+NULL	F	S	4	1	4.0	4	4	1	1	1.0	1	1	4	1	4.0	4	4
+NULL	F	S	4	1	4.0	4	4	2	1	2.0	2	2	4	1	4.0	4	4
+NULL	F	S	5	1	5.0	5	5	6	1	6.0	6	6	0	1	0.0	0	0
+NULL	F	U	0	1	0.0	0	0	1	1	1.0	1	1	3	1	3.0	3	3
+NULL	F	U	0	1	0.0	0	0	3	1	3.0	3	3	0	1	0.0	0	0
+NULL	F	U	1	1	1.0	1	1	3	1	3.0	3	3	2	1	2.0	2	2
+NULL	F	U	1	1	1.0	1	1	5	1	5.0	5	5	6	1	6.0	6	6
+NULL	F	U	2	1	2.0	2	2	0	1	0.0	0	0	1	1	1.0	1	1
+NULL	F	U	2	1	2.0	2	2	4	1	4.0	4	4	4	1	4.0	4	4
+NULL	F	U	3	2	3.0	3	6	1	2	1.0	1	2	6	2	6.0	6	12
+NULL	F	U	4	1	4.0	4	4	0	1	0.0	0	0	4	1	4.0	4	4
+NULL	F	U	5	1	5.0	5	5	3	1	3.0	3	3	6	1	6.0	6	6
+NULL	F	U	6	1	6.0	6	6	2	1	2.0	2	2	2	1	2.0	2	2
+NULL	F	U	6	1	6.0	6	6	4	1	4.0	4	4	4	1	4.0	4	4
+NULL	F	U	6	1	6.0	6	6	5	1	5.0	5	5	0	1	0.0	0	0
+NULL	F	U	6	1	6.0	6	6	5	1	5.0	5	5	6	1	6.0	6	6
+NULL	F	W	0	1	0.0	0	0	0	1	0.0	0	0	4	1	4.0	4	4
+NULL	F	W	0	1	0.0	0	0	5	1	5.0	5	5	5	1	5.0	5	5
+NULL	F	W	1	1	1.0	1	1	3	1	3.0	3	3	4	1	4.0	4	4
+NULL	F	W	2	1	2.0	2	2	0	1	0.0	0	0	5	1	5.0	5	5
+NULL	F	W	3	1	3.0	3	3	3	1	3.0	3	3	6	1	6.0	6	6
+NULL	F	W	3	1	3.0	3	3	6	1	6.0	6	6	6	1	6.0	6	6
+NULL	F	W	4	1	4.0	4	4	3	1	3.0	3	3	1	1	1.0	1	1
+NULL	F	W	5	1	5.0	5	5	1	1	1.0	1	1	1	1	1.0	1	1
+NULL	F	W	5	1	5.0	5	5	1	1	1.0	1	1	4	1	4.0	4	4
+NULL	F	W	5	1	5.0	5	5	3	1	3.0	3	3	6	1	6.0	6	6
+NULL	F	W	5	1	5.0	5	5	4	1	4.0	4	4	6	1	6.0	6	6
+NULL	F	W	6	1	6.0	6	6	0	1	0.0	0	0	5	1	5.0	5	5
+NULL	F	W	6	1	6.0	6	6	2	1	2.0	2	2	3	1	3.0	3	3
+NULL	F	W	6	1	6.0	6	6	5	1	5.0	5	5	5	1	5.0	5	5
+NULL	M	D	0	1	0.0	0	0	3	1	3.0	3	3	0	1	0.0	0	0
+NULL	M	D	1	1	1.0	1	1	3	1	3.0	3	3	0	1	0.0	0	0
+NULL	M	D	1	1	1.0	1	1	3	1	3.0	3	3	2	1	2.0	2	2
+NULL	M	D	2	1	2.0	2	2	0	1	0.0	0	0	6	1	6.0	6	6
+NULL	M	D	2	1	2.0	2	2	4	1	4.0	4	4	4	1	4.0	4	4
+NULL	M	D	2	1	2.0	2	2	5	1	5.0	5	5	3	1	3.0	3	3
+NULL	M	D	3	1	3.0	3	3	1	1	1.0	1	1	5	1	5.0	5	5
+NULL	M	D	3	1	3.0	3	3	2	1	2.0	2	2	3	1	3.0	3	3
+NULL	M	D	4	1	4.0	4	4	5	1	5.0	5	5	2	1	2.0	2	2
+NULL	M	D	6	1	6.0	6	6	1	1	1.0	1	1	6	1	6.0	6	6
+NULL	M	D	6	1	6.0	6	6	3	1	3.0	3	3	1	1	1.0	1	1
+NULL	M	M	0	1	0.0	0	0	0	1	0.0	0	0	1	1	1.0	1	1
+NULL	M	M	0	2	0.0	0	0	1	2	1.0	1	2	2	2	2.0	2	4
+NULL	M	M	0	1	0.0	0	0	2	1	2.0	2	2	1	1	1.0	1	1
+NULL	M	M	0	1	0.0	0	0	3	1	3.0	3	3	5	1	5.0	5	5
+NULL	M	M	0	1	0.0	0	0	5	1	5.0	5	5	0	1	0.0	0	0
+NULL	M	M	1	1	1.0	1	1	0	1	0.0	0	0	1	1	1.0	1	1
+NULL	M	M	1	1	1.0	1	1	0	1	0.0	0	0	2	1	2.0	2	2
+NULL	M	M	2	1	2.0	2	2	6	1	6.0	6	6	5	1	5.0	5	5
+NULL	M	M	3	1	3.0	3	3	5	1	5.0	5	5	1	1	1.0	1	1
+NULL	M	M	3	1	3.0	3	3	6	1	6.0	6	6	4	1	4.0	4	4
+NULL	M	M	4	1	4.0	4	4	1	1	1.0	1	1	3	1	3.0	3	3
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q35a.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q35a.sql.out
new file mode 100644
index 0000000000000..5d57fd93ef7d2
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q35a.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<ca_state:string,cd_gender:string,cd_marital_status:string,cd_dep_count:int,cnt1:bigint,avg(cd_dep_count):double,max(cd_dep_count):int,sum(cd_dep_count):bigint,cd_dep_employed_count:int,cnt2:bigint,avg(cd_dep_employed_count):double,max(cd_dep_employed_count):int,sum(cd_dep_employed_count):bigint,cd_dep_college_count:int,cnt3:bigint,avg(cd_dep_college_count):double,max(cd_dep_college_count):int,sum(cd_dep_college_count):bigint>
+-- !query output
+NULL	F	D	0	1	0.0	0	0	2	1	2.0	2	2	2	1	2.0	2	2
+NULL	F	D	1	1	1.0	1	1	1	1	1.0	1	1	1	1	1.0	1	1
+NULL	F	D	1	1	1.0	1	1	4	1	4.0	4	4	3	1	3.0	3	3
+NULL	F	D	1	1	1.0	1	1	4	1	4.0	4	4	6	1	6.0	6	6
+NULL	F	D	2	1	2.0	2	2	2	1	2.0	2	2	5	1	5.0	5	5
+NULL	F	D	2	1	2.0	2	2	6	1	6.0	6	6	3	1	3.0	3	3
+NULL	F	D	3	1	3.0	3	3	0	1	0.0	0	0	2	1	2.0	2	2
+NULL	F	D	3	1	3.0	3	3	6	1	6.0	6	6	3	1	3.0	3	3
+NULL	F	D	4	1	4.0	4	4	1	1	1.0	1	1	6	1	6.0	6	6
+NULL	F	D	4	1	4.0	4	4	3	1	3.0	3	3	1	1	1.0	1	1
+NULL	F	D	4	1	4.0	4	4	4	1	4.0	4	4	4	1	4.0	4	4
+NULL	F	D	4	2	4.0	4	8	5	2	5.0	5	10	4	2	4.0	4	8
+NULL	F	D	5	1	5.0	5	5	5	1	5.0	5	5	2	1	2.0	2	2
+NULL	F	D	5	1	5.0	5	5	6	1	6.0	6	6	4	1	4.0	4	4
+NULL	F	D	6	1	6.0	6	6	2	1	2.0	2	2	1	1	1.0	1	1
+NULL	F	D	6	1	6.0	6	6	3	1	3.0	3	3	0	1	0.0	0	0
+NULL	F	D	6	1	6.0	6	6	5	1	5.0	5	5	1	1	1.0	1	1
+NULL	F	D	6	1	6.0	6	6	5	1	5.0	5	5	4	1	4.0	4	4
+NULL	F	M	0	1	0.0	0	0	1	1	1.0	1	1	1	1	1.0	1	1
+NULL	F	M	0	1	0.0	0	0	2	1	2.0	2	2	6	1	6.0	6	6
+NULL	F	M	0	1	0.0	0	0	3	1	3.0	3	3	4	1	4.0	4	4
+NULL	F	M	0	1	0.0	0	0	5	1	5.0	5	5	3	1	3.0	3	3
+NULL	F	M	1	1	1.0	1	1	6	1	6.0	6	6	0	1	0.0	0	0
+NULL	F	M	1	1	1.0	1	1	6	1	6.0	6	6	3	1	3.0	3	3
+NULL	F	M	3	1	3.0	3	3	0	1	0.0	0	0	1	1	1.0	1	1
+NULL	F	M	3	1	3.0	3	3	1	1	1.0	1	1	4	1	4.0	4	4
+NULL	F	M	3	2	3.0	3	6	4	2	4.0	4	8	5	2	5.0	5	10
+NULL	F	M	3	1	3.0	3	3	5	1	5.0	5	5	6	1	6.0	6	6
+NULL	F	M	3	1	3.0	3	3	6	1	6.0	6	6	1	1	1.0	1	1
+NULL	F	M	4	1	4.0	4	4	3	1	3.0	3	3	2	1	2.0	2	2
+NULL	F	M	4	1	4.0	4	4	6	1	6.0	6	6	1	1	1.0	1	1
+NULL	F	M	4	1	4.0	4	4	6	1	6.0	6	6	4	1	4.0	4	4
+NULL	F	M	5	1	5.0	5	5	1	1	1.0	1	1	5	1	5.0	5	5
+NULL	F	M	5	1	5.0	5	5	2	1	2.0	2	2	2	1	2.0	2	2
+NULL	F	M	6	1	6.0	6	6	0	1	0.0	0	0	5	1	5.0	5	5
+NULL	F	M	6	1	6.0	6	6	1	1	1.0	1	1	4	1	4.0	4	4
+NULL	F	M	6	1	6.0	6	6	6	1	6.0	6	6	3	1	3.0	3	3
+NULL	F	S	0	1	0.0	0	0	3	1	3.0	3	3	3	1	3.0	3	3
+NULL	F	S	0	1	0.0	0	0	4	1	4.0	4	4	2	1	2.0	2	2
+NULL	F	S	0	1	0.0	0	0	5	1	5.0	5	5	1	1	1.0	1	1
+NULL	F	S	0	1	0.0	0	0	5	1	5.0	5	5	2	1	2.0	2	2
+NULL	F	S	1	1	1.0	1	1	1	1	1.0	1	1	4	1	4.0	4	4
+NULL	F	S	1	1	1.0	1	1	2	1	2.0	2	2	3	1	3.0	3	3
+NULL	F	S	1	1	1.0	1	1	3	1	3.0	3	3	6	1	6.0	6	6
+NULL	F	S	2	1	2.0	2	2	0	1	0.0	0	0	6	1	6.0	6	6
+NULL	F	S	2	1	2.0	2	2	5	1	5.0	5	5	4	1	4.0	4	4
+NULL	F	S	2	1	2.0	2	2	6	1	6.0	6	6	6	1	6.0	6	6
+NULL	F	S	4	1	4.0	4	4	3	1	3.0	3	3	5	1	5.0	5	5
+NULL	F	U	0	1	0.0	0	0	2	1	2.0	2	2	0	1	0.0	0	0
+NULL	F	U	0	1	0.0	0	0	3	1	3.0	3	3	0	1	0.0	0	0
+NULL	F	U	0	1	0.0	0	0	3	1	3.0	3	3	6	1	6.0	6	6
+NULL	F	U	0	1	0.0	0	0	6	1	6.0	6	6	6	1	6.0	6	6
+NULL	F	U	1	1	1.0	1	1	0	1	0.0	0	0	6	1	6.0	6	6
+NULL	F	U	1	1	1.0	1	1	3	1	3.0	3	3	5	1	5.0	5	5
+NULL	F	U	1	1	1.0	1	1	6	1	6.0	6	6	5	1	5.0	5	5
+NULL	F	U	2	1	2.0	2	2	0	1	0.0	0	0	5	1	5.0	5	5
+NULL	F	U	2	2	2.0	2	4	1	2	1.0	1	2	0	2	0.0	0	0
+NULL	F	U	2	1	2.0	2	2	3	1	3.0	3	3	4	1	4.0	4	4
+NULL	F	U	2	1	2.0	2	2	4	1	4.0	4	4	6	1	6.0	6	6
+NULL	F	U	3	1	3.0	3	3	0	1	0.0	0	0	6	1	6.0	6	6
+NULL	F	U	3	1	3.0	3	3	2	1	2.0	2	2	0	1	0.0	0	0
+NULL	F	U	4	1	4.0	4	4	0	1	0.0	0	0	6	1	6.0	6	6
+NULL	F	U	4	1	4.0	4	4	1	1	1.0	1	1	0	1	0.0	0	0
+NULL	F	U	5	1	5.0	5	5	0	1	0.0	0	0	2	1	2.0	2	2
+NULL	F	U	5	1	5.0	5	5	1	1	1.0	1	1	2	1	2.0	2	2
+NULL	F	U	5	1	5.0	5	5	4	1	4.0	4	4	4	1	4.0	4	4
+NULL	F	U	6	1	6.0	6	6	0	1	0.0	0	0	0	1	0.0	0	0
+NULL	F	U	6	1	6.0	6	6	6	1	6.0	6	6	4	1	4.0	4	4
+NULL	F	W	1	1	1.0	1	1	3	1	3.0	3	3	4	1	4.0	4	4
+NULL	F	W	3	1	3.0	3	3	4	1	4.0	4	4	1	1	1.0	1	1
+NULL	F	W	4	1	4.0	4	4	3	1	3.0	3	3	4	1	4.0	4	4
+NULL	F	W	4	1	4.0	4	4	5	1	5.0	5	5	3	1	3.0	3	3
+NULL	F	W	5	1	5.0	5	5	2	1	2.0	2	2	3	1	3.0	3	3
+NULL	F	W	6	1	6.0	6	6	2	1	2.0	2	2	0	1	0.0	0	0
+NULL	F	W	6	1	6.0	6	6	2	1	2.0	2	2	2	1	2.0	2	2
+NULL	F	W	6	1	6.0	6	6	2	1	2.0	2	2	3	1	3.0	3	3
+NULL	F	W	6	1	6.0	6	6	2	1	2.0	2	2	6	1	6.0	6	6
+NULL	F	W	6	1	6.0	6	6	4	1	4.0	4	4	6	1	6.0	6	6
+NULL	F	W	6	1	6.0	6	6	5	1	5.0	5	5	0	1	0.0	0	0
+NULL	F	W	6	1	6.0	6	6	6	1	6.0	6	6	5	1	5.0	5	5
+NULL	M	D	0	1	0.0	0	0	0	1	0.0	0	0	6	1	6.0	6	6
+NULL	M	D	0	1	0.0	0	0	3	1	3.0	3	3	0	1	0.0	0	0
+NULL	M	D	0	1	0.0	0	0	5	1	5.0	5	5	5	1	5.0	5	5
+NULL	M	D	1	1	1.0	1	1	2	1	2.0	2	2	0	1	0.0	0	0
+NULL	M	D	1	1	1.0	1	1	2	1	2.0	2	2	1	1	1.0	1	1
+NULL	M	D	1	1	1.0	1	1	2	1	2.0	2	2	6	1	6.0	6	6
+NULL	M	D	1	1	1.0	1	1	3	1	3.0	3	3	4	1	4.0	4	4
+NULL	M	D	2	1	2.0	2	2	1	1	1.0	1	1	0	1	0.0	0	0
+NULL	M	D	2	1	2.0	2	2	1	1	1.0	1	1	4	1	4.0	4	4
+NULL	M	D	2	1	2.0	2	2	4	1	4.0	4	4	6	1	6.0	6	6
+NULL	M	D	2	1	2.0	2	2	6	1	6.0	6	6	1	1	1.0	1	1
+NULL	M	D	3	1	3.0	3	3	0	1	0.0	0	0	6	1	6.0	6	6
+NULL	M	D	3	1	3.0	3	3	3	1	3.0	3	3	0	1	0.0	0	0
+NULL	M	D	3	1	3.0	3	3	3	1	3.0	3	3	3	1	3.0	3	3
+NULL	M	D	3	1	3.0	3	3	5	1	5.0	5	5	2	1	2.0	2	2
+NULL	M	D	3	1	3.0	3	3	5	1	5.0	5	5	5	1	5.0	5	5
+NULL	M	D	4	1	4.0	4	4	4	1	4.0	4	4	5	1	5.0	5	5
+NULL	M	D	4	1	4.0	4	4	6	1	6.0	6	6	5	1	5.0	5	5
+NULL	M	D	5	1	5.0	5	5	2	1	2.0	2	2	3	1	3.0	3	3
+NULL	M	D	6	1	6.0	6	6	3	1	3.0	3	3	1	1	1.0	1	1
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q36a.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q36a.sql.out
new file mode 100644
index 0000000000000..a74e6211b49af
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q36a.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<gross_margin:decimal(38,20),i_category:string,i_class:string,lochierarchy:int,rank_within_parent:int>
+-- !query output
+-0.43310777865000000000	NULL	NULL	2	1
+-0.44057752675000000000	Home                                              	NULL	1	1
+-0.43759152110000000000	Music                                             	NULL	1	2
+-0.43708103961000000000	NULL	NULL	1	3
+-0.43616253139000000000	Shoes                                             	NULL	1	4
+-0.43567118609000000000	Children                                          	NULL	1	5
+-0.43423932352000000000	Sports                                            	NULL	1	6
+-0.43342977300000000000	Electronics                                       	NULL	1	7
+-0.43243283121000000000	Women                                             	NULL	1	8
+-0.43164166900000000000	Men                                               	NULL	1	9
+-0.42516187690000000000	Books                                             	NULL	1	10
+-0.42448713381000000000	Jewelry                                           	NULL	1	11
+-0.73902664238792748962	NULL	shirts                                            	0	1
+-0.61125804873635587486	NULL	country                                           	0	2
+-0.53129803597069255822	NULL	dresses                                           	0	3
+-0.51266635289382758517	NULL	athletic                                          	0	4
+-0.45290387783638603924	NULL	mens                                              	0	5
+-0.41288056661656330013	NULL	accessories                                       	0	6
+-0.40784754677005682440	NULL	NULL	0	7
+-0.34254844860867375832	NULL	baseball                                          	0	8
+-0.32511461675631534897	NULL	infants                                           	0	9
+-0.44733955704648003493	Books                                             	computers                                         	0	1
+-0.44221358112622373783	Books                                             	home repair                                       	0	2
+-0.44131129175272951442	Books                                             	romance                                           	0	3
+-0.43954111564375046074	Books                                             	history                                           	0	4
+-0.43921337505389731821	Books                                             	mystery                                           	0	5
+-0.43904020269360481109	Books                                             	sports                                            	0	6
+-0.42821476999837619396	Books                                             	travel                                            	0	7
+-0.42609067296303848297	Books                                             	cooking                                           	0	8
+-0.42538995145338568328	Books                                             	fiction                                           	0	9
+-0.42446563616188232944	Books                                             	arts                                              	0	10
+-0.42424821311884350413	Books                                             	parenting                                         	0	11
+-0.41822014479424203008	Books                                             	reference                                         	0	12
+-0.41350839325516811781	Books                                             	business                                          	0	13
+-0.40935208137315013129	Books                                             	science                                           	0	14
+-0.40159380735731858928	Books                                             	self-help                                         	0	15
+-0.36957884843305744526	Books                                             	entertainments                                    	0	16
+-0.44602461556731552282	Children                                          	school-uniforms                                   	0	1
+-0.44141106040000560852	Children                                          	toddlers                                          	0	2
+-0.43479886701046623711	Children                                          	infants                                           	0	3
+-0.41900662971936329442	Children                                          	newborn                                           	0	4
+-0.41526603781609697786	Children                                          	NULL	0	5
+-0.45347482218635333366	Electronics                                       	personal                                          	0	1
+-0.44349670349829474271	Electronics                                       	stereo                                            	0	2
+-0.44262427232850112058	Electronics                                       	automotive                                        	0	3
+-0.44115886172705231970	Electronics                                       	portable                                          	0	4
+-0.43972786651639318010	Electronics                                       	memory                                            	0	5
+-0.43889275271590953040	Electronics                                       	scanners                                          	0	6
+-0.43879181695132886061	Electronics                                       	karoke                                            	0	7
+-0.43743655149948399284	Electronics                                       	dvd/vcr players                                   	0	8
+-0.43737666390514154910	Electronics                                       	cameras                                           	0	9
+-0.43390499017233926812	Electronics                                       	wireless                                          	0	10
+-0.43163869754114299547	Electronics                                       	audio                                             	0	11
+-0.42958938669780912634	Electronics                                       	camcorders                                        	0	12
+-0.42872845803629855724	Electronics                                       	musical                                           	0	13
+-0.42228240153396399656	Electronics                                       	televisions                                       	0	14
+-0.41893847772039275795	Electronics                                       	monitors                                          	0	15
+-0.39793878022746331540	Electronics                                       	disk drives                                       	0	16
+-0.49051156860507320113	Home                                              	NULL	0	1
+-0.48431476750686752965	Home                                              	blinds/shades                                     	0	2
+-0.47545837941951440918	Home                                              	bathroom                                          	0	3
+-0.45726228921216284093	Home                                              	rugs                                              	0	4
+-0.45540507568891021759	Home                                              	furniture                                         	0	5
+-0.45303572267019508501	Home                                              	flatware                                          	0	6
+-0.44755542058111800358	Home                                              	tables                                            	0	7
+-0.44419847780930149402	Home                                              	wallpaper                                         	0	8
+-0.44092345226680695671	Home                                              	glassware                                         	0	9
+-0.43877591834074789745	Home                                              	decor                                             	0	10
+-0.43765482553654514822	Home                                              	accent                                            	0	11
+-0.43188199218974854630	Home                                              	bedding                                           	0	12
+-0.43107417904272222899	Home                                              	kids                                              	0	13
+-0.42474436355625900935	Home                                              	lighting                                          	0	14
+-0.41783311109052416746	Home                                              	curtains/drapes                                   	0	15
+-0.41767111806961188479	Home                                              	mattresses                                        	0	16
+-0.40562188698541221499	Home                                              	paint                                             	0	17
+-0.45165056505480816921	Jewelry                                           	jewelry boxes                                     	0	1
+-0.44372227804836590137	Jewelry                                           	estate                                            	0	2
+-0.44251815032563188894	Jewelry                                           	gold                                              	0	3
+-0.43978127753996883542	Jewelry                                           	consignment                                       	0	4
+-0.43821750044359339153	Jewelry                                           	custom                                            	0	5
+-0.43439645036479672989	Jewelry                                           	bracelets                                         	0	6
+-0.43208398325687772942	Jewelry                                           	loose stones                                      	0	7
+-0.43060897375114375156	Jewelry                                           	diamonds                                          	0	8
+-0.42847505748860847066	Jewelry                                           	costume                                           	0	9
+-0.42667449062277843561	Jewelry                                           	rings                                             	0	10
+-0.41987969011585456826	Jewelry                                           	mens watch                                        	0	11
+-0.41624621972944533035	Jewelry                                           	semi-precious                                     	0	12
+-0.41148949162100715771	Jewelry                                           	womens watch                                      	0	13
+-0.39725668174847694299	Jewelry                                           	birdal                                            	0	14
+-0.39665274051903254057	Jewelry                                           	pendants                                          	0	15
+-0.38423525233438861010	Jewelry                                           	earings                                           	0	16
+-0.44464388887858793403	Men                                               	shirts                                            	0	1
+-0.43719860800637369827	Men                                               	accessories                                       	0	2
+-0.43164606665359630905	Men                                               	sports-apparel                                    	0	3
+-0.41530906677293519754	Men                                               	pants                                             	0	4
+-0.38332708894803499123	Men                                               	NULL	0	5
+-0.47339698705534020269	Music                                             	NULL	0	1
+-0.44193214675249008923	Music                                             	rock                                              	0	2
+-0.44008174913565459246	Music                                             	country                                           	0	3
+-0.43863444992223641373	Music                                             	pop                                               	0	4
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q47.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q47.sql.out
new file mode 100644
index 0000000000000..0bdc42a4751cb
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q47.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_category:string,d_year:int,d_moy:int,avg_monthly_sales:decimal(21,6),sum_sales:decimal(17,2),psum:decimal(17,2),nsum:decimal(17,2)>
+-- !query output
+Shoes                                             	1999	3	5607.487500	2197.48	3271.66	2831.67
+Shoes                                             	1999	2	5643.938333	2393.31	4463.11	2652.44
+Shoes                                             	1999	4	5640.362500	2416.57	3348.90	2987.78
+Men                                               	1999	6	4702.116667	1534.95	2666.37	2514.13
+Men                                               	1999	7	5330.618333	2218.63	3182.74	7436.83
+Shoes                                             	1999	4	5338.852500	2233.60	3470.43	2832.41
+Music                                             	1999	5	5139.465000	2034.96	3149.72	3648.17
+Men                                               	1999	7	5748.707500	2645.88	3432.67	7646.91
+Men                                               	1999	3	4915.190000	1815.92	2884.81	2956.00
+Women                                             	1999	6	4586.300000	1508.07	2992.12	3059.37
+Shoes                                             	1999	4	5374.032500	2322.80	2484.06	3313.69
+Men                                               	1999	6	4596.057500	1577.69	2457.43	2439.68
+Men                                               	1999	3	5839.670833	2825.07	3157.46	3531.36
+Men                                               	1999	4	5342.149167	2347.82	2787.31	3588.16
+Shoes                                             	1999	3	5643.938333	2652.44	2393.31	3008.88
+Men                                               	1999	3	5475.719167	2515.22	2709.11	2702.85
+Shoes                                             	1999	6	5338.852500	2388.17	2832.41	4216.10
+Shoes                                             	1999	2	5065.706667	2119.85	3439.05	2640.59
+Shoes                                             	1999	2	4713.009167	1774.19	3366.21	1940.05
+Shoes                                             	1999	3	5374.032500	2484.06	2994.92	2322.80
+Men                                               	1999	7	5342.149167	2454.72	2766.42	7665.06
+Men                                               	1999	4	4741.993333	1879.47	3419.59	2634.29
+Music                                             	1999	4	4723.742500	1866.30	2384.82	2931.55
+Shoes                                             	1999	4	4732.205000	1875.08	2686.46	3422.03
+Children                                          	1999	6	4849.967500	2002.59	2590.23	3380.67
+Men                                               	1999	6	4920.419167	2077.36	3402.55	3347.44
+Men                                               	1999	4	4596.057500	1762.17	2728.14	2457.43
+Children                                          	1999	4	4739.001667	1923.04	2309.91	2849.64
+Music                                             	1999	3	4816.848333	2010.47	2539.57	2940.38
+Shoes                                             	1999	2	4707.697500	1903.70	2693.49	3474.34
+Shoes                                             	1999	2	4443.995833	1642.32	3972.50	2319.04
+Men                                               	1999	6	5839.670833	3053.78	3151.27	3622.65
+Shoes                                             	1999	4	5607.487500	2831.67	2197.48	4187.54
+Men                                               	1999	3	5598.894167	2824.13	3154.80	3135.60
+Shoes                                             	1999	3	4713.009167	1940.05	1774.19	2496.18
+Men                                               	1999	4	5475.719167	2702.85	2515.22	4364.56
+Shoes                                             	1999	4	4596.537500	1825.00	2777.56	3234.34
+Music                                             	1999	6	4332.550833	1563.26	2484.37	2460.11
+Men                                               	1999	2	5102.436667	2333.32	3417.85	2536.68
+Men                                               	1999	2	5475.719167	2709.11	4740.72	2515.22
+Men                                               	1999	6	5475.719167	2723.08	4364.56	4000.42
+Shoes                                             	1999	2	5338.852500	2587.72	3837.18	3470.43
+Children                                          	1999	6	4426.704167	1683.86	2186.50	1970.76
+Shoes                                             	1999	6	5643.938333	2902.72	3046.01	3994.18
+Shoes                                             	1999	6	5374.032500	2637.31	3313.69	3158.65
+Men                                               	1999	3	4920.419167	2183.89	3598.63	2841.78
+Men                                               	1999	3	5330.618333	2602.65	2613.45	2624.00
+Women                                             	1999	6	4366.627500	1640.10	3159.43	2882.91
+Music                                             	1999	2	5139.465000	2413.63	4627.89	2773.28
+Men                                               	1999	2	4929.611667	2207.89	3940.06	2246.76
+Women                                             	1999	2	4551.444167	1833.08	3729.30	2940.96
+Men                                               	1999	2	5330.618333	2613.45	3859.71	2602.65
+Children                                          	1999	2	4640.935833	1927.95	3524.16	2584.03
+Children                                          	1999	7	4565.828333	1853.57	2846.40	6447.36
+Women                                             	1999	6	4551.444167	1840.98	2478.75	3174.74
+Men                                               	1999	4	5330.618333	2624.00	2602.65	3722.18
+Children                                          	1999	3	5024.325000	2319.29	2691.31	2350.53
+Children                                          	1999	2	4836.831667	2132.80	3984.13	2182.12
+Children                                          	1999	4	4640.935833	1950.03	2584.03	2919.51
+Shoes                                             	1999	3	4791.740833	2100.87	2211.40	3278.75
+Men                                               	1999	5	5839.670833	3151.27	3531.36	3053.78
+Men                                               	1999	3	4929.611667	2246.76	2207.89	3208.14
+Men                                               	1999	2	5839.670833	3157.46	3425.68	2825.07
+Men                                               	1999	5	5311.965833	2631.11	2819.12	3022.54
+Music                                             	1999	2	4388.770833	1714.25	2228.69	2407.93
+Shoes                                             	1999	2	4989.784167	2315.46	4430.70	2976.26
+Children                                          	1999	4	5024.325000	2350.53	2319.29	2984.92
+Women                                             	1999	3	4586.300000	1917.37	3282.12	2557.46
+Children                                          	1999	3	4836.831667	2182.12	2132.80	3466.25
+Shoes                                             	1999	5	5640.362500	2987.78	2416.57	3437.87
+Men                                               	1999	4	5071.261667	2425.64	2696.60	2641.17
+Shoes                                             	1999	6	4443.995833	1801.87	2297.76	3078.40
+Children                                          	1999	2	4480.353333	1838.88	3250.53	2375.04
+Men                                               	1999	7	5102.436667	2464.06	2798.80	6978.09
+Shoes                                             	1999	4	5643.938333	3008.88	2652.44	3046.01
+Men                                               	1999	2	5071.261667	2438.50	4282.24	2696.60
+Children                                          	1999	6	4525.318333	1896.34	2883.95	2727.36
+Children                                          	1999	4	4212.889167	1588.72	2296.49	3077.12
+Shoes                                             	1999	4	4884.260000	2261.01	2502.36	3210.47
+Men                                               	1999	3	5748.707500	3132.90	3747.34	3768.23
+Shoes                                             	1999	2	5640.362500	3027.85	4285.15	3348.90
+Music                                             	1999	3	4490.221667	1883.06	2321.31	2211.82
+Children                                          	1999	2	4423.506667	1816.43	3192.19	3334.11
+Music                                             	1999	6	4894.035000	2288.21	3156.77	2845.45
+Music                                             	1999	5	4829.768333	2225.46	3380.94	2782.42
+Women                                             	1999	3	4268.562500	1665.68	1758.65	2082.99
+Children                                          	1999	3	4267.374167	1667.03	2579.77	2802.05
+Men                                               	1999	3	4987.821667	2387.78	2962.40	2928.83
+Women                                             	1999	3	4309.575833	1710.42	2004.22	2742.41
+Shoes                                             	1999	5	5643.938333	3046.01	3008.88	2902.72
+Music                                             	1999	4	4518.543333	1922.23	2683.26	2227.02
+Children                                          	1999	3	4441.555000	1846.53	3827.47	3623.30
+Men                                               	1999	5	5748.707500	3156.57	3768.23	3432.67
+Music                                             	1999	2	4302.831667	1712.70	3561.46	2414.41
+Women                                             	1999	7	4494.382500	1904.98	2308.58	5603.14
+Women                                             	1999	4	4582.713333	1994.71	2408.40	2321.48
+Music                                             	1999	3	4426.623333	1840.51	2707.80	3147.10
+Shoes                                             	1999	2	4791.740833	2211.40	3912.90	2100.87
+Shoes                                             	1999	7	5640.362500	3062.31	3437.87	6376.30
+Children                                          	1999	3	4733.152500	2155.79	2710.50	2685.74
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q49.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q49.sql.out
new file mode 100644
index 0000000000000..31b31b7bdc02b
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q49.sql.out
@@ -0,0 +1,37 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<channel:string,item:int,return_ratio:decimal(35,20),return_rank:int,currency_rank:int>
+-- !query output
+catalog	16735	0.50505050505050505051	1	1
+catalog	12633	0.69662921348314606742	2	2
+catalog	13967	0.70000000000000000000	3	3
+catalog	12819	0.70129870129870129870	4	8
+catalog	16155	0.72043010752688172043	5	4
+catalog	17681	0.75268817204301075269	6	5
+catalog	5975	0.76404494382022471910	7	6
+catalog	11451	0.76744186046511627907	8	7
+catalog	1689	0.80219780219780219780	9	9
+catalog	10311	0.81818181818181818182	10	10
+store	5111	0.78947368421052631579	1	1
+store	11073	0.83505154639175257732	2	3
+store	14429	0.84782608695652173913	3	2
+store	15927	0.86419753086419753086	4	4
+store	10171	0.86868686868686868687	5	5
+store	12783	0.88775510204081632653	6	6
+store	11075	0.89743589743589743590	7	7
+store	12889	0.95652173913043478261	8	8
+store	1939	0.99000000000000000000	9	9
+store	4333	1.00000000000000000000	10	10
+store	10455	1.00000000000000000000	10	10
+store	12975	1.00000000000000000000	10	10
+web	10485	0.48863636363636363636	1	1
+web	4483	0.52688172043010752688	2	2
+web	8833	0.58241758241758241758	3	3
+web	1165	0.61458333333333333333	4	4
+web	17197	0.73076923076923076923	5	5
+web	10319	0.73469387755102040816	6	6
+web	13159	0.75257731958762886598	7	7
+web	9629	0.77894736842105263158	8	8
+web	5909	0.78378378378378378378	9	9
+web	7057	0.86746987951807228916	10	10
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q51a.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q51a.sql.out
new file mode 100644
index 0000000000000..62b22b5ca706f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q51a.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<item_sk:int,d_date:date,web_sales:decimal(27,2),store_sales:decimal(27,2),web_cumulative:decimal(27,2),store_cumulative:decimal(27,2)>
+-- !query output
+9	2001-01-20	155.31	NULL	155.31	4.26
+9	2001-02-01	168.10	28.81	168.10	28.81
+9	2001-02-04	NULL	34.51	168.10	34.51
+9	2001-02-15	NULL	144.79	168.10	144.79
+9	2001-04-02	249.70	NULL	249.70	181.21
+9	2001-04-09	NULL	183.64	249.70	183.64
+9	2001-04-17	260.21	230.47	260.21	230.47
+29	2001-01-07	145.14	NULL	145.14	12.18
+29	2001-01-10	NULL	53.52	145.14	53.52
+29	2001-01-11	NULL	55.81	145.14	55.81
+29	2001-02-05	NULL	88.85	145.14	88.85
+29	2001-02-08	NULL	140.80	145.14	140.80
+29	2001-02-28	215.28	NULL	215.28	212.84
+29	2001-03-30	340.91	NULL	340.91	279.82
+29	2001-04-07	NULL	306.91	340.91	306.91
+29	2001-05-26	392.06	389.83	392.06	389.83
+29	2001-06-12	NULL	391.96	392.06	391.96
+29	2001-06-29	580.42	NULL	580.42	407.31
+29	2001-07-08	NULL	429.12	580.42	429.12
+29	2001-07-12	NULL	444.37	580.42	444.37
+29	2001-07-26	NULL	501.13	580.42	501.13
+29	2001-07-28	NULL	523.55	580.42	523.55
+29	2001-07-31	NULL	547.65	580.42	547.65
+29	2001-09-15	755.81	NULL	755.81	710.20
+29	2001-09-16	NULL	742.83	755.81	742.83
+31	2001-01-23	160.83	143.12	160.83	143.12
+31	2001-01-24	247.53	NULL	247.53	143.12
+31	2001-02-04	NULL	143.12	247.53	143.12
+31	2001-02-07	NULL	166.46	247.53	166.46
+31	2001-02-15	NULL	224.58	247.53	224.58
+31	2001-02-22	NULL	243.99	247.53	243.99
+31	2001-02-26	NULL	245.24	247.53	245.24
+33	2001-02-06	143.86	NULL	143.86	100.10
+33	2001-03-06	260.39	NULL	260.39	100.10
+33	2001-03-08	311.65	NULL	311.65	100.10
+33	2001-03-17	NULL	166.59	311.65	166.59
+33	2001-04-04	NULL	195.94	311.65	195.94
+33	2001-04-11	NULL	218.41	311.65	218.41
+33	2001-04-15	NULL	258.16	311.65	258.16
+33	2001-04-26	NULL	260.73	311.65	260.73
+35	2001-04-11	218.63	NULL	218.63	175.11
+35	2001-04-13	NULL	200.14	218.63	200.14
+35	2001-04-15	NULL	213.01	218.63	213.01
+35	2001-04-21	236.74	NULL	236.74	213.01
+35	2001-05-12	250.14	NULL	250.14	213.01
+35	2001-06-03	315.73	NULL	315.73	213.01
+35	2001-06-09	NULL	213.01	315.73	213.01
+35	2001-06-12	350.47	NULL	350.47	213.01
+35	2001-06-16	NULL	240.39	350.47	240.39
+35	2001-06-29	NULL	251.30	350.47	251.30
+35	2001-07-03	NULL	251.62	350.47	251.62
+35	2001-07-06	NULL	279.62	350.47	279.62
+35	2001-07-10	NULL	281.30	350.47	281.30
+35	2001-07-19	NULL	285.09	350.47	285.09
+35	2001-07-22	NULL	306.31	350.47	306.31
+35	2001-07-28	421.29	NULL	421.29	306.31
+35	2001-07-29	434.60	NULL	434.60	306.31
+37	2001-03-07	104.29	NULL	104.29	100.87
+47	2001-01-01	7.00	5.55	7.00	5.55
+47	2001-01-15	72.85	NULL	72.85	48.96
+47	2001-01-22	NULL	58.39	72.85	58.39
+47	2001-02-17	139.16	NULL	139.16	119.31
+49	2001-05-05	361.90	NULL	361.90	340.45
+49	2001-05-21	397.02	NULL	397.02	340.45
+49	2001-05-25	479.86	NULL	479.86	340.45
+49	2001-06-03	NULL	350.29	479.86	350.29
+49	2001-06-10	488.44	NULL	488.44	350.29
+49	2001-06-20	NULL	362.21	488.44	362.21
+49	2001-06-28	527.37	NULL	527.37	362.21
+49	2001-07-19	NULL	490.09	527.37	490.09
+49	2001-07-23	532.83	NULL	532.83	490.09
+49	2001-07-26	NULL	511.76	532.83	511.76
+49	2001-07-31	556.70	NULL	556.70	511.76
+51	2001-03-24	142.03	NULL	142.03	141.46
+51	2001-04-11	221.62	NULL	221.62	208.13
+53	2001-02-01	150.99	NULL	150.99	15.99
+53	2001-02-03	NULL	92.68	150.99	92.68
+53	2001-02-09	NULL	96.53	150.99	96.53
+53	2001-02-18	NULL	98.68	150.99	98.68
+53	2001-02-20	NULL	129.22	150.99	129.22
+57	2001-03-15	147.84	NULL	147.84	108.81
+65	2001-01-07	NULL	7.39	143.43	7.39
+65	2001-01-08	NULL	21.35	143.43	21.35
+65	2001-01-16	NULL	102.46	143.43	102.46
+67	2001-02-02	120.54	13.19	120.54	13.19
+67	2001-02-19	219.36	NULL	219.36	120.94
+67	2001-03-12	NULL	203.25	219.36	203.25
+67	2001-04-13	330.07	NULL	330.07	277.79
+67	2001-04-23	NULL	284.23	330.07	284.23
+67	2001-04-27	NULL	290.21	330.07	290.21
+67	2001-04-28	NULL	320.26	330.07	320.26
+69	2001-02-22	54.17	NULL	54.17	36.00
+69	2001-02-27	NULL	45.08	54.17	45.08
+73	2001-01-18	184.35	NULL	184.35	178.75
+73	2001-01-19	185.11	NULL	185.11	178.75
+73	2001-02-09	NULL	180.42	185.11	180.42
+75	2001-01-15	NULL	9.11	19.68	9.11
+75	2001-01-31	36.37	NULL	36.37	9.11
+75	2001-02-03	NULL	14.06	36.37	14.06
+83	2001-02-03	72.95	NULL	72.95	55.10
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q57.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q57.sql.out
new file mode 100644
index 0000000000000..ea23526b956ee
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q57.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_category:string,i_brand:string,d_year:int,d_moy:int,avg_monthly_sales:decimal(21,6),sum_sales:decimal(17,2),psum:decimal(17,2),nsum:decimal(17,2)>
+-- !query output
+Music                                             	amalgscholar #x                                   	1999	2	6662.669167	1961.57	4348.07	3386.25
+Shoes                                             	amalgedu pack #x                                  	1999	2	6493.071667	2044.05	4348.88	3443.20
+Shoes                                             	exportiedu pack #x                                	1999	3	7416.141667	2980.15	4654.22	5157.83
+Children                                          	importoexporti #x                                 	1999	4	6577.143333	2152.15	3291.07	3659.32
+Shoes                                             	importoedu pack #x                                	1999	6	6926.960833	2523.33	4014.93	4254.99
+Men                                               	importoimporto #x                                 	1999	2	6707.315833	2449.22	4311.11	3583.31
+Men                                               	amalgimporto #x                                   	1999	4	7098.680833	2965.42	3526.45	4923.53
+Men                                               	exportiimporto #x                                 	1999	2	7146.240000	3013.99	6183.83	5091.17
+Children                                          	amalgexporti #x                                   	1999	4	6364.467500	2270.79	3330.83	3817.50
+Men                                               	edu packimporto #x                                	1999	4	7386.333333	3329.74	3488.01	4860.20
+Men                                               	edu packimporto #x                                	1999	2	7386.333333	3347.65	4007.40	3488.01
+Music                                             	edu packscholar #x                                	1999	7	6639.040000	2653.55	4219.52	10071.22
+Music                                             	amalgscholar #x                                   	1999	4	6719.304167	2739.33	3690.54	3872.98
+Men                                               	importoimporto #x                                 	1999	3	6610.034167	2645.24	3661.14	4282.01
+Music                                             	exportischolar #x                                 	1999	2	7043.051667	3115.83	4457.95	5258.95
+Men                                               	edu packimporto #x                                	1999	3	7386.333333	3488.01	3347.65	3329.74
+Shoes                                             	exportiedu pack #x                                	1999	3	7255.790000	3411.07	4194.64	3624.85
+Women                                             	exportiamalg #x                                   	1999	2	5646.671667	1809.52	4198.70	2172.85
+Music                                             	importoscholar #x                                 	1999	6	6279.081667	2456.98	4361.44	4256.24
+Children                                          	importoexporti #x                                 	1999	7	6786.750000	2978.82	3942.59	7809.22
+Music                                             	exportischolar #x                                 	1999	2	7041.705833	3245.77	3608.31	4127.40
+Shoes                                             	importoedu pack #x                                	1999	2	6864.320833	3104.48	3135.52	3606.28
+Shoes                                             	importoedu pack #x                                	1999	1	6864.320833	3135.52	14580.30	3104.48
+Children                                          	edu packexporti #x                                	1999	5	6511.800833	2785.92	3956.90	3906.63
+Women                                             	amalgamalg #x                                     	1999	2	6480.683333	2769.69	3353.67	3876.51
+Men                                               	exportiimporto #x                                 	1999	5	7146.240000	3440.57	3561.11	3971.13
+Women                                             	importoamalg #x                                   	1999	3	6512.794167	2808.28	3789.51	4335.27
+Men                                               	amalgimporto #x                                   	1999	2	6720.550000	3018.90	4328.03	3810.74
+Shoes                                             	exportiedu pack #x                                	1999	7	7255.790000	3557.87	4937.98	9496.49
+Music                                             	exportischolar #x                                 	1999	5	6791.260833	3096.02	3918.04	3801.90
+Men                                               	exportiimporto #x                                 	1999	1	7343.719167	3652.72	13689.13	3984.13
+Men                                               	importoimporto #x                                 	1999	5	6707.315833	3030.18	4977.33	4620.75
+Children                                          	exportiexporti #x                                 	1999	2	6386.880833	2717.07	4809.11	3355.48
+Men                                               	amalgimporto #x                                   	1999	2	7098.680833	3440.27	5293.69	3526.45
+Men                                               	importoimporto #x                                 	1999	5	6610.034167	2954.71	4282.01	3166.43
+Shoes                                             	exportiedu pack #x                                	1999	4	7255.790000	3624.85	3411.07	5169.09
+Men                                               	exportiimporto #x                                 	1999	5	7343.719167	3727.75	3729.62	4580.93
+Men                                               	exportiimporto #x                                 	1999	4	7343.719167	3729.62	4033.37	3727.75
+Music                                             	edu packscholar #x                                	1999	2	6489.175000	2875.98	4299.82	4028.97
+Men                                               	edu packimporto #x                                	1999	1	7202.242500	3614.07	15582.63	4234.79
+Music                                             	importoscholar #x                                 	1999	2	5816.271667	2229.79	2919.29	4298.41
+Men                                               	exportiimporto #x                                 	1999	4	7146.240000	3561.11	5091.17	3440.57
+Shoes                                             	exportiedu pack #x                                	1999	7	7073.462500	3493.41	4534.31	8701.59
+Music                                             	exportischolar #x                                 	1999	3	6791.260833	3218.13	3847.87	3918.04
+Shoes                                             	edu packedu pack #x                               	1999	4	6203.331667	2631.02	4424.18	4186.85
+Men                                               	amalgimporto #x                                   	1999	3	7098.680833	3526.45	3440.27	2965.42
+Men                                               	edu packimporto #x                                	1999	3	7202.242500	3639.93	4234.79	4016.25
+Children                                          	amalgexporti #x                                   	1999	2	6364.467500	2825.09	4111.08	3330.83
+Shoes                                             	edu packedu pack #x                               	1999	2	6464.239167	2928.99	4233.86	3840.97
+Shoes                                             	amalgedu pack #x                                  	1999	4	6493.071667	2962.20	3443.20	4212.60
+Music                                             	importoscholar #x                                 	1999	4	5707.844167	2179.41	3789.16	4317.53
+Shoes                                             	exportiedu pack #x                                	1999	1	7416.141667	3892.54	14170.68	4654.22
+Women                                             	importoamalg #x                                   	1999	5	6512.794167	2991.07	4335.27	4624.86
+Music                                             	exportischolar #x                                 	1999	4	7043.051667	3521.99	5258.95	4135.21
+Women                                             	edu packamalg #x                                  	1999	2	6354.045833	2836.23	3719.67	3527.07
+Music                                             	amalgscholar #x                                   	1999	3	6123.475000	2617.39	3080.43	4919.93
+Shoes                                             	amalgedu pack #x                                  	1999	7	6674.896667	3178.12	3342.98	8050.81
+Men                                               	amalgimporto #x                                   	1999	5	6618.534167	3127.28	4291.66	4669.62
+Women                                             	amalgamalg #x                                     	1999	6	6874.250000	3387.53	4798.69	4329.48
+Women                                             	exportiamalg #x                                   	1999	3	5646.671667	2172.85	1809.52	4461.31
+Children                                          	edu packexporti #x                                	1999	2	6112.954167	2641.54	3567.58	3196.45
+Children                                          	amalgexporti #x                                   	1999	5	6294.100833	2834.48	3317.60	3803.79
+Women                                             	edu packamalg #x                                  	1999	5	6027.880000	2575.81	2750.96	4459.01
+Music                                             	exportischolar #x                                 	1999	6	7041.705833	3589.97	4134.03	4892.26
+Music                                             	exportischolar #x                                 	1999	4	7041.705833	3593.86	4127.40	4134.03
+Men                                               	importoimporto #x                                 	1999	6	6610.034167	3166.43	2954.71	3673.39
+Music                                             	exportischolar #x                                 	1999	1	7041.705833	3608.31	15046.54	3245.77
+Music                                             	edu packscholar #x                                	1999	2	6602.385000	3173.44	3434.91	3929.40
+Music                                             	amalgscholar #x                                   	1999	6	6123.475000	2699.75	4038.47	3330.87
+Children                                          	importoexporti #x                                 	1999	4	6786.750000	3366.25	3847.60	4259.57
+Men                                               	edu packimporto #x                                	1999	1	7230.493333	3811.64	14668.93	4497.31
+Shoes                                             	importoedu pack #x                                	1999	5	6864.320833	3449.62	3869.15	3531.93
+Children                                          	edu packexporti #x                                	1999	2	6739.498333	3328.01	4986.50	3623.32
+Children                                          	importoexporti #x                                 	1999	1	6786.750000	3376.55	12504.93	5018.22
+Children                                          	edu packexporti #x                                	1999	7	6112.954167	2711.04	3254.04	9465.10
+Shoes                                             	importoedu pack #x                                	1999	3	6588.741667	3187.25	4283.21	3212.76
+Men                                               	importoimporto #x                                 	1999	3	6702.415000	3310.55	3981.06	4901.56
+Men                                               	edu packimporto #x                                	1999	1	7386.333333	4007.40	14005.45	3347.65
+Shoes                                             	importoedu pack #x                                	1999	4	6588.741667	3212.76	3187.25	3974.78
+Shoes                                             	edu packedu pack #x                               	1999	6	6203.331667	2835.78	4186.85	3192.53
+Men                                               	exportiimporto #x                                 	1999	2	7343.719167	3984.13	3652.72	4033.37
+Men                                               	amalgimporto #x                                   	1999	4	6720.550000	3364.32	3810.74	4333.58
+Children                                          	edu packexporti #x                                	1999	4	6739.498333	3389.03	3623.32	3605.25
+Shoes                                             	importoedu pack #x                                	1999	6	6864.320833	3531.93	3449.62	4414.17
+Shoes                                             	amalgedu pack #x                                  	1999	6	6674.896667	3342.98	4458.26	3178.12
+Children                                          	edu packexporti #x                                	1999	2	6511.800833	3185.28	3581.75	3410.75
+Children                                          	amalgexporti #x                                   	1999	4	6854.405833	3541.62	3854.33	3938.42
+Men                                               	exportiimporto #x                                 	1999	3	7343.719167	4033.37	3984.13	3729.62
+Men                                               	amalgimporto #x                                   	1999	3	6618.534167	3313.62	4044.36	4291.66
+Shoes                                             	exportiedu pack #x                                	1999	7	7416.141667	4121.24	4239.08	8658.42
+Women                                             	importoamalg #x                                   	1999	6	6395.326667	3102.55	4234.22	3650.03
+Children                                          	importoexporti #x                                 	1999	3	6577.143333	3291.07	3773.61	2152.15
+Women                                             	edu packamalg #x                                  	1999	4	6027.880000	2750.96	3199.35	2575.81
+Music                                             	amalgscholar #x                                   	1999	3	6662.669167	3386.25	1961.57	4799.18
+Men                                               	amalgimporto #x                                   	1999	6	7098.680833	3834.90	4923.53	4115.57
+Shoes                                             	importoedu pack #x                                	1999	3	6864.320833	3606.28	3104.48	3869.15
+Music                                             	exportischolar #x                                 	1999	6	7043.051667	3793.48	4135.21	5006.69
+Shoes                                             	edu packedu pack #x                               	1999	1	6711.753333	3473.10	15060.83	4085.86
+Men                                               	exportiimporto #x                                 	1999	1	7419.459167	4188.88	16358.86	4366.77
+Women                                             	amalgamalg #x                                     	1999	2	6362.709167	3137.03	4180.91	3181.26
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q5a.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q5a.sql.out
new file mode 100644
index 0000000000000..600c51b691b35
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q5a.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<channel:string,id:string,sales:decimal(37,2),returns:decimal(37,2),profit:decimal(38,2)>
+-- !query output
+NULL	NULL	115594110.84	2284876.41	-30784735.42
+catalog channel	NULL	39386111.98	835889.43	-4413707.25
+catalog channel	catalog_pageAAAAAAAAAABAAAAA	142709.95	0.00	-33829.11
+catalog channel	catalog_pageAAAAAAAAAADAAAAA	78358.50	0.00	-9578.35
+catalog channel	catalog_pageAAAAAAAAABAAAAAA	0.00	4176.90	-492.29
+catalog channel	catalog_pageAAAAAAAAABBAAAAA	71027.83	0.00	-27264.22
+catalog channel	catalog_pageAAAAAAAAABDAAAAA	93670.35	0.00	-3768.98
+catalog channel	catalog_pageAAAAAAAAACAAAAAA	0.00	1776.17	-1252.65
+catalog channel	catalog_pageAAAAAAAAACBAAAAA	112153.16	0.00	-19004.12
+catalog channel	catalog_pageAAAAAAAAACDAAAAA	102298.11	0.00	-14768.21
+catalog channel	catalog_pageAAAAAAAAADAAAAAA	0.00	8773.82	-4372.46
+catalog channel	catalog_pageAAAAAAAAADBAAAAA	23722.97	0.00	-22193.13
+catalog channel	catalog_pageAAAAAAAAADDAAAAA	47126.10	0.00	-6735.40
+catalog channel	catalog_pageAAAAAAAAAEAAAAAA	0.00	6234.69	-2690.18
+catalog channel	catalog_pageAAAAAAAAAEBAAAAA	45880.56	679.36	-16578.99
+catalog channel	catalog_pageAAAAAAAAAEDAAAAA	66004.24	0.00	6074.29
+catalog channel	catalog_pageAAAAAAAAAFAAAAAA	0.00	463.68	-1521.36
+catalog channel	catalog_pageAAAAAAAAAFDAAAAA	88022.65	0.00	-769.81
+catalog channel	catalog_pageAAAAAAAAAGAAAAAA	0.00	8947.80	-2922.65
+catalog channel	catalog_pageAAAAAAAAAGDAAAAA	124552.05	0.00	24880.33
+catalog channel	catalog_pageAAAAAAAAAHAAAAAA	188643.97	0.00	-33045.93
+catalog channel	catalog_pageAAAAAAAAAIAAAAAA	219950.84	518.83	-1089.03
+catalog channel	catalog_pageAAAAAAAAAJAAAAAA	196607.25	7494.81	-41407.88
+catalog channel	catalog_pageAAAAAAAAAJCAAAAA	23556.72	0.00	-5599.24
+catalog channel	catalog_pageAAAAAAAAAKAAAAAA	155095.31	0.00	-24248.39
+catalog channel	catalog_pageAAAAAAAAAKCAAAAA	13054.83	0.00	-528.84
+catalog channel	catalog_pageAAAAAAAAALAAAAAA	114177.20	1866.02	-48127.76
+catalog channel	catalog_pageAAAAAAAAALCAAAAA	10602.13	0.00	-3956.02
+catalog channel	catalog_pageAAAAAAAAAMAAAAAA	180993.77	144.00	-26262.58
+catalog channel	catalog_pageAAAAAAAAAMCAAAAA	29191.46	0.00	-8019.34
+catalog channel	catalog_pageAAAAAAAAANAAAAAA	202640.83	0.00	14603.23
+catalog channel	catalog_pageAAAAAAAAANBAAAAA	0.00	3103.80	-1162.99
+catalog channel	catalog_pageAAAAAAAAANCAAAAA	14627.25	61.80	1219.16
+catalog channel	catalog_pageAAAAAAAAAOAAAAAA	49178.54	0.00	-27661.86
+catalog channel	catalog_pageAAAAAAAAAOCAAAAA	9621.16	0.00	-4213.22
+catalog channel	catalog_pageAAAAAAAAAPAAAAAA	142216.57	0.00	21113.94
+catalog channel	catalog_pageAAAAAAAAAPBAAAAA	0.00	4876.83	-2903.30
+catalog channel	catalog_pageAAAAAAAAAPCAAAAA	17230.02	689.60	-1816.99
+catalog channel	catalog_pageAAAAAAAABAAAAAAA	0.00	1902.91	-1516.52
+catalog channel	catalog_pageAAAAAAAABABAAAAA	100018.02	0.00	-9370.45
+catalog channel	catalog_pageAAAAAAAABADAAAAA	115622.82	0.00	12126.64
+catalog channel	catalog_pageAAAAAAAABBAAAAAA	0.00	1786.21	-891.28
+catalog channel	catalog_pageAAAAAAAABBBAAAAA	76388.45	1153.20	2393.80
+catalog channel	catalog_pageAAAAAAAABBCAAAAA	0.00	831.90	-1077.03
+catalog channel	catalog_pageAAAAAAAABBDAAAAA	47684.96	0.00	-8371.50
+catalog channel	catalog_pageAAAAAAAABCAAAAAA	0.00	4371.58	-4222.90
+catalog channel	catalog_pageAAAAAAAABCBAAAAA	93859.97	1753.50	-1196.59
+catalog channel	catalog_pageAAAAAAAABCDAAAAA	73347.51	0.00	-11342.92
+catalog channel	catalog_pageAAAAAAAABDAAAAAA	0.00	3474.90	-1008.22
+catalog channel	catalog_pageAAAAAAAABDBAAAAA	42173.47	0.00	-26341.79
+catalog channel	catalog_pageAAAAAAAABDCAAAAA	0.00	305.46	-585.35
+catalog channel	catalog_pageAAAAAAAABDDAAAAA	83810.14	0.00	-1482.89
+catalog channel	catalog_pageAAAAAAAABEAAAAAA	0.00	347.39	-197.16
+catalog channel	catalog_pageAAAAAAAABEBAAAAA	97527.42	0.00	2333.17
+catalog channel	catalog_pageAAAAAAAABEDAAAAA	60506.22	0.00	-1212.87
+catalog channel	catalog_pageAAAAAAAABFAAAAAA	0.00	6528.33	-2187.98
+catalog channel	catalog_pageAAAAAAAABFDAAAAA	68737.67	0.00	-11721.44
+catalog channel	catalog_pageAAAAAAAABHAAAAAA	199767.35	1044.68	-6811.92
+catalog channel	catalog_pageAAAAAAAABIAAAAAA	205306.84	411.12	2555.23
+catalog channel	catalog_pageAAAAAAAABICAAAAA	0.00	68.02	-99.54
+catalog channel	catalog_pageAAAAAAAABJAAAAAA	187825.12	230.88	-21543.64
+catalog channel	catalog_pageAAAAAAAABJBAAAAA	0.00	727.89	-209.17
+catalog channel	catalog_pageAAAAAAAABJCAAAAA	14241.32	0.00	4283.83
+catalog channel	catalog_pageAAAAAAAABKAAAAAA	215820.15	1121.48	-7030.88
+catalog channel	catalog_pageAAAAAAAABKBAAAAA	0.00	3063.15	-3543.44
+catalog channel	catalog_pageAAAAAAAABKCAAAAA	1871.32	0.00	-2063.64
+catalog channel	catalog_pageAAAAAAAABLAAAAAA	171643.10	779.60	-15082.45
+catalog channel	catalog_pageAAAAAAAABLCAAAAA	8855.03	0.00	3013.67
+catalog channel	catalog_pageAAAAAAAABMAAAAAA	202476.57	1954.44	10034.15
+catalog channel	catalog_pageAAAAAAAABMCAAAAA	13837.48	0.00	6008.90
+catalog channel	catalog_pageAAAAAAAABNAAAAAA	197464.73	1397.02	-34977.14
+catalog channel	catalog_pageAAAAAAAABNCAAAAA	14801.05	0.00	-6910.60
+catalog channel	catalog_pageAAAAAAAABOAAAAAA	98871.66	0.00	-6493.20
+catalog channel	catalog_pageAAAAAAAABOCAAAAA	15216.80	0.00	-5722.57
+catalog channel	catalog_pageAAAAAAAABPAAAAAA	99238.38	100.20	-6353.28
+catalog channel	catalog_pageAAAAAAAABPCAAAAA	26783.54	0.00	4129.36
+catalog channel	catalog_pageAAAAAAAACAAAAAAA	0.00	8072.86	-3459.85
+catalog channel	catalog_pageAAAAAAAACABAAAAA	139938.46	0.00	8612.80
+catalog channel	catalog_pageAAAAAAAACADAAAAA	36613.51	0.00	-9705.44
+catalog channel	catalog_pageAAAAAAAACBAAAAAA	0.00	9373.27	-7982.93
+catalog channel	catalog_pageAAAAAAAACBBAAAAA	111531.45	55.38	-15921.86
+catalog channel	catalog_pageAAAAAAAACBDAAAAA	103545.90	0.00	-168.10
+catalog channel	catalog_pageAAAAAAAACCAAAAAA	0.00	3879.26	-1783.40
+catalog channel	catalog_pageAAAAAAAACCBAAAAA	78290.87	0.00	-12841.84
+catalog channel	catalog_pageAAAAAAAACCDAAAAA	37841.79	0.00	-26568.14
+catalog channel	catalog_pageAAAAAAAACDAAAAAA	0.00	923.32	-1166.75
+catalog channel	catalog_pageAAAAAAAACDBAAAAA	113988.86	212.31	-9856.53
+catalog channel	catalog_pageAAAAAAAACDDAAAAA	64516.71	0.00	-27577.04
+catalog channel	catalog_pageAAAAAAAACEAAAAAA	0.00	2955.12	-1700.94
+catalog channel	catalog_pageAAAAAAAACEBAAAAA	91555.04	0.00	-23392.02
+catalog channel	catalog_pageAAAAAAAACEDAAAAA	84993.13	0.00	-24259.30
+catalog channel	catalog_pageAAAAAAAACFAAAAAA	0.00	6507.25	-3250.26
+catalog channel	catalog_pageAAAAAAAACFDAAAAA	73749.97	0.00	1468.23
+catalog channel	catalog_pageAAAAAAAACGAAAAAA	0.00	8814.60	-3790.45
+catalog channel	catalog_pageAAAAAAAACHAAAAAA	157602.51	907.12	-50822.23
+catalog channel	catalog_pageAAAAAAAACHCAAAAA	0.00	59.58	-63.15
+catalog channel	catalog_pageAAAAAAAACIAAAAAA	132342.50	4686.24	-45195.01
+catalog channel	catalog_pageAAAAAAAACIBAAAAA	0.00	6029.56	-3462.80
+catalog channel	catalog_pageAAAAAAAACICAAAAA	0.00	530.41	-535.89
+catalog channel	catalog_pageAAAAAAAACJAAAAAA	193536.42	2487.88	-29573.52
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q6.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q6.sql.out
new file mode 100644
index 0000000000000..86d2d430218f7
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q6.sql.out
@@ -0,0 +1,51 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<state:string,cnt:bigint>
+-- !query output
+VT	10
+MA	11
+NH	11
+NJ	14
+NV	16
+OR	16
+WY	16
+AK	20
+ME	22
+MD	26
+WA	29
+ID	31
+NM	31
+UT	36
+ND	38
+SC	41
+SD	43
+WV	44
+CA	52
+FL	53
+LA	56
+PA	57
+NY	59
+AR	61
+CO	61
+WI	61
+MT	66
+MS	68
+OK	68
+MN	71
+OH	75
+MO	79
+IL	80
+AL	81
+NC	81
+IA	82
+MI	84
+KS	86
+NE	86
+NULL	88
+IN	94
+TN	108
+KY	110
+VA	128
+GA	138
+TX	250
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q64.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q64.sql.out
new file mode 100644
index 0000000000000..a803af3a52c21
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q64.sql.out
@@ -0,0 +1,14 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<product_name:string,store_name:string,store_zip:string,b_street_number:string,b_streen_name:string,b_city:string,b_zip:string,c_street_number:string,c_street_name:string,c_city:string,c_zip:string,syear:int,cnt:bigint,s1:decimal(17,2),s2:decimal(17,2),s3:decimal(17,2),s1:decimal(17,2),s2:decimal(17,2),s3:decimal(17,2),syear:int,cnt:bigint>
+-- !query output
+ablepricallyantiought                             	ation	35709     	996       	NULL	Bridgeport	65817     	752       	Lakeview Lincoln	Friendship	74536     	1999	1	15.78	24.93	0.00	17.09	26.31	0.00	2000	1
+ablepricallyantiought                             	ation	35709     	71        	River River	Friendship	34536     	NULL	NULL	Newport	NULL	1999	1	22.60	38.87	0.00	17.09	26.31	0.00	2000	1
+ablepricallyantiought                             	bar	31904     	128       	East 	Franklin	19101     	990       	2nd 	Friendship	94536     	1999	1	54.76	78.30	0.00	15.80	23.54	0.00	2000	1
+ationbarpri                                       	ation	35709     	362       	Central Ridge	Oakland	69843     	666       	13th Ridge	Shiloh	29275     	1999	1	74.00	95.46	0.00	11.32	20.94	0.00	2000	1
+ationbarpri                                       	ese	31904     	759       	Elm Pine	Belmont	20191     	35        	Madison 	Waterloo	31675     	1999	1	12.92	22.22	0.00	24.15	36.70	0.00	2000	1
+ationbarpri                                       	ese	31904     	759       	Elm Pine	Belmont	20191     	35        	Madison 	Waterloo	31675     	1999	1	12.92	22.22	0.00	83.87	147.61	0.00	2000	1
+ationbarpri                                       	ought	31904     	754       	NULL	NULL	65804     	897       	8th 	Ashland	54244     	1999	1	74.70	90.38	0.00	12.02	12.74	0.00	2000	1
+ationbarpri                                       	ought	31904     	754       	NULL	NULL	65804     	897       	8th 	Ashland	54244     	1999	1	74.70	90.38	0.00	28.08	38.18	0.00	2000	1
+ationbarpri                                       	ought	31904     	754       	NULL	NULL	65804     	897       	8th 	Ashland	54244     	1999	1	74.70	90.38	0.00	56.60	63.39	0.00	2000	1
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q67a.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q67a.sql.out
new file mode 100644
index 0000000000000..2c460414e8623
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q67a.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_category:string,i_class:string,i_brand:string,i_product_name:string,d_year:int,d_qoy:int,d_moy:int,s_store_id:string,sumsales:decimal(38,2),rk:int>
+-- !query output
+NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	681149.47	5
+NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	1474997.55	4
+NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	1577864.42	3
+NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	2536204.81	2
+NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	999668082.59	1
+NULL	NULL	NULL	NULL	2001	NULL	NULL	NULL	681149.47	5
+NULL	NULL	NULL	NULL	2001	1	NULL	NULL	111437.42	32
+NULL	NULL	NULL	NULL	2001	1	1	NULL	36235.73	90
+NULL	NULL	NULL	NULL	2001	1	3	NULL	49252.88	77
+NULL	NULL	NULL	NULL	2001	2	NULL	NULL	78346.18	63
+NULL	NULL	NULL	NULL	2001	2	6	NULL	34840.48	96
+NULL	NULL	NULL	NULL	2001	3	NULL	NULL	141691.87	12
+NULL	NULL	NULL	NULL	2001	3	7	NULL	31988.52	100
+NULL	NULL	NULL	NULL	2001	3	8	NULL	57778.28	69
+NULL	NULL	NULL	NULL	2001	3	9	NULL	51925.07	73
+NULL	NULL	NULL	NULL	2001	4	NULL	NULL	349674.00	7
+NULL	NULL	NULL	NULL	2001	4	10	NULL	89521.66	52
+NULL	NULL	NULL	NULL	2001	4	11	NULL	107989.95	35
+NULL	NULL	NULL	NULL	2001	4	11	AAAAAAAAHAAAAAAA	38673.65	85
+NULL	NULL	NULL	NULL	2001	4	12	NULL	152162.39	11
+NULL	NULL	NULL	NULL	2001	4	12	AAAAAAAACAAAAAAA	36936.71	88
+NULL	NULL	NULL	NULL	2001	4	12	AAAAAAAAKAAAAAAA	35885.68	92
+NULL	NULL	NULL	ationesebareing                                   	NULL	NULL	NULL	NULL	130971.51	13
+NULL	NULL	NULL	ationesebareing                                   	2001	NULL	NULL	NULL	130971.51	13
+NULL	NULL	NULL	ationesebareing                                   	2001	3	NULL	NULL	49334.88	76
+NULL	NULL	NULL	ationesebareing                                   	2001	4	NULL	NULL	36033.41	91
+NULL	NULL	NULL	n stcallyn stn st                                 	NULL	NULL	NULL	NULL	110926.60	33
+NULL	NULL	NULL	n stcallyn stn st                                 	2001	NULL	NULL	NULL	110926.60	33
+NULL	NULL	NULL	n stcallyn stn st                                 	2001	3	NULL	NULL	33118.05	98
+NULL	NULL	NULL	n stcallyn stn st                                 	2001	4	NULL	NULL	41603.21	83
+NULL	NULL	NULL	n stought                                         	NULL	NULL	NULL	NULL	157465.43	9
+NULL	NULL	NULL	n stought                                         	2001	NULL	NULL	NULL	157465.43	9
+NULL	NULL	NULL	n stought                                         	2001	3	NULL	NULL	46371.14	78
+NULL	NULL	NULL	n stought                                         	2001	4	NULL	NULL	65619.67	66
+NULL	NULL	NULL	n stought                                         	2001	4	11	NULL	45528.69	79
+NULL	NULL	NULL	oughtableantiable                                 	NULL	NULL	NULL	NULL	103187.19	40
+NULL	NULL	NULL	oughtableantiable                                 	2001	NULL	NULL	NULL	103187.19	40
+NULL	NULL	NULL	oughtableantiable                                 	2001	4	NULL	NULL	42408.80	82
+NULL	NULL	NULL	oughtablen stationought                           	NULL	NULL	NULL	NULL	86348.22	57
+NULL	NULL	NULL	oughtablen stationought                           	2001	NULL	NULL	NULL	86348.22	57
+NULL	NULL	NULL	oughtablen stationought                           	2001	4	NULL	NULL	35714.86	93
+NULL	NULL	NULL	oughtesebaration                                  	NULL	NULL	NULL	NULL	76325.38	64
+NULL	NULL	NULL	oughtesebaration                                  	2001	NULL	NULL	NULL	76325.38	64
+NULL	NULL	NULL	oughtesebaration                                  	2001	4	NULL	NULL	36610.75	89
+NULL	NULL	NULL	oughteseoughtation                                	NULL	NULL	NULL	NULL	128623.75	19
+NULL	NULL	NULL	oughteseoughtation                                	2001	NULL	NULL	NULL	128623.75	19
+NULL	NULL	NULL	oughteseoughtation                                	2001	3	NULL	NULL	35495.82	94
+NULL	NULL	NULL	oughteseoughtation                                	2001	4	NULL	NULL	57203.17	70
+NULL	NULL	importoamalg #x                                   	NULL	NULL	NULL	NULL	NULL	102866.87	42
+NULL	NULL	importoamalg #x                                   	NULL	NULL	NULL	NULL	NULL	102866.87	42
+NULL	NULL	importoamalg #x                                   	NULL	2001	NULL	NULL	NULL	102866.87	42
+NULL	NULL	importoamalg #x                                   	NULL	2001	4	NULL	NULL	51717.30	74
+NULL	accessories                                       	NULL	NULL	NULL	NULL	NULL	NULL	98994.39	48
+NULL	accessories                                       	NULL	NULL	NULL	NULL	NULL	NULL	98994.39	48
+NULL	accessories                                       	NULL	prin stn stoughtought                             	NULL	NULL	NULL	NULL	98994.39	48
+NULL	accessories                                       	NULL	prin stn stoughtought                             	2001	NULL	NULL	NULL	98994.39	48
+NULL	accessories                                       	NULL	prin stn stoughtought                             	2001	3	NULL	NULL	37246.83	87
+NULL	athletic                                          	NULL	NULL	NULL	NULL	NULL	NULL	122120.16	21
+NULL	athletic                                          	NULL	NULL	NULL	NULL	NULL	NULL	122120.16	21
+NULL	athletic                                          	NULL	NULL	NULL	NULL	NULL	NULL	122120.16	21
+NULL	athletic                                          	NULL	NULL	2001	NULL	NULL	NULL	122120.16	21
+NULL	athletic                                          	NULL	NULL	2001	4	NULL	NULL	63278.49	67
+NULL	athletic                                          	NULL	NULL	2001	4	11	NULL	38216.51	86
+NULL	baseball                                          	NULL	NULL	NULL	NULL	NULL	NULL	105806.96	36
+NULL	baseball                                          	NULL	NULL	NULL	NULL	NULL	NULL	105806.96	36
+NULL	baseball                                          	NULL	NULL	NULL	NULL	NULL	NULL	105806.96	36
+NULL	baseball                                          	NULL	NULL	2001	NULL	NULL	NULL	105806.96	36
+NULL	baseball                                          	NULL	NULL	2001	4	NULL	NULL	58206.33	68
+NULL	country                                           	NULL	NULL	NULL	NULL	NULL	NULL	89164.69	53
+NULL	country                                           	importoscholar #x                                 	NULL	NULL	NULL	NULL	NULL	89164.69	53
+NULL	country                                           	importoscholar #x                                 	NULL	NULL	NULL	NULL	NULL	89164.69	53
+NULL	country                                           	importoscholar #x                                 	NULL	2001	NULL	NULL	NULL	89164.69	53
+NULL	country                                           	importoscholar #x                                 	NULL	2001	4	NULL	NULL	55674.96	71
+NULL	dresses                                           	NULL	NULL	NULL	NULL	NULL	NULL	116126.30	25
+NULL	dresses                                           	NULL	NULL	NULL	NULL	NULL	NULL	217647.20	8
+NULL	dresses                                           	NULL	oughteingantieing                                 	NULL	NULL	NULL	NULL	116126.30	25
+NULL	dresses                                           	NULL	oughteingantieing                                 	2001	NULL	NULL	NULL	116126.30	25
+NULL	dresses                                           	NULL	oughteingantieing                                 	2001	3	NULL	NULL	35421.52	95
+NULL	dresses                                           	NULL	oughteingantieing                                 	2001	4	NULL	NULL	53283.33	72
+NULL	dresses                                           	amalgamalg #x                                     	NULL	NULL	NULL	NULL	NULL	101520.90	45
+NULL	dresses                                           	amalgamalg #x                                     	NULL	NULL	NULL	NULL	NULL	101520.90	45
+NULL	dresses                                           	amalgamalg #x                                     	NULL	2001	NULL	NULL	NULL	101520.90	45
+NULL	dresses                                           	amalgamalg #x                                     	NULL	2001	3	NULL	NULL	32017.28	99
+NULL	dresses                                           	amalgamalg #x                                     	NULL	2001	4	NULL	NULL	34451.25	97
+NULL	infants                                           	NULL	NULL	NULL	NULL	NULL	NULL	128750.23	15
+NULL	infants                                           	importoexporti #x                                 	NULL	NULL	NULL	NULL	NULL	128750.23	15
+NULL	infants                                           	importoexporti #x                                 	antiationeseese                                   	NULL	NULL	NULL	NULL	128750.23	15
+NULL	infants                                           	importoexporti #x                                 	antiationeseese                                   	2001	NULL	NULL	NULL	128750.23	15
+NULL	infants                                           	importoexporti #x                                 	antiationeseese                                   	2001	3	NULL	NULL	40113.56	84
+NULL	infants                                           	importoexporti #x                                 	antiationeseese                                   	2001	4	NULL	NULL	50724.87	75
+NULL	mens                                              	NULL	NULL	NULL	NULL	NULL	NULL	113853.60	28
+NULL	mens                                              	importoedu pack #x                                	NULL	NULL	NULL	NULL	NULL	113853.60	28
+NULL	mens                                              	importoedu pack #x                                	oughtablen steseought                             	NULL	NULL	NULL	NULL	113853.60	28
+NULL	mens                                              	importoedu pack #x                                	oughtablen steseought                             	2001	NULL	NULL	NULL	113853.60	28
+NULL	mens                                              	importoedu pack #x                                	oughtablen steseought                             	2001	3	NULL	NULL	42447.63	81
+NULL	mens                                              	importoedu pack #x                                	oughtablen steseought                             	2001	4	NULL	NULL	43089.74	80
+NULL	shirts                                            	NULL	NULL	NULL	NULL	NULL	NULL	82003.16	59
+NULL	shirts                                            	NULL	NULL	NULL	NULL	NULL	NULL	82003.16	59
+NULL	shirts                                            	NULL	oughtcallyeseantiought                            	NULL	NULL	NULL	NULL	82003.16	59
+NULL	shirts                                            	NULL	oughtcallyeseantiought                            	2001	NULL	NULL	NULL	82003.16	59
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q70a.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q70a.sql.out
new file mode 100644
index 0000000000000..067238b4407b6
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q70a.sql.out
@@ -0,0 +1,8 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<total_sum:decimal(27,2),s_state:string,s_county:string,lochierarchy:int,rank_within_parent:int>
+-- !query output
+-435656177.80	NULL	NULL	2	1
+-435656177.80	TN	NULL	1	1
+-435656177.80	TN	Williamson County	0	1
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q72.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q72.sql.out
new file mode 100644
index 0000000000000..f4050e94d4c53
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q72.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_item_desc:string,w_warehouse_name:string,d_week_seq:int,no_promo:bigint,promo:bigint,total_cnt:bigint>
+-- !query output
+Again perfect sons used to put always. Europea	Significantly	5305	2	2	2
+Ago new arguments accept previously european parents; fo	Selective,	5313	2	2	2
+Difficult, rapid sizes say so; initial banks stress high single sports; prisoners used to think likely firms. Good, current services must take human, precise persons. Signals m	Selective,	5322	2	2	2
+Eventually soft issues will see pp.; possible children pay completely future tasks. Only women will not rehearse more old parts; different movements sponsor most. Please political allowance	Significantly	5273	2	2	2
+Happy, nuclear obligations should not leave little payments. About able relationships impress thus only original a	Significantly	5313	2	2	2
+Healthy, delighted conclusions may offer experienced condi	Significantly	5305	2	2	2
+Hitherto certain kinds evade also by a months. Poor points might make even just selective passengers. Old, general qualities could overcome over; recent variables might s	Just good amou	5280	2	2	2
+Ideas total sadly. International members contribute perhaps lucky cells. Texts would acknowled	Selective,	5280	2	2	2
+Important audiences shall murder.	Selective,	5322	2	2	2
+In order suitable conditions used to eat more on a americans. As open rates shall go sometimes big authorities. Tickets respond for example pregnant, good women. Banks could exploit also full, old cr	Matches produce	5319	2	2	2
+Little, num	Matches produce	5322	2	2	2
+National, l	Operations	5318	2	2	2
+Obvious, present cells may not define appointed, military boys. Answers can get little. Over there comparative days become in a police. Particularly divine prop	Just good amou	5322	2	2	2
+Outside, useful animals find again also british decisions; now small attitudes shall n	Significantly	5322	2	2	2
+Particular, british wa	Operations	5313	2	2	2
+Parts will recruit si	Selective,	5322	2	2	2
+Rational, given goods would stay just equal materials. Very alternative months might not buy voc	Just good amou	5319	2	2	2
+Royal, military notions will not find very very wet acids. Funny actions take western, remaining homes. Great patients will replace simply. Signs can think equivalent reasons. Campaigns 	Matches produce	5313	2	2	2
+Short companies get that is for a days. Early, special hands might keep to the women. Present blocks co-ordinate so sure results. Seasons lend still recent friends. Dead 	Just good amou	5273	2	2	2
+Short companies get that is for a days. Early, special hands might keep to the women. Present blocks co-ordinate so sure results. Seasons lend still recent friends. Dead 	Selective,	5273	2	2	2
+Specialist, identical factors should not answer so main shares. Sales might reduce then free hours. Ethic	Just good amou	5306	2	2	2
+Then sp	Selective,	5308	2	2	2
+NULL	Just good amou	5297	1	1	1
+NULL	Matches produce	5282	1	1	1
+NULL	Operations	5308	1	1	1
+NULL	Selective,	5318	1	1	1
+A bit important 	Significantly	5284	1	1	1
+A bit liable flowers change also writings. Currently soviet ministers come. Hotels telephone before aggressive, economic eyes. Blue changes improve. Overal	Operations	5317	1	1	1
+A bit liable flowers change also writings. Currently soviet ministers come. Hotels telephone before aggressive, economic eyes. Blue changes improve. Overal	Selective,	5317	1	1	1
+A little local letters think over like a children; nevertheless particular powers damage now suddenly absent prote	Significantly	5317	1	1	1
+Able systems merge from a areas. Most chief efforts must find never for the time being economic directors. Activities sit there. Available polic	Selective,	5305	1	1	1
+Able troubles dust into the styles. Independent feet kill wounds. Fundamental months should exploit arms. Massive years read only modern courses; twin forms shall become products. Even h	Matches produce	5309	1	1	1
+Able troubles dust into the styles. Independent feet kill wounds. Fundamental months should exploit arms. Massive years read only modern courses; twin forms shall become products. Even h	Operations	5309	1	1	1
+Able, active jobs might not play upstairs. Electoral crimes could not worry for the solutions. Wholly capitalist effects would not get 	Matches produce	5291	1	1	1
+Able, initial men cannot assume then rational, new references; shares could support physical, internati	Operations	5304	1	1	1
+Able, reasonable standards make forward. Strategic, 	Just good amou	5319	1	1	1
+About military programmes identify all in a thousands; able sentences mean also flats. Branches know secrets; right, increased interactions tour a little. High, lov	Selective,	5299	1	1	1
+About supreme days tell then for a consequences. Ill items force meals; years may not mean quite social structures. Goals	Significantly	5311	1	1	1
+Above, new groups will not like much local bodies. However traditional sessions can walk slowly big, young aspects. Quite close companies ought to take in a rules. Leaders must not like of cou	Just good amou	5314	1	1	1
+Above, new groups will not like much local bodies. However traditional sessions can walk slowly big, young aspects. Quite close companies ought to take in a rules. Leaders must not like of cou	Matches produce	5314	1	1	1
+Above, new groups will not like much local bodies. However traditional sessions can walk slowly big, young aspects. Quite close companies ought to take in a rules. Leaders must not like of cou	Matches produce	5315	1	1	1
+Absolute	Matches produce	5322	1	1	1
+Absolutely angry odds put strongly. Telecommunications help only recent, 	Selective,	5316	1	1	1
+Ac	Significantly	5306	1	1	1
+Accessible, likel	Matches produce	5301	1	1	1
+Accessible, likel	Selective,	5301	1	1	1
+Accessible, old walls profit here. Wars form therefore as effective servants. Secrets could not feel meanwhile regional theories. Perfect, new service	Operations	5273	1	1	1
+Accidents can include below other, marginal plans. Comparable, welsh exceptions argue most as usual physical claims. Certain months may smell far from in a cases. Active seconds used to restore t	Matches produce	5304	1	1	1
+Accidents fly bet	Matches produce	5321	1	1	1
+Accounts rank only high plans. Days sho	Just good amou	5302	1	1	1
+Accurately economic workers play clearly. Deliberately other stands recapture social, cultural prices. Full paths used to make twice alw	Matches produce	5301	1	1	1
+Actions must not compare so economi	Matches produce	5320	1	1	1
+Actions see of course informal phrases. Markedly right men buy honest, additional stations. In order imaginative factors used to move human thanks. Centres shall catch altogether succe	Significantly	5289	1	1	1
+Actual arrangements should introduce never in a unions. Ultimately d	Just good amou	5273	1	1	1
+Actual, possible sides employ here future hands. Powerful intervals ought to respond new, particular marks. Appointed, spiritual accidents sustain but modern, coming findings. Male, national year	Selective,	5304	1	1	1
+Actually subtle subjects mark as tories. Yet possible areas 	Just good amou	5321	1	1	1
+Actually subtle subjects mark as tories. Yet possible areas 	Matches produce	5320	1	1	1
+Acutely possible kilometres cannot trim fully justly original visitors. Owners can transport from the connections. Then controversial girls might tell yet more big kinds. More typical houses g	Just good amou	5272	1	1	1
+Acutely possible kilometres cannot trim fully justly original visitors. Owners can transport from the connections. Then controversial girls might tell yet more big kinds. More typical houses g	Selective,	5318	1	1	1
+Additional companies visit. Grey opportunities may not look numbers. Entire, british models assist also great quarters. Little males show	Just good amou	5284	1	1	1
+Additional companies visit. Grey opportunities may not look numbers. Entire, british models assist also great quarters. Little males show	Selective,	5284	1	1	1
+Additional figures consult relationships. Sole addresses convict right, 	Operations	5322	1	1	1
+Advantages emerge moves; special, expected operations pass etc natural preferences; very posit	Selective,	5313	1	1	1
+Afterwards defensive standards answer just almost informal officers. Now constant rights shall hear courses. Signs go on a budgets	Just good amou	5280	1	1	1
+Afterwards oth	Just good amou	5277	1	1	1
+Afterwards oth	Selective,	5277	1	1	1
+Afterwards rich options go unlikely, welsh elections. Just gentle authors must not provi	Operations	5285	1	1	1
+Afterwards written skills influence; english, level departments like just. Really legal rocks would 	Just good amou	5300	1	1	1
+Again	Just good amou	5319	1	1	1
+Again brief things should remember only in a patients. Deals reply soon other points. Increasingly religious times necessitate farther troops. Both added programmes must come wonderfully solid pupi	Matches produce	5308	1	1	1
+Again brief things should remember only in a patients. Deals reply soon other points. Increasingly religious times necessitate farther troops. Both added programmes must come wonderfully solid pupi	Operations	5308	1	1	1
+Again brief things should remember only in a patients. Deals reply soon other points. Increasingly religious times necessitate farther troops. Both added programmes must come wonderfully solid pupi	Selective,	5308	1	1	1
+Again new teeth heat delicately. Just future officers	Just good amou	5319	1	1	1
+Again new teeth heat delicately. Just future officers	Operations	5294	1	1	1
+Again new teeth heat delicately. Just future officers	Significantly	5294	1	1	1
+Again old police could work in the skills. Points announce agents. Pieces conform slowly to a hea	Significantly	5307	1	1	1
+Again scottish women shall ag	Significantly	5308	1	1	1
+Agencies will not move criminal issues. Years mean very largel	Selective,	5305	1	1	1
+Agents invest often things. French cars ought to get locally distinctive, local powers. More american entries compensate only	Operations	5317	1	1	1
+Ago correct profits must not handle else. Healthy children may not go only ancient words. Later just characters ought to drink about. British parts must watch soon ago other clients. So vital d	Just good amou	5317	1	1	1
+Ago correct profits must not handle else. Healthy children may not go only ancient words. Later just characters ought to drink about. British parts must watch soon ago other clients. So vital d	Operations	5304	1	1	1
+Ago interested doctors meet really fair, cold minds. Fine children understand original procedures. So other services ought to	Operations	5303	1	1	1
+Ago interested doctors meet really fair, cold minds. Fine children understand original procedures. So other services ought to	Significantly	5303	1	1	1
+Ago new studies shall not apply of course small forces. Dead parts used to point on a students. Then other students should pay only	Matches produce	5307	1	1	1
+Ago sexual courts may attract. Important, alone observations expect. New, available ways represent years. Excell	Matches produce	5301	1	1	1
+Ahead national cir	Matches produce	5320	1	1	1
+All attractive ways develop originally lucky sites. New, single sounds might excuse enough senior savings. Other bacteria live across a concerns. Dark minutes s	Significantly	5316	1	1	1
+All capital bacteria make jobs. Again appropriate eyes may not leave others. There fixed ways	Just good amou	5285	1	1	1
+All difficult emotions supervise. Mere 	Just good amou	5322	1	1	1
+All environmental lips cannot catch; social, broad authorities add for no customers. Interes	Just good amou	5318	1	1	1
+All following systems develop home different words. Old minutes will come never independent, real duties. Policies used to distinguish all rats. E	Matches produce	5311	1	1	1
+All full things will not administer quickly difficult women. Ready, new arrangements ma	Just good amou	5284	1	1	1
+All full things will not administer quickly difficult women. Ready, new arrangements ma	Significantly	5284	1	1	1
+All real fam	Operations	5318	1	1	1
+All real fam	Selective,	5318	1	1	1
+All right other details might distrib	Selective,	5277	1	1	1
+Almost busy threats go together recent sides; still tired wines shall not admit on a	Significantly	5300	1	1	1
+Alone arms happen again real documents. Paintings might not invite steps. Internal pairs may increase only rural rooms. Men must not deal here long, heavy patients; merely e	Matches produce	5310	1	1	1
+Already early meetings cannot go animals. As comprehensive evenings w	Just good amou	5320	1	1	1
+Already professional senses encourage broad theories. Nearly eastern eyes would describe correct, complex proposals. Friends change crimin	Selective,	5291	1	1	1
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q74.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q74.sql.out
new file mode 100644
index 0000000000000..ac13c0dabf0c7
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q74.sql.out
@@ -0,0 +1,94 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<customer_id:string,customer_first_name:string,customer_last_name:string>
+-- !query output
+AAAAAAAAHEIFBAAA	NULL	Jones                         
+AAAAAAAAGLDMAAAA	Alex                	Norris                        
+AAAAAAAAOEDIAAAA	Alexander           	Rich                          
+AAAAAAAAKNMEBAAA	Amber               	Gonzalez                      
+AAAAAAAAFGNEAAAA	Andrew              	Silva                         
+AAAAAAAACGLDAAAA	Angelo              	Sloan                         
+AAAAAAAAOCDCAAAA	Armando             	Jackson                       
+AAAAAAAABJEDBAAA	Arthur              	Bryan                         
+AAAAAAAAHGKLAAAA	Arthur              	Christensen                   
+AAAAAAAAPAEEBAAA	Audria              	Mattson                       
+AAAAAAAAFAIEAAAA	Betty               	Gipson                        
+AAAAAAAAOCFLAAAA	Bill                	Freeman                       
+AAAAAAAAILLJAAAA	Billy               	Ortiz                         
+AAAAAAAAGMFHAAAA	Bruce               	Howe                          
+AAAAAAAAKAKPAAAA	Carolann            	Royer                         
+AAAAAAAAAGLPAAAA	Charlene            	Marcus                        
+AAAAAAAABIIHAAAA	Charles             	Quarles                       
+AAAAAAAAIPKJAAAA	Charles             	Jones                         
+AAAAAAAAPIGBBAAA	Charles             	Welch                         
+AAAAAAAACPDFBAAA	Cheryl              	Barry                         
+AAAAAAAACOEHBAAA	Christine           	Gonzalez                      
+AAAAAAAAPNMGAAAA	Christine           	Olds                          
+AAAAAAAAEFCEBAAA	Cornelius           	Martino                       
+AAAAAAAAPEFLAAAA	David               	Martinez                      
+AAAAAAAAIBJDBAAA	Dean                	Velez                         
+AAAAAAAAOOKKAAAA	Deborah             	Early                         
+AAAAAAAALMGGBAAA	Dedra               	Rainey                        
+AAAAAAAANKBBAAAA	Diann               	Saunders                      
+AAAAAAAADKMBAAAA	Donald              	Nelson                        
+AAAAAAAAFGMHBAAA	Donald              	Coleman                       
+AAAAAAAALPHGBAAA	Dorothy             	Heller                        
+AAAAAAAALEAHBAAA	Eddie               	Pena                          
+AAAAAAAAJINGAAAA	Ella                	Moore                         
+AAAAAAAAIANDAAAA	Elva                	Wade                          
+AAAAAAAAEBFHAAAA	Esther              	Merrill                       
+AAAAAAAAAAECBAAA	Frank               	Wenzel                        
+AAAAAAAAJGDLAAAA	Fredrick            	Davis                         
+AAAAAAAAHLEAAAAA	Geneva              	Sims                          
+AAAAAAAAFMOKAAAA	Harry               	Brown                         
+AAAAAAAAEIAHAAAA	Henry               	Desantis                      
+AAAAAAAALMAJAAAA	Ileen               	Linn                          
+AAAAAAAACEMIAAAA	James               	Hernandez                     
+AAAAAAAAIBBFBAAA	James               	Compton                       
+AAAAAAAALNLABAAA	Janie               	Garcia                        
+AAAAAAAABBEAAAAA	Jason               	Gallegos                      
+AAAAAAAAIODCBAAA	Jennifer            	Crane                         
+AAAAAAAAGEKLAAAA	Jerilyn             	Walker                        
+AAAAAAAAMFMKAAAA	John                	Sanders                       
+AAAAAAAAJCNBBAAA	Johnnie             	Cox                           
+AAAAAAAAOJBPAAAA	Jonathan            	Mcbride                       
+AAAAAAAAABGKAAAA	Jonna               	King                          
+AAAAAAAAGGMHAAAA	Julia               	Fisher                        
+AAAAAAAAHEPFBAAA	Kathryn             	Kinney                        
+AAAAAAAAOMOKAAAA	Laurette            	Gary                          
+AAAAAAAABIABAAAA	Letha               	Stone                         
+AAAAAAAAPFKDAAAA	Linda               	Simmons                       
+AAAAAAAAJDEFAAAA	Loretta             	Serrano                       
+AAAAAAAAFOEDAAAA	Lori                	Erwin                         
+AAAAAAAABAAGAAAA	Luis                	James                         
+AAAAAAAAEIPIAAAA	Luke                	Rios                          
+AAAAAAAAGCGIAAAA	Mae                 	Williams                      
+AAAAAAAAFMPPAAAA	Manuel              	Bryant                        
+AAAAAAAAMJFAAAAA	Marcus              	Espinal                       
+AAAAAAAAJADIAAAA	Margaret            	Roberts                       
+AAAAAAAAHLJCAAAA	Marlene             	Grover                        
+AAAAAAAABGMHBAAA	Michael             	Gillespie                     
+AAAAAAAAIPGJAAAA	Michael             	NULL
+AAAAAAAANBECBAAA	Michael             	Lombardi                      
+AAAAAAAAMLOEAAAA	Miguel              	Jackson                       
+AAAAAAAAJHGFAAAA	Pamela              	Gannon                        
+AAAAAAAADFJBBAAA	Patrick             	Jones                         
+AAAAAAAADHNHBAAA	Patrick             	Cooper                        
+AAAAAAAAOPMDAAAA	Peggy               	Smith                         
+AAAAAAAAAFAGAAAA	Robert              	Chang                         
+AAAAAAAAAFBNAAAA	Robert              	Baines                        
+AAAAAAAAKMHPAAAA	Robert              	Jones                         
+AAAAAAAAJMIDAAAA	Sally               	Thurman                       
+AAAAAAAAJBELAAAA	Sean                	Busby                         
+AAAAAAAAJIAHAAAA	Shawna              	Delgado                       
+AAAAAAAAFDIMAAAA	Stephanie           	Cowan                         
+AAAAAAAAPBIGBAAA	Susie               	Zavala                        
+AAAAAAAAGMGEBAAA	Tamika              	Potts                         
+AAAAAAAAMHOLAAAA	Terri               	Cook                          
+AAAAAAAABILCAAAA	Theresa             	Mullins                       
+AAAAAAAADHCBAAAA	Therese             	Perez                         
+AAAAAAAAIBHHBAAA	Vennie              	Loya                          
+AAAAAAAAFHNDAAAA	Virgil              	Mcdonald                      
+AAAAAAAAAHKEAAAA	William             	Stafford                      
+AAAAAAAAHIEIAAAA	William             	Roberts
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q75.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q75.sql.out
new file mode 100644
index 0000000000000..ca273a867c15b
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q75.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<prev_year:int,year:int,i_brand_id:int,i_class_id:int,i_category_id:int,i_manufact_id:int,prev_yr_cnt:bigint,curr_yr_cnt:bigint,sales_cnt_diff:bigint,sales_amt_diff:decimal(19,2)>
+-- !query output
+2001	2002	7009003	16	9	545	6720	3891	-2829	-117008.58
+2001	2002	8005007	5	9	512	6454	4227	-2227	-135735.94
+2001	2002	9001010	1	9	456	5643	3436	-2207	-93617.05
+2001	2002	10013015	3	9	127	6225	4143	-2082	-14567.99
+2001	2002	9015004	10	9	118	7332	5259	-2073	-77291.51
+2001	2002	9004002	4	9	435	6625	4581	-2044	-122670.86
+2001	2002	9014004	3	9	306	5855	3858	-1997	-73620.65
+2001	2002	5002001	5	9	170	5700	3736	-1964	-109018.53
+2001	2002	9014010	1	9	987	6970	5148	-1822	-75086.23
+2001	2002	9015008	15	9	327	6526	4739	-1787	-93540.56
+2001	2002	9016002	16	9	197	6053	4343	-1710	-97667.31
+2001	2002	9015002	15	9	856	10449	8783	-1666	-20233.37
+2001	2002	3001001	1	9	380	6554	4895	-1659	-71747.73
+2001	2002	5002001	14	9	75	6649	5024	-1625	-114258.52
+2001	2002	9004008	4	9	787	6493	4911	-1582	-133842.63
+2001	2002	3001001	4	9	652	6331	4766	-1565	-74417.45
+2001	2002	5004001	11	9	963	6179	4628	-1551	-85648.92
+2001	2002	3004001	7	9	237	6097	4560	-1537	-118933.63
+2001	2002	1001001	8	9	271	5623	4128	-1495	-50180.45
+2001	2002	10004009	4	9	513	6500	5024	-1476	-67288.17
+2001	2002	4004001	10	9	126	5667	4194	-1473	-88247.10
+2001	2002	9007002	7	9	144	5474	4020	-1454	-42908.70
+2001	2002	9016008	16	9	53	6099	4671	-1428	-67851.70
+2001	2002	9007008	7	9	110	5655	4230	-1425	-28768.95
+2001	2002	9002008	2	9	97	5979	4581	-1398	-101148.97
+2001	2002	9008004	8	9	171	5952	4559	-1393	-41622.99
+2001	2002	10010001	10	9	368	5714	4353	-1361	-78871.35
+2001	2002	3003001	8	9	266	5785	4431	-1354	-99666.06
+2001	2002	9011002	11	9	581	6045	4721	-1324	-61169.99
+2001	2002	9012004	12	9	233	5370	4050	-1320	-135207.43
+2001	2002	9008008	8	9	84	5431	4114	-1317	-48625.72
+2001	2002	9010010	7	9	962	6123	4819	-1304	-34943.08
+2001	2002	9009008	9	9	135	5984	4698	-1286	-53184.41
+2001	2002	9014008	14	9	52	5835	4555	-1280	-47886.49
+2001	2002	9005008	5	9	326	5773	4494	-1279	-47219.15
+2001	2002	9011002	11	9	806	5487	4208	-1279	-34285.63
+2001	2002	1003001	3	9	57	5893	4631	-1262	-24156.11
+2001	2002	5003001	3	9	456	5617	4359	-1258	-46563.26
+2001	2002	9015008	15	9	829	4998	3781	-1217	15209.89
+2001	2002	7007001	7	9	211	6277	5071	-1206	-74262.01
+2001	2002	9011008	11	9	541	5654	4449	-1205	-50137.27
+2001	2002	6010001	4	9	91	6373	5169	-1204	-40582.11
+2001	2002	4003001	3	9	175	6106	4907	-1199	-84819.82
+2001	2002	9009008	9	9	83	5880	4689	-1191	-54463.08
+2001	2002	9014010	14	9	201	6106	4919	-1187	-5741.72
+2001	2002	9012010	16	9	173	6576	5390	-1186	-57504.67
+2001	2002	9014002	14	9	966	6066	4915	-1151	-31274.42
+2001	2002	9003004	2	9	29	5232	4084	-1148	-85408.38
+2001	2002	9008002	8	9	586	6212	5079	-1133	-45166.64
+2001	2002	9004010	4	9	87	5551	4428	-1123	-104064.28
+2001	2002	9013002	13	9	18	5087	3977	-1110	34375.04
+2001	2002	9001008	1	9	65	5429	4333	-1096	-112773.38
+2001	2002	8007003	4	9	290	6148	5075	-1073	-48541.94
+2001	2002	9004002	4	9	506	5503	4432	-1071	-47389.52
+2001	2002	9001009	1	9	73	5116	4053	-1063	-12805.29
+2001	2002	9003008	3	9	156	5719	4665	-1054	-27241.44
+2001	2002	9011008	11	9	324	5293	4253	-1040	-78455.59
+2001	2002	9011008	11	9	525	5269	4243	-1026	-8416.01
+2001	2002	9009004	9	9	599	5642	4622	-1020	-7244.59
+2001	2002	9004010	4	9	146	5387	4373	-1014	-96437.31
+2001	2002	9002010	2	9	64	4911	3898	-1013	-70357.40
+2001	2002	2003001	3	9	4	6089	5078	-1011	-14326.99
+2001	2002	9008002	8	9	433	5707	4697	-1010	-59856.93
+2001	2002	9015002	15	9	120	5419	4409	-1010	-25557.35
+2001	2002	9008010	3	9	64	6024	5020	-1004	15466.29
+2001	2002	1003001	5	9	139	6059	5060	-999	-23720.17
+2001	2002	9003002	3	9	281	5640	4651	-989	-62224.16
+2001	2002	9013008	13	9	660	5400	4411	-989	-6728.84
+2001	2002	9013002	13	9	30	4967	3997	-970	-74733.18
+2001	2002	6014001	14	9	977	5339	4369	-970	-34999.44
+2001	2002	8012009	12	9	149	5872	4905	-967	-25425.45
+2001	2002	9004009	4	9	618	5522	4576	-946	-59306.75
+2001	2002	7006009	8	9	116	5471	4528	-943	-65664.87
+2001	2002	9014010	15	9	151	5633	4701	-932	7429.42
+2001	2002	9003002	3	9	877	6303	5379	-924	-45324.79
+2001	2002	9006008	6	9	162	5169	4251	-918	-69642.81
+2001	2002	9008010	8	9	503	5526	4608	-918	-49021.02
+2001	2002	9005002	5	9	530	4962	4049	-913	-51406.91
+2001	2002	9010002	10	9	236	5107	4195	-912	-68196.83
+2001	2002	9015004	15	9	134	4996	4092	-904	-20839.46
+2001	2002	9009009	6	9	181	5283	4380	-903	-50577.89
+2001	2002	9009010	16	9	590	4684	3787	-897	-42032.78
+2001	2002	9006008	6	9	221	5238	4347	-891	-110703.28
+2001	2002	10009017	9	9	640	5562	4682	-880	-91021.88
+2001	2002	7013003	13	9	313	5161	4284	-877	-55538.42
+2001	2002	3004001	12	9	570	5053	4181	-872	-81469.82
+2001	2002	9009004	9	9	587	5131	4262	-869	-31202.41
+2001	2002	9010008	10	9	86	5074	4211	-863	-31897.54
+2001	2002	9006002	6	9	230	5616	4756	-860	-39142.42
+2001	2002	9001008	1	9	533	6196	5337	-859	-70174.46
+2001	2002	9004010	4	9	102	6146	5291	-855	-66567.87
+2001	2002	9002004	2	9	179	5203	4351	-852	-26624.64
+2001	2002	9016008	16	9	69	5922	5072	-850	-31016.10
+2001	2002	9008010	8	9	81	5520	4672	-848	-44217.60
+2001	2002	9001002	1	9	927	5858	5013	-845	8179.45
+2001	2002	1001001	1	9	116	5773	4930	-843	700.02
+2001	2002	9012008	12	9	274	6154	5316	-838	-85490.33
+2001	2002	6008007	8	9	22	5439	4605	-834	-25617.83
+2001	2002	9016008	16	9	601	6463	5633	-830	22715.94
+2001	2002	9001008	1	9	170	5970	5142	-828	-2265.18
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q77a.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q77a.sql.out
new file mode 100644
index 0000000000000..1f822ce67889f
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q77a.sql.out
@@ -0,0 +1,49 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<channel:string,id:int,sales:decimal(37,2),returns:decimal(37,2),profit:decimal(38,2)>
+-- !query output
+NULL	NULL	239062306.14	9940693.53	-67351905.74
+catalog channel	NULL	81456313.49	6721169.80	-11963308.94
+catalog channel	NULL	120443.39	1680292.45	-994006.90
+catalog channel	1	25511213.21	1680292.45	-4013845.35
+catalog channel	2	28320909.41	1680292.45	-3815679.20
+catalog channel	4	27503747.48	1680292.45	-3139777.49
+store channel	NULL	114950675.73	2650053.56	-51250807.34
+store channel	1	19900057.08	430318.96	-9021290.46
+store channel	2	19357997.15	420826.73	-8260880.77
+store channel	4	18625533.75	413145.75	-8693775.33
+store channel	7	19392254.22	523129.28	-8781756.70
+store channel	8	18443448.06	453191.53	-8177970.58
+store channel	10	19231385.47	409441.31	-8315133.50
+web channel	NULL	42655316.92	569470.17	-4137789.46
+web channel	1	1484216.46	22578.30	-145384.36
+web channel	2	1434288.77	8368.29	-74179.21
+web channel	4	1366508.91	23624.40	-200293.36
+web channel	7	1357762.92	40137.83	-123205.40
+web channel	8	1488902.03	21963.58	-135533.22
+web channel	10	1568120.92	11055.99	-148192.16
+web channel	13	1322895.39	16655.94	-110269.99
+web channel	14	1367483.46	8838.41	-184084.28
+web channel	16	1393205.82	26800.78	-87656.13
+web channel	19	1364048.12	13570.75	-121815.95
+web channel	20	1313170.80	14044.48	-194480.66
+web channel	22	1372505.20	19743.54	-82372.11
+web channel	25	1483839.68	17464.70	12637.58
+web channel	26	1351984.72	16290.97	-210017.25
+web channel	28	1515110.49	34693.25	-87241.61
+web channel	31	1472020.01	6383.43	-163741.34
+web channel	32	1371605.44	23901.58	-201414.74
+web channel	34	1550210.89	15290.60	-189498.78
+web channel	37	1292707.71	17894.26	-179114.08
+web channel	38	1271096.72	16767.70	-211804.95
+web channel	40	1529058.38	21329.09	-126745.86
+web channel	43	1489558.57	24760.75	-100645.79
+web channel	44	1506795.25	9375.29	-161279.12
+web channel	46	1311086.33	13347.60	-144246.40
+web channel	49	1492170.04	21268.41	-97796.30
+web channel	50	1486817.09	14070.53	-109583.01
+web channel	52	1564493.01	26771.44	-74766.72
+web channel	55	1330931.73	18908.13	-235881.43
+web channel	56	1448998.86	26396.32	-131860.48
+web channel	58	1353723.20	17173.83	-117322.35
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q78.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q78.sql.out
new file mode 100644
index 0000000000000..df28fcb255ae8
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q78.sql.out
@@ -0,0 +1,6 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<ratio:double,store_qty:bigint,store_wholesale_cost:decimal(17,2),store_sales_price:decimal(17,2),other_chan_qty:bigint,other_chan_wholesale_cost:decimal(18,2),other_chan_sales_price:decimal(18,2)>
+-- !query output
+
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q80a.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q80a.sql.out
new file mode 100644
index 0000000000000..2aeebd88f1d79
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q80a.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<channel:string,id:string,sales:decimal(37,2),returns:decimal(38,2),profit:decimal(38,2)>
+-- !query output
+NULL	NULL	12394002.28	579360.90	-3565544.29
+catalog channel	NULL	4336403.99	190045.19	-496576.74
+catalog channel	catalog_pageAAAAAAAAAABAAAAA	4438.43	383.01	-4797.32
+catalog channel	catalog_pageAAAAAAAAAADAAAAA	1459.86	0.00	-538.24
+catalog channel	catalog_pageAAAAAAAAABBAAAAA	6092.40	0.00	-37.09
+catalog channel	catalog_pageAAAAAAAAABDAAAAA	878.66	0.00	401.04
+catalog channel	catalog_pageAAAAAAAAACBAAAAA	15564.33	0.00	-743.39
+catalog channel	catalog_pageAAAAAAAAACDAAAAA	19434.58	0.00	-1241.86
+catalog channel	catalog_pageAAAAAAAAADBAAAAA	3357.08	0.00	-223.80
+catalog channel	catalog_pageAAAAAAAAADDAAAAA	17773.31	0.00	10583.67
+catalog channel	catalog_pageAAAAAAAAAEBAAAAA	11505.75	0.00	1975.14
+catalog channel	catalog_pageAAAAAAAAAEDAAAAA	14132.65	0.00	-3283.42
+catalog channel	catalog_pageAAAAAAAAAFDAAAAA	8636.84	216.08	-1736.94
+catalog channel	catalog_pageAAAAAAAAAGDAAAAA	17045.61	145.32	1162.68
+catalog channel	catalog_pageAAAAAAAAAHAAAAAA	19082.44	2556.07	-16790.11
+catalog channel	catalog_pageAAAAAAAAAIAAAAAA	7785.09	33.84	-3860.11
+catalog channel	catalog_pageAAAAAAAAAJAAAAAA	33875.64	0.00	6085.44
+catalog channel	catalog_pageAAAAAAAAAKAAAAAA	4069.90	3395.00	-57.06
+catalog channel	catalog_pageAAAAAAAAALAAAAAA	10778.98	0.00	-6320.25
+catalog channel	catalog_pageAAAAAAAAAMAAAAAA	23566.51	0.00	1270.50
+catalog channel	catalog_pageAAAAAAAAANAAAAAA	11997.42	0.00	-7811.93
+catalog channel	catalog_pageAAAAAAAAANCAAAAA	1574.03	0.00	170.61
+catalog channel	catalog_pageAAAAAAAAAOAAAAAA	12768.66	0.00	-1180.46
+catalog channel	catalog_pageAAAAAAAAAPAAAAAA	15591.17	0.00	617.97
+catalog channel	catalog_pageAAAAAAAAAPCAAAAA	0.00	0.00	-2448.27
+catalog channel	catalog_pageAAAAAAAABABAAAAA	22630.47	0.00	4803.56
+catalog channel	catalog_pageAAAAAAAABADAAAAA	4689.30	0.00	1920.53
+catalog channel	catalog_pageAAAAAAAABBBAAAAA	8799.41	1594.88	-1551.23
+catalog channel	catalog_pageAAAAAAAABBDAAAAA	2900.27	0.00	-1813.33
+catalog channel	catalog_pageAAAAAAAABCBAAAAA	19972.81	189.75	5730.83
+catalog channel	catalog_pageAAAAAAAABCDAAAAA	24932.09	2122.08	3205.82
+catalog channel	catalog_pageAAAAAAAABDBAAAAA	5321.51	0.00	143.34
+catalog channel	catalog_pageAAAAAAAABDDAAAAA	2502.32	0.00	-1101.24
+catalog channel	catalog_pageAAAAAAAABHAAAAAA	3734.33	0.00	-3819.22
+catalog channel	catalog_pageAAAAAAAABIAAAAAA	37573.12	586.70	5857.09
+catalog channel	catalog_pageAAAAAAAABJAAAAAA	12330.42	0.00	-1486.17
+catalog channel	catalog_pageAAAAAAAABJCAAAAA	6097.67	0.00	3736.59
+catalog channel	catalog_pageAAAAAAAABKAAAAAA	18001.80	41.67	-7542.13
+catalog channel	catalog_pageAAAAAAAABLAAAAAA	21210.22	0.00	2386.64
+catalog channel	catalog_pageAAAAAAAABMAAAAAA	27450.22	35.95	-6130.18
+catalog channel	catalog_pageAAAAAAAABNAAAAAA	24544.71	0.00	-19634.16
+catalog channel	catalog_pageAAAAAAAABNCAAAAA	1254.78	23.70	-2121.59
+catalog channel	catalog_pageAAAAAAAABOAAAAAA	7397.94	0.00	-2259.68
+catalog channel	catalog_pageAAAAAAAABPAAAAAA	23154.42	0.00	744.22
+catalog channel	catalog_pageAAAAAAAACABAAAAA	4702.71	0.00	-13312.05
+catalog channel	catalog_pageAAAAAAAACADAAAAA	1010.39	0.00	-2765.23
+catalog channel	catalog_pageAAAAAAAACBBAAAAA	74.70	0.00	-1114.49
+catalog channel	catalog_pageAAAAAAAACBDAAAAA	9217.35	0.00	3327.65
+catalog channel	catalog_pageAAAAAAAACCBAAAAA	11097.18	0.00	3019.32
+catalog channel	catalog_pageAAAAAAAACCDAAAAA	771.30	0.00	-365.85
+catalog channel	catalog_pageAAAAAAAACDBAAAAA	3370.78	61.32	-1489.09
+catalog channel	catalog_pageAAAAAAAACEBAAAAA	3646.15	0.00	1089.86
+catalog channel	catalog_pageAAAAAAAACEDAAAAA	2402.24	0.00	-1262.38
+catalog channel	catalog_pageAAAAAAAACFDAAAAA	2351.04	581.76	-3631.49
+catalog channel	catalog_pageAAAAAAAACHAAAAAA	14363.16	72.40	-6022.88
+catalog channel	catalog_pageAAAAAAAACIAAAAAA	17233.80	290.61	-1149.33
+catalog channel	catalog_pageAAAAAAAACJAAAAAA	17334.55	0.00	-5325.06
+catalog channel	catalog_pageAAAAAAAACKAAAAAA	33391.87	0.00	5427.63
+catalog channel	catalog_pageAAAAAAAACLAAAAAA	29852.16	55.65	-4206.17
+catalog channel	catalog_pageAAAAAAAACMAAAAAA	13596.19	0.00	1289.63
+catalog channel	catalog_pageAAAAAAAACNAAAAAA	12652.74	2776.86	-10512.27
+catalog channel	catalog_pageAAAAAAAACOAAAAAA	15417.46	0.00	8.02
+catalog channel	catalog_pageAAAAAAAACPAAAAAA	6001.13	0.00	-948.15
+catalog channel	catalog_pageAAAAAAAACPCAAAAA	6110.72	0.00	-34.56
+catalog channel	catalog_pageAAAAAAAADABAAAAA	0.00	0.00	-674.57
+catalog channel	catalog_pageAAAAAAAADADAAAAA	13815.26	0.00	4963.37
+catalog channel	catalog_pageAAAAAAAADBBAAAAA	4417.61	0.00	1955.36
+catalog channel	catalog_pageAAAAAAAADBDAAAAA	9645.90	0.00	2941.05
+catalog channel	catalog_pageAAAAAAAADCBAAAAA	4124.08	0.00	-7880.15
+catalog channel	catalog_pageAAAAAAAADCDAAAAA	20198.02	0.00	-3874.19
+catalog channel	catalog_pageAAAAAAAADDBAAAAA	7697.49	875.52	3573.72
+catalog channel	catalog_pageAAAAAAAADDDAAAAA	81.18	0.00	-40.95
+catalog channel	catalog_pageAAAAAAAADEBAAAAA	7221.62	0.00	-2080.43
+catalog channel	catalog_pageAAAAAAAADEDAAAAA	31633.70	0.00	1203.47
+catalog channel	catalog_pageAAAAAAAADFDAAAAA	4486.06	0.00	-194.53
+catalog channel	catalog_pageAAAAAAAADHAAAAAA	29478.34	0.00	2568.57
+catalog channel	catalog_pageAAAAAAAADIAAAAAA	3541.34	0.00	-5860.49
+catalog channel	catalog_pageAAAAAAAADJAAAAAA	27011.83	995.61	-13747.67
+catalog channel	catalog_pageAAAAAAAADKAAAAAA	22422.65	0.00	10022.26
+catalog channel	catalog_pageAAAAAAAADLAAAAAA	25580.22	1878.40	3693.93
+catalog channel	catalog_pageAAAAAAAADMAAAAAA	14964.93	0.00	-3775.51
+catalog channel	catalog_pageAAAAAAAADNAAAAAA	7954.26	7.41	-11362.44
+catalog channel	catalog_pageAAAAAAAADOAAAAAA	20282.62	389.76	20.54
+catalog channel	catalog_pageAAAAAAAADOCAAAAA	2056.15	1394.00	-96.79
+catalog channel	catalog_pageAAAAAAAADPAAAAAA	2325.25	0.00	-3003.94
+catalog channel	catalog_pageAAAAAAAADPCAAAAA	5.79	0.00	0.11
+catalog channel	catalog_pageAAAAAAAAEABAAAAA	5870.11	0.00	-2224.89
+catalog channel	catalog_pageAAAAAAAAEADAAAAA	15887.40	294.38	1553.93
+catalog channel	catalog_pageAAAAAAAAEBBAAAAA	750.60	48.60	-1121.83
+catalog channel	catalog_pageAAAAAAAAEBDAAAAA	2076.12	1787.04	-7886.05
+catalog channel	catalog_pageAAAAAAAAECBAAAAA	18310.64	2076.12	1185.01
+catalog channel	catalog_pageAAAAAAAAECDAAAAA	6384.54	0.00	-2290.17
+catalog channel	catalog_pageAAAAAAAAEDBAAAAA	6470.98	0.00	3092.44
+catalog channel	catalog_pageAAAAAAAAEDDAAAAA	7165.64	0.00	-169.94
+catalog channel	catalog_pageAAAAAAAAEEBAAAAA	8572.26	697.82	-1507.83
+catalog channel	catalog_pageAAAAAAAAEEDAAAAA	9693.59	811.58	-7987.68
+catalog channel	catalog_pageAAAAAAAAEFDAAAAA	5850.24	0.00	2898.90
+catalog channel	catalog_pageAAAAAAAAEHAAAAAA	18074.80	2563.92	-2315.60
+catalog channel	catalog_pageAAAAAAAAEIAAAAAA	1842.65	0.00	-4113.10
+catalog channel	catalog_pageAAAAAAAAEJAAAAAA	31976.85	194.76	4337.39
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q86a.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q86a.sql.out
new file mode 100644
index 0000000000000..76865be96e9de
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q86a.sql.out
@@ -0,0 +1,105 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<total_sum:decimal(27,2),i_category:string,i_class:string,lochierarchy:int,rank_within_parent:int>
+-- !query output
+329332948.59	NULL	NULL	2	1
+35765359.17	Books                                             	NULL	1	1
+34301963.73	Men                                               	NULL	1	2
+34299263.29	Home                                              	NULL	1	3
+34185638.04	Children                                          	NULL	1	4
+33632557.65	Electronics                                       	NULL	1	5
+32679955.12	Music                                             	NULL	1	6
+31276464.49	Women                                             	NULL	1	7
+30973438.42	Shoes                                             	NULL	1	8
+30893463.64	Jewelry                                           	NULL	1	9
+30492050.58	Sports                                            	NULL	1	10
+832794.46	NULL	NULL	1	11
+487784.65	NULL	NULL	0	1
+91077.43	NULL	dresses                                           	0	2
+48174.49	NULL	athletic                                          	0	3
+45303.17	NULL	accessories                                       	0	4
+38544.63	NULL	country                                           	0	5
+36691.45	NULL	infants                                           	0	6
+33102.66	NULL	shirts                                            	0	7
+31974.19	NULL	baseball                                          	0	8
+20141.79	NULL	mens                                              	0	9
+2788922.44	Books                                             	history                                           	0	1
+2627303.83	Books                                             	fiction                                           	0	2
+2559899.95	Books                                             	business                                          	0	3
+2523972.68	Books                                             	sports                                            	0	4
+2478882.85	Books                                             	parenting                                         	0	5
+2454802.70	Books                                             	home repair                                       	0	6
+2311733.05	Books                                             	science                                           	0	7
+2267995.48	Books                                             	reference                                         	0	8
+2199284.41	Books                                             	travel                                            	0	9
+2143489.54	Books                                             	self-help                                         	0	10
+2136284.47	Books                                             	romance                                           	0	11
+2047067.66	Books                                             	computers                                         	0	12
+2024086.51	Books                                             	mystery                                           	0	13
+1841858.53	Books                                             	cooking                                           	0	14
+1718405.56	Books                                             	arts                                              	0	15
+1641369.51	Books                                             	entertainments                                    	0	16
+9726340.87	Children                                          	school-uniforms                                   	0	1
+8423026.60	Children                                          	infants                                           	0	2
+8110705.74	Children                                          	toddlers                                          	0	3
+7857828.46	Children                                          	newborn                                           	0	4
+67736.37	Children                                          	NULL	0	5
+2545288.50	Electronics                                       	televisions                                       	0	1
+2432288.94	Electronics                                       	memory                                            	0	2
+2393855.24	Electronics                                       	stereo                                            	0	3
+2322478.19	Electronics                                       	karoke                                            	0	4
+2312404.32	Electronics                                       	audio                                             	0	5
+2297793.09	Electronics                                       	automotive                                        	0	6
+2175340.77	Electronics                                       	camcorders                                        	0	7
+2155848.29	Electronics                                       	dvd/vcr players                                   	0	8
+2151409.30	Electronics                                       	portable                                          	0	9
+2126590.68	Electronics                                       	monitors                                          	0	10
+2049010.73	Electronics                                       	personal                                          	0	11
+1975162.86	Electronics                                       	disk drives                                       	0	12
+1824298.91	Electronics                                       	wireless                                          	0	13
+1747334.28	Electronics                                       	musical                                           	0	14
+1647065.60	Electronics                                       	scanners                                          	0	15
+1476387.95	Electronics                                       	cameras                                           	0	16
+2878895.80	Home                                              	rugs                                              	0	1
+2433373.20	Home                                              	wallpaper                                         	0	2
+2394255.96	Home                                              	bedding                                           	0	3
+2360342.61	Home                                              	mattresses                                        	0	4
+2349809.40	Home                                              	paint                                             	0	5
+2342593.19	Home                                              	tables                                            	0	6
+2255327.66	Home                                              	accent                                            	0	7
+2238644.85	Home                                              	lighting                                          	0	8
+2195356.00	Home                                              	glassware                                         	0	9
+2001058.99	Home                                              	curtains/drapes                                   	0	10
+1995854.06	Home                                              	bathroom                                          	0	11
+1932033.64	Home                                              	decor                                             	0	12
+1789688.36	Home                                              	flatware                                          	0	13
+1767061.55	Home                                              	blinds/shades                                     	0	14
+1730423.41	Home                                              	kids                                              	0	15
+1585801.82	Home                                              	furniture                                         	0	16
+48742.79	Home                                              	NULL	0	17
+2309049.65	Jewelry                                           	loose stones                                      	0	1
+2291438.75	Jewelry                                           	bracelets                                         	0	2
+2204443.86	Jewelry                                           	custom                                            	0	3
+1975110.14	Jewelry                                           	mens watch                                        	0	4
+1964522.90	Jewelry                                           	diamonds                                          	0	5
+1950628.31	Jewelry                                           	estate                                            	0	6
+1950233.65	Jewelry                                           	womens watch                                      	0	7
+1948966.11	Jewelry                                           	birdal                                            	0	8
+1934844.79	Jewelry                                           	rings                                             	0	9
+1920939.20	Jewelry                                           	gold                                              	0	10
+1912539.77	Jewelry                                           	semi-precious                                     	0	11
+1816107.81	Jewelry                                           	consignment                                       	0	12
+1793972.84	Jewelry                                           	earings                                           	0	13
+1783332.27	Jewelry                                           	pendants                                          	0	14
+1720324.59	Jewelry                                           	costume                                           	0	15
+1417009.00	Jewelry                                           	jewelry boxes                                     	0	16
+8970878.34	Men                                               	sports-apparel                                    	0	1
+8843683.02	Men                                               	pants                                             	0	2
+8340416.89	Men                                               	accessories                                       	0	3
+8054511.00	Men                                               	shirts                                            	0	4
+92474.48	Men                                               	NULL	0	5
+8987804.92	Music                                             	rock                                              	0	1
+8141376.53	Music                                             	country                                           	0	2
+7793743.27	Music                                             	classical                                         	0	3
+7727726.97	Music                                             	pop                                               	0	4
diff --git a/sql/core/src/test/resources/tpcds-query-results/v2_7/q98.sql.out b/sql/core/src/test/resources/tpcds-query-results/v2_7/q98.sql.out
new file mode 100644
index 0000000000000..efd1a7af3b008
--- /dev/null
+++ b/sql/core/src/test/resources/tpcds-query-results/v2_7/q98.sql.out
@@ -0,0 +1,2534 @@
+-- Automatically generated by TPCDSQueryTestSuite
+
+-- !query schema
+struct<i_item_id:string,i_item_desc:string,i_category:string,i_class:string,i_current_price:decimal(7,2),itemrevenue:decimal(17,2),revenueratio:decimal(38,17)>
+-- !query output
+AAAAAAAAMJJBAAAA	NULL	Books                                             	NULL	NULL	557.55	2.41577919115267179
+AAAAAAAAMLGDAAAA	NULL	Books                                             	NULL	6.35	361.02	1.56424464817493959
+AAAAAAAAAELBAAAA	Precisely elderly bodies	Books                                             	arts                                              	1.40	4303.08	1.75015577438987686
+AAAAAAAAAHMAAAAA	Abilities could affect cruel parts. Predominantly other events telephone strong signs. Accurate mate	Books                                             	arts                                              	25.69	9236.96	3.75687156218529913
+AAAAAAAAAIDDAAAA	Germa	Books                                             	arts                                              	5.82	3191.92	1.29822295179047002
+AAAAAAAAALNCAAAA	Great, contemporary workers would not remove of course cultural values. Then due children might see positive seconds. Significant problems w	Books                                             	arts                                              	0.55	2096.07	0.85251703756969175
+AAAAAAAAANKAAAAA	Small objects stop etc mediterranean patterns; liberal, free initiatives would not leave less clear british attitudes; good, blue relationships find softly very	Books                                             	arts                                              	58.41	5760.88	2.34307458786895754
+AAAAAAAABGDAAAAA	Newly national rights head curiously all electrical cells. Chinese, long values might not pull bad lines. High fun clothes ough	Books                                             	arts                                              	3.28	571.10	0.23227873122369528
+AAAAAAAABKLDAAAA	Forward psychological plants establish closely yet eastern changes. Likewise necessary techniques might drop. Pleasant operations like lonely things; dogs let regions. Forces might not result clearl	Books                                             	arts                                              	2.43	3457.82	1.40637023708618106
+AAAAAAAACMFDAAAA	Early, short v	Books                                             	arts                                              	75.57	850.08	0.34574593563060564
+AAAAAAAADLLDAAAA	Black, following services justify by a investors; dirty, different charts will fly however prizes. Temporary, l	Books                                             	arts                                              	5.56	4798.20	1.95153179505784395
+AAAAAAAAEIPCAAAA	Scientific, difficult polls would not achieve. Countries reach of course. Bad, new churches realize most english	Books                                             	arts                                              	3.98	17.50	0.00711762878027433
+AAAAAAAAFCIBAAAA	United, important objectives put similarly large, previous phenomena; old, present days receive. Happy detectives assi	Books                                             	arts                                              	1.26	11646.43	4.73685516316858938
+AAAAAAAAFFIBAAAA	Naturally new years put serious, negative vehicles. Fin	Books                                             	arts                                              	3.34	20302.86	8.25761260902173683
+AAAAAAAAFJGCAAAA	Ago correct profits must not handle else. Healthy children may not go only ancient words. Later just characters ought to drink about. British parts must watch soon ago other clients. So vital d	Books                                             	arts                                              	4.03	9410.73	3.82754758236520025
+AAAAAAAAFLNCAAAA	Much new waters 	Books                                             	arts                                              	1.85	2864.61	1.16509889030066491
+AAAAAAAAGLEBAAAA	Tall, following actions keep widely willing, secondary groups. Heads could afford however; agricultural, square pri	Books                                             	arts                                              	9.99	1903.68	0.77426786036757875
+AAAAAAAAGMFAAAAA	Anonymous, useful women provoke slightly present persons. Ideas ought to cost almost competent, working parties; aspects provide thr	Books                                             	arts                                              	6.73	7841.73	3.18940132200803357
+AAAAAAAAHHEBAAAA	Powerful walls will find; there scottish decades must not	Books                                             	arts                                              	4.16	5934.47	2.41367739815283298
+AAAAAAAAHMCEAAAA	Too executive doors progress mainly seemingly possible parts; hundreds stay virtually simple workers. Sola	Books                                             	arts                                              	34.32	10139.18	4.12382396436467639
+AAAAAAAAIBOCAAAA	Careful privileges ought to live rather to a boards. Possible, broad p	Books                                             	arts                                              	3.93	10008.95	4.07085660459009779
+AAAAAAAAICMBAAAA	Aside legitimate decisions may not stand probably sexual g	Books                                             	arts                                              	3.88	874.84	0.35581636355058234
+AAAAAAAAIFPBAAAA	Specially interesting crews continue current, foreign directions; only social men would not call at least political children; circumstances could not understand now in a assessme	Books                                             	arts                                              	2.13	15380.51	6.25558632178840388
+AAAAAAAAIHNAAAAA	Unlikely states take later in general extra inf	Books                                             	arts                                              	0.32	6478.12	2.63479162023261224
+AAAAAAAAINFDAAAA	Sometimes careful things state probably so	Books                                             	arts                                              	5.08	17118.92	6.96263529595507190
+AAAAAAAAJGHDAAAA	Circumstances would not use. Principles seem writers. Times go from a hands. Members find grounds. Central, only teachers pursue properly into a p	Books                                             	arts                                              	5.95	7863.28	3.19816617344888566
+AAAAAAAAJLHBAAAA	Inches may lose from a problems. Firm, other corporations shall protect ashamed, important practices. Materials shall not make then by a police. Weeks used	Books                                             	arts                                              	0.84	6519.57	2.65165023240074772
+AAAAAAAAKFGBAAAA	Systems cannot await regions. Home appropr	Books                                             	arts                                              	7.30	889.75	0.36188058327137607
+AAAAAAAAKHJDAAAA	Relevant lips take so sure, manufacturing 	Books                                             	arts                                              	8.80	932.33	0.37919879089789497
+AAAAAAAAKHLBAAAA	Extra, primitive weeks look obviou	Books                                             	arts                                              	1.18	10006.38	4.06981132996350893
+AAAAAAAALCFBAAAA	More than key reasons should remain. Words used to offer slowly british	Books                                             	arts                                              	0.28	1075.86	0.43757554854548205
+AAAAAAAALGEEAAAA	Children may turn also above, historical aspects. Surveys migh	Books                                             	arts                                              	7.22	3864.60	1.57181646767132336
+AAAAAAAALOKCAAAA	Trustees know operations. Now past issues cut today german governments. British lines go critical, individual structures. Tonight adequate problems should no	Books                                             	arts                                              	4.05	3061.36	1.24512137387317768
+AAAAAAAAMACDAAAA	Useful observers start often white colleagues; simple pro	Books                                             	arts                                              	3.47	724.40	0.29462915933889837
+AAAAAAAAMNPAAAAA	Members should say earnings. Detailed departments would not move just at the hopes. Figures can take. Actually open houses want. Good teachers combine the	Books                                             	arts                                              	3.09	7805.88	3.17482035104958588
+AAAAAAAAMPFCAAAA	Major, senior words afford economic libraries; successful seconds need outside. Clinical, new ideas put now red c	Books                                             	arts                                              	5.87	616.62	0.25079270048530027
+AAAAAAAANABCAAAA	Likely states feel astonishingly working roads. Parents put so somewhere able policies. Others may rely shortly instead interesting bodies; bri	Books                                             	arts                                              	7.50	8829.29	3.59106334933647431
+AAAAAAAANMECAAAA	Floors could not go only for a years. Special reasons shape consequently black, concerned instances. Mutual depths encourage both simple teachers. Cards favour massive 	Books                                             	arts                                              	1.83	6344.62	2.58049428068023382
+AAAAAAAAOAHCAAAA	Accurate years want then other organisations. Simple lines mean as well so red results. Orthodox, central scales will not in	Books                                             	arts                                              	7.69	3640.38	1.48062134052200283
+AAAAAAAAODBEAAAA	Certain customers think exactly already necessary factories. Awkward doubts shall not forget fine	Books                                             	arts                                              	0.30	12867.38	5.23344195512721440
+AAAAAAAAOEIDAAAA	Weak effects set far in the effects. Positive, true classes sell frankly ever open studies. Unique problems must mean as yet new genes. Essential businesses agree deep current stages. Not	Books                                             	arts                                              	4.40	4471.87	1.81880632077973420
+AAAAAAAAOKEDAAAA	Visitors could not allow glad wages. Communist, real figures used to apply factors. Aggressive, optimistic days must mean about trees. Detailed courts consider really large pro	Books                                             	arts                                              	9.08	1737.00	0.70647549664780021
+AAAAAAAAOODBAAAA	Deep, big areas take for a facilities. Words could replace certainly cases; lights test. Nevertheless practical arts cross. Fa	Books                                             	arts                                              	7.37	8016.08	3.26031324074179520
+AAAAAAAAPEBCAAAA	Similar situations come separate programmes. National, large others could not ask opportunities. Severe, large findings accept; twins go more. Tiny rights may see specifi	Books                                             	arts                                              	1.27	2413.11	0.98146406776958731
+AAAAAAAABBLDAAAA	Natural plans might not like n	Books                                             	business                                          	4.29	8813.76	2.98246237973343018
+AAAAAAAABINDAAAA	Years shall want free objects. Old residents use absolutely so residential steps. Letters will share variables. Sure fres	Books                                             	business                                          	40.76	1861.77	0.62999888636816844
+AAAAAAAACAADAAAA	Whole, important problems make. Indeed industrial members go skills. Soft	Books                                             	business                                          	3.22	6877.01	2.32709123121693768
+AAAAAAAACCMBAAAA	Cheap depths calm as in a traditi	Books                                             	business                                          	7.92	2554.82	0.86451804190159048
+AAAAAAAACDDCAAAA	Simple, great shops glance from a years. Lessons deepen here previous clients. Increased, silent flights open more great rocks. Brill	Books                                             	business                                          	8.92	8014.35	2.71195237594586375
+AAAAAAAACGCAAAAA	Just sudden ideas ought to serve full sources; uncertain, open qualifications shout questions; chronic, informal	Books                                             	business                                          	4.62	4172.49	1.41191664565564981
+AAAAAAAACGIDAAAA	Groups must not put new, civil moves. Correct men laugh slightly total novels. Relatively public girls set even scott	Books                                             	business                                          	3.36	414.96	0.14041709657333354
+AAAAAAAACNEDAAAA	Just young degrees stop posts. More than tight artists buy to a arts. European, essential techniques ought to sell to a offences. Sentences be	Books                                             	business                                          	2.58	6286.70	2.12733796276165399
+AAAAAAAACPBBAAAA	Other, black houses flow. New soldiers put only eastern hours. Applications reserve there methods; sources cry pretty scarcely special workers. Never british opportunities 	Books                                             	business                                          	8.20	3426.05	1.15933100471146462
+AAAAAAAADEDAAAAA	Junior, severe restrictions ought to want principles. Sure,	Books                                             	business                                          	9.77	3899.69	1.31960465427044307
+AAAAAAAAEBPAAAAA	Rows could not	Books                                             	business                                          	1.65	15875.48	5.37205708576254358
+AAAAAAAAEEFDAAAA	Remaining subjects handle even only certain ladies; eagerly literary days could not provide. Very different articles cut then. Boys see out of a houses. Governme	Books                                             	business                                          	9.03	916.17	0.31002007751973922
+AAAAAAAAEGCCAAAA	Manufacturing, ready concerns see already then new pupils. Both stable types used to manage otherw	Books                                             	business                                          	1.18	8723.00	2.95175036969632840
+AAAAAAAAEGKCAAAA	Russian windows should see in a weapons. New, considerable branches walk. English regions apply neither alone police; very new years w	Books                                             	business                                          	2.79	8434.64	2.85417307557668685
+AAAAAAAAEKDAAAAA	Long groups used to create more tiny feet; tools used to dare still	Books                                             	business                                          	57.04	795.66	0.26924105229308502
+AAAAAAAAEPLBAAAA	Drugs must compensate dark, modest houses. Small pubs claim naturally accessible relationships. Distinguished	Books                                             	business                                          	1.66	11559.25	3.91150068335575881
+AAAAAAAAFCGDAAAA	Small, capable centres	Books                                             	business                                          	2.98	19190.81	6.49392187461561344
+AAAAAAAAFDLAAAAA	Popular, different parameters might take open, used modules. Prisoners use pretty alternative lovers. Annual, professional others spend once true men. Other, small subsidies seem politically	Books                                             	business                                          	7.25	1761.61	0.59610603791823330
+AAAAAAAAFEGEAAAA	Supreme, free uses handle even in the customers. Other minutes might not make of course social neighbours. So environmental rights come other, able sales	Books                                             	business                                          	8.08	3500.18	1.18441563785437289
+AAAAAAAAFHFCAAAA	Sound, original activities consider quite to a attitudes. In order weak improvements marry available, hard studie	Books                                             	business                                          	71.27	11431.86	3.86839355512056274
+AAAAAAAAGIJAAAAA	Always other hours used to use. Women should jump then. Civil samples take therefore other offices. Concrete, major demands	Books                                             	business                                          	1.42	2038.40	0.68976819369356825
+AAAAAAAAGLADAAAA	Changes ensure different clients. Distinct, alone attacks think directly previous feelings. White children tell so medieval, responsible yea	Books                                             	business                                          	5.89	5116.38	1.73131681262259552
+AAAAAAAAHDKCAAAA	Visual fragments 	Books                                             	business                                          	6.77	2739.02	0.92684893931051673
+AAAAAAAAHDLBAAAA	Classic issues will draw as european, engl	Books                                             	business                                          	75.64	14735.99	4.98646840884344817
+AAAAAAAAHJAAAAAA	Again british shareholders see shares. American lives ought to establish horses. Then ideal conservatives might charge even nec	Books                                             	business                                          	2.44	9396.38	3.17961345165736401
+AAAAAAAAHKACAAAA	Critical cases tell anywhere to the circumstances. Dependent, new numbers must not	Books                                             	business                                          	3.72	726.75	0.24592279963049486
+AAAAAAAAHLKAAAAA	Confident, video-tape	Books                                             	business                                          	3.17	6173.95	2.08918482116091330
+AAAAAAAAIHNDAAAA	Of course fundamental children will not deal still including a suppliers. More crucial powers will not keep enough. As good comments used to devote even convenient electric problems. Publi	Books                                             	business                                          	8.85	2672.80	0.90444094785330122
+AAAAAAAAIMJAAAAA	Departments could seek now for a commu	Books                                             	business                                          	5.93	3205.71	1.08477079876638965
+AAAAAAAAJFBEAAAA	Paintings must not know primary, royal stands; similar, available others ough	Books                                             	business                                          	0.39	12939.97	4.37871847201185356
+AAAAAAAAJJGBAAAA	Most present eyes restore fat, central relationships; again considerable habits must face in a discussions. Engineers help at all direct occasions. Curiously del	Books                                             	business                                          	80.10	6877.89	2.32738901183430931
+AAAAAAAAKBMDAAAA	So white countries could secure more angry items. National feet must not defend too by the types; guidelines would not view more so flexible authorities. Critics will handle closely lig	Books                                             	business                                          	2.50	2135.27	0.72254774869901171
+AAAAAAAAKJHDAAAA	Simple changes ought to vote almost sudden techniques. Partial, golden faces mean in a officials; vertically minor 	Books                                             	business                                          	8.73	5996.87	2.02926323965617573
+AAAAAAAAKJOBAAAA	Christian lines stand once deep formal aspirations. National, fine islands play together with a patterns. New journals lose etc positive armie	Books                                             	business                                          	4.89	6106.50	2.06636061361350790
+AAAAAAAAKKDAAAAA	Children would not mean in favour of a parts. Heavy, whole others shall mean on	Books                                             	business                                          	3.13	5521.91	1.86854291917113983
+AAAAAAAAKLCCAAAA	Lips will n	Books                                             	business                                          	8.48	7806.43	2.64159493735051117
+AAAAAAAAKNJCAAAA	White fees might combine reports. Tr	Books                                             	business                                          	2.09	6566.98	2.22218108939451963
+AAAAAAAALAJCAAAA	Asleep children invite more. Wealthy forms could expect as. Indeed statistical examinations could la	Books                                             	business                                          	3.71	11535.83	3.90357565828889099
+AAAAAAAALDHBAAAA	Most new weeks go yet members. Also encouraging delegates make publications. Different competitors run resources; somehow common views m	Books                                             	business                                          	1.07	9334.32	3.15861315039135987
+AAAAAAAALHMBAAAA	Local, bloody names 	Books                                             	business                                          	4.40	2927.75	0.99071273012477651
+AAAAAAAALJJCAAAA	Large, larg	Books                                             	business                                          	3.50	5826.76	1.97170021599584758
+AAAAAAAALOMDAAAA	Only new systems might join late speeches. Materials could stay on a benefits. Corporate regulations must crawl definitely practical deaths. Windows might soothe despite a organisations. Old	Books                                             	business                                          	0.67	123.41	0.04176034771571981
+AAAAAAAAMBECAAAA	Professional managers form later initial grounds. Conscious, big risks restore. American, full rises say for a systems. Already	Books                                             	business                                          	5.27	1616.40	0.54696885218126163
+AAAAAAAAMKGDAAAA	Memories can earn particularly over quick contexts; alone differences make separate years; irish men mea	Books                                             	business                                          	4.23	6855.84	2.31992757704675870
+AAAAAAAANJLBAAAA	Only, gothic	Books                                             	business                                          	1.68	7807.37	2.64191302119179451
+AAAAAAAANKCAAAAA	Low, large clouds will not visit for example as the notions. Small, unacceptable drugs might not negotiate environmental, happy keys.	Books                                             	business                                          	3.11	3933.56	1.33106582416859905
+AAAAAAAAOAPAAAAA	Silver, critical operations could help howev	Books                                             	business                                          	5.56	9087.69	3.07515674850230731
+AAAAAAAAOBAEAAAA	Terrible, psychiatric bones will destroy also used studies; solely usual windows should not make shares. Advances continue sufficiently. As key days might not use far artists. Offici	Books                                             	business                                          	5.83	2024.32	0.68500370381562209
+AAAAAAAAOCHCAAAA	Too white addresses end by the talks. Hands get only companies. Statements know. Sentences would pay around for a payments; papers wait actually drinks; men would 	Books                                             	business                                          	6.06	5178.86	1.75245923645598158
+AAAAAAAAAGLDAAAA	New, big arguments may not win since by a tenant	Books                                             	computers                                         	1.00	1686.72	0.45326198032962534
+AAAAAAAAALNBAAAA	Else substantial problems slip months. Just unique corporations put vast areas. Supporters like far perfect chapters. Now young reports become wrong trials. Available ears shall	Books                                             	computers                                         	51.46	471.00	0.12656895793922734
+AAAAAAAABEBEAAAA	Cheap, desirable members take immediate, estimated debts; months must track typica	Books                                             	computers                                         	3.26	7226.83	1.94202195818247621
+AAAAAAAABHOAAAAA	Expert, scottish terms will ask quiet demands; poor bits attempt northern, dangerous si	Books                                             	computers                                         	2.66	4463.91	1.19955931429829364
+AAAAAAAACCDBAAAA	Gradually serious visitors bear no doubt technical hearts. Critics continue earlier soviet, standard minute	Books                                             	computers                                         	6.40	3244.45	0.87186126451364360
+AAAAAAAACCPBAAAA	Clear, general goods must know never women. Communications meet about. Other rewards spot wide in a skills. Relative, empty drawings facilitate too rooms. Still asian police end speedily comp	Books                                             	computers                                         	7.64	5385.04	1.44708896233770017
+AAAAAAAACDFEAAAA	Wide, essential activities make steadily procedures. Modules	Books                                             	computers                                         	35.95	7285.54	1.95779873848101557
+AAAAAAAACFMBAAAA	At least remaining results shall keep cuts. Clients should meet policies. Glorious, local times could use enough; clever styles will live political parents. Single, gradual contracts will describe ho	Books                                             	computers                                         	9.51	14393.77	3.86795004186180949
+AAAAAAAACLPDAAAA	Environmental, new women pay again fingers. Different, uncomfortable records miss far russian, dependent members. Enough double men will go here immediatel	Books                                             	computers                                         	89.89	9487.37	2.54948308807619376
+AAAAAAAACOFCAAAA	Years learn here. Days make too. Only moving systems avoid old groups; short movements cannot see respectiv	Books                                             	computers                                         	0.60	1706.66	0.45862033493962150
+AAAAAAAACONDAAAA	Magnetic	Books                                             	computers                                         	57.19	7638.33	2.05260184394042112
+AAAAAAAADAHAAAAA	Ga	Books                                             	computers                                         	5.53	7904.13	2.12402865714688954
+AAAAAAAADDBAAAAA	S	Books                                             	computers                                         	65.78	578.19	0.15537347301673430
+AAAAAAAAEAHCAAAA	Simple year	Books                                             	computers                                         	3.01	5038.44	1.35394925783295241
+AAAAAAAAECEEAAAA	Agricultural players shall smoke. So full reasons undertake 	Books                                             	computers                                         	0.70	5739.18	1.54225484506508439
+AAAAAAAAECGEAAAA	Then basic years can encourage later traditions. For example christian parts subscribe informal, valuable gr	Books                                             	computers                                         	2.75	11359.94	3.05268740563088364
+AAAAAAAAECHAAAAA	Boxes batt	Books                                             	computers                                         	0.83	6659.54	1.78957757569979198
+AAAAAAAAEFFAAAAA	Blocks extend ev	Books                                             	computers                                         	9.29	11249.90	3.02311702743208836
+AAAAAAAAEIGCAAAA	Separate, dead buildings think possibly english, net policies. Big divisions can use almost	Books                                             	computers                                         	9.46	3529.22	0.94838577014496795
+AAAAAAAAEJECAAAA	Artists make times. Rather ready functions must pre	Books                                             	computers                                         	5.71	7805.93	2.09763996995021836
+AAAAAAAAFDPCAAAA	Limited, capable cities shall try during the bodies. Specially economic services ought to prevent old area	Books                                             	computers                                         	2.93	6458.26	1.73548882866368225
+AAAAAAAAFGLAAAAA	Since other birds shall blame sudden	Books                                             	computers                                         	6.74	2404.97	0.64627292308939187
+AAAAAAAAFHNAAAAA	Legs throw then. Old-fashioned develo	Books                                             	computers                                         	2.66	12518.22	3.36394492707854445
+AAAAAAAAFJBEAAAA	Only careful men define judicial, special lawyers. Now able funds will not put too black, economic terms. Objectives know both points. Teeth pay.	Books                                             	computers                                         	9.85	911.50	0.24494183686115864
+AAAAAAAAGHEAAAAA	Men should not turn shadows. Different, single concessions guarantee only therefore alone products.	Books                                             	computers                                         	8.38	11864.77	3.18834729318175442
+AAAAAAAAGIFEAAAA	Educational, white teachers should not fix. Considerable, other services might not cover today on a forms. Successful genes fall otherwise so	Books                                             	computers                                         	1.65	7042.80	1.89256869845942737
+AAAAAAAAGKKAAAAA	Women note days. Other, efficient qualificati	Books                                             	computers                                         	7.64	8012.26	2.15308577269247054
+AAAAAAAAHGCEAAAA	Present 	Books                                             	computers                                         	2.84	4786.32	1.28619858760866792
+AAAAAAAAHHFDAAAA	Multiple, dark feet mean more complex girls; schools may not answer frequently blue assets. Spiritual, dry patients may reply personnel	Books                                             	computers                                         	2.04	2973.19	0.79896721880112808
+AAAAAAAAIBDEAAAA	Private teachers ap	Books                                             	computers                                         	5.27	8109.35	2.17917617635769258
+AAAAAAAAIDCDAAAA	Daily numbers sense interesting players. General advantages would speak here. Shelves shall know with the reductions. Again wrong mothers provide ways; as hot pr	Books                                             	computers                                         	7.56	13142.36	3.53166626340166409
+AAAAAAAAIECAAAAA	Inc, corporate ships slow evident degrees. Chosen, acute prices throw always. Budgets spend points. Commonly large events may mean. Bottles c	Books                                             	computers                                         	68.38	12405.10	3.33354687926095337
+AAAAAAAAINGCAAAA	European, possible problems ought to restore then unfair interests. States would sleep in respect of the questions. Ideal stages affect only pressures. About spanish employees might kno	Books                                             	computers                                         	3.42	6760.19	1.81662463645686890
+AAAAAAAAIOJBAAAA	Upper others narrow deaths. Situations could e	Books                                             	computers                                         	5.42	10932.74	2.93788855460829783
+AAAAAAAAIOKCAAAA	However old hours ma	Books                                             	computers                                         	8.84	5208.75	1.39971562561772907
+AAAAAAAAJDOCAAAA	Indeed other actions should provide after a ideas; exhibitio	Books                                             	computers                                         	6.95	3949.76	1.06139491997885895
+AAAAAAAAJJFCAAAA	Effective females must answer too english projects. Firm, political experiments see in terms of 	Books                                             	computers                                         	0.76	246.87	0.06633986973770075
+AAAAAAAAJPBDAAAA	Of course responsible fears tell. Now clear substances might develop at least independent civil tourists.	Books                                             	computers                                         	4.95	619.44	0.16645833398274943
+AAAAAAAAKDGEAAAA	Perfect days find at all. Crimes might develop hopes. Much socialist grants drive current, useful walls. Emissions open naturally. Combinations shall not know. Tragic things shall not receive just	Books                                             	computers                                         	6.71	8038.78	2.16021233057898500
+AAAAAAAAKEAAAAAA	Advantages apply almost on a services; materials defeat today individual ideas. Domestic divisions used to win smoothly irish talks. Subsequent quantities make only, automatic pounds. Flower	Books                                             	computers                                         	7.87	442.26	0.11884583298981461
+AAAAAAAAKJDBAAAA	Close, historic tactics lead ago large, typical stars. Generally significant facilities check leading years; yesterday general years 	Books                                             	computers                                         	3.87	8448.38	2.27028164092273769
+AAAAAAAAKJECAAAA	Internal seats used to sell dark words. Universal items show now in the roles. Most wonderf	Books                                             	computers                                         	2.57	870.24	0.23385428865612144
+AAAAAAAAKKACAAAA	Likely, separate attacks prefer seats. Informally equal women could use easy prime, big forces. Long technical women save conditions; fast alone rooms sell. Ne	Books                                             	computers                                         	3.77	344.40	0.09254851191989362
+AAAAAAAAKOJBAAAA	Economic customs should not put unpleasant shops. Colonial, middle goods used to see. Closely explicit legs continue	Books                                             	computers                                         	3.32	8481.54	2.27919252551990282
+AAAAAAAAKOKBAAAA	Human windows take right, variable steps. Years should buy often. Indeed thin figures may beat even up to a cars. Details may tell enough. Impossible, sufficient differences ought to return 	Books                                             	computers                                         	4.47	5466.16	1.46888784528468556
+AAAAAAAAKPNDAAAA	Left diff	Books                                             	computers                                         	0.74	3269.32	0.87854442796151745
+AAAAAAAALBNAAAAA	Male levels shall reduce else high, local conditions; further personal agencies control. Successful days wake eve	Books                                             	computers                                         	6.55	2376.38	0.63859010672531010
+AAAAAAAALEJBAAAA	Wide governments conform widely in proportion to a friends. So living rooms wear too clothes; most essential measures should not bring previously pains. Real accounts become also gue	Books                                             	computers                                         	9.35	11110.42	2.98563541755233586
+AAAAAAAALIHCAAAA	Places transform 	Books                                             	computers                                         	3.10	5805.20	1.55999599708875273
+AAAAAAAAMADEAAAA	Appropriate effects beg passages. Running contracts must keep only upper sons. Safely available reports intend perhaps w	Books                                             	computers                                         	5.81	8969.60	2.41034591323077181
+AAAAAAAAMDKBAAAA	Friendly, hot computers tax elsewhere units. New, real officials should l	Books                                             	computers                                         	3.19	2999.57	0.80605615534133364
+AAAAAAAAMELCAAAA	Perfect members state democratic schools. Genuine, enormous knees must afford around the implications. Matters will indicate with a months. Still regular machines would not l	Books                                             	computers                                         	4.07	2110.95	0.56726272136265806
+AAAAAAAAMENDAAAA	Kinds relieve really major practices. Then capable reserves could not approve foundations. Pos	Books                                             	computers                                         	7.23	1522.62	0.40916438797755059
+AAAAAAAANANCAAAA	Only increased errors must submit as rich, main 	Books                                             	computers                                         	6.94	8287.27	2.22698753303826016
+AAAAAAAANFHDAAAA	Meals ought to test. Round days might need most urban years. Political, english pages must see on a eyes. Only subsequent women may come better methods; difficult, social childr	Books                                             	computers                                         	7.23	15325.54	4.11833891222069241
+AAAAAAAANHFDAAAA	Systems cannot see fairly practitioners. Little ca	Books                                             	computers                                         	1.73	2428.71	0.65265242852777245
+AAAAAAAANKLDAAAA	Past beautiful others might not like more than legislative, small products. Close, wh	Books                                             	computers                                         	3.02	4174.86	1.12188467036552578
+AAAAAAAAOGDDAAAA	Main problems wait properly. Everyday, foreign offenders can worry activities. Social, important shoes will afford okay physical parts. Very	Books                                             	computers                                         	1.40	939.26	0.25240161238640906
+AAAAAAAAOGMDAAAA	Schools offer quickly others. Further main buildings satisfy sadly great, productive figures. Years contribute acti	Books                                             	computers                                         	4.11	885.92	0.23806787944271822
+AAAAAAAAOKLAAAAA	Chief me	Books                                             	computers                                         	2.62	9675.59	2.60006230094948754
+AAAAAAAAOMDAAAAA	Tiny, rare leaders mention old, precious areas; students will improve much multiple stars. Even confident solutions will include clearly single women. Please little rights will not mention harder com	Books                                             	computers                                         	1.45	3092.13	0.83092923972956056
+AAAAAAAAONDCAAAA	Guidelines should investigate so. Usual personnel look now old, modern aspects. Discussions could appear once br	Books                                             	computers                                         	2.44	9923.72	2.66674076280396839
+AAAAAAAAONHDAAAA	Flat pleasant groups would go private, redundant eyes. Main devic	Books                                             	computers                                         	2.83	2445.21	0.65708637291417851
+AAAAAAAAOPDAAAAA	Popular, obvious copies should believe still difficult parts. Forms ought to soften characteristic	Books                                             	computers                                         	1.05	2156.19	0.57941979069847684
+AAAAAAAAPAKAAAAA	Real, domestic facilities turn often guilty symptoms. Winds get naturally intense islands. Products shall not travel a little clear shares; improved children may not apply wrong c	Books                                             	computers                                         	5.28	1338.00	0.35955258115219995
+AAAAAAAAAADCAAAA	Directions would ask yet profits. Forthcoming, specified discussions ought	Books                                             	cooking                                           	0.58	5750.02	2.05632295473197783
+AAAAAAAAABIBAAAA	Then irish champions must look now forward good women. Future, big models sign. Then different o	Books                                             	cooking                                           	85.81	2279.71	0.81527020830049933
+AAAAAAAAACLDAAAA	Black ears see sensibly glad months. Equal members must afford approximately o	Books                                             	cooking                                           	8.37	10363.44	3.70617485886789408
+AAAAAAAAAEFBAAAA	Considerable benefits should govern. Well experienced years provide please in an towns. Exc	Books                                             	cooking                                           	4.18	0.00	0.00000000000000000
+AAAAAAAAAGHBAAAA	Valuable studies should persist so concerned parties. Always polite songs include then from the holes. There conventional areas might not explain theore	Books                                             	cooking                                           	1.58	1326.45	0.47436523408687831
+AAAAAAAAAIJCAAAA	Meanings occur in a things. Also essential features may not satisfy by the potatoes; happy words change childre	Books                                             	cooking                                           	3.46	1262.55	0.45151330717055917
+AAAAAAAAAJDBAAAA	Then dominant goods should combine probably american items. Important artists guess only sill	Books                                             	cooking                                           	6.67	5569.20	1.99165808110116677
+AAAAAAAACDIBAAAA	Libraries shall note still. Children would not concentrate. Local, public modes must mean low children. Outer, good years should vis	Books                                             	cooking                                           	1.42	2178.99	0.77925070784648269
+AAAAAAAACJJAAAAA	Children ought to miss historical effects. Honest girls may not think activities. Woo	Books                                             	cooking                                           	4.42	348.88	0.12476651428114901
+AAAAAAAACLFEAAAA	Single, past rates mark blue, evident discussions. Only literary children ought to publish exactly really recent themes; conscious, ready conditions would adopt advanced, ideal provisions. A	Books                                             	cooking                                           	4.99	9499.97	3.39738059698316657
+AAAAAAAACPEEAAAA	Standards could lead no longer ago great tactics; difficult lives might feel french, easy costs. Students drop certainly unabl	Books                                             	cooking                                           	3.05	16321.01	5.83672187356046718
+AAAAAAAADDNAAAAA	Individual, remarkable services take by the interest	Books                                             	cooking                                           	6.05	1054.65	0.37716408016112647
+AAAAAAAADIGCAAAA	Positions shall	Books                                             	cooking                                           	4.21	2629.53	0.94037288551281172
+AAAAAAAAEGFEAAAA	Ultimately senior elections marry at l	Books                                             	cooking                                           	5.06	7756.87	2.77401293175881769
+AAAAAAAAEINDAAAA	Hence young effects shall not solve however months. In order small activities must not return almost national foods. International decades take contributions. Sessions must see 	Books                                             	cooking                                           	1.43	19276.07	6.89351084309627374
+AAAAAAAAEKJCAAAA	Members need for a regions. Leading needs go at least under the others; old police could play on a drinks. Very similar machines must consider fully effec	Books                                             	cooking                                           	9.86	10833.86	3.87440652490818908
+AAAAAAAAEKPDAAAA	Mainly catholic activities could assume just fat, o	Books                                             	cooking                                           	2.68	2262.61	0.80915490391444210
+AAAAAAAAENDCAAAA	Points trace so simple eyes. Short advisers shall not say limitations. Keys stretch in full now blue wings. Immediately strategic students would not make strangely for the players.	Books                                             	cooking                                           	1.69	5132.94	1.83564271902740482
+AAAAAAAAEPJDAAAA	Projects become more from a pupils. Details may precede generally; good, marvellous birds could suffer fair	Books                                             	cooking                                           	9.88	628.36	0.22471419087853357
+AAAAAAAAFDIAAAAA	Great pp. will not r	Books                                             	cooking                                           	1.91	2941.23	1.05184308300603044
+AAAAAAAAFPEBAAAA	New, general students raise therefore to a women; united letters would start black positio	Books                                             	cooking                                           	4.03	3747.49	1.34017789670793138
+AAAAAAAAGFGCAAAA	Products may not resist further specif	Books                                             	cooking                                           	5.37	8721.33	3.11892325153523644
+AAAAAAAAHBGCAAAA	Dramatic months deal broadly in a films. Almost new occasions may get together sources. Under dry orders wor	Books                                             	cooking                                           	3.92	1412.78	0.50523858073297895
+AAAAAAAAHCGEAAAA	Thus certain stars appear totally even local guests. Urban friends might not take properly various vehicles	Books                                             	cooking                                           	4.55	1446.44	0.51727607462974425
+AAAAAAAAICAAAAAA	Somet	Books                                             	cooking                                           	7.34	6593.72	2.35804706645808830
+AAAAAAAAIFHDAAAA	Genetic properties might describe therefore leaves; right other organisers must not talk even lives; methods carry thus wrong minutes. Proud worke	Books                                             	cooking                                           	1.08	119.92	0.04288580713309846
+AAAAAAAAIHHDAAAA	Urgent agencies mean over as a plants; then	Books                                             	cooking                                           	6.47	9566.59	3.42120525067902230
+AAAAAAAAIJJAAAAA	Men could require evolutionary falls; taxes teach dead parents; only financial servants might not buy eastern things. Different payments develop. New inhabitants might not eat w	Books                                             	cooking                                           	80.50	3855.42	1.37877583836799906
+AAAAAAAAINMBAAAA	Hours ought to cope thus into the eyes. Dark states reduce most for the feelings. National, tragic children shall establish enough typical boats. In order secret hours must mean; sin	Books                                             	cooking                                           	2.30	12966.63	4.63712802990534045
+AAAAAAAAJHGDAAAA	Guests agree around trying, young costs. Here annual banks appeas	Books                                             	cooking                                           	58.88	8031.52	2.87223330308224573
+AAAAAAAAJIHBAAAA	Wonderful qualities suffer of course light leaders. True clubs used to see early living operat	Books                                             	cooking                                           	9.91	4482.62	1.60307518988467144
+AAAAAAAAKABAAAAA	High big appeals may	Books                                             	cooking                                           	36.23	675.62	0.24161531867298181
+AAAAAAAAKCGEAAAA	Final women should establish on a pupils. Full, northern years might not avoid full	Books                                             	cooking                                           	60.38	5877.02	2.10174071245298770
+AAAAAAAAKECDAAAA	Little part	Books                                             	cooking                                           	9.90	4729.36	1.69131438311366337
+AAAAAAAAKFIAAAAA	Here other affairs afford directly effective leads. Plants cannot undertake as coming, huge photographs; d	Books                                             	cooking                                           	0.87	20785.39	7.43327407210001090
+AAAAAAAAKFNBAAAA	Stairs might bring early others. Large forms rel	Books                                             	cooking                                           	1.88	2350.18	0.84047169953356678
+AAAAAAAAKHADAAAA	Now available m	Books                                             	cooking                                           	3.55	1102.96	0.39444070910208700
+AAAAAAAAKJNDAAAA	Major instructions put flatly british, other democrats. Operations represent well upon a stores. Thousands will not appear surely 	Books                                             	cooking                                           	1.29	582.88	0.20844962693245854
+AAAAAAAAKLACAAAA	New, single products raise too extreme, efficient minutes; hands support leaders. Additional, english things prefer halfway private, slow churches. High white things could f	Books                                             	cooking                                           	4.13	2472.08	0.88406559454294555
+AAAAAAAAKLEDAAAA	Golden, sure days fill of course. Early free minutes must not express only, cap	Books                                             	cooking                                           	9.44	4521.21	1.61687575106934680
+AAAAAAAAKLOCAAAA	Purposes hide tears. Small laws award good eyes. 	Books                                             	cooking                                           	55.11	5382.78	1.92499053468895684
+AAAAAAAALCJDAAAA	Yet religious animals ensure also. Rough, real heads resist dead. Civil, evolutionary votes dissuade rapidly left cars. Buyers 	Books                                             	cooking                                           	2.20	11624.81	4.15726617427380135
+AAAAAAAALLNCAAAA	Here comprehensive years should tend sensibly particular front sales. Official, coherent tears regulate animals. Rewards cannot w	Books                                             	cooking                                           	2.50	2499.59	0.89390372458156745
+AAAAAAAAMAODAAAA	Widely splendid others deprive only. Different, main soldiers discover then other periods. Too main birds must change public, terrible houses. Different, armed females may foster; science	Books                                             	cooking                                           	4.26	6853.89	2.45108909816104214
+AAAAAAAAMGBDAAAA	New women add however. Scottish managers place mostly. Normal, financial purposes should lea	Books                                             	cooking                                           	4.74	319.20	0.11415234853973505
+AAAAAAAAMINCAAAA	Extra theories drop. Other resources shall know eventually anyway open students. Long-term, liable di	Books                                             	cooking                                           	6.96	5834.64	2.08658477093947276
+AAAAAAAAMJIAAAAA	Special, public skills agree at a parent	Books                                             	cooking                                           	5.87	4713.66	1.68569974692295585
+AAAAAAAAMKGAAAAA	Gentle fans cannot pay else can	Books                                             	cooking                                           	2.45	7576.48	2.70950183478800689
+AAAAAAAANHBBAAAA	Sound, new offices might equip hot, new reports; calculations fight great scientists. Professional, little issues learn of c	Books                                             	cooking                                           	66.16	6628.48	2.37047794250834265
+AAAAAAAAOLDBAAAA	Well angry rebels drop in a methods. Studies argue most sometimes residential organisations. Rural, different children know o	Books                                             	cooking                                           	4.42	453.06	0.16202338041795852
+AAAAAAAAPAFCAAAA	Half general features used to defend as ready medical pounds. Turkish, trying rooms secure with a ci	Books                                             	cooking                                           	7.08	683.53	0.24444409397670770
+AAAAAAAAPHHCAAAA	African, elected carers would examine proba	Books                                             	cooking                                           	6.20	15598.69	5.57840569437117702
+AAAAAAAAAFBEAAAA	Already accessible clubs match all enough o	Books                                             	entertainments                                    	5.00	1196.30	0.46493128593083651
+AAAAAAAAAGKBAAAA	Likely, various days develop no longer. Officials say before agricultural, rare ho	Books                                             	entertainments                                    	2.67	23516.84	9.13960934734576042
+AAAAAAAAAIKBAAAA	Less progressive experiences would silence as economic, soviet specialists. Alone legal brothers fight only ears. Methods could not return records. E	Books                                             	entertainments                                    	8.36	5931.28	2.30513887621487248
+AAAAAAAAAJCDAAAA	Strict heads discuss as categories. Alone, specific markets wait single, human numbers. External, various changes want very relatively nuclear orders. Old, pre	Books                                             	entertainments                                    	2.32	4525.09	1.75863572068274594
+AAAAAAAAAMFCAAAA	Instances used to lower out of a costs. Irish supporters sign in a types. Bad things shall participate clear 	Books                                             	entertainments                                    	1.58	3570.57	1.38767006737947580
+AAAAAAAAANJDAAAA	Trustees may encourage today necessary, political tears; inner, foreign times pay in the historians. Areas may belie	Books                                             	entertainments                                    	1.79	17322.75	6.73233171726021741
+AAAAAAAABAJDAAAA	Rare, radical beds say over readers; han	Books                                             	entertainments                                    	7.10	7808.46	3.03468807902658165
+AAAAAAAACKAEAAAA	L	Books                                             	entertainments                                    	1.63	4264.23	1.65725481685601518
+AAAAAAAACLMDAAAA	Always constitutional advertisements go for a spaces. Cars spend bad difficulties. Rights encourage further great qualities. Blue, high homes would produce u	Books                                             	entertainments                                    	2.63	3974.52	1.54466161878945775
+AAAAAAAACMOAAAAA	Companies ought to record now detailed, good roads. Muscles shall not argue significantly territorial months. Clearly new periods could write in a committees. Figures will not find more there	Books                                             	entertainments                                    	3.07	7276.45	2.82792715498740725
+AAAAAAAACMPAAAAA	Falsely large trees shall reflect against a 	Books                                             	entertainments                                    	0.70	957.09	0.37196446079707792
+AAAAAAAAEGNDAAAA	Deep patterns shall find british, american expectations. Sufficient patients must see. English, large assets could not meet for the proceedings. White, chinese matches shal	Books                                             	entertainments                                    	0.56	1499.01	0.58257681762365897
+AAAAAAAAEKGAAAAA	Particular, deliberate things rain however original ways. Virtually old deaths consider women. Notably w	Books                                             	entertainments                                    	9.71	1611.84	0.62642718708915783
+AAAAAAAAEPCEAAAA	New, previous police outline right in a persons. Wealthy quest	Books                                             	entertainments                                    	2.07	5037.58	1.95781037146155928
+AAAAAAAAFAKBAAAA	Doors cannot happen here severe, old rates. Inevitable, int	Books                                             	entertainments                                    	2.29	1047.84	0.40723363591888968
+AAAAAAAAFGJCAAAA	Limitations respond. Bare rivers will not create yesterday. Well local persons may unders	Books                                             	entertainments                                    	8.95	2096.28	0.81470045646668390
+AAAAAAAAGCBCAAAA	So perfect changes would twist again; legal standards like improvements; rights used to tell working stations. Official, immediate loans listen much possible pictures. Always d	Books                                             	entertainments                                    	6.32	1017.52	0.39545003933824690
+AAAAAAAAHFABAAAA	Prisons take angry, logical sums. Now old grounds cannot help so increased problems. Blue, negative designs would freeze. Small payments ask alike to a hundreds. Exte	Books                                             	entertainments                                    	2.62	11202.91	4.35391068500161131
+AAAAAAAAHGMAAAAA	High, official employees shall not start too left circumstances. Patients used to touch obviously popular, senior members. British, impossible theories make only. Young, international wo	Books                                             	entertainments                                    	4.85	1041.70	0.40484737988309988
+AAAAAAAAHPPDAAAA	Now old tears give. Other kids coincide up a animals; specific procedures remove future, french levels. Coming, strong values a	Books                                             	entertainments                                    	5.08	24460.84	9.50648649682223761
+AAAAAAAAIACBAAAA	Large women establish today polite, easy horses. Details sha	Books                                             	entertainments                                    	5.06	1748.58	0.67956996401650263
+AAAAAAAAIDDAAAAA	Plans would not claim; most particular horses will not tell simply cases; more british enquiries could not smile blue men. Old, dangerous play	Books                                             	entertainments                                    	0.95	6942.27	2.69805108950854163
+AAAAAAAAIKFAAAAA	Pieces threaten	Books                                             	entertainments                                    	0.69	1273.35	0.49487607869266126
+AAAAAAAAINCCAAAA	Cases can accept gmt sudden services; tools show all also capable meals; important, spatial days would not happen human, cold backs. Red, economic effects must s	Books                                             	entertainments                                    	9.58	1334.73	0.51873086622959576
+AAAAAAAAIPLAAAAA	Financial gods might presume divine, tiny 	Books                                             	entertainments                                    	8.42	731.84	0.28442306469583164
+AAAAAAAAKBFEAAAA	Marginal, available teeth pay recently little services. Then british times could require more scottish fair tea	Books                                             	entertainments                                    	95.74	3018.65	1.17317130007115240
+AAAAAAAAKFGCAAAA	Now complete others shall pass. Just front advantages could exercise more establish	Books                                             	entertainments                                    	6.51	5281.66	2.05266987849992639
+AAAAAAAAKKIDAAAA	Young reasons could not say impossible experiences. Prisoners cancel particularly; forms might k	Books                                             	entertainments                                    	3.77	3626.88	1.40955444480216694
+AAAAAAAAKLNAAAAA	Just particular actions seem very; necessarily poor eleme	Books                                             	entertainments                                    	0.26	6872.96	2.67111437845958545
+AAAAAAAAKMOCAAAA	Japanese, efficient sports withdraw recently severe days; factors mean originally impossible items. Quiet workers become from a officers. Pieces explore. For example o	Books                                             	entertainments                                    	3.74	16796.75	6.52790652592057016
+AAAAAAAALMJCAAAA	Never able feet go on the provisions. Services play brown studies. Cruel,	Books                                             	entertainments                                    	9.79	12846.63	4.99272774870656373
+AAAAAAAAMDOCAAAA	Internal claims speculate perhaps through a expectations. Immediate courts appeal to a councils; transactions materialise entirely; fine, serious conditions may not use to a types. Short, large lip	Books                                             	entertainments                                    	3.11	5231.34	2.03311346095579892
+AAAAAAAAMJABAAAA	Front, possible foundations hear well. Old, close inches change pointedly for the employees; odd, financial members work under on the arrangements; st	Books                                             	entertainments                                    	92.23	225.66	0.08770073893099771
+AAAAAAAAMJDCAAAA	Local words co	Books                                             	entertainments                                    	2.95	9381.26	3.64594271959501737
+AAAAAAAAMOFBAAAA	Hardly local women should tell easily tall, able issues. Important, available conditions could no	Books                                             	entertainments                                    	2.21	15740.54	6.11741996442846214
+AAAAAAAAMOHBAAAA	General, raw tests would not buy heavy, considerable blues. High, regional modules meet often young, responsible calculations. Things hesitat	Books                                             	entertainments                                    	2.00	5567.90	2.16391449212931922
+AAAAAAAAOAODAAAA	H	Books                                             	entertainments                                    	4.80	2493.52	0.96908422644341674
+AAAAAAAAOBODAAAA	New hours borrow new poets. Youngsters mind especially. Laws must add there in a ends. Factors must not take strategic, royal tr	Books                                             	entertainments                                    	2.30	4109.90	1.59727584389128560
+AAAAAAAAOEGCAAAA	Clear materials will ship evidently literally scottish targets. Residential heads make prominent times. Internal, open subjects feel subsequent 	Books                                             	entertainments                                    	0.75	263.40	0.10236805208909332
+AAAAAAAAOEMBAAAA	Other practices get feet. Numbers will not increase now large, simple foreigners. Flowers cover	Books                                             	entertainments                                    	1.00	315.51	0.12262013710945267
+AAAAAAAAOFEAAAAA	Heavy, formal factors could want then truly serious players. Be	Books                                             	entertainments                                    	4.31	8757.62	3.40357061631163789
+AAAAAAAAPCKBAAAA	Men call tonight particularly mental lines. Recent markets must dress children. Multiple relations should seem relatively about a arts. Funny, real proteins shall keep citie	Books                                             	entertainments                                    	5.20	3090.94	1.20126616144366780
+AAAAAAAAPEGAAAAA	Dirty trials should get. Balls shall win later national programmes. Elements ought to explain apart poss	Books                                             	entertainments                                    	1.62	290.34	0.11283804192690719
+AAAAAAAAPFIAAAAA	Subsequent, 	Books                                             	entertainments                                    	1.29	9603.95	3.73248919461293761
+AAAAAAAAPNPAAAAA	Countries turn more actually scientific patients. Good writers could not drag perhaps. Suddenly left months cannot announce more overall loans; beds transform far 	Books                                             	entertainments                                    	1.32	2401.56	0.93334479565331415
+AAAAAAAAACCEAAAA	Royal, blue men used to convey jobs. Other, technical things would say as mere children; ab	Books                                             	fiction                                           	0.62	555.50	0.18274906106295868
+AAAAAAAAAEGEAAAA	Exclusively ready fields invest right in a courts. Quite glad facts would not imitate usually by a types. More large managers can continue both small matters. Additional, basic scholars s	Books                                             	fiction                                           	1.11	3969.44	1.30587116641899316
+AAAAAAAAAIDCAAAA	Dollars get on a years; separate economies can say. Firms know even sons. Simple, definite members will say most cold, big policies; main, true agents might repeat too. Elements know goods. Great 	Books                                             	fiction                                           	5.03	149.04	0.04903135924540659
+AAAAAAAAAMOCAAAA	Wild officials will not watch onl	Books                                             	fiction                                           	0.47	6954.51	2.28790310108543073
+AAAAAAAAAODAAAAA	Just minor eyes exc	Books                                             	fiction                                           	7.11	16681.12	5.48777500896227056
+AAAAAAAABGFEAAAA	Married circumstances face human, compulsory hours. Years make sometimes national problems. Difficulties should invest far running, medical centuries; perf	Books                                             	fiction                                           	2.71	10221.52	3.36268799754501063
+AAAAAAAABJAAAAAA	Other horses apply able schools; possible enquiries would not describe easily r	Books                                             	fiction                                           	3.83	10067.63	3.31206107944063852
+AAAAAAAABKFAAAAA	Firm, local examinations may not sponsor most rural charges. Countries shall help beautiful, different terms	Books                                             	fiction                                           	7.72	5090.34	1.67462620250444840
+AAAAAAAABOJBAAAA	As joint men would so	Books                                             	fiction                                           	2.13	2773.11	0.91230107781152357
+AAAAAAAABPGDAAAA	Pictures get with a conditions; still gross eyes go that. Personal beings contact thereafter in a systems. New, medium goals might not tell; as official years mu	Books                                             	fiction                                           	5.52	2061.58	0.67822107885899974
+AAAAAAAACEBDAAAA	Essential, alternative fans let unlikel	Books                                             	fiction                                           	1.52	2460.17	0.80934969856932323
+AAAAAAAACMNBAAAA	Basic changes may not see; afraid names seek in excess of a characteristics. Awful scientists shall not want now right eyes. Here used workers will not pray in part	Books                                             	fiction                                           	2.27	6034.24	1.98515156476786280
+AAAAAAAACNBBAAAA	Local companies would restrict yet most imaginative days. Married, str	Books                                             	fiction                                           	99.71	7003.69	2.30408239689654919
+AAAAAAAADCGCAAAA	Different stations may smell; weapons disguise cons	Books                                             	fiction                                           	1.47	1671.19	0.54979010505455611
+AAAAAAAADCODAAAA	Private, quiet populations shall receive more somewhat proposed machines. Heads protect abroad parent	Books                                             	fiction                                           	74.86	3243.16	1.06693869464796593
+AAAAAAAADDBDAAAA	Circumstances should include parties. Good investigations fall as deposits. Characters might force at all convenient, special years; 	Books                                             	fiction                                           	5.18	12.59	0.00414187340914968
+AAAAAAAADNMAAAAA	Old, official cases look enough; actual emotions go statistical, wild limits. Mental cities hear above mod	Books                                             	fiction                                           	2.55	769.44	0.25313130070978025
+AAAAAAAADPHDAAAA	Times should not get on a lists; different students undermine suddenly groups. Even actual modules may stay for a 	Books                                             	fiction                                           	8.31	638.38	0.21001502358482729
+AAAAAAAAEBJAAAAA	Techniques render eventually dark tiles. Only, other centres would bid at the falls. Sorry, full days write for a groups. Both 	Books                                             	fiction                                           	2.99	6665.04	2.19267291079579140
+AAAAAAAAEEEEAAAA	Towns see even afraid, mean factors. Soldiers spend areas; resu	Books                                             	fiction                                           	48.40	9444.91	3.10719790157362568
+AAAAAAAAEJDEAAAA	Loud young standards remove enough green values; important students cannot receive particular police; significant authorities should not expect	Books                                             	fiction                                           	52.22	8870.17	2.91811924206809036
+AAAAAAAAFHDBAAAA	Good, bad cats could not finance libraries. Concerned names get at 	Books                                             	fiction                                           	0.13	5959.16	1.96045165566866039
+AAAAAAAAFMHBAAAA	Years take critics. Again academic areas look high under a w	Books                                             	fiction                                           	90.57	742.90	0.24440013944855446
+AAAAAAAAGFHBAAAA	Ambitious, isolated mines should	Books                                             	fiction                                           	9.67	5292.65	1.74118239070183305
+AAAAAAAAGFODAAAA	Wives must file upon a respects; anywhere growing wounds may not develop yet for a demands; quite key sides could not make fresh men. Dead times	Books                                             	fiction                                           	18.03	6121.11	2.01373016230978759
+AAAAAAAAGIOAAAAA	Thus separate stars will touch lightly commercial great institutions. Personal, brief hands will not concern always smart rules. Dead 	Books                                             	fiction                                           	4.96	2769.10	0.91098186316730672
+AAAAAAAAHDHCAAAA	Difficult decisions retain concerns. Accordingly parliamentary cases worry only inadequate, good scores. Responsible adults exist still well silly	Books                                             	fiction                                           	2.74	2397.93	0.78887390818127904
+AAAAAAAAHFFCAAAA	Necessarily royal losses ought to say courses. True, current 	Books                                             	fiction                                           	0.62	5056.32	1.66343426180712733
+AAAAAAAAIBMCAAAA	Others reflect much up to a paintings; twice narrow cases cannot wear however hard major wings. Popular bacteria go	Books                                             	fiction                                           	8.71	3061.36	1.00712991102736127
+AAAAAAAAIFBAAAAA	Usually sure students give. Popular resources may give especially full, fine paintings. Ever possible borders shall not free. New bodies help apart. Further main readers could esca	Books                                             	fiction                                           	3.51	11100.42	3.65182958128620664
+AAAAAAAAIGADAAAA	Communications move afterwards different errors; warm goods give at all. Twins could return f	Books                                             	fiction                                           	0.34	5726.99	1.88407208859937665
+AAAAAAAAIJPDAAAA	New, united books ought to earn things. Home domestic bands shal	Books                                             	fiction                                           	3.36	8480.61	2.78996132266631505
+AAAAAAAAIMOAAAAA	Different, expensive years used to learn humans. Normally parliamentary cards benefit. Certain consequences used to encourage. More new proposals could not prom	Books                                             	fiction                                           	3.33	8887.28	2.92374811053755431
+AAAAAAAAIOMDAAAA	Good levels ask quiet, particular objects. Previously rural re	Books                                             	fiction                                           	4.72	3395.05	1.11690765033626979
+AAAAAAAAJHGAAAAA	Large hearts used to say annually. For example separate criteria should admit gay ministers. Growing, ordinary	Books                                             	fiction                                           	1.92	3430.77	1.12865885908724888
+AAAAAAAAJHIBAAAA	Plans mi	Books                                             	fiction                                           	4.76	533.80	0.17561016884861808
+AAAAAAAAJJBAAAAA	Citizens can b	Books                                             	fiction                                           	4.61	584.00	0.19212502549193136
+AAAAAAAAJKFEAAAA	Personal, sympathetic text	Books                                             	fiction                                           	0.15	3428.40	1.12787917362420799
+AAAAAAAAJLODAAAA	Social, private books ought to demand merely social companies. Alive, swiss police will rest again victorian, married commentators. Standard, european territories attend to a comments. Books atte	Books                                             	fiction                                           	2.81	3504.94	1.15305939528714023
+AAAAAAAAJOKBAAAA	Favourably present words can make small, economic cases. About eastern years give less views. Only possible workers may accept even requirements. Negative goods imp	Books                                             	fiction                                           	4.00	4392.10	1.44491836380669814
+AAAAAAAAKDGAAAAA	Provinces complement more. Participants cannot lie swiftly then total muscles. Unions surprise perio	Books                                             	fiction                                           	2.17	1757.38	0.57814499537501769
+AAAAAAAAKGGBAAAA	New, novel individuals used to pay at the rates. Especially social values sleep too unaware cattle. Also immediate changes give almost chains. Swee	Books                                             	fiction                                           	1.98	11006.58	3.62095798472428397
+AAAAAAAAKHNCAAAA	Also good forms	Books                                             	fiction                                           	4.30	2992.89	0.98460456771326445
+AAAAAAAAKIJCAAAA	Mo	Books                                             	fiction                                           	6.72	9516.74	3.13082862174671717
+AAAAAAAAKMMAAAAA	Then wild sciences will know in a chemicals. Extremely	Books                                             	fiction                                           	5.84	10044.66	3.30450438109209457
+AAAAAAAAKNJAAAAA	Likewise high penalties might afford never square, thin	Books                                             	fiction                                           	1.65	209.10	0.06878997059993638
+AAAAAAAALDNBAAAA	Enough little accountants light only important, great systems. Determined sk	Books                                             	fiction                                           	0.36	6117.14	2.01242410691389210
+AAAAAAAALGFAAAAA	Primary, good features assess then early, bad c	Books                                             	fiction                                           	4.63	2352.74	0.77400724739021675
+AAAAAAAALHNDAAAA	Mass attitudes may like occupational state	Books                                             	fiction                                           	6.40	528.87	0.17398829149300982
+AAAAAAAALIOBAAAA	Additional officers shall not apply so poin	Books                                             	fiction                                           	9.09	6890.24	2.26675947884507726
+AAAAAAAAMAEAAAAA	In order financial glasses must kill convenient, important papers. Shy cities like below fragments. Patients ma	Books                                             	fiction                                           	6.94	8176.49	2.68991155767897573
+AAAAAAAAMBOBAAAA	Goods keep points. Again sensitive windows must not cause closely female, individual powers; gaps derive suddenly sincerely other hands; other houses may not imagine under for a data	Books                                             	fiction                                           	7.80	6049.19	1.99006983382797303
+AAAAAAAAMFPDAAAA	Pretty realistic facts may not work without a guidelines. Overall patterns t	Books                                             	fiction                                           	15.95	13032.24	4.28736205859069780
+AAAAAAAAMOODAAAA	Mechanically whole rooms might like then please specialist relatives. Als	Books                                             	fiction                                           	3.90	6774.40	2.22865029570640375
+AAAAAAAANGNCAAAA	Important enterprises could flow without a countries; ugly, previous things see even de	Books                                             	fiction                                           	0.82	887.04	0.29181949077459382
+AAAAAAAANIEDAAAA	Excellent, relevant concentrations seem exciting, local children. Units should not reinforce current lips; pure feet shall show always into a minutes. Commonly primit	Books                                             	fiction                                           	2.70	4113.69	1.35332670567791628
+AAAAAAAANLEDAAAA	Conservative, available	Books                                             	fiction                                           	2.01	2510.09	0.82577244047438695
+AAAAAAAANOAEAAAA	Black women treat really users. Expert, hard authorities should produce good indians; little, other details could waste. Ideas shall build. Low day	Books                                             	fiction                                           	0.72	9472.17	3.11616592930463604
+AAAAAAAANOBAAAAA	Houses appear again scientific tests. Naked pieces shall not believe experiences. Coming, good measu	Books                                             	fiction                                           	1.86	2113.81	0.69540376735462230
+AAAAAAAAPFODAAAA	Rates should not turn exactly enormous flowers. Happy practitioners should believe suddenly natural organisms; al	Books                                             	fiction                                           	2.51	3437.58	1.13089922111396129
+AAAAAAAAPJKAAAAA	Constitutional, good pupils might not begin below level devices. External savings fit hardly. Parents shall dry. Actually literary companies improve a	Books                                             	fiction                                           	4.22	439.55	0.14460368999140142
+AAAAAAAAPPHDAAAA	Eyes come no longer. Commercia	Books                                             	fiction                                           	0.20	5344.48	1.75823348671424196
+AAAAAAAAAAJBAAAA	Famous authorities will demand at last growing teachers. Over immediate schools should go only so 	Books                                             	history                                           	2.40	4151.41	1.32043953348399043
+AAAAAAAAAFCDAAAA	Civil, english books could search either young institutions; incidentally major difficulties could not clinch little nevertheless old papers. Special subjects sail late workers. Low, national part	Books                                             	history                                           	1.01	1167.75	0.37142639855517278
+AAAAAAAAALGDAAAA	At first close areas may 	Books                                             	history                                           	0.09	9795.83	3.11576095719008192
+AAAAAAAAALPCAAAA	Onwards current types may allow; other sectors might carry nowadays marginal conditions. Minutes add well faces. Urban, possible women could not oppose never markets; galleries must favour gently vehe	Books                                             	history                                           	59.17	3685.92	1.17238106697707767
+AAAAAAAAANBDAAAA	Weapons wi	Books                                             	history                                           	3.85	1690.46	0.53768483810882242
+AAAAAAAAAPFDAAAA	Odd, only premises present previously obvious strengths. Widely different times should not ke	Books                                             	history                                           	1.88	8472.00	2.69469017217677053
+AAAAAAAABAJAAAAA	All female calls see ever fresh, widespread lawyers. Results could not want initially 	Books                                             	history                                           	1.77	439.46	0.13977910092832903
+AAAAAAAABHDCAAAA	Logical suggestions should evacuate in common equivalent, distinctive women. Fruits know formal pensioners	Books                                             	history                                           	1.85	10800.83	3.43542144149575407
+AAAAAAAABJCEAAAA	Regular, elderly circumstances should not stop sole, different sites. New group	Books                                             	history                                           	2.98	383.28	0.12190992082057514
+AAAAAAAABKDCAAAA	Also quiet users fall. Other, current sources would c	Books                                             	history                                           	0.43	10191.59	3.24164039327845288
+AAAAAAAABLLCAAAA	Similarly legislative games could expect others. Central, special miles get all to a problems. Rights pass different, glad eyes. Most local tanks	Books                                             	history                                           	9.29	367.56	0.11690985831979388
+AAAAAAAACAJDAAAA	Military areas used to help sometimes sooner certain children. Unlikely proceedings say; wages recognize now managerial years. New events stay full, royal communities	Books                                             	history                                           	6.86	9156.39	2.91237419093692870
+AAAAAAAADANAAAAA	Wildly sexual powers achieve local, comfortable songs; artistic, very shares might start. Miners used to sleep very italian partners. Book	Books                                             	history                                           	4.58	3997.52	1.27149172061851791
+AAAAAAAADJPBAAAA	Architects influence around enough visual interests. Days think already other issues. Regardless lucky rules mean to a shoulders. Women accept only.	Books                                             	history                                           	1.44	5541.90	1.76271287360557656
+AAAAAAAADNIBAAAA	Never possible applications will not contribute still bad, golden resources; force	Books                                             	history                                           	5.60	5573.65	1.77281160034856670
+AAAAAAAAEIMBAAAA	Armed profits forget now s	Books                                             	history                                           	9.04	494.12	0.15716481443295395
+AAAAAAAAEJAAAAAA	Hundreds go over electronic fa	Books                                             	history                                           	7.68	898.62	0.28582418348931652
+AAAAAAAAEJCCAAAA	In short new acres marry perfectly for a c	Books                                             	history                                           	1.58	186.93	0.05945685008085502
+AAAAAAAAEMGBAAAA	Hostile, certain contents would carry; others can get great, prime rates. Expensive, national shows produc	Books                                             	history                                           	1.95	3076.78	0.97863182577314023
+AAAAAAAAFLAAAAAA	Origins help still already common hands. Probably official increases could inform more recent, 	Books                                             	history                                           	34.26	5002.56	1.59116492772953555
+AAAAAAAAGBGEAAAA	Safe films go behind amo	Books                                             	history                                           	4.48	6872.36	2.18589246360490448
+AAAAAAAAGEBCAAAA	Ancient, yellow sets anger other men. Beautiful, vari	Books                                             	history                                           	3.24	2349.53	0.74731532108527947
+AAAAAAAAGEDDAAAA	Wheels shall include tables; only central days shall see lovely, jewish artists. Genes ought to climb therefore;	Books                                             	history                                           	2.02	6800.22	2.16294688416429633
+AAAAAAAAGMEAAAAA	Branches attend fair true banks. Rigid cigarettes like by a places. Stations shall not let thus. Kids hold into a achievements. Streets used to set twice actual, wonderful areas; surroundings r	Books                                             	history                                           	6.21	12377.05	3.93676994753783023
+AAAAAAAAHIICAAAA	Then sp	Books                                             	history                                           	1.91	8909.36	2.83380132582446085
+AAAAAAAAHKBEAAAA	Parliamentary pieces shine never tragic patterns. Great human eyes would not get groups. Plant	Books                                             	history                                           	6.03	953.70	0.30334348645006918
+AAAAAAAAHOEDAAAA	Tropical, different relations would not work eyes. Level customs might aff	Books                                             	history                                           	0.31	10335.72	3.28748384163962355
+AAAAAAAAHONAAAAA	Ready, imperial shops see impossible assumptions. Clinical holders ask. Other rules would not avoid at a panels. Unusual, particular rights cannot go yet golden substance	Books                                             	history                                           	4.56	2768.79	0.88066940531413131
+AAAAAAAAIAPCAAAA	Very valid police should not like away pictures. New, special principles survive from a	Books                                             	history                                           	4.76	8944.55	2.84499421382716393
+AAAAAAAAIEHBAAAA	Fully classical offices cannot go different, new roads; proceedings mean asian, only groups. Earlier academic affairs 	Books                                             	history                                           	3.37	10650.60	3.38763776531939474
+AAAAAAAAIHCCAAAA	Big, special things find however happy agencies. Current firms reduce especially at a eyes. Imports want reasons. Little controversial de	Books                                             	history                                           	4.36	1262.68	0.40162079634137920
+AAAAAAAAIJBEAAAA	Additional, human standards should not dream also silly forms. More independent friends may believ	Books                                             	history                                           	4.39	5255.61	1.67165257504650106
+AAAAAAAAIKGCAAAA	Confidential, full terms make incorrectly elderly, real methods; teeth slip much today unknown conditions. Years shall not undermine occasionally local, industrial lips; restrictions beat most things	Books                                             	history                                           	1.38	7182.03	2.28438924188842437
+AAAAAAAAIKOAAAAA	Independently mean findings must not take today police. White, yellow features try even grateful examples. Sweet 	Books                                             	history                                           	2.06	4957.80	1.57692810854792173
+AAAAAAAAINEEAAAA	Films cope 	Books                                             	history                                           	1.22	14315.87	4.55345068403685835
+AAAAAAAAINKAAAAA	Hours used to use always local, upper budgets. Only royal strategies confuse already key windows. Open, short habits broadcast just. Working-class lights will display previous measures. Soviet scho	Books                                             	history                                           	0.75	4671.20	1.48576920824741861
+AAAAAAAAJENCAAAA	Opponents bring also waiting, other things. There massive characters contact	Books                                             	history                                           	58.48	1594.66	0.50721371930635138
+AAAAAAAAJGICAAAA	Boys form so go	Books                                             	history                                           	4.24	12750.46	4.05554051613940340
+AAAAAAAAKADBAAAA	Tomorrow soft actors could not go for the needs. Considerable times used to allow following visitors; months must not avoid about economic farmers. Tears start at firs	Books                                             	history                                           	1.76	10852.02	3.45170345163665691
+AAAAAAAAKCACAAAA	Years would land in a trees. Areas establish above civil tests. Within so-called thanks like just. Ill acts prevent. Most 	Books                                             	history                                           	8.83	11890.89	3.78213697136863066
+AAAAAAAAKDBAAAAA	Allegedly great plans respond able, cheap facts. Today local banks might allow at least tr	Books                                             	history                                           	7.32	75.87	0.02413198103907597
+AAAAAAAAKMCEAAAA	Effects shall not come in short southern firms. High, afraid years smell anyway governors. Wages can think deep, educational participants. Quietly probable 	Books                                             	history                                           	88.42	7756.02	2.46695831789500422
+AAAAAAAAKOPBAAAA	Particularly particular contents destroy feet. Essential, fatal wo	Books                                             	history                                           	2.76	7308.24	2.32453287345481131
+AAAAAAAALGCAAAAA	Popular, current dogs shall not nominate respectively. More labour connections take further feet; holy, neighbouring months can leave. Attempts should investigate 	Books                                             	history                                           	0.64	2234.94	0.71086766447176010
+AAAAAAAALKABAAAA	Green discussions might offer most. Grateful feet ought to go still 	Books                                             	history                                           	47.36	12676.50	4.03201604905557503
+AAAAAAAAMEPDAAAA	Major, grateful charts talk system	Books                                             	history                                           	3.78	1685.71	0.53617400497404436
+AAAAAAAAMMEDAAAA	Forward slight interests provide on a cases; successful areas must come internal, present en	Books                                             	history                                           	4.36	1180.89	0.37560584011116933
+AAAAAAAANCDEAAAA	Soon sure forests cope; guilty, e	Books                                             	history                                           	6.82	3323.19	1.05700748740275284
+AAAAAAAANHACAAAA	Grey words need. English, swiss measures help separat	Books                                             	history                                           	3.59	4100.58	1.30427202859119708
+AAAAAAAANHIAAAAA	Parliamentary, monetary charges shall evaluate by a observations. Urgent, suitable problems give just at the rises; earlier big others stay always guilty terms. S	Books                                             	history                                           	1.16	6557.12	2.08562403467702379
+AAAAAAAANJJDAAAA	Lovely years help. Possible, good years must imagine even necessar	Books                                             	history                                           	35.72	11655.58	3.70729188822239413
+AAAAAAAAOCBCAAAA	Other, current movements would get in a products.	Books                                             	history                                           	8.87	18347.84	5.83589992075918761
+AAAAAAAAOPNCAAAA	Legal, independent teachers cut. Perhaps common wives might carry already states. Courts rally regions. Besides financial ways could not suffer notably political	Books                                             	history                                           	3.66	1239.86	0.39436243589177180
+AAAAAAAAPINBAAAA	Major, front faces wonder very desirable teachers. Prospective, national plans take industrial, separate locations. Capitalist children save head, economic features. Techniques l	Books                                             	history                                           	1.92	1668.04	0.53055370571267001
+AAAAAAAAPONBAAAA	Trends work to a co	Books                                             	history                                           	4.91	3816.03	1.21376517206465081
+AAAAAAAAAFFDAAAA	Alone sole services keep only; stairs shall eliminate for the woods. Methods must need yet. Other students can	Books                                             	home repair                                       	2.39	1754.10	0.73033351855711644
+AAAAAAAAAFMCAAAA	Alive reforms remember to a rocks. Neighbours could find together with a maps. So anxious circum	Books                                             	home repair                                       	2.84	819.94	0.34138855550180837
+AAAAAAAAAJFDAAAA	Refugees can help as natural publications. Serious, active feet carry alone as well sharp coins. New reasons pay absolutely cautious changes. Practical memb	Books                                             	home repair                                       	4.33	4572.72	1.90388842538994214
+AAAAAAAAAPKDAAAA	Above northern firms can restore either in a tories. Then natural children used to supply publicly chosen things; extra, available circumstances must pay 	Books                                             	home repair                                       	0.40	2992.66	1.24601784826699738
+AAAAAAAABHNBAAAA	Here different	Books                                             	home repair                                       	4.50	3368.22	1.40238524820389416
+AAAAAAAABJMDAAAA	Chief 	Books                                             	home repair                                       	4.04	3930.58	1.63652831729675090
+AAAAAAAACALDAAAA	Black, relative workers make soft, important cases. Previous p	Books                                             	home repair                                       	9.53	10606.18	4.41596759469250173
+AAAAAAAACDEEAAAA	Taxes disregard earlier for the aims. In part heavy years continue less settings. Breasts accomplish. Weak, appropriate duties mu	Books                                             	home repair                                       	9.96	6044.52	2.51668408847207200
+AAAAAAAACHACAAAA	Members defeat at all new, only bills; original abilities convince; already exciting systems lead shapes. New, real travellers should pursue again short vehicles. Important, only	Books                                             	home repair                                       	80.60	1171.18	0.48763012956144099
+AAAAAAAACHNDAAAA	Professional managers take at least at a applicants. Vulnerable areas must regulate more with a employees. 	Books                                             	home repair                                       	0.38	2026.22	0.84363284987788637
+AAAAAAAACIKAAAAA	Completely foreign parties cope with the terms. Children would take terribly visual, total things. Yet good songs will work all right m	Books                                             	home repair                                       	2.78	1190.62	0.49572412853570149
+AAAAAAAACLLBAAAA	Activities bring brief, yellow practitioners. Polish representatives will not prevent for the examples. Annual, ashamed standards use	Books                                             	home repair                                       	7.44	5309.96	2.21084417661338922
+AAAAAAAADABDAAAA	Perhaps european sectors may say practices. Just true years can tell interesting relations. Then private years could not persuade before quickly continuous levels; pale, constitu	Books                                             	home repair                                       	4.28	61.23	0.02549359862108901
+AAAAAAAADAGBAAAA	Chief levels must attack about for a parties. Branches complete really. Just following aims shall not arrive together experienced friends. Actually 	Books                                             	home repair                                       	7.44	7424.19	3.09112069160056914
+AAAAAAAADHHCAAAA	States should not hold services. Western manufacturers could not mean even large exercises. Facilities maint	Books                                             	home repair                                       	7.52	5601.60	2.33227081554617381
+AAAAAAAAEECEAAAA	Free, particular nurses get either. Great, evolutionary million	Books                                             	home repair                                       	0.89	1230.96	0.51252000912323588
+AAAAAAAAFKGBAAAA	Military, inc computers ought to maintain entirely even burning sections. Able, outer papers may not cause thus useless, pretty walls. Always im	Books                                             	home repair                                       	73.73	6564.64	2.73324019683073308
+AAAAAAAAGGDBAAAA	Diverse, remaining bits ought to listen along a relationships. Distant stages jail relatively. Short, false applications could appear p	Books                                             	home repair                                       	1.52	1742.72	0.72559536483658741
+AAAAAAAAGKKCAAAA	Houses help general, new attitudes. All central shoes cannot watch. Effects boost to a details. Figures get intently normal, common leaders. Ne	Books                                             	home repair                                       	1.01	19637.84	8.17637123542653418
+AAAAAAAAGNNAAAAA	Even able courses should not vote. Appropriate findings might wait legal things. Sheer, interested levels inform in a meetings.	Books                                             	home repair                                       	2.99	3714.58	1.54659499536052312
+AAAAAAAAHAAAAAAA	Tomorrow different years mean highly in a circumstances. Financial fi	Books                                             	home repair                                       	0.35	7727.05	3.21721886697837445
+AAAAAAAAHEHBAAAA	Open, l	Books                                             	home repair                                       	6.35	1419.57	0.59104928620838367
+AAAAAAAAHGECAAAA	Expenses look away both complete manufacturers. Male advantages use here books. Right rich falls used to say; simple visitors mind records. Conventional profits might arrange	Books                                             	home repair                                       	7.60	414.17	0.17244298123299750
+AAAAAAAAHGFBAAAA	European, local terms bring even everywhere working days; much nice choices grow merely major, black rates; final, other talks can know for example also industrial	Books                                             	home repair                                       	8.57	772.24	0.32152828024089140
+AAAAAAAAIAIDAAAA	Internal exhibitions shall die soon direct movies; services could follow at once social, outer sciences	Books                                             	home repair                                       	2.25	1729.95	0.72027847353507987
+AAAAAAAAIDEEAAAA	However broad boots may not obtain extraordinarily da	Books                                             	home repair                                       	2.68	2701.11	1.12462868155168622
+AAAAAAAAIDPDAAAA	Political, standard statements damage as elegant preferences. Tremendous girl	Books                                             	home repair                                       	4.06	16118.92	6.71124084085324406
+AAAAAAAAIHHBAAAA	British runs wish underneath appropriate pounds. Unable, complex results must not look at the origins. Extra employees find so early thanks. Competent	Books                                             	home repair                                       	5.60	15.48	0.00644522140542966
+AAAAAAAAIJLDAAAA	New, immediate seconds may not give also lines; relevant groups break little golden, political eyebrows. Able cattle doub	Books                                             	home repair                                       	3.96	1518.63	0.63229370690747035
+AAAAAAAAJJHAAAAA	Vast, delicate tem	Books                                             	home repair                                       	0.83	336.52	0.14011278471286747
+AAAAAAAAKDEAAAAA	Corporate stones relieve together early things; forward line	Books                                             	home repair                                       	8.20	7293.74	3.03680679416269454
+AAAAAAAAKDLBAAAA	Words should agree completely level times. Very gentle hours would not interpret. Gr	Books                                             	home repair                                       	8.23	3906.80	1.62662732472432730
+AAAAAAAAKHGAAAAA	However great occupations find very academic homes. Surprised writings suit as free, short shows. Originally possible preparations should accept as yet similar children. Hours re	Books                                             	home repair                                       	1.86	2705.71	1.12654392822255033
+AAAAAAAAKNGDAAAA	Members may not cut probably area	Books                                             	home repair                                       	0.87	8868.24	3.69236242096172529
+AAAAAAAAKPJAAAAA	Similar seats would not see now light soldiers. Rather possible countries take white, proposed boys. Guilty, famous models would not invest often like a fears. Plainly new classes prevent little	Books                                             	home repair                                       	3.02	3962.44	1.64979348228234450
+AAAAAAAALBABAAAA	External hours will not begin never old, empty word	Books                                             	home repair                                       	1.92	275.50	0.11470662126588312
+AAAAAAAALBCCAAAA	Sections will not kick for a systems. Political, lacking arms used to say other authorities. Savi	Books                                             	home repair                                       	53.64	8876.73	3.69589730014338536
+AAAAAAAALBNDAAAA	Planes play sometimes economic, wonderful comments. Responsible, primary costs can bring stra	Books                                             	home repair                                       	8.00	3496.76	1.45590390191538823
+AAAAAAAALEBAAAAA	Of course british lawyers shall describe at least extremely active men. Proposals may gain. Also lexical differences attend bad teams; academic, major contexts could not hold less stead	Books                                             	home repair                                       	4.97	855.34	0.35612762770802348
+AAAAAAAALJLAAAAA	Political, local children will distinguish as necessarily new managers. Directly resulting questions 	Books                                             	home repair                                       	6.97	13643.34	5.68051337271024974
+AAAAAAAALJNBAAAA	Issues become at a materials; more complete others should apply seco	Books                                             	home repair                                       	3.96	2603.64	1.08404627002796343
+AAAAAAAALNICAAAA	Real earnings exceed there from a shoulders. Practical days shall not spend now systems. Ages might not sit much. Probably 	Books                                             	home repair                                       	0.86	1450.51	0.60393140185980444
+AAAAAAAALOPAAAAA	Scientific contracts transform only variable contacts; just important relations could tell generally on a values. Possible	Books                                             	home repair                                       	1.94	8305.21	3.45794039202767748
+AAAAAAAALPKBAAAA	Extraordinary, economic obligations intend multiple, public patients; again enthusiastic supporters should stop greatly labour, mad trus	Books                                             	home repair                                       	2.73	1640.87	0.68318930539582445
+AAAAAAAAMBGEAAAA	Remarkably political plans would locate separate problems. Sensible areas will not join home social 	Books                                             	home repair                                       	6.39	3591.09	1.49517894940726030
+AAAAAAAAMCFDAAAA	Hours might need etc with the holders. Early demands drive usually; at all other responsibilities see so equally italian issues. Simple, senior operations must t	Books                                             	home repair                                       	6.30	4254.02	1.77119513973681346
+AAAAAAAAMOEEAAAA	Spanish, unique colleagues put through a applications. Years will confront normally by no appearances; colleagues will not own still. Sympa	Books                                             	home repair                                       	2.68	5243.74	2.18327295171238458
+AAAAAAAANCICAAAA	British demands can travel easy conditions. Inevitably small pat	Books                                             	home repair                                       	0.78	3069.27	1.27791503249632335
+AAAAAAAANMFEAAAA	Able prices would leave mainly in a matters. Ostensibly necessary schools get far private sales. Laboratories question possibly rare sectors. Likely hands could respond up to good	Books                                             	home repair                                       	2.22	5893.46	2.45378905323278233
+AAAAAAAAOFKCAAAA	Systems cannot show. Global pains sha	Books                                             	home repair                                       	6.41	748.19	0.31151487101604752
+AAAAAAAAOGJAAAAA	Dark, fun calculations must not take away interested feet. High, local films could show normal, visual glasses. Concerned, indian chiefs stick at least. Cultural condition	Books                                             	home repair                                       	1.87	2172.50	0.90453769401136507
+AAAAAAAAOHGBAAAA	Sentences might treat in a persons. Prisoners look best heavy investigations. Western, emotio	Books                                             	home repair                                       	2.92	1731.95	0.72111118947893383
+AAAAAAAAOOOCAAAA	Japane	Books                                             	home repair                                       	8.75	326.81	0.13606994880545649
+AAAAAAAAOPDCAAAA	Democratic, sure places lose in a friends. Other, essential volunteers borrow other, other nurses; foreign hours get indeed enormous designers. Helpful, professional powers lower far from. C	Books                                             	home repair                                       	4.46	7443.09	3.09898985726998908
+AAAAAAAAPHADAAAA	Dutch, quick households ring fortunately small, automatic pubs; objectives st	Books                                             	home repair                                       	93.40	4131.30	1.72009968942193442
+AAAAAAAAPLACAAAA	Industrial, difficult children shall use crops; errors can reach frankly boards. Apparent, special arms may not see always other inter	Books                                             	home repair                                       	3.19	722.52	0.30082696187668193
+AAAAAAAAPMKDAAAA	Suddenly various forms must not involve then local, other economies; continuing, still others cannot know directly only comprehensive products. Odd books go enough southern cases	Books                                             	home repair                                       	7.64	10446.87	4.34963760618481448
+AAAAAAAAAAICAAAA	Rather little years should not reach more new magistrates. Political speakers may lower considerably gates. Kinds would not depend well. Provisions raise. Almost difficult pensions pick yet organi	Books                                             	mystery                                           	4.25	327.20	0.10733361870342104
+AAAAAAAAAANBAAAA	Royal plants find however workers. About genetic peasants come welsh, marine men. So federal eyes develop. Home old services 	Books                                             	mystery                                           	4.32	7859.96	2.57835559188307223
+AAAAAAAAADIAAAAA	White changes come much matters.	Books                                             	mystery                                           	3.16	3490.58	1.14503845591010823
+AAAAAAAAAEMAAAAA	Later other operations see; expected, honest animals show respons	Books                                             	mystery                                           	2.82	18416.84	6.04140000697406092
+AAAAAAAAAENDAAAA	Royal advantages succumb again english, new regulat	Books                                             	mystery                                           	0.58	3081.67	1.01090095583671001
+AAAAAAAAAFEBAAAA	Centra	Books                                             	mystery                                           	1.36	6619.98	2.17159660496416018
+AAAAAAAABEBBAAAA	Countries keep much french, addit	Books                                             	mystery                                           	4.87	25157.14	8.25246599152989476
+AAAAAAAABHODAAAA	Always silver months must capture only left mass miles. Characteristics should fall total ways. Courses might work in a spirits; key sources would live again up the records; thoughts can inspect ofte	Books                                             	mystery                                           	9.69	3901.52	1.27984187054942315
+AAAAAAAACDLBAAAA	Primary, single engineers seem new centuries. Close ladies date. Necessary, likely hands cannot retain generally inc prices. Opini	Books                                             	mystery                                           	1.81	10328.03	3.38797320897766992
+AAAAAAAACGMCAAAA	A	Books                                             	mystery                                           	0.11	6325.20	2.07489793711148765
+AAAAAAAACHLAAAAA	Hills may not die reforms. Better 	Books                                             	mystery                                           	5.64	2254.23	0.73947024232827876
+AAAAAAAACHLDAAAA	Only present circumstances cannot fall from a players. Sharp relations will blame late eyes. Closest different problems should not write i	Books                                             	mystery                                           	4.33	9175.56	3.00992071647421134
+AAAAAAAACJMAAAAA	Also strategic consultants proceed slightly eyes. Almost stran	Books                                             	mystery                                           	2.26	23865.71	7.82882951475068011
+AAAAAAAACMLBAAAA	Now top documents might mitigate usually ethnic sheets. Big times come partly high records. Social years can seek social, major r	Books                                             	mystery                                           	2.68	5730.79	1.87990962325604602
+AAAAAAAACNOAAAAA	Double workers ought to face with the objects. Satisfactory, other participants help politically urgent, 	Books                                             	mystery                                           	3.56	2094.56	0.68709261733324441
+AAAAAAAADBBCAAAA	National specialists go practical chapters. Enough right women stare again for example literary cameras. Most industrial cells shall improve possible, availab	Books                                             	mystery                                           	3.03	4124.34	1.35293501516891054
+AAAAAAAADEFBAAAA	Fortunes could meet emotional meetings. Beautiful women replace beautifully in the things; less previous year	Books                                             	mystery                                           	5.11	102.48	0.03361720429317417
+AAAAAAAADJEAAAAA	Available solicitors emerge. Further true weeks manufacture changes; families save up to right things. Gre	Books                                             	mystery                                           	3.50	2151.90	0.70590224354490139
+AAAAAAAAECCAAAAA	Present, regular adults cannot l	Books                                             	mystery                                           	7.59	522.99	0.17155993045752497
+AAAAAAAAEDPBAAAA	Especially simple sources absorb shortly accessible, new years; glad chapters restrict so southern districts. Modest, particular years could not discard only free men. Now black things could ge	Books                                             	mystery                                           	3.35	3104.40	1.01835723075458519
+AAAAAAAAEHBEAAAA	Days must appear on the police. Direct, late developments should serve always for the papers. Meetings take yesterday women. Medium periods 	Books                                             	mystery                                           	7.03	1997.98	0.65541082975874440
+AAAAAAAAEIDBAAAA	Sufficient, whole judges may not show even almost vo	Books                                             	mystery                                           	75.13	1924.56	0.63132637289687040
+AAAAAAAAEIDCAAAA	Words take here free goods. Efficient sales could not ask only. Please local women can talk less than useful permanent colleges. Always free members mus	Books                                             	mystery                                           	5.23	4082.90	1.33934117299571443
+AAAAAAAAEKFDAAAA	Regional, able services should transfer old, social preferences. Other courts might talk a li	Books                                             	mystery                                           	1.16	954.39	0.31307497663312349
+AAAAAAAAEMAEAAAA	Huge, difficult init	Books                                             	mystery                                           	34.65	621.18	0.20376985717051064
+AAAAAAAAENCDAAAA	Difficulties would offer changes. Payable pounds give electric, sure weeks. Tired houses shall not get together most important pools. Bones shall not give foreign, new troops. 	Books                                             	mystery                                           	4.33	12111.11	3.97288894503419799
+AAAAAAAAFMACAAAA	Very dead processes may enable drugs. Early units work long police. Easily difficult opportunities ought to seem extra, common eyes. Just quiet subjects must finance ch	Books                                             	mystery                                           	4.70	475.66	0.15603395193297449
+AAAAAAAAGADCAAAA	Also rich lines want noticeably often social difficulties. Animals go; sexual, central cats ought to tolerate. Groups sha	Books                                             	mystery                                           	3.23	150.35	0.04932032265299313
+AAAAAAAAGDFDAAAA	Also significant 	Books                                             	mystery                                           	4.93	1060.69	0.34794528124245618
+AAAAAAAAGGEEAAAA	Fine, sure centuries would not form now angry, dead insects; customers cannot pray totally as male laws. Unique procedures reinforce rarely also	Books                                             	mystery                                           	2.81	5986.79	1.96388702664258571
+AAAAAAAAGGOCAAAA	Intermediate, subj	Books                                             	mystery                                           	9.70	10978.67	3.60140702827227219
+AAAAAAAAGHMBAAAA	Hot eyes must invest patently common laws. Whole arts discourage small studies. Policies could need. Reasons hope really independent, international departments. Effective, afraid attitudes	Books                                             	mystery                                           	0.97	251.85	0.08261605094882821
+AAAAAAAAGPFDAAAA	Prices find under way around the languages. Civil, effective products should last really at a hundreds. Main, capable groups will contribute; only indian regulations take now in a feet; total	Books                                             	mystery                                           	2.73	625.40	0.20515417217946063
+AAAAAAAAHLEDAAAA	Advances accept. Lists must not act also old comments. Objectives shall know as to the months; live years can pay possible, inc attempts. Russian years see further pro	Books                                             	mystery                                           	1.42	15186.66	4.98178231607119854
+AAAAAAAAIABCAAAA	Clean systems can realise often s	Books                                             	mystery                                           	2.73	3145.42	1.03181329750035026
+AAAAAAAAIJCAAAAA	Distinguished, huge levels return pretty characters. Months cannot ask right. Overseas studies c	Books                                             	mystery                                           	6.54	1642.06	0.53865599611289594
+AAAAAAAAIKNBAAAA	Voluntary, clear techniques go. Before domestic students ought to live supreme, easy journalists; hands will run overseas such as the skills. Technical, official doctors would	Books                                             	mystery                                           	5.72	1966.05	0.64493661690666545
+AAAAAAAAIPDAAAAA	Good, local rules follow normally high lines. Whole, male activities know again. 	Books                                             	mystery                                           	4.01	5929.90	1.94522501696031914
+AAAAAAAAJABDAAAA	Years will appear original	Books                                             	mystery                                           	4.79	1653.40	0.54237593265353407
+AAAAAAAAJBJDAAAA	Problems eat very in a persons; dead ideas happen british things. Short bags should test usually to a others. Also inner visitors expose nevertheless coming, peaceful me	Books                                             	mystery                                           	4.72	5511.42	1.80794820536188504
+AAAAAAAAJOBEAAAA	Expensive rates take as at once white careers. Parts drive all weeks. Therefore other years s	Books                                             	mystery                                           	0.55	181.72	0.05961083493516403
+AAAAAAAAKFDCAAAA	Furthermore little classes say spots. Like days used to provide costs. Friends	Books                                             	mystery                                           	4.03	13223.74	4.33787245413562633
+AAAAAAAAKIABAAAA	Years might give also. Ultimately private stars should make 	Books                                             	mystery                                           	2.78	1284.36	0.42131725708412545
+AAAAAAAAKIKDAAAA	Good, low facilities suggest too thereafter asian senses. Far holidays defend delicate members. Cautious reports treat on a talks	Books                                             	mystery                                           	0.25	5386.71	1.76703874451682502
+AAAAAAAAKNLDAAAA	Strange, necessary weeks hope all. Dead sons know too. Heavy, social waters used to move pupils. Heels provide. Eastern trees used to allow currently bad children. Articles would not clear	Books                                             	mystery                                           	4.09	5477.40	1.79678839573997066
+AAAAAAAALFFBAAAA	Bitter, nice students like general books; maximum, holy members draw indeed sure, strong lines; forests must not adapt opposite, r	Books                                             	mystery                                           	6.38	2322.45	0.76184890818386367
+AAAAAAAALPOAAAAA	Everyday, low cases could contribute again through a developments. Useful, unable answers might not assign local da	Books                                             	mystery                                           	1.87	8562.04	2.80866362067065732
+AAAAAAAAMJLAAAAA	Free, peaceful years should not help ahead animals. Then helpful others 	Books                                             	mystery                                           	27.03	92.46	0.03033027623874789
+AAAAAAAANIHDAAAA	Howev	Books                                             	mystery                                           	3.41	6376.36	2.09168029631951644
+AAAAAAAANPECAAAA	Sorry theories decide there wages.	Books                                             	mystery                                           	2.59	4969.90	1.63030975426079530
+AAAAAAAAOEEDAAAA	Other courses discuss full leaves. Connections excuse; objective, international sessions go. All expensive surve	Books                                             	mystery                                           	3.01	1617.54	0.53061253544477894
+AAAAAAAAOEHBAAAA	Banks will employ of course real, dead resources. Sisters shall not go short effects. Hopes run c	Books                                             	mystery                                           	3.63	4915.26	1.61238582722548074
+AAAAAAAAOEMCAAAA	Seconds preve	Books                                             	mystery                                           	4.51	2037.80	0.66847325242613507
+AAAAAAAAOGLDAAAA	Right developments would not seek variables; numbers like impatiently 	Books                                             	mystery                                           	3.84	11928.22	3.91289430712261892
+AAAAAAAAOIAAAAAA	Limits ought to eat less; actual costs would smash more main rules; magnetic, constitutional expressions can head years. Quickly western children may not wonder also useless, other millions; comm	Books                                             	mystery                                           	10.39	6043.00	1.98232597134710679
+AAAAAAAAOOGDAAAA	British, quiet residents trace particularly. Years should reduce now libraries. Special, general figures gain	Books                                             	mystery                                           	2.22	6385.64	2.09472447719227850
+AAAAAAAAPNKCAAAA	Most small ministers appear agencies. Industries review so much as solicitors. Far from distant children hear still terms. Particular, available days learn already long-t	Books                                             	mystery                                           	3.79	3704.73	1.21528752206334055
+AAAAAAAAPPEDAAAA	Sizes could not continue home; obligations will not lack notably current buildings. Measures burn there then useful thousands. Historic,	Books                                             	mystery                                           	7.35	5443.06	1.78552361436382311
+AAAAAAAAAAABAAAA	Inches c	Books                                             	parenting                                         	0.16	4582.16	1.47127016656624148
+AAAAAAAAAAFCAAAA	Certain signs prepare societies. Economic reasons can i	Books                                             	parenting                                         	0.98	1989.28	0.63873114796229133
+AAAAAAAAADLCAAAA	Golden dogs could hear only available feet. Big, serious patterns used to use here with a days; otherwise long reasons should not trave	Books                                             	parenting                                         	1.58	566.43	0.18187308178852684
+AAAAAAAABCEDAAAA	Luckily economic c	Books                                             	parenting                                         	9.18	122.92	0.03946796464425564
+AAAAAAAABEMCAAAA	Men become most so living studies; private nurses come frequently in a feet. Points will 	Books                                             	parenting                                         	1.38	4878.48	1.56641454732922415
+AAAAAAAACCOCAAAA	Other changes mean. Miles form. Local, illegal authorities take again inside the figures. Players would love properly	Books                                             	parenting                                         	14.38	2483.90	0.79754700113786669
+AAAAAAAACPIAAAAA	Popular circumstances should not take relations. Secret questions should get after the players. Automatic methods cope please in a effects; unli	Books                                             	parenting                                         	5.60	9646.64	3.09740682115084758
+AAAAAAAADLKBAAAA	Original, able troops reduce jointly. Crowds move american feet. Cities move. Legs transfer loudly so central germans. Households could c	Books                                             	parenting                                         	4.02	877.39	0.28171817034838474
+AAAAAAAAEFBAAAAA	Typical, right programmes tell against a reforms. Outside friends can inhibit again either military stairs. International men must launch legall	Books                                             	parenting                                         	65.75	4078.44	1.30953242534752647
+AAAAAAAAEKGDAAAA	Favorite, small son	Books                                             	parenting                                         	1.77	4476.61	1.43737947613180297
+AAAAAAAAELEBAAAA	Improved loans read years. Now constant tears perform now local negotiations. Specifically concerned problems ought to know more than previous steep plants. Cont	Books                                             	parenting                                         	0.48	5231.60	1.67979664686696862
+AAAAAAAAGCJDAAAA	So plain prisoners make improvements. Contemporary roots will resume in the computers. Firms accept modern, present names. Essential, collective sons cannot examine in the d	Books                                             	parenting                                         	5.38	18382.40	5.90234228178136019
+AAAAAAAAGEOCAAAA	Soft friends could make clean, brave feet. Rapid standards should not spread problems. Careers use quantities; british, other visitors should pursue wide, sudden sh	Books                                             	parenting                                         	4.17	7509.00	2.41103926548743546
+AAAAAAAAHCDBAAAA	Crazy years could cry even clergy. Other, philosophical sides might take years. Already senior hours cannot believe early strengths. Fields will not find little jewish councils. Events might not o	Books                                             	parenting                                         	1.37	8851.94	2.84223930160325602
+AAAAAAAAHIEDAAAA	Prime, flexible records say upwards at least easy schools. Here good investors can spend more at a cus	Books                                             	parenting                                         	7.33	6260.65	2.01021081069035995
+AAAAAAAAHMOBAAAA	Arms shall get thus famous, clear conditions. Royal languages might not understand in a films. Scientific, notable views would achieve like a years. Large, nervous students 	Books                                             	parenting                                         	2.05	2365.43	0.75950787185536616
+AAAAAAAAIBOAAAAA	Main contents set within a communities; rules date at	Books                                             	parenting                                         	1.39	1973.40	0.63363229278371356
+AAAAAAAAICADAAAA	Leaders restructure so. Years used to go from a years. Shoulders supply thus original tracks. Securely necessary	Books                                             	parenting                                         	2.01	2314.86	0.74327052258706151
+AAAAAAAAICBDAAAA	Faces may occur existing houses. Ruling, annual arguments allow all but for a elections. Future, spanish subjects take. Then prim	Books                                             	parenting                                         	8.01	13033.96	4.18502987678687100
+AAAAAAAAIIKDAAAA	High fields shall join then. Diffi	Books                                             	parenting                                         	1.11	3833.50	1.23088547399734770
+AAAAAAAAINOCAAAA	Narrow, 	Books                                             	parenting                                         	7.17	950.12	0.30507079863163167
+AAAAAAAAJFNBAAAA	Very strong arrangements should not cover parliamentary, fundamental implications. Parents renew then; major, basic structures settle; only long-te	Books                                             	parenting                                         	7.59	3460.43	1.11109769682656629
+AAAAAAAAJLLAAAAA	Pretty eastern facts should not join. Too labour things mean in particular. Closer intensive problems	Books                                             	parenting                                         	1.18	11548.91	3.70820022420834975
+AAAAAAAAJNFBAAAA	New friends must not gather by a blocks. Empty opportunities ought to remind else single families. Early years should not use suddenly abou	Books                                             	parenting                                         	4.28	11681.79	3.75086621137015165
+AAAAAAAAKBJDAAAA	Source	Books                                             	parenting                                         	8.78	5480.98	1.75986922271292103
+AAAAAAAAKDPAAAAA	Good countries need once again. Most economic patients appear there only real trees. Apparently jewish policies	Books                                             	parenting                                         	9.76	3680.94	1.18190050258400862
+AAAAAAAAKGNCAAAA	Small, true kids can go genuine objectives. Scottish games give ever. Scientific, similar trees remark. Boot	Books                                             	parenting                                         	8.58	10853.90	3.48504182763005404
+AAAAAAAAKHDDAAAA	Widespread lands get curious, certain reasons; issues ought to accept sales. Easy, other others might bomb large payments. Econo	Books                                             	parenting                                         	4.78	8024.99	2.57671673926541680
+AAAAAAAAKHJAAAAA	Forces can measure now groups. Resources form rat	Books                                             	parenting                                         	4.43	6742.48	2.16491996627563242
+AAAAAAAAKNFAAAAA	Equal voices build. High streets would harm simply individual, black methods. Substantial rooms land as current savings. Again very opportunit	Books                                             	parenting                                         	7.81	26.70	0.00857301217053063
+AAAAAAAALEICAAAA	Overall, high heads cannot see explicit, bad bodies; opportunities can accommodate little leaders. Light times u	Books                                             	parenting                                         	6.61	13341.53	4.28378648177900984
+AAAAAAAAMABDAAAA	Meanwhile thorough roads put also more other trees. Never religious costs want just especially direct nights. Young, excellent aud	Books                                             	parenting                                         	2.67	3546.05	1.13858913135993082
+AAAAAAAAMAFCAAAA	Common circles may win children. Tiny things must need as beside a words. Permanent yards remain fully. Slight, general ways avoid new, possible arts; therefore educational conditions ou	Books                                             	parenting                                         	4.26	9853.55	3.16384284917348778
+AAAAAAAAMCFBAAAA	Sites will not manage most generally immense woods. Fine employers avoid in a men; reasons ought to think here; only corresponding areas	Books                                             	parenting                                         	58.45	12923.27	4.14948880123795580
+AAAAAAAAMGCDAAAA	Records face long lips. Main researchers will know unequivocally ameri	Books                                             	parenting                                         	1.24	16478.74	5.29110256835243338
+AAAAAAAAMLACAAAA	Corners would not descend often plain new activities. Just local trusts think 	Books                                             	parenting                                         	8.15	9940.76	3.19184481139790637
+AAAAAAAAMMJBAAAA	Open, large roads might tell friends. Used, old arms will drop as good as natural others. Sad programmes participate	Books                                             	parenting                                         	4.27	2597.90	0.83415087332664917
+AAAAAAAAMNKDAAAA	Days could meet just. Folk might alter possibly tories; serious, basic things wait suffici	Books                                             	parenting                                         	5.54	8776.83	2.81812248721641872
+AAAAAAAAMPBEAAAA	Stations may not reme	Books                                             	parenting                                         	0.88	3316.92	1.06501855912645951
+AAAAAAAAMPNAAAAA	Economic, free bits post quite issues. Perhaps back sales used to affect d	Books                                             	parenting                                         	0.09	19263.00	6.18509114010979749
+AAAAAAAANDECAAAA	Genuine cities say. Practices prove together elsewhere simple	Books                                             	parenting                                         	1.52	1712.57	0.54988327538897554
+AAAAAAAANPFEAAAA	Se	Books                                             	parenting                                         	3.22	2194.90	0.70475297427332163
+AAAAAAAAOCAEAAAA	Partners will not locate. General, other losses cannot restrict else protective kilometres; children carry unusual, long groups. Yet true reservations differ never long-term	Books                                             	parenting                                         	1.02	6482.66	2.08149524634502309
+AAAAAAAAOFDDAAAA	Profits could not cling through a terms. Later democratic arms might not work all. Sometimes apparent arti	Books                                             	parenting                                         	6.57	0.00	0.00000000000000000
+AAAAAAAAOFICAAAA	Else emotional lives declare also c	Books                                             	parenting                                         	7.67	4780.68	1.53501227803042655
+AAAAAAAAOKJCAAAA	Previous floors keep complex computers.	Books                                             	parenting                                         	9.60	5787.26	1.85821162599344996
+AAAAAAAAOKKBAAAA	Lists used to miss little names. Prime roads should not help from the minutes; in order various exceptions help 	Books                                             	parenting                                         	1.19	4186.16	1.34411987369994445
+AAAAAAAAONDEAAAA	Theories look. Just young regions 	Books                                             	parenting                                         	45.83	1849.39	0.59381434374747746
+AAAAAAAAPCGCAAAA	Foreign, simple stocks may draw still; 	Books                                             	parenting                                         	2.55	18500.06	5.94012133091936148
+AAAAAAAAAKICAAAA	Careful, long customers may think about just professional meetings. Students could not drink. British, basic commentators remember espec	Books                                             	reference                                         	1.77	6207.69	2.15509748883540916
+AAAAAAAAALADAAAA	Bills emerge later in a yards. Ev	Books                                             	reference                                         	2.72	1496.80	0.51963772696266090
+AAAAAAAACEDCAAAA	Examples will talk there estimated, short initiatives. Benefits ought to prove too negative 	Books                                             	reference                                         	0.17	6141.90	2.13225745272044827
+AAAAAAAACGFDAAAA	Sorry services must not recall much main details. Sexual, major secrets will not go results. P	Books                                             	reference                                         	7.54	1423.78	0.49428768231887850
+AAAAAAAACGMDAAAA	Flexible, previous patterns must not manipulate essential, dull criteria. Much possible players will include firmly working, important duties. Far english busi	Books                                             	reference                                         	6.38	13587.29	4.71704201709145697
+AAAAAAAACLJBAAAA	Funds shall call more able countries. 	Books                                             	reference                                         	0.39	913.90	0.31727479868464444
+AAAAAAAACNAEAAAA	Indivi	Books                                             	reference                                         	3.76	2162.13	0.75061752979541556
+AAAAAAAACOOBAAAA	Hitherto certain kinds evade also by a months. Poor points might make even just selective passengers. Old, general qualities could overcome over; recent variables might s	Books                                             	reference                                         	56.16	1298.61	0.45083294268504882
+AAAAAAAADAJBAAAA	Difficult, rapid sizes say so; initial banks stress high single sports; prisoners used to think likely firms. Good, current services must take human, precise persons. Signals m	Books                                             	reference                                         	7.77	9585.22	3.32766029745927077
+AAAAAAAAEDAEAAAA	Royal, educational days can add black, long-term matters. Different executives should not remai	Books                                             	reference                                         	4.86	9194.30	3.19194625401709854
+AAAAAAAAEEDCAAAA	Classical, labour books make in addition finally significant suggestions. Ethical figures could sell as the levels. Regardless plain scholars set in a companie	Books                                             	reference                                         	80.47	2466.20	0.85618022597228374
+AAAAAAAAEHNCAAAA	Cruelly shared examples shall not investigate then in vit	Books                                             	reference                                         	0.28	610.19	0.21183708218555990
+AAAAAAAAEJABAAAA	Male, small legs allocate today to a programs. Video-taped circumstances afford short, royal changes. Planned, appropriate names can enter usual periods. Very consta	Books                                             	reference                                         	4.40	9663.14	3.35471145438399721
+AAAAAAAAELGBAAAA	Often other ideas must not understand possible, static groups. Late	Books                                             	reference                                         	8.13	705.22	0.24482824546272563
+AAAAAAAAENCEAAAA	Possible solutio	Books                                             	reference                                         	2.63	10773.86	3.74031542023913264
+AAAAAAAAFGKBAAAA	Still short documents ought to give more longer individual parties. Brief, expensive reforms should give now. As perfect sect	Books                                             	reference                                         	1.16	4401.20	1.52794599405936875
+AAAAAAAAGLODAAAA	Great speeches would draw too particular, full things. Available, real lives shall like long, supreme skills. Grim men would n	Books                                             	reference                                         	4.95	7141.72	2.47936073450278901
+AAAAAAAAGPGBAAAA	Ever only sides should not ensure clearly familiar, running points. Persons bear free, huge products. Organizations blame. Recent, parliamentary communities complain both perfect, l	Books                                             	reference                                         	5.85	4618.08	1.60323930660858167
+AAAAAAAAHJBCAAAA	Dead, blue homes should write more small objectives. Systems could underpin all so blue exchanges. Better adult arts make very governments. Quick managers talk and 	Books                                             	reference                                         	2.83	3913.25	1.35854645579678832
+AAAAAAAAHKEBAAAA	Damp, happy roads 	Books                                             	reference                                         	4.29	12407.36	4.30741070818241603
+AAAAAAAAIEPCAAAA	Italian pati	Books                                             	reference                                         	4.42	7902.99	2.74364762146488472
+AAAAAAAAIFNDAAAA	Classes used t	Books                                             	reference                                         	1.61	7530.59	2.61436308811313771
+AAAAAAAAIIECAAAA	Dangerous parents would not advise almost previous, important matters.	Books                                             	reference                                         	7.62	1064.34	0.36950241736734266
+AAAAAAAAIMACAAAA	Utterly free reasons control powers. Resources think too systematic sy	Books                                             	reference                                         	5.69	6131.92	2.12879273831966837
+AAAAAAAAINEAAAAA	Therefore secondary countries get eventually prospective lives. Directly complete wings see as g	Books                                             	reference                                         	6.19	4028.40	1.39852259439897325
+AAAAAAAAJEFEAAAA	At present pink police would not endorse yet bright rules. Photographs shall te	Books                                             	reference                                         	5.24	7033.41	2.44175920977849331
+AAAAAAAAJOGCAAAA	Equal, strong requirements use broadly remote pictures.	Books                                             	reference                                         	96.89	15194.39	5.27497212866393982
+AAAAAAAAKAMAAAAA	Relative, possible papers may change only current, tropical services; following procedures bring ever delicious questions; never convenient women may want secondary ch	Books                                             	reference                                         	3.67	2.16	0.00074987806670186
+AAAAAAAAKELAAAAA	Eyes alleviate yet; major women get that blue scientists. Wild interests suffer forthwith years. Women might complete in a commitments. Japanese, victorian	Books                                             	reference                                         	8.24	12242.59	4.25020820399238554
+AAAAAAAAKFHAAAAA	Clear points create however from a bases. Social, wrong rates contribute. More whole legs find now now unha	Books                                             	reference                                         	0.65	9377.23	3.25545328861977061
+AAAAAAAAKGCCAAAA	Glad, certain others ought to protect narrow, american friends; thi	Books                                             	reference                                         	9.25	2557.68	0.88793895076019410
+AAAAAAAAKMJBAAAA	Long son	Books                                             	reference                                         	6.53	13751.99	4.77422021967747397
+AAAAAAAAKNPDAAAA	Historical arguments can point much big times. Lines bri	Books                                             	reference                                         	7.40	4482.72	1.55624694776193163
+AAAAAAAALDIAAAAA	Types shall serve quite possible emotions; hard weekends appear months. There difficult colours form probably. Rules know however green manufac	Books                                             	reference                                         	4.01	2684.41	0.93193526899775290
+AAAAAAAALHBAAAAA	Also real addresses give in a advantages. Perfect, interested humans could fall never at a years. Sophisticated interp	Books                                             	reference                                         	8.60	936.71	0.32519364993532475
+AAAAAAAAMAMBAAAA	Much political attitudes must not understand more. Holy years shall not link large friends. Now occasional supporters may write also. Southern difficulties used 	Books                                             	reference                                         	3.32	7569.18	2.62776021524000108
+AAAAAAAAMDGCAAAA	Actions cannot go perhaps publications; huge, willing girls wo	Books                                             	reference                                         	9.60	2251.62	0.78168539469779966
+AAAAAAAAMHHAAAAA	Successful solutions find clearly as socialist problems; individual systems	Books                                             	reference                                         	9.20	2974.66	1.03270013421081565
+AAAAAAAAMKOCAAAA	Too nuclear windows ought to contemplate for example active, constitutional appeals. Again short partners clear to the issues. There political sheets end s	Books                                             	reference                                         	3.51	295.80	0.10269163524556059
+AAAAAAAAMLJDAAAA	Cities regard only. Operations used to make later; personal, written years used to interfere for a agreements. Obvious, sufficient protests tell. Issues pay effective own	Books                                             	reference                                         	2.70	445.16	0.15454431489490789
+AAAAAAAAMPPBAAAA	Here special fruits sti	Books                                             	reference                                         	2.31	6938.36	2.40876110318589515
+AAAAAAAANCABAAAA	Years decide pot	Books                                             	reference                                         	4.03	15341.75	5.32613047677004465
+AAAAAAAANINDAAAA	Structures drop home revolutionary, formal hands. Ears	Books                                             	reference                                         	3.42	1450.10	0.50342508542794934
+AAAAAAAAOAFDAAAA	Predominantly on	Books                                             	reference                                         	8.46	11177.59	3.88047665721577287
+AAAAAAAAOIPBAAAA	Really different purposes answ	Books                                             	reference                                         	81.85	4832.22	1.67758138494355241
+AAAAAAAAOKBDAAAA	Kinds play sooner; old causes would publish. Great,	Books                                             	reference                                         	2.90	463.44	0.16089050520014402
+AAAAAAAAOMPAAAAA	Relations preclude most primary records. Hardly common f	Books                                             	reference                                         	3.01	45.64	0.01584464581679305
+AAAAAAAAPDEAAAAA	Particularly natural children put hardly. Parties weep into a days. Heavy hands will not take mad, lonely children. Ye	Books                                             	reference                                         	4.55	1000.50	0.34733935450704318
+AAAAAAAAPEKCAAAA	Little, num	Books                                             	reference                                         	4.79	11088.98	3.84971429819241545
+AAAAAAAAPFFAAAAA	Democratic, fresh operations shall not explain fully decisions; contra	Books                                             	reference                                         	1.68	140.25	0.04868999946987787
+AAAAAAAAPOIDAAAA	As progressive minutes apply as firms. Involved,	Books                                             	reference                                         	4.35	18398.21	6.38722877109947712
+AAAAAAAAAAGCAAAA	Both gross guns ought t	Books                                             	romance                                           	22.07	2932.20	1.53691964340235494
+AAAAAAAAAAJCAAAA	Matters care too expressions; economic	Books                                             	romance                                           	5.87	4968.70	2.60435598941862117
+AAAAAAAAACNCAAAA	Internal, additional structures pretend trains. Useful payments should make fingers. 	Books                                             	romance                                           	0.64	4689.33	2.45792353570560163
+AAAAAAAAADEEAAAA	Following, very poli	Books                                             	romance                                           	1.59	7979.33	4.18238490491430082
+AAAAAAAAAGBDAAAA	Likely weapons see. Items improve half. Short, human resources depend white, local texts; fully permanent way	Books                                             	romance                                           	6.42	22088.52	11.57775059057560371
+AAAAAAAAALIAAAAA	Full days keep full, visible bottles. Big, domestic countr	Books                                             	romance                                           	4.62	11680.82	6.12252974184813303
+AAAAAAAAANADAAAA	Teachers arise clear often old services. Other minutes could cost by a attempts; open conscious goods detect yet disastrous stones; thus slight men tell for a countries. Capitalist bodies wou	Books                                             	romance                                           	0.25	4832.22	2.53281967097801228
+AAAAAAAABCBDAAAA	New, small beds will come instead in a stories. Female, other systems could not	Books                                             	romance                                           	4.36	9867.04	5.17183261654620160
+AAAAAAAACFGAAAAA	Part-time architects buy. Silently national skills understand free parts. Only european millions shall not attend at all other informal words. Empty, redundant holes contain again acceptable relatio	Books                                             	romance                                           	1.12	1104.46	0.57890535071010332
+AAAAAAAACFJAAAAA	Similar consumers will live once on a eyes. More likely teams pass particularly. Just other workshops 	Books                                             	romance                                           	3.59	1239.88	0.64988606761534406
+AAAAAAAACGKCAAAA	Future years can reform as before social suppliers; particular, judicial individuals resume vaguely remaining aff	Books                                             	romance                                           	0.52	6031.54	3.16144611757964666
+AAAAAAAACHJCAAAA	Crucial, different affairs could not forgo; public p	Books                                             	romance                                           	5.62	4775.42	2.50304781512054902
+AAAAAAAACIDBAAAA	For example new resources find perhaps necessary opportunities. Main systems move spontaneously necessary m	Books                                             	romance                                           	6.68	3560.08	1.86602444720136955
+AAAAAAAACIJDAAAA	Rather aware thanks may not work with a chi	Books                                             	romance                                           	2.35	2220.62	1.16394328440493058
+AAAAAAAAEGDDAAAA	Islands meet only for	Books                                             	romance                                           	6.79	2450.58	1.28447736843630822
+AAAAAAAAEIKBAAAA	Minutes will defend. Now new courses could know definitely international forces. There capital accounts should not lift more pro	Books                                             	romance                                           	72.49	1876.47	0.98355623874743093
+AAAAAAAAFLJDAAAA	More simple principl	Books                                             	romance                                           	6.44	6567.15	3.44218738018203917
+AAAAAAAAFOECAAAA	Late, dark looks would not make citizens. Safe, great curtains use as by the children. Signs would prove neither romantic moveme	Books                                             	romance                                           	4.68	2862.64	1.50045960302479959
+AAAAAAAAGBGBAAAA	Problems inherit. Sure edges must become enough revolutionary years. Systems burst however slowly strong issues; cultural site	Books                                             	romance                                           	1.60	775.70	0.40658501036327902
+AAAAAAAAGDNDAAAA	Possible, common bars cannot rid mainly ultimate years. Drugs could bring of course large, good rules. S	Books                                             	romance                                           	3.33	273.51	0.14336092069673900
+AAAAAAAAGFLAAAAA	Standard, geographical scales may hope equal, sure problems. Strong associati	Books                                             	romance                                           	7.58	4049.00	2.12229303462797052
+AAAAAAAAGKDDAAAA	Probably just results receive perfectly on the countries. Bold girls will pass religious years. Here public conditions ought to consider most sources. Different, able years go rarely ita	Books                                             	romance                                           	5.44	1710.73	0.89668322132109361
+AAAAAAAAGLMDAAAA	Even sure children build there imaginative novels. Real, quick members shall not exercise unlikely, vast times. Open regulations buy all catholic days. Domestic, palest	Books                                             	romance                                           	6.42	49.14	0.02575684853584057
+AAAAAAAAGOPDAAAA	Silver, political interviews might know in common families. Far possible houses shall insist in a places. Whole, political gardens would adopt eggs. Others might live even offi	Books                                             	romance                                           	6.13	5432.94	2.84768849581419762
+AAAAAAAAHHLAAAAA	Cultural, harsh conditions describe	Books                                             	romance                                           	4.72	1495.08	0.78364975801718601
+AAAAAAAAIAACAAAA	Distinctive hours work more federal, proper plants; crimes may ensure therefore; players work increasingly previous, genuine needs. Hostile, young schools will offer very new, implicit changes;	Books                                             	romance                                           	47.76	1911.06	1.00168666998175583
+AAAAAAAAIBFAAAAA	Particular bombs could illustrate suddenly planes. Western months expect just special, relevant readers. Able demands ought to achieve for a cars. Suitable counties must stud	Books                                             	romance                                           	0.88	1663.75	0.87205854195166361
+AAAAAAAAICDAAAAA	Levels tear only. Colleagues may not see hot forests. So effective residents must help completely in a hands. However professional classes ought to seem very; political	Books                                             	romance                                           	4.81	1069.40	0.56052856785160575
+AAAAAAAAIHBAAAAA	So only things know prac	Books                                             	romance                                           	2.71	3443.44	1.80488731221519852
+AAAAAAAAIHDEAAAA	Ways used to contain only double cigarettes. Intensely increased feelings 	Books                                             	romance                                           	76.83	18974.38	9.94546666099883214
+AAAAAAAAIJFDAAAA	Views balance quite other degrees. Slow passages promote due major animals. Sons would say. Possible, other schemes cannot restart either important, new 	Books                                             	romance                                           	3.75	745.80	0.39091285384676227
+AAAAAAAAIKODAAAA	Premier, good budgets could put high, slow members; traditions could not join however. Students laugh for a effects. Carefu	Books                                             	romance                                           	9.00	1184.75	0.62098954625228157
+AAAAAAAAILNCAAAA	Contacts remove basically blue, labour details. Full measures hold then families. G	Books                                             	romance                                           	66.85	845.81	0.44333333455635558
+AAAAAAAAIMDCAAAA	Subject children would not like sufficiently great levels. Yet busy hotels must not help behind	Books                                             	romance                                           	9.33	1361.15	0.71345002817581182
+AAAAAAAAJDMBAAAA	Large thoughts make	Books                                             	romance                                           	0.85	2228.59	1.16812077896802885
+AAAAAAAAJGBAAAAA	Specially clinical muscles can pass causal, following changes. Dishes could use at present areas; even c	Books                                             	romance                                           	5.00	276.00	0.14466606015246230
+AAAAAAAAJJPBAAAA	Teachers play apparent indians. Professional corners accept consequences; extensively necessary men will not know only economic clean stairs. Divisions could 	Books                                             	romance                                           	0.78	379.40	0.19886341747044999
+AAAAAAAAJLBBAAAA	Stages choose physically to a families	Books                                             	romance                                           	6.13	1969.70	1.03242296624023550
+AAAAAAAAKBEBAAAA	Illegal technologies might distinguish that on a change	Books                                             	romance                                           	2.73	1019.24	0.53423708387607130
+AAAAAAAAKBLBAAAA	As single women would get ideas. Rural classes may hear quite available, high sequen	Books                                             	romance                                           	1.38	894.27	0.46873375946573356
+AAAAAAAALCADAAAA	Senior fans cook frequently. Fin	Books                                             	romance                                           	4.36	5607.44	2.93915308819320006
+AAAAAAAALMLAAAAA	Mammals take at all. Profound weeks must know parts. Too low earnings can share directly new gaps. Equal block	Books                                             	romance                                           	4.99	179.00	0.09382327814235780
+AAAAAAAAMABAAAAA	Fine, real rows could think short, united others. Twice moving molecules list enough really vague assessments. Days put with a lines. Importa	Books                                             	romance                                           	4.85	950.33	0.49811774255322283
+AAAAAAAAMAOAAAAA	Associated words produce simply. Frantically tough forms take there across right years. Recent fears appear also fierce examples. Incredibly coastal te	Books                                             	romance                                           	2.28	99.82	0.05232089175514053
+AAAAAAAAMDNBAAAA	Historical, new notes should say levels; largely low prisons present at once enough useful winners. Yet worthwhile sons give different, social beaches. Minutes want guns. Industrial	Books                                             	romance                                           	65.28	3120.61	1.63567519555208473
+AAAAAAAAMHDAAAAA	Complete, foreign makers prevent conservative gardens; full prisoners would look so good goods. Then only cir	Books                                             	romance                                           	3.56	510.48	0.26756931299503245
+AAAAAAAAMLEEAAAA	Local, strong letters should not make also ba	Books                                             	romance                                           	6.39	3270.83	1.71441336785680534
+AAAAAAAANDMDAAAA	At all chemical branches make as existing things. Directly civil students must not afford much beautiful companies. Past police offer well perhaps chan	Books                                             	romance                                           	36.28	3753.37	1.96733786302336027
+AAAAAAAANIKAAAAA	Minor democrats can wonder impatiently real backs. Early,	Books                                             	romance                                           	2.77	1091.04	0.57187122561138576
+AAAAAAAANMGDAAAA	Surely local universities may know perhaps primitive computers. About bad sides will provide carefully about a workshops. National, sheer references ought to develop already also long-t	Books                                             	romance                                           	5.58	112.88	0.05916632199278965
+AAAAAAAANNDCAAAA	Financial things will die only pai	Books                                             	romance                                           	1.33	1782.43	0.93426494781722240
+AAAAAAAAODHCAAAA	Debts should not go into a eyes. Legal troops pursue wholly friends. Inc families will meet never; potatoes should give all various users. New women st	Books                                             	romance                                           	4.80	6935.94	3.63548954077488907
+AAAAAAAAPDEDAAAA	Also genuine men identify. Gradual, useful things used to see below patterns; superb, hidden ways would fail even huge yea	Books                                             	romance                                           	2.08	1555.12	0.81511986762426513
+AAAAAAAAPENCAAAA	Gains keep still. Possible, final clothes kill perhaps in the conclusions. Methods would proceed for a hopes. Other, particular ways find perhaps in a demands. Adverse, other men admit eviden	Books                                             	romance                                           	1.93	3352.42	1.75717896150839737
+AAAAAAAAPLHBAAAA	Racial minutes used to come enough teenag	Books                                             	romance                                           	3.47	4982.66	2.61167315680894137
+AAAAAAAAACCAAAAA	Then modern features should improve otherwise available qualifications. Personal purposes go with a years. Ministers remove big arts. Linear, poli	Books                                             	science                                           	4.66	527.85	0.17402980157734269
+AAAAAAAAAEJDAAAA	Organizations make enough horrible requirements. Grateful, only funds reassure anxiously yesterday great years. Extra	Books                                             	science                                           	5.13	36276.15	11.96008560479287668
+AAAAAAAAAGIDAAAA	Ac	Books                                             	science                                           	1.13	11382.07	3.75261794759766011
+AAAAAAAAAIBBAAAA	P	Books                                             	science                                           	7.15	115.77	0.03816885503193893
+AAAAAAAAAMOAAAAA	Confident views gain to the resources. Jobs could direct kings. Attitudes might not support as aware jobs. Happy accounts cannot test. Professional, joint interests will support in	Books                                             	science                                           	78.67	7479.68	2.46601728949894583
+AAAAAAAAAPLDAAAA	Continuous members shall look usually about careful supplies. More than negative sports become probably other leaves. L	Books                                             	science                                           	47.51	97.92	0.03228378927811575
+AAAAAAAABEGCAAAA	Obvious relationships put originally. Pounds give well central, british leaves. Differences ought to ask also central states. Tests grant for a chapters. Soon active heads should want 	Books                                             	science                                           	4.26	2414.14	0.79593124027645368
+AAAAAAAABEHBAAAA	Gently independent fears call now statutory sciences. Friendly, quiet needs stumble too. So famous cattle teach too only services; public forces collect pure friends. Arms might make im	Books                                             	science                                           	4.68	5668.22	1.86878696958743084
+AAAAAAAACAECAAAA	Later other words comfort historic, social birds. Large, english interests muster there ag	Books                                             	science                                           	1.74	2463.16	0.81209291664913785
+AAAAAAAACAOAAAAA	Ways create things. Popular opportunities regard eyes. Intact conditions show years. Variable banks could run legally. Sexual, mechanical dates shall not carry however fingers. Forms	Books                                             	science                                           	2.88	10151.52	3.34691107570034261
+AAAAAAAACDKBAAAA	Now educational levels lift perhaps men. Types use not. Very environments might go for sure at once common p	Books                                             	science                                           	71.85	6430.06	2.11996223535172516
+AAAAAAAADCEEAAAA	Little, able companies could not combine particles. Private kids participate in common; unable, only detectives introduce; very good skills go. Copies miss. Strategic m	Books                                             	science                                           	1.07	7269.76	2.39680759745174345
+AAAAAAAADNCBAAAA	Regular teachers serve together events. Other arms would not use. Dou	Books                                             	science                                           	3.59	8847.06	2.91683640493103230
+AAAAAAAAEEEBAAAA	Aware parts hang experienced, new groups. Handsome, perfect forms will grasp tonight in terms of the tears. Effective, economic subjects deny in the o	Books                                             	science                                           	3.18	38.60	0.01272624863291736
+AAAAAAAAENIAAAAA	Just essential errors permit never too bad applications. Ideas could buy men. Anxious wives would not pull royal, common towns. Adults	Books                                             	science                                           	3.22	10051.00	3.31377007796508735
+AAAAAAAAFCPAAAAA	Domestic copies cannot get additional victims. Pieces should not determine now british, gold depths. Local, available stocks punc	Books                                             	science                                           	3.99	3769.53	1.24279730593888526
+AAAAAAAAFPOAAAAA	Complaints can involve very vital adults. A little practical initiatives remain traditionally important months. Clear new transactions create perhaps new, personal princip	Books                                             	science                                           	1.15	3928.72	1.29528154220505402
+AAAAAAAAGCCDAAAA	Distinguished, assis	Books                                             	science                                           	6.29	16.68	0.00549932194811040
+AAAAAAAAGCCEAAAA	Old prices help general trials. National, prime men ought to compete about a posts. Suspicious, extreme mistakes might not make gently other characters. Acc	Books                                             	science                                           	1.53	3227.96	1.06424408127232946
+AAAAAAAAGEHDAAAA	Spanish ranks can deal all but conservatives. Local metres shall not go no longer with a processes	Books                                             	science                                           	2.91	4385.32	1.44582053510116972
+AAAAAAAAGGBAAAAA	Particular ears ought to know streets; tears could pr	Books                                             	science                                           	1.38	4417.02	1.45627188436706299
+AAAAAAAAGIAAAAAA	Useful examples might understand evidently. Royal shops ought to leave in order. Also huge experts stay continuous, long organisers. Often burning services flee global circumstances. Fine, ex	Books                                             	science                                           	2.78	7923.96	2.61249443309046200
+AAAAAAAAGJGBAAAA	Accounts accept	Books                                             	science                                           	1.24	4454.22	1.46853655921536677
+AAAAAAAAGKEDAAAA	Small years turn as beside a problems. Famous, significant attitudes defend again subtle machines. Pp. double less. Human men appear in a regions. Exclusively warm 	Books                                             	science                                           	1.75	3606.79	1.18914265043316062
+AAAAAAAAHFDEAAAA	Certain, long councillors smile then fresh eyes. Lights attend initially after a preferences; national genes admit. Wide single plans improve never	Books                                             	science                                           	2.09	2209.49	0.72845904383276100
+AAAAAAAAHGDAAAAA	Problems could not find small, late years. Demands might get only normal, available communications. Quiet mothers leave women. Fair interes	Books                                             	science                                           	0.21	8916.11	2.93960188337929509
+AAAAAAAAHJPDAAAA	Marks remember	Books                                             	science                                           	1.41	1407.04	0.46389484135906840
+AAAAAAAAHMDDAAAA	Things prejudice unfortunately. Available lives used to get for an readers. Roughly good articles might express open years. Black m	Books                                             	science                                           	9.38	11566.26	3.81334457287478571
+AAAAAAAAHNIDAAAA	Small, stupid members lack hands. Literary terms would understand sure ordinary acids. Lovely,	Books                                             	science                                           	0.22	2581.68	0.85116843447228203
+AAAAAAAAIHEAAAAA	Conditions must like most still desperate concessions. Parts shall not raise sometimes places. Local, prof	Books                                             	science                                           	4.37	214.32	0.07066035251313079
+AAAAAAAAIJHBAAAA	Male, major regulations could get. Books may not bring. Upper, musical girls take well special, curious parents. Criminal, equal knees stop just a	Books                                             	science                                           	3.41	7411.80	2.44363755485639582
+AAAAAAAAILGAAAAA	Courts receive high male limitations. Political, little parents may establish tomorrow unique minu	Books                                             	science                                           	9.26	10412.18	3.43284952048418299
+AAAAAAAAIMADAAAA	Local, contemporary tanks provoke yet. Well red quantities should spend only deaf new firms. 	Books                                             	science                                           	2.13	6975.01	2.29962983101256232
+AAAAAAAAIMMAAAAA	Young officers depend very well unnecessary players. Personnel will consider apart types. Most universal courses could enable arrangements. Magic, equal responsibilities detect; value	Books                                             	science                                           	5.89	6948.34	2.29083685041567357
+AAAAAAAAIOHAAAAA	Pounds realise fairly formal, casual residents. Good areas shall stick etc disputes. So small police find variable, certain programs. Results think children; dogs will take prices. Old, traditi	Books                                             	science                                           	44.25	3791.67	1.25009676564698863
+AAAAAAAAIOOBAAAA	Left times used to tell trees. Right t	Books                                             	science                                           	7.96	2094.92	0.69068582347334800
+AAAAAAAAIPCBAAAA	So clear employees could tell experiments. Hands would control demands; well ethnic sites afford then bottom programmes; times flow easily premises. Alter	Books                                             	science                                           	1.28	10461.12	3.44898482121203209
+AAAAAAAAJLLDAAAA	However major deb	Books                                             	science                                           	0.66	2219.28	0.73168676336945170
+AAAAAAAAJNDDAAAA	Thereafter strange rates shall not inhibit now on a heroes; eyes may not provide.	Books                                             	science                                           	8.37	11495.90	3.79014719324234879
+AAAAAAAALAPCAAAA	Due proposed concepts afford indeed yesterda	Books                                             	science                                           	1.34	10405.19	3.43054494851671946
+AAAAAAAALKJBAAAA	Earnings feel possibilities. Single, poor problems make full, sho	Books                                             	science                                           	2.75	17541.34	5.78330192213830518
+AAAAAAAALNGBAAAA	Direct schemes rival pa	Books                                             	science                                           	78.33	9776.79	3.22336425833730836
+AAAAAAAAMBLCAAAA	M	Books                                             	science                                           	42.63	5228.32	1.72375389255063431
+AAAAAAAAMCPCAAAA	Clear spirits shall not co	Books                                             	science                                           	2.11	1098.47	0.36216068227463034
+AAAAAAAAMLBEAAAA	New, political bish	Books                                             	science                                           	1.33	1836.00	0.60532104896467022
+AAAAAAAANKOAAAAA	Professionally uncomfortable groups would not protect again there dependent users. Standard fields avoid likely families. Independent, intact fortunes work in the	Books                                             	science                                           	8.28	64.98	0.02142361751727901
+AAAAAAAAOIDEAAAA	Future, solar deaths stand much confident, prime horses. New, other hundr	Books                                             	science                                           	0.22	7461.07	2.45988165511918956
+AAAAAAAAOPDDAAAA	Acts will not reflect as with the problems. General governments distract new, soft fires. Useful proposals restrict hard trees. Large, black customs go official	Books                                             	science                                           	3.05	12762.28	4.20766705707016963
+AAAAAAAAPGEDAAAA	Royal, considerable rooms reply then often c	Books                                             	science                                           	0.79	3487.40	1.14978029747243514
+AAAAAAAAAECEAAAA	Symptoms could not take else. Now rich	Books                                             	self-help                                         	8.22	4725.36	1.53069603755177003
+AAAAAAAAAFHBAAAA	Normal sports will not afford from a women. Nearly past families would permit 	Books                                             	self-help                                         	4.46	6912.33	2.23912593775928744
+AAAAAAAABFOCAAAA	There main prices could bowl acres. Radical, domestic plants take long. Fresh developments wave sanctions. British, able men cover goals. There other men	Books                                             	self-help                                         	7.22	5298.60	1.71638690482244922
+AAAAAAAACCGEAAAA	Results	Books                                             	self-help                                         	0.29	6602.84	2.13887217578942752
+AAAAAAAACDACAAAA	About statistical blocks shall point so brothers. Even new affairs spend hopefully even old contexts. Possible officers wait absolutely with	Books                                             	self-help                                         	3.51	7809.11	2.52962181374665694
+AAAAAAAACDJDAAAA	Facts shall provide al	Books                                             	self-help                                         	5.02	1138.39	0.36876112342521194
+AAAAAAAACDLDAAAA	Men shall accept yet. Indians can continue obviously global, efficient times. Profit	Books                                             	self-help                                         	5.85	4729.95	1.53218288613311888
+AAAAAAAACIDEAAAA	Proper, mutual feelings would bring right over the days. Prices ought to see thus electronic owners; most surprising definitions might not see in part big lads. Responsible, tory doors read good, a	Books                                             	self-help                                         	6.84	4062.63	1.31601648192708015
+AAAAAAAACMIBAAAA	Early alternatives lie meanwhile european, new makers. Suspicious purposes speak new, overseas critics. Generally important police must refer approximately virtually other firms. British, appointed c	Books                                             	self-help                                         	2.07	157.85	0.05113269031937184
+AAAAAAAACPGDAAAA	Settlements can see so scientific sales; jeans ought to disco	Books                                             	self-help                                         	0.78	10137.10	3.28373262614193372
+AAAAAAAADIFDAAAA	Now christian papers believe very major, new branches. Annual wars include harshly so-called sites. 	Books                                             	self-help                                         	5.23	8239.18	2.66893531470105824
+AAAAAAAADNCEAAAA	Much g	Books                                             	self-help                                         	4.52	725.34	0.23496094771145497
+AAAAAAAADPNAAAAA	Particular prisoners wait at a wag	Books                                             	self-help                                         	1.99	210.35	0.06813912834133586
+AAAAAAAAEAAEAAAA	Good others run considerably excelle	Books                                             	self-help                                         	2.72	567.97	0.18398374482542681
+AAAAAAAAECBBAAAA	Very concerned shares must form also rather nice gardens. Quietly available games may see quite. Short eyes repay. As useful variables should not produce there. Managers use so also total versions	Books                                             	self-help                                         	26.11	239.20	0.07748457094959609
+AAAAAAAAEHBCAAAA	Creative churches like. Walls objec	Books                                             	self-help                                         	6.05	3579.99	1.15967386770001887
+AAAAAAAAEJCEAAAA	Now environmental examples enter banks. Royal, new attitudes go prices; almost living tre	Books                                             	self-help                                         	7.75	779.81	0.25260553207443365
+AAAAAAAAEJJBAAAA	Hot steps help right able councils. Provincial mammals ought to establish from a others; forests used to offer true, open practitioners. Key theories could not imagine exact, other races.	Books                                             	self-help                                         	4.63	8643.42	2.79988164814865324
+AAAAAAAAENMCAAAA	Aware, a	Books                                             	self-help                                         	2.74	1189.77	0.38540475743604073
+AAAAAAAAEOFDAAAA	Cultural notes ignore usuall	Books                                             	self-help                                         	9.32	5567.49	1.80348902138865697
+AAAAAAAAEPICAAAA	Positive, recent adults cannot tell fortunately laboratories. Frequent performances may get labour buildings; vocational windows will talk; similar seeds must replace better. Other merch	Books                                             	self-help                                         	9.69	10154.63	3.28941115678050571
+AAAAAAAAFEAEAAAA	Tonight single claims used to compete seriously. Frequently magic advances concentrate very political men. Again damp types will apply also pol	Books                                             	self-help                                         	0.56	8790.86	2.84764220475738421
+AAAAAAAAFFGAAAAA	Areas increase well final, peculiar findings. Fat possibilities will say now at all sure dogs	Books                                             	self-help                                         	5.11	3770.90	1.22151575499093605
+AAAAAAAAGEPAAAAA	Clearly legal servants should not investigate however early difficult women. Increased laboratories tell home samples. Still wooden institutions avoid undoubtedly. Policies will 	Books                                             	self-help                                         	9.11	9124.75	2.95579991125554742
+AAAAAAAAGKLBAAAA	Physical, political issues must not increase. Teeth go there particular prices. Words mi	Books                                             	self-help                                         	4.82	1881.44	0.60945890956274278
+AAAAAAAAGLECAAAA	Old, acceptable respects imply around banks. Rights will not spare so existing reasons. Old eggs must claim. Patients might not stop there military,	Books                                             	self-help                                         	7.89	15529.28	5.03043310182334282
+AAAAAAAAGNJBAAAA	National, dreadful opportunities give. Lucky, low rules should start away from the girls. Available words will not leave now. Stor	Books                                             	self-help                                         	5.53	6895.58	2.23370007419989892
+AAAAAAAAGPFAAAAA	Dominant, useful restaurants might not say contrary eyes. Modest years may not confirm again just other stage	Books                                             	self-help                                         	3.87	12631.86	4.09186560365955223
+AAAAAAAAHAFBAAAA	Various	Books                                             	self-help                                         	6.24	3437.60	1.11354916846292444
+AAAAAAAAHBBEAAAA	There political deta	Books                                             	self-help                                         	8.83	4867.67	1.57679482221664051
+AAAAAAAAICMDAAAA	Other, established programmes used to avoid good organisations. Forward, simple changes might enter straight. Papers cal	Books                                             	self-help                                         	1.63	3028.98	0.98118401218606844
+AAAAAAAAIECDAAAA	Cards insist sad males. Instruments turn later instructions. Economic, white 	Books                                             	self-help                                         	2.64	3883.30	1.25792572896557903
+AAAAAAAAIEDBAAAA	Other, precious services can stem; grounds will set in particular friendly factors. Ports will provide. So complete moments diversify morally different, open pupi	Books                                             	self-help                                         	6.72	NULL	NULL
+AAAAAAAAIHIDAAAA	Metres must not go more soft attacks. Northern, central changes see all right inherent metres; women shall reduce together always private efforts. Extra, secret dates ought to sa	Books                                             	self-help                                         	36.51	215.49	0.06980413960672434
+AAAAAAAAIPODAAAA	Outside, remaining problems must come only new politicians. Readers would not tell right, modern products. Particular threats become legally among a beaches	Books                                             	self-help                                         	1.38	24121.05	7.81358365427991146
+AAAAAAAAJCEEAAAA	In order excellent words could go old costs. Surp	Books                                             	self-help                                         	1.45	3398.74	1.10096116500514307
+AAAAAAAAJCMCAAAA	Logic	Books                                             	self-help                                         	1.29	3676.91	1.19106937194932846
+AAAAAAAAJJOCAAAA	Sufficiently great tears may see. Much short standards duck over a pap	Books                                             	self-help                                         	8.57	1508.73	0.48872615689291017
+AAAAAAAAJMABAAAA	Again right years welcome to the months. Once competitive years could benefit great, social projects. Actually old expectations must not spin 	Books                                             	self-help                                         	2.42	1824.90	0.59114378564346952
+AAAAAAAAKCEAAAAA	Actions need qualifications. Expert sales see. Guests look evidently dead roots. Activities 	Books                                             	self-help                                         	2.20	1248.95	0.40457506223870418
+AAAAAAAAKDCEAAAA	Still social transactions provide both most existing vi	Books                                             	self-help                                         	6.50	2330.32	0.75486557431129919
+AAAAAAAAKHEBAAAA	Prime even	Books                                             	self-help                                         	4.28	3438.17	1.11373380979002005
+AAAAAAAAKHMAAAAA	Confidential, japanese reports discuss ever forms. Initiatives say now pregnant, sad sites. Neither round eyes may ask more w	Books                                             	self-help                                         	1.72	3385.13	1.09655244840554440
+AAAAAAAAKLCAAAAA	Clever, informal negotiations study sharply with a leaders. Professionals come noble officials. Plans continue pa	Books                                             	self-help                                         	4.69	2768.44	0.89678672909573497
+AAAAAAAAKLEAAAAA	British, 	Books                                             	self-help                                         	1.52	4014.40	1.30039323419756920
+AAAAAAAALBBAAAAA	Highly other times could stay no longer huge symbolic results. Most narrow police chan	Books                                             	self-help                                         	7.99	660.44	0.21393775099477944
+AAAAAAAAMBHAAAAA	Hands can ensure. Dead schools concentrate by a years. Increased authorities should not stop natural, following guards. Principal years might secure. Long, criti	Books                                             	self-help                                         	4.23	4140.99	1.34139980542043446
+AAAAAAAAMCODAAAA	Rights could not talk. Miners shall clear various outcomes. Relative, western forms locate communist, local prices. Items would not disappear probably likely women. Bare conclusions mark in gener	Books                                             	self-help                                         	8.57	3116.42	1.00950863962684053
+AAAAAAAAMHEDAAAA	Other changes shall seek 	Books                                             	self-help                                         	2.51	2862.54	0.92726874467415049
+AAAAAAAAMLOBAAAA	So ashamed periods could give there on the operations. Potatoes must order very noble systems; labour years should not escape so formal, ready 	Books                                             	self-help                                         	1.94	11014.72	3.56802196208166835
+AAAAAAAANBMCAAAA	Also crucial weeks will consider just then close parts. Long values prove then reco	Books                                             	self-help                                         	3.91	65.52	0.02122403465141110
+AAAAAAAANDDDAAAA	Sincerely important experiments should hear surprised, unchanged sorts. Else financial democrats will not start so major bodies. E	Books                                             	self-help                                         	1.90	5855.42	1.89675880614416367
+AAAAAAAAOACAAAAA	Cities practise a	Books                                             	self-help                                         	2.94	9089.11	2.94425496932977984
+AAAAAAAAOJMCAAAA	Nearly northern eyes would not use further buyers. Ever independent advertisements comment also nice, old schemes. Firm members would restore as a doors. Problems 	Books                                             	self-help                                         	8.02	14009.14	4.53801087906699247
+AAAAAAAAOKBEAAAA	Essential, modern goods help friendly roads. Cultures	Books                                             	self-help                                         	1.13	8764.28	2.83903208813597843
+AAAAAAAAOLEDAAAA	Gentlemen construct. Inevitable proposals tell more subject troops. Feelings used to come thus a	Books                                             	self-help                                         	1.73	8962.10	2.90311234660273887
+AAAAAAAAONJCAAAA	Miles kiss silently difficult streets. Less social rules see never 	Books                                             	self-help                                         	7.03	283.44	0.09181532938943778
+AAAAAAAAONPCAAAA	Yards shall build gradually steep, possible players. Foreign, wild lines used to understand vital layers. Problems shall go likely, parliamentary rats. Suspicious, wrong thousands 	Books                                             	self-help                                         	7.63	7823.86	2.53439981300044683
+AAAAAAAAPEECAAAA	Results	Books                                             	self-help                                         	9.21	3280.19	1.06255900829078431
+AAAAAAAAPPNDAAAA	Smooth, othe	Books                                             	self-help                                         	8.62	11533.69	3.73613303141992873
+AAAAAAAAABJAAAAA	Available, other responsibilities ban common, english authorities. Participants save little for a years. Well local plans look. As entir	Books                                             	sports                                            	2.98	624.68	0.24146901355107034
+AAAAAAAAAIOAAAAA	Now beautiful results scream just official payments. Carefully 	Books                                             	sports                                            	4.89	12518.36	4.83895120778186737
+AAAAAAAAAJABAAAA	Agricultural elections go users. Popular customers could threaten upside down hard, able pages. European, interesting bases spend at a fingers. 	Books                                             	sports                                            	2.47	7461.50	2.88423039734153702
+AAAAAAAAALMDAAAA	Levels should rethink really typically other women. Elections respond long numbers. Firms might sum nearly present, personal homes. Again clear	Books                                             	sports                                            	3.91	6886.83	2.66209266599525798
+AAAAAAAAAMGBAAAA	Very social engineers ask facilities. Numerous, stupid 	Books                                             	sports                                            	7.36	4152.23	1.60503759066587821
+AAAAAAAABAGAAAAA	Green levels provide. Students would agree. Very upper states get finally for a	Books                                             	sports                                            	1.29	4251.46	1.64339478189126194
+AAAAAAAABLKAAAAA	In order	Books                                             	sports                                            	9.54	5723.96	2.21258720433787633
+AAAAAAAABMIBAAAA	As specific characteristics contain for the hours. Free, double men avoid in the meals. Trying, potential institutions share above from the months. Contemporary problems could cheer only heav	Books                                             	sports                                            	1.58	1246.89	0.48198325271610120
+AAAAAAAABNPCAAAA	Grounds ought 	Books                                             	sports                                            	1.69	6467.35	2.49994337066900616
+AAAAAAAABOPBAAAA	Completely particular voices shall not say straight. Used ideas must recall current colonies. New techniques could not make naturally old, great versions; great adults test	Books                                             	sports                                            	2.88	6653.24	2.57179884055600185
+AAAAAAAACBHBAAAA	Procedures order here shops. Late static sciences shall not see cultures. Polite implications cover underway. That is right communications might not say cool principles. Strange keys	Books                                             	sports                                            	1.34	2498.12	0.96564412520362400
+AAAAAAAACDJAAAAA	More big results develop again on a politicians. Characteristics live flowers. Children wipe perhaps appropriate roles. Wrong, external shows want somewhat little ways. Then difficult	Books                                             	sports                                            	3.64	4362.77	1.68642147699654727
+AAAAAAAACGPAAAAA	Basic, functional circumstances must 	Books                                             	sports                                            	7.87	2947.46	1.13933575379592397
+AAAAAAAACLNAAAAA	Neighbours shall not represent overall dramatic trees. Random chiefs could not interfere basic, special fruits. A little poli	Books                                             	sports                                            	5.46	3974.85	1.53647164710487281
+AAAAAAAACPDDAAAA	Immediately impossible teachers cut kindly busy, national products. Important, principal communities could die all very video-taped words. Short children doubt windows. Sometimes russian developm	Books                                             	sports                                            	96.08	4160.79	1.60834644440858994
+AAAAAAAAFBKDAAAA	Twice detailed customers know women; economic, intense values listen wide industr	Books                                             	sports                                            	0.74	6802.45	2.62947571753614401
+AAAAAAAAFIECAAAA	Sad, very sales could gather hence on a pounds. Issues see just within a things. Eastern directors put very in a initiatives. 	Books                                             	sports                                            	3.99	5533.59	2.13899999791263899
+AAAAAAAAGBBAAAAA	Sick organizations cannot cause to the situations. Direct nations seek to a genes. Able, invisible polls c	Books                                             	sports                                            	52.92	10879.04	4.20527479218581719
+AAAAAAAAGBECAAAA	Letters help; international directions should hu	Books                                             	sports                                            	37.74	460.35	0.17794752575436260
+AAAAAAAAGCFDAAAA	Appointments might not hold to a tickets. Proper, private areas describe and so on prime, natural calls. Miners shall receive typically safe units. Little times will develop pointedly very mus	Books                                             	sports                                            	6.13	3351.79	1.29562884185557735
+AAAAAAAAGJJBAAAA	Ministers prove perhaps social processes. Aggressive characters could get open signals. Products try at all public, loyal councils; wholly historical respondents see there from a statements. C	Books                                             	sports                                            	7.24	13142.40	5.08017283039890319
+AAAAAAAAGJKBAAAA	Likely days shall get. Great users would love even. However acceptable walls	Books                                             	sports                                            	8.23	2406.70	0.93030587647013029
+AAAAAAAAGPODAAAA	Just average men might make so faintly free parents. J	Books                                             	sports                                            	1.41	9937.58	3.84135499725434718
+AAAAAAAAHACBAAAA	Papers conceive over immediate victims. Local, expert members add ill over immediate tiles. Profits pay even. Tall classes begin for instance grand fields; ru	Books                                             	sports                                            	0.25	3880.85	1.50013610366855243
+AAAAAAAAHEJCAAAA	Great, reliable children see french, proper dates. Public passages like closely traditionally academic books. Values used to distinguish leaders. Much key oper	Books                                             	sports                                            	31.97	1293.62	0.50004665638396557
+AAAAAAAAHLHDAAAA	Dual months should survive only large, political eyes; new, new merchants pass fairly conseque	Books                                             	sports                                            	6.26	4192.74	1.62069666369359458
+AAAAAAAAIACEAAAA	Conversely good eggs would not call too. Police happen present courses. Fine procedures finish well forward private	Books                                             	sports                                            	6.31	6912.27	2.67192645562313022
+AAAAAAAAIAMDAAAA	Real, japanese systems would need downstairs for the phrases; level waters might not go about existing, little friends. Nation	Books                                             	sports                                            	5.90	2794.92	1.08037167086213344
+AAAAAAAAIBLDAAAA	Devices take truly afraid, great men. Both true parties hurt even with a proposals. All internal candidates prevent more. Distinctive, prime women would say. Little, english departme	Books                                             	sports                                            	0.63	1050.56	0.40609221821766738
+AAAAAAAAIHLCAAAA	Parents prevent alone little children. Cases might dispose again lives; very strange windows violate officially. Improved, cheap critics should alert plates. Expressions build c	Books                                             	sports                                            	5.56	4342.45	1.67856681484095121
+AAAAAAAAJBBCAAAA	Wrong others miss less to the respects. Especially other standards start in order regula	Books                                             	sports                                            	7.53	11059.22	4.27492307108322362
+AAAAAAAAJCCDAAAA	Adults will foresee most left, social children. Different eyes make personal counties. Readers would not admit more musical proceedings; titles take here away fast institutions; bird	Books                                             	sports                                            	3.83	10985.10	4.24627210853535058
+AAAAAAAAKEOBAAAA	International, coloured contexts think. Relevant, british conservatives ought to happen ago. Perhaps human shops must see animals; rights must h	Books                                             	sports                                            	44.83	10933.78	4.22643444801245737
+AAAAAAAAKMFBAAAA	Years should comment then central, internal implications; directly collective feet may find around extra, victorian crimes. Short	Books                                             	sports                                            	2.75	1868.42	0.72223463901372038
+AAAAAAAAKNODAAAA	So single phrases could not sum; desirable friends see times. French efforts think	Books                                             	sports                                            	4.59	4611.30	1.78249033455217177
+AAAAAAAALEHDAAAA	Central, visible moments 	Books                                             	sports                                            	57.64	1362.54	0.52668756759280813
+AAAAAAAALJLDAAAA	Old, straight enemies obtain however years. Largely social questions disrupt never. Measures rule fine, extensive trees. Already economic friends would not show more beautiful engines. Systems ret	Books                                             	sports                                            	9.99	4644.12	1.79517685088812959
+AAAAAAAALNHAAAAA	Freely proud children cannot continue countries. Rates shall not look applications. Compl	Books                                             	sports                                            	4.13	886.97	0.34285677618843706
+AAAAAAAAMDOAAAAA	Already secondary year	Books                                             	sports                                            	72.51	8152.72	3.15142033706550904
+AAAAAAAAMLOAAAAA	Developers ought to recognize again. British, fast artists shall experi	Books                                             	sports                                            	79.00	2317.17	0.89569820408870728
+AAAAAAAAMOFEAAAA	Paths check still international attitudes. Immediate	Books                                             	sports                                            	0.37	2211.39	0.85480912127281399
+AAAAAAAANDFBAAAA	All capital bacteria make jobs. Again appropriate eyes may not leave others. There fixed ways	Books                                             	sports                                            	0.32	7910.07	3.05762438371632671
+AAAAAAAANDNCAAAA	Papers occur critically relatively happy numbers; related, soviet genes experiment governments; voluntary devices	Books                                             	sports                                            	2.52	3864.91	1.49397452321775512
+AAAAAAAANMJDAAAA	Indeed similar changes might drink too national careful areas. Wise, good rooms give large opportunities. Various patients shall research directly plants. International hands can get pieces	Books                                             	sports                                            	9.31	3710.53	1.43429919134861534
+AAAAAAAAOCLBAAAA	Here familiar rooms would not believe particularly new, fresh rights. Levels allow then wives; temporary, big ears may sound always others. Lovely, essentia	Books                                             	sports                                            	9.23	1808.93	0.69923887859854273
+AAAAAAAAOKLCAAAA	Lines might clear too high eyes. Great women balance as the things. Natural requirements	Books                                             	sports                                            	8.76	5395.16	2.08549011197764081
+AAAAAAAAOLJBAAAA	General, local thanks must soar actually about p	Books                                             	sports                                            	22.08	7752.94	2.99688604392750734
+AAAAAAAAOMFAAAAA	Inc others look in the varieties. Cold methods write values. Partners will make often times. Democratic, dramatic personnel shall not see	Books                                             	sports                                            	3.64	473.00	0.18283736218488870
+AAAAAAAAOMKBAAAA	Others used to coincide there so as historical sites; syste	Books                                             	sports                                            	4.08	4391.31	1.69745356646114923
+AAAAAAAAOMMAAAAA	Poor, major pairs affect complex, redundant results. Different animals could find so great, honest designs. Dull, linguistic studies might not get more cons	Books                                             	sports                                            	33.21	1010.07	0.39044087615663959
+AAAAAAAAPLNBAAAA	Open prod	Books                                             	sports                                            	2.74	12438.41	4.80804666844427361
+AAAAAAAAACIBAAAA	Bloody masters pull only women; shops take aggressively also legal cells. Continually underlying grounds would interfere. Entries shall not separate. Senior techniques see in 	Books                                             	travel                                            	2.25	4171.41	1.64665291182793628
+AAAAAAAAACKCAAAA	Active, mi	Books                                             	travel                                            	1.40	12936.19	5.10652631399441219
+AAAAAAAAADDEAAAA	Voluntary others will imply again international, important birds; ill old publishers can get dark powers. Features stretch now progressive procedures. Tough n	Books                                             	travel                                            	1.83	3612.43	1.42599705573765030
+AAAAAAAAAGAEAAAA	Cold terms shall comply only early claims; head, different politicians shall not commend good, foreign organizations; criminal, po	Books                                             	travel                                            	1.03	5504.86	2.17302872367020583
+AAAAAAAACACEAAAA	Operations s	Books                                             	travel                                            	1.00	193.62	0.07643097580629212
+AAAAAAAACBLAAAAA	Applications might gather rather out of a problems. Scales could observe presumably for a directors; totally empty questions will forget. Just, symbolic question	Books                                             	travel                                            	21.48	5351.75	2.11258896173599765
+AAAAAAAACDDDAAAA	For example influential subjects shall work for example. Modules should persuade aside overall preliminary relatives. American, available reasons may use to the weekends; streets used t	Books                                             	travel                                            	2.18	6997.28	2.76215751673304277
+AAAAAAAACGADAAAA	Similar sides assess more yet complete improvements. Bacteria would stay; general, curious trends used to reac	Books                                             	travel                                            	1.61	221.43	0.08740889873353613
+AAAAAAAACHBBAAAA	Communist, small cases may not turn other rules. Little, forward men should assist quite available technique	Books                                             	travel                                            	2.29	16204.92	6.39684871636659094
+AAAAAAAACPDCAAAA	Conflicts could give really sole institutions. Then advanced proceedings could not receive. Black experiences shall 	Books                                             	travel                                            	1.91	5880.48	2.32130371144192077
+AAAAAAAADACCAAAA	Leading players will sa	Books                                             	travel                                            	4.51	262.65	0.10368038320174892
+AAAAAAAADADBAAAA	There european members turn; industrial, honest leaders cut exactly happy, consistent reasons. Incidentally european millions worry at first aware 	Books                                             	travel                                            	3.81	2395.24	0.94551456714318326
+AAAAAAAADEPDAAAA	Deliberately ordinary procedures will not pay by a months. Feet reach very s	Books                                             	travel                                            	9.43	1776.74	0.70136335065629308
+AAAAAAAAEEHCAAAA	Good, national parts remove animals; 	Books                                             	travel                                            	2.57	3370.45	1.33047609960911726
+AAAAAAAAEIICAAAA	Odd, artistic databases open now; female, left days use all obligations. Simple, early sites may not hesitate statements. Left, free s	Books                                             	travel                                            	2.31	9717.76	3.83605970174234756
+AAAAAAAAEJPAAAAA	However solid hours visit painfully things. Clubs must take most other words; officials will follow necessary developers. Alternative, great decisio	Books                                             	travel                                            	2.68	1892.11	0.74690534879063830
+AAAAAAAAFEBAAAAA	Finally surprising cells cannot look better points. Elections might choo	Books                                             	travel                                            	1.98	3145.02	1.24148821160161580
+AAAAAAAAFGBEAAAA	Right schools go now; average, invisible hands should get also good persons. Usually good ministers will make. Notes ought to stem average words. Heavy, certain suggestions summaris	Books                                             	travel                                            	4.55	337.50	0.13322721999082528
+AAAAAAAAGEEDAAAA	Thanks could talk well individually national records; just simple officials go then encouraging, remarkable needs. Signals assess now. Upper, cheap pp. would not see. Hard trousers shall send whol	Books                                             	travel                                            	4.23	6920.66	2.73191197719023675
+AAAAAAAAGFHCAAAA	Reports used to think characteristics. True types break extremely deliberately white tasks. Courses must cost. Economic, nervous resou	Books                                             	travel                                            	0.74	1273.19	0.50258833842998175
+AAAAAAAAGMGCAAAA	Dear signals know finally. Positions answer payable payments. Superior babies can exis	Books                                             	travel                                            	1.78	16390.16	6.46997170964392568
+AAAAAAAAGNFBAAAA	Horizontal problems continue members; modern, other interactio	Books                                             	travel                                            	8.51	2371.88	0.93629326978322569
+AAAAAAAAHAECAAAA	Open conditio	Books                                             	travel                                            	8.17	9456.37	3.73287670016189772
+AAAAAAAAIACAAAAA	Practical writers used to succeed recent arms. 	Books                                             	travel                                            	9.48	10115.82	3.99319281934100804
+AAAAAAAAIFJAAAAA	Members show yards. Economic stones get newspapers. Only magic views lea	Books                                             	travel                                            	9.23	1653.26	0.65261995176898313
+AAAAAAAAIGDAAAAA	Investments ought to use still also professional developments. Only fresh visitors know steadily never main occ	Books                                             	travel                                            	1.37	4036.41	1.59336202383160616
+AAAAAAAAIGEEAAAA	Conclusions might take on a ch	Books                                             	travel                                            	4.48	4341.46	1.71377969333738765
+AAAAAAAAILBCAAAA	Small, original things announce in addition at last other functions. Best political women make even old materials. Downstairs wet arr	Books                                             	travel                                            	0.34	8289.45	3.27223815926799005
+AAAAAAAAIMLAAAAA	Again english deals cut for the cases. Yet normal systems reach biological, original reasons. So other remains spread steadily. Much inadequate members consider	Books                                             	travel                                            	1.92	7192.94	2.83939377706905721
+AAAAAAAAINFCAAAA	Later severe rules would	Books                                             	travel                                            	1.57	3713.31	1.46581916522705609
+AAAAAAAAINIDAAAA	Movements may describe quite southern, nervous views. Young notes imagine. Sensitive women might excuse then sales. Proportions may not exist only from a controls. Are	Books                                             	travel                                            	2.49	6651.86	2.62580389797976612
+AAAAAAAAJGKDAAAA	That is fine terms know to the goods; useful colleagues us	Books                                             	travel                                            	6.31	6202.60	2.44845971767434933
+AAAAAAAAKBODAAAA	Yesterday long babies may not include as else able companies. Large, true d	Books                                             	travel                                            	4.19	1813.84	0.71600847617232157
+AAAAAAAAKEKBAAAA	Words see low courts. National, democratic plants avoid. Days should go stupid, apparent days. Dependent hours should not want police. Also urban wages shall not define so great, typic	Books                                             	travel                                            	8.88	8312.77	3.28144366673520796
+AAAAAAAAKGPBAAAA	Masses can contain as. Military men retain in a earnings; british, related instructions shall know different, precise needs; favorite	Books                                             	travel                                            	5.09	959.36	0.37870478746784635
+AAAAAAAAKIIAAAAA	Behind relevant areas find then necessary papers. Copies might come envi	Books                                             	travel                                            	7.07	7437.38	2.93588581160107894
+AAAAAAAAKLHBAAAA	Remarkably good bishops would deprive transactions. I	Books                                             	travel                                            	0.59	7014.30	2.76887611609376528
+AAAAAAAAKNEEAAAA	Running businesses find emotions; 	Books                                             	travel                                            	4.40	2300.61	0.90815962839434831
+AAAAAAAALJDCAAAA	Pink, central countries shall defend rapidly 	Books                                             	travel                                            	6.87	6536.14	2.58012373828394893
+AAAAAAAALNEAAAAA	Local, conservati	Books                                             	travel                                            	1.68	8121.86	3.20608245616202735
+AAAAAAAALNHDAAAA	Strong women know also also obvious votes. Private, natural areas should play strongly for 	Books                                             	travel                                            	2.11	184.12	0.07268087628062445
+AAAAAAAAMENAAAAA	Colours meet certainly hours; aw	Books                                             	travel                                            	1.63	5441.98	2.14820701228347073
+AAAAAAAAMHJAAAAA	Too full weeks might obtain most today vital cities. Police shall take for example full sto	Books                                             	travel                                            	3.82	5904.69	2.33086054402259597
+AAAAAAAAMKAAAAAA	Exceptional hundreds compare else then previous scientists. Rapid, popular differences get exactly now social persons. Naturally fundamental dreams hold on a changes. Brilliant birds pursue te	Books                                             	travel                                            	5.39	3124.51	1.23339194409935853
+AAAAAAAAOBIDAAAA	British leaders can focus. Different workers cannot breathe only in an objectives; arrangements might enter predictably hours; reduced, effective phases operate ready men. Others say o	Books                                             	travel                                            	4.95	1624.50	0.64126701888917236
+AAAAAAAAOHHBAAAA	Yesterday public notes work at least students; accidents might not apply today rural, subject premises. National, particular organisations could not endorse simply under a respondents. Sti	Books                                             	travel                                            	9.83	531.86	0.20995030881280099
+AAAAAAAAONIAAAAA	Maybe gastric variations will see as. However physical plants would not choose for example wi	Books                                             	travel                                            	6.36	1691.34	0.66765192965713314
+AAAAAAAAPMICAAAA	Little arts can grow directly rights. Full, slim argum	Books                                             	travel                                            	4.77	16542.31	6.53003251415238218
+AAAAAAAAPMNDAAAA	About right clothes must get thoughtfully to a cases. Eastern improvements 	Books                                             	travel                                            	98.75	2730.37	1.07780623598918408
+AAAAAAAAPPDEAAAA	Countries want incorr	Books                                             	travel                                            	63.33	473.46	0.18689706541290708
+AAAAAAAADKHCAAAA	Fields would die clear horses. However new problems go nasty, smooth ways. Interested others go great societies. Familiar patients shall seem trends. Yellow, r	Home                                              	NULL	NULL	7995.48	34.64319649767261090
+AAAAAAAAGAMCAAAA	NULL	Home                                              	NULL	0.87	14048.70	60.87087637475838958
+AAAAAAAAHGGDAAAA	NULL	Home                                              	NULL	NULL	116.76	0.50590328824138814
+AAAAAAAAAEPBAAAA	Neat, desirable words make especially gradu	Home                                              	accent                                            	7.11	1583.88	0.73384072874422647
+AAAAAAAABCDBAAAA	Common males protest probably statements. Subsequent, main ways begin then titles. Rights come therefore interesting, ordinary thin	Home                                              	accent                                            	8.82	1429.40	0.66226730413099308
+AAAAAAAABCNAAAAA	Offers go odds. Black, certain readers prove again in a cases. Public, black things watch as else modern forces. Difficult, new crops comp	Home                                              	accent                                            	3.59	4707.69	2.18115934307012370
+AAAAAAAABDMDAAAA	National, round fields would not accomp	Home                                              	accent                                            	0.17	1970.93	0.91316811090730250
+AAAAAAAABDPDAAAA	More general applications work also moves. Final, equal instruction	Home                                              	accent                                            	33.79	1466.94	0.67966027642501678
+AAAAAAAABIDBAAAA	Severe plants filter fair with the days. Both great hills bring still. Military standards ask now for a conditions. Ago new proposals may like particularly men. Then alone a	Home                                              	accent                                            	5.54	6369.32	2.95102307649896240
+AAAAAAAABMNCAAAA	Present, good grounds fall students. Big, long nerves remain events. Important, black years must not use principles. Fatal mines cannot order hospitals. Forces apply elsewhere; now final members	Home                                              	accent                                            	5.37	187.59	0.08691389644741359
+AAAAAAAACBIAAAAA	Terms must work slow signs. Just american movements make surprisingly	Home                                              	accent                                            	0.26	481.20	0.22294880841460324
+AAAAAAAACBIDAAAA	Discussions could inform; legitimately potential miles remember again from the factors. Then administrative changes may	Home                                              	accent                                            	2.20	1475.60	0.68367261366705848
+AAAAAAAACLEDAAAA	Ago light fingers blame enough green, british years. Children go general stands. Economic, great numbers affect deputies. Purposes urge annually. Always electrical ways vote judicial, regular ac	Home                                              	accent                                            	6.86	11873.28	5.50110895256222018
+AAAAAAAADCIDAAAA	Days shall want later romantic, american changes. Reasons read; great reasons may occupy economically. Strong, new films go then objects. English relations would resolve over. New, crazy feelin	Home                                              	accent                                            	1.78	715.86	0.33167110139583931
+AAAAAAAADIJCAAAA	New, large words stop more strong cars. Back views leave other, young shoes. White conte	Home                                              	accent                                            	2.81	9585.07	4.44093918343840622
+AAAAAAAADKJDAAAA	Decades try then. Different leaders stray examples. Things would not participate too good, good messages. Exactly new thanks can forget; companies u	Home                                              	accent                                            	3.51	4955.85	2.29613643429241784
+AAAAAAAADNPDAAAA	Very afraid concepts will not disentangle with a days. Long-term, civil points c	Home                                              	accent                                            	8.15	3501.80	1.62244833189164095
+AAAAAAAAEEPAAAAA	New measures shall pay under a agencies; comparatively heavy police shall beat similarly concepts. However japanese times cannot check like a police. Long, long-term auth	Home                                              	accent                                            	1.87	5547.93	2.57045798559357804
+AAAAAAAAELJCAAAA	Useful, n	Home                                              	accent                                            	9.44	3014.70	1.39676594498650122
+AAAAAAAAFAJCAAAA	Days give briefly vulnerable months. Sexual feelings create just animals. Charts study; changes knock rapidly aware sites. Schemes include sufficiently. For example speci	Home                                              	accent                                            	7.15	303.87	0.14078855863039378
+AAAAAAAAFKLCAAAA	Connections must not come right finally certain parties. Wild parties fi	Home                                              	accent                                            	2.55	1293.30	0.59920967149336320
+AAAAAAAAFOADAAAA	Little powers reach by a subjects; traditional insects make also others. Numbers shall make. Products take serious, military rules. Curiously economic methods approac	Home                                              	accent                                            	3.52	99.03	0.04588241998607265
+AAAAAAAAGCJAAAAA	Old buildings must proceed;	Home                                              	accent                                            	9.33	595.01	0.27567907417866391
+AAAAAAAAGEPDAAAA	Additional eyes give nationally. Territorial groups should talk previously strange differences. Small discus	Home                                              	accent                                            	6.07	18159.55	8.41365343691896978
+AAAAAAAAGHFBAAAA	Almost busy pounds lose at last for an factors. Good mothers would	Home                                              	accent                                            	1.45	2292.51	1.06216203819318802
+AAAAAAAAGKMDAAAA	Benefits might choose only by a directors. Continued eggs must not make much black, back arrangements. Living,	Home                                              	accent                                            	1.62	9494.68	4.39905983432661074
+AAAAAAAAGNOBAAAA	Holes may avoid of course genuine	Home                                              	accent                                            	3.27	409.64	0.18979374455311320
+AAAAAAAAGOEEAAAA	Supporters will laugh well indirect, old reductions. Men can increase critical words. Eyes ought to drift better parties. Other, social goods avoid costs; similar, substantial days learn;	Home                                              	accent                                            	63.79	5475.88	2.53707589572185700
+AAAAAAAAHKFAAAAA	Main, powerful kilometres should like certainly political directors. Left families go tall, clear organizatio	Home                                              	accent                                            	0.18	11613.93	5.38094732857567124
+AAAAAAAAHOAEAAAA	Promptly soviet faces could confirm now consistent new procedure	Home                                              	accent                                            	1.85	5675.68	2.62964690968951645
+AAAAAAAAHPCEAAAA	Old events can try far natural genes. Primary months explain at all par	Home                                              	accent                                            	0.15	20335.22	9.42168135463177076
+AAAAAAAAIEODAAAA	Women should hear among a pages. Everywhere main techniques go just unlikely principles. Broad, willing differences can make also short, modern roots. Together sorry thoug	Home                                              	accent                                            	8.25	1632.64	0.75643213335415177
+AAAAAAAAIKDBAAAA	Attractive, pale rights stop in a delegates. Answers go as; variable, alone roles ought to relax quickly concerned, detailed parents. Poor, physical matches would send as for a details; cent	Home                                              	accent                                            	1.45	989.82	0.45860180703437776
+AAAAAAAAILOBAAAA	Ancient periods will not see in a affairs. Fun	Home                                              	accent                                            	4.09	8014.62	3.71332082064806196
+AAAAAAAAJNKCAAAA	Perhaps material e	Home                                              	accent                                            	6.64	2552.44	1.18259238684490834
+AAAAAAAAKMBDAAAA	Here german thanks trust further remarkable towns. Other years	Home                                              	accent                                            	2.04	7200.88	3.33630011541261051
+AAAAAAAAKOEAAAAA	Supreme others can decide. Unfair, short presents give. Activities give simply police. Dark, impossible 	Home                                              	accent                                            	0.13	2033.98	0.94238033528498482
+AAAAAAAAKOEBAAAA	Still different holes ought to enjoy early problems. Mammals see usually. Powerful, public 	Home                                              	accent                                            	6.84	1085.87	0.50310353822353537
+AAAAAAAALGMCAAAA	Always potential wages shall not restart sometimes at the efforts. Mere, high weapons would not go there physical pr	Home                                              	accent                                            	66.58	7246.44	3.35740890118021093
+AAAAAAAALIMDAAAA	Boys ought to answer. International citizens call areas. All quick cuts might back most white, central amounts. Strong mice make on a lines. Cultures would dismiss changes. Left chil	Home                                              	accent                                            	5.45	18131.76	8.40077781891015469
+AAAAAAAALOADAAAA	Most main firms would know highly for an companies. D	Home                                              	accent                                            	1.31	5733.85	2.65659814033265334
+AAAAAAAAMBBDAAAA	New investors think especially secondary parties. Farmers detect adequately. Hum	Home                                              	accent                                            	38.04	1460.72	0.67677843605024781
+AAAAAAAAMDCAAAAA	International, nice forces will turn modest ways. Trees might not deal eastern others. Responsibilities ought t	Home                                              	accent                                            	2.75	6806.25	3.15346077986677743
+AAAAAAAAMOFDAAAA	Quite political women like home seriously formal chains. Certainly male lips 	Home                                              	accent                                            	4.86	1551.13	0.71866705152980782
+AAAAAAAANGKCAAAA	Rules meet as; authorities shall not kill moreover near a 	Home                                              	accent                                            	3.55	651.58	0.30188899540063836
+AAAAAAAANMBCAAAA	Also possible systems could go forward. Local, british babies d	Home                                              	accent                                            	2.53	2797.54	1.29615172379922932
+AAAAAAAAOCADAAAA	British results cou	Home                                              	accent                                            	4.30	118.60	0.05494956084366572
+AAAAAAAAOFFEAAAA	Simply perfect shareholders come others. Other, tired eyes contact therefore educational jobs. Over cathol	Home                                              	accent                                            	7.12	11929.65	5.52722621010654933
+AAAAAAAAOIKDAAAA	Enough labour losses demonstrate also quickly happy women; near available things might surrender also ge	Home                                              	accent                                            	1.26	1093.19	0.50649502882535352
+AAAAAAAAPABAAAAA	Royal children 	Home                                              	accent                                            	3.70	188.00	0.08710385698658647
+AAAAAAAAAALAAAAA	Future, real fears mean far interests; ill, mean payments speak far so labour lights. Already other applicants might not go so powerful lengths; japanese, central modes boil. Old homes ough	Home                                              	bathroom                                          	1.70	19546.11	7.34362930968507144
+AAAAAAAAAAOAAAAA	Also eastern matters should not enable now irish, 	Home                                              	bathroom                                          	3.46	2574.19	0.96714369931910820
+AAAAAAAAABFEAAAA	Quite public shoulders help even ministers. Short, tall groups cannot overcome too other notes. Thus surprising reasons find	Home                                              	bathroom                                          	1.77	11046.40	4.15022051991445731
+AAAAAAAAAEHCAAAA	In	Home                                              	bathroom                                          	0.42	1225.60	0.46046768804381146
+AAAAAAAAAHKDAAAA	Necessary, p	Home                                              	bathroom                                          	8.13	5680.58	2.13423918027734537
+AAAAAAAAAJCAAAAA	Letter	Home                                              	bathroom                                          	9.54	6366.89	2.39209131717465953
+AAAAAAAAALCEAAAA	Modern companies shall not become also old, grateful agents. Enough joint programs approve titles. Jeans will not fall already wrong teachers. High, silver children manage a	Home                                              	bathroom                                          	2.28	16790.19	6.30820820097611185
+AAAAAAAAANBAAAAA	Detailed, unhappy groups play old, human others. Well anxious councils will study whole, democratic employees. Educational, english customers get more. Explicitly cold deci	Home                                              	bathroom                                          	79.37	2249.42	0.84512502189907830
+AAAAAAAAAPICAAAA	Pp. may not record also human rocks. Extraordinary, industrial measures may not operate only out of a officials. Ready subjects show clearly new things. Projects should enable	Home                                              	bathroom                                          	3.56	11356.89	4.26687408752274959
+AAAAAAAABLEAAAAA	Here economic areas develop too sole processes; grateful, new children pass shares; fat, proposed aspects affect gmt on the terms. Years remind e	Home                                              	bathroom                                          	6.16	5399.13	2.02849617211813296
+AAAAAAAACGECAAAA	Appropriate, active areas change alternative books. Clients will not look now only, other rates. Usually effecti	Home                                              	bathroom                                          	2.89	2344.36	0.88079473657179327
+AAAAAAAACLKCAAAA	Employees watch never at the imports. Cases resist actually reliable prices. Alive, var	Home                                              	bathroom                                          	7.17	2759.95	1.03693521182809843
+AAAAAAAACONAAAAA	Very oral hands ought to smoke military, independent issues. Moving sons play. Patients contradict to a measures. Other cattle enable significant goods. Initial, possible groups let soci	Home                                              	bathroom                                          	7.17	3821.04	1.43559518172562445
+AAAAAAAADHLBAAAA	New sports will give now students. Scarcely free countries damage there prime, necessary members. Big units should not fill probably mental child	Home                                              	bathroom                                          	4.29	1777.37	0.66777207465602902
+AAAAAAAAEABDAAAA	Unions last moving pur	Home                                              	bathroom                                          	2.72	3881.21	1.45820153028110433
+AAAAAAAAEBAAAAAA	Indeed political miles imagine. Urgent, able males can explain companies. Accor	Home                                              	bathroom                                          	5.47	2914.22	1.09489568036148517
+AAAAAAAAEDMBAAAA	Almost other bodies call cars. So international benefits ought to suppose in a points. Officers can ensure also for a books. Carefully different police sleep. Irish, u	Home                                              	bathroom                                          	9.17	4471.44	1.67995564541989254
+AAAAAAAAEIJCAAAA	Labour, japanese economies care more minor, great gardens; events may m	Home                                              	bathroom                                          	5.15	5956.38	2.23785943840600333
+AAAAAAAAEMBAAAAA	Smal	Home                                              	bathroom                                          	3.40	1261.44	0.47393306168895686
+AAAAAAAAGALAAAAA	Free, sad bits might not speed then. Troubles	Home                                              	bathroom                                          	5.76	175.15	0.06580525094718797
+AAAAAAAAGCLDAAAA	Hard players show empty troops. Expectations used to know even; alternative organs could not consume historical, direct practices. Material restrictions could count deep. Gifts could s	Home                                              	bathroom                                          	4.64	8640.19	3.24618824539756797
+AAAAAAAAGECCAAAA	Mere, alternativ	Home                                              	bathroom                                          	6.84	4069.67	1.52900745430912057
+AAAAAAAAGKPAAAAA	Strong taxes represent nece	Home                                              	bathroom                                          	3.36	2436.99	0.91559656583378597
+AAAAAAAAGLOAAAAA	Simply costly processes should not believe therefore by the weeks. Instead earl	Home                                              	bathroom                                          	7.28	419.52	0.15761700757844303
+AAAAAAAAGONBAAAA	Joint lovers can mention tomorrow minor techniques. Major markets may no	Home                                              	bathroom                                          	17.20	2682.86	1.00797188442005549
+AAAAAAAAHKDCAAAA	Pretty figures ought to join that things. Extra authorities find dramatic items. Over mutual cases give for the time being as successful lines; permanent arms return publi	Home                                              	bathroom                                          	0.31	15228.27	5.72138240845865918
+AAAAAAAAIEOAAAAA	Both long tories will not get together; problems seem by now special,	Home                                              	bathroom                                          	5.62	8655.20	3.25182762202741263
+AAAAAAAAIJAAAAAA	Sanctions will know black quarters. Cent	Home                                              	bathroom                                          	4.35	2089.84	0.78516954404494038
+AAAAAAAAILJBAAAA	Comfortable clothes ought to carry violently. New, united services must look always. Common, recent workers could prevent. New, local languages need very often young kinds. Structures might	Home                                              	bathroom                                          	1.84	4089.18	1.53633751680400859
+AAAAAAAAILOCAAAA	Drivers might put 	Home                                              	bathroom                                          	7.91	1583.75	0.59502749750276305
+AAAAAAAAIMGCAAAA	Financial forces may bring yet. Unknown, expensive assets offer enough securities; female movements ought to grow great, aware modules. Normal contacts mus	Home                                              	bathroom                                          	2.10	4156.11	1.56148365123675362
+AAAAAAAAIMGDAAAA	By now developing masses used to flourish subtle methods. Much 	Home                                              	bathroom                                          	9.84	4755.08	1.78652145403342606
+AAAAAAAAIOEDAAAA	Thereby social children should report to a days. Times meet anyway as a whole liable reasons. Physical, region	Home                                              	bathroom                                          	5.82	12047.28	4.52625911293770307
+AAAAAAAAJBIBAAAA	So present rises l	Home                                              	bathroom                                          	5.86	3137.27	1.17869734307213477
+AAAAAAAAJKPAAAAA	Philosophical,	Home                                              	bathroom                                          	6.72	3878.46	1.45716833336357782
+AAAAAAAAJODCAAAA	Single p	Home                                              	bathroom                                          	3.92	6593.22	2.47712530202694074
+AAAAAAAAKBHCAAAA	Areas ride perhaps even leading women. High sides cannot get then throughout the officers. Long signs may not embrace to the friends. Very, tory	Home                                              	bathroom                                          	9.18	6130.98	2.30345804996968600
+AAAAAAAAKBIBAAAA	Hi	Home                                              	bathroom                                          	2.13	440.85	0.16563085857874860
+AAAAAAAAKECAAAAA	Force	Home                                              	bathroom                                          	0.20	6396.38	2.40317094521024374
+AAAAAAAAMBGBAAAA	Hard programmes make as other goods. Rational, similar computers could go to the streets. Options mi	Home                                              	bathroom                                          	7.10	4799.14	1.80307514719205068
+AAAAAAAAMBJDAAAA	So straightforwar	Home                                              	bathroom                                          	1.16	1899.26	0.71356711912050371
+AAAAAAAAMHGBAAAA	Properties go industrial troops; sweet companies would start more constant negotiations. Groups will protect. Public so	Home                                              	bathroom                                          	5.64	10621.64	3.99063480257316377
+AAAAAAAANAJDAAAA	Especially linguistic games cover to a officials. Minor, main days know completely variations	Home                                              	bathroom                                          	1.60	3572.22	1.34211152462782650
+AAAAAAAANAKCAAAA	From time to time successful books decide important, active elements. Parts will hear on a clubs. Firstly following supplies take barely upon a years. Other cases may find	Home                                              	bathroom                                          	3.90	218.22	0.08198699321550305
+AAAAAAAANBLAAAAA	Important kinds can catch again slim areas. Good, past men must 	Home                                              	bathroom                                          	5.17	6013.16	2.25919213694315054
+AAAAAAAANBPCAAAA	Formal, positive soldiers co-operate long along a offices. Great, able details must overtake responsible, remaining papers. Lives would think acute, labour shapes. Representative	Home                                              	bathroom                                          	10.92	3002.22	1.12795798172233325
+AAAAAAAANIKDAAAA	Social	Home                                              	bathroom                                          	5.38	4680.62	1.75854623858650847
+AAAAAAAAOHDBAAAA	Main forms matter constitutional, popular animals; ministers might not allow hardly. Officials will think so. Soon brief relations interfere for example old terms. Co	Home                                              	bathroom                                          	8.37	867.00	0.32573880999835553
+AAAAAAAAPABDAAAA	Probably awful sales require massively as annual notes. A little national devices arrest sharply short, grateful legs. Trees may protect immediately in a courses. Indians will not get i	Home                                              	bathroom                                          	4.33	1138.62	0.42778860881237321
+AAAAAAAAPGFCAAAA	Military characters would	Home                                              	bathroom                                          	2.10	8317.61	3.12499236843185918
+AAAAAAAAPKECAAAA	In particular acute origins could like thousands; impatiently small stones might give away female, crucial models. Colleagues might accompany bes	Home                                              	bathroom                                          	3.25	4807.80	1.80632877821233414
+AAAAAAAAPLOAAAAA	Afterwards oth	Home                                              	bathroom                                          	0.24	7197.60	2.70419568494136532
+AAAAAAAAAMLAAAAA	Material officials tackle employers. Clear shareholders go very products. Areas imagine systems; superior, precise tonnes will make much minutes. Milita	Home                                              	bedding                                           	18.44	3038.10	1.25620354127751860
+AAAAAAAABBEEAAAA	Large tests complain dark, pales	Home                                              	bedding                                           	37.80	10472.58	4.33023668816435133
+AAAAAAAABELDAAAA	Great servants deal primarily certainly possible gates. Problems ca	Home                                              	bedding                                           	4.62	4172.20	1.72513492476154936
+AAAAAAAABFBAAAAA	Usually large paintings might not go beautifully local appeals. Clothes bring partially different, very orders. Fruits provide except a schools. R	Home                                              	bedding                                           	33.55	1050.47	0.43435177709943549
+AAAAAAAABHLDAAAA	Well healthy	Home                                              	bedding                                           	7.46	10368.46	4.28718480945140073
+AAAAAAAACANCAAAA	Conditions know both popular	Home                                              	bedding                                           	2.48	18121.95	7.49312325626349635
+AAAAAAAACAPAAAAA	Payable, mutual pictures will not help new women; mole	Home                                              	bedding                                           	49.59	591.36	0.24451747018527152
+AAAAAAAACCKDAAAA	Increasingly sexual	Home                                              	bedding                                           	0.50	233.74	0.09664758096777828
+AAAAAAAACHPCAAAA	Thus angry stations would not demonstrate forward; single, political winds must not accept then dark profits. Patterns used to know obviously. Wars use particular met	Home                                              	bedding                                           	64.50	744.66	0.30790445641937955
+AAAAAAAACICEAAAA	Notes shall say slightly to a files. Important suggestions stay today acts. New, true powers make in particular; awkwardly left prices g	Home                                              	bedding                                           	0.79	546.70	0.22605130707232133
+AAAAAAAACIFBAAAA	About political men	Home                                              	bedding                                           	3.09	589.74	0.24384762727790521
+AAAAAAAACJACAAAA	Yet personal children answer; sp	Home                                              	bedding                                           	4.17	1458.28	0.60297439194699971
+AAAAAAAACJLDAAAA	Sacred, other police run competent, poor solutions. Just subsequent lips allow far all small sentences; programmes used to develop with a conditions. Properties m	Home                                              	bedding                                           	1.39	2951.80	1.22051993454559739
+AAAAAAAACMAAAAAA	Attractive, dead situations shall enter also great, forward groups; thus compatible sections give still troubles. Cold, known waters can ho	Home                                              	bedding                                           	5.95	634.78	0.26247091403579318
+AAAAAAAACOKDAAAA	New, hard children say needs. Particular, horrible sports can clean. Corporate, adminis	Home                                              	bedding                                           	8.14	2691.36	1.11283235010455958
+AAAAAAAACOMAAAAA	Female abilities remove hard, happy customs. Really current shoulders lead to a heads. Vast advantages ought to explai	Home                                              	bedding                                           	2.45	2906.03	1.20159480499611843
+AAAAAAAACOPBAAAA	Clearly profitable ages cancel above evolutionary lessons. Steps would live better; labour women can bounce inst	Home                                              	bedding                                           	3.09	4184.78	1.73033654437554205
+AAAAAAAADAEDAAAA	Usefully clinical hours laugh almost attractive instruments. Responsible, obvious results follow even powers. Away big cups should d	Home                                              	bedding                                           	9.21	12113.91	5.00889919381098232
+AAAAAAAADHMDAAAA	Of course political others should turn social, low charges. Thoughts must not expand. Prime letters will not correspond alone 	Home                                              	bedding                                           	3.60	3509.07	1.45094175984684579
+AAAAAAAAEKJDAAAA	Immediately legitimate details may not laugh over bad, great publications. Pale conditions cost high, commercial arms; new problems should gai	Home                                              	bedding                                           	1.16	272.24	0.11256668709963190
+AAAAAAAAELEEAAAA	Criminal faces can exercise always to a members. And so on likely lines can know patients. New premises used to top also yesterday physical relatives. Organisational, alone operations 	Home                                              	bedding                                           	93.25	255.70	0.10572767371207712
+AAAAAAAAFHAEAAAA	Expensive parents could become very over the implications; prominent reasons bring 	Home                                              	bedding                                           	92.94	4461.34	1.84468947922815077
+AAAAAAAAGDJBAAAA	Just joint transactions might take now still national tests. Cells vary less so orange texts	Home                                              	bedding                                           	6.63	7559.57	3.12575576990069165
+AAAAAAAAGFDDAAAA	Important, local transactions set overhead single prices. Available, white particles shall develop concerned, remote comments. Whole efforts m	Home                                              	bedding                                           	1.47	361.08	0.14930054135297930
+AAAAAAAAGFFEAAAA	Eager, low years shall report clearly. Others should operate since a meanings. Directors would know holes. Poor boundaries hear early hours. Important countries make of course small, rec	Home                                              	bedding                                           	2.90	15764.84	6.51849769121275679
+AAAAAAAAGKMBAAAA	Goods want special children. Personal plans remain. Payable, royal things go always concessions. Free, academic dogs raise still ra	Home                                              	bedding                                           	2.19	10328.90	4.27082741104682595
+AAAAAAAAGLLAAAAA	Public applications will include less elderly, double businessmen. Federal cards impose partners. Places pay completely. Quite old ways deny ac	Home                                              	bedding                                           	6.98	7984.50	3.30145721843597883
+AAAAAAAAHBFAAAAA	Good benefits pretend completely; s	Home                                              	bedding                                           	1.31	2239.67	0.92606608909944376
+AAAAAAAAHPMAAAAA	Ways become just together close centuries; shots account also perhaps lengthy profits. Both eastern efforts might grab together tight countries. Police will express today for	Home                                              	bedding                                           	1.95	405.51	0.16767160331241453
+AAAAAAAAIEACAAAA	Electronic, long-term theories would give especially; elderly forms know yet later old risks. Different m	Home                                              	bedding                                           	82.96	15743.55	6.50969463226347981
+AAAAAAAAIFMBAAAA	Double services precipitate finally demands. Authorities write early with a things. Full changes may not see in the doll	Home                                              	bedding                                           	4.48	1865.76	0.77146055731343376
+AAAAAAAAIIDEAAAA	Critical, whole men forget in a industries. Alone lips look soon for a natio	Home                                              	bedding                                           	5.35	3628.30	1.50024137086245375
+AAAAAAAAIJEBAAAA	Total, unlikely images get either open measures. Politicians visualise economically children. Able, ready states could not go in addition small 	Home                                              	bedding                                           	1.42	334.80	0.13843420085570364
+AAAAAAAAIJIAAAAA	Firm managers will not walk at a g	Home                                              	bedding                                           	3.23	1994.75	0.82479576510428565
+AAAAAAAAKNNCAAAA	There controversial beings upset sure at a arms. Broad circumstances see pale memb	Home                                              	bedding                                           	0.56	8534.56	3.52889782931617102
+AAAAAAAALCCBAAAA	Difficulties will not feel most. Like things used to avoid both favor	Home                                              	bedding                                           	0.82	2845.65	1.17662868478205813
+AAAAAAAALFMAAAAA	Special, true decades cannot convert cool normal, old-fashioned books. Old ministers become. Substantial, economic recordings see particularly patients. Mass, absolute thanks could not su	Home                                              	bedding                                           	3.58	8483.58	3.50781845189793992
+AAAAAAAAMAHAAAAA	Areas cannot get just. Horses achieve finally sad fans; tough examinations will not love also concrete mines. Experts shall l	Home                                              	bedding                                           	6.67	1746.36	0.72209065414087995
+AAAAAAAAMKFEAAAA	Questions will encourage finally final, small institutions. Additional holes enjoy alread	Home                                              	bedding                                           	4.45	7157.46	2.95949000972719407
+AAAAAAAAMLBCAAAA	Able, small executives would complete ne	Home                                              	bedding                                           	5.70	11277.99	4.66326025360996743
+AAAAAAAAMNNAAAAA	Shortly official associations find however weeks. Empty subjects draw much linguistic, whole powers. Typical, payable feet shall sink also narrow boys. Permanent, i	Home                                              	bedding                                           	4.13	10215.08	4.22376474455520053
+AAAAAAAAMOPAAAAA	Nevertheless left things must appear for instance again h	Home                                              	bedding                                           	6.76	6935.76	2.86782076740428637
+AAAAAAAANDMAAAAA	Enough lost problems manage before excellent champions	Home                                              	bedding                                           	0.97	425.46	0.17592059467164776
+AAAAAAAANEBBAAAA	Crude dates should convin	Home                                              	bedding                                           	9.48	2442.81	1.01006108181696956
+AAAAAAAANJBCAAAA	Personal, major issues say palestinian, german gods; angry styles keep surprising, pleased years. Authori	Home                                              	bedding                                           	8.78	375.34	0.15519681287090742
+AAAAAAAANLKAAAAA	Final off	Home                                              	bedding                                           	4.48	10411.01	4.30477852285167011
+AAAAAAAAOGEBAAAA	Children used to solve all right required, military a	Home                                              	bedding                                           	4.08	5342.86	2.20918325682169878
+AAAAAAAAOJADAAAA	Able, red references might hire so direct children. Experiments ban too different, labour met	Home                                              	bedding                                           	4.41	1941.93	0.80295557845793480
+AAAAAAAAOPMAAAAA	Then distant children plot. Previous roads w	Home                                              	bedding                                           	8.48	514.40	0.21269579725261037
+AAAAAAAAPILDAAAA	Powerful, happy companies seem also very national implications; children scan natural charts; really single subjects used to preserve. New re	Home                                              	bedding                                           	1.99	9617.02	3.97647693641971033
+AAAAAAAAACJDAAAA	Slight, royal projects will ask audiences. Elabora	Home                                              	blinds/shades                                     	5.27	7981.68	2.95699007390289399
+AAAAAAAAADAEAAAA	Years say much at a eyes; surely different theories may hear well powerful, free wars. Well little conservatives weave physical, fundamental servants; c	Home                                              	blinds/shades                                     	4.42	1284.84	0.47599742492224623
+AAAAAAAAAJAEAAAA	States must not harm maybe late changes. Good, original steps must abandon incredible, useful neighbours. Sure annual shareholders could analyse absolutely patently dark 	Home                                              	blinds/shades                                     	7.32	10474.36	3.88045856893354741
+AAAAAAAAANNDAAAA	Very able governments m	Home                                              	blinds/shades                                     	2.20	7440.10	2.75634977208368684
+AAAAAAAAAOADAAAA	Companies want as from the reports. Often different purposes will not work cases; principal towns guess 	Home                                              	blinds/shades                                     	9.34	5385.32	1.99511102735147651
+AAAAAAAAAPLCAAAA	Cells cannot give. Indeed english trees shall talk ever. In particular foreign things may catch too soviet, rich situations. N	Home                                              	blinds/shades                                     	0.28	8695.50	3.22144049719139513
+AAAAAAAABDJDAAAA	Tiny, exi	Home                                              	blinds/shades                                     	7.04	7025.12	2.60261124324411636
+AAAAAAAACBDBAAAA	Women must sleep in a scales. Agents can get generally extraordinary, general studies. Central systems profit; either comprehensive rivers use in the cars; cases shall ke	Home                                              	blinds/shades                                     	0.63	5940.92	2.20094534857964501
+AAAAAAAACBFCAAAA	Theories employ more specific offenders. Modes must see preferences. Certainly main studies see from the varieties. Pleasant elements 	Home                                              	blinds/shades                                     	97.19	4156.26	1.53977853842294381
+AAAAAAAACEIAAAAA	Young opinions make fine weeks; copies would reply soon from the accountants. Interesting losses would go only slightly old families. Most famous patterns ex	Home                                              	blinds/shades                                     	2.76	8530.68	3.16037927900416200
+AAAAAAAADAMBAAAA	Industrial losses take letters; organic, likely yards could know all possible questions. Old studies retrie	Home                                              	blinds/shades                                     	9.59	8586.88	3.18119981329686010
+AAAAAAAADBMDAAAA	New, light associations feel on a officials. Potential, enormous customers find in the me	Home                                              	blinds/shades                                     	4.62	4568.78	1.69260570579703321
+AAAAAAAADDKDAAAA	Certainly tory systems read now in a prisons; evenings may seduce anywhere other months; new customers talk with the cells. Police lead more other exports. Young provisions 	Home                                              	blinds/shades                                     	7.50	11150.34	4.13089032642781908
+AAAAAAAADFLDAAAA	Common, interesting figures would not see high; naked studies would get both. Changes might face over prayers. Tremendous, intact considerations shall choose just.	Home                                              	blinds/shades                                     	1.19	3490.71	1.29321080535345580
+AAAAAAAADHECAAAA	True, impossible trees could get no longer exclusive cel	Home                                              	blinds/shades                                     	7.65	13982.16	5.18000074316711372
+AAAAAAAADPDEAAAA	Less whole balls should purchase often difficult specialists. Impossible, international intentions will not counter completely during a trees. Important sciences look initia	Home                                              	blinds/shades                                     	0.25	4673.99	1.73158307969266965
+AAAAAAAAEBEEAAAA	National, electric sections must market in the decisions; b	Home                                              	blinds/shades                                     	3.94	13578.70	5.03053005338540591
+AAAAAAAAECOAAAAA	Thin, financial others can mobilize from a stories. Anywhere related others should remain following patients. Equations sh	Home                                              	blinds/shades                                     	5.47	1070.00	0.39640519027023090
+AAAAAAAAEDHAAAAA	Steep, slow terms get. Affairs will decide upwards dominant courts. Familiar, serious years add	Home                                              	blinds/shades                                     	2.80	2331.69	0.86382618514130345
+AAAAAAAAFFOBAAAA	Available laws get worldwide waste, new policies; then other societies understand by a interests; often local problems can help whole hours. Certain, m	Home                                              	blinds/shades                                     	8.96	9879.49	3.66007580675032100
+AAAAAAAAFLMAAAAA	Clear accounts will not play even spectacular offices. Christian, impossible changes say for ins	Home                                              	blinds/shades                                     	0.25	7864.42	2.91354851071496196
+AAAAAAAAGBKCAAAA	Rural, top years must accept again unusual shelves. Directors used to move later known, form	Home                                              	blinds/shades                                     	4.05	3163.86	1.17212198625081564
+AAAAAAAAGDMBAAAA	Healthy directors understand at least young conditions. Excellent members prevent well meetings. Obvious	Home                                              	blinds/shades                                     	4.77	821.24	0.30424654061450881
+AAAAAAAAGOODAAAA	Thoughts must not achieve forward from the eyes. Powers seem recent	Home                                              	blinds/shades                                     	1.53	8071.29	2.99018808240767473
+AAAAAAAAHLJBAAAA	Services must move amongst a bedrooms. Small markets used to advance in a courses. New levels could say from a centres. In particular present buyers must not transfer again. Indian, net courses s	Home                                              	blinds/shades                                     	0.19	3825.58	1.41727081102242049
+AAAAAAAAIBCCAAAA	Different, upper days receive thorough, personal couples. Social, new girls must not prove strangely in a words; feet shall help however now full th	Home                                              	blinds/shades                                     	4.79	7716.79	2.85885570862188328
+AAAAAAAAIHMCAAAA	Scarcely crucial groups may bring especially effective, important losses. Now new drugs wan	Home                                              	blinds/shades                                     	3.48	2706.56	1.00270507642784686
+AAAAAAAAIJMCAAAA	Short candidates shed women. Involved, wooden needs might violate then long-term times. Students must not	Home                                              	blinds/shades                                     	5.18	NULL	NULL
+AAAAAAAAIODEAAAA	Only normal subjects might create over in the teachers. Main hours used t	Home                                              	blinds/shades                                     	4.63	2891.18	1.07110164299578147
+AAAAAAAAJBMDAAAA	Bars like full, traditional politicians. Things used to show properly at the holidays; less specific relations may say possibly. Forces could 	Home                                              	blinds/shades                                     	6.30	144.44	0.05351099596507678
+AAAAAAAAKALCAAAA	Prime, international results test ever conditions. Territorial users should love never barely emotional detectives. Firms resi	Home                                              	blinds/shades                                     	3.79	5465.05	2.02464877110871531
+AAAAAAAAKCPBAAAA	Conditions make patients. New, various eggs will not watch appropri	Home                                              	blinds/shades                                     	2.22	360.68	0.13362189161370737
+AAAAAAAAKGECAAAA	Already early meetings cannot go animals. As comprehensive evenings w	Home                                              	blinds/shades                                     	4.11	511.70	0.18957059426287584
+AAAAAAAAKHIAAAAA	Serious, free symptoms used to remember certainly able runs. Feelings shall pro	Home                                              	blinds/shades                                     	5.48	2291.60	0.84897395703108517
+AAAAAAAAKJMDAAAA	Also long lines make further near a dogs. Rather foreign jobs can sit in the trends. Chronic police shall experience apparently diverse, proper years. Only notable companies migrate also years. Free,	Home                                              	blinds/shades                                     	73.55	6931.61	2.56796839339162169
+AAAAAAAALGLDAAAA	Complete costs become again industrial theories. Populations vary trustees. Countr	Home                                              	blinds/shades                                     	3.42	4143.26	1.53496240059723073
+AAAAAAAAMIMDAAAA	P	Home                                              	blinds/shades                                     	2.11	8507.90	3.15193992364495091
+AAAAAAAANDPDAAAA	Minutes might not reply polish, main days. Main beans make properly agencies. As new years de	Home                                              	blinds/shades                                     	9.78	8403.34	3.11320335664060012
+AAAAAAAANEDCAAAA	Lives would look. Things exist for a patterns. Local, palestinian members should get problems; statements may not make yet nasty, specific men; numbers find clear women. Groups shall seem m	Home                                              	blinds/shades                                     	3.38	2112.47	0.78261128251416324
+AAAAAAAANILCAAAA	Appropriate, extensive scenes stem openly now financial personnel. More concerned signs stay now members; also full days could prepare subtle horses. Ancient achievements say america	Home                                              	blinds/shades                                     	2.98	14371.92	5.32439596462480082
+AAAAAAAAOADAAAAA	Primary, occupational regions set particularly here prime ideas. Clinical, sophisticated minutes allocate just. Needs want interested let	Home                                              	blinds/shades                                     	4.77	5863.19	2.17214854910328515
+AAAAAAAAOCKAAAAA	Large colours must win over; months assess extreme days. Blacks might signify then fully concerned points; here political potatoes might not die 	Home                                              	blinds/shades                                     	0.55	3969.07	1.47042985845407977
+AAAAAAAAOEBAAAAA	Sad increases ought to mean too levels. Organs used to present; other, sympathetic controls like always new interests. Other, small women deal in a edges. Outcomes run 	Home                                              	blinds/shades                                     	8.43	7535.76	2.79178913703812636
+AAAAAAAAOFBCAAAA	New parts come also old, tiny chains; responsible seats involve now properly small secrets; eligible chains get complete communications. Talks beat about married, liable books. Big,	Home                                              	blinds/shades                                     	7.11	1861.92	0.68978948772705450
+AAAAAAAAOLLCAAAA	Social, central lights warn police. 	Home                                              	blinds/shades                                     	7.78	6660.62	2.46757414805393022
+AAAAAAAAOMOBAAAA	Subjects sha	Home                                              	blinds/shades                                     	0.26	360.45	0.13353668302140629
+AAAAAAAAPBLBAAAA	Free, educational times ensure practically. So linguistic officers need. N	Home                                              	blinds/shades                                     	9.32	4744.02	1.75752724368764560
+AAAAAAAAPMHAAAAA	Just possible women say. Reasonably strong employers light almost degrees. Palestinian, smart rules help really visual	Home                                              	blinds/shades                                     	3.71	8398.39	3.11136951954542476
+AAAAAAAAAAACAAAA	Labour taxes could get even lab	Home                                              	curtains/drapes                                   	4.54	24984.53	7.47827965622433549
+AAAAAAAAAIACAAAA	All real copies loosen more definite doors.	Home                                              	curtains/drapes                                   	9.49	736.67	0.22049741477429358
+AAAAAAAABIJBAAAA	Very, various goods should turn local arran	Home                                              	curtains/drapes                                   	3.04	3989.59	1.19414972919947050
+AAAAAAAACEHAAAAA	Unlikely sides sell sometimes friends; mutual floors used to say i	Home                                              	curtains/drapes                                   	3.70	11830.01	3.54091604348492652
+AAAAAAAACIHAAAAA	Roads help less functions. Relevant, increased procedures may not respond. All labour children ought to say workers. Given findings could decide thus royal shareholders	Home                                              	curtains/drapes                                   	4.28	5979.42	1.78973848785712263
+AAAAAAAACJNBAAAA	Weak girls swim; provinces may introduce. Nervous, green tracks say better british, public rebels. Houses must not s	Home                                              	curtains/drapes                                   	8.21	9746.45	2.91727235835165499
+AAAAAAAACLCBAAAA	Mainly alternative politicians will not maintain from a matters. Principles should not tell always details; suddenly democratic years formulate far. Western, wise years ge	Home                                              	curtains/drapes                                   	2.73	3116.99	0.93296623573285915
+AAAAAAAACLDDAAAA	Public metres want; characteristics shoul	Home                                              	curtains/drapes                                   	0.82	6428.18	1.92405971697478996
+AAAAAAAACPAAAAAA	Services decide only easy, single bases. Now british solicitors ought to transfer now over a drawings. Thorough elections run still religious, tough parameters. Complete, sole consequences ac	Home                                              	curtains/drapes                                   	4.49	6448.14	1.93003407238344634
+AAAAAAAAECKDAAAA	New, intimate hours go unfortunately forms. Subsequently experienced advisers must feed n	Home                                              	curtains/drapes                                   	0.70	188.16	0.05631937443350629
+AAAAAAAAEJKCAAAA	Words might correct long old, major relations. Visible, desperate policemen may become extra agreements. General, other students include so	Home                                              	curtains/drapes                                   	3.90	10122.80	3.02992008671076475
+AAAAAAAAEOLAAAAA	Centres look nevertheless with a advertisements. Naked users address to a reports. Im	Home                                              	curtains/drapes                                   	3.82	6381.83	1.91018640168464850
+AAAAAAAAEPFEAAAA	Clear partners ought to take effective, black books. Circumstances become hospitals. Forces answer gradua	Home                                              	curtains/drapes                                   	1.32	1013.02	0.30321350280947356
+AAAAAAAAFBMBAAAA	Certain, conservativ	Home                                              	curtains/drapes                                   	0.28	11983.75	3.58693294731893617
+AAAAAAAAFGEBAAAA	Private years forgive then in the computers; more exclusive differences get sources. Minutes meet insects. Small circumstances will contact sudd	Home                                              	curtains/drapes                                   	1.69	2179.00	0.65221044265843012
+AAAAAAAAFHHDAAAA	Known, possible years may approve. Forth wrong aspects see again local girls. Excellent peasants can run usually with a exchanges;	Home                                              	curtains/drapes                                   	3.79	4760.53	1.42490471711277482
+AAAAAAAAGALDAAAA	Prime, national features claim different, great views. Versions might not sign european 	Home                                              	curtains/drapes                                   	0.67	9131.87	2.73331848324884729
+AAAAAAAAGAPBAAAA	Free funds cause still new,	Home                                              	curtains/drapes                                   	4.69	8170.69	2.44562154278329893
+AAAAAAAAGBIBAAAA	Years must not enable existing others; other, political ties like then short products. Quite	Home                                              	curtains/drapes                                   	4.35	696.96	0.20861156040166106
+AAAAAAAAGCMDAAAA	Private parents carry really british dreams; writings look probab	Home                                              	curtains/drapes                                   	9.60	2216.28	0.66336895817119114
+AAAAAAAAGJDBAAAA	Responses used to bring of course video-taped loans. Hot, positive systems would remember. New, personal words may not answer on	Home                                              	curtains/drapes                                   	6.31	2854.74	0.85447050898335328
+AAAAAAAAGJJCAAAA	Germans will throw perhaps with a	Home                                              	curtains/drapes                                   	6.68	11036.19	3.30331269626550706
+AAAAAAAAGNDAAAAA	Generally left questions bri	Home                                              	curtains/drapes                                   	93.18	2295.48	0.68707481730774354
+AAAAAAAAGPACAAAA	Particular, british wa	Home                                              	curtains/drapes                                   	3.20	6421.72	1.92212613300986409
+AAAAAAAAHCDEAAAA	Democratic, likely appearances might expand both good, certain pounds; american values can pick. Only previous figures will not repa	Home                                              	curtains/drapes                                   	6.11	15070.04	4.51071016947234888
+AAAAAAAAHEFAAAAA	Different, local measures say there political doors. Open assets progress minus th	Home                                              	curtains/drapes                                   	9.40	2024.63	0.60600496949037970
+AAAAAAAAHILCAAAA	Statements might not test financial, concerned authorities. United scenes back just bare publishers. More simple things could cope 	Home                                              	curtains/drapes                                   	0.37	4710.47	1.40992093796661557
+AAAAAAAAHLFCAAAA	Accountants look equally marvellous, british schemes. Things shall study tiny events. Both normal courses could appeal faintly. Then black practices used to die hardly. Advisor	Home                                              	curtains/drapes                                   	2.23	9441.66	2.82604371180834938
+AAAAAAAAIBDBAAAA	Valid resources ought to say still tears. M	Home                                              	curtains/drapes                                   	1.25	8697.98	2.60344808904734832
+AAAAAAAAIGCDAAAA	Electronic reports try in comparison with the problems. Germans might not go as. Common, social cups come sure about intact	Home                                              	curtains/drapes                                   	3.25	817.84	0.24479292722522739
+AAAAAAAAILHDAAAA	Outside mammals can ignore eyes. Amounts stand always that is ready notes. Structures remember most attractive issues. Subjective difficulties cause very. Adequate, di	Home                                              	curtains/drapes                                   	1.51	3062.90	0.91677621148164553
+AAAAAAAAILJAAAAA	Small females would allow topics; local, local tears find	Home                                              	curtains/drapes                                   	0.60	123.41	0.03693863732376175
+AAAAAAAAIMIAAAAA	Problems must not hate there in a stars. Fully forward teams may work yet white, concerned personnel. Merely common years stem methods; measures could introduce more on a areas. L	Home                                              	curtains/drapes                                   	3.73	15982.27	4.78375557199933360
+AAAAAAAAJOCAAAAA	Here other years may like later. Terms call yesterday also	Home                                              	curtains/drapes                                   	1.50	1201.77	0.35970947392089103
+AAAAAAAAKCGBAAAA	Free, competitive aspects get even specific, medical times. Other, free days 	Home                                              	curtains/drapes                                   	4.40	3406.63	1.01966023876708940
+AAAAAAAAKNBBAAAA	National features sing then really magnificent values. Light, shallow advertisements should acknowledge. Possible, good forms should move anyway political, irish estates. Simply	Home                                              	curtains/drapes                                   	2.02	2017.71	0.60393369997996376
+AAAAAAAAKPLCAAAA	Linguistic, appropriate degrees shout. Educational poles will study now in a names. Full arms look in a ways. Minute, modest systems deal unique experiments; automatically regular 	Home                                              	curtains/drapes                                   	2.54	6407.34	1.91782196313128299
+AAAAAAAALGACAAAA	Active books find; important, remarkable personnel may turn alone prices; public eyes administer different, financial waters. Obvious, weekly managers cannot make so. Proble	Home                                              	curtains/drapes                                   	8.93	25.68	0.00768644523518517
+AAAAAAAALIMAAAAA	Socially extra interpretations continue other men. Also odd initiatives must need now by a hills. So gross rules can divide. Significant, impossible parent	Home                                              	curtains/drapes                                   	4.37	100.62	0.03011721649393815
+AAAAAAAALJODAAAA	Effects might tolerate reasonably. Comparisons take other, clear others. French, christian 	Home                                              	curtains/drapes                                   	1.91	6527.01	1.95364115710692977
+AAAAAAAAMBMBAAAA	New, different elections kill arms. As good as new yards would calcula	Home                                              	curtains/drapes                                   	0.59	4150.32	1.24225885469212285
+AAAAAAAAMKHCAAAA	Events explore away. Unusual rights should affect so in a posts. New journalists might not find wrong scientists. For example tall authorities shall not con	Home                                              	curtains/drapes                                   	6.84	1245.00	0.37264892203292588
+AAAAAAAAMMKBAAAA	Tall, whole women would not create. Still national hands bear around flat, poor attacks. Fiel	Home                                              	curtains/drapes                                   	6.19	2226.86	0.66653572571746292
+AAAAAAAAMOCAAAAA	Months shall not find also intact forces; super ju	Home                                              	curtains/drapes                                   	0.99	6731.10	2.01472864184403808
+AAAAAAAANAJAAAAA	Superbly loyal police would contemplate twice sure nights. Even succ	Home                                              	curtains/drapes                                   	0.44	49.08	0.01469044907098474
+AAAAAAAANIJBAAAA	Legs solve by a women. Early, early weekends neglect again loans; proposals	Home                                              	curtains/drapes                                   	57.92	10980.48	3.28663777944104577
+AAAAAAAANOABAAAA	Likely, normal policies believe very children. Twice old knees should suggest with a details. Lives take students; questions will not look as deeply ready areas; valuable members wor	Home                                              	curtains/drapes                                   	5.17	249.22	0.07459563401529782
+AAAAAAAAOACEAAAA	Budgets keep so lesser women. Stairs determine 	Home                                              	curtains/drapes                                   	1.55	4402.52	1.31774645158907378
+AAAAAAAAOCCCAAAA	Di	Home                                              	curtains/drapes                                   	6.03	5657.98	1.69352622319988272
+AAAAAAAAOJKBAAAA	Particularly old assumptions might learn repeatedly fine sessions; payments compete more bad times. Days will plan formerly; all right simple jeans reject weeks. Today national representati	Home                                              	curtains/drapes                                   	24.89	14029.64	4.19930138354218335
+AAAAAAAAOLJDAAAA	Goals commit then obvious tracks. Excellent days k	Home                                              	curtains/drapes                                   	6.14	1920.32	0.57478327546848854
+AAAAAAAAPCIAAAAA	Human drinks	Home                                              	curtains/drapes                                   	0.71	1522.69	0.45576609404844651
+AAAAAAAAPDGBAAAA	Dead, obvious terms would serve more through a forces; worthy, possible arms decide for the falls. Rules 	Home                                              	curtains/drapes                                   	2.34	14312.02	4.28382234948889629
+AAAAAAAAPGKDAAAA	Small branches cause smoothly right duties. Outstanding ministers give real policies. Increased, japanese settlements used to protect electoral, large offices; clouds 	Home                                              	curtains/drapes                                   	3.90	15202.77	4.55043843567430089
+AAAAAAAAPHNCAAAA	Specific, small functions make about a children. Other, hot notes request however new things. Very slight eyes should want always serious, normal	Home                                              	curtains/drapes                                   	6.32	1409.34	0.42183857974127210
+AAAAAAAAPJBAAAAA	Somehow surprising officials eat important cells. Mature police operate close, permanent flights. Old, fine engineers will pay away fingers. Hardly cultural activities watch gay, new 	Home                                              	curtains/drapes                                   	0.25	6118.86	1.83147516712481033
+AAAAAAAAPLBBAAAA	New, perfect clothes let. High centuries could go months. Part-time, legal things think even there new systems. Aware losses come yet that wide functions. Big, british ears send please economic hee	Home                                              	curtains/drapes                                   	7.09	4208.63	1.25971199416500631
+AAAAAAAAPOFDAAAA	Less than dark patients teach however national senses; as positive problems can take instead liberal collective sectors; urgent resources raise so southern motives. Private p	Home                                              	curtains/drapes                                   	0.67	7346.83	2.19902673081057097
+AAAAAAAAABDDAAAA	Still available arguments	Home                                              	decor                                             	6.57	7479.82	2.46562464048976131
+AAAAAAAAAJBBAAAA	Then adequate experiments ought to need pp.. Able unions could need please on a countries. Women continue previously british ways.	Home                                              	decor                                             	0.96	3319.93	1.09437141705297364
+AAAAAAAAAJIAAAAA	Now imaginative boys shall look. Experiments tell main confl	Home                                              	decor                                             	3.59	1502.18	0.49517395103771343
+AAAAAAAAALBBAAAA	Independent, limited numbers claim nonetheless to a firms; never managerial sources would want only special terms. Changing, present homes shall suffer 	Home                                              	decor                                             	6.24	1843.18	0.60758013225691504
+AAAAAAAAANIBAAAA	Fre	Home                                              	decor                                             	2.65	4396.90	1.44938046393755886
+AAAAAAAAAPCCAAAA	Wonderful, brief ships continue; less vital o	Home                                              	decor                                             	9.80	3685.64	1.21492292594937898
+AAAAAAAABAECAAAA	Perhaps spanish	Home                                              	decor                                             	7.44	2152.90	0.70967527139829663
+AAAAAAAABHJCAAAA	Regional circumstances see really matters. Again sexual years secure adjacent trials. Old animals will solve new, necessary eyes. Level views migh	Home                                              	decor                                             	7.80	157.04	0.05176617800194552
+AAAAAAAABLCCAAAA	Old fruits tak	Home                                              	decor                                             	2.26	7882.54	2.59837601087274335
+AAAAAAAABPCEAAAA	Parliamentary, favorite months escape almost necessary, environmental beliefs; closely high doctors used to run far exact contributions. Kinds accept never european trades. Sorry, great tho	Home                                              	decor                                             	2.64	8778.45	2.89370100153577829
+AAAAAAAACFLCAAAA	Much red years would not repeat by the others. Particularly environ	Home                                              	decor                                             	1.45	2736.60	0.90208432705122327
+AAAAAAAACIICAAAA	Sol	Home                                              	decor                                             	1.01	9042.00	2.98057680523173309
+AAAAAAAACJDDAAAA	Schemes wield usually other 	Home                                              	decor                                             	1.43	5016.00	1.65345866567599792
+AAAAAAAACOGBAAAA	Helpful, very colleagues shall provide members. Concessions go other, tired eyes. Accurate windows ride slowly different hours. Speciali	Home                                              	decor                                             	1.48	2381.42	0.78500389465991526
+AAAAAAAACPLBAAAA	Frequently small crimes spend as primary regions; exactly small students simplify very. Early workers make interpretations. Late direct pensioners ca	Home                                              	decor                                             	2.82	6192.37	2.04123361993063780
+AAAAAAAADAOCAAAA	Maps form houses. Whole assumptions used to know for a premises; black titles 	Home                                              	decor                                             	5.19	6005.87	1.97975633899990144
+AAAAAAAADDGBAAAA	Contacts choose to the governments. Over angry contracts could sell as yet national technical tables; violent, toxic patterns cannot express solid crops. Feet shall use	Home                                              	decor                                             	9.88	1269.31	0.41841140728253607
+AAAAAAAADFNBAAAA	Formerly prime men influence incentives; new bars support children. Machines end certainly so economic drawings; other, christian eff	Home                                              	decor                                             	2.26	5503.23	1.81406765006142784
+AAAAAAAADGMBAAAA	As 	Home                                              	decor                                             	2.03	7855.62	2.58950218565743277
+AAAAAAAAEAFEAAAA	Long-term st	Home                                              	decor                                             	8.22	2874.12	0.94741599286138340
+AAAAAAAAEGMDAAAA	Contemporary feet used to go still political, late lives. Statutory, scottish genes must smell. Good lips establish quite. Old women must avoid with the places. Too wet l	Home                                              	decor                                             	4.58	710.24	0.23412130835520749
+AAAAAAAAEKLAAAAA	Citizens can keep for the most part at the things. Branches visit terms. Available, slight problems may avoid. Problems help more. Social years feel inherent acres. Individuals use	Home                                              	decor                                             	49.10	5668.87	1.86866870536098372
+AAAAAAAAFAPCAAAA	Workers shall not control never on a studies. Sophisticated activities go separately according to a bodies; co	Home                                              	decor                                             	40.34	2145.78	0.70732825670539131
+AAAAAAAAFBLCAAAA	Prematurely other systems assume nearly under w	Home                                              	decor                                             	0.88	9056.13	2.98523457455908593
+AAAAAAAAFHDEAAAA	Always cool temperatures meet there social grounds. Threats h	Home                                              	decor                                             	5.44	3350.86	1.10456708621751882
+AAAAAAAAFIPAAAAA	Too complete events try environmental, national topi	Home                                              	decor                                             	3.31	7994.82	2.63538764145131214
+AAAAAAAAGCLBAAAA	Fresh, beautiful functions give empty, fast origins. Sons get other companies. Lights say delightful, native services. Small, soviet things could go already also dead systems. Medical, comm	Home                                              	decor                                             	34.78	11689.03	3.85313555559144935
+AAAAAAAAHFLCAAAA	Resulting, distinct clients shall tell intellectually difficult gardens. Villages turn then by a things; fresh, supreme powers succeed here. Historical hands st	Home                                              	decor                                             	4.30	269.93	0.08897888708650760
+AAAAAAAAHJGDAAAA	Possible shoes render undoubt	Home                                              	decor                                             	8.28	13638.47	4.49574290431860593
+AAAAAAAAIELCAAAA	However old figures ask only good, large sources. Yet naked researchers shall deal to a women. Right, common miles describe there also prime bags. Readily significant shares	Home                                              	decor                                             	7.78	NULL	NULL
+AAAAAAAAIIMDAAAA	Relatively regional months wish then needs. Eyes follo	Home                                              	decor                                             	66.29	7883.31	2.59862983128194800
+AAAAAAAAIJFAAAAA	Deposits shall leave more skills. Close ce	Home                                              	decor                                             	5.30	5555.19	1.83119558312931557
+AAAAAAAAIOFEAAAA	Regular findings put. Little, national cattle should say most mothers. Asleep eyes stay over thoughts. Western, golden walls might not move distinct, small boxes. Swiss, go	Home                                              	decor                                             	3.83	3030.40	0.99893164682307498
+AAAAAAAAJDHDAAAA	Gentlemen work always. Religious, spiritual variations think fairly so electronic resources. Diplomatic, civil others split both mathematical, new contacts. Ultimate	Home                                              	decor                                             	9.53	6205.11	2.04543319397384199
+AAAAAAAAJLFAAAAA	There final techniques wear so old winners. Old, particular prices will return especially motives. Around early members shall pay systems. Unions call rather. Else old ter	Home                                              	decor                                             	2.10	13195.83	4.34983242908439067
+AAAAAAAAKAFCAAAA	Similar, ready forces play often arms. Marrie	Home                                              	decor                                             	7.68	7302.41	2.40714375893522009
+AAAAAAAAKFEAAAAA	Nearly delighted services know then eventually political p	Home                                              	decor                                             	0.48	4915.69	1.62039278873142867
+AAAAAAAALIPAAAAA	Top modules ought to go. Funds shall offer in	Home                                              	decor                                             	4.71	13454.30	4.43503367735338493
+AAAAAAAAMBPDAAAA	Important rights justify now still e	Home                                              	decor                                             	53.89	3370.57	1.11106422941936768
+AAAAAAAAMGEDAAAA	Fields divorce hardl	Home                                              	decor                                             	1.25	14250.34	4.69743783130568185
+AAAAAAAAMIBCAAAA	Able, assistant positions should die	Home                                              	decor                                             	4.24	3308.46	1.09059048186650958
+AAAAAAAAMJOAAAAA	British, electric ye	Home                                              	decor                                             	4.13	6855.95	2.25997407076183372
+AAAAAAAAMMDEAAAA	Immediate designs reward more speedily expected things. Good, happy feet create interesting, political signals. Still general stations help. Remote, flat ideas ma	Home                                              	decor                                             	0.10	6799.02	2.24120784232544325
+AAAAAAAAMOPCAAAA	Sa	Home                                              	decor                                             	2.03	474.81	0.15651489414864844
+AAAAAAAANGFBAAAA	Minutes must not reduce in addition conditions. Australian, likely methods miss on a grou	Home                                              	decor                                             	25.40	111.84	0.03686659034473756
+AAAAAAAANJGDAAAA	Quickl	Home                                              	decor                                             	9.23	2919.06	0.96222987492587290
+AAAAAAAAOAEEAAAA	Abroad great methods can call all labour main clubs. Minerals may make often countries. Apparently good pairs used to write terrible accounts; able funds close again with the times; earlier average 	Home                                              	decor                                             	4.93	5327.91	1.75627570961758494
+AAAAAAAAOBJBAAAA	Minor, usual members come more before good waters. Circumstances cannot take interests	Home                                              	decor                                             	0.15	15519.10	5.11566793829592889
+AAAAAAAAOBPCAAAA	Present, responsible rates contribute at all records. Eyes ought to wait political, national awards. Politically int	Home                                              	decor                                             	0.18	20899.05	6.88909795193300723
+AAAAAAAAOCEAAAAA	Nations realize on a shadows. Managerial, disabled systems stay between the councils. Capitalist girls might live 	Home                                              	decor                                             	4.02	1089.18	0.35903391337340180
+AAAAAAAAOPADAAAA	Military issues face rather once previous thanks. Then famous sources ought to transport boats; readily impossible requirements trust again with 	Home                                              	decor                                             	5.27	7325.56	2.41477485305611310
+AAAAAAAAPHLBAAAA	Private, direct rates increase furious meals. Italian values buy for instance random members. Available reforms work financial, impossible adults. Immediate, good experimen	Home                                              	decor                                             	6.40	7796.60	2.57004701611034397
+AAAAAAAAPNPDAAAA	So far conditions may r	Home                                              	decor                                             	8.95	1175.16	0.38737609361160401
+AAAAAAAAABCBAAAA	Suspiciou	Home                                              	flatware                                          	8.91	11913.78	5.12961692885790892
+AAAAAAAAABCCAAAA	Material, rough relations think cities. As popular studies should not ask at a boo	Home                                              	flatware                                          	0.28	1925.64	0.82910676064909237
+AAAAAAAAABGBAAAA	Real times could cultivate honours. Great carers enter like a drugs. Sufficient years read o	Home                                              	flatware                                          	3.21	32.10	0.01382102938079593
+AAAAAAAAAFCAAAAA	Long, other grounds give now clinical, essential areas. Possible languages make. So similar costs would say. More similar propos	Home                                              	flatware                                          	3.20	180.81	0.07784985427855798
+AAAAAAAAALOBAAAA	Present variables shall raise royal, american structures. 	Home                                              	flatware                                          	1.03	26390.07	11.36255242464987910
+AAAAAAAABDAAAAAA	Remarkable m	Home                                              	flatware                                          	20.08	15671.25	6.74743946055445923
+AAAAAAAABJOBAAAA	Changes like old, perfect streets. Thousands say. Whole factors work particular	Home                                              	flatware                                          	1.83	3396.31	1.46232088150439278
+AAAAAAAACBAEAAAA	Police succeed schools; supplies calculate far countries; new words move shares; officers must complete years. Asian things may bear warm things. Aw	Home                                              	flatware                                          	6.66	2788.28	1.20052647357899259
+AAAAAAAACBHAAAAA	Suppo	Home                                              	flatware                                          	2.16	18092.16	7.78979049601435527
+AAAAAAAACEDEAAAA	Streets will marry. Agencies tell regularly students. Years study here colonial, economic transactions. Cards shall not hide of course inside technical sons; else environmental 	Home                                              	flatware                                          	58.71	3036.50	1.30740048955722201
+AAAAAAAACLBAAAAA	Early, particular conditions fulfil just women. All new sales might not feel large, active books; current children should take. Generally di	Home                                              	flatware                                          	14.12	22.62	0.00973930481600012
+AAAAAAAACNCAAAAA	Foreign parties could not keep ston	Home                                              	flatware                                          	1.70	4789.08	2.06199424881564327
+AAAAAAAACPDAAAAA	Patient 	Home                                              	flatware                                          	1.87	9772.43	4.20763371189319384
+AAAAAAAADMMBAAAA	Years know more medical citizens. Then comprehensive observers come finally by a processes. Small voters must waste others. Statistical levels study. Ex	Home                                              	flatware                                          	0.33	741.75	0.31936911349549462
+AAAAAAAAEGHDAAAA	Arrangements keep simply close large terms. Projects might not live true, easy others. So new years take labour members. Original towns travel away away americ	Home                                              	flatware                                          	9.19	2252.25	0.96973250538621876
+AAAAAAAAEHGDAAAA	Possible, thick acids shall not go in a c	Home                                              	flatware                                          	3.98	5764.14	2.48181770389473594
+AAAAAAAAEIODAAAA	Random influences can force low for a subjects; young days will not travel historic hills. Unlikely, huge guards arrest now by th	Home                                              	flatware                                          	3.46	5434.00	2.33967207648738495
+AAAAAAAAEPMAAAAA	Domestic, new tasks show here very various farms. Internal, old homes used to impose long traditional, high 	Home                                              	flatware                                          	1.93	627.94	0.27036689063479730
+AAAAAAAAFCHCAAAA	More special scots ought to see just on a pupils. Grounds might shut complex writers. Empty, actual eyes may get little wrong, odd words; social, full tact	Home                                              	flatware                                          	3.31	2123.58	0.91433213621403771
+AAAAAAAAGDLDAAAA	Legal ci	Home                                              	flatware                                          	4.71	5052.16	2.17526641110535642
+AAAAAAAAGEADAAAA	Hom	Home                                              	flatware                                          	8.19	3362.38	1.44771192428039261
+AAAAAAAAGOCAAAAA	Leaves cannot lose colours; european, dynamic sentences will 	Home                                              	flatware                                          	96.77	1428.58	0.61509178046160258
+AAAAAAAAGPEAAAAA	Further o	Home                                              	flatware                                          	5.51	11480.35	4.94299858728412768
+AAAAAAAAHLBAAAAA	Thus internal planes would not apply never rather than a	Home                                              	flatware                                          	2.06	4826.77	2.07822211789234727
+AAAAAAAAINBEAAAA	European seconds wou	Home                                              	flatware                                          	5.97	12128.66	5.22213601899328053
+AAAAAAAAJAAEAAAA	Labour, likely area	Home                                              	flatware                                          	84.74	7106.28	3.05969173421066874
+AAAAAAAAJBGDAAAA	Particular, healthy talks get written, apparent months; then great attacks used to secure characteristically to a agencies. Accounts answer prod	Home                                              	flatware                                          	3.87	179.28	0.07719109493423967
+AAAAAAAAJIKBAAAA	Yesterday angry obligations defi	Home                                              	flatware                                          	3.77	1418.04	0.61055366053407644
+AAAAAAAAJMNDAAAA	European, rigid voters believe in common including a meetings. Complete trends broadcast directly;	Home                                              	flatware                                          	2.19	10595.74	4.56211943461914690
+AAAAAAAAKEECAAAA	Likely, odd offences shall ease enough true, chinese eyes. Other indi	Home                                              	flatware                                          	4.09	3818.90	1.64427193465176194
+AAAAAAAAKFCDAAAA	Left, white ways might intervene es	Home                                              	flatware                                          	9.19	416.05	0.17913517987165560
+AAAAAAAAKOKAAAAA	Later substantial changes give wisely. Minor taxes would shed forward reasons; yet long shareholders will live close small constitutional bags; supplies rea	Home                                              	flatware                                          	3.08	1033.24	0.44487353262970659
+AAAAAAAALBOCAAAA	Rather inc researchers might not answer sure. Most actual lives 	Home                                              	flatware                                          	4.89	317.32	0.13662582688829168
+AAAAAAAALEBDAAAA	Forces used to adapt in a musicians. Rather political t	Home                                              	flatware                                          	89.07	4073.22	1.75377237677400555
+AAAAAAAAMGKBAAAA	Other, white years get meanwhile plans; more royal sciences would not contain triumphantly splendid specific concepts; free months 	Home                                              	flatware                                          	1.62	21553.63	9.28016677547677492
+AAAAAAAAMMABAAAA	Financial, black securities may support vague, late offices. So marginal incomes make on the men. Hotly close occupation	Home                                              	flatware                                          	6.87	280.44	0.12074671275857973
+AAAAAAAAMPBDAAAA	Actively fierce lines should not feel quite confident new rules. Arms pay long settings. Wide, black women should pick real talks. Important friends make today between the revenues. Noisily expe	Home                                              	flatware                                          	4.53	8713.76	3.75181099617458879
+AAAAAAAAOOPAAAAA	Brief regions ought to inclu	Home                                              	flatware                                          	4.98	5812.86	2.50279466811381312
+AAAAAAAAOPJCAAAA	Forward general regulations can begin forward women; galleries consist typic	Home                                              	flatware                                          	8.74	2672.21	1.15055118136002115
+AAAAAAAAPAGEAAAA	Uncertain, statistical jobs walk there; agreements show to a rights. Useless years may not resist locally only marginal experts. Concerned,	Home                                              	flatware                                          	0.14	7564.70	3.25706981174164905
+AAAAAAAAPCHBAAAA	Beneficial, moving years ought to see difficult, political stocks; attitudes can say british questions. Upper, educational chapters should end then back lives. Workers talk there in a boundaries; pro	Home                                              	flatware                                          	2.02	609.71	0.26251775151916148
+AAAAAAAAPNCEAAAA	Busy, new things go satisfactory services. Now old years must take. Scottish procedure	Home                                              	flatware                                          	0.85	2855.80	1.22959799706158888
+AAAAAAAAABPDAAAA	Mislea	Home                                              	furniture                                         	1.06	2910.97	1.06321050660037366
+AAAAAAAAADKAAAAA	Papers check other, industrial boards. Violent, social things give cars. Local councillors give ther	Home                                              	furniture                                         	3.38	3631.97	1.32655048442868154
+AAAAAAAAAHCBAAAA	Dutch, busy firms must not return thereof full, naval plants. Parts shall get ashore early politicians. Good organisms try rather also close boys. Positive, big ingredients foster greatly local grou	Home                                              	furniture                                         	1.71	1113.86	0.40682922011628158
+AAAAAAAAAKLBAAAA	Arrangements will trade however in every negotia	Home                                              	furniture                                         	3.24	15049.37	5.49667234692094570
+AAAAAAAAALDCAAAA	Black, perfect visitors should test more low english interests; about major wives believe examples. Other, available gro	Home                                              	furniture                                         	0.66	10969.33	4.00646757141663321
+AAAAAAAAANGAAAAA	Marine, new services shall reach more more significant elements. Late, solid rights would like also. Notes complete elements; continually personal armies will compare clearly curre	Home                                              	furniture                                         	3.59	965.34	0.35258337613977633
+AAAAAAAAAOBEAAAA	Ways become worldwide specially common gene	Home                                              	furniture                                         	8.57	791.04	0.28892157567448637
+AAAAAAAAAPBBAAAA	Very likely areas should k	Home                                              	furniture                                         	2.37	3579.84	1.30751038311912580
+AAAAAAAABAOBAAAA	Arms fail other faces; leaders could arise good characteristics; gol	Home                                              	furniture                                         	8.75	2288.09	0.83570814128872814
+AAAAAAAABKCAAAAA	Stones tell. Still brown relationships put initially long r	Home                                              	furniture                                         	9.54	5599.90	2.04532252682488396
+AAAAAAAACCKCAAAA	Private, young standards find even so in the women. Sheer, expert classes cannot present men. Small, sure enquiries must support mildly p	Home                                              	furniture                                         	4.99	2942.39	1.07468643184775984
+AAAAAAAACMLDAAAA	Authorities used to consider; general weapons seek particularly economic papers; much american walls	Home                                              	furniture                                         	1.27	2216.17	0.80943988718968251
+AAAAAAAAEFLAAAAA	Severe, likely areas make on board formal, new conditions. Democratic, individual numbers should not fight workers. Poor options think. Independent feelings squeeze only ideas. Thin prob	Home                                              	furniture                                         	8.47	3094.07	1.13008644271738222
+AAAAAAAAEFPCAAAA	Adults might not surrender doubtful, upper industries; earnings insist m	Home                                              	furniture                                         	1.61	6969.96	2.54572692352870019
+AAAAAAAAEJJAAAAA	Shareholders mean; more very teams believe necessary, charming words. Courses would not suggest as popular, similar assets. Subjects must make on the things. Liabilities used to get very to a lines; 	Home                                              	furniture                                         	8.45	3751.07	1.37005088853319121
+AAAAAAAAEPPAAAAA	Directly high lines move calmly also international files. Pounds cannot ensure creditors. Similar, favorable colleagues could gather written police. Free days might provide so. Probably other rock	Home                                              	furniture                                         	6.83	5386.33	1.96731764601379975
+AAAAAAAAGCFAAAAA	Streets know half. National, 	Home                                              	furniture                                         	0.39	9772.83	3.56945469558921243
+AAAAAAAAGGCCAAAA	Soviet, evident ways change able, huge woods. Smart sales ask sales. Thus possible transactions can want below effective, available families. Also external 	Home                                              	furniture                                         	4.84	145.90	0.05328890813474358
+AAAAAAAAGLJDAAAA	Usual tools happen little young children. Dramatic,	Home                                              	furniture                                         	1.68	11143.74	4.07016954857756966
+AAAAAAAAGMOAAAAA	Judicial operations cannot kick currently h	Home                                              	furniture                                         	6.22	9022.42	3.29537293031578591
+AAAAAAAAGPCAAAAA	Too young things leave individually skills. Contexts suffer enormously so romantic	Home                                              	furniture                                         	29.66	20545.03	7.50392197598047208
+AAAAAAAAHEIDAAAA	Superb lights occur with a standards. Bright services specify at the sides. Then urgent versions get earlier prisoners. Available heroes would not believe civil sides. Banks could t	Home                                              	furniture                                         	0.12	16046.32	5.86080104441877032
+AAAAAAAAHPPAAAAA	Royal, military notions will not find very very wet acids. Funny actions take western, remaining homes. Great patients will replace simply. Signs can think equivalent reasons. Campaigns 	Home                                              	furniture                                         	7.54	1334.66	0.48747480555940278
+AAAAAAAAICHCAAAA	Yet huge priests think today unlikely, absolute things. Whole, modern changes might not manipulate most only, desirable companies; accused, particular girls may take serious, central hours	Home                                              	furniture                                         	0.52	10920.86	3.98876425834404225
+AAAAAAAAIHDBAAAA	Local blocks shall not get natural things; already post-war patients may exploit british, sexual grounds. Easy centuries would not	Home                                              	furniture                                         	3.75	2996.52	1.09445701853270617
+AAAAAAAAIKCAAAAA	Ago new arguments accept previously european parents; fo	Home                                              	furniture                                         	3.03	6882.58	2.51381201747788529
+AAAAAAAAIMCEAAAA	Walls s	Home                                              	furniture                                         	4.80	1253.04	0.45766369738971278
+AAAAAAAAIPIBAAAA	Late general supporters see more informal, blank employees; very similar methods shall help complex, likely schemes. More than new groups reconsider unanimously. Physical incenti	Home                                              	furniture                                         	37.53	2259.23	0.82516723732184192
+AAAAAAAAJKLBAAAA	Mountains ought to join pressures. Bright countries used to pay there owners. Imperial issues might establish thus calmly senior members. Just regular 	Home                                              	furniture                                         	7.01	10713.70	3.91310058316108488
+AAAAAAAAKGPAAAAA	Contacts open considerable, suprem	Home                                              	furniture                                         	7.01	1997.51	0.72957592109822925
+AAAAAAAAKOGCAAAA	Effects must quit about small values; full paths must get. Problem	Home                                              	furniture                                         	1.87	4806.19	1.75542575317425115
+AAAAAAAAKOOAAAAA	Political girls used to ask hands. Large-scale, chief areas can produce including the children. Sufficiently new areas will	Home                                              	furniture                                         	2.26	3164.50	1.15581048521176187
+AAAAAAAALFDDAAAA	Now late makers used to	Home                                              	furniture                                         	0.85	7607.78	2.77868601459451341
+AAAAAAAALMDCAAAA	Greatly commercial animals used to live now as wide personnel. Enough hot wars keep. Min	Home                                              	furniture                                         	4.37	894.54	0.32672419385094943
+AAAAAAAAMBPAAAAA	Better high children	Home                                              	furniture                                         	4.48	4768.72	1.74174010966630844
+AAAAAAAAMCNCAAAA	Thus light firms expect anyway in a toys. Laws used to ab	Home                                              	furniture                                         	2.06	12227.85	4.46613279873491621
+AAAAAAAAMFBEAAAA	Widespread others hold quickly new teachers. Societies wou	Home                                              	furniture                                         	3.01	1696.19	0.61952099444188288
+AAAAAAAAMMDCAAAA	Hot, small levels ought to arrive only there other features. Often irish columns used to spend now new natural months. Once british flowers shall penetrate funds. 	Home                                              	furniture                                         	5.70	20519.61	7.49463750685925767
+AAAAAAAANGDAAAAA	Electronic organizations transfer still natural, whole posts. Plants ought to curl just animals; already huge women can dream eventua	Home                                              	furniture                                         	3.59	6214.52	2.26980798753616633
+AAAAAAAANNLDAAAA	Increasingly other policies happen previously under a targets. Efficient, experienced points will see mostly even english machines. Fine states must remedy also good thoughts; normally clear years i	Home                                              	furniture                                         	5.85	9156.23	3.34424605435629337
+AAAAAAAANPOBAAAA	Natural costs assist during the types. Sometimes possible concerns make as real, right forms. 	Home                                              	furniture                                         	6.28	1707.15	0.62352405429902331
+AAAAAAAAOCFCAAAA	Therefore early eyes stay recent, expert studies; varieties halt in a parts. Unable i	Home                                              	furniture                                         	7.52	742.08	0.27103929368492471
+AAAAAAAAOFECAAAA	Funds drink much common months. Royal, long trees will expect sometimes front coins. Old ears can allow very similar, short members. Even public rules act common, open 	Home                                              	furniture                                         	17.29	6237.51	2.27820491692628117
+AAAAAAAAOGODAAAA	Intensive minutes might see like a boys. Questions might know more young communications. Ready, southern others may result. Lonely, trying seeds love probably good farms. 	Home                                              	furniture                                         	9.12	11445.81	4.18049840724968750
+AAAAAAAAOKNAAAAA	At least competitive notions may not convince white, familiar principles. Valuable, fat books convince further cases. Yet ordinary cities cannot need so as. Ri	Home                                              	furniture                                         	8.51	1332.65	0.48674066775713524
+AAAAAAAAOOKDAAAA	Women should not knock doubtless details. Sure northern products must go very cruel, other tickets. Poor, physical objectives highlight only by the discussions; now slow crowds must 	Home                                              	furniture                                         	0.77	87.87	0.03209387496778559
+AAAAAAAAPKAAAAAA	Little, evil parties would not act subject	Home                                              	furniture                                         	7.63	1108.98	0.40504683580032854
+AAAAAAAAPLEEAAAA	Easy, philosophical levels must 	Home                                              	furniture                                         	2.32	3778.34	1.38001105662664191
+AAAAAAAAAEPAAAAA	Now additional reasons hate. Original, use	Home                                              	glassware                                         	4.41	6349.14	1.56441659290736902
+AAAAAAAAAFBBAAAA	Jobs notify about future boxes. Now main policies will think above offers. Criminal men used to think soon national women. Sure talks ought to appreciate there companies. So appropri	Home                                              	glassware                                         	1.19	7756.30	1.91113826747676477
+AAAAAAAAAFIAAAAA	Seats will cope similarly new shares; massive deals explore semantic, important thi	Home                                              	glassware                                         	1.53	4412.81	1.08730838906490754
+AAAAAAAAAGGCAAAA	Powerful hours take worth a authorities. Respondents must generate aside c	Home                                              	glassware                                         	31.97	10526.17	2.59362921714811148
+AAAAAAAAAHPDAAAA	Unfair, possible hands will not arrive surely tight russian employers. Really necessary walls should decide varieties. Talks would raise probably moral meetings. Bright, necessary 	Home                                              	glassware                                         	1.54	3919.44	0.96574291493097623
+AAAAAAAAAINCAAAA	Old	Home                                              	glassware                                         	1.47	1351.66	0.33304657512185499
+AAAAAAAAANFBAAAA	Conditions criticise enough more particular shops. Be	Home                                              	glassware                                         	6.38	1038.40	0.25585987867254652
+AAAAAAAAANMBAAAA	Countries ensure in a christians. Expected ends used to run high holes. Broad, unlike women specify therefore. Lit	Home                                              	glassware                                         	2.94	153.37	0.03779009013097887
+AAAAAAAABGKCAAAA	Onc	Home                                              	glassware                                         	4.53	1345.23	0.33146223477144621
+AAAAAAAABHEBAAAA	Western, complete meetings follow also educational shareho	Home                                              	glassware                                         	7.67	2508.40	0.61806521539119384
+AAAAAAAABMBCAAAA	Similar, low sites remember peaceful days. Faster permanent views give then churches. Others make well public processes. Eventually other schemes can trus	Home                                              	glassware                                         	0.29	105.75	0.02605660840680065
+AAAAAAAABPMAAAAA	Statistical bedrooms analyse there good, little residents. 	Home                                              	glassware                                         	8.08	5239.63	1.29103533906879324
+AAAAAAAACCGDAAAA	Less than outside students go more. Military views should not let more different, big steps. Average, black animals ought to begin automatically with a notes. Needs	Home                                              	glassware                                         	3.76	13328.83	3.28419956341197821
+AAAAAAAACCKAAAAA	Wide, great premises mean ever severe courses. Used ministers face there about a things.	Home                                              	glassware                                         	0.83	1275.20	0.31420696964872045
+AAAAAAAACGLAAAAA	Faintly actual prices may not wait dramatic terms. Others shall see shortly priests. Very na	Home                                              	glassware                                         	27.85	6812.75	1.67864925695915955
+AAAAAAAACIFCAAAA	Agents invest often things. French cars ought to get locally distinctive, local powers. More american entries compensate only	Home                                              	glassware                                         	6.43	10473.16	2.58056764918929822
+AAAAAAAACJFDAAAA	Again other wheels ought to find on a employees. Developments make really together new groups. Drinks would not assess bright women; special, australian t	Home                                              	glassware                                         	3.25	516.63	0.12729669599248624
+AAAAAAAACKODAAAA	Words visit authorities. American occasions must need available, pure c	Home                                              	glassware                                         	5.43	5888.06	1.45080731627183575
+AAAAAAAACNKBAAAA	Purposes look events. Words convert over complete sites. New notes tell up a 	Home                                              	glassware                                         	9.93	9702.28	2.39062421383578063
+AAAAAAAADLHBAAAA	Free kids would become only 	Home                                              	glassware                                         	1.05	8484.78	2.09063441964873770
+AAAAAAAADNADAAAA	Interested, square savings change off	Home                                              	glassware                                         	2.10	8572.37	2.11221643695702771
+AAAAAAAADOKBAAAA	Exactly single cities used to deserve ago false services. Suddenly psychological managers could sustain far together big changes. Parents should r	Home                                              	glassware                                         	0.64	2997.09	0.73847754600414333
+AAAAAAAAEGOBAAAA	Heavy, desperate standards could produce still fine, important weeks. Accordingly 	Home                                              	glassware                                         	9.90	11317.37	2.78857946368674669
+AAAAAAAAEIIAAAAA	Long, surprised sections keep positive sports. Strategies go northern, precious forms; readers emerge about reports. Large, unusual legs might show affairs; as usual ac	Home                                              	glassware                                         	4.43	12838.25	3.16332154022324760
+AAAAAAAAEIKAAAAA	Red rooms could not apply	Home                                              	glassware                                         	4.96	1551.75	0.38234838860759250
+AAAAAAAAEJOCAAAA	Present materials would say real, rare relationships. Particular conclusions contribute well to a hand	Home                                              	glassware                                         	4.07	8454.05	2.08306260332400026
+AAAAAAAAFCECAAAA	Separate moments come months. Avail	Home                                              	glassware                                         	0.58	5564.41	1.37106054264667234
+AAAAAAAAFIDDAAAA	Professional, local chemicals can feel eyes. Familiar shops bear early in a accounts. Western arrangements ride reserves. Sorry, scottish ministers might not keep constantly w	Home                                              	glassware                                         	6.13	5921.40	1.45902223186788996
+AAAAAAAAGAIDAAAA	Rows come 	Home                                              	glassware                                         	0.29	840.56	0.20711246111035795
+AAAAAAAAGFLDAAAA	White, local attitudes ca	Home                                              	glassware                                         	1.74	1012.36	0.24944366985067333
+AAAAAAAAGHMDAAAA	Seconds may make ahead quite little lips. Young, criminal consumers s	Home                                              	glassware                                         	7.17	1471.96	0.36268827716760552
+AAAAAAAAGJNAAAAA	Recently nice particles hear above in a candidates. Human errors register. American, old days live. 	Home                                              	glassware                                         	8.16	528.66	0.13026086619706129
+AAAAAAAAGKNDAAAA	Traditional, old-fashioned men show too final,	Home                                              	glassware                                         	4.84	6698.16	1.65041448856828214
+AAAAAAAAGLGCAAAA	Years must share new, white loans. Able 	Home                                              	glassware                                         	1.64	1410.40	0.34752000469930625
+AAAAAAAAGLOBAAAA	Single, roman facts may hear by a rights; different, able preferences must produce as internal surveys. Similar heads might stabilize direct na	Home                                              	glassware                                         	6.70	8825.39	2.17456010654651897
+AAAAAAAAGNBAAAAA	Stones should send perhaps at the groups. Perhaps individual facts 	Home                                              	glassware                                         	4.18	26041.20	6.41650449969907389
+AAAAAAAAGPPBAAAA	More black members would run more central poor phases. Personal responsibiliti	Home                                              	glassware                                         	8.30	423.06	0.10424121751849724
+AAAAAAAAHBKBAAAA	Safe, distinct millions must not deliver at the men. Indeed old claims might put exercises; particular, wooden households should learn clear, lucky votes. Mean, level terms might write bot	Home                                              	glassware                                         	9.86	7952.69	1.95952840766599957
+AAAAAAAAHOBDAAAA	Significant difficulties could observe numbers. Very alone centuries affect forwards by a matters. Glad fields ought to spread hardly british str	Home                                              	glassware                                         	3.06	501.96	0.12368203457094708
+AAAAAAAAIACDAAAA	Novel, small attitudes may warn now however good terms. Aware earnings must eat much; lat	Home                                              	glassware                                         	2.84	5534.76	1.36375483636523840
+AAAAAAAAIBHDAAAA	Cold, old days stem thereby difficult, nuclear men; likely contents shall threaten often outer years. All real or	Home                                              	glassware                                         	9.08	11902.21	2.93268298009935465
+AAAAAAAAIELDAAAA	Now strong fields may not feel. Again	Home                                              	glassware                                         	3.96	9805.52	2.41606236279008890
+AAAAAAAAIGPDAAAA	Even sexual men can clear thereby always male members. Shoulders extract. Negotiations used to alter else 	Home                                              	glassware                                         	3.47	1371.15	0.33784887581073012
+AAAAAAAAIJJDAAAA	Conditions could not estimate following problems. Theories get sure; extremely complete scholars ought to thrive only strong, european businesses. Important, social p	Home                                              	glassware                                         	1.56	6751.07	1.66345141670827100
+AAAAAAAAIMBEAAAA	Holes buy then markets. Practical themes ought to escape above australian children.	Home                                              	glassware                                         	1.43	3401.20	0.83804951785541719
+AAAAAAAAINDCAAAA	Willing, due values will chat hardly gmt central records. Necessary, adult stairs make fast in terms of a years. Views would not dig	Home                                              	glassware                                         	0.24	2373.76	0.58489016332602467
+AAAAAAAAINPCAAAA	Moments used to contract really boats. A	Home                                              	glassware                                         	68.94	1997.56	0.49219516490864023
+AAAAAAAAJIIDAAAA	Insects indicate germans. Other, particular properties might 	Home                                              	glassware                                         	4.52	2374.24	0.58500843445638178
+AAAAAAAAJKCEAAAA	Persons might live here doctors. Chil	Home                                              	glassware                                         	2.86	15578.10	3.83841561628351009
+AAAAAAAAJNOBAAAA	Materials make apart colonies. Rates make naturally poor, appropriate companies; c	Home                                              	glassware                                         	0.80	1956.16	0.48199427991533955
+AAAAAAAAJPDBAAAA	Used groups ought to fail high from the districts. Immediate, main walls could exploit rights. Therefore late friends ought to try away. In short widespread lakes sh	Home                                              	glassware                                         	80.17	9287.91	2.28852419657312357
+AAAAAAAAKIDBAAAA	Too only affairs put nonetheless big numbers. Rapid students appeal for the 	Home                                              	glassware                                         	9.29	13621.22	3.35624392967263487
+AAAAAAAAKKHCAAAA	Good windows say widely actions. Simple, imaginative findings see to a recommendations. Environmental, l	Home                                              	glassware                                         	4.66	12892.65	3.17672560166371999
+AAAAAAAAKNMDAAAA	Japanese emotions speak disabled, new techniques. Experts should not tell only refugees. Years cannot afford well head quarters. Offices make conscious, primary stories	Home                                              	glassware                                         	7.31	4129.01	1.01738058324126665
+AAAAAAAAKPJBAAAA	Full goods should find then. Only able years exploit completely mode	Home                                              	glassware                                         	2.13	3040.36	0.74913919560946025
+AAAAAAAAMDMBAAAA	Sexual, due tensions take quite lucky circumstances. For ever formal districts respond ways. Poor relations should not come correctly in an facilities; important times could look away common 	Home                                              	glassware                                         	42.90	1247.40	0.30735710001553787
+AAAAAAAAMDNAAAAA	Bad boys might claim shortly italian, good lines. Times learn additional, sick cards; measures work sometimes pleasant male	Home                                              	glassware                                         	2.10	3225.77	0.79482388369177617
+AAAAAAAAMHFBAAAA	Children want on a paintings. Over nice teachers must not sell. Richly accurate pp. hate as african, fiscal days. Claims eat part	Home                                              	glassware                                         	7.95	6793.78	1.67397508332817129
+AAAAAAAAMLGAAAAA	Always sad weeks would not put close to a masses. Fresh, atomic sides will not help together previous 	Home                                              	glassware                                         	0.83	6893.14	1.69845720731209292
+AAAAAAAAMMPBAAAA	As other 	Home                                              	glassware                                         	4.88	2352.12	0.57955810653242499
+AAAAAAAAMNECAAAA	Serious branches use. Rich, english bombs keep much vulnerable consequences. Little, furious sales can keep to a gentlemen. As gold customers overlap betwee	Home                                              	glassware                                         	2.54	3062.18	0.75451560407694385
+AAAAAAAAMNIBAAAA	Really different shares ought to help clearly p	Home                                              	glassware                                         	2.82	6640.72	1.63626137663554805
+AAAAAAAANGAAAAAA	There possible newspapers experiment. Annual accounts might visit possible, prime groups; competitive, universal pr	Home                                              	glassware                                         	1.12	63.36	0.01561178920713843
+AAAAAAAANPPAAAAA	Recent, labour complaints must read in a units. Softly old courts rely even. Actual	Home                                              	glassware                                         	8.70	2861.55	0.70508073556955459
+AAAAAAAAOAIBAAAA	Well new carers shall give together with a samples. Individual, central birds find there weapons. Kind details proceed ultimate miles. Unlike, independent months mus	Home                                              	glassware                                         	0.46	6486.44	1.59824706415326716
+AAAAAAAAOMGDAAAA	Overseas businesses conceal gmt in a farmers. Level functions could support all right dreadful processes. Walls buy furth	Home                                              	glassware                                         	3.81	10274.91	2.53171920836992962
+AAAAAAAAONFAAAAA	Mental techniques might prohibit by a chiefs; other, waiting defendants vary else. Now old skills would see. Common jobs will no	Home                                              	glassware                                         	89.76	2200.15	0.54211297386498769
+AAAAAAAAOOFCAAAA	Dogs will cover never. Bitter children restore cheaply upper, short views; other teams shall exist too high customs. Yards must not help now present, coming mines. However federal method	Home                                              	glassware                                         	3.22	2352.77	0.57971826535478358
+AAAAAAAAPBFBAAAA	More than divine areas will control together from 	Home                                              	glassware                                         	4.90	563.56	0.13886016296677611
+AAAAAAAAPBMDAAAA	Surely national arguments address working, soviet effects. Again central parents say english rules; carefully military chang	Home                                              	glassware                                         	8.61	13637.98	3.36037356330760394
+AAAAAAAAPGHAAAAA	Classical, attractive employers want only prices. Financial approaches used to hear considerable votes. Bo	Home                                              	glassware                                         	2.50	13555.23	3.33998411323041478
+AAAAAAAAPKCBAAAA	Other patients see normal colleagues	Home                                              	glassware                                         	4.62	1970.54	0.48553748586228795
+AAAAAAAAPKHCAAAA	Newspapers ought to pursue. Well rare criticisms used to tell so. Powerful, new matters touch. Home magic brothers can read now rather supreme rats. As evolu	Home                                              	glassware                                         	4.99	1537.58	0.37885692628017534
+AAAAAAAAAANAAAAA	Surely additional years work never remote, great bits; women deal in a judges. Far ethnic hands might help afterwards already dead awards. Rich, social experts target social children. National	Home                                              	kids                                              	0.50	361.08	0.11815869948988022
+AAAAAAAAABBAAAAA	Yet black costs must not judge here lively variables. Full, po	Home                                              	kids                                              	1.68	3938.44	1.28880289248621866
+AAAAAAAAABNCAAAA	Proud investors may not visit extremely. Alone, everyday houses move widely global countries. Only single gardens come further shadows. Scottish, wo	Home                                              	kids                                              	2.68	31.68	0.01036686496022877
+AAAAAAAAAIEEAAAA	Total, new savings would make short, popular consultants. Short, other contracts might discuss for a	Home                                              	kids                                              	9.91	1600.56	0.52376229105883094
+AAAAAAAAAKADAAAA	Effective, free arrangements will build social, possible agreemen	Home                                              	kids                                              	4.30	2319.90	0.75915688198341950
+AAAAAAAAAKGDAAAA	Enterprises shall not influence perhaps delighted, big police. Novels keep early temporary bacteria; rates will not cope men	Home                                              	kids                                              	3.57	6583.08	2.15422668504996302
+AAAAAAAABAAAAAAA	Agricultural sites will not provide skills. Again	Home                                              	kids                                              	0.55	5015.40	1.64122394323015739
+AAAAAAAABDKCAAAA	Conservatives tell effectively in a parties. Dir	Home                                              	kids                                              	6.35	8063.47	2.63866491795631001
+AAAAAAAABFABAAAA	Too old	Home                                              	kids                                              	0.95	114.66	0.03752098283900982
+AAAAAAAABLDBAAAA	Following occasions see then only real lovers	Home                                              	kids                                              	5.63	6310.36	2.06498263795546836
+AAAAAAAACCPDAAAA	Permanent details would help also off a owners. External children used to listen like a years	Home                                              	kids                                              	30.73	6001.32	1.96385334668939829
+AAAAAAAACFKCAAAA	Farmers might not assume now to the tanks. For	Home                                              	kids                                              	3.80	11826.88	3.87019153601106270
+AAAAAAAACGOAAAAA	Local farmers skip also shoulders; things ought to seem so only applications. Foreign, voluntary voices may not find new	Home                                              	kids                                              	3.96	2251.62	0.73681314651989612
+AAAAAAAACHLCAAAA	Now close items become already against a groups. Authorities would work as well natural, dependent parties. Operators should not fall l	Home                                              	kids                                              	5.59	7257.25	2.37483998524685165
+AAAAAAAACIAEAAAA	Appropriate items take mediterranean centuries. High, very days see ways. Careful, technical minds remai	Home                                              	kids                                              	4.98	10259.21	3.35719206656024705
+AAAAAAAACJGAAAAA	Dire	Home                                              	kids                                              	4.41	1733.90	0.56739605917110697
+AAAAAAAACKPBAAAA	Short areas would not improve below to the measurements. Vo	Home                                              	kids                                              	0.36	18342.34	6.00229046195084044
+AAAAAAAACMDCAAAA	As beautiful children strike really natural services. Too assistant pow	Home                                              	kids                                              	3.30	2799.11	0.91597207635182954
+AAAAAAAACPKBAAAA	Even growing seats may build for a times. Obvious, different systems require american settlements. Evil yards support worldwide possible members. Courses could build also users. Alm	Home                                              	kids                                              	4.28	2619.47	0.85718723981598684
+AAAAAAAADDCCAAAA	Gold, young schools shall not go far human hands. Aware terms brush almost. Real years treat early. Edges cannot stop still british assessments. Very royal skills shall say already other	Home                                              	kids                                              	5.63	4448.98	1.45587041890020849
+AAAAAAAADGDBAAAA	Dogs hang perhaps chief, visual brothers. Minimum, small families shall work strong mountains. Small, defensive factors make by	Home                                              	kids                                              	5.44	2978.61	0.97471109972181264
+AAAAAAAADJHAAAAA	So dependent things distinguish again new subjects. Critical, firm centuries increase then institutions. Effects allo	Home                                              	kids                                              	1.59	10537.48	3.44825227844417572
+AAAAAAAADMDBAAAA	Turkish, old women must improve far from full, new changes. Days keep again exactly secondary visitors. Things used to make great, other notes. General, hig	Home                                              	kids                                              	1.38	355.77	0.11642107155620551
+AAAAAAAAECJCAAAA	Examinations reduce other, late things. Police should help very strong boxes. Annual, sole reports might benefit fortunate, total seats. Never rural shapes shall cease pictures. Physical periods wi	Home                                              	kids                                              	3.60	1189.98	0.38940536506859327
+AAAAAAAAEHDAAAAA	Likely products ought to work other, considerable arrangements. Also other funds kill possible, royal patterns. Old, good files know palestinian colours. Northern	Home                                              	kids                                              	1.60	3252.96	1.06448854296167261
+AAAAAAAAEKMDAAAA	Minds could not decide later avail	Home                                              	kids                                              	2.36	7178.10	2.34893918469122957
+AAAAAAAAEKNCAAAA	Teams make never features. Now russian individuals may reproduce indeed other visual lakes. International legs drive also married views. Catholic populat	Home                                              	kids                                              	8.74	5328.40	1.74364909261625606
+AAAAAAAAEMNAAAAA	Healthy, delighted conclusions may offer experienced condi	Home                                              	kids                                              	4.30	1952.10	0.63879915053227863
+AAAAAAAAFENAAAAA	Reasonable pictures could not try features. Unexpected politicians remember always. Serious buildings pay thereafter aged a offers. Large, material products go tomorrow interesting, individual re	Home                                              	kids                                              	44.54	107.20	0.03507979557249130
+AAAAAAAAFJEBAAAA	Equal supplies could get easily still new years. Equivalent, national policemen shall appeal. Tables would 	Home                                              	kids                                              	7.14	13784.20	4.51069886315610630
+AAAAAAAAGDBDAAAA	Hours get skills; foreign, positive events disguise currently apparent prices; other programmes may sink honours. For instance var	Home                                              	kids                                              	7.04	2430.74	0.79542781986826031
+AAAAAAAAGNIBAAAA	Apparently effective deals could stand 	Home                                              	kids                                              	0.92	1924.93	0.62990812398652687
+AAAAAAAAHBJCAAAA	Funny times go actually much old details. Military parameters tell so studies. Values u	Home                                              	kids                                              	4.41	1907.42	0.62417820588508729
+AAAAAAAAHDIBAAAA	Levels contact in a sides. Companies must not count with an boxes; yet physical days happen never from a opera	Home                                              	kids                                              	8.77	13024.65	4.26214607652284354
+AAAAAAAAICLCAAAA	Questions seem strongly. Political years establish guilty centres. Necessary, pale eyes used to generate social, particular assets. Conditions help as firm directors. Persona	Home                                              	kids                                              	9.37	8639.50	2.82716318888562125
+AAAAAAAAIEGEAAAA	Subsequent qualities say broadly good objectives. Odd workers ought to make commonly therefore intact times. Objectives will not hold just with the types. B	Home                                              	kids                                              	0.64	3035.53	0.99333742401272873
+AAAAAAAAIKMBAAAA	Soon artificial notions think no longer lights; clearly late members could not trace good countries. Cultures can proceed away wealthy 	Home                                              	kids                                              	2.38	3035.43	0.99330470032282902
+AAAAAAAAJHMDAAAA	Appropriate, new ad	Home                                              	kids                                              	3.99	396.54	0.12976251992831810
+AAAAAAAAJNADAAAA	Ruthlessly empty times shall not focus to a lectures. Skills involve even; boring periods re	Home                                              	kids                                              	0.63	1007.86	0.32980898102323771
+AAAAAAAAKBACAAAA	Lists could play round, new roads. Soon national calculations think usually at first similar benefits. Skilfully specific practitioners will believe that is bars. More immediate	Home                                              	kids                                              	8.24	3098.01	1.01378318546206881
+AAAAAAAAKINDAAAA	Suggestions must see much large assessments. Disabled charges might claim besides wide, white passengers. Democratic, wide relationships test little years. Working, bri	Home                                              	kids                                              	0.50	934.46	0.30578979263684908
+AAAAAAAAKMDDAAAA	Strong settlements should close here. Forms may seem quickly other unions. Places employ difficult banks. Women must not accept too areas. Vast possibilities know; never healthy subjects cancel most j	Home                                              	kids                                              	1.95	10592.00	3.46609323417749873
+AAAAAAAAKNJDAAAA	English requests serve also intervals. More late cards might make only other companies. Tragic lights learn more royal, attractive studies. Businessmen ought to defend close po	Home                                              	kids                                              	1.59	17495.72	5.72524515852189842
+AAAAAAAALDODAAAA	Goals help still human plates. Practical groups t	Home                                              	kids                                              	4.79	16455.90	5.38497768620671273
+AAAAAAAALIFEAAAA	Full, good fans might not pose of course parts. Daily	Home                                              	kids                                              	85.83	7041.80	2.30433679535792207
+AAAAAAAALJACAAAA	Due years show just ashamed homes. Large, australian parties suit there automatic grounds. Sexual steps might not mean today technical molecules. Al	Home                                              	kids                                              	6.52	4853.82	1.58834900509020269
+AAAAAAAALLDAAAAA	Then dark tactics should not follow then. Ashamed, g	Home                                              	kids                                              	1.43	11882.09	3.88825828520469372
+AAAAAAAAMCNBAAAA	Vocational, political styles run incorrectly indeed only hands. Complete, confident employers expect big owners. Inc times should stop more; consi	Home                                              	kids                                              	8.09	3606.10	1.18004898147351569
+AAAAAAAAMKJAAAAA	Formal matters must admire much. Capable rules rise however. Harder only studies would show more. Old stones oppose common, secure police. Opinions come grey, appropriate systems. Eye	Home                                              	kids                                              	6.02	261.24	0.08548736749400772
+AAAAAAAANBJCAAAA	Soft, good estates must not join most likely, accused pieces. Coming, historical pictures arrange; best old loans cannot	Home                                              	kids                                              	6.24	6536.61	2.13901998635356684
+AAAAAAAANGIBAAAA	About american members provide certainly increased, special experienc	Home                                              	kids                                              	0.99	5029.15	1.64572345059136780
+AAAAAAAAODPDAAAA	Trying, ti	Home                                              	kids                                              	3.34	16043.89	5.25015281145090918
+AAAAAAAAOGJCAAAA	New, other companies could take always political years. Important charges wait sure other aspects. Legal grounds may not worry to	Home                                              	kids                                              	6.49	5131.46	1.67920305772776318
+AAAAAAAAOPCBAAAA	Windows recommend never internal cells. Mutual, other moments should not see levels. Necessary, national costs shall not walk light, high types; more digital days might continue.	Home                                              	kids                                              	2.75	8373.49	2.74011490138339726
+AAAAAAAAPADEAAAA	Fresh, f	Home                                              	kids                                              	1.45	4190.94	1.37143020948299155
+AAAAAAAAPICDAAAA	Quickly wrong facilities prepare as. Similar surveys look hopelessl	Home                                              	kids                                              	3.16	116.22	0.03803147240144533
+AAAAAAAAAHECAAAA	Remote, left figures used to feed on a records. Over economic depths must understand in particular at the ranks; degrees can think go	Home                                              	lighting                                          	2.60	5654.38	2.08346575200781715
+AAAAAAAABMMDAAAA	Lovely letters would require now questions; communities will add years. Emotional, traditional times make for a patterns. Perhap	Home                                              	lighting                                          	8.69	2656.29	0.97876146321981272
+AAAAAAAACBPBAAAA	Moving, powerful drugs use so blind honours. Efficient, other seconds look just rare, planned homes. German, specified sons reside further red weeks. Available lists undergo young, milit	Home                                              	lighting                                          	0.67	10412.96	3.83685665573012774
+AAAAAAAACGIBAAAA	Different men may not inform by now between a eyes. Members can cause new minds. Strong, chief rooms will carry high lessons; natural molecules accept here because of a talks. Eyes may disc	Home                                              	lighting                                          	0.56	7704.59	2.83890530849746709
+AAAAAAAACHHAAAAA	Incidentally immediate flames earn. Friends influence certain, potential men. Early, opening conventions should see per a agencies. Economic, senior practitioner	Home                                              	lighting                                          	1.62	616.89	0.22730506045863602
+AAAAAAAACJGDAAAA	Original others get industrial yar	Home                                              	lighting                                          	1.48	6297.95	2.32060157486013180
+AAAAAAAACJJDAAAA	So soviet years get. Good things must appreciate well real churches. Overseas, constant boxes complete for no months. Subjects may not suffer widel	Home                                              	lighting                                          	5.50	178.36	0.06572019417303299
+AAAAAAAACMCDAAAA	Important, toxic commun	Home                                              	lighting                                          	0.33	431.67	0.15905716650971714
+AAAAAAAADFFDAAAA	Prisoners give fundamental months. Opportunities grasp capital actions. British iss	Home                                              	lighting                                          	5.72	5860.48	2.15940728609091930
+AAAAAAAADKDDAAAA	Under way long interpretations might take yesterday. Little little shares get quickly families. Measures occur. Forward daily hands m	Home                                              	lighting                                          	2.56	2458.11	0.90573820642898698
+AAAAAAAADNMDAAAA	New, future communities should make yesterday particular, primary relations. Significant students mea	Home                                              	lighting                                          	83.07	7959.15	2.93270286752800800
+AAAAAAAAEBDAAAAA	Opportunities drop cars. Officials change as for a inches. Other, american societies take straight leading, total posts. Agreements get 	Home                                              	lighting                                          	65.24	13670.55	5.03717874216279499
+AAAAAAAAEBFBAAAA	Vital problems may lig	Home                                              	lighting                                          	60.33	6799.66	2.50546633500003077
+AAAAAAAAECIAAAAA	Rather american gentlemen might generate rather in a studies. Enough current negotiations used to co-operate nearly rough main rivals. Dramatic, overall weeks used to provide too other, great meal	Home                                              	lighting                                          	7.69	3528.80	1.30025466022538018
+AAAAAAAAEDCEAAAA	Also new colonies go unhappily eggs; typically modern centres would provide then	Home                                              	lighting                                          	0.51	5329.54	1.96377216670187391
+AAAAAAAAEJNCAAAA	Prayers increase ever depths. International, official member	Home                                              	lighting                                          	7.88	4324.07	1.59328728424415089
+AAAAAAAAEMPDAAAA	Sick, old-fashioned birds might think there imports. Grant	Home                                              	lighting                                          	7.01	5314.03	1.95805720700449927
+AAAAAAAAFDFDAAAA	Common contracts will undergo for the goods. Generous, long laws shall not reach less traditional men. All pla	Home                                              	lighting                                          	3.29	973.56	0.35872702533694772
+AAAAAAAAFDJCAAAA	Front shelves produce more at a principles; previously everyday birds avoid on a matters. Up	Home                                              	lighting                                          	18.01	4993.08	1.83979696748983826
+AAAAAAAAFGIAAAAA	Problems should prevent finally in a effects. Now economic men sign. Royal, permanent teeth can start colonies. Geographical eyes wi	Home                                              	lighting                                          	9.41	5689.57	2.09643218861327258
+AAAAAAAAFJJCAAAA	Essentially everyday lines sing s	Home                                              	lighting                                          	6.37	2165.33	0.79785774864708186
+AAAAAAAAFLECAAAA	Famous, attractive arms shall go publicly just democratic men. Importantly private ministers ought to write. Levels bring most true, adjacent days. Successful, particular constraints may pl	Home                                              	lighting                                          	3.16	2680.48	0.98767473691932868
+AAAAAAAAFNBAAAAA	Just familiar police work virtually rare fruits; blind police might not succeed possible, stable churches. Senior communications light old, economic activities; almost direct characters ca	Home                                              	lighting                                          	2.42	14392.73	5.30327994101837339
+AAAAAAAAGAOAAAAA	New kinds will go wholly great, occasional models; efforts may seem then too local homes. However religious co	Home                                              	lighting                                          	4.11	408.39	0.15047919992332890
+AAAAAAAAGBNCAAAA	More possible newspapers	Home                                              	lighting                                          	9.78	3183.02	1.17284532662394854
+AAAAAAAAGEJBAAAA	Of course high 	Home                                              	lighting                                          	4.02	405.11	0.14927062043864877
+AAAAAAAAGFGAAAAA	Further high men can give stairs. Controversial, great fingers hate sometimes generally ancient books. Other dogs woul	Home                                              	lighting                                          	6.69	1549.44	0.57092115754353125
+AAAAAAAAGMLCAAAA	Visual, sensible rates know instead excellent honours. Other, inc christians fill plans. Girls may not make to a institutions. Days could build appropriate, small statements. Left, runnin	Home                                              	lighting                                          	1.12	8531.28	3.14351523965302125
+AAAAAAAAHBCAAAAA	Propos	Home                                              	lighting                                          	1.14	5525.76	2.03607322355673225
+AAAAAAAAHLICAAAA	Significantly severe hundreds let right. Domestic, good approaches like of course later main records. General firms will preve	Home                                              	lighting                                          	17.01	2134.46	0.78648309965559538
+AAAAAAAAHOHAAAAA	More great societies press. Years make still other, lively standards. Decisions may strike to	Home                                              	lighting                                          	0.43	2644.48	0.97440984013625407
+AAAAAAAAICJBAAAA	Unusual, fierce imports can press fine rural contents. Perhaps public	Home                                              	lighting                                          	4.21	7474.73	2.75420894253753570
+AAAAAAAAIFGBAAAA	Middle-class years record also recent problems; certain, mild others can show. Matters will influence solely books. Loca	Home                                              	lighting                                          	6.43	2611.80	0.96236826161206301
+AAAAAAAAILCBAAAA	Able, double cells monitor quickly tomorrow direct men. Different weeks used to become n	Home                                              	lighting                                          	7.19	187.35	0.06903273367525079
+AAAAAAAAJABAAAAA	Legal conventions ought to work in accordance with a cases. Together left books may not come sure subsequent things. Short, real products deal excessive, im	Home                                              	lighting                                          	5.79	9924.55	3.65689253801286467
+AAAAAAAAJBDDAAAA	International, final writers must learn political eyes. Immediate times reach also also wrong requests. Isolated years may not plan yesterday minutes. Long, old researc	Home                                              	lighting                                          	0.62	4542.45	1.67375362200770182
+AAAAAAAAKEPAAAAA	Alone new conditions will recognise personal, hot others. Sooner scottish eyes permit probably only advanced cases. Never impossible services use again direct	Home                                              	lighting                                          	4.82	8731.18	3.21717226373459388
+AAAAAAAAKFFCAAAA	Usually severe kinds cost incidentally conclusions; normally continuing concentrations ought to tell amer	Home                                              	lighting                                          	0.90	8588.69	3.16466906532847440
+AAAAAAAALENDAAAA	Empty, splendid pounds make relatively skills; public, simple exchanges might exploit simply. Basically quiet perceptions shall not sleep. Old, alone individuals get permanent, new minerals. Fo	Home                                              	lighting                                          	2.10	4427.11	1.63125436659215111
+AAAAAAAAMALAAAAA	White, fair artists take. Simply silent years could create general, alternative issues. Deliberately natural moves take so n	Home                                              	lighting                                          	5.13	1353.00	0.49853903743055412
+AAAAAAAAMBDBAAAA	Regular villages will raise finally small, rich terms; working-class, smooth states may violate european interests; discussions should not tell particularly times. Delightful, previous obje	Home                                              	lighting                                          	2.57	1509.56	0.55622659966272526
+AAAAAAAAMGMBAAAA	Happy sorts should care. Definite, sensitive pages should happen else smooth clouds. Local, legal years might not represent easy unfair clothes. Poor, other powers change only fo	Home                                              	lighting                                          	8.25	6600.48	2.43207460885411963
+AAAAAAAAMMIAAAAA	Plates shall think; new, economic pupils collect entirely. Really powerful books develop yet girls. Best unlik	Home                                              	lighting                                          	3.44	2151.42	0.79273233991784386
+AAAAAAAAMNDEAAAA	Writers say. Spanish, local targets find able weapons. Figures would win most into the effects; as steady workers shall understand. Social point	Home                                              	lighting                                          	5.26	5754.60	2.12039375077447653
+AAAAAAAAMNNCAAAA	Fiscal, occasional subjects ought to provide ill altogether royal stocks. Individual students save within a students.	Home                                              	lighting                                          	2.33	6565.32	2.41911922632931676
+AAAAAAAANDPAAAAA	Villages	Home                                              	lighting                                          	3.15	5303.78	1.95428039611487386
+AAAAAAAANEJCAAAA	Rich, logical 	Home                                              	lighting                                          	7.93	2820.76	1.03936361805070942
+AAAAAAAANMKCAAAA	Residents a	Home                                              	lighting                                          	4.83	13929.25	5.13250176432338949
+AAAAAAAANPJDAAAA	Ridicu	Home                                              	lighting                                          	4.71	6980.98	2.57227719846411656
+AAAAAAAAODEEAAAA	Behind aware variables cannot bring into a contents. Different, electronic mistakes measure; however additional students should like. Interesting sales wo	Home                                              	lighting                                          	1.37	1624.72	0.59865953059436060
+AAAAAAAAOFJAAAAA	Common feet cannot send at a engines. Orders should think prime, conservative cell	Home                                              	lighting                                          	2.52	2080.16	0.76647521367445784
+AAAAAAAAOJJDAAAA	Emotional areas make then new targets. Difficulties should halt much. Military circumstances might mount very much white systems. Other holidays drag further through a	Home                                              	lighting                                          	6.63	10785.78	3.97422940069306875
+AAAAAAAAPAEDAAAA	Yet young minutes could not walk here; enough pale others may not	Home                                              	lighting                                          	1.89	7242.84	2.66876458378678093
+AAAAAAAAPGDBAAAA	Difficulties apply just initially high surroundings. Enough usual requirements assist repeatedly for a students. Directions make too through the flowers. More national historia	Home                                              	lighting                                          	9.68	372.50	0.13725483476931368
+AAAAAAAAPHCEAAAA	Always top problems change almost expensive women. Supreme, industrial discussions 	Home                                              	lighting                                          	4.16	1004.00	0.36994323250574748
+AAAAAAAAPIBBAAAA	Discussions emerge so annual lessons. Good, early faces play really legislative products. Cold, private societies understand clearly ahead fat manufacturers. Abstract causes become so as executi	Home                                              	lighting                                          	9.11	4351.81	1.60350862415422005
+AAAAAAAAABOCAAAA	Approximately senior colours cannot encomp	Home                                              	mattresses                                        	4.73	2262.11	0.80877478687478841
+AAAAAAAAAKBDAAAA	Facilities shall look just much quiet clients. Specific prices should explain on a ways. Aspects ought to establish ill high chains. Suitable, enormous areas c	Home                                              	mattresses                                        	0.21	4913.00	1.75655053375646430
+AAAAAAAAAMNBAAAA	Sufficient, united companies earn either for no months. Comfortable, big tears come spiritual, old bir	Home                                              	mattresses                                        	6.95	6514.82	2.32925107843014222
+AAAAAAAABLHAAAAA	Complex, social miles cannot tie faces; probably future universities get objectives. Given settlements cannot g	Home                                              	mattresses                                        	4.30	100.50	0.03593188044830545
+AAAAAAAACHEEAAAA	Even widespread figures help also new, coloured trees. American, potential chapters may get actually years. Genes alter sides. Fingers develop par	Home                                              	mattresses                                        	4.87	NULL	NULL
+AAAAAAAADGBAAAAA	Dark companies stem in a offices. However multiple hours will preserve most long copies. Over mil	Home                                              	mattresses                                        	4.19	265.00	0.09474575441592979
+AAAAAAAADHAAAAAA	Early children shall not burst environmental	Home                                              	mattresses                                        	29.32	1972.12	0.70509432905186207
+AAAAAAAAEAIBAAAA	Strong t	Home                                              	mattresses                                        	3.26	972.30	0.34762753591927748
+AAAAAAAAEHGAAAAA	Also unknown books want very structural eyes. Well existing years could not buy much constant simple clients. Clouds find; ordinary, magic years prevent surely. Pensioners	Home                                              	mattresses                                        	0.47	5228.57	1.86937663836414340
+AAAAAAAAEHLBAAAA	Central, new children agree strangely legitimate, full values. Underneath adequate rights avoid just rough museums; dead, local shareholders spare various forces. Small letters force finally women.	Home                                              	mattresses                                        	2.58	4991.57	1.78464175611291563
+AAAAAAAAEJBCAAAA	Terms connect too all personal doctors. Current, new hours used to revive for the schools; practical, willing leaders discuss cases. Ago new structures must answer. More willing minutes claim more. F	Home                                              	mattresses                                        	5.91	5652.60	2.02098057136409324
+AAAAAAAAEPKDAAAA	Physically useless findings continue other critics; perhaps young forms substitute coins; arms command 	Home                                              	mattresses                                        	0.77	13274.08	4.74589707085813303
+AAAAAAAAFBOCAAAA	Groups make in t	Home                                              	mattresses                                        	4.98	5572.29	1.99226724480883542
+AAAAAAAAGEGBAAAA	Skills should spend twins. Certain, industrial homes will get to a rights. Decisions could buy politically so difficult differences. Running magistrates cannot respect thickl	Home                                              	mattresses                                        	7.20	4964.20	1.77485612857191941
+AAAAAAAAGGFDAAAA	Here extra efforts ensure eyes; merely little periods will not loosen home past a boys. Just local aspects must reclaim. Standard qualities might not roll today. Military, national clothes must go wid	Home                                              	mattresses                                        	3.34	4129.43	1.47639985153876580
+AAAAAAAAGHKAAAAA	Possible, rich d	Home                                              	mattresses                                        	4.63	10156.22	3.63116500344963929
+AAAAAAAAGIBEAAAA	Japanese years escape so good objects. Tiny features see then proud heads; abroad full secrets might not re	Home                                              	mattresses                                        	0.95	2753.98	0.98463363300521627
+AAAAAAAAGOMDAAAA	Past, interior years fetch accidents. Away internal feet would not organise so square collective rocks. M	Home                                              	mattresses                                        	6.31	3321.81	1.18765054519388575
+AAAAAAAAGPDCAAAA	National, difficult pain	Home                                              	mattresses                                        	0.37	987.66	0.35311921436391401
+AAAAAAAAHCGBAAAA	British differences discuss almost in the advantages; in particular international operations go then in a architects. Regional, fair costs commit merely political items. Then difficult travel	Home                                              	mattresses                                        	3.06	430.92	0.15406732261476401
+AAAAAAAAHNGCAAAA	Never arab policies follow only. Valuable employees might shed. Recently relative costs order just with a areas; sessions may come somewh	Home                                              	mattresses                                        	6.84	7661.12	2.73908903423006793
+AAAAAAAAIEFAAAAA	Perhaps blank models work certain	Home                                              	mattresses                                        	4.17	1990.47	0.71165502563122929
+AAAAAAAAIEJBAAAA	Keys must not read political, italian farmers. Red, single years should play however at the dates. Authors disturb no longer for a purposes. Ever essential agencies will answer as fundame	Home                                              	mattresses                                        	42.14	5401.80	1.93131175926026233
+AAAAAAAAIHLDAAAA	Payments forget. Doubts make respects. Considerable, available states should go here. Only public pages might differ. In	Home                                              	mattresses                                        	3.45	2289.13	0.81843527851372585
+AAAAAAAAIJOBAAAA	Well able areas examine respectively previous services. Surprised computers ought to love british, sole appeals. Common, similar inhabitants finish from a seco	Home                                              	mattresses                                        	7.94	3465.86	1.23915290716979022
+AAAAAAAAIMDAAAAA	Social councils used to determine yet at the boats. Persons ask alive months. Individual, considerable rooms note cases. Then only policies may look to a 	Home                                              	mattresses                                        	4.91	4363.94	1.56024448122963257
+AAAAAAAAJCPCAAAA	Films must ta	Home                                              	mattresses                                        	6.04	6064.00	2.16806888595546499
+AAAAAAAAKJGDAAAA	Educational hopes appear more high others; black thoughts might close always in a officials; close years base top workers. Regulations ask over high widespread	Home                                              	mattresses                                        	3.52	15000.77	5.36324253007489455
+AAAAAAAAKNFDAAAA	Vital arms generate slow, neat judges. Specially simi	Home                                              	mattresses                                        	4.42	10296.27	3.68123724083058633
+AAAAAAAAKNIBAAAA	Closely blind winners might come similar, local crops. Very difficult evenings can stretch only ago naked hands. Sufficient, similar 	Home                                              	mattresses                                        	6.05	13831.69	4.94526001470668627
+AAAAAAAAKOCCAAAA	Natural beans look often bacteria. Square, small items must negotiate for the forces. Hence chief ha	Home                                              	mattresses                                        	6.40	161.10	0.05759826806191052
+AAAAAAAAKPKCAAAA	Large, very materials like otherwise long, rough concepts. Sources give as local children. Rapid customers remove gently downwards short expressions. Behind national crimes confess n	Home                                              	mattresses                                        	7.74	1076.05	0.38472139260098583
+AAAAAAAALEOCAAAA	Growing, social objectiv	Home                                              	mattresses                                        	7.70	8.96	0.00320347909270464
+AAAAAAAALFIBAAAA	Ago new studies shall not apply of course small forces. Dead parts used to point on a students. Then other students should pay only	Home                                              	mattresses                                        	8.92	16657.18	5.95546070015825401
+AAAAAAAALGIDAAAA	Good, ethical interests stress now because of the eyes; patients used to give so hills. Social operations will pronounce basic ideas. British friends store too p	Home                                              	mattresses                                        	0.68	2433.04	0.86988758612880682
+AAAAAAAALKOCAAAA	Following, combined cells must ease of course continued changes. German te	Home                                              	mattresses                                        	5.91	785.92	0.28099088041723599
+AAAAAAAALMHBAAAA	Old words cannot force. Equal, capital problems would not produce; great, competitive things congratulate only times. Vice versa unemployed complaints will say previous gardens. Difficult, uncomfort	Home                                              	mattresses                                        	1.57	1412.84	0.50513430818491411
+AAAAAAAAMDIDAAAA	Now comfortable grounds bowl much only double groups. Good talks must not support somewhat; used, linear	Home                                              	mattresses                                        	5.00	5416.79	1.93667115117986530
+AAAAAAAAMMCCAAAA	Respectively excellent things speak reliable, historical movements. Masters respond. Cheap ideas should featu	Home                                              	mattresses                                        	3.37	5563.35	1.98907091633910557
+AAAAAAAAMMEEAAAA	Prisoners ought to leave. Main items should not strengthen ago allowances. Ideas provide together between a patients. Regional, english conditions argue also in a minutes; ordinary trials become lon	Home                                              	mattresses                                        	36.96	6326.30	2.26184930626979851
+AAAAAAAAMPNBAAAA	Cases move so very natural tories. Therefore political cells win there already eastern events. Extra questions encourage skilled efforts. Serious, physical clothes would 	Home                                              	mattresses                                        	80.68	751.60	0.26872041139250123
+AAAAAAAANAFBAAAA	Individuals recognise. Really elegant relations should extend totally types; attitudes would relate muc	Home                                              	mattresses                                        	7.09	1139.56	0.40742819585742244
+AAAAAAAANCMBAAAA	Evidently super tales may risk just; others match maybe. Lovers describe anywhere 	Home                                              	mattresses                                        	2.32	9619.86	3.43939959651179740
+AAAAAAAANLJBAAAA	Minimum words	Home                                              	mattresses                                        	6.86	4696.59	1.67917721785777990
+AAAAAAAANOFCAAAA	Other, extra notes alter so social ways. Different, preliminary parts take so diffic	Home                                              	mattresses                                        	3.40	10150.42	3.62909132278695101
+AAAAAAAAOAFAAAAA	So social decisions fulfil again comparative times. Academic governments ought to arise then on a decades. 	Home                                              	mattresses                                        	1.81	1346.52	0.48142284240051991
+AAAAAAAAOICBAAAA	Often presidential councillors used to take in the friends. Exact, rich visits used to want sophi	Home                                              	mattresses                                        	0.41	8719.30	3.11742134520308145
+AAAAAAAAOIIBAAAA	Constant, domestic things might worry like a minutes. Literary, kind sales tell however emotional branches. Too specific troops may not bring most fair unknown owners. Issues look official	Home                                              	mattresses                                        	0.51	148.32	0.05302901998102153
+AAAAAAAAOJJCAAAA	Even successful questions continue indian areas; good, good jobs get nice, famous interests. Labour, generous circumstances help good changes. Strict, vulnera	Home                                              	mattresses                                        	2.55	2079.26	0.74340021632779686
+AAAAAAAAOLCEAAAA	Good, full governments consider elsewhere genuinely	Home                                              	mattresses                                        	0.33	11909.49	4.25801364059989293
+AAAAAAAAOMLDAAAA	Days should conti	Home                                              	mattresses                                        	3.57	1697.42	0.60688052249316052
+AAAAAAAAOODAAAAA	Personal, national arts ought to rely still strategic, dead instruments. Finally federal spots remember. Laws 	Home                                              	mattresses                                        	3.72	13796.99	4.93285368384543056
+AAAAAAAAOOJBAAAA	New products should not see. Much separate subjects give at least existing implications. Similar corporations might turn years; local	Home                                              	mattresses                                        	3.84	1888.50	0.67519757439427698
+AAAAAAAAPBMAAAAA	Other parties will add regional, special owners. Little administrative horses may indicate; 	Home                                              	mattresses                                        	1.41	23082.32	8.25264838516945064
+AAAAAAAAACMAAAAA	Often local men ought to suppress trousers. Angry studies might cool seeming	Home                                              	paint                                             	0.70	4572.36	1.91646328201692969
+AAAAAAAAAGODAAAA	Worthy, rich types force both shy years. Tropical, personal views might work. Other eyes ought to administer neve	Home                                              	paint                                             	0.28	12758.19	5.34747978724238078
+AAAAAAAAAHAEAAAA	Rural others come as through a estimates. Publications should worry really powerful	Home                                              	paint                                             	3.24	4960.42	2.07911511634744823
+AAAAAAAAAHEAAAAA	Early, dangerous weeks live still to a changes. Vari	Home                                              	paint                                             	2.74	12614.97	5.28745042138963413
+AAAAAAAABECDAAAA	Perhaps de	Home                                              	paint                                             	1.44	1475.69	0.61852209813740890
+AAAAAAAABMKCAAAA	Clinical, national residents might cry; objects ought to justify only relatives	Home                                              	paint                                             	7.77	2688.57	1.12688976505180184
+AAAAAAAABOOCAAAA	Equal forces tell together feet. Never new police cannot place hardly big, independent years. Then old choices ought to afford especially; parties accept	Home                                              	paint                                             	6.51	6336.50	2.65588658515520978
+AAAAAAAACBPCAAAA	Always huge concessions express directly ameri	Home                                              	paint                                             	4.52	9357.30	3.92202754569128769
+AAAAAAAACCEDAAAA	Average decisions shall see. Lovely, wide temperatures prepare in a regulations. Right arms ought to make now applic	Home                                              	paint                                             	57.27	3310.24	1.38745711507049343
+AAAAAAAACJBCAAAA	Vast, separate feet wear financially other, dangerous workers. Other, old genes spin for instance ordinary purposes. Events could focus anywhere under fresh countries	Home                                              	paint                                             	7.37	10616.13	4.44965473893533925
+AAAAAAAACNPBAAAA	Quickly far walls shall see gold, true patients. Above bad pensions will insist as round detailed degrees. Free,	Home                                              	paint                                             	0.70	809.31	0.33921495655834654
+AAAAAAAADCHBAAAA	Probably political hands may park easily. Little, suitable officials apply today men; women ought to take to the provi	Home                                              	paint                                             	6.55	2700.80	1.13201585878437474
+AAAAAAAAEBOBAAAA	Special words should tell by a follower	Home                                              	paint                                             	1.68	592.00	0.24813143824065086
+AAAAAAAAECABAAAA	Both usual effects include repeatedly low, possible practices. Professional, past countries retain yesterday ways. Equally old	Home                                              	paint                                             	0.84	1006.06	0.42168093708849528
+AAAAAAAAECEDAAAA	Capital areas judge almost active, numerous f	Home                                              	paint                                             	9.32	661.29	0.27717371417932434
+AAAAAAAAEEODAAAA	Pale, original yards agree social, little generations. Weeks used to include now oral shows; 	Home                                              	paint                                             	2.40	5882.28	2.46550438603752661
+AAAAAAAAFKDBAAAA	Appropriate, like v	Home                                              	paint                                             	4.82	372.76	0.15623897790301523
+AAAAAAAAGABEAAAA	Attitudes must build ge	Home                                              	paint                                             	45.77	9930.33	4.16220788024372040
+AAAAAAAAGBEDAAAA	Very difficult parts ought to know else areas. Members could not comment of course male, popular girls. Primary, worried actions might send indirectly elsewhere hard children. New resou	Home                                              	paint                                             	3.98	770.04	0.32275529172775471
+AAAAAAAAGCJCAAAA	Careful universities may find cultural methods; artificial, apparent sections ought to tell highly reforms. Medical, glorious studies shall not agree straight almost actual states. Enough n	Home                                              	paint                                             	4.20	103.50	0.04338108759781649
+AAAAAAAAGEGDAAAA	Players shall mean with a rights. Occasionally popular enemies worry. In general basic patients get perhaps parts. Other varieties enjoy thousands; classes shall not spend as for the families. New f	Home                                              	paint                                             	2.13	5837.14	2.44658436387167698
+AAAAAAAAGKBDAAAA	Students wi	Home                                              	paint                                             	2.79	4724.08	1.98005534588495595
+AAAAAAAAGKHDAAAA	For example clear witnesses used to enjoy yet international, environmental computers. Ill	Home                                              	paint                                             	9.67	59.46	0.02492212046923835
+AAAAAAAAIABEAAAA	Opposite youngsters see altogether. Plans may not say to the problems. Popular, new lands might create cha	Home                                              	paint                                             	4.08	7043.01	2.95201385277582167
+AAAAAAAAIHFCAAAA	Objects sell so yellow ru	Home                                              	paint                                             	1.47	1136.47	0.47634110746174406
+AAAAAAAAIMCDAAAA	Only horses can forget meanwhile animals. Rich exception	Home                                              	paint                                             	67.74	386.10	0.16183031808228935
+AAAAAAAAIOCBAAAA	Responsible, useless sources explore there. Serious, conventional fields could defend once again famous efforts. Officials call as notions. Big, ap	Home                                              	paint                                             	3.14	8952.05	3.75217067855104485
+AAAAAAAAJCIAAAAA	Aware groups could finish services. Companies make also glad, top ways; t	Home                                              	paint                                             	3.27	1574.90	0.66010507108986663
+AAAAAAAAJEGAAAAA	Ever insufficien	Home                                              	paint                                             	2.77	3898.21	1.63389941531095878
+AAAAAAAAJGABAAAA	Se	Home                                              	paint                                             	7.48	13291.94	5.57119626555479193
+AAAAAAAAJJHDAAAA	Windows avoid. Always noble funds should lead nowhere able initiatives. Under new groups wait plans. High enterprises could know inadvertently different, main	Home                                              	paint                                             	8.31	804.05	0.33701027519830292
+AAAAAAAAKEFEAAAA	Human honours tell requests. Effective, late crimes know on a courses. Adequate, typical men should not tend already in a nerves. 	Home                                              	paint                                             	1.35	7526.60	3.15470622138865334
+AAAAAAAAKEJDAAAA	Patterns might not maintain. Great, vast eyes say still different views. Easily national plants develop together with the cities. Able g	Home                                              	paint                                             	21.04	8770.96	3.67626844518787008
+AAAAAAAAKGOBAAAA	Possible guests want only; organisations weigh however explicitly c	Home                                              	paint                                             	4.69	2761.50	1.15745771402290094
+AAAAAAAAKLDBAAAA	Letters state on a chains. General, criminal cases shall look unknown months. Special, poor nights give as ever	Home                                              	paint                                             	7.47	3235.66	1.35619758354348711
+AAAAAAAAKOBBAAAA	Also inc goods could not lay 	Home                                              	paint                                             	2.41	2540.30	1.06474373743703612
+AAAAAAAALCBCAAAA	Additional companies visit. Grey opportunities may not look numbers. Entire, british models assist also great quarters. Little males show	Home                                              	paint                                             	51.57	13562.60	5.68464095318015436
+AAAAAAAALFPDAAAA	Communist, different demands die perhaps kinds; likely, public forests should make moral, nice faces. Efficient, central services can p	Home                                              	paint                                             	0.27	668.17	0.28005740386698596
+AAAAAAAALPPCAAAA	Effectively initial representatives amount dark areas; comprehensive, christian words will not want hearts. There judicial men explain r	Home                                              	paint                                             	4.54	5116.69	2.14461427150600652
+AAAAAAAAMDABAAAA	Reasons will look probably key students. Now very bones us	Home                                              	paint                                             	3.58	54.00	0.02263361092059991
+AAAAAAAAMFDCAAAA	Features need stages; french cells come hours. Still small beliefs look scarcely electric, good producers. Churches receive for the seats; businesses get appropriate, high ways. Purpo	Home                                              	paint                                             	2.89	7559.52	3.16850434123135981
+AAAAAAAAMGJBAAAA	Managers ought to express so new faces. Universities should not appear at a stories. Accidents dismiss only single times. Other, current companies could not meet effectively that is to say perfe	Home                                              	paint                                             	0.74	6272.75	2.62916635004061266
+AAAAAAAANGCBAAAA	There blue items see in a conditions; lives ask silent countries. Here necessary months may encourage free 	Home                                              	paint                                             	7.02	4828.00	2.02361247267882156
+AAAAAAAAOEABAAAA	New dollars might end old relationships. Other, gentle groups 	Home                                              	paint                                             	8.34	2369.97	0.99335146062026237
+AAAAAAAAONBBAAAA	International years collect respectively affairs. Exter	Home                                              	paint                                             	69.84	5908.06	2.47630983954739820
+AAAAAAAAONNBAAAA	Colleagues attach th	Home                                              	paint                                             	9.80	2499.83	1.04778110347487541
+AAAAAAAAOPEEAAAA	Furthermore additional 	Home                                              	paint                                             	8.18	1563.59	0.65536458702482987
+AAAAAAAAPDOCAAAA	Months find there costly foreigners. White, particular changes used to share away in a subjects. Muscles make fully less gold fingers. Norm	Home                                              	paint                                             	4.97	14512.01	6.08257755584916844
+AAAAAAAAPIIDAAAA	English persons last there golden units. Special services help far vital fingers. Very complicated birds sho	Home                                              	paint                                             	0.74	1043.89	0.43753703896120444
+AAAAAAAAPKDDAAAA	Hands might contact enough growing things. Criteria used to make convincing forms. Particular organizations sha	Home                                              	paint                                             	48.89	8562.98	3.58909551186812250
+AAAAAAAAACDAAAAA	New, american owners might not appear. Parties move heavily also high variations. Unable, american terms might create indeed years. Nations absorb over normal experienc	Home                                              	rugs                                              	0.89	2701.48	0.99827241978850362
+AAAAAAAAAINBAAAA	Concepts shall allow cautiously there	Home                                              	rugs                                              	4.82	8082.19	2.98659526203801105
+AAAAAAAABDJAAAAA	Awards might mention better real, video-taped fires. Familiar patients must yield finally never net rules. Courses should attend; black ac	Home                                              	rugs                                              	0.79	120.11	0.04438400444970800
+AAAAAAAABHIAAAAA	Smoothly main organisations yield here pensioners; subtle, british rights say public books. Only, social pairs take up to the police. Important, other men could go mor	Home                                              	rugs                                              	6.67	21599.16	7.98149374365127852
+AAAAAAAABNNBAAAA	For example brief children must change almost. Fierce manufacturers ought to throw comfortably alone, subsequent loans; other boots switch. Very main men k	Home                                              	rugs                                              	7.88	1113.44	0.41144722266657961
+AAAAAAAABOGBAAAA	Forms carry here american negotiations. Partly subject drivers should tell only stiffly local orders. Quite clean forces will enhance intentionally full ministers; stories mus	Home                                              	rugs                                              	7.64	9195.42	3.39796488383093785
+AAAAAAAACCFCAAAA	Royal, comprehensive reports cost loyal, critical minutes. Exciting, short areas ought to pay for a appearances. Public, large institutions can	Home                                              	rugs                                              	4.30	2726.74	1.00760669630502701
+AAAAAAAACCHAAAAA	Of course institutional forces occur violently from a governments. Patient, western teams count 	Home                                              	rugs                                              	1.97	500.94	0.18511134117922509
+AAAAAAAACCPAAAAA	Great images may not pay only, certain plans. Internationally new years command so in the days. Stairs tell teams; else unlike customers see elected, different numbe	Home                                              	rugs                                              	2.11	8294.23	3.06494997274915987
+AAAAAAAACDPCAAAA	Organizations understand also instead accurate settlements. Costs become co	Home                                              	rugs                                              	7.44	12898.01	4.76617544944116470
+AAAAAAAACELAAAAA	Broad, political premises must not continue even. Short local levels stay in a germans. Encouraging, poor priorities i	Home                                              	rugs                                              	9.98	13098.17	4.84014016787138328
+AAAAAAAACIGAAAAA	Consumers must light now human schools; systems take 	Home                                              	rugs                                              	37.18	2295.76	0.84834753189127999
+AAAAAAAADANDAAAA	Hardly happy reforms may not try quickly along a pp.; sure sources use then now different	Home                                              	rugs                                              	3.58	2396.96	0.88574376243253759
+AAAAAAAADLCAAAAA	However magic things should not take for a firms. Estimates supply; able, doubtful children must maintain left, lacking banks; simple sons c	Home                                              	rugs                                              	1.73	113.88	0.04208184519800805
+AAAAAAAADMFCAAAA	Ideological members get sometimes modest abilities. Used, certain services would make all victorian, angry regulations. Even voluntary directions must sail however equations. Other, specific others ge	Home                                              	rugs                                              	8.46	4771.52	1.76321009834210907
+AAAAAAAADNGDAAAA	Turkish members shall know to a subjects. No doubt decisive millions might know virtually public industries. Good, artificial 	Home                                              	rugs                                              	1.62	4557.68	1.68419023728536476
+AAAAAAAAEDBAAAAA	Softly social men get only with a miles. Only afraid difficulties should emerge t	Home                                              	rugs                                              	4.09	5355.01	1.97882597342628292
+AAAAAAAAEJFCAAAA	Others could withdraw buildings. Clothes know partly. Inner prese	Home                                              	rugs                                              	4.44	7946.40	2.93641705902222677
+AAAAAAAAEJLBAAAA	Parallel dead relations check further international men. Types improve apart half way steady ways; back metres shall not support at leas	Home                                              	rugs                                              	1.00	9684.36	3.57864188937285967
+AAAAAAAAELABAAAA	Good, alone centuries might not see gently subjective ships. Less ambitious 	Home                                              	rugs                                              	6.42	3762.17	1.39022704204943760
+AAAAAAAAFBPBAAAA	Also other republics could not prescribe almost permanent mental p	Home                                              	rugs                                              	3.56	1252.71	0.46291138301718183
+AAAAAAAAFDIDAAAA	Coastal agencies encourage. Obviously other events understand local students. Western subjects cannot set in a e	Home                                              	rugs                                              	6.19	3558.04	1.31479529757921118
+AAAAAAAAFICEAAAA	Existing services make talks. Concerned, running	Home                                              	rugs                                              	30.02	2214.66	0.81837881354250538
+AAAAAAAAFOOBAAAA	However major months execute either elements. Enough left provisions used to prove so away gastric police. Animals shall add faintly things. Well modern principles might pay suddenly other, soc	Home                                              	rugs                                              	1.32	16957.77	6.26637032001602569
+AAAAAAAAGBCBAAAA	Mental horses could grab 	Home                                              	rugs                                              	1.74	1044.31	0.38590175411601501
+AAAAAAAAGJOAAAAA	Other, initial companies could know definitely mere funds. Italian years get already thereafte	Home                                              	rugs                                              	8.14	4357.37	1.61017008965967989
+AAAAAAAAHJCEAAAA	Additional, interior police provide words. Different, long qualities answer really concerns; then other words state dry, political services. Awfully di	Home                                              	rugs                                              	9.78	7977.70	2.94798328447619281
+AAAAAAAAHLNDAAAA	Firm, main skills can measure already electoral, white activities. Fairly disciplinary men protest there new changes. Strong, good reactions might prompt arbitrarily wild product	Home                                              	rugs                                              	6.42	9423.50	3.48224682317728204
+AAAAAAAAIBLCAAAA	Origins used to play very on a matters. Long, important shows tackle more. Further vast fingers succeed only. Much dead values must rem	Home                                              	rugs                                              	4.71	7612.23	2.81293189736242391
+AAAAAAAAIEIAAAAA	Possibly southern complaints would not produce to a years; months take. Services give; always professional days might develop quickly major urba	Home                                              	rugs                                              	36.03	10189.52	3.76531263858453641
+AAAAAAAAIHNBAAAA	British stories ought to read furt	Home                                              	rugs                                              	2.05	1296.18	0.47897476386331293
+AAAAAAAAIKFCAAAA	Better silent colleges protect never concessions. Certainly material words 	Home                                              	rugs                                              	2.45	7108.50	2.62678957314752580
+AAAAAAAAINHAAAAA	Still global systems would find real forces. Facts get rivals. Ahead british features must not rest nearly. Flats will restrict always subsequent miles. Then new children can allay only ordi	Home                                              	rugs                                              	8.72	430.95	0.15924807857465376
+AAAAAAAAINPBAAAA	Possible	Home                                              	rugs                                              	0.41	9833.88	3.63389371141365844
+AAAAAAAAJEBCAAAA	True 	Home                                              	rugs                                              	55.56	1867.47	0.69008239771622846
+AAAAAAAAJINBAAAA	Difficult writings improve full charges. Western incidents run in a options. Parts happen possible, forw	Home                                              	rugs                                              	4.45	2413.98	0.89203312847811273
+AAAAAAAAJKECAAAA	Past losses will feel nowhere options. Political, free situations must produce selectively classes. Difficult ways believe sometimes enormous scientists. Interesting, simple rights ought to flush ago	Home                                              	rugs                                              	4.83	1972.51	0.72889761566142310
+AAAAAAAAJMMBAAAA	Minds apply reluctantly dirty goods; therefore extended unions make secret, working men. Followin	Home                                              	rugs                                              	0.63	215.67	0.07969609724143306
+AAAAAAAAKAADAAAA	Possible, false publications produce toda	Home                                              	rugs                                              	62.90	1868.41	0.69042975400781722
+AAAAAAAAKGMAAAAA	Wonderful, scottish unions go nearby for a teams. Gladly current systems cannot look so major, emotional p	Home                                              	rugs                                              	7.31	5243.52	1.93762730007603777
+AAAAAAAAKLJCAAAA	Dead names spend as a schools. Polit	Home                                              	rugs                                              	1.98	718.90	0.26565365747144353
+AAAAAAAALEGBAAAA	Standard, foreign hospitals say later adult difficulties. Things ask very into a metals. Enough public persons will not give however ago sweet c	Home                                              	rugs                                              	0.57	5940.00	2.19499614046511968
+AAAAAAAALKFCAAAA	Single institutions place also local ideas; variations used to appear yesterday domestic, corresponding attempts. Unlike, possible amounts open locally. National, main cig	Home                                              	rugs                                              	7.07	11038.74	4.07912318107709347
+AAAAAAAAMCLBAAAA	Also noble characteristics might sound about a miles. Again social funds would stretch en	Home                                              	rugs                                              	7.90	2544.16	0.94013827958345773
+AAAAAAAAMFEBAAAA	International metres minimise originally small allowances. Eminently favorite lines compare just never bottom things. British poets take countries; individual, in	Home                                              	rugs                                              	1.63	3135.51	1.15865864451006522
+AAAAAAAANBLDAAAA	Colourful bones may adjust so. Pupils might catch so. Final,	Home                                              	rugs                                              	86.39	282.42	0.10436208922393251
+AAAAAAAANDFEAAAA	Able armies bring certain, pretty requirements. Dogs pay weeks. Simi	Home                                              	rugs                                              	46.20	4674.82	1.72747674366484020
+AAAAAAAAOAACAAAA	Foreign, absolute bills sh	Home                                              	rugs                                              	0.23	4232.41	1.56399387455656182
+AAAAAAAAOEPCAAAA	Levels look only steep, cold results. Examples used to ensure together only expensi	Home                                              	rugs                                              	5.36	2875.57	1.06260354404668084
+AAAAAAAAOGBCAAAA	African days incorporate economic, similar cells; vast, automatic stations ought to plan previously in a judges. Blank times would pay into the workers. Gradually ultima	Home                                              	rugs                                              	2.42	1831.70	0.67686438223736696
+AAAAAAAAOGEEAAAA	Hands order. Pl	Home                                              	rugs                                              	91.05	5998.14	2.21648049662785404
+AAAAAAAAOGKBAAAA	Magic facilities should not fight only likely months. Later present members absorb once more	Home                                              	rugs                                              	8.11	1193.91	0.44118313839439580
+AAAAAAAAOGLBAAAA	As active accounts talk slowly. Big implications make as a children. Rounds should not check. Likely, military	Home                                              	rugs                                              	5.59	2607.00	0.96335941720413586
+AAAAAAAAONFDAAAA	Prime members must need so regulations. Only injuries might run adequately to a shares; inevitably orthodox poets think yesterday protests. Thinking, full changes could put more. Months 	Home                                              	rugs                                              	9.27	2740.60	1.01272835396611229
+AAAAAAAAPKJDAAAA	Clinical photographs look also popular, common men. Loose, concerned earnings must go maybe only able enquiries; black unions observe exactly by a	Home                                              	rugs                                              	24.08	2749.12	1.01587673226859761
+AAAAAAAAACBEAAAA	Directly green hours will maintain also	Home                                              	tables                                            	1.10	1433.48	0.74353680625971805
+AAAAAAAAAEDDAAAA	Then legal services may bother circumstances; obvious, original years worry scottish, static areas; much fresh journals mean exactly routes. I	Home                                              	tables                                            	4.46	15267.45	7.91912758652365733
+AAAAAAAAAFFBAAAA	Small motives shall use large, patient payments. Answers refer here odd, average officers. Always powerful sections might yield into a 	Home                                              	tables                                            	4.41	5271.29	2.73418403568155059
+AAAAAAAAAGPBAAAA	Odd, poor times could recycle suddenly eyes. Fa	Home                                              	tables                                            	0.33	2225.20	1.15419685052398680
+AAAAAAAAALACAAAA	Perfect grants fight highly as great minutes. Severe, available millions like counties. Young legs cook however from a years. Early armed services reject yet with 	Home                                              	tables                                            	4.31	7602.83	3.94353875654740364
+AAAAAAAAALJAAAAA	True, particular parties drop for a times. Too mad 	Home                                              	tables                                            	56.61	2020.10	1.04781280682343418
+AAAAAAAAANECAAAA	Usually complete artists will give from the weeks. Units swallow political minutes; books might not arrest continually lips. Modest, royal problems must behave consequently genera	Home                                              	tables                                            	4.25	4496.26	2.33218098648974514
+AAAAAAAAAOPDAAAA	Particularly popular detectives avoid rather free, major relations. Financial servants may know also widely surprising children. Delegates cannot get. However separate thousands discuss alway	Home                                              	tables                                            	4.93	7737.75	4.01352088807387150
+AAAAAAAACBODAAAA	Nuclear needs can want. Overwhelmingly clo	Home                                              	tables                                            	0.43	930.32	0.48255096799365244
+AAAAAAAACDHBAAAA	Enough bad rounds arrange later well black places. Courses reduce then in a experts. Also poor systems offer wonderful performances. Economic, unlikel	Home                                              	tables                                            	21.49	7678.11	3.98258600574183368
+AAAAAAAACFBDAAAA	Actions see of course informal phrases. Markedly right men buy honest, additional stations. In order imaginative factors used to move human thanks. Centres shall catch altogether succe	Home                                              	tables                                            	1.61	33.06	0.01714800821423827
+AAAAAAAADNLBAAAA	Federal, clear months please. New lips take slightly interesting 	Home                                              	tables                                            	3.47	361.20	0.18735210426445445
+AAAAAAAAEEPDAAAA	Roots should not lend overnight in a feet; fine children retire once usually evident forests. Sometimes novel effects might not go tons. Casualties involve more. Correct, perfect deleg	Home                                              	tables                                            	3.13	10251.08	5.31716890637669900
+AAAAAAAAEFEBAAAA	Provincial, important tr	Home                                              	tables                                            	3.22	2399.31	1.24450658162444130
+AAAAAAAAEOFAAAAA	Western, complex eyes can tell only regular acts. Perhaps high processes could put. Changes stay in the prisoners. Ages give now fascinating methods. British, quick words shall not expect new 	Home                                              	tables                                            	4.27	9672.26	5.01693871537351095
+AAAAAAAAEPHAAAAA	Now professional schools will not visit useful lists. Beautiful plans can recommen	Home                                              	tables                                            	2.52	408.50	0.21188630839432348
+AAAAAAAAFNJBAAAA	Personal dimensions can dissolve final variations. Gradual sounds migh	Home                                              	tables                                            	1.19	5519.07	2.86270591938765946
+AAAAAAAAGBKDAAAA	Hard sheets share so books. Permanent	Home                                              	tables                                            	31.00	443.40	0.22998871271001966
+AAAAAAAAGBMAAAAA	Current degrees see in particular be	Home                                              	tables                                            	2.99	2250.99	1.16757395675039954
+AAAAAAAAGCDCAAAA	Vast girls call benefits. Good, difficult makers deliver local things. High, formal hours play for a payments; well new men increase all equal newspapers. Top, total rights	Home                                              	tables                                            	2.62	10786.92	5.59510564931431049
+AAAAAAAAGHNAAAAA	Just responsible poems ask only just joint patients. Solid, equal books prevent. Never universal fields must ignore black, main cameras	Home                                              	tables                                            	0.32	6835.22	3.54538441337343388
+AAAAAAAAGJHAAAAA	Most official languages might not feel anywhere careful points; good, post-war prices refer originally ruling varieties. Increased lands would not get we	Home                                              	tables                                            	0.35	13164.59	6.82838770287595335
+AAAAAAAAGNHDAAAA	Important, small girls should realise only high numbers. Previous, statutory products can give rather scientific methods. Isolated, living estates move now old trees; univ	Home                                              	tables                                            	2.85	3966.40	2.05734603088187185
+AAAAAAAAHFHDAAAA	More german bags might not give always about a words. Recently new guests ought to 	Home                                              	tables                                            	8.63	4805.11	2.49237948428065532
+AAAAAAAAIGDDAAAA	Too labour operators tell more	Home                                              	tables                                            	3.43	9131.41	4.73640331783356027
+AAAAAAAAIIBDAAAA	Families must not hear more great, english feelings. Proper faces justify extremely skills. Immediate discussions undertake often pa	Home                                              	tables                                            	0.18	2677.96	1.38904053470664016
+AAAAAAAAIODBAAAA	Experts should not offer; low easy cities flourish particularly integrated, decisive 	Home                                              	tables                                            	9.66	3549.82	1.84126867873766800
+AAAAAAAAIPJCAAAA	Simply different statements complete always social, international speakers. Early serious buildings shall overcome just by a husbands; complex, common criteria will work little, fair countr	Home                                              	tables                                            	2.23	2835.45	1.47072957928196943
+AAAAAAAAJANDAAAA	Only long brothers detect in a years; commitments can imagine near little great fields. Civil, soviet patients profit already just long arrangements. Often indi	Home                                              	tables                                            	8.94	690.05	0.35792447272338536
+AAAAAAAAJKHCAAAA	Central houses increase actually essential payments. Minor organizations take subsequently careful players; good, molecular righ	Home                                              	tables                                            	7.94	13582.01	7.04490075758821408
+AAAAAAAAKFABAAAA	Women get also chairs. Full, integrated paintings sit 	Home                                              	tables                                            	6.34	1123.11	0.58254989429803830
+AAAAAAAALJOAAAAA	Wild volunteers expand approximately sales. Specific, close versions must stress longer able powers. Far me	Home                                              	tables                                            	3.86	2363.26	1.22580767974533392
+AAAAAAAAMEKDAAAA	Bold parties could revert newly equal plans. Also other products cry as at least lovely discussions. Manufacturing, french letters lay economically ready duties; serious, stron	Home                                              	tables                                            	1.02	2741.71	1.42210724746095625
+AAAAAAAAMGGBAAAA	Areas ought to calculate slowly charges. Difficult, national participants might not write away bus	Home                                              	tables                                            	4.21	5457.26	2.83064547208814138
+AAAAAAAAMLKCAAAA	Closely young offic	Home                                              	tables                                            	8.10	25.92	0.01344453638575487
+AAAAAAAANAIBAAAA	Wide, new changes reduce highly on a notes. Nurses re	Home                                              	tables                                            	0.25	1860.34	0.96494632792728456
+AAAAAAAANFABAAAA	Critical, neighbouring feelings should achieve unusual, hungry types; po	Home                                              	tables                                            	5.93	619.20	0.32117503588192191
+AAAAAAAANJNCAAAA	A	Home                                              	tables                                            	4.83	2031.72	1.05384002568155423
+AAAAAAAANOJBAAAA	New situations seem. National missiles will cater departments. Women come astonishingly. Spanish mont	Home                                              	tables                                            	5.87	8171.71	4.23861313382858538
+AAAAAAAAODODAAAA	Highly tory votes could no	Home                                              	tables                                            	8.80	3686.85	1.91234525361961205
+AAAAAAAAOGPCAAAA	Slight, present techniques run writers. Schemes make. Grand boys could help fine, past re	Home                                              	tables                                            	1.51	332.04	0.17222700083048022
+AAAAAAAAONNDAAAA	Dead, big talks will rest old offers. Dead, competitive authorities occupy alone	Home                                              	tables                                            	0.38	2425.28	1.25797705268686622
+AAAAAAAAPDGEAAAA	Almost working things shall not see import	Home                                              	tables                                            	3.78	3316.68	1.72034046836055031
+AAAAAAAAPHDAAAAA	Police know more families. Atlantic birds might keep there far presen	Home                                              	tables                                            	40.62	0.00	0.00000000000000000
+AAAAAAAAPHDDAAAA	Obviously elaborate members would not retu	Home                                              	tables                                            	3.94	610.39	0.31660534585265877
+AAAAAAAAPLFDAAAA	Quiet levels must achieve. Local, national metres fill to a businessmen. Real, key boots could not determine at best. Young groups may know ever happy, magnetic difficulties	Home                                              	tables                                            	2.15	NULL	NULL
+AAAAAAAAAMODAAAA	Labour, middle children might produce useful signals. Surprising farmers kill on the costs. Trees return recent, single animals. Original governments read over there. Previous	Home                                              	wallpaper                                         	3.08	5699.40	1.39109945794862842
+AAAAAAAAAPGCAAAA	Once again only measures shall destroy independent, normal prisons. Present, industrial ambitions can prevent as employers. Large, previous origins say inside 	Home                                              	wallpaper                                         	3.32	262.60	0.06409494291632625
+AAAAAAAABBHBAAAA	Reports can say. Constant, other keys will analyse here white months. Dreams would not change to a neighbours; visual, financial wages set in a girls. Fingers 	Home                                              	wallpaper                                         	4.24	9127.17	2.22774348871898495
+AAAAAAAABCPBAAAA	Nearer regular men like in a ministers; children come therefore female views. Only financial events must not allow old miles. Very british forces get. 	Home                                              	wallpaper                                         	9.72	5545.66	1.35357487103333520
+AAAAAAAABPNCAAAA	Great, strategic live	Home                                              	wallpaper                                         	2.35	12111.89	2.95624866016307208
+AAAAAAAACCOBAAAA	Groups can consent close. Awful, soft friends pursue comfortable departments. C	Home                                              	wallpaper                                         	6.57	1777.90	0.43394668320996359
+AAAAAAAACDBEAAAA	Empty, additional russians should ensure commonly in a books. Sure, close difficulties follow always on a weeks. Royal y	Home                                              	wallpaper                                         	0.85	328.29	0.08012844177456491
+AAAAAAAACFPBAAAA	Educational, reasonable rooms mi	Home                                              	wallpaper                                         	2.73	737.08	0.17990518097778275
+AAAAAAAACGHCAAAA	Then french ministers aid	Home                                              	wallpaper                                         	3.16	7027.37	1.71522802361730232
+AAAAAAAACHCDAAAA	Old eyes would not develop to a parents; nice, red games come now to a molecules. Sheer centuries could follow as usually late powers; backs affect police. Almost tiny trees shall buy fro	Home                                              	wallpaper                                         	1.22	20810.71	5.07944123952101991
+AAAAAAAACKBDAAAA	American, long organisations terminate for a agents. Facilities determine open. Now general students rebuild even particular pounds. Good teachers might not press names. Guidelines evaluate clear	Home                                              	wallpaper                                         	4.09	293.44	0.07162231549644621
+AAAAAAAACNCCAAAA	Public 	Home                                              	wallpaper                                         	0.64	1015.94	0.24796883589646797
+AAAAAAAACNICAAAA	Initial unions agree still otherwise individual councillors. Leading minutes bring mathematical conditions. Full, huge banks must not feel exclusively special lines. Ago other cases will hold	Home                                              	wallpaper                                         	8.36	1699.28	0.41475725285169409
+AAAAAAAACNOCAAAA	Fresh, othe	Home                                              	wallpaper                                         	8.40	501.78	0.12247357371117359
+AAAAAAAADBGDAAAA	Ahead national cir	Home                                              	wallpaper                                         	14.29	13998.80	3.41680231110840781
+AAAAAAAAEDDBAAAA	Still fortun	Home                                              	wallpaper                                         	4.83	4391.94	1.07197693675525478
+AAAAAAAAEDKDAAAA	Minor, single things could cry too profits. Examples focus material, young observations. Existing tensions would stop away. Facilities reply most thoroughly small	Home                                              	wallpaper                                         	3.85	6735.50	1.64398891094027208
+AAAAAAAAEEPCAAAA	Wooden, clear considerations will not become now proceedings. A bit institutional firms will 	Home                                              	wallpaper                                         	4.94	9408.96	2.29652229284842735
+AAAAAAAAEKDCAAAA	Thick, other ways come completely. Careful men would find later there valid children. Interesting owners allow a bit best wide polls. Miles behave other, considerable heads; inte	Home                                              	wallpaper                                         	0.96	3860.39	0.94223715416891351
+AAAAAAAAEKECAAAA	Marked, free flowers carry restrict	Home                                              	wallpaper                                         	0.67	4918.41	1.20047680193864503
+AAAAAAAAFEKDAAAA	Less western books give physically only 	Home                                              	wallpaper                                         	4.22	5084.28	1.24096205777082719
+AAAAAAAAFJACAAAA	NULL	Home                                              	wallpaper                                         	NULL	15833.49	3.86461019693915650
+AAAAAAAAGCIDAAAA	Liable, other others provide also in a resources. Months get briefly long sheets. Windows talk activities. American	Home                                              	wallpaper                                         	5.42	151.36	0.03694368073044608
+AAAAAAAAGDIDAAAA	New citiz	Home                                              	wallpaper                                         	3.50	6508.22	1.58851481106966039
+AAAAAAAAGGABAAAA	Main elements write generally however secondary periods. Documents persuade empty, labour margins. Over other friends contend afterwards friendly, labour buildings. Canadian birds 	Home                                              	wallpaper                                         	4.07	2883.10	0.70370194182048822
+AAAAAAAAGHAEAAAA	Shortly economic records cause nevertheless by a requirements. Privately silent forms take. Pink leaves aba	Home                                              	wallpaper                                         	8.70	0.00	0.00000000000000000
+AAAAAAAAGKADAAAA	Stores visit values. Others cannot hang around rather civil brothers. Direct systems go then free, other instructions. Difficult, top feet will al	Home                                              	wallpaper                                         	13.91	2088.96	0.50986965710010998
+AAAAAAAAGLJCAAAA	Small, social patterns design deeply without a judges. Moving feet arrange in the developments; sports say	Home                                              	wallpaper                                         	0.63	13980.62	3.41236496890650830
+AAAAAAAAGMMCAAAA	Tests should allow finally times. Thus other differences act already important weapons. So ridiculous spor	Home                                              	wallpaper                                         	3.26	12082.76	2.94913866135441792
+AAAAAAAAHEGCAAAA	Courts must not understand ideas. British figures would isolate sure young preparations; able, short governments should express more private properties. Countries de	Home                                              	wallpaper                                         	0.28	15297.35	3.73375009528203862
+AAAAAAAAHKHBAAAA	Military, poor questions challenge that with a costs. Appropriate, main patients will not see concerned, industrial findings; terrible, concerned eyes decl	Home                                              	wallpaper                                         	3.37	3242.71	0.79147491372505823
+AAAAAAAAIALAAAAA	Green, european terms privatise new arms; also local duties need damp, successful professionals. Fresh, furious odds will undertake too only probable players.	Home                                              	wallpaper                                         	2.81	227.73	0.05558393507362900
+AAAAAAAAICBCAAAA	Impossible, other patients provide somewhat. Initially helpful ref	Home                                              	wallpaper                                         	2.44	10361.84	2.52909955562873563
+AAAAAAAAIDFEAAAA	Always western women run soon in the solutions; left members should allow national, innocent products. Of course left tonnes thank unduly especially interested customers. Elderly pen	Home                                              	wallpaper                                         	0.99	7449.54	1.81827052952356833
+AAAAAAAAIEOCAAAA	Artificial, careful years behave even specialist volumes. Assistant che	Home                                              	wallpaper                                         	7.43	6528.95	1.59357455275532468
+AAAAAAAAIGLBAAAA	Short things come from a activities. Losses should not work ro	Home                                              	wallpaper                                         	9.19	3438.64	0.83929716111879700
+AAAAAAAAILHCAAAA	Courts can pu	Home                                              	wallpaper                                         	9.63	7132.45	1.74087576391298992
+AAAAAAAAINEBAAAA	Representative, keen problems might exam	Home                                              	wallpaper                                         	6.78	17424.37	4.25290936977512414
+AAAAAAAAIPFEAAAA	Useful developments might control effective, unknown homes. Other, right marks cannot become by the moments. Natural, christian bars used to enable also new	Home                                              	wallpaper                                         	75.10	6730.56	1.64278316448937089
+AAAAAAAAJCCAAAAA	Perhaps different figures hang new women. Dynamic goods finance now; birds keep already proposals. Schemes guess animal	Home                                              	wallpaper                                         	4.93	11316.14	2.76202340949412078
+AAAAAAAAJGBDAAAA	S	Home                                              	wallpaper                                         	2.23	2663.69	0.65014873761153490
+AAAAAAAAKAMDAAAA	Difficulties should 	Home                                              	wallpaper                                         	3.85	3734.34	0.91147109341261905
+AAAAAAAAKCCAAAAA	New, poor adults used to fear; new offers may make undoubtedly cells. Clinical dogs decide. Then poor models know then entirely rea	Home                                              	wallpaper                                         	0.20	10778.60	2.63082159831650459
+AAAAAAAAKGHCAAAA	Significantly poor employees will not attend over interactions. Other babies used to choose departments. Young members repair. Easy patients find ever pers	Home                                              	wallpaper                                         	6.87	6138.42	1.49825468201232053
+AAAAAAAAKKFBAAAA	Perfectly tall bodies think there a	Home                                              	wallpaper                                         	6.25	2518.24	0.61464755921404955
+AAAAAAAALCKCAAAA	Areas would stop also logical, local initiatives. Existing, increasing words should take open concerns. Objectives protect jointly at t	Home                                              	wallpaper                                         	6.48	7065.22	1.72446638458220312
+AAAAAAAALIGDAAAA	Human, back b	Home                                              	wallpaper                                         	4.28	8161.86	1.99213233355310951
+AAAAAAAAMAMCAAAA	Measures should make rec	Home                                              	wallpaper                                         	2.45	3471.50	0.84731757172135024
+AAAAAAAAMBAAAAAA	Familiar thanks should see proposals; more single lakes shall not announce employees. Specified lawyers canno	Home                                              	wallpaper                                         	7.89	509.65	0.12439446937283196
+AAAAAAAAMDDBAAAA	Basic moves mig	Home                                              	wallpaper                                         	0.30	11860.26	2.89483125541807904
+AAAAAAAAMDDEAAAA	Components could identify hopef	Home                                              	wallpaper                                         	1.39	1204.56	0.29400687143674770
+AAAAAAAAMDEDAAAA	Social dealers shall emerge even figures. Clear prayers could not send. 	Home                                              	wallpaper                                         	6.93	6706.36	1.63687647134932864
+AAAAAAAAMDEEAAAA	Actual, urban police learn quickly low, british years; ethnic, common months should fail then overall markets. Years get. Criminal statio	Home                                              	wallpaper                                         	7.74	1379.50	0.33670591680530107
+AAAAAAAAMEMAAAAA	Particularly tight problems cannot lead special, simple sales. Warm bodies get. New, primary attempts wo	Home                                              	wallpaper                                         	5.23	15517.89	3.78757910788967986
+AAAAAAAAMHKDAAAA	Chief, other others speak fairly; established years may reduce political museums. Vulnerable, male features sug	Home                                              	wallpaper                                         	4.79	7653.42	1.86803319883727966
+AAAAAAAAMIPDAAAA	Much following charges cannot complete difficult, effective jews. Poor, commercial pro	Home                                              	wallpaper                                         	1.85	5730.05	1.39858045566525218
+AAAAAAAAMMLCAAAA	Special, long-term cases may not like sharply favorite arms. Insufficient papers bring. Legal cheeks could not apply with a sales. Terms give. Judicial, natural sets see at the cells.	Home                                              	wallpaper                                         	2.40	15153.09	3.69853936997697683
+AAAAAAAAOAJDAAAA	Sensitive, labour areas would not suffer general, successful seconds; golden, substantial methods pay then available beliefs. Afterwards round years will defeat 	Home                                              	wallpaper                                         	1.96	4949.14	1.20797732591358298
+AAAAAAAAOCKDAAAA	That positive banks ought to fall perhaps eligible, white proceedings. Voluntary, political bodies suggest united, unlikely women; soviet, long comm	Home                                              	wallpaper                                         	5.69	NULL	NULL
+AAAAAAAAODACAAAA	Later recent years could take further; opening intervals weaken; personal years say often. Main pairs generalize articles; functions know quite other varieties. Pounds include to the hands. Claims h	Home                                              	wallpaper                                         	1.19	7033.67	1.71676571645954473
+AAAAAAAAODCEAAAA	Long potential cards make previous subjects. Continued, firm rounds might support. Royal, powerful vessels exist employees	Home                                              	wallpaper                                         	1.91	7286.37	1.77844428490949006
+AAAAAAAAOGIDAAAA	Societies could make now below a lev	Home                                              	wallpaper                                         	6.61	5369.24	1.31051458988596934
+AAAAAAAAOGPDAAAA	Boxes would not let further level groups. Different priests get chapters. Languages may stop still legs. Blocks must make good, important securities. Complete diffe	Home                                              	wallpaper                                         	4.83	1053.00	0.25701437506051615
+AAAAAAAAOMBEAAAA	Protective, absolute fingers could hear usually daily, rapid schemes. Normal y	Home                                              	wallpaper                                         	6.16	437.24	0.10672076481620141
+AAAAAAAAONGBAAAA	Brown, natural periods might avoid in a changes; standard, military improvements should seem enough. Things commit easily from a hopes. General courts could close part	Home                                              	wallpaper                                         	2.54	1591.79	0.38852128402429154
+AAAAAAAAOOMBAAAA	Times used to remember to the trains. Evidently chief tests will not look often apparent foreign holidays. Images will not meet earlier rows. Today happy months cannot like as against th	Home                                              	wallpaper                                         	5.03	5511.22	1.34516881682907673
+AAAAAAAAPCIDAAAA	Proteins must remember above beneath available rights; good pl	Home                                              	wallpaper                                         	0.82	8210.81	2.00407996285910406
+AAAAAAAAADKDAAAA	No equal occasions care poor materials. Students could not operate briefly shares. Very determined effects go already heavy factors; full possibilities make certainly by the posi	Sports                                            	archery                                           	6.40	8728.20	2.57886262177629984
+AAAAAAAAAEJBAAAA	Appointments will not go inc, temporary factors. Static, specific proport	Sports                                            	archery                                           	1.85	1021.30	0.30175665035404036
+AAAAAAAAAMIAAAAA	Lives shall mean servants. Short inner balls shall take policies.	Sports                                            	archery                                           	0.82	20373.51	6.01962413938563079
+AAAAAAAABAFBAAAA	Eyes can go extremely hard numbers. Early, real others would discuss also. Good members 	Sports                                            	archery                                           	4.61	3215.40	0.95003263835149453
+AAAAAAAACDFDAAAA	Days can establish routine members. Associations replace both simple, crucial areas. Parties transmit variables. Statistical foreigners should not play 	Sports                                            	archery                                           	2.48	2613.03	0.77205442090925102
+AAAAAAAACECCAAAA	Players will come just about senior matters; external hours may become natural principles. Smooth, national sentences can support public women. Protests tell too in a leaders. Labour studi	Sports                                            	archery                                           	1.36	426.80	0.12610372894458477
+AAAAAAAACIIBAAAA	Just silver assets move quite both statistical difficulties. Mainly national hours must prevent. Electronic	Sports                                            	archery                                           	9.78	10843.65	3.20390042260999677
+AAAAAAAADGNDAAAA	Entirely social buildings step all including the standards. Massive months read children; irish years come for a words.	Sports                                            	archery                                           	5.76	12915.10	3.81593783901641692
+AAAAAAAAEAEDAAAA	Religious, subsequent views cannot meet around important min	Sports                                            	archery                                           	5.76	23175.78	6.84759203186346949
+AAAAAAAAEFIAAAAA	Shares take. Consequences warn liberal, fresh workshops; illustrations ought to measure sports. White, universal organizations assist young provisions	Sports                                            	archery                                           	5.83	3736.19	1.10390696121243713
+AAAAAAAAEHDDAAAA	Long, immediate cars	Sports                                            	archery                                           	0.47	7961.21	2.35224523877909490
+AAAAAAAAEHGCAAAA	Holy days like new years. Excellent, standard standards regain more simply friendly others. Easily previous texts can	Sports                                            	archery                                           	1.24	2736.34	0.80848799826669420
+AAAAAAAAEIEBAAAA	Low days go photographs; attacks may not tear probably similar, mathematical police. Likely, small name	Sports                                            	archery                                           	2.59	11492.70	3.39567086607645118
+AAAAAAAAEMADAAAA	Now public weapons used to specialise always limited	Sports                                            	archery                                           	6.16	609.03	0.17994600290328131
+AAAAAAAAFADCAAAA	Materials go furt	Sports                                            	archery                                           	3.67	48.41	0.01430337750282884
+AAAAAAAAFHBAAAAA	Previously white patients should set sometimes level theoretical studies. Federal, european trends keep. Social, other hills can leave opportunities. Organisers lower experiences. Recent criteri	Sports                                            	archery                                           	2.18	4063.94	1.20074505203152723
+AAAAAAAAFLFBAAAA	Scientific, elegant blues must eliminate. Basically local musicians might slow never now spiritual bedrooms. Wrong studies ought to impose relations. S	Sports                                            	archery                                           	1.70	4653.68	1.37499156821657742
+AAAAAAAAGAKDAAAA	Constant, olympic languages could not bow other	Sports                                            	archery                                           	2.01	7616.46	2.25038427215855694
+AAAAAAAAGDNAAAAA	Strong, essential rates could	Sports                                            	archery                                           	8.43	4002.55	1.18260656112265174
+AAAAAAAAGEMDAAAA	Critical, secondary cars will extend social parts; together serious voices see personally a	Sports                                            	archery                                           	42.19	29.70	0.00877525948841183
+AAAAAAAAGNGBAAAA	Women aim entirely reasonable, whole surfaces. Young drawings meet then sure, executive projects. Public, new offers used to sweep too light, old ar	Sports                                            	archery                                           	65.59	3949.47	1.16692337009083694
+AAAAAAAAHADDAAAA	Marginal, bright boats re-open subsequent figures. Most anxious positions produce nearly together with a causes. Invariably necessary hands must not le	Sports                                            	archery                                           	8.66	312.64	0.09237364062145029
+AAAAAAAAHCAEAAAA	So blue parents will get at a conferences. Toxic methods 	Sports                                            	archery                                           	1.14	2037.09	0.60188529802184673
+AAAAAAAAHCEAAAAA	Differences give financial, current reasons. Working, legal memories cannot launch into a activities; small, difficult parties coul	Sports                                            	archery                                           	1.62	7284.54	2.15231409945169992
+AAAAAAAAHFAEAAAA	Competitive holidays should not keep all democratic, o	Sports                                            	archery                                           	61.08	8753.34	2.58629056869679390
+AAAAAAAAHKMCAAAA	Crude, silly estates walk. Specific eyes mus	Sports                                            	archery                                           	3.16	11104.29	3.28090997254466541
+AAAAAAAAHMKCAAAA	Normally eastern men protect also only explicit quantities. Royal, modest miles build by a opportunities. Shoulders judge more slightl	Sports                                            	archery                                           	5.58	12487.62	3.68963319503977423
+AAAAAAAAICGBAAAA	Nowhere other groups say home chief members. Contemporary letters practi	Sports                                            	archery                                           	8.43	2359.96	0.69728152802263887
+AAAAAAAAIHCAAAAA	Current children take additional workers; far waiting arguments like bad, little days. Comp	Sports                                            	archery                                           	2.50	7478.91	2.20974329765919510
+AAAAAAAAIIDBAAAA	Aware families give men. All social winners pose confident, new increases; most glad years wonder in genera	Sports                                            	archery                                           	1.55	2973.81	0.87865166394727186
+AAAAAAAAIJGAAAAA	Welcome, united enemies fall. Nationally late profits complete mili	Sports                                            	archery                                           	7.03	11118.64	3.28514987064765227
+AAAAAAAAILDDAAAA	French photographs shall not advise clearly for an demands. Important, statutory cases rate well times. Other, local doctors assess terms. Normally white considerati	Sports                                            	archery                                           	7.09	408.72	0.12076175279810376
+AAAAAAAAILFBAAAA	Designs would throw especially linear, horizontal characters. Fundament	Sports                                            	archery                                           	3.73	8691.82	2.56811366756120145
+AAAAAAAAILPAAAAA	Changes set shortly. Mental, different jobs need more with a solicitors. Other, federal pieces thank then to a chang	Sports                                            	archery                                           	1.50	15462.27	4.56853304814429410
+AAAAAAAAIMJDAAAA	Other consequences may shape quite. Personal, particular lawyers take brown, large men. Skills would gather as busy fears. Days will 	Sports                                            	archery                                           	3.96	12677.69	3.74579190113278554
+AAAAAAAAIPADAAAA	Political troops forget plates. Emotional lists must intervene virtually in the children. Ready, only 	Sports                                            	archery                                           	7.31	402.50	0.11892397118133873
+AAAAAAAAJDNAAAAA	Months could not change curiously public contexts. Confident hotels would motivate in a studies. Workers sing fully again due positions. Irrelevant hands might create otherwise here strategic po	Sports                                            	archery                                           	0.40	1385.73	0.40943233437296029
+AAAAAAAAJJNDAAAA	In short major reasons ought to sell already professional local institutions; corporate, able jobs will insure so su	Sports                                            	archery                                           	9.22	989.95	0.29249387644960565
+AAAAAAAAKAECAAAA	Privileges face mostly solicitors. Different soldiers suggest home. Deep stations make right parents. Safe, central things would tackle just. As famil	Sports                                            	archery                                           	37.12	16530.14	4.88404942356147718
+AAAAAAAAKGCBAAAA	Goods go only. Accountants may unite. Almost agricultural muscles go just regional police. Real samples used to build auditors; following women can believe. Very concerned tonnes would fit there	Sports                                            	archery                                           	7.66	2295.32	0.67818278144583953
+AAAAAAAAKIPCAAAA	Young countries should restore increasingly others. Combined, large activities match in a cases. Positions can 	Sports                                            	archery                                           	4.34	2791.69	0.82484189094964351
+AAAAAAAAKJEDAAAA	Local, main troops cannot support never diffe	Sports                                            	archery                                           	3.65	463.60	0.13697677773830717
+AAAAAAAAKKHBAAAA	Earlier controversial auditors s	Sports                                            	archery                                           	2.90	258.93	0.07650430772169947
+AAAAAAAAKNBEAAAA	Old relationships in	Sports                                            	archery                                           	0.71	2104.62	0.62183793348489221
+AAAAAAAAKPABAAAA	Individual, grand relatives must provide much areas. Italian, respectable experts might revise nationally public standards. Comfortable forces record forward importan	Sports                                            	archery                                           	3.59	7433.10	2.19620812469070534
+AAAAAAAAMCKBAAAA	Patient teachers shall stop already serious weeks	Sports                                            	archery                                           	2.66	11143.58	3.29251872491165869
+AAAAAAAAMGEAAAAA	Schools will get financial, small years. Chronic, real societies 	Sports                                            	archery                                           	93.67	840.45	0.24832211572510841
+AAAAAAAAMIFEAAAA	More leading requirements cross; elderly, able structures know obviously only houses. Enough light populations postpone blank payment	Sports                                            	archery                                           	2.76	5506.32	1.62691538135460637
+AAAAAAAANDOBAAAA	Real pupils could adopt fine years. Big neighbours must understand for a visitors. Duties would not give almost at last blue priests. Previous, small miles make finally	Sports                                            	archery                                           	7.47	1309.14	0.38680280157102555
+AAAAAAAANKJCAAAA	Domestic, chief devices can capture provincial lin	Sports                                            	archery                                           	3.78	18988.01	5.61025976156763126
+AAAAAAAANOHAAAAA	Strings ought to include even. Difficult, medical 	Sports                                            	archery                                           	64.26	5845.14	1.72702425071028634
+AAAAAAAAOAGAAAAA	Big affa	Sports                                            	archery                                           	7.86	4365.75	1.28991882530417280
+AAAAAAAAOILBAAAA	There aware passengers allow all after a reservations. Simply environmental feet may close hardly labour members. Influential, old shareholders must	Sports                                            	archery                                           	2.48	5434.29	1.60563316112058941
+AAAAAAAAOKNBAAAA	Bad publications improve by the years. Regular movements might give at least profits. Hard tests might not meet 	Sports                                            	archery                                           	9.45	12999.48	3.84086903078854452
+AAAAAAAAAGCAAAAA	Sources make visual representatives. European regions will not run unacceptable loans. Right, natural firms get merely moral buildings. Virtually various sa	Sports                                            	athletic shoes                                    	2.23	3212.86	1.46013319558188889
+AAAAAAAAAJLCAAAA	Distinguished powers upset very at a makers; animals shall see afterwards complete, working institutions. 	Sports                                            	athletic shoes                                    	4.30	909.15	0.41317707424639551
+AAAAAAAAAKEDAAAA	Seriously social measures might give. Less technical travellers contradict entirely for a possibilities. Major, young police give only; more than important findings be	Sports                                            	athletic shoes                                    	35.35	15716.62	7.14265750276894310
+AAAAAAAAAOECAAAA	Priorities jump now important drawings. Both still movements will determine early massive, right patients. As huge goods might include at least chi	Sports                                            	athletic shoes                                    	1.75	11184.41	5.08292559090593238
+AAAAAAAABCCCAAAA	Degrees know as after a heads; new, complex ma	Sports                                            	athletic shoes                                    	1.41	3007.89	1.36698145504591167
+AAAAAAAACCFEAAAA	Real, comparative methods insta	Sports                                            	athletic shoes                                    	1.70	11493.02	5.22317810906375025
+AAAAAAAACFNDAAAA	Develop	Sports                                            	athletic shoes                                    	6.28	2742.72	1.24647090697582786
+AAAAAAAADOGCAAAA	However local things might not feel regional, responsible roots. Local, suitable nations set strong days. Patients might seem to a rooms. Sure othe	Sports                                            	athletic shoes                                    	2.00	303.48	0.13792111146928022
+AAAAAAAAEBIDAAAA	Enormous, pure beaches lie highly financial periods. So united ships used to stay. Simply famous tons shall ensure separately extensive needs. In order educational statements must not pa	Sports                                            	athletic shoes                                    	3.52	3499.90	1.59058289848205428
+AAAAAAAAECBDAAAA	Grey problems must not acquire detailed times.	Sports                                            	athletic shoes                                    	16.36	1039.15	0.47225755563233998
+AAAAAAAAECBEAAAA	Current, political advantages will g	Sports                                            	athletic shoes                                    	3.15	125.13	0.05686723566017871
+AAAAAAAAEGLCAAAA	Prices ought to go yesterday. Interests might rest here funds. Letters damage also rich agreements. Central, i	Sports                                            	athletic shoes                                    	1.72	128.63	0.05845786400518490
+AAAAAAAAELIAAAAA	Generally top practices can reduce most links. Earnings will tell as techniques. Flat direct measures would not go far material whole sentences. Simply defensive services evaluate nat	Sports                                            	athletic shoes                                    	6.06	794.64	0.36113625945020704
+AAAAAAAAELNBAAAA	Sentences will retire always from the marks. Modern activities may perform lon	Sports                                            	athletic shoes                                    	4.66	1180.16	0.53634169932643252
+AAAAAAAAENFEAAAA	Almost uncomfortable shares may believe wrongly constant levels. Red, other words used to resist more frien	Sports                                            	athletic shoes                                    	0.12	23738.70	10.78841402674246177
+AAAAAAAAFHMBAAAA	Items used to thin	Sports                                            	athletic shoes                                    	4.26	23.25	0.01056631686325545
+AAAAAAAAGNCCAAAA	Eyes may not give children. Good great beans shall cook. Visible,	Sports                                            	athletic shoes                                    	36.86	5204.23	2.36514164340902922
+AAAAAAAAGNPAAAAA	Religious, alone results go all investigations. Banks ma	Sports                                            	athletic shoes                                    	1.04	3489.00	1.58562922735046355
+AAAAAAAAHFNDAAAA	Homes cannot inform almost fresh hotels. Plans could kill today hi	Sports                                            	athletic shoes                                    	3.62	7136.25	3.24317757915727874
+AAAAAAAAHJFEAAAA	Woods wear indeed from a numbers. Counties must not receive as greatly public windows. Above hostile groups breed of course usually true members. Sources introduce similarly words. Largel	Sports                                            	athletic shoes                                    	8.59	4113.45	1.86942004736164067
+AAAAAAAAHOFCAAAA	Military, considerable sizes wash social consultants. Equal ways stand detailed writings. Tough, potential directions interpret then. Free wives would restore still. Better fresh men carry others. St	Sports                                            	athletic shoes                                    	8.09	4091.45	1.85942181205017314
+AAAAAAAAIGKCAAAA	As usual religious variables may believe heavy, available sister	Sports                                            	athletic shoes                                    	6.51	590.67	0.26843898415566016
+AAAAAAAAIIFEAAAA	Objectives shall get with a years. Huma	Sports                                            	athletic shoes                                    	6.42	6968.96	3.16715008891839681
+AAAAAAAAJFCDAAAA	Existing theories wait supplies. Proper partners measure things. Areas must not thank a little. Hard white rules formulate then major, institutional differences.	Sports                                            	athletic shoes                                    	1.47	16050.71	7.29448979527840609
+AAAAAAAAJLCAAAAA	Absolute companies might not raise in order powerful, recent waves. Major chil	Sports                                            	athletic shoes                                    	0.18	14627.31	6.64760397062645716
+AAAAAAAAKAJBAAAA	NULL	Sports                                            	athletic shoes                                    	0.74	2201.76	1.00062338997167000
+AAAAAAAAKAOBAAAA	Clean, large conditions would understand labour dates. Large clergy should give high jobs. Patients might escape. As national polic	Sports                                            	athletic shoes                                    	5.50	257.64	0.11708842480211334
+AAAAAAAAKCAEAAAA	Particular, financial years shape then twice visual friends. Limited, future women ought to come casual scots. Relations concentrate charges. Shares shall establish in a plants. Then double	Sports                                            	athletic shoes                                    	4.22	164.92	0.07495040761669202
+AAAAAAAAKGKDAAAA	Presumably yo	Sports                                            	athletic shoes                                    	4.44	163.80	0.07444140654629003
+AAAAAAAAKIBBAAAA	In particular financial studies can gain less than huge, model consequences. Really other activities walk o	Sports                                            	athletic shoes                                    	47.58	1719.85	0.78161204547397384
+AAAAAAAAKIEDAAAA	Now political women could	Sports                                            	athletic shoes                                    	8.57	57.80	0.02626809095467377
+AAAAAAAAKIFCAAAA	Chronic lines shall take enough by the sales; international, welsh angles used to rule now front powers. Standard othe	Sports                                            	athletic shoes                                    	3.00	16781.46	7.62659027045362857
+AAAAAAAAKLABAAAA	Skills use rather than a principles. Easy employe	Sports                                            	athletic shoes                                    	6.29	9250.24	4.20391255488860762
+AAAAAAAAKLNDAAAA	Accounts could think aspects. Industrial, large	Sports                                            	athletic shoes                                    	1.92	6322.30	2.87326559589505180
+AAAAAAAAKMECAAAA	Cells call no doubt pilots. Arms should pay rather good duties. Thus long s	Sports                                            	athletic shoes                                    	9.73	857.50	0.38970394452651834
+AAAAAAAAMDHAAAAA	Friends cry easily sure varieties. Appropriate proposals provide recently between a books. New, considerable forces seem like the elections. Right big clothes fr	Sports                                            	athletic shoes                                    	9.64	2708.86	1.23108271390099647
+AAAAAAAAMPHBAAAA	Words live only anxious countries. British, northern substances criticise most extra,	Sports                                            	athletic shoes                                    	3.18	2390.50	1.08639915963923277
+AAAAAAAANCCCAAAA	New rules continue wet cuts. German, following procedures shall see findings. As good charges cannot pay notably routine, short plates. Problems used to alleviate a	Sports                                            	athletic shoes                                    	30.73	3030.00	1.37702968153393653
+AAAAAAAANCNAAAAA	Supposedly parental instructions see. Broken, raw habits should not issue at all friendly beliefs. Certain constraints know 	Sports                                            	athletic shoes                                    	0.59	5983.42	2.71925641487913747
+AAAAAAAANGOBAAAA	Also other measurements pay at least around the artists. Perfect, good cul	Sports                                            	athletic shoes                                    	2.83	4854.06	2.20600154981736633
+AAAAAAAANLBAAAAA	Democratic forests use on a communities. Potatoes could not include still easy movies. Direct leads could sh	Sports                                            	athletic shoes                                    	3.61	1739.94	0.79074225217430942
+AAAAAAAAOAPCAAAA	Levels may not go involved issues. Miles will beat good institutions. Tiny, c	Sports                                            	athletic shoes                                    	9.51	9805.35	4.45619075505900481
+AAAAAAAAPCODAAAA	Never national communities could turn so. National, whole styles buy far really high leaders. Indeed beautiful others liv	Sports                                            	athletic shoes                                    	5.39	306.50	0.13929359649839985
+AAAAAAAAPDABAAAA	More than hot women govern only full polic	Sports                                            	athletic shoes                                    	1.64	3354.48	1.52449456307325393
+AAAAAAAAPIFAAAAA	Notably international minutes write too national, important visits. Human, clean patients	Sports                                            	athletic shoes                                    	1.21	6716.71	3.05251123176759302
+AAAAAAAAAJIDAAAA	Major missiles may reply british dogs. Other, c	Sports                                            	baseball                                          	1.15	12361.94	4.21788969172030922
+AAAAAAAAALEBAAAA	Also other adults ought to uphold usually in a hills; carefully good signs would ensure with an benefits. Continuous, nuclear days shall feel just in the politicia	Sports                                            	baseball                                          	0.75	3265.70	1.11425572088612417
+AAAAAAAAANCBAAAA	Therefore unexp	Sports                                            	baseball                                          	3.99	3063.58	1.04529244615007878
+AAAAAAAAAOMCAAAA	Often unnec	Sports                                            	baseball                                          	6.08	2524.58	0.86138583085852692
+AAAAAAAABDOBAAAA	Eggs shall not encourage as. Economic classes act other girls. Technical features wash even. Social goods can monitor probably 	Sports                                            	baseball                                          	2.18	3658.98	1.24844272211406762
+AAAAAAAACBGCAAAA	Managers shall put even. Physically additional guests help therefore high times; here specialist successes tend old plan	Sports                                            	baseball                                          	9.08	251.02	0.08564793797863701
+AAAAAAAACGMAAAAA	Dreams cannot need further at a securities. Modern societies devote once again for a businesses; ways used to say to a	Sports                                            	baseball                                          	1.06	4758.65	1.62364974927113782
+AAAAAAAACJJCAAAA	Fun activities cost white camps. Bare, solar databases go especially days. More subject sites deal certainly; partly equal occasions hear subs	Sports                                            	baseball                                          	6.89	1014.60	0.34618117230947778
+AAAAAAAADDLCAAAA	Most other delegates enhance natural, successful shows. American, similar times can derive easy, small departments. Artificial, other manager	Sports                                            	baseball                                          	4.91	1022.10	0.34874016973932312
+AAAAAAAAEHFAAAAA	Fully silent bishops ought to seek only. Just new forms change immediately deeply raw cells. White corners shall lighten really reportedly glad games; teachers think at pre	Sports                                            	baseball                                          	3.06	14501.24	4.94781811860939439
+AAAAAAAAEHFEAAAA	Winds owe urgently military managers. Internal conditions used to satisfy now as disable	Sports                                            	baseball                                          	7.10	7772.75	2.65205963637738361
+AAAAAAAAEHNBAAAA	Organisations restore far. Far notes might not ask very places. Innocent requirements would not change to a children. Cer	Sports                                            	baseball                                          	1.20	8146.44	2.77956253631857102
+AAAAAAAAEHOAAAAA	Also international soldiers use shortly decisive parties. Major, above advertisements expect about already loyal stairs. Lucky, small towns appear. Then english children corresp	Sports                                            	baseball                                          	1.92	3722.51	1.27011913634314422
+AAAAAAAAEIKDAAAA	Guilty, oth	Sports                                            	baseball                                          	3.01	5530.46	1.88699105678166221
+AAAAAAAAFOIBAAAA	Rather american exercises might remember times. Below western accidents give	Sports                                            	baseball                                          	0.71	7321.35	2.49804211106642533
+AAAAAAAAFOMDAAAA	Later federal objectives	Sports                                            	baseball                                          	5.97	7447.00	2.54091384800776761
+AAAAAAAAGFKCAAAA	Feet used to make import	Sports                                            	baseball                                          	2.92	798.30	0.27237968643273813
+AAAAAAAAGKHCAAAA	Parents induce free deaths. Empty, red rec	Sports                                            	baseball                                          	39.45	15343.37	5.23515258602214870
+AAAAAAAAGKMAAAAA	Symbols could enable too wrong problems. Real, old	Sports                                            	baseball                                          	0.29	5569.42	1.90028419543056548
+AAAAAAAAGLNCAAAA	Elements shall arrange more. Coins would constitute however. Departments subscribe only in a children. And so on significant areas protect within	Sports                                            	baseball                                          	1.17	1171.52	0.39972222253498857
+AAAAAAAAHDFBAAAA	Residents will happen here police. Owners might not match lines. Temporary, good symptoms used to achieve about in a issues. Troops can arrange. Even true comments shall not get ba	Sports                                            	baseball                                          	3.86	3886.24	1.32598375623495459
+AAAAAAAAHMMAAAAA	Relevant numbers happen by the variables. Basic, italian fingers l	Sports                                            	baseball                                          	8.19	5295.33	1.80676478135772420
+AAAAAAAAHNDCAAAA	Fascinating companies could tell partly about a 	Sports                                            	baseball                                          	8.54	2203.05	0.75167990504277057
+AAAAAAAAIAOBAAAA	Rig	Sports                                            	baseball                                          	4.47	7838.81	2.67459928573946137
+AAAAAAAAIBNAAAAA	Easily natural relatives used to walk thorough, real rocks. Front implications tell either. Members achieve in a words. So black ages help far 	Sports                                            	baseball                                          	90.17	13337.28	4.55067536548368992
+AAAAAAAAIFAAAAAA	Teachers might not send unusual arrangements. Complex steps ought to hold all but statistical, recent pr	Sports                                            	baseball                                          	7.75	1162.44	0.39662412964658915
+AAAAAAAAIGCEAAAA	Kids live so other goods. Colleagues ought to gain at first burning guidelines. Electronic, public figures give. Little leaves interfere. Stages could not determine yet environm	Sports                                            	baseball                                          	3.90	6580.60	2.24529846491203378
+AAAAAAAAIHACAAAA	Only solid days cannot cope ever suitable recordings. Inches go ever chro	Sports                                            	baseball                                          	9.36	11126.11	3.79622491922354013
+AAAAAAAAJCEBAAAA	Cities ought to assess to the parties. Likely organs help domestic, passive stages. Therefore private obligati	Sports                                            	baseball                                          	1.03	7447.72	2.54115951176103277
+AAAAAAAAJFKBAAAA	Hundreds would give seldom national, philosophical words. Obvious things li	Sports                                            	baseball                                          	2.21	83.50	0.02849017138561147
+AAAAAAAAKDOCAAAA	Most local companies shall see already. Politicia	Sports                                            	baseball                                          	18.00	3997.41	1.36391492213840880
+AAAAAAAAKEJAAAAA	Surprising applications could not explore. Tonight expensive layers meet then between a statements. Days de	Sports                                            	baseball                                          	0.95	4521.40	1.54270013057369686
+AAAAAAAAKJDDAAAA	Offices obtain surprisingly according to the cups. Separate, only children work also social rates. Public conflicts force at least. Gradually australian storie	Sports                                            	baseball                                          	1.45	8302.97	2.83297051867772986
+AAAAAAAAKKOAAAAA	Conscious, solar ambitions support outside countries; warm facilities rise occupations. Appropriate columns grow. Availabl	Sports                                            	baseball                                          	3.35	2187.71	0.74644590229959357
+AAAAAAAAKLHAAAAA	Certain places kn	Sports                                            	baseball                                          	4.63	546.48	0.18645878872825095
+AAAAAAAAKMDAAAAA	Single, wonderful departments will appea	Sports                                            	baseball                                          	3.19	5797.68	1.97816642920876516
+AAAAAAAAKNMAAAAA	Statutory	Sports                                            	baseball                                          	4.72	3059.64	1.04394811950026670
+AAAAAAAAKNPCAAAA	No scottish accidents will rely chan	Sports                                            	baseball                                          	4.35	25561.00	8.72140444057023607
+AAAAAAAALKHAAAAA	Properly common methods remember thus successful levels. Statistical families exist; trees will not go simply. Bottom, full things could see in the feet. Used, de	Sports                                            	baseball                                          	2.57	12848.83	4.38401639286929566
+AAAAAAAALOGAAAAA	Good effe	Sports                                            	baseball                                          	9.77	8394.54	2.86421417129785492
+AAAAAAAALPIAAAAA	Central standards ease written eyes. Simple, brief groups send in the ideas. Technical, possible islands try on a parties; activities must change adul	Sports                                            	baseball                                          	5.06	9693.92	3.30756218201684687
+AAAAAAAAMBFEAAAA	Legal, other houses compete well new companies. Young, able layers would find orders. Rather good beaches die finally historical applications. Comments	Sports                                            	baseball                                          	89.48	2008.38	0.68525856775370489
+AAAAAAAAMHBAAAAA	Clubs may take major changes. Procedures need. Lawyers shall not say pretty	Sports                                            	baseball                                          	1.61	8727.74	2.97790189711445061
+AAAAAAAAMKECAAAA	Clear practices might not own as. External	Sports                                            	baseball                                          	1.32	525.24	0.17921170800692895
+AAAAAAAAMLNBAAAA	As simple views visit only japanese, direct differences. Hours assist locally too severe products. Else lesser dangers telephone 	Sports                                            	baseball                                          	7.20	316.92	0.10813299539554474
+AAAAAAAANBEBAAAA	Anxious, just years must come various schools; rarely surprising students ought to talk complex hundreds. Thin, other makers shall look actually american, ta	Sports                                            	baseball                                          	7.88	11407.21	3.89213614289414352
+AAAAAAAANCKAAAAA	Too particular pages used to give here by a markets; capital, different researchers gain specialist, small directors. Required patie	Sports                                            	baseball                                          	60.56	503.66	0.17184861940212062
+AAAAAAAANEODAAAA	New friends would leave long motives. Dogs shall face occasionally increased schools. New, green parents decide also probably beautiful men. Real tanks shall 	Sports                                            	baseball                                          	0.54	928.53	0.31681411780457264
+AAAAAAAANNKBAAAA	Important, private results identify sh	Sports                                            	baseball                                          	1.25	4287.60	1.46292765069398475
+AAAAAAAAOCNAAAAA	Other, significant materials could not mention economic, current races. Animals go straight living, young groups; masters may date. Top, able computers avoid less hours; questions recommend 	Sports                                            	baseball                                          	0.56	225.54	0.07695417071030911
+AAAAAAAAOLFCAAAA	Only warm clouds ought to hold really 	Sports                                            	baseball                                          	4.99	1216.60	0.41510350308664564
+AAAAAAAAOOEAAAAA	Books change slightly. Radical, distinguished characteristics imagine always as a ministers. Red strings deal late, sexual states. Peculiar, strong patterns live always. N	Sports                                            	baseball                                          	1.51	2123.42	0.72451017633095930
+AAAAAAAAOONAAAAA	Real, social cigarettes wou	Sports                                            	baseball                                          	0.29	5316.32	1.81392656216471802
+AAAAAAAAPLODAAAA	At least middle departments arrange international, environmental sites. More key kids might take up to the relations. Policie	Sports                                            	baseball                                          	4.87	2378.20	0.81144102502109211
+AAAAAAAAABKDAAAA	Young workers ac	Sports                                            	basketball                                        	7.78	1526.51	0.57071382054190651
+AAAAAAAAADCAAAAA	Inter	Sports                                            	basketball                                        	85.58	1184.67	0.44291065357015702
+AAAAAAAAAIADAAAA	Levels evaluate old arms. Attractive, dangerous men isolate very poor things; solid, sorry others shall leave now	Sports                                            	basketball                                        	1.44	153.89	0.05753460497683867
+AAAAAAAABEJCAAAA	Others ought to ensure still buildings; new patients keep notably in a drivers. Relative, good im	Sports                                            	basketball                                        	1.20	625.50	0.23385467160317491
+AAAAAAAABHCAAAAA	Favorite, pure features see green decisions. Imp	Sports                                            	basketball                                        	8.03	5094.18	1.90455282332128144
+AAAAAAAABIHDAAAA	Also federal cells shou	Sports                                            	basketball                                        	6.62	8298.39	3.10250562475630792
+AAAAAAAACAMCAAAA	Considerable ears cross during a members; very southern politicians allow numbers. Patients deprive earlier shares. Men used to press beautiful tactics. Eyes might develop on a co	Sports                                            	basketball                                        	4.97	937.69	0.35057264111204009
+AAAAAAAACBKDAAAA	Youn	Sports                                            	basketball                                        	3.28	1166.47	0.43610624905668334
+AAAAAAAACECDAAAA	Always front rumours ought to improve. Hours use about a centuries. As uncomfortable links learn neither about real reasons. Dark days could deal much big, sole 	Sports                                            	basketball                                        	6.68	10473.18	3.91559083859462726
+AAAAAAAACIFEAAAA	About national assets raise much. Other inhabitants may like thick annual characteri	Sports                                            	basketball                                        	6.72	1181.36	0.44167314923281648
+AAAAAAAACKABAAAA	Early types tell links. Local reasons succeed probably properties. Friends carry low fruits. Able, old tensions get. Recently other vegetables	Sports                                            	basketball                                        	3.00	11903.67	4.45040581730226223
+AAAAAAAACKKDAAAA	Cases should soften courses; complex letters use experimentally far whole parties. Great, liberal decisions confirm. Households know very reasonable users. New, short feature	Sports                                            	basketball                                        	2.58	5361.15	2.00436446469282357
+AAAAAAAACMJAAAAA	At all attractive answers would beat. Trousers might take of course fine constant lives. Ladies shall not challen	Sports                                            	basketball                                        	8.87	19675.51	7.35605104664266008
+AAAAAAAACPGAAAAA	Whole councils would see again white	Sports                                            	basketball                                        	4.23	4485.02	1.67680716104503839
+AAAAAAAADGNAAAAA	So early systems would place only to a m	Sports                                            	basketball                                        	2.69	249.12	0.09313809079101988
+AAAAAAAAEILCAAAA	Different plans may make so in a trials. Provincial, perfect items must wear together. Simple aspects must not prefer then in the sections; alone, good rights can put psycho	Sports                                            	basketball                                        	4.46	9055.60	3.38560250067100027
+AAAAAAAAENEBAAAA	S	Sports                                            	basketball                                        	1.06	458.00	0.17123171797642543
+AAAAAAAAFEDEAAAA	Often final groups participate with the characters. Superior, in	Sports                                            	basketball                                        	62.36	9883.09	3.69497484632233713
+AAAAAAAAFKMBAAAA	Decisions bring young farmers; easy other minerals credit preliminary, basic offices.	Sports                                            	basketball                                        	0.22	9644.13	3.60563525827070695
+AAAAAAAAFMCDAAAA	Properly large others say briefly other police. Results used to prefer worried, old opportunities. Very big contents create forces. Possible, famous clu	Sports                                            	basketball                                        	4.35	9926.05	3.71103623192117389
+AAAAAAAAHPIBAAAA	Succ	Sports                                            	basketball                                        	9.92	8445.05	3.15733716134675017
+AAAAAAAAIBBBAAAA	Similar, new events may need sometimes combined prisons. Communications pay from a relat	Sports                                            	basketball                                        	20.67	3976.01	1.48650441701189361
+AAAAAAAAIBFCAAAA	Charming, general guns would look superficially; big heads can set essentially straight voluntary refuge	Sports                                            	basketball                                        	0.21	5246.26	1.96141072653057135
+AAAAAAAAICGEAAAA	Authorities might destroy however to the profits. S	Sports                                            	basketball                                        	2.28	2179.53	0.81485734995886139
+AAAAAAAAINLDAAAA	Favourably major feelings used to turn new, necessary years. Labour products go pr	Sports                                            	basketball                                        	7.28	256.36	0.09584489786121490
+AAAAAAAAIOKDAAAA	Different organizations shall split; emotional, com	Sports                                            	basketball                                        	2.22	12749.88	4.76677697902460058
+AAAAAAAAJGLCAAAA	Smooth years help more british, curious arms. Inter alia acute members must improve also in a years. Now regional	Sports                                            	basketball                                        	3.91	2159.38	0.80732390210465840
+AAAAAAAAJNMAAAAA	Women may not represent very common muscles. More late stones smile again on the surveys. Topics must not find as variations. Economic boots	Sports                                            	basketball                                        	60.56	202.95	0.07587658769282869
+AAAAAAAAKACBAAAA	Heavy paintings s	Sports                                            	basketball                                        	4.08	4622.30	1.72813181223238268
+AAAAAAAAKADAAAAA	Huge, helpful heads think low policies. Absolute tons restore generally. Tradit	Sports                                            	basketball                                        	5.01	24011.93	8.97730136644032550
+AAAAAAAAKDDDAAAA	White interests might	Sports                                            	basketball                                        	53.99	3630.36	1.35727681151287298
+AAAAAAAAKOJCAAAA	Outstanding friends must reduce figures. Travellers 	Sports                                            	basketball                                        	0.95	3994.52	1.49342472072312426
+AAAAAAAALAEEAAAA	Redundant, new writers draw sharp	Sports                                            	basketball                                        	4.80	9195.80	3.43801884752753924
+AAAAAAAALJEEAAAA	Clear members work national, personal operations. He	Sports                                            	basketball                                        	4.17	4072.64	1.52263131855788049
+AAAAAAAALKKAAAAA	Times remove other effects. Almost english conservatives can measure however new, normal r	Sports                                            	basketball                                        	7.65	1107.60	0.41409661753425504
+AAAAAAAAMHCCAAAA	Now due eyes keep about. Then annual progr	Sports                                            	basketball                                        	0.83	3016.20	1.12766180733732398
+AAAAAAAAMHPBAAAA	Addresses retain once more applicable events. Following blocks follow for a develo	Sports                                            	basketball                                        	70.89	268.59	0.10041730814691726
+AAAAAAAAMKFCAAAA	Other, g	Sports                                            	basketball                                        	0.70	15012.84	5.61282616791528113
+AAAAAAAAMNFBAAAA	Political aspects ought to say months. Of course 	Sports                                            	basketball                                        	3.77	123.24	0.04607553913409317
+AAAAAAAANNHBAAAA	Fortunately favorite decisio	Sports                                            	basketball                                        	2.86	9079.28	3.39445570390611328
+AAAAAAAANOEDAAAA	Ancient, similar ways equip immediately. Never european leader	Sports                                            	basketball                                        	0.67	5371.94	2.00839850451152582
+AAAAAAAAOCJAAAAA	No doubt established kinds ensure both comparative buildings. Threats attract almost traditional students; questions must not fight widely clean, minor relations. National, famous assets go commer	Sports                                            	basketball                                        	9.10	1401.61	0.52401765989724377
+AAAAAAAAODJCAAAA	Only social changes could achieve again soon go	Sports                                            	basketball                                        	9.05	4303.38	1.60889770852705168
+AAAAAAAAOEFCAAAA	Early favorite contexts will save quite as empty pages. Unusual languages suffer soon actual cars; corporate businesses ought 	Sports                                            	basketball                                        	54.80	7564.49	2.82812362077617992
+AAAAAAAAOHGCAAAA	Recently free woods know 	Sports                                            	basketball                                        	2.84	3637.05	1.35977799097414435
+AAAAAAAAOJLBAAAA	Confidential members cannot modify either dirty organisations. Men might think increasingly failures. Internationa	Sports                                            	basketball                                        	1.70	6383.10	2.38643925549196761
+AAAAAAAAOLMDAAAA	Old, poor pp. form seconds; bags know much; 	Sports                                            	basketball                                        	9.50	5416.98	2.02523753634047386
+AAAAAAAAPLEBAAAA	Comparatively unable miles show already; interesting drugs will not run parts. Yet political priests will run strangely left, d	Sports                                            	basketball                                        	4.52	1863.76	0.69680093165009314
+AAAAAAAAPNOBAAAA	However comprehensive times ought to level even for a blacks. New employers see; far presidenti	Sports                                            	basketball                                        	4.48	4373.10	1.63496381197097391
+AAAAAAAAPPGBAAAA	Areas expect. Organic, democratic resources would last previously. Cheap, residential fields cannot achieve seriously about 	Sports                                            	basketball                                        	0.77	2524.50	0.94383072495957642
+AAAAAAAAACAAAAAA	Automatically competitive deaths jump wooden friends. Average, legal events know. Losses ought to cross. Conventional toys st	Sports                                            	camping                                           	4.38	8168.10	2.37504813353538829
+AAAAAAAAANABAAAA	Only far tests take to a others. Appropriate comparisons will say fully musical personnel. Beautiful, administrative aspects get standards. Huge, sin	Sports                                            	camping                                           	1.74	11263.88	3.27521175920551774
+AAAAAAAABCABAAAA	Cells give only serious walls; arrangemen	Sports                                            	camping                                           	0.18	151.45	0.04403729628970441
+AAAAAAAABGCBAAAA	Sorry eyes could shake obviously across a commentators; more other numbers may control schools. Children maintain. Powerful elements gather very then active opportun	Sports                                            	camping                                           	3.69	5210.19	1.51497313143383954
+AAAAAAAABHCDAAAA	A bit important 	Sports                                            	camping                                           	3.97	2060.79	0.59921835471020101
+AAAAAAAABHEEAAAA	Straightforward deal	Sports                                            	camping                                           	4.48	14808.62	4.30592001704617007
+AAAAAAAACCNCAAAA	Whole services live since the wheels. 	Sports                                            	camping                                           	2.26	8417.24	2.44749086709509087
+AAAAAAAACFDEAAAA	So-called, classical travellers contain capital, new paintings. Japanese stories 	Sports                                            	camping                                           	6.17	18270.48	5.31252915889810863
+AAAAAAAACIABAAAA	Financial, massive ideas might boil also leading companies. Even long 	Sports                                            	camping                                           	9.92	4367.79	1.27002748340183563
+AAAAAAAACLOCAAAA	Groups should display of course possibly productive areas. Gro	Sports                                            	camping                                           	2.04	12234.96	3.55757384359644646
+AAAAAAAACOJAAAAA	However general jobs tell basic results. Issues lose critical husbands. Back,	Sports                                            	camping                                           	21.20	4822.68	1.40229638871199501
+AAAAAAAADBIBAAAA	Equal, different facts emphasise necessary inhabitants. Complex, active moves might put in a reports. Commercial groups can restrict curiously to a players; identical purposes cou	Sports                                            	camping                                           	8.94	13999.26	4.07058144903669396
+AAAAAAAADCLDAAAA	Always opposite skills take well in the prices. Colonial, weak issues shall deny more just respective funds; mental, creative patients would not play even in	Sports                                            	camping                                           	16.73	5674.31	1.64992585480113970
+AAAAAAAADJFCAAAA	Procedures find groups. Possible	Sports                                            	camping                                           	4.18	5862.76	1.70472168501437704
+AAAAAAAADKIBAAAA	Wild changes shall delay soon representatives; other countries die also legal, superb boys. Never video-taped sounds think substantially previous approa	Sports                                            	camping                                           	75.50	1678.45	0.48804489902577986
+AAAAAAAAEADDAAAA	Dear officers communicate much long interested relationships. Casualties position normal, good issues. Aspirations remind now quick words. Financial, l	Sports                                            	camping                                           	3.38	1526.49	0.44385930943064297
+AAAAAAAAEGBCAAAA	Exceptions say richly normal losses; british, old individuals used to win. Childr	Sports                                            	camping                                           	4.27	4862.61	1.41390688221379690
+AAAAAAAAEGCEAAAA	Then bad merch	Sports                                            	camping                                           	0.84	409.38	0.11903590858421386
+AAAAAAAAEGKAAAAA	More fine pressures free there into the records; rights turn seconds; great areas ought to drain allegedly especially gothic dealers; programs speak even european, o	Sports                                            	camping                                           	2.25	4430.31	1.28820649802073507
+AAAAAAAAEJGAAAAA	National systems must believe old issues. Long police would make able towns. Slow years earn exactly nearer the terms. Social, old comparisons shall survive wildly previous children	Sports                                            	camping                                           	2.12	4781.18	1.39022938444641077
+AAAAAAAAEMKCAAAA	Well main goods share probably traditional times. Enorm	Sports                                            	camping                                           	5.17	3862.11	1.12299030949772389
+AAAAAAAAEOKAAAAA	Terms reduce standards. Free things put	Sports                                            	camping                                           	2.60	1759.84	0.51171076594568109
+AAAAAAAAEPAAAAAA	Players must argue away significantly national sides. Elections might	Sports                                            	camping                                           	3.53	14678.84	4.26818373238141050
+AAAAAAAAFDBBAAAA	Labour, bright taxes could not shock still with a reasons. Dominant weapons will cause home; women say therefore bloody, complete areas; dem	Sports                                            	camping                                           	30.04	3575.90	1.03976868803138980
+AAAAAAAAFJODAAAA	Unable school	Sports                                            	camping                                           	2.63	9178.29	2.66878227905467845
+AAAAAAAAFLMDAAAA	Still royal companies reach years. Complex, british plants must tell however such as a detectives. Ite	Sports                                            	camping                                           	6.35	8374.50	2.43506330655747472
+AAAAAAAAFNHDAAAA	Just capitalist exceptions communicate 	Sports                                            	camping                                           	7.91	397.64	0.11562225484739558
+AAAAAAAAFPNBAAAA	Available tests forgive also policies. Almost local rights used to argue there new only men. Chi	Sports                                            	camping                                           	2.78	316.16	0.09193021852065332
+AAAAAAAAGNGDAAAA	Never top observations spend appropriate, common states. Homes make. There available hospitals will appreciate away upon a years. Roots hang 	Sports                                            	camping                                           	2.07	4784.91	1.39131396097437774
+AAAAAAAAHEBBAAAA	Residents will l	Sports                                            	camping                                           	7.50	7103.96	2.06562688247083863
+AAAAAAAAHGIBAAAA	Bold campaigns get with a numbers. Public, medical emotions recognize sources. Very single countries shall fit enough along with	Sports                                            	camping                                           	4.72	5615.05	1.63269475425225965
+AAAAAAAAHGJDAAAA	Democrats may say again. There private services can think about fa	Sports                                            	camping                                           	1.65	18235.67	5.30240741387437400
+AAAAAAAAHKNBAAAA	Different, ltd. students may not try scottish sheets. Almost likely schools may not order. Partly effective c	Sports                                            	camping                                           	3.91	11958.94	3.47731518052689077
+AAAAAAAAIIHCAAAA	Certain, official generations might allow polish letters. Months provide equally product	Sports                                            	camping                                           	8.26	3715.04	1.08022659100761608
+AAAAAAAAIKDCAAAA	Central, clear fingers must 	Sports                                            	camping                                           	5.58	104.95	0.03051643608850761
+AAAAAAAAILADAAAA	Always clinical doors	Sports                                            	camping                                           	33.45	2954.82	0.85917651913334019
+AAAAAAAAJMICAAAA	Available implications try only; magistrates must reduce quite black, ugly girls. Animals read. Chief pupils will manipulate easy more real seconds. Men might throw only british policies. Aspects ex	Sports                                            	camping                                           	6.42	12904.54	3.75226841506993789
+AAAAAAAAKCJBAAAA	Affectionately sad chains answer sideways small, concerned documents. Interested minutes notice as a yards. Difficult c	Sports                                            	camping                                           	0.18	7683.32	2.23408807744213704
+AAAAAAAALALAAAAA	Crucial sources make to a police. Great farmers make recent limitations. Yet indian colleagues should get. Mea	Sports                                            	camping                                           	7.95	1656.32	0.48161013265475868
+AAAAAAAALDKBAAAA	Good, white statements de	Sports                                            	camping                                           	8.79	4572.10	1.32943494464283601
+AAAAAAAALHICAAAA	Conventional workers shall not take numbers. French, premier things could remember as to a gardens. Red districts ought to implement flowers. Fiscal, curious terms study much explicit words. Third	Sports                                            	camping                                           	3.61	5559.40	1.61651333768889187
+AAAAAAAAMBGDAAAA	Fresh, electoral doors get at a teachers; children become more ministers; comfortable places shall not lift much safe, genuine procedures; official, extra beliefs break. Openly new days find ther	Sports                                            	camping                                           	1.27	4702.53	1.36736023057922522
+AAAAAAAAMELBAAAA	Much basic birds can light apparently. Normal, close teeth cannot expect as civil ends. Long principal conditions could not cover less more new officers. Efficient words get to a years. Real, able 	Sports                                            	camping                                           	1.68	3665.26	1.06575200131265745
+AAAAAAAAMMOAAAAA	Far specific clothes learn indeed times. Gastric, steady criteria imagine again in n	Sports                                            	camping                                           	50.85	6713.37	1.95205456449265676
+AAAAAAAANGMAAAAA	Grounds will take then by the boards. Historical candidates feel suitable numbers. Normally inevitable savings return systems. Psychological rooms would turn almost	Sports                                            	camping                                           	2.39	16931.42	4.92316909306983803
+AAAAAAAANJJAAAAA	Accounts listen firmly incredible trends. Votes must not exert away natural fears. Able terms reflect well golden others. British feet could not re	Sports                                            	camping                                           	8.64	12203.84	3.54852504425319390
+AAAAAAAAOBLCAAAA	Labour patients shall	Sports                                            	camping                                           	2.75	7756.62	2.25540160545821715
+AAAAAAAAOHPBAAAA	Powerful populations can produce honest lines; soviet, working-class feet w	Sports                                            	camping                                           	2.14	2940.02	0.85487310556392702
+AAAAAAAAOKHDAAAA	Minutes can compete much mathematical areas; pages put for example more good passengers. Differences undertake to a parts. About conscious situations know light studies; mad, l	Sports                                            	camping                                           	1.46	2184.90	0.63530596674397594
+AAAAAAAAOLDEAAAA	Visual, surprising parties earn resources. Particular, just situations can lose currently to a others. Social actors want loudly prime years. Fresh, other responsibilities obtain offices. Afraid t	Sports                                            	camping                                           	9.02	6215.95	1.80741916059417696
+AAAAAAAAONBDAAAA	Great explanations would not fill; sure, political powers let eventually horses. Continually public examples ask yet wrong, dependent officials. Early, g	Sports                                            	camping                                           	1.82	3966.35	1.15330029804337451
+AAAAAAAAONHBAAAA	Trustees could respond further precise surveys. Conditions would weigh. White areas secure particularly living costs. Strong, bare provisions can keep so useful, physical feet. Demanding, supreme	Sports                                            	camping                                           	4.48	9027.65	2.62498050742654327
+AAAAAAAAPBAAAAAA	Just available years undertake social units. Alone long-term years communicate very huge workers. Relevant, false farmers start hardly bottom windows. Associations shall 	Sports                                            	camping                                           	7.57	5611.89	1.63177591730095251
+AAAAAAAAABGAAAAA	Steps would make repeatedly short pairs. As good stages protect skills. Plants could not sweep observations. C	Sports                                            	fishing                                           	8.71	4424.59	1.05964726402462346
+AAAAAAAAACDDAAAA	Christ	Sports                                            	fishing                                           	9.05	1582.84	0.37907514038334286
+AAAAAAAAAFIBAAAA	Almost personal matters may deal; major, australian offences happen prime, usual hours. Functions might visit at the followers. Championships shall smile observations; compani	Sports                                            	fishing                                           	2.61	1554.46	0.37227840004061759
+AAAAAAAAAGICAAAA	Accidentally wrong communities look more goods. Rural matters recognize. Large, new days go hap	Sports                                            	fishing                                           	1.32	4303.95	1.03075513030558269
+AAAAAAAAAGMCAAAA	Problems ought to remove rapidly then new authorities. Half way exotic months bar systems. Front, new models cause too; difficult, full others comprehend eve	Sports                                            	fishing                                           	2.89	2105.84	0.50432867101214193
+AAAAAAAAAJPBAAAA	Delightful, married guns should go much tremendous, clear networks. Again just hours shall know there. Large, whole years cannot want 	Sports                                            	fishing                                           	9.33	2187.51	0.52388786001109799
+AAAAAAAAALDAAAAA	Very modern weeks must prevent hotly to a situations. Points look strongly regulations. Times take good groups. As af	Sports                                            	fishing                                           	68.83	2026.90	0.48542329107363830
+AAAAAAAABEFDAAAA	Members support general, mysterious programmes. Front times want with the services. Now new details should impose never cheap live activiti	Sports                                            	fishing                                           	4.96	11202.69	2.68293781078382606
+AAAAAAAABHGCAAAA	Tests shall see famous, good words; sexual, significant theo	Sports                                            	fishing                                           	8.63	11684.99	2.79844407813042221
+AAAAAAAACBEEAAAA	Personal, lacking artists cut pieces. Prices make quickly for a rooms. High, overall types ought to use together supposed women; reductions shall give prices. Lengthy, fundamental meas	Sports                                            	fishing                                           	9.23	13101.80	3.13775661107533389
+AAAAAAAACEMBAAAA	Other offices shall embark blindly resources. Spectacular copies may look also old, other offices. Properties fill better important others. Very wrong supplies will not own both aspects. Certainly	Sports                                            	fishing                                           	7.25	386.95	0.09267084833042791
+AAAAAAAACKDBAAAA	Sheets identify in a persons. Successful studies cannot solve for instance impressive governments; public buildings can move to a women. Substances sweep even on a tales; however great spac	Sports                                            	fishing                                           	4.50	5339.33	1.27871880247087137
+AAAAAAAACLCEAAAA	Inherent, public girls run. Opposite, similar players might adjust though central ties. Entirely small generations achieve rats. At all western boxes prosecute almost suspicious, ordinary v	Sports                                            	fishing                                           	0.46	2861.92	0.68540264699268189
+AAAAAAAACPJAAAAA	Difficult skills can tell specifically significant applicants. Irish women find si	Sports                                            	fishing                                           	8.65	0.00	0.00000000000000000
+AAAAAAAADCPCAAAA	Usually english commentators will indicate still dangerous, spiritu	Sports                                            	fishing                                           	9.90	13087.32	3.13428878865945433
+AAAAAAAADDDBAAAA	Early, associated parents continue stories. Alive, key costs will not supply. For example excellent wi	Sports                                            	fishing                                           	0.65	9375.15	2.24525934545809862
+AAAAAAAADEDDAAAA	Just left grounds would not shoot other, accessible readers. Long, true winners shall vary; male conditions must hear never local, clean studies. Major, generous pp. must not get always gre	Sports                                            	fishing                                           	3.62	8.19	0.00196142718135729
+AAAAAAAADFMCAAAA	Groups deserve also only members. Inevitable, rare dreams worry much old enquiries. Please clear nerves turn necessar	Sports                                            	fishing                                           	2.58	3603.80	0.86307585789687587
+AAAAAAAADFPCAAAA	Foreign advances expand never new, colonial players. Colours confess lines. Urgent, massive items sit then men. Different countries cut however. Effectively old ideas suggest only actually particul	Sports                                            	fishing                                           	4.19	20.28	0.00485686730621806
+AAAAAAAADGICAAAA	Sole, public skills require long opportunities. Parents destroy how	Sports                                            	fishing                                           	4.84	1396.88	0.33453948731311060
+AAAAAAAADKFBAAAA	Courses try military parents. Fast, w	Sports                                            	fishing                                           	1.64	6454.18	1.54571478453878082
+AAAAAAAADLIAAAAA	New parties strengthen please at all current things. Similar teams must lead most real firms. Simply tiny planes will set moving advantages. Concerned, average memories use	Sports                                            	fishing                                           	2.13	5552.34	1.32973267352104439
+AAAAAAAADOHBAAAA	International, new heads succeed altogether. Inc men see about accord	Sports                                            	fishing                                           	4.11	4917.54	1.17770410517847910
+AAAAAAAAEABAAAAA	Illegal campaigns ought to become here western, certain abilities. Indirect teachers would not tend no longer long, main agreements. Twice sweet patients ought to enjoy	Sports                                            	fishing                                           	0.33	2469.18	0.59134514867689882
+AAAAAAAAEAOBAAAA	Common, preliminary children will not maintain early further international 	Sports                                            	fishing                                           	3.67	4265.38	1.02151798178483168
+AAAAAAAAEBGCAAAA	Northern, de	Sports                                            	fishing                                           	15.22	1489.04	0.35661093163959266
+AAAAAAAAEJHDAAAA	Unable occasions command more effective, other birds. Proper songs know in a ports. Later wealthy details look now hours. Aware, black issues 	Sports                                            	fishing                                           	0.59	4257.58	1.01964995589782473
+AAAAAAAAFEICAAAA	Points can appoint even pregnant ideas. Other, basic bodies shall frighten too modern laws; features accompa	Sports                                            	fishing                                           	1.97	15202.78	3.64092135826557149
+AAAAAAAAFFMAAAAA	Home available features need with a questions. Hard waters can operate still more content bands. Organic, large ideas contribute points. Difficult, right produc	Sports                                            	fishing                                           	2.47	7589.73	1.81766821992220870
+AAAAAAAAFJCAAAAA	Collective, full signals will assume only services. Political villages think children. So old	Sports                                            	fishing                                           	2.56	2552.33	0.61125878361338953
+AAAAAAAAGDABAAAA	Industrial, slight needs would disturb too for a folk. Now known buildings ought to suggest so. Papers create colours. Good levels tell into a r	Sports                                            	fishing                                           	2.72	5261.10	1.25998346078618504
+AAAAAAAAGNJAAAAA	Norma	Sports                                            	fishing                                           	1.01	8662.39	2.07456009786539724
+AAAAAAAAGOCBAAAA	Onwards horizontal sports find. Normal, powerful eyes come american, commercial situations. Major, enormo	Sports                                            	fishing                                           	1.89	13071.78	3.13056710631534049
+AAAAAAAAHACEAAAA	Shoes give more now annual ch	Sports                                            	fishing                                           	1.18	6235.99	1.49346035270723652
+AAAAAAAAHIJBAAAA	As modern women may find only with a bones. However simple minutes end changes. Catholic hands provide hard able rights. Weeks used to affect then tiny c	Sports                                            	fishing                                           	2.55	3728.50	0.89294032303915358
+AAAAAAAAHODBAAAA	Strong, southern weeks use to a exceptions. Shoulders write natural, particular courses. Cold, labour things will hang. New authorities may bu	Sports                                            	fishing                                           	1.08	5888.16	1.41015837267164344
+AAAAAAAAHPCBAAAA	Automatically private stands go always easy terms. Well distinctive depar	Sports                                            	fishing                                           	1.17	5365.88	1.28507727520164501
+AAAAAAAAIBBDAAAA	Internatio	Sports                                            	fishing                                           	1.86	8437.51	2.02070347459999698
+AAAAAAAAIBIAAAAA	Apparent,	Sports                                            	fishing                                           	7.13	2649.10	0.63443427913719237
+AAAAAAAAICNDAAAA	Special, easy things invest here hot 	Sports                                            	fishing                                           	4.61	8905.67	2.13282334630014721
+AAAAAAAAIDMDAAAA	Leaves could not help accounts; maximum, supreme expenses may not build in a officers; r	Sports                                            	fishing                                           	0.44	13341.40	3.19513853447621392
+AAAAAAAAIGFAAAAA	Still original workers solve merely villages. Only long years punish already. Scottish features should not take from th	Sports                                            	fishing                                           	4.81	3.50	0.00083821674416978
+AAAAAAAAJFODAAAA	Settlements must make significa	Sports                                            	fishing                                           	7.42	7154.29	1.71338447732755427
+AAAAAAAAJHDDAAAA	Shortly new terms would recover yet satisfactory, previou	Sports                                            	fishing                                           	2.86	3393.96	0.81282117172642234
+AAAAAAAAJHKCAAAA	Public, certain lives could not choose indeed in a tools. Then bad things gain women.	Sports                                            	fishing                                           	2.62	392.55	0.09401199512109957
+AAAAAAAAJJEAAAAA	Circumstances cannot take lines. Modern goods would make corresponding tools. Subsequently toxic practices see annually alm	Sports                                            	fishing                                           	3.56	12846.92	3.07671527285990692
+AAAAAAAAKGAEAAAA	Also normal groups must not keep possibly others. Rates will not depend centuries. Fields could indicate already in a months; important arti	Sports                                            	fishing                                           	64.57	16106.48	3.85734892161020958
+AAAAAAAAKIACAAAA	Crops shall argue already for the responses. Easy committees like as with a figures. Easy current groups should not meet nevertheless; evident, international forces sen	Sports                                            	fishing                                           	6.00	1274.25	0.30517076750238473
+AAAAAAAAKILCAAAA	Elements take further vital, typical 	Sports                                            	fishing                                           	1.73	6796.42	1.62767801268868558
+AAAAAAAAKIMBAAAA	Good, silent examples close so literary, financial years. Often foreign interests discourage best suddenly whi	Sports                                            	fishing                                           	4.19	4776.06	1.14382098947415311
+AAAAAAAAKKBCAAAA	Projects support indeed in a departments. Populations involve even with a terms; fine, classical miles visit continuously crucial, great days. Steady, sc	Sports                                            	fishing                                           	0.68	7528.93	1.80310719762348789
+AAAAAAAAKOACAAAA	Directions use none the less. Military, new recordings pass yellow tasks. Frequently wor	Sports                                            	fishing                                           	1.49	1880.44	0.45034751268760788
+AAAAAAAAKOIDAAAA	Poor networks explain personally to a funds. Already federal words accelerate companie	Sports                                            	fishing                                           	2.01	7024.79	1.68237045779327228
+AAAAAAAALBICAAAA	Sectors might not know properly. Large, electric workers used to drop even as ca	Sports                                            	fishing                                           	6.89	1774.46	0.42496630967414683
+AAAAAAAALOIBAAAA	Old others will cut other activities. Sharp passages avoid allegedly orthodox, additional firms. High officers must form.	Sports                                            	fishing                                           	0.25	2612.13	0.62558031541377612
+AAAAAAAAMAEBAAAA	Very acids should depend much a little christian tons. Warm rules defeat at large details. Banks should not seek then. Times can back stiffly ordinary, chemical	Sports                                            	fishing                                           	6.07	10778.84	2.58142976306486528
+AAAAAAAAMEMDAAAA	Factors might assist now absent, voluntary demands; political companies might know no longer concerned things; autonomous, possible events can dry at 	Sports                                            	fishing                                           	6.68	6637.53	1.58962536740836076
+AAAAAAAAMIIDAAAA	Pale, other animals assist in a words. Is	Sports                                            	fishing                                           	3.40	1226.34	0.29369677772719206
+AAAAAAAAMKLCAAAA	Necessary women know little international troops. Immediate, possible drugs can try effectively too gentle spots. Northern, german ideas tell to a areas. False appropriat	Sports                                            	fishing                                           	2.18	505.79	0.12113189915246708
+AAAAAAAAMNAAAAAA	Western, social things will go in order. Warm, male cards used to describe. High, briti	Sports                                            	fishing                                           	0.51	2346.30	0.56191655624158939
+AAAAAAAAMNEDAAAA	Different, common buildings could not comply even. Impossible transactions build always qualities. Police move tiles. Options must use just different stages; words	Sports                                            	fishing                                           	8.87	4167.10	0.99798085560854416
+AAAAAAAANCGBAAAA	Members like very. Then interested principles could remember yet important, new agents. Necessarily due assets generate across. Areas give anyway as social projects. Main, 	Sports                                            	fishing                                           	1.79	7991.56	1.91390268686784986
+AAAAAAAANECDAAAA	Blind sessions hurt traditionally public, various clothes. High, southern schools might not tal	Sports                                            	fishing                                           	1.43	1122.60	0.26885203342999968
+AAAAAAAANMFBAAAA	Practical roads mean either dishes. Necessary issues determine simply fund	Sports                                            	fishing                                           	3.40	4810.52	1.15207383204675046
+AAAAAAAAOCHDAAAA	Just formal teams ask still, certain interests. Well l	Sports                                            	fishing                                           	9.79	2218.77	0.53137433298902583
+AAAAAAAAOEKDAAAA	Books must work major, able forces. Clearly future teachers would measure certain, direct measures. Hard tears go main nurses. Cruel patients used to leave much only days. Yet social defence	Sports                                            	fishing                                           	8.56	1810.80	0.43366939438361253
+AAAAAAAAOGNCAAAA	Comprehensive, able acts must not resign. British, red forces convict perhaps; years want as well problems	Sports                                            	fishing                                           	54.91	119.66	0.02865743303067322
+AAAAAAAAOIOCAAAA	New companies must benefit always to a companies; adults might get yet international, comfortable indicators. Dual bones shall find ever like parents. Wars need new, heavy purposes	Sports                                            	fishing                                           	3.43	7734.29	1.85228896636140409
+AAAAAAAAOLKDAAAA	Backs think now back, british wines. Very fine shows get often serious, fatal prisoners. Good terms ought to come far easy, obvious shoulders. Machines play more ac	Sports                                            	fishing                                           	2.94	7583.99	1.81629354446177025
+AAAAAAAAOOFEAAAA	Tiny values allow equations. 	Sports                                            	fishing                                           	4.39	7729.84	1.85122323364381680
+AAAAAAAAPBKCAAAA	Ill, simple objects shall bear solid trees. Ears should use there minimum, inappropriate personnel. Available practices should not apply increasingly pr	Sports                                            	fishing                                           	7.87	15557.69	3.72591893102937088
+AAAAAAAAPDKAAAAA	Sure reliable suppliers show upright other women. Maybe	Sports                                            	fishing                                           	1.11	12392.70	2.96793389870653577
+AAAAAAAAPEFEAAAA	Much common times achieve existing, continuing positions. Los	Sports                                            	fishing                                           	8.20	9965.46	2.38663298152977430
+AAAAAAAAPHKCAAAA	Good, whole facilities maintain for a points. More worthwhile directors battle annual hours. Yes	Sports                                            	fishing                                           	8.90	603.00	0.14441277049553697
+AAAAAAAAPLGCAAAA	Rules offer. Important, italian goo	Sports                                            	fishing                                           	4.06	3150.39	0.75448847104715544
+AAAAAAAAPLIAAAAA	Vital, similar activities change thickly. Seats would sit essentially brilliant words. Hig	Sports                                            	fishing                                           	68.38	6302.32	1.50934575746174558
+AAAAAAAAPMMBAAAA	Even useless times make old, old studies. Early public employees must open together warm consequences. Sufficient, evident men would operate stars. Various, other sections control l	Sports                                            	fishing                                           	89.62	2679.48	0.64171000047658609
+AAAAAAAAAFKCAAAA	A	Sports                                            	fitness                                           	7.12	10468.61	4.22441966945963305
+AAAAAAAAAHMDAAAA	Fast bizarre situations fulfil all as political plans. Thus labour conventions run more part-time experiments. Early considerable own	Sports                                            	fitness                                           	0.81	5713.17	2.30544721056249987
+AAAAAAAAAPNAAAAA	Other, cultural differences might take. Musical branches take only new defences. 	Sports                                            	fitness                                           	3.76	18567.33	7.49251276543379958
+AAAAAAAABDGDAAAA	Increased machines claim 	Sports                                            	fitness                                           	1.76	2327.22	0.93910786084875139
+AAAAAAAACAHCAAAA	New parties survive	Sports                                            	fitness                                           	1.06	5055.94	2.04023384036732070
+AAAAAAAACEFDAAAA	Abruptly real years cope together; significant accounts provide at a others. Twice competent languages cannot impose most protests. Identical leaders 	Sports                                            	fitness                                           	3.76	11311.78	4.56466578930728034
+AAAAAAAACGPCAAAA	Clinical, real figures figure effects. Full, pleased bacteria used to fit immediately more main things. Windows will not present perhaps	Sports                                            	fitness                                           	4.25	1715.39	0.69221484579083182
+AAAAAAAACMADAAAA	Concerned clothes comment less small probl	Sports                                            	fitness                                           	0.73	1855.00	0.74855195549816254
+AAAAAAAACOKCAAAA	Large, working matters oppose etc far remote aspects; today amer	Sports                                            	fitness                                           	3.52	11563.15	4.66610164108818229
+AAAAAAAADKMDAAAA	Physical questions confirm much to the marks. Irish, pleased eyes would know in an subsi	Sports                                            	fitness                                           	2.86	8639.15	3.48617392255630775
+AAAAAAAADOCAAAAA	Little, national services will buy young molecules. In part video-taped activities join now	Sports                                            	fitness                                           	5.91	408.38	0.16479441918401058
+AAAAAAAAEBBEAAAA	Intelligent trends used to bother open. Bedrooms will not hit all senior, economic boys; objects would sum. Often blue times should deal in a	Sports                                            	fitness                                           	3.84	1925.10	0.77683955230701493
+AAAAAAAAEBMDAAAA	Absolutely wild standards impose only so scottish schools. New, complex incomes can establish children. Certainly free groups will rest. Impressive teeth must go front s	Sports                                            	fitness                                           	4.00	2927.91	1.18150552885316716
+AAAAAAAAEHJDAAAA	Policies think races. Loc	Sports                                            	fitness                                           	40.32	1793.89	0.72389211183212873
+AAAAAAAAEIOAAAAA	Shares could release barely months. Aware writings used to use so very impossible authorities. 	Sports                                            	fitness                                           	6.66	3449.47	1.39197170562385268
+AAAAAAAAEMMBAAAA	Boys might not fill in a problems. Military, young ways will encourage somehow inner, large matters. Ways will begin today whole firm	Sports                                            	fitness                                           	3.62	2731.00	1.10204603259594711
+AAAAAAAAFFPDAAAA	Corporate heroes examine forth technical, formal shares; buildings may not emphasize abo	Sports                                            	fitness                                           	68.11	4428.60	1.78708204319092324
+AAAAAAAAFHBDAAAA	Below old resources could cover lo	Sports                                            	fitness                                           	2.86	2908.84	1.17381017263141516
+AAAAAAAAFKKAAAAA	Running children may continue common, small wives; great, subtle teams shall change bad, good lines; others may want; parties used to like near a sty	Sports                                            	fitness                                           	2.32	2591.76	1.04585822974766455
+AAAAAAAAFLCBAAAA	Labour, dominant dreams come. Please various symptoms cannot persuade so owners. Primary colours would argue once small posts. Live, asia	Sports                                            	fitness                                           	48.03	4332.46	1.74828647176149287
+AAAAAAAAFMOAAAAA	Deep, light measures could ask around experimental sections. Days attend social, wise cases. Children should find; as	Sports                                            	fitness                                           	3.91	12590.50	5.08067029417769025
+AAAAAAAAFNPBAAAA	Times force also years. Emotional solutions ought to allow elderly differences. Too urban parents shall accommodate so. Traditional, effective im	Sports                                            	fitness                                           	3.60	8417.45	3.39671086674286159
+AAAAAAAAFOGAAAAA	Principal eyes should pay frequently relevant areas. Light police m	Sports                                            	fitness                                           	3.17	451.78	0.18230771021830721
+AAAAAAAAGAABAAAA	Original hands know as. So prime things might not identify. Less than little journals let very hard things; nurses see; large bodies name once political, national c	Sports                                            	fitness                                           	6.83	1540.63	0.62169358447392677
+AAAAAAAAGCMAAAAA	Methods develop warm males. Governments depend please with the hospitals. At random tory weaknesses enter approximately simply young me	Sports                                            	fitness                                           	6.01	24.98	0.01008023064600760
+AAAAAAAAGENAAAAA	Also new activities would not drop immediately fina	Sports                                            	fitness                                           	6.42	9171.55	3.70101438676505262
+AAAAAAAAGPBAAAAA	Beings should affect close projects. In common labour metres might call directly 	Sports                                            	fitness                                           	2.85	837.90	0.33811950593633983
+AAAAAAAAIAFBAAAA	Men could not escape so old victims. Tiny horses give together effective teeth; little, beneficial bones used to forget again days. Of course	Sports                                            	fitness                                           	71.90	2421.19	0.97702776772646693
+AAAAAAAAIKEEAAAA	Regions see in the cop	Sports                                            	fitness                                           	1.90	8595.06	3.46838219440648889
+AAAAAAAAINBDAAAA	Asleep, fat topics pick into a rul	Sports                                            	fitness                                           	2.70	3452.62	1.39324283158601937
+AAAAAAAAJDJBAAAA	Conscious, central results play above about the hands. Stages stay so available universities. Tomorrow professional birds decide; enthusiastically big views appear new window	Sports                                            	fitness                                           	9.62	412.47	0.16644486527456987
+AAAAAAAAJPAEAAAA	Please positive sys	Sports                                            	fitness                                           	0.31	4494.44	1.81365059346046449
+AAAAAAAAKABCAAAA	Simply necessary girls could not take supreme hospitals. Issues ought to 	Sports                                            	fitness                                           	93.50	342.93	0.13838324641454710
+AAAAAAAAKAGEAAAA	Overseas campaigns must finance just. Researchers believe sure, positive days. Workers appear from a values. Periods can lift ago related, extens	Sports                                            	fitness                                           	8.92	691.02	0.27884871821473869
+AAAAAAAAKDJAAAAA	Regular, gold effects take gently for a terms. Good, strong difficulties attract articles. Ultimate farmers develop 	Sports                                            	fitness                                           	1.12	3313.24	1.33699853425052940
+AAAAAAAAKFPCAAAA	Round prisoners go at all into a lives. Streets find again places. Kindly liable men offer plainly on a contents. Early accurate regions should no	Sports                                            	fitness                                           	4.49	3281.89	1.32434780443658472
+AAAAAAAAKLAEAAAA	More 	Sports                                            	fitness                                           	0.82	1089.45	0.43962799348650845
+AAAAAAAAKNGAAAAA	Solid, romantic feet would come so equations. Only economic feet will n	Sports                                            	fitness                                           	0.36	6592.06	2.66010749528906595
+AAAAAAAAKOICAAAA	Only subjects think for a goods. Windows wo	Sports                                            	fitness                                           	3.66	9334.78	3.76688292352837611
+AAAAAAAALCIBAAAA	Special miles must ease under across a conditions. Points might continue australian, australian places. Entirely 	Sports                                            	fitness                                           	3.17	0.00	0.00000000000000000
+AAAAAAAALLDDAAAA	Men mean also weapons. Individual proposals ought to mean farmers. Sometimes valuable eyes might take rights. Rough, different rewards cost real, alone ministers. Requirements may no	Sports                                            	fitness                                           	64.89	3913.00	1.57902091744706739
+AAAAAAAAMDKDAAAA	Together working cases used to buy in a structures. Millions must 	Sports                                            	fitness                                           	1.88	3472.20	1.40114398915402693
+AAAAAAAAMGABAAAA	Sure, coming sessions could not pass very. Concerned children pick on a individuals. Easy pairs shall return. Reports consider subsequently rough sites. Vital, normal w	Sports                                            	fitness                                           	2.27	5967.84	2.40821471811329074
+AAAAAAAAMHMAAAAA	Girls move ways. Other, human actors should participate serious families. New di	Sports                                            	fitness                                           	4.79	10717.00	4.32465299572712017
+AAAAAAAAMKDDAAAA	Quick reasons could set only distant a	Sports                                            	fitness                                           	1.29	968.12	0.39066744968025936
+AAAAAAAANBGCAAAA	So close miles would seem american, emotional horses. Other, alive operations ought to want further red layers. Parameters might faint bad, significant stations. So prime newspapers wou	Sports                                            	fitness                                           	2.97	9281.14	3.74523746428690903
+AAAAAAAANNEBAAAA	Royal speeches take evil, front margins. For example hard events ought to go angles. Possible, foreign lakes shall not reconsider. Other honours hear momen	Sports                                            	fitness                                           	8.13	0.00	0.00000000000000000
+AAAAAAAAPDHAAAAA	Points force into the symptoms. Local, strong negotiations get examples. For the time being fat result	Sports                                            	fitness                                           	5.61	19543.75	7.88652953114135530
+AAAAAAAAPFFDAAAA	Subject, dead qualifications benefit more real nurses. Up to special writers give most responses; social circumstances de	Sports                                            	fitness                                           	2.69	12178.65	4.91447561877503891
+AAAAAAAAAIMAAAAA	Just ready clothes try live skills. Girls investigate up 	Sports                                            	football                                          	1.80	3028.92	1.26615656780156976
+AAAAAAAAAKKAAAAA	Mostly furious applications cut in a workers; successful, substantia	Sports                                            	football                                          	3.20	4690.04	1.96054202463322710
+AAAAAAAAAMBDAAAA	Dynamic, technical problems cannot go important, general sources. Overall inevitable subjects may take. Recent ends would n	Sports                                            	football                                          	2.51	10300.92	4.30601584472305176
+AAAAAAAAAOCEAAAA	Allowances might lay at best children. Academic sections burst hot times. Short-term, warm goods	Sports                                            	football                                          	4.96	652.80	0.27288505720219244
+AAAAAAAABDPAAAAA	Sophisticated, unfair questions may remove separate premises. Typical patterns intervene typically walls. Naked areas ought to return now military, necessary children; young met	Sports                                            	football                                          	33.19	7921.58	3.31139830182558766
+AAAAAAAACAGAAAAA	Only available cars could not allow during a films. Cuts might not grow also unfortunately poor names. Windows go at first so key effects. Leading, possible relationships used to rec	Sports                                            	football                                          	1.80	5455.78	2.28063853765713464
+AAAAAAAACCDDAAAA	Pupils talk tonight even expected rights. However federal costs may not borrow large decisions. Social, american soldiers repair legal, economi	Sports                                            	football                                          	11.06	1681.47	0.70289221374658476
+AAAAAAAACEBEAAAA	British components must go. Wrong, overseas jobs explain with a towns. Quite ideological habits may	Sports                                            	football                                          	0.63	8173.32	3.41663127409899441
+AAAAAAAACGFAAAAA	Girls would face criminal, special offenders. Healthy principles get very greek, ade	Sports                                            	football                                          	1.47	435.76	0.18215746404170861
+AAAAAAAACIECAAAA	Delicate readers gain too able officers. Feet see as international appearances; just prominent samples halt just. Substantia	Sports                                            	football                                          	94.83	12471.06	5.21318309049015641
+AAAAAAAACNDCAAAA	Daily, level areas fetch known, other 	Sports                                            	football                                          	69.68	818.79	0.34227260414611390
+AAAAAAAACPNBAAAA	More reasonable opp	Sports                                            	football                                          	3.70	3418.34	1.42894287137950754
+AAAAAAAADEKCAAAA	Awful eyes get now like a gentlemen. Final countries may become french, turkish sciences. French lives repeat great, big standards. Large, able roads cl	Sports                                            	football                                          	6.18	5009.22	2.09396643112494858
+AAAAAAAADGHAAAAA	Thanks may add suddenly strong weeks. Times abandon as files. Systems feel cheap targets. Green, formal events understand french, rea	Sports                                            	football                                          	0.97	2280.64	0.95335872680393409
+AAAAAAAADGKDAAAA	Miserable officers introduce clearly. Much mathematical eyes could change so before prominent plans. Prices i	Sports                                            	football                                          	4.67	20055.07	8.38346955291662626
+AAAAAAAADLIDAAAA	Else social offenders will not support mines. Gently intelligent expressions speed days. Sometimes old houses offer really important, local month	Sports                                            	football                                          	2.19	15388.53	6.43275105592471583
+AAAAAAAADPEAAAAA	Critics can cover only str	Sports                                            	football                                          	1.79	10295.54	4.30376688392686948
+AAAAAAAAEBNBAAAA	Sources negotiate never books.	Sports                                            	football                                          	12.71	1473.07	0.61577633457848288
+AAAAAAAAEHMDAAAA	Young, previous metals keep here due, equal churches. Strong temperatures avoid. Established, average children could help also technical aspects. Feelings navigate now weekl	Sports                                            	football                                          	1.45	8988.48	3.75738645674136449
+AAAAAAAAEIACAAAA	White, vital departments should become aga	Sports                                            	football                                          	2.88	4166.35	1.74162784631488126
+AAAAAAAAEJPDAAAA	Daily, marked years may not save players. Then hot families please universally always parental opportunities. Closely medic	Sports                                            	football                                          	3.21	1605.80	0.67126045474154508
+AAAAAAAAELMCAAAA	Popular, strong farms worry certainly followers. New documents will argue considerably under a men. Catholic, exist	Sports                                            	football                                          	0.10	1110.81	0.46434352081919024
+AAAAAAAAEOODAAAA	Clearly great options cannot believe. Responsible products ought to condemn at a systems. Dull types assure; real ser	Sports                                            	football                                          	3.03	8226.16	3.43871958050610814
+AAAAAAAAGDEAAAAA	Succ	Sports                                            	football                                          	4.47	9246.93	3.86542435967320677
+AAAAAAAAGFFDAAAA	Almost busy threats go together recent sides; still tired wines shall not admit on a	Sports                                            	football                                          	3.88	7510.88	3.13971648045159802
+AAAAAAAAHAFEAAAA	Economic, crude hands put available payments; irish months pay main, tropical members. Neither soft syste	Sports                                            	football                                          	4.23	2877.00	1.20265059676885365
+AAAAAAAAHAGAAAAA	International, profitable schools sit rather di	Sports                                            	football                                          	81.85	205.56	0.08592869540208744
+AAAAAAAAIPLBAAAA	Young features may seem actually for the plans. Unduly	Sports                                            	football                                          	9.86	3012.65	1.25935534249415605
+AAAAAAAAJLIAAAAA	Standards must pa	Sports                                            	football                                          	3.63	836.01	0.34947095078370849
+AAAAAAAAJLOAAAAA	Very aspects use then. Popular, weste	Sports                                            	football                                          	6.30	1501.17	0.62752276550278069
+AAAAAAAAJPFCAAAA	Models may register still digital, professional birds. There necessary things can fail never irish forces. All corporate readers identify more	Sports                                            	football                                          	68.59	9190.37	3.84178100974159524
+AAAAAAAAKGCEAAAA	Again sexual officials shall not	Sports                                            	football                                          	7.81	11678.56	4.88190029662873252
+AAAAAAAAKNDCAAAA	Ages must answer even such as a citizens. Fatal candidates say also. Thus great friends create normally 	Sports                                            	football                                          	19.60	1325.80	0.55421416795138901
+AAAAAAAALDPCAAAA	Successive, joint	Sports                                            	football                                          	4.67	4363.92	1.82421654231892103
+AAAAAAAALHDBAAAA	Democrats take before. Joint years woul	Sports                                            	football                                          	65.80	7674.39	3.20806733171252094
+AAAAAAAALHEAAAAA	Hours take so. Now new things want common, recent drugs. Ships will st	Sports                                            	football                                          	3.32	1013.26	0.42356543054640551
+AAAAAAAALMIDAAAA	Quiet, small objectives should stay as matches. In particular formal students allow then. Professional, other demands drop	Sports                                            	football                                          	1.58	2487.00	1.03962184016827912
+AAAAAAAAMDLCAAAA	Super stars might like approximately stories. Major practices might allow more fresh decisions. Advanced organisations wield. Towns must not protect quickly. Active, righ	Sports                                            	football                                          	4.05	6655.69	2.78222785902276383
+AAAAAAAAMHAAAAAA	Cheaply financial tales allow unfortunately safe, red meals. Who	Sports                                            	football                                          	2.91	5952.36	2.48822012727947644
+AAAAAAAAMIEBAAAA	Hard figures will not help twice central principles. Collective, impor	Sports                                            	football                                          	2.33	468.64	0.19590204229049551
+AAAAAAAAMJKAAAAA	Advanced, foreign stories would greet always corporate games. Recent dev	Sports                                            	football                                          	3.00	634.63	0.26528958923441696
+AAAAAAAANIEAAAAA	Very questions make secret stocks. Aggressive, major years qualify for example senio	Sports                                            	football                                          	4.39	292.60	0.12231336969571310
+AAAAAAAAODBBAAAA	Matters reserve more proper, concerned birds. True months result together more chemical columns. Social views reduce in a affairs. Medieval, serious sports may n	Sports                                            	football                                          	0.16	7175.77	2.99963297628642230
+AAAAAAAAOGADAAAA	Proud things mus	Sports                                            	football                                          	28.70	17469.96	7.30283552990198210
+AAAAAAAAACEEAAAA	Unacceptable flowers should not give reasonable, ethnic governments. Employees shall complain 	Sports                                            	golf                                              	8.39	4100.46	1.45417454300510042
+AAAAAAAAAHOCAAAA	Crucial products would carry silently double groups. Really full systems run usual structures. Financial departments must meet well c	Sports                                            	golf                                              	1.50	12212.90	4.33114535351326216
+AAAAAAAAANGBAAAA	Different hours must not know towards a weapons. Facilities shall not know items. Today established fl	Sports                                            	golf                                              	5.73	437.77	0.15524940852766344
+AAAAAAAABEEBAAAA	Educational terms must apply automatic, other objectives. Indeed financial sources pass very unacceptabl	Sports                                            	golf                                              	6.99	16143.50	5.72508126771211978
+AAAAAAAABPHCAAAA	More black mothers shall repea	Sports                                            	golf                                              	14.90	7660.56	2.71671747490846200
+AAAAAAAACCMAAAAA	Admini	Sports                                            	golf                                              	9.35	2840.01	1.00717242550345943
+AAAAAAAACGGCAAAA	Separate, rapid bodies will start too religious surveys. Geographical, loyal things involve in order. Notes need dead for a members; at last economic managers look once more nervous skills; joint	Sports                                            	golf                                              	6.57	2341.31	0.83031498887521685
+AAAAAAAACIGDAAAA	European quantities would wait	Sports                                            	golf                                              	0.73	9236.58	3.27563236818065546
+AAAAAAAACKEAAAAA	Wet, suitable projects shall follow voluntarily all of a sudden resulting negotiations. High, video-taped services should not take all full eyes; wrong representatives follow royal, full figures. Fre	Sports                                            	golf                                              	3.35	7298.73	2.58839919478975935
+AAAAAAAACNHBAAAA	Good, interior faces contribute with a rights. Social, certain versions pick furiously between a troops. Forward political countries bec	Sports                                            	golf                                              	7.89	4757.12	1.68705042898124194
+AAAAAAAACPMCAAAA	Great, new errors w	Sports                                            	golf                                              	3.21	791.01	0.28052135742391451
+AAAAAAAADJICAAAA	Stairs say long words. Newspapers will go exceedingly. Other, empty numbers must not provide therefore environmental months. Entirely bare groups buy. New days	Sports                                            	golf                                              	20.77	1505.63	0.53395199982069557
+AAAAAAAAEBEBAAAA	Labour parties worry far well clear files. Finally domestic generations would not announce too; continuous, possible patterns might conceal	Sports                                            	golf                                              	4.32	2152.66	0.76341273216794201
+AAAAAAAAENPDAAAA	Live processes review home at pres	Sports                                            	golf                                              	2.74	4204.30	1.49100004174076658
+AAAAAAAAFCFEAAAA	Judicial models should not pick. Close dogs can refuse exactly. European, r	Sports                                            	golf                                              	5.70	6536.36	2.31803463902021193
+AAAAAAAAFIJAAAAA	Pages could watch fundamental, literary components. Financial, royal elements should overcome environmental trustees. Shared areas 	Sports                                            	golf                                              	3.07	4544.08	1.61149857756900853
+AAAAAAAAFOBCAAAA	Demands could treat lines. Conditions suck studies. Documents could not hide local things; gold calls see together. Preferences may refuse indeed in a pieces. Old, unknown boys emerge more opposite, 	Sports                                            	golf                                              	2.87	625.67	0.22188568753798383
+AAAAAAAAGBJDAAAA	New sources play just. English groups evaluate here indian changes. Familiar, able authorities get direct important, emotional orde	Sports                                            	golf                                              	6.52	7170.18	2.54281061753176740
+AAAAAAAAGCHBAAAA	Most angry years help intimate conditions. By far urgent police would agree 	Sports                                            	golf                                              	1.81	13747.41	4.87533926785135024
+AAAAAAAAGFCAAAAA	Then growing levels light sometimes human, fellow cities. Users may derive odd championships. Stages support right 	Sports                                            	golf                                              	8.86	5586.76	1.98127141098295675
+AAAAAAAAGHBCAAAA	Brown customers can detect too. Then human numbers cannot prepare never victorian long accountants; interests share open in the years. Full-time, underlying 	Sports                                            	golf                                              	92.44	6716.33	2.38185864718140065
+AAAAAAAAGPGCAAAA	Secondary, normal 	Sports                                            	golf                                              	6.04	7486.01	2.65481559890393074
+AAAAAAAAHCABAAAA	Wishes might behave environmental regions. Statements conflict now nuclear	Sports                                            	golf                                              	7.46	16077.73	5.70175679687386128
+AAAAAAAAHLDBAAAA	Horses say. Other peasants can keep at first large kilometres. Necessarily new miles separate for an poems; interestingly indian teeth used to make further.	Sports                                            	golf                                              	3.40	752.00	0.26668697081299062
+AAAAAAAAIBLAAAAA	Russians receive then definit	Sports                                            	golf                                              	8.76	20347.14	7.21584724907956645
+AAAAAAAAIDOBAAAA	Independent, scientific subsidies might contain. Here certain instructions shall not imagine exhibitions. Either other attitudes buy finally. Public, right p	Sports                                            	golf                                              	4.05	198.74	0.07048054332363531
+AAAAAAAAIIAEAAAA	Married professionals clarify plans. All basic children could prove more religious big trees.	Sports                                            	golf                                              	4.01	7501.44	2.66028764672260686
+AAAAAAAAIKPCAAAA	Roles shall not remember primary, inc years. Young feelings can s	Sports                                            	golf                                              	5.74	3892.36	1.38037459802347363
+AAAAAAAAILFEAAAA	Particular, complete artists belong much enough active cheeks; profits may see able, complete processes. Here available officials take aside at a eyebrows. 	Sports                                            	golf                                              	4.07	10080.46	3.57490338005521200
+AAAAAAAAINHDAAAA	Poles decide over for a managers. Properly other views include slim functions. Bright, other minutes should talk exactly certain weeks.	Sports                                            	golf                                              	6.56	1356.03	0.48089831520151552
+AAAAAAAAKBNCAAAA	Inevitably dead trees establish original, primary events. Other women ought to issue almost long medical achievements. Catholic, hard cars need here difficult humans. Great,	Sports                                            	golf                                              	0.80	5928.82	2.10257851900994022
+AAAAAAAAKGHAAAAA	Strong changes stay. Future claims will not recoup fo	Sports                                            	golf                                              	2.23	9989.59	3.54267752229221140
+AAAAAAAAKGMDAAAA	Impressive records lie easy origins. Social schools shall bend else different details. Novel chemicals present primarily by a bags. Molecules shall see repeated	Sports                                            	golf                                              	3.63	4279.32	1.51760490417479657
+AAAAAAAAKJGAAAAA	Also major pieces resign never. Substan	Sports                                            	golf                                              	4.63	55.04	0.01951921658716357
+AAAAAAAAKNCBAAAA	Assets may not engage heavily always formal groups. Local, genetic offices cannot keep still sad, annual troops; supreme, natural gaps can see. Nearl	Sports                                            	golf                                              	7.20	4005.33	1.42043793192339857
+AAAAAAAAMCBBAAAA	So overall investor	Sports                                            	golf                                              	2.54	15395.25	5.45972418538390139
+AAAAAAAAMDBDAAAA	Brothers appoint even.	Sports                                            	golf                                              	3.65	3103.75	1.10070436922981335
+AAAAAAAAMDEAAAAA	Closely substantial instructions wait for a companies; members may bring then characters; recent views should indicate near early days; objectives could not arrive categories. High gains speak	Sports                                            	golf                                              	7.73	77.67	0.02754465029660237
+AAAAAAAAMFADAAAA	Neighbours shall send important, excellent games. Plain important ways note monthly, japanese figures; routinely 	Sports                                            	golf                                              	4.81	616.44	0.21861238868079779
+AAAAAAAANGBCAAAA	Certainly persistent players move often respective minutes; amer	Sports                                            	golf                                              	7.78	74.48	0.02641335849222279
+AAAAAAAANMHCAAAA	Impossible, natural cases may wait then products. Political sectors go here sure consultants. Me	Sports                                            	golf                                              	2.14	2979.66	1.05669747267637717
+AAAAAAAANOGBAAAA	Classical, small perceptions finance again ideas. Obligations determine. Clear, useful crowds could take thus formal, genetic individuals. Int	Sports                                            	golf                                              	0.68	14169.23	5.02493221735711581
+AAAAAAAAOEIAAAAA	Forward working birds ought to try already then public pounds. Black, similar hands cover still at a rights. Right contracts save for example general, able feet. Systems could not t	Sports                                            	golf                                              	8.61	291.36	0.10332701571286296
+AAAAAAAAOKCDAAAA	Young, severe parts must not act therefore rath	Sports                                            	golf                                              	2.17	1012.25	0.35898123165618319
+AAAAAAAAOLHDAAAA	Only concerned times used to know however in the trees. Developers might not wear in the times. Studies see far variations. Calculations must not transport hardl	Sports                                            	golf                                              	0.15	8494.93	3.01261588958563618
+AAAAAAAAPEOBAAAA	Sales include easier from the times. Significant, young features should not keep hardly social	Sports                                            	golf                                              	4.30	403.10	0.14295414618978261
+AAAAAAAAPJLCAAAA	Likely, exciting negotiations disrupt even communications; all normal girls may think about years; allegedly old hands end darkly musical years. Individual, similar 	Sports                                            	golf                                              	4.26	9885.12	3.50562860229110351
+AAAAAAAAPNFBAAAA	Basic differences stem 	Sports                                            	golf                                              	0.88	12915.95	4.58047284663835931
+AAAAAAAAAILAAAAA	Continental issues need famous areas. Thus christian years shall agree just foreign negotiations. Sensitive centres may not assess large remains. Men eat from the ideas. Other, specific plants say 	Sports                                            	guns                                              	0.19	6159.12	2.82446517920513238
+AAAAAAAAAJGCAAAA	Revolutionary son	Sports                                            	guns                                              	4.83	7287.25	3.34180595233776919
+AAAAAAAACCDEAAAA	Businesses may keep also behind a workers. Early, previous objectives hit wet, bottom requests. Under true hours touch similar, long sources. Widely able attitudes must appear now politica	Sports                                            	guns                                              	2.73	6762.87	3.10133441571052580
+AAAAAAAACHGBAAAA	Occasional, biological questions make usually for a tools; parts will use between a machines. Languages swim alive commitments. Other russians shall finish b	Sports                                            	guns                                              	4.12	2865.32	1.31398585630415545
+AAAAAAAADDFCAAAA	Again dull trials ensure suddenly; communities should produce terms. Too extra notes might choose properly social, absolute talks	Sports                                            	guns                                              	6.99	8342.32	3.82564268171208874
+AAAAAAAADKLBAAAA	Only other packages shall not lift procedures. Available, only types result obviously rough parts. Deep, back boundaries assert english, blue police; findings will declare restaurants. Little, daily s	Sports                                            	guns                                              	2.81	10686.60	4.90068866722739088
+AAAAAAAADMJBAAAA	Complicated, right projects forget naturally british, true weapons. Employers step also as continuous tickets. Ev	Sports                                            	guns                                              	5.02	8567.83	3.92905764075860015
+AAAAAAAAEBCBAAAA	Then vague profits used to buy tonnes. I	Sports                                            	guns                                              	0.44	2445.30	1.12137199838780706
+AAAAAAAAEECDAAAA	NULL	Sports                                            	guns                                              	8.03	272.49	0.12495916895296837
+AAAAAAAAEHLAAAAA	Vital, possible communications go yet operational effects; 	Sports                                            	guns                                              	1.48	11987.62	5.49731378371310009
+AAAAAAAAEIBBAAAA	Now good properties see quite mere exceptions; long publications ought to make alone facilities. Certa	Sports                                            	guns                                              	4.20	3874.40	1.77673237253249895
+AAAAAAAAEKKBAAAA	Negative patients may not get for a eyes. Past little questions perform highly only, afraid acts. Again co	Sports                                            	guns                                              	1.13	5931.38	2.72002758099107309
+AAAAAAAAEMHCAAAA	Differences imagine up a feet. Tender methods shall complet	Sports                                            	guns                                              	93.05	1128.12	0.51733618730677336
+AAAAAAAAEMOBAAAA	Annual communications use enough in a standards; only famous conservatives used to kill new, public children. Men dance so examples. Christian patients shall cause as busy te	Sports                                            	guns                                              	2.43	22127.23	10.14716236203600213
+AAAAAAAAENIBAAAA	Courts define so. Appropriate tables surprise well to a agreemen	Sports                                            	guns                                              	7.17	131.70	0.06039532662154917
+AAAAAAAAENNBAAAA	Examples should not monitor firms. Fo	Sports                                            	guns                                              	3.84	535.99	0.24579568045470114
+AAAAAAAAEONAAAAA	New years can lend elements. Other, typical figures return under a flowers. Due, following others used to reject in full strong, lik	Sports                                            	guns                                              	0.78	4193.11	1.92288722862630256
+AAAAAAAAFJLAAAAA	Other aspects might appear quite good	Sports                                            	guns                                              	0.21	5214.14	2.39111380676146088
+AAAAAAAAFKFCAAAA	Strong chips meet to a connections; necessary, suprem	Sports                                            	guns                                              	2.74	4156.55	1.90612144926955361
+AAAAAAAAGACCAAAA	Artistic children can stay significant	Sports                                            	guns                                              	5.71	4613.16	2.11551484402024129
+AAAAAAAAGDCAAAAA	Old ideas must withdraw holy pensioners. Additional bo	Sports                                            	guns                                              	7.83	1028.06	0.47145041371715901
+AAAAAAAAGFCDAAAA	High, capital clothes can show. Prob	Sports                                            	guns                                              	28.98	231.55	0.10618479786803121
+AAAAAAAAGMDBAAAA	Settlements relocate colleagues. Well 	Sports                                            	guns                                              	0.15	9689.92	4.44362857506971716
+AAAAAAAAHCLCAAAA	Major, late transactions ought to determine interested, industrial group	Sports                                            	guns                                              	3.27	2963.68	1.35909203949698443
+AAAAAAAAHMCBAAAA	Films exclude british, young members; spots decide other, poor agents. Black, 	Sports                                            	guns                                              	7.63	834.49	0.38268258247848571
+AAAAAAAAIDNCAAAA	Badly heavy reports shall keep there important, given women. Vice versa pure plants reliev	Sports                                            	guns                                              	2.78	1558.80	0.71483853559355238
+AAAAAAAAIKACAAAA	Upwards new instructions help enough firms. Funds see then. Mines might play girls; odd difficulties bid complaints. Others go slightly at a fees. Empty awards find necessarily fi	Sports                                            	guns                                              	5.31	4316.40	1.97942587569669586
+AAAAAAAAIKAEAAAA	Political, appointed actors might not take formal resources. Possibly new programmes might not use in a waves. Racial, suspicious reader	Sports                                            	guns                                              	1.08	15990.81	7.33310700754088619
+AAAAAAAAJLEBAAAA	Golden, royal counties work then jobs. Patterns would take efficiently compl	Sports                                            	guns                                              	42.09	2480.64	1.13757830698921593
+AAAAAAAAKBECAAAA	Girls help diverse, clear workers. Classes improve no longer	Sports                                            	guns                                              	3.07	147.44	0.06761341653060903
+AAAAAAAAKJICAAAA	Social, large demands may attend subsequent, french sales. Small, able others will react in a principles. Enormous procedures could not move terms. Important members take so	Sports                                            	guns                                              	6.84	266.10	0.12202882622622805
+AAAAAAAAKLICAAAA	Wooden, english birds say so to a states; key, video-taped trends check largely ago fast ways. Urban patients promote and so on political minu	Sports                                            	guns                                              	7.33	4309.42	1.97622496924401239
+AAAAAAAALKBAAAAA	Alone, fortunate minutes can put particularly out of a consequences. Darling costs run already in a laws. Molecules discover. Temporary, political ty	Sports                                            	guns                                              	5.47	1876.47	0.86051646579755789
+AAAAAAAALKKDAAAA	Good definitions deliver a bit international childre	Sports                                            	guns                                              	4.27	10401.45	4.76992384273130321
+AAAAAAAAMABBAAAA	Suggestions go instead reasonable figures. More fat practices imagine 	Sports                                            	guns                                              	1.92	7358.08	3.37428735692853857
+AAAAAAAAMAKBAAAA	However old days hold perhaps new, gentle bones. Rules achieve also. Fine, vocational proble	Sports                                            	guns                                              	7.68	1967.40	0.90221538037384845
+AAAAAAAAMFDDAAAA	Chips ought to finish. Bottles may not clear. Right, white wives used to accommodate about a words. Courts choose well new, future rewards. Permanent tourists serve ahead polit	Sports                                            	guns                                              	5.55	2717.44	1.24617066343555491
+AAAAAAAAMGMCAAAA	Cold clients see lengthy, only spirits; numbers must not want once again tall leads; once naked lads make. Minutes lose front expenses. Probably alive p	Sports                                            	guns                                              	0.47	3757.58	1.72316075479575351
+AAAAAAAAMIIAAAAA	Right, vital dreams vary most; documents	Sports                                            	guns                                              	4.18	2652.80	1.21652788505425697
+AAAAAAAANCGEAAAA	Directly essential organisations introduce onwards atomic words. Much famous steps ma	Sports                                            	guns                                              	62.90	380.00	0.17426138281084803
+AAAAAAAAOBHBAAAA	Today keen pages wil	Sports                                            	guns                                              	8.17	1181.16	0.54165940768647699
+AAAAAAAAOILCAAAA	Possible roots must reveal at least upper, previous populations. So gr	Sports                                            	guns                                              	3.01	21554.07	9.88432116684688198
+AAAAAAAAOJOAAAAA	Unusually global cattle shall tempt great prices. Worlds would not sign certainly deposits. Contributions predict als	Sports                                            	guns                                              	4.06	1782.00	0.81719416886560838
+AAAAAAAAPGHDAAAA	In full possible products bear to a components. Lovely boards help alongside at the possibilities. True, dry papers should disagree into a c	Sports                                            	guns                                              	0.52	763.63	0.35018742041012600
+AAAAAAAAPOBEAAAA	Resources go in a records. Permanent, flat applications would work	Sports                                            	guns                                              	7.43	571.34	0.26200657488197345
+AAAAAAAAAAFDAAAA	Negative in	Sports                                            	hockey                                            	1.63	5985.40	2.60825063748619267
+AAAAAAAAABMDAAAA	Modern facilities see; certain procedures lure for a features. Still dependent companies put little persons; procedures find to a employers. Public boards know almost also tory considerations.	Sports                                            	hockey                                            	8.87	6280.74	2.73695059793581544
+AAAAAAAAAEABAAAA	Contracts will improve just by a services. Strange, educational passengers resist only english days. Difficulties should debate then impressive, linguistic applications; fine, new eyes build; roya	Sports                                            	hockey                                            	6.73	11482.83	5.00385916858448520
+AAAAAAAAAFADAAAA	Following parts treat perhaps appearances. Coming studies perform loudly so professional streets. Lesser, elderly years wear equ	Sports                                            	hockey                                            	2.07	8396.19	3.65879772779683831
+AAAAAAAAAICDAAAA	Girls would not enhance here inner authorities. Commercial others might not think normally problems. Loudly bright peasants see yellow candidates. Comfortable sessions may	Sports                                            	hockey                                            	5.75	3982.08	1.73526626433003944
+AAAAAAAAALPDAAAA	Depen	Sports                                            	hockey                                            	3.19	1800.84	0.78474990443589989
+AAAAAAAACBICAAAA	Then sophisticated numbers might not facilitate alway	Sports                                            	hockey                                            	1.14	1035.30	0.45115144935834786
+AAAAAAAACDBBAAAA	Speakers get more with a	Sports                                            	hockey                                            	37.55	4112.16	1.79195107118074348
+AAAAAAAACGHBAAAA	Public, available symptoms take somewhat in a minutes; nerves seem. Curious, certain islands contact again vital respects; mass rules might recognise primary,	Sports                                            	hockey                                            	8.68	334.35	0.14569930174148904
+AAAAAAAACMEEAAAA	Foreign children increase about so tall leaders. Available, domestic telecommunications mess subsequently primary characteristics. Cities risk businesses. Elegant views cannot use f	Sports                                            	hockey                                            	7.88	2922.03	1.27332953691545754
+AAAAAAAADGAEAAAA	All british ways trap stages. Accidents welcom	Sports                                            	hockey                                            	3.21	4828.96	2.10431015444169561
+AAAAAAAAECKAAAAA	Much catholic guests invite highest problems. Long men must assume maps. Passive applications want independen	Sports                                            	hockey                                            	5.63	10772.75	4.69442845172910449
+AAAAAAAAEKMCAAAA	Eyes must increase roughly. Services should love now senior, rapid sales. 	Sports                                            	hockey                                            	0.88	9712.50	4.23240457055245201
+AAAAAAAAELCAAAAA	International places	Sports                                            	hockey                                            	7.18	5185.35	2.25961380076336237
+AAAAAAAAEOBBAAAA	Reasonable laws shall pay significant boys. Widespread operations would not run then words. Substantial paintings make stil	Sports                                            	hockey                                            	0.88	10680.29	4.65413726752387621
+AAAAAAAAFALAAAAA	Military, special factors may adopt often young names. Actually large-scale workers make here advantages. Precious, odd customers study in the careers; usual women win then firms. S	Sports                                            	hockey                                            	3.48	7195.62	3.13562676715146818
+AAAAAAAAFIKCAAAA	Parts work only windows. Positive, vital eyes could happen without a minds; common payments must not investigate only important seeds. Here different	Sports                                            	hockey                                            	8.94	1422.63	0.61993778267233306
+AAAAAAAAGJKAAAAA	Colleagues come so; great places finish only large years. Regulations would know genuinely most other services. Opi	Sports                                            	hockey                                            	9.08	3086.02	1.34479126412522810
+AAAAAAAAHIDBAAAA	Main months answer weapons. Little, norma	Sports                                            	hockey                                            	1.15	619.92	0.27014180091396407
+AAAAAAAAICCCAAAA	Workers ought to widen late, close benefits. Final eyes restore yesterday high, public funds. Quickly educational days go perhap	Sports                                            	hockey                                            	3.55	11162.51	4.86427370325224722
+AAAAAAAAJBHCAAAA	Then suspicious authorities can advertise perhaps important massive mammals. Easy lawyers will put. Respectively responsible pounds might acknowledge ti	Sports                                            	hockey                                            	4.00	4553.02	1.98406410891291892
+AAAAAAAAJIMCAAAA	Flights might work bits. Appropriate powers ought to lie just very parental pounds	Sports                                            	hockey                                            	3.03	1200.96	0.52334091048140775
+AAAAAAAAJNLBAAAA	Little hearts must not get here. Best professional hospitals achieve there foreign shoulders. Women should not forestall certainly able deals. Projects sound years. Facilities shall find dry, 	Sports                                            	hockey                                            	47.20	1750.77	0.76293096010153065
+AAAAAAAAKFEBAAAA	As able participants arise. As red years must make often versus a models. Alone techni	Sports                                            	hockey                                            	0.13	10294.75	4.48613096038042269
+AAAAAAAAKHDAAAAA	Small regions allow so new deaths; slowly late attacks would install automatically acc	Sports                                            	hockey                                            	5.69	12283.72	5.35286205110801192
+AAAAAAAAKHEEAAAA	Interesting, complete times join secure reports. Ancient, traditional markets go lessons. Rapid terms figh	Sports                                            	hockey                                            	3.26	12950.49	5.64341962078700893
+AAAAAAAAKOBAAAAA	Reports may develop relevant, clear cells. Intently inc	Sports                                            	hockey                                            	7.52	1084.78	0.47271329009460889
+AAAAAAAALLPDAAAA	Forces trust together from the systems. Reasons exploit even mar	Sports                                            	hockey                                            	3.36	2768.45	1.20640416302146057
+AAAAAAAALMPCAAAA	Annual priests look often practical genes. Needs may n	Sports                                            	hockey                                            	0.72	2604.48	1.13495115118789706
+AAAAAAAAMBDDAAAA	Tenants shall not know so realistic years. Recommendations tell. Successful, proposed actions used to link also. Holes will not become only later previo	Sports                                            	hockey                                            	5.91	6583.03	2.86867915161739080
+AAAAAAAAMEGAAAAA	Then royal plans would afford certain, terrible days. Priests ought to care rarely 	Sports                                            	hockey                                            	4.15	6918.52	3.01487522980268214
+AAAAAAAAMEJBAAAA	Complete clubs engage to a classes; other, small estates rob sl	Sports                                            	hockey                                            	8.86	2201.70	0.95943218975395970
+AAAAAAAAMFBBAAAA	Details accompany ok. Black savings go ju	Sports                                            	hockey                                            	7.28	15049.92	6.55828573430617849
+AAAAAAAAMGAEAAAA	Issues recognise only previous 	Sports                                            	hockey                                            	75.67	4488.20	1.95581757462584454
+AAAAAAAAMHLBAAAA	Very old efforts bring sorry supporters. Almost other subjects sha	Sports                                            	hockey                                            	1.96	7640.40	3.32944801862022696
+AAAAAAAAMIKCAAAA	Too female dates will achieve also national, capable statements. Actual, small lights see then cheap effects. Free peasants used 	Sports                                            	hockey                                            	3.59	8586.28	3.74163302095681932
+AAAAAAAANACBAAAA	As national managers shall respect years. Other police could not consider. Therefore true bodies continue in the factors. Special relations would reach on 	Sports                                            	hockey                                            	3.94	1856.04	0.80880434276737946
+AAAAAAAANMPDAAAA	Tonight certain authorities hang with a cattle. Internationa	Sports                                            	hockey                                            	0.61	9094.17	3.96295564204694903
+AAAAAAAAOBHDAAAA	Psychological, ill activities talk rather right windows. Leaders would know adequately sacred, ordinary offenders; important minutes could affect again norma	Sports                                            	hockey                                            	7.66	794.92	0.34640134272571996
+AAAAAAAAOCECAAAA	British observations speak great quantities. Personal, ready th	Sports                                            	hockey                                            	1.66	274.86	0.11977541521359557
+AAAAAAAAOJFBAAAA	Late, chief standards guarantee publicly police. Also political years might come curious years. Systems may not follow so with a times. Central, silent towns must apologis	Sports                                            	hockey                                            	40.41	5501.55	2.39740389859694645
+AAAAAAAAAFGAAAAA	Columns blame rapidly. English users may not get excellent, female manufactu	Sports                                            	optics                                            	0.25	1588.38	0.64760161773605996
+AAAAAAAAAGABAAAA	Softly old women ask perhaps as a questions; relevant needs used to fall. Entries would not call together questions. N	Sports                                            	optics                                            	3.85	6270.40	2.55651744787279515
+AAAAAAAAALMAAAAA	Projects mount in general perhaps busy things. Accounts will fail. Often d	Sports                                            	optics                                            	56.35	1751.04	0.71392005484868258
+AAAAAAAAAPEBAAAA	Good duties cannot determine gifts. Today social others succeed really quick eggs. Asleep, liable observers understand more after a operations. States must wish just similar women. Questio	Sports                                            	optics                                            	4.66	2203.00	0.89818957923956490
+AAAAAAAABKPCAAAA	Solid police must lift increasingly western girls. However central days choose widely over a drivers. Able years release commonly christian, aware muscles; sometimes important	Sports                                            	optics                                            	2.47	24705.19	10.07260291018316218
+AAAAAAAACCFBAAAA	Mad, social circles could arrive increased eggs. Shareholders search very low carers. Fast, significant patients will not seize then capital memorie	Sports                                            	optics                                            	1.38	6498.54	2.64953286803063189
+AAAAAAAACEIDAAAA	Obvious eyes talk lives. Neutral, real guests must stay in a departments. Hands can drop in the rounds. Flexible, mutual margins may pass like women; large clubs try. Old, sure records would 	Sports                                            	optics                                            	6.07	1813.00	0.73918189158480761
+AAAAAAAACMJDAAAA	Circumstances join by a members. Human, personal priests will not obtain again wide, statutory days. Whole, new kids shall not encourage	Sports                                            	optics                                            	4.53	6033.35	2.45986931362007665
+AAAAAAAACODDAAAA	Nurses should see certainly eyes. Clubs shall go individual procedures. New, internal police might read too international children; healthy, sufficient years break well only new agent	Sports                                            	optics                                            	8.75	9654.45	3.93623530789351671
+AAAAAAAADIDCAAAA	Identical solicitors must maintain sources. Factors take already unusual minutes. Just various sales sell agricultural, long states. 	Sports                                            	optics                                            	3.77	1573.11	0.64137585519634677
+AAAAAAAADJDEAAAA	New hotels join increases. Agencies might not prov	Sports                                            	optics                                            	40.19	2052.76	0.83693492541071686
+AAAAAAAAEAFCAAAA	Aware, single times would ring to the men. Again double months cover that. Accurate politicians send so social hotels. Other, urban feelings upset just wild eyebrows. True, magnificent p	Sports                                            	optics                                            	3.24	642.52	0.26196312685111450
+AAAAAAAAENJAAAAA	Other, international colours s	Sports                                            	optics                                            	3.14	11101.71	4.52630060541973219
+AAAAAAAAGFADAAAA	Quick artists must hope tough teachers. Social conflicts find rapidly from a shareholders; other tools	Sports                                            	optics                                            	3.81	10100.29	4.11800963472427822
+AAAAAAAAGFECAAAA	New, able officers may believe often. Losses weep fast excellent, old hours. Able, only regulations shall not let by a countries. Dreams back a little. Sophisticated, 	Sports                                            	optics                                            	8.41	1446.65	0.58981659319424265
+AAAAAAAAGHPBAAAA	Acute, serious forms change just premises. Above causal buildings may pay so open, traditional consequen	Sports                                            	optics                                            	4.49	7490.92	3.05413812206865251
+AAAAAAAAGMKDAAAA	Ago sexual courts may attract. Important, alone observations expect. New, available ways represent years. Excell	Sports                                            	optics                                            	8.59	3179.49	1.29631628928570322
+AAAAAAAAHKEEAAAA	Bombs shall not help. Angles pull sometimes. Measures train still african pictures. Teachers wear by the motives. Attractive months shall give 	Sports                                            	optics                                            	0.92	NULL	NULL
+AAAAAAAAIGKBAAAA	Other, different problems spread importantly only likely commitment	Sports                                            	optics                                            	3.10	8596.18	3.50476590888223467
+AAAAAAAAIGNCAAAA	Possible opponents can inform also foreign, new heads. Losses face most qualifications. High difficulties will not walk results. Direct, ou	Sports                                            	optics                                            	0.27	149.24	0.06084694180922046
+AAAAAAAAIIPDAAAA	Drugs hold years. Cells might reconsider now. Wrong players meet too rapid, integrated parents. Complete, social women used to includ	Sports                                            	optics                                            	4.94	13154.62	5.36329668763339318
+AAAAAAAAIJHCAAAA	Holidays will find soon so international expectations; furious children would not talk in order reasons; there current stones shall give as firms. Central drugs ought to love european, following 	Sports                                            	optics                                            	9.08	13906.80	5.66996951455686841
+AAAAAAAAIJJCAAAA	European nights accompany however expensi	Sports                                            	optics                                            	1.37	3255.97	1.32749810454682075
+AAAAAAAAILPDAAAA	Earnings used to connect of course. Only big branches show into the men. Tiny trousers mediate. Highest proposed m	Sports                                            	optics                                            	8.81	3903.78	1.59161802798176516
+AAAAAAAAKBPAAAAA	Wild, other services change less at a hours. Inherently southern days would win almost remarkable, separate firms; strong, professional children might damage other fea	Sports                                            	optics                                            	1.25	10597.58	4.32076074496487887
+AAAAAAAAKDJDAAAA	Industrial, sexual minutes must cure crowds. 	Sports                                            	optics                                            	3.33	503.37	0.20522999931993635
+AAAAAAAAKHPAAAAA	Sad recordings will borrow most long teachers; then bold shares show markets. Common, dark skills watch really to a le	Sports                                            	optics                                            	8.63	838.35	0.34180537165478404
+AAAAAAAAKKJAAAAA	National, little grounds must not hate broadly. Teachers define abroad normally tall researchers. Cultures handle centres. Major addresses used to look 	Sports                                            	optics                                            	1.61	12110.40	4.93755564249787867
+AAAAAAAAKLKDAAAA	Excellent, difficult relations attempt. Boots dismantle really social sheets. Literary sp	Sports                                            	optics                                            	1.67	2628.08	1.07149980454285779
+AAAAAAAAKLPBAAAA	Obvious clubs should finance at leas	Sports                                            	optics                                            	5.51	1283.02	0.52310267542258128
+AAAAAAAAMAICAAAA	Alleged books ought to go altogether different databases; artists will listen years. Forward cold others check effectively. Quite numerous d	Sports                                            	optics                                            	5.42	3201.52	1.30529818507809887
+AAAAAAAAMDGBAAAA	Teams judge conscious shareholders. Else local areas imagine ea	Sports                                            	optics                                            	2.39	6080.10	2.47892985053766615
+AAAAAAAAMFPAAAAA	Tall students should encompass much true women. Rough birds ought to protect as possible families. Political, dead proceedings 	Sports                                            	optics                                            	1.06	5878.74	2.39683295826545608
+AAAAAAAAMMJCAAAA	Natural, political manufacturers must not pr	Sports                                            	optics                                            	2.60	1879.45	0.76627435528906048
+AAAAAAAANFCCAAAA	Physical, nationa	Sports                                            	optics                                            	52.14	5315.52	2.16720139457080890
+AAAAAAAANFDEAAAA	Rules share briefly ago specific subsidies. Maybe new subjects should scor	Sports                                            	optics                                            	1.12	NULL	NULL
+AAAAAAAANHEEAAAA	Exchanges see with a costs. Possible controls achieve yet high similar machines. Rights would not sum suit	Sports                                            	optics                                            	4.85	337.31	0.13752534134057995
+AAAAAAAANIBDAAAA	Legal, local prices ask central instruments. Structures cover for a parents. International tourists should 	Sports                                            	optics                                            	1.84	3778.91	1.54070702809086890
+AAAAAAAANJMDAAAA	Wings can go yellow, expected eyes.	Sports                                            	optics                                            	8.93	5543.20	2.26002926719961695
+AAAAAAAANPBCAAAA	Hot grounds shall pass. Impressive methods could change very basic voices. Concrete, desirable centres pay again in a ingredients. Positio	Sports                                            	optics                                            	1.04	2610.25	1.06423029923289799
+AAAAAAAAOBOCAAAA	Small aspects can allow obvious, redundant colours. Past, sound individuals give both; soft, religious months improve; customers use once for a fore	Sports                                            	optics                                            	0.82	1475.16	0.60144046287382504
+AAAAAAAAOJFAAAAA	Injuries answer so good issues. Aside aware definitions m	Sports                                            	optics                                            	1.71	6407.03	2.61222314111451179
+AAAAAAAAOMIAAAAA	Scenes should not learn. Magistrates produce somewhat on a businesses; extremely national values see everywhere. Northern engines shall not aim; rom	Sports                                            	optics                                            	1.88	6498.82	2.64964702739612762
+AAAAAAAAONGCAAAA	Colonies give. Even formal payments may follow comparative, frequent years. Perhaps residential messages face times. Late houses talk then conditions. Officials may includ	Sports                                            	optics                                            	76.62	15211.44	6.20188692384379802
+AAAAAAAAPBPDAAAA	Great structures should not survive even here various areas. Cultural results choose likely, female hours. Gold feelings ou	Sports                                            	optics                                            	9.72	3879.70	1.58180032254913297
+AAAAAAAAPKMDAAAA	Social cases need. Inc, right products can know states. Whole, economic years should run relatively new notes. Markets can stop just keen words. Now common services abuse only new, narrow feelings. Ye	Sports                                            	optics                                            	0.97	8141.82	3.31951787564424615
+AAAAAAAAAAEAAAAA	Only economic shares last too white patients. Ever environmental markets might come slightly w	Sports                                            	outdoor                                           	1.07	1920.21	0.69563739953531432
+AAAAAAAAADPCAAAA	Strict results wonder indeed ago possible factors; wrong tables survive for example known differences. Featur	Sports                                            	outdoor                                           	3.18	7506.80	2.71949986242738947
+AAAAAAAAAHADAAAA	Total, happy arrangements control indeed. Particularly internatio	Sports                                            	outdoor                                           	4.20	5584.92	2.02325746945009538
+AAAAAAAAAJMBAAAA	Easy, local stages may not get elected, alone pages; clean mem	Sports                                            	outdoor                                           	1.93	11116.50	4.02719137590905246
+AAAAAAAAALFEAAAA	Public questions call under way far essential taxes; 	Sports                                            	outdoor                                           	1.23	9780.48	3.54318937689479327
+AAAAAAAAALGAAAAA	Preliminary, central jobs would attend unhappily personal members; as blue duties must sound remaining, slow voices. Bad years can seem short drugs. Major problems fit more middle countries. S	Sports                                            	outdoor                                           	3.62	276.60	0.10020430302491287
+AAAAAAAAAOOBAAAA	Houses decide quite. Elements cannot assume simply; simple, cruel days could know. 	Sports                                            	outdoor                                           	7.17	NULL	NULL
+AAAAAAAABAKCAAAA	Principles take hardly perhaps financial women. Men revive so in a classes. Only domestic miles perform relations. Urgent, male developers relax major po	Sports                                            	outdoor                                           	2.50	7845.25	2.84211065909688245
+AAAAAAAABBCDAAAA	Costs use again successfully coming weeks. Processes can stress less heavy, oral issues. Personally cheap officials shall go current events. Natural parties imagine powerfully without the we	Sports                                            	outdoor                                           	4.07	3610.83	1.30810088030168523
+AAAAAAAABIDEAAAA	Ago natural taxes could protect rats. More local days shall tend closely. Proteins may intervene very perfect men. Procedures make expens	Sports                                            	outdoor                                           	8.79	12330.06	4.46682960432160944
+AAAAAAAABIKAAAAA	European	Sports                                            	outdoor                                           	29.44	11343.15	4.10930021640289375
+AAAAAAAABOEAAAAA	Numbers choose special bodies. Main pictures offset like a changes; beautiful, large elections must suspend. Electronic p	Sports                                            	outdoor                                           	5.79	6902.40	2.50054295444381268
+AAAAAAAACBKAAAAA	Yet green experiments think wonderful minutes. Scottish years may remove twice parental features. Good boundaries look please. French, e	Sports                                            	outdoor                                           	8.75	3992.78	1.44647048818442374
+AAAAAAAACFMAAAAA	Good products may say pp.. Substantial, front flats become actually. Bills tr	Sports                                            	outdoor                                           	9.06	3258.39	1.18042190503740363
+AAAAAAAADCMCAAAA	Modern personnel would keep 	Sports                                            	outdoor                                           	0.48	6309.82	2.28586809585197296
+AAAAAAAADFGCAAAA	Initial, real signals keep perfect, free sectors; just funny deposits can understand sufficiently. Entire relations shall not relate; poor views must reach probably. Years 	Sports                                            	outdoor                                           	2.66	17724.56	6.42110333052512525
+AAAAAAAADPBDAAAA	Unacceptable events must not persuade at least but for a companies; horses would try also crude skills. Turkish, new animals go further scottish lands. European elements believe 	Sports                                            	outdoor                                           	9.19	702.52	0.25450298973630437
+AAAAAAAAEDGAAAAA	Eyes should jump rapidly closer explicit things. Green, radical children could ensure middle consumers. Likely minutes think very pa	Sports                                            	outdoor                                           	2.37	8733.77	3.16399615195189179
+AAAAAAAAEDNCAAAA	So competent candidates would enter suddenly almost cold situations; eyebrows could read enough rational sales. Impossible 	Sports                                            	outdoor                                           	0.33	2072.27	0.75072440719246635
+AAAAAAAAEHHCAAAA	However subsequent steps share terribly existing communications; less great responsibilities speed at all long-term mountains. Of 	Sports                                            	outdoor                                           	4.39	3486.57	1.26308502096012459
+AAAAAAAAEIPBAAAA	Industries give much proposals. Possible, strong goals ought to live most total criteria. The	Sports                                            	outdoor                                           	96.84	5462.95	1.97907121189424352
+AAAAAAAAEJIBAAAA	Only single galleries discover in the countries. Clean front products ought to shoot even. Ready, educational questions ought to sense shortly tests. Sciences stop. Upright variou	Sports                                            	outdoor                                           	1.53	1332.46	0.48271231239542806
+AAAAAAAAELICAAAA	Economic elements used to hear as 	Sports                                            	outdoor                                           	0.40	396.48	0.14363341309948465
+AAAAAAAAEMBCAAAA	Social, joint functions should suit. Best absolute goods might not lose still western wonderful hundreds. Inches feel certain years. Diverse lives put breasts; very good police shall 	Sports                                            	outdoor                                           	5.91	1973.74	0.71502979411565989
+AAAAAAAAEOIAAAAA	Trees work	Sports                                            	outdoor                                           	3.30	8407.66	3.04585578586565052
+AAAAAAAAFHKAAAAA	Steps cannot stay only able transaction	Sports                                            	outdoor                                           	6.89	702.30	0.25442329000143278
+AAAAAAAAGLMAAAAA	Stars divorce there s	Sports                                            	outdoor                                           	2.51	7314.38	2.64979157613652275
+AAAAAAAAGMCAAAAA	Original women shall know here necessarily national goods. Accounts will make as. Independent members will find a little dreams. Short jobs assist widely new moments. Ago passive represen	Sports                                            	outdoor                                           	9.83	5957.43	2.15820723416379853
+AAAAAAAAGNEDAAAA	Distinctive things used to pick today symbolic pictures. Helpful lips know still. Concerned theories must accommodate very in the ph	Sports                                            	outdoor                                           	27.94	9643.98	3.49373931412219527
+AAAAAAAAHIEAAAAA	Even short boards can expel anywhere secure charming details. Specia	Sports                                            	outdoor                                           	6.91	8327.04	3.01664945575043550
+AAAAAAAAIDAAAAAA	Ideas form on the needs. Firstly rough operations might begin worldwide obvious activities. Twins	Sports                                            	outdoor                                           	4.30	2362.14	0.85573605331622446
+AAAAAAAAIDADAAAA	Creative teachers may close concerned, foreign parts. Alone desirable fires put pupils; areas begin behind a countries. Kindly able rates lead employers. Songs point thoroughly; large, acute others sa	Sports                                            	outdoor                                           	2.27	10905.96	3.95091872963694416
+AAAAAAAAIMEBAAAA	Obviously base children must seem most for a years. Just available	Sports                                            	outdoor                                           	5.16	5010.90	1.81530637030924041
+AAAAAAAAIMNCAAAA	Always small authorities make after a nations; forms will retrieve now. Financial, giant words render american, sensitive activities. Written eggs might not grant now really existing entries; grounds	Sports                                            	outdoor                                           	6.44	4934.08	1.78747667197817097
+AAAAAAAAJNIBAAAA	Apparently realistic minutes see. Ful	Sports                                            	outdoor                                           	2.79	3360.22	1.21731201413728388
+AAAAAAAAJPEAAAAA	Less social teeth play instead as social children. Advances mean very now slow bases. Small fit managers must think about sites; full, civil weap	Sports                                            	outdoor                                           	96.73	8555.01	3.09923649465350631
+AAAAAAAAKFACAAAA	Moreover overall miles say. Leaves may order faintly sure trees. Political, certain drinks protect to a parents. New minutes remember satisfied, exciting feet. Cri	Sports                                            	outdoor                                           	5.71	3006.51	1.08917295403987994
+AAAAAAAAKHGDAAAA	Alone healthy sales might meet far other roots. French groups look up to a workers. Fully average miners would walk inadequate considerations. Small, sure goods may admire more app	Sports                                            	outdoor                                           	0.48	1427.56	0.51716433415128205
+AAAAAAAAKJBCAAAA	True champions get all the same police. Especially clear issues move further great homes. Better environmental sessions burn. Bonds shall test already elderly areas. Imperial, close schools press	Sports                                            	outdoor                                           	1.71	724.38	0.26242224521036292
+AAAAAAAAKMNAAAAA	Public, great addresses must prefer thick maybe dangerous problems. Public pages may shoot now injuries. Flat groups know rather special responsibilities; nuclear months can see dou	Sports                                            	outdoor                                           	9.74	6478.02	2.34680216587652229
+AAAAAAAALEDEAAAA	Quite significant levels move chiefly dirty, actual beliefs. Away significant views bury. Practical proceedings build a bit. Funds think about prime s	Sports                                            	outdoor                                           	9.44	3562.95	1.29075531982145086
+AAAAAAAAMAHBAAAA	Independent, different attitudes include greatly other, bottom waters. Twin others should exert. Extraordinary, bottom tables could go only results. Good, early pupils shall say per	Sports                                            	outdoor                                           	98.21	5097.92	1.84683123816617431
+AAAAAAAAMFEEAAAA	Theories must not	Sports                                            	outdoor                                           	0.92	453.25	0.16419956741157541
+AAAAAAAAMFKCAAAA	Great, possible children used to	Sports                                            	outdoor                                           	4.00	8014.65	2.90347945494800407
+AAAAAAAAMJBEAAAA	Truly growing visitors shall not receive open, personal times. Large societies	Sports                                            	outdoor                                           	12.35	2130.34	0.77176151448334375
+AAAAAAAAMNBAAAAA	So	Sports                                            	outdoor                                           	2.12	6574.51	2.38175774504815585
+AAAAAAAAMNFEAAAA	Very major companies would not remedy ever future, clear movies. Famous, equal fees know open, active rights. Original hours apply so. Social, technical rates could 	Sports                                            	outdoor                                           	3.18	1551.09	0.56191573528167788
+AAAAAAAAMOJDAAAA	Social thousands choose especially blue claims. Social, right professionals can go tons. General projects must ma	Sports                                            	outdoor                                           	0.64	1598.82	0.57920695503359072
+AAAAAAAAOBJCAAAA	Prominent, regional tonnes ought to replace extremely. Women could make very young, equal hours. Q	Sports                                            	outdoor                                           	4.73	NULL	NULL
+AAAAAAAAOELDAAAA	Most whole councils arise already so social customers. More sc	Sports                                            	outdoor                                           	2.11	1583.53	0.57366782346001546
+AAAAAAAAOGCAAAAA	Various pockets can get. Areas conduct photographs. Ever 	Sports                                            	outdoor                                           	1.85	1513.96	0.54846459366448694
+AAAAAAAAOHACAAAA	Scientific risks would use. Quiet minutes imagine times; arms cut inner appeals. Areas happen straight in a changes. Fears kick very currently silent 	Sports                                            	outdoor                                           	4.22	474.41	0.17186523282013346
+AAAAAAAAOKHAAAAA	Clothes realise almost necessary females. Foreign, cultural others may give bad ya	Sports                                            	outdoor                                           	7.21	4335.56	1.57064992054479841
+AAAAAAAAOKIDAAAA	Heavy years could come much through a genes. Dealers come so sincerely educational characters. Studies must handle	Sports                                            	outdoor                                           	2.12	7347.30	2.66171755464548924
+AAAAAAAAOKOCAAAA	Various, personal benefits must not remember at le	Sports                                            	outdoor                                           	0.34	6983.49	2.52991955217443519
+AAAAAAAAONOAAAAA	Losses try a little cho	Sports                                            	outdoor                                           	4.86	1698.82	0.61543410724794823
+AAAAAAAAPKLBAAAA	Industr	Sports                                            	outdoor                                           	8.35	1902.72	0.68930127061302319
+AAAAAAAAPOPCAAAA	Nearly cultural sheets might decide to a years. Loudly new marks create lives. Local, new arrangements must not face b	Sports                                            	outdoor                                           	1.39	431.65	0.15637450253327419
+AAAAAAAAAGBBAAAA	Also religious bits might hear so extensive western talks. Sometimes complete settings mean also minutes. Other, available theories admit both just old years. Considerable seconds will prepare che	Sports                                            	pools                                             	0.62	10914.03	4.26659608077049963
+AAAAAAAAANEDAAAA	Other sports take prime tables; sources think in a priests. Fine, key eyes keep always important branches. Still local effects shall get much; black, final metho	Sports                                            	pools                                             	2.25	1716.96	0.67120713492996785
+AAAAAAAAAOJAAAAA	Factors would impose that is free, liable thoughts; significant wives buy useful sports; russians make nearly outstanding animals. Problems write. Finally per	Sports                                            	pools                                             	2.04	10920.36	4.26907065278388765
+AAAAAAAAAPEDAAAA	Popular systems associate evenly public rights. Unlike mothers experiment around languages. Chea	Sports                                            	pools                                             	8.52	3232.70	1.26375180848016674
+AAAAAAAABDBCAAAA	Subsequent feet can accept regardless. Individual, following arms hold prime officials. Assistant acids might not get however necessary times. Sometimes new times shall not take about. Small	Sports                                            	pools                                             	1.90	9375.14	3.66500143216343934
+AAAAAAAABNOAAAAA	Bonds will set ever into the nations. Distinguished, philosophical employees may not include. General, existing tiles must continue only quiet missiles. Small ve	Sports                                            	pools                                             	12.34	9502.98	3.71497762271502301
+AAAAAAAACAIDAAAA	Western products become grea	Sports                                            	pools                                             	8.19	12699.99	4.96477722342934165
+AAAAAAAACGOBAAAA	Very old circumstances explore fairly upon a lines. Crucial, active looks mean alone bloody recordings; poor bacteria could not transfer both at a properties. States could not understand really at a 	Sports                                            	pools                                             	3.35	2713.46	1.06076653640566500
+AAAAAAAACIOCAAAA	Years ought to know then. Associated, simple activities would not indicate now for a brothers. Workers get organizations. S	Sports                                            	pools                                             	20.43	4211.26	1.64629796794635660
+AAAAAAAACJCBAAAA	Supreme injuries could think conditions. Basic, eventual c	Sports                                            	pools                                             	9.13	3177.04	1.24199277557887491
+AAAAAAAADHCBAAAA	Able systems merge from a areas. Most chief efforts must find never for the time being economic directors. Activities sit there. Available polic	Sports                                            	pools                                             	3.10	4811.17	1.88081937340474643
+AAAAAAAAECEBAAAA	Carers get m	Sports                                            	pools                                             	5.77	4684.53	1.83131229603105623
+AAAAAAAAEEFBAAAA	Privileges cut perhaps reasons. Ideas finish times. Women envy general programmes. Hands shall unveil never to a facilities; official proposals conform. Scot	Sports                                            	pools                                             	7.52	8558.76	3.34585591868955110
+AAAAAAAAEIJBAAAA	Central, clear awards announce. Single, very proposals help dry maps. New questions	Sports                                            	pools                                             	2.90	2934.22	1.14706772403213253
+AAAAAAAAFBDEAAAA	Able troubles dust into the styles. Independent feet kill wounds. Fundamental months should exploit arms. Massive years read only modern courses; twin forms shall become products. Even h	Sports                                            	pools                                             	6.81	6802.61	2.65932832922487921
+AAAAAAAAFICBAAAA	Far good grounds change clearly rocks. Growing,	Sports                                            	pools                                             	1.99	5753.89	2.24935468595785151
+AAAAAAAAFPBBAAAA	Secret, familiar questions ought to influence historical values. Central, net investors can hope. So chief arrangements shoul	Sports                                            	pools                                             	6.13	4628.51	1.80941252917639637
+AAAAAAAAGCFEAAAA	Fine, high letters see now suddenly prime forces. Things used to know temporary men. Late, special methods provide fr	Sports                                            	pools                                             	2.85	2565.78	1.00303434131290940
+AAAAAAAAGCPAAAAA	Directors could involve. No longer local patients see waste lovers. Only direct aims canno	Sports                                            	pools                                             	60.43	1100.10	0.43005950583383284
+AAAAAAAAGEKDAAAA	Similarly direct changes can alienate men; ways surrender forms. Players must develop deep. Social, serious thousands walk. Thanks will not say organisations. Natur	Sports                                            	pools                                             	3.39	3166.29	1.23779030336024597
+AAAAAAAAGJEBAAAA	Simple, environmental rights ought to detail thick disabled days; also old drinks move to a conditions. 	Sports                                            	pools                                             	8.46	825.24	0.32260913243733498
+AAAAAAAAGNDDAAAA	Previous, significant flats give all formally co	Sports                                            	pools                                             	2.82	6467.74	2.52841838765722572
+AAAAAAAAGNECAAAA	Dangerous, other ladies may know neatly. Effortlessly growing services might encourage in the citizens. Banks use secondly other, similar responses. Indirect branches shall not buy i	Sports                                            	pools                                             	4.74	1246.28	0.48720530945422161
+AAAAAAAAGOJDAAAA	Literary, sensitive pages could not know now; very public program	Sports                                            	pools                                             	3.36	2399.19	0.93790970439184930
+AAAAAAAAGPKAAAAA	Christian, red laboratories prevent; shoes allow most to a positions. Now religious passengers will not know always in a elections. Southern ages abandon northern terms. Thoughts go as	Sports                                            	pools                                             	2.22	6752.13	2.63959430154149417
+AAAAAAAAHNMCAAAA	Things used to reappear. Good powers lead. Rare, traditional months may pay too. Shows tend anywhere extra pp.; canadian, proper questions can investigate only small, certain countrie	Sports                                            	pools                                             	4.95	478.95	0.18723479712672870
+AAAAAAAAIEGBAAAA	Like records start clear, likely un	Sports                                            	pools                                             	0.52	127.98	0.05003092042233790
+AAAAAAAAIHIAAAAA	Problems might introduce therefore now public details. Early future children shall annoy ever sharp services; civil lines must fly. Finally other serv	Sports                                            	pools                                             	4.38	3165.54	1.23749710762406255
+AAAAAAAAJNFEAAAA	Exclusive, different friends find for the features. Procedures comprehend totally ey	Sports                                            	pools                                             	3.90	7853.37	3.07009946489432581
+AAAAAAAAKGFCAAAA	Direct, different traders woul	Sports                                            	pools                                             	4.53	3602.83	1.40844585892492317
+AAAAAAAAKIGBAAAA	Southern hours see	Sports                                            	pools                                             	7.73	2352.82	0.91978238934274937
+AAAAAAAAKJEAAAAA	Unable centuries may think away individuals. True, additional feet appear generally recent, pri	Sports                                            	pools                                             	3.10	741.45	0.28985330479092388
+AAAAAAAAKOABAAAA	Basic levels look early, video-taped rights. Employees might not prevail later. Causal, permanent arms could not know here public vessels	Sports                                            	pools                                             	13.28	4827.92	1.88736741151284270
+AAAAAAAALGOAAAAA	Thus aware parties would conduct either at the poems. Things plan. Instead old organizations should show rather necessary, b	Sports                                            	pools                                             	77.38	4657.72	1.82083152578161976
+AAAAAAAALMEBAAAA	Thoughtfully fine 	Sports                                            	pools                                             	4.43	6849.91	2.67781920698684657
+AAAAAAAAMANAAAAA	Types can scratch like a 	Sports                                            	pools                                             	9.69	3733.27	1.45943846136194267
+AAAAAAAAMGHDAAAA	Only sexual functions would avoid special pati	Sports                                            	pools                                             	8.64	4120.56	1.61084083025057563
+AAAAAAAAMJACAAAA	Still male versions will get in a colonies. Wide wages would com	Sports                                            	pools                                             	1.46	5664.01	2.21421810893363108
+AAAAAAAAMLDDAAAA	Then available police rememb	Sports                                            	pools                                             	0.40	1103.32	0.43131829286118030
+AAAAAAAAMLMAAAAA	Pressure	Sports                                            	pools                                             	5.42	3879.88	1.51675236387107660
+AAAAAAAAMPDCAAAA	Consumers remind related, slight customers. Large purposes like with a systems; types must go programmes. Main followers shall reduce al	Sports                                            	pools                                             	15.70	1464.58	0.57254481506600755
+AAAAAAAANHHBAAAA	Final holes agree really probably clear children. So good feet must imply birds. Newly british forces ought to raise nevertheless supreme, fine problems. Necessarily good units may push only 	Sports                                            	pools                                             	2.20	1319.87	0.51597367508853827
+AAAAAAAANNFAAAAA	Men make only. Flat, distant depths would assert local,	Sports                                            	pools                                             	7.24	10909.61	4.26486818056525871
+AAAAAAAAOCGAAAAA	Apparently other offenders should approach	Sports                                            	pools                                             	0.36	15958.64	6.23867360438145453
+AAAAAAAAODLBAAAA	Workers relieve fast quite female photographs. Other, automatic shares want away right games. 	Sports                                            	pools                                             	1.82	3069.94	1.20012442445188328
+AAAAAAAAOHDCAAAA	Here ready critics stay services. Excellent years ought to 	Sports                                            	pools                                             	55.17	2208.60	0.86340280391291993
+AAAAAAAAOHOAAAAA	Never future depths c	Sports                                            	pools                                             	23.19	4555.50	1.78087090157806155
+AAAAAAAAOLIBAAAA	Real ships suspend for instance worth the arms; ago econo	Sports                                            	pools                                             	3.46	38.42	0.01501944024555573
+AAAAAAAAOODDAAAA	Famous, busy shoes will not secure. Dark, extraordinary thousands might not look then. Numbers ought to e	Sports                                            	pools                                             	6.47	7750.63	3.02993555831368042
+AAAAAAAAPJDBAAAA	Massive, military measures must get standards. Services make as well fine 	Sports                                            	pools                                             	0.51	10656.29	4.16583838871194852
+AAAAAAAAPLCDAAAA	Critics shall not print still black parents. Multiple, accessible responses exclude against a areas. Mo	Sports                                            	pools                                             	6.14	4995.43	1.95285170187028778
+AAAAAAAAPLIDAAAA	Forces eliminate away. New, large characteristics should reconsider right, used firms. Peculiar principles establish degrees. More growing arts may not say about. Actual animals move here	Sports                                            	pools                                             	2.65	1461.99	0.57153231245705415
+AAAAAAAAPOBBAAAA	Senior disputes can bring tonight controversial houses. Heavy, real examples should not offer nearly free effects. Worlds will not add. Agricultural, rare defendants draw maybe possibl	Sports                                            	pools                                             	3.45	7092.42	2.77262307096263314
+AAAAAAAAAFCEAAAA	Free plans ca	Sports                                            	sailing                                           	0.98	6984.42	2.34770798957927730
+AAAAAAAAAOFBAAAA	Special thousands take so reforms. Finally reliable acids used to go pale, small days; great, foreign judges show vice versa fair, true arrangements	Sports                                            	sailing                                           	0.90	11949.72	4.01671908579886112
+AAAAAAAABAFEAAAA	References should make private women. Additional, northern values ar	Sports                                            	sailing                                           	0.63	14040.42	4.71947652218060722
+AAAAAAAABFJBAAAA	More critical photographs balance just now serious values. Scottish, practical views suppl	Sports                                            	sailing                                           	5.19	2863.69	0.96258642703020159
+AAAAAAAABLHDAAAA	Quite british tonnes could buy successfully surprising processes; local interests used to suggest suddenly other solicitors. Shares return just real, royal companies. Crucial, old groups study. Child	Sports                                            	sailing                                           	95.70	6541.62	2.19886741329868364
+AAAAAAAACDEBAAAA	Then other rates may make more at once above councils. Camps could give 	Sports                                            	sailing                                           	0.61	8648.26	2.90698284151853421
+AAAAAAAACEEDAAAA	Scottish, british colleagues enable about a workers. Most good persons could read with a years. Indeed specific damages believe organisations. Immediate facilitie	Sports                                            	sailing                                           	1.74	7276.84	2.44600058514380124
+AAAAAAAACLDEAAAA	Easy, natural leaves contin	Sports                                            	sailing                                           	1.73	12739.66	4.28224556463149924
+AAAAAAAACLFBAAAA	New routes cannot test over a others. Armed, brown fans make so in a techniques. Electronic, subsequent professionals used to follow in a matters. Enough substantial standards	Sports                                            	sailing                                           	3.07	5349.42	1.79812727092803377
+AAAAAAAACNFDAAAA	Open times ought to add actually soviet attitudes. Women must imagine of course inner streets. Rightly big records enable yesterday st	Sports                                            	sailing                                           	6.43	2470.80	0.83052234840580583
+AAAAAAAACOPDAAAA	External, definite securities might know then particular others; always local years must buy right children. British effects used to enable powerful, 	Sports                                            	sailing                                           	5.35	NULL	NULL
+AAAAAAAADFOAAAAA	Important, broad investors can see dearly vulnerable troops. Eastern, poor lists need genuine facilities. Figures meet equally children. Other, defensive changes go old, new companies; 	Sports                                            	sailing                                           	71.43	17348.99	5.83160268628332577
+AAAAAAAADOIAAAAA	Young, black boys spread too wealthy, major numbers. Profitable drawings might think better purposes. Industr	Sports                                            	sailing                                           	3.24	12918.54	4.34237339273690257
+AAAAAAAADOODAAAA	Joint texts take only local, asleep shareholders. Detailed courses fast programmes. Soft students know settlements; just b	Sports                                            	sailing                                           	4.70	1007.64	0.33870306748730216
+AAAAAAAAEAGEAAAA	Only american aspirations will not provide then on a subjec	Sports                                            	sailing                                           	9.32	2524.02	0.84841145289915090
+AAAAAAAAECFAAAAA	Equal songs will overcome slight contracts. Large, inner odds go even good women. Feet could not find hard strong models. Bloody machines see dark heads. Huge, only men make at the advis	Sports                                            	sailing                                           	2.07	2504.57	0.84187362722467586
+AAAAAAAAEJPBAAAA	Prisoners raise both. Medical children sell; activities 	Sports                                            	sailing                                           	1.25	8453.80	2.84161803017362852
+AAAAAAAAELBEAAAA	Benefits may hold	Sports                                            	sailing                                           	8.02	5687.08	1.91162661371688936
+AAAAAAAAEMLCAAAA	Ethnic positions must buy years. Other efforts should get; common goods show exactly aware eyes; foreign, unfair fans may carry thus daily, national actions.	Sports                                            	sailing                                           	4.63	4728.78	1.58950844693799842
+AAAAAAAAENACAAAA	Criteria shall announce far about other waves. Farmers see possibly; just english managers clean. Head files see both. Comparisons may n	Sports                                            	sailing                                           	4.18	1308.47	0.43982255836916981
+AAAAAAAAEOEBAAAA	Connections present high secondary benefits. Levels could compete. Psychological students ought to wonder advanced seats. Of course rich functions would see items; unlikely id	Sports                                            	sailing                                           	9.39	2534.25	0.85185011390942748
+AAAAAAAAEOJBAAAA	Well bad areas seem	Sports                                            	sailing                                           	0.39	2413.53	0.81127189717818704
+AAAAAAAAEPOCAAAA	Blue, united ministers know childr	Sports                                            	sailing                                           	4.68	530.93	0.17846415348838210
+AAAAAAAAFGBBAAAA	Dear, continuous problems	Sports                                            	sailing                                           	5.90	8982.06	3.01918470322237831
+AAAAAAAAGBNDAAAA	Prices acquire more out of a christians. Efficiently local prices 	Sports                                            	sailing                                           	2.11	8027.95	2.69847494207721747
+AAAAAAAAGOPCAAAA	Good, capable studies might like bad apparently new years. Modest, payable plants could feed there english women. New, local recommendations last public novels. Candidates must save as orange pla	Sports                                            	sailing                                           	4.28	1617.69	0.54376222186845881
+AAAAAAAAHHHBAAAA	Mothers may not obtain p	Sports                                            	sailing                                           	9.99	205.80	0.06917658220087212
+AAAAAAAAICFEAAAA	British figures can tell much white methods. New, french men could think marginally nuclear relatives. Electronic, differ	Sports                                            	sailing                                           	7.39	13316.13	4.47601730584304808
+AAAAAAAAIEJCAAAA	Real appearances could join miles. A	Sports                                            	sailing                                           	2.44	1182.16	0.39736534700963551
+AAAAAAAAIJIDAAAA	At present financial areas used to link very purposes. Likely members can retaliate true, blac	Sports                                            	sailing                                           	1.69	7800.18	2.62191347401165555
+AAAAAAAAIKLCAAAA	Special birds will not answer especially then public walls. Most human areas could require major groups. Particularly diverse children could continue to the readers	Sports                                            	sailing                                           	4.71	7976.59	2.68121104867664997
+AAAAAAAAIPPAAAAA	Students would rise broad obligations. Good, statistical children would not see. Gradually elegant cases can look w	Sports                                            	sailing                                           	4.63	391.82	0.13170441417855061
+AAAAAAAAJBADAAAA	Reliable stages cannot see similarly. Feelings repeat together significant, available notes. Rich, basic roots provide instinctively before the talks. Parties arrest there other investigations. Bom	Sports                                            	sailing                                           	7.89	7983.29	2.68346315315063365
+AAAAAAAAJKOBAAAA	Demands can imagine also purely fresh eyebrows. Busy skills become almost; complete pa	Sports                                            	sailing                                           	4.98	12443.47	4.18268574013161433
+AAAAAAAAJNPAAAAA	Proper applications stand now very limited arms. Angrily slow boys shall aid too previous old masses. Mechanical contents think through the times. Sequences may not agree. Old, working stren	Sports                                            	sailing                                           	0.63	679.89	0.22853482250996573
+AAAAAAAAKNHBAAAA	Successful, able hearts cite then contents. Urban rights will use long important, suspicious ideas; police speak for a methods. Plans seek no longer good gardens	Sports                                            	sailing                                           	4.39	8675.35	2.91608873856334289
+AAAAAAAAKNNBAAAA	Scientific packages make banks. Then important parents must get front, little bact	Sports                                            	sailing                                           	4.23	6135.42	2.06232937787597103
+AAAAAAAALGNBAAAA	Also long ways should not give only now good resources. Previous, economic units s	Sports                                            	sailing                                           	4.65	389.74	0.13100525338662731
+AAAAAAAALPEBAAAA	Social years attend. Bloody wee	Sports                                            	sailing                                           	1.94	3178.08	1.06826390845941533
+AAAAAAAAMFBCAAAA	Capital, foreign problems 	Sports                                            	sailing                                           	3.60	1277.78	0.42950657533834004
+AAAAAAAAMLMCAAAA	Original, major nations should come once more now permanent feet. Prizes revise thus with the spots. Aside ordinary studies can learn l	Sports                                            	sailing                                           	1.46	7468.82	2.51053178169833686
+AAAAAAAAMOMDAAAA	Industrial, open sites would throw before a men. Also p	Sports                                            	sailing                                           	7.20	1089.19	0.36611487642064095
+AAAAAAAANJDDAAAA	Loose patients used to look at all companies. Old, low centres may illustr	Sports                                            	sailing                                           	6.35	7701.71	2.58881426094401766
+AAAAAAAAOGBEAAAA	Especially moral students used to keep guilty, bizarre things. Unknown trends reduce later terms; general mothers can find as right n	Sports                                            	sailing                                           	3.35	12086.74	4.06277630296680815
+AAAAAAAAOIKCAAAA	Origins would come sales. Educational eyes could invite actually stupid, forei	Sports                                            	sailing                                           	3.77	9292.44	3.12351428331716300
+AAAAAAAAOKFDAAAA	Legal, secondary sales elect. Big years appeal low with a doubts. Military videos might describe; comparable, long companies would not extend now industrial tools. Even ne	Sports                                            	sailing                                           	5.45	1828.50	0.61462284039987695
+AAAAAAAAOPACAAAA	Additional organisations will adopt usually schemes. Conventional problems should not create attacks. Generally european powers win very human, busy months; fai	Sports                                            	sailing                                           	0.87	6498.29	2.18430268391693540
+AAAAAAAAOPGCAAAA	Wrong, local indians train excellent, comprehensive holidays. Meals c	Sports                                            	sailing                                           	60.65	1510.40	0.50769829813506926
+AAAAAAAAOPLAAAAA	National shareholders learn. Effective proceedings will develop now other, informal days; new, big waves last americans. Solicitors ought to sue flames; interested conservatives might understand just	Sports                                            	sailing                                           	0.24	5784.43	1.94434935558887624
+AAAAAAAAPHAAAAAA	Ambitious exceptions appoint. V	Sports                                            	sailing                                           	7.35	9044.55	3.04018977912972767
+AAAAAAAAPNIBAAAA	Proceedings mi	Sports                                            	sailing                                           	7.11	4105.60	1.38003584005782598
+AAAAAAAAABOBAAAA	Again standard families change literally. Narrow lips work certainly carefully vast stages. Drugs see also right factors. Financial, western examples ought to let desperately ago sudden	Sports                                            	tennis                                            	9.39	6556.29	1.81601129267527792
+AAAAAAAAACFDAAAA	Late global concepts shall understand very quiet, upper heads. Already english buildings make women. Others try. Please minimal agreements conflict largely forthcoming police. 	Sports                                            	tennis                                            	4.33	7426.08	2.05693237186122454
+AAAAAAAAACPDAAAA	Seriously complete characteristics make forward in a projects. Industries should rise then also new departments. Physical babies encourage under to a workers. Personal, beautiful ministers cont	Sports                                            	tennis                                            	0.82	14172.38	3.92557408596710262
+AAAAAAAAAJOCAAAA	Whole, new meetings may last; free plans broaden there mammals. Public, honest areas may risk on a profits. Good, normal generations ought to walk almost over a reductions. Otherwise basic s	Sports                                            	tennis                                            	4.88	8723.48	2.41629613568450044
+AAAAAAAAAKOCAAAA	Economic, content activit	Sports                                            	tennis                                            	5.07	16087.57	4.45605804375706699
+AAAAAAAAAPDCAAAA	Women would come fair unaware, current bars. Villages may go then on a neighbours. Early numbers should not change however cr	Sports                                            	tennis                                            	2.92	13912.86	3.85369025369685708
+AAAAAAAABAPAAAAA	Women should leave also annual, marginal techniques; intellectual, appropriate factors could think profits. Neverthe	Sports                                            	tennis                                            	8.24	23633.13	6.54608489881669218
+AAAAAAAABFHAAAAA	Of course equal nee	Sports                                            	tennis                                            	3.49	11949.65	3.30990534944566741
+AAAAAAAABJPAAAAA	Free representatives can fall much prime, useful banks. Recent, secondary practitioners can talk times; libraries take from now on young prices. Bodies appear only yellow rates. Second	Sports                                            	tennis                                            	6.85	7304.83	2.02334762054045053
+AAAAAAAABMJDAAAA	Costly offices collect officially for a debts; readers greet. Women get by a write	Sports                                            	tennis                                            	3.22	2864.47	0.79342278446035080
+AAAAAAAACKLDAAAA	Rapidly main banks shall not bring extremely decades. For example main clothes might not see less. Certainly co	Sports                                            	tennis                                            	3.15	5004.38	1.38615140465694887
+AAAAAAAACNIDAAAA	Just able pounds should join then successful modern pieces. Associated, sorry clubs pay close issues. Resources will e	Sports                                            	tennis                                            	7.67	7567.71	2.09616213128028617
+AAAAAAAADHGDAAAA	Necessary times believe probably. Cruel traders know ho	Sports                                            	tennis                                            	92.95	7731.85	2.14162688247032202
+AAAAAAAADLEBAAAA	Funny, armed savings go yet thin	Sports                                            	tennis                                            	3.97	3362.82	0.93145957473422897
+AAAAAAAADPICAAAA	Elected, marvellous advisers may not pass all in a programmes. Directly soviet studies could not stress more than; convenient, public	Sports                                            	tennis                                            	4.67	18.70	0.00517966886349257
+AAAAAAAAEAGBAAAA	Men could remove only; economic, clear children raise public, extensive poli	Sports                                            	tennis                                            	5.04	2721.49	0.75381909172761457
+AAAAAAAAECHBAAAA	Able, common villages read. Only social grounds remember e	Sports                                            	tennis                                            	2.08	2677.23	0.74155961879188295
+AAAAAAAAEIEDAAAA	Successful parties see once on a ideas. Scottish, natural men would not examine regulatory, multiple payments. Steadily loc	Sports                                            	tennis                                            	2.55	8031.03	2.22449604453340795
+AAAAAAAAEODCAAAA	Current, 	Sports                                            	tennis                                            	0.47	18310.05	5.07165753336856247
+AAAAAAAAFAODAAAA	Years may speak to a	Sports                                            	tennis                                            	2.02	3056.11	0.84650469574375807
+AAAAAAAAFLGDAAAA	Separate, comfortable consumers get. Tests work even high, different faces. Hars	Sports                                            	tennis                                            	8.09	11878.41	3.29017274998923903
+AAAAAAAAGIIBAAAA	Much critical possibilities might ensure; hence northern ways may persuade much japanese, running notes. Small, ed	Sports                                            	tennis                                            	8.53	8171.42	2.26338233927916847
+AAAAAAAAGLPDAAAA	As specific ears worry also labour components. Duly proper articles would attend more easy shapes; years wait head convention	Sports                                            	tennis                                            	0.85	11273.32	3.12257029904748936
+AAAAAAAAGMEBAAAA	Early, experimental factors mean usually suitable detectives; just black assets must not store only. So british employers must see elaborate, complete pages. Mental years should t	Sports                                            	tennis                                            	88.56	15092.59	4.18046088194969605
+AAAAAAAAHNOAAAAA	Social, substantial orders would not offset however to a colleagues. Small students give for sure husbands. Subjects shall not make generations; acceptable lights g	Sports                                            	tennis                                            	56.30	5682.58	1.57400442194147617
+AAAAAAAAIDKCAAAA	I	Sports                                            	tennis                                            	1.04	4973.48	1.37759248658839698
+AAAAAAAAIFCBAAAA	Automatic amounts may find more in a regulations. Boys can give available, current seasons; here complicated camps may spot even generous open individuals. Channels remain currently protest	Sports                                            	tennis                                            	8.43	3330.22	0.92242977767808685
+AAAAAAAAIHODAAAA	Points used to find cool typical managers. However military horses understand indeed inc periods. Coloured developments could make very roots. 	Sports                                            	tennis                                            	8.52	11481.61	3.18026405453288334
+AAAAAAAAIMKBAAAA	Sides express even new women. Also joint markets should switch authorities. Trees would play always about a skills. Teams deprive future pubs; ways recover national, old days. Rea	Sports                                            	tennis                                            	90.25	3634.02	1.00657862263685918
+AAAAAAAAKADEAAAA	Secret children will start in short familie	Sports                                            	tennis                                            	38.21	13612.04	3.77036683190456646
+AAAAAAAAKEBCAAAA	Other, general countries keep successfully teachers. Major, traditional relationships could not become in a subjects. Constant observers wil	Sports                                            	tennis                                            	99.16	7979.51	2.21022564133302628
+AAAAAAAAKKMAAAAA	Upper, industrial years shall opera	Sports                                            	tennis                                            	1.58	369.36	0.10230815462136981
+AAAAAAAALBGEAAAA	Afraid, spanish matt	Sports                                            	tennis                                            	3.06	141.37	0.03915774263272431
+AAAAAAAALBKDAAAA	Light, social animals resist instead then female societies. Also informal minutes shall not implement. Servants win. Hands will a	Sports                                            	tennis                                            	8.30	3341.21	0.92547387183903783
+AAAAAAAALGIAAAAA	Modest, educational principles would 	Sports                                            	tennis                                            	6.42	18707.39	5.18171580215038800
+AAAAAAAALHPBAAAA	Far little eyes can happen pp.. Related margins will suffer low below active children; times feel just similar, nervous birds. Vegetabl	Sports                                            	tennis                                            	9.01	813.78	0.22540700148304722
+AAAAAAAALKHDAAAA	Then various shoes date good, bad shops. Here open rats match badly well dual games. No doubt small kids answer much points. Completely free services shall understand. Following patients	Sports                                            	tennis                                            	5.46	1154.69	0.31983485775327459
+AAAAAAAALODAAAAA	Widely free parties would find in a problems. Men like parties; straight a	Sports                                            	tennis                                            	8.95	10297.10	2.85216942536199653
+AAAAAAAAMDACAAAA	Tired rights free. Paintings sell	Sports                                            	tennis                                            	8.06	5429.22	1.50382683353214583
+AAAAAAAANAAAAAAA	Meetings improve early women. Even likely variables might want approxi	Sports                                            	tennis                                            	2.56	7342.79	2.03386207134570068
+AAAAAAAANKODAAAA	Growing jews see only grey tactics. Also indian parts ought to provide pretty other, canadian ways. Countries shall correspond really to a doubts. Star sounds ought to mean further at a steps. 	Sports                                            	tennis                                            	8.04	4423.03	1.22512464028307694
+AAAAAAAAOCCBAAAA	Else single arrangements will not keep approximately from a teachers. Large levels tolerate daily financial, critical others. Properties make a	Sports                                            	tennis                                            	0.30	5475.18	1.51655718545546767
+AAAAAAAAOEBEAAAA	Equivalent, important points would not reject foreign, high mountains. Always alive cups mark near the games. Sons will not stay extremely. Unfortunatel	Sports                                            	tennis                                            	0.19	5314.97	1.47218099568968454
+AAAAAAAAOEDBAAAA	Confidential companies can write highly; potentially new children mix sympathetically military, economic gains. Various, traditional designers make in a measurements. Individuals tell only se	Sports                                            	tennis                                            	7.12	1837.86	0.50906450360740392
+AAAAAAAAONEBAAAA	Examples show waves. Currently representative farmers should put like a customers. Both full rights practise with a police. Legal re	Sports                                            	tennis                                            	4.24	735.27	0.20366070188557120
+AAAAAAAAPCOAAAAA	Part	Sports                                            	tennis                                            	6.53	4928.46	1.36512250304644856
+AAAAAAAAPGOCAAAA	Great, big arts will not let brilliant pp.. Real, new or	Sports                                            	tennis                                            	0.88	13772.83	3.81490367450140978
+AAAAAAAAPHPDAAAA	Inc presents cannot break often subjects. Of course capital services would pay. Systems cannot	Sports                                            	tennis                                            	9.67	3395.45	0.94049768141956387
+AAAAAAAAPJNAAAAA	Parts may refuse primarily old holidays. Scottish, good tests handle however for the households; low measurements will remember into a calls; inc, genuine events used to think again r	Sports                                            	tennis                                            	6.88	733.87	0.20327291918990865
+AAAAAAAAPMOCAAAA	Literary pai	Sports                                            	tennis                                            	2.68	3317.04	0.91877908058606374
+AAAAAAAAPOHBAAAA	Themes would not reflect on the jeans. Traditional relations would not force mildly smal	Sports                                            	tennis                                            	9.89	1274.76	0.35309276365913303
diff --git a/sql/core/src/test/scala/org/apache/spark/deploy/history/Utils.scala b/sql/core/src/test/scala/org/apache/spark/deploy/history/Utils.scala
new file mode 100644
index 0000000000000..f73305b1b001e
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/deploy/history/Utils.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.deploy.history
+
+import org.apache.spark.SparkConf
+import org.apache.spark.internal.config.History.HISTORY_LOG_DIR
+import org.apache.spark.util.ManualClock
+
+object Utils {
+  def withFsHistoryProvider(logDir: String)(fn: FsHistoryProvider => Unit): Unit = {
+    var provider: FsHistoryProvider = null
+    try {
+      val clock = new ManualClock()
+      val conf = new SparkConf().set(HISTORY_LOG_DIR, logDir)
+      val provider = new FsHistoryProvider(conf, clock)
+      provider.checkForLogs()
+      fn(provider)
+    } finally {
+      if (provider != null) {
+        provider.stop()
+        provider = null
+      }
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ApproximatePercentileQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ApproximatePercentileQuerySuite.scala
index 2b4abed645910..4991e397eb11c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ApproximatePercentileQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ApproximatePercentileQuerySuite.scala
@@ -150,7 +150,7 @@ class ApproximatePercentileQuerySuite extends QueryTest with SharedSparkSession
       (1 to 1000).toDF("col").createOrReplaceTempView(table)
       checkAnswer(
         spark.sql(s"SELECT percentile_approx(col, array(0.25 + 0.25D), 200 + 800) FROM $table"),
-        Row(Seq(499))
+        Row(Seq(500))
       )
     }
   }
@@ -296,4 +296,23 @@ class ApproximatePercentileQuerySuite extends QueryTest with SharedSparkSession
     buffer.quantileSummaries
     assert(buffer.isCompressed)
   }
+
+  test("SPARK-32908: maximum target error in percentile_approx") {
+    withTempView(table) {
+      spark.read
+        .schema("col int")
+        .csv(testFile("test-data/percentile_approx-input.csv.bz2"))
+        .repartition(1)
+        .createOrReplaceTempView(table)
+      checkAnswer(
+        spark.sql(
+          s"""SELECT
+             |  percentile_approx(col, 0.77, 1000),
+             |  percentile_approx(col, 0.77, 10000),
+             |  percentile_approx(col, 0.77, 100000),
+             |  percentile_approx(col, 0.77, 1000000)
+             |FROM $table""".stripMargin),
+        Row(18, 17, 17, 17))
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala
index 2c3b37a1498ec..d58bf2c6260b1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala
@@ -63,11 +63,17 @@ abstract class BenchmarkQueryTest extends QueryTest with SharedSparkSession {
 
   protected def checkGeneratedCode(plan: SparkPlan, checkMethodCodeSize: Boolean = true): Unit = {
     val codegenSubtrees = new collection.mutable.HashSet[WholeStageCodegenExec]()
-    plan foreach {
-      case s: WholeStageCodegenExec =>
-        codegenSubtrees += s
-      case _ =>
+
+    def findSubtrees(plan: SparkPlan): Unit = {
+      plan foreach {
+        case s: WholeStageCodegenExec =>
+          codegenSubtrees += s
+        case s =>
+          s.subqueries.foreach(findSubtrees)
+      }
     }
+
+    findSubtrees(plan)
     codegenSubtrees.toSeq.foreach { subtree =>
       val code = subtree.doCodeGen()._2
       val (_, ByteCodeStats(maxMethodCodeSize, _, _)) = try {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CTEHintSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CTEHintSuite.scala
new file mode 100644
index 0000000000000..13039bbbf6bd2
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CTEHintSuite.scala
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import org.apache.log4j.Level
+
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.test.SharedSparkSession
+
+class CTEHintSuite extends QueryTest with SharedSparkSession {
+
+  def verifyCoalesceOrRepartitionHint(df: DataFrame): Unit = {
+    def checkContainsRepartition(plan: LogicalPlan): Unit = {
+      val repartitions = plan collect {
+        case r: Repartition => r
+        case r: RepartitionByExpression => r
+      }
+      assert(repartitions.nonEmpty)
+    }
+    val analyzed = df.queryExecution.analyzed
+    val optimized = df.queryExecution.optimizedPlan
+    checkContainsRepartition(analyzed)
+    checkContainsRepartition(optimized)
+    optimized collect {
+      case _: ResolvedHint => fail("ResolvedHint should not appear after optimize.")
+    }
+  }
+
+  def verifyJoinHint(df: DataFrame, expectedHints: Seq[JoinHint]): Unit = {
+    val analyzed = df.queryExecution.analyzed
+    val resolvedHints = analyzed collect {
+      case r: ResolvedHint => r
+    }
+    assert(resolvedHints.nonEmpty)
+    val optimized = df.queryExecution.optimizedPlan
+    val joinHints = optimized collect {
+      case Join(_, _, _, _, hint) => hint
+      case _: ResolvedHint => fail("ResolvedHint should not appear after optimize.")
+    }
+    assert(joinHints == expectedHints)
+  }
+
+  def verifyJoinHintWithWarnings(
+      df: => DataFrame,
+      expectedHints: Seq[JoinHint],
+      warnings: Seq[String]): Unit = {
+    val logAppender = new LogAppender("join hints")
+    withLogAppender(logAppender) {
+      verifyJoinHint(df, expectedHints)
+    }
+    val warningMessages = logAppender.loggingEvents
+      .filter(_.getLevel == Level.WARN)
+      .map(_.getRenderedMessage)
+      .filter(_.contains("hint"))
+    assert(warningMessages.size == warnings.size)
+    warnings.foreach { w =>
+      assert(warningMessages.contains(w))
+    }
+  }
+
+  def msgNoJoinForJoinHint(strategy: String): String =
+    s"A join hint (strategy=$strategy) is specified but it is not part of a join relation."
+
+  def msgJoinHintOverridden(strategy: String): String =
+    s"Hint (strategy=$strategy) is overridden by another hint and will not take effect."
+
+  test("Resolve coalesce hint in CTE") {
+    // COALESCE,
+    // REPARTITION,
+    // REPARTITION_BY_RANGE
+    withTable("t") {
+      sql("CREATE TABLE t USING PARQUET AS SELECT 1 AS id")
+      verifyCoalesceOrRepartitionHint(
+        sql("WITH cte AS (SELECT /*+ COALESCE(1) */ * FROM t) SELECT * FROM cte"))
+      verifyCoalesceOrRepartitionHint(
+        sql("WITH cte AS (SELECT /*+ REPARTITION(3) */ * FROM t) SELECT * FROM cte"))
+      verifyCoalesceOrRepartitionHint(
+        sql("WITH cte AS (SELECT /*+ REPARTITION(id) */ * FROM t) SELECT * FROM cte"))
+      verifyCoalesceOrRepartitionHint(
+        sql("WITH cte AS (SELECT /*+ REPARTITION(3, id) */ * FROM t) SELECT * FROM cte"))
+      verifyCoalesceOrRepartitionHint(
+        sql("WITH cte AS (SELECT /*+ REPARTITION_BY_RANGE(id) */ * FROM t) SELECT * FROM cte"))
+      verifyCoalesceOrRepartitionHint(
+        sql("WITH cte AS (SELECT /*+ REPARTITION_BY_RANGE(3, id) */ * FROM t) SELECT * FROM cte"))
+    }
+  }
+
+  test("Resolve join hint in CTE") {
+    // BROADCAST,
+    // SHUFFLE_MERGE,
+    // SHUFFLE_HASH,
+    // SHUFFLE_REPLICATE_NL
+    withTable("t", "s") {
+      sql("CREATE TABLE a USING PARQUET AS SELECT 1 AS a1")
+      sql("CREATE TABLE b USING PARQUET AS SELECT 1 AS b1")
+      sql("CREATE TABLE c USING PARQUET AS SELECT 1 AS c1")
+      verifyJoinHint(
+        sql(
+          """
+            |WITH cte AS (
+            |  SELECT /*+ BROADCAST(a) */ * FROM a JOIN b ON a.a1 = b.b1
+            |)
+            |SELECT * FROM cte
+            |""".stripMargin),
+        JoinHint(
+          Some(HintInfo(strategy = Some(BROADCAST))),
+          None) :: Nil
+      )
+      verifyJoinHint(
+        sql(
+          """
+            |WITH cte AS (
+            |  SELECT /*+ SHUFFLE_HASH(a) */ * FROM a JOIN b ON a.a1 = b.b1
+            |)
+            |SELECT * FROM cte
+            |""".stripMargin),
+        JoinHint(
+          Some(HintInfo(strategy = Some(SHUFFLE_HASH))),
+          None) :: Nil
+      )
+      verifyJoinHintWithWarnings(
+        sql(
+          """
+            |WITH cte AS (
+            |  SELECT /*+ SHUFFLE_HASH MERGE(a, c) BROADCAST(a, b)*/ * FROM a, b, c
+            |  WHERE a.a1 = b.b1 AND b.b1 = c.c1
+            |)
+            |SELECT * FROM cte
+            |""".stripMargin),
+        JoinHint(
+          None,
+          Some(HintInfo(strategy = Some(SHUFFLE_MERGE)))) ::
+          JoinHint(
+            Some(HintInfo(strategy = Some(SHUFFLE_MERGE))),
+            Some(HintInfo(strategy = Some(BROADCAST)))) :: Nil,
+          msgNoJoinForJoinHint("shuffle_hash") ::
+            msgJoinHintOverridden("broadcast") :: Nil
+      )
+      verifyJoinHint(
+        sql(
+          """
+            |WITH cte AS (
+            |  SELECT /*+ SHUFFLE_REPLICATE_NL(a) SHUFFLE_HASH(b) */ * FROM a JOIN b ON a.a1 = b.b1
+            |)
+            |SELECT * FROM cte
+            |""".stripMargin),
+        JoinHint(
+          Some(HintInfo(strategy = Some(SHUFFLE_REPLICATE_NL))),
+          Some(HintInfo(strategy = Some(SHUFFLE_HASH)))) :: Nil
+      )
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
index 20f2a7f947b81..19150446488da 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
@@ -17,14 +17,21 @@
 
 package org.apache.spark.sql
 
+import java.io.{File, FilenameFilter}
+import java.nio.file.{Files, Paths}
+import java.time.{Duration, Period}
+
 import scala.collection.mutable.HashSet
 import scala.concurrent.duration._
 
+import org.apache.commons.io.FileUtils
+
 import org.apache.spark.CleanerListener
 import org.apache.spark.executor.DataReadMethod._
 import org.apache.spark.executor.DataReadMethod.DataReadMethod
 import org.apache.spark.scheduler.{SparkListener, SparkListenerJobStart}
 import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.analysis.TempTableAlreadyExistsException
 import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
 import org.apache.spark.sql.catalyst.plans.logical.{BROADCAST, Join, JoinStrategyHint, SHUFFLE_HASH}
 import org.apache.spark.sql.catalyst.util.DateTimeConstants
@@ -140,6 +147,16 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
     }
   }
 
+  test("cache table as select - existing temp view") {
+    withTempView("tempView") {
+      sql("CREATE TEMPORARY VIEW tempView as SELECT 1")
+      val e = intercept[TempTableAlreadyExistsException] {
+        sql("CACHE TABLE tempView AS SELECT 1")
+      }
+      assert(e.getMessage.contains("Temporary view 'tempView' already exists"))
+    }
+  }
+
   test("uncaching temp table") {
     withTempView("tempTable1", "tempTable2") {
       testData.select("key").createOrReplaceTempView("tempTable1")
@@ -824,7 +841,7 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
     }
   }
 
-  test("SPARK-19993 nested subquery caching and scalar + predicate subqueris") {
+  test("SPARK-19993 nested subquery caching and scalar + predicate subqueries") {
     withTempView("t1", "t2", "t3", "t4") {
       Seq(1).toDF("c1").createOrReplaceTempView("t1")
       Seq(2).toDF("c1").createOrReplaceTempView("t2")
@@ -875,17 +892,17 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
   }
 
   private def checkIfNoJobTriggered[T](f: => T): T = {
-    var numJobTrigered = 0
+    var numJobTriggered = 0
     val jobListener = new SparkListener {
       override def onJobStart(jobStart: SparkListenerJobStart): Unit = {
-        numJobTrigered += 1
+        numJobTriggered += 1
       }
     }
     sparkContext.addSparkListener(jobListener)
     try {
       val result = f
       sparkContext.listenerBus.waitUntilEmpty()
-      assert(numJobTrigered === 0)
+      assert(numJobTriggered === 0)
       result
     } finally {
       sparkContext.removeSparkListener(jobListener)
@@ -912,33 +929,61 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
     }
   }
 
-  test("SPARK-24596 Non-cascading Cache Invalidation - drop temporary view") {
-    withTempView("t1", "t2") {
-      sql("CACHE TABLE t1 AS SELECT * FROM testData WHERE key > 1")
-      sql("CACHE TABLE t2 as SELECT * FROM t1 WHERE value > 1")
-
-      assert(spark.catalog.isCached("t1"))
-      assert(spark.catalog.isCached("t2"))
-      sql("DROP VIEW t1")
-      assert(spark.catalog.isCached("t2"))
-    }
-  }
-
-  test("SPARK-24596 Non-cascading Cache Invalidation - drop persistent view") {
-    withTable("t") {
-      spark.range(1, 10).toDF("key").withColumn("value", $"key" * 2)
-        .write.format("json").saveAsTable("t")
-      withView("t1") {
-        withTempView("t2") {
-          sql("CREATE VIEW t1 AS SELECT * FROM t WHERE key > 1")
-
-          sql("CACHE TABLE t1")
-          sql("CACHE TABLE t2 AS SELECT * FROM t1 WHERE value > 1")
+  test("SPARK-24596, SPARK-34052: cascading cache invalidation - drop temporary view") {
+    Seq(true, false).foreach { storeAnalyzed =>
+      withSQLConf(SQLConf.STORE_ANALYZED_PLAN_FOR_VIEW.key -> storeAnalyzed.toString) {
+        withTempView("t1", "t2") {
+          sql("CACHE TABLE t1 AS SELECT * FROM testData WHERE key > 1")
+          sql("CACHE TABLE t2 as SELECT * FROM t1 WHERE value > 1")
 
           assert(spark.catalog.isCached("t1"))
           assert(spark.catalog.isCached("t2"))
+
+          val oldView = spark.table("t2")
           sql("DROP VIEW t1")
-          assert(!spark.catalog.isCached("t2"))
+
+          // dropping a temp view trigger cache invalidation on dependents iff the config is
+          // turned off
+          assert(storeAnalyzed ==
+            spark.sharedState.cacheManager.lookupCachedData(oldView).isDefined)
+          if (!storeAnalyzed) {
+            // t2 should become invalid after t1 is dropped
+            val e = intercept[AnalysisException](spark.catalog.isCached("t2"))
+            assert(e.message.contains(s"Table or view not found"))
+          }
+        }
+      }
+    }
+  }
+
+  test("SPARK-24596, SPARK-34052: cascading cache invalidation - drop persistent view") {
+    Seq(true, false).foreach { storeAnalyzed =>
+      withSQLConf(SQLConf.STORE_ANALYZED_PLAN_FOR_VIEW.key -> storeAnalyzed.toString) {
+        withTable("t") {
+          spark.range(1, 10).toDF("key").withColumn("value", $"key" * 2)
+            .write.format("json").saveAsTable("t")
+          withView("t1") {
+            withTempView("t2") {
+              sql("CREATE VIEW t1 AS SELECT * FROM t WHERE key > 1")
+
+              sql("CACHE TABLE t1")
+              sql("CACHE TABLE t2 AS SELECT * FROM t1 WHERE value > 1")
+
+              assert(spark.catalog.isCached("t1"))
+              assert(spark.catalog.isCached("t2"))
+
+              val oldView = spark.table("t2")
+              sql("DROP VIEW t1")
+
+              // dropping a permanent view always trigger cache invalidation on dependents
+              assert(spark.sharedState.cacheManager.lookupCachedData(oldView).isEmpty)
+              if (!storeAnalyzed) {
+                // t2 should become invalid after t1 is dropped
+                val e = intercept[AnalysisException](spark.catalog.isCached("t2"))
+                assert(e.message.contains(s"Table or view not found"))
+              }
+            }
+          }
         }
       }
     }
@@ -1131,7 +1176,7 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
   }
 
   test("cache supports for intervals") {
-    withTable("interval_cache") {
+    withTable("interval_cache", "t1") {
       Seq((1, "1 second"), (2, "2 seconds"), (2, null))
         .toDF("k", "v").write.saveAsTable("interval_cache")
       sql("CACHE TABLE t1 AS SELECT k, cast(v as interval) FROM interval_cache")
@@ -1143,6 +1188,20 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
     }
   }
 
+  test("SPARK-35243: cache supports for YearMonthIntervalType and DayTimeIntervalType") {
+    withTempView("ymi_dti_interval_cache") {
+      Seq((1, Period.ofYears(1), Duration.ofDays(1)),
+        (2, Period.ofYears(2), Duration.ofDays(2)))
+        .toDF("k", "v1", "v2").createTempView("ymi_dti_interval_cache")
+      sql("CACHE TABLE tmp AS SELECT k, v1, v2 FROM ymi_dti_interval_cache")
+      assert(spark.catalog.isCached("tmp"))
+      checkAnswer(sql("SELECT * FROM tmp WHERE k = 1"),
+        Row(1, Period.ofYears(1), Duration.ofDays(1)))
+      sql("UNCACHE TABLE tmp")
+      assert(!spark.catalog.isCached("tmp"))
+    }
+  }
+
   test("SPARK-30494 Fix the leak of cached data when replace an existing view") {
     withTempView("tempView") {
       spark.catalog.clearCache()
@@ -1184,4 +1243,391 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
       assert(spark.sharedState.cacheManager.isEmpty)
     }
   }
+
+  test("SPARK-33228: Don't uncache data when replacing an existing view having the same plan") {
+    withTempView("tempView") {
+      spark.catalog.clearCache()
+      val df = spark.range(1).selectExpr("id a", "id b")
+      df.cache()
+      assert(spark.sharedState.cacheManager.lookupCachedData(df).isDefined)
+      df.createOrReplaceTempView("tempView")
+      assert(spark.sharedState.cacheManager.lookupCachedData(df).isDefined)
+      df.createOrReplaceTempView("tempView")
+      assert(spark.sharedState.cacheManager.lookupCachedData(df).isDefined)
+    }
+
+    withTempView("tempGlobalTempView") {
+      spark.catalog.clearCache()
+      val df = spark.range(1).selectExpr("id a", "id b")
+      df.cache()
+      assert(spark.sharedState.cacheManager.lookupCachedData(df).isDefined)
+      df.createOrReplaceGlobalTempView("tempGlobalTempView")
+      assert(spark.sharedState.cacheManager.lookupCachedData(df).isDefined)
+      df.createOrReplaceGlobalTempView("tempGlobalTempView")
+      assert(spark.sharedState.cacheManager.lookupCachedData(df).isDefined)
+    }
+  }
+
+  test("SPARK-33290: REFRESH TABLE should invalidate all caches referencing the table") {
+    withTable("t") {
+      withTempPath { path =>
+        withTempView("tempView1", "tempView2") {
+          Seq((1 -> "a")).toDF("i", "j").write.parquet(path.getCanonicalPath)
+          sql(s"CREATE TABLE t USING parquet LOCATION '${path.toURI}'")
+          sql("CREATE TEMPORARY VIEW tempView1 AS SELECT * FROM t")
+          sql("CACHE TABLE tempView2 AS SELECT i FROM tempView1")
+          checkAnswer(sql("SELECT * FROM tempView1"), Seq(Row(1, "a")))
+          checkAnswer(sql("SELECT * FROM tempView2"), Seq(Row(1)))
+
+          Utils.deleteRecursively(path)
+          sql("REFRESH TABLE tempView1")
+          checkAnswer(sql("SELECT * FROM tempView1"), Seq.empty)
+          checkAnswer(sql("SELECT * FROM tempView2"), Seq.empty)
+        }
+      }
+    }
+  }
+
+  test("SPARK-33729: REFRESH TABLE should not use cached/stale plan") {
+    def moveParquetFiles(src: File, dst: File): Unit = {
+      src.listFiles(new FilenameFilter {
+        override def accept(dir: File, name: String): Boolean = name.endsWith("parquet")
+      }).foreach { f =>
+        Files.move(f.toPath, Paths.get(dst.getAbsolutePath, f.getName))
+      }
+      // cleanup the rest of the files
+      src.listFiles().foreach(_.delete())
+      src.delete()
+    }
+
+    withTable("t") {
+      withTempDir { dir =>
+        val path1 = new File(dir, "path1")
+        Seq((1 -> "a")).toDF("i", "j").write.parquet(path1.getCanonicalPath)
+        moveParquetFiles(path1, dir)
+        sql(s"CREATE TABLE t (i INT, j STRING) USING parquet LOCATION '${dir.toURI}'")
+        sql("CACHE TABLE t")
+        checkAnswer(sql("SELECT * FROM t"), Row(1, "a") :: Nil)
+
+        val path2 = new File(dir, "path2")
+        Seq(2 -> "b").toDF("i", "j").write.parquet(path2.getCanonicalPath)
+        moveParquetFiles(path2, dir)
+        sql("REFRESH TABLE t")
+        checkAnswer(sql("SELECT * FROM t"), Row(1, "a") :: Row(2, "b") :: Nil)
+      }
+    }
+  }
+
+  test("SPARK-33647: cache table support for permanent view") {
+    withView("v1") {
+      spark.catalog.clearCache()
+      sql("create or replace view v1 as select 1")
+      sql("cache table v1")
+      assert(spark.sharedState.cacheManager.lookupCachedData(sql("select 1")).isDefined)
+      sql("create or replace view v1 as select 1, 2")
+      assert(spark.sharedState.cacheManager.lookupCachedData(sql("select 1")).isEmpty)
+      sql("cache table v1")
+      assert(spark.sharedState.cacheManager.lookupCachedData(sql("select 1, 2")).isDefined)
+    }
+  }
+
+  test("SPARK-34269: cache lookup with ORDER BY / LIMIT clause") {
+    Seq("ORDER BY key", "LIMIT 10").foreach { clause =>
+      withTable("t") {
+        withTempView("v1") {
+          sql("CREATE TABLE t (key bigint, value string) USING parquet")
+          sql(s"CACHE TABLE v1 AS SELECT * FROM t $clause")
+
+          val query = sql(s"SELECT * FROM t $clause")
+          assert(spark.sharedState.cacheManager.lookupCachedData(query).isDefined)
+        }
+      }
+    }
+  }
+
+  test("SPARK-34027: refresh cache in partitions recovering") {
+    withTable("t") {
+      sql("CREATE TABLE t (id int, part int) USING parquet PARTITIONED BY (part)")
+      sql("INSERT INTO t PARTITION (part=0) SELECT 0")
+      assert(!spark.catalog.isCached("t"))
+      sql("CACHE TABLE t")
+      assert(spark.catalog.isCached("t"))
+      checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 0)))
+
+      // Create new partition (part = 1) in the filesystem
+      val information = sql("SHOW TABLE EXTENDED LIKE 't' PARTITION (part = 0)")
+        .select("information")
+        .first().getString(0)
+      val part0Loc = information
+        .split("\\r?\\n")
+        .filter(_.startsWith("Location:"))
+        .head
+        .replace("Location: file:", "")
+      FileUtils.copyDirectory(
+        new File(part0Loc),
+        new File(part0Loc.replace("part=0", "part=1")))
+
+      sql("ALTER TABLE t RECOVER PARTITIONS")
+      assert(spark.catalog.isCached("t"))
+      checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 0), Row(0, 1)))
+    }
+  }
+
+  test("SPARK-34052: cascading cache invalidation - CatalogImpl.dropTempView") {
+    Seq(true, false).foreach { storeAnalyzed =>
+      withSQLConf(SQLConf.STORE_ANALYZED_PLAN_FOR_VIEW.key -> storeAnalyzed.toString) {
+        withTempView("view1", "view2") {
+          sql("CREATE TEMPORARY VIEW view1 AS SELECT * FROM testData WHERE key > 1")
+          sql("CACHE TABLE view2 AS SELECT * FROM view1 WHERE value > 1")
+          assert(spark.catalog.isCached("view2"))
+
+          val oldView = spark.table("view2")
+          spark.catalog.dropTempView("view1")
+          assert(storeAnalyzed ==
+            spark.sharedState.cacheManager.lookupCachedData(oldView).isDefined)
+        }
+      }
+    }
+  }
+
+  test("SPARK-34052: cascading cache invalidation - CatalogImpl.dropGlobalTempView") {
+    Seq(true, false).foreach { storeAnalyzed =>
+      withSQLConf(SQLConf.STORE_ANALYZED_PLAN_FOR_VIEW.key -> storeAnalyzed.toString) {
+        withGlobalTempView("view1") {
+          withTempView("view2") {
+            val db = spark.sharedState.globalTempViewManager.database
+            sql("CREATE GLOBAL TEMPORARY VIEW view1 AS SELECT * FROM testData WHERE key > 1")
+            sql(s"CACHE TABLE view2 AS SELECT * FROM ${db}.view1 WHERE value > 1")
+            assert(spark.catalog.isCached("view2"))
+
+            val oldView = spark.table("view2")
+            spark.catalog.dropGlobalTempView("view1")
+            assert(storeAnalyzed ==
+              spark.sharedState.cacheManager.lookupCachedData(oldView).isDefined)
+          }
+        }
+      }
+    }
+  }
+
+  test("SPARK-34052: cached temp view should become invalid after the source table is dropped") {
+    val t = "t"
+    withTable(t) {
+      sql(s"CREATE TABLE $t USING parquet AS SELECT * FROM VALUES(1, 'a') AS $t(a, b)")
+      sql(s"CACHE TABLE v AS SELECT a FROM $t")
+      checkAnswer(sql("SELECT * FROM v"), Row(1) :: Nil)
+      sql(s"DROP TABLE $t")
+      val e = intercept[AnalysisException](sql("SELECT * FROM v"))
+      assert(e.message.contains(s"Table or view not found: $t"))
+    }
+  }
+
+  test("SPARK-34347: cascading cache invalidation - SQLContext.uncacheTable") {
+    Seq(true, false).foreach { storeAnalyzed =>
+      withSQLConf(SQLConf.STORE_ANALYZED_PLAN_FOR_VIEW.key -> storeAnalyzed.toString) {
+        withTempView("view1", "view2") {
+          sql("CREATE TEMPORARY VIEW view1 AS SELECT * FROM testData WHERE key > 1")
+          sql("CACHE TABLE view2 AS SELECT * FROM view1 WHERE value > 1")
+          assert(spark.catalog.isCached("view2"))
+
+          val oldView = spark.table("view2")
+          spark.sqlContext.uncacheTable("view1")
+          assert(storeAnalyzed ==
+            spark.sharedState.cacheManager.lookupCachedData(oldView).isDefined,
+            s"when storeAnalyzed = $storeAnalyzed")
+        }
+      }
+    }
+  }
+
+  test("SPARK-34347: cascading cache invalidation - SQLContext.uncacheTable (global temp view)") {
+    Seq(true, false).foreach { storeAnalyzed =>
+      withSQLConf(SQLConf.STORE_ANALYZED_PLAN_FOR_VIEW.key -> storeAnalyzed.toString) {
+        withGlobalTempView("view1") {
+          withTempView("view2") {
+            val db = spark.sharedState.globalTempViewManager.database
+            sql("CREATE GLOBAL TEMPORARY VIEW view1 AS SELECT * FROM testData WHERE key > 1")
+            sql(s"CACHE TABLE view2 AS SELECT * FROM $db.view1 WHERE value > 1")
+            assert(spark.catalog.isCached("view2"))
+
+            val oldView = spark.table("view2")
+            spark.sqlContext.uncacheTable(s"$db.view1")
+            assert(storeAnalyzed ==
+              spark.sharedState.cacheManager.lookupCachedData(oldView).isDefined,
+              s"when storeAnalyzed = $storeAnalyzed")
+          }
+        }
+      }
+    }
+  }
+
+  test("SPARK-34546: ALTER VIEW AS should uncache if a temp view is cached") {
+    Seq(true, false).foreach { storeAnalyzed =>
+      withSQLConf(SQLConf.STORE_ANALYZED_PLAN_FOR_VIEW.key -> storeAnalyzed.toString) {
+        withTempView("tv") {
+          testAlterTemporaryViewAsWithCache(TableIdentifier("tv"), storeAnalyzed)
+        }
+      }
+    }
+  }
+
+  test("SPARK-34546: ALTER VIEW AS should uncache if a global temp view is cached") {
+    Seq(true, false).foreach { storeAnalyzed =>
+      withSQLConf(SQLConf.STORE_ANALYZED_PLAN_FOR_VIEW.key -> storeAnalyzed.toString) {
+        withGlobalTempView("global_tv") {
+          val db = spark.sharedState.globalTempViewManager.database
+          testAlterTemporaryViewAsWithCache(TableIdentifier("global_tv", Some(db)), storeAnalyzed)
+        }
+      }
+    }
+  }
+
+  private def testAlterTemporaryViewAsWithCache(
+      ident: TableIdentifier,
+      storeAnalyzed: Boolean): Unit = {
+    val (tempViewStr, viewName) = if (ident.database.nonEmpty) {
+      ("GLOBAL TEMPORARY", s"${ident.database.get}.${ident.table}")
+    } else {
+      ("TEMPORARY", ident.table)
+    }
+
+    sql(s"CREATE $tempViewStr VIEW ${ident.table} AS SELECT 1")
+
+    sql(s"CACHE TABLE $viewName")
+    assert(spark.catalog.isCached(viewName))
+    assert(spark.sharedState.cacheManager.lookupCachedData(sql("SELECT 1")).nonEmpty)
+
+    if (storeAnalyzed) {
+      // Altered temporary view will have the same plan, thus it will not be uncached.
+      // Note that this check is done only if a temporary view stores an analyzed view.
+      sql(s"ALTER VIEW $viewName as SELECT 1")
+      assert(spark.catalog.isCached(viewName))
+      assert(spark.sharedState.cacheManager.lookupCachedData(sql("SELECT 1")).nonEmpty)
+    }
+
+    sql(s"ALTER VIEW $viewName as SELECT 2")
+    assert(!spark.catalog.isCached(viewName))
+    assert(spark.sharedState.cacheManager.lookupCachedData(sql("SELECT 1")).isEmpty)
+  }
+
+  test("SPARK-34546: ALTER VIEW AS should uncache if a permanent view is cached") {
+    withView("view") {
+      sql("CREATE VIEW view AS SELECT 1")
+      sql("CACHE TABLE view")
+      assert(spark.catalog.isCached("view"))
+      assert(spark.sharedState.cacheManager.lookupCachedData(sql("SELECT 1")).nonEmpty)
+
+      // ALTER VIEW AS on a permanent view should uncache even if the replacing view produces
+      // the same result.
+      sql("ALTER VIEW view as SELECT 1")
+      assert(!spark.catalog.isCached("view"))
+      assert(spark.sharedState.cacheManager.lookupCachedData(sql("SELECT 1")).isEmpty)
+    }
+  }
+
+  test("SPARK-34699: CREATE TEMP VIEW USING should uncache correctly") {
+    withTempView("tv") {
+      testCreateTemporaryViewUsingWithCache(TableIdentifier("tv"))
+    }
+  }
+
+  test("SPARK-34699: CREATE GLOBAL TEMP VIEW USING should uncache correctly") {
+    withGlobalTempView("global_tv") {
+      val db = spark.sharedState.globalTempViewManager.database
+      testCreateTemporaryViewUsingWithCache(TableIdentifier("global_tv", Some(db)))
+    }
+  }
+
+  private def testCreateTemporaryViewUsingWithCache(ident: TableIdentifier): Unit = {
+    withTempDir { dir =>
+      val path1 = new File(dir, "t1").getCanonicalPath
+      val path2 = new File(dir, "t2").getCanonicalPath
+      Seq(1).toDF.write.parquet(path1)
+      Seq(1).toDF.write.parquet(path2)
+
+      val (tempViewStr, viewName) = if (ident.database.nonEmpty) {
+        ("GLOBAL TEMPORARY VIEW", s"${ident.database.get}.${ident.table}")
+      } else {
+        ("TEMPORARY VIEW", ident.table)
+      }
+
+      sql(s"CREATE $tempViewStr ${ident.table} USING parquet OPTIONS (path '$path1')")
+
+      sql(s"CACHE TABLE $viewName")
+      assert(spark.catalog.isCached(viewName))
+
+      // Replacing with the same relation. The cache shouldn't be uncached.
+      sql(s"CREATE OR REPLACE $tempViewStr ${ident.table} USING parquet OPTIONS (path '$path1')")
+      assert(spark.catalog.isCached(viewName))
+
+      // Replacing with a different relation. The cache should be cleared.
+      sql(s"CREATE OR REPLACE $tempViewStr ${ident.table} USING parquet OPTIONS (path '$path2')")
+      assert(!spark.catalog.isCached(viewName))
+
+      // Validate that the cache is cleared by creating a temp view with the same relation.
+      sql(s"CREATE OR REPLACE $tempViewStr ${ident.table} USING parquet OPTIONS (path '$path1')")
+      assert(!spark.catalog.isCached(viewName))
+    }
+  }
+
+  test("SPARK-35332: Make cache plan disable configs configurable - check AQE") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "2",
+      SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key -> "1",
+      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") {
+
+      withTempView("t1", "t2", "t3") {
+        withSQLConf(SQLConf.CAN_CHANGE_CACHED_PLAN_OUTPUT_PARTITIONING.key -> "false") {
+          sql("CACHE TABLE t1 as SELECT /*+ REPARTITION */ * FROM values(1) as t(c)")
+          assert(spark.table("t1").rdd.partitions.length == 2)
+        }
+
+        withSQLConf(SQLConf.CAN_CHANGE_CACHED_PLAN_OUTPUT_PARTITIONING.key -> "true") {
+          assert(spark.table("t1").rdd.partitions.length == 2)
+          sql("CACHE TABLE t2 as SELECT /*+ REPARTITION */ * FROM values(2) as t(c)")
+          assert(spark.table("t2").rdd.partitions.length == 1)
+        }
+
+        withSQLConf(SQLConf.CAN_CHANGE_CACHED_PLAN_OUTPUT_PARTITIONING.key -> "false") {
+          assert(spark.table("t1").rdd.partitions.length == 2)
+          assert(spark.table("t2").rdd.partitions.length == 1)
+          sql("CACHE TABLE t3 as SELECT /*+ REPARTITION */ * FROM values(3) as t(c)")
+          assert(spark.table("t3").rdd.partitions.length == 2)
+        }
+      }
+    }
+  }
+
+  test("SPARK-35332: Make cache plan disable configs configurable - check bucket scan") {
+    withTable("t1", "t2", "t3") {
+      Seq(1, 2, 3).foreach { i =>
+        spark.range(1, 2)
+          .write
+          .format("parquet")
+          .bucketBy(2, "id")
+          .saveAsTable(s"t$i")
+      }
+
+      withCache("t1", "t2", "t3") {
+        withSQLConf(SQLConf.BUCKETING_ENABLED.key -> "true",
+          SQLConf.FILES_MIN_PARTITION_NUM.key -> "1",
+          SQLConf.CAN_CHANGE_CACHED_PLAN_OUTPUT_PARTITIONING.key -> "false") {
+          sql("CACHE TABLE t1")
+          assert(spark.table("t1").rdd.partitions.length == 2)
+
+          withSQLConf(SQLConf.CAN_CHANGE_CACHED_PLAN_OUTPUT_PARTITIONING.key -> "true") {
+            assert(spark.table("t1").rdd.partitions.length == 2)
+            sql("CACHE TABLE t2")
+            assert(spark.table("t2").rdd.partitions.length == 1)
+          }
+
+          withSQLConf(SQLConf.CAN_CHANGE_CACHED_PLAN_OUTPUT_PARTITIONING.key -> "false") {
+            assert(spark.table("t1").rdd.partitions.length == 2)
+            assert(spark.table("t2").rdd.partitions.length == 1)
+            sql("CACHE TABLE t3")
+            assert(spark.table("t3").rdd.partitions.length == 2)
+          }
+        }
+      }
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala
new file mode 100644
index 0000000000000..c06544ee00621
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala
@@ -0,0 +1,876 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import org.apache.spark.{SparkConf, SparkException}
+import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
+import org.apache.spark.sql.catalyst.util.CharVarcharUtils
+import org.apache.spark.sql.connector.SchemaRequiredDataSource
+import org.apache.spark.sql.connector.catalog.InMemoryPartitionTableCatalog
+import org.apache.spark.sql.execution.datasources.LogicalRelation
+import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.sources.SimpleInsertSource
+import org.apache.spark.sql.test.{SharedSparkSession, SQLTestUtils}
+import org.apache.spark.sql.types._
+
+// The base trait for char/varchar tests that need to be run with different table implementations.
+trait CharVarcharTestSuite extends QueryTest with SQLTestUtils {
+
+  def format: String
+
+  def checkColType(f: StructField, dt: DataType): Unit = {
+    assert(f.dataType == CharVarcharUtils.replaceCharVarcharWithString(dt))
+    assert(CharVarcharUtils.getRawType(f.metadata) == Some(dt))
+  }
+
+  def checkPlainResult(df: DataFrame, dt: String, insertVal: String): Unit = {
+    val dataType = CatalystSqlParser.parseDataType(dt)
+    checkColType(df.schema(1), dataType)
+    dataType match {
+      case CharType(len) =>
+        // char value will be padded if (<= len) or trimmed if (> len)
+        val fixLenStr = if (insertVal != null) {
+          insertVal.take(len).padTo(len, " ").mkString
+        } else null
+        checkAnswer(df, Row("1", fixLenStr))
+      case VarcharType(len) =>
+        // varchar value will be remained if (<= len) or trimmed if (> len)
+        val varLenStrWithUpperBound = if (insertVal != null) {
+          insertVal.take(len)
+        } else null
+        checkAnswer(df, Row("1", varLenStrWithUpperBound))
+    }
+  }
+
+  test("apply char padding/trimming and varchar trimming: top-level columns") {
+    Seq("CHAR(5)", "VARCHAR(5)").foreach { typ =>
+      withTable("t") {
+        sql(s"CREATE TABLE t(i STRING, c $typ) USING $format")
+        (0 to 5).map(n => "a" + " " * n).foreach { v =>
+          sql(s"INSERT OVERWRITE t VALUES ('1', '$v')")
+          checkPlainResult(spark.table("t"), typ, v)
+        }
+        sql("INSERT OVERWRITE t VALUES ('1', null)")
+        checkPlainResult(spark.table("t"), typ, null)
+      }
+    }
+  }
+
+  test("char type values should be padded or trimmed: partitioned columns") {
+    // via dynamic partitioned columns
+    withTable("t") {
+      sql(s"CREATE TABLE t(i STRING, c CHAR(5)) USING $format PARTITIONED BY (c)")
+      (0 to 5).map(n => "a" + " " * n).foreach { v =>
+        sql(s"INSERT OVERWRITE t VALUES ('1', '$v')")
+        checkPlainResult(spark.table("t"), "CHAR(5)", v)
+      }
+    }
+
+    withTable("t") {
+      sql(s"CREATE TABLE t(i STRING, c CHAR(5)) USING $format PARTITIONED BY (c)")
+      (0 to 5).map(n => "a" + " " * n).foreach { v =>
+        // via dynamic partitioned columns with drop partition command
+        sql(s"INSERT INTO t VALUES ('1', '$v')")
+        checkPlainResult(spark.table("t"), "CHAR(5)", v)
+        sql(s"ALTER TABLE t DROP PARTITION(c='a')")
+        checkAnswer(spark.table("t"), Nil)
+
+        // via static partitioned columns with drop partition command
+        sql(s"INSERT INTO t PARTITION (c ='$v') VALUES ('1')")
+        checkPlainResult(spark.table("t"), "CHAR(5)", v)
+        sql(s"ALTER TABLE t DROP PARTITION(c='a')")
+        checkAnswer(spark.table("t"), Nil)
+      }
+    }
+  }
+
+  test("varchar type values length check and trim: partitioned columns") {
+    (0 to 5).foreach { n =>
+      // SPARK-34192: we need to create a a new table for each round of test because of
+      // trailing spaces in partition column will be treated differently.
+      // This is because Mysql and Derby(used in tests) considers 'a' = 'a '
+      // whereas others like (Postgres, Oracle) doesn't exhibit this problem.
+      // see more at:
+      // https://issues.apache.org/jira/browse/HIVE-13618
+      // https://issues.apache.org/jira/browse/SPARK-34192
+      withTable("t") {
+        sql(s"CREATE TABLE t(i STRING, c VARCHAR(5)) USING $format PARTITIONED BY (c)")
+        val v = "a" + " " * n
+        // via dynamic partitioned columns
+        sql(s"INSERT INTO t VALUES ('1', '$v')")
+        checkPlainResult(spark.table("t"), "VARCHAR(5)", v)
+        sql(s"ALTER TABLE t DROP PARTITION(c='$v')")
+        checkAnswer(spark.table("t"), Nil)
+
+        // via static partitioned columns
+        sql(s"INSERT INTO t PARTITION (c='$v') VALUES ('1')")
+        checkPlainResult(spark.table("t"), "VARCHAR(5)", v)
+        sql(s"ALTER TABLE t DROP PARTITION(c='$v')")
+        checkAnswer(spark.table("t"), Nil)
+      }
+    }
+  }
+
+  test("oversize char/varchar values for alter table partition operations") {
+    Seq("CHAR(5)", "VARCHAR(5)").foreach { typ =>
+      withTable("t") {
+        sql(s"CREATE TABLE t(i STRING, c $typ) USING $format PARTITIONED BY (c)")
+        Seq("ADD", "DROP").foreach { op =>
+          val e = intercept[RuntimeException](sql(s"ALTER TABLE t $op PARTITION(c='abcdef')"))
+          assert(e.getMessage.contains("Exceeds char/varchar type length limitation: 5"))
+        }
+        val e1 = intercept[RuntimeException] {
+          sql(s"ALTER TABLE t PARTITION (c='abcdef') RENAME TO PARTITION (c='2')")
+        }
+        assert(e1.getMessage.contains("Exceeds char/varchar type length limitation: 5"))
+        val e2 = intercept[RuntimeException] {
+          sql(s"ALTER TABLE t PARTITION (c='1') RENAME TO PARTITION (c='abcdef')")
+        }
+        assert(e2.getMessage.contains("Exceeds char/varchar type length limitation: 5"))
+      }
+    }
+  }
+
+  test("SPARK-34233: char/varchar with null value for partitioned columns") {
+    Seq("CHAR(5)", "VARCHAR(5)").foreach { typ =>
+      withTable("t") {
+        sql(s"CREATE TABLE t(i STRING, c $typ) USING $format PARTITIONED BY (c)")
+        sql("INSERT INTO t VALUES ('1', null)")
+        checkPlainResult(spark.table("t"), typ, null)
+        sql("INSERT OVERWRITE t VALUES ('1', null)")
+        checkPlainResult(spark.table("t"), typ, null)
+        sql("INSERT OVERWRITE t PARTITION (c=null) VALUES ('1')")
+        checkPlainResult(spark.table("t"), typ, null)
+        sql("ALTER TABLE t DROP PARTITION(c=null)")
+        checkAnswer(spark.table("t"), Nil)
+      }
+    }
+  }
+
+  test("char/varchar type values length check: partitioned columns of other types") {
+    Seq("CHAR(5)", "VARCHAR(5)").foreach { typ =>
+      withTable("t") {
+        sql(s"CREATE TABLE t(i STRING, c $typ) USING $format PARTITIONED BY (c)")
+        Seq(1, 10, 100, 1000, 10000).foreach { v =>
+          sql(s"INSERT OVERWRITE t VALUES ('1', $v)")
+          checkPlainResult(spark.table("t"), typ, v.toString)
+          sql(s"ALTER TABLE t DROP PARTITION(c=$v)")
+          checkAnswer(spark.table("t"), Nil)
+        }
+
+        val e1 = intercept[SparkException](sql(s"INSERT OVERWRITE t VALUES ('1', 100000)"))
+        assert(e1.getCause.getMessage.contains("Exceeds char/varchar type length limitation: 5"))
+
+        val e2 = intercept[RuntimeException](sql("ALTER TABLE t DROP PARTITION(c=100000)"))
+        assert(e2.getMessage.contains("Exceeds char/varchar type length limitation: 5"))
+      }
+    }
+  }
+
+  test("char type values should be padded: nested in struct") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(i STRING, c STRUCT<c: CHAR(5)>) USING $format")
+      sql("INSERT INTO t VALUES ('1', struct('a'))")
+      checkAnswer(spark.table("t"), Row("1", Row("a" + " " * 4)))
+      checkColType(spark.table("t").schema(1), new StructType().add("c", CharType(5)))
+
+      sql("INSERT OVERWRITE t VALUES ('1', null)")
+      checkAnswer(spark.table("t"), Row("1", null))
+      sql("INSERT OVERWRITE t VALUES ('1', struct(null))")
+      checkAnswer(spark.table("t"), Row("1", Row(null)))
+    }
+  }
+
+  test("char type values should be padded: nested in array") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(i STRING, c ARRAY<CHAR(5)>) USING $format")
+      sql("INSERT INTO t VALUES ('1', array('a', 'ab'))")
+      checkAnswer(spark.table("t"), Row("1", Seq("a" + " " * 4, "ab" + " " * 3)))
+      checkColType(spark.table("t").schema(1), ArrayType(CharType(5)))
+
+      sql("INSERT OVERWRITE t VALUES ('1', null)")
+      checkAnswer(spark.table("t"), Row("1", null))
+      sql("INSERT OVERWRITE t VALUES ('1', array(null))")
+      checkAnswer(spark.table("t"), Row("1", Seq(null)))
+    }
+  }
+
+  test("char type values should be padded: nested in map key") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(i STRING, c MAP<CHAR(5), STRING>) USING $format")
+      sql("INSERT INTO t VALUES ('1', map('a', 'ab'))")
+      checkAnswer(spark.table("t"), Row("1", Map(("a" + " " * 4, "ab"))))
+      checkColType(spark.table("t").schema(1), MapType(CharType(5), StringType))
+
+      sql("INSERT OVERWRITE t VALUES ('1', null)")
+      checkAnswer(spark.table("t"), Row("1", null))
+    }
+  }
+
+  test("char type values should be padded: nested in map value") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(i STRING, c MAP<STRING, CHAR(5)>) USING $format")
+      sql("INSERT INTO t VALUES ('1', map('a', 'ab'))")
+      checkAnswer(spark.table("t"), Row("1", Map(("a", "ab" + " " * 3))))
+      checkColType(spark.table("t").schema(1), MapType(StringType, CharType(5)))
+
+      sql("INSERT OVERWRITE t VALUES ('1', null)")
+      checkAnswer(spark.table("t"), Row("1", null))
+      sql("INSERT OVERWRITE t VALUES ('1', map('a', null))")
+      checkAnswer(spark.table("t"), Row("1", Map("a" -> null)))
+    }
+  }
+
+  test("char type values should be padded: nested in both map key and value") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(i STRING, c MAP<CHAR(5), CHAR(10)>) USING $format")
+      sql("INSERT INTO t VALUES ('1', map('a', 'ab'))")
+      checkAnswer(spark.table("t"), Row("1", Map(("a" + " " * 4, "ab" + " " * 8))))
+      checkColType(spark.table("t").schema(1), MapType(CharType(5), CharType(10)))
+
+      sql("INSERT OVERWRITE t VALUES ('1', null)")
+      checkAnswer(spark.table("t"), Row("1", null))
+    }
+  }
+
+  test("char type values should be padded: nested in struct of array") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(i STRING, c STRUCT<c: ARRAY<CHAR(5)>>) USING $format")
+      sql("INSERT INTO t VALUES ('1', struct(array('a', 'ab')))")
+      checkAnswer(spark.table("t"), Row("1", Row(Seq("a" + " " * 4, "ab" + " " * 3))))
+      checkColType(spark.table("t").schema(1),
+        new StructType().add("c", ArrayType(CharType(5))))
+
+      sql("INSERT OVERWRITE t VALUES ('1', null)")
+      checkAnswer(spark.table("t"), Row("1", null))
+      sql("INSERT OVERWRITE t VALUES ('1', struct(null))")
+      checkAnswer(spark.table("t"), Row("1", Row(null)))
+      sql("INSERT OVERWRITE t VALUES ('1', struct(array(null)))")
+      checkAnswer(spark.table("t"), Row("1", Row(Seq(null))))
+    }
+  }
+
+  test("char type values should be padded: nested in array of struct") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(i STRING, c ARRAY<STRUCT<c: CHAR(5)>>) USING $format")
+      sql("INSERT INTO t VALUES ('1', array(struct('a'), struct('ab')))")
+      checkAnswer(spark.table("t"), Row("1", Seq(Row("a" + " " * 4), Row("ab" + " " * 3))))
+      checkColType(spark.table("t").schema(1),
+        ArrayType(new StructType().add("c", CharType(5))))
+
+      sql("INSERT OVERWRITE t VALUES ('1', null)")
+      checkAnswer(spark.table("t"), Row("1", null))
+      sql("INSERT OVERWRITE t VALUES ('1', array(null))")
+      checkAnswer(spark.table("t"), Row("1", Seq(null)))
+      sql("INSERT OVERWRITE t VALUES ('1', array(struct(null)))")
+      checkAnswer(spark.table("t"), Row("1", Seq(Row(null))))
+    }
+  }
+
+  test("char type values should be padded: nested in array of array") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(i STRING, c ARRAY<ARRAY<CHAR(5)>>) USING $format")
+      sql("INSERT INTO t VALUES ('1', array(array('a', 'ab')))")
+      checkAnswer(spark.table("t"), Row("1", Seq(Seq("a" + " " * 4, "ab" + " " * 3))))
+      checkColType(spark.table("t").schema(1), ArrayType(ArrayType(CharType(5))))
+
+      sql("INSERT OVERWRITE t VALUES ('1', null)")
+      checkAnswer(spark.table("t"), Row("1", null))
+      sql("INSERT OVERWRITE t VALUES ('1', array(null))")
+      checkAnswer(spark.table("t"), Row("1", Seq(null)))
+      sql("INSERT OVERWRITE t VALUES ('1', array(array(null)))")
+      checkAnswer(spark.table("t"), Row("1", Seq(Seq(null))))
+    }
+  }
+
+  private def testTableWrite(f: String => Unit): Unit = {
+    withTable("t") { f("char") }
+    withTable("t") { f("varchar") }
+  }
+
+  test("length check for input string values: top-level columns") {
+    testTableWrite { typeName =>
+      sql(s"CREATE TABLE t(c $typeName(5)) USING $format")
+      sql("INSERT INTO t VALUES (null)")
+      checkAnswer(spark.table("t"), Row(null))
+      val e = intercept[SparkException](sql("INSERT INTO t VALUES ('123456')"))
+      assert(e.getCause.getMessage.contains(s"Exceeds char/varchar type length limitation: 5"))
+    }
+  }
+
+  test("length check for input string values: partitioned columns") {
+    // DS V2 doesn't support partitioned table.
+    if (!conf.contains(SQLConf.DEFAULT_CATALOG.key)) {
+      testTableWrite { typeName =>
+        sql(s"CREATE TABLE t(i INT, c $typeName(5)) USING $format PARTITIONED BY (c)")
+        sql("INSERT INTO t VALUES (1, null)")
+        checkAnswer(spark.table("t"), Row(1, null))
+        val e = intercept[SparkException](sql("INSERT INTO t VALUES (1, '123456')"))
+        assert(e.getCause.getMessage.contains(s"Exceeds char/varchar type length limitation: 5"))
+      }
+    }
+  }
+
+  test("length check for input string values: nested in struct") {
+    testTableWrite { typeName =>
+      sql(s"CREATE TABLE t(c STRUCT<c: $typeName(5)>) USING $format")
+      sql("INSERT INTO t SELECT struct(null)")
+      checkAnswer(spark.table("t"), Row(Row(null)))
+      val e = intercept[SparkException](sql("INSERT INTO t SELECT struct('123456')"))
+      assert(e.getCause.getMessage.contains(s"Exceeds char/varchar type length limitation: 5"))
+    }
+  }
+
+  test("length check for input string values: nested in array") {
+    testTableWrite { typeName =>
+      sql(s"CREATE TABLE t(c ARRAY<$typeName(5)>) USING $format")
+      sql("INSERT INTO t VALUES (array(null))")
+      checkAnswer(spark.table("t"), Row(Seq(null)))
+      val e = intercept[SparkException](sql("INSERT INTO t VALUES (array('a', '123456'))"))
+      assert(e.getCause.getMessage.contains(s"Exceeds char/varchar type length limitation: 5"))
+    }
+  }
+
+  test("length check for input string values: nested in map key") {
+    testTableWrite { typeName =>
+      sql(s"CREATE TABLE t(c MAP<$typeName(5), STRING>) USING $format")
+      val e = intercept[SparkException](sql("INSERT INTO t VALUES (map('123456', 'a'))"))
+      assert(e.getCause.getMessage.contains(s"Exceeds char/varchar type length limitation: 5"))
+    }
+  }
+
+  test("length check for input string values: nested in map value") {
+    testTableWrite { typeName =>
+      sql(s"CREATE TABLE t(c MAP<STRING, $typeName(5)>) USING $format")
+      sql("INSERT INTO t VALUES (map('a', null))")
+      checkAnswer(spark.table("t"), Row(Map("a" -> null)))
+      val e = intercept[SparkException](sql("INSERT INTO t VALUES (map('a', '123456'))"))
+      assert(e.getCause.getMessage.contains(s"Exceeds char/varchar type length limitation: 5"))
+    }
+  }
+
+  test("length check for input string values: nested in both map key and value") {
+    testTableWrite { typeName =>
+      sql(s"CREATE TABLE t(c MAP<$typeName(5), $typeName(5)>) USING $format")
+      val e1 = intercept[SparkException](sql("INSERT INTO t VALUES (map('123456', 'a'))"))
+      assert(e1.getCause.getMessage.contains(s"Exceeds char/varchar type length limitation: 5"))
+      val e2 = intercept[SparkException](sql("INSERT INTO t VALUES (map('a', '123456'))"))
+      assert(e2.getCause.getMessage.contains(s"Exceeds char/varchar type length limitation: 5"))
+    }
+  }
+
+  test("length check for input string values: nested in struct of array") {
+    testTableWrite { typeName =>
+      sql(s"CREATE TABLE t(c STRUCT<c: ARRAY<$typeName(5)>>) USING $format")
+      sql("INSERT INTO t SELECT struct(array(null))")
+      checkAnswer(spark.table("t"), Row(Row(Seq(null))))
+      val e = intercept[SparkException](sql("INSERT INTO t SELECT struct(array('123456'))"))
+      assert(e.getCause.getMessage.contains(s"Exceeds char/varchar type length limitation: 5"))
+    }
+  }
+
+  test("length check for input string values: nested in array of struct") {
+    testTableWrite { typeName =>
+      sql(s"CREATE TABLE t(c ARRAY<STRUCT<c: $typeName(5)>>) USING $format")
+      sql("INSERT INTO t VALUES (array(struct(null)))")
+      checkAnswer(spark.table("t"), Row(Seq(Row(null))))
+      val e = intercept[SparkException](sql("INSERT INTO t VALUES (array(struct('123456')))"))
+      assert(e.getCause.getMessage.contains(s"Exceeds char/varchar type length limitation: 5"))
+    }
+  }
+
+  test("length check for input string values: nested in array of array") {
+    testTableWrite { typeName =>
+      sql(s"CREATE TABLE t(c ARRAY<ARRAY<$typeName(5)>>) USING $format")
+      sql("INSERT INTO t VALUES (array(array(null)))")
+      checkAnswer(spark.table("t"), Row(Seq(Seq(null))))
+      val e = intercept[SparkException](sql("INSERT INTO t VALUES (array(array('123456')))"))
+      assert(e.getCause.getMessage.contains(s"Exceeds char/varchar type length limitation: 5"))
+    }
+  }
+
+  test("length check for input string values: with trailing spaces") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(c1 CHAR(5), c2 VARCHAR(5)) USING $format")
+      sql("INSERT INTO t VALUES ('12 ', '12 ')")
+      sql("INSERT INTO t VALUES ('1234  ', '1234  ')")
+      checkAnswer(spark.table("t"), Seq(
+        Row("12" + " " * 3, "12 "),
+        Row("1234 ", "1234 ")))
+    }
+  }
+
+  test("length check for input string values: with implicit cast") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(c1 CHAR(5), c2 VARCHAR(5)) USING $format")
+      sql("INSERT INTO t VALUES (1234, 1234)")
+      checkAnswer(spark.table("t"), Row("1234 ", "1234"))
+      val e1 = intercept[SparkException](sql("INSERT INTO t VALUES (123456, 1)"))
+      assert(e1.getCause.getMessage.contains("Exceeds char/varchar type length limitation: 5"))
+      val e2 = intercept[SparkException](sql("INSERT INTO t VALUES (1, 123456)"))
+      assert(e2.getCause.getMessage.contains("Exceeds char/varchar type length limitation: 5"))
+    }
+  }
+
+  private def testConditions(df: DataFrame, conditions: Seq[(String, Boolean)]): Unit = {
+    checkAnswer(df.selectExpr(conditions.map(_._1): _*), Row.fromSeq(conditions.map(_._2)))
+  }
+
+  test("char type comparison: top-level columns") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(c1 CHAR(2), c2 CHAR(5)) USING $format")
+      sql("INSERT INTO t VALUES ('a', 'a')")
+      testConditions(spark.table("t"), Seq(
+        ("c1 = 'a'", true),
+        ("'a' = c1", true),
+        ("c1 = 'a  '", true),
+        ("c1 > 'a'", false),
+        ("c1 IN ('a', 'b')", true),
+        ("c1 = c2", true),
+        ("c1 < c2", false),
+        ("c1 IN (c2)", true),
+        ("c1 <=> null", false)))
+    }
+  }
+
+  test("char type comparison: partitioned columns") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(i INT, c1 CHAR(2), c2 CHAR(5)) USING $format PARTITIONED BY (c1, c2)")
+      sql("INSERT INTO t VALUES (1, 'a', 'a')")
+      testConditions(spark.table("t"), Seq(
+        ("c1 = 'a'", true),
+        ("'a' = c1", true),
+        ("c1 = 'a  '", true),
+        ("c1 > 'a'", false),
+        ("c1 IN ('a', 'b')", true),
+        ("c1 = c2", true),
+        ("c1 < c2", false),
+        ("c1 IN (c2)", true),
+        ("c1 <=> null", false)))
+    }
+  }
+
+  private def testNullConditions(df: DataFrame, conditions: Seq[String]): Unit = {
+    conditions.foreach { cond =>
+      checkAnswer(df.selectExpr(cond), Row(null))
+    }
+  }
+
+  test("SPARK-34233: char type comparison with null values") {
+    val conditions = Seq("c = null", "c IN ('e', null)", "c IN (null)")
+    withTable("t") {
+      sql(s"CREATE TABLE t(c CHAR(2)) USING $format")
+      sql("INSERT INTO t VALUES ('a')")
+      testNullConditions(spark.table("t"), conditions)
+    }
+
+    withTable("t") {
+      sql(s"CREATE TABLE t(i INT, c CHAR(2)) USING $format PARTITIONED BY (c)")
+      sql("INSERT INTO t VALUES (1, 'a')")
+      testNullConditions(spark.table("t"), conditions)
+    }
+  }
+
+  test("char type comparison: partition pruning") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(i INT, c1 CHAR(2), c2 VARCHAR(5)) USING $format PARTITIONED BY (c1, c2)")
+      sql("INSERT INTO t VALUES (1, 'a', 'a')")
+      Seq(("c1 = 'a'", true),
+        ("'a' = c1", true),
+        ("c1 = 'a  '", true),
+        ("c1 > 'a'", false),
+        ("c1 IN ('a', 'b')", true),
+        ("c2 = 'a  '", false),
+        ("c2 = 'a'", true),
+        ("c2 IN ('a', 'b')", true)).foreach { case (con, res) =>
+        val df = spark.table("t")
+        withClue(con) {
+          checkAnswer(df.where(con), df.where(res.toString))
+        }
+      }
+    }
+  }
+
+  test("char type comparison: join") {
+    withTable("t1", "t2") {
+      sql(s"CREATE TABLE t1(c CHAR(2)) USING $format")
+      sql(s"CREATE TABLE t2(c CHAR(5)) USING $format")
+      sql("INSERT INTO t1 VALUES ('a')")
+      sql("INSERT INTO t2 VALUES ('a')")
+      checkAnswer(sql("SELECT t1.c FROM t1 JOIN t2 ON t1.c = t2.c"), Row("a "))
+    }
+  }
+
+  test("char type comparison: nested in struct") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(c1 STRUCT<c: CHAR(2)>, c2 STRUCT<c: CHAR(5)>) USING $format")
+      sql("INSERT INTO t VALUES (struct('a'), struct('a'))")
+      testConditions(spark.table("t"), Seq(
+        ("c1 = c2", true),
+        ("c1 < c2", false),
+        ("c1 IN (c2)", true)))
+    }
+  }
+
+  test("char type comparison: nested in array") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(c1 ARRAY<CHAR(2)>, c2 ARRAY<CHAR(5)>) USING $format")
+      sql("INSERT INTO t VALUES (array('a', 'b'), array('a', 'b'))")
+      testConditions(spark.table("t"), Seq(
+        ("c1 = c2", true),
+        ("c1 < c2", false),
+        ("c1 IN (c2)", true)))
+    }
+  }
+
+  test("char type comparison: nested in struct of array") {
+    withTable("t") {
+      sql("CREATE TABLE t(c1 STRUCT<a: ARRAY<CHAR(2)>>, c2 STRUCT<a: ARRAY<CHAR(5)>>) " +
+        s"USING $format")
+      sql("INSERT INTO t VALUES (struct(array('a', 'b')), struct(array('a', 'b')))")
+      testConditions(spark.table("t"), Seq(
+        ("c1 = c2", true),
+        ("c1 < c2", false),
+        ("c1 IN (c2)", true)))
+    }
+  }
+
+  test("char type comparison: nested in array of struct") {
+    withTable("t") {
+      sql("CREATE TABLE t(c1 ARRAY<STRUCT<c: CHAR(2)>>, c2 ARRAY<STRUCT<c: CHAR(5)>>) " +
+        s"USING $format")
+      sql("INSERT INTO t VALUES (array(struct('a')), array(struct('a')))")
+      testConditions(spark.table("t"), Seq(
+        ("c1 = c2", true),
+        ("c1 < c2", false),
+        ("c1 IN (c2)", true)))
+    }
+  }
+
+  test("char type comparison: nested in array of array") {
+    withTable("t") {
+      sql("CREATE TABLE t(c1 ARRAY<ARRAY<CHAR(2)>>, c2 ARRAY<ARRAY<CHAR(5)>>) " +
+        s"USING $format")
+      sql("INSERT INTO t VALUES (array(array('a')), array(array('a')))")
+      testConditions(spark.table("t"), Seq(
+        ("c1 = c2", true),
+        ("c1 < c2", false),
+        ("c1 IN (c2)", true)))
+    }
+  }
+
+  test("SPARK-33892: DESCRIBE TABLE w/ char/varchar") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(v VARCHAR(3), c CHAR(5)) USING $format")
+      checkAnswer(sql("desc t").selectExpr("data_type").where("data_type like '%char%'"),
+        Seq(Row("char(5)"), Row("varchar(3)")))
+    }
+  }
+
+  test("SPARK-34003: fix char/varchar fails w/ both group by and order by ") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(v VARCHAR(3), i INT) USING $format")
+      sql("INSERT INTO t VALUES ('c', 1)")
+      checkAnswer(sql("SELECT v, sum(i) FROM t GROUP BY v ORDER BY v"), Row("c", 1))
+    }
+  }
+
+  test("SPARK-34003: fix char/varchar fails w/ order by functions") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(v VARCHAR(3), i INT) USING $format")
+      sql("INSERT INTO t VALUES ('c', 1)")
+      checkAnswer(sql("SELECT substr(v, 1, 2), sum(i) FROM t GROUP BY v ORDER BY substr(v, 1, 2)"),
+        Row("c", 1))
+      checkAnswer(sql("SELECT sum(i) FROM t GROUP BY v ORDER BY substr(v, 1, 2)"),
+        Row(1))
+    }
+  }
+
+  test("SPARK-34114: varchar type will strip tailing spaces to certain length at write time") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(v VARCHAR(3)) USING $format")
+      sql("INSERT INTO t VALUES ('c      ')")
+      checkAnswer(spark.table("t"), Row("c  "))
+    }
+  }
+
+  test("SPARK-34114: varchar type will remain the value length with spaces at read time") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(v VARCHAR(3)) USING $format")
+      sql("INSERT INTO t VALUES ('c ')")
+      checkAnswer(spark.table("t"), Row("c "))
+    }
+  }
+
+  test("SPARK-34833: right-padding applied correctly for correlated subqueries - join keys") {
+    withTable("t1", "t2") {
+      sql(s"CREATE TABLE t1(v VARCHAR(3), c CHAR(5)) USING $format")
+      sql(s"CREATE TABLE t2(v VARCHAR(5), c CHAR(8)) USING $format")
+      sql("INSERT INTO t1 VALUES ('c', 'b')")
+      sql("INSERT INTO t2 VALUES ('a', 'b')")
+      Seq("t1.c = t2.c", "t2.c = t1.c",
+        "t1.c = 'b'", "'b' = t1.c", "t1.c = 'b    '", "'b    ' = t1.c",
+        "t1.c = 'b      '", "'b      ' = t1.c").foreach { predicate =>
+        checkAnswer(sql(
+          s"""
+             |SELECT v FROM t1
+             |WHERE 'a' IN (SELECT v FROM t2 WHERE $predicate)
+           """.stripMargin),
+          Row("c"))
+      }
+    }
+  }
+
+  test("SPARK-34833: right-padding applied correctly for correlated subqueries - other preds") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(c0 INT, c1 CHAR(5), c2 CHAR(7)) USING $format")
+      sql("INSERT INTO t VALUES (1, 'abc', 'abc')")
+      Seq("c1 = 'abc'", "'abc' = c1", "c1 = 'abc  '", "'abc  ' = c1",
+        "c1 = 'abc    '", "'abc    ' = c1", "c1 = c2", "c2 = c1",
+        "c1 IN ('xxx', 'abc', 'xxxxx')", "c1 IN ('xxx', 'abc  ', 'xxxxx')",
+        "c1 IN ('xxx', 'abc    ', 'xxxxx')",
+        "c1 IN (c2)", "c2 IN (c1)").foreach { predicate =>
+        checkAnswer(sql(
+          s"""
+             |SELECT c0 FROM t t1
+             |WHERE (
+             |  SELECT count(*) AS c
+             |  FROM t
+             |  WHERE c0 = t1.c0 AND $predicate
+             |) > 0
+         """.stripMargin),
+          Row(1))
+      }
+    }
+  }
+}
+
+// Some basic char/varchar tests which doesn't rely on table implementation.
+class BasicCharVarcharTestSuite extends QueryTest with SharedSparkSession {
+  import testImplicits._
+
+  test("user-specified schema in cast") {
+    def assertNoCharType(df: DataFrame): Unit = {
+      checkAnswer(df, Row("0"))
+      assert(df.schema.map(_.dataType) == Seq(StringType))
+    }
+
+    val logAppender = new LogAppender("The Spark cast operator does not support char/varchar" +
+      " type and simply treats them as string type. Please use string type directly to avoid" +
+      " confusion.")
+    withLogAppender(logAppender) {
+      assertNoCharType(spark.range(1).select($"id".cast("char(5)")))
+      assertNoCharType(spark.range(1).select($"id".cast(CharType(5))))
+      assertNoCharType(spark.range(1).selectExpr("CAST(id AS CHAR(5))"))
+      assertNoCharType(sql("SELECT CAST(id AS CHAR(5)) FROM range(1)"))
+    }
+  }
+
+  def failWithInvalidCharUsage[T](fn: => T): Unit = {
+    val e = intercept[AnalysisException](fn)
+    assert(e.getMessage contains "char/varchar type can only be used in the table schema")
+  }
+
+  test("invalidate char/varchar in functions") {
+    failWithInvalidCharUsage(sql("""SELECT from_json('{"a": "str"}', 'a CHAR(5)')"""))
+    withSQLConf((SQLConf.LEGACY_CHAR_VARCHAR_AS_STRING.key, "true")) {
+      val df = sql("""SELECT from_json('{"a": "str"}', 'a CHAR(5)')""")
+      checkAnswer(df, Row(Row("str")))
+      val schema = df.schema.head.dataType.asInstanceOf[StructType]
+      assert(schema.map(_.dataType) == Seq(StringType))
+    }
+  }
+
+  test("invalidate char/varchar in SparkSession createDataframe") {
+    val df = spark.range(10).map(_.toString).toDF()
+    val schema = new StructType().add("id", CharType(5))
+    failWithInvalidCharUsage(spark.createDataFrame(df.collectAsList(), schema))
+    failWithInvalidCharUsage(spark.createDataFrame(df.rdd, schema))
+    failWithInvalidCharUsage(spark.createDataFrame(df.toJavaRDD, schema))
+    withSQLConf((SQLConf.LEGACY_CHAR_VARCHAR_AS_STRING.key, "true")) {
+      val df1 = spark.createDataFrame(df.collectAsList(), schema)
+      checkAnswer(df1, df)
+      assert(df1.schema.head.dataType === StringType)
+    }
+  }
+
+  test("invalidate char/varchar in spark.read.schema") {
+    failWithInvalidCharUsage(spark.read.schema(new StructType().add("id", CharType(5))))
+    failWithInvalidCharUsage(spark.read.schema("id char(5)"))
+    withSQLConf((SQLConf.LEGACY_CHAR_VARCHAR_AS_STRING.key, "true")) {
+      val ds = spark.range(10).map(_.toString)
+      val df1 = spark.read.schema(new StructType().add("id", CharType(5))).csv(ds)
+      assert(df1.schema.map(_.dataType) == Seq(StringType))
+      val df2 = spark.read.schema("id char(5)").csv(ds)
+      assert(df2.schema.map(_.dataType) == Seq(StringType))
+
+      def checkSchema(df: DataFrame): Unit = {
+        val schemas = df.queryExecution.analyzed.collect {
+          case l: LogicalRelation => l.relation.schema
+          case d: DataSourceV2Relation => d.table.schema()
+        }
+        assert(schemas.length == 1)
+        assert(schemas.head.map(_.dataType) == Seq(StringType))
+      }
+
+      // user-specified schema in DataFrameReader: DSV1
+      checkSchema(spark.read.schema(new StructType().add("id", CharType(5)))
+        .format(classOf[SimpleInsertSource].getName).load())
+      checkSchema(spark.read.schema("id char(5)")
+        .format(classOf[SimpleInsertSource].getName).load())
+
+      // user-specified schema in DataFrameReader: DSV2
+      checkSchema(spark.read.schema(new StructType().add("id", CharType(5)))
+        .format(classOf[SchemaRequiredDataSource].getName).load())
+      checkSchema(spark.read.schema("id char(5)")
+        .format(classOf[SchemaRequiredDataSource].getName).load())
+    }
+  }
+
+  test("invalidate char/varchar in udf's result type") {
+    failWithInvalidCharUsage(spark.udf.register("testchar", () => "B", VarcharType(1)))
+    failWithInvalidCharUsage(spark.udf.register("testchar2", (x: String) => x, VarcharType(1)))
+    withSQLConf((SQLConf.LEGACY_CHAR_VARCHAR_AS_STRING.key, "true")) {
+      spark.udf.register("testchar", () => "B", VarcharType(1))
+      spark.udf.register("testchar2", (x: String) => x, VarcharType(1))
+      val df1 = spark.sql("select testchar()")
+      checkAnswer(df1, Row("B"))
+      assert(df1.schema.head.dataType === StringType)
+      val df2 = spark.sql("select testchar2('abc')")
+      checkAnswer(df2, Row("abc"))
+      assert(df2.schema.head.dataType === StringType)
+    }
+  }
+
+  test("invalidate char/varchar in spark.readStream.schema") {
+    failWithInvalidCharUsage(spark.readStream.schema(new StructType().add("id", CharType(5))))
+    failWithInvalidCharUsage(spark.readStream.schema("id char(5)"))
+    withSQLConf((SQLConf.LEGACY_CHAR_VARCHAR_AS_STRING.key, "true")) {
+      withTempPath { dir =>
+        spark.range(2).write.save(dir.toString)
+        val df1 = spark.readStream.schema(new StructType().add("id", CharType(5)))
+          .load(dir.toString)
+        assert(df1.schema.map(_.dataType) == Seq(StringType))
+        val df2 = spark.readStream.schema("id char(5)").load(dir.toString)
+        assert(df2.schema.map(_.dataType) == Seq(StringType))
+      }
+    }
+  }
+}
+
+class FileSourceCharVarcharTestSuite extends CharVarcharTestSuite with SharedSparkSession {
+  override def format: String = "parquet"
+  override protected def sparkConf: SparkConf = {
+    super.sparkConf.set(SQLConf.USE_V1_SOURCE_LIST, "parquet")
+  }
+
+  test("create table w/ location and fit length values") {
+    Seq("char", "varchar").foreach { typ =>
+      withTempPath { dir =>
+        withTable("t") {
+          sql("SELECT '12' as col").write.format(format).save(dir.toString)
+          sql(s"CREATE TABLE t (col $typ(2)) using $format LOCATION '$dir'")
+          val df = sql("select * from t")
+          checkAnswer(sql("select * from t"), Row("12"))
+        }
+      }
+    }
+  }
+
+  test("create table w/ location and over length values") {
+    Seq("char", "varchar").foreach { typ =>
+      withTempPath { dir =>
+        withTable("t") {
+          sql("SELECT '123456' as col").write.format(format).save(dir.toString)
+          sql(s"CREATE TABLE t (col $typ(2)) using $format LOCATION '$dir'")
+          checkAnswer(sql("select * from t"), Row("123456"))
+        }
+      }
+    }
+  }
+
+  test("alter table set location w/ fit length values") {
+    Seq("char", "varchar").foreach { typ =>
+      withTempPath { dir =>
+        withTable("t") {
+          sql("SELECT '12' as col").write.format(format).save(dir.toString)
+          sql(s"CREATE TABLE t (col $typ(2)) using $format")
+          sql(s"ALTER TABLE t SET LOCATION '$dir'")
+          checkAnswer(spark.table("t"), Row("12"))
+        }
+      }
+    }
+  }
+
+  test("alter table set location w/ over length values") {
+    Seq("char", "varchar").foreach { typ =>
+      withTempPath { dir =>
+        withTable("t") {
+          sql("SELECT '123456' as col").write.format(format).save(dir.toString)
+          sql(s"CREATE TABLE t (col $typ(2)) using $format")
+          sql(s"ALTER TABLE t SET LOCATION '$dir'")
+          checkAnswer(spark.table("t"), Row("123456"))
+        }
+      }
+    }
+  }
+
+  // TODO(SPARK-33875): Move these tests to super after DESCRIBE COLUMN v2 implemented
+  test("SPARK-33892: DESCRIBE COLUMN w/ char/varchar") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(v VARCHAR(3), c CHAR(5)) USING $format")
+      checkAnswer(sql("desc t v").selectExpr("info_value").where("info_value like '%char%'"),
+        Row("varchar(3)"))
+      checkAnswer(sql("desc t c").selectExpr("info_value").where("info_value like '%char%'"),
+        Row("char(5)"))
+    }
+  }
+
+  // TODO(SPARK-33898): Move these tests to super after SHOW CREATE TABLE for v2 implemented
+  test("SPARK-33892: SHOW CREATE TABLE w/ char/varchar") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(v VARCHAR(3), c CHAR(5)) USING $format")
+      val rest = sql("SHOW CREATE TABLE t").head().getString(0)
+      assert(rest.contains("VARCHAR(3)"))
+      assert(rest.contains("CHAR(5)"))
+    }
+  }
+
+  test("SPARK-34114: should not trim right for read-side length check and char padding") {
+    Seq("char", "varchar").foreach { typ =>
+      withTempPath { dir =>
+        withTable("t") {
+          sql("SELECT '12  ' as col").write.format(format).save(dir.toString)
+          sql(s"CREATE TABLE t (col $typ(2)) using $format LOCATION '$dir'")
+          checkAnswer(spark.table("t"), Row("12  "))
+        }
+      }
+    }
+  }
+}
+
+class DSV2CharVarcharTestSuite extends CharVarcharTestSuite
+  with SharedSparkSession {
+  override def format: String = "foo"
+  protected override def sparkConf = {
+    super.sparkConf
+      .set("spark.sql.catalog.testcat", classOf[InMemoryPartitionTableCatalog].getName)
+      .set(SQLConf.DEFAULT_CATALOG.key, "testcat")
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
index fa06484a73d95..ad5d73c774274 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
@@ -18,15 +18,19 @@
 package org.apache.spark.sql
 
 import java.sql.{Date, Timestamp}
+import java.time.{Duration, Instant, LocalDate, LocalDateTime, Period}
+import java.time.temporal.ChronoUnit
 import java.util.Locale
 
-import scala.collection.JavaConverters._
-
 import org.apache.hadoop.io.{LongWritable, Text}
 import org.apache.hadoop.mapreduce.lib.input.{TextInputFormat => NewTextInputFormat}
-import org.scalatest.Matchers._
+import org.scalatest.matchers.should.Matchers._
 
+import org.apache.spark.SparkException
+import org.apache.spark.sql.UpdateFieldsBenchmark._
 import org.apache.spark.sql.catalyst.expressions.{InSet, Literal, NamedExpression}
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{outstandingTimezonesIds, outstandingZoneIds}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.execution.ProjectExec
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
@@ -155,6 +159,28 @@ class ColumnExpressionSuite extends QueryTest with SharedSparkSession {
     checkAnswer(testData.as("testData").select($"testData.*"), testData.collect().toSeq)
   }
 
+  test("SPARK-34199: star can be qualified by table name inside a non-count function") {
+    checkAnswer(
+      testData.as("testData").selectExpr("hash(testData.*)"),
+      testData.as("testData").selectExpr("hash(testData.key, testData.value)")
+    )
+  }
+
+  test("SPARK-34199: star cannot be qualified by table name inside a count function") {
+    val e = intercept[AnalysisException] {
+      testData.as("testData").selectExpr("count(testData.*)").collect()
+    }
+    assert(e.getMessage.contains(
+      "count(testData.*) is not allowed. Please use count(*) or expand the columns manually"))
+  }
+
+  test("SPARK-34199: table star can be qualified inside a count function with multiple arguments") {
+    checkAnswer(
+      testData.as("testData").selectExpr("count(testData.*, testData.key)"),
+      testData.as("testData").selectExpr("count(testData.key, testData.value, testData.key)")
+    )
+  }
+
   test("+") {
     checkAnswer(
       testData2.select($"a" + 1),
@@ -923,4 +949,1933 @@ class ColumnExpressionSuite extends QueryTest with SharedSparkSession {
     val inSet = InSet(Literal("a"), Set("a", "b").map(UTF8String.fromString))
     assert(inSet.sql === "('a' IN ('a', 'b'))")
   }
+
+  def checkAnswer(
+      df: => DataFrame,
+      expectedAnswer: Seq[Row],
+      expectedSchema: StructType): Unit = {
+    checkAnswer(df, expectedAnswer)
+    assert(df.schema == expectedSchema)
+  }
+
+  private lazy val structType = StructType(Seq(
+    StructField("a", IntegerType, nullable = false),
+    StructField("b", IntegerType, nullable = true),
+    StructField("c", IntegerType, nullable = false)))
+
+  private lazy val structLevel1: DataFrame = spark.createDataFrame(
+    sparkContext.parallelize(Row(Row(1, null, 3)) :: Nil),
+    StructType(Seq(StructField("a", structType, nullable = false))))
+
+  private lazy val nullableStructLevel1: DataFrame = spark.createDataFrame(
+    sparkContext.parallelize(Row(null) :: Row(Row(1, null, 3)) :: Nil),
+    StructType(Seq(StructField("a", structType, nullable = true))))
+
+  private lazy val structLevel2: DataFrame = spark.createDataFrame(
+    sparkContext.parallelize(Row(Row(Row(1, null, 3))) :: Nil),
+    StructType(Seq(
+      StructField("a", StructType(Seq(
+        StructField("a", structType, nullable = false))),
+        nullable = false))))
+
+  private lazy val nullableStructLevel2: DataFrame = spark.createDataFrame(
+    sparkContext.parallelize(Row(null) :: Row(Row(null)) :: Row(Row(Row(1, null, 3))) :: Nil),
+    StructType(Seq(
+      StructField("a", StructType(Seq(
+        StructField("a", structType, nullable = true))),
+        nullable = true))))
+
+  private lazy val structLevel3: DataFrame = spark.createDataFrame(
+    sparkContext.parallelize(Row(Row(Row(Row(1, null, 3)))) :: Nil),
+    StructType(Seq(
+      StructField("a", StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", structType, nullable = false))),
+          nullable = false))),
+        nullable = false))))
+
+  test("withField should throw an exception if called on a non-StructType column") {
+    intercept[AnalysisException] {
+      testData.withColumn("key", $"key".withField("a", lit(2)))
+    }.getMessage should include("struct argument should be struct type, got: int")
+  }
+
+  test("withField should throw an exception if either fieldName or col argument are null") {
+    intercept[IllegalArgumentException] {
+      structLevel1.withColumn("a", $"a".withField(null, lit(2)))
+    }.getMessage should include("fieldName cannot be null")
+
+    intercept[IllegalArgumentException] {
+      structLevel1.withColumn("a", $"a".withField("b", null))
+    }.getMessage should include("col cannot be null")
+
+    intercept[IllegalArgumentException] {
+      structLevel1.withColumn("a", $"a".withField(null, null))
+    }.getMessage should include("fieldName cannot be null")
+  }
+
+  test("withField should throw an exception if any intermediate structs don't exist") {
+    intercept[AnalysisException] {
+      structLevel2.withColumn("a", 'a.withField("x.b", lit(2)))
+    }.getMessage should include("No such struct field x in a")
+
+    intercept[AnalysisException] {
+      structLevel3.withColumn("a", 'a.withField("a.x.b", lit(2)))
+    }.getMessage should include("No such struct field x in a")
+  }
+
+  test("withField should throw an exception if intermediate field is not a struct") {
+    intercept[AnalysisException] {
+      structLevel1.withColumn("a", 'a.withField("b.a", lit(2)))
+    }.getMessage should include("struct argument should be struct type, got: int")
+  }
+
+  test("withField should throw an exception if intermediate field reference is ambiguous") {
+    intercept[AnalysisException] {
+      val structLevel2: DataFrame = spark.createDataFrame(
+        sparkContext.parallelize(Row(Row(Row(1, null, 3), 4)) :: Nil),
+        StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", structType, nullable = false),
+            StructField("a", structType, nullable = false))),
+            nullable = false))))
+
+      structLevel2.withColumn("a", 'a.withField("a.b", lit(2)))
+    }.getMessage should include("Ambiguous reference to fields")
+  }
+
+  test("withField should add field with no name") {
+    checkAnswer(
+      structLevel1.withColumn("a", $"a".withField("", lit(4))),
+      Row(Row(1, null, 3, 4)) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", IntegerType, nullable = false),
+          StructField("b", IntegerType, nullable = true),
+          StructField("c", IntegerType, nullable = false),
+          StructField("", IntegerType, nullable = false))),
+          nullable = false))))
+  }
+
+  test("withField should add field to struct") {
+    checkAnswer(
+      structLevel1.withColumn("a", 'a.withField("d", lit(4))),
+      Row(Row(1, null, 3, 4)) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", IntegerType, nullable = false),
+          StructField("b", IntegerType, nullable = true),
+          StructField("c", IntegerType, nullable = false),
+          StructField("d", IntegerType, nullable = false))),
+          nullable = false))))
+  }
+
+  test("withField should add field to nullable struct") {
+    checkAnswer(
+      nullableStructLevel1.withColumn("a", $"a".withField("d", lit(4))),
+      Row(null) :: Row(Row(1, null, 3, 4)) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", IntegerType, nullable = false),
+          StructField("b", IntegerType, nullable = true),
+          StructField("c", IntegerType, nullable = false),
+          StructField("d", IntegerType, nullable = false))),
+          nullable = true))))
+  }
+
+  test("withField should add field to nested nullable struct") {
+    checkAnswer(
+      nullableStructLevel2.withColumn("a", $"a".withField("a.d", lit(4))),
+      Row(null) :: Row(Row(null)) :: Row(Row(Row(1, null, 3, 4))) :: Nil,
+      StructType(
+        Seq(StructField("a", StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", IntegerType, nullable = false),
+            StructField("b", IntegerType, nullable = true),
+            StructField("c", IntegerType, nullable = false),
+            StructField("d", IntegerType, nullable = false))),
+            nullable = true))),
+          nullable = true))))
+  }
+
+  test("withField should add null field to struct") {
+    checkAnswer(
+      structLevel1.withColumn("a", 'a.withField("d", lit(null).cast(IntegerType))),
+      Row(Row(1, null, 3, null)) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", IntegerType, nullable = false),
+          StructField("b", IntegerType, nullable = true),
+          StructField("c", IntegerType, nullable = false),
+          StructField("d", IntegerType, nullable = true))),
+          nullable = false))))
+  }
+
+  test("withField should add multiple fields to struct") {
+    checkAnswer(
+      structLevel1.withColumn("a", 'a.withField("d", lit(4)).withField("e", lit(5))),
+      Row(Row(1, null, 3, 4, 5)) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", IntegerType, nullable = false),
+          StructField("b", IntegerType, nullable = true),
+          StructField("c", IntegerType, nullable = false),
+          StructField("d", IntegerType, nullable = false),
+          StructField("e", IntegerType, nullable = false))),
+          nullable = false))))
+  }
+
+  test("withField should add multiple fields to nullable struct") {
+    checkAnswer(
+      nullableStructLevel1.withColumn("a", 'a.withField("d", lit(4)).withField("e", lit(5))),
+      Row(null) :: Row(Row(1, null, 3, 4, 5)) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", IntegerType, nullable = false),
+          StructField("b", IntegerType, nullable = true),
+          StructField("c", IntegerType, nullable = false),
+          StructField("d", IntegerType, nullable = false),
+          StructField("e", IntegerType, nullable = false))),
+          nullable = true))))
+  }
+
+  test("withField should add field to nested struct") {
+    Seq(
+      structLevel2.withColumn("a", 'a.withField("a.d", lit(4))),
+      structLevel2.withColumn("a", 'a.withField("a", $"a.a".withField("d", lit(4))))
+    ).foreach { df =>
+      checkAnswer(
+        df,
+        Row(Row(Row(1, null, 3, 4))) :: Nil,
+        StructType(
+          Seq(StructField("a", StructType(Seq(
+            StructField("a", StructType(Seq(
+              StructField("a", IntegerType, nullable = false),
+              StructField("b", IntegerType, nullable = true),
+              StructField("c", IntegerType, nullable = false),
+              StructField("d", IntegerType, nullable = false))),
+              nullable = false))),
+            nullable = false))))
+    }
+  }
+
+  test("withField should add multiple fields to nested struct") {
+    Seq(
+      col("a").withField("a", $"a.a".withField("d", lit(4)).withField("e", lit(5))),
+      col("a").withField("a.d", lit(4)).withField("a.e", lit(5))
+    ).foreach { column =>
+      checkAnswer(
+        structLevel2.select(column.as("a")),
+        Row(Row(Row(1, null, 3, 4, 5))) :: Nil,
+        StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", StructType(Seq(
+              StructField("a", IntegerType, nullable = false),
+              StructField("b", IntegerType, nullable = true),
+              StructField("c", IntegerType, nullable = false),
+              StructField("d", IntegerType, nullable = false),
+              StructField("e", IntegerType, nullable = false))),
+              nullable = false))),
+            nullable = false))))
+    }
+  }
+
+  test("withField should add multiple fields to nested nullable struct") {
+    Seq(
+      col("a").withField("a", $"a.a".withField("d", lit(4)).withField("e", lit(5))),
+      col("a").withField("a.d", lit(4)).withField("a.e", lit(5))
+    ).foreach { column =>
+      checkAnswer(
+        nullableStructLevel2.select(column.as("a")),
+        Row(null) :: Row(Row(null)) :: Row(Row(Row(1, null, 3, 4, 5))) :: Nil,
+        StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", StructType(Seq(
+              StructField("a", IntegerType, nullable = false),
+              StructField("b", IntegerType, nullable = true),
+              StructField("c", IntegerType, nullable = false),
+              StructField("d", IntegerType, nullable = false),
+              StructField("e", IntegerType, nullable = false))),
+              nullable = true))),
+            nullable = true))))
+    }
+  }
+
+  test("withField should add field to deeply nested struct") {
+    checkAnswer(
+      structLevel3.withColumn("a", 'a.withField("a.a.d", lit(4))),
+      Row(Row(Row(Row(1, null, 3, 4)))) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", StructType(Seq(
+              StructField("a", IntegerType, nullable = false),
+              StructField("b", IntegerType, nullable = true),
+              StructField("c", IntegerType, nullable = false),
+              StructField("d", IntegerType, nullable = false))),
+              nullable = false))),
+            nullable = false))),
+          nullable = false))))
+  }
+
+  test("withField should replace field in struct") {
+    checkAnswer(
+      structLevel1.withColumn("a", 'a.withField("b", lit(2))),
+      Row(Row(1, 2, 3)) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", IntegerType, nullable = false),
+          StructField("b", IntegerType, nullable = false),
+          StructField("c", IntegerType, nullable = false))),
+          nullable = false))))
+  }
+
+  test("withField should replace field in nullable struct") {
+    checkAnswer(
+      nullableStructLevel1.withColumn("a", 'a.withField("b", lit("foo"))),
+      Row(null) :: Row(Row(1, "foo", 3)) ::  Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", IntegerType, nullable = false),
+          StructField("b", StringType, nullable = false),
+          StructField("c", IntegerType, nullable = false))),
+          nullable = true))))
+  }
+
+  test("withField should replace field in nested nullable struct") {
+    checkAnswer(
+      nullableStructLevel2.withColumn("a", $"a".withField("a.b", lit("foo"))),
+      Row(null) :: Row(Row(null)) :: Row(Row(Row(1, "foo", 3))) :: Nil,
+      StructType(
+        Seq(StructField("a", StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", IntegerType, nullable = false),
+            StructField("b", StringType, nullable = false),
+            StructField("c", IntegerType, nullable = false))),
+            nullable = true))),
+          nullable = true))))
+  }
+
+  test("withField should replace field with null value in struct") {
+    checkAnswer(
+      structLevel1.withColumn("a", 'a.withField("c", lit(null).cast(IntegerType))),
+      Row(Row(1, null, null)) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", IntegerType, nullable = false),
+          StructField("b", IntegerType, nullable = true),
+          StructField("c", IntegerType, nullable = true))),
+          nullable = false))))
+  }
+
+  test("withField should replace multiple fields in struct") {
+    checkAnswer(
+      structLevel1.withColumn("a", 'a.withField("a", lit(10)).withField("b", lit(20))),
+      Row(Row(10, 20, 3)) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", IntegerType, nullable = false),
+          StructField("b", IntegerType, nullable = false),
+          StructField("c", IntegerType, nullable = false))),
+          nullable = false))))
+  }
+
+  test("withField should replace multiple fields in nullable struct") {
+    checkAnswer(
+      nullableStructLevel1.withColumn("a", 'a.withField("a", lit(10)).withField("b", lit(20))),
+      Row(null) :: Row(Row(10, 20, 3)) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", IntegerType, nullable = false),
+          StructField("b", IntegerType, nullable = false),
+          StructField("c", IntegerType, nullable = false))),
+          nullable = true))))
+  }
+
+  test("withField should replace field in nested struct") {
+    Seq(
+      structLevel2.withColumn("a", $"a".withField("a.b", lit(2))),
+      structLevel2.withColumn("a", 'a.withField("a", $"a.a".withField("b", lit(2))))
+    ).foreach { df =>
+      checkAnswer(
+        df,
+        Row(Row(Row(1, 2, 3))) :: Nil,
+        StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", StructType(Seq(
+              StructField("a", IntegerType, nullable = false),
+              StructField("b", IntegerType, nullable = false),
+              StructField("c", IntegerType, nullable = false))),
+              nullable = false))),
+            nullable = false))))
+    }
+  }
+
+  test("withField should replace multiple fields in nested struct") {
+    Seq(
+      col("a").withField("a", $"a.a".withField("a", lit(10)).withField("b", lit(20))),
+      col("a").withField("a.a", lit(10)).withField("a.b", lit(20))
+    ).foreach { column =>
+      checkAnswer(
+        structLevel2.select(column.as("a")),
+        Row(Row(Row(10, 20, 3))) :: Nil,
+        StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", StructType(Seq(
+              StructField("a", IntegerType, nullable = false),
+              StructField("b", IntegerType, nullable = false),
+              StructField("c", IntegerType, nullable = false))),
+              nullable = false))),
+            nullable = false))))
+    }
+  }
+
+  test("withField should replace multiple fields in nested nullable struct") {
+    Seq(
+      col("a").withField("a", $"a.a".withField("a", lit(10)).withField("b", lit(20))),
+      col("a").withField("a.a", lit(10)).withField("a.b", lit(20))
+    ).foreach { column =>
+      checkAnswer(
+        nullableStructLevel2.select(column.as("a")),
+        Row(null) :: Row(Row(null)) :: Row(Row(Row(10, 20, 3))) :: Nil,
+        StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", StructType(Seq(
+              StructField("a", IntegerType, nullable = false),
+              StructField("b", IntegerType, nullable = false),
+              StructField("c", IntegerType, nullable = false))),
+              nullable = true))),
+            nullable = true))))
+    }
+  }
+
+  test("withField should replace field in deeply nested struct") {
+    checkAnswer(
+      structLevel3.withColumn("a", $"a".withField("a.a.b", lit(2))),
+      Row(Row(Row(Row(1, 2, 3)))) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", StructType(Seq(
+              StructField("a", IntegerType, nullable = false),
+              StructField("b", IntegerType, nullable = false),
+              StructField("c", IntegerType, nullable = false))),
+              nullable = false))),
+            nullable = false))),
+          nullable = false))))
+  }
+
+  test("withField should replace all fields with given name in struct") {
+    val structLevel1 = spark.createDataFrame(
+      sparkContext.parallelize(Row(Row(1, 2, 3)) :: Nil),
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", IntegerType, nullable = false),
+          StructField("b", IntegerType, nullable = false),
+          StructField("b", IntegerType, nullable = false))),
+          nullable = false))))
+
+    checkAnswer(
+      structLevel1.withColumn("a", 'a.withField("b", lit(100))),
+      Row(Row(1, 100, 100)) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", IntegerType, nullable = false),
+          StructField("b", IntegerType, nullable = false),
+          StructField("b", IntegerType, nullable = false))),
+          nullable = false))))
+  }
+
+  test("withField should replace fields in struct in given order") {
+    checkAnswer(
+      structLevel1.withColumn("a", 'a.withField("b", lit(2)).withField("b", lit(20))),
+      Row(Row(1, 20, 3)) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", IntegerType, nullable = false),
+          StructField("b", IntegerType, nullable = false),
+          StructField("c", IntegerType, nullable = false))),
+          nullable = false))))
+  }
+
+  test("withField should add field and then replace same field in struct") {
+    checkAnswer(
+      structLevel1.withColumn("a", 'a.withField("d", lit(4)).withField("d", lit(5))),
+      Row(Row(1, null, 3, 5)) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", IntegerType, nullable = false),
+          StructField("b", IntegerType, nullable = true),
+          StructField("c", IntegerType, nullable = false),
+          StructField("d", IntegerType, nullable = false))),
+          nullable = false))))
+  }
+
+  test("withField should handle fields with dots in their name if correctly quoted") {
+    val df: DataFrame = spark.createDataFrame(
+      sparkContext.parallelize(Row(Row(Row(1, null, 3))) :: Nil),
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a.b", StructType(Seq(
+            StructField("c.d", IntegerType, nullable = false),
+            StructField("e.f", IntegerType, nullable = true),
+            StructField("g.h", IntegerType, nullable = false))),
+            nullable = false))),
+          nullable = false))))
+
+    checkAnswer(
+      df.withColumn("a", 'a.withField("`a.b`.`e.f`", lit(2))),
+      Row(Row(Row(1, 2, 3))) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a.b", StructType(Seq(
+            StructField("c.d", IntegerType, nullable = false),
+            StructField("e.f", IntegerType, nullable = false),
+            StructField("g.h", IntegerType, nullable = false))),
+            nullable = false))),
+          nullable = false))))
+
+    intercept[AnalysisException] {
+      df.withColumn("a", 'a.withField("a.b.e.f", lit(2)))
+    }.getMessage should include("No such struct field a in a.b")
+  }
+
+  private lazy val mixedCaseStructLevel1: DataFrame = spark.createDataFrame(
+    sparkContext.parallelize(Row(Row(1, 1)) :: Nil),
+    StructType(Seq(
+      StructField("a", StructType(Seq(
+        StructField("a", IntegerType, nullable = false),
+        StructField("B", IntegerType, nullable = false))),
+        nullable = false))))
+
+  test("withField should replace field in struct even if casing is different") {
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+      checkAnswer(
+        mixedCaseStructLevel1.withColumn("a", 'a.withField("A", lit(2))),
+        Row(Row(2, 1)) :: Nil,
+        StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("A", IntegerType, nullable = false),
+            StructField("B", IntegerType, nullable = false))),
+            nullable = false))))
+
+      checkAnswer(
+        mixedCaseStructLevel1.withColumn("a", 'a.withField("b", lit(2))),
+        Row(Row(1, 2)) :: Nil,
+        StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", IntegerType, nullable = false),
+            StructField("b", IntegerType, nullable = false))),
+            nullable = false))))
+    }
+  }
+
+  test("withField should add field to struct because casing is different") {
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+      checkAnswer(
+        mixedCaseStructLevel1.withColumn("a", 'a.withField("A", lit(2))),
+        Row(Row(1, 1, 2)) :: Nil,
+        StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", IntegerType, nullable = false),
+            StructField("B", IntegerType, nullable = false),
+            StructField("A", IntegerType, nullable = false))),
+            nullable = false))))
+
+      checkAnswer(
+        mixedCaseStructLevel1.withColumn("a", 'a.withField("b", lit(2))),
+        Row(Row(1, 1, 2)) :: Nil,
+        StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", IntegerType, nullable = false),
+            StructField("B", IntegerType, nullable = false),
+            StructField("b", IntegerType, nullable = false))),
+            nullable = false))))
+    }
+  }
+
+  private lazy val mixedCaseStructLevel2: DataFrame = spark.createDataFrame(
+    sparkContext.parallelize(Row(Row(Row(1, 1), Row(1, 1))) :: Nil),
+    StructType(Seq(
+      StructField("a", StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", IntegerType, nullable = false),
+          StructField("b", IntegerType, nullable = false))),
+          nullable = false),
+        StructField("B", StructType(Seq(
+          StructField("a", IntegerType, nullable = false),
+          StructField("b", IntegerType, nullable = false))),
+          nullable = false))),
+        nullable = false))))
+
+  test("withField should replace nested field in struct even if casing is different") {
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+      checkAnswer(
+        mixedCaseStructLevel2.withColumn("a", 'a.withField("A.a", lit(2))),
+        Row(Row(Row(2, 1), Row(1, 1))) :: Nil,
+        StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("A", StructType(Seq(
+              StructField("a", IntegerType, nullable = false),
+              StructField("b", IntegerType, nullable = false))),
+              nullable = false),
+            StructField("B", StructType(Seq(
+              StructField("a", IntegerType, nullable = false),
+              StructField("b", IntegerType, nullable = false))),
+              nullable = false))),
+            nullable = false))))
+
+      checkAnswer(
+        mixedCaseStructLevel2.withColumn("a", 'a.withField("b.a", lit(2))),
+        Row(Row(Row(1, 1), Row(2, 1))) :: Nil,
+        StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", StructType(Seq(
+              StructField("a", IntegerType, nullable = false),
+              StructField("b", IntegerType, nullable = false))),
+              nullable = false),
+            StructField("b", StructType(Seq(
+              StructField("a", IntegerType, nullable = false),
+              StructField("b", IntegerType, nullable = false))),
+              nullable = false))),
+            nullable = false))))
+    }
+  }
+
+  test("withField should throw an exception because casing is different") {
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+      intercept[AnalysisException] {
+        mixedCaseStructLevel2.withColumn("a", 'a.withField("A.a", lit(2)))
+      }.getMessage should include("No such struct field A in a, B")
+
+      intercept[AnalysisException] {
+        mixedCaseStructLevel2.withColumn("a", 'a.withField("b.a", lit(2)))
+      }.getMessage should include("No such struct field b in a, B")
+    }
+  }
+
+  test("withField user-facing examples") {
+    checkAnswer(
+      sql("SELECT named_struct('a', 1, 'b', 2) struct_col")
+        .select($"struct_col".withField("c", lit(3))),
+      Row(Row(1, 2, 3)))
+
+    checkAnswer(
+      sql("SELECT named_struct('a', 1, 'b', 2) struct_col")
+        .select($"struct_col".withField("b", lit(3))),
+      Row(Row(1, 3)))
+
+    checkAnswer(
+      sql("SELECT CAST(NULL AS struct<a:int,b:int>) struct_col")
+        .select($"struct_col".withField("c", lit(3))),
+      Row(null))
+
+    checkAnswer(
+      sql("SELECT named_struct('a', 1, 'b', 2, 'b', 3) struct_col")
+        .select($"struct_col".withField("b", lit(100))),
+      Row(Row(1, 100, 100)))
+
+    checkAnswer(
+      sql("SELECT named_struct('a', named_struct('a', 1, 'b', 2)) struct_col")
+        .select($"struct_col".withField("a.c", lit(3))),
+      Row(Row(Row(1, 2, 3))))
+
+    intercept[AnalysisException] {
+      sql("SELECT named_struct('a', named_struct('b', 1), 'a', named_struct('c', 2)) struct_col")
+        .select($"struct_col".withField("a.c", lit(3)))
+    }.getMessage should include("Ambiguous reference to fields")
+
+    checkAnswer(
+      sql("SELECT named_struct('a', named_struct('a', 1, 'b', 2)) struct_col")
+        .select($"struct_col".withField("a.c", lit(3)).withField("a.d", lit(4))),
+      Row(Row(Row(1, 2, 3, 4))))
+
+    checkAnswer(
+      sql("SELECT named_struct('a', named_struct('a', 1, 'b', 2)) struct_col")
+        .select($"struct_col".withField("a",
+          $"struct_col.a".withField("c", lit(3)).withField("d", lit(4)))),
+      Row(Row(Row(1, 2, 3, 4))))
+  }
+
+  test("SPARK-32641: extracting field from non-null struct column after withField should return " +
+    "field value") {
+    // extract newly added field
+    checkAnswer(
+      structLevel1.withColumn("a", $"a".withField("d", lit(4)).getField("d")),
+      Row(4) :: Nil,
+      StructType(Seq(StructField("a", IntegerType, nullable = false))))
+
+    // extract newly replaced field
+    checkAnswer(
+      structLevel1.withColumn("a", $"a".withField("a", lit(4)).getField("a")),
+      Row(4) :: Nil,
+      StructType(Seq(StructField("a", IntegerType, nullable = false))))
+
+    // add new field, extract another field from original struct
+    checkAnswer(
+      structLevel1.withColumn("a", $"a".withField("d", lit(4)).getField("c")),
+      Row(3):: Nil,
+      StructType(Seq(StructField("a", IntegerType, nullable = false))))
+
+    // replace field, extract another field from original struct
+    checkAnswer(
+      structLevel1.withColumn("a", $"a".withField("a", lit(4)).getField("c")),
+      Row(3):: Nil,
+      StructType(Seq(StructField("a", IntegerType, nullable = false))))
+  }
+
+  test("SPARK-32641: extracting field from null struct column after withField should return " +
+    "null if the original struct was null") {
+    val nullStructLevel1 = spark.createDataFrame(
+      sparkContext.parallelize(Row(null) :: Nil),
+      StructType(Seq(StructField("a", structType, nullable = true))))
+
+    // extract newly added field
+    checkAnswer(
+      nullStructLevel1.withColumn("a", $"a".withField("d", lit(4)).getField("d")),
+      Row(null) :: Nil,
+      StructType(Seq(StructField("a", IntegerType, nullable = true))))
+
+    // extract newly replaced field
+    checkAnswer(
+      nullStructLevel1.withColumn("a", $"a".withField("a", lit(4)).getField("a")),
+      Row(null):: Nil,
+      StructType(Seq(StructField("a", IntegerType, nullable = true))))
+
+    // add new field, extract another field from original struct
+    checkAnswer(
+      nullStructLevel1.withColumn("a", $"a".withField("d", lit(4)).getField("c")),
+      Row(null):: Nil,
+      StructType(Seq(StructField("a", IntegerType, nullable = true))))
+
+    // replace field, extract another field from original struct
+    checkAnswer(
+      nullStructLevel1.withColumn("a", $"a".withField("a", lit(4)).getField("c")),
+      Row(null):: Nil,
+      StructType(Seq(StructField("a", IntegerType, nullable = true))))
+  }
+
+  test("SPARK-32641: extracting field from nullable struct column which contains both null and " +
+    "non-null values after withField should return null if the original struct was null") {
+    val df = spark.createDataFrame(
+      sparkContext.parallelize(Row(Row(1, null, 3)) :: Row(null) :: Nil),
+      StructType(Seq(StructField("a", structType, nullable = true))))
+
+    // extract newly added field
+    checkAnswer(
+      df.withColumn("a", $"a".withField("d", lit(4)).getField("d")),
+      Row(4) :: Row(null) :: Nil,
+      StructType(Seq(StructField("a", IntegerType, nullable = true))))
+
+    // extract newly replaced field
+    checkAnswer(
+      df.withColumn("a", $"a".withField("a", lit(4)).getField("a")),
+      Row(4) :: Row(null):: Nil,
+      StructType(Seq(StructField("a", IntegerType, nullable = true))))
+
+    // add new field, extract another field from original struct
+    checkAnswer(
+      df.withColumn("a", $"a".withField("d", lit(4)).getField("c")),
+      Row(3) :: Row(null):: Nil,
+      StructType(Seq(StructField("a", IntegerType, nullable = true))))
+
+    // replace field, extract another field from original struct
+    checkAnswer(
+      df.withColumn("a", $"a".withField("a", lit(4)).getField("c")),
+      Row(3) :: Row(null):: Nil,
+      StructType(Seq(StructField("a", IntegerType, nullable = true))))
+  }
+
+  test("SPARK-35213: chained withField operations should have correct schema for new columns") {
+    val df = spark.createDataFrame(
+      sparkContext.parallelize(Row(null) :: Nil),
+      StructType(Seq(StructField("data", NullType))))
+
+    checkAnswer(
+      df.withColumn("data", struct()
+        .withField("a", struct())
+        .withField("b", struct())
+        .withField("a.aa", lit("aa1"))
+        .withField("b.ba", lit("ba1"))
+        .withField("a.ab", lit("ab1"))),
+        Row(Row(Row("aa1", "ab1"), Row("ba1"))) :: Nil,
+        StructType(Seq(
+          StructField("data", StructType(Seq(
+            StructField("a", StructType(Seq(
+              StructField("aa", StringType, nullable = false),
+              StructField("ab", StringType, nullable = false)
+            )), nullable = false),
+            StructField("b", StructType(Seq(
+              StructField("ba", StringType, nullable = false)
+            )), nullable = false)
+          )), nullable = false)
+        ))
+    )
+  }
+
+  test("SPARK-35213: optimized withField operations should maintain correct nested struct " +
+    "ordering") {
+    val df = spark.createDataFrame(
+      sparkContext.parallelize(Row(null) :: Nil),
+      StructType(Seq(StructField("data", NullType))))
+
+    checkAnswer(
+      df.withColumn("data", struct()
+          .withField("a", struct().withField("aa", lit("aa1")))
+          .withField("b", struct().withField("ba", lit("ba1")))
+        )
+        .withColumn("data", col("data").withField("b.bb", lit("bb1")))
+        .withColumn("data", col("data").withField("a.ab", lit("ab1"))),
+        Row(Row(Row("aa1", "ab1"), Row("ba1", "bb1"))) :: Nil,
+        StructType(Seq(
+          StructField("data", StructType(Seq(
+            StructField("a", StructType(Seq(
+              StructField("aa", StringType, nullable = false),
+              StructField("ab", StringType, nullable = false)
+            )), nullable = false),
+            StructField("b", StructType(Seq(
+              StructField("ba", StringType, nullable = false),
+              StructField("bb", StringType, nullable = false)
+            )), nullable = false)
+          )), nullable = false)
+        ))
+    )
+  }
+
+  test("dropFields should throw an exception if called on a non-StructType column") {
+    intercept[AnalysisException] {
+      testData.withColumn("key", $"key".dropFields("a"))
+    }.getMessage should include("struct argument should be struct type, got: int")
+  }
+
+  test("dropFields should throw an exception if fieldName argument is null") {
+    intercept[IllegalArgumentException] {
+      structLevel1.withColumn("a", $"a".dropFields(null))
+    }.getMessage should include("fieldName cannot be null")
+  }
+
+  test("dropFields should throw an exception if any intermediate structs don't exist") {
+    intercept[AnalysisException] {
+      structLevel2.withColumn("a", 'a.dropFields("x.b"))
+    }.getMessage should include("No such struct field x in a")
+
+    intercept[AnalysisException] {
+      structLevel3.withColumn("a", 'a.dropFields("a.x.b"))
+    }.getMessage should include("No such struct field x in a")
+  }
+
+  test("dropFields should throw an exception if intermediate field is not a struct") {
+    intercept[AnalysisException] {
+      structLevel1.withColumn("a", 'a.dropFields("b.a"))
+    }.getMessage should include("struct argument should be struct type, got: int")
+  }
+
+  test("dropFields should throw an exception if intermediate field reference is ambiguous") {
+    intercept[AnalysisException] {
+      val structLevel2: DataFrame = spark.createDataFrame(
+        sparkContext.parallelize(Row(Row(Row(1, null, 3), 4)) :: Nil),
+        StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", structType, nullable = false),
+            StructField("a", structType, nullable = false))),
+            nullable = false))))
+
+      structLevel2.withColumn("a", 'a.dropFields("a.b"))
+    }.getMessage should include("Ambiguous reference to fields")
+  }
+
+  test("dropFields should drop field in struct") {
+    checkAnswer(
+      structLevel1.withColumn("a", 'a.dropFields("b")),
+      Row(Row(1, 3)) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", IntegerType, nullable = false),
+          StructField("c", IntegerType, nullable = false))),
+          nullable = false))))
+  }
+
+  test("dropFields should drop field in nullable struct") {
+    checkAnswer(
+      nullableStructLevel1.withColumn("a", $"a".dropFields("b")),
+      Row(null) :: Row(Row(1, 3)) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", IntegerType, nullable = false),
+          StructField("c", IntegerType, nullable = false))),
+          nullable = true))))
+  }
+
+  test("dropFields should drop multiple fields in struct") {
+    Seq(
+      structLevel1.withColumn("a", $"a".dropFields("b", "c")),
+      structLevel1.withColumn("a", 'a.dropFields("b").dropFields("c"))
+    ).foreach { df =>
+      checkAnswer(
+        df,
+        Row(Row(1)) :: Nil,
+        StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", IntegerType, nullable = false))),
+            nullable = false))))
+    }
+  }
+
+  test("dropFields should throw an exception if no fields will be left in struct") {
+    intercept[AnalysisException] {
+      structLevel1.withColumn("a", 'a.dropFields("a", "b", "c"))
+    }.getMessage should include("cannot drop all fields in struct")
+  }
+
+  test("dropFields should drop field with no name in struct") {
+    val structType = StructType(Seq(
+      StructField("a", IntegerType, nullable = false),
+      StructField("", IntegerType, nullable = false)))
+
+    val structLevel1: DataFrame = spark.createDataFrame(
+      sparkContext.parallelize(Row(Row(1, 2)) :: Nil),
+      StructType(Seq(StructField("a", structType, nullable = false))))
+
+    checkAnswer(
+      structLevel1.withColumn("a", $"a".dropFields("")),
+      Row(Row(1)) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", IntegerType, nullable = false))),
+          nullable = false))))
+  }
+
+  test("dropFields should drop field in nested struct") {
+    checkAnswer(
+      structLevel2.withColumn("a", 'a.dropFields("a.b")),
+      Row(Row(Row(1, 3))) :: Nil,
+      StructType(
+        Seq(StructField("a", StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", IntegerType, nullable = false),
+            StructField("c", IntegerType, nullable = false))),
+            nullable = false))),
+          nullable = false))))
+  }
+
+  test("dropFields should drop multiple fields in nested struct") {
+    checkAnswer(
+      structLevel2.withColumn("a", 'a.dropFields("a.b", "a.c")),
+      Row(Row(Row(1))) :: Nil,
+      StructType(
+        Seq(StructField("a", StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", IntegerType, nullable = false))),
+            nullable = false))),
+          nullable = false))))
+  }
+
+  test("dropFields should drop field in nested nullable struct") {
+    checkAnswer(
+      nullableStructLevel2.withColumn("a", $"a".dropFields("a.b")),
+      Row(null) :: Row(Row(null)) :: Row(Row(Row(1, 3))) :: Nil,
+      StructType(
+        Seq(StructField("a", StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", IntegerType, nullable = false),
+            StructField("c", IntegerType, nullable = false))),
+            nullable = true))),
+          nullable = true))))
+  }
+
+  test("dropFields should drop multiple fields in nested nullable struct") {
+    checkAnswer(
+      nullableStructLevel2.withColumn("a", $"a".dropFields("a.b", "a.c")),
+      Row(null) :: Row(Row(null)) :: Row(Row(Row(1))) :: Nil,
+      StructType(
+        Seq(StructField("a", StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", IntegerType, nullable = false))),
+            nullable = true))),
+          nullable = true))))
+  }
+
+  test("dropFields should drop field in deeply nested struct") {
+    checkAnswer(
+      structLevel3.withColumn("a", 'a.dropFields("a.a.b")),
+      Row(Row(Row(Row(1, 3)))) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", StructType(Seq(
+              StructField("a", IntegerType, nullable = false),
+              StructField("c", IntegerType, nullable = false))),
+              nullable = false))),
+            nullable = false))),
+          nullable = false))))
+  }
+
+  test("dropFields should drop all fields with given name in struct") {
+    val structLevel1 = spark.createDataFrame(
+      sparkContext.parallelize(Row(Row(1, 2, 3)) :: Nil),
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", IntegerType, nullable = false),
+          StructField("b", IntegerType, nullable = false),
+          StructField("b", IntegerType, nullable = false))),
+          nullable = false))))
+
+    checkAnswer(
+      structLevel1.withColumn("a", 'a.dropFields("b")),
+      Row(Row(1)) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", IntegerType, nullable = false))),
+          nullable = false))))
+  }
+
+  test("dropFields should drop field in struct even if casing is different") {
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+      checkAnswer(
+        mixedCaseStructLevel1.withColumn("a", 'a.dropFields("A")),
+        Row(Row(1)) :: Nil,
+        StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("B", IntegerType, nullable = false))),
+            nullable = false))))
+
+      checkAnswer(
+        mixedCaseStructLevel1.withColumn("a", 'a.dropFields("b")),
+        Row(Row(1)) :: Nil,
+        StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", IntegerType, nullable = false))),
+            nullable = false))))
+    }
+  }
+
+  test("dropFields should not drop field in struct because casing is different") {
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+      checkAnswer(
+        mixedCaseStructLevel1.withColumn("a", 'a.dropFields("A")),
+        Row(Row(1, 1)) :: Nil,
+        StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", IntegerType, nullable = false),
+            StructField("B", IntegerType, nullable = false))),
+            nullable = false))))
+
+      checkAnswer(
+        mixedCaseStructLevel1.withColumn("a", 'a.dropFields("b")),
+        Row(Row(1, 1)) :: Nil,
+        StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", IntegerType, nullable = false),
+            StructField("B", IntegerType, nullable = false))),
+            nullable = false))))
+    }
+  }
+
+  test("dropFields should drop nested field in struct even if casing is different") {
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+      checkAnswer(
+        mixedCaseStructLevel2.withColumn("a", 'a.dropFields("A.a")),
+        Row(Row(Row(1), Row(1, 1))) :: Nil,
+        StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("A", StructType(Seq(
+              StructField("b", IntegerType, nullable = false))),
+              nullable = false),
+            StructField("B", StructType(Seq(
+              StructField("a", IntegerType, nullable = false),
+              StructField("b", IntegerType, nullable = false))),
+              nullable = false))),
+            nullable = false))))
+
+      checkAnswer(
+        mixedCaseStructLevel2.withColumn("a", 'a.dropFields("b.a")),
+        Row(Row(Row(1, 1), Row(1))) :: Nil,
+        StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", StructType(Seq(
+              StructField("a", IntegerType, nullable = false),
+              StructField("b", IntegerType, nullable = false))),
+              nullable = false),
+            StructField("b", StructType(Seq(
+              StructField("b", IntegerType, nullable = false))),
+              nullable = false))),
+            nullable = false))))
+    }
+  }
+
+  test("dropFields should throw an exception because casing is different") {
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+      intercept[AnalysisException] {
+        mixedCaseStructLevel2.withColumn("a", 'a.dropFields("A.a"))
+      }.getMessage should include("No such struct field A in a, B")
+
+      intercept[AnalysisException] {
+        mixedCaseStructLevel2.withColumn("a", 'a.dropFields("b.a"))
+      }.getMessage should include("No such struct field b in a, B")
+    }
+  }
+
+  test("dropFields should drop only fields that exist") {
+    checkAnswer(
+      structLevel1.withColumn("a", 'a.dropFields("d")),
+      Row(Row(1, null, 3)) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", IntegerType, nullable = false),
+          StructField("b", IntegerType, nullable = true),
+          StructField("c", IntegerType, nullable = false))),
+          nullable = false))))
+
+    checkAnswer(
+      structLevel1.withColumn("a", 'a.dropFields("b", "d")),
+      Row(Row(1, 3)) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", IntegerType, nullable = false),
+          StructField("c", IntegerType, nullable = false))),
+          nullable = false))))
+
+    checkAnswer(
+      structLevel2.withColumn("a", $"a".dropFields("a.b", "a.d")),
+      Row(Row(Row(1, 3))) :: Nil,
+      StructType(
+        Seq(StructField("a", StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", IntegerType, nullable = false),
+            StructField("c", IntegerType, nullable = false))),
+            nullable = false))),
+          nullable = false))))
+  }
+
+  test("dropFields should drop multiple fields at arbitrary levels of nesting in a single call") {
+    val df: DataFrame = spark.createDataFrame(
+      sparkContext.parallelize(Row(Row(Row(1, null, 3), 4)) :: Nil),
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", structType, nullable = false),
+          StructField("b", IntegerType, nullable = false))),
+          nullable = false))))
+
+    checkAnswer(
+      df.withColumn("a", $"a".dropFields("a.b", "b")),
+      Row(Row(Row(1, 3))) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", IntegerType, nullable = false),
+            StructField("c", IntegerType, nullable = false))), nullable = false))),
+          nullable = false))))
+  }
+
+  test("dropFields user-facing examples") {
+    checkAnswer(
+      sql("SELECT named_struct('a', 1, 'b', 2) struct_col")
+        .select($"struct_col".dropFields("b")),
+      Row(Row(1)))
+
+    checkAnswer(
+      sql("SELECT named_struct('a', 1, 'b', 2) struct_col")
+        .select($"struct_col".dropFields("c")),
+      Row(Row(1, 2)))
+
+    checkAnswer(
+      sql("SELECT named_struct('a', 1, 'b', 2, 'c', 3) struct_col")
+        .select($"struct_col".dropFields("b", "c")),
+      Row(Row(1)))
+
+    intercept[AnalysisException] {
+      sql("SELECT named_struct('a', 1, 'b', 2) struct_col")
+        .select($"struct_col".dropFields("a", "b"))
+    }.getMessage should include("cannot drop all fields in struct")
+
+    checkAnswer(
+      sql("SELECT CAST(NULL AS struct<a:int,b:int>) struct_col")
+        .select($"struct_col".dropFields("b")),
+      Row(null))
+
+    checkAnswer(
+      sql("SELECT named_struct('a', 1, 'b', 2, 'b', 3) struct_col")
+        .select($"struct_col".dropFields("b")),
+      Row(Row(1)))
+
+    checkAnswer(
+      sql("SELECT named_struct('a', named_struct('a', 1, 'b', 2)) struct_col")
+        .select($"struct_col".dropFields("a.b")),
+      Row(Row(Row(1))))
+
+    intercept[AnalysisException] {
+      sql("SELECT named_struct('a', named_struct('b', 1), 'a', named_struct('c', 2)) struct_col")
+        .select($"struct_col".dropFields("a.c"))
+    }.getMessage should include("Ambiguous reference to fields")
+
+    checkAnswer(
+      sql("SELECT named_struct('a', named_struct('a', 1, 'b', 2, 'c', 3)) struct_col")
+        .select($"struct_col".dropFields("a.b", "a.c")),
+      Row(Row(Row(1))))
+
+    checkAnswer(
+      sql("SELECT named_struct('a', named_struct('a', 1, 'b', 2, 'c', 3)) struct_col")
+        .select($"struct_col".withField("a", $"struct_col.a".dropFields("b", "c"))),
+      Row(Row(Row(1))))
+  }
+
+  test("should correctly handle different dropField + withField + getField combinations") {
+    val structType = StructType(Seq(
+      StructField("a", IntegerType, nullable = false),
+      StructField("b", IntegerType, nullable = false)))
+
+    val structLevel1: DataFrame = spark.createDataFrame(
+      sparkContext.parallelize(Row(Row(1, 2)) :: Nil),
+      StructType(Seq(StructField("a", structType, nullable = false))))
+
+    val nullStructLevel1: DataFrame = spark.createDataFrame(
+      sparkContext.parallelize(Row(null) :: Nil),
+      StructType(Seq(StructField("a", structType, nullable = true))))
+
+    val nullableStructLevel1: DataFrame = spark.createDataFrame(
+      sparkContext.parallelize(Row(Row(1, 2)) :: Row(null) :: Nil),
+      StructType(Seq(StructField("a", structType, nullable = true))))
+
+    def check(
+      fieldOps: Column => Column,
+      getFieldName: String,
+      expectedValue: Option[Int]): Unit = {
+
+      def query(df: DataFrame): DataFrame =
+        df.select(fieldOps(col("a")).getField(getFieldName).as("res"))
+
+      checkAnswer(
+        query(structLevel1),
+        Row(expectedValue.orNull) :: Nil,
+        StructType(Seq(StructField("res", IntegerType, nullable = expectedValue.isEmpty))))
+
+      checkAnswer(
+        query(nullStructLevel1),
+        Row(null) :: Nil,
+        StructType(Seq(StructField("res", IntegerType, nullable = true))))
+
+      checkAnswer(
+        query(nullableStructLevel1),
+        Row(expectedValue.orNull) :: Row(null) :: Nil,
+        StructType(Seq(StructField("res", IntegerType, nullable = true))))
+    }
+
+    // add attribute, extract an attribute from the original struct
+    check(_.withField("c", lit(3)), "a", Some(1))
+    check(_.withField("c", lit(3)), "b", Some(2))
+
+    // add attribute, extract added attribute
+    check(_.withField("c", lit(3)), "c", Some(3))
+    check(_.withField("c", col("a.a")), "c", Some(1))
+    check(_.withField("c", col("a.b")), "c", Some(2))
+    check(_.withField("c", lit(null).cast(IntegerType)), "c", None)
+
+    // replace attribute, extract an attribute from the original struct
+    check(_.withField("b", lit(3)), "a", Some(1))
+    check(_.withField("a", lit(3)), "b", Some(2))
+
+    // replace attribute, extract replaced attribute
+    check(_.withField("b", lit(3)), "b", Some(3))
+    check(_.withField("b", lit(null).cast(IntegerType)), "b", None)
+    check(_.withField("a", lit(3)), "a", Some(3))
+    check(_.withField("a", lit(null).cast(IntegerType)), "a", None)
+
+    // drop attribute, extract an attribute from the original struct
+    check(_.dropFields("b"), "a", Some(1))
+    check(_.dropFields("a"), "b", Some(2))
+
+    // drop attribute, add attribute, extract an attribute from the original struct
+    check(_.dropFields("b").withField("c", lit(3)), "a", Some(1))
+    check(_.dropFields("a").withField("c", lit(3)), "b", Some(2))
+
+    // drop attribute, add another attribute, extract added attribute
+    check(_.dropFields("a").withField("c", lit(3)), "c", Some(3))
+    check(_.dropFields("b").withField("c", lit(3)), "c", Some(3))
+
+    // add attribute, drop attribute, extract an attribute from the original struct
+    check(_.withField("c", lit(3)).dropFields("a"), "b", Some(2))
+    check(_.withField("c", lit(3)).dropFields("b"), "a", Some(1))
+
+    // add attribute, drop another attribute, extract added attribute
+    check(_.withField("c", lit(3)).dropFields("a"), "c", Some(3))
+    check(_.withField("c", lit(3)).dropFields("b"), "c", Some(3))
+
+    // replace attribute, drop same attribute, extract an attribute from the original struct
+    check(_.withField("b", lit(3)).dropFields("b"), "a", Some(1))
+    check(_.withField("a", lit(3)).dropFields("a"), "b", Some(2))
+
+    // add attribute, drop same attribute, extract an attribute from the original struct
+    check(_.withField("c", lit(3)).dropFields("c"), "a", Some(1))
+    check(_.withField("c", lit(3)).dropFields("c"), "b", Some(2))
+
+    // add attribute, drop another attribute, extract added attribute
+    check(_.withField("b", lit(3)).dropFields("a"), "b", Some(3))
+    check(_.withField("a", lit(3)).dropFields("b"), "a", Some(3))
+    check(_.withField("b", lit(null).cast(IntegerType)).dropFields("a"), "b", None)
+    check(_.withField("a", lit(null).cast(IntegerType)).dropFields("b"), "a", None)
+
+    // drop attribute, add same attribute, extract added attribute
+    check(_.dropFields("b").withField("b", lit(3)), "b", Some(3))
+    check(_.dropFields("a").withField("a", lit(3)), "a", Some(3))
+    check(_.dropFields("b").withField("b", lit(null).cast(IntegerType)), "b", None)
+    check(_.dropFields("a").withField("a", lit(null).cast(IntegerType)), "a", None)
+    check(_.dropFields("c").withField("c", lit(3)), "c", Some(3))
+
+    // add attribute, drop same attribute, add same attribute again, extract added attribute
+    check(_.withField("c", lit(3)).dropFields("c").withField("c", lit(4)), "c", Some(4))
+  }
+
+  test("should move field up one level of nesting") {
+    // move a field up one level
+    checkAnswer(
+      nullableStructLevel2.select(
+        col("a").withField("c", col("a.a.c")).dropFields("a.c").as("res")),
+      Row(null) :: Row(Row(null, null)) ::  Row(Row(Row(1, null), 3)) :: Nil,
+      StructType(Seq(
+        StructField("res", StructType(Seq(
+          StructField("a", StructType(Seq(
+            StructField("a", IntegerType, nullable = false),
+            StructField("b", IntegerType, nullable = true))),
+            nullable = true),
+          StructField("c", IntegerType, nullable = true))),
+          nullable = true))))
+
+    // move a field up one level and then extract it
+    checkAnswer(
+      nullableStructLevel2.select(
+        col("a").withField("c", col("a.a.c")).dropFields("a.c").getField("c").as("res")),
+      Row(null) :: Row(null) :: Row(3) :: Nil,
+      StructType(Seq(StructField("res", IntegerType, nullable = true))))
+  }
+
+  test("should be able to refer to newly added nested column") {
+    intercept[AnalysisException] {
+      structLevel1.select($"a".withField("d", lit(4)).withField("e", $"a.d" + 1).as("a"))
+    }.getMessage should include("No such struct field d in a, b, c")
+
+    checkAnswer(
+      structLevel1
+        .select($"a".withField("d", lit(4)).as("a"))
+        .select($"a".withField("e", $"a.d" + 1).as("a")),
+      Row(Row(1, null, 3, 4, 5)) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", IntegerType, nullable = false),
+          StructField("b", IntegerType, nullable = true),
+          StructField("c", IntegerType, nullable = false),
+          StructField("d", IntegerType, nullable = false),
+          StructField("e", IntegerType, nullable = false))),
+          nullable = false))))
+  }
+
+  test("should be able to drop newly added nested column") {
+    Seq(
+      structLevel1.select($"a".withField("d", lit(4)).dropFields("d").as("a")),
+      structLevel1
+        .select($"a".withField("d", lit(4)).as("a"))
+        .select($"a".dropFields("d").as("a"))
+    ).foreach { query =>
+      checkAnswer(
+        query,
+        Row(Row(1, null, 3)) :: Nil,
+        StructType(Seq(
+          StructField("a", structType, nullable = false))))
+    }
+  }
+
+  test("should still be able to refer to dropped column within the same select statement") {
+    // we can still access the nested column even after dropping it within the same select statement
+    checkAnswer(
+      structLevel1.select($"a".dropFields("c").withField("z", $"a.c").as("a")),
+      Row(Row(1, null, 3)) :: Nil,
+      StructType(Seq(
+        StructField("a", StructType(Seq(
+          StructField("a", IntegerType, nullable = false),
+          StructField("b", IntegerType, nullable = true),
+          StructField("z", IntegerType, nullable = false))),
+          nullable = false))))
+
+    // we can't access the nested column in subsequent select statement after dropping it in a
+    // previous select statement
+    intercept[AnalysisException]{
+      structLevel1
+        .select($"a".dropFields("c").as("a"))
+        .select($"a".withField("z", $"a.c")).as("a")
+    }.getMessage should include("No such struct field c in a, b")
+  }
+
+  test("nestedDf should generate nested DataFrames") {
+    checkAnswer(
+      emptyNestedDf(1, 1, nullable = false),
+      Seq.empty[Row],
+      StructType(Seq(StructField("nested0Col0", StructType(Seq(
+        StructField("nested1Col0", IntegerType, nullable = false))),
+        nullable = false))))
+
+    checkAnswer(
+      emptyNestedDf(1, 2, nullable = false),
+      Seq.empty[Row],
+      StructType(Seq(StructField("nested0Col0", StructType(Seq(
+        StructField("nested1Col0", IntegerType, nullable = false),
+        StructField("nested1Col1", IntegerType, nullable = false))),
+        nullable = false))))
+
+    checkAnswer(
+      emptyNestedDf(2, 1, nullable = false),
+      Seq.empty[Row],
+      StructType(Seq(StructField("nested0Col0", StructType(Seq(
+        StructField("nested1Col0", StructType(Seq(
+          StructField("nested2Col0", IntegerType, nullable = false))),
+          nullable = false))),
+        nullable = false))))
+
+    checkAnswer(
+      emptyNestedDf(2, 2, nullable = false),
+      Seq.empty[Row],
+      StructType(Seq(StructField("nested0Col0", StructType(Seq(
+        StructField("nested1Col0", StructType(Seq(
+          StructField("nested2Col0", IntegerType, nullable = false),
+          StructField("nested2Col1", IntegerType, nullable = false))),
+          nullable = false),
+        StructField("nested1Col1", IntegerType, nullable = false))),
+        nullable = false))))
+
+    checkAnswer(
+      emptyNestedDf(2, 2, nullable = true),
+      Seq.empty[Row],
+      StructType(Seq(StructField("nested0Col0", StructType(Seq(
+        StructField("nested1Col0", StructType(Seq(
+          StructField("nested2Col0", IntegerType, nullable = false),
+          StructField("nested2Col1", IntegerType, nullable = false))),
+          nullable = true),
+        StructField("nested1Col1", IntegerType, nullable = false))),
+        nullable = true))))
+  }
+
+  Seq(Performant, NonPerformant).foreach { method =>
+    Seq(false, true).foreach { nullable =>
+      test(s"should add and drop 1 column at each depth of nesting using ${method.name} method, " +
+        s"nullable = $nullable") {
+        val maxDepth = 3
+
+        // dataframe with nested*Col0 to nested*Col2 at each depth
+        val inputDf = emptyNestedDf(maxDepth, 3, nullable)
+
+        // add nested*Col3 and drop nested*Col2
+        val modifiedColumn = method(
+          column = col(nestedColName(0, 0)),
+          numsToAdd = Seq(3),
+          numsToDrop = Seq(2),
+          maxDepth = maxDepth
+        ).as(nestedColName(0, 0))
+        val resultDf = inputDf.select(modifiedColumn)
+
+        // dataframe with nested*Col0, nested*Col1, nested*Col3 at each depth
+        val expectedDf = {
+          val colNums = Seq(0, 1, 3)
+          val nestedColumnDataType = nestedStructType(colNums, nullable, maxDepth)
+
+          spark.createDataFrame(
+            spark.sparkContext.emptyRDD[Row],
+            StructType(Seq(StructField(nestedColName(0, 0), nestedColumnDataType, nullable))))
+        }
+
+        checkAnswer(resultDf, expectedDf.collect(), expectedDf.schema)
+      }
+    }
+  }
+
+  test("assert_true") {
+    // assert_true(condition, errMsgCol)
+    val booleanDf = Seq((true), (false)).toDF("cond")
+    checkAnswer(
+      booleanDf.filter("cond = true").select(assert_true($"cond")),
+      Row(null) :: Nil
+    )
+    val e1 = intercept[SparkException] {
+      booleanDf.select(assert_true($"cond", lit(null.asInstanceOf[String]))).collect()
+    }
+    assert(e1.getCause.isInstanceOf[RuntimeException])
+    assert(e1.getCause.getMessage == null)
+
+    val nullDf = Seq(("first row", None), ("second row", Some(true))).toDF("n", "cond")
+    checkAnswer(
+      nullDf.filter("cond = true").select(assert_true($"cond", $"cond")),
+      Row(null) :: Nil
+    )
+    val e2 = intercept[SparkException] {
+      nullDf.select(assert_true($"cond", $"n")).collect()
+    }
+    assert(e2.getCause.isInstanceOf[RuntimeException])
+    assert(e2.getCause.getMessage == "first row")
+
+    // assert_true(condition)
+    val intDf = Seq((0, 1)).toDF("a", "b")
+    checkAnswer(intDf.select(assert_true($"a" < $"b")), Row(null) :: Nil)
+    val e3 = intercept[SparkException] {
+      intDf.select(assert_true($"a" > $"b")).collect()
+    }
+    assert(e3.getCause.isInstanceOf[RuntimeException])
+    assert(e3.getCause.getMessage == "'('a > 'b)' is not true!")
+  }
+
+  test("raise_error") {
+    val strDf = Seq(("hello")).toDF("a")
+
+    val e1 = intercept[SparkException] {
+      strDf.select(raise_error(lit(null.asInstanceOf[String]))).collect()
+    }
+    assert(e1.getCause.isInstanceOf[RuntimeException])
+    assert(e1.getCause.getMessage == null)
+
+    val e2 = intercept[SparkException] {
+      strDf.select(raise_error($"a")).collect()
+    }
+    assert(e2.getCause.isInstanceOf[RuntimeException])
+    assert(e2.getCause.getMessage == "hello")
+  }
+
+  test("SPARK-34677: negate/add/subtract year-month and day-time intervals") {
+    import testImplicits._
+    val df = Seq((Period.ofMonths(10), Duration.ofDays(10), Period.ofMonths(1), Duration.ofDays(1)))
+      .toDF("year-month-A", "day-time-A", "year-month-B", "day-time-B")
+    val negatedDF = df.select(-$"year-month-A", -$"day-time-A")
+    checkAnswer(negatedDF, Row(Period.ofMonths(-10), Duration.ofDays(-10)))
+    val addDF = df.select($"year-month-A" + $"year-month-B", $"day-time-A" + $"day-time-B")
+    checkAnswer(addDF, Row(Period.ofMonths(11), Duration.ofDays(11)))
+    val subDF = df.select($"year-month-A" - $"year-month-B", $"day-time-A" - $"day-time-B")
+    checkAnswer(subDF, Row(Period.ofMonths(9), Duration.ofDays(9)))
+  }
+
+  test("SPARK-34721: add a year-month interval to a date") {
+    withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
+      outstandingTimezonesIds.foreach { zid =>
+        withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> zid) {
+          Seq(
+            (LocalDate.of(1900, 10, 1), Period.ofMonths(0)) -> LocalDate.of(1900, 10, 1),
+            (LocalDate.of(1970, 1, 1), Period.ofMonths(-1)) -> LocalDate.of(1969, 12, 1),
+            (LocalDate.of(2021, 3, 11), Period.ofMonths(1)) -> LocalDate.of(2021, 4, 11),
+            (LocalDate.of(2020, 12, 31), Period.ofMonths(2)) -> LocalDate.of(2021, 2, 28),
+            (LocalDate.of(2021, 5, 31), Period.ofMonths(-3)) -> LocalDate.of(2021, 2, 28),
+            (LocalDate.of(2020, 2, 29), Period.ofYears(1)) -> LocalDate.of(2021, 2, 28),
+            (LocalDate.of(1, 1, 1), Period.ofYears(2020)) -> LocalDate.of(2021, 1, 1)
+          ).foreach { case ((date, period), result) =>
+            val df = Seq((date, period)).toDF("date", "interval")
+            checkAnswer(
+              df.select($"date" + $"interval", $"interval" + $"date"),
+              Row(result, result))
+          }
+        }
+      }
+
+      val e = intercept[SparkException] {
+        Seq((LocalDate.of(2021, 3, 11), Period.ofMonths(Int.MaxValue)))
+          .toDF("date", "interval")
+          .select($"date" + $"interval")
+          .collect()
+      }.getCause
+      assert(e.isInstanceOf[ArithmeticException])
+      assert(e.getMessage.contains("integer overflow"))
+    }
+  }
+
+  test("SPARK-34721: subtract a year-month interval from a date") {
+    withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
+      outstandingTimezonesIds.foreach { zid =>
+        withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> zid) {
+          Seq(
+            (LocalDate.of(1582, 10, 4), Period.ofMonths(0)) -> LocalDate.of(1582, 10, 4),
+            (LocalDate.of(1582, 10, 15), Period.ofMonths(1)) -> LocalDate.of(1582, 9, 15),
+            (LocalDate.of(1, 1, 1), Period.ofMonths(-1)) -> LocalDate.of(1, 2, 1),
+            (LocalDate.of(9999, 10, 31), Period.ofMonths(-2)) -> LocalDate.of(9999, 12, 31),
+            (LocalDate.of(2021, 5, 31), Period.ofMonths(3)) -> LocalDate.of(2021, 2, 28),
+            (LocalDate.of(2021, 2, 28), Period.ofYears(1)) -> LocalDate.of(2020, 2, 28),
+            (LocalDate.of(2020, 2, 29), Period.ofYears(4)) -> LocalDate.of(2016, 2, 29)
+          ).foreach { case ((date, period), result) =>
+            val df = Seq((date, period)).toDF("date", "interval")
+            checkAnswer(df.select($"date" - $"interval"), Row(result))
+          }
+        }
+      }
+
+      val e = intercept[SparkException] {
+        Seq((LocalDate.of(2021, 3, 11), Period.ofMonths(Int.MaxValue)))
+          .toDF("date", "interval")
+          .select($"date" - $"interval")
+          .collect()
+      }.getCause
+      assert(e.isInstanceOf[ArithmeticException])
+      assert(e.getMessage.contains("integer overflow"))
+    }
+  }
+
+  test("SPARK-34739: add a year-month interval to a timestamp") {
+    withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
+      outstandingZoneIds.foreach { zid =>
+        withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> zid.getId) {
+          Seq(
+            (LocalDateTime.of(1900, 1, 1, 0, 0, 0, 123456000), Period.ofMonths(0)) ->
+              LocalDateTime.of(1900, 1, 1, 0, 0, 0, 123456000),
+            (LocalDateTime.of(1970, 1, 1, 0, 0, 0, 1000), Period.ofMonths(-1)) ->
+              LocalDateTime.of(1969, 12, 1, 0, 0, 0, 1000),
+            (LocalDateTime.of(2021, 3, 14, 1, 2, 3, 0), Period.ofMonths(1)) ->
+              LocalDateTime.of(2021, 4, 14, 1, 2, 3, 0),
+            (LocalDateTime.of(2020, 12, 31, 23, 59, 59, 999999000), Period.ofMonths(2)) ->
+              LocalDateTime.of(2021, 2, 28, 23, 59, 59, 999999000),
+            (LocalDateTime.of(2021, 5, 31, 0, 0, 1, 0), Period.ofMonths(-3)) ->
+              LocalDateTime.of(2021, 2, 28, 0, 0, 1, 0),
+            (LocalDateTime.of(2020, 2, 29, 12, 13, 14), Period.ofYears(1)) ->
+              LocalDateTime.of(2021, 2, 28, 12, 13, 14),
+            (LocalDateTime.of(1, 1, 1, 1, 1, 1, 1000), Period.ofYears(2020)) ->
+              LocalDateTime.of(2021, 1, 1, 1, 1, 1, 1000)
+          ).foreach { case ((ldt, period), expected) =>
+            val df = Seq((ldt.atZone(zid).toInstant, period)).toDF("ts", "interval")
+            val result = expected.atZone(zid).toInstant
+            checkAnswer(df.select($"ts" + $"interval", $"interval" + $"ts"), Row(result, result))
+          }
+        }
+      }
+
+      val e = intercept[SparkException] {
+        Seq((Instant.parse("2021-03-14T18:55:00Z"), Period.ofMonths(Int.MaxValue)))
+          .toDF("ts", "interval")
+          .select($"ts" + $"interval")
+          .collect()
+      }.getCause
+      assert(e.isInstanceOf[ArithmeticException])
+      assert(e.getMessage.contains("long overflow"))
+    }
+  }
+
+  test("SPARK-34739: subtract a year-month interval from a timestamp") {
+    withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
+      outstandingZoneIds.foreach { zid =>
+        withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> zid.getId) {
+          Seq(
+            (LocalDateTime.of(1582, 10, 4, 0, 0, 0), Period.ofMonths(0)) ->
+              LocalDateTime.of(1582, 10, 4, 0, 0, 0),
+            (LocalDateTime.of(1582, 10, 15, 23, 59, 59, 999999000), Period.ofMonths(1)) ->
+              LocalDateTime.of(1582, 9, 15, 23, 59, 59, 999999000),
+            (LocalDateTime.of(1, 1, 1, 1, 1, 1, 1000), Period.ofMonths(-1)) ->
+              LocalDateTime.of(1, 2, 1, 1, 1, 1, 1000),
+            (LocalDateTime.of(9999, 10, 31, 23, 59, 59, 999000000), Period.ofMonths(-2)) ->
+              LocalDateTime.of(9999, 12, 31, 23, 59, 59, 999000000),
+            (LocalDateTime.of(2021, 5, 31, 0, 0, 0, 1000), Period.ofMonths(3)) ->
+              LocalDateTime.of(2021, 2, 28, 0, 0, 0, 1000),
+            (LocalDateTime.of(2021, 2, 28, 11, 12, 13, 123456000), Period.ofYears(1)) ->
+              LocalDateTime.of(2020, 2, 28, 11, 12, 13, 123456000),
+            (LocalDateTime.of(2020, 2, 29, 1, 2, 3, 5000), Period.ofYears(4)) ->
+              LocalDateTime.of(2016, 2, 29, 1, 2, 3, 5000)
+          ).foreach { case ((ldt, period), expected) =>
+            val df = Seq((ldt.atZone(zid).toInstant, period)).toDF("ts", "interval")
+            checkAnswer(df.select($"ts" - $"interval"), Row(expected.atZone(zid).toInstant))
+          }
+        }
+      }
+
+      val e = intercept[SparkException] {
+        Seq((Instant.parse("2021-03-14T18:55:00Z"), Period.ofMonths(Int.MaxValue)))
+          .toDF("ts", "interval")
+          .select($"ts" - $"interval")
+          .collect()
+      }.getCause
+      assert(e.isInstanceOf[ArithmeticException])
+      assert(e.getMessage.contains("long overflow"))
+    }
+  }
+
+  test("SPARK-34761, SPARK-34903: add/subtract a day-time interval to/from a timestamp") {
+    withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
+      outstandingZoneIds.foreach { zid =>
+        withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> zid.getId) {
+          Seq(
+            (LocalDateTime.of(1900, 1, 1, 0, 0, 0, 123456000), Duration.ofDays(0)) ->
+              LocalDateTime.of(1900, 1, 1, 0, 0, 0, 123456000),
+            (LocalDateTime.of(1970, 1, 1, 0, 0, 0, 100000000), Duration.ofDays(-1)) ->
+              LocalDateTime.of(1969, 12, 31, 0, 0, 0, 100000000),
+            (LocalDateTime.of(2021, 3, 14, 1, 2, 3), Duration.ofDays(1)) ->
+              LocalDateTime.of(2021, 3, 15, 1, 2, 3),
+            (LocalDateTime.of(2020, 12, 31, 23, 59, 59, 999000000),
+              Duration.ofDays(2 * 30).plusMillis(1)) -> LocalDateTime.of(2021, 3, 2, 0, 0, 0),
+            (LocalDateTime.of(2020, 3, 16, 0, 0, 0, 1000), Duration.of(-1, ChronoUnit.MICROS)) ->
+              LocalDateTime.of(2020, 3, 16, 0, 0, 0),
+            (LocalDateTime.of(2020, 2, 29, 12, 13, 14), Duration.ofDays(365)) ->
+              LocalDateTime.of(2021, 2, 28, 12, 13, 14),
+            (LocalDateTime.of(1582, 10, 4, 1, 2, 3, 40000000),
+              Duration.ofDays(10).plusMillis(60)) ->
+              LocalDateTime.of(1582, 10, 14, 1, 2, 3, 100000000)
+          ).foreach { case ((ldt, duration), expected) =>
+            val ts = ldt.atZone(zid).toInstant
+            val result = expected.atZone(zid).toInstant
+            val df = Seq((ts, duration, result)).toDF("ts", "interval", "result")
+            checkAnswer(
+              df.select($"ts" + $"interval", $"interval" + $"ts", $"result" - $"interval",
+                $"result" - $"ts"),
+              Row(result, result, ts, duration))
+          }
+        }
+      }
+
+      Seq(
+        "2021-03-16T18:56:00Z" -> "ts + i",
+        "1900-03-16T18:56:00Z" -> "ts - i").foreach { case (instant, op) =>
+        val e = intercept[SparkException] {
+          Seq(
+            (Instant.parse(instant), Duration.of(Long.MaxValue, ChronoUnit.MICROS)))
+            .toDF("ts", "i")
+            .selectExpr(op)
+            .collect()
+        }.getCause
+        assert(e.isInstanceOf[ArithmeticException])
+        assert(e.getMessage.contains("long overflow"))
+      }
+    }
+  }
+
+  test("SPARK-34824: multiply year-month interval by numeric") {
+    checkAnswer(
+      Seq((Period.ofYears(0), 0)).toDF("i", "n").select($"i" * $"n"),
+      Row(Period.ofYears(0)))
+    checkAnswer(
+      Seq((Period.ofMonths(0), 10.toByte)).toDF("i", "n").select($"i" * $"n"),
+      Row(Period.ofMonths(0)))
+    checkAnswer(
+      Seq((Period.ofMonths(5), 3.toShort)).toDF("i", "n").select($"n" * $"i"),
+      Row(Period.ofYears(1).plusMonths(3)))
+    checkAnswer(
+      Seq((Period.ofYears(1000), "2")).toDF("i", "n").select($"i" * $"n"),
+      Row(Period.ofYears(2000)))
+    checkAnswer(
+      Seq((Period.ofMonths(1), 12L)).toDF("i", "n").select($"n" * $"i"),
+      Row(Period.ofYears(1)))
+    checkAnswer(
+      Seq((Period.ofYears(100).plusMonths(11), Short.MaxValue)).toDF("i", "n").select($"n" * $"i"),
+      Row(Period.ofYears(100).plusMonths(11).multipliedBy(Short.MaxValue).normalized()))
+    checkAnswer(
+      Seq((Period.ofMonths(-1), 0.499f)).toDF("i", "n").select($"i" * $"n"),
+      Row(Period.ofMonths(0)))
+    checkAnswer(
+      Seq((Period.ofMonths(10000000), 0.0000001d)).toDF("i", "n").select($"i" * $"n"),
+      Row(Period.ofMonths(1)))
+    checkAnswer(
+      Seq((Period.ofMonths(-10000000), BigDecimal(0.0000001d))).toDF("i", "n").select($"i" * $"n"),
+      Row(Period.ofMonths(-1)))
+    checkAnswer(
+      Seq((Period.ofMonths(-1), BigDecimal(0.5))).toDF("i", "n").select($"i" * $"n"),
+      Row(Period.ofMonths(-1)))
+
+    val e = intercept[SparkException] {
+      Seq((Period.ofYears(9999), Long.MinValue)).toDF("i", "n").select($"n" * $"i").collect()
+    }.getCause
+    assert(e.isInstanceOf[ArithmeticException])
+    assert(e.getMessage.contains("overflow"))
+  }
+
+  test("SPARK-34850: multiply day-time interval by numeric") {
+    checkAnswer(
+      Seq((Duration.ofDays(0), 0)).toDF("i", "n").select($"i" * $"n"),
+      Row(Duration.ofDays(0)))
+    checkAnswer(
+      Seq((Duration.ofDays(0), 10.toByte)).toDF("i", "n").select($"i" * $"n"),
+      Row(Duration.ofDays(0)))
+    checkAnswer(
+      Seq((Duration.ofHours(12), 3.toShort)).toDF("i", "n").select($"n" * $"i"),
+      Row(Duration.ofDays(1).plusHours(12)))
+    checkAnswer(
+      Seq((Duration.ofMinutes(1000), "2")).toDF("i", "n").select($"i" * $"n"),
+      Row(Duration.ofMinutes(2000)))
+    checkAnswer(
+      Seq((Duration.ofSeconds(1), 60L)).toDF("i", "n").select($"n" * $"i"),
+      Row(Duration.ofMinutes(1)))
+    checkAnswer(
+      Seq((Duration.of(-1, ChronoUnit.MICROS), 0.499f)).toDF("i", "n").select($"i" * $"n"),
+      Row(Duration.of(0, ChronoUnit.MICROS)))
+    checkAnswer(
+      Seq((Duration.of(-1, ChronoUnit.MICROS), 0.51d)).toDF("i", "n").select($"i" * $"n"),
+      Row(Duration.of(-1, ChronoUnit.MICROS)))
+    checkAnswer(
+      Seq((Duration.of(-10000000, ChronoUnit.MICROS), BigDecimal(0.0000001d)))
+        .toDF("i", "n").select($"i" * $"n"),
+      Row(Duration.of(-1, ChronoUnit.MICROS)))
+
+    val e = intercept[SparkException] {
+      Seq((Duration.ofDays(9999), Long.MinValue)).toDF("i", "n").select($"n" * $"i").collect()
+    }.getCause
+    assert(e.isInstanceOf[ArithmeticException])
+    assert(e.getMessage.contains("overflow"))
+  }
+
+  test("SPARK-34868: divide year-month interval by numeric") {
+    checkAnswer(
+      Seq((Period.ofYears(0), 10.toByte)).toDF("i", "n").select($"i" / $"n"),
+      Row(Period.ofYears(0)))
+    checkAnswer(
+      Seq((Period.ofYears(10), 3.toShort)).toDF("i", "n").select($"i" / $"n"),
+      Row(Period.ofYears(3).plusMonths(4)))
+    checkAnswer(
+      Seq((Period.ofYears(1000), "2")).toDF("i", "n").select($"i" / $"n"),
+      Row(Period.ofYears(500)))
+    checkAnswer(
+      Seq((Period.ofMonths(1).multipliedBy(Int.MaxValue), Int.MaxValue))
+        .toDF("i", "n").select($"i" / $"n"),
+      Row(Period.ofMonths(1)))
+    checkAnswer(
+      Seq((Period.ofYears(-1), 12L)).toDF("i", "n").select($"i" / $"n"),
+      Row(Period.ofMonths(-1)))
+    checkAnswer(
+      Seq((Period.ofMonths(-1), 0.499f)).toDF("i", "n").select($"i" / $"n"),
+      Row(Period.ofMonths(-2)))
+    checkAnswer(
+      Seq((Period.ofMonths(10000000), 10000000d)).toDF("i", "n").select($"i" / $"n"),
+      Row(Period.ofMonths(1)))
+    checkAnswer(
+      Seq((Period.ofMonths(-1), BigDecimal(0.5))).toDF("i", "n").select($"i" / $"n"),
+      Row(Period.ofMonths(-2)))
+
+    val e = intercept[SparkException] {
+      Seq((Period.ofYears(9999), 0)).toDF("i", "n").select($"i" / $"n").collect()
+    }.getCause
+    assert(e.isInstanceOf[ArithmeticException])
+    assert(e.getMessage.contains("/ by zero"))
+  }
+
+  test("SPARK-34875: divide day-time interval by numeric") {
+    checkAnswer(
+      Seq((Duration.ZERO, 10.toByte)).toDF("i", "n").select($"i" / $"n"),
+      Row(Duration.ZERO))
+    checkAnswer(
+      Seq((Duration.ofDays(10), 3.toShort)).toDF("i", "n").select($"i" / $"n"),
+      Row(Duration.ofDays(10).dividedBy(3)))
+    checkAnswer(
+      Seq((Duration.ofHours(1000), "2")).toDF("i", "n").select($"i" / $"n"),
+      Row(Duration.ofHours(500)))
+    checkAnswer(
+      Seq((Duration.of(1, ChronoUnit.MICROS).multipliedBy(Long.MaxValue), Long.MaxValue))
+        .toDF("i", "n").select($"i" / $"n"),
+      Row(Duration.of(1, ChronoUnit.MICROS)))
+    checkAnswer(
+      Seq((Duration.ofMinutes(-1), 60L)).toDF("i", "n").select($"i" / $"n"),
+      Row(Duration.ofSeconds(-1)))
+    checkAnswer(
+      Seq((Duration.ofDays(-1), 0.5f)).toDF("i", "n").select($"i" / $"n"),
+      Row(Duration.ofDays(-2)))
+    checkAnswer(
+      Seq((Duration.ofMillis(10000000), 10000000d)).toDF("i", "n").select($"i" / $"n"),
+      Row(Duration.ofMillis(1)))
+    checkAnswer(
+      Seq((Duration.of(-1, ChronoUnit.MICROS), BigDecimal(10000.0001)))
+        .toDF("i", "n").select($"i" / $"n"),
+      Row(Duration.of(-1, ChronoUnit.MICROS).multipliedBy(10000).dividedBy(100000001)))
+
+    val e = intercept[SparkException] {
+      Seq((Duration.ofDays(9999), 0)).toDF("i", "n").select($"i" / $"n").collect()
+    }.getCause
+    assert(e.isInstanceOf[ArithmeticException])
+    assert(e.getMessage.contains("/ by zero"))
+  }
+
+  test("SPARK-34896: return day-time interval from dates subtraction") {
+    withSQLConf(
+      SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true",
+      SQLConf.LEGACY_INTERVAL_ENABLED.key -> "false") {
+      outstandingTimezonesIds.foreach { zid =>
+        withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> zid) {
+          Seq(
+            (LocalDate.of(1582, 10, 15), LocalDate.of(1582, 10, 4)),
+            (LocalDate.of(1900, 10, 1), LocalDate.of(1900, 10, 1)),
+            (LocalDate.of(1969, 12, 1), LocalDate.of(1970, 1, 1)),
+            (LocalDate.of(2021, 3, 1), LocalDate.of(2020, 2, 29)),
+            (LocalDate.of(2021, 3, 15), LocalDate.of(2021, 3, 14)),
+            (LocalDate.of(1, 1, 1), LocalDate.of(2021, 3, 29))
+          ).foreach { case (end, start) =>
+            val df = Seq((end, start)).toDF("end", "start")
+            val daysBetween = Duration.ofDays(ChronoUnit.DAYS.between(start, end))
+            checkAnswer(df.select($"end" - $"start"), Row(daysBetween))
+          }
+        }
+      }
+
+      val e = intercept[SparkException] {
+        Seq((LocalDate.ofEpochDay(0), LocalDate.of(500000, 1, 1)))
+          .toDF("start", "end")
+          .select($"end" - $"start")
+          .collect()
+      }.getCause
+      assert(e.isInstanceOf[ArithmeticException])
+      assert(e.getMessage.contains("long overflow"))
+    }
+  }
+
+  test("SPARK-34903: Return day-time interval from timestamps subtraction") {
+    outstandingTimezonesIds.foreach { tz =>
+      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz) {
+        checkAnswer(
+          sql("select timestamp '2021-03-31 19:11:10' - timestamp '2021-03-01 19:11:10'"),
+          Row(Duration.ofDays(30)))
+        checkAnswer(
+          Seq((Instant.parse("2021-03-31T00:01:02Z"), Instant.parse("2021-04-01T00:00:00Z")))
+            .toDF("start", "end").select($"end" - $"start" < Duration.ofDays(1)),
+          Row(true))
+        checkAnswer(
+          Seq((Instant.parse("2021-03-31T00:01:02.777Z"), Duration.ofMillis(333)))
+            .toDF("ts", "i")
+            .select(($"ts" + $"i") - $"ts"),
+          Row(Duration.ofMillis(333)))
+        checkAnswer(
+          Seq((LocalDateTime.of(2021, 3, 31, 10, 0, 0)
+              .atZone(DateTimeUtils.getZoneId(tz)).toInstant, LocalDate.of(2020, 3, 31)))
+            .toDF("ts", "d")
+          .select($"ts" - $"d"),
+          Row(Duration.ofDays(365).plusHours(10)))
+      }
+    }
+  }
+
+  test("SPARK-35051: add/subtract a day-time interval to/from a date") {
+    withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
+      outstandingZoneIds.foreach { zid =>
+        withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> zid.getId) {
+          Seq(
+            (LocalDate.of(1, 1, 1), Duration.ofDays(31)) -> LocalDateTime.of(1, 2, 1, 0, 0, 0),
+            (LocalDate.of(1582, 9, 15), Duration.ofDays(30).plus(1, ChronoUnit.MICROS)) ->
+              LocalDateTime.of(1582, 10, 15, 0, 0, 0, 1000),
+            (LocalDate.of(1900, 1, 1), Duration.ofDays(0).plusHours(1)) ->
+              LocalDateTime.of(1900, 1, 1, 1, 0, 0),
+            (LocalDate.of(1970, 1, 1), Duration.ofDays(-1).minusMinutes(1)) ->
+              LocalDateTime.of(1969, 12, 30, 23, 59, 0),
+            (LocalDate.of(2021, 3, 14), Duration.ofDays(1)) ->
+              LocalDateTime.of(2021, 3, 15, 0, 0, 0),
+            (LocalDate.of(2020, 12, 31), Duration.ofDays(4 * 30).plusMinutes(30)) ->
+              LocalDateTime.of(2021, 4, 30, 0, 30, 0),
+            (LocalDate.of(2020, 2, 29), Duration.ofDays(365).plusSeconds(59)) ->
+              LocalDateTime.of(2021, 2, 28, 0, 0, 59),
+            (LocalDate.of(10000, 1, 1), Duration.ofDays(-2)) ->
+              LocalDateTime.of(9999, 12, 30, 0, 0, 0)
+          ).foreach { case ((date, duration), expected) =>
+            val result = expected.atZone(zid).toInstant
+            val ts = date.atStartOfDay(zid).toInstant
+            val df = Seq((date, duration, result)).toDF("date", "interval", "result")
+            checkAnswer(
+              df.select($"date" + $"interval", $"interval" + $"date", $"result" - $"interval",
+                $"result" - $"date"),
+              Row(result, result, ts, duration))
+          }
+        }
+      }
+
+      Seq(
+        "2021-04-14" -> "date + i",
+        "1900-04-14" -> "date - i").foreach { case (date, op) =>
+        val e = intercept[SparkException] {
+          Seq(
+            (LocalDate.parse(date), Duration.of(Long.MaxValue, ChronoUnit.MICROS)))
+            .toDF("date", "i")
+            .selectExpr(op)
+            .collect()
+        }.getCause
+        assert(e.isInstanceOf[ArithmeticException])
+        assert(e.getMessage.contains("long overflow"))
+      }
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ComplexTypesSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ComplexTypesSuite.scala
index 6b503334f9f23..bdcf7230e3211 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ComplexTypesSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ComplexTypesSuite.scala
@@ -17,11 +17,15 @@
 
 package org.apache.spark.sql
 
+import scala.collection.JavaConverters._
+
 import org.apache.spark.sql.catalyst.expressions.CreateNamedStruct
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.{ArrayType, StructType}
 
 class ComplexTypesSuite extends QueryTest with SharedSparkSession {
+  import testImplicits._
 
   override def beforeAll(): Unit = {
     super.beforeAll()
@@ -106,4 +110,11 @@ class ComplexTypesSuite extends QueryTest with SharedSparkSession {
     checkAnswer(df1, Row(10, 12) :: Row(11, 13) :: Nil)
     checkNamedStruct(df.queryExecution.optimizedPlan, expectedCount = 0)
   }
+
+  test("SPARK-32167: get field from an array of struct") {
+    val innerStruct = new StructType().add("i", "int", nullable = true)
+    val schema = new StructType().add("arr", ArrayType(innerStruct, containsNull = false))
+    val df = spark.createDataFrame(List(Row(Seq(Row(1), Row(null)))).asJava, schema)
+    checkAnswer(df.select($"arr".getField("i")), Row(Seq(1, null)))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala
index 800e294cca8c4..16b92d6d11c91 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala
@@ -80,16 +80,16 @@ class CsvFunctionsSuite extends QueryTest with SharedSparkSession {
   test("schema_of_csv - infers schemas") {
     checkAnswer(
       spark.range(1).select(schema_of_csv(lit("0.1,1"))),
-      Seq(Row("struct<_c0:double,_c1:int>")))
+      Seq(Row("STRUCT<`_c0`: DOUBLE, `_c1`: INT>")))
     checkAnswer(
       spark.range(1).select(schema_of_csv("0.1,1")),
-      Seq(Row("struct<_c0:double,_c1:int>")))
+      Seq(Row("STRUCT<`_c0`: DOUBLE, `_c1`: INT>")))
   }
 
   test("schema_of_csv - infers schemas using options") {
     val df = spark.range(1)
       .select(schema_of_csv(lit("0.1 1"), Map("sep" -> " ").asJava))
-    checkAnswer(df, Seq(Row("struct<_c0:double,_c1:int>")))
+    checkAnswer(df, Seq(Row("STRUCT<`_c0`: DOUBLE, `_c1`: INT>")))
   }
 
   test("to_csv - struct") {
@@ -236,7 +236,7 @@ class CsvFunctionsSuite extends QueryTest with SharedSparkSession {
     val input = concat_ws(",", lit(0.1), lit(1))
     checkAnswer(
       spark.range(1).select(schema_of_csv(input)),
-      Seq(Row("struct<_c0:double,_c1:int>")))
+      Seq(Row("STRUCT<`_c0`: DOUBLE, `_c1`: INT>")))
   }
 
   test("optional datetime parser does not affect csv time formatting") {
@@ -250,4 +250,52 @@ class CsvFunctionsSuite extends QueryTest with SharedSparkSession {
          | """.stripMargin)
     checkAnswer(toDF("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"), toDF("yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]"))
   }
+
+  test("SPARK-32968: Pruning csv field should not change result") {
+    Seq("true", "false").foreach { enabled =>
+      withSQLConf(SQLConf.CSV_EXPRESSION_OPTIMIZATION.key -> enabled) {
+        val df1 = sparkContext.parallelize(Seq("a,b")).toDF("csv")
+          .selectExpr("from_csv(csv, 'a string, b string', map('mode', 'failfast')) as parsed")
+        checkAnswer(df1.selectExpr("parsed.a"), Seq(Row("a")))
+        checkAnswer(df1.selectExpr("parsed.b"), Seq(Row("b")))
+
+        val df2 = sparkContext.parallelize(Seq("a,b")).toDF("csv")
+          .selectExpr("from_csv(csv, 'a string, b string') as parsed")
+        checkAnswer(df2.selectExpr("parsed.a"), Seq(Row("a")))
+        checkAnswer(df2.selectExpr("parsed.b"), Seq(Row("b")))
+      }
+    }
+  }
+
+  test("SPARK-32968: bad csv input with csv pruning optimization") {
+    Seq("true", "false").foreach { enabled =>
+      withSQLConf(SQLConf.CSV_EXPRESSION_OPTIMIZATION.key -> enabled) {
+        val df = sparkContext.parallelize(Seq("1,\u0001\u0000\u0001234")).toDF("csv")
+          .selectExpr("from_csv(csv, 'a int, b int', map('mode', 'failfast')) as parsed")
+
+        val err1 = intercept[SparkException] {
+          df.selectExpr("parsed.a").collect
+        }
+
+        val err2 = intercept[SparkException] {
+          df.selectExpr("parsed.b").collect
+        }
+
+        assert(err1.getMessage.contains("Malformed records are detected in record parsing"))
+        assert(err2.getMessage.contains("Malformed records are detected in record parsing"))
+      }
+    }
+  }
+
+  test("SPARK-32968: csv pruning optimization with corrupt record field") {
+    Seq("true", "false").foreach { enabled =>
+      withSQLConf(SQLConf.CSV_EXPRESSION_OPTIMIZATION.key -> enabled) {
+        val df = sparkContext.parallelize(Seq("a,b,c,d")).toDF("csv")
+          .selectExpr("from_csv(csv, 'a string, b string, _corrupt_record string') as parsed")
+          .selectExpr("parsed._corrupt_record")
+
+        checkAnswer(df, Seq(Row("a,b,c,d")))
+      }
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
index 2293d4ae61aff..c6f6cbdbf02c6 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
@@ -17,10 +17,13 @@
 
 package org.apache.spark.sql
 
+import java.time.{Duration, Period}
+
 import scala.util.Random
 
-import org.scalatest.Matchers.the
+import org.scalatest.matchers.must.Matchers.the
 
+import org.apache.spark.SparkException
 import org.apache.spark.sql.execution.WholeStageCodegenExec
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
 import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec}
@@ -31,7 +34,6 @@ import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.test.SQLTestData.DecimalData
 import org.apache.spark.sql.types._
-import org.apache.spark.unsafe.types.CalendarInterval
 
 case class Fact(date: Int, hour: Int, minute: Int, room_name: String, temp: Double)
 
@@ -298,7 +300,7 @@ class DataFrameAggregateSuite extends QueryTest
       Row(2.0, 2.0))
 
     checkAnswer(
-      testData2.agg(avg($"a"), sumDistinct($"a")), // non-partial
+      testData2.agg(avg($"a"), sumDistinct($"a")), // non-partial and test deprecated version
       Row(2.0, 6.0) :: Nil)
 
     checkAnswer(
@@ -306,7 +308,7 @@ class DataFrameAggregateSuite extends QueryTest
       Row(new java.math.BigDecimal(2)))
 
     checkAnswer(
-      decimalData.agg(avg($"a"), sumDistinct($"a")), // non-partial
+      decimalData.agg(avg($"a"), sum_distinct($"a")), // non-partial
       Row(new java.math.BigDecimal(2), new java.math.BigDecimal(6)) :: Nil)
 
     checkAnswer(
@@ -315,7 +317,7 @@ class DataFrameAggregateSuite extends QueryTest
     // non-partial
     checkAnswer(
       decimalData.agg(
-        avg($"a" cast DecimalType(10, 2)), sumDistinct($"a" cast DecimalType(10, 2))),
+        avg($"a" cast DecimalType(10, 2)), sum_distinct($"a" cast DecimalType(10, 2))),
       Row(new java.math.BigDecimal(2), new java.math.BigDecimal(6)) :: Nil)
   }
 
@@ -325,11 +327,11 @@ class DataFrameAggregateSuite extends QueryTest
       Row(2.0))
 
     checkAnswer(
-      testData3.agg(avg($"b"), countDistinct($"b")),
+      testData3.agg(avg($"b"), count_distinct($"b")),
       Row(2.0, 1))
 
     checkAnswer(
-      testData3.agg(avg($"b"), sumDistinct($"b")), // non-partial
+      testData3.agg(avg($"b"), sum_distinct($"b")), // non-partial
       Row(2.0, 2.0))
   }
 
@@ -340,7 +342,7 @@ class DataFrameAggregateSuite extends QueryTest
       Row(null))
 
     checkAnswer(
-      emptyTableData.agg(avg($"a"), sumDistinct($"b")), // non-partial
+      emptyTableData.agg(avg($"a"), sum_distinct($"b")), // non-partial
       Row(null, null))
   }
 
@@ -348,7 +350,7 @@ class DataFrameAggregateSuite extends QueryTest
     assert(testData2.count() === testData2.rdd.map(_ => 1).count())
 
     checkAnswer(
-      testData2.agg(count($"a"), sumDistinct($"a")), // non-partial
+      testData2.agg(count($"a"), sum_distinct($"a")), // non-partial
       Row(6, 6.0))
   }
 
@@ -365,12 +367,12 @@ class DataFrameAggregateSuite extends QueryTest
 
     checkAnswer(
       testData3.agg(
-        count($"a"), count($"b"), count(lit(1)), countDistinct($"a"), countDistinct($"b")),
+        count($"a"), count($"b"), count(lit(1)), count_distinct($"a"), count_distinct($"b")),
       Row(2, 1, 2, 2, 1)
     )
 
     checkAnswer(
-      testData3.agg(count($"b"), countDistinct($"b"), sumDistinct($"b")), // non-partial
+      testData3.agg(count($"b"), count_distinct($"b"), sum_distinct($"b")), // non-partial
       Row(1, 1, 2)
     )
   }
@@ -385,17 +387,17 @@ class DataFrameAggregateSuite extends QueryTest
       .toDF("key1", "key2", "key3")
 
     checkAnswer(
-      df1.agg(countDistinct($"key1", $"key2")),
+      df1.agg(count_distinct($"key1", $"key2")),
       Row(3)
     )
 
     checkAnswer(
-      df1.agg(countDistinct($"key1", $"key2", $"key3")),
+      df1.agg(count_distinct($"key1", $"key2", $"key3")),
       Row(3)
     )
 
     checkAnswer(
-      df1.groupBy($"key1").agg(countDistinct($"key2", $"key3")),
+      df1.groupBy($"key1").agg(count_distinct($"key2", $"key3")),
       Seq(Row("a", 2), Row("x", 1))
     )
   }
@@ -403,7 +405,7 @@ class DataFrameAggregateSuite extends QueryTest
   test("zero count") {
     val emptyTableData = Seq.empty[(Int, Int)].toDF("a", "b")
     checkAnswer(
-      emptyTableData.agg(count($"a"), sumDistinct($"a")), // non-partial
+      emptyTableData.agg(count($"a"), sum_distinct($"a")), // non-partial
       Row(0, null))
   }
 
@@ -434,7 +436,7 @@ class DataFrameAggregateSuite extends QueryTest
   test("zero sum distinct") {
     val emptyTableData = Seq.empty[(Int, Int)].toDF("a", "b")
     checkAnswer(
-      emptyTableData.agg(sumDistinct($"a")),
+      emptyTableData.agg(sum_distinct($"a")),
       Row(null))
   }
 
@@ -457,25 +459,51 @@ class DataFrameAggregateSuite extends QueryTest
   }
 
   test("zero moments") {
-    val input = Seq((1, 2)).toDF("a", "b")
-    checkAnswer(
-      input.agg(stddev($"a"), stddev_samp($"a"), stddev_pop($"a"), variance($"a"),
-        var_samp($"a"), var_pop($"a"), skewness($"a"), kurtosis($"a")),
-      Row(Double.NaN, Double.NaN, 0.0, Double.NaN, Double.NaN, 0.0,
-        Double.NaN, Double.NaN))
+    withSQLConf(SQLConf.LEGACY_STATISTICAL_AGGREGATE.key -> "true") {
+      val input = Seq((1, 2)).toDF("a", "b")
+      checkAnswer(
+        input.agg(stddev($"a"), stddev_samp($"a"), stddev_pop($"a"), variance($"a"),
+          var_samp($"a"), var_pop($"a"), skewness($"a"), kurtosis($"a")),
+        Row(Double.NaN, Double.NaN, 0.0, Double.NaN, Double.NaN, 0.0,
+          Double.NaN, Double.NaN))
 
-    checkAnswer(
-      input.agg(
-        expr("stddev(a)"),
-        expr("stddev_samp(a)"),
-        expr("stddev_pop(a)"),
-        expr("variance(a)"),
-        expr("var_samp(a)"),
-        expr("var_pop(a)"),
-        expr("skewness(a)"),
-        expr("kurtosis(a)")),
-      Row(Double.NaN, Double.NaN, 0.0, Double.NaN, Double.NaN, 0.0,
-        Double.NaN, Double.NaN))
+      checkAnswer(
+        input.agg(
+          expr("stddev(a)"),
+          expr("stddev_samp(a)"),
+          expr("stddev_pop(a)"),
+          expr("variance(a)"),
+          expr("var_samp(a)"),
+          expr("var_pop(a)"),
+          expr("skewness(a)"),
+          expr("kurtosis(a)")),
+        Row(Double.NaN, Double.NaN, 0.0, Double.NaN, Double.NaN, 0.0,
+          Double.NaN, Double.NaN))
+    }
+  }
+
+  test("SPARK-13860: zero moments LEGACY_STATISTICAL_AGGREGATE off") {
+    withSQLConf(SQLConf.LEGACY_STATISTICAL_AGGREGATE.key -> "false") {
+      val input = Seq((1, 2)).toDF("a", "b")
+      checkAnswer(
+        input.agg(stddev($"a"), stddev_samp($"a"), stddev_pop($"a"), variance($"a"),
+          var_samp($"a"), var_pop($"a"), skewness($"a"), kurtosis($"a")),
+        Row(null, null, 0.0, null, null, 0.0,
+          null, null))
+
+      checkAnswer(
+        input.agg(
+          expr("stddev(a)"),
+          expr("stddev_samp(a)"),
+          expr("stddev_pop(a)"),
+          expr("variance(a)"),
+          expr("var_samp(a)"),
+          expr("var_pop(a)"),
+          expr("skewness(a)"),
+          expr("kurtosis(a)")),
+        Row(null, null, 0.0, null, null, 0.0,
+          null, null))
+    }
   }
 
   test("null moments") {
@@ -597,7 +625,7 @@ class DataFrameAggregateSuite extends QueryTest
     val df = Seq((1, 3, "a"), (1, 2, "b"), (3, 4, "c"), (3, 4, "c"), (3, 5, "d"))
       .toDF("x", "y", "z")
     checkAnswer(
-      df.groupBy($"x").agg(countDistinct($"y"), sort_array(collect_list($"z"))),
+      df.groupBy($"x").agg(count_distinct($"y"), sort_array(collect_list($"z"))),
       Seq(Row(1, 2, Seq("a", "b")), Row(3, 2, Seq("c", "c", "d"))))
   }
 
@@ -812,7 +840,7 @@ class DataFrameAggregateSuite extends QueryTest
     )
   }
 
-  test("SPARK-27581: DataFrame countDistinct(\"*\") shouldn't fail with AnalysisException") {
+  test("SPARK-27581: DataFrame count_distinct(\"*\") shouldn't fail with AnalysisException") {
     val df = sql("select id % 100 from range(100000)")
     val distinctCount1 = df.select(expr("count(distinct(*))"))
     val distinctCount2 = df.select(countDistinct("*"))
@@ -976,7 +1004,8 @@ class DataFrameAggregateSuite extends QueryTest
 
   Seq(true, false).foreach { value =>
     test(s"SPARK-31620: agg with subquery (whole-stage-codegen = $value)") {
-      withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> value.toString) {
+      withSQLConf(
+        SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> value.toString) {
         withTempView("t1", "t2") {
           sql("create temporary view t1 as select * from values (1, 2) as t1(a, b)")
           sql("create temporary view t2 as select * from values (3, 4) as t2(c, d)")
@@ -999,17 +1028,175 @@ class DataFrameAggregateSuite extends QueryTest
 
           // test SortAggregateExec
           var df = sql("select max(if(c > (select a from t1), 'str1', 'str2')) as csum from t2")
-          assert(df.queryExecution.executedPlan
-            .find { case _: SortAggregateExec => true }.isDefined)
+          assert(find(df.queryExecution.executedPlan)(_.isInstanceOf[SortAggregateExec]).isDefined)
           checkAnswer(df, Row("str1") :: Nil)
 
           // test ObjectHashAggregateExec
           df = sql("select collect_list(d), sum(if(c > (select a from t1), d, 0)) as csum from t2")
-          assert(df.queryExecution.executedPlan
-            .find { case _: ObjectHashAggregateExec => true }.isDefined)
+          assert(
+            find(df.queryExecution.executedPlan)(_.isInstanceOf[ObjectHashAggregateExec]).isDefined)
           checkAnswer(df, Row(Array(4), 4) :: Nil)
         }
       }
     }
   }
+
+  test("SPARK-32038: NormalizeFloatingNumbers should work on distinct aggregate") {
+    withTempView("view") {
+      val nan1 = java.lang.Float.intBitsToFloat(0x7f800001)
+      val nan2 = java.lang.Float.intBitsToFloat(0x7fffffff)
+
+      Seq(("mithunr", Float.NaN),
+        ("mithunr", nan1),
+        ("mithunr", nan2),
+        ("abellina", 1.0f),
+        ("abellina", 2.0f)).toDF("uid", "score").createOrReplaceTempView("view")
+
+      val df = spark.sql("select uid, count(distinct score) from view group by 1 order by 1 asc")
+      checkAnswer(df, Row("abellina", 2) :: Row("mithunr", 1) :: Nil)
+    }
+  }
+
+  test("SPARK-32136: NormalizeFloatingNumbers should work on null struct") {
+    val df = Seq(
+      A(None),
+      A(Some(B(None))),
+      A(Some(B(Some(1.0))))).toDF
+    val groupBy = df.groupBy("b").agg(count("*"))
+    checkAnswer(groupBy, Row(null, 1) :: Row(Row(null), 1) :: Row(Row(1.0), 1) :: Nil)
+  }
+
+  test("SPARK-32344: Unevaluable's set to FIRST/LAST ignoreNullsExpr in distinct aggregates") {
+    val queryTemplate = (agg: String) =>
+      s"SELECT $agg(DISTINCT v) FROM (SELECT v FROM VALUES 1, 2, 3 t(v) ORDER BY v)"
+    checkAnswer(sql(queryTemplate("FIRST")), Row(1))
+    checkAnswer(sql(queryTemplate("LAST")), Row(3))
+  }
+
+  test("SPARK-32906: struct field names should not change after normalizing floats") {
+    val df = Seq(Tuple1(Tuple2(-0.0d, Double.NaN)), Tuple1(Tuple2(0.0d, Double.NaN))).toDF("k")
+    val aggs = df.distinct().queryExecution.sparkPlan.collect { case a: HashAggregateExec => a }
+    assert(aggs.length == 2)
+    assert(aggs.head.output.map(_.dataType.simpleString).head ===
+      aggs.last.output.map(_.dataType.simpleString).head)
+  }
+
+  test("SPARK-33726: Aggregation on a table where a column name is reused") {
+    val query =
+      """|with T as (
+         |select id as a, -id as x from range(3)),
+         |U as (
+         |select id as b, cast(id as string) as x from range(3))
+         |select T.x, U.x, min(a) as ma, min(b) as mb
+         |from T join U on a=b
+         |group by U.x, T.x
+      """.stripMargin
+    val df = spark.sql(query)
+    checkAnswer(df, Row(0, "0", 0, 0) :: Row(-1, "1", 1, 1) :: Row(-2, "2", 2, 2) :: Nil)
+  }
+
+  test("SPARK-34713: group by CreateStruct with ExtractValue") {
+    val structDF = Seq(Tuple1(1 -> 1)).toDF("col")
+    checkAnswer(structDF.groupBy(struct($"col._1")).count().select("count"), Row(1))
+
+    val arrayOfStructDF = Seq(Tuple1(Seq(1 -> 1))).toDF("col")
+    checkAnswer(arrayOfStructDF.groupBy(struct($"col._1")).count().select("count"), Row(1))
+
+    val mapDF = Seq(Tuple1(Map("a" -> "a"))).toDF("col")
+    checkAnswer(mapDF.groupBy(struct($"col.a")).count().select("count"), Row(1))
+
+    val nonStringMapDF = Seq(Tuple1(Map(1 -> 1))).toDF("col")
+    // Spark implicit casts string literal "a" to int to match the key type.
+    checkAnswer(nonStringMapDF.groupBy(struct($"col.a")).count().select("count"), Row(1))
+
+    val arrayDF = Seq(Tuple1(Seq(1))).toDF("col")
+    val e = intercept[AnalysisException](arrayDF.groupBy(struct($"col.a")).count())
+    assert(e.message.contains("requires integral type"))
+  }
+
+  test("SPARK-34716: Support ANSI SQL intervals by the aggregate function `sum`") {
+    val df = Seq((1, Period.ofMonths(10), Duration.ofDays(10)),
+      (2, Period.ofMonths(1), Duration.ofDays(1)),
+      (2, null, null),
+      (3, Period.ofMonths(-3), Duration.ofDays(-6)),
+      (3, Period.ofMonths(21), Duration.ofDays(-5)))
+      .toDF("class", "year-month", "day-time")
+
+    val df2 = Seq((Period.ofMonths(Int.MaxValue), Duration.ofDays(106751991)),
+      (Period.ofMonths(10), Duration.ofDays(10)))
+      .toDF("year-month", "day-time")
+
+    val sumDF = df.select(sum($"year-month"), sum($"day-time"))
+    checkAnswer(sumDF, Row(Period.of(2, 5, 0), Duration.ofDays(0)))
+    assert(find(sumDF.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined)
+    assert(sumDF.schema == StructType(Seq(StructField("sum(year-month)", YearMonthIntervalType),
+      StructField("sum(day-time)", DayTimeIntervalType))))
+
+    val sumDF2 = df.groupBy($"class").agg(sum($"year-month"), sum($"day-time"))
+    checkAnswer(sumDF2, Row(1, Period.ofMonths(10), Duration.ofDays(10)) ::
+      Row(2, Period.ofMonths(1), Duration.ofDays(1)) ::
+      Row(3, Period.of(1, 6, 0), Duration.ofDays(-11)) :: Nil)
+    assert(find(sumDF2.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined)
+    assert(sumDF2.schema == StructType(Seq(StructField("class", IntegerType, false),
+      StructField("sum(year-month)", YearMonthIntervalType),
+      StructField("sum(day-time)", DayTimeIntervalType))))
+
+    val error = intercept[SparkException] {
+      checkAnswer(df2.select(sum($"year-month")), Nil)
+    }
+    assert(error.toString contains "java.lang.ArithmeticException: integer overflow")
+
+    val error2 = intercept[SparkException] {
+      checkAnswer(df2.select(sum($"day-time")), Nil)
+    }
+    assert(error2.toString contains "java.lang.ArithmeticException: long overflow")
+  }
+
+  test("SPARK-34837: Support ANSI SQL intervals by the aggregate function `avg`") {
+    val df = Seq((1, Period.ofMonths(10), Duration.ofDays(10)),
+      (2, Period.ofMonths(1), Duration.ofDays(1)),
+      (2, null, null),
+      (3, Period.ofMonths(-3), Duration.ofDays(-6)),
+      (3, Period.ofMonths(21), Duration.ofDays(-5)))
+      .toDF("class", "year-month", "day-time")
+
+    val df2 = Seq((Period.ofMonths(Int.MaxValue), Duration.ofDays(106751991)),
+      (Period.ofMonths(10), Duration.ofDays(10)))
+      .toDF("year-month", "day-time")
+
+    val avgDF = df.select(avg($"year-month"), avg($"day-time"))
+    checkAnswer(avgDF, Row(Period.ofMonths(7), Duration.ofDays(0)))
+    assert(find(avgDF.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined)
+    assert(avgDF.schema == StructType(Seq(StructField("avg(year-month)", YearMonthIntervalType),
+      StructField("avg(day-time)", DayTimeIntervalType))))
+
+    val avgDF2 = df.groupBy($"class").agg(avg($"year-month"), avg($"day-time"))
+    checkAnswer(avgDF2, Row(1, Period.ofMonths(10), Duration.ofDays(10)) ::
+      Row(2, Period.ofMonths(1), Duration.ofDays(1)) ::
+      Row(3, Period.ofMonths(9), Duration.ofDays(-5).plusHours(-12)) :: Nil)
+    assert(find(avgDF2.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined)
+    assert(avgDF2.schema == StructType(Seq(StructField("class", IntegerType, false),
+      StructField("avg(year-month)", YearMonthIntervalType),
+      StructField("avg(day-time)", DayTimeIntervalType))))
+
+    val error = intercept[SparkException] {
+      checkAnswer(df2.select(avg($"year-month")), Nil)
+    }
+    assert(error.toString contains "java.lang.ArithmeticException: integer overflow")
+
+    val error2 = intercept[SparkException] {
+      checkAnswer(df2.select(avg($"day-time")), Nil)
+    }
+    assert(error2.toString contains "java.lang.ArithmeticException: long overflow")
+
+    val df3 = df.filter($"class" > 4)
+    val avgDF3 = df3.select(avg($"year-month"), avg($"day-time"))
+    checkAnswer(avgDF3, Row(null, null) :: Nil)
+
+    val avgDF4 = df3.groupBy($"class").agg(avg($"year-month"), avg($"day-time"))
+    checkAnswer(avgDF4, Nil)
+  }
 }
+
+case class B(c: Option[Double])
+case class A(b: Option[B])
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
index ac98d3f0c7095..eb71f5966a9c9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
@@ -19,17 +19,15 @@ package org.apache.spark.sql
 
 import java.nio.charset.StandardCharsets
 import java.sql.{Date, Timestamp}
-import java.util.TimeZone
 
 import scala.util.Random
 
 import org.apache.spark.SparkException
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.{Expression, UnaryExpression}
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
 import org.apache.spark.sql.catalyst.plans.logical.OneRowRelation
-import org.apache.spark.sql.catalyst.util.DateTimeTestUtils
-import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.UTC
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, UTC}
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
@@ -173,10 +171,10 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
     )
   }
 
-  test("bitwiseNOT") {
+  test("bitwise_not") {
     checkAnswer(
-      testData2.select(bitwiseNOT($"a")),
-      testData2.collect().toSeq.map(r => Row(~r.getInt(0))))
+      testData2.select(bitwiseNOT($"a"), bitwise_not($"a")),
+      testData2.collect().toSeq.map(r => Row(~r.getInt(0), ~r.getInt(0))))
   }
 
   test("bin") {
@@ -465,7 +463,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
 
     assert(intercept[AnalysisException] {
       df3.selectExpr("array_sort(a)").collect()
-    }.getMessage().contains("argument 1 requires array type, however, '`a`' is of string type"))
+    }.getMessage().contains("argument 1 requires array type, however, 'a' is of string type"))
   }
 
   def testSizeOfArray(sizeOfNull: Any): Unit = {
@@ -938,11 +936,21 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
     val answer = Seq(Row(Seq(2, 3)), Row(Seq(5)))
 
     checkAnswer(df.select(slice(df("x"), 2, 2)), answer)
+    checkAnswer(df.select(slice(df("x"), lit(2), lit(2))), answer)
     checkAnswer(df.selectExpr("slice(x, 2, 2)"), answer)
 
     val answerNegative = Seq(Row(Seq(3)), Row(Seq(5)))
     checkAnswer(df.select(slice(df("x"), -1, 1)), answerNegative)
+    checkAnswer(df.select(slice(df("x"), lit(-1), lit(1))), answerNegative)
     checkAnswer(df.selectExpr("slice(x, -1, 1)"), answerNegative)
+
+    val answerStartExpr = Seq(Row(Seq(2)), Row(Seq(4)))
+    checkAnswer(df.select(slice(df("x"), size($"x") - 1, lit(1))), answerStartExpr)
+    checkAnswer(df.selectExpr("slice(x, size(x) - 1, 1)"), answerStartExpr)
+
+    val answerLengthExpr = Seq(Row(Seq(1, 2)), Row(Seq(4)))
+    checkAnswer(df.select(slice(df("x"), lit(1), size($"x") - 1)), answerLengthExpr)
+    checkAnswer(df.selectExpr("slice(x, 1, size(x) - 1)"), answerLengthExpr)
   }
 
   test("array_join function") {
@@ -1029,7 +1037,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
         Timestamp.valueOf("2018-01-01 12:00:00"),
         Timestamp.valueOf("2018-01-02 00:00:00")))))
 
-    DateTimeTestUtils.withDefaultTimeZone(UTC) {
+    withDefaultTimeZone(UTC) {
       checkAnswer(
         spark.sql("select sequence(" +
           "   cast('2018-01-01' as date)" +
@@ -2250,7 +2258,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
     val ex3 = intercept[AnalysisException] {
       df.selectExpr("transform(a, x -> x)")
     }
-    assert(ex3.getMessage.contains("cannot resolve '`a`'"))
+    assert(ex3.getMessage.contains("cannot resolve 'a'"))
   }
 
   test("map_filter") {
@@ -2321,7 +2329,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
     val ex4 = intercept[AnalysisException] {
       df.selectExpr("map_filter(a, (k, v) -> k > v)")
     }
-    assert(ex4.getMessage.contains("cannot resolve '`a`'"))
+    assert(ex4.getMessage.contains("cannot resolve 'a'"))
   }
 
   test("filter function - array for primitive type not containing null") {
@@ -2480,7 +2488,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
     val ex4 = intercept[AnalysisException] {
       df.selectExpr("filter(a, x -> x)")
     }
-    assert(ex4.getMessage.contains("cannot resolve '`a`'"))
+    assert(ex4.getMessage.contains("cannot resolve 'a'"))
   }
 
   test("exists function - array for primitive type not containing null") {
@@ -2612,7 +2620,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
     val ex4 = intercept[AnalysisException] {
       df.selectExpr("exists(a, x -> x)")
     }
-    assert(ex4.getMessage.contains("cannot resolve '`a`'"))
+    assert(ex4.getMessage.contains("cannot resolve 'a'"))
   }
 
   test("forall function - array for primitive type not containing null") {
@@ -2758,12 +2766,12 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
     val ex4 = intercept[AnalysisException] {
       df.selectExpr("forall(a, x -> x)")
     }
-    assert(ex4.getMessage.contains("cannot resolve '`a`'"))
+    assert(ex4.getMessage.contains("cannot resolve 'a'"))
 
     val ex4a = intercept[AnalysisException] {
       df.select(forall(col("a"), x => x))
     }
-    assert(ex4a.getMessage.contains("cannot resolve '`a`'"))
+    assert(ex4a.getMessage.contains("cannot resolve 'a'"))
   }
 
   test("aggregate function - array for primitive type not containing null") {
@@ -2940,7 +2948,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
     val ex5 = intercept[AnalysisException] {
       df.selectExpr("aggregate(a, 0, (acc, x) -> x)")
     }
-    assert(ex5.getMessage.contains("cannot resolve '`a`'"))
+    assert(ex5.getMessage.contains("cannot resolve 'a'"))
   }
 
   test("map_zip_with function - map of primitive types") {
@@ -3493,7 +3501,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
     val ex4 = intercept[AnalysisException] {
       df.selectExpr("zip_with(a1, a, (acc, x) -> x)")
     }
-    assert(ex4.getMessage.contains("cannot resolve '`a`'"))
+    assert(ex4.getMessage.contains("cannot resolve 'a'"))
   }
 
   private def assertValuesDoNotChangeAfterCoalesceOrUnion(v: Column): Unit = {
@@ -3621,14 +3629,38 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
       df.select(map(map_entries($"m"), lit(1))),
       Row(Map(Seq(Row(1, "a")) -> 1)))
   }
+
+  test("SPARK-34794: lambda variable name issues in nested functions") {
+    val df1 = Seq((Seq(1, 2), Seq("a", "b"))).toDF("numbers", "letters")
+
+    checkAnswer(df1.select(flatten(transform($"numbers", (number: Column) =>
+      transform($"letters", (letter: Column) =>
+        struct(number, letter))))),
+      Seq(Row(Seq(Row(1, "a"), Row(1, "b"), Row(2, "a"), Row(2, "b"))))
+    )
+    checkAnswer(df1.select(flatten(transform($"numbers", (number: Column, i: Column) =>
+      transform($"letters", (letter: Column, j: Column) =>
+        struct(number + j, concat(letter, i)))))),
+      Seq(Row(Seq(Row(1, "a0"), Row(2, "b0"), Row(2, "a1"), Row(3, "b1"))))
+    )
+
+    val df2 = Seq((Map("a" -> 1, "b" -> 2), Map("a" -> 2, "b" -> 3))).toDF("m1", "m2")
+
+    checkAnswer(df2.select(map_zip_with($"m1", $"m2", (k1: Column, ov1: Column, ov2: Column) =>
+      map_zip_with($"m1", $"m2", (k2: Column, iv1: Column, iv2: Column) =>
+        ov1 + iv1 + ov2 + iv2))),
+      Seq(Row(Map("a" -> Map("a" -> 6, "b" -> 8), "b" -> Map("a" -> 8, "b" -> 10))))
+    )
+  }
 }
 
 object DataFrameFunctionsSuite {
-  case class CodegenFallbackExpr(child: Expression) extends Expression with CodegenFallback {
-    override def children: Seq[Expression] = Seq(child)
+  case class CodegenFallbackExpr(child: Expression) extends UnaryExpression with CodegenFallback {
     override def nullable: Boolean = child.nullable
     override def dataType: DataType = child.dataType
     override lazy val resolved = true
     override def eval(input: InternalRow): Any = child.eval(input)
+    override protected def withNewChildInternal(newChild: Expression): CodegenFallbackExpr =
+      copy(child = newChild)
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
index 6b772e53ac184..a803fa88ed313 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
@@ -17,18 +17,22 @@
 
 package org.apache.spark.sql
 
+import scala.collection.JavaConverters._
+
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.plans.{Inner, InnerLike, LeftOuter, RightOuter}
 import org.apache.spark.sql.catalyst.plans.logical.{BROADCAST, Filter, HintInfo, Join, JoinHint, LogicalPlan, Project}
 import org.apache.spark.sql.connector.catalog.CatalogManager
 import org.apache.spark.sql.execution.FileSourceScanExec
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
+import org.apache.spark.sql.execution.analysis.DetectAmbiguousSelfJoin.LogicalPlanWithDatasetId
 import org.apache.spark.sql.execution.datasources.LogicalRelation
 import org.apache.spark.sql.execution.exchange.BroadcastExchangeExec
 import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types._
 
 class DataFrameJoinSuite extends QueryTest
   with SharedSparkSession
@@ -119,6 +123,16 @@ class DataFrameJoinSuite extends QueryTest
       df2.crossJoin(df1),
       Row(2, "2", 1, "1") :: Row(2, "2", 3, "3") ::
         Row(4, "4", 1, "1") :: Row(4, "4", 3, "3") :: Nil)
+
+    checkAnswer(
+      df1.join(df2, Nil, "cross"),
+      Row(1, "1", 2, "2") :: Row(1, "1", 4, "4") ::
+        Row(3, "3", 2, "2") :: Row(3, "3", 4, "4") :: Nil)
+
+    checkAnswer(
+      df2.join(df1, Nil, "cross"),
+      Row(2, "2", 1, "1") :: Row(2, "2", 3, "3") ::
+        Row(4, "4", 1, "1") :: Row(4, "4", 3, "3") :: Nil)
   }
 
   test("broadcast join hint using broadcast function") {
@@ -261,7 +275,16 @@ class DataFrameJoinSuite extends QueryTest
     withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "false") {
       val df = spark.range(2)
       // this throws an exception before the fix
-      df.join(df, df("id") <=> df("id")).queryExecution.optimizedPlan
+      val plan = df.join(df, df("id") <=> df("id")).queryExecution.optimizedPlan
+
+      plan match {
+        // SPARK-34178: we can't match the plan before the fix due to
+        // the right side plan doesn't contains dataset id.
+        case Join(
+          LogicalPlanWithDatasetId(_, leftId),
+          LogicalPlanWithDatasetId(_, rightId), _, _, _) =>
+          assert(leftId === rightId)
+      }
     }
   }
 
@@ -338,14 +361,14 @@ class DataFrameJoinSuite extends QueryTest
 
           def checkIfHintApplied(df: DataFrame): Unit = {
             val sparkPlan = df.queryExecution.executedPlan
-            val broadcastHashJoins = sparkPlan.collect { case p: BroadcastHashJoinExec => p }
+            val broadcastHashJoins = collect(sparkPlan) { case p: BroadcastHashJoinExec => p }
             assert(broadcastHashJoins.size == 1)
             val broadcastExchanges = broadcastHashJoins.head.collect {
               case p: BroadcastExchangeExec => p
             }
             assert(broadcastExchanges.size == 1)
             val tables = broadcastExchanges.head.collect {
-              case FileSourceScanExec(_, _, _, _, _, _, Some(tableIdent)) => tableIdent
+              case FileSourceScanExec(_, _, _, _, _, _, _, Some(tableIdent), _) => tableIdent
             }
             assert(tables.size == 1)
             assert(tables.head === TableIdentifier(table1Name, Some(dbName)))
@@ -353,7 +376,7 @@ class DataFrameJoinSuite extends QueryTest
 
           def checkIfHintNotApplied(df: DataFrame): Unit = {
             val sparkPlan = df.queryExecution.executedPlan
-            val broadcastHashJoins = sparkPlan.collect { case p: BroadcastHashJoinExec => p }
+            val broadcastHashJoins = collect(sparkPlan) { case p: BroadcastHashJoinExec => p }
             assert(broadcastHashJoins.isEmpty)
           }
 
@@ -418,4 +441,62 @@ class DataFrameJoinSuite extends QueryTest
       }
     }
   }
+
+  test("SPARK-32693: Compare two dataframes with same schema except nullable property") {
+    val schema1 = StructType(
+      StructField("a", IntegerType, false) ::
+        StructField("b", IntegerType, false) ::
+        StructField("c", IntegerType, false) :: Nil)
+    val rowSeq1: List[Row] = List(Row(10, 1, 1), Row(10, 50, 2))
+    val df1 = spark.createDataFrame(rowSeq1.asJava, schema1)
+
+    val schema2 = StructType(
+      StructField("a", IntegerType) ::
+        StructField("b", IntegerType) ::
+        StructField("c", IntegerType) :: Nil)
+    val rowSeq2: List[Row] = List(Row(10, 1, 1))
+    val df2 = spark.createDataFrame(rowSeq2.asJava, schema2)
+
+    checkAnswer(df1.except(df2), Row(10, 50, 2))
+
+    val schema3 = StructType(
+      StructField("a", IntegerType, false) ::
+        StructField("b", IntegerType, false) ::
+        StructField("c", IntegerType, false) ::
+        StructField("d", schema1, false) :: Nil)
+    val rowSeq3: List[Row] = List(Row(10, 1, 1, Row(10, 1, 1)), Row(10, 50, 2, Row(10, 50, 2)))
+    val df3 = spark.createDataFrame(rowSeq3.asJava, schema3)
+
+    val schema4 = StructType(
+      StructField("a", IntegerType) ::
+        StructField("b", IntegerType) ::
+        StructField("b", IntegerType) ::
+        StructField("d", schema2) :: Nil)
+    val rowSeq4: List[Row] = List(Row(10, 1, 1, Row(10, 1, 1)))
+    val df4 = spark.createDataFrame(rowSeq4.asJava, schema4)
+
+    checkAnswer(df3.except(df4), Row(10, 50, 2, Row(10, 50, 2)))
+  }
+
+  test("SPARK-34527: Resolve common columns from USING JOIN") {
+    val joinDf = testData2.as("testData2").join(
+      testData3.as("testData3"), usingColumns = Seq("a"), joinType = "fullouter")
+    val dfQuery = joinDf.select(
+      $"a", $"testData2.a", $"testData2.b", $"testData3.a", $"testData3.b")
+    val dfQuery2 = joinDf.select(
+      $"a", testData2.col("a"), testData2.col("b"), testData3.col("a"), testData3.col("b"))
+
+    Seq(dfQuery, dfQuery2).map { query =>
+      checkAnswer(query,
+        Seq(
+          Row(1, 1, 1, 1, null),
+          Row(1, 1, 2, 1, null),
+          Row(2, 2, 1, 2, 2),
+          Row(2, 2, 2, 2, 2),
+          Row(3, 3, 1, null, null),
+          Row(3, 3, 2, null, null)
+        )
+      )
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionsSuite.scala
index 091877f7cac37..20ae995af628b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionsSuite.scala
@@ -460,4 +460,66 @@ class DataFrameNaFunctionsSuite extends QueryTest with SharedSparkSession {
       Row(0, 0L, 0.toShort, 0.toByte, Float.NaN, Double.NaN) ::
       Row(0, 0L, 0.toShort, 0.toByte, Float.NaN, Double.NaN) :: Nil)
   }
+
+  test("SPARK-34417: test fillMap() for column with a dot in the name") {
+    val na = "n/a"
+    checkAnswer(
+      Seq(("abc", 23L), ("def", 44L), (null, 0L)).toDF("ColWith.Dot", "Col")
+        .na.fill(Map("`ColWith.Dot`" -> na)),
+      Row("abc", 23) :: Row("def", 44L) :: Row(na, 0L) :: Nil)
+  }
+
+  test("SPARK-34417: test fillMap() for qualified-column with a dot in the name") {
+    val na = "n/a"
+    checkAnswer(
+      Seq(("abc", 23L), ("def", 44L), (null, 0L)).toDF("ColWith.Dot", "Col").as("testDF")
+        .na.fill(Map("testDF.`ColWith.Dot`" -> na)),
+      Row("abc", 23) :: Row("def", 44L) :: Row(na, 0L) :: Nil)
+  }
+
+  test("SPARK-34417: test fillMap() for column without a dot in the name" +
+    " and dataframe with another column having a dot in the name") {
+    val na = "n/a"
+    checkAnswer(
+      Seq(("abc", 23L), ("def", 44L), (null, 0L)).toDF("Col", "ColWith.Dot")
+        .na.fill(Map("Col" -> na)),
+      Row("abc", 23) :: Row("def", 44L) :: Row(na, 0L) :: Nil)
+  }
+
+  test("SPARK-34649: replace value of a column with dot in the name") {
+    checkAnswer(
+      Seq(("abc", 23), ("def", 44), ("n/a", 0)).toDF("Col.1", "Col.2")
+        .na.replace("`Col.1`", Map( "n/a" -> "unknown")),
+      Row("abc", 23) :: Row("def", 44L) :: Row("unknown", 0L) :: Nil)
+  }
+
+  test("SPARK-34649: replace value of a qualified-column with dot in the name") {
+    checkAnswer(
+      Seq(("abc", 23), ("def", 44), ("n/a", 0)).toDF("Col.1", "Col.2").as("testDf")
+        .na.replace("testDf.`Col.1`", Map( "n/a" -> "unknown")),
+      Row("abc", 23) :: Row("def", 44L) :: Row("unknown", 0L) :: Nil)
+  }
+
+  test("SPARK-34649: replace value of a dataframe having dot in the all column names") {
+    checkAnswer(
+      Seq(("abc", 23), ("def", 44), ("n/a", 0)).toDF("Col.1", "Col.2")
+        .na.replace("*", Map( "n/a" -> "unknown")),
+      Row("abc", 23) :: Row("def", 44L) :: Row("unknown", 0L) :: Nil)
+  }
+
+  test("SPARK-34649: replace value of a column not present in the dataframe") {
+    val df = Seq(("abc", 23), ("def", 44), ("n/a", 0)).toDF("Col.1", "Col.2")
+    val exception = intercept[AnalysisException] {
+      df.na.replace("aa", Map( "n/a" -> "unknown"))
+    }
+    assert(exception.getMessage.equals("Cannot resolve column name \"aa\" among (Col.1, Col.2)"))
+  }
+
+  test("SPARK-34649: replace value of a nested column") {
+    val df = createDFWithNestedColumns
+    val exception = intercept[UnsupportedOperationException] {
+      df.na.replace("c1.c1-1", Map("b1" ->"a1"))
+    }
+    assert(exception.getMessage.equals("Nested field c1.c1-1 is not supported."))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSelfJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSelfJoinSuite.scala
index 250ec7dc0ba5a..76f07b5b0132d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSelfJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSelfJoinSuite.scala
@@ -17,9 +17,11 @@
 
 package org.apache.spark.sql
 
+import org.apache.spark.sql.expressions.Window
 import org.apache.spark.sql.functions.{count, sum}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.test.SQLTestData.TestData
 
 class DataFrameSelfJoinSuite extends QueryTest with SharedSparkSession {
   import testImplicits._
@@ -202,4 +204,57 @@ class DataFrameSelfJoinSuite extends QueryTest with SharedSparkSession {
       assertAmbiguousSelfJoin(df1.join(df4).join(df2).select(df2("id")))
     }
   }
+
+  test("SPARK-28344: don't fail if there is no ambiguous self join") {
+    withSQLConf(
+      SQLConf.FAIL_AMBIGUOUS_SELF_JOIN_ENABLED.key -> "true") {
+      val df = Seq(1, 1, 2, 2).toDF("a")
+      val w = Window.partitionBy(df("a"))
+      checkAnswer(
+        df.select(df("a").alias("x"), sum(df("a")).over(w)),
+        Seq((1, 2), (1, 2), (2, 4), (2, 4)).map(Row.fromTuple))
+
+      val joined = df.join(spark.range(1)).select($"a")
+      checkAnswer(
+        joined.select(joined("a").alias("x"), sum(joined("a")).over(w)),
+        Seq((1, 2), (1, 2), (2, 4), (2, 4)).map(Row.fromTuple))
+    }
+  }
+
+  test("SPARK-33071/SPARK-33536: Avoid changing dataset_id of LogicalPlan in join() " +
+    "to not break DetectAmbiguousSelfJoin") {
+    val emp1 = Seq[TestData](
+      TestData(1, "sales"),
+      TestData(2, "personnel"),
+      TestData(3, "develop"),
+      TestData(4, "IT")).toDS()
+    val emp2 = Seq[TestData](
+      TestData(1, "sales"),
+      TestData(2, "personnel"),
+      TestData(3, "develop")).toDS()
+    val emp3 = emp1.join(emp2, emp1("key") === emp2("key")).select(emp1("*"))
+    assertAmbiguousSelfJoin(emp1.join(emp3, emp1.col("key") === emp3.col("key"),
+      "left_outer").select(emp1.col("*"), emp3.col("key").as("e2")))
+  }
+
+  test("df.show() should also not change dataset_id of LogicalPlan") {
+    val df = Seq[TestData](
+      TestData(1, "sales"),
+      TestData(2, "personnel"),
+      TestData(3, "develop"),
+      TestData(4, "IT")).toDF()
+    val ds_id1 = df.logicalPlan.getTagValue(Dataset.DATASET_ID_TAG)
+    df.show(0)
+    val ds_id2 = df.logicalPlan.getTagValue(Dataset.DATASET_ID_TAG)
+    assert(ds_id1 === ds_id2)
+  }
+
+  test("SPARK-34200: ambiguous column reference should consider attribute availability") {
+    withTable("t") {
+      sql("CREATE TABLE t USING json AS SELECT 1 a, 2 b")
+      val df1 = spark.table("t")
+      val df2 = df1.select("a")
+      checkAnswer(df1.join(df2, df1("b") === 2), Row(1, 2, 1))
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSetOperationsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSetOperationsSuite.scala
index bd3f48078374d..797673ae15ba8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSetOperationsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSetOperationsSuite.scala
@@ -19,10 +19,11 @@ package org.apache.spark.sql
 
 import java.sql.{Date, Timestamp}
 
+import org.apache.spark.sql.catalyst.optimizer.RemoveNoopUnion
 import org.apache.spark.sql.catalyst.plans.logical.Union
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.test.{ExamplePoint, ExamplePointUDT, SharedSparkSession}
+import org.apache.spark.sql.test.{ExamplePoint, ExamplePointUDT, SharedSparkSession, SQLTestData}
 import org.apache.spark.sql.test.SQLTestData.NullStrings
 import org.apache.spark.sql.types._
 
@@ -297,13 +298,12 @@ class DataFrameSetOperationsSuite extends QueryTest with SharedSparkSession {
 
     // When generating expected results at here, we need to follow the implementation of
     // Rand expression.
-    def expected(df: DataFrame): Seq[Row] = {
+    def expected(df: DataFrame): Seq[Row] =
       df.rdd.collectPartitions().zipWithIndex.flatMap {
         case (data, index) =>
           val rng = new org.apache.spark.util.random.XORShiftRandom(7 + index)
           data.filter(_.getInt(0) < rng.nextDouble() * 10)
-      }
-    }
+      }.toSeq
 
     val union = df1.union(df2)
     checkAnswer(
@@ -372,7 +372,7 @@ class DataFrameSetOperationsSuite extends QueryTest with SharedSparkSession {
 
     // Before optimizer, Union should be combined.
     assert(unionDF.queryExecution.analyzed.collect {
-      case j: Union if j.children.size == 5 => j }.size === 1)
+      case u: Union if u.children.size == 5 => u }.size === 1)
 
     checkAnswer(
       unionDF.agg(avg("key"), max("key"), min("key"), sum("key")),
@@ -386,6 +386,109 @@ class DataFrameSetOperationsSuite extends QueryTest with SharedSparkSession {
     checkAnswer(unionDF, unionAllDF)
   }
 
+  test("SPARK-34283: SQL-style union using Dataset, " +
+    "remove unnecessary deduplicate in multiple unions") {
+    withSQLConf(SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> RemoveNoopUnion.ruleName) {
+      val unionDF = testData.union(testData).distinct().union(testData).distinct()
+        .union(testData).distinct().union(testData).distinct()
+
+      // Before optimizer, there are three 'union.deduplicate' operations should be combined.
+      assert(unionDF.queryExecution.analyzed.collect {
+        case u: Union if u.children.size == 4 => u
+      }.size === 1)
+
+      // After optimizer, four 'union.deduplicate' operations should be combined.
+      assert(unionDF.queryExecution.optimizedPlan.collect {
+        case u: Union if u.children.size == 5 => u
+      }.size === 1)
+
+      checkAnswer(
+        unionDF.agg(avg("key"), max("key"), min("key"),
+          sum("key")), Row(50.5, 100, 1, 5050) :: Nil
+      )
+
+      // The result of SQL-style union
+      val unionSQLResult = sql(
+        """
+          | select key, value from testData
+          | union
+          | select key, value from testData
+          | union
+          | select key, value from testData
+          | union
+          | select key, value from testData
+          | union
+          | select key, value from testData
+          |""".stripMargin)
+      checkAnswer(unionDF, unionSQLResult)
+    }
+  }
+
+  test("SPARK-34283: SQL-style union using Dataset, " +
+    "keep necessary deduplicate in multiple unions") {
+    withSQLConf(SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> RemoveNoopUnion.ruleName) {
+      val df1 = Seq((1, 2, 3)).toDF("a", "b", "c")
+      var df2 = Seq((6, 2, 5)).toDF("a", "b", "c")
+      var df3 = Seq((2, 4, 3)).toDF("c", "a", "b")
+      var df4 = Seq((1, 4, 5)).toDF("b", "a", "c")
+
+      val unionDF = df1.unionByName(df2).dropDuplicates(Seq("a"))
+        .unionByName(df3).dropDuplicates("c").unionByName(df4)
+        .dropDuplicates("b")
+
+      // In this case, there is no 'union.deduplicate' operation will be combined.
+      assert(unionDF.queryExecution.analyzed.collect {
+        case u: Union if u.children.size == 2 => u
+      }.size === 3)
+
+      assert(unionDF.queryExecution.optimizedPlan.collect {
+        case u: Union if u.children.size == 2 => u
+      }.size === 3)
+
+      checkAnswer(
+        unionDF,
+        Row(4, 3, 2) :: Row(4, 1, 5) :: Row(1, 2, 3) :: Nil
+      )
+
+      val unionDF1 = df1.unionByName(df2).dropDuplicates(Seq("B", "A", "c"))
+        .unionByName(df3).dropDuplicates().unionByName(df4)
+        .dropDuplicates("A")
+
+      // In this case, there are two 'union.deduplicate' operations will be combined.
+      assert(unionDF1.queryExecution.analyzed.collect {
+        case u: Union if u.children.size == 2 => u
+      }.size === 1)
+      assert(unionDF1.queryExecution.analyzed.collect {
+        case u: Union if u.children.size == 3 => u
+      }.size === 1)
+
+      assert(unionDF1.queryExecution.optimizedPlan.collect {
+        case u: Union if u.children.size == 2 => u
+      }.size === 1)
+      assert(unionDF1.queryExecution.optimizedPlan.collect {
+        case u: Union if u.children.size == 3 => u
+      }.size === 1)
+
+      checkAnswer(
+        unionDF1,
+        Row(4, 3, 2) :: Row(6, 2, 5) :: Row(1, 2, 3) :: Nil
+      )
+
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+        df2 = Seq((6, 2, 5)).toDF("a", "B", "C")
+        df3 = Seq((2, 1, 3)).toDF("b", "a", "c")
+        df4 = Seq((1, 4, 5)).toDF("b", "a", "c")
+
+        val unionDF2 = df1.unionByName(df2, true).distinct()
+          .unionByName(df3, true).dropDuplicates(Seq("a")).unionByName(df4, true).distinct()
+
+        checkAnswer(unionDF2,
+          Row(4, 1, 5, null, null) :: Row(1, 2, 3, null, null) :: Row(6, null, null, 2, 5) :: Nil)
+        assert(unionDF2.schema.fieldNames === Array("a", "b", "c", "B", "C"))
+      }
+    }
+  }
+
   test("union should union DataFrames with UDTs (SPARK-13410)") {
     val rowRDD1 = sparkContext.parallelize(Seq(Row(1, new ExamplePoint(1.0, 2.0))))
     val schema1 = StructType(Array(StructField("label", IntegerType, false),
@@ -506,4 +609,280 @@ class DataFrameSetOperationsSuite extends QueryTest with SharedSparkSession {
     check(lit(2).cast("int"), $"c" === 2, Seq(Row(1, 1, 2, 0), Row(1, 1, 2, 1)))
     check(lit(2).cast("int"), $"c" =!= 2, Seq())
   }
+
+  test("SPARK-29358: Make unionByName optionally fill missing columns with nulls") {
+    var df1 = Seq(1, 2, 3).toDF("a")
+    var df2 = Seq(3, 1, 2).toDF("b")
+    val df3 = Seq(2, 3, 1).toDF("c")
+    val unionDf = df1.unionByName(df2.unionByName(df3, true), true)
+    checkAnswer(unionDf,
+      Row(1, null, null) :: Row(2, null, null) :: Row(3, null, null) :: // df1
+        Row(null, 3, null) :: Row(null, 1, null) :: Row(null, 2, null) :: // df2
+        Row(null, null, 2) :: Row(null, null, 3) :: Row(null, null, 1) :: Nil // df3
+    )
+
+    df1 = Seq((1, 2)).toDF("a", "c")
+    df2 = Seq((3, 4, 5)).toDF("a", "b", "c")
+    checkAnswer(df1.unionByName(df2, true),
+      Row(1, 2, null) :: Row(3, 5, 4) :: Nil)
+    checkAnswer(df2.unionByName(df1, true),
+      Row(3, 4, 5) :: Row(1, null, 2) :: Nil)
+
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+      df2 = Seq((3, 4, 5)).toDF("a", "B", "C")
+      val union1 = df1.unionByName(df2, true)
+      val union2 = df2.unionByName(df1, true)
+
+      checkAnswer(union1, Row(1, 2, null, null) :: Row(3, null, 4, 5) :: Nil)
+      checkAnswer(union2, Row(3, 4, 5, null) :: Row(1, null, null, 2) :: Nil)
+
+      assert(union1.schema.fieldNames === Array("a", "c", "B", "C"))
+      assert(union2.schema.fieldNames === Array("a", "B", "C", "c"))
+    }
+  }
+
+  test("SPARK-32376: Make unionByName null-filling behavior work with struct columns - simple") {
+    val df1 = Seq(((1, 2, 3), 0), ((2, 3, 4), 1), ((3, 4, 5), 2)).toDF("a", "idx")
+    val df2 = Seq(((3, 4), 0), ((1, 2), 1), ((2, 3), 2)).toDF("a", "idx")
+    val df3 = Seq(((100, 101, 102, 103), 0), ((110, 111, 112, 113), 1), ((120, 121, 122, 123), 2))
+      .toDF("a", "idx")
+
+    var unionDf = df1.unionByName(df2, true)
+
+    checkAnswer(unionDf,
+      Row(Row(1, 2, 3), 0) :: Row(Row(2, 3, 4), 1) :: Row(Row(3, 4, 5), 2) ::
+        Row(Row(3, 4, null), 0) :: Row(Row(1, 2, null), 1) :: Row(Row(2, 3, null), 2) :: Nil
+    )
+
+    assert(unionDf.schema.toDDL == "`a` STRUCT<`_1`: INT, `_2`: INT, `_3`: INT>,`idx` INT")
+
+    unionDf = df1.unionByName(df2, true).unionByName(df3, true)
+
+    checkAnswer(unionDf,
+      Row(Row(1, 2, 3, null), 0) ::
+        Row(Row(2, 3, 4, null), 1) ::
+        Row(Row(3, 4, 5, null), 2) :: // df1
+        Row(Row(3, 4, null, null), 0) ::
+        Row(Row(1, 2, null, null), 1) ::
+        Row(Row(2, 3, null, null), 2) :: // df2
+        Row(Row(100, 101, 102, 103), 0) ::
+        Row(Row(110, 111, 112, 113), 1) ::
+        Row(Row(120, 121, 122, 123), 2) :: Nil // df3
+    )
+    assert(unionDf.schema.toDDL ==
+      "`a` STRUCT<`_1`: INT, `_2`: INT, `_3`: INT, `_4`: INT>,`idx` INT")
+  }
+
+  test("SPARK-32376: Make unionByName null-filling behavior work with struct columns - nested") {
+    val df1 = Seq((0, UnionClass1a(0, 1L, UnionClass2(1, "2")))).toDF("id", "a")
+    val df2 = Seq((1, UnionClass1b(1, 2L, UnionClass3(2, 3L)))).toDF("id", "a")
+
+    val expectedSchema = "`id` INT,`a` STRUCT<`a`: INT, `b`: BIGINT, " +
+      "`nested`: STRUCT<`a`: INT, `b`: BIGINT, `c`: STRING>>"
+
+    var unionDf = df1.unionByName(df2, true)
+    checkAnswer(unionDf,
+      Row(0, Row(0, 1, Row(1, null, "2"))) ::
+        Row(1, Row(1, 2, Row(2, 3L, null))) :: Nil)
+    assert(unionDf.schema.toDDL == expectedSchema)
+
+    unionDf = df2.unionByName(df1, true)
+    checkAnswer(unionDf,
+      Row(1, Row(1, 2, Row(2, 3L, null))) ::
+        Row(0, Row(0, 1, Row(1, null, "2"))) :: Nil)
+    assert(unionDf.schema.toDDL == expectedSchema)
+
+    val df3 = Seq((2, UnionClass1b(2, 3L, null))).toDF("id", "a")
+    unionDf = df1.unionByName(df3, true)
+    checkAnswer(unionDf,
+      Row(0, Row(0, 1, Row(1, null, "2"))) ::
+        Row(2, Row(2, 3, null)) :: Nil)
+    assert(unionDf.schema.toDDL == expectedSchema)
+  }
+
+  test("SPARK-32376: Make unionByName null-filling behavior work with struct columns" +
+      " - case-sensitive cases") {
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+      val df1 = Seq((0, UnionClass1a(0, 1L, UnionClass2(1, "2")))).toDF("id", "a")
+      val df2 = Seq((1, UnionClass1c(1, 2L, UnionClass4(2, 3L)))).toDF("id", "a")
+
+      var unionDf = df1.unionByName(df2, true)
+      checkAnswer(unionDf,
+        Row(0, Row(0, 1, Row(null, 1, null, "2"))) ::
+          Row(1, Row(1, 2, Row(2, null, 3L, null))) :: Nil)
+      assert(unionDf.schema.toDDL ==
+        "`id` INT,`a` STRUCT<`a`: INT, `b`: BIGINT, " +
+          "`nested`: STRUCT<`A`: INT, `a`: INT, `b`: BIGINT, `c`: STRING>>")
+
+      unionDf = df2.unionByName(df1, true)
+      checkAnswer(unionDf,
+        Row(1, Row(1, 2, Row(2, null, 3L, null))) ::
+          Row(0, Row(0, 1, Row(null, 1, null, "2"))) :: Nil)
+      assert(unionDf.schema.toDDL ==
+        "`id` INT,`a` STRUCT<`a`: INT, `b`: BIGINT, " +
+          "`nested`: STRUCT<`A`: INT, `a`: INT, `b`: BIGINT, `c`: STRING>>")
+
+      val df3 = Seq((2, UnionClass1b(2, 3L, UnionClass3(4, 5L)))).toDF("id", "a")
+      unionDf = df2.unionByName(df3, true)
+      checkAnswer(unionDf,
+        Row(1, Row(1, 2, Row(2, null, 3L))) ::
+          Row(2, Row(2, 3, Row(null, 4, 5L))) :: Nil)
+      assert(unionDf.schema.toDDL ==
+        "`id` INT,`a` STRUCT<`a`: INT, `b`: BIGINT, " +
+          "`nested`: STRUCT<`A`: INT, `a`: INT, `b`: BIGINT>>")
+    }
+  }
+
+  test("SPARK-32376: Make unionByName null-filling behavior work with struct columns - edge case") {
+    val nestedStructType1 = StructType(Seq(
+      StructField("b", StringType)))
+    val nestedStructValues1 = Row("b")
+
+    val nestedStructType2 = StructType(Seq(
+      StructField("b", StringType),
+      StructField("a", StringType)))
+    val nestedStructValues2 = Row("b", "a")
+
+    val df1: DataFrame = spark.createDataFrame(
+      sparkContext.parallelize(Row(nestedStructValues1) :: Nil),
+      StructType(Seq(StructField("topLevelCol", nestedStructType1))))
+
+    val df2: DataFrame = spark.createDataFrame(
+      sparkContext.parallelize(Row(nestedStructValues2) :: Nil),
+      StructType(Seq(StructField("topLevelCol", nestedStructType2))))
+
+    val union = df1.unionByName(df2, allowMissingColumns = true)
+    checkAnswer(union, Row(Row(null, "b")) :: Row(Row("a", "b")) :: Nil)
+    assert(union.schema.toDDL == "`topLevelCol` STRUCT<`a`: STRING, `b`: STRING>")
+  }
+
+  test("SPARK-32376: Make unionByName null-filling behavior work with struct columns - deep expr") {
+    def nestedDf(depth: Int, numColsAtEachDepth: Int): DataFrame = {
+      val initialNestedStructType = StructType(
+        (0 to numColsAtEachDepth).map(i =>
+          StructField(s"nested${depth}Col$i", IntegerType, nullable = false))
+      )
+      val initialNestedValues = Row(0 to numColsAtEachDepth: _*)
+
+      var depthCounter = depth - 1
+      var structType = initialNestedStructType
+      var struct = initialNestedValues
+      while (depthCounter != 0) {
+        struct = Row((struct +: (1 to numColsAtEachDepth)): _*)
+        structType = StructType(
+          StructField(s"nested${depthCounter}Col0", structType, nullable = false) +:
+            (1 to numColsAtEachDepth).map(i =>
+              StructField(s"nested${depthCounter}Col$i", IntegerType, nullable = false))
+        )
+        depthCounter -= 1
+      }
+
+      val df: DataFrame = spark.createDataFrame(
+        sparkContext.parallelize(Row(struct) :: Nil),
+        StructType(Seq(StructField("nested0Col0", structType))))
+
+      df
+    }
+
+    val df1 = nestedDf(depth = 10, numColsAtEachDepth = 1)
+    val df2 = nestedDf(depth = 10, numColsAtEachDepth = 20)
+    val union = df1.unionByName(df2, allowMissingColumns = true)
+    // scalastyle:off
+    val row1 = Row(Row(Row(Row(Row(Row(Row(Row(Row(Row(
+      Row(0, 1, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null),
+      1, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null),
+      1, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null),
+      1, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null),
+      1, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null),
+      1, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null),
+      1, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null),
+      1, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null),
+      1, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null),
+      1, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null))
+    val row2 = Row(Row(Row(Row(Row(Row(Row(Row(Row(Row(
+      Row(0, 1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 2, 20, 3, 4, 5, 6, 7, 8, 9),
+      1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 2, 20, 3, 4, 5, 6, 7, 8, 9),
+      1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 2, 20, 3, 4, 5, 6, 7, 8, 9),
+      1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 2, 20, 3, 4, 5, 6, 7, 8, 9),
+      1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 2, 20, 3, 4, 5, 6, 7, 8, 9),
+      1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 2, 20, 3, 4, 5, 6, 7, 8, 9),
+      1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 2, 20, 3, 4, 5, 6, 7, 8, 9),
+      1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 2, 20, 3, 4, 5, 6, 7, 8, 9),
+      1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 2, 20, 3, 4, 5, 6, 7, 8, 9),
+      1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 2, 20, 3, 4, 5, 6, 7, 8, 9))
+    // scalastyle:on
+    checkAnswer(union, row1 :: row2 :: Nil)
+  }
+
+  test("SPARK-34474: Remove unnecessary Union under Distinct") {
+    Seq(RemoveNoopUnion.ruleName, "").map { ruleName =>
+      withSQLConf(SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> ruleName) {
+        val distinctUnionDF1 = testData.union(testData).distinct()
+        checkAnswer(distinctUnionDF1, testData.distinct())
+
+
+        val distinctUnionDF2 = testData.union(testData).dropDuplicates(Seq("key"))
+        checkAnswer(distinctUnionDF2, testData.dropDuplicates(Seq("key")))
+
+        val distinctUnionDF3 = sql(
+          """
+            |select key, value from testData
+            |union
+            |select key, value from testData
+            |""".stripMargin)
+        checkAnswer(distinctUnionDF3, testData.distinct())
+
+        val distinctUnionDF4 = sql(
+          """
+            |select distinct key, expr
+            |from
+            |(
+            |  select key, key + 1 as expr
+            |  from testData
+            |  union all
+            |  select key, key + 2 as expr
+            |  from testData
+            |)
+            |""".stripMargin)
+        val expected = sql(
+          """
+            |select key, expr
+            |from
+            |(
+            |  select key, key + 1 as expr
+            |  from testData
+            |  union all
+            |  select key, key + 2 as expr
+            |  from testData
+            |) group by key, expr
+            |""".stripMargin)
+        checkAnswer(distinctUnionDF4, expected)
+      }
+    }
+  }
+
+  test("SPARK-34548: Remove unnecessary children from Union") {
+    Seq(RemoveNoopUnion.ruleName, "").map { ruleName =>
+      withSQLConf(SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> ruleName) {
+        val testDataCopy = spark.sparkContext.parallelize(
+          (1 to 100).map(i => SQLTestData.TestData(i, i.toString))).toDF()
+
+        val distinctUnionDF1 = testData.union(testData).union(testDataCopy).distinct()
+        val expected = testData.union(testDataCopy).distinct()
+        checkAnswer(distinctUnionDF1, expected)
+
+        val distinctUnionDF2 = testData.union(testData).union(testDataCopy)
+          .dropDuplicates(Seq("key"))
+        checkAnswer(distinctUnionDF2, expected)
+      }
+    }
+  }
 }
+
+case class UnionClass1a(a: Int, b: Long, nested: UnionClass2)
+case class UnionClass1b(a: Int, b: Long, nested: UnionClass3)
+case class UnionClass1c(a: Int, b: Long, nested: UnionClass4)
+
+case class UnionClass2(a: Int, c: String)
+case class UnionClass3(a: Int, b: Long)
+case class UnionClass4(A: Int, b: Long)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala
index 19601726fa393..cdd256877157c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql
 
 import java.util.Random
 
-import org.scalatest.Matchers._
+import org.scalatest.matchers.must.Matchers._
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.execution.stat.StatFunctions
@@ -412,8 +412,8 @@ class DataFrameStatSuite extends QueryTest with SharedSparkSession {
     // Original bug was a NullPointerException exception caused by calling collect(), test for this
     val resultRow = result.collect()(0)
 
-    assert(resultRow.get(0).asInstanceOf[Seq[String]].toSet == Set("1", "2", "3"))
-    assert(resultRow.get(1).asInstanceOf[Seq[String]].toSet == Set("a", "b", null))
+    assert(resultRow.get(0).asInstanceOf[scala.collection.Seq[String]].toSet == Set("1", "2", "3"))
+    assert(resultRow.get(1).asInstanceOf[scala.collection.Seq[String]].toSet == Set("a", "b", null))
   }
 
   test("sampleBy") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index 4e91a7c7bb0f4..d7d85d43544e0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -18,15 +18,16 @@
 package org.apache.spark.sql
 
 import java.io.{ByteArrayOutputStream, File}
+import java.lang.{Long => JLong}
 import java.nio.charset.StandardCharsets
 import java.sql.{Date, Timestamp}
-import java.util.UUID
+import java.util.{Locale, UUID}
 import java.util.concurrent.atomic.AtomicLong
 
 import scala.reflect.runtime.universe.TypeTag
 import scala.util.Random
 
-import org.scalatest.Matchers._
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.SparkException
 import org.apache.spark.scheduler.{SparkListener, SparkListenerJobEnd}
@@ -36,10 +37,12 @@ import org.apache.spark.sql.catalyst.expressions.Uuid
 import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation
 import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, OneRowRelation}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.connector.FakeV2Provider
 import org.apache.spark.sql.execution.{FilterExec, QueryExecution, WholeStageCodegenExec}
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
 import org.apache.spark.sql.execution.aggregate.HashAggregateExec
 import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ReusedExchangeExec, ShuffleExchangeExec}
+import org.apache.spark.sql.expressions.{Aggregator, Window}
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.{ExamplePoint, ExamplePointUDT, SharedSparkSession}
@@ -132,7 +135,7 @@ class DataFrameSuite extends QueryTest
       df2
         .select('_1 as 'letter, 'number)
         .groupBy('letter)
-        .agg(countDistinct('number)),
+        .agg(count_distinct('number)),
       Row("a", 3) :: Row("b", 2) :: Row("c", 1) :: Nil
     )
   }
@@ -192,6 +195,20 @@ class DataFrameSuite extends QueryTest
       structDf.select(xxhash64($"a", $"record.*")))
   }
 
+  private def assertDecimalSumOverflow(
+      df: DataFrame, ansiEnabled: Boolean, expectedAnswer: Row): Unit = {
+    if (!ansiEnabled) {
+      checkAnswer(df, expectedAnswer)
+    } else {
+      val e = intercept[SparkException] {
+        df.collect()
+      }
+      assert(e.getCause.isInstanceOf[ArithmeticException])
+      assert(e.getCause.getMessage.contains("cannot be represented as Decimal") ||
+        e.getCause.getMessage.contains("Overflow in sum of decimals"))
+    }
+  }
+
   test("SPARK-28224: Aggregate sum big decimal overflow") {
     val largeDecimals = spark.sparkContext.parallelize(
       DecimalData(BigDecimal("1"* 20 + ".123"), BigDecimal("1"* 20 + ".123")) ::
@@ -200,14 +217,90 @@ class DataFrameSuite extends QueryTest
     Seq(true, false).foreach { ansiEnabled =>
       withSQLConf((SQLConf.ANSI_ENABLED.key, ansiEnabled.toString)) {
         val structDf = largeDecimals.select("a").agg(sum("a"))
-        if (!ansiEnabled) {
-          checkAnswer(structDf, Row(null))
-        } else {
-          val e = intercept[SparkException] {
-            structDf.collect
+        assertDecimalSumOverflow(structDf, ansiEnabled, Row(null))
+      }
+    }
+  }
+
+  test("SPARK-28067: sum of null decimal values") {
+    Seq("true", "false").foreach { wholeStageEnabled =>
+      withSQLConf((SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, wholeStageEnabled)) {
+        Seq("true", "false").foreach { ansiEnabled =>
+          withSQLConf((SQLConf.ANSI_ENABLED.key, ansiEnabled)) {
+            val df = spark.range(1, 4, 1).select(expr(s"cast(null as decimal(38,18)) as d"))
+            checkAnswer(df.agg(sum($"d")), Row(null))
+          }
+        }
+      }
+    }
+  }
+
+  test("SPARK-28067: Aggregate sum should not return wrong results for decimal overflow") {
+    Seq("true", "false").foreach { wholeStageEnabled =>
+      withSQLConf((SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, wholeStageEnabled)) {
+        Seq(true, false).foreach { ansiEnabled =>
+          withSQLConf((SQLConf.ANSI_ENABLED.key, ansiEnabled.toString)) {
+            val df0 = Seq(
+              (BigDecimal("10000000000000000000"), 1),
+              (BigDecimal("10000000000000000000"), 1),
+              (BigDecimal("10000000000000000000"), 2)).toDF("decNum", "intNum")
+            val df1 = Seq(
+              (BigDecimal("10000000000000000000"), 2),
+              (BigDecimal("10000000000000000000"), 2),
+              (BigDecimal("10000000000000000000"), 2),
+              (BigDecimal("10000000000000000000"), 2),
+              (BigDecimal("10000000000000000000"), 2),
+              (BigDecimal("10000000000000000000"), 2),
+              (BigDecimal("10000000000000000000"), 2),
+              (BigDecimal("10000000000000000000"), 2),
+              (BigDecimal("10000000000000000000"), 2)).toDF("decNum", "intNum")
+            val df = df0.union(df1)
+            val df2 = df.withColumnRenamed("decNum", "decNum2").
+              join(df, "intNum").agg(sum("decNum"))
+
+            val expectedAnswer = Row(null)
+            assertDecimalSumOverflow(df2, ansiEnabled, expectedAnswer)
+
+            val decStr = "1" + "0" * 19
+            val d1 = spark.range(0, 12, 1, 1)
+            val d2 = d1.select(expr(s"cast('$decStr' as decimal (38, 18)) as d")).agg(sum($"d"))
+            assertDecimalSumOverflow(d2, ansiEnabled, expectedAnswer)
+
+            val d3 = spark.range(0, 1, 1, 1).union(spark.range(0, 11, 1, 1))
+            val d4 = d3.select(expr(s"cast('$decStr' as decimal (38, 18)) as d")).agg(sum($"d"))
+            assertDecimalSumOverflow(d4, ansiEnabled, expectedAnswer)
+
+            val d5 = d3.select(expr(s"cast('$decStr' as decimal (38, 18)) as d"),
+              lit(1).as("key")).groupBy("key").agg(sum($"d").alias("sumd")).select($"sumd")
+            assertDecimalSumOverflow(d5, ansiEnabled, expectedAnswer)
+
+            val nullsDf = spark.range(1, 4, 1).select(expr(s"cast(null as decimal(38,18)) as d"))
+
+            val largeDecimals = Seq(BigDecimal("1"* 20 + ".123"), BigDecimal("9"* 20 + ".123")).
+              toDF("d")
+            assertDecimalSumOverflow(
+              nullsDf.union(largeDecimals).agg(sum($"d")), ansiEnabled, expectedAnswer)
+
+            val df3 = Seq(
+              (BigDecimal("10000000000000000000"), 1),
+              (BigDecimal("50000000000000000000"), 1),
+              (BigDecimal("10000000000000000000"), 2)).toDF("decNum", "intNum")
+
+            val df4 = Seq(
+              (BigDecimal("10000000000000000000"), 1),
+              (BigDecimal("10000000000000000000"), 1),
+              (BigDecimal("10000000000000000000"), 2)).toDF("decNum", "intNum")
+
+            val df5 = Seq(
+              (BigDecimal("10000000000000000000"), 1),
+              (BigDecimal("10000000000000000000"), 1),
+              (BigDecimal("20000000000000000000"), 2)).toDF("decNum", "intNum")
+
+            val df6 = df3.union(df4).union(df5)
+            val df7 = df6.groupBy("intNum").agg(sum("decNum"), countDistinct("decNum")).
+              filter("intNum == 1")
+            assertDecimalSumOverflow(df7, ansiEnabled, Row(1, null, 2))
           }
-          assert(e.getCause.getClass.equals(classOf[ArithmeticException]))
-          assert(e.getCause.getMessage.contains("cannot be represented as Decimal"))
         }
       }
     }
@@ -422,7 +515,7 @@ class DataFrameSuite extends QueryTest
         Row(5, false)))
 
     checkAnswer(
-      testData2.select(sumDistinct($"a")),
+      testData2.select(sum_distinct($"a")),
       Row(6))
   }
 
@@ -516,7 +609,7 @@ class DataFrameSuite extends QueryTest
     val df = Seq(("id1", 1), ("id2", 4), ("id3", 5)).toDF("id", "value")
     df.sparkSession.udf.register("simpleUDF", (v: Int) => v * v)
     checkAnswer(
-      df.select($"id", callUDF("simpleUDF", $"value")),
+      df.select($"id", callUDF("simpleUDF", $"value")), // test deprecated one
       Row("id1", 1) :: Row("id2", 16) :: Row("id3", 25) :: Nil)
   }
 
@@ -714,7 +807,7 @@ class DataFrameSuite extends QueryTest
     assert(df2.drop("`a.b`").columns.size == 2)
   }
 
-  test("drop(name: String) search and drop all top level columns that matchs the name") {
+  test("drop(name: String) search and drop all top level columns that matches the name") {
     val df1 = Seq((1, 2)).toDF("a", "b")
     val df2 = Seq((3, 4)).toDF("a", "b")
     checkAnswer(df1.crossJoin(df2), Row(1, 2, 3, 4))
@@ -739,6 +832,15 @@ class DataFrameSuite extends QueryTest
     ("David", 60, 192),
     ("Amy", 24, 180)).toDF("name", "age", "height")
 
+  private lazy val person3: DataFrame = Seq(
+    ("Luis", 1, 99),
+    ("Luis", 16, 99),
+    ("Luis", 16, 176),
+    ("Fernando", 32, 99),
+    ("Fernando", 32, 164),
+    ("David", 60, 99),
+    ("Amy", 24, 99)).toDF("name", "age", "height")
+
   test("describe") {
     val describeResult = Seq(
       Row("count", "4", "4", "4"),
@@ -830,6 +932,25 @@ class DataFrameSuite extends QueryTest
     checkAnswer(emptyDescription, emptySummaryResult)
   }
 
+  test("SPARK-34165: Add count_distinct to summary") {
+    val summaryDF = person3.summary("count", "count_distinct")
+
+    val summaryResult = Seq(
+      Row("count", "7", "7", "7"),
+      Row("count_distinct", "4", "5", "3"))
+
+    def getSchemaAsSeq(df: DataFrame): Seq[String] = df.schema.map(_.name)
+    assert(getSchemaAsSeq(summaryDF) === Seq("summary", "name", "age", "height"))
+    checkAnswer(summaryDF, summaryResult)
+
+    val approxSummaryDF = person3.summary("count", "approx_count_distinct")
+    val approxSummaryResult = Seq(
+      Row("count", "7", "7", "7"),
+      Row("approx_count_distinct", "4", "5", "3"))
+    assert(getSchemaAsSeq(summaryDF) === Seq("summary", "name", "age", "height"))
+    checkAnswer(approxSummaryDF, approxSummaryResult)
+  }
+
   test("summary advanced") {
     val stats = Array("count", "50.01%", "max", "mean", "min", "25%")
     val orderMatters = person2.summary(stats: _*)
@@ -1144,6 +1265,99 @@ class DataFrameSuite extends QueryTest
     assert(df.showString(10, vertical = true) === expectedAnswer)
   }
 
+  test("SPARK-33690: showString: escape meta-characters") {
+    val df1 = spark.sql("SELECT 'aaa\nbbb\tccc\rddd\feee\bfff\u000Bggg\u0007hhh'")
+    assert(df1.showString(1, truncate = 0) ===
+      """+--------------------------------------+
+        ||aaa\nbbb\tccc\rddd\feee\bfff\vggg\ahhh|
+        |+--------------------------------------+
+        ||aaa\nbbb\tccc\rddd\feee\bfff\vggg\ahhh|
+        |+--------------------------------------+
+        |""".stripMargin)
+
+    val df2 = spark.sql("SELECT array('aaa\nbbb\tccc\rddd\feee\bfff\u000Bggg\u0007hhh')")
+    assert(df2.showString(1, truncate = 0) ===
+      """+---------------------------------------------+
+        ||array(aaa\nbbb\tccc\rddd\feee\bfff\vggg\ahhh)|
+        |+---------------------------------------------+
+        ||[aaa\nbbb\tccc\rddd\feee\bfff\vggg\ahhh]     |
+        |+---------------------------------------------+
+        |""".stripMargin)
+
+    val df3 =
+      spark.sql("SELECT map('aaa\nbbb\tccc', 'aaa\nbbb\tccc\rddd\feee\bfff\u000Bggg\u0007hhh')")
+    assert(df3.showString(1, truncate = 0) ===
+      """+----------------------------------------------------------+
+        ||map(aaa\nbbb\tccc, aaa\nbbb\tccc\rddd\feee\bfff\vggg\ahhh)|
+        |+----------------------------------------------------------+
+        ||{aaa\nbbb\tccc -> aaa\nbbb\tccc\rddd\feee\bfff\vggg\ahhh} |
+        |+----------------------------------------------------------+
+        |""".stripMargin)
+
+    val df4 =
+      spark.sql("SELECT named_struct('v', 'aaa\nbbb\tccc\rddd\feee\bfff\u000Bggg\u0007hhh')")
+    assert(df4.showString(1, truncate = 0) ===
+      """+-------------------------------------------------------+
+        ||named_struct(v, aaa\nbbb\tccc\rddd\feee\bfff\vggg\ahhh)|
+        |+-------------------------------------------------------+
+        ||{aaa\nbbb\tccc\rddd\feee\bfff\vggg\ahhh}               |
+        |+-------------------------------------------------------+
+        |""".stripMargin)
+  }
+
+  test("SPARK-34308: printSchema: escape meta-characters") {
+    val captured = new ByteArrayOutputStream()
+
+    val df1 = spark.sql("SELECT 'aaa\nbbb\tccc\rddd\feee\bfff\u000Bggg\u0007hhh'")
+    Console.withOut(captured) {
+      df1.printSchema()
+    }
+    assert(captured.toString ===
+      """root
+        | |-- aaa\nbbb\tccc\rddd\feee\bfff\vggg\ahhh: string (nullable = false)
+        |
+        |""".stripMargin)
+    captured.reset()
+
+    val df2 = spark.sql("SELECT array('aaa\nbbb\tccc\rddd\feee\bfff\u000Bggg\u0007hhh')")
+    Console.withOut(captured) {
+      df2.printSchema()
+    }
+    assert(captured.toString ===
+      """root
+        | |-- array(aaa\nbbb\tccc\rddd\feee\bfff\vggg\ahhh): array (nullable = false)
+        | |    |-- element: string (containsNull = false)
+        |
+        |""".stripMargin)
+    captured.reset()
+
+    val df3 =
+      spark.sql("SELECT map('aaa\nbbb\tccc', 'aaa\nbbb\tccc\rddd\feee\bfff\u000Bggg\u0007hhh')")
+    Console.withOut(captured) {
+      df3.printSchema()
+    }
+    assert(captured.toString ===
+      """root
+        | |-- map(aaa\nbbb\tccc, aaa\nbbb\tccc\rddd\feee\bfff\vggg\ahhh): map (nullable = false)
+        | |    |-- key: string
+        | |    |-- value: string (valueContainsNull = false)
+        |
+        |""".stripMargin)
+    captured.reset()
+
+    val df4 =
+      spark.sql("SELECT named_struct('v', 'aaa\nbbb\tccc\rddd\feee\bfff\u000Bggg\u0007hhh')")
+    Console.withOut(captured) {
+      df4.printSchema()
+    }
+    assert(captured.toString ===
+      """root
+        | |-- named_struct(v, aaa\nbbb\tccc\rddd\feee\bfff\vggg\ahhh): struct (nullable = false)
+        | |    |-- v: string (nullable = false)
+        |
+        |""".stripMargin)
+  }
+
   test("SPARK-7319 showString") {
     val expectedAnswer = """+---+-----+
                            ||key|value|
@@ -1177,7 +1391,7 @@ class DataFrameSuite extends QueryTest
       s"""+----------------+
          ||               a|
          |+----------------+
-         ||[1 -> a, 2 -> b]|
+         ||{1 -> a, 2 -> b}|
          |+----------------+
          |""".stripMargin)
     val df3 = Seq(((1, "a"), 0), ((2, "b"), 0)).toDF("a", "b")
@@ -1185,8 +1399,8 @@ class DataFrameSuite extends QueryTest
       s"""+------+---+
          ||     a|  b|
          |+------+---+
-         ||[1, a]|  0|
-         ||[2, b]|  0|
+         ||{1, a}|  0|
+         ||{2, b}|  0|
          |+------+---+
          |""".stripMargin)
   }
@@ -2339,7 +2553,7 @@ class DataFrameSuite extends QueryTest
     val err = intercept[AnalysisException] {
       df.groupBy($"d", $"b").as[GroupByKey, Row]
     }
-    assert(err.getMessage.contains("cannot resolve '`d`'"))
+    assert(err.getMessage.contains("cannot resolve 'd'"))
   }
 
   test("emptyDataFrame should be foldable") {
@@ -2361,6 +2575,14 @@ class DataFrameSuite extends QueryTest
     assert(e.getMessage.contains("Table or view not found:"))
   }
 
+  test("SPARK-32680: Don't analyze CTAS with unresolved query") {
+    val v2Source = classOf[FakeV2Provider].getName
+    val e = intercept[AnalysisException] {
+      sql(s"CREATE TABLE t USING $v2Source AS SELECT * from nonexist")
+    }
+    assert(e.getMessage.contains("Table or view not found:"))
+  }
+
   test("CalendarInterval reflection support") {
     val df = Seq((1, new CalendarInterval(1, 2, 3))).toDF("a", "b")
     checkAnswer(df.selectExpr("b"), Row(new CalendarInterval(1, 2, 3)))
@@ -2439,6 +2661,208 @@ class DataFrameSuite extends QueryTest
     val nestedDecArray = Array(decSpark)
     checkAnswer(Seq(nestedDecArray).toDF(), Row(Array(wrapRefArray(decJava))))
   }
+
+  test("SPARK-31750: eliminate UpCast if child's dataType is DecimalType") {
+    withTempPath { f =>
+      sql("select cast(1 as decimal(38, 0)) as d")
+        .write.mode("overwrite")
+        .parquet(f.getAbsolutePath)
+
+      val df = spark.read.parquet(f.getAbsolutePath).as[BigDecimal]
+      assert(df.schema === new StructType().add(StructField("d", DecimalType(38, 0))))
+    }
+  }
+
+  test("SPARK-32640: ln(NaN) should return NaN") {
+    val df = Seq(Double.NaN).toDF("d")
+    checkAnswer(df.selectExpr("ln(d)"), Row(Double.NaN))
+  }
+
+  test("SPARK-32761: aggregating multiple distinct CONSTANT columns") {
+     checkAnswer(sql("select count(distinct 2), count(distinct 2,3)"), Row(1, 1))
+  }
+
+  test("SPARK-32764: -0.0 and 0.0 should be equal") {
+    val df = Seq(0.0 -> -0.0).toDF("pos", "neg")
+    checkAnswer(df.select($"pos" > $"neg"), Row(false))
+  }
+
+  test("SPARK-32635: Replace references with foldables coming only from the node's children") {
+    val a = Seq("1").toDF("col1").withColumn("col2", lit("1"))
+    val b = Seq("2").toDF("col1").withColumn("col2", lit("2"))
+    val aub = a.union(b)
+    val c = aub.filter($"col1" === "2").cache()
+    val d = Seq("2").toDF("col4")
+    val r = d.join(aub, $"col2" === $"col4").select("col4")
+    val l = c.select("col2")
+    val df = l.join(r, $"col2" === $"col4", "LeftOuter")
+    checkAnswer(df, Row("2", "2"))
+  }
+
+  test("SPARK-33939: Make Column.named use UnresolvedAlias to assign name") {
+    val df = spark.range(1).selectExpr("id as id1", "id as id2")
+    val df1 = df.selectExpr("cast(struct(id1, id2).id1 as int)")
+    assert(df1.schema.head.name == "CAST(struct(id1, id2).id1 AS INT)")
+
+    val df2 = df.selectExpr("cast(array(struct(id1, id2))[0].id1 as int)")
+    assert(df2.schema.head.name == "CAST(array(struct(id1, id2))[0].id1 AS INT)")
+
+    val df3 = df.select(hex(expr("struct(id1, id2).id1")))
+    assert(df3.schema.head.name == "hex(struct(id1, id2).id1)")
+
+    // this test is to make sure we don't have a regression.
+    val df4 = df.selectExpr("id1 == null")
+    assert(df4.schema.head.name == "(id1 = NULL)")
+  }
+
+  test("SPARK-33989: Strip auto-generated cast when using Cast.sql") {
+    Seq("SELECT id == null FROM VALUES(1) AS t(id)",
+      "SELECT floor(1)",
+      "SELECT split(struct(c1, c2).c1, ',') FROM VALUES(1, 2) AS t(c1, c2)").foreach { sqlStr =>
+      assert(!sql(sqlStr).schema.fieldNames.head.toLowerCase(Locale.getDefault).contains("cast"))
+    }
+
+    Seq("SELECT id == CAST(null AS int) FROM VALUES(1) AS t(id)",
+      "SELECT floor(CAST(1 AS double))",
+      "SELECT split(CAST(struct(c1, c2).c1 AS string), ',') FROM VALUES(1, 2) AS t(c1, c2)"
+    ).foreach { sqlStr =>
+      assert(sql(sqlStr).schema.fieldNames.head.toLowerCase(Locale.getDefault).contains("cast"))
+    }
+  }
+
+  test("SPARK-34318: colRegex should work with column names & qualifiers which contain newlines") {
+    val df = Seq(1, 2, 3).toDF("test\n_column").as("test\n_table")
+    val col1 = df.colRegex("`tes.*\n.*mn`")
+    checkAnswer(df.select(col1), Row(1) :: Row(2) :: Row(3) :: Nil)
+
+    val col2 = df.colRegex("test\n_table.`tes.*\n.*mn`")
+    checkAnswer(df.select(col2), Row(1) :: Row(2) :: Row(3) :: Nil)
+  }
+
+  test("SPARK-34763: col(), $\"<name>\", df(\"name\") should handle quoted column name properly") {
+    val df1 = spark.sql("SELECT 'col1' AS `a``b.c`")
+    checkAnswer(df1.selectExpr("`a``b.c`"), Row("col1"))
+    checkAnswer(df1.select(df1("`a``b.c`")), Row("col1"))
+    checkAnswer(df1.select(col("`a``b.c`")), Row("col1"))
+    checkAnswer(df1.select($"`a``b.c`"), Row("col1"))
+
+    val df2 = df1.as("d.e`f")
+    checkAnswer(df2.selectExpr("`a``b.c`"), Row("col1"))
+    checkAnswer(df2.select(df2("`a``b.c`")), Row("col1"))
+    checkAnswer(df2.select(col("`a``b.c`")), Row("col1"))
+    checkAnswer(df2.select($"`a``b.c`"), Row("col1"))
+
+    checkAnswer(df2.selectExpr("`d.e``f`.`a``b.c`"), Row("col1"))
+    checkAnswer(df2.select(df2("`d.e``f`.`a``b.c`")), Row("col1"))
+    checkAnswer(df2.select(col("`d.e``f`.`a``b.c`")), Row("col1"))
+    checkAnswer(df2.select($"`d.e``f`.`a``b.c`"), Row("col1"))
+
+    val df3 = df1.as("*-#&% ?")
+    checkAnswer(df3.selectExpr("`*-#&% ?`.`a``b.c`"), Row("col1"))
+    checkAnswer(df3.select(df3("*-#&% ?.`a``b.c`")), Row("col1"))
+    checkAnswer(df3.select(col("*-#&% ?.`a``b.c`")), Row("col1"))
+    checkAnswer(df3.select($"*-#&% ?.`a``b.c`"), Row("col1"))
+  }
+
+  test("SPARK-34776: Nested column pruning should not prune Window produced attributes") {
+    val df = Seq(
+      ("t1", "123", "bob"),
+      ("t1", "456", "bob"),
+      ("t2", "123", "sam")
+    ).toDF("type", "value", "name")
+
+    val test = df.select(
+      $"*",
+      struct(count($"*").over(Window.partitionBy($"type", $"value", $"name"))
+        .as("count"), $"name").as("name_count")
+    ).select(
+      $"*",
+      max($"name_count").over(Window.partitionBy($"type", $"value")).as("best_name")
+    )
+    checkAnswer(test.select($"best_name.name"), Row("bob") :: Row("bob") :: Row("sam") :: Nil)
+  }
+
+  test("SPARK-34829: Multiple applications of typed ScalaUDFs in higher order functions work") {
+    val reverse = udf((s: String) => s.reverse)
+    val reverse2 = udf((b: Bar2) => Bar2(b.s.reverse))
+
+    val df = Seq(Array("abc", "def")).toDF("array")
+    val test = df.select(transform(col("array"), s => reverse(s)))
+    checkAnswer(test, Row(Array("cba", "fed")) :: Nil)
+
+    val df2 = Seq(Array(Bar2("abc"), Bar2("def"))).toDF("array")
+    val test2 = df2.select(transform(col("array"), b => reverse2(b)))
+    checkAnswer(test2, Row(Array(Row("cba"), Row("fed"))) :: Nil)
+
+    val df3 = Seq(Map("abc" -> 1, "def" -> 2)).toDF("map")
+    val test3 = df3.select(transform_keys(col("map"), (s, _) => reverse(s)))
+    checkAnswer(test3, Row(Map("cba" -> 1, "fed" -> 2)) :: Nil)
+
+    val df4 = Seq(Map(Bar2("abc") -> 1, Bar2("def") -> 2)).toDF("map")
+    val test4 = df4.select(transform_keys(col("map"), (b, _) => reverse2(b)))
+    checkAnswer(test4, Row(Map(Row("cba") -> 1, Row("fed") -> 2)) :: Nil)
+
+    val df5 = Seq(Map(1 -> "abc", 2 -> "def")).toDF("map")
+    val test5 = df5.select(transform_values(col("map"), (_, s) => reverse(s)))
+    checkAnswer(test5, Row(Map(1 -> "cba", 2 -> "fed")) :: Nil)
+
+    val df6 = Seq(Map(1 -> Bar2("abc"), 2 -> Bar2("def"))).toDF("map")
+    val test6 = df6.select(transform_values(col("map"), (_, b) => reverse2(b)))
+    checkAnswer(test6, Row(Map(1 -> Row("cba"), 2 -> Row("fed"))) :: Nil)
+
+    val reverseThenConcat = udf((s1: String, s2: String) => s1.reverse ++ s2.reverse)
+    val reverseThenConcat2 = udf((b1: Bar2, b2: Bar2) => Bar2(b1.s.reverse ++ b2.s.reverse))
+
+    val df7 = Seq((Map(1 -> "abc", 2 -> "def"), Map(1 -> "ghi", 2 -> "jkl"))).toDF("map1", "map2")
+    val test7 =
+      df7.select(map_zip_with(col("map1"), col("map2"), (_, s1, s2) => reverseThenConcat(s1, s2)))
+    checkAnswer(test7, Row(Map(1 -> "cbaihg", 2 -> "fedlkj")) :: Nil)
+
+    val df8 = Seq((Map(1 -> Bar2("abc"), 2 -> Bar2("def")),
+      Map(1 -> Bar2("ghi"), 2 -> Bar2("jkl")))).toDF("map1", "map2")
+    val test8 =
+      df8.select(map_zip_with(col("map1"), col("map2"), (_, b1, b2) => reverseThenConcat2(b1, b2)))
+    checkAnswer(test8, Row(Map(1 -> Row("cbaihg"), 2 -> Row("fedlkj"))) :: Nil)
+
+    val df9 = Seq((Array("abc", "def"), Array("ghi", "jkl"))).toDF("array1", "array2")
+    val test9 =
+      df9.select(zip_with(col("array1"), col("array2"), (s1, s2) => reverseThenConcat(s1, s2)))
+    checkAnswer(test9, Row(Array("cbaihg", "fedlkj")) :: Nil)
+
+    val df10 = Seq((Array(Bar2("abc"), Bar2("def")), Array(Bar2("ghi"), Bar2("jkl"))))
+      .toDF("array1", "array2")
+    val test10 =
+      df10.select(zip_with(col("array1"), col("array2"), (b1, b2) => reverseThenConcat2(b1, b2)))
+    checkAnswer(test10, Row(Array(Row("cbaihg"), Row("fedlkj"))) :: Nil)
+  }
+
+  test("SPARK-34882: Aggregate with multiple distinct null sensitive aggregators") {
+    withUserDefinedFunction(("countNulls", true)) {
+      spark.udf.register("countNulls", udaf(new Aggregator[JLong, JLong, JLong] {
+        def zero: JLong = 0L
+        def reduce(b: JLong, a: JLong): JLong = if (a == null) {
+          b + 1
+        } else {
+          b
+        }
+        def merge(b1: JLong, b2: JLong): JLong = b1 + b2
+        def finish(r: JLong): JLong = r
+        def bufferEncoder: Encoder[JLong] = Encoders.LONG
+        def outputEncoder: Encoder[JLong] = Encoders.LONG
+      }))
+
+      val result = testData.selectExpr(
+        "countNulls(key)",
+        "countNulls(DISTINCT key)",
+        "countNulls(key) FILTER (WHERE key > 50)",
+        "countNulls(DISTINCT key) FILTER (WHERE key > 50)",
+        "count(DISTINCT key)")
+
+      checkAnswer(result, Row(0, 0, 0, 0, 100))
+    }
+  }
 }
 
 case class GroupByKey(a: Int, b: Int)
+
+case class Bar2(s: String)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameTimeWindowingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameTimeWindowingSuite.scala
index 8b0f46b9d1ddb..4fdaeb57ad50e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameTimeWindowingSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameTimeWindowingSuite.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.sql
 
-import org.scalatest.BeforeAndAfterEach
-
 import org.apache.spark.sql.catalyst.plans.logical.Expand
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.test.SharedSparkSession
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFramesSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFramesSuite.scala
index 8c998290b5044..fd408c37ef6cd 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFramesSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFramesSuite.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.sql
 
-import java.sql.Date
-
 import org.apache.spark.sql.expressions.Window
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.test.SharedSparkSession
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
index 09ce43e4e2b27..666bf739ca9c9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql
 
-import org.scalatest.Matchers.the
+import org.scalatest.matchers.must.Matchers.the
 
 import org.apache.spark.TestUtils.{assertNotSpilled, assertSpilled}
 import org.apache.spark.sql.catalyst.optimizer.TransposeWindow
@@ -94,89 +94,187 @@ class DataFrameWindowFunctionsSuite extends QueryTest
   }
 
   test("corr, covar_pop, stddev_pop functions in specific window") {
-    val df = Seq(
-      ("a", "p1", 10.0, 20.0),
-      ("b", "p1", 20.0, 10.0),
-      ("c", "p2", 20.0, 20.0),
-      ("d", "p2", 20.0, 20.0),
-      ("e", "p3", 0.0, 0.0),
-      ("f", "p3", 6.0, 12.0),
-      ("g", "p3", 6.0, 12.0),
-      ("h", "p3", 8.0, 16.0),
-      ("i", "p4", 5.0, 5.0)).toDF("key", "partitionId", "value1", "value2")
-    checkAnswer(
-      df.select(
-        $"key",
-        corr("value1", "value2").over(Window.partitionBy("partitionId")
-          .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)),
-        covar_pop("value1", "value2")
-          .over(Window.partitionBy("partitionId")
+    withSQLConf(SQLConf.LEGACY_STATISTICAL_AGGREGATE.key -> "true") {
+      val df = Seq(
+        ("a", "p1", 10.0, 20.0),
+        ("b", "p1", 20.0, 10.0),
+        ("c", "p2", 20.0, 20.0),
+        ("d", "p2", 20.0, 20.0),
+        ("e", "p3", 0.0, 0.0),
+        ("f", "p3", 6.0, 12.0),
+        ("g", "p3", 6.0, 12.0),
+        ("h", "p3", 8.0, 16.0),
+        ("i", "p4", 5.0, 5.0)).toDF("key", "partitionId", "value1", "value2")
+      checkAnswer(
+        df.select(
+          $"key",
+          corr("value1", "value2").over(Window.partitionBy("partitionId")
             .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)),
-        var_pop("value1")
-          .over(Window.partitionBy("partitionId")
+          covar_pop("value1", "value2")
+            .over(Window.partitionBy("partitionId")
+              .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)),
+          var_pop("value1")
+            .over(Window.partitionBy("partitionId")
+              .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)),
+          stddev_pop("value1")
+            .over(Window.partitionBy("partitionId")
+              .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)),
+          var_pop("value2")
+            .over(Window.partitionBy("partitionId")
+              .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)),
+          stddev_pop("value2")
+            .over(Window.partitionBy("partitionId")
+              .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing))),
+
+        // As stddev_pop(expr) = sqrt(var_pop(expr))
+        // the "stddev_pop" column can be calculated from the "var_pop" column.
+        //
+        // As corr(expr1, expr2) = covar_pop(expr1, expr2) / (stddev_pop(expr1) * stddev_pop(expr2))
+        // the "corr" column can be calculated from the "covar_pop" and the two "stddev_pop" columns
+        Seq(
+          Row("a", -1.0, -25.0, 25.0, 5.0, 25.0, 5.0),
+          Row("b", -1.0, -25.0, 25.0, 5.0, 25.0, 5.0),
+          Row("c", null, 0.0, 0.0, 0.0, 0.0, 0.0),
+          Row("d", null, 0.0, 0.0, 0.0, 0.0, 0.0),
+          Row("e", 1.0, 18.0, 9.0, 3.0, 36.0, 6.0),
+          Row("f", 1.0, 18.0, 9.0, 3.0, 36.0, 6.0),
+          Row("g", 1.0, 18.0, 9.0, 3.0, 36.0, 6.0),
+          Row("h", 1.0, 18.0, 9.0, 3.0, 36.0, 6.0),
+          Row("i", Double.NaN, 0.0, 0.0, 0.0, 0.0, 0.0)))
+    }
+  }
+
+  test("SPARK-13860: " +
+    "corr, covar_pop, stddev_pop functions in specific window " +
+    "LEGACY_STATISTICAL_AGGREGATE off") {
+    withSQLConf(SQLConf.LEGACY_STATISTICAL_AGGREGATE.key -> "false") {
+      val df = Seq(
+        ("a", "p1", 10.0, 20.0),
+        ("b", "p1", 20.0, 10.0),
+        ("c", "p2", 20.0, 20.0),
+        ("d", "p2", 20.0, 20.0),
+        ("e", "p3", 0.0, 0.0),
+        ("f", "p3", 6.0, 12.0),
+        ("g", "p3", 6.0, 12.0),
+        ("h", "p3", 8.0, 16.0),
+        ("i", "p4", 5.0, 5.0)).toDF("key", "partitionId", "value1", "value2")
+      checkAnswer(
+        df.select(
+          $"key",
+          corr("value1", "value2").over(Window.partitionBy("partitionId")
             .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)),
-        stddev_pop("value1")
-          .over(Window.partitionBy("partitionId")
+          covar_pop("value1", "value2")
+            .over(Window.partitionBy("partitionId")
+              .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)),
+          var_pop("value1")
+            .over(Window.partitionBy("partitionId")
+              .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)),
+          stddev_pop("value1")
+            .over(Window.partitionBy("partitionId")
+              .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)),
+          var_pop("value2")
+            .over(Window.partitionBy("partitionId")
+              .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)),
+          stddev_pop("value2")
+            .over(Window.partitionBy("partitionId")
+              .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing))),
+
+        // As stddev_pop(expr) = sqrt(var_pop(expr))
+        // the "stddev_pop" column can be calculated from the "var_pop" column.
+        //
+        // As corr(expr1, expr2) = covar_pop(expr1, expr2) / (stddev_pop(expr1) * stddev_pop(expr2))
+        // the "corr" column can be calculated from the "covar_pop" and the two "stddev_pop" columns
+        Seq(
+          Row("a", -1.0, -25.0, 25.0, 5.0, 25.0, 5.0),
+          Row("b", -1.0, -25.0, 25.0, 5.0, 25.0, 5.0),
+          Row("c", null, 0.0, 0.0, 0.0, 0.0, 0.0),
+          Row("d", null, 0.0, 0.0, 0.0, 0.0, 0.0),
+          Row("e", 1.0, 18.0, 9.0, 3.0, 36.0, 6.0),
+          Row("f", 1.0, 18.0, 9.0, 3.0, 36.0, 6.0),
+          Row("g", 1.0, 18.0, 9.0, 3.0, 36.0, 6.0),
+          Row("h", 1.0, 18.0, 9.0, 3.0, 36.0, 6.0),
+          Row("i", null, 0.0, 0.0, 0.0, 0.0, 0.0)))
+    }
+  }
+
+  test("covar_samp, var_samp (variance), stddev_samp (stddev) functions in specific window") {
+    withSQLConf(SQLConf.LEGACY_STATISTICAL_AGGREGATE.key -> "true") {
+      val df = Seq(
+        ("a", "p1", 10.0, 20.0),
+        ("b", "p1", 20.0, 10.0),
+        ("c", "p2", 20.0, 20.0),
+        ("d", "p2", 20.0, 20.0),
+        ("e", "p3", 0.0, 0.0),
+        ("f", "p3", 6.0, 12.0),
+        ("g", "p3", 6.0, 12.0),
+        ("h", "p3", 8.0, 16.0),
+        ("i", "p4", 5.0, 5.0)).toDF("key", "partitionId", "value1", "value2")
+      checkAnswer(
+        df.select(
+          $"key",
+          covar_samp("value1", "value2").over(Window.partitionBy("partitionId")
             .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)),
-        var_pop("value2")
-          .over(Window.partitionBy("partitionId")
+          var_samp("value1").over(Window.partitionBy("partitionId")
             .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)),
-        stddev_pop("value2")
-          .over(Window.partitionBy("partitionId")
-            .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing))),
-
-      // As stddev_pop(expr) = sqrt(var_pop(expr))
-      // the "stddev_pop" column can be calculated from the "var_pop" column.
-      //
-      // As corr(expr1, expr2) = covar_pop(expr1, expr2) / (stddev_pop(expr1) * stddev_pop(expr2))
-      // the "corr" column can be calculated from the "covar_pop" and the two "stddev_pop" columns.
-      Seq(
-        Row("a", -1.0, -25.0, 25.0, 5.0, 25.0, 5.0),
-        Row("b", -1.0, -25.0, 25.0, 5.0, 25.0, 5.0),
-        Row("c", null, 0.0, 0.0, 0.0, 0.0, 0.0),
-        Row("d", null, 0.0, 0.0, 0.0, 0.0, 0.0),
-        Row("e", 1.0, 18.0, 9.0, 3.0, 36.0, 6.0),
-        Row("f", 1.0, 18.0, 9.0, 3.0, 36.0, 6.0),
-        Row("g", 1.0, 18.0, 9.0, 3.0, 36.0, 6.0),
-        Row("h", 1.0, 18.0, 9.0, 3.0, 36.0, 6.0),
-        Row("i", Double.NaN, 0.0, 0.0, 0.0, 0.0, 0.0)))
+          variance("value1").over(Window.partitionBy("partitionId")
+            .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)),
+          stddev_samp("value1").over(Window.partitionBy("partitionId")
+            .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)),
+          stddev("value1").over(Window.partitionBy("partitionId")
+            .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing))
+        ),
+        Seq(
+          Row("a", -50.0, 50.0, 50.0, 7.0710678118654755, 7.0710678118654755),
+          Row("b", -50.0, 50.0, 50.0, 7.0710678118654755, 7.0710678118654755),
+          Row("c", 0.0, 0.0, 0.0, 0.0, 0.0),
+          Row("d", 0.0, 0.0, 0.0, 0.0, 0.0),
+          Row("e", 24.0, 12.0, 12.0, 3.4641016151377544, 3.4641016151377544),
+          Row("f", 24.0, 12.0, 12.0, 3.4641016151377544, 3.4641016151377544),
+          Row("g", 24.0, 12.0, 12.0, 3.4641016151377544, 3.4641016151377544),
+          Row("h", 24.0, 12.0, 12.0, 3.4641016151377544, 3.4641016151377544),
+          Row("i", Double.NaN, Double.NaN, Double.NaN, Double.NaN, Double.NaN)))
+    }
   }
 
-  test("covar_samp, var_samp (variance), stddev_samp (stddev) functions in specific window") {
-    val df = Seq(
-      ("a", "p1", 10.0, 20.0),
-      ("b", "p1", 20.0, 10.0),
-      ("c", "p2", 20.0, 20.0),
-      ("d", "p2", 20.0, 20.0),
-      ("e", "p3", 0.0, 0.0),
-      ("f", "p3", 6.0, 12.0),
-      ("g", "p3", 6.0, 12.0),
-      ("h", "p3", 8.0, 16.0),
-      ("i", "p4", 5.0, 5.0)).toDF("key", "partitionId", "value1", "value2")
-    checkAnswer(
-      df.select(
-        $"key",
-        covar_samp("value1", "value2").over(Window.partitionBy("partitionId")
-          .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)),
-        var_samp("value1").over(Window.partitionBy("partitionId")
-          .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)),
-        variance("value1").over(Window.partitionBy("partitionId")
-          .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)),
-        stddev_samp("value1").over(Window.partitionBy("partitionId")
-          .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)),
-        stddev("value1").over(Window.partitionBy("partitionId")
-          .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing))
-      ),
-      Seq(
-        Row("a", -50.0, 50.0, 50.0, 7.0710678118654755, 7.0710678118654755),
-        Row("b", -50.0, 50.0, 50.0, 7.0710678118654755, 7.0710678118654755),
-        Row("c", 0.0, 0.0, 0.0, 0.0, 0.0),
-        Row("d", 0.0, 0.0, 0.0, 0.0, 0.0),
-        Row("e", 24.0, 12.0, 12.0, 3.4641016151377544, 3.4641016151377544),
-        Row("f", 24.0, 12.0, 12.0, 3.4641016151377544, 3.4641016151377544),
-        Row("g", 24.0, 12.0, 12.0, 3.4641016151377544, 3.4641016151377544),
-        Row("h", 24.0, 12.0, 12.0, 3.4641016151377544, 3.4641016151377544),
-        Row("i", Double.NaN, Double.NaN, Double.NaN, Double.NaN, Double.NaN)))
+  test("SPARK-13860: " +
+    "covar_samp, var_samp (variance), stddev_samp (stddev) functions in specific window " +
+    "LEGACY_STATISTICAL_AGGREGATE off") {
+    withSQLConf(SQLConf.LEGACY_STATISTICAL_AGGREGATE.key -> "false") {
+      val df = Seq(
+        ("a", "p1", 10.0, 20.0),
+        ("b", "p1", 20.0, 10.0),
+        ("c", "p2", 20.0, 20.0),
+        ("d", "p2", 20.0, 20.0),
+        ("e", "p3", 0.0, 0.0),
+        ("f", "p3", 6.0, 12.0),
+        ("g", "p3", 6.0, 12.0),
+        ("h", "p3", 8.0, 16.0),
+        ("i", "p4", 5.0, 5.0)).toDF("key", "partitionId", "value1", "value2")
+      checkAnswer(
+        df.select(
+          $"key",
+          covar_samp("value1", "value2").over(Window.partitionBy("partitionId")
+            .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)),
+          var_samp("value1").over(Window.partitionBy("partitionId")
+            .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)),
+          variance("value1").over(Window.partitionBy("partitionId")
+            .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)),
+          stddev_samp("value1").over(Window.partitionBy("partitionId")
+            .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)),
+          stddev("value1").over(Window.partitionBy("partitionId")
+            .orderBy("key").rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing))
+        ),
+        Seq(
+          Row("a", -50.0, 50.0, 50.0, 7.0710678118654755, 7.0710678118654755),
+          Row("b", -50.0, 50.0, 50.0, 7.0710678118654755, 7.0710678118654755),
+          Row("c", 0.0, 0.0, 0.0, 0.0, 0.0),
+          Row("d", 0.0, 0.0, 0.0, 0.0, 0.0),
+          Row("e", 24.0, 12.0, 12.0, 3.4641016151377544, 3.4641016151377544),
+          Row("f", 24.0, 12.0, 12.0, 3.4641016151377544, 3.4641016151377544),
+          Row("g", 24.0, 12.0, 12.0, 3.4641016151377544, 3.4641016151377544),
+          Row("h", 24.0, 12.0, 12.0, 3.4641016151377544, 3.4641016151377544),
+          Row("i", null, null, null, null, null)))
+    }
   }
 
   test("collect_list in ascending ordered window") {
@@ -301,7 +399,7 @@ class DataFrameWindowFunctionsSuite extends QueryTest
     val df = Seq((1, "1")).toDF("key", "value")
     val e = intercept[AnalysisException](
       df.select($"key", count("invalid").over()))
-    assert(e.message.contains("cannot resolve '`invalid`' given input columns: [key, value]"))
+    assert(e.message.contains("cannot resolve 'invalid' given input columns: [key, value]"))
   }
 
   test("numerical aggregate functions on string column") {
@@ -541,6 +639,157 @@ class DataFrameWindowFunctionsSuite extends QueryTest
         Row("b", 3, null, null, null, null, null, null)))
   }
 
+  test("nth_value with ignoreNulls") {
+    val nullStr: String = null
+    val df = Seq(
+      ("a", 0, nullStr),
+      ("a", 1, "x"),
+      ("a", 2, "y"),
+      ("a", 3, "z"),
+      ("a", 4, nullStr),
+      ("b", 1, nullStr),
+      ("b", 2, nullStr)).
+      toDF("key", "order", "value")
+    val window = Window.partitionBy($"key").orderBy($"order")
+    checkAnswer(
+      df.select(
+        $"key",
+        $"order",
+        nth_value($"value", 2).over(window),
+        nth_value($"value", 2, ignoreNulls = false).over(window),
+        nth_value($"value", 2, ignoreNulls = true).over(window),
+        nth_value($"value", 3, ignoreNulls = false).over(window)),
+      Seq(
+        Row("a", 0, null, null, null, null),
+        Row("a", 1, "x", "x", null, null),
+        Row("a", 2, "x", "x", "y", "y"),
+        Row("a", 3, "x", "x", "y", "y"),
+        Row("a", 4, "x", "x", "y", "y"),
+        Row("b", 1, null, null, null, null),
+        Row("b", 2, null, null, null, null)))
+  }
+
+  test("nth_value with ignoreNulls over offset window frame") {
+    val nullStr: String = null
+    val df = Seq(
+      ("a", 0, nullStr),
+      ("a", 1, "x"),
+      ("a", 2, "y"),
+      ("a", 3, "z"),
+      ("a", 4, nullStr),
+      ("b", 1, nullStr),
+      ("b", 2, nullStr)).
+      toDF("key", "order", "value")
+    val window1 = Window.partitionBy($"key").orderBy($"order")
+      .rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)
+    val window2 = Window.partitionBy($"key").orderBy($"order")
+      .rowsBetween(Window.unboundedPreceding, Window.currentRow)
+    checkAnswer(
+      df.select(
+        $"key",
+        $"order",
+        nth_value($"value", 2).over(window1),
+        nth_value($"value", 2, ignoreNulls = false).over(window1),
+        nth_value($"value", 2, ignoreNulls = true).over(window1),
+        nth_value($"value", 2).over(window2),
+        nth_value($"value", 2, ignoreNulls = false).over(window2),
+        nth_value($"value", 2, ignoreNulls = true).over(window2)),
+      Seq(
+        Row("a", 0, "x", "x", "y", null, null, null),
+        Row("a", 1, "x", "x", "y", "x", "x", null),
+        Row("a", 2, "x", "x", "y", "x", "x", "y"),
+        Row("a", 3, "x", "x", "y", "x", "x", "y"),
+        Row("a", 4, "x", "x", "y", "x", "x", "y"),
+        Row("b", 1, null, null, null, null, null, null),
+        Row("b", 2, null, null, null, null, null, null)))
+  }
+
+  test("nth_value on descending ordered window") {
+    val nullStr: String = null
+    val df = Seq(
+      ("a", 0, nullStr),
+      ("a", 1, "x"),
+      ("a", 2, "y"),
+      ("a", 3, "z"),
+      ("a", 4, "v"),
+      ("b", 1, "k"),
+      ("b", 2, "l"),
+      ("b", 3, nullStr)).
+      toDF("key", "order", "value")
+    val window = Window.partitionBy($"key").orderBy($"order".desc)
+    checkAnswer(
+      df.select(
+        $"key",
+        $"order",
+        nth_value($"value", 2).over(window),
+        nth_value($"value", 2, ignoreNulls = false).over(window),
+        nth_value($"value", 2, ignoreNulls = true).over(window)),
+      Seq(
+        Row("a", 0, "z", "z", "z"),
+        Row("a", 1, "z", "z", "z"),
+        Row("a", 2, "z", "z", "z"),
+        Row("a", 3, "z", "z", "z"),
+        Row("a", 4, null, null, null),
+        Row("b", 1, "l", "l", "k"),
+        Row("b", 2, "l", "l", null),
+        Row("b", 3, null, null, null)))
+  }
+
+  test("lead/lag with ignoreNulls") {
+    val nullStr: String = null
+    val df = Seq(
+      ("a", 0, nullStr),
+      ("a", 1, "x"),
+      ("b", 2, nullStr),
+      ("c", 3, nullStr),
+      ("a", 4, "y"),
+      ("b", 5, nullStr),
+      ("a", 6, "z"),
+      ("a", 7, "v"),
+      ("a", 8, nullStr)).
+      toDF("key", "order", "value")
+    val window = Window.orderBy($"order")
+    checkAnswer(
+      df.select(
+        $"key",
+        $"order",
+        $"value",
+        lead($"value", 1).over(window),
+        lead($"value", 2).over(window),
+        lead($"value", 0, null, true).over(window),
+        lead($"value", 1, null, true).over(window),
+        lead($"value", 2, null, true).over(window),
+        lead($"value", 3, null, true).over(window),
+        lead(concat($"value", $"key"), 1, null, true).over(window),
+        lag($"value", 1).over(window),
+        lag($"value", 2).over(window),
+        lag($"value", 0, null, true).over(window),
+        lag($"value", 1, null, true).over(window),
+        lag($"value", 2, null, true).over(window),
+        lag($"value", 3, null, true).over(window),
+        lag(concat($"value", $"key"), 1, null, true).over(window))
+        .orderBy($"order"),
+      Seq(
+        Row("a", 0, null, "x", null, null, "x", "y", "z", "xa",
+          null, null, null, null, null, null, null),
+        Row("a", 1, "x", null, null, "x", "y", "z", "v", "ya",
+          null, null, "x", null, null, null, null),
+        Row("b", 2, null, null, "y", null, "y", "z", "v", "ya",
+          "x", null, null, "x", null, null, "xa"),
+        Row("c", 3, null, "y", null, null, "y", "z", "v", "ya",
+          null, "x", null, "x", null, null, "xa"),
+        Row("a", 4, "y", null, "z", "y", "z", "v", null, "za",
+          null, null, "y", "x", null, null, "xa"),
+        Row("b", 5, null, "z", "v", null, "z", "v", null, "za",
+          "y", null, null, "y", "x", null, "ya"),
+        Row("a", 6, "z", "v", null, "z", "v", null, null, "va",
+          null, "y", "z", "y", "x", null, "ya"),
+        Row("a", 7, "v", null, null, "v", null, null, null, null,
+          "z", null, "v", "z", "y", "x", "za"),
+        Row("a", 8, null, null, null, null, null, null, null, null,
+          "v", "z", null, "v", "z", "y", "va")))
+  }
+
   test("SPARK-12989 ExtractWindowExpressions treats alias as regular attribute") {
     val src = Seq((0, 3, 5)).toDF("a", "b", "c")
       .withColumn("Data", struct("a", "b"))
@@ -795,4 +1044,30 @@ class DataFrameWindowFunctionsSuite extends QueryTest
         Row(Seq(-0.0f, 0.0f), Row(-0.0d, Double.NaN), Seq(Row(-0.0d, Double.NaN)), 2),
         Row(Seq(0.0f, -0.0f), Row(0.0d, Double.NaN), Seq(Row(0.0d, 0.0/0.0)), 2)))
   }
+
+  test("SPARK-34227: WindowFunctionFrame should clear its states during preparation") {
+    // This creates a single partition dataframe with 3 records:
+    //   "a", 0, null
+    //   "a", 1, "x"
+    //   "b", 0, null
+    val df = spark.range(0, 3, 1, 1).select(
+      when($"id" < 2, lit("a")).otherwise(lit("b")).as("key"),
+      ($"id" % 2).cast("int").as("order"),
+      when($"id" % 2 === 0, lit(null)).otherwise(lit("x")).as("value"))
+
+    val window1 = Window.partitionBy($"key").orderBy($"order")
+      .rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)
+    val window2 = Window.partitionBy($"key").orderBy($"order")
+      .rowsBetween(Window.unboundedPreceding, Window.currentRow)
+    checkAnswer(
+      df.select(
+        $"key",
+        $"order",
+        nth_value($"value", 1, ignoreNulls = true).over(window1),
+        nth_value($"value", 1, ignoreNulls = true).over(window2)),
+      Seq(
+        Row("a", 0, "x", null),
+        Row("a", 1, "x", "x"),
+        Row("b", 0, null, null)))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala
index ac2ebd8bd748b..8aef27a1b6692 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala
@@ -23,16 +23,15 @@ import scala.collection.JavaConverters._
 
 import org.scalatest.BeforeAndAfter
 
-import org.apache.spark.sql.catalyst.analysis.{CannotReplaceMissingTableException, NoSuchTableException, TableAlreadyExistsException}
+import org.apache.spark.sql.catalyst.analysis.{CannotReplaceMissingTableException, TableAlreadyExistsException}
 import org.apache.spark.sql.catalyst.plans.logical.{AppendData, LogicalPlan, OverwriteByExpression, OverwritePartitionsDynamic}
-import org.apache.spark.sql.connector.{InMemoryTable, InMemoryTableCatalog}
-import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog}
+import org.apache.spark.sql.connector.catalog.{Identifier, InMemoryTable, InMemoryTableCatalog, TableCatalog}
 import org.apache.spark.sql.connector.expressions.{BucketTransform, DaysTransform, FieldReference, HoursTransform, IdentityTransform, LiteralValue, MonthsTransform, YearsTransform}
 import org.apache.spark.sql.execution.QueryExecution
 import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
+import org.apache.spark.sql.sources.FakeSourceOne
 import org.apache.spark.sql.test.SharedSparkSession
-import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructType}
-import org.apache.spark.sql.types.TimestampType
+import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructType, TimestampType}
 import org.apache.spark.sql.util.QueryExecutionListener
 import org.apache.spark.unsafe.types.UTF8String
 import org.apache.spark.util.Utils
@@ -58,6 +57,7 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
   }
 
   after {
+    spark.sessionState.catalog.reset()
     spark.sessionState.catalogManager.reset()
     spark.sessionState.conf.clear()
   }
@@ -119,6 +119,18 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
       Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c"), Row(4L, "d"), Row(5L, "e"), Row(6L, "f")))
   }
 
+  test("Append: write to a temp view of v2 relation") {
+    spark.sql("CREATE TABLE testcat.table_name (id bigint, data string) USING foo")
+    spark.table("testcat.table_name").createOrReplaceTempView("temp_view")
+    spark.table("source").writeTo("temp_view").append()
+    checkAnswer(
+      spark.table("testcat.table_name"),
+      Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c")))
+    checkAnswer(
+      spark.table("temp_view"),
+      Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c")))
+  }
+
   test("Append: by name not position") {
     spark.sql("CREATE TABLE testcat.table_name (id bigint, data string) USING foo")
 
@@ -137,11 +149,36 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
   }
 
   test("Append: fail if table does not exist") {
-    val exc = intercept[NoSuchTableException] {
+    val exc = intercept[AnalysisException] {
       spark.table("source").writeTo("testcat.table_name").append()
     }
 
-    assert(exc.getMessage.contains("table_name"))
+    assert(exc.getMessage.contains("Table or view not found: testcat.table_name"))
+  }
+
+  test("Append: fail if it writes to a temp view that is not v2 relation") {
+    spark.range(10).createOrReplaceTempView("temp_view")
+    val exc = intercept[AnalysisException] {
+      spark.table("source").writeTo("temp_view").append()
+    }
+    assert(exc.getMessage.contains("Cannot write into temp view temp_view as it's not a " +
+      "data source v2 relation"))
+  }
+
+  test("Append: fail if it writes to a view") {
+    spark.sql("CREATE VIEW v AS SELECT 1")
+    val exc = intercept[AnalysisException] {
+      spark.table("source").writeTo("v").append()
+    }
+    assert(exc.getMessage.contains("Writing into a view is not allowed"))
+  }
+
+  test("Append: fail if it writes to a v1 table") {
+    sql(s"CREATE TABLE table_name USING ${classOf[FakeSourceOne].getName}")
+    val exc = intercept[AnalysisException] {
+      spark.table("source").writeTo("table_name").append()
+    }
+    assert(exc.getMessage.contains("Cannot write into v1 table: `default`.`table_name`"))
   }
 
   test("Overwrite: overwrite by expression: true") {
@@ -182,6 +219,20 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
       Seq(Row(1L, "a"), Row(2L, "b"), Row(4L, "d"), Row(5L, "e"), Row(6L, "f")))
   }
 
+  test("Overwrite: write to a temp view of v2 relation") {
+    spark.sql("CREATE TABLE testcat.table_name (id bigint, data string) USING foo")
+    spark.table("source").writeTo("testcat.table_name").append()
+    spark.table("testcat.table_name").createOrReplaceTempView("temp_view")
+
+    spark.table("source2").writeTo("testcat.table_name").overwrite(lit(true))
+    checkAnswer(
+      spark.table("testcat.table_name"),
+      Seq(Row(4L, "d"), Row(5L, "e"), Row(6L, "f")))
+    checkAnswer(
+      spark.table("temp_view"),
+      Seq(Row(4L, "d"), Row(5L, "e"), Row(6L, "f")))
+  }
+
   test("Overwrite: by name not position") {
     spark.sql("CREATE TABLE testcat.table_name (id bigint, data string) USING foo")
 
@@ -201,11 +252,36 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
   }
 
   test("Overwrite: fail if table does not exist") {
-    val exc = intercept[NoSuchTableException] {
+    val exc = intercept[AnalysisException] {
       spark.table("source").writeTo("testcat.table_name").overwrite(lit(true))
     }
 
-    assert(exc.getMessage.contains("table_name"))
+    assert(exc.getMessage.contains("Table or view not found: testcat.table_name"))
+  }
+
+  test("Overwrite: fail if it writes to a temp view that is not v2 relation") {
+    spark.range(10).createOrReplaceTempView("temp_view")
+    val exc = intercept[AnalysisException] {
+      spark.table("source").writeTo("temp_view").overwrite(lit(true))
+    }
+    assert(exc.getMessage.contains("Cannot write into temp view temp_view as it's not a " +
+      "data source v2 relation"))
+  }
+
+  test("Overwrite: fail if it writes to a view") {
+    spark.sql("CREATE VIEW v AS SELECT 1")
+    val exc = intercept[AnalysisException] {
+      spark.table("source").writeTo("v").overwrite(lit(true))
+    }
+    assert(exc.getMessage.contains("Writing into a view is not allowed"))
+  }
+
+  test("Overwrite: fail if it writes to a v1 table") {
+    sql(s"CREATE TABLE table_name USING ${classOf[FakeSourceOne].getName}")
+    val exc = intercept[AnalysisException] {
+      spark.table("source").writeTo("table_name").overwrite(lit(true))
+    }
+    assert(exc.getMessage.contains("Cannot write into v1 table: `default`.`table_name`"))
   }
 
   test("OverwritePartitions: overwrite conflicting partitions") {
@@ -246,6 +322,20 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
       Seq(Row(4L, "d"), Row(5L, "e"), Row(6L, "f")))
   }
 
+  test("OverwritePartitions: write to a temp view of v2 relation") {
+    spark.sql("CREATE TABLE testcat.table_name (id bigint, data string) USING foo")
+    spark.table("source").writeTo("testcat.table_name").append()
+    spark.table("testcat.table_name").createOrReplaceTempView("temp_view")
+
+    spark.table("source2").writeTo("testcat.table_name").overwritePartitions()
+    checkAnswer(
+      spark.table("testcat.table_name"),
+      Seq(Row(4L, "d"), Row(5L, "e"), Row(6L, "f")))
+    checkAnswer(
+      spark.table("temp_view"),
+      Seq(Row(4L, "d"), Row(5L, "e"), Row(6L, "f")))
+  }
+
   test("OverwritePartitions: by name not position") {
     spark.sql("CREATE TABLE testcat.table_name (id bigint, data string) USING foo")
 
@@ -265,11 +355,36 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
   }
 
   test("OverwritePartitions: fail if table does not exist") {
-    val exc = intercept[NoSuchTableException] {
+    val exc = intercept[AnalysisException] {
       spark.table("source").writeTo("testcat.table_name").overwritePartitions()
     }
 
-    assert(exc.getMessage.contains("table_name"))
+    assert(exc.getMessage.contains("Table or view not found: testcat.table_name"))
+  }
+
+  test("OverwritePartitions: fail if it writes to a temp view that is not v2 relation") {
+    spark.range(10).createOrReplaceTempView("temp_view")
+    val exc = intercept[AnalysisException] {
+      spark.table("source").writeTo("temp_view").overwritePartitions()
+    }
+    assert(exc.getMessage.contains("Cannot write into temp view temp_view as it's not a " +
+      "data source v2 relation"))
+  }
+
+  test("OverwritePartitions: fail if it writes to a view") {
+    spark.sql("CREATE VIEW v AS SELECT 1")
+    val exc = intercept[AnalysisException] {
+      spark.table("source").writeTo("v").overwritePartitions()
+    }
+    assert(exc.getMessage.contains("Writing into a view is not allowed"))
+  }
+
+  test("OverwritePartitions: fail if it writes to a v1 table") {
+    sql(s"CREATE TABLE table_name USING ${classOf[FakeSourceOne].getName}")
+    val exc = intercept[AnalysisException] {
+      spark.table("source").writeTo("table_name").overwritePartitions()
+    }
+    assert(exc.getMessage.contains("Cannot write into v1 table: `default`.`table_name`"))
   }
 
   test("Create: basic behavior") {
@@ -336,7 +451,6 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
     spark.table("source")
         .withColumn("ts", lit("2019-06-01 10:00:00.000000").cast("timestamp"))
         .writeTo("testcat.table_name")
-        .tableProperty("allow-unsupported-transforms", "true")
         .partitionedBy(years($"ts"))
         .create()
 
@@ -350,7 +464,6 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
     spark.table("source")
         .withColumn("ts", lit("2019-06-01 10:00:00.000000").cast("timestamp"))
         .writeTo("testcat.table_name")
-        .tableProperty("allow-unsupported-transforms", "true")
         .partitionedBy(months($"ts"))
         .create()
 
@@ -364,7 +477,6 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
     spark.table("source")
         .withColumn("ts", lit("2019-06-01 10:00:00.000000").cast("timestamp"))
         .writeTo("testcat.table_name")
-        .tableProperty("allow-unsupported-transforms", "true")
         .partitionedBy(days($"ts"))
         .create()
 
@@ -378,7 +490,6 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
     spark.table("source")
         .withColumn("ts", lit("2019-06-01 10:00:00.000000").cast("timestamp"))
         .writeTo("testcat.table_name")
-        .tableProperty("allow-unsupported-transforms", "true")
         .partitionedBy(hours($"ts"))
         .create()
 
@@ -391,7 +502,6 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
   test("Create: partitioned by bucket(4, id)") {
     spark.table("source")
         .writeTo("testcat.table_name")
-        .tableProperty("allow-unsupported-transforms", "true")
         .partitionedBy(bucket(4, $"id"))
         .create()
 
@@ -596,7 +706,6 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
         lit("2019-09-02 07:00:00.000000").cast("timestamp") as "modified",
         lit("America/Los_Angeles") as "timezone"))
       .writeTo("testcat.table_name")
-      .tableProperty("allow-unsupported-transforms", "true")
       .partitionedBy(
         years($"ts.created"), months($"ts.created"), days($"ts.created"), hours($"ts.created"),
         years($"ts.modified"), months($"ts.modified"), days($"ts.modified"), hours($"ts.modified")
@@ -624,7 +733,6 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
         lit("2019-09-02 07:00:00.000000").cast("timestamp") as "modified",
         lit("America/Los_Angeles") as "timezone"))
       .writeTo("testcat.table_name")
-      .tableProperty("allow-unsupported-transforms", "true")
       .partitionedBy(bucket(4, $"ts.timezone"))
       .create()
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetBenchmark.scala
index e47a6a68a0a9c..955d0f9193e24 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetBenchmark.scala
@@ -28,7 +28,9 @@ import org.apache.spark.sql.types.StringType
  * Benchmark for Dataset typed operations comparing with DataFrame and RDD versions.
  * To run this benchmark:
  * {{{
- *   1. without sbt: bin/spark-submit --class <this class> <spark sql test jar>
+ *   1. without sbt:
+ *      bin/spark-submit --class <this class>
+ *        --jars <spark core test jar>,<spark catalyst test jar> <spark sql test jar>
  *   2. build/sbt "sql/test:runMain <this class>"
  *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
  *      Results will be written to "benchmarks/DatasetBenchmark-results.txt".
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetCacheSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetCacheSuite.scala
index 5c144dad23c30..009ccb9a45354 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetCacheSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetCacheSuite.scala
@@ -102,18 +102,19 @@ class DatasetCacheSuite extends QueryTest
   test("persist and then groupBy columns asKey, map") {
     val ds = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDS()
     val grouped = ds.groupByKey(_._1)
-    val agged = grouped.mapGroups { (g, iter) => (g, iter.map(_._2).sum) }
-    agged.persist()
+    val aggregated = grouped.mapGroups { (g, iter) => (g, iter.map(_._2).sum) }
+    aggregated.persist()
 
     checkDataset(
-      agged.filter(_._1 == "b"),
+      aggregated.filter(_._1 == "b"),
       ("b", 3))
-    assertCached(agged.filter(_._1 == "b"))
+    assertCached(aggregated.filter(_._1 == "b"))
 
     ds.unpersist(blocking = true)
     assert(ds.storageLevel == StorageLevel.NONE, "The Dataset ds should not be cached.")
-    agged.unpersist(blocking = true)
-    assert(agged.storageLevel == StorageLevel.NONE, "The Dataset agged should not be cached.")
+    aggregated.unpersist(blocking = true)
+    assert(aggregated.storageLevel == StorageLevel.NONE,
+           "The Dataset aggregated should not be cached.")
   }
 
   test("persist and then withColumn") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetOptimizationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetOptimizationSuite.scala
index 0ac99905f35f4..5b8c80b471bb4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetOptimizationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetOptimizationSuite.scala
@@ -18,12 +18,13 @@
 package org.apache.spark.sql
 
 import org.apache.spark.metrics.source.CodegenMetrics
+import org.apache.spark.sql.catalyst.encoders.RowEncoder
 import org.apache.spark.sql.catalyst.expressions.{CreateNamedStruct, Expression}
-import org.apache.spark.sql.catalyst.expressions.objects.ExternalMapToCatalyst
 import org.apache.spark.sql.catalyst.plans.logical.SerializeFromObject
-import org.apache.spark.sql.functions.expr
+import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.StructType
 
 class DatasetOptimizationSuite extends QueryTest with SharedSparkSession {
   import testImplicits._
@@ -195,4 +196,12 @@ class DatasetOptimizationSuite extends QueryTest with SharedSparkSession {
       checkCodegenCache(() => Seq(Seq(Map("abc" -> 1))).toDS())
     }
   }
+
+  test("SPARK-32652: Pruned nested serializers: RowEncoder") {
+    val df = Seq(("a", 1), ("b", 2), ("c", 3)).toDF("i", "j")
+    val encoder = RowEncoder(new StructType().add("s", df.schema))
+    val query = df.map(row => Row(row))(encoder).select("s.i")
+    testSerializer(query, Seq(Seq("i")))
+    checkAnswer(query, Seq(Row("a"), Row("b"), Row("c")))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetPrimitiveSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetPrimitiveSuite.scala
index 124b58483d24f..8547d96e0f457 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetPrimitiveSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetPrimitiveSuite.scala
@@ -20,7 +20,6 @@ package org.apache.spark.sql
 import scala.collection.immutable.{HashSet => HSet}
 import scala.collection.immutable.Queue
 import scala.collection.mutable.{LinkedHashMap => LHMap}
-import scala.collection.mutable.ArrayBuffer
 
 import org.apache.spark.sql.test.SharedSparkSession
 
@@ -171,23 +170,23 @@ class DatasetPrimitiveSuite extends QueryTest with SharedSparkSession {
   test("groupBy function, map") {
     val ds = Seq(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11).toDS()
     val grouped = ds.groupByKey(_ % 2)
-    val agged = grouped.mapGroups { (g, iter) =>
+    val aggregated = grouped.mapGroups { (g, iter) =>
       val name = if (g == 0) "even" else "odd"
       (name, iter.size)
     }
 
     checkDatasetUnorderly(
-      agged,
+      aggregated,
       ("even", 5), ("odd", 6))
   }
 
   test("groupBy function, flatMap") {
     val ds = Seq("a", "b", "c", "xyz", "hello").toDS()
     val grouped = ds.groupByKey(_.length)
-    val agged = grouped.flatMapGroups { (g, iter) => Iterator(g.toString, iter.mkString) }
+    val aggregated = grouped.flatMapGroups { (g, iter) => Iterator(g.toString, iter.mkString) }
 
     checkDatasetUnorderly(
-      agged,
+      aggregated,
       "1", "abc", "3", "xyz", "5", "hello")
   }
 
@@ -223,16 +222,6 @@ class DatasetPrimitiveSuite extends QueryTest with SharedSparkSession {
     checkDataset(Seq(Queue(true)).toDS(), Queue(true))
     checkDataset(Seq(Queue("test")).toDS(), Queue("test"))
     checkDataset(Seq(Queue(Tuple1(1))).toDS(), Queue(Tuple1(1)))
-
-    checkDataset(Seq(ArrayBuffer(1)).toDS(), ArrayBuffer(1))
-    checkDataset(Seq(ArrayBuffer(1.toLong)).toDS(), ArrayBuffer(1.toLong))
-    checkDataset(Seq(ArrayBuffer(1.toDouble)).toDS(), ArrayBuffer(1.toDouble))
-    checkDataset(Seq(ArrayBuffer(1.toFloat)).toDS(), ArrayBuffer(1.toFloat))
-    checkDataset(Seq(ArrayBuffer(1.toByte)).toDS(), ArrayBuffer(1.toByte))
-    checkDataset(Seq(ArrayBuffer(1.toShort)).toDS(), ArrayBuffer(1.toShort))
-    checkDataset(Seq(ArrayBuffer(true)).toDS(), ArrayBuffer(true))
-    checkDataset(Seq(ArrayBuffer("test")).toDS(), ArrayBuffer("test"))
-    checkDataset(Seq(ArrayBuffer(Tuple1(1))).toDS(), ArrayBuffer(Tuple1(1)))
   }
 
   test("sequence and product combinations") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSerializerRegistratorSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSerializerRegistratorSuite.scala
index 43de2663b1236..b20d050f2fc4a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSerializerRegistratorSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSerializerRegistratorSuite.scala
@@ -34,7 +34,7 @@ class DatasetSerializerRegistratorSuite extends QueryTest with SharedSparkSessio
 
   override protected def sparkConf: SparkConf = {
     // Make sure we use the KryoRegistrator
-    super.sparkConf.set(KRYO_USER_REGISTRATORS, TestRegistrator().getClass.getCanonicalName)
+    super.sparkConf.set(KRYO_USER_REGISTRATORS, Seq(TestRegistrator().getClass.getCanonicalName))
   }
 
   test("Kryo registrator") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
index af65957691b37..e933b4488b8c5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
@@ -25,7 +25,7 @@ import org.scalatest.exceptions.TestFailedException
 import org.scalatest.prop.TableDrivenPropertyChecks._
 
 import org.apache.spark.{SparkException, TaskContext}
-import org.apache.spark.sql.catalyst.ScroogeLikeExample
+import org.apache.spark.sql.catalyst.{FooClassWithEnum, FooEnum, ScroogeLikeExample}
 import org.apache.spark.sql.catalyst.encoders.{OuterScopes, RowEncoder}
 import org.apache.spark.sql.catalyst.plans.{LeftAnti, LeftSemi}
 import org.apache.spark.sql.catalyst.util.sideBySide
@@ -528,42 +528,42 @@ class DatasetSuite extends QueryTest
   test("groupBy function, map") {
     val ds = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDS()
     val grouped = ds.groupByKey(v => (v._1, "word"))
-    val agged = grouped.mapGroups { (g, iter) => (g._1, iter.map(_._2).sum) }
+    val aggregated = grouped.mapGroups { (g, iter) => (g._1, iter.map(_._2).sum) }
 
     checkDatasetUnorderly(
-      agged,
+      aggregated,
       ("a", 30), ("b", 3), ("c", 1))
   }
 
   test("groupBy function, flatMap") {
     val ds = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDS()
     val grouped = ds.groupByKey(v => (v._1, "word"))
-    val agged = grouped.flatMapGroups { (g, iter) =>
+    val aggregated = grouped.flatMapGroups { (g, iter) =>
       Iterator(g._1, iter.map(_._2).sum.toString)
     }
 
     checkDatasetUnorderly(
-      agged,
+      aggregated,
       "a", "30", "b", "3", "c", "1")
   }
 
   test("groupBy function, mapValues, flatMap") {
     val ds = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDS()
     val keyValue = ds.groupByKey(_._1).mapValues(_._2)
-    val agged = keyValue.mapGroups { (g, iter) => (g, iter.sum) }
-    checkDataset(agged, ("a", 30), ("b", 3), ("c", 1))
+    val aggregated = keyValue.mapGroups { (g, iter) => (g, iter.sum) }
+    checkDataset(aggregated, ("a", 30), ("b", 3), ("c", 1))
 
     val keyValue1 = ds.groupByKey(t => (t._1, "key")).mapValues(t => (t._2, "value"))
-    val agged1 = keyValue1.mapGroups { (g, iter) => (g._1, iter.map(_._1).sum) }
-    checkDataset(agged1, ("a", 30), ("b", 3), ("c", 1))
+    val aggregated1 = keyValue1.mapGroups { (g, iter) => (g._1, iter.map(_._1).sum) }
+    checkDataset(aggregated1, ("a", 30), ("b", 3), ("c", 1))
   }
 
   test("groupBy function, reduce") {
     val ds = Seq("abc", "xyz", "hello").toDS()
-    val agged = ds.groupByKey(_.length).reduceGroups(_ + _)
+    val aggregated = ds.groupByKey(_.length).reduceGroups(_ + _)
 
     checkDatasetUnorderly(
-      agged,
+      aggregated,
       3 -> "abcxyz", 5 -> "hello")
   }
 
@@ -864,7 +864,7 @@ class DatasetSuite extends QueryTest
     val e = intercept[AnalysisException] {
       ds.as[ClassData2]
     }
-    assert(e.getMessage.contains("cannot resolve '`c`' given input columns: [a, b]"), e.getMessage)
+    assert(e.getMessage.contains("cannot resolve 'c' given input columns: [a, b]"), e.getMessage)
   }
 
   test("runtime nullability check") {
@@ -914,11 +914,11 @@ class DatasetSuite extends QueryTest
 
   test("grouping key and grouped value has field with same name") {
     val ds = Seq(ClassData("a", 1), ClassData("a", 2)).toDS()
-    val agged = ds.groupByKey(d => ClassNullableData(d.a, null)).mapGroups {
+    val aggregated = ds.groupByKey(d => ClassNullableData(d.a, null)).mapGroups {
       (key, values) => key.a + values.map(_.b).sum
     }
 
-    checkDataset(agged, "a3")
+    checkDataset(aggregated, "a3")
   }
 
   test("cogroup's left and right side has field with same name") {
@@ -1116,8 +1116,8 @@ class DatasetSuite extends QueryTest
       """+--------+
         ||       f|
         |+--------+
-        ||[foo, 1]|
-        ||[bar, 2]|
+        ||{foo, 1}|
+        ||{bar, 2}|
         |+--------+
         |""".stripMargin
 
@@ -1286,7 +1286,7 @@ class DatasetSuite extends QueryTest
       Route("b", "c", 6))
     val ds = sparkContext.parallelize(data).toDF.as[Route]
 
-    val grped = ds.map(r => GroupedRoutes(r.src, r.dest, Seq(r)))
+    val grouped = ds.map(r => GroupedRoutes(r.src, r.dest, Seq(r)))
       .groupByKey(r => (r.src, r.dest))
       .reduceGroups { (g1: GroupedRoutes, g2: GroupedRoutes) =>
         GroupedRoutes(g1.src, g1.dest, g1.routes ++ g2.routes)
@@ -1303,7 +1303,7 @@ class DatasetSuite extends QueryTest
     implicit def ordering[GroupedRoutes]: Ordering[GroupedRoutes] =
       (x: GroupedRoutes, y: GroupedRoutes) => x.toString.compareTo(y.toString)
 
-    checkDatasetUnorderly(grped, expected: _*)
+    checkDatasetUnorderly(grouped, expected: _*)
   }
 
   test("SPARK-18189: Fix serialization issue in KeyValueGroupedDataset") {
@@ -1383,7 +1383,7 @@ class DatasetSuite extends QueryTest
               }
             }
           } else {
-            // Local checkpoints dont require checkpoint_dir
+            // Local checkpoints don't require checkpoint_dir
             f
           }
         }
@@ -1474,7 +1474,7 @@ class DatasetSuite extends QueryTest
   }
 
   test("SPARK-18717: code generation works for both scala.collection.Map" +
-    " and scala.collection.imutable.Map") {
+    " and scala.collection.immutable.Map") {
     val ds = Seq(WithImmutableMap("hi", Map(42L -> "foo"))).toDS
     checkDataset(ds.map(t => t), WithImmutableMap("hi", Map(42L -> "foo")))
 
@@ -1693,6 +1693,33 @@ class DatasetSuite extends QueryTest
     checkDataset(ds1.select("_2._2"), ds2.select("_2._2").collect(): _*)
   }
 
+  test("SPARK-23862: Spark ExpressionEncoder should support Java Enum type from Scala") {
+    val saveModeSeq =
+      Seq(SaveMode.Append, SaveMode.Overwrite, SaveMode.ErrorIfExists, SaveMode.Ignore, null)
+    assert(saveModeSeq.toDS().collect().toSeq === saveModeSeq)
+    assert(saveModeSeq.toDS().schema === new StructType().add("value", StringType, nullable = true))
+
+    val saveModeCaseSeq = saveModeSeq.map(SaveModeCase.apply)
+    assert(saveModeCaseSeq.toDS().collect().toSet === saveModeCaseSeq.toSet)
+    assert(saveModeCaseSeq.toDS().schema ===
+      new StructType().add("mode", StringType, nullable = true))
+
+    val saveModeArrayCaseSeq =
+      Seq(SaveModeArrayCase(Array()), SaveModeArrayCase(saveModeSeq.toArray))
+    val collected = saveModeArrayCaseSeq.toDS().collect()
+    assert(collected.length === 2)
+    val sortedByLength = collected.sortBy(_.modes.length)
+    assert(sortedByLength(0).modes === Array())
+    assert(sortedByLength(1).modes === saveModeSeq.toArray)
+    assert(saveModeArrayCaseSeq.toDS().schema ===
+      new StructType().add("modes", ArrayType(StringType, containsNull = true), nullable = true))
+
+    // Enum is stored as string, so it is possible to convert to/from string
+    val stringSeq = saveModeSeq.map(Option.apply).map(_.map(_.toString).orNull)
+    assert(stringSeq.toDS().as[SaveMode].collect().toSet === saveModeSeq.toSet)
+    assert(saveModeSeq.toDS().as[String].collect().toSet === stringSeq.toSet)
+  }
+
   test("SPARK-24571: filtering of string values by char literal") {
     val df = Seq("Amsterdam", "San Francisco", "X").toDF("city")
     checkAnswer(df.where($"city" === 'X'), Seq(Row("X")))
@@ -1826,7 +1853,7 @@ class DatasetSuite extends QueryTest
         .map(b => b - 1)
         .collect()
     }
-    assert(thrownException.message.contains("Cannot up cast `id` from bigint to tinyint"))
+    assert(thrownException.message.contains("Cannot up cast id from bigint to tinyint"))
   }
 
   test("SPARK-26690: checkpoints should be executed with an execution id") {
@@ -1916,8 +1943,84 @@ class DatasetSuite extends QueryTest
     assert(df1.semanticHash !== df3.semanticHash)
     assert(df3.semanticHash === df4.semanticHash)
   }
+
+  test("SPARK-31854: Invoke in MapElementsExec should not propagate null") {
+    Seq("true", "false").foreach { wholeStage =>
+      withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> wholeStage) {
+        val ds = Seq(1.asInstanceOf[Integer], null.asInstanceOf[Integer]).toDS()
+        val expectedAnswer = Seq[(Integer, Integer)]((1, 1), (null, null))
+        checkDataset(ds.map(v => (v, v)), expectedAnswer: _*)
+      }
+    }
+  }
+
+  test("SPARK-32585: Support scala enumeration in ScalaReflection") {
+    checkDataset(
+      Seq(FooClassWithEnum(1, FooEnum.E1), FooClassWithEnum(2, FooEnum.E2)).toDS(),
+      Seq(FooClassWithEnum(1, FooEnum.E1), FooClassWithEnum(2, FooEnum.E2)): _*
+    )
+
+    // test null
+    checkDataset(
+      Seq(FooClassWithEnum(1, null), FooClassWithEnum(2, FooEnum.E2)).toDS(),
+      Seq(FooClassWithEnum(1, null), FooClassWithEnum(2, FooEnum.E2)): _*
+    )
+  }
+
+  test("SPARK-33390: Make Literal support char array") {
+    val df = Seq("aa", "bb", "cc", "abc").toDF("zoo")
+    checkAnswer(df.where($"zoo" === Array('a', 'a')), Seq(Row("aa")))
+    checkAnswer(
+      df.where($"zoo".contains(Array('a', 'b'))),
+      Seq(Row("abc")))
+  }
+
+  test("SPARK-33469: Add current_timezone function") {
+    val df = Seq(1).toDF("c")
+    withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "Asia/Shanghai") {
+      val timezone = df.selectExpr("current_timezone()").collect().head.getString(0)
+      assert(timezone == "Asia/Shanghai")
+    }
+  }
+
+  test("SPARK-34002: Fix broken Option input/output in UDF") {
+    def f1(bar: Bar): Option[Bar] = {
+      None
+    }
+
+    def f2(bar: Option[Bar]): Option[Bar] = {
+      bar
+    }
+
+    val udf1 = udf(f1 _).withName("f1")
+    val udf2 = udf(f2 _).withName("f2")
+
+    val df = (1 to 2).map(i => Tuple1(Bar(1))).toDF("c0")
+    val withUDF = df
+      .withColumn("c1", udf1(col("c0")))
+      .withColumn("c2", udf2(col("c1")))
+
+    assert(withUDF.schema == StructType(
+      StructField("c0", StructType(StructField("a", IntegerType, false) :: Nil)) ::
+        StructField("c1", StructType(StructField("a", IntegerType, false) :: Nil)) ::
+        StructField("c2", StructType(StructField("a", IntegerType, false) :: Nil)) :: Nil))
+
+    checkAnswer(withUDF, Row(Row(1), null, null) :: Row(Row(1), null, null) :: Nil)
+  }
+
+  test("SPARK-34605: implicit encoder for java.time.Duration") {
+    val duration = java.time.Duration.ofMinutes(10)
+    assert(spark.range(1).map { _ => duration }.head === duration)
+  }
+
+  test("SPARK-34615: implicit encoder for java.time.Period") {
+    val period = java.time.Period.ofYears(9999).withMonths(11)
+    assert(spark.range(1).map { _ => period }.head === period)
+  }
 }
 
+case class Bar(a: Int)
+
 object AssertExecutionId {
   def apply(id: Long): Long = {
     assert(TaskContext.get().getLocalProperty(SQLExecution.EXECUTION_ID_KEY) != null)
@@ -2014,3 +2117,7 @@ case class CircularReferenceClassD(map: Map[String, CircularReferenceClassE])
 case class CircularReferenceClassE(id: String, list: List[CircularReferenceClassD])
 
 case class SpecialCharClass(`field.1`: String, `field 2`: String)
+
+/** Used to test Java Enums from Scala code */
+case class SaveModeCase(mode: SaveMode)
+case class SaveModeArrayCase(modes: Array[SaveMode])
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
index 14e6ee2b04c14..d927953677d03 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
@@ -23,7 +23,7 @@ import java.time.{Instant, LocalDateTime, ZoneId}
 import java.util.{Locale, TimeZone}
 import java.util.concurrent.TimeUnit
 
-import org.apache.spark.SparkException
+import org.apache.spark.{SparkException, SparkUpgradeException}
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{CEST, LA}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.functions._
@@ -47,7 +47,7 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
 
   test("function current_timestamp and now") {
     val df1 = Seq((1, 2), (3, 1)).toDF("a", "b")
-    checkAnswer(df1.select(countDistinct(current_timestamp())), Row(1))
+    checkAnswer(df1.select(count_distinct(current_timestamp())), Row(1))
 
     // Execution in one query should return the same value
     checkAnswer(sql("""SELECT CURRENT_TIMESTAMP() = CURRENT_TIMESTAMP()"""), Row(true))
@@ -323,6 +323,46 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
         Row(Timestamp.valueOf("2015-12-27 00:00:00"))))
   }
 
+  test("function make_interval") {
+    val t1 = Timestamp.valueOf("2015-10-01 00:00:01")
+    val t2 = Timestamp.valueOf("2016-02-29 00:00:02")
+    val df = Seq((t1), (t2)).toDF("t")
+    // adds two hours to times
+    checkAnswer(
+      df.select(col("t") + make_interval(hours = lit(2))),
+      Seq(Row(Timestamp.valueOf("2015-10-01 02:00:01")),
+        Row(Timestamp.valueOf("2016-02-29 02:00:02"))))
+    // adds four days and two hours to times
+    checkAnswer(
+      df.select(col("t") + make_interval(hours = lit(2), days = lit(4))),
+      Seq(Row(Timestamp.valueOf("2015-10-05 02:00:01")),
+        Row(Timestamp.valueOf("2016-03-04 02:00:02"))))
+    // subtracts two hours from times
+    checkAnswer(
+      df.select(col("t") + make_interval(hours = lit(-2))),
+      Seq(Row(Timestamp.valueOf("2015-09-30 22:00:01")),
+        Row(Timestamp.valueOf("2016-02-28 22:00:02"))))
+
+    val d1 = Date.valueOf("2015-08-31")
+    val d2 = Date.valueOf("2015-02-28")
+    val df2 = Seq((d1), (d2)).toDF("d")
+    // adding an hour to a date does nothing
+    checkAnswer(
+      df2.select(col("d") + make_interval(hours = lit(1))),
+      Seq(Row(Date.valueOf("2015-08-31")),
+        Row(Date.valueOf("2015-02-28"))))
+    // adds three years to date
+    checkAnswer(
+      df2.select(col("d") + make_interval(years = lit(3))),
+      Seq(Row(Date.valueOf("2018-08-31")),
+        Row(Date.valueOf("2018-02-28"))))
+    // subtracts 1 week, one day from date
+    checkAnswer(
+      df2.select(col("d") - make_interval(weeks = lit(1), days = lit(1))),
+      Seq(Row(Date.valueOf("2015-08-23")),
+        Row(Date.valueOf("2015-02-20"))))
+  }
+
   test("function add_months") {
     val d1 = Date.valueOf("2015-08-31")
     val d2 = Date.valueOf("2015-02-28")
@@ -372,11 +412,21 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
     val df1 = Seq(("mon", "2015-07-23"), ("tuesday", "2015-07-20")).toDF("dow", "d")
     val df2 = Seq(("th", "2015-07-23 00:11:22"), ("xx", "2015-07-24 11:22:33")).toDF("dow", "t")
     checkAnswer(
-      df1.select(next_day(col("d"), "MONDAY")),
-      Seq(Row(Date.valueOf("2015-07-27")), Row(Date.valueOf("2015-07-27"))))
+      df1.select(
+        next_day(col("d"), "MONDAY"),
+        next_day(col("d"), col("dow")),
+        next_day(col("d"), "NonValidDay")),
+      Seq(
+        Row(Date.valueOf("2015-07-27"), Date.valueOf("2015-07-27"), null),
+        Row(Date.valueOf("2015-07-27"), Date.valueOf("2015-07-21"), null)))
     checkAnswer(
-      df2.select(next_day(col("t"), "th")),
-      Seq(Row(Date.valueOf("2015-07-30")), Row(Date.valueOf("2015-07-30"))))
+      df2.select(
+        next_day(col("t"), "th"),
+        next_day(col("t"), col("dow")),
+        next_day(col("t"), "NonValidDay")),
+      Seq(
+        Row(Date.valueOf("2015-07-30"), Date.valueOf("2015-07-30"), null),
+        Row(Date.valueOf("2015-07-30"), null, null)))
   }
 
   def checkExceptionMessage(df: DataFrame): Unit = {
@@ -450,11 +500,11 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df.select(to_date(col("s"), "yyyy-hh-MM")),
       Seq(Row(null), Row(null), Row(null)))
-    checkAnswer(
-      df.select(to_date(col("s"), "yyyy-dd-aa")),
-      Seq(Row(null), Row(null), Row(null)))
+    val e = intercept[SparkUpgradeException](df.select(to_date(col("s"), "yyyy-dd-aa")).collect())
+    assert(e.getCause.isInstanceOf[IllegalArgumentException])
+    assert(e.getMessage.contains("You may get a different result due to the upgrading of Spark"))
 
-    // february
+    // February
     val x1 = "2016-02-29"
     val x2 = "2017-02-29"
     val df1 = Seq(x1, x2).toDF("x")
@@ -618,10 +668,18 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
           Row(secs(ts4.getTime)), Row(null), Row(secs(ts3.getTime)), Row(null)))
 
         // invalid format
-        checkAnswer(df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd aa:HH:ss')"), Seq(
-          Row(null), Row(null), Row(null), Row(null)))
+        val invalid = df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd aa:HH:ss')")
+        if (legacyParserPolicy == "legacy") {
+          checkAnswer(invalid,
+            Seq(Row(null), Row(null), Row(null), Row(null)))
+        } else {
+          val e = intercept[SparkUpgradeException](invalid.collect())
+          assert(e.getCause.isInstanceOf[IllegalArgumentException])
+          assert(
+            e.getMessage.contains("You may get a different result due to the upgrading of Spark"))
+        }
 
-        // february
+        // February
         val y1 = "2016-02-29"
         val y2 = "2017-02-29"
         val ts5 = Timestamp.valueOf("2016-02-29 00:00:00")
@@ -631,7 +689,7 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
 
         val now = sql("select unix_timestamp()").collect().head.getLong(0)
         checkAnswer(
-          sql(s"select cast ($now as timestamp)"),
+          sql(s"select timestamp_seconds($now)"),
           Row(new java.util.Date(TimeUnit.SECONDS.toMillis(now))))
       }
     }
@@ -672,7 +730,7 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
         checkAnswer(df1.selectExpr(s"to_unix_timestamp(x, 'yyyy-MM-dd mm:HH:ss')"), Seq(
           Row(secs(ts4.getTime)), Row(null), Row(secs(ts3.getTime)), Row(null)))
 
-        // february
+        // February
         val y1 = "2016-02-29"
         val y2 = "2017-02-29"
         val ts5 = Timestamp.valueOf("2016-02-29 00:00:00")
@@ -681,8 +739,9 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
           Row(secs(ts5.getTime)), Row(null)))
 
         // invalid format
-        checkAnswer(df1.selectExpr(s"to_unix_timestamp(x, 'yyyy-MM-dd bb:HH:ss')"), Seq(
-          Row(null), Row(null), Row(null), Row(null)))
+        val invalid = df1.selectExpr(s"to_unix_timestamp(x, 'yyyy-MM-dd bb:HH:ss')")
+        val e = intercept[IllegalArgumentException](invalid.collect())
+        assert(e.getMessage.contains('b'))
       }
     }
   }
@@ -707,7 +766,7 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
         val df = Seq((date1, ts1, s1, ss1), (date2, ts2, s2, ss2)).toDF("d", "ts", "s", "ss")
 
         checkAnswer(df.select(to_timestamp(col("ss"))),
-          df.select(unix_timestamp(col("ss")).cast("timestamp")))
+          df.select(timestamp_seconds(unix_timestamp(col("ss")))))
         checkAnswer(df.select(to_timestamp(col("ss"))), Seq(
           Row(ts1), Row(ts2)))
         if (legacyParserPolicy == "legacy") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
index cdf9ea4b31ee7..3b88bd58d925f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
@@ -20,10 +20,11 @@ package org.apache.spark.sql
 import org.scalatest.GivenWhenThen
 
 import org.apache.spark.sql.catalyst.expressions.{DynamicPruningExpression, Expression}
+import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode._
 import org.apache.spark.sql.catalyst.plans.ExistenceJoin
 import org.apache.spark.sql.execution._
-import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AdaptiveSparkPlanHelper}
-import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ReusedExchangeExec}
+import org.apache.spark.sql.execution.adaptive._
+import org.apache.spark.sql.execution.exchange.{BroadcastExchangeLike, ReusedExchangeExec}
 import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec
 import org.apache.spark.sql.execution.streaming.{MemoryStream, StreamingQueryWrapper}
 import org.apache.spark.sql.functions._
@@ -43,14 +44,9 @@ abstract class DynamicPartitionPruningSuiteBase
 
   import testImplicits._
 
-  val adaptiveExecutionOn: Boolean
-
   override def beforeAll(): Unit = {
     super.beforeAll()
 
-    spark.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED, adaptiveExecutionOn)
-    spark.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_FORCE_APPLY, true)
-
     val factData = Seq[(Int, Int, Int, Int)](
       (1000, 1, 1, 10),
       (1010, 2, 1, 10),
@@ -172,6 +168,8 @@ abstract class DynamicPartitionPruningSuiteBase
       df: DataFrame,
       withSubquery: Boolean,
       withBroadcast: Boolean): Unit = {
+    df.collect()
+
     val plan = df.queryExecution.executedPlan
     val dpExprs = collectDynamicPruningExpressions(plan)
     val hasSubquery = dpExprs.exists {
@@ -192,12 +190,24 @@ abstract class DynamicPartitionPruningSuiteBase
     subqueryBroadcast.foreach { s =>
       s.child match {
         case _: ReusedExchangeExec => // reuse check ok.
-        case b: BroadcastExchangeExec =>
+        case BroadcastQueryStageExec(_, _: ReusedExchangeExec, _) => // reuse check ok.
+        case b: BroadcastExchangeLike =>
           val hasReuse = plan.find {
             case ReusedExchangeExec(_, e) => e eq b
             case _ => false
           }.isDefined
           assert(hasReuse, s"$s\nshould have been reused in\n$plan")
+        case a: AdaptiveSparkPlanExec =>
+          val broadcastQueryStage = collectFirst(a) {
+            case b: BroadcastQueryStageExec => b
+          }
+          val broadcastPlan = broadcastQueryStage.get.broadcast
+          val hasReuse = find(plan) {
+            case ReusedExchangeExec(_, e) => e eq broadcastPlan
+            case b: BroadcastExchangeLike => b eq broadcastPlan
+            case _ => false
+          }.isDefined
+          assert(hasReuse, s"$s\nshould have been reused in\n$plan")
         case _ =>
           fail(s"Invalid child node found in\n$s")
       }
@@ -205,7 +215,11 @@ abstract class DynamicPartitionPruningSuiteBase
 
     val isMainQueryAdaptive = plan.isInstanceOf[AdaptiveSparkPlanExec]
     subqueriesAll(plan).filterNot(subqueryBroadcast.contains).foreach { s =>
-      assert(s.find(_.isInstanceOf[AdaptiveSparkPlanExec]).isDefined == isMainQueryAdaptive)
+      val subquery = s match {
+        case r: ReusedSubqueryExec => r.child
+        case o => o
+      }
+      assert(subquery.find(_.isInstanceOf[AdaptiveSparkPlanExec]).isDefined == isMainQueryAdaptive)
     }
   }
 
@@ -213,6 +227,8 @@ abstract class DynamicPartitionPruningSuiteBase
    * Check if the plan has the given number of distinct broadcast exchange subqueries.
    */
   def checkDistinctSubqueries(df: DataFrame, n: Int): Unit = {
+    df.collect()
+
     val buf = collectDynamicPruningExpressions(df.queryExecution.executedPlan).collect {
       case InSubqueryExec(_, b: SubqueryBroadcastExec, _, _) =>
         b.index
@@ -224,7 +240,7 @@ abstract class DynamicPartitionPruningSuiteBase
    * Collect the children of all correctly pushed down dynamic pruning expressions in a spark plan.
    */
   private def collectDynamicPruningExpressions(plan: SparkPlan): Seq[Expression] = {
-    plan.flatMap {
+    flatMap(plan) {
       case s: FileSourceScanExec => s.partitionFilters.collect {
         case d: DynamicPruningExpression => d.child
       }
@@ -236,7 +252,7 @@ abstract class DynamicPartitionPruningSuiteBase
    * Check if the plan contains unpushed dynamic pruning filters.
    */
   def checkUnpushedFilters(df: DataFrame): Boolean = {
-    df.queryExecution.executedPlan.find {
+    find(df.queryExecution.executedPlan) {
       case FilterExec(condition, _) =>
         splitConjunctivePredicates(condition).exists {
           case _: DynamicPruningExpression => true
@@ -249,33 +265,20 @@ abstract class DynamicPartitionPruningSuiteBase
   /**
    * Test the result of a simple join on mock-up tables
    */
-  test("simple inner join triggers DPP with mock-up tables") {
+  test("simple inner join triggers DPP with mock-up tables",
+    DisableAdaptiveExecution("DPP in AQE must reuse broadcast")) {
     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
       SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
       SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
-      withTable("df1", "df2") {
-        spark.range(1000)
-          .select(col("id"), col("id").as("k"))
-          .write
-          .partitionBy("k")
-          .format(tableFormat)
-          .mode("overwrite")
-          .saveAsTable("df1")
-
-        spark.range(100)
-          .select(col("id"), col("id").as("k"))
-          .write
-          .partitionBy("k")
-          .format(tableFormat)
-          .mode("overwrite")
-          .saveAsTable("df2")
-
-        val df = sql("SELECT df1.id, df2.k FROM df1 JOIN df2 ON df1.k = df2.k AND df2.id < 2")
+      val df = sql(
+        """
+          |SELECT f.date_id, f.store_id FROM fact_sk f
+          |JOIN dim_store s ON f.store_id = s.store_id AND s.country = 'NL'
+        """.stripMargin)
 
-        checkPartitionPruningPredicate(df, true, false)
+      checkPartitionPruningPredicate(df, true, false)
 
-        checkAnswer(df, Row(0, 0) :: Row(1, 1) :: Nil)
-      }
+      checkAnswer(df, Row(1000, 1) :: Row(1010, 2) :: Row(1020, 2) :: Nil)
     }
   }
 
@@ -314,11 +317,14 @@ abstract class DynamicPartitionPruningSuiteBase
   /**
    * Check the static scan metrics with and without DPP
    */
-  test("static scan metrics") {
+  test("static scan metrics",
+    DisableAdaptiveExecution("DPP in AQE must reuse broadcast")) {
     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
       SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
       SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
       withTable("fact", "dim") {
+        val numPartitions = 10
+
         spark.range(10)
           .map { x => Tuple3(x, x + 1, 0) }
           .toDF("did", "d1", "d2")
@@ -328,7 +334,7 @@ abstract class DynamicPartitionPruningSuiteBase
           .saveAsTable("dim")
 
         spark.range(100)
-          .map { x => Tuple2(x, x % 10) }
+          .map { x => Tuple2(x, x % numPartitions) }
           .toDF("f1", "fid")
           .write.partitionBy("fid")
           .format(tableFormat)
@@ -355,6 +361,8 @@ abstract class DynamicPartitionPruningSuiteBase
         assert(!scan1.metrics.contains("staticFilesSize"))
         val allFilesNum = scan1.metrics("numFiles").value
         val allFilesSize = scan1.metrics("filesSize").value
+        assert(scan1.metrics("numPartitions").value === numPartitions)
+        assert(scan1.metrics("pruningTime").value === -1)
 
         // No dynamic partition pruning, so no static metrics
         // Only files from fid = 5 partition are scanned
@@ -367,6 +375,8 @@ abstract class DynamicPartitionPruningSuiteBase
         val partFilesSize = scan2.metrics("filesSize").value
         assert(0 < partFilesNum && partFilesNum < allFilesNum)
         assert(0 < partFilesSize && partFilesSize < allFilesSize)
+        assert(scan2.metrics("numPartitions").value === 1)
+        assert(scan2.metrics("pruningTime").value === -1)
 
         // Dynamic partition pruning is used
         // Static metrics are as-if reading the whole fact table
@@ -378,6 +388,8 @@ abstract class DynamicPartitionPruningSuiteBase
         assert(scan3.metrics("staticFilesSize").value == allFilesSize)
         assert(scan3.metrics("numFiles").value == partFilesNum)
         assert(scan3.metrics("filesSize").value == partFilesSize)
+        assert(scan3.metrics("numPartitions").value === 1)
+        assert(scan3.metrics("pruningTime").value !== -1)
       }
     }
   }
@@ -396,7 +408,7 @@ abstract class DynamicPartitionPruningSuiteBase
        """.stripMargin)
 
       val found = df.queryExecution.executedPlan.find {
-        case BroadcastHashJoinExec(_, _, p: ExistenceJoin, _, _, _, _) => true
+        case BroadcastHashJoinExec(_, _, p: ExistenceJoin, _, _, _, _, _) => true
         case _ => false
       }
 
@@ -409,9 +421,11 @@ abstract class DynamicPartitionPruningSuiteBase
    * (2) DPP should be triggered only for certain join types
    * (3) DPP should trigger only when we have attributes on both sides of the join condition
    */
-  test("DPP triggers only for certain types of query") {
+  test("DPP triggers only for certain types of query",
+    DisableAdaptiveExecution("DPP in AQE must reuse broadcast")) {
     withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false") {
+      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
+      SQLConf.DYNAMIC_PARTITION_PRUNING_PRUNING_SIDE_EXTRA_FILTER_RATIO.key -> "1") {
       Given("dynamic partition pruning disabled")
       withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "false") {
         val df = sql(
@@ -461,6 +475,19 @@ abstract class DynamicPartitionPruningSuiteBase
         checkPartitionPruningPredicate(df, true, false)
       }
 
+      Given("left-semi join with partition column on the right side")
+      withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
+        SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
+        val df = sql(
+          """
+            |SELECT * FROM dim_store s
+            |LEFT SEMI JOIN fact_sk f
+            |ON f.store_id = s.store_id AND s.country = 'NL'
+          """.stripMargin)
+
+        checkPartitionPruningPredicate(df, true, false)
+      }
+
       Given("left outer with partition column on the left side")
       withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") {
         val df = sql(
@@ -490,7 +517,8 @@ abstract class DynamicPartitionPruningSuiteBase
   /**
    * The filtering policy has a fallback when the stats are unavailable
    */
-  test("filtering ratio policy fallback") {
+  test("filtering ratio policy fallback",
+    DisableAdaptiveExecution("DPP in AQE must reuse broadcast")) {
     withSQLConf(
       SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
       SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
@@ -560,7 +588,8 @@ abstract class DynamicPartitionPruningSuiteBase
   /**
    *  The filtering ratio policy performs best when it uses cardinality estimates
    */
-  test("filtering ratio policy with stats when the broadcast pruning is disabled") {
+  test("filtering ratio policy with stats when the broadcast pruning is disabled",
+    DisableAdaptiveExecution("DPP in AQE must reuse broadcast")) {
     withSQLConf(
       SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
       SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
@@ -753,7 +782,8 @@ abstract class DynamicPartitionPruningSuiteBase
     }
   }
 
-  test("partition pruning in broadcast hash joins") {
+  test("partition pruning in broadcast hash joins",
+    DisableAdaptiveExecution("DPP in AQE must reuse broadcast")) {
     Given("disable broadcast pruning and disable subquery duplication")
     withSQLConf(
       SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true",
@@ -1060,7 +1090,8 @@ abstract class DynamicPartitionPruningSuiteBase
     }
   }
 
-  test("avoid reordering broadcast join keys to match input hash partitioning") {
+  test("avoid reordering broadcast join keys to match input hash partitioning",
+    DisableAdaptiveExecution("DPP in AQE must reuse broadcast")) {
     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
       withTable("large", "dimTwo", "dimThree") {
@@ -1151,7 +1182,7 @@ abstract class DynamicPartitionPruningSuiteBase
   test("cleanup any DPP filter that isn't pushed down due to expression id clashes") {
     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       withTable("fact", "dim") {
-        spark.range(1000).select($"id".as("A"), $"id".as("AA"))
+        spark.range(20).select($"id".as("A"), $"id".as("AA"))
           .write.partitionBy("A").format(tableFormat).mode("overwrite").saveAsTable("fact")
         spark.range(10).select($"id".as("B"), $"id".as("BB"))
           .write.format(tableFormat).mode("overwrite").saveAsTable("dim")
@@ -1184,7 +1215,8 @@ abstract class DynamicPartitionPruningSuiteBase
     }
   }
 
-  test("join key with multiple references on the filtering plan") {
+  test("join key with multiple references on the filtering plan",
+    DisableAdaptiveExecution("DPP in AQE must reuse broadcast")) {
     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       // when enable AQE, the reusedExchange is inserted when executed.
       withTable("fact", "dim") {
@@ -1234,9 +1266,42 @@ abstract class DynamicPartitionPruningSuiteBase
 
       val plan = df.queryExecution.executedPlan
       val countSubqueryBroadcasts =
-        plan.collectWithSubqueries({ case _: SubqueryBroadcastExec => 1 }).sum
+        collectWithSubqueries(plan)({ case _: SubqueryBroadcastExec => 1 }).sum
+
+      if (conf.getConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED)) {
+        val countReusedSubqueryBroadcasts =
+          collectWithSubqueries(plan)({ case ReusedSubqueryExec(_: SubqueryBroadcastExec) => 1}).sum
+
+        assert(countSubqueryBroadcasts == 1)
+        assert(countReusedSubqueryBroadcasts == 1)
+      } else {
+        assert(countSubqueryBroadcasts == 2)
+      }
+    }
+  }
+
+  test("SPARK-32509: Unused Dynamic Pruning filter shouldn't affect " +
+    "canonicalization and exchange reuse",
+    DisableAdaptiveExecution("DPP in AQE must reuse broadcast")) {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
+      withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+        val df = sql(
+          """ WITH view1 as (
+            |   SELECT f.store_id FROM fact_stats f WHERE f.units_sold = 70
+            | )
+            |
+            | SELECT * FROM view1 v1 join view1 v2 WHERE v1.store_id = v2.store_id
+          """.stripMargin)
 
-      assert(countSubqueryBroadcasts == 2)
+        checkPartitionPruningPredicate(df, false, false)
+        val reuseExchangeNodes = df.queryExecution.executedPlan.collect {
+          case se: ReusedExchangeExec => se
+        }
+        assert(reuseExchangeNodes.size == 1, "Expected plan to contain 1 ReusedExchangeExec " +
+          s"nodes. Found ${reuseExchangeNodes.size}")
+
+        checkAnswer(df, Row(15, 15) :: Nil)
+      }
     }
   }
 
@@ -1280,12 +1345,170 @@ abstract class DynamicPartitionPruningSuiteBase
       )
     }
   }
-}
 
-class DynamicPartitionPruningSuiteAEOff extends DynamicPartitionPruningSuiteBase {
-  override val adaptiveExecutionOn: Boolean = false
-}
+  test("SPARK-32659: Fix the data issue when pruning DPP on non-atomic type") {
+    Seq(NO_CODEGEN, CODEGEN_ONLY).foreach { mode =>
+      Seq(true, false).foreach { pruning =>
+        withSQLConf(
+          SQLConf.CODEGEN_FACTORY_MODE.key -> mode.toString,
+          SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> s"$pruning") {
+          Seq("struct", "array").foreach { dataType =>
+            val df = sql(
+              s"""
+                 |SELECT f.date_id, f.product_id, f.units_sold, f.store_id FROM fact_stats f
+                 |JOIN dim_stats s
+                 |ON $dataType(f.store_id) = $dataType(s.store_id) WHERE s.country = 'DE'
+              """.stripMargin)
+
+            if (pruning) {
+              checkPartitionPruningPredicate(df, false, true)
+            } else {
+              checkPartitionPruningPredicate(df, false, false)
+            }
+
+            checkAnswer(df,
+              Row(1030, 2, 10, 3) ::
+              Row(1040, 2, 50, 3) ::
+              Row(1050, 2, 50, 3) ::
+              Row(1060, 2, 50, 3) :: Nil
+            )
+          }
+        }
+      }
+    }
+  }
 
-class DynamicPartitionPruningSuiteAEOn extends DynamicPartitionPruningSuiteBase {
-  override val adaptiveExecutionOn: Boolean = true
+  test("SPARK-32817: DPP throws error when the broadcast side is empty") {
+    withSQLConf(
+      SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
+      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true",
+      SQLConf.ADAPTIVE_OPTIMIZER_EXCLUDED_RULES.key -> EliminateUnnecessaryJoin.ruleName) {
+      val df = sql(
+        """
+          |SELECT * FROM fact_sk f
+          |JOIN dim_store s
+          |ON f.store_id = s.store_id WHERE s.country = 'XYZ'
+        """.stripMargin)
+
+      checkPartitionPruningPredicate(df, false, true)
+
+      checkAnswer(df, Nil)
+    }
+  }
+
+  test("SPARK-34436: DPP support LIKE ANY/ALL expression") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") {
+      val df = sql(
+        """
+          |SELECT date_id, product_id FROM fact_sk f
+          |JOIN dim_store s
+          |ON f.store_id = s.store_id WHERE s.country LIKE ANY ('%D%E%', '%A%B%')
+        """.stripMargin)
+
+      checkPartitionPruningPredicate(df, false, true)
+
+      checkAnswer(df,
+        Row(1030, 2) ::
+        Row(1040, 2) ::
+        Row(1050, 2) ::
+        Row(1060, 2) :: Nil
+      )
+    }
+  }
+
+  test("SPARK-34595: DPP support RLIKE expression") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") {
+      val df = sql(
+        """
+          |SELECT date_id, product_id FROM fact_sk f
+          |JOIN dim_store s
+          |ON f.store_id = s.store_id WHERE s.country RLIKE '[DE|US]'
+        """.stripMargin)
+
+      checkPartitionPruningPredicate(df, false, true)
+
+      checkAnswer(df,
+        Row(1030, 2) ::
+        Row(1040, 2) ::
+        Row(1050, 2) ::
+        Row(1060, 2) ::
+        Row(1070, 2) ::
+        Row(1080, 3) ::
+        Row(1090, 3) ::
+        Row(1100, 3) ::
+        Row(1110, 3) ::
+        Row(1120, 4) :: Nil
+      )
+    }
+  }
+
+  test("SPARK-32855: Filtering side can not broadcast by join type",
+    DisableAdaptiveExecution("DPP in AQE must reuse broadcast")) {
+    withSQLConf(
+      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
+      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
+      SQLConf.DYNAMIC_PARTITION_PRUNING_PRUNING_SIDE_EXTRA_FILTER_RATIO.key -> "1") {
+
+      val sqlStr =
+        """
+          |SELECT s.store_id,f. product_id FROM dim_store s
+          |LEFT JOIN fact_sk f
+          |ON f.store_id = s.store_id WHERE s.country = 'NL'
+          """.stripMargin
+
+      // DPP will only apply if disable reuseBroadcastOnly
+      Seq(true, false).foreach { reuseBroadcastOnly =>
+        withSQLConf(
+          SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> s"$reuseBroadcastOnly") {
+          val df = sql(sqlStr)
+          checkPartitionPruningPredicate(df, !reuseBroadcastOnly, false)
+        }
+      }
+
+      // DPP will only apply if left side can broadcast by size
+      Seq(1L, 100000L).foreach { threshold =>
+        withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> s"$threshold") {
+          val df = sql(sqlStr)
+          checkPartitionPruningPredicate(df, threshold > 10L, false)
+        }
+      }
+    }
+  }
+
+  test("SPARK-34637: DPP side broadcast query stage is created firstly") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
+      val df = sql(
+        """ WITH v as (
+          |   SELECT f.store_id FROM fact_stats f WHERE f.units_sold = 70 group by f.store_id
+          | )
+          |
+          | SELECT * FROM v v1 join v v2 WHERE v1.store_id = v2.store_id
+        """.stripMargin)
+
+      // A possible resulting query plan:
+      // BroadcastHashJoin
+      // +- HashAggregate
+      //    +- ShuffleQueryStage
+      //       +- Exchange
+      //          +- HashAggregate
+      //             +- Filter
+      //                +- FileScan [PartitionFilters: dynamicpruning#3385]
+      //                     +- SubqueryBroadcast dynamicpruning#3385
+      //                        +- AdaptiveSparkPlan
+      //                           +- BroadcastQueryStage
+      //                              +- BroadcastExchange
+      //
+      // +- BroadcastQueryStage
+      //    +- ReusedExchange
+
+      checkPartitionPruningPredicate(df, false, true)
+      checkAnswer(df, Row(15, 15) :: Nil)
+    }
+  }
 }
+
+class DynamicPartitionPruningSuiteAEOff extends DynamicPartitionPruningSuiteBase
+  with DisableAdaptiveExecutionSuite
+
+class DynamicPartitionPruningSuiteAEOn extends DynamicPartitionPruningSuiteBase
+  with EnableAdaptiveExecutionSuite
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
index 5aeecd2df91e9..3edc40133c616 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
@@ -19,10 +19,12 @@ package org.apache.spark.sql
 
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.execution.adaptive.{DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
+import org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.sources.TestOptionsSource
 import org.apache.spark.sql.test.SharedSparkSession
-import org.apache.spark.sql.types.StructType
+import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
 
 trait ExplainSuiteHelper extends QueryTest with SharedSparkSession {
 
@@ -69,6 +71,18 @@ trait ExplainSuiteHelper extends QueryTest with SharedSparkSession {
   protected def checkKeywordsExistsInExplain(df: DataFrame, keywords: String*): Unit = {
     checkKeywordsExistsInExplain(df, ExtendedMode, keywords: _*)
   }
+
+  /**
+   * Runs the plan and makes sure the plans does not contain any of the keywords.
+   */
+  protected def checkKeywordsNotExistsInExplain(
+      df: DataFrame, mode: ExplainMode, keywords: String*): Unit = {
+    withNormalizedExplain(df, mode) { normalizedOutput =>
+      for (key <- keywords) {
+        assert(!normalizedOutput.contains(key))
+      }
+    }
+  }
 }
 
 class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite {
@@ -196,14 +210,13 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
     // OR                                                disjunction
     // ---------------------------------------------------------------------------------------
     checkKeywordsExistsInExplain(sql("select 'a' || 1 + 2"),
-      "Project [null AS (CAST(concat(a, CAST(1 AS STRING)) AS DOUBLE) + CAST(2 AS DOUBLE))#x]")
+      "Project [null AS (concat(a, 1) + 2)#x]")
     checkKeywordsExistsInExplain(sql("select 1 - 2 || 'b'"),
-      "Project [-1b AS concat(CAST((1 - 2) AS STRING), b)#x]")
+      "Project [-1b AS concat((1 - 2), b)#x]")
     checkKeywordsExistsInExplain(sql("select 2 * 4  + 3 || 'b'"),
-      "Project [11b AS concat(CAST(((2 * 4) + 3) AS STRING), b)#x]")
+      "Project [11b AS concat(((2 * 4) + 3), b)#x]")
     checkKeywordsExistsInExplain(sql("select 3 + 1 || 'a' || 4 / 2"),
-      "Project [4a2.0 AS concat(concat(CAST((3 + 1) AS STRING), a), " +
-        "CAST((CAST(4 AS DOUBLE) / CAST(2 AS DOUBLE)) AS STRING))#x]")
+      "Project [4a2.0 AS concat(concat((3 + 1), a), (4 / 2))#x]")
     checkKeywordsExistsInExplain(sql("select 1 == 1 OR 'a' || 'b' ==  'ab'"),
       "Project [true AS ((1 = 1) OR (concat(a, b) = ab))#x]")
     checkKeywordsExistsInExplain(sql("select 'a' || 'c' == 'ac' AND 2 == 3"),
@@ -228,12 +241,27 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
     }
   }
 
+  test("SPARK-33853: explain codegen - check presence of subquery") {
+    withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true") {
+      withTempView("df") {
+        val df1 = spark.range(1, 100)
+        df1.createTempView("df")
+
+        val sqlText = "EXPLAIN CODEGEN SELECT (SELECT min(id) FROM df)"
+        val expectedText = "Found 3 WholeStageCodegen subtrees."
+
+        withNormalizedExplain(sqlText) { normalizedOutput =>
+          assert(normalizedOutput.contains(expectedText))
+        }
+      }
+    }
+  }
+
   test("explain formatted - check presence of subquery in case of DPP") {
     withTable("df1", "df2") {
       withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
         SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
         SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
-        withTable("df1", "df2") {
           spark.range(1000).select(col("id"), col("id").as("k"))
             .write
             .partitionBy("k")
@@ -261,11 +289,11 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
             "PartitionFilters: \\[isnotnull\\(k#xL\\), dynamicpruningexpression\\(k#xL " +
               "IN subquery#x\\)\\]"
           val expected_pattern3 =
-            "Location: InMemoryFileIndex \\[.*org.apache.spark.sql.ExplainSuite" +
-              "/df2/.*, ... 99 entries\\]"
+            "Location: InMemoryFileIndex \\[\\S*org.apache.spark.sql.ExplainSuite" +
+              "/df2/\\S*, ... 99 entries\\]"
           val expected_pattern4 =
-            "Location: InMemoryFileIndex \\[.*org.apache.spark.sql.ExplainSuite" +
-              "/df1/.*, ... 999 entries\\]"
+            "Location: InMemoryFileIndex \\[\\S*org.apache.spark.sql.ExplainSuite" +
+              "/df1/\\S*, ... 999 entries\\]"
           withNormalizedExplain(sqlText) { normalizedOutput =>
             assert(expected_pattern1.r.findAllMatchIn(normalizedOutput).length == 1)
             assert(expected_pattern2.r.findAllMatchIn(normalizedOutput).length == 1)
@@ -273,6 +301,22 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
             assert(expected_pattern4.r.findAllMatchIn(normalizedOutput).length == 1)
           }
         }
+    }
+  }
+
+  test("SPARK-33850: explain formatted - check presence of subquery in case of AQE") {
+    withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") {
+      withTempView("df") {
+        val df = spark.range(1, 100)
+        df.createTempView("df")
+
+        val sqlText = "EXPLAIN FORMATTED SELECT (SELECT min(id) FROM df) as v"
+        val expected_pattern =
+          "Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#x"
+
+        withNormalizedExplain(sqlText) { normalizedOutput =>
+          assert(expected_pattern.r.findAllMatchIn(normalizedOutput).length == 1)
+        }
       }
     }
   }
@@ -316,6 +360,45 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
         Nil: _*)
   }
 
+  test("SPARK-34970: Redact Map type options in explain output") {
+    val password = "MyPassWord"
+    val token = "MyToken"
+    val value = "value"
+    val options = Map("password" -> password, "token" -> token, "key" -> value)
+    val cmd = SaveIntoDataSourceCommand(spark.range(10).logicalPlan, new TestOptionsSource,
+      options, SaveMode.Overwrite)
+
+    Seq(SimpleMode, ExtendedMode, FormattedMode).foreach { mode =>
+      checkKeywordsExistsInExplain(cmd, mode, value)
+    }
+    Seq(SimpleMode, ExtendedMode, CodegenMode, CostMode, FormattedMode).foreach { mode =>
+      checkKeywordsNotExistsInExplain(cmd, mode, password)
+      checkKeywordsNotExistsInExplain(cmd, mode, token)
+    }
+  }
+
+  test("SPARK-34970: Redact CaseInsensitiveMap type options in explain output") {
+    val password = "MyPassWord"
+    val token = "MyToken"
+    val value = "value"
+    val tableName = "t"
+    withTable(tableName) {
+      val df1 = spark.range(10).toDF()
+      df1.write.format("json").saveAsTable(tableName)
+      val df2 = spark.read
+        .option("key", value)
+        .option("password", password)
+        .option("token", token)
+        .table(tableName)
+
+      checkKeywordsExistsInExplain(df2, ExtendedMode, value)
+      Seq(SimpleMode, ExtendedMode, CodegenMode, CostMode, FormattedMode).foreach { mode =>
+        checkKeywordsNotExistsInExplain(df2, mode, password)
+        checkKeywordsNotExistsInExplain(df2, mode, token)
+      }
+    }
+  }
+
   test("Dataset.toExplainString has mode as string") {
     val df = spark.range(10).toDF
     def assertExplainOutput(mode: ExplainMode): Unit = {
@@ -343,6 +426,94 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
       assert(getNormalizedExplain(df1, FormattedMode) === getNormalizedExplain(df2, FormattedMode))
     }
   }
+
+  test("Coalesced bucket info should be a part of explain string") {
+    withTable("t1", "t2") {
+      withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "0",
+        SQLConf.COALESCE_BUCKETS_IN_JOIN_ENABLED.key -> "true") {
+        Seq(1, 2).toDF("i").write.bucketBy(8, "i").saveAsTable("t1")
+        Seq(2, 3).toDF("i").write.bucketBy(4, "i").saveAsTable("t2")
+        val df1 = spark.table("t1")
+        val df2 = spark.table("t2")
+        val joined = df1.join(df2, df1("i") === df2("i"))
+        checkKeywordsExistsInExplain(
+          joined,
+          SimpleMode,
+          "SelectedBucketsCount: 8 out of 8 (Coalesced to 4)" :: Nil: _*)
+      }
+    }
+  }
+
+  test("Explain formatted output for scan operator for datasource V2") {
+    withTempDir { dir =>
+      Seq("parquet", "orc", "csv", "json").foreach { fmt =>
+        val basePath = dir.getCanonicalPath + "/" + fmt
+        val pushFilterMaps = Map (
+          "parquet" ->
+            "|PushedFilters: \\[IsNotNull\\(value\\), GreaterThan\\(value,2\\)\\]",
+          "orc" ->
+            "|PushedFilters: \\[.*\\(id\\), .*\\(value\\), .*\\(id,1\\), .*\\(value,2\\)\\]",
+          "csv" ->
+            "|PushedFilters: \\[IsNotNull\\(value\\), GreaterThan\\(value,2\\)\\]",
+          "json" ->
+            "|remove_marker"
+        )
+        val expected_plan_fragment1 =
+          s"""
+             |\\(1\\) BatchScan
+             |Output \\[2\\]: \\[value#x, id#x\\]
+             |DataFilters: \\[isnotnull\\(value#x\\), \\(value#x > 2\\)\\]
+             |Format: $fmt
+             |Location: InMemoryFileIndex\\([0-9]+ paths\\)\\[.*\\]
+             |PartitionFilters: \\[isnotnull\\(id#x\\), \\(id#x > 1\\)\\]
+             ${pushFilterMaps.get(fmt).get}
+             |ReadSchema: struct\\<value:int\\>
+             |""".stripMargin.replaceAll("\nremove_marker", "").trim
+
+        spark.range(10)
+          .select(col("id"), col("id").as("value"))
+          .write.option("header", true)
+          .partitionBy("id")
+          .format(fmt)
+          .save(basePath)
+        val readSchema =
+          StructType(Seq(StructField("id", IntegerType), StructField("value", IntegerType)))
+        withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") {
+          val df = spark
+            .read
+            .schema(readSchema)
+            .option("header", true)
+            .format(fmt)
+            .load(basePath).where($"id" > 1 && $"value" > 2)
+          val normalizedOutput = getNormalizedExplain(df, FormattedMode)
+          assert(expected_plan_fragment1.r.findAllMatchIn(normalizedOutput).length == 1)
+        }
+      }
+    }
+  }
+
+  test("Explain UnresolvedRelation with CaseInsensitiveStringMap options") {
+    val tableName = "test"
+    withTable(tableName) {
+      val df1 = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("id", "data")
+      df1.write.saveAsTable(tableName)
+      val df2 = spark.read
+        .option("key1", "value1")
+        .option("KEY2", "VALUE2")
+        .table(tableName)
+      // == Parsed Logical Plan ==
+      // 'UnresolvedRelation [test], [key1=value1, KEY2=VALUE2]
+      checkKeywordsExistsInExplain(df2, keywords = "[key1=value1, KEY2=VALUE2]")
+    }
+  }
+
+  test("SPARK-35225: Handle empty output for analyzed plan") {
+    withTempView("test") {
+      checkKeywordsExistsInExplain(
+        sql("CREATE TEMPORARY VIEW test AS SELECT 1"),
+        "== Analyzed Logical Plan ==\nCreateViewCommand")
+    }
+  }
 }
 
 class ExplainSuiteAE extends ExplainSuiteHelper with EnableAdaptiveExecutionSuite {
@@ -394,6 +565,33 @@ class ExplainSuiteAE extends ExplainSuiteHelper with EnableAdaptiveExecutionSuit
          |""".stripMargin
     )
   }
+
+  test("SPARK-35133: explain codegen should work with AQE") {
+    withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true") {
+      withTempView("df") {
+        val df = spark.range(5).select(col("id").as("key"), col("id").as("value"))
+        df.createTempView("df")
+
+        val sqlText = "EXPLAIN CODEGEN SELECT key, MAX(value) FROM df GROUP BY key"
+        val expectedCodegenText = "Found 2 WholeStageCodegen subtrees."
+        val expectedNoCodegenText = "Found 0 WholeStageCodegen subtrees."
+        withNormalizedExplain(sqlText) { normalizedOutput =>
+          assert(normalizedOutput.contains(expectedNoCodegenText))
+        }
+
+        val aggDf = df.groupBy('key).agg(max('value))
+        withNormalizedExplain(aggDf, CodegenMode) { normalizedOutput =>
+          assert(normalizedOutput.contains(expectedNoCodegenText))
+        }
+
+        // trigger the final plan for AQE
+        aggDf.collect()
+        withNormalizedExplain(aggDf, CodegenMode) { normalizedOutput =>
+          assert(normalizedOutput.contains(expectedCodegenText))
+        }
+      }
+    }
+  }
 }
 
 case class ExplainSingleData(id: Int)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala
index 4c9ba9455c33f..f8071e6cda175 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala
@@ -32,12 +32,12 @@ import org.apache.spark.tags.ExtendedSQLTest
  *
  * To run the entire test suite:
  * {{{
- *   build/sbt "sql/test-only *ExpressionsSchemaSuite"
+ *   build/sbt "sql/testOnly *ExpressionsSchemaSuite"
  * }}}
  *
  * To re-generate golden files for entire suite, run:
  * {{{
- *   SPARK_GENERATE_GOLDEN_FILES=1 build/sbt "sql/test-only *ExpressionsSchemaSuite"
+ *   SPARK_GENERATE_GOLDEN_FILES=1 build/sbt "sql/testOnly *ExpressionsSchemaSuite"
  * }}}
  *
  * For example:
@@ -72,14 +72,7 @@ class ExpressionsSchemaSuite extends QueryTest with SharedSparkSession {
     // We use a path based on Spark home for 2 reasons:
     //   1. Maven can't get correct resource directory when resources in other jars.
     //   2. We test subclasses in the hive-thriftserver module.
-    val sparkHome = {
-      assert(sys.props.contains("spark.test.home") ||
-        sys.env.contains("SPARK_HOME"), "spark.test.home or SPARK_HOME is not set.")
-      sys.props.getOrElse("spark.test.home", sys.env("SPARK_HOME"))
-    }
-
-    java.nio.file.Paths.get(sparkHome,
-      "sql", "core", "src", "test", "resources", "sql-functions").toFile
+    getWorkspaceFilePath("sql", "core", "src", "test", "resources", "sql-functions").toFile
   }
 
   private val resultFile = new File(baseResourcePath, "sql-expression-schema.md")
@@ -101,7 +94,9 @@ class ExpressionsSchemaSuite extends QueryTest with SharedSparkSession {
       spark.sessionState.catalog.lookupFunctionInfo(funcId)
     }
 
-    val classFunsMap = funInfos.groupBy(_.getClassName).toSeq.sortBy(_._1)
+    val classFunsMap = funInfos.groupBy(_.getClassName).toSeq.sortBy(_._1).map {
+      case (className, infos) => (className, infos.sortBy(_.getName))
+    }
     val outputBuffer = new ArrayBuffer[String]
     val outputs = new ArrayBuffer[QueryOutput]
     val missingExamples = new ArrayBuffer[String]
@@ -158,16 +153,17 @@ class ExpressionsSchemaSuite extends QueryTest with SharedSparkSession {
 
     val outputSize = outputs.size
     val headerSize = header.size
-    val expectedOutputs: Seq[QueryOutput] = {
+    val (expectedMissingExamples, expectedOutputs) = {
       val expectedGoldenOutput = fileToString(resultFile)
       val lines = expectedGoldenOutput.split("\n")
       val expectedSize = lines.size
 
       assert(expectedSize == outputSize + headerSize,
         s"Expected $expectedSize blocks in result file but got " +
-          s"${outputSize + headerSize}. Try regenerate the result files.")
+          s"${outputSize + headerSize}. Try regenerating the result files.")
 
-      Seq.tabulate(outputSize) { i =>
+      val numberOfQueries = lines(2).split(":")(1).trim.toInt
+      val expectedOutputs = Seq.tabulate(outputSize) { i =>
         val segments = lines(i + headerSize).split('|')
         QueryOutput(
           className = segments(1).trim,
@@ -175,6 +171,28 @@ class ExpressionsSchemaSuite extends QueryTest with SharedSparkSession {
           sql = segments(3).trim,
           schema = segments(4).trim)
       }
+
+      assert(numberOfQueries == expectedOutputs.size,
+        s"expected outputs size: ${expectedOutputs.size} not same as numberOfQueries: " +
+          s"$numberOfQueries record in result file. Try regenerating the result files.")
+
+      val numberOfMissingExamples = lines(3).split(":")(1).trim.toInt
+      val expectedMissingExamples = {
+        val missingExamples = lines(4).split(":")(1).trim
+        // Splitting on a empty string would return [""]
+        if (missingExamples.nonEmpty) {
+          missingExamples.split(",")
+        } else {
+          Array.empty[String]
+        }
+      }
+
+      assert(numberOfMissingExamples == expectedMissingExamples.size,
+        s"expected missing examples size: ${expectedMissingExamples.size} not same as " +
+          s"numberOfMissingExamples: $numberOfMissingExamples " +
+          "record in result file. Try regenerating the result files.")
+
+      (expectedMissingExamples, expectedOutputs)
     }
 
     // Compare results.
@@ -185,5 +203,13 @@ class ExpressionsSchemaSuite extends QueryTest with SharedSparkSession {
       assert(expected.sql == output.sql, "SQL query did not match")
       assert(expected.schema == output.schema, s"Schema did not match for query ${expected.sql}")
     }
+
+    // Compare expressions missing examples
+    assert(expectedMissingExamples.length == missingExamples.size,
+      "The number of missing examples not equals the number of expected missing examples.")
+
+    missingExamples.zip(expectedMissingExamples).foreach { case (output, expected) =>
+      assert(expected == output, "Missing example expression not match")
+    }
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExtraStrategiesSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExtraStrategiesSuite.scala
index 9192370cfa620..bec68fae08719 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ExtraStrategiesSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExtraStrategiesSuite.scala
@@ -21,10 +21,10 @@ import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project}
-import org.apache.spark.sql.execution.SparkPlan
+import org.apache.spark.sql.execution.{LeafExecNode, SparkPlan}
 import org.apache.spark.sql.test.SharedSparkSession
 
-case class FastOperator(output: Seq[Attribute]) extends SparkPlan {
+case class FastOperator(output: Seq[Attribute]) extends LeafExecNode {
 
   override protected def doExecute(): RDD[InternalRow] = {
     val str = Literal("so fast").value
@@ -35,7 +35,6 @@ case class FastOperator(output: Seq[Attribute]) extends SparkPlan {
   }
 
   override def producedAttributes: AttributeSet = outputSet
-  override def children: Seq[SparkPlan] = Nil
 }
 
 object TestStrategy extends Strategy {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
index cb410b4f0d7dc..876f62803dc7c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
@@ -18,23 +18,27 @@
 package org.apache.spark.sql
 
 import java.io.{File, FileNotFoundException}
+import java.net.URI
 import java.nio.file.{Files, StandardOpenOption}
 import java.util.Locale
 
 import scala.collection.mutable
 
-import org.apache.hadoop.fs.Path
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.{LocalFileSystem, Path}
 
 import org.apache.spark.SparkException
 import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd}
 import org.apache.spark.sql.TestingUDT.{IntervalUDT, NullData, NullUDT}
 import org.apache.spark.sql.catalyst.expressions.AttributeReference
-import org.apache.spark.sql.catalyst.planning.PhysicalOperation
+import org.apache.spark.sql.catalyst.expressions.IntegralLiteralTestUtils.{negativeInt, positiveInt}
 import org.apache.spark.sql.catalyst.plans.logical.Filter
+import org.apache.spark.sql.execution.SimpleMode
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
 import org.apache.spark.sql.execution.datasources.FilePartition
-import org.apache.spark.sql.execution.datasources.v2.{BatchScanExec, DataSourceV2ScanRelation, FileScan}
-import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetTable
+import org.apache.spark.sql.execution.datasources.v2.{BatchScanExec, FileScan}
+import org.apache.spark.sql.execution.datasources.v2.orc.OrcScan
+import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetScan
 import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, SortMergeJoinExec}
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
@@ -229,6 +233,20 @@ class FileBasedDataSourceSuite extends QueryTest
     }
   }
 
+  Seq("json", "orc").foreach { format =>
+    test(s"SPARK-32889: column name supports special characters using $format") {
+      Seq("$", " ", ",", ";", "{", "}", "(", ")", "\n", "\t", "=").foreach { name =>
+        withTempDir { dir =>
+          val dataDir = new File(dir, "file").getCanonicalPath
+          Seq(1).toDF(name).write.format(format).save(dataDir)
+          val schema = spark.read.format(format).load(dataDir).schema
+          assert(schema.size == 1)
+          assertResult(name)(schema.head.name)
+        }
+      }
+    }
+  }
+
   // Text file format only supports string type
   test("SPARK-24691 error handling for unsupported types - text") {
     withTempDir { dir =>
@@ -559,38 +577,6 @@ class FileBasedDataSourceSuite extends QueryTest
     }
   }
 
-  test("Option pathGlobFilter: filter files correctly") {
-    withTempPath { path =>
-      val dataDir = path.getCanonicalPath
-      Seq("foo").toDS().write.text(dataDir)
-      Seq("bar").toDS().write.mode("append").orc(dataDir)
-      val df = spark.read.option("pathGlobFilter", "*.txt").text(dataDir)
-      checkAnswer(df, Row("foo"))
-
-      // Both glob pattern in option and path should be effective to filter files.
-      val df2 = spark.read.option("pathGlobFilter", "*.txt").text(dataDir + "/*.orc")
-      checkAnswer(df2, Seq.empty)
-
-      val df3 = spark.read.option("pathGlobFilter", "*.txt").text(dataDir + "/*xt")
-      checkAnswer(df3, Row("foo"))
-    }
-  }
-
-  test("Option pathGlobFilter: simple extension filtering should contains partition info") {
-    withTempPath { path =>
-      val input = Seq(("foo", 1), ("oof", 2)).toDF("a", "b")
-      input.write.partitionBy("b").text(path.getCanonicalPath)
-      Seq("bar").toDS().write.mode("append").orc(path.getCanonicalPath + "/b=1")
-
-      // If we use glob pattern in the path, the partition column won't be shown in the result.
-      val df = spark.read.text(path.getCanonicalPath + "/*/*.txt")
-      checkAnswer(df, input.select("a"))
-
-      val df2 = spark.read.option("pathGlobFilter", "*.txt").text(path.getCanonicalPath)
-      checkAnswer(df2, input)
-    }
-  }
-
   test("Option recursiveFileLookup: recursive loading correctly") {
 
     val expectedFileList = mutable.ListBuffer[String]()
@@ -631,13 +617,15 @@ class FileBasedDataSourceSuite extends QueryTest
 
       assert(fileList.toSet === expectedFileList.toSet)
 
-      val fileList2 = spark.read.format("binaryFile")
-        .option("recursiveFileLookup", true)
-        .option("pathGlobFilter", "*.bin")
-        .load(dataPath)
-        .select("path").collect().map(_.getString(0))
+      withClue("SPARK-32368: 'recursiveFileLookup' and 'pathGlobFilter' can be case insensitive") {
+        val fileList2 = spark.read.format("binaryFile")
+          .option("RecuRsivefileLookup", true)
+          .option("PaThglobFilter", "*.bin")
+          .load(dataPath)
+          .select("path").collect().map(_.getString(0))
 
-      assert(fileList2.toSet === expectedFileList.filter(_.endsWith(".bin")).toSet)
+        assert(fileList2.toSet === expectedFileList.filter(_.endsWith(".bin")).toSet)
+      }
     }
   }
 
@@ -822,27 +810,6 @@ class FileBasedDataSourceSuite extends QueryTest
     }
   }
 
-  test("File table location should include both values of option `path` and `paths`") {
-    withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") {
-      withTempPaths(3) { paths =>
-        paths.zipWithIndex.foreach { case (path, index) =>
-          Seq(index).toDF("a").write.mode("overwrite").parquet(path.getCanonicalPath)
-        }
-        val df = spark
-          .read
-          .option("path", paths.head.getCanonicalPath)
-          .parquet(paths(1).getCanonicalPath, paths(2).getCanonicalPath)
-        df.queryExecution.optimizedPlan match {
-          case PhysicalOperation(_, _, DataSourceV2ScanRelation(table: ParquetTable, _, _)) =>
-            assert(table.paths.toSet == paths.map(_.getCanonicalPath).toSet)
-          case _ =>
-            throw new AnalysisException("Can not match ParquetTable in the query.")
-        }
-        checkAnswer(df, Seq(0, 1, 2).map(Row(_)))
-      }
-    }
-  }
-
   test("SPARK-31116: Select nested schema with case insensitive mode") {
     // This test case failed at only Parquet. ORC is added for test coverage parity.
     Seq("orc", "parquet").foreach { format =>
@@ -882,6 +849,114 @@ class FileBasedDataSourceSuite extends QueryTest
       }
     }
   }
+
+  test("test casts pushdown on orc/parquet for integral types") {
+    def checkPushedFilters(
+        format: String,
+        df: DataFrame,
+        filters: Array[sources.Filter],
+        noScan: Boolean = false): Unit = {
+      val scanExec = df.queryExecution.sparkPlan.find(_.isInstanceOf[BatchScanExec])
+      if (noScan) {
+        assert(scanExec.isEmpty)
+        return
+      }
+      val scan = scanExec.get.asInstanceOf[BatchScanExec].scan
+      format match {
+        case "orc" =>
+          assert(scan.isInstanceOf[OrcScan])
+          assert(scan.asInstanceOf[OrcScan].pushedFilters === filters)
+        case "parquet" =>
+          assert(scan.isInstanceOf[ParquetScan])
+          assert(scan.asInstanceOf[ParquetScan].pushedFilters === filters)
+        case _ =>
+          fail(s"unknown format $format")
+      }
+    }
+
+    Seq("orc", "parquet").foreach { format =>
+      withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") {
+        withTempPath { dir =>
+          spark.range(100).map(i => (i.toShort, i.toString)).toDF("id", "s")
+            .write
+            .format(format)
+            .save(dir.getCanonicalPath)
+          val df = spark.read.format(format).load(dir.getCanonicalPath)
+
+          // cases when value == MAX
+          var v = Short.MaxValue
+          checkPushedFilters(format, df.where('id > v.toInt), Array(), noScan = true)
+          checkPushedFilters(format, df.where('id >= v.toInt), Array(sources.IsNotNull("id"),
+            sources.EqualTo("id", v)))
+          checkPushedFilters(format, df.where('id === v.toInt), Array(sources.IsNotNull("id"),
+            sources.EqualTo("id", v)))
+          checkPushedFilters(format, df.where('id <=> v.toInt),
+            Array(sources.EqualNullSafe("id", v)))
+          checkPushedFilters(format, df.where('id <= v.toInt), Array(sources.IsNotNull("id")))
+          checkPushedFilters(format, df.where('id < v.toInt), Array(sources.IsNotNull("id"),
+            sources.Not(sources.EqualTo("id", v))))
+
+          // cases when value > MAX
+          var v1: Int = positiveInt
+          checkPushedFilters(format, df.where('id > v1), Array(), noScan = true)
+          checkPushedFilters(format, df.where('id >= v1), Array(), noScan = true)
+          checkPushedFilters(format, df.where('id === v1), Array(), noScan = true)
+          checkPushedFilters(format, df.where('id <=> v1), Array(), noScan = true)
+          checkPushedFilters(format, df.where('id <= v1), Array(sources.IsNotNull("id")))
+          checkPushedFilters(format, df.where('id < v1), Array(sources.IsNotNull("id")))
+
+          // cases when value = MIN
+          v = Short.MinValue
+          checkPushedFilters(format, df.where(lit(v.toInt) < 'id), Array(sources.IsNotNull("id"),
+            sources.Not(sources.EqualTo("id", v))))
+          checkPushedFilters(format, df.where(lit(v.toInt) <= 'id), Array(sources.IsNotNull("id")))
+          checkPushedFilters(format, df.where(lit(v.toInt) === 'id), Array(sources.IsNotNull("id"),
+            sources.EqualTo("id", v)))
+          checkPushedFilters(format, df.where(lit(v.toInt) <=> 'id),
+            Array(sources.EqualNullSafe("id", v)))
+          checkPushedFilters(format, df.where(lit(v.toInt) >= 'id), Array(sources.IsNotNull("id"),
+            sources.EqualTo("id", v)))
+          checkPushedFilters(format, df.where(lit(v.toInt) > 'id), Array(), noScan = true)
+
+          // cases when value < MIN
+          v1 = negativeInt
+          checkPushedFilters(format, df.where(lit(v1) < 'id), Array(sources.IsNotNull("id")))
+          checkPushedFilters(format, df.where(lit(v1) <= 'id), Array(sources.IsNotNull("id")))
+          checkPushedFilters(format, df.where(lit(v1) === 'id), Array(), noScan = true)
+          checkPushedFilters(format, df.where(lit(v1) >= 'id), Array(), noScan = true)
+          checkPushedFilters(format, df.where(lit(v1) > 'id), Array(), noScan = true)
+
+          // cases when value is within range (MIN, MAX)
+          checkPushedFilters(format, df.where('id > 30), Array(sources.IsNotNull("id"),
+            sources.GreaterThan("id", 30)))
+          checkPushedFilters(format, df.where(lit(100) >= 'id), Array(sources.IsNotNull("id"),
+            sources.LessThanOrEqual("id", 100)))
+        }
+      }
+    }
+  }
+
+  test("SPARK-32827: Set max metadata string length") {
+    withTempDir { dir =>
+      val tableName = "t"
+      val path = s"${dir.getCanonicalPath}/$tableName"
+      withTable(tableName) {
+        sql(s"CREATE TABLE $tableName(c INT) USING PARQUET LOCATION '$path'")
+        withSQLConf(SQLConf.MAX_METADATA_STRING_LENGTH.key -> "5") {
+          val explain = spark.table(tableName).queryExecution.explainString(SimpleMode)
+          assert(!explain.contains(path))
+          // metadata has abbreviated by ...
+          assert(explain.contains("..."))
+        }
+
+        withSQLConf(SQLConf.MAX_METADATA_STRING_LENGTH.key -> "1000") {
+          val explain = spark.table(tableName).queryExecution.explainString(SimpleMode)
+          assert(explain.contains(path))
+          assert(!explain.contains("..."))
+        }
+      }
+    }
+  }
 }
 
 object TestingUDT {
@@ -912,3 +987,10 @@ object TestingUDT {
     override def userClass: Class[NullData] = classOf[NullData]
   }
 }
+
+class FakeFileSystemRequiringDSOption extends LocalFileSystem {
+  override def initialize(name: URI, conf: Configuration): Unit = {
+    super.initialize(name, conf)
+    require(conf.get("ds_option", "") == "value")
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/FileScanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/FileScanSuite.scala
new file mode 100644
index 0000000000000..4e7fe8455ff93
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/FileScanSuite.scala
@@ -0,0 +1,374 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import scala.collection.mutable
+
+import com.google.common.collect.ImmutableMap
+import org.apache.hadoop.fs.{FileStatus, Path}
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.expressions.{And, Expression, IsNull, LessThan}
+import org.apache.spark.sql.execution.datasources.{PartitioningAwareFileIndex, PartitionSpec}
+import org.apache.spark.sql.execution.datasources.v2.FileScan
+import org.apache.spark.sql.execution.datasources.v2.csv.CSVScan
+import org.apache.spark.sql.execution.datasources.v2.json.JsonScan
+import org.apache.spark.sql.execution.datasources.v2.orc.OrcScan
+import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetScan
+import org.apache.spark.sql.execution.datasources.v2.text.TextScan
+import org.apache.spark.sql.sources.Filter
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.StructType
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
+
+trait FileScanSuiteBase extends SharedSparkSession {
+  private def newPartitioningAwareFileIndex() = {
+    new PartitioningAwareFileIndex(spark, Map.empty, None) {
+      override def partitionSpec(): PartitionSpec = {
+        PartitionSpec.emptySpec
+      }
+
+      override protected def leafFiles: mutable.LinkedHashMap[Path, FileStatus] = {
+        mutable.LinkedHashMap.empty
+      }
+
+      override protected def leafDirToChildrenFiles: Map[Path, Array[FileStatus]] = {
+        Map.empty
+      }
+
+      override def rootPaths: Seq[Path] = {
+        Seq.empty
+      }
+
+      override def refresh(): Unit = {}
+    }
+  }
+
+  type ScanBuilder = (
+    SparkSession,
+      PartitioningAwareFileIndex,
+      StructType,
+      StructType,
+      StructType,
+      Array[Filter],
+      CaseInsensitiveStringMap,
+      Seq[Expression],
+      Seq[Expression]) => FileScan
+
+  def run(scanBuilders: Seq[(String, ScanBuilder, Seq[String])]): Unit = {
+    val dataSchema = StructType.fromDDL("data INT, partition INT, other INT")
+    val dataSchemaNotEqual = StructType.fromDDL("data INT, partition INT, other INT, new INT")
+    val readDataSchema = StructType.fromDDL("data INT")
+    val readDataSchemaNotEqual = StructType.fromDDL("data INT, other INT")
+    val readPartitionSchema = StructType.fromDDL("partition INT")
+    val readPartitionSchemaNotEqual = StructType.fromDDL("partition INT, other INT")
+    val pushedFilters =
+      Array[Filter](sources.And(sources.IsNull("data"), sources.LessThan("data", 0)))
+    val pushedFiltersNotEqual =
+      Array[Filter](sources.And(sources.IsNull("data"), sources.LessThan("data", 1)))
+    val optionsMap = ImmutableMap.of("key", "value")
+    val options = new CaseInsensitiveStringMap(ImmutableMap.copyOf(optionsMap))
+    val optionsNotEqual =
+      new CaseInsensitiveStringMap(ImmutableMap.copyOf(ImmutableMap.of("key2", "value2")))
+    val partitionFilters = Seq(And(IsNull('data.int), LessThan('data.int, 0)))
+    val partitionFiltersNotEqual = Seq(And(IsNull('data.int), LessThan('data.int, 1)))
+    val dataFilters = Seq(And(IsNull('data.int), LessThan('data.int, 0)))
+    val dataFiltersNotEqual = Seq(And(IsNull('data.int), LessThan('data.int, 1)))
+
+    scanBuilders.foreach { case (name, scanBuilder, exclusions) =>
+      test(s"SPARK-33482: Test $name equals") {
+        val partitioningAwareFileIndex = newPartitioningAwareFileIndex()
+
+        val scan = scanBuilder(
+          spark,
+          partitioningAwareFileIndex,
+          dataSchema,
+          readDataSchema,
+          readPartitionSchema,
+          pushedFilters,
+          options,
+          partitionFilters,
+          dataFilters)
+
+        val scanEquals = scanBuilder(
+          spark,
+          partitioningAwareFileIndex,
+          dataSchema.copy(),
+          readDataSchema.copy(),
+          readPartitionSchema.copy(),
+          pushedFilters.clone(),
+          new CaseInsensitiveStringMap(ImmutableMap.copyOf(optionsMap)),
+          Seq(partitionFilters: _*),
+          Seq(dataFilters: _*))
+
+        assert(scan === scanEquals)
+      }
+
+      test(s"SPARK-33482: Test $name fileIndex not equals") {
+        val partitioningAwareFileIndex = newPartitioningAwareFileIndex()
+
+        val scan = scanBuilder(
+          spark,
+          partitioningAwareFileIndex,
+          dataSchema,
+          readDataSchema,
+          readPartitionSchema,
+          pushedFilters,
+          options,
+          partitionFilters,
+          dataFilters)
+
+        val partitioningAwareFileIndexNotEqual = newPartitioningAwareFileIndex()
+
+        val scanNotEqual = scanBuilder(
+          spark,
+          partitioningAwareFileIndexNotEqual,
+          dataSchema,
+          readDataSchema,
+          readPartitionSchema,
+          pushedFilters,
+          options,
+          partitionFilters,
+          dataFilters)
+
+        assert(scan !== scanNotEqual)
+      }
+
+      if (!exclusions.contains("dataSchema")) {
+        test(s"SPARK-33482: Test $name dataSchema not equals") {
+          val partitioningAwareFileIndex = newPartitioningAwareFileIndex()
+
+          val scan = scanBuilder(
+            spark,
+            partitioningAwareFileIndex,
+            dataSchema,
+            readDataSchema,
+            readPartitionSchema,
+            pushedFilters,
+            options,
+            partitionFilters,
+            dataFilters)
+
+          val scanNotEqual = scanBuilder(
+            spark,
+            partitioningAwareFileIndex,
+            dataSchemaNotEqual,
+            readDataSchema,
+            readPartitionSchema,
+            pushedFilters,
+            options,
+            partitionFilters,
+            dataFilters)
+
+          assert(scan !== scanNotEqual)
+        }
+      }
+
+      test(s"SPARK-33482: Test $name readDataSchema not equals") {
+        val partitioningAwareFileIndex = newPartitioningAwareFileIndex()
+
+        val scan = scanBuilder(
+          spark,
+          partitioningAwareFileIndex,
+          dataSchema,
+          readDataSchema,
+          readPartitionSchema,
+          pushedFilters,
+          options,
+          partitionFilters,
+          dataFilters)
+
+        val scanNotEqual = scanBuilder(
+          spark,
+          partitioningAwareFileIndex,
+          dataSchema,
+          readDataSchemaNotEqual,
+          readPartitionSchema,
+          pushedFilters,
+          options,
+          partitionFilters,
+          dataFilters)
+
+        assert(scan !== scanNotEqual)
+      }
+
+      test(s"SPARK-33482: Test $name readPartitionSchema not equals") {
+        val partitioningAwareFileIndex = newPartitioningAwareFileIndex()
+
+        val scan = scanBuilder(
+          spark,
+          partitioningAwareFileIndex,
+          dataSchema,
+          readDataSchema,
+          readPartitionSchema,
+          pushedFilters,
+          options,
+          partitionFilters,
+          dataFilters)
+
+        val scanNotEqual = scanBuilder(
+          spark,
+          partitioningAwareFileIndex,
+          dataSchema,
+          readDataSchema,
+          readPartitionSchemaNotEqual,
+          pushedFilters,
+          options,
+          partitionFilters,
+          dataFilters)
+
+        assert(scan !== scanNotEqual)
+      }
+
+      if (!exclusions.contains("pushedFilters")) {
+        test(s"SPARK-33482: Test $name pushedFilters not equals") {
+          val partitioningAwareFileIndex = newPartitioningAwareFileIndex()
+
+          val scan = scanBuilder(
+            spark,
+            partitioningAwareFileIndex,
+            dataSchema,
+            readDataSchema,
+            readPartitionSchema,
+            pushedFilters,
+            options,
+            partitionFilters,
+            dataFilters)
+
+          val scanNotEqual = scanBuilder(
+            spark,
+            partitioningAwareFileIndex,
+            dataSchema,
+            readDataSchema,
+            readPartitionSchema,
+            pushedFiltersNotEqual,
+            options,
+            partitionFilters,
+            dataFilters)
+
+          assert(scan !== scanNotEqual)
+        }
+      }
+
+      test(s"SPARK-33482: Test $name options not equals") {
+        val partitioningAwareFileIndex = newPartitioningAwareFileIndex()
+
+        val scan = scanBuilder(
+          spark,
+          partitioningAwareFileIndex,
+          dataSchema,
+          readDataSchema,
+          readPartitionSchema,
+          pushedFilters,
+          options,
+          partitionFilters,
+          dataFilters)
+
+        val scanNotEqual = scanBuilder(
+          spark,
+          partitioningAwareFileIndex,
+          dataSchema,
+          readDataSchema,
+          readPartitionSchema,
+          pushedFilters,
+          optionsNotEqual,
+          partitionFilters,
+          dataFilters)
+
+        assert(scan !== scanNotEqual)
+      }
+
+      test(s"SPARK-33482: Test $name partitionFilters not equals") {
+        val partitioningAwareFileIndex = newPartitioningAwareFileIndex()
+
+        val scan = scanBuilder(
+          spark,
+          partitioningAwareFileIndex,
+          dataSchema,
+          readDataSchema,
+          readPartitionSchema,
+          pushedFilters,
+          options,
+          partitionFilters,
+          dataFilters)
+
+        val scanNotEqual = scanBuilder(
+          spark,
+          partitioningAwareFileIndex,
+          dataSchema,
+          readDataSchema,
+          readPartitionSchema,
+          pushedFilters,
+          options,
+          partitionFiltersNotEqual,
+          dataFilters)
+        assert(scan !== scanNotEqual)
+      }
+
+      test(s"SPARK-33482: Test $name dataFilters not equals") {
+        val partitioningAwareFileIndex = newPartitioningAwareFileIndex()
+
+        val scan = scanBuilder(
+          spark,
+          partitioningAwareFileIndex,
+          dataSchema,
+          readDataSchema,
+          readPartitionSchema,
+          pushedFilters,
+          options,
+          partitionFilters,
+          dataFilters)
+
+        val scanNotEqual = scanBuilder(
+          spark,
+          partitioningAwareFileIndex,
+          dataSchema,
+          readDataSchema,
+          readPartitionSchema,
+          pushedFilters,
+          options,
+          partitionFilters,
+          dataFiltersNotEqual)
+        assert(scan !== scanNotEqual)
+      }
+    }
+  }
+}
+
+class FileScanSuite extends FileScanSuiteBase {
+  val scanBuilders = Seq[(String, ScanBuilder, Seq[String])](
+    ("ParquetScan",
+      (s, fi, ds, rds, rps, f, o, pf, df) =>
+        ParquetScan(s, s.sessionState.newHadoopConf(), fi, ds, rds, rps, f, o, pf, df),
+      Seq.empty),
+    ("OrcScan",
+      (s, fi, ds, rds, rps, f, o, pf, df) =>
+        OrcScan(s, s.sessionState.newHadoopConf(), fi, ds, rds, rps, o, f, pf, df),
+      Seq.empty),
+    ("CSVScan",
+      (s, fi, ds, rds, rps, f, o, pf, df) => CSVScan(s, fi, ds, rds, rps, o, f, pf, df),
+      Seq.empty),
+    ("JsonScan",
+      (s, fi, ds, rds, rps, f, o, pf, df) => JsonScan(s, fi, ds, rds, rps, o, f, pf, df),
+      Seq.empty),
+    ("TextScan",
+      (s, fi, _, rds, rps, _, o, pf, df) => TextScan(s, fi, rds, rps, o, pf, df),
+      Seq("dataSchema", "pushedFilters")))
+
+  run(scanBuilders)
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/GenTPCDSData.scala b/sql/core/src/test/scala/org/apache/spark/sql/GenTPCDSData.scala
new file mode 100644
index 0000000000000..104ea6c9cfefd
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/GenTPCDSData.scala
@@ -0,0 +1,444 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import java.util.concurrent.LinkedBlockingQueue
+
+import scala.collection.immutable.Stream
+import scala.sys.process._
+import scala.util.Try
+
+import org.apache.spark.SparkContext
+import org.apache.spark.internal.Logging
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.functions.{col, rpad}
+import org.apache.spark.sql.types.{CharType, StringType, StructField, StructType, VarcharType}
+
+// The classes in this file are basically moved from https://github.com/databricks/spark-sql-perf
+
+/**
+ * Using ProcessBuilder.lineStream produces a stream, that uses
+ * a LinkedBlockingQueue with a default capacity of Integer.MAX_VALUE.
+ *
+ * This causes OOM if the consumer cannot keep up with the producer.
+ *
+ * See scala.sys.process.ProcessBuilderImpl.lineStream
+ */
+object BlockingLineStream {
+
+  // See scala.sys.process.Streamed
+  private final class BlockingStreamed[T](
+    val process: T => Unit,
+    val done: Int => Unit,
+    val stream: () => Stream[T])
+
+  // See scala.sys.process.Streamed
+  private object BlockingStreamed {
+    // scala.process.sys.Streamed uses default of Integer.MAX_VALUE,
+    // which causes OOMs if the consumer cannot keep up with producer.
+    val maxQueueSize = 65536
+
+    def apply[T](nonzeroException: Boolean): BlockingStreamed[T] = {
+      val q = new LinkedBlockingQueue[Either[Int, T]](maxQueueSize)
+
+      def next(): Stream[T] = q.take match {
+        case Left(0) => Stream.empty
+        case Left(code) =>
+          if (nonzeroException) scala.sys.error("Nonzero exit code: " + code) else Stream.empty
+        case Right(s) => Stream.cons(s, next())
+      }
+
+      new BlockingStreamed((s: T) => q put Right(s), code => q put Left(code), () => next())
+    }
+  }
+
+  // See scala.sys.process.ProcessImpl.Spawn
+  private object Spawn {
+    def apply(f: => Unit): Thread = apply(f, daemon = false)
+    def apply(f: => Unit, daemon: Boolean): Thread = {
+      val thread = new Thread() { override def run() = { f } }
+      thread.setDaemon(daemon)
+      thread.start()
+      thread
+    }
+  }
+
+  def apply(command: Seq[String]): Stream[String] = {
+    val streamed = BlockingStreamed[String](true)
+    val process = command.run(BasicIO(false, streamed.process, None))
+    Spawn(streamed.done(process.exitValue()))
+    streamed.stream()
+  }
+}
+
+class Dsdgen(dsdgenDir: String) extends Serializable {
+  private val dsdgen = s"$dsdgenDir/dsdgen"
+
+  def generate(
+      sparkContext: SparkContext,
+      tableName: String,
+      partitions: Int,
+      scaleFactor: Int): RDD[String] = {
+    val generatedData = {
+      sparkContext.parallelize(1 to partitions, partitions).flatMap { i =>
+        val localToolsDir = if (new java.io.File(dsdgen).exists) {
+          dsdgenDir
+        } else if (new java.io.File(s"/$dsdgen").exists) {
+          s"/$dsdgenDir"
+        } else {
+          throw new IllegalStateException(
+            s"Could not find dsdgen at $dsdgen or /$dsdgen. Run install")
+        }
+
+        // NOTE: RNGSEED is the RNG seed used by the data generator. Right now, it is fixed to 100.
+        val parallel = if (partitions > 1) s"-parallel $partitions -child $i" else ""
+        val commands = Seq(
+          "bash", "-c",
+          s"cd $localToolsDir && ./dsdgen -table $tableName -filter Y -scale $scaleFactor " +
+          s"-RNGSEED 100 $parallel")
+        BlockingLineStream(commands)
+      }
+    }
+
+    generatedData.setName(s"$tableName, sf=$scaleFactor, strings")
+    generatedData
+  }
+}
+
+class TPCDSTables(sqlContext: SQLContext, dsdgenDir: String, scaleFactor: Int)
+  extends TPCDSSchema with Logging with Serializable {
+
+  private val dataGenerator = new Dsdgen(dsdgenDir)
+
+  private def tables: Seq[Table] = tableColumns.map { case (tableName, schemaString) =>
+    val partitionColumns = tablePartitionColumns.getOrElse(tableName, Nil)
+      .map(_.stripPrefix("`").stripSuffix("`"))
+    Table(tableName, partitionColumns, StructType.fromDDL(schemaString))
+  }.toSeq
+
+  private case class Table(name: String, partitionColumns: Seq[String], schema: StructType) {
+    def nonPartitioned: Table = {
+      Table(name, Nil, schema)
+    }
+
+    private def df(numPartition: Int) = {
+      val generatedData = dataGenerator.generate(
+        sqlContext.sparkContext, name, numPartition, scaleFactor)
+      val rows = generatedData.mapPartitions { iter =>
+        iter.map { l =>
+          val values = l.split("\\|", -1).dropRight(1).map { v =>
+            if (v.equals("")) {
+              // If the string value is an empty string, we turn it to a null
+              null
+            } else {
+              v
+            }
+          }
+          Row.fromSeq(values)
+        }
+      }
+
+      val stringData =
+        sqlContext.createDataFrame(
+          rows,
+          StructType(schema.fields.map(f => StructField(f.name, StringType))))
+
+      val convertedData = {
+        val columns = schema.fields.map { f =>
+          val c = f.dataType match {
+            // Needs right-padding for char types
+            case CharType(n) => rpad(Column(f.name), n, " ")
+            // Don't need a cast for varchar types
+            case _: VarcharType => col(f.name)
+            case _ => col(f.name).cast(f.dataType)
+          }
+          c.as(f.name)
+        }
+        stringData.select(columns: _*)
+      }
+
+      convertedData
+    }
+
+    def genData(
+        location: String,
+        format: String,
+        overwrite: Boolean,
+        clusterByPartitionColumns: Boolean,
+        filterOutNullPartitionValues: Boolean,
+        numPartitions: Int): Unit = {
+      val mode = if (overwrite) SaveMode.Overwrite else SaveMode.Ignore
+
+      val data = df(numPartitions)
+      val tempTableName = s"${name}_text"
+      data.createOrReplaceTempView(tempTableName)
+
+      val writer = if (partitionColumns.nonEmpty) {
+        if (clusterByPartitionColumns) {
+          val columnString = data.schema.fields.map { field =>
+            field.name
+          }.mkString(",")
+          val partitionColumnString = partitionColumns.mkString(",")
+          val predicates = if (filterOutNullPartitionValues) {
+            partitionColumns.map(col => s"$col IS NOT NULL").mkString("WHERE ", " AND ", "")
+          } else {
+            ""
+          }
+
+          val query =
+            s"""
+               |SELECT
+               |  $columnString
+               |FROM
+               |  $tempTableName
+               |$predicates
+               |DISTRIBUTE BY
+               |  $partitionColumnString
+            """.stripMargin
+          val grouped = sqlContext.sql(query)
+          logInfo(s"Pre-clustering with partitioning columns with query $query.")
+          grouped.write
+        } else {
+          data.write
+        }
+      } else {
+        // treat non-partitioned tables as "one partition" that we want to coalesce
+        if (clusterByPartitionColumns) {
+          // in case data has more than maxRecordsPerFile, split into multiple writers to improve
+          // datagen speed files will be truncated to maxRecordsPerFile value, so the final
+          // result will be the same.
+          val numRows = data.count
+          val maxRecordPerFile = Try {
+            sqlContext.getConf("spark.sql.files.maxRecordsPerFile").toInt
+          }.getOrElse(0)
+
+          if (maxRecordPerFile > 0 && numRows > maxRecordPerFile) {
+            val numFiles = (numRows.toDouble/maxRecordPerFile).ceil.toInt
+            logInfo(s"Coalescing into $numFiles files")
+            data.coalesce(numFiles).write
+          } else {
+            data.coalesce(1).write
+          }
+        } else {
+          data.write
+        }
+      }
+      writer.format(format).mode(mode)
+      if (partitionColumns.nonEmpty) {
+        writer.partitionBy(partitionColumns: _*)
+      }
+      logInfo(s"Generating table $name in database to $location with save mode $mode.")
+      writer.save(location)
+      sqlContext.dropTempTable(tempTableName)
+    }
+  }
+
+  def genData(
+      location: String,
+      format: String,
+      overwrite: Boolean,
+      partitionTables: Boolean,
+      clusterByPartitionColumns: Boolean,
+      filterOutNullPartitionValues: Boolean,
+      tableFilter: String = "",
+      numPartitions: Int = 100): Unit = {
+    var tablesToBeGenerated = if (partitionTables) {
+      tables
+    } else {
+      tables.map(_.nonPartitioned)
+    }
+
+    if (!tableFilter.isEmpty) {
+      tablesToBeGenerated = tablesToBeGenerated.filter(_.name == tableFilter)
+      if (tablesToBeGenerated.isEmpty) {
+        throw new RuntimeException("Bad table name filter: " + tableFilter)
+      }
+    }
+
+    tablesToBeGenerated.foreach { table =>
+      val tableLocation = s"$location/${table.name}"
+      table.genData(tableLocation, format, overwrite, clusterByPartitionColumns,
+        filterOutNullPartitionValues, numPartitions)
+    }
+  }
+}
+
+class GenTPCDSDataConfig(args: Array[String]) {
+  var master: String = "local[*]"
+  var dsdgenDir: String = null
+  var location: String = null
+  var scaleFactor: Int = 1
+  var format: String = "parquet"
+  var overwrite: Boolean = false
+  var partitionTables: Boolean = false
+  var clusterByPartitionColumns: Boolean = false
+  var filterOutNullPartitionValues: Boolean = false
+  var tableFilter: String = ""
+  var numPartitions: Int = 100
+
+  parseArgs(args.toList)
+
+  private def parseArgs(inputArgs: List[String]): Unit = {
+    var args = inputArgs
+
+    while (args.nonEmpty) {
+      args match {
+        case "--master" :: value :: tail =>
+          master = value
+          args = tail
+
+        case "--dsdgenDir" :: value :: tail =>
+          dsdgenDir = value
+          args = tail
+
+        case "--location" :: value :: tail =>
+          location = value
+          args = tail
+
+        case "--scaleFactor" :: value :: tail =>
+          scaleFactor = toPositiveIntValue("Scale factor", value)
+          args = tail
+
+        case "--format" :: value :: tail =>
+          format = value
+          args = tail
+
+        case "--overwrite" :: tail =>
+          overwrite = true
+          args = tail
+
+        case "--partitionTables" :: tail =>
+          partitionTables = true
+          args = tail
+
+        case "--clusterByPartitionColumns" :: tail =>
+          clusterByPartitionColumns = true
+          args = tail
+
+        case "--filterOutNullPartitionValues" :: tail =>
+          filterOutNullPartitionValues = true
+          args = tail
+
+        case "--tableFilter" :: value :: tail =>
+          tableFilter = value
+          args = tail
+
+        case "--numPartitions" :: value :: tail =>
+          numPartitions = toPositiveIntValue("Number of partitions", value)
+          args = tail
+
+        case "--help" :: tail =>
+          printUsageAndExit(0)
+
+        case _ =>
+          // scalastyle:off println
+          System.err.println("Unknown/unsupported param " + args)
+          // scalastyle:on println
+          printUsageAndExit(1)
+      }
+    }
+
+    checkRequiredArguments()
+  }
+
+  private def printUsageAndExit(exitCode: Int): Unit = {
+    // scalastyle:off
+    System.err.println("""
+      |build/sbt "test:runMain <this class> [Options]"
+      |Options:
+      |  --master                        the Spark master to use, default to local[*]
+      |  --dsdgenDir                     location of dsdgen
+      |  --location                      root directory of location to generate data in
+      |  --scaleFactor                   size of the dataset to generate (in GB)
+      |  --format                        generated data format, Parquet, ORC ...
+      |  --overwrite                     whether to overwrite the data that is already there
+      |  --partitionTables               whether to create the partitioned fact tables
+      |  --clusterByPartitionColumns     whether to shuffle to get partitions coalesced into single files
+      |  --filterOutNullPartitionValues  whether to filter out the partition with NULL key value
+      |  --tableFilter                   comma-separated list of table names to generate (e.g., store_sales,store_returns),
+      |                                  all the tables are generated by default
+      |  --numPartitions                 how many dsdgen partitions to run - number of input tasks
+      """.stripMargin)
+    // scalastyle:on
+    System.exit(exitCode)
+  }
+
+  private def toPositiveIntValue(name: String, v: String): Int = {
+    if (Try(v.toInt).getOrElse(-1) <= 0) {
+      // scalastyle:off println
+      System.err.println(s"$name must be a positive number")
+      // scalastyle:on println
+      printUsageAndExit(-1)
+    }
+    v.toInt
+  }
+
+  private def checkRequiredArguments(): Unit = {
+    if (dsdgenDir == null) {
+      // scalastyle:off println
+      System.err.println("Must specify a dsdgen path")
+      // scalastyle:on println
+      printUsageAndExit(-1)
+    }
+    if (location == null) {
+      // scalastyle:off println
+      System.err.println("Must specify an output location")
+      // scalastyle:on println
+      printUsageAndExit(-1)
+    }
+  }
+}
+
+/**
+ * This class generates TPCDS table data by using tpcds-kit:
+ *  - https://github.com/databricks/tpcds-kit
+ *
+ * To run this:
+ * {{{
+ *   build/sbt "sql/test:runMain <this class> --dsdgenDir <path> --location <path> --scaleFactor 1"
+ * }}}
+ */
+object GenTPCDSData {
+
+  def main(args: Array[String]): Unit = {
+    val config = new GenTPCDSDataConfig(args)
+
+    val spark = SparkSession
+      .builder()
+      .appName(getClass.getName)
+      .master(config.master)
+      .getOrCreate()
+
+    val tables = new TPCDSTables(
+      spark.sqlContext,
+      dsdgenDir = config.dsdgenDir,
+      scaleFactor = config.scaleFactor)
+
+    tables.genData(
+      location = config.location,
+      format = config.format,
+      overwrite = config.overwrite,
+      partitionTables = config.partitionTables,
+      clusterByPartitionColumns = config.clusterByPartitionColumns,
+      filterOutNullPartitionValues = config.filterOutNullPartitionValues,
+      tableFilter = config.tableFilter,
+      numPartitions = config.numPartitions)
+
+    spark.stop()
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala
index 8f449037a5e01..311bc52515827 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala
@@ -21,6 +21,7 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{Expression, Generator}
 import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
+import org.apache.spark.sql.catalyst.trees.LeafLike
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.{IntegerType, StructType}
@@ -358,8 +359,7 @@ class GeneratorFunctionSuite extends QueryTest with SharedSparkSession {
   }
 }
 
-case class EmptyGenerator() extends Generator {
-  override def children: Seq[Expression] = Nil
+case class EmptyGenerator() extends Generator with LeafLike[Expression] {
   override def elementSchema: StructType = new StructType().add("id", IntegerType)
   override def eval(input: InternalRow): TraversableOnce[InternalRow] = Seq.empty
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala
index 4a4504a075060..a090eba430061 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala
@@ -27,7 +27,6 @@ import org.scalatest.Assertions._
 import org.apache.spark.TestUtils
 import org.apache.spark.api.python.{PythonBroadcast, PythonEvalType, PythonFunction, PythonUtils}
 import org.apache.spark.broadcast.Broadcast
-import org.apache.spark.internal.config.Tests
 import org.apache.spark.sql.catalyst.expressions.{Cast, Expression}
 import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.execution.python.UserDefinedPythonFunction
@@ -75,14 +74,7 @@ object IntegratedUDFTestUtils extends SQLHelper {
   import scala.sys.process._
 
   private lazy val pythonPath = sys.env.getOrElse("PYTHONPATH", "")
-  private lazy val sparkHome = if (sys.props.contains(Tests.IS_TESTING.key)) {
-    assert(sys.props.contains("spark.test.home") ||
-      sys.env.contains("SPARK_HOME"), "spark.test.home or SPARK_HOME is not set.")
-    sys.props.getOrElse("spark.test.home", sys.env("SPARK_HOME"))
-  } else {
-    assert(sys.env.contains("SPARK_HOME"), "SPARK_HOME is not set.")
-    sys.env("SPARK_HOME")
-  }
+
   // Note that we will directly refer pyspark's source, not the zip from a regular build.
   // It is possible the test is being ran without the build.
   private lazy val sourcePath = Paths.get(sparkHome, "python").toAbsolutePath
@@ -204,7 +196,7 @@ object IntegratedUDFTestUtils extends SQLHelper {
 
   lazy val pythonExec: String = {
     val pythonExec = sys.env.getOrElse(
-      "PYSPARK_DRIVER_PYTHON", sys.env.getOrElse("PYSPARK_PYTHON", "python3.6"))
+      "PYSPARK_DRIVER_PYTHON", sys.env.getOrElse("PYSPARK_PYTHON", "python3"))
     if (TestUtils.testCommandAvailable(pythonExec)) {
       pythonExec
     } else {
@@ -329,25 +321,33 @@ object IntegratedUDFTestUtils extends SQLHelper {
    *   casted_col.cast(df.schema("col").dataType)
    * }}}
    */
-  case class TestScalaUDF(name: String) extends TestUDF {
-    private[IntegratedUDFTestUtils] lazy val udf = new SparkUserDefinedFunction(
-      (input: Any) => if (input == null) {
-        null
-      } else {
-        input.toString
-      },
-      StringType,
-      inputEncoders = Seq.fill(1)(None),
-      name = Some(name)) {
-
-      override def apply(exprs: Column*): Column = {
-        assert(exprs.length == 1, "Defined UDF only has one column")
-        val expr = exprs.head.expr
-        assert(expr.resolved, "column should be resolved to use the same type " +
-          "as input. Try df(name) or df.col(name)")
-        Column(Cast(createScalaUDF(Cast(expr, StringType) :: Nil), expr.dataType))
-      }
+  class TestInternalScalaUDF(name: String) extends SparkUserDefinedFunction(
+    (input: Any) => if (input == null) {
+      null
+    } else {
+      input.toString
+    },
+    StringType,
+    inputEncoders = Seq.fill(1)(None),
+    name = Some(name)) {
+
+    override def apply(exprs: Column*): Column = {
+      assert(exprs.length == 1, "Defined UDF only has one column")
+      val expr = exprs.head.expr
+      assert(expr.resolved, "column should be resolved to use the same type " +
+        "as input. Try df(name) or df.col(name)")
+      Column(Cast(createScalaUDF(Cast(expr, StringType) :: Nil), expr.dataType))
+    }
+
+    override def withName(name: String): TestInternalScalaUDF = {
+      // "withName" should overridden to return TestInternalScalaUDF. Otherwise, the current object
+      // is sliced and the overridden "apply" is not invoked.
+      new TestInternalScalaUDF(name)
     }
+  }
+
+  case class TestScalaUDF(name: String) extends TestUDF {
+    private[IntegratedUDFTestUtils] lazy val udf = new TestInternalScalaUDF(name)
 
     def apply(exprs: Column*): Column = udf(exprs: _*)
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala
index f68c416941266..9f4c24b46a9b9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql
 
 import org.apache.log4j.Level
 
-import org.apache.spark.sql.catalyst.optimizer.EliminateResolvedHint
+import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide, EliminateResolvedHint}
 import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules.RuleExecutor
@@ -570,4 +570,31 @@ class JoinHintSuite extends PlanTest with SharedSparkSession with AdaptiveSparkP
       assert(joinHints == expectedHints)
     }
   }
+
+  test("SPARK-32220: Non Cartesian Product Join Result Correct with SHUFFLE_REPLICATE_NL hint") {
+    withTempView("t1", "t2") {
+      Seq((1, "4"), (2, "2")).toDF("key", "value").createTempView("t1")
+      Seq((1, "1"), (2, "12.3"), (2, "123")).toDF("key", "value").createTempView("t2")
+      val df1 = sql("SELECT /*+ shuffle_replicate_nl(t1) */ * from t1 join t2 ON t1.key = t2.key")
+      val df2 = sql("SELECT * from t1 join t2 ON t1.key = t2.key")
+      assert(df1.collect().size == df2.collect().size)
+
+      val df3 = sql("SELECT /*+ shuffle_replicate_nl(t1) */ * from t1 join t2")
+      val df4 = sql("SELECT * from t1 join t2")
+      assert(df3.collect().size == df4.collect().size)
+
+      val df5 = sql("SELECT /*+ shuffle_replicate_nl(t1) */ * from t1 join t2 ON t1.key < t2.key")
+      val df6 = sql("SELECT * from t1 join t2 ON t1.key < t2.key")
+      assert(df5.collect().size == df6.collect().size)
+
+      val df7 = sql("SELECT /*+ shuffle_replicate_nl(t1) */ * from t1 join t2 ON t1.key < 2")
+      val df8 = sql("SELECT * from t1 join t2 ON t1.key < 2")
+      assert(df7.collect().size == df8.collect().size)
+
+
+      val df9 = sql("SELECT /*+ shuffle_replicate_nl(t1) */ * from t1 join t2 ON t2.key < 2")
+      val df10 = sql("SELECT * from t1 join t2 ON t2.key < 2")
+      assert(df9.collect().size == df10.collect().size)
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
index fe6775cc7f9b9..acbf30089a92e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
@@ -29,8 +29,9 @@ import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
 import org.apache.spark.sql.catalyst.expressions.{Ascending, GenericRow, SortOrder}
 import org.apache.spark.sql.catalyst.plans.logical.Filter
-import org.apache.spark.sql.execution.{BinaryExecNode, FilterExec, SortExec, SparkPlan}
+import org.apache.spark.sql.execution.{BinaryExecNode, FilterExec, ProjectExec, SortExec, SparkPlan, WholeStageCodegenExec}
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
+import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
 import org.apache.spark.sql.execution.joins._
 import org.apache.spark.sql.execution.python.BatchEvalPythonExec
 import org.apache.spark.sql.internal.SQLConf
@@ -88,6 +89,7 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
     if (operators.head.getClass != c) {
       fail(s"$sqlString expected operator: $c, but got ${operators.head}\n physical: \n$physical")
     }
+    operators.head
   }
 
   test("join operator selection") {
@@ -712,7 +714,7 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
               |ON
               |  big.key = small.a
             """.stripMargin),
-          expected
+          expected.toSeq
         )
       }
 
@@ -729,7 +731,7 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
               |ON
               |  big.key = small.a
             """.stripMargin),
-          expected
+          expected.toSeq
         )
       }
     }
@@ -747,6 +749,22 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
         )
       }
 
+      // LEFT SEMI JOIN without bound condition does not spill
+      assertNotSpilled(sparkContext, "left semi join") {
+        checkAnswer(
+          sql("SELECT * FROM testData LEFT SEMI JOIN testData2 ON key = a WHERE key = 2"),
+          Row(2, "2") :: Nil
+        )
+      }
+
+      // LEFT ANTI JOIN without bound condition does not spill
+      assertNotSpilled(sparkContext, "left anti join") {
+        checkAnswer(
+          sql("SELECT * FROM testData LEFT ANTI JOIN testData2 ON key = a WHERE key = 2"),
+          Nil
+        )
+      }
+
       val expected = new ListBuffer[Row]()
       expected.append(
         Row(1, "1", 1, 1), Row(1, "1", 1, 2),
@@ -770,7 +788,7 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
               |ON
               |  big.key = small.a
             """.stripMargin),
-          expected
+          expected.toSeq
         )
       }
 
@@ -787,7 +805,7 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
               |ON
               |  big.key = small.a
             """.stripMargin),
-          expected
+          expected.toSeq
         )
       }
 
@@ -806,7 +824,7 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
               |ON
               |  big.key = small.a
             """.stripMargin),
-          expected
+          expected.toSeq
         )
       }
     }
@@ -1086,4 +1104,294 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
       assert(df2.join(df1, "id").collect().isEmpty)
     }
   }
+
+  test("SPARK-32330: Preserve shuffled hash join build side partitioning") {
+    val df1 = spark.range(10).select($"id".as("k1"))
+    val df2 = spark.range(30).select($"id".as("k2"))
+    Seq("inner", "cross").foreach(joinType => {
+      val plan = df1.join(df2.hint("SHUFFLE_HASH"), $"k1" === $"k2", joinType)
+        .groupBy($"k1").count()
+        .queryExecution.executedPlan
+      assert(collect(plan) { case _: ShuffledHashJoinExec => true }.size === 1)
+      // No extra shuffle before aggregate
+      assert(collect(plan) { case _: ShuffleExchangeExec => true }.size === 2)
+    })
+  }
+
+  test("SPARK-32383: Preserve hash join (BHJ and SHJ) stream side ordering") {
+    val df1 = spark.range(100).select($"id".as("k1"))
+    val df2 = spark.range(100).select($"id".as("k2"))
+    val df3 = spark.range(3).select($"id".as("k3"))
+    val df4 = spark.range(100).select($"id".as("k4"))
+
+    // Test broadcast hash join
+    Seq("inner", "left_outer").foreach(joinType => {
+      val plan = df1.join(df2.hint("SHUFFLE_MERGE"), $"k1" === $"k2", joinType)
+        .join(df3.hint("BROADCAST"), $"k1" === $"k3", joinType)
+        .join(df4.hint("SHUFFLE_MERGE"), $"k1" === $"k4", joinType)
+        .queryExecution
+        .executedPlan
+      assert(collect(plan) { case _: SortMergeJoinExec => true }.size === 2)
+      assert(collect(plan) { case _: BroadcastHashJoinExec => true }.size === 1)
+      // No extra sort before last sort merge join
+      assert(collect(plan) { case _: SortExec => true }.size === 3)
+    })
+
+    // Test shuffled hash join
+    Seq("inner", "left_outer").foreach(joinType => {
+      val plan = df1.join(df2.hint("SHUFFLE_MERGE"), $"k1" === $"k2", joinType)
+        .join(df3.hint("SHUFFLE_HASH"), $"k1" === $"k3", joinType)
+        .join(df4.hint("SHUFFLE_MERGE"), $"k1" === $"k4", joinType)
+        .queryExecution
+        .executedPlan
+      assert(collect(plan) { case _: SortMergeJoinExec => true }.size === 2)
+      assert(collect(plan) { case _: ShuffledHashJoinExec => true }.size === 1)
+      // No extra sort before last sort merge join
+      assert(collect(plan) { case _: SortExec => true }.size === 3)
+    })
+  }
+
+  test("SPARK-32290: SingleColumn Null Aware Anti Join Optimize") {
+    withSQLConf(SQLConf.OPTIMIZE_NULL_AWARE_ANTI_JOIN.key -> "true",
+      SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> Long.MaxValue.toString) {
+      // positive not in subquery case
+      var joinExec = assertJoin((
+        "select * from testData where key not in (select a from testData2)",
+        classOf[BroadcastHashJoinExec]))
+      assert(joinExec.asInstanceOf[BroadcastHashJoinExec].isNullAwareAntiJoin)
+
+      // negative not in subquery case since multi-column is not supported
+      assertJoin((
+        "select * from testData where (key, key + 1) not in (select * from testData2)",
+        classOf[BroadcastNestedLoopJoinExec]))
+
+      // positive hand-written left anti join
+      // testData.key nullable false
+      // testData3.b nullable true
+      joinExec = assertJoin((
+        "select * from testData left anti join testData3 ON key = b or isnull(key = b)",
+        classOf[BroadcastHashJoinExec]))
+      assert(joinExec.asInstanceOf[BroadcastHashJoinExec].isNullAwareAntiJoin)
+
+      // negative hand-written left anti join
+      // testData.key nullable false
+      // testData2.a nullable false
+      // isnull(key = a) will be optimized to true literal and removed
+      joinExec = assertJoin((
+        "select * from testData left anti join testData2 ON key = a or isnull(key = a)",
+        classOf[BroadcastHashJoinExec]))
+      assert(!joinExec.asInstanceOf[BroadcastHashJoinExec].isNullAwareAntiJoin)
+
+      // negative hand-written left anti join
+      // not match pattern Or(EqualTo(a=b), IsNull(EqualTo(a=b))
+      assertJoin((
+        "select * from testData2 left anti join testData3 ON testData2.a = testData3.b or " +
+          "isnull(testData2.b = testData3.b)",
+        classOf[BroadcastNestedLoopJoinExec]))
+    }
+  }
+
+  test("SPARK-32399: Full outer shuffled hash join") {
+    val inputDFs = Seq(
+      // Test unique join key
+      (spark.range(10).selectExpr("id as k1"),
+        spark.range(30).selectExpr("id as k2"),
+        $"k1" === $"k2"),
+      // Test non-unique join key
+      (spark.range(10).selectExpr("id % 5 as k1"),
+        spark.range(30).selectExpr("id % 5 as k2"),
+        $"k1" === $"k2"),
+      // Test empty build side
+      (spark.range(10).selectExpr("id as k1").filter("k1 < -1"),
+        spark.range(30).selectExpr("id as k2"),
+        $"k1" === $"k2"),
+      // Test empty stream side
+      (spark.range(10).selectExpr("id as k1"),
+        spark.range(30).selectExpr("id as k2").filter("k2 < -1"),
+        $"k1" === $"k2"),
+      // Test empty build and stream side
+      (spark.range(10).selectExpr("id as k1").filter("k1 < -1"),
+        spark.range(30).selectExpr("id as k2").filter("k2 < -1"),
+        $"k1" === $"k2"),
+      // Test string join key
+      (spark.range(10).selectExpr("cast(id * 3 as string) as k1"),
+        spark.range(30).selectExpr("cast(id as string) as k2"),
+        $"k1" === $"k2"),
+      // Test build side at right
+      (spark.range(30).selectExpr("cast(id / 3 as string) as k1"),
+        spark.range(10).selectExpr("cast(id as string) as k2"),
+        $"k1" === $"k2"),
+      // Test NULL join key
+      (spark.range(10).map(i => if (i % 2 == 0) i else null).selectExpr("value as k1"),
+        spark.range(30).map(i => if (i % 4 == 0) i else null).selectExpr("value as k2"),
+        $"k1" === $"k2"),
+      (spark.range(10).map(i => if (i % 3 == 0) i else null).selectExpr("value as k1"),
+        spark.range(30).map(i => if (i % 5 == 0) i else null).selectExpr("value as k2"),
+        $"k1" === $"k2"),
+      // Test multiple join keys
+      (spark.range(10).map(i => if (i % 2 == 0) i else null).selectExpr(
+        "value as k1", "cast(value % 5 as short) as k2", "cast(value * 3 as long) as k3"),
+        spark.range(30).map(i => if (i % 3 == 0) i else null).selectExpr(
+          "value as k4", "cast(value % 5 as short) as k5", "cast(value * 3 as long) as k6"),
+        $"k1" === $"k4" && $"k2" === $"k5" && $"k3" === $"k6")
+    )
+    inputDFs.foreach { case (df1, df2, joinExprs) =>
+      val smjDF = df1.join(df2.hint("SHUFFLE_MERGE"), joinExprs, "full")
+      assert(collect(smjDF.queryExecution.executedPlan) {
+        case _: SortMergeJoinExec => true }.size === 1)
+      val smjResult = smjDF.collect()
+
+      val shjDF = df1.join(df2.hint("SHUFFLE_HASH"), joinExprs, "full")
+      assert(collect(shjDF.queryExecution.executedPlan) {
+        case _: ShuffledHashJoinExec => true }.size === 1)
+      // Same result between shuffled hash join and sort merge join
+      checkAnswer(shjDF, smjResult)
+    }
+  }
+
+  test("SPARK-32649: Optimize BHJ/SHJ inner/semi join with empty hashed relation") {
+    val inputDFs = Seq(
+      // Test empty build side for inner join
+      (spark.range(30).selectExpr("id as k1"),
+        spark.range(10).selectExpr("id as k2").filter("k2 < -1"),
+        "inner"),
+      // Test empty build side for semi join
+      (spark.range(30).selectExpr("id as k1"),
+        spark.range(10).selectExpr("id as k2").filter("k2 < -1"),
+        "semi")
+    )
+    inputDFs.foreach { case (df1, df2, joinType) =>
+      // Test broadcast hash join
+      withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
+        val bhjCodegenDF = df1.join(df2.hint("BROADCAST"), $"k1" === $"k2", joinType)
+        assert(bhjCodegenDF.queryExecution.executedPlan.collect {
+          case WholeStageCodegenExec(_ : BroadcastHashJoinExec) => true
+          case WholeStageCodegenExec(ProjectExec(_, _ : BroadcastHashJoinExec)) => true
+        }.size === 1)
+        checkAnswer(bhjCodegenDF, Seq.empty)
+
+        withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false") {
+          val bhjNonCodegenDF = df1.join(df2, $"k1" === $"k2", joinType)
+          assert(bhjNonCodegenDF.queryExecution.executedPlan.collect {
+            case _: BroadcastHashJoinExec => true }.size === 1)
+          checkAnswer(bhjNonCodegenDF, Seq.empty)
+        }
+      }
+
+      // Test shuffled hash join
+      withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
+        val shjCodegenDF = df1.join(df2.hint("SHUFFLE_HASH"), $"k1" === $"k2", joinType)
+        assert(shjCodegenDF.queryExecution.executedPlan.collect {
+          case WholeStageCodegenExec(_ : ShuffledHashJoinExec) => true
+          case WholeStageCodegenExec(ProjectExec(_, _ : ShuffledHashJoinExec)) => true
+        }.size === 1)
+        checkAnswer(shjCodegenDF, Seq.empty)
+
+        withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false") {
+          val shjNonCodegenDF = df1.join(df2.hint("SHUFFLE_HASH"), $"k1" === $"k2", joinType)
+          assert(shjNonCodegenDF.queryExecution.executedPlan.collect {
+            case _: ShuffledHashJoinExec => true }.size === 1)
+          checkAnswer(shjNonCodegenDF, Seq.empty)
+        }
+      }
+    }
+  }
+
+  test("SPARK-34593: Preserve broadcast nested loop join partitioning and ordering") {
+    withTable("t1", "t2", "t3", "t4", "t5") {
+      spark.range(15).toDF("k").write.bucketBy(4, "k").saveAsTable("t1")
+      spark.range(6).toDF("k").write.bucketBy(4, "k").saveAsTable("t2")
+      spark.range(8).toDF("k").write.saveAsTable("t3")
+      spark.range(9).toDF("k").write.saveAsTable("t4")
+      spark.range(11).toDF("k").write.saveAsTable("t5")
+
+      def getAggQuery(selectExpr: String, joinType: String): String = {
+        s"""
+           |SELECT k, COUNT(*)
+           |FROM (SELECT $selectExpr FROM t1 $joinType JOIN t2)
+           |GROUP BY k
+         """.stripMargin
+      }
+
+      // Test output partitioning is preserved
+      Seq("INNER", "LEFT OUTER", "RIGHT OUTER", "LEFT SEMI", "LEFT ANTI").foreach {
+        joinType =>
+          val selectExpr = if (joinType == "RIGHT OUTER") {
+            "/*+ BROADCAST(t1) */ t2.k AS k"
+          } else {
+            "/*+ BROADCAST(t2) */ t1.k as k"
+          }
+          val plan = sql(getAggQuery(selectExpr, joinType)).queryExecution.executedPlan
+          assert(collect(plan) { case _: BroadcastNestedLoopJoinExec => true }.size === 1)
+          // No extra shuffle before aggregation
+          assert(collect(plan) { case _: ShuffleExchangeExec => true }.size === 0)
+      }
+
+      // Test output partitioning is not preserved
+      Seq("LEFT OUTER", "RIGHT OUTER", "LEFT SEMI", "LEFT ANTI", "FULL OUTER").foreach {
+        joinType =>
+          val selectExpr = if (joinType == "RIGHT OUTER") {
+            "/*+ BROADCAST(t2) */ t1.k AS k"
+          } else {
+            "/*+ BROADCAST(t1) */ t1.k as k"
+          }
+          val plan = sql(getAggQuery(selectExpr, joinType)).queryExecution.executedPlan
+          assert(collect(plan) { case _: BroadcastNestedLoopJoinExec => true }.size === 1)
+          // Have shuffle before aggregation
+          assert(collect(plan) { case _: ShuffleExchangeExec => true }.size === 1)
+      }
+
+      def getJoinQuery(selectExpr: String, joinType: String): String = {
+        s"""
+           |SELECT /*+ MERGE(t3) */ t3.k
+           |FROM
+           |(
+           |  SELECT $selectExpr
+           |  FROM
+           |    (SELECT /*+ MERGE(t4) */ t1.k AS k1 FROM t1 JOIN t4 ON t1.k = t4.k) AS left_t
+           |  $joinType JOIN
+           |    (SELECT /*+ MERGE(t5) */ t2.k AS k2 FROM t2 JOIN t5 ON t2.k = t5.k) AS right_t
+           |)
+           |JOIN t3
+           |ON t3.k = k0
+         """.stripMargin
+      }
+
+      // Test output ordering is preserved
+      Seq("INNER", "LEFT OUTER", "RIGHT OUTER", "LEFT SEMI", "LEFT ANTI").foreach {
+        joinType =>
+          val selectExpr = if (joinType == "RIGHT OUTER") {
+            "/*+ BROADCAST(left_t) */ k2 AS k0"
+          } else {
+            "/*+ BROADCAST(right_t) */ k1 as k0"
+          }
+          val plan = sql(getJoinQuery(selectExpr, joinType)).queryExecution.executedPlan
+          assert(collect(plan) { case _: BroadcastNestedLoopJoinExec => true }.size === 1)
+          assert(collect(plan) { case _: SortMergeJoinExec => true }.size === 3)
+          // No extra sort on left side before last sort merge join
+          assert(collect(plan) { case _: SortExec => true }.size === 5)
+      }
+
+      // Test output ordering is not preserved
+      Seq("LEFT OUTER", "FULL OUTER").foreach {
+        joinType =>
+          val selectExpr = "/*+ BROADCAST(left_t) */ k1 as k0"
+          val plan = sql(getJoinQuery(selectExpr, joinType)).queryExecution.executedPlan
+          assert(collect(plan) { case _: BroadcastNestedLoopJoinExec => true }.size === 1)
+          assert(collect(plan) { case _: SortMergeJoinExec => true }.size === 3)
+          // Have sort on left side before last sort merge join
+          assert(collect(plan) { case _: SortExec => true }.size === 6)
+      }
+
+      // Test singe partition
+      val fullJoinDF = sql(
+        s"""
+           |SELECT /*+ BROADCAST(t1) */ COUNT(*)
+           |FROM range(0, 10, 1, 1) t1 FULL OUTER JOIN range(0, 10, 1, 1) t2
+           |""".stripMargin)
+      val plan = fullJoinDF.queryExecution.executedPlan
+      assert(collect(plan) { case _: ShuffleExchangeExec => true}.size == 1)
+      checkAnswer(fullJoinDF, Row(100))
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
index 03b48451c7495..310e170e8c1b1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
@@ -411,7 +411,7 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession {
   test("infers schemas using options") {
     val df = spark.range(1)
       .select(schema_of_json(lit("{a:1}"), Map("allowUnquotedFieldNames" -> "true").asJava))
-    checkAnswer(df, Seq(Row("struct<a:bigint>")))
+    checkAnswer(df, Seq(Row("STRUCT<`a`: BIGINT>")))
   }
 
   test("from_json - array of primitive types") {
@@ -684,14 +684,14 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession {
     val input = regexp_replace(lit("""{"item_id": 1, "item_price": 0.1}"""), "item_", "")
     checkAnswer(
       spark.range(1).select(schema_of_json(input)),
-      Seq(Row("struct<id:bigint,price:double>")))
+      Seq(Row("STRUCT<`id`: BIGINT, `price`: DOUBLE>")))
   }
 
   test("SPARK-31065: schema_of_json - null and empty strings as strings") {
     Seq("""{"id": null}""", """{"id": ""}""").foreach { input =>
       checkAnswer(
         spark.range(1).select(schema_of_json(input)),
-        Seq(Row("struct<id:string>")))
+        Seq(Row("STRUCT<`id`: STRING>")))
     }
   }
 
@@ -703,7 +703,7 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession {
         schema_of_json(
           lit("""{"id": "a", "drop": {"drop": null}}"""),
           options.asJava)),
-      Seq(Row("struct<id:string>")))
+      Seq(Row("STRUCT<`id`: STRING>")))
 
     // Array of structs
     checkAnswer(
@@ -711,7 +711,7 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession {
         schema_of_json(
           lit("""[{"id": "a", "drop": {"drop": null}}]"""),
           options.asJava)),
-      Seq(Row("array<struct<id:string>>")))
+      Seq(Row("ARRAY<STRUCT<`id`: STRING>>")))
 
     // Other types are not affected.
     checkAnswer(
@@ -719,7 +719,7 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession {
         schema_of_json(
           lit("""null"""),
           options.asJava)),
-      Seq(Row("string")))
+      Seq(Row("STRING")))
   }
 
   test("optional datetime parser does not affect json time formatting") {
@@ -733,4 +733,111 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession {
          | """.stripMargin)
     checkAnswer(toDF("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"), toDF("yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]"))
   }
+
+  test("SPARK-33134: return partial results only for root JSON objects") {
+    val st = new StructType()
+      .add("c1", LongType)
+      .add("c2", ArrayType(new StructType().add("c3", LongType).add("c4", StringType)))
+    val df1 = Seq("""{"c2": [19], "c1": 123456}""").toDF("c0")
+    checkAnswer(df1.select(from_json($"c0", st)), Row(Row(123456, null)))
+    val df2 = Seq("""{"data": {"c2": [19], "c1": 123456}}""").toDF("c0")
+    checkAnswer(df2.select(from_json($"c0", new StructType().add("data", st))), Row(Row(null)))
+    val df3 = Seq("""[{"c2": [19], "c1": 123456}]""").toDF("c0")
+    checkAnswer(df3.select(from_json($"c0", ArrayType(st))), Row(null))
+    val df4 = Seq("""{"c2": [19]}""").toDF("c0")
+    checkAnswer(df4.select(from_json($"c0", MapType(StringType, st))), Row(null))
+  }
+
+  test("SPARK-33270: infers schema for JSON field with spaces and pass them to from_json") {
+    val in = Seq("""{"a b": 1}""").toDS()
+    val out = in.select(from_json('value, schema_of_json("""{"a b": 100}""")) as "parsed")
+    val expected = new StructType().add("parsed", new StructType().add("a b", LongType))
+    assert(out.schema == expected)
+  }
+
+  test("SPARK-33286: from_json - combined error messages") {
+    val df = Seq("""{"a":1}""").toDF("json")
+    val invalidJsonSchema = """{"fields": [{"a":123}], "type": "struct"}"""
+    val errMsg1 = intercept[AnalysisException] {
+      df.select(from_json($"json", invalidJsonSchema, Map.empty[String, String])).collect()
+    }.getMessage
+    assert(errMsg1.contains("""Failed to convert the JSON string '{"a":123}' to a field"""))
+
+    val invalidDataType = "MAP<INT, cow>"
+    val errMsg2 = intercept[AnalysisException] {
+      df.select(from_json($"json", invalidDataType, Map.empty[String, String])).collect()
+    }.getMessage
+    assert(errMsg2.contains("DataType cow is not supported"))
+
+    val invalidTableSchema = "x INT, a cow"
+    val errMsg3 = intercept[AnalysisException] {
+      df.select(from_json($"json", invalidTableSchema, Map.empty[String, String])).collect()
+    }.getMessage
+    assert(errMsg3.contains("DataType cow is not supported"))
+  }
+
+  test("SPARK-33907: bad json input with json pruning optimization: GetStructField") {
+    Seq("true", "false").foreach { enabled =>
+      withSQLConf(SQLConf.JSON_EXPRESSION_OPTIMIZATION.key -> enabled) {
+        val schema = new StructType()
+          .add("a", IntegerType)
+          .add("b", IntegerType)
+        val badRec = """{"a" 1, "b": 11}"""
+        val df = Seq(badRec, """{"a": 2, "b": 12}""").toDS()
+
+        val exception1 = intercept[SparkException] {
+          df.select(from_json($"value", schema, Map("mode" -> "FAILFAST"))("b")).collect()
+        }.getMessage
+        assert(exception1.contains(
+          "Malformed records are detected in record parsing. Parse Mode: FAILFAST."))
+
+        val exception2 = intercept[SparkException] {
+          df.select(from_json($"value", schema, Map("mode" -> "FAILFAST"))("a")).collect()
+        }.getMessage
+        assert(exception2.contains(
+          "Malformed records are detected in record parsing. Parse Mode: FAILFAST."))
+      }
+    }
+  }
+
+  test("SPARK-33907: bad json input with json pruning optimization: GetArrayStructFields") {
+    Seq("true", "false").foreach { enabled =>
+      withSQLConf(SQLConf.JSON_EXPRESSION_OPTIMIZATION.key -> enabled) {
+        val schema = ArrayType(new StructType()
+          .add("a", IntegerType)
+          .add("b", IntegerType))
+        val badRec = """{"a" 1, "b": 11}"""
+        val df = Seq(s"""[$badRec, {"a": 2, "b": 12}]""").toDS()
+
+        val exception1 = intercept[SparkException] {
+          df.select(from_json($"value", schema, Map("mode" -> "FAILFAST"))("b")).collect()
+        }.getMessage
+        assert(exception1.contains(
+          "Malformed records are detected in record parsing. Parse Mode: FAILFAST."))
+
+        val exception2 = intercept[SparkException] {
+          df.select(from_json($"value", schema, Map("mode" -> "FAILFAST"))("a")).collect()
+        }.getMessage
+        assert(exception2.contains(
+          "Malformed records are detected in record parsing. Parse Mode: FAILFAST."))
+      }
+    }
+  }
+
+  test("SPARK-33907: json pruning optimization with corrupt record field") {
+    Seq("true", "false").foreach { enabled =>
+      withSQLConf(SQLConf.JSON_EXPRESSION_OPTIMIZATION.key -> enabled) {
+        val schema = new StructType()
+          .add("a", IntegerType)
+          .add("b", IntegerType)
+        val badRec = """{"a" 1, "b": 11}"""
+
+        val df = Seq(badRec, """{"a": 2, "b": 12}""").toDS()
+          .selectExpr("from_json(value, 'a int, b int, _corrupt_record string') as parsed")
+          .selectExpr("parsed._corrupt_record")
+
+        checkAnswer(df, Seq(Row("""{"a" 1, "b": 11}"""), Row(null)))
+      }
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala
index bd86c2ec075b0..7c46dd08acff0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala
@@ -125,6 +125,11 @@ class MathFunctionsSuite extends QueryTest with SharedSparkSession {
     testOneToOneMathFunction(sinh, math.sinh)
   }
 
+  test("asinh") {
+    testOneToOneMathFunction(asinh,
+      (x: Double) => math.log(x + math.sqrt(x * x + 1)) )
+  }
+
   test("cos") {
     testOneToOneMathFunction(cos, math.cos)
   }
@@ -137,6 +142,11 @@ class MathFunctionsSuite extends QueryTest with SharedSparkSession {
     testOneToOneMathFunction(cosh, math.cosh)
   }
 
+  test("acosh") {
+    testOneToOneMathFunction(acosh,
+      (x: Double) => math.log(x + math.sqrt(x * x - 1)) )
+  }
+
   test("tan") {
     testOneToOneMathFunction(tan, math.tan)
   }
@@ -149,6 +159,11 @@ class MathFunctionsSuite extends QueryTest with SharedSparkSession {
     testOneToOneMathFunction(tanh, math.tanh)
   }
 
+  test("atanh") {
+    testOneToOneMathFunction(atanh,
+      (x: Double) => (0.5 * (math.log1p(x) - math.log1p(-x))) )
+  }
+
   test("degrees") {
     testOneToOneMathFunction(degrees, math.toDegrees)
     checkAnswer(
@@ -351,14 +366,14 @@ class MathFunctionsSuite extends QueryTest with SharedSparkSession {
 
     checkAnswer(
       df.select(
-        shiftLeft('a, 1), shiftLeft('b, 1), shiftLeft('c, 1), shiftLeft('d, 1),
-        shiftLeft('f, 1)),
+        shiftleft('a, 1), shiftleft('b, 1), shiftleft('c, 1), shiftleft('d, 1),
+        shiftLeft('f, 1)), // test deprecated one.
         Row(42.toLong, 42, 42.toShort, 42.toByte, null))
 
     checkAnswer(
       df.selectExpr(
-        "shiftLeft(a, 1)", "shiftLeft(b, 1)", "shiftLeft(b, 1)", "shiftLeft(d, 1)",
-        "shiftLeft(f, 1)"),
+        "shiftleft(a, 1)", "shiftleft(b, 1)", "shiftleft(b, 1)", "shiftleft(d, 1)",
+        "shiftleft(f, 1)"),
       Row(42.toLong, 42, 42.toShort, 42.toByte, null))
   }
 
@@ -368,14 +383,14 @@ class MathFunctionsSuite extends QueryTest with SharedSparkSession {
 
     checkAnswer(
       df.select(
-        shiftRight('a, 1), shiftRight('b, 1), shiftRight('c, 1), shiftRight('d, 1),
-        shiftRight('f, 1)),
+        shiftright('a, 1), shiftright('b, 1), shiftright('c, 1), shiftright('d, 1),
+        shiftRight('f, 1)), // test deprecated one.
       Row(21.toLong, 21, 21.toShort, 21.toByte, null))
 
     checkAnswer(
       df.selectExpr(
-        "shiftRight(a, 1)", "shiftRight(b, 1)", "shiftRight(c, 1)", "shiftRight(d, 1)",
-        "shiftRight(f, 1)"),
+        "shiftright(a, 1)", "shiftright(b, 1)", "shiftright(c, 1)", "shiftright(d, 1)",
+        "shiftright(f, 1)"),
       Row(21.toLong, 21, 21.toShort, 21.toByte, null))
   }
 
@@ -385,14 +400,14 @@ class MathFunctionsSuite extends QueryTest with SharedSparkSession {
 
     checkAnswer(
       df.select(
-        shiftRightUnsigned('a, 1), shiftRightUnsigned('b, 1), shiftRightUnsigned('c, 1),
-        shiftRightUnsigned('d, 1), shiftRightUnsigned('f, 1)),
+        shiftrightunsigned('a, 1), shiftrightunsigned('b, 1), shiftrightunsigned('c, 1),
+        shiftrightunsigned('d, 1), shiftRightUnsigned('f, 1)), // test deprecated one.
       Row(9223372036854775787L, 21, 21.toShort, 21.toByte, null))
 
     checkAnswer(
       df.selectExpr(
-        "shiftRightUnsigned(a, 1)", "shiftRightUnsigned(b, 1)", "shiftRightUnsigned(c, 1)",
-        "shiftRightUnsigned(d, 1)", "shiftRightUnsigned(f, 1)"),
+        "shiftrightunsigned(a, 1)", "shiftrightunsigned(b, 1)", "shiftrightunsigned(c, 1)",
+        "shiftrightunsigned(d, 1)", "shiftrightunsigned(f, 1)"),
       Row(9223372036854775787L, 21, 21.toShort, 21.toByte, null))
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/MetadataCacheSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/MetadataCacheSuite.scala
index a9f443be69cb2..956bd7861d99d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/MetadataCacheSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/MetadataCacheSuite.scala
@@ -20,7 +20,6 @@ package org.apache.spark.sql
 import java.io.File
 
 import org.apache.spark.{SparkConf, SparkException}
-import org.apache.spark.sql.execution.adaptive.AdaptiveTestUtils.assertExceptionMessage
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 
@@ -56,8 +55,8 @@ abstract class MetadataCacheSuite extends QueryTest with SharedSparkSession {
       val e = intercept[SparkException] {
         df.count()
       }
-      assertExceptionMessage(e, "FileNotFoundException")
-      assertExceptionMessage(e, "recreating the Dataset/DataFrame involved")
+      assert(e.getMessage.contains("FileNotFoundException"))
+      assert(e.getMessage.contains("recreating the Dataset/DataFrame involved"))
     }
   }
 }
@@ -85,8 +84,8 @@ class MetadataCacheV1Suite extends MetadataCacheSuite {
       val e = intercept[SparkException] {
         sql("select count(*) from view_refresh").first()
       }
-      assertExceptionMessage(e, "FileNotFoundException")
-      assertExceptionMessage(e, "REFRESH")
+      assert(e.getMessage.contains("FileNotFoundException"))
+      assert(e.getMessage.contains("REFRESH"))
 
       // Refresh and we should be able to read it again.
       spark.catalog.refreshTable("view_refresh")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/NestedDataSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/NestedDataSourceSuite.scala
new file mode 100644
index 0000000000000..78b314272aa07
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/NestedDataSourceSuite.scala
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.{LongType, StructType}
+
+// Datasource tests for nested schemas
+trait NestedDataSourceSuiteBase extends QueryTest with SharedSparkSession {
+  protected val nestedDataSources: Seq[String] = Seq("orc", "parquet", "json")
+  protected def readOptions(schema: StructType): Map[String, String] = Map.empty
+  protected def save(selectExpr: Seq[String], format: String, path: String): Unit = {
+    spark
+      .range(1L)
+      .selectExpr(selectExpr: _*)
+      .write.mode("overwrite")
+      .format(format)
+      .save(path)
+  }
+  protected val colType: String = "in the data schema"
+
+  test("SPARK-32431: consistent error for nested and top-level duplicate columns") {
+    Seq(
+      Seq("id AS lowercase", "id + 1 AS camelCase") ->
+        new StructType()
+          .add("LowerCase", LongType)
+          .add("camelcase", LongType)
+          .add("CamelCase", LongType),
+      Seq("NAMED_STRUCT('lowercase', id, 'camelCase', id + 1) AS StructColumn") ->
+        new StructType().add("StructColumn",
+          new StructType()
+            .add("LowerCase", LongType)
+            .add("camelcase", LongType)
+            .add("CamelCase", LongType))
+    ).foreach { case (selectExpr: Seq[String], caseInsensitiveSchema: StructType) =>
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+        nestedDataSources.map { format =>
+          withClue(s"format = $format select = ${selectExpr.mkString(",")}") {
+            withTempPath { dir =>
+              val path = dir.getCanonicalPath
+              save(selectExpr, format, path)
+              val e = intercept[AnalysisException] {
+                spark
+                  .read
+                  .options(readOptions(caseInsensitiveSchema))
+                  .schema(caseInsensitiveSchema)
+                  .format(format)
+                  .load(path)
+                  .show
+              }
+              assert(e.getMessage.contains(s"Found duplicate column(s) $colType: `camelcase`"))
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+class NestedDataSourceV1Suite extends NestedDataSourceSuiteBase {
+  override protected def sparkConf: SparkConf =
+    super
+      .sparkConf
+      .set(SQLConf.USE_V1_SOURCE_LIST, nestedDataSources.mkString(","))
+}
+
+class NestedDataSourceV2Suite extends NestedDataSourceSuiteBase {
+  override protected def sparkConf: SparkConf =
+    super
+      .sparkConf
+      .set(SQLConf.USE_V1_SOURCE_LIST, "")
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/PlanStabilitySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/PlanStabilitySuite.scala
new file mode 100644
index 0000000000000..0df18723921e8
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/PlanStabilitySuite.scala
@@ -0,0 +1,333 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import java.io.File
+import java.nio.charset.StandardCharsets
+
+import scala.collection.mutable
+
+import org.apache.commons.io.FileUtils
+
+import org.apache.spark.sql.catalyst.expressions.AttributeSet
+import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.execution._
+import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecutionSuite
+import org.apache.spark.sql.execution.exchange.{Exchange, ReusedExchangeExec}
+import org.apache.spark.sql.internal.SQLConf
+
+// scalastyle:off line.size.limit
+/**
+ * Check that TPC-DS SparkPlans don't change.
+ * If there are plan differences, the error message looks like this:
+ *   Plans did not match:
+ *   last approved simplified plan: /path/to/tpcds-plan-stability/approved-plans-xxx/q1/simplified.txt
+ *   last approved explain plan: /path/to/tpcds-plan-stability/approved-plans-xxx/q1/explain.txt
+ *   [last approved simplified plan]
+ *
+ *   actual simplified plan: /path/to/tmp/q1.actual.simplified.txt
+ *   actual explain plan: /path/to/tmp/q1.actual.explain.txt
+ *   [actual simplified plan]
+ *
+ *
+ * To run the entire test suite:
+ * {{{
+ *   build/sbt "sql/testOnly *PlanStability[WithStats]Suite"
+ * }}}
+ *
+ * To run a single test file upon change:
+ * {{{
+ *   build/sbt "sql/testOnly *PlanStability[WithStats]Suite -- -z (tpcds-v1.4/q49)"
+ * }}}
+ *
+ * To re-generate golden files for entire suite, run:
+ * {{{
+ *   SPARK_GENERATE_GOLDEN_FILES=1 build/sbt "sql/testOnly *PlanStability[WithStats]Suite"
+ * }}}
+ *
+ * To re-generate golden file for a single test, run:
+ * {{{
+ *   SPARK_GENERATE_GOLDEN_FILES=1 build/sbt "sql/testOnly *PlanStability[WithStats]Suite -- -z (tpcds-v1.4/q49)"
+ * }}}
+ */
+// scalastyle:on line.size.limit
+trait PlanStabilitySuite extends TPCDSBase with DisableAdaptiveExecutionSuite {
+
+  private val originalMaxToStringFields = conf.maxToStringFields
+
+  override def beforeAll(): Unit = {
+    conf.setConf(SQLConf.MAX_TO_STRING_FIELDS, Int.MaxValue)
+    super.beforeAll()
+  }
+
+  override def afterAll(): Unit = {
+    super.afterAll()
+    conf.setConf(SQLConf.MAX_TO_STRING_FIELDS, originalMaxToStringFields)
+  }
+
+  private val regenerateGoldenFiles: Boolean = System.getenv("SPARK_GENERATE_GOLDEN_FILES") == "1"
+
+  protected val baseResourcePath = {
+    // use the same way as `SQLQueryTestSuite` to get the resource path
+    getWorkspaceFilePath("sql", "core", "src", "test", "resources", "tpcds-plan-stability").toFile
+  }
+
+  private val referenceRegex = "#\\d+".r
+  private val normalizeRegex = "#\\d+L?".r
+
+  private val clsName = this.getClass.getCanonicalName
+
+  def goldenFilePath: String
+
+  private def getDirForTest(name: String): File = {
+    new File(goldenFilePath, name)
+  }
+
+  private def isApproved(
+      dir: File, actualSimplifiedPlan: String, actualExplain: String): Boolean = {
+    val simplifiedFile = new File(dir, "simplified.txt")
+    val expectedSimplified = FileUtils.readFileToString(simplifiedFile, StandardCharsets.UTF_8)
+    lazy val explainFile = new File(dir, "explain.txt")
+    lazy val expectedExplain = FileUtils.readFileToString(explainFile, StandardCharsets.UTF_8)
+    expectedSimplified == actualSimplifiedPlan && expectedExplain == actualExplain
+  }
+
+  /**
+   * Serialize and save this SparkPlan.
+   * The resulting file is used by [[checkWithApproved]] to check stability.
+   *
+   * @param plan    the SparkPlan
+   * @param name    the name of the query
+   * @param explain the full explain output; this is saved to help debug later as the simplified
+   *                plan is not too useful for debugging
+   */
+  private def generateGoldenFile(plan: SparkPlan, name: String, explain: String): Unit = {
+    val dir = getDirForTest(name)
+    val simplified = getSimplifiedPlan(plan)
+    val foundMatch = dir.exists() && isApproved(dir, simplified, explain)
+
+    if (!foundMatch) {
+      FileUtils.deleteDirectory(dir)
+      assert(dir.mkdirs())
+
+      val file = new File(dir, "simplified.txt")
+      FileUtils.writeStringToFile(file, simplified, StandardCharsets.UTF_8)
+      val fileOriginalPlan = new File(dir, "explain.txt")
+      FileUtils.writeStringToFile(fileOriginalPlan, explain, StandardCharsets.UTF_8)
+      logDebug(s"APPROVED: $file $fileOriginalPlan")
+    }
+  }
+
+  private def checkWithApproved(plan: SparkPlan, name: String, explain: String): Unit = {
+    val dir = getDirForTest(name)
+    val tempDir = FileUtils.getTempDirectory
+    val actualSimplified = getSimplifiedPlan(plan)
+    val foundMatch = isApproved(dir, actualSimplified, explain)
+
+    if (!foundMatch) {
+      // show diff with last approved
+      val approvedSimplifiedFile = new File(dir, "simplified.txt")
+      val approvedExplainFile = new File(dir, "explain.txt")
+
+      val actualSimplifiedFile = new File(tempDir, s"$name.actual.simplified.txt")
+      val actualExplainFile = new File(tempDir, s"$name.actual.explain.txt")
+
+      val approvedSimplified = FileUtils.readFileToString(
+        approvedSimplifiedFile, StandardCharsets.UTF_8)
+      // write out for debugging
+      FileUtils.writeStringToFile(actualSimplifiedFile, actualSimplified, StandardCharsets.UTF_8)
+      FileUtils.writeStringToFile(actualExplainFile, explain, StandardCharsets.UTF_8)
+
+      fail(
+        s"""
+          |Plans did not match:
+          |last approved simplified plan: ${approvedSimplifiedFile.getAbsolutePath}
+          |last approved explain plan: ${approvedExplainFile.getAbsolutePath}
+          |
+          |$approvedSimplified
+          |
+          |actual simplified plan: ${actualSimplifiedFile.getAbsolutePath}
+          |actual explain plan: ${actualExplainFile.getAbsolutePath}
+          |
+          |$actualSimplified
+        """.stripMargin)
+    }
+  }
+
+  /**
+   * Get the simplified plan for a specific SparkPlan. In the simplified plan, the node only has
+   * its name and all the sorted reference and produced attributes names(without ExprId) and its
+   * simplified children as well. And we'll only identify the performance sensitive nodes, e.g.,
+   * Exchange, Subquery, in the simplified plan. Given such a identical but simplified plan, we'd
+   * expect to avoid frequent plan changing and catch the possible meaningful regression.
+   */
+  private def getSimplifiedPlan(plan: SparkPlan): String = {
+    val exchangeIdMap = new mutable.HashMap[Int, Int]()
+    val subqueriesMap = new mutable.HashMap[Int, Int]()
+
+    def getId(plan: SparkPlan): Int = plan match {
+      case exchange: Exchange => exchangeIdMap.getOrElseUpdate(exchange.id, exchangeIdMap.size + 1)
+      case ReusedExchangeExec(_, exchange) =>
+        exchangeIdMap.getOrElseUpdate(exchange.id, exchangeIdMap.size + 1)
+      case subquery: SubqueryExec =>
+        subqueriesMap.getOrElseUpdate(subquery.id, subqueriesMap.size + 1)
+      case subquery: SubqueryBroadcastExec =>
+        subqueriesMap.getOrElseUpdate(subquery.id, subqueriesMap.size + 1)
+      case ReusedSubqueryExec(subquery) =>
+        subqueriesMap.getOrElseUpdate(subquery.id, subqueriesMap.size + 1)
+      case _ => -1
+    }
+
+    /**
+     * Some expression names have ExprId in them due to using things such as
+     * "sum(sr_return_amt#14)", so we remove all of these using regex
+     */
+    def cleanUpReferences(references: AttributeSet): String = {
+      referenceRegex.replaceAllIn(references.map(_.name).mkString(","), "")
+    }
+
+    /**
+     * Generate a simplified plan as a string
+     * Example output:
+     * TakeOrderedAndProject [c_customer_id]
+     *   WholeStageCodegen
+     *     Project [c_customer_id]
+     */
+    def simplifyNode(node: SparkPlan, depth: Int): String = {
+      val padding = "  " * depth
+      var thisNode = node.nodeName
+      if (node.references.nonEmpty) {
+        thisNode += s" [${cleanUpReferences(node.references)}]"
+      }
+      if (node.producedAttributes.nonEmpty) {
+        thisNode += s" [${cleanUpReferences(node.producedAttributes)}]"
+      }
+      val id = getId(node)
+      if (id > 0) {
+        thisNode += s" #$id"
+      }
+      val childrenSimplified = node.children.map(simplifyNode(_, depth + 1))
+      val subqueriesSimplified = node.subqueries.map(simplifyNode(_, depth + 1))
+      s"$padding$thisNode\n${subqueriesSimplified.mkString("")}${childrenSimplified.mkString("")}"
+    }
+
+    simplifyNode(plan, 0)
+  }
+
+  private def normalizeIds(plan: String): String = {
+    val map = new mutable.HashMap[String, String]()
+    normalizeRegex.findAllMatchIn(plan).map(_.toString)
+      .foreach(map.getOrElseUpdate(_, (map.size + 1).toString))
+    normalizeRegex.replaceAllIn(plan, regexMatch => s"#${map(regexMatch.toString)}")
+  }
+
+  private def normalizeLocation(plan: String): String = {
+    plan.replaceAll(s"Location.*$clsName/",
+      "Location [not included in comparison]/{warehouse_dir}/")
+  }
+
+  /**
+   * Test a TPC-DS query. Depending on the settings this test will either check if the plan matches
+   * a golden file or it will create a new golden file.
+   */
+  protected def testQuery(tpcdsGroup: String, query: String, suffix: String = ""): Unit = {
+    val queryString = resourceToString(s"$tpcdsGroup/$query.sql",
+      classLoader = Thread.currentThread().getContextClassLoader)
+    val qe = sql(queryString).queryExecution
+    val plan = qe.executedPlan
+    val explain = normalizeLocation(normalizeIds(qe.explainString(FormattedMode)))
+
+    if (regenerateGoldenFiles) {
+      generateGoldenFile(plan, query + suffix, explain)
+    } else {
+      checkWithApproved(plan, query + suffix, explain)
+    }
+  }
+}
+
+class TPCDSV1_4_PlanStabilitySuite extends PlanStabilitySuite {
+  override val goldenFilePath: String =
+    new File(baseResourcePath, s"approved-plans-v1_4").getAbsolutePath
+
+  tpcdsQueries.foreach { q =>
+    test(s"check simplified (tpcds-v1.4/$q)") {
+      testQuery("tpcds", q)
+    }
+  }
+}
+
+class TPCDSV1_4_PlanStabilityWithStatsSuite extends PlanStabilitySuite {
+  override def injectStats: Boolean = true
+
+  override val goldenFilePath: String =
+    new File(baseResourcePath, s"approved-plans-v1_4").getAbsolutePath
+
+  tpcdsQueries.foreach { q =>
+    test(s"check simplified sf100 (tpcds-v1.4/$q)") {
+      testQuery("tpcds", q, ".sf100")
+    }
+  }
+}
+
+class TPCDSV2_7_PlanStabilitySuite extends PlanStabilitySuite {
+  override val goldenFilePath: String =
+    new File(baseResourcePath, s"approved-plans-v2_7").getAbsolutePath
+
+  tpcdsQueriesV2_7_0.foreach { q =>
+    test(s"check simplified (tpcds-v2.7.0/$q)") {
+      testQuery("tpcds-v2.7.0", q)
+    }
+  }
+}
+
+class TPCDSV2_7_PlanStabilityWithStatsSuite extends PlanStabilitySuite {
+  override def injectStats: Boolean = true
+
+  override val goldenFilePath: String =
+    new File(baseResourcePath, s"approved-plans-v2_7").getAbsolutePath
+
+  tpcdsQueriesV2_7_0.foreach { q =>
+    test(s"check simplified sf100 (tpcds-v2.7.0/$q)") {
+      testQuery("tpcds-v2.7.0", q, ".sf100")
+    }
+  }
+}
+
+class TPCDSModifiedPlanStabilitySuite extends PlanStabilitySuite {
+  override val goldenFilePath: String =
+    new File(baseResourcePath, s"approved-plans-modified").getAbsolutePath
+
+  modifiedTPCDSQueries.foreach { q =>
+    test(s"check simplified (tpcds-modifiedQueries/$q)") {
+      testQuery("tpcds-modifiedQueries", q)
+    }
+  }
+}
+
+class TPCDSModifiedPlanStabilityWithStatsSuite extends PlanStabilitySuite {
+  override def injectStats: Boolean = true
+
+  override val goldenFilePath: String =
+    new File(baseResourcePath, s"approved-plans-modified").getAbsolutePath
+
+  modifiedTPCDSQueries.foreach { q =>
+    test(s"check simplified sf100 (tpcds-modifiedQueries/$q)") {
+      testQuery("tpcds-modifiedQueries", q, ".sf100")
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ProductAggSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ProductAggSuite.scala
new file mode 100644
index 0000000000000..7fff53087e66f
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ProductAggSuite.scala
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import org.apache.spark.sql.expressions.Window
+import org.apache.spark.sql.functions.{col, lit, product}
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.{ByteType, DoubleType, FloatType, IntegerType, ShortType}
+
+
+class ProductAggSuite extends QueryTest
+  with SharedSparkSession {
+
+  // Sequence of integers small enough that factorial is representable exactly as DoubleType:
+  private lazy val data16 = spark.range(1, 17).toDF("x")
+
+  private lazy val factorials = (1 to 16).scanLeft(1L) { case (f, x) => f * x }
+
+  test("bare factorial") {
+    checkAnswer(
+      data16.agg(product(col("x"))),
+      Row((1L to 16L).reduce { _ * _ }.toDouble)
+    )
+
+    checkAnswer(
+      data16.agg(product(col("x"))),
+      Row(factorials(16))
+    )
+  }
+
+  test("type flexibility") {
+    val bytes16 = spark.createDataset((1 to 16).map { _.toByte })(Encoders.scalaByte).toDF("x")
+
+    val variants = Map(
+      "int8" -> ByteType, "int16" -> ShortType, "int32" -> IntegerType,
+      "float32" -> FloatType, "float64" -> DoubleType)
+
+    val prods = variants.foldLeft(bytes16) { case (df, (id, typ)) =>
+      df.withColumn(id, df.col("x").cast(typ))
+    }.agg(
+      lit(1) as "dummy",
+      variants.keys.toSeq.map { id => product(col(id)) as id } : _*)
+
+    variants.keys.foreach { typ =>
+      checkAnswer(
+        prods.select(typ),
+        Row(factorials(16))
+      )
+    }
+  }
+
+  test("windowed factorials") {
+    val win = Window.partitionBy(lit(1)).orderBy("x")
+
+    val prodFactorials = data16.withColumn("f", product(col("x")).over(win)).orderBy(col("x"))
+
+    assert(prodFactorials.count === 16)
+
+    checkAnswer(
+      prodFactorials.limit(5),
+      Seq(
+        Row(1, 1.0),
+        Row(2, 2.0),
+        Row(3, 6.0),
+        Row(4, 24.0),
+        Row(5, 120.0)
+      )
+    )
+
+    checkAnswer(
+      prodFactorials,
+      factorials.zipWithIndex.drop(1).map { case (fac, idx) =>
+        Row(idx, fac)
+      }
+    )
+  }
+
+  test("grouped factorials") {
+    val grouped = data16.groupBy((col("x") % 3) as "mod3")
+      .agg(
+        product(col("x")) as "product",
+        product(col("x") * 0.5) as "product_scaled",
+        product(col("x") * -1.0) as "product_minus")
+      .orderBy("mod3")
+
+    val expectedBase = Seq(
+      (3 * 6 * 9 * 12 * 15),
+      (1 * 4 * 7 * 10 * 13 * 16),
+      (2 * 5 * 8 * 11 * 14))
+
+    checkAnswer(
+      grouped.select("product"),
+      expectedBase.map { n => Row(n.toDouble) }
+    )
+
+    checkAnswer(
+      grouped.select("product_scaled"),
+      expectedBase.zip(Seq(0.03125, 0.015625, 0.03125)).map { case(a, b) => Row(a * b) }
+    )
+
+    checkAnswer(
+      grouped.select("product_minus"),
+      expectedBase.zip(Seq(-1.0, 1.0, -1.0)).map { case(a, b) => Row(a * b) }
+    )
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
index e52d2262a6bf8..8469216901b05 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
@@ -418,7 +418,7 @@ object QueryTest extends Assertions {
   }
 
   def checkAnswer(df: DataFrame, expectedAnswer: java.util.List[Row]): Unit = {
-    getErrorMessageInCheckAnswer(df, expectedAnswer.asScala) match {
+    getErrorMessageInCheckAnswer(df, expectedAnswer.asScala.toSeq) match {
       case Some(errorMessage) => Assert.fail(errorMessage)
       case None =>
     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ReplaceNullWithFalseInPredicateEndToEndSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ReplaceNullWithFalseInPredicateEndToEndSuite.scala
index bdbb741f24bc6..739b4052ee90d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ReplaceNullWithFalseInPredicateEndToEndSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ReplaceNullWithFalseInPredicateEndToEndSuite.scala
@@ -27,6 +27,12 @@ import org.apache.spark.sql.types.BooleanType
 class ReplaceNullWithFalseInPredicateEndToEndSuite extends QueryTest with SharedSparkSession {
   import testImplicits._
 
+  private def checkPlanIsEmptyLocalScan(df: DataFrame): Unit =
+    df.queryExecution.executedPlan match {
+      case s: LocalTableScanExec => assert(s.rows.isEmpty)
+      case p => fail(s"$p is not LocalTableScanExec")
+    }
+
   test("SPARK-25860: Replace Literal(null, _) with FalseLiteral whenever possible") {
     withTable("t1", "t2") {
       Seq((1, true), (2, false)).toDF("l", "b").write.saveAsTable("t1")
@@ -64,11 +70,6 @@ class ReplaceNullWithFalseInPredicateEndToEndSuite extends QueryTest with Shared
 
       checkAnswer(df1.where("IF(l > 10, false, b OR null)"), Row(1, true))
     }
-
-    def checkPlanIsEmptyLocalScan(df: DataFrame): Unit = df.queryExecution.executedPlan match {
-      case s: LocalTableScanExec => assert(s.rows.isEmpty)
-      case p => fail(s"$p is not LocalTableScanExec")
-    }
   }
 
   test("SPARK-26107: Replace Literal(null, _) with FalseLiteral in higher-order functions") {
@@ -112,4 +113,14 @@ class ReplaceNullWithFalseInPredicateEndToEndSuite extends QueryTest with Shared
       assertNoLiteralNullInPlan(q3)
     }
   }
+
+  test("SPARK-33847: replace None of elseValue inside CaseWhen to FalseLiteral") {
+    withTable("t1") {
+      Seq((1, 1), (2, 2)).toDF("a", "b").write.saveAsTable("t1")
+      val t1 = spark.table("t1")
+      val q1 = t1.filter("(CASE WHEN a > 1 THEN 1 END) = 0")
+      checkAnswer(q1, Seq.empty)
+      checkPlanIsEmptyLocalScan(q1)
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala
index 720d570ca8384..4052130720811 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.internal.config
 import org.apache.spark.sql.internal.SQLConf.CHECKPOINT_LOCATION
-import org.apache.spark.sql.internal.StaticSQLConf.SCHEMA_STRING_LENGTH_THRESHOLD
+import org.apache.spark.sql.internal.StaticSQLConf.GLOBAL_TEMP_DATABASE
 
 class RuntimeConfigSuite extends SparkFunSuite {
 
@@ -62,7 +62,7 @@ class RuntimeConfigSuite extends SparkFunSuite {
     val conf = newConf()
 
     // SQL configs
-    assert(!conf.isModifiable(SCHEMA_STRING_LENGTH_THRESHOLD.key))
+    assert(!conf.isModifiable(GLOBAL_TEMP_DATABASE.key))
     assert(conf.isModifiable(CHECKPOINT_LOCATION.key))
     // Core configs
     assert(!conf.isModifiable(config.CPUS_PER_TASK.key))
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala
index a1799829932b8..d81768c0077eb 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala
@@ -122,7 +122,7 @@ class SQLContextSuite extends SparkFunSuite with SharedSparkContext {
     df.createOrReplaceTempView("listtablessuitetable")
 
     val expectedSchema = StructType(
-      StructField("database", StringType, false) ::
+      StructField("namespace", StringType, false) ::
         StructField("tableName", StringType, false) ::
         StructField("isTemporary", BooleanType, false) :: Nil)
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLInsertTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLInsertTestSuite.scala
new file mode 100644
index 0000000000000..2f56fbaf7f821
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLInsertTestSuite.scala
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.catalyst.expressions.Hex
+import org.apache.spark.sql.connector.catalog.InMemoryPartitionTableCatalog
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.test.{SharedSparkSession, SQLTestUtils}
+import org.apache.spark.unsafe.types.UTF8String
+
+/**
+ * The base trait for SQL INSERT.
+ */
+trait SQLInsertTestSuite extends QueryTest with SQLTestUtils {
+
+  import testImplicits._
+
+  def format: String
+
+  protected def createTable(
+      table: String,
+      cols: Seq[String],
+      colTypes: Seq[String],
+      partCols: Seq[String] = Nil): Unit = {
+    val values = cols.zip(colTypes).map(tuple => tuple._1 + " " + tuple._2).mkString("(", ", ", ")")
+    val partitionSpec = if (partCols.nonEmpty) {
+      partCols.mkString("PARTITIONED BY (", ",", ")")
+    } else ""
+    sql(s"CREATE TABLE $table$values USING $format $partitionSpec")
+  }
+
+  protected def processInsert(
+      tableName: String,
+      input: DataFrame,
+      cols: Seq[String] = Nil,
+      partitionExprs: Seq[String] = Nil,
+      overwrite: Boolean): Unit = {
+    val tmpView = "tmp_view"
+    val columnList = if (cols.nonEmpty) cols.mkString("(", ",", ")") else ""
+    val partitionList = if (partitionExprs.nonEmpty) {
+      partitionExprs.mkString("PARTITION (", ",", ")")
+    } else ""
+    withTempView(tmpView) {
+      input.createOrReplaceTempView(tmpView)
+      val overwriteStr = if (overwrite) "OVERWRITE" else "INTO"
+      sql(
+        s"INSERT $overwriteStr TABLE $tableName $partitionList $columnList SELECT * FROM $tmpView")
+    }
+  }
+
+  protected def verifyTable(tableName: String, expected: DataFrame): Unit = {
+    checkAnswer(spark.table(tableName), expected)
+  }
+
+  test("insert with column list - follow table output order") {
+    withTable("t1") {
+      val df = Seq((1, 2L, "3")).toDF()
+      val cols = Seq("c1", "c2", "c3")
+      createTable("t1", cols, Seq("int", "long", "string"))
+      Seq(false, true).foreach { m =>
+        processInsert("t1", df, cols, overwrite = m)
+        verifyTable("t1", df)
+      }
+    }
+  }
+
+  test("insert with column list - follow table output order + partitioned table") {
+    val cols = Seq("c1", "c2", "c3", "c4")
+    val df = Seq((1, 2, 3, 4)).toDF(cols: _*)
+    withTable("t1") {
+      createTable("t1", cols, Seq("int", "int", "int", "int"), cols.takeRight(2))
+      Seq(false, true).foreach { m =>
+        processInsert("t1", df, cols, overwrite = m)
+        verifyTable("t1", df)
+      }
+    }
+
+    withTable("t1") {
+      createTable("t1", cols, Seq("int", "int", "int", "int"), cols.takeRight(2))
+      Seq(false, true).foreach { m =>
+        processInsert(
+          "t1", df.selectExpr("c1", "c2"), cols.take(2), Seq("c3=3", "c4=4"), overwrite = m)
+        verifyTable("t1", df)
+      }
+    }
+
+    withTable("t1") {
+      createTable("t1", cols, Seq("int", "int", "int", "int"), cols.takeRight(2))
+      Seq(false, true).foreach { m =>
+        processInsert("t1", df.selectExpr("c1", "c2", "c4"),
+          cols.filterNot(_ == "c3"), Seq("c3=3", "c4"), overwrite = m)
+        verifyTable("t1", df)
+      }
+    }
+  }
+
+  test("insert with column list - table output reorder") {
+    withTable("t1") {
+      val cols = Seq("c1", "c2", "c3")
+      val df = Seq((1, 2, 3)).toDF(cols: _*)
+      createTable("t1", cols, Seq("int", "int", "int"))
+      Seq(false, true).foreach { m =>
+        processInsert("t1", df, cols.reverse, overwrite = m)
+        verifyTable("t1", df.selectExpr(cols.reverse: _*))
+      }
+    }
+  }
+
+  test("insert with column list - table output reorder + partitioned table") {
+    val cols = Seq("c1", "c2", "c3", "c4")
+    val df = Seq((1, 2, 3, 4)).toDF(cols: _*)
+    withTable("t1") {
+      createTable("t1", cols, Seq("int", "int", "int", "int"), cols.takeRight(2))
+      Seq(false, true).foreach { m =>
+        processInsert("t1", df, cols.reverse, overwrite = m)
+        verifyTable("t1", df.selectExpr(cols.reverse: _*))
+      }
+    }
+
+    withTable("t1") {
+      createTable("t1", cols, Seq("int", "int", "int", "int"), cols.takeRight(2))
+      Seq(false, true).foreach { m =>
+        processInsert(
+          "t1", df.selectExpr("c1", "c2"), cols.take(2).reverse, Seq("c3=3", "c4=4"), overwrite = m)
+        verifyTable("t1", df.selectExpr("c2", "c1", "c3", "c4"))
+      }
+    }
+
+    withTable("t1") {
+      createTable("t1", cols, Seq("int", "int", "int", "int"), cols.takeRight(2))
+      Seq(false, true).foreach { m =>
+        processInsert("t1",
+          df.selectExpr("c1", "c2", "c4"), Seq("c4", "c2", "c1"), Seq("c3=3", "c4"), overwrite = m)
+        verifyTable("t1", df.selectExpr("c4", "c2", "c3", "c1"))
+      }
+    }
+  }
+
+  test("insert with column list - duplicated columns") {
+    withTable("t1") {
+      val cols = Seq("c1", "c2", "c3")
+      createTable("t1", cols, Seq("int", "long", "string"))
+      val e1 = intercept[AnalysisException](sql(s"INSERT INTO t1 (c1, c2, c2) values(1, 2, 3)"))
+      assert(e1.getMessage.contains("Found duplicate column(s) in the column list: `c2`"))
+    }
+  }
+
+  test("insert with column list - invalid columns") {
+    withTable("t1") {
+      val cols = Seq("c1", "c2", "c3")
+      createTable("t1", cols, Seq("int", "long", "string"))
+      val e1 = intercept[AnalysisException](sql(s"INSERT INTO t1 (c1, c2, c4) values(1, 2, 3)"))
+      assert(e1.getMessage.contains("Cannot resolve column name c4"))
+    }
+  }
+
+  test("insert with column list - mismatched column list size") {
+    val msg = "Cannot write to table due to mismatched user specified column size"
+    withTable("t1") {
+      val cols = Seq("c1", "c2", "c3")
+      createTable("t1", cols, Seq("int", "long", "string"))
+      val e1 = intercept[AnalysisException](sql(s"INSERT INTO t1 (c1, c2) values(1, 2, 3)"))
+      assert(e1.getMessage.contains(msg))
+      val e2 = intercept[AnalysisException](sql(s"INSERT INTO t1 (c1, c2, c3) values(1, 2)"))
+      assert(e2.getMessage.contains(msg))
+    }
+  }
+
+  test("insert with column list - mismatched target table out size after rewritten query") {
+    val v2Msg = "Cannot write to 'testcat.t1', not enough data columns:"
+    val cols = Seq("c1", "c2", "c3", "c4")
+
+    withTable("t1") {
+      createTable("t1", cols, Seq.fill(4)("int"))
+      val e1 = intercept[AnalysisException](sql(s"INSERT INTO t1 (c1) values(1)"))
+      assert(e1.getMessage.contains("target table has 4 column(s) but the inserted data has 1") ||
+        e1.getMessage.contains(v2Msg))
+    }
+
+    withTable("t1") {
+      createTable("t1", cols, Seq.fill(4)("int"), cols.takeRight(2))
+      val e1 = intercept[AnalysisException] {
+        sql(s"INSERT INTO t1 partition(c3=3, c4=4) (c1) values(1)")
+      }
+      assert(e1.getMessage.contains("target table has 4 column(s) but the inserted data has 3") ||
+        e1.getMessage.contains(v2Msg))
+    }
+  }
+
+  test("SPARK-34223: static partition with null raise NPE") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(i STRING, c string) USING PARQUET PARTITIONED BY (c)")
+      sql("INSERT OVERWRITE t PARTITION (c=null) VALUES ('1')")
+      checkAnswer(spark.table("t"), Row("1", null))
+    }
+  }
+
+  test("SPARK-33474: Support typed literals as partition spec values") {
+    withTable("t1") {
+      val binaryStr = "Spark SQL"
+      val binaryHexStr = Hex.hex(UTF8String.fromString(binaryStr).getBytes).toString
+      sql(
+        """
+          | CREATE TABLE t1(name STRING, part1 DATE, part2 TIMESTAMP, part3 BINARY,
+          |  part4 STRING, part5 STRING, part6 STRING, part7 STRING)
+          | USING PARQUET PARTITIONED BY (part1, part2, part3, part4, part5, part6, part7)
+         """.stripMargin)
+
+      sql(
+        s"""
+           | INSERT OVERWRITE t1 PARTITION(
+           | part1 = date'2019-01-01',
+           | part2 = timestamp'2019-01-01 11:11:11',
+           | part3 = X'$binaryHexStr',
+           | part4 = 'p1',
+           | part5 = date'2019-01-01',
+           | part6 = timestamp'2019-01-01 11:11:11',
+           | part7 = X'$binaryHexStr'
+           | ) VALUES('a')
+        """.stripMargin)
+      checkAnswer(sql(
+        """
+          | SELECT
+          |   name,
+          |   CAST(part1 AS STRING),
+          |   CAST(part2 as STRING),
+          |   CAST(part3 as STRING),
+          |   part4,
+          |   part5,
+          |   part6,
+          |   part7
+          | FROM t1
+        """.stripMargin),
+        Row("a", "2019-01-01", "2019-01-01 11:11:11", "Spark SQL", "p1",
+          "2019-01-01", "2019-01-01 11:11:11", "Spark SQL"))
+
+      val e = intercept[AnalysisException] {
+        sql("CREATE TABLE t2(name STRING, part INTERVAL) USING PARQUET PARTITIONED BY (part)")
+      }.getMessage
+      assert(e.contains("Cannot use interval"))
+    }
+  }
+
+  test("SPARK-34556: " +
+    "checking duplicate static partition columns should respect case sensitive conf") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(i STRING, c string) USING PARQUET PARTITIONED BY (c)")
+      val e = intercept[AnalysisException] {
+        sql("INSERT OVERWRITE t PARTITION (c='2', C='3') VALUES (1)")
+      }
+      assert(e.getMessage.contains("Found duplicate keys 'c'"))
+    }
+    // The following code is skipped for Hive because columns stored in Hive Metastore is always
+    // case insensitive and we cannot create such table in Hive Metastore.
+    if (!format.startsWith("hive")) {
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+        withTable("t") {
+          sql(s"CREATE TABLE t(i int, c string, C string) USING PARQUET PARTITIONED BY (c, C)")
+          sql("INSERT OVERWRITE t PARTITION (c='2', C='3') VALUES (1)")
+          checkAnswer(spark.table("t"), Row(1, "2", "3"))
+        }
+      }
+    }
+  }
+
+  test("SPARK-30844: static partition should also follow StoreAssignmentPolicy") {
+    val testingPolicies = if (format == "foo") {
+      // DS v2 doesn't support the legacy policy
+      Seq(SQLConf.StoreAssignmentPolicy.ANSI, SQLConf.StoreAssignmentPolicy.STRICT)
+    } else {
+      SQLConf.StoreAssignmentPolicy.values
+    }
+    testingPolicies.foreach { policy =>
+      withSQLConf(
+        SQLConf.STORE_ASSIGNMENT_POLICY.key -> policy.toString) {
+        withTable("t") {
+          sql("create table t(a int, b string) using parquet partitioned by (a)")
+          policy match {
+            case SQLConf.StoreAssignmentPolicy.ANSI | SQLConf.StoreAssignmentPolicy.STRICT =>
+              val errorMsg = intercept[NumberFormatException] {
+                sql("insert into t partition(a='ansi') values('ansi')")
+              }.getMessage
+              assert(errorMsg.contains("invalid input syntax for type numeric: ansi"))
+            case SQLConf.StoreAssignmentPolicy.LEGACY =>
+              sql("insert into t partition(a='ansi') values('ansi')")
+              checkAnswer(sql("select * from t"), Row("ansi", null) :: Nil)
+          }
+        }
+      }
+    }
+  }
+}
+
+class FileSourceSQLInsertTestSuite extends SQLInsertTestSuite with SharedSparkSession {
+  override def format: String = "parquet"
+  override protected def sparkConf: SparkConf = {
+    super.sparkConf.set(SQLConf.USE_V1_SOURCE_LIST, format)
+  }
+}
+
+class DSV2SQLInsertTestSuite extends SQLInsertTestSuite with SharedSparkSession {
+
+  override def format: String = "foo"
+
+  protected override def sparkConf: SparkConf = {
+    super.sparkConf
+      .set("spark.sql.catalog.testcat", classOf[InMemoryPartitionTableCatalog].getName)
+      .set(SQLConf.DEFAULT_CATALOG.key, "testcat")
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 788a07370195d..26af0b9f81127 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -22,20 +22,25 @@ import java.net.{MalformedURLException, URL}
 import java.sql.{Date, Timestamp}
 import java.util.concurrent.atomic.AtomicBoolean
 
+import org.apache.commons.io.FileUtils
+
 import org.apache.spark.{AccumulatorSuite, SparkException}
 import org.apache.spark.scheduler.{SparkListener, SparkListenerJobStart}
 import org.apache.spark.sql.catalyst.expressions.GenericRow
 import org.apache.spark.sql.catalyst.expressions.aggregate.{Complete, Partial}
 import org.apache.spark.sql.catalyst.optimizer.{ConvertToLocalRelation, NestedColumnAliasingSuite}
-import org.apache.spark.sql.catalyst.plans.logical.Project
+import org.apache.spark.sql.catalyst.plans.logical.{LocalLimit, Project, RepartitionByExpression, Sort}
 import org.apache.spark.sql.catalyst.util.StringUtils
+import org.apache.spark.sql.execution.UnionExec
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
 import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec}
 import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
 import org.apache.spark.sql.execution.command.FunctionsCommand
+import org.apache.spark.sql.execution.datasources.LogicalRelation
 import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
 import org.apache.spark.sql.execution.datasources.v2.orc.OrcScan
 import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetScan
+import org.apache.spark.sql.execution.exchange.ReusedExchangeExec
 import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, CartesianProductExec, SortMergeJoinExec}
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
@@ -43,8 +48,10 @@ import org.apache.spark.sql.test.{SharedSparkSession, TestSQLContext}
 import org.apache.spark.sql.test.SQLTestData._
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.CalendarInterval
+import org.apache.spark.util.ResetSystemProperties
 
-class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlanHelper {
+class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlanHelper
+    with ResetSystemProperties {
   import testImplicits._
 
   setupTestData()
@@ -117,10 +124,18 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
     checkKeywordsExist(sql("describe functioN abcadf"), "Function: abcadf not found.")
   }
 
+  test("SPARK-34678: describe functions for table-valued functions") {
+    checkKeywordsExist(sql("describe function range"),
+      "Function: range",
+      "Class: org.apache.spark.sql.catalyst.plans.logical.Range",
+      "range(end: long)"
+    )
+  }
+
   test("SPARK-14415: All functions should have own descriptions") {
     for (f <- spark.sessionState.functionRegistry.listFunction()) {
       if (!Seq("cube", "grouping", "grouping_id", "rollup", "window").contains(f.unquotedString)) {
-        checkKeywordsNotExist(sql(s"describe function `$f`"), "N/A.")
+        checkKeywordsNotExist(sql(s"describe function $f"), "N/A.")
       }
     }
   }
@@ -1054,6 +1069,21 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
     spark.sessionState.conf.clear()
   }
 
+  test("SPARK-35044: SET command shall display default value for hadoop conf correctly") {
+    val key = "hadoop.this.is.a.test.key"
+    val value = "2018-11-17 13:33:33.333"
+    // these keys are located at `src/test/resources/hive-site.xml`
+    checkAnswer(sql(s"SET $key"), Row(key, value))
+    checkAnswer(sql("SET hadoop.tmp.dir"), Row("hadoop.tmp.dir", "/tmp/hive_one"))
+
+    // these keys does not exist as default yet
+    checkAnswer(sql(s"SET ${key}no"), Row(key + "no", "<undefined>"))
+    checkAnswer(sql("SET dfs.hosts"), Row("dfs.hosts", "<undefined>"))
+
+    // io.file.buffer.size has a default value from `SparkHadoopUtil.newConfiguration`
+    checkAnswer(sql("SET io.file.buffer.size"), Row("io.file.buffer.size", "65536"))
+  }
+
   test("apply schema") {
     withTempView("applySchema1", "applySchema2", "applySchema3") {
       val schema1 = StructType(
@@ -1175,7 +1205,7 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
             |order by struct.a, struct.b
             |""".stripMargin)
     }
-    assert(error.message contains "cannot resolve '`struct.a`' given input columns: [a, b]")
+    assert(error.message contains "cannot resolve 'struct.a' given input columns: [a, b]")
 
   }
 
@@ -1316,7 +1346,7 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
     )
   }
 
-  test("oder by asc by default when not specify ascending and descending") {
+  test("order by asc by default when not specify ascending and descending") {
     checkAnswer(
       sql("SELECT a, b FROM testData2 ORDER BY a desc, b"),
       Seq(Row(3, 1), Row(3, 2), Row(2, 1), Row(2, 2), Row(1, 1), Row(1, 2))
@@ -2763,8 +2793,8 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
         }.message
         assert(
           m.contains(
-            "cannot resolve '(spark_catalog.default.t.`c` = spark_catalog.default.S.`C`)' " +
-            "due to data type mismatch"))
+            "cannot resolve '(spark_catalog.default.t.c = " +
+            "spark_catalog.default.S.C)' due to data type mismatch"))
       }
     }
   }
@@ -2776,7 +2806,7 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
 
       val e = intercept[AnalysisException](sql("SELECT v.i from (SELECT i FROM v)"))
       assert(e.message ==
-        "cannot resolve '`v.i`' given input columns: [__auto_generated_subquery_name.i]")
+        "cannot resolve 'v.i' given input columns: [__auto_generated_subquery_name.i]")
 
       checkAnswer(sql("SELECT __auto_generated_subquery_name.i from (SELECT i FROM v)"), Row(1))
     }
@@ -2812,7 +2842,7 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
     }
   }
 
-  test("SRARK-22266: the same aggregate function was calculated multiple times") {
+  test("SPARK-22266: the same aggregate function was calculated multiple times") {
     val query = "SELECT a, max(b+1), max(b+1) + 1 FROM testData2 GROUP BY a"
     val df = sql(query)
     val physical = df.queryExecution.sparkPlan
@@ -3092,7 +3122,7 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
           assert(scan.isInstanceOf[ParquetScan])
           assert(scan.asInstanceOf[ParquetScan].pushedFilters === filters)
         case _ =>
-          fail(s"unknow format $format")
+          fail(s"unknown format $format")
       }
     }
 
@@ -3495,6 +3525,593 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
     assert(df4.schema.head.name === "randn(1)")
     checkIfSeedExistsInExplain(df2)
   }
+
+  test("SPARK-31670: Trim unnecessary Struct field alias in Aggregate/GroupingSets") {
+    withTempView("t") {
+      sql(
+        """
+          |CREATE TEMPORARY VIEW t(a, b, c) AS
+          |SELECT * FROM VALUES
+          |('A', 1, NAMED_STRUCT('row_id', 1, 'json_string', '{"i": 1}')),
+          |('A', 2, NAMED_STRUCT('row_id', 2, 'json_string', '{"i": 1}')),
+          |('A', 2, NAMED_STRUCT('row_id', 2, 'json_string', '{"i": 2}')),
+          |('B', 1, NAMED_STRUCT('row_id', 3, 'json_string', '{"i": 1}')),
+          |('C', 3, NAMED_STRUCT('row_id', 4, 'json_string', '{"i": 1}'))
+        """.stripMargin)
+
+      checkAnswer(
+        sql(
+          """
+            |SELECT a, c.json_string, SUM(b)
+            |FROM t
+            |GROUP BY a, c.json_string
+            |""".stripMargin),
+        Row("A", "{\"i\": 1}", 3) :: Row("A", "{\"i\": 2}", 2) ::
+          Row("B", "{\"i\": 1}", 1) :: Row("C", "{\"i\": 1}", 3) :: Nil)
+
+      checkAnswer(
+        sql(
+          """
+            |SELECT a, c.json_string, SUM(b)
+            |FROM t
+            |GROUP BY a, c.json_string
+            |WITH CUBE
+            |""".stripMargin),
+        Row("A", "{\"i\": 1}", 3) :: Row("A", "{\"i\": 2}", 2) :: Row("A", null, 5) ::
+          Row("B", "{\"i\": 1}", 1) :: Row("B", null, 1) ::
+          Row("C", "{\"i\": 1}", 3) :: Row("C", null, 3) ::
+          Row(null, "{\"i\": 1}", 7) :: Row(null, "{\"i\": 2}", 2) :: Row(null, null, 9) :: Nil)
+
+      checkAnswer(
+        sql(
+          """
+            |SELECT a, get_json_object(c.json_string, '$.i'), SUM(b)
+            |FROM t
+            |GROUP BY a, get_json_object(c.json_string, '$.i')
+            |WITH CUBE
+            |""".stripMargin),
+        Row("A", "1", 3) :: Row("A", "2", 2) :: Row("A", null, 5) ::
+          Row("B", "1", 1) :: Row("B", null, 1) ::
+          Row("C", "1", 3) :: Row("C", null, 3) ::
+          Row(null, "1", 7) :: Row(null, "2", 2) :: Row(null, null, 9) :: Nil)
+
+      checkAnswer(
+        sql(
+          """
+            |SELECT a, c.json_string AS json_string, SUM(b)
+            |FROM t
+            |GROUP BY a, c.json_string
+            |WITH CUBE
+            |""".stripMargin),
+        Row("A", null, 5) :: Row("A", "{\"i\": 1}", 3) :: Row("A", "{\"i\": 2}", 2) ::
+          Row("B", null, 1) :: Row("B", "{\"i\": 1}", 1) ::
+          Row("C", null, 3) :: Row("C", "{\"i\": 1}", 3) ::
+          Row(null, null, 9) :: Row(null, "{\"i\": 1}", 7) :: Row(null, "{\"i\": 2}", 2) :: Nil)
+
+      checkAnswer(
+        sql(
+          """
+            |SELECT a, c.json_string as js, SUM(b)
+            |FROM t
+            |GROUP BY a, c.json_string
+            |WITH CUBE
+            |""".stripMargin),
+        Row("A", null, 5) :: Row("A", "{\"i\": 1}", 3) :: Row("A", "{\"i\": 2}", 2) ::
+          Row("B", null, 1) :: Row("B", "{\"i\": 1}", 1) ::
+          Row("C", null, 3) :: Row("C", "{\"i\": 1}", 3) ::
+          Row(null, null, 9) :: Row(null, "{\"i\": 1}", 7) :: Row(null, "{\"i\": 2}", 2) :: Nil)
+
+      checkAnswer(
+        sql(
+          """
+            |SELECT a, c.json_string as js, SUM(b)
+            |FROM t
+            |GROUP BY a, c.json_string
+            |WITH ROLLUP
+            |""".stripMargin),
+        Row("A", null, 5) :: Row("A", "{\"i\": 1}", 3) :: Row("A", "{\"i\": 2}", 2) ::
+          Row("B", null, 1) :: Row("B", "{\"i\": 1}", 1) ::
+          Row("C", null, 3) :: Row("C", "{\"i\": 1}", 3) ::
+          Row(null, null, 9) :: Nil)
+
+      checkAnswer(
+        sql(
+          """
+            |SELECT a, c.json_string, SUM(b)
+            |FROM t
+            |GROUP BY a, c.json_string
+            |GROUPING sets((a),(a, c.json_string))
+            |""".stripMargin),
+        Row("A", null, 5) :: Row("A", "{\"i\": 1}", 3) :: Row("A", "{\"i\": 2}", 2) ::
+          Row("B", null, 1) :: Row("B", "{\"i\": 1}", 1) ::
+          Row("C", null, 3) :: Row("C", "{\"i\": 1}", 3) :: Nil)
+    }
+  }
+
+  test("SPARK-31761: test byte, short, integer overflow for (Divide) integral type") {
+    checkAnswer(sql("Select -2147483648 DIV -1"), Seq(Row(Integer.MIN_VALUE.toLong * -1)))
+    checkAnswer(sql("select CAST(-128 as Byte) DIV CAST (-1 as Byte)"),
+      Seq(Row(Byte.MinValue.toLong * -1)))
+    checkAnswer(sql("select CAST(-32768 as short) DIV CAST (-1 as short)"),
+      Seq(Row(Short.MinValue.toLong * -1)))
+  }
+
+  test("normalize special floating numbers in subquery") {
+    withTempView("v1", "v2", "v3") {
+      Seq(-0.0).toDF("d").createTempView("v1")
+      Seq(0.0).toDF("d").createTempView("v2")
+      spark.range(2).createTempView("v3")
+
+      // non-correlated subquery
+      checkAnswer(sql("SELECT (SELECT v1.d FROM v1 JOIN v2 ON v1.d = v2.d)"), Row(-0.0))
+      // correlated subquery
+      checkAnswer(
+        sql(
+          """
+            |SELECT id FROM v3 WHERE EXISTS
+            |  (SELECT v1.d FROM v1 JOIN v2 ON v1.d = v2.d WHERE id > 0)
+            |""".stripMargin), Row(1))
+    }
+  }
+
+  test("SPARK-31875: remove hints from plan when spark.sql.optimizer.disableHints = true") {
+    withSQLConf(SQLConf.DISABLE_HINTS.key -> "true") {
+      withTempView("t1", "t2") {
+        Seq[Integer](1, 2).toDF("c1").createOrReplaceTempView("t1")
+        Seq[Integer](1, 2).toDF("c1").createOrReplaceTempView("t2")
+        val repartitionHints = Seq(
+          "COALESCE(2)",
+          "REPARTITION(c1)",
+          "REPARTITION(c1, 2)",
+          "REPARTITION_BY_RANGE(c1, 2)",
+          "REPARTITION_BY_RANGE(c1)"
+        )
+        val joinHints = Seq(
+          "BROADCASTJOIN (t1)",
+          "MAPJOIN(t1)",
+          "SHUFFLE_MERGE(t1)",
+          "MERGEJOIN(t1)",
+          "SHUFFLE_REPLICATE_NL(t1)"
+        )
+
+        repartitionHints.foreach { hintName =>
+          val sqlText = s"SELECT /*+ $hintName */ * FROM t1"
+          val sqlTextWithoutHint = "SELECT * FROM t1"
+          val expectedPlan = sql(sqlTextWithoutHint)
+          val actualPlan = sql(sqlText)
+          comparePlans(actualPlan.queryExecution.analyzed, expectedPlan.queryExecution.analyzed)
+        }
+
+        joinHints.foreach { hintName =>
+          val sqlText = s"SELECT /*+ $hintName */ * FROM t1 INNER JOIN t2 ON t1.c1 = t2.c1"
+          val sqlTextWithoutHint = "SELECT * FROM t1 INNER JOIN t2 ON t1.c1 = t2.c1"
+          val expectedPlan = sql(sqlTextWithoutHint)
+          val actualPlan = sql(sqlText)
+          comparePlans(actualPlan.queryExecution.analyzed, expectedPlan.queryExecution.analyzed)
+        }
+      }
+    }
+  }
+
+  test("SPARK-32372: ResolveReferences.dedupRight should only rewrite attributes for ancestor " +
+    "plans of the conflict plan") {
+    sql("SELECT name, avg(age) as avg_age FROM person GROUP BY name")
+      .createOrReplaceTempView("person_a")
+    sql("SELECT p1.name, p2.avg_age FROM person p1 JOIN person_a p2 ON p1.name = p2.name")
+      .createOrReplaceTempView("person_b")
+    sql("SELECT * FROM person_a UNION SELECT * FROM person_b")
+      .createOrReplaceTempView("person_c")
+    checkAnswer(
+      sql("SELECT p1.name, p2.avg_age FROM person_c p1 JOIN person_c p2 ON p1.name = p2.name"),
+      Row("jim", 20.0) :: Row("mike", 30.0) :: Nil)
+  }
+
+  test("SPARK-32280: Avoid duplicate rewrite attributes when there're multiple JOINs") {
+    sql("SELECT 1 AS id").createOrReplaceTempView("A")
+    sql("SELECT id, 'foo' AS kind FROM A").createOrReplaceTempView("B")
+    sql("SELECT l.id as id FROM B AS l LEFT SEMI JOIN B AS r ON l.kind = r.kind")
+      .createOrReplaceTempView("C")
+    checkAnswer(sql("SELECT 0 FROM ( SELECT * FROM B JOIN C USING (id)) " +
+      "JOIN ( SELECT * FROM B JOIN C USING (id)) USING (id)"), Row(0))
+  }
+
+  test("SPARK-32788: non-partitioned table scan should not have partition filter") {
+    withTable("t") {
+      spark.range(1).write.saveAsTable("t")
+      checkAnswer(sql("SELECT id FROM t WHERE (SELECT true)"), Row(0L))
+    }
+  }
+
+  test("SPARK-33306: Timezone is needed when cast Date to String") {
+    withTempView("t1", "t2") {
+      spark.sql("select to_date(concat('2000-01-0', id)) as d from range(1, 2)")
+        .createOrReplaceTempView("t1")
+      spark.sql("select concat('2000-01-0', id) as d from range(1, 2)")
+        .createOrReplaceTempView("t2")
+      val result = Date.valueOf("2000-01-01")
+
+      checkAnswer(sql("select t1.d from t1 join t2 on t1.d = t2.d"), Row(result))
+      withSQLConf(SQLConf.LEGACY_CAST_DATETIME_TO_STRING.key -> "true") {
+        checkAnswer(sql("select t1.d from t1 join t2 on t1.d = t2.d"), Row(result))
+      }
+    }
+  }
+
+  test("SPARK-33338: GROUP BY using literal map should not fail") {
+    withTempDir { dir =>
+      sql(s"CREATE TABLE t USING ORC LOCATION '${dir.toURI}' AS SELECT map('k1', 'v1') m, 'k1' k")
+      Seq(
+        "SELECT map('k1', 'v1')[k] FROM t GROUP BY 1",
+        "SELECT map('k1', 'v1')[k] FROM t GROUP BY map('k1', 'v1')[k]",
+        "SELECT map('k1', 'v1')[k] a FROM t GROUP BY a").foreach { statement =>
+        checkAnswer(sql(statement), Row("v1"))
+      }
+    }
+  }
+
+  test("SPARK-33084: Add jar support Ivy URI in SQL") {
+    val sc = spark.sparkContext
+    val hiveVersion = "2.3.8"
+    // transitive=false, only download specified jar
+    sql(s"ADD JAR ivy://org.apache.hive.hcatalog:hive-hcatalog-core:$hiveVersion?transitive=false")
+    assert(sc.listJars()
+      .exists(_.contains(s"org.apache.hive.hcatalog_hive-hcatalog-core-$hiveVersion.jar")))
+
+    // default transitive=true, test download ivy URL jar return multiple jars
+    sql("ADD JAR ivy://org.scala-js:scalajs-test-interface_2.12:1.2.0")
+    assert(sc.listJars().exists(_.contains("scalajs-library_2.12")))
+    assert(sc.listJars().exists(_.contains("scalajs-test-interface_2.12")))
+
+    sql(s"ADD JAR ivy://org.apache.hive:hive-contrib:$hiveVersion" +
+      "?exclude=org.pentaho:pentaho-aggdesigner-algorithm&transitive=true")
+    assert(sc.listJars().exists(_.contains(s"org.apache.hive_hive-contrib-$hiveVersion.jar")))
+    assert(sc.listJars().exists(_.contains(s"org.apache.hive_hive-exec-$hiveVersion.jar")))
+    assert(!sc.listJars().exists(_.contains("org.pentaho.pentaho_aggdesigner-algorithm")))
+  }
+
+  test("SPARK-33677: LikeSimplification should be skipped if pattern contains any escapeChar") {
+    withTempView("df") {
+      Seq("m@ca").toDF("s").createOrReplaceTempView("df")
+
+      val e = intercept[AnalysisException] {
+        sql("SELECT s LIKE 'm%@ca' ESCAPE '%' FROM df").collect()
+      }
+      assert(e.message.contains("the pattern 'm%@ca' is invalid, " +
+        "the escape character is not allowed to precede '@'"))
+
+      checkAnswer(sql("SELECT s LIKE 'm@@ca' ESCAPE '@' FROM df"), Row(true))
+    }
+  }
+
+  test("limit partition num to 1 when distributing by foldable expressions") {
+    withSQLConf((SQLConf.SHUFFLE_PARTITIONS.key, "5")) {
+      Seq(1, "1, 2", null, "version()").foreach { expr =>
+        val plan = sql(s"select * from values (1), (2), (3) t(a) distribute by $expr")
+          .queryExecution.optimizedPlan
+        val res = plan.collect {
+          case r: RepartitionByExpression if r.numPartitions == 1 => true
+        }
+        assert(res.nonEmpty)
+      }
+    }
+  }
+
+  test("Fold RepartitionExpression num partition should check if partition expression is empty") {
+    withSQLConf((SQLConf.SHUFFLE_PARTITIONS.key, "5")) {
+      val df = spark.range(1).hint("REPARTITION_BY_RANGE")
+      val plan = df.queryExecution.optimizedPlan
+      val res = plan.collect {
+        case r: RepartitionByExpression if r.numPartitions == 5 => true
+      }
+      assert(res.nonEmpty)
+    }
+  }
+
+  test("SPARK-34030: Fold RepartitionExpression num partition should at Optimizer") {
+    withSQLConf((SQLConf.SHUFFLE_PARTITIONS.key, "2")) {
+      Seq(1, "1, 2", null, "version()").foreach { expr =>
+        val plan = sql(s"select * from values (1), (2), (3) t(a) distribute by $expr")
+          .queryExecution.analyzed
+        val res = plan.collect {
+          case r: RepartitionByExpression if r.numPartitions == 2 => true
+        }
+        assert(res.nonEmpty)
+      }
+    }
+  }
+
+  test("SPARK-33591: null as string partition literal value 'null' after setting legacy conf") {
+    withSQLConf(SQLConf.LEGACY_PARSE_NULL_PARTITION_SPEC_AS_STRING_LITERAL.key -> "true") {
+      val t = "tbl"
+      withTable("tbl") {
+        sql(s"CREATE TABLE $t (col1 INT, p1 STRING) USING PARQUET PARTITIONED BY (p1)")
+        sql(s"INSERT INTO TABLE $t PARTITION (p1 = null) SELECT 0")
+        checkAnswer(spark.sql(s"SELECT * FROM $t"), Row(0, "null"))
+      }
+    }
+  }
+
+  test("SPARK-33593: Vector reader got incorrect data with binary partition value") {
+    Seq("false", "true").foreach(value => {
+      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> value) {
+        withTable("t1") {
+          sql(
+            """CREATE TABLE t1(name STRING, id BINARY, part BINARY)
+              |USING PARQUET PARTITIONED BY (part)""".stripMargin)
+          sql("INSERT INTO t1 PARTITION(part = 'Spark SQL') VALUES('a', X'537061726B2053514C')")
+          checkAnswer(sql("SELECT name, cast(id as string), cast(part as string) FROM t1"),
+            Row("a", "Spark SQL", "Spark SQL"))
+        }
+      }
+
+      withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> value) {
+        withTable("t2") {
+          sql(
+            """CREATE TABLE t2(name STRING, id BINARY, part BINARY)
+              |USING ORC PARTITIONED BY (part)""".stripMargin)
+          sql("INSERT INTO t2 PARTITION(part = 'Spark SQL') VALUES('a', X'537061726B2053514C')")
+          checkAnswer(sql("SELECT name, cast(id as string), cast(part as string) FROM t2"),
+            Row("a", "Spark SQL", "Spark SQL"))
+        }
+      }
+    })
+  }
+
+  test("SPARK-33084: Add jar support Ivy URI in SQL -- jar contains udf class") {
+    val sumFuncClass = "org.apache.spark.examples.sql.Spark33084"
+    val functionName = "test_udf"
+    withTempDir { dir =>
+      System.setProperty("ivy.home", dir.getAbsolutePath)
+      val sourceJar = new File(Thread.currentThread().getContextClassLoader
+        .getResource("SPARK-33084.jar").getFile)
+      val targetCacheJarDir = new File(dir.getAbsolutePath +
+        "/local/org.apache.spark/SPARK-33084/1.0/jars/")
+      targetCacheJarDir.mkdir()
+      // copy jar to local cache
+      FileUtils.copyFileToDirectory(sourceJar, targetCacheJarDir)
+      withTempView("v1") {
+        withUserDefinedFunction(
+          s"default.$functionName" -> false,
+          functionName -> true) {
+          // create temporary function without class
+          val e = intercept[AnalysisException] {
+            sql(s"CREATE TEMPORARY FUNCTION $functionName AS '$sumFuncClass'")
+          }.getMessage
+          assert(e.contains("Can not load class 'org.apache.spark.examples.sql.Spark33084"))
+          sql("ADD JAR ivy://org.apache.spark:SPARK-33084:1.0")
+          sql(s"CREATE TEMPORARY FUNCTION $functionName AS '$sumFuncClass'")
+          // create a view using a function in 'default' database
+          sql(s"CREATE TEMPORARY VIEW v1 AS SELECT $functionName(col1) FROM VALUES (1), (2), (3)")
+          // view v1 should still using function defined in `default` database
+          checkAnswer(sql("SELECT * FROM v1"), Seq(Row(2.0)))
+        }
+      }
+    }
+  }
+
+  test("SPARK-33964: Combine distinct unions that have noop project between them") {
+    val df = sql("""
+      |SELECT a, b FROM (
+      |  SELECT a, b FROM testData2
+      |  UNION
+      |  SELECT a, sum(b) FROM testData2 GROUP BY a
+      |  UNION
+      |  SELECT null AS a, sum(b) FROM testData2
+      |)""".stripMargin)
+
+    val unions = df.queryExecution.sparkPlan.collect {
+      case u: UnionExec => u
+    }
+
+    assert(unions.size == 1)
+  }
+
+  test("SPARK-34421: Resolve temporary objects in temporary views with CTEs") {
+    val tempFuncName = "temp_func"
+    withUserDefinedFunction(tempFuncName -> true) {
+      spark.udf.register(tempFuncName, identity[Int](_))
+
+      val tempViewName = "temp_view"
+      withTempView(tempViewName) {
+        sql(s"CREATE TEMPORARY VIEW $tempViewName AS SELECT 1")
+
+        val testViewName = "test_view"
+
+        withTempView(testViewName) {
+          sql(
+            s"""
+              |CREATE TEMPORARY VIEW $testViewName AS
+              |WITH cte AS (
+              |  SELECT $tempFuncName(0)
+              |)
+              |SELECT * FROM cte
+              |""".stripMargin)
+          checkAnswer(sql(s"SELECT * FROM $testViewName"), Row(0))
+        }
+
+        withTempView(testViewName) {
+          sql(
+            s"""
+              |CREATE TEMPORARY VIEW $testViewName AS
+              |WITH cte AS (
+              |  SELECT * FROM $tempViewName
+              |)
+              |SELECT * FROM cte
+              |""".stripMargin)
+          checkAnswer(sql(s"SELECT * FROM $testViewName"), Row(1))
+        }
+      }
+    }
+  }
+
+  test("SPARK-34421: Resolve temporary objects in permanent views with CTEs") {
+    val tempFuncName = "temp_func"
+    withUserDefinedFunction((tempFuncName, true)) {
+      spark.udf.register(tempFuncName, identity[Int](_))
+
+      val tempViewName = "temp_view"
+      withTempView(tempViewName) {
+        sql(s"CREATE TEMPORARY VIEW $tempViewName AS SELECT 1")
+
+        val testViewName = "test_view"
+
+        val e = intercept[AnalysisException] {
+          sql(
+            s"""
+              |CREATE VIEW $testViewName AS
+              |WITH cte AS (
+              |  SELECT * FROM $tempViewName
+              |)
+              |SELECT * FROM cte
+              |""".stripMargin)
+        }
+        assert(e.message.contains("Not allowed to create a permanent view " +
+          s"`default`.`$testViewName` by referencing a temporary view $tempViewName"))
+
+        val e2 = intercept[AnalysisException] {
+          sql(
+            s"""
+              |CREATE VIEW $testViewName AS
+              |WITH cte AS (
+              |  SELECT $tempFuncName(0)
+              |)
+              |SELECT * FROM cte
+              |""".stripMargin)
+        }
+        assert(e2.message.contains("Not allowed to create a permanent view " +
+          s"`default`.`$testViewName` by referencing a temporary function `$tempFuncName`"))
+      }
+    }
+  }
+
+  test("SPARK-26138 Pushdown limit through InnerLike when condition is empty") {
+    withTable("t1", "t2") {
+      spark.range(5).repartition(1).write.saveAsTable("t1")
+      spark.range(5).repartition(1).write.saveAsTable("t2")
+      val df = spark.sql("SELECT * FROM t1 CROSS JOIN t2 LIMIT 3")
+      val pushedLocalLimits = df.queryExecution.optimizedPlan.collect {
+        case l @ LocalLimit(_, _: LogicalRelation) => l
+      }
+      assert(pushedLocalLimits.length === 2)
+      checkAnswer(df, Row(0, 0) :: Row(0, 1) :: Row(0, 2) :: Nil)
+    }
+  }
+
+  test("SPARK-34514: Push down limit through LEFT SEMI and LEFT ANTI join") {
+    withTable("left_table", "nonempty_right_table", "empty_right_table") {
+      spark.range(5).toDF().repartition(1).write.saveAsTable("left_table")
+      spark.range(3).write.saveAsTable("nonempty_right_table")
+      spark.range(0).write.saveAsTable("empty_right_table")
+      Seq("LEFT SEMI", "LEFT ANTI").foreach { joinType =>
+        val joinWithNonEmptyRightDf = spark.sql(
+          s"SELECT * FROM left_table $joinType JOIN nonempty_right_table LIMIT 3")
+        val joinWithEmptyRightDf = spark.sql(
+          s"SELECT * FROM left_table $joinType JOIN empty_right_table LIMIT 3")
+
+        Seq(joinWithNonEmptyRightDf, joinWithEmptyRightDf).foreach { df =>
+          val pushedLocalLimits = df.queryExecution.optimizedPlan.collect {
+            case l @ LocalLimit(_, _: LogicalRelation) => l
+          }
+          assert(pushedLocalLimits.length === 1)
+        }
+
+        val expectedAnswer = Seq(Row(0), Row(1), Row(2))
+        if (joinType == "LEFT SEMI") {
+          checkAnswer(joinWithNonEmptyRightDf, expectedAnswer)
+          checkAnswer(joinWithEmptyRightDf, Seq.empty)
+        } else {
+          checkAnswer(joinWithNonEmptyRightDf, Seq.empty)
+          checkAnswer(joinWithEmptyRightDf, expectedAnswer)
+        }
+      }
+    }
+  }
+
+  test("SPARK-34575 Push down limit through window when partitionSpec is empty") {
+    withTable("t1") {
+      val numRows = 10
+      spark.range(numRows)
+        .selectExpr("if (id % 2 = 0, null, id) AS a", s"$numRows - id AS b")
+        .write
+        .saveAsTable("t1")
+
+      val df1 = spark.sql(
+        """
+          |SELECT a, b, ROW_NUMBER() OVER(ORDER BY a, b) AS rn
+          |FROM t1 LIMIT 3
+          |""".stripMargin)
+      val pushedLocalLimits1 = df1.queryExecution.optimizedPlan.collect {
+        case l @ LocalLimit(_, _: Sort) => l
+      }
+      assert(pushedLocalLimits1.length === 1)
+      checkAnswer(df1, Seq(Row(null, 2, 1), Row(null, 4, 2), Row(null, 6, 3)))
+
+      val df2 = spark.sql(
+        """
+          |SELECT b, RANK() OVER(ORDER BY a, b) AS rk, DENSE_RANK(b) OVER(ORDER BY a, b) AS s
+          |FROM t1 LIMIT 2
+          |""".stripMargin)
+      val pushedLocalLimits2 = df2.queryExecution.optimizedPlan.collect {
+        case l @ LocalLimit(_, _: Sort) => l
+      }
+      assert(pushedLocalLimits2.length === 1)
+      checkAnswer(df2, Seq(Row(2, 1, 1), Row(4, 2, 2)))
+    }
+  }
+
+  test("SPARK-34796: Avoid code-gen compilation error for LIMIT query") {
+    withTable("left_table", "empty_right_table", "output_table") {
+      spark.range(5).toDF("k").write.saveAsTable("left_table")
+      spark.range(0).toDF("k").write.saveAsTable("empty_right_table")
+
+      withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
+        spark.sql("CREATE TABLE output_table (k INT) USING parquet")
+        spark.sql(
+          """
+            |INSERT INTO TABLE output_table
+            |SELECT t1.k FROM left_table t1
+            |JOIN empty_right_table t2
+            |ON t1.k = t2.k
+            |LIMIT 3
+          """.stripMargin)
+      }
+    }
+  }
+
+  test("SPARK-33482: Fix FileScan canonicalization") {
+    withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") {
+      withTempPath { path =>
+        spark.range(5).toDF().write.mode("overwrite").parquet(path.toString)
+        withTempView("t") {
+          spark.read.parquet(path.toString).createOrReplaceTempView("t")
+          val df = sql(
+            """
+              |SELECT *
+              |FROM t AS t1
+              |JOIN t AS t2 ON t2.id = t1.id
+              |JOIN t AS t3 ON t3.id = t2.id
+              |""".stripMargin)
+          df.collect()
+          val reusedExchanges = collect(df.queryExecution.executedPlan) {
+            case r: ReusedExchangeExec => r
+          }
+          assert(reusedExchanges.size == 1)
+        }
+      }
+    }
+  }
+
+  test("SPARK-35331: Fix resolving original expression in RepartitionByExpression after aliased") {
+    Seq("CLUSTER", "DISTRIBUTE").foreach { keyword =>
+      Seq("a", "substr(a, 0, 3)").foreach { expr =>
+        val clause = keyword + " by " + expr
+        withClue(clause) {
+          checkAnswer(sql(s"select a b from values('123') t(a) $clause"), Row("123"))
+        }
+      }
+    }
+    checkAnswer(sql(s"select /*+ REPARTITION(3, a) */ a b from values('123') t(a)"), Row("123"))
+  }
 }
 
 case class Foo(bar: Option[String])
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestHelper.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestHelper.scala
new file mode 100644
index 0000000000000..b9ca2a0f034e5
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestHelper.scala
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import scala.util.control.NonFatal
+
+import org.apache.spark.SparkException
+import org.apache.spark.sql.catalyst.planning.PhysicalOperation
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.execution.HiveResult.hiveResultString
+import org.apache.spark.sql.execution.SQLExecution
+import org.apache.spark.sql.execution.command.{DescribeColumnCommand, DescribeCommandBase}
+import org.apache.spark.sql.types.StructType
+
+trait SQLQueryTestHelper {
+
+  private val notIncludedMsg = "[not included in comparison]"
+  private val clsName = this.getClass.getCanonicalName
+  protected val emptySchema = StructType(Seq.empty).catalogString
+
+  protected def replaceNotIncludedMsg(line: String): String = {
+    line.replaceAll("#\\d+", "#x")
+      .replaceAll(
+        s"Location.*$clsName/",
+        s"Location $notIncludedMsg/{warehouse_dir}/")
+      .replaceAll("Created By.*", s"Created By $notIncludedMsg")
+      .replaceAll("Created Time.*", s"Created Time $notIncludedMsg")
+      .replaceAll("Last Access.*", s"Last Access $notIncludedMsg")
+      .replaceAll("Partition Statistics\t\\d+", s"Partition Statistics\t$notIncludedMsg")
+      .replaceAll("\\*\\(\\d+\\) ", "*") // remove the WholeStageCodegen codegenStageIds
+  }
+
+
+  /** Executes a query and returns the result as (schema of the output, normalized output). */
+  protected def getNormalizedResult(session: SparkSession, sql: String): (String, Seq[String]) = {
+    // Returns true if the plan is supposed to be sorted.
+    def isSorted(plan: LogicalPlan): Boolean = plan match {
+      case _: Join | _: Aggregate | _: Generate | _: Sample | _: Distinct => false
+      case _: DescribeCommandBase
+          | _: DescribeColumnCommand
+          | _: DescribeRelation
+          | _: DescribeColumn => true
+      case PhysicalOperation(_, _, Sort(_, true, _)) => true
+      case _ => plan.children.iterator.exists(isSorted)
+    }
+
+    val df = session.sql(sql)
+    val schema = df.schema.catalogString
+    // Get answer, but also get rid of the #1234 expression ids that show up in explain plans
+    val answer = SQLExecution.withNewExecutionId(df.queryExecution, Some(sql)) {
+      hiveResultString(df.queryExecution.executedPlan).map(replaceNotIncludedMsg)
+    }
+
+    // If the output is not pre-sorted, sort it.
+    if (isSorted(df.queryExecution.analyzed)) (schema, answer) else (schema, answer.sorted)
+  }
+
+  /**
+   * This method handles exceptions occurred during query execution as they may need special care
+   * to become comparable to the expected output.
+   *
+   * @param result a function that returns a pair of schema and output
+   */
+  protected def handleExceptions(result: => (String, Seq[String])): (String, Seq[String]) = {
+    try {
+      result
+    } catch {
+      case a: AnalysisException =>
+        // Do not output the logical plan tree which contains expression IDs.
+        // Also implement a crude way of masking expression IDs in the error message
+        // with a generic pattern "###".
+        val msg = if (a.plan.nonEmpty) a.getSimpleMessage else a.getMessage
+        (emptySchema, Seq(a.getClass.getName, msg.replaceAll("#\\d+", "#x")))
+      case s: SparkException if s.getCause != null =>
+        // For a runtime exception, it is hard to match because its message contains
+        // information of stage, task ID, etc.
+        // To make result matching simpler, here we match the cause of the exception if it exists.
+        val cause = s.getCause
+        (emptySchema, Seq(cause.getClass.getName, cause.getMessage))
+      case NonFatal(e) =>
+        // If there is an exception, put the exception class followed by the message.
+        (emptySchema, Seq(e.getClass.getName, e.getMessage))
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
index 92da58c27a141..609e0094f9ee4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
@@ -18,26 +18,22 @@
 package org.apache.spark.sql
 
 import java.io.File
+import java.net.URI
 import java.util.Locale
-import java.util.regex.Pattern
 
-import scala.collection.mutable.{ArrayBuffer, HashMap}
-import scala.util.control.NonFatal
+import scala.collection.mutable.ArrayBuffer
 
-import org.apache.spark.{SparkConf, SparkException}
+import org.apache.spark.{SparkConf, TestUtils}
 import org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator
-import org.apache.spark.sql.catalyst.planning.PhysicalOperation
-import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.catalyst.rules.RuleExecutor
 import org.apache.spark.sql.catalyst.util.{fileToString, stringToFile}
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.NANOS_PER_SECOND
-import org.apache.spark.sql.execution.{SQLExecution, WholeStageCodegenExec}
-import org.apache.spark.sql.execution.HiveResult.hiveResultString
-import org.apache.spark.sql.execution.command.{DescribeColumnCommand, DescribeCommandBase}
+import org.apache.spark.sql.execution.WholeStageCodegenExec
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
-import org.apache.spark.sql.types.StructType
 import org.apache.spark.tags.ExtendedSQLTest
+import org.apache.spark.util.Utils
 
 /**
  * End-to-end test cases for SQL queries.
@@ -47,22 +43,22 @@ import org.apache.spark.tags.ExtendedSQLTest
  *
  * To run the entire test suite:
  * {{{
- *   build/sbt "sql/test-only *SQLQueryTestSuite"
+ *   build/sbt "sql/testOnly *SQLQueryTestSuite"
  * }}}
  *
  * To run a single test file upon change:
  * {{{
- *   build/sbt "~sql/test-only *SQLQueryTestSuite -- -z inline-table.sql"
+ *   build/sbt "~sql/testOnly *SQLQueryTestSuite -- -z inline-table.sql"
  * }}}
  *
  * To re-generate golden files for entire suite, run:
  * {{{
- *   SPARK_GENERATE_GOLDEN_FILES=1 build/sbt "sql/test-only *SQLQueryTestSuite"
+ *   SPARK_GENERATE_GOLDEN_FILES=1 build/sbt "sql/testOnly *SQLQueryTestSuite"
  * }}}
  *
  * To re-generate golden file for a single test, run:
  * {{{
- *   SPARK_GENERATE_GOLDEN_FILES=1 build/sbt "sql/test-only *SQLQueryTestSuite -- -z describe.sql"
+ *   SPARK_GENERATE_GOLDEN_FILES=1 build/sbt "sql/testOnly *SQLQueryTestSuite -- -z describe.sql"
  * }}}
  *
  * The format for input files is simple:
@@ -124,7 +120,8 @@ import org.apache.spark.tags.ExtendedSQLTest
  * different types of UDFs. See 'udf/udf-inner-join.sql' as an example.
  */
 @ExtendedSQLTest
-class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
+class SQLQueryTestSuite extends QueryTest with SharedSparkSession with SQLHelper
+    with SQLQueryTestHelper {
 
   import IntegratedUDFTestUtils._
 
@@ -134,14 +131,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
     // We use a path based on Spark home for 2 reasons:
     //   1. Maven can't get correct resource directory when resources in other jars.
     //   2. We test subclasses in the hive-thriftserver module.
-    val sparkHome = {
-      assert(sys.props.contains("spark.test.home") ||
-        sys.env.contains("SPARK_HOME"), "spark.test.home or SPARK_HOME is not set.")
-      sys.props.getOrElse("spark.test.home", sys.env("SPARK_HOME"))
-    }
-
-    java.nio.file.Paths.get(sparkHome,
-      "sql", "core", "src", "test", "resources", "sql-tests").toFile
+    getWorkspaceFilePath("sql", "core", "src", "test", "resources", "sql-tests").toFile
   }
 
   protected val inputFilePath = new File(baseResourcePath, "inputs").getAbsolutePath
@@ -149,19 +139,18 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
 
   protected val validFileExtensions = ".sql"
 
-  private val notIncludedMsg = "[not included in comparison]"
-  private val clsName = this.getClass.getCanonicalName
-
-  protected val emptySchema = StructType(Seq.empty).catalogString
-
   protected override def sparkConf: SparkConf = super.sparkConf
     // Fewer shuffle partitions to speed up testing.
     .set(SQLConf.SHUFFLE_PARTITIONS, 4)
 
+  // SPARK-32106 Since we add SQL test 'transform.sql' will use `cat` command,
+  // here we need to ignore it.
+  private val otherIgnoreList =
+    if (TestUtils.testCommandAvailable("/bin/bash")) Nil else Set("transform.sql")
   /** List of test cases to ignore, in lower cases. */
-  protected def blackList: Set[String] = Set(
-    "blacklist.sql"   // Do NOT remove this one. It is here to test the blacklist functionality.
-  )
+  protected def ignoreList: Set[String] = Set(
+    "ignored.sql" // Do NOT remove this one. It is here to test the ignore functionality.
+  ) ++ otherIgnoreList
 
   // Create all the test cases.
   listTestCases.foreach(createScalaTestCase)
@@ -228,7 +217,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
       name: String, inputFile: String, resultFile: String) extends TestCase with AnsiTest
 
   protected def createScalaTestCase(testCase: TestCase): Unit = {
-    if (blackList.exists(t =>
+    if (ignoreList.exists(t =>
         testCase.name.toLowerCase(Locale.ROOT).contains(t.toLowerCase(Locale.ROOT)))) {
       // Create a test case to ignore this case.
       ignore(testCase.name) { /* Do nothing */ }
@@ -282,18 +271,18 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
     val allCode = importedCode ++ code
     val tempQueries = if (allCode.exists(_.trim.startsWith("--QUERY-DELIMITER"))) {
       // Although the loop is heavy, only used for bracketed comments test.
-      val querys = new ArrayBuffer[String]
+      val queries = new ArrayBuffer[String]
       val otherCodes = new ArrayBuffer[String]
       var tempStr = ""
       var start = false
       for (c <- allCode) {
         if (c.trim.startsWith("--QUERY-DELIMITER-START")) {
           start = true
-          querys ++= splitWithSemicolon(otherCodes.toSeq)
+          queries ++= splitWithSemicolon(otherCodes.toSeq)
           otherCodes.clear()
         } else if (c.trim.startsWith("--QUERY-DELIMITER-END")) {
           start = false
-          querys += s"\n${tempStr.stripSuffix(";")}"
+          queries += s"\n${tempStr.stripSuffix(";")}"
           tempStr = ""
         } else if (start) {
           tempStr += s"\n$c"
@@ -302,9 +291,9 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
         }
       }
       if (otherCodes.nonEmpty) {
-        querys ++= splitWithSemicolon(otherCodes.toSeq)
+        queries ++= splitWithSemicolon(otherCodes.toSeq)
       }
-      querys.toSeq
+      queries.toSeq
     } else {
       splitWithSemicolon(allCode).toSeq
     }
@@ -378,6 +367,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
         // vol used by boolean.sql and case.sql.
         localSparkSession.udf.register("vol", (s: String) => s)
         localSparkSession.conf.set(SQLConf.ANSI_ENABLED.key, true)
+        localSparkSession.conf.set(SQLConf.LEGACY_INTERVAL_ENABLED.key, true)
       case _: AnsiTest =>
         localSparkSession.conf.set(SQLConf.ANSI_ENABLED.key, true)
       case _ =>
@@ -405,7 +395,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
       val goldenOutput = {
         s"-- Automatically generated by ${getClass.getSimpleName}\n" +
         s"-- Number of queries: ${outputs.size}\n\n\n" +
-        outputs.zipWithIndex.map{case (qr, i) => qr.toString}.mkString("\n\n\n") + "\n"
+        outputs.mkString("\n\n\n") + "\n"
       }
       val resultFile = new File(testCase.resultFile)
       val parent = resultFile.getParentFile
@@ -470,70 +460,6 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
     }
   }
 
-  /**
-   * This method handles exceptions occurred during query execution as they may need special care
-   * to become comparable to the expected output.
-   *
-   * @param result a function that returns a pair of schema and output
-   */
-  protected def handleExceptions(result: => (String, Seq[String])): (String, Seq[String]) = {
-    try {
-      result
-    } catch {
-      case a: AnalysisException =>
-        // Do not output the logical plan tree which contains expression IDs.
-        // Also implement a crude way of masking expression IDs in the error message
-        // with a generic pattern "###".
-        val msg = if (a.plan.nonEmpty) a.getSimpleMessage else a.getMessage
-        (emptySchema, Seq(a.getClass.getName, msg.replaceAll("#\\d+", "#x")))
-      case s: SparkException if s.getCause != null =>
-        // For a runtime exception, it is hard to match because its message contains
-        // information of stage, task ID, etc.
-        // To make result matching simpler, here we match the cause of the exception if it exists.
-        val cause = s.getCause
-        (emptySchema, Seq(cause.getClass.getName, cause.getMessage))
-      case NonFatal(e) =>
-        // If there is an exception, put the exception class followed by the message.
-        (emptySchema, Seq(e.getClass.getName, e.getMessage))
-    }
-  }
-
-  /** Executes a query and returns the result as (schema of the output, normalized output). */
-  private def getNormalizedResult(session: SparkSession, sql: String): (String, Seq[String]) = {
-    // Returns true if the plan is supposed to be sorted.
-    def isSorted(plan: LogicalPlan): Boolean = plan match {
-      case _: Join | _: Aggregate | _: Generate | _: Sample | _: Distinct => false
-      case _: DescribeCommandBase
-          | _: DescribeColumnCommand
-          | _: DescribeRelation
-          | _: DescribeColumnStatement => true
-      case PhysicalOperation(_, _, Sort(_, true, _)) => true
-      case _ => plan.children.iterator.exists(isSorted)
-    }
-
-    val df = session.sql(sql)
-    val schema = df.schema.catalogString
-    // Get answer, but also get rid of the #1234 expression ids that show up in explain plans
-    val answer = SQLExecution.withNewExecutionId(df.queryExecution, Some(sql)) {
-      hiveResultString(df.queryExecution.executedPlan).map(replaceNotIncludedMsg)
-    }
-
-    // If the output is not pre-sorted, sort it.
-    if (isSorted(df.queryExecution.analyzed)) (schema, answer) else (schema, answer.sorted)
-  }
-
-  protected def replaceNotIncludedMsg(line: String): String = {
-    line.replaceAll("#\\d+", "#x")
-      .replaceAll(
-        s"Location.*$clsName/",
-        s"Location $notIncludedMsg/{warehouse_dir}/")
-      .replaceAll("Created By.*", s"Created By $notIncludedMsg")
-      .replaceAll("Created Time.*", s"Created Time $notIncludedMsg")
-      .replaceAll("Last Access.*", s"Last Access $notIncludedMsg")
-      .replaceAll("Partition Statistics\t\\d+", s"Partition Statistics\t$notIncludedMsg")
-      .replaceAll("\\*\\(\\d+\\) ", "*") // remove the WholeStageCodegen codegenStageIds
-  }
-
   protected lazy val listTestCases: Seq[TestCase] = {
     listFilesRecursively(new File(inputFilePath)).flatMap { file =>
       val resultFile = file.getAbsolutePath.replace(inputFilePath, goldenFilePath) + ".out"
@@ -574,6 +500,14 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
   private def createTestTables(session: SparkSession): Unit = {
     import session.implicits._
 
+    // Before creating test tables, deletes orphan directories in warehouse dir
+    Seq("testdata", "arraydata", "mapdata", "aggtest", "onek", "tenk1").foreach { dirName =>
+      val f = new File(new URI(s"${conf.warehousePath}/$dirName"))
+      if (f.exists()) {
+        Utils.deleteRecursively(f)
+      }
+    }
+
     (1 to 100).map(i => (i, i.toString)).toDF("key", "value")
       .repartition(1)
       .write
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala
index 4e85f739b95a2..5ce5d36c5e8f7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala
@@ -155,16 +155,19 @@ abstract class ShowCreateTableSuite extends QueryTest with SQLTestUtils {
       val ex = intercept[AnalysisException] {
         sql(s"SHOW CREATE TABLE $viewName")
       }
-      assert(ex.getMessage.contains("SHOW CREATE TABLE is not supported on a temporary view"))
+      assert(ex.getMessage.contains(
+        s"$viewName is a temp view. 'SHOW CREATE TABLE' expects a table or permanent view."))
     }
 
     withGlobalTempView(viewName) {
       sql(s"CREATE GLOBAL TEMPORARY VIEW $viewName AS SELECT 1 AS a")
+      val globalTempViewDb = spark.sessionState.catalog.globalTempViewManager.database
       val ex = intercept[AnalysisException] {
-        val globalTempViewDb = spark.sessionState.catalog.globalTempViewManager.database
         sql(s"SHOW CREATE TABLE $globalTempViewDb.$viewName")
       }
-      assert(ex.getMessage.contains("SHOW CREATE TABLE is not supported on a temporary view"))
+      assert(ex.getMessage.contains(
+        s"$globalTempViewDb.$viewName is a temp view. " +
+          "'SHOW CREATE TABLE' expects a table or permanent view."))
     }
   }
 
@@ -220,29 +223,6 @@ abstract class ShowCreateTableSuite extends QueryTest with SQLTestUtils {
   }
 
   protected def checkCatalogTables(expected: CatalogTable, actual: CatalogTable): Unit = {
-    def normalize(table: CatalogTable): CatalogTable = {
-      val nondeterministicProps = Set(
-        "CreateTime",
-        "transient_lastDdlTime",
-        "grantTime",
-        "lastUpdateTime",
-        "last_modified_by",
-        "last_modified_time",
-        "Owner:",
-        // The following are hive specific schema parameters which we do not need to match exactly.
-        "totalNumberFiles",
-        "maxFileSize",
-        "minFileSize"
-      )
-
-      table.copy(
-        createTime = 0L,
-        lastAccessTime = 0L,
-        properties = table.properties.filterKeys(!nondeterministicProps.contains(_)),
-        stats = None,
-        ignoredProperties = Map.empty
-      )
-    }
-    assert(normalize(actual) == normalize(expected))
+    assert(CatalogTable.normalize(actual) == CatalogTable.normalize(expected))
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
index 7b76d0702d835..f12f866073d61 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
@@ -17,17 +17,25 @@
 
 package org.apache.spark.sql
 
+import scala.collection.JavaConverters._
+
+import org.apache.hadoop.fs.Path
 import org.scalatest.BeforeAndAfterEach
+import org.scalatest.concurrent.Eventually
+import org.scalatest.time.SpanSugar._
 
-import org.apache.spark.{SparkConf, SparkContext, SparkFunSuite}
+import org.apache.spark.{SparkConf, SparkContext, SparkException, SparkFunSuite}
+import org.apache.spark.internal.config.EXECUTOR_ALLOW_SPARK_CONTEXT
 import org.apache.spark.internal.config.UI.UI_ENABLED
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.StaticSQLConf._
+import org.apache.spark.sql.util.ExecutionListenerBus
+import org.apache.spark.util.ThreadUtils
 
 /**
  * Test cases for the builder pattern of [[SparkSession]].
  */
-class SparkSessionBuilderSuite extends SparkFunSuite with BeforeAndAfterEach {
+class SparkSessionBuilderSuite extends SparkFunSuite with BeforeAndAfterEach with Eventually {
 
   override def afterEach(): Unit = {
     // This suite should not interfere with the other test suites.
@@ -37,6 +45,34 @@ class SparkSessionBuilderSuite extends SparkFunSuite with BeforeAndAfterEach {
     SparkSession.clearDefaultSession()
   }
 
+  test("SPARK-34087: Fix memory leak of ExecutionListenerBus") {
+    val spark = SparkSession.builder()
+      .master("local")
+      .getOrCreate()
+
+    @inline def listenersNum(): Int = {
+      spark.sparkContext
+        .listenerBus
+        .listeners
+        .asScala
+        .count(_.isInstanceOf[ExecutionListenerBus])
+    }
+
+    (1 to 10).foreach { _ =>
+      spark.cloneSession()
+      SparkSession.clearActiveSession()
+    }
+
+    eventually(timeout(10.seconds), interval(1.seconds)) {
+      System.gc()
+      // After GC, the number of ExecutionListenerBus should be less than 11 (we created 11
+      // SparkSessions in total).
+      // Since GC can't 100% guarantee all out-of-referenced objects be cleaned at one time,
+      // here, we check at least one listener is cleaned up to prove the mechanism works.
+      assert(listenersNum() < 11)
+    }
+  }
+
   test("create with config options and propagate them to SparkContext and SparkSession") {
     val session = SparkSession.builder()
       .master("local")
@@ -169,6 +205,31 @@ class SparkSessionBuilderSuite extends SparkFunSuite with BeforeAndAfterEach {
     assert(session.sessionState.conf.getConf(GLOBAL_TEMP_DATABASE) === "globaltempdb-spark-31234")
   }
 
+  test("SPARK-31354: SparkContext only register one SparkSession ApplicationEnd listener") {
+    val conf = new SparkConf()
+      .setMaster("local")
+      .setAppName("test-app-SPARK-31354-1")
+    val context = new SparkContext(conf)
+    SparkSession
+      .builder()
+      .sparkContext(context)
+      .master("local")
+      .getOrCreate()
+    val postFirstCreation = context.listenerBus.listeners.size()
+    SparkSession.clearActiveSession()
+    SparkSession.clearDefaultSession()
+
+    SparkSession
+      .builder()
+      .sparkContext(context)
+      .master("local")
+      .getOrCreate()
+    val postSecondCreation = context.listenerBus.listeners.size()
+    SparkSession.clearActiveSession()
+    SparkSession.clearDefaultSession()
+    assert(postFirstCreation == postSecondCreation)
+  }
+
   test("SPARK-31532: should not propagate static sql configs to the existing" +
     " active/default SparkSession") {
     val session = SparkSession.builder()
@@ -213,6 +274,202 @@ class SparkSessionBuilderSuite extends SparkFunSuite with BeforeAndAfterEach {
       .getOrCreate()
     assert(session.conf.get("spark.app.name") === "test-app-SPARK-31532-2")
     assert(session.conf.get(GLOBAL_TEMP_DATABASE) === "globaltempdb-spark-31532-2")
-    assert(session.conf.get(WAREHOUSE_PATH) === "SPARK-31532-db-2")
+    assert(session.conf.get(WAREHOUSE_PATH) contains "SPARK-31532-db-2")
+  }
+
+  test("SPARK-32062: reset listenerRegistered in SparkSession") {
+    (1 to 2).foreach { i =>
+      val conf = new SparkConf()
+        .setMaster("local")
+        .setAppName(s"test-SPARK-32062-$i")
+      val context = new SparkContext(conf)
+      val beforeListenerSize = context.listenerBus.listeners.size()
+      SparkSession
+        .builder()
+        .sparkContext(context)
+        .getOrCreate()
+      val afterListenerSize = context.listenerBus.listeners.size()
+      assert(beforeListenerSize + 1 == afterListenerSize)
+      context.stop()
+    }
+  }
+
+  test("SPARK-32160: Disallow to create SparkSession in executors") {
+    val session = SparkSession.builder().master("local-cluster[3, 1, 1024]").getOrCreate()
+
+    val error = intercept[SparkException] {
+      session.range(1).foreach { v =>
+        SparkSession.builder.master("local").getOrCreate()
+        ()
+      }
+    }.getMessage()
+
+    assert(error.contains("SparkSession should only be created and accessed on the driver."))
+  }
+
+  test("SPARK-32160: Allow to create SparkSession in executors if the config is set") {
+    val session = SparkSession.builder().master("local-cluster[3, 1, 1024]").getOrCreate()
+
+    session.range(1).foreach { v =>
+      SparkSession.builder.master("local")
+        .config(EXECUTOR_ALLOW_SPARK_CONTEXT.key, true).getOrCreate().stop()
+      ()
+    }
+  }
+
+  test("SPARK-32991: Use conf in shared state as the original configuration for RESET") {
+    val wh = "spark.sql.warehouse.dir"
+    val td = "spark.sql.globalTempDatabase"
+    val custom = "spark.sql.custom"
+
+    val conf = new SparkConf()
+      .setMaster("local")
+      .setAppName("SPARK-32991")
+      .set(wh, "./data1")
+      .set(td, "bob")
+
+    val sc = new SparkContext(conf)
+
+    val spark = SparkSession.builder()
+      .config(wh, "./data2")
+      .config(td, "alice")
+      .config(custom, "kyao")
+      .getOrCreate()
+
+    // When creating the first session like above, we will update the shared spark conf to the
+    // newly specified values
+    val sharedWH = spark.sharedState.conf.get(wh)
+    val sharedTD = spark.sharedState.conf.get(td)
+    assert(sharedWH contains "data2",
+      "The warehouse dir in shared state should be determined by the 1st created spark session")
+    assert(sharedTD === "alice",
+      "Static sql configs in shared state should be determined by the 1st created spark session")
+    assert(spark.sharedState.conf.getOption(custom).isEmpty,
+      "Dynamic sql configs is session specific")
+
+    assert(spark.conf.get(wh) contains sharedWH,
+      "The warehouse dir in session conf and shared state conf should be consistent")
+    assert(spark.conf.get(td) === sharedTD,
+      "Static sql configs in session conf and shared state conf should be consistent")
+    assert(spark.conf.get(custom) === "kyao", "Dynamic sql configs is session specific")
+
+    spark.sql("RESET")
+
+    assert(spark.conf.get(wh) contains sharedWH,
+      "The warehouse dir in shared state should be respect after RESET")
+    assert(spark.conf.get(td) === sharedTD,
+      "Static sql configs in shared state should be respect after RESET")
+    assert(spark.conf.get(custom) === "kyao",
+      "Dynamic sql configs in session initial map should be respect after RESET")
+
+    val spark2 = SparkSession.builder()
+      .config(wh, "./data3")
+      .config(custom, "kyaoo").getOrCreate()
+    assert(spark2.conf.get(wh) contains sharedWH)
+    assert(spark2.conf.get(td) === sharedTD)
+    assert(spark2.conf.get(custom) === "kyaoo")
+  }
+
+  test("SPARK-32991: RESET should work properly with multi threads") {
+    val wh = "spark.sql.warehouse.dir"
+    val td = "spark.sql.globalTempDatabase"
+    val custom = "spark.sql.custom"
+    val spark = ThreadUtils.runInNewThread("new session 0", false) {
+      SparkSession.builder()
+        .master("local")
+        .config(wh, "./data0")
+        .config(td, "bob")
+        .config(custom, "c0")
+        .getOrCreate()
+    }
+
+    spark.sql(s"SET $custom=c1")
+    assert(spark.conf.get(custom) === "c1")
+    spark.sql("RESET")
+    assert(spark.conf.get(wh) contains "data0",
+      "The warehouse dir in shared state should be respect after RESET")
+    assert(spark.conf.get(td) === "bob",
+      "Static sql configs in shared state should be respect after RESET")
+    assert(spark.conf.get(custom) === "c0",
+      "Dynamic sql configs in shared state should be respect after RESET")
+
+    val spark1 = ThreadUtils.runInNewThread("new session 1", false) {
+      SparkSession.builder().getOrCreate()
+    }
+
+    assert(spark === spark1)
+
+    // TODO: SPARK-33718: After clear sessions, the SharedState will be unreachable, then all
+    // the new static will take effect.
+    SparkSession.clearDefaultSession()
+    val spark2 = ThreadUtils.runInNewThread("new session 2", false) {
+      SparkSession.builder()
+        .master("local")
+        .config(wh, "./data1")
+        .config(td, "alice")
+        .config(custom, "c2")
+        .getOrCreate()
+    }
+
+    assert(spark2 !== spark)
+    spark2.sql(s"SET $custom=c1")
+    assert(spark2.conf.get(custom) === "c1")
+    spark2.sql("RESET")
+    assert(spark2.conf.get(wh) contains "data1")
+    assert(spark2.conf.get(td) === "alice")
+    assert(spark2.conf.get(custom) === "c2")
+
+  }
+
+  test("SPARK-33944: warning setting hive.metastore.warehouse.dir using session options") {
+    val msg = "Not allowing to set hive.metastore.warehouse.dir in SparkSession's options"
+    val logAppender = new LogAppender(msg)
+    withLogAppender(logAppender) {
+      SparkSession.builder()
+        .master("local")
+        .config("hive.metastore.warehouse.dir", "any")
+        .getOrCreate()
+        .sharedState
+    }
+    assert(logAppender.loggingEvents.exists(_.getRenderedMessage.contains(msg)))
+  }
+
+  test("SPARK-33944: no warning setting spark.sql.warehouse.dir using session options") {
+    val msg = "Not allowing to set hive.metastore.warehouse.dir in SparkSession's options"
+    val logAppender = new LogAppender(msg)
+    withLogAppender(logAppender) {
+      SparkSession.builder()
+        .master("local")
+        .config("spark.sql.warehouse.dir", "any")
+        .getOrCreate()
+        .sharedState
+    }
+    assert(!logAppender.loggingEvents.exists(_.getRenderedMessage.contains(msg)))
+  }
+
+  Seq(".", "..", "dir0", "dir0/dir1", "/dir0/dir1", "./dir0").foreach { pathStr =>
+    test(s"SPARK-34558: warehouse path ($pathStr) should be qualified for spark/hadoop conf") {
+      val path = new Path(pathStr)
+      val conf = new SparkConf().set(WAREHOUSE_PATH, pathStr)
+      val session = SparkSession.builder()
+        .master("local")
+        .config(conf)
+        .getOrCreate()
+      val hadoopConf = session.sessionState.newHadoopConf()
+      val expected = path.getFileSystem(hadoopConf).makeQualified(path).toString
+      // session related configs
+      assert(hadoopConf.get("hive.metastore.warehouse.dir") === expected)
+      assert(session.conf.get(WAREHOUSE_PATH) === expected)
+      assert(session.sessionState.conf.warehousePath === expected)
+
+      // shared configs
+      assert(session.sharedState.conf.get(WAREHOUSE_PATH) === expected)
+      assert(session.sharedState.hadoopConf.get("hive.metastore.warehouse.dir") === expected)
+
+      // spark context configs
+      assert(session.sparkContext.conf.get(WAREHOUSE_PATH) === expected)
+      assert(session.sparkContext.hadoopConfiguration.get("hive.metastore.warehouse.dir") ===
+        expected)
+    }
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
index d9c90c7dbd085..c8768ec2c5af1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
@@ -16,17 +16,24 @@
  */
 package org.apache.spark.sql
 
-import java.util.Locale
+import java.util.{Locale, UUID}
 
-import org.apache.spark.{SparkFunSuite, TaskContext}
+import scala.concurrent.Future
+
+import org.apache.spark.{MapOutputStatistics, SparkFunSuite, TaskContext}
+import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.internal.Logging
 import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
+import org.apache.spark.sql.catalyst.{FunctionIdentifier, InternalRow, TableIdentifier}
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, ParserInterface}
-import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan, UnresolvedHint}
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan, Statistics, UnresolvedHint}
+import org.apache.spark.sql.catalyst.plans.physical.Partitioning
 import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreeNodeTag
 import org.apache.spark.sql.execution._
+import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, QueryStageExec}
+import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, BroadcastExchangeLike, ShuffleExchangeExec, ShuffleExchangeLike, ShuffleOrigin}
 import org.apache.spark.sql.execution.vectorized.OnHeapColumnVector
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.COLUMN_BATCH_SIZE
@@ -39,8 +46,8 @@ import org.apache.spark.unsafe.types.UTF8String
  * Test cases for the [[SparkSessionExtensions]].
  */
 class SparkSessionExtensionSuite extends SparkFunSuite {
-  type ExtensionsBuilder = SparkSessionExtensions => Unit
-  private def create(builder: ExtensionsBuilder): Seq[ExtensionsBuilder] = Seq(builder)
+  private def create(
+      builder: SparkSessionExtensionsProvider): Seq[SparkSessionExtensionsProvider] = Seq(builder)
 
   private def stop(spark: SparkSession): Unit = {
     spark.stop()
@@ -48,7 +55,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite {
     SparkSession.clearDefaultSession()
   }
 
-  private def withSession(builders: Seq[ExtensionsBuilder])(f: SparkSession => Unit): Unit = {
+  private def withSession(
+      builders: Seq[SparkSessionExtensionsProvider])(f: SparkSession => Unit): Unit = {
     val builder = SparkSession.builder().master("local[1]")
     builders.foreach(builder.withExtensions)
     val spark = builder.getOrCreate()
@@ -81,6 +89,12 @@ class SparkSessionExtensionSuite extends SparkFunSuite {
     }
   }
 
+  test("SPARK-33621: inject a pre CBO rule") {
+    withSession(Seq(_.injectPreCBORule(MyRule))) { session =>
+      assert(session.sessionState.optimizer.preCBORules.contains(MyRule(session)))
+    }
+  }
+
   test("inject spark planner strategy") {
     withSession(Seq(_.injectPlannerStrategy(MySparkStrategy))) { session =>
       assert(session.sessionState.planner.strategies.contains(MySparkStrategy(session)))
@@ -145,33 +159,83 @@ class SparkSessionExtensionSuite extends SparkFunSuite {
     }
   }
 
-  test("inject columnar") {
+  test("inject adaptive query prep rule") {
     val extensions = create { extensions =>
+      // inject rule that will run during AQE query stage preparation and will add custom tags
+      // to the plan
+      extensions.injectQueryStagePrepRule(session => MyQueryStagePrepRule())
+      // inject rule that will run during AQE query stage optimization and will verify that the
+      // custom tags were written in the preparation phase
       extensions.injectColumnar(session =>
-        MyColumarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule()))
+        MyColumnarRule(MyNewQueryStageRule(), MyNewQueryStageRule()))
     }
     withSession(extensions) { session =>
-      // The ApplyColumnarRulesAndInsertTransitions rule is not applied when enable AQE
-      session.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED, false)
+      session.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED, true)
+      assert(session.sessionState.queryStagePrepRules.contains(MyQueryStagePrepRule()))
       assert(session.sessionState.columnarRules.contains(
-        MyColumarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())))
+        MyColumnarRule(MyNewQueryStageRule(), MyNewQueryStageRule())))
       import session.sqlContext.implicits._
-      // repartitioning avoids having the add operation pushed up into the LocalTableScan
       val data = Seq((100L), (200L), (300L)).toDF("vals").repartition(1)
       val df = data.selectExpr("vals + 1")
-      // Verify that both pre and post processing of the plan worked.
-      val found = df.queryExecution.executedPlan.collect {
-        case rep: ReplacedRowToColumnarExec => 1
-        case proj: ColumnarProjectExec => 10
-        case c2r: ColumnarToRowExec => 100
-      }.sum
-      assert(found == 111)
+      df.collect()
+    }
+  }
+
+  test("inject columnar AQE on") {
+    testInjectColumnar(true)
+  }
+
+  test("inject columnar AQE off") {
+    testInjectColumnar(false)
+  }
+
+  private def testInjectColumnar(enableAQE: Boolean): Unit = {
+    def collectPlanSteps(plan: SparkPlan): Seq[Int] = plan match {
+      case a: AdaptiveSparkPlanExec =>
+        assert(a.toString.startsWith("AdaptiveSparkPlan isFinalPlan=true"))
+        collectPlanSteps(a.executedPlan)
+      case _ => plan.collect {
+        case _: ReplacedRowToColumnarExec => 1
+        case _: ColumnarProjectExec => 10
+        case _: ColumnarToRowExec => 100
+        case s: QueryStageExec => collectPlanSteps(s.plan).sum
+        case _: MyShuffleExchangeExec => 1000
+        case _: MyBroadcastExchangeExec => 10000
+      }
+    }
+
+    val extensions = create { extensions =>
+      extensions.injectColumnar(session =>
+        MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule()))
+    }
+    withSession(extensions) { session =>
+      session.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED, enableAQE)
+      assert(session.sessionState.columnarRules.contains(
+        MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())))
+      import session.sqlContext.implicits._
+      // perform a join to inject a broadcast exchange
+      val left = Seq((1, 50L), (2, 100L), (3, 150L)).toDF("l1", "l2")
+      val right = Seq((1, 50L), (2, 100L), (3, 150L)).toDF("r1", "r2")
+      val data = left.join(right, $"l1" === $"r1")
+        // repartitioning avoids having the add operation pushed up into the LocalTableScan
+        .repartition(1)
+      val df = data.selectExpr("l2 + r2")
+      // execute the plan so that the final adaptive plan is available when AQE is on
+      df.collect()
+      val found = collectPlanSteps(df.queryExecution.executedPlan).sum
+      // 1 MyBroadcastExchangeExec
+      // 1 MyShuffleExchangeExec
+      // 1 ColumnarToRowExec
+      // 2 ColumnarProjectExec
+      // 1 ReplacedRowToColumnarExec
+      // so 11121 is expected.
+      assert(found == 11121)
 
       // Verify that we get back the expected, wrong, result
       val result = df.collect()
-      assert(result(0).getLong(0) == 102L) // Check that broken columnar Add was used.
-      assert(result(1).getLong(0) == 202L)
-      assert(result(2).getLong(0) == 302L)
+      assert(result(0).getLong(0) == 101L) // Check that broken columnar Add was used.
+      assert(result(1).getLong(0) == 201L)
+      assert(result(2).getLong(0) == 301L)
     }
   }
 
@@ -181,12 +245,12 @@ class SparkSessionExtensionSuite extends SparkFunSuite {
       .config(COLUMN_BATCH_SIZE.key, 2)
       .withExtensions { extensions =>
         extensions.injectColumnar(session =>
-          MyColumarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())) }
+          MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())) }
       .getOrCreate()
 
     try {
       assert(session.sessionState.columnarRules.contains(
-        MyColumarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())))
+        MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())))
       import session.sqlContext.implicits._
 
       val input = Seq((100L), (200L), (300L))
@@ -214,7 +278,7 @@ class SparkSessionExtensionSuite extends SparkFunSuite {
       assert(session.sessionState.functionRegistry
         .lookupFunction(MyExtensions.myFunction._1).isDefined)
       assert(session.sessionState.columnarRules.contains(
-        MyColumarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())))
+        MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())))
     } finally {
       stop(session)
     }
@@ -292,6 +356,20 @@ class SparkSessionExtensionSuite extends SparkFunSuite {
       stop(session)
     }
   }
+
+  test("SPARK-35380: Loading extensions from ServiceLoader") {
+    val builder = SparkSession.builder().master("local[1]")
+
+    Seq(None, Some(classOf[YourExtensions].getName)).foreach { ext =>
+      ext.foreach(builder.config(SPARK_SESSION_EXTENSIONS.key, _))
+      val session = builder.getOrCreate()
+      try {
+        assert(session.sql("select get_fake_app_name()").head().getString(0) === "Fake App Name")
+      } finally {
+        stop(session)
+      }
+    }
+  }
 }
 
 case class MyRule(spark: SparkSession) extends Rule[LogicalPlan] {
@@ -327,9 +405,6 @@ case class MyParser(spark: SparkSession, delegate: ParserInterface) extends Pars
 
   override def parseDataType(sqlText: String): DataType =
     delegate.parseDataType(sqlText)
-
-  override def parseRawDataType(sqlText: String): DataType =
-    delegate.parseRawDataType(sqlText)
 }
 
 object MyExtensions {
@@ -516,11 +591,16 @@ class ColumnarBoundReference(ordinal: Int, dataType: DataType, nullable: Boolean
 class ColumnarAlias(child: ColumnarExpression, name: String)(
     override val exprId: ExprId = NamedExpression.newExprId,
     override val qualifier: Seq[String] = Seq.empty,
-    override val explicitMetadata: Option[Metadata] = None)
-  extends Alias(child, name)(exprId, qualifier, explicitMetadata)
+    override val explicitMetadata: Option[Metadata] = None,
+    override val nonInheritableMetadataKeys: Seq[String] = Seq.empty)
+  extends Alias(child, name)(exprId, qualifier, explicitMetadata, nonInheritableMetadataKeys)
   with ColumnarExpression {
 
   override def columnarEval(batch: ColumnarBatch): Any = child.columnarEval(batch)
+
+  override protected def withNewChildInternal(newChild: Expression): ColumnarAlias =
+    new ColumnarAlias(newChild.asInstanceOf[ColumnarExpression], name)(exprId, qualifier,
+      explicitMetadata, nonInheritableMetadataKeys)
 }
 
 class ColumnarAttributeReference(
@@ -580,14 +660,20 @@ class ColumnarProjectExec(projectList: Seq[NamedExpression], child: SparkPlan)
   }
 
   override def hashCode(): Int = super.hashCode()
+
+  override def withNewChildInternal(newChild: SparkPlan): ColumnarProjectExec =
+    new ColumnarProjectExec(projectList, newChild)
 }
 
 /**
  * A version of add that supports columnar processing for longs.  This version is broken
  * on purpose so it adds the numbers plus 1 so that the tests can show that it was replaced.
  */
-class BrokenColumnarAdd(left: ColumnarExpression, right: ColumnarExpression)
-  extends Add(left, right) with ColumnarExpression {
+class BrokenColumnarAdd(
+    left: ColumnarExpression,
+    right: ColumnarExpression,
+    failOnError: Boolean = false)
+  extends Add(left, right, failOnError) with ColumnarExpression {
 
   override def supportsColumnar(): Boolean = left.supportsColumnar && right.supportsColumnar
 
@@ -641,6 +727,12 @@ class BrokenColumnarAdd(left: ColumnarExpression, right: ColumnarExpression)
     }
     ret
   }
+
+  override def withNewChildrenInternal(
+      newLeft: Expression, newRight: Expression): BrokenColumnarAdd =
+    new BrokenColumnarAdd(
+      left = newLeft.asInstanceOf[ColumnarExpression],
+      right = newRight.asInstanceOf[ColumnarExpression], failOnError)
 }
 
 class CannotReplaceException(str: String) extends RuntimeException(str) {
@@ -651,7 +743,7 @@ case class PreRuleReplaceAddWithBrokenVersion() extends Rule[SparkPlan] {
   def replaceWithColumnarExpression(exp: Expression): ColumnarExpression = exp match {
     case a: Alias =>
       new ColumnarAlias(replaceWithColumnarExpression(a.child),
-        a.name)(a.exprId, a.qualifier, a.explicitMetadata)
+        a.name)(a.exprId, a.qualifier, a.explicitMetadata, a.nonInheritableMetadataKeys)
     case att: AttributeReference =>
       new ColumnarAttributeReference(att.name, att.dataType, att.nullable,
         att.metadata)(att.exprId, att.qualifier)
@@ -671,6 +763,16 @@ case class PreRuleReplaceAddWithBrokenVersion() extends Rule[SparkPlan] {
   def replaceWithColumnarPlan(plan: SparkPlan): SparkPlan =
     try {
       plan match {
+        case e: ShuffleExchangeExec =>
+          // note that this is not actually columnar but demonstrates that exchanges can
+          // be replaced.
+          val replaced = e.withNewChildren(e.children.map(replaceWithColumnarPlan))
+          MyShuffleExchangeExec(replaced.asInstanceOf[ShuffleExchangeExec])
+        case e: BroadcastExchangeExec =>
+          // note that this is not actually columnar but demonstrates that exchanges can
+          // be replaced.
+          val replaced = e.withNewChildren(e.children.map(replaceWithColumnarPlan))
+          MyBroadcastExchangeExec(replaced.asInstanceOf[BroadcastExchangeExec])
         case plan: ProjectExec =>
           new ColumnarProjectExec(plan.projectList.map((exp) =>
             replaceWithColumnarExpression(exp).asInstanceOf[NamedExpression]),
@@ -689,6 +791,48 @@ case class PreRuleReplaceAddWithBrokenVersion() extends Rule[SparkPlan] {
   override def apply(plan: SparkPlan): SparkPlan = replaceWithColumnarPlan(plan)
 }
 
+/**
+ * Custom Exchange used in tests to demonstrate that shuffles can be replaced regardless of
+ * whether AQE is enabled.
+ */
+case class MyShuffleExchangeExec(delegate: ShuffleExchangeExec) extends ShuffleExchangeLike {
+  override def numMappers: Int = delegate.numMappers
+  override def numPartitions: Int = delegate.numPartitions
+  override def shuffleOrigin: ShuffleOrigin = {
+    delegate.shuffleOrigin
+  }
+  override def mapOutputStatisticsFuture: Future[MapOutputStatistics] =
+    delegate.mapOutputStatisticsFuture
+  override def getShuffleRDD(partitionSpecs: Array[ShufflePartitionSpec]): RDD[_] =
+    delegate.getShuffleRDD(partitionSpecs)
+  override def runtimeStatistics: Statistics = delegate.runtimeStatistics
+  override def child: SparkPlan = delegate.child
+  override protected def doExecute(): RDD[InternalRow] = delegate.execute()
+  override def outputPartitioning: Partitioning = delegate.outputPartitioning
+  override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan =
+    super.legacyWithNewChildren(Seq(newChild))
+}
+
+/**
+ * Custom Exchange used in tests to demonstrate that broadcasts can be replaced regardless of
+ * whether AQE is enabled.
+ */
+case class MyBroadcastExchangeExec(delegate: BroadcastExchangeExec) extends BroadcastExchangeLike {
+  override def runId: UUID = delegate.runId
+  override def relationFuture: java.util.concurrent.Future[Broadcast[Any]] =
+    delegate.relationFuture
+  override def completionFuture: Future[Broadcast[Any]] = delegate.completionFuture
+  override def runtimeStatistics: Statistics = delegate.runtimeStatistics
+  override def child: SparkPlan = delegate.child
+  override protected def doPrepare(): Unit = delegate.prepare()
+  override protected def doExecute(): RDD[InternalRow] = delegate.execute()
+  override def doExecuteBroadcast[T](): Broadcast[T] = delegate.executeBroadcast()
+  override def outputPartitioning: Partitioning = delegate.outputPartitioning
+
+  override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan =
+    super.legacyWithNewChildren(Seq(newChild))
+}
+
 class ReplacedRowToColumnarExec(override val child: SparkPlan)
   extends RowToColumnarExec(child) {
 
@@ -704,6 +848,9 @@ class ReplacedRowToColumnarExec(override val child: SparkPlan)
   }
 
   override def hashCode(): Int = super.hashCode()
+
+  override def withNewChildInternal(newChild: SparkPlan): ReplacedRowToColumnarExec =
+    new ReplacedRowToColumnarExec(newChild)
 }
 
 case class MyPostRule() extends Rule[SparkPlan] {
@@ -713,7 +860,7 @@ case class MyPostRule() extends Rule[SparkPlan] {
   }
 }
 
-case class MyColumarRule(pre: Rule[SparkPlan], post: Rule[SparkPlan]) extends ColumnarRule {
+case class MyColumnarRule(pre: Rule[SparkPlan], post: Rule[SparkPlan]) extends ColumnarRule {
   override def preColumnarTransitions: Rule[SparkPlan] = pre
   override def postColumnarTransitions: Rule[SparkPlan] = post
 }
@@ -727,7 +874,32 @@ class MyExtensions extends (SparkSessionExtensions => Unit) {
     e.injectOptimizerRule(MyRule)
     e.injectParser(MyParser)
     e.injectFunction(MyExtensions.myFunction)
-    e.injectColumnar(session => MyColumarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule()))
+    e.injectColumnar(session => MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule()))
+  }
+}
+
+object QueryPrepRuleHelper {
+  val myPrepTag: TreeNodeTag[String] = TreeNodeTag[String]("myPrepTag")
+  val myPrepTagValue: String = "myPrepTagValue"
+}
+
+// this rule will run during AQE query preparation and will write custom tags to each node
+case class MyQueryStagePrepRule() extends Rule[SparkPlan] {
+  override def apply(plan: SparkPlan): SparkPlan = plan.transformDown {
+    case plan =>
+      plan.setTagValue(QueryPrepRuleHelper.myPrepTag, QueryPrepRuleHelper.myPrepTagValue)
+      plan
+  }
+}
+
+// this rule will run during AQE query stage optimization and will verify custom tags were
+// already written during query preparation phase
+case class MyNewQueryStageRule() extends Rule[SparkPlan] {
+  override def apply(plan: SparkPlan): SparkPlan = plan.transformDown {
+    case plan if !plan.isInstanceOf[AdaptiveSparkPlanExec] =>
+      assert(plan.getTagValue(QueryPrepRuleHelper.myPrepTag).get ==
+          QueryPrepRuleHelper.myPrepTagValue)
+      plan
   }
 }
 
@@ -802,3 +974,16 @@ class MyExtensions2Duplicate extends (SparkSessionExtensions => Unit) {
     e.injectFunction(MyExtensions2Duplicate.myFunction)
   }
 }
+
+class YourExtensions extends SparkSessionExtensionsProvider {
+  val getAppName = (FunctionIdentifier("get_fake_app_name"),
+    new ExpressionInfo(
+      "zzz.zzz.zzz",
+      "",
+      "get_fake_app_name"),
+    (_: Seq[Expression]) => Literal("Fake App Name"))
+
+  override def apply(v1: SparkSessionExtensions): Unit = {
+    v1.injectFunction(getAppName)
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
index 708b98e8fe15a..9f8000a08f7af 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
@@ -25,10 +25,11 @@ import java.util.concurrent.TimeUnit
 import scala.collection.mutable
 
 import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
 import org.apache.spark.sql.catalyst.catalog.CatalogColumnStat
 import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, DateTimeUtils}
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.TimeZoneUTC
+import org.apache.spark.sql.functions.timestamp_seconds
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.test.SQLTestData.ArrayData
@@ -173,6 +174,15 @@ class StatisticsCollectionSuite extends StatisticsCollectionTestBase with Shared
     }
   }
 
+  test("SPARK-33812: column stats round trip serialization with splitting histogram property") {
+    withSQLConf(SQLConf.HIVE_TABLE_PROPERTY_LENGTH_THRESHOLD.key -> "10") {
+      statsWithHgms.foreach { case (k, v) =>
+        val roundtrip = CatalogColumnStat.fromMap("t", k, v.toMap(k))
+        assert(roundtrip == Some(v))
+      }
+    }
+  }
+
   test("analyze column command - result verification") {
     // (data.head.productArity - 1) because the last column does not support stats collection.
     assert(stats.size == data.head.productArity - 1)
@@ -247,24 +257,6 @@ class StatisticsCollectionSuite extends StatisticsCollectionTestBase with Shared
     }
   }
 
-  test("change stats after truncate command") {
-    val table = "change_stats_truncate_table"
-    withTable(table) {
-      spark.range(100).select($"id", $"id" % 5 as "value").write.saveAsTable(table)
-      // analyze to get initial stats
-      sql(s"ANALYZE TABLE $table COMPUTE STATISTICS FOR COLUMNS id, value")
-      val fetched1 = checkTableStats(table, hasSizeInBytes = true, expectedRowCounts = Some(100))
-      assert(fetched1.get.sizeInBytes > 0)
-      assert(fetched1.get.colStats.size == 2)
-
-      // truncate table command
-      sql(s"TRUNCATE TABLE $table")
-      val fetched2 = checkTableStats(table, hasSizeInBytes = true, expectedRowCounts = Some(0))
-      assert(fetched2.get.sizeInBytes == 0)
-      assert(fetched2.get.colStats.isEmpty)
-    }
-  }
-
   test("change stats after set location command") {
     val table = "change_stats_set_location_table"
     val tableLoc = new File(spark.sessionState.catalog.defaultTablePath(TableIdentifier(table)))
@@ -375,22 +367,6 @@ class StatisticsCollectionSuite extends StatisticsCollectionTestBase with Shared
     }
   }
 
-  test("invalidation of tableRelationCache after table truncation") {
-    val table = "invalidate_catalog_cache_table"
-    Seq(false, true).foreach { autoUpdate =>
-      withSQLConf(SQLConf.AUTO_SIZE_UPDATE_ENABLED.key -> autoUpdate.toString) {
-        withTable(table) {
-          spark.range(100).write.saveAsTable(table)
-          sql(s"ANALYZE TABLE $table COMPUTE STATISTICS")
-          spark.table(table)
-          sql(s"TRUNCATE TABLE $table")
-          spark.table(table)
-          assert(getTableFromCatalogCache(table).stats.sizeInBytes == 0)
-        }
-      }
-    }
-  }
-
   test("invalidation of tableRelationCache after alter table add partition") {
     val table = "invalidate_catalog_cache_table"
     Seq(false, true).foreach { autoUpdate =>
@@ -467,7 +443,7 @@ class StatisticsCollectionSuite extends StatisticsCollectionTestBase with Shared
         withTable(table) {
           TimeZone.setDefault(srcTimeZone)
           spark.range(start, end)
-            .select('id.cast(TimestampType).cast(t).as(column))
+            .select(timestamp_seconds($"id").cast(t).as(column))
             .write.saveAsTable(table)
           sql(s"ANALYZE TABLE $table COMPUTE STATISTICS FOR COLUMNS $column")
 
@@ -483,11 +459,11 @@ class StatisticsCollectionSuite extends StatisticsCollectionTestBase with Shared
 
     DateTimeTestUtils.outstandingZoneIds.foreach { zid =>
       val timeZone = TimeZone.getTimeZone(zid)
-      checkTimestampStats(DateType, DateTimeUtils.TimeZoneUTC, timeZone) { stats =>
+      checkTimestampStats(DateType, TimeZoneUTC, timeZone) { stats =>
         assert(stats.min.get.asInstanceOf[Int] == TimeUnit.SECONDS.toDays(start))
         assert(stats.max.get.asInstanceOf[Int] == TimeUnit.SECONDS.toDays(end - 1))
       }
-      checkTimestampStats(TimestampType, DateTimeUtils.TimeZoneUTC, timeZone) { stats =>
+      checkTimestampStats(TimestampType, TimeZoneUTC, timeZone) { stats =>
         assert(stats.min.get.asInstanceOf[Long] == TimeUnit.SECONDS.toMicros(start))
         assert(stats.max.get.asInstanceOf[Long] == TimeUnit.SECONDS.toMicros(end - 1))
       }
@@ -521,11 +497,11 @@ class StatisticsCollectionSuite extends StatisticsCollectionTestBase with Shared
   test("analyzes column statistics in cached local temporary view") {
     withTempView("tempView") {
       // Analyzes in a temporary view
-      sql("CREATE TEMPORARY VIEW tempView AS SELECT * FROM range(1, 30)")
+      sql("CREATE TEMPORARY VIEW tempView AS SELECT 1 id")
       val errMsg = intercept[AnalysisException] {
         sql("ANALYZE TABLE tempView COMPUTE STATISTICS FOR COLUMNS id")
       }.getMessage
-      assert(errMsg.contains(s"Table or view 'tempView' not found in database 'default'"))
+      assert(errMsg.contains("Temporary view `tempView` is not cached for analyzing columns"))
 
       // Cache the view then analyze it
       sql("CACHE TABLE tempView")
@@ -538,16 +514,18 @@ class StatisticsCollectionSuite extends StatisticsCollectionTestBase with Shared
   test("analyzes column statistics in cached global temporary view") {
     withGlobalTempView("gTempView") {
       val globalTempDB = spark.sharedState.globalTempViewManager.database
-      val errMsg1 = intercept[NoSuchTableException] {
+      val errMsg1 = intercept[AnalysisException] {
         sql(s"ANALYZE TABLE $globalTempDB.gTempView COMPUTE STATISTICS FOR COLUMNS id")
       }.getMessage
-      assert(errMsg1.contains(s"Table or view 'gTempView' not found in database '$globalTempDB'"))
+      assert(errMsg1.contains("Table or view not found: " +
+        s"$globalTempDB.gTempView"))
       // Analyzes in a global temporary view
-      sql("CREATE GLOBAL TEMP VIEW gTempView AS SELECT * FROM range(1, 30)")
+      sql("CREATE GLOBAL TEMP VIEW gTempView AS SELECT 1 id")
       val errMsg2 = intercept[AnalysisException] {
         sql(s"ANALYZE TABLE $globalTempDB.gTempView COMPUTE STATISTICS FOR COLUMNS id")
       }.getMessage
-      assert(errMsg2.contains(s"Table or view 'gTempView' not found in database '$globalTempDB'"))
+      assert(errMsg2.contains(
+        s"Temporary view `$globalTempDB`.`gTempView` is not cached for analyzing columns"))
 
       // Cache the view then analyze it
       sql(s"CACHE TABLE $globalTempDB.gTempView")
@@ -668,4 +646,73 @@ class StatisticsCollectionSuite extends StatisticsCollectionTestBase with Shared
       }
     }
   }
+
+  test("SPARK-34119: Keep necessary stats after PruneFileSourcePartitions") {
+    withTable("SPARK_34119") {
+      withSQLConf(SQLConf.CBO_ENABLED.key -> "true") {
+        sql(s"CREATE TABLE SPARK_34119 using parquet PARTITIONED BY (p) AS " +
+          "(SELECT id, CAST(id % 5 AS STRING) AS p FROM range(10))")
+        sql(s"ANALYZE TABLE SPARK_34119 COMPUTE STATISTICS FOR ALL COLUMNS")
+
+        checkOptimizedPlanStats(sql(s"SELECT id FROM SPARK_34119"),
+          160L,
+          Some(10),
+          Seq(ColumnStat(
+            distinctCount = Some(10),
+            min = Some(0),
+            max = Some(9),
+            nullCount = Some(0),
+            avgLen = Some(LongType.defaultSize),
+            maxLen = Some(LongType.defaultSize))))
+
+        checkOptimizedPlanStats(sql("SELECT id FROM SPARK_34119 WHERE p = '2'"),
+          32L,
+          Some(2),
+          Seq(ColumnStat(
+            distinctCount = Some(2),
+            min = Some(0),
+            max = Some(9),
+            nullCount = Some(0),
+            avgLen = Some(LongType.defaultSize),
+            maxLen = Some(LongType.defaultSize))))
+      }
+    }
+  }
+
+  test("SPARK-33687: analyze all tables in a specific database") {
+    withTempDatabase { database =>
+      spark.catalog.setCurrentDatabase(database)
+      withTempDir { dir =>
+        withTable("t1", "t2") {
+          spark.range(10).write.saveAsTable("t1")
+          sql(s"CREATE EXTERNAL TABLE t2 USING parquet LOCATION '${dir.toURI}' " +
+            "AS SELECT * FROM range(20)")
+          withView("v1", "v2") {
+            sql("CREATE VIEW v1 AS SELECT 1 c1")
+            sql("CREATE VIEW v2 AS SELECT 2 c2")
+            sql("CACHE TABLE v1")
+            sql("CACHE LAZY TABLE v2")
+
+            sql(s"ANALYZE TABLES IN $database COMPUTE STATISTICS NOSCAN")
+            checkTableStats("t1", hasSizeInBytes = true, expectedRowCounts = None)
+            checkTableStats("t2", hasSizeInBytes = true, expectedRowCounts = None)
+            assert(getCatalogTable("v1").stats.isEmpty)
+            checkOptimizedPlanStats(spark.table("v1"), 4, Some(1), Seq.empty)
+            checkOptimizedPlanStats(spark.table("v2"), 1, None, Seq.empty)
+
+            sql("ANALYZE TABLES COMPUTE STATISTICS")
+            checkTableStats("t1", hasSizeInBytes = true, expectedRowCounts = Some(10))
+            checkTableStats("t2", hasSizeInBytes = true, expectedRowCounts = Some(20))
+            checkOptimizedPlanStats(spark.table("v1"), 4, Some(1), Seq.empty)
+            checkOptimizedPlanStats(spark.table("v2"), 4, Some(1), Seq.empty)
+          }
+        }
+      }
+    }
+
+    val errMsg = intercept[AnalysisException] {
+      sql(s"ANALYZE TABLES IN db_not_exists COMPUTE STATISTICS")
+    }.getMessage
+    assert(errMsg.contains("Database 'db_not_exists' not found"))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionTestBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionTestBase.scala
index d3fae6a14915b..6c6ef1a118f48 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionTestBase.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionTestBase.scala
@@ -27,7 +27,8 @@ import scala.util.Random
 import org.apache.spark.sql.catalyst.{QualifiedTableName, TableIdentifier}
 import org.apache.spark.sql.catalyst.catalog.{CatalogColumnStat, CatalogStatistics, CatalogTable, HiveTableRelation}
 import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
-import org.apache.spark.sql.catalyst.plans.logical.{ColumnStat, Histogram, HistogramBin, HistogramSerializer, LogicalPlan}
+import org.apache.spark.sql.catalyst.expressions.AttributeMap
+import org.apache.spark.sql.catalyst.plans.logical.{ColumnStat, Histogram, HistogramBin, HistogramSerializer, LogicalPlan, Statistics}
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.execution.datasources.LogicalRelation
@@ -320,6 +321,17 @@ abstract class StatisticsCollectionTestBase extends QueryTest with SQLTestUtils
     }
   }
 
+  def checkOptimizedPlanStats(
+      df: DataFrame,
+      sizeInBytes: BigInt,
+      rowCount: Option[BigInt],
+      colStat: Seq[ColumnStat]): Unit = {
+    val optimizedPlan = df.queryExecution.optimizedPlan
+    val attributeStats = optimizedPlan.references.zip(colStat)
+    assert(optimizedPlan.stats ===
+      Statistics(sizeInBytes, rowCount, AttributeMap(attributeStats.toSeq)))
+  }
+
   // Filter out the checksum file refer to ChecksumFileSystem#isChecksumFile.
   def getDataSize(file: File): Long = {
     file.listFiles.filter { f =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
index ec698818a0d85..10f2b3004d708 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql
 
 import org.apache.spark.sql.functions._
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 
 
@@ -48,6 +49,65 @@ class StringFunctionsSuite extends QueryTest with SharedSparkSession {
       Row("a||b"))
   }
 
+  test("SPARK-31993: concat_ws in agg function with plenty of string/array types columns") {
+    withSQLConf(SQLConf.CODEGEN_METHOD_SPLIT_THRESHOLD.key -> "1024",
+      SQLConf.CODEGEN_FACTORY_MODE.key -> "CODEGEN_ONLY") {
+
+      val (df, genColNames, genColValues) = prepareTestConcatWsColumns()
+      val groupedCols = Seq($"a") ++ genColNames.map(col)
+      val concatCols = Seq(collect_list($"b"), collect_list($"c")) ++ genColNames.map(col)
+      val df2 = df
+        .groupBy(groupedCols: _*)
+        .agg(concat_ws(",", concatCols: _*).as("con"))
+        .select("con")
+
+      val expected = Seq(
+        Row((Seq("b1", "b2") ++ genColValues).mkString(",")),
+        Row((Seq("b3", "b4") ++ genColValues).mkString(","))
+      )
+
+      checkAnswer(df2, expected)
+    }
+  }
+
+  // This test doesn't fail without SPARK-31993, but still be useful for regression test.
+  test("SPARK-31993: concat_ws in agg function with plenty of string types columns") {
+    withSQLConf(SQLConf.CODEGEN_METHOD_SPLIT_THRESHOLD.key -> "1024",
+      SQLConf.CODEGEN_FACTORY_MODE.key -> "CODEGEN_ONLY") {
+
+      val (df, genColNames, genColValues) = prepareTestConcatWsColumns()
+      val groupedCols = Seq($"a") ++ genColNames.map(col)
+      val concatCols = groupedCols
+      val df2 = df
+        .groupBy(groupedCols: _*)
+        .agg(concat_ws(",", concatCols: _*).as("con"))
+        .select("con")
+
+      val expected = Seq(
+        Row((Seq("a") ++ genColValues).mkString(",")),
+        Row((Seq("b") ++ genColValues).mkString(","))
+      )
+
+      checkAnswer(df2, expected)
+    }
+  }
+
+  private def prepareTestConcatWsColumns(): (DataFrame, Seq[String], Seq[String]) = {
+    val genColNames = (1 to 30).map { idx => s"col_$idx" }
+    val genColValues = (1 to 30).map { _.toString }
+    val genCols = genColValues.map(lit)
+
+    val df = Seq[(String, String, String)](
+      ("a", "b1", null),
+      ("a", "b2", null),
+      ("b", "b3", null),
+      ("b", "b4", null))
+      .toDF("a", "b", "c")
+      .withColumns(genColNames, genCols)
+
+    (df, genColNames, genColValues)
+  }
+
   test("string elt") {
     val df = Seq[(String, String, String, Int)](("hello", "world", null, 15))
       .toDF("a", "b", "c", "d")
@@ -96,6 +156,7 @@ class StringFunctionsSuite extends QueryTest with SharedSparkSession {
 
   test("non-matching optional group") {
     val df = Seq(Tuple1("aaaac")).toDF("s")
+
     checkAnswer(
       df.select(regexp_extract($"s", "(foo)", 1)),
       Row("")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
index 8118eb4f79c17..bb6b402e8156d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
@@ -24,6 +24,7 @@ import org.apache.spark.sql.catalyst.plans.logical.{Join, LogicalPlan, Sort}
 import org.apache.spark.sql.execution.{ColumnarToRowExec, ExecSubqueryExpression, FileSourceScanExec, InputAdapter, ReusedSubqueryExec, ScalarSubquery, SubqueryExec, WholeStageCodegenExec}
 import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecution}
 import org.apache.spark.sql.execution.datasources.FileScanRDD
+import org.apache.spark.sql.execution.joins.{BaseJoinExec, BroadcastHashJoinExec, BroadcastNestedLoopJoinExec}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 
@@ -511,6 +512,18 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
       Row(1, null) :: Row(2, 6.0) :: Row(3, 2.0) :: Row(null, null) :: Row(6, null) :: Nil)
   }
 
+  test("SPARK-34269: correlated subquery with view in aggregate's grouping expression") {
+    withTable("tr") {
+      withView("vr") {
+        r.write.saveAsTable("tr")
+        sql("create view vr as select * from tr")
+        checkAnswer(
+          sql("select a, (select sum(d) from vr where a = c) sum_d from l l1 group by 1, 2"),
+          Row(1, null) :: Row(2, 6.0) :: Row(3, 2.0) :: Row(null, null) :: Row(6, null) :: Nil)
+      }
+    }
+  }
+
   test("SPARK-18504 extra GROUP BY column in correlated scalar subquery is not permitted") {
     withTempView("t") {
       Seq((1, 1), (1, 2)).toDF("c1", "c2").createOrReplaceTempView("t")
@@ -541,7 +554,7 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
       sql("select a, (select sum(b) from l l2 where l2.a < l1.a) sum_b from l l1")
     }
     assert(msg1.getMessage.contains(
-      "Correlated column is not allowed in a non-equality predicate:"))
+      "Correlated column is not allowed in predicate (l2.a < outer(l1.a))"))
   }
 
   test("disjunctive correlated scalar subquery") {
@@ -882,7 +895,7 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
     withTempView("t") {
       Seq(1 -> "a").toDF("i", "j").createOrReplaceTempView("t")
       val e = intercept[AnalysisException](sql("SELECT (SELECT count(*) FROM t WHERE a = 1)"))
-      assert(e.message.contains("cannot resolve '`a`' given input columns: [t.i, t.j]"))
+      assert(e.message.contains("cannot resolve 'a' given input columns: [t.i, t.j]"))
     }
   }
 
@@ -992,7 +1005,7 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
         subqueryExpressions ++= (getSubqueryExpressions(s.plan) :+ s)
         s
     }
-    subqueryExpressions
+    subqueryExpressions.toSeq
   }
 
   private def getNumSorts(plan: LogicalPlan): Int = {
@@ -1102,12 +1115,12 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
            |SELECT c1 FROM t1
            |WHERE
            |c1 IN ((
-           |        SELECT c1 FROM t2
+           |        SELECT c1 + 1 AS c1 FROM t2
            |        ORDER BY c1
            |       )
            |       UNION
            |       (
-           |         SELECT c1 FROM t2
+           |         SELECT c1 + 2 AS c1 FROM t2
            |         ORDER BY c1
            |       ))
         """.stripMargin
@@ -1313,7 +1326,7 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
       // need to execute the query before we can examine fs.inputRDDs()
       assert(stripAQEPlan(df.queryExecution.executedPlan) match {
         case WholeStageCodegenExec(ColumnarToRowExec(InputAdapter(
-            fs @ FileSourceScanExec(_, _, _, partitionFilters, _, _, _)))) =>
+            fs @ FileSourceScanExec(_, _, _, partitionFilters, _, _, _, _, _)))) =>
           partitionFilters.exists(ExecSubqueryExpression.hasSubquery) &&
             fs.inputRDDs().forall(
               _.asInstanceOf[FileScanRDD].filePartitions.forall(
@@ -1646,4 +1659,181 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
     checkAnswer(df, df2)
     checkAnswer(df, Nil)
   }
+
+  test("SPARK-32290: SingleColumn Null Aware Anti Join Optimize") {
+    Seq(true, false).foreach { enableNAAJ =>
+      Seq(true, false).foreach { enableAQE =>
+        Seq(true, false).foreach { enableCodegen =>
+          withSQLConf(
+            SQLConf.OPTIMIZE_NULL_AWARE_ANTI_JOIN.key -> enableNAAJ.toString,
+            SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> enableAQE.toString,
+            SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> enableCodegen.toString) {
+
+            def findJoinExec(df: DataFrame): BaseJoinExec = {
+              df.queryExecution.sparkPlan.collectFirst {
+                case j: BaseJoinExec => j
+              }.get
+            }
+
+            var df: DataFrame = null
+            var joinExec: BaseJoinExec = null
+
+            // single column not in subquery -- empty sub-query
+            df = sql("select * from l where a not in (select c from r where c > 10)")
+            checkAnswer(df, spark.table("l"))
+            if (enableNAAJ) {
+              joinExec = findJoinExec(df)
+              assert(joinExec.isInstanceOf[BroadcastHashJoinExec])
+              assert(joinExec.asInstanceOf[BroadcastHashJoinExec].isNullAwareAntiJoin)
+            } else {
+              assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec])
+            }
+
+            // single column not in subquery -- sub-query include null
+            df = sql("select * from l where a not in (select c from r where d < 6.0)")
+            checkAnswer(df, Seq.empty)
+            if (enableNAAJ) {
+              joinExec = findJoinExec(df)
+              assert(joinExec.isInstanceOf[BroadcastHashJoinExec])
+              assert(joinExec.asInstanceOf[BroadcastHashJoinExec].isNullAwareAntiJoin)
+            } else {
+              assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec])
+            }
+
+            // single column not in subquery -- streamedSide row is null
+            df =
+              sql("select * from l where b = 5.0 and a not in(select c from r where c is not null)")
+            checkAnswer(df, Seq.empty)
+            if (enableNAAJ) {
+              joinExec = findJoinExec(df)
+              assert(joinExec.isInstanceOf[BroadcastHashJoinExec])
+              assert(joinExec.asInstanceOf[BroadcastHashJoinExec].isNullAwareAntiJoin)
+            } else {
+              assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec])
+            }
+
+            // single column not in subquery -- streamedSide row is not null, match found
+            df =
+              sql("select * from l where a = 6 and a not in (select c from r where c is not null)")
+            checkAnswer(df, Seq.empty)
+            if (enableNAAJ) {
+              joinExec = findJoinExec(df)
+              assert(joinExec.isInstanceOf[BroadcastHashJoinExec])
+              assert(joinExec.asInstanceOf[BroadcastHashJoinExec].isNullAwareAntiJoin)
+            } else {
+              assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec])
+            }
+
+            // single column not in subquery -- streamedSide row is not null, match not found
+            df =
+              sql("select * from l where a = 1 and a not in (select c from r where c is not null)")
+            checkAnswer(df, Row(1, 2.0) :: Row(1, 2.0) :: Nil)
+            if (enableNAAJ) {
+              joinExec = findJoinExec(df)
+              assert(joinExec.isInstanceOf[BroadcastHashJoinExec])
+              assert(joinExec.asInstanceOf[BroadcastHashJoinExec].isNullAwareAntiJoin)
+            } else {
+              assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec])
+            }
+
+            // single column not in subquery -- d = b + 10 joinKey found, match ExtractEquiJoinKeys
+            df = sql("select * from l where a not in (select c from r where d = b + 10)")
+            checkAnswer(df, spark.table("l"))
+            joinExec = findJoinExec(df)
+            assert(joinExec.isInstanceOf[BroadcastHashJoinExec])
+            assert(!joinExec.asInstanceOf[BroadcastHashJoinExec].isNullAwareAntiJoin)
+
+            // single column not in subquery -- d = b + 10 and b = 5.0 => d = 15, joinKey not found
+            // match ExtractSingleColumnNullAwareAntiJoin
+            df =
+              sql("select * from l where b = 5.0 and a not in (select c from r where d = b + 10)")
+            checkAnswer(df, Row(null, 5.0) :: Nil)
+            if (enableNAAJ) {
+              joinExec = findJoinExec(df)
+              assert(joinExec.isInstanceOf[BroadcastHashJoinExec])
+              assert(joinExec.asInstanceOf[BroadcastHashJoinExec].isNullAwareAntiJoin)
+            } else {
+              assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec])
+            }
+
+            // multi column not in subquery
+            df = sql("select * from l where (a, b) not in (select c, d from r where c > 10)")
+            checkAnswer(df, spark.table("l"))
+            assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec])
+          }
+        }
+      }
+    }
+  }
+
+  test("SPARK-28379: non-aggregated zero row scalar subquery") {
+    checkAnswer(
+      sql("select a, (select id from range(0) where id = a) from l where a = 3"),
+      Row(3, null))
+    checkAnswer(
+      sql("select a, (select c from (select * from r limit 0) where c = a) from l where a = 3"),
+      Row(3, null))
+  }
+
+  test("SPARK-28379: non-aggregated single row correlated scalar subquery") {
+    withTempView("t") {
+      Seq((0, 1), (1, 2)).toDF("c1", "c2").createOrReplaceTempView("t")
+      // inline table
+      checkAnswer(
+        sql("select c1, c2, (select col1 from values (0, 1) where col2 = c2) from t"),
+        Row(0, 1, 0) :: Row(1, 2, null) :: Nil)
+      // one row relation
+      checkAnswer(
+        sql("select c1, c2, (select a from (select 1 as a) where a = c2) from t"),
+        Row(0, 1, 1) :: Row(1, 2, null) :: Nil)
+      // limit 1 with order by
+      checkAnswer(
+        sql(
+          """
+            |select c1, c2, (
+            |  select b from (select * from l order by a asc nulls last limit 1) where a = c2
+            |) from t
+            |""".stripMargin),
+        Row(0, 1, 2.0) :: Row(1, 2, null) :: Nil)
+      // limit 1 with window
+      checkAnswer(
+        sql(
+          """
+            |select c1, c2, (
+            |  select w from (
+            |    select a, sum(b) over (partition by a) w from l order by a asc nulls last limit 1
+            |  ) where a = c1 + c2
+            |) from t
+            |""".stripMargin),
+        Row(0, 1, 4.0) :: Row(1, 2, null) :: Nil)
+      // set operations
+      checkAnswer(
+        sql(
+          """
+            |select c1, c2, (
+            |  select a from ((select 1 as a) intersect (select 1 as a)) where a = c2
+            |) from t
+            |""".stripMargin),
+        Row(0, 1, 1) :: Row(1, 2, null) :: Nil)
+      // join
+      checkAnswer(
+        sql(
+          """
+            |select c1, c2, (
+            |  select a from (select * from (select 1 as a) join (select 1 as b) on a = b)
+            |  where a = c2
+            |) from t
+            |""".stripMargin),
+        Row(0, 1, 1) :: Row(1, 2, null) :: Nil)
+    }
+  }
+
+  test("SPARK-35080: correlated equality predicates contain only outer references") {
+    withTempView("t") {
+      Seq((0, 1), (1, 1)).toDF("c1", "c2").createOrReplaceTempView("t")
+      checkAnswer(
+        sql("select c1, c2, (select count(*) from l where c1 = c2) from t"),
+        Row(0, 1, 0) :: Row(1, 1, 8) :: Nil)
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSBase.scala
new file mode 100644
index 0000000000000..20cfcecc229ae
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSBase.scala
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.test.SharedSparkSession
+
+trait TPCDSBase extends SharedSparkSession with TPCDSSchema {
+
+  // The TPCDS queries below are based on v1.4
+  private val tpcdsAllQueries: Seq[String] = Seq(
+    "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11",
+    "q12", "q13", "q14a", "q14b", "q15", "q16", "q17", "q18", "q19", "q20",
+    "q21", "q22", "q23a", "q23b", "q24a", "q24b", "q25", "q26", "q27", "q28", "q29", "q30",
+    "q31", "q32", "q33", "q34", "q35", "q36", "q37", "q38", "q39a", "q39b", "q40",
+    "q41", "q42", "q43", "q44", "q45", "q46", "q47", "q48", "q49", "q50",
+    "q51", "q52", "q53", "q54", "q55", "q56", "q57", "q58", "q59", "q60",
+    "q61", "q62", "q63", "q64", "q65", "q66", "q67", "q68", "q69", "q70",
+    "q71", "q72", "q73", "q74", "q75", "q76", "q77", "q78", "q79", "q80",
+    "q81", "q82", "q83", "q84", "q85", "q86", "q87", "q88", "q89", "q90",
+    "q91", "q92", "q93", "q94", "q95", "q96", "q97", "q98", "q99")
+
+  // Since `tpcdsQueriesV2_7_0` has almost the same queries with these ones below,
+  // we skip them in the TPCDS-related tests.
+  // NOTE: q6" and "q75" can cause flaky test results, so we must exclude them.
+  // For more details, see SPARK-35327.
+  private val excludedTpcdsQueries: Set[String] = Set("q6", "q34", "q64", "q74", "q75", "q78")
+
+  val tpcdsQueries: Seq[String] = tpcdsAllQueries.filterNot(excludedTpcdsQueries.contains)
+
+  // This list only includes TPCDS v2.7 queries that are different from v1.4 ones
+  val tpcdsQueriesV2_7_0 = Seq(
+    "q5a", "q6", "q10a", "q11", "q12", "q14", "q14a", "q18a",
+    "q20", "q22", "q22a", "q24", "q27a", "q34", "q35", "q35a", "q36a", "q47", "q49",
+    "q51a", "q57", "q64", "q67a", "q70a", "q72", "q74", "q75", "q77a", "q78",
+    "q80a", "q86a", "q98")
+
+  // These queries are from https://github.com/cloudera/impala-tpcds-kit/tree/master/queries
+  val modifiedTPCDSQueries = Seq(
+    "q3", "q7", "q10", "q19", "q27", "q34", "q42", "q43", "q46", "q52", "q53", "q55", "q59",
+    "q63", "q65", "q68", "q73", "q79", "q89", "q98", "ss_max")
+
+  protected def partitionedByClause(tableName: String): String = {
+    tablePartitionColumns.get(tableName) match {
+      case Some(cols) if cols.nonEmpty => s"PARTITIONED BY (${cols.mkString(", ")})"
+      case _ => ""
+    }
+  }
+
+  val tableNames: Iterable[String] = tableColumns.keys
+
+  def createTable(
+      spark: SparkSession,
+      tableName: String,
+      format: String = "parquet",
+      options: Seq[String] = Nil): Unit = {
+    spark.sql(
+      s"""
+         |CREATE TABLE `$tableName` (${tableColumns(tableName)})
+         |USING $format
+         |${partitionedByClause(tableName)}
+         |${options.mkString("\n")}
+       """.stripMargin)
+  }
+
+  private val originalCBCEnabled = conf.cboEnabled
+  private val originalPlanStatsEnabled = conf.planStatsEnabled
+  private val originalJoinReorderEnabled = conf.joinReorderEnabled
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    if (injectStats) {
+      // Sets configurations for enabling the optimization rules that
+      // exploit data statistics.
+      conf.setConf(SQLConf.CBO_ENABLED, true)
+      conf.setConf(SQLConf.PLAN_STATS_ENABLED, true)
+      conf.setConf(SQLConf.JOIN_REORDER_ENABLED, true)
+    }
+    tableNames.foreach { tableName =>
+      createTable(spark, tableName)
+      if (injectStats) {
+        // To simulate plan generation on actual TPC-DS data, injects data stats here
+        spark.sessionState.catalog.alterTableStats(
+          TableIdentifier(tableName), Some(TPCDSTableStats.sf100TableStats(tableName)))
+      }
+    }
+  }
+
+  override def afterAll(): Unit = {
+    conf.setConf(SQLConf.CBO_ENABLED, originalCBCEnabled)
+    conf.setConf(SQLConf.PLAN_STATS_ENABLED, originalPlanStatsEnabled)
+    conf.setConf(SQLConf.JOIN_REORDER_ENABLED, originalJoinReorderEnabled)
+    tableNames.foreach { tableName =>
+      spark.sessionState.catalog.dropTable(TableIdentifier(tableName), true, true)
+    }
+    super.afterAll()
+  }
+
+  protected def injectStats: Boolean = false
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala
index aacb625d7921f..22e1b838f3f3f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala
@@ -17,75 +17,42 @@
 
 package org.apache.spark.sql
 
+import org.apache.spark.SparkConf
 import org.apache.spark.sql.catalyst.util.resourceToString
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.tags.ExtendedSQLTest
 
 /**
  * This test suite ensures all the TPC-DS queries can be successfully analyzed, optimized
  * and compiled without hitting the max iteration threshold.
  */
-class TPCDSQuerySuite extends BenchmarkQueryTest with TPCDSSchema {
-
-  override def beforeAll(): Unit = {
-    super.beforeAll()
-    for (tableName <- tableNames) {
-      createTable(spark, tableName)
-    }
-  }
-
-  // The TPCDS queries below are based on v1.4
-  val tpcdsQueries = Seq(
-    "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11",
-    "q12", "q13", "q14a", "q14b", "q15", "q16", "q17", "q18", "q19", "q20",
-    "q21", "q22", "q23a", "q23b", "q24a", "q24b", "q25", "q26", "q27", "q28", "q29", "q30",
-    "q31", "q32", "q33", "q34", "q35", "q36", "q37", "q38", "q39a", "q39b", "q40",
-    "q41", "q42", "q43", "q44", "q45", "q46", "q47", "q48", "q49", "q50",
-    "q51", "q52", "q53", "q54", "q55", "q56", "q57", "q58", "q59", "q60",
-    "q61", "q62", "q63", "q64", "q65", "q66", "q67", "q68", "q69", "q70",
-    "q71", "q72", "q73", "q74", "q75", "q76", "q77", "q78", "q79", "q80",
-    "q81", "q82", "q83", "q84", "q85", "q86", "q87", "q88", "q89", "q90",
-    "q91", "q92", "q93", "q94", "q95", "q96", "q97", "q98", "q99")
+@ExtendedSQLTest
+class TPCDSQuerySuite extends BenchmarkQueryTest with TPCDSBase {
 
   tpcdsQueries.foreach { name =>
     val queryString = resourceToString(s"tpcds/$name.sql",
       classLoader = Thread.currentThread().getContextClassLoader)
     test(name) {
-      withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "true") {
-        // check the plans can be properly generated
-        val plan = sql(queryString).queryExecution.executedPlan
-        checkGeneratedCode(plan)
-      }
+      // check the plans can be properly generated
+      val plan = sql(queryString).queryExecution.executedPlan
+      checkGeneratedCode(plan)
     }
   }
 
-  // This list only includes TPCDS v2.7 queries that are different from v1.4 ones
-  val tpcdsQueriesV2_7_0 = Seq(
-    "q5a", "q6", "q10a", "q11", "q12", "q14", "q14a", "q18a",
-    "q20", "q22", "q22a", "q24", "q27a", "q34", "q35", "q35a", "q36a", "q47", "q49",
-    "q51a", "q57", "q64", "q67a", "q70a", "q72", "q74", "q75", "q77a", "q78",
-    "q80a", "q86a", "q98")
-
   tpcdsQueriesV2_7_0.foreach { name =>
     val queryString = resourceToString(s"tpcds-v2.7.0/$name.sql",
       classLoader = Thread.currentThread().getContextClassLoader)
     test(s"$name-v2.7") {
-      withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "true") {
-        // check the plans can be properly generated
-        val plan = sql(queryString).queryExecution.executedPlan
-        checkGeneratedCode(plan)
-      }
+      // check the plans can be properly generated
+      val plan = sql(queryString).queryExecution.executedPlan
+      checkGeneratedCode(plan)
     }
   }
 
-  // These queries are from https://github.com/cloudera/impala-tpcds-kit/tree/master/queries
-  val modifiedTPCDSQueries = Seq(
-    "q3", "q7", "q10", "q19", "q27", "q34", "q42", "q43", "q46", "q52", "q53", "q55", "q59",
-    "q63", "q65", "q68", "q73", "q79", "q89", "q98", "ss_max")
-
   // List up the known queries having too large code in a generated function.
   // A JIRA file for `modified-q3` is as follows;
   // [SPARK-29128] Split predicate code in OR expressions
-  val blackListForMethodCodeSizeCheck = Set("modified-q3")
+  val excludeListForMethodCodeSizeCheck = Set("modified-q3")
 
   modifiedTPCDSQueries.foreach { name =>
     val queryString = resourceToString(s"tpcds-modifiedQueries/$name.sql",
@@ -94,7 +61,18 @@ class TPCDSQuerySuite extends BenchmarkQueryTest with TPCDSSchema {
     test(testName) {
       // check the plans can be properly generated
       val plan = sql(queryString).queryExecution.executedPlan
-      checkGeneratedCode(plan, !blackListForMethodCodeSizeCheck.contains(testName))
+      checkGeneratedCode(plan, !excludeListForMethodCodeSizeCheck.contains(testName))
     }
   }
 }
+
+@ExtendedSQLTest
+class TPCDSQueryWithStatsSuite extends TPCDSQuerySuite {
+  override def injectStats: Boolean = true
+}
+
+@ExtendedSQLTest
+class TPCDSQueryANSISuite extends TPCDSQuerySuite {
+  override protected def sparkConf: SparkConf =
+    super.sparkConf.set(SQLConf.ANSI_ENABLED, true)
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala
new file mode 100644
index 0000000000000..952e8968020aa
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import java.io.File
+import java.nio.file.{Files, Paths}
+
+import org.apache.spark.{SparkConf, SparkContext}
+import org.apache.spark.sql.catalyst.util.{fileToString, resourceToString, stringToFile}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.test.TestSparkSession
+
+/**
+ * End-to-end tests to check TPCDS query results.
+ *
+ * To run this test suite:
+ * {{{
+ *   SPARK_TPCDS_DATA=<path of TPCDS SF=1 data> build/sbt "sql/testOnly *TPCDSQueryTestSuite"
+ * }}}
+ *
+ * To run a single test file upon change:
+ * {{{
+ *   SPARK_TPCDS_DATA=<path of TPCDS SF=1 data>
+ *     build/sbt "~sql/testOnly *TPCDSQueryTestSuite -- -z q79"
+ * }}}
+ *
+ * To re-generate golden files for this suite, run:
+ * {{{
+ *   SPARK_GENERATE_GOLDEN_FILES=1 SPARK_TPCDS_DATA=<path of TPCDS SF=1 data>
+ *     build/sbt "sql/testOnly *TPCDSQueryTestSuite"
+ * }}}
+ *
+ * To re-generate golden file for a single test, run:
+ * {{{
+ *   SPARK_GENERATE_GOLDEN_FILES=1 SPARK_TPCDS_DATA=<path of TPCDS SF=1 data>
+ *     build/sbt "sql/testOnly *TPCDSQueryTestSuite -- -z q79"
+ * }}}
+ */
+class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelper {
+
+  private val tpcdsDataPath = sys.env.get("SPARK_TPCDS_DATA")
+  private val regenerateGoldenFiles = sys.env.get("SPARK_GENERATE_GOLDEN_FILES").exists(_ == "1")
+
+  // To make output results deterministic
+  protected override def sparkConf: SparkConf = super.sparkConf
+    .set(SQLConf.SHUFFLE_PARTITIONS.key, "1")
+
+  protected override def createSparkSession: TestSparkSession = {
+    new TestSparkSession(new SparkContext("local[1]", this.getClass.getSimpleName, sparkConf))
+  }
+
+  // We use SF=1 table data here, so we cannot use SF=100 stats
+  protected override val injectStats: Boolean = false
+
+  if (tpcdsDataPath.nonEmpty) {
+    val nonExistentTables = tableNames.filterNot { tableName =>
+      Files.exists(Paths.get(s"${tpcdsDataPath.get}/$tableName"))
+    }
+    if (nonExistentTables.nonEmpty) {
+      fail(s"Non-existent TPCDS table paths found in ${tpcdsDataPath.get}: " +
+        nonExistentTables.mkString(", "))
+    }
+  }
+
+  protected val baseResourcePath = {
+    // use the same way as `SQLQueryTestSuite` to get the resource path
+    getWorkspaceFilePath("sql", "core", "src", "test", "resources", "tpcds-query-results")
+      .toFile.getAbsolutePath
+  }
+
+  override def createTable(
+      spark: SparkSession,
+      tableName: String,
+      format: String = "parquet",
+      options: Seq[String] = Nil): Unit = {
+    spark.sql(
+      s"""
+         |CREATE TABLE `$tableName` (${tableColumns(tableName)})
+         |USING $format
+         |LOCATION '${tpcdsDataPath.get}/$tableName'
+         |${options.mkString("\n")}
+       """.stripMargin)
+  }
+
+  private def runQuery(query: String, goldenFile: File): Unit = {
+    val (schema, output) = handleExceptions(getNormalizedResult(spark, query))
+    val queryString = query.trim
+    val outputString = output.mkString("\n").replaceAll("\\s+$", "")
+    if (regenerateGoldenFiles) {
+      val goldenOutput = {
+        s"-- Automatically generated by ${getClass.getSimpleName}\n\n" +
+          s"-- !query schema\n" +
+          schema + "\n" +
+          s"-- !query output\n" +
+          outputString +
+          "\n"
+      }
+      val parent = goldenFile.getParentFile
+      if (!parent.exists()) {
+        assert(parent.mkdirs(), "Could not create directory: " + parent)
+      }
+      stringToFile(goldenFile, goldenOutput)
+    }
+
+    // Read back the golden file.
+    val (expectedSchema, expectedOutput) = {
+      val goldenOutput = fileToString(goldenFile)
+      val segments = goldenOutput.split("-- !query.*\n")
+
+      // query has 3 segments, plus the header
+      assert(segments.size == 3,
+        s"Expected 3 blocks in result file but got ${segments.size}. " +
+          "Try regenerate the result files.")
+
+      (segments(1).trim, segments(2).replaceAll("\\s+$", ""))
+    }
+
+    assertResult(expectedSchema, s"Schema did not match\n$queryString") { schema }
+    assertResult(expectedOutput, s"Result did not match\n$queryString") { outputString }
+  }
+
+  if (tpcdsDataPath.nonEmpty) {
+    tpcdsQueries.foreach { name =>
+      val queryString = resourceToString(s"tpcds/$name.sql",
+        classLoader = Thread.currentThread().getContextClassLoader)
+      test(name) {
+        val goldenFile = new File(s"$baseResourcePath/v1_4", s"$name.sql.out")
+        runQuery(queryString, goldenFile)
+      }
+    }
+
+    tpcdsQueriesV2_7_0.foreach { name =>
+      val queryString = resourceToString(s"tpcds-v2.7.0/$name.sql",
+        classLoader = Thread.currentThread().getContextClassLoader)
+      test(s"$name-v2.7") {
+        val goldenFile = new File(s"$baseResourcePath/v2_7", s"$name.sql.out")
+        runQuery(queryString, goldenFile)
+      }
+    }
+  } else {
+    ignore("skipped because env `SPARK_TPCDS_DATA` is not set") {}
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala
index 43974ad22f2ef..7b2ed8d28274d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala
@@ -17,243 +17,539 @@
 
 package org.apache.spark.sql
 
+/**
+ * Base trait for TPC-DS related tests.
+ *
+ * Datatype mapping for TPC-DS and Spark SQL, fully matching schemas defined in `tpcds.sql` of the
+ * official tpcds toolkit
+ * see more at:
+ *   http://www.tpc.org/tpc_documents_current_versions/pdf/tpc-ds_v2.9.0.pdf
+ *
+ *    |---------------|---------------|
+ *    |    TPC-DS     |  Spark  SQL   |
+ *    |---------------|---------------|
+ *    |  Identifier   |      INT      |
+ *    |---------------|---------------|
+ *    |    Integer    |      INT      |
+ *    |---------------|---------------|
+ *    | Decimal(d, f) | Decimal(d, f) |
+ *    |---------------|---------------|
+ *    |    Char(N)    |    Char(N)    |
+ *    |---------------|---------------|
+ *    |  Varchar(N)   |  Varchar(N)   |
+ *    |---------------|---------------|
+ *    |     Date      |     Date      |
+ *    |---------------|---------------|
+ */
 trait TPCDSSchema {
 
-  private val tableColumns = Map(
+  protected val tableColumns: Map[String, String] = Map(
     "store_sales" ->
       """
-        |`ss_sold_date_sk` INT, `ss_sold_time_sk` INT, `ss_item_sk` INT, `ss_customer_sk` INT,
-        |`ss_cdemo_sk` INT, `ss_hdemo_sk` INT, `ss_addr_sk` INT, `ss_store_sk` INT,
-        |`ss_promo_sk` INT, `ss_ticket_number` INT, `ss_quantity` INT,
-        |`ss_wholesale_cost` DECIMAL(7,2), `ss_list_price` DECIMAL(7,2),
-        |`ss_sales_price` DECIMAL(7,2), `ss_ext_discount_amt` DECIMAL(7,2),
-        |`ss_ext_sales_price` DECIMAL(7,2), `ss_ext_wholesale_cost` DECIMAL(7,2),
-        |`ss_ext_list_price` DECIMAL(7,2), `ss_ext_tax` DECIMAL(7,2),
-        |`ss_coupon_amt` DECIMAL(7,2), `ss_net_paid` DECIMAL(7,2),
-        |`ss_net_paid_inc_tax` DECIMAL(7,2), `ss_net_profit` DECIMAL(7,2)
+        |`ss_sold_date_sk` INT,
+        |`ss_sold_time_sk` INT,
+        |`ss_item_sk` INT,
+        |`ss_customer_sk` INT,
+        |`ss_cdemo_sk` INT,
+        |`ss_hdemo_sk` INT,
+        |`ss_addr_sk` INT,
+        |`ss_store_sk` INT,
+        |`ss_promo_sk` INT,
+        |`ss_ticket_number` INT,
+        |`ss_quantity` INT,
+        |`ss_wholesale_cost` DECIMAL(7,2),
+        |`ss_list_price` DECIMAL(7,2),
+        |`ss_sales_price` DECIMAL(7,2),
+        |`ss_ext_discount_amt` DECIMAL(7,2),
+        |`ss_ext_sales_price` DECIMAL(7,2),
+        |`ss_ext_wholesale_cost` DECIMAL(7,2),
+        |`ss_ext_list_price` DECIMAL(7,2),
+        |`ss_ext_tax` DECIMAL(7,2),
+        |`ss_coupon_amt` DECIMAL(7,2),
+        |`ss_net_paid` DECIMAL(7,2),
+        |`ss_net_paid_inc_tax` DECIMAL(7,2),
+        |`ss_net_profit` DECIMAL(7,2)
       """.stripMargin,
     "store_returns" ->
       """
-        |`sr_returned_date_sk` BIGINT, `sr_return_time_sk` BIGINT, `sr_item_sk` BIGINT,
-        |`sr_customer_sk` BIGINT, `sr_cdemo_sk` BIGINT, `sr_hdemo_sk` BIGINT, `sr_addr_sk` BIGINT,
-        |`sr_store_sk` BIGINT, `sr_reason_sk` BIGINT, `sr_ticket_number` BIGINT,
-        |`sr_return_quantity` INT, `sr_return_amt` DECIMAL(7,2), `sr_return_tax` DECIMAL(7,2),
-        |`sr_return_amt_inc_tax` DECIMAL(7,2), `sr_fee` DECIMAL(7,2),
-        |`sr_return_ship_cost` DECIMAL(7,2), `sr_refunded_cash` DECIMAL(7,2),
-        |`sr_reversed_charge` DECIMAL(7,2), `sr_store_credit` DECIMAL(7,2),
+        |`sr_returned_date_sk` INT,
+        |`sr_return_time_sk` INT,
+        |`sr_item_sk` INT,
+        |`sr_customer_sk` INT,
+        |`sr_cdemo_sk` INT,
+        |`sr_hdemo_sk` INT,
+        |`sr_addr_sk` INT,
+        |`sr_store_sk` INT,
+        |`sr_reason_sk` INT,
+        |`sr_ticket_number` INT,
+        |`sr_return_quantity` INT,
+        |`sr_return_amt` DECIMAL(7,2),
+        |`sr_return_tax` DECIMAL(7,2),
+        |`sr_return_amt_inc_tax` DECIMAL(7,2),
+        |`sr_fee` DECIMAL(7,2),
+        |`sr_return_ship_cost` DECIMAL(7,2),
+        |`sr_refunded_cash` DECIMAL(7,2),
+        |`sr_reversed_charge` DECIMAL(7,2),
+        |`sr_store_credit` DECIMAL(7,2),
         |`sr_net_loss` DECIMAL(7,2)
       """.stripMargin,
     "catalog_sales" ->
       """
-        |`cs_sold_date_sk` INT, `cs_sold_time_sk` INT, `cs_ship_date_sk` INT,
-        |`cs_bill_customer_sk` INT, `cs_bill_cdemo_sk` INT, `cs_bill_hdemo_sk` INT,
-        |`cs_bill_addr_sk` INT, `cs_ship_customer_sk` INT, `cs_ship_cdemo_sk` INT,
-        |`cs_ship_hdemo_sk` INT, `cs_ship_addr_sk` INT, `cs_call_center_sk` INT,
-        |`cs_catalog_page_sk` INT, `cs_ship_mode_sk` INT, `cs_warehouse_sk` INT,
-        |`cs_item_sk` INT, `cs_promo_sk` INT, `cs_order_number` INT, `cs_quantity` INT,
-        |`cs_wholesale_cost` DECIMAL(7,2), `cs_list_price` DECIMAL(7,2),
-        |`cs_sales_price` DECIMAL(7,2), `cs_ext_discount_amt` DECIMAL(7,2),
-        |`cs_ext_sales_price` DECIMAL(7,2), `cs_ext_wholesale_cost` DECIMAL(7,2),
-        |`cs_ext_list_price` DECIMAL(7,2), `cs_ext_tax` DECIMAL(7,2),
-        |`cs_coupon_amt` DECIMAL(7,2), `cs_ext_ship_cost` DECIMAL(7,2),
-        |`cs_net_paid` DECIMAL(7,2), `cs_net_paid_inc_tax` DECIMAL(7,2),
-        |`cs_net_paid_inc_ship` DECIMAL(7,2), `cs_net_paid_inc_ship_tax` DECIMAL(7,2),
+        |`cs_sold_date_sk` INT,
+        |`cs_sold_time_sk` INT,
+        |`cs_ship_date_sk` INT,
+        |`cs_bill_customer_sk` INT,
+        |`cs_bill_cdemo_sk` INT,
+        |`cs_bill_hdemo_sk` INT,
+        |`cs_bill_addr_sk` INT,
+        |`cs_ship_customer_sk` INT,
+        |`cs_ship_cdemo_sk` INT,
+        |`cs_ship_hdemo_sk` INT,
+        |`cs_ship_addr_sk` INT,
+        |`cs_call_center_sk` INT,
+        |`cs_catalog_page_sk` INT,
+        |`cs_ship_mode_sk` INT,
+        |`cs_warehouse_sk` INT,
+        |`cs_item_sk` INT,
+        |`cs_promo_sk` INT,
+        |`cs_order_number` INT,
+        |`cs_quantity` INT,
+        |`cs_wholesale_cost` DECIMAL(7,2),
+        |`cs_list_price` DECIMAL(7,2),
+        |`cs_sales_price` DECIMAL(7,2),
+        |`cs_ext_discount_amt` DECIMAL(7,2),
+        |`cs_ext_sales_price` DECIMAL(7,2),
+        |`cs_ext_wholesale_cost` DECIMAL(7,2),
+        |`cs_ext_list_price` DECIMAL(7,2),
+        |`cs_ext_tax` DECIMAL(7,2),
+        |`cs_coupon_amt` DECIMAL(7,2),
+        |`cs_ext_ship_cost` DECIMAL(7,2),
+        |`cs_net_paid` DECIMAL(7,2),
+        |`cs_net_paid_inc_tax` DECIMAL(7,2),
+        |`cs_net_paid_inc_ship` DECIMAL(7,2),
+        |`cs_net_paid_inc_ship_tax` DECIMAL(7,2),
         |`cs_net_profit` DECIMAL(7,2)
       """.stripMargin,
     "catalog_returns" ->
       """
-        |`cr_returned_date_sk` INT, `cr_returned_time_sk` INT, `cr_item_sk` INT,
-        |`cr_refunded_customer_sk` INT, `cr_refunded_cdemo_sk` INT, `cr_refunded_hdemo_sk` INT,
-        |`cr_refunded_addr_sk` INT, `cr_returning_customer_sk` INT, `cr_returning_cdemo_sk` INT,
-        |`cr_returning_hdemo_sk` INT, `cr_returning_addr_sk` INT, `cr_call_center_sk` INT,
-        |`cr_catalog_page_sk` INT, `cr_ship_mode_sk` INT, `cr_warehouse_sk` INT,
-        |`cr_reason_sk` INT,`cr_order_number` INT, `cr_return_quantity` INT,
-        |`cr_return_amount` DECIMAL(7,2), `cr_return_tax` DECIMAL(7,2),
-        |`cr_return_amt_inc_tax` DECIMAL(7,2), `cr_fee` DECIMAL(7,2),
-        |`cr_return_ship_cost` DECIMAL(7,2), `cr_refunded_cash` DECIMAL(7,2),
-        |`cr_reversed_charge` DECIMAL(7,2), `cr_store_credit` DECIMAL(7,2),
+        |`cr_returned_date_sk` INT,
+        |`cr_returned_time_sk` INT,
+        |`cr_item_sk` INT,
+        |`cr_refunded_customer_sk` INT,
+        |`cr_refunded_cdemo_sk` INT,
+        |`cr_refunded_hdemo_sk` INT,
+        |`cr_refunded_addr_sk` INT,
+        |`cr_returning_customer_sk` INT,
+        |`cr_returning_cdemo_sk` INT,
+        |`cr_returning_hdemo_sk` INT,
+        |`cr_returning_addr_sk` INT,
+        |`cr_call_center_sk` INT,
+        |`cr_catalog_page_sk` INT,
+        |`cr_ship_mode_sk` INT,
+        |`cr_warehouse_sk` INT,
+        |`cr_reason_sk` INT,`cr_order_number` INT,
+        |`cr_return_quantity` INT,
+        |`cr_return_amount` DECIMAL(7,2),
+        |`cr_return_tax` DECIMAL(7,2),
+        |`cr_return_amt_inc_tax` DECIMAL(7,2),
+        |`cr_fee` DECIMAL(7,2),
+        |`cr_return_ship_cost` DECIMAL(7,2),
+        |`cr_refunded_cash` DECIMAL(7,2),
+        |`cr_reversed_charge` DECIMAL(7,2),
+        |`cr_store_credit` DECIMAL(7,2),
         |`cr_net_loss` DECIMAL(7,2)
       """.stripMargin,
     "web_sales" ->
       """
-        |`ws_sold_date_sk` INT, `ws_sold_time_sk` INT, `ws_ship_date_sk` INT, `ws_item_sk` INT,
-        |`ws_bill_customer_sk` INT, `ws_bill_cdemo_sk` INT, `ws_bill_hdemo_sk` INT,
-        |`ws_bill_addr_sk` INT, `ws_ship_customer_sk` INT, `ws_ship_cdemo_sk` INT,
-        |`ws_ship_hdemo_sk` INT, `ws_ship_addr_sk` INT, `ws_web_page_sk` INT,
-        |`ws_web_site_sk` INT, `ws_ship_mode_sk` INT, `ws_warehouse_sk` INT, `ws_promo_sk` INT,
-        |`ws_order_number` INT, `ws_quantity` INT, `ws_wholesale_cost` DECIMAL(7,2),
-        |`ws_list_price` DECIMAL(7,2), `ws_sales_price` DECIMAL(7,2),
-        |`ws_ext_discount_amt` DECIMAL(7,2), `ws_ext_sales_price` DECIMAL(7,2),
-        |`ws_ext_wholesale_cost` DECIMAL(7,2), `ws_ext_list_price` DECIMAL(7,2),
-        |`ws_ext_tax` DECIMAL(7,2), `ws_coupon_amt` DECIMAL(7,2), `ws_ext_ship_cost` DECIMAL(7,2),
-        |`ws_net_paid` DECIMAL(7,2), `ws_net_paid_inc_tax` DECIMAL(7,2),
-        |`ws_net_paid_inc_ship` DECIMAL(7,2), `ws_net_paid_inc_ship_tax` DECIMAL(7,2),
+        |`ws_sold_date_sk` INT,
+        |`ws_sold_time_sk` INT,
+        |`ws_ship_date_sk` INT,
+        |`ws_item_sk` INT,
+        |`ws_bill_customer_sk` INT,
+        |`ws_bill_cdemo_sk` INT,
+        |`ws_bill_hdemo_sk` INT,
+        |`ws_bill_addr_sk` INT,
+        |`ws_ship_customer_sk` INT,
+        |`ws_ship_cdemo_sk` INT,
+        |`ws_ship_hdemo_sk` INT,
+        |`ws_ship_addr_sk` INT,
+        |`ws_web_page_sk` INT,
+        |`ws_web_site_sk` INT,
+        |`ws_ship_mode_sk` INT,
+        |`ws_warehouse_sk` INT,
+        |`ws_promo_sk` INT,
+        |`ws_order_number` INT,
+        |`ws_quantity` INT,
+        |`ws_wholesale_cost` DECIMAL(7,2),
+        |`ws_list_price` DECIMAL(7,2),
+        |`ws_sales_price` DECIMAL(7,2),
+        |`ws_ext_discount_amt` DECIMAL(7,2),
+        |`ws_ext_sales_price` DECIMAL(7,2),
+        |`ws_ext_wholesale_cost` DECIMAL(7,2),
+        |`ws_ext_list_price` DECIMAL(7,2),
+        |`ws_ext_tax` DECIMAL(7,2),
+        |`ws_coupon_amt` DECIMAL(7,2),
+        |`ws_ext_ship_cost` DECIMAL(7,2),
+        |`ws_net_paid` DECIMAL(7,2),
+        |`ws_net_paid_inc_tax` DECIMAL(7,2),
+        |`ws_net_paid_inc_ship` DECIMAL(7,2),
+        |`ws_net_paid_inc_ship_tax` DECIMAL(7,2),
         |`ws_net_profit` DECIMAL(7,2)
       """.stripMargin,
     "web_returns" ->
       """
-        |`wr_returned_date_sk` BIGINT, `wr_returned_time_sk` BIGINT, `wr_item_sk` BIGINT,
-        |`wr_refunded_customer_sk` BIGINT, `wr_refunded_cdemo_sk` BIGINT,
-        |`wr_refunded_hdemo_sk` BIGINT, `wr_refunded_addr_sk` BIGINT,
-        |`wr_returning_customer_sk` BIGINT, `wr_returning_cdemo_sk` BIGINT,
-        |`wr_returning_hdemo_sk` BIGINT, `wr_returning_addr_sk` BIGINT, `wr_web_page_sk` BIGINT,
-        |`wr_reason_sk` BIGINT, `wr_order_number` BIGINT, `wr_return_quantity` INT,
-        |`wr_return_amt` DECIMAL(7,2), `wr_return_tax` DECIMAL(7,2),
-        |`wr_return_amt_inc_tax` DECIMAL(7,2), `wr_fee` DECIMAL(7,2),
-        |`wr_return_ship_cost` DECIMAL(7,2), `wr_refunded_cash` DECIMAL(7,2),
-        |`wr_reversed_charge` DECIMAL(7,2), `wr_account_credit` DECIMAL(7,2),
+        |`wr_returned_date_sk` INT,
+        |`wr_returned_time_sk` INT,
+        |`wr_item_sk` INT,
+        |`wr_refunded_customer_sk` INT,
+        |`wr_refunded_cdemo_sk` INT,
+        |`wr_refunded_hdemo_sk` INT,
+        |`wr_refunded_addr_sk` INT,
+        |`wr_returning_customer_sk` INT,
+        |`wr_returning_cdemo_sk` INT,
+        |`wr_returning_hdemo_sk` INT,
+        |`wr_returning_addr_sk` INT,
+        |`wr_web_page_sk` INT,
+        |`wr_reason_sk` INT,
+        |`wr_order_number` INT,
+        |`wr_return_quantity` INT,
+        |`wr_return_amt` DECIMAL(7,2),
+        |`wr_return_tax` DECIMAL(7,2),
+        |`wr_return_amt_inc_tax` DECIMAL(7,2),
+        |`wr_fee` DECIMAL(7,2),
+        |`wr_return_ship_cost` DECIMAL(7,2),
+        |`wr_refunded_cash` DECIMAL(7,2),
+        |`wr_reversed_charge` DECIMAL(7,2),
+        |`wr_account_credit` DECIMAL(7,2),
         |`wr_net_loss` DECIMAL(7,2)
       """.stripMargin,
     "inventory" ->
       """
-        |`inv_date_sk` INT, `inv_item_sk` INT, `inv_warehouse_sk` INT,
+        |`inv_date_sk` INT,
+        |`inv_item_sk` INT,
+        |`inv_warehouse_sk` INT,
         |`inv_quantity_on_hand` INT
       """.stripMargin,
     "store" ->
       """
-        |`s_store_sk` INT, `s_store_id` STRING, `s_rec_start_date` DATE,
-        |`s_rec_end_date` DATE, `s_closed_date_sk` INT, `s_store_name` STRING,
-        |`s_number_employees` INT, `s_floor_space` INT, `s_hours` STRING, `s_manager` STRING,
-        |`s_market_id` INT, `s_geography_class` STRING, `s_market_desc` STRING,
-        |`s_market_manager` STRING, `s_division_id` INT, `s_division_name` STRING,
-        |`s_company_id` INT, `s_company_name` STRING, `s_street_number` STRING,
-        |`s_street_name` STRING, `s_street_type` STRING, `s_suite_number` STRING, `s_city` STRING,
-        |`s_county` STRING, `s_state` STRING, `s_zip` STRING, `s_country` STRING,
-        |`s_gmt_offset` DECIMAL(5,2), `s_tax_percentage` DECIMAL(5,2)
+        |`s_store_sk` INT,
+        |`s_store_id` CHAR(16),
+        |`s_rec_start_date` DATE,
+        |`s_rec_end_date` DATE,
+        |`s_closed_date_sk` INT,
+        |`s_store_name` VARCHAR(50),
+        |`s_number_employees` INT,
+        |`s_floor_space` INT,
+        |`s_hours` CHAR(20),
+        |`s_manager` VARCHAR(40),
+        |`s_market_id` INT,
+        |`s_geography_class` VARCHAR(100),
+        |`s_market_desc` VARCHAR(100),
+        |`s_market_manager` VARCHAR(40),
+        |`s_division_id` INT,
+        |`s_division_name` VARCHAR(50),
+        |`s_company_id` INT,
+        |`s_company_name` VARCHAR(50),
+        |`s_street_number` VARCHAR(10),
+        |`s_street_name` VARCHAR(60),
+        |`s_street_type` CHAR(15),
+        |`s_suite_number` CHAR(10),
+        |`s_city` VARCHAR(60),
+        |`s_county` VARCHAR(30),
+        |`s_state` CHAR(2),
+        |`s_zip` CHAR(10),
+        |`s_country` VARCHAR(20),
+        |`s_gmt_offset` DECIMAL(5,2),
+        |`s_tax_percentage` DECIMAL(5,2)
       """.stripMargin,
     "call_center" ->
       """
-        |`cc_call_center_sk` INT, `cc_call_center_id` STRING, `cc_rec_start_date` DATE,
-        |`cc_rec_end_date` DATE, `cc_closed_date_sk` INT, `cc_open_date_sk` INT, `cc_name` STRING,
-        |`cc_class` STRING, `cc_employees` INT, `cc_sq_ft` INT, `cc_hours` STRING,
-        |`cc_manager` STRING, `cc_mkt_id` INT, `cc_mkt_class` STRING, `cc_mkt_desc` STRING,
-        |`cc_market_manager` STRING, `cc_division` INT, `cc_division_name` STRING,
-        |`cc_company` INT, `cc_company_name` STRING, `cc_street_number` STRING,
-        |`cc_street_name` STRING, `cc_street_type` STRING, `cc_suite_number` STRING,
-        |`cc_city` STRING, `cc_county` STRING, `cc_state` STRING, `cc_zip` STRING,
-        |`cc_country` STRING, `cc_gmt_offset` DECIMAL(5,2), `cc_tax_percentage` DECIMAL(5,2)
+        |`cc_call_center_sk` INT,
+        |`cc_call_center_id` CHAR(16),
+        |`cc_rec_start_date` DATE,
+        |`cc_rec_end_date` DATE,
+        |`cc_closed_date_sk` INT,
+        |`cc_open_date_sk` INT,
+        |`cc_name` VARCHAR(50),
+        |`cc_class` VARCHAR(50),
+        |`cc_employees` INT,
+        |`cc_sq_ft` INT,
+        |`cc_hours` CHAR(20),
+        |`cc_manager` VARCHAR(40),
+        |`cc_mkt_id` INT,
+        |`cc_mkt_class` CHAR(50),
+        |`cc_mkt_desc` VARCHAR(100),
+        |`cc_market_manager` VARCHAR(40),
+        |`cc_division` INT,
+        |`cc_division_name` VARCHAR(50),
+        |`cc_company` INT,
+        |`cc_company_name` CHAR(50),
+        |`cc_street_number` CHAR(10),
+        |`cc_street_name` VARCHAR(60),
+        |`cc_street_type` CHAR(15),
+        |`cc_suite_number` CHAR(10),
+        |`cc_city` VARCHAR(60),
+        |`cc_county` VARCHAR(30),
+        |`cc_state` CHAR(2),
+        |`cc_zip` CHAR(10),
+        |`cc_country` VARCHAR(20),
+        |`cc_gmt_offset` DECIMAL(5,2),
+        |`cc_tax_percentage` DECIMAL(5,2)
       """.stripMargin,
     "catalog_page" ->
       """
-        |`cp_catalog_page_sk` INT, `cp_catalog_page_id` STRING, `cp_start_date_sk` INT,
-        |`cp_end_date_sk` INT, `cp_department` STRING, `cp_catalog_number` INT,
-        |`cp_catalog_page_number` INT, `cp_description` STRING, `cp_type` STRING
+        |`cp_catalog_page_sk` INT,
+        |`cp_catalog_page_id` CHAR(16),
+        |`cp_start_date_sk` INT,
+        |`cp_end_date_sk` INT,
+        |`cp_department` VARCHAR(50),
+        |`cp_catalog_number` INT,
+        |`cp_catalog_page_number` INT,
+        |`cp_description` VARCHAR(100),
+        |`cp_type` VARCHAR(100)
       """.stripMargin,
     "web_site" ->
       """
-        |`web_site_sk` INT, `web_site_id` STRING, `web_rec_start_date` DATE,
-        |`web_rec_end_date` DATE, `web_name` STRING, `web_open_date_sk` INT,
-        |`web_close_date_sk` INT, `web_class` STRING, `web_manager` STRING, `web_mkt_id` INT,
-        |`web_mkt_class` STRING, `web_mkt_desc` STRING, `web_market_manager` STRING,
-        |`web_company_id` INT, `web_company_name` STRING, `web_street_number` STRING,
-        |`web_street_name` STRING, `web_street_type` STRING, `web_suite_number` STRING,
-        |`web_city` STRING, `web_county` STRING, `web_state` STRING, `web_zip` STRING,
-        |`web_country` STRING, `web_gmt_offset` DECIMAL(5,2), `web_tax_percentage` DECIMAL(5,2)
+        |`web_site_sk` INT,
+        |`web_site_id` CHAR(16),
+        |`web_rec_start_date` DATE,
+        |`web_rec_end_date` DATE,
+        |`web_name` VARCHAR(50),
+        |`web_open_date_sk` INT,
+        |`web_close_date_sk` INT,
+        |`web_class` VARCHAR(50),
+        |`web_manager` VARCHAR(40),
+        |`web_mkt_id` INT,
+        |`web_mkt_class` VARCHAR(50),
+        |`web_mkt_desc` VARCHAR(100),
+        |`web_market_manager` VARCHAR(40),
+        |`web_company_id` INT,
+        |`web_company_name` CHAR(50),
+        |`web_street_number` CHAR(10),
+        |`web_street_name` VARCHAR(60),
+        |`web_street_type` CHAR(15),
+        |`web_suite_number` CHAR(10),
+        |`web_city` VARCHAR(60),
+        |`web_county` VARCHAR(30),
+        |`web_state` CHAR(2),
+        |`web_zip` CHAR(10),
+        |`web_country` VARCHAR(20),
+        |`web_gmt_offset` DECIMAL(5,2),
+        |`web_tax_percentage` DECIMAL(5,2)
       """.stripMargin,
     "web_page" ->
       """
-        |`wp_web_page_sk` INT, `wp_web_page_id` STRING,
-        |`wp_rec_start_date` DATE, `wp_rec_end_date` DATE, `wp_creation_date_sk` INT,
-        |`wp_access_date_sk` INT, `wp_autogen_flag` STRING, `wp_customer_sk` INT,
-        |`wp_url` STRING, `wp_type` STRING, `wp_char_count` INT, `wp_link_count` INT,
-        |`wp_image_count` INT, `wp_max_ad_count` INT
+        |`wp_web_page_sk` INT,
+        |`wp_web_page_id` CHAR(16),
+        |`wp_rec_start_date` DATE,
+        |`wp_rec_end_date` DATE,
+        |`wp_creation_date_sk` INT,
+        |`wp_access_date_sk` INT,
+        |`wp_autogen_flag` CHAR(1),
+        |`wp_customer_sk` INT,
+        |`wp_url` VARCHAR(100),
+        |`wp_type` CHAR(50),
+        |`wp_char_count` INT,
+        |`wp_link_count` INT,
+        |`wp_image_count` INT,
+        |`wp_max_ad_count` INT
       """.stripMargin,
     "warehouse" ->
       """
-        |`w_warehouse_sk` INT, `w_warehouse_id` STRING, `w_warehouse_name` STRING,
-        |`w_warehouse_sq_ft` INT, `w_street_number` STRING, `w_street_name` STRING,
-        |`w_street_type` STRING, `w_suite_number` STRING, `w_city` STRING, `w_county` STRING,
-        |`w_state` STRING, `w_zip` STRING, `w_country` STRING, `w_gmt_offset` DECIMAL(5,2)
+        |`w_warehouse_sk` INT,
+        |`w_warehouse_id` CHAR(16),
+        |`w_warehouse_name` VARCHAR(20),
+        |`w_warehouse_sq_ft` INT,
+        |`w_street_number` CHAR(10),
+        |`w_street_name` VARCHAR(20),
+        |`w_street_type` CHAR(15),
+        |`w_suite_number` CHAR(10),
+        |`w_city` VARCHAR(60),
+        |`w_county` VARCHAR(30),
+        |`w_state` CHAR(2),
+        |`w_zip` CHAR(10),
+        |`w_country` VARCHAR(20),
+        |`w_gmt_offset` DECIMAL(5,2)
       """.stripMargin,
     "customer" ->
       """
-        |`c_customer_sk` INT, `c_customer_id` STRING, `c_current_cdemo_sk` INT,
-        |`c_current_hdemo_sk` INT, `c_current_addr_sk` INT, `c_first_shipto_date_sk` INT,
-        |`c_first_sales_date_sk` INT, `c_salutation` STRING, `c_first_name` STRING,
-        |`c_last_name` STRING, `c_preferred_cust_flag` STRING, `c_birth_day` INT,
-        |`c_birth_month` INT, `c_birth_year` INT, `c_birth_country` STRING, `c_login` STRING,
-        |`c_email_address` STRING, `c_last_review_date` INT
+        |`c_customer_sk` INT,
+        |`c_customer_id` CHAR(16),
+        |`c_current_cdemo_sk` INT,
+        |`c_current_hdemo_sk` INT,
+        |`c_current_addr_sk` INT,
+        |`c_first_shipto_date_sk` INT,
+        |`c_first_sales_date_sk` INT,
+        |`c_salutation` CHAR(10),
+        |`c_first_name` CHAR(20),
+        |`c_last_name` CHAR(30),
+        |`c_preferred_cust_flag` CHAR(1),
+        |`c_birth_day` INT,
+        |`c_birth_month` INT,
+        |`c_birth_year` INT,
+        |`c_birth_country` VARCHAR(20),
+        |`c_login` CHAR(13),
+        |`c_email_address` CHAR(50),
+        |`c_last_review_date` INT
       """.stripMargin,
     "customer_address" ->
       """
-        |`ca_address_sk` INT, `ca_address_id` STRING, `ca_street_number` STRING,
-        |`ca_street_name` STRING, `ca_street_type` STRING, `ca_suite_number` STRING,
-        |`ca_city` STRING, `ca_county` STRING, `ca_state` STRING, `ca_zip` STRING,
-        |`ca_country` STRING, `ca_gmt_offset` DECIMAL(5,2), `ca_location_type` STRING
+        |`ca_address_sk` INT,
+        |`ca_address_id` CHAR(16),
+        |`ca_street_number` CHAR(10),
+        |`ca_street_name` VARCHAR(60),
+        |`ca_street_type` CHAR(15),
+        |`ca_suite_number` CHAR(10),
+        |`ca_city` VARCHAR(60),
+        |`ca_county` VARCHAR(30),
+        |`ca_state` CHAR(2),
+        |`ca_zip` CHAR(10),
+        |`ca_country` VARCHAR(20),
+        |`ca_gmt_offset` DECIMAL(5,2),
+        |`ca_location_type` CHAR(20)
       """.stripMargin,
     "customer_demographics" ->
       """
-        |`cd_demo_sk` INT, `cd_gender` STRING, `cd_marital_status` STRING,
-        |`cd_education_status` STRING, `cd_purchase_estimate` INT, `cd_credit_rating` STRING,
-        |`cd_dep_count` INT, `cd_dep_employed_count` INT, `cd_dep_college_count` INT
+        |`cd_demo_sk` INT,
+        |`cd_gender` CHAR(1),
+        |`cd_marital_status` CHAR(1),
+        |`cd_education_status` CHAR(20),
+        |`cd_purchase_estimate` INT,
+        |`cd_credit_rating` CHAR(10),
+        |`cd_dep_count` INT,
+        |`cd_dep_employed_count` INT,
+        |`cd_dep_college_count` INT
       """.stripMargin,
     "date_dim" ->
       """
-        |`d_date_sk` INT, `d_date_id` STRING, `d_date` DATE,
-        |`d_month_seq` INT, `d_week_seq` INT, `d_quarter_seq` INT, `d_year` INT, `d_dow` INT,
-        |`d_moy` INT, `d_dom` INT, `d_qoy` INT, `d_fy_year` INT, `d_fy_quarter_seq` INT,
-        |`d_fy_week_seq` INT, `d_day_name` STRING, `d_quarter_name` STRING, `d_holiday` STRING,
-        |`d_weekend` STRING, `d_following_holiday` STRING, `d_first_dom` INT, `d_last_dom` INT,
-        |`d_same_day_ly` INT, `d_same_day_lq` INT, `d_current_day` STRING,
-        |`d_current_week` STRING, `d_current_month` STRING, `d_current_quarter` STRING,
-        |`d_current_year` STRING
+        |`d_date_sk` INT,
+        |`d_date_id` CHAR(16),
+        |`d_date` DATE,
+        |`d_month_seq` INT,
+        |`d_week_seq` INT,
+        |`d_quarter_seq` INT,
+        |`d_year` INT,
+        |`d_dow` INT,
+        |`d_moy` INT,
+        |`d_dom` INT,
+        |`d_qoy` INT,
+        |`d_fy_year` INT,
+        |`d_fy_quarter_seq` INT,
+        |`d_fy_week_seq` INT,
+        |`d_day_name` CHAR(9),
+        |`d_quarter_name` CHAR(6),
+        |`d_holiday` CHAR(1),
+        |`d_weekend` CHAR(1),
+        |`d_following_holiday` CHAR(1),
+        |`d_first_dom` INT,
+        |`d_last_dom` INT,
+        |`d_same_day_ly` INT,
+        |`d_same_day_lq` INT,
+        |`d_current_day` CHAR(1),
+        |`d_current_week` CHAR(1),
+        |`d_current_month` CHAR(1),
+        |`d_current_quarter` CHAR(1),
+        |`d_current_year` CHAR(1)
       """.stripMargin,
     "household_demographics" ->
       """
-        |`hd_demo_sk` INT, `hd_income_band_sk` INT, `hd_buy_potential` STRING, `hd_dep_count` INT,
+        |`hd_demo_sk` INT,
+        |`hd_income_band_sk` INT,
+        |`hd_buy_potential` CHAR(15),
+        |`hd_dep_count` INT,
         |`hd_vehicle_count` INT
       """.stripMargin,
     "item" ->
       """
-        |`i_item_sk` INT, `i_item_id` STRING, `i_rec_start_date` DATE,
-        |`i_rec_end_date` DATE, `i_item_desc` STRING, `i_current_price` DECIMAL(7,2),
-        |`i_wholesale_cost` DECIMAL(7,2), `i_brand_id` INT, `i_brand` STRING, `i_class_id` INT,
-        |`i_class` STRING, `i_category_id` INT, `i_category` STRING, `i_manufact_id` INT,
-        |`i_manufact` STRING, `i_size` STRING, `i_formulation` STRING, `i_color` STRING,
-        |`i_units` STRING, `i_container` STRING, `i_manager_id` INT, `i_product_name` STRING
+        |`i_item_sk` INT,
+        |`i_item_id` CHAR(16),
+        |`i_rec_start_date` DATE,
+        |`i_rec_end_date` DATE,
+        |`i_item_desc` VARCHAR(200),
+        |`i_current_price` DECIMAL(7,2),
+        |`i_wholesale_cost` DECIMAL(7,2),
+        |`i_brand_id` INT,
+        |`i_brand` CHAR(50),
+        |`i_class_id` INT,
+        |`i_class` CHAR(50),
+        |`i_category_id` INT,
+        |`i_category` CHAR(50),
+        |`i_manufact_id` INT,
+        |`i_manufact` CHAR(50),
+        |`i_size` CHAR(20),
+        |`i_formulation` CHAR(20),
+        |`i_color` CHAR(20),
+        |`i_units` CHAR(10),
+        |`i_container` CHAR(10),
+        |`i_manager_id` INT,
+        |`i_product_name` CHAR(50)
       """.stripMargin,
     "income_band" ->
       """
-        |`ib_income_band_sk` INT, `ib_lower_bound` INT, `ib_upper_bound` INT
+        |`ib_income_band_sk` INT,
+        |`ib_lower_bound` INT,
+        |`ib_upper_bound` INT
       """.stripMargin,
     "promotion" ->
       """
-        |`p_promo_sk` INT, `p_promo_id` STRING, `p_start_date_sk` INT, `p_end_date_sk` INT,
-        |`p_item_sk` INT, `p_cost` DECIMAL(15,2), `p_response_target` INT, `p_promo_name` STRING,
-        |`p_channel_dmail` STRING, `p_channel_email` STRING, `p_channel_catalog` STRING,
-        |`p_channel_tv` STRING, `p_channel_radio` STRING, `p_channel_press` STRING,
-        |`p_channel_event` STRING, `p_channel_demo` STRING, `p_channel_details` STRING,
-        |`p_purpose` STRING, `p_discount_active` STRING
+        |`p_promo_sk` INT,
+        |`p_promo_id` CHAR(16),
+        |`p_start_date_sk` INT,
+        |`p_end_date_sk` INT,
+        |`p_item_sk` INT,
+        |`p_cost` DECIMAL(15,2),
+        |`p_response_target` INT,
+        |`p_promo_name` CHAR(50),
+        |`p_channel_dmail` CHAR(1),
+        |`p_channel_email` CHAR(1),
+        |`p_channel_catalog` CHAR(1),
+        |`p_channel_tv` CHAR(1),
+        |`p_channel_radio` CHAR(1),
+        |`p_channel_press` CHAR(1),
+        |`p_channel_event` CHAR(1),
+        |`p_channel_demo` CHAR(1),
+        |`p_channel_details` VARCHAR(100),
+        |`p_purpose` CHAR(15),
+        |`p_discount_active` CHAR(1)
       """.stripMargin,
     "reason" ->
       """
-        |`r_reason_sk` INT, `r_reason_id` STRING, `r_reason_desc` STRING
+        |`r_reason_sk` INT,
+        |`r_reason_id` CHAR(16),
+        |`r_reason_desc` CHAR(100)
       """.stripMargin,
     "ship_mode" ->
       """
-        |`sm_ship_mode_sk` INT, `sm_ship_mode_id` STRING, `sm_type` STRING, `sm_code` STRING,
-        |`sm_carrier` STRING, `sm_contract` STRING
+        |`sm_ship_mode_sk` INT,
+        |`sm_ship_mode_id` CHAR(16),
+        |`sm_type` CHAR(30),
+        |`sm_code` CHAR(10),
+        |`sm_carrier` CHAR(20),
+        |`sm_contract` CHAR(20)
       """.stripMargin,
     "time_dim" ->
       """
-        |`t_time_sk` INT, `t_time_id` STRING, `t_time` INT, `t_hour` INT, `t_minute` INT,
-        |`t_second` INT, `t_am_pm` STRING, `t_shift` STRING, `t_sub_shift` STRING,
-        |`t_meal_time` STRING
+        |`t_time_sk` INT,
+        |`t_time_id` CHAR(16),
+        |`t_time` INT,
+        |`t_hour` INT,
+        |`t_minute` INT,
+        |`t_second` INT,
+        |`t_am_pm` CHAR(2),
+        |`t_shift` CHAR(20),
+        |`t_sub_shift` CHAR(20),
+        |`t_meal_time` CHAR(20)
       """.stripMargin
   )
 
-  val tableNames: Iterable[String] = tableColumns.keys
-
-  def createTable(
-      spark: SparkSession,
-      tableName: String,
-      format: String = "parquet",
-      options: Seq[String] = Nil): Unit = {
-    spark.sql(
-      s"""
-         |CREATE TABLE `$tableName` (${tableColumns(tableName)})
-         |USING $format
-         |${options.mkString("\n")}
-       """.stripMargin)
-  }
+  // The partition column is consistent with the databricks/spark-sql-perf project.
+  protected val tablePartitionColumns = Map(
+    "catalog_sales" -> Seq("`cs_sold_date_sk`"),
+    "catalog_returns" -> Seq("`cr_returned_date_sk`"),
+    "inventory" -> Seq("`inv_date_sk`"),
+    "store_sales" -> Seq("`ss_sold_date_sk`"),
+    "store_returns" -> Seq("`sr_returned_date_sk`"),
+    "web_sales" -> Seq("`ws_sold_date_sk`"),
+    "web_returns" -> Seq("`wr_returned_date_sk`")
+  )
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSTableStats.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSTableStats.scala
new file mode 100644
index 0000000000000..ee9cf7b67225f
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSTableStats.scala
@@ -0,0 +1,503 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import org.apache.spark.sql.catalyst.catalog.{CatalogColumnStat, CatalogStatistics}
+
+object TPCDSTableStats {
+
+  // These data statistics are extracted from generated TPCDS data with SF=100
+
+  // scalastyle:off line.size.limit
+  val sf100TableStats = Map(
+    "customer" -> CatalogStatistics(500000000L, Some(2000000L), Map(
+      "c_birth_country" -> CatalogColumnStat(Some(196L), None, None, Some(69626), Some(9), Some(20), None, CatalogColumnStat.VERSION),
+      "c_current_cdemo_sk" -> CatalogColumnStat(Some(1184426L), Some("1"), Some("1920798"), Some(69943), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_customer_id" -> CatalogColumnStat(Some(2000000L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "c_birth_day" -> CatalogColumnStat(Some(32L), Some("1"), Some("31"), Some(70166), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_last_name" -> CatalogColumnStat(Some(5250L), None, None, Some(70098), Some(7), Some(13), None, CatalogColumnStat.VERSION),
+      "c_login" -> CatalogColumnStat(Some(0L), None, None, Some(2000000), Some(20), Some(20), None, CatalogColumnStat.VERSION),
+      "c_first_shipto_date_sk" -> CatalogColumnStat(Some(3755L), Some("2449028"), Some("2452678"), Some(70080), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_first_name" -> CatalogColumnStat(Some(5146L), None, None, Some(69769), Some(6), Some(11), None, CatalogColumnStat.VERSION),
+      "c_birth_month" -> CatalogColumnStat(Some(12L), Some("1"), Some("12"), Some(69896), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_email_address" -> CatalogColumnStat(Some(1929800L), None, None, Some(70200), Some(28), Some(47), None, CatalogColumnStat.VERSION),
+      "c_current_addr_sk" -> CatalogColumnStat(Some(824389L), Some("1"), Some("1000000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_salutation" -> CatalogColumnStat(Some(6L), None, None, Some(69840), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_current_hdemo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(69657), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_birth_year" -> CatalogColumnStat(Some(67L), Some("1924"), Some("1992"), Some(69986), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_customer_sk" -> CatalogColumnStat(Some(1903054L), Some("1"), Some("2000000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_last_review_date" -> CatalogColumnStat(Some(357L), None, None, Some(70102), Some(7), Some(7), None, CatalogColumnStat.VERSION),
+      "c_preferred_cust_flag" -> CatalogColumnStat(Some(2L), None, None, Some(69778), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "c_first_sales_date_sk" -> CatalogColumnStat(Some(3758L), Some("2448998"), Some("2452648"), Some(69950), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "store_sales" -> CatalogStatistics(42623559552L, Some(287997024L), Map(
+      "ss_ext_sales_price" -> CatalogColumnStat(Some(712602L), Some("0.00"), Some("19878.00"), Some(12955462), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_item_sk" -> CatalogColumnStat(Some(206807L), Some("1"), Some("204000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_coupon_amt" -> CatalogColumnStat(Some(925847L), Some("0.00"), Some("19225.00"), Some(12958053), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_store_sk" -> CatalogColumnStat(Some(199L), Some("1"), Some("400"), Some(12950651), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_quantity" -> CatalogColumnStat(Some(99L), Some("1"), Some("100"), Some(12953654), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_ext_discount_amt" -> CatalogColumnStat(Some(925847L), Some("0.00"), Some("19225.00"), Some(12958053), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_hdemo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(12957139), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_ext_tax" -> CatalogColumnStat(Some(135620L), Some("0.00"), Some("1762.38"), Some(12957705), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_sold_time_sk" -> CatalogColumnStat(Some(47827L), Some("28800"), Some("75599"), Some(12953300), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_promo_sk" -> CatalogColumnStat(Some(1026L), Some("1"), Some("1000"), Some(12954088), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_net_paid_inc_tax" -> CatalogColumnStat(Some(1404501L), Some("0.00"), Some("21344.38"), Some(12958941), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_wholesale_cost" -> CatalogColumnStat(Some(9503L), Some("1.00"), Some("100.00"), Some(12958327), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_cdemo_sk" -> CatalogColumnStat(Some(1847065L), Some("1"), Some("1920800"), Some(12955252), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_sold_date_sk" -> CatalogColumnStat(Some(1781L), Some("2450816"), Some("2452642"), Some(12955025), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_list_price" -> CatalogColumnStat(Some(19079L), Some("1.00"), Some("200.00"), Some(12952108), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_ext_list_price" -> CatalogColumnStat(Some(752016L), Some("1.00"), Some("20000.00"), Some(12955040), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_net_profit" -> CatalogColumnStat(Some(1388760L), Some("-10000.00"), Some("9889.00"), Some(12955156), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_ticket_number" -> CatalogColumnStat(Some(24596280L), Some("1"), Some("24000000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_net_paid" -> CatalogColumnStat(Some(1058635L), Some("0.00"), Some("19878.00"), Some(12954554), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_ext_wholesale_cost" -> CatalogColumnStat(Some(382964L), Some("1.00"), Some("10000.00"), Some(12960060), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_customer_sk" -> CatalogColumnStat(Some(1903054L), Some("1"), Some("2000000"), Some(12952082), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_addr_sk" -> CatalogColumnStat(Some(943039L), Some("1"), Some("1000000"), Some(12956686), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_sales_price" -> CatalogColumnStat(Some(19261L), Some("0.00"), Some("200.00"), Some(12958587), Some(8), Some(8), None, CatalogColumnStat.VERSION)
+    )),
+    "web_sales" -> CatalogStatistics(14688252348L, Some(72001237L), Map(
+      "ws_sold_time_sk" -> CatalogColumnStat(Some(83505L), Some("0"), Some("86399"), Some(17931), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_ext_wholesale_cost" -> CatalogColumnStat(Some(382964L), Some("1.00"), Some("10000.00"), Some(17814), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_bill_cdemo_sk" -> CatalogColumnStat(Some(1707373L), Some("1"), Some("1920800"), Some(17833), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_net_paid_inc_ship" -> CatalogColumnStat(Some(1629659L), Some("0.00"), Some("43468.92"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_sales_price" -> CatalogColumnStat(Some(29143L), Some("0.00"), Some("300.00"), Some(18005), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_ship_mode_sk" -> CatalogColumnStat(Some(20L), Some("1"), Some("20"), Some(17823), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_ship_hdemo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(17833), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_order_number" -> CatalogColumnStat(Some(6073146L), Some("1"), Some("6000000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_sold_date_sk" -> CatalogColumnStat(Some(1781L), Some("2450816"), Some("2452642"), Some(17922), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_web_site_sk" -> CatalogColumnStat(Some(25L), Some("1"), Some("24"), Some(18030), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_bill_customer_sk" -> CatalogColumnStat(Some(1800267L), Some("1"), Some("2000000"), Some(17882), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_coupon_amt" -> CatalogColumnStat(Some(863374L), Some("0.00"), Some("27591.16"), Some(18027), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_ship_addr_sk" -> CatalogColumnStat(Some(943039L), Some("1"), Some("1000000"), Some(17931), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_net_paid_inc_tax" -> CatalogColumnStat(Some(1695042L), Some("0.00"), Some("32492.90"), Some(18102), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_quantity" -> CatalogColumnStat(Some(99L), Some("1"), Some("100"), Some(18014), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_bill_addr_sk" -> CatalogColumnStat(Some(940457L), Some("1"), Some("1000000"), Some(17801), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_net_paid" -> CatalogColumnStat(Some(1247989L), Some("0.00"), Some("29810.00"), Some(17968), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_ext_discount_amt" -> CatalogColumnStat(Some(934889L), Some("0.00"), Some("29982.00"), Some(17890), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_item_sk" -> CatalogColumnStat(Some(206807L), Some("1"), Some("204000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_ext_tax" -> CatalogColumnStat(Some(168876L), Some("0.00"), Some("2682.90"), Some(17800), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_wholesale_cost" -> CatalogColumnStat(Some(9503L), Some("1.00"), Some("100.00"), Some(17850), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_ship_customer_sk" -> CatalogColumnStat(Some(1811873L), Some("1"), Some("2000000"), Some(17886), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_net_paid_inc_ship_tax" -> CatalogColumnStat(Some(2186741L), Some("0.00"), Some("44479.52"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_web_page_sk" -> CatalogColumnStat(Some(2108L), Some("1"), Some("2040"), Some(17920), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_ext_sales_price" -> CatalogColumnStat(Some(908147L), Some("0.00"), Some("29810.00"), Some(17843), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_ext_ship_cost" -> CatalogColumnStat(Some(482711L), Some("0.00"), Some("14927.00"), Some(17923), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_bill_hdemo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(18011), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_ship_date_sk" -> CatalogColumnStat(Some(1898L), Some("2450817"), Some("2452762"), Some(17883), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_ext_list_price" -> CatalogColumnStat(Some(1071558L), Some("1.02"), Some("29997.00"), Some(18001), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_ship_cdemo_sk" -> CatalogColumnStat(Some(1755797L), Some("1"), Some("1920800"), Some(17903), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_warehouse_sk" -> CatalogColumnStat(Some(15L), Some("1"), Some("15"), Some(17812), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_list_price" -> CatalogColumnStat(Some(29447L), Some("1.00"), Some("300.00"), Some(17824), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_promo_sk" -> CatalogColumnStat(Some(1026L), Some("1"), Some("1000"), Some(18116), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_net_profit" -> CatalogColumnStat(Some(1482074L), Some("-9997.00"), Some("19840.00"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION)
+    )),
+    "date_dim" -> CatalogStatistics(19284936L, Some(73049L), Map(
+      "d_dom" -> CatalogColumnStat(Some(32L), Some("1"), Some("31"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_current_quarter" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "d_fy_week_seq" -> CatalogColumnStat(Some(10010L), Some("1"), Some("10436"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_holiday" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "d_current_day" -> CatalogColumnStat(Some(1L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "d_week_seq" -> CatalogColumnStat(Some(10010L), Some("1"), Some("10436"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_current_month" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "d_moy" -> CatalogColumnStat(Some(12L), Some("1"), Some("12"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_date" -> CatalogColumnStat(Some(73049L), None, None, Some(0), Some(10), Some(10), None, CatalogColumnStat.VERSION),
+      "d_same_day_lq" -> CatalogColumnStat(Some(73049L), Some("2414930"), Some("2487978"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_weekend" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "d_dow" -> CatalogColumnStat(Some(7L), Some("0"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_year" -> CatalogColumnStat(Some(202L), Some("1900"), Some("2100"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_current_week" -> CatalogColumnStat(Some(1L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "d_quarter_name" -> CatalogColumnStat(Some(774L), None, None, Some(0), Some(6), Some(6), None, CatalogColumnStat.VERSION),
+      "d_month_seq" -> CatalogColumnStat(Some(2431L), Some("0"), Some("2400"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_fy_year" -> CatalogColumnStat(Some(202L), Some("1900"), Some("2100"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_following_holiday" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "d_same_day_ly" -> CatalogColumnStat(Some(73049L), Some("2414657"), Some("2487705"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_day_name" -> CatalogColumnStat(Some(7L), None, None, Some(0), Some(8), Some(9), None, CatalogColumnStat.VERSION),
+      "d_qoy" -> CatalogColumnStat(Some(4L), Some("1"), Some("4"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_date_sk" -> CatalogColumnStat(Some(73049L), Some("2415022"), Some("2488070"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_fy_quarter_seq" -> CatalogColumnStat(Some(787L), Some("1"), Some("801"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_current_year" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "d_date_id" -> CatalogColumnStat(Some(73049L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "d_quarter_seq" -> CatalogColumnStat(Some(787L), Some("1"), Some("801"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_last_dom" -> CatalogColumnStat(Some(2386L), Some("2415020"), Some("2488372"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_first_dom" -> CatalogColumnStat(Some(2329L), Some("2415021"), Some("2488070"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "household_demographics" -> CatalogStatistics(316800L, Some(7200L), Map(
+      "hd_buy_potential" -> CatalogColumnStat(Some(6L), None, None, Some(0), Some(8), Some(10), None, CatalogColumnStat.VERSION),
+      "hd_income_band_sk" -> CatalogColumnStat(Some(20L), Some("1"), Some("20"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "hd_demo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "hd_vehicle_count" -> CatalogColumnStat(Some(6L), Some("-1"), Some("4"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "hd_dep_count" -> CatalogColumnStat(Some(10L), Some("0"), Some("9"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "call_center" -> CatalogStatistics(16020L, Some(30L), Map(
+      "cc_rec_start_date" -> CatalogColumnStat(Some(4L), Some("1998-01-01"), Some("2002-01-01"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_mkt_desc" -> CatalogColumnStat(Some(21L), None, None, Some(0), Some(59), Some(93), None, CatalogColumnStat.VERSION),
+      "cc_hours" -> CatalogColumnStat(Some(3L), None, None, Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cc_street_name" -> CatalogColumnStat(Some(14L), None, None, Some(0), Some(8), Some(16), None, CatalogColumnStat.VERSION),
+      "cc_manager" -> CatalogColumnStat(Some(22L), None, None, Some(0), Some(13), Some(17), None, CatalogColumnStat.VERSION),
+      "cc_city" -> CatalogColumnStat(Some(12L), None, None, Some(0), Some(10), Some(15), None, CatalogColumnStat.VERSION),
+      "cc_class" -> CatalogColumnStat(Some(3L), None, None, Some(0), Some(6), Some(6), None, CatalogColumnStat.VERSION),
+      "cc_country" -> CatalogColumnStat(Some(1L), None, None, Some(0), Some(13), Some(13), None, CatalogColumnStat.VERSION),
+      "cc_open_date_sk" -> CatalogColumnStat(Some(15L), Some("2450794"), Some("2451146"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_market_manager" -> CatalogColumnStat(Some(25L), None, None, Some(0), Some(13), Some(17), None, CatalogColumnStat.VERSION),
+      "cc_street_number" -> CatalogColumnStat(Some(15L), None, None, Some(0), Some(3), Some(3), None, CatalogColumnStat.VERSION),
+      "cc_call_center_sk" -> CatalogColumnStat(Some(30L), Some("1"), Some("30"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_name" -> CatalogColumnStat(Some(15L), None, None, Some(0), Some(14), Some(19), None, CatalogColumnStat.VERSION),
+      "cc_suite_number" -> CatalogColumnStat(Some(14L), None, None, Some(0), Some(8), Some(9), None, CatalogColumnStat.VERSION),
+      "cc_mkt_id" -> CatalogColumnStat(Some(6L), Some("1"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_gmt_offset" -> CatalogColumnStat(Some(2L), Some("-6.00"), Some("-5.00"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cc_company_name" -> CatalogColumnStat(Some(6L), None, None, Some(0), Some(4), Some(5), None, CatalogColumnStat.VERSION),
+      "cc_division" -> CatalogColumnStat(Some(6L), Some("1"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_closed_date_sk" -> CatalogColumnStat(Some(0L), None, None, Some(30), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_state" -> CatalogColumnStat(Some(8L), None, None, Some(0), Some(2), Some(2), None, CatalogColumnStat.VERSION),
+      "cc_street_type" -> CatalogColumnStat(Some(9L), None, None, Some(0), Some(5), Some(9), None, CatalogColumnStat.VERSION),
+      "cc_call_center_id" -> CatalogColumnStat(Some(14L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "cc_sq_ft" -> CatalogColumnStat(Some(22L), Some("1670015"), Some("31896816"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_mkt_class" -> CatalogColumnStat(Some(26L), None, None, Some(0), Some(35), Some(50), None, CatalogColumnStat.VERSION),
+      "cc_tax_percentage" -> CatalogColumnStat(Some(10L), Some("0.00"), Some("0.12"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cc_division_name" -> CatalogColumnStat(Some(6L), None, None, Some(0), Some(5), Some(5), None, CatalogColumnStat.VERSION),
+      "cc_zip" -> CatalogColumnStat(Some(14L), None, None, Some(0), Some(5), Some(5), None, CatalogColumnStat.VERSION),
+      "cc_rec_end_date" -> CatalogColumnStat(Some(3L), Some("2000-01-01"), Some("2001-12-31"), Some(15), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_employees" -> CatalogColumnStat(Some(22L), Some("2935"), Some("69020"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_county" -> CatalogColumnStat(Some(8L), None, None, Some(0), Some(15), Some(17), None, CatalogColumnStat.VERSION),
+      "cc_company" -> CatalogColumnStat(Some(6L), Some("1"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "catalog_page" -> CatalogStatistics(3774000L, Some(20400L), Map(
+      "cp_catalog_page_id" -> CatalogColumnStat(Some(20400L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "cp_department" -> CatalogColumnStat(Some(1L), None, None, Some(205), Some(10), Some(10), None, CatalogColumnStat.VERSION),
+      "cp_description" -> CatalogColumnStat(Some(19304L), None, None, Some(190), Some(75), Some(99), None, CatalogColumnStat.VERSION),
+      "cp_start_date_sk" -> CatalogColumnStat(Some(88L), Some("2450815"), Some("2453005"), Some(196), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cp_end_date_sk" -> CatalogColumnStat(Some(101L), Some("2450844"), Some("2453186"), Some(206), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cp_catalog_page_number" -> CatalogColumnStat(Some(186L), Some("1"), Some("188"), Some(208), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cp_type" -> CatalogColumnStat(Some(3L), None, None, Some(197), Some(8), Some(9), None, CatalogColumnStat.VERSION),
+      "cp_catalog_page_sk" -> CatalogColumnStat(Some(18915L), Some("1"), Some("20400"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cp_catalog_number" -> CatalogColumnStat(Some(110L), Some("1"), Some("109"), Some(197), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "catalog_returns" -> CatalogStatistics(2189464848L, Some(14404374L), Map(
+      "cr_return_amount" -> CatalogColumnStat(Some(563788L), Some("0.00"), Some("28778.31"), Some(288408), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_returning_hdemo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(288369), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_refunded_hdemo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(288053), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_catalog_page_sk" -> CatalogColumnStat(Some(11224L), Some("1"), Some("17108"), Some(288041), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_reason_sk" -> CatalogColumnStat(Some(55L), Some("1"), Some("55"), Some(287890), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_returning_cdemo_sk" -> CatalogColumnStat(Some(1839372L), Some("1"), Some("1920800"), Some(288128), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_fee" -> CatalogColumnStat(Some(9571L), Some("0.50"), Some("100.00"), Some(288038), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_refunded_cash" -> CatalogColumnStat(Some(592855L), Some("0.00"), Some("24544.84"), Some(287638), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_refunded_cdemo_sk" -> CatalogColumnStat(Some(1842661L), Some("1"), Some("1920800"), Some(287556), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_order_number" -> CatalogColumnStat(Some(8816832L), Some("2"), Some("16000000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_ship_mode_sk" -> CatalogColumnStat(Some(20L), Some("1"), Some("20"), Some(287768), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_refunded_customer_sk" -> CatalogColumnStat(Some(1894309L), Some("1"), Some("2000000"), Some(287207), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_return_amt_inc_tax" -> CatalogColumnStat(Some(855338L), Some("0.00"), Some("29353.87"), Some(288246), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_returning_customer_sk" -> CatalogColumnStat(Some(1903054L), Some("1"), Some("2000000"), Some(287581), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_returned_date_sk" -> CatalogColumnStat(Some(2027L), Some("2450821"), Some("2452921"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_return_ship_cost" -> CatalogColumnStat(Some(347391L), Some("0.00"), Some("14130.96"), Some(287952), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_returning_addr_sk" -> CatalogColumnStat(Some(943039L), Some("1"), Some("1000000"), Some(288203), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_return_quantity" -> CatalogColumnStat(Some(99L), Some("1"), Some("100"), Some(287844), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_store_credit" -> CatalogColumnStat(Some(415989L), Some("0.00"), Some("22167.49"), Some(288118), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_item_sk" -> CatalogColumnStat(Some(206807L), Some("1"), Some("204000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_return_tax" -> CatalogColumnStat(Some(101557L), Some("0.00"), Some("2390.75"), Some(288599), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_refunded_addr_sk" -> CatalogColumnStat(Some(943039L), Some("1"), Some("1000000"), Some(287752), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_call_center_sk" -> CatalogColumnStat(Some(31L), Some("1"), Some("30"), Some(288179), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_returned_time_sk" -> CatalogColumnStat(Some(83505L), Some("0"), Some("86399"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_net_loss" -> CatalogColumnStat(Some(550908L), Some("0.50"), Some("15781.83"), Some(287954), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_reversed_charge" -> CatalogColumnStat(Some(410432L), Some("0.00"), Some("23801.24"), Some(288476), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_warehouse_sk" -> CatalogColumnStat(Some(15L), Some("1"), Some("15"), Some(288581), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "catalog_sales" -> CatalogStatistics(29375401260L, Some(143997065L), Map(
+      "cs_ship_hdemo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(720450), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_bill_addr_sk" -> CatalogColumnStat(Some(943039L), Some("1"), Some("1000000"), Some(718886), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_net_paid_inc_ship_tax" -> CatalogColumnStat(Some(2570239L), Some("0.00"), Some("45460.80"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_ext_sales_price" -> CatalogColumnStat(Some(977146L), Some("0.00"), Some("29808.00"), Some(719228), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_sold_date_sk" -> CatalogColumnStat(Some(1772L), Some("2450815"), Some("2452654"), Some(719581), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_ext_tax" -> CatalogColumnStat(Some(189612L), Some("0.00"), Some("2619.36"), Some(719627), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_sold_time_sk" -> CatalogColumnStat(Some(83505L), Some("0"), Some("86399"), Some(720917), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_net_paid_inc_ship" -> CatalogColumnStat(Some(1870941L), Some("0.00"), Some("43725.00"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_warehouse_sk" -> CatalogColumnStat(Some(15L), Some("1"), Some("15"), Some(719624), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_ext_list_price" -> CatalogColumnStat(Some(1101138L), Some("1.00"), Some("29997.00"), Some(719642), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_quantity" -> CatalogColumnStat(Some(99L), Some("1"), Some("100"), Some(720147), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_ext_discount_amt" -> CatalogColumnStat(Some(965247L), Some("0.00"), Some("29765.00"), Some(719820), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_net_profit" -> CatalogColumnStat(Some(1640663L), Some("-10000.00"), Some("19840.00"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_bill_hdemo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(719849), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_ext_ship_cost" -> CatalogColumnStat(Some(508914L), Some("0.00"), Some("14896.00"), Some(719848), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_item_sk" -> CatalogColumnStat(Some(206807L), Some("1"), Some("204000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_ship_cdemo_sk" -> CatalogColumnStat(Some(1847065L), Some("1"), Some("1920800"), Some(720292), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_wholesale_cost" -> CatalogColumnStat(Some(9503L), Some("1.00"), Some("100.00"), Some(721114), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_ship_date_sk" -> CatalogColumnStat(Some(1887L), Some("2450817"), Some("2452744"), Some(719625), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_ext_wholesale_cost" -> CatalogColumnStat(Some(382964L), Some("1.00"), Some("10000.00"), Some(719924), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_bill_cdemo_sk" -> CatalogColumnStat(Some(1847065L), Some("1"), Some("1920800"), Some(720208), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_sales_price" -> CatalogColumnStat(Some(29282L), Some("0.00"), Some("300.00"), Some(719781), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_net_paid" -> CatalogColumnStat(Some(1330351L), Some("0.00"), Some("29760.00"), Some(719706), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_promo_sk" -> CatalogColumnStat(Some(1026L), Some("1"), Some("1000"), Some(720194), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_call_center_sk" -> CatalogColumnStat(Some(31L), Some("1"), Some("30"), Some(719767), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_catalog_page_sk" -> CatalogColumnStat(Some(11224L), Some("1"), Some("17108"), Some(719180), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_bill_customer_sk" -> CatalogColumnStat(Some(1903054L), Some("1"), Some("2000000"), Some(719473), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_list_price" -> CatalogColumnStat(Some(29447L), Some("1.00"), Some("300.00"), Some(720328), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_ship_customer_sk" -> CatalogColumnStat(Some(1903054L), Some("1"), Some("2000000"), Some(720582), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_coupon_amt" -> CatalogColumnStat(Some(982009L), Some("0.00"), Some("28422.94"), Some(719631), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_ship_addr_sk" -> CatalogColumnStat(Some(943039L), Some("1"), Some("1000000"), Some(718680), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_order_number" -> CatalogColumnStat(Some(15603123L), Some("1"), Some("16000000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_net_paid_inc_tax" -> CatalogColumnStat(Some(1807594L), Some("0.00"), Some("31745.52"), Some(719354), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_ship_mode_sk" -> CatalogColumnStat(Some(20L), Some("1"), Some("20"), Some(720146), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "customer_address" -> CatalogStatistics(245000000L, Some(1000000L), Map(
+      "ca_country" -> CatalogColumnStat(Some(1L), None, None, Some(30097), Some(13), Some(13), None, CatalogColumnStat.VERSION),
+      "ca_address_id" -> CatalogColumnStat(Some(1000000L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "ca_address_sk" -> CatalogColumnStat(Some(943039L), Some("1"), Some("1000000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ca_county" -> CatalogColumnStat(Some(1957L), None, None, Some(30028), Some(14), Some(28), None, CatalogColumnStat.VERSION),
+      "ca_gmt_offset" -> CatalogColumnStat(Some(6L), Some("-10.00"), Some("-5.00"), Some(30131), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ca_street_type" -> CatalogColumnStat(Some(20L), None, None, Some(30124), Some(5), Some(9), None, CatalogColumnStat.VERSION),
+      "ca_street_name" -> CatalogColumnStat(Some(8292L), None, None, Some(30178), Some(9), Some(21), None, CatalogColumnStat.VERSION),
+      "ca_city" -> CatalogColumnStat(Some(977L), None, None, Some(30183), Some(9), Some(20), None, CatalogColumnStat.VERSION),
+      "ca_location_type" -> CatalogColumnStat(Some(3L), None, None, Some(30172), Some(9), Some(13), None, CatalogColumnStat.VERSION),
+      "ca_suite_number" -> CatalogColumnStat(Some(76L), None, None, Some(30047), Some(8), Some(9), None, CatalogColumnStat.VERSION),
+      "ca_zip" -> CatalogColumnStat(Some(8311L), None, None, Some(30370), Some(5), Some(5), None, CatalogColumnStat.VERSION),
+      "ca_state" -> CatalogColumnStat(Some(54L), None, None, Some(30124), Some(2), Some(2), None, CatalogColumnStat.VERSION),
+      "ca_street_number" -> CatalogColumnStat(Some(1034L), None, None, Some(30226), Some(3), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "customer_demographics" -> CatalogStatistics(182476000L, Some(1920800L), Map(
+      "cd_dep_employed_count" -> CatalogColumnStat(Some(7L), Some("0"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cd_dep_count" -> CatalogColumnStat(Some(7L), Some("0"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cd_education_status" -> CatalogColumnStat(Some(7L), None, None, Some(0), Some(10), Some(15), None, CatalogColumnStat.VERSION),
+      "cd_marital_status" -> CatalogColumnStat(Some(5L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "cd_gender" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "cd_purchase_estimate" -> CatalogColumnStat(Some(20L), Some("500"), Some("10000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cd_demo_sk" -> CatalogColumnStat(Some(1847065L), Some("1"), Some("1920800"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cd_credit_rating" -> CatalogColumnStat(Some(4L), None, None, Some(0), Some(7), Some(9), None, CatalogColumnStat.VERSION),
+      "cd_dep_college_count" -> CatalogColumnStat(Some(7L), Some("0"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "income_band" -> CatalogStatistics(400L, Some(20L), Map(
+      "ib_income_band_sk" -> CatalogColumnStat(Some(20L), Some("1"), Some("20"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ib_lower_bound" -> CatalogColumnStat(Some(19L), Some("0"), Some("190001"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ib_upper_bound" -> CatalogColumnStat(Some(20L), Some("10000"), Some("200000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "inventory" -> CatalogStatistics(9583920000L, Some(399330000L), Map(
+      "inv_date_sk" -> CatalogColumnStat(Some(267L), Some("2450815"), Some("2452635"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "inv_item_sk" -> CatalogColumnStat(Some(206807L), Some("1"), Some("204000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "inv_warehouse_sk" -> CatalogColumnStat(Some(15L), Some("1"), Some("15"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "inv_quantity_on_hand" -> CatalogColumnStat(Some(1026L), Some("0"), Some("1000"), Some(19969395), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "item" -> CatalogStatistics(94248000L, Some(204000L), Map(
+      "i_container" -> CatalogColumnStat(Some(1L), None, None, Some(510), Some(7), Some(7), None, CatalogColumnStat.VERSION),
+      "i_formulation" -> CatalogColumnStat(Some(158873L), None, None, Some(530), Some(20), Some(20), None, CatalogColumnStat.VERSION),
+      "i_brand_id" -> CatalogColumnStat(Some(862L), Some("1001001"), Some("10016017"), Some(516), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "i_manager_id" -> CatalogColumnStat(Some(99L), Some("1"), Some("100"), Some(506), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "i_rec_end_date" -> CatalogColumnStat(Some(3L), None, None, Some(102000), Some(10), Some(10), None, CatalogColumnStat.VERSION),
+      "i_class" -> CatalogColumnStat(Some(100L), None, None, Some(499), Some(8), Some(15), None, CatalogColumnStat.VERSION),
+      "i_wholesale_cost" -> CatalogColumnStat(Some(6297L), Some("0.02"), Some("88.91"), Some(489), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "i_item_id" -> CatalogColumnStat(Some(104042L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "i_manufact_id" -> CatalogColumnStat(Some(1026L), Some("1"), Some("1000"), Some(498), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "i_item_desc" -> CatalogColumnStat(Some(158754L), None, None, Some(508), Some(101), Some(200), None, CatalogColumnStat.VERSION),
+      "i_rec_start_date" -> CatalogColumnStat(Some(4L), None, None, Some(522), Some(10), Some(10), None, CatalogColumnStat.VERSION),
+      "i_color" -> CatalogColumnStat(Some(88L), None, None, Some(524), Some(6), Some(10), None, CatalogColumnStat.VERSION),
+      "i_product_name" -> CatalogColumnStat(Some(203486L), None, None, Some(514), Some(23), Some(30), None, CatalogColumnStat.VERSION),
+      "i_category" -> CatalogColumnStat(Some(10L), None, None, Some(482), Some(6), Some(11), None, CatalogColumnStat.VERSION),
+      "i_class_id" -> CatalogColumnStat(Some(16L), Some("1"), Some("16"), Some(491), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "i_item_sk" -> CatalogColumnStat(Some(204000L), Some("1"), Some("204000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "i_manufact" -> CatalogColumnStat(Some(997L), None, None, Some(544), Some(12), Some(15), None, CatalogColumnStat.VERSION),
+      "i_brand" -> CatalogColumnStat(Some(671L), None, None, Some(510), Some(17), Some(22), None, CatalogColumnStat.VERSION),
+      "i_current_price" -> CatalogColumnStat(Some(8736L), Some("0.09"), Some("99.99"), Some(518), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "i_category_id" -> CatalogColumnStat(Some(10L), Some("1"), Some("10"), Some(515), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "i_size" -> CatalogColumnStat(Some(7L), None, None, Some(515), Some(5), Some(11), None, CatalogColumnStat.VERSION),
+      "i_units" -> CatalogColumnStat(Some(21L), None, None, Some(503), Some(5), Some(7), None, CatalogColumnStat.VERSION)
+    )),
+    "promotion" -> CatalogStatistics(268000L, Some(1000L), Map(
+      "p_promo_name" -> CatalogColumnStat(Some(10L), None, None, Some(18), Some(4), Some(5), None, CatalogColumnStat.VERSION),
+      "p_end_date_sk" -> CatalogColumnStat(Some(564L), Some("2450116"), Some("2450967"), Some(12), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "p_channel_radio" -> CatalogColumnStat(Some(1L), None, None, Some(13), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_channel_demo" -> CatalogColumnStat(Some(1L), None, None, Some(16), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_item_sk" -> CatalogColumnStat(Some(986L), Some("280"), Some("203966"), Some(14), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "p_channel_dmail" -> CatalogColumnStat(Some(2L), None, None, Some(13), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_discount_active" -> CatalogColumnStat(Some(1L), None, None, Some(19), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_promo_id" -> CatalogColumnStat(Some(1000L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "p_channel_tv" -> CatalogColumnStat(Some(1L), None, None, Some(14), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_cost" -> CatalogColumnStat(Some(1L), Some("1000.00"), Some("1000.00"), Some(14), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "p_channel_email" -> CatalogColumnStat(Some(1L), None, None, Some(13), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_channel_event" -> CatalogColumnStat(Some(1L), None, None, Some(14), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_channel_catalog" -> CatalogColumnStat(Some(1L), None, None, Some(14), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_promo_sk" -> CatalogColumnStat(Some(1000L), Some("1"), Some("1000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "p_purpose" -> CatalogColumnStat(Some(1L), None, None, Some(13), Some(7), Some(7), None, CatalogColumnStat.VERSION),
+      "p_start_date_sk" -> CatalogColumnStat(Some(577L), Some("2450100"), Some("2450915"), Some(17), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "p_channel_press" -> CatalogColumnStat(Some(1L), None, None, Some(15), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_channel_details" -> CatalogColumnStat(Some(990L), None, None, Some(10), Some(40), Some(60), None, CatalogColumnStat.VERSION),
+      "p_response_target" -> CatalogColumnStat(Some(1L), Some("1"), Some("1"), Some(19), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "reason" -> CatalogStatistics(3630L, Some(55L), Map(
+      "r_reason_sk" -> CatalogColumnStat(Some(55L), Some("1"), Some("55"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "r_reason_id" -> CatalogColumnStat(Some(55L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "r_reason_desc" -> CatalogColumnStat(Some(55L), None, None, Some(0), Some(14), Some(43), None, CatalogColumnStat.VERSION)
+    )),
+    "ship_mode" -> CatalogStatistics(2420L, Some(20L), Map(
+      "sm_carrier" -> CatalogColumnStat(Some(19L), None, None, Some(0), Some(7), Some(14), None, CatalogColumnStat.VERSION),
+      "sm_ship_mode_sk" -> CatalogColumnStat(Some(20L), Some("1"), Some("20"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "sm_code" -> CatalogColumnStat(Some(4L), None, None, Some(0), Some(5), Some(7), None, CatalogColumnStat.VERSION),
+      "sm_type" -> CatalogColumnStat(Some(5L), None, None, Some(0), Some(8), Some(9), None, CatalogColumnStat.VERSION),
+      "sm_contract" -> CatalogColumnStat(Some(18L), None, None, Some(0), Some(13), Some(20), None, CatalogColumnStat.VERSION),
+      "sm_ship_mode_id" -> CatalogColumnStat(Some(20L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION)
+    )),
+    "store" -> CatalogStatistics(207432L, Some(402L), Map(
+      "s_rec_end_date" -> CatalogColumnStat(Some(3L), None, None, Some(201), Some(10), Some(10), None, CatalogColumnStat.VERSION),
+      "s_state" -> CatalogColumnStat(Some(9L), None, None, Some(2), Some(2), Some(2), None, CatalogColumnStat.VERSION),
+      "s_street_type" -> CatalogColumnStat(Some(20L), None, None, Some(6), Some(5), Some(9), None, CatalogColumnStat.VERSION),
+      "s_floor_space" -> CatalogColumnStat(Some(300L), Some("5004767"), Some("9997773"), Some(3), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "s_division_name" -> CatalogColumnStat(Some(1L), None, None, Some(5), Some(7), Some(7), None, CatalogColumnStat.VERSION),
+      "s_market_desc" -> CatalogColumnStat(Some(318L), None, None, Some(5), Some(59), Some(100), None, CatalogColumnStat.VERSION),
+      "s_street_name" -> CatalogColumnStat(Some(255L), None, None, Some(6), Some(9), Some(16), None, CatalogColumnStat.VERSION),
+      "s_county" -> CatalogColumnStat(Some(9L), None, None, Some(4), Some(15), Some(17), None, CatalogColumnStat.VERSION),
+      "s_rec_start_date" -> CatalogColumnStat(Some(4L), None, None, Some(4), Some(10), Some(10), None, CatalogColumnStat.VERSION),
+      "s_company_id" -> CatalogColumnStat(Some(1L), Some("1"), Some("1"), Some(4), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "s_market_manager" -> CatalogColumnStat(Some(275L), None, None, Some(7), Some(13), Some(20), None, CatalogColumnStat.VERSION),
+      "s_number_employees" -> CatalogColumnStat(Some(94L), Some("200"), Some("300"), Some(5), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "s_manager" -> CatalogColumnStat(Some(300L), None, None, Some(5), Some(13), Some(20), None, CatalogColumnStat.VERSION),
+      "s_country" -> CatalogColumnStat(Some(1L), None, None, Some(4), Some(13), Some(13), None, CatalogColumnStat.VERSION),
+      "s_store_name" -> CatalogColumnStat(Some(10L), None, None, Some(2), Some(4), Some(5), None, CatalogColumnStat.VERSION),
+      "s_division_id" -> CatalogColumnStat(Some(1L), Some("1"), Some("1"), Some(2), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "s_street_number" -> CatalogColumnStat(Some(274L), None, None, Some(5), Some(3), Some(3), None, CatalogColumnStat.VERSION),
+      "s_company_name" -> CatalogColumnStat(Some(1L), None, None, Some(3), Some(7), Some(7), None, CatalogColumnStat.VERSION),
+      "s_gmt_offset" -> CatalogColumnStat(Some(2L), Some("-6.00"), Some("-5.00"), Some(4), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "s_store_sk" -> CatalogColumnStat(Some(399L), Some("1"), Some("402"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "s_city" -> CatalogColumnStat(Some(18L), None, None, Some(5), Some(10), Some(15), None, CatalogColumnStat.VERSION),
+      "s_zip" -> CatalogColumnStat(Some(99L), None, None, Some(6), Some(5), Some(5), None, CatalogColumnStat.VERSION),
+      "s_market_id" -> CatalogColumnStat(Some(10L), Some("1"), Some("10"), Some(6), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "s_hours" -> CatalogColumnStat(Some(3L), None, None, Some(4), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "s_suite_number" -> CatalogColumnStat(Some(75L), None, None, Some(2), Some(8), Some(9), None, CatalogColumnStat.VERSION),
+      "s_closed_date_sk" -> CatalogColumnStat(Some(70L), Some("2450823"), Some("2451313"), Some(296), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "s_store_id" -> CatalogColumnStat(Some(210L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "s_geography_class" -> CatalogColumnStat(Some(1L), None, None, Some(3), Some(7), Some(7), None, CatalogColumnStat.VERSION),
+      "s_tax_percentage" -> CatalogColumnStat(Some(12L), Some("0.00"), Some("0.11"), Some(5), Some(8), Some(8), None, CatalogColumnStat.VERSION)
+    )),
+    "store_returns" -> CatalogStatistics(4837573440L, Some(28795080L), Map(
+      "sr_item_sk" -> CatalogColumnStat(Some(197284L), Some("1"), Some("204000"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_reversed_charge" -> CatalogColumnStat(Some(423824L), Some("0.00"), Some("16099.52"), Some(1009035), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_return_amt_inc_tax" -> CatalogColumnStat(Some(807709L), Some("0.00"), Some("20002.89"), Some(1006919), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_store_sk" -> CatalogColumnStat(Some(199L), Some("1"), Some("400"), Some(1007164), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_return_quantity" -> CatalogColumnStat(Some(103L), Some("1"), Some("100"), Some(1007948), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_fee" -> CatalogColumnStat(Some(9571L), Some("0.50"), Some("100.00"), Some(1008291), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_refunded_cash" -> CatalogColumnStat(Some(559685L), Some("0.00"), Some("17556.95"), Some(1008003), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_return_time_sk" -> CatalogColumnStat(Some(31932L), Some("28799"), Some("61199"), Some(1009330), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_addr_sk" -> CatalogColumnStat(Some(925738L), Some("1"), Some("1000000"), Some(1008253), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_return_amt" -> CatalogColumnStat(Some(492928L), Some("0.00"), Some("18973.20"), Some(1007419), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_returned_date_sk" -> CatalogColumnStat(Some(2010L), Some("2450820"), Some("2452822"), Some(1007464), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_return_tax" -> CatalogColumnStat(Some(86618L), Some("0.00"), Some("1611.71"), Some(1008618), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_return_ship_cost" -> CatalogColumnStat(Some(290195L), Some("0.00"), Some("9578.25"), Some(1007846), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_reason_sk" -> CatalogColumnStat(Some(57L), Some("1"), Some("55"), Some(1008299), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_net_loss" -> CatalogColumnStat(Some(487073L), Some("0.50"), Some("10447.72"), Some(1007153), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_cdemo_sk" -> CatalogColumnStat(Some(1807132L), Some("1"), Some("1920800"), Some(1006835), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_hdemo_sk" -> CatalogColumnStat(Some(6609L), Some("1"), Some("7200"), Some(1008547), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_store_credit" -> CatalogColumnStat(Some(410133L), Some("0.00"), Some("15642.11"), Some(1007102), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_customer_sk" -> CatalogColumnStat(Some(1860981L), Some("1"), Some("2000000"), Some(1008429), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_ticket_number" -> CatalogColumnStat(Some(15853105L), Some("1"), Some("23999996"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION)
+    )),
+    "time_dim" -> CatalogStatistics(10886400L, Some(86400L), Map(
+      "t_sub_shift" -> CatalogColumnStat(Some(4L), None, None, Some(0), Some(7), Some(9), None, CatalogColumnStat.VERSION),
+      "t_time" -> CatalogColumnStat(Some(83505L), Some("0"), Some("86399"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "t_second" -> CatalogColumnStat(Some(60L), Some("0"), Some("59"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "t_hour" -> CatalogColumnStat(Some(25L), Some("0"), Some("23"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "t_am_pm" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(2), Some(2), None, CatalogColumnStat.VERSION),
+      "t_minute" -> CatalogColumnStat(Some(60L), Some("0"), Some("59"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "t_shift" -> CatalogColumnStat(Some(3L), None, None, Some(0), Some(6), Some(6), None, CatalogColumnStat.VERSION),
+      "t_time_id" -> CatalogColumnStat(Some(80197L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "t_time_sk" -> CatalogColumnStat(Some(83505L), Some("0"), Some("86399"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "t_meal_time" -> CatalogColumnStat(Some(3L), None, None, Some(50400), Some(7), Some(9), None, CatalogColumnStat.VERSION)
+    )),
+    "warehouse" -> CatalogStatistics(3855L, Some(15L), Map(
+      "w_state" -> CatalogColumnStat(Some(8L), None, None, Some(0), Some(2), Some(2), None, CatalogColumnStat.VERSION),
+      "w_street_name" -> CatalogColumnStat(Some(14L), None, None, Some(1), Some(10), Some(14), None, CatalogColumnStat.VERSION),
+      "w_warehouse_name" -> CatalogColumnStat(Some(14L), None, None, Some(1), Some(17), Some(20), None, CatalogColumnStat.VERSION),
+      "w_county" -> CatalogColumnStat(Some(8L), None, None, Some(0), Some(14), Some(16), None, CatalogColumnStat.VERSION),
+      "w_street_number" -> CatalogColumnStat(Some(14L), None, None, Some(1), Some(3), Some(3), None, CatalogColumnStat.VERSION),
+      "w_warehouse_sq_ft" -> CatalogColumnStat(Some(14L), Some("73065"), Some("977787"), Some(1), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "w_city" -> CatalogColumnStat(Some(11L), None, None, Some(0), Some(8), Some(13), None, CatalogColumnStat.VERSION),
+      "w_warehouse_sk" -> CatalogColumnStat(Some(15L), Some("1"), Some("15"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "w_suite_number" -> CatalogColumnStat(Some(13L), None, None, Some(1), Some(8), Some(9), None, CatalogColumnStat.VERSION),
+      "w_zip" -> CatalogColumnStat(Some(14L), None, None, Some(0), Some(5), Some(5), None, CatalogColumnStat.VERSION),
+      "w_gmt_offset" -> CatalogColumnStat(Some(2L), Some("-6.00"), Some("-5.00"), Some(1), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "w_street_type" -> CatalogColumnStat(Some(10L), None, None, Some(1), Some(5), Some(7), None, CatalogColumnStat.VERSION),
+      "w_warehouse_id" -> CatalogColumnStat(Some(15L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "w_country" -> CatalogColumnStat(Some(1L), None, None, Some(0), Some(13), Some(13), None, CatalogColumnStat.VERSION)
+    )),
+    "web_page" -> CatalogStatistics(281520L, Some(2040L), Map(
+      "wp_type" -> CatalogColumnStat(Some(7L), None, None, Some(19), Some(7), Some(9), None, CatalogColumnStat.VERSION),
+      "wp_autogen_flag" -> CatalogColumnStat(Some(2L), None, None, Some(25), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "wp_url" -> CatalogColumnStat(Some(1L), None, None, Some(25), Some(18), Some(18), None, CatalogColumnStat.VERSION),
+      "wp_image_count" -> CatalogColumnStat(Some(7L), Some("1"), Some("7"), Some(20), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_max_ad_count" -> CatalogColumnStat(Some(5L), Some("0"), Some("4"), Some(21), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_customer_sk" -> CatalogColumnStat(Some(486L), Some("711"), Some("1996257"), Some(1471), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_rec_end_date" -> CatalogColumnStat(Some(3L), Some("1999-09-03"), Some("2001-09-02"), Some(1020), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_creation_date_sk" -> CatalogColumnStat(Some(121L), Some("2450672"), Some("2450815"), Some(20), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_link_count" -> CatalogColumnStat(Some(25L), Some("2"), Some("25"), Some(16), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_web_page_id" -> CatalogColumnStat(Some(1004L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "wp_web_page_sk" -> CatalogColumnStat(Some(2040L), Some("1"), Some("2040"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_rec_start_date" -> CatalogColumnStat(Some(4L), Some("1997-09-03"), Some("2001-09-03"), Some(21), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_access_date_sk" -> CatalogColumnStat(Some(100L), Some("2452548"), Some("2452648"), Some(19), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_char_count" -> CatalogColumnStat(Some(1493L), Some("303"), Some("8523"), Some(25), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "web_returns" -> CatalogStatistics(1439534000L, Some(7197670L), Map(
+      "wr_returning_addr_sk" -> CatalogColumnStat(Some(925738L), Some("1"), Some("1000000"), Some(323850), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_reversed_charge" -> CatalogColumnStat(Some(346909L), Some("0.00"), Some("22972.36"), Some(323810), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_returned_date_sk" -> CatalogColumnStat(Some(2189L), Some("2450820"), Some("2453002"), Some(324185), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_order_number" -> CatalogColumnStat(Some(4098425L), Some("1"), Some("5999999"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_fee" -> CatalogColumnStat(Some(9571L), Some("0.50"), Some("100.00"), Some(324065), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_account_credit" -> CatalogColumnStat(Some(334119L), Some("0.00"), Some("23028.27"), Some(324422), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_refunded_customer_sk" -> CatalogColumnStat(Some(1808850L), Some("1"), Some("2000000"), Some(324191), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_returning_customer_sk" -> CatalogColumnStat(Some(1813094L), Some("1"), Some("2000000"), Some(324024), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_return_tax" -> CatalogColumnStat(Some(88469L), Some("0.00"), Some("2551.16"), Some(323621), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_web_page_sk" -> CatalogColumnStat(Some(1994L), Some("1"), Some("2040"), Some(324900), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_item_sk" -> CatalogColumnStat(Some(197284L), Some("1"), Some("204000"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_reason_sk" -> CatalogColumnStat(Some(57L), Some("1"), Some("55"), Some(323666), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_return_amt_inc_tax" -> CatalogColumnStat(Some(683544L), Some("0.00"), Some("29493.38"), Some(323171), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_net_loss" -> CatalogColumnStat(Some(494707L), Some("0.50"), Some("15068.96"), Some(324438), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_refunded_cdemo_sk" -> CatalogColumnStat(Some(1755065L), Some("1"), Some("1920800"), Some(323863), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_returned_time_sk" -> CatalogColumnStat(Some(84232L), Some("0"), Some("86399"), Some(323677), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_return_amt" -> CatalogColumnStat(Some(453015L), Some("0.00"), Some("28346.31"), Some(323473), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_returning_cdemo_sk" -> CatalogColumnStat(Some(1755065L), Some("1"), Some("1920800"), Some(323899), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_returning_hdemo_sk" -> CatalogColumnStat(Some(6609L), Some("1"), Some("7200"), Some(323999), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_refunded_cash" -> CatalogColumnStat(Some(484316L), Some("0.00"), Some("26466.56"), Some(324693), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_refunded_hdemo_sk" -> CatalogColumnStat(Some(6609L), Some("1"), Some("7200"), Some(324230), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_return_quantity" -> CatalogColumnStat(Some(103L), Some("1"), Some("100"), Some(323764), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_return_ship_cost" -> CatalogColumnStat(Some(302038L), Some("0.00"), Some("13602.60"), Some(323341), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_refunded_addr_sk" -> CatalogColumnStat(Some(925738L), Some("1"), Some("1000000"), Some(324482), Some(8), Some(8), None, CatalogColumnStat.VERSION)
+    )),
+    "web_site" -> CatalogStatistics(11760L, Some(24L), Map(
+      "web_rec_end_date" -> CatalogColumnStat(Some(3L), Some("1999-08-16"), Some("2001-08-15"), Some(12), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "web_market_manager" -> CatalogColumnStat(Some(21L), None, None, Some(0), Some(13), Some(15), None, CatalogColumnStat.VERSION),
+      "web_country" -> CatalogColumnStat(Some(1L), None, None, Some(0), Some(13), Some(13), None, CatalogColumnStat.VERSION),
+      "web_street_name" -> CatalogColumnStat(Some(24L), None, None, Some(0), Some(10), Some(14), None, CatalogColumnStat.VERSION),
+      "web_city" -> CatalogColumnStat(Some(11L), None, None, Some(0), Some(10), Some(15), None, CatalogColumnStat.VERSION),
+      "web_mkt_id" -> CatalogColumnStat(Some(6L), Some("1"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "web_close_date_sk" -> CatalogColumnStat(Some(8L), Some("2443328"), Some("2447131"), Some(4), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "web_street_number" -> CatalogColumnStat(Some(14L), None, None, Some(0), Some(3), Some(3), None, CatalogColumnStat.VERSION),
+      "web_gmt_offset" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(2), Some(2), None, CatalogColumnStat.VERSION),
+      "web_rec_start_date" -> CatalogColumnStat(Some(4L), Some("1997-08-16"), Some("2001-08-16"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "web_mkt_class" -> CatalogColumnStat(Some(17L), None, None, Some(0), Some(32), Some(47), None, CatalogColumnStat.VERSION),
+      "web_county" -> CatalogColumnStat(Some(9L), None, None, Some(0), Some(14), Some(17), None, CatalogColumnStat.VERSION),
+      "web_class" -> CatalogColumnStat(Some(1L), None, None, Some(0), Some(7), Some(7), None, CatalogColumnStat.VERSION),
+      "web_site_sk" -> CatalogColumnStat(Some(24L), Some("1"), Some("24"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "web_manager" -> CatalogColumnStat(Some(19L), None, None, Some(0), Some(13), Some(16), None, CatalogColumnStat.VERSION),
+      "web_suite_number" -> CatalogColumnStat(Some(20L), None, None, Some(0), Some(9), Some(9), None, CatalogColumnStat.VERSION),
+      "web_site_id" -> CatalogColumnStat(Some(12L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "web_company_name" -> CatalogColumnStat(Some(6L), None, None, Some(0), Some(5), Some(5), None, CatalogColumnStat.VERSION),
+      "web_state" -> CatalogColumnStat(Some(9L), None, None, Some(0), Some(2), Some(2), None, CatalogColumnStat.VERSION),
+      "web_mkt_desc" -> CatalogColumnStat(Some(15L), None, None, Some(0), Some(66), Some(92), None, CatalogColumnStat.VERSION),
+      "web_name" -> CatalogColumnStat(Some(4L), None, None, Some(0), Some(6), Some(6), None, CatalogColumnStat.VERSION),
+      "web_company_id" -> CatalogColumnStat(Some(6L), Some("1"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "web_street_type" -> CatalogColumnStat(Some(15L), None, None, Some(0), Some(4), Some(9), None, CatalogColumnStat.VERSION),
+      "web_open_date_sk" -> CatalogColumnStat(Some(12L), Some("2450628"), Some("2450807"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "web_tax_percentage" -> CatalogColumnStat(Some(8L), Some("0.00"), Some("0.12"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "web_zip" -> CatalogColumnStat(Some(14L), None, None, Some(0), Some(5), Some(5), None, CatalogColumnStat.VERSION)
+    ))
+  )
+  // scalastyle:on line.size.limit
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TypedImperativeAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/TypedImperativeAggregateSuite.scala
index f2b608bbcb21b..986e625137a77 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/TypedImperativeAggregateSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/TypedImperativeAggregateSuite.scala
@@ -23,6 +23,7 @@ import org.apache.spark.sql.TypedImperativeAggregateSuite.TypedMax
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{BoundReference, Expression, GenericInternalRow, ImplicitCastInputTypes, SpecificInternalRow}
 import org.apache.spark.sql.catalyst.expressions.aggregate.TypedImperativeAggregate
+import org.apache.spark.sql.catalyst.trees.UnaryLike
 import org.apache.spark.sql.execution.aggregate.HashAggregateExec
 import org.apache.spark.sql.expressions.Window
 import org.apache.spark.sql.functions._
@@ -147,9 +148,9 @@ class TypedImperativeAggregateSuite extends QueryTest with SharedSparkSession {
     val query = df.select(typedMax($"key"), count($"key"), typedMax($"value"),
       count($"value"))
     val maxKey = nullableData.map(_._1).filter(_ != null).max
-    val countKey = nullableData.map(_._1).filter(_ != null).size
+    val countKey = nullableData.map(_._1).count(_ != null)
     val maxValue = nullableData.map(_._2).filter(_ != null).max
-    val countValue = nullableData.map(_._2).filter(_ != null).size
+    val countValue = nullableData.map(_._2).count(_ != null)
     val expected = Seq(Row(maxKey, countKey, maxValue, countValue))
     checkAnswer(query, expected)
   }
@@ -232,8 +233,8 @@ object TypedImperativeAggregateSuite {
       nullable: Boolean = false,
       mutableAggBufferOffset: Int = 0,
       inputAggBufferOffset: Int = 0)
-    extends TypedImperativeAggregate[MaxValue] with ImplicitCastInputTypes {
-
+    extends TypedImperativeAggregate[MaxValue] with ImplicitCastInputTypes
+    with UnaryLike[Expression] {
 
     override def createAggregationBuffer(): MaxValue = {
       // Returns Int.MinValue if all inputs are null
@@ -270,8 +271,6 @@ object TypedImperativeAggregateSuite {
 
     override lazy val deterministic: Boolean = true
 
-    override def children: Seq[Expression] = Seq(child)
-
     override def inputTypes: Seq[AbstractDataType] = Seq(IntegerType)
 
     override def dataType: DataType = IntegerType
@@ -297,6 +296,9 @@ object TypedImperativeAggregateSuite {
       val value = stream.readInt()
       new MaxValue(value, isValueSet)
     }
+
+    override protected def withNewChildInternal(newChild: Expression): TypedMax =
+      copy(child = newChild)
   }
 
   private class MaxValue(var value: Int, var isValueSet: Boolean = false)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
index e2747d7db9f3f..116e1ef1f9445 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
@@ -18,14 +18,26 @@
 package org.apache.spark.sql
 
 import java.math.BigDecimal
+import java.sql.Timestamp
+import java.time.{Instant, LocalDate}
+import java.time.format.DateTimeFormatter
 
+import scala.collection.mutable.{ArrayBuffer, WrappedArray}
+
+import org.apache.spark.SparkException
 import org.apache.spark.sql.api.java._
+import org.apache.spark.sql.catalyst.FunctionIdentifier
+import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, OuterScopes}
+import org.apache.spark.sql.catalyst.expressions.{Literal, ScalaUDF}
 import org.apache.spark.sql.catalyst.plans.logical.Project
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.execution.{QueryExecution, SimpleMode}
+import org.apache.spark.sql.execution.aggregate.{ScalaAggregator, ScalaUDAF}
 import org.apache.spark.sql.execution.columnar.InMemoryRelation
 import org.apache.spark.sql.execution.command.{CreateDataSourceTableAsSelectCommand, ExplainCommand}
 import org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand
-import org.apache.spark.sql.functions.{lit, udf}
+import org.apache.spark.sql.expressions.{Aggregator, MutableAggregationBuffer, SparkUserDefinedFunction, UserDefinedAggregateFunction}
+import org.apache.spark.sql.functions.{lit, struct, udaf, udf}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.test.SQLTestData._
@@ -33,6 +45,8 @@ import org.apache.spark.sql.types._
 import org.apache.spark.sql.util.QueryExecutionListener
 
 private case class FunctionResult(f1: String, f2: String)
+private case class LocalDateInstantType(date: LocalDate, instant: Instant)
+private case class TimestampInstantType(t: Timestamp, instant: Instant)
 
 class UDFSuite extends QueryTest with SharedSparkSession {
   import testImplicits._
@@ -330,7 +344,7 @@ class UDFSuite extends QueryTest with SharedSparkSession {
       Console.withOut(outputStream) {
         spark.sql("SELECT f(a._1) FROM x").show
       }
-      assert(outputStream.toString.contains("f(a._1 AS `_1`)"))
+      assert(outputStream.toString.contains("f(a._1)"))
     }
   }
 
@@ -483,7 +497,7 @@ class UDFSuite extends QueryTest with SharedSparkSession {
       sparkContext.parallelize(Seq(Row(Map("a" -> new BigDecimal("2011000000000002456556"))))),
       StructType(Seq(StructField("col1", MapType(StringType, DecimalType(30, 0))))))
     val udf2 = org.apache.spark.sql.functions.udf((map: Map[String, BigDecimal]) => {
-      map.mapValues(value => if (value == null) null else value.toBigInteger.toString)
+      map.mapValues(value => if (value == null) null else value.toBigInteger.toString).toMap
     })
     checkAnswer(df2.select(udf2($"col1")), Seq(Row(Map("a" -> "2011000000000002456556"))))
   }
@@ -504,23 +518,94 @@ class UDFSuite extends QueryTest with SharedSparkSession {
   }
 
   test("Using java.time.Instant in UDF") {
-    withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
-      val expected = java.time.Instant.parse("2019-02-27T00:00:00Z")
-      val plusSec = udf((i: java.time.Instant) => i.plusSeconds(1))
-      val df = spark.sql("SELECT TIMESTAMP '2019-02-26 23:59:59Z' as t")
-        .select(plusSec('t))
-      assert(df.collect().toSeq === Seq(Row(expected)))
-    }
+    val dtf = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
+    val expected = java.time.Instant.parse("2019-02-27T00:00:00Z")
+      .atZone(DateTimeUtils.getZoneId(conf.sessionLocalTimeZone))
+      .toLocalDateTime
+      .format(dtf)
+    val plusSec = udf((i: java.time.Instant) => i.plusSeconds(1))
+    val df = spark.sql("SELECT TIMESTAMP '2019-02-26 23:59:59Z' as t")
+      .select(plusSec('t).cast(StringType))
+    checkAnswer(df, Row(expected) :: Nil)
   }
 
   test("Using java.time.LocalDate in UDF") {
-    withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
-      val expected = java.time.LocalDate.parse("2019-02-27")
-      val plusDay = udf((i: java.time.LocalDate) => i.plusDays(1))
-      val df = spark.sql("SELECT DATE '2019-02-26' as d")
-        .select(plusDay('d))
-      assert(df.collect().toSeq === Seq(Row(expected)))
-    }
+    val expected = java.time.LocalDate.parse("2019-02-27").toString
+    val plusDay = udf((i: java.time.LocalDate) => i.plusDays(1))
+    val df = spark.sql("SELECT DATE '2019-02-26' as d")
+      .select(plusDay('d).cast(StringType))
+    checkAnswer(df, Row(expected) :: Nil)
+  }
+
+  test("Using combined types of Instant/LocalDate in UDF") {
+    val dtf = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
+    val date = LocalDate.parse("2019-02-26")
+    val instant = Instant.parse("2019-02-26T23:59:59Z")
+    val expectedDate = date.toString
+    val expectedInstant =
+      instant.atZone(DateTimeUtils.getZoneId(conf.sessionLocalTimeZone))
+        .toLocalDateTime
+        .format(dtf)
+    val df = Seq((date, instant)).toDF("d", "i")
+
+    // test normal case
+    spark.udf.register("buildLocalDateInstantType",
+      udf((d: LocalDate, i: Instant) => LocalDateInstantType(d, i)))
+    checkAnswer(df.selectExpr(s"buildLocalDateInstantType(d, i) as di")
+      .select('di.cast(StringType)),
+      Row(s"{$expectedDate, $expectedInstant}") :: Nil)
+
+    // test null cases
+    spark.udf.register("buildLocalDateInstantType",
+      udf((d: LocalDate, i: Instant) => LocalDateInstantType(null, null)))
+    checkAnswer(df.selectExpr("buildLocalDateInstantType(d, i) as di"),
+      Row(Row(null, null)))
+
+    spark.udf.register("buildLocalDateInstantType",
+      udf((d: LocalDate, i: Instant) => null.asInstanceOf[LocalDateInstantType]))
+    checkAnswer(df.selectExpr("buildLocalDateInstantType(d, i) as di"),
+      Row(null))
+  }
+
+  test("Using combined types of Instant/Timestamp in UDF") {
+    val dtf = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
+    val timestamp = Timestamp.valueOf("2019-02-26 23:59:59")
+    val instant = Instant.parse("2019-02-26T23:59:59Z")
+    val expectedTimestamp = timestamp.toLocalDateTime.format(dtf)
+    val expectedInstant =
+      instant.atZone(DateTimeUtils.getZoneId(conf.sessionLocalTimeZone))
+      .toLocalDateTime
+      .format(dtf)
+    val df = Seq((timestamp, instant)).toDF("t", "i")
+
+    // test normal case
+    spark.udf.register("buildTimestampInstantType",
+      udf((t: Timestamp, i: Instant) => TimestampInstantType(t, i)))
+    checkAnswer(df.selectExpr("buildTimestampInstantType(t, i) as ti")
+      .select('ti.cast(StringType)),
+      Row(s"{$expectedTimestamp, $expectedInstant}"))
+
+    // test null cases
+    spark.udf.register("buildTimestampInstantType",
+      udf((t: Timestamp, i: Instant) => TimestampInstantType(null, null)))
+    checkAnswer(df.selectExpr("buildTimestampInstantType(t, i) as ti"),
+      Row(Row(null, null)))
+
+    spark.udf.register("buildTimestampInstantType",
+      udf((t: Timestamp, i: Instant) => null.asInstanceOf[TimestampInstantType]))
+    checkAnswer(df.selectExpr("buildTimestampInstantType(t, i) as ti"),
+      Row(null))
+  }
+
+  test("SPARK-32154: return null with or without explicit type") {
+    // without explicit type
+    val udf1 = udf((i: String) => null)
+    assert(udf1.asInstanceOf[SparkUserDefinedFunction] .dataType === NullType)
+    checkAnswer(Seq("1").toDF("a").select(udf1('a)), Row(null) :: Nil)
+    // with explicit type
+    val udf2 = udf((i: String) => null.asInstanceOf[String])
+    assert(udf2.asInstanceOf[SparkUserDefinedFunction].dataType === StringType)
+    checkAnswer(Seq("1").toDF("a").select(udf1('a)), Row(null) :: Nil)
   }
 
   test("SPARK-28321 0-args Java UDF should not be called only once") {
@@ -581,4 +666,234 @@ class UDFSuite extends QueryTest with SharedSparkSession {
       .toDF("col1", "col2")
     checkAnswer(df.select(myUdf(Column("col1"), Column("col2"))), Row(2020) :: Nil)
   }
+
+  test("case class as element type of Seq/Array") {
+    val f1 = (s: Seq[TestData]) => s.map(d => d.key * d.value.toInt).sum
+    val myUdf1 = udf(f1)
+    val df1 = Seq(("data", Seq(TestData(50, "2")))).toDF("col1", "col2")
+    checkAnswer(df1.select(myUdf1(Column("col2"))), Row(100) :: Nil)
+
+    val f2 = (s: Array[TestData]) => s.map(d => d.key * d.value.toInt).sum
+    val myUdf2 = udf(f2)
+    val df2 = Seq(("data", Array(TestData(50, "2")))).toDF("col1", "col2")
+    checkAnswer(df2.select(myUdf2(Column("col2"))), Row(100) :: Nil)
+  }
+
+  test("case class as key/value type of Map") {
+    val f1 = (s: Map[TestData, Int]) => s.keys.head.key * s.keys.head.value.toInt
+    val myUdf1 = udf(f1)
+    val df1 = Seq(("data", Map(TestData(50, "2") -> 502))).toDF("col1", "col2")
+    checkAnswer(df1.select(myUdf1(Column("col2"))), Row(100) :: Nil)
+
+    val f2 = (s: Map[Int, TestData]) => s.values.head.key * s.values.head.value.toInt
+    val myUdf2 = udf(f2)
+    val df2 = Seq(("data", Map(502 -> TestData(50, "2")))).toDF("col1", "col2")
+    checkAnswer(df2.select(myUdf2(Column("col2"))), Row(100) :: Nil)
+
+    val f3 = (s: Map[TestData, TestData]) => s.keys.head.key * s.values.head.value.toInt
+    val myUdf3 = udf(f3)
+    val df3 = Seq(("data", Map(TestData(50, "2") -> TestData(50, "2")))).toDF("col1", "col2")
+    checkAnswer(df3.select(myUdf3(Column("col2"))), Row(100) :: Nil)
+  }
+
+  test("case class as element of tuple") {
+    val f = (s: (TestData, Int)) => s._1.key * s._2
+    val myUdf = udf(f)
+    val df = Seq(("data", (TestData(50, "2"), 2))).toDF("col1", "col2")
+    checkAnswer(df.select(myUdf(Column("col2"))), Row(100) :: Nil)
+  }
+
+  test("case class as generic type of Option") {
+    val f = (o: Option[TestData]) => o.map(t => t.key * t.value.toInt)
+    val myUdf = udf(f)
+    val df1 = Seq(("data", Some(TestData(50, "2")))).toDF("col1", "col2")
+    checkAnswer(df1.select(myUdf(Column("col2"))), Row(100) :: Nil)
+    val df2 = Seq(("data", None: Option[TestData])).toDF("col1", "col2")
+    checkAnswer(df2.select(myUdf(Column("col2"))), Row(null) :: Nil)
+  }
+
+  test("more input fields than expect for case class") {
+    val f = (t: TestData2) => t.a * t.b
+    val myUdf = udf(f)
+    val df = spark.range(1)
+      .select(lit(50).as("a"), lit(2).as("b"), lit(2).as("c"))
+      .select(struct("a", "b", "c").as("col"))
+    checkAnswer(df.select(myUdf(Column("col"))), Row(100) :: Nil)
+  }
+
+  test("less input fields than expect for case class") {
+    val f = (t: TestData2) => t.a * t.b
+    val myUdf = udf(f)
+    val df = spark.range(1)
+      .select(lit(50).as("a"))
+      .select(struct("a").as("col"))
+    val error = intercept[AnalysisException](df.select(myUdf(Column("col"))))
+    assert(error.getMessage.contains("cannot resolve 'b' given input columns: [a]"))
+  }
+
+  test("wrong order of input fields for case class") {
+    val f = (t: TestData) => t.key * t.value.toInt
+    val myUdf = udf(f)
+    val df = spark.range(1)
+      .select(lit("2").as("value"), lit(50).as("key"))
+      .select(struct("value", "key").as("col"))
+    checkAnswer(df.select(myUdf(Column("col"))), Row(100) :: Nil)
+  }
+
+  test("top level Option primitive type") {
+    val f = (i: Option[Int]) => i.map(_ * 10)
+    val myUdf = udf(f)
+    val df = Seq(Some(10), None).toDF("col")
+    checkAnswer(df.select(myUdf(Column("col"))), Row(100) :: Row(null) :: Nil)
+  }
+
+  test("array Option") {
+    val f = (i: Array[Option[TestData]]) =>
+      i.map(_.map(t => t.key * t.value.toInt).getOrElse(0)).sum
+    val myUdf = udf(f)
+    val df = Seq(Array(Some(TestData(50, "2")), None)).toDF("col")
+    checkAnswer(df.select(myUdf(Column("col"))), Row(100) :: Nil)
+  }
+
+  object MalformedClassObject extends Serializable {
+    class MalformedNonPrimitiveFunction extends (String => Int) with Serializable {
+      override def apply(v1: String): Int = v1.toInt / 0
+    }
+
+    class MalformedPrimitiveFunction extends (Int => Int) with Serializable {
+      override def apply(v1: Int): Int = v1 / 0
+    }
+  }
+
+  test("SPARK-32238: Use Utils.getSimpleName to avoid hitting Malformed class name") {
+    OuterScopes.addOuterScope(MalformedClassObject)
+    val f1 = new MalformedClassObject.MalformedNonPrimitiveFunction()
+    val f2 = new MalformedClassObject.MalformedPrimitiveFunction()
+
+    val e1 = intercept[SparkException] {
+      Seq("20").toDF("col").select(udf(f1).apply(Column("col"))).collect()
+    }
+    assert(e1.getMessage.contains("UDFSuite$MalformedClassObject$MalformedNonPrimitiveFunction"))
+
+    val e2 = intercept[SparkException] {
+      Seq(20).toDF("col").select(udf(f2).apply(Column("col"))).collect()
+    }
+    assert(e2.getMessage.contains("UDFSuite$MalformedClassObject$MalformedPrimitiveFunction"))
+  }
+
+  test("SPARK-32307: Aggregation that use map type input UDF as group expression") {
+    spark.udf.register("key", udf((m: Map[String, String]) => m.keys.head.toInt))
+    Seq(Map("1" -> "one", "2" -> "two")).toDF("a").createOrReplaceTempView("t")
+    checkAnswer(sql("SELECT key(a) AS k FROM t GROUP BY key(a)"), Row(1) :: Nil)
+  }
+
+  test("SPARK-32307: Aggregation that use array type input UDF as group expression") {
+    spark.udf.register("key", udf((m: Array[Int]) => m.head))
+    Seq(Array(1)).toDF("a").createOrReplaceTempView("t")
+    checkAnswer(sql("SELECT key(a) AS k FROM t GROUP BY key(a)"), Row(1) :: Nil)
+  }
+
+  test("SPARK-32459: UDF should not fail on WrappedArray") {
+    val myUdf = udf((a: WrappedArray[Int]) =>
+      WrappedArray.make[Int](Array(a.head + 99)))
+    checkAnswer(Seq(Array(1))
+      .toDF("col")
+      .select(myUdf(Column("col"))),
+      Row(ArrayBuffer(100)))
+  }
+
+  test("SPARK-34388: UDF name is propagated with registration for ScalaUDF") {
+    spark.udf.register("udf34388", udf((value: Int) => value > 2))
+    spark.sessionState.catalog.lookupFunction(
+      FunctionIdentifier("udf34388"), Seq(Literal(1))) match {
+      case udf: ScalaUDF => assert(udf.name === "udf34388")
+    }
+  }
+
+  test("SPARK-34388: UDF name is propagated with registration for ScalaAggregator") {
+    val agg = new Aggregator[Long, Long, Long] {
+      override def zero: Long = 0L
+      override def reduce(b: Long, a: Long): Long = a + b
+      override def merge(b1: Long, b2: Long): Long = b1 + b2
+      override def finish(reduction: Long): Long = reduction
+      override def bufferEncoder: Encoder[Long] = ExpressionEncoder[Long]()
+      override def outputEncoder: Encoder[Long] = ExpressionEncoder[Long]()
+    }
+
+    spark.udf.register("agg34388", udaf(agg))
+    spark.sessionState.catalog.lookupFunction(
+      FunctionIdentifier("agg34388"), Seq(Literal(1))) match {
+      case agg: ScalaAggregator[_, _, _] => assert(agg.name === "agg34388")
+    }
+  }
+
+  test("SPARK-34388: UDF name is propagated with registration for ScalaUDAF") {
+    val udaf = new UserDefinedAggregateFunction {
+      def inputSchema: StructType = new StructType().add("a", LongType)
+      def bufferSchema: StructType = new StructType().add("product", LongType)
+      def dataType: DataType = LongType
+      def deterministic: Boolean = true
+      def initialize(buffer: MutableAggregationBuffer): Unit = {}
+      def update(buffer: MutableAggregationBuffer, input: Row): Unit = {}
+      def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {}
+      def evaluate(buffer: Row): Any = buffer.getLong(0)
+    }
+    spark.udf.register("udaf34388", udaf)
+    spark.sessionState.catalog.lookupFunction(
+      FunctionIdentifier("udaf34388"), Seq(Literal(1))) match {
+      case udaf: ScalaUDAF => assert(udaf.name === "udaf34388")
+    }
+  }
+
+  test("SPARK-34663: using java.time.Duration in UDF") {
+    // Regular case
+    val input = Seq(java.time.Duration.ofHours(23)).toDF("d")
+    val plusHour = udf((d: java.time.Duration) => d.plusHours(1))
+    val result = input.select(plusHour($"d").as("new_d"))
+    checkAnswer(result, Row(java.time.Duration.ofDays(1)) :: Nil)
+    assert(result.schema === new StructType().add("new_d", DayTimeIntervalType))
+    // UDF produces `null`
+    val nullFunc = udf((_: java.time.Duration) => null.asInstanceOf[java.time.Duration])
+    val nullResult = input.select(nullFunc($"d").as("null_d"))
+    checkAnswer(nullResult, Row(null) :: Nil)
+    assert(nullResult.schema === new StructType().add("null_d", DayTimeIntervalType))
+    // Input parameter of UDF is null
+    val nullInput = Seq(null.asInstanceOf[java.time.Duration]).toDF("null_d")
+    val constDuration = udf((_: java.time.Duration) => java.time.Duration.ofMinutes(10))
+    val constResult = nullInput.select(constDuration($"null_d").as("10_min"))
+    checkAnswer(constResult, Row(java.time.Duration.ofMinutes(10)) :: Nil)
+    assert(constResult.schema === new StructType().add("10_min", DayTimeIntervalType))
+    // Error in the conversion of UDF result to the internal representation of day-time interval
+    val overflowFunc = udf((d: java.time.Duration) => d.plusDays(Long.MaxValue))
+    val e = intercept[SparkException] {
+      input.select(overflowFunc($"d")).collect()
+    }.getCause.getCause
+    assert(e.isInstanceOf[java.lang.ArithmeticException])
+  }
+
+  test("SPARK-34663: using java.time.Period in UDF") {
+    // Regular case
+    val input = Seq(java.time.Period.ofMonths(11)).toDF("p")
+    val incMonth = udf((p: java.time.Period) => p.plusMonths(1))
+    val result = input.select(incMonth($"p").as("new_p"))
+    checkAnswer(result, Row(java.time.Period.ofYears(1)) :: Nil)
+    assert(result.schema === new StructType().add("new_p", YearMonthIntervalType))
+    // UDF produces `null`
+    val nullFunc = udf((_: java.time.Period) => null.asInstanceOf[java.time.Period])
+    val nullResult = input.select(nullFunc($"p").as("null_p"))
+    checkAnswer(nullResult, Row(null) :: Nil)
+    assert(nullResult.schema === new StructType().add("null_p", YearMonthIntervalType))
+    // Input parameter of UDF is null
+    val nullInput = Seq(null.asInstanceOf[java.time.Period]).toDF("null_p")
+    val constPeriod = udf((_: java.time.Period) => java.time.Period.ofYears(10))
+    val constResult = nullInput.select(constPeriod($"null_p").as("10_years"))
+    checkAnswer(constResult, Row(java.time.Period.ofYears(10)) :: Nil)
+    assert(constResult.schema === new StructType().add("10_years", YearMonthIntervalType))
+    // Error in the conversion of UDF result to the internal representation of year-month interval
+    val overflowFunc = udf((p: java.time.Period) => p.plusYears(Long.MaxValue))
+    val e = intercept[SparkException] {
+      input.select(overflowFunc($"p")).collect()
+    }.getCause.getCause
+    assert(e.isInstanceOf[java.lang.ArithmeticException])
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala
index a5f904c621e6e..9daa69ce9f155 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala
@@ -178,4 +178,14 @@ class UnsafeRowSuite extends SparkFunSuite {
     // Makes sure hashCode on unsafe array won't crash
     unsafeRow.getArray(0).hashCode()
   }
+
+  test("SPARK-32018: setDecimal with overflowed value") {
+    val d1 = new Decimal().set(BigDecimal("10000000000000000000")).toPrecision(38, 18)
+    val row = InternalRow.apply(d1)
+    val unsafeRow = UnsafeProjection.create(Array[DataType](DecimalType(38, 18))).apply(row)
+    assert(unsafeRow.getDecimal(0, 38, 18) === d1)
+    val d2 = (d1 * Decimal(10)).toPrecision(39, 18)
+    unsafeRow.setDecimal(0, d2, 38)
+    assert(unsafeRow.getDecimal(0, 38, 18) === null)
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UnwrapCastInComparisonEndToEndSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UnwrapCastInComparisonEndToEndSuite.scala
new file mode 100644
index 0000000000000..e6f0426428bd4
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UnwrapCastInComparisonEndToEndSuite.scala
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import org.apache.spark.sql.catalyst.expressions.IntegralLiteralTestUtils.{negativeInt, positiveInt}
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.Decimal
+
+class UnwrapCastInComparisonEndToEndSuite extends QueryTest with SharedSparkSession {
+  import testImplicits._
+
+  val t = "test_table"
+
+  test("cases when literal is max") {
+    withTable(t) {
+      Seq[(Integer, java.lang.Short, java.lang.Float)](
+        (1, 100.toShort, 3.14.toFloat),
+        (2, Short.MaxValue, Float.NaN),
+        (3, Short.MinValue, Float.PositiveInfinity),
+        (4, 0.toShort, Float.MaxValue),
+        (5, null, null))
+        .toDF("c1", "c2", "c3").write.saveAsTable(t)
+      val df = spark.table(t)
+
+      val lit = Short.MaxValue.toInt
+      checkAnswer(df.where(s"c2 > $lit").select("c1"), Seq.empty)
+      checkAnswer(df.where(s"c2 >= $lit").select("c1"), Row(2))
+      checkAnswer(df.where(s"c2 == $lit").select("c1"), Row(2))
+      checkAnswer(df.where(s"c2 <=> $lit").select("c1"), Row(2))
+      checkAnswer(df.where(s"c2 != $lit").select("c1"), Row(1) :: Row(3) :: Row(4) :: Nil)
+      checkAnswer(df.where(s"c2 <= $lit").select("c1"), Row(1) :: Row(2) :: Row(3) :: Row(4) :: Nil)
+      checkAnswer(df.where(s"c2 < $lit").select("c1"), Row(1) :: Row(3) :: Row(4) :: Nil)
+
+      checkAnswer(df.where(s"c3 > double('nan')").select("c1"), Seq.empty)
+      checkAnswer(df.where(s"c3 >= double('nan')").select("c1"), Row(2))
+      checkAnswer(df.where(s"c3 == double('nan')").select("c1"), Row(2))
+      checkAnswer(df.where(s"c3 <=> double('nan')").select("c1"), Row(2))
+      checkAnswer(df.where(s"c3 != double('nan')").select("c1"), Row(1) :: Row(3) :: Row(4) :: Nil)
+      checkAnswer(df.where(s"c3 <= double('nan')").select("c1"),
+        Row(1) :: Row(2) :: Row(3) :: Row(4) :: Nil)
+      checkAnswer(df.where(s"c3 < double('nan')").select("c1"), Row(1) :: Row(3) :: Row(4) :: Nil)
+    }
+  }
+
+  test("cases when literal is > max") {
+    withTable(t) {
+      Seq[(Integer, java.lang.Short)](
+        (1, 100.toShort),
+        (2, Short.MaxValue),
+        (3, null))
+        .toDF("c1", "c2").write.saveAsTable(t)
+      val df = spark.table(t)
+      val lit = positiveInt
+      checkAnswer(df.where(s"c2 > $lit").select("c1"), Seq.empty)
+      checkAnswer(df.where(s"c2 >= $lit").select("c1"), Seq.empty)
+      checkAnswer(df.where(s"c2 == $lit").select("c1"), Seq.empty)
+      checkAnswer(df.where(s"c2 <=> $lit").select("c1"), Seq.empty)
+      checkAnswer(df.where(s"c2 != $lit").select("c1"), Row(1) :: Row(2) :: Nil)
+      checkAnswer(df.where(s"c2 <= $lit").select("c1"), Row(1) :: Row(2) :: Nil)
+      checkAnswer(df.where(s"c2 < $lit").select("c1"), Row(1) :: Row(2) :: Nil)
+
+      // No test for float case since NaN is greater than any other numeric value
+    }
+  }
+
+  test("cases when literal is min") {
+    withTable(t) {
+      Seq[(Integer, java.lang.Short, java.lang.Float)](
+        (1, 100.toShort, 3.14.toFloat),
+        (2, Short.MinValue, Float.NegativeInfinity),
+        (3, Short.MaxValue, Float.MinValue),
+        (4, null, null))
+        .toDF("c1", "c2", "c3").write.saveAsTable(t)
+      val df = spark.table(t)
+
+      val lit = Short.MinValue.toInt
+      checkAnswer(df.where(s"c2 > $lit").select("c1"), Row(1) :: Row(3) :: Nil)
+      checkAnswer(df.where(s"c2 >= $lit").select("c1"), Row(1) :: Row(2) :: Row(3) :: Nil)
+      checkAnswer(df.where(s"c2 == $lit").select("c1"), Row(2))
+      checkAnswer(df.where(s"c2 <=> $lit").select("c1"), Row(2))
+      checkAnswer(df.where(s"c2 != $lit").select("c1"), Row(1) :: Row(3) :: Nil)
+      checkAnswer(df.where(s"c2 <= $lit").select("c1"), Row(2))
+      checkAnswer(df.where(s"c2 < $lit").select("c1"), Seq.empty)
+
+      checkAnswer(df.where(s"c3 > double('-inf')").select("c1"), Row(1) :: Row(3) :: Nil)
+      checkAnswer(df.where(s"c3 >= double('-inf')").select("c1"), Row(1) :: Row(2) :: Row(3) :: Nil)
+      checkAnswer(df.where(s"c3 == double('-inf')").select("c1"), Row(2))
+      checkAnswer(df.where(s"c3 <=> double('-inf')").select("c1"), Row(2))
+      checkAnswer(df.where(s"c3 != double('-inf')").select("c1"), Row(1) :: Row(3) :: Nil)
+      checkAnswer(df.where(s"c3 <= double('-inf')").select("c1"), Row(2) :: Nil)
+      checkAnswer(df.where(s"c3 < double('-inf')").select("c1"), Seq.empty)
+    }
+  }
+
+  test("cases when literal is < min") {
+    val t = "test_table"
+    withTable(t) {
+      Seq[(Integer, java.lang.Short)](
+        (1, 100.toShort),
+        (2, Short.MinValue),
+        (3, null))
+        .toDF("c1", "c2").write.saveAsTable(t)
+      val df = spark.table(t)
+
+      val lit = negativeInt
+      checkAnswer(df.where(s"c2 > $lit").select("c1"), Row(1) :: Row(2) :: Nil)
+      checkAnswer(df.where(s"c2 >= $lit").select("c1"), Row(1) :: Row(2) :: Nil)
+      checkAnswer(df.where(s"c2 == $lit").select("c1"), Seq.empty)
+      checkAnswer(df.where(s"c2 <=> $lit").select("c1"), Seq.empty)
+      checkAnswer(df.where(s"c2 != $lit").select("c1"), Row(1) :: Row(2) :: Nil)
+      checkAnswer(df.where(s"c2 <= $lit").select("c1"), Seq.empty)
+      checkAnswer(df.where(s"c2 < $lit").select("c1"), Seq.empty)
+    }
+  }
+
+  test("cases when literal is within range (min, max)") {
+    withTable(t) {
+      Seq((1, 300.toShort), (2, 500.toShort)).toDF("c1", "c2").write.saveAsTable(t)
+      val df = spark.table(t)
+
+      checkAnswer(df.where("c2 < 200").select("c1"), Seq.empty)
+      checkAnswer(df.where("c2 < 400").select("c1"), Row(1) :: Nil)
+      checkAnswer(df.where("c2 < 600").select("c1"), Row(1) :: Row(2) :: Nil)
+
+      checkAnswer(df.where("c2 <= 100").select("c1"), Seq.empty)
+      checkAnswer(df.where("c2 <= 300").select("c1"), Row(1) :: Nil)
+      checkAnswer(df.where("c2 <= 500").select("c1"), Row(1) :: Row(2) :: Nil)
+
+      checkAnswer(df.where("c2 == 100").select("c1"), Seq.empty)
+      checkAnswer(df.where("c2 == 300").select("c1"), Row(1) :: Nil)
+      checkAnswer(df.where("c2 == 500").select("c1"), Row(2) :: Nil)
+
+      checkAnswer(df.where("c2 <=> 100").select("c1"), Seq.empty)
+      checkAnswer(df.where("c2 <=> 300").select("c1"), Row(1) :: Nil)
+      checkAnswer(df.where("c2 <=> 500").select("c1"), Row(2) :: Nil)
+      checkAnswer(df.where("c2 <=> null").select("c1"), Seq.empty)
+
+      checkAnswer(df.where("c2 >= 200").select("c1"), Row(1) :: Row(2) :: Nil)
+      checkAnswer(df.where("c2 >= 400").select("c1"), Row(2) :: Nil)
+      checkAnswer(df.where("c2 >= 600").select("c1"), Seq.empty)
+
+      checkAnswer(df.where("c2 > 100").select("c1"), Row(1) :: Row(2) :: Nil)
+      checkAnswer(df.where("c2 > 300").select("c1"), Row(2) :: Nil)
+      checkAnswer(df.where("c2 > 500").select("c1"), Seq.empty)
+    }
+  }
+
+  test("cases when literal is within range (min, max) and has rounding up or down") {
+    withTable(t) {
+      Seq((1, 100, 3.14.toFloat, decimal(200.12)))
+        .toDF("c1", "c2", "c3", "c4").write.saveAsTable(t)
+      val df = spark.table(t)
+
+      checkAnswer(df.where("c2 > 99.6").select("c1"), Row(1))
+      checkAnswer(df.where("c2 > 100.4").select("c1"), Seq.empty)
+      checkAnswer(df.where("c2 == 100.4").select("c1"), Seq.empty)
+      checkAnswer(df.where("c2 <=> 100.4").select("c1"), Seq.empty)
+      checkAnswer(df.where("c2 < 99.6").select("c1"), Seq.empty)
+      checkAnswer(df.where("c2 < 100.4").select("c1"), Row(1))
+
+      checkAnswer(df.where("c3 >= 3.14").select("c1"), Row(1))
+      // float(3.14) is casted to double(3.140000104904175)
+      checkAnswer(df.where("c3 >= 3.14000010").select("c1"), Row(1))
+      checkAnswer(df.where("c3 == 3.14").select("c1"), Seq.empty)
+      checkAnswer(df.where("c3 <=> 3.14").select("c1"), Seq.empty)
+      checkAnswer(df.where("c3 < 3.14000010").select("c1"), Seq.empty)
+      checkAnswer(df.where("c3 <= 3.14").select("c1"), Seq.empty)
+
+      checkAnswer(df.where("c4 > cast(200.1199 as decimal(10, 4))").select("c1"), Row(1))
+      checkAnswer(df.where("c4 >= cast(200.1201 as decimal(10, 4))").select("c1"), Seq.empty)
+      checkAnswer(df.where("c4 == cast(200.1156 as decimal(10, 4))").select("c1"), Seq.empty)
+      checkAnswer(df.where("c4 <=> cast(200.1201 as decimal(10, 4))").select("c1"), Seq.empty)
+      checkAnswer(df.where("c4 <= cast(200.1201 as decimal(10, 4))").select("c1"), Row(1))
+      checkAnswer(df.where("c4 < cast(200.1159 as decimal(10, 4))").select("c1"), Seq.empty)
+    }
+  }
+
+  private def decimal(v: BigDecimal): Decimal = Decimal(v, 5, 2)
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UpdateFieldsBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/UpdateFieldsBenchmark.scala
new file mode 100644
index 0000000000000..13221c9c79c86
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UpdateFieldsBenchmark.scala
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import org.apache.spark.benchmark.Benchmark
+import org.apache.spark.sql.execution.benchmark.SqlBasedBenchmark
+import org.apache.spark.sql.functions.{col, lit}
+import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
+
+/**
+ * Benchmark to measure Spark's performance analyzing and optimizing long UpdateFields chains.
+ *
+ * {{{
+ *   To run this benchmark:
+ *   1. without sbt:
+ *      bin/spark-submit --class <this class>
+ *        --jars <spark core test jar>,<spark catalyst test jar> <spark sql test jar>
+ *   2. with sbt:
+ *      build/sbt "sql/test:runMain <this class>"
+ *   3. generate result:
+ *      SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
+ *   Results will be written to "benchmarks/UpdateFieldsBenchmark-results.txt".
+ * }}}
+ */
+object UpdateFieldsBenchmark extends SqlBasedBenchmark {
+
+  def nestedColName(d: Int, colNum: Int): String = s"nested${d}Col$colNum"
+
+  def nestedStructType(
+      colNums: Seq[Int],
+      nullable: Boolean,
+      maxDepth: Int,
+      currDepth: Int = 1): StructType = {
+
+    if (currDepth == maxDepth) {
+      val fields = colNums.map { colNum =>
+        val name = nestedColName(currDepth, colNum)
+        StructField(name, IntegerType, nullable = false)
+      }
+      StructType(fields)
+    } else {
+      val fields = colNums.foldLeft(Seq.empty[StructField]) {
+        case (structFields, colNum) if colNum == 0 =>
+          val nested = nestedStructType(colNums, nullable, maxDepth, currDepth + 1)
+          structFields :+ StructField(nestedColName(currDepth, colNum), nested, nullable)
+        case (structFields, colNum) =>
+          val name = nestedColName(currDepth, colNum)
+          structFields :+ StructField(name, IntegerType, nullable = false)
+      }
+      StructType(fields)
+    }
+  }
+
+  /**
+   * Utility function for generating an empty DataFrame with nested columns.
+   *
+   * @param maxDepth: The depth to which to create nested columns.
+   * @param numColsAtEachDepth: The number of columns to create at each depth.
+   * @param nullable: This value is used to set the nullability of any StructType columns.
+   */
+  def emptyNestedDf(maxDepth: Int, numColsAtEachDepth: Int, nullable: Boolean): DataFrame = {
+    require(maxDepth > 0)
+    require(numColsAtEachDepth > 0)
+
+    val nestedColumnDataType = nestedStructType(0 until numColsAtEachDepth, nullable, maxDepth)
+    spark.createDataFrame(
+      spark.sparkContext.emptyRDD[Row],
+      StructType(Seq(StructField(nestedColName(0, 0), nestedColumnDataType, nullable))))
+  }
+
+  trait ModifyNestedColumns {
+    val name: String
+    def apply(column: Column, numsToAdd: Seq[Int], numsToDrop: Seq[Int], maxDepth: Int): Column
+  }
+
+  object Performant extends ModifyNestedColumns {
+    override val name: String = "performant"
+
+    override def apply(
+        column: Column,
+        numsToAdd: Seq[Int],
+        numsToDrop: Seq[Int],
+        maxDepth: Int): Column = helper(column, numsToAdd, numsToDrop, maxDepth, 1)
+
+    private def helper(
+        column: Column,
+        numsToAdd: Seq[Int],
+        numsToDrop: Seq[Int],
+        maxDepth: Int,
+        currDepth: Int): Column = {
+
+      // drop columns at the current depth
+      val dropped = if (numsToDrop.nonEmpty) {
+        column.dropFields(numsToDrop.map(num => nestedColName(currDepth, num)): _*)
+      } else column
+
+      // add columns at the current depth
+      val added = numsToAdd.foldLeft(dropped) {
+        (res, num) => res.withField(nestedColName(currDepth, num), lit(num))
+      }
+
+      if (currDepth == maxDepth) {
+        added
+      } else {
+        // add/drop columns at the next depth
+        val newValue = helper(
+          column = col((0 to currDepth).map(d => nestedColName(d, 0)).mkString(".")),
+          numsToAdd = numsToAdd,
+          numsToDrop = numsToDrop,
+          currDepth = currDepth + 1,
+          maxDepth = maxDepth)
+        added.withField(nestedColName(currDepth, 0), newValue)
+      }
+    }
+  }
+
+  object NonPerformant extends ModifyNestedColumns {
+    override val name: String = "non-performant"
+
+    override def apply(
+        column: Column,
+        numsToAdd: Seq[Int],
+        numsToDrop: Seq[Int],
+        maxDepth: Int): Column = {
+
+      val dropped = if (numsToDrop.nonEmpty) {
+        val colsToDrop = (1 to maxDepth).flatMap { depth =>
+          numsToDrop.map(num => s"${prefix(depth)}${nestedColName(depth, num)}")
+        }
+        column.dropFields(colsToDrop: _*)
+      } else column
+
+      val added = {
+        val colsToAdd = (1 to maxDepth).flatMap { depth =>
+          numsToAdd.map(num => (s"${prefix(depth)}${nestedColName(depth, num)}", lit(num)))
+        }
+        colsToAdd.foldLeft(dropped)((col, add) => col.withField(add._1, add._2))
+      }
+
+      added
+    }
+
+    private def prefix(depth: Int): String =
+      if (depth == 1) ""
+      else (1 until depth).map(d => nestedColName(d, 0)).mkString("", ".", ".")
+  }
+
+  private def updateFieldsBenchmark(
+      methods: Seq[ModifyNestedColumns],
+      maxDepth: Int,
+      initialNumberOfColumns: Int,
+      numsToAdd: Seq[Int] = Seq.empty,
+      numsToDrop: Seq[Int] = Seq.empty): Unit = {
+
+    val name = s"Add ${numsToAdd.length} columns and drop ${numsToDrop.length} columns " +
+      s"at $maxDepth different depths of nesting"
+
+    runBenchmark(name) {
+      val benchmark = new Benchmark(
+        name = name,
+        // The purpose of this benchmark is to ensure Spark is able to analyze and optimize long
+        // UpdateFields chains quickly so it runs over 0 rows of data.
+        valuesPerIteration = 0,
+        output = output)
+
+      val nonNullableStructsDf = emptyNestedDf(maxDepth, initialNumberOfColumns, nullable = false)
+      val nullableStructsDf = emptyNestedDf(maxDepth, initialNumberOfColumns, nullable = true)
+
+      methods.foreach { method =>
+        val modifiedColumn = method(
+          column = col(nestedColName(0, 0)),
+          numsToAdd = numsToAdd,
+          numsToDrop = numsToDrop,
+          maxDepth = maxDepth
+        ).as(nestedColName(0, 0))
+
+        benchmark.addCase(s"To non-nullable StructTypes using ${method.name} method") { _ =>
+          nonNullableStructsDf.select(modifiedColumn).queryExecution.optimizedPlan
+        }
+
+        benchmark.addCase(s"To nullable StructTypes using ${method.name} method") { _ =>
+          nullableStructsDf.select(modifiedColumn).queryExecution.optimizedPlan
+        }
+      }
+
+      benchmark.run()
+    }
+  }
+
+  override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
+    // This benchmark compares the performant and non-performant methods of writing the same query.
+    // We use small values for maxDepth, numsToAdd, and numsToDrop because the NonPerformant method
+    // scales extremely poorly with the number of nested columns being added/dropped.
+    updateFieldsBenchmark(
+      methods = Seq(Performant, NonPerformant),
+      maxDepth = 3,
+      initialNumberOfColumns = 5,
+      numsToAdd = 5 to 6,
+      numsToDrop = 3 to 4)
+
+    // This benchmark is to show that the performant method of writing a query when we want to add
+    // and drop a large number of nested columns scales nicely.
+    updateFieldsBenchmark(
+      methods = Seq(Performant),
+      maxDepth = 100,
+      initialNumberOfColumns = 51,
+      numsToAdd = 51 to 100,
+      numsToDrop = 1 to 50)
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
index ed8ab1cb3a603..b6ab60a91955d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
@@ -21,8 +21,8 @@ import java.time.{LocalDateTime, ZoneOffset}
 import java.util.Arrays
 
 import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
-import org.apache.spark.sql.catalyst.expressions.{Cast, ExpressionEvalHelper, GenericInternalRow, Literal}
+import org.apache.spark.sql.catalyst.CatalystTypeConverters
+import org.apache.spark.sql.catalyst.expressions.{Cast, ExpressionEvalHelper, Literal}
 import org.apache.spark.sql.execution.datasources.parquet.ParquetTest
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.test.SharedSparkSession
@@ -33,77 +33,6 @@ private[sql] case class MyLabeledPoint(label: Double, features: TestUDT.MyDenseV
   def getFeatures: TestUDT.MyDenseVector = features
 }
 
-// object and classes to test SPARK-19311
-
-// Trait/Interface for base type
-sealed trait IExampleBaseType extends Serializable {
-  def field: Int
-}
-
-// Trait/Interface for derived type
-sealed trait IExampleSubType extends IExampleBaseType
-
-// a base class
-class ExampleBaseClass(override val field: Int) extends IExampleBaseType
-
-// a derived class
-class ExampleSubClass(override val field: Int)
-  extends ExampleBaseClass(field) with IExampleSubType
-
-// UDT for base class
-class ExampleBaseTypeUDT extends UserDefinedType[IExampleBaseType] {
-
-  override def sqlType: StructType = {
-    StructType(Seq(
-      StructField("intfield", IntegerType, nullable = false)))
-  }
-
-  override def serialize(obj: IExampleBaseType): InternalRow = {
-    val row = new GenericInternalRow(1)
-    row.setInt(0, obj.field)
-    row
-  }
-
-  override def deserialize(datum: Any): IExampleBaseType = {
-    datum match {
-      case row: InternalRow =>
-        require(row.numFields == 1,
-          "ExampleBaseTypeUDT requires row with length == 1")
-        val field = row.getInt(0)
-        new ExampleBaseClass(field)
-    }
-  }
-
-  override def userClass: Class[IExampleBaseType] = classOf[IExampleBaseType]
-}
-
-// UDT for derived class
-private[spark] class ExampleSubTypeUDT extends UserDefinedType[IExampleSubType] {
-
-  override def sqlType: StructType = {
-    StructType(Seq(
-      StructField("intfield", IntegerType, nullable = false)))
-  }
-
-  override def serialize(obj: IExampleSubType): InternalRow = {
-    val row = new GenericInternalRow(1)
-    row.setInt(0, obj.field)
-    row
-  }
-
-  override def deserialize(datum: Any): IExampleSubType = {
-    datum match {
-      case row: InternalRow =>
-        require(row.numFields == 1,
-          "ExampleSubTypeUDT requires row with length == 1")
-        val field = row.getInt(0)
-        new ExampleSubClass(field)
-    }
-  }
-
-  override def userClass: Class[IExampleSubType] = classOf[IExampleSubType]
-}
-
 private[sql] case class FooWithDate(date: LocalDateTime, s: String, i: Int)
 
 private[sql] class LocalDateTimeUDT extends UserDefinedType[LocalDateTime] {
@@ -134,6 +63,24 @@ class UserDefinedTypeSuite extends QueryTest with SharedSparkSession with Parque
     MyLabeledPoint(1.0, new TestUDT.MyDenseVector(Array(0.1, 1.0))),
     MyLabeledPoint(0.0, new TestUDT.MyDenseVector(Array(0.3, 3.0)))).toDF()
 
+
+  test("SPARK-32090: equal") {
+    val udt1 = new ExampleBaseTypeUDT
+    val udt2 = new ExampleSubTypeUDT
+    val udt3 = new ExampleSubTypeUDT
+    assert(udt1 !== udt2)
+    assert(udt2 !== udt1)
+    assert(udt2 === udt3)
+    assert(udt3 === udt2)
+  }
+
+  test("SPARK-32090: acceptsType") {
+    val udt1 = new ExampleBaseTypeUDT
+    val udt2 = new ExampleSubTypeUDT
+    assert(udt1.acceptsType(udt2))
+    assert(!udt2.acceptsType(udt1))
+  }
+
   test("register user type: MyDenseVector for MyLabeledPoint") {
     val labels: RDD[Double] = pointsRDD.select('label).rdd.map { case Row(v: Double) => v }
     val labelsArrays: Array[Double] = labels.collect()
@@ -275,11 +222,11 @@ class UserDefinedTypeSuite extends QueryTest with SharedSparkSession with Parque
 
     // this worked already before the fix SPARK-19311:
     // return type of doUDF equals parameter type of doOtherUDF
-    sql("SELECT doOtherUDF(doUDF(41))")
+    checkAnswer(sql("SELECT doOtherUDF(doUDF(41))"), Row(41) :: Nil)
 
     // this one passes only with the fix SPARK-19311:
     // return type of doSubUDF is a subtype of the parameter type of doOtherUDF
-    sql("SELECT doOtherUDF(doSubTypeUDF(42))")
+    checkAnswer(sql("SELECT doOtherUDF(doSubTypeUDF(42))"), Row(42) :: Nil)
   }
 
   test("except on UDT") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSessionCatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSessionCatalogSuite.scala
index 6b25d7c61663c..91ac7db335cc3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSessionCatalogSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSessionCatalogSuite.scala
@@ -23,14 +23,14 @@ import org.scalatest.BeforeAndAfter
 
 import org.apache.spark.sql.{DataFrame, QueryTest, SaveMode}
 import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.{NoSuchTableException, TableAlreadyExistsException}
+import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
 import org.apache.spark.sql.connector.catalog._
 import org.apache.spark.sql.connector.catalog.CatalogManager.SESSION_CATALOG_NAME
 import org.apache.spark.sql.connector.expressions.Transform
+import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.internal.SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.StructType
-import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
 class DataSourceV2DataFrameSessionCatalogSuite
   extends InsertIntoTests(supportsDynamicOverwrite = true, includeSQLOnlyTests = false)
@@ -126,7 +126,7 @@ class InMemoryTableSessionCatalog extends TestV2SessionCatalogBase[InMemoryTable
 
         newTable
       case _ =>
-        throw new NoSuchTableException(ident)
+        throw QueryCompilationErrors.noSuchTableError(ident)
     }
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala
index 7c7afa9cfbd41..d83d1a2755928 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala
@@ -22,8 +22,9 @@ import java.util.Collections
 import org.apache.spark.sql.{AnalysisException, DataFrame, Row, SaveMode}
 import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
 import org.apache.spark.sql.catalyst.plans.logical.{AppendData, LogicalPlan}
-import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.catalog.{Identifier, InMemoryTableCatalog}
 import org.apache.spark.sql.execution.QueryExecution
+import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.QueryExecutionListener
 
@@ -186,4 +187,21 @@ class DataSourceV2DataFrameSuite
       assert(e3.getMessage.contains(s"Cannot use interval type in the table schema."))
     }
   }
+
+  test("options to scan v2 table should be passed to DataSourceV2Relation") {
+    val t1 = "testcat.ns1.ns2.tbl"
+    withTable(t1) {
+      val df1 = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("id", "data")
+      df1.write.saveAsTable(t1)
+
+      val optionName = "fakeOption"
+      val df2 = spark.read
+        .option(optionName, false)
+        .table(t1)
+      val options = df2.queryExecution.analyzed.collectFirst {
+        case d: DataSourceV2Relation => d.options
+      }.get
+      assert(options.get(optionName) === "false")
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2FunctionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2FunctionSuite.scala
new file mode 100644
index 0000000000000..b269da39daf38
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2FunctionSuite.scala
@@ -0,0 +1,442 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector
+
+import java.util
+import java.util.Collections
+
+import test.org.apache.spark.sql.connector.catalog.functions.{JavaAverage, JavaStrLen}
+import test.org.apache.spark.sql.connector.catalog.functions.JavaStrLen._
+
+import org.apache.spark.SparkException
+import org.apache.spark.sql.{AnalysisException, Row}
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.connector.catalog.{BasicInMemoryTableCatalog, Identifier, InMemoryCatalog, SupportsNamespaces}
+import org.apache.spark.sql.connector.catalog.functions._
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.UTF8String
+
+class DataSourceV2FunctionSuite extends DatasourceV2SQLBase {
+  private val emptyProps: util.Map[String, String] = Collections.emptyMap[String, String]
+
+  private def addFunction(ident: Identifier, fn: UnboundFunction): Unit = {
+    catalog("testcat").asInstanceOf[InMemoryCatalog].createFunction(ident, fn)
+  }
+
+  test("undefined function") {
+    assert(intercept[AnalysisException](
+      sql("SELECT testcat.non_exist('abc')").collect()
+    ).getMessage.contains("Undefined function"))
+  }
+
+  test("non-function catalog") {
+    withSQLConf("spark.sql.catalog.testcat" -> classOf[BasicInMemoryTableCatalog].getName) {
+      assert(intercept[AnalysisException](
+        sql("SELECT testcat.strlen('abc')").collect()
+      ).getMessage.contains("is not a FunctionCatalog"))
+    }
+  }
+
+  test("built-in with non-function catalog should still work") {
+    withSQLConf(SQLConf.DEFAULT_CATALOG.key -> "testcat",
+      "spark.sql.catalog.testcat" -> classOf[BasicInMemoryTableCatalog].getName) {
+      checkAnswer(sql("SELECT length('abc')"), Row(3))
+    }
+  }
+
+  test("built-in with default v2 function catalog") {
+    withSQLConf(SQLConf.DEFAULT_CATALOG.key -> "testcat") {
+      checkAnswer(sql("SELECT length('abc')"), Row(3))
+    }
+  }
+
+  test("looking up higher-order function with non-session catalog") {
+    checkAnswer(sql("SELECT transform(array(1, 2, 3), x -> x + 1)"),
+      Row(Array(2, 3, 4)) :: Nil)
+  }
+
+  test("built-in override with default v2 function catalog") {
+    // a built-in function with the same name should take higher priority
+    withSQLConf(SQLConf.DEFAULT_CATALOG.key -> "testcat") {
+      addFunction(Identifier.of(Array.empty, "length"), new JavaStrLen(new JavaStrLenNoImpl))
+      checkAnswer(sql("SELECT length('abc')"), Row(3))
+    }
+  }
+
+  test("built-in override with non-session catalog") {
+    addFunction(Identifier.of(Array.empty, "length"), new JavaStrLen(new JavaStrLenNoImpl))
+    checkAnswer(sql("SELECT length('abc')"), Row(3))
+  }
+
+  test("temp function override with default v2 function catalog") {
+    val className = "test.org.apache.spark.sql.JavaStringLength"
+    sql(s"CREATE FUNCTION length AS '$className'")
+
+    withSQLConf(SQLConf.DEFAULT_CATALOG.key -> "testcat") {
+      addFunction(Identifier.of(Array.empty, "length"), new JavaStrLen(new JavaStrLenNoImpl))
+      checkAnswer(sql("SELECT length('abc')"), Row(3))
+    }
+  }
+
+  test("view should use captured catalog and namespace for function lookup") {
+    val viewName = "my_view"
+    withView(viewName) {
+      withSQLConf(SQLConf.DEFAULT_CATALOG.key -> "testcat") {
+        catalog("testcat").asInstanceOf[SupportsNamespaces].createNamespace(Array("ns"), emptyProps)
+        addFunction(Identifier.of(Array("ns"), "my_avg"), IntegralAverage)
+        sql("USE ns")
+        sql(s"CREATE TEMPORARY VIEW $viewName AS SELECT my_avg(col1) FROM values (1), (2), (3)")
+      }
+
+      // change default catalog and namespace and add a function with the same name but with no
+      // implementation
+      withSQLConf(SQLConf.DEFAULT_CATALOG.key -> "testcat2") {
+        catalog("testcat2").asInstanceOf[SupportsNamespaces]
+          .createNamespace(Array("ns2"), emptyProps)
+        addFunction(Identifier.of(Array("ns2"), "my_avg"), NoImplAverage)
+        sql("USE ns2")
+        checkAnswer(sql(s"SELECT * FROM $viewName"), Row(2.0) :: Nil)
+      }
+    }
+  }
+
+  test("scalar function: with default produceResult method") {
+    catalog("testcat").asInstanceOf[SupportsNamespaces].createNamespace(Array("ns"), emptyProps)
+    addFunction(Identifier.of(Array("ns"), "strlen"), StrLen(StrLenDefault))
+    checkAnswer(sql("SELECT testcat.ns.strlen('abc')"), Row(3) :: Nil)
+  }
+
+  test("scalar function: with default produceResult method w/ expression") {
+    catalog("testcat").asInstanceOf[SupportsNamespaces].createNamespace(Array("ns"), emptyProps)
+    addFunction(Identifier.of(Array("ns"), "strlen"), StrLen(StrLenDefault))
+    checkAnswer(sql("SELECT testcat.ns.strlen(substr('abcde', 3))"), Row(3) :: Nil)
+  }
+
+  test("scalar function: lookup magic method") {
+    catalog("testcat").asInstanceOf[SupportsNamespaces].createNamespace(Array("ns"), emptyProps)
+    addFunction(Identifier.of(Array("ns"), "strlen"), StrLen(StrLenMagic))
+    checkAnswer(sql("SELECT testcat.ns.strlen('abc')"), Row(3) :: Nil)
+  }
+
+  test("scalar function: lookup magic method w/ expression") {
+    catalog("testcat").asInstanceOf[SupportsNamespaces].createNamespace(Array("ns"), emptyProps)
+    addFunction(Identifier.of(Array("ns"), "strlen"), StrLen(StrLenMagic))
+    checkAnswer(sql("SELECT testcat.ns.strlen(substr('abcde', 3))"), Row(3) :: Nil)
+  }
+
+  test("scalar function: bad magic method") {
+    catalog("testcat").asInstanceOf[SupportsNamespaces].createNamespace(Array("ns"), emptyProps)
+    addFunction(Identifier.of(Array("ns"), "strlen"), StrLen(StrLenBadMagic))
+    assert(intercept[SparkException](sql("SELECT testcat.ns.strlen('abc')").collect())
+      .getMessage.contains("Cannot find a compatible"))
+  }
+
+  test("scalar function: bad magic method with default impl") {
+    catalog("testcat").asInstanceOf[SupportsNamespaces].createNamespace(Array("ns"), emptyProps)
+    addFunction(Identifier.of(Array("ns"), "strlen"), StrLen(StrLenBadMagicWithDefault))
+    checkAnswer(sql("SELECT testcat.ns.strlen('abc')"), Row(3) :: Nil)
+  }
+
+  test("scalar function: no implementation found") {
+    catalog("testcat").asInstanceOf[SupportsNamespaces].createNamespace(Array("ns"), emptyProps)
+    addFunction(Identifier.of(Array("ns"), "strlen"), StrLen(StrLenNoImpl))
+    intercept[SparkException](sql("SELECT testcat.ns.strlen('abc')").collect())
+  }
+
+  test("scalar function: invalid parameter type or length") {
+    catalog("testcat").asInstanceOf[SupportsNamespaces].createNamespace(Array("ns"), emptyProps)
+    addFunction(Identifier.of(Array("ns"), "strlen"), StrLen(StrLenDefault))
+
+    assert(intercept[AnalysisException](sql("SELECT testcat.ns.strlen(42)"))
+      .getMessage.contains("Expect StringType"))
+    assert(intercept[AnalysisException](sql("SELECT testcat.ns.strlen('a', 'b')"))
+      .getMessage.contains("Expect exactly one argument"))
+  }
+
+  test("scalar function: default produceResult in Java") {
+    catalog("testcat").asInstanceOf[SupportsNamespaces].createNamespace(Array("ns"), emptyProps)
+    addFunction(Identifier.of(Array("ns"), "strlen"),
+      new JavaStrLen(new JavaStrLenDefault))
+    checkAnswer(sql("SELECT testcat.ns.strlen('abc')"), Row(3) :: Nil)
+  }
+
+  test("scalar function: magic method in Java") {
+    catalog("testcat").asInstanceOf[SupportsNamespaces].createNamespace(Array("ns"), emptyProps)
+    addFunction(Identifier.of(Array("ns"), "strlen"),
+      new JavaStrLen(new JavaStrLenMagic))
+    checkAnswer(sql("SELECT testcat.ns.strlen('abc')"), Row(3) :: Nil)
+  }
+
+  test("scalar function: static magic method in Java") {
+    catalog("testcat").asInstanceOf[SupportsNamespaces].createNamespace(Array("ns"), emptyProps)
+    addFunction(Identifier.of(Array("ns"), "strlen"),
+      new JavaStrLen(new JavaStrLenStaticMagic))
+    checkAnswer(sql("SELECT testcat.ns.strlen('abc')"), Row(3) :: Nil)
+  }
+
+  test("scalar function: magic method should take higher precedence in Java") {
+    catalog("testcat").asInstanceOf[SupportsNamespaces].createNamespace(Array("ns"), emptyProps)
+    addFunction(Identifier.of(Array("ns"), "strlen"),
+      new JavaStrLen(new JavaStrLenBoth))
+    // to differentiate, the static method returns string length + 100
+    checkAnswer(sql("SELECT testcat.ns.strlen('abc')"), Row(103) :: Nil)
+  }
+
+  test("scalar function: bad static magic method should fallback to non-static") {
+    catalog("testcat").asInstanceOf[SupportsNamespaces].createNamespace(Array("ns"), emptyProps)
+    addFunction(Identifier.of(Array("ns"), "strlen"),
+      new JavaStrLen(new JavaStrLenBadStaticMagic))
+    checkAnswer(sql("SELECT testcat.ns.strlen('abc')"), Row(103) :: Nil)
+  }
+
+  test("scalar function: no implementation found in Java") {
+    catalog("testcat").asInstanceOf[SupportsNamespaces].createNamespace(Array("ns"), emptyProps)
+    addFunction(Identifier.of(Array("ns"), "strlen"),
+      new JavaStrLen(new JavaStrLenNoImpl))
+    assert(intercept[AnalysisException](sql("SELECT testcat.ns.strlen('abc')").collect())
+      .getMessage.contains("neither implement magic method nor override 'produceResult'"))
+  }
+
+  test("bad bound function (neither scalar nor aggregate)") {
+    catalog("testcat").asInstanceOf[SupportsNamespaces].createNamespace(Array("ns"), emptyProps)
+    addFunction(Identifier.of(Array("ns"), "strlen"), StrLen(BadBoundFunction))
+
+    assert(intercept[AnalysisException](sql("SELECT testcat.ns.strlen('abc')"))
+      .getMessage.contains("does not implement ScalarFunction or AggregateFunction"))
+  }
+
+  test("aggregate function: lookup int average") {
+    import testImplicits._
+    val t = "testcat.ns.t"
+    withTable(t) {
+      addFunction(Identifier.of(Array("ns"), "avg"), IntegralAverage)
+
+      (1 to 100).toDF("i").write.saveAsTable(t)
+      checkAnswer(sql(s"SELECT testcat.ns.avg(i) from $t"), Row(50) :: Nil)
+    }
+  }
+
+  test("aggregate function: lookup long average") {
+    import testImplicits._
+    val t = "testcat.ns.t"
+    withTable(t) {
+      addFunction(Identifier.of(Array("ns"), "avg"), IntegralAverage)
+
+      (1L to 100L).toDF("i").write.saveAsTable(t)
+      checkAnswer(sql(s"SELECT testcat.ns.avg(i) from $t"), Row(50) :: Nil)
+    }
+  }
+
+  test("aggregate function: lookup double average in Java") {
+    import testImplicits._
+    val t = "testcat.ns.t"
+    withTable(t) {
+      addFunction(Identifier.of(Array("ns"), "avg"), new JavaAverage)
+
+      Seq(1.toDouble, 2.toDouble, 3.toDouble).toDF("i").write.saveAsTable(t)
+      checkAnswer(sql(s"SELECT testcat.ns.avg(i) from $t"), Row(2.0) :: Nil)
+    }
+  }
+
+  test("aggregate function: lookup int average w/ expression") {
+    import testImplicits._
+    val t = "testcat.ns.t"
+    withTable(t) {
+      addFunction(Identifier.of(Array("ns"), "avg"), IntegralAverage)
+
+      (1 to 100).toDF("i").write.saveAsTable(t)
+      checkAnswer(sql(s"SELECT testcat.ns.avg(i * 10) from $t"), Row(505) :: Nil)
+    }
+  }
+
+  test("aggregate function: unsupported input type") {
+    import testImplicits._
+    val t = "testcat.ns.t"
+    withTable(t) {
+      addFunction(Identifier.of(Array("ns"), "avg"), IntegralAverage)
+
+      Seq(1.toShort, 2.toShort).toDF("i").write.saveAsTable(t)
+      assert(intercept[AnalysisException](sql(s"SELECT testcat.ns.avg(i) from $t"))
+        .getMessage.contains("Unsupported non-integral type: ShortType"))
+    }
+  }
+
+  private case class StrLen(impl: BoundFunction) extends UnboundFunction {
+    override def description(): String =
+      """strlen: returns the length of the input string
+        |  strlen(string) -> int""".stripMargin
+    override def name(): String = "strlen"
+
+    override def bind(inputType: StructType): BoundFunction = {
+      if (inputType.fields.length != 1) {
+        throw new UnsupportedOperationException("Expect exactly one argument");
+      }
+      inputType.fields(0).dataType match {
+        case StringType => impl
+        case _ =>
+          throw new UnsupportedOperationException("Expect StringType")
+      }
+    }
+  }
+
+  private case object StrLenDefault extends ScalarFunction[Int] {
+    override def inputTypes(): Array[DataType] = Array(StringType)
+    override def resultType(): DataType = IntegerType
+    override def name(): String = "strlen_default"
+
+    override def produceResult(input: InternalRow): Int = {
+      val s = input.getString(0)
+      s.length
+    }
+  }
+
+  private case object StrLenMagic extends ScalarFunction[Int] {
+    override def inputTypes(): Array[DataType] = Array(StringType)
+    override def resultType(): DataType = IntegerType
+    override def name(): String = "strlen_magic"
+
+    def invoke(input: UTF8String): Int = {
+      input.toString.length
+    }
+  }
+
+  private case object StrLenBadMagic extends ScalarFunction[Int] {
+    override def inputTypes(): Array[DataType] = Array(StringType)
+    override def resultType(): DataType = IntegerType
+    override def name(): String = "strlen_bad_magic"
+
+    def invoke(input: String): Int = {
+      input.length
+    }
+  }
+
+  private case object StrLenBadMagicWithDefault extends ScalarFunction[Int] {
+    override def inputTypes(): Array[DataType] = Array(StringType)
+    override def resultType(): DataType = IntegerType
+    override def name(): String = "strlen_bad_magic"
+
+    def invoke(input: String): Int = {
+      input.length
+    }
+
+    override def produceResult(input: InternalRow): Int = {
+      val s = input.getString(0)
+      s.length
+    }
+  }
+
+  private case object StrLenNoImpl extends ScalarFunction[Int] {
+    override def inputTypes(): Array[DataType] = Array(StringType)
+    override def resultType(): DataType = IntegerType
+    override def name(): String = "strlen_noimpl"
+  }
+
+  private case object BadBoundFunction extends BoundFunction {
+    override def inputTypes(): Array[DataType] = Array(StringType)
+    override def resultType(): DataType = IntegerType
+    override def name(): String = "bad_bound_func"
+  }
+
+  object IntegralAverage extends UnboundFunction {
+    override def name(): String = "iavg"
+
+    override def bind(inputType: StructType): BoundFunction = {
+      if (inputType.fields.length > 1) {
+        throw new UnsupportedOperationException("Too many arguments")
+      }
+
+      inputType.fields(0).dataType match {
+        case _: IntegerType => IntAverage
+        case _: LongType => LongAverage
+        case dataType =>
+          throw new UnsupportedOperationException(s"Unsupported non-integral type: $dataType")
+      }
+    }
+
+    override def description(): String =
+      """iavg: produces an average using integer division, ignoring nulls
+        |  iavg(int) -> int
+        |  iavg(bigint) -> bigint""".stripMargin
+  }
+
+  object IntAverage extends AggregateFunction[(Int, Int), Int] {
+    override def name(): String = "iavg"
+    override def inputTypes(): Array[DataType] = Array(IntegerType)
+    override def resultType(): DataType = IntegerType
+
+    override def newAggregationState(): (Int, Int) = (0, 0)
+
+    override def update(state: (Int, Int), input: InternalRow): (Int, Int) = {
+      if (input.isNullAt(0)) {
+        state
+      } else {
+        val i = input.getInt(0)
+        state match {
+          case (_, 0) =>
+            (i, 1)
+          case (total, count) =>
+            (total + i, count + 1)
+        }
+      }
+    }
+
+    override def merge(leftState: (Int, Int), rightState: (Int, Int)): (Int, Int) = {
+      (leftState._1 + rightState._1, leftState._2 + rightState._2)
+    }
+
+    override def produceResult(state: (Int, Int)): Int = state._1 / state._2
+  }
+
+  object LongAverage extends AggregateFunction[(Long, Long), Long] {
+    override def name(): String = "iavg"
+    override def inputTypes(): Array[DataType] = Array(LongType)
+    override def resultType(): DataType = LongType
+
+    override def newAggregationState(): (Long, Long) = (0L, 0L)
+
+    override def update(state: (Long, Long), input: InternalRow): (Long, Long) = {
+      if (input.isNullAt(0)) {
+        state
+      } else {
+        val l = input.getLong(0)
+        state match {
+          case (_, 0L) =>
+            (l, 1)
+          case (total, count) =>
+            (total + l, count + 1L)
+        }
+      }
+    }
+
+    override def merge(leftState: (Long, Long), rightState: (Long, Long)): (Long, Long) = {
+      (leftState._1 + rightState._1, leftState._2 + rightState._2)
+    }
+
+    override def produceResult(state: (Long, Long)): Long = state._1 / state._2
+  }
+
+  object NoImplAverage extends UnboundFunction {
+    override def name(): String = "no_impl_avg"
+    override def description(): String = name()
+
+    override def bind(inputType: StructType): BoundFunction = {
+      throw new UnsupportedOperationException(s"Not implemented")
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala
index cf00b3b5e4410..44fbc639a5803 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.connector
 
 import org.apache.spark.sql.{DataFrame, Row, SaveMode}
-import org.apache.spark.sql.connector.catalog.{Identifier, Table, TableCatalog}
+import org.apache.spark.sql.connector.catalog.{Identifier, InMemoryTable, Table, TableCatalog}
 
 class DataSourceV2SQLSessionCatalogSuite
   extends InsertIntoTests(supportsDynamicOverwrite = true, includeSQLOnlyTests = true)
@@ -79,4 +79,12 @@ class DataSourceV2SQLSessionCatalogSuite
         Row("keyX", s"Table default.$t1 does not have property: keyX"))
     }
   }
+
+  test("SPARK-33651: allow CREATE EXTERNAL TABLE without LOCATION") {
+    withTable("t") {
+      val prop = TestV2SessionCatalogBase.SIMULATE_ALLOW_EXTERNAL_PROPERTY + "=true"
+      // The following should not throw AnalysisException.
+      sql(s"CREATE EXTERNAL TABLE t (i INT) USING $v2Format TBLPROPERTIES($prop)")
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
index 8462ce5a6c44f..13facc36876b2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
@@ -17,26 +17,33 @@
 
 package org.apache.spark.sql.connector
 
+import java.sql.Timestamp
+import java.time.LocalDate
+
 import scala.collection.JavaConverters._
 
 import org.apache.spark.SparkException
 import org.apache.spark.sql._
-import org.apache.spark.sql.catalyst.analysis.{CannotReplaceMissingTableException, NamespaceAlreadyExistsException, NoSuchDatabaseException, NoSuchNamespaceException, NoSuchTableException, TableAlreadyExistsException}
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.{CannotReplaceMissingTableException, NamespaceAlreadyExistsException, NoSuchDatabaseException, NoSuchNamespaceException, TableAlreadyExistsException}
 import org.apache.spark.sql.catalyst.parser.ParseException
 import org.apache.spark.sql.connector.catalog._
 import org.apache.spark.sql.connector.catalog.CatalogManager.SESSION_CATALOG_NAME
 import org.apache.spark.sql.connector.catalog.CatalogV2Util.withDefaultOwnership
+import org.apache.spark.sql.execution.columnar.InMemoryRelation
+import org.apache.spark.sql.execution.streaming.MemoryStream
 import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
-import org.apache.spark.sql.internal.SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION
+import org.apache.spark.sql.internal.SQLConf.{PARTITION_OVERWRITE_MODE, PartitionOverwriteMode, V2_SESSION_CATALOG_IMPLEMENTATION}
 import org.apache.spark.sql.internal.connector.SimpleTableProvider
 import org.apache.spark.sql.sources.SimpleScanSource
-import org.apache.spark.sql.types.{BooleanType, LongType, StringType, StructField, StructType}
+import org.apache.spark.sql.types.{BooleanType, LongType, MetadataBuilder, StringType, StructField, StructType}
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
+import org.apache.spark.unsafe.types.UTF8String
 import org.apache.spark.util.Utils
 
 class DataSourceV2SQLSuite
   extends InsertIntoTests(supportsDynamicOverwrite = true, includeSQLOnlyTests = true)
-  with AlterTableTests {
+  with AlterTableTests with DatasourceV2SQLBase {
 
   import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
 
@@ -45,10 +52,6 @@ class DataSourceV2SQLSuite
   override protected val catalogAndNamespace = "testcat.ns1.ns2."
   private val defaultUser: String = Utils.getCurrentUserName()
 
-  private def catalog(name: String): CatalogPlugin = {
-    spark.sessionState.catalogManager.catalog(name)
-  }
-
   protected def doInsert(tableName: String, insert: DataFrame, mode: SaveMode): Unit = {
     val tmpView = "tmp_view"
     withTempView(tmpView) {
@@ -69,26 +72,6 @@ class DataSourceV2SQLSuite
     v2Catalog.loadTable(Identifier.of(namespace, nameParts.last))
   }
 
-  before {
-    spark.conf.set("spark.sql.catalog.testcat", classOf[InMemoryTableCatalog].getName)
-    spark.conf.set(
-        "spark.sql.catalog.testcat_atomic", classOf[StagingInMemoryTableCatalog].getName)
-    spark.conf.set("spark.sql.catalog.testcat2", classOf[InMemoryTableCatalog].getName)
-    spark.conf.set(
-      V2_SESSION_CATALOG_IMPLEMENTATION.key, classOf[InMemoryTableSessionCatalog].getName)
-
-    val df = spark.createDataFrame(Seq((1L, "a"), (2L, "b"), (3L, "c"))).toDF("id", "data")
-    df.createOrReplaceTempView("source")
-    val df2 = spark.createDataFrame(Seq((4L, "d"), (5L, "e"), (6L, "f"))).toDF("id", "data")
-    df2.createOrReplaceTempView("source2")
-  }
-
-  after {
-    spark.sessionState.catalog.reset()
-    spark.sessionState.catalogManager.reset()
-    spark.sessionState.conf.clear()
-  }
-
   test("CreateTable: use v2 plan because catalog is set") {
     spark.sql("CREATE TABLE testcat.table_name (id bigint NOT NULL, data string) USING foo")
 
@@ -158,6 +141,10 @@ class DataSourceV2SQLSuite
       Array("# Partitioning", "", ""),
       Array("Part 0", "id", ""),
       Array("", "", ""),
+      Array("# Metadata Columns", "", ""),
+      Array("index", "int", "Metadata column used to conflict with a data column"),
+      Array("_partition", "string", "Partition key used to store the row"),
+      Array("", "", ""),
       Array("# Detailed Table Information", "", ""),
       Array("Name", "testcat.table_name", ""),
       Array("Comment", "this is a test table", ""),
@@ -165,7 +152,72 @@ class DataSourceV2SQLSuite
       Array("Provider", "foo", ""),
       Array(TableCatalog.PROP_OWNER.capitalize, defaultUser, ""),
       Array("Table Properties", "[bar=baz]", "")))
+  }
+
+  test("Describe column for v2 catalog") {
+    val t = "testcat.tbl"
+    withTable(t) {
+      sql(s"CREATE TABLE $t (id bigint, data string COMMENT 'hello') USING foo")
+      val df1 = sql(s"DESCRIBE $t id")
+      assert(df1.schema.map(field => (field.name, field.dataType))
+        === Seq(("info_name", StringType), ("info_value", StringType)))
+      assert(df1.collect === Seq(
+        Row("col_name", "id"),
+        Row("data_type", "bigint"),
+        Row("comment", "NULL")))
+      val df2 = sql(s"DESCRIBE $t data")
+      assert(df2.schema.map(field => (field.name, field.dataType))
+        === Seq(("info_name", StringType), ("info_value", StringType)))
+      assert(df2.collect === Seq(
+        Row("col_name", "data"),
+        Row("data_type", "string"),
+        Row("comment", "hello")))
 
+      assertAnalysisError(
+        s"DESCRIBE $t invalid_col",
+        "cannot resolve 'invalid_col' given input columns: [testcat.tbl.data, testcat.tbl.id]")
+    }
+  }
+
+  test("Describe column for v2 catalog should work with qualified columns") {
+    val t = "testcat.ns.tbl"
+    withTable(t) {
+      sql(s"CREATE TABLE $t (id bigint) USING foo")
+      Seq("testcat.ns.tbl.id", "ns.tbl.id", "tbl.id", "id").foreach { col =>
+        val df = sql(s"DESCRIBE $t $col")
+        assert(df.schema.map(field => (field.name, field.dataType))
+          === Seq(("info_name", StringType), ("info_value", StringType)))
+        assert(df.collect === Seq(
+          Row("col_name", "id"),
+          Row("data_type", "bigint"),
+          Row("comment", "NULL")))
+      }
+    }
+  }
+
+  test("Describing nested column for v2 catalog is not supported") {
+    val t = "testcat.tbl"
+    withTable(t) {
+      sql(s"CREATE TABLE $t (d struct<a: INT, b: INT>) USING foo")
+      assertAnalysisError(
+        s"describe $t d.a",
+        "DESC TABLE COLUMN does not support nested column")
+    }
+  }
+
+  test("SPARK-33004: Describe column should resolve to a temporary view first") {
+    withTable("testcat.ns.t") {
+      withTempView("t") {
+        sql("CREATE TABLE testcat.ns.t (id bigint) USING foo")
+        sql("CREATE TEMPORARY VIEW t AS SELECT 2 as i")
+        sql("USE testcat.ns")
+        checkAnswer(
+          sql("DESCRIBE t i"),
+          Seq(Row("col_name", "i"),
+            Row("data_type", "int"),
+            Row("comment", "NULL")))
+      }
+    }
   }
 
   test("CreateTable: use v2 plan and session catalog when provider is v2") {
@@ -256,25 +308,26 @@ class DataSourceV2SQLSuite
     checkAnswer(spark.internalCreateDataFrame(rdd, table.schema), Seq.empty)
   }
 
-  // TODO: ignored by SPARK-31707, restore the test after create table syntax unification
-  ignore("CreateTable: without USING clause") {
-    // unset this config to use the default v2 session catalog.
-    spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key)
-    val testCatalog = catalog("testcat").asTableCatalog
+  test("CreateTable: without USING clause") {
+    withSQLConf(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT.key -> "false") {
+      // unset this config to use the default v2 session catalog.
+      spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key)
+      val testCatalog = catalog("testcat").asTableCatalog
 
-    sql("CREATE TABLE testcat.t1 (id int)")
-    val t1 = testCatalog.loadTable(Identifier.of(Array(), "t1"))
-    // Spark shouldn't set the default provider for catalog plugins.
-    assert(!t1.properties.containsKey(TableCatalog.PROP_PROVIDER))
+      sql("CREATE TABLE testcat.t1 (id int)")
+      val t1 = testCatalog.loadTable(Identifier.of(Array(), "t1"))
+      // Spark shouldn't set the default provider for catalog plugins.
+      assert(!t1.properties.containsKey(TableCatalog.PROP_PROVIDER))
 
-    sql("CREATE TABLE t2 (id int)")
-    val t2 = spark.sessionState.catalogManager.v2SessionCatalog.asTableCatalog
-      .loadTable(Identifier.of(Array("default"), "t2")).asInstanceOf[V1Table]
-    // Spark should set the default provider as DEFAULT_DATA_SOURCE_NAME for the session catalog.
-    assert(t2.v1Table.provider == Some(conf.defaultDataSourceName))
+      sql("CREATE TABLE t2 (id int)")
+      val t2 = spark.sessionState.catalogManager.v2SessionCatalog.asTableCatalog
+        .loadTable(Identifier.of(Array("default"), "t2")).asInstanceOf[V1Table]
+      // Spark should set the default provider as DEFAULT_DATA_SOURCE_NAME for the session catalog.
+      assert(t2.v1Table.provider == Some(conf.defaultDataSourceName))
+    }
   }
 
-  test("CreateTable/RepalceTable: invalid schema if has interval type") {
+  test("CreateTable/ReplaceTable: invalid schema if has interval type") {
     Seq("CREATE", "REPLACE").foreach { action =>
       val e1 = intercept[AnalysisException](
         sql(s"$action TABLE table_name (id int, value interval) USING $v2Format"))
@@ -423,7 +476,7 @@ class DataSourceV2SQLSuite
 
     intercept[Exception] {
       spark.sql("REPLACE TABLE testcat.table_name" +
-        s" USING foo OPTIONS (`${InMemoryTable.SIMULATE_FAILED_WRITE_OPTION}`=true)" +
+        s" USING foo TBLPROPERTIES (`${InMemoryTable.SIMULATE_FAILED_WRITE_OPTION}`=true)" +
         s" AS SELECT id FROM source")
     }
 
@@ -456,7 +509,7 @@ class DataSourceV2SQLSuite
 
     intercept[Exception] {
       spark.sql("REPLACE TABLE testcat_atomic.table_name" +
-        s" USING foo OPTIONS (`${InMemoryTable.SIMULATE_FAILED_WRITE_OPTION}=true)" +
+        s" USING foo TBLPROPERTIES (`${InMemoryTable.SIMULATE_FAILED_WRITE_OPTION}=true)" +
         s" AS SELECT id FROM source")
     }
 
@@ -699,28 +752,128 @@ class DataSourceV2SQLSuite
     assert(t2.v1Table.provider == Some(conf.defaultDataSourceName))
   }
 
-  test("DropTable: basic") {
-    val tableName = "testcat.ns1.ns2.tbl"
-    val ident = Identifier.of(Array("ns1", "ns2"), "tbl")
-    sql(s"CREATE TABLE $tableName USING foo AS SELECT id, data FROM source")
-    assert(catalog("testcat").asTableCatalog.tableExists(ident) === true)
-    sql(s"DROP TABLE $tableName")
-    assert(catalog("testcat").asTableCatalog.tableExists(ident) === false)
+  test("SPARK-34039: ReplaceTable (atomic or non-atomic) should invalidate cache") {
+    Seq("testcat.ns.t", "testcat_atomic.ns.t").foreach { t =>
+      val view = "view"
+      withTable(t) {
+        withTempView(view) {
+          sql(s"CREATE TABLE $t USING foo AS SELECT id, data FROM source")
+          sql(s"CACHE TABLE $view AS SELECT id FROM $t")
+          checkAnswer(sql(s"SELECT * FROM $t"), spark.table("source"))
+          checkAnswer(sql(s"SELECT * FROM $view"), spark.table("source").select("id"))
+
+          val oldView = spark.table(view)
+          sql(s"REPLACE TABLE $t (a bigint) USING foo")
+          assert(spark.sharedState.cacheManager.lookupCachedData(oldView).isEmpty)
+        }
+      }
+    }
   }
 
-  test("DropTable: table qualified with the session catalog name") {
-    val ident = Identifier.of(Array("default"), "tbl")
-    sql("CREATE TABLE tbl USING json AS SELECT 1 AS i")
-    assert(catalog("spark_catalog").asTableCatalog.tableExists(ident) === true)
-    sql("DROP TABLE spark_catalog.default.tbl")
-    assert(catalog("spark_catalog").asTableCatalog.tableExists(ident) === false)
+  test("SPARK-33492: ReplaceTableAsSelect (atomic or non-atomic) should invalidate cache") {
+    Seq("testcat.ns.t", "testcat_atomic.ns.t").foreach { t =>
+      val view = "view"
+      withTable(t) {
+        withTempView(view) {
+          sql(s"CREATE TABLE $t USING foo AS SELECT id, data FROM source")
+          sql(s"CACHE TABLE $view AS SELECT id FROM $t")
+          checkAnswer(sql(s"SELECT * FROM $t"), spark.table("source"))
+          checkAnswer(sql(s"SELECT * FROM $view"), spark.table("source").select("id"))
+
+          sql(s"REPLACE TABLE $t USING foo AS SELECT id FROM source")
+          assert(spark.sharedState.cacheManager.lookupCachedData(spark.table(view)).isEmpty)
+        }
+      }
+    }
   }
 
-  test("DropTable: if exists") {
-    intercept[NoSuchTableException] {
-      sql(s"DROP TABLE testcat.db.notbl")
+  test("SPARK-33492: AppendData should refresh cache") {
+    import testImplicits._
+
+    val t = "testcat.ns.t"
+    val view = "view"
+    withTable(t) {
+      withTempView(view) {
+        Seq((1, "a")).toDF("i", "j").write.saveAsTable(t)
+        sql(s"CACHE TABLE $view AS SELECT i FROM $t")
+        checkAnswer(sql(s"SELECT * FROM $t"), Row(1, "a") :: Nil)
+        checkAnswer(sql(s"SELECT * FROM $view"), Row(1) :: Nil)
+
+        Seq((2, "b")).toDF("i", "j").write.mode(SaveMode.Append).saveAsTable(t)
+
+        assert(spark.sharedState.cacheManager.lookupCachedData(spark.table(view)).isDefined)
+        checkAnswer(sql(s"SELECT * FROM $t"), Row(1, "a") :: Row(2, "b") :: Nil)
+        checkAnswer(sql(s"SELECT * FROM $view"), Row(1) :: Row(2) :: Nil)
+      }
+    }
+  }
+
+  test("SPARK-33492: OverwriteByExpression should refresh cache") {
+    val t = "testcat.ns.t"
+    val view = "view"
+    withTable(t) {
+      withTempView(view) {
+        sql(s"CREATE TABLE $t USING foo AS SELECT id, data FROM source")
+        sql(s"CACHE TABLE $view AS SELECT id FROM $t")
+        checkAnswer(sql(s"SELECT * FROM $t"), spark.table("source"))
+        checkAnswer(sql(s"SELECT * FROM $view"), spark.table("source").select("id"))
+
+        sql(s"INSERT OVERWRITE TABLE $t VALUES (1, 'a')")
+
+        assert(spark.sharedState.cacheManager.lookupCachedData(spark.table(view)).isDefined)
+        checkAnswer(sql(s"SELECT * FROM $t"), Row(1, "a") :: Nil)
+        checkAnswer(sql(s"SELECT * FROM $view"), Row(1) :: Nil)
+      }
+    }
+  }
+
+  test("SPARK-33492: OverwritePartitionsDynamic should refresh cache") {
+    import testImplicits._
+
+    val t = "testcat.ns.t"
+    val view = "view"
+    withTable(t) {
+      withTempView(view) {
+        Seq((1, "a", 1)).toDF("i", "j", "k").write.partitionBy("k") saveAsTable(t)
+        sql(s"CACHE TABLE $view AS SELECT i FROM $t")
+        checkAnswer(sql(s"SELECT * FROM $t"), Row(1, "a", 1) :: Nil)
+        checkAnswer(sql(s"SELECT * FROM $view"), Row(1) :: Nil)
+
+        Seq((2, "b", 1)).toDF("i", "j", "k").writeTo(t).overwritePartitions()
+
+        assert(spark.sharedState.cacheManager.lookupCachedData(spark.table(view)).isDefined)
+        checkAnswer(sql(s"SELECT * FROM $t"), Row(2, "b", 1) :: Nil)
+        checkAnswer(sql(s"SELECT * FROM $view"), Row(2) :: Nil)
+      }
+    }
+  }
+
+  test("SPARK-34947: micro batch streaming write should invalidate cache") {
+    import testImplicits._
+
+    val t = "testcat.ns.t"
+    withTable(t) {
+      withTempDir { checkpointDir =>
+        sql(s"CREATE TABLE $t (id bigint, data string) USING foo")
+        sql(s"INSERT INTO $t VALUES (1L, 'a')")
+        sql(s"CACHE TABLE $t")
+
+        val inputData = MemoryStream[(Long, String)]
+        val df = inputData.toDF().toDF("id", "data")
+        val query = df
+          .writeStream
+          .option("checkpointLocation", checkpointDir.getAbsolutePath)
+          .toTable(t)
+
+        val newData = Seq((2L, "b"))
+        inputData.addData(newData)
+        query.processAllAvailable()
+        query.stop()
+
+        assert(!spark.catalog.isCached("testcat.ns.t"))
+        checkAnswer(sql(s"SELECT * FROM $t"), Row(1L, "a") :: Row(2L, "b") :: Nil)
+      }
     }
-    sql(s"DROP TABLE IF EXISTS testcat.db.notbl")
   }
 
   test("Relation: basic") {
@@ -755,8 +908,9 @@ class DataSourceV2SQLSuite
 
   test("Relation: view text") {
     val t1 = "testcat.ns1.ns2.tbl"
+    val v1 = "view1"
     withTable(t1) {
-      withView("view1") { v1: String =>
+      withView(v1) {
         sql(s"CREATE TABLE $t1 USING foo AS SELECT id, data FROM source")
         sql(s"CREATE VIEW $v1 AS SELECT * from $t1")
         checkAnswer(sql(s"TABLE $v1"), spark.table("source"))
@@ -808,7 +962,7 @@ class DataSourceV2SQLSuite
       val ex = intercept[AnalysisException] {
         sql(s"SELECT ns1.ns2.ns3.tbl.id from $t")
       }
-      assert(ex.getMessage.contains("cannot resolve '`ns1.ns2.ns3.tbl.id`"))
+      assert(ex.getMessage.contains("cannot resolve 'ns1.ns2.ns3.tbl.id"))
     }
   }
 
@@ -847,74 +1001,13 @@ class DataSourceV2SQLSuite
     }
   }
 
-  test("ShowTables: using v2 catalog") {
-    spark.sql("CREATE TABLE testcat.db.table_name (id bigint, data string) USING foo")
-    spark.sql("CREATE TABLE testcat.n1.n2.db.table_name (id bigint, data string) USING foo")
-
-    runShowTablesSql("SHOW TABLES FROM testcat.db", Seq(Row("db", "table_name")))
-
-    runShowTablesSql(
-      "SHOW TABLES FROM testcat.n1.n2.db",
-      Seq(Row("n1.n2.db", "table_name")))
-  }
-
-  test("ShowTables: using v2 catalog with a pattern") {
-    spark.sql("CREATE TABLE testcat.db.table (id bigint, data string) USING foo")
-    spark.sql("CREATE TABLE testcat.db.table_name_1 (id bigint, data string) USING foo")
-    spark.sql("CREATE TABLE testcat.db.table_name_2 (id bigint, data string) USING foo")
-    spark.sql("CREATE TABLE testcat.db2.table_name_2 (id bigint, data string) USING foo")
-
-    runShowTablesSql(
-      "SHOW TABLES FROM testcat.db",
-      Seq(
-        Row("db", "table"),
-        Row("db", "table_name_1"),
-        Row("db", "table_name_2")))
-
-    runShowTablesSql(
-      "SHOW TABLES FROM testcat.db LIKE '*name*'",
-      Seq(Row("db", "table_name_1"), Row("db", "table_name_2")))
-
-    runShowTablesSql(
-      "SHOW TABLES FROM testcat.db LIKE '*2'",
-      Seq(Row("db", "table_name_2")))
-  }
-
-  test("ShowTables: using v2 catalog, namespace doesn't exist") {
-    runShowTablesSql("SHOW TABLES FROM testcat.unknown", Seq())
-  }
-
-  test("ShowTables: using v1 catalog") {
-    runShowTablesSql(
-      "SHOW TABLES FROM default",
-      Seq(Row("", "source", true), Row("", "source2", true)),
-      expectV2Catalog = false)
-  }
-
-  test("ShowTables: using v1 catalog, db doesn't exist ") {
-    // 'db' below resolves to a database name for v1 catalog because there is no catalog named
-    // 'db' and there is no default catalog set.
-    val exception = intercept[NoSuchDatabaseException] {
-      runShowTablesSql("SHOW TABLES FROM db", Seq(), expectV2Catalog = false)
-    }
-
-    assert(exception.getMessage.contains("Database 'db' not found"))
-  }
-
-  test("ShowTables: using v1 catalog, db name with multipartIdentifier ('a.b') is not allowed.") {
-    val exception = intercept[AnalysisException] {
-      runShowTablesSql("SHOW TABLES FROM a.b", Seq(), expectV2Catalog = false)
-    }
-
-    assert(exception.getMessage.contains("The database name is not valid: a.b"))
-  }
-
   test("ShowViews: using v1 catalog, db name with multipartIdentifier ('a.b') is not allowed.") {
     val exception = intercept[AnalysisException] {
-      sql("SHOW TABLES FROM a.b")
+      sql("SHOW VIEWS FROM a.b")
     }
 
-    assert(exception.getMessage.contains("The database name is not valid: a.b"))
+    assert(exception.getMessage.contains(
+      "Nested databases are not supported by v1 session catalog: a.b"))
   }
 
   test("ShowViews: using v2 catalog, command not supported.") {
@@ -926,112 +1019,6 @@ class DataSourceV2SQLSuite
       " only SessionCatalog supports this command."))
   }
 
-  test("ShowTables: using v2 catalog with empty namespace") {
-    spark.sql("CREATE TABLE testcat.table (id bigint, data string) USING foo")
-    runShowTablesSql("SHOW TABLES FROM testcat", Seq(Row("", "table")))
-  }
-
-  test("ShowTables: namespace is not specified and default v2 catalog is set") {
-    spark.conf.set(SQLConf.DEFAULT_CATALOG.key, "testcat")
-    spark.sql("CREATE TABLE testcat.table (id bigint, data string) USING foo")
-
-    // v2 catalog is used where default namespace is empty for TestInMemoryTableCatalog.
-    runShowTablesSql("SHOW TABLES", Seq(Row("", "table")))
-  }
-
-  test("ShowTables: namespace not specified and default v2 catalog not set - fallback to v1") {
-    runShowTablesSql(
-      "SHOW TABLES",
-      Seq(Row("", "source", true), Row("", "source2", true)),
-      expectV2Catalog = false)
-
-    runShowTablesSql(
-      "SHOW TABLES LIKE '*2'",
-      Seq(Row("", "source2", true)),
-      expectV2Catalog = false)
-  }
-
-  test("ShowTables: change current catalog and namespace with USE statements") {
-    sql("CREATE TABLE testcat.ns1.ns2.table (id bigint) USING foo")
-
-    // Initially, the v2 session catalog (current catalog) is used.
-    runShowTablesSql(
-      "SHOW TABLES", Seq(Row("", "source", true), Row("", "source2", true)),
-      expectV2Catalog = false)
-
-    // Update the current catalog, and no table is matched since the current namespace is Array().
-    sql("USE testcat")
-    runShowTablesSql("SHOW TABLES", Seq())
-
-    // Update the current namespace to match ns1.ns2.table.
-    sql("USE testcat.ns1.ns2")
-    runShowTablesSql("SHOW TABLES", Seq(Row("ns1.ns2", "table")))
-  }
-
-  private def runShowTablesSql(
-      sqlText: String,
-      expected: Seq[Row],
-      expectV2Catalog: Boolean = true): Unit = {
-    val schema = if (expectV2Catalog) {
-      new StructType()
-        .add("namespace", StringType, nullable = false)
-        .add("tableName", StringType, nullable = false)
-    } else {
-      new StructType()
-        .add("database", StringType, nullable = false)
-        .add("tableName", StringType, nullable = false)
-        .add("isTemporary", BooleanType, nullable = false)
-    }
-
-    val df = spark.sql(sqlText)
-    assert(df.schema === schema)
-    assert(expected === df.collect())
-  }
-
-  test("SHOW TABLE EXTENDED not valid v1 database") {
-    def testV1CommandNamespace(sqlCommand: String, namespace: String): Unit = {
-      val e = intercept[AnalysisException] {
-        sql(sqlCommand)
-      }
-      assert(e.message.contains(s"The database name is not valid: ${namespace}"))
-    }
-
-    val namespace = "testcat.ns1.ns2"
-    val table = "tbl"
-    withTable(s"$namespace.$table") {
-      sql(s"CREATE TABLE $namespace.$table (id bigint, data string) " +
-        s"USING foo PARTITIONED BY (id)")
-
-      testV1CommandNamespace(s"SHOW TABLE EXTENDED FROM $namespace LIKE 'tb*'",
-        namespace)
-      testV1CommandNamespace(s"SHOW TABLE EXTENDED IN $namespace LIKE 'tb*'",
-        namespace)
-      testV1CommandNamespace("SHOW TABLE EXTENDED " +
-        s"FROM $namespace LIKE 'tb*' PARTITION(id=1)",
-        namespace)
-      testV1CommandNamespace("SHOW TABLE EXTENDED " +
-        s"IN $namespace LIKE 'tb*' PARTITION(id=1)",
-        namespace)
-    }
-  }
-
-  test("SHOW TABLE EXTENDED valid v1") {
-    val expected = Seq(Row("", "source", true), Row("", "source2", true))
-    val schema = new StructType()
-      .add("database", StringType, nullable = false)
-      .add("tableName", StringType, nullable = false)
-      .add("isTemporary", BooleanType, nullable = false)
-      .add("information", StringType, nullable = false)
-
-    val df = sql("SHOW TABLE EXTENDED FROM default LIKE '*source*'")
-    val result = df.collect()
-    val resultWithoutInfo = result.map{ case Row(db, table, temp, _) => Row(db, table, temp)}
-
-    assert(df.schema === schema)
-    assert(resultWithoutInfo === expected)
-    result.foreach{ case Row(_, _, _, info: String) => assert(info.nonEmpty)}
-  }
-
   test("CreateNameSpace: basic tests") {
     // Session catalog is used.
     withNamespace("ns") {
@@ -1251,8 +1238,8 @@ class DataSourceV2SQLSuite
       val descriptionDf = sql("DESCRIBE NAMESPACE testcat.ns1.ns2")
       assert(descriptionDf.schema.map(field => (field.name, field.dataType)) ===
         Seq(
-          ("name", StringType),
-          ("value", StringType)
+          ("info_name", StringType),
+          ("info_value", StringType)
         ))
       val description = descriptionDf.collect()
       assert(description === Seq(
@@ -1264,7 +1251,7 @@ class DataSourceV2SQLSuite
     }
   }
 
-  test("AlterNamespaceSetProperties using v2 catalog") {
+  test("ALTER NAMESPACE .. SET PROPERTIES using v2 catalog") {
     withNamespace("testcat.ns1.ns2") {
       sql("CREATE NAMESPACE IF NOT EXISTS testcat.ns1.ns2 COMMENT " +
         "'test namespace' LOCATION '/tmp/ns_test' WITH PROPERTIES ('a'='a','b'='b','c'='c')")
@@ -1280,7 +1267,7 @@ class DataSourceV2SQLSuite
     }
   }
 
-  test("AlterNamespaceSetProperties: reserved properties") {
+  test("ALTER NAMESPACE .. SET PROPERTIES reserved properties") {
     import SupportsNamespaces._
     withSQLConf((SQLConf.LEGACY_PROPERTY_NON_RESERVED.key, "false")) {
       CatalogV2Util.NAMESPACE_RESERVED_PROPERTIES.filterNot(_ == PROP_COMMENT).foreach { key =>
@@ -1311,7 +1298,7 @@ class DataSourceV2SQLSuite
     }
   }
 
-  test("AlterNamespaceSetLocation using v2 catalog") {
+  test("ALTER NAMESPACE .. SET LOCATION using v2 catalog") {
     withNamespace("testcat.ns1.ns2") {
       sql("CREATE NAMESPACE IF NOT EXISTS testcat.ns1.ns2 COMMENT " +
         "'test namespace' LOCATION '/tmp/ns_test_1'")
@@ -1326,95 +1313,6 @@ class DataSourceV2SQLSuite
     }
   }
 
-  test("ShowNamespaces: show root namespaces with default v2 catalog") {
-    spark.conf.set(SQLConf.DEFAULT_CATALOG.key, "testcat")
-
-    testShowNamespaces("SHOW NAMESPACES", Seq())
-
-    spark.sql("CREATE TABLE testcat.ns1.table (id bigint) USING foo")
-    spark.sql("CREATE TABLE testcat.ns1.ns1_1.table (id bigint) USING foo")
-    spark.sql("CREATE TABLE testcat.ns2.table (id bigint) USING foo")
-
-    testShowNamespaces("SHOW NAMESPACES", Seq("ns1", "ns2"))
-    testShowNamespaces("SHOW NAMESPACES LIKE '*1*'", Seq("ns1"))
-  }
-
-  test("ShowNamespaces: show namespaces with v2 catalog") {
-    spark.sql("CREATE TABLE testcat.ns1.table (id bigint) USING foo")
-    spark.sql("CREATE TABLE testcat.ns1.ns1_1.table (id bigint) USING foo")
-    spark.sql("CREATE TABLE testcat.ns1.ns1_2.table (id bigint) USING foo")
-    spark.sql("CREATE TABLE testcat.ns2.table (id bigint) USING foo")
-    spark.sql("CREATE TABLE testcat.ns2.ns2_1.table (id bigint) USING foo")
-
-    // Look up only with catalog name, which should list root namespaces.
-    testShowNamespaces("SHOW NAMESPACES IN testcat", Seq("ns1", "ns2"))
-
-    // Look up sub-namespaces.
-    testShowNamespaces("SHOW NAMESPACES IN testcat.ns1", Seq("ns1.ns1_1", "ns1.ns1_2"))
-    testShowNamespaces("SHOW NAMESPACES IN testcat.ns1 LIKE '*2*'", Seq("ns1.ns1_2"))
-    testShowNamespaces("SHOW NAMESPACES IN testcat.ns2", Seq("ns2.ns2_1"))
-
-    // Try to look up namespaces that do not exist.
-    testShowNamespaces("SHOW NAMESPACES IN testcat.ns3", Seq())
-    testShowNamespaces("SHOW NAMESPACES IN testcat.ns1.ns3", Seq())
-  }
-
-  test("ShowNamespaces: default v2 catalog is not set") {
-    spark.sql("CREATE TABLE testcat.ns.table (id bigint) USING foo")
-
-    // The current catalog is resolved to a v2 session catalog.
-    testShowNamespaces("SHOW NAMESPACES", Seq("default"))
-  }
-
-  test("ShowNamespaces: default v2 catalog doesn't support namespace") {
-    spark.conf.set(
-      "spark.sql.catalog.testcat_no_namspace",
-      classOf[BasicInMemoryTableCatalog].getName)
-    spark.conf.set(SQLConf.DEFAULT_CATALOG.key, "testcat_no_namspace")
-
-    val exception = intercept[AnalysisException] {
-      sql("SHOW NAMESPACES")
-    }
-
-    assert(exception.getMessage.contains("does not support namespaces"))
-  }
-
-  test("ShowNamespaces: v2 catalog doesn't support namespace") {
-    spark.conf.set(
-      "spark.sql.catalog.testcat_no_namspace",
-      classOf[BasicInMemoryTableCatalog].getName)
-
-    val exception = intercept[AnalysisException] {
-      sql("SHOW NAMESPACES in testcat_no_namspace")
-    }
-
-    assert(exception.getMessage.contains("does not support namespaces"))
-  }
-
-  test("ShowNamespaces: session catalog is used and namespace doesn't exist") {
-    val exception = intercept[AnalysisException] {
-      sql("SHOW NAMESPACES in dummy")
-    }
-
-    assert(exception.getMessage.contains("Namespace 'dummy' not found"))
-  }
-
-  test("ShowNamespaces: change catalog and namespace with USE statements") {
-    sql("CREATE TABLE testcat.ns1.ns2.table (id bigint) USING foo")
-
-    // Initially, the current catalog is a v2 session catalog.
-    testShowNamespaces("SHOW NAMESPACES", Seq("default"))
-
-    // Update the current catalog to 'testcat'.
-    sql("USE testcat")
-    testShowNamespaces("SHOW NAMESPACES", Seq("ns1"))
-
-    // Update the current namespace to 'ns1'.
-    sql("USE ns1")
-    // 'SHOW NAMESPACES' is not affected by the current namespace and lists root namespaces.
-    testShowNamespaces("SHOW NAMESPACES", Seq("ns1"))
-  }
-
   private def testShowNamespaces(
       sqlText: String,
       expected: Seq[String]): Unit = {
@@ -1479,11 +1377,26 @@ class DataSourceV2SQLSuite
     assert(exception.getMessage.contains("Database 'ns1' not found"))
   }
 
-  test("Use: v2 catalog is used and namespace does not exist") {
-    // Namespaces are not required to exist for v2 catalogs.
-    sql("USE testcat.ns1.ns2")
-    val catalogManager = spark.sessionState.catalogManager
-    assert(catalogManager.currentNamespace === Array("ns1", "ns2"))
+  test("SPARK-31100: Use: v2 catalog that implements SupportsNamespaces is used " +
+      "and namespace not exists") {
+    // Namespaces are required to exist for v2 catalogs that implements SupportsNamespaces.
+    val exception = intercept[NoSuchNamespaceException] {
+      sql("USE testcat.ns1.ns2")
+    }
+    assert(exception.getMessage.contains("Namespace 'ns1.ns2' not found"))
+  }
+
+  test("SPARK-31100: Use: v2 catalog that does not implement SupportsNameSpaces is used " +
+      "and namespace does not exist") {
+    // Namespaces are not required to exist for v2 catalogs
+    // that does not implement SupportsNamespaces.
+    withSQLConf("spark.sql.catalog.dummy" -> classOf[BasicInMemoryTableCatalog].getName) {
+      val catalogManager = spark.sessionState.catalogManager
+
+      sql("USE dummy.ns1")
+      assert(catalogManager.currentCatalog.name() == "dummy")
+      assert(catalogManager.currentNamespace === Array("ns1"))
+    }
   }
 
   test("ShowCurrentNamespace: basic tests") {
@@ -1505,6 +1418,8 @@ class DataSourceV2SQLSuite
 
     sql("USE testcat")
     testShowCurrentNamespace("testcat", "")
+
+    sql("CREATE NAMESPACE testcat.ns1.ns2")
     sql("USE testcat.ns1.ns2")
     testShowCurrentNamespace("testcat", "ns1.ns2")
   }
@@ -1630,7 +1545,6 @@ class DataSourceV2SQLSuite
         """
           |CREATE TABLE testcat.t (id int, `a.b` string) USING foo
           |CLUSTERED BY (`a.b`) INTO 4 BUCKETS
-          |OPTIONS ('allow-unsupported-transforms'=true)
         """.stripMargin)
 
       val testCatalog = catalog("testcat").asTableCatalog.asInstanceOf[InMemoryTableCatalog]
@@ -1709,6 +1623,63 @@ class DataSourceV2SQLSuite
     }
   }
 
+  test("SPARK-32990: REFRESH TABLE should resolve to a temporary view first") {
+    withTable("testcat.ns.t") {
+      withTempView("t") {
+        sql("CREATE TABLE testcat.ns.t (id bigint) USING foo")
+        sql("CREATE TEMPORARY VIEW t AS SELECT 2")
+        sql("USE testcat.ns")
+
+        val testCatalog = catalog("testcat").asTableCatalog.asInstanceOf[InMemoryTableCatalog]
+        val identifier = Identifier.of(Array("ns"), "t")
+
+        assert(!testCatalog.isTableInvalidated(identifier))
+        sql("REFRESH TABLE t")
+        assert(!testCatalog.isTableInvalidated(identifier))
+      }
+    }
+  }
+
+  test("SPARK-33435, SPARK-34099: REFRESH TABLE should refresh all caches referencing the table") {
+    val tblName = "testcat.ns.t"
+    withTable(tblName) {
+      withTempView("t") {
+        sql(s"CREATE TABLE $tblName (id bigint) USING foo")
+        sql(s"INSERT INTO $tblName SELECT 0")
+        sql(s"CACHE TABLE t AS SELECT id FROM $tblName")
+        checkAnswer(spark.table(tblName), Row(0))
+        checkAnswer(spark.table("t"), Row(0))
+
+        sql(s"INSERT INTO $tblName SELECT 1")
+
+        assert(spark.sharedState.cacheManager.lookupCachedData(spark.table("t")).isDefined)
+        sql(s"REFRESH TABLE $tblName")
+        assert(spark.sharedState.cacheManager.lookupCachedData(spark.table("t")).isDefined)
+        checkAnswer(spark.table(tblName), Seq(Row(0), Row(1)))
+        checkAnswer(spark.table("t"), Seq(Row(0), Row(1)))
+      }
+    }
+  }
+
+  test("SPARK-33653: REFRESH TABLE should recache the target table itself") {
+    val tblName = "testcat.ns.t"
+    withTable(tblName) {
+      sql(s"CREATE TABLE $tblName (id bigint) USING foo")
+
+      // if the table is not cached, refreshing it should not recache it
+      assert(spark.sharedState.cacheManager.lookupCachedData(spark.table(tblName)).isEmpty)
+      sql(s"REFRESH TABLE $tblName")
+      assert(spark.sharedState.cacheManager.lookupCachedData(spark.table(tblName)).isEmpty)
+
+      sql(s"CACHE TABLE $tblName")
+
+      // after caching & refreshing the table should be recached
+      assert(spark.sharedState.cacheManager.lookupCachedData(spark.table(tblName)).isDefined)
+      sql(s"REFRESH TABLE $tblName")
+      assert(spark.sharedState.cacheManager.lookupCachedData(spark.table(tblName)).isDefined)
+    }
+  }
+
   test("REPLACE TABLE: v1 table") {
     val e = intercept[AnalysisException] {
       sql(s"CREATE OR REPLACE TABLE tbl (a int) USING ${classOf[SimpleScanSource].getName}")
@@ -1773,6 +1744,20 @@ class DataSourceV2SQLSuite
     }
   }
 
+  test("DeleteFrom: delete with unsupported predicates") {
+    val t = "testcat.ns1.ns2.tbl"
+    withTable(t) {
+      sql(s"CREATE TABLE $t (id bigint, data string, p int) USING foo")
+      sql(s"INSERT INTO $t VALUES (2L, 'a', 2), (2L, 'b', 3), (3L, 'c', 3)")
+      val exc = intercept[AnalysisException] {
+        sql(s"DELETE FROM $t WHERE id > 3 AND p > 3")
+      }
+
+      assert(spark.table(t).count === 3)
+      assert(exc.getMessage.contains(s"Cannot delete from table $t"))
+    }
+  }
+
   test("DeleteFrom: DELETE is only supported with v2 tables") {
     // unset this config to use the default v2 session catalog.
     spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key)
@@ -1788,6 +1773,22 @@ class DataSourceV2SQLSuite
     }
   }
 
+  test("SPARK-33652: DeleteFrom should refresh caches referencing the table") {
+    val t = "testcat.ns1.ns2.tbl"
+    val view = "view"
+    withTable(t) {
+      withTempView(view) {
+        sql(s"CREATE TABLE $t (id bigint, data string, p int) USING foo PARTITIONED BY (id, p)")
+        sql(s"INSERT INTO $t VALUES (2L, 'a', 2), (2L, 'b', 3), (3L, 'c', 3)")
+        sql(s"CACHE TABLE view AS SELECT id FROM $t")
+        assert(spark.table(view).count() == 3)
+
+        sql(s"DELETE FROM $t WHERE id = 2")
+        assert(spark.table(view).count() == 1)
+      }
+    }
+  }
+
   test("UPDATE TABLE") {
     val t = "testcat.ns1.ns2.tbl"
     withTable(t) {
@@ -1904,14 +1905,16 @@ class DataSourceV2SQLSuite
     }
   }
 
-  test("AlterTable: rename table basic test") {
+  test("rename table by ALTER VIEW") {
     withTable("testcat.ns1.new") {
-      sql(s"CREATE TABLE testcat.ns1.ns2.old USING foo AS SELECT id, data FROM source")
-      checkAnswer(sql("SHOW TABLES FROM testcat.ns1.ns2"), Seq(Row("ns1.ns2", "old")))
+      sql("CREATE TABLE testcat.ns1.ns2.old USING foo AS SELECT id, data FROM source")
+      checkAnswer(sql("SHOW TABLES FROM testcat.ns1.ns2"), Seq(Row("ns1.ns2", "old", false)))
 
-      sql(s"ALTER TABLE testcat.ns1.ns2.old RENAME TO ns1.new")
-      checkAnswer(sql("SHOW TABLES FROM testcat.ns1.ns2"), Seq.empty)
-      checkAnswer(sql("SHOW TABLES FROM testcat.ns1"), Seq(Row("ns1", "new")))
+      val e = intercept[AnalysisException] {
+        sql("ALTER VIEW testcat.ns1.ns2.old RENAME TO ns1.new")
+      }
+      assert(e.getMessage.contains(
+        "Cannot rename a table with ALTER VIEW. Please use ALTER TABLE instead"))
     }
   }
 
@@ -1919,15 +1922,16 @@ class DataSourceV2SQLSuite
     val e = intercept[AnalysisException] {
       sql(s"ALTER VIEW testcat.ns.tbl RENAME TO ns.view")
     }
-    assert(e.getMessage.contains("Renaming view is not supported in v2 catalogs"))
+    assert(e.getMessage.contains(
+      "Table or view not found: testcat.ns.tbl"))
   }
 
   test("ANALYZE TABLE") {
     val t = "testcat.ns1.ns2.tbl"
     withTable(t) {
       spark.sql(s"CREATE TABLE $t (id bigint, data string) USING foo")
-      testV1Command("ANALYZE TABLE", s"$t COMPUTE STATISTICS")
-      testV1CommandSupportingTempView("ANALYZE TABLE", s"$t COMPUTE STATISTICS FOR ALL COLUMNS")
+      testNotSupportedV2Command("ANALYZE TABLE", s"$t COMPUTE STATISTICS")
+      testNotSupportedV2Command("ANALYZE TABLE", s"$t COMPUTE STATISTICS FOR ALL COLUMNS")
     }
   }
 
@@ -1935,37 +1939,7 @@ class DataSourceV2SQLSuite
     val t = "testcat.ns1.ns2.tbl"
     withTable(t) {
       spark.sql(s"CREATE TABLE $t (id bigint, data string) USING foo")
-      testV1Command("MSCK REPAIR TABLE", t)
-    }
-  }
-
-  test("TRUNCATE TABLE") {
-    val t = "testcat.ns1.ns2.tbl"
-    withTable(t) {
-      sql(
-        s"""
-           |CREATE TABLE $t (id bigint, data string)
-           |USING foo
-           |PARTITIONED BY (id)
-         """.stripMargin)
-
-      testV1Command("TRUNCATE TABLE", t)
-      testV1Command("TRUNCATE TABLE", s"$t PARTITION(id='1')")
-    }
-  }
-
-  test("SHOW PARTITIONS") {
-    val t = "testcat.ns1.ns2.tbl"
-    withTable(t) {
-      sql(
-        s"""
-           |CREATE TABLE $t (id bigint, data string)
-           |USING foo
-           |PARTITIONED BY (id)
-         """.stripMargin)
-
-      testV1Command("SHOW PARTITIONS", t)
-      testV1Command("SHOW PARTITIONS", s"$t PARTITION(id='1')")
+      testNotSupportedV2Command("MSCK REPAIR TABLE", t)
     }
   }
 
@@ -1979,10 +1953,10 @@ class DataSourceV2SQLSuite
            |PARTITIONED BY (id)
          """.stripMargin)
 
-      testV1Command("LOAD DATA", s"INPATH 'filepath' INTO TABLE $t")
-      testV1Command("LOAD DATA", s"LOCAL INPATH 'filepath' INTO TABLE $t")
-      testV1Command("LOAD DATA", s"LOCAL INPATH 'filepath' OVERWRITE INTO TABLE $t")
-      testV1Command("LOAD DATA",
+      testNotSupportedV2Command("LOAD DATA", s"INPATH 'filepath' INTO TABLE $t")
+      testNotSupportedV2Command("LOAD DATA", s"LOCAL INPATH 'filepath' INTO TABLE $t")
+      testNotSupportedV2Command("LOAD DATA", s"LOCAL INPATH 'filepath' OVERWRITE INTO TABLE $t")
+      testNotSupportedV2Command("LOAD DATA",
         s"LOCAL INPATH 'filepath' OVERWRITE INTO TABLE $t PARTITION(id=1)")
     }
   }
@@ -1991,91 +1965,44 @@ class DataSourceV2SQLSuite
     val t = "testcat.ns1.ns2.tbl"
     withTable(t) {
       spark.sql(s"CREATE TABLE $t (id bigint, data string) USING foo")
-      testV1CommandSupportingTempView("SHOW CREATE TABLE", t)
+      testNotSupportedV2Command("SHOW CREATE TABLE", t)
+      testNotSupportedV2Command("SHOW CREATE TABLE", s"$t AS SERDE")
     }
   }
 
-  test("CACHE TABLE") {
+  test("CACHE/UNCACHE TABLE") {
     val t = "testcat.ns1.ns2.tbl"
     withTable(t) {
-      spark.sql(s"CREATE TABLE $t (id bigint, data string) USING foo")
-
-      testV1CommandSupportingTempView("CACHE TABLE", t)
-
-      val e = intercept[AnalysisException] {
-        sql(s"CACHE LAZY TABLE $t")
+      def isCached(table: String): Boolean = {
+        spark.table(table).queryExecution.withCachedData.isInstanceOf[InMemoryRelation]
       }
-      assert(e.message.contains("CACHE TABLE is only supported with temp views or v1 tables"))
-    }
-  }
-
-  test("UNCACHE TABLE") {
-    val t = "testcat.ns1.ns2.tbl"
-    withTable(t) {
-      sql(s"CREATE TABLE $t (id bigint, data string) USING foo")
-
-      testV1CommandSupportingTempView("UNCACHE TABLE", t)
-      testV1CommandSupportingTempView("UNCACHE TABLE", s"IF EXISTS $t")
-    }
-  }
 
-  test("SHOW COLUMNS") {
-    val t = "testcat.ns1.ns2.tbl"
-    withTable(t) {
       spark.sql(s"CREATE TABLE $t (id bigint, data string) USING foo")
+      sql(s"CACHE TABLE $t")
+      assert(isCached(t))
 
-      testV1CommandSupportingTempView("SHOW COLUMNS", s"FROM $t")
-      testV1CommandSupportingTempView("SHOW COLUMNS", s"IN $t")
-
-      val e3 = intercept[AnalysisException] {
-        sql(s"SHOW COLUMNS FROM tbl IN testcat.ns1.ns2")
-      }
-      assert(e3.message.contains("Namespace name should have " +
-        "only one part if specified: testcat.ns1.ns2"))
+      sql(s"UNCACHE TABLE $t")
+      assert(!isCached(t))
     }
-  }
 
-  test("ALTER TABLE RECOVER PARTITIONS") {
-    val t = "testcat.ns1.ns2.tbl"
-    withTable(t) {
-      spark.sql(s"CREATE TABLE $t (id bigint, data string) USING foo")
-      val e = intercept[AnalysisException] {
-        sql(s"ALTER TABLE $t RECOVER PARTITIONS")
-      }
-      assert(e.message.contains("ALTER TABLE RECOVER PARTITIONS is only supported with v1 tables"))
+    // Test a scenario where a table does not exist.
+    val e = intercept[AnalysisException] {
+      sql(s"UNCACHE TABLE $t")
     }
-  }
+    assert(e.message.contains("Table or view not found: testcat.ns1.ns2.tbl"))
 
-  test("ALTER TABLE ADD PARTITION") {
-    val t = "testcat.ns1.ns2.tbl"
-    withTable(t) {
-      spark.sql(s"CREATE TABLE $t (id bigint, data string) USING foo PARTITIONED BY (id)")
-      val e = intercept[AnalysisException] {
-        sql(s"ALTER TABLE $t ADD PARTITION (id=1) LOCATION 'loc'")
-      }
-      assert(e.message.contains("ALTER TABLE ADD PARTITION is only supported with v1 tables"))
-    }
+    // If "IF EXISTS" is set, UNCACHE TABLE will not throw an exception.
+    sql(s"UNCACHE TABLE IF EXISTS $t")
   }
 
-  test("ALTER TABLE RENAME PARTITION") {
+  test("SHOW COLUMNS") {
     val t = "testcat.ns1.ns2.tbl"
     withTable(t) {
-      spark.sql(s"CREATE TABLE $t (id bigint, data string) USING foo PARTITIONED BY (id)")
-      val e = intercept[AnalysisException] {
-        sql(s"ALTER TABLE $t PARTITION (id=1) RENAME TO PARTITION (id=2)")
-      }
-      assert(e.message.contains("ALTER TABLE RENAME PARTITION is only supported with v1 tables"))
-    }
-  }
+      spark.sql(s"CREATE TABLE $t (id bigint, data string) USING foo")
 
-  test("ALTER TABLE DROP PARTITIONS") {
-    val t = "testcat.ns1.ns2.tbl"
-    withTable(t) {
-      spark.sql(s"CREATE TABLE $t (id bigint, data string) USING foo PARTITIONED BY (id)")
-      val e = intercept[AnalysisException] {
-        sql(s"ALTER TABLE $t DROP PARTITION (id=1)")
-      }
-      assert(e.message.contains("ALTER TABLE DROP PARTITION is only supported with v1 tables"))
+      testNotSupportedV2Command("SHOW COLUMNS", s"FROM $t")
+      testNotSupportedV2Command("SHOW COLUMNS", s"IN $t")
+      testNotSupportedV2Command("SHOW COLUMNS", "FROM tbl IN testcat.ns1.ns2")
     }
   }
 
@@ -2086,22 +2013,15 @@ class DataSourceV2SQLSuite
       val e = intercept[AnalysisException] {
         sql(s"ALTER TABLE $t SET SERDEPROPERTIES ('columns'='foo,bar', 'field.delim' = ',')")
       }
-      assert(e.message.contains("ALTER TABLE SerDe Properties is only supported with v1 tables"))
+      assert(e.message.contains(
+        "ALTER TABLE ... SET [SERDE|SERDEPROPERTIES] is not supported for v2 tables"))
     }
   }
 
-  test("ALTER VIEW AS QUERY") {
-    val v = "testcat.ns1.ns2.v"
-    val e = intercept[AnalysisException] {
-      sql(s"ALTER VIEW $v AS SELECT 1")
-    }
-    assert(e.message.contains("ALTER VIEW QUERY is only supported with temp views or v1 tables"))
-  }
-
   test("CREATE VIEW") {
     val v = "testcat.ns1.ns2.v"
     val e = intercept[AnalysisException] {
-      sql(s"CREATE VIEW $v AS SELECT * FROM tab1")
+      sql(s"CREATE VIEW $v AS SELECT 1")
     }
     assert(e.message.contains("CREATE VIEW is only supported with v1 tables"))
   }
@@ -2166,13 +2086,12 @@ class DataSourceV2SQLSuite
     val e = intercept[AnalysisException] {
       sql("DESCRIBE FUNCTION testcat.ns1.ns2.fun")
     }
-    assert(e.message.contains("DESCRIBE FUNCTION is only supported in v1 catalog"))
+    assert(e.message.contains("function is only supported in v1 catalog"))
 
     val e1 = intercept[AnalysisException] {
       sql("DESCRIBE FUNCTION default.ns1.ns2.fun")
     }
-    assert(e1.message.contains(
-      "The namespace in session catalog must have exactly one name part: default.ns1.ns2.fun"))
+    assert(e1.message.contains("Unsupported function name 'default.ns1.ns2.fun'"))
   }
 
   test("SHOW FUNCTIONS not valid v1 namespace") {
@@ -2181,33 +2100,44 @@ class DataSourceV2SQLSuite
     val e = intercept[AnalysisException] {
       sql(s"SHOW FUNCTIONS LIKE $function")
     }
-    assert(e.message.contains("SHOW FUNCTIONS is only supported in v1 catalog"))
+    assert(e.message.contains("function is only supported in v1 catalog"))
   }
 
   test("DROP FUNCTION: only support session catalog") {
     val e = intercept[AnalysisException] {
       sql("DROP FUNCTION testcat.ns1.ns2.fun")
     }
-    assert(e.message.contains("DROP FUNCTION is only supported in v1 catalog"))
+    assert(e.message.contains("function is only supported in v1 catalog"))
 
     val e1 = intercept[AnalysisException] {
       sql("DROP FUNCTION default.ns1.ns2.fun")
     }
-    assert(e1.message.contains(
-      "The namespace in session catalog must have exactly one name part: default.ns1.ns2.fun"))
+    assert(e1.message.contains("Unsupported function name 'default.ns1.ns2.fun'"))
   }
 
   test("CREATE FUNCTION: only support session catalog") {
     val e = intercept[AnalysisException] {
       sql("CREATE FUNCTION testcat.ns1.ns2.fun as 'f'")
     }
-    assert(e.message.contains("CREATE FUNCTION is only supported in v1 catalog"))
+    assert(e.message.contains("function is only supported in v1 catalog"))
 
     val e1 = intercept[AnalysisException] {
       sql("CREATE FUNCTION default.ns1.ns2.fun as 'f'")
     }
+    assert(e1.message.contains("Unsupported function name 'default.ns1.ns2.fun'"))
+  }
+
+  test("REFRESH FUNCTION: only support session catalog") {
+    val e = intercept[AnalysisException] {
+      sql("REFRESH FUNCTION testcat.ns1.ns2.fun")
+    }
+    assert(e.message.contains("function is only supported in v1 catalog"))
+
+    val e1 = intercept[AnalysisException] {
+      sql("REFRESH FUNCTION default.ns1.ns2.fun")
+    }
     assert(e1.message.contains(
-      "The namespace in session catalog must have exactly one name part: default.ns1.ns2.fun"))
+      "Unsupported function name 'default.ns1.ns2.fun'"))
   }
 
   test("global temp view should not be masked by v2 catalog") {
@@ -2241,11 +2171,11 @@ class DataSourceV2SQLSuite
 
     val e = intercept[AnalysisException] {
       // Since the following multi-part name starts with `globalTempDB`, it is resolved to
-      // the session catalog, not the `gloabl_temp` v2 catalog.
+      // the session catalog, not the `global_temp` v2 catalog.
       sql(s"CREATE TABLE $globalTempDB.ns1.ns2.tbl (id bigint, data string) USING json")
     }
     assert(e.message.contains(
-      "The namespace in session catalog must have exactly one name part: global_temp.ns1.ns2.tbl"))
+      "global_temp.ns1.ns2.tbl is not a valid TableIdentifier as it has more than 2 name parts."))
   }
 
   test("table name same as catalog can be used") {
@@ -2274,10 +2204,29 @@ class DataSourceV2SQLSuite
         sql("CREATE TABLE t USING json AS SELECT 1 AS i")
 
         val t = "spark_catalog.t"
+
         def verify(sql: String): Unit = {
           val e = intercept[AnalysisException](spark.sql(sql))
-          assert(e.message.contains(
-            s"The namespace in session catalog must have exactly one name part: $t"))
+          assert(e.message.contains(s"Table or view not found: $t"),
+            s"Error message did not contain expected text while evaluting $sql")
+        }
+
+        def verifyView(sql: String): Unit = {
+          val e = intercept[AnalysisException](spark.sql(sql))
+          assert(e.message.contains(s"View not found: $t"),
+            s"Error message did not contain expected text while evaluting $sql")
+        }
+
+        def verifyTable(sql: String): Unit = {
+          val e = intercept[AnalysisException](spark.sql(sql))
+          assert(e.message.contains(s"Table not found: $t"),
+            s"Error message did not contain expected text while evaluting $sql")
+        }
+
+        def verifyGeneric(sql: String): Unit = {
+          val e = intercept[AnalysisException](spark.sql(sql))
+          assert(e.message.contains(s"not found: $t"),
+            s"Error message did not contain expected text while evaluting $sql")
         }
 
         verify(s"select * from $t")
@@ -2285,17 +2234,16 @@ class DataSourceV2SQLSuite
         verify(s"REFRESH TABLE $t")
         verify(s"DESCRIBE $t i")
         verify(s"DROP TABLE $t")
-        verify(s"DROP VIEW $t")
-        verify(s"ANALYZE TABLE $t COMPUTE STATISTICS")
-        verify(s"ANALYZE TABLE $t COMPUTE STATISTICS FOR ALL COLUMNS")
-        verify(s"MSCK REPAIR TABLE $t")
-        verify(s"LOAD DATA INPATH 'filepath' INTO TABLE $t")
-        verify(s"SHOW CREATE TABLE $t")
-        verify(s"SHOW CREATE TABLE $t AS SERDE")
-        verify(s"CACHE TABLE $t")
-        verify(s"UNCACHE TABLE $t")
-        verify(s"TRUNCATE TABLE $t")
-        verify(s"SHOW PARTITIONS $t")
+        verifyView(s"DROP VIEW $t")
+        verifyGeneric(s"ANALYZE TABLE $t COMPUTE STATISTICS")
+        verifyGeneric(s"ANALYZE TABLE $t COMPUTE STATISTICS FOR ALL COLUMNS")
+        verifyTable(s"MSCK REPAIR TABLE $t")
+        verifyTable(s"LOAD DATA INPATH 'filepath' INTO TABLE $t")
+        verifyGeneric(s"SHOW CREATE TABLE $t")
+        verifyGeneric(s"SHOW CREATE TABLE $t AS SERDE")
+        verifyGeneric(s"CACHE TABLE $t")
+        verifyGeneric(s"UNCACHE TABLE $t")
+        verifyGeneric(s"TRUNCATE TABLE $t")
         verify(s"SHOW COLUMNS FROM $t")
       }
     }
@@ -2342,6 +2290,7 @@ class DataSourceV2SQLSuite
     spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key)
     val sessionCatalogName = CatalogManager.SESSION_CATALOG_NAME
 
+    sql("CREATE NAMESPACE testcat.ns1.ns2")
     sql("USE testcat.ns1.ns2")
     sql("CREATE TABLE t USING foo AS SELECT 1 col")
     checkAnswer(spark.table("t"), Row(1))
@@ -2421,7 +2370,8 @@ class DataSourceV2SQLSuite
     withTempView("v") {
       sql("create global temp view v as select 1")
       val e = intercept[AnalysisException](sql("COMMENT ON TABLE global_temp.v IS NULL"))
-      assert(e.getMessage.contains("global_temp.v is a temp view not table."))
+      assert(e.getMessage.contains(
+        "global_temp.v is a temp view. 'COMMENT ON TABLE' expects a table"))
     }
   }
 
@@ -2441,7 +2391,7 @@ class DataSourceV2SQLSuite
         sql(s"CACHE TABLE $sessionCatalogName.v")
       )
       assert(e1.message.contains(
-        "The namespace in session catalog must have exactly one name part: spark_catalog.v"))
+        "Table or view not found: spark_catalog.v"))
     }
     val e2 = intercept[AnalysisException] {
       sql(s"CREATE TEMP VIEW $sessionCatalogName.v AS SELECT 1")
@@ -2476,18 +2426,407 @@ class DataSourceV2SQLSuite
     }
   }
 
-  private def testV1Command(sqlCommand: String, sqlParams: String): Unit = {
-    val e = intercept[AnalysisException] {
-      sql(s"$sqlCommand $sqlParams")
+  test("SPARK-32168: INSERT OVERWRITE - hidden days partition - dynamic mode") {
+    def testTimestamp(daysOffset: Int): Timestamp = {
+      Timestamp.valueOf(LocalDate.of(2020, 1, 1 + daysOffset).atStartOfDay())
+    }
+
+    withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.DYNAMIC.toString) {
+      val t1 = s"${catalogAndNamespace}tbl"
+      withTable(t1) {
+        val df = spark.createDataFrame(Seq(
+          (testTimestamp(1), "a"),
+          (testTimestamp(2), "b"),
+          (testTimestamp(3), "c"))).toDF("ts", "data")
+        df.createOrReplaceTempView("source_view")
+
+        sql(s"CREATE TABLE $t1 (ts timestamp, data string) " +
+            s"USING $v2Format PARTITIONED BY (days(ts))")
+        sql(s"INSERT INTO $t1 VALUES " +
+            s"(CAST(date_add('2020-01-01', 2) AS timestamp), 'dummy'), " +
+            s"(CAST(date_add('2020-01-01', 4) AS timestamp), 'keep')")
+        sql(s"INSERT OVERWRITE TABLE $t1 SELECT ts, data FROM source_view")
+
+        val expected = spark.createDataFrame(Seq(
+          (testTimestamp(1), "a"),
+          (testTimestamp(2), "b"),
+          (testTimestamp(3), "c"),
+          (testTimestamp(4), "keep"))).toDF("ts", "data")
+
+        verifyTable(t1, expected)
+      }
+    }
+  }
+
+  test("SPARK-31255: Project a metadata column") {
+    val t1 = s"${catalogAndNamespace}table"
+    withTable(t1) {
+      sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format " +
+          "PARTITIONED BY (bucket(4, id), id)")
+      sql(s"INSERT INTO $t1 VALUES (1, 'a'), (2, 'b'), (3, 'c')")
+
+      val sqlQuery = spark.sql(s"SELECT id, data, index, _partition FROM $t1")
+      val dfQuery = spark.table(t1).select("id", "data", "index", "_partition")
+
+      Seq(sqlQuery, dfQuery).foreach { query =>
+        checkAnswer(query, Seq(Row(1, "a", 0, "3/1"), Row(2, "b", 0, "0/2"), Row(3, "c", 0, "1/3")))
+      }
+    }
+  }
+
+  test("SPARK-31255: Projects data column when metadata column has the same name") {
+    val t1 = s"${catalogAndNamespace}table"
+    withTable(t1) {
+      sql(s"CREATE TABLE $t1 (index bigint, data string) USING $v2Format " +
+          "PARTITIONED BY (bucket(4, index), index)")
+      sql(s"INSERT INTO $t1 VALUES (3, 'c'), (2, 'b'), (1, 'a')")
+
+      val sqlQuery = spark.sql(s"SELECT index, data, _partition FROM $t1")
+      val dfQuery = spark.table(t1).select("index", "data", "_partition")
+
+      Seq(sqlQuery, dfQuery).foreach { query =>
+        checkAnswer(query, Seq(Row(3, "c", "1/3"), Row(2, "b", "0/2"), Row(1, "a", "3/1")))
+      }
+    }
+  }
+
+  test("SPARK-31255: * expansion does not include metadata columns") {
+    val t1 = s"${catalogAndNamespace}table"
+    withTable(t1) {
+      sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format " +
+          "PARTITIONED BY (bucket(4, id), id)")
+      sql(s"INSERT INTO $t1 VALUES (3, 'c'), (2, 'b'), (1, 'a')")
+
+      val sqlQuery = spark.sql(s"SELECT * FROM $t1")
+      val dfQuery = spark.table(t1)
+
+      Seq(sqlQuery, dfQuery).foreach { query =>
+        checkAnswer(query, Seq(Row(3, "c"), Row(2, "b"), Row(1, "a")))
+      }
+    }
+  }
+
+  test("SPARK-31255: metadata column should only be produced when necessary") {
+    val t1 = s"${catalogAndNamespace}table"
+    withTable(t1) {
+      sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format " +
+        "PARTITIONED BY (bucket(4, id), id)")
+
+      val sqlQuery = spark.sql(s"SELECT * FROM $t1 WHERE index = 0")
+      val dfQuery = spark.table(t1).filter("index = 0")
+
+      Seq(sqlQuery, dfQuery).foreach { query =>
+        assert(query.schema.fieldNames.toSeq == Seq("id", "data"))
+      }
+    }
+  }
+
+  test("SPARK-34547: metadata columns are resolved last") {
+    val t1 = s"${catalogAndNamespace}tableOne"
+    val t2 = "t2"
+    withTable(t1) {
+      sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format " +
+        "PARTITIONED BY (bucket(4, id), id)")
+      sql(s"INSERT INTO $t1 VALUES (1, 'a'), (2, 'b'), (3, 'c')")
+      withTempView(t2) {
+        sql(s"CREATE TEMPORARY VIEW $t2 AS SELECT * FROM " +
+          s"VALUES (1, -1), (2, -2), (3, -3) AS $t2(id, index)")
+
+        val sqlQuery = spark.sql(s"SELECT $t1.id, $t2.id, data, index, $t1.index, $t2.index FROM " +
+          s"$t1 JOIN $t2 WHERE $t1.id = $t2.id")
+        val t1Table = spark.table(t1)
+        val t2Table = spark.table(t2)
+        val dfQuery = t1Table.join(t2Table, t1Table.col("id") === t2Table.col("id"))
+          .select(s"$t1.id", s"$t2.id", "data", "index", s"$t1.index", s"$t2.index")
+
+        Seq(sqlQuery, dfQuery).foreach { query =>
+          checkAnswer(query,
+            Seq(
+              Row(1, 1, "a", -1, 0, -1),
+              Row(2, 2, "b", -2, 0, -2),
+              Row(3, 3, "c", -3, 0, -3)
+            )
+          )
+        }
+      }
+    }
+  }
+
+  test("SPARK-33505: insert into partitioned table") {
+    val t = "testpart.ns1.ns2.tbl"
+    withTable(t) {
+      sql(s"""
+        |CREATE TABLE $t (id bigint, city string, data string)
+        |USING foo
+        |PARTITIONED BY (id, city)""".stripMargin)
+      val partTable = catalog("testpart").asTableCatalog
+        .loadTable(Identifier.of(Array("ns1", "ns2"), "tbl")).asInstanceOf[InMemoryPartitionTable]
+      val expectedPartitionIdent = InternalRow.fromSeq(Seq(1, UTF8String.fromString("NY")))
+      assert(!partTable.partitionExists(expectedPartitionIdent))
+      sql(s"INSERT INTO $t PARTITION(id = 1, city = 'NY') SELECT 'abc'")
+      assert(partTable.partitionExists(expectedPartitionIdent))
+      // Insert into the existing partition must not fail
+      sql(s"INSERT INTO $t PARTITION(id = 1, city = 'NY') SELECT 'def'")
+      assert(partTable.partitionExists(expectedPartitionIdent))
+    }
+  }
+
+  test("View commands are not supported in v2 catalogs") {
+    def validateViewCommand(
+        sql: String,
+        catalogName: String,
+        viewName: String,
+        cmdName: String): Unit = {
+      assertAnalysisError(
+        sql,
+        s"Cannot specify catalog `$catalogName` for view $viewName because view support " +
+          s"in v2 catalog has not been implemented yet. $cmdName expects a view.")
+    }
+
+    validateViewCommand("DROP VIEW testcat.v", "testcat", "v", "DROP VIEW")
+    validateViewCommand(
+      "ALTER VIEW testcat.v SET TBLPROPERTIES ('key' = 'val')",
+      "testcat",
+      "v",
+      "ALTER VIEW ... SET TBLPROPERTIES")
+    validateViewCommand(
+      "ALTER VIEW testcat.v UNSET TBLPROPERTIES ('key')",
+      "testcat",
+      "v",
+      "ALTER VIEW ... UNSET TBLPROPERTIES")
+    validateViewCommand(
+      "ALTER VIEW testcat.v AS SELECT 1",
+      "testcat",
+      "v",
+      "ALTER VIEW ... AS")
+  }
+
+  test("SPARK-33924: INSERT INTO .. PARTITION preserves the partition location") {
+    val t = "testpart.ns1.ns2.tbl"
+    withTable(t) {
+      sql(s"""
+        |CREATE TABLE $t (id bigint, city string, data string)
+        |USING foo
+        |PARTITIONED BY (id, city)""".stripMargin)
+      val partTable = catalog("testpart").asTableCatalog
+        .loadTable(Identifier.of(Array("ns1", "ns2"), "tbl")).asInstanceOf[InMemoryPartitionTable]
+
+      val loc = "partition_location"
+      sql(s"ALTER TABLE $t ADD PARTITION (id = 1, city = 'NY') LOCATION '$loc'")
+
+      val ident = InternalRow.fromSeq(Seq(1, UTF8String.fromString("NY")))
+      assert(partTable.loadPartitionMetadata(ident).get("location") === loc)
+
+      sql(s"INSERT INTO $t PARTITION(id = 1, city = 'NY') SELECT 'abc'")
+      assert(partTable.loadPartitionMetadata(ident).get("location") === loc)
+    }
+  }
+
+  test("SPARK-34468: rename table in place when the destination name has single part") {
+    val tbl = s"${catalogAndNamespace}src_tbl"
+    withTable(tbl) {
+      sql(s"CREATE TABLE $tbl (c0 INT) USING $v2Format")
+      sql(s"INSERT INTO $tbl SELECT 0")
+      checkAnswer(sql(s"SHOW TABLES FROM testcat.ns1.ns2 LIKE 'new_tbl'"), Nil)
+      sql(s"ALTER TABLE $tbl RENAME TO new_tbl")
+      checkAnswer(
+        sql(s"SHOW TABLES FROM testcat.ns1.ns2 LIKE 'new_tbl'"),
+        Row("ns1.ns2", "new_tbl", false))
+      checkAnswer(sql(s"SELECT c0 FROM ${catalogAndNamespace}new_tbl"), Row(0))
+    }
+  }
+
+  test("SPARK-34555: Resolve DataFrame metadata column") {
+    val tbl = s"${catalogAndNamespace}table"
+    withTable(tbl) {
+      sql(s"CREATE TABLE $tbl (id bigint, data string) USING $v2Format " +
+        "PARTITIONED BY (bucket(4, id), id)")
+      sql(s"INSERT INTO $tbl VALUES (1, 'a'), (2, 'b'), (3, 'c')")
+      val table = spark.table(tbl)
+      val dfQuery = table.select(
+        table.col("id"),
+        table.col("data"),
+        table.col("index"),
+        table.col("_partition")
+      )
+
+      checkAnswer(
+        dfQuery,
+        Seq(Row(1, "a", 0, "3/1"), Row(2, "b", 0, "0/2"), Row(3, "c", 0, "1/3"))
+      )
+    }
+  }
+
+  test("SPARK-34561: drop/add columns to a dataset of `DESCRIBE TABLE`") {
+    val tbl = s"${catalogAndNamespace}tbl"
+    withTable(tbl) {
+      sql(s"CREATE TABLE $tbl (c0 INT) USING $v2Format")
+      val description = sql(s"DESCRIBE TABLE $tbl")
+      val noCommentDataset = description.drop("comment")
+      val expectedSchema = new StructType()
+        .add(
+          name = "col_name",
+          dataType = StringType,
+          nullable = false,
+          metadata = new MetadataBuilder().putString("comment", "name of the column").build())
+        .add(
+          name = "data_type",
+          dataType = StringType,
+          nullable = false,
+          metadata = new MetadataBuilder().putString("comment", "data type of the column").build())
+      assert(noCommentDataset.schema === expectedSchema)
+      val isNullDataset = noCommentDataset
+        .withColumn("is_null", noCommentDataset("col_name").isNull)
+      assert(isNullDataset.schema === expectedSchema.add("is_null", BooleanType, false))
+    }
+  }
+
+  test("SPARK-34576: drop/add columns to a dataset of `DESCRIBE COLUMN`") {
+    val tbl = s"${catalogAndNamespace}tbl"
+    withTable(tbl) {
+      sql(s"CREATE TABLE $tbl (c0 INT) USING $v2Format")
+      val description = sql(s"DESCRIBE TABLE $tbl c0")
+      val noCommentDataset = description.drop("info_value")
+      val expectedSchema = new StructType()
+        .add(
+          name = "info_name",
+          dataType = StringType,
+          nullable = false,
+          metadata = new MetadataBuilder().putString("comment", "name of the column info").build())
+      assert(noCommentDataset.schema === expectedSchema)
+      val isNullDataset = noCommentDataset
+        .withColumn("is_null", noCommentDataset("info_name").isNull)
+      assert(isNullDataset.schema === expectedSchema.add("is_null", BooleanType, false))
+    }
+  }
+
+  test("SPARK-34577: drop/add columns to a dataset of `DESCRIBE NAMESPACE`") {
+    withNamespace("ns") {
+      sql("CREATE NAMESPACE ns")
+      val description = sql(s"DESCRIBE NAMESPACE ns")
+      val noCommentDataset = description.drop("info_name")
+      val expectedSchema = new StructType()
+        .add(
+          name = "info_value",
+          dataType = StringType,
+          nullable = true,
+          metadata = new MetadataBuilder()
+            .putString("comment", "value of the namespace info").build())
+      assert(noCommentDataset.schema === expectedSchema)
+      val isNullDataset = noCommentDataset
+        .withColumn("is_null", noCommentDataset("info_value").isNull)
+      assert(isNullDataset.schema === expectedSchema.add("is_null", BooleanType, false))
+    }
+  }
+
+  test("SPARK-34923: do not propagate metadata columns through Project") {
+    val t1 = s"${catalogAndNamespace}table"
+    withTable(t1) {
+      sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format " +
+        "PARTITIONED BY (bucket(4, id), id)")
+      sql(s"INSERT INTO $t1 VALUES (1, 'a'), (2, 'b'), (3, 'c')")
+
+      assertThrows[AnalysisException] {
+        sql(s"SELECT index, _partition from (SELECT id, data FROM $t1)")
+      }
+      assertThrows[AnalysisException] {
+        spark.table(t1).select("id", "data").select("index", "_partition")
+      }
+    }
+  }
+
+  test("SPARK-34923: do not propagate metadata columns through View") {
+    val t1 = s"${catalogAndNamespace}table"
+    val view = "view"
+
+    withTable(t1) {
+      withTempView(view) {
+        sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format " +
+          "PARTITIONED BY (bucket(4, id), id)")
+        sql(s"INSERT INTO $t1 VALUES (1, 'a'), (2, 'b'), (3, 'c')")
+        sql(s"CACHE TABLE $view AS SELECT * FROM $t1")
+        assertThrows[AnalysisException] {
+          sql(s"SELECT index, _partition FROM $view")
+        }
+      }
+    }
+  }
+
+  test("SPARK-34923: propagate metadata columns through Filter") {
+    val t1 = s"${catalogAndNamespace}table"
+    withTable(t1) {
+      sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format " +
+        "PARTITIONED BY (bucket(4, id), id)")
+      sql(s"INSERT INTO $t1 VALUES (1, 'a'), (2, 'b'), (3, 'c')")
+
+      val sqlQuery = spark.sql(s"SELECT id, data, index, _partition FROM $t1 WHERE id > 1")
+      val dfQuery = spark.table(t1).where("id > 1").select("id", "data", "index", "_partition")
+
+      Seq(sqlQuery, dfQuery).foreach { query =>
+        checkAnswer(query, Seq(Row(2, "b", 0, "0/2"), Row(3, "c", 0, "1/3")))
+      }
+    }
+  }
+
+  test("SPARK-34923: propagate metadata columns through Sort") {
+    val t1 = s"${catalogAndNamespace}table"
+    withTable(t1) {
+      sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format " +
+        "PARTITIONED BY (bucket(4, id), id)")
+      sql(s"INSERT INTO $t1 VALUES (1, 'a'), (2, 'b'), (3, 'c')")
+
+      val sqlQuery = spark.sql(s"SELECT id, data, index, _partition FROM $t1 ORDER BY id")
+      val dfQuery = spark.table(t1).orderBy("id").select("id", "data", "index", "_partition")
+
+      Seq(sqlQuery, dfQuery).foreach { query =>
+        checkAnswer(query, Seq(Row(1, "a", 0, "3/1"), Row(2, "b", 0, "0/2"), Row(3, "c", 0, "1/3")))
+      }
+    }
+  }
+
+  test("SPARK-34923: propagate metadata columns through RepartitionBy") {
+    val t1 = s"${catalogAndNamespace}table"
+    withTable(t1) {
+      sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format " +
+        "PARTITIONED BY (bucket(4, id), id)")
+      sql(s"INSERT INTO $t1 VALUES (1, 'a'), (2, 'b'), (3, 'c')")
+
+      val sqlQuery = spark.sql(
+        s"SELECT /*+ REPARTITION_BY_RANGE(3, id) */ id, data, index, _partition FROM $t1")
+      val tbl = spark.table(t1)
+      val dfQuery = tbl.repartitionByRange(3, tbl.col("id"))
+        .select("id", "data", "index", "_partition")
+
+      Seq(sqlQuery, dfQuery).foreach { query =>
+        checkAnswer(query, Seq(Row(1, "a", 0, "3/1"), Row(2, "b", 0, "0/2"), Row(3, "c", 0, "1/3")))
+      }
+    }
+  }
+
+  test("SPARK-34923: propagate metadata columns through SubqueryAlias") {
+    val t1 = s"${catalogAndNamespace}table"
+    val sbq = "sbq"
+    withTable(t1) {
+      sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format " +
+        "PARTITIONED BY (bucket(4, id), id)")
+      sql(s"INSERT INTO $t1 VALUES (1, 'a'), (2, 'b'), (3, 'c')")
+
+      val sqlQuery = spark.sql(
+        s"SELECT $sbq.id, $sbq.data, $sbq.index, $sbq._partition FROM $t1 as $sbq")
+      val dfQuery = spark.table(t1).as(sbq).select(
+        s"$sbq.id", s"$sbq.data", s"$sbq.index", s"$sbq._partition")
+
+      Seq(sqlQuery, dfQuery).foreach { query =>
+        checkAnswer(query, Seq(Row(1, "a", 0, "3/1"), Row(2, "b", 0, "0/2"), Row(3, "c", 0, "1/3")))
+      }
     }
-    assert(e.message.contains(s"$sqlCommand is only supported with v1 tables"))
   }
 
-  private def testV1CommandSupportingTempView(sqlCommand: String, sqlParams: String): Unit = {
+  private def testNotSupportedV2Command(sqlCommand: String, sqlParams: String): Unit = {
     val e = intercept[AnalysisException] {
       sql(s"$sqlCommand $sqlParams")
     }
-    assert(e.message.contains(s"$sqlCommand is only supported with temp views or v1 tables"))
+    assert(e.message.contains(s"$sqlCommand is not supported for v2 tables"))
   }
 
   private def assertAnalysisError(sqlStatement: String, expectedError: String): Unit = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala
index 2d8761f872da7..49a1078800552 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala
@@ -157,6 +157,19 @@ class DataSourceV2Suite extends QueryTest with SharedSparkSession with AdaptiveS
     }
   }
 
+  test("SPARK-33369: Skip schema inference in DataframeWriter.save() if table provider " +
+    "supports external metadata") {
+    withTempDir { dir =>
+      val cls = classOf[SupportsExternalMetadataWritableDataSource].getName
+      spark.range(10).select('id as 'i, -'id as 'j).write.format(cls)
+          .option("path", dir.getCanonicalPath).mode("append").save()
+      val schema = new StructType().add("i", "long").add("j", "long")
+        checkAnswer(
+          spark.read.format(cls).option("path", dir.getCanonicalPath).schema(schema).load(),
+          spark.range(10).select('id, -'id))
+    }
+  }
+
   test("partitioning reporting") {
     import org.apache.spark.sql.functions.{count, sum}
     Seq(classOf[PartitionAwareDataSource], classOf[JavaPartitionAwareDataSource]).foreach { cls =>
@@ -215,8 +228,7 @@ class DataSourceV2Suite extends QueryTest with SharedSparkSession with AdaptiveS
   }
 
   test("simple writable data source") {
-    // TODO: java implementation.
-    Seq(classOf[SimpleWritableDataSource]).foreach { cls =>
+    Seq(classOf[SimpleWritableDataSource], classOf[JavaSimpleWritableDataSource]).foreach { cls =>
       withTempPath { file =>
         val path = file.getCanonicalPath
         assert(spark.read.format(cls.getName).option("path", path).load().collect().isEmpty)
@@ -268,7 +280,7 @@ class DataSourceV2Suite extends QueryTest with SharedSparkSession with AdaptiveS
           }
         }
         // this input data will fail to read middle way.
-        val input = spark.range(10).select(failingUdf('id).as('i)).select('i, -'i as 'j)
+        val input = spark.range(15).select(failingUdf('id).as('i)).select('i, -'i as 'j)
         val e3 = intercept[SparkException] {
           input.write.format(cls.getName).option("path", path).mode("overwrite").save()
         }
@@ -394,6 +406,35 @@ class DataSourceV2Suite extends QueryTest with SharedSparkSession with AdaptiveS
       checkAnswer(df, (0 until 3).map(i => Row(i)))
     }
   }
+
+  test("SPARK-32609: DataSourceV2 with different pushedfilters should be different") {
+    def getScanExec(query: DataFrame): BatchScanExec = {
+      query.queryExecution.executedPlan.collect {
+        case d: BatchScanExec => d
+      }.head
+    }
+
+    Seq(classOf[AdvancedDataSourceV2], classOf[JavaAdvancedDataSourceV2]).foreach { cls =>
+      withClue(cls.getName) {
+        val df = spark.read.format(cls.getName).load()
+        val q1 = df.select('i).filter('i > 6)
+        val q2 = df.select('i).filter('i > 5)
+        val scan1 = getScanExec(q1)
+        val scan2 = getScanExec(q2)
+        assert(!scan1.equals(scan2))
+      }
+    }
+  }
+
+  test("SPARK-33267: push down with condition 'in (..., null)' should not throw NPE") {
+    Seq(classOf[AdvancedDataSourceV2], classOf[JavaAdvancedDataSourceV2]).foreach { cls =>
+      withClue(cls.getName) {
+        val df = spark.read.format(cls.getName).load()
+        // before SPARK-33267 below query just threw NPE
+        df.select('i).where("i in (1, null)").collect()
+      }
+    }
+  }
 }
 
 
@@ -742,6 +783,16 @@ class SimpleWriteOnlyDataSource extends SimpleWritableDataSource {
   }
 }
 
+class SupportsExternalMetadataWritableDataSource extends SimpleWritableDataSource {
+  override def supportsExternalMetadata(): Boolean = true
+
+  override def inferSchema(options: CaseInsensitiveStringMap): StructType = {
+    throw new IllegalArgumentException(
+      "Dataframe writer should not require inferring table schema the data source supports" +
+        " external metadata.")
+  }
+}
+
 class ReportStatisticsDataSource extends SimpleWritableDataSource {
 
   class MyScanBuilder extends SimpleScanBuilder
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2UtilsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2UtilsSuite.scala
index 01fcced5b12a8..a58bab276a41b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2UtilsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2UtilsSuite.scala
@@ -37,8 +37,8 @@ class DataSourceV2UtilsSuite extends SparkFunSuite {
     val source = new DataSourceV2WithSessionConfig
     val confs = DataSourceV2Utils.extractSessionConfigs(source, conf)
     assert(confs.size == 2)
-    assert(confs.keySet.filter(_.startsWith("spark.datasource")).size == 0)
-    assert(confs.keySet.filter(_.startsWith("not.exist.prefix")).size == 0)
+    assert(!confs.keySet.exists(_.startsWith("spark.datasource")))
+    assert(!confs.keySet.exists(_.startsWith("not.exist.prefix")))
     assert(confs.keySet.contains("foo.bar"))
     assert(confs.keySet.contains("whateverConfigName"))
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DatasourceV2SQLBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DatasourceV2SQLBase.scala
new file mode 100644
index 0000000000000..77a515b55ce76
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DatasourceV2SQLBase.scala
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector
+
+import org.scalatest.BeforeAndAfter
+
+import org.apache.spark.sql.QueryTest
+import org.apache.spark.sql.connector.catalog.{CatalogPlugin, InMemoryCatalog, InMemoryPartitionTableCatalog, StagingInMemoryTableCatalog}
+import org.apache.spark.sql.internal.SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION
+import org.apache.spark.sql.test.SharedSparkSession
+
+trait DatasourceV2SQLBase
+  extends QueryTest with SharedSparkSession with BeforeAndAfter {
+
+  protected def catalog(name: String): CatalogPlugin = {
+    spark.sessionState.catalogManager.catalog(name)
+  }
+
+  before {
+    spark.conf.set("spark.sql.catalog.testcat", classOf[InMemoryCatalog].getName)
+    spark.conf.set("spark.sql.catalog.testpart", classOf[InMemoryPartitionTableCatalog].getName)
+    spark.conf.set(
+      "spark.sql.catalog.testcat_atomic", classOf[StagingInMemoryTableCatalog].getName)
+    spark.conf.set("spark.sql.catalog.testcat2", classOf[InMemoryCatalog].getName)
+    spark.conf.set(
+      V2_SESSION_CATALOG_IMPLEMENTATION.key, classOf[InMemoryTableSessionCatalog].getName)
+
+    val df = spark.createDataFrame(Seq((1L, "a"), (2L, "b"), (3L, "c"))).toDF("id", "data")
+    df.createOrReplaceTempView("source")
+    val df2 = spark.createDataFrame(Seq((4L, "d"), (5L, "e"), (6L, "f"))).toDF("id", "data")
+    df2.createOrReplaceTempView("source2")
+  }
+
+  after {
+    spark.sessionState.catalog.reset()
+    spark.sessionState.catalogManager.reset()
+    spark.sessionState.conf.clear()
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala
index b88ad5218fcd2..ad730376b2e3a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala
@@ -446,21 +446,18 @@ trait InsertIntoSQLOnlyTests
       }
     }
 
-    test("InsertInto: overwrite - multiple static partitions - dynamic mode") {
-      // Since all partitions are provided statically, this should be supported by everyone
-      withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.DYNAMIC.toString) {
-        val t1 = s"${catalogAndNamespace}tbl"
-        withTableAndData(t1) { view =>
-          sql(s"CREATE TABLE $t1 (id bigint, data string, p int) " +
-            s"USING $v2Format PARTITIONED BY (id, p)")
-          sql(s"INSERT INTO $t1 VALUES (2L, 'dummy', 2), (4L, 'keep', 2)")
-          sql(s"INSERT OVERWRITE TABLE $t1 PARTITION (id = 2, p = 2) SELECT data FROM $view")
-          verifyTable(t1, Seq(
-            (2, "a", 2),
-            (2, "b", 2),
-            (2, "c", 2),
-            (4, "keep", 2)).toDF("id", "data", "p"))
-        }
+    dynamicOverwriteTest("InsertInto: overwrite - multiple static partitions - dynamic mode") {
+      val t1 = s"${catalogAndNamespace}tbl"
+      withTableAndData(t1) { view =>
+        sql(s"CREATE TABLE $t1 (id bigint, data string, p int) " +
+          s"USING $v2Format PARTITIONED BY (id, p)")
+        sql(s"INSERT INTO $t1 VALUES (2L, 'dummy', 2), (4L, 'keep', 2)")
+        sql(s"INSERT OVERWRITE TABLE $t1 PARTITION (id = 2, p = 2) SELECT data FROM $view")
+        verifyTable(t1, Seq(
+          (2, "a", 2),
+          (2, "b", 2),
+          (2, "c", 2),
+          (4, "keep", 2)).toDF("id", "data", "p"))
       }
     }
 
@@ -480,5 +477,15 @@ trait InsertIntoSQLOnlyTests
         verifyTable(t1, spark.table(view))
       }
     }
+
+    test("SPARK-34599: InsertInto: overwrite - dot in the partition column name - static mode") {
+      import testImplicits._
+      val t1 = "tbl"
+      withTable(t1) {
+        sql(s"CREATE TABLE $t1 (`a.b` string, `c.d` string) USING $v2Format PARTITIONED BY (`a.b`)")
+        sql(s"INSERT OVERWRITE $t1 PARTITION (`a.b` = 'a') (`c.d`) VALUES('b')")
+        verifyTable(t1, Seq("a" -> "b").toDF("id", "data"))
+      }
+    }
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/SimpleWritableDataSource.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/SimpleWritableDataSource.scala
index f9306ba28e7f2..49a6742a85269 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/SimpleWritableDataSource.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/SimpleWritableDataSource.scala
@@ -27,12 +27,10 @@ import org.apache.hadoop.fs.{FileSystem, Path}
 
 import org.apache.spark.SparkContext
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.connector.catalog.{SessionConfigSupport, SupportsWrite, Table, TableCapability}
+import org.apache.spark.sql.connector.catalog.{SupportsWrite, Table, TableCapability}
 import org.apache.spark.sql.connector.catalog.TableCapability._
 import org.apache.spark.sql.connector.read.{InputPartition, PartitionReader, PartitionReaderFactory, ScanBuilder}
 import org.apache.spark.sql.connector.write._
-import org.apache.spark.sql.internal.connector.SimpleTableProvider
-import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 import org.apache.spark.util.SerializableConfiguration
 
@@ -41,11 +39,7 @@ import org.apache.spark.util.SerializableConfiguration
  * Each task writes data to `target/_temporary/uniqueId/$jobId-$partitionId-$attemptNumber`.
  * Each job moves files from `target/_temporary/uniqueId/` to `target`.
  */
-class SimpleWritableDataSource extends SimpleTableProvider with SessionConfigSupport {
-
-  private val tableSchema = new StructType().add("i", "long").add("j", "long")
-
-  override def keyPrefix: String = "simpleWritableDataSource"
+class SimpleWritableDataSource extends TestingV2Source {
 
   class MyScanBuilder(path: String, conf: Configuration) extends SimpleScanBuilder {
     override def planInputPartitions(): Array[InputPartition] = {
@@ -67,8 +61,6 @@ class SimpleWritableDataSource extends SimpleTableProvider with SessionConfigSup
       val serializableConf = new SerializableConfiguration(conf)
       new CSVReaderFactory(serializableConf)
     }
-
-    override def readSchema(): StructType = tableSchema
   }
 
   class MyWriteBuilder(path: String, info: LogicalWriteInfo)
@@ -81,17 +73,21 @@ class SimpleWritableDataSource extends SimpleTableProvider with SessionConfigSup
       this
     }
 
-    override def buildForBatch(): BatchWrite = {
-      val hadoopPath = new Path(path)
-      val hadoopConf = SparkContext.getActive.get.hadoopConfiguration
-      val fs = hadoopPath.getFileSystem(hadoopConf)
+    override def build(): Write = {
+      new Write {
+        override def toBatch: BatchWrite = {
+          val hadoopPath = new Path(path)
+          val hadoopConf = SparkContext.getActive.get.hadoopConfiguration
+          val fs = hadoopPath.getFileSystem(hadoopConf)
 
-      if (needTruncate) {
-        fs.delete(hadoopPath, true)
-      }
+          if (needTruncate) {
+            fs.delete(hadoopPath, true)
+          }
 
-      val pathStr = hadoopPath.toUri.toString
-      new MyBatchWrite(queryId, pathStr, hadoopConf)
+          val pathStr = hadoopPath.toUri.toString
+          new MyBatchWrite(queryId, pathStr, hadoopConf)
+        }
+      }
     }
   }
 
@@ -134,8 +130,6 @@ class SimpleWritableDataSource extends SimpleTableProvider with SessionConfigSup
     private val path = options.get("path")
     private val conf = SparkContext.getActive.get.hadoopConfiguration
 
-    override def schema(): StructType = tableSchema
-
     override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = {
       new MyScanBuilder(new Path(path).toUri.toString, conf)
     }
@@ -179,7 +173,7 @@ class CSVReaderFactory(conf: SerializableConfiguration)
         }
       }
 
-      override def get(): InternalRow = InternalRow(currentLine.split(",").map(_.trim.toLong): _*)
+      override def get(): InternalRow = InternalRow(currentLine.split(",").map(_.trim.toInt): _*)
 
       override def close(): Unit = {
         inputStream.close()
@@ -222,7 +216,7 @@ class CSVDataWriter(fs: FileSystem, file: Path) extends DataWriter[InternalRow]
   private val out = fs.create(file)
 
   override def write(record: InternalRow): Unit = {
-    out.writeBytes(s"${record.getLong(0)},${record.getLong(1)}\n")
+    out.writeBytes(s"${record.getInt(0)},${record.getInt(1)}\n")
   }
 
   override def commit(): WriterCommitMessage = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala
index 550bec7505422..076dad7530807 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala
@@ -22,12 +22,13 @@ import scala.util.Try
 
 import org.scalatest.BeforeAndAfter
 
+import org.apache.spark.SparkException
 import org.apache.spark.sql.{DataFrame, QueryTest, SaveMode}
 import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
 import org.apache.spark.sql.catalyst.plans.logical.{AppendData, LogicalPlan, OverwriteByExpression}
-import org.apache.spark.sql.connector.catalog.{Identifier, SupportsCatalogOptions, TableCatalog}
+import org.apache.spark.sql.connector.catalog.{Identifier, InMemoryTableCatalog, SupportsCatalogOptions, TableCatalog}
 import org.apache.spark.sql.connector.catalog.CatalogManager.SESSION_CATALOG_NAME
-import org.apache.spark.sql.connector.expressions.{FieldReference, IdentityTransform, Transform}
+import org.apache.spark.sql.connector.expressions.{FieldReference, IdentityTransform}
 import org.apache.spark.sql.execution.QueryExecution
 import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
 import org.apache.spark.sql.internal.SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION
@@ -254,6 +255,22 @@ class SupportsCatalogOptionsSuite extends QueryTest with SharedSparkSession with
     }
   }
 
+  test("SPARK-33240: fail the query when instantiation on session catalog fails") {
+    try {
+      spark.sessionState.catalogManager.reset()
+      spark.conf.set(
+        V2_SESSION_CATALOG_IMPLEMENTATION.key, "InvalidCatalogClass")
+      val e = intercept[SparkException] {
+        sql(s"create table t1 (id bigint) using $format")
+      }
+
+      assert(e.getMessage.contains("Cannot find catalog plugin class"))
+      assert(e.getMessage.contains("InvalidCatalogClass"))
+    } finally {
+      spark.sessionState.catalogManager.reset()
+    }
+  }
+
   private def checkV2Identifiers(
       plan: LogicalPlan,
       identifier: String = "t1",
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/TableCapabilityCheckSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/TableCapabilityCheckSuite.scala
index 23e4c293cbc28..ce94d3b5c2fc0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/TableCapabilityCheckSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/TableCapabilityCheckSuite.scala
@@ -22,30 +22,33 @@ import java.util
 import scala.collection.JavaConverters._
 
 import org.apache.spark.sql.{AnalysisException, DataFrame, SQLContext}
-import org.apache.spark.sql.catalyst.analysis.{AnalysisSuite, NamedRelation}
+import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, NamedRelation}
 import org.apache.spark.sql.catalyst.expressions.{AttributeReference, EqualTo, Literal}
 import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.connector.catalog.{CatalogPlugin, Identifier, Table, TableCapability, TableProvider}
+import org.apache.spark.sql.catalyst.streaming.StreamingRelationV2
+import org.apache.spark.sql.connector.catalog.{Table, TableCapability}
 import org.apache.spark.sql.connector.catalog.TableCapability._
 import org.apache.spark.sql.execution.datasources.DataSource
 import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Relation, TableCapabilityCheck}
-import org.apache.spark.sql.execution.streaming.{Offset, Source, StreamingRelation, StreamingRelationV2}
+import org.apache.spark.sql.execution.streaming.{Offset, Source, StreamingRelation}
 import org.apache.spark.sql.sources.StreamSourceProvider
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.{LongType, StringType, StructType}
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
-class TableCapabilityCheckSuite extends AnalysisSuite with SharedSparkSession {
+class TableCapabilityCheckSuite extends AnalysisTest with SharedSparkSession {
 
   private val emptyMap = CaseInsensitiveStringMap.empty
   private def createStreamingRelation(table: Table, v1Relation: Option[StreamingRelation]) = {
     StreamingRelationV2(
-      new FakeV2Provider,
+      Some(new FakeV2Provider),
       "fake",
       table,
       CaseInsensitiveStringMap.empty(),
       TableCapabilityCheckSuite.schema.toAttributes,
-      v1Relation)(spark)
+      None,
+      None,
+      v1Relation)
   }
 
   private def createStreamingRelationV1() = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/TestV2SessionCatalogBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/TestV2SessionCatalogBase.scala
index 637cf2fd16515..bf2749d1afc53 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/TestV2SessionCatalogBase.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/TestV2SessionCatalogBase.scala
@@ -19,11 +19,12 @@ package org.apache.spark.sql.connector
 
 import java.util
 import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.atomic.AtomicBoolean
 
 import scala.collection.JavaConverters._
 
-import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
-import org.apache.spark.sql.connector.catalog.{DelegatingCatalogExtension, Identifier, Table}
+import org.apache.spark.sql.catalyst.catalog.CatalogTableType
+import org.apache.spark.sql.connector.catalog.{DelegatingCatalogExtension, Identifier, Table, TableCatalog, V1Table}
 import org.apache.spark.sql.connector.expressions.Transform
 import org.apache.spark.sql.types.StructType
 
@@ -36,6 +37,13 @@ private[connector] trait TestV2SessionCatalogBase[T <: Table] extends Delegating
 
   protected val tables: util.Map[Identifier, T] = new ConcurrentHashMap[Identifier, T]()
 
+  private val tableCreated: AtomicBoolean = new AtomicBoolean(false)
+
+  private def addTable(ident: Identifier, table: T): Unit = {
+    tableCreated.set(true)
+    tables.put(ident, table)
+  }
+
   protected def newTable(
       name: String,
       schema: StructType,
@@ -47,10 +55,13 @@ private[connector] trait TestV2SessionCatalogBase[T <: Table] extends Delegating
       tables.get(ident)
     } else {
       // Table was created through the built-in catalog
-      val t = super.loadTable(ident)
-      val table = newTable(t.name(), t.schema(), t.partitioning(), t.properties())
-      tables.put(ident, table)
-      table
+      super.loadTable(ident) match {
+        case v1Table: V1Table if v1Table.v1Table.tableType == CatalogTableType.VIEW => v1Table
+        case t =>
+          val table = newTable(t.name(), t.schema(), t.partitioning(), t.properties())
+          addTable(ident, table)
+          table
+      }
     }
   }
 
@@ -59,9 +70,23 @@ private[connector] trait TestV2SessionCatalogBase[T <: Table] extends Delegating
       schema: StructType,
       partitions: Array[Transform],
       properties: util.Map[String, String]): Table = {
-    val created = super.createTable(ident, schema, partitions, properties)
-    val t = newTable(created.name(), schema, partitions, properties)
-    tables.put(ident, t)
+    val key = TestV2SessionCatalogBase.SIMULATE_ALLOW_EXTERNAL_PROPERTY
+    val propsWithLocation = if (properties.containsKey(key)) {
+      // Always set a location so that CREATE EXTERNAL TABLE won't fail with LOCATION not specified.
+      if (!properties.containsKey(TableCatalog.PROP_LOCATION)) {
+        val newProps = new util.HashMap[String, String]()
+        newProps.putAll(properties)
+        newProps.put(TableCatalog.PROP_LOCATION, "file:/abc")
+        newProps
+      } else {
+        properties
+      }
+    } else {
+      properties
+    }
+    val created = super.createTable(ident, schema, partitions, propsWithLocation)
+    val t = newTable(created.name(), schema, partitions, propsWithLocation)
+    addTable(ident, t)
     t
   }
 
@@ -71,8 +96,15 @@ private[connector] trait TestV2SessionCatalogBase[T <: Table] extends Delegating
   }
 
   def clearTables(): Unit = {
-    assert(!tables.isEmpty, "Tables were empty, maybe didn't use the session catalog code path?")
+    assert(
+      tableCreated.get,
+      "Tables are not created, maybe didn't use the session catalog code path?")
     tables.keySet().asScala.foreach(super.dropTable)
     tables.clear()
+    tableCreated.set(false)
   }
 }
+
+object TestV2SessionCatalogBase {
+  val SIMULATE_ALLOW_EXTERNAL_PROPERTY = "spark.sql.test.simulateAllowExternal"
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/V1ReadFallbackSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/V1ReadFallbackSuite.scala
index 74f2ca14234d2..847953e09cef7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/V1ReadFallbackSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/V1ReadFallbackSuite.scala
@@ -23,7 +23,7 @@ import scala.collection.JavaConverters._
 
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{DataFrame, QueryTest, Row, SparkSession, SQLContext}
-import org.apache.spark.sql.connector.catalog.{Identifier, SupportsRead, Table, TableCapability, TableProvider}
+import org.apache.spark.sql.connector.catalog.{BasicInMemoryTableCatalog, Identifier, SupportsRead, Table, TableCapability}
 import org.apache.spark.sql.connector.expressions.Transform
 import org.apache.spark.sql.connector.read.{Scan, ScanBuilder, SupportsPushDownFilters, SupportsPushDownRequiredColumns, V1Scan}
 import org.apache.spark.sql.execution.RowDataSourceScanExec
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/V1WriteFallbackSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/V1WriteFallbackSuite.scala
index 10ed2048dbf61..7effc747ab323 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/V1WriteFallbackSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/V1WriteFallbackSuite.scala
@@ -24,11 +24,18 @@ import scala.collection.mutable
 
 import org.scalatest.BeforeAndAfter
 
+import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row, SaveMode, SparkSession, SQLContext}
-import org.apache.spark.sql.connector.catalog.{SupportsWrite, Table, TableCapability}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreeNodeTag
+import org.apache.spark.sql.connector.catalog.{Identifier, InMemoryTable, SupportsRead, SupportsWrite, Table, TableCapability}
 import org.apache.spark.sql.connector.expressions.{FieldReference, IdentityTransform, Transform}
-import org.apache.spark.sql.connector.write.{LogicalWriteInfo, LogicalWriteInfoImpl, SupportsOverwrite, SupportsTruncate, V1WriteBuilder, WriteBuilder}
+import org.apache.spark.sql.connector.read.{Scan, ScanBuilder, V1Scan}
+import org.apache.spark.sql.connector.write.{LogicalWriteInfo, LogicalWriteInfoImpl, SupportsOverwrite, SupportsTruncate, V1Write, WriteBuilder}
 import org.apache.spark.sql.execution.datasources.DataSourceUtils
+import org.apache.spark.sql.functions.lit
+import org.apache.spark.sql.internal.SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION
 import org.apache.spark.sql.internal.connector.SimpleTableProvider
 import org.apache.spark.sql.sources._
 import org.apache.spark.sql.test.SharedSparkSession
@@ -124,6 +131,69 @@ class V1WriteFallbackSuite extends QueryTest with SharedSparkSession with Before
     }
     assert(e3.getMessage.contains("schema"))
   }
+
+  test("fallback writes should only analyze plan once") {
+    SparkSession.clearActiveSession()
+    SparkSession.clearDefaultSession()
+    try {
+      val session = SparkSession.builder()
+        .master("local[1]")
+        .withExtensions(_.injectPostHocResolutionRule(_ => OnlyOnceRule))
+        .config(V2_SESSION_CATALOG_IMPLEMENTATION.key, classOf[V1FallbackTableCatalog].getName)
+        .getOrCreate()
+      val df = session.createDataFrame(Seq((1, "x"), (2, "y"), (3, "z")))
+      df.write.mode("append").option("name", "t1").format(v2Format).saveAsTable("test")
+    } finally {
+      SparkSession.setActiveSession(spark)
+      SparkSession.setDefaultSession(spark)
+    }
+  }
+
+  test("SPARK-33492: append fallback should refresh cache") {
+    SparkSession.clearActiveSession()
+    SparkSession.clearDefaultSession()
+    try {
+      val session = SparkSession.builder()
+        .master("local[1]")
+        .config(V2_SESSION_CATALOG_IMPLEMENTATION.key, classOf[V1FallbackTableCatalog].getName)
+        .getOrCreate()
+      val df = session.createDataFrame(Seq((1, "x")))
+      df.write.mode("append").option("name", "t1").format(v2Format).saveAsTable("test")
+      session.catalog.cacheTable("test")
+      checkAnswer(session.read.table("test"), Row(1, "x") :: Nil)
+
+      val df2 = session.createDataFrame(Seq((2, "y")))
+      df2.writeTo("test").append()
+      checkAnswer(session.read.table("test"), Row(1, "x") :: Row(2, "y") :: Nil)
+
+    } finally {
+      SparkSession.setActiveSession(spark)
+      SparkSession.setDefaultSession(spark)
+    }
+  }
+
+  test("SPARK-33492: overwrite fallback should refresh cache") {
+    SparkSession.clearActiveSession()
+    SparkSession.clearDefaultSession()
+    try {
+      val session = SparkSession.builder()
+        .master("local[1]")
+        .config(V2_SESSION_CATALOG_IMPLEMENTATION.key, classOf[V1FallbackTableCatalog].getName)
+        .getOrCreate()
+      val df = session.createDataFrame(Seq((1, "x")))
+      df.write.mode("append").option("name", "t1").format(v2Format).saveAsTable("test")
+      session.catalog.cacheTable("test")
+      checkAnswer(session.read.table("test"), Row(1, "x") :: Nil)
+
+      val df2 = session.createDataFrame(Seq((2, "y")))
+      df2.writeTo("test").overwrite(lit(true))
+      checkAnswer(session.read.table("test"), Row(2, "y") :: Nil)
+
+    } finally {
+      SparkSession.setActiveSession(spark)
+      SparkSession.setDefaultSession(spark)
+    }
+  }
 }
 
 class V1WriteFallbackSessionCatalogSuite
@@ -156,6 +226,7 @@ class V1FallbackTableCatalog extends TestV2SessionCatalogBase[InMemoryTableWithV
       properties: util.Map[String, String]): InMemoryTableWithV1Fallback = {
     val t = new InMemoryTableWithV1Fallback(name, schema, partitions, properties)
     InMemoryV1Provider.tables.put(name, t)
+    tables.put(Identifier.of(Array("default"), name), t)
     t
   }
 }
@@ -240,7 +311,8 @@ class InMemoryV1Provider
     if (mode == SaveMode.Overwrite) {
       writer.asInstanceOf[SupportsTruncate].truncate()
     }
-    writer.asInstanceOf[V1WriteBuilder].buildForV1Write().insert(data, overwrite = false)
+    val write = writer.build()
+    write.asInstanceOf[V1Write].toInsertableRelation.insert(data, overwrite = false)
     getRelation
   }
 }
@@ -251,7 +323,7 @@ class InMemoryTableWithV1Fallback(
     override val partitioning: Array[Transform],
     override val properties: util.Map[String, String])
   extends Table
-  with SupportsWrite {
+  with SupportsWrite with SupportsRead {
 
   partitioning.foreach { t =>
     if (!t.isInstanceOf[IdentityTransform]) {
@@ -260,6 +332,7 @@ class InMemoryTableWithV1Fallback(
   }
 
   override def capabilities: util.Set[TableCapability] = Set(
+    TableCapability.BATCH_READ,
     TableCapability.V1_BATCH_WRITE,
     TableCapability.OVERWRITE_BY_FILTER,
     TableCapability.TRUNCATE).asJava
@@ -276,7 +349,6 @@ class InMemoryTableWithV1Fallback(
 
   private class FallbackWriteBuilder(options: CaseInsensitiveStringMap)
     extends WriteBuilder
-    with V1WriteBuilder
     with SupportsTruncate
     with SupportsOverwrite {
 
@@ -299,9 +371,9 @@ class InMemoryTableWithV1Fallback(
       partIndexes.map(row.get)
     }
 
-    override def buildForV1Write(): InsertableRelation = {
-      new InsertableRelation {
-        override def insert(data: DataFrame, overwrite: Boolean): Unit = {
+    override def build(): V1Write = new V1Write {
+      override def toInsertableRelation: InsertableRelation = {
+        (data: DataFrame, overwrite: Boolean) => {
           assert(!overwrite, "V1 write fallbacks cannot be called with overwrite=true")
           val rows = data.collect()
           rows.groupBy(getPartitionValues).foreach { case (partition, elements) =>
@@ -317,4 +389,49 @@ class InMemoryTableWithV1Fallback(
       }
     }
   }
+
+  override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder =
+    new V1ReadFallbackScanBuilder(schema)
+
+  private class V1ReadFallbackScanBuilder(schema: StructType) extends ScanBuilder {
+    override def build(): Scan = new V1ReadFallbackScan(schema)
+  }
+
+  private class V1ReadFallbackScan(schema: StructType) extends V1Scan {
+    override def readSchema(): StructType = schema
+    override def toV1TableScan[T <: BaseRelation with TableScan](context: SQLContext): T =
+      new V1TableScan(context, schema).asInstanceOf[T]
+  }
+
+  private class V1TableScan(
+      context: SQLContext,
+      requiredSchema: StructType) extends BaseRelation with TableScan {
+    override def sqlContext: SQLContext = context
+    override def schema: StructType = requiredSchema
+    override def buildScan(): RDD[Row] = {
+      val data = InMemoryV1Provider.getTableData(context.sparkSession, name).collect()
+      context.sparkContext.makeRDD(data)
+    }
+  }
+}
+
+/** A rule that fails if a query plan is analyzed twice. */
+object OnlyOnceRule extends Rule[LogicalPlan] {
+  private val tag = TreeNodeTag[String]("test")
+  private val counts = new mutable.HashMap[LogicalPlan, Int]()
+
+  override def apply(plan: LogicalPlan): LogicalPlan = {
+    if (plan.getTagValue(tag).isEmpty) {
+      plan.setTagValue(tag, "abc")
+      plan
+    } else {
+      val cnt = counts.getOrElseUpdate(plan, 0) + 1
+      // This rule will be run as injectPostHocResolutionRule, and is supposed to be run only twice.
+      // Once during planning and once during checkBatchIdempotence
+      assert(cnt <= 1, "This rule shouldn't have been called again")
+      counts.put(plan, cnt)
+      plan
+    }
+
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/WriteDistributionAndOrderingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/WriteDistributionAndOrderingSuite.scala
new file mode 100644
index 0000000000000..db4a9c153c0ff
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/WriteDistributionAndOrderingSuite.scala
@@ -0,0 +1,801 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector
+
+import java.util.Collections
+
+import org.scalatest.BeforeAndAfter
+
+import org.apache.spark.sql.{catalyst, AnalysisException, DataFrame, QueryTest}
+import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
+import org.apache.spark.sql.catalyst.plans.physical
+import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, RangePartitioning, UnknownPartitioning}
+import org.apache.spark.sql.connector.catalog.{Identifier, InMemoryTableCatalog}
+import org.apache.spark.sql.connector.distributions.{Distribution, Distributions}
+import org.apache.spark.sql.connector.expressions.{Expression, FieldReference, NullOrdering, SortDirection, SortOrder}
+import org.apache.spark.sql.connector.expressions.LogicalExpressions._
+import org.apache.spark.sql.execution.{QueryExecution, SortExec, SparkPlan}
+import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
+import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec
+import org.apache.spark.sql.execution.exchange.ShuffleExchangeLike
+import org.apache.spark.sql.functions.lit
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.{IntegerType, StringType, StructType}
+import org.apache.spark.sql.util.QueryExecutionListener
+
+class WriteDistributionAndOrderingSuite
+  extends QueryTest with SharedSparkSession with BeforeAndAfter with AdaptiveSparkPlanHelper {
+
+  import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+  before {
+    spark.conf.set("spark.sql.catalog.testcat", classOf[InMemoryTableCatalog].getName)
+  }
+
+  after {
+    spark.sessionState.catalogManager.reset()
+    spark.sessionState.conf.unsetConf("spark.sql.catalog.testcat")
+  }
+
+  private val namespace = Array("ns1")
+  private val ident = Identifier.of(namespace, "test_table")
+  private val tableNameAsString = "testcat." + ident.toString
+  private val emptyProps = Collections.emptyMap[String, String]
+  private val schema = new StructType()
+    .add("id", IntegerType)
+    .add("data", StringType)
+
+  private val resolver = conf.resolver
+
+  test("ordered distribution and sort with same exprs: append") {
+    checkOrderedDistributionAndSortWithSameExprs("append")
+  }
+
+  test("ordered distribution and sort with same exprs: overwrite") {
+    checkOrderedDistributionAndSortWithSameExprs("overwrite")
+  }
+
+  test("ordered distribution and sort with same exprs: overwriteDynamic") {
+    checkOrderedDistributionAndSortWithSameExprs("overwriteDynamic")
+  }
+
+  test("ordered distribution and sort with same exprs with numPartitions: append") {
+    checkOrderedDistributionAndSortWithSameExprs("append", Some(10))
+  }
+
+  test("ordered distribution and sort with same exprs with numPartitions: overwrite") {
+    checkOrderedDistributionAndSortWithSameExprs("overwrite", Some(10))
+  }
+
+  test("ordered distribution and sort with same exprs with numPartitions: overwriteDynamic") {
+    checkOrderedDistributionAndSortWithSameExprs("overwriteDynamic", Some(10))
+  }
+
+  private def checkOrderedDistributionAndSortWithSameExprs(command: String): Unit = {
+    checkOrderedDistributionAndSortWithSameExprs(command, None)
+  }
+
+  private def checkOrderedDistributionAndSortWithSameExprs(
+      command: String,
+      targetNumPartitions: Option[Int]): Unit = {
+    val tableOrdering = Array[SortOrder](
+      sort(FieldReference("data"), SortDirection.ASCENDING, NullOrdering.NULLS_FIRST)
+    )
+    val tableDistribution = Distributions.ordered(tableOrdering)
+
+    val writeOrdering = Seq(
+      catalyst.expressions.SortOrder(
+        attr("data"),
+        catalyst.expressions.Ascending,
+        catalyst.expressions.NullsFirst,
+        Seq.empty
+      )
+    )
+    val writePartitioning = orderedWritePartitioning(writeOrdering, targetNumPartitions)
+
+    checkWriteRequirements(
+      tableDistribution,
+      tableOrdering,
+      targetNumPartitions,
+      expectedWritePartitioning = writePartitioning,
+      expectedWriteOrdering = writeOrdering,
+      writeCommand = command)
+  }
+
+  test("clustered distribution and sort with same exprs: append") {
+    checkClusteredDistributionAndSortWithSameExprs("append")
+  }
+
+  test("clustered distribution and sort with same exprs: overwrite") {
+    checkClusteredDistributionAndSortWithSameExprs("overwrite")
+  }
+
+  test("clustered distribution and sort with same exprs: overwriteDynamic") {
+    checkClusteredDistributionAndSortWithSameExprs("overwriteDynamic")
+  }
+
+  test("clustered distribution and sort with same exprs with numPartitions: append") {
+    checkClusteredDistributionAndSortWithSameExprs("append", Some(10))
+  }
+
+  test("clustered distribution and sort with same exprs with numPartitions: overwrite") {
+    checkClusteredDistributionAndSortWithSameExprs("overwrite", Some(10))
+  }
+
+  test("clustered distribution and sort with same exprs with numPartitions: overwriteDynamic") {
+    checkClusteredDistributionAndSortWithSameExprs("overwriteDynamic", Some(10))
+  }
+
+  private def checkClusteredDistributionAndSortWithSameExprs(command: String): Unit = {
+    checkClusteredDistributionAndSortWithSameExprs(command, None)
+  }
+
+  private def checkClusteredDistributionAndSortWithSameExprs(
+      command: String,
+      targetNumPartitions: Option[Int]): Unit = {
+    val tableOrdering = Array[SortOrder](
+      sort(FieldReference("data"), SortDirection.DESCENDING, NullOrdering.NULLS_FIRST),
+      sort(FieldReference("id"), SortDirection.ASCENDING, NullOrdering.NULLS_FIRST)
+    )
+    val clustering = Array[Expression](FieldReference("data"), FieldReference("id"))
+    val tableDistribution = Distributions.clustered(clustering)
+
+    val writeOrdering = Seq(
+      catalyst.expressions.SortOrder(
+        attr("data"),
+        catalyst.expressions.Descending,
+        catalyst.expressions.NullsFirst,
+        Seq.empty
+      ),
+      catalyst.expressions.SortOrder(
+        attr("id"),
+        catalyst.expressions.Ascending,
+        catalyst.expressions.NullsFirst,
+        Seq.empty
+      )
+    )
+    val writePartitioningExprs = Seq(attr("data"), attr("id"))
+    val writePartitioning = clusteredWritePartitioning(writePartitioningExprs, targetNumPartitions)
+
+    checkWriteRequirements(
+      tableDistribution,
+      tableOrdering,
+      targetNumPartitions,
+      expectedWritePartitioning = writePartitioning,
+      expectedWriteOrdering = writeOrdering,
+      writeCommand = command)
+  }
+
+  test("clustered distribution and sort with extended exprs: append") {
+    checkClusteredDistributionAndSortWithExtendedExprs("append")
+  }
+
+  test("clustered distribution and sort with extended exprs: overwrite") {
+    checkClusteredDistributionAndSortWithExtendedExprs("overwrite")
+  }
+
+  test("clustered distribution and sort with extended exprs: overwriteDynamic") {
+    checkClusteredDistributionAndSortWithExtendedExprs("overwriteDynamic")
+  }
+
+  test("clustered distribution and sort with extended exprs with numPartitions: append") {
+    checkClusteredDistributionAndSortWithExtendedExprs("append", Some(10))
+  }
+
+  test("clustered distribution and sort with extended exprs with numPartitions: overwrite") {
+    checkClusteredDistributionAndSortWithExtendedExprs("overwrite", Some(10))
+  }
+
+  test("clustered distribution and sort with extended exprs with numPartitions: " +
+    "overwriteDynamic") {
+    checkClusteredDistributionAndSortWithExtendedExprs("overwriteDynamic", Some(10))
+  }
+
+  private def checkClusteredDistributionAndSortWithExtendedExprs(command: String): Unit = {
+    checkClusteredDistributionAndSortWithExtendedExprs(command, None)
+  }
+
+  private def checkClusteredDistributionAndSortWithExtendedExprs(
+      command: String,
+      targetNumPartitions: Option[Int]): Unit = {
+    val tableOrdering = Array[SortOrder](
+      sort(FieldReference("data"), SortDirection.DESCENDING, NullOrdering.NULLS_FIRST),
+      sort(FieldReference("id"), SortDirection.ASCENDING, NullOrdering.NULLS_FIRST)
+    )
+    val clustering = Array[Expression](FieldReference("data"))
+    val tableDistribution = Distributions.clustered(clustering)
+
+    val writeOrdering = Seq(
+      catalyst.expressions.SortOrder(
+        attr("data"),
+        catalyst.expressions.Descending,
+        catalyst.expressions.NullsFirst,
+        Seq.empty
+      ),
+      catalyst.expressions.SortOrder(
+        attr("id"),
+        catalyst.expressions.Ascending,
+        catalyst.expressions.NullsFirst,
+        Seq.empty
+      )
+    )
+    val writePartitioningExprs = Seq(attr("data"))
+    val writePartitioning = clusteredWritePartitioning(writePartitioningExprs, targetNumPartitions)
+
+    checkWriteRequirements(
+      tableDistribution,
+      tableOrdering,
+      targetNumPartitions,
+      expectedWritePartitioning = writePartitioning,
+      expectedWriteOrdering = writeOrdering,
+      writeCommand = command)
+  }
+
+  test("unspecified distribution and local sort: append") {
+    checkUnspecifiedDistributionAndLocalSort("append")
+  }
+
+  test("unspecified distribution and local sort: overwrite") {
+    checkUnspecifiedDistributionAndLocalSort("overwrite")
+  }
+
+  test("unspecified distribution and local sort: overwriteDynamic") {
+    checkUnspecifiedDistributionAndLocalSort("overwriteDynamic")
+  }
+
+  test("unspecified distribution and local sort with numPartitions: append") {
+    checkUnspecifiedDistributionAndLocalSort("append", Some(10))
+  }
+
+  test("unspecified distribution and local sort with numPartitions: overwrite") {
+    checkUnspecifiedDistributionAndLocalSort("overwrite", Some(10))
+  }
+
+  test("unspecified distribution and local sort with numPartitions: overwriteDynamic") {
+    checkUnspecifiedDistributionAndLocalSort("overwriteDynamic", Some(10))
+  }
+
+  private def checkUnspecifiedDistributionAndLocalSort(command: String): Unit = {
+    checkUnspecifiedDistributionAndLocalSort(command, None)
+  }
+
+  private def checkUnspecifiedDistributionAndLocalSort(
+      command: String,
+      targetNumPartitions: Option[Int]): Unit = {
+    val tableOrdering = Array[SortOrder](
+      sort(FieldReference("data"), SortDirection.DESCENDING, NullOrdering.NULLS_FIRST)
+    )
+    val tableDistribution = Distributions.unspecified()
+
+    val writeOrdering = Seq(
+      catalyst.expressions.SortOrder(
+        attr("data"),
+        catalyst.expressions.Descending,
+        catalyst.expressions.NullsFirst,
+        Seq.empty
+      )
+    )
+
+    val writePartitioning = UnknownPartitioning(0)
+
+    checkWriteRequirements(
+      tableDistribution,
+      tableOrdering,
+      targetNumPartitions,
+      expectedWritePartitioning = writePartitioning,
+      expectedWriteOrdering = writeOrdering,
+      writeCommand = command,
+      // if the number of partitions is specified, we expect query to fail
+      expectAnalysisException = targetNumPartitions.isDefined)
+  }
+
+  test("unspecified distribution and no sort: append") {
+    checkUnspecifiedDistributionAndNoSort("append")
+  }
+
+  test("unspecified distribution and no sort: overwrite") {
+    checkUnspecifiedDistributionAndNoSort("overwrite")
+  }
+
+  test("unspecified distribution and no sort: overwriteDynamic") {
+    checkUnspecifiedDistributionAndNoSort("overwriteDynamic")
+  }
+
+  test("unspecified distribution and no sort with numPartitions: append") {
+    checkUnspecifiedDistributionAndNoSort("append", Some(10))
+  }
+
+  test("unspecified distribution and no sort with numPartitions: overwrite") {
+    checkUnspecifiedDistributionAndNoSort("overwrite", Some(10))
+  }
+
+  test("unspecified distribution and no sort with numPartitions: overwriteDynamic") {
+    checkUnspecifiedDistributionAndNoSort("overwriteDynamic", Some(10))
+  }
+
+  private def checkUnspecifiedDistributionAndNoSort(command: String): Unit = {
+    checkUnspecifiedDistributionAndNoSort(command, None)
+  }
+
+  private def checkUnspecifiedDistributionAndNoSort(
+      command: String,
+      targetNumPartitions: Option[Int]): Unit = {
+    val tableOrdering = Array.empty[SortOrder]
+    val tableDistribution = Distributions.unspecified()
+
+    val writeOrdering = Seq.empty[catalyst.expressions.SortOrder]
+    val writePartitioning = UnknownPartitioning(0)
+
+    checkWriteRequirements(
+      tableDistribution,
+      tableOrdering,
+      targetNumPartitions,
+      expectedWritePartitioning = writePartitioning,
+      expectedWriteOrdering = writeOrdering,
+      writeCommand = command,
+      // if the number of partitions is specified, we expect query to fail
+      expectAnalysisException = targetNumPartitions.isDefined)
+  }
+
+  test("ordered distribution and sort with manual global sort: append") {
+    checkOrderedDistributionAndSortWithManualGlobalSort("append")
+  }
+
+  test("ordered distribution and sort with manual global sort: overwrite") {
+    checkOrderedDistributionAndSortWithManualGlobalSort("overwrite")
+  }
+
+  test("ordered distribution and sort with manual global sort: overwriteDynamic") {
+    checkOrderedDistributionAndSortWithManualGlobalSort("overwriteDynamic")
+  }
+
+  test("ordered distribution and sort with manual global sort with numPartitions: append") {
+    checkOrderedDistributionAndSortWithManualGlobalSort("append", Some(10))
+  }
+
+  test("ordered distribution and sort with manual global sort with numPartitions: overwrite") {
+    checkOrderedDistributionAndSortWithManualGlobalSort("overwrite", Some(10))
+  }
+
+  test("ordered distribution and sort with manual global sort with numPartitions: " +
+    "overwriteDynamic") {
+    checkOrderedDistributionAndSortWithManualGlobalSort("overwriteDynamic", Some(10))
+  }
+
+  private def checkOrderedDistributionAndSortWithManualGlobalSort(command: String): Unit = {
+    checkOrderedDistributionAndSortWithManualGlobalSort(command, None)
+  }
+
+  private def checkOrderedDistributionAndSortWithManualGlobalSort(
+      command: String,
+      targetNumPartitions: Option[Int]): Unit = {
+    val tableOrdering = Array[SortOrder](
+      sort(FieldReference("data"), SortDirection.ASCENDING, NullOrdering.NULLS_FIRST),
+      sort(FieldReference("id"), SortDirection.ASCENDING, NullOrdering.NULLS_FIRST)
+    )
+    val tableDistribution = Distributions.ordered(tableOrdering)
+
+    val writeOrdering = Seq(
+      catalyst.expressions.SortOrder(
+        attr("data"),
+        catalyst.expressions.Ascending,
+        catalyst.expressions.NullsFirst,
+        Seq.empty
+      ),
+      catalyst.expressions.SortOrder(
+        attr("id"),
+        catalyst.expressions.Ascending,
+        catalyst.expressions.NullsFirst,
+        Seq.empty
+      )
+    )
+    val writePartitioning = orderedWritePartitioning(writeOrdering, targetNumPartitions)
+
+    checkWriteRequirements(
+      tableDistribution,
+      tableOrdering,
+      targetNumPartitions,
+      expectedWritePartitioning = writePartitioning,
+      expectedWriteOrdering = writeOrdering,
+      writeTransform = df => df.orderBy("data", "id"),
+      writeCommand = command)
+  }
+
+  test("ordered distribution and sort with incompatible global sort: append") {
+    checkOrderedDistributionAndSortWithIncompatibleGlobalSort("append")
+  }
+
+  test("ordered distribution and sort with incompatible global sort: overwrite") {
+    checkOrderedDistributionAndSortWithIncompatibleGlobalSort("overwrite")
+  }
+
+  test("ordered distribution and sort with incompatible global sort: overwriteDynamic") {
+    checkOrderedDistributionAndSortWithIncompatibleGlobalSort("overwriteDynamic")
+  }
+
+  test("ordered distribution and sort with incompatible global sort with numPartitions: append") {
+    checkOrderedDistributionAndSortWithIncompatibleGlobalSort("append", Some(10))
+  }
+
+  test("ordered distribution and sort with incompatible global sort with numPartitions: " +
+    "overwrite") {
+    checkOrderedDistributionAndSortWithIncompatibleGlobalSort("overwrite", Some(10))
+  }
+
+  test("ordered distribution and sort with incompatible global sort with numPartitions: " +
+    "overwriteDynamic") {
+    checkOrderedDistributionAndSortWithIncompatibleGlobalSort("overwriteDynamic", Some(10))
+  }
+
+  private def checkOrderedDistributionAndSortWithIncompatibleGlobalSort(command: String): Unit = {
+    checkOrderedDistributionAndSortWithIncompatibleGlobalSort(command, None)
+  }
+
+  private def checkOrderedDistributionAndSortWithIncompatibleGlobalSort(
+      command: String,
+      targetNumPartitions: Option[Int]): Unit = {
+    val tableOrdering = Array[SortOrder](
+      sort(FieldReference("data"), SortDirection.ASCENDING, NullOrdering.NULLS_FIRST),
+      sort(FieldReference("id"), SortDirection.ASCENDING, NullOrdering.NULLS_FIRST)
+    )
+    val tableDistribution = Distributions.ordered(tableOrdering)
+
+    val writeOrdering = Seq(
+      catalyst.expressions.SortOrder(
+        attr("data"),
+        catalyst.expressions.Ascending,
+        catalyst.expressions.NullsFirst,
+        Seq.empty
+      ),
+      catalyst.expressions.SortOrder(
+        attr("id"),
+        catalyst.expressions.Ascending,
+        catalyst.expressions.NullsFirst,
+        Seq.empty
+      )
+    )
+    val writePartitioning = orderedWritePartitioning(writeOrdering, targetNumPartitions)
+
+    checkWriteRequirements(
+      tableDistribution,
+      tableOrdering,
+      targetNumPartitions,
+      expectedWritePartitioning = writePartitioning,
+      expectedWriteOrdering = writeOrdering,
+      writeTransform = df => df.orderBy(df("data").desc, df("id").asc),
+      writeCommand = command)
+  }
+
+  test("ordered distribution and sort with manual local sort: append") {
+    checkOrderedDistributionAndSortWithManualLocalSort("append")
+  }
+
+  test("ordered distribution and sort with manual local sort: overwrite") {
+    checkOrderedDistributionAndSortWithManualLocalSort("overwrite")
+  }
+
+  test("ordered distribution and sort with manual local sort: overwriteDynamic") {
+    checkOrderedDistributionAndSortWithManualLocalSort("overwriteDynamic")
+  }
+
+  test("ordered distribution and sort with manual local sort with numPartitions: append") {
+    checkOrderedDistributionAndSortWithManualLocalSort("append", Some(10))
+  }
+
+  test("ordered distribution and sort with manual local sort with numPartitions: overwrite") {
+    checkOrderedDistributionAndSortWithManualLocalSort("overwrite", Some(10))
+  }
+
+  test("ordered distribution and sort with manual local sort with numPartitions: " +
+    "overwriteDynamic") {
+    checkOrderedDistributionAndSortWithManualLocalSort("overwriteDynamic", Some(10))
+  }
+
+  private def checkOrderedDistributionAndSortWithManualLocalSort(command: String): Unit = {
+    checkOrderedDistributionAndSortWithManualLocalSort(command, None)
+  }
+
+  private def checkOrderedDistributionAndSortWithManualLocalSort(
+      command: String,
+      targetNumPartitions: Option[Int]): Unit = {
+    val tableOrdering = Array[SortOrder](
+      sort(FieldReference("data"), SortDirection.ASCENDING, NullOrdering.NULLS_FIRST),
+      sort(FieldReference("id"), SortDirection.ASCENDING, NullOrdering.NULLS_FIRST)
+    )
+    val tableDistribution = Distributions.ordered(tableOrdering)
+
+    val writeOrdering = Seq(
+      catalyst.expressions.SortOrder(
+        attr("data"),
+        catalyst.expressions.Ascending,
+        catalyst.expressions.NullsFirst,
+        Seq.empty
+      ),
+      catalyst.expressions.SortOrder(
+        attr("id"),
+        catalyst.expressions.Ascending,
+        catalyst.expressions.NullsFirst,
+        Seq.empty
+      )
+    )
+    val writePartitioning = orderedWritePartitioning(writeOrdering, targetNumPartitions)
+
+    checkWriteRequirements(
+      tableDistribution,
+      tableOrdering,
+      targetNumPartitions,
+      expectedWritePartitioning = writePartitioning,
+      expectedWriteOrdering = writeOrdering,
+      writeTransform = df => df.sortWithinPartitions("data", "id"),
+      writeCommand = command)
+  }
+
+  test("clustered distribution and local sort with manual global sort: append") {
+    checkClusteredDistributionAndLocalSortWithManualGlobalSort("append")
+  }
+
+  test("clustered distribution and local sort with manual global sort: overwrite") {
+    checkClusteredDistributionAndLocalSortWithManualGlobalSort("overwrite")
+  }
+
+  test("clustered distribution and local sort with manual global sort: overwriteDynamic") {
+    checkClusteredDistributionAndLocalSortWithManualGlobalSort("overwriteDynamic")
+  }
+
+  test("clustered distribution and local sort with manual global sort with numPartitions: append") {
+    checkClusteredDistributionAndLocalSortWithManualGlobalSort("append", Some(10))
+  }
+
+  test("clustered distribution and local sort with manual global sort with numPartitions: " +
+    "overwrite") {
+    checkClusteredDistributionAndLocalSortWithManualGlobalSort("overwrite", Some(10))
+  }
+
+  test("clustered distribution and local sort with manual global sort with numPartitions: " +
+    "overwriteDynamic") {
+    checkClusteredDistributionAndLocalSortWithManualGlobalSort("overwriteDynamic", Some(10))
+  }
+
+  private def checkClusteredDistributionAndLocalSortWithManualGlobalSort(command: String): Unit = {
+    checkClusteredDistributionAndLocalSortWithManualGlobalSort(command, None)
+  }
+
+  private def checkClusteredDistributionAndLocalSortWithManualGlobalSort(
+      command: String,
+      targetNumPartitions: Option[Int]): Unit = {
+    val tableOrdering = Array[SortOrder](
+      sort(FieldReference("data"), SortDirection.DESCENDING, NullOrdering.NULLS_FIRST),
+      sort(FieldReference("id"), SortDirection.ASCENDING, NullOrdering.NULLS_FIRST)
+    )
+    val tableDistribution = Distributions.clustered(Array(FieldReference("data")))
+
+    val writeOrdering = Seq(
+      catalyst.expressions.SortOrder(
+        attr("data"),
+        catalyst.expressions.Descending,
+        catalyst.expressions.NullsFirst,
+        Seq.empty
+      ),
+      catalyst.expressions.SortOrder(
+        attr("id"),
+        catalyst.expressions.Ascending,
+        catalyst.expressions.NullsFirst,
+        Seq.empty
+      )
+    )
+    val writePartitioningExprs = Seq(attr("data"))
+    val writePartitioning = clusteredWritePartitioning(writePartitioningExprs, targetNumPartitions)
+
+    checkWriteRequirements(
+      tableDistribution,
+      tableOrdering,
+      targetNumPartitions,
+      expectedWritePartitioning = writePartitioning,
+      expectedWriteOrdering = writeOrdering,
+      writeTransform = df => df.orderBy("data", "id"),
+      writeCommand = command)
+  }
+
+  test("clustered distribution and local sort with manual local sort: append") {
+    checkClusteredDistributionAndLocalSortWithManualLocalSort("append")
+  }
+
+  test("clustered distribution and local sort with manual local sort: overwrite") {
+    checkClusteredDistributionAndLocalSortWithManualLocalSort("overwrite")
+  }
+
+  test("clustered distribution and local sort with manual local sort: overwriteDynamic") {
+    checkClusteredDistributionAndLocalSortWithManualLocalSort("overwriteDynamic")
+  }
+
+  test("clustered distribution and local sort with manual local sort with numPartitions: append") {
+    checkClusteredDistributionAndLocalSortWithManualLocalSort("append", Some(10))
+  }
+
+  test("clustered distribution and local sort with manual local sort with numPartitions: " +
+    "overwrite") {
+    checkClusteredDistributionAndLocalSortWithManualLocalSort("overwrite", Some(10))
+  }
+
+  test("clustered distribution and local sort with manual local sort with numPartitions: " +
+    "overwriteDynamic") {
+    checkClusteredDistributionAndLocalSortWithManualLocalSort("overwriteDynamic", Some(10))
+  }
+
+  private def checkClusteredDistributionAndLocalSortWithManualLocalSort(command: String): Unit = {
+    checkClusteredDistributionAndLocalSortWithManualLocalSort(command, None)
+  }
+
+  private def checkClusteredDistributionAndLocalSortWithManualLocalSort(
+      command: String,
+      targetNumPartitions: Option[Int]): Unit = {
+    val tableOrdering = Array[SortOrder](
+      sort(FieldReference("data"), SortDirection.DESCENDING, NullOrdering.NULLS_FIRST),
+      sort(FieldReference("id"), SortDirection.ASCENDING, NullOrdering.NULLS_FIRST)
+    )
+    val tableDistribution = Distributions.clustered(Array(FieldReference("data")))
+
+    val writeOrdering = Seq(
+      catalyst.expressions.SortOrder(
+        attr("data"),
+        catalyst.expressions.Descending,
+        catalyst.expressions.NullsFirst,
+        Seq.empty
+      ),
+      catalyst.expressions.SortOrder(
+        attr("id"),
+        catalyst.expressions.Ascending,
+        catalyst.expressions.NullsFirst,
+        Seq.empty
+      )
+    )
+    val writePartitioningExprs = Seq(attr("data"))
+    val writePartitioning = clusteredWritePartitioning(writePartitioningExprs, targetNumPartitions)
+
+    checkWriteRequirements(
+      tableDistribution,
+      tableOrdering,
+      targetNumPartitions,
+      expectedWritePartitioning = writePartitioning,
+      expectedWriteOrdering = writeOrdering,
+      writeTransform = df => df.orderBy("data", "id"),
+      writeCommand = command)
+  }
+
+  private def checkWriteRequirements(
+      tableDistribution: Distribution,
+      tableOrdering: Array[SortOrder],
+      tableNumPartitions: Option[Int],
+      expectedWritePartitioning: physical.Partitioning,
+      expectedWriteOrdering: Seq[catalyst.expressions.SortOrder],
+      writeTransform: DataFrame => DataFrame = df => df,
+      writeCommand: String = "append",
+      expectAnalysisException: Boolean = false): Unit = {
+
+    catalog.createTable(ident, schema, Array.empty, emptyProps, tableDistribution,
+      tableOrdering, tableNumPartitions)
+
+    val df = spark.createDataFrame(Seq((1, "a"), (2, "b"), (3, "c"))).toDF("id", "data")
+    val writer = writeTransform(df).writeTo(tableNameAsString)
+
+    def executeCommand(): SparkPlan = writeCommand match {
+      case "append" => execute(writer.append())
+      case "overwrite" => execute(writer.overwrite(lit(true)))
+      case "overwriteDynamic" => execute(writer.overwritePartitions())
+    }
+
+    if (expectAnalysisException) {
+      intercept[AnalysisException] {
+        executeCommand()
+      }
+    } else {
+      val executedPlan = executeCommand()
+
+      checkPartitioningAndOrdering(executedPlan, expectedWritePartitioning, expectedWriteOrdering)
+
+      checkAnswer(spark.table(tableNameAsString), df)
+    }
+  }
+
+  private def checkPartitioningAndOrdering(
+      plan: SparkPlan,
+      partitioning: physical.Partitioning,
+      ordering: Seq[catalyst.expressions.SortOrder]): Unit = {
+
+    val sorts = collect(plan) { case s: SortExec => s }
+    assert(sorts.size <= 1, "must be at most one sort")
+    val shuffles = collect(plan) { case s: ShuffleExchangeLike => s }
+    assert(shuffles.size <= 1, "must be at most one shuffle")
+
+    val actualPartitioning = plan.outputPartitioning
+    val expectedPartitioning = partitioning match {
+      case p: physical.RangePartitioning =>
+        val resolvedOrdering = p.ordering.map(resolveAttrs(_, plan))
+        p.copy(ordering = resolvedOrdering.asInstanceOf[Seq[catalyst.expressions.SortOrder]])
+      case p: physical.HashPartitioning =>
+        val resolvedExprs = p.expressions.map(resolveAttrs(_, plan))
+        p.copy(expressions = resolvedExprs)
+      case other => other
+    }
+    assert(actualPartitioning == expectedPartitioning, "partitioning must match")
+
+    val actualOrdering = plan.outputOrdering
+    val expectedOrdering = ordering.map(resolveAttrs(_, plan))
+    assert(actualOrdering == expectedOrdering, "ordering must match")
+  }
+
+  private def resolveAttrs(
+      expr: catalyst.expressions.Expression,
+      plan: SparkPlan): catalyst.expressions.Expression = {
+
+    expr.transform {
+      case UnresolvedAttribute(Seq(attrName)) =>
+        plan.output.find(attr => resolver(attr.name, attrName)).get
+      case UnresolvedAttribute(nameParts) =>
+        val attrName = nameParts.mkString(".")
+        fail(s"cannot resolve a nested attr: $attrName")
+    }
+  }
+
+  private def attr(name: String): UnresolvedAttribute = {
+    UnresolvedAttribute(name)
+  }
+
+  private def catalog: InMemoryTableCatalog = {
+    val catalog = spark.sessionState.catalogManager.catalog("testcat")
+    catalog.asTableCatalog.asInstanceOf[InMemoryTableCatalog]
+  }
+
+  // executes a write operation and keeps the executed physical plan
+  private def execute(writeFunc: => Unit): SparkPlan = {
+    var executedPlan: SparkPlan = null
+
+    val listener = new QueryExecutionListener {
+      override def onSuccess(funcName: String, qe: QueryExecution, durationNs: Long): Unit = {
+        executedPlan = qe.executedPlan
+      }
+      override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {
+      }
+    }
+    spark.listenerManager.register(listener)
+
+    writeFunc
+
+    sparkContext.listenerBus.waitUntilEmpty()
+
+    executedPlan match {
+      case w: V2TableWriteExec =>
+        stripAQEPlan(w.query)
+      case _ =>
+        fail("expected V2TableWriteExec")
+    }
+  }
+
+  private def orderedWritePartitioning(
+      writeOrdering: Seq[catalyst.expressions.SortOrder],
+      targetNumPartitions: Option[Int]): physical.Partitioning = {
+    RangePartitioning(writeOrdering, targetNumPartitions.getOrElse(conf.numShufflePartitions))
+  }
+
+  private def clusteredWritePartitioning(
+      writePartitioningExprs: Seq[catalyst.expressions.Expression],
+      targetNumPartitions: Option[Int]): physical.Partitioning = {
+    HashPartitioning(writePartitioningExprs,
+      targetNumPartitions.getOrElse(conf.numShufflePartitions))
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/functions/V2FunctionBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/functions/V2FunctionBenchmark.scala
new file mode 100644
index 0000000000000..16e86a7597e76
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/functions/V2FunctionBenchmark.scala
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.functions
+
+import test.org.apache.spark.sql.connector.catalog.functions.JavaLongAdd
+import test.org.apache.spark.sql.connector.catalog.functions.JavaLongAdd.{JavaLongAddDefault, JavaLongAddMagic, JavaLongAddStaticMagic}
+
+import org.apache.spark.benchmark.Benchmark
+import org.apache.spark.sql.Column
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.expressions.{BinaryArithmetic, Expression}
+import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode._
+import org.apache.spark.sql.catalyst.util.TypeUtils
+import org.apache.spark.sql.connector.catalog.{Identifier, InMemoryCatalog}
+import org.apache.spark.sql.connector.catalog.functions.{BoundFunction, ScalarFunction, UnboundFunction}
+import org.apache.spark.sql.execution.benchmark.SqlBasedBenchmark
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types.{AbstractDataType, DataType, LongType, NumericType, StructType}
+
+/**
+ * Benchmark to measure DataSourceV2 UDF performance
+ * {{{
+ *   To run this benchmark:
+ *   1. without sbt:
+ *      bin/spark-submit --class <this class>
+ *        --jars <spark core test jar>,<spark catalyst test jar> <sql core test jar>
+ *   2. build/sbt "sql/test:runMain <this class>"
+ *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
+ *      Results will be written to "benchmarks/V2FunctionBenchmark-results.txt".
+ * }}}
+ * '''NOTE''': to update the result of this benchmark, please use Github benchmark action:
+ *   https://spark.apache.org/developer-tools.html#github-workflow-benchmarks
+ */
+object V2FunctionBenchmark extends SqlBasedBenchmark {
+  val catalogName: String = "benchmark_catalog"
+
+  override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
+    val N = 500L * 1000 * 1000
+    Seq(true, false).foreach { codegenEnabled =>
+      Seq(true, false).foreach { resultNullable =>
+        scalarFunctionBenchmark(N, codegenEnabled = codegenEnabled,
+          resultNullable = resultNullable)
+      }
+    }
+  }
+
+  private def scalarFunctionBenchmark(
+      N: Long,
+      codegenEnabled: Boolean,
+      resultNullable: Boolean): Unit = {
+    withSQLConf(s"spark.sql.catalog.$catalogName" -> classOf[InMemoryCatalog].getName) {
+      createFunction("java_long_add_default",
+        new JavaLongAdd(new JavaLongAddDefault(resultNullable)))
+      createFunction("java_long_add_magic", new JavaLongAdd(new JavaLongAddMagic(resultNullable)))
+      createFunction("java_long_add_static_magic",
+        new JavaLongAdd(new JavaLongAddStaticMagic(resultNullable)))
+      createFunction("scala_long_add_default",
+        LongAddUnbound(new LongAddWithProduceResult(resultNullable)))
+      createFunction("scala_long_add_magic", LongAddUnbound(new LongAddWithMagic(resultNullable)))
+
+      val codeGenFactoryMode = if (codegenEnabled) FALLBACK else NO_CODEGEN
+      withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> codegenEnabled.toString,
+          SQLConf.CODEGEN_FACTORY_MODE.key -> codeGenFactoryMode.toString) {
+        val name = s"scalar function (long + long) -> long, result_nullable = $resultNullable " +
+            s"codegen = $codegenEnabled"
+        val benchmark = new Benchmark(name, N, output = output)
+        benchmark.addCase(s"native_long_add", numIters = 3) { _ =>
+          spark.range(N).select(Column(NativeAdd($"id".expr, $"id".expr, resultNullable))).noop()
+        }
+        Seq("java_long_add_default", "java_long_add_magic", "java_long_add_static_magic",
+            "scala_long_add_default", "scala_long_add_magic").foreach { functionName =>
+          benchmark.addCase(s"$functionName", numIters = 3) { _ =>
+            spark.range(N).selectExpr(s"$catalogName.$functionName(id, id)").noop()
+          }
+        }
+        benchmark.run()
+      }
+    }
+  }
+
+  private def createFunction(name: String, fn: UnboundFunction): Unit = {
+    val catalog = spark.sessionState.catalogManager.catalog(catalogName)
+    val ident = Identifier.of(Array.empty, name)
+    catalog.asInstanceOf[InMemoryCatalog].createFunction(ident, fn)
+  }
+
+  case class NativeAdd(
+      left: Expression,
+      right: Expression,
+      override val nullable: Boolean) extends BinaryArithmetic {
+    override protected val failOnError: Boolean = false
+    override def inputType: AbstractDataType = NumericType
+    override def symbol: String = "+"
+
+    private lazy val numeric = TypeUtils.getNumeric(dataType, failOnError)
+    protected override def nullSafeEval(input1: Any, input2: Any): Any =
+      numeric.plus(input1, input2)
+
+    override protected def withNewChildrenInternal(
+        newLeft: Expression,
+        newRight: Expression): NativeAdd = copy(left = newLeft, right = newRight)
+  }
+
+  case class LongAddUnbound(impl: ScalarFunction[Long]) extends UnboundFunction {
+    override def bind(inputType: StructType): BoundFunction = impl
+    override def description(): String = name()
+    override def name(): String = "long_add_unbound"
+  }
+
+  abstract class LongAddBase(resultNullable: Boolean) extends ScalarFunction[Long] {
+    override def inputTypes(): Array[DataType] = Array(LongType, LongType)
+    override def resultType(): DataType = LongType
+    override def isResultNullable: Boolean = resultNullable
+  }
+
+  class LongAddWithProduceResult(resultNullable: Boolean) extends LongAddBase(resultNullable) {
+    override def produceResult(input: InternalRow): Long = {
+      input.getLong(0) + input.getLong(1)
+    }
+    override def name(): String = "long_add_default"
+  }
+
+  class LongAddWithMagic(resultNullable: Boolean) extends LongAddBase(resultNullable) {
+    def invoke(left: Long, right: Long): Long = {
+      left + right
+    }
+    override def name(): String = "long_add_magic"
+  }
+}
+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
new file mode 100644
index 0000000000000..c845dd81f3e32
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
@@ -0,0 +1,707 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import java.sql.{Date, Timestamp}
+import java.time.{Duration, Period}
+
+import org.json4s.DefaultFormats
+import org.json4s.JsonDSL._
+import org.json4s.jackson.JsonMethods._
+import org.scalatest.Assertions._
+import org.scalatest.BeforeAndAfterEach
+import org.scalatest.exceptions.TestFailedException
+
+import org.apache.spark.{SparkException, TaskContext, TestUtils}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, GenericInternalRow}
+import org.apache.spark.sql.catalyst.plans.physical.Partitioning
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.test.SQLTestUtils
+import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.CalendarInterval
+
+abstract class BaseScriptTransformationSuite extends SparkPlanTest with SQLTestUtils
+  with BeforeAndAfterEach {
+  import testImplicits._
+  import ScriptTransformationIOSchema._
+
+  protected def defaultSerDe(): String
+
+  protected val uncaughtExceptionHandler = new TestUncaughtExceptionHandler
+
+  private var defaultUncaughtExceptionHandler: Thread.UncaughtExceptionHandler = _
+
+  protected override def beforeAll(): Unit = {
+    super.beforeAll()
+    defaultUncaughtExceptionHandler = Thread.getDefaultUncaughtExceptionHandler
+    Thread.setDefaultUncaughtExceptionHandler(uncaughtExceptionHandler)
+  }
+
+  protected override def afterAll(): Unit = {
+    super.afterAll()
+    Thread.setDefaultUncaughtExceptionHandler(defaultUncaughtExceptionHandler)
+  }
+
+  override protected def afterEach(): Unit = {
+    super.afterEach()
+    uncaughtExceptionHandler.cleanStatus()
+  }
+
+  def createScriptTransformationExec(
+      script: String,
+      output: Seq[Attribute],
+      child: SparkPlan,
+      ioschema: ScriptTransformationIOSchema): BaseScriptTransformationExec
+
+  test("cat without SerDe") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+
+    val rowsDf = Seq("a", "b", "c").map(Tuple1.apply).toDF("a")
+    checkAnswer(
+      rowsDf,
+      (child: SparkPlan) => createScriptTransformationExec(
+        script = "cat",
+        output = Seq(AttributeReference("a", StringType)()),
+        child = child,
+        ioschema = defaultIOSchema
+      ),
+      rowsDf.collect())
+    assert(uncaughtExceptionHandler.exception.isEmpty)
+  }
+
+  test("script transformation should not swallow errors from upstream operators (no serde)") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+
+    val rowsDf = Seq("a", "b", "c").map(Tuple1.apply).toDF("a")
+    val e = intercept[TestFailedException] {
+      checkAnswer(
+        rowsDf,
+        (child: SparkPlan) => createScriptTransformationExec(
+          script = "cat",
+          output = Seq(AttributeReference("a", StringType)()),
+          child = ExceptionInjectingOperator(child),
+          ioschema = defaultIOSchema
+        ),
+        rowsDf.collect())
+    }
+    assert(e.getMessage().contains("intentional exception"))
+    // Before SPARK-25158, uncaughtExceptionHandler will catch IllegalArgumentException
+    assert(uncaughtExceptionHandler.exception.isEmpty)
+  }
+
+  test("SPARK-25990: TRANSFORM should handle different data types correctly") {
+    assume(TestUtils.testCommandAvailable("python"))
+    val scriptFilePath = copyAndGetResourceFile("test_script.py", ".py").getAbsoluteFile
+
+    withTempView("v") {
+      val df = Seq(
+        (1, "1", 1.0, BigDecimal(1.0), new Timestamp(1)),
+        (2, "2", 2.0, BigDecimal(2.0), new Timestamp(2)),
+        (3, "3", 3.0, BigDecimal(3.0), new Timestamp(3))
+      ).toDF("a", "b", "c", "d", "e") // Note column d's data type is Decimal(38, 18)
+      df.createTempView("v")
+
+      val query = sql(
+        s"""
+           |SELECT
+           |TRANSFORM(a, b, c, d, e)
+           |  ROW FORMAT DELIMITED
+           |  FIELDS TERMINATED BY '\t'
+           |  USING 'python $scriptFilePath' AS (a, b, c, d, e)
+           |  ROW FORMAT DELIMITED
+           |  FIELDS TERMINATED BY '\t'
+           |FROM v
+        """.stripMargin)
+
+      checkAnswer(query, identity, df.select(
+        'a.cast("string"),
+        'b.cast("string"),
+        'c.cast("string"),
+        'd.cast("string"),
+        'e.cast("string")).collect())
+    }
+  }
+
+  test("SPARK-32388: TRANSFORM should handle schema less correctly (no serde)") {
+    withTempView("v") {
+      val df = Seq(
+        (1, "1", 1.0, BigDecimal(1.0), new Timestamp(1)),
+        (2, "2", 2.0, BigDecimal(2.0), new Timestamp(2)),
+        (3, "3", 3.0, BigDecimal(3.0), new Timestamp(3))
+      ).toDF("a", "b", "c", "d", "e") // Note column d's data type is Decimal(38, 18)
+
+      checkAnswer(
+        df,
+        (child: SparkPlan) => createScriptTransformationExec(
+          script = "cat",
+          output = Seq(
+            AttributeReference("key", StringType)(),
+            AttributeReference("value", StringType)()),
+          child = child,
+          ioschema = defaultIOSchema.copy(schemaLess = true)
+        ),
+        df.select(
+          'a.cast("string").as("key"),
+          'b.cast("string").as("value")).collect())
+
+      checkAnswer(
+        df.select('a, 'b),
+        (child: SparkPlan) => createScriptTransformationExec(
+          script = "cat",
+          output = Seq(
+            AttributeReference("key", StringType)(),
+            AttributeReference("value", StringType)()),
+          child = child,
+          ioschema = defaultIOSchema.copy(schemaLess = true)
+        ),
+        df.select(
+          'a.cast("string").as("key"),
+          'b.cast("string").as("value")).collect())
+
+      checkAnswer(
+        df.select('a),
+        (child: SparkPlan) => createScriptTransformationExec(
+          script = "cat",
+          output = Seq(
+            AttributeReference("key", StringType)(),
+            AttributeReference("value", StringType)()),
+          child = child,
+          ioschema = defaultIOSchema.copy(schemaLess = true)
+        ),
+        df.select(
+          'a.cast("string").as("key"),
+          lit(null)).collect())
+    }
+  }
+
+  test("SPARK-30973: TRANSFORM should wait for the termination of the script (no serde)") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+
+    val rowsDf = Seq("a", "b", "c").map(Tuple1.apply).toDF("a")
+    val e = intercept[SparkException] {
+      val plan =
+        createScriptTransformationExec(
+          script = "some_non_existent_command",
+          output = Seq(AttributeReference("a", StringType)()),
+          child = rowsDf.queryExecution.sparkPlan,
+          ioschema = defaultIOSchema)
+      SparkPlanTest.executePlan(plan, spark.sqlContext)
+    }
+    assert(e.getMessage.contains("Subprocess exited with status"))
+    assert(uncaughtExceptionHandler.exception.isEmpty)
+  }
+
+  def testBasicInputDataTypesWith(serde: ScriptTransformationIOSchema, testName: String): Unit = {
+    test(s"SPARK-32400: TRANSFORM should support basic data types as input ($testName)") {
+      assume(TestUtils.testCommandAvailable("python"))
+      withTempView("v") {
+        val df = Seq(
+          (1, "1", 1.0f, 1.0, 11.toByte, BigDecimal(1.0), new Timestamp(1),
+            new Date(2020, 7, 1), true),
+          (2, "2", 2.0f, 2.0, 22.toByte, BigDecimal(2.0), new Timestamp(2),
+            new Date(2020, 7, 2), true),
+          (3, "3", 3.0f, 3.0, 33.toByte, BigDecimal(3.0), new Timestamp(3),
+            new Date(2020, 7, 3), false)
+        ).toDF("a", "b", "c", "d", "e", "f", "g", "h", "i")
+          .withColumn("j", lit("abc").cast("binary"))
+
+        checkAnswer(
+          df,
+          (child: SparkPlan) => createScriptTransformationExec(
+            script = "cat",
+            output = Seq(
+              AttributeReference("a", IntegerType)(),
+              AttributeReference("b", StringType)(),
+              AttributeReference("c", FloatType)(),
+              AttributeReference("d", DoubleType)(),
+              AttributeReference("e", ByteType)(),
+              AttributeReference("f", DecimalType(38, 18))(),
+              AttributeReference("g", TimestampType)(),
+              AttributeReference("h", DateType)(),
+              AttributeReference("i", BooleanType)(),
+              AttributeReference("j", BinaryType)()),
+            child = child,
+            ioschema = serde
+          ),
+          df.select('a, 'b, 'c, 'd, 'e, 'f, 'g, 'h, 'i, 'j).collect())
+      }
+    }
+  }
+
+  testBasicInputDataTypesWith(defaultIOSchema, "no serde")
+
+  test("SPARK-32400: TRANSFORM should support more data types (interval, array, map, struct " +
+    "and udt) as input (no serde)") {
+    assume(TestUtils.testCommandAvailable("python"))
+    withTempView("v") {
+      val df = Seq(
+        (new CalendarInterval(7, 1, 1000), Array(0, 1, 2), Map("a" -> 1), (1, 2),
+          new SimpleTuple(1, 1L)),
+        (new CalendarInterval(7, 2, 2000), Array(3, 4, 5), Map("b" -> 2), (3, 4),
+          new SimpleTuple(1, 1L)),
+        (new CalendarInterval(7, 3, 3000), Array(6, 7, 8), Map("c" -> 3), (5, 6),
+          new SimpleTuple(1, 1L))
+      ).toDF("a", "b", "c", "d", "e")
+
+      // Can't support convert script output data to ArrayType/MapType/StructType now,
+      // return these column still as string.
+      // For UserDefinedType, if user defined deserialize method to support convert string
+      // to UserType like [[SimpleTupleUDT]], we can support convert to this UDT, else we
+      // will return null value as column.
+      checkAnswer(
+        df,
+        (child: SparkPlan) => createScriptTransformationExec(
+          script = "cat",
+          output = Seq(
+            AttributeReference("a", CalendarIntervalType)(),
+            AttributeReference("b", ArrayType(IntegerType))(),
+            AttributeReference("c", MapType(StringType, IntegerType))(),
+            AttributeReference("d", StructType(
+              Array(StructField("_1", IntegerType),
+                StructField("_2", IntegerType))))(),
+            AttributeReference("e", new SimpleTupleUDT)()),
+          child = child,
+          ioschema = defaultIOSchema
+        ),
+        df.select('a, 'b, 'c, 'd, 'e).collect())
+    }
+  }
+
+  test("SPARK-32400: TRANSFORM should respect DATETIME_JAVA8API_ENABLED (no serde)") {
+    assume(TestUtils.testCommandAvailable("python"))
+    Array(false, true).foreach { java8AapiEnable =>
+      withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8AapiEnable.toString) {
+        withTempView("v") {
+          val df = Seq(
+            (new Timestamp(1), new Date(2020, 7, 1)),
+            (new Timestamp(2), new Date(2020, 7, 2)),
+            (new Timestamp(3), new Date(2020, 7, 3))
+          ).toDF("a", "b")
+          df.createTempView("v")
+
+          val query = sql(
+            """
+              |SELECT TRANSFORM (a, b)
+              |USING 'cat' AS (a timestamp, b date)
+              |FROM v
+            """.stripMargin)
+          checkAnswer(query, identity, df.select('a, 'b).collect())
+        }
+      }
+    }
+  }
+
+  test("SPARK-32608: Script Transform ROW FORMAT DELIMIT value should format value") {
+    withTempView("v") {
+      val df = Seq(
+        (1, "1", 1.0, BigDecimal(1.0), new Timestamp(1)),
+        (2, "2", 2.0, BigDecimal(2.0), new Timestamp(2)),
+        (3, "3", 3.0, BigDecimal(3.0), new Timestamp(3))
+      ).toDF("a", "b", "c", "d", "e") // Note column d's data type is Decimal(38, 18)
+      df.createTempView("v")
+
+      // input/output with same delimit
+      checkAnswer(
+        sql(
+          s"""
+             |SELECT TRANSFORM(a, b, c, d, e)
+             |  ROW FORMAT DELIMITED
+             |  FIELDS TERMINATED BY ','
+             |  COLLECTION ITEMS TERMINATED BY '#'
+             |  MAP KEYS TERMINATED BY '@'
+             |  LINES TERMINATED BY '\n'
+             |  NULL DEFINED AS 'null'
+             |  USING 'cat' AS (a, b, c, d, e)
+             |  ROW FORMAT DELIMITED
+             |  FIELDS TERMINATED BY ','
+             |  COLLECTION ITEMS TERMINATED BY '#'
+             |  MAP KEYS TERMINATED BY '@'
+             |  LINES TERMINATED BY '\n'
+             |  NULL DEFINED AS 'NULL'
+             |FROM v
+        """.stripMargin), identity, df.select(
+          'a.cast("string"),
+          'b.cast("string"),
+          'c.cast("string"),
+          'd.cast("string"),
+          'e.cast("string")).collect())
+
+      // input/output with different delimit and show result
+      checkAnswer(
+        sql(
+          s"""
+             |SELECT TRANSFORM(a, b, c, d, e)
+             |  ROW FORMAT DELIMITED
+             |  FIELDS TERMINATED BY ','
+             |  LINES TERMINATED BY '\n'
+             |  NULL DEFINED AS 'null'
+             |  USING 'cat' AS (value)
+             |  ROW FORMAT DELIMITED
+             |  FIELDS TERMINATED BY '&'
+             |  LINES TERMINATED BY '\n'
+             |  NULL DEFINED AS 'NULL'
+             |FROM v
+        """.stripMargin), identity, df.select(
+          concat_ws(",",
+            'a.cast("string"),
+            'b.cast("string"),
+            'c.cast("string"),
+            'd.cast("string"),
+            'e.cast("string"))).collect())
+    }
+  }
+
+  test("SPARK-32667: SCRIPT TRANSFORM pad null value to fill column" +
+    " when without schema less (no-serde)") {
+    val df = Seq(
+      (1, "1", 1.0, BigDecimal(1.0), new Timestamp(1)),
+      (2, "2", 2.0, BigDecimal(2.0), new Timestamp(2)),
+      (3, "3", 3.0, BigDecimal(3.0), new Timestamp(3))
+    ).toDF("a", "b", "c", "d", "e") // Note column d's data type is Decimal(38, 18)
+
+    checkAnswer(
+      df.select('a, 'b),
+      (child: SparkPlan) => createScriptTransformationExec(
+        script = "cat",
+        output = Seq(
+          AttributeReference("a", StringType)(),
+          AttributeReference("b", StringType)(),
+          AttributeReference("c", StringType)(),
+          AttributeReference("d", StringType)()),
+        child = child,
+        ioschema = defaultIOSchema
+      ),
+      df.select(
+        'a.cast("string").as("a"),
+        'b.cast("string").as("b"),
+        lit(null), lit(null)).collect())
+  }
+
+  test("SPARK-32106: TRANSFORM with non-existent command/file") {
+    Seq(
+      s"""
+         |SELECT TRANSFORM(a)
+         |USING 'some_non_existent_command' AS (a)
+         |FROM VALUES (1) t(a)
+       """.stripMargin,
+      s"""
+         |SELECT TRANSFORM(a)
+         |USING 'python some_non_existent_file' AS (a)
+         |FROM VALUES (1) t(a)
+       """.stripMargin).foreach { query =>
+      intercept[SparkException] {
+        // Since an error message is shell-dependent, this test just checks
+        // if the expected exception will be thrown.
+        sql(query).collect()
+      }
+    }
+  }
+
+  test("SPARK-33930: Script Transform default FIELD DELIMIT should be \u0001 (no serde)") {
+    withTempView("v") {
+      val df = Seq(
+        (1, 2, 3),
+        (2, 3, 4),
+        (3, 4, 5)
+      ).toDF("a", "b", "c")
+      df.createTempView("v")
+
+      checkAnswer(
+        sql(
+          s"""
+             |SELECT TRANSFORM(a, b, c)
+             |  ROW FORMAT DELIMITED
+             |  USING 'cat' AS (a)
+             |  ROW FORMAT DELIMITED
+             |  FIELDS TERMINATED BY '&'
+             |FROM v
+        """.stripMargin), identity,
+        Row("1\u00012\u00013") ::
+          Row("2\u00013\u00014") ::
+          Row("3\u00014\u00015") :: Nil)
+    }
+  }
+
+  test("SPARK-31936: Script transform support ArrayType/MapType/StructType (no serde)") {
+    assume(TestUtils.testCommandAvailable("python"))
+    withTempView("v") {
+      val df = Seq(
+        (Array(0, 1, 2), Array(Array(0, 1), Array(2)),
+          Map("a" -> 1), Map("b" -> Array("a", "b"))),
+        (Array(3, 4, 5), Array(Array(3, 4), Array(5)),
+          Map("b" -> 2), Map("c" -> Array("c", "d"))),
+        (Array(6, 7, 8), Array(Array(6, 7), Array(8)),
+          Map("c" -> 3), Map("d" -> Array("e", "f")))
+      ).toDF("a", "b", "c", "d")
+        .select('a, 'b, 'c, 'd,
+          struct('a, 'b).as("e"),
+          struct('a, 'd).as("f"),
+          struct(struct('a, 'b), struct('a, 'd)).as("g")
+        )
+
+      checkAnswer(
+        df,
+        (child: SparkPlan) => createScriptTransformationExec(
+          script = "cat",
+          output = Seq(
+            AttributeReference("a", ArrayType(IntegerType))(),
+            AttributeReference("b", ArrayType(ArrayType(IntegerType)))(),
+            AttributeReference("c", MapType(StringType, IntegerType))(),
+            AttributeReference("d", MapType(StringType, ArrayType(StringType)))(),
+            AttributeReference("e", StructType(
+              Array(StructField("a", ArrayType(IntegerType)),
+                StructField("b", ArrayType(ArrayType(IntegerType))))))(),
+            AttributeReference("f", StructType(
+              Array(StructField("a", ArrayType(IntegerType)),
+                StructField("d", MapType(StringType, ArrayType(StringType))))))(),
+            AttributeReference("g", StructType(
+              Array(StructField("col1", StructType(
+                Array(StructField("a", ArrayType(IntegerType)),
+                  StructField("b", ArrayType(ArrayType(IntegerType)))))),
+                StructField("col2", StructType(
+                  Array(StructField("a", ArrayType(IntegerType)),
+                    StructField("d", MapType(StringType, ArrayType(StringType)))))))))()),
+          child = child,
+          ioschema = defaultIOSchema
+        ),
+        df.select('a, 'b, 'c, 'd, 'e, 'f, 'g).collect())
+    }
+  }
+
+  test("SPARK-33934: Add SparkFile's root dir to env property PATH") {
+    assume(TestUtils.testCommandAvailable("python"))
+    val scriptFilePath = copyAndGetResourceFile("test_script.py", ".py").getAbsoluteFile
+    withTempView("v") {
+      val df = Seq(
+        (1, "1", 1.0, BigDecimal(1.0), new Timestamp(1)),
+        (2, "2", 2.0, BigDecimal(2.0), new Timestamp(2)),
+        (3, "3", 3.0, BigDecimal(3.0), new Timestamp(3))
+      ).toDF("a", "b", "c", "d", "e") // Note column d's data type is Decimal(38, 18)
+      df.createTempView("v")
+
+      // test 'python /path/to/script.py' with local file
+      checkAnswer(
+        sql(
+          s"""
+             |SELECT
+             |TRANSFORM(a, b, c, d, e)
+             |  ROW FORMAT DELIMITED
+             |  FIELDS TERMINATED BY '\t'
+             |  USING 'python $scriptFilePath' AS (a, b, c, d, e)
+             |  ROW FORMAT DELIMITED
+             |  FIELDS TERMINATED BY '\t'
+             |FROM v
+        """.stripMargin), identity, df.select(
+          'a.cast("string"),
+          'b.cast("string"),
+          'c.cast("string"),
+          'd.cast("string"),
+          'e.cast("string")).collect())
+
+      // test '/path/to/script.py' with script not executable
+      val e1 = intercept[TestFailedException] {
+        checkAnswer(
+          sql(
+            s"""
+               |SELECT
+               |TRANSFORM(a, b, c, d, e)
+               |  ROW FORMAT DELIMITED
+               |  FIELDS TERMINATED BY '\t'
+               |  USING '$scriptFilePath' AS (a, b, c, d, e)
+               |  ROW FORMAT DELIMITED
+               |  FIELDS TERMINATED BY '\t'
+               |FROM v
+        """.stripMargin), identity, df.select(
+            'a.cast("string"),
+            'b.cast("string"),
+            'c.cast("string"),
+            'd.cast("string"),
+            'e.cast("string")).collect())
+      }.getMessage
+      // Check with status exit code since in GA test, it may lose detail failed root cause.
+      // Different root cause's exitcode is not same.
+      // In this test, root cause is `Permission denied`
+      assert(e1.contains("Subprocess exited with status 126"))
+
+      // test `/path/to/script.py' with script executable
+      scriptFilePath.setExecutable(true)
+      checkAnswer(
+        sql(
+          s"""
+             |SELECT
+             |TRANSFORM(a, b, c, d, e)
+             |  ROW FORMAT DELIMITED
+             |  FIELDS TERMINATED BY '\t'
+             |  USING '$scriptFilePath' AS (a, b, c, d, e)
+             |  ROW FORMAT DELIMITED
+             |  FIELDS TERMINATED BY '\t'
+             |FROM v
+        """.stripMargin), identity, df.select(
+          'a.cast("string"),
+          'b.cast("string"),
+          'c.cast("string"),
+          'd.cast("string"),
+          'e.cast("string")).collect())
+
+      scriptFilePath.setExecutable(false)
+      sql(s"ADD FILE ${scriptFilePath.getAbsolutePath}")
+
+      // test `script.py` when file added
+      checkAnswer(
+        sql(
+          s"""
+             |SELECT TRANSFORM(a, b, c, d, e)
+             |  ROW FORMAT DELIMITED
+             |  FIELDS TERMINATED BY '\t'
+             |  USING '${scriptFilePath.getName}' AS (a, b, c, d, e)
+             |  ROW FORMAT DELIMITED
+             |  FIELDS TERMINATED BY '\t'
+             |FROM v
+        """.stripMargin), identity, df.select(
+          'a.cast("string"),
+          'b.cast("string"),
+          'c.cast("string"),
+          'd.cast("string"),
+          'e.cast("string")).collect())
+
+      // test `python script.py` when file added
+      checkAnswer(
+        sql(
+          s"""
+             |SELECT TRANSFORM(a, b, c, d, e)
+             |  ROW FORMAT DELIMITED
+             |  FIELDS TERMINATED BY '\t'
+             |  USING 'python ${scriptFilePath.getName}' AS (a, b, c, d, e)
+             |  ROW FORMAT DELIMITED
+             |  FIELDS TERMINATED BY '\t'
+             |FROM v
+        """.stripMargin), identity, df.select(
+          'a.cast("string"),
+          'b.cast("string"),
+          'c.cast("string"),
+          'd.cast("string"),
+          'e.cast("string")).collect())
+    }
+  }
+
+  test("SPARK-35220: DayTimeIntervalType/YearMonthIntervalType show different " +
+    "between hive serde and row format delimited\t") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+    withTempView("v") {
+      val df = Seq(
+        (Duration.ofDays(1), Period.ofMonths(10))
+      ).toDF("a", "b")
+      df.createTempView("v")
+
+      if (defaultSerDe == "hive-serde") {
+        checkAnswer(sql(
+          """
+            |SELECT TRANSFORM(a, b)
+            |  USING 'cat' AS (a, b)
+            |FROM v
+            |""".stripMargin),
+          identity,
+          Row("1 00:00:00.000000000", "0-10") :: Nil)
+      } else {
+        checkAnswer(sql(
+          """
+            |SELECT TRANSFORM(a, b)
+            |  USING 'cat' AS (a, b)
+            |FROM v
+            |""".stripMargin),
+          identity,
+          Row("INTERVAL '1 00:00:00' DAY TO SECOND", "INTERVAL '0-10' YEAR TO MONTH") :: Nil)
+      }
+    }
+  }
+}
+
+case class ExceptionInjectingOperator(child: SparkPlan) extends UnaryExecNode {
+  override protected def doExecute(): RDD[InternalRow] = {
+    child.execute().map { x =>
+      assert(TaskContext.get() != null) // Make sure that TaskContext is defined.
+      Thread.sleep(1000) // This sleep gives the external process time to start.
+      throw new IllegalArgumentException("intentional exception")
+    }
+  }
+
+  override def output: Seq[Attribute] = child.output
+
+  override def outputPartitioning: Partitioning = child.outputPartitioning
+
+  override protected def withNewChildInternal(newChild: SparkPlan): ExceptionInjectingOperator =
+    copy(child = newChild)
+}
+
+@SQLUserDefinedType(udt = classOf[SimpleTupleUDT])
+private class SimpleTuple(val id: Int, val size: Long) extends Serializable {
+
+  override def hashCode(): Int = getClass.hashCode()
+
+  override def equals(other: Any): Boolean = other match {
+    case v: SimpleTuple => this.id == v.id && this.size == v.size
+    case _ => false
+  }
+
+  override def toString: String =
+    compact(render(
+      ("id" -> id) ~
+        ("size" -> size)
+    ))
+}
+
+private class SimpleTupleUDT extends UserDefinedType[SimpleTuple] {
+
+  override def sqlType: DataType = StructType(
+    StructField("id", IntegerType, false) ::
+      StructField("size", LongType, false) ::
+      Nil)
+
+  override def serialize(sql: SimpleTuple): Any = {
+    val row = new GenericInternalRow(2)
+    row.setInt(0, sql.id)
+    row.setLong(1, sql.size)
+    row
+  }
+
+  override def deserialize(datum: Any): SimpleTuple = {
+    datum match {
+      case str: String =>
+        implicit val format = DefaultFormats
+        val json = parse(str)
+        new SimpleTuple((json \ "id").extract[Int], (json \ "size").extract[Long])
+      case data: InternalRow if data.numFields == 2 =>
+        new SimpleTuple(data.getInt(0), data.getLong(1))
+      case _ => null
+    }
+  }
+
+  override def userClass: Class[SimpleTuple] = classOf[SimpleTuple]
+
+  override def asNullable: SimpleTupleUDT = this
+
+  override def hashCode(): Int = getClass.hashCode()
+
+  override def equals(other: Any): Boolean = {
+    other.isInstanceOf[SimpleTupleUDT]
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala
index 22c5b651f7e12..e3688686a187b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.execution
 import org.scalatest.BeforeAndAfterAll
 
 import org.apache.spark.{SparkConf, SparkFunSuite}
+import org.apache.spark.internal.config.IO_ENCRYPTION_ENABLED
 import org.apache.spark.internal.config.UI.UI_ENABLED
 import org.apache.spark.sql._
 import org.apache.spark.sql.execution.adaptive._
@@ -57,15 +58,18 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterAl
   def withSparkSession(
       f: SparkSession => Unit,
       targetPostShuffleInputSize: Int,
-      minNumPostShufflePartitions: Option[Int]): Unit = {
+      minNumPostShufflePartitions: Option[Int],
+      enableIOEncryption: Boolean = false): Unit = {
     val sparkConf =
       new SparkConf(false)
         .setMaster("local[*]")
         .setAppName("test")
         .set(UI_ENABLED, false)
+        .set(IO_ENCRYPTION_ENABLED, enableIOEncryption)
         .set(SQLConf.SHUFFLE_PARTITIONS.key, "5")
         .set(SQLConf.COALESCE_PARTITIONS_INITIAL_PARTITION_NUM.key, "5")
         .set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "true")
+        .set(SQLConf.FETCH_SHUFFLE_BLOCKS_IN_BATCH.key, "true")
         .set(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key, "-1")
         .set(
           SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key,
@@ -90,7 +94,7 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterAl
     }
 
     test(s"determining the number of reducers: aggregate operator$testNameNote") {
-      val test = { spark: SparkSession =>
+      val test: SparkSession => Unit = { spark: SparkSession =>
         val df =
           spark
             .range(0, 1000, 1, numInputPartitions)
@@ -109,14 +113,13 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterAl
         val shuffleReaders = finalPlan.collect {
           case r @ CoalescedShuffleReader() => r
         }
-        assert(shuffleReaders.length === 1)
+
         minNumPostShufflePartitions match {
           case Some(numPartitions) =>
-            shuffleReaders.foreach { reader =>
-              assert(reader.outputPartitioning.numPartitions === numPartitions)
-            }
+            assert(shuffleReaders.isEmpty)
 
           case None =>
+            assert(shuffleReaders.length === 1)
             shuffleReaders.foreach { reader =>
               assert(reader.outputPartitioning.numPartitions === 3)
             }
@@ -127,7 +130,7 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterAl
     }
 
     test(s"determining the number of reducers: join operator$testNameNote") {
-      val test = { spark: SparkSession =>
+      val test: SparkSession => Unit = { spark: SparkSession =>
         val df1 =
           spark
             .range(0, 1000, 1, numInputPartitions)
@@ -156,14 +159,13 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterAl
         val shuffleReaders = finalPlan.collect {
           case r @ CoalescedShuffleReader() => r
         }
-        assert(shuffleReaders.length === 2)
+
         minNumPostShufflePartitions match {
           case Some(numPartitions) =>
-            shuffleReaders.foreach { reader =>
-              assert(reader.outputPartitioning.numPartitions === numPartitions)
-            }
+            assert(shuffleReaders.isEmpty)
 
           case None =>
+            assert(shuffleReaders.length === 2)
             shuffleReaders.foreach { reader =>
               assert(reader.outputPartitioning.numPartitions === 2)
             }
@@ -208,14 +210,13 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterAl
         val shuffleReaders = finalPlan.collect {
           case r @ CoalescedShuffleReader() => r
         }
-        assert(shuffleReaders.length === 2)
+
         minNumPostShufflePartitions match {
           case Some(numPartitions) =>
-            shuffleReaders.foreach { reader =>
-              assert(reader.outputPartitioning.numPartitions === numPartitions)
-            }
+            assert(shuffleReaders.isEmpty)
 
           case None =>
+            assert(shuffleReaders.length === 2)
             shuffleReaders.foreach { reader =>
               assert(reader.outputPartitioning.numPartitions === 2)
             }
@@ -260,14 +261,13 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterAl
         val shuffleReaders = finalPlan.collect {
           case r @ CoalescedShuffleReader() => r
         }
-        assert(shuffleReaders.length === 2)
+
         minNumPostShufflePartitions match {
           case Some(numPartitions) =>
-            shuffleReaders.foreach { reader =>
-              assert(reader.outputPartitioning.numPartitions === numPartitions)
-            }
+            assert(shuffleReaders.isEmpty)
 
           case None =>
+            assert(shuffleReaders.length === 2)
             shuffleReaders.foreach { reader =>
               assert(reader.outputPartitioning.numPartitions === 3)
             }
@@ -327,7 +327,7 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterAl
       val finalPlan = resultDf.queryExecution.executedPlan
         .asInstanceOf[AdaptiveSparkPlanExec].executedPlan
       assert(finalPlan.collect {
-        case ShuffleQueryStageExec(_, r: ReusedExchangeExec) => r
+        case ShuffleQueryStageExec(_, r: ReusedExchangeExec, _) => r
       }.length == 2)
       assert(
         finalPlan.collect {
@@ -363,7 +363,7 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterAl
 
       val reusedStages = level1Stages.flatMap { stage =>
         stage.plan.collect {
-          case ShuffleQueryStageExec(_, r: ReusedExchangeExec) => r
+          case ShuffleQueryStageExec(_, r: ReusedExchangeExec, _) => r
         }
       }
       assert(reusedStages.length == 1)
@@ -408,6 +408,25 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterAl
     }
     withSparkSession(test, 100, None)
   }
+
+  test("SPARK-34790: enable IO encryption in AQE partition coalescing") {
+    val test: SparkSession => Unit = { spark: SparkSession =>
+      val ds = spark.range(0, 100, 1, numInputPartitions)
+      val resultDf = ds.repartition(ds.col("id"))
+      resultDf.collect()
+
+      val finalPlan = resultDf.queryExecution.executedPlan
+        .asInstanceOf[AdaptiveSparkPlanExec].executedPlan
+      assert(
+        finalPlan.collect {
+          case r @ CoalescedShuffleReader() => r
+        }.isDefinedAt(0))
+    }
+    Seq(true, false).foreach { enableIOEncryption =>
+      // Before SPARK-34790, it will throw an exception when io encryption enabled.
+      withSparkSession(test, Int.MaxValue, None, enableIOEncryption)
+    }
+  }
 }
 
 object CoalescedShuffleReader {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ColumnarRulesSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ColumnarRulesSuite.scala
new file mode 100644
index 0000000000000..df08acd35ef17
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ColumnarRulesSuite.scala
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.test.SharedSparkSession
+
+class ColumnarRulesSuite extends PlanTest with SharedSparkSession {
+
+  test("Idempotency of columnar rules - RowToColumnar/ColumnarToRow") {
+    val rules = ApplyColumnarRulesAndInsertTransitions(
+      spark.sessionState.columnarRules)
+
+    val plan = UnaryOp(UnaryOp(LeafOp(false), true), false)
+    val expected =
+      UnaryOp(ColumnarToRowExec(UnaryOp(RowToColumnarExec(LeafOp(false)), true)), false)
+    val appliedOnce = rules.apply(plan)
+    assert(appliedOnce == expected)
+    val appliedTwice = rules.apply(appliedOnce)
+    assert(appliedTwice == expected)
+  }
+
+  test("Idempotency of columnar rules - ColumnarToRow/RowToColumnar") {
+    val rules = ApplyColumnarRulesAndInsertTransitions(
+      spark.sessionState.columnarRules)
+
+    val plan = UnaryOp(UnaryOp(LeafOp(true), false), true)
+    val expected = ColumnarToRowExec(
+      UnaryOp(RowToColumnarExec(UnaryOp(ColumnarToRowExec(LeafOp(true)), false)), true))
+    val appliedOnce = rules.apply(plan)
+    assert(appliedOnce == expected)
+    val appliedTwice = rules.apply(appliedOnce)
+    assert(appliedTwice == expected)
+  }
+}
+
+case class LeafOp(override val supportsColumnar: Boolean) extends LeafExecNode {
+  override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException()
+  override def output: Seq[Attribute] = Seq.empty
+}
+
+case class UnaryOp(child: SparkPlan, override val supportsColumnar: Boolean) extends UnaryExecNode {
+  override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException()
+  override def output: Seq[Attribute] = child.output
+  override protected def withNewChildInternal(newChild: SparkPlan): UnaryOp = copy(child = newChild)
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala
index f1411b263c77b..612cd6f0d891b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala
@@ -16,7 +16,10 @@
  */
 package org.apache.spark.sql.execution
 
+import java.io.File
+
 import scala.collection.mutable
+import scala.util.Random
 
 import org.apache.hadoop.fs.Path
 
@@ -116,6 +119,48 @@ class DataSourceScanExecRedactionSuite extends DataSourceScanRedactionTest {
       assert(isIncluded(df.queryExecution, "Location"))
     }
   }
+
+  test("SPARK-31793: FileSourceScanExec metadata should contain limited file paths") {
+    withTempPath { path =>
+      // create a sub-directory with long name so that each root path will always exceed the limit
+      // this is to ensure we always test the case for the path truncation
+      val dataDirName = Random.alphanumeric.take(100).toList.mkString
+      val dataDir = new File(path, dataDirName)
+      dataDir.mkdir()
+
+      val partitionCol = "partitionCol"
+      spark.range(10)
+        .select("id", "id")
+        .toDF("value", partitionCol)
+        .write
+        .partitionBy(partitionCol)
+        .orc(dataDir.getCanonicalPath)
+      val paths = (0 to 9).map(i => new File(dataDir, s"$partitionCol=$i").getCanonicalPath)
+      val plan = spark.read.orc(paths: _*).queryExecution.executedPlan
+      val location = plan collectFirst {
+        case f: FileSourceScanExec => f.metadata("Location")
+      }
+      assert(location.isDefined)
+      // The location metadata should at least contain one path
+      assert(location.get.contains(paths.head))
+
+      // The location metadata should have the number of root paths
+      assert(location.get.contains("(10 paths)"))
+
+      // The location metadata should have bracket wrapping paths
+      assert(location.get.indexOf('[') > -1)
+      assert(location.get.indexOf(']') > -1)
+
+      // extract paths in location metadata (removing classname, brackets, separators)
+      val pathsInLocation = location.get.substring(
+        location.get.indexOf('[') + 1, location.get.indexOf(']')).split(", ").toSeq
+
+      // the only one path should be available
+      assert(pathsInLocation.size == 2)
+      // indicator ("...") should be available
+      assert(pathsInLocation.exists(_.contains("...")))
+    }
+  }
 }
 
 /**
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ExchangeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ExchangeSuite.scala
index fb97e15e4df63..9776e76b541ee 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ExchangeSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ExchangeSuite.scala
@@ -40,6 +40,9 @@ case class ColumnarExchange(child: SparkPlan) extends Exchange {
   override protected def doExecute(): RDD[InternalRow] = throw new RanRowBased
 
   override protected def doExecuteColumnar(): RDD[ColumnarBatch] = throw new RanColumnar
+
+  override protected def withNewChildInternal(newChild: SparkPlan): ColumnarExchange =
+    copy(child = newChild)
 }
 
 class ExchangeSuite extends SparkPlanTest with SharedSparkSession {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArrayBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArrayBenchmark.scala
index 0869e25674e69..e7f83cb7eb4bc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArrayBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArrayBenchmark.scala
@@ -47,6 +47,9 @@ object ExternalAppendOnlyUnsafeRowArrayBenchmark extends BenchmarkBase {
     // for a bug we had with bytes written past the last object in a batch (SPARK-2792)
     .set(config.SERIALIZER_OBJECT_STREAM_RESET, 1)
     .set(config.SERIALIZER, "org.apache.spark.serializer.JavaSerializer")
+    // SPARK-34832: Add this configuration to allow `withFakeTaskContext` method
+    // to create `SparkContext` on the executor side.
+    .set(config.EXECUTOR_ALLOW_SPARK_CONTEXT, true)
 
   private def withFakeTaskContext(f: => Unit): Unit = {
     val sc = new SparkContext("local", "test", conf)
@@ -54,6 +57,7 @@ object ExternalAppendOnlyUnsafeRowArrayBenchmark extends BenchmarkBase {
     TaskContext.setTaskContext(taskContext)
     f
     sc.stop()
+    TaskContext.unset()
   }
 
   private def testRows(numRows: Int): Seq[UnsafeRow] = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArraySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArraySuite.scala
index b29de9c4adbaa..98aba3ba25f17 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArraySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArraySuite.scala
@@ -27,32 +27,29 @@ import org.apache.spark.sql.catalyst.expressions.UnsafeRow
 
 class ExternalAppendOnlyUnsafeRowArraySuite extends SparkFunSuite with LocalSparkContext {
   private val random = new java.util.Random()
-  private var taskContext: TaskContext = _
-
-  override def afterAll(): Unit = try {
-    TaskContext.unset()
-  } finally {
-    super.afterAll()
-  }
 
   private def withExternalArray(inMemoryThreshold: Int, spillThreshold: Int)
                                (f: ExternalAppendOnlyUnsafeRowArray => Unit): Unit = {
     sc = new SparkContext("local", "test", new SparkConf(false))
 
-    taskContext = MemoryTestingUtils.fakeTaskContext(SparkEnv.get)
+    val taskContext = MemoryTestingUtils.fakeTaskContext(SparkEnv.get)
     TaskContext.setTaskContext(taskContext)
 
-    val array = new ExternalAppendOnlyUnsafeRowArray(
-      taskContext.taskMemoryManager(),
-      SparkEnv.get.blockManager,
-      SparkEnv.get.serializerManager,
-      taskContext,
-      1024,
-      SparkEnv.get.memoryManager.pageSizeBytes,
-      inMemoryThreshold,
-      spillThreshold)
-    try f(array) finally {
-      array.clear()
+    try {
+      val array = new ExternalAppendOnlyUnsafeRowArray(
+        taskContext.taskMemoryManager(),
+        SparkEnv.get.blockManager,
+        SparkEnv.get.serializerManager,
+        taskContext,
+        1024,
+        SparkEnv.get.memoryManager.pageSizeBytes,
+        inMemoryThreshold,
+        spillThreshold)
+      try f(array) finally {
+        array.clear()
+      }
+    } finally {
+      TaskContext.unset()
     }
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala
index 5e81c74420fd0..f8366b3f7c5fa 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala
@@ -17,21 +17,30 @@
 
 package org.apache.spark.sql.execution
 
-import org.apache.spark.sql.connector.InMemoryTableCatalog
+import java.time.{Duration, Period}
+
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils
+import org.apache.spark.sql.connector.catalog.InMemoryTableCatalog
+import org.apache.spark.sql.execution.HiveResult._
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.{ExamplePoint, ExamplePointUDT, SharedSparkSession}
 
 class HiveResultSuite extends SharedSparkSession {
   import testImplicits._
 
   test("date formatting in hive result") {
-    val dates = Seq("2018-12-28", "1582-10-03", "1582-10-04", "1582-10-15")
-    val df = dates.toDF("a").selectExpr("cast(a as date) as b")
-    val executedPlan1 = df.queryExecution.executedPlan
-    val result = HiveResult.hiveResultString(executedPlan1)
-    assert(result == dates)
-    val executedPlan2 = df.selectExpr("array(b)").queryExecution.executedPlan
-    val result2 = HiveResult.hiveResultString(executedPlan2)
-    assert(result2 == dates.map(x => s"[$x]"))
+    DateTimeTestUtils.outstandingTimezonesIds.foreach { zoneId =>
+      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> zoneId) {
+        val dates = Seq("2018-12-28", "1582-10-03", "1582-10-04", "1582-10-15")
+        val df = dates.toDF("a").selectExpr("cast(a as date) as b")
+        val executedPlan1 = df.queryExecution.executedPlan
+        val result = hiveResultString(executedPlan1)
+        assert(result == dates)
+        val executedPlan2 = df.selectExpr("array(b)").queryExecution.executedPlan
+        val result2 = hiveResultString(executedPlan2)
+        assert(result2 == dates.map(x => s"[$x]"))
+      }
+    }
   }
 
   test("timestamp formatting in hive result") {
@@ -42,17 +51,17 @@ class HiveResultSuite extends SharedSparkSession {
       "1582-10-15 01:02:03")
     val df = timestamps.toDF("a").selectExpr("cast(a as timestamp) as b")
     val executedPlan1 = df.queryExecution.executedPlan
-    val result = HiveResult.hiveResultString(executedPlan1)
+    val result = hiveResultString(executedPlan1)
     assert(result == timestamps)
     val executedPlan2 = df.selectExpr("array(b)").queryExecution.executedPlan
-    val result2 = HiveResult.hiveResultString(executedPlan2)
+    val result2 = hiveResultString(executedPlan2)
     assert(result2 == timestamps.map(x => s"[$x]"))
   }
 
   test("toHiveString correctly handles UDTs") {
     val point = new ExamplePoint(50.0, 50.0)
     val tpe = new ExamplePointUDT()
-    assert(HiveResult.toHiveString((point, tpe)) === "(50.0, 50.0)")
+    assert(toHiveString((point, tpe), false, getTimeFormatters) === "(50.0, 50.0)")
   }
 
   test("decimal formatting in hive result") {
@@ -60,13 +69,13 @@ class HiveResultSuite extends SharedSparkSession {
     Seq(2, 6, 18).foreach { scala =>
       val executedPlan =
         df.selectExpr(s"CAST(value AS decimal(38, $scala))").queryExecution.executedPlan
-      val result = HiveResult.hiveResultString(executedPlan)
+      val result = hiveResultString(executedPlan)
       assert(result.head.split("\\.").last.length === scala)
     }
 
     val executedPlan = Seq(java.math.BigDecimal.ZERO).toDS()
       .selectExpr(s"CAST(value AS decimal(38, 8))").queryExecution.executedPlan
-    val result = HiveResult.hiveResultString(executedPlan)
+    val result = hiveResultString(executedPlan)
     assert(result.head === "0.00000000")
   }
 
@@ -78,7 +87,7 @@ class HiveResultSuite extends SharedSparkSession {
             spark.sql(s"CREATE TABLE $ns.$tbl (id bigint) USING $source")
             val df = spark.sql(s"SHOW TABLES FROM $ns")
             val executedPlan = df.queryExecution.executedPlan
-            assert(HiveResult.hiveResultString(executedPlan).head == tbl)
+            assert(hiveResultString(executedPlan).head == tbl)
           }
       }
     }
@@ -95,9 +104,25 @@ class HiveResultSuite extends SharedSparkSession {
             val expected = "id                  " +
               "\tbigint              " +
               "\tcol1                "
-            assert(HiveResult.hiveResultString(executedPlan).head == expected)
+            assert(hiveResultString(executedPlan).head == expected)
           }
       }
     }
   }
+
+  test("SPARK-34984, SPARK-35016: year-month interval formatting in hive result") {
+    val df = Seq(Period.ofYears(-10).minusMonths(1)).toDF("i")
+    val plan1 = df.queryExecution.executedPlan
+    assert(hiveResultString(plan1) === Seq("-10-1"))
+    val plan2 = df.selectExpr("array(i)").queryExecution.executedPlan
+    assert(hiveResultString(plan2) === Seq("[-10-1]"))
+  }
+
+  test("SPARK-34984, SPARK-35016: day-time interval formatting in hive result") {
+    val df = Seq(Duration.ofDays(5).plusMillis(10)).toDF("i")
+    val plan1 = df.queryExecution.executedPlan
+    assert(hiveResultString(plan1) === Seq("5 00:00:00.010000000"))
+    val plan2 = df.selectExpr("array(i)").queryExecution.executedPlan
+    assert(hiveResultString(plan2) === Seq("[5 00:00:00.010000000]"))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala
index d428b7ebc0e91..0b30b8cdf2644 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala
@@ -22,7 +22,7 @@ import org.apache.spark.sql.{execution, DataFrame, Row}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans._
-import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan, Range, Repartition, Sort, Union}
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan, Range, Repartition, RepartitionOperation, Union}
 import org.apache.spark.sql.catalyst.plans.physical._
 import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecution}
 import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec}
@@ -61,13 +61,13 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
   }
 
   test("count distinct is partially aggregated") {
-    val query = testData.groupBy('value).agg(countDistinct('key)).queryExecution.analyzed
+    val query = testData.groupBy('value).agg(count_distinct('key)).queryExecution.analyzed
     testPartialAggregationPlan(query)
   }
 
   test("mixed aggregates are partially aggregated") {
     val query =
-      testData.groupBy('value).agg(count('value), countDistinct('key)).queryExecution.analyzed
+      testData.groupBy('value).agg(count('value), count_distinct('key)).queryExecution.analyzed
     testPartialAggregationPlan(query)
   }
 
@@ -234,19 +234,6 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
     }
   }
 
-  test("SPARK-23375: Cached sorted data doesn't need to be re-sorted") {
-    val query = testData.select('key, 'value).sort('key.desc).cache()
-    assert(query.queryExecution.optimizedPlan.isInstanceOf[InMemoryRelation])
-    val resorted = query.sort('key.desc)
-    assert(resorted.queryExecution.optimizedPlan.collect { case s: Sort => s}.isEmpty)
-    assert(resorted.select('key).collect().map(_.getInt(0)).toSeq ==
-      (1 to 100).reverse)
-    // with a different order, the sort is needed
-    val sortedAsc = query.sort('key)
-    assert(sortedAsc.queryExecution.optimizedPlan.collect { case s: Sort => s}.size == 1)
-    assert(sortedAsc.select('key).collect().map(_.getInt(0)).toSeq == (1 to 100))
-  }
-
   test("PartitioningCollection") {
     withTempView("normal", "small", "tiny") {
       testData.createOrReplaceTempView("normal")
@@ -342,7 +329,7 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
       requiredChildDistribution = Seq(distribution, distribution),
       requiredChildOrdering = Seq(Seq.empty, Seq.empty)
     )
-    val outputPlan = EnsureRequirements(spark.sessionState.conf).apply(inputPlan)
+    val outputPlan = EnsureRequirements.apply(inputPlan)
     assertDistributionRequirementsAreSatisfied(outputPlan)
   }
 
@@ -360,7 +347,7 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
       requiredChildDistribution = Seq(distribution, distribution),
       requiredChildOrdering = Seq(Seq.empty, Seq.empty)
     )
-    val outputPlan = EnsureRequirements(spark.sessionState.conf).apply(inputPlan)
+    val outputPlan = EnsureRequirements.apply(inputPlan)
     assertDistributionRequirementsAreSatisfied(outputPlan)
     if (outputPlan.collect { case e: ShuffleExchangeExec => true }.isEmpty) {
       fail(s"Exchange should have been added:\n$outputPlan")
@@ -380,7 +367,7 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
       requiredChildDistribution = Seq(distribution, distribution),
       requiredChildOrdering = Seq(Seq.empty, Seq.empty)
     )
-    val outputPlan = EnsureRequirements(spark.sessionState.conf).apply(inputPlan)
+    val outputPlan = EnsureRequirements.apply(inputPlan)
     assertDistributionRequirementsAreSatisfied(outputPlan)
     if (outputPlan.collect { case e: ShuffleExchangeExec => true }.nonEmpty) {
       fail(s"Exchange should not have been added:\n$outputPlan")
@@ -403,7 +390,7 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
       requiredChildDistribution = Seq(distribution, distribution),
       requiredChildOrdering = Seq(outputOrdering, outputOrdering)
     )
-    val outputPlan = EnsureRequirements(spark.sessionState.conf).apply(inputPlan)
+    val outputPlan = EnsureRequirements.apply(inputPlan)
     assertDistributionRequirementsAreSatisfied(outputPlan)
     if (outputPlan.collect { case e: ShuffleExchangeExec => true }.nonEmpty) {
       fail(s"No Exchanges should have been added:\n$outputPlan")
@@ -418,7 +405,7 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
     val inputPlan = ShuffleExchangeExec(
       partitioning,
       DummySparkPlan(outputPartitioning = partitioning))
-    val outputPlan = EnsureRequirements(spark.sessionState.conf).apply(inputPlan)
+    val outputPlan = EnsureRequirements.apply(inputPlan)
     assertDistributionRequirementsAreSatisfied(outputPlan)
     if (outputPlan.collect { case e: ShuffleExchangeExec => true }.size == 2) {
       fail(s"Topmost Exchange should have been eliminated:\n$outputPlan")
@@ -433,7 +420,7 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
     val inputPlan = ShuffleExchangeExec(
       partitioning,
       DummySparkPlan(outputPartitioning = partitioning))
-    val outputPlan = EnsureRequirements(spark.sessionState.conf).apply(inputPlan)
+    val outputPlan = EnsureRequirements.apply(inputPlan)
     assertDistributionRequirementsAreSatisfied(outputPlan)
     if (outputPlan.collect { case e: ShuffleExchangeExec => true }.size == 1) {
       fail(s"Topmost Exchange should not have been eliminated:\n$outputPlan")
@@ -451,7 +438,7 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
         requiredChildDistribution = Seq(distribution),
         requiredChildOrdering = Seq(Seq.empty))
 
-    val outputPlan = EnsureRequirements(spark.sessionState.conf).apply(inputPlan)
+    val outputPlan = EnsureRequirements.apply(inputPlan)
     val shuffle = outputPlan.collect { case e: ShuffleExchangeExec => e }
     assert(shuffle.size === 1)
     assert(shuffle.head.outputPartitioning === finalPartitioning)
@@ -476,7 +463,7 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
       shuffle,
       shuffle)
 
-    val outputPlan = ReuseExchange(spark.sessionState.conf).apply(inputPlan)
+    val outputPlan = ReuseExchange.apply(inputPlan)
     if (outputPlan.collect { case e: ReusedExchangeExec => true }.size != 1) {
       fail(s"Should re-use the shuffle:\n$outputPlan")
     }
@@ -493,7 +480,7 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
       ShuffleExchangeExec(finalPartitioning, inputPlan),
       ShuffleExchangeExec(finalPartitioning, inputPlan))
 
-    val outputPlan2 = ReuseExchange(spark.sessionState.conf).apply(inputPlan2)
+    val outputPlan2 = ReuseExchange.apply(inputPlan2)
     if (outputPlan2.collect { case e: ReusedExchangeExec => true }.size != 2) {
       fail(s"Should re-use the two shuffles:\n$outputPlan2")
     }
@@ -530,7 +517,7 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
       requiredChildOrdering = Seq(requiredOrdering),
       requiredChildDistribution = Seq(UnspecifiedDistribution)
     )
-    val outputPlan = EnsureRequirements(spark.sessionState.conf).apply(inputPlan)
+    val outputPlan = EnsureRequirements.apply(inputPlan)
     assertDistributionRequirementsAreSatisfied(outputPlan)
     if (shouldHaveSort) {
       if (outputPlan.collect { case s: SortExec => true }.isEmpty) {
@@ -691,7 +678,7 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
     val smjExec = SortMergeJoinExec(
       exprA :: exprA :: Nil, exprB :: exprC :: Nil, Inner, None, plan1, plan2)
 
-    val outputPlan = EnsureRequirements(spark.sessionState.conf).apply(smjExec)
+    val outputPlan = EnsureRequirements.apply(smjExec)
     outputPlan match {
       case SortMergeJoinExec(leftKeys, rightKeys, _, _, _, _, _) =>
         assert(leftKeys == Seq(exprA, exprA))
@@ -711,7 +698,7 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
       condition = None,
       left = plan1,
       right = plan2)
-    val outputPlan = EnsureRequirements(spark.sessionState.conf).apply(smjExec)
+    val outputPlan = EnsureRequirements.apply(smjExec)
     outputPlan match {
       case SortMergeJoinExec(leftKeys, rightKeys, _, _,
              SortExec(_, _,
@@ -741,8 +728,6 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
       case r: Range => r
     }
     assert(ranges.length == 2)
-    // Ensure the two Range instances are equal according to their equal method
-    assert(ranges.head == ranges.last)
     val execRanges = df.queryExecution.sparkPlan.collect {
       case r: RangeExec => r
     }
@@ -902,8 +887,225 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
             |  (SELECT key AS k from df2) t2
             |ON t1.k = t2.k
           """.stripMargin).queryExecution.executedPlan
-        val exchanges = planned.collect { case s: ShuffleExchangeExec => s }
+        val exchanges = collect(planned) { case s: ShuffleExchangeExec => s }
+        assert(exchanges.size == 2)
+      }
+    }
+  }
+
+  test("SPARK-33399: aliases should be handled properly in PartitioningCollection output" +
+    " partitioning") {
+    withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+      withTempView("t1", "t2", "t3") {
+        spark.range(10).repartition($"id").createTempView("t1")
+        spark.range(20).repartition($"id").createTempView("t2")
+        spark.range(30).repartition($"id").createTempView("t3")
+        val planned = sql(
+          """
+            |SELECT t3.id as t3id
+            |FROM (
+            |    SELECT t1.id as t1id, t2.id as t2id
+            |    FROM t1, t2
+            |    WHERE t1.id = t2.id
+            |) t12, t3
+            |WHERE t1id = t3.id
+          """.stripMargin).queryExecution.executedPlan
+        val exchanges = collect(planned) { case s: ShuffleExchangeExec => s }
+        assert(exchanges.size == 3)
+
+        val projects = collect(planned) { case p: ProjectExec => p }
+        assert(projects.exists(_.outputPartitioning match {
+          case HashPartitioning(Seq(k1: AttributeReference), _) if k1.name == "t1id" =>
+            true
+          case _ =>
+            false
+        }))
+      }
+    }
+  }
+
+  test("SPARK-33399: aliases should be handled properly in HashPartitioning") {
+    withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+      withTempView("t1", "t2", "t3") {
+        spark.range(10).repartition($"id").createTempView("t1")
+        spark.range(20).repartition($"id").createTempView("t2")
+        spark.range(30).repartition($"id").createTempView("t3")
+        val planned = sql(
+          """
+            |SELECT t1id, t3.id as t3id
+            |FROM (
+            |    SELECT t1.id as t1id
+            |    FROM t1 LEFT SEMI JOIN t2
+            |    ON t1.id = t2.id
+            |) t12 INNER JOIN t3
+            |WHERE t1id = t3.id
+          """.stripMargin).queryExecution.executedPlan
+        val exchanges = collect(planned) { case s: ShuffleExchangeExec => s }
+        assert(exchanges.size == 3)
+
+        val projects = collect(planned) { case p: ProjectExec => p }
+        assert(projects.exists(_.outputPartitioning match {
+          case HashPartitioning(Seq(a: AttributeReference), _) => a.name == "t1id"
+          case _ => false
+        }))
+      }
+    }
+  }
+
+  test("SPARK-33399: alias handling should happen properly for RangePartitioning") {
+    withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+      val df = spark.range(1, 100)
+        .select(col("id").as("id1")).groupBy("id1").count()
+      // Plan for this will be Range -> ProjectWithAlias -> HashAggregate -> HashAggregate
+      // if Project normalizes alias in its Range outputPartitioning, then no Exchange should come
+      // in between HashAggregates
+      val planned = df.queryExecution.executedPlan
+      val exchanges = collect(planned) { case s: ShuffleExchangeExec => s }
+      assert(exchanges.isEmpty)
+
+      val projects = collect(planned) { case p: ProjectExec => p }
+      assert(projects.exists(_.outputPartitioning match {
+        case RangePartitioning(Seq(SortOrder(ar: AttributeReference, _, _, _)), _) =>
+          ar.name == "id1"
+        case _ => false
+      }))
+    }
+  }
+
+  test("SPARK-33399: aliased should be handled properly " +
+    "for partitioning and sortorder involving complex expressions") {
+    withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+      withTempView("t1", "t2", "t3") {
+        spark.range(10).select(col("id").as("id1")).createTempView("t1")
+        spark.range(20).select(col("id").as("id2")).createTempView("t2")
+        spark.range(30).select(col("id").as("id3")).createTempView("t3")
+        val planned = sql(
+          """
+            |SELECT t3.id3 as t3id
+            |FROM (
+            |    SELECT t1.id1 as t1id, t2.id2 as t2id
+            |    FROM t1, t2
+            |    WHERE t1.id1 * 10 = t2.id2 * 10
+            |) t12, t3
+            |WHERE t1id * 10 = t3.id3 * 10
+          """.stripMargin).queryExecution.executedPlan
+        val sortNodes = collect(planned) { case s: SortExec => s }
+        assert(sortNodes.size == 3)
+        val exchangeNodes = collect(planned) { case e: ShuffleExchangeExec => e }
+        assert(exchangeNodes.size == 3)
+
+        val projects = collect(planned) { case p: ProjectExec => p }
+        assert(projects.exists(_.outputPartitioning match {
+          case HashPartitioning(Seq(Multiply(ar1: AttributeReference, _, _)), _) =>
+            ar1.name == "t1id"
+          case _ =>
+            false
+        }))
+      }
+    }
+  }
+
+  test("SPARK-33399: alias handling should happen properly for SinglePartition") {
+    withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+      val df = spark.range(1, 100, 1, 1)
+        .select(col("id").as("id1")).groupBy("id1").count()
+      val planned = df.queryExecution.executedPlan
+      val exchanges = collect(planned) { case s: ShuffleExchangeExec => s }
+      assert(exchanges.isEmpty)
+
+      val projects = collect(planned) { case p: ProjectExec => p }
+      assert(projects.exists(_.outputPartitioning match {
+        case SinglePartition => true
+        case _ => false
+      }))
+    }
+  }
+
+  test("SPARK-33399: No extra exchanges in case of" +
+    " [Inner Join -> Project with aliases -> HashAggregate]") {
+    withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+      withTempView("t1", "t2") {
+        spark.range(10).repartition($"id").createTempView("t1")
+        spark.range(20).repartition($"id").createTempView("t2")
+        val planned = sql(
+          """
+            |SELECT t1id, t2id
+            |FROM (
+            |  SELECT t1.id as t1id, t2.id as t2id
+            |  FROM t1 INNER JOIN t2
+            |  WHERE t1.id = t2.id
+            |) t12
+            |GROUP BY t1id, t2id
+          """.stripMargin).queryExecution.executedPlan
+        val exchanges = collect(planned) { case s: ShuffleExchangeExec => s }
         assert(exchanges.size == 2)
+
+        val projects = collect(planned) { case p: ProjectExec => p }
+        assert(projects.exists(_.outputPartitioning match {
+          case PartitioningCollection(Seq(HashPartitioning(Seq(k1: AttributeReference), _),
+          HashPartitioning(Seq(k2: AttributeReference), _))) =>
+            k1.name == "t1id" && k2.name == "t2id"
+          case _ => false
+        }))
+      }
+    }
+  }
+
+  test("SPARK-33400: Normalization of sortOrder should take care of sameOrderExprs") {
+    withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+      withTempView("t1", "t2", "t3") {
+        spark.range(10).repartition($"id").createTempView("t1")
+        spark.range(20).repartition($"id").createTempView("t2")
+        spark.range(30).repartition($"id").createTempView("t3")
+        val planned = sql(
+          """
+            |SELECT t2id, t3.id as t3id
+            |FROM (
+            |    SELECT t1.id as t1id, t2.id as t2id
+            |    FROM t1, t2
+            |    WHERE t1.id = t2.id
+            |) t12, t3
+            |WHERE t2id = t3.id
+          """.stripMargin).queryExecution.executedPlan
+
+        val sortNodes = collect(planned) { case s: SortExec => s }
+        assert(sortNodes.size == 3)
+
+        val projects = collect(planned) { case p: ProjectExec => p }
+        assert(projects.exists(_.outputOrdering match {
+          case Seq(SortOrder(_, Ascending, NullsFirst, sameOrderExprs)) =>
+            sameOrderExprs.size == 1 && sameOrderExprs.head.isInstanceOf[AttributeReference] &&
+              sameOrderExprs.head.asInstanceOf[AttributeReference].name == "t2id"
+          case _ => false
+        }))
+      }
+    }
+  }
+
+  test("sort order doesn't have repeated expressions") {
+    withSQLConf(
+      SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
+      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
+      withTempView("t1", "t2") {
+        spark.range(10).repartition($"id").createTempView("t1")
+        spark.range(20).repartition($"id").createTempView("t2")
+        val planned = sql(
+          """
+            | SELECT t12.id, t1.id
+            | FROM (SELECT t1.id FROM t1, t2 WHERE t1.id * 2 = t2.id) t12, t1
+            | where 2 * t12.id = t1.id
+        """.stripMargin).queryExecution.executedPlan
+
+        // t12 is already sorted on `t1.id * 2`. and we need to sort it on `2 * t12.id`
+        // for 2nd join. So sorting on t12 can be avoided
+        val sortNodes = planned.collect { case s: SortExec => s }
+        assert(sortNodes.size == 3)
+        val outputOrdering = planned.outputOrdering
+        assert(outputOrdering.size == 1)
+        // Sort order should have 3 childrens, not 4. This is because t1.id*2 and 2*t1.id are same
+        assert(outputOrdering.head.children.size == 3)
+        assert(outputOrdering.head.children.count(_.isInstanceOf[AttributeReference]) == 2)
+        assert(outputOrdering.head.children.count(_.isInstanceOf[Multiply]) == 1)
       }
     }
   }
@@ -921,7 +1123,7 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
             |  (SELECT key + 1 AS k2 from df2) t2
             |ON t1.k1 = t2.k2
             |""".stripMargin).queryExecution.executedPlan
-        val exchanges = planned.collect { case s: ShuffleExchangeExec => s }
+        val exchanges = collect(planned) { case s: ShuffleExchangeExec => s }
 
         // Make sure aliases to an expression (key + 1) are not replaced.
         Seq("k1", "k2").foreach { alias =>
@@ -944,9 +1146,9 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
 
       val planned = agg1.join(agg2, $"k1" === $"k3").queryExecution.executedPlan
 
-      assert(planned.collect { case h: HashAggregateExec => h }.nonEmpty)
+      assert(collect(planned) { case h: HashAggregateExec => h }.nonEmpty)
 
-      val exchanges = planned.collect { case s: ShuffleExchangeExec => s }
+      val exchanges = collect(planned) { case s: ShuffleExchangeExec => s }
       assert(exchanges.size == 2)
     }
   }
@@ -964,12 +1166,12 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
           val planned = agg1.join(agg2, $"k1" === $"k3").queryExecution.executedPlan
 
           if (useObjectHashAgg) {
-            assert(planned.collect { case o: ObjectHashAggregateExec => o }.nonEmpty)
+            assert(collect(planned) { case o: ObjectHashAggregateExec => o }.nonEmpty)
           } else {
-            assert(planned.collect { case s: SortAggregateExec => s }.nonEmpty)
+            assert(collect(planned) { case s: SortAggregateExec => s }.nonEmpty)
           }
 
-          val exchanges = planned.collect { case s: ShuffleExchangeExec => s }
+          val exchanges = collect(planned) { case s: ShuffleExchangeExec => s }
           assert(exchanges.size == 2)
         }
       }
@@ -977,23 +1179,79 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
   }
 
   test("aliases in the sort aggregate expressions should not introduce extra sort") {
-    withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
-      withSQLConf(SQLConf.USE_OBJECT_HASH_AGG.key -> "false") {
-        val t1 = spark.range(10).selectExpr("floor(id/4) as k1")
-        val t2 = spark.range(20).selectExpr("floor(id/4) as k2")
+    withSQLConf(
+      SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
+      SQLConf.USE_OBJECT_HASH_AGG.key -> "false") {
+      val t1 = spark.range(10).selectExpr("floor(id/4) as k1")
+      val t2 = spark.range(20).selectExpr("floor(id/4) as k2")
+
+      val agg1 = t1.groupBy("k1").agg(collect_list("k1")).withColumnRenamed("k1", "k3")
+      val agg2 = t2.groupBy("k2").agg(collect_list("k2"))
 
-        val agg1 = t1.groupBy("k1").agg(collect_list("k1")).withColumnRenamed("k1", "k3")
-        val agg2 = t2.groupBy("k2").agg(collect_list("k2"))
+      val planned = agg1.join(agg2, $"k3" === $"k2").queryExecution.executedPlan
+      assert(collect(planned) { case s: SortAggregateExec => s }.nonEmpty)
 
-        val planned = agg1.join(agg2, $"k3" === $"k2").queryExecution.executedPlan
-        assert(planned.collect { case s: SortAggregateExec => s }.nonEmpty)
+      // We expect two SortExec nodes on each side of join.
+      val sorts = collect(planned) { case s: SortExec => s }
+      assert(sorts.size == 4)
+    }
+  }
 
-        // We expect two SortExec nodes on each side of join.
-        val sorts = planned.collect { case s: SortExec => s }
-        assert(sorts.size == 4)
+  testWithWholeStageCodegenOnAndOff("Change the number of partitions to zero " +
+    "when a range is empty") { _ =>
+    val range = spark.range(1, 1, 1, 1000)
+    val numPartitions = range.rdd.getNumPartitions
+    assert(numPartitions == 0)
+  }
+
+  test("SPARK-33758: Prune unnecessary output partitioning") {
+    withSQLConf(
+      SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
+      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
+      withTempView("t1", "t2") {
+        spark.range(10).repartition($"id").createTempView("t1")
+        spark.range(20).repartition($"id").createTempView("t2")
+        val planned = sql(
+          """
+            | SELECT t1.id as t1id, t2.id as t2id
+            | FROM t1, t2
+            | WHERE t1.id = t2.id
+          """.stripMargin).queryExecution.executedPlan
+
+        assert(planned.outputPartitioning match {
+          case PartitioningCollection(Seq(HashPartitioning(Seq(k1: AttributeReference), _),
+          HashPartitioning(Seq(k2: AttributeReference), _))) =>
+            k1.name == "t1id" && k2.name == "t2id"
+        })
+
+        val planned2 = sql(
+          """
+            | SELECT t1.id as t1id
+            | FROM t1, t2
+            | WHERE t1.id = t2.id
+          """.stripMargin).queryExecution.executedPlan
+        assert(planned2.outputPartitioning match {
+          case HashPartitioning(Seq(k1: AttributeReference), _) if k1.name == "t1id" =>
+            true
+        })
       }
     }
   }
+
+  test("SPARK-34919: Change partitioning to SinglePartition if partition number is 1") {
+    def checkSinglePartitioning(df: DataFrame): Unit = {
+      assert(
+        df.queryExecution.analyzed.collect {
+          case r: RepartitionOperation => r
+        }.size == 1)
+      assert(
+        collect(df.queryExecution.executedPlan) {
+          case s: ShuffleExchangeExec if s.outputPartitioning == SinglePartition => s
+        }.size == 1)
+    }
+    checkSinglePartitioning(sql("SELECT /*+ REPARTITION(1) */ * FROM VALUES(1),(2),(3) AS t(c)"))
+    checkSinglePartitioning(sql("SELECT /*+ REPARTITION(1, c) */ * FROM VALUES(1),(2),(3) AS t(c)"))
+  }
 }
 
 // Used for unit-testing EnsureRequirements
@@ -1006,4 +1264,6 @@ private case class DummySparkPlan(
   ) extends SparkPlan {
   override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException
   override def output: Seq[Attribute] = Seq.empty
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[SparkPlan]): SparkPlan =
+    copy(children = newChildren)
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala
index eca39f3f81726..eb93a5eca6560 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala
@@ -24,6 +24,7 @@ import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, OneRowRelation}
 import org.apache.spark.sql.catalyst.trees.TreeNodeTag
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.util.Utils
 
 case class QueryExecutionTestRecord(
     c0: Int, c1: Int, c2: Int, c3: Int, c4: Int,
@@ -36,8 +37,9 @@ case class QueryExecutionTestRecord(
 class QueryExecutionSuite extends SharedSparkSession {
   import testImplicits._
 
-  def checkDumpedPlans(path: String, expected: Int): Unit = {
-    assert(Source.fromFile(path).getLines.toList
+  def checkDumpedPlans(path: String, expected: Int): Unit = Utils.tryWithResource(
+    Source.fromFile(path)) { source =>
+    assert(source.getLines.toList
       .takeWhile(_ != "== Whole Stage Codegen ==") == List(
       "== Parsed Logical Plan ==",
       s"Range (0, $expected, step=1, splits=Some(2))",
@@ -53,6 +55,7 @@ class QueryExecutionSuite extends SharedSparkSession {
       s"*(1) Range (0, $expected, step=1, splits=2)",
       ""))
   }
+
   test("dumping query execution info to a file") {
     withTempDir { dir =>
       val path = dir.getCanonicalPath + "/plans.txt"
@@ -93,6 +96,26 @@ class QueryExecutionSuite extends SharedSparkSession {
     assert(exception.getMessage.contains("Illegal character in scheme name"))
   }
 
+  test("dumping query execution info to a file - explainMode=formatted") {
+    withTempDir { dir =>
+      val path = dir.getCanonicalPath + "/plans.txt"
+      val df = spark.range(0, 10)
+      df.queryExecution.debug.toFile(path, explainMode = Option("formatted"))
+      val lines = Utils.tryWithResource(Source.fromFile(path))(_.getLines().toList)
+      assert(lines
+        .takeWhile(_ != "== Whole Stage Codegen ==").map(_.replaceAll("#\\d+", "#x")) == List(
+        "== Physical Plan ==",
+        s"* Range (1)",
+        "",
+        "",
+        s"(1) Range [codegen id : 1]",
+        "Output [1]: [id#xL]",
+        s"Arguments: Range (0, 10, step=1, splits=Some(2))",
+        "",
+        ""))
+    }
+  }
+
   test("limit number of fields by sql config") {
     def relationPlans: String = {
       val ds = spark.createDataset(Seq(QueryExecutionTestRecord(
@@ -115,9 +138,10 @@ class QueryExecutionSuite extends SharedSparkSession {
         0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
         16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26)))
       ds.queryExecution.debug.toFile(path)
-      val localRelations = Source.fromFile(path).getLines().filter(_.contains("LocalRelation"))
-
-      assert(!localRelations.exists(_.contains("more fields")))
+      Utils.tryWithResource(Source.fromFile(path)) { source =>
+        val localRelations = source.getLines().filter(_.contains("LocalRelation"))
+        assert(!localRelations.exists(_.contains("more fields")))
+      }
     }
   }
 
@@ -191,4 +215,25 @@ class QueryExecutionSuite extends SharedSparkSession {
     df.queryExecution.executedPlan.setTagValue(tag5, "v")
     assertNoTag(tag5, df.queryExecution.sparkPlan)
   }
+
+  test("Logging plan changes for execution") {
+    val testAppender = new LogAppender("plan changes")
+    withLogAppender(testAppender) {
+      withSQLConf(SQLConf.PLAN_CHANGE_LOG_LEVEL.key -> "INFO") {
+        spark.range(1).groupBy("id").count().queryExecution.executedPlan
+      }
+    }
+    Seq("=== Applying Rule org.apache.spark.sql.execution",
+        "=== Result of Batch Preparations ===").foreach { expectedMsg =>
+      assert(testAppender.loggingEvents.exists(_.getRenderedMessage.contains(expectedMsg)))
+    }
+  }
+
+  test("SPARK-34129: Add table name to LogicalRelation.simpleString") {
+    withTable("spark_34129") {
+      spark.sql("CREATE TABLE spark_34129(id INT) using parquet")
+      val df = spark.table("spark_34129")
+      assert(df.queryExecution.optimizedPlan.toString.startsWith("Relation default.spark_34129["))
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReferenceSort.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ReferenceSort.scala
index 25b4464823e5f..1592949fe9a9b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReferenceSort.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ReferenceSort.scala
@@ -20,7 +20,6 @@ package org.apache.spark.sql.execution
 import org.apache.spark.TaskContext
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.errors._
 import org.apache.spark.sql.catalyst.expressions.{Attribute, RowOrdering, SortOrder}
 import org.apache.spark.sql.catalyst.plans.physical._
 import org.apache.spark.util.CompletionIterator
@@ -39,7 +38,7 @@ case class ReferenceSort(
   override def requiredChildDistribution: Seq[Distribution] =
     if (global) OrderedDistribution(sortOrder) :: Nil else UnspecifiedDistribution :: Nil
 
-  protected override def doExecute(): RDD[InternalRow] = attachTree(this, "sort") {
+  protected override def doExecute(): RDD[InternalRow] = {
     child.execute().mapPartitions( { iterator =>
       val ordering = RowOrdering.create(sortOrder, child.output)
       val sorter = new ExternalSorter[InternalRow, Null, InternalRow](
@@ -59,4 +58,7 @@ case class ReferenceSort(
   override def outputOrdering: Seq[SortOrder] = sortOrder
 
   override def outputPartitioning: Partitioning = child.outputPartitioning
+
+  override protected def withNewChildInternal(newChild: SparkPlan): ReferenceSort =
+    copy(child = newChild)
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantProjectsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantProjectsSuite.scala
new file mode 100644
index 0000000000000..944aa963cc4be
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantProjectsSuite.scala
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import org.apache.spark.sql.{DataFrame, QueryTest, Row}
+import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.StructType
+import org.apache.spark.util.Utils
+
+abstract class RemoveRedundantProjectsSuiteBase
+  extends QueryTest
+    with SharedSparkSession
+    with AdaptiveSparkPlanHelper {
+
+  private def assertProjectExecCount(df: DataFrame, expected: Int): Unit = {
+    withClue(df.queryExecution) {
+      val plan = df.queryExecution.executedPlan
+      val actual = collectWithSubqueries(plan) { case p: ProjectExec => p }.size
+      assert(actual == expected)
+    }
+  }
+
+  private def assertProjectExec(query: String, enabled: Int, disabled: Int): Unit = {
+    val df = sql(query)
+    // When enabling AQE, the DPP subquery filters is replaced in runtime.
+    df.collect()
+    assertProjectExecCount(df, enabled)
+    val result = df.collect()
+    withSQLConf(SQLConf.REMOVE_REDUNDANT_PROJECTS_ENABLED.key -> "false") {
+      val df2 = sql(query)
+      df2.collect()
+      assertProjectExecCount(df2, disabled)
+      checkAnswer(df2, result)
+    }
+  }
+
+  private val tmpPath = Utils.createTempDir()
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    tmpPath.delete()
+    val path = tmpPath.getAbsolutePath
+    spark.range(100).selectExpr("id % 10 as key", "cast(id * 2 as int) as a",
+      "cast(id * 3 as int) as b", "cast(id as string) as c", "array(id, id + 1, id + 3) as d")
+      .write.partitionBy("key").parquet(path)
+    spark.read.parquet(path).createOrReplaceTempView("testView")
+  }
+
+  override def afterAll(): Unit = {
+    Utils.deleteRecursively(tmpPath)
+    super.afterAll()
+  }
+
+  test("project") {
+    val query = "select * from testView"
+    assertProjectExec(query, 0, 0)
+  }
+
+  test("project with filter") {
+    val query = "select * from testView where a > 5"
+    assertProjectExec(query, 0, 1)
+  }
+
+  test("project with specific column ordering") {
+    val query = "select key, a, b, c from testView"
+    assertProjectExec(query, 1, 1)
+  }
+
+  test("project with extra columns") {
+    val query = "select a, b, c, key, a from testView"
+    assertProjectExec(query, 1, 1)
+  }
+
+  test("project with fewer columns") {
+    val query = "select a from testView where a > 3"
+    assertProjectExec(query, 1, 1)
+  }
+
+  test("aggregate without ordering requirement") {
+    val query = "select sum(a) as sum_a, key, last(b) as last_b " +
+      "from (select key, a, b from testView where a > 100) group by key"
+    assertProjectExec(query, 0, 1)
+  }
+
+  test("aggregate with ordering requirement") {
+    val query = "select a, sum(b) as sum_b from testView group by a"
+    assertProjectExec(query, 1, 1)
+  }
+
+  test("join without ordering requirement") {
+    val query = "select t1.key, t2.key, t1.a, t2.b from (select key, a, b, c from testView)" +
+      " as t1 join (select key, a, b, c from testView) as t2 on t1.c > t2.c and t1.key > 10"
+    assertProjectExec(query, 1, 3)
+  }
+
+  test("join with ordering requirement") {
+    val query = "select * from (select key, a, c, b from testView) as t1 join " +
+      "(select key, a, b, c from testView) as t2 on t1.key = t2.key where t2.a > 50"
+    assertProjectExec(query, 2, 2)
+  }
+
+  test("window function") {
+    val query = "select key, b, avg(a) over (partition by key order by a " +
+      "rows between 1 preceding and 1 following) as avg from testView"
+    assertProjectExec(query, 1, 2)
+  }
+
+  test("generate should require column ordering") {
+    withTempView("testData") {
+      spark.range(0, 10, 1)
+        .selectExpr("id as key", "id * 2 as a", "id * 3 as b")
+        .createOrReplaceTempView("testData")
+
+      val data = sql("select key, a, b, count(*) from testData group by key, a, b limit 2")
+      val df = data.selectExpr("a", "b", "key", "explode(array(key, a, b)) as d").filter("d > 0")
+      df.collect()
+      val plan = df.queryExecution.executedPlan
+      val numProjects = collectWithSubqueries(plan) { case p: ProjectExec => p }.length
+
+      // Create a new plan that reverse the GenerateExec output and add a new ProjectExec between
+      // GenerateExec and its child. This is to test if the ProjectExec is removed, the output of
+      // the query will be incorrect.
+      val newPlan = stripAQEPlan(plan) transform {
+        case g @ GenerateExec(_, requiredChildOutput, _, _, child) =>
+          g.copy(requiredChildOutput = requiredChildOutput.reverse,
+            child = ProjectExec(requiredChildOutput.reverse, child))
+      }
+
+      // Re-apply remove redundant project rule.
+      val rule = RemoveRedundantProjects
+      val newExecutedPlan = rule.apply(newPlan)
+      // The manually added ProjectExec node shouldn't be removed.
+      assert(collectWithSubqueries(newExecutedPlan) {
+        case p: ProjectExec => p
+      }.size == numProjects + 1)
+
+      // Check the original plan's output and the new plan's output are the same.
+      val expectedRows = plan.executeCollect()
+      val actualRows = newExecutedPlan.executeCollect()
+      assert(expectedRows.length == actualRows.length)
+      expectedRows.zip(actualRows).foreach { case (expected, actual) => assert(expected == actual) }
+    }
+  }
+
+  test("subquery") {
+    withTempView("testData") {
+      val data = spark.sparkContext.parallelize((1 to 100).map(i => Row(i, i.toString)))
+      val schema = new StructType().add("key", "int").add("value", "string")
+      spark.createDataFrame(data, schema).createOrReplaceTempView("testData")
+      val query = "select key, value from testData where key in " +
+        "(select sum(a) from testView where a > 5 group by key)"
+      assertProjectExec(query, 0, 1)
+    }
+  }
+
+  test("SPARK-33697: UnionExec should require column ordering") {
+    withTable("t1", "t2") {
+      spark.range(-10, 20)
+        .selectExpr(
+          "id",
+          "date_add(date '1950-01-01', cast(id as int)) as datecol",
+          "cast(id as string) strcol")
+        .write.mode("overwrite").format("parquet").saveAsTable("t1")
+      spark.range(-10, 20)
+        .selectExpr(
+          "cast(id as string) strcol",
+          "id",
+          "date_add(date '1950-01-01', cast(id as int)) as datecol")
+        .write.mode("overwrite").format("parquet").saveAsTable("t2")
+
+      val queryTemplate =
+        """
+          |SELECT DISTINCT datecol, strcol FROM
+          |(
+          |(SELECT datecol, id, strcol from t1)
+          | %s
+          |(SELECT datecol, id, strcol from t2)
+          |)
+          |""".stripMargin
+
+      Seq(("UNION", 1, 2), ("UNION ALL", 1, 2)).foreach { case (setOperation, enabled, disabled) =>
+        val query = queryTemplate.format(setOperation)
+        assertProjectExec(query, enabled = enabled, disabled = disabled)
+      }
+    }
+  }
+
+  test("SPARK-33697: remove redundant projects under expand") {
+    val query =
+      """
+        |SELECT t1.key, t2.key, sum(t1.a) AS s1, sum(t2.b) AS s2 FROM
+        |(SELECT a, key FROM testView) t1
+        |JOIN
+        |(SELECT b, key FROM testView) t2
+        |ON t1.key = t2.key
+        |GROUP BY t1.key, t2.key GROUPING SETS(t1.key, t2.key)
+        |ORDER BY t1.key, t2.key, s1, s2
+        |LIMIT 10
+        |""".stripMargin
+    assertProjectExec(query, 0, 3)
+  }
+}
+
+class RemoveRedundantProjectsSuite extends RemoveRedundantProjectsSuiteBase
+  with DisableAdaptiveExecutionSuite
+
+class RemoveRedundantProjectsSuiteAE extends RemoveRedundantProjectsSuiteBase
+  with EnableAdaptiveExecutionSuite
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantSortsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantSortsSuite.scala
new file mode 100644
index 0000000000000..751078d08fda9
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantSortsSuite.scala
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import org.apache.spark.sql.{DataFrame, QueryTest}
+import org.apache.spark.sql.catalyst.plans.physical.{RangePartitioning, UnknownPartitioning}
+import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
+import org.apache.spark.sql.execution.joins.ShuffledJoin
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.test.SharedSparkSession
+
+
+abstract class RemoveRedundantSortsSuiteBase
+    extends QueryTest
+    with SharedSparkSession
+    with AdaptiveSparkPlanHelper {
+  import testImplicits._
+
+  private def checkNumSorts(df: DataFrame, count: Int): Unit = {
+    val plan = df.queryExecution.executedPlan
+    assert(collectWithSubqueries(plan) { case s: SortExec => s }.length == count)
+  }
+
+  private def checkSorts(query: String, enabledCount: Int, disabledCount: Int): Unit = {
+    withSQLConf(SQLConf.REMOVE_REDUNDANT_SORTS_ENABLED.key -> "true") {
+      val df = sql(query)
+      checkNumSorts(df, enabledCount)
+      val result = df.collect()
+      withSQLConf(SQLConf.REMOVE_REDUNDANT_SORTS_ENABLED.key -> "false") {
+        val df = sql(query)
+        checkNumSorts(df, disabledCount)
+        checkAnswer(df, result)
+      }
+    }
+  }
+
+  test("remove redundant sorts with limit") {
+    withTempView("t") {
+      spark.range(100).select('id as "key").createOrReplaceTempView("t")
+      val query =
+        """
+          |SELECT key FROM
+          | (SELECT key FROM t WHERE key > 10 ORDER BY key DESC LIMIT 10)
+          |ORDER BY key DESC
+          |""".stripMargin
+      checkSorts(query, 0, 1)
+    }
+  }
+
+  test("remove redundant sorts with broadcast hash join") {
+    withTempView("t1", "t2") {
+      spark.range(1000).select('id as "key").createOrReplaceTempView("t1")
+      spark.range(1000).select('id as "key").createOrReplaceTempView("t2")
+
+      val queryTemplate = """
+        |SELECT /*+ BROADCAST(%s) */ t1.key FROM
+        | (SELECT key FROM t1 WHERE key > 10 ORDER BY key DESC LIMIT 10) t1
+        |JOIN
+        | (SELECT key FROM t2 WHERE key > 50 ORDER BY key DESC LIMIT 100) t2
+        |ON t1.key = t2.key
+        |ORDER BY %s
+      """.stripMargin
+
+      // No sort should be removed since the stream side (t2) order DESC
+      // does not satisfy the required sort order ASC.
+      val buildLeftOrderByRightAsc = queryTemplate.format("t1", "t2.key ASC")
+      checkSorts(buildLeftOrderByRightAsc, 1, 1)
+
+      // The top sort node should be removed since the stream side (t2) order DESC already
+      // satisfies the required sort order DESC.
+      val buildLeftOrderByRightDesc = queryTemplate.format("t1", "t2.key DESC")
+      checkSorts(buildLeftOrderByRightDesc, 0, 1)
+
+      // No sort should be removed since the sort ordering from broadcast-hash join is based
+      // on the stream side (t2) and the required sort order is from t1.
+      val buildLeftOrderByLeftDesc = queryTemplate.format("t1", "t1.key DESC")
+      checkSorts(buildLeftOrderByLeftDesc, 1, 1)
+
+      // The top sort node should be removed since the stream side (t1) order DESC already
+      // satisfies the required sort order DESC.
+      val buildRightOrderByLeftDesc = queryTemplate.format("t2", "t1.key DESC")
+      checkSorts(buildRightOrderByLeftDesc, 0, 1)
+    }
+  }
+
+  test("remove redundant sorts with sort merge join") {
+    withTempView("t1", "t2") {
+      spark.range(1000).select('id as "key").createOrReplaceTempView("t1")
+      spark.range(1000).select('id as "key").createOrReplaceTempView("t2")
+      val query = """
+        |SELECT /*+ MERGE(t1) */ t1.key FROM
+        | (SELECT key FROM t1 WHERE key > 10 ORDER BY key DESC LIMIT 10) t1
+        |JOIN
+        | (SELECT key FROM t2 WHERE key > 50 ORDER BY key DESC LIMIT 100) t2
+        |ON t1.key = t2.key
+        |ORDER BY t1.key
+      """.stripMargin
+
+      val queryAsc = query + " ASC"
+      checkSorts(queryAsc, 2, 3)
+
+      // The top level sort should not be removed since the child output ordering is ASC and
+      // the required ordering is DESC.
+      val queryDesc = query + " DESC"
+      checkSorts(queryDesc, 3, 3)
+    }
+  }
+
+  test("cached sorted data doesn't need to be re-sorted") {
+    withSQLConf(SQLConf.REMOVE_REDUNDANT_SORTS_ENABLED.key -> "true") {
+      val df = spark.range(1000).select('id as "key").sort('key.desc).cache()
+      val resorted = df.sort('key.desc)
+      val sortedAsc = df.sort('key.asc)
+      checkNumSorts(df, 0)
+      checkNumSorts(resorted, 0)
+      checkNumSorts(sortedAsc, 1)
+      val result = resorted.collect()
+      withSQLConf(SQLConf.REMOVE_REDUNDANT_SORTS_ENABLED.key -> "false") {
+        val resorted = df.sort('key.desc)
+        checkNumSorts(resorted, 1)
+        checkAnswer(resorted, result)
+      }
+    }
+  }
+
+  test("SPARK-33472: shuffled join with different left and right side partition numbers") {
+    withTempView("t1", "t2") {
+      spark.range(0, 100, 1, 2).select('id as "key").createOrReplaceTempView("t1")
+      (0 to 100).toDF("key").createOrReplaceTempView("t2")
+
+      val queryTemplate = """
+        |SELECT /*+ %s(t1) */ t1.key
+        |FROM t1 JOIN t2 ON t1.key = t2.key
+        |WHERE t1.key > 10 AND t2.key < 50
+        |ORDER BY t1.key ASC
+      """.stripMargin
+
+      Seq(("MERGE", 3), ("SHUFFLE_HASH", 1)).foreach { case (hint, count) =>
+        val query = queryTemplate.format(hint)
+        val df = sql(query)
+        val sparkPlan = df.queryExecution.sparkPlan
+        val join = sparkPlan.collect { case j: ShuffledJoin => j }.head
+        val leftPartitioning = join.left.outputPartitioning
+        assert(leftPartitioning.isInstanceOf[RangePartitioning])
+        assert(leftPartitioning.numPartitions == 2)
+        assert(join.right.outputPartitioning == UnknownPartitioning(0))
+        checkSorts(query, count, count)
+      }
+    }
+  }
+}
+
+class RemoveRedundantSortsSuite extends RemoveRedundantSortsSuiteBase
+  with DisableAdaptiveExecutionSuite
+
+class RemoveRedundantSortsSuiteAE extends RemoveRedundantSortsSuiteBase
+  with EnableAdaptiveExecutionSuite
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLExecutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLExecutionSuite.scala
index 8bf7fe62cd49b..81e692076b432 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLExecutionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLExecutionSuite.scala
@@ -17,11 +17,17 @@
 
 package org.apache.spark.sql.execution
 
+import java.util.concurrent.Executors
+
 import scala.collection.parallel.immutable.ParRange
+import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.duration._
 
 import org.apache.spark.{SparkConf, SparkContext, SparkFunSuite}
 import org.apache.spark.scheduler.{SparkListener, SparkListenerJobStart}
-import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.{Row, SparkSession}
+import org.apache.spark.sql.types._
+import org.apache.spark.util.ThreadUtils
 
 class SQLExecutionSuite extends SparkFunSuite {
 
@@ -119,6 +125,38 @@ class SQLExecutionSuite extends SparkFunSuite {
 
     spark.stop()
   }
+
+  test("SPARK-32813: Table scan should work in different thread") {
+    val executor1 = Executors.newSingleThreadExecutor()
+    val executor2 = Executors.newSingleThreadExecutor()
+    var session: SparkSession = null
+    SparkSession.cleanupAnyExistingSession()
+
+    withTempDir { tempDir =>
+      try {
+        val tablePath = tempDir.toString + "/table"
+        val df = ThreadUtils.awaitResult(Future {
+          session = SparkSession.builder().appName("test").master("local[*]").getOrCreate()
+
+          session.createDataFrame(
+            session.sparkContext.parallelize(Row(Array(1, 2, 3)) :: Nil),
+            StructType(Seq(
+              StructField("a", ArrayType(IntegerType, containsNull = false), nullable = false))))
+            .write.parquet(tablePath)
+
+          session.read.parquet(tablePath)
+        }(ExecutionContext.fromExecutorService(executor1)), 1.minute)
+
+        ThreadUtils.awaitResult(Future {
+          assert(df.rdd.collect()(0) === Row(Seq(1, 2, 3)))
+        }(ExecutionContext.fromExecutorService(executor2)), 1.minute)
+      } finally {
+        executor1.shutdown()
+        executor2.shutdown()
+        session.stop()
+      }
+    }
+  }
 }
 
 object SQLExecutionSuite {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala
index 575efec364812..adb212d653ce9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala
@@ -17,10 +17,12 @@
 
 package org.apache.spark.sql.execution
 
+import org.apache.spark.SparkException
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
-import org.apache.spark.sql.internal.SQLConf.MAX_NESTED_VIEW_DEPTH
+import org.apache.spark.sql.catalyst.parser.ParseException
+import org.apache.spark.sql.internal.SQLConf._
 import org.apache.spark.sql.test.{SharedSparkSession, SQLTestUtils}
 
 class SimpleSQLViewSuite extends SQLViewSuite with SharedSparkSession
@@ -109,7 +111,7 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils {
       e = intercept[AnalysisException] {
         sql("ALTER VIEW tab1 AS SELECT * FROM jt")
       }.getMessage
-      assert(e.contains("`tab1` is not a view"))
+      assert(e.contains("tab1 is a table. 'ALTER VIEW ... AS' expects a view."))
     }
   }
 
@@ -125,8 +127,12 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils {
     val viewName = "testView"
     withTempView(viewName) {
       spark.range(10).createTempView(viewName)
-      assertNoSuchTable(s"ALTER VIEW $viewName SET TBLPROPERTIES ('p' = 'an')")
-      assertNoSuchTable(s"ALTER VIEW $viewName UNSET TBLPROPERTIES ('p')")
+      assertAnalysisError(
+        s"ALTER VIEW $viewName SET TBLPROPERTIES ('p' = 'an')",
+        "testView is a temp view. 'ALTER VIEW ... SET TBLPROPERTIES' expects a permanent view.")
+      assertAnalysisError(
+        s"ALTER VIEW $viewName UNSET TBLPROPERTIES ('p')",
+        "testView is a temp view. 'ALTER VIEW ... UNSET TBLPROPERTIES' expects a permanent view.")
     }
   }
 
@@ -134,24 +140,50 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils {
     val viewName = "testView"
     withTempView(viewName) {
       spark.range(10).createTempView(viewName)
-      assertNoSuchTable(s"ALTER TABLE $viewName SET SERDE 'whatever'")
-      assertNoSuchTable(s"ALTER TABLE $viewName PARTITION (a=1, b=2) SET SERDE 'whatever'")
-      assertNoSuchTable(s"ALTER TABLE $viewName SET SERDEPROPERTIES ('p' = 'an')")
-      assertNoSuchTable(s"ALTER TABLE $viewName ADD IF NOT EXISTS PARTITION (a='4', b='8')")
-      assertNoSuchTable(s"ALTER TABLE $viewName DROP PARTITION (a='4', b='8')")
-      assertNoSuchTable(s"ALTER TABLE $viewName PARTITION (a='4') RENAME TO PARTITION (a='5')")
-      assertNoSuchTable(s"ALTER TABLE $viewName RECOVER PARTITIONS")
-
-      // For v2 ALTER TABLE statements, we have better error message saying view is not supported.
-      assertAnalysisError(
+      assertErrorForAlterTableOnTempView(
+        s"ALTER TABLE $viewName SET SERDE 'whatever'",
+        viewName,
+        "ALTER TABLE ... SET [SERDE|SERDEPROPERTIES]")
+      assertErrorForAlterTableOnTempView(
+        s"ALTER TABLE $viewName PARTITION (a=1, b=2) SET SERDE 'whatever'",
+        viewName,
+        "ALTER TABLE ... SET [SERDE|SERDEPROPERTIES]")
+      assertErrorForAlterTableOnTempView(
+        s"ALTER TABLE $viewName SET SERDEPROPERTIES ('p' = 'an')",
+        viewName,
+        "ALTER TABLE ... SET [SERDE|SERDEPROPERTIES]")
+      assertErrorForAlterTableOnTempView(
+        s"ALTER TABLE $viewName PARTITION (a='4') RENAME TO PARTITION (a='5')",
+        viewName,
+        "ALTER TABLE ... RENAME TO PARTITION")
+      assertErrorForAlterTableOnTempView(
+        s"ALTER TABLE $viewName RECOVER PARTITIONS",
+        viewName,
+        "ALTER TABLE ... RECOVER PARTITIONS")
+      assertErrorForAlterTableOnTempView(
         s"ALTER TABLE $viewName SET LOCATION '/path/to/your/lovely/heart'",
-        s"'$viewName' is a view not a table")
-
-      // For the following v2 ALERT TABLE statements, unsupported operations are checked first
-      // before resolving the relations.
-      assertAnalysisError(
+        viewName,
+        "ALTER TABLE ... SET LOCATION ...")
+      assertErrorForAlterTableOnTempView(
         s"ALTER TABLE $viewName PARTITION (a='4') SET LOCATION '/path/to/home'",
-        "ALTER TABLE SET LOCATION does not support partition for v2 tables")
+        viewName,
+        "ALTER TABLE ... SET LOCATION ...")
+      assertErrorForAlterTableOnTempView(
+        s"ALTER TABLE $viewName ADD IF NOT EXISTS PARTITION (a='4', b='8')",
+        viewName,
+        "ALTER TABLE ... ADD PARTITION ...")
+      assertErrorForAlterTableOnTempView(
+        s"ALTER TABLE $viewName DROP PARTITION (a='4', b='8')",
+        viewName,
+        "ALTER TABLE ... DROP PARTITION ...")
+      assertErrorForAlterTableOnTempView(
+        s"ALTER TABLE $viewName SET TBLPROPERTIES ('p' = 'an')",
+        viewName,
+        "ALTER TABLE ... SET TBLPROPERTIES")
+      assertErrorForAlterTableOnTempView(
+        s"ALTER TABLE $viewName UNSET TBLPROPERTIES ('p')",
+        viewName,
+        "ALTER TABLE ... UNSET TBLPROPERTIES")
     }
   }
 
@@ -167,15 +199,24 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils {
 
       val dataFilePath =
         Thread.currentThread().getContextClassLoader.getResource("data/files/employee.dat")
-      assertNoSuchTable(s"""LOAD DATA LOCAL INPATH "$dataFilePath" INTO TABLE $viewName""")
-      assertNoSuchTable(s"TRUNCATE TABLE $viewName")
       val e2 = intercept[AnalysisException] {
+        sql(s"""LOAD DATA LOCAL INPATH "$dataFilePath" INTO TABLE $viewName""")
+      }.getMessage
+      assert(e2.contains(s"$viewName is a temp view. 'LOAD DATA' expects a table"))
+      val e3 = intercept[AnalysisException] {
         sql(s"SHOW CREATE TABLE $viewName")
       }.getMessage
-      assert(e2.contains("SHOW CREATE TABLE is not supported on a temporary view"))
-      assertNoSuchTable(s"SHOW PARTITIONS $viewName")
-      assertNoSuchTable(s"ANALYZE TABLE $viewName COMPUTE STATISTICS")
-      assertNoSuchTable(s"ANALYZE TABLE $viewName COMPUTE STATISTICS FOR COLUMNS id")
+      assert(e3.contains(
+        s"$viewName is a temp view. 'SHOW CREATE TABLE' expects a table or permanent view."))
+      val e4 = intercept[AnalysisException] {
+        sql(s"ANALYZE TABLE $viewName COMPUTE STATISTICS")
+      }.getMessage
+      assert(e4.contains(
+        s"$viewName is a temp view. 'ANALYZE TABLE' expects a table or permanent view."))
+      val e5 = intercept[AnalysisException] {
+        sql(s"ANALYZE TABLE $viewName COMPUTE STATISTICS FOR COLUMNS id")
+      }.getMessage
+      assert(e5.contains(s"Temporary view `$viewName` is not cached for analyzing columns."))
     }
   }
 
@@ -190,7 +231,14 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils {
     assert(e.message.contains(message))
   }
 
-  test("error handling: insert/load/truncate table commands against a view") {
+  private def assertErrorForAlterTableOnTempView(
+    sqlText: String, viewName: String, cmdName: String): Unit = {
+    assertAnalysisError(
+      sqlText,
+      s"$viewName is a temp view. '$cmdName' expects a table. Please use ALTER VIEW instead.")
+  }
+
+  test("error handling: insert/load table commands against a view") {
     val viewName = "testView"
     withView(viewName) {
       sql(s"CREATE VIEW $viewName AS SELECT id FROM jt")
@@ -204,12 +252,7 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils {
       e = intercept[AnalysisException] {
         sql(s"""LOAD DATA LOCAL INPATH "$dataFilePath" INTO TABLE $viewName""")
       }.getMessage
-      assert(e.contains(s"Target table in LOAD DATA cannot be a view: `default`.`testview`"))
-
-      e = intercept[AnalysisException] {
-        sql(s"TRUNCATE TABLE $viewName")
-      }.getMessage
-      assert(e.contains(s"Operation not allowed: TRUNCATE TABLE on views: `default`.`testview`"))
+      assert(e.contains("default.testView is a view. 'LOAD DATA' expects a table"))
     }
   }
 
@@ -266,6 +309,16 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils {
     }
   }
 
+  test("SPARK-32374: disallow setting properties for CREATE TEMPORARY VIEW") {
+    withTempView("myabcdview") {
+      val e = intercept[ParseException] {
+        sql("CREATE TEMPORARY VIEW myabcdview TBLPROPERTIES ('a' = 'b') AS SELECT * FROM jt")
+      }
+      assert(e.message.contains(
+        "Operation not allowed: TBLPROPERTIES can't coexist with CREATE TEMPORARY VIEW"))
+    }
+  }
+
   test("correctly parse CREATE VIEW statement") {
     withView("testView") {
       sql(
@@ -301,7 +354,6 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils {
       sql(
         """CREATE TEMPORARY VIEW
           |testView (c1 COMMENT 'blabla', c2 COMMENT 'blabla')
-          |TBLPROPERTIES ('a' = 'b')
           |AS SELECT * FROM jt
           |""".stripMargin)
       checkAnswer(sql("SELECT c1, c2 FROM testView ORDER BY c1"), (1 to 9).map(i => Row(i, i)))
@@ -416,8 +468,13 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils {
   }
 
   test("should not allow ALTER VIEW AS when the view does not exist") {
-    assertNoSuchTable("ALTER VIEW testView AS SELECT 1, 2")
-    assertNoSuchTable("ALTER VIEW default.testView AS SELECT 1, 2")
+    assertAnalysisError(
+      "ALTER VIEW testView AS SELECT 1, 2",
+      "View not found: testView")
+
+    assertAnalysisError(
+      "ALTER VIEW default.testView AS SELECT 1, 2",
+      "View not found: default.testView")
   }
 
   test("ALTER VIEW AS should try to alter temp view first if view name has no database part") {
@@ -676,31 +733,6 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils {
     }
   }
 
-  test("restrict the nested level of a view") {
-    val viewNames = Array.range(0, 11).map(idx => s"view$idx")
-    withView(viewNames: _*) {
-      sql("CREATE VIEW view0 AS SELECT * FROM jt")
-      Array.range(0, 10).foreach { idx =>
-        sql(s"CREATE VIEW view${idx + 1} AS SELECT * FROM view$idx")
-      }
-
-      withSQLConf(MAX_NESTED_VIEW_DEPTH.key -> "10") {
-        val e = intercept[AnalysisException] {
-          sql("SELECT * FROM view10")
-        }.getMessage
-        assert(e.contains("The depth of view `default`.`view0` exceeds the maximum view " +
-          "resolution depth (10). Analysis is aborted to avoid errors. Increase the value " +
-          s"of ${MAX_NESTED_VIEW_DEPTH.key} to work around this."))
-      }
-
-      val e = intercept[IllegalArgumentException] {
-        withSQLConf(MAX_NESTED_VIEW_DEPTH.key -> "0") {}
-      }.getMessage
-      assert(e.contains("The maximum depth of a view reference in a nested view must be " +
-        "positive."))
-    }
-  }
-
   test("permanent view should be case-preserving") {
     withView("v") {
       sql("CREATE VIEW v AS SELECT 1 as aBc")
@@ -713,7 +745,7 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils {
 
   test("sparkSession API view resolution with different default database") {
     withDatabase("db2") {
-      withView("v1") {
+      withView("default.v1") {
         withTable("t1") {
           sql("USE default")
           sql("CREATE TABLE t1 USING parquet AS SELECT 1 AS c0")
@@ -735,4 +767,147 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils {
       }
     }
   }
+
+  test("temporary view should ignore useCurrentSQLConfigsForView config") {
+    withTable("t") {
+      Seq(2, 3, 1).toDF("c1").write.format("parquet").saveAsTable("t")
+      withTempView("v1") {
+        sql("CREATE TEMPORARY VIEW v1 AS SELECT 1/0")
+        withSQLConf(
+          USE_CURRENT_SQL_CONFIGS_FOR_VIEW.key -> "true",
+          ANSI_ENABLED.key -> "true") {
+          checkAnswer(sql("SELECT * FROM v1"), Seq(Row(null)))
+        }
+      }
+    }
+  }
+
+  test("alter temporary view should follow current storeAnalyzedPlanForView config") {
+    withTable("t") {
+      Seq(2, 3, 1).toDF("c1").write.format("parquet").saveAsTable("t")
+      withView("v1") {
+        withSQLConf(STORE_ANALYZED_PLAN_FOR_VIEW.key -> "true") {
+          sql("CREATE TEMPORARY VIEW v1 AS SELECT * FROM t")
+          Seq(4, 6, 5).toDF("c1").write.mode("overwrite").format("parquet").saveAsTable("t")
+          val e = intercept[SparkException] {
+            sql("SELECT * FROM v1").collect()
+          }.getMessage
+          assert(e.contains("does not exist"))
+        }
+
+        withSQLConf(STORE_ANALYZED_PLAN_FOR_VIEW.key -> "false") {
+          // alter view from legacy to non-legacy config
+          sql("ALTER VIEW v1 AS SELECT * FROM t")
+          Seq(1, 3, 5).toDF("c1").write.mode("overwrite").format("parquet").saveAsTable("t")
+          checkAnswer(sql("SELECT * FROM v1"), Seq(Row(1), Row(3), Row(5)))
+        }
+
+        withSQLConf(STORE_ANALYZED_PLAN_FOR_VIEW.key -> "true") {
+          // alter view from non-legacy to legacy config
+          sql("ALTER VIEW v1 AS SELECT * FROM t")
+          Seq(2, 4, 6).toDF("c1").write.mode("overwrite").format("parquet").saveAsTable("t")
+          val e = intercept[SparkException] {
+            sql("SELECT * FROM v1").collect()
+          }.getMessage
+          assert(e.contains("does not exist"))
+        }
+      }
+    }
+  }
+
+  test("local temp view refers global temp view") {
+    withGlobalTempView("v1") {
+      withTempView("v2") {
+        val globalTempDB = spark.sharedState.globalTempViewManager.database
+        sql("CREATE GLOBAL TEMPORARY VIEW v1 AS SELECT 1")
+        sql(s"CREATE TEMPORARY VIEW v2 AS SELECT * FROM ${globalTempDB}.v1")
+        checkAnswer(sql("SELECT * FROM v2"), Seq(Row(1)))
+      }
+    }
+  }
+
+  test("global temp view refers local temp view") {
+    withTempView("v1") {
+      withGlobalTempView("v2") {
+        val globalTempDB = spark.sharedState.globalTempViewManager.database
+        sql("CREATE TEMPORARY VIEW v1 AS SELECT 1")
+        sql(s"CREATE GLOBAL TEMPORARY VIEW v2 AS SELECT * FROM v1")
+        checkAnswer(sql(s"SELECT * FROM ${globalTempDB}.v2"), Seq(Row(1)))
+      }
+    }
+  }
+
+  test("SPARK-33141: view should be parsed and analyzed with configs set when creating") {
+    withTable("t") {
+      withView("v1", "v2", "v3", "v4", "v5") {
+        Seq(2, 3, 1).toDF("c1").write.format("parquet").saveAsTable("t")
+        sql("CREATE VIEW v1 (c1) AS SELECT C1 FROM t")
+        sql("CREATE VIEW v2 (c1) AS SELECT c1 FROM t ORDER BY 1 ASC, c1 DESC")
+        sql("CREATE VIEW v3 (c1, count) AS SELECT c1, count(c1) FROM t GROUP BY 1")
+        sql("CREATE VIEW v4 (a, count) AS SELECT c1 as a, count(c1) FROM t GROUP BY a")
+        sql("CREATE VIEW v5 (c1) AS SELECT 1/0")
+
+        withSQLConf(CASE_SENSITIVE.key -> "true") {
+          checkAnswer(sql("SELECT * FROM v1"), Seq(Row(2), Row(3), Row(1)))
+        }
+        withSQLConf(ORDER_BY_ORDINAL.key -> "false") {
+          checkAnswer(sql("SELECT * FROM v2"), Seq(Row(1), Row(2), Row(3)))
+        }
+        withSQLConf(GROUP_BY_ORDINAL.key -> "false") {
+          checkAnswer(sql("SELECT * FROM v3"),
+            Seq(Row(1, 1), Row(2, 1), Row(3, 1)))
+        }
+        withSQLConf(GROUP_BY_ALIASES.key -> "false") {
+          checkAnswer(sql("SELECT * FROM v4"),
+            Seq(Row(1, 1), Row(2, 1), Row(3, 1)))
+        }
+        withSQLConf(ANSI_ENABLED.key -> "true") {
+          checkAnswer(sql("SELECT * FROM v5"), Seq(Row(null)))
+        }
+
+        withSQLConf(USE_CURRENT_SQL_CONFIGS_FOR_VIEW.key -> "true") {
+          withSQLConf(CASE_SENSITIVE.key -> "true") {
+            val e = intercept[AnalysisException] {
+              sql("SELECT * FROM v1")
+            }.getMessage
+            assert(e.contains("cannot resolve 'C1' given input columns: " +
+              "[spark_catalog.default.t.c1]"))
+          }
+          withSQLConf(ORDER_BY_ORDINAL.key -> "false") {
+            checkAnswer(sql("SELECT * FROM v2"), Seq(Row(3), Row(2), Row(1)))
+          }
+          withSQLConf(GROUP_BY_ORDINAL.key -> "false") {
+            val e = intercept[AnalysisException] {
+              sql("SELECT * FROM v3")
+            }.getMessage
+            assert(e.contains(
+              "expression 'spark_catalog.default.t.c1' is neither present " +
+              "in the group by, nor is it an aggregate function. Add to group by or wrap in " +
+              "first() (or first_value) if you don't care which value you get."))
+          }
+          withSQLConf(GROUP_BY_ALIASES.key -> "false") {
+            val e = intercept[AnalysisException] {
+              sql("SELECT * FROM v4")
+            }.getMessage
+            assert(e.contains("cannot resolve 'a' given input columns: " +
+              "[spark_catalog.default.t.c1]"))
+          }
+          withSQLConf(ANSI_ENABLED.key -> "true") {
+            val e = intercept[ArithmeticException] {
+              sql("SELECT * FROM v5").collect()
+            }.getMessage
+            assert(e.contains("divide by zero"))
+          }
+        }
+
+        withSQLConf(ANSI_ENABLED.key -> "true") {
+          sql("ALTER VIEW v1 AS SELECT 1/0")
+        }
+        val e = intercept[ArithmeticException] {
+          sql("SELECT * FROM v1").collect()
+        }.getMessage
+        assert(e.contains("divide by zero"))
+      }
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewTestSuite.scala
new file mode 100644
index 0000000000000..5fb2e2cd3d433
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewTestSuite.scala
@@ -0,0 +1,406 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
+import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.plans.logical.Repartition
+import org.apache.spark.sql.internal.SQLConf._
+import org.apache.spark.sql.test.{SharedSparkSession, SQLTestUtils}
+
+/**
+ * A base suite contains a set of view related test cases for different kind of views
+ * Currently, the test cases in this suite should have same behavior across all kind of views
+ * TODO: Combine this with [[SQLViewSuite]]
+ */
+abstract class SQLViewTestSuite extends QueryTest with SQLTestUtils {
+  import testImplicits._
+
+  protected def viewTypeString: String
+  protected def formattedViewName(viewName: String): String
+  protected def tableIdentifier(viewName: String): TableIdentifier
+
+  def createView(
+      viewName: String,
+      sqlText: String,
+      columnNames: Seq[String] = Seq.empty,
+      replace: Boolean = false): String = {
+    val replaceString = if (replace) "OR REPLACE" else ""
+    val columnString = if (columnNames.nonEmpty) columnNames.mkString("(", ",", ")") else ""
+    sql(s"CREATE $replaceString $viewTypeString $viewName $columnString AS $sqlText")
+    formattedViewName(viewName)
+  }
+
+  def checkViewOutput(viewName: String, expectedAnswer: Seq[Row]): Unit = {
+    checkAnswer(sql(s"SELECT * FROM $viewName"), expectedAnswer)
+  }
+
+  test("change SQLConf should not change view behavior - caseSensitiveAnalysis") {
+    withTable("t") {
+      Seq(2, 3, 1).toDF("c1").write.format("parquet").saveAsTable("t")
+      val viewName = createView("v1", "SELECT c1 FROM t", Seq("C1"))
+      withView(viewName) {
+        Seq("true", "false").foreach { flag =>
+          withSQLConf(CASE_SENSITIVE.key -> flag) {
+            checkViewOutput(viewName, Seq(Row(2), Row(3), Row(1)))
+          }
+        }
+      }
+    }
+  }
+
+  test("change SQLConf should not change view behavior - orderByOrdinal") {
+    withTable("t") {
+      Seq(2, 3, 1).toDF("c1").write.format("parquet").saveAsTable("t")
+      val viewName = createView("v1", "SELECT c1 FROM t ORDER BY 1 ASC, c1 DESC", Seq("c1"))
+      withView(viewName) {
+        Seq("true", "false").foreach { flag =>
+          withSQLConf(ORDER_BY_ORDINAL.key -> flag) {
+            checkViewOutput(viewName, Seq(Row(1), Row(2), Row(3)))
+          }
+        }
+      }
+    }
+  }
+
+  test("change SQLConf should not change view behavior - groupByOrdinal") {
+    withTable("t") {
+      Seq(2, 3, 1).toDF("c1").write.format("parquet").saveAsTable("t")
+      val viewName = createView("v1", "SELECT c1, count(c1) FROM t GROUP BY 1", Seq("c1", "count"))
+      withView(viewName) {
+        Seq("true", "false").foreach { flag =>
+          withSQLConf(GROUP_BY_ORDINAL.key -> flag) {
+            checkViewOutput(viewName, Seq(Row(1, 1), Row(2, 1), Row(3, 1)))
+          }
+        }
+      }
+    }
+  }
+
+  test("change SQLConf should not change view behavior - groupByAliases") {
+    withTable("t") {
+      Seq(2, 3, 1).toDF("c1").write.format("parquet").saveAsTable("t")
+      val viewName = createView(
+        "v1", "SELECT c1 as a, count(c1) FROM t GROUP BY a", Seq("a", "count"))
+      withView(viewName) {
+        Seq("true", "false").foreach { flag =>
+          withSQLConf(GROUP_BY_ALIASES.key -> flag) {
+            checkViewOutput(viewName, Seq(Row(1, 1), Row(2, 1), Row(3, 1)))
+          }
+        }
+      }
+    }
+  }
+
+  test("change SQLConf should not change view behavior - ansiEnabled") {
+    withTable("t") {
+      Seq(2, 3, 1).toDF("c1").write.format("parquet").saveAsTable("t")
+      val viewName = createView("v1", "SELECT 1/0", Seq("c1"))
+      withView(viewName) {
+        Seq("true", "false").foreach { flag =>
+          withSQLConf(ANSI_ENABLED.key -> flag) {
+            checkViewOutput(viewName, Seq(Row(null)))
+          }
+        }
+      }
+    }
+  }
+
+  test("change current database should not change view behavior") {
+    withTable("t") {
+      Seq(2, 3, 1).toDF("c1").write.format("parquet").saveAsTable("t")
+      val viewName = createView("v1", "SELECT * FROM t")
+      withView(viewName) {
+        withTempDatabase { db =>
+          sql(s"USE $db")
+          Seq(4, 5, 6).toDF("c1").write.format("parquet").saveAsTable("t")
+          checkViewOutput(viewName, Seq(Row(2), Row(3), Row(1)))
+        }
+      }
+    }
+  }
+
+  test("view should read the new data if table is updated") {
+    withTable("t") {
+      Seq(2, 3, 1).toDF("c1").write.format("parquet").saveAsTable("t")
+      val viewName = createView("v1", "SELECT c1 FROM t", Seq("c1"))
+      withView(viewName) {
+        Seq(9, 7, 8).toDF("c1").write.mode("overwrite").format("parquet").saveAsTable("t")
+        checkViewOutput(viewName, Seq(Row(9), Row(7), Row(8)))
+      }
+    }
+  }
+
+  test("add column for table should not affect view output") {
+    withTable("t") {
+      Seq(2, 3, 1).toDF("c1").write.format("parquet").saveAsTable("t")
+      val viewName = createView("v1", "SELECT * FROM t")
+      withView(viewName) {
+        sql("ALTER TABLE t ADD COLUMN (c2 INT)")
+        checkViewOutput(viewName, Seq(Row(2), Row(3), Row(1)))
+      }
+    }
+  }
+
+  test("check cyclic view reference on CREATE OR REPLACE VIEW") {
+    withTable("t") {
+      Seq(2, 3, 1).toDF("c1").write.format("parquet").saveAsTable("t")
+      val viewName1 = createView("v1", "SELECT * FROM t")
+      val viewName2 = createView("v2", s"SELECT * FROM $viewName1")
+      withView(viewName2, viewName1) {
+        val e = intercept[AnalysisException] {
+          createView("v1", s"SELECT * FROM $viewName2", replace = true)
+        }.getMessage
+        assert(e.contains("Recursive view"))
+      }
+    }
+  }
+
+  test("check cyclic view reference on ALTER VIEW") {
+    withTable("t") {
+      Seq(2, 3, 1).toDF("c1").write.format("parquet").saveAsTable("t")
+      val viewName1 = createView("v1", "SELECT * FROM t")
+      val viewName2 = createView("v2", s"SELECT * FROM $viewName1")
+      withView(viewName2, viewName1) {
+        val e = intercept[AnalysisException] {
+          sql(s"ALTER VIEW $viewName1 AS SELECT * FROM $viewName2")
+        }.getMessage
+        assert(e.contains("Recursive view"))
+      }
+    }
+  }
+
+  test("restrict the nested level of a view") {
+    val viewNames = scala.collection.mutable.ArrayBuffer.empty[String]
+    val view0 = createView("view0", "SELECT 1")
+    viewNames += view0
+    for (i <- 1 to 10) {
+      viewNames += createView(s"view$i", s"SELECT * FROM ${viewNames.last}")
+    }
+    withView(viewNames.reverse.toSeq: _*) {
+      withSQLConf(MAX_NESTED_VIEW_DEPTH.key -> "10") {
+        val e = intercept[AnalysisException] {
+          sql(s"SELECT * FROM ${viewNames.last}")
+        }.getMessage
+        assert(e.contains("exceeds the maximum view resolution depth (10)"))
+        assert(e.contains(s"Increase the value of ${MAX_NESTED_VIEW_DEPTH.key}"))
+      }
+    }
+  }
+
+  test("view should use captured catalog and namespace to resolve relation") {
+    withTempDatabase { dbName =>
+      withTable("default.t", s"$dbName.t") {
+        withTempView("t") {
+          // create a table in default database
+          sql("USE DEFAULT")
+          Seq(2, 3, 1).toDF("c1").write.format("parquet").saveAsTable("t")
+          // create a view refer the created table in default database
+          val viewName = createView("v1", "SELECT * FROM t")
+          // using another database to create a table with same name
+          sql(s"USE $dbName")
+          Seq(4, 5, 6).toDF("c1").write.format("parquet").saveAsTable("t")
+          // create a temporary view with the same name
+          sql("CREATE TEMPORARY VIEW t AS SELECT 1")
+          withView(viewName) {
+            // view v1 should still refer the table defined in `default` database
+            checkViewOutput(viewName, Seq(Row(2), Row(3), Row(1)))
+          }
+        }
+      }
+    }
+  }
+
+  test("SPARK-33692: view should use captured catalog and namespace to lookup function") {
+    val avgFuncClass = "test.org.apache.spark.sql.MyDoubleAvg"
+    val sumFuncClass = "test.org.apache.spark.sql.MyDoubleSum"
+    val functionName = "test_udf"
+    withTempDatabase { dbName =>
+      withUserDefinedFunction(
+        s"default.$functionName" -> false,
+        s"$dbName.$functionName" -> false,
+        functionName -> true) {
+        // create a function in default database
+        sql("USE DEFAULT")
+        sql(s"CREATE FUNCTION $functionName AS '$avgFuncClass'")
+        // create a view using a function in 'default' database
+        val viewName = createView("v1", s"SELECT $functionName(col1) FROM VALUES (1), (2), (3)")
+        // create function in another database with the same function name
+        sql(s"USE $dbName")
+        sql(s"CREATE FUNCTION $functionName AS '$sumFuncClass'")
+        // create temporary function with the same function name
+        sql(s"CREATE TEMPORARY FUNCTION $functionName AS '$sumFuncClass'")
+        withView(viewName) {
+          // view v1 should still using function defined in `default` database
+          checkViewOutput(viewName, Seq(Row(102.0)))
+        }
+      }
+    }
+  }
+
+  test("SPARK-34260: replace existing view using CREATE OR REPLACE") {
+    val viewName = createView("testView", "SELECT * FROM (SELECT 1)")
+    withView(viewName) {
+      checkViewOutput(viewName, Seq(Row(1)))
+      createView("testView", "SELECT * FROM (SELECT 2)", replace = true)
+      checkViewOutput(viewName, Seq(Row(2)))
+    }
+  }
+
+  test("SPARK-34490 - query should fail if the view refers a dropped table") {
+    withTable("t") {
+      Seq(2, 3, 1).toDF("c1").write.format("parquet").saveAsTable("t")
+      val viewName = createView("testView", "SELECT * FROM t")
+      withView(viewName) {
+        // Always create a temp view in this case, not use `createView` on purpose
+        sql("CREATE TEMP VIEW t AS SELECT 1 AS c1")
+        withTempView("t") {
+          checkViewOutput(viewName, Seq(Row(2), Row(3), Row(1)))
+          // Manually drop table `t` to see if the query will fail
+          sql("DROP TABLE IF EXISTS default.t")
+          val e = intercept[AnalysisException] {
+            sql(s"SELECT * FROM $viewName").collect()
+          }.getMessage
+          assert(e.contains("Table or view not found: t"))
+        }
+      }
+    }
+  }
+
+  test("SPARK-34613: Fix view does not capture disable hint config") {
+    withSQLConf(DISABLE_HINTS.key -> "true") {
+      val viewName = createView("v1", "SELECT /*+ repartition(1) */ 1")
+      withView(viewName) {
+        assert(
+          sql(s"SELECT * FROM $viewName").queryExecution.analyzed.collect {
+            case e: Repartition => e
+          }.isEmpty
+        )
+        checkViewOutput(viewName, Seq(Row(1)))
+      }
+    }
+  }
+
+  test("SPARK-34152: view's identifier should be correctly stored") {
+    Seq(true, false).foreach { storeAnalyzed =>
+      withSQLConf(STORE_ANALYZED_PLAN_FOR_VIEW.key -> storeAnalyzed.toString) {
+        val viewName = createView("v", "SELECT 1")
+        withView(viewName) {
+          val tblIdent = tableIdentifier("v")
+          val metadata = spark.sessionState.catalog.getTempViewOrPermanentTableMetadata(tblIdent)
+          assert(metadata.identifier == tblIdent)
+        }
+      }
+    }
+  }
+
+  test("SPARK-34504: drop an invalid view") {
+    withTable("t") {
+      sql("CREATE TABLE t(s STRUCT<i: INT, j: INT>) USING json")
+      val viewName = createView("v", "SELECT s.i FROM t")
+      withView(viewName) {
+        assert(spark.table(viewName).collect().isEmpty)
+
+        // re-create the table without nested field `i` which is referred by the view.
+        sql("DROP TABLE t")
+        sql("CREATE TABLE t(s STRUCT<j: INT>) USING json")
+        val e = intercept[AnalysisException](spark.table(viewName))
+        assert(e.message.contains("No such struct field i in j"))
+
+        // drop invalid view should be fine
+        sql(s"DROP VIEW $viewName")
+      }
+    }
+  }
+
+  test("SPARK-34719: view query with duplicated output column names") {
+    Seq(true, false).foreach { caseSensitive =>
+      withSQLConf(CASE_SENSITIVE.key -> caseSensitive.toString) {
+        withView("v1", "v2") {
+          sql("CREATE VIEW v1 AS SELECT 1 a, 2 b")
+          sql("CREATE VIEW v2 AS SELECT 1 col")
+
+          val viewName = createView(
+            viewName = "testView",
+            sqlText = "SELECT *, 1 col, 2 col FROM v1",
+            columnNames = Seq("c1", "c2", "c3", "c4"))
+          withView(viewName) {
+            checkViewOutput(viewName, Seq(Row(1, 2, 1, 2)))
+
+            // One more duplicated column `COL` if caseSensitive=false.
+            sql("CREATE OR REPLACE VIEW v1 AS SELECT 1 a, 2 b, 3 COL")
+            if (caseSensitive) {
+              checkViewOutput(viewName, Seq(Row(1, 2, 1, 2)))
+            } else {
+              val e = intercept[AnalysisException](spark.table(viewName).collect())
+              assert(e.message.contains("incompatible schema change"))
+            }
+          }
+
+          // v1 has 3 columns [a, b, COL], v2 has one column [col], so `testView2` has duplicated
+          // output column names if caseSensitive=false.
+          val viewName2 = createView(
+            viewName = "testView2",
+            sqlText = "SELECT * FROM v1, v2",
+            columnNames = Seq("c1", "c2", "c3", "c4"))
+          withView(viewName2) {
+            checkViewOutput(viewName2, Seq(Row(1, 2, 3, 1)))
+
+            // One less duplicated column if caseSensitive=false.
+            sql("CREATE OR REPLACE VIEW v1 AS SELECT 1 a, 2 b")
+            if (caseSensitive) {
+              val e = intercept[AnalysisException](spark.table(viewName2).collect())
+              assert(e.message.contains("cannot resolve 'COL'"))
+            } else {
+              val e = intercept[AnalysisException](spark.table(viewName2).collect())
+              assert(e.message.contains("incompatible schema change"))
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+class LocalTempViewTestSuite extends SQLViewTestSuite with SharedSparkSession {
+  override protected def viewTypeString: String = "TEMPORARY VIEW"
+  override protected def formattedViewName(viewName: String): String = viewName
+  override protected def tableIdentifier(viewName: String): TableIdentifier = {
+    TableIdentifier(viewName)
+  }
+}
+
+class GlobalTempViewTestSuite extends SQLViewTestSuite with SharedSparkSession {
+  private def db: String = spark.sharedState.globalTempViewManager.database
+  override protected def viewTypeString: String = "GLOBAL TEMPORARY VIEW"
+  override protected def formattedViewName(viewName: String): String = {
+    s"$db.$viewName"
+  }
+  override protected def tableIdentifier(viewName: String): TableIdentifier = {
+    TableIdentifier(viewName, Some(db))
+  }
+}
+
+class PersistedViewTestSuite extends SQLViewTestSuite with SharedSparkSession {
+  private def db: String = "default"
+  override protected def viewTypeString: String = "VIEW"
+  override protected def formattedViewName(viewName: String): String = s"$db.$viewName"
+  override protected def tableIdentifier(viewName: String): TableIdentifier = {
+    TableIdentifier(viewName, Some(db))
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala
index 67ec1028f1998..eec396b2e3998 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala
@@ -372,7 +372,7 @@ class SQLWindowFunctionSuite extends QueryTest with SharedSparkSession {
     spark.catalog.dropTempView("nums")
   }
 
-  test("window function: mutiple window expressions specified by range in a single expression") {
+  test("window function: multiple window expressions specified by range in a single expression") {
     val nums = sparkContext.parallelize(1 to 10).map(x => (x, x % 2)).toDF("x", "y")
     nums.createOrReplaceTempView("nums")
     withTempView("nums") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SameResultSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SameResultSuite.scala
index ddaa2687eaf1a..d2406aa59e21e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SameResultSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SameResultSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.execution
 import org.apache.spark.sql.{DataFrame, QueryTest}
 import org.apache.spark.sql.catalyst.expressions.AttributeReference
 import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, Project}
-import org.apache.spark.sql.execution.datasources.v2.{BatchScanExec, FileScan}
+import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
@@ -120,8 +120,8 @@ class SameResultSuite extends QueryTest with SharedSparkSession {
     val df2 = spark.range(10).agg(sum($"id"))
     assert(df1.queryExecution.executedPlan.sameResult(df2.queryExecution.executedPlan))
 
-    val df3 = spark.range(10).agg(sumDistinct($"id"))
-    val df4 = spark.range(10).agg(sumDistinct($"id"))
+    val df3 = spark.range(10).agg(sum_distinct($"id"))
+    val df4 = spark.range(10).agg(sum_distinct($"id"))
     assert(df3.queryExecution.executedPlan.sameResult(df4.queryExecution.executedPlan))
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsUtilSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsUtilSuite.scala
index f5c3b7816f5ea..94e22a414a628 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsUtilSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsUtilSuite.scala
@@ -198,9 +198,11 @@ class ShufflePartitionsUtilSuite extends SparkFunSuite {
       // the size of data is 0.
       val bytesByPartitionId1 = Array[Long](0, 0, 0, 0, 0)
       val bytesByPartitionId2 = Array[Long](0, 0, 0, 0, 0)
+      // Create at least one partition spec
+      val expectedPartitionSpecs = Seq(CoalescedPartitionSpec(0, 5))
       checkEstimation(
         Array(bytesByPartitionId1, bytesByPartitionId2),
-        Seq.empty, targetSize, minNumPartitions)
+        expectedPartitionSpecs, targetSize, minNumPartitions)
     }
 
 
@@ -248,16 +250,19 @@ class ShufflePartitionsUtilSuite extends SparkFunSuite {
     val minNumPartitions = 2
 
     {
-      // 1 shuffle: All bytes per partition are 0, no partition spec created.
+      // 1 shuffle: All bytes per partition are 0, 1 empty partition spec created.
       val bytesByPartitionId = Array[Long](0, 0, 0, 0, 0)
-      checkEstimation(Array(bytesByPartitionId), Seq.empty, targetSize)
+      val expectedPartitionSpecs = Seq(CoalescedPartitionSpec(0, 5))
+      checkEstimation(Array(bytesByPartitionId), expectedPartitionSpecs, targetSize)
     }
 
     {
-      // 2 shuffles: All bytes per partition are 0, no partition spec created.
+      // 2 shuffles: All bytes per partition are 0, 1 empty partition spec created.
       val bytesByPartitionId1 = Array[Long](0, 0, 0, 0, 0)
       val bytesByPartitionId2 = Array[Long](0, 0, 0, 0, 0)
-      checkEstimation(Array(bytesByPartitionId1, bytesByPartitionId2), Seq.empty, targetSize)
+      val expectedPartitionSpecs = Seq(CoalescedPartitionSpec(0, 5))
+      checkEstimation(
+        Array(bytesByPartitionId1, bytesByPartitionId2), expectedPartitionSpecs, targetSize)
     }
 
     {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala
index 7654a9d982059..6a4f3f62641f8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala
@@ -97,6 +97,19 @@ class SortSuite extends SparkPlanTest with SharedSparkSession {
     }
   }
 
+  test("SPARK-33260: sort order is a Stream") {
+    val input = Seq(
+      ("Hello", 4, 2.0),
+      ("Hello", 1, 1.0),
+      ("World", 8, 3.0)
+    )
+    checkAnswer(
+      input.toDF("a", "b", "c"),
+      (child: SparkPlan) => SortExec(Stream('a.asc, 'b.asc, 'c.asc), global = true, child = child),
+      input.sortBy(t => (t._1, t._2, t._3)).map(Row.fromTuple),
+      sortAnswers = false)
+  }
+
   // Test sorting on different data types
   for (
     dataType <- DataTypeTestUtils.atomicTypes ++ Set(NullType);
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala
index 56fff1107ae39..7687bfdd14fdb 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.execution
 
 import org.apache.spark.SparkEnv
 import org.apache.spark.sql.QueryTest
+import org.apache.spark.sql.catalyst.plans.logical.Deduplicate
 import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
@@ -88,4 +89,24 @@ class SparkPlanSuite extends QueryTest with SharedSparkSession {
   test("SPARK-30780 empty LocalTableScan should use RDD without partitions") {
     assert(LocalTableScanExec(Nil, Nil).execute().getNumPartitions == 0)
   }
+
+  test("SPARK-33617: change default parallelism of LocalTableScan") {
+    Seq(1, 4).foreach { minPartitionNum =>
+      withSQLConf(SQLConf.LEAF_NODE_DEFAULT_PARALLELISM.key -> minPartitionNum.toString) {
+        val df = spark.sql("SELECT * FROM VALUES (1), (2), (3), (4), (5), (6), (7), (8)")
+        assert(df.rdd.partitions.length === minPartitionNum)
+      }
+    }
+  }
+
+  test("SPARK-34420: Throw exception if non-streaming Deduplicate is not replaced by aggregate") {
+    val df = spark.range(10)
+    val planner = spark.sessionState.planner
+    val deduplicate = Deduplicate(df.queryExecution.analyzed.output, df.queryExecution.analyzed)
+    val err = intercept[IllegalStateException] {
+      planner.plan(deduplicate)
+    }
+    assert(err.getMessage.contains("Deduplicate operator for non streaming data source " +
+      "should have been replaced by aggregate in the optimizer"))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala
index 7ddf9d87a6aca..f1fcf3bc5125e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala
@@ -17,7 +17,6 @@
 
 package org.apache.spark.sql.execution
 
-import scala.language.implicitConversions
 import scala.util.control.NonFatal
 
 import org.apache.spark.SparkFunSuite
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlannerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlannerSuite.scala
index 9107f8afa83d7..b4cb7e3fce3cf 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlannerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlannerSuite.scala
@@ -37,9 +37,9 @@ class SparkPlannerSuite extends SharedSparkSession {
         case ReturnAnswer(child) =>
           planned += 1
           planLater(child) :: planLater(NeverPlanned) :: Nil
-        case Union(children) =>
+        case u: Union =>
           planned += 1
-          UnionExec(children.map(planLater)) :: planLater(NeverPlanned) :: Nil
+          UnionExec(u.children.map(planLater)) :: planLater(NeverPlanned) :: Nil
         case LocalRelation(output, data, _) =>
           planned += 1
           LocalTableScanExec(output, data) :: planLater(NeverPlanned) :: Nil
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkScriptTransformationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkScriptTransformationSuite.scala
new file mode 100644
index 0000000000000..5638743b7633d
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkScriptTransformationSuite.scala
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import org.apache.spark.TestUtils
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.parser.ParseException
+import org.apache.spark.sql.test.SharedSparkSession
+
+class SparkScriptTransformationSuite extends BaseScriptTransformationSuite with SharedSparkSession {
+  import testImplicits._
+
+  override protected def defaultSerDe(): String = "row-format-delimited"
+
+  override def createScriptTransformationExec(
+      script: String,
+      output: Seq[Attribute],
+      child: SparkPlan,
+      ioschema: ScriptTransformationIOSchema): BaseScriptTransformationExec = {
+    SparkScriptTransformationExec(
+      script = script,
+      output = output,
+      child = child,
+      ioschema = ioschema
+    )
+  }
+
+  test("SPARK-32106: TRANSFORM with serde without hive should throw exception") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+    withTempView("v") {
+      val df = Seq("a", "b", "c").map(Tuple1.apply).toDF("a")
+      df.createTempView("v")
+
+      val e = intercept[ParseException] {
+        sql(
+          """
+            |SELECT TRANSFORM (a)
+            |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+            |USING 'cat' AS (a)
+            |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+            |FROM v
+          """.stripMargin)
+      }.getMessage
+      assert(e.contains("TRANSFORM with serde is only supported in hive mode"))
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala
index 06574a9f8fd2c..e7d630d1abfbd 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala
@@ -17,16 +17,18 @@
 
 package org.apache.spark.sql.execution
 
-import org.apache.spark.sql.SaveMode
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, UnresolvedAlias, UnresolvedAttribute, UnresolvedRelation, UnresolvedStar}
-import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStorageFormat, CatalogTable, CatalogTableType}
-import org.apache.spark.sql.catalyst.expressions.{Ascending, Concat, SortOrder}
-import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project, RepartitionByExpression, Sort}
+import scala.collection.JavaConverters._
+
+import org.apache.spark.internal.config.ConfigEntry
+import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
+import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, UnresolvedAlias, UnresolvedAttribute, UnresolvedFunction, UnresolvedGenerator, UnresolvedHaving, UnresolvedRelation, UnresolvedStar}
+import org.apache.spark.sql.catalyst.expressions.{Ascending, AttributeReference, Concat, GreaterThan, Literal, NullsFirst, SortOrder, UnresolvedWindowExpression, UnspecifiedFrame, WindowSpecDefinition, WindowSpecReference}
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.connector.catalog.TableCatalog
 import org.apache.spark.sql.execution.command._
-import org.apache.spark.sql.execution.datasources.{CreateTable, RefreshResource}
-import org.apache.spark.sql.internal.{HiveSerDe, SQLConf}
-import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructType}
+import org.apache.spark.sql.execution.datasources.{CreateTempViewUsing, RefreshResource}
+import org.apache.spark.sql.internal.StaticSQLConf
+import org.apache.spark.sql.types.StringType
 
 /**
  * Parser test cases for rules defined in [[SparkSqlParser]].
@@ -35,32 +37,135 @@ import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructType
  * defined in the Catalyst module.
  */
 class SparkSqlParserSuite extends AnalysisTest {
+  import org.apache.spark.sql.catalyst.dsl.expressions._
 
-  val newConf = new SQLConf
-  private lazy val parser = new SparkSqlParser(newConf)
-
-  /**
-   * Normalizes plans:
-   * - CreateTable the createTime in tableDesc will replaced by -1L.
-   */
-  override def normalizePlan(plan: LogicalPlan): LogicalPlan = {
-    plan match {
-      case CreateTable(tableDesc, mode, query) =>
-        val newTableDesc = tableDesc.copy(createTime = -1L)
-        CreateTable(newTableDesc, mode, query)
-      case _ => plan // Don't transform
-    }
-  }
+  private lazy val parser = new SparkSqlParser()
 
   private def assertEqual(sqlCommand: String, plan: LogicalPlan): Unit = {
-    val normalized1 = normalizePlan(parser.parsePlan(sqlCommand))
-    val normalized2 = normalizePlan(plan)
-    comparePlans(normalized1, normalized2)
+    comparePlans(parser.parsePlan(sqlCommand), plan)
   }
 
   private def intercept(sqlCommand: String, messages: String*): Unit =
     interceptParseException(parser.parsePlan)(sqlCommand, messages: _*)
 
+  test("Checks if SET/RESET can parse all the configurations") {
+    // Force to build static SQL configurations
+    StaticSQLConf
+    ConfigEntry.knownConfigs.values.asScala.foreach { config =>
+      assertEqual(s"SET ${config.key}", SetCommand(Some(config.key -> None)))
+      assertEqual(s"SET `${config.key}`", SetCommand(Some(config.key -> None)))
+
+      val defaultValueStr = config.defaultValueString
+      if (config.defaultValue.isDefined && defaultValueStr != null) {
+        assertEqual(s"SET ${config.key}=`$defaultValueStr`",
+          SetCommand(Some(config.key -> Some(defaultValueStr))))
+        assertEqual(s"SET `${config.key}`=`$defaultValueStr`",
+          SetCommand(Some(config.key -> Some(defaultValueStr))))
+
+        if (!defaultValueStr.contains(";")) {
+          assertEqual(s"SET ${config.key}=$defaultValueStr",
+            SetCommand(Some(config.key -> Some(defaultValueStr))))
+          assertEqual(s"SET `${config.key}`=$defaultValueStr",
+            SetCommand(Some(config.key -> Some(defaultValueStr))))
+        }
+      }
+      assertEqual(s"RESET ${config.key}", ResetCommand(Some(config.key)))
+    }
+  }
+
+  test("Report Error for invalid usage of SET command") {
+    assertEqual("SET", SetCommand(None))
+    assertEqual("SET -v", SetCommand(Some("-v", None)))
+    assertEqual("SET spark.sql.key", SetCommand(Some("spark.sql.key" -> None)))
+    assertEqual("SET  spark.sql.key   ", SetCommand(Some("spark.sql.key" -> None)))
+    assertEqual("SET spark:sql:key=false", SetCommand(Some("spark:sql:key" -> Some("false"))))
+    assertEqual("SET spark:sql:key=", SetCommand(Some("spark:sql:key" -> Some(""))))
+    assertEqual("SET spark:sql:key=  ", SetCommand(Some("spark:sql:key" -> Some(""))))
+    assertEqual("SET spark:sql:key=-1 ", SetCommand(Some("spark:sql:key" -> Some("-1"))))
+    assertEqual("SET spark:sql:key = -1", SetCommand(Some("spark:sql:key" -> Some("-1"))))
+    assertEqual("SET 1.2.key=value", SetCommand(Some("1.2.key" -> Some("value"))))
+    assertEqual("SET spark.sql.3=4", SetCommand(Some("spark.sql.3" -> Some("4"))))
+    assertEqual("SET 1:2:key=value", SetCommand(Some("1:2:key" -> Some("value"))))
+    assertEqual("SET spark:sql:3=4", SetCommand(Some("spark:sql:3" -> Some("4"))))
+    assertEqual("SET 5=6", SetCommand(Some("5" -> Some("6"))))
+    assertEqual("SET spark:sql:key = va l u  e ",
+      SetCommand(Some("spark:sql:key" -> Some("va l u  e"))))
+    assertEqual("SET `spark.sql.    key`=value",
+      SetCommand(Some("spark.sql.    key" -> Some("value"))))
+    assertEqual("SET `spark.sql.    key`= v  a lu e ",
+      SetCommand(Some("spark.sql.    key" -> Some("v  a lu e"))))
+    assertEqual("SET `spark.sql.    key`=  -1",
+      SetCommand(Some("spark.sql.    key" -> Some("-1"))))
+    assertEqual("SET key=", SetCommand(Some("key" -> Some(""))))
+
+    val expectedErrMsg = "Expected format is 'SET', 'SET key', or " +
+      "'SET key=value'. If you want to include special characters in key, or include semicolon " +
+      "in value, please use quotes, e.g., SET `ke y`=`v;alue`."
+    intercept("SET spark.sql.key value", expectedErrMsg)
+    intercept("SET spark.sql.key   'value'", expectedErrMsg)
+    intercept("SET    spark.sql.key \"value\" ", expectedErrMsg)
+    intercept("SET spark.sql.key value1 value2", expectedErrMsg)
+    intercept("SET spark.   sql.key=value", expectedErrMsg)
+    intercept("SET spark   :sql:key=value", expectedErrMsg)
+    intercept("SET spark .  sql.key=value", expectedErrMsg)
+    intercept("SET spark.sql.   key=value", expectedErrMsg)
+    intercept("SET spark.sql   :key=value", expectedErrMsg)
+    intercept("SET spark.sql .  key=value", expectedErrMsg)
+    intercept("SET =", expectedErrMsg)
+    intercept("SET =value", expectedErrMsg)
+  }
+
+  test("Report Error for invalid usage of RESET command") {
+    assertEqual("RESET", ResetCommand(None))
+    assertEqual("RESET spark.sql.key", ResetCommand(Some("spark.sql.key")))
+    assertEqual("RESET  spark.sql.key  ", ResetCommand(Some("spark.sql.key")))
+    assertEqual("RESET 1.2.key ", ResetCommand(Some("1.2.key")))
+    assertEqual("RESET spark.sql.3", ResetCommand(Some("spark.sql.3")))
+    assertEqual("RESET 1:2:key ", ResetCommand(Some("1:2:key")))
+    assertEqual("RESET spark:sql:3", ResetCommand(Some("spark:sql:3")))
+    assertEqual("RESET `spark.sql.    key`", ResetCommand(Some("spark.sql.    key")))
+
+    val expectedErrMsg = "Expected format is 'RESET' or 'RESET key'. " +
+      "If you want to include special characters in key, " +
+      "please use quotes, e.g., RESET `ke y`."
+    intercept("RESET spark.sql.key1 key2", expectedErrMsg)
+    intercept("RESET spark.  sql.key1 key2", expectedErrMsg)
+    intercept("RESET spark.sql.key1 key2 key3", expectedErrMsg)
+    intercept("RESET spark:   sql:key", expectedErrMsg)
+    intercept("RESET spark   .sql.key", expectedErrMsg)
+    intercept("RESET spark :  sql:key", expectedErrMsg)
+    intercept("RESET spark.sql:   key", expectedErrMsg)
+    intercept("RESET spark.sql   .key", expectedErrMsg)
+    intercept("RESET spark.sql :  key", expectedErrMsg)
+  }
+
+  test("SPARK-33419: Semicolon handling in SET command") {
+    assertEqual("SET a=1;", SetCommand(Some("a" -> Some("1"))))
+    assertEqual("SET a=1;;", SetCommand(Some("a" -> Some("1"))))
+
+    assertEqual("SET a=`1`;", SetCommand(Some("a" -> Some("1"))))
+    assertEqual("SET a=`1;`", SetCommand(Some("a" -> Some("1;"))))
+    assertEqual("SET a=`1;`;", SetCommand(Some("a" -> Some("1;"))))
+
+    assertEqual("SET `a`=1;;", SetCommand(Some("a" -> Some("1"))))
+    assertEqual("SET `a`=`1;`", SetCommand(Some("a" -> Some("1;"))))
+    assertEqual("SET `a`=`1;`;", SetCommand(Some("a" -> Some("1;"))))
+
+    val expectedErrMsg = "Expected format is 'SET', 'SET key', or " +
+      "'SET key=value'. If you want to include special characters in key, or include semicolon " +
+      "in value, please use quotes, e.g., SET `ke y`=`v;alue`."
+
+    intercept("SET a=1; SELECT 1", expectedErrMsg)
+    intercept("SET a=1;2;;", expectedErrMsg)
+
+    intercept("SET a b=`1;;`",
+      "'a b' is an invalid property key, please use quotes, e.g. SET `a b`=`1;;`")
+
+    intercept("SET `a`=1;2;;",
+      "'1;2;;' is an invalid property value, please use quotes, e.g." +
+        " SET `a`=`1;2;;`")
+  }
+
   test("refresh resource") {
     assertEqual("REFRESH prefix_path", RefreshResource("prefix_path"))
     assertEqual("REFRESH /", RefreshResource("/"))
@@ -80,108 +185,13 @@ class SparkSqlParserSuite extends AnalysisTest {
     intercept("REFRESH", "Resource paths cannot be empty in REFRESH statements")
   }
 
-  private def createTableUsing(
-      table: String,
-      database: Option[String] = None,
-      tableType: CatalogTableType = CatalogTableType.MANAGED,
-      storage: CatalogStorageFormat = CatalogStorageFormat.empty,
-      schema: StructType = new StructType,
-      provider: Option[String] = Some("parquet"),
-      partitionColumnNames: Seq[String] = Seq.empty,
-      bucketSpec: Option[BucketSpec] = None,
-      mode: SaveMode = SaveMode.ErrorIfExists,
-      query: Option[LogicalPlan] = None): CreateTable = {
-    CreateTable(
-      CatalogTable(
-        identifier = TableIdentifier(table, database),
-        tableType = tableType,
-        storage = storage,
-        schema = schema,
-        provider = provider,
-        partitionColumnNames = partitionColumnNames,
-        bucketSpec = bucketSpec
-      ), mode, query
-    )
-  }
-
-  private def createTable(
-      table: String,
-      database: Option[String] = None,
-      tableType: CatalogTableType = CatalogTableType.MANAGED,
-      storage: CatalogStorageFormat = CatalogStorageFormat.empty.copy(
-        inputFormat = HiveSerDe.sourceToSerDe("textfile").get.inputFormat,
-        outputFormat = HiveSerDe.sourceToSerDe("textfile").get.outputFormat,
-        serde = Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe")),
-      schema: StructType = new StructType,
-      provider: Option[String] = Some("hive"),
-      partitionColumnNames: Seq[String] = Seq.empty,
-      comment: Option[String] = None,
-      mode: SaveMode = SaveMode.ErrorIfExists,
-      query: Option[LogicalPlan] = None): CreateTable = {
-    CreateTable(
-      CatalogTable(
-        identifier = TableIdentifier(table, database),
-        tableType = tableType,
-        storage = storage,
-        schema = schema,
-        provider = provider,
-        partitionColumnNames = partitionColumnNames,
-        comment = comment
-      ), mode, query
-    )
-  }
-
-  test("create table - schema") {
-    assertEqual("CREATE TABLE my_tab(a INT COMMENT 'test', b STRING) STORED AS textfile",
-      createTable(
-        table = "my_tab",
-        schema = (new StructType)
-          .add("a", IntegerType, nullable = true, "test")
-          .add("b", StringType)
-      )
-    )
-    assertEqual("CREATE TABLE my_tab(a INT COMMENT 'test', b STRING) " +
-      "PARTITIONED BY (c INT, d STRING COMMENT 'test2')",
-      createTable(
-        table = "my_tab",
-        schema = (new StructType)
-          .add("a", IntegerType, nullable = true, "test")
-          .add("b", StringType)
-          .add("c", IntegerType)
-          .add("d", StringType, nullable = true, "test2"),
-        partitionColumnNames = Seq("c", "d")
-      )
-    )
-    assertEqual("CREATE TABLE my_tab(id BIGINT, nested STRUCT<col1: STRING,col2: INT>) " +
-      "STORED AS textfile",
-      createTable(
-        table = "my_tab",
-        schema = (new StructType)
-          .add("id", LongType)
-          .add("nested", (new StructType)
-            .add("col1", StringType)
-            .add("col2", IntegerType)
-          )
-      )
-    )
-    // Partitioned by a StructType should be accepted by `SparkSqlParser` but will fail an analyze
-    // rule in `AnalyzeCreateTable`.
-    assertEqual("CREATE TABLE my_tab(a INT COMMENT 'test', b STRING) " +
-      "PARTITIONED BY (nested STRUCT<col1: STRING,col2: INT>)",
-      createTable(
-        table = "my_tab",
-        schema = (new StructType)
-          .add("a", IntegerType, nullable = true, "test")
-          .add("b", StringType)
-          .add("nested", (new StructType)
-            .add("col1", StringType)
-            .add("col2", IntegerType)
-          ),
-        partitionColumnNames = Seq("nested")
-      )
-    )
-    intercept("CREATE TABLE my_tab(a: INT COMMENT 'test', b: STRING)",
-      "no viable alternative at input")
+  test("SPARK-33118 CREATE TEMPORARY TABLE with LOCATION") {
+    assertEqual("CREATE TEMPORARY TABLE t USING parquet OPTIONS (path '/data/tmp/testspark1')",
+      CreateTempViewUsing(TableIdentifier("t", None), None, false, false, "parquet",
+        Map("path" -> "/data/tmp/testspark1")))
+    assertEqual("CREATE TEMPORARY TABLE t USING parquet LOCATION '/data/tmp/testspark1'",
+      CreateTempViewUsing(TableIdentifier("t", None), None, false, false, "parquet",
+        Map("path" -> "/data/tmp/testspark1")))
   }
 
   test("describe query") {
@@ -199,20 +209,20 @@ class SparkSqlParserSuite extends AnalysisTest {
     assertEqual(s"$baseSql distribute by a, b",
       RepartitionByExpression(UnresolvedAttribute("a") :: UnresolvedAttribute("b") :: Nil,
         basePlan,
-        numPartitions = newConf.numShufflePartitions))
+        None))
     assertEqual(s"$baseSql distribute by a sort by b",
       Sort(SortOrder(UnresolvedAttribute("b"), Ascending) :: Nil,
         global = false,
         RepartitionByExpression(UnresolvedAttribute("a") :: Nil,
           basePlan,
-          numPartitions = newConf.numShufflePartitions)))
+          None)))
     assertEqual(s"$baseSql cluster by a, b",
       Sort(SortOrder(UnresolvedAttribute("a"), Ascending) ::
           SortOrder(UnresolvedAttribute("b"), Ascending) :: Nil,
         global = false,
         RepartitionByExpression(UnresolvedAttribute("a") :: UnresolvedAttribute("b") :: Nil,
           basePlan,
-          numPartitions = newConf.numShufflePartitions)))
+          None)))
   }
 
   test("pipeline concatenation") {
@@ -235,20 +245,226 @@ class SparkSqlParserSuite extends AnalysisTest {
   }
 
   test("manage resources") {
-    assertEqual("ADD FILE abc.txt", AddFileCommand("abc.txt"))
-    assertEqual("ADD FILE 'abc.txt'", AddFileCommand("abc.txt"))
-    assertEqual("ADD FILE \"/path/to/abc.txt\"", AddFileCommand("/path/to/abc.txt"))
+    assertEqual("ADD FILE abc.txt", AddFilesCommand(Seq("abc.txt")))
+    assertEqual("ADD FILE 'abc.txt'", AddFilesCommand(Seq("abc.txt")))
+    assertEqual("ADD FILE \"/path/to/abc.txt\"", AddFilesCommand("/path/to/abc.txt"::Nil))
     assertEqual("LIST FILE abc.txt", ListFilesCommand(Array("abc.txt")))
     assertEqual("LIST FILE '/path//abc.txt'", ListFilesCommand(Array("/path//abc.txt")))
     assertEqual("LIST FILE \"/path2/abc.txt\"", ListFilesCommand(Array("/path2/abc.txt")))
-    assertEqual("ADD JAR /path2/_2/abc.jar", AddJarCommand("/path2/_2/abc.jar"))
-    assertEqual("ADD JAR '/test/path_2/jar/abc.jar'", AddJarCommand("/test/path_2/jar/abc.jar"))
-    assertEqual("ADD JAR \"abc.jar\"", AddJarCommand("abc.jar"))
+    assertEqual("ADD JAR /path2/_2/abc.jar", AddJarsCommand(Seq("/path2/_2/abc.jar")))
+    assertEqual("ADD JAR '/test/path_2/jar/abc.jar'",
+      AddJarsCommand(Seq("/test/path_2/jar/abc.jar")))
+    assertEqual("ADD JAR \"abc.jar\"", AddJarsCommand(Seq("abc.jar")))
     assertEqual("LIST JAR /path-with-dash/abc.jar",
       ListJarsCommand(Array("/path-with-dash/abc.jar")))
     assertEqual("LIST JAR 'abc.jar'", ListJarsCommand(Array("abc.jar")))
     assertEqual("LIST JAR \"abc.jar\"", ListJarsCommand(Array("abc.jar")))
-    assertEqual("ADD FILE /path with space/abc.txt", AddFileCommand("/path with space/abc.txt"))
-    assertEqual("ADD JAR /path with space/abc.jar", AddJarCommand("/path with space/abc.jar"))
+    assertEqual("ADD FILE '/path with space/abc.txt'",
+      AddFilesCommand(Seq("/path with space/abc.txt")))
+    assertEqual("ADD JAR '/path with space/abc.jar'",
+      AddJarsCommand(Seq("/path with space/abc.jar")))
+  }
+
+  test("SPARK-32608: script transform with row format delimit") {
+    val rowFormat =
+      """
+        |  ROW FORMAT DELIMITED
+        |  FIELDS TERMINATED BY ','
+        |  COLLECTION ITEMS TERMINATED BY '#'
+        |  MAP KEYS TERMINATED BY '@'
+        |  LINES TERMINATED BY '\n'
+        |  NULL DEFINED AS 'null'
+      """.stripMargin
+
+    val ioSchema =
+      ScriptInputOutputSchema(
+        Seq(("TOK_TABLEROWFORMATFIELD", ","),
+          ("TOK_TABLEROWFORMATCOLLITEMS", "#"),
+          ("TOK_TABLEROWFORMATMAPKEYS", "@"),
+          ("TOK_TABLEROWFORMATNULL", "null"),
+          ("TOK_TABLEROWFORMATLINES", "\n")),
+        Seq(("TOK_TABLEROWFORMATFIELD", ","),
+          ("TOK_TABLEROWFORMATCOLLITEMS", "#"),
+          ("TOK_TABLEROWFORMATMAPKEYS", "@"),
+          ("TOK_TABLEROWFORMATNULL", "null"),
+          ("TOK_TABLEROWFORMATLINES", "\n")), None, None,
+        List.empty, List.empty, None, None, false)
+
+    assertEqual(
+      s"""
+         |SELECT TRANSFORM(a, b, c)
+         |  $rowFormat
+         |  USING 'cat' AS (a, b, c)
+         |  $rowFormat
+         |FROM testData
+      """.stripMargin,
+      ScriptTransformation(
+        "cat",
+        Seq(AttributeReference("a", StringType)(),
+          AttributeReference("b", StringType)(),
+          AttributeReference("c", StringType)()),
+        Project(Seq('a, 'b, 'c),
+          UnresolvedRelation(TableIdentifier("testData"))),
+        ioSchema))
+
+    assertEqual(
+      s"""
+         |SELECT TRANSFORM(a, sum(b), max(c))
+         |  $rowFormat
+         |  USING 'cat' AS (a, b, c)
+         |  $rowFormat
+         |FROM testData
+         |GROUP BY a
+         |HAVING sum(b) > 10
+      """.stripMargin,
+      ScriptTransformation(
+        "cat",
+        Seq(AttributeReference("a", StringType)(),
+          AttributeReference("b", StringType)(),
+          AttributeReference("c", StringType)()),
+        UnresolvedHaving(
+          GreaterThan(
+            UnresolvedFunction("sum", Seq(UnresolvedAttribute("b")), isDistinct = false),
+            Literal(10)),
+          Aggregate(
+            Seq('a),
+            Seq(
+              'a,
+              UnresolvedAlias(
+                UnresolvedFunction("sum", Seq(UnresolvedAttribute("b")), isDistinct = false), None),
+              UnresolvedAlias(
+                UnresolvedFunction("max", Seq(UnresolvedAttribute("c")), isDistinct = false), None)
+            ),
+            UnresolvedRelation(TableIdentifier("testData")))),
+        ioSchema))
+
+    assertEqual(
+      s"""
+         |SELECT TRANSFORM(a, sum(b) OVER w, max(c) OVER w)
+         |  $rowFormat
+         |  USING 'cat' AS (a, b, c)
+         |  $rowFormat
+         |FROM testData
+         |WINDOW w AS (PARTITION BY a ORDER BY b)
+      """.stripMargin,
+      ScriptTransformation(
+        "cat",
+        Seq(AttributeReference("a", StringType)(),
+          AttributeReference("b", StringType)(),
+          AttributeReference("c", StringType)()),
+        WithWindowDefinition(
+          Map("w" -> WindowSpecDefinition(
+            Seq('a),
+            Seq(SortOrder('b, Ascending, NullsFirst, Seq.empty)),
+            UnspecifiedFrame)),
+          Project(
+            Seq(
+              'a,
+              UnresolvedAlias(
+                UnresolvedWindowExpression(
+                  UnresolvedFunction("sum", Seq(UnresolvedAttribute("b")), isDistinct = false),
+                  WindowSpecReference("w")), None),
+              UnresolvedAlias(
+                UnresolvedWindowExpression(
+                  UnresolvedFunction("max", Seq(UnresolvedAttribute("c")), isDistinct = false),
+                  WindowSpecReference("w")), None)
+            ),
+            UnresolvedRelation(TableIdentifier("testData")))),
+        ioSchema))
+
+    assertEqual(
+      s"""
+         |SELECT TRANSFORM(a, sum(b), max(c))
+         |  $rowFormat
+         |  USING 'cat' AS (a, b, c)
+         |  $rowFormat
+         |FROM testData
+         |LATERAL VIEW explode(array(array(1,2,3))) myTable AS myCol
+         |LATERAL VIEW explode(myTable.myCol) myTable2 AS myCol2
+         |GROUP BY a, myCol, myCol2
+         |HAVING sum(b) > 10
+      """.stripMargin,
+      ScriptTransformation(
+        "cat",
+        Seq(AttributeReference("a", StringType)(),
+          AttributeReference("b", StringType)(),
+          AttributeReference("c", StringType)()),
+        UnresolvedHaving(
+          GreaterThan(
+            UnresolvedFunction("sum", Seq(UnresolvedAttribute("b")), isDistinct = false),
+            Literal(10)),
+          Aggregate(
+            Seq('a, 'myCol, 'myCol2),
+            Seq(
+              'a,
+              UnresolvedAlias(
+                UnresolvedFunction("sum", Seq(UnresolvedAttribute("b")), isDistinct = false), None),
+              UnresolvedAlias(
+                UnresolvedFunction("max", Seq(UnresolvedAttribute("c")), isDistinct = false), None)
+            ),
+            Generate(
+              UnresolvedGenerator(
+                FunctionIdentifier("explode"),
+                Seq(UnresolvedAttribute("myTable.myCol"))),
+              Nil, false, Option("mytable2"), Seq('myCol2),
+              Generate(
+                UnresolvedGenerator(
+                  FunctionIdentifier("explode"),
+                  Seq(UnresolvedFunction("array",
+                    Seq(
+                      UnresolvedFunction("array", Seq(Literal(1), Literal(2), Literal(3)), false)),
+                    false))),
+                Nil, false, Option("mytable"), Seq('myCol),
+                UnresolvedRelation(TableIdentifier("testData")))))),
+        ioSchema))
+  }
+
+  test("SPARK-32607: Script Transformation ROW FORMAT DELIMITED" +
+    " `TOK_TABLEROWFORMATLINES` only support '\\n'") {
+
+      // test input format TOK_TABLEROWFORMATLINES
+      intercept(
+          s"""
+             |SELECT TRANSFORM(a, b, c, d, e)
+             |  ROW FORMAT DELIMITED
+             |  FIELDS TERMINATED BY ','
+             |  LINES TERMINATED BY '@'
+             |  NULL DEFINED AS 'null'
+             |  USING 'cat' AS (value)
+             |  ROW FORMAT DELIMITED
+             |  FIELDS TERMINATED BY '&'
+             |  LINES TERMINATED BY '\n'
+             |  NULL DEFINED AS 'NULL'
+             |FROM v
+        """.stripMargin,
+      "LINES TERMINATED BY only supports newline '\\n' right now")
+
+    // test output format TOK_TABLEROWFORMATLINES
+    intercept(
+      s"""
+         |SELECT TRANSFORM(a, b, c, d, e)
+         |  ROW FORMAT DELIMITED
+         |  FIELDS TERMINATED BY ','
+         |  LINES TERMINATED BY '\n'
+         |  NULL DEFINED AS 'null'
+         |  USING 'cat' AS (value)
+         |  ROW FORMAT DELIMITED
+         |  FIELDS TERMINATED BY '&'
+         |  LINES TERMINATED BY '@'
+         |  NULL DEFINED AS 'NULL'
+         |FROM v
+        """.stripMargin,
+      "LINES TERMINATED BY only supports newline '\\n' right now")
+  }
+
+  test("CLEAR CACHE") {
+    assertEqual("CLEAR CACHE", ClearCacheCommand)
+  }
+
+  test("CREATE TABLE LIKE COMMAND should reject reserved properties") {
+    Seq(TableCatalog.PROP_OWNER, TableCatalog.PROP_PROVIDER).foreach { reserved =>
+      intercept(s"CREATE TABLE target LIKE source TBLPROPERTIES ($reserved='howdy')",
+        "reserved")
+    }
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SubExprEliminationBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SubExprEliminationBenchmark.scala
new file mode 100644
index 0000000000000..c025670fb895e
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SubExprEliminationBenchmark.scala
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution
+
+import org.apache.spark.benchmark.Benchmark
+import org.apache.spark.sql.Column
+import org.apache.spark.sql.catalyst.expressions.{Expression, Literal, Or}
+import org.apache.spark.sql.execution.benchmark.SqlBasedBenchmark
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * The benchmarks aims to measure performance of the queries where there are subexpression
+ * elimination or not.
+ * To run this benchmark:
+ * {{{
+ *   1. without sbt:
+ *      bin/spark-submit --class <this class> --jars <spark core test jar>,
+ *        <spark catalyst test jar> <spark sql test jar>
+ *   2. build/sbt "sql/test:runMain <this class>"
+ *   3. generate result:
+ *      SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
+ *      Results will be written to "benchmarks/SubExprEliminationBenchmark-results.txt".
+ * }}}
+ */
+object SubExprEliminationBenchmark extends SqlBasedBenchmark {
+  import spark.implicits._
+
+  def withFromJson(rowsNum: Int, numIters: Int): Unit = {
+    val benchmark = new Benchmark("from_json as subExpr in Project", rowsNum, output = output)
+
+    withTempPath { path =>
+      prepareDataInfo(benchmark)
+      val numCols = 500
+      val schema = writeWideRow(path.getAbsolutePath, rowsNum, numCols)
+
+      val cols = (0 until numCols).map { idx =>
+        from_json('value, schema).getField(s"col$idx")
+      }
+
+      Seq(
+        ("false", "true", "CODEGEN_ONLY"),
+        ("false", "false", "NO_CODEGEN"),
+        ("true", "true", "CODEGEN_ONLY"),
+        ("true", "false", "NO_CODEGEN")
+      ).foreach { case (subExprEliminationEnabled, codegenEnabled, codegenFactory) =>
+        // We only benchmark subexpression performance under codegen/non-codegen, so disabling
+        // json optimization.
+        val caseName = s"subExprElimination $subExprEliminationEnabled, codegen: $codegenEnabled"
+        benchmark.addCase(caseName, numIters) { _ =>
+          withSQLConf(
+            SQLConf.SUBEXPRESSION_ELIMINATION_ENABLED.key -> subExprEliminationEnabled,
+            SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> codegenEnabled,
+            SQLConf.CODEGEN_FACTORY_MODE.key -> codegenFactory,
+            SQLConf.JSON_EXPRESSION_OPTIMIZATION.key -> "false") {
+            val df = spark.read
+              .text(path.getAbsolutePath)
+              .select(cols: _*)
+            df.write.mode("overwrite").format("noop").save()
+          }
+        }
+      }
+
+      benchmark.run()
+    }
+  }
+
+  def withFilter(rowsNum: Int, numIters: Int): Unit = {
+    val benchmark = new Benchmark("from_json as subExpr in Filter", rowsNum, output = output)
+
+    withTempPath { path =>
+      prepareDataInfo(benchmark)
+      val numCols = 500
+      val schema = writeWideRow(path.getAbsolutePath, rowsNum, numCols)
+
+      val predicate = (0 until numCols).map { idx =>
+        (from_json('value, schema).getField(s"col$idx") >= Literal(100000)).expr
+      }.asInstanceOf[Seq[Expression]].reduce(Or)
+
+      Seq(
+        ("false", "true", "CODEGEN_ONLY"),
+        ("false", "false", "NO_CODEGEN"),
+        ("true", "true", "CODEGEN_ONLY"),
+        ("true", "false", "NO_CODEGEN")
+      ).foreach { case (subExprEliminationEnabled, codegenEnabled, codegenFactory) =>
+        // We only benchmark subexpression performance under codegen/non-codegen, so disabling
+        // json optimization.
+        val caseName = s"subExprElimination $subExprEliminationEnabled, codegen: $codegenEnabled"
+        benchmark.addCase(caseName, numIters) { _ =>
+          withSQLConf(
+            SQLConf.SUBEXPRESSION_ELIMINATION_ENABLED.key -> subExprEliminationEnabled,
+            SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> codegenEnabled,
+            SQLConf.CODEGEN_FACTORY_MODE.key -> codegenFactory,
+            SQLConf.JSON_EXPRESSION_OPTIMIZATION.key -> "false") {
+            val df = spark.read
+              .text(path.getAbsolutePath)
+              .where(Column(predicate))
+            df.write.mode("overwrite").format("noop").save()
+          }
+        }
+      }
+
+      benchmark.run()
+    }
+  }
+
+  override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
+    val numIters = 3
+    runBenchmark("Benchmark for performance of subexpression elimination") {
+      withFromJson(100, numIters)
+      withFilter(100, numIters)
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/TakeOrderedAndProjectSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/TakeOrderedAndProjectSuite.scala
index 376d330ebeb70..6ec5c6287eed1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/TakeOrderedAndProjectSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/TakeOrderedAndProjectSuite.scala
@@ -37,12 +37,18 @@ class TakeOrderedAndProjectSuite extends SparkPlanTest with SharedSparkSession {
     rand = new Random(seed)
   }
 
-  private def generateRandomInputData(): DataFrame = {
+  private def generateRandomInputData(numRows: Int, numParts: Int): DataFrame = {
     val schema = new StructType()
       .add("a", IntegerType, nullable = false)
       .add("b", IntegerType, nullable = false)
-    val inputData = Seq.fill(10000)(Row(rand.nextInt(), rand.nextInt()))
-    spark.createDataFrame(sparkContext.parallelize(Random.shuffle(inputData), 10), schema)
+    val rdd = if (numParts > 0) {
+      val inputData = Seq.fill(numRows)(Row(rand.nextInt(), rand.nextInt()))
+      sparkContext.parallelize(Random.shuffle(inputData), numParts)
+    } else {
+      sparkContext.emptyRDD[Row]
+    }
+    assert(rdd.getNumPartitions == numParts)
+    spark.createDataFrame(rdd, schema)
   }
 
   /**
@@ -56,31 +62,35 @@ class TakeOrderedAndProjectSuite extends SparkPlanTest with SharedSparkSession {
 
   test("TakeOrderedAndProject.doExecute without project") {
     withClue(s"seed = $seed") {
-      checkThatPlansAgree(
-        generateRandomInputData(),
-        input =>
-          noOpFilter(TakeOrderedAndProjectExec(limit, sortOrder, input.output, input)),
-        input =>
-          GlobalLimitExec(limit,
-            LocalLimitExec(limit,
-              SortExec(sortOrder, true, input))),
-        sortAnswers = false)
+      Seq((0, 0), (10000, 1), (10000, 10)).foreach { case (n, m) =>
+        checkThatPlansAgree(
+          generateRandomInputData(n, m),
+          input =>
+            noOpFilter(TakeOrderedAndProjectExec(limit, sortOrder, input.output, input)),
+          input =>
+            GlobalLimitExec(limit,
+              LocalLimitExec(limit,
+                SortExec(sortOrder, true, input))),
+          sortAnswers = false)
+      }
     }
   }
 
   test("TakeOrderedAndProject.doExecute with project") {
     withClue(s"seed = $seed") {
-      checkThatPlansAgree(
-        generateRandomInputData(),
-        input =>
-          noOpFilter(
-            TakeOrderedAndProjectExec(limit, sortOrder, Seq(input.output.last), input)),
-        input =>
-          GlobalLimitExec(limit,
-            LocalLimitExec(limit,
-              ProjectExec(Seq(input.output.last),
-                SortExec(sortOrder, true, input)))),
-        sortAnswers = false)
+      Seq((0, 0), (10000, 1), (10000, 10)).foreach { case (n, m) =>
+        checkThatPlansAgree(
+          generateRandomInputData(n, m),
+          input =>
+            noOpFilter(
+              TakeOrderedAndProjectExec(limit, sortOrder, Seq(input.output.last), input)),
+          input =>
+            GlobalLimitExec(limit,
+              LocalLimitExec(limit,
+                ProjectExec(Seq(input.output.last),
+                  SortExec(sortOrder, true, input)))),
+          sortAnswers = false)
+      }
     }
   }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/TestUncaughtExceptionHandler.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/TestUncaughtExceptionHandler.scala
similarity index 96%
rename from sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/TestUncaughtExceptionHandler.scala
rename to sql/core/src/test/scala/org/apache/spark/sql/execution/TestUncaughtExceptionHandler.scala
index 681eb4e255dbc..360f4658345e9 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/TestUncaughtExceptionHandler.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/TestUncaughtExceptionHandler.scala
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.spark.sql.hive.execution
+package org.apache.spark.sql.execution
 
 class TestUncaughtExceptionHandler extends Thread.UncaughtExceptionHandler {
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/UnsafeFixedWidthAggregationMapSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/UnsafeFixedWidthAggregationMapSuite.scala
index ef81f1b788496..97e5c1148c244 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/UnsafeFixedWidthAggregationMapSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/UnsafeFixedWidthAggregationMapSuite.scala
@@ -24,7 +24,8 @@ import scala.util.{Random, Try}
 import scala.util.control.NonFatal
 
 import org.mockito.Mockito._
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.{SparkConf, SparkFunSuite, TaskContext, TaskContextImpl}
 import org.apache.spark.internal.config.MEMORY_OFFHEAP_ENABLED
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/UnsafeKVExternalSorterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/UnsafeKVExternalSorterSuite.scala
index 8aa003a3dfeb0..f630cd8322c61 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/UnsafeKVExternalSorterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/UnsafeKVExternalSorterSuite.scala
@@ -210,23 +210,8 @@ class UnsafeKVExternalSorterSuite extends SparkFunSuite with SharedSparkSession
   test("SPARK-23376: Create UnsafeKVExternalSorter with BytesToByteMap having duplicated keys") {
     val memoryManager = new TestMemoryManager(new SparkConf())
     val taskMemoryManager = new TaskMemoryManager(memoryManager, 0)
-    val map = new BytesToBytesMap(taskMemoryManager, 64, taskMemoryManager.pageSizeBytes())
-
-    // Key/value are a unsafe rows with a single int column
+    val map = createBytesToBytesMapWithDuplicateKeys(taskMemoryManager)
     val schema = new StructType().add("i", IntegerType)
-    val key = new UnsafeRow(1)
-    key.pointTo(new Array[Byte](32), 32)
-    key.setInt(0, 1)
-    val value = new UnsafeRow(1)
-    value.pointTo(new Array[Byte](32), 32)
-    value.setInt(0, 2)
-
-    for (_ <- 1 to 65) {
-      val loc = map.lookup(key.getBaseObject, key.getBaseOffset, key.getSizeInBytes)
-      loc.append(
-        key.getBaseObject, key.getBaseOffset, key.getSizeInBytes,
-        value.getBaseObject, value.getBaseOffset, value.getSizeInBytes)
-    }
 
     // Make sure we can successfully create a UnsafeKVExternalSorter with a `BytesToBytesMap`
     // which has duplicated keys and the number of entries exceeds its capacity.
@@ -245,4 +230,82 @@ class UnsafeKVExternalSorterSuite extends SparkFunSuite with SharedSparkSession
       TaskContext.unset()
     }
   }
+
+  test("SPARK-31952: create UnsafeKVExternalSorter with existing map should count spilled memory " +
+    "size correctly") {
+    val memoryManager = new TestMemoryManager(new SparkConf())
+    val taskMemoryManager = new TaskMemoryManager(memoryManager, 0)
+    val map = createBytesToBytesMapWithDuplicateKeys(taskMemoryManager)
+    val schema = new StructType().add("i", IntegerType)
+
+    try {
+      val context = new TaskContextImpl(0, 0, 0, 0, 0, taskMemoryManager, new Properties(), null)
+      TaskContext.setTaskContext(context)
+      val expectedSpillSize = map.getTotalMemoryConsumption
+      val sorter = new UnsafeKVExternalSorter(
+        schema,
+        schema,
+        sparkContext.env.blockManager,
+        sparkContext.env.serializerManager,
+        taskMemoryManager.pageSizeBytes(),
+        Int.MaxValue,
+        map)
+      assert(sorter.getSpillSize === expectedSpillSize)
+    } finally {
+      TaskContext.unset()
+    }
+  }
+
+  test("SPARK-31952: UnsafeKVExternalSorter.merge should accumulate totalSpillBytes") {
+    val memoryManager = new TestMemoryManager(new SparkConf())
+    val taskMemoryManager = new TaskMemoryManager(memoryManager, 0)
+    val map1 = createBytesToBytesMapWithDuplicateKeys(taskMemoryManager)
+    val map2 = createBytesToBytesMapWithDuplicateKeys(taskMemoryManager)
+    val schema = new StructType().add("i", IntegerType)
+
+    try {
+      val context = new TaskContextImpl(0, 0, 0, 0, 0, taskMemoryManager, new Properties(), null)
+      TaskContext.setTaskContext(context)
+      val expectedSpillSize = map1.getTotalMemoryConsumption + map2.getTotalMemoryConsumption
+      val sorter1 = new UnsafeKVExternalSorter(
+        schema,
+        schema,
+        sparkContext.env.blockManager,
+        sparkContext.env.serializerManager,
+        taskMemoryManager.pageSizeBytes(),
+        Int.MaxValue,
+        map1)
+      val sorter2 = new UnsafeKVExternalSorter(
+        schema,
+        schema,
+        sparkContext.env.blockManager,
+        sparkContext.env.serializerManager,
+        taskMemoryManager.pageSizeBytes(),
+        Int.MaxValue,
+        map2)
+      sorter1.merge(sorter2)
+      assert(sorter1.getSpillSize === expectedSpillSize)
+    } finally {
+      TaskContext.unset()
+    }
+  }
+
+  private def createBytesToBytesMapWithDuplicateKeys(taskMemoryManager: TaskMemoryManager)
+    : BytesToBytesMap = {
+    val map = new BytesToBytesMap(taskMemoryManager, 64, taskMemoryManager.pageSizeBytes())
+    // Key/value are a unsafe rows with a single int column
+    val key = new UnsafeRow(1)
+    key.pointTo(new Array[Byte](32), 32)
+    key.setInt(0, 1)
+    val value = new UnsafeRow(1)
+    value.pointTo(new Array[Byte](32), 32)
+    value.setInt(0, 2)
+    for (_ <- 1 to 65) {
+      val loc = map.lookup(key.getBaseObject, key.getBaseOffset, key.getSizeInBytes)
+      loc.append(
+        key.getBaseObject, key.getBaseOffset, key.getSizeInBytes,
+        value.getBaseObject, value.getBaseOffset, value.getSizeInBytes)
+    }
+    map
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSparkSubmitSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSparkSubmitSuite.scala
index c5a01de911962..ee5e0e09fca21 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSparkSubmitSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSparkSubmitSuite.scala
@@ -17,14 +17,14 @@
 
 package org.apache.spark.sql.execution
 
-import org.scalatest.{Assertions, BeforeAndAfterEach, Matchers}
-import org.scalatest.concurrent.TimeLimits
+import org.scalatest.{Assertions, BeforeAndAfterEach}
+import org.scalatest.matchers.must.Matchers
 import org.scalatest.time.SpanSugar._
 
 import org.apache.spark.{SparkFunSuite, TestUtils}
 import org.apache.spark.deploy.SparkSubmitSuite
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.{LocalSparkSession, QueryTest, Row, SparkSession}
+import org.apache.spark.sql.{QueryTest, Row, SparkSession}
 import org.apache.spark.sql.functions.{array, col, count, lit}
 import org.apache.spark.sql.types.IntegerType
 import org.apache.spark.unsafe.Platform
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
index f7396ee2a89c8..f019e34b60118 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
@@ -22,8 +22,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.{ByteCodeStats, CodeAnd
 import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecutionSuite
 import org.apache.spark.sql.execution.aggregate.HashAggregateExec
 import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
-import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec
-import org.apache.spark.sql.execution.joins.SortMergeJoinExec
+import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BroadcastNestedLoopJoinExec, ShuffledHashJoinExec, SortMergeJoinExec}
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
@@ -51,6 +50,85 @@ class WholeStageCodegenSuite extends QueryTest with SharedSparkSession
     assert(df.collect() === Array(Row(9, 4.5)))
   }
 
+  testWithWholeStageCodegenOnAndOff("GenerateExec should be" +
+    " included in WholeStageCodegen") { codegenEnabled =>
+    import testImplicits._
+    val arrayData = Seq(("James", Seq("Java", "Scala"), Map("hair" -> "black", "eye" -> "brown")))
+    val df = arrayData.toDF("name", "knownLanguages", "properties")
+
+    // Array - explode
+    var expDF = df.select($"name", explode($"knownLanguages"), $"properties")
+    var plan = expDF.queryExecution.executedPlan
+    assert(plan.find {
+      case stage: WholeStageCodegenExec =>
+        stage.find(_.isInstanceOf[GenerateExec]).isDefined
+      case _ => !codegenEnabled.toBoolean
+    }.isDefined)
+    checkAnswer(expDF, Array(Row("James", "Java", Map("hair" -> "black", "eye" -> "brown")),
+      Row("James", "Scala", Map("hair" -> "black", "eye" -> "brown"))))
+
+    // Map - explode
+    expDF = df.select($"name", $"knownLanguages", explode($"properties"))
+    plan = expDF.queryExecution.executedPlan
+    assert(plan.find {
+      case stage: WholeStageCodegenExec =>
+        stage.find(_.isInstanceOf[GenerateExec]).isDefined
+      case _ => !codegenEnabled.toBoolean
+    }.isDefined)
+    checkAnswer(expDF,
+      Array(Row("James", List("Java", "Scala"), "hair", "black"),
+        Row("James", List("Java", "Scala"), "eye", "brown")))
+
+    // Array - posexplode
+    expDF = df.select($"name", posexplode($"knownLanguages"))
+    plan = expDF.queryExecution.executedPlan
+    assert(plan.find {
+      case stage: WholeStageCodegenExec =>
+        stage.find(_.isInstanceOf[GenerateExec]).isDefined
+      case _ => !codegenEnabled.toBoolean
+    }.isDefined)
+    checkAnswer(expDF,
+      Array(Row("James", 0, "Java"), Row("James", 1, "Scala")))
+
+    // Map - posexplode
+    expDF = df.select($"name", posexplode($"properties"))
+    plan = expDF.queryExecution.executedPlan
+    assert(plan.find {
+      case stage: WholeStageCodegenExec =>
+        stage.find(_.isInstanceOf[GenerateExec]).isDefined
+      case _ => !codegenEnabled.toBoolean
+    }.isDefined)
+    checkAnswer(expDF,
+      Array(Row("James", 0, "hair", "black"), Row("James", 1, "eye", "brown")))
+
+    // Array - explode , selecting all columns
+    expDF = df.select($"*", explode($"knownLanguages"))
+    plan = expDF.queryExecution.executedPlan
+    assert(plan.find {
+      case stage: WholeStageCodegenExec =>
+        stage.find(_.isInstanceOf[GenerateExec]).isDefined
+      case _ => !codegenEnabled.toBoolean
+    }.isDefined)
+    checkAnswer(expDF,
+      Array(Row("James", Seq("Java", "Scala"), Map("hair" -> "black", "eye" -> "brown"), "Java"),
+        Row("James", Seq("Java", "Scala"), Map("hair" -> "black", "eye" -> "brown"), "Scala")))
+
+    // Map - explode, selecting all columns
+    expDF = df.select($"*", explode($"properties"))
+    plan = expDF.queryExecution.executedPlan
+    assert(plan.find {
+      case stage: WholeStageCodegenExec =>
+        stage.find(_.isInstanceOf[GenerateExec]).isDefined
+      case _ => !codegenEnabled.toBoolean
+    }.isDefined)
+    checkAnswer(expDF,
+      Array(
+        Row("James", List("Java", "Scala"),
+          Map("hair" -> "black", "eye" -> "brown"), "hair", "black"),
+        Row("James", List("Java", "Scala"),
+          Map("hair" -> "black", "eye" -> "brown"), "eye", "brown")))
+  }
+
   test("Aggregate with grouping keys should be included in WholeStageCodegen") {
     val df = spark.range(3).groupBy(col("id") * 2).count().orderBy(col("id") * 2)
     val plan = df.queryExecution.executedPlan
@@ -71,6 +149,205 @@ class WholeStageCodegenSuite extends QueryTest with SharedSparkSession
     assert(df.collect() === Array(Row(1, 1, "1"), Row(1, 1, "1"), Row(2, 2, "2")))
   }
 
+  test("ShuffledHashJoin should be included in WholeStageCodegen") {
+    val df1 = spark.range(5).select($"id".as("k1"))
+    val df2 = spark.range(15).select($"id".as("k2"))
+    val df3 = spark.range(6).select($"id".as("k3"))
+
+    // test one shuffled hash join
+    val oneJoinDF = df1.join(df2.hint("SHUFFLE_HASH"), $"k1" === $"k2")
+    assert(oneJoinDF.queryExecution.executedPlan.collect {
+      case WholeStageCodegenExec(_ : ShuffledHashJoinExec) => true
+    }.size === 1)
+    checkAnswer(oneJoinDF, Seq(Row(0, 0), Row(1, 1), Row(2, 2), Row(3, 3), Row(4, 4)))
+
+    // test two shuffled hash joins
+    val twoJoinsDF = df1.join(df2.hint("SHUFFLE_HASH"), $"k1" === $"k2")
+      .join(df3.hint("SHUFFLE_HASH"), $"k1" === $"k3")
+    assert(twoJoinsDF.queryExecution.executedPlan.collect {
+      case WholeStageCodegenExec(_ : ShuffledHashJoinExec) => true
+    }.size === 2)
+    checkAnswer(twoJoinsDF,
+      Seq(Row(0, 0, 0), Row(1, 1, 1), Row(2, 2, 2), Row(3, 3, 3), Row(4, 4, 4)))
+  }
+
+  test("Left/Right Outer SortMergeJoin should be included in WholeStageCodegen") {
+    val df1 = spark.range(10).select($"id".as("k1"))
+    val df2 = spark.range(4).select($"id".as("k2"))
+    val df3 = spark.range(6).select($"id".as("k3"))
+
+    // test one left outer sort merge join
+    val oneLeftOuterJoinDF = df1.join(df2.hint("SHUFFLE_MERGE"), $"k1" === $"k2", "left_outer")
+    assert(oneLeftOuterJoinDF.queryExecution.executedPlan.collect {
+      case WholeStageCodegenExec(_ : SortMergeJoinExec) => true
+    }.size === 1)
+    checkAnswer(oneLeftOuterJoinDF, Seq(Row(0, 0), Row(1, 1), Row(2, 2), Row(3, 3), Row(4, null),
+      Row(5, null), Row(6, null), Row(7, null), Row(8, null), Row(9, null)))
+
+    // test one right outer sort merge join
+    val oneRightOuterJoinDF = df2.join(df3.hint("SHUFFLE_MERGE"), $"k2" === $"k3", "right_outer")
+    assert(oneRightOuterJoinDF.queryExecution.executedPlan.collect {
+      case WholeStageCodegenExec(_ : SortMergeJoinExec) => true
+    }.size === 1)
+    checkAnswer(oneRightOuterJoinDF, Seq(Row(0, 0), Row(1, 1), Row(2, 2), Row(3, 3), Row(null, 4),
+      Row(null, 5)))
+
+    // test two sort merge joins
+    val twoJoinsDF = df3.join(df2.hint("SHUFFLE_MERGE"), $"k3" === $"k2", "left_outer")
+      .join(df1.hint("SHUFFLE_MERGE"), $"k3" === $"k1", "right_outer")
+    assert(twoJoinsDF.queryExecution.executedPlan.collect {
+      case WholeStageCodegenExec(_ : SortMergeJoinExec) => true
+    }.size === 2)
+    checkAnswer(twoJoinsDF,
+      Seq(Row(0, 0, 0), Row(1, 1, 1), Row(2, 2, 2), Row(3, 3, 3), Row(4, null, 4), Row(5, null, 5),
+        Row(null, null, 6), Row(null, null, 7), Row(null, null, 8), Row(null, null, 9)))
+  }
+
+  test("Left Semi SortMergeJoin should be included in WholeStageCodegen") {
+    val df1 = spark.range(10).select($"id".as("k1"))
+    val df2 = spark.range(4).select($"id".as("k2"))
+    val df3 = spark.range(6).select($"id".as("k3"))
+
+    // test one left semi sort merge join
+    val oneJoinDF = df1.join(df2.hint("SHUFFLE_MERGE"), $"k1" === $"k2", "left_semi")
+    assert(oneJoinDF.queryExecution.executedPlan.collect {
+      case WholeStageCodegenExec(ProjectExec(_, _ : SortMergeJoinExec)) => true
+    }.size === 1)
+    checkAnswer(oneJoinDF, Seq(Row(0), Row(1), Row(2), Row(3)))
+
+    // test two left semi sort merge joins
+    val twoJoinsDF = df3.join(df2.hint("SHUFFLE_MERGE"), $"k3" === $"k2", "left_semi")
+      .join(df1.hint("SHUFFLE_MERGE"), $"k3" === $"k1", "left_semi")
+    assert(twoJoinsDF.queryExecution.executedPlan.collect {
+      case WholeStageCodegenExec(ProjectExec(_, _ : SortMergeJoinExec)) |
+           WholeStageCodegenExec(_ : SortMergeJoinExec) => true
+    }.size === 2)
+    checkAnswer(twoJoinsDF, Seq(Row(0), Row(1), Row(2), Row(3)))
+  }
+
+  test("Inner/Cross BroadcastNestedLoopJoinExec should be included in WholeStageCodegen") {
+    val df1 = spark.range(4).select($"id".as("k1"))
+    val df2 = spark.range(3).select($"id".as("k2"))
+    val df3 = spark.range(2).select($"id".as("k3"))
+
+    Seq(true, false).foreach { codegenEnabled =>
+      withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> codegenEnabled.toString) {
+        // test broadcast nested loop join without condition
+        val oneJoinDF = df1.join(df2)
+        var hasJoinInCodegen = oneJoinDF.queryExecution.executedPlan.collect {
+          case WholeStageCodegenExec(_ : BroadcastNestedLoopJoinExec) => true
+        }.size === 1
+        assert(hasJoinInCodegen == codegenEnabled)
+        checkAnswer(oneJoinDF,
+          Seq(Row(0, 0), Row(0, 1), Row(0, 2), Row(1, 0), Row(1, 1), Row(1, 2),
+            Row(2, 0), Row(2, 1), Row(2, 2), Row(3, 0), Row(3, 1), Row(3, 2)))
+
+        // test broadcast nested loop join with condition
+        val oneJoinDFWithCondition = df1.join(df2, $"k1" + 1 =!= $"k2")
+        hasJoinInCodegen = oneJoinDFWithCondition.queryExecution.executedPlan.collect {
+          case WholeStageCodegenExec(_ : BroadcastNestedLoopJoinExec) => true
+        }.size === 1
+        assert(hasJoinInCodegen == codegenEnabled)
+        checkAnswer(oneJoinDFWithCondition,
+          Seq(Row(0, 0), Row(0, 2), Row(1, 0), Row(1, 1), Row(2, 0), Row(2, 1),
+            Row(2, 2), Row(3, 0), Row(3, 1), Row(3, 2)))
+
+        // test two broadcast nested loop joins
+        val twoJoinsDF = df1.join(df2, $"k1" < $"k2").crossJoin(df3)
+        hasJoinInCodegen = twoJoinsDF.queryExecution.executedPlan.collect {
+          case WholeStageCodegenExec(BroadcastNestedLoopJoinExec(
+            _: BroadcastNestedLoopJoinExec, _, _, _, _)) => true
+        }.size === 1
+        assert(hasJoinInCodegen == codegenEnabled)
+        checkAnswer(twoJoinsDF,
+          Seq(Row(0, 1, 0), Row(0, 2, 0), Row(1, 2, 0), Row(0, 1, 1), Row(0, 2, 1), Row(1, 2, 1)))
+      }
+    }
+  }
+
+  test("Left/Right outer BroadcastNestedLoopJoinExec should be included in WholeStageCodegen") {
+    val df1 = spark.range(4).select($"id".as("k1"))
+    val df2 = spark.range(3).select($"id".as("k2"))
+    val df3 = spark.range(2).select($"id".as("k3"))
+    val df4 = spark.range(0).select($"id".as("k4"))
+
+    Seq(true, false).foreach { codegenEnabled =>
+      withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> codegenEnabled.toString) {
+        // test left outer join
+        val leftOuterJoinDF = df1.join(df2, $"k1" > $"k2", "left_outer")
+        var hasJoinInCodegen = leftOuterJoinDF.queryExecution.executedPlan.collect {
+          case WholeStageCodegenExec(_: BroadcastNestedLoopJoinExec) => true
+        }.size === 1
+        assert(hasJoinInCodegen == codegenEnabled)
+        checkAnswer(leftOuterJoinDF,
+          Seq(Row(0, null), Row(1, 0), Row(2, 0), Row(2, 1), Row(3, 0), Row(3, 1), Row(3, 2)))
+
+        // test right outer join
+        val rightOuterJoinDF = df1.join(df2, $"k1" < $"k2", "right_outer")
+        hasJoinInCodegen = rightOuterJoinDF.queryExecution.executedPlan.collect {
+          case WholeStageCodegenExec(_: BroadcastNestedLoopJoinExec) => true
+        }.size === 1
+        assert(hasJoinInCodegen == codegenEnabled)
+        checkAnswer(rightOuterJoinDF, Seq(Row(null, 0), Row(0, 1), Row(0, 2), Row(1, 2)))
+
+        // test a combination of left outer and right outer joins
+        val twoJoinsDF = df1.join(df2, $"k1" > $"k2" + 1, "right_outer")
+          .join(df3, $"k1" <= $"k3", "left_outer")
+        hasJoinInCodegen = twoJoinsDF.queryExecution.executedPlan.collect {
+          case WholeStageCodegenExec(BroadcastNestedLoopJoinExec(
+            _: BroadcastNestedLoopJoinExec, _, _, _, _)) => true
+        }.size === 1
+        assert(hasJoinInCodegen == codegenEnabled)
+        checkAnswer(twoJoinsDF,
+          Seq(Row(2, 0, null), Row(3, 0, null), Row(3, 1, null), Row(null, 2, null)))
+
+        // test build side is empty
+        val buildSideIsEmptyDF = df3.join(df4, $"k3" > $"k4", "left_outer")
+        hasJoinInCodegen = buildSideIsEmptyDF.queryExecution.executedPlan.collect {
+          case WholeStageCodegenExec(_: BroadcastNestedLoopJoinExec) => true
+        }.size === 1
+        assert(hasJoinInCodegen == codegenEnabled)
+        checkAnswer(buildSideIsEmptyDF, Seq(Row(0, null), Row(1, null)))
+      }
+    }
+  }
+
+  test("Left semi/anti BroadcastNestedLoopJoinExec should be included in WholeStageCodegen") {
+    val df1 = spark.range(4).select($"id".as("k1"))
+    val df2 = spark.range(3).select($"id".as("k2"))
+    val df3 = spark.range(2).select($"id".as("k3"))
+
+    Seq(true, false).foreach { codegenEnabled =>
+      withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> codegenEnabled.toString) {
+        // test left semi join
+        val semiJoinDF = df1.join(df2, $"k1" + 1 <= $"k2", "left_semi")
+        var hasJoinInCodegen = semiJoinDF.queryExecution.executedPlan.collect {
+          case WholeStageCodegenExec(ProjectExec(_, _ : BroadcastNestedLoopJoinExec)) => true
+        }.size === 1
+        assert(hasJoinInCodegen == codegenEnabled)
+        checkAnswer(semiJoinDF, Seq(Row(0), Row(1)))
+
+        // test left anti join
+        val antiJoinDF = df1.join(df2, $"k1" + 1 <= $"k2", "left_anti")
+        hasJoinInCodegen = antiJoinDF.queryExecution.executedPlan.collect {
+          case WholeStageCodegenExec(ProjectExec(_, _ : BroadcastNestedLoopJoinExec)) => true
+        }.size === 1
+        assert(hasJoinInCodegen == codegenEnabled)
+        checkAnswer(antiJoinDF, Seq(Row(2), Row(3)))
+
+        // test a combination of left semi and left anti joins
+        val twoJoinsDF = df1.join(df2, $"k1" < $"k2", "left_semi")
+          .join(df3, $"k1" > $"k3", "left_anti")
+        hasJoinInCodegen = twoJoinsDF.queryExecution.executedPlan.collect {
+          case WholeStageCodegenExec(ProjectExec(_, BroadcastNestedLoopJoinExec(
+          _: BroadcastNestedLoopJoinExec, _, _, _, _))) => true
+        }.size === 1
+        assert(hasJoinInCodegen == codegenEnabled)
+        checkAnswer(twoJoinsDF, Seq(Row(0)))
+      }
+    }
+  }
+
   test("Sort should be included in WholeStageCodegen") {
     val df = spark.range(3, 0, -1).toDF().sort(col("id"))
     val plan = df.queryExecution.executedPlan
@@ -342,7 +619,7 @@ class WholeStageCodegenSuite extends QueryTest with SharedSparkSession
         .join(baseTable, "idx")
       assert(distinctWithId.queryExecution.executedPlan.collectFirst {
         case WholeStageCodegenExec(
-          ProjectExec(_, BroadcastHashJoinExec(_, _, _, _, _, _: HashAggregateExec, _))) => true
+          ProjectExec(_, BroadcastHashJoinExec(_, _, _, _, _, _: HashAggregateExec, _, _))) => true
       }.isDefined)
       checkAnswer(distinctWithId, Seq(Row(1, 0), Row(1, 0)))
 
@@ -353,7 +630,7 @@ class WholeStageCodegenSuite extends QueryTest with SharedSparkSession
         .join(baseTable, "idx")
       assert(groupByWithId.queryExecution.executedPlan.collectFirst {
         case WholeStageCodegenExec(
-          ProjectExec(_, BroadcastHashJoinExec(_, _, _, _, _, _: HashAggregateExec, _))) => true
+          ProjectExec(_, BroadcastHashJoinExec(_, _, _, _, _, _: HashAggregateExec, _, _))) => true
       }.isDefined)
       checkAnswer(groupByWithId, Seq(Row(1, 2, 0), Row(1, 2, 0)))
     }
@@ -374,8 +651,8 @@ class WholeStageCodegenSuite extends QueryTest with SharedSparkSession
     // Case2: The parent of a LocalTableScanExec supports WholeStageCodegen.
     // In this case, the LocalTableScanExec should be within a WholeStageCodegen domain
     // and no more InputAdapter is inserted as the direct parent of the LocalTableScanExec.
-    val aggedDF = Seq(1, 2, 3).toDF.groupBy("value").sum()
-    val executedPlan = aggedDF.queryExecution.executedPlan
+    val aggregatedDF = Seq(1, 2, 3).toDF.groupBy("value").sum()
+    val executedPlan = aggregatedDF.queryExecution.executedPlan
 
     // HashAggregateExec supports WholeStageCodegen and it's the parent of
     // LocalTableScanExec so LocalTableScanExec should be within a WholeStageCodegen domain.
@@ -424,7 +701,7 @@ class WholeStageCodegenSuite extends QueryTest with SharedSparkSession
             "GROUP BY k").foreach { query =>
           val e = intercept[Exception] {
             sql(query).collect
-          }.getCause
+          }
           assert(e.isInstanceOf[IllegalStateException])
           assert(e.getMessage.contains(expectedErrMsg))
         }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index a80fc410f5033..454d3aa148a44 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -21,25 +21,34 @@ import java.io.File
 import java.net.URI
 
 import org.apache.log4j.Level
+import org.scalatest.PrivateMethodTester
 
 import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent, SparkListenerJobStart}
-import org.apache.spark.sql.{QueryTest, Row, SparkSession, Strategy}
+import org.apache.spark.sql.{Dataset, QueryTest, Row, SparkSession, Strategy}
+import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight}
 import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LogicalPlan}
-import org.apache.spark.sql.execution.{ReusedSubqueryExec, ShuffledRowRDD, SparkPlan}
+import org.apache.spark.sql.execution.{PartialReducerPartitionSpec, QueryExecution, ReusedSubqueryExec, ShuffledRowRDD, SparkPlan, UnaryExecNode}
 import org.apache.spark.sql.execution.command.DataWritingCommandExec
-import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, Exchange, ReusedExchangeExec}
-import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BuildLeft, BuildRight, SortMergeJoinExec}
+import org.apache.spark.sql.execution.datasources.noop.NoopDataSource
+import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec
+import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ENSURE_REQUIREMENTS, Exchange, REPARTITION, REPARTITION_WITH_NUM, ReusedExchangeExec, ShuffleExchangeExec, ShuffleExchangeLike, ShuffleOrigin}
+import org.apache.spark.sql.execution.joins.{BaseJoinExec, BroadcastHashJoinExec, ShuffledHashJoinExec, ShuffledJoin, SortMergeJoinExec}
+import org.apache.spark.sql.execution.metric.SQLShuffleReadMetricsReporter
 import org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.PartitionOverwriteMode
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.test.SQLTestData.TestData
 import org.apache.spark.sql.types.{IntegerType, StructType}
+import org.apache.spark.sql.util.QueryExecutionListener
 import org.apache.spark.util.Utils
 
 class AdaptiveQueryExecSuite
   extends QueryTest
   with SharedSparkSession
-  with AdaptiveSparkPlanHelper {
+  with AdaptiveSparkPlanHelper
+  with PrivateMethodTester {
 
   import testImplicits._
 
@@ -67,7 +76,7 @@ class AdaptiveQueryExecSuite
     val result = dfAdaptive.collect()
     withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
       val df = sql(query)
-      QueryTest.sameRows(result.toSeq, df.collect().toSeq)
+      checkAnswer(df, result)
     }
     val planAfter = dfAdaptive.queryExecution.executedPlan
     assert(planAfter.toString.startsWith("AdaptiveSparkPlan isFinalPlan=true"))
@@ -99,10 +108,22 @@ class AdaptiveQueryExecSuite
     }
   }
 
+  private def findTopLevelShuffledHashJoin(plan: SparkPlan): Seq[ShuffledHashJoinExec] = {
+    collect(plan) {
+      case j: ShuffledHashJoinExec => j
+    }
+  }
+
+  private def findTopLevelBaseJoin(plan: SparkPlan): Seq[BaseJoinExec] = {
+    collect(plan) {
+      case j: BaseJoinExec => j
+    }
+  }
+
   private def findReusedExchange(plan: SparkPlan): Seq[ReusedExchangeExec] = {
     collectWithSubqueries(plan) {
-      case ShuffleQueryStageExec(_, e: ReusedExchangeExec) => e
-      case BroadcastQueryStageExec(_, e: ReusedExchangeExec) => e
+      case ShuffleQueryStageExec(_, e: ReusedExchangeExec, _) => e
+      case BroadcastQueryStageExec(_, e: ReusedExchangeExec, _) => e
     }
   }
 
@@ -129,6 +150,17 @@ class AdaptiveQueryExecSuite
     assert(numShuffles === (numLocalReaders.length + numShufflesWithoutLocalReader))
   }
 
+  private def checkInitialPartitionNum(df: Dataset[_], numPartition: Int): Unit = {
+    // repartition obeys initialPartitionNum when adaptiveExecutionEnabled
+    val plan = df.queryExecution.executedPlan
+    assert(plan.isInstanceOf[AdaptiveSparkPlanExec])
+    val shuffle = plan.asInstanceOf[AdaptiveSparkPlanExec].executedPlan.collect {
+      case s: ShuffleExchangeExec => s
+    }
+    assert(shuffle.size == 1)
+    assert(shuffle(0).outputPartitioning.numPartitions == numPartition)
+  }
+
   test("Change merge join to broadcast join") {
     withSQLConf(
         SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
@@ -201,34 +233,36 @@ class AdaptiveQueryExecSuite
     }
   }
 
-  test("Empty stage coalesced to 0-partition RDD") {
+  test("Empty stage coalesced to 1-partition RDD") {
     withSQLConf(
       SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
       SQLConf.COALESCE_PARTITIONS_ENABLED.key -> "true") {
       val df1 = spark.range(10).withColumn("a", 'id)
       val df2 = spark.range(10).withColumn("b", 'id)
       withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
-        val testDf = df1.where('a > 10).join(df2.where('b > 10), "id").groupBy('a).count()
+        val testDf = df1.where('a > 10).join(df2.where('b > 10), Seq("id"), "left_outer")
+          .groupBy('a).count()
         checkAnswer(testDf, Seq())
         val plan = testDf.queryExecution.executedPlan
         assert(find(plan)(_.isInstanceOf[SortMergeJoinExec]).isDefined)
         val coalescedReaders = collect(plan) {
           case r: CustomShuffleReaderExec => r
         }
-        assert(coalescedReaders.length == 2)
-        coalescedReaders.foreach(r => assert(r.partitionSpecs.isEmpty))
+        assert(coalescedReaders.length == 3)
+        coalescedReaders.foreach(r => assert(r.partitionSpecs.length == 1))
       }
 
       withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "1") {
-        val testDf = df1.where('a > 10).join(df2.where('b > 10), "id").groupBy('a).count()
+        val testDf = df1.where('a > 10).join(df2.where('b > 10), Seq("id"), "left_outer")
+          .groupBy('a).count()
         checkAnswer(testDf, Seq())
         val plan = testDf.queryExecution.executedPlan
         assert(find(plan)(_.isInstanceOf[BroadcastHashJoinExec]).isDefined)
         val coalescedReaders = collect(plan) {
           case r: CustomShuffleReaderExec => r
         }
-        assert(coalescedReaders.length == 2, s"$plan")
-        coalescedReaders.foreach(r => assert(r.partitionSpecs.isEmpty))
+        assert(coalescedReaders.length == 3, s"$plan")
+        coalescedReaders.foreach(r => assert(r.isLocalReader || r.partitionSpecs.length == 1))
       }
     }
   }
@@ -545,7 +579,8 @@ class AdaptiveQueryExecSuite
 
     withSQLConf(
       SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-      SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80") {
+      SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80",
+      SQLConf.BROADCAST_HASH_JOIN_OUTPUT_PARTITIONING_EXPAND_LIMIT.key -> "0") {
       val (plan, adaptivePlan) = runAdaptiveAndVerifyResult(
         "SELECT * FROM testData " +
           "join testData2 t2 ON key = t2.a " +
@@ -628,98 +663,102 @@ class AdaptiveQueryExecSuite
     withSQLConf(
       SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
-      SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "700") {
+      SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "100",
+      SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "100") {
       withTempView("skewData1", "skewData2") {
         spark
           .range(0, 1000, 1, 10)
-          .selectExpr("id % 2 as key1", "id as value1")
+          .selectExpr("id % 3 as key1", "id as value1")
           .createOrReplaceTempView("skewData1")
         spark
           .range(0, 1000, 1, 10)
           .selectExpr("id % 1 as key2", "id as value2")
           .createOrReplaceTempView("skewData2")
-        val (_, innerAdaptivePlan) = runAdaptiveAndVerifyResult(
-          "SELECT key1 FROM skewData1 join skewData2 ON key1 = key2 group by key1")
+
+        def checkSkewJoin(query: String, optimizeSkewJoin: Boolean): Unit = {
+          val (_, innerAdaptivePlan) = runAdaptiveAndVerifyResult(query)
+          val innerSmj = findTopLevelSortMergeJoin(innerAdaptivePlan)
+          assert(innerSmj.size == 1 && innerSmj.head.isSkewJoin == optimizeSkewJoin)
+        }
+
+        checkSkewJoin(
+          "SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2", true)
         // Additional shuffle introduced, so disable the "OptimizeSkewedJoin" optimization
-        val innerSmj = findTopLevelSortMergeJoin(innerAdaptivePlan)
-        assert(innerSmj.size == 1 && !innerSmj.head.isSkewJoin)
+        checkSkewJoin(
+          "SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 GROUP BY key1", false)
       }
     }
   }
 
-  // TODO: we need a way to customize data distribution after shuffle, to improve test coverage
-  //       of this case.
   test("SPARK-29544: adaptive skew join with different join types") {
-    withSQLConf(
-      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-      SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
-      SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "2000",
-      SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "2000") {
-      withTempView("skewData1", "skewData2") {
-        spark
-          .range(0, 1000, 1, 10)
-          .selectExpr("id % 2 as key1", "id as value1")
-          .createOrReplaceTempView("skewData1")
-        spark
-          .range(0, 1000, 1, 10)
-          .selectExpr("id % 1 as key2", "id as value2")
-          .createOrReplaceTempView("skewData2")
+    Seq("SHUFFLE_MERGE", "SHUFFLE_HASH").foreach { joinHint =>
+      def getJoinNode(plan: SparkPlan): Seq[ShuffledJoin] = if (joinHint == "SHUFFLE_MERGE") {
+        findTopLevelSortMergeJoin(plan)
+      } else {
+        findTopLevelShuffledHashJoin(plan)
+      }
+      withSQLConf(
+        SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+        SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
+        SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key -> "1",
+        SQLConf.SHUFFLE_PARTITIONS.key -> "100",
+        SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "800",
+        SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "800") {
+        withTempView("skewData1", "skewData2") {
+          spark
+            .range(0, 1000, 1, 10)
+            .select(
+              when('id < 250, 249)
+                .when('id >= 750, 1000)
+                .otherwise('id).as("key1"),
+              'id as "value1")
+            .createOrReplaceTempView("skewData1")
+          spark
+            .range(0, 1000, 1, 10)
+            .select(
+              when('id < 250, 249)
+                .otherwise('id).as("key2"),
+              'id as "value2")
+            .createOrReplaceTempView("skewData2")
 
-        def checkSkewJoin(joins: Seq[SortMergeJoinExec], expectedNumPartitions: Int): Unit = {
-          assert(joins.size == 1 && joins.head.isSkewJoin)
-          assert(joins.head.left.collect {
-            case r: CustomShuffleReaderExec => r
-          }.head.partitionSpecs.length == expectedNumPartitions)
-          assert(joins.head.right.collect {
-            case r: CustomShuffleReaderExec => r
-          }.head.partitionSpecs.length == expectedNumPartitions)
-        }
+          def checkSkewJoin(
+              joins: Seq[ShuffledJoin],
+              leftSkewNum: Int,
+              rightSkewNum: Int): Unit = {
+            assert(joins.size == 1 && joins.head.isSkewJoin)
+            assert(joins.head.left.collect {
+              case r: CustomShuffleReaderExec => r
+            }.head.partitionSpecs.collect {
+              case p: PartialReducerPartitionSpec => p.reducerIndex
+            }.distinct.length == leftSkewNum)
+            assert(joins.head.right.collect {
+              case r: CustomShuffleReaderExec => r
+            }.head.partitionSpecs.collect {
+              case p: PartialReducerPartitionSpec => p.reducerIndex
+            }.distinct.length == rightSkewNum)
+          }
 
-        // skewed inner join optimization
-        val (_, innerAdaptivePlan) = runAdaptiveAndVerifyResult(
-          "SELECT * FROM skewData1 join skewData2 ON key1 = key2")
-        // left stats: [3496, 0, 0, 0, 4014]
-        // right stats:[6292, 0, 0, 0, 0]
-        // Partition 0: both left and right sides are skewed, left side is divided
-        //              into 2 splits and right side is divided into 4 splits, so
-        //              2 x 4 sub-partitions.
-        // Partition 1, 2, 3: not skewed, and coalesced into 1 partition, but it's ignored as the
-        //                    size is 0.
-        // Partition 4: only left side is skewed, and divide into 2 splits, so
-        //              2 sub-partitions.
-        // So total (8 + 0 + 2) partitions.
-        val innerSmj = findTopLevelSortMergeJoin(innerAdaptivePlan)
-        checkSkewJoin(innerSmj, 8 + 0 + 2)
-
-        // skewed left outer join optimization
-        val (_, leftAdaptivePlan) = runAdaptiveAndVerifyResult(
-          "SELECT * FROM skewData1 left outer join skewData2 ON key1 = key2")
-        // left stats: [3496, 0, 0, 0, 4014]
-        // right stats:[6292, 0, 0, 0, 0]
-        // Partition 0: both left and right sides are skewed, but left join can't split right side,
-        //              so only left side is divided into 2 splits, and thus 2 sub-partitions.
-        // Partition 1, 2, 3: not skewed, and coalesced into 1 partition, but it's ignored as the
-        //                    size is 0.
-        // Partition 4: only left side is skewed, and divide into 2 splits, so
-        //              2 sub-partitions.
-        // So total (2 + 0 + 2) partitions.
-        val leftSmj = findTopLevelSortMergeJoin(leftAdaptivePlan)
-        checkSkewJoin(leftSmj, 2 + 0 + 2)
-
-        // skewed right outer join optimization
-        val (_, rightAdaptivePlan) = runAdaptiveAndVerifyResult(
-          "SELECT * FROM skewData1 right outer join skewData2 ON key1 = key2")
-        // left stats: [3496, 0, 0, 0, 4014]
-        // right stats:[6292, 0, 0, 0, 0]
-        // Partition 0: both left and right sides are skewed, but right join can't split left side,
-        //              so only right side is divided into 4 splits, and thus 4 sub-partitions.
-        // Partition 1, 2, 3: not skewed, and coalesced into 1 partition, but it's ignored as the
-        //                    size is 0.
-        // Partition 4: only left side is skewed, but right join can't split left side, so just
-        //              1 partition.
-        // So total (4 + 0 + 1) partitions.
-        val rightSmj = findTopLevelSortMergeJoin(rightAdaptivePlan)
-        checkSkewJoin(rightSmj, 4 + 0 + 1)
+          // skewed inner join optimization
+          val (_, innerAdaptivePlan) = runAdaptiveAndVerifyResult(
+            s"SELECT /*+ $joinHint(skewData1) */ * FROM skewData1 " +
+              "JOIN skewData2 ON key1 = key2")
+          val inner = getJoinNode(innerAdaptivePlan)
+          checkSkewJoin(inner, 2, 1)
+
+          // skewed left outer join optimization
+          val (_, leftAdaptivePlan) = runAdaptiveAndVerifyResult(
+            s"SELECT /*+ $joinHint(skewData2) */ * FROM skewData1 " +
+              "LEFT OUTER JOIN skewData2 ON key1 = key2")
+          val leftJoin = getJoinNode(leftAdaptivePlan)
+          checkSkewJoin(leftJoin, 2, 0)
+
+          // skewed right outer join optimization
+          val (_, rightAdaptivePlan) = runAdaptiveAndVerifyResult(
+            s"SELECT /*+ $joinHint(skewData1) */ * FROM skewData1 " +
+              "RIGHT OUTER JOIN skewData2 ON key1 = key2")
+          val rightJoin = getJoinNode(rightAdaptivePlan)
+          checkSkewJoin(rightJoin, 0, 1)
+        }
       }
     }
   }
@@ -736,11 +775,11 @@ class AdaptiveQueryExecSuite
         Utils.deleteRecursively(tableDir)
         df1.write.parquet(tableDir.getAbsolutePath)
 
-        val agged = spark.table("bucketed_table").groupBy("i").count()
+        val aggregated = spark.table("bucketed_table").groupBy("i").count()
         val error = intercept[Exception] {
-          agged.count()
+          aggregated.count()
         }
-        assert(error.getCause().toString contains "Early failed query stage found")
+        assert(error.toString contains "Invalid bucket file")
         assert(error.getSuppressed.size === 0)
       }
     }
@@ -823,6 +862,19 @@ class AdaptiveQueryExecSuite
     }
   }
 
+  test("tree string output") {
+    withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") {
+      val df = sql("SELECT * FROM testData join testData2 ON key = a where value = '1'")
+      val planBefore = df.queryExecution.executedPlan
+      assert(!planBefore.toString.contains("== Current Plan =="))
+      assert(!planBefore.toString.contains("== Initial Plan =="))
+      df.collect()
+      val planAfter = df.queryExecution.executedPlan
+      assert(planAfter.toString.contains("== Final Plan =="))
+      assert(planAfter.toString.contains("== Initial Plan =="))
+    }
+  }
+
   test("SPARK-31384: avoid NPE in OptimizeSkewedJoin when there's 0 partition plan") {
     withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
@@ -836,6 +888,25 @@ class AdaptiveQueryExecSuite
     }
   }
 
+  test("SPARK-34682: CustomShuffleReaderExec operating on canonicalized plan") {
+    withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") {
+      val (_, adaptivePlan) = runAdaptiveAndVerifyResult(
+        "SELECT key FROM testData GROUP BY key")
+      val readers = collect(adaptivePlan) {
+        case r: CustomShuffleReaderExec => r
+      }
+      assert(readers.length == 1)
+      val reader = readers.head
+      val c = reader.canonicalized.asInstanceOf[CustomShuffleReaderExec]
+      // we can't just call execute() because that has separate checks for canonicalized plans
+      val ex = intercept[IllegalStateException] {
+        val doExecute = PrivateMethod[Unit](Symbol("doExecute"))
+        c.invokePrivate(doExecute())
+      }
+      assert(ex.getMessage === "operating on canonicalized plan")
+    }
+  }
+
   test("metrics of the shuffle reader") {
     withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") {
       val (_, adaptivePlan) = runAdaptiveAndVerifyResult(
@@ -873,28 +944,40 @@ class AdaptiveQueryExecSuite
 
       withSQLConf(
         SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
-        SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "2000",
-        SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "2000") {
+        SQLConf.SHUFFLE_PARTITIONS.key -> "100",
+        SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "800",
+        SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "800") {
         withTempView("skewData1", "skewData2") {
           spark
             .range(0, 1000, 1, 10)
-            .selectExpr("id % 2 as key1", "id as value1")
+            .select(
+              when('id < 250, 249)
+                .when('id >= 750, 1000)
+                .otherwise('id).as("key1"),
+              'id as "value1")
             .createOrReplaceTempView("skewData1")
           spark
             .range(0, 1000, 1, 10)
-            .selectExpr("id % 1 as key2", "id as value2")
+            .select(
+              when('id < 250, 249)
+                .otherwise('id).as("key2"),
+              'id as "value2")
             .createOrReplaceTempView("skewData2")
           val (_, adaptivePlan) = runAdaptiveAndVerifyResult(
             "SELECT * FROM skewData1 join skewData2 ON key1 = key2")
-          val reader = collect(adaptivePlan) {
+          val readers = collect(adaptivePlan) {
             case r: CustomShuffleReaderExec => r
-          }.head
-          assert(!reader.isLocalReader)
-          assert(reader.hasSkewedPartition)
-          assert(!reader.hasCoalescedPartition) // 0-size partitions are ignored.
-          assert(reader.metrics.contains("numSkewedPartitions"))
-          assert(reader.metrics("numSkewedPartitions").value > 0)
-          assert(reader.metrics("numSkewedSplits").value > 0)
+          }
+          readers.foreach { reader =>
+            assert(!reader.isLocalReader)
+            assert(reader.hasCoalescedPartition)
+            assert(reader.hasSkewedPartition)
+            assert(reader.metrics.contains("numSkewedPartitions"))
+          }
+          assert(readers(0).metrics("numSkewedPartitions").value == 2)
+          assert(readers(0).metrics("numSkewedSplits").value == 15)
+          assert(readers(1).metrics("numSkewedPartitions").value == 1)
+          assert(readers(1).metrics("numSkewedSplits").value == 12)
         }
       }
     }
@@ -918,9 +1001,9 @@ class AdaptiveQueryExecSuite
       withSQLConf(SQLConf.UI_EXPLAIN_MODE.key -> mode,
           SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
           SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80") {
-        val dfApdaptive = sql("SELECT * FROM testData JOIN testData2 ON key = a WHERE value = '1'")
+        val dfAdaptive = sql("SELECT * FROM testData JOIN testData2 ON key = a WHERE value = '1'")
         try {
-          checkAnswer(dfApdaptive, Row(1, "1", 1, 1) :: Row(1, "1", 1, 2) :: Nil)
+          checkAnswer(dfAdaptive, Row(1, "1", 1, 1) :: Row(1, "1", 1, 2) :: Nil)
           spark.sparkContext.listenerBus.waitUntilEmpty()
           assert(checkDone)
         } finally {
@@ -1014,4 +1097,548 @@ class AdaptiveQueryExecSuite
       }
     }
   }
+
+  test("SPARK-31220, SPARK-32056: repartition by expression with AQE") {
+    Seq(true, false).foreach { enableAQE =>
+      withSQLConf(
+        SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> enableAQE.toString,
+        SQLConf.COALESCE_PARTITIONS_ENABLED.key -> "true",
+        SQLConf.COALESCE_PARTITIONS_INITIAL_PARTITION_NUM.key -> "10",
+        SQLConf.SHUFFLE_PARTITIONS.key -> "10") {
+
+        val df1 = spark.range(10).repartition($"id")
+        val df2 = spark.range(10).repartition($"id" + 1)
+
+        val partitionsNum1 = df1.rdd.collectPartitions().length
+        val partitionsNum2 = df2.rdd.collectPartitions().length
+
+        if (enableAQE) {
+          assert(partitionsNum1 < 10)
+          assert(partitionsNum2 < 10)
+
+          checkInitialPartitionNum(df1, 10)
+          checkInitialPartitionNum(df2, 10)
+        } else {
+          assert(partitionsNum1 === 10)
+          assert(partitionsNum2 === 10)
+        }
+
+
+        // Don't coalesce partitions if the number of partitions is specified.
+        val df3 = spark.range(10).repartition(10, $"id")
+        val df4 = spark.range(10).repartition(10)
+        assert(df3.rdd.collectPartitions().length == 10)
+        assert(df4.rdd.collectPartitions().length == 10)
+      }
+    }
+  }
+
+  test("SPARK-31220, SPARK-32056: repartition by range with AQE") {
+    Seq(true, false).foreach { enableAQE =>
+      withSQLConf(
+        SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> enableAQE.toString,
+        SQLConf.COALESCE_PARTITIONS_ENABLED.key -> "true",
+        SQLConf.COALESCE_PARTITIONS_INITIAL_PARTITION_NUM.key -> "10",
+        SQLConf.SHUFFLE_PARTITIONS.key -> "10") {
+
+        val df1 = spark.range(10).toDF.repartitionByRange($"id".asc)
+        val df2 = spark.range(10).toDF.repartitionByRange(($"id" + 1).asc)
+
+        val partitionsNum1 = df1.rdd.collectPartitions().length
+        val partitionsNum2 = df2.rdd.collectPartitions().length
+
+        if (enableAQE) {
+          assert(partitionsNum1 < 10)
+          assert(partitionsNum2 < 10)
+
+          checkInitialPartitionNum(df1, 10)
+          checkInitialPartitionNum(df2, 10)
+        } else {
+          assert(partitionsNum1 === 10)
+          assert(partitionsNum2 === 10)
+        }
+
+        // Don't coalesce partitions if the number of partitions is specified.
+        val df3 = spark.range(10).repartitionByRange(10, $"id".asc)
+        assert(df3.rdd.collectPartitions().length == 10)
+      }
+    }
+  }
+
+  test("SPARK-31220, SPARK-32056: repartition using sql and hint with AQE") {
+    Seq(true, false).foreach { enableAQE =>
+      withTempView("test") {
+        withSQLConf(
+          SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> enableAQE.toString,
+          SQLConf.COALESCE_PARTITIONS_ENABLED.key -> "true",
+          SQLConf.COALESCE_PARTITIONS_INITIAL_PARTITION_NUM.key -> "10",
+          SQLConf.SHUFFLE_PARTITIONS.key -> "10") {
+
+          spark.range(10).toDF.createTempView("test")
+
+          val df1 = spark.sql("SELECT /*+ REPARTITION(id) */ * from test")
+          val df2 = spark.sql("SELECT /*+ REPARTITION_BY_RANGE(id) */ * from test")
+          val df3 = spark.sql("SELECT * from test DISTRIBUTE BY id")
+          val df4 = spark.sql("SELECT * from test CLUSTER BY id")
+
+          val partitionsNum1 = df1.rdd.collectPartitions().length
+          val partitionsNum2 = df2.rdd.collectPartitions().length
+          val partitionsNum3 = df3.rdd.collectPartitions().length
+          val partitionsNum4 = df4.rdd.collectPartitions().length
+
+          if (enableAQE) {
+            assert(partitionsNum1 < 10)
+            assert(partitionsNum2 < 10)
+            assert(partitionsNum3 < 10)
+            assert(partitionsNum4 < 10)
+
+            checkInitialPartitionNum(df1, 10)
+            checkInitialPartitionNum(df2, 10)
+            checkInitialPartitionNum(df3, 10)
+            checkInitialPartitionNum(df4, 10)
+          } else {
+            assert(partitionsNum1 === 10)
+            assert(partitionsNum2 === 10)
+            assert(partitionsNum3 === 10)
+            assert(partitionsNum4 === 10)
+          }
+
+          // Don't coalesce partitions if the number of partitions is specified.
+          val df5 = spark.sql("SELECT /*+ REPARTITION(10, id) */ * from test")
+          val df6 = spark.sql("SELECT /*+ REPARTITION_BY_RANGE(10, id) */ * from test")
+          assert(df5.rdd.collectPartitions().length == 10)
+          assert(df6.rdd.collectPartitions().length == 10)
+        }
+      }
+    }
+  }
+
+  test("SPARK-32573: Eliminate NAAJ when BuildSide is HashedRelationWithAllNullKeys") {
+    withSQLConf(
+      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+      SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> Long.MaxValue.toString) {
+      val (plan, adaptivePlan) = runAdaptiveAndVerifyResult(
+        "SELECT * FROM testData2 t1 WHERE t1.b NOT IN (SELECT b FROM testData3)")
+      val bhj = findTopLevelBroadcastHashJoin(plan)
+      assert(bhj.size == 1)
+      val join = findTopLevelBaseJoin(adaptivePlan)
+      assert(join.isEmpty)
+      checkNumLocalShuffleReaders(adaptivePlan)
+    }
+  }
+
+  test("SPARK-32717: AQEOptimizer should respect excludedRules configuration") {
+    withSQLConf(
+      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+      SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> Long.MaxValue.toString,
+      // This test is a copy of test(SPARK-32573), in order to test the configuration
+      // `spark.sql.adaptive.optimizer.excludedRules` works as expect.
+      SQLConf.ADAPTIVE_OPTIMIZER_EXCLUDED_RULES.key -> EliminateUnnecessaryJoin.ruleName) {
+      val (plan, adaptivePlan) = runAdaptiveAndVerifyResult(
+        "SELECT * FROM testData2 t1 WHERE t1.b NOT IN (SELECT b FROM testData3)")
+      val bhj = findTopLevelBroadcastHashJoin(plan)
+      assert(bhj.size == 1)
+      val join = findTopLevelBaseJoin(adaptivePlan)
+      // this is different compares to test(SPARK-32573) due to the rule
+      // `EliminateUnnecessaryJoin` has been excluded.
+      assert(join.nonEmpty)
+      checkNumLocalShuffleReaders(adaptivePlan)
+    }
+  }
+
+  test("SPARK-32649: Eliminate inner and semi join to empty relation") {
+    withSQLConf(
+      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+      SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80") {
+      Seq(
+        // inner join (small table at right side)
+        "SELECT * FROM testData t1 join testData3 t2 ON t1.key = t2.a WHERE t2.b = 1",
+        // inner join (small table at left side)
+        "SELECT * FROM testData3 t1 join testData t2 ON t1.a = t2.key WHERE t1.b = 1",
+        // left semi join
+        "SELECT * FROM testData t1 left semi join testData3 t2 ON t1.key = t2.a AND t2.b = 1"
+      ).foreach(query => {
+        val (plan, adaptivePlan) = runAdaptiveAndVerifyResult(query)
+        val smj = findTopLevelSortMergeJoin(plan)
+        assert(smj.size == 1)
+        val join = findTopLevelBaseJoin(adaptivePlan)
+        assert(join.isEmpty)
+        checkNumLocalShuffleReaders(adaptivePlan)
+      })
+    }
+  }
+
+  test("SPARK-34533: Eliminate left anti join to empty relation") {
+    withSQLConf(
+      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") {
+      Seq(
+        // broadcast non-empty right side
+        ("SELECT /*+ broadcast(testData3) */ * FROM testData LEFT ANTI JOIN testData3", true),
+        // broadcast empty right side
+        ("SELECT /*+ broadcast(emptyTestData) */ * FROM testData LEFT ANTI JOIN emptyTestData",
+          true),
+        // broadcast left side
+        ("SELECT /*+ broadcast(testData) */ * FROM testData LEFT ANTI JOIN testData3", false)
+      ).foreach { case (query, isEliminated) =>
+        val (plan, adaptivePlan) = runAdaptiveAndVerifyResult(query)
+        assert(findTopLevelBaseJoin(plan).size == 1)
+        assert(findTopLevelBaseJoin(adaptivePlan).isEmpty == isEliminated)
+      }
+    }
+  }
+
+  test("SPARK-34781: Eliminate left semi/anti join to its left side") {
+    withSQLConf(
+      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") {
+      Seq(
+        // left semi join and non-empty right side
+        ("SELECT * FROM testData LEFT SEMI JOIN testData3", true),
+        // left semi join, non-empty right side and non-empty join condition
+        ("SELECT * FROM testData t1 LEFT SEMI JOIN testData3 t2 ON t1.key = t2.a", false),
+        // left anti join and empty right side
+        ("SELECT * FROM testData LEFT ANTI JOIN emptyTestData", true),
+        // left anti join, empty right side and non-empty join condition
+        ("SELECT * FROM testData t1 LEFT ANTI JOIN emptyTestData t2 ON t1.key = t2.key", true)
+      ).foreach { case (query, isEliminated) =>
+        val (plan, adaptivePlan) = runAdaptiveAndVerifyResult(query)
+        assert(findTopLevelBaseJoin(plan).size == 1)
+        assert(findTopLevelBaseJoin(adaptivePlan).isEmpty == isEliminated)
+      }
+    }
+  }
+
+  test("SPARK-32753: Only copy tags to node with no tags") {
+    withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") {
+      withTempView("v1") {
+        spark.range(10).union(spark.range(10)).createOrReplaceTempView("v1")
+
+        val (_, adaptivePlan) = runAdaptiveAndVerifyResult(
+          "SELECT id FROM v1 GROUP BY id DISTRIBUTE BY id")
+        assert(collect(adaptivePlan) {
+          case s: ShuffleExchangeExec => s
+        }.length == 1)
+      }
+    }
+  }
+
+  test("Logging plan changes for AQE") {
+    val testAppender = new LogAppender("plan changes")
+    withLogAppender(testAppender) {
+      withSQLConf(
+          SQLConf.PLAN_CHANGE_LOG_LEVEL.key -> "INFO",
+          SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+          SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80") {
+        sql("SELECT * FROM testData JOIN testData2 ON key = a " +
+          "WHERE value = (SELECT max(a) FROM testData3)").collect()
+      }
+      Seq("=== Result of Batch AQE Preparations ===",
+          "=== Result of Batch AQE Post Stage Creation ===",
+          "=== Result of Batch AQE Replanning ===",
+          "=== Result of Batch AQE Query Stage Optimization ===",
+          "=== Result of Batch AQE Final Query Stage Optimization ===").foreach { expectedMsg =>
+        assert(testAppender.loggingEvents.exists(_.getRenderedMessage.contains(expectedMsg)))
+      }
+    }
+  }
+
+  test("SPARK-32932: Do not use local shuffle reader at final stage on write command") {
+    withSQLConf(SQLConf.PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.DYNAMIC.toString,
+      SQLConf.SHUFFLE_PARTITIONS.key -> "5",
+      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") {
+      val data = for (
+        i <- 1L to 10L;
+        j <- 1L to 3L
+      ) yield (i, j)
+
+      val df = data.toDF("i", "j").repartition($"j")
+      var noLocalReader: Boolean = false
+      val listener = new QueryExecutionListener {
+        override def onSuccess(funcName: String, qe: QueryExecution, durationNs: Long): Unit = {
+          qe.executedPlan match {
+            case plan@(_: DataWritingCommandExec | _: V2TableWriteExec) =>
+              assert(plan.asInstanceOf[UnaryExecNode].child.isInstanceOf[AdaptiveSparkPlanExec])
+              noLocalReader = collect(plan) {
+                case exec: CustomShuffleReaderExec if exec.isLocalReader => exec
+              }.isEmpty
+            case _ => // ignore other events
+          }
+        }
+        override def onFailure(funcName: String, qe: QueryExecution,
+          exception: Exception): Unit = {}
+      }
+      spark.listenerManager.register(listener)
+
+      withTable("t") {
+        df.write.partitionBy("j").saveAsTable("t")
+        sparkContext.listenerBus.waitUntilEmpty()
+        assert(noLocalReader)
+        noLocalReader = false
+      }
+
+      // Test DataSource v2
+      val format = classOf[NoopDataSource].getName
+      df.write.format(format).mode("overwrite").save()
+      sparkContext.listenerBus.waitUntilEmpty()
+      assert(noLocalReader)
+      noLocalReader = false
+
+      spark.listenerManager.unregister(listener)
+    }
+  }
+
+  test("SPARK-33494: Do not use local shuffle reader for repartition") {
+    withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") {
+      val df = spark.table("testData").repartition('key)
+      df.collect()
+      // local shuffle reader breaks partitioning and shouldn't be used for repartition operation
+      // which is specified by users.
+      checkNumLocalShuffleReaders(df.queryExecution.executedPlan, numShufflesWithoutLocalReader = 1)
+    }
+  }
+
+  test("SPARK-33551: Do not use custom shuffle reader for repartition") {
+    def hasRepartitionShuffle(plan: SparkPlan): Boolean = {
+      find(plan) {
+        case s: ShuffleExchangeLike =>
+          s.shuffleOrigin == REPARTITION || s.shuffleOrigin == REPARTITION_WITH_NUM
+        case _ => false
+      }.isDefined
+    }
+
+    withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+      SQLConf.SHUFFLE_PARTITIONS.key -> "5") {
+      val df = sql(
+        """
+          |SELECT * FROM (
+          |  SELECT * FROM testData WHERE key = 1
+          |)
+          |RIGHT OUTER JOIN testData2
+          |ON value = b
+        """.stripMargin)
+
+      withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80") {
+        // Repartition with no partition num specified.
+        val dfRepartition = df.repartition('b)
+        dfRepartition.collect()
+        val plan = dfRepartition.queryExecution.executedPlan
+        // The top shuffle from repartition is optimized out.
+        assert(!hasRepartitionShuffle(plan))
+        val bhj = findTopLevelBroadcastHashJoin(plan)
+        assert(bhj.length == 1)
+        checkNumLocalShuffleReaders(plan, 1)
+        // Probe side is coalesced.
+        val customReader = bhj.head.right.find(_.isInstanceOf[CustomShuffleReaderExec])
+        assert(customReader.isDefined)
+        assert(customReader.get.asInstanceOf[CustomShuffleReaderExec].hasCoalescedPartition)
+
+        // Repartition with partition default num specified.
+        val dfRepartitionWithNum = df.repartition(5, 'b)
+        dfRepartitionWithNum.collect()
+        val planWithNum = dfRepartitionWithNum.queryExecution.executedPlan
+        // The top shuffle from repartition is optimized out.
+        assert(!hasRepartitionShuffle(planWithNum))
+        val bhjWithNum = findTopLevelBroadcastHashJoin(planWithNum)
+        assert(bhjWithNum.length == 1)
+        checkNumLocalShuffleReaders(planWithNum, 1)
+        // Probe side is not coalesced.
+        assert(bhjWithNum.head.right.find(_.isInstanceOf[CustomShuffleReaderExec]).isEmpty)
+
+        // Repartition with partition non-default num specified.
+        val dfRepartitionWithNum2 = df.repartition(3, 'b)
+        dfRepartitionWithNum2.collect()
+        val planWithNum2 = dfRepartitionWithNum2.queryExecution.executedPlan
+        // The top shuffle from repartition is not optimized out, and this is the only shuffle that
+        // does not have local shuffle reader.
+        assert(hasRepartitionShuffle(planWithNum2))
+        val bhjWithNum2 = findTopLevelBroadcastHashJoin(planWithNum2)
+        assert(bhjWithNum2.length == 1)
+        checkNumLocalShuffleReaders(planWithNum2, 1)
+        val customReader2 = bhjWithNum2.head.right.find(_.isInstanceOf[CustomShuffleReaderExec])
+        assert(customReader2.isDefined)
+        assert(customReader2.get.asInstanceOf[CustomShuffleReaderExec].isLocalReader)
+      }
+
+      // Force skew join
+      withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
+        SQLConf.SKEW_JOIN_ENABLED.key -> "true",
+        SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "1",
+        SQLConf.SKEW_JOIN_SKEWED_PARTITION_FACTOR.key -> "0",
+        SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "10") {
+        // Repartition with no partition num specified.
+        val dfRepartition = df.repartition('b)
+        dfRepartition.collect()
+        val plan = dfRepartition.queryExecution.executedPlan
+        // The top shuffle from repartition is optimized out.
+        assert(!hasRepartitionShuffle(plan))
+        val smj = findTopLevelSortMergeJoin(plan)
+        assert(smj.length == 1)
+        // No skew join due to the repartition.
+        assert(!smj.head.isSkewJoin)
+        // Both sides are coalesced.
+        val customReaders = collect(smj.head) {
+          case c: CustomShuffleReaderExec if c.hasCoalescedPartition => c
+        }
+        assert(customReaders.length == 2)
+
+        // Repartition with default partition num specified.
+        val dfRepartitionWithNum = df.repartition(5, 'b)
+        dfRepartitionWithNum.collect()
+        val planWithNum = dfRepartitionWithNum.queryExecution.executedPlan
+        // The top shuffle from repartition is optimized out.
+        assert(!hasRepartitionShuffle(planWithNum))
+        val smjWithNum = findTopLevelSortMergeJoin(planWithNum)
+        assert(smjWithNum.length == 1)
+        // No skew join due to the repartition.
+        assert(!smjWithNum.head.isSkewJoin)
+        // No coalesce due to the num in repartition.
+        val customReadersWithNum = collect(smjWithNum.head) {
+          case c: CustomShuffleReaderExec if c.hasCoalescedPartition => c
+        }
+        assert(customReadersWithNum.isEmpty)
+
+        // Repartition with default non-partition num specified.
+        val dfRepartitionWithNum2 = df.repartition(3, 'b)
+        dfRepartitionWithNum2.collect()
+        val planWithNum2 = dfRepartitionWithNum2.queryExecution.executedPlan
+        // The top shuffle from repartition is not optimized out.
+        assert(hasRepartitionShuffle(planWithNum2))
+        val smjWithNum2 = findTopLevelSortMergeJoin(planWithNum2)
+        assert(smjWithNum2.length == 1)
+        // Skew join can apply as the repartition is not optimized out.
+        assert(smjWithNum2.head.isSkewJoin)
+      }
+    }
+  }
+
+  test("SPARK-34091: Batch shuffle fetch in AQE partition coalescing") {
+    withSQLConf(
+      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+      SQLConf.SHUFFLE_PARTITIONS.key -> "10000",
+      SQLConf.FETCH_SHUFFLE_BLOCKS_IN_BATCH.key -> "true") {
+      withTable("t1") {
+        spark.range(100).selectExpr("id + 1 as a").write.format("parquet").saveAsTable("t1")
+        val query = "SELECT SUM(a) FROM t1 GROUP BY a"
+        val (_, adaptivePlan) = runAdaptiveAndVerifyResult(query)
+        val metricName = SQLShuffleReadMetricsReporter.LOCAL_BLOCKS_FETCHED
+        val blocksFetchedMetric = collectFirst(adaptivePlan) {
+          case p if p.metrics.contains(metricName) => p.metrics(metricName)
+        }
+        assert(blocksFetchedMetric.isDefined)
+        val blocksFetched = blocksFetchedMetric.get.value
+        withSQLConf(SQLConf.FETCH_SHUFFLE_BLOCKS_IN_BATCH.key -> "false") {
+          val (_, adaptivePlan2) = runAdaptiveAndVerifyResult(query)
+          val blocksFetchedMetric2 = collectFirst(adaptivePlan2) {
+            case p if p.metrics.contains(metricName) => p.metrics(metricName)
+          }
+          assert(blocksFetchedMetric2.isDefined)
+          val blocksFetched2 = blocksFetchedMetric2.get.value
+          assert(blocksFetched < blocksFetched2)
+        }
+      }
+    }
+  }
+
+  test("SPARK-33933: Materialize BroadcastQueryStage first in AQE") {
+    val testAppender = new LogAppender("aqe query stage materialization order test")
+    val df = spark.range(1000).select($"id" % 26, $"id" % 10)
+      .toDF("index", "pv")
+    val dim = Range(0, 26).map(x => (x, ('a' + x).toChar.toString))
+      .toDF("index", "name")
+    val testDf = df.groupBy("index")
+      .agg(sum($"pv").alias("pv"))
+      .join(dim, Seq("index"))
+    withLogAppender(testAppender, level = Some(Level.DEBUG)) {
+      withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") {
+        val result = testDf.collect()
+        assert(result.length == 26)
+      }
+    }
+    val materializeLogs = testAppender.loggingEvents
+      .map(_.getRenderedMessage)
+      .filter(_.startsWith("Materialize query stage"))
+      .toArray
+    assert(materializeLogs(0).startsWith("Materialize query stage BroadcastQueryStageExec"))
+    assert(materializeLogs(1).startsWith("Materialize query stage ShuffleQueryStageExec"))
+  }
+
+  test("SPARK-34899: Use origin plan if we can not coalesce shuffle partition") {
+    def checkNoCoalescePartitions(ds: Dataset[Row], origin: ShuffleOrigin): Unit = {
+      assert(collect(ds.queryExecution.executedPlan) {
+        case s: ShuffleExchangeExec if s.shuffleOrigin == origin && s.numPartitions == 2 => s
+      }.size == 1)
+      ds.collect()
+      val plan = ds.queryExecution.executedPlan
+      assert(collect(plan) {
+        case c: CustomShuffleReaderExec => c
+      }.isEmpty)
+      assert(collect(plan) {
+        case s: ShuffleExchangeExec if s.shuffleOrigin == origin && s.numPartitions == 2 => s
+      }.size == 1)
+      checkAnswer(ds, testData)
+    }
+
+    withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+      SQLConf.COALESCE_PARTITIONS_ENABLED.key -> "true",
+      SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "2258",
+      SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key -> "1",
+      SQLConf.SHUFFLE_PARTITIONS.key -> "2") {
+      val df = spark.sparkContext.parallelize(
+        (1 to 100).map(i => TestData(i, i.toString)), 10).toDF()
+
+      // partition size [1420, 1420]
+      checkNoCoalescePartitions(df.repartition(), REPARTITION)
+      // partition size [1140, 1119]
+      checkNoCoalescePartitions(df.sort($"key"), ENSURE_REQUIREMENTS)
+    }
+  }
+
+  test("SPARK-35239: Coalesce shuffle partition should handle empty input RDD") {
+    withTable("t") {
+      withSQLConf(SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key -> "1",
+        SQLConf.SHUFFLE_PARTITIONS.key -> "2") {
+        spark.sql("CREATE TABLE t (c1 int) USING PARQUET")
+        val (_, adaptive) = runAdaptiveAndVerifyResult("SELECT c1, count(*) FROM t GROUP BY c1")
+        assert(
+          collect(adaptive) {
+            case c @ CustomShuffleReaderExec(_, partitionSpecs) if partitionSpecs.length == 1 => c
+          }.length == 1
+        )
+      }
+    }
+  }
+
+  test("SPARK-35264: Support AQE side broadcastJoin threshold") {
+    withTempView("t1", "t2") {
+      def checkJoinStrategy(shouldBroadcast: Boolean): Unit = {
+        withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+          val (origin, adaptive) = runAdaptiveAndVerifyResult(
+            "SELECT t1.c1, t2.c1 FROM t1 JOIN t2 ON t1.c1 = t2.c1")
+          assert(findTopLevelSortMergeJoin(origin).size == 1)
+          if (shouldBroadcast) {
+            assert(findTopLevelBroadcastHashJoin(adaptive).size == 1)
+          } else {
+            assert(findTopLevelSortMergeJoin(adaptive).size == 1)
+          }
+        }
+      }
+
+      // t1: 1600 bytes
+      // t2: 160 bytes
+      spark.sparkContext.parallelize(
+        (1 to 100).map(i => TestData(i, i.toString)), 10)
+        .toDF("c1", "c2").createOrReplaceTempView("t1")
+      spark.sparkContext.parallelize(
+        (1 to 10).map(i => TestData(i, i.toString)), 5)
+        .toDF("c1", "c2").createOrReplaceTempView("t2")
+
+      checkJoinStrategy(false)
+      withSQLConf(SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+        checkJoinStrategy(false)
+      }
+
+      withSQLConf(SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "160") {
+        checkJoinStrategy(true)
+      }
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveTestUtils.scala
index ddaeb57d31547..ad3ec85e984c8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveTestUtils.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveTestUtils.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.sql.execution.adaptive
 
-import java.io.{PrintWriter, StringWriter}
-
 import org.scalactic.source.Position
 import org.scalatest.Tag
 
@@ -69,25 +67,3 @@ trait DisableAdaptiveExecutionSuite extends SQLTestUtils {
     }
   }
 }
-
-object AdaptiveTestUtils {
-  def assertExceptionMessage(e: Exception, expected: String): Unit = {
-    val stringWriter = new StringWriter()
-    e.printStackTrace(new PrintWriter(stringWriter))
-    val errorMsg = stringWriter.toString
-    assert(errorMsg.contains(expected))
-  }
-
-  def assertExceptionCause(t: Throwable, causeClass: Class[_]): Unit = {
-    var c = t.getCause
-    var foundCause = false
-    while (c != null && !foundCause) {
-      if (causeClass.isAssignableFrom(c.getClass)) {
-        foundCause = true
-      } else {
-        c = c.getCause
-      }
-    }
-    assert(foundCause, s"Can not find cause: $causeClass")
-  }
-}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersSuite.scala
index 1e6e59456c887..a5ac2d5aa70c9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersSuite.scala
@@ -34,7 +34,7 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
-import org.apache.spark.sql.types.{BinaryType, Decimal, IntegerType, StructField, StructType}
+import org.apache.spark.sql.types.{ArrayType, BinaryType, Decimal, IntegerType, NullType, StructField, StructType}
 import org.apache.spark.sql.util.ArrowUtils
 import org.apache.spark.util.Utils
 
@@ -1071,6 +1071,61 @@ class ArrowConvertersSuite extends SharedSparkSession {
     collectAndValidate(df, json, "structData.json")
   }
 
+  test("null type conversion") {
+    val json =
+      s"""
+         |{
+         |  "schema" : {
+         |    "fields": [ {
+         |      "name" : "a",
+         |      "type" : {
+         |        "name" : "null"
+         |      },
+         |      "nullable" : true,
+         |      "children" : [ ]
+         |    }, {
+         |      "name" : "b",
+         |      "type" : {
+         |        "name" : "list"
+         |      },
+         |      "nullable" : true,
+         |      "children" : [ {
+         |        "name" : "element",
+         |        "type" : {
+         |          "name" : "null"
+         |        },
+         |        "nullable" : true,
+         |        "children" : [ ]
+         |      } ]
+         |    } ]
+         |  },
+         |  "batches" : [ {
+         |    "count" : 3,
+         |    "columns" : [ {
+         |      "name" : "a",
+         |      "count" : 3
+         |    }, {
+         |      "name" : "b",
+         |      "count" : 3,
+         |      "VALIDITY" : [ 1, 1, 1 ],
+         |      "OFFSET" : [ 0, 2, 4, 6 ],
+         |      "children" : [ {
+         |        "name" : "element",
+         |        "count" : 6
+         |      } ]
+         |    } ]
+         |  } ]
+         |}
+       """.stripMargin
+
+    val data = Seq(null, null, null)
+    val rdd = sparkContext.parallelize(data.map(n => Row(n, Seq(n, n))))
+    val schema = new StructType().add("a", NullType).add("b", ArrayType(NullType))
+    val df = spark.createDataFrame(rdd, schema)
+
+    collectAndValidate(df, json, "nullData.json")
+  }
+
   test("partitioned DataFrame") {
     val json1 =
       s"""
@@ -1210,7 +1265,7 @@ class ArrowConvertersSuite extends SharedSparkSession {
 
   testQuietly("interval is unsupported for arrow") {
     val e = intercept[SparkException] {
-      calenderIntervalData.toDF().toArrowBatchRdd.collect()
+      calendarIntervalData.toDF().toArrowBatchRdd.collect()
     }
 
     assert(e.getCause.isInstanceOf[UnsupportedOperationException])
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowWriterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowWriterSuite.scala
index bdc3b5eed7d8d..1684633c92854 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowWriterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowWriterSuite.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql.execution.arrow
 
+import org.apache.arrow.vector.IntervalDayVector
+
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.util._
@@ -54,6 +56,8 @@ class ArrowWriterSuite extends SparkFunSuite {
             case BinaryType => reader.getBinary(rowId)
             case DateType => reader.getInt(rowId)
             case TimestampType => reader.getLong(rowId)
+            case YearMonthIntervalType => reader.getInt(rowId)
+            case DayTimeIntervalType => reader.getLong(rowId)
           }
           assert(value === datum)
       }
@@ -72,6 +76,34 @@ class ArrowWriterSuite extends SparkFunSuite {
     check(BinaryType, Seq("a".getBytes(), "b".getBytes(), null, "d".getBytes()))
     check(DateType, Seq(0, 1, 2, null, 4))
     check(TimestampType, Seq(0L, 3.6e9.toLong, null, 8.64e10.toLong), "America/Los_Angeles")
+    check(NullType, Seq(null, null, null))
+    check(YearMonthIntervalType, Seq(null, 0, 1, -1, Int.MaxValue, Int.MinValue))
+    check(DayTimeIntervalType, Seq(null, 0L, 1000L, -1000L, (Long.MaxValue - 807L),
+      (Long.MinValue + 808L)))
+  }
+
+  test("long overflow for DayTimeIntervalType")
+  {
+    val schema = new StructType().add("value", DayTimeIntervalType, nullable = true)
+    val writer = ArrowWriter.create(schema, null)
+    val reader = new ArrowColumnVector(writer.root.getFieldVectors().get(0))
+    val valueVector = writer.root.getFieldVectors().get(0).asInstanceOf[IntervalDayVector]
+
+    valueVector.set(0, 106751992, 0)
+    valueVector.set(1, 106751991, Int.MaxValue)
+
+    // first long overflow for test Math.multiplyExact()
+    val msg = intercept[java.lang.ArithmeticException] {
+      reader.getLong(0)
+    }.getMessage
+    assert(msg.equals("long overflow"))
+
+    // second long overflow for test Math.addExact()
+    val msg1 = intercept[java.lang.ArithmeticException] {
+      reader.getLong(1)
+    }.getMessage
+    assert(msg1.equals("long overflow"))
+    writer.root.close()
   }
 
   test("get multiple") {
@@ -96,6 +128,8 @@ class ArrowWriterSuite extends SparkFunSuite {
         case DoubleType => reader.getDoubles(0, data.size)
         case DateType => reader.getInts(0, data.size)
         case TimestampType => reader.getLongs(0, data.size)
+        case YearMonthIntervalType => reader.getInts(0, data.size)
+        case DayTimeIntervalType => reader.getLongs(0, data.size)
       }
       assert(values === data)
 
@@ -110,6 +144,8 @@ class ArrowWriterSuite extends SparkFunSuite {
     check(DoubleType, (0 until 10).map(_.toDouble))
     check(DateType, (0 until 10))
     check(TimestampType, (0 until 10).map(_ * 4.32e10.toLong), "America/Los_Angeles")
+    check(YearMonthIntervalType, (0 until 10))
+    check(DayTimeIntervalType, (-10 until 10).map(_ * 1000.toLong))
   }
 
   test("array") {
@@ -202,6 +238,46 @@ class ArrowWriterSuite extends SparkFunSuite {
     writer.root.close()
   }
 
+  test("null array") {
+    val schema = new StructType()
+      .add("arr", ArrayType(NullType, containsNull = true), nullable = true)
+    val writer = ArrowWriter.create(schema, null)
+    assert(writer.schema === schema)
+
+    writer.write(InternalRow(ArrayData.toArrayData(Array(null, null, null))))
+    writer.write(InternalRow(ArrayData.toArrayData(Array(null, null))))
+    writer.write(InternalRow(null))
+    writer.write(InternalRow(ArrayData.toArrayData(Array.empty[Int])))
+    writer.write(InternalRow(ArrayData.toArrayData(Array(null, null, null))))
+    writer.finish()
+
+    val reader = new ArrowColumnVector(writer.root.getFieldVectors().get(0))
+
+    val array0 = reader.getArray(0)
+    assert(array0.numElements() === 3)
+    assert(array0.isNullAt(0))
+    assert(array0.isNullAt(1))
+    assert(array0.isNullAt(2))
+
+    val array1 = reader.getArray(1)
+    assert(array1.numElements() === 2)
+    assert(array1.isNullAt(0))
+    assert(array1.isNullAt(1))
+
+    assert(reader.isNullAt(2))
+
+    val array3 = reader.getArray(3)
+    assert(array3.numElements() === 0)
+
+    val array4 = reader.getArray(4)
+    assert(array4.numElements() === 3)
+    assert(array4.isNullAt(0))
+    assert(array4.isNullAt(1))
+    assert(array4.isNullAt(2))
+
+    writer.root.close()
+  }
+
   test("struct") {
     val schema = new StructType()
       .add("struct", new StructType().add("i", IntegerType).add("str", StringType))
@@ -268,6 +344,32 @@ class ArrowWriterSuite extends SparkFunSuite {
     writer.root.close()
   }
 
+  test("null struct") {
+    val schema = new StructType()
+      .add("struct", new StructType().add("n1", NullType).add("n2", NullType))
+    val writer = ArrowWriter.create(schema, null)
+    assert(writer.schema === schema)
+
+    writer.write(InternalRow(InternalRow(null, null)))
+    writer.write(InternalRow(null))
+    writer.write(InternalRow(InternalRow(null, null)))
+    writer.finish()
+
+    val reader = new ArrowColumnVector(writer.root.getFieldVectors().get(0))
+
+    val struct0 = reader.getStruct(0)
+    assert(struct0.isNullAt(0))
+    assert(struct0.isNullAt(1))
+
+    assert(reader.isNullAt(1))
+
+    val struct2 = reader.getStruct(2)
+    assert(struct2.isNullAt(0))
+    assert(struct2.isNullAt(1))
+
+    writer.root.close()
+  }
+
   test("map") {
     val schema = new StructType()
       .add("map", MapType(IntegerType, StringType), nullable = true)
@@ -326,6 +428,37 @@ class ArrowWriterSuite extends SparkFunSuite {
     writer.root.close()
   }
 
+  test("null value map") {
+    val schema = new StructType()
+      .add("map", MapType(IntegerType, NullType), nullable = true)
+    val writer = ArrowWriter.create(schema, null)
+    assert(writer.schema == schema)
+
+    writer.write(InternalRow(ArrayBasedMapData(
+      keys = Array(1, 2, 3),
+      values = Array(null, null, null)
+    )))
+    writer.write(InternalRow(ArrayBasedMapData(Array(43), Array(null))))
+    writer.write(InternalRow(null))
+
+    writer.finish()
+
+    val reader = new ArrowColumnVector(writer.root.getFieldVectors.get(0))
+    val map0 = reader.getMap(0)
+    assert(map0.numElements() == 3)
+    assert(map0.keyArray().array().mkString(",") == Array(1, 2, 3).mkString(","))
+    assert(map0.valueArray().array().mkString(",") == Array(null, null, null).mkString(","))
+
+    val map1 = reader.getMap(1)
+    assert(map1.numElements() == 1)
+    assert(map1.keyArray().array().mkString(",") == Array(43).mkString(","))
+    assert(map1.valueArray().array().mkString(",") == Array(null).mkString(","))
+
+    val map2 = reader.getMap(3)
+    assert(map2 == null)
+    writer.root.close()
+  }
+
   test("nested map") {
     val valueSchema = new StructType()
       .add("name", StringType)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/AggregateBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/AggregateBenchmark.scala
index 965d78227c335..b8d7b774d832d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/AggregateBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/AggregateBenchmark.scala
@@ -36,7 +36,8 @@ import org.apache.spark.unsafe.map.BytesToBytesMap
  * Benchmark to measure performance for aggregate primitives.
  * To run this benchmark:
  * {{{
- *   1. without sbt: bin/spark-submit --class <this class> <spark sql test jar>
+ *   1. without sbt: bin/spark-submit --class <this class>
+ *      --jars <spark core test jar>,<spark catalyst test jar> <spark sql test jar>
  *   2. build/sbt "sql/test:runMain <this class>"
  *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
  *      Results will be written to "benchmarks/AggregateBenchmark-results.txt".
@@ -79,7 +80,7 @@ object AggregateBenchmark extends SqlBasedBenchmark {
         }
       }
 
-      benchmark.addCase("codegen = T hashmap = F", numIters = 3) { _ =>
+      benchmark.addCase("codegen = T, hashmap = F", numIters = 3) { _ =>
         withSQLConf(
           SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true",
           SQLConf.ENABLE_TWOLEVEL_AGG_MAP.key -> "false",
@@ -88,7 +89,16 @@ object AggregateBenchmark extends SqlBasedBenchmark {
         }
       }
 
-      benchmark.addCase("codegen = T hashmap = T", numIters = 5) { _ =>
+      benchmark.addCase("codegen = T, row-based hashmap = T", numIters = 5) { _ =>
+        withSQLConf(
+          SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true",
+          SQLConf.ENABLE_TWOLEVEL_AGG_MAP.key -> "true",
+          SQLConf.ENABLE_VECTORIZED_HASH_MAP.key -> "false") {
+          f()
+        }
+      }
+
+      benchmark.addCase("codegen = T, vectorized hashmap = T", numIters = 5) { _ =>
         withSQLConf(
           SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true",
           SQLConf.ENABLE_TWOLEVEL_AGG_MAP.key -> "true",
@@ -115,7 +125,7 @@ object AggregateBenchmark extends SqlBasedBenchmark {
         }
       }
 
-      benchmark.addCase("codegen = T hashmap = F", numIters = 3) { _ =>
+      benchmark.addCase("codegen = T, hashmap = F", numIters = 3) { _ =>
         withSQLConf(
           SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true",
           SQLConf.ENABLE_TWOLEVEL_AGG_MAP.key -> "false",
@@ -124,7 +134,16 @@ object AggregateBenchmark extends SqlBasedBenchmark {
         }
       }
 
-      benchmark.addCase("codegen = T hashmap = T", numIters = 5) { _ =>
+      benchmark.addCase("codegen = T, row-based hashmap = T", numIters = 5) { _ =>
+        withSQLConf(
+          SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true",
+          SQLConf.ENABLE_TWOLEVEL_AGG_MAP.key -> "true",
+          SQLConf.ENABLE_VECTORIZED_HASH_MAP.key -> "false") {
+          f()
+        }
+      }
+
+      benchmark.addCase("codegen = T, vectorized hashmap = T", numIters = 5) { _ =>
         withSQLConf(
           SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true",
           SQLConf.ENABLE_TWOLEVEL_AGG_MAP.key -> "true",
@@ -150,7 +169,7 @@ object AggregateBenchmark extends SqlBasedBenchmark {
         }
       }
 
-      benchmark.addCase("codegen = T hashmap = F", numIters = 3) { _ =>
+      benchmark.addCase("codegen = T, hashmap = F", numIters = 3) { _ =>
         withSQLConf(
           SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true",
           SQLConf.ENABLE_TWOLEVEL_AGG_MAP.key -> "false",
@@ -159,7 +178,16 @@ object AggregateBenchmark extends SqlBasedBenchmark {
         }
       }
 
-      benchmark.addCase("codegen = T hashmap = T", numIters = 5) { _ =>
+      benchmark.addCase("codegen = T, row-based hashmap = T", numIters = 5) { _ =>
+        withSQLConf(
+          SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true",
+          SQLConf.ENABLE_TWOLEVEL_AGG_MAP.key -> "true",
+          SQLConf.ENABLE_VECTORIZED_HASH_MAP.key -> "false") {
+          f()
+        }
+      }
+
+      benchmark.addCase("codegen = T, vectorized hashmap = T", numIters = 5) { _ =>
         withSQLConf(
           SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true",
           SQLConf.ENABLE_TWOLEVEL_AGG_MAP.key -> "true",
@@ -185,7 +213,7 @@ object AggregateBenchmark extends SqlBasedBenchmark {
         }
       }
 
-      benchmark.addCase("codegen = T hashmap = F") { _ =>
+      benchmark.addCase("codegen = T, hashmap = F") { _ =>
         withSQLConf(
           SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true",
           SQLConf.ENABLE_TWOLEVEL_AGG_MAP.key -> "false",
@@ -194,7 +222,16 @@ object AggregateBenchmark extends SqlBasedBenchmark {
         }
       }
 
-      benchmark.addCase("codegen = T hashmap = T") { _ =>
+      benchmark.addCase("codegen = T, row-based hashmap = T") { _ =>
+        withSQLConf(
+          SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true",
+          SQLConf.ENABLE_TWOLEVEL_AGG_MAP.key -> "true",
+          SQLConf.ENABLE_VECTORIZED_HASH_MAP.key -> "false") {
+          f()
+        }
+      }
+
+      benchmark.addCase("codegen = T, vectorized hashmap = T") { _ =>
         withSQLConf(
           SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true",
           SQLConf.ENABLE_TWOLEVEL_AGG_MAP.key -> "true",
@@ -230,7 +267,7 @@ object AggregateBenchmark extends SqlBasedBenchmark {
         }
       }
 
-      benchmark.addCase("codegen = T hashmap = F") { _ =>
+      benchmark.addCase("codegen = T, hashmap = F") { _ =>
         withSQLConf(
           SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true",
           SQLConf.ENABLE_TWOLEVEL_AGG_MAP.key -> "false",
@@ -239,7 +276,16 @@ object AggregateBenchmark extends SqlBasedBenchmark {
         }
       }
 
-      benchmark.addCase("codegen = T hashmap = T") { _ =>
+      benchmark.addCase("codegen = T, row-based hashmap = T") { _ =>
+        withSQLConf(
+          SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true",
+          SQLConf.ENABLE_TWOLEVEL_AGG_MAP.key -> "true",
+          SQLConf.ENABLE_VECTORIZED_HASH_MAP.key -> "false") {
+          f()
+        }
+      }
+
+      benchmark.addCase("codegen = T, vectorized hashmap = T") { _ =>
         withSQLConf(
           SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true",
           SQLConf.ENABLE_TWOLEVEL_AGG_MAP.key -> "true",
@@ -290,7 +336,7 @@ object AggregateBenchmark extends SqlBasedBenchmark {
         }
       }
 
-      benchmark.addCase("codegen = T hugeMethodLimit = 10000") { _ =>
+      benchmark.addCase("codegen = T, hugeMethodLimit = 10000") { _ =>
         withSQLConf(
           SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true",
           SQLConf.WHOLESTAGE_HUGE_METHOD_LIMIT.key -> "10000") {
@@ -298,7 +344,7 @@ object AggregateBenchmark extends SqlBasedBenchmark {
         }
       }
 
-      benchmark.addCase("codegen = T hugeMethodLimit = 1500") { _ =>
+      benchmark.addCase("codegen = T, hugeMethodLimit = 1500") { _ =>
         withSQLConf(
           SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true",
           SQLConf.WHOLESTAGE_HUGE_METHOD_LIMIT.key -> "1500") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/BloomFilterBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/BloomFilterBenchmark.scala
index ae241b3625d02..f78ccf9569a0f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/BloomFilterBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/BloomFilterBenchmark.scala
@@ -29,7 +29,8 @@ import org.apache.spark.benchmark.Benchmark
  *
  * To run this benchmark:
  * {{{
- *   1. without sbt: bin/spark-submit --class <this class> <spark sql test jar>
+ *   1. without sbt: bin/spark-submit --class <this class>
+ *     --jars <spark core test jar>,<spark catalyst test jar> <spark sql test jar>
  *   2. build/sbt "sql/test:runMain <this class>"
  *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
  *      Results will be written to "benchmarks/BloomFilterBenchmark-results.txt".
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/BuiltInDataSourceWriteBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/BuiltInDataSourceWriteBenchmark.scala
index 6925bdd72674f..361deb0d3e3b6 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/BuiltInDataSourceWriteBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/BuiltInDataSourceWriteBenchmark.scala
@@ -31,7 +31,8 @@ import org.apache.spark.sql.internal.SQLConf
  *
  *   To measure specified formats, run it with arguments.
  *   1. without sbt:
- *        bin/spark-submit --class <this class> <spark sql test jar> format1 [format2] [...]
+ *        bin/spark-submit --class <this class> --jars <spark core test jar>,
+ *        <spark catalyst test jar> <spark sql test jar> format1 [format2] [...]
  *   2. build/sbt "sql/test:runMain <this class> format1 [format2] [...]"
  *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt
  *        "sql/test:runMain <this class> format1 [format2] [...]"
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/CharVarcharBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/CharVarcharBenchmark.scala
new file mode 100644
index 0000000000000..112fd98d7016a
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/CharVarcharBenchmark.scala
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.benchmark
+
+import org.apache.spark.benchmark.Benchmark
+
+/**
+ * Benchmark for measure writing and reading char/varchar values with implicit length check
+ * and padding.
+ * To run this benchmark:
+ * {{{
+ *   1. without sbt:
+ *      bin/spark-submit --class <this class>
+ *        --jars <spark core test jar>,<spark catalyst test jar> <sql core test jar>
+ *   2. build/sbt "sql/test:runMain <this class>"
+ *   3. generate result:
+ *      SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
+ *      Results will be written to "benchmarks/CharVarcharBenchmark-results.txt".
+ * }}}
+ */
+object CharVarcharBenchmark extends SqlBasedBenchmark {
+  import spark.implicits._
+
+  private def withTable(tableNames: String*)(f: => Unit): Unit = {
+    try f finally {
+      tableNames.foreach { name =>
+        spark.sql(s"DROP TABLE IF EXISTS $name")
+      }
+    }
+  }
+
+  private def createTable(tblName: String, colType: String, path: String): Unit = {
+    spark.sql(s"CREATE TABLE $tblName (c $colType) USING PARQUET LOCATION '$path'")
+  }
+
+  def writeBenchmark(card: Long, length: Int, hasSpaces: Boolean): Unit = {
+    withTempPath { dir =>
+      val path = dir.getCanonicalPath
+      val benchmark =
+        new Benchmark(s"Write with length $length", card, output = output)
+      Seq("string", "char", "varchar").foreach { typ =>
+        val colType = if (typ == "string") typ else s"$typ($length)"
+        val tblName = s"${typ}_${length}_$card"
+
+        benchmark.addCase(s"write $typ with length $length", 3) { _ =>
+          withTable(tblName) {
+            createTable(tblName, colType, path)
+            spark.range(card).map { _ =>
+              if (hasSpaces) {
+                "st" + " " * length
+              } else {
+                "st"
+              }
+            }.write.insertInto(tblName)
+          }
+        }
+      }
+      benchmark.run()
+    }
+  }
+
+  override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
+    val N = 200L * 1000 * 1000
+    val range = Range(20, 101, 20)
+
+    runBenchmark("Char Varchar Write Side Perf w/o Tailing Spaces") {
+      for (len <- Seq(5, 10) ++ range) {
+        writeBenchmark(N / len, len, hasSpaces = false)
+      }
+    }
+
+    runBenchmark("Char Varchar Write Side Perf w/ Tailing Spaces") {
+      for (len <- Seq(5, 10) ++ range) {
+        // in write side length check, we only visit the last few spaces
+        writeBenchmark(N / len, len, hasSpaces = true)
+      }
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
index f56efa3bba600..4e42330088da5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
@@ -21,8 +21,10 @@ import java.sql.{Date, Timestamp}
 import java.time.{Instant, LocalDate}
 
 import org.apache.spark.benchmark.Benchmark
+import org.apache.spark.sql.Dataset
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.MILLIS_PER_DAY
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, LA}
+import org.apache.spark.sql.execution.HiveResult
 import org.apache.spark.sql.internal.SQLConf
 
 /**
@@ -30,7 +32,8 @@ import org.apache.spark.sql.internal.SQLConf
  * To run this benchmark:
  * {{{
  *   1. without sbt:
- *      bin/spark-submit --class <this class> --jars <spark core test jar> <sql core test jar>
+ *      bin/spark-submit --class <this class>
+ *        --jars <spark core test jar>,<spark catalyst test jar> <sql core test jar>
  *   2. build/sbt "sql/test:runMain <this class>"
  *   3. generate result:
  *      SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
@@ -52,7 +55,7 @@ object DateTimeBenchmark extends SqlBasedBenchmark {
 
   private def run(cardinality: Int, func: String): Unit = {
     codegenBenchmark(s"$func of timestamp", cardinality) {
-      doBenchmark(cardinality, s"$func(cast(id as timestamp))")
+      doBenchmark(cardinality, s"$func(timestamp_seconds(id))")
     }
   }
 
@@ -62,7 +65,7 @@ object DateTimeBenchmark extends SqlBasedBenchmark {
         val N = 10000000
         runBenchmark("datetime +/- interval") {
           val benchmark = new Benchmark("datetime +/- interval", N, output = output)
-          val ts = "cast(id as timestamp)"
+          val ts = "timestamp_seconds(id)"
           val dt = s"cast($ts as date)"
           benchmark.addCase("date + interval(m)") { _ =>
             doBenchmark(N, s"$dt + interval 1 month")
@@ -103,7 +106,7 @@ object DateTimeBenchmark extends SqlBasedBenchmark {
           benchmark.run()
         }
         runBenchmark("Extract components") {
-          run(N, "cast to timestamp", "cast(id as timestamp)")
+          run(N, "cast to timestamp", "timestamp_seconds(id)")
           run(N, "year")
           run(N, "quarter")
           run(N, "month")
@@ -122,7 +125,7 @@ object DateTimeBenchmark extends SqlBasedBenchmark {
           run(N, "current_timestamp", "current_timestamp")
         }
         runBenchmark("Date arithmetic") {
-          val dateExpr = "cast(cast(id as timestamp) as date)"
+          val dateExpr = "cast(timestamp_seconds(id) as date)"
           run(N, "cast to date", dateExpr)
           run(N, "last_day", s"last_day($dateExpr)")
           run(N, "next_day", s"next_day($dateExpr, 'TU')")
@@ -131,31 +134,31 @@ object DateTimeBenchmark extends SqlBasedBenchmark {
           run(N, "add_months", s"add_months($dateExpr, 10)")
         }
         runBenchmark("Formatting dates") {
-          val dateExpr = "cast(cast(id as timestamp) as date)"
+          val dateExpr = "cast(timestamp_seconds(id) as date)"
           run(N, "format date", s"date_format($dateExpr, 'MMM yyyy')")
         }
         runBenchmark("Formatting timestamps") {
           run(N, "from_unixtime", "from_unixtime(id, 'yyyy-MM-dd HH:mm:ss.SSSSSS')")
         }
         runBenchmark("Convert timestamps") {
-          val timestampExpr = "cast(id as timestamp)"
+          val timestampExpr = "timestamp_seconds(id)"
           run(N, "from_utc_timestamp", s"from_utc_timestamp($timestampExpr, 'CET')")
           run(N, "to_utc_timestamp", s"to_utc_timestamp($timestampExpr, 'CET')")
         }
         runBenchmark("Intervals") {
-          val (start, end) = ("cast(id as timestamp)", "cast((id+8640000) as timestamp)")
+          val (start, end) = ("timestamp_seconds(id)", "timestamp_seconds(id+8640000)")
           run(N, "cast interval", start, end)
           run(N, "datediff", s"datediff($start, $end)")
           run(N, "months_between", s"months_between($start, $end)")
           run(1000000, "window", s"window($start, 100, 10, 1)")
         }
         runBenchmark("Truncation") {
-          val timestampExpr = "cast(id as timestamp)"
+          val timestampExpr = "timestamp_seconds(id)"
           Seq("YEAR", "YYYY", "YY", "MON", "MONTH", "MM", "DAY", "DD", "HOUR", "MINUTE",
             "SECOND", "WEEK", "QUARTER").foreach { level =>
             run(N, s"date_trunc $level", s"date_trunc('$level', $timestampExpr)")
           }
-          val dateExpr = "cast(cast(id as timestamp) as date)"
+          val dateExpr = "cast(timestamp_seconds(id) as date)"
           Seq("year", "yyyy", "yy", "mon", "month", "mm").foreach { level =>
             run(N, s"trunc $level", s"trunc('$level', $dateExpr)")
           }
@@ -182,14 +185,19 @@ object DateTimeBenchmark extends SqlBasedBenchmark {
           benchmark.addCase("From java.time.LocalDate", numIters) { _ =>
             spark.range(rowsNum).map(millis => LocalDate.ofEpochDay(millis / MILLIS_PER_DAY)).noop()
           }
+          def dates = {
+            spark.range(0, rowsNum, 1, 1).map(millis => new Date(millis))
+          }
           benchmark.addCase("Collect java.sql.Date", numIters) { _ =>
-            spark.range(0, rowsNum, 1, 1).map(millis => new Date(millis)).collect()
+            dates.collect()
+          }
+          def localDates = {
+            spark.range(0, rowsNum, 1, 1)
+              .map(millis => LocalDate.ofEpochDay(millis / MILLIS_PER_DAY))
           }
           benchmark.addCase("Collect java.time.LocalDate", numIters) { _ =>
             withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
-              spark.range(0, rowsNum, 1, 1)
-                .map(millis => LocalDate.ofEpochDay(millis / MILLIS_PER_DAY))
-                .collect()
+              localDates.collect()
             }
           }
           benchmark.addCase("From java.sql.Timestamp", numIters) { _ =>
@@ -202,14 +210,37 @@ object DateTimeBenchmark extends SqlBasedBenchmark {
             spark.range(0, rowsNum, 1, 1)
               .collect()
           }
+          def timestamps = {
+            spark.range(0, rowsNum, 1, 1).map(millis => new Timestamp(millis))
+          }
           benchmark.addCase("Collect java.sql.Timestamp", numIters) { _ =>
-            spark.range(0, rowsNum, 1, 1).map(millis => new Timestamp(millis)).collect()
+            timestamps.collect()
+          }
+          def instants = {
+            spark.range(0, rowsNum, 1, 1).map(millis => Instant.ofEpochMilli(millis))
           }
           benchmark.addCase("Collect java.time.Instant", numIters) { _ =>
             withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
-              spark.range(0, rowsNum, 1, 1)
-                .map(millis => Instant.ofEpochMilli(millis))
-                .collect()
+              instants.collect()
+            }
+          }
+          def toHiveString(df: Dataset[_]): Unit = {
+            HiveResult.hiveResultString(df.queryExecution.executedPlan)
+          }
+          benchmark.addCase("java.sql.Date to Hive string", numIters) { _ =>
+            toHiveString(dates)
+          }
+          benchmark.addCase("java.time.LocalDate to Hive string", numIters) { _ =>
+            withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
+              toHiveString(localDates)
+            }
+          }
+          benchmark.addCase("java.sql.Timestamp to Hive string", numIters) { _ =>
+            toHiveString(timestamps)
+          }
+          benchmark.addCase("java.time.Instant to Hive string", numIters) { _ =>
+            withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
+              toHiveString(instants)
             }
           }
           benchmark.run()
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
index d6167f98b5a51..090ef34e1cf0d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
@@ -25,8 +25,7 @@ import org.apache.spark.sql.DataFrame
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.SECONDS_PER_DAY
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, LA}
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy._
-import org.apache.spark.sql.internal.SQLConf.ParquetOutputTimestampType
+import org.apache.spark.sql.internal.SQLConf.{LegacyBehaviorPolicy, ParquetOutputTimestampType}
 
 object DateTime extends Enumeration {
   type DateTime = Value
@@ -38,7 +37,8 @@ object DateTime extends Enumeration {
  * To run this benchmark:
  * {{{
  *   1. without sbt:
- *      bin/spark-submit --class <this class> --jars <spark core test jar> <sql core test jar>
+ *      bin/spark-submit --class <this class>
+ *        --jars <spark core test jar>,<spark catalyst test jar> <sql core test jar>
  *   2. build/sbt "sql/test:runMain <this class>"
  *   3. generate result:
  *      SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
@@ -118,12 +118,12 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
   private def caseName(
       modernDates: Boolean,
       dateTime: DateTime,
-      rebase: Option[Boolean] = None,
+      mode: Option[LegacyBehaviorPolicy.Value] = None,
       vec: Option[Boolean] = None): String = {
     val period = if (modernDates) "after" else "before"
     val year = if (dateTime == DATE) 1582 else 1900
     val vecFlag = vec.map(flagToStr).map(flag => s", vec $flag").getOrElse("")
-    val rebaseFlag = rebase.map(flagToStr).map(flag => s", rebase $flag").getOrElse("")
+    val rebaseFlag = mode.map(_.toString).map(m => s", rebase $m").getOrElse("")
     s"$period $year$vecFlag$rebaseFlag"
   }
 
@@ -131,10 +131,10 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
       basePath: File,
       dateTime: DateTime,
       modernDates: Boolean,
-      rebase: Option[Boolean] = None): String = {
+      mode: Option[LegacyBehaviorPolicy.Value] = None): String = {
     val period = if (modernDates) "after" else "before"
     val year = if (dateTime == DATE) 1582 else 1900
-    val rebaseFlag = rebase.map(flagToStr).map(flag => s"_$flag").getOrElse("")
+    val rebaseFlag = mode.map(_.toString).map(m => s"_$m").getOrElse("")
     basePath.getAbsolutePath + s"/${dateTime}_${period}_$year$rebaseFlag"
   }
 
@@ -160,17 +160,19 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
                 output = output)
               benchmarkInputs(benchmark, rowsNum, dateTime)
               Seq(true, false).foreach { modernDates =>
-                Seq(false, true).foreach { rebase =>
-                  benchmark.addCase(caseName(modernDates, dateTime, Some(rebase)), 1) { _ =>
-                    val mode = if (rebase) LEGACY else CORRECTED
+                LegacyBehaviorPolicy.values
+                  .filterNot(v => !modernDates && v == LegacyBehaviorPolicy.EXCEPTION)
+                  .foreach { mode =>
+                  benchmark.addCase(caseName(modernDates, dateTime, Some(mode)), 1) { _ =>
                     withSQLConf(
                       SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> getOutputType(dateTime),
-                      SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> mode.toString) {
+                      SQLConf.PARQUET_REBASE_MODE_IN_WRITE.key -> mode.toString,
+                      SQLConf.PARQUET_INT96_REBASE_MODE_IN_WRITE.key -> mode.toString) {
                       genDF(rowsNum, dateTime, modernDates)
                         .write
                         .mode("overwrite")
                         .format("parquet")
-                        .save(getPath(path, dateTime, modernDates, Some(rebase)))
+                        .save(getPath(path, dateTime, modernDates, Some(mode)))
                     }
                   }
                 }
@@ -181,13 +183,15 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
                 s"Load $dateTime from parquet", rowsNum, output = output)
               Seq(true, false).foreach { modernDates =>
                 Seq(false, true).foreach { vec =>
-                  Seq(false, true).foreach { rebase =>
-                    val name = caseName(modernDates, dateTime, Some(rebase), Some(vec))
+                  LegacyBehaviorPolicy.values
+                    .filterNot(v => !modernDates && v == LegacyBehaviorPolicy.EXCEPTION)
+                    .foreach { mode =>
+                    val name = caseName(modernDates, dateTime, Some(mode), Some(vec))
                     benchmark2.addCase(name, 3) { _ =>
                       withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vec.toString) {
                         spark.read
                           .format("parquet")
-                          .load(getPath(path, dateTime, modernDates, Some(rebase)))
+                          .load(getPath(path, dateTime, modernDates, Some(mode)))
                           .noop()
                       }
                     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ExtractBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ExtractBenchmark.scala
index 287854dc3646c..6af20e8696aae 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ExtractBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ExtractBenchmark.scala
@@ -27,7 +27,8 @@ import org.apache.spark.sql.internal.SQLConf
  * To run this benchmark:
  * {{{
  *   1. without sbt:
- *      bin/spark-submit --class <this class> --jars <spark core test jar> <sql core test jar>
+ *      bin/spark-submit --class <this class>
+ *        --jars <spark core test jar>,<spark catalyst test jar> <sql core test jar>
  *   2. build/sbt "sql/test:runMain <this class>"
  *   3. generate result:
  *      SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
@@ -38,7 +39,9 @@ object ExtractBenchmark extends SqlBasedBenchmark {
 
   private def doBenchmark(cardinality: Long, exprs: String*): Unit = {
     val sinceSecond = Instant.parse("2010-01-01T00:00:00Z").getEpochSecond
-    withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true") {
+    withSQLConf(
+      SQLConf.LEGACY_INTERVAL_ENABLED.key -> "true",
+      SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true") {
       spark
         .range(sinceSecond, sinceSecond + cardinality, 1, 1)
         .selectExpr(exprs: _*)
@@ -59,10 +62,10 @@ object ExtractBenchmark extends SqlBasedBenchmark {
   }
 
   private def castExpr(from: String): String = from match {
-    case "timestamp" => "cast(id as timestamp)"
-    case "date" => "cast(cast(id as timestamp) as date)"
-    case "interval" => "(cast(cast(id as timestamp) as date) - date'0001-01-01') + " +
-      "(cast(id as timestamp) - timestamp'1000-01-01 01:02:03.123456')"
+    case "timestamp" => "timestamp_seconds(id)"
+    case "date" => "cast(timestamp_seconds(id) as date)"
+    case "interval" => "(cast(timestamp_seconds(id) as date) - date'0001-01-01') + " +
+      "(timestamp_seconds(id) - timestamp'1000-01-01 01:02:03.123456')"
     case other => throw new IllegalArgumentException(
       s"Unsupported column type $other. Valid column types are 'timestamp' and 'date'")
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/FilterPushdownBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/FilterPushdownBenchmark.scala
index b3f65d40ad95b..2bd03b6cb758b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/FilterPushdownBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/FilterPushdownBenchmark.scala
@@ -24,16 +24,17 @@ import scala.util.Random
 import org.apache.spark.SparkConf
 import org.apache.spark.benchmark.Benchmark
 import org.apache.spark.sql.{DataFrame, SparkSession}
-import org.apache.spark.sql.functions.monotonically_increasing_id
+import org.apache.spark.sql.functions.{monotonically_increasing_id, timestamp_seconds}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.ParquetOutputTimestampType
-import org.apache.spark.sql.types.{ByteType, Decimal, DecimalType, TimestampType}
+import org.apache.spark.sql.types.{ByteType, Decimal, DecimalType}
 
 /**
  * Benchmark to measure read performance with Filter pushdown.
  * To run this benchmark:
  * {{{
- *   1. without sbt: bin/spark-submit --class <this class> <spark sql test jar>
+ *   1. without sbt: bin/spark-submit --class <this class>
+ *      --jars <spark core test jar>,<spark catalyst test jar> <spark sql test jar>
  *   2. build/sbt "sql/test:runMain <this class>"
  *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
  *      Results will be written to "benchmarks/FilterPushdownBenchmark-results.txt".
@@ -332,11 +333,11 @@ object FilterPushdownBenchmark extends SqlBasedBenchmark {
             withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> fileType) {
               val columns = (1 to width).map(i => s"CAST(id AS string) c$i")
               val df = spark.range(numRows).selectExpr(columns: _*)
-                .withColumn("value", monotonically_increasing_id().cast(TimestampType))
+                .withColumn("value", timestamp_seconds(monotonically_increasing_id()))
               withTempTable("orcTable", "parquetTable") {
                 saveAsTable(df, dir)
 
-                Seq(s"value = CAST($mid AS timestamp)").foreach { whereExpr =>
+                Seq(s"value = timestamp_seconds($mid)").foreach { whereExpr =>
                   val title = s"Select 1 timestamp stored as $fileType row ($whereExpr)"
                     .replace("value AND value", "value")
                   filterPushDownBenchmark(numRows, title, whereExpr)
@@ -348,8 +349,8 @@ object FilterPushdownBenchmark extends SqlBasedBenchmark {
                   filterPushDownBenchmark(
                     numRows,
                     s"Select $percent% timestamp stored as $fileType rows " +
-                      s"(value < CAST(${numRows * percent / 100} AS timestamp))",
-                    s"value < CAST(${numRows * percent / 100} as timestamp)",
+                      s"(value < timestamp_seconds(${numRows * percent / 100}))",
+                    s"value < timestamp_seconds(${numRows * percent / 100})",
                     selectExpr
                   )
                 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/GenerateExecBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/GenerateExecBenchmark.scala
new file mode 100644
index 0000000000000..97bf95090f104
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/GenerateExecBenchmark.scala
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.benchmark
+
+import org.apache.spark.sql.functions.explode
+
+/**
+ * Benchmark to measure performance for generate exec operator.
+ * To run this benchmark:
+ * {{{
+ *   1. without sbt:
+ *      bin/spark-submit --class <this class>
+ *        --jars <spark core test jar>,<spark catalyst test jar> <sql core test jar>
+ *   2. build/sbt "sql/test:runMain <this class>"
+ *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
+ *      Results will be written to "benchmarks/GenerateExecBenchmark-results.txt".
+ * }}}
+ */
+case class Data(value1: Float, value2: Map[String, String], value3: String)
+
+object GenerateExecBenchmark extends SqlBasedBenchmark {
+
+  override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
+    runBenchmark("GenerateExec benchmark") {
+      import spark.implicits._
+      val numRecords = 100000000
+      codegenBenchmark("GenerateExec Benchmark", numRecords) {
+        val srcDF = spark.range(numRecords).map {
+          x => Data(x.toFloat, Map(x.toString -> x.toString), s"value3$x")
+        }.select($"value1", explode($"value2"), $"value3")
+        srcDF.noop()
+      }
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/HashedRelationMetricsBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/HashedRelationMetricsBenchmark.scala
index f3647b3bb2631..f03c22aecfce3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/HashedRelationMetricsBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/HashedRelationMetricsBenchmark.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.sql.execution.benchmark
 
-import org.scalatest.Assertions._
-
 import org.apache.spark.SparkConf
 import org.apache.spark.benchmark.Benchmark
 import org.apache.spark.internal.config.MEMORY_OFFHEAP_ENABLED
@@ -32,7 +30,8 @@ import org.apache.spark.sql.types.LongType
  * Benchmark to measure metrics performance at HashedRelation.
  * To run this benchmark:
  * {{{
- *   1. without sbt: bin/spark-submit --class <this class> <spark sql test jar>
+ *   1. without sbt: bin/spark-submit --class <this class>
+ *      --jars <spark core test jar>,<spark catalyst test jar> <spark sql test jar>
  *   2. build/sbt "sql/test:runMain <this class>"
  *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
  *      Results will be written to "benchmarks/HashedRelationMetricsBenchmark-results.txt".
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/InExpressionBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/InExpressionBenchmark.scala
index caf3387875813..f5a3483bdf3ce 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/InExpressionBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/InExpressionBenchmark.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.execution.benchmark
 
 import org.apache.spark.benchmark.Benchmark
 import org.apache.spark.sql.DataFrame
-import org.apache.spark.sql.functions.{array, struct}
+import org.apache.spark.sql.functions.{array, struct, timestamp_seconds}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
@@ -31,7 +31,9 @@ import org.apache.spark.sql.types._
  *
  * To run this benchmark:
  * {{{
- *   1. without sbt: bin/spark-submit --class <this class> <spark sql test jar>
+ *   1. without sbt:
+ *      bin/spark-submit --class <this class>
+ *        --jars <spark core test jar>,<spark catalyst test jar> <spark sql test jar>
  *   2. build/sbt "sql/test:runMain <this class>"
  *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
  *      Results will be written to "benchmarks/InExpressionBenchmark-results.txt".
@@ -128,15 +130,15 @@ object InExpressionBenchmark extends SqlBasedBenchmark {
 
   private def runTimestampBenchmark(numItems: Int, numRows: Long, minNumIters: Int): Unit = {
     val name = s"$numItems timestamps"
-    val values = (1 to numItems).map(m => s"CAST('1970-01-01 01:00:00.$m' AS timestamp)")
-    val df = spark.range(0, numRows).select($"id".cast(TimestampType))
+    val values = (1 to numItems).map(m => s"timestamp'1970-01-01 01:00:00.$m'")
+    val df = spark.range(0, numRows).select(timestamp_seconds($"id").as("id"))
     runBenchmark(name, df, values, numRows, minNumIters)
   }
 
   private def runDateBenchmark(numItems: Int, numRows: Long, minNumIters: Int): Unit = {
     val name = s"$numItems dates"
-    val values = (1 to numItems).map(n => 1970 + n).map(y => s"CAST('$y-01-01' AS date)")
-    val df = spark.range(0, numRows).select($"id".cast(TimestampType).cast(DateType))
+    val values = (1 to numItems).map(n => 1970 + n).map(y => s"date'$y-01-01'")
+    val df = spark.range(0, numRows).select(timestamp_seconds($"id").cast(DateType).as("id"))
     runBenchmark(name, df, values, numRows, minNumIters)
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/InsertTableWithDynamicPartitionsBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/InsertTableWithDynamicPartitionsBenchmark.scala
new file mode 100644
index 0000000000000..32bc2001dcd08
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/InsertTableWithDynamicPartitionsBenchmark.scala
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.benchmark
+
+import org.apache.spark.benchmark.Benchmark
+
+/**
+ * Benchmark to measure insert into table with dynamic partition columns.
+ * To run this benchmark:
+ * {{{
+ *   1. without sbt:
+ *      bin/spark-submit --class <this class>
+ *        --jars <spark core test jar>,<spark catalyst test jar> < spark sql test jar>
+ *   2. build/sbt "sql/test:runMain <this class>"
+ *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
+ *      Results will be written to
+ *      "benchmarks/InsertTableWithDynamicPartitionsBenchmark-results.txt".
+ * }}}
+ */
+object InsertTableWithDynamicPartitionsBenchmark extends DataSourceWriteBenchmark {
+
+  def prepareSourceTableAndGetTotalRows(numberRows: Long, sourceTable: String,
+      part1Step: Int, part2Step: Int, part3Step: Int): Long = {
+    val dataFrame = spark.range(0, numberRows, 1, 4)
+    val dataFrame1 = spark.range(0, numberRows, part1Step, 4)
+    val dataFrame2 = spark.range(0, numberRows, part2Step, 4)
+    val dataFrame3 = spark.range(0, numberRows, part3Step, 4)
+
+    val data = dataFrame.join(dataFrame1).join(dataFrame2).join(dataFrame3)
+      .toDF("id", "part1", "part2", "part3")
+    data.write.saveAsTable(sourceTable)
+    data.count()
+  }
+
+  def writeOnePartitionColumnTable(tableName: String,
+      partitionNumber: Long, benchmark: Benchmark): Unit = {
+    spark.sql(s"create table $tableName(i bigint, part bigint) " +
+      "using parquet partitioned by (part)")
+    benchmark.addCase(s"one partition column, $partitionNumber partitions") { _ =>
+      spark.sql(s"insert overwrite table $tableName partition(part) " +
+        "select id, part1 as part from sourceTable")
+    }
+  }
+
+  def writeTwoPartitionColumnTable(tableName: String,
+      partitionNumber: Long, benchmark: Benchmark): Unit = {
+    spark.sql(s"create table $tableName(i bigint, part1 bigint, part2 bigint) " +
+      "using parquet partitioned by (part1, part2)")
+    benchmark.addCase(s"two partition columns, $partitionNumber partitions") { _ =>
+      spark.sql(s"insert overwrite table $tableName partition(part1, part2) " +
+        "select id, part1, part2 from sourceTable")
+    }
+  }
+
+  def writeThreePartitionColumnTable(tableName: String,
+      partitionNumber: Long, benchmark: Benchmark): Unit = {
+    spark.sql(s"create table $tableName(i bigint, part1 bigint, part2 bigint, part3 bigint) " +
+      "using parquet partitioned by (part1, part2, part3)")
+    benchmark.addCase(s"three partition columns, $partitionNumber partitions") { _ =>
+      spark.sql(s"insert overwrite table $tableName partition(part1, part2, part3) " +
+        "select id, part1, part2, part3 from sourceTable")
+    }
+  }
+
+  override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
+    val sourceTable = "sourceTable"
+    val onePartColTable = "onePartColTable"
+    val twoPartColTable = "twoPartColTable"
+    val threePartColTable = "threePartColTable"
+    val numberRows = 100L
+    val part1Step = 1
+    val part2Step = 20
+    val part3Step = 25
+    val part1Number = numberRows / part1Step
+    val part2Number = numberRows / part2Step *  part1Number
+    val part3Number = numberRows / part3Step *  part2Number
+
+    withTable(sourceTable, onePartColTable, twoPartColTable, threePartColTable) {
+      val totalRows =
+        prepareSourceTableAndGetTotalRows(numberRows, sourceTable, part1Step, part2Step, part3Step)
+      val benchmark =
+        new Benchmark(s"dynamic insert table benchmark, totalRows = $totalRows",
+          totalRows, output = output)
+      writeOnePartitionColumnTable(onePartColTable, part1Number, benchmark)
+      writeTwoPartitionColumnTable(twoPartColTable, part2Number, benchmark)
+      writeThreePartitionColumnTable(threePartColTable, part3Number, benchmark)
+      benchmark.run()
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/IntervalBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/IntervalBenchmark.scala
index 907e3f40c1911..a9696e6718de8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/IntervalBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/IntervalBenchmark.scala
@@ -39,11 +39,11 @@ import org.apache.spark.sql.internal.SQLConf
 object IntervalBenchmark extends SqlBasedBenchmark {
   import spark.implicits._
 
-  private def doBenchmark(cardinality: Long, exprs: Column*): Unit = {
+  private def doBenchmark(cardinality: Long, columns: Column*): Unit = {
     withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true") {
       spark
         .range(0, cardinality, 1, 1)
-        .select(exprs: _*)
+        .select(columns: _*)
         .queryExecution
         .toRdd
         .foreach(_ => ())
@@ -60,6 +60,26 @@ object IntervalBenchmark extends SqlBasedBenchmark {
     }
   }
 
+  private def doBenchmarkExpr(cardinality: Long, exprs: String*): Unit = {
+    withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true") {
+      spark
+        .range(0, cardinality, 1, 1)
+        .selectExpr(exprs: _*)
+        .queryExecution
+        .toRdd
+        .foreach(_ => ())
+    }
+  }
+
+  private def addCaseExpr(
+      benchmark: Benchmark,
+      cardinality: Long,
+      name: String,
+      exprs: String*): Unit = {
+    benchmark.addCase(name, numIters = 3) { _ => doBenchmarkExpr(cardinality, exprs: _*) }
+  }
+
+
   private def buildString(withPrefix: Boolean, units: Seq[String] = Seq.empty): Column = {
     val init = lit(if (withPrefix) "interval" else "") ::
       ($"id" % 10000).cast("string") ::
@@ -78,25 +98,68 @@ object IntervalBenchmark extends SqlBasedBenchmark {
     }
   }
 
-  override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
-    val N = 1000000
+  private def benchmarkIntervalStringParsing(cardinality: Long): Unit = {
     val timeUnits = Seq(
       "13 months", "                      1                     months",
       "100 weeks", "9 days", "12 hours", "-                    3 hours",
       "5 minutes", "45 seconds", "123 milliseconds", "567 microseconds")
     val intervalToTest = ListBuffer[String]()
 
-    val benchmark = new Benchmark("cast strings to intervals", N, output = output)
+    val benchmark = new Benchmark("cast strings to intervals", cardinality, output = output)
     // The first 2 cases are used to show the overhead of preparing the interval string.
-    addCase(benchmark, N, "prepare string w/ interval", buildString(true, timeUnits))
-    addCase(benchmark, N, "prepare string w/o interval", buildString(false, timeUnits))
-    addCase(benchmark, N, intervalToTest) // Only years
+    addCase(benchmark, cardinality, "prepare string w/ interval", buildString(true, timeUnits))
+    addCase(benchmark, cardinality, "prepare string w/o interval", buildString(false, timeUnits))
+    addCase(benchmark, cardinality, intervalToTest.toSeq) // Only years
 
     for (unit <- timeUnits) {
       intervalToTest.append(unit)
-      addCase(benchmark, N, intervalToTest)
+      addCase(benchmark, cardinality, intervalToTest.toSeq)
     }
 
     benchmark.run()
   }
+
+  private def benchmarkMakeInterval(cardinality: Long): Unit = {
+    val benchmark = new Benchmark("make_interval()", cardinality, output = output)
+    val hmExprs = Seq("id % 24", "id % 60")
+    val hmsExprs = hmExprs ++ Seq("cast((id % 500000000) / 1000000.0 as decimal(18, 6))")
+    val ymExprs = Seq("(2000 + (id % 30))", "((id % 12) + 1)")
+    val wdExpr = Seq("((id % 54) + 1)", "((id % 1000) + 1)")
+    val args = ymExprs ++ wdExpr ++ hmsExprs
+
+    addCaseExpr(
+      benchmark,
+      cardinality,
+      "prepare make_interval()",
+      args: _*)
+    val foldableExpr = "make_interval(0, 1, 2, 3, 4, 5, 50.123456)"
+    addCaseExpr(benchmark, cardinality, foldableExpr, foldableExpr)
+    addCaseExpr(
+      benchmark,
+      cardinality,
+      "make_interval(*, *, 2, 3, 4, 5, 50.123456)",
+      s"make_interval(${ymExprs.mkString(",")}, 2, 3, 4, 5, 50.123456)")
+    addCaseExpr(
+      benchmark,
+      cardinality,
+      "make_interval(0, 1, *, *, 4, 5, 50.123456)",
+      s"make_interval(0, 1, ${wdExpr.mkString(",")}, 4, 5, 50.123456)")
+    addCaseExpr(
+      benchmark,
+      cardinality,
+      "make_interval(0, 1, 2, 3, *, *, *)",
+      s"make_interval(0, 1, 2, 3, ${hmsExprs.mkString(",")})")
+    addCaseExpr(
+      benchmark,
+      cardinality,
+      "make_interval(*, *, *, *, *, *, *)",
+      s"make_interval(${args.mkString(",")})")
+
+    benchmark.run()
+  }
+
+  override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
+    benchmarkIntervalStringParsing(1000000)
+    benchmarkMakeInterval(1000000)
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/JoinBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/JoinBenchmark.scala
index 1cc92892fe122..849c41307245e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/JoinBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/JoinBenchmark.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.sql.execution.benchmark
 
-import org.scalatest.Assertions._
-
 import org.apache.spark.sql.execution.joins._
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
@@ -29,7 +27,8 @@ import org.apache.spark.sql.types.IntegerType
  * To run this benchmark:
  * {{{
  *   1. without sbt:
- *      bin/spark-submit --class <this class> --jars <spark core test jar> <spark sql test jar>
+ *      bin/spark-submit --class <this class>
+ *        --jars <spark core test jar>,<spark catalyst test jar> <spark sql test jar>
  *   2. build/sbt "sql/test:runMain <this class>"
  *   3. generate result:
  *      SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
@@ -166,6 +165,19 @@ object JoinBenchmark extends SqlBasedBenchmark {
     }
   }
 
+  def broadcastNestedLoopJoin(): Unit = {
+    val N = 20 << 20
+    val M = 1 << 4
+
+    val dim = broadcast(spark.range(M).selectExpr("id as k", "cast(id as string) as v"))
+    codegenBenchmark("broadcast nested loop join", N) {
+      val df = spark.range(N).join(dim)
+      assert(df.queryExecution.sparkPlan.find(
+        _.isInstanceOf[BroadcastNestedLoopJoinExec]).isDefined)
+      df.noop()
+    }
+  }
+
   override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
     runBenchmark("Join Benchmark") {
       broadcastHashJoinLongKey()
@@ -178,6 +190,7 @@ object JoinBenchmark extends SqlBasedBenchmark {
       sortMergeJoin()
       sortMergeJoinWithDuplicates()
       shuffleHashJoin()
+      broadcastNestedLoopJoin()
     }
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MakeDateTimeBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MakeDateTimeBenchmark.scala
index c92098c93aa1e..6ab4fcc9c00fe 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MakeDateTimeBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MakeDateTimeBenchmark.scala
@@ -25,7 +25,8 @@ import org.apache.spark.sql.internal.SQLConf
  * To run this benchmark:
  * {{{
  *   1. without sbt:
- *      bin/spark-submit --class <this class> --jars <spark core test jar> <sql core test jar>
+ *      bin/spark-submit --class <this class>
+ *        --jars <spark core test jar>,<spark catalyst test jar> <sql core test jar>
  *   2. build/sbt "sql/test:runMain <this class>"
  *   3. generate result:
  *      SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MiscBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MiscBenchmark.scala
index 2aecf553d75a2..18605468adba7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MiscBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/MiscBenchmark.scala
@@ -24,7 +24,9 @@ import org.apache.spark.sql.internal.SQLConf
  * Benchmark to measure whole stage codegen performance.
  * To run this benchmark:
  * {{{
- *   1. without sbt: bin/spark-submit --class <this class> <spark sql test jar>
+ *   1. without sbt:
+ *      bin/spark-submit --class <this class>
+ *        --jars <spark core test jar>,<spark catalyst test jar> <sql core test jar>
  *   2. build/sbt "sql/test:runMain <this class>"
  *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
  *      Results will be written to "benchmarks/MiscBenchmark-results.txt".
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/OrcNestedSchemaPruningBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/OrcNestedSchemaPruningBenchmark.scala
index eeb2605c47c20..a59da45f1fe17 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/OrcNestedSchemaPruningBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/OrcNestedSchemaPruningBenchmark.scala
@@ -24,7 +24,8 @@ import org.apache.spark.sql.internal.SQLConf
  * To run this benchmark:
  * {{{
  *   1. without sbt:
- *      bin/spark-submit --class <this class> --jars <spark core test jar> <sql core test jar>
+ *      bin/spark-submit --class <this class>
+ *        --jars <spark core test jar>,<spark catalyst test jar> <spark sql test jar>
  *   2. build/sbt "sql/test:runMain <this class>"
  *   3. generate result:
  *      SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/OrcV2NestedSchemaPruningBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/OrcV2NestedSchemaPruningBenchmark.scala
index e735d1cf257d0..d0289df380d98 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/OrcV2NestedSchemaPruningBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/OrcV2NestedSchemaPruningBenchmark.scala
@@ -22,7 +22,8 @@ package org.apache.spark.sql.execution.benchmark
  * To run this benchmark:
  * {{{
  *   1. without sbt:
- *      bin/spark-submit --class <this class> --jars <spark core test jar> <sql core test jar>
+ *      bin/spark-submit --class <this class>
+ *        --jars <spark core test jar>,<spark catalyst test jar> <sql core test jar>
  *   2. build/sbt "sql/test:runMain <this class>"
  *   3. generate result:
  *      SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ParquetNestedPredicatePushDownBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ParquetNestedPredicatePushDownBenchmark.scala
index d2bd962b50654..5604f6856b042 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ParquetNestedPredicatePushDownBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ParquetNestedPredicatePushDownBenchmark.scala
@@ -17,9 +17,8 @@
 
 package org.apache.spark.sql.execution.benchmark
 
-import org.apache.spark.SparkConf
 import org.apache.spark.benchmark.Benchmark
-import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}
+import org.apache.spark.sql.{DataFrame, SaveMode}
 import org.apache.spark.sql.internal.SQLConf
 
 /**
@@ -27,7 +26,8 @@ import org.apache.spark.sql.internal.SQLConf
  * To run this benchmark:
  * {{{
  *   1. without sbt:
- *      bin/spark-submit --class <this class> --jars <spark core test jar> <sql core test jar>
+ *      bin/spark-submit --class <this class>
+ *        --jars <spark core test jar>,<spark catalyst test jar> <sql core test jar>
  *   2. build/sbt "sql/test:runMain <this class>"
  *   3. generate result:
  *      SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ParquetNestedSchemaPruningBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ParquetNestedSchemaPruningBenchmark.scala
index 1c9cc2c371176..f8f0ab10b24ef 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ParquetNestedSchemaPruningBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ParquetNestedSchemaPruningBenchmark.scala
@@ -22,7 +22,8 @@ package org.apache.spark.sql.execution.benchmark
  * To run this benchmark:
  * {{{
  *   1. without sbt:
- *      bin/spark-submit --class <this class> --jars <spark core test jar> <sql core test jar>
+ *      bin/spark-submit --class <this class>
+ *        --jars <spark core test jar>,<spark catalyst test jar> <sql core test jar>
  *   2. build/sbt "sql/test:runMain <this class>"
  *   3. generate result:
  *      SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/PrimitiveArrayBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/PrimitiveArrayBenchmark.scala
index e07921bf3aa74..b09549be167f5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/PrimitiveArrayBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/PrimitiveArrayBenchmark.scala
@@ -23,7 +23,9 @@ import org.apache.spark.sql.SparkSession
 /**
  * Benchmark primitive arrays via DataFrame and Dataset program using primitive arrays
  * To run this benchmark:
- * 1. without sbt: bin/spark-submit --class <this class> <spark sql test jar>
+ * 1. without sbt:
+ *    bin/spark-submit --class <this class>
+ *      --jars <spark core test jar>,<spark catalyst test jar> < spark sql test jar>
  * 2. build/sbt "sql/test:runMain <this class>"
  * 3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
  *    Results will be written to "benchmarks/PrimitiveArrayBenchmark-results.txt".
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/RangeBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/RangeBenchmark.scala
index e566f5d5adee6..e9bdff5853a51 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/RangeBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/RangeBenchmark.scala
@@ -24,7 +24,8 @@ import org.apache.spark.benchmark.Benchmark
  * To run this benchmark:
  * {{{
  *   1. without sbt:
- *      bin/spark-submit --class <this class> --jars <spark core test jar> <spark sql test jar>
+ *      bin/spark-submit --class <this class>
+ *        --jars <spark core test jar>,<spark catalyst test jar> <spark sql test jar>
  *   2. build/sbt "sql/test:runMain <this class>"
  *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
  *      Results will be written to "benchmarks/RangeBenchmark-results.txt".
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/SortBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/SortBenchmark.scala
index 2c9e8a909633c..8b8710d96b5ce 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/SortBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/SortBenchmark.scala
@@ -30,7 +30,8 @@ import org.apache.spark.util.random.XORShiftRandom
  * Benchmark to measure performance for aggregate primitives.
  * {{{
  *   To run this benchmark:
- *   1. without sbt: bin/spark-submit --class <this class> <spark sql test jar>
+ *   1. without sbt:
+ *      bin/spark-submit --class <this class> --jars <spark core test jar> <sql core test jar>
  *   2. build/sbt "sql/test:runMain <this class>"
  *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
  *      Results will be written to "benchmarks/<this class>-results.txt".
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/SqlBasedBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/SqlBasedBenchmark.scala
index 28387dcef125b..f84172278bef6 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/SqlBasedBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/SqlBasedBenchmark.scala
@@ -22,7 +22,9 @@ import org.apache.spark.internal.config.UI.UI_ENABLED
 import org.apache.spark.sql.{Dataset, SparkSession}
 import org.apache.spark.sql.SaveMode.Overwrite
 import org.apache.spark.sql.catalyst.plans.SQLHelper
+import org.apache.spark.sql.functions.lit
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types._
 
 /**
  * Common base trait to run benchmark with the Dataset and DataFrame API.
@@ -66,4 +68,29 @@ trait SqlBasedBenchmark extends BenchmarkBase with SQLHelper {
       ds.write.format("noop").mode(Overwrite).save()
     }
   }
+
+  protected def prepareDataInfo(benchmark: Benchmark): Unit = {
+    // scalastyle:off println
+    benchmark.out.println("Preparing data for benchmarking ...")
+    // scalastyle:on println
+  }
+
+  /**
+   * Prepares a table with wide row for benchmarking. The table will be written into
+   * the given path.
+   */
+  protected  def writeWideRow(path: String, rowsNum: Int, numCols: Int): StructType = {
+    val fields = Seq.tabulate(numCols)(i => StructField(s"col$i", IntegerType))
+    val schema = StructType(fields)
+
+    spark.range(rowsNum)
+      .select(Seq.tabulate(numCols)(i => lit(i).as(s"col$i")): _*)
+      .write.json(path)
+
+    schema
+  }
+
+  override def afterAll(): Unit = {
+    spark.stop()
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmark.scala
index ad3d79760adf0..90d3f96f8b0f5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmark.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql.execution.benchmark
 
+import scala.util.Try
+
 import org.apache.spark.SparkConf
 import org.apache.spark.benchmark.Benchmark
 import org.apache.spark.internal.Logging
@@ -24,21 +26,24 @@ import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.catalog.HiveTableRelation
 import org.apache.spark.sql.catalyst.plans.logical.SubqueryAlias
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.catalyst.util.DateTimeConstants.NANOS_PER_SECOND
 import org.apache.spark.sql.execution.datasources.LogicalRelation
+import org.apache.spark.sql.internal.SQLConf
 
 /**
  * Benchmark to measure TPCDS query performance.
  * To run this:
  * {{{
  *   1. without sbt:
- *        bin/spark-submit --class <this class> <spark sql test jar> --data-location <location>
+ *        bin/spark-submit --jars <spark core test jar>,<spark catalyst test jar>
+ *          --class <this class> <spark sql test jar> --data-location <location>
  *   2. build/sbt "sql/test:runMain <this class> --data-location <TPCDS data location>"
  *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt
  *        "sql/test:runMain <this class> --data-location <location>"
  *      Results will be written to "benchmarks/TPCDSQueryBenchmark-results.txt".
  * }}}
  */
-object TPCDSQueryBenchmark extends SqlBasedBenchmark {
+object TPCDSQueryBenchmark extends SqlBasedBenchmark with Logging {
 
   override def getSparkSession: SparkSession = {
     val conf = new SparkConf()
@@ -60,9 +65,20 @@ object TPCDSQueryBenchmark extends SqlBasedBenchmark {
     "web_returns", "web_site", "reason", "call_center", "warehouse", "ship_mode", "income_band",
     "time_dim", "web_page")
 
-  def setupTables(dataLocation: String): Map[String, Long] = {
+  def setupTables(dataLocation: String, createTempView: Boolean): Map[String, Long] = {
     tables.map { tableName =>
-      spark.read.parquet(s"$dataLocation/$tableName").createOrReplaceTempView(tableName)
+      if (createTempView) {
+        spark.read.parquet(s"$dataLocation/$tableName").createOrReplaceTempView(tableName)
+      } else {
+        spark.sql(s"DROP TABLE IF EXISTS $tableName")
+        spark.catalog.createTable(tableName, s"$dataLocation/$tableName", "parquet")
+        // Recover partitions but don't fail if a table is not partitioned.
+        Try {
+          spark.sql(s"ALTER TABLE $tableName RECOVER PARTITIONS")
+        }.getOrElse {
+          logInfo(s"Recovering partitions of table $tableName failed")
+        }
+      }
       tableName -> spark.table(tableName).count()
     }.toMap
   }
@@ -97,11 +113,16 @@ object TPCDSQueryBenchmark extends SqlBasedBenchmark {
     }
   }
 
-  def filterQueries(
+  private def filterQueries(
       origQueries: Seq[String],
-      args: TPCDSQueryBenchmarkArguments): Seq[String] = {
-    if (args.queryFilter.nonEmpty) {
-      origQueries.filter(args.queryFilter.contains)
+      queryFilter: Set[String],
+      nameSuffix: String = ""): Seq[String] = {
+    if (queryFilter.nonEmpty) {
+      if (nameSuffix.nonEmpty) {
+        origQueries.filter { name => queryFilter.contains(s"$name$nameSuffix") }
+      } else {
+        origQueries.filter(queryFilter.contains)
+      }
     } else {
       origQueries
     }
@@ -124,6 +145,7 @@ object TPCDSQueryBenchmark extends SqlBasedBenchmark {
       "q91", "q92", "q93", "q94", "q95", "q96", "q97", "q98", "q99")
 
     // This list only includes TPC-DS v2.7 queries that are different from v1.4 ones
+    val nameSuffixForQueriesV2_7 = "-v2.7"
     val tpcdsQueriesV2_7 = Seq(
       "q5a", "q6", "q10a", "q11", "q12", "q14", "q14a", "q18a",
       "q20", "q22", "q22a", "q24", "q27a", "q34", "q35", "q35a", "q36a", "q47", "q49",
@@ -131,17 +153,36 @@ object TPCDSQueryBenchmark extends SqlBasedBenchmark {
       "q80a", "q86a", "q98")
 
     // If `--query-filter` defined, filters the queries that this option selects
-    val queriesV1_4ToRun = filterQueries(tpcdsQueries, benchmarkArgs)
-    val queriesV2_7ToRun = filterQueries(tpcdsQueriesV2_7, benchmarkArgs)
+    val queriesV1_4ToRun = filterQueries(tpcdsQueries, benchmarkArgs.queryFilter)
+    val queriesV2_7ToRun = filterQueries(tpcdsQueriesV2_7, benchmarkArgs.queryFilter,
+      nameSuffix = nameSuffixForQueriesV2_7)
 
     if ((queriesV1_4ToRun ++ queriesV2_7ToRun).isEmpty) {
       throw new RuntimeException(
         s"Empty queries to run. Bad query name filter: ${benchmarkArgs.queryFilter}")
     }
 
-    val tableSizes = setupTables(benchmarkArgs.dataLocation)
+    val tableSizes = setupTables(benchmarkArgs.dataLocation,
+      createTempView = !benchmarkArgs.cboEnabled)
+    if (benchmarkArgs.cboEnabled) {
+      spark.sql(s"SET ${SQLConf.CBO_ENABLED.key}=true")
+      spark.sql(s"SET ${SQLConf.PLAN_STATS_ENABLED.key}=true")
+      spark.sql(s"SET ${SQLConf.JOIN_REORDER_ENABLED.key}=true")
+      spark.sql(s"SET ${SQLConf.HISTOGRAM_ENABLED.key}=true")
+
+      // Analyze all the tables before running TPCDS queries
+      val startTime = System.nanoTime()
+      tables.foreach { tableName =>
+        spark.sql(s"ANALYZE TABLE $tableName COMPUTE STATISTICS FOR ALL COLUMNS")
+      }
+      logInfo("The elapsed time to analyze all the tables is " +
+        s"${(System.nanoTime() - startTime) / NANOS_PER_SECOND.toDouble} seconds")
+    } else {
+      spark.sql(s"SET ${SQLConf.CBO_ENABLED.key}=false")
+    }
+
     runTpcdsQueries(queryLocation = "tpcds", queries = queriesV1_4ToRun, tableSizes)
     runTpcdsQueries(queryLocation = "tpcds-v2.7.0", queries = queriesV2_7ToRun, tableSizes,
-      nameSuffix = "-v2.7")
+      nameSuffix = nameSuffixForQueriesV2_7)
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmarkArguments.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmarkArguments.scala
index 184ffff94298a..80a6bffc61ea4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmarkArguments.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmarkArguments.scala
@@ -23,6 +23,7 @@ import java.util.Locale
 class TPCDSQueryBenchmarkArguments(val args: Array[String]) {
   var dataLocation: String = null
   var queryFilter: Set[String] = Set.empty
+  var cboEnabled: Boolean = false
 
   parseArgs(args.toList)
   validateArguments()
@@ -44,6 +45,10 @@ class TPCDSQueryBenchmarkArguments(val args: Array[String]) {
           queryFilter = value.toLowerCase(Locale.ROOT).split(",").map(_.trim).toSet
           args = tail
 
+        case optName :: tail if optionMatch("--cbo", optName) =>
+          cboEnabled = true
+          args = tail
+
         case _ =>
           // scalastyle:off println
           System.err.println("Unknown/unsupported param " + args)
@@ -60,6 +65,7 @@ class TPCDSQueryBenchmarkArguments(val args: Array[String]) {
       |Options:
       |  --data-location      Path to TPCDS data
       |  --query-filter       Queries to filter, e.g., q3,q5,q13
+      |  --cbo                Whether to enable cost-based optimization
       |
       |------------------------------------------------------------------------------------------------------------------
       |In order to run this benchmark, please follow the instructions at
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/UDFBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/UDFBenchmark.scala
index ee8a6e787c36c..34e51efc1f796 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/UDFBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/UDFBenchmark.scala
@@ -28,7 +28,8 @@ import org.apache.spark.sql.types.{IntegerType, StringType}
  * To run this benchmark:
  * {{{
  *   1. without sbt:
- *      bin/spark-submit --class <this class> --jars <spark core test jar> <sql core test jar>
+ *      bin/spark-submit --class <this class>
+ *        --jars <spark core test jar>,<spark catalyst test jar> <sql core test jar>
  *   2. build/sbt "sql/test:runMain <this class>"
  *   3. generate result:
  *      SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/UnsafeArrayDataBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/UnsafeArrayDataBenchmark.scala
index 9b0389c6d1ea4..67eb20c42a9ef 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/UnsafeArrayDataBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/UnsafeArrayDataBenchmark.scala
@@ -27,7 +27,8 @@ import org.apache.spark.sql.catalyst.expressions.UnsafeArrayData
  * Benchmark [[UnsafeArrayDataBenchmark]] for UnsafeArrayData
  * To run this benchmark:
  * {{{
- *   1. without sbt: bin/spark-submit --class <this class> <spark sql test jar>
+ *   1. without sbt:
+ *      bin/spark-submit --class <this class> --jars <spark core test jar> <spark sql test jar>
  *   2. build/sbt "sql/test:runMain <this class>"
  *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
  *      Results will be written to "benchmarks/UnsafeArrayDataBenchmark-results.txt".
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/WideSchemaBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/WideSchemaBenchmark.scala
index 77dc3a10f8033..c209a6da3a3d5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/WideSchemaBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/WideSchemaBenchmark.scala
@@ -19,8 +19,6 @@ package org.apache.spark.sql.execution.benchmark
 
 import java.io.File
 
-import org.scalatest.Assertions._
-
 import org.apache.spark.benchmark.Benchmark
 import org.apache.spark.sql.DataFrame
 import org.apache.spark.util.Utils
@@ -30,7 +28,8 @@ import org.apache.spark.util.Utils
  * To run this benchmark:
  * {{{
  *   1. without sbt:
- *      bin/spark-submit --class <this class> --jars <spark core test jar> <spark sql test jar>
+ *      bin/spark-submit --class <this class>
+ *        --jars <spark core test jar>,<spark catalyst test jar> <spark sql test jar>
  *   2. build/sbt "sql/test:runMain <this class>"
  *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
  *      Results will be written to "benchmarks/WideSchemaBenchmark-results.txt".
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/bucketing/CoalesceBucketsInJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/bucketing/CoalesceBucketsInJoinSuite.scala
new file mode 100644
index 0000000000000..63964665fc81c
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/bucketing/CoalesceBucketsInJoinSuite.scala
@@ -0,0 +1,272 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.bucketing
+
+import org.apache.spark.sql.catalyst.catalog.BucketSpec
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
+import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide}
+import org.apache.spark.sql.catalyst.plans.Inner
+import org.apache.spark.sql.execution.{BinaryExecNode, FileSourceScanExec, SparkPlan}
+import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, InMemoryFileIndex, PartitionSpec}
+import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
+import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, ShuffledHashJoinExec, SortMergeJoinExec}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.test.{SharedSparkSession, SQLTestUtils}
+import org.apache.spark.sql.types.{IntegerType, StructType}
+
+class CoalesceBucketsInJoinSuite extends SQLTestUtils with SharedSparkSession {
+  private val SORT_MERGE_JOIN = "sortMergeJoin"
+  private val SHUFFLED_HASH_JOIN = "shuffledHashJoin"
+  private val BROADCAST_HASH_JOIN = "broadcastHashJoin"
+
+  case class RelationSetting(
+      cols: Seq[Attribute],
+      numBuckets: Int,
+      expectedCoalescedNumBuckets: Option[Int])
+
+  object RelationSetting {
+    def apply(numBuckets: Int, expectedCoalescedNumBuckets: Option[Int]): RelationSetting = {
+      val cols = Seq(AttributeReference("i", IntegerType)())
+      RelationSetting(cols, numBuckets, expectedCoalescedNumBuckets)
+    }
+  }
+
+  case class JoinSetting(
+      leftKeys: Seq[Attribute],
+      rightKeys: Seq[Attribute],
+      leftRelation: RelationSetting,
+      rightRelation: RelationSetting,
+      joinOperator: String,
+      shjBuildSide: Option[BuildSide])
+
+  object JoinSetting {
+    def apply(
+        l: RelationSetting,
+        r: RelationSetting,
+        joinOperator: String = SORT_MERGE_JOIN,
+        shjBuildSide: Option[BuildSide] = None): JoinSetting = {
+      JoinSetting(l.cols, r.cols, l, r, joinOperator, shjBuildSide)
+    }
+  }
+
+  private def newFileSourceScanExec(setting: RelationSetting): FileSourceScanExec = {
+    val relation = HadoopFsRelation(
+      location = new InMemoryFileIndex(spark, Nil, Map.empty, None),
+      partitionSchema = PartitionSpec.emptySpec.partitionColumns,
+      dataSchema = StructType.fromAttributes(setting.cols),
+      bucketSpec = Some(BucketSpec(setting.numBuckets, setting.cols.map(_.name), Nil)),
+      fileFormat = new ParquetFileFormat(),
+      options = Map.empty)(spark)
+    FileSourceScanExec(relation, setting.cols, relation.dataSchema, Nil, None, None, Nil, None)
+  }
+
+  private def run(setting: JoinSetting): Unit = {
+    val swappedSetting = setting.copy(
+      leftKeys = setting.rightKeys,
+      rightKeys = setting.leftKeys,
+      leftRelation = setting.rightRelation,
+      rightRelation = setting.leftRelation)
+
+    val settings = if (setting.joinOperator != SHUFFLED_HASH_JOIN) {
+      Seq(setting, swappedSetting)
+    } else {
+      Seq(setting)
+    }
+    settings.foreach { s =>
+      val lScan = newFileSourceScanExec(s.leftRelation)
+      val rScan = newFileSourceScanExec(s.rightRelation)
+      val join = if (s.joinOperator == SORT_MERGE_JOIN) {
+        SortMergeJoinExec(s.leftKeys, s.rightKeys, Inner, None, lScan, rScan)
+      } else if (s.joinOperator == SHUFFLED_HASH_JOIN) {
+        ShuffledHashJoinExec(s.leftKeys, s.rightKeys, Inner, s.shjBuildSide.get, None, lScan, rScan)
+      } else {
+        BroadcastHashJoinExec(
+          s.leftKeys, s.rightKeys, Inner, BuildLeft, None, lScan, rScan)
+      }
+
+      val plan = CoalesceBucketsInJoin(join)
+
+      def verify(expected: Option[Int], subPlan: SparkPlan): Unit = {
+        val coalesced = subPlan.collect {
+          case f: FileSourceScanExec if f.optionalNumCoalescedBuckets.nonEmpty =>
+            f.optionalNumCoalescedBuckets.get
+        }
+        if (expected.isDefined) {
+          assert(coalesced.size == 1 && coalesced.head == expected.get)
+        } else {
+          assert(coalesced.isEmpty)
+        }
+      }
+
+      verify(s.leftRelation.expectedCoalescedNumBuckets, plan.asInstanceOf[BinaryExecNode].left)
+      verify(s.rightRelation.expectedCoalescedNumBuckets, plan.asInstanceOf[BinaryExecNode].right)
+    }
+  }
+
+  test("bucket coalescing - basic") {
+    withSQLConf(SQLConf.COALESCE_BUCKETS_IN_JOIN_ENABLED.key -> "true") {
+      run(JoinSetting(
+        RelationSetting(4, None), RelationSetting(8, Some(4)), joinOperator = SORT_MERGE_JOIN))
+      run(JoinSetting(
+        RelationSetting(4, None), RelationSetting(8, Some(4)), joinOperator = SHUFFLED_HASH_JOIN,
+        shjBuildSide = Some(BuildLeft)))
+    }
+
+    withSQLConf(SQLConf.COALESCE_BUCKETS_IN_JOIN_ENABLED.key -> "false") {
+      run(JoinSetting(
+        RelationSetting(4, None), RelationSetting(8, None), joinOperator = SORT_MERGE_JOIN))
+      run(JoinSetting(
+        RelationSetting(4, None), RelationSetting(8, None), joinOperator = SHUFFLED_HASH_JOIN,
+        shjBuildSide = Some(BuildLeft)))
+    }
+  }
+
+  test("bucket coalescing should work only for sort merge join and shuffled hash join") {
+    Seq(true, false).foreach { enabled =>
+      withSQLConf(SQLConf.COALESCE_BUCKETS_IN_JOIN_ENABLED.key -> enabled.toString) {
+        run(JoinSetting(
+          RelationSetting(4, None), RelationSetting(8, None), joinOperator = BROADCAST_HASH_JOIN))
+      }
+    }
+  }
+
+  test("bucket coalescing shouldn't be applied to shuffled hash join build side") {
+    withSQLConf(SQLConf.COALESCE_BUCKETS_IN_JOIN_ENABLED.key -> "true") {
+      run(JoinSetting(
+        RelationSetting(4, None), RelationSetting(8, None), joinOperator = SHUFFLED_HASH_JOIN,
+        shjBuildSide = Some(BuildRight)))
+    }
+  }
+
+  test("bucket coalescing shouldn't be applied when the number of buckets are the same") {
+    withSQLConf(SQLConf.COALESCE_BUCKETS_IN_JOIN_ENABLED.key -> "true") {
+      run(JoinSetting(
+        RelationSetting(8, None), RelationSetting(8, None), joinOperator = SORT_MERGE_JOIN))
+      run(JoinSetting(
+        RelationSetting(8, None), RelationSetting(8, None), joinOperator = SHUFFLED_HASH_JOIN,
+        shjBuildSide = Some(BuildLeft)))
+    }
+  }
+
+  test("number of bucket is not divisible by other number of bucket") {
+    withSQLConf(SQLConf.COALESCE_BUCKETS_IN_JOIN_ENABLED.key -> "true") {
+      run(JoinSetting(
+        RelationSetting(3, None), RelationSetting(8, None), joinOperator = SORT_MERGE_JOIN))
+      run(JoinSetting(
+        RelationSetting(3, None), RelationSetting(8, None), joinOperator = SHUFFLED_HASH_JOIN,
+        shjBuildSide = Some(BuildLeft)))
+    }
+  }
+
+  test("the ratio of the number of buckets is greater than max allowed") {
+    withSQLConf(SQLConf.COALESCE_BUCKETS_IN_JOIN_ENABLED.key -> "true",
+      SQLConf.COALESCE_BUCKETS_IN_JOIN_MAX_BUCKET_RATIO.key -> "2") {
+      run(JoinSetting(
+        RelationSetting(4, None), RelationSetting(16, None), joinOperator = SORT_MERGE_JOIN))
+      run(JoinSetting(
+        RelationSetting(4, None), RelationSetting(16, None), joinOperator = SHUFFLED_HASH_JOIN,
+        shjBuildSide = Some(BuildLeft)))
+    }
+  }
+
+  test("join keys should match with output partitioning") {
+    withSQLConf(SQLConf.COALESCE_BUCKETS_IN_JOIN_ENABLED.key -> "true") {
+      val lCols = Seq(
+        AttributeReference("l1", IntegerType)(),
+        AttributeReference("l2", IntegerType)())
+      val rCols = Seq(
+        AttributeReference("r1", IntegerType)(),
+        AttributeReference("r2", IntegerType)())
+
+      val lRel = RelationSetting(lCols, 4, None)
+      val rRel = RelationSetting(rCols, 8, None)
+
+      // The following should not be coalesced because join keys do not match with output
+      // partitioning (missing one expression).
+      run(JoinSetting(
+        leftKeys = Seq(lCols.head),
+        rightKeys = Seq(rCols.head),
+        leftRelation = lRel,
+        rightRelation = rRel,
+        joinOperator = SORT_MERGE_JOIN,
+        shjBuildSide = None))
+
+      run(JoinSetting(
+        leftKeys = Seq(lCols.head),
+        rightKeys = Seq(rCols.head),
+        leftRelation = lRel,
+        rightRelation = rRel,
+        joinOperator = SHUFFLED_HASH_JOIN,
+        shjBuildSide = Some(BuildLeft)))
+
+      // The following should not be coalesced because join keys do not match with output
+      // partitioning (more expressions).
+      run(JoinSetting(
+        leftKeys = lCols :+ AttributeReference("l3", IntegerType)(),
+        rightKeys = rCols :+ AttributeReference("r3", IntegerType)(),
+        leftRelation = lRel,
+        rightRelation = rRel,
+        joinOperator = SORT_MERGE_JOIN,
+        shjBuildSide = None))
+
+      run(JoinSetting(
+        leftKeys = lCols :+ AttributeReference("l3", IntegerType)(),
+        rightKeys = rCols :+ AttributeReference("r3", IntegerType)(),
+        leftRelation = lRel,
+        rightRelation = rRel,
+        joinOperator = SHUFFLED_HASH_JOIN,
+        shjBuildSide = Some(BuildLeft)))
+
+      // The following will be coalesced since ordering should not matter because it will be
+      // adjusted in `EnsureRequirements`.
+      run(JoinSetting(
+        leftKeys = lCols.reverse,
+        rightKeys = rCols.reverse,
+        leftRelation = lRel,
+        rightRelation = RelationSetting(rCols, 8, Some(4)),
+        joinOperator = SORT_MERGE_JOIN,
+        shjBuildSide = None))
+
+      run(JoinSetting(
+        leftKeys = lCols.reverse,
+        rightKeys = rCols.reverse,
+        leftRelation = lRel,
+        rightRelation = RelationSetting(rCols, 8, Some(4)),
+        joinOperator = SHUFFLED_HASH_JOIN,
+        shjBuildSide = Some(BuildLeft)))
+
+      run(JoinSetting(
+        leftKeys = rCols.reverse,
+        rightKeys = lCols.reverse,
+        leftRelation = RelationSetting(rCols, 8, Some(4)),
+        rightRelation = lRel,
+        joinOperator = SHUFFLED_HASH_JOIN,
+        shjBuildSide = Some(BuildRight)))
+    }
+  }
+
+  test("FileSourceScanExec's metadata should be updated with coalesced info") {
+    val scan = newFileSourceScanExec(RelationSetting(8, None))
+    val value = scan.metadata("SelectedBucketsCount")
+    assert(value === "8 out of 8")
+
+    val scanWithCoalescing = scan.copy(optionalNumCoalescedBuckets = Some(4))
+    val valueWithCoalescing = scanWithCoalescing.metadata("SelectedBucketsCount")
+    assert(valueWithCoalescing == "8 out of 8 (Coalesced to 4)")
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/CachedBatchSerializerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/CachedBatchSerializerSuite.scala
new file mode 100644
index 0000000000000..099a1aa996c11
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/CachedBatchSerializerSuite.scala
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.columnar
+
+import org.apache.spark.SparkConf
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{QueryTest, Row}
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression}
+import org.apache.spark.sql.columnar.{CachedBatch, CachedBatchSerializer}
+import org.apache.spark.sql.execution.columnar.InMemoryRelation.clearSerializer
+import org.apache.spark.sql.execution.vectorized.OnHeapColumnVector
+import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.{IntegerType, StructType}
+import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector}
+import org.apache.spark.storage.StorageLevel
+
+case class SingleIntCachedBatch(data: Array[Int]) extends CachedBatch {
+  override def numRows: Int = data.length
+  override def sizeInBytes: Long = 4 * data.length
+}
+
+/**
+ * Very simple serializer that only supports a single int column, but does support columnar.
+ */
+class TestSingleIntColumnarCachedBatchSerializer extends CachedBatchSerializer {
+  override def supportsColumnarInput(schema: Seq[Attribute]): Boolean = true
+
+  override def convertInternalRowToCachedBatch(
+      input: RDD[InternalRow],
+      schema: Seq[Attribute],
+      storageLevel: StorageLevel,
+      conf: SQLConf): RDD[CachedBatch] = {
+    throw new IllegalStateException("This does not work. This is only for testing")
+  }
+
+  override def convertColumnarBatchToCachedBatch(
+      input: RDD[ColumnarBatch],
+      schema: Seq[Attribute],
+      storageLevel: StorageLevel,
+      conf: SQLConf): RDD[CachedBatch] = {
+    if (schema.length != 1 || schema.head.dataType != IntegerType) {
+      throw new IllegalArgumentException("Only a single column of non-nullable ints works. " +
+          s"This is for testing $schema")
+    }
+    input.map { cb =>
+      val column = cb.column(0)
+      val data = column.getInts(0, cb.numRows())
+      SingleIntCachedBatch(data)
+    }
+  }
+
+  override def supportsColumnarOutput(schema: StructType): Boolean = true
+  override def vectorTypes(attributes: Seq[Attribute], conf: SQLConf): Option[Seq[String]] =
+    Some(attributes.map(_ => classOf[OnHeapColumnVector].getName))
+
+  override def convertCachedBatchToColumnarBatch(
+      input: RDD[CachedBatch],
+      cacheAttributes: Seq[Attribute],
+      selectedAttributes: Seq[Attribute],
+      conf: SQLConf): RDD[ColumnarBatch] = {
+    if (selectedAttributes.isEmpty) {
+      input.map { cached =>
+        val single = cached.asInstanceOf[SingleIntCachedBatch]
+        new ColumnarBatch(new Array[ColumnVector](0), single.numRows)
+      }
+    } else {
+      if (selectedAttributes.length > 1 ||
+          selectedAttributes.head.dataType != IntegerType) {
+        throw new IllegalArgumentException("Only a single column of non-nullable ints works. " +
+            s"This is for testing")
+      }
+      input.map { cached =>
+        val single = cached.asInstanceOf[SingleIntCachedBatch]
+        val cv = OnHeapColumnVector.allocateColumns(single.numRows, selectedAttributes.toStructType)
+        val data = single.data
+        cv(0).putInts(0, data.length, data, 0)
+        new ColumnarBatch(cv.toArray, single.numRows)
+      }
+    }
+  }
+
+  override def convertCachedBatchToInternalRow(
+      input: RDD[CachedBatch],
+      cacheAttributes: Seq[Attribute],
+      selectedAttributes: Seq[Attribute],
+      conf: SQLConf): RDD[InternalRow] = {
+    throw new IllegalStateException("This does not work. This is only for testing")
+  }
+
+  override def buildFilter(
+      predicates: Seq[Expression],
+      cachedAttributes: Seq[Attribute]): (Int, Iterator[CachedBatch]) => Iterator[CachedBatch] = {
+    def ret(index: Int, cb: Iterator[CachedBatch]): Iterator[CachedBatch] = cb
+    ret
+  }
+}
+
+class CachedBatchSerializerSuite  extends QueryTest with SharedSparkSession {
+  import testImplicits._
+
+  override protected def sparkConf: SparkConf = {
+    super.sparkConf.set(
+      StaticSQLConf.SPARK_CACHE_SERIALIZER.key,
+      classOf[TestSingleIntColumnarCachedBatchSerializer].getName)
+  }
+
+  protected override def beforeAll(): Unit = {
+    super.beforeAll()
+    clearSerializer()
+  }
+
+  protected override def afterAll(): Unit = {
+    clearSerializer()
+    super.afterAll()
+  }
+
+  test("Columnar Cache Plugin") {
+    withTempPath { workDir =>
+      val workDirPath = workDir.getAbsolutePath
+      val input = Seq(100, 200, 300).toDF("count")
+      input.write.parquet(workDirPath)
+      val data = spark.read.parquet(workDirPath)
+      data.cache()
+      assert(data.count() == 3)
+      checkAnswer(data, Row(100) :: Row(200) :: Row(300) :: Nil)
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ColumnStatsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ColumnStatsSuite.scala
index 847e0ec4f3195..0abb3cb6a2ed0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ColumnStatsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ColumnStatsSuite.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.execution.columnar
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.types._
-import org.apache.spark.unsafe.types.CalendarInterval
 
 class ColumnStatsSuite extends SparkFunSuite {
   testColumnStats(classOf[BooleanColumnStats], BOOLEAN, Array(true, false, 0))
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ColumnTypeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ColumnTypeSuite.scala
index b25aa6e308657..e6dfc7c99abcc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ColumnTypeSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ColumnTypeSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.execution.columnar
 
 import java.nio.{ByteBuffer, ByteOrder}
 import java.nio.charset.StandardCharsets
+import java.time.{Duration, Period}
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.internal.Logging
@@ -79,6 +80,8 @@ class ColumnTypeSuite extends SparkFunSuite with Logging {
     checkActualSize(MAP_TYPE, Map(1 -> "a"), 4 + (8 + 8 + 8 + 8) + (8 + 8 + 8 + 8))
     checkActualSize(STRUCT_TYPE, Row("hello"), 28)
     checkActualSize(CALENDAR_INTERVAL, new CalendarInterval(0, 0, 0), 4 + 4 + 8)
+    checkActualSize(YEAR_MONTH_INTERVAL, Period.ofMonths(Int.MaxValue).normalized(), 4)
+    checkActualSize(DAY_TIME_INTERVAL, Duration.ofDays(106751991), 8)
   }
 
   testNativeColumnType(BOOLEAN)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala
index 18f29f7b90ad5..b8f73f4563ef1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala
@@ -20,18 +20,32 @@ package org.apache.spark.sql.execution.columnar
 import java.nio.charset.StandardCharsets
 import java.sql.{Date, Timestamp}
 
+import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{DataFrame, QueryTest, Row}
-import org.apache.spark.sql.catalyst.expressions.{AttributeReference, AttributeSet, In}
-import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, AttributeSet, In}
 import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning
-import org.apache.spark.sql.execution.{ColumnarToRowExec, FilterExec, InputAdapter, LocalTableScanExec, WholeStageCodegenExec}
+import org.apache.spark.sql.columnar.CachedBatch
+import org.apache.spark.sql.execution.{ColumnarToRowExec, FilterExec, InputAdapter, WholeStageCodegenExec}
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.test.SQLTestData._
 import org.apache.spark.sql.types._
+import org.apache.spark.storage.StorageLevel
 import org.apache.spark.storage.StorageLevel._
-import org.apache.spark.util.Utils
+
+class TestCachedBatchSerializer(
+    useCompression: Boolean,
+    batchSize: Int) extends DefaultCachedBatchSerializer {
+
+  override def convertInternalRowToCachedBatch(input: RDD[InternalRow],
+      schema: Seq[Attribute],
+      storageLevel: StorageLevel,
+      conf: SQLConf): RDD[CachedBatch] = {
+    convertForCacheInternal(input, schema, batchSize, useCompression)
+  }
+}
 
 class InMemoryColumnarQuerySuite extends QueryTest with SharedSparkSession {
   import testImplicits._
@@ -42,12 +56,12 @@ class InMemoryColumnarQuerySuite extends QueryTest with SharedSparkSession {
     data.createOrReplaceTempView(s"testData$dataType")
     val storageLevel = MEMORY_ONLY
     val plan = spark.sessionState.executePlan(data.logicalPlan).sparkPlan
-    val inMemoryRelation = InMemoryRelation(useCompression = true, 5, storageLevel, plan, None,
-      data.logicalPlan)
+    val inMemoryRelation = InMemoryRelation(new TestCachedBatchSerializer(useCompression = true, 5),
+      storageLevel, plan, None, data.logicalPlan)
 
     assert(inMemoryRelation.cacheBuilder.cachedColumnBuffers.getStorageLevel == storageLevel)
     inMemoryRelation.cacheBuilder.cachedColumnBuffers.collect().head match {
-      case _: CachedBatch =>
+      case _: DefaultCachedBatch =>
       case other => fail(s"Unexpected cached batch type: ${other.getClass.getName}")
     }
     checkAnswer(inMemoryRelation, data.collect().toSeq)
@@ -119,8 +133,8 @@ class InMemoryColumnarQuerySuite extends QueryTest with SharedSparkSession {
 
   test("simple columnar query") {
     val plan = spark.sessionState.executePlan(testData.logicalPlan).sparkPlan
-    val scan = InMemoryRelation(useCompression = true, 5, MEMORY_ONLY, plan, None,
-      testData.logicalPlan)
+    val scan = InMemoryRelation(new TestCachedBatchSerializer(useCompression = true, 5),
+      MEMORY_ONLY, plan, None, testData.logicalPlan)
 
     checkAnswer(scan, testData.collect().toSeq)
   }
@@ -140,8 +154,8 @@ class InMemoryColumnarQuerySuite extends QueryTest with SharedSparkSession {
   test("projection") {
     val logicalPlan = testData.select('value, 'key).logicalPlan
     val plan = spark.sessionState.executePlan(logicalPlan).sparkPlan
-    val scan = InMemoryRelation(useCompression = true, 5, MEMORY_ONLY, plan, None,
-      logicalPlan)
+    val scan = InMemoryRelation(new TestCachedBatchSerializer(useCompression = true, 5),
+      MEMORY_ONLY, plan, None, logicalPlan)
 
     checkAnswer(scan, testData.collect().map {
       case Row(key: Int, value: String) => value -> key
@@ -157,8 +171,8 @@ class InMemoryColumnarQuerySuite extends QueryTest with SharedSparkSession {
 
   test("SPARK-1436 regression: in-memory columns must be able to be accessed multiple times") {
     val plan = spark.sessionState.executePlan(testData.logicalPlan).sparkPlan
-    val scan = InMemoryRelation(useCompression = true, 5, MEMORY_ONLY, plan, None,
-      testData.logicalPlan)
+    val scan = InMemoryRelation(new TestCachedBatchSerializer(useCompression = true, 5),
+      MEMORY_ONLY, plan, None, testData.logicalPlan)
 
     checkAnswer(scan, testData.collect().toSeq)
     checkAnswer(scan, testData.collect().toSeq)
@@ -336,7 +350,8 @@ class InMemoryColumnarQuerySuite extends QueryTest with SharedSparkSession {
   test("SPARK-17549: cached table size should be correctly calculated") {
     val data = spark.sparkContext.parallelize(1 to 10, 5).toDF()
     val plan = spark.sessionState.executePlan(data.logicalPlan).sparkPlan
-    val cached = InMemoryRelation(true, 5, MEMORY_ONLY, plan, None, data.logicalPlan)
+    val cached = InMemoryRelation(new TestCachedBatchSerializer(true, 5),
+      MEMORY_ONLY, plan, None, data.logicalPlan)
 
     // Materialize the data.
     val expectedAnswer = data.collect()
@@ -349,7 +364,8 @@ class InMemoryColumnarQuerySuite extends QueryTest with SharedSparkSession {
    test("cached row count should be calculated") {
     val data = spark.range(6).toDF
     val plan = spark.sessionState.executePlan(data.logicalPlan).sparkPlan
-    val cached = InMemoryRelation(true, 5, MEMORY_ONLY, plan, None, data.logicalPlan)
+    val cached = InMemoryRelation(new TestCachedBatchSerializer(true, 5),
+      MEMORY_ONLY, plan, None, data.logicalPlan)
 
     // Materialize the data.
     val expectedAnswer = data.collect()
@@ -474,12 +490,8 @@ class InMemoryColumnarQuerySuite extends QueryTest with SharedSparkSession {
 
   test("SPARK-22249: buildFilter should not throw exception when In contains an empty list") {
     val attribute = AttributeReference("a", IntegerType)()
-    val localTableScanExec = LocalTableScanExec(Seq(attribute), Nil)
-    val testRelation = InMemoryRelation(false, 1, MEMORY_ONLY, localTableScanExec, None,
-      LocalRelation(Seq(attribute), Nil))
-    val tableScanExec = InMemoryTableScanExec(Seq(attribute),
-      Seq(In(attribute, Nil)), testRelation)
-    assert(tableScanExec.partitionFilters.isEmpty)
+    val testSerializer = new TestCachedBatchSerializer(false, 1)
+    testSerializer.buildFilter(Seq(In(attribute, Nil)), Seq(attribute))
   }
 
   testWithWholeStageCodegenOnAndOff("SPARK-22348: table cache " +
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/BooleanBitSetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/BooleanBitSetSuite.scala
index 192db0e910d03..111a620df8c24 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/BooleanBitSetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/BooleanBitSetSuite.scala
@@ -156,4 +156,30 @@ class BooleanBitSetSuite extends SparkFunSuite {
   test(s"$BooleanBitSet: multiple words and 1 more bit for decompression()") {
     skeletonForDecompress(BITS_PER_LONG * 2 + 1)
   }
+
+  test(s"$BooleanBitSet: Only nulls for decompression()") {
+    val builder = TestCompressibleColumnBuilder(new NoopColumnStats, BOOLEAN, BooleanBitSet)
+    val numRows = 10
+
+    val rows = Seq.fill[InternalRow](numRows)({
+      val row = new GenericInternalRow(1)
+      row.setNullAt(0)
+      row
+    })
+    rows.foreach(builder.appendFrom(_, 0))
+    val buffer = builder.build()
+
+    // Rewinds, skips column header and 4 more bytes for compression scheme ID
+    val headerSize = CompressionScheme.columnHeaderSize(buffer)
+    buffer.position(headerSize)
+    assertResult(BooleanBitSet.typeId, "Wrong compression scheme ID")(buffer.getInt())
+
+    val decoder = BooleanBitSet.decoder(buffer, BOOLEAN)
+    val columnVector = new OnHeapColumnVector(numRows, BooleanType)
+    decoder.decompress(columnVector, numRows)
+
+    (0 until numRows).foreach { rowNum =>
+      assert(columnVector.isNullAt(rowNum))
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/CompressionSchemeBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/CompressionSchemeBenchmark.scala
index fcb18392235c3..cad7f7a11211b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/CompressionSchemeBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/CompressionSchemeBenchmark.scala
@@ -22,7 +22,6 @@ import java.nio.charset.StandardCharsets
 
 import org.apache.commons.lang3.RandomStringUtils
 import org.apache.commons.math3.distribution.LogNormalDistribution
-import org.scalatest.Assertions._
 
 import org.apache.spark.benchmark.{Benchmark, BenchmarkBase}
 import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
@@ -34,7 +33,9 @@ import org.apache.spark.util.Utils._
  * Benchmark to decoders using various compression schemes.
  * To run this benchmark:
  * {{{
- *   1. without sbt: bin/spark-submit --class <this class> <spark sql test jar>
+ *   1. without sbt:
+ *      bin/spark-submit --class <this class>
+ *        --jars <spark core test jar>,<spark catalyst test jar> <spark sql test jar>
  *   2. build/sbt "sql/test:runMain <this class>"
  *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
  *      Results will be written to "benchmarks/CompressionSchemeBenchmark-results.txt".
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/PassThroughEncodingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/PassThroughEncodingSuite.scala
index f946a6779ec95..c6fe64d1058ab 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/PassThroughEncodingSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/PassThroughEncodingSuite.scala
@@ -46,7 +46,7 @@ class PassThroughSuite extends SparkFunSuite {
 
       val builder = TestCompressibleColumnBuilder(columnStats, columnType, PassThrough)
 
-      input.map { value =>
+      input.foreach { value =>
         val row = new GenericInternalRow(1)
         columnType.setField(row, 0, value)
         builder.appendFrom(row, 0)
@@ -98,7 +98,7 @@ class PassThroughSuite extends SparkFunSuite {
       val row = new GenericInternalRow(1)
       val nullRow = new GenericInternalRow(1)
       nullRow.setNullAt(0)
-      input.map { value =>
+      input.foreach { value =>
         if (value == nullValue) {
           builder.appendFrom(nullRow, 0)
         } else {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableAddPartitionParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableAddPartitionParserSuite.scala
new file mode 100644
index 0000000000000..1694c73b10f2c
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableAddPartitionParserSuite.scala
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, UnresolvedPartitionSpec, UnresolvedTable}
+import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan
+import org.apache.spark.sql.catalyst.plans.logical.AddPartitions
+import org.apache.spark.sql.test.SharedSparkSession
+
+class AlterTableAddPartitionParserSuite extends AnalysisTest with SharedSparkSession {
+  test("add partition if not exists") {
+    val sql = """
+      |ALTER TABLE a.b.c ADD IF NOT EXISTS PARTITION
+      |(dt='2008-08-08', country='us') LOCATION 'location1' PARTITION
+      |(dt='2009-09-09', country='uk')""".stripMargin
+    val parsed = parsePlan(sql)
+    val expected = AddPartitions(
+      UnresolvedTable(
+        Seq("a", "b", "c"),
+        "ALTER TABLE ... ADD PARTITION ...",
+        Some("Please use ALTER VIEW instead.")),
+      Seq(
+        UnresolvedPartitionSpec(Map("dt" -> "2008-08-08", "country" -> "us"), Some("location1")),
+        UnresolvedPartitionSpec(Map("dt" -> "2009-09-09", "country" -> "uk"), None)),
+      ifNotExists = true)
+    comparePlans(parsed, expected)
+  }
+
+  test("add partition") {
+    val sql = "ALTER TABLE a.b.c ADD PARTITION (dt='2008-08-08') LOCATION 'loc'"
+    val parsed = parsePlan(sql)
+    val expected = AddPartitions(
+      UnresolvedTable(
+        Seq("a", "b", "c"),
+        "ALTER TABLE ... ADD PARTITION ...",
+        Some("Please use ALTER VIEW instead.")),
+      Seq(UnresolvedPartitionSpec(Map("dt" -> "2008-08-08"), Some("loc"))),
+      ifNotExists = false)
+
+    comparePlans(parsed, expected)
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableAddPartitionSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableAddPartitionSuiteBase.scala
new file mode 100644
index 0000000000000..e2e15917d7ea7
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableAddPartitionSuiteBase.scala
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.{AnalysisException, QueryTest}
+import org.apache.spark.sql.catalyst.analysis.PartitionsAlreadyExistException
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * This base suite contains unified tests for the `ALTER TABLE .. ADD PARTITION` command that
+ * check V1 and V2 table catalogs. The tests that cannot run for all supported catalogs are
+ * located in more specific test suites:
+ *
+ *   - V2 table catalog tests:
+ *     `org.apache.spark.sql.execution.command.v2.AlterTableAddPartitionSuite`
+ *   - V1 table catalog tests:
+ *     `org.apache.spark.sql.execution.command.v1.AlterTableAddPartitionSuiteBase`
+ *     - V1 In-Memory catalog:
+ *       `org.apache.spark.sql.execution.command.v1.AlterTableAddPartitionSuite`
+ *     - V1 Hive External catalog:
+ *       `org.apache.spark.sql.hive.execution.command.AlterTableAddPartitionSuite`
+ */
+trait AlterTableAddPartitionSuiteBase extends QueryTest with DDLCommandTestUtils {
+  override val command = "ALTER TABLE .. ADD PARTITION"
+
+  test("one partition") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (id bigint, data string) $defaultUsing PARTITIONED BY (id)")
+      Seq("", "IF NOT EXISTS").foreach { exists =>
+        sql(s"ALTER TABLE $t ADD $exists PARTITION (id=1) LOCATION 'loc'")
+
+        checkPartitions(t, Map("id" -> "1"))
+        checkLocation(t, Map("id" -> "1"), "loc")
+      }
+    }
+  }
+
+  test("multiple partitions") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (id bigint, data string) $defaultUsing PARTITIONED BY (id)")
+      Seq("", "IF NOT EXISTS").foreach { exists =>
+        sql(s"""
+          |ALTER TABLE $t ADD $exists
+          |PARTITION (id=1) LOCATION 'loc'
+          |PARTITION (id=2) LOCATION 'loc1'""".stripMargin)
+
+        checkPartitions(t, Map("id" -> "1"), Map("id" -> "2"))
+        checkLocation(t, Map("id" -> "1"), "loc")
+        checkLocation(t, Map("id" -> "2"), "loc1")
+      }
+    }
+  }
+
+  test("multi-part partition") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (id bigint, a int, b string) $defaultUsing PARTITIONED BY (a, b)")
+      Seq("", "IF NOT EXISTS").foreach { exists =>
+        sql(s"ALTER TABLE $t ADD $exists PARTITION (a=2, b='abc')")
+
+        checkPartitions(t, Map("a" -> "2", "b" -> "abc"))
+      }
+    }
+  }
+
+  test("table to alter does not exist") {
+    withNamespaceAndTable("ns", "does_not_exist") { t =>
+      val errMsg = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $t ADD IF NOT EXISTS PARTITION (a='4', b='9')")
+      }.getMessage
+      assert(errMsg.contains("Table not found"))
+    }
+  }
+
+  test("case sensitivity in resolving partition specs") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      spark.sql(s"CREATE TABLE $t (id bigint, data string) $defaultUsing PARTITIONED BY (id)")
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+        val errMsg = intercept[AnalysisException] {
+          spark.sql(s"ALTER TABLE $t ADD PARTITION (ID=1) LOCATION 'loc1'")
+        }.getMessage
+        assert(errMsg.contains("ID is not a valid partition column"))
+      }
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+        spark.sql(s"ALTER TABLE $t ADD PARTITION (ID=1) LOCATION 'loc1'")
+        checkPartitions(t, Map("id" -> "1"))
+        checkLocation(t, Map("id" -> "1"), "loc1")
+      }
+    }
+  }
+
+  test("SPARK-33521: universal type conversions of partition values") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"""
+        |CREATE TABLE $t (
+        |  id int,
+        |  part0 tinyint,
+        |  part1 smallint,
+        |  part2 int,
+        |  part3 bigint,
+        |  part4 float,
+        |  part5 double,
+        |  part6 string,
+        |  part7 boolean,
+        |  part8 date,
+        |  part9 timestamp
+        |) $defaultUsing
+        |PARTITIONED BY (part0, part1, part2, part3, part4, part5, part6, part7, part8, part9)
+        |""".stripMargin)
+      val partSpec = """
+        |  part0 = -1,
+        |  part1 = 0,
+        |  part2 = 1,
+        |  part3 = 2,
+        |  part4 = 3.14,
+        |  part5 = 3.14,
+        |  part6 = 'abc',
+        |  part7 = true,
+        |  part8 = '2020-11-23',
+        |  part9 = '2020-11-23 22:13:10.123456'
+        |""".stripMargin
+      sql(s"ALTER TABLE $t ADD PARTITION ($partSpec)")
+      val expected = Map(
+        "part0" -> "-1",
+        "part1" -> "0",
+        "part2" -> "1",
+        "part3" -> "2",
+        "part4" -> "3.14",
+        "part5" -> "3.14",
+        "part6" -> "abc",
+        "part7" -> "true",
+        "part8" -> "2020-11-23",
+        "part9" -> "2020-11-23 22:13:10.123456")
+      checkPartitions(t, expected)
+      sql(s"ALTER TABLE $t DROP PARTITION ($partSpec)")
+      checkPartitions(t) // no partitions
+    }
+  }
+
+  test("SPARK-33676: not fully specified partition spec") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"""
+        |CREATE TABLE $t (id bigint, part0 int, part1 string)
+        |$defaultUsing
+        |PARTITIONED BY (part0, part1)""".stripMargin)
+      val errMsg = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $t ADD PARTITION (part0 = 1)")
+      }.getMessage
+      assert(errMsg.contains("Partition spec is invalid. " +
+        "The spec (part0) must match the partition spec (part0, part1)"))
+    }
+  }
+
+  test("partition already exists") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (id bigint, data string) $defaultUsing PARTITIONED BY (id)")
+      sql(s"ALTER TABLE $t ADD PARTITION (id=2) LOCATION 'loc1'")
+
+      val errMsg = intercept[PartitionsAlreadyExistException] {
+        sql(s"ALTER TABLE $t ADD PARTITION (id=1) LOCATION 'loc'" +
+          " PARTITION (id=2) LOCATION 'loc1'")
+      }.getMessage
+      assert(errMsg.contains("The following partitions already exists"))
+
+      sql(s"ALTER TABLE $t ADD IF NOT EXISTS PARTITION (id=1) LOCATION 'loc'" +
+        " PARTITION (id=2) LOCATION 'loc1'")
+      checkPartitions(t, Map("id" -> "1"), Map("id" -> "2"))
+    }
+  }
+
+  test("SPARK-33474: Support typed literals as partition spec values") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t(name STRING, part DATE) USING PARQUET PARTITIONED BY (part)")
+      sql(s"ALTER TABLE $t ADD PARTITION(part = date'2020-01-01')")
+      checkPartitions(t, Map("part" ->"2020-01-01"))
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableDropPartitionParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableDropPartitionParserSuite.scala
new file mode 100644
index 0000000000000..4c60c80f4e054
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableDropPartitionParserSuite.scala
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, UnresolvedPartitionSpec, UnresolvedTable}
+import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan
+import org.apache.spark.sql.catalyst.parser.ParseException
+import org.apache.spark.sql.catalyst.plans.logical.DropPartitions
+import org.apache.spark.sql.test.SharedSparkSession
+
+class AlterTableDropPartitionParserSuite extends AnalysisTest with SharedSparkSession {
+  test("drop partition") {
+    val sql = """
+      |ALTER TABLE table_name DROP PARTITION
+      |(dt='2008-08-08', country='us'), PARTITION (dt='2009-09-09', country='uk')
+      """.stripMargin
+    val expected = DropPartitions(
+      UnresolvedTable(
+        Seq("table_name"),
+        "ALTER TABLE ... DROP PARTITION ...",
+        Some("Please use ALTER VIEW instead.")),
+      Seq(
+        UnresolvedPartitionSpec(Map("dt" -> "2008-08-08", "country" -> "us")),
+        UnresolvedPartitionSpec(Map("dt" -> "2009-09-09", "country" -> "uk"))),
+      ifExists = false,
+      purge = false)
+
+    comparePlans(parsePlan(sql), expected)
+  }
+
+  test("drop partition if exists") {
+    val sql = """
+      |ALTER TABLE table_name DROP IF EXISTS
+      |PARTITION (dt='2008-08-08', country='us'),
+      |PARTITION (dt='2009-09-09', country='uk')
+      """.stripMargin
+    val expected = DropPartitions(
+      UnresolvedTable(
+        Seq("table_name"),
+        "ALTER TABLE ... DROP PARTITION ...",
+        Some("Please use ALTER VIEW instead.")),
+      Seq(
+        UnresolvedPartitionSpec(Map("dt" -> "2008-08-08", "country" -> "us")),
+        UnresolvedPartitionSpec(Map("dt" -> "2009-09-09", "country" -> "uk"))),
+      ifExists = true,
+      purge = false)
+    comparePlans(parsePlan(sql), expected)
+  }
+
+  test("drop partition in a table with multi-part identifier") {
+    val sql = "ALTER TABLE a.b.c DROP IF EXISTS PARTITION (ds='2017-06-10')"
+    val expected = DropPartitions(
+      UnresolvedTable(
+        Seq("a", "b", "c"),
+        "ALTER TABLE ... DROP PARTITION ...",
+        Some("Please use ALTER VIEW instead.")),
+      Seq(UnresolvedPartitionSpec(Map("ds" -> "2017-06-10"))),
+      ifExists = true,
+      purge = false)
+
+    comparePlans(parsePlan(sql), expected)
+  }
+
+  test("drop partition with PURGE") {
+    val sql = "ALTER TABLE table_name DROP PARTITION (p=1) PURGE"
+    val expected = DropPartitions(
+      UnresolvedTable(
+        Seq("table_name"),
+        "ALTER TABLE ... DROP PARTITION ...",
+        Some("Please use ALTER VIEW instead.")),
+      Seq(UnresolvedPartitionSpec(Map("p" -> "1"))),
+      ifExists = false,
+      purge = true)
+
+    comparePlans(parsePlan(sql), expected)
+  }
+
+  test("drop partition from view") {
+    val sql = "ALTER VIEW table_name DROP PARTITION (p=1)"
+    val errMsg = intercept[ParseException] {
+      parsePlan(sql)
+    }.getMessage
+    assert(errMsg.contains("Operation not allowed"))
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableDropPartitionSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableDropPartitionSuiteBase.scala
new file mode 100644
index 0000000000000..41ac62ed333aa
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableDropPartitionSuiteBase.scala
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
+import org.apache.spark.sql.catalyst.analysis.NoSuchPartitionsException
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * This base suite contains unified tests for the `ALTER TABLE .. DROP PARTITION` command that
+ * check V1 and V2 table catalogs. The tests that cannot run for all supported catalogs are
+ * located in more specific test suites:
+ *
+ *   - V2 table catalog tests:
+ *     `org.apache.spark.sql.execution.command.v2.AlterTableDropPartitionSuite`
+ *   - V1 table catalog tests:
+ *     `org.apache.spark.sql.execution.command.v1.AlterTableDropPartitionSuiteBase`
+ *     - V1 In-Memory catalog:
+ *       `org.apache.spark.sql.execution.command.v1.AlterTableDropPartitionSuite`
+ *     - V1 Hive External catalog:
+ *       `org.apache.spark.sql.hive.execution.command.AlterTableDropPartitionSuite`
+ */
+trait AlterTableDropPartitionSuiteBase extends QueryTest with DDLCommandTestUtils {
+  override val command = "ALTER TABLE .. DROP PARTITION"
+
+  protected def notFullPartitionSpecErr: String
+  protected def nullPartitionValue: String
+
+  protected def checkDropPartition(
+      t: String,
+      ifExists: String,
+      specs: Map[String, Any]*): Unit = {
+    checkPartitions(t, specs.map(_.mapValues(_.toString).toMap): _*)
+    val specStr = specs.map(partSpecToString).mkString(", ")
+    sql(s"ALTER TABLE $t DROP $ifExists $specStr")
+    checkPartitions(t)
+  }
+
+  test("single partition") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (id bigint, data string) $defaultUsing PARTITIONED BY (id)")
+      Seq("", "IF EXISTS").foreach { ifExists =>
+        sql(s"ALTER TABLE $t ADD PARTITION (id=1) LOCATION 'loc'")
+        checkDropPartition(t, ifExists, Map("id" -> 1))
+      }
+    }
+  }
+
+  test("multiple partitions") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (id bigint, data string) $defaultUsing PARTITIONED BY (id)")
+      Seq("", "IF EXISTS").foreach { ifExists =>
+        sql(s"""
+          |ALTER TABLE $t ADD
+          |PARTITION (id=1) LOCATION 'loc'
+          |PARTITION (id=2) LOCATION 'loc1'""".stripMargin)
+        checkDropPartition(t, ifExists, Map("id" -> 1), Map("id" -> 2))
+      }
+    }
+  }
+
+  test("multi-part partition") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (id bigint, a int, b string) $defaultUsing PARTITIONED BY (a, b)")
+      Seq("", "IF EXISTS").foreach { ifExists =>
+        sql(s"ALTER TABLE $t ADD PARTITION (a = 2, b = 'abc')")
+        checkDropPartition(t, ifExists, Map("a" -> 2, "b" -> "abc"))
+      }
+    }
+  }
+
+  test("table to alter does not exist") {
+    withNamespaceAndTable("ns", "does_not_exist") { t =>
+      val errMsg = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $t DROP PARTITION (a='4', b='9')")
+      }.getMessage
+      assert(errMsg.contains("Table not found"))
+    }
+  }
+
+  test("case sensitivity in resolving partition specs") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (id bigint, data string) $defaultUsing PARTITIONED BY (id)")
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+        val errMsg = intercept[AnalysisException] {
+          sql(s"ALTER TABLE $t DROP PARTITION (ID=1)")
+        }.getMessage
+        assert(errMsg.contains("ID is not a valid partition column"))
+      }
+
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+        Seq("", "IF EXISTS").foreach { ifExists =>
+          sql(s"ALTER TABLE $t ADD PARTITION (ID=1) LOCATION 'loc1'")
+          checkDropPartition(t, ifExists, Map("id" -> 1))
+        }
+      }
+    }
+  }
+
+  test("SPARK-33676: not fully specified partition spec") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"""
+        |CREATE TABLE $t (id bigint, part0 int, part1 string)
+        |$defaultUsing
+        |PARTITIONED BY (part0, part1)""".stripMargin)
+      val errMsg = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $t DROP PARTITION (part0 = 1)")
+      }.getMessage
+      assert(errMsg.contains(notFullPartitionSpecErr))
+    }
+  }
+
+  test("partition not exists") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (id bigint, data string) $defaultUsing PARTITIONED BY (id)")
+      sql(s"ALTER TABLE $t ADD PARTITION (id=1) LOCATION 'loc'")
+
+      val errMsg = intercept[NoSuchPartitionsException] {
+        sql(s"ALTER TABLE $t DROP PARTITION (id=1), PARTITION (id=2)")
+      }.getMessage
+      assert(errMsg.contains("partitions not found in table"))
+
+      checkPartitions(t, Map("id" -> "1"))
+      sql(s"ALTER TABLE $t DROP IF EXISTS PARTITION (id=1), PARTITION (id=2)")
+      checkPartitions(t)
+    }
+  }
+
+  test("SPARK-33990: don not return data from dropped partition") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (id int, part int) $defaultUsing PARTITIONED BY (part)")
+      sql(s"INSERT INTO $t PARTITION (part=0) SELECT 0")
+      sql(s"INSERT INTO $t PARTITION (part=1) SELECT 1")
+      QueryTest.checkAnswer(sql(s"SELECT * FROM $t"), Seq(Row(0, 0), Row(1, 1)))
+      sql(s"ALTER TABLE $t DROP PARTITION (part=0)")
+      QueryTest.checkAnswer(sql(s"SELECT * FROM $t"), Seq(Row(1, 1)))
+    }
+  }
+
+  test("SPARK-33950, SPARK-33987: refresh cache after partition dropping") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (id int, part int) $defaultUsing PARTITIONED BY (part)")
+      sql(s"INSERT INTO $t PARTITION (part=0) SELECT 0")
+      sql(s"INSERT INTO $t PARTITION (part=1) SELECT 1")
+      assert(!spark.catalog.isCached(t))
+      sql(s"CACHE TABLE $t")
+      assert(spark.catalog.isCached(t))
+      QueryTest.checkAnswer(sql(s"SELECT * FROM $t"), Seq(Row(0, 0), Row(1, 1)))
+      sql(s"ALTER TABLE $t DROP PARTITION (part=0)")
+      assert(spark.catalog.isCached(t))
+      QueryTest.checkAnswer(sql(s"SELECT * FROM $t"), Seq(Row(1, 1)))
+    }
+  }
+
+  test("SPARK-33591: null as a partition value") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (col1 INT, p1 STRING) $defaultUsing PARTITIONED BY (p1)")
+      sql(s"ALTER TABLE $t ADD PARTITION (p1 = null)")
+      checkPartitions(t, Map("p1" -> nullPartitionValue))
+      sql(s"ALTER TABLE $t DROP PARTITION (p1 = null)")
+      checkPartitions(t)
+    }
+  }
+
+  test("SPARK-33591, SPARK-34203: insert and drop partitions with null values") {
+    def insertAndDropNullPart(t: String, insertCmd: String): Unit = {
+      sql(s"CREATE TABLE $t (col1 INT, p1 STRING) $defaultUsing PARTITIONED BY (p1)")
+      sql(insertCmd)
+      checkPartitions(t, Map("p1" -> nullPartitionValue))
+      sql(s"ALTER TABLE $t DROP PARTITION (p1 = null)")
+      checkPartitions(t)
+    }
+
+    withNamespaceAndTable("ns", "tbl") { t =>
+      insertAndDropNullPart(t, s"INSERT INTO TABLE $t PARTITION (p1 = null) SELECT 0")
+    }
+
+    withSQLConf("hive.exec.dynamic.partition.mode" -> "nonstrict") {
+      withNamespaceAndTable("ns", "tbl") { t =>
+        insertAndDropNullPart(t, s"INSERT OVERWRITE TABLE $t VALUES (0, null)")
+      }
+    }
+  }
+
+  test("SPARK-34161, SPARK-34138, SPARK-34099: keep dependents cached after table altering") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (id int, part int) $defaultUsing PARTITIONED BY (part)")
+      sql(s"INSERT INTO $t PARTITION (part=0) SELECT 0")
+      sql(s"INSERT INTO $t PARTITION (part=1) SELECT 1")
+      sql(s"INSERT INTO $t PARTITION (part=2) SELECT 2")
+      sql(s"INSERT INTO $t PARTITION (part=3) SELECT 3")
+      cacheRelation(t)
+      checkCachedRelation(t, Seq(Row(0, 0), Row(1, 1), Row(2, 2), Row(3, 3)))
+
+      withView("v0") {
+        sql(s"CREATE VIEW v0 AS SELECT * FROM $t")
+        cacheRelation("v0")
+        sql(s"ALTER TABLE $t DROP PARTITION (part=1)")
+        checkCachedRelation("v0", Seq(Row(0, 0), Row(2, 2), Row(3, 3)))
+      }
+
+      withTempView("v1") {
+        sql(s"CREATE TEMP VIEW v1 AS SELECT * FROM $t")
+        cacheRelation("v1")
+        sql(s"ALTER TABLE $t DROP PARTITION (part=2)")
+        checkCachedRelation("v1", Seq(Row(0, 0), Row(3, 3)))
+      }
+
+      val v2 = s"${spark.sharedState.globalTempViewManager.database}.v2"
+      withGlobalTempView("v2") {
+        sql(s"CREATE GLOBAL TEMP VIEW v2 AS SELECT * FROM $t")
+        cacheRelation(v2)
+        sql(s"ALTER TABLE $t DROP PARTITION (part=3)")
+        checkCachedRelation(v2, Seq(Row(0, 0)))
+      }
+    }
+  }
+
+  test("SPARK-33474: Support typed literals as partition spec values") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t(name STRING, part DATE) USING PARQUET PARTITIONED BY (part)")
+      sql(s"ALTER TABLE $t ADD PARTITION(part = date'2020-01-01')")
+      checkPartitions(t, Map("part" -> "2020-01-01"))
+      sql(s"ALTER TABLE $t DROP PARTITION (part = date'2020-01-01')")
+      checkPartitions(t)
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableRecoverPartitionsParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableRecoverPartitionsParserSuite.scala
new file mode 100644
index 0000000000000..ebc1bd3468837
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableRecoverPartitionsParserSuite.scala
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, UnresolvedTable}
+import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan
+import org.apache.spark.sql.catalyst.parser.ParseException
+import org.apache.spark.sql.catalyst.plans.logical.RecoverPartitions
+import org.apache.spark.sql.test.SharedSparkSession
+
+class AlterTableRecoverPartitionsParserSuite extends AnalysisTest with SharedSparkSession {
+
+  test("recover partitions without table") {
+    val errMsg = intercept[ParseException] {
+      parsePlan("ALTER TABLE RECOVER PARTITIONS")
+    }.getMessage
+    assert(errMsg.contains("no viable alternative at input 'ALTER TABLE RECOVER PARTITIONS'"))
+  }
+
+  test("recover partitions of a table") {
+    comparePlans(
+      parsePlan("ALTER TABLE tbl RECOVER PARTITIONS"),
+      RecoverPartitions(
+        UnresolvedTable(
+          Seq("tbl"),
+          "ALTER TABLE ... RECOVER PARTITIONS",
+          Some("Please use ALTER VIEW instead."))))
+  }
+
+  test("recover partitions of a table in a database") {
+    comparePlans(
+      parsePlan("alter table db.tbl recover partitions"),
+      RecoverPartitions(
+        UnresolvedTable(
+          Seq("db", "tbl"),
+          "ALTER TABLE ... RECOVER PARTITIONS",
+          Some("Please use ALTER VIEW instead."))))
+  }
+
+  test("recover partitions of a table spark_catalog") {
+    comparePlans(
+      parsePlan("alter table spark_catalog.db.TBL recover partitions"),
+      RecoverPartitions(
+        UnresolvedTable(
+          Seq("spark_catalog", "db", "TBL"),
+          "ALTER TABLE ... RECOVER PARTITIONS",
+          Some("Please use ALTER VIEW instead."))))
+  }
+
+  test("recover partitions of a table in nested namespaces") {
+    comparePlans(
+      parsePlan("Alter Table ns1.ns2.ns3.ns4.ns5.ns6.ns7.ns8.t Recover Partitions"),
+      RecoverPartitions(
+        UnresolvedTable(
+          Seq("ns1", "ns2", "ns3", "ns4", "ns5", "ns6", "ns7", "ns8", "t"),
+          "ALTER TABLE ... RECOVER PARTITIONS",
+          Some("Please use ALTER VIEW instead."))))
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableRecoverPartitionsSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableRecoverPartitionsSuiteBase.scala
new file mode 100644
index 0000000000000..b33817e84b2e8
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableRecoverPartitionsSuiteBase.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.QueryTest
+
+/**
+ * This base suite contains unified tests for the `ALTER TABLE .. RECOVER PARTITIONS` command that
+ * check V1 and V2 table catalogs. The tests that cannot run for all supported catalogs are
+ * located in more specific test suites:
+ *
+ *   - V2 table catalog tests:
+ *     `org.apache.spark.sql.execution.command.v2.AlterTableRecoverPartitionsSuite`
+ *   - V1 table catalog tests:
+ *     `org.apache.spark.sql.execution.command.v1.AlterTableRecoverPartitionsSuiteBase`
+ *     - V1 In-Memory catalog (sequential):
+ *       `org.apache.spark.sql.execution.command.v1.AlterTableRecoverPartitionsSuite`
+ *     - V1 In-Memory catalog (parallel):
+ *       `org.apache.spark.sql.execution.command.v1.AlterTableRecoverPartitionsParallelSuite`
+ *     - V1 Hive External catalog:
+ *       `org.apache.spark.sql.hive.execution.command.AlterTableRecoverPartitionsSuite`
+ */
+trait AlterTableRecoverPartitionsSuiteBase extends QueryTest with DDLCommandTestUtils {
+  override val command = "ALTER TABLE .. RECOVER PARTITIONS"
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableRenameParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableRenameParserSuite.scala
new file mode 100644
index 0000000000000..e4087e63797d2
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableRenameParserSuite.scala
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, UnresolvedTableOrView}
+import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan
+import org.apache.spark.sql.catalyst.parser.ParseException
+import org.apache.spark.sql.catalyst.plans.logical.RenameTable
+
+class AlterTableRenameParserSuite extends AnalysisTest {
+  test("rename table") {
+    comparePlans(
+      parsePlan("ALTER TABLE a.b.c RENAME TO x.y.z"),
+      RenameTable(
+        UnresolvedTableOrView(Seq("a", "b", "c"), "ALTER TABLE ... RENAME TO", true),
+        Seq("x", "y", "z"),
+        isView = false))
+  }
+
+  test("case sensitivity") {
+    comparePlans(
+      parsePlan("alter Table spark_catalog.ns.tbl RENAME to tbl"),
+      RenameTable(
+        UnresolvedTableOrView(Seq("spark_catalog", "ns", "tbl"), "ALTER TABLE ... RENAME TO", true),
+        Seq("tbl"),
+        isView = false))
+  }
+
+  test("invalid table identifiers") {
+    Seq(
+      "ALTER TABLE RENAME TO x.y.z",
+      "ALTER TABLE _ RENAME TO .z").foreach { renameCmd =>
+      intercept[ParseException] { parsePlan(renameCmd) }
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableRenamePartitionParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableRenamePartitionParserSuite.scala
new file mode 100644
index 0000000000000..4148798d6cdb4
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableRenamePartitionParserSuite.scala
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, UnresolvedPartitionSpec, UnresolvedTable}
+import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan
+import org.apache.spark.sql.catalyst.plans.logical.RenamePartitions
+import org.apache.spark.sql.test.SharedSparkSession
+
+class AlterTableRenamePartitionParserSuite extends AnalysisTest with SharedSparkSession {
+  test("rename a partition with single part") {
+    val sql = """
+      |ALTER TABLE a.b.c PARTITION (ds='2017-06-10')
+      |RENAME TO PARTITION (ds='2018-06-10')
+      """.stripMargin
+    val parsed = parsePlan(sql)
+    val expected = RenamePartitions(
+      UnresolvedTable(
+        Seq("a", "b", "c"),
+        "ALTER TABLE ... RENAME TO PARTITION",
+        Some("Please use ALTER VIEW instead.")),
+      UnresolvedPartitionSpec(Map("ds" -> "2017-06-10")),
+      UnresolvedPartitionSpec(Map("ds" -> "2018-06-10")))
+    comparePlans(parsed, expected)
+  }
+
+  test("rename a partition with multi parts") {
+    val sql = """
+      |ALTER TABLE table_name PARTITION (dt='2008-08-08', country='us')
+      |RENAME TO PARTITION (dt='2008-09-09', country='uk')
+      """.stripMargin
+    val parsed = parsePlan(sql)
+    val expected = RenamePartitions(
+      UnresolvedTable(
+        Seq("table_name"),
+        "ALTER TABLE ... RENAME TO PARTITION",
+        Some("Please use ALTER VIEW instead.")),
+      UnresolvedPartitionSpec(Map("dt" -> "2008-08-08", "country" -> "us")),
+      UnresolvedPartitionSpec(Map("dt" -> "2008-09-09", "country" -> "uk")))
+    comparePlans(parsed, expected)
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableRenamePartitionSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableRenamePartitionSuiteBase.scala
new file mode 100644
index 0000000000000..080cd89c4a209
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableRenamePartitionSuiteBase.scala
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
+import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionException, PartitionAlreadyExistsException}
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * This base suite contains unified tests for the `ALTER TABLE .. RENAME PARTITION` command that
+ * check V1 and V2 table catalogs. The tests that cannot run for all supported catalogs are
+ * located in more specific test suites:
+ *
+ *   - V2 table catalog tests:
+ *     `org.apache.spark.sql.execution.command.v2.AlterTableRenamePartitionSuite`
+ *   - V1 table catalog tests:
+ *     `org.apache.spark.sql.execution.command.v1.AlterTableRenamePartitionSuiteBase`
+ *     - V1 In-Memory catalog:
+ *       `org.apache.spark.sql.execution.command.v1.AlterTableRenamePartitionSuite`
+ *     - V1 Hive External catalog:
+ *       `org.apache.spark.sql.hive.execution.command.AlterTableRenamePartitionSuite`
+ */
+trait AlterTableRenamePartitionSuiteBase extends QueryTest with DDLCommandTestUtils {
+  override val command = "ALTER TABLE .. RENAME PARTITION"
+
+  protected def createSinglePartTable(t: String): Unit = {
+    sql(s"CREATE TABLE $t (id bigint, data string) $defaultUsing PARTITIONED BY (id)")
+    sql(s"INSERT INTO $t PARTITION (id = 1) SELECT 'abc'")
+  }
+
+  test("rename without explicitly specifying database") {
+    withSQLConf(SQLConf.DEFAULT_CATALOG.key -> catalog) {
+      createSinglePartTable("t")
+      checkPartitions("t", Map("id" -> "1"))
+
+      sql(s"ALTER TABLE t PARTITION (id = 1) RENAME TO PARTITION (id = 2)")
+      checkPartitions("t", Map("id" -> "2"))
+      checkAnswer(sql(s"SELECT id, data FROM t"), Row(2, "abc"))
+    }
+  }
+
+  test("table to alter does not exist") {
+    withNamespace(s"$catalog.ns") {
+      sql(s"CREATE NAMESPACE $catalog.ns")
+      val errMsg = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $catalog.ns.no_tbl PARTITION (id=1) RENAME TO PARTITION (id=2)")
+      }.getMessage
+      assert(errMsg.contains("Table not found"))
+    }
+  }
+
+  test("partition to rename does not exist") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      createSinglePartTable(t)
+      checkPartitions(t, Map("id" -> "1"))
+      val errMsg = intercept[NoSuchPartitionException] {
+        sql(s"ALTER TABLE $t PARTITION (id = 3) RENAME TO PARTITION (id = 2)")
+      }.getMessage
+      assert(errMsg.contains("Partition not found in table"))
+    }
+  }
+
+  test("target partition exists") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      createSinglePartTable(t)
+      sql(s"INSERT INTO $t PARTITION (id = 2) SELECT 'def'")
+      checkPartitions(t, Map("id" -> "1"), Map("id" -> "2"))
+      val errMsg = intercept[PartitionAlreadyExistsException] {
+        sql(s"ALTER TABLE $t PARTITION (id = 1) RENAME TO PARTITION (id = 2)")
+      }.getMessage
+      assert(errMsg.contains("Partition already exists"))
+    }
+  }
+
+  test("single part partition") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      createSinglePartTable(t)
+      checkPartitions(t, Map("id" -> "1"))
+
+      sql(s"ALTER TABLE $t PARTITION (id = 1) RENAME TO PARTITION (id = 2)")
+      checkPartitions(t, Map("id" -> "2"))
+      checkAnswer(sql(s"SELECT id, data FROM $t"), Row(2, "abc"))
+    }
+  }
+
+  test("multi part partition") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      createWideTable(t)
+      checkPartitions(t,
+        Map(
+          "year" -> "2016",
+          "month" -> "3",
+          "hour" -> "10",
+          "minute" -> "10",
+          "sec" -> "10",
+          "extra" -> "1"),
+        Map(
+          "year" -> "2016",
+          "month" -> "4",
+          "hour" -> "10",
+          "minute" -> "10",
+          "sec" -> "10",
+          "extra" -> "1"))
+
+      sql(s"""
+        |ALTER TABLE $t
+        |PARTITION (
+        |  year = 2016, month = 3, hour = 10, minute = 10, sec = 10, extra = 1
+        |) RENAME TO PARTITION (
+        |  year = 2016, month = 3, hour = 10, minute = 10, sec = 123, extra = 1
+        |)""".stripMargin)
+      checkPartitions(t,
+        Map(
+          "year" -> "2016",
+          "month" -> "3",
+          "hour" -> "10",
+          "minute" -> "10",
+          "sec" -> "123",
+          "extra" -> "1"),
+        Map(
+          "year" -> "2016",
+          "month" -> "4",
+          "hour" -> "10",
+          "minute" -> "10",
+          "sec" -> "10",
+          "extra" -> "1"))
+      checkAnswer(sql(s"SELECT month, sec, price FROM $t"), Row(3, 123, 3))
+    }
+  }
+
+  test("partition spec in RENAME PARTITION should be case insensitive") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      createSinglePartTable(t)
+      checkPartitions(t, Map("id" -> "1"))
+
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+        val errMsg = intercept[AnalysisException] {
+          sql(s"ALTER TABLE $t PARTITION (ID = 1) RENAME TO PARTITION (id = 2)")
+        }.getMessage
+        assert(errMsg.contains("ID is not a valid partition column"))
+        checkPartitions(t, Map("id" -> "1"))
+      }
+
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+        sql(s"ALTER TABLE $t PARTITION (ID = 1) RENAME TO PARTITION (id = 2)")
+        checkPartitions(t, Map("id" -> "2"))
+        checkAnswer(sql(s"SELECT id, data FROM $t"), Row(2, "abc"))
+      }
+    }
+  }
+
+  test("SPARK-34011: refresh cache after partition renaming") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (id int, part int) $defaultUsing PARTITIONED BY (part)")
+      sql(s"INSERT INTO $t PARTITION (part=0) SELECT 0")
+      sql(s"INSERT INTO $t PARTITION (part=1) SELECT 1")
+      assert(!spark.catalog.isCached(t))
+      sql(s"CACHE TABLE $t")
+      assert(spark.catalog.isCached(t))
+      QueryTest.checkAnswer(sql(s"SELECT * FROM $t"), Seq(Row(0, 0), Row(1, 1)))
+      sql(s"ALTER TABLE $t PARTITION (part=0) RENAME TO PARTITION (part=2)")
+      assert(spark.catalog.isCached(t))
+      QueryTest.checkAnswer(sql(s"SELECT * FROM $t"), Seq(Row(0, 2), Row(1, 1)))
+    }
+  }
+
+  test("SPARK-34161, SPARK-34138, SPARK-34099: keep dependents cached after table altering") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (id int, part int) $defaultUsing PARTITIONED BY (part)")
+      sql(s"INSERT INTO $t PARTITION (part=0) SELECT 0")
+      sql(s"INSERT INTO $t PARTITION (part=1) SELECT 1")
+      cacheRelation(t)
+      checkCachedRelation(t, Seq(Row(0, 0), Row(1, 1)))
+
+      withView("v0") {
+        sql(s"CREATE VIEW v0 AS SELECT * FROM $t")
+        cacheRelation("v0")
+        sql(s"ALTER TABLE $t PARTITION (part=0) RENAME TO PARTITION (part=2)")
+        checkCachedRelation("v0", Seq(Row(0, 2), Row(1, 1)))
+      }
+
+      withTempView("v1") {
+        sql(s"CREATE TEMP VIEW v1 AS SELECT * FROM $t")
+        cacheRelation("v1")
+        sql(s"ALTER TABLE $t PARTITION (part=1) RENAME TO PARTITION (part=3)")
+        checkCachedRelation("v1", Seq(Row(0, 2), Row(1, 3)))
+      }
+
+      val v2 = s"${spark.sharedState.globalTempViewManager.database}.v2"
+      withGlobalTempView("v2") {
+        sql(s"CREATE GLOBAL TEMP VIEW v2 AS SELECT * FROM $t")
+        cacheRelation(v2)
+        sql(s"ALTER TABLE $t PARTITION (part=2) RENAME TO PARTITION (part=4)")
+        checkCachedRelation(v2, Seq(Row(0, 4), Row(1, 3)))
+      }
+    }
+  }
+
+  test("SPARK-33474: Support typed literals as partition spec values") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t(name STRING, part DATE) USING PARQUET PARTITIONED BY (part)")
+      sql(s"ALTER TABLE $t ADD PARTITION(part = date'2020-01-01')")
+      checkPartitions(t, Map("part" -> "2020-01-01"))
+      sql(s"ALTER TABLE $t PARTITION (part = date'2020-01-01')" +
+        s" RENAME TO PARTITION (part = date'2020-01-02')")
+      checkPartitions(t, Map("part" -> "2020-01-02"))
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableRenameSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableRenameSuiteBase.scala
new file mode 100644
index 0000000000000..6370939cef6a2
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableRenameSuiteBase.scala
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
+import org.apache.spark.storage.StorageLevel
+
+/**
+ * This base suite contains unified tests for the `ALTER TABLE .. RENAME` command that check V1
+ * and V2 table catalogs. The tests that cannot run for all supported catalogs are located in more
+ * specific test suites:
+ *
+ *   - V2 table catalog tests: `org.apache.spark.sql.execution.command.v2.AlterTableRenameSuite`
+ *   - V1 table catalog tests: `org.apache.spark.sql.execution.command.v1.AlterTableRenameSuiteBase`
+ *     - V1 In-Memory catalog: `org.apache.spark.sql.execution.command.v1.AlterTableRenameSuite`
+ *     - V1 Hive External catalog:
+ *       `org.apache.spark.sql.hive.execution.command.AlterTableRenameSuite`
+ */
+trait AlterTableRenameSuiteBase extends QueryTest with DDLCommandTestUtils {
+  override val command = "ALTER TABLE .. RENAME"
+
+  test("rename a table in a database/namespace") {
+    withNamespaceAndTable("ns", "dst_tbl") { dst =>
+      val src = dst.replace("dst", "src")
+      sql(s"CREATE TABLE $src (c0 INT) $defaultUsing")
+      sql(s"INSERT INTO $src SELECT 0")
+
+      sql(s"ALTER TABLE $src RENAME TO ns.dst_tbl")
+      checkTables("ns", "dst_tbl")
+      QueryTest.checkAnswer(sql(s"SELECT c0 FROM $dst"), Seq(Row(0)))
+    }
+  }
+
+  test("table to rename does not exist") {
+    val errMsg = intercept[AnalysisException] {
+      sql(s"ALTER TABLE $catalog.dbx.does_not_exist RENAME TO dbx.tab2")
+    }.getMessage
+    assert(errMsg.contains("Table or view not found"))
+  }
+
+  test("omit namespace in the destination table") {
+    withNamespaceAndTable("ns", "dst_tbl") { dst =>
+      val src = dst.replace("dst", "src")
+      sql(s"CREATE TABLE $src (c0 INT) $defaultUsing")
+      sql(s"INSERT INTO $src SELECT 0")
+
+      sql(s"ALTER TABLE $src RENAME TO dst_tbl")
+      checkTables("ns", "dst_tbl")
+      QueryTest.checkAnswer(sql(s"SELECT c0 FROM $dst"), Seq(Row(0)))
+    }
+  }
+
+  test("SPARK-33786: Cache's storage level should be respected when a table name is altered") {
+    withNamespaceAndTable("ns", "dst_tbl") { dst =>
+      val src = dst.replace("dst", "src")
+      def getStorageLevel(tableName: String): StorageLevel = {
+        val table = spark.table(tableName)
+        val cachedData = spark.sharedState.cacheManager.lookupCachedData(table).get
+        cachedData.cachedRepresentation.cacheBuilder.storageLevel
+      }
+      sql(s"CREATE TABLE $src (c0 INT) $defaultUsing")
+      sql(s"INSERT INTO $src SELECT 0")
+      sql(s"CACHE TABLE $src OPTIONS('storageLevel' 'MEMORY_ONLY')")
+      val oldStorageLevel = getStorageLevel(src)
+
+      sql(s"ALTER TABLE $src RENAME TO ns.dst_tbl")
+      QueryTest.checkAnswer(sql(s"SELECT c0 FROM $dst"), Seq(Row(0)))
+      val newStorageLevel = getStorageLevel(dst)
+      assert(oldStorageLevel === newStorageLevel)
+    }
+  }
+
+  test("rename cached table") {
+    withNamespaceAndTable("ns", "students") { students =>
+      sql(s"CREATE TABLE $students (age INT, name STRING) $defaultUsing")
+      sql(s"INSERT INTO $students SELECT 19, 'Ana'")
+
+      spark.catalog.cacheTable(students)
+      val expected = Seq(Row(19, "Ana"))
+      QueryTest.checkAnswer(spark.table(students), expected)
+      assert(spark.catalog.isCached(students),
+        "bad test: table was not cached in the first place")
+      val teachers = s"$catalog.ns.teachers"
+      withTable(teachers) {
+        // After the command below we have both students and teachers.
+        sql(s"ALTER TABLE $students RENAME TO ns.teachers")
+        // The cached data for the old students table should not be read by
+        // the new students table.
+        sql(s"CREATE TABLE $students (age INT, name STRING) $defaultUsing")
+        assert(!spark.catalog.isCached(students))
+        assert(spark.catalog.isCached(teachers))
+        assert(spark.table(students).collect().isEmpty)
+        QueryTest.checkAnswer(spark.table(teachers), expected)
+      }
+    }
+  }
+
+  test("rename without explicitly specifying database") {
+    try {
+      withNamespaceAndTable("ns", "dst_tbl") { dst =>
+        val src = dst.replace("dst", "src")
+        sql(s"CREATE TABLE $src (c0 INT) $defaultUsing")
+        sql(s"INSERT INTO $src SELECT 0")
+
+        sql(s"USE $catalog.ns")
+        sql(s"ALTER TABLE src_tbl RENAME TO dst_tbl")
+        checkTables("ns", "dst_tbl")
+        checkAnswer(sql(s"SELECT c0 FROM $dst"), Seq(Row(0)))
+      }
+    } finally {
+      spark.sessionState.catalogManager.reset()
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/CharVarcharDDLTestBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/CharVarcharDDLTestBase.scala
new file mode 100644
index 0000000000000..ba683c049a631
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/CharVarcharDDLTestBase.scala
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
+import org.apache.spark.sql.catalyst.util.CharVarcharUtils
+import org.apache.spark.sql.connector.catalog.InMemoryPartitionTableCatalog
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.test.{SharedSparkSession, SQLTestUtils}
+import org.apache.spark.sql.types._
+
+trait CharVarcharDDLTestBase extends QueryTest with SQLTestUtils {
+
+  def format: String
+
+  def checkColType(f: StructField, dt: DataType): Unit = {
+    assert(f.dataType == CharVarcharUtils.replaceCharVarcharWithString(dt))
+    assert(CharVarcharUtils.getRawType(f.metadata).contains(dt))
+  }
+
+  test("allow to change column for char(x) to char(y), x == y") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(i STRING, c CHAR(4)) USING $format")
+      sql("ALTER TABLE t CHANGE COLUMN c TYPE CHAR(4)")
+      checkColType(spark.table("t").schema(1), CharType(4))
+    }
+  }
+
+  test("not allow to change column for char(x) to char(y), x != y") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(i STRING, c CHAR(4)) USING $format")
+      val e = intercept[AnalysisException] {
+        sql("ALTER TABLE t CHANGE COLUMN c TYPE CHAR(5)")
+      }
+      val v1 = e.getMessage contains "'CharType(4)' to 'c' with type 'CharType(5)'"
+      val v2 = e.getMessage contains "char(4) cannot be cast to char(5)"
+      assert(v1 || v2)
+    }
+  }
+
+  test("not allow to change column from string to char type") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(i STRING, c STRING) USING $format")
+      val e = intercept[AnalysisException] {
+        sql("ALTER TABLE t CHANGE COLUMN c TYPE CHAR(5)")
+      }
+      val v1 = e.getMessage contains "'StringType' to 'c' with type 'CharType(5)'"
+      val v2 = e.getMessage contains "string cannot be cast to char(5)"
+      assert(v1 || v2)
+    }
+  }
+
+  test("not allow to change column from int to char type") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(i int, c CHAR(4)) USING $format")
+      val e = intercept[AnalysisException] {
+        sql("ALTER TABLE t CHANGE COLUMN i TYPE CHAR(5)")
+      }
+      val v1 = e.getMessage contains "'IntegerType' to 'i' with type 'CharType(5)'"
+      val v2 = e.getMessage contains "int cannot be cast to char(5)"
+      assert(v1 || v2)
+    }
+  }
+
+  test("allow to change column for varchar(x) to varchar(y), x == y") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(i STRING, c VARCHAR(4)) USING $format")
+      sql("ALTER TABLE t CHANGE COLUMN c TYPE VARCHAR(4)")
+      checkColType(spark.table("t").schema(1), VarcharType(4))
+    }
+  }
+
+  test("not allow to change column for varchar(x) to varchar(y), x > y") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(i STRING, c VARCHAR(4)) USING $format")
+      val e = intercept[AnalysisException] {
+        sql("ALTER TABLE t CHANGE COLUMN c TYPE VARCHAR(3)")
+      }
+      val v1 = e.getMessage contains "'VarcharType(4)' to 'c' with type 'VarcharType(3)'"
+      val v2 = e.getMessage contains "varchar(4) cannot be cast to varchar(3)"
+      assert(v1 || v2)
+    }
+  }
+
+  def checkTableSchemaTypeStr(expected: Seq[Row]): Unit = {
+    checkAnswer(sql("desc t").selectExpr("data_type").where("data_type like '%char%'"), expected)
+  }
+
+  test("SPARK-33901: alter table add columns should not change original table's schema") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(i CHAR(5), c VARCHAR(4)) USING $format")
+      sql("ALTER TABLE t ADD COLUMNS (d VARCHAR(5))")
+      checkTableSchemaTypeStr(Seq(Row("char(5)"), Row("varchar(4)"), Row("varchar(5)")))
+    }
+  }
+
+  test("SPARK-33901: ctas should should not change table's schema") {
+    withTable("t", "tt") {
+      sql(s"CREATE TABLE tt(i CHAR(5), c VARCHAR(4)) USING $format")
+      sql(s"CREATE TABLE t USING $format AS SELECT * FROM tt")
+      checkTableSchemaTypeStr(Seq(Row("char(5)"), Row("varchar(4)")))
+    }
+  }
+}
+
+class FileSourceCharVarcharDDLTestSuite extends CharVarcharDDLTestBase with SharedSparkSession {
+  override def format: String = "parquet"
+  override protected def sparkConf: SparkConf = {
+    super.sparkConf.set(SQLConf.USE_V1_SOURCE_LIST, "parquet")
+  }
+
+  // TODO(SPARK-33902): MOVE TO SUPER CLASS AFTER THE TARGET TICKET RESOLVED
+  test("SPARK-33901: create table like should should not change table's schema") {
+    withTable("t", "tt") {
+      sql(s"CREATE TABLE tt(i CHAR(5), c VARCHAR(4)) USING $format")
+      sql("CREATE TABLE t LIKE tt")
+      checkTableSchemaTypeStr(Seq(Row("char(5)"), Row("varchar(4)")))
+    }
+  }
+
+  // TODO(SPARK-33903): MOVE TO SUPER CLASS AFTER THE TARGET TICKET RESOLVED
+  test("SPARK-33901: cvas should should not change view's schema") {
+    withTable( "tt") {
+      sql(s"CREATE TABLE tt(i CHAR(5), c VARCHAR(4)) USING $format")
+      withView("t") {
+        sql("CREATE VIEW t AS SELECT * FROM tt")
+        checkTableSchemaTypeStr(Seq(Row("char(5)"), Row("varchar(4)")))
+      }
+    }
+  }
+}
+
+class DSV2CharVarcharDDLTestSuite extends CharVarcharDDLTestBase
+  with SharedSparkSession {
+  override def format: String = "foo"
+  protected override def sparkConf = {
+    super.sparkConf
+      .set("spark.sql.catalog.testcat", classOf[InMemoryPartitionTableCatalog].getName)
+      .set(SQLConf.DEFAULT_CATALOG.key, "testcat")
+  }
+
+  test("allow to change change column from char to string type") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(i STRING, c CHAR(4)) USING $format")
+      sql("ALTER TABLE t CHANGE COLUMN c TYPE STRING")
+      assert(spark.table("t").schema(1).dataType === StringType)
+    }
+  }
+
+  test("allow to change column from char(x) to varchar(y) type x <= y") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(i STRING, c CHAR(4)) USING $format")
+      sql("ALTER TABLE t CHANGE COLUMN c TYPE VARCHAR(4)")
+      checkColType(spark.table("t").schema(1), VarcharType(4))
+    }
+    withTable("t") {
+      sql(s"CREATE TABLE t(i STRING, c CHAR(4)) USING $format")
+      sql("ALTER TABLE t CHANGE COLUMN c TYPE VARCHAR(5)")
+      checkColType(spark.table("t").schema(1), VarcharType(5))
+    }
+  }
+
+  test("allow to change column from varchar(x) to varchar(y) type x <= y") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(i STRING, c VARCHAR(4)) USING $format")
+      sql("ALTER TABLE t CHANGE COLUMN c TYPE VARCHAR(4)")
+      checkColType(spark.table("t").schema(1), VarcharType(4))
+      sql("ALTER TABLE t CHANGE COLUMN c TYPE VARCHAR(5)")
+      checkColType(spark.table("t").schema(1), VarcharType(5))
+
+    }
+  }
+
+  test("not allow to change column from char(x) to varchar(y) type x > y") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(i STRING, c CHAR(4)) USING $format")
+      val e = intercept[AnalysisException] {
+        sql("ALTER TABLE t CHANGE COLUMN c TYPE VARCHAR(3)")
+      }
+      assert(e.getMessage contains "char(4) cannot be cast to varchar(3)")
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandTestUtils.scala
new file mode 100644
index 0000000000000..f9e26f8277d8b
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandTestUtils.scala
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import java.io.File
+
+import org.apache.commons.io.FileUtils
+import org.apache.hadoop.fs.{FileSystem, Path}
+import org.scalactic.source.Position
+import org.scalatest.Tag
+
+import org.apache.spark.sql.{QueryTest, Row}
+import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
+import org.apache.spark.sql.execution.datasources.PartitioningUtils
+import org.apache.spark.sql.test.SQLTestUtils
+
+/**
+ * The common settings and utility functions for all v1 and v2 test suites. When a function
+ * is not applicable to all supported catalogs, it should be placed to a specific trait:
+ *
+ *   - V1 In-Memory catalog: `org.apache.spark.sql.execution.command.v1.CommandSuiteBase`
+ *   - V1 Hive External catalog: `org.apache.spark.sql.hive.execution.command.CommandSuiteBase`
+ *   - V2 In-Memory catalog: `org.apache.spark.sql.execution.command.v2.CommandSuiteBase`
+ */
+trait DDLCommandTestUtils extends SQLTestUtils {
+  // The version of the catalog under testing such as "V1", "V2", "Hive V1".
+  protected def version: String
+  // Name of the command as SQL statement, for instance "SHOW PARTITIONS"
+  protected def command: String
+  // The catalog name which can be used in SQL statements under testing
+  protected def catalog: String
+  // The clause is used in creating tables for testing
+  protected def defaultUsing: String
+
+  // Overrides the `test` method, and adds a prefix to easily find identify the catalog to which
+  // the failed test in logs belongs to.
+  override def test(testName: String, testTags: Tag*)(testFun: => Any)
+    (implicit pos: Position): Unit = {
+    super.test(s"$command $version: " + testName, testTags: _*)(testFun)
+  }
+
+  protected def withNamespaceAndTable(ns: String, tableName: String, cat: String = catalog)
+      (f: String => Unit): Unit = {
+    val nsCat = s"$cat.$ns"
+    withNamespace(nsCat) {
+      sql(s"CREATE NAMESPACE $nsCat")
+      val t = s"$nsCat.$tableName"
+      withTable(t) {
+        f(t)
+      }
+    }
+  }
+
+  // Checks that the table `t` contains only the `expected` partitions.
+  protected def checkPartitions(t: String, expected: Map[String, String]*): Unit = {
+    val partitions = sql(s"SHOW PARTITIONS $t")
+      .collect()
+      .toSet
+      .map((row: Row) => row.getString(0))
+      .map(PartitioningUtils.parsePathFragment)
+    assert(partitions === expected.toSet)
+  }
+
+  protected def createWideTable(table: String): Unit = {
+    sql(s"""
+      |CREATE TABLE $table (
+      |  price int, qty int,
+      |  year int, month int, hour int, minute int, sec int, extra int)
+      |$defaultUsing
+      |PARTITIONED BY (year, month, hour, minute, sec, extra)
+      |""".stripMargin)
+    sql(s"""
+      |INSERT INTO $table
+      |PARTITION(year = 2016, month = 3, hour = 10, minute = 10, sec = 10, extra = 1) SELECT 3, 3
+      |""".stripMargin)
+    sql(s"""
+      |ALTER TABLE $table
+      |ADD PARTITION(year = 2016, month = 4, hour = 10, minute = 10, sec = 10, extra = 1)
+      |""".stripMargin)
+  }
+
+  protected def checkLocation(t: String, spec: TablePartitionSpec, expected: String): Unit
+
+  // Getting the total table size in the filesystem in bytes
+  def getTableSize(tableName: String): Int = {
+    val stats =
+      sql(s"DESCRIBE TABLE EXTENDED $tableName")
+        .where("col_name = 'Statistics'")
+        .select("data_type")
+    if (stats.isEmpty) {
+      throw new IllegalArgumentException(s"The table $tableName does not have stats")
+    }
+    val tableSizeInStats = "^(\\d+) bytes.*$".r
+    val size = stats.first().getString(0) match {
+      case tableSizeInStats(s) => s.toInt
+      case _ => throw new IllegalArgumentException("Not found table size in stats")
+    }
+    size
+  }
+
+  def partSpecToString(spec: Map[String, Any]): String = {
+    spec.map {
+      case (k, v: String) => s"$k = '$v'"
+      case (k, v) => s"$k = $v"
+    }.mkString("PARTITION (", ", ", ")")
+  }
+
+  def cacheRelation(name: String): Unit = {
+    assert(!spark.catalog.isCached(name))
+    sql(s"CACHE TABLE $name")
+    assert(spark.catalog.isCached(name))
+  }
+
+  def checkCachedRelation(name: String, expected: Seq[Row]): Unit = {
+    assert(spark.catalog.isCached(name))
+    QueryTest.checkAnswer(sql(s"SELECT * FROM $name"), expected)
+  }
+
+  def checkTables(namespace: String, expectedTables: String*): Unit = {
+    val tables = sql(s"SHOW TABLES IN $catalog.$namespace").select("tableName")
+    val rows = expectedTables.map(Row(_))
+    QueryTest.checkAnswer(tables, rows)
+  }
+
+  def withTableDir(tableName: String)(f: (FileSystem, Path) => Unit): Unit = {
+    val location = sql(s"DESCRIBE TABLE EXTENDED $tableName")
+      .where("col_name = 'Location'")
+      .select("data_type")
+      .first()
+      .getString(0)
+    val root = new Path(location)
+    val fs = root.getFileSystem(spark.sessionState.newHadoopConf())
+    f(fs, root)
+  }
+
+  def getPartitionLocation(tableName: String, part: String): String = {
+    val idents = tableName.split('.')
+    val table = idents.last
+    val catalogAndNs = idents.init
+    val in = if (catalogAndNs.isEmpty) "" else s"IN ${catalogAndNs.mkString(".")}"
+    val information = sql(s"SHOW TABLE EXTENDED $in LIKE '$table' PARTITION ($part)")
+      .select("information")
+      .first().getString(0)
+    information
+      .split("\\r?\\n")
+      .filter(_.startsWith("Location:"))
+      .head
+      .replace("Location: file:", "")
+  }
+
+  def copyPartition(tableName: String, from: String, to: String): String = {
+    val part0Loc = getPartitionLocation(tableName, from)
+    val part1Loc = part0Loc.replace(from, to)
+    FileUtils.copyDirectory(new File(part0Loc), new File(part1Loc))
+    part1Loc
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala
index c6a533dfae4d0..6c337e3e8275c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala
@@ -17,30 +17,21 @@
 
 package org.apache.spark.sql.execution.command
 
-import java.net.URI
 import java.util.Locale
 
-import scala.reflect.{classTag, ClassTag}
-
-import org.apache.spark.sql.{AnalysisException, SaveMode}
-import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, UnresolvedAttribute}
-import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans
 import org.apache.spark.sql.catalyst.dsl.plans.DslLogicalPlan
 import org.apache.spark.sql.catalyst.expressions.JsonTuple
 import org.apache.spark.sql.catalyst.parser.ParseException
 import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.connector.expressions.{FieldReference, IdentityTransform}
 import org.apache.spark.sql.execution.SparkSqlParser
-import org.apache.spark.sql.execution.datasources.CreateTable
-import org.apache.spark.sql.internal.{HiveSerDe, SQLConf}
 import org.apache.spark.sql.test.SharedSparkSession
-import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
 
 class DDLParserSuite extends AnalysisTest with SharedSparkSession {
-  private lazy val parser = new SparkSqlParser(new SQLConf)
+  private lazy val parser = new SparkSqlParser()
 
   private def assertUnsupported(sql: String, containsThesePhrases: Seq[String] = Seq()): Unit = {
     val e = intercept[ParseException] {
@@ -52,159 +43,17 @@ class DDLParserSuite extends AnalysisTest with SharedSparkSession {
     }
   }
 
-  private def intercept(sqlCommand: String, messages: String*): Unit =
-    interceptParseException(parser.parsePlan)(sqlCommand, messages: _*)
-
-  private def parseAs[T: ClassTag](query: String): T = {
-    parser.parsePlan(query) match {
-      case t: T => t
-      case other =>
-        fail(s"Expected to parse ${classTag[T].runtimeClass} from query," +
-          s"got ${other.getClass.getName}: $query")
-    }
-  }
-
   private def compareTransformQuery(sql: String, expected: LogicalPlan): Unit = {
     val plan = parser.parsePlan(sql).asInstanceOf[ScriptTransformation].copy(ioschema = null)
     comparePlans(plan, expected, checkAnalysis = false)
   }
 
-  private def extractTableDesc(sql: String): (CatalogTable, Boolean) = {
-    parser.parsePlan(sql).collect {
-      case CreateTable(tableDesc, mode, _) => (tableDesc, mode == SaveMode.Ignore)
-    }.head
-  }
-
   test("alter database - property values must be set") {
     assertUnsupported(
       sql = "ALTER DATABASE my_db SET DBPROPERTIES('key_without_value', 'key_with_value'='x')",
       containsThesePhrases = Seq("key_without_value"))
   }
 
-  test("create hive table - table file format") {
-    val allSources = Seq("parquet", "parquetfile", "orc", "orcfile", "avro", "avrofile",
-      "sequencefile", "rcfile", "textfile")
-
-    allSources.foreach { s =>
-      val query = s"CREATE TABLE my_tab STORED AS $s"
-      val ct = parseAs[CreateTable](query)
-      val hiveSerde = HiveSerDe.sourceToSerDe(s)
-      assert(hiveSerde.isDefined)
-      assert(ct.tableDesc.storage.serde ==
-        hiveSerde.get.serde.orElse(Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe")))
-      assert(ct.tableDesc.storage.inputFormat == hiveSerde.get.inputFormat)
-      assert(ct.tableDesc.storage.outputFormat == hiveSerde.get.outputFormat)
-    }
-  }
-
-  test("create hive table - row format and table file format") {
-    val createTableStart = "CREATE TABLE my_tab ROW FORMAT"
-    val fileFormat = s"STORED AS INPUTFORMAT 'inputfmt' OUTPUTFORMAT 'outputfmt'"
-    val query1 = s"$createTableStart SERDE 'anything' $fileFormat"
-    val query2 = s"$createTableStart DELIMITED FIELDS TERMINATED BY ' ' $fileFormat"
-
-    // No conflicting serdes here, OK
-    val parsed1 = parseAs[CreateTable](query1)
-    assert(parsed1.tableDesc.storage.serde == Some("anything"))
-    assert(parsed1.tableDesc.storage.inputFormat == Some("inputfmt"))
-    assert(parsed1.tableDesc.storage.outputFormat == Some("outputfmt"))
-
-    val parsed2 = parseAs[CreateTable](query2)
-    assert(parsed2.tableDesc.storage.serde ==
-      Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"))
-    assert(parsed2.tableDesc.storage.inputFormat == Some("inputfmt"))
-    assert(parsed2.tableDesc.storage.outputFormat == Some("outputfmt"))
-  }
-
-  test("create hive table - row format serde and generic file format") {
-    val allSources = Seq("parquet", "orc", "avro", "sequencefile", "rcfile", "textfile")
-    val supportedSources = Set("sequencefile", "rcfile", "textfile")
-
-    allSources.foreach { s =>
-      val query = s"CREATE TABLE my_tab ROW FORMAT SERDE 'anything' STORED AS $s"
-      if (supportedSources.contains(s)) {
-        val ct = parseAs[CreateTable](query)
-        val hiveSerde = HiveSerDe.sourceToSerDe(s)
-        assert(hiveSerde.isDefined)
-        assert(ct.tableDesc.storage.serde == Some("anything"))
-        assert(ct.tableDesc.storage.inputFormat == hiveSerde.get.inputFormat)
-        assert(ct.tableDesc.storage.outputFormat == hiveSerde.get.outputFormat)
-      } else {
-        assertUnsupported(query, Seq("row format serde", "incompatible", s))
-      }
-    }
-  }
-
-  test("create hive table - row format delimited and generic file format") {
-    val allSources = Seq("parquet", "orc", "avro", "sequencefile", "rcfile", "textfile")
-    val supportedSources = Set("textfile")
-
-    allSources.foreach { s =>
-      val query = s"CREATE TABLE my_tab ROW FORMAT DELIMITED FIELDS TERMINATED BY ' ' STORED AS $s"
-      if (supportedSources.contains(s)) {
-        val ct = parseAs[CreateTable](query)
-        val hiveSerde = HiveSerDe.sourceToSerDe(s)
-        assert(hiveSerde.isDefined)
-        assert(ct.tableDesc.storage.serde ==
-          hiveSerde.get.serde.orElse(Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe")))
-        assert(ct.tableDesc.storage.inputFormat == hiveSerde.get.inputFormat)
-        assert(ct.tableDesc.storage.outputFormat == hiveSerde.get.outputFormat)
-      } else {
-        assertUnsupported(query, Seq("row format delimited", "only compatible with 'textfile'", s))
-      }
-    }
-  }
-
-  test("create hive external table - location must be specified") {
-    assertUnsupported(
-      sql = "CREATE EXTERNAL TABLE my_tab STORED AS parquet",
-      containsThesePhrases = Seq("create external table", "location"))
-    val query = "CREATE EXTERNAL TABLE my_tab STORED AS parquet LOCATION '/something/anything'"
-    val ct = parseAs[CreateTable](query)
-    assert(ct.tableDesc.tableType == CatalogTableType.EXTERNAL)
-    assert(ct.tableDesc.storage.locationUri == Some(new URI("/something/anything")))
-  }
-
-  test("create hive table - property values must be set") {
-    assertUnsupported(
-      sql = "CREATE TABLE my_tab STORED AS parquet " +
-        "TBLPROPERTIES('key_without_value', 'key_with_value'='x')",
-      containsThesePhrases = Seq("key_without_value"))
-    assertUnsupported(
-      sql = "CREATE TABLE my_tab ROW FORMAT SERDE 'serde' " +
-        "WITH SERDEPROPERTIES('key_without_value', 'key_with_value'='x')",
-      containsThesePhrases = Seq("key_without_value"))
-  }
-
-  test("create hive table - location implies external") {
-    val query = "CREATE TABLE my_tab STORED AS parquet LOCATION '/something/anything'"
-    val ct = parseAs[CreateTable](query)
-    assert(ct.tableDesc.tableType == CatalogTableType.EXTERNAL)
-    assert(ct.tableDesc.storage.locationUri == Some(new URI("/something/anything")))
-  }
-
-  test("Duplicate clauses - create hive table") {
-    def createTableHeader(duplicateClause: String): String = {
-      s"CREATE TABLE my_tab(a INT, b STRING) STORED AS parquet $duplicateClause $duplicateClause"
-    }
-
-    intercept(createTableHeader("TBLPROPERTIES('test' = 'test2')"),
-      "Found duplicate clauses: TBLPROPERTIES")
-    intercept(createTableHeader("LOCATION '/tmp/file'"),
-      "Found duplicate clauses: LOCATION")
-    intercept(createTableHeader("COMMENT 'a table'"),
-      "Found duplicate clauses: COMMENT")
-    intercept(createTableHeader("CLUSTERED BY(b) INTO 256 BUCKETS"),
-      "Found duplicate clauses: CLUSTERED BY")
-    intercept(createTableHeader("PARTITIONED BY (k int)"),
-      "Found duplicate clauses: PARTITIONED BY")
-    intercept(createTableHeader("STORED AS parquet"),
-      "Found duplicate clauses: STORED AS/BY")
-    intercept(
-      createTableHeader("ROW FORMAT SERDE 'parquet.hive.serde.ParquetHiveSerDe'"),
-      "Found duplicate clauses: ROW FORMAT")
-  }
-
   test("insert overwrite directory") {
     val v1 = "INSERT OVERWRITE DIRECTORY '/tmp/file' USING parquet SELECT 1 as a"
     parser.parsePlan(v1) match {
@@ -361,188 +210,6 @@ class DDLParserSuite extends AnalysisTest with SharedSparkSession {
     assert(e.contains("Found duplicate keys 'a'"))
   }
 
-  test("empty values in non-optional partition specs") {
-    val e = intercept[ParseException] {
-      parser.parsePlan(
-        "SHOW PARTITIONS dbx.tab1 PARTITION (a='1', b)")
-    }.getMessage
-    assert(e.contains("Found an empty partition key 'b'"))
-  }
-
-  test("Test CTAS #1") {
-    val s1 =
-      """
-        |CREATE EXTERNAL TABLE IF NOT EXISTS mydb.page_view
-        |COMMENT 'This is the staging page view table'
-        |STORED AS RCFILE
-        |LOCATION '/user/external/page_view'
-        |TBLPROPERTIES ('p1'='v1', 'p2'='v2')
-        |AS SELECT * FROM src
-      """.stripMargin
-
-    val s2 =
-      """
-        |CREATE EXTERNAL TABLE IF NOT EXISTS mydb.page_view
-        |STORED AS RCFILE
-        |COMMENT 'This is the staging page view table'
-        |TBLPROPERTIES ('p1'='v1', 'p2'='v2')
-        |LOCATION '/user/external/page_view'
-        |AS SELECT * FROM src
-      """.stripMargin
-
-    val s3 =
-      """
-        |CREATE EXTERNAL TABLE IF NOT EXISTS mydb.page_view
-        |TBLPROPERTIES ('p1'='v1', 'p2'='v2')
-        |LOCATION '/user/external/page_view'
-        |STORED AS RCFILE
-        |COMMENT 'This is the staging page view table'
-        |AS SELECT * FROM src
-      """.stripMargin
-
-    checkParsing(s1)
-    checkParsing(s2)
-    checkParsing(s3)
-
-    def checkParsing(sql: String): Unit = {
-      val (desc, exists) = extractTableDesc(sql)
-      assert(exists)
-      assert(desc.identifier.database == Some("mydb"))
-      assert(desc.identifier.table == "page_view")
-      assert(desc.tableType == CatalogTableType.EXTERNAL)
-      assert(desc.storage.locationUri == Some(new URI("/user/external/page_view")))
-      assert(desc.schema.isEmpty) // will be populated later when the table is actually created
-      assert(desc.comment == Some("This is the staging page view table"))
-      // TODO will be SQLText
-      assert(desc.viewText.isEmpty)
-      assert(desc.viewCatalogAndNamespace.isEmpty)
-      assert(desc.viewQueryColumnNames.isEmpty)
-      assert(desc.partitionColumnNames.isEmpty)
-      assert(desc.storage.inputFormat == Some("org.apache.hadoop.hive.ql.io.RCFileInputFormat"))
-      assert(desc.storage.outputFormat == Some("org.apache.hadoop.hive.ql.io.RCFileOutputFormat"))
-      assert(desc.storage.serde ==
-        Some("org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe"))
-      assert(desc.properties == Map("p1" -> "v1", "p2" -> "v2"))
-    }
-  }
-
-  test("Test CTAS #2") {
-    val s1 =
-      """
-        |CREATE EXTERNAL TABLE IF NOT EXISTS mydb.page_view
-        |COMMENT 'This is the staging page view table'
-        |ROW FORMAT SERDE 'parquet.hive.serde.ParquetHiveSerDe'
-        | STORED AS
-        | INPUTFORMAT 'parquet.hive.DeprecatedParquetInputFormat'
-        | OUTPUTFORMAT 'parquet.hive.DeprecatedParquetOutputFormat'
-        |LOCATION '/user/external/page_view'
-        |TBLPROPERTIES ('p1'='v1', 'p2'='v2')
-        |AS SELECT * FROM src
-      """.stripMargin
-
-    val s2 =
-      """
-        |CREATE EXTERNAL TABLE IF NOT EXISTS mydb.page_view
-        |LOCATION '/user/external/page_view'
-        |TBLPROPERTIES ('p1'='v1', 'p2'='v2')
-        |ROW FORMAT SERDE 'parquet.hive.serde.ParquetHiveSerDe'
-        | STORED AS
-        | INPUTFORMAT 'parquet.hive.DeprecatedParquetInputFormat'
-        | OUTPUTFORMAT 'parquet.hive.DeprecatedParquetOutputFormat'
-        |COMMENT 'This is the staging page view table'
-        |AS SELECT * FROM src
-      """.stripMargin
-
-    checkParsing(s1)
-    checkParsing(s2)
-
-    def checkParsing(sql: String): Unit = {
-      val (desc, exists) = extractTableDesc(sql)
-      assert(exists)
-      assert(desc.identifier.database == Some("mydb"))
-      assert(desc.identifier.table == "page_view")
-      assert(desc.tableType == CatalogTableType.EXTERNAL)
-      assert(desc.storage.locationUri == Some(new URI("/user/external/page_view")))
-      assert(desc.schema.isEmpty) // will be populated later when the table is actually created
-      // TODO will be SQLText
-      assert(desc.comment == Some("This is the staging page view table"))
-      assert(desc.viewText.isEmpty)
-      assert(desc.viewCatalogAndNamespace.isEmpty)
-      assert(desc.viewQueryColumnNames.isEmpty)
-      assert(desc.partitionColumnNames.isEmpty)
-      assert(desc.storage.properties == Map())
-      assert(desc.storage.inputFormat == Some("parquet.hive.DeprecatedParquetInputFormat"))
-      assert(desc.storage.outputFormat == Some("parquet.hive.DeprecatedParquetOutputFormat"))
-      assert(desc.storage.serde == Some("parquet.hive.serde.ParquetHiveSerDe"))
-      assert(desc.properties == Map("p1" -> "v1", "p2" -> "v2"))
-    }
-  }
-
-  test("Test CTAS #3") {
-    val s3 = """CREATE TABLE page_view AS SELECT * FROM src"""
-    val (desc, exists) = extractTableDesc(s3)
-    assert(exists == false)
-    assert(desc.identifier.database == None)
-    assert(desc.identifier.table == "page_view")
-    assert(desc.tableType == CatalogTableType.MANAGED)
-    assert(desc.storage.locationUri == None)
-    assert(desc.schema.isEmpty)
-    assert(desc.viewText == None) // TODO will be SQLText
-    assert(desc.viewQueryColumnNames.isEmpty)
-    assert(desc.storage.properties == Map())
-    assert(desc.storage.inputFormat == Some("org.apache.hadoop.mapred.TextInputFormat"))
-    assert(desc.storage.outputFormat ==
-      Some("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"))
-    assert(desc.storage.serde == Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"))
-    assert(desc.properties == Map())
-  }
-
-  test("Test CTAS #4") {
-    val s4 =
-      """CREATE TABLE page_view
-        |STORED BY 'storage.handler.class.name' AS SELECT * FROM src""".stripMargin
-    intercept[AnalysisException] {
-      extractTableDesc(s4)
-    }
-  }
-
-  test("Test CTAS #5") {
-    val s5 = """CREATE TABLE ctas2
-               | ROW FORMAT SERDE "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe"
-               | WITH SERDEPROPERTIES("serde_p1"="p1","serde_p2"="p2")
-               | STORED AS RCFile
-               | TBLPROPERTIES("tbl_p1"="p11", "tbl_p2"="p22")
-               | AS
-               |   SELECT key, value
-               |   FROM src
-               |   ORDER BY key, value""".stripMargin
-    val (desc, exists) = extractTableDesc(s5)
-    assert(exists == false)
-    assert(desc.identifier.database == None)
-    assert(desc.identifier.table == "ctas2")
-    assert(desc.tableType == CatalogTableType.MANAGED)
-    assert(desc.storage.locationUri == None)
-    assert(desc.schema.isEmpty)
-    assert(desc.viewText == None) // TODO will be SQLText
-    assert(desc.viewCatalogAndNamespace.isEmpty)
-    assert(desc.viewQueryColumnNames.isEmpty)
-    assert(desc.storage.properties == Map(("serde_p1" -> "p1"), ("serde_p2" -> "p2")))
-    assert(desc.storage.inputFormat == Some("org.apache.hadoop.hive.ql.io.RCFileInputFormat"))
-    assert(desc.storage.outputFormat == Some("org.apache.hadoop.hive.ql.io.RCFileOutputFormat"))
-    assert(desc.storage.serde == Some("org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe"))
-    assert(desc.properties == Map(("tbl_p1" -> "p11"), ("tbl_p2" -> "p22")))
-  }
-
-  test("CTAS statement with a PARTITIONED BY clause is not allowed") {
-    assertUnsupported(s"CREATE TABLE ctas1 PARTITIONED BY (k int)" +
-      " AS SELECT key, value FROM (SELECT 1 as key, 2 as value) tmp")
-  }
-
-  test("CTAS statement with schema") {
-    assertUnsupported(s"CREATE TABLE ctas1 (age INT, name STRING) AS SELECT * FROM src")
-    assertUnsupported(s"CREATE TABLE ctas1 (age INT, name STRING) AS SELECT 1, 'hello'")
-  }
-
   test("unsupported operations") {
     intercept[ParseException] {
       parser.parsePlan(
@@ -612,16 +279,16 @@ class DDLParserSuite extends AnalysisTest with SharedSparkSession {
   }
 
   test("transform query spec") {
-    val p = ScriptTransformation(
-      Seq(UnresolvedAttribute("a"), UnresolvedAttribute("b")),
-      "func", Seq.empty, plans.table("e"), null)
+    val p = Project(Seq(UnresolvedAttribute("a"), UnresolvedAttribute("b")), plans.table("e"))
+    val s = ScriptTransformation("func", Seq.empty, p, null)
 
     compareTransformQuery("select transform(a, b) using 'func' from e where f < 10",
-      p.copy(child = p.child.where('f < 10), output = Seq('key.string, 'value.string)))
+      s.copy(child = p.copy(child = p.child.where('f < 10)),
+        output = Seq('key.string, 'value.string)))
     compareTransformQuery("map a, b using 'func' as c, d from e",
-      p.copy(output = Seq('c.string, 'd.string)))
+      s.copy(output = Seq('c.string, 'd.string)))
     compareTransformQuery("reduce a, b using 'func' as (c int, d decimal(10, 0)) from e",
-      p.copy(output = Seq('c.int, 'd.decimal(10, 0))))
+      s.copy(output = Seq('c.int, 'd.decimal(10, 0))))
   }
 
   test("use backticks in output of Script Transform") {
@@ -652,205 +319,6 @@ class DDLParserSuite extends AnalysisTest with SharedSparkSession {
       """.stripMargin)
   }
 
-  test("create table - basic") {
-    val query = "CREATE TABLE my_table (id int, name string)"
-    val (desc, allowExisting) = extractTableDesc(query)
-    assert(!allowExisting)
-    assert(desc.identifier.database.isEmpty)
-    assert(desc.identifier.table == "my_table")
-    assert(desc.tableType == CatalogTableType.MANAGED)
-    assert(desc.schema == new StructType().add("id", "int").add("name", "string"))
-    assert(desc.partitionColumnNames.isEmpty)
-    assert(desc.bucketSpec.isEmpty)
-    assert(desc.viewText.isEmpty)
-    assert(desc.viewQueryColumnNames.isEmpty)
-    assert(desc.storage.locationUri.isEmpty)
-    assert(desc.storage.inputFormat ==
-      Some("org.apache.hadoop.mapred.TextInputFormat"))
-    assert(desc.storage.outputFormat ==
-      Some("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"))
-    assert(desc.storage.serde == Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"))
-    assert(desc.storage.properties.isEmpty)
-    assert(desc.properties.isEmpty)
-    assert(desc.comment.isEmpty)
-  }
-
-  test("create table - with database name") {
-    val query = "CREATE TABLE dbx.my_table (id int, name string)"
-    val (desc, _) = extractTableDesc(query)
-    assert(desc.identifier.database == Some("dbx"))
-    assert(desc.identifier.table == "my_table")
-  }
-
-  test("create table - temporary") {
-    val query = "CREATE TEMPORARY TABLE tab1 (id int, name string)"
-    val e = intercept[ParseException] { parser.parsePlan(query) }
-    assert(e.message.contains("CREATE TEMPORARY TABLE is not supported yet"))
-  }
-
-  test("create table - external") {
-    val query = "CREATE EXTERNAL TABLE tab1 (id int, name string) LOCATION '/path/to/nowhere'"
-    val (desc, _) = extractTableDesc(query)
-    assert(desc.tableType == CatalogTableType.EXTERNAL)
-    assert(desc.storage.locationUri == Some(new URI("/path/to/nowhere")))
-  }
-
-  test("create table - if not exists") {
-    val query = "CREATE TABLE IF NOT EXISTS tab1 (id int, name string)"
-    val (_, allowExisting) = extractTableDesc(query)
-    assert(allowExisting)
-  }
-
-  test("create table - comment") {
-    val query = "CREATE TABLE my_table (id int, name string) COMMENT 'its hot as hell below'"
-    val (desc, _) = extractTableDesc(query)
-    assert(desc.comment == Some("its hot as hell below"))
-  }
-
-  test("create table - partitioned columns") {
-    val query = "CREATE TABLE my_table (id int, name string) PARTITIONED BY (month int)"
-    val (desc, _) = extractTableDesc(query)
-    assert(desc.schema == new StructType()
-      .add("id", "int")
-      .add("name", "string")
-      .add("month", "int"))
-    assert(desc.partitionColumnNames == Seq("month"))
-  }
-
-  test("create table - clustered by") {
-    val numBuckets = 10
-    val bucketedColumn = "id"
-    val sortColumn = "id"
-    val baseQuery =
-      s"""
-         CREATE TABLE my_table (
-           $bucketedColumn int,
-           name string)
-         CLUSTERED BY($bucketedColumn)
-       """
-
-    val query1 = s"$baseQuery INTO $numBuckets BUCKETS"
-    val (desc1, _) = extractTableDesc(query1)
-    assert(desc1.bucketSpec.isDefined)
-    val bucketSpec1 = desc1.bucketSpec.get
-    assert(bucketSpec1.numBuckets == numBuckets)
-    assert(bucketSpec1.bucketColumnNames.head.equals(bucketedColumn))
-    assert(bucketSpec1.sortColumnNames.isEmpty)
-
-    val query2 = s"$baseQuery SORTED BY($sortColumn) INTO $numBuckets BUCKETS"
-    val (desc2, _) = extractTableDesc(query2)
-    assert(desc2.bucketSpec.isDefined)
-    val bucketSpec2 = desc2.bucketSpec.get
-    assert(bucketSpec2.numBuckets == numBuckets)
-    assert(bucketSpec2.bucketColumnNames.head.equals(bucketedColumn))
-    assert(bucketSpec2.sortColumnNames.head.equals(sortColumn))
-  }
-
-  test("create table(hive) - skewed by") {
-    val baseQuery = "CREATE TABLE my_table (id int, name string) SKEWED BY"
-    val query1 = s"$baseQuery(id) ON (1, 10, 100)"
-    val query2 = s"$baseQuery(id, name) ON ((1, 'x'), (2, 'y'), (3, 'z'))"
-    val query3 = s"$baseQuery(id, name) ON ((1, 'x'), (2, 'y'), (3, 'z')) STORED AS DIRECTORIES"
-    val e1 = intercept[ParseException] { parser.parsePlan(query1) }
-    val e2 = intercept[ParseException] { parser.parsePlan(query2) }
-    val e3 = intercept[ParseException] { parser.parsePlan(query3) }
-    assert(e1.getMessage.contains("Operation not allowed"))
-    assert(e2.getMessage.contains("Operation not allowed"))
-    assert(e3.getMessage.contains("Operation not allowed"))
-  }
-
-  test("create table(hive) - row format") {
-    val baseQuery = "CREATE TABLE my_table (id int, name string) ROW FORMAT"
-    val query1 = s"$baseQuery SERDE 'org.apache.poof.serde.Baff'"
-    val query2 = s"$baseQuery SERDE 'org.apache.poof.serde.Baff' WITH SERDEPROPERTIES ('k1'='v1')"
-    val query3 =
-      s"""
-         |$baseQuery DELIMITED FIELDS TERMINATED BY 'x' ESCAPED BY 'y'
-         |COLLECTION ITEMS TERMINATED BY 'a'
-         |MAP KEYS TERMINATED BY 'b'
-         |LINES TERMINATED BY '\n'
-         |NULL DEFINED AS 'c'
-      """.stripMargin
-    val (desc1, _) = extractTableDesc(query1)
-    val (desc2, _) = extractTableDesc(query2)
-    val (desc3, _) = extractTableDesc(query3)
-    assert(desc1.storage.serde == Some("org.apache.poof.serde.Baff"))
-    assert(desc1.storage.properties.isEmpty)
-    assert(desc2.storage.serde == Some("org.apache.poof.serde.Baff"))
-    assert(desc2.storage.properties == Map("k1" -> "v1"))
-    assert(desc3.storage.properties == Map(
-      "field.delim" -> "x",
-      "escape.delim" -> "y",
-      "serialization.format" -> "x",
-      "line.delim" -> "\n",
-      "colelction.delim" -> "a", // yes, it's a typo from Hive :)
-      "mapkey.delim" -> "b"))
-  }
-
-  test("create table(hive) - file format") {
-    val baseQuery = "CREATE TABLE my_table (id int, name string) STORED AS"
-    val query1 = s"$baseQuery INPUTFORMAT 'winput' OUTPUTFORMAT 'wowput'"
-    val query2 = s"$baseQuery ORC"
-    val (desc1, _) = extractTableDesc(query1)
-    val (desc2, _) = extractTableDesc(query2)
-    assert(desc1.storage.inputFormat == Some("winput"))
-    assert(desc1.storage.outputFormat == Some("wowput"))
-    assert(desc1.storage.serde == Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"))
-    assert(desc2.storage.inputFormat == Some("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"))
-    assert(desc2.storage.outputFormat == Some("org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat"))
-    assert(desc2.storage.serde == Some("org.apache.hadoop.hive.ql.io.orc.OrcSerde"))
-  }
-
-  test("create table(hive) - storage handler") {
-    val baseQuery = "CREATE TABLE my_table (id int, name string) STORED BY"
-    val query1 = s"$baseQuery 'org.papachi.StorageHandler'"
-    val query2 = s"$baseQuery 'org.mamachi.StorageHandler' WITH SERDEPROPERTIES ('k1'='v1')"
-    val e1 = intercept[ParseException] { parser.parsePlan(query1) }
-    val e2 = intercept[ParseException] { parser.parsePlan(query2) }
-    assert(e1.getMessage.contains("Operation not allowed"))
-    assert(e2.getMessage.contains("Operation not allowed"))
-  }
-
-  test("create table - properties") {
-    val query = "CREATE TABLE my_table (id int, name string) TBLPROPERTIES ('k1'='v1', 'k2'='v2')"
-    val (desc, _) = extractTableDesc(query)
-    assert(desc.properties == Map("k1" -> "v1", "k2" -> "v2"))
-  }
-
-  test("create table(hive) - everything!") {
-    val query =
-      """
-        |CREATE EXTERNAL TABLE IF NOT EXISTS dbx.my_table (id int, name string)
-        |COMMENT 'no comment'
-        |PARTITIONED BY (month int)
-        |ROW FORMAT SERDE 'org.apache.poof.serde.Baff' WITH SERDEPROPERTIES ('k1'='v1')
-        |STORED AS INPUTFORMAT 'winput' OUTPUTFORMAT 'wowput'
-        |LOCATION '/path/to/mercury'
-        |TBLPROPERTIES ('k1'='v1', 'k2'='v2')
-      """.stripMargin
-    val (desc, allowExisting) = extractTableDesc(query)
-    assert(allowExisting)
-    assert(desc.identifier.database == Some("dbx"))
-    assert(desc.identifier.table == "my_table")
-    assert(desc.tableType == CatalogTableType.EXTERNAL)
-    assert(desc.schema == new StructType()
-      .add("id", "int")
-      .add("name", "string")
-      .add("month", "int"))
-    assert(desc.partitionColumnNames == Seq("month"))
-    assert(desc.bucketSpec.isEmpty)
-    assert(desc.viewText.isEmpty)
-    assert(desc.viewCatalogAndNamespace.isEmpty)
-    assert(desc.viewQueryColumnNames.isEmpty)
-    assert(desc.storage.locationUri == Some(new URI("/path/to/mercury")))
-    assert(desc.storage.inputFormat == Some("winput"))
-    assert(desc.storage.outputFormat == Some("wowput"))
-    assert(desc.storage.serde == Some("org.apache.poof.serde.Baff"))
-    assert(desc.storage.properties == Map("k1" -> "v1"))
-    assert(desc.properties == Map("k1" -> "v1", "k2" -> "v2"))
-    assert(desc.comment == Some("no comment"))
-  }
-
   test("create table like") {
     val v1 = "CREATE TABLE table1 LIKE table2"
     val (target, source, fileFormat, provider, properties, exists) =
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
index 10ad8acc68937..59f321d0015e9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
@@ -22,14 +22,13 @@ import java.net.URI
 import java.util.Locale
 
 import org.apache.hadoop.fs.{Path, RawLocalFileSystem}
-import org.apache.hadoop.fs.permission.{AclEntry, AclEntryScope, AclEntryType, AclStatus, FsAction, FsPermission}
+import org.apache.hadoop.fs.permission.{AclEntry, AclStatus}
 
 import org.apache.spark.{SparkException, SparkFiles}
 import org.apache.spark.internal.config
-import org.apache.spark.internal.config.RDD_PARALLEL_LISTING_THRESHOLD
 import org.apache.spark.sql.{AnalysisException, QueryTest, Row, SaveMode}
-import org.apache.spark.sql.catalyst.{QualifiedTableName, TableIdentifier}
-import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, NoSuchDatabaseException, NoSuchPartitionException, NoSuchTableException, TempTableAlreadyExistsException}
+import org.apache.spark.sql.catalyst.{FunctionIdentifier, QualifiedTableName, TableIdentifier}
+import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, NoSuchDatabaseException, NoSuchFunctionException, TableFunctionRegistry, TempTableAlreadyExistsException}
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
 import org.apache.spark.sql.connector.catalog.SupportsNamespaces.PROP_OWNER
@@ -334,22 +333,6 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     testChangeColumn(isDatasourceTable = true)
   }
 
-  test("alter table: add partition (datasource table)") {
-    testAddPartitions(isDatasourceTable = true)
-  }
-
-  test("alter table: drop partition (datasource table)") {
-    testDropPartitions(isDatasourceTable = true)
-  }
-
-  test("alter table: rename partition (datasource table)") {
-    testRenamePartitions(isDatasourceTable = true)
-  }
-
-  test("drop table - data source table") {
-    testDropTable(isDatasourceTable = true)
-  }
-
   test("the qualified path of a database is stored in the catalog") {
     val catalog = spark.sessionState.catalog
 
@@ -497,19 +480,6 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     }
   }
 
-  test("rename a managed table with existing empty directory") {
-    withEmptyDirInTablePath("tab2") { tableLoc =>
-      withTable("tab1") {
-        sql(s"CREATE TABLE tab1 USING $dataSource AS SELECT 1, 'a'")
-        val ex = intercept[AnalysisException] {
-          sql("ALTER TABLE tab1 RENAME TO tab2")
-        }.getMessage
-        assert(ex.contains(
-          "Can not rename the managed table('`default`.`tab1`'). The associated location"))
-      }
-    }
-  }
-
   private def checkSchemaInCreatedDataSourceTable(
       path: File,
       userSpecifiedSchema: Option[String],
@@ -549,9 +519,9 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     import testImplicits._
     val df = sparkContext.parallelize(1 to 10).map(i => (i, i.toString)).toDF("num", "str")
 
-    // Case 1: with partitioning columns but no schema: Option("inexistentColumns")
+    // Case 1: with partitioning columns but no schema: Option("nonexistentColumns")
     // Case 2: without schema and partitioning columns: None
-    Seq(Option("inexistentColumns"), None).foreach { partitionCols =>
+    Seq(Option("nonexistentColumns"), None).foreach { partitionCols =>
       withTempPath { pathToPartitionedTable =>
         df.write.format("parquet").partitionBy("num")
           .save(pathToPartitionedTable.getCanonicalPath)
@@ -589,9 +559,9 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     import testImplicits._
     val df = sparkContext.parallelize(1 to 10).map(i => (i, i.toString)).toDF("num", "str")
 
-    // Case 1: with partitioning columns but no schema: Option("inexistentColumns")
+    // Case 1: with partitioning columns but no schema: Option("nonexistentColumns")
     // Case 2: without schema and partitioning columns: None
-    Seq(Option("inexistentColumns"), None).foreach { partitionCols =>
+    Seq(Option("nonexistentColumns"), None).foreach { partitionCols =>
       withTempPath { pathToNonPartitionedTable =>
         df.write.format("parquet").save(pathToNonPartitionedTable.getCanonicalPath)
         checkSchemaInCreatedDataSourceTable(
@@ -608,7 +578,7 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     import testImplicits._
     val df = sparkContext.parallelize(1 to 10).map(i => (i, i.toString)).toDF("num", "str")
 
-    // Case 1: with partitioning columns but no schema: Option("inexistentColumns")
+    // Case 1: with partitioning columns but no schema: Option("nonexistentColumns")
     // Case 2: without schema and partitioning columns: None
     Seq(Option("num"), None).foreach { partitionCols =>
       withTempPath { pathToNonPartitionedTable =>
@@ -995,55 +965,6 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     }
   }
 
-  test("alter table: rename") {
-    val catalog = spark.sessionState.catalog
-    val tableIdent1 = TableIdentifier("tab1", Some("dbx"))
-    val tableIdent2 = TableIdentifier("tab2", Some("dbx"))
-    createDatabase(catalog, "dbx")
-    createDatabase(catalog, "dby")
-    createTable(catalog, tableIdent1)
-
-    assert(catalog.listTables("dbx") == Seq(tableIdent1))
-    sql("ALTER TABLE dbx.tab1 RENAME TO dbx.tab2")
-    assert(catalog.listTables("dbx") == Seq(tableIdent2))
-
-    // The database in destination table name can be omitted, and we will use the database of source
-    // table for it.
-    sql("ALTER TABLE dbx.tab2 RENAME TO tab1")
-    assert(catalog.listTables("dbx") == Seq(tableIdent1))
-
-    catalog.setCurrentDatabase("dbx")
-    // rename without explicitly specifying database
-    sql("ALTER TABLE tab1 RENAME TO tab2")
-    assert(catalog.listTables("dbx") == Seq(tableIdent2))
-    // table to rename does not exist
-    intercept[AnalysisException] {
-      sql("ALTER TABLE dbx.does_not_exist RENAME TO dbx.tab2")
-    }
-    // destination database is different
-    intercept[AnalysisException] {
-      sql("ALTER TABLE dbx.tab1 RENAME TO dby.tab2")
-    }
-  }
-
-  test("alter table: rename cached table") {
-    import testImplicits._
-    sql("CREATE TABLE students (age INT, name STRING) USING parquet")
-    val df = (1 to 2).map { i => (i, i.toString) }.toDF("age", "name")
-    df.write.insertInto("students")
-    spark.catalog.cacheTable("students")
-    checkAnswer(spark.table("students"), df)
-    assert(spark.catalog.isCached("students"), "bad test: table was not cached in the first place")
-    sql("ALTER TABLE students RENAME TO teachers")
-    sql("CREATE TABLE students (age INT, name STRING) USING parquet")
-    // Now we have both students and teachers.
-    // The cached data for the old students table should not be read by the new students table.
-    assert(!spark.catalog.isCached("students"))
-    assert(spark.catalog.isCached("teachers"))
-    assert(spark.table("students").collect().isEmpty)
-    checkAnswer(spark.table("teachers"), df)
-  }
-
   test("rename temporary view - destination table with database name") {
     withTempView("tab1") {
       sql(
@@ -1202,79 +1123,6 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     assertUnsupported("ALTER TABLE dbx.tab1 NOT STORED AS DIRECTORIES")
   }
 
-  test("alter table: recover partitions (sequential)") {
-    withSQLConf(RDD_PARALLEL_LISTING_THRESHOLD.key -> "10") {
-      testRecoverPartitions()
-    }
-  }
-
-  test("alter table: recover partition (parallel)") {
-    withSQLConf(RDD_PARALLEL_LISTING_THRESHOLD.key -> "0") {
-      testRecoverPartitions()
-    }
-  }
-
-  protected def testRecoverPartitions(): Unit = {
-    val catalog = spark.sessionState.catalog
-    // table to alter does not exist
-    intercept[AnalysisException] {
-      sql("ALTER TABLE does_not_exist RECOVER PARTITIONS")
-    }
-
-    val tableIdent = TableIdentifier("tab1")
-    createTable(catalog, tableIdent, partitionCols = Seq("a", "b", "c"))
-    val part1 = Map("a" -> "1", "b" -> "5", "c" -> "19")
-    createTablePartition(catalog, part1, tableIdent)
-    assert(catalog.listPartitions(tableIdent).map(_.spec).toSet == Set(part1))
-
-    val part2 = Map("a" -> "2", "b" -> "6", "c" -> "31")
-    val root = new Path(catalog.getTableMetadata(tableIdent).location)
-    val fs = root.getFileSystem(spark.sessionState.newHadoopConf())
-    // valid
-    fs.mkdirs(new Path(new Path(new Path(root, "a=1"), "b=5"), "c=19"))
-    fs.createNewFile(new Path(new Path(root, "a=1/b=5/c=19"), "a.csv"))  // file
-    fs.createNewFile(new Path(new Path(root, "a=1/b=5/c=19"), "_SUCCESS"))  // file
-
-    fs.mkdirs(new Path(new Path(new Path(root, "A=2"), "B=6"), "C=31"))
-    fs.createNewFile(new Path(new Path(root, "A=2/B=6/C=31"), "b.csv"))  // file
-    fs.createNewFile(new Path(new Path(root, "A=2/B=6/C=31"), "c.csv"))  // file
-    fs.createNewFile(new Path(new Path(root, "A=2/B=6/C=31"), ".hiddenFile"))  // file
-    fs.mkdirs(new Path(new Path(root, "A=2/B=6/C=31"), "_temporary"))
-
-    val parts = (10 to 100).map { a =>
-      val part = Map("a" -> a.toString, "b" -> "5", "c" -> "42")
-      fs.mkdirs(new Path(new Path(new Path(root, s"a=$a"), "b=5"), "c=42"))
-      fs.createNewFile(new Path(new Path(root, s"a=$a/b=5/c=42"), "a.csv"))  // file
-      createTablePartition(catalog, part, tableIdent)
-      part
-    }
-
-    // invalid
-    fs.mkdirs(new Path(new Path(root, "a"), "b"))  // bad name
-    fs.mkdirs(new Path(new Path(root, "b=1"), "a=1"))  // wrong order
-    fs.mkdirs(new Path(root, "a=4")) // not enough columns
-    fs.createNewFile(new Path(new Path(root, "a=1"), "b=4"))  // file
-    fs.createNewFile(new Path(new Path(root, "a=1"), "_SUCCESS"))  // _SUCCESS
-    fs.mkdirs(new Path(new Path(root, "a=1"), "_temporary"))  // _temporary
-    fs.mkdirs(new Path(new Path(root, "a=1"), ".b=4"))  // start with .
-
-    try {
-      sql("ALTER TABLE tab1 RECOVER PARTITIONS")
-      assert(catalog.listPartitions(tableIdent).map(_.spec).toSet ==
-        Set(part1, part2) ++ parts)
-      if (!isUsingHiveMetastore) {
-        assert(catalog.getPartition(tableIdent, part1).parameters("numFiles") == "1")
-        assert(catalog.getPartition(tableIdent, part2).parameters("numFiles") == "2")
-      } else {
-        // After ALTER TABLE, the statistics of the first partition is removed by Hive megastore
-        assert(catalog.getPartition(tableIdent, part1).parameters.get("numFiles").isEmpty)
-        assert(catalog.getPartition(tableIdent, part2).parameters("numFiles") == "2")
-      }
-    } finally {
-      fs.delete(root, true)
-    }
-  }
-
   test("alter table: add partition is not supported for views") {
     assertUnsupported("ALTER VIEW dbx.tab1 ADD IF NOT EXISTS PARTITION (b='2')")
   }
@@ -1283,36 +1131,6 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     assertUnsupported("ALTER VIEW dbx.tab1 DROP IF EXISTS PARTITION (b='2')")
   }
 
-
-  test("show databases") {
-    sql("CREATE DATABASE showdb2B")
-    sql("CREATE DATABASE showdb1A")
-
-    // check the result as well as its order
-    checkDataset(sql("SHOW DATABASES"), Row("default"), Row("showdb1a"), Row("showdb2b"))
-
-    checkAnswer(
-      sql("SHOW DATABASES LIKE '*db1A'"),
-      Row("showdb1a") :: Nil)
-
-    checkAnswer(
-      sql("SHOW DATABASES '*db1A'"),
-      Row("showdb1a") :: Nil)
-
-    checkAnswer(
-      sql("SHOW DATABASES LIKE 'showdb1A'"),
-      Row("showdb1a") :: Nil)
-
-    checkAnswer(
-      sql("SHOW DATABASES LIKE '*db1A|*db2B'"),
-      Row("showdb1a") ::
-        Row("showdb2b") :: Nil)
-
-    checkAnswer(
-      sql("SHOW DATABASES LIKE 'non-existentdb'"),
-      Nil)
-  }
-
   test("drop view - temporary view") {
     val catalog = spark.sessionState.catalog
     sql(
@@ -1330,35 +1148,17 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     assert(catalog.listTables("default") == Nil)
   }
 
-  protected def testDropTable(isDatasourceTable: Boolean): Unit = {
-    if (!isUsingHiveMetastore) {
-      assert(isDatasourceTable, "InMemoryCatalog only supports data source tables")
-    }
-    val catalog = spark.sessionState.catalog
-    val tableIdent = TableIdentifier("tab1", Some("dbx"))
-    createDatabase(catalog, "dbx")
-    createTable(catalog, tableIdent, isDatasourceTable)
-    assert(catalog.listTables("dbx") == Seq(tableIdent))
-    sql("DROP TABLE dbx.tab1")
-    assert(catalog.listTables("dbx") == Nil)
-    sql("DROP TABLE IF EXISTS dbx.tab1")
-    intercept[AnalysisException] {
-      sql("DROP TABLE dbx.tab1")
-    }
-  }
-
   test("drop view") {
     val catalog = spark.sessionState.catalog
     val tableIdent = TableIdentifier("tab1", Some("dbx"))
     createDatabase(catalog, "dbx")
     createTable(catalog, tableIdent)
     assert(catalog.listTables("dbx") == Seq(tableIdent))
-
     val e = intercept[AnalysisException] {
       sql("DROP VIEW dbx.tab1")
     }
-    assert(
-      e.getMessage.contains("Cannot drop a table with DROP VIEW. Please use DROP TABLE instead"))
+    assert(e.getMessage.contains(
+      "dbx.tab1 is a table. 'DROP VIEW' expects a view. Please use DROP TABLE instead."))
   }
 
   protected def testSetProperties(isDatasourceTable: Boolean): Unit = {
@@ -1461,6 +1261,17 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     // set table partition location
     sql("ALTER TABLE dbx.tab1 PARTITION (a='1', b='2') SET LOCATION '/path/to/part/ways'")
     verifyLocation(new URI("/path/to/part/ways"), Some(partSpec))
+    // set location for partition spec in the upper case
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+      sql("ALTER TABLE dbx.tab1 PARTITION (A='1', B='2') SET LOCATION '/path/to/part/ways2'")
+      verifyLocation(new URI("/path/to/part/ways2"), Some(partSpec))
+    }
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+      val errMsg = intercept[AnalysisException] {
+        sql("ALTER TABLE dbx.tab1 PARTITION (A='1', B='2') SET LOCATION '/path/to/part/ways3'")
+      }.getMessage
+      assert(errMsg.contains("not a valid partition column"))
+    }
     // set table location without explicitly specifying database
     catalog.setCurrentDatabase("dbx")
     sql("ALTER TABLE tab1 SET LOCATION '/swanky/steak/place'")
@@ -1612,160 +1423,6 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     }
   }
 
-  protected def testAddPartitions(isDatasourceTable: Boolean): Unit = {
-    if (!isUsingHiveMetastore) {
-      assert(isDatasourceTable, "InMemoryCatalog only supports data source tables")
-    }
-    val catalog = spark.sessionState.catalog
-    val tableIdent = TableIdentifier("tab1", Some("dbx"))
-    val part1 = Map("a" -> "1", "b" -> "5")
-    val part2 = Map("a" -> "2", "b" -> "6")
-    val part3 = Map("a" -> "3", "b" -> "7")
-    val part4 = Map("a" -> "4", "b" -> "8")
-    val part5 = Map("a" -> "9", "b" -> "9")
-    createDatabase(catalog, "dbx")
-    createTable(catalog, tableIdent, isDatasourceTable)
-    createTablePartition(catalog, part1, tableIdent)
-    assert(catalog.listPartitions(tableIdent).map(_.spec).toSet == Set(part1))
-
-    // basic add partition
-    sql("ALTER TABLE dbx.tab1 ADD IF NOT EXISTS " +
-      "PARTITION (a='2', b='6') LOCATION 'paris' PARTITION (a='3', b='7')")
-    assert(catalog.listPartitions(tableIdent).map(_.spec).toSet == Set(part1, part2, part3))
-    assert(catalog.getPartition(tableIdent, part1).storage.locationUri.isDefined)
-
-    val tableLocation = catalog.getTableMetadata(tableIdent).storage.locationUri
-    assert(tableLocation.isDefined)
-    val partitionLocation = makeQualifiedPath(
-      new Path(tableLocation.get.toString, "paris").toString)
-
-    assert(catalog.getPartition(tableIdent, part2).storage.locationUri == Option(partitionLocation))
-    assert(catalog.getPartition(tableIdent, part3).storage.locationUri.isDefined)
-
-    // add partitions without explicitly specifying database
-    catalog.setCurrentDatabase("dbx")
-    sql("ALTER TABLE tab1 ADD IF NOT EXISTS PARTITION (a='4', b='8')")
-    assert(catalog.listPartitions(tableIdent).map(_.spec).toSet ==
-      Set(part1, part2, part3, part4))
-
-    // table to alter does not exist
-    intercept[AnalysisException] {
-      sql("ALTER TABLE does_not_exist ADD IF NOT EXISTS PARTITION (a='4', b='9')")
-    }
-
-    // partition to add already exists
-    intercept[AnalysisException] {
-      sql("ALTER TABLE tab1 ADD PARTITION (a='4', b='8')")
-    }
-
-    // partition to add already exists when using IF NOT EXISTS
-    sql("ALTER TABLE tab1 ADD IF NOT EXISTS PARTITION (a='4', b='8')")
-    assert(catalog.listPartitions(tableIdent).map(_.spec).toSet ==
-      Set(part1, part2, part3, part4))
-
-    // partition spec in ADD PARTITION should be case insensitive by default
-    sql("ALTER TABLE tab1 ADD PARTITION (A='9', B='9')")
-    assert(catalog.listPartitions(tableIdent).map(_.spec).toSet ==
-      Set(part1, part2, part3, part4, part5))
-  }
-
-  protected def testDropPartitions(isDatasourceTable: Boolean): Unit = {
-    if (!isUsingHiveMetastore) {
-      assert(isDatasourceTable, "InMemoryCatalog only supports data source tables")
-    }
-    val catalog = spark.sessionState.catalog
-    val tableIdent = TableIdentifier("tab1", Some("dbx"))
-    val part1 = Map("a" -> "1", "b" -> "5")
-    val part2 = Map("a" -> "2", "b" -> "6")
-    val part3 = Map("a" -> "3", "b" -> "7")
-    val part4 = Map("a" -> "4", "b" -> "8")
-    val part5 = Map("a" -> "9", "b" -> "9")
-    createDatabase(catalog, "dbx")
-    createTable(catalog, tableIdent, isDatasourceTable)
-    createTablePartition(catalog, part1, tableIdent)
-    createTablePartition(catalog, part2, tableIdent)
-    createTablePartition(catalog, part3, tableIdent)
-    createTablePartition(catalog, part4, tableIdent)
-    createTablePartition(catalog, part5, tableIdent)
-    assert(catalog.listPartitions(tableIdent).map(_.spec).toSet ==
-      Set(part1, part2, part3, part4, part5))
-
-    // basic drop partition
-    sql("ALTER TABLE dbx.tab1 DROP IF EXISTS PARTITION (a='4', b='8'), PARTITION (a='3', b='7')")
-    assert(catalog.listPartitions(tableIdent).map(_.spec).toSet == Set(part1, part2, part5))
-
-    // drop partitions without explicitly specifying database
-    catalog.setCurrentDatabase("dbx")
-    sql("ALTER TABLE tab1 DROP IF EXISTS PARTITION (a='2', b ='6')")
-    assert(catalog.listPartitions(tableIdent).map(_.spec).toSet == Set(part1, part5))
-
-    // table to alter does not exist
-    intercept[AnalysisException] {
-      sql("ALTER TABLE does_not_exist DROP IF EXISTS PARTITION (a='2')")
-    }
-
-    // partition to drop does not exist
-    intercept[AnalysisException] {
-      sql("ALTER TABLE tab1 DROP PARTITION (a='300')")
-    }
-
-    // partition to drop does not exist when using IF EXISTS
-    sql("ALTER TABLE tab1 DROP IF EXISTS PARTITION (a='300')")
-    assert(catalog.listPartitions(tableIdent).map(_.spec).toSet == Set(part1, part5))
-
-    // partition spec in DROP PARTITION should be case insensitive by default
-    sql("ALTER TABLE tab1 DROP PARTITION (A='1', B='5')")
-    assert(catalog.listPartitions(tableIdent).map(_.spec).toSet == Set(part5))
-
-    // use int literal as partition value for int type partition column
-    sql("ALTER TABLE tab1 DROP PARTITION (a=9, b=9)")
-    assert(catalog.listPartitions(tableIdent).isEmpty)
-  }
-
-  protected def testRenamePartitions(isDatasourceTable: Boolean): Unit = {
-    if (!isUsingHiveMetastore) {
-      assert(isDatasourceTable, "InMemoryCatalog only supports data source tables")
-    }
-    val catalog = spark.sessionState.catalog
-    val tableIdent = TableIdentifier("tab1", Some("dbx"))
-    val part1 = Map("a" -> "1", "b" -> "q")
-    val part2 = Map("a" -> "2", "b" -> "c")
-    val part3 = Map("a" -> "3", "b" -> "p")
-    createDatabase(catalog, "dbx")
-    createTable(catalog, tableIdent, isDatasourceTable)
-    createTablePartition(catalog, part1, tableIdent)
-    createTablePartition(catalog, part2, tableIdent)
-    createTablePartition(catalog, part3, tableIdent)
-    assert(catalog.listPartitions(tableIdent).map(_.spec).toSet == Set(part1, part2, part3))
-
-    // basic rename partition
-    sql("ALTER TABLE dbx.tab1 PARTITION (a='1', b='q') RENAME TO PARTITION (a='100', b='p')")
-    sql("ALTER TABLE dbx.tab1 PARTITION (a='2', b='c') RENAME TO PARTITION (a='20', b='c')")
-    assert(catalog.listPartitions(tableIdent).map(_.spec).toSet ==
-      Set(Map("a" -> "100", "b" -> "p"), Map("a" -> "20", "b" -> "c"), Map("a" -> "3", "b" -> "p")))
-
-    // rename without explicitly specifying database
-    catalog.setCurrentDatabase("dbx")
-    sql("ALTER TABLE tab1 PARTITION (a='100', b='p') RENAME TO PARTITION (a='10', b='p')")
-    assert(catalog.listPartitions(tableIdent).map(_.spec).toSet ==
-      Set(Map("a" -> "10", "b" -> "p"), Map("a" -> "20", "b" -> "c"), Map("a" -> "3", "b" -> "p")))
-
-    // table to alter does not exist
-    intercept[NoSuchTableException] {
-      sql("ALTER TABLE does_not_exist PARTITION (c='3') RENAME TO PARTITION (c='333')")
-    }
-
-    // partition to rename does not exist
-    intercept[NoSuchPartitionException] {
-      sql("ALTER TABLE tab1 PARTITION (a='not_found', b='1') RENAME TO PARTITION (a='1', b='2')")
-    }
-
-    // partition spec in RENAME PARTITION should be case insensitive by default
-    sql("ALTER TABLE tab1 PARTITION (A='10', B='p') RENAME TO PARTITION (A='1', B='p')")
-    assert(catalog.listPartitions(tableIdent).map(_.spec).toSet ==
-      Set(Map("a" -> "1", "b" -> "p"), Map("a" -> "20", "b" -> "c"), Map("a" -> "3", "b" -> "p")))
-  }
-
   protected def testChangeColumn(isDatasourceTable: Boolean): Unit = {
     if (!isUsingHiveMetastore) {
       assert(isDatasourceTable, "InMemoryCatalog only supports data source tables")
@@ -1853,6 +1510,7 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
             "Returns the concatenation of col1, col2, ..., colN.") :: Nil
     )
     // extended mode
+    // scalastyle:off whitespace.end.of.line
     checkAnswer(
       sql("DESCRIBE FUNCTION EXTENDED ^"),
       Row("Class: org.apache.spark.sql.catalyst.expressions.BitwiseXor") ::
@@ -1861,11 +1519,14 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
             |    Examples:
             |      > SELECT 3 ^ 5;
             |       6
-            |  """.stripMargin) ::
+            |  
+            |    Since: 1.4.0
+            |""".stripMargin) ::
         Row("Function: ^") ::
         Row("Usage: expr1 ^ expr2 - Returns the result of " +
           "bitwise exclusive OR of `expr1` and `expr2`.") :: Nil
     )
+    // scalastyle:on whitespace.end.of.line
   }
 
   test("create a data source table without schema") {
@@ -1897,7 +1558,7 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
              |OPTIONS (
              |  path '${tempDir.getCanonicalPath}'
              |)
-             |CLUSTERED BY (inexistentColumnA) SORTED BY (inexistentColumnB) INTO 2 BUCKETS
+             |CLUSTERED BY (nonexistentColumnA) SORTED BY (nonexistentColumnB) INTO 2 BUCKETS
            """.stripMargin)
         }
         assert(e.message == "Cannot specify bucketing information if the table schema is not " +
@@ -1951,160 +1612,6 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     }
   }
 
-  test("truncate table - datasource table") {
-    import testImplicits._
-
-    val data = (1 to 10).map { i => (i, i) }.toDF("width", "length")
-    // Test both a Hive compatible and incompatible code path.
-    Seq("json", "parquet").foreach { format =>
-      withTable("rectangles") {
-        data.write.format(format).saveAsTable("rectangles")
-        assert(spark.table("rectangles").collect().nonEmpty,
-          "bad test; table was empty to begin with")
-
-        sql("TRUNCATE TABLE rectangles")
-        assert(spark.table("rectangles").collect().isEmpty)
-
-        // not supported since the table is not partitioned
-        assertUnsupported("TRUNCATE TABLE rectangles PARTITION (width=1)")
-      }
-    }
-  }
-
-  test("truncate partitioned table - datasource table") {
-    import testImplicits._
-
-    val data = (1 to 10).map { i => (i % 3, i % 5, i) }.toDF("width", "length", "height")
-
-    withTable("partTable") {
-      data.write.partitionBy("width", "length").saveAsTable("partTable")
-      // supported since partitions are stored in the metastore
-      sql("TRUNCATE TABLE partTable PARTITION (width=1, length=1)")
-      assert(spark.table("partTable").filter($"width" === 1).collect().nonEmpty)
-      assert(spark.table("partTable").filter($"width" === 1 && $"length" === 1).collect().isEmpty)
-    }
-
-    withTable("partTable") {
-      data.write.partitionBy("width", "length").saveAsTable("partTable")
-      // support partial partition spec
-      sql("TRUNCATE TABLE partTable PARTITION (width=1)")
-      assert(spark.table("partTable").collect().nonEmpty)
-      assert(spark.table("partTable").filter($"width" === 1).collect().isEmpty)
-    }
-
-    withTable("partTable") {
-      data.write.partitionBy("width", "length").saveAsTable("partTable")
-      // do nothing if no partition is matched for the given partial partition spec
-      sql("TRUNCATE TABLE partTable PARTITION (width=100)")
-      assert(spark.table("partTable").count() == data.count())
-
-      // throw exception if no partition is matched for the given non-partial partition spec.
-      intercept[NoSuchPartitionException] {
-        sql("TRUNCATE TABLE partTable PARTITION (width=100, length=100)")
-      }
-
-      // throw exception if the column in partition spec is not a partition column.
-      val e = intercept[AnalysisException] {
-        sql("TRUNCATE TABLE partTable PARTITION (unknown=1)")
-      }
-      assert(e.message.contains("unknown is not a valid partition column"))
-    }
-  }
-
-  test("SPARK-30312: truncate table - keep acl/permission") {
-    import testImplicits._
-    val ignorePermissionAcl = Seq(true, false)
-
-    ignorePermissionAcl.foreach { ignore =>
-      withSQLConf(
-        "fs.file.impl" -> classOf[FakeLocalFsFileSystem].getName,
-        "fs.file.impl.disable.cache" -> "true",
-        SQLConf.TRUNCATE_TABLE_IGNORE_PERMISSION_ACL.key -> ignore.toString) {
-        withTable("tab1") {
-          sql("CREATE TABLE tab1 (col INT) USING parquet")
-          sql("INSERT INTO tab1 SELECT 1")
-          checkAnswer(spark.table("tab1"), Row(1))
-
-          val tablePath = new Path(spark.sessionState.catalog
-            .getTableMetadata(TableIdentifier("tab1")).storage.locationUri.get)
-
-          val hadoopConf = spark.sessionState.newHadoopConf()
-          val fs = tablePath.getFileSystem(hadoopConf)
-          val fileStatus = fs.getFileStatus(tablePath);
-
-          fs.setPermission(tablePath, new FsPermission("777"))
-          assert(fileStatus.getPermission().toString() == "rwxrwxrwx")
-
-          // Set ACL to table path.
-          val customAcl = new java.util.ArrayList[AclEntry]()
-          customAcl.add(new AclEntry.Builder()
-            .setName("test")
-            .setType(AclEntryType.USER)
-            .setScope(AclEntryScope.ACCESS)
-            .setPermission(FsAction.READ).build())
-          fs.setAcl(tablePath, customAcl)
-          assert(fs.getAclStatus(tablePath).getEntries().get(0) == customAcl.get(0))
-
-          sql("TRUNCATE TABLE tab1")
-          assert(spark.table("tab1").collect().isEmpty)
-
-          val fileStatus2 = fs.getFileStatus(tablePath)
-          if (ignore) {
-            assert(fileStatus2.getPermission().toString() != "rwxrwxrwx")
-          } else {
-            assert(fileStatus2.getPermission().toString() == "rwxrwxrwx")
-          }
-          val aclEntries = fs.getAclStatus(tablePath).getEntries()
-          if (ignore) {
-            assert(aclEntries.size() == 0)
-          } else {
-            assert(aclEntries.size() == 4)
-            assert(aclEntries.get(0) == customAcl.get(0))
-
-            // Setting ACLs will also set user/group/other permissions
-            // as ACL entries.
-            val user = new AclEntry.Builder()
-              .setType(AclEntryType.USER)
-              .setScope(AclEntryScope.ACCESS)
-              .setPermission(FsAction.ALL).build()
-            val group = new AclEntry.Builder()
-              .setType(AclEntryType.GROUP)
-              .setScope(AclEntryScope.ACCESS)
-              .setPermission(FsAction.ALL).build()
-            val other = new AclEntry.Builder()
-              .setType(AclEntryType.OTHER)
-              .setScope(AclEntryScope.ACCESS)
-              .setPermission(FsAction.ALL).build()
-            assert(aclEntries.get(1) == user)
-            assert(aclEntries.get(2) == group)
-            assert(aclEntries.get(3) == other)
-          }
-        }
-      }
-    }
-  }
-
-  test("SPARK-31163: acl/permission should handle non-existed path when truncating table") {
-    withSQLConf(SQLConf.TRUNCATE_TABLE_IGNORE_PERMISSION_ACL.key -> "false") {
-      withTable("tab1") {
-        sql("CREATE TABLE tab1 (col1 STRING, col2 INT) USING parquet PARTITIONED BY (col2)")
-        sql("INSERT INTO tab1 SELECT 'one', 1")
-        checkAnswer(spark.table("tab1"), Row("one", 1))
-        val part = spark.sessionState.catalog.listPartitions(TableIdentifier("tab1")).head
-        val path = new File(part.location.getPath)
-        sql("TRUNCATE TABLE tab1")
-        // simulate incomplete/unsuccessful truncate
-        assert(path.exists())
-        path.delete()
-        assert(!path.exists())
-        // execute without java.io.FileNotFoundException
-        sql("TRUNCATE TABLE tab1")
-        // partition path should be re-created
-        assert(path.exists())
-      }
-    }
-  }
-
   test("create temporary view with mismatched schema") {
     withTable("tab1") {
       spark.range(10).write.saveAsTable("tab1")
@@ -2148,31 +1655,6 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     }
   }
 
-  test("truncate table - external table, temporary table, view (not allowed)") {
-    import testImplicits._
-    withTempPath { tempDir =>
-      withTable("my_ext_tab") {
-        (("a", "b") :: Nil).toDF().write.parquet(tempDir.getCanonicalPath)
-        (1 to 10).map { i => (i, i) }.toDF("a", "b").createTempView("my_temp_tab")
-        sql(s"CREATE TABLE my_ext_tab using parquet LOCATION '${tempDir.toURI}'")
-        sql(s"CREATE VIEW my_view AS SELECT 1")
-        intercept[NoSuchTableException] {
-          sql("TRUNCATE TABLE my_temp_tab")
-        }
-        assertUnsupported("TRUNCATE TABLE my_ext_tab")
-        assertUnsupported("TRUNCATE TABLE my_view")
-      }
-    }
-  }
-
-  test("truncate table - non-partitioned table (not allowed)") {
-    withTable("my_tab") {
-      sql("CREATE TABLE my_tab (age INT, name STRING) using parquet")
-      sql("INSERT INTO my_tab values (10, 'a')")
-      assertUnsupported("TRUNCATE TABLE my_tab PARTITION (age=10)")
-    }
-  }
-
   test("SPARK-16034 Partition columns should match when appending to existing data source tables") {
     import testImplicits._
     val df = Seq((1, 2, 3)).toDF("a", "b", "c")
@@ -2217,6 +1699,7 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
   test("show functions") {
     withUserDefinedFunction("add_one" -> true) {
       val numFunctions = FunctionRegistry.functionSet.size.toLong +
+        TableFunctionRegistry.functionSet.size.toLong +
         FunctionsCommand.virtualOperators.size.toLong
       assert(sql("show functions").count() === numFunctions)
       assert(sql("show system functions").count() === numFunctions)
@@ -2249,6 +1732,17 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     }
   }
 
+  test("show columns - invalid db name") {
+    withTable("tbl") {
+      sql("CREATE TABLE tbl(col1 int, col2 string) USING parquet ")
+      val message = intercept[AnalysisException] {
+        sql("SHOW COLUMNS IN tbl FROM a.b.c")
+      }.getMessage
+      assert(message.contains(
+        "Table or view not found: a.b.c.tbl"))
+    }
+  }
+
   test("SPARK-18009 calling toLocalIterator on commands") {
     import scala.collection.JavaConverters._
     val df = sql("show databases")
@@ -3020,6 +2514,77 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
       }
     }
   }
+
+  test("REFRESH FUNCTION") {
+    val msg = intercept[AnalysisException] {
+      sql("REFRESH FUNCTION md5")
+    }.getMessage
+    assert(msg.contains("Cannot refresh built-in function"))
+    val msg2 = intercept[NoSuchFunctionException] {
+      sql("REFRESH FUNCTION default.md5")
+    }.getMessage
+    assert(msg2.contains(s"Undefined function: 'md5'. This function is neither a registered " +
+      s"temporary function nor a permanent function registered in the database 'default'."))
+
+    withUserDefinedFunction("func1" -> true) {
+      sql("CREATE TEMPORARY FUNCTION func1 AS 'test.org.apache.spark.sql.MyDoubleAvg'")
+      val msg = intercept[AnalysisException] {
+        sql("REFRESH FUNCTION func1")
+      }.getMessage
+      assert(msg.contains("Cannot refresh temporary function"))
+    }
+
+    withUserDefinedFunction("func1" -> false) {
+      val func = FunctionIdentifier("func1", Some("default"))
+      assert(!spark.sessionState.catalog.isRegisteredFunction(func))
+      intercept[NoSuchFunctionException] {
+        sql("REFRESH FUNCTION func1")
+      }
+      assert(!spark.sessionState.catalog.isRegisteredFunction(func))
+
+      sql("CREATE FUNCTION func1 AS 'test.org.apache.spark.sql.MyDoubleAvg'")
+      assert(!spark.sessionState.catalog.isRegisteredFunction(func))
+      sql("REFRESH FUNCTION func1")
+      assert(spark.sessionState.catalog.isRegisteredFunction(func))
+      val msg = intercept[NoSuchFunctionException] {
+        sql("REFRESH FUNCTION func2")
+      }.getMessage
+      assert(msg.contains(s"Undefined function: 'func2'. This function is neither a registered " +
+        s"temporary function nor a permanent function registered in the database 'default'."))
+      assert(spark.sessionState.catalog.isRegisteredFunction(func))
+
+      spark.sessionState.catalog.externalCatalog.dropFunction("default", "func1")
+      assert(spark.sessionState.catalog.isRegisteredFunction(func))
+      intercept[NoSuchFunctionException] {
+        sql("REFRESH FUNCTION func1")
+      }
+      assert(!spark.sessionState.catalog.isRegisteredFunction(func))
+
+      val function = CatalogFunction(func, "test.non.exists.udf", Seq.empty)
+      spark.sessionState.catalog.createFunction(function, false)
+      assert(!spark.sessionState.catalog.isRegisteredFunction(func))
+      val err = intercept[AnalysisException] {
+        sql("REFRESH FUNCTION func1")
+      }.getMessage
+      assert(err.contains("Can not load class"))
+      assert(!spark.sessionState.catalog.isRegisteredFunction(func))
+    }
+  }
+
+  test("REFRESH FUNCTION persistent function with the same name as the built-in function") {
+    withUserDefinedFunction("default.rand" -> false) {
+      val rand = FunctionIdentifier("rand", Some("default"))
+      sql("CREATE FUNCTION rand AS 'test.org.apache.spark.sql.MyDoubleAvg'")
+      assert(!spark.sessionState.catalog.isRegisteredFunction(rand))
+      val msg = intercept[AnalysisException] {
+        sql("REFRESH FUNCTION rand")
+      }.getMessage
+      assert(msg.contains("Cannot refresh built-in function"))
+      assert(!spark.sessionState.catalog.isRegisteredFunction(rand))
+      sql("REFRESH FUNCTION default.rand")
+      assert(spark.sessionState.catalog.isRegisteredFunction(rand))
+    }
+  }
 }
 
 object FakeLocalFsFileSystem {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DropTableParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DropTableParserSuite.scala
new file mode 100644
index 0000000000000..60c7cd8dd6f8b
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DropTableParserSuite.scala
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, UnresolvedTableOrView}
+import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan
+import org.apache.spark.sql.catalyst.plans.logical.{DropTable, LogicalPlan}
+import org.apache.spark.sql.test.SharedSparkSession
+
+class DropTableParserSuite extends AnalysisTest with SharedSparkSession {
+  private def parseCompare(sql: String, expected: LogicalPlan): Unit = {
+    comparePlans(parsePlan(sql), expected, checkAnalysis = false)
+  }
+
+  test("drop table") {
+    parseCompare("DROP TABLE testcat.ns1.ns2.tbl",
+      DropTable(
+        UnresolvedTableOrView(Seq("testcat", "ns1", "ns2", "tbl"), "DROP TABLE", true),
+        ifExists = false,
+        purge = false))
+    parseCompare(s"DROP TABLE db.tab",
+      DropTable(
+        UnresolvedTableOrView(Seq("db", "tab"), "DROP TABLE", true),
+        ifExists = false,
+        purge = false))
+    parseCompare(s"DROP TABLE IF EXISTS db.tab",
+      DropTable(
+        UnresolvedTableOrView(Seq("db", "tab"), "DROP TABLE", true),
+        ifExists = true,
+        purge = false))
+    parseCompare(s"DROP TABLE tab",
+      DropTable(
+        UnresolvedTableOrView(Seq("tab"), "DROP TABLE", true), ifExists = false, purge = false))
+    parseCompare(s"DROP TABLE IF EXISTS tab",
+      DropTable(
+        UnresolvedTableOrView(Seq("tab"), "DROP TABLE", true), ifExists = true, purge = false))
+    parseCompare(s"DROP TABLE tab PURGE",
+      DropTable(
+        UnresolvedTableOrView(Seq("tab"), "DROP TABLE", true), ifExists = false, purge = true))
+    parseCompare(s"DROP TABLE IF EXISTS tab PURGE",
+      DropTable(
+        UnresolvedTableOrView(Seq("tab"), "DROP TABLE", true), ifExists = true, purge = true))
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DropTableSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DropTableSuiteBase.scala
new file mode 100644
index 0000000000000..3c9b39af8ef22
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DropTableSuiteBase.scala
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
+
+/**
+ * This base suite contains unified tests for the `DROP TABLE` command that check V1 and V2
+ * table catalogs. The tests that cannot run for all supported catalogs are located in more
+ * specific test suites:
+ *
+ *   - V2 table catalog tests: `org.apache.spark.sql.execution.command.v2.DropTableSuite`
+ *   - V1 table catalog tests: `org.apache.spark.sql.execution.command.v1.DropTableSuiteBase`
+ *     - V1 In-Memory catalog: `org.apache.spark.sql.execution.command.v1.DropTableSuite`
+ *     - V1 Hive External catalog: `org.apache.spark.sql.hive.execution.command.DropTableSuite`
+ */
+trait DropTableSuiteBase extends QueryTest with DDLCommandTestUtils {
+  override val command = "DROP TABLE"
+
+  protected def createTable(tableName: String): Unit = {
+    sql(s"CREATE TABLE $tableName (c int) $defaultUsing")
+    sql(s"INSERT INTO $tableName SELECT 0")
+  }
+
+  test("basic") {
+    withNamespace(s"$catalog.ns") {
+      sql(s"CREATE NAMESPACE $catalog.ns")
+
+      createTable(s"$catalog.ns.tbl")
+      checkTables("ns", "tbl")
+
+      sql(s"DROP TABLE $catalog.ns.tbl")
+      checkTables("ns") // no tables
+    }
+  }
+
+  test("try to drop a nonexistent table") {
+    withNamespace(s"$catalog.ns") {
+      sql(s"CREATE NAMESPACE $catalog.ns")
+      checkTables("ns") // no tables
+
+      val errMsg = intercept[AnalysisException] {
+        sql(s"DROP TABLE $catalog.ns.tbl")
+      }.getMessage
+      assert(errMsg.contains("Table or view not found"))
+    }
+  }
+
+  test("with IF EXISTS") {
+    withNamespace(s"$catalog.ns") {
+      sql(s"CREATE NAMESPACE $catalog.ns")
+
+      createTable(s"$catalog.ns.tbl")
+      checkTables("ns", "tbl")
+      sql(s"DROP TABLE IF EXISTS $catalog.ns.tbl")
+      checkTables("ns")
+
+      // It must not throw any exceptions
+      sql(s"DROP TABLE IF EXISTS $catalog.ns.tbl")
+      checkTables("ns")
+    }
+  }
+
+  test("SPARK-33174: DROP TABLE should resolve to a temporary view first") {
+    withNamespaceAndTable("ns", "t") { t =>
+      withTempView("t") {
+        sql(s"CREATE TABLE $t (id bigint) $defaultUsing")
+        sql("CREATE TEMPORARY VIEW t AS SELECT 2")
+        sql(s"USE $catalog.ns")
+        try {
+          // Check the temporary view 't' exists.
+          checkAnswer(
+            sql("SHOW TABLES FROM spark_catalog.default LIKE 't'")
+              .select("tableName", "isTemporary"),
+            Row("t", true))
+          sql("DROP TABLE t")
+          // Verify that the temporary view 't' is resolved first and dropped.
+          checkAnswer(
+            sql("SHOW TABLES FROM spark_catalog.default LIKE 't'")
+              .select("tableName", "isTemporary"),
+            Seq.empty)
+        } finally {
+          sql(s"USE spark_catalog")
+        }
+      }
+    }
+  }
+
+  test("SPARK-33305: DROP TABLE should also invalidate cache") {
+    val t = s"$catalog.ns.tbl"
+    val view = "view"
+    withNamespace(s"$catalog.ns") {
+      sql(s"CREATE NAMESPACE $catalog.ns")
+      withTempView(view, "source") {
+        val df = spark.createDataFrame(Seq((1L, "a"), (2L, "b"), (3L, "c"))).toDF("id", "data")
+        df.createOrReplaceTempView("source")
+        sql(s"CREATE TABLE $t $defaultUsing AS SELECT id, data FROM source")
+        sql(s"CACHE TABLE $view AS SELECT id FROM $t")
+        checkAnswer(sql(s"SELECT * FROM $t"), spark.table("source").collect())
+        checkAnswer(
+          sql(s"SELECT * FROM $view"),
+          spark.table("source").select("id").collect())
+
+        val oldTable = spark.table(view)
+        assert(spark.sharedState.cacheManager.lookupCachedData(oldTable).isDefined)
+        sql(s"DROP TABLE $t")
+        assert(spark.sharedState.cacheManager.lookupCachedData(oldTable).isEmpty)
+      }
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/MsckRepairTableParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/MsckRepairTableParserSuite.scala
new file mode 100644
index 0000000000000..458b3a4fc3c8d
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/MsckRepairTableParserSuite.scala
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, UnresolvedTable}
+import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan
+import org.apache.spark.sql.catalyst.plans.logical.RepairTable
+
+class MsckRepairTableParserSuite extends AnalysisTest {
+  test("repair a table") {
+    comparePlans(
+      parsePlan("MSCK REPAIR TABLE a.b.c"),
+      RepairTable(
+        UnresolvedTable(Seq("a", "b", "c"), "MSCK REPAIR TABLE", None),
+        enableAddPartitions = true,
+        enableDropPartitions = false))
+  }
+
+  test("add partitions") {
+    comparePlans(
+      parsePlan("msck repair table ns.tbl add partitions"),
+      RepairTable(
+        UnresolvedTable(
+          Seq("ns", "tbl"),
+          "MSCK REPAIR TABLE ... ADD PARTITIONS",
+          None),
+        enableAddPartitions = true,
+        enableDropPartitions = false))
+  }
+
+  test("drop partitions") {
+    comparePlans(
+      parsePlan("MSCK repair table TBL Drop Partitions"),
+      RepairTable(
+        UnresolvedTable(
+          Seq("TBL"),
+          "MSCK REPAIR TABLE ... DROP PARTITIONS",
+          None),
+        enableAddPartitions = false,
+        enableDropPartitions = true))
+  }
+
+  test("sync partitions") {
+    comparePlans(
+      parsePlan("MSCK REPAIR TABLE spark_catalog.ns.tbl SYNC PARTITIONS"),
+      RepairTable(
+        UnresolvedTable(
+          Seq("spark_catalog", "ns", "tbl"),
+          "MSCK REPAIR TABLE ... SYNC PARTITIONS",
+          None),
+        enableAddPartitions = true,
+        enableDropPartitions = true))
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/MsckRepairTableSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/MsckRepairTableSuiteBase.scala
new file mode 100644
index 0000000000000..b8b0d003a314c
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/MsckRepairTableSuiteBase.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.QueryTest
+
+/**
+ * This base suite contains unified tests for the `MSCK REPAIR TABLE` command that
+ * check V1 and V2 table catalogs. The tests that cannot run for all supported catalogs are
+ * located in more specific test suites:
+ *
+ *   - V2 table catalog tests:
+ *     `org.apache.spark.sql.execution.command.v2.MsckRepairTableSuite`
+ *   - V1 table catalog tests:
+ *     `org.apache.spark.sql.execution.command.v1.MsckRepairTableSuiteBase`
+ *     - V1 In-Memory catalog:
+ *       `org.apache.spark.sql.execution.command.v1.MsckRepairTableSuite`
+ *     - V1 Hive External catalog:
+ *       `org.apache.spark.sql.hive.execution.command.MsckRepairTableSuite`
+ */
+trait MsckRepairTableSuiteBase extends QueryTest with DDLCommandTestUtils {
+  override val command = "MSCK REPAIR TABLE"
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
index 2d6a5da6d67f7..d48264c6d1c43 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
@@ -26,19 +26,21 @@ import org.mockito.invocation.InvocationOnMock
 
 import org.apache.spark.sql.{AnalysisException, SaveMode}
 import org.apache.spark.sql.catalyst.{AliasIdentifier, TableIdentifier}
-import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, Analyzer, CTESubstitution, EmptyFunctionRegistry, NoSuchTableException, ResolveCatalogs, ResolvedTable, ResolveInlineTables, ResolveSessionCatalog, UnresolvedAttribute, UnresolvedRelation, UnresolvedStar, UnresolvedSubqueryColumnAliases, UnresolvedV2Relation}
+import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, Analyzer, EmptyFunctionRegistry, NoSuchTableException, ResolvedTable, ResolveSessionCatalog, UnresolvedAttribute, UnresolvedRelation, UnresolvedSubqueryColumnAliases, UnresolvedTable}
 import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStorageFormat, CatalogTable, CatalogTableType, InMemoryCatalog, SessionCatalog}
-import org.apache.spark.sql.catalyst.expressions.{AttributeReference, EqualTo, Expression, InSubquery, IntegerLiteral, ListQuery, StringLiteral}
-import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
-import org.apache.spark.sql.catalyst.plans.logical.{AlterTable, Assignment, CreateTableAsSelect, CreateV2Table, DeleteAction, DeleteFromTable, DescribeRelation, DropTable, InsertAction, InsertIntoStatement, LocalRelation, LogicalPlan, MergeIntoTable, OneRowRelation, Project, ShowTableProperties, SubqueryAlias, UpdateAction, UpdateTable}
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, EqualTo, Expression, InSubquery, IntegerLiteral, ListQuery, Literal, StringLiteral}
+import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, ParseException}
+import org.apache.spark.sql.catalyst.plans.logical.{AlterTable, AnalysisOnlyCommand, AppendData, Assignment, CreateTableAsSelect, CreateTableStatement, CreateV2Table, DeleteAction, DeleteFromTable, DescribeRelation, DropTable, InsertAction, LocalRelation, LogicalPlan, MergeIntoTable, OneRowRelation, Project, SetTableLocation, SetTableProperties, ShowTableProperties, SubqueryAlias, UnsetTableProperties, UpdateAction, UpdateTable}
+import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.connector.FakeV2Provider
 import org.apache.spark.sql.connector.catalog.{CatalogManager, CatalogNotFoundException, Identifier, Table, TableCapability, TableCatalog, TableChange, V1Table}
 import org.apache.spark.sql.connector.catalog.TableChange.{UpdateColumnComment, UpdateColumnType}
+import org.apache.spark.sql.connector.expressions.Transform
 import org.apache.spark.sql.execution.datasources.CreateTable
 import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.{HiveSerDe, SQLConf}
 import org.apache.spark.sql.sources.SimpleScanSource
-import org.apache.spark.sql.types.{CharType, DoubleType, HIVE_TYPE_STRING, IntegerType, LongType, MetadataBuilder, StringType, StructField, StructType}
+import org.apache.spark.sql.types.{BooleanType, CharType, DoubleType, IntegerType, LongType, StringType, StructField, StructType}
 
 class PlanResolutionSuite extends AnalysisTest {
   import CatalystSqlParser._
@@ -49,6 +51,21 @@ class PlanResolutionSuite extends AnalysisTest {
   private val table: Table = {
     val t = mock(classOf[Table])
     when(t.schema()).thenReturn(new StructType().add("i", "int").add("s", "string"))
+    when(t.partitioning()).thenReturn(Array.empty[Transform])
+    t
+  }
+
+  private val table1: Table = {
+    val t = mock(classOf[Table])
+    when(t.schema()).thenReturn(new StructType().add("s", "string").add("i", "int"))
+    when(t.partitioning()).thenReturn(Array.empty[Transform])
+    t
+  }
+
+  private val table2: Table = {
+    val t = mock(classOf[Table])
+    when(t.schema()).thenReturn(new StructType().add("i", "int").add("x", "string"))
+    when(t.partitioning()).thenReturn(Array.empty[Transform])
     t
   }
 
@@ -75,12 +92,21 @@ class PlanResolutionSuite extends AnalysisTest {
     V1Table(t)
   }
 
+  private val view: V1Table = {
+    val t = mock(classOf[CatalogTable])
+    when(t.schema).thenReturn(new StructType().add("i", "int").add("s", "string"))
+    when(t.tableType).thenReturn(CatalogTableType.VIEW)
+    when(t.provider).thenReturn(Some(v1Format))
+    V1Table(t)
+  }
+
   private val testCat: TableCatalog = {
     val newCatalog = mock(classOf[TableCatalog])
     when(newCatalog.loadTable(any())).thenAnswer((invocation: InvocationOnMock) => {
       invocation.getArgument[Identifier](0).name match {
         case "tab" => table
-        case "tab1" => table
+        case "tab1" => table1
+        case "tab2" => table2
         case name => throw new NoSuchTableException(name)
       }
     })
@@ -96,8 +122,9 @@ class PlanResolutionSuite extends AnalysisTest {
         case "v1Table1" => v1Table
         case "v1HiveTable" => v1HiveTable
         case "v2Table" => table
-        case "v2Table1" => table
+        case "v2Table1" => table1
         case "v2TableWithAcceptAnySchemaCapability" => tableWithAcceptAnySchemaCapability
+        case "view" => view
         case name => throw new NoSuchTableException(name)
       }
     })
@@ -109,7 +136,7 @@ class PlanResolutionSuite extends AnalysisTest {
     new InMemoryCatalog,
     EmptyFunctionRegistry,
     new SQLConf().copy(SQLConf.CASE_SENSITIVE -> true))
-  v1SessionCatalog.createTempView("v", LocalRelation(Nil), false)
+  createTempView(v1SessionCatalog, "v", LocalRelation(Nil), false)
 
   private val catalogManagerWithDefault = {
     val manager = mock(classOf[CatalogManager])
@@ -145,28 +172,26 @@ class PlanResolutionSuite extends AnalysisTest {
     manager
   }
 
-  def parseAndResolve(query: String, withDefault: Boolean = false): LogicalPlan = {
+  def parseAndResolve(
+      query: String,
+      withDefault: Boolean = false,
+      checkAnalysis: Boolean = false): LogicalPlan = {
     val catalogManager = if (withDefault) {
       catalogManagerWithDefault
     } else {
       catalogManagerWithoutDefault
     }
-    val analyzer = new Analyzer(catalogManager, conf)
-    // TODO: run the analyzer directly.
-    val rules = Seq(
-      CTESubstitution,
-      ResolveInlineTables(conf),
-      analyzer.ResolveRelations,
-      new ResolveCatalogs(catalogManager),
-      new ResolveSessionCatalog(catalogManager, conf, _ == Seq("v"), _ => false),
-      analyzer.ResolveTables,
-      analyzer.ResolveReferences,
-      analyzer.ResolveSubqueryColumnAliases,
-      analyzer.ResolveReferences,
-      analyzer.ResolveAlterTableChanges)
-    rules.foldLeft(parsePlan(query)) {
-      case (plan, rule) => rule.apply(plan)
+    val analyzer = new Analyzer(catalogManager) {
+      override val extendedResolutionRules: Seq[Rule[LogicalPlan]] = Seq(
+        new ResolveSessionCatalog(catalogManager))
+    }
+    // We don't check analysis here by default, as we expect the plan to be unresolved
+    // such as `CreateTable`.
+    val analyzed = analyzer.execute(CatalystSqlParser.parsePlan(query))
+    if (checkAnalysis) {
+      analyzer.checkAnalysis(analyzed)
     }
+    analyzed
   }
 
   private def parseResolveCompare(query: String, expected: LogicalPlan): Unit =
@@ -178,6 +203,16 @@ class PlanResolutionSuite extends AnalysisTest {
     }.head
   }
 
+  private def assertUnsupported(sql: String, containsThesePhrases: Seq[String] = Seq()): Unit = {
+    val e = intercept[ParseException] {
+      parsePlan(sql)
+    }
+    assert(e.getMessage.toLowerCase(Locale.ROOT).contains("operation not allowed"))
+    containsThesePhrases.foreach { p =>
+      assert(e.getMessage.toLowerCase(Locale.ROOT).contains(p.toLowerCase(Locale.ROOT)))
+    }
+  }
+
   test("create table - with partitioned by") {
     val query = "CREATE TABLE my_tab(a INT comment 'test', b STRING) " +
         "USING parquet PARTITIONED BY (a)"
@@ -428,10 +463,11 @@ class PlanResolutionSuite extends AnalysisTest {
     val expectedProperties = Map(
       "p1" -> "v1",
       "p2" -> "v2",
-      "other" -> "20",
+      "option.other" -> "20",
       "provider" -> "parquet",
       "location" -> "s3://bucket/path/to/data",
-      "comment" -> "table comment")
+      "comment" -> "table comment",
+      "other" -> "20")
 
     parseAndResolve(sql) match {
       case create: CreateV2Table =>
@@ -467,10 +503,11 @@ class PlanResolutionSuite extends AnalysisTest {
     val expectedProperties = Map(
       "p1" -> "v1",
       "p2" -> "v2",
-      "other" -> "20",
+      "option.other" -> "20",
       "provider" -> "parquet",
       "location" -> "s3://bucket/path/to/data",
-      "comment" -> "table comment")
+      "comment" -> "table comment",
+      "other" -> "20")
 
     parseAndResolve(sql, withDefault = true) match {
       case create: CreateV2Table =>
@@ -542,10 +579,11 @@ class PlanResolutionSuite extends AnalysisTest {
     val expectedProperties = Map(
       "p1" -> "v1",
       "p2" -> "v2",
-      "other" -> "20",
+      "option.other" -> "20",
       "provider" -> "parquet",
       "location" -> "s3://bucket/path/to/data",
-      "comment" -> "table comment")
+      "comment" -> "table comment",
+      "other" -> "20")
 
     parseAndResolve(sql) match {
       case ctas: CreateTableAsSelect =>
@@ -576,10 +614,11 @@ class PlanResolutionSuite extends AnalysisTest {
     val expectedProperties = Map(
       "p1" -> "v1",
       "p2" -> "v2",
-      "other" -> "20",
+      "option.other" -> "20",
       "provider" -> "parquet",
       "location" -> "s3://bucket/path/to/data",
-      "comment" -> "table comment")
+      "comment" -> "table comment",
+      "other" -> "20")
 
     parseAndResolve(sql, withDefault = true) match {
       case ctas: CreateTableAsSelect =>
@@ -630,10 +669,10 @@ class PlanResolutionSuite extends AnalysisTest {
   }
 
   test("drop table") {
-    val tableName1 = "db.tab"
-    val tableIdent1 = TableIdentifier("tab", Option("db"))
-    val tableName2 = "tab"
-    val tableIdent2 = TableIdentifier("tab", Some("default"))
+    val tableName1 = "db.v1Table"
+    val tableIdent1 = TableIdentifier("v1Table", Option("db"))
+    val tableName2 = "v1Table"
+    val tableIdent2 = TableIdentifier("v1Table", Some("default"))
 
     parseResolveCompare(s"DROP TABLE $tableName1",
       DropTableCommand(tableIdent1, ifExists = false, isView = false, purge = false))
@@ -656,13 +695,13 @@ class PlanResolutionSuite extends AnalysisTest {
     val tableIdent2 = Identifier.of(Array.empty, "tab")
 
     parseResolveCompare(s"DROP TABLE $tableName1",
-      DropTable(testCat, tableIdent1, ifExists = false))
+      DropTable(ResolvedTable.create(testCat, tableIdent1, table), ifExists = false, purge = false))
     parseResolveCompare(s"DROP TABLE IF EXISTS $tableName1",
-      DropTable(testCat, tableIdent1, ifExists = true))
+      DropTable(ResolvedTable.create(testCat, tableIdent1, table), ifExists = true, purge = false))
     parseResolveCompare(s"DROP TABLE $tableName2",
-      DropTable(testCat, tableIdent2, ifExists = false))
+      DropTable(ResolvedTable.create(testCat, tableIdent2, table), ifExists = false, purge = false))
     parseResolveCompare(s"DROP TABLE IF EXISTS $tableName2",
-      DropTable(testCat, tableIdent2, ifExists = true))
+      DropTable(ResolvedTable.create(testCat, tableIdent2, table), ifExists = true, purge = false))
   }
 
   test("drop view") {
@@ -670,6 +709,8 @@ class PlanResolutionSuite extends AnalysisTest {
     val viewIdent1 = TableIdentifier("view", Option("db"))
     val viewName2 = "view"
     val viewIdent2 = TableIdentifier("view", Option("default"))
+    val tempViewName = "v"
+    val tempViewIdent = TableIdentifier("v")
 
     parseResolveCompare(s"DROP VIEW $viewName1",
       DropTableCommand(viewIdent1, ifExists = false, isView = true, purge = false))
@@ -679,11 +720,15 @@ class PlanResolutionSuite extends AnalysisTest {
       DropTableCommand(viewIdent2, ifExists = false, isView = true, purge = false))
     parseResolveCompare(s"DROP VIEW IF EXISTS $viewName2",
       DropTableCommand(viewIdent2, ifExists = true, isView = true, purge = false))
+    parseResolveCompare(s"DROP VIEW $tempViewName",
+      DropTableCommand(tempViewIdent, ifExists = false, isView = true, purge = false))
+    parseResolveCompare(s"DROP VIEW IF EXISTS $tempViewName",
+      DropTableCommand(tempViewIdent, ifExists = true, isView = true, purge = false))
   }
 
   test("drop view in v2 catalog") {
     intercept[AnalysisException] {
-      parseAndResolve("DROP VIEW testcat.db.view")
+      parseAndResolve("DROP VIEW testcat.db.view", checkAnalysis = true)
     }.getMessage.toLowerCase(Locale.ROOT).contains(
       "view support in catalog has not been implemented")
   }
@@ -691,16 +736,16 @@ class PlanResolutionSuite extends AnalysisTest {
   // ALTER VIEW view_name SET TBLPROPERTIES ('comment' = new_comment);
   // ALTER VIEW view_name UNSET TBLPROPERTIES [IF EXISTS] ('comment', 'key');
   test("alter view: alter view properties") {
-    val sql1_view = "ALTER VIEW table_name SET TBLPROPERTIES ('test' = 'test', " +
+    val sql1_view = "ALTER VIEW view SET TBLPROPERTIES ('test' = 'test', " +
         "'comment' = 'new_comment')"
-    val sql2_view = "ALTER VIEW table_name UNSET TBLPROPERTIES ('comment', 'test')"
-    val sql3_view = "ALTER VIEW table_name UNSET TBLPROPERTIES IF EXISTS ('comment', 'test')"
+    val sql2_view = "ALTER VIEW view UNSET TBLPROPERTIES ('comment', 'test')"
+    val sql3_view = "ALTER VIEW view UNSET TBLPROPERTIES IF EXISTS ('comment', 'test')"
 
     val parsed1_view = parseAndResolve(sql1_view)
     val parsed2_view = parseAndResolve(sql2_view)
     val parsed3_view = parseAndResolve(sql3_view)
 
-    val tableIdent = TableIdentifier("table_name", Some("default"))
+    val tableIdent = TableIdentifier("view", Some("default"))
     val expected1_view = AlterTableSetPropertiesCommand(
       tableIdent, Map("test" -> "test", "comment" -> "new_comment"), isView = true)
     val expected2_view = AlterTableUnsetPropertiesCommand(
@@ -741,27 +786,23 @@ class PlanResolutionSuite extends AnalysisTest {
           comparePlans(parsed3, expected3)
         } else {
           parsed1 match {
-            case AlterTable(_, _, _: DataSourceV2Relation, changes) =>
-              assert(changes == Seq(
-                TableChange.setProperty("test", "test"),
-                TableChange.setProperty("comment", "new_comment")))
-            case _ => fail("expect AlterTable")
+            case SetTableProperties(_: ResolvedTable, properties) =>
+              assert(properties == Map(("test", "test"), ("comment", "new_comment")))
+            case _ => fail(s"expect ${SetTableProperties.getClass.getName}")
           }
 
           parsed2 match {
-            case AlterTable(_, _, _: DataSourceV2Relation, changes) =>
-              assert(changes == Seq(
-                TableChange.removeProperty("comment"),
-                TableChange.removeProperty("test")))
-            case _ => fail("expect AlterTable")
+            case UnsetTableProperties(_: ResolvedTable, propertyKeys, ifExists) =>
+              assert(propertyKeys == Seq("comment", "test"))
+              assert(!ifExists)
+            case _ => fail(s"expect ${UnsetTableProperties.getClass.getName}")
           }
 
           parsed3 match {
-            case AlterTable(_, _, _: DataSourceV2Relation, changes) =>
-              assert(changes == Seq(
-                TableChange.removeProperty("comment"),
-                TableChange.removeProperty("test")))
-            case _ => fail("expect AlterTable")
+            case UnsetTableProperties(_: ResolvedTable, propertyKeys, ifExists) =>
+              assert(propertyKeys == Seq("comment", "test"))
+              assert(ifExists)
+            case _ => fail(s"expect ${UnsetTableProperties.getClass.getName}")
           }
         }
     }
@@ -773,12 +814,14 @@ class PlanResolutionSuite extends AnalysisTest {
 
     // For non-existing tables, we convert it to v2 command with `UnresolvedV2Table`
     parsed4 match {
-      case AlterTable(_, _, _: UnresolvedV2Relation, _) => // OK
-      case _ => fail("Expect AlterTable, but got:\n" + parsed4.treeString)
+      case SetTableProperties(_: UnresolvedTable, _) => // OK
+      case _ =>
+        fail(s"Expect ${SetTableProperties.getClass.getName}, but got:\n" + parsed4.treeString)
     }
     parsed5 match {
-      case AlterTable(_, _, _: UnresolvedV2Relation, _) => // OK
-      case _ => fail("Expect AlterTable, but got:\n" + parsed5.treeString)
+      case UnsetTableProperties(_: UnresolvedTable, _, _) => // OK
+      case _ =>
+        fail(s"Expect ${UnsetTableProperties.getClass.getName}, but got:\n" + parsed5.treeString)
     }
   }
 
@@ -800,11 +843,8 @@ class PlanResolutionSuite extends AnalysisTest {
           comparePlans(parsed, expected)
         } else {
           parsed match {
-            case AlterTable(_, _, _: DataSourceV2Relation, changes) =>
-              assert(changes == Seq(
-                TableChange.setProperty("a", "1"),
-                TableChange.setProperty("b", "0.1"),
-                TableChange.setProperty("c", "true")))
+            case SetTableProperties(_: ResolvedTable, changes) =>
+              assert(changes == Map(("a", "1"), ("b", "0.1"), ("c", "true")))
             case _ => fail("Expect AlterTable, but got:\n" + parsed.treeString)
           }
         }
@@ -824,9 +864,10 @@ class PlanResolutionSuite extends AnalysisTest {
           comparePlans(parsed, expected)
         } else {
           parsed match {
-            case AlterTable(_, _, _: DataSourceV2Relation, changes) =>
-              assert(changes == Seq(TableChange.setProperty("location", "new location")))
-            case _ => fail("Expect AlterTable, but got:\n" + parsed.treeString)
+            case SetTableLocation(_: ResolvedTable, _, location) =>
+              assert(location === "new location")
+            case _ =>
+              fail(s"Expect ${SetTableLocation.getClass.getName}, but got:\n" + parsed.treeString)
           }
         }
     }
@@ -841,21 +882,21 @@ class PlanResolutionSuite extends AnalysisTest {
         val parsed2 = parseAndResolve(sql2)
         if (useV1Command) {
           val expected1 = DescribeTableCommand(
-            TableIdentifier(tblName, Some("default")), Map.empty, false)
+            TableIdentifier(tblName, Some("default")), Map.empty, false, parsed1.output)
           val expected2 = DescribeTableCommand(
-            TableIdentifier(tblName, Some("default")), Map.empty, true)
+            TableIdentifier(tblName, Some("default")), Map.empty, true, parsed2.output)
 
           comparePlans(parsed1, expected1)
           comparePlans(parsed2, expected2)
         } else {
           parsed1 match {
-            case DescribeRelation(_: ResolvedTable, _, isExtended) =>
+            case DescribeRelation(_: ResolvedTable, _, isExtended, _) =>
               assert(!isExtended)
             case _ => fail("Expect DescribeTable, but got:\n" + parsed1.treeString)
           }
 
           parsed2 match {
-            case DescribeRelation(_: ResolvedTable, _, isExtended) =>
+            case DescribeRelation(_: ResolvedTable, _, isExtended, _) =>
               assert(isExtended)
             case _ => fail("Expect DescribeTable, but got:\n" + parsed2.treeString)
           }
@@ -865,11 +906,11 @@ class PlanResolutionSuite extends AnalysisTest {
         val parsed3 = parseAndResolve(sql3)
         if (useV1Command) {
           val expected3 = DescribeTableCommand(
-            TableIdentifier(tblName, Some("default")), Map("a" -> "1"), false)
+            TableIdentifier(tblName, Some("default")), Map("a" -> "1"), false, parsed3.output)
           comparePlans(parsed3, expected3)
         } else {
           parsed3 match {
-            case DescribeRelation(_: ResolvedTable, partitionSpec, isExtended) =>
+            case DescribeRelation(_: ResolvedTable, partitionSpec, isExtended, _) =>
               assert(!isExtended)
               assert(partitionSpec == Map("a" -> "1"))
             case _ => fail("Expect DescribeTable, but got:\n" + parsed2.treeString)
@@ -1076,9 +1117,7 @@ class PlanResolutionSuite extends AnalysisTest {
     }
 
     val sql = s"ALTER TABLE v1HiveTable ALTER COLUMN i TYPE char(1)"
-    val builder = new MetadataBuilder
-    builder.putString(HIVE_TYPE_STRING, CharType(1).catalogString)
-    val newColumnWithCleanedType = StructField("i", StringType, true, builder.build())
+    val newColumnWithCleanedType = StructField("i", CharType(1), true)
     val expected = AlterTableChangeColumnCommand(
       TableIdentifier("v1HiveTable", Some("default")), "i", newColumnWithCleanedType)
     val parsed = parseAndResolve(sql)
@@ -1144,9 +1183,9 @@ class PlanResolutionSuite extends AnalysisTest {
       ("ALTER TABLE testcat.tab ALTER COLUMN i TYPE bigint", false),
       ("ALTER TABLE tab ALTER COLUMN i TYPE bigint", false),
       (s"ALTER TABLE $v2SessionCatalogTable ALTER COLUMN i TYPE bigint", true),
-      ("INSERT INTO TABLE tab VALUES (1)", false),
-      ("INSERT INTO TABLE testcat.tab VALUES (1)", false),
-      (s"INSERT INTO TABLE $v2SessionCatalogTable VALUES (1)", true),
+      ("INSERT INTO TABLE tab VALUES (1, 'a')", false),
+      ("INSERT INTO TABLE testcat.tab VALUES (1, 'a')", false),
+      (s"INSERT INTO TABLE $v2SessionCatalogTable VALUES (1, 'a')", true),
       ("DESC TABLE tab", false),
       ("DESC TABLE testcat.tab", false),
       (s"DESC TABLE $v2SessionCatalogTable", true),
@@ -1159,28 +1198,28 @@ class PlanResolutionSuite extends AnalysisTest {
     )
   }
 
-  DSV2ResolutionTests.foreach { case (sql, isSessionCatlog) =>
+  DSV2ResolutionTests.foreach { case (sql, isSessionCatalog) =>
     test(s"Data source V2 relation resolution '$sql'") {
       val parsed = parseAndResolve(sql, withDefault = true)
-      val catlogIdent = if (isSessionCatlog) v2SessionCatalog else testCat
-      val tableIdent = if (isSessionCatlog) "v2Table" else "tab"
+      val catalogIdent = if (isSessionCatalog) v2SessionCatalog else testCat
+      val tableIdent = if (isSessionCatalog) "v2Table" else "tab"
       parsed match {
         case AlterTable(_, _, r: DataSourceV2Relation, _) =>
-          assert(r.catalog.exists(_ == catlogIdent))
+          assert(r.catalog.exists(_ == catalogIdent))
           assert(r.identifier.exists(_.name() == tableIdent))
         case Project(_, AsDataSourceV2Relation(r)) =>
-          assert(r.catalog.exists(_ == catlogIdent))
+          assert(r.catalog.exists(_ == catalogIdent))
           assert(r.identifier.exists(_.name() == tableIdent))
-        case InsertIntoStatement(r: DataSourceV2Relation, _, _, _, _) =>
-          assert(r.catalog.exists(_ == catlogIdent))
+        case AppendData(r: DataSourceV2Relation, _, _, _, _) =>
+          assert(r.catalog.exists(_ == catalogIdent))
           assert(r.identifier.exists(_.name() == tableIdent))
-        case DescribeRelation(r: ResolvedTable, _, _) =>
-          assert(r.catalog == catlogIdent)
+        case DescribeRelation(r: ResolvedTable, _, _, _) =>
+          assert(r.catalog == catalogIdent)
           assert(r.identifier.name() == tableIdent)
-        case ShowTableProperties(r: ResolvedTable, _) =>
-          assert(r.catalog == catlogIdent)
+        case ShowTableProperties(r: ResolvedTable, _, _) =>
+          assert(r.catalog == catalogIdent)
           assert(r.identifier.name() == tableIdent)
-        case ShowTablePropertiesCommand(t: TableIdentifier, _) =>
+        case ShowTablePropertiesCommand(t: TableIdentifier, _, _) =>
           assert(t.identifier == tableIdent)
       }
     }
@@ -1205,6 +1244,7 @@ class PlanResolutionSuite extends AnalysisTest {
       mergeCondition match {
         case EqualTo(l: AttributeReference, r: AttributeReference) =>
           assert(l.sameRef(ti) && r.sameRef(si))
+        case Literal(_, BooleanType) => // this is acceptable as a merge condition
         case other => fail("unexpected merge condition " + other)
       }
 
@@ -1285,6 +1325,28 @@ class PlanResolutionSuite extends AnalysisTest {
           case other => fail("Expect MergeIntoTable, but got:\n" + other.treeString)
         }
 
+        // merge with star should get resolved into specific actions even if there
+        // is no other unresolved expression in the merge
+        parseAndResolve(s"""
+             |MERGE INTO $target AS target
+             |USING $source AS source
+             |ON true
+             |WHEN MATCHED THEN UPDATE SET *
+             |WHEN NOT MATCHED THEN INSERT *
+           """.stripMargin) match {
+          case MergeIntoTable(
+              SubqueryAlias(AliasIdentifier("target", Seq()), AsDataSourceV2Relation(target)),
+              SubqueryAlias(AliasIdentifier("source", Seq()), AsDataSourceV2Relation(source)),
+              mergeCondition,
+              Seq(UpdateAction(None, updateAssigns)),
+              Seq(InsertAction(None, insertAssigns))) =>
+
+            checkResolution(target, source, mergeCondition, None, None, None,
+              updateAssigns, insertAssigns, starInUpdate = true)
+
+          case other => fail("Expect MergeIntoTable, but got:\n" + other.treeString)
+        }
+
         // no additional conditions
         val sql3 =
           s"""
@@ -1338,7 +1400,7 @@ class PlanResolutionSuite extends AnalysisTest {
         // cte
         val sql5 =
           s"""
-             |WITH source(i, s) AS
+             |WITH source(s, i) AS
              | (SELECT * FROM $source)
              |MERGE INTO $target AS target
              |USING source
@@ -1358,7 +1420,7 @@ class PlanResolutionSuite extends AnalysisTest {
                   updateAssigns)),
               Seq(InsertAction(Some(EqualTo(il: AttributeReference, StringLiteral("insert"))),
                 insertAssigns))) =>
-            assert(source.output.map(_.name) == Seq("i", "s"))
+            assert(source.output.map(_.name) == Seq("s", "i"))
             checkResolution(target, source, mergeCondition, Some(dl), Some(ul), Some(il),
               updateAssigns, insertAssigns)
 
@@ -1367,8 +1429,7 @@ class PlanResolutionSuite extends AnalysisTest {
     }
 
     // no aliases
-    Seq(("v2Table", "v2Table1"),
-      ("testcat.tab", "testcat.tab1")).foreach { pair =>
+    Seq(("v2Table", "v2Table1"), ("testcat.tab", "testcat.tab1")).foreach { pair =>
 
       val target = pair._1
       val source = pair._2
@@ -1460,7 +1521,7 @@ class PlanResolutionSuite extends AnalysisTest {
       assert(e5.message.contains("Reference 's' is ambiguous"))
     }
 
-    val sql6 =
+    val sql1 =
       s"""
          |MERGE INTO non_exist_target
          |USING non_exist_source
@@ -1469,13 +1530,37 @@ class PlanResolutionSuite extends AnalysisTest {
          |WHEN MATCHED THEN UPDATE SET *
          |WHEN NOT MATCHED THEN INSERT *
        """.stripMargin
-    val parsed = parseAndResolve(sql6)
+    val parsed = parseAndResolve(sql1)
     parsed match {
       case u: MergeIntoTable =>
         assert(u.targetTable.isInstanceOf[UnresolvedRelation])
         assert(u.sourceTable.isInstanceOf[UnresolvedRelation])
       case _ => fail("Expect MergeIntoTable, but got:\n" + parsed.treeString)
     }
+
+    // UPDATE * with incompatible schema between source and target tables.
+    val sql2 =
+      """
+         |MERGE INTO testcat.tab
+         |USING testcat.tab2
+         |ON 1 = 1
+         |WHEN MATCHED THEN UPDATE SET *
+         |""".stripMargin
+    val e2 = intercept[AnalysisException](parseAndResolve(sql2))
+    assert(e2.message.contains(
+      "cannot resolve s in MERGE command given columns [testcat.tab2.i, testcat.tab2.x]"))
+
+    // INSERT * with incompatible schema between source and target tables.
+    val sql3 =
+      """
+        |MERGE INTO testcat.tab
+        |USING testcat.tab2
+        |ON 1 = 1
+        |WHEN NOT MATCHED THEN INSERT *
+        |""".stripMargin
+    val e3 = intercept[AnalysisException](parseAndResolve(sql3))
+    assert(e3.message.contains(
+      "cannot resolve s in MERGE command given columns [testcat.tab2.i, testcat.tab2.x]"))
   }
 
   test("MERGE INTO TABLE - skip resolution on v2 tables that accept any schema") {
@@ -1519,42 +1604,639 @@ class PlanResolutionSuite extends AnalysisTest {
     }
   }
 
-  test("SPARK-31147: forbid CHAR type in non-Hive tables") {
-    def checkFailure(t: String, provider: String): Unit = {
-      val types = Seq(
-        "CHAR(2)",
-        "ARRAY<CHAR(2)>",
-        "MAP<INT, CHAR(2)>",
-        "MAP<CHAR(2), INT>",
-        "STRUCT<s: CHAR(2)>")
-      types.foreach { tpe =>
-        intercept[AnalysisException] {
-          parseAndResolve(s"CREATE TABLE $t(col $tpe) USING $provider")
-        }
-        intercept[AnalysisException] {
-          parseAndResolve(s"REPLACE TABLE $t(col $tpe) USING $provider")
-        }
-        intercept[AnalysisException] {
-          parseAndResolve(s"CREATE OR REPLACE TABLE $t(col $tpe) USING $provider")
-        }
-        intercept[AnalysisException] {
-          parseAndResolve(s"ALTER TABLE $t ADD COLUMN col $tpe")
-        }
-        intercept[AnalysisException] {
-          parseAndResolve(s"ALTER TABLE $t ADD COLUMN col $tpe")
-        }
-        intercept[AnalysisException] {
-          parseAndResolve(s"ALTER TABLE $t ALTER COLUMN col TYPE $tpe")
+  private def compareNormalized(plan1: LogicalPlan, plan2: LogicalPlan): Unit = {
+    /**
+     * Normalizes plans:
+     * - CreateTable the createTime in tableDesc will replaced by -1L.
+     */
+    def normalizePlan(plan: LogicalPlan): LogicalPlan = {
+      plan match {
+        case CreateTable(tableDesc, mode, query) =>
+          val newTableDesc = tableDesc.copy(createTime = -1L)
+          CreateTable(newTableDesc, mode, query)
+        case _ => plan // Don't transform
+      }
+    }
+    comparePlans(normalizePlan(plan1), normalizePlan(plan2))
+  }
+
+  test("create table - schema") {
+    def createTable(
+        table: String,
+        database: Option[String] = None,
+        tableType: CatalogTableType = CatalogTableType.MANAGED,
+        storage: CatalogStorageFormat = CatalogStorageFormat.empty.copy(
+          inputFormat = HiveSerDe.sourceToSerDe("textfile").get.inputFormat,
+          outputFormat = HiveSerDe.sourceToSerDe("textfile").get.outputFormat,
+          serde = Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe")),
+        schema: StructType = new StructType,
+        provider: Option[String] = Some("hive"),
+        partitionColumnNames: Seq[String] = Seq.empty,
+        comment: Option[String] = None,
+        mode: SaveMode = SaveMode.ErrorIfExists,
+        query: Option[LogicalPlan] = None): CreateTable = {
+      CreateTable(
+        CatalogTable(
+          identifier = TableIdentifier(table, database),
+          tableType = tableType,
+          storage = storage,
+          schema = schema,
+          provider = provider,
+          partitionColumnNames = partitionColumnNames,
+          comment = comment
+        ), mode, query
+      )
+    }
+
+    def compare(sql: String, plan: LogicalPlan): Unit = {
+      compareNormalized(parseAndResolve(sql), plan)
+    }
+
+    compare("CREATE TABLE my_tab(a INT COMMENT 'test', b STRING) STORED AS textfile",
+      createTable(
+        table = "my_tab",
+        database = Some("default"),
+        schema = (new StructType)
+            .add("a", IntegerType, nullable = true, "test")
+            .add("b", StringType)
+      )
+    )
+    compare("CREATE TABLE my_tab(a INT COMMENT 'test', b STRING) STORED AS textfile " +
+        "PARTITIONED BY (c INT, d STRING COMMENT 'test2')",
+      createTable(
+        table = "my_tab",
+        database = Some("default"),
+        schema = (new StructType)
+            .add("a", IntegerType, nullable = true, "test")
+            .add("b", StringType)
+            .add("c", IntegerType)
+            .add("d", StringType, nullable = true, "test2"),
+        partitionColumnNames = Seq("c", "d")
+      )
+    )
+    compare("CREATE TABLE my_tab(id BIGINT, nested STRUCT<col1: STRING,col2: INT>) " +
+        "STORED AS textfile",
+      createTable(
+        table = "my_tab",
+        database = Some("default"),
+        schema = (new StructType)
+            .add("id", LongType)
+            .add("nested", (new StructType)
+                .add("col1", StringType)
+                .add("col2", IntegerType)
+            )
+      )
+    )
+    // Partitioned by a StructType should be accepted by `SparkSqlParser` but will fail an analyze
+    // rule in `AnalyzeCreateTable`.
+    compare("CREATE TABLE my_tab(a INT COMMENT 'test', b STRING) STORED AS textfile " +
+        "PARTITIONED BY (nested STRUCT<col1: STRING,col2: INT>)",
+      createTable(
+        table = "my_tab",
+        database = Some("default"),
+        schema = (new StructType)
+            .add("a", IntegerType, nullable = true, "test")
+            .add("b", StringType)
+            .add("nested", (new StructType)
+                .add("col1", StringType)
+                .add("col2", IntegerType)
+            ),
+        partitionColumnNames = Seq("nested")
+      )
+    )
+
+    interceptParseException(parsePlan)(
+      "CREATE TABLE my_tab(a: INT COMMENT 'test', b: STRING)",
+      "extraneous input ':'")
+  }
+
+  test("create hive table - table file format") {
+    val allSources = Seq("parquet", "parquetfile", "orc", "orcfile", "avro", "avrofile",
+      "sequencefile", "rcfile", "textfile")
+
+    allSources.foreach { s =>
+      val query = s"CREATE TABLE my_tab STORED AS $s"
+      parseAndResolve(query) match {
+        case ct: CreateTable =>
+          val hiveSerde = HiveSerDe.sourceToSerDe(s)
+          assert(hiveSerde.isDefined)
+          assert(ct.tableDesc.storage.serde ==
+            hiveSerde.get.serde.orElse(Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe")))
+          assert(ct.tableDesc.storage.inputFormat == hiveSerde.get.inputFormat)
+          assert(ct.tableDesc.storage.outputFormat == hiveSerde.get.outputFormat)
+      }
+    }
+  }
+
+  test("create hive table - row format and table file format") {
+    val createTableStart = "CREATE TABLE my_tab ROW FORMAT"
+    val fileFormat = s"STORED AS INPUTFORMAT 'inputfmt' OUTPUTFORMAT 'outputfmt'"
+    val query1 = s"$createTableStart SERDE 'anything' $fileFormat"
+    val query2 = s"$createTableStart DELIMITED FIELDS TERMINATED BY ' ' $fileFormat"
+
+    // No conflicting serdes here, OK
+    parseAndResolve(query1) match {
+      case parsed1: CreateTable =>
+        assert(parsed1.tableDesc.storage.serde == Some("anything"))
+        assert(parsed1.tableDesc.storage.inputFormat == Some("inputfmt"))
+        assert(parsed1.tableDesc.storage.outputFormat == Some("outputfmt"))
+    }
+
+    parseAndResolve(query2) match {
+      case parsed2: CreateTable =>
+        assert(parsed2.tableDesc.storage.serde ==
+            Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"))
+        assert(parsed2.tableDesc.storage.inputFormat == Some("inputfmt"))
+        assert(parsed2.tableDesc.storage.outputFormat == Some("outputfmt"))
+    }
+  }
+
+  test("create hive table - row format serde and generic file format") {
+    val allSources = Seq("parquet", "orc", "avro", "sequencefile", "rcfile", "textfile")
+    val supportedSources = Set("sequencefile", "rcfile", "textfile")
+
+    allSources.foreach { s =>
+      val query = s"CREATE TABLE my_tab ROW FORMAT SERDE 'anything' STORED AS $s"
+      if (supportedSources.contains(s)) {
+        parseAndResolve(query) match {
+          case ct: CreateTable =>
+            val hiveSerde = HiveSerDe.sourceToSerDe(s)
+            assert(hiveSerde.isDefined)
+            assert(ct.tableDesc.storage.serde == Some("anything"))
+            assert(ct.tableDesc.storage.inputFormat == hiveSerde.get.inputFormat)
+            assert(ct.tableDesc.storage.outputFormat == hiveSerde.get.outputFormat)
         }
-        intercept[AnalysisException] {
-          parseAndResolve(s"ALTER TABLE $t REPLACE COLUMNS (col $tpe)")
+      } else {
+        assertUnsupported(query, Seq("row format serde", "incompatible", s))
+      }
+    }
+  }
+
+  test("create hive table - row format delimited and generic file format") {
+    val allSources = Seq("parquet", "orc", "avro", "sequencefile", "rcfile", "textfile")
+    val supportedSources = Set("textfile")
+
+    allSources.foreach { s =>
+      val query = s"CREATE TABLE my_tab ROW FORMAT DELIMITED FIELDS TERMINATED BY ' ' STORED AS $s"
+      if (supportedSources.contains(s)) {
+        parseAndResolve(query) match {
+          case ct: CreateTable =>
+            val hiveSerde = HiveSerDe.sourceToSerDe(s)
+            assert(hiveSerde.isDefined)
+            assert(ct.tableDesc.storage.serde == hiveSerde.get.serde
+                .orElse(Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe")))
+            assert(ct.tableDesc.storage.inputFormat == hiveSerde.get.inputFormat)
+            assert(ct.tableDesc.storage.outputFormat == hiveSerde.get.outputFormat)
         }
+      } else {
+        assertUnsupported(query, Seq("row format delimited", "only compatible with 'textfile'", s))
       }
     }
+  }
+
+  test("create hive external table") {
+    val withoutLoc = "CREATE EXTERNAL TABLE my_tab STORED AS parquet"
+    parseAndResolve(withoutLoc) match {
+      case ct: CreateTable =>
+        assert(ct.tableDesc.tableType == CatalogTableType.EXTERNAL)
+        assert(ct.tableDesc.storage.locationUri.isEmpty)
+    }
+
+    val withLoc = "CREATE EXTERNAL TABLE my_tab STORED AS parquet LOCATION '/something/anything'"
+    parseAndResolve(withLoc) match {
+      case ct: CreateTable =>
+        assert(ct.tableDesc.tableType == CatalogTableType.EXTERNAL)
+        assert(ct.tableDesc.storage.locationUri == Some(new URI("/something/anything")))
+    }
+  }
+
+  test("create hive table - property values must be set") {
+    assertUnsupported(
+      sql = "CREATE TABLE my_tab STORED AS parquet " +
+          "TBLPROPERTIES('key_without_value', 'key_with_value'='x')",
+      containsThesePhrases = Seq("key_without_value"))
+    assertUnsupported(
+      sql = "CREATE TABLE my_tab ROW FORMAT SERDE 'serde' " +
+          "WITH SERDEPROPERTIES('key_without_value', 'key_with_value'='x')",
+      containsThesePhrases = Seq("key_without_value"))
+  }
+
+  test("create hive table - location implies external") {
+    val query = "CREATE TABLE my_tab STORED AS parquet LOCATION '/something/anything'"
+    parseAndResolve(query) match {
+      case ct: CreateTable =>
+        assert(ct.tableDesc.tableType == CatalogTableType.EXTERNAL)
+        assert(ct.tableDesc.storage.locationUri == Some(new URI("/something/anything")))
+    }
+  }
+
+  test("Duplicate clauses - create hive table") {
+    def intercept(sqlCommand: String, messages: String*): Unit =
+      interceptParseException(parsePlan)(sqlCommand, messages: _*)
+
+    def createTableHeader(duplicateClause: String): String = {
+      s"CREATE TABLE my_tab(a INT, b STRING) STORED AS parquet $duplicateClause $duplicateClause"
+    }
+
+    intercept(createTableHeader("TBLPROPERTIES('test' = 'test2')"),
+      "Found duplicate clauses: TBLPROPERTIES")
+    intercept(createTableHeader("LOCATION '/tmp/file'"),
+      "Found duplicate clauses: LOCATION")
+    intercept(createTableHeader("COMMENT 'a table'"),
+      "Found duplicate clauses: COMMENT")
+    intercept(createTableHeader("CLUSTERED BY(b) INTO 256 BUCKETS"),
+      "Found duplicate clauses: CLUSTERED BY")
+    intercept(createTableHeader("PARTITIONED BY (k int)"),
+      "Found duplicate clauses: PARTITIONED BY")
+    intercept(createTableHeader("STORED AS parquet"),
+      "Found duplicate clauses: STORED AS/BY")
+    intercept(
+      createTableHeader("ROW FORMAT SERDE 'parquet.hive.serde.ParquetHiveSerDe'"),
+      "Found duplicate clauses: ROW FORMAT")
+  }
+
+  test("Test CTAS #1") {
+    val s1 =
+      """
+        |CREATE EXTERNAL TABLE IF NOT EXISTS mydb.page_view
+        |COMMENT 'This is the staging page view table'
+        |STORED AS RCFILE
+        |LOCATION '/user/external/page_view'
+        |TBLPROPERTIES ('p1'='v1', 'p2'='v2')
+        |AS SELECT * FROM src
+      """.stripMargin
+
+    val s2 =
+      """
+        |CREATE EXTERNAL TABLE IF NOT EXISTS mydb.page_view
+        |STORED AS RCFILE
+        |COMMENT 'This is the staging page view table'
+        |TBLPROPERTIES ('p1'='v1', 'p2'='v2')
+        |LOCATION '/user/external/page_view'
+        |AS SELECT * FROM src
+      """.stripMargin
+
+    val s3 =
+      """
+        |CREATE EXTERNAL TABLE IF NOT EXISTS mydb.page_view
+        |TBLPROPERTIES ('p1'='v1', 'p2'='v2')
+        |LOCATION '/user/external/page_view'
+        |STORED AS RCFILE
+        |COMMENT 'This is the staging page view table'
+        |AS SELECT * FROM src
+      """.stripMargin
+
+    checkParsing(s1)
+    checkParsing(s2)
+    checkParsing(s3)
+
+    def checkParsing(sql: String): Unit = {
+      val (desc, exists) = extractTableDesc(sql)
+      assert(exists)
+      assert(desc.identifier.database == Some("mydb"))
+      assert(desc.identifier.table == "page_view")
+      assert(desc.tableType == CatalogTableType.EXTERNAL)
+      assert(desc.storage.locationUri == Some(new URI("/user/external/page_view")))
+      assert(desc.schema.isEmpty) // will be populated later when the table is actually created
+      assert(desc.comment == Some("This is the staging page view table"))
+      // TODO will be SQLText
+      assert(desc.viewText.isEmpty)
+      assert(desc.viewCatalogAndNamespace.isEmpty)
+      assert(desc.viewQueryColumnNames.isEmpty)
+      assert(desc.partitionColumnNames.isEmpty)
+      assert(desc.storage.inputFormat == Some("org.apache.hadoop.hive.ql.io.RCFileInputFormat"))
+      assert(desc.storage.outputFormat == Some("org.apache.hadoop.hive.ql.io.RCFileOutputFormat"))
+      assert(desc.storage.serde ==
+          Some("org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe"))
+      assert(desc.properties == Map("p1" -> "v1", "p2" -> "v2"))
+    }
+  }
+
+  test("Test CTAS #2") {
+    val s1 =
+      """
+        |CREATE EXTERNAL TABLE IF NOT EXISTS mydb.page_view
+        |COMMENT 'This is the staging page view table'
+        |ROW FORMAT SERDE 'parquet.hive.serde.ParquetHiveSerDe'
+        | STORED AS
+        | INPUTFORMAT 'parquet.hive.DeprecatedParquetInputFormat'
+        | OUTPUTFORMAT 'parquet.hive.DeprecatedParquetOutputFormat'
+        |LOCATION '/user/external/page_view'
+        |TBLPROPERTIES ('p1'='v1', 'p2'='v2')
+        |AS SELECT * FROM src
+      """.stripMargin
+
+    val s2 =
+      """
+        |CREATE EXTERNAL TABLE IF NOT EXISTS mydb.page_view
+        |LOCATION '/user/external/page_view'
+        |TBLPROPERTIES ('p1'='v1', 'p2'='v2')
+        |ROW FORMAT SERDE 'parquet.hive.serde.ParquetHiveSerDe'
+        | STORED AS
+        | INPUTFORMAT 'parquet.hive.DeprecatedParquetInputFormat'
+        | OUTPUTFORMAT 'parquet.hive.DeprecatedParquetOutputFormat'
+        |COMMENT 'This is the staging page view table'
+        |AS SELECT * FROM src
+      """.stripMargin
+
+    checkParsing(s1)
+    checkParsing(s2)
 
-    checkFailure("v1Table", v1Format)
-    checkFailure("v2Table", v2Format)
-    checkFailure("testcat.tab", "foo")
+    def checkParsing(sql: String): Unit = {
+      val (desc, exists) = extractTableDesc(sql)
+      assert(exists)
+      assert(desc.identifier.database == Some("mydb"))
+      assert(desc.identifier.table == "page_view")
+      assert(desc.tableType == CatalogTableType.EXTERNAL)
+      assert(desc.storage.locationUri == Some(new URI("/user/external/page_view")))
+      assert(desc.schema.isEmpty) // will be populated later when the table is actually created
+      // TODO will be SQLText
+      assert(desc.comment == Some("This is the staging page view table"))
+      assert(desc.viewText.isEmpty)
+      assert(desc.viewCatalogAndNamespace.isEmpty)
+      assert(desc.viewQueryColumnNames.isEmpty)
+      assert(desc.partitionColumnNames.isEmpty)
+      assert(desc.storage.properties == Map())
+      assert(desc.storage.inputFormat == Some("parquet.hive.DeprecatedParquetInputFormat"))
+      assert(desc.storage.outputFormat == Some("parquet.hive.DeprecatedParquetOutputFormat"))
+      assert(desc.storage.serde == Some("parquet.hive.serde.ParquetHiveSerDe"))
+      assert(desc.properties == Map("p1" -> "v1", "p2" -> "v2"))
+    }
+  }
+
+  test("Test CTAS #3") {
+    val s3 = """CREATE TABLE page_view STORED AS textfile AS SELECT * FROM src"""
+    val (desc, exists) = extractTableDesc(s3)
+    assert(exists == false)
+    assert(desc.identifier.database == Some("default"))
+    assert(desc.identifier.table == "page_view")
+    assert(desc.tableType == CatalogTableType.MANAGED)
+    assert(desc.storage.locationUri == None)
+    assert(desc.schema.isEmpty)
+    assert(desc.viewText == None) // TODO will be SQLText
+    assert(desc.viewQueryColumnNames.isEmpty)
+    assert(desc.storage.properties == Map())
+    assert(desc.storage.inputFormat == Some("org.apache.hadoop.mapred.TextInputFormat"))
+    assert(desc.storage.outputFormat ==
+        Some("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"))
+    assert(desc.storage.serde == Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"))
+    assert(desc.properties == Map())
+  }
+
+  test("Test CTAS #4") {
+    val s4 =
+      """CREATE TABLE page_view
+        |STORED BY 'storage.handler.class.name' AS SELECT * FROM src""".stripMargin
+    intercept[AnalysisException] {
+      extractTableDesc(s4)
+    }
+  }
+
+  test("Test CTAS #5") {
+    val s5 = """CREATE TABLE ctas2
+               | ROW FORMAT SERDE "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe"
+               | WITH SERDEPROPERTIES("serde_p1"="p1","serde_p2"="p2")
+               | STORED AS RCFile
+               | TBLPROPERTIES("tbl_p1"="p11", "tbl_p2"="p22")
+               | AS
+               |   SELECT key, value
+               |   FROM src
+               |   ORDER BY key, value""".stripMargin
+    val (desc, exists) = extractTableDesc(s5)
+    assert(exists == false)
+    assert(desc.identifier.database == Some("default"))
+    assert(desc.identifier.table == "ctas2")
+    assert(desc.tableType == CatalogTableType.MANAGED)
+    assert(desc.storage.locationUri == None)
+    assert(desc.schema.isEmpty)
+    assert(desc.viewText == None) // TODO will be SQLText
+    assert(desc.viewCatalogAndNamespace.isEmpty)
+    assert(desc.viewQueryColumnNames.isEmpty)
+    assert(desc.storage.properties == Map(("serde_p1" -> "p1"), ("serde_p2" -> "p2")))
+    assert(desc.storage.inputFormat == Some("org.apache.hadoop.hive.ql.io.RCFileInputFormat"))
+    assert(desc.storage.outputFormat == Some("org.apache.hadoop.hive.ql.io.RCFileOutputFormat"))
+    assert(desc.storage.serde == Some("org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe"))
+    assert(desc.properties == Map(("tbl_p1" -> "p11"), ("tbl_p2" -> "p22")))
+  }
+
+  test("CTAS statement with a PARTITIONED BY clause is not allowed") {
+    assertUnsupported(s"CREATE TABLE ctas1 PARTITIONED BY (k int)" +
+        " AS SELECT key, value FROM (SELECT 1 as key, 2 as value) tmp")
+  }
+
+  test("CTAS statement with schema") {
+    assertUnsupported(s"CREATE TABLE ctas1 (age INT, name STRING) AS SELECT * FROM src")
+    assertUnsupported(s"CREATE TABLE ctas1 (age INT, name STRING) AS SELECT 1, 'hello'")
+  }
+
+  test("create table - basic") {
+    val query = "CREATE TABLE my_table (id int, name string)"
+    val (desc, allowExisting) = extractTableDesc(query)
+    assert(!allowExisting)
+    assert(desc.identifier.database == Some("default"))
+    assert(desc.identifier.table == "my_table")
+    assert(desc.tableType == CatalogTableType.MANAGED)
+    assert(desc.schema == new StructType().add("id", "int").add("name", "string"))
+    assert(desc.partitionColumnNames.isEmpty)
+    assert(desc.bucketSpec.isEmpty)
+    assert(desc.viewText.isEmpty)
+    assert(desc.viewQueryColumnNames.isEmpty)
+    assert(desc.storage.locationUri.isEmpty)
+    assert(desc.storage.inputFormat ==
+        Some("org.apache.hadoop.mapred.TextInputFormat"))
+    assert(desc.storage.outputFormat ==
+        Some("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"))
+    assert(desc.storage.serde == Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"))
+    assert(desc.storage.properties.isEmpty)
+    assert(desc.properties.isEmpty)
+    assert(desc.comment.isEmpty)
+  }
+
+  test("create table - with database name") {
+    val query = "CREATE TABLE dbx.my_table (id int, name string)"
+    val (desc, _) = extractTableDesc(query)
+    assert(desc.identifier.database == Some("dbx"))
+    assert(desc.identifier.table == "my_table")
+  }
+
+  test("create table - temporary") {
+    val query = "CREATE TEMPORARY TABLE tab1 (id int, name string)"
+    val e = intercept[ParseException] { parsePlan(query) }
+    assert(e.message.contains("Operation not allowed: CREATE TEMPORARY TABLE"))
+  }
+
+  test("create table - external") {
+    val query = "CREATE EXTERNAL TABLE tab1 (id int, name string) LOCATION '/path/to/nowhere'"
+    val (desc, _) = extractTableDesc(query)
+    assert(desc.tableType == CatalogTableType.EXTERNAL)
+    assert(desc.storage.locationUri == Some(new URI("/path/to/nowhere")))
+  }
+
+  test("create table - if not exists") {
+    val query = "CREATE TABLE IF NOT EXISTS tab1 (id int, name string)"
+    val (_, allowExisting) = extractTableDesc(query)
+    assert(allowExisting)
+  }
+
+  test("create table - comment") {
+    val query = "CREATE TABLE my_table (id int, name string) COMMENT 'its hot as hell below'"
+    val (desc, _) = extractTableDesc(query)
+    assert(desc.comment == Some("its hot as hell below"))
+  }
+
+  test("create table - partitioned columns") {
+    val query = "CREATE TABLE my_table (id int, name string) PARTITIONED BY (month int)"
+    val (desc, _) = extractTableDesc(query)
+    assert(desc.schema == new StructType()
+        .add("id", "int")
+        .add("name", "string")
+        .add("month", "int"))
+    assert(desc.partitionColumnNames == Seq("month"))
+  }
+
+  test("create table - clustered by") {
+    val numBuckets = 10
+    val bucketedColumn = "id"
+    val sortColumn = "id"
+    val baseQuery =
+      s"""
+         CREATE TABLE my_table (
+           $bucketedColumn int,
+           name string)
+         CLUSTERED BY($bucketedColumn)
+       """
+
+    val query1 = s"$baseQuery INTO $numBuckets BUCKETS"
+    val (desc1, _) = extractTableDesc(query1)
+    assert(desc1.bucketSpec.isDefined)
+    val bucketSpec1 = desc1.bucketSpec.get
+    assert(bucketSpec1.numBuckets == numBuckets)
+    assert(bucketSpec1.bucketColumnNames.head.equals(bucketedColumn))
+    assert(bucketSpec1.sortColumnNames.isEmpty)
+
+    val query2 = s"$baseQuery SORTED BY($sortColumn) INTO $numBuckets BUCKETS"
+    val (desc2, _) = extractTableDesc(query2)
+    assert(desc2.bucketSpec.isDefined)
+    val bucketSpec2 = desc2.bucketSpec.get
+    assert(bucketSpec2.numBuckets == numBuckets)
+    assert(bucketSpec2.bucketColumnNames.head.equals(bucketedColumn))
+    assert(bucketSpec2.sortColumnNames.head.equals(sortColumn))
+  }
+
+  test("create table(hive) - skewed by") {
+    val baseQuery = "CREATE TABLE my_table (id int, name string) SKEWED BY"
+    val query1 = s"$baseQuery(id) ON (1, 10, 100)"
+    val query2 = s"$baseQuery(id, name) ON ((1, 'x'), (2, 'y'), (3, 'z'))"
+    val query3 = s"$baseQuery(id, name) ON ((1, 'x'), (2, 'y'), (3, 'z')) STORED AS DIRECTORIES"
+    val e1 = intercept[ParseException] { parsePlan(query1) }
+    val e2 = intercept[ParseException] { parsePlan(query2) }
+    val e3 = intercept[ParseException] { parsePlan(query3) }
+    assert(e1.getMessage.contains("Operation not allowed"))
+    assert(e2.getMessage.contains("Operation not allowed"))
+    assert(e3.getMessage.contains("Operation not allowed"))
+  }
+
+  test("create table(hive) - row format") {
+    val baseQuery = "CREATE TABLE my_table (id int, name string) ROW FORMAT"
+    val query1 = s"$baseQuery SERDE 'org.apache.poof.serde.Baff'"
+    val query2 = s"$baseQuery SERDE 'org.apache.poof.serde.Baff' WITH SERDEPROPERTIES ('k1'='v1')"
+    val query3 =
+      s"""
+         |$baseQuery DELIMITED FIELDS TERMINATED BY 'x' ESCAPED BY 'y'
+         |COLLECTION ITEMS TERMINATED BY 'a'
+         |MAP KEYS TERMINATED BY 'b'
+         |LINES TERMINATED BY '\n'
+         |NULL DEFINED AS 'c'
+      """.stripMargin
+    val (desc1, _) = extractTableDesc(query1)
+    val (desc2, _) = extractTableDesc(query2)
+    val (desc3, _) = extractTableDesc(query3)
+    assert(desc1.storage.serde == Some("org.apache.poof.serde.Baff"))
+    assert(desc1.storage.properties.isEmpty)
+    assert(desc2.storage.serde == Some("org.apache.poof.serde.Baff"))
+    assert(desc2.storage.properties == Map("k1" -> "v1"))
+    assert(desc3.storage.properties == Map(
+      "field.delim" -> "x",
+      "escape.delim" -> "y",
+      "serialization.format" -> "x",
+      "line.delim" -> "\n",
+      "colelction.delim" -> "a", // yes, it's a typo from Hive :)
+      "mapkey.delim" -> "b"))
+  }
+
+  test("create table(hive) - file format") {
+    val baseQuery = "CREATE TABLE my_table (id int, name string) STORED AS"
+    val query1 = s"$baseQuery INPUTFORMAT 'winput' OUTPUTFORMAT 'wowput'"
+    val query2 = s"$baseQuery ORC"
+    val (desc1, _) = extractTableDesc(query1)
+    val (desc2, _) = extractTableDesc(query2)
+    assert(desc1.storage.inputFormat == Some("winput"))
+    assert(desc1.storage.outputFormat == Some("wowput"))
+    assert(desc1.storage.serde == Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"))
+    assert(desc2.storage.inputFormat == Some("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"))
+    assert(desc2.storage.outputFormat == Some("org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat"))
+    assert(desc2.storage.serde == Some("org.apache.hadoop.hive.ql.io.orc.OrcSerde"))
+  }
+
+  test("create table(hive) - storage handler") {
+    val baseQuery = "CREATE TABLE my_table (id int, name string) STORED BY"
+    val query1 = s"$baseQuery 'org.papachi.StorageHandler'"
+    val query2 = s"$baseQuery 'org.mamachi.StorageHandler' WITH SERDEPROPERTIES ('k1'='v1')"
+    val e1 = intercept[ParseException] { parsePlan(query1) }
+    val e2 = intercept[ParseException] { parsePlan(query2) }
+    assert(e1.getMessage.contains("Operation not allowed"))
+    assert(e2.getMessage.contains("Operation not allowed"))
+  }
+
+  test("create table - properties") {
+    val query = "CREATE TABLE my_table (id int, name string) TBLPROPERTIES ('k1'='v1', 'k2'='v2')"
+    parsePlan(query) match {
+      case state: CreateTableStatement =>
+        assert(state.properties == Map("k1" -> "v1", "k2" -> "v2"))
+    }
+  }
+
+  test("create table(hive) - everything!") {
+    val query =
+      """
+        |CREATE EXTERNAL TABLE IF NOT EXISTS dbx.my_table (id int, name string)
+        |COMMENT 'no comment'
+        |PARTITIONED BY (month int)
+        |ROW FORMAT SERDE 'org.apache.poof.serde.Baff' WITH SERDEPROPERTIES ('k1'='v1')
+        |STORED AS INPUTFORMAT 'winput' OUTPUTFORMAT 'wowput'
+        |LOCATION '/path/to/mercury'
+        |TBLPROPERTIES ('k1'='v1', 'k2'='v2')
+      """.stripMargin
+    val (desc, allowExisting) = extractTableDesc(query)
+    assert(allowExisting)
+    assert(desc.identifier.database == Some("dbx"))
+    assert(desc.identifier.table == "my_table")
+    assert(desc.tableType == CatalogTableType.EXTERNAL)
+    assert(desc.schema == new StructType()
+        .add("id", "int")
+        .add("name", "string")
+        .add("month", "int"))
+    assert(desc.partitionColumnNames == Seq("month"))
+    assert(desc.bucketSpec.isEmpty)
+    assert(desc.viewText.isEmpty)
+    assert(desc.viewCatalogAndNamespace.isEmpty)
+    assert(desc.viewQueryColumnNames.isEmpty)
+    assert(desc.storage.locationUri == Some(new URI("/path/to/mercury")))
+    assert(desc.storage.inputFormat == Some("winput"))
+    assert(desc.storage.outputFormat == Some("wowput"))
+    assert(desc.storage.serde == Some("org.apache.poof.serde.Baff"))
+    assert(desc.storage.properties == Map("k1" -> "v1"))
+    assert(desc.properties == Map("k1" -> "v1", "k2" -> "v2"))
+    assert(desc.comment == Some("no comment"))
+  }
+
+  test("SPARK-34701: children/innerChildren should be mutually exclusive for AnalysisOnlyCommand") {
+    val cmdNotAnalyzed = DummyAnalysisOnlyCommand(isAnalyzed = false, childrenToAnalyze = Seq(null))
+    assert(cmdNotAnalyzed.innerChildren.isEmpty)
+    assert(cmdNotAnalyzed.children.length == 1)
+    val cmdAnalyzed = cmdNotAnalyzed.markAsAnalyzed()
+    assert(cmdAnalyzed.innerChildren.length == 1)
+    assert(cmdAnalyzed.children.isEmpty)
   }
 
   // TODO: add tests for more commands.
@@ -1566,3 +2248,13 @@ object AsDataSourceV2Relation {
     case _ => None
   }
 }
+
+case class DummyAnalysisOnlyCommand(
+    isAnalyzed: Boolean,
+    childrenToAnalyze: Seq[LogicalPlan]) extends AnalysisOnlyCommand {
+  override def markAsAnalyzed(): LogicalPlan = copy(isAnalyzed = true)
+  override protected def withNewChildrenInternal(
+      newChildren: IndexedSeq[LogicalPlan]): LogicalPlan = {
+    copy(childrenToAnalyze = newChildren)
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowNamespacesParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowNamespacesParserSuite.scala
new file mode 100644
index 0000000000000..c9e5d33fea87a
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowNamespacesParserSuite.scala
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, UnresolvedNamespace}
+import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan
+import org.apache.spark.sql.catalyst.parser.ParseException
+import org.apache.spark.sql.catalyst.plans.logical.ShowNamespaces
+import org.apache.spark.sql.test.SharedSparkSession
+
+class ShowNamespacesParserSuite extends AnalysisTest with SharedSparkSession {
+  test("all namespaces") {
+    Seq("SHOW NAMESPACES", "SHOW DATABASES").foreach { sqlCmd =>
+      comparePlans(
+        parsePlan(sqlCmd),
+        ShowNamespaces(UnresolvedNamespace(Seq.empty[String]), None))
+    }
+  }
+
+  test("basic pattern") {
+    Seq(
+      "SHOW DATABASES LIKE 'defau*'",
+      "SHOW NAMESPACES LIKE 'defau*'").foreach { sqlCmd =>
+      comparePlans(
+        parsePlan(sqlCmd),
+        ShowNamespaces(UnresolvedNamespace(Seq.empty[String]), Some("defau*")))
+    }
+  }
+
+  test("FROM/IN operator is not allowed by SHOW DATABASES") {
+    Seq(
+      "SHOW DATABASES FROM testcat.ns1.ns2",
+      "SHOW DATABASES IN testcat.ns1.ns2").foreach { sqlCmd =>
+      val errMsg = intercept[ParseException] {
+        parsePlan(sqlCmd)
+      }.getMessage
+      assert(errMsg.contains("FROM/IN operator is not allowed in SHOW DATABASES"))
+    }
+  }
+
+  test("show namespaces in/from a namespace") {
+    comparePlans(
+      parsePlan("SHOW NAMESPACES FROM testcat.ns1.ns2"),
+      ShowNamespaces(UnresolvedNamespace(Seq("testcat", "ns1", "ns2")), None))
+    comparePlans(
+      parsePlan("SHOW NAMESPACES IN testcat.ns1.ns2"),
+      ShowNamespaces(UnresolvedNamespace(Seq("testcat", "ns1", "ns2")), None))
+  }
+
+  test("namespaces by a pattern from another namespace") {
+    comparePlans(
+      parsePlan("SHOW NAMESPACES IN testcat.ns1 LIKE '*pattern*'"),
+      ShowNamespaces(UnresolvedNamespace(Seq("testcat", "ns1")), Some("*pattern*")))
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowNamespacesSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowNamespacesSuiteBase.scala
new file mode 100644
index 0000000000000..790489e0d47ce
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowNamespacesSuiteBase.scala
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.{QueryTest, Row}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types.{StringType, StructType}
+
+/**
+ * This base suite contains unified tests for the `SHOW NAMESPACES` and `SHOW DATABASES` commands
+ * that check V1 and V2 table catalogs. The tests that cannot run for all supported catalogs are
+ * located in more specific test suites:
+ *
+ *   - V2 table catalog tests: `org.apache.spark.sql.execution.command.v2.ShowNamespacesSuite`
+ *   - V1 table catalog tests: `org.apache.spark.sql.execution.command.v1.ShowNamespacesSuiteBase`
+ *     - V1 In-Memory catalog: `org.apache.spark.sql.execution.command.v1.ShowNamespacesSuite`
+ *     - V1 Hive External catalog: `org.apache.spark.sql.hive.execution.command.ShowNamespacesSuite`
+ */
+trait ShowNamespacesSuiteBase extends QueryTest with DDLCommandTestUtils {
+  override val command = "SHOW NAMESPACES"
+
+  protected def runShowNamespacesSql(sqlText: String, expected: Seq[String]): Unit = {
+    val df = spark.sql(sqlText)
+    assert(df.schema === new StructType().add("namespace", StringType, false))
+    checkAnswer(df, expected.map(Row(_)))
+  }
+
+  protected def builtinTopNamespaces: Seq[String] = Seq.empty
+
+  test("default namespace") {
+    withSQLConf(SQLConf.DEFAULT_CATALOG.key -> catalog) {
+      runShowNamespacesSql("SHOW NAMESPACES", builtinTopNamespaces)
+    }
+    runShowNamespacesSql(s"SHOW NAMESPACES IN $catalog", builtinTopNamespaces)
+  }
+
+  test("at the top level") {
+    withNamespace(s"$catalog.ns1", s"$catalog.ns2") {
+      sql(s"CREATE DATABASE $catalog.ns1")
+      sql(s"CREATE NAMESPACE $catalog.ns2")
+
+      runShowNamespacesSql(
+        s"SHOW NAMESPACES IN $catalog",
+        Seq("ns1", "ns2") ++ builtinTopNamespaces)
+    }
+  }
+
+  test("exact matching") {
+    withNamespace(s"$catalog.ns1", s"$catalog.ns2") {
+      sql(s"CREATE NAMESPACE $catalog.ns1")
+      sql(s"CREATE NAMESPACE $catalog.ns2")
+      Seq(
+        s"SHOW NAMESPACES IN $catalog LIKE 'ns2'",
+        s"SHOW NAMESPACES IN $catalog 'ns2'",
+        s"SHOW NAMESPACES FROM $catalog LIKE 'ns2'",
+        s"SHOW NAMESPACES FROM $catalog 'ns2'").foreach { sqlCmd =>
+        withClue(sqlCmd) {
+          runShowNamespacesSql(sqlCmd, Seq("ns2"))
+        }
+      }
+    }
+  }
+
+  test("does not match to any namespace") {
+    Seq(
+      "SHOW DATABASES LIKE 'non-existentdb'",
+      "SHOW NAMESPACES 'non-existentdb'").foreach { sqlCmd =>
+      runShowNamespacesSql(sqlCmd, Seq.empty)
+    }
+  }
+
+  test("show root namespaces with the default catalog") {
+    withSQLConf(SQLConf.DEFAULT_CATALOG.key -> catalog) {
+      runShowNamespacesSql("SHOW NAMESPACES", builtinTopNamespaces)
+
+      withNamespace("ns1", "ns2") {
+        sql(s"CREATE NAMESPACE ns1")
+        sql(s"CREATE NAMESPACE ns2")
+
+        runShowNamespacesSql("SHOW NAMESPACES", Seq("ns1", "ns2") ++ builtinTopNamespaces)
+        runShowNamespacesSql("SHOW NAMESPACES LIKE '*1*'", Seq("ns1"))
+      }
+    }
+  }
+
+  test("complex namespace patterns") {
+    withNamespace(s"$catalog.showdb2b", s"$catalog.showdb1a") {
+      sql(s"CREATE NAMESPACE $catalog.showdb2b")
+      sql(s"CREATE NAMESPACE $catalog.showdb1a")
+
+      Seq(
+        "'*db1A'" -> Seq("showdb1a"),
+        "'*2*'" -> Seq("showdb2b"),
+        "'*db1A|*db2B'" -> Seq("showdb1a", "showdb2b")
+      ).foreach { case (pattern, expected) =>
+        runShowNamespacesSql(s"SHOW NAMESPACES IN $catalog LIKE $pattern", expected)
+      }
+    }
+  }
+
+  test("change catalog and namespace with USE statements") {
+    try {
+      withNamespace(s"$catalog.ns") {
+        sql(s"CREATE NAMESPACE $catalog.ns")
+        sql(s"USE $catalog")
+        runShowNamespacesSql("SHOW NAMESPACES", Seq("ns") ++ builtinTopNamespaces)
+
+        sql("USE ns")
+        // 'SHOW NAMESPACES' is not affected by the current namespace and lists root namespaces.
+        runShowNamespacesSql("SHOW NAMESPACES", Seq("ns") ++ builtinTopNamespaces)
+      }
+    } finally {
+      spark.sessionState.catalogManager.reset()
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowPartitionsParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowPartitionsParserSuite.scala
new file mode 100644
index 0000000000000..1c7b1282fde42
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowPartitionsParserSuite.scala
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, UnresolvedPartitionSpec, UnresolvedTable}
+import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan
+import org.apache.spark.sql.catalyst.parser.ParseException
+import org.apache.spark.sql.catalyst.plans.logical.ShowPartitions
+import org.apache.spark.sql.execution.SparkSqlParser
+import org.apache.spark.sql.test.SharedSparkSession
+
+class ShowPartitionsParserSuite extends AnalysisTest with SharedSparkSession {
+  test("SHOW PARTITIONS") {
+    val commandName = "SHOW PARTITIONS"
+    Seq(
+      "SHOW PARTITIONS t1" -> ShowPartitions(UnresolvedTable(Seq("t1"), commandName, None), None),
+      "SHOW PARTITIONS db1.t1" -> ShowPartitions(
+        UnresolvedTable(Seq("db1", "t1"), commandName, None), None),
+      "SHOW PARTITIONS t1 PARTITION(partcol1='partvalue', partcol2='partvalue')" ->
+        ShowPartitions(
+          UnresolvedTable(Seq("t1"), commandName, None),
+          Some(UnresolvedPartitionSpec(Map("partcol1" -> "partvalue", "partcol2" -> "partvalue")))),
+      "SHOW PARTITIONS a.b.c" -> ShowPartitions(
+        UnresolvedTable(Seq("a", "b", "c"), commandName, None), None),
+      "SHOW PARTITIONS a.b.c PARTITION(ds='2017-06-10')" ->
+        ShowPartitions(
+          UnresolvedTable(Seq("a", "b", "c"), commandName, None),
+          Some(UnresolvedPartitionSpec(Map("ds" -> "2017-06-10"))))
+    ).foreach { case (sql, expected) =>
+      val parsed = parsePlan(sql)
+      comparePlans(parsed, expected)
+    }
+  }
+
+  test("empty values in non-optional partition specs") {
+    val e = intercept[ParseException] {
+      new SparkSqlParser().parsePlan(
+        "SHOW PARTITIONS dbx.tab1 PARTITION (a='1', b)")
+    }.getMessage
+    assert(e.contains("Found an empty partition key 'b'"))
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowPartitionsSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowPartitionsSuiteBase.scala
new file mode 100644
index 0000000000000..27d2eb9854302
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowPartitionsSuiteBase.scala
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.{AnalysisException, QueryTest, Row, SaveMode}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types.{StringType, StructType}
+
+/**
+ * This base suite contains unified tests for the `SHOW PARTITIONS` command that check V1 and V2
+ * table catalogs. The tests that cannot run for all supported catalogs are located in more
+ * specific test suites:
+ *
+ *   - V2 table catalog tests: `org.apache.spark.sql.execution.command.v2.ShowPartitionsSuite`
+ *   - V1 table catalog tests: `org.apache.spark.sql.execution.command.v1.ShowPartitionsSuiteBase`
+ *     - V1 In-Memory catalog: `org.apache.spark.sql.execution.command.v1.ShowPartitionsSuite`
+ *     - V1 Hive External catalog:
+ *       `org.apache.spark.sql.hive.execution.command.ShowPartitionsSuite`
+ */
+trait ShowPartitionsSuiteBase extends QueryTest with DDLCommandTestUtils {
+  override val command = "SHOW PARTITIONS"
+  // Gets the schema of `SHOW PARTITIONS`
+  private val showSchema: StructType = new StructType().add("partition", StringType, false)
+  protected def runShowPartitionsSql(sqlText: String, expected: Seq[Row]): Unit = {
+    val df = spark.sql(sqlText)
+    assert(df.schema === showSchema)
+    checkAnswer(df, expected)
+  }
+
+  protected def createDateTable(table: String): Unit = {
+    sql(s"""
+      |CREATE TABLE $table (price int, qty int, year int, month int)
+      |$defaultUsing
+      |partitioned by (year, month)""".stripMargin)
+    sql(s"INSERT INTO $table PARTITION(year = 2015, month = 1) SELECT 1, 1")
+    sql(s"INSERT INTO $table PARTITION(year = 2015, month = 2) SELECT 2, 2")
+    sql(s"ALTER TABLE $table ADD PARTITION(year = 2016, month = 2)")
+    sql(s"ALTER TABLE $table ADD PARTITION(year = 2016, month = 3)")
+  }
+
+  protected def createNullPartTable(table: String, format: String): Unit = {
+    import testImplicits._
+    val df = Seq((0, ""), (1, null)).toDF("a", "part")
+    df.write
+      .partitionBy("part")
+      .format(format)
+      .mode(SaveMode.Overwrite)
+      .saveAsTable(table)
+  }
+
+  test("show partitions of non-partitioned table") {
+    withNamespaceAndTable("ns", "not_partitioned_table") { t =>
+      sql(s"CREATE TABLE $t (col1 int) $defaultUsing")
+      val errMsg = intercept[AnalysisException] {
+        sql(s"SHOW PARTITIONS $t")
+      }.getMessage
+      assert(errMsg.contains("not allowed on a table that is not partitioned") ||
+        // V2 error message.
+        errMsg.contains(s"Table $t is not partitioned"))
+    }
+  }
+
+  test("non-partitioning columns") {
+    withNamespaceAndTable("ns", "dateTable") { t =>
+      createDateTable(t)
+      val errMsg = intercept[AnalysisException] {
+        sql(s"SHOW PARTITIONS $t PARTITION(abcd=2015, xyz=1)")
+      }.getMessage
+      assert(errMsg.contains("abcd is not a valid partition column"))
+    }
+  }
+
+  test("show everything") {
+    withNamespaceAndTable("ns", "dateTable") { t =>
+      createDateTable(t)
+      runShowPartitionsSql(
+        s"show partitions $t",
+        Row("year=2015/month=1") ::
+        Row("year=2015/month=2") ::
+        Row("year=2016/month=2") ::
+        Row("year=2016/month=3") :: Nil)
+    }
+  }
+
+  test("filter by partitions") {
+    withNamespaceAndTable("ns", "dateTable") { t =>
+      createDateTable(t)
+      runShowPartitionsSql(
+        s"show partitions $t PARTITION(year=2015)",
+        Row("year=2015/month=1") ::
+        Row("year=2015/month=2") :: Nil)
+      runShowPartitionsSql(
+        s"show partitions $t PARTITION(year=2015, month=1)",
+        Row("year=2015/month=1") :: Nil)
+      runShowPartitionsSql(
+        s"show partitions $t PARTITION(month=2)",
+        Row("year=2015/month=2") ::
+        Row("year=2016/month=2") :: Nil)
+    }
+  }
+
+  test("show everything more than 5 part keys") {
+    withNamespaceAndTable("ns", "wideTable") { t =>
+      createWideTable(t)
+      runShowPartitionsSql(
+        s"show partitions $t",
+        Row("year=2016/month=3/hour=10/minute=10/sec=10/extra=1") ::
+        Row("year=2016/month=4/hour=10/minute=10/sec=10/extra=1") :: Nil)
+    }
+  }
+
+  test("SPARK-33667: case sensitivity of partition spec") {
+    withNamespaceAndTable("ns", "part_table") { t =>
+      sql(s"""
+        |CREATE TABLE $t (price int, qty int, year int, month int)
+        |$defaultUsing
+        |PARTITIONED BY (year, month)""".stripMargin)
+      sql(s"INSERT INTO $t PARTITION(year = 2015, month = 1) SELECT 1, 1")
+      Seq(
+        true -> "PARTITION(year = 2015, month = 1)",
+        false -> "PARTITION(YEAR = 2015, Month = 1)"
+      ).foreach { case (caseSensitive, partitionSpec) =>
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
+          runShowPartitionsSql(
+            s"SHOW PARTITIONS $t $partitionSpec",
+            Row("year=2015/month=1") :: Nil)
+        }
+      }
+    }
+  }
+
+  test("SPARK-33777: sorted output") {
+    withNamespaceAndTable("ns", "dateTable") { t =>
+      sql(s"""
+        |CREATE TABLE $t (id int, part string)
+        |$defaultUsing
+        |PARTITIONED BY (part)""".stripMargin)
+      sql(s"ALTER TABLE $t ADD PARTITION(part = 'b')")
+      sql(s"ALTER TABLE $t ADD PARTITION(part = 'a')")
+      val partitions = sql(s"show partitions $t")
+      assert(partitions.first().getString(0) === "part=a")
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesParserSuite.scala
new file mode 100644
index 0000000000000..d68e1233f7ab2
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesParserSuite.scala
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, UnresolvedNamespace, UnresolvedPartitionSpec}
+import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan
+import org.apache.spark.sql.catalyst.plans.logical.{ShowTableExtended, ShowTables}
+import org.apache.spark.sql.test.SharedSparkSession
+
+class ShowTablesParserSuite extends AnalysisTest with SharedSparkSession {
+  private val catalog = "test_catalog"
+
+  test("show tables") {
+    comparePlans(
+      parsePlan("SHOW TABLES"),
+      ShowTables(UnresolvedNamespace(Seq.empty[String]), None))
+    comparePlans(
+      parsePlan("SHOW TABLES '*test*'"),
+      ShowTables(UnresolvedNamespace(Seq.empty[String]), Some("*test*")))
+    comparePlans(
+      parsePlan("SHOW TABLES LIKE '*test*'"),
+      ShowTables(UnresolvedNamespace(Seq.empty[String]), Some("*test*")))
+    comparePlans(
+      parsePlan(s"SHOW TABLES FROM $catalog.ns1.ns2.tbl"),
+      ShowTables(UnresolvedNamespace(Seq(catalog, "ns1", "ns2", "tbl")), None))
+    comparePlans(
+      parsePlan(s"SHOW TABLES IN $catalog.ns1.ns2.tbl"),
+      ShowTables(UnresolvedNamespace(Seq(catalog, "ns1", "ns2", "tbl")), None))
+    comparePlans(
+      parsePlan("SHOW TABLES IN ns1 '*test*'"),
+      ShowTables(UnresolvedNamespace(Seq("ns1")), Some("*test*")))
+    comparePlans(
+      parsePlan("SHOW TABLES IN ns1 LIKE '*test*'"),
+      ShowTables(UnresolvedNamespace(Seq("ns1")), Some("*test*")))
+  }
+
+  test("show table extended") {
+    comparePlans(
+      parsePlan("SHOW TABLE EXTENDED LIKE '*test*'"),
+      ShowTableExtended(UnresolvedNamespace(Seq.empty[String]), "*test*", None))
+    comparePlans(
+      parsePlan(s"SHOW TABLE EXTENDED FROM $catalog.ns1.ns2 LIKE '*test*'"),
+      ShowTableExtended(UnresolvedNamespace(Seq(catalog, "ns1", "ns2")), "*test*", None))
+    comparePlans(
+      parsePlan(s"SHOW TABLE EXTENDED IN $catalog.ns1.ns2 LIKE '*test*'"),
+      ShowTableExtended(UnresolvedNamespace(Seq(catalog, "ns1", "ns2")), "*test*", None))
+    comparePlans(
+      parsePlan("SHOW TABLE EXTENDED LIKE '*test*' PARTITION(ds='2008-04-09', hr=11)"),
+      ShowTableExtended(
+        UnresolvedNamespace(Seq.empty[String]),
+        "*test*",
+        Some(UnresolvedPartitionSpec(Map("ds" -> "2008-04-09", "hr" -> "11")))))
+    comparePlans(
+      parsePlan(s"SHOW TABLE EXTENDED FROM $catalog.ns1.ns2 LIKE '*test*' " +
+        "PARTITION(ds='2008-04-09')"),
+      ShowTableExtended(
+        UnresolvedNamespace(Seq(catalog, "ns1", "ns2")),
+        "*test*",
+        Some(UnresolvedPartitionSpec(Map("ds" -> "2008-04-09")))))
+    comparePlans(
+      parsePlan(s"SHOW TABLE EXTENDED IN $catalog.ns1.ns2 LIKE '*test*' " +
+        "PARTITION(ds='2008-04-09')"),
+      ShowTableExtended(
+        UnresolvedNamespace(Seq(catalog, "ns1", "ns2")),
+        "*test*",
+        Some(UnresolvedPartitionSpec(Map("ds" -> "2008-04-09")))))
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala
new file mode 100644
index 0000000000000..06385017bbd64
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.{QueryTest, Row}
+import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * This base suite contains unified tests for the `SHOW TABLES` command that check V1 and V2
+ * table catalogs. The tests that cannot run for all supported catalogs are located in more
+ * specific test suites:
+ *
+ *   - V2 table catalog tests: `org.apache.spark.sql.execution.command.v2.ShowTablesSuite`
+ *   - V1 table catalog tests: `org.apache.spark.sql.execution.command.v1.ShowTablesSuiteBase`
+ *     - V1 In-Memory catalog: `org.apache.spark.sql.execution.command.v1.ShowTablesSuite`
+ *     - V1 Hive External catalog: `org.apache.spark.sql.hive.execution.command.ShowTablesSuite`
+ */
+trait ShowTablesSuiteBase extends QueryTest with DDLCommandTestUtils {
+  override val command = "SHOW TABLES"
+  protected def defaultNamespace: Seq[String]
+
+  protected def runShowTablesSql(sqlText: String, expected: Seq[Row]): Unit = {
+    val df = spark.sql(sqlText)
+    checkAnswer(df, expected)
+  }
+
+  test("show an existing table") {
+    withNamespaceAndTable("ns", "table") { t =>
+      sql(s"CREATE TABLE $t (name STRING, id INT) $defaultUsing")
+      runShowTablesSql(s"SHOW TABLES IN $catalog.ns", Seq(Row("ns", "table", false)))
+    }
+  }
+
+  test("show table in a not existing namespace") {
+    val msg = intercept[NoSuchNamespaceException] {
+      runShowTablesSql(s"SHOW TABLES IN $catalog.unknown", Seq())
+    }.getMessage
+    assert(msg.matches("(Database|Namespace) 'unknown' not found"))
+  }
+
+  test("show tables with a pattern") {
+    withNamespace(s"$catalog.ns1", s"$catalog.ns2") {
+      sql(s"CREATE NAMESPACE $catalog.ns1")
+      sql(s"CREATE NAMESPACE $catalog.ns2")
+      withTable(
+        s"$catalog.ns1.table",
+        s"$catalog.ns1.table_name_1a",
+        s"$catalog.ns1.table_name_2b",
+        s"$catalog.ns2.table_name_2b") {
+        sql(s"CREATE TABLE $catalog.ns1.table (id bigint, data string) $defaultUsing")
+        sql(s"CREATE TABLE $catalog.ns1.table_name_1a (id bigint, data string) $defaultUsing")
+        sql(s"CREATE TABLE $catalog.ns1.table_name_2b (id bigint, data string) $defaultUsing")
+        sql(s"CREATE TABLE $catalog.ns2.table_name_2b (id bigint, data string) $defaultUsing")
+
+        runShowTablesSql(
+          s"SHOW TABLES FROM $catalog.ns1",
+          Seq(
+            Row("ns1", "table", false),
+            Row("ns1", "table_name_1a", false),
+            Row("ns1", "table_name_2b", false)))
+
+        runShowTablesSql(
+          s"SHOW TABLES FROM $catalog.ns1 LIKE '*name*'",
+          Seq(
+            Row("ns1", "table_name_1a", false),
+            Row("ns1", "table_name_2b", false)))
+
+        runShowTablesSql(
+          s"SHOW TABLES FROM $catalog.ns1 LIKE 'table_name_1*|table_name_2*'",
+          Seq(
+            Row("ns1", "table_name_1a", false),
+            Row("ns1", "table_name_2b", false)))
+
+        runShowTablesSql(
+          s"SHOW TABLES FROM $catalog.ns1 LIKE '*2b'",
+          Seq(Row("ns1", "table_name_2b", false)))
+      }
+    }
+  }
+
+  test("show tables with current catalog and namespace") {
+    withSQLConf(SQLConf.DEFAULT_CATALOG.key -> catalog) {
+      val tblName = (catalog +: defaultNamespace :+ "table").quoted
+      withTable(tblName) {
+        sql(s"CREATE TABLE $tblName (name STRING, id INT) $defaultUsing")
+        val ns = defaultNamespace.mkString(".")
+        runShowTablesSql("SHOW TABLES", Seq(Row(ns, "table", false)))
+      }
+    }
+  }
+
+  test("SPARK-34560: unique attribute references") {
+    withNamespaceAndTable("ns1", "tbl1") { t1 =>
+      sql(s"CREATE TABLE $t1 (col INT) $defaultUsing")
+      val show1 = sql(s"SHOW TABLES IN $catalog.ns1")
+      withNamespaceAndTable("ns2", "tbl2") { t2 =>
+        sql(s"CREATE TABLE $t2 (col INT) $defaultUsing")
+        val show2 = sql(s"SHOW TABLES IN $catalog.ns2")
+        assert(!show1.join(show2).where(show1("tableName") =!= show2("tableName")).isEmpty)
+      }
+    }
+  }
+
+  test("change current catalog and namespace with USE statements") {
+    withNamespaceAndTable("ns", "table") { t =>
+      sql(s"CREATE TABLE $t (name STRING, id INT) $defaultUsing")
+
+      sql(s"USE $catalog")
+      // No table is matched since the current namespace is not ["ns"]
+      assert(defaultNamespace != Seq("ns"))
+      runShowTablesSql("SHOW TABLES", Seq())
+
+      // Update the current namespace to match "ns.tbl".
+      sql(s"USE $catalog.ns")
+      runShowTablesSql("SHOW TABLES", Seq(Row("ns", "table", false)))
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/TruncateTableParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/TruncateTableParserSuite.scala
new file mode 100644
index 0000000000000..7f4a48023c16e
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/TruncateTableParserSuite.scala
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, UnresolvedPartitionSpec, UnresolvedTable}
+import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan
+import org.apache.spark.sql.catalyst.parser.ParseException
+import org.apache.spark.sql.catalyst.plans.logical.{TruncatePartition, TruncateTable}
+import org.apache.spark.sql.test.SharedSparkSession
+
+class TruncateTableParserSuite extends AnalysisTest with SharedSparkSession {
+  test("truncate table") {
+    comparePlans(
+      parsePlan("TRUNCATE TABLE a.b.c"),
+      TruncateTable(UnresolvedTable(Seq("a", "b", "c"), "TRUNCATE TABLE", None)))
+  }
+
+  test("truncate a single part partition") {
+    comparePlans(
+      parsePlan("TRUNCATE TABLE a.b.c PARTITION(ds='2017-06-10')"),
+      TruncatePartition(
+        UnresolvedTable(Seq("a", "b", "c"), "TRUNCATE TABLE", None),
+        UnresolvedPartitionSpec(Map("ds" -> "2017-06-10"), None)))
+  }
+
+  test("truncate a multi parts partition") {
+    comparePlans(
+      parsePlan("TRUNCATE TABLE ns.tbl PARTITION(a = 1, B = 'ABC')"),
+      TruncatePartition(
+        UnresolvedTable(Seq("ns", "tbl"), "TRUNCATE TABLE", None),
+        UnresolvedPartitionSpec(Map("a" -> "1", "B" -> "ABC"), None)))
+  }
+
+  test("empty values in non-optional partition specs") {
+    val errMsg = intercept[ParseException] {
+      parsePlan("TRUNCATE TABLE dbx.tab1 PARTITION (a='1', b)")
+    }.getMessage
+    assert(errMsg.contains("Found an empty partition key 'b'"))
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/TruncateTableSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/TruncateTableSuiteBase.scala
new file mode 100644
index 0000000000000..001ec8e250def
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/TruncateTableSuiteBase.scala
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
+import org.apache.spark.sql.catalyst.analysis.NoSuchPartitionException
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * This base suite contains unified tests for the `TRUNCATE TABLE` command that check V1 and V2
+ * table catalogs. The tests that cannot run for all supported catalogs are located in more
+ * specific test suites:
+ *
+ *   - V2 table catalog tests: `org.apache.spark.sql.execution.command.v2.TruncateTableSuite`
+ *   - V1 table catalog tests: `org.apache.spark.sql.execution.command.v1.TruncateTableSuiteBase`
+ *     - V1 In-Memory catalog: `org.apache.spark.sql.execution.command.v1.TruncateTableSuite`
+ *     - V1 Hive External catalog: `org.apache.spark.sql.hive.execution.command.TruncateTableSuite`
+ */
+trait TruncateTableSuiteBase extends QueryTest with DDLCommandTestUtils {
+  override val command = "TRUNCATE TABLE"
+
+  test("table does not exist") {
+    withNamespaceAndTable("ns", "does_not_exist") { t =>
+      val errMsg = intercept[AnalysisException] {
+        sql(s"TRUNCATE TABLE $t")
+      }.getMessage
+      assert(errMsg.contains("Table not found"))
+    }
+  }
+
+  test("truncate non-partitioned table") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (c0 INT, c1 INT) $defaultUsing")
+      sql(s"INSERT INTO $t SELECT 0, 1")
+
+      sql(s"TRUNCATE TABLE $t")
+      QueryTest.checkAnswer(sql(s"SELECT * FROM $t"), Nil)
+    }
+  }
+
+  protected def createPartTable(t: String): Unit = {
+    sql(s"""
+      |CREATE TABLE $t (width INT, length INT, height INT)
+      |$defaultUsing
+      |PARTITIONED BY (width, length)""".stripMargin)
+    sql(s"INSERT INTO $t PARTITION (width = 0, length = 0) SELECT 0")
+    sql(s"INSERT INTO $t PARTITION (width = 1, length = 1) SELECT 1")
+    sql(s"INSERT INTO $t PARTITION (width = 1, length = 2) SELECT 3")
+  }
+
+  test("SPARK-34418: truncate partitioned tables") {
+    withNamespaceAndTable("ns", "partTable") { t =>
+      createPartTable(t)
+      sql(s"TRUNCATE TABLE $t PARTITION (width = 1, length = 1)")
+      checkAnswer(sql(s"SELECT width, length, height FROM $t"), Seq(Row(0, 0, 0), Row(1, 2, 3)))
+      checkPartitions(t,
+        Map("width" -> "0", "length" -> "0"),
+        Map("width" -> "1", "length" -> "1"),
+        Map("width" -> "1", "length" -> "2"))
+    }
+
+    withNamespaceAndTable("ns", "partTable") { t =>
+      createPartTable(t)
+      // support partial partition spec
+      sql(s"TRUNCATE TABLE $t PARTITION (width = 1)")
+      QueryTest.checkAnswer(sql(s"SELECT * FROM $t"), Row(0, 0, 0) :: Nil)
+      checkPartitions(t,
+        Map("width" -> "0", "length" -> "0"),
+        Map("width" -> "1", "length" -> "1"),
+        Map("width" -> "1", "length" -> "2"))
+    }
+
+    withNamespaceAndTable("ns", "partTable") { t =>
+      createPartTable(t)
+      // do nothing if no partition is matched for the given partial partition spec
+      sql(s"TRUNCATE TABLE $t PARTITION (width = 100)")
+      QueryTest.checkAnswer(
+        sql(s"SELECT width, length, height FROM $t"),
+        Seq(Row(0, 0, 0), Row(1, 1, 1), Row(1, 2, 3)))
+
+      // throw exception if no partition is matched for the given non-partial partition spec.
+      intercept[NoSuchPartitionException] {
+        sql(s"TRUNCATE TABLE $t PARTITION (width = 100, length = 100)")
+      }
+
+      // throw exception if the column in partition spec is not a partition column.
+      val errMsg = intercept[AnalysisException] {
+        sql(s"TRUNCATE TABLE $t PARTITION (unknown = 1)")
+      }.getMessage
+      assert(errMsg.contains("unknown is not a valid partition column"))
+    }
+  }
+
+  protected def invalidPartColumnError: String
+
+  test("truncate a partition of non partitioned table") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (c0 INT) $defaultUsing")
+      sql(s"INSERT INTO $t SELECT 0")
+
+      val errMsg = intercept[AnalysisException] {
+        sql(s"TRUNCATE TABLE $t PARTITION (c0=1)")
+      }.getMessage
+      assert(errMsg.contains(invalidPartColumnError))
+    }
+  }
+
+  test("SPARK-34418: preserve partitions in truncated table") {
+    withNamespaceAndTable("ns", "partTable") { t =>
+      createPartTable(t)
+      checkAnswer(
+        sql(s"SELECT width, length, height FROM $t"),
+        Seq(Row(0, 0, 0), Row(1, 1, 1), Row(1, 2, 3)))
+      sql(s"TRUNCATE TABLE $t")
+      checkAnswer(sql(s"SELECT width, length, height FROM $t"), Nil)
+      checkPartitions(t,
+        Map("width" -> "0", "length" -> "0"),
+        Map("width" -> "1", "length" -> "1"),
+        Map("width" -> "1", "length" -> "2"))
+    }
+  }
+
+  test("case sensitivity in resolving partition specs") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (id bigint, data string) $defaultUsing PARTITIONED BY (id)")
+      sql(s"INSERT INTO $t PARTITION (id=0) SELECT 'abc'")
+      sql(s"INSERT INTO $t PARTITION (id=1) SELECT 'def'")
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+        val errMsg = intercept[AnalysisException] {
+          sql(s"TRUNCATE TABLE $t PARTITION (ID=1)")
+        }.getMessage
+        assert(errMsg.contains("ID is not a valid partition column"))
+      }
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+        sql(s"TRUNCATE TABLE $t PARTITION (ID=1)")
+        QueryTest.checkAnswer(sql(s"SELECT id, data FROM $t"), Row(0, "abc") :: Nil)
+      }
+    }
+  }
+
+  test("SPARK-34215: keep table cached after truncation") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (c0 int) $defaultUsing")
+      sql(s"INSERT INTO $t SELECT 0")
+      sql(s"CACHE TABLE $t")
+      assert(spark.catalog.isCached(t))
+      QueryTest.checkAnswer(sql(s"SELECT * FROM $t"), Row(0) :: Nil)
+      sql(s"TRUNCATE TABLE $t")
+      assert(spark.catalog.isCached(t))
+      QueryTest.checkAnswer(sql(s"SELECT * FROM $t"), Nil)
+    }
+  }
+
+  test("truncation of views is not allowed") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (id int, part int) $defaultUsing PARTITIONED BY (part)")
+      sql(s"INSERT INTO $t PARTITION (part=0) SELECT 0")
+
+      withView("v0") {
+        sql(s"CREATE VIEW v0 AS SELECT * FROM $t")
+        val errMsg = intercept[AnalysisException] {
+          sql("TRUNCATE TABLE v0")
+        }.getMessage
+        assert(errMsg.contains("'TRUNCATE TABLE' expects a table"))
+      }
+
+      withTempView("v1") {
+        sql(s"CREATE TEMP VIEW v1 AS SELECT * FROM $t")
+        val errMsg = intercept[AnalysisException] {
+          sql("TRUNCATE TABLE v1")
+        }.getMessage
+        assert(errMsg.contains("'TRUNCATE TABLE' expects a table"))
+      }
+
+      val v2 = s"${spark.sharedState.globalTempViewManager.database}.v2"
+      withGlobalTempView("v2") {
+        sql(s"CREATE GLOBAL TEMP VIEW v2 AS SELECT * FROM $t")
+        val errMsg = intercept[AnalysisException] {
+          sql(s"TRUNCATE TABLE $v2")
+        }.getMessage
+        assert(errMsg.contains("'TRUNCATE TABLE' expects a table"))
+      }
+    }
+  }
+
+  test("keep dependents as cached after table truncation") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      createPartTable(t)
+      cacheRelation(t)
+      QueryTest.checkAnswer(
+        sql(s"SELECT width, length, height FROM $t"),
+        Seq(Row(0, 0, 0), Row(1, 1, 1), Row(1, 2, 3)))
+
+      withView("v0") {
+        sql(s"CREATE VIEW v0 AS SELECT * FROM $t")
+        cacheRelation("v0")
+        sql(s"TRUNCATE TABLE $t PARTITION (width = 1, length = 2)")
+        checkCachedRelation("v0", Seq(Row(0, 0, 0), Row(1, 1, 1)))
+      }
+
+      withTempView("v1") {
+        sql(s"CREATE TEMP VIEW v1 AS SELECT * FROM $t")
+        cacheRelation("v1")
+        sql(s"TRUNCATE TABLE $t PARTITION (width = 1, length = 1)")
+        checkCachedRelation("v1", Seq(Row(0, 0, 0)))
+      }
+
+      val v2 = s"${spark.sharedState.globalTempViewManager.database}.v2"
+      withGlobalTempView("v2") {
+        sql(s"INSERT INTO $t PARTITION (width = 10, length = 10) SELECT 10")
+        sql(s"CREATE GLOBAL TEMP VIEW v2 AS SELECT * FROM $t")
+        cacheRelation(v2)
+        sql(s"TRUNCATE TABLE $t PARTITION (width = 10, length = 10)")
+        checkCachedRelation(v2, Seq(Row(0, 0, 0)))
+      }
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/AlterTableAddPartitionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/AlterTableAddPartitionSuite.scala
new file mode 100644
index 0000000000000..b2e626be1b180
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/AlterTableAddPartitionSuite.scala
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v1
+
+import org.apache.spark.sql.{AnalysisException, Row}
+import org.apache.spark.sql.execution.command
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * This base suite contains unified tests for the `ALTER TABLE .. ADD PARTITION` command that
+ * check V1 table catalogs. The tests that cannot run for all V1 catalogs are located in more
+ * specific test suites:
+ *
+ *   - V1 In-Memory catalog:
+ *     `org.apache.spark.sql.execution.command.v1.AlterTableAddPartitionSuite`
+ *   - V1 Hive External catalog:
+ *     `org.apache.spark.sql.hive.execution.command.AlterTableAddPartitionSuite`
+ */
+trait AlterTableAddPartitionSuiteBase extends command.AlterTableAddPartitionSuiteBase {
+  test("empty string as partition value") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (col1 INT, p1 STRING) $defaultUsing PARTITIONED BY (p1)")
+      val errMsg = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $t ADD PARTITION (p1 = '')")
+      }.getMessage
+      assert(errMsg.contains("Partition spec is invalid. " +
+        "The spec ([p1=]) contains an empty partition column value"))
+    }
+  }
+
+  test("SPARK-34055: refresh cache in partition adding") {
+    withTable("t") {
+      sql(s"CREATE TABLE t (id int, part int) $defaultUsing PARTITIONED BY (part)")
+      sql("INSERT INTO t PARTITION (part=0) SELECT 0")
+      assert(!spark.catalog.isCached("t"))
+      sql("CACHE TABLE t")
+      assert(spark.catalog.isCached("t"))
+      checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 0)))
+
+      // Create new partition (part = 1) in the filesystem
+      val part1Loc = copyPartition("t", "part=0", "part=1")
+
+      sql(s"ALTER TABLE t ADD PARTITION (part=1) LOCATION '$part1Loc'")
+      assert(spark.catalog.isCached("t"))
+      checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 0), Row(0, 1)))
+    }
+  }
+
+  test("SPARK-34084: auto update table stats") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      withSQLConf(SQLConf.AUTO_SIZE_UPDATE_ENABLED.key -> "false") {
+        sql(s"CREATE TABLE $t (col0 int, part int) $defaultUsing PARTITIONED BY (part)")
+        sql(s"INSERT INTO $t PARTITION (part=0) SELECT 0")
+        val errMsg = intercept[IllegalArgumentException] {
+          getTableSize(t)
+        }.getMessage
+        assert(errMsg.contains(s"The table $t does not have stats"))
+      }
+      withSQLConf(SQLConf.AUTO_SIZE_UPDATE_ENABLED.key -> "true") {
+        sql(s"ALTER TABLE $t ADD PARTITION (part=1)")
+        assert(getTableSize(t) > 0)
+      }
+    }
+  }
+
+  test("SPARK-34060, SPARK-34071: update stats of cached table") {
+    withSQLConf(SQLConf.AUTO_SIZE_UPDATE_ENABLED.key -> "true") {
+      withNamespaceAndTable("ns", "tbl") { t =>
+        sql(s"CREATE TABLE $t (id int, part int) $defaultUsing PARTITIONED BY (part)")
+        sql(s"INSERT INTO $t PARTITION (part=0) SELECT 0")
+        assert(!spark.catalog.isCached(t))
+        sql(s"CACHE TABLE $t")
+        assert(spark.catalog.isCached(t))
+        checkAnswer(sql(s"SELECT * FROM $t"), Seq(Row(0, 0)))
+        val onePartSize = getTableSize(t)
+        assert(onePartSize > 0)
+
+        // Create new partition (part = 1) in the filesystem
+        val part1Loc = copyPartition(t, "part=0", "part=1")
+
+        sql(s"ALTER TABLE $t ADD PARTITION (part=1) LOCATION '$part1Loc'")
+        assert(spark.catalog.isCached(t))
+        val twoPartSize = getTableSize(t)
+        assert(onePartSize < twoPartSize)
+        checkAnswer(sql(s"SELECT * FROM $t"), Seq(Row(0, 0), Row(0, 1)))
+      }
+    }
+  }
+
+  test("SPARK-34138: keep dependents cached after table altering") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (id int, part int) $defaultUsing PARTITIONED BY (part)")
+      sql(s"INSERT INTO $t PARTITION (part=0) SELECT 0")
+      cacheRelation(t)
+      checkCachedRelation(t, Seq(Row(0, 0)))
+
+      withView("v0") {
+        sql(s"CREATE VIEW v0 AS SELECT * FROM $t")
+        cacheRelation("v0")
+        val part1Loc = copyPartition(t, "part=0", "part=1")
+        sql(s"ALTER TABLE $t ADD PARTITION (part=1) LOCATION '$part1Loc'")
+        checkCachedRelation("v0", Seq(Row(0, 0), Row(0, 1)))
+      }
+
+      withTempView("v1") {
+        sql(s"CREATE TEMP VIEW v1 AS SELECT * FROM $t")
+        cacheRelation("v1")
+        val part2Loc = copyPartition(t, "part=0", "part=2")
+        sql(s"ALTER TABLE $t ADD PARTITION (part=2) LOCATION '$part2Loc'")
+        checkCachedRelation("v1", Seq(Row(0, 0), Row(0, 1), Row(0, 2)))
+      }
+
+      val v2 = s"${spark.sharedState.globalTempViewManager.database}.v2"
+      withGlobalTempView("v2") {
+        sql(s"CREATE GLOBAL TEMP VIEW v2 AS SELECT * FROM $t")
+        cacheRelation(v2)
+        val part3Loc = copyPartition(t, "part=0", "part=3")
+        sql(s"ALTER TABLE $t ADD PARTITION (part=3) LOCATION '$part3Loc'")
+        checkCachedRelation(v2, Seq(Row(0, 0), Row(0, 1), Row(0, 2), Row(0, 3)))
+      }
+    }
+  }
+}
+
+/**
+ * The class contains tests for the `ALTER TABLE .. ADD PARTITION` command to check
+ * V1 In-Memory table catalog.
+ */
+class AlterTableAddPartitionSuite extends AlterTableAddPartitionSuiteBase with CommandSuiteBase
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/AlterTableDropPartitionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/AlterTableDropPartitionSuite.scala
new file mode 100644
index 0000000000000..cc57e10a16887
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/AlterTableDropPartitionSuite.scala
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v1
+
+import org.apache.spark.sql.{AnalysisException, Row}
+import org.apache.spark.sql.execution.command
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * This base suite contains unified tests for the `ALTER TABLE .. DROP PARTITION` command that
+ * check V1 table catalogs. The tests that cannot run for all V1 catalogs are located in more
+ * specific test suites:
+ *
+ *   - V1 In-Memory catalog:
+ *     `org.apache.spark.sql.execution.command.v1.AlterTableDropPartitionSuite`
+ *   - V1 Hive External catalog:
+ *     `org.apache.spark.sql.hive.execution.command.AlterTableDropPartitionSuite`
+ */
+trait AlterTableDropPartitionSuiteBase extends command.AlterTableDropPartitionSuiteBase {
+  override protected val notFullPartitionSpecErr = "The following partitions not found in table"
+  override protected def nullPartitionValue: String = "__HIVE_DEFAULT_PARTITION__"
+
+  test("purge partition data") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (id bigint, data string) $defaultUsing PARTITIONED BY (id)")
+      sql(s"ALTER TABLE $t ADD PARTITION (id = 1)")
+      checkPartitions(t, Map("id" -> "1"))
+      sql(s"ALTER TABLE $t DROP PARTITION (id = 1) PURGE")
+      checkPartitions(t) // no partitions
+    }
+  }
+
+  test("SPARK-34060, SPARK-34071: update stats of cached table") {
+    withSQLConf(SQLConf.AUTO_SIZE_UPDATE_ENABLED.key -> "true") {
+      withNamespaceAndTable("ns", "tbl") { t =>
+        sql(s"CREATE TABLE $t (id int, part int) $defaultUsing PARTITIONED BY (part)")
+        sql(s"INSERT INTO $t PARTITION (part=0) SELECT 0")
+        sql(s"INSERT INTO $t PARTITION (part=1) SELECT 1")
+        assert(!spark.catalog.isCached(t))
+        sql(s"CACHE TABLE $t")
+        assert(spark.catalog.isCached(t))
+        checkAnswer(sql(s"SELECT * FROM $t"), Seq(Row(0, 0), Row(1, 1)))
+        val twoPartSize = getTableSize(t)
+        assert(twoPartSize > 0)
+
+        sql(s"ALTER TABLE $t DROP PARTITION (part=0)")
+        assert(spark.catalog.isCached(t))
+        val onePartSize = getTableSize(t)
+        assert(0 < onePartSize && onePartSize < twoPartSize)
+        checkAnswer(sql(s"SELECT * FROM $t"), Seq(Row(1, 1)))
+      }
+    }
+  }
+}
+
+/**
+ * The class contains tests for the `ALTER TABLE .. DROP PARTITION` command to check
+ * V1 In-Memory table catalog.
+ */
+class AlterTableDropPartitionSuite
+  extends AlterTableDropPartitionSuiteBase
+  with CommandSuiteBase {
+
+  test("empty string as partition value") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (col1 INT, p1 STRING) $defaultUsing PARTITIONED BY (p1)")
+      val errMsg = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $t DROP PARTITION (p1 = '')")
+      }.getMessage
+      assert(errMsg.contains("Partition spec is invalid. " +
+        "The spec ([p1=]) contains an empty partition column value"))
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/AlterTableRecoverPartitionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/AlterTableRecoverPartitionsSuite.scala
new file mode 100644
index 0000000000000..16dd6a3a03f1d
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/AlterTableRecoverPartitionsSuite.scala
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v1
+
+import org.apache.hadoop.fs.Path
+
+import org.apache.spark.internal.config.RDD_PARALLEL_LISTING_THRESHOLD
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.execution.command
+
+/**
+ * This base suite contains unified tests for the `ALTER TABLE .. RECOVER PARTITIONS` command that
+ * check V1 table catalogs. The tests that cannot run for all V1 catalogs are located in more
+ * specific test suites:
+ *
+ *   - V1 In-Memory catalog (sequential):
+ *     `org.apache.spark.sql.execution.command.v1.AlterTableRecoverPartitionsSuite`
+ *   - V1 In-Memory catalog (parallel):
+ *     `org.apache.spark.sql.execution.command.v1.AlterTableRecoverPartitionsParallelSuite`
+ *   - V1 Hive External catalog:
+ *     `org.apache.spark.sql.hive.execution.command.AlterTableRecoverPartitionsSuite`
+ */
+trait AlterTableRecoverPartitionsSuiteBase extends command.AlterTableRecoverPartitionsSuiteBase {
+  test("table does not exist") {
+    val errMsg = intercept[AnalysisException] {
+      sql("ALTER TABLE does_not_exist RECOVER PARTITIONS")
+    }.getMessage
+    assert(errMsg.contains("Table not found"))
+  }
+
+  test("valid locations") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (a INT, b INT, c INT, d INT) $defaultUsing PARTITIONED BY (a, b, c)")
+      checkPartitions(t)
+
+      withTableDir(t) { case (fs, root) =>
+        fs.mkdirs(new Path(new Path(new Path(root, "a=1"), "b=5"), "c=19"))
+        fs.createNewFile(new Path(new Path(root, "a=1/b=5/c=19"), "a.csv"))  // file
+        fs.createNewFile(new Path(new Path(root, "a=1/b=5/c=19"), "_SUCCESS"))  // file
+
+        fs.mkdirs(new Path(new Path(new Path(root, "A=2"), "B=6"), "C=31"))
+        fs.createNewFile(new Path(new Path(root, "A=2/B=6/C=31"), "b.csv"))  // file
+        fs.createNewFile(new Path(new Path(root, "A=2/B=6/C=31"), "c.csv"))  // file
+        fs.createNewFile(new Path(new Path(root, "A=2/B=6/C=31"), ".hiddenFile"))  // file
+        fs.mkdirs(new Path(new Path(root, "A=2/B=6/C=31"), "_temporary"))
+      }
+
+      sql(s"ALTER TABLE $t RECOVER PARTITIONS")
+      checkPartitions(t,
+        Map("a" -> "1", "b" -> "5", "c" -> "19"),
+        Map("a" -> "2", "b" -> "6", "c" -> "31"))
+    }
+  }
+
+  test("invalid locations") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (a INT, b INT, c INT, d INT) $defaultUsing PARTITIONED BY (a, b, c)")
+      sql(s"INSERT INTO $t PARTITION (a=0, b=1, c=2) SELECT 3")
+      checkPartitions(t, Map("a" -> "0", "b" -> "1", "c" -> "2"))
+
+      withTableDir(t) { case (fs, root) =>
+        fs.mkdirs(new Path(new Path(root, "a"), "b"))  // bad name
+        fs.mkdirs(new Path(new Path(root, "b=1"), "a=1"))  // wrong order
+        fs.mkdirs(new Path(root, "a=4")) // not enough columns
+        fs.createNewFile(new Path(new Path(root, "a=1"), "b=4"))  // file
+        fs.createNewFile(new Path(new Path(root, "a=1"), "_SUCCESS"))  // _SUCCESS
+        fs.mkdirs(new Path(new Path(root, "a=1"), "_temporary"))  // _temporary
+        fs.mkdirs(new Path(new Path(root, "a=1"), ".b=4"))  // start with .
+      }
+
+      sql(s"ALTER TABLE $t RECOVER PARTITIONS")
+      checkPartitions(t, Map("a" -> "0", "b" -> "1", "c" -> "2"))
+    }
+  }
+
+  test("multiple locations") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (a INT, b INT, c INT, d INT) $defaultUsing PARTITIONED BY (a, b, c)")
+      sql(s"INSERT INTO $t PARTITION (a=0, b=1, c=2) SELECT 3")
+      val initPart = Map("a" -> "0", "b" -> "1", "c" -> "2")
+      checkPartitions(t, initPart)
+
+      withTableDir(t) { case (fs, root) =>
+        (0 to 100).foreach { a =>
+          val part = Map("a" -> a.toString, "b" -> "5", "c" -> "42")
+          fs.mkdirs(new Path(new Path(new Path(root, s"a=$a"), "b=5"), "c=42"))
+          val loc = s"a=$a/b=5/c=42"
+          fs.createNewFile(new Path(new Path(root, loc), "a.csv"))  // file
+          if (a >= 10) {
+            sql(s"ALTER TABLE $t ADD ${partSpecToString(part)} LOCATION '$loc'")
+          }
+        }
+      }
+
+      sql(s"ALTER TABLE $t RECOVER PARTITIONS")
+      val expected = (0 to 100)
+        .map(a => Map("a" -> a.toString, "b" -> "5", "c" -> "42")) :+ initPart
+      checkPartitions(t, expected: _*)
+    }
+  }
+}
+
+/**
+ * The class contains tests for the `ALTER TABLE .. RECOVER PARTITIONS` command to check
+ * V1 In-Memory table catalog (sequential).
+ */
+class AlterTableRecoverPartitionsSuite
+  extends AlterTableRecoverPartitionsSuiteBase
+  with CommandSuiteBase {
+
+  override protected def sparkConf = super.sparkConf
+    .set(RDD_PARALLEL_LISTING_THRESHOLD, 0)
+}
+
+/**
+ * The class contains tests for the `ALTER TABLE .. RECOVER PARTITIONS` command to check
+ * V1 In-Memory table catalog (parallel).
+ */
+class AlterTableRecoverPartitionsParallelSuite
+  extends AlterTableRecoverPartitionsSuiteBase
+  with CommandSuiteBase {
+
+  override protected def sparkConf = super.sparkConf
+    .set(RDD_PARALLEL_LISTING_THRESHOLD, 10)
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/AlterTableRenamePartitionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/AlterTableRenamePartitionSuite.scala
new file mode 100644
index 0000000000000..b7f3696351cfb
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/AlterTableRenamePartitionSuite.scala
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v1
+
+import org.apache.spark.sql.{QueryTest, Row}
+import org.apache.spark.sql.execution.command
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * This base suite contains unified tests for the `ALTER TABLE .. RENAME PARTITION` command that
+ * check V1 table catalogs. The tests that cannot run for all V1 catalogs are located in more
+ * specific test suites:
+ *
+ *   - V1 In-Memory catalog:
+ *     `org.apache.spark.sql.execution.command.v1.AlterTableRenamePartitionSuite`
+ *   - V1 Hive External catalog:
+ *     `org.apache.spark.sql.hive.execution.command.AlterTableRenamePartitionSuite`
+ */
+trait AlterTableRenamePartitionSuiteBase extends command.AlterTableRenamePartitionSuiteBase {
+  test("with location") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      createSinglePartTable(t)
+      sql(s"ALTER TABLE $t ADD PARTITION (id = 2) LOCATION 'loc1'")
+      sql(s"INSERT INTO $t PARTITION (id = 2) SELECT 'def'")
+      checkPartitions(t, Map("id" -> "1"), Map("id" -> "2"))
+      checkLocation(t, Map("id" -> "2"), "loc1")
+
+      sql(s"ALTER TABLE $t PARTITION (id = 2) RENAME TO PARTITION (id = 3)")
+      checkPartitions(t, Map("id" -> "1"), Map("id" -> "3"))
+      // V1 catalogs rename the partition location of managed tables
+      checkLocation(t, Map("id" -> "3"), "id=3")
+      checkAnswer(sql(s"SELECT id, data FROM $t WHERE id = 3"), Row(3, "def"))
+    }
+  }
+
+  test("SPARK-34060, SPARK-34071: update stats of cached table") {
+    withSQLConf(SQLConf.AUTO_SIZE_UPDATE_ENABLED.key -> "true") {
+      withNamespaceAndTable("ns", "tbl") { t =>
+        sql(s"CREATE TABLE $t (id int, part int) $defaultUsing PARTITIONED BY (part)")
+        sql(s"INSERT INTO $t PARTITION (part=0) SELECT 0")
+        sql(s"INSERT INTO $t PARTITION (part=1) SELECT 1")
+        assert(!spark.catalog.isCached(t))
+        sql(s"CACHE TABLE $t")
+        assert(spark.catalog.isCached(t))
+        QueryTest.checkAnswer(sql(s"SELECT * FROM $t"), Seq(Row(0, 0), Row(1, 1)))
+        val tableSize = getTableSize(t)
+        assert(tableSize > 0)
+
+        sql(s"ALTER TABLE $t PARTITION (part=0) RENAME TO PARTITION (part=2)")
+        assert(spark.catalog.isCached(t))
+        assert(tableSize == getTableSize(t))
+        QueryTest.checkAnswer(sql(s"SELECT * FROM $t"), Seq(Row(0, 2), Row(1, 1)))
+      }
+    }
+  }
+}
+
+/**
+ * The class contains tests for the `ALTER TABLE .. RENAME PARTITION` command to check
+ * V1 In-Memory table catalog.
+ */
+class AlterTableRenamePartitionSuite
+  extends AlterTableRenamePartitionSuiteBase
+  with CommandSuiteBase
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/AlterTableRenameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/AlterTableRenameSuite.scala
new file mode 100644
index 0000000000000..abc99db441db2
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/AlterTableRenameSuite.scala
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v1
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.execution.command
+
+/**
+ * This base suite contains unified tests for the `ALTER TABLE .. RENAME` command that check V1
+ * table catalogs. The tests that cannot run for all V1 catalogs are located in more
+ * specific test suites:
+ *
+ *   - V1 In-Memory catalog: `org.apache.spark.sql.execution.command.v1.AlterTableRenameSuite`
+ *   - V1 Hive External catalog: `org.apache.spark.sql.hive.execution.command.AlterTableRenameSuite`
+ */
+trait AlterTableRenameSuiteBase extends command.AlterTableRenameSuiteBase {
+  test("destination database is different") {
+    withNamespaceAndTable("dst_ns", "dst_tbl") { dst =>
+      withNamespace("src_ns") {
+        sql(s"CREATE NAMESPACE $catalog.src_ns")
+        val src = dst.replace("dst", "src")
+        sql(s"CREATE TABLE $src (c0 INT) $defaultUsing")
+        val errMsg = intercept[AnalysisException] {
+          sql(s"ALTER TABLE $src RENAME TO dst_ns.dst_tbl")
+        }.getMessage
+        assert(errMsg.contains("source and destination databases do not match"))
+      }
+    }
+  }
+
+  test("preserve table stats") {
+    withNamespaceAndTable("ns", "dst_tbl") { dst =>
+      val src = dst.replace("dst", "src")
+      sql(s"CREATE TABLE $src (c0 INT) $defaultUsing")
+      sql(s"INSERT INTO $src SELECT 0")
+      sql(s"ANALYZE TABLE $src COMPUTE STATISTICS")
+      val size = getTableSize(src)
+      assert(size > 0)
+      sql(s"ALTER TABLE $src RENAME TO ns.dst_tbl")
+      assert(size === getTableSize(dst))
+    }
+  }
+
+  test("the destination folder exists already") {
+    withNamespaceAndTable("ns", "dst_tbl") { dst =>
+      val src = dst.replace("dst", "src")
+      sql(s"CREATE TABLE $src (c0 INT) $defaultUsing")
+      sql(s"INSERT INTO $src SELECT 0")
+
+      sql(s"CREATE TABLE $dst (c0 INT) $defaultUsing")
+      withTableDir(dst) { (fs, dst_dir) =>
+        sql(s"DROP TABLE $dst")
+        fs.mkdirs(dst_dir)
+        val errMsg = intercept[AnalysisException] {
+          sql(s"ALTER TABLE $src RENAME TO ns.dst_tbl")
+        }.getMessage
+        assert(errMsg.matches("Can not rename the managed table(.+). " +
+          "The associated location(.+) already exists."))
+      }
+    }
+  }
+}
+
+/**
+ * The class contains tests for the `ALTER TABLE .. RENAME` command to check
+ * V1 In-Memory table catalog.
+ */
+class AlterTableRenameSuite extends AlterTableRenameSuiteBase with CommandSuiteBase
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/CommandSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/CommandSuiteBase.scala
new file mode 100644
index 0000000000000..80c552de567ba
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/CommandSuiteBase.scala
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v1
+
+import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
+import org.apache.spark.sql.connector.catalog.CatalogManager
+import org.apache.spark.sql.test.SharedSparkSession
+
+/**
+ * The trait contains settings and utility functions. It can be mixed to the test suites for
+ * datasource v1 In-Memory catalog. This trait complements the common trait
+ * `org.apache.spark.sql.execution.command.DDLCommandTestUtils` with utility functions and
+ * settings for all unified datasource V1 and V2 test suites.
+ */
+trait CommandSuiteBase extends SharedSparkSession {
+  def version: String = "V1" // The prefix is added to test names
+  def catalog: String = CatalogManager.SESSION_CATALOG_NAME
+  def defaultUsing: String = "USING parquet" // The clause is used in creating tables under testing
+
+  // TODO(SPARK-33393): Move this to `DDLCommandTestUtils`
+  def checkLocation(
+      t: String,
+      spec: TablePartitionSpec,
+      expected: String): Unit = {
+    val tablePath = t.split('.')
+    val tableName = tablePath.last
+    val ns = tablePath.init.mkString(".")
+    val partSpec = spec.map { case (key, value) => s"$key = $value"}.mkString(", ")
+    val information = sql(s"SHOW TABLE EXTENDED IN $ns LIKE '$tableName' PARTITION($partSpec)")
+      .select("information")
+      .first().getString(0)
+    val location = information.split("\\r?\\n").filter(_.startsWith("Location:")).head
+    assert(location.endsWith(expected))
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DropTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DropTableSuite.scala
new file mode 100644
index 0000000000000..497624f0a18de
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DropTableSuite.scala
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v1
+
+import org.apache.spark.sql.execution.command
+
+/**
+ * This base suite contains unified tests for the `DROP TABLE` command that check V1
+ * table catalogs. The tests that cannot run for all V1 catalogs are located in more
+ * specific test suites:
+ *
+ *   - V1 In-Memory catalog: `org.apache.spark.sql.execution.command.v1.DropTableSuite`
+ *   - V1 Hive External catalog: `org.apache.spark.sql.hive.execution.command.DropTableSuite`
+ */
+trait DropTableSuiteBase extends command.DropTableSuiteBase {
+  test("purge option") {
+    withNamespace(s"$catalog.ns") {
+      sql(s"CREATE NAMESPACE $catalog.ns")
+
+      createTable(s"$catalog.ns.tbl")
+      checkTables("ns", "tbl")
+
+      sql(s"DROP TABLE $catalog.ns.tbl PURGE")
+      checkTables("ns") // no tables
+    }
+  }
+}
+
+/**
+ * The class contains tests for the `DROP TABLE` command to check V1 In-Memory table catalog.
+ */
+class DropTableSuite extends DropTableSuiteBase with CommandSuiteBase
+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/MsckRepairTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/MsckRepairTableSuite.scala
new file mode 100644
index 0000000000000..45dc9e0e00f63
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/MsckRepairTableSuite.scala
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v1
+
+import java.io.File
+
+import org.apache.commons.io.FileUtils
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.execution.command
+
+/**
+ * This base suite contains unified tests for the `MSCK REPAIR TABLE` command that
+ * check V1 table catalogs. The tests that cannot run for all V1 catalogs are located in more
+ * specific test suites:
+ *
+ *   - V1 In-Memory catalog:
+ *     `org.apache.spark.sql.execution.command.v1.MsckRepairTableSuite`
+ *   - V1 Hive External catalog:
+ *     `org.apache.spark.sql.hive.execution.command.MsckRepairTableSuite`
+ */
+trait MsckRepairTableSuiteBase extends command.MsckRepairTableSuiteBase {
+  def deletePartitionDir(tableName: String, part: String): Unit = {
+    val partLoc = getPartitionLocation(tableName, part)
+    FileUtils.deleteDirectory(new File(partLoc))
+  }
+
+  test("drop partitions") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (col INT, part INT) $defaultUsing PARTITIONED BY (part)")
+      sql(s"INSERT INTO $t PARTITION (part=0) SELECT 0")
+      sql(s"INSERT INTO $t PARTITION (part=1) SELECT 1")
+
+      checkAnswer(spark.table(t), Seq(Row(0, 0), Row(1, 1)))
+      deletePartitionDir(t, "part=1")
+      sql(s"MSCK REPAIR TABLE $t DROP PARTITIONS")
+      checkPartitions(t, Map("part" -> "0"))
+      checkAnswer(spark.table(t), Seq(Row(0, 0)))
+    }
+  }
+
+  test("sync partitions") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (col INT, part INT) $defaultUsing PARTITIONED BY (part)")
+      sql(s"INSERT INTO $t PARTITION (part=0) SELECT 0")
+      sql(s"INSERT INTO $t PARTITION (part=1) SELECT 1")
+
+      checkAnswer(sql(s"SELECT col, part FROM $t"), Seq(Row(0, 0), Row(1, 1)))
+      copyPartition(t, "part=0", "part=2")
+      deletePartitionDir(t, "part=0")
+      sql(s"MSCK REPAIR TABLE $t SYNC PARTITIONS")
+      checkPartitions(t, Map("part" -> "1"), Map("part" -> "2"))
+      checkAnswer(sql(s"SELECT col, part FROM $t"), Seq(Row(1, 1), Row(0, 2)))
+    }
+  }
+}
+
+/**
+ * The class contains tests for the `MSCK REPAIR TABLE` command to check
+ * V1 In-Memory table catalog.
+ */
+class MsckRepairTableSuite extends MsckRepairTableSuiteBase with CommandSuiteBase
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowNamespacesSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowNamespacesSuite.scala
new file mode 100644
index 0000000000000..52742a2f2332e
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowNamespacesSuite.scala
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v1
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.execution.command
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * This base suite contains unified tests for the `SHOW NAMESPACES` and `SHOW DATABASES` commands
+ * that check V1 table catalogs. The tests that cannot run for all V1 catalogs are located in more
+ * specific test suites:
+ *
+ *   - V1 In-Memory catalog: `org.apache.spark.sql.execution.command.v1.ShowNamespacesSuite`
+ *   - V1 Hive External catalog: `org.apache.spark.sql.hive.execution.command.ShowNamespacesSuite`
+ */
+trait ShowNamespacesSuiteBase extends command.ShowNamespacesSuiteBase {
+  override protected def builtinTopNamespaces: Seq[String] = Seq("default")
+
+  test("IN namespace doesn't exist") {
+    val errMsg = intercept[AnalysisException] {
+      sql("SHOW NAMESPACES in dummy")
+    }.getMessage
+    assert(errMsg.contains("Namespace 'dummy' not found"))
+  }
+
+  test("SPARK-34359: keep the legacy output schema") {
+    withSQLConf(SQLConf.LEGACY_KEEP_COMMAND_OUTPUT_SCHEMA.key -> "true") {
+      assert(sql("SHOW NAMESPACES").schema.fieldNames.toSeq == Seq("databaseName"))
+    }
+  }
+}
+
+class ShowNamespacesSuite extends ShowNamespacesSuiteBase with CommandSuiteBase {
+  test("case sensitivity") {
+    Seq(true, false).foreach { caseSensitive =>
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
+        withNamespace(s"$catalog.AAA", s"$catalog.bbb") {
+          sql(s"CREATE NAMESPACE $catalog.AAA")
+          sql(s"CREATE NAMESPACE $catalog.bbb")
+          val expected = if (caseSensitive) "AAA" else "aaa"
+          runShowNamespacesSql(
+            s"SHOW NAMESPACES IN $catalog",
+            Seq(expected, "bbb") ++ builtinTopNamespaces)
+          runShowNamespacesSql(s"SHOW NAMESPACES IN $catalog LIKE 'AAA'", Seq(expected))
+          runShowNamespacesSql(s"SHOW NAMESPACES IN $catalog LIKE 'aaa'", Seq(expected))
+        }
+      }
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowPartitionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowPartitionsSuite.scala
new file mode 100644
index 0000000000000..a26e29706e147
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowPartitionsSuite.scala
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v1
+
+import org.apache.spark.sql.{AnalysisException, Row, SaveMode}
+import org.apache.spark.sql.execution.command
+
+/**
+ * This base suite contains unified tests for the `SHOW PARTITIONS` command that check V1
+ * table catalogs. The tests that cannot run for all V1 catalogs are located in more
+ * specific test suites:
+ *
+ *   - V1 In-Memory catalog: `org.apache.spark.sql.execution.command.v1.ShowPartitionsSuite`
+ *   - V1 Hive External catalog: `org.apache.spark.sql.hive.execution.command.ShowPartitionsSuite`
+ */
+trait ShowPartitionsSuiteBase extends command.ShowPartitionsSuiteBase {
+  test("show everything in the default database") {
+    val table = "dateTable"
+    withTable(table) {
+      createDateTable(table)
+      runShowPartitionsSql(
+        s"show partitions default.$table",
+        Row("year=2015/month=1") ::
+        Row("year=2015/month=2") ::
+        Row("year=2016/month=2") ::
+        Row("year=2016/month=3") :: Nil)
+    }
+  }
+
+  // The test fails for V2 Table Catalogs with the exception:
+  // org.apache.spark.sql.AnalysisException: CREATE VIEW is only supported with v1 tables.
+  test("show partitions of a view") {
+    val table = "dateTable"
+    withTable(table) {
+      createDateTable(table)
+      val view = "view1"
+      withView(view) {
+        sql(s"CREATE VIEW $view as select * from $table")
+        val errMsg = intercept[AnalysisException] {
+          sql(s"SHOW PARTITIONS $view")
+        }.getMessage
+        assert(errMsg.contains("'SHOW PARTITIONS' expects a table"))
+      }
+    }
+  }
+
+  test("show partitions of a temporary view") {
+    val viewName = "test_view"
+    withTempView(viewName) {
+      spark.range(10).createTempView(viewName)
+      val errMsg = intercept[AnalysisException] {
+        sql(s"SHOW PARTITIONS $viewName")
+      }.getMessage
+      assert(errMsg.contains("'SHOW PARTITIONS' expects a table"))
+    }
+  }
+
+  test("SPARK-33591: null as a partition value") {
+    val t = "part_table"
+    withTable(t) {
+      sql(s"CREATE TABLE $t (col1 INT, p1 STRING) $defaultUsing PARTITIONED BY (p1)")
+      sql(s"INSERT INTO TABLE $t PARTITION (p1 = null) SELECT 0")
+      checkAnswer(sql(s"SHOW PARTITIONS $t"), Row("p1=__HIVE_DEFAULT_PARTITION__"))
+      checkAnswer(
+        sql(s"SHOW PARTITIONS $t PARTITION (p1 = null)"),
+        Row("p1=__HIVE_DEFAULT_PARTITION__"))
+    }
+  }
+}
+
+/**
+ * The class contains tests for the `SHOW PARTITIONS` command to check V1 In-Memory table catalog.
+ */
+class ShowPartitionsSuite extends ShowPartitionsSuiteBase with CommandSuiteBase {
+  // The test is placed here because it fails with `USING HIVE`:
+  // org.apache.spark.sql.AnalysisException:
+  //   Hive data source can only be used with tables, you can't use it with CREATE TEMP VIEW USING
+  test("issue exceptions on the temporary view") {
+    val viewName = "test_view"
+    withTempView(viewName) {
+      sql(s"""
+        |CREATE TEMPORARY VIEW $viewName (c1 INT, c2 STRING)
+        |$defaultUsing""".stripMargin)
+      val errMsg = intercept[AnalysisException] {
+        sql(s"SHOW PARTITIONS $viewName")
+      }.getMessage
+      assert(errMsg.contains("'SHOW PARTITIONS' expects a table"))
+    }
+  }
+
+  test("show partitions from a datasource") {
+    import testImplicits._
+    withTable("part_datasrc") {
+      val df = (1 to 3).map(i => (i, s"val_$i", i * 2)).toDF("a", "b", "c")
+      df.write
+        .partitionBy("a")
+        .format("parquet")
+        .mode(SaveMode.Overwrite)
+        .saveAsTable("part_datasrc")
+
+      assert(sql("SHOW PARTITIONS part_datasrc").count() == 3)
+    }
+  }
+
+  test("SPARK-33904: null and empty string as partition values") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      createNullPartTable(t, "parquet")
+      runShowPartitionsSql(
+        s"SHOW PARTITIONS $t",
+        Row("part=__HIVE_DEFAULT_PARTITION__") :: Nil)
+      checkAnswer(spark.table(t), Row(0, null) :: Row(1, null) :: Nil)
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowTablesSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowTablesSuite.scala
new file mode 100644
index 0000000000000..4efb7c85e5066
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowTablesSuite.scala
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v1
+
+import org.apache.spark.sql.{AnalysisException, Row, SaveMode}
+import org.apache.spark.sql.execution.command
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * This base suite contains unified tests for the `SHOW TABLES` command that check V1
+ * table catalogs. The tests that cannot run for all V1 catalogs are located in more
+ * specific test suites:
+ *
+ *   - V1 In-Memory catalog: `org.apache.spark.sql.execution.command.v1.ShowTablesSuite`
+ *   - V1 Hive External catalog: `org.apache.spark.sql.hive.execution.command.ShowTablesSuite`
+ */
+trait ShowTablesSuiteBase extends command.ShowTablesSuiteBase {
+  override def defaultNamespace: Seq[String] = Seq("default")
+
+  private def withSourceViews(f: => Unit): Unit = {
+    withTable("source", "source2") {
+      val df = spark.createDataFrame(Seq((1L, "a"), (2L, "b"), (3L, "c"))).toDF("id", "data")
+      df.createOrReplaceTempView("source")
+      val df2 = spark.createDataFrame(Seq((4L, "d"), (5L, "e"), (6L, "f"))).toDF("id", "data")
+      df2.createOrReplaceTempView("source2")
+      f
+    }
+  }
+
+  // `SHOW TABLES` from v2 catalog returns empty result.
+  test("v1 SHOW TABLES list the temp views") {
+    withSourceViews {
+      runShowTablesSql(
+        "SHOW TABLES FROM default",
+        Seq(Row("", "source", true), Row("", "source2", true)))
+    }
+  }
+
+  test("only support single-level namespace") {
+    val errMsg = intercept[AnalysisException] {
+      runShowTablesSql("SHOW TABLES FROM a.b", Seq())
+    }.getMessage
+    assert(errMsg.contains("Nested databases are not supported by v1 session catalog: a.b"))
+  }
+
+  test("SHOW TABLE EXTENDED from default") {
+    withSourceViews {
+      val expected = Seq(Row("", "source", true), Row("", "source2", true))
+
+      val df = sql("SHOW TABLE EXTENDED FROM default LIKE '*source*'")
+      val result = df.collect()
+      val resultWithoutInfo = result.map { case Row(db, table, temp, _) => Row(db, table, temp) }
+
+      assert(resultWithoutInfo === expected)
+      result.foreach { case Row(_, _, _, info: String) => assert(info.nonEmpty) }
+    }
+  }
+
+  test("case sensitivity of partition spec") {
+    withNamespaceAndTable("ns", "part_table") { t =>
+      sql(s"""
+        |CREATE TABLE $t (price int, qty int, year int, month int)
+        |$defaultUsing
+        |partitioned by (year, month)""".stripMargin)
+      sql(s"INSERT INTO $t PARTITION(year = 2015, month = 1) SELECT 1, 1")
+      Seq(
+        true -> "PARTITION(year = 2015, month = 1)",
+        false -> "PARTITION(YEAR = 2015, Month = 1)"
+      ).foreach { case (caseSensitive, partitionSpec) =>
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
+          val df = sql(s"SHOW TABLE EXTENDED LIKE 'part_table' $partitionSpec")
+          val information = df.select("information").first().getString(0)
+          assert(information.contains("Partition Values: [year=2015, month=1]"))
+        }
+      }
+    }
+  }
+
+  test("no database specified") {
+    Seq(
+      s"SHOW TABLES IN $catalog",
+      s"SHOW TABLE EXTENDED IN $catalog LIKE '*tbl'").foreach { showTableCmd =>
+      val errMsg = intercept[AnalysisException] {
+        sql(showTableCmd)
+      }.getMessage
+      assert(errMsg.contains("Database from v1 session catalog is not specified"))
+    }
+  }
+
+  test("SPARK-34157: Unify output of SHOW TABLES and pass output attributes properly") {
+    withNamespace(s"$catalog.ns") {
+      sql(s"CREATE NAMESPACE $catalog.ns")
+      sql(s"USE $catalog.ns")
+      withTable("tbl") {
+        sql("CREATE TABLE tbl(col1 int, col2 string) USING parquet")
+        checkAnswer(sql("show tables"), Row("ns", "tbl", false))
+        assert(sql("show tables").schema.fieldNames ===
+          Seq("namespace", "tableName", "isTemporary"))
+        assert(sql("show table extended like 'tbl'").collect()(0).length == 4)
+        assert(sql("show table extended like 'tbl'").schema.fieldNames ===
+          Seq("namespace", "tableName", "isTemporary", "information"))
+
+        // Keep the legacy output schema
+        withSQLConf(SQLConf.LEGACY_KEEP_COMMAND_OUTPUT_SCHEMA.key -> "true") {
+          checkAnswer(sql("show tables"), Row("ns", "tbl", false))
+          assert(sql("show tables").schema.fieldNames ===
+            Seq("database", "tableName", "isTemporary"))
+          assert(sql("show table extended like 'tbl'").collect()(0).length == 4)
+          assert(sql("show table extended like 'tbl'").schema.fieldNames ===
+            Seq("database", "tableName", "isTemporary", "information"))
+        }
+      }
+    }
+  }
+}
+
+/**
+ * The class contains tests for the `SHOW TABLES` command to check V1 In-Memory table catalog.
+ */
+class ShowTablesSuite extends ShowTablesSuiteBase with CommandSuiteBase {
+  test("SPARK-33670: show partitions from a datasource table") {
+    import testImplicits._
+    withNamespace(s"$catalog.ns") {
+      sql(s"CREATE NAMESPACE $catalog.ns")
+      sql(s"USE $catalog.ns")
+      val t = "part_datasrc"
+      withTable(t) {
+        val df = (1 to 3).map(i => (i, s"val_$i", i * 2)).toDF("a", "b", "c")
+        df.write.partitionBy("a").format("parquet").mode(SaveMode.Overwrite).saveAsTable(t)
+        assert(sql(s"SHOW TABLE EXTENDED LIKE '$t' PARTITION(a = 1)").count() === 1)
+      }
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/TruncateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/TruncateTableSuite.scala
new file mode 100644
index 0000000000000..7da03db6f7371
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/TruncateTableSuite.scala
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v1
+
+import java.io.File
+
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.fs.permission.{AclEntry, AclEntryScope, AclEntryType, FsAction, FsPermission}
+
+import org.apache.spark.sql.{AnalysisException, Row}
+import org.apache.spark.sql.catalyst.{QualifiedTableName, TableIdentifier}
+import org.apache.spark.sql.execution.command
+import org.apache.spark.sql.execution.command.FakeLocalFsFileSystem
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * This base suite contains unified tests for the `TRUNCATE TABLE` command that check V1
+ * table catalogs. The tests that cannot run for all V1 catalogs are located in more
+ * specific test suites:
+ *
+ *   - V1 In-Memory catalog: `org.apache.spark.sql.execution.command.v1.TruncateTableSuite`
+ *   - V1 Hive External catalog: `org.apache.spark.sql.hive.execution.command.TruncateTableSuite`
+ */
+trait TruncateTableSuiteBase extends command.TruncateTableSuiteBase {
+
+  override val invalidPartColumnError =
+    "TRUNCATE TABLE ... PARTITION is not supported for tables that are not partitioned"
+
+  test("SPARK-30312: truncate table - keep acl/permission") {
+    Seq(true, false).foreach { ignore =>
+      withSQLConf(
+        "fs.file.impl" -> classOf[FakeLocalFsFileSystem].getName,
+        "fs.file.impl.disable.cache" -> "true",
+        SQLConf.TRUNCATE_TABLE_IGNORE_PERMISSION_ACL.key -> ignore.toString) {
+        withNamespaceAndTable("ns", "tbl") { t =>
+          sql(s"CREATE TABLE $t (col INT) $defaultUsing")
+          sql(s"INSERT INTO $t SELECT 1")
+          checkAnswer(spark.table(t), Row(1))
+
+          val tablePath = new Path(spark.sessionState.catalog
+            .getTableMetadata(TableIdentifier("tbl", Some("ns")))
+            .storage.locationUri.get)
+
+          val hadoopConf = spark.sessionState.newHadoopConf()
+          val fs = tablePath.getFileSystem(hadoopConf)
+          val fileStatus = fs.getFileStatus(tablePath);
+
+          fs.setPermission(tablePath, new FsPermission("777"))
+          assert(fileStatus.getPermission().toString() == "rwxrwxrwx")
+
+          // Set ACL to table path.
+          val customAcl = new java.util.ArrayList[AclEntry]()
+          customAcl.add(new AclEntry.Builder()
+            .setName("test")
+            .setType(AclEntryType.USER)
+            .setScope(AclEntryScope.ACCESS)
+            .setPermission(FsAction.READ).build())
+          fs.setAcl(tablePath, customAcl)
+          assert(fs.getAclStatus(tablePath).getEntries().get(0) == customAcl.get(0))
+
+          sql(s"TRUNCATE TABLE $t")
+          assert(spark.table(t).collect().isEmpty)
+
+          val fileStatus2 = fs.getFileStatus(tablePath)
+          if (ignore) {
+            assert(fileStatus2.getPermission().toString() != "rwxrwxrwx")
+          } else {
+            assert(fileStatus2.getPermission().toString() == "rwxrwxrwx")
+          }
+          val aclEntries = fs.getAclStatus(tablePath).getEntries()
+          if (ignore) {
+            assert(aclEntries.size() == 0)
+          } else {
+            assert(aclEntries.size() == 4)
+            assert(aclEntries.get(0) == customAcl.get(0))
+
+            // Setting ACLs will also set user/group/other permissions
+            // as ACL entries.
+            val user = new AclEntry.Builder()
+              .setType(AclEntryType.USER)
+              .setScope(AclEntryScope.ACCESS)
+              .setPermission(FsAction.ALL).build()
+            val group = new AclEntry.Builder()
+              .setType(AclEntryType.GROUP)
+              .setScope(AclEntryScope.ACCESS)
+              .setPermission(FsAction.ALL).build()
+            val other = new AclEntry.Builder()
+              .setType(AclEntryType.OTHER)
+              .setScope(AclEntryScope.ACCESS)
+              .setPermission(FsAction.ALL).build()
+            assert(aclEntries.get(1) == user)
+            assert(aclEntries.get(2) == group)
+            assert(aclEntries.get(3) == other)
+          }
+        }
+      }
+    }
+  }
+
+  test("SPARK-31163: acl/permission should handle non-existed path when truncating table") {
+    withSQLConf(SQLConf.TRUNCATE_TABLE_IGNORE_PERMISSION_ACL.key -> "false") {
+      withNamespaceAndTable("ns", "tbl") { t =>
+        sql(s"CREATE TABLE $t (col1 STRING, col2 INT) $defaultUsing PARTITIONED BY (col2)")
+        sql(s"INSERT INTO $t PARTITION (col2 = 1) SELECT 'one'")
+        checkAnswer(spark.table(t), Row("one", 1))
+        val part = spark.sessionState.catalog
+          .listPartitions(TableIdentifier("tbl", Some("ns")))
+          .head
+        val path = new File(part.location.getPath)
+        sql(s"TRUNCATE TABLE $t")
+        // simulate incomplete/unsuccessful truncate
+        assert(path.exists())
+        path.delete()
+        assert(!path.exists())
+        // execute without java.io.FileNotFoundException
+        sql(s"TRUNCATE TABLE $t")
+        // partition path should be re-created
+        assert(path.exists())
+      }
+    }
+  }
+
+  test("invalidation of tableRelationCache after table truncation") {
+    Seq(false, true).foreach { autoUpdate =>
+      withSQLConf(SQLConf.AUTO_SIZE_UPDATE_ENABLED.key -> autoUpdate.toString) {
+        withNamespaceAndTable("ns", "tbl") { t =>
+          spark.range(100).write.saveAsTable(t)
+          sql(s"ANALYZE TABLE $t COMPUTE STATISTICS")
+          spark.table(t)
+          sql(s"TRUNCATE TABLE $t")
+          spark.table(t)
+
+          val catalog = spark.sessionState.catalog
+          val qualifiedTableName = QualifiedTableName("ns", "tbl")
+          val cachedPlan = catalog.getCachedTable(qualifiedTableName)
+          assert(cachedPlan.stats.sizeInBytes == 0)
+        }
+      }
+    }
+  }
+
+  test("change stats after truncate command") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (id INT, value INT) $defaultUsing")
+      sql(s"INSERT INTO $t SELECT 0, 100")
+      // analyze to get initial stats
+      sql(s"ANALYZE TABLE $t COMPUTE STATISTICS FOR COLUMNS id, value")
+      assert(getTableSize(t) > 0)
+
+      withSQLConf(SQLConf.AUTO_SIZE_UPDATE_ENABLED.key -> "true") {
+        sql(s"TRUNCATE TABLE $t")
+        assert(getTableSize(t) == 0)
+      }
+    }
+  }
+
+  test("SPARK-34251: stats in truncated non-empty table") {
+    withSQLConf(SQLConf.AUTO_SIZE_UPDATE_ENABLED.key -> "true") {
+      withNamespaceAndTable("ns", "tbl") { t =>
+        sql(s"CREATE TABLE $t (c0 int, part int) $defaultUsing PARTITIONED BY (part)")
+        sql(s"INSERT INTO $t PARTITION (part=0) SELECT 0")
+        sql(s"INSERT INTO $t PARTITION (part=1) SELECT 1")
+        val sizeOfTwoParts = getTableSize(t)
+        assert(sizeOfTwoParts > 0)
+        sql(s"TRUNCATE TABLE $t PARTITION (part=1)")
+        val sizeOfOnePart = getTableSize(t)
+        assert(0 < sizeOfOnePart && sizeOfOnePart < sizeOfTwoParts)
+      }
+    }
+  }
+}
+
+/**
+ * The class contains tests for the `TRUNCATE TABLE` command to check V1 In-Memory table catalog.
+ */
+class TruncateTableSuite extends TruncateTableSuiteBase with CommandSuiteBase {
+  test("truncation of external tables is not allowed") {
+    import testImplicits._
+    withTempPath { tempDir =>
+      withNamespaceAndTable("ns", "tbl") { t =>
+        (("a", "b") :: Nil).toDF().write.parquet(tempDir.getCanonicalPath)
+        sql(s"CREATE TABLE $t $defaultUsing LOCATION '${tempDir.toURI}'")
+        val errMsg = intercept[AnalysisException] {
+          sql(s"TRUNCATE TABLE $t")
+        }.getMessage
+        assert(errMsg.contains("Operation not allowed: TRUNCATE TABLE on external tables"))
+      }
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/AlterTableAddPartitionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/AlterTableAddPartitionSuite.scala
new file mode 100644
index 0000000000000..fabe399c340ae
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/AlterTableAddPartitionSuite.scala
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v2
+
+import org.apache.spark.sql.{AnalysisException, Row}
+import org.apache.spark.sql.execution.command
+
+/**
+ * The class contains tests for the `ALTER TABLE .. ADD PARTITION` command
+ * to check V2 table catalogs.
+ */
+class AlterTableAddPartitionSuite
+  extends command.AlterTableAddPartitionSuiteBase
+  with CommandSuiteBase {
+  test("SPARK-33650: add partition into a table which doesn't support partition management") {
+    withNamespaceAndTable("ns", "tbl", s"non_part_$catalog") { t =>
+      sql(s"CREATE TABLE $t (id bigint, data string) $defaultUsing")
+      val errMsg = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $t ADD PARTITION (id=1)")
+      }.getMessage
+      assert(errMsg.contains(s"Table $t does not support partition management"))
+    }
+  }
+
+  test("empty string as partition value") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (col1 INT, p1 STRING) $defaultUsing PARTITIONED BY (p1)")
+      sql(s"ALTER TABLE $t ADD PARTITION (p1 = '')")
+      checkPartitions(t, Map("p1" -> ""))
+    }
+  }
+
+  test("SPARK-34143: add a partition to fully partitioned table") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (p0 INT, p1 STRING) $defaultUsing PARTITIONED BY (p0, p1)")
+      sql(s"ALTER TABLE $t ADD PARTITION (p0 = 0, p1 = 'abc')")
+      checkPartitions(t, Map("p0" -> "0", "p1" -> "abc"))
+      checkAnswer(sql(s"SELECT * FROM $t"), Row(0, "abc"))
+    }
+  }
+
+  test("SPARK-34149: refresh cache in partition adding") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (part int) $defaultUsing PARTITIONED BY (part)")
+      sql(s"ALTER TABLE $t ADD PARTITION (part=0)")
+      assert(!spark.catalog.isCached(t))
+      sql(s"CACHE TABLE $t")
+      assert(spark.catalog.isCached(t))
+      checkAnswer(sql(s"SELECT * FROM $t"), Row(0))
+
+      sql(s"ALTER TABLE $t ADD PARTITION (part=1)")
+      assert(spark.catalog.isCached(t))
+      checkAnswer(sql(s"SELECT * FROM $t"), Seq(Row(0), Row(1)))
+    }
+  }
+
+  test("SPARK-34099, SPARK-34161: keep dependents cached after table altering") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (id int, part int) $defaultUsing PARTITIONED BY (id, part)")
+      sql(s"INSERT INTO $t PARTITION (part=0) SELECT 0")
+      cacheRelation(t)
+      checkCachedRelation(t, Seq(Row(0, 0)))
+
+      withView("v0") {
+        sql(s"CREATE VIEW v0 AS SELECT * FROM $t")
+        cacheRelation("v0")
+        sql(s"ALTER TABLE $t ADD PARTITION (id=0, part=1)")
+        checkCachedRelation("v0", Seq(Row(0, 0), Row(0, 1)))
+      }
+
+      withTempView("v1") {
+        sql(s"CREATE TEMP VIEW v1 AS SELECT * FROM $t")
+        cacheRelation("v1")
+        sql(s"ALTER TABLE $t ADD PARTITION (id=1, part=2)")
+        checkCachedRelation("v1", Seq(Row(0, 0), Row(0, 1), Row(1, 2)))
+      }
+
+      val v2 = s"${spark.sharedState.globalTempViewManager.database}.v2"
+      withGlobalTempView(v2) {
+        sql(s"CREATE GLOBAL TEMP VIEW v2 AS SELECT * FROM $t")
+        cacheRelation(v2)
+        sql(s"ALTER TABLE $t ADD PARTITION (id=2, part=3)")
+        checkCachedRelation(v2, Seq(Row(0, 0), Row(0, 1), Row(1, 2), Row(2, 3)))
+      }
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/AlterTableDropPartitionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/AlterTableDropPartitionSuite.scala
new file mode 100644
index 0000000000000..b03c8fb17f542
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/AlterTableDropPartitionSuite.scala
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v2
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.execution.command
+
+/**
+ * The class contains tests for the `ALTER TABLE .. DROP PARTITION` command
+ * to check V2 table catalogs.
+ */
+class AlterTableDropPartitionSuite
+  extends command.AlterTableDropPartitionSuiteBase
+  with CommandSuiteBase {
+  override protected val notFullPartitionSpecErr = "Partition spec is invalid"
+  override protected def nullPartitionValue: String = "null"
+
+  test("SPARK-33650: drop partition into a table which doesn't support partition management") {
+    withNamespaceAndTable("ns", "tbl", s"non_part_$catalog") { t =>
+      sql(s"CREATE TABLE $t (id bigint, data string) $defaultUsing")
+      val errMsg = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $t DROP PARTITION (id=1)")
+      }.getMessage
+      assert(errMsg.contains(s"Table $t does not support partition management"))
+    }
+  }
+
+  test("purge partition data") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (id bigint, data string) $defaultUsing PARTITIONED BY (id)")
+      sql(s"ALTER TABLE $t ADD PARTITION (id=1)")
+      try {
+        val errMsg = intercept[UnsupportedOperationException] {
+          sql(s"ALTER TABLE $t DROP PARTITION (id=1) PURGE")
+        }.getMessage
+        assert(errMsg.contains("purge is not supported"))
+      } finally {
+        sql(s"ALTER TABLE $t DROP PARTITION (id=1)")
+      }
+    }
+  }
+
+  test("empty string as partition value") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (col1 INT, p1 STRING) $defaultUsing PARTITIONED BY (p1)")
+      sql(s"ALTER TABLE $t ADD PARTITION (p1 = '')")
+      sql(s"ALTER TABLE $t DROP PARTITION (p1 = '')")
+      checkPartitions(t)
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/AlterTableRecoverPartitionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/AlterTableRecoverPartitionsSuite.scala
new file mode 100644
index 0000000000000..a44e346d0348c
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/AlterTableRecoverPartitionsSuite.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v2
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.execution.command
+
+/**
+ * The class contains tests for the `ALTER TABLE .. RECOVER PARTITIONS` command
+ * to check V2 table catalogs.
+ */
+class AlterTableRecoverPartitionsSuite
+  extends command.AlterTableRecoverPartitionsSuiteBase
+  with CommandSuiteBase {
+
+  test("partition recovering of v2 tables is not supported") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      spark.sql(s"CREATE TABLE $t (id bigint, data string) $defaultUsing")
+      val errMsg = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $t RECOVER PARTITIONS")
+      }.getMessage
+      assert(errMsg.contains("ALTER TABLE ... RECOVER PARTITIONS is not supported for v2 tables"))
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/AlterTableRenamePartitionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/AlterTableRenamePartitionSuite.scala
new file mode 100644
index 0000000000000..bb06818da48b1
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/AlterTableRenamePartitionSuite.scala
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v2
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.execution.command
+
+/**
+ * The class contains tests for the `ALTER TABLE .. RENAME PARTITION` command
+ * to check V2 table catalogs.
+ */
+class AlterTableRenamePartitionSuite
+  extends command.AlterTableRenamePartitionSuiteBase
+  with CommandSuiteBase {
+
+  test("with location") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      createSinglePartTable(t)
+      val loc = "location1"
+      sql(s"ALTER TABLE $t ADD PARTITION (id = 2) LOCATION '$loc'")
+      sql(s"INSERT INTO $t PARTITION (id = 2) SELECT 'def'")
+      checkPartitions(t, Map("id" -> "1"), Map("id" -> "2"))
+      checkLocation(t, Map("id" -> "2"), loc)
+
+      sql(s"ALTER TABLE $t PARTITION (id = 2) RENAME TO PARTITION (id = 3)")
+      checkPartitions(t, Map("id" -> "1"), Map("id" -> "3"))
+      // `InMemoryPartitionTableCatalog` should keep the original location
+      checkLocation(t, Map("id" -> "3"), loc)
+      checkAnswer(sql(s"SELECT id, data FROM $t WHERE id = 3"), Row(3, "def"))
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/AlterTableRenameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/AlterTableRenameSuite.scala
new file mode 100644
index 0000000000000..9e89c079dc982
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/AlterTableRenameSuite.scala
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v2
+
+import org.apache.spark.sql.execution.command
+
+/**
+ * The class contains tests for the `ALTER TABLE .. RENAME` command to check V2 table catalogs.
+ */
+class AlterTableRenameSuite extends command.AlterTableRenameSuiteBase with CommandSuiteBase {
+  test("destination namespace is different") {
+    withNamespaceAndTable("dst_ns", "dst_tbl") { dst =>
+      withNamespace("src_ns") {
+        sql(s"CREATE NAMESPACE $catalog.src_ns")
+        val src = dst.replace("dst", "src")
+        sql(s"CREATE TABLE $src (c0 INT) $defaultUsing")
+        sql(s"ALTER TABLE $src RENAME TO dst_ns.dst_tbl")
+        checkTables("dst_ns", "dst_tbl")
+      }
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CommandSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CommandSuiteBase.scala
new file mode 100644
index 0000000000000..bed04f4f2659b
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CommandSuiteBase.scala
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v2
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.catalyst.analysis.ResolvePartitionSpec
+import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
+import org.apache.spark.sql.connector.catalog.{CatalogV2Implicits, Identifier, InMemoryPartitionTable, InMemoryPartitionTableCatalog, InMemoryTableCatalog}
+import org.apache.spark.sql.test.SharedSparkSession
+
+/**
+ * The trait contains settings and utility functions. It can be mixed to the test suites for
+ * datasource v2 catalogs (in-memory test catalogs). This trait complements the trait
+ * `org.apache.spark.sql.execution.command.DDLCommandTestUtils` with common utility functions
+ * for all unified datasource V1 and V2 test suites.
+ */
+trait CommandSuiteBase extends SharedSparkSession {
+  def version: String = "V2" // The prefix is added to test names
+  def catalog: String = "test_catalog" // The default V2 catalog for testing
+  def defaultUsing: String = "USING _" // The clause is used in creating v2 tables under testing
+
+  // V2 catalogs created and used especially for testing
+  override def sparkConf: SparkConf = super.sparkConf
+    .set(s"spark.sql.catalog.$catalog", classOf[InMemoryPartitionTableCatalog].getName)
+    .set(s"spark.sql.catalog.non_part_$catalog", classOf[InMemoryTableCatalog].getName)
+
+  def checkLocation(
+      t: String,
+      spec: TablePartitionSpec,
+      expected: String): Unit = {
+    import CatalogV2Implicits._
+
+    val tablePath = t.split('.')
+    val catalogName = tablePath.head
+    val namespaceWithTable = tablePath.tail
+    val namespaces = namespaceWithTable.init
+    val tableName = namespaceWithTable.last
+    val catalogPlugin = spark.sessionState.catalogManager.catalog(catalogName)
+    val partTable = catalogPlugin.asTableCatalog
+      .loadTable(Identifier.of(namespaces, tableName))
+      .asInstanceOf[InMemoryPartitionTable]
+    val ident = ResolvePartitionSpec.convertToPartIdent(spec, partTable.partitionSchema.fields)
+    val partMetadata = partTable.loadPartitionMetadata(ident)
+
+    assert(partMetadata.containsKey("location"))
+    assert(partMetadata.get("location") === expected)
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DropTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DropTableSuite.scala
new file mode 100644
index 0000000000000..9c9b7d3049c7a
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DropTableSuite.scala
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v2
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.connector.InMemoryTableSessionCatalog
+import org.apache.spark.sql.execution.command
+import org.apache.spark.sql.internal.SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION
+
+/**
+ * The class contains tests for the `DROP TABLE` command to check V2 table catalogs.
+ */
+class DropTableSuite extends command.DropTableSuiteBase with CommandSuiteBase {
+  test("purge option") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      createTable(t)
+      val errMsg = intercept[UnsupportedOperationException] {
+        sql(s"DROP TABLE $catalog.ns.tbl PURGE")
+      }.getMessage
+      // The default TableCatalog.purgeTable implementation throws an exception.
+      assert(errMsg.contains("Purge table is not supported"))
+    }
+  }
+
+  test("table qualified with the session catalog name") {
+    withSQLConf(
+      V2_SESSION_CATALOG_IMPLEMENTATION.key -> classOf[InMemoryTableSessionCatalog].getName) {
+
+      sql("CREATE TABLE tbl USING json AS SELECT 1 AS i")
+      checkAnswer(
+        sql("SHOW TABLES IN spark_catalog.default").select("tableName"),
+        Row("tbl"))
+
+      sql("DROP TABLE spark_catalog.default.tbl")
+      checkAnswer(
+        sql("SHOW TABLES IN spark_catalog.default").select("tableName"),
+        Seq.empty)
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/MsckRepairTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/MsckRepairTableSuite.scala
new file mode 100644
index 0000000000000..d4b23e50786eb
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/MsckRepairTableSuite.scala
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v2
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.execution.command
+
+/**
+ * The class contains tests for the `MSCK REPAIR TABLE` command
+ * to check V2 table catalogs.
+ */
+class MsckRepairTableSuite
+  extends command.MsckRepairTableSuiteBase
+  with CommandSuiteBase {
+
+  // TODO(SPARK-34397): Support v2 `MSCK REPAIR TABLE`
+  test("repairing of v2 tables is not supported") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      spark.sql(s"CREATE TABLE $t (id bigint, data string) $defaultUsing")
+      val errMsg = intercept[AnalysisException] {
+        sql(s"MSCK REPAIR TABLE $t")
+      }.getMessage
+      assert(errMsg.contains("MSCK REPAIR TABLE is not supported for v2 tables"))
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowNamespacesSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowNamespacesSuite.scala
new file mode 100644
index 0000000000000..bafb6608c8e6c
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowNamespacesSuite.scala
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v2
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.connector.catalog.BasicInMemoryTableCatalog
+import org.apache.spark.sql.execution.command
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * The class contains tests for the `SHOW NAMESPACES` command to check V2 table catalogs.
+ */
+class ShowNamespacesSuite extends command.ShowNamespacesSuiteBase with CommandSuiteBase {
+  override def sparkConf: SparkConf = super.sparkConf
+    .set("spark.sql.catalog.testcat_no_namespace", classOf[BasicInMemoryTableCatalog].getName)
+
+  test("IN namespace doesn't exist") {
+    withSQLConf(SQLConf.DEFAULT_CATALOG.key -> catalog) {
+      runShowNamespacesSql("SHOW NAMESPACES in dummy", Seq.empty)
+    }
+    runShowNamespacesSql(s"SHOW NAMESPACES in $catalog.ns1", Seq.empty)
+    runShowNamespacesSql(s"SHOW NAMESPACES in $catalog.ns1.ns3", Seq.empty)
+  }
+
+  test("default v2 catalog doesn't support namespace") {
+    withSQLConf(SQLConf.DEFAULT_CATALOG.key -> "testcat_no_namespace") {
+      val errMsg = intercept[AnalysisException] {
+        sql("SHOW NAMESPACES")
+      }.getMessage
+      assert(errMsg.contains("does not support namespaces"))
+    }
+  }
+
+  test("v2 catalog doesn't support namespace") {
+    val errMsg = intercept[AnalysisException] {
+      sql("SHOW NAMESPACES in testcat_no_namespace")
+    }.getMessage
+    assert(errMsg.contains("does not support namespaces"))
+  }
+
+  test("case sensitivity") {
+    Seq(true, false).foreach { caseSensitive =>
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
+        withNamespace(s"$catalog.AAA", s"$catalog.bbb") {
+          sql(s"CREATE NAMESPACE $catalog.AAA")
+          sql(s"CREATE NAMESPACE $catalog.bbb")
+          runShowNamespacesSql(
+            s"SHOW NAMESPACES IN $catalog",
+            Seq("AAA", "bbb") ++ builtinTopNamespaces)
+          runShowNamespacesSql(s"SHOW NAMESPACES IN $catalog LIKE 'AAA'", Seq("AAA"))
+          runShowNamespacesSql(s"SHOW NAMESPACES IN $catalog LIKE 'aaa'", Seq("AAA"))
+        }
+      }
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowPartitionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowPartitionsSuite.scala
new file mode 100644
index 0000000000000..8ae8171924c69
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowPartitionsSuite.scala
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v2
+
+import org.apache.spark.sql.{AnalysisException, Row}
+import org.apache.spark.sql.execution.command
+
+/**
+ * The class contains tests for the `SHOW PARTITIONS` command to check V2 table catalogs.
+ */
+class ShowPartitionsSuite extends command.ShowPartitionsSuiteBase with CommandSuiteBase {
+  test("a table does not support partitioning") {
+    val table = s"non_part_$catalog.tab1"
+    withTable(table) {
+      sql(s"""
+        |CREATE TABLE $table (price int, qty int, year int, month int)
+        |$defaultUsing""".stripMargin)
+      val errMsg = intercept[AnalysisException] {
+        sql(s"SHOW PARTITIONS $table")
+      }.getMessage
+      assert(errMsg.contains(s"Table $table does not support partition management"))
+    }
+  }
+
+  test("SPARK-33889, SPARK-33904: null and empty string as partition values") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      createNullPartTable(t, "parquet")
+      runShowPartitionsSql(s"SHOW PARTITIONS $t", Row("part=") :: Row("part=null") :: Nil)
+      checkAnswer(spark.table(t), Row(0, "") :: Row(1, null) :: Nil)
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowTablesSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowTablesSuite.scala
new file mode 100644
index 0000000000000..702c1744a68ef
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowTablesSuite.scala
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v2
+
+import org.apache.spark.sql.{AnalysisException, Row}
+import org.apache.spark.sql.execution.command
+
+/**
+ * The class contains tests for the `SHOW TABLES` command to check V2 table catalogs.
+ */
+class ShowTablesSuite extends command.ShowTablesSuiteBase with CommandSuiteBase {
+  override def defaultNamespace: Seq[String] = Nil
+
+  // The test fails for V1 catalog with the error:
+  // org.apache.spark.sql.AnalysisException:
+  //   The namespace in session catalog must have exactly one name part: spark_catalog.n1.n2.db
+  test("show tables in nested namespaces") {
+    withTable(s"$catalog.n1.n2.db") {
+      spark.sql(s"CREATE TABLE $catalog.n1.n2.db.table_name (id bigint, data string) $defaultUsing")
+      runShowTablesSql(
+        s"SHOW TABLES FROM $catalog.n1.n2.db",
+        Seq(Row("n1.n2.db", "table_name", false)))
+    }
+  }
+
+  // The test fails for V1 catalog with the error:
+  // org.apache.spark.sql.AnalysisException:
+  //   The namespace in session catalog must have exactly one name part: spark_catalog.table
+  test("using v2 catalog with empty namespace") {
+    withTable(s"$catalog.table") {
+      spark.sql(s"CREATE TABLE $catalog.table (id bigint, data string) $defaultUsing")
+      runShowTablesSql(s"SHOW TABLES FROM $catalog", Seq(Row("", "table", false)))
+    }
+  }
+
+  // The test fails for V1 catalog with the error:
+  // org.apache.spark.sql.AnalysisException:
+  //   The namespace in session catalog must have exactly one name part: spark_catalog.ns1.ns2.tbl
+  test("SHOW TABLE EXTENDED not valid v1 database") {
+    def testV1CommandNamespace(sqlCommand: String, namespace: String): Unit = {
+      val e = intercept[AnalysisException] {
+        sql(sqlCommand)
+      }
+      assert(e.message.contains(s"SHOW TABLE EXTENDED is not supported for v2 tables"))
+    }
+
+    val namespace = s"$catalog.ns1.ns2"
+    val table = "tbl"
+    withTable(s"$namespace.$table") {
+      sql(s"CREATE TABLE $namespace.$table (id bigint, data string) " +
+        s"$defaultUsing PARTITIONED BY (id)")
+
+      testV1CommandNamespace(s"SHOW TABLE EXTENDED FROM $namespace LIKE 'tb*'",
+        namespace)
+      testV1CommandNamespace(s"SHOW TABLE EXTENDED IN $namespace LIKE 'tb*'",
+        namespace)
+      testV1CommandNamespace("SHOW TABLE EXTENDED " +
+        s"FROM $namespace LIKE 'tb*' PARTITION(id=1)",
+        namespace)
+      testV1CommandNamespace("SHOW TABLE EXTENDED " +
+        s"IN $namespace LIKE 'tb*' PARTITION(id=1)",
+        namespace)
+    }
+  }
+
+  // TODO(SPARK-33393): Support SHOW TABLE EXTENDED in DSv2
+  test("SHOW TABLE EXTENDED: an existing table") {
+    val table = "people"
+    withTable(s"$catalog.$table") {
+      sql(s"CREATE TABLE $catalog.$table (name STRING, id INT) $defaultUsing")
+      val errMsg = intercept[AnalysisException] {
+        sql(s"SHOW TABLE EXTENDED FROM $catalog LIKE '*$table*'").collect()
+      }.getMessage
+      assert(errMsg.contains("SHOW TABLE EXTENDED is not supported for v2 tables"))
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/TruncateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/TruncateTableSuite.scala
new file mode 100644
index 0000000000000..f125a72bd32a5
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/TruncateTableSuite.scala
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v2
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.execution.command
+
+/**
+ * The class contains tests for the `TRUNCATE TABLE` command to check V2 table catalogs.
+ */
+class TruncateTableSuite extends command.TruncateTableSuiteBase with CommandSuiteBase {
+
+  override val invalidPartColumnError = "not a valid partition column in table"
+
+  test("truncate a partition of a table which does not support partitions") {
+    withNamespaceAndTable("ns", "tbl", s"non_part_$catalog") { t =>
+      sql(s"CREATE TABLE $t (c0 INT) $defaultUsing")
+      sql(s"INSERT INTO $t SELECT 0")
+
+      val errMsg = intercept[AnalysisException] {
+        sql(s"TRUNCATE TABLE $t PARTITION (c0=1)")
+      }.getMessage
+      assert(errMsg.contains(
+        "Table non_part_test_catalog.ns.tbl does not support partition management"))
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/BasicWriteJobStatsTrackerMetricSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/BasicWriteJobStatsTrackerMetricSuite.scala
new file mode 100644
index 0000000000000..3e58c225d8c7a
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/BasicWriteJobStatsTrackerMetricSuite.scala
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.{LocalSparkSession, SparkSession}
+
+class BasicWriteJobStatsTrackerMetricSuite extends SparkFunSuite with LocalSparkSession {
+
+  test("SPARK-32978: make sure the number of dynamic part metric is correct") {
+    try {
+      val partitions = "50"
+      spark = SparkSession.builder().master("local[4]").getOrCreate()
+      val statusStore = spark.sharedState.statusStore
+
+      spark.sql("create table dynamic_partition(i bigint, part bigint) " +
+        "using parquet partitioned by (part)")
+      val oldExecutionsSize = statusStore.executionsList().size
+      spark.sql("insert overwrite table dynamic_partition partition(part) " +
+        s"select id, id % $partitions as part from range(10000)")
+
+      // Wait for listener to finish computing the metrics for the executions.
+      while (statusStore.executionsList().size - oldExecutionsSize < 1 ||
+        statusStore.executionsList().last.metricValues == null) {
+        Thread.sleep(100)
+      }
+
+      // There should be 2 SQLExecutionUIData in executionsList and the 2nd item is we need,
+      // but the executionId is indeterminate in maven test,
+      // so the `statusStore.execution(executionId)` API is not used.
+      assert(statusStore.executionsCount() == 2)
+      val executionData = statusStore.executionsList()(1)
+      val accumulatorIdOpt =
+        executionData.metrics.find(_.name == "number of dynamic part").map(_.accumulatorId)
+      assert(accumulatorIdOpt.isDefined)
+      val numPartsOpt = executionData.metricValues.get(accumulatorIdOpt.get)
+      assert(numPartsOpt.isDefined && numPartsOpt.get == partitions)
+
+    } finally {
+      spark.sql("drop table if exists dynamic_partition")
+      spark.stop()
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/BasicWriteTaskStatsTrackerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/BasicWriteTaskStatsTrackerSuite.scala
index 32941d8d2cd11..0237679942714 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/BasicWriteTaskStatsTrackerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/BasicWriteTaskStatsTrackerSuite.scala
@@ -17,12 +17,13 @@
 
 package org.apache.spark.sql.execution.datasources
 
-import java.nio.charset.Charset
+import java.nio.charset.{Charset, StandardCharsets}
 
 import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
+import org.apache.hadoop.fs.{FileSystem, FilterFileSystem, Path}
 
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.execution.datasources.BasicWriteJobStatsTracker.FILE_LENGTH_XATTR
 import org.apache.spark.util.Utils
 
 /**
@@ -148,7 +149,7 @@ class BasicWriteTaskStatsTrackerSuite extends SparkFunSuite {
   test("Three files, last one empty") {
     val file1 = new Path(tempDirPath, "f-3-1")
     val file2 = new Path(tempDirPath, "f-3-2")
-    val file3 = new Path(tempDirPath, "f-3-2")
+    val file3 = new Path(tempDirPath, "f-3-3")
     val tracker = new BasicWriteTaskStatsTracker(conf)
     tracker.newFile(file1.toString)
     write1(file1)
@@ -221,4 +222,127 @@ class BasicWriteTaskStatsTrackerSuite extends SparkFunSuite {
     write(file, data2)
   }
 
+  /**
+   * Does a length specified in the XAttr header get picked up?
+   */
+  test("XAttr sourced length") {
+    val file = new Path(tempDirPath, "file")
+    touch(file)
+    val xattrFS = new FsWithFakeXAttrs(localfs)
+    val bigLong = 34359738368L
+    xattrFS.set(FILE_LENGTH_XATTR, s"$bigLong")
+    val tracker = new BasicWriteTaskStatsTracker(conf)
+    assert(Some(bigLong) === tracker.getFileSize(xattrFS, file),
+      "Size not collected from XAttr entry")
+  }
+
+  /**
+   * If a file is non-empty then the XAttr size declaration
+   * is not used.
+   */
+  test("XAttr sourced length only used for 0-byte-files") {
+    val file = new Path(tempDirPath, "file")
+    write2(file)
+    val xattrFS = new FsWithFakeXAttrs(localfs)
+    val bigLong = 34359738368L
+    xattrFS.set(FILE_LENGTH_XATTR, s"$bigLong")
+    val tracker = new BasicWriteTaskStatsTracker(conf)
+    assert(Some(len2) === tracker.getFileSize(xattrFS, file),
+      "Size not collected from XAttr entry")
+  }
+
+  /**
+   * Any FS which supports XAttr must raise an FNFE if the
+   * file is missing. This verifies resilience on a path
+   * which the the local FS would not normally take.
+   */
+  test("Missing File with XAttr") {
+    val missing = new Path(tempDirPath, "missing")
+    val xattrFS = new FsWithFakeXAttrs(localfs)
+    val tracker = new BasicWriteTaskStatsTracker(conf)
+    tracker.newFile(missing.toString)
+    assert(None === tracker.getFileSize(xattrFS, missing))
+  }
+
+  /**
+   * If there are any problems parsing/validating the
+   * header attribute, fall back to the file length.
+   */
+  test("XAttr error recovery") {
+    val file = new Path(tempDirPath, "file")
+    touch(file)
+    val xattrFS = new FsWithFakeXAttrs(localfs)
+
+    val tracker = new BasicWriteTaskStatsTracker(conf)
+
+    // without a header
+    assert(Some(0) === tracker.getFileSize(xattrFS, file))
+
+    // will fail to parse as a long
+    xattrFS.set(FILE_LENGTH_XATTR, "Not-a-long")
+    assert(Some(0) === tracker.getFileSize(xattrFS, file))
+
+    // a negative value
+    xattrFS.set(FILE_LENGTH_XATTR, "-1")
+    assert(Some(0) === tracker.getFileSize(xattrFS, file))
+
+    // empty string
+    xattrFS.set(FILE_LENGTH_XATTR, "")
+    assert(Some(0) === tracker.getFileSize(xattrFS, file))
+
+    // then a zero byte array
+    xattrFS.setXAttr(file, FILE_LENGTH_XATTR,
+      new Array[Byte](0))
+    assert(Some(0) === tracker.getFileSize(xattrFS, file))
+  }
+
+  /**
+   * Extend any FS with a mock get/setXAttr.
+   * A map of attributes is used, these are returned on a getXAttr(path, key)
+   * call to any path; the other XAttr list/get calls are not implemented.
+   */
+  class FsWithFakeXAttrs(fs: FileSystem) extends FilterFileSystem(fs) {
+
+    private val xattrs = scala.collection.mutable.Map[String, Array[Byte]]()
+
+    /**
+     * Mock implementation of setAttr.
+     *
+     * @param path path (ignored)
+     * @param name attribute name.
+     * @param value byte array value
+     */
+    override def setXAttr(
+      path: Path,
+      name: String,
+      value: Array[Byte]): Unit = {
+
+      xattrs.put(name, value)
+    }
+
+    /**
+     * Set an attribute to the UTF-8 byte value of a string.
+     *
+     * @param name  attribute name.
+     * @param value string value
+     */
+    def set(name: String, value: String): Unit = {
+      setXAttr(null, name, value.getBytes(StandardCharsets.UTF_8))
+    }
+
+    /**
+     * Get any attribute if it is found in the map, else null.
+     * @param path path (ignored)
+     * @param name attribute name.
+     * @return the byte[] value or null.
+     */
+    override def getXAttr(
+      path: Path,
+      name: String): Array[Byte] = {
+      // force a check for the file and raise an FNFE if not found
+      getFileStatus(path)
+
+      xattrs.getOrElse(name, null)
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/CommonFileDataSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/CommonFileDataSourceSuite.scala
new file mode 100644
index 0000000000000..b7d0a7fc306ad
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/CommonFileDataSourceSuite.scala
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources
+
+import org.scalatest.funsuite.AnyFunSuite
+
+import org.apache.spark.sql.{Dataset, Encoders, FakeFileSystemRequiringDSOption, SparkSession}
+import org.apache.spark.sql.catalyst.plans.SQLHelper
+
+/**
+ * The trait contains tests for all file-based data sources.
+ * The tests that are not applicable to all file-based data sources should be placed to
+ * [[org.apache.spark.sql.FileBasedDataSourceSuite]].
+ */
+trait CommonFileDataSourceSuite extends SQLHelper { self: AnyFunSuite =>
+
+  protected def spark: SparkSession
+  protected def dataSourceFormat: String
+  protected def inputDataset: Dataset[_] = spark.createDataset(Seq("abc"))(Encoders.STRING)
+
+  test(s"Propagate Hadoop configs from $dataSourceFormat options to underlying file system") {
+    withSQLConf(
+      "fs.file.impl" -> classOf[FakeFileSystemRequiringDSOption].getName,
+      "fs.file.impl.disable.cache" -> "true") {
+      Seq(false, true).foreach { mergeSchema =>
+        withTempPath { dir =>
+          val path = dir.getAbsolutePath
+          val conf = Map("ds_option" -> "value", "mergeSchema" -> mergeSchema.toString)
+          inputDataset
+            .write
+            .options(conf)
+            .format(dataSourceFormat)
+            .save(path)
+          Seq(path, "file:" + path.stripPrefix("file:")).foreach { p =>
+            val readback = spark
+              .read
+              .options(conf)
+              .format(dataSourceFormat)
+              .load(p)
+            // Checks that read doesn't throw the exception from `FakeFileSystemRequiringDSOption`
+            readback.write.mode("overwrite").format("noop").save()
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/CustomWriteTaskStatsTrackerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/CustomWriteTaskStatsTrackerSuite.scala
new file mode 100644
index 0000000000000..82d873a2cd81b
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/CustomWriteTaskStatsTrackerSuite.scala
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources
+
+import scala.collection.mutable
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.InternalRow
+
+class CustomWriteTaskStatsTrackerSuite extends SparkFunSuite {
+
+  def checkFinalStats(tracker: CustomWriteTaskStatsTracker, result: Map[String, Int]): Unit = {
+    assert(tracker.getFinalStats().asInstanceOf[CustomWriteTaskStats].numRowsPerFile == result)
+  }
+
+  test("sequential file writing") {
+    val tracker = new CustomWriteTaskStatsTracker
+    tracker.newFile("a")
+    tracker.newRow("a", null)
+    tracker.newRow("a", null)
+    tracker.newFile("b")
+    checkFinalStats(tracker, Map("a" -> 2, "b" -> 0))
+  }
+
+  test("random file writing") {
+    val tracker = new CustomWriteTaskStatsTracker
+    tracker.newFile("a")
+    tracker.newRow("a", null)
+    tracker.newFile("b")
+    tracker.newRow("a", null)
+    tracker.newRow("b", null)
+    checkFinalStats(tracker, Map("a" -> 2, "b" -> 1))
+  }
+}
+
+class CustomWriteTaskStatsTracker extends WriteTaskStatsTracker {
+
+  val numRowsPerFile = mutable.Map.empty[String, Int]
+
+  override def newPartition(partitionValues: InternalRow): Unit = {}
+
+  override def newFile(filePath: String): Unit = {
+    numRowsPerFile.put(filePath, 0)
+  }
+
+  override def closeFile(filePath: String): Unit = {}
+
+  override def newRow(filePath: String, row: InternalRow): Unit = {
+    numRowsPerFile(filePath) += 1
+  }
+
+  override def getFinalStats(): WriteTaskStats = {
+    CustomWriteTaskStats(numRowsPerFile.toMap)
+  }
+}
+
+case class CustomWriteTaskStats(numRowsPerFile: Map[String, Int]) extends WriteTaskStats
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceSuite.scala
index 1e3c660e09454..6ba3d2723412b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceSuite.scala
@@ -19,11 +19,12 @@ package org.apache.spark.sql.execution.datasources
 
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.{FileStatus, Path, RawLocalFileSystem}
+import org.scalatest.PrivateMethodTester
 
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.test.SharedSparkSession
 
-class DataSourceSuite extends SharedSparkSession {
+class DataSourceSuite extends SharedSparkSession with PrivateMethodTester {
   import TestPaths._
 
   test("test glob and non glob paths") {
@@ -36,7 +37,8 @@ class DataSourceSuite extends SharedSparkSession {
       ),
       hadoopConf,
       checkEmptyGlobPath = true,
-      checkFilesExist = true
+      checkFilesExist = true,
+      enableGlobbing = true
     )
 
     assert(resultPaths.toSet === allPathsInFs.toSet)
@@ -50,7 +52,8 @@ class DataSourceSuite extends SharedSparkSession {
       ),
       hadoopConf,
       checkEmptyGlobPath = true,
-      checkFilesExist = true
+      checkFilesExist = true,
+      enableGlobbing = true
     )
 
     assert(
@@ -71,7 +74,8 @@ class DataSourceSuite extends SharedSparkSession {
       ),
       hadoopConf,
       checkEmptyGlobPath = true,
-      checkFilesExist = true
+      checkFilesExist = true,
+      enableGlobbing = true
     )
 
     assert(
@@ -91,7 +95,8 @@ class DataSourceSuite extends SharedSparkSession {
       ),
       hadoopConf,
       checkEmptyGlobPath = true,
-      checkFilesExist = false
+      checkFilesExist = false,
+      enableGlobbing = true
     )
 
     assert(
@@ -113,7 +118,8 @@ class DataSourceSuite extends SharedSparkSession {
         ),
         hadoopConf,
         checkEmptyGlobPath = true,
-        checkFilesExist = true
+        checkFilesExist = true,
+        enableGlobbing = true
       )
     )
   }
@@ -128,10 +134,23 @@ class DataSourceSuite extends SharedSparkSession {
         ),
         hadoopConf,
         checkEmptyGlobPath = true,
-        checkFilesExist = true
+        checkFilesExist = true,
+        enableGlobbing = true
       )
     )
   }
+
+  test("Data source options should be propagated in method checkAndGlobPathIfNecessary") {
+    val dataSourceOptions = Map("fs.defaultFS" -> "nonexistentFs://nonexistentFs")
+    val dataSource = DataSource(spark, "parquet", Seq("/path3"), options = dataSourceOptions)
+    val checkAndGlobPathIfNecessary = PrivateMethod[Seq[Path]]('checkAndGlobPathIfNecessary)
+
+    val message = intercept[java.io.IOException] {
+      dataSource invokePrivate checkAndGlobPathIfNecessary(false, false)
+    }.getMessage
+    val expectMessage = "No FileSystem for scheme nonexistentFs"
+    assert(message.filterNot(Set(':', '"').contains) == expectMessage)
+  }
 }
 
 object TestPaths {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileBasedDataSourceTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileBasedDataSourceTest.scala
index bdb161d59a33e..c2dc20b0099a3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileBasedDataSourceTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileBasedDataSourceTest.scala
@@ -22,8 +22,10 @@ import java.io.File
 import scala.reflect.ClassTag
 import scala.reflect.runtime.universe.TypeTag
 
-import org.apache.spark.sql.{DataFrame, SaveMode}
+import org.apache.spark.sql.{DataFrame, Row, SaveMode}
+import org.apache.spark.sql.functions.struct
 import org.apache.spark.sql.test.SQLTestUtils
+import org.apache.spark.sql.types.StructType
 
 /**
  * A helper trait that provides convenient facilities for file-based data source testing.
@@ -103,4 +105,40 @@ private[sql] trait FileBasedDataSourceTest extends SQLTestUtils {
       df: DataFrame, path: File): Unit = {
     df.write.mode(SaveMode.Overwrite).format(dataSourceName).save(path.getCanonicalPath)
   }
+
+  /**
+   * Takes single level `inputDF` dataframe to generate multi-level nested
+   * dataframes as new test data. It tests both non-nested and nested dataframes
+   * which are written and read back with specified datasource.
+   */
+  protected def withNestedDataFrame(inputDF: DataFrame): Seq[(DataFrame, String, Any => Any)] = {
+    assert(inputDF.schema.fields.length == 1)
+    assert(!inputDF.schema.fields.head.dataType.isInstanceOf[StructType])
+    val df = inputDF.toDF("temp")
+    Seq(
+      (
+        df.withColumnRenamed("temp", "a"),
+        "a", // zero nesting
+        (x: Any) => x),
+      (
+        df.withColumn("a", struct(df("temp") as "b")).drop("temp"),
+        "a.b", // one level nesting
+        (x: Any) => Row(x)),
+      (
+        df.withColumn("a", struct(struct(df("temp") as "c") as "b")).drop("temp"),
+        "a.b.c", // two level nesting
+        (x: Any) => Row(Row(x))
+      ),
+      (
+        df.withColumnRenamed("temp", "a.b"),
+        "`a.b`", // zero nesting with column name containing `dots`
+        (x: Any) => x
+      ),
+      (
+        df.withColumn("a.b", struct(df("temp") as "c.d") ).drop("temp"),
+        "`a.b`.`c.d`", // one level nesting with column names containing `dots`
+        (x: Any) => Row(x)
+      )
+    )
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileFormatWriterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileFormatWriterSuite.scala
index ce511842e6356..f492fc653653e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileFormatWriterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileFormatWriterSuite.scala
@@ -61,4 +61,17 @@ class FileFormatWriterSuite
       checkAnswer(spark.table("t2").sort("id"), Seq(Row(0, null), Row(1, null), Row(2, null)))
     }
   }
+
+  test("SPARK-33904: save and insert into a table in a namespace of spark_catalog") {
+    val ns = "spark_catalog.ns"
+    withNamespace(ns) {
+      spark.sql(s"CREATE NAMESPACE $ns")
+      val t = s"$ns.tbl"
+      withTable(t) {
+        spark.range(1).write.saveAsTable(t)
+        Seq(100).toDF().write.insertInto(t)
+        checkAnswer(spark.table(t), Seq(Row(0), Row(100)))
+      }
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala
index ea15f1891b006..fcaf8df4f9a02 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala
@@ -21,6 +21,7 @@ import java.io.{File, FileNotFoundException}
 import java.net.URI
 
 import scala.collection.mutable
+import scala.concurrent.duration._
 
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.{BlockLocation, FileStatus, LocatedFileStatus, Path, RawLocalFileSystem, RemoteIterator}
@@ -33,7 +34,7 @@ import org.apache.spark.metrics.source.HiveCatalogMetrics
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.util._
 import org.apache.spark.sql.functions.col
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
 import org.apache.spark.util.KnownSizeEstimation
@@ -296,17 +297,6 @@ class FileIndexSuite extends SharedSparkSession {
     }
   }
 
-  test("InMemoryFileIndex - file filtering") {
-    assert(!InMemoryFileIndex.shouldFilterOut("abcd"))
-    assert(InMemoryFileIndex.shouldFilterOut(".ab"))
-    assert(InMemoryFileIndex.shouldFilterOut("_cd"))
-    assert(!InMemoryFileIndex.shouldFilterOut("_metadata"))
-    assert(!InMemoryFileIndex.shouldFilterOut("_common_metadata"))
-    assert(InMemoryFileIndex.shouldFilterOut("_ab_metadata"))
-    assert(InMemoryFileIndex.shouldFilterOut("_cd_common_metadata"))
-    assert(InMemoryFileIndex.shouldFilterOut("a._COPYING_"))
-  }
-
   test("SPARK-17613 - PartitioningAwareFileIndex: base path w/o '/' at end") {
     class MockCatalog(
       override val rootPaths: Seq[Path])
@@ -366,13 +356,15 @@ class FileIndexSuite extends SharedSparkSession {
       val wrongBasePath = new File(dir, "unknown")
       // basePath must be a directory
       wrongBasePath.mkdir()
-      val parameters = Map("basePath" -> wrongBasePath.getCanonicalPath)
-      val fileIndex = new InMemoryFileIndex(spark, Seq(path), parameters, None)
-      val msg = intercept[IllegalArgumentException] {
-        // trigger inferPartitioning()
-        fileIndex.partitionSpec()
-      }.getMessage
-      assert(msg === s"Wrong basePath ${wrongBasePath.getCanonicalPath} for the root path: $path")
+      withClue("SPARK-32368: 'basePath' can be case insensitive") {
+        val parameters = Map("bAsepAtH" -> wrongBasePath.getCanonicalPath)
+        val fileIndex = new InMemoryFileIndex(spark, Seq(path), parameters, None)
+        val msg = intercept[IllegalArgumentException] {
+          // trigger inferPartitioning()
+          fileIndex.partitionSpec()
+        }.getMessage
+        assert(msg === s"Wrong basePath ${wrongBasePath.getCanonicalPath} for the root path: $path")
+      }
     }
   }
 
@@ -413,6 +405,21 @@ class FileIndexSuite extends SharedSparkSession {
     fileStatusCache.putLeafFiles(new Path("/tmp", "abc"), files.toArray)
   }
 
+  test("SPARK-34075: InMemoryFileIndex filters out hidden file on partition inference") {
+    withTempPath { path =>
+      spark
+        .range(2)
+        .select(col("id").as("p"), col("id"))
+        .write
+        .partitionBy("p")
+        .parquet(path.getAbsolutePath)
+      val targetPath = new File(path, "p=1")
+      val hiddenPath = new File(path, "_hidden_path")
+      targetPath.renameTo(hiddenPath)
+      assert(spark.read.parquet(path.getAbsolutePath).count() == 1L)
+    }
+  }
+
   test("SPARK-20367 - properly unescape column names in inferPartitioning") {
     withTempPath { path =>
       val colToUnescape = "Column/#%'?"
@@ -488,6 +495,31 @@ class FileIndexSuite extends SharedSparkSession {
     val fileIndex = new TestInMemoryFileIndex(spark, path)
     assert(fileIndex.leafFileStatuses.toSeq == statuses)
   }
+
+  test("expire FileStatusCache if TTL is configured") {
+    val previousValue = SQLConf.get.getConf(StaticSQLConf.METADATA_CACHE_TTL_SECONDS)
+    try {
+      // using 'SQLConf.get.setConf' instead of 'withSQLConf' to set a static config at runtime
+      SQLConf.get.setConf(StaticSQLConf.METADATA_CACHE_TTL_SECONDS, 1L)
+
+      val path = new Path("/dummy_tmp", "abc")
+      val files = (1 to 3).map(_ => new FileStatus())
+
+      FileStatusCache.resetForTesting()
+      val fileStatusCache = FileStatusCache.getOrCreate(spark)
+      fileStatusCache.putLeafFiles(path, files.toArray)
+
+      // Exactly 3 files are cached.
+      assert(fileStatusCache.getLeafFiles(path).get.length === 3)
+      // Wait until the cache expiration.
+      eventually(timeout(3.seconds)) {
+        // And the cache is gone.
+        assert(fileStatusCache.getLeafFiles(path).isEmpty === true)
+      }
+    } finally {
+      SQLConf.get.setConf(StaticSQLConf.METADATA_CACHE_TTL_SECONDS, previousValue)
+    }
+  }
 }
 
 object DeletionRaceFileSystem {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategySuite.scala
index 812305ba24403..50f32126e5dec 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategySuite.scala
@@ -262,20 +262,22 @@ class FileSourceStrategySuite extends QueryTest with SharedSparkSession with Pre
           "p1=2/file7_0000" -> 1),
         buckets = 3)
 
-    // No partition pruning
-    checkScan(table) { partitions =>
-      assert(partitions.size == 3)
-      assert(partitions(0).files.size == 5)
-      assert(partitions(1).files.size == 0)
-      assert(partitions(2).files.size == 2)
-    }
+    withSQLConf(SQLConf.AUTO_BUCKETED_SCAN_ENABLED.key -> "false") {
+      // No partition pruning
+      checkScan(table) { partitions =>
+        assert(partitions.size == 3)
+        assert(partitions(0).files.size == 5)
+        assert(partitions(1).files.size == 0)
+        assert(partitions(2).files.size == 2)
+      }
 
-    // With partition pruning
-    checkScan(table.where("p1=2")) { partitions =>
-      assert(partitions.size == 3)
-      assert(partitions(0).files.size == 3)
-      assert(partitions(1).files.size == 0)
-      assert(partitions(2).files.size == 1)
+      // With partition pruning
+      checkScan(table.where("p1=2")) { partitions =>
+        assert(partitions.size == 3)
+        assert(partitions(0).files.size == 3)
+        assert(partitions(1).files.size == 0)
+        assert(partitions(2).files.size == 1)
+      }
     }
   }
 
@@ -528,6 +530,76 @@ class FileSourceStrategySuite extends QueryTest with SharedSparkSession with Pre
     }
   }
 
+  test("SPARK-32019: Add spark.sql.files.minPartitionNum config") {
+    withSQLConf(SQLConf.FILES_MIN_PARTITION_NUM.key -> "1") {
+      val table =
+        createTable(files = Seq(
+          "file1" -> 1,
+          "file2" -> 1,
+          "file3" -> 1
+        ))
+      assert(table.rdd.partitions.length == 1)
+    }
+
+    withSQLConf(SQLConf.FILES_MIN_PARTITION_NUM.key -> "10") {
+      val table =
+        createTable(files = Seq(
+          "file1" -> 1,
+          "file2" -> 1,
+          "file3" -> 1
+        ))
+      assert(table.rdd.partitions.length == 3)
+    }
+
+    withSQLConf(
+      SQLConf.FILES_MAX_PARTITION_BYTES.key -> "2MB",
+      SQLConf.FILES_OPEN_COST_IN_BYTES.key -> String.valueOf(4 * 1024 * 1024)) {
+
+      withSQLConf(SQLConf.FILES_MIN_PARTITION_NUM.key -> "8") {
+        val partitions = (1 to 12).map(i => s"file$i" -> 2 * 1024 * 1024)
+        val table = createTable(files = partitions)
+        // partition is limited by filesMaxPartitionBytes(2MB)
+        assert(table.rdd.partitions.length == 12)
+      }
+
+      withSQLConf(SQLConf.FILES_MIN_PARTITION_NUM.key -> "16") {
+        val partitions = (1 to 12).map(i => s"file$i" -> 4 * 1024 * 1024)
+        val table = createTable(files = partitions)
+        assert(table.rdd.partitions.length == 24)
+      }
+    }
+  }
+
+  test("SPARK-32352: Partially push down support data filter if it mixed in partition filters") {
+    val table =
+      createTable(
+        files = Seq(
+          "p1=1/file1" -> 10,
+          "p1=2/file2" -> 10,
+          "p1=3/file3" -> 10,
+          "p1=4/file4" -> 10))
+
+    checkScan(table.where("(c1 = 1) OR (c1 = 2)")) { partitions =>
+      assert(partitions.size == 1, "when checking partitions")
+    }
+    checkDataFilters(Set(Or(EqualTo("c1", 1), EqualTo("c1", 2))))
+
+    checkScan(table.where("(p1 = 1 AND c1 = 1) OR (p1 = 2 and c1 = 2)")) { partitions =>
+      assert(partitions.size == 1, "when checking partitions")
+    }
+    checkDataFilters(Set(Or(EqualTo("c1", 1), EqualTo("c1", 2))))
+
+    checkScan(table.where("(p1 = '1' AND c1 = 2) OR (c1 = 1 OR p1 = '2')")) { partitions =>
+      assert(partitions.size == 1, "when checking partitions")
+    }
+    checkDataFilters(Set.empty)
+
+    checkScan(table.where("p1 = '1' OR (p1 = '2' AND c1 = 1)")) { partitions =>
+      assert(partitions.size == 1, "when checking partitions")
+    }
+    checkDataFilters(Set.empty)
+  }
+
   // Helpers for checking the arguments passed to the FileFormat.
 
   protected val checkPartitionSchema =
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/HadoopFileLinesReaderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/HadoopFileLinesReaderSuite.scala
index 3c83388461ce1..771ddbd6523bc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/HadoopFileLinesReaderSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/HadoopFileLinesReaderSuite.scala
@@ -36,13 +36,13 @@ class HadoopFileLinesReaderSuite extends SharedSparkSession {
     val delimOpt = delimiter.map(_.getBytes(StandardCharsets.UTF_8))
     Files.write(path.toPath, text.getBytes(StandardCharsets.UTF_8))
 
-    val lines = ranges.map { case (start, length) =>
+    val lines = ranges.flatMap { case (start, length) =>
       val file = PartitionedFile(InternalRow.empty, path.getCanonicalPath, start, length)
       val hadoopConf = conf.getOrElse(spark.sessionState.newHadoopConf())
       val reader = new HadoopFileLinesReader(file, delimOpt, hadoopConf)
 
       reader.map(_.toString)
-    }.flatten
+    }
 
     lines
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/PathFilterStrategySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/PathFilterStrategySuite.scala
new file mode 100644
index 0000000000000..b965a78c9eec0
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/PathFilterStrategySuite.scala
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources
+
+import org.apache.spark.sql.QueryTest
+import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
+import org.apache.spark.sql.test.SharedSparkSession
+
+class PathFilterStrategySuite extends QueryTest with SharedSparkSession {
+
+  test("SPARK-31962: PathFilterStrategies - modifiedAfter option") {
+    val options =
+      CaseInsensitiveMap[String](Map("modifiedAfter" -> "2010-10-01T01:01:00"))
+    val strategy = PathFilterFactory.create(options)
+    assert(strategy.head.isInstanceOf[ModifiedAfterFilter])
+    assert(strategy.size == 1)
+  }
+
+  test("SPARK-31962: PathFilterStrategies - modifiedBefore option") {
+    val options =
+      CaseInsensitiveMap[String](Map("modifiedBefore" -> "2020-10-01T01:01:00"))
+    val strategy = PathFilterFactory.create(options)
+    assert(strategy.head.isInstanceOf[ModifiedBeforeFilter])
+    assert(strategy.size == 1)
+  }
+
+  test("SPARK-31962: PathFilterStrategies - pathGlobFilter option") {
+    val options = CaseInsensitiveMap[String](Map("pathGlobFilter" -> "*.txt"))
+    val strategy = PathFilterFactory.create(options)
+    assert(strategy.head.isInstanceOf[PathGlobFilter])
+    assert(strategy.size == 1)
+  }
+
+  test("SPARK-31962: PathFilterStrategies - no options") {
+    val options = CaseInsensitiveMap[String](Map.empty)
+    val strategy = PathFilterFactory.create(options)
+    assert(strategy.isEmpty)
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/PathFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/PathFilterSuite.scala
new file mode 100644
index 0000000000000..1af2adfd8640c
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/PathFilterSuite.scala
@@ -0,0 +1,307 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources
+
+import java.io.File
+import java.time.{LocalDateTime, ZoneId, ZoneOffset}
+import java.time.format.DateTimeFormatter
+
+import scala.util.Random
+
+import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
+import org.apache.spark.sql.catalyst.util.{stringToFile, DateTimeUtils}
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.{StringType, StructField, StructType}
+
+class PathFilterSuite extends QueryTest with SharedSparkSession {
+  import testImplicits._
+
+  test("SPARK-31962: modifiedBefore specified" +
+      " and sharing same timestamp with file last modified time.") {
+    withTempDir { dir =>
+      val curTime = LocalDateTime.now(ZoneOffset.UTC)
+      executeTest(dir, Seq(curTime), 0, modifiedBefore = Some(formatTime(curTime)))
+    }
+  }
+
+  test("SPARK-31962: modifiedAfter specified" +
+      " and sharing same timestamp with file last modified time.") {
+    withTempDir { dir =>
+      val curTime = LocalDateTime.now(ZoneOffset.UTC)
+      executeTest(dir, Seq(curTime), 0, modifiedAfter = Some(formatTime(curTime)))
+    }
+  }
+
+  test("SPARK-31962: modifiedBefore and modifiedAfter option" +
+      " share same timestamp with file last modified time.") {
+    withTempDir { dir =>
+      val curTime = LocalDateTime.now(ZoneOffset.UTC)
+      val formattedTime = formatTime(curTime)
+      executeTest(dir, Seq(curTime), 0, modifiedBefore = Some(formattedTime),
+        modifiedAfter = Some(formattedTime))
+    }
+  }
+
+  test("SPARK-31962: modifiedBefore and modifiedAfter option" +
+      " share same timestamp with earlier file last modified time.") {
+    withTempDir { dir =>
+      val curTime = LocalDateTime.now(ZoneOffset.UTC)
+      val fileTime = curTime.minusDays(3)
+      val formattedTime = formatTime(curTime)
+      executeTest(dir, Seq(fileTime), 0, modifiedBefore = Some(formattedTime),
+        modifiedAfter = Some(formattedTime))
+    }
+  }
+
+  test("SPARK-31962: modifiedBefore and modifiedAfter option" +
+      " share same timestamp with later file last modified time.") {
+    withTempDir { dir =>
+      val curTime = LocalDateTime.now(ZoneOffset.UTC)
+      val formattedTime = formatTime(curTime)
+      executeTest(dir, Seq(curTime), 0, modifiedBefore = Some(formattedTime),
+        modifiedAfter = Some(formattedTime))
+    }
+  }
+
+  test("SPARK-31962: when modifiedAfter specified with a past date") {
+    withTempDir { dir =>
+      val curTime = LocalDateTime.now(ZoneOffset.UTC)
+      val pastTime = curTime.minusYears(1)
+      val formattedTime = formatTime(pastTime)
+      executeTest(dir, Seq(curTime), 1, modifiedAfter = Some(formattedTime))
+    }
+  }
+
+  test("SPARK-31962: when modifiedBefore specified with a future date") {
+    withTempDir { dir =>
+      val curTime = LocalDateTime.now(ZoneOffset.UTC)
+      val futureTime = curTime.plusYears(1)
+      val formattedTime = formatTime(futureTime)
+      executeTest(dir, Seq(curTime), 1, modifiedBefore = Some(formattedTime))
+    }
+  }
+
+  test("SPARK-31962: with modifiedBefore option provided using a past date") {
+    withTempDir { dir =>
+      val curTime = LocalDateTime.now(ZoneOffset.UTC)
+      val pastTime = curTime.minusYears(1)
+      val formattedTime = formatTime(pastTime)
+      executeTest(dir, Seq(curTime), 0, modifiedBefore = Some(formattedTime))
+    }
+  }
+
+  test("SPARK-31962: modifiedAfter specified with a past date, multiple files, one valid") {
+    withTempDir { dir =>
+      val fileTime1 = LocalDateTime.now(ZoneOffset.UTC)
+      val fileTime2 = LocalDateTime.ofEpochSecond(0, 0, ZoneOffset.UTC)
+      val pastTime = fileTime1.minusYears(1)
+      val formattedTime = formatTime(pastTime)
+      executeTest(dir, Seq(fileTime1, fileTime2), 1, modifiedAfter = Some(formattedTime))
+    }
+  }
+
+  test("SPARK-31962: modifiedAfter specified with a past date, multiple files, both valid") {
+    withTempDir { dir =>
+      val curTime = LocalDateTime.now(ZoneOffset.UTC)
+      val pastTime = curTime.minusYears(1)
+      val formattedTime = formatTime(pastTime)
+      executeTest(dir, Seq(curTime, curTime), 2, modifiedAfter = Some(formattedTime))
+    }
+  }
+
+  test("SPARK-31962: modifiedAfter specified with a past date, multiple files, none valid") {
+    withTempDir { dir =>
+      val fileTime = LocalDateTime.ofEpochSecond(0, 0, ZoneOffset.UTC)
+      val pastTime = LocalDateTime.now(ZoneOffset.UTC).minusYears(1)
+      val formattedTime = formatTime(pastTime)
+      executeTest(dir, Seq(fileTime, fileTime), 0, modifiedAfter = Some(formattedTime))
+    }
+  }
+
+  test("SPARK-31962: modifiedBefore specified with a future date, multiple files, both valid") {
+    withTempDir { dir =>
+      val fileTime = LocalDateTime.ofEpochSecond(0, 0, ZoneOffset.UTC)
+      val futureTime = LocalDateTime.now(ZoneOffset.UTC).plusYears(1)
+      val formattedTime = formatTime(futureTime)
+      executeTest(dir, Seq(fileTime, fileTime), 2, modifiedBefore = Some(formattedTime))
+    }
+  }
+
+  test("SPARK-31962: modifiedBefore specified with a future date, multiple files, one valid") {
+    withTempDir { dir =>
+      val curTime = LocalDateTime.now(ZoneOffset.UTC)
+      val fileTime1 = LocalDateTime.ofEpochSecond(0, 0, ZoneOffset.UTC)
+      val fileTime2 = curTime.plusDays(3)
+      val formattedTime = formatTime(curTime)
+      executeTest(dir, Seq(fileTime1, fileTime2), 1, modifiedBefore = Some(formattedTime))
+    }
+  }
+
+  test("SPARK-31962: modifiedBefore specified with a future date, multiple files, none valid") {
+    withTempDir { dir =>
+      val fileTime = LocalDateTime.now(ZoneOffset.UTC).minusDays(1)
+      val formattedTime = formatTime(fileTime)
+      executeTest(dir, Seq(fileTime, fileTime), 0, modifiedBefore = Some(formattedTime))
+    }
+  }
+
+  test("SPARK-31962: modifiedBefore/modifiedAfter is specified with an invalid date") {
+    executeTestWithBadOption(
+      Map("modifiedBefore" -> "2024-05+1 01:00:00"),
+      Seq("The timestamp provided", "modifiedbefore", "2024-05+1 01:00:00"))
+
+    executeTestWithBadOption(
+      Map("modifiedAfter" -> "2024-05+1 01:00:00"),
+      Seq("The timestamp provided", "modifiedafter", "2024-05+1 01:00:00"))
+  }
+
+  test("SPARK-31962: modifiedBefore/modifiedAfter - empty option") {
+    executeTestWithBadOption(
+      Map("modifiedBefore" -> ""),
+      Seq("The timestamp provided", "modifiedbefore"))
+
+    executeTestWithBadOption(
+      Map("modifiedAfter" -> ""),
+      Seq("The timestamp provided", "modifiedafter"))
+  }
+
+  test("SPARK-31962: modifiedBefore/modifiedAfter filter takes into account local timezone " +
+      "when specified as an option.") {
+    Seq("modifiedbefore", "modifiedafter").foreach { filterName =>
+      // CET = UTC + 1 hour, HST = UTC - 10 hours
+      Seq("CET", "HST").foreach { tzId =>
+        testModifiedDateFilterWithTimezone(tzId, filterName)
+      }
+    }
+  }
+
+  test("Option pathGlobFilter: filter files correctly") {
+    withTempPath { path =>
+      val dataDir = path.getCanonicalPath
+      Seq("foo").toDS().write.text(dataDir)
+      Seq("bar").toDS().write.mode("append").orc(dataDir)
+      val df = spark.read.option("pathGlobFilter", "*.txt").text(dataDir)
+      checkAnswer(df, Row("foo"))
+
+      // Both glob pattern in option and path should be effective to filter files.
+      val df2 = spark.read.option("pathGlobFilter", "*.txt").text(dataDir + "/*.orc")
+      checkAnswer(df2, Seq.empty)
+
+      val df3 = spark.read.option("pathGlobFilter", "*.txt").text(dataDir + "/*xt")
+      checkAnswer(df3, Row("foo"))
+    }
+  }
+
+  test("Option pathGlobFilter: simple extension filtering should contains partition info") {
+    withTempPath { path =>
+      val input = Seq(("foo", 1), ("oof", 2)).toDF("a", "b")
+      input.write.partitionBy("b").text(path.getCanonicalPath)
+      Seq("bar").toDS().write.mode("append").orc(path.getCanonicalPath + "/b=1")
+
+      // If we use glob pattern in the path, the partition column won't be shown in the result.
+      val df = spark.read.text(path.getCanonicalPath + "/*/*.txt")
+      checkAnswer(df, input.select("a"))
+
+      val df2 = spark.read.option("pathGlobFilter", "*.txt").text(path.getCanonicalPath)
+      checkAnswer(df2, input)
+    }
+  }
+
+  private def executeTest(
+      dir: File,
+      fileDates: Seq[LocalDateTime],
+      expectedCount: Long,
+      modifiedBefore: Option[String] = None,
+      modifiedAfter: Option[String] = None): Unit = {
+    fileDates.foreach { fileDate =>
+      val file = createSingleFile(dir)
+      setFileTime(fileDate, file)
+    }
+
+    val schema = StructType(Seq(StructField("a", StringType)))
+
+    var dfReader = spark.read.format("csv").option("timeZone", "UTC").schema(schema)
+    modifiedBefore.foreach { opt => dfReader = dfReader.option("modifiedBefore", opt) }
+    modifiedAfter.foreach { opt => dfReader = dfReader.option("modifiedAfter", opt) }
+
+    if (expectedCount > 0) {
+      // without pathGlobFilter
+      val df1 = dfReader.load(dir.getCanonicalPath)
+      assert(df1.count() === expectedCount)
+
+      // pathGlobFilter matched
+      val df2 = dfReader.option("pathGlobFilter", "*.csv").load(dir.getCanonicalPath)
+      assert(df2.count() === expectedCount)
+
+      // pathGlobFilter mismatched
+      val df3 = dfReader.option("pathGlobFilter", "*.txt").load(dir.getCanonicalPath)
+      assert(df3.count() === 0)
+    } else {
+      val df = dfReader.load(dir.getCanonicalPath)
+      assert(df.count() === 0)
+    }
+  }
+
+  private def executeTestWithBadOption(
+      options: Map[String, String],
+      expectedMsgParts: Seq[String]): Unit = {
+    withTempDir { dir =>
+      createSingleFile(dir)
+      val exc = intercept[AnalysisException] {
+        var dfReader = spark.read.format("csv")
+        options.foreach { case (key, value) =>
+          dfReader = dfReader.option(key, value)
+        }
+        dfReader.load(dir.getCanonicalPath)
+      }
+      expectedMsgParts.foreach { msg => assert(exc.getMessage.contains(msg)) }
+    }
+  }
+
+  private def testModifiedDateFilterWithTimezone(
+      timezoneId: String,
+      filterParamName: String): Unit = {
+    val curTime = LocalDateTime.now(ZoneOffset.UTC)
+    val zoneId: ZoneId = DateTimeUtils.getTimeZone(timezoneId).toZoneId
+    val strategyTimeInMicros =
+      ModifiedDateFilter.toThreshold(
+        curTime.toString,
+        timezoneId,
+        filterParamName)
+    val strategyTimeInSeconds = strategyTimeInMicros / 1000 / 1000
+
+    val curTimeAsSeconds = curTime.atZone(zoneId).toEpochSecond
+    withClue(s"timezone: $timezoneId / param: $filterParamName,") {
+      assert(strategyTimeInSeconds === curTimeAsSeconds)
+    }
+  }
+
+  private def createSingleFile(dir: File): File = {
+    val file = new File(dir, "temp" + Random.nextInt(1000000) + ".csv")
+    stringToFile(file, "text")
+  }
+
+  private def setFileTime(time: LocalDateTime, file: File): Boolean = {
+    val sameTime = time.toEpochSecond(ZoneOffset.UTC)
+    file.setLastModified(sameTime * 1000)
+  }
+
+  private def formatTime(time: LocalDateTime): String = {
+    time.format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss"))
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/ReadSchemaTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/ReadSchemaTest.scala
index fd70b6529ff51..22db55afc27c9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/ReadSchemaTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/ReadSchemaTest.scala
@@ -21,7 +21,7 @@ import java.io.File
 
 import org.apache.spark.sql.{QueryTest, Row}
 import org.apache.spark.sql.functions._
-import org.apache.spark.sql.test.{SharedSparkSession, SQLTestUtils}
+import org.apache.spark.sql.test.SharedSparkSession
 
 /**
  * The reader schema is said to be evolved (or projected) when it changed after the data is
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/RowDataSourceStrategySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/RowDataSourceStrategySuite.scala
index 6420081a9757b..3e8a4fe290502 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/RowDataSourceStrategySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/RowDataSourceStrategySuite.scala
@@ -22,15 +22,10 @@ import java.util.Properties
 
 import org.scalatest.BeforeAndAfter
 
-import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.{DataFrame, Row}
-import org.apache.spark.sql.sources._
 import org.apache.spark.sql.test.SharedSparkSession
-import org.apache.spark.sql.types._
 import org.apache.spark.util.Utils
 
 class RowDataSourceStrategySuite extends SharedSparkSession with BeforeAndAfter {
-  import testImplicits._
 
   val url = "jdbc:h2:mem:testdb0"
   val urlWithUserAndPass = "jdbc:h2:mem:testdb0;user=testUser;password=testPass"
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SaveIntoDataSourceCommandSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SaveIntoDataSourceCommandSuite.scala
index 233978289f068..e843d1d328425 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SaveIntoDataSourceCommandSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SaveIntoDataSourceCommandSuite.scala
@@ -17,7 +17,6 @@
 
 package org.apache.spark.sql.execution.datasources
 
-import org.apache.spark.SparkConf
 import org.apache.spark.sql.SaveMode
 import org.apache.spark.sql.test.SharedSparkSession
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
index 80061dc84efbc..ac5c28953a5d9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
@@ -23,7 +23,9 @@ import org.scalactic.Equality
 
 import org.apache.spark.sql.{DataFrame, QueryTest, Row}
 import org.apache.spark.sql.catalyst.SchemaPruningTest
+import org.apache.spark.sql.catalyst.expressions.Concat
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
+import org.apache.spark.sql.catalyst.plans.logical.Expand
 import org.apache.spark.sql.execution.FileSourceScanExec
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
 import org.apache.spark.sql.functions._
@@ -49,6 +51,11 @@ abstract class SchemaPruningSuite
     relatives: Map[String, FullName] = Map.empty,
     employer: Employer = null,
     relations: Map[FullName, String] = Map.empty)
+  case class Department(
+    depId: Int,
+    depName: String,
+    contactId: Int,
+    employer: Employer)
 
   val janeDoe = FullName("Jane", "X.", "Doe")
   val johnDoe = FullName("John", "Y.", "Doe")
@@ -56,6 +63,7 @@ abstract class SchemaPruningSuite
 
   val employer = Employer(0, Company("abc", "123 Business Street"))
   val employerWithNullCompany = Employer(1, null)
+  val employerWithNullCompany2 = Employer(2, null)
 
   val contacts =
     Contact(0, janeDoe, "123 Main Street", 1, friends = Array(susanSmith),
@@ -64,6 +72,11 @@ abstract class SchemaPruningSuite
     Contact(1, johnDoe, "321 Wall Street", 3, relatives = Map("sister" -> janeDoe),
       employer = employerWithNullCompany, relations = Map(janeDoe -> "sister")) :: Nil
 
+  val departments =
+    Department(0, "Engineering", 0, employer) ::
+    Department(1, "Marketing", 1, employerWithNullCompany) ::
+    Department(2, "Operation", 4, employerWithNullCompany2) :: Nil
+
   case class Name(first: String, last: String)
   case class BriefContact(id: Int, name: Name, address: String)
 
@@ -338,6 +351,272 @@ abstract class SchemaPruningSuite
     }
   }
 
+  testSchemaPruning("SPARK-34638: nested column prune on generator output") {
+    val query1 = spark.table("contacts")
+      .select(explode(col("friends")).as("friend"))
+      .select("friend.first")
+    checkScan(query1, "struct<friends:array<struct<first:string>>>")
+    checkAnswer(query1, Row("Susan") :: Nil)
+
+    // Currently we don't prune multiple field case.
+    val query2 = spark.table("contacts")
+      .select(explode(col("friends")).as("friend"))
+      .select("friend.first", "friend.middle")
+    checkScan(query2, "struct<friends:array<struct<first:string,middle:string,last:string>>>")
+    checkAnswer(query2, Row("Susan", "Z.") :: Nil)
+
+    val query3 = spark.table("contacts")
+      .select(explode(col("friends")).as("friend"))
+      .select("friend.first", "friend.middle", "friend")
+    checkScan(query3, "struct<friends:array<struct<first:string,middle:string,last:string>>>")
+    checkAnswer(query3, Row("Susan", "Z.", Row("Susan", "Z.", "Smith")) :: Nil)
+  }
+
+  testSchemaPruning("SPARK-34638: nested column prune on generator output - case-sensitivity") {
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+      val query1 = spark.table("contacts")
+        .select(explode(col("friends")).as("friend"))
+        .select("friend.First")
+      checkScan(query1, "struct<friends:array<struct<first:string>>>")
+      checkAnswer(query1, Row("Susan") :: Nil)
+
+      val query2 = spark.table("contacts")
+        .select(explode(col("friends")).as("friend"))
+        .select("friend.MIDDLE")
+      checkScan(query2, "struct<friends:array<struct<middle:string>>>")
+      checkAnswer(query2, Row("Z.") :: Nil)
+    }
+  }
+
+  testSchemaPruning("select one deep nested complex field after repartition") {
+    val query = sql("select * from contacts")
+      .repartition(100)
+      .where("employer.company.address is not null")
+      .selectExpr("employer.id as employer_id")
+    checkScan(query,
+      "struct<employer:struct<id:int,company:struct<address:string>>>")
+    checkAnswer(query, Row(0) :: Nil)
+  }
+
+  testSchemaPruning("select one deep nested complex field after repartition by expression") {
+    val query1 = sql("select * from contacts")
+      .repartition(100, col("id"))
+      .where("employer.company.address is not null")
+      .selectExpr("employer.id as employer_id")
+    checkScan(query1,
+      "struct<id:int,employer:struct<id:int,company:struct<address:string>>>")
+    checkAnswer(query1, Row(0) :: Nil)
+
+    val query2 = sql("select * from contacts")
+      .repartition(100, col("employer"))
+      .where("employer.company.address is not null")
+      .selectExpr("employer.id as employer_id")
+    checkScan(query2,
+      "struct<employer:struct<id:int,company:struct<name:string,address:string>>>")
+    checkAnswer(query2, Row(0) :: Nil)
+
+    val query3 = sql("select * from contacts")
+      .repartition(100, col("employer.company"))
+      .where("employer.company.address is not null")
+      .selectExpr("employer.company as employer_company")
+    checkScan(query3,
+      "struct<employer:struct<company:struct<name:string,address:string>>>")
+    checkAnswer(query3, Row(Row("abc", "123 Business Street")) :: Nil)
+
+    val query4 = sql("select * from contacts")
+      .repartition(100, col("employer.company.address"))
+      .where("employer.company.address is not null")
+      .selectExpr("employer.company.address as employer_company_addr")
+    checkScan(query4,
+      "struct<employer:struct<company:struct<address:string>>>")
+    checkAnswer(query4, Row("123 Business Street") :: Nil)
+  }
+
+  testSchemaPruning("select one deep nested complex field after join") {
+    val query1 = sql("select contacts.name.middle from contacts, departments where " +
+        "contacts.id = departments.contactId")
+    checkScan(query1,
+      "struct<id:int,name:struct<middle:string>>",
+    "struct<contactId:int>")
+    checkAnswer(query1, Row("X.") :: Row("Y.") :: Nil)
+
+    val query2 = sql("select contacts.name.middle from contacts, departments where " +
+      "contacts.employer = departments.employer")
+    checkScan(query2,
+      "struct<name:struct<middle:string>," +
+        "employer:struct<id:int,company:struct<name:string,address:string>>>",
+      "struct<employer:struct<id:int,company:struct<name:string,address:string>>>")
+    checkAnswer(query2, Row("X.") :: Row("Y.") :: Nil)
+
+    val query3 = sql("select contacts.employer.company.name from contacts, departments where " +
+      "contacts.employer = departments.employer")
+    checkScan(query3,
+      "struct<employer:struct<id:int,company:struct<name:string,address:string>>>",
+      "struct<employer:struct<id:int,company:struct<name:string,address:string>>>")
+    checkAnswer(query3, Row("abc") :: Row(null) :: Nil)
+  }
+
+  testSchemaPruning("select one deep nested complex field after outer join") {
+    val query1 = sql("select departments.contactId, contacts.name.middle from departments " +
+      "left outer join contacts on departments.contactId = contacts.id")
+    checkScan(query1,
+      "struct<contactId:int>",
+      "struct<id:int,name:struct<middle:string>>")
+    checkAnswer(query1, Row(0, "X.") :: Row(1, "Y.") :: Row(4, null) :: Nil)
+
+    val query2 = sql("select contacts.name.first, departments.employer.company.name " +
+      "from contacts right outer join departments on contacts.id = departments.contactId")
+    checkScan(query2,
+      "struct<id:int,name:struct<first:string>>",
+      "struct<contactId:int,employer:struct<company:struct<name:string>>>")
+    checkAnswer(query2,
+      Row("Jane", "abc") ::
+        Row("John", null) ::
+        Row(null, null) :: Nil)
+  }
+
+  testSchemaPruning("select nested field in aggregation function of Aggregate") {
+    val query1 = sql("select count(name.first) from contacts group by name.last")
+    checkScan(query1, "struct<name:struct<first:string,last:string>>")
+    checkAnswer(query1, Row(2) :: Row(2) :: Nil)
+
+    val query2 = sql("select count(name.first), sum(pets) from contacts group by id")
+    checkScan(query2, "struct<id:int,name:struct<first:string>,pets:int>")
+    checkAnswer(query2, Row(1, 1) :: Row(1, null) :: Row(1, 3) :: Row(1, null) :: Nil)
+
+    val query3 = sql("select count(name.first), first(name) from contacts group by id")
+    checkScan(query3, "struct<id:int,name:struct<first:string,middle:string,last:string>>")
+    checkAnswer(query3,
+      Row(1, Row("Jane", "X.", "Doe")) ::
+        Row(1, Row("Jim", null, "Jones")) ::
+        Row(1, Row("John", "Y.", "Doe")) ::
+        Row(1, Row("Janet", null, "Jones")) :: Nil)
+
+    val query4 = sql("select count(name.first), sum(pets) from contacts group by name.last")
+    checkScan(query4, "struct<name:struct<first:string,last:string>,pets:int>")
+    checkAnswer(query4, Row(2, null) :: Row(2, 4) :: Nil)
+  }
+
+  testSchemaPruning("select nested field in window function") {
+    val windowSql =
+      """
+        |with contact_rank as (
+        |  select row_number() over (partition by address order by id desc) as rank,
+        |  contacts.*
+        |  from contacts
+        |)
+        |select name.first, rank from contact_rank
+        |where name.first = 'Jane' AND rank = 1
+        |""".stripMargin
+    val query = sql(windowSql)
+    checkScan(query, "struct<id:int,name:struct<first:string>,address:string>")
+    checkAnswer(query, Row("Jane", 1) :: Nil)
+  }
+
+  testSchemaPruning("select nested field in window function and then order by") {
+    val windowSql =
+      """
+        |with contact_rank as (
+        |  select row_number() over (partition by address order by id desc) as rank,
+        |  contacts.*
+        |  from contacts
+        |  order by name.last, name.first
+        |)
+        |select name.first, rank from contact_rank
+        |""".stripMargin
+    val query = sql(windowSql)
+    checkScan(query, "struct<id:int,name:struct<first:string,last:string>,address:string>")
+    checkAnswer(query,
+      Row("Jane", 1) ::
+        Row("John", 1) ::
+        Row("Janet", 1) ::
+        Row("Jim", 1) :: Nil)
+  }
+
+  testSchemaPruning("select nested field in Sort") {
+    val query1 = sql("select name.first, name.last from contacts order by name.first, name.last")
+    checkScan(query1, "struct<name:struct<first:string,last:string>>")
+    checkAnswer(query1,
+      Row("Jane", "Doe") ::
+        Row("Janet", "Jones") ::
+        Row("Jim", "Jones") ::
+        Row("John", "Doe") :: Nil)
+
+    withTempView("tmp_contacts") {
+      // Create a repartitioned view because `SORT BY` is a local sort
+      sql("select * from contacts").repartition(1).createOrReplaceTempView("tmp_contacts")
+      val sortBySql =
+        """
+          |select name.first, name.last from tmp_contacts
+          |sort by name.first, name.last
+          |""".stripMargin
+      val query2 = sql(sortBySql)
+      checkScan(query2, "struct<name:struct<first:string,last:string>>")
+      checkAnswer(query2,
+        Row("Jane", "Doe") ::
+          Row("Janet", "Jones") ::
+          Row("Jim", "Jones") ::
+          Row("John", "Doe") :: Nil)
+    }
+  }
+
+  testSchemaPruning("select nested field in Expand") {
+    import org.apache.spark.sql.catalyst.dsl.expressions._
+
+    val query1 = Expand(
+      Seq(
+        Seq($"name.first", $"name.last"),
+        Seq(Concat(Seq($"name.first", $"name.last")),
+          Concat(Seq($"name.last", $"name.first")))
+      ),
+      Seq('a.string, 'b.string),
+      sql("select * from contacts").logicalPlan
+    ).toDF()
+    checkScan(query1, "struct<name:struct<first:string,last:string>>")
+    checkAnswer(query1,
+      Row("Jane", "Doe") ::
+        Row("JaneDoe", "DoeJane") ::
+        Row("John", "Doe") ::
+        Row("JohnDoe", "DoeJohn") ::
+        Row("Jim", "Jones") ::
+        Row("JimJones", "JonesJim") ::
+        Row("Janet", "Jones") ::
+        Row("JanetJones", "JonesJanet") :: Nil)
+
+    val name = StructType.fromDDL("first string, middle string, last string")
+    val query2 = Expand(
+      Seq(Seq($"name", $"name.last")),
+      Seq('a.struct(name), 'b.string),
+      sql("select * from contacts").logicalPlan
+    ).toDF()
+    checkScan(query2, "struct<name:struct<first:string,middle:string,last:string>>")
+    checkAnswer(query2,
+      Row(Row("Jane", "X.", "Doe"), "Doe") ::
+        Row(Row("John", "Y.", "Doe"), "Doe") ::
+        Row(Row("Jim", null, "Jones"), "Jones") ::
+        Row(Row("Janet", null, "Jones"), "Jones") ::Nil)
+  }
+
+  testSchemaPruning("SPARK-32163: nested pruning should work even with cosmetic variations") {
+    withTempView("contact_alias") {
+      sql("select * from contacts")
+        .repartition(100, col("name.first"), col("name.last"))
+        .selectExpr("name").createOrReplaceTempView("contact_alias")
+
+      val query1 = sql("select name.first from contact_alias")
+      checkScan(query1, "struct<name:struct<first:string,last:string>>")
+      checkAnswer(query1, Row("Jane") :: Row("John") :: Row("Jim") :: Row("Janet") ::Nil)
+
+      sql("select * from contacts")
+        .select(explode(col("friends.first")), col("friends"))
+        .createOrReplaceTempView("contact_alias")
+
+      val query2 = sql("select friends.middle, col from contact_alias")
+      checkScan(query2, "struct<friends:array<struct<first:string,middle:string>>>")
+      checkAnswer(query2, Row(Array("Z."), "Susan") :: Nil)
+    }
+  }
+
   protected def testSchemaPruning(testName: String)(testThunk: => Unit): Unit = {
     test(s"Spark vectorized reader - without partition data column - $testName") {
       withSQLConf(vectorizedReaderEnabledKey -> "true") {
@@ -368,6 +647,7 @@ abstract class SchemaPruningSuite
 
       makeDataSourceFile(contacts, new File(path + "/contacts/p=1"))
       makeDataSourceFile(briefContacts, new File(path + "/contacts/p=2"))
+      makeDataSourceFile(departments, new File(path + "/departments"))
 
       // Providing user specified schema. Inferred schema from different data sources might
       // be different.
@@ -380,6 +660,11 @@ abstract class SchemaPruningSuite
       spark.read.format(dataSourceName).schema(schema).load(path + "/contacts")
         .createOrReplaceTempView("contacts")
 
+      val departmentSchema = "`depId` INT,`depName` STRING,`contactId` INT, " +
+        "`employer` STRUCT<`id`: INT, `company`: STRUCT<`name`: STRING, `address`: STRING>>"
+      spark.read.format(dataSourceName).schema(departmentSchema).load(path + "/departments")
+        .createOrReplaceTempView("departments")
+
       testThunk
     }
   }
@@ -390,6 +675,7 @@ abstract class SchemaPruningSuite
 
       makeDataSourceFile(contactsWithDataPartitionColumn, new File(path + "/contacts/p=1"))
       makeDataSourceFile(briefContactsWithDataPartitionColumn, new File(path + "/contacts/p=2"))
+      makeDataSourceFile(departments, new File(path + "/departments"))
 
       // Providing user specified schema. Inferred schema from different data sources might
       // be different.
@@ -402,6 +688,11 @@ abstract class SchemaPruningSuite
       spark.read.format(dataSourceName).schema(schema).load(path + "/contacts")
         .createOrReplaceTempView("contacts")
 
+      val departmentSchema = "`depId` INT,`depName` STRING,`contactId` INT, " +
+        "`employer` STRUCT<`id`: INT, `company`: STRUCT<`name`: STRING, `address`: STRING>>"
+      spark.read.format(dataSourceName).schema(departmentSchema).load(path + "/departments")
+        .createOrReplaceTempView("departments")
+
       testThunk
     }
   }
@@ -520,4 +811,63 @@ abstract class SchemaPruningSuite
         assert(scanSchema === expectedScanSchema)
     }
   }
+
+  testSchemaPruning("SPARK-34963: extract case-insensitive struct field from array") {
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+      val query1 = spark.table("contacts")
+        .select("friends.First", "friends.MiDDle")
+      checkScan(query1, "struct<friends:array<struct<first:string,middle:string>>>")
+      checkAnswer(query1,
+        Row(Array.empty[String], Array.empty[String]) ::
+          Row(Array("Susan"), Array("Z.")) ::
+          Row(null, null) ::
+          Row(null, null) :: Nil)
+
+      val query2 = spark.table("contacts")
+        .where("friends.First is not null")
+        .select("friends.First", "friends.MiDDle")
+      checkScan(query2, "struct<friends:array<struct<first:string,middle:string>>>")
+      checkAnswer(query2,
+        Row(Array.empty[String], Array.empty[String]) ::
+          Row(Array("Susan"), Array("Z.")) :: Nil)
+    }
+  }
+
+  testSchemaPruning("SPARK-34963: extract case-insensitive struct field from struct") {
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+      val query1 = spark.table("contacts")
+        .select("Name.First", "NAME.MiDDle")
+      checkScan(query1, "struct<name:struct<first:string,middle:string>>")
+      checkAnswer(query1,
+        Row("Jane", "X.") ::
+          Row("Janet", null) ::
+          Row("Jim", null) ::
+          Row("John", "Y.") :: Nil)
+
+      val query2 = spark.table("contacts")
+        .where("Name.MIDDLE is not null")
+        .select("Name.First", "NAME.MiDDle")
+      checkScan(query2, "struct<name:struct<first:string,middle:string>>")
+      checkAnswer(query2,
+        Row("Jane", "X.") ::
+          Row("John", "Y.") :: Nil)
+    }
+  }
+
+  test("SPARK-34638: queries should not fail on unsupported cases") {
+    withTable("nested_array") {
+      sql("select * from values array(array(named_struct('a', 1, 'b', 3), " +
+        "named_struct('a', 2, 'b', 4))) T(items)").write.saveAsTable("nested_array")
+      val query = sql("select d.a from (select explode(c) d from " +
+        "(select explode(items) c from nested_array))")
+      checkAnswer(query, Row(1) :: Row(2) :: Nil)
+    }
+
+    withTable("map") {
+      sql("select * from values map(1, named_struct('a', 1, 'b', 3), " +
+        "2, named_struct('a', 2, 'b', 4)) T(items)").write.saveAsTable("map")
+      val query = sql("select d.a from (select explode(items) (c, d) from map)")
+      checkAnswer(query, Row(1) :: Row(2) :: Nil)
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala
index 8462916daaab8..86ff026d7b1e9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala
@@ -34,7 +34,7 @@ import org.apache.spark.sql.execution.datasources.PartitionedFile
 import org.apache.spark.sql.functions.col
 import org.apache.spark.sql.internal.SQLConf.SOURCES_BINARY_FILE_MAX_LENGTH
 import org.apache.spark.sql.sources._
-import org.apache.spark.sql.test.{SharedSparkSession, SQLTestUtils}
+import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types._
 import org.apache.spark.util.Utils
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
index 366cf11871fa0..fd25a79619d24 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
@@ -36,13 +36,21 @@ import org.apache.hadoop.io.compress.GzipCodec
 import org.apache.spark.{SparkConf, SparkException, TestUtils}
 import org.apache.spark.sql.{AnalysisException, Column, DataFrame, QueryTest, Row}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.execution.datasources.CommonFileDataSourceSuite
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types._
 
-abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvData {
+abstract class CSVSuite
+  extends QueryTest
+  with SharedSparkSession
+  with TestCsvData
+  with CommonFileDataSourceSuite {
+
   import testImplicits._
 
+  override protected def dataSourceFormat = "csv"
+
   private val carsFile = "test-data/cars.csv"
   private val carsMalformedFile = "test-data/cars-malformed.csv"
   private val carsFile8859 = "test-data/cars_iso-8859-1.csv"
@@ -1450,10 +1458,22 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
       val ds = sampledTestData.coalesce(1)
       ds.write.text(path.getAbsolutePath)
 
-      val readback = spark.read
+      val readback1 = spark.read
         .option("inferSchema", true).option("samplingRatio", 0.1)
         .csv(path.getCanonicalPath)
-      assert(readback.schema == new StructType().add("_c0", IntegerType))
+      assert(readback1.schema == new StructType().add("_c0", IntegerType))
+
+      withClue("SPARK-32621: 'path' option can cause issues while inferring schema") {
+        // During infer, "path" option gets added again to the paths that have already been listed.
+        // This results in reading more data than necessary and causes different schema to be
+        // inferred when sampling ratio is involved.
+        val readback2 = spark.read
+          .option("inferSchema", true).option("samplingRatio", 0.1)
+          .option("path", path.getCanonicalPath)
+          .format("csv")
+          .load
+        assert(readback2.schema == new StructType().add("_c0", IntegerType))
+      }
     })
   }
 
@@ -1890,25 +1910,26 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
 
   test("SPARK-25387: bad input should not cause NPE") {
     val schema = StructType(StructField("a", IntegerType) :: Nil)
-    val input = spark.createDataset(Seq("\u0000\u0000\u0001234"))
+    val input = spark.createDataset(Seq("\u0001\u0000\u0001234"))
 
     checkAnswer(spark.read.schema(schema).csv(input), Row(null))
     checkAnswer(spark.read.option("multiLine", true).schema(schema).csv(input), Row(null))
-    assert(spark.read.csv(input).collect().toSet == Set(Row()))
+    assert(spark.read.schema(schema).csv(input).collect().toSet == Set(Row(null)))
   }
 
   test("SPARK-31261: bad csv input with `columnNameCorruptRecord` should not cause NPE") {
     val schema = StructType(
       StructField("a", IntegerType) :: StructField("_corrupt_record", StringType) :: Nil)
-    val input = spark.createDataset(Seq("\u0000\u0000\u0001234"))
+    val input = spark.createDataset(Seq("\u0001\u0000\u0001234"))
 
     checkAnswer(
       spark.read
         .option("columnNameOfCorruptRecord", "_corrupt_record")
         .schema(schema)
         .csv(input),
-      Row(null, null))
-    assert(spark.read.csv(input).collect().toSet == Set(Row()))
+      Row(null, "\u0001\u0000\u0001234"))
+    assert(spark.read.schema(schema).csv(input).collect().toSet ==
+      Set(Row(null, "\u0001\u0000\u0001234")))
   }
 
   test("field names of inferred schema shouldn't compare to the first row") {
@@ -2341,6 +2362,107 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
       checkAnswer(csv, Row(null))
     }
   }
+
+  test("SPARK-32025: infer the schema from mixed-type values") {
+    withTempPath { path =>
+      Seq("col_mixed_types", "2012", "1997", "True").toDS.write.text(path.getCanonicalPath)
+      val df = spark.read.format("csv")
+        .option("header", "true")
+        .option("inferSchema", "true")
+        .load(path.getCanonicalPath)
+
+      assert(df.schema.last == StructField("col_mixed_types", StringType, true))
+    }
+  }
+
+  test("SPARK-32614: don't treat rows starting with null char as comment") {
+    withTempPath { path =>
+      Seq("\u0000foo", "bar", "baz").toDS.write.text(path.getCanonicalPath)
+      val df = spark.read.format("csv")
+        .option("header", "false")
+        .option("inferSchema", "true")
+        .load(path.getCanonicalPath)
+      assert(df.count() == 3)
+    }
+  }
+
+  test("case sensitivity of filters references") {
+    Seq(true, false).foreach { filterPushdown =>
+      withSQLConf(SQLConf.CSV_FILTER_PUSHDOWN_ENABLED.key -> filterPushdown.toString) {
+        withTempPath { path =>
+          Seq(
+            """aaa,BBB""",
+            """0,1""",
+            """2,3""").toDF().repartition(1).write.text(path.getCanonicalPath)
+          withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+            val readback = spark.read.schema("aaa integer, BBB integer")
+              .option("header", true)
+              .csv(path.getCanonicalPath)
+            checkAnswer(readback, Seq(Row(2, 3), Row(0, 1)))
+            checkAnswer(readback.filter($"AAA" === 2 && $"bbb" === 3), Seq(Row(2, 3)))
+          }
+          withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+            val readback = spark.read.schema("aaa integer, BBB integer")
+              .option("header", true)
+              .csv(path.getCanonicalPath)
+            checkAnswer(readback, Seq(Row(2, 3), Row(0, 1)))
+            val errorMsg = intercept[AnalysisException] {
+              readback.filter($"AAA" === 2 && $"bbb" === 3).collect()
+            }.getMessage
+            assert(errorMsg.contains("cannot resolve 'AAA'"))
+          }
+        }
+      }
+    }
+  }
+
+  test("SPARK-32810: CSV data source should be able to read files with " +
+    "escaped glob metacharacter in the paths") {
+    withTempDir { dir =>
+      val basePath = dir.getCanonicalPath
+      // test CSV writer / reader without specifying schema
+      val csvTableName = "[abc]"
+      spark.range(3).coalesce(1).write.csv(s"$basePath/$csvTableName")
+      val readback = spark.read
+        .csv(s"$basePath/${"""(\[|\]|\{|\})""".r.replaceAllIn(csvTableName, """\\$1""")}")
+      assert(readback.collect sameElements Array(Row("0"), Row("1"), Row("2")))
+    }
+  }
+
+  test("SPARK-33566: configure UnescapedQuoteHandling to parse " +
+    "unescaped quotes and unescaped delimiter data correctly") {
+    withTempPath { path =>
+      val dataPath = path.getCanonicalPath
+      val row1 = Row("""a,""b,c""", "xyz")
+      val row2 = Row("""a,b,c""", """x""yz""")
+      // Generate the test data, use `,` as delimiter and `"` as quotes, but they didn't escape.
+      Seq(
+        """c1,c2""",
+        s""""${row1.getString(0)}","${row1.getString(1)}"""",
+        s""""${row2.getString(0)}","${row2.getString(1)}"""")
+        .toDF().repartition(1).write.text(dataPath)
+      // Without configure UnescapedQuoteHandling to STOP_AT_CLOSING_QUOTE,
+      // the result will be Row(""""a,""b""", """c""""), Row("""a,b,c""", """"x""yz"""")
+      val result = spark.read
+        .option("inferSchema", "true")
+        .option("header", "true")
+        .option("unescapedQuoteHandling", "STOP_AT_CLOSING_QUOTE")
+        .csv(dataPath).collect()
+      val exceptResults = Array(row1, row2)
+      assert(result.sameElements(exceptResults))
+    }
+  }
+
+  test("SPARK-34768: counting a long record with ignoreTrailingWhiteSpace set to true") {
+    val bufSize = 128
+    val line = "X" * (bufSize - 1) + "| |"
+    withTempPath { path =>
+      Seq(line).toDF.write.text(path.getAbsolutePath)
+      assert(spark.read.format("csv")
+        .option("delimiter", "|")
+        .option("ignoreTrailingWhiteSpace", "true").load(path.getAbsolutePath).count() == 1)
+    }
+  }
 }
 
 class CSVv1Suite extends CSVSuite {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/DriverRegistrySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/DriverRegistrySuite.scala
new file mode 100644
index 0000000000000..51dbdacb5e0fe
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/DriverRegistrySuite.scala
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.jdbc
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.execution.datasources.jdbc.connection.TestDriver
+
+class DriverRegistrySuite extends SparkFunSuite {
+  test("SPARK-32229: get must give back wrapped driver if wrapped") {
+    val className = classOf[TestDriver].getName
+    DriverRegistry.register(className)
+    assert(DriverRegistry.get(className).isInstanceOf[TestDriver])
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/ConnectionProviderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/ConnectionProviderSuite.scala
index ff5fe4f620a1d..71b0325f93732 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/ConnectionProviderSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/ConnectionProviderSuite.scala
@@ -19,27 +19,61 @@ package org.apache.spark.sql.execution.datasources.jdbc.connection
 
 import javax.security.auth.login.Configuration
 
-class ConnectionProviderSuite extends ConnectionProviderSuiteBase {
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.test.SharedSparkSession
+
+class ConnectionProviderSuite extends ConnectionProviderSuiteBase with SharedSparkSession {
+  test("All built-in providers must be loaded") {
+    IntentionallyFaultyConnectionProvider.constructed = false
+    val providers = ConnectionProvider.loadProviders()
+    assert(providers.exists(_.isInstanceOf[BasicConnectionProvider]))
+    assert(providers.exists(_.isInstanceOf[DB2ConnectionProvider]))
+    assert(providers.exists(_.isInstanceOf[MariaDBConnectionProvider]))
+    assert(providers.exists(_.isInstanceOf[MSSQLConnectionProvider]))
+    assert(providers.exists(_.isInstanceOf[PostgresConnectionProvider]))
+    assert(providers.exists(_.isInstanceOf[OracleConnectionProvider]))
+    assert(IntentionallyFaultyConnectionProvider.constructed)
+    assert(!providers.exists(_.isInstanceOf[IntentionallyFaultyConnectionProvider]))
+    assert(providers.size === 6)
+  }
+
+  test("Disabled provider must not be loaded") {
+    withSQLConf(SQLConf.DISABLED_JDBC_CONN_PROVIDER_LIST.key -> "db2") {
+      val providers = ConnectionProvider.loadProviders()
+      assert(!providers.exists(_.isInstanceOf[DB2ConnectionProvider]))
+      assert(providers.size === 5)
+    }
+  }
+
   test("Multiple security configs must be reachable") {
     Configuration.setConfiguration(null)
-    val postgresDriver = registerDriver(PostgresConnectionProvider.driverClass)
-    val postgresProvider = new PostgresConnectionProvider(
-      postgresDriver, options("jdbc:postgresql://localhost/postgres"))
-    val db2Driver = registerDriver(DB2ConnectionProvider.driverClass)
-    val db2Provider = new DB2ConnectionProvider(db2Driver, options("jdbc:db2://localhost/db2"))
+    val postgresProvider = new PostgresConnectionProvider()
+    val postgresDriver = registerDriver(postgresProvider.driverClass)
+    val postgresOptions = options("jdbc:postgresql://localhost/postgres")
+    val postgresAppEntry = postgresProvider.appEntry(postgresDriver, postgresOptions)
+    val db2Provider = new DB2ConnectionProvider()
+    val db2Driver = registerDriver(db2Provider.driverClass)
+    val db2Options = options("jdbc:db2://localhost/db2")
+    val db2AppEntry = db2Provider.appEntry(db2Driver, db2Options)
 
     // Make sure no authentication for the databases are set
-    val oldConfig = Configuration.getConfiguration
-    assert(oldConfig.getAppConfigurationEntry(postgresProvider.appEntry) == null)
-    assert(oldConfig.getAppConfigurationEntry(db2Provider.appEntry) == null)
+    val rootConfig = Configuration.getConfiguration
+    assert(rootConfig.getAppConfigurationEntry(postgresAppEntry) == null)
+    assert(rootConfig.getAppConfigurationEntry(db2AppEntry) == null)
+
+    postgresProvider.setAuthenticationConfig(postgresDriver, postgresOptions)
+    val postgresConfig = Configuration.getConfiguration
 
-    postgresProvider.setAuthenticationConfigIfNeeded()
-    db2Provider.setAuthenticationConfigIfNeeded()
+    db2Provider.setAuthenticationConfig(db2Driver, db2Options)
+    val db2Config = Configuration.getConfiguration
 
     // Make sure authentication for the databases are set
-    val newConfig = Configuration.getConfiguration
-    assert(oldConfig != newConfig)
-    assert(newConfig.getAppConfigurationEntry(postgresProvider.appEntry) != null)
-    assert(newConfig.getAppConfigurationEntry(db2Provider.appEntry) != null)
+    assert(rootConfig != postgresConfig)
+    assert(rootConfig != db2Config)
+    // The topmost config in the chain is linked with all the subsequent entries
+    assert(db2Config.getAppConfigurationEntry(postgresAppEntry) != null)
+    assert(db2Config.getAppConfigurationEntry(db2AppEntry) != null)
+
+    Configuration.setConfiguration(null)
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/ConnectionProviderSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/ConnectionProviderSuiteBase.scala
index d18a3088c4f2f..f42b17abf31bc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/ConnectionProviderSuiteBase.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/ConnectionProviderSuiteBase.scala
@@ -42,7 +42,7 @@ abstract class ConnectionProviderSuiteBase extends SparkFunSuite with BeforeAndA
     JDBCOptions.JDBC_PRINCIPAL -> "principal"
   ))
 
-  override def afterEach(): Unit = {
+  protected override def afterEach(): Unit = {
     try {
       Configuration.setConfiguration(null)
     } finally {
@@ -50,20 +50,21 @@ abstract class ConnectionProviderSuiteBase extends SparkFunSuite with BeforeAndA
     }
   }
 
-  protected def testSecureConnectionProvider(provider: SecureConnectionProvider): Unit = {
+  protected def testSecureConnectionProvider(
+      provider: SecureConnectionProvider,
+      driver: Driver,
+      options: JDBCOptions): Unit = {
+    val providerAppEntry = provider.appEntry(driver, options)
+
     // Make sure no authentication for the database is set
-    assert(Configuration.getConfiguration.getAppConfigurationEntry(provider.appEntry) == null)
+    assert(Configuration.getConfiguration.getAppConfigurationEntry(providerAppEntry) == null)
 
-    // Make sure the first call sets authentication properly
+    // Make sure setAuthenticationConfig call sets authentication properly
     val savedConfig = Configuration.getConfiguration
-    provider.setAuthenticationConfigIfNeeded()
+    provider.setAuthenticationConfig(driver, options)
     val config = Configuration.getConfiguration
     assert(savedConfig != config)
-    val appEntry = config.getAppConfigurationEntry(provider.appEntry)
+    val appEntry = config.getAppConfigurationEntry(providerAppEntry)
     assert(appEntry != null)
-
-    // Make sure a second call is not modifying the existing authentication
-    provider.setAuthenticationConfigIfNeeded()
-    assert(config.getAppConfigurationEntry(provider.appEntry) === appEntry)
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/DB2ConnectionProviderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/DB2ConnectionProviderSuite.scala
index d656f83e2ebb9..895b3d85d960b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/DB2ConnectionProviderSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/DB2ConnectionProviderSuite.scala
@@ -18,10 +18,10 @@
 package org.apache.spark.sql.execution.datasources.jdbc.connection
 
 class DB2ConnectionProviderSuite extends ConnectionProviderSuiteBase {
-  test("setAuthenticationConfigIfNeeded must set authentication if not set") {
-    val driver = registerDriver(DB2ConnectionProvider.driverClass)
-    val provider = new DB2ConnectionProvider(driver, options("jdbc:db2://localhost/db2"))
+  test("setAuthenticationConfig must set authentication all the time") {
+    val provider = new DB2ConnectionProvider()
+    val driver = registerDriver(provider.driverClass)
 
-    testSecureConnectionProvider(provider)
+    testSecureConnectionProvider(provider, driver, options("jdbc:db2://localhost/db2"))
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/IntentionallyFaultyConnectionProvider.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/IntentionallyFaultyConnectionProvider.scala
new file mode 100644
index 0000000000000..329d79cae62e8
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/IntentionallyFaultyConnectionProvider.scala
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.jdbc.connection
+
+import java.sql.{Connection, Driver}
+
+import org.apache.spark.sql.jdbc.JdbcConnectionProvider
+
+private class IntentionallyFaultyConnectionProvider extends JdbcConnectionProvider {
+  IntentionallyFaultyConnectionProvider.constructed = true
+  throw new IllegalArgumentException("Intentional Exception")
+  override val name: String = "IntentionallyFaultyConnectionProvider"
+  override def canHandle(driver: Driver, options: Map[String, String]): Boolean = true
+  override def getConnection(driver: Driver, options: Map[String, String]): Connection = null
+}
+
+private object IntentionallyFaultyConnectionProvider {
+  var constructed = false
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/MSSQLConnectionProviderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/MSSQLConnectionProviderSuite.scala
new file mode 100644
index 0000000000000..a0b9af2d82e13
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/MSSQLConnectionProviderSuite.scala
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.jdbc.connection
+
+import java.sql.Driver
+
+import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions
+
+class MSSQLConnectionProviderSuite extends ConnectionProviderSuiteBase {
+  test("setAuthenticationConfig default parser must set authentication all the time") {
+    val provider = new MSSQLConnectionProvider()
+    val driver = registerDriver(provider.driverClass)
+
+    testProviders(driver, provider, options("jdbc:sqlserver://localhost/mssql"),
+      options("jdbc:sqlserver://localhost/mssql;jaasConfigurationName=custommssql"))
+  }
+
+  test("setAuthenticationConfig custom parser must set authentication all the time") {
+    val provider = new MSSQLConnectionProvider() {
+      override val parserMethod: String = "IntentionallyNotExistingMethod"
+    }
+    val driver = registerDriver(provider.driverClass)
+
+    testProviders(driver, provider, options("jdbc:sqlserver://localhost/mssql"),
+      options("jdbc:sqlserver://localhost/mssql;jaasConfigurationName=custommssql"))
+  }
+
+  private def testProviders(
+      driver: Driver,
+      provider: SecureConnectionProvider,
+      defaultOptions: JDBCOptions,
+      customOptions: JDBCOptions) = {
+    testSecureConnectionProvider(provider, driver, defaultOptions)
+    testSecureConnectionProvider(provider, driver, customOptions)
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/MariaDBConnectionProviderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/MariaDBConnectionProviderSuite.scala
index 70cad2097eb43..d8bdf26b35c7d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/MariaDBConnectionProviderSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/MariaDBConnectionProviderSuite.scala
@@ -18,10 +18,10 @@
 package org.apache.spark.sql.execution.datasources.jdbc.connection
 
 class MariaDBConnectionProviderSuite extends ConnectionProviderSuiteBase {
-  test("setAuthenticationConfigIfNeeded must set authentication if not set") {
-    val driver = registerDriver(MariaDBConnectionProvider.driverClass)
-    val provider = new MariaDBConnectionProvider(driver, options("jdbc:mysql://localhost/mysql"))
+  test("setAuthenticationConfig must set authentication all the time") {
+    val provider = new MariaDBConnectionProvider()
+    val driver = registerDriver(provider.driverClass)
 
-    testSecureConnectionProvider(provider)
+    testSecureConnectionProvider(provider, driver, options("jdbc:mysql://localhost/mysql"))
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/OracleConnectionProviderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/OracleConnectionProviderSuite.scala
new file mode 100644
index 0000000000000..4aaaf8168eb53
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/OracleConnectionProviderSuite.scala
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.jdbc.connection
+
+class OracleConnectionProviderSuite extends ConnectionProviderSuiteBase {
+  test("setAuthenticationConfig must set authentication all the time") {
+    val provider = new OracleConnectionProvider()
+    val driver = registerDriver(provider.driverClass)
+
+    testSecureConnectionProvider(provider, driver, options("jdbc:oracle:thin:@//localhost/xe"))
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/PostgresConnectionProviderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/PostgresConnectionProviderSuite.scala
index 8cef7652f9c54..5006bf4091380 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/PostgresConnectionProviderSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/PostgresConnectionProviderSuite.scala
@@ -18,15 +18,15 @@
 package org.apache.spark.sql.execution.datasources.jdbc.connection
 
 class PostgresConnectionProviderSuite extends ConnectionProviderSuiteBase {
-  test("setAuthenticationConfigIfNeeded must set authentication if not set") {
-    val driver = registerDriver(PostgresConnectionProvider.driverClass)
-    val defaultProvider = new PostgresConnectionProvider(
-      driver, options("jdbc:postgresql://localhost/postgres"))
-    val customProvider = new PostgresConnectionProvider(
-      driver, options(s"jdbc:postgresql://localhost/postgres?jaasApplicationName=custompgjdbc"))
+  test("setAuthenticationConfig must set authentication all the time") {
+    val provider = new PostgresConnectionProvider()
+    val defaultOptions = options("jdbc:postgresql://localhost/postgres")
+    val customOptions =
+      options(s"jdbc:postgresql://localhost/postgres?jaasApplicationName=custompgjdbc")
+    val driver = registerDriver(provider.driverClass)
 
-    assert(defaultProvider.appEntry !== customProvider.appEntry)
-    testSecureConnectionProvider(defaultProvider)
-    testSecureConnectionProvider(customProvider)
+    assert(provider.appEntry(driver, defaultOptions) !== provider.appEntry(driver, customOptions))
+    testSecureConnectionProvider(provider, driver, defaultOptions)
+    testSecureConnectionProvider(provider, driver, customOptions)
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/TestDriver.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/TestDriver.scala
new file mode 100644
index 0000000000000..6b57a95ed458b
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/TestDriver.scala
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.jdbc.connection
+
+import java.sql.{Connection, Driver, DriverPropertyInfo}
+import java.util.Properties
+import java.util.logging.Logger
+
+private[jdbc] class TestDriver() extends Driver {
+  override def connect(url: String, info: Properties): Connection = null
+  override def acceptsURL(url: String): Boolean = false
+  override def getPropertyInfo(url: String, info: Properties): Array[DriverPropertyInfo] =
+    Array.empty
+  override def getMajorVersion: Int = 0
+  override def getMinorVersion: Int = 0
+  override def jdbcCompliant(): Boolean = false
+  override def getParentLogger: Logger = null
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmark.scala
index 56930880ed5da..e4f6ccaa9a621 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmark.scala
@@ -20,9 +20,10 @@ import java.io.File
 import java.time.{Instant, LocalDate}
 
 import org.apache.spark.benchmark.Benchmark
-import org.apache.spark.sql.{Dataset, Row}
+import org.apache.spark.sql.{Column, Dataset, Row}
 import org.apache.spark.sql.execution.benchmark.SqlBasedBenchmark
 import org.apache.spark.sql.functions._
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 /**
@@ -38,16 +39,9 @@ import org.apache.spark.sql.types._
  *      Results will be written to "benchmarks/JSONBenchmark-results.txt".
  * }}}
  */
-
 object JsonBenchmark extends SqlBasedBenchmark {
   import spark.implicits._
 
-  private def prepareDataInfo(benchmark: Benchmark): Unit = {
-    // scalastyle:off println
-    benchmark.out.println("Preparing data for benchmarking ...")
-    // scalastyle:on println
-  }
-
   def schemaInferring(rowsNum: Int, numIters: Int): Unit = {
     val benchmark = new Benchmark("JSON schema inferring", rowsNum, output = output)
 
@@ -127,18 +121,6 @@ object JsonBenchmark extends SqlBasedBenchmark {
       .add("z", StringType)
   }
 
-  def writeWideRow(path: String, rowsNum: Int): StructType = {
-    val colsNum = 1000
-    val fields = Seq.tabulate(colsNum)(i => StructField(s"col$i", IntegerType))
-    val schema = StructType(fields)
-
-    spark.range(rowsNum)
-      .select(Seq.tabulate(colsNum)(i => lit(i).as(s"col$i")): _*)
-      .write.json(path)
-
-    schema
-  }
-
   def countWideColumn(rowsNum: Int, numIters: Int): Unit = {
     val benchmark = new Benchmark("count a wide column", rowsNum, output = output)
 
@@ -170,7 +152,7 @@ object JsonBenchmark extends SqlBasedBenchmark {
 
     withTempPath { path =>
       prepareDataInfo(benchmark)
-      val schema = writeWideRow(path.getAbsolutePath, rowsNum)
+      val schema = writeWideRow(path.getAbsolutePath, rowsNum, 1000)
 
       benchmark.addCase("No encoding", numIters) { _ =>
         spark.read
@@ -430,7 +412,7 @@ object JsonBenchmark extends SqlBasedBenchmark {
       }
 
       readBench.addCase("infer timestamps from files", numIters) { _ =>
-        spark.read.json(timestampDir).noop()
+        spark.read.option("inferTimestamp", true).json(timestampDir).noop()
       }
 
       val dateSchema = new StructType().add("date", DateType)
@@ -460,7 +442,7 @@ object JsonBenchmark extends SqlBasedBenchmark {
       }
 
       readBench.addCase("infer timestamps from Dataset[String]", numIters) { _ =>
-        spark.read.json(timestampStr).noop()
+        spark.read.option("inferTimestamp", true).json(timestampStr).noop()
       }
 
       def dateStr: Dataset[String] = {
@@ -495,19 +477,61 @@ object JsonBenchmark extends SqlBasedBenchmark {
     }
   }
 
+  private def filtersPushdownBenchmark(rowsNum: Int, numIters: Int): Unit = {
+    val benchmark = new Benchmark("Filters pushdown", rowsNum, output = output)
+    val colsNum = 100
+    val fields = Seq.tabulate(colsNum)(i => StructField(s"col$i", TimestampType))
+    val schema = StructType(StructField("key", IntegerType) +: fields)
+    def columns(): Seq[Column] = {
+      val ts = Seq.tabulate(colsNum) { i =>
+        lit(Instant.ofEpochSecond(i * 12345678)).as(s"col$i")
+      }
+      ($"id" % 1000).as("key") +: ts
+    }
+    withTempPath { path =>
+      spark.range(rowsNum).select(columns(): _*).write.json(path.getAbsolutePath)
+      def readback = {
+        spark.read.schema(schema).json(path.getAbsolutePath)
+      }
+
+      benchmark.addCase("w/o filters", numIters) { _ =>
+        readback.noop()
+      }
+
+      def withFilter(configEnabled: Boolean): Unit = {
+        withSQLConf(SQLConf.JSON_FILTER_PUSHDOWN_ENABLED.key -> configEnabled.toString()) {
+          readback.filter($"key" === 0).noop()
+        }
+      }
+
+      benchmark.addCase("pushdown disabled", numIters) { _ =>
+        withFilter(configEnabled = false)
+      }
+
+      benchmark.addCase("w/ filters", numIters) { _ =>
+        withFilter(configEnabled = true)
+      }
+
+      benchmark.run()
+    }
+  }
+
   override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
     val numIters = 3
     runBenchmark("Benchmark for performance of JSON parsing") {
-      schemaInferring(100 * 1000 * 1000, numIters)
-      countShortColumn(100 * 1000 * 1000, numIters)
-      countWideColumn(10 * 1000 * 1000, numIters)
-      countWideRow(500 * 1000, numIters)
-      selectSubsetOfColumns(10 * 1000 * 1000, numIters)
-      jsonParserCreation(10 * 1000 * 1000, numIters)
-      jsonFunctions(10 * 1000 * 1000, numIters)
-      jsonInDS(50 * 1000 * 1000, numIters)
-      jsonInFile(50 * 1000 * 1000, numIters)
-      datetimeBenchmark(rowsNum = 10 * 1000 * 1000, numIters)
+      schemaInferring(5 * 1000 * 1000, numIters)
+      countShortColumn(5 * 1000 * 1000, numIters)
+      countWideColumn(1000 * 1000, numIters)
+      countWideRow(50 * 1000, numIters)
+      selectSubsetOfColumns(1000 * 1000, numIters)
+      jsonParserCreation(1000 * 1000, numIters)
+      jsonFunctions(1000 * 1000, numIters)
+      jsonInDS(5 * 1000 * 1000, numIters)
+      jsonInFile(5 * 1000 * 1000, numIters)
+      datetimeBenchmark(rowsNum = 1000 * 1000, numIters)
+      // Benchmark pushdown filters that refer to top-level columns.
+      // TODO (SPARK-32325): Add benchmarks for filters with nested column attributes.
+      filtersPushdownBenchmark(rowsNum = 100 * 1000, numIters)
     }
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonParsingOptionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonParsingOptionsSuite.scala
index 7592809d7c85b..e9fe79a0641b9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonParsingOptionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonParsingOptionsSuite.scala
@@ -17,12 +17,12 @@
 
 package org.apache.spark.sql.execution.datasources.json
 
-import org.apache.spark.sql.QueryTest
-import org.apache.spark.sql.catalyst.json.JSONOptions
+import org.apache.spark.sql.{QueryTest, Row}
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.{StringType, StructType}
 
 /**
- * Test cases for various [[JSONOptions]].
+ * Test cases for various [[org.apache.spark.sql.catalyst.json.JSONOptions]].
  */
 class JsonParsingOptionsSuite extends QueryTest with SharedSparkSession {
   import testImplicits._
@@ -73,14 +73,14 @@ class JsonParsingOptionsSuite extends QueryTest with SharedSparkSession {
   }
 
   test("allowUnquotedControlChars off") {
-    val str = """{"name": "a\u0001b"}"""
+    val str = "{\"name\": \"a\u0001b\"}"
     val df = spark.read.json(Seq(str).toDS())
 
     assert(df.schema.head.name == "_corrupt_record")
   }
 
   test("allowUnquotedControlChars on") {
-    val str = """{"name": "a\u0001b"}"""
+    val str = "{\"name\": \"a\u0001b\"}"
     val df = spark.read.option("allowUnquotedControlChars", "true").json(Seq(str).toDS())
 
     assert(df.schema.head.name == "name")
@@ -102,21 +102,32 @@ class JsonParsingOptionsSuite extends QueryTest with SharedSparkSession {
     assert(df.first().getLong(0) == 18)
   }
 
-  // The following two tests are not really working - need to look into Jackson's
-  // JsonReadFeature.ALLOW_NON_NUMERIC_NUMBERS.
-  ignore("allowNonNumericNumbers off") {
+  test("allowNonNumericNumbers off") {
     val str = """{"age": NaN}"""
-    val df = spark.read.json(Seq(str).toDS())
+    val df = spark.read.option("allowNonNumericNumbers", false).json(Seq(str).toDS())
 
-    assert(df.schema.head.name == "_corrupt_record")
+    assert(df.schema === new StructType().add("_corrupt_record", StringType))
+    checkAnswer(df, Row(str))
   }
 
-  ignore("allowNonNumericNumbers on") {
-    val str = """{"age": NaN}"""
-    val df = spark.read.option("allowNonNumericNumbers", "true").json(Seq(str).toDS())
-
-    assert(df.schema.head.name == "age")
-    assert(df.first().getDouble(0).isNaN)
+  test("allowNonNumericNumbers on") {
+    val str = """{"c0":NaN, "c1":+INF, "c2":+Infinity, "c3":Infinity, "c4":-INF, "c5":-Infinity}"""
+    val df = spark.read.option("allowNonNumericNumbers", true).json(Seq(str).toDS())
+
+    assert(df.schema ===
+      new StructType()
+        .add("c0", "double")
+        .add("c1", "double")
+        .add("c2", "double")
+        .add("c3", "double")
+        .add("c4", "double")
+        .add("c5", "double"))
+    checkAnswer(
+      df,
+      Row(
+        Double.NaN,
+        Double.PositiveInfinity, Double.PositiveInfinity, Double.PositiveInfinity,
+        Double.NegativeInfinity, Double.NegativeInfinity))
   }
 
   test("allowBackslashEscapingAnyCharacter off") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
index dcea4835b5f2a..b707a48413647 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
@@ -21,7 +21,7 @@ import java.io._
 import java.nio.charset.{Charset, StandardCharsets, UnsupportedCharsetException}
 import java.nio.file.Files
 import java.sql.{Date, Timestamp}
-import java.time.LocalDate
+import java.time.{LocalDate, ZoneId}
 import java.util.Locale
 
 import com.fasterxml.jackson.core.JsonFactory
@@ -35,22 +35,30 @@ import org.apache.spark.sql.{functions => F, _}
 import org.apache.spark.sql.catalyst.json._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.execution.ExternalRDD
-import org.apache.spark.sql.execution.adaptive.AdaptiveTestUtils.assertExceptionMessage
-import org.apache.spark.sql.execution.datasources.DataSource
+import org.apache.spark.sql.execution.datasources.{CommonFileDataSourceSuite, DataSource, InMemoryFileIndex, NoopCache}
+import org.apache.spark.sql.execution.datasources.v2.json.JsonScanBuilder
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.types.StructType.fromDDL
 import org.apache.spark.sql.types.TestUDT.{MyDenseVector, MyDenseVectorUDT}
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
 import org.apache.spark.util.Utils
 
 class TestFileFilter extends PathFilter {
   override def accept(path: Path): Boolean = path.getParent.getName != "p=2"
 }
 
-abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJsonData {
+abstract class JsonSuite
+  extends QueryTest
+  with SharedSparkSession
+  with TestJsonData
+  with CommonFileDataSourceSuite {
+
   import testImplicits._
 
+  override protected def dataSourceFormat = "json"
+
   test("Type promotion") {
     def checkTypePromotion(expected: Any, actual: Any): Unit = {
       assert(expected.getClass == actual.getClass,
@@ -126,7 +134,7 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
           Map("timestampFormat" -> "yyyy-MM-dd'T'HH:mm:ssXXX")))
 
     val ISO8601Date = "1970-01-01"
-    checkTypePromotion(DateTimeUtils.microsToDays(32400000000L),
+    checkTypePromotion(DateTimeUtils.microsToDays(32400000000L, ZoneId.systemDefault),
       enforceCorrectType(ISO8601Date, DateType))
   }
 
@@ -1557,6 +1565,15 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
         "mapreduce.input.pathFilter.class" -> classOf[TestFileFilter].getName
       )
       assert(spark.read.options(extraOptions).json(path).count() === 2)
+
+      withClue("SPARK-32621: 'path' option can cause issues while inferring schema") {
+        // During infer, "path" option is used again on top of the paths that have already been
+        // listed. When a partition is removed by TestFileFilter, this will cause a conflict while
+        // inferring partitions because the original path in the "path" option will list the
+        // partition directory that has been removed.
+        assert(
+          spark.read.options(extraOptions).format("json").option("path", path).load.count() === 2)
+      }
     }
   }
 
@@ -2142,9 +2159,18 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
     )(withTempPath { path =>
       val ds = sampledTestData.coalesce(1)
       ds.write.text(path.getAbsolutePath)
-      val readback = spark.read.option("samplingRatio", 0.1).json(path.getCanonicalPath)
-
-      assert(readback.schema == new StructType().add("f1", LongType))
+      val readback1 = spark.read.option("samplingRatio", 0.1).json(path.getCanonicalPath)
+      assert(readback1.schema == new StructType().add("f1", LongType))
+
+      withClue("SPARK-32621: 'path' option can cause issues while inferring schema") {
+        // During infer, "path" option gets added again to the paths that have already been listed.
+        // This results in reading more data than necessary and causes different schema to be
+        // inferred when sampling ratio is involved.
+        val readback2 = spark.read
+          .option("samplingRatio", 0.1).option("path", path.getCanonicalPath)
+          .format("json").load
+        assert(readback2.schema == new StructType().add("f1", LongType))
+      }
     })
   }
 
@@ -2239,7 +2265,7 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
         .count()
     }
 
-    assertExceptionMessage(exception, "Malformed records are detected in record parsing")
+    assert(exception.getMessage.contains("Malformed records are detected in record parsing"))
   }
 
   def checkEncoding(expectedEncoding: String, pathToJsonFiles: String,
@@ -2489,7 +2515,7 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
           .json(testFile("test-data/utf16LE.json"))
           .count()
       }
-      assert(exception.getMessage.contains("encoding must not be included in the blacklist"))
+      assert(exception.getMessage.contains("encoding must not be included in the denyList"))
     }
   }
 
@@ -2611,7 +2637,9 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
   }
 
   test("inferring timestamp type") {
-    def schemaOf(jsons: String*): StructType = spark.read.json(jsons.toDS).schema
+    def schemaOf(jsons: String*): StructType = {
+      spark.read.option("inferTimestamp", true).json(jsons.toDS).schema
+    }
 
     assert(schemaOf(
       """{"a":"2018-12-17T10:11:12.123-01:00"}""",
@@ -2634,6 +2662,7 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
       val timestampsWithFormatPath = s"${dir.getCanonicalPath}/timestampsWithFormat.json"
       val timestampsWithFormat = spark.read
         .option("timestampFormat", "dd/MM/yyyy HH:mm")
+        .option("inferTimestamp", true)
         .json(datesRecords)
       assert(timestampsWithFormat.schema === customSchema)
 
@@ -2646,6 +2675,7 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
       val readBack = spark.read
         .option("timestampFormat", "yyyy-MM-dd HH:mm:ss")
         .option(DateTimeUtils.TIMEZONE_OPTION, "UTC")
+        .option("inferTimestamp", true)
         .json(timestampsWithFormatPath)
 
       assert(readBack.schema === customSchema)
@@ -2685,6 +2715,215 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
       checkAnswer(json, Row(null))
     }
   }
+
+  test("filters push down") {
+    withTempPath { path =>
+      val t = "2019-12-17 00:01:02"
+      Seq(
+        """{"c0": "abc", "c1": {"c2": 1, "c3": "2019-11-14 20:35:30"}}""",
+        s"""{"c0": "def", "c1": {"c2": 2, "c3": "$t"}}""",
+        s"""{"c0": "defa", "c1": {"c2": 3, "c3": "$t"}}""",
+        s"""{"c0": "define", "c1": {"c2": 2, "c3": "$t"}}""").toDF("data")
+        .repartition(1)
+        .write.text(path.getAbsolutePath)
+      Seq(true, false).foreach { filterPushdown =>
+        withSQLConf(SQLConf.JSON_FILTER_PUSHDOWN_ENABLED.key -> filterPushdown.toString) {
+          Seq("PERMISSIVE", "DROPMALFORMED", "FAILFAST").foreach { mode =>
+            val readback = spark.read
+              .option("mode", mode)
+              .option("timestampFormat", "yyyy-MM-dd HH:mm:ss")
+              .schema("c0 string, c1 struct<c2:integer,c3:timestamp>")
+              .json(path.getAbsolutePath)
+              .where($"c1.c2" === 2 && $"c0".startsWith("def"))
+              .select($"c1.c3")
+            assert(readback.count() === 2)
+            checkAnswer(readback, Seq(Row(Timestamp.valueOf(t)), Row(Timestamp.valueOf(t))))
+          }
+        }
+      }
+    }
+  }
+
+  test("apply filters to malformed rows") {
+    withSQLConf(SQLConf.JSON_FILTER_PUSHDOWN_ENABLED.key -> "true") {
+      withTempPath { path =>
+        Seq(
+          "{}",
+          """{"invalid": 0}""",
+          """{"i":}""",
+          """{"i": 0}""",
+          """{"i": 1, "t": "2020-01-28 01:00:00"}""",
+          """{"t": "2020-01-28 02:00:00"}""",
+          """{"i": "abc", "t": "2020-01-28 03:00:00"}""",
+          """{"i": 2, "t": "2020-01-28 04:00:00", "d": 3.14}""").toDF("data")
+          .repartition(1)
+          .write.text(path.getAbsolutePath)
+        val schema = "i INTEGER, t TIMESTAMP"
+        val readback = spark.read
+          .schema(schema)
+          .option("timestampFormat", "yyyy-MM-dd HH:mm:ss")
+          .json(path.getAbsolutePath)
+        // readback:
+        // +----+-------------------+
+        // |i   |t                  |
+        // +----+-------------------+
+        // |null|null               |
+        // |null|null               |
+        // |null|null               |
+        // |0   |null               |
+        // |1   |2020-01-28 01:00:00|
+        // |null|2020-01-28 02:00:00|
+        // |null|2020-01-28 03:00:00|
+        // |2   |2020-01-28 04:00:00|
+        // +----+-------------------+
+        checkAnswer(
+          readback.where($"i".isNull && $"t".isNotNull),
+          Seq(
+            Row(null, Timestamp.valueOf("2020-01-28 02:00:00")),
+            Row(null, Timestamp.valueOf("2020-01-28 03:00:00"))))
+        checkAnswer(
+          readback.where($"i" >= 0 && $"t" > "2020-01-28 00:00:00"),
+          Seq(
+            Row(1, Timestamp.valueOf("2020-01-28 01:00:00")),
+            Row(2, Timestamp.valueOf("2020-01-28 04:00:00"))))
+        checkAnswer(
+          readback.where($"t".isNull).select($"i"),
+          Seq(Row(null), Row(null), Row(null), Row(0)))
+      }
+    }
+  }
+
+  test("case sensitivity of filters references") {
+    Seq(true, false).foreach { filterPushdown =>
+      withSQLConf(SQLConf.JSON_FILTER_PUSHDOWN_ENABLED.key -> filterPushdown.toString) {
+        withTempPath { path =>
+          Seq(
+            """{"aaa": 0, "BBB": 1}""",
+            """{"AAA": 2, "bbb": 3}""").toDF().write.text(path.getCanonicalPath)
+          withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+            val readback = spark.read.schema("aaa integer, BBB integer")
+              .json(path.getCanonicalPath)
+            checkAnswer(readback, Seq(Row(null, null), Row(0, 1)))
+            checkAnswer(readback.filter($"AAA" === 0 && $"bbb" === 1), Seq(Row(0, 1)))
+            checkAnswer(readback.filter($"AAA" === 2 && $"bbb" === 3), Seq())
+            // Schema inferring
+            val errorMsg = intercept[AnalysisException] {
+              spark.read.json(path.getCanonicalPath).collect()
+            }.getMessage
+            assert(errorMsg.contains("Found duplicate column(s) in the data schema"))
+          }
+          withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+            val readback = spark.read.schema("aaa integer, BBB integer")
+              .json(path.getCanonicalPath)
+            checkAnswer(readback, Seq(Row(null, null), Row(0, 1)))
+            val errorMsg = intercept[AnalysisException] {
+              readback.filter($"AAA" === 0 && $"bbb" === 1).collect()
+            }.getMessage
+            assert(errorMsg.contains("cannot resolve 'AAA'"))
+            // Schema inferring
+            val readback2 = spark.read.json(path.getCanonicalPath)
+            checkAnswer(
+              readback2.filter($"AAA" === 2).select($"AAA", $"bbb"),
+              Seq(Row(2, 3)))
+            checkAnswer(readback2.filter($"aaa" === 2).select($"AAA", $"bbb"), Seq())
+          }
+        }
+      }
+    }
+  }
+
+  test("SPARK-32810: JSON data source should be able to read files with " +
+    "escaped glob metacharacter in the paths") {
+    withTempDir { dir =>
+      val basePath = dir.getCanonicalPath
+      // test JSON writer / reader without specifying schema
+      val jsonTableName = "{def}"
+      spark.range(3).coalesce(1).write.json(s"$basePath/$jsonTableName")
+      val readback = spark.read
+        .json(s"$basePath/${"""(\[|\]|\{|\})""".r.replaceAllIn(jsonTableName, """\\$1""")}")
+      assert(readback.collect sameElements Array(Row(0), Row(1), Row(2)))
+    }
+  }
+
+  test("SPARK-35047: Write Non-ASCII character as codepoint") {
+    // scalastyle:off nonascii
+    withTempPaths(2) { paths =>
+      paths.foreach(_.delete())
+      val seq = Seq("a", "\n", "\u3042")
+      val df = seq.toDF
+
+      val basePath1 = paths(0).getCanonicalPath
+      df.write.option("writeNonAsciiCharacterAsCodePoint", "true")
+        .option("pretty", "false").json(basePath1)
+      val actualText1 = spark.read.option("wholetext", "true").text(basePath1)
+        .sort("value").map(_.getString(0)).collect().mkString
+      val expectedText1 =
+        s"""{"value":"\\n"}
+           |{"value":"\\u3042"}
+           |{"value":"a"}
+           |""".stripMargin
+      assert(actualText1 === expectedText1)
+
+      val actualJson1 = spark.read.json(basePath1)
+        .sort("value").map(_.getString(0)).collect().mkString
+      val expectedJson1 = "\na\u3042"
+      assert(actualJson1 === expectedJson1)
+
+      // Test for pretty printed JSON.
+      // If multiLine option is set to true, the format should be should be
+      // one JSON record per file. So LEAF_NODE_DEFAULT_PARALLELISM is set here.
+      withSQLConf(SQLConf.LEAF_NODE_DEFAULT_PARALLELISM.key -> s"${seq.length}") {
+        val basePath2 = paths(1).getCanonicalPath
+        df.write.option("writeNonAsciiCharacterAsCodePoint", "true")
+          .option("pretty", "true").json(basePath2)
+        val actualText2 = spark.read.option("wholetext", "true").text(basePath2)
+          .sort("value").map(_.getString(0)).collect().mkString
+        val expectedText2 =
+          s"""{
+             |  "value" : "\\n"
+             |}
+             |{
+             |  "value" : "\\u3042"
+             |}
+             |{
+             |  "value" : "a"
+             |}
+             |""".stripMargin
+        assert(actualText2 === expectedText2)
+
+        val actualJson2 = spark.read.option("multiLine", "true").json(basePath2)
+          .sort("value").map(_.getString(0)).collect().mkString
+        val expectedJson2 = "\na\u3042"
+        assert(actualJson2 === expectedJson2)
+      }
+    }
+    // scalastyle:on nonascii
+  }
+
+  test("SPARK-35104: Fix wrong indentation for multiple JSON even if `pretty` option is true") {
+    withSQLConf(SQLConf.LEAF_NODE_DEFAULT_PARALLELISM.key -> "1") {
+      withTempPath { path =>
+        val basePath = path.getCanonicalPath
+        val df = Seq("a", "b", "c").toDF
+        df.write.option("pretty", "true").json(basePath)
+
+        val expectedText =
+          s"""{
+             |  "value" : "a"
+             |}
+             |{
+             |  "value" : "b"
+             |}
+             |{
+             |  "value" : "c"
+             |}
+             |""".stripMargin
+        val actualText = spark.read.option("wholetext", "true")
+          .text(basePath).map(_.getString(0)).collect().mkString
+        assert(actualText === expectedText)
+      }
+    }
+  }
 }
 
 class JsonV1Suite extends JsonSuite {
@@ -2699,6 +2938,37 @@ class JsonV2Suite extends JsonSuite {
     super
       .sparkConf
       .set(SQLConf.USE_V1_SOURCE_LIST, "")
+
+  test("get pushed filters") {
+    val attr = "col"
+    def getBuilder(path: String): JsonScanBuilder = {
+      val fileIndex = new InMemoryFileIndex(
+        spark,
+        Seq(new org.apache.hadoop.fs.Path(path, "file.json")),
+        Map.empty,
+        None,
+        NoopCache)
+      val schema = new StructType().add(attr, IntegerType)
+      val options = CaseInsensitiveStringMap.empty()
+      new JsonScanBuilder(spark, fileIndex, schema, schema, options)
+    }
+    val filters: Array[sources.Filter] = Array(sources.IsNotNull(attr))
+    withSQLConf(SQLConf.JSON_FILTER_PUSHDOWN_ENABLED.key -> "true") {
+      withTempPath { file =>
+        val scanBuilder = getBuilder(file.getCanonicalPath)
+        assert(scanBuilder.pushFilters(filters) === filters)
+        assert(scanBuilder.pushedFilters() === filters)
+      }
+    }
+
+    withSQLConf(SQLConf.JSON_FILTER_PUSHDOWN_ENABLED.key -> "false") {
+      withTempPath { file =>
+        val scanBuilder = getBuilder(file.getCanonicalPath)
+        assert(scanBuilder.pushFilters(filters) === filters)
+        assert(scanBuilder.pushedFilters() === Array.empty[sources.Filter])
+      }
+    }
+  }
 }
 
 class JsonLegacyTimeParserSuite extends JsonSuite {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReaderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReaderSuite.scala
index 719bf91e1786b..bfcef46339908 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReaderSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReaderSuite.scala
@@ -17,16 +17,29 @@
 
 package org.apache.spark.sql.execution.datasources.orc
 
+import java.io.File
+
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.mapreduce.{JobID, TaskAttemptID, TaskID, TaskType}
+import org.apache.hadoop.mapreduce.lib.input.FileSplit
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
 import org.apache.orc.TypeDescription
 
 import org.apache.spark.sql.QueryTest
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.execution.datasources.parquet.SpecificParquetRecordReaderBase
 import org.apache.spark.sql.execution.vectorized.{OnHeapColumnVector, WritableColumnVector}
 import org.apache.spark.sql.test.SharedSparkSession
-import org.apache.spark.sql.types.{StructField, StructType}
+import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.UTF8String
 import org.apache.spark.unsafe.types.UTF8String.fromString
 
 class OrcColumnarBatchReaderSuite extends QueryTest with SharedSparkSession {
+
+  import testImplicits._
+
   private val dataSchema = StructType.fromDDL("col1 int, col2 int")
   private val partitionSchema = StructType.fromDDL("p1 string, p2 string")
   private val partitionValues = InternalRow(fromString("partValue1"), fromString("partValue2"))
@@ -77,4 +90,66 @@ class OrcColumnarBatchReaderSuite extends QueryTest with SharedSparkSession {
       assert(p1.getUTF8String(0) === partitionValues.getUTF8String(0))
     }
   }
+
+  test("SPARK-33593: partition column types") {
+    withTempPath { dir =>
+      Seq(1).toDF().repartition(1).write.orc(dir.getCanonicalPath)
+
+      val dataTypes =
+        Seq(StringType, BooleanType, ByteType, BinaryType, ShortType, IntegerType, LongType,
+          FloatType, DoubleType, DecimalType(25, 5), DateType, TimestampType)
+
+      val constantValues =
+        Seq(
+          UTF8String.fromString("a string"),
+          true,
+          1.toByte,
+          "Spark SQL".getBytes,
+          2.toShort,
+          3,
+          Long.MaxValue,
+          0.25.toFloat,
+          0.75D,
+          Decimal("1234.23456"),
+          DateTimeUtils.fromJavaDate(java.sql.Date.valueOf("2015-01-01")),
+          DateTimeUtils.fromJavaTimestamp(java.sql.Timestamp.valueOf("2015-01-01 23:50:59.123")))
+
+      dataTypes.zip(constantValues).foreach { case (dt, v) =>
+        val schema = StructType(StructField("col1", IntegerType) :: StructField("pcol", dt) :: Nil)
+        val partitionValues = new GenericInternalRow(Array(v))
+        val file = new File(SpecificParquetRecordReaderBase.listDirectory(dir).get(0))
+        val fileSplit = new FileSplit(new Path(file.getCanonicalPath), 0L, file.length, Array.empty)
+        val taskConf = sqlContext.sessionState.newHadoopConf()
+        val orcFileSchema = TypeDescription.fromString(schema.simpleString)
+        val vectorizedReader = new OrcColumnarBatchReader(4096)
+        val attemptId = new TaskAttemptID(new TaskID(new JobID(), TaskType.MAP, 0), 0)
+        val taskAttemptContext = new TaskAttemptContextImpl(taskConf, attemptId)
+
+        try {
+          vectorizedReader.initialize(fileSplit, taskAttemptContext)
+          vectorizedReader.initBatch(
+            orcFileSchema,
+            schema.toArray,
+            Array(0, -1),
+            Array(-1, 0),
+            partitionValues)
+          vectorizedReader.nextKeyValue()
+          val row = vectorizedReader.getCurrentValue.getRow(0)
+
+          // Use `GenericMutableRow` by explicitly copying rather than `ColumnarBatch`
+          // in order to use get(...) method which is not implemented in `ColumnarBatch`.
+          val actual = row.copy().get(1, dt)
+          val expected = v
+          if (dt.isInstanceOf[BinaryType]) {
+            assert(actual.asInstanceOf[Array[Byte]]
+              sameElements expected.asInstanceOf[Array[Byte]])
+          } else {
+            assert(actual == expected)
+          }
+        } finally {
+          vectorizedReader.close()
+        }
+      }
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcEncryptionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcEncryptionSuite.scala
new file mode 100644
index 0000000000000..ed7a11c923f01
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcEncryptionSuite.scala
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.orc
+
+import java.util.Random
+
+import org.apache.orc.impl.HadoopShimsFactory
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.SharedSparkSession
+
+class OrcEncryptionSuite extends OrcTest with SharedSparkSession {
+  import testImplicits._
+
+  val originalData = Seq(("123456789", "dongjoon@apache.org", "Dongjoon Hyun"))
+  val rowDataWithoutKey =
+    Row(null, "841626795E7D351555B835A002E3BF10669DE9B81C95A3D59E10865AC37EA7C3", "Dongjoon Hyun")
+
+  test("Write and read an encrypted file") {
+    val conf = spark.sessionState.newHadoopConf()
+    val provider = HadoopShimsFactory.get.getHadoopKeyProvider(conf, new Random)
+    assume(!provider.getKeyNames.isEmpty,
+      s"$provider doesn't has the test keys. ORC shim is created with old Hadoop libraries")
+
+    val df = originalData.toDF("ssn", "email", "name")
+
+    withTempPath { dir =>
+      val path = dir.getAbsolutePath
+      withSQLConf(
+        "hadoop.security.key.provider.path" -> "test:///",
+        "orc.key.provider" -> "hadoop",
+        "orc.encrypt" -> "pii:ssn,email",
+        "orc.mask" -> "nullify:ssn;sha256:email") {
+        df.write.mode("overwrite").orc(path)
+        checkAnswer(spark.read.orc(path), df)
+      }
+
+      withSQLConf(
+        "orc.key.provider" -> "memory",
+        "orc.encrypt" -> "pii:ssn,email",
+        "orc.mask" -> "nullify:ssn;sha256:email") {
+        checkAnswer(spark.read.orc(path), rowDataWithoutKey)
+      }
+    }
+  }
+
+  test("Write and read an encrypted table") {
+    val conf = spark.sessionState.newHadoopConf()
+    val provider = HadoopShimsFactory.get.getHadoopKeyProvider(conf, new Random)
+    assume(!provider.getKeyNames.isEmpty,
+      s"$provider doesn't has the test keys. ORC shim is created with old Hadoop libraries")
+
+    val df = originalData.toDF("ssn", "email", "name")
+
+    withTempDir { dir =>
+      val path = dir.getAbsolutePath
+      withTable("encrypted") {
+        sql(
+          s"""
+            |CREATE TABLE encrypted (
+            |  ssn STRING,
+            |  email STRING,
+            |  name STRING
+            |)
+            |USING ORC
+            |LOCATION "$path"
+            |OPTIONS (
+            |  hadoop.security.key.provider.path "test:///",
+            |  orc.key.provider "hadoop",
+            |  orc.encrypt "pii:ssn,email",
+            |  orc.mask "nullify:ssn;sha256:email"
+            |)
+            |""".stripMargin)
+        sql("INSERT INTO encrypted VALUES('123456789', 'dongjoon@apache.org', 'Dongjoon Hyun')")
+        checkAnswer(sql("SELECT * FROM encrypted"), df)
+      }
+      withTable("normal") {
+        sql(
+          s"""
+            |CREATE TABLE normal (
+            |  ssn STRING,
+            |  email STRING,
+            |  name STRING
+            |)
+            |USING ORC
+            |LOCATION "$path"
+            |OPTIONS (
+            |  orc.key.provider "memory",
+            |  orc.encrypt "pii:ssn,email",
+            |  orc.mask "nullify:ssn;sha256:email"
+            |)
+            |""".stripMargin)
+        checkAnswer(sql("SELECT * FROM normal"), rowDataWithoutKey)
+      }
+    }
+  }
+
+  test("SPARK-35325: Write and read encrypted nested columns") {
+    val conf = spark.sessionState.newHadoopConf()
+    val provider = HadoopShimsFactory.get.getHadoopKeyProvider(conf, new Random)
+    assume(!provider.getKeyNames.isEmpty,
+      s"$provider doesn't has the test keys. ORC shim is created with old Hadoop libraries")
+
+    val originalNestedData = Row(1, Row("123456789", "dongjoon@apache.org", "Dongjoon"))
+    val rowNestedDataWithoutKey =
+      Row(1, Row(null, "841626795E7D351555B835A002E3BF10669DE9B81C95A3D59E10865AC37EA7C3",
+        "Dongjoon"))
+
+    withTempDir { dir =>
+      val path = dir.getAbsolutePath
+      withTable("encrypted") {
+        sql(
+          s"""
+            |CREATE TABLE encrypted (
+            |  id INT,
+            |  contact struct<ssn:STRING, email:STRING, name:STRING>
+            |)
+            |USING ORC
+            |LOCATION "$path"
+            |OPTIONS (
+            |  hadoop.security.key.provider.path "test:///",
+            |  orc.key.provider "hadoop",
+            |  orc.encrypt "pii:contact.ssn,contact.email",
+            |  orc.mask "nullify:contact.ssn;sha256:contact.email"
+            |)
+            |""".stripMargin)
+        sql("INSERT INTO encrypted VALUES(1, ('123456789', 'dongjoon@apache.org', 'Dongjoon'))")
+        checkAnswer(sql("SELECT * FROM encrypted"), originalNestedData)
+      }
+      withTable("normal") {
+        sql(
+          s"""
+            |CREATE TABLE normal (
+            |  id INT,
+            |  contact struct<ssn:STRING, email:STRING, name:STRING>
+            |)
+            |USING ORC
+            |LOCATION "$path"
+            |OPTIONS (
+            |  orc.key.provider "memory"
+            |)
+            |""".stripMargin)
+        checkAnswer(sql("SELECT * FROM normal"), rowNestedDataWithoutKey)
+      }
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
new file mode 100644
index 0000000000000..681ed91afaa12
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.orc
+
+import java.math.MathContext
+import java.nio.charset.StandardCharsets
+import java.sql.{Date, Timestamp}
+
+import scala.collection.JavaConverters._
+
+import org.apache.hadoop.hive.ql.io.sarg.{PredicateLeaf, SearchArgument}
+import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory.newBuilder
+
+import org.apache.spark.{SparkConf, SparkException}
+import org.apache.spark.sql.{AnalysisException, Column, DataFrame, Row}
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.planning.PhysicalOperation
+import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanRelation
+import org.apache.spark.sql.execution.datasources.v2.orc.OrcScan
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types._
+
+/**
+ * A test suite that tests Apache ORC filter API based filter pushdown optimization.
+ */
+class OrcFilterSuite extends OrcTest with SharedSparkSession {
+
+  override protected def sparkConf: SparkConf =
+    super
+      .sparkConf
+      .set(SQLConf.USE_V1_SOURCE_LIST, "")
+
+  protected def checkFilterPredicate(
+      df: DataFrame,
+      predicate: Predicate,
+      checker: (SearchArgument) => Unit): Unit = {
+    val output = predicate.collect { case a: Attribute => a }.distinct
+    val query = df
+      .select(output.map(e => Column(e)): _*)
+      .where(Column(predicate))
+
+    query.queryExecution.optimizedPlan match {
+      case PhysicalOperation(_, filters, DataSourceV2ScanRelation(_, o: OrcScan, _)) =>
+        assert(filters.nonEmpty, "No filter is analyzed from the given query")
+        assert(o.pushedFilters.nonEmpty, "No filter is pushed down")
+        val maybeFilter = OrcFilters.createFilter(query.schema, o.pushedFilters)
+        assert(maybeFilter.isDefined, s"Couldn't generate filter predicate for ${o.pushedFilters}")
+        checker(maybeFilter.get)
+
+      case _ =>
+        throw new AnalysisException("Can not match OrcTable in the query.")
+    }
+  }
+
+  protected def checkFilterPredicate
+      (predicate: Predicate, filterOperator: PredicateLeaf.Operator)
+      (implicit df: DataFrame): Unit = {
+    def checkComparisonOperator(filter: SearchArgument) = {
+      val operator = filter.getLeaves.asScala
+      assert(operator.map(_.getOperator).contains(filterOperator))
+    }
+    checkFilterPredicate(df, predicate, checkComparisonOperator)
+  }
+
+  protected def checkFilterPredicate
+      (predicate: Predicate, stringExpr: String)
+      (implicit df: DataFrame): Unit = {
+    def checkLogicalOperator(filter: SearchArgument) = {
+      assert(filter.toString == stringExpr)
+    }
+    checkFilterPredicate(df, predicate, checkLogicalOperator)
+  }
+
+  test("filter pushdown - integer") {
+    withNestedOrcDataFrame((1 to 4).map(i => Tuple1(Option(i)))) { case (inputDF, colName, _) =>
+      implicit val df: DataFrame = inputDF
+
+      val intAttr = df(colName).expr
+      assert(df(colName).expr.dataType === IntegerType)
+
+      checkFilterPredicate(intAttr.isNull, PredicateLeaf.Operator.IS_NULL)
+
+      checkFilterPredicate(intAttr === 1, PredicateLeaf.Operator.EQUALS)
+      checkFilterPredicate(intAttr <=> 1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+
+      checkFilterPredicate(intAttr < 2, PredicateLeaf.Operator.LESS_THAN)
+      checkFilterPredicate(intAttr > 3, PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(intAttr <= 1, PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(intAttr >= 4, PredicateLeaf.Operator.LESS_THAN)
+
+      checkFilterPredicate(Literal(1) === intAttr, PredicateLeaf.Operator.EQUALS)
+      checkFilterPredicate(Literal(1) <=> intAttr, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+      checkFilterPredicate(Literal(2) > intAttr, PredicateLeaf.Operator.LESS_THAN)
+      checkFilterPredicate(Literal(3) < intAttr, PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(Literal(1) >= intAttr, PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(Literal(4) <= intAttr, PredicateLeaf.Operator.LESS_THAN)
+    }
+  }
+
+  test("filter pushdown - long") {
+    withNestedOrcDataFrame(
+        (1 to 4).map(i => Tuple1(Option(i.toLong)))) { case (inputDF, colName, _) =>
+      implicit val df: DataFrame = inputDF
+
+      val longAttr = df(colName).expr
+      assert(df(colName).expr.dataType === LongType)
+
+      checkFilterPredicate(longAttr.isNull, PredicateLeaf.Operator.IS_NULL)
+
+      checkFilterPredicate(longAttr === 1, PredicateLeaf.Operator.EQUALS)
+      checkFilterPredicate(longAttr <=> 1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+
+      checkFilterPredicate(longAttr < 2, PredicateLeaf.Operator.LESS_THAN)
+      checkFilterPredicate(longAttr > 3, PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(longAttr <= 1, PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(longAttr >= 4, PredicateLeaf.Operator.LESS_THAN)
+
+      checkFilterPredicate(Literal(1) === longAttr, PredicateLeaf.Operator.EQUALS)
+      checkFilterPredicate(Literal(1) <=> longAttr, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+      checkFilterPredicate(Literal(2) > longAttr, PredicateLeaf.Operator.LESS_THAN)
+      checkFilterPredicate(Literal(3) < longAttr, PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(Literal(1) >= longAttr, PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(Literal(4) <= longAttr, PredicateLeaf.Operator.LESS_THAN)
+    }
+  }
+
+  test("filter pushdown - float") {
+    withNestedOrcDataFrame(
+        (1 to 4).map(i => Tuple1(Option(i.toFloat)))) { case (inputDF, colName, _) =>
+      implicit val df: DataFrame = inputDF
+
+      val floatAttr = df(colName).expr
+      assert(df(colName).expr.dataType === FloatType)
+
+      checkFilterPredicate(floatAttr.isNull, PredicateLeaf.Operator.IS_NULL)
+
+      checkFilterPredicate(floatAttr === 1, PredicateLeaf.Operator.EQUALS)
+      checkFilterPredicate(floatAttr <=> 1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+
+      checkFilterPredicate(floatAttr < 2, PredicateLeaf.Operator.LESS_THAN)
+      checkFilterPredicate(floatAttr > 3, PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(floatAttr <= 1, PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(floatAttr >= 4, PredicateLeaf.Operator.LESS_THAN)
+
+      checkFilterPredicate(Literal(1) === floatAttr, PredicateLeaf.Operator.EQUALS)
+      checkFilterPredicate(Literal(1) <=> floatAttr, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+      checkFilterPredicate(Literal(2) > floatAttr, PredicateLeaf.Operator.LESS_THAN)
+      checkFilterPredicate(Literal(3) < floatAttr, PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(Literal(1) >= floatAttr, PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(Literal(4) <= floatAttr, PredicateLeaf.Operator.LESS_THAN)
+    }
+  }
+
+  test("filter pushdown - double") {
+    withNestedOrcDataFrame(
+        (1 to 4).map(i => Tuple1(Option(i.toDouble)))) { case (inputDF, colName, _) =>
+      implicit val df: DataFrame = inputDF
+
+      val doubleAttr = df(colName).expr
+      assert(df(colName).expr.dataType === DoubleType)
+
+      checkFilterPredicate(doubleAttr.isNull, PredicateLeaf.Operator.IS_NULL)
+
+      checkFilterPredicate(doubleAttr === 1, PredicateLeaf.Operator.EQUALS)
+      checkFilterPredicate(doubleAttr <=> 1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+
+      checkFilterPredicate(doubleAttr < 2, PredicateLeaf.Operator.LESS_THAN)
+      checkFilterPredicate(doubleAttr > 3, PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(doubleAttr <= 1, PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(doubleAttr >= 4, PredicateLeaf.Operator.LESS_THAN)
+
+      checkFilterPredicate(Literal(1) === doubleAttr, PredicateLeaf.Operator.EQUALS)
+      checkFilterPredicate(Literal(1) <=> doubleAttr, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+      checkFilterPredicate(Literal(2) > doubleAttr, PredicateLeaf.Operator.LESS_THAN)
+      checkFilterPredicate(Literal(3) < doubleAttr, PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(Literal(1) >= doubleAttr, PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(Literal(4) <= doubleAttr, PredicateLeaf.Operator.LESS_THAN)
+    }
+  }
+
+  test("filter pushdown - string") {
+    withNestedOrcDataFrame(
+        (1 to 4).map(i => Tuple1(i.toString))) { case (inputDF, colName, _) =>
+      implicit val df: DataFrame = inputDF
+
+      val strAttr = df(colName).expr
+      assert(df(colName).expr.dataType === StringType)
+
+      checkFilterPredicate(strAttr.isNull, PredicateLeaf.Operator.IS_NULL)
+
+      checkFilterPredicate(strAttr === "1", PredicateLeaf.Operator.EQUALS)
+      checkFilterPredicate(strAttr <=> "1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+
+      checkFilterPredicate(strAttr < "2", PredicateLeaf.Operator.LESS_THAN)
+      checkFilterPredicate(strAttr > "3", PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(strAttr <= "1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(strAttr >= "4", PredicateLeaf.Operator.LESS_THAN)
+
+      checkFilterPredicate(Literal("1") === strAttr, PredicateLeaf.Operator.EQUALS)
+      checkFilterPredicate(Literal("1") <=> strAttr, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+      checkFilterPredicate(Literal("2") > strAttr, PredicateLeaf.Operator.LESS_THAN)
+      checkFilterPredicate(Literal("3") < strAttr, PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(Literal("1") >= strAttr, PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(Literal("4") <= strAttr, PredicateLeaf.Operator.LESS_THAN)
+    }
+  }
+
+  test("filter pushdown - boolean") {
+    withNestedOrcDataFrame(
+        (true :: false :: Nil).map(b => Tuple1.apply(Option(b)))) { case (inputDF, colName, _) =>
+      implicit val df: DataFrame = inputDF
+
+      val booleanAttr = df(colName).expr
+      assert(df(colName).expr.dataType === BooleanType)
+
+      checkFilterPredicate(booleanAttr.isNull, PredicateLeaf.Operator.IS_NULL)
+
+      checkFilterPredicate(booleanAttr === true, PredicateLeaf.Operator.EQUALS)
+      checkFilterPredicate(booleanAttr <=> true, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+
+      checkFilterPredicate(booleanAttr < true, PredicateLeaf.Operator.LESS_THAN)
+      checkFilterPredicate(booleanAttr > false, PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(booleanAttr <= false, PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(booleanAttr >= false, PredicateLeaf.Operator.LESS_THAN)
+
+      checkFilterPredicate(Literal(false) === booleanAttr, PredicateLeaf.Operator.EQUALS)
+      checkFilterPredicate(Literal(false) <=> booleanAttr,
+        PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+      checkFilterPredicate(Literal(false) > booleanAttr, PredicateLeaf.Operator.LESS_THAN)
+      checkFilterPredicate(Literal(true) < booleanAttr, PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(Literal(true) >= booleanAttr, PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(Literal(true) <= booleanAttr, PredicateLeaf.Operator.LESS_THAN)
+    }
+  }
+
+  test("filter pushdown - decimal") {
+    withNestedOrcDataFrame(
+        (1 to 4).map(i => Tuple1.apply(BigDecimal.valueOf(i)))) { case (inputDF, colName, _) =>
+      implicit val df: DataFrame = inputDF
+
+      val decimalAttr = df(colName).expr
+      assert(df(colName).expr.dataType === DecimalType(38, 18))
+
+      checkFilterPredicate(decimalAttr.isNull, PredicateLeaf.Operator.IS_NULL)
+
+      checkFilterPredicate(decimalAttr === BigDecimal.valueOf(1), PredicateLeaf.Operator.EQUALS)
+      checkFilterPredicate(decimalAttr <=> BigDecimal.valueOf(1),
+        PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+
+      checkFilterPredicate(decimalAttr < BigDecimal.valueOf(2), PredicateLeaf.Operator.LESS_THAN)
+      checkFilterPredicate(decimalAttr > BigDecimal.valueOf(3),
+        PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(decimalAttr <= BigDecimal.valueOf(1),
+        PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(decimalAttr >= BigDecimal.valueOf(4), PredicateLeaf.Operator.LESS_THAN)
+
+      checkFilterPredicate(
+        Literal(BigDecimal.valueOf(1)) === decimalAttr, PredicateLeaf.Operator.EQUALS)
+      checkFilterPredicate(
+        Literal(BigDecimal.valueOf(1)) <=> decimalAttr, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+      checkFilterPredicate(
+        Literal(BigDecimal.valueOf(2)) > decimalAttr, PredicateLeaf.Operator.LESS_THAN)
+      checkFilterPredicate(
+        Literal(BigDecimal.valueOf(3)) < decimalAttr, PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(
+        Literal(BigDecimal.valueOf(1)) >= decimalAttr, PredicateLeaf.Operator.LESS_THAN_EQUALS)
+      checkFilterPredicate(
+        Literal(BigDecimal.valueOf(4)) <= decimalAttr, PredicateLeaf.Operator.LESS_THAN)
+    }
+  }
+
+  test("filter pushdown - timestamp") {
+    val input = Seq(
+      "1000-01-01 01:02:03",
+      "1582-10-01 00:11:22",
+      "1900-01-01 23:59:59",
+      "2020-05-25 10:11:12").map(Timestamp.valueOf)
+
+    withOrcFile(input.map(Tuple1(_))) { path =>
+      Seq(false, true).foreach { java8Api =>
+        withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
+          readFile(path) { implicit df =>
+            val timestamps = input.map(Literal(_))
+            checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
+
+            checkFilterPredicate($"_1" === timestamps(0), PredicateLeaf.Operator.EQUALS)
+            checkFilterPredicate($"_1" <=> timestamps(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+
+            checkFilterPredicate($"_1" < timestamps(1), PredicateLeaf.Operator.LESS_THAN)
+            checkFilterPredicate($"_1" > timestamps(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate($"_1" <= timestamps(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate($"_1" >= timestamps(3), PredicateLeaf.Operator.LESS_THAN)
+
+            checkFilterPredicate(Literal(timestamps(0)) === $"_1", PredicateLeaf.Operator.EQUALS)
+            checkFilterPredicate(
+              Literal(timestamps(0)) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+            checkFilterPredicate(Literal(timestamps(1)) > $"_1", PredicateLeaf.Operator.LESS_THAN)
+            checkFilterPredicate(
+              Literal(timestamps(2)) < $"_1",
+              PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate(
+              Literal(timestamps(0)) >= $"_1",
+              PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate(Literal(timestamps(3)) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
+          }
+        }
+      }
+    }
+  }
+
+  test("filter pushdown - combinations with logical operators") {
+    withOrcDataFrame((1 to 4).map(i => Tuple1(Option(i)))) { implicit df =>
+      checkFilterPredicate(
+        $"_1".isNotNull,
+        "leaf-0 = (IS_NULL _1), expr = (not leaf-0)"
+      )
+      checkFilterPredicate(
+        $"_1" =!= 1,
+        "leaf-0 = (IS_NULL _1), leaf-1 = (EQUALS _1 1), expr = (and (not leaf-0) (not leaf-1))"
+      )
+      checkFilterPredicate(
+        !($"_1" < 4),
+        "leaf-0 = (IS_NULL _1), leaf-1 = (LESS_THAN _1 4), expr = (and (not leaf-0) (not leaf-1))"
+      )
+      checkFilterPredicate(
+        $"_1" < 2 || $"_1" > 3,
+        "leaf-0 = (LESS_THAN _1 2), leaf-1 = (LESS_THAN_EQUALS _1 3), " +
+          "expr = (or leaf-0 (not leaf-1))"
+      )
+      checkFilterPredicate(
+        $"_1" < 2 && $"_1" > 3,
+        "leaf-0 = (IS_NULL _1), leaf-1 = (LESS_THAN _1 2), leaf-2 = (LESS_THAN_EQUALS _1 3), " +
+          "expr = (and (not leaf-0) leaf-1 (not leaf-2))"
+      )
+    }
+  }
+
+  test("filter pushdown - date") {
+    val input = Seq("2017-08-18", "2017-08-19", "2017-08-20", "2017-08-21").map { day =>
+      Date.valueOf(day)
+    }
+    withOrcFile(input.map(Tuple1(_))) { path =>
+      Seq(false, true).foreach { java8Api =>
+        withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
+          readFile(path) { implicit df =>
+            val dates = input.map(Literal(_))
+            checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
+
+            checkFilterPredicate($"_1" === dates(0), PredicateLeaf.Operator.EQUALS)
+            checkFilterPredicate($"_1" <=> dates(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+
+            checkFilterPredicate($"_1" < dates(1), PredicateLeaf.Operator.LESS_THAN)
+            checkFilterPredicate($"_1" > dates(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate($"_1" <= dates(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate($"_1" >= dates(3), PredicateLeaf.Operator.LESS_THAN)
+
+            checkFilterPredicate(dates(0) === $"_1", PredicateLeaf.Operator.EQUALS)
+            checkFilterPredicate(dates(0) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+            checkFilterPredicate(dates(1) > $"_1", PredicateLeaf.Operator.LESS_THAN)
+            checkFilterPredicate(dates(2) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate(dates(0) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate(dates(3) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
+          }
+        }
+      }
+    }
+  }
+
+  test("no filter pushdown - non-supported types") {
+    implicit class IntToBinary(int: Int) {
+      def b: Array[Byte] = int.toString.getBytes(StandardCharsets.UTF_8)
+    }
+    // ArrayType
+    withOrcDataFrame((1 to 4).map(i => Tuple1(Array(i)))) { implicit df =>
+      checkNoFilterPredicate($"_1".isNull, noneSupported = true)
+    }
+    // BinaryType
+    withOrcDataFrame((1 to 4).map(i => Tuple1(i.b))) { implicit df =>
+      checkNoFilterPredicate($"_1" <=> 1.b, noneSupported = true)
+    }
+    // MapType
+    withOrcDataFrame((1 to 4).map(i => Tuple1(Map(i -> i)))) { implicit df =>
+      checkNoFilterPredicate($"_1".isNotNull, noneSupported = true)
+    }
+  }
+
+  test("SPARK-12218 and SPARK-25699 Converting conjunctions into ORC SearchArguments") {
+    import org.apache.spark.sql.sources._
+    // The `LessThan` should be converted while the `StringContains` shouldn't
+    val schema = new StructType(
+      Array(
+        StructField("a", IntegerType, nullable = true),
+        StructField("b", StringType, nullable = true)))
+    assertResult("leaf-0 = (LESS_THAN a 10), expr = leaf-0") {
+      OrcFilters.createFilter(schema, Array(
+        LessThan("a", 10),
+        StringContains("b", "prefix")
+      )).get.toString
+    }
+
+    // The `LessThan` should be converted while the whole inner `And` shouldn't
+    assertResult("leaf-0 = (LESS_THAN a 10), expr = leaf-0") {
+      OrcFilters.createFilter(schema, Array(
+        LessThan("a", 10),
+        Not(And(
+          GreaterThan("a", 1),
+          StringContains("b", "prefix")
+        ))
+      )).get.toString
+    }
+
+    // Safely remove unsupported `StringContains` predicate and push down `LessThan`
+    assertResult("leaf-0 = (LESS_THAN a 10), expr = leaf-0") {
+      OrcFilters.createFilter(schema, Array(
+        And(
+          LessThan("a", 10),
+          StringContains("b", "prefix")
+        )
+      )).get.toString
+    }
+
+    // Safely remove unsupported `StringContains` predicate, push down `LessThan` and `GreaterThan`.
+    assertResult("leaf-0 = (LESS_THAN a 10), leaf-1 = (LESS_THAN_EQUALS a 1)," +
+      " expr = (and leaf-0 (not leaf-1))") {
+      OrcFilters.createFilter(schema, Array(
+        And(
+          And(
+            LessThan("a", 10),
+            StringContains("b", "prefix")
+          ),
+          GreaterThan("a", 1)
+        )
+      )).get.toString
+    }
+  }
+
+  test("SPARK-27699 Converting disjunctions into ORC SearchArguments") {
+    import org.apache.spark.sql.sources._
+    // The `LessThan` should be converted while the `StringContains` shouldn't
+    val schema = new StructType(
+      Array(
+        StructField("a", IntegerType, nullable = true),
+        StructField("b", StringType, nullable = true)))
+
+    // The predicate `StringContains` predicate is not able to be pushed down.
+    assertResult("leaf-0 = (LESS_THAN_EQUALS a 10), leaf-1 = (LESS_THAN a 1)," +
+      " expr = (or (not leaf-0) leaf-1)") {
+      OrcFilters.createFilter(schema, Array(
+        Or(
+          GreaterThan("a", 10),
+          And(
+            StringContains("b", "prefix"),
+            LessThan("a", 1)
+          )
+        )
+      )).get.toString
+    }
+
+    assertResult("leaf-0 = (LESS_THAN_EQUALS a 10), leaf-1 = (LESS_THAN a 1)," +
+      " expr = (or (not leaf-0) leaf-1)") {
+      OrcFilters.createFilter(schema, Array(
+        Or(
+          And(
+            GreaterThan("a", 10),
+            StringContains("b", "foobar")
+          ),
+          And(
+            StringContains("b", "prefix"),
+            LessThan("a", 1)
+          )
+        )
+      )).get.toString
+    }
+
+    assert(OrcFilters.createFilter(schema, Array(
+      Or(
+        StringContains("b", "foobar"),
+        And(
+          StringContains("b", "prefix"),
+          LessThan("a", 1)
+        )
+      )
+    )).isEmpty)
+  }
+
+  test("SPARK-27160: Fix casting of the DecimalType literal") {
+    import org.apache.spark.sql.sources._
+    val schema = StructType(Array(StructField("a", DecimalType(3, 2))))
+    assertResult("leaf-0 = (LESS_THAN a 3.14), expr = leaf-0") {
+      OrcFilters.createFilter(schema, Array(
+        LessThan(
+          "a",
+          new java.math.BigDecimal(3.14, MathContext.DECIMAL64).setScale(2)))
+      ).get.toString
+    }
+  }
+
+  test("SPARK-32622: case sensitivity in predicate pushdown") {
+    withTempPath { dir =>
+      val count = 10
+      val tableName = "spark_32622"
+      val tableDir1 = dir.getAbsoluteFile + "/table1"
+
+      // Physical ORC files have both `A` and `a` fields.
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+        spark.range(count).repartition(count).selectExpr("id - 1 as A", "id as a")
+          .write.mode("overwrite").orc(tableDir1)
+      }
+
+      // Metastore table has both `A` and `a` fields too.
+      withTable(tableName) {
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+          sql(
+            s"""
+               |CREATE TABLE $tableName (A LONG, a LONG) USING ORC LOCATION '$tableDir1'
+             """.stripMargin)
+
+          checkAnswer(sql(s"select a, A from $tableName"), (0 until count).map(c => Row(c, c - 1)))
+
+          val actual1 = stripSparkFilter(sql(s"select A from $tableName where A < 0"))
+          assert(actual1.count() == 1)
+
+          val actual2 = stripSparkFilter(sql(s"select A from $tableName where a < 0"))
+          assert(actual2.count() == 0)
+        }
+
+        // Exception thrown for ambiguous case.
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+          val e = intercept[AnalysisException] {
+            sql(s"select a from $tableName where a < 0").collect()
+          }
+          assert(e.getMessage.contains(
+            "Reference 'a' is ambiguous"))
+        }
+      }
+
+      // Metastore table has only `A` field.
+      withTable(tableName) {
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+          sql(
+            s"""
+               |CREATE TABLE $tableName (A LONG) USING ORC LOCATION '$tableDir1'
+             """.stripMargin)
+
+          val e = intercept[SparkException] {
+            sql(s"select A from $tableName where A < 0").collect()
+          }
+          assert(e.getCause.isInstanceOf[RuntimeException] && e.getCause.getMessage.contains(
+            """Found duplicate field(s) "A": [A, a] in case-insensitive mode"""))
+        }
+      }
+
+      // Physical ORC files have only `A` field.
+      val tableDir2 = dir.getAbsoluteFile + "/table2"
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+        spark.range(count).repartition(count).selectExpr("id - 1 as A")
+          .write.mode("overwrite").orc(tableDir2)
+      }
+
+      withTable(tableName) {
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+          sql(
+            s"""
+               |CREATE TABLE $tableName (a LONG) USING ORC LOCATION '$tableDir2'
+             """.stripMargin)
+
+          checkAnswer(sql(s"select a from $tableName"), (0 until count).map(c => Row(c - 1)))
+
+          val actual = stripSparkFilter(sql(s"select a from $tableName where a < 0"))
+          assert(actual.count() == 1)
+        }
+      }
+
+      withTable(tableName) {
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+          sql(
+            s"""
+               |CREATE TABLE $tableName (A LONG) USING ORC LOCATION '$tableDir2'
+             """.stripMargin)
+
+          checkAnswer(sql(s"select A from $tableName"), (0 until count).map(c => Row(c - 1)))
+
+          val actual = stripSparkFilter(sql(s"select A from $tableName where A < 0"))
+          assert(actual.count() == 1)
+        }
+      }
+    }
+  }
+
+  test("SPARK-32646: Case-insensitive field resolution for pushdown when reading ORC") {
+    import org.apache.spark.sql.sources._
+
+    def getOrcFilter(
+        schema: StructType,
+        filters: Seq[Filter],
+        caseSensitive: String): Option[SearchArgument] = {
+      var orcFilter: Option[SearchArgument] = None
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive) {
+        orcFilter =
+          OrcFilters.createFilter(schema, filters)
+      }
+      orcFilter
+    }
+
+    def testFilter(
+        schema: StructType,
+        filters: Seq[Filter],
+        expected: SearchArgument): Unit = {
+      val caseSensitiveFilters = getOrcFilter(schema, filters, "true")
+      val caseInsensitiveFilters = getOrcFilter(schema, filters, "false")
+
+      assert(caseSensitiveFilters.isEmpty)
+      assert(caseInsensitiveFilters.isDefined)
+
+      assert(caseInsensitiveFilters.get.getLeaves().size() > 0)
+      assert(caseInsensitiveFilters.get.getLeaves().size() == expected.getLeaves().size())
+      (0 until expected.getLeaves().size()).foreach { index =>
+        assert(caseInsensitiveFilters.get.getLeaves().get(index) == expected.getLeaves().get(index))
+      }
+    }
+
+    val schema1 = StructType(Seq(StructField("cint", IntegerType)))
+    testFilter(schema1, Seq(GreaterThan("CINT", 1)),
+      newBuilder.startNot()
+        .lessThanEquals("cint", OrcFilters.getPredicateLeafType(IntegerType), 1L).`end`().build())
+    testFilter(schema1, Seq(
+      And(GreaterThan("CINT", 1), EqualTo("Cint", 2))),
+      newBuilder.startAnd()
+        .startNot()
+        .lessThanEquals("cint", OrcFilters.getPredicateLeafType(IntegerType), 1L).`end`()
+        .equals("cint", OrcFilters.getPredicateLeafType(IntegerType), 2L)
+        .`end`().build())
+
+    // Nested column case
+    val schema2 = StructType(Seq(StructField("a",
+      StructType(Seq(StructField("cint", IntegerType))))))
+
+    testFilter(schema2, Seq(GreaterThan("A.CINT", 1)),
+      newBuilder.startNot()
+        .lessThanEquals("a.cint", OrcFilters.getPredicateLeafType(IntegerType), 1L).`end`().build())
+    testFilter(schema2, Seq(GreaterThan("a.CINT", 1)),
+      newBuilder.startNot()
+        .lessThanEquals("a.cint", OrcFilters.getPredicateLeafType(IntegerType), 1L).`end`().build())
+    testFilter(schema2, Seq(GreaterThan("A.cint", 1)),
+      newBuilder.startNot()
+        .lessThanEquals("a.cint", OrcFilters.getPredicateLeafType(IntegerType), 1L).`end`().build())
+    testFilter(schema2, Seq(
+      And(GreaterThan("a.CINT", 1), EqualTo("a.Cint", 2))),
+      newBuilder.startAnd()
+        .startNot()
+        .lessThanEquals("a.cint", OrcFilters.getPredicateLeafType(IntegerType), 1L).`end`()
+        .equals("a.cint", OrcFilters.getPredicateLeafType(IntegerType), 2L)
+        .`end`().build())
+  }
+}
+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala
index 60f278b8e5bb0..ead2c2cf1b70f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala
@@ -34,7 +34,6 @@ import org.apache.orc.mapreduce.OrcInputFormat
 import org.apache.spark.{SparkConf, SparkException}
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.execution.adaptive.AdaptiveTestUtils.assertExceptionMessage
 import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation, RecordReaderIterator}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
@@ -137,7 +136,7 @@ abstract class OrcQueryTest extends OrcTest {
       assertResult(10) {
         sql("SELECT name, contacts FROM t where age > 5")
           .rdd
-          .flatMap(_.getAs[Seq[_]]("contacts"))
+          .flatMap(_.getAs[scala.collection.Seq[_]]("contacts"))
           .count()
       }
 
@@ -149,7 +148,7 @@ abstract class OrcQueryTest extends OrcTest {
         val df = sql("SELECT name, contacts FROM t WHERE age > 5 AND age < 8")
         assert(df.count() === 2)
         assertResult(4) {
-          df.rdd.flatMap(_.getAs[Seq[_]]("contacts")).count()
+          df.rdd.flatMap(_.getAs[scala.collection.Seq[_]]("contacts")).count()
         }
       }
 
@@ -161,7 +160,7 @@ abstract class OrcQueryTest extends OrcTest {
         val df = sql("SELECT name, contacts FROM t WHERE age < 2 OR age > 8")
         assert(df.count() === 3)
         assertResult(6) {
-          df.rdd.flatMap(_.getAs[Seq[_]]("contacts")).count()
+          df.rdd.flatMap(_.getAs[scala.collection.Seq[_]]("contacts")).count()
         }
       }
     }
@@ -218,7 +217,6 @@ abstract class OrcQueryTest extends OrcTest {
     }
   }
 
-  // Hive supports zlib, snappy and none for Hive 1.2.1.
   test("Compression options for writing to an ORC file (SNAPPY, ZLIB and NONE)") {
     withTempPath { file =>
       spark.range(0, 10).write
@@ -599,19 +597,19 @@ abstract class OrcQueryTest extends OrcTest {
       val e1 = intercept[SparkException] {
         testIgnoreCorruptFiles()
       }
-      assertExceptionMessage(e1, "Malformed ORC file")
+      assert(e1.getMessage.contains("Malformed ORC file"))
       val e2 = intercept[SparkException] {
         testIgnoreCorruptFilesWithoutSchemaInfer()
       }
-      assertExceptionMessage(e2, "Malformed ORC file")
+      assert(e2.getMessage.contains("Malformed ORC file"))
       val e3 = intercept[SparkException] {
         testAllCorruptFiles()
       }
-      assertExceptionMessage(e3, "Could not read footer for file")
+      assert(e3.getMessage.contains("Could not read footer for file"))
       val e4 = intercept[SparkException] {
         testAllCorruptFilesWithoutSchemaInfer()
       }
-      assertExceptionMessage(e4, "Malformed ORC file")
+      assert(e4.getMessage.contains("Malformed ORC file"))
     }
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
index 73873684f6aaf..eee8e2ecc9fbd 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
@@ -33,7 +33,8 @@ import org.scalatest.BeforeAndAfterAll
 
 import org.apache.spark.{SPARK_VERSION_SHORT, SparkException}
 import org.apache.spark.sql.{Row, SPARK_VERSION_METADATA_KEY}
-import org.apache.spark.sql.execution.datasources.SchemaMergeUtils
+import org.apache.spark.sql.execution.FileSourceScanExec
+import org.apache.spark.sql.execution.datasources.{CommonFileDataSourceSuite, SchemaMergeUtils}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.{LongType, StructField, StructType}
@@ -41,9 +42,11 @@ import org.apache.spark.util.Utils
 
 case class OrcData(intField: Int, stringField: String)
 
-abstract class OrcSuite extends OrcTest with BeforeAndAfterAll {
+abstract class OrcSuite extends OrcTest with BeforeAndAfterAll with CommonFileDataSourceSuite {
   import testImplicits._
 
+  override protected def dataSourceFormat = "orc"
+
   var orcTableDir: File = null
   var orcTableAsDir: File = null
 
@@ -120,8 +123,7 @@ abstract class OrcSuite extends OrcTest with BeforeAndAfterAll {
     }
   }
 
-  protected def testSelectiveDictionaryEncoding(isSelective: Boolean,
-      isHive23: Boolean = false): Unit = {
+  protected def testSelectiveDictionaryEncoding(isSelective: Boolean, isHiveOrc: Boolean): Unit = {
     val tableName = "orcTable"
 
     withTempDir { dir =>
@@ -174,7 +176,7 @@ abstract class OrcSuite extends OrcTest with BeforeAndAfterAll {
           // Hive 0.11 and RLE v2 is introduced in Hive 0.12 ORC with more improvements.
           // For more details, see https://orc.apache.org/specification/
           assert(stripe.getColumns(1).getKind === DICTIONARY_V2)
-          if (isSelective || isHive23) {
+          if (isSelective || isHiveOrc) {
             assert(stripe.getColumns(2).getKind === DIRECT_V2)
           } else {
             assert(stripe.getColumns(2).getKind === DICTIONARY_V2)
@@ -213,9 +215,7 @@ abstract class OrcSuite extends OrcTest with BeforeAndAfterAll {
           Seq(fs.listStatus(path1), fs.listStatus(path2), fs.listStatus(path3)).flatten
 
         val schema = SchemaMergeUtils.mergeSchemasInParallel(
-          spark,
-          fileStatuses,
-          schemaReader)
+          spark, Map.empty, fileStatuses, schemaReader)
 
         assert(schema.isDefined)
         assert(schema.get == StructType(Seq(
@@ -338,7 +338,7 @@ abstract class OrcSuite extends OrcTest with BeforeAndAfterAll {
     }
 
     // Test all the valid options of spark.sql.orc.compression.codec
-    Seq("NONE", "UNCOMPRESSED", "SNAPPY", "ZLIB", "LZO").foreach { c =>
+    Seq("NONE", "UNCOMPRESSED", "SNAPPY", "ZLIB", "LZO", "ZSTD").foreach { c =>
       withSQLConf(SQLConf.ORC_COMPRESSION.key -> c) {
         val expected = if (c == "UNCOMPRESSED") "NONE" else c
         assert(new OrcOptions(Map.empty[String, String], conf).compressionCodec == expected)
@@ -543,6 +543,7 @@ abstract class OrcSuite extends OrcTest with BeforeAndAfterAll {
 }
 
 class OrcSourceSuite extends OrcSuite with SharedSparkSession {
+  import testImplicits._
 
   protected override def beforeAll(): Unit = {
     super.beforeAll()
@@ -583,7 +584,7 @@ class OrcSourceSuite extends OrcSuite with SharedSparkSession {
   }
 
   test("Enforce direct encoding column-wise selectively") {
-    testSelectiveDictionaryEncoding(isSelective = true)
+    testSelectiveDictionaryEncoding(isSelective = true, isHiveOrc = false)
   }
 
   test("SPARK-11412 read and merge orc schemas in parallel") {
@@ -595,4 +596,57 @@ class OrcSourceSuite extends OrcSuite with SharedSparkSession {
     val df = readResourceOrcFile("test-data/TestStringDictionary.testRowIndex.orc")
     assert(df.where("str < 'row 001000'").count() === 1000)
   }
+
+  test("SPARK-33978: Write and read a file with ZSTD compression") {
+    withTempPath { dir =>
+      val path = dir.getAbsolutePath
+      spark.range(3).write.option("compression", "zstd").orc(path)
+      checkAnswer(spark.read.orc(path), Seq(Row(0), Row(1), Row(2)))
+      val files = OrcUtils.listOrcFiles(path, spark.sessionState.newHadoopConf())
+      assert(files.nonEmpty && files.forall(_.getName.contains("zstd")))
+    }
+  }
+
+  test("SPARK-34862: Support ORC vectorized reader for nested column") {
+    withTempPath { dir =>
+      val path = dir.getCanonicalPath
+      val df = spark.range(10).map { x =>
+        val stringColumn = s"$x" * 10
+        val structColumn = (x, s"$x" * 100)
+        val arrayColumn = (0 until 5).map(i => (x + i, s"$x" * 5))
+        val mapColumn = Map(
+          s"$x" -> (x * 0.1, (x, s"$x" * 100)),
+          (s"$x" * 2) -> (x * 0.2, (x, s"$x" * 200)),
+          (s"$x" * 3) -> (x * 0.3, (x, s"$x" * 300)))
+        (x, stringColumn, structColumn, arrayColumn, mapColumn)
+      }.toDF("int_col", "string_col", "struct_col", "array_col", "map_col")
+      df.write.format("orc").save(path)
+
+      withSQLConf(SQLConf.ORC_VECTORIZED_READER_NESTED_COLUMN_ENABLED.key -> "true") {
+        val readDf = spark.read.orc(path)
+        val vectorizationEnabled = readDf.queryExecution.executedPlan.find {
+          case scan: FileSourceScanExec => scan.supportsColumnar
+          case _ => false
+        }.isDefined
+        assert(vectorizationEnabled)
+        checkAnswer(readDf, df)
+      }
+    }
+  }
+
+  test("SPARK-34897: Support reconcile schemas based on index after nested column pruning") {
+    withTable("t1") {
+      spark.sql(
+        """
+          |CREATE TABLE t1 (
+          |  _col0 INT,
+          |  _col1 STRING,
+          |  _col2 STRUCT<c1: STRING, c2: STRING, c3: STRING, c4: BIGINT>)
+          |USING ORC
+          |""".stripMargin)
+
+      spark.sql("INSERT INTO t1 values(1, '2', struct('a', 'b', 'c', 10L))")
+      checkAnswer(spark.sql("SELECT _col0, _col2.c1 FROM t1"), Seq(Row(1, "a")))
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcTest.scala
index e929f904c798d..4243318ac1dd8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcTest.scala
@@ -46,7 +46,6 @@ import org.apache.spark.sql.internal.SQLConf.ORC_IMPLEMENTATION
  *       -> OrcPartitionDiscoverySuite
  *       -> HiveOrcPartitionDiscoverySuite
  *   -> OrcFilterSuite
- *   -> HiveOrcFilterSuite
  */
 abstract class OrcTest extends QueryTest with FileBasedDataSourceTest with BeforeAndAfterAll {
 
@@ -143,4 +142,26 @@ abstract class OrcTest extends QueryTest with FileBasedDataSourceTest with Befor
     FileUtils.copyURLToFile(url, file)
     spark.read.orc(file.getAbsolutePath)
   }
+
+  /**
+   * Takes a sequence of products `data` to generate multi-level nested
+   * dataframes as new test data. It tests both non-nested and nested dataframes
+   * which are written and read back with Orc datasource.
+   *
+   * This is different from [[withOrcDataFrame]] which does not
+   * test nested cases.
+   */
+  protected def withNestedOrcDataFrame[T <: Product: ClassTag: TypeTag](data: Seq[T])
+      (runTest: (DataFrame, String, Any => Any) => Unit): Unit =
+    withNestedOrcDataFrame(spark.createDataFrame(data))(runTest)
+
+  protected def withNestedOrcDataFrame(inputDF: DataFrame)
+      (runTest: (DataFrame, String, Any => Any) => Unit): Unit = {
+    withNestedDataFrame(inputDF).foreach { case (newDF, colName, resultFun) =>
+      withTempPath { file =>
+        newDF.write.format(dataSourceName).save(file.getCanonicalPath)
+        readFile(file.getCanonicalPath, true) { df => runTest(df, colName, resultFun) }
+      }
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV2SchemaPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV2SchemaPruningSuite.scala
index 6c9bd32913178..378b52f9c6c8c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV2SchemaPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV2SchemaPruningSuite.scala
@@ -17,7 +17,7 @@
 package org.apache.spark.sql.execution.datasources.orc
 
 import org.apache.spark.SparkConf
-import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.DataFrame
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
 import org.apache.spark.sql.execution.datasources.SchemaPruningSuite
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCommitterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCommitterSuite.scala
index 4b2437803d645..7f408dbba5099 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCommitterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCommitterSuite.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.sql.execution.datasources.parquet
 
-import java.io.FileNotFoundException
-
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.{FileStatus, Path}
 import org.apache.hadoop.mapreduce.{JobContext, TaskAttemptContext}
@@ -149,7 +147,7 @@ private object MarkingFileOutput {
    * @param outputPath destination directory
    * @param conf configuration to create the FS with
    * @return the status of the marker
-   * @throws FileNotFoundException if the marker is absent
+   * @throws java.io.FileNotFoundException if the marker is absent
    */
   def checkMarker(outputPath: Path, conf: Configuration): FileStatus = {
     outputPath.getFileSystem(conf).getFileStatus(new Path(outputPath, "marker"))
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormatSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormatSuite.scala
index e65f4d12bf7f2..c52b57eb31e4d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormatSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormatSuite.scala
@@ -19,12 +19,19 @@ package org.apache.spark.sql.execution.datasources.parquet
 
 import org.apache.hadoop.fs.{FileSystem, Path}
 
-import org.apache.spark.SparkException
+import org.apache.spark.{SparkConf, SparkException}
 import org.apache.spark.sql.QueryTest
+import org.apache.spark.sql.execution.datasources.CommonFileDataSourceSuite
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 
-class ParquetFileFormatSuite extends QueryTest with ParquetTest with SharedSparkSession {
+abstract class ParquetFileFormatSuite
+  extends QueryTest
+  with ParquetTest
+  with SharedSparkSession
+  with CommonFileDataSourceSuite {
+
+  override protected def dataSourceFormat = "parquet"
 
   test("read parquet footers in parallel") {
     def testReadFooters(ignoreCorruptFiles: Boolean): Unit = {
@@ -57,3 +64,17 @@ class ParquetFileFormatSuite extends QueryTest with ParquetTest with SharedSpark
     assert(exception.getMessage().contains("Could not read footer for file"))
   }
 }
+
+class ParquetFileFormatV1Suite extends ParquetFileFormatSuite {
+  override protected def sparkConf: SparkConf =
+    super
+      .sparkConf
+      .set(SQLConf.USE_V1_SOURCE_LIST, "parquet")
+}
+
+class ParquetFileFormatV2Suite extends ParquetFileFormatSuite {
+  override protected def sparkConf: SparkConf =
+    super
+      .sparkConf
+      .set(SQLConf.USE_V1_SOURCE_LIST, "")
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
index 5cf21293fd07f..94bda56bc8738 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
@@ -20,11 +20,17 @@ package org.apache.spark.sql.execution.datasources.parquet
 import java.math.{BigDecimal => JBigDecimal}
 import java.nio.charset.StandardCharsets
 import java.sql.{Date, Timestamp}
-import java.time.LocalDate
+import java.time.{LocalDate, LocalDateTime, ZoneId}
 
+import scala.reflect.ClassTag
+import scala.reflect.runtime.universe.TypeTag
+
+import org.apache.hadoop.fs.Path
 import org.apache.parquet.filter2.predicate.{FilterApi, FilterPredicate, Operators}
 import org.apache.parquet.filter2.predicate.FilterApi._
 import org.apache.parquet.filter2.predicate.Operators.{Column => _, _}
+import org.apache.parquet.hadoop.{ParquetFileReader, ParquetInputFormat, ParquetOutputFormat}
+import org.apache.parquet.hadoop.util.HadoopInputFile
 import org.apache.parquet.schema.MessageType
 
 import org.apache.spark.{SparkConf, SparkException}
@@ -42,7 +48,8 @@ import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.ParquetOutputTimestampType
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types._
-import org.apache.spark.util.{AccumulatorContext, AccumulatorV2}
+import org.apache.spark.tags.ExtendedSQLTest
+import org.apache.spark.util.{AccumulatorContext, AccumulatorV2, Utils}
 
 /**
  * A test suite that tests Parquet filter2 API based filter pushdown optimization.
@@ -106,44 +113,31 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
   }
 
   /**
-   * Takes single level `inputDF` dataframe to generate multi-level nested
-   * dataframes as new test data.
+   * Takes a sequence of products `data` to generate multi-level nested
+   * dataframes as new test data. It tests both non-nested and nested dataframes
+   * which are written and read back with Parquet datasource.
+   *
+   * This is different from [[ParquetTest.withParquetDataFrame]] which does not
+   * test nested cases.
    */
-  private def withNestedDataFrame(inputDF: DataFrame)
+  private def withNestedParquetDataFrame[T <: Product: ClassTag: TypeTag](data: Seq[T])
+      (runTest: (DataFrame, String, Any => Any) => Unit): Unit =
+    withNestedParquetDataFrame(spark.createDataFrame(data))(runTest)
+
+  private def withNestedParquetDataFrame(inputDF: DataFrame)
       (runTest: (DataFrame, String, Any => Any) => Unit): Unit = {
-    assert(inputDF.schema.fields.length == 1)
-    assert(!inputDF.schema.fields.head.dataType.isInstanceOf[StructType])
-    val df = inputDF.toDF("temp")
-    Seq(
-      (
-        df.withColumnRenamed("temp", "a"),
-        "a", // zero nesting
-        (x: Any) => x),
-      (
-        df.withColumn("a", struct(df("temp") as "b")).drop("temp"),
-        "a.b", // one level nesting
-        (x: Any) => Row(x)),
-      (
-        df.withColumn("a", struct(struct(df("temp") as "c") as "b")).drop("temp"),
-        "a.b.c", // two level nesting
-        (x: Any) => Row(Row(x))
-      ),
-      (
-        df.withColumnRenamed("temp", "a.b"),
-        "`a.b`", // zero nesting with column name containing `dots`
-        (x: Any) => x
-      ),
-      (
-        df.withColumn("a.b", struct(df("temp") as "c.d") ).drop("temp"),
-        "`a.b`.`c.d`", // one level nesting with column names containing `dots`
-        (x: Any) => Row(x)
-      )
-    ).foreach { case (df, colName, resultFun) =>
-      runTest(df, colName, resultFun)
+    withNestedDataFrame(inputDF).foreach { case (newDF, colName, resultFun) =>
+      withTempPath { file =>
+        newDF.write.format(dataSourceName).save(file.getCanonicalPath)
+        readParquetFile(file.getCanonicalPath) { df => runTest(df, colName, resultFun) }
+      }
     }
   }
 
-  private def testTimestampPushdown(data: Seq[Timestamp]): Unit = {
+  private def testTimestampPushdown(data: Seq[String], java8Api: Boolean): Unit = {
+    implicit class StringToTs(s: String) {
+      def ts: Timestamp = Timestamp.valueOf(s)
+    }
     assert(data.size === 4)
     val ts1 = data.head
     val ts2 = data(1)
@@ -151,37 +145,49 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
     val ts4 = data(3)
 
     import testImplicits._
-    withNestedDataFrame(data.map(i => Tuple1(i)).toDF()) { case (inputDF, colName, resultFun) =>
-      withParquetDataFrame(inputDF) { implicit df =>
-        val tsAttr = df(colName).expr
-        assert(df(colName).expr.dataType === TimestampType)
-
-        checkFilterPredicate(tsAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
-        checkFilterPredicate(tsAttr.isNotNull, classOf[NotEq[_]],
-          data.map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(tsAttr === ts1, classOf[Eq[_]], resultFun(ts1))
-        checkFilterPredicate(tsAttr <=> ts1, classOf[Eq[_]], resultFun(ts1))
-        checkFilterPredicate(tsAttr =!= ts1, classOf[NotEq[_]],
-          Seq(ts2, ts3, ts4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(tsAttr < ts2, classOf[Lt[_]], resultFun(ts1))
-        checkFilterPredicate(tsAttr > ts1, classOf[Gt[_]],
-          Seq(ts2, ts3, ts4).map(i => Row.apply(resultFun(i))))
-        checkFilterPredicate(tsAttr <= ts1, classOf[LtEq[_]], resultFun(ts1))
-        checkFilterPredicate(tsAttr >= ts4, classOf[GtEq[_]], resultFun(ts4))
-
-        checkFilterPredicate(Literal(ts1) === tsAttr, classOf[Eq[_]], resultFun(ts1))
-        checkFilterPredicate(Literal(ts1) <=> tsAttr, classOf[Eq[_]], resultFun(ts1))
-        checkFilterPredicate(Literal(ts2) > tsAttr, classOf[Lt[_]], resultFun(ts1))
-        checkFilterPredicate(Literal(ts3) < tsAttr, classOf[Gt[_]], resultFun(ts4))
-        checkFilterPredicate(Literal(ts1) >= tsAttr, classOf[LtEq[_]], resultFun(ts1))
-        checkFilterPredicate(Literal(ts4) <= tsAttr, classOf[GtEq[_]], resultFun(ts4))
-
-        checkFilterPredicate(!(tsAttr < ts4), classOf[GtEq[_]], resultFun(ts4))
-        checkFilterPredicate(tsAttr < ts2 || tsAttr > ts3, classOf[Operators.Or],
-          Seq(Row(resultFun(ts1)), Row(resultFun(ts4))))
+    val df = data.map(i => Tuple1(Timestamp.valueOf(i))).toDF()
+    withNestedParquetDataFrame(df) { case (parquetDF, colName, fun) =>
+      implicit val df: DataFrame = parquetDF
+
+      def resultFun(tsStr: String): Any = {
+        val parsed = if (java8Api) {
+          LocalDateTime.parse(tsStr.replace(" ", "T"))
+            .atZone(ZoneId.systemDefault())
+            .toInstant
+        } else {
+          Timestamp.valueOf(tsStr)
+        }
+        fun(parsed)
       }
+
+      val tsAttr = df(colName).expr
+      assert(df(colName).expr.dataType === TimestampType)
+
+      checkFilterPredicate(tsAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(tsAttr.isNotNull, classOf[NotEq[_]],
+        data.map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(tsAttr === ts1.ts, classOf[Eq[_]], resultFun(ts1))
+      checkFilterPredicate(tsAttr <=> ts1.ts, classOf[Eq[_]], resultFun(ts1))
+      checkFilterPredicate(tsAttr =!= ts1.ts, classOf[NotEq[_]],
+        Seq(ts2, ts3, ts4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(tsAttr < ts2.ts, classOf[Lt[_]], resultFun(ts1))
+      checkFilterPredicate(tsAttr > ts1.ts, classOf[Gt[_]],
+        Seq(ts2, ts3, ts4).map(i => Row.apply(resultFun(i))))
+      checkFilterPredicate(tsAttr <= ts1.ts, classOf[LtEq[_]], resultFun(ts1))
+      checkFilterPredicate(tsAttr >= ts4.ts, classOf[GtEq[_]], resultFun(ts4))
+
+      checkFilterPredicate(Literal(ts1.ts) === tsAttr, classOf[Eq[_]], resultFun(ts1))
+      checkFilterPredicate(Literal(ts1.ts) <=> tsAttr, classOf[Eq[_]], resultFun(ts1))
+      checkFilterPredicate(Literal(ts2.ts) > tsAttr, classOf[Lt[_]], resultFun(ts1))
+      checkFilterPredicate(Literal(ts3.ts) < tsAttr, classOf[Gt[_]], resultFun(ts4))
+      checkFilterPredicate(Literal(ts1.ts) >= tsAttr, classOf[LtEq[_]], resultFun(ts1))
+      checkFilterPredicate(Literal(ts4.ts) <= tsAttr, classOf[GtEq[_]], resultFun(ts4))
+
+      checkFilterPredicate(!(tsAttr < ts4.ts), classOf[GtEq[_]], resultFun(ts4))
+      checkFilterPredicate(tsAttr < ts2.ts || tsAttr > ts3.ts, classOf[Operators.Or],
+        Seq(Row(resultFun(ts1)), Row(resultFun(ts4))))
     }
   }
 
@@ -212,272 +218,264 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
 
   test("filter pushdown - boolean") {
     val data = (true :: false :: Nil).map(b => Tuple1.apply(Option(b)))
-    import testImplicits._
-    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
-      withParquetDataFrame(inputDF) { implicit df =>
-        val booleanAttr = df(colName).expr
-        assert(df(colName).expr.dataType === BooleanType)
-
-        checkFilterPredicate(booleanAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
-        checkFilterPredicate(booleanAttr.isNotNull, classOf[NotEq[_]],
-          Seq(Row(resultFun(true)), Row(resultFun(false))))
-
-        checkFilterPredicate(booleanAttr === true, classOf[Eq[_]], resultFun(true))
-        checkFilterPredicate(booleanAttr <=> true, classOf[Eq[_]], resultFun(true))
-        checkFilterPredicate(booleanAttr =!= true, classOf[NotEq[_]], resultFun(false))
-      }
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val booleanAttr = df(colName).expr
+      assert(df(colName).expr.dataType === BooleanType)
+
+      checkFilterPredicate(booleanAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(booleanAttr.isNotNull, classOf[NotEq[_]],
+        Seq(Row(resultFun(true)), Row(resultFun(false))))
+
+      checkFilterPredicate(booleanAttr === true, classOf[Eq[_]], resultFun(true))
+      checkFilterPredicate(booleanAttr <=> true, classOf[Eq[_]], resultFun(true))
+      checkFilterPredicate(booleanAttr =!= true, classOf[NotEq[_]], resultFun(false))
     }
   }
 
   test("filter pushdown - tinyint") {
     val data = (1 to 4).map(i => Tuple1(Option(i.toByte)))
-    import testImplicits._
-    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
-      withParquetDataFrame(inputDF) { implicit df =>
-        val tinyIntAttr = df(colName).expr
-        assert(df(colName).expr.dataType === ByteType)
-
-        checkFilterPredicate(tinyIntAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
-        checkFilterPredicate(tinyIntAttr.isNotNull, classOf[NotEq[_]],
-          (1 to 4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(tinyIntAttr === 1.toByte, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(tinyIntAttr <=> 1.toByte, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(tinyIntAttr =!= 1.toByte, classOf[NotEq[_]],
-          (2 to 4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(tinyIntAttr < 2.toByte, classOf[Lt[_]], resultFun(1))
-        checkFilterPredicate(tinyIntAttr > 3.toByte, classOf[Gt[_]], resultFun(4))
-        checkFilterPredicate(tinyIntAttr <= 1.toByte, classOf[LtEq[_]], resultFun(1))
-        checkFilterPredicate(tinyIntAttr >= 4.toByte, classOf[GtEq[_]], resultFun(4))
-
-        checkFilterPredicate(Literal(1.toByte) === tinyIntAttr, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(Literal(1.toByte) <=> tinyIntAttr, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(Literal(2.toByte) > tinyIntAttr, classOf[Lt[_]], resultFun(1))
-        checkFilterPredicate(Literal(3.toByte) < tinyIntAttr, classOf[Gt[_]], resultFun(4))
-        checkFilterPredicate(Literal(1.toByte) >= tinyIntAttr, classOf[LtEq[_]], resultFun(1))
-        checkFilterPredicate(Literal(4.toByte) <= tinyIntAttr, classOf[GtEq[_]], resultFun(4))
-
-        checkFilterPredicate(!(tinyIntAttr < 4.toByte), classOf[GtEq[_]], resultFun(4))
-        checkFilterPredicate(tinyIntAttr < 2.toByte || tinyIntAttr > 3.toByte,
-          classOf[Operators.Or], Seq(Row(resultFun(1)), Row(resultFun(4))))
-      }
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val tinyIntAttr = df(colName).expr
+      assert(df(colName).expr.dataType === ByteType)
+
+      checkFilterPredicate(tinyIntAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(tinyIntAttr.isNotNull, classOf[NotEq[_]],
+        (1 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(tinyIntAttr === 1.toByte, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(tinyIntAttr <=> 1.toByte, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(tinyIntAttr =!= 1.toByte, classOf[NotEq[_]],
+        (2 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(tinyIntAttr < 2.toByte, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(tinyIntAttr > 3.toByte, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(tinyIntAttr <= 1.toByte, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(tinyIntAttr >= 4.toByte, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(Literal(1.toByte) === tinyIntAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(1.toByte) <=> tinyIntAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(2.toByte) > tinyIntAttr, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(Literal(3.toByte) < tinyIntAttr, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(Literal(1.toByte) >= tinyIntAttr, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(Literal(4.toByte) <= tinyIntAttr, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(!(tinyIntAttr < 4.toByte), classOf[GtEq[_]], resultFun(4))
+      checkFilterPredicate(tinyIntAttr < 2.toByte || tinyIntAttr > 3.toByte,
+        classOf[Operators.Or], Seq(Row(resultFun(1)), Row(resultFun(4))))
     }
   }
 
   test("filter pushdown - smallint") {
     val data = (1 to 4).map(i => Tuple1(Option(i.toShort)))
-    import testImplicits._
-    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
-      withParquetDataFrame(inputDF) { implicit df =>
-        val smallIntAttr = df(colName).expr
-        assert(df(colName).expr.dataType === ShortType)
-
-        checkFilterPredicate(smallIntAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
-        checkFilterPredicate(smallIntAttr.isNotNull, classOf[NotEq[_]],
-          (1 to 4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(smallIntAttr === 1.toShort, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(smallIntAttr <=> 1.toShort, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(smallIntAttr =!= 1.toShort, classOf[NotEq[_]],
-          (2 to 4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(smallIntAttr < 2.toShort, classOf[Lt[_]], resultFun(1))
-        checkFilterPredicate(smallIntAttr > 3.toShort, classOf[Gt[_]], resultFun(4))
-        checkFilterPredicate(smallIntAttr <= 1.toShort, classOf[LtEq[_]], resultFun(1))
-        checkFilterPredicate(smallIntAttr >= 4.toShort, classOf[GtEq[_]], resultFun(4))
-
-        checkFilterPredicate(Literal(1.toShort) === smallIntAttr, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(Literal(1.toShort) <=> smallIntAttr, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(Literal(2.toShort) > smallIntAttr, classOf[Lt[_]], resultFun(1))
-        checkFilterPredicate(Literal(3.toShort) < smallIntAttr, classOf[Gt[_]], resultFun(4))
-        checkFilterPredicate(Literal(1.toShort) >= smallIntAttr, classOf[LtEq[_]], resultFun(1))
-        checkFilterPredicate(Literal(4.toShort) <= smallIntAttr, classOf[GtEq[_]], resultFun(4))
-
-        checkFilterPredicate(!(smallIntAttr < 4.toShort), classOf[GtEq[_]], resultFun(4))
-        checkFilterPredicate(smallIntAttr < 2.toShort || smallIntAttr > 3.toShort,
-          classOf[Operators.Or], Seq(Row(resultFun(1)), Row(resultFun(4))))
-      }
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val smallIntAttr = df(colName).expr
+      assert(df(colName).expr.dataType === ShortType)
+
+      checkFilterPredicate(smallIntAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(smallIntAttr.isNotNull, classOf[NotEq[_]],
+        (1 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(smallIntAttr === 1.toShort, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(smallIntAttr <=> 1.toShort, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(smallIntAttr =!= 1.toShort, classOf[NotEq[_]],
+        (2 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(smallIntAttr < 2.toShort, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(smallIntAttr > 3.toShort, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(smallIntAttr <= 1.toShort, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(smallIntAttr >= 4.toShort, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(Literal(1.toShort) === smallIntAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(1.toShort) <=> smallIntAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(2.toShort) > smallIntAttr, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(Literal(3.toShort) < smallIntAttr, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(Literal(1.toShort) >= smallIntAttr, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(Literal(4.toShort) <= smallIntAttr, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(!(smallIntAttr < 4.toShort), classOf[GtEq[_]], resultFun(4))
+      checkFilterPredicate(smallIntAttr < 2.toShort || smallIntAttr > 3.toShort,
+        classOf[Operators.Or], Seq(Row(resultFun(1)), Row(resultFun(4))))
     }
   }
 
   test("filter pushdown - integer") {
     val data = (1 to 4).map(i => Tuple1(Option(i)))
-    import testImplicits._
-    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
-      withParquetDataFrame(inputDF) { implicit df =>
-        val intAttr = df(colName).expr
-        assert(df(colName).expr.dataType === IntegerType)
-
-        checkFilterPredicate(intAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
-        checkFilterPredicate(intAttr.isNotNull, classOf[NotEq[_]],
-          (1 to 4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(intAttr === 1, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(intAttr <=> 1, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(intAttr =!= 1, classOf[NotEq[_]],
-          (2 to 4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(intAttr < 2, classOf[Lt[_]], resultFun(1))
-        checkFilterPredicate(intAttr > 3, classOf[Gt[_]], resultFun(4))
-        checkFilterPredicate(intAttr <= 1, classOf[LtEq[_]], resultFun(1))
-        checkFilterPredicate(intAttr >= 4, classOf[GtEq[_]], resultFun(4))
-
-        checkFilterPredicate(Literal(1) === intAttr, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(Literal(1) <=> intAttr, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(Literal(2) > intAttr, classOf[Lt[_]], resultFun(1))
-        checkFilterPredicate(Literal(3) < intAttr, classOf[Gt[_]], resultFun(4))
-        checkFilterPredicate(Literal(1) >= intAttr, classOf[LtEq[_]], resultFun(1))
-        checkFilterPredicate(Literal(4) <= intAttr, classOf[GtEq[_]], resultFun(4))
-
-        checkFilterPredicate(!(intAttr < 4), classOf[GtEq[_]], resultFun(4))
-        checkFilterPredicate(intAttr < 2 || intAttr > 3, classOf[Operators.Or],
-          Seq(Row(resultFun(1)), Row(resultFun(4))))
-      }
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val intAttr = df(colName).expr
+      assert(df(colName).expr.dataType === IntegerType)
+
+      checkFilterPredicate(intAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(intAttr.isNotNull, classOf[NotEq[_]],
+        (1 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(intAttr === 1, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(intAttr <=> 1, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(intAttr =!= 1, classOf[NotEq[_]],
+        (2 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(intAttr < 2, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(intAttr > 3, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(intAttr <= 1, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(intAttr >= 4, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(Literal(1) === intAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(1) <=> intAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(2) > intAttr, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(Literal(3) < intAttr, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(Literal(1) >= intAttr, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(Literal(4) <= intAttr, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(!(intAttr < 4), classOf[GtEq[_]], resultFun(4))
+      checkFilterPredicate(intAttr < 2 || intAttr > 3, classOf[Operators.Or],
+        Seq(Row(resultFun(1)), Row(resultFun(4))))
     }
   }
 
   test("filter pushdown - long") {
     val data = (1 to 4).map(i => Tuple1(Option(i.toLong)))
-    import testImplicits._
-    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
-      withParquetDataFrame(inputDF) { implicit df =>
-        val longAttr = df(colName).expr
-        assert(df(colName).expr.dataType === LongType)
-
-        checkFilterPredicate(longAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
-        checkFilterPredicate(longAttr.isNotNull, classOf[NotEq[_]],
-          (1 to 4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(longAttr === 1, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(longAttr <=> 1, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(longAttr =!= 1, classOf[NotEq[_]],
-          (2 to 4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(longAttr < 2, classOf[Lt[_]], resultFun(1))
-        checkFilterPredicate(longAttr > 3, classOf[Gt[_]], resultFun(4))
-        checkFilterPredicate(longAttr <= 1, classOf[LtEq[_]], resultFun(1))
-        checkFilterPredicate(longAttr >= 4, classOf[GtEq[_]], resultFun(4))
-
-        checkFilterPredicate(Literal(1) === longAttr, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(Literal(1) <=> longAttr, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(Literal(2) > longAttr, classOf[Lt[_]], resultFun(1))
-        checkFilterPredicate(Literal(3) < longAttr, classOf[Gt[_]], resultFun(4))
-        checkFilterPredicate(Literal(1) >= longAttr, classOf[LtEq[_]], resultFun(1))
-        checkFilterPredicate(Literal(4) <= longAttr, classOf[GtEq[_]], resultFun(4))
-
-        checkFilterPredicate(!(longAttr < 4), classOf[GtEq[_]], resultFun(4))
-        checkFilterPredicate(longAttr < 2 || longAttr > 3, classOf[Operators.Or],
-          Seq(Row(resultFun(1)), Row(resultFun(4))))
-      }
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val longAttr = df(colName).expr
+      assert(df(colName).expr.dataType === LongType)
+
+      checkFilterPredicate(longAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(longAttr.isNotNull, classOf[NotEq[_]],
+        (1 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(longAttr === 1, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(longAttr <=> 1, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(longAttr =!= 1, classOf[NotEq[_]],
+        (2 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(longAttr < 2, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(longAttr > 3, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(longAttr <= 1, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(longAttr >= 4, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(Literal(1) === longAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(1) <=> longAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(2) > longAttr, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(Literal(3) < longAttr, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(Literal(1) >= longAttr, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(Literal(4) <= longAttr, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(!(longAttr < 4), classOf[GtEq[_]], resultFun(4))
+      checkFilterPredicate(longAttr < 2 || longAttr > 3, classOf[Operators.Or],
+        Seq(Row(resultFun(1)), Row(resultFun(4))))
     }
   }
 
   test("filter pushdown - float") {
     val data = (1 to 4).map(i => Tuple1(Option(i.toFloat)))
-    import testImplicits._
-    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
-      withParquetDataFrame(inputDF) { implicit df =>
-        val floatAttr = df(colName).expr
-        assert(df(colName).expr.dataType === FloatType)
-
-        checkFilterPredicate(floatAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
-        checkFilterPredicate(floatAttr.isNotNull, classOf[NotEq[_]],
-          (1 to 4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(floatAttr === 1, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(floatAttr <=> 1, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(floatAttr =!= 1, classOf[NotEq[_]],
-          (2 to 4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(floatAttr < 2, classOf[Lt[_]], resultFun(1))
-        checkFilterPredicate(floatAttr > 3, classOf[Gt[_]], resultFun(4))
-        checkFilterPredicate(floatAttr <= 1, classOf[LtEq[_]], resultFun(1))
-        checkFilterPredicate(floatAttr >= 4, classOf[GtEq[_]], resultFun(4))
-
-        checkFilterPredicate(Literal(1) === floatAttr, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(Literal(1) <=> floatAttr, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(Literal(2) > floatAttr, classOf[Lt[_]], resultFun(1))
-        checkFilterPredicate(Literal(3) < floatAttr, classOf[Gt[_]], resultFun(4))
-        checkFilterPredicate(Literal(1) >= floatAttr, classOf[LtEq[_]], resultFun(1))
-        checkFilterPredicate(Literal(4) <= floatAttr, classOf[GtEq[_]], resultFun(4))
-
-        checkFilterPredicate(!(floatAttr < 4), classOf[GtEq[_]], resultFun(4))
-        checkFilterPredicate(floatAttr < 2 || floatAttr > 3, classOf[Operators.Or],
-          Seq(Row(resultFun(1)), Row(resultFun(4))))
-      }
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val floatAttr = df(colName).expr
+      assert(df(colName).expr.dataType === FloatType)
+
+      checkFilterPredicate(floatAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(floatAttr.isNotNull, classOf[NotEq[_]],
+        (1 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(floatAttr === 1, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(floatAttr <=> 1, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(floatAttr =!= 1, classOf[NotEq[_]],
+        (2 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(floatAttr < 2, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(floatAttr > 3, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(floatAttr <= 1, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(floatAttr >= 4, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(Literal(1) === floatAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(1) <=> floatAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(2) > floatAttr, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(Literal(3) < floatAttr, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(Literal(1) >= floatAttr, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(Literal(4) <= floatAttr, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(!(floatAttr < 4), classOf[GtEq[_]], resultFun(4))
+      checkFilterPredicate(floatAttr < 2 || floatAttr > 3, classOf[Operators.Or],
+        Seq(Row(resultFun(1)), Row(resultFun(4))))
     }
   }
 
   test("filter pushdown - double") {
     val data = (1 to 4).map(i => Tuple1(Option(i.toDouble)))
-    import testImplicits._
-    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
-      withParquetDataFrame(inputDF) { implicit df =>
-        val doubleAttr = df(colName).expr
-        assert(df(colName).expr.dataType === DoubleType)
-
-        checkFilterPredicate(doubleAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
-        checkFilterPredicate(doubleAttr.isNotNull, classOf[NotEq[_]],
-          (1 to 4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(doubleAttr === 1, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(doubleAttr <=> 1, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(doubleAttr =!= 1, classOf[NotEq[_]],
-          (2 to 4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(doubleAttr < 2, classOf[Lt[_]], resultFun(1))
-        checkFilterPredicate(doubleAttr > 3, classOf[Gt[_]], resultFun(4))
-        checkFilterPredicate(doubleAttr <= 1, classOf[LtEq[_]], resultFun(1))
-        checkFilterPredicate(doubleAttr >= 4, classOf[GtEq[_]], resultFun(4))
-
-        checkFilterPredicate(Literal(1) === doubleAttr, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(Literal(1) <=> doubleAttr, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(Literal(2) > doubleAttr, classOf[Lt[_]], resultFun(1))
-        checkFilterPredicate(Literal(3) < doubleAttr, classOf[Gt[_]], resultFun(4))
-        checkFilterPredicate(Literal(1) >= doubleAttr, classOf[LtEq[_]], resultFun(1))
-        checkFilterPredicate(Literal(4) <= doubleAttr, classOf[GtEq[_]], resultFun(4))
-
-        checkFilterPredicate(!(doubleAttr < 4), classOf[GtEq[_]], resultFun(4))
-        checkFilterPredicate(doubleAttr < 2 || doubleAttr > 3, classOf[Operators.Or],
-          Seq(Row(resultFun(1)), Row(resultFun(4))))
-      }
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val doubleAttr = df(colName).expr
+      assert(df(colName).expr.dataType === DoubleType)
+
+      checkFilterPredicate(doubleAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(doubleAttr.isNotNull, classOf[NotEq[_]],
+        (1 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(doubleAttr === 1, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(doubleAttr <=> 1, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(doubleAttr =!= 1, classOf[NotEq[_]],
+        (2 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(doubleAttr < 2, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(doubleAttr > 3, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(doubleAttr <= 1, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(doubleAttr >= 4, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(Literal(1) === doubleAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(1) <=> doubleAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(2) > doubleAttr, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(Literal(3) < doubleAttr, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(Literal(1) >= doubleAttr, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(Literal(4) <= doubleAttr, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(!(doubleAttr < 4), classOf[GtEq[_]], resultFun(4))
+      checkFilterPredicate(doubleAttr < 2 || doubleAttr > 3, classOf[Operators.Or],
+        Seq(Row(resultFun(1)), Row(resultFun(4))))
     }
   }
 
   test("filter pushdown - string") {
     val data = (1 to 4).map(i => Tuple1(Option(i.toString)))
-    import testImplicits._
-    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
-      withParquetDataFrame(inputDF) { implicit df =>
-        val stringAttr = df(colName).expr
-        assert(df(colName).expr.dataType === StringType)
-
-        checkFilterPredicate(stringAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
-        checkFilterPredicate(stringAttr.isNotNull, classOf[NotEq[_]],
-          (1 to 4).map(i => Row.apply(resultFun(i.toString))))
-
-        checkFilterPredicate(stringAttr === "1", classOf[Eq[_]], resultFun("1"))
-        checkFilterPredicate(stringAttr <=> "1", classOf[Eq[_]], resultFun("1"))
-        checkFilterPredicate(stringAttr =!= "1", classOf[NotEq[_]],
-          (2 to 4).map(i => Row.apply(resultFun(i.toString))))
-
-        checkFilterPredicate(stringAttr < "2", classOf[Lt[_]], resultFun("1"))
-        checkFilterPredicate(stringAttr > "3", classOf[Gt[_]], resultFun("4"))
-        checkFilterPredicate(stringAttr <= "1", classOf[LtEq[_]], resultFun("1"))
-        checkFilterPredicate(stringAttr >= "4", classOf[GtEq[_]], resultFun("4"))
-
-        checkFilterPredicate(Literal("1") === stringAttr, classOf[Eq[_]], resultFun("1"))
-        checkFilterPredicate(Literal("1") <=> stringAttr, classOf[Eq[_]], resultFun("1"))
-        checkFilterPredicate(Literal("2") > stringAttr, classOf[Lt[_]], resultFun("1"))
-        checkFilterPredicate(Literal("3") < stringAttr, classOf[Gt[_]], resultFun("4"))
-        checkFilterPredicate(Literal("1") >= stringAttr, classOf[LtEq[_]], resultFun("1"))
-        checkFilterPredicate(Literal("4") <= stringAttr, classOf[GtEq[_]], resultFun("4"))
-
-        checkFilterPredicate(!(stringAttr < "4"), classOf[GtEq[_]], resultFun("4"))
-        checkFilterPredicate(stringAttr < "2" || stringAttr > "3", classOf[Operators.Or],
-          Seq(Row(resultFun("1")), Row(resultFun("4"))))
-      }
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val stringAttr = df(colName).expr
+      assert(df(colName).expr.dataType === StringType)
+
+      checkFilterPredicate(stringAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(stringAttr.isNotNull, classOf[NotEq[_]],
+        (1 to 4).map(i => Row.apply(resultFun(i.toString))))
+
+      checkFilterPredicate(stringAttr === "1", classOf[Eq[_]], resultFun("1"))
+      checkFilterPredicate(stringAttr <=> "1", classOf[Eq[_]], resultFun("1"))
+      checkFilterPredicate(stringAttr =!= "1", classOf[NotEq[_]],
+        (2 to 4).map(i => Row.apply(resultFun(i.toString))))
+
+      checkFilterPredicate(stringAttr < "2", classOf[Lt[_]], resultFun("1"))
+      checkFilterPredicate(stringAttr > "3", classOf[Gt[_]], resultFun("4"))
+      checkFilterPredicate(stringAttr <= "1", classOf[LtEq[_]], resultFun("1"))
+      checkFilterPredicate(stringAttr >= "4", classOf[GtEq[_]], resultFun("4"))
+
+      checkFilterPredicate(Literal("1") === stringAttr, classOf[Eq[_]], resultFun("1"))
+      checkFilterPredicate(Literal("1") <=> stringAttr, classOf[Eq[_]], resultFun("1"))
+      checkFilterPredicate(Literal("2") > stringAttr, classOf[Lt[_]], resultFun("1"))
+      checkFilterPredicate(Literal("3") < stringAttr, classOf[Gt[_]], resultFun("4"))
+      checkFilterPredicate(Literal("1") >= stringAttr, classOf[LtEq[_]], resultFun("1"))
+      checkFilterPredicate(Literal("4") <= stringAttr, classOf[GtEq[_]], resultFun("4"))
+
+      checkFilterPredicate(!(stringAttr < "4"), classOf[GtEq[_]], resultFun("4"))
+      checkFilterPredicate(stringAttr < "2" || stringAttr > "3", classOf[Operators.Or],
+        Seq(Row(resultFun("1")), Row(resultFun("4"))))
     }
   }
 
@@ -487,38 +485,37 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
     }
 
     val data = (1 to 4).map(i => Tuple1(Option(i.b)))
-    import testImplicits._
-    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
-      withParquetDataFrame(inputDF) { implicit df =>
-        val binaryAttr: Expression = df(colName).expr
-        assert(df(colName).expr.dataType === BinaryType)
-
-        checkFilterPredicate(binaryAttr === 1.b, classOf[Eq[_]], resultFun(1.b))
-        checkFilterPredicate(binaryAttr <=> 1.b, classOf[Eq[_]], resultFun(1.b))
-
-        checkFilterPredicate(binaryAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
-        checkFilterPredicate(binaryAttr.isNotNull, classOf[NotEq[_]],
-          (1 to 4).map(i => Row.apply(resultFun(i.b))))
-
-        checkFilterPredicate(binaryAttr =!= 1.b, classOf[NotEq[_]],
-          (2 to 4).map(i => Row.apply(resultFun(i.b))))
-
-        checkFilterPredicate(binaryAttr < 2.b, classOf[Lt[_]], resultFun(1.b))
-        checkFilterPredicate(binaryAttr > 3.b, classOf[Gt[_]], resultFun(4.b))
-        checkFilterPredicate(binaryAttr <= 1.b, classOf[LtEq[_]], resultFun(1.b))
-        checkFilterPredicate(binaryAttr >= 4.b, classOf[GtEq[_]], resultFun(4.b))
-
-        checkFilterPredicate(Literal(1.b) === binaryAttr, classOf[Eq[_]], resultFun(1.b))
-        checkFilterPredicate(Literal(1.b) <=> binaryAttr, classOf[Eq[_]], resultFun(1.b))
-        checkFilterPredicate(Literal(2.b) > binaryAttr, classOf[Lt[_]], resultFun(1.b))
-        checkFilterPredicate(Literal(3.b) < binaryAttr, classOf[Gt[_]], resultFun(4.b))
-        checkFilterPredicate(Literal(1.b) >= binaryAttr, classOf[LtEq[_]], resultFun(1.b))
-        checkFilterPredicate(Literal(4.b) <= binaryAttr, classOf[GtEq[_]], resultFun(4.b))
-
-        checkFilterPredicate(!(binaryAttr < 4.b), classOf[GtEq[_]], resultFun(4.b))
-        checkFilterPredicate(binaryAttr < 2.b || binaryAttr > 3.b, classOf[Operators.Or],
-          Seq(Row(resultFun(1.b)), Row(resultFun(4.b))))
-      }
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val binaryAttr: Expression = df(colName).expr
+      assert(df(colName).expr.dataType === BinaryType)
+
+      checkFilterPredicate(binaryAttr === 1.b, classOf[Eq[_]], resultFun(1.b))
+      checkFilterPredicate(binaryAttr <=> 1.b, classOf[Eq[_]], resultFun(1.b))
+
+      checkFilterPredicate(binaryAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(binaryAttr.isNotNull, classOf[NotEq[_]],
+        (1 to 4).map(i => Row.apply(resultFun(i.b))))
+
+      checkFilterPredicate(binaryAttr =!= 1.b, classOf[NotEq[_]],
+        (2 to 4).map(i => Row.apply(resultFun(i.b))))
+
+      checkFilterPredicate(binaryAttr < 2.b, classOf[Lt[_]], resultFun(1.b))
+      checkFilterPredicate(binaryAttr > 3.b, classOf[Gt[_]], resultFun(4.b))
+      checkFilterPredicate(binaryAttr <= 1.b, classOf[LtEq[_]], resultFun(1.b))
+      checkFilterPredicate(binaryAttr >= 4.b, classOf[GtEq[_]], resultFun(4.b))
+
+      checkFilterPredicate(Literal(1.b) === binaryAttr, classOf[Eq[_]], resultFun(1.b))
+      checkFilterPredicate(Literal(1.b) <=> binaryAttr, classOf[Eq[_]], resultFun(1.b))
+      checkFilterPredicate(Literal(2.b) > binaryAttr, classOf[Lt[_]], resultFun(1.b))
+      checkFilterPredicate(Literal(3.b) < binaryAttr, classOf[Gt[_]], resultFun(4.b))
+      checkFilterPredicate(Literal(1.b) >= binaryAttr, classOf[LtEq[_]], resultFun(1.b))
+      checkFilterPredicate(Literal(4.b) <= binaryAttr, classOf[GtEq[_]], resultFun(4.b))
+
+      checkFilterPredicate(!(binaryAttr < 4.b), classOf[GtEq[_]], resultFun(4.b))
+      checkFilterPredicate(binaryAttr < 2.b || binaryAttr > 3.b, classOf[Operators.Or],
+        Seq(Row(resultFun(1.b)), Row(resultFun(4.b))))
     }
   }
 
@@ -532,90 +529,104 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
 
     Seq(false, true).foreach { java8Api =>
       withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
-        val df = data.map(i => Tuple1(Date.valueOf(i))).toDF()
-        withNestedDataFrame(df) { case (inputDF, colName, fun) =>
+        val dates = data.map(i => Tuple1(Date.valueOf(i))).toDF()
+        withNestedParquetDataFrame(dates) { case (inputDF, colName, fun) =>
+          implicit val df: DataFrame = inputDF
+
           def resultFun(dateStr: String): Any = {
             val parsed = if (java8Api) LocalDate.parse(dateStr) else Date.valueOf(dateStr)
             fun(parsed)
           }
-          withParquetDataFrame(inputDF) { implicit df =>
-            val dateAttr: Expression = df(colName).expr
-            assert(df(colName).expr.dataType === DateType)
-
-            checkFilterPredicate(dateAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
-            checkFilterPredicate(dateAttr.isNotNull, classOf[NotEq[_]],
-              data.map(i => Row.apply(resultFun(i))))
-
-            checkFilterPredicate(dateAttr === "2018-03-18".date, classOf[Eq[_]],
-              resultFun("2018-03-18"))
-            checkFilterPredicate(dateAttr <=> "2018-03-18".date, classOf[Eq[_]],
-              resultFun("2018-03-18"))
-            checkFilterPredicate(dateAttr =!= "2018-03-18".date, classOf[NotEq[_]],
-              Seq("2018-03-19", "2018-03-20", "2018-03-21").map(i => Row.apply(resultFun(i))))
-
-            checkFilterPredicate(dateAttr < "2018-03-19".date, classOf[Lt[_]],
-              resultFun("2018-03-18"))
-            checkFilterPredicate(dateAttr > "2018-03-20".date, classOf[Gt[_]],
-              resultFun("2018-03-21"))
-            checkFilterPredicate(dateAttr <= "2018-03-18".date, classOf[LtEq[_]],
-              resultFun("2018-03-18"))
-            checkFilterPredicate(dateAttr >= "2018-03-21".date, classOf[GtEq[_]],
-              resultFun("2018-03-21"))
-
-            checkFilterPredicate(Literal("2018-03-18".date) === dateAttr, classOf[Eq[_]],
-              resultFun("2018-03-18"))
-            checkFilterPredicate(Literal("2018-03-18".date) <=> dateAttr, classOf[Eq[_]],
-              resultFun("2018-03-18"))
-            checkFilterPredicate(Literal("2018-03-19".date) > dateAttr, classOf[Lt[_]],
-              resultFun("2018-03-18"))
-            checkFilterPredicate(Literal("2018-03-20".date) < dateAttr, classOf[Gt[_]],
-              resultFun("2018-03-21"))
-            checkFilterPredicate(Literal("2018-03-18".date) >= dateAttr, classOf[LtEq[_]],
-              resultFun("2018-03-18"))
-            checkFilterPredicate(Literal("2018-03-21".date) <= dateAttr, classOf[GtEq[_]],
-              resultFun("2018-03-21"))
-
-            checkFilterPredicate(!(dateAttr < "2018-03-21".date), classOf[GtEq[_]],
-              resultFun("2018-03-21"))
-            checkFilterPredicate(
-              dateAttr < "2018-03-19".date || dateAttr > "2018-03-20".date,
-              classOf[Operators.Or],
-              Seq(Row(resultFun("2018-03-18")), Row(resultFun("2018-03-21"))))
-          }
+
+          val dateAttr: Expression = df(colName).expr
+          assert(df(colName).expr.dataType === DateType)
+
+          checkFilterPredicate(dateAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+          checkFilterPredicate(dateAttr.isNotNull, classOf[NotEq[_]],
+            data.map(i => Row.apply(resultFun(i))))
+
+          checkFilterPredicate(dateAttr === "2018-03-18".date, classOf[Eq[_]],
+            resultFun("2018-03-18"))
+          checkFilterPredicate(dateAttr <=> "2018-03-18".date, classOf[Eq[_]],
+            resultFun("2018-03-18"))
+          checkFilterPredicate(dateAttr =!= "2018-03-18".date, classOf[NotEq[_]],
+            Seq("2018-03-19", "2018-03-20", "2018-03-21").map(i => Row.apply(resultFun(i))))
+
+          checkFilterPredicate(dateAttr < "2018-03-19".date, classOf[Lt[_]],
+            resultFun("2018-03-18"))
+          checkFilterPredicate(dateAttr > "2018-03-20".date, classOf[Gt[_]],
+            resultFun("2018-03-21"))
+          checkFilterPredicate(dateAttr <= "2018-03-18".date, classOf[LtEq[_]],
+            resultFun("2018-03-18"))
+          checkFilterPredicate(dateAttr >= "2018-03-21".date, classOf[GtEq[_]],
+            resultFun("2018-03-21"))
+
+          checkFilterPredicate(Literal("2018-03-18".date) === dateAttr, classOf[Eq[_]],
+            resultFun("2018-03-18"))
+          checkFilterPredicate(Literal("2018-03-18".date) <=> dateAttr, classOf[Eq[_]],
+            resultFun("2018-03-18"))
+          checkFilterPredicate(Literal("2018-03-19".date) > dateAttr, classOf[Lt[_]],
+            resultFun("2018-03-18"))
+          checkFilterPredicate(Literal("2018-03-20".date) < dateAttr, classOf[Gt[_]],
+            resultFun("2018-03-21"))
+          checkFilterPredicate(Literal("2018-03-18".date) >= dateAttr, classOf[LtEq[_]],
+            resultFun("2018-03-18"))
+          checkFilterPredicate(Literal("2018-03-21".date) <= dateAttr, classOf[GtEq[_]],
+            resultFun("2018-03-21"))
+
+          checkFilterPredicate(!(dateAttr < "2018-03-21".date), classOf[GtEq[_]],
+            resultFun("2018-03-21"))
+          checkFilterPredicate(
+            dateAttr < "2018-03-19".date || dateAttr > "2018-03-20".date,
+            classOf[Operators.Or],
+            Seq(Row(resultFun("2018-03-18")), Row(resultFun("2018-03-21"))))
         }
       }
     }
   }
 
   test("filter pushdown - timestamp") {
-    // spark.sql.parquet.outputTimestampType = TIMESTAMP_MILLIS
-    val millisData = Seq(Timestamp.valueOf("2018-06-14 08:28:53.123"),
-      Timestamp.valueOf("2018-06-15 08:28:53.123"),
-      Timestamp.valueOf("2018-06-16 08:28:53.123"),
-      Timestamp.valueOf("2018-06-17 08:28:53.123"))
-    withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
-      ParquetOutputTimestampType.TIMESTAMP_MILLIS.toString) {
-      testTimestampPushdown(millisData)
-    }
-
-    // spark.sql.parquet.outputTimestampType = TIMESTAMP_MICROS
-    val microsData = Seq(Timestamp.valueOf("2018-06-14 08:28:53.123456"),
-      Timestamp.valueOf("2018-06-15 08:28:53.123456"),
-      Timestamp.valueOf("2018-06-16 08:28:53.123456"),
-      Timestamp.valueOf("2018-06-17 08:28:53.123456"))
-    withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
-      ParquetOutputTimestampType.TIMESTAMP_MICROS.toString) {
-      testTimestampPushdown(microsData)
-    }
-
-    // spark.sql.parquet.outputTimestampType = INT96 doesn't support pushdown
-    withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
-      ParquetOutputTimestampType.INT96.toString) {
-      import testImplicits._
-      withParquetDataFrame(millisData.map(i => Tuple1(i)).toDF()) { implicit df =>
-        val schema = new SparkToParquetSchemaConverter(conf).convert(df.schema)
-        assertResult(None) {
-          createParquetFilters(schema).createFilter(sources.IsNull("_1"))
+    Seq(true, false).foreach { java8Api =>
+      withSQLConf(
+        SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString,
+        SQLConf.PARQUET_REBASE_MODE_IN_WRITE.key -> "CORRECTED",
+        SQLConf.PARQUET_INT96_REBASE_MODE_IN_WRITE.key -> "CORRECTED") {
+        // spark.sql.parquet.outputTimestampType = TIMESTAMP_MILLIS
+        val millisData = Seq(
+          "1000-06-14 08:28:53.123",
+          "1582-06-15 08:28:53.001",
+          "1900-06-16 08:28:53.0",
+          "2018-06-17 08:28:53.999")
+        withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
+          ParquetOutputTimestampType.TIMESTAMP_MILLIS.toString) {
+          testTimestampPushdown(millisData, java8Api)
+        }
+
+        // spark.sql.parquet.outputTimestampType = TIMESTAMP_MICROS
+        val microsData = Seq(
+          "1000-06-14 08:28:53.123456",
+          "1582-06-15 08:28:53.123456",
+          "1900-06-16 08:28:53.123456",
+          "2018-06-17 08:28:53.123456")
+        withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
+          ParquetOutputTimestampType.TIMESTAMP_MICROS.toString) {
+          testTimestampPushdown(microsData, java8Api)
+        }
+
+        // spark.sql.parquet.outputTimestampType = INT96 doesn't support pushdown
+        withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
+          ParquetOutputTimestampType.INT96.toString) {
+          import testImplicits._
+          withTempPath { file =>
+            millisData.map(i => Tuple1(Timestamp.valueOf(i))).toDF
+              .write.format(dataSourceName).save(file.getCanonicalPath)
+            readParquetFile(file.getCanonicalPath) { df =>
+              val schema = new SparkToParquetSchemaConverter(conf).convert(df.schema)
+              assertResult(None) {
+                createParquetFilters(schema).createFilter(sources.IsNull("_1"))
+              }
+            }
+          }
         }
       }
     }
@@ -632,36 +643,36 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
         val rdd =
           spark.sparkContext.parallelize((1 to 4).map(i => Row(new java.math.BigDecimal(i))))
         val dataFrame = spark.createDataFrame(rdd, StructType.fromDDL(s"a decimal($precision, 2)"))
-        withNestedDataFrame(dataFrame) { case (inputDF, colName, resultFun) =>
-          withParquetDataFrame(inputDF) { implicit df =>
-            val decimalAttr: Expression = df(colName).expr
-            assert(df(colName).expr.dataType === DecimalType(precision, 2))
-
-            checkFilterPredicate(decimalAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
-            checkFilterPredicate(decimalAttr.isNotNull, classOf[NotEq[_]],
-              (1 to 4).map(i => Row.apply(resultFun(i))))
-
-            checkFilterPredicate(decimalAttr === 1, classOf[Eq[_]], resultFun(1))
-            checkFilterPredicate(decimalAttr <=> 1, classOf[Eq[_]], resultFun(1))
-            checkFilterPredicate(decimalAttr =!= 1, classOf[NotEq[_]],
-              (2 to 4).map(i => Row.apply(resultFun(i))))
-
-            checkFilterPredicate(decimalAttr < 2, classOf[Lt[_]], resultFun(1))
-            checkFilterPredicate(decimalAttr > 3, classOf[Gt[_]], resultFun(4))
-            checkFilterPredicate(decimalAttr <= 1, classOf[LtEq[_]], resultFun(1))
-            checkFilterPredicate(decimalAttr >= 4, classOf[GtEq[_]], resultFun(4))
-
-            checkFilterPredicate(Literal(1) === decimalAttr, classOf[Eq[_]], resultFun(1))
-            checkFilterPredicate(Literal(1) <=> decimalAttr, classOf[Eq[_]], resultFun(1))
-            checkFilterPredicate(Literal(2) > decimalAttr, classOf[Lt[_]], resultFun(1))
-            checkFilterPredicate(Literal(3) < decimalAttr, classOf[Gt[_]], resultFun(4))
-            checkFilterPredicate(Literal(1) >= decimalAttr, classOf[LtEq[_]], resultFun(1))
-            checkFilterPredicate(Literal(4) <= decimalAttr, classOf[GtEq[_]], resultFun(4))
-
-            checkFilterPredicate(!(decimalAttr < 4), classOf[GtEq[_]], resultFun(4))
-            checkFilterPredicate(decimalAttr < 2 || decimalAttr > 3, classOf[Operators.Or],
-              Seq(Row(resultFun(1)), Row(resultFun(4))))
-          }
+        withNestedParquetDataFrame(dataFrame) { case (inputDF, colName, resultFun) =>
+          implicit val df: DataFrame = inputDF
+
+          val decimalAttr: Expression = df(colName).expr
+          assert(df(colName).expr.dataType === DecimalType(precision, 2))
+
+          checkFilterPredicate(decimalAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+          checkFilterPredicate(decimalAttr.isNotNull, classOf[NotEq[_]],
+            (1 to 4).map(i => Row.apply(resultFun(i))))
+
+          checkFilterPredicate(decimalAttr === 1, classOf[Eq[_]], resultFun(1))
+          checkFilterPredicate(decimalAttr <=> 1, classOf[Eq[_]], resultFun(1))
+          checkFilterPredicate(decimalAttr =!= 1, classOf[NotEq[_]],
+            (2 to 4).map(i => Row.apply(resultFun(i))))
+
+          checkFilterPredicate(decimalAttr < 2, classOf[Lt[_]], resultFun(1))
+          checkFilterPredicate(decimalAttr > 3, classOf[Gt[_]], resultFun(4))
+          checkFilterPredicate(decimalAttr <= 1, classOf[LtEq[_]], resultFun(1))
+          checkFilterPredicate(decimalAttr >= 4, classOf[GtEq[_]], resultFun(4))
+
+          checkFilterPredicate(Literal(1) === decimalAttr, classOf[Eq[_]], resultFun(1))
+          checkFilterPredicate(Literal(1) <=> decimalAttr, classOf[Eq[_]], resultFun(1))
+          checkFilterPredicate(Literal(2) > decimalAttr, classOf[Lt[_]], resultFun(1))
+          checkFilterPredicate(Literal(3) < decimalAttr, classOf[Gt[_]], resultFun(4))
+          checkFilterPredicate(Literal(1) >= decimalAttr, classOf[LtEq[_]], resultFun(1))
+          checkFilterPredicate(Literal(4) <= decimalAttr, classOf[GtEq[_]], resultFun(4))
+
+          checkFilterPredicate(!(decimalAttr < 4), classOf[GtEq[_]], resultFun(4))
+          checkFilterPredicate(decimalAttr < 2 || decimalAttr > 3, classOf[Operators.Or],
+            Seq(Row(resultFun(1)), Row(resultFun(4))))
         }
       }
     }
@@ -781,10 +792,9 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
 
   test("Filter applied on merged Parquet schema with new column should work") {
     import testImplicits._
-    Seq("true", "false").foreach { vectorized =>
+    withAllParquetReaders {
       withSQLConf(SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED.key -> "true",
-        SQLConf.PARQUET_SCHEMA_MERGING_ENABLED.key -> "true",
-        SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized) {
+        SQLConf.PARQUET_SCHEMA_MERGING_ENABLED.key -> "true") {
         withTempPath { dir =>
           val path1 = s"${dir.getCanonicalPath}/table1"
           (1 to 3).map(i => (i, i.toString)).toDF("a", "b").write.parquet(path1)
@@ -1175,8 +1185,7 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
   }
 
   test("SPARK-16371 Do not push down filters when inner name and outer name are the same") {
-    import testImplicits._
-    withParquetDataFrame((1 to 4).map(i => Tuple1(Tuple1(i))).toDF()) { implicit df =>
+    withParquetDataFrame((1 to 4).map(i => Tuple1(Tuple1(i)))) { implicit df =>
       // Here the schema becomes as below:
       //
       // root
@@ -1219,24 +1228,22 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
   }
 
   test("SPARK-17213: Broken Parquet filter push-down for string columns") {
-    Seq(true, false).foreach { vectorizedEnabled =>
-      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorizedEnabled.toString) {
-        withTempPath { dir =>
-          import testImplicits._
+    withAllParquetReaders {
+      withTempPath { dir =>
+        import testImplicits._
 
-          val path = dir.getCanonicalPath
-          // scalastyle:off nonascii
-          Seq("a", "é").toDF("name").write.parquet(path)
-          // scalastyle:on nonascii
+        val path = dir.getCanonicalPath
+        // scalastyle:off nonascii
+        Seq("a", "é").toDF("name").write.parquet(path)
+        // scalastyle:on nonascii
 
-          assert(spark.read.parquet(path).where("name > 'a'").count() == 1)
-          assert(spark.read.parquet(path).where("name >= 'a'").count() == 2)
+        assert(spark.read.parquet(path).where("name > 'a'").count() == 1)
+        assert(spark.read.parquet(path).where("name >= 'a'").count() == 2)
 
-          // scalastyle:off nonascii
-          assert(spark.read.parquet(path).where("name < 'é'").count() == 1)
-          assert(spark.read.parquet(path).where("name <= 'é'").count() == 2)
-          // scalastyle:on nonascii
-        }
+        // scalastyle:off nonascii
+        assert(spark.read.parquet(path).where("name < 'é'").count() == 1)
+        assert(spark.read.parquet(path).where("name <= 'é'").count() == 2)
+        // scalastyle:on nonascii
       }
     }
   }
@@ -1244,8 +1251,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
   test("SPARK-31026: Parquet predicate pushdown for fields having dots in the names") {
     import testImplicits._
 
-    Seq(true, false).foreach { vectorized =>
-      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString,
+    withAllParquetReaders {
+      withSQLConf(
           SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED.key -> true.toString,
           SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "false") {
         withTempPath { path =>
@@ -1255,7 +1262,7 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
         }
       }
 
-      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString,
+      withSQLConf(
           // Makes sure disabling 'spark.sql.parquet.recordFilter' still enables
           // row group level filtering.
           SQLConf.PARQUET_RECORD_FILTER_ENABLED.key -> "false",
@@ -1318,10 +1325,7 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
   }
 
   test("filter pushdown - StringStartsWith") {
-    withParquetDataFrame {
-      import testImplicits._
-      (1 to 4).map(i => Tuple1(i + "str" + i)).toDF()
-    } { implicit df =>
+    withParquetDataFrame((1 to 4).map(i => Tuple1(i + "str" + i))) { implicit df =>
       checkFilterPredicate(
         '_1.startsWith("").asInstanceOf[Predicate],
         classOf[UserDefinedByInstance[_, _]],
@@ -1367,10 +1371,7 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
     }
 
     // SPARK-28371: make sure filter is null-safe.
-    withParquetDataFrame {
-      import testImplicits._
-      Seq(Tuple1[String](null)).toDF()
-    } { implicit df =>
+    withParquetDataFrame(Seq(Tuple1[String](null))) { implicit df =>
       checkFilterPredicate(
         '_1.startsWith("blah").asInstanceOf[Predicate],
         classOf[UserDefinedByInstance[_, _]],
@@ -1573,8 +1574,98 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
       }
     }
   }
+
+  test("Support Parquet column index") {
+    // block 1:
+    //                      null count  min                                       max
+    // page-0                         0  0                                         99
+    // page-1                         0  100                                       199
+    // page-2                         0  200                                       299
+    // page-3                         0  300                                       399
+    // page-4                         0  400                                       449
+    //
+    // block 2:
+    //                      null count  min                                       max
+    // page-0                         0  450                                       549
+    // page-1                         0  550                                       649
+    // page-2                         0  650                                       749
+    // page-3                         0  750                                       849
+    // page-4                         0  850                                       899
+    withTempPath { path =>
+      spark.range(900)
+        .repartition(1)
+        .write
+        .option(ParquetOutputFormat.PAGE_SIZE, "500")
+        .option(ParquetOutputFormat.BLOCK_SIZE, "2000")
+        .parquet(path.getCanonicalPath)
+
+      val parquetFile = path.listFiles().filter(_.getName.startsWith("part")).last
+      val in = HadoopInputFile.fromPath(
+        new Path(parquetFile.getCanonicalPath),
+        spark.sessionState.newHadoopConf())
+
+      Utils.tryWithResource(ParquetFileReader.open(in)) { reader =>
+        val blocks = reader.getFooter.getBlocks
+        assert(blocks.size() > 1)
+        val columns = blocks.get(0).getColumns
+        assert(columns.size() === 1)
+        val columnIndex = reader.readColumnIndex(columns.get(0))
+        assert(columnIndex.getMinValues.size() > 1)
+
+        val rowGroupCnt = blocks.get(0).getRowCount
+        // Page count = Second page min value - first page min value
+        val pageCnt = columnIndex.getMinValues.get(1).asLongBuffer().get() -
+          columnIndex.getMinValues.get(0).asLongBuffer().get()
+        assert(pageCnt < rowGroupCnt)
+        Seq(true, false).foreach { columnIndex =>
+          withSQLConf(ParquetInputFormat.COLUMN_INDEX_FILTERING_ENABLED -> s"$columnIndex") {
+            val df = spark.read.parquet(parquetFile.getCanonicalPath).where("id = 1")
+            df.collect()
+            val plan = df.queryExecution.executedPlan
+            val metrics = plan.collectLeaves().head.metrics
+            val numOutputRows = metrics("numOutputRows").value
+            if (columnIndex) {
+              assert(numOutputRows === pageCnt)
+            } else {
+              assert(numOutputRows === rowGroupCnt)
+            }
+          }
+        }
+      }
+    }
+  }
+
+  test("SPARK-34562: Bloom filter push down") {
+    withTempPath { dir =>
+      val path = dir.getCanonicalPath
+      spark.range(100).selectExpr("id * 2 AS id")
+        .write
+        .option(ParquetOutputFormat.BLOOM_FILTER_ENABLED + "#id", true)
+        // Disable dictionary because the distinct values less than 40000.
+        .option(ParquetOutputFormat.ENABLE_DICTIONARY, false)
+        .parquet(path)
+
+      Seq(true, false).foreach { bloomFilterEnabled =>
+        withSQLConf(ParquetInputFormat.BLOOM_FILTERING_ENABLED -> bloomFilterEnabled.toString) {
+          val accu = new NumRowGroupsAcc
+          sparkContext.register(accu)
+
+          val df = spark.read.parquet(path).filter("id = 19")
+          df.foreachPartition((it: Iterator[Row]) => it.foreach(_ => accu.add(0)))
+          if (bloomFilterEnabled) {
+            assert(accu.value === 0)
+          } else {
+            assert(accu.value > 0)
+          }
+
+          AccumulatorContext.remove(accu.id)
+        }
+      }
+    }
+  }
 }
 
+@ExtendedSQLTest
 class ParquetV1FilterSuite extends ParquetFilterSuite {
   override protected def sparkConf: SparkConf =
     super
@@ -1589,7 +1680,7 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
       expected: Seq[Row]): Unit = {
     val output = predicate.collect { case a: Attribute => a }.distinct
 
-    Seq(("parquet", true), ("", false)).map { case (pushdownDsList, nestedPredicatePushdown) =>
+    Seq(("parquet", true), ("", false)).foreach { case (pushdownDsList, nestedPredicatePushdown) =>
       withSQLConf(
         SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED.key -> "true",
         SQLConf.PARQUET_FILTER_PUSHDOWN_DATE_ENABLED.key -> "true",
@@ -1654,6 +1745,7 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
   }
 }
 
+@ExtendedSQLTest
 class ParquetV2FilterSuite extends ParquetFilterSuite {
   // TODO: enable Parquet V2 write path after file source V2 writers are workable.
   override protected def sparkConf: SparkConf =
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
index 87b4db3fe087a..4d0ea3f5a75b2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.sql.execution.datasources.parquet
 
-import java.sql.{Date, Timestamp}
-import java.time._
 import java.util.Locale
 
 import scala.collection.JavaConverters._
@@ -26,22 +24,20 @@ import scala.collection.mutable
 import scala.reflect.ClassTag
 import scala.reflect.runtime.universe.TypeTag
 
-import org.apache.hadoop.conf.Configuration
+import com.google.common.primitives.UnsignedLong
 import org.apache.hadoop.fs.{FileSystem, Path}
 import org.apache.hadoop.mapreduce.{JobContext, TaskAttemptContext}
-import org.apache.parquet.HadoopReadOptions
 import org.apache.parquet.column.{Encoding, ParquetProperties}
-import org.apache.parquet.example.data.{Group, GroupWriter}
-import org.apache.parquet.example.data.simple.SimpleGroup
+import org.apache.parquet.column.ParquetProperties.WriterVersion.PARQUET_1_0
+import org.apache.parquet.example.data.Group
+import org.apache.parquet.example.data.simple.{SimpleGroup, SimpleGroupFactory}
 import org.apache.parquet.hadoop._
-import org.apache.parquet.hadoop.api.WriteSupport
-import org.apache.parquet.hadoop.api.WriteSupport.WriteContext
+import org.apache.parquet.hadoop.example.ExampleParquetWriter
 import org.apache.parquet.hadoop.metadata.CompressionCodecName
-import org.apache.parquet.hadoop.util.HadoopInputFile
-import org.apache.parquet.io.api.RecordConsumer
+import org.apache.parquet.hadoop.metadata.CompressionCodecName.GZIP
 import org.apache.parquet.schema.{MessageType, MessageTypeParser}
 
-import org.apache.spark.{SPARK_VERSION_SHORT, SparkException, SparkUpgradeException}
+import org.apache.spark.{SPARK_VERSION_SHORT, SparkException}
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.{InternalRow, ScalaReflection}
 import org.apache.spark.sql.catalyst.expressions.{GenericInternalRow, UnsafeRow}
@@ -49,31 +45,10 @@ import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy._
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
-// Write support class for nested groups: ParquetWriter initializes GroupWriteSupport
-// with an empty configuration (it is after all not intended to be used in this way?)
-// and members are private so we need to make our own in order to pass the schema
-// to the writer.
-private[parquet] class TestGroupWriteSupport(schema: MessageType) extends WriteSupport[Group] {
-  var groupWriter: GroupWriter = null
-
-  override def prepareForWrite(recordConsumer: RecordConsumer): Unit = {
-    groupWriter = new GroupWriter(recordConsumer, schema)
-  }
-
-  override def init(configuration: Configuration): WriteContext = {
-    new WriteContext(schema, new java.util.HashMap[String, String]())
-  }
-
-  override def write(record: Group): Unit = {
-    groupWriter.write(record)
-  }
-}
-
 /**
  * A test suite that tests basic Parquet I/O.
  */
@@ -84,7 +59,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
    * Writes `data` to a Parquet file, reads it back and check file contents.
    */
   protected def checkParquetFile[T <: Product : ClassTag: TypeTag](data: Seq[T]): Unit = {
-    withParquetDataFrame(data.toDF())(r => checkAnswer(r, data.map(Row.fromTuple)))
+    withParquetDataFrame(data)(r => checkAnswer(r, data.map(Row.fromTuple)))
   }
 
   test("basic data types (without binary)") {
@@ -96,7 +71,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
 
   test("raw binary") {
     val data = (1 to 4).map(i => Tuple1(Array.fill(3)(i.toByte)))
-    withParquetDataFrame(data.toDF()) { df =>
+    withParquetDataFrame(data) { df =>
       assertResult(data.map(_._1.mkString(",")).sorted) {
         df.collect().map(_.getAs[Array[Byte]](0).mkString(",")).sorted
       }
@@ -199,7 +174,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
 
   testStandardAndLegacyModes("struct") {
     val data = (1 to 4).map(i => Tuple1((i, s"val_$i")))
-    withParquetDataFrame(data.toDF()) { df =>
+    withParquetDataFrame(data) { df =>
       // Structs are converted to `Row`s
       checkAnswer(df, data.map { case Tuple1(struct) =>
         Row(Row(struct.productIterator.toSeq: _*))
@@ -216,7 +191,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
         )
       )
     }
-    withParquetDataFrame(data.toDF()) { df =>
+    withParquetDataFrame(data) { df =>
       // Structs are converted to `Row`s
       checkAnswer(df, data.map { case Tuple1(array) =>
         Row(array.map(struct => Row(struct.productIterator.toSeq: _*)))
@@ -235,7 +210,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
         )
       )
     }
-    withParquetDataFrame(data.toDF()) { df =>
+    withParquetDataFrame(data) { df =>
       // Structs are converted to `Row`s
       checkAnswer(df, data.map { case Tuple1(array) =>
         Row(array.map { case Tuple1(Tuple1(str)) => Row(Row(str))})
@@ -245,7 +220,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
 
   testStandardAndLegacyModes("nested struct with array of array as field") {
     val data = (1 to 4).map(i => Tuple1((i, Seq(Seq(s"val_$i")))))
-    withParquetDataFrame(data.toDF()) { df =>
+    withParquetDataFrame(data) { df =>
       // Structs are converted to `Row`s
       checkAnswer(df, data.map { case Tuple1(struct) =>
         Row(Row(struct.productIterator.toSeq: _*))
@@ -262,7 +237,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
         )
       )
     }
-    withParquetDataFrame(data.toDF()) { df =>
+    withParquetDataFrame(data) { df =>
       // Structs are converted to `Row`s
       checkAnswer(df, data.map { case Tuple1(m) =>
         Row(m.map { case (k, v) => Row(k.productIterator.toSeq: _*) -> v })
@@ -279,7 +254,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
         )
       )
     }
-    withParquetDataFrame(data.toDF()) { df =>
+    withParquetDataFrame(data) { df =>
       // Structs are converted to `Row`s
       checkAnswer(df, data.map { case Tuple1(m) =>
         Row(m.mapValues(struct => Row(struct.productIterator.toSeq: _*)))
@@ -295,7 +270,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
       null.asInstanceOf[java.lang.Float],
       null.asInstanceOf[java.lang.Double])
 
-    withParquetDataFrame((allNulls :: Nil).toDF()) { df =>
+    withParquetDataFrame(allNulls :: Nil) { df =>
       val rows = df.collect()
       assert(rows.length === 1)
       assert(rows.head === Row(Seq.fill(5)(null): _*))
@@ -308,28 +283,31 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
       None.asInstanceOf[Option[Long]],
       None.asInstanceOf[Option[String]])
 
-    withParquetDataFrame((allNones :: Nil).toDF()) { df =>
+    withParquetDataFrame(allNones :: Nil) { df =>
       val rows = df.collect()
       assert(rows.length === 1)
       assert(rows.head === Row(Seq.fill(3)(null): _*))
     }
   }
 
-  test("SPARK-10113 Support for unsigned Parquet logical types") {
+  test("SPARK-34817: Support for unsigned Parquet logical types") {
     val parquetSchema = MessageTypeParser.parseMessageType(
       """message root {
-        |  required int32 c(UINT_32);
+        |  required INT32 a(UINT_8);
+        |  required INT32 b(UINT_16);
+        |  required INT32 c(UINT_32);
+        |  required INT64 d(UINT_64);
         |}
       """.stripMargin)
 
+    val expectedSparkTypes = Seq(ShortType, IntegerType, LongType, DecimalType.LongDecimal)
+
     withTempPath { location =>
       val path = new Path(location.getCanonicalPath)
       val conf = spark.sessionState.newHadoopConf()
       writeMetadata(parquetSchema, path, conf)
-      val errorMessage = intercept[Throwable] {
-        spark.read.parquet(path.toString).printSchema()
-      }.toString
-      assert(errorMessage.contains("Parquet type not supported"))
+      val sparkTypes = spark.read.parquet(path.toString).schema.map(_.dataType)
+      assert(sparkTypes === expectedSparkTypes)
     }
   }
 
@@ -384,11 +362,30 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
     checkCompressionCodec(CompressionCodecName.UNCOMPRESSED)
     checkCompressionCodec(CompressionCodecName.GZIP)
     checkCompressionCodec(CompressionCodecName.SNAPPY)
+    checkCompressionCodec(CompressionCodecName.ZSTD)
+  }
+
+  private def createParquetWriter(
+      schema: MessageType,
+      path: Path,
+      dictionaryEnabled: Boolean = false): ParquetWriter[Group] = {
+    val hadoopConf = spark.sessionState.newHadoopConf()
+
+    ExampleParquetWriter
+      .builder(path)
+      .withDictionaryEncoding(dictionaryEnabled)
+      .withType(schema)
+      .withWriterVersion(PARQUET_1_0)
+      .withCompressionCodec(GZIP)
+      .withRowGroupSize(1024 * 1024)
+      .withPageSize(1024)
+      .withConf(hadoopConf)
+      .build()
   }
 
   test("read raw Parquet file") {
     def makeRawParquetFile(path: Path): Unit = {
-      val schema = MessageTypeParser.parseMessageType(
+      val schemaStr =
         """
           |message root {
           |  required boolean _1;
@@ -397,22 +394,11 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
           |  required float   _4;
           |  required double  _5;
           |}
-        """.stripMargin)
-
-      val testWriteSupport = new TestGroupWriteSupport(schema)
-      /**
-       * Provide a builder for constructing a parquet writer - after PARQUET-248 directly
-       * constructing the writer is deprecated and should be done through a builder. The default
-       * builders include Avro - but for raw Parquet writing we must create our own builder.
-       */
-      class ParquetWriterBuilder() extends
-          ParquetWriter.Builder[Group, ParquetWriterBuilder](path) {
-        override def getWriteSupport(conf: Configuration) = testWriteSupport
-
-        override def self() = this
-      }
+        """.stripMargin
+      val schema = MessageTypeParser.parseMessageType(schemaStr)
+
 
-      val writer = new ParquetWriterBuilder().build()
+      val writer = createParquetWriter(schema, path)
 
       (0 until 10).foreach { i =>
         val record = new SimpleGroup(schema)
@@ -437,6 +423,79 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
     }
   }
 
+  test("SPARK-34817: Read UINT_8/UINT_16/UINT_32 from parquet") {
+    Seq(true, false).foreach { dictionaryEnabled =>
+      def makeRawParquetFile(path: Path): Unit = {
+        val schemaStr =
+          """message root {
+            |  required INT32 a(UINT_8);
+            |  required INT32 b(UINT_16);
+            |  required INT32 c(UINT_32);
+            |}
+        """.stripMargin
+        val schema = MessageTypeParser.parseMessageType(schemaStr)
+
+        val writer = createParquetWriter(schema, path, dictionaryEnabled)
+
+        val factory = new SimpleGroupFactory(schema)
+        (0 until 1000).foreach { i =>
+          val group = factory.newGroup()
+            .append("a", i % 100 + Byte.MaxValue)
+            .append("b", i % 100 + Short.MaxValue)
+            .append("c", i % 100 + Int.MaxValue)
+          writer.write(group)
+        }
+        writer.close()
+      }
+
+      withTempDir { dir =>
+        val path = new Path(dir.toURI.toString, "part-r-0.parquet")
+        makeRawParquetFile(path)
+        readParquetFile(path.toString) { df =>
+          checkAnswer(df, (0 until 1000).map { i =>
+            Row(i % 100 + Byte.MaxValue,
+              i % 100 + Short.MaxValue,
+              i % 100 + Int.MaxValue.toLong)
+          })
+        }
+      }
+    }
+  }
+
+  test("SPARK-34817: Read UINT_64 as Decimal from parquet") {
+    Seq(true, false).foreach { dictionaryEnabled =>
+      def makeRawParquetFile(path: Path): Unit = {
+        val schemaStr =
+          """message root {
+            |  required INT64 a(UINT_64);
+            |}
+        """.stripMargin
+        val schema = MessageTypeParser.parseMessageType(schemaStr)
+
+        val writer = createParquetWriter(schema, path, dictionaryEnabled)
+
+        val factory = new SimpleGroupFactory(schema)
+        (-500 until 500).foreach { i =>
+          val group = factory.newGroup()
+            .append("a", i % 100L)
+          writer.write(group)
+        }
+        writer.close()
+      }
+
+      withTempDir { dir =>
+        val path = new Path(dir.toURI.toString, "part-r-0.parquet")
+        makeRawParquetFile(path)
+        readParquetFile(path.toString) { df =>
+          checkAnswer(df, (-500 until 500).map { i =>
+            val bi = UnsignedLong.fromLongBits(i % 100L).bigIntegerValue()
+            Row(new java.math.BigDecimal(bi))
+          })
+        }
+      }
+    }
+  }
+
   test("write metadata") {
     val hadoopConf = spark.sessionState.newHadoopConf()
     withTempPath { file =>
@@ -647,47 +706,39 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
   }
 
   test("read dictionary encoded decimals written as INT32") {
-    ("true" :: "false" :: Nil).foreach { vectorized =>
-      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized) {
-        checkAnswer(
-          // Decimal column in this file is encoded using plain dictionary
-          readResourceParquetFile("test-data/dec-in-i32.parquet"),
-          spark.range(1 << 4).select('id % 10 cast DecimalType(5, 2) as 'i32_dec))
-      }
+    withAllParquetReaders {
+      checkAnswer(
+        // Decimal column in this file is encoded using plain dictionary
+        readResourceParquetFile("test-data/dec-in-i32.parquet"),
+        spark.range(1 << 4).select('id % 10 cast DecimalType(5, 2) as 'i32_dec))
     }
   }
 
   test("read dictionary encoded decimals written as INT64") {
-    ("true" :: "false" :: Nil).foreach { vectorized =>
-      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized) {
-        checkAnswer(
-          // Decimal column in this file is encoded using plain dictionary
-          readResourceParquetFile("test-data/dec-in-i64.parquet"),
-          spark.range(1 << 4).select('id % 10 cast DecimalType(10, 2) as 'i64_dec))
-      }
+    withAllParquetReaders {
+      checkAnswer(
+        // Decimal column in this file is encoded using plain dictionary
+        readResourceParquetFile("test-data/dec-in-i64.parquet"),
+        spark.range(1 << 4).select('id % 10 cast DecimalType(10, 2) as 'i64_dec))
     }
   }
 
   test("read dictionary encoded decimals written as FIXED_LEN_BYTE_ARRAY") {
-    ("true" :: "false" :: Nil).foreach { vectorized =>
-      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized) {
-        checkAnswer(
-          // Decimal column in this file is encoded using plain dictionary
-          readResourceParquetFile("test-data/dec-in-fixed-len.parquet"),
-          spark.range(1 << 4).select('id % 10 cast DecimalType(10, 2) as 'fixed_len_dec))
-      }
+    withAllParquetReaders {
+      checkAnswer(
+        // Decimal column in this file is encoded using plain dictionary
+        readResourceParquetFile("test-data/dec-in-fixed-len.parquet"),
+        spark.range(1 << 4).select('id % 10 cast DecimalType(10, 2) as 'fixed_len_dec))
     }
   }
 
   test("read dictionary and plain encoded timestamp_millis written as INT64") {
-    ("true" :: "false" :: Nil).foreach { vectorized =>
-      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized) {
-        checkAnswer(
-          // timestamp column in this file is encoded using combination of plain
-          // and dictionary encodings.
-          readResourceParquetFile("test-data/timemillis-in-i64.parquet"),
-          (1 to 3).map(i => Row(new java.sql.Timestamp(10))))
-      }
+    withAllParquetReaders {
+      checkAnswer(
+        // timestamp column in this file is encoded using combination of plain
+        // and dictionary encodings.
+        readResourceParquetFile("test-data/timemillis-in-i64.parquet"),
+        (1 to 3).map(i => Row(new java.sql.Timestamp(10))))
     }
   }
 
@@ -798,7 +849,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
       Seq(1).toDF().repartition(1).write.parquet(dir.getCanonicalPath)
 
       val dataTypes =
-        Seq(StringType, BooleanType, ByteType, ShortType, IntegerType, LongType,
+        Seq(StringType, BooleanType, ByteType, BinaryType, ShortType, IntegerType, LongType,
           FloatType, DoubleType, DecimalType(25, 5), DateType, TimestampType)
 
       val constantValues =
@@ -806,6 +857,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
           UTF8String.fromString("a string"),
           true,
           1.toByte,
+          "Spark SQL".getBytes,
           2.toShort,
           3,
           Long.MaxValue,
@@ -833,7 +885,11 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
           // in order to use get(...) method which is not implemented in `ColumnarBatch`.
           val actual = row.copy().get(1, dt)
           val expected = v
-          assert(actual == expected)
+          if (dt.isInstanceOf[BinaryType]) {
+            assert(actual.asInstanceOf[Array[Byte]] sameElements expected.asInstanceOf[Array[Byte]])
+          } else {
+            assert(actual == expected)
+          }
         } finally {
           vectorizedReader.close()
         }
@@ -868,186 +924,43 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
 
   test("Write Spark version into Parquet metadata") {
     withTempPath { dir =>
-      val path = dir.getAbsolutePath
-      spark.range(1).repartition(1).write.parquet(path)
-      val file = SpecificParquetRecordReaderBase.listDirectory(dir).get(0)
-
-      val conf = new Configuration()
-      val hadoopInputFile = HadoopInputFile.fromPath(new Path(file), conf)
-      val parquetReadOptions = HadoopReadOptions.builder(conf).build()
-      val m = ParquetFileReader.open(hadoopInputFile, parquetReadOptions)
-      val metaData = m.getFileMetaData.getKeyValueMetaData
-      m.close()
-
-      assert(metaData.get(SPARK_VERSION_METADATA_KEY) === SPARK_VERSION_SHORT)
-    }
-  }
-
-  test("SPARK-31159: compatibility with Spark 2.4 in reading dates/timestamps") {
-    // test reading the existing 2.4 files and new 3.0 files (with rebase on/off) together.
-    def checkReadMixedFiles(fileName: String, dt: String, dataStr: String): Unit = {
-      withTempPaths(2) { paths =>
-        paths.foreach(_.delete())
-        val path2_4 = getResourceParquetFilePath("test-data/" + fileName)
-        val path3_0 = paths(0).getCanonicalPath
-        val path3_0_rebase = paths(1).getCanonicalPath
-        if (dt == "date") {
-          val df = Seq(dataStr).toDF("str").select($"str".cast("date").as("date"))
-
-          // By default we should fail to write ancient datetime values.
-          var e = intercept[SparkException](df.write.parquet(path3_0))
-          assert(e.getCause.getCause.getCause.isInstanceOf[SparkUpgradeException])
-          // By default we should fail to read ancient datetime values.
-          e = intercept[SparkException](spark.read.parquet(path2_4).collect())
-          assert(e.getCause.isInstanceOf[SparkUpgradeException])
-
-          withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString) {
-            df.write.mode("overwrite").parquet(path3_0)
-          }
-          withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
-            df.write.parquet(path3_0_rebase)
-          }
-
-          // For Parquet files written by Spark 3.0, we know the writer info and don't need the
-          // config to guide the rebase behavior.
-          withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ.key -> LEGACY.toString) {
-            checkAnswer(
-              spark.read.format("parquet").load(path2_4, path3_0, path3_0_rebase),
-              1.to(3).map(_ => Row(java.sql.Date.valueOf(dataStr))))
-          }
-        } else {
-          val df = Seq(dataStr).toDF("str").select($"str".cast("timestamp").as("ts"))
-          withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> dt) {
-            // By default we should fail to write ancient datetime values.
-            var e = intercept[SparkException](df.write.parquet(path3_0))
-            assert(e.getCause.getCause.getCause.isInstanceOf[SparkUpgradeException])
-            // By default we should fail to read ancient datetime values.
-            e = intercept[SparkException](spark.read.parquet(path2_4).collect())
-            assert(e.getCause.isInstanceOf[SparkUpgradeException])
-
-            withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString) {
-              df.write.mode("overwrite").parquet(path3_0)
-            }
-            withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
-              df.write.parquet(path3_0_rebase)
-            }
-          }
-          // For Parquet files written by Spark 3.0, we know the writer info and don't need the
-          // config to guide the rebase behavior.
-          withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ.key -> LEGACY.toString) {
-            checkAnswer(
-              spark.read.format("parquet").load(path2_4, path3_0, path3_0_rebase),
-              1.to(3).map(_ => Row(java.sql.Timestamp.valueOf(dataStr))))
-          }
-        }
+      spark.range(1).repartition(1).write.parquet(dir.getAbsolutePath)
+      assert(getMetaData(dir)(SPARK_VERSION_METADATA_KEY) === SPARK_VERSION_SHORT)
+    }
+  }
+
+  Seq(true, false).foreach { vec =>
+    test(s"SPARK-34167: read LongDecimals with precision < 10, VectorizedReader $vec") {
+      // decimal32-written-as-64-bit.snappy.parquet was generated using a 3rd-party library. It has
+      // 10 rows of Decimal(9, 1) written as LongDecimal instead of an IntDecimal
+      readParquetFile(testFile("test-data/decimal32-written-as-64-bit.snappy.parquet"), vec) {
+        df =>
+          assert(10 == df.collect().length)
+          val first10Df = df.head(10)
+          assert(
+            Seq(792059492, 986842987, 540247998, null, 357991078,
+              494131059, 92536396, 426847157, -999999999, 204486094)
+              .zip(first10Df).forall(d =>
+              d._2.isNullAt(0) && d._1 == null ||
+                d._1 == d._2.getDecimal(0).unscaledValue().intValue()
+            ))
       }
-    }
-
-    Seq(false, true).foreach { vectorized =>
-      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
-        checkReadMixedFiles("before_1582_date_v2_4.snappy.parquet", "date", "1001-01-01")
-        checkReadMixedFiles(
-          "before_1582_timestamp_micros_v2_4.snappy.parquet",
-          "TIMESTAMP_MICROS",
-          "1001-01-01 01:02:03.123456")
-        checkReadMixedFiles(
-          "before_1582_timestamp_millis_v2_4.snappy.parquet",
-          "TIMESTAMP_MILLIS",
-          "1001-01-01 01:02:03.123")
-
-        // INT96 is a legacy timestamp format and we always rebase the seconds for it.
-        checkAnswer(readResourceParquetFile(
-          "test-data/before_1582_timestamp_int96_v2_4.snappy.parquet"),
-          Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")))
-      }
-    }
-  }
-
-  test("SPARK-31159: rebasing timestamps in write") {
-    val N = 8
-    Seq(false, true).foreach { dictionaryEncoding =>
-      Seq(
-        ("TIMESTAMP_MILLIS", "1001-01-01 01:02:03.123", "1001-01-07 01:09:05.123"),
-        ("TIMESTAMP_MICROS", "1001-01-01 01:02:03.123456", "1001-01-07 01:09:05.123456"),
-        ("INT96", "1001-01-01 01:02:03.123456", "1001-01-01 01:02:03.123456")
-      ).foreach { case (outType, tsStr, nonRebased) =>
-        withClue(s"output type $outType") {
-          withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> outType) {
-            withTempPath { dir =>
-              val path = dir.getAbsolutePath
-              withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
-                Seq.tabulate(N)(_ => tsStr).toDF("tsS")
-                  .select($"tsS".cast("timestamp").as("ts"))
-                  .repartition(1)
-                  .write
-                  .option("parquet.enable.dictionary", dictionaryEncoding)
-                  .parquet(path)
-              }
-
-              Seq(false, true).foreach { vectorized =>
-                withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
-                  // The file metadata indicates if it needs rebase or not, so we can always get the
-                  // correct result regardless of the "rebase mode" config.
-                  Seq(LEGACY, CORRECTED, EXCEPTION).foreach { mode =>
-                    withSQLConf(
-                      SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ.key -> mode.toString) {
-                      checkAnswer(
-                        spark.read.parquet(path),
-                        Seq.tabulate(N)(_ => Row(Timestamp.valueOf(tsStr))))
-                    }
-                  }
-
-                  // Force to not rebase to prove the written datetime values are rebased
-                  // and we will get wrong result if we don't rebase while reading.
-                  withSQLConf("spark.test.forceNoRebase" -> "true") {
-                    checkAnswer(
-                      spark.read.parquet(path),
-                      Seq.tabulate(N)(_ => Row(Timestamp.valueOf(nonRebased))))
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  }
-
-  test("SPARK-31159: rebasing dates in write") {
-    val N = 8
-    Seq(false, true).foreach { dictionaryEncoding =>
-      withTempPath { dir =>
-        val path = dir.getAbsolutePath
-        withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
-          Seq.tabulate(N)(_ => "1001-01-01").toDF("dateS")
-            .select($"dateS".cast("date").as("date"))
-            .repartition(1)
-            .write
-            .option("parquet.enable.dictionary", dictionaryEncoding)
-            .parquet(path)
-        }
-
-        Seq(false, true).foreach { vectorized =>
-          withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
-            // The file metadata indicates if it needs rebase or not, so we can always get the
-            // correct result regardless of the "rebase mode" config.
-            Seq(LEGACY, CORRECTED, EXCEPTION).foreach { mode =>
-              withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ.key -> mode.toString) {
-                checkAnswer(
-                  spark.read.parquet(path),
-                  Seq.tabulate(N)(_ => Row(Date.valueOf("1001-01-01"))))
-              }
-            }
-
-            // Force to not rebase to prove the written datetime values are rebased and we will get
-            // wrong result if we don't rebase while reading.
-            withSQLConf("spark.test.forceNoRebase" -> "true") {
-              checkAnswer(
-                spark.read.parquet(path),
-                Seq.tabulate(N)(_ => Row(Date.valueOf("1001-01-07"))))
-            }
-          }
-        }
+      // decimal32-written-as-64-bit-dict.snappy.parquet was generated using a 3rd-party library. It
+      // has 2048 rows of Decimal(3, 1) written as LongDecimal instead of an IntDecimal
+      readParquetFile(
+        testFile("test-data/decimal32-written-as-64-bit-dict.snappy.parquet"), vec) {
+        df =>
+          assert(2048 == df.collect().length)
+          val first10Df = df.head(10)
+          assert(Seq(751, 937, 511, null, 337, 467, 84, 403, -999, 190)
+            .zip(first10Df).forall(d =>
+            d._2.isNullAt(0) && d._1 == null ||
+              d._1 == d._2.getDecimal(0).unscaledValue().intValue()))
+
+          val last10Df = df.tail(10)
+          assert(Seq(866, 20, 492, 76, 824, 604, 343, 820, 864, 243)
+            .zip(last10Df).forall(d =>
+            d._1 == d._2.getDecimal(0).unscaledValue().intValue()))
       }
     }
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala
index 7d75077a9732a..b1d930acb097e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala
@@ -21,9 +21,8 @@ import java.io.File
 import java.time.ZoneOffset
 
 import org.apache.commons.io.FileUtils
-import org.apache.hadoop.fs.{FileSystem, Path, PathFilter}
+import org.apache.hadoop.fs.{Path, PathFilter}
 import org.apache.parquet.format.converter.ParquetMetadataConverter.NO_FILTER
-import org.apache.parquet.hadoop.ParquetFileReader
 import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName
 
 import org.apache.spark.sql.Row
@@ -124,12 +123,11 @@ class ParquetInteroperabilitySuite extends ParquetCompatibilityTest with SharedS
       FileUtils.copyFile(new File(impalaPath), new File(tableDir, "part-00001.parq"))
 
       Seq(false, true).foreach { int96TimestampConversion =>
-        Seq(false, true).foreach { vectorized =>
+        withAllParquetReaders {
           withSQLConf(
               (SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key,
                 SQLConf.ParquetOutputTimestampType.INT96.toString),
-              (SQLConf.PARQUET_INT96_TIMESTAMP_CONVERSION.key, int96TimestampConversion.toString()),
-              (SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key, vectorized.toString())
+              (SQLConf.PARQUET_INT96_TIMESTAMP_CONVERSION.key, int96TimestampConversion.toString())
           ) {
             val readBack = spark.read.parquet(tableDir.getAbsolutePath).collect()
             assert(readBack.size === 6)
@@ -149,7 +147,8 @@ class ParquetInteroperabilitySuite extends ParquetCompatibilityTest with SharedS
             val fullExpectations = (ts ++ impalaExpectations).map(_.toString).sorted.toArray
             val actual = readBack.map(_.getTimestamp(0).toString).sorted
             withClue(
-              s"int96TimestampConversion = $int96TimestampConversion; vectorized = $vectorized") {
+              s"int96TimestampConversion = $int96TimestampConversion; " +
+              s"vectorized = ${SQLConf.get.parquetVectorizedReaderEnabled}") {
               assert(fullExpectations === actual)
 
               // Now test that the behavior is still correct even with a filter which could get
@@ -165,7 +164,7 @@ class ParquetInteroperabilitySuite extends ParquetCompatibilityTest with SharedS
               // the assumption on column stats, and also the end-to-end behavior.
 
               val hadoopConf = spark.sessionState.newHadoopConf()
-              val fs = FileSystem.get(hadoopConf)
+              val fs = new Path(tableDir.getAbsolutePath).getFileSystem(hadoopConf)
               val parts = fs.listStatus(new Path(tableDir.getAbsolutePath), new PathFilter {
                 override def accept(path: Path): Boolean = !path.getName.startsWith("_")
               })
@@ -174,7 +173,7 @@ class ParquetInteroperabilitySuite extends ParquetCompatibilityTest with SharedS
               assert(parts.size == 2)
               parts.foreach { part =>
                 val oneFooter =
-                  ParquetFileReader.readFooter(hadoopConf, part.getPath, NO_FILTER)
+                  ParquetFooterReader.readFooter(hadoopConf, part.getPath, NO_FILTER)
                 assert(oneFooter.getFileMetaData.getSchema.getColumns.size === 1)
                 val typeName = oneFooter
                   .getFileMetaData.getSchema.getColumns.get(0).getPrimitiveType.getPrimitiveTypeName
@@ -183,7 +182,7 @@ class ParquetInteroperabilitySuite extends ParquetCompatibilityTest with SharedS
                 val oneBlockColumnMeta = oneBlockMeta.getColumns().get(0)
                 // This is the important assert.  Column stats are written, but they are ignored
                 // when the data is read back as mentioned above, b/c int96 is unsigned.  This
-                // assert makes sure this holds even if we change parquet versions (if eg. there
+                // assert makes sure this holds even if we change parquet versions (if e.g. there
                 // were ever statistics even on unsigned columns).
                 assert(!oneBlockColumnMeta.getStatistics.hasNonNullValue)
               }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala
index d3301ced2ba19..5ea8c61749e36 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala
@@ -19,11 +19,9 @@ package org.apache.spark.sql.execution.datasources.parquet
 
 import java.io.File
 import java.math.BigInteger
-import java.sql.{Date, Timestamp}
+import java.sql.Timestamp
 import java.time.{ZoneId, ZoneOffset}
-import java.util.{Calendar, Locale, TimeZone}
-
-import scala.collection.mutable.ArrayBuffer
+import java.util.{Calendar, Locale}
 
 import com.google.common.io.Files
 import org.apache.hadoop.fs.Path
@@ -33,8 +31,8 @@ import org.apache.spark.SparkConf
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils
-import org.apache.spark.sql.catalyst.expressions.Literal
 import org.apache.spark.sql.catalyst.util.{DateFormatter, DateTimeUtils, TimestampFormatter}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.TimeZoneUTC
 import org.apache.spark.sql.execution.datasources._
 import org.apache.spark.sql.execution.datasources.{PartitionPath => Partition}
 import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Relation, FileTable}
@@ -60,7 +58,7 @@ abstract class ParquetPartitionDiscoverySuite
   val timeZoneId = ZoneId.systemDefault()
   val df = DateFormatter(timeZoneId)
   val tf = TimestampFormatter(
-    timestampPartitionPattern, timeZoneId, needVarLengthSecondFraction = true)
+    timestampPartitionPattern, timeZoneId, isParsing = true)
 
   protected override def beforeAll(): Unit = {
     super.beforeAll()
@@ -73,29 +71,25 @@ abstract class ParquetPartitionDiscoverySuite
   }
 
   test("column type inference") {
-    def check(raw: String, literal: Literal, zoneId: ZoneId = timeZoneId): Unit = {
-      assert(inferPartitionColumnValue(raw, true, zoneId, df, tf) === literal)
+    def check(raw: String, dataType: DataType, zoneId: ZoneId = timeZoneId): Unit = {
+      assert(inferPartitionColumnValue(raw, true, zoneId, df, tf) === dataType)
     }
 
-    check("10", Literal.create(10, IntegerType))
-    check("1000000000000000", Literal.create(1000000000000000L, LongType))
+    check("10", IntegerType)
+    check("1000000000000000", LongType)
     val decimal = Decimal("1" * 20)
-    check("1" * 20,
-      Literal.create(decimal, DecimalType(decimal.precision, decimal.scale)))
-    check("1.5", Literal.create(1.5, DoubleType))
-    check("hello", Literal.create("hello", StringType))
-    check("1990-02-24", Literal.create(Date.valueOf("1990-02-24"), DateType))
-    check("1990-02-24 12:00:30",
-      Literal.create(Timestamp.valueOf("1990-02-24 12:00:30"), TimestampType))
-
-    val c = Calendar.getInstance(TimeZone.getTimeZone("UTC"))
+    check("1" * 20, DecimalType(decimal.precision, decimal.scale))
+    check("1.5", DoubleType)
+    check("hello", StringType)
+    check("1990-02-24", DateType)
+    check("1990-02-24 12:00:30", TimestampType)
+
+    val c = Calendar.getInstance(TimeZoneUTC)
     c.set(1990, 1, 24, 12, 0, 30)
     c.set(Calendar.MILLISECOND, 0)
-    check("1990-02-24 12:00:30",
-      Literal.create(new Timestamp(c.getTimeInMillis), TimestampType),
-      ZoneOffset.UTC)
+    check("1990-02-24 12:00:30", TimestampType, ZoneOffset.UTC)
 
-    check(defaultPartitionName, Literal.create(null, NullType))
+    check(defaultPartitionName, NullType)
   }
 
   test("parse invalid partitioned directories") {
@@ -217,22 +211,22 @@ abstract class ParquetPartitionDiscoverySuite
     check("file://path/a=10", Some {
       PartitionValues(
         Seq("a"),
-        Seq(Literal.create(10, IntegerType)))
+        Seq(TypedPartValue("10", IntegerType)))
     })
 
     check("file://path/a=10/b=hello/c=1.5", Some {
       PartitionValues(
         Seq("a", "b", "c"),
         Seq(
-          Literal.create(10, IntegerType),
-          Literal.create("hello", StringType),
-          Literal.create(1.5, DoubleType)))
+          TypedPartValue("10", IntegerType),
+          TypedPartValue("hello", StringType),
+          TypedPartValue("1.5", DoubleType)))
     })
 
     check("file://path/a=10/b_hello/c=1.5", Some {
       PartitionValues(
         Seq("c"),
-        Seq(Literal.create(1.5, DoubleType)))
+        Seq(TypedPartValue("1.5", DoubleType)))
     })
 
     check("file:///", None)
@@ -274,7 +268,7 @@ abstract class ParquetPartitionDiscoverySuite
     assert(partitionSpec2 ==
       Option(PartitionValues(
         Seq("a"),
-        Seq(Literal.create(10, IntegerType)))))
+        Seq(TypedPartValue("10", IntegerType)))))
   }
 
   test("parse partitions") {
@@ -912,15 +906,19 @@ abstract class ParquetPartitionDiscoverySuite
     assert(
       listConflictingPartitionColumns(
         Seq(
-          (new Path("file:/tmp/foo/a=1"), PartitionValues(Seq("a"), Seq(Literal(1)))),
-          (new Path("file:/tmp/foo/b=1"), PartitionValues(Seq("b"), Seq(Literal(1)))))).trim ===
+          (new Path("file:/tmp/foo/a=1"),
+            PartitionValues(Seq("a"), Seq(TypedPartValue("1", IntegerType)))),
+          (new Path("file:/tmp/foo/b=1"),
+            PartitionValues(Seq("b"), Seq(TypedPartValue("1", IntegerType)))))).trim ===
         makeExpectedMessage(Seq("a", "b"), Seq("file:/tmp/foo/a=1", "file:/tmp/foo/b=1")))
 
     assert(
       listConflictingPartitionColumns(
         Seq(
-          (new Path("file:/tmp/foo/a=1/_temporary"), PartitionValues(Seq("a"), Seq(Literal(1)))),
-          (new Path("file:/tmp/foo/a=1"), PartitionValues(Seq("a"), Seq(Literal(1)))))).trim ===
+          (new Path("file:/tmp/foo/a=1/_temporary"),
+            PartitionValues(Seq("a"), Seq(TypedPartValue("1", IntegerType)))),
+          (new Path("file:/tmp/foo/a=1"),
+            PartitionValues(Seq("a"), Seq(TypedPartValue("1", IntegerType)))))).trim ===
         makeExpectedMessage(
           Seq("a"),
           Seq("file:/tmp/foo/a=1/_temporary", "file:/tmp/foo/a=1")))
@@ -929,9 +927,10 @@ abstract class ParquetPartitionDiscoverySuite
       listConflictingPartitionColumns(
         Seq(
           (new Path("file:/tmp/foo/a=1"),
-            PartitionValues(Seq("a"), Seq(Literal(1)))),
+            PartitionValues(Seq("a"), Seq(TypedPartValue("1", IntegerType)))),
           (new Path("file:/tmp/foo/a=1/b=foo"),
-            PartitionValues(Seq("a", "b"), Seq(Literal(1), Literal("foo")))))).trim ===
+            PartitionValues(Seq("a", "b"),
+              Seq(TypedPartValue("1", IntegerType), TypedPartValue("foo", StringType)))))).trim ===
         makeExpectedMessage(
           Seq("a", "a, b"),
           Seq("file:/tmp/foo/a=1", "file:/tmp/foo/a=1/b=foo")))
@@ -1040,6 +1039,21 @@ abstract class ParquetPartitionDiscoverySuite
       checkAnswer(input, data)
     }
   }
+
+  test("SPARK-34314: preserve partition values of the string type") {
+    import testImplicits._
+    withTempPath { file =>
+      val path = file.getCanonicalPath
+      val df = Seq((0, "AA"), (1, "-0")).toDF("id", "part")
+      df.write
+        .partitionBy("part")
+        .format("parquet")
+        .save(path)
+      val readback = spark.read.parquet(path)
+      assert(readback.schema("part").dataType === StringType)
+      checkAnswer(readback, Row(0, "AA") :: Row(1, "-0") :: Nil)
+    }
+  }
 }
 
 class ParquetV1PartitionDiscoverySuite extends ParquetPartitionDiscoverySuite {
@@ -1158,7 +1172,7 @@ class ParquetV1PartitionDiscoverySuite extends ParquetPartitionDiscoverySuite {
   test("SPARK-21463: MetadataLogFileIndex should respect userSpecifiedSchema for partition cols") {
     withTempDir { tempDir =>
       val output = new File(tempDir, "output").toString
-      val checkpoint = new File(tempDir, "chkpoint").toString
+      val checkpoint = new File(tempDir, "checkpoint").toString
       try {
         val stream = MemoryStream[(String, Int)]
         val df = stream.toDS().toDF("time", "value")
@@ -1304,7 +1318,7 @@ class ParquetV2PartitionDiscoverySuite extends ParquetPartitionDiscoverySuite {
   test("SPARK-21463: MetadataLogFileIndex should respect userSpecifiedSchema for partition cols") {
     withTempDir { tempDir =>
       val output = new File(tempDir, "output").toString
-      val checkpoint = new File(tempDir, "chkpoint").toString
+      val checkpoint = new File(tempDir, "checkpoint").toString
       try {
         val stream = MemoryStream[(String, Int)]
         val df = stream.toDS().toDF("time", "value")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
index 917aaba2669ce..9ef43995467c6 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
@@ -28,7 +28,7 @@ import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier}
 import org.apache.spark.sql.catalyst.expressions.SpecificInternalRow
 import org.apache.spark.sql.execution.FileSourceScanExec
-import org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol
+import org.apache.spark.sql.execution.datasources.{SchemaColumnConvertNotSupportedException, SQLHadoopMapReduceCommitProtocol}
 import org.apache.spark.sql.execution.datasources.parquet.TestingUDT.{NestedStruct, NestedStructUDT, SingleElement}
 import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
 import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetScan
@@ -168,11 +168,9 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
       withTempPath { file =>
         val df = spark.createDataFrame(sparkContext.parallelize(data), schema)
         df.write.parquet(file.getCanonicalPath)
-        ("true" :: "false" :: Nil).foreach { vectorized =>
-          withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized) {
-            val df2 = spark.read.parquet(file.getCanonicalPath)
-            checkAnswer(df2, df.collect().toSeq)
-          }
+        withAllParquetReaders {
+          val df2 = spark.read.parquet(file.getCanonicalPath)
+          checkAnswer(df2, df.collect().toSeq)
         }
       }
     }
@@ -791,15 +789,13 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
   }
 
   test("SPARK-26677: negated null-safe equality comparison should not filter matched row groups") {
-    (true :: false :: Nil).foreach { vectorized =>
-      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
-        withTempPath { path =>
-          // Repeated values for dictionary encoding.
-          Seq(Some("A"), Some("A"), None).toDF.repartition(1)
-            .write.parquet(path.getAbsolutePath)
-          val df = spark.read.parquet(path.getAbsolutePath)
-          checkAnswer(stripSparkFilter(df.where("NOT (value <=> 'A')")), df)
-        }
+    withAllParquetReaders {
+      withTempPath { path =>
+        // Repeated values for dictionary encoding.
+        Seq(Some("A"), Some("A"), None).toDF.repartition(1)
+          .write.parquet(path.getAbsolutePath)
+        val df = spark.read.parquet(path.getAbsolutePath)
+        checkAnswer(stripSparkFilter(df.where("NOT (value <=> 'A')")), df)
       }
     }
   }
@@ -821,10 +817,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
         withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> toTsType) {
           write(df2.write.mode(SaveMode.Append))
         }
-        Seq("true", "false").foreach { vectorized =>
-          withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized) {
-            checkAnswer(readback, df1.unionAll(df2))
-          }
+        withAllParquetReaders {
+          checkAnswer(readback, df1.unionAll(df2))
         }
       }
 
@@ -846,6 +840,67 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
     testMigration(fromTsType = "INT96", toTsType = "TIMESTAMP_MICROS")
     testMigration(fromTsType = "TIMESTAMP_MICROS", toTsType = "INT96")
   }
+
+  test("SPARK-34212 Parquet should read decimals correctly") {
+    def readParquet(schema: String, path: File): DataFrame = {
+      spark.read.schema(schema).parquet(path.toString)
+    }
+
+    withTempPath { path =>
+      // a is int-decimal (4 bytes), b is long-decimal (8 bytes), c is binary-decimal (16 bytes)
+      val df = sql("SELECT 1.0 a, CAST(1.23 AS DECIMAL(17, 2)) b, CAST(1.23 AS DECIMAL(36, 2)) c")
+      df.write.parquet(path.toString)
+
+      withAllParquetReaders {
+        // We can read the decimal parquet field with a larger precision, if scale is the same.
+        val schema = "a DECIMAL(9, 1), b DECIMAL(18, 2), c DECIMAL(38, 2)"
+        checkAnswer(readParquet(schema, path), df)
+      }
+
+      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false") {
+        val schema1 = "a DECIMAL(3, 2), b DECIMAL(18, 3), c DECIMAL(37, 3)"
+        checkAnswer(readParquet(schema1, path), df)
+        val schema2 = "a DECIMAL(3, 0), b DECIMAL(18, 1), c DECIMAL(37, 1)"
+        checkAnswer(readParquet(schema2, path), Row(1, 1.2, 1.2))
+      }
+
+      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true") {
+        Seq("a DECIMAL(3, 2)", "b DECIMAL(18, 1)", "c DECIMAL(37, 1)").foreach { schema =>
+          val e = intercept[SparkException] {
+            readParquet(schema, path).collect()
+          }.getCause.getCause
+          assert(e.isInstanceOf[SchemaColumnConvertNotSupportedException])
+        }
+      }
+    }
+
+    // tests for parquet types without decimal metadata.
+    withTempPath { path =>
+      val df = sql(s"SELECT 1 a, 123456 b, ${Int.MaxValue.toLong * 10} c, CAST('1.2' AS BINARY) d")
+      df.write.parquet(path.toString)
+
+      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false") {
+        checkAnswer(readParquet("a DECIMAL(3, 2)", path), sql("SELECT 1.00"))
+        checkAnswer(readParquet("b DECIMAL(3, 2)", path), Row(null))
+        checkAnswer(readParquet("b DECIMAL(11, 1)", path), sql("SELECT 123456.0"))
+        checkAnswer(readParquet("c DECIMAL(11, 1)", path), Row(null))
+        checkAnswer(readParquet("c DECIMAL(13, 0)", path), df.select("c"))
+        val e = intercept[SparkException] {
+          readParquet("d DECIMAL(3, 2)", path).collect()
+        }.getCause
+        assert(e.getMessage.contains("Please read this column/field as Spark BINARY type"))
+      }
+
+      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true") {
+        Seq("a DECIMAL(3, 2)", "c DECIMAL(18, 1)", "d DECIMAL(37, 1)").foreach { schema =>
+          val e = intercept[SparkException] {
+            readParquet(schema, path).collect()
+          }.getCause.getCause
+          assert(e.isInstanceOf[SchemaColumnConvertNotSupportedException])
+        }
+      }
+    }
+  }
 }
 
 class ParquetV1QuerySuite extends ParquetQuerySuite {
@@ -863,7 +918,7 @@ class ParquetV1QuerySuite extends ParquetQuerySuite {
         val df = spark.range(10).select(Seq.tabulate(11) {i => ('id + i).as(s"c$i")} : _*)
         df.write.mode(SaveMode.Overwrite).parquet(path)
 
-        // donot return batch, because whole stage codegen is disabled for wide table (>200 columns)
+        // do not return batch - whole stage codegen is disabled for wide table (>200 columns)
         val df2 = spark.read.parquet(path)
         val fileScan2 = df2.queryExecution.sparkPlan.find(_.isInstanceOf[FileSourceScanExec]).get
         assert(!fileScan2.asInstanceOf[FileSourceScanExec].supportsColumnar)
@@ -896,7 +951,7 @@ class ParquetV2QuerySuite extends ParquetQuerySuite {
         val df = spark.range(10).select(Seq.tabulate(11) {i => ('id + i).as(s"c$i")} : _*)
         df.write.mode(SaveMode.Overwrite).parquet(path)
 
-        // donot return batch, because whole stage codegen is disabled for wide table (>200 columns)
+        // do not return batch - whole stage codegen is disabled for wide table (>200 columns)
         val df2 = spark.read.parquet(path)
         val fileScan2 = df2.queryExecution.sparkPlan.find(_.isInstanceOf[BatchScanExec]).get
         val parquetScan2 = fileScan2.asInstanceOf[BatchScanExec].scan.asInstanceOf[ParquetScan]
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRebaseDatetimeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRebaseDatetimeSuite.scala
new file mode 100644
index 0000000000000..6a49f3aea994e
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRebaseDatetimeSuite.scala
@@ -0,0 +1,391 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.parquet
+
+import java.nio.file.{Files, Paths, StandardCopyOption}
+import java.sql.{Date, Timestamp}
+
+import org.apache.spark.{SparkConf, SparkException, SparkUpgradeException}
+import org.apache.spark.sql.{QueryTest, Row, SPARK_LEGACY_DATETIME, SPARK_LEGACY_INT96}
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.{LegacyBehaviorPolicy, ParquetOutputTimestampType}
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy.{CORRECTED, EXCEPTION, LEGACY}
+import org.apache.spark.sql.test.SharedSparkSession
+
+abstract class ParquetRebaseDatetimeSuite
+  extends QueryTest
+  with ParquetTest
+  with SharedSparkSession {
+
+  import testImplicits._
+
+  // It generates input files for the test below:
+  // "SPARK-31159: compatibility with Spark 2.4 in reading dates/timestamps"
+  ignore("SPARK-31806: generate test files for checking compatibility with Spark 2.4") {
+    val resourceDir = "sql/core/src/test/resources/test-data"
+    val version = "2_4_5"
+    val N = 8
+    def save(
+        in: Seq[(String, String)],
+        t: String,
+        dstFile: String,
+        options: Map[String, String] = Map.empty): Unit = {
+      withTempDir { dir =>
+        in.toDF("dict", "plain")
+          .select($"dict".cast(t), $"plain".cast(t))
+          .repartition(1)
+          .write
+          .mode("overwrite")
+          .options(options)
+          .parquet(dir.getCanonicalPath)
+        Files.copy(
+          dir.listFiles().filter(_.getName.endsWith(".snappy.parquet")).head.toPath,
+          Paths.get(resourceDir, dstFile),
+          StandardCopyOption.REPLACE_EXISTING)
+      }
+    }
+    DateTimeTestUtils.withDefaultTimeZone(DateTimeTestUtils.LA) {
+      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> DateTimeTestUtils.LA.getId) {
+        save(
+          (1 to N).map(i => ("1001-01-01", s"1001-01-0$i")),
+          "date",
+          s"before_1582_date_v$version.snappy.parquet")
+        withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> "TIMESTAMP_MILLIS") {
+          save(
+            (1 to N).map(i => ("1001-01-01 01:02:03.123", s"1001-01-0$i 01:02:03.123")),
+            "timestamp",
+            s"before_1582_timestamp_millis_v$version.snappy.parquet")
+        }
+        val usTs = (1 to N).map(i => ("1001-01-01 01:02:03.123456", s"1001-01-0$i 01:02:03.123456"))
+        withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> "TIMESTAMP_MICROS") {
+          save(usTs, "timestamp", s"before_1582_timestamp_micros_v$version.snappy.parquet")
+        }
+        withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> "INT96") {
+          // Comparing to other logical types, Parquet-MR chooses dictionary encoding for the
+          // INT96 logical type because it consumes less memory for small column cardinality.
+          // Huge parquet files doesn't make sense to place to the resource folder. That's why
+          // we explicitly set `parquet.enable.dictionary` and generate two files w/ and w/o
+          // dictionary encoding.
+          save(
+            usTs,
+            "timestamp",
+            s"before_1582_timestamp_int96_plain_v$version.snappy.parquet",
+            Map("parquet.enable.dictionary" -> "false"))
+          save(
+            usTs,
+            "timestamp",
+            s"before_1582_timestamp_int96_dict_v$version.snappy.parquet",
+            Map("parquet.enable.dictionary" -> "true"))
+        }
+      }
+    }
+  }
+
+  private def inReadConfToOptions(
+      conf: String,
+      mode: LegacyBehaviorPolicy.Value): Map[String, String] = conf match {
+    case SQLConf.PARQUET_INT96_REBASE_MODE_IN_READ.key =>
+      Map(ParquetOptions.INT96_REBASE_MODE -> mode.toString)
+    case _ => Map(ParquetOptions.DATETIME_REBASE_MODE -> mode.toString)
+  }
+
+  private def runInMode(
+      conf: String,
+      modes: Seq[LegacyBehaviorPolicy.Value])(f: Map[String, String] => Unit): Unit = {
+    modes.foreach { mode =>
+      withSQLConf(conf -> mode.toString) { f(Map.empty) }
+    }
+    withSQLConf(conf -> EXCEPTION.toString) {
+      modes.foreach { mode =>
+        f(inReadConfToOptions(conf, mode))
+      }
+    }
+  }
+
+  test("SPARK-31159: compatibility with Spark 2.4 in reading dates/timestamps") {
+    val N = 8
+    // test reading the existing 2.4 files and new 3.0 files (with rebase on/off) together.
+    def checkReadMixedFiles[T](
+        fileName: String,
+        catalystType: String,
+        rowFunc: Int => (String, String),
+        toJavaType: String => T,
+        checkDefaultLegacyRead: String => Unit,
+        tsOutputType: String = "TIMESTAMP_MICROS",
+        inWriteConf: String = SQLConf.PARQUET_REBASE_MODE_IN_WRITE.key,
+        inReadConf: String = SQLConf.PARQUET_REBASE_MODE_IN_READ.key): Unit = {
+      withTempPaths(2) { paths =>
+        paths.foreach(_.delete())
+        val path2_4 = getResourceParquetFilePath("test-data/" + fileName)
+        val path3_0 = paths(0).getCanonicalPath
+        val path3_0_rebase = paths(1).getCanonicalPath
+        val df = Seq.tabulate(N)(rowFunc).toDF("dict", "plain")
+          .select($"dict".cast(catalystType), $"plain".cast(catalystType))
+        withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> tsOutputType) {
+          checkDefaultLegacyRead(path2_4)
+          // By default we should fail to write ancient datetime values.
+          if (tsOutputType != "INT96") {
+            val e = intercept[SparkException](df.write.parquet(path3_0))
+            assert(e.getCause.getCause.getCause.isInstanceOf[SparkUpgradeException])
+          }
+          withSQLConf(inWriteConf -> CORRECTED.toString) {
+            df.write.mode("overwrite").parquet(path3_0)
+          }
+          withSQLConf(inWriteConf -> LEGACY.toString) {
+            df.write.parquet(path3_0_rebase)
+          }
+        }
+        // For Parquet files written by Spark 3.0, we know the writer info and don't need the
+        // config to guide the rebase behavior.
+        runInMode(inReadConf, Seq(LEGACY)) { options =>
+          checkAnswer(
+            spark.read.format("parquet").options(options).load(path2_4, path3_0, path3_0_rebase),
+            (0 until N).flatMap { i =>
+              val (dictS, plainS) = rowFunc(i)
+              Seq.tabulate(3) { _ =>
+                Row(toJavaType(dictS), toJavaType(plainS))
+              }
+            })
+        }
+      }
+    }
+    def successInRead(path: String): Unit = spark.read.parquet(path).collect()
+    def failInRead(path: String): Unit = {
+      val e = intercept[SparkException](spark.read.parquet(path).collect())
+      assert(e.getCause.isInstanceOf[SparkUpgradeException])
+    }
+    Seq(
+      // By default we should fail to read ancient datetime values when parquet files don't
+      // contain Spark version.
+      "2_4_5" -> failInRead _,
+      "2_4_6" -> successInRead _).foreach { case (version, checkDefaultRead) =>
+      withAllParquetReaders {
+        checkReadMixedFiles(
+          s"before_1582_date_v$version.snappy.parquet",
+          "date",
+          (i: Int) => ("1001-01-01", s"1001-01-0${i + 1}"),
+          java.sql.Date.valueOf,
+          checkDefaultRead)
+        checkReadMixedFiles(
+          s"before_1582_timestamp_micros_v$version.snappy.parquet",
+          "timestamp",
+          (i: Int) => ("1001-01-01 01:02:03.123456", s"1001-01-0${i + 1} 01:02:03.123456"),
+          java.sql.Timestamp.valueOf,
+          checkDefaultRead)
+        checkReadMixedFiles(
+          s"before_1582_timestamp_millis_v$version.snappy.parquet",
+          "timestamp",
+          (i: Int) => ("1001-01-01 01:02:03.123", s"1001-01-0${i + 1} 01:02:03.123"),
+          java.sql.Timestamp.valueOf,
+          checkDefaultRead,
+          tsOutputType = "TIMESTAMP_MILLIS")
+      }
+    }
+    Seq(
+      "2_4_5" -> failInRead _,
+      "2_4_6" -> successInRead _).foreach { case (version, checkDefaultRead) =>
+      withAllParquetReaders {
+        Seq("plain", "dict").foreach { enc =>
+          checkReadMixedFiles(
+            s"before_1582_timestamp_int96_${enc}_v$version.snappy.parquet",
+            "timestamp",
+            (i: Int) => ("1001-01-01 01:02:03.123456", s"1001-01-0${i + 1} 01:02:03.123456"),
+            java.sql.Timestamp.valueOf,
+            checkDefaultRead,
+            tsOutputType = "INT96",
+            inWriteConf = SQLConf.PARQUET_INT96_REBASE_MODE_IN_WRITE.key,
+            inReadConf = SQLConf.PARQUET_INT96_REBASE_MODE_IN_READ.key)
+        }
+      }
+    }
+  }
+
+  test("SPARK-31159: rebasing timestamps in write") {
+    val N = 8
+    Seq(false, true).foreach { dictionaryEncoding =>
+      Seq(
+        (
+          "TIMESTAMP_MILLIS",
+          "1001-01-01 01:02:03.123",
+          "1001-01-07 01:09:05.123",
+          SQLConf.PARQUET_REBASE_MODE_IN_WRITE.key,
+          SQLConf.PARQUET_REBASE_MODE_IN_READ.key),
+        (
+          "TIMESTAMP_MICROS",
+          "1001-01-01 01:02:03.123456",
+          "1001-01-07 01:09:05.123456",
+          SQLConf.PARQUET_REBASE_MODE_IN_WRITE.key,
+          SQLConf.PARQUET_REBASE_MODE_IN_READ.key),
+        (
+          "INT96",
+          "1001-01-01 01:02:03.123456",
+          "1001-01-07 01:09:05.123456",
+          SQLConf.PARQUET_INT96_REBASE_MODE_IN_WRITE.key,
+          SQLConf.PARQUET_INT96_REBASE_MODE_IN_READ.key
+        )
+      ).foreach { case (outType, tsStr, nonRebased, inWriteConf, inReadConf) =>
+        withClue(s"output type $outType") {
+          withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> outType) {
+            withTempPath { dir =>
+              val path = dir.getAbsolutePath
+              withSQLConf(inWriteConf -> LEGACY.toString) {
+                Seq.tabulate(N)(_ => tsStr).toDF("tsS")
+                  .select($"tsS".cast("timestamp").as("ts"))
+                  .repartition(1)
+                  .write
+                  .option("parquet.enable.dictionary", dictionaryEncoding)
+                  .parquet(path)
+              }
+
+              withAllParquetReaders {
+                // The file metadata indicates if it needs rebase or not, so we can always get the
+                // correct result regardless of the "rebase mode" config.
+                runInMode(inReadConf, Seq(LEGACY, CORRECTED, EXCEPTION)) { options =>
+                  checkAnswer(
+                    spark.read.options(options).parquet(path),
+                    Seq.tabulate(N)(_ => Row(Timestamp.valueOf(tsStr))))
+                }
+
+                // Force to not rebase to prove the written datetime values are rebased
+                // and we will get wrong result if we don't rebase while reading.
+                withSQLConf("spark.test.forceNoRebase" -> "true") {
+                  checkAnswer(
+                    spark.read.parquet(path),
+                    Seq.tabulate(N)(_ => Row(Timestamp.valueOf(nonRebased))))
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  test("SPARK-31159: rebasing dates in write") {
+    val N = 8
+    Seq(false, true).foreach { dictionaryEncoding =>
+      withTempPath { dir =>
+        val path = dir.getAbsolutePath
+        withSQLConf(SQLConf.PARQUET_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
+          Seq.tabulate(N)(_ => "1001-01-01").toDF("dateS")
+            .select($"dateS".cast("date").as("date"))
+            .repartition(1)
+            .write
+            .option("parquet.enable.dictionary", dictionaryEncoding)
+            .parquet(path)
+        }
+
+        withAllParquetReaders {
+          // The file metadata indicates if it needs rebase or not, so we can always get the
+          // correct result regardless of the "rebase mode" config.
+          runInMode(
+            SQLConf.PARQUET_REBASE_MODE_IN_READ.key,
+            Seq(LEGACY, CORRECTED, EXCEPTION)) { options =>
+            checkAnswer(
+              spark.read.options(options).parquet(path),
+              Seq.tabulate(N)(_ => Row(Date.valueOf("1001-01-01"))))
+          }
+
+          // Force to not rebase to prove the written datetime values are rebased and we will get
+          // wrong result if we don't rebase while reading.
+          withSQLConf("spark.test.forceNoRebase" -> "true") {
+            checkAnswer(
+              spark.read.parquet(path),
+              Seq.tabulate(N)(_ => Row(Date.valueOf("1001-01-07"))))
+          }
+        }
+      }
+    }
+  }
+
+  test("SPARK-33163: write the metadata key 'org.apache.spark.legacyDateTime'") {
+    def checkMetadataKey(dir: java.io.File, exists: Boolean): Unit = {
+      Seq("timestamp '1000-01-01 01:02:03'", "date '1000-01-01'").foreach { dt =>
+        withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
+          ParquetOutputTimestampType.TIMESTAMP_MICROS.toString) {
+          sql(s"SELECT $dt AS dt")
+            .repartition(1)
+            .write
+            .mode("overwrite")
+            .parquet(dir.getAbsolutePath)
+          val metaData = getMetaData(dir)
+          val expected = if (exists) Some("") else None
+          assert(metaData.get(SPARK_LEGACY_DATETIME) === expected)
+        }
+      }
+    }
+    withSQLConf(SQLConf.PARQUET_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
+      withTempPath { dir =>
+        checkMetadataKey(dir, exists = true)
+      }
+    }
+    withSQLConf(SQLConf.PARQUET_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString) {
+      withTempPath { dir =>
+        checkMetadataKey(dir, exists = false)
+      }
+    }
+    withSQLConf(SQLConf.PARQUET_REBASE_MODE_IN_WRITE.key -> EXCEPTION.toString) {
+      withTempPath { dir => intercept[SparkException] { checkMetadataKey(dir, exists = false) } }
+    }
+  }
+
+  test("SPARK-33160: write the metadata key 'org.apache.spark.legacyINT96'") {
+    def saveTs(dir: java.io.File, ts: String = "1000-01-01 01:02:03"): Unit = {
+      Seq(Timestamp.valueOf(ts)).toDF()
+        .repartition(1)
+        .write
+        .parquet(dir.getAbsolutePath)
+    }
+    withSQLConf(SQLConf.PARQUET_INT96_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
+      withTempPath { dir =>
+        saveTs(dir)
+        assert(getMetaData(dir)(SPARK_LEGACY_INT96) === "")
+      }
+    }
+    withSQLConf(SQLConf.PARQUET_INT96_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString) {
+      withTempPath { dir =>
+        saveTs(dir)
+        assert(getMetaData(dir).get(SPARK_LEGACY_INT96).isEmpty)
+      }
+    }
+    withSQLConf(SQLConf.PARQUET_INT96_REBASE_MODE_IN_WRITE.key -> EXCEPTION.toString) {
+      withTempPath { dir => intercept[SparkException] { saveTs(dir) } }
+    }
+    withSQLConf(SQLConf.PARQUET_INT96_REBASE_MODE_IN_WRITE.key -> EXCEPTION.toString) {
+      withTempPath { dir =>
+        saveTs(dir, "2020-10-22 01:02:03")
+        assert(getMetaData(dir).get(SPARK_LEGACY_INT96).isEmpty)
+      }
+    }
+  }
+}
+
+class ParquetRebaseDatetimeV1Suite extends ParquetRebaseDatetimeSuite {
+  override protected def sparkConf: SparkConf =
+    super
+      .sparkConf
+      .set(SQLConf.USE_V1_SOURCE_LIST, "parquet")
+}
+
+class ParquetRebaseDatetimeV2Suite extends ParquetRebaseDatetimeSuite {
+  override protected def sparkConf: SparkConf =
+    super
+      .sparkConf
+      .set(SQLConf.USE_V1_SOURCE_LIST, "")
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala
index c64e95078e916..cab93bd96fff4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala
@@ -25,6 +25,7 @@ import org.apache.spark.sql.execution.datasources.SchemaPruningSuite
 import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
 import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetScan
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.tags.ExtendedSQLTest
 
 abstract class ParquetSchemaPruningSuite extends SchemaPruningSuite with AdaptiveSparkPlanHelper {
   override protected val dataSourceName: String = "parquet"
@@ -33,6 +34,7 @@ abstract class ParquetSchemaPruningSuite extends SchemaPruningSuite with Adaptiv
 
 }
 
+@ExtendedSQLTest
 class ParquetV1SchemaPruningSuite extends ParquetSchemaPruningSuite {
   override protected def sparkConf: SparkConf =
     super
@@ -40,6 +42,7 @@ class ParquetV1SchemaPruningSuite extends ParquetSchemaPruningSuite {
       .set(SQLConf.USE_V1_SOURCE_LIST, "parquet")
 }
 
+@ExtendedSQLTest
 class ParquetV2SchemaPruningSuite extends ParquetSchemaPruningSuite {
   // TODO: enable Parquet V2 write path after file source V2 writers are workable.
   override protected def sparkConf: SparkConf =
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
index 1274995fd6779..fcc08ee16e805 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
@@ -23,7 +23,7 @@ import scala.reflect.runtime.universe.TypeTag
 import org.apache.parquet.io.ParquetDecodingException
 import org.apache.parquet.schema.{MessageType, MessageTypeParser}
 
-import org.apache.spark.{SparkConf, SparkException}
+import org.apache.spark.SparkException
 import org.apache.spark.sql.catalyst.ScalaReflection
 import org.apache.spark.sql.execution.QueryExecutionException
 import org.apache.spark.sql.execution.datasources.SchemaColumnConvertNotSupportedException
@@ -251,7 +251,7 @@ class ParquetSchemaInferenceSuite extends ParquetSchemaTest {
     """
       |message root {
       |  optional group _1 (MAP) {
-      |    repeated group map (MAP_KEY_VALUE) {
+      |    repeated group key_value (MAP_KEY_VALUE) {
       |      required int32 key;
       |      optional binary value (UTF8);
       |    }
@@ -262,6 +262,25 @@ class ParquetSchemaInferenceSuite extends ParquetSchemaTest {
     int96AsTimestamp = true,
     writeLegacyParquetFormat = true)
 
+  testSchemaInference[Tuple1[Map[(String, String), String]]](
+    "map - group type key",
+    """
+      |message root {
+      |  optional group _1 (MAP) {
+      |    repeated group key_value (MAP_KEY_VALUE) {
+      |      required group key {
+      |        optional binary _1 (UTF8);
+      |        optional binary _2 (UTF8);
+      |      }
+      |      optional binary value (UTF8);
+      |    }
+      |  }
+      |}
+    """.stripMargin,
+    binaryAsString = true,
+    int96AsTimestamp = true,
+    writeLegacyParquetFormat = true)
+
   testSchemaInference[Tuple1[(Int, String)]](
     "struct",
     """
@@ -281,7 +300,7 @@ class ParquetSchemaInferenceSuite extends ParquetSchemaTest {
     """
       |message root {
       |  optional group _1 (MAP_KEY_VALUE) {
-      |    repeated group map {
+      |    repeated group key_value {
       |      required int32 key;
       |      optional group value {
       |        optional binary _1 (UTF8);
@@ -721,7 +740,7 @@ class ParquetSchemaSuite extends ParquetSchemaTest {
         nullable = true))),
     """message root {
       |  optional group f1 (MAP_KEY_VALUE) {
-      |    repeated group map {
+      |    repeated group key_value {
       |      required int32 num;
       |      required binary str (UTF8);
       |    }
@@ -740,7 +759,7 @@ class ParquetSchemaSuite extends ParquetSchemaTest {
         nullable = true))),
     """message root {
       |  optional group f1 (MAP) {
-      |    repeated group map (MAP_KEY_VALUE) {
+      |    repeated group key_value (MAP_KEY_VALUE) {
       |      required int32 key;
       |      required binary value (UTF8);
       |    }
@@ -778,7 +797,7 @@ class ParquetSchemaSuite extends ParquetSchemaTest {
         nullable = true))),
     """message root {
       |  optional group f1 (MAP_KEY_VALUE) {
-      |    repeated group map {
+      |    repeated group key_value {
       |      required int32 num;
       |      optional binary str (UTF8);
       |    }
@@ -797,7 +816,7 @@ class ParquetSchemaSuite extends ParquetSchemaTest {
         nullable = true))),
     """message root {
       |  optional group f1 (MAP) {
-      |    repeated group map (MAP_KEY_VALUE) {
+      |    repeated group key_value (MAP_KEY_VALUE) {
       |      required int32 key;
       |      optional binary value (UTF8);
       |    }
@@ -838,7 +857,7 @@ class ParquetSchemaSuite extends ParquetSchemaTest {
         nullable = true))),
     """message root {
       |  optional group f1 (MAP) {
-      |    repeated group map (MAP_KEY_VALUE) {
+      |    repeated group key_value (MAP_KEY_VALUE) {
       |      required int32 key;
       |      required binary value (UTF8);
       |    }
@@ -874,7 +893,7 @@ class ParquetSchemaSuite extends ParquetSchemaTest {
         nullable = true))),
     """message root {
       |  optional group f1 (MAP) {
-      |    repeated group map (MAP_KEY_VALUE) {
+      |    repeated group key_value (MAP_KEY_VALUE) {
       |      required int32 key;
       |      optional binary value (UTF8);
       |    }
@@ -1428,7 +1447,7 @@ class ParquetSchemaSuite extends ParquetSchemaTest {
     parquetSchema =
       """message root {
         |  required group f0 (MAP) {
-        |    repeated group map (MAP_KEY_VALUE) {
+        |    repeated group key_value (MAP_KEY_VALUE) {
         |      required int32 key;
         |      required group value {
         |        required int32 value_f0;
@@ -1453,7 +1472,7 @@ class ParquetSchemaSuite extends ParquetSchemaTest {
     expectedSchema =
       """message root {
         |  required group f0 (MAP) {
-        |    repeated group map (MAP_KEY_VALUE) {
+        |    repeated group key_value (MAP_KEY_VALUE) {
         |      required int32 key;
         |      required group value {
         |        required int64 value_f1;
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
index c833d5f1ab1f6..7a4a382f7f5ce 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
@@ -25,9 +25,11 @@ import scala.reflect.runtime.universe.TypeTag
 
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.Path
+import org.apache.parquet.HadoopReadOptions
 import org.apache.parquet.format.converter.ParquetMetadataConverter
 import org.apache.parquet.hadoop.{Footer, ParquetFileReader, ParquetFileWriter}
 import org.apache.parquet.hadoop.metadata.{BlockMetaData, FileMetaData, ParquetMetadata}
+import org.apache.parquet.hadoop.util.HadoopInputFile
 import org.apache.parquet.schema.MessageType
 
 import org.apache.spark.sql.DataFrame
@@ -63,16 +65,12 @@ private[sql] trait ParquetTest extends FileBasedDataSourceTest {
       (f: String => Unit): Unit = withDataSourceFile(data)(f)
 
   /**
-   * Writes `df` dataframe to a Parquet file and reads it back as a [[DataFrame]],
+   * Writes `data` to a Parquet file and reads it back as a [[DataFrame]],
    * which is then passed to `f`. The Parquet file will be deleted after `f` returns.
    */
-  protected def withParquetDataFrame(df: DataFrame, testVectorized: Boolean = true)
-      (f: DataFrame => Unit): Unit = {
-    withTempPath { file =>
-      df.write.format(dataSourceName).save(file.getCanonicalPath)
-      readFile(file.getCanonicalPath, testVectorized)(f)
-    }
-  }
+  protected def withParquetDataFrame[T <: Product: ClassTag: TypeTag]
+      (data: Seq[T], testVectorized: Boolean = true)
+      (f: DataFrame => Unit): Unit = withDataSourceDataFrame(data, testVectorized)(f)
 
   /**
    * Writes `data` to a Parquet file, reads it back as a [[DataFrame]] and registers it as a
@@ -139,7 +137,7 @@ private[sql] trait ParquetTest extends FileBasedDataSourceTest {
   }
 
   protected def readFooter(path: Path, configuration: Configuration): ParquetMetadata = {
-    ParquetFileReader.readFooter(
+    ParquetFooterReader.readFooter(
       configuration,
       new Path(path, ParquetFileWriter.PARQUET_METADATA_FILE),
       ParquetMetadataConverter.NO_FILTER)
@@ -162,4 +160,25 @@ private[sql] trait ParquetTest extends FileBasedDataSourceTest {
   protected def getResourceParquetFilePath(name: String): String = {
     Thread.currentThread().getContextClassLoader.getResource(name).toString
   }
+
+  def withAllParquetReaders(code: => Unit): Unit = {
+    // test the row-based reader
+    withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false")(code)
+    // test the vectorized reader
+    withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true")(code)
+  }
+
+  def getMetaData(dir: java.io.File): Map[String, String] = {
+    val file = SpecificParquetRecordReaderBase.listDirectory(dir).get(0)
+    val conf = new Configuration()
+    val hadoopInputFile = HadoopInputFile.fromPath(new Path(file), conf)
+    val parquetReadOptions = HadoopReadOptions.builder(conf).build()
+    val m = ParquetFileReader.open(hadoopInputFile, parquetReadOptions)
+    val metadata = try {
+      m.getFileMetaData.getKeyValueMetaData
+    } finally {
+      m.close()
+    }
+    metadata.asScala.toMap
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala
index 7e97994476694..1eb32ed285799 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala
@@ -26,14 +26,17 @@ import org.apache.hadoop.io.compress.GzipCodec
 
 import org.apache.spark.{SparkConf, TestUtils}
 import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row, SaveMode}
+import org.apache.spark.sql.execution.datasources.CommonFileDataSourceSuite
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.{StringType, StructType}
 import org.apache.spark.util.Utils
 
-abstract class TextSuite extends QueryTest with SharedSparkSession {
+abstract class TextSuite extends QueryTest with SharedSparkSession with CommonFileDataSourceSuite {
   import testImplicits._
 
+  override protected def dataSourceFormat = "text"
+
   test("reading text file") {
     verifyFrame(spark.read.format("text").load(testFile))
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala
index c399a011f9073..1a4f08418f8d3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala
@@ -17,17 +17,19 @@
 
 package org.apache.spark.sql.execution.datasources.v2
 
+import java.net.URI
 import java.util
 import java.util.Collections
 
 import scala.collection.JavaConverters._
 
+import org.apache.hadoop.fs.Path
 import org.scalatest.BeforeAndAfter
 
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.analysis.{NamespaceAlreadyExistsException, NoSuchNamespaceException, NoSuchTableException, TableAlreadyExistsException}
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
-import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Identifier, NamespaceChange, SupportsNamespaces, TableChange}
+import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Identifier, NamespaceChange, TableCatalog, TableChange, V1Table}
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.{DoubleType, IntegerType, LongType, StringType, StructField, StructType, TimestampType}
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
@@ -44,7 +46,7 @@ abstract class V2SessionCatalogBaseSuite extends SharedSparkSession with BeforeA
   val testIdent: Identifier = Identifier.of(testNs, "test_table")
 
   def newCatalog(): V2SessionCatalog = {
-    val newCatalog = new V2SessionCatalog(spark.sessionState.catalog, spark.sessionState.conf)
+    val newCatalog = new V2SessionCatalog(spark.sessionState.catalog)
     newCatalog.initialize("test", CaseInsensitiveStringMap.empty())
     newCatalog
   }
@@ -160,6 +162,36 @@ class V2SessionCatalogTableSuite extends V2SessionCatalogBaseSuite {
     assert(catalog.tableExists(testIdent))
   }
 
+  private def makeQualifiedPathWithWarehouse(path: String): URI = {
+    val p = new Path(spark.sessionState.conf.warehousePath, path)
+    val fs = p.getFileSystem(spark.sessionState.newHadoopConf())
+    fs.makeQualified(p).toUri
+
+  }
+
+  test("createTable: location") {
+    val catalog = newCatalog()
+    val properties = new util.HashMap[String, String]()
+    assert(!catalog.tableExists(testIdent))
+
+    // default location
+    val t1 = catalog.createTable(testIdent, schema, Array.empty, properties).asInstanceOf[V1Table]
+    assert(t1.catalogTable.location ===
+      spark.sessionState.catalog.defaultTablePath(testIdent.asTableIdentifier))
+    catalog.dropTable(testIdent)
+
+    // relative path
+    properties.put(TableCatalog.PROP_LOCATION, "relative/path")
+    val t2 = catalog.createTable(testIdent, schema, Array.empty, properties).asInstanceOf[V1Table]
+    assert(t2.catalogTable.location === makeQualifiedPathWithWarehouse("db.db/relative/path"))
+    catalog.dropTable(testIdent)
+
+    // absolute path
+    properties.put(TableCatalog.PROP_LOCATION, "/absolute/path")
+    val t3 = catalog.createTable(testIdent, schema, Array.empty, properties).asInstanceOf[V1Table]
+    assert(t3.catalogTable.location.toString === "file:/absolute/path")
+  }
+
   test("tableExists") {
     val catalog = newCatalog()
 
@@ -640,6 +672,26 @@ class V2SessionCatalogTableSuite extends V2SessionCatalogBaseSuite {
     assert(exc.message.contains("not found"))
   }
 
+  test("alterTable: location") {
+    val catalog = newCatalog()
+    assert(!catalog.tableExists(testIdent))
+
+    // default location
+    val t1 = catalog.createTable(testIdent, schema, Array.empty, emptyProps).asInstanceOf[V1Table]
+    assert(t1.catalogTable.location ===
+      spark.sessionState.catalog.defaultTablePath(testIdent.asTableIdentifier))
+
+    // relative path
+    val t2 = catalog.alterTable(testIdent,
+      TableChange.setProperty(TableCatalog.PROP_LOCATION, "relative/path")).asInstanceOf[V1Table]
+    assert(t2.catalogTable.location === makeQualifiedPathWithWarehouse("db.db/relative/path"))
+
+    // absolute path
+    val t3 = catalog.alterTable(testIdent,
+      TableChange.setProperty(TableCatalog.PROP_LOCATION, "/absolute/path")).asInstanceOf[V1Table]
+    assert(t3.catalogTable.location.toString === "file:/absolute/path")
+  }
+
   test("dropTable") {
     val catalog = newCatalog()
 
@@ -812,11 +864,15 @@ class V2SessionCatalogNamespaceSuite extends V2SessionCatalogBaseSuite {
 
   test("createNamespace: basic behavior") {
     val catalog = newCatalog()
-    val expectedPath = sqlContext.sessionState.catalog.getDefaultDBPath(testNs(0)).toString
+
+    val sessionCatalog = sqlContext.sessionState.catalog
+    val expectedPath =
+      new Path(spark.sessionState.conf.warehousePath,
+        sessionCatalog.getDefaultDBPath(testNs(0)).toString).toString
 
     catalog.createNamespace(testNs, Map("property" -> "value").asJava)
 
-    assert(expectedPath === spark.catalog.getDatabase(testNs(0)).locationUri.toString)
+    assert(expectedPath === spark.catalog.getDatabase(testNs(0)).locationUri)
 
     assert(catalog.namespaceExists(testNs) === true)
     val metadata = catalog.loadNamespaceMetadata(testNs).asScala
@@ -842,6 +898,23 @@ class V2SessionCatalogNamespaceSuite extends V2SessionCatalogBaseSuite {
     catalog.dropNamespace(testNs)
   }
 
+  test("createNamespace: relative location") {
+    val catalog = newCatalog()
+    val expectedPath =
+      new Path(spark.sessionState.conf.warehousePath, "a/b/c").toString
+
+    catalog.createNamespace(testNs, Map("location" -> "a/b/c").asJava)
+
+    assert(expectedPath === spark.catalog.getDatabase(testNs(0)).locationUri)
+
+    assert(catalog.namespaceExists(testNs) === true)
+    val metadata = catalog.loadNamespaceMetadata(testNs).asScala
+    checkMetadata(metadata, Map.empty)
+    assert(expectedPath === metadata("location"))
+
+    catalog.dropNamespace(testNs)
+  }
+
   test("createNamespace: fail if namespace already exists") {
     val catalog = newCatalog()
 
@@ -954,16 +1027,23 @@ class V2SessionCatalogNamespaceSuite extends V2SessionCatalogBaseSuite {
 
   test("alterNamespace: update namespace location") {
     val catalog = newCatalog()
-    val initialPath = sqlContext.sessionState.catalog.getDefaultDBPath(testNs(0)).toString
-    val newPath = "file:/tmp/db.db"
+    val initialPath =
+      new Path(spark.sessionState.conf.warehousePath,
+        spark.sessionState.catalog.getDefaultDBPath(testNs(0)).toString).toString
 
+    val newAbsoluteUri = "file:/tmp/db.db"
     catalog.createNamespace(testNs, emptyProps)
+    assert(initialPath === spark.catalog.getDatabase(testNs(0)).locationUri)
+    catalog.alterNamespace(testNs, NamespaceChange.setProperty("location", newAbsoluteUri))
+    assert(newAbsoluteUri === spark.catalog.getDatabase(testNs(0)).locationUri)
 
-    assert(initialPath === spark.catalog.getDatabase(testNs(0)).locationUri.toString)
-
-    catalog.alterNamespace(testNs, NamespaceChange.setProperty("location", newPath))
+    val newAbsolutePath = "/tmp/newAbsolutePath"
+    catalog.alterNamespace(testNs, NamespaceChange.setProperty("location", newAbsolutePath))
+    assert("file:" + newAbsolutePath === spark.catalog.getDatabase(testNs(0)).locationUri)
 
-    assert(newPath === spark.catalog.getDatabase(testNs(0)).locationUri.toString)
+    val newRelativePath = new Path(spark.sessionState.conf.warehousePath, "relativeP").toString
+    catalog.alterNamespace(testNs, NamespaceChange.setProperty("location", "relativeP"))
+    assert(newRelativePath === spark.catalog.getDatabase(testNs(0)).locationUri)
 
     catalog.dropNamespace(testNs)
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalogSuite.scala
new file mode 100644
index 0000000000000..b94d868120b28
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalogSuite.scala
@@ -0,0 +1,428 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2.jdbc
+
+import java.sql.{Connection, DriverManager}
+import java.util.Properties
+
+import org.apache.log4j.Level
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
+import org.apache.spark.sql.catalyst.analysis.{NoSuchNamespaceException, TableAlreadyExistsException}
+import org.apache.spark.sql.catalyst.parser.ParseException
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types._
+import org.apache.spark.util.Utils
+
+class JDBCTableCatalogSuite extends QueryTest with SharedSparkSession {
+
+  val tempDir = Utils.createTempDir()
+  val url = s"jdbc:h2:${tempDir.getCanonicalPath};user=testUser;password=testPass"
+  val defaultMetadata = new MetadataBuilder().putLong("scale", 0).build()
+  var conn: java.sql.Connection = null
+
+  override def sparkConf: SparkConf = super.sparkConf
+    .set("spark.sql.catalog.h2", classOf[JDBCTableCatalog].getName)
+    .set("spark.sql.catalog.h2.url", url)
+    .set("spark.sql.catalog.h2.driver", "org.h2.Driver")
+
+  private def withConnection[T](f: Connection => T): T = {
+    val conn = DriverManager.getConnection(url, new Properties())
+    try {
+      f(conn)
+    } finally {
+      conn.close()
+    }
+  }
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    Utils.classForName("org.h2.Driver")
+    withConnection { conn =>
+      conn.prepareStatement("""CREATE SCHEMA "test"""").executeUpdate()
+      conn.prepareStatement(
+        """CREATE TABLE "test"."people" (name TEXT(32) NOT NULL, id INTEGER NOT NULL)""")
+        .executeUpdate()
+    }
+  }
+
+  override def afterAll(): Unit = {
+    Utils.deleteRecursively(tempDir)
+    super.afterAll()
+  }
+
+  test("show tables") {
+    checkAnswer(sql("SHOW TABLES IN h2.test"), Seq(Row("test", "people", false)))
+    // Check not existing namespace
+    checkAnswer(sql("SHOW TABLES IN h2.bad_test"), Seq())
+  }
+
+  test("drop a table and test whether the table exists") {
+    withConnection { conn =>
+      conn.prepareStatement("""CREATE TABLE "test"."to_drop" (id INTEGER)""").executeUpdate()
+    }
+    checkAnswer(sql("SHOW TABLES IN h2.test"),
+      Seq(Row("test", "to_drop", false), Row("test", "people", false)))
+    sql("DROP TABLE h2.test.to_drop")
+    checkAnswer(sql("SHOW TABLES IN h2.test"), Seq(Row("test", "people", false)))
+    Seq(
+      "h2.test.not_existing_table" ->
+        "Table or view not found: h2.test.not_existing_table",
+      "h2.bad_test.not_existing_table" ->
+        "Table or view not found: h2.bad_test.not_existing_table"
+    ).foreach { case (table, expectedMsg) =>
+      val msg = intercept[AnalysisException] {
+        sql(s"DROP TABLE $table")
+      }.getMessage
+      assert(msg.contains(expectedMsg))
+    }
+  }
+
+  test("rename a table") {
+    withTable("h2.test.dst_table") {
+      withConnection { conn =>
+        conn.prepareStatement("""CREATE TABLE "test"."src_table" (id INTEGER)""").executeUpdate()
+      }
+      checkAnswer(
+        sql("SHOW TABLES IN h2.test"),
+        Seq(Row("test", "src_table", false), Row("test", "people", false)))
+      sql("ALTER TABLE h2.test.src_table RENAME TO test.dst_table")
+      checkAnswer(
+        sql("SHOW TABLES IN h2.test"),
+        Seq(Row("test", "dst_table", false), Row("test", "people", false)))
+    }
+    // Rename not existing table or namespace
+    val exp1 = intercept[AnalysisException] {
+      sql("ALTER TABLE h2.test.not_existing_table RENAME TO test.dst_table")
+    }
+    assert(exp1.getMessage.contains(
+      "Table or view not found: h2.test.not_existing_table"))
+    val exp2 = intercept[AnalysisException] {
+      sql("ALTER TABLE h2.bad_test.not_existing_table RENAME TO test.dst_table")
+    }
+    assert(exp2.getMessage.contains(
+      "Table or view not found: h2.bad_test.not_existing_table"))
+    // Rename to an existing table
+    withTable("h2.test.dst_table") {
+      withConnection { conn =>
+        conn.prepareStatement("""CREATE TABLE "test"."dst_table" (id INTEGER)""").executeUpdate()
+      }
+      withTable("h2.test.src_table") {
+        withConnection { conn =>
+          conn.prepareStatement("""CREATE TABLE "test"."src_table" (id INTEGER)""").executeUpdate()
+        }
+        val exp = intercept[TableAlreadyExistsException] {
+          sql("ALTER TABLE h2.test.src_table RENAME TO test.dst_table")
+        }
+        assert(exp.getMessage.contains(
+          "Failed table renaming from test.src_table to test.dst_table"))
+        assert(exp.cause.get.getMessage.contains("Table \"dst_table\" already exists"))
+      }
+    }
+  }
+
+  test("load a table") {
+    val t = spark.table("h2.test.people")
+    val expectedSchema = new StructType()
+      .add("NAME", StringType, true, defaultMetadata)
+      .add("ID", IntegerType, true, defaultMetadata)
+    assert(t.schema === expectedSchema)
+    Seq("h2.test.not_existing_table", "h2.bad_test.not_existing_table").foreach { table =>
+      val msg = intercept[AnalysisException] {
+        spark.table(table).schema
+      }.getMessage
+      assert(msg.contains("Table or view not found"))
+    }
+  }
+
+  test("create a table") {
+    withTable("h2.test.new_table") {
+      sql("CREATE TABLE h2.test.new_table(i INT, j STRING)")
+      checkAnswer(
+        sql("SHOW TABLES IN h2.test"),
+        Seq(Row("test", "people", false), Row("test", "new_table", false)))
+    }
+    withTable("h2.test.new_table") {
+      sql("CREATE TABLE h2.test.new_table(i INT, j STRING)")
+      val msg = intercept[AnalysisException] {
+        sql("CREATE TABLE h2.test.new_table(i INT, j STRING)")
+      }.getMessage
+      assert(msg.contains("Table test.new_table already exists"))
+    }
+    val exp = intercept[NoSuchNamespaceException] {
+      sql("CREATE TABLE h2.bad_test.new_table(i INT, j STRING)")
+    }
+    assert(exp.getMessage.contains("Failed table creation: bad_test.new_table"))
+    assert(exp.cause.get.getMessage.contains("Schema \"bad_test\" not found"))
+  }
+
+  test("ALTER TABLE ... add column") {
+    val tableName = "h2.test.alt_table"
+    withTable(tableName) {
+      sql(s"CREATE TABLE $tableName (ID INTEGER)")
+      sql(s"ALTER TABLE $tableName ADD COLUMNS (C1 INTEGER, C2 STRING)")
+      var t = spark.table(tableName)
+      var expectedSchema = new StructType()
+        .add("ID", IntegerType, true, defaultMetadata)
+        .add("C1", IntegerType, true, defaultMetadata)
+        .add("C2", StringType, true, defaultMetadata)
+      assert(t.schema === expectedSchema)
+      sql(s"ALTER TABLE $tableName ADD COLUMNS (c3 DOUBLE)")
+      t = spark.table(tableName)
+      expectedSchema = expectedSchema.add("c3", DoubleType, true, defaultMetadata)
+      assert(t.schema === expectedSchema)
+      // Add already existing column
+      val msg = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $tableName ADD COLUMNS (c3 DOUBLE)")
+      }.getMessage
+      assert(msg.contains("Cannot add column, because c3 already exists"))
+    }
+    // Add a column to not existing table and namespace
+    Seq("h2.test.not_existing_table", "h2.bad_test.not_existing_table").foreach { table =>
+      val msg = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $table ADD COLUMNS (C4 STRING)")
+      }.getMessage
+      assert(msg.contains("Table not found"))
+    }
+  }
+
+  test("ALTER TABLE ... rename column") {
+    val tableName = "h2.test.alt_table"
+    withTable(tableName) {
+      sql(s"CREATE TABLE $tableName (id INTEGER, C0 INTEGER)")
+      sql(s"ALTER TABLE $tableName RENAME COLUMN id TO C")
+      val t = spark.table(tableName)
+      val expectedSchema = new StructType()
+        .add("C", IntegerType, true, defaultMetadata)
+        .add("C0", IntegerType, true, defaultMetadata)
+      assert(t.schema === expectedSchema)
+      // Rename to already existing column
+      val msg = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $tableName RENAME COLUMN C TO C0")
+      }.getMessage
+      assert(msg.contains("Cannot rename column, because C0 already exists"))
+    }
+    // Rename a column in not existing table and namespace
+    Seq("h2.test.not_existing_table", "h2.bad_test.not_existing_table").foreach { table =>
+      val msg = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $table RENAME COLUMN ID TO C")
+      }.getMessage
+      assert(msg.contains("Table not found"))
+    }
+  }
+
+  test("ALTER TABLE ... drop column") {
+    val tableName = "h2.test.alt_table"
+    withTable(tableName) {
+      sql(s"CREATE TABLE $tableName (C1 INTEGER, C2 INTEGER, c3 INTEGER)")
+      sql(s"ALTER TABLE $tableName DROP COLUMN C1")
+      sql(s"ALTER TABLE $tableName DROP COLUMN c3")
+      val t = spark.table(tableName)
+      val expectedSchema = new StructType().add("C2", IntegerType, true, defaultMetadata)
+      assert(t.schema === expectedSchema)
+      // Drop not existing column
+      val msg = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $tableName DROP COLUMN bad_column")
+      }.getMessage
+      assert(msg.contains("Cannot delete missing field bad_column in test.alt_table schema"))
+    }
+    // Drop a column to not existing table and namespace
+    Seq("h2.test.not_existing_table", "h2.bad_test.not_existing_table").foreach { table =>
+      val msg = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $table DROP COLUMN C1")
+      }.getMessage
+      assert(msg.contains("Table not found"))
+    }
+  }
+
+  test("ALTER TABLE ... update column type") {
+    val tableName = "h2.test.alt_table"
+    withTable(tableName) {
+      sql(s"CREATE TABLE $tableName (ID INTEGER, deptno INTEGER)")
+      sql(s"ALTER TABLE $tableName ALTER COLUMN id TYPE DOUBLE")
+      sql(s"ALTER TABLE $tableName ALTER COLUMN deptno TYPE DOUBLE")
+      val t = spark.table(tableName)
+      val expectedSchema = new StructType()
+        .add("ID", DoubleType, true, defaultMetadata)
+        .add("deptno", DoubleType, true, defaultMetadata)
+      assert(t.schema === expectedSchema)
+      // Update not existing column
+      val msg1 = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $tableName ALTER COLUMN bad_column TYPE DOUBLE")
+      }.getMessage
+      assert(msg1.contains("Cannot update missing field bad_column in test.alt_table schema"))
+      // Update column to wrong type
+      val msg2 = intercept[ParseException] {
+        sql(s"ALTER TABLE $tableName ALTER COLUMN id TYPE bad_type")
+      }.getMessage
+      assert(msg2.contains("DataType bad_type is not supported"))
+    }
+    // Update column type in not existing table and namespace
+    Seq("h2.test.not_existing_table", "h2.bad_test.not_existing_table").foreach { table =>
+      val msg = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $table ALTER COLUMN id TYPE DOUBLE")
+      }.getMessage
+      assert(msg.contains("Table not found"))
+    }
+  }
+
+  test("ALTER TABLE ... update column nullability") {
+    val tableName = "h2.test.alt_table"
+    withTable(tableName) {
+      sql(s"CREATE TABLE $tableName (ID INTEGER NOT NULL, deptno INTEGER NOT NULL)")
+      sql(s"ALTER TABLE $tableName ALTER COLUMN ID DROP NOT NULL")
+      sql(s"ALTER TABLE $tableName ALTER COLUMN deptno DROP NOT NULL")
+      val t = spark.table(tableName)
+      val expectedSchema = new StructType()
+        .add("ID", IntegerType, true, defaultMetadata)
+        .add("deptno", IntegerType, true, defaultMetadata)
+      assert(t.schema === expectedSchema)
+      // Update nullability of not existing column
+      val msg = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $tableName ALTER COLUMN bad_column DROP NOT NULL")
+      }.getMessage
+      assert(msg.contains("Cannot update missing field bad_column in test.alt_table"))
+    }
+    // Update column nullability in not existing table and namespace
+    Seq("h2.test.not_existing_table", "h2.bad_test.not_existing_table").foreach { table =>
+      val msg = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $table ALTER COLUMN ID DROP NOT NULL")
+      }.getMessage
+      assert(msg.contains("Table not found"))
+    }
+  }
+
+  test("ALTER TABLE ... update column comment not supported") {
+    val tableName = "h2.test.alt_table"
+    withTable(tableName) {
+      sql(s"CREATE TABLE $tableName (ID INTEGER)")
+      val exp = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $tableName ALTER COLUMN ID COMMENT 'test'")
+      }
+      assert(exp.getMessage.contains("Failed table altering: test.alt_table"))
+      assert(exp.cause.get.getMessage.contains("Unsupported TableChange"))
+      // Update comment for not existing column
+      val msg = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $tableName ALTER COLUMN bad_column COMMENT 'test'")
+      }.getMessage
+      assert(msg.contains("Cannot update missing field bad_column in test.alt_table"))
+    }
+    // Update column comments in not existing table and namespace
+    Seq("h2.test.not_existing_table", "h2.bad_test.not_existing_table").foreach { table =>
+      val msg = intercept[AnalysisException] {
+        sql(s"ALTER TABLE $table ALTER COLUMN ID COMMENT 'test'")
+      }.getMessage
+      assert(msg.contains("Table not found"))
+    }
+  }
+
+  test("ALTER TABLE case sensitivity") {
+    val tableName = "h2.test.alt_table"
+    withTable(tableName) {
+      sql(s"CREATE TABLE $tableName (c1 INTEGER NOT NULL, c2 INTEGER)")
+      var t = spark.table(tableName)
+      var expectedSchema = new StructType()
+        .add("c1", IntegerType, true, defaultMetadata)
+        .add("c2", IntegerType, true, defaultMetadata)
+      assert(t.schema === expectedSchema)
+
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+        val msg = intercept[AnalysisException] {
+          sql(s"ALTER TABLE $tableName RENAME COLUMN C2 TO c3")
+        }.getMessage
+        assert(msg.contains("Cannot rename missing field C2 in test.alt_table schema"))
+      }
+
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+        sql(s"ALTER TABLE $tableName RENAME COLUMN C2 TO c3")
+        expectedSchema = new StructType()
+          .add("c1", IntegerType, true, defaultMetadata)
+          .add("c3", IntegerType, true, defaultMetadata)
+        t = spark.table(tableName)
+        assert(t.schema === expectedSchema)
+      }
+
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+        val msg = intercept[AnalysisException] {
+          sql(s"ALTER TABLE $tableName DROP COLUMN C3")
+        }.getMessage
+        assert(msg.contains("Cannot delete missing field C3 in test.alt_table schema"))
+      }
+
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+        sql(s"ALTER TABLE $tableName DROP COLUMN C3")
+        expectedSchema = new StructType().add("c1", IntegerType, true, defaultMetadata)
+        t = spark.table(tableName)
+        assert(t.schema === expectedSchema)
+      }
+
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+        val msg = intercept[AnalysisException] {
+          sql(s"ALTER TABLE $tableName ALTER COLUMN C1 TYPE DOUBLE")
+        }.getMessage
+        assert(msg.contains("Cannot update missing field C1 in test.alt_table schema"))
+      }
+
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+        sql(s"ALTER TABLE $tableName ALTER COLUMN C1 TYPE DOUBLE")
+        expectedSchema = new StructType().add("c1", DoubleType, true, defaultMetadata)
+        t = spark.table(tableName)
+        assert(t.schema === expectedSchema)
+      }
+
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+        val msg = intercept[AnalysisException] {
+          sql(s"ALTER TABLE $tableName ALTER COLUMN C1 DROP NOT NULL")
+        }.getMessage
+        assert(msg.contains("Cannot update missing field C1 in test.alt_table schema"))
+      }
+
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+        sql(s"ALTER TABLE $tableName ALTER COLUMN C1 DROP NOT NULL")
+        expectedSchema = new StructType().add("c1", DoubleType, true, defaultMetadata)
+        t = spark.table(tableName)
+        assert(t.schema === expectedSchema)
+      }
+    }
+  }
+
+  test("CREATE TABLE with table comment") {
+    withTable("h2.test.new_table") {
+      val logAppender = new LogAppender("table comment")
+      withLogAppender(logAppender) {
+        sql("CREATE TABLE h2.test.new_table(i INT, j STRING) COMMENT 'this is a comment'")
+      }
+      val createCommentWarning = logAppender.loggingEvents
+        .filter(_.getLevel == Level.WARN)
+        .map(_.getRenderedMessage)
+        .exists(_.contains("Cannot create JDBC table comment"))
+      assert(createCommentWarning === false)
+    }
+  }
+
+  test("CREATE TABLE with table property") {
+    withTable("h2.test.new_table") {
+      val m = intercept[AnalysisException] {
+        sql("CREATE TABLE h2.test.new_table(i INT, j STRING)" +
+          " TBLPROPERTIES('ENGINE'='tableEngineName')")
+      }.cause.get.getMessage
+      assert(m.contains("\"TABLEENGINENAME\" not found"))
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
index e9ef7c1a0c540..5085c7483b370 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
@@ -24,14 +24,13 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
 import org.apache.spark.sql.execution.{CodegenSupport, LeafExecNode, WholeStageCodegenExec}
-import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecutionSuite
+import org.apache.spark.sql.execution.adaptive.{DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.test.SQLTestData.TestData
 import org.apache.spark.sql.types.StructType
 
-// Disable AQE because the WholeStageCodegenExec is added when running QueryStageExec
-class DebuggingSuite extends SharedSparkSession with DisableAdaptiveExecutionSuite {
+abstract class DebuggingSuiteBase extends SharedSparkSession {
 
   test("DataFrame.debug()") {
     testData.debug()
@@ -43,62 +42,23 @@ class DebuggingSuite extends SharedSparkSession with DisableAdaptiveExecutionSui
   }
 
   test("debugCodegen") {
-    val res = codegenString(spark.range(10).groupBy(col("id") * 2).count()
-      .queryExecution.executedPlan)
+    val df = spark.range(10).groupBy(col("id") * 2).count()
+    df.collect()
+    val res = codegenString(df.queryExecution.executedPlan)
     assert(res.contains("Subtree 1 / 2"))
     assert(res.contains("Subtree 2 / 2"))
     assert(res.contains("Object[]"))
   }
 
   test("debugCodegenStringSeq") {
-    val res = codegenStringSeq(spark.range(10).groupBy(col("id") * 2).count()
-      .queryExecution.executedPlan)
+    val df = spark.range(10).groupBy(col("id") * 2).count()
+    df.collect()
+    val res = codegenStringSeq(df.queryExecution.executedPlan)
     assert(res.length == 2)
     assert(res.forall{ case (subtree, code, _) =>
       subtree.contains("Range") && code.contains("Object[]")})
   }
 
-  test("SPARK-28537: DebugExec cannot debug broadcast related queries") {
-    val rightDF = spark.range(10)
-    val leftDF = spark.range(10)
-    val joinedDF = leftDF.join(rightDF, leftDF("id") === rightDF("id"))
-
-    val captured = new ByteArrayOutputStream()
-    Console.withOut(captured) {
-      joinedDF.debug()
-    }
-
-    val output = captured.toString()
-    assert(output.replaceAll("\\[id=#\\d+\\]", "[id=#x]").contains(
-      """== BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, false])), [id=#x] ==
-        |Tuples output: 0
-        | id LongType: {}
-        |== WholeStageCodegen (1) ==
-        |Tuples output: 10
-        | id LongType: {java.lang.Long}
-        |== Range (0, 10, step=1, splits=2) ==
-        |Tuples output: 0
-        | id LongType: {}""".stripMargin))
-  }
-
-  test("SPARK-28537: DebugExec cannot debug columnar related queries") {
-    val df = spark.range(5)
-    df.persist()
-
-    val captured = new ByteArrayOutputStream()
-    Console.withOut(captured) {
-      df.debug()
-    }
-    df.unpersist()
-
-    val output = captured.toString().replaceAll("#\\d+", "#x")
-    assert(output.contains(
-      """== InMemoryTableScan [id#xL] ==
-        |Tuples output: 0
-        | id LongType: {}
-        |""".stripMargin))
-  }
-
   case class DummyCodeGeneratorPlan(useInnerClass: Boolean)
       extends CodegenSupport with LeafExecNode {
     override def output: Seq[Attribute] = StructType.fromDDL("d int").toAttributes
@@ -136,3 +96,51 @@ class DebuggingSuite extends SharedSparkSession with DisableAdaptiveExecutionSui
     }
   }
 }
+
+// Disable AQE because the WholeStageCodegenExec is added when running QueryStageExec
+class DebuggingSuite extends DebuggingSuiteBase with DisableAdaptiveExecutionSuite {
+
+  test("SPARK-28537: DebugExec cannot debug broadcast related queries") {
+    val rightDF = spark.range(10)
+    val leftDF = spark.range(10)
+    val joinedDF = leftDF.join(rightDF, leftDF("id") === rightDF("id"))
+
+    val captured = new ByteArrayOutputStream()
+    Console.withOut(captured) {
+      joinedDF.debug()
+    }
+
+    val output = captured.toString()
+    val hashedModeString = "HashedRelationBroadcastMode(List(input[0, bigint, false]),false)"
+    assert(output.replaceAll("\\[id=#\\d+\\]", "[id=#x]").contains(
+      s"""== BroadcastExchange $hashedModeString, [id=#x] ==
+         |Tuples output: 0
+         | id LongType: {}
+         |== WholeStageCodegen (1) ==
+         |Tuples output: 10
+         | id LongType: {java.lang.Long}
+         |== Range (0, 10, step=1, splits=2) ==
+         |Tuples output: 0
+         | id LongType: {}""".stripMargin))
+  }
+
+  test("SPARK-28537: DebugExec cannot debug columnar related queries") {
+    val df = spark.range(5)
+    df.persist()
+
+    val captured = new ByteArrayOutputStream()
+    Console.withOut(captured) {
+      df.debug()
+    }
+    df.unpersist()
+
+    val output = captured.toString().replaceAll("#\\d+", "#x")
+    assert(output.contains(
+      """== InMemoryTableScan [id#xL] ==
+        |Tuples output: 0
+        | id LongType: {}
+        |""".stripMargin))
+  }
+}
+
+class DebuggingSuiteAE extends DebuggingSuiteBase with EnableAdaptiveExecutionSuite
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/exchange/EnsureRequirementsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/exchange/EnsureRequirementsSuite.scala
new file mode 100644
index 0000000000000..061799f439e5b
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/exchange/EnsureRequirementsSuite.scala
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.exchange
+
+import org.apache.spark.sql.catalyst.expressions.Literal
+import org.apache.spark.sql.catalyst.plans.Inner
+import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, PartitioningCollection}
+import org.apache.spark.sql.execution.{DummySparkPlan, SortExec}
+import org.apache.spark.sql.execution.joins.SortMergeJoinExec
+import org.apache.spark.sql.test.SharedSparkSession
+
+class EnsureRequirementsSuite extends SharedSparkSession {
+  private val exprA = Literal(1)
+  private val exprB = Literal(2)
+  private val exprC = Literal(3)
+
+  test("reorder should handle PartitioningCollection") {
+    val plan1 = DummySparkPlan(
+      outputPartitioning = PartitioningCollection(Seq(
+        HashPartitioning(exprA :: exprB :: Nil, 5),
+        HashPartitioning(exprA :: Nil, 5))))
+    val plan2 = DummySparkPlan()
+
+    // Test PartitioningCollection on the left side of join.
+    val smjExec1 = SortMergeJoinExec(
+      exprB :: exprA :: Nil, exprA :: exprB :: Nil, Inner, None, plan1, plan2)
+    EnsureRequirements.apply(smjExec1) match {
+      case SortMergeJoinExec(leftKeys, rightKeys, _, _,
+        SortExec(_, _, DummySparkPlan(_, _, _: PartitioningCollection, _, _), _),
+        SortExec(_, _, ShuffleExchangeExec(_: HashPartitioning, _, _), _), _) =>
+        assert(leftKeys === Seq(exprA, exprB))
+        assert(rightKeys === Seq(exprB, exprA))
+      case other => fail(other.toString)
+    }
+
+    // Test PartitioningCollection on the right side of join.
+    val smjExec2 = SortMergeJoinExec(
+      exprA :: exprB :: Nil, exprB :: exprA :: Nil, Inner, None, plan2, plan1)
+    EnsureRequirements.apply(smjExec2) match {
+      case SortMergeJoinExec(leftKeys, rightKeys, _, _,
+        SortExec(_, _, ShuffleExchangeExec(_: HashPartitioning, _, _), _),
+        SortExec(_, _, DummySparkPlan(_, _, _: PartitioningCollection, _, _), _), _) =>
+        assert(leftKeys === Seq(exprB, exprA))
+        assert(rightKeys === Seq(exprA, exprB))
+      case other => fail(other.toString)
+    }
+
+    // Both sides are PartitioningCollection, but left side cannot be reordered to match
+    // and it should fall back to the right side.
+    val smjExec3 = SortMergeJoinExec(
+      exprA :: exprC :: Nil, exprB :: exprA :: Nil, Inner, None, plan1, plan1)
+    EnsureRequirements.apply(smjExec3) match {
+      case SortMergeJoinExec(leftKeys, rightKeys, _, _,
+        SortExec(_, _, ShuffleExchangeExec(_: HashPartitioning, _, _), _),
+        SortExec(_, _, DummySparkPlan(_, _, _: PartitioningCollection, _, _), _), _) =>
+        assert(leftKeys === Seq(exprC, exprA))
+        assert(rightKeys === Seq(exprA, exprB))
+      case other => fail(other.toString)
+    }
+  }
+
+  test("reorder should fallback to the other side partitioning") {
+    val plan1 = DummySparkPlan(
+      outputPartitioning = HashPartitioning(exprA :: exprB :: exprC :: Nil, 5))
+    val plan2 = DummySparkPlan(
+      outputPartitioning = HashPartitioning(exprB :: exprC :: Nil, 5))
+
+    // Test fallback to the right side, which has HashPartitioning.
+    val smjExec1 = SortMergeJoinExec(
+      exprA :: exprB :: Nil, exprC :: exprB :: Nil, Inner, None, plan1, plan2)
+    EnsureRequirements.apply(smjExec1) match {
+      case SortMergeJoinExec(leftKeys, rightKeys, _, _,
+        SortExec(_, _, ShuffleExchangeExec(_: HashPartitioning, _, _), _),
+        SortExec(_, _, DummySparkPlan(_, _, _: HashPartitioning, _, _), _), _) =>
+        assert(leftKeys === Seq(exprB, exprA))
+        assert(rightKeys === Seq(exprB, exprC))
+      case other => fail(other.toString)
+    }
+
+    // Test fallback to the right side, which has PartitioningCollection.
+    val plan3 = DummySparkPlan(
+      outputPartitioning = PartitioningCollection(Seq(HashPartitioning(exprB :: exprC :: Nil, 5))))
+    val smjExec2 = SortMergeJoinExec(
+      exprA :: exprB :: Nil, exprC :: exprB :: Nil, Inner, None, plan1, plan3)
+    EnsureRequirements.apply(smjExec2) match {
+      case SortMergeJoinExec(leftKeys, rightKeys, _, _,
+        SortExec(_, _, ShuffleExchangeExec(_: HashPartitioning, _, _), _),
+        SortExec(_, _, DummySparkPlan(_, _, _: PartitioningCollection, _, _), _), _) =>
+        assert(leftKeys === Seq(exprB, exprA))
+        assert(rightKeys === Seq(exprB, exprC))
+      case other => fail(other.toString)
+    }
+
+    // The right side has HashPartitioning, so it is matched first, but no reordering match is
+    // found, and it should fall back to the left side, which has a PartitioningCollection.
+    val smjExec3 = SortMergeJoinExec(
+      exprC :: exprB :: Nil, exprA :: exprB :: Nil, Inner, None, plan3, plan1)
+    EnsureRequirements.apply(smjExec3) match {
+      case SortMergeJoinExec(leftKeys, rightKeys, _, _,
+        SortExec(_, _, DummySparkPlan(_, _, _: PartitioningCollection, _, _), _),
+        SortExec(_, _, ShuffleExchangeExec(_: HashPartitioning, _, _), _), _) =>
+        assert(leftKeys === Seq(exprB, exprC))
+        assert(rightKeys === Seq(exprB, exprA))
+      case other => fail(other.toString)
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
index 1be9308c06d8c..98a1089709b92 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
@@ -21,12 +21,15 @@ import scala.reflect.ClassTag
 
 import org.apache.spark.AccumulatorSuite
 import org.apache.spark.sql.{Dataset, QueryTest, Row, SparkSession}
-import org.apache.spark.sql.catalyst.expressions.{BitwiseAnd, BitwiseOr, Cast, Literal, ShiftLeft}
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BitwiseAnd, BitwiseOr, Cast, Expression, Literal, ShiftLeft}
+import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide}
+import org.apache.spark.sql.catalyst.plans.Inner
 import org.apache.spark.sql.catalyst.plans.logical.BROADCAST
-import org.apache.spark.sql.execution.{SparkPlan, WholeStageCodegenExec}
-import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, AdaptiveTestUtils, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
+import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, PartitioningCollection}
+import org.apache.spark.sql.execution.{DummySparkPlan, SparkPlan, WholeStageCodegenExec}
+import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
 import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
-import org.apache.spark.sql.execution.exchange.EnsureRequirements
+import org.apache.spark.sql.execution.exchange.{EnsureRequirements, ShuffleExchangeExec}
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SQLTestUtils
@@ -88,7 +91,7 @@ abstract class BroadcastJoinSuiteBase extends QueryTest with SQLTestUtils
     } else {
       df1.join(df2, joinExpression, joinType)
     }
-    val plan = EnsureRequirements(spark.sessionState.conf).apply(df3.queryExecution.sparkPlan)
+    val plan = EnsureRequirements.apply(df3.queryExecution.sparkPlan)
     assert(plan.collect { case p: T => p }.size === 1)
     plan
   }
@@ -168,7 +171,7 @@ abstract class BroadcastJoinSuiteBase extends QueryTest with SQLTestUtils
       val df4 = Seq((1, "5"), (2, "5")).toDF("key", "value")
       val df5 = df4.join(df3, Seq("key"), "inner")
 
-      val plan = EnsureRequirements(spark.sessionState.conf).apply(df5.queryExecution.sparkPlan)
+      val plan = EnsureRequirements.apply(df5.queryExecution.sparkPlan)
 
       assert(plan.collect { case p: BroadcastHashJoinExec => p }.size === 1)
       assert(plan.collect { case p: SortMergeJoinExec => p }.size === 1)
@@ -179,7 +182,7 @@ abstract class BroadcastJoinSuiteBase extends QueryTest with SQLTestUtils
     val df1 = Seq((1, "4"), (2, "2")).toDF("key", "value")
     val joined = df1.join(df, Seq("key"), "inner")
 
-    val plan = EnsureRequirements(spark.sessionState.conf).apply(joined.queryExecution.sparkPlan)
+    val plan = EnsureRequirements.apply(joined.queryExecution.sparkPlan)
 
     assert(plan.collect { case p: BroadcastHashJoinExec => p }.size === 1)
   }
@@ -239,33 +242,40 @@ abstract class BroadcastJoinSuiteBase extends QueryTest with SQLTestUtils
     assert(HashJoin.rewriteKeyExpr(l :: l :: Nil) === l :: l :: Nil)
     assert(HashJoin.rewriteKeyExpr(l :: i :: Nil) === l :: i :: Nil)
 
-    assert(HashJoin.rewriteKeyExpr(i :: Nil) === Cast(i, LongType) :: Nil)
+    assert(HashJoin.rewriteKeyExpr(i :: Nil) ===
+      Cast(i, LongType, Some(conf.sessionLocalTimeZone)) :: Nil)
     assert(HashJoin.rewriteKeyExpr(i :: l :: Nil) === i :: l :: Nil)
     assert(HashJoin.rewriteKeyExpr(i :: i :: Nil) ===
-      BitwiseOr(ShiftLeft(Cast(i, LongType), Literal(32)),
-        BitwiseAnd(Cast(i, LongType), Literal((1L << 32) - 1))) :: Nil)
+      BitwiseOr(ShiftLeft(Cast(i, LongType, Some(conf.sessionLocalTimeZone)), Literal(32)),
+        BitwiseAnd(Cast(i, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 32) - 1))) ::
+        Nil)
     assert(HashJoin.rewriteKeyExpr(i :: i :: i :: Nil) === i :: i :: i :: Nil)
 
-    assert(HashJoin.rewriteKeyExpr(s :: Nil) === Cast(s, LongType) :: Nil)
+    assert(HashJoin.rewriteKeyExpr(s :: Nil) ===
+      Cast(s, LongType, Some(conf.sessionLocalTimeZone)) :: Nil)
     assert(HashJoin.rewriteKeyExpr(s :: l :: Nil) === s :: l :: Nil)
     assert(HashJoin.rewriteKeyExpr(s :: s :: Nil) ===
-      BitwiseOr(ShiftLeft(Cast(s, LongType), Literal(16)),
-        BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))) :: Nil)
+      BitwiseOr(ShiftLeft(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal(16)),
+        BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 16) - 1))) ::
+        Nil)
     assert(HashJoin.rewriteKeyExpr(s :: s :: s :: Nil) ===
       BitwiseOr(ShiftLeft(
-        BitwiseOr(ShiftLeft(Cast(s, LongType), Literal(16)),
-          BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))),
+        BitwiseOr(ShiftLeft(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal(16)),
+          BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 16) - 1))),
         Literal(16)),
-        BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))) :: Nil)
+        BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 16) - 1))) ::
+        Nil)
     assert(HashJoin.rewriteKeyExpr(s :: s :: s :: s :: Nil) ===
       BitwiseOr(ShiftLeft(
         BitwiseOr(ShiftLeft(
-          BitwiseOr(ShiftLeft(Cast(s, LongType), Literal(16)),
-            BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))),
+          BitwiseOr(ShiftLeft(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal(16)),
+            BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)),
+              Literal((1L << 16) - 1))),
           Literal(16)),
-          BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))),
+          BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 16) - 1))),
         Literal(16)),
-        BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))) :: Nil)
+        BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 16) - 1))) ::
+        Nil)
     assert(HashJoin.rewriteKeyExpr(s :: s :: s :: s :: s :: Nil) ===
       s :: s :: s :: s :: s :: Nil)
 
@@ -411,9 +421,221 @@ abstract class BroadcastJoinSuiteBase extends QueryTest with SQLTestUtils
       val e = intercept[Exception] {
         testDf.collect()
       }
-      AdaptiveTestUtils.assertExceptionMessage(e, s"Could not execute broadcast in $timeout secs.")
+      assert(e.getMessage.contains(s"Could not execute broadcast in $timeout secs."))
     }
   }
+
+  test("broadcast join where streamed side's output partitioning is HashPartitioning") {
+    withTable("t1", "t3") {
+      withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "500") {
+        val df1 = (0 until 100).map(i => (i % 5, i % 13)).toDF("i1", "j1")
+        val df2 = (0 until 20).map(i => (i % 7, i % 11)).toDF("i2", "j2")
+        val df3 = (0 until 100).map(i => (i % 5, i % 13)).toDF("i3", "j3")
+        df1.write.format("parquet").bucketBy(8, "i1", "j1").saveAsTable("t1")
+        df3.write.format("parquet").bucketBy(8, "i3", "j3").saveAsTable("t3")
+        val t1 = spark.table("t1")
+        val t3 = spark.table("t3")
+
+        // join1 is a broadcast join where df2 is broadcasted. Note that output partitioning on the
+        // streamed side (t1) is HashPartitioning (bucketed files).
+        val join1 = t1.join(df2, t1("i1") === df2("i2") && t1("j1") === df2("j2"))
+        withSQLConf(SQLConf.AUTO_BUCKETED_SCAN_ENABLED.key -> "false") {
+          val plan1 = join1.queryExecution.executedPlan
+          assert(collect(plan1) { case e: ShuffleExchangeExec => e }.isEmpty)
+          val broadcastJoins = collect(plan1) { case b: BroadcastHashJoinExec => b }
+          assert(broadcastJoins.size == 1)
+          assert(broadcastJoins(0).outputPartitioning.isInstanceOf[PartitioningCollection])
+          val p = broadcastJoins(0).outputPartitioning.asInstanceOf[PartitioningCollection]
+          assert(p.partitionings.size == 4)
+          // Verify all the combinations of output partitioning.
+          Seq(Seq(t1("i1"), t1("j1")),
+            Seq(t1("i1"), df2("j2")),
+            Seq(df2("i2"), t1("j1")),
+            Seq(df2("i2"), df2("j2"))).foreach { expected =>
+            val expectedExpressions = expected.map(_.expr)
+            assert(p.partitionings.exists {
+              case h: HashPartitioning => expressionsEqual(h.expressions, expectedExpressions)
+            })
+          }
+        }
+
+        // Join on the column from the broadcasted side (i2, j2) and make sure output partitioning
+        // is maintained by checking no shuffle exchange is introduced.
+        val join2 = join1.join(t3, join1("i2") === t3("i3") && join1("j2") === t3("j3"))
+        val plan2 = join2.queryExecution.executedPlan
+        assert(collect(plan2) { case s: SortMergeJoinExec => s }.size == 1)
+        assert(collect(plan2) { case b: BroadcastHashJoinExec => b }.size == 1)
+        assert(collect(plan2) { case e: ShuffleExchangeExec => e }.isEmpty)
+
+        // Validate the data with broadcast join off.
+        withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+          val df = join1.join(t3, join1("i2") === t3("i3") && join1("j2") === t3("j3"))
+          checkAnswer(join2, df)
+        }
+      }
+    }
+  }
+
+  test("broadcast join where streamed side's output partitioning is PartitioningCollection") {
+    withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "500") {
+      val t1 = (0 until 100).map(i => (i % 5, i % 13)).toDF("i1", "j1")
+      val t2 = (0 until 100).map(i => (i % 5, i % 13)).toDF("i2", "j2")
+      val t3 = (0 until 20).map(i => (i % 7, i % 11)).toDF("i3", "j3")
+      val t4 = (0 until 100).map(i => (i % 5, i % 13)).toDF("i4", "j4")
+
+      // join1 is a sort merge join (shuffle on the both sides).
+      val join1 = t1.join(t2, t1("i1") === t2("i2"))
+      val plan1 = join1.queryExecution.executedPlan
+      assert(collect(plan1) { case s: SortMergeJoinExec => s }.size == 1)
+      assert(collect(plan1) { case e: ShuffleExchangeExec => e }.size == 2)
+
+      // join2 is a broadcast join where t3 is broadcasted. Note that output partitioning on the
+      // streamed side (join1) is PartitioningCollection (sort merge join)
+      val join2 = join1.join(t3, join1("i1") === t3("i3"))
+      val plan2 = join2.queryExecution.executedPlan
+      assert(collect(plan2) { case s: SortMergeJoinExec => s }.size == 1)
+      assert(collect(plan2) { case e: ShuffleExchangeExec => e }.size == 2)
+      val broadcastJoins = collect(plan2) { case b: BroadcastHashJoinExec => b }
+      assert(broadcastJoins.size == 1)
+      assert(broadcastJoins(0).outputPartitioning.isInstanceOf[PartitioningCollection])
+      val p = broadcastJoins(0).outputPartitioning.asInstanceOf[PartitioningCollection]
+      assert(p.partitionings.size == 3)
+      // Verify all the combinations of output partitioning.
+      Seq(Seq(t1("i1")), Seq(t2("i2")), Seq(t3("i3"))).foreach { expected =>
+        val expectedExpressions = expected.map(_.expr)
+        assert(p.partitionings.exists {
+          case h: HashPartitioning => expressionsEqual(h.expressions, expectedExpressions)
+        })
+      }
+
+      // Join on the column from the broadcasted side (i3) and make sure output partitioning
+      // is maintained by checking no shuffle exchange is introduced. Note that one extra
+      // ShuffleExchangeExec is from t4, not from join2.
+      val join3 = join2.join(t4, join2("i3") === t4("i4"))
+      val plan3 = join3.queryExecution.executedPlan
+      assert(collect(plan3) { case s: SortMergeJoinExec => s }.size == 2)
+      assert(collect(plan3) { case b: BroadcastHashJoinExec => b }.size == 1)
+      assert(collect(plan3) { case e: ShuffleExchangeExec => e }.size == 3)
+
+      // Validate the data with broadcast join off.
+      withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+        val df = join2.join(t4, join2("i3") === t4("i4"))
+        checkAnswer(join3, df)
+      }
+    }
+  }
+
+  test("BroadcastHashJoinExec output partitioning scenarios for inner join") {
+    val l1 = AttributeReference("l1", LongType)()
+    val l2 = AttributeReference("l2", LongType)()
+    val l3 = AttributeReference("l3", LongType)()
+    val r1 = AttributeReference("r1", LongType)()
+    val r2 = AttributeReference("r2", LongType)()
+    val r3 = AttributeReference("r3", LongType)()
+
+    // Streamed side has a HashPartitioning.
+    var bhj = BroadcastHashJoinExec(
+      leftKeys = Seq(l2, l3),
+      rightKeys = Seq(r1, r2),
+      Inner,
+      BuildRight,
+      None,
+      left = DummySparkPlan(outputPartitioning = HashPartitioning(Seq(l1, l2, l3), 1)),
+      right = DummySparkPlan())
+    var expected = PartitioningCollection(Seq(
+      HashPartitioning(Seq(l1, l2, l3), 1),
+      HashPartitioning(Seq(l1, l2, r2), 1),
+      HashPartitioning(Seq(l1, r1, l3), 1),
+      HashPartitioning(Seq(l1, r1, r2), 1)))
+    assert(bhj.outputPartitioning === expected)
+
+    // Streamed side has a PartitioningCollection.
+    bhj = BroadcastHashJoinExec(
+      leftKeys = Seq(l1, l2, l3),
+      rightKeys = Seq(r1, r2, r3),
+      Inner,
+      BuildRight,
+      None,
+      left = DummySparkPlan(outputPartitioning = PartitioningCollection(Seq(
+        HashPartitioning(Seq(l1, l2), 1), HashPartitioning(Seq(l3), 1)))),
+      right = DummySparkPlan())
+    expected = PartitioningCollection(Seq(
+      HashPartitioning(Seq(l1, l2), 1),
+      HashPartitioning(Seq(l1, r2), 1),
+      HashPartitioning(Seq(r1, l2), 1),
+      HashPartitioning(Seq(r1, r2), 1),
+      HashPartitioning(Seq(l3), 1),
+      HashPartitioning(Seq(r3), 1)))
+    assert(bhj.outputPartitioning === expected)
+
+    // Streamed side has a nested PartitioningCollection.
+    bhj = BroadcastHashJoinExec(
+      leftKeys = Seq(l1, l2, l3),
+      rightKeys = Seq(r1, r2, r3),
+      Inner,
+      BuildRight,
+      None,
+      left = DummySparkPlan(outputPartitioning = PartitioningCollection(Seq(
+        PartitioningCollection(Seq(HashPartitioning(Seq(l1), 1), HashPartitioning(Seq(l2), 1))),
+        HashPartitioning(Seq(l3), 1)))),
+      right = DummySparkPlan())
+    expected = PartitioningCollection(Seq(
+      PartitioningCollection(Seq(
+        HashPartitioning(Seq(l1), 1),
+        HashPartitioning(Seq(r1), 1),
+        HashPartitioning(Seq(l2), 1),
+        HashPartitioning(Seq(r2), 1))),
+      HashPartitioning(Seq(l3), 1),
+      HashPartitioning(Seq(r3), 1)))
+    assert(bhj.outputPartitioning === expected)
+
+    // One-to-mapping case ("l1" = "r1" AND "l1" = "r2")
+    bhj = BroadcastHashJoinExec(
+      leftKeys = Seq(l1, l1),
+      rightKeys = Seq(r1, r2),
+      Inner,
+      BuildRight,
+      None,
+      left = DummySparkPlan(outputPartitioning = HashPartitioning(Seq(l1, l2), 1)),
+      right = DummySparkPlan())
+    expected = PartitioningCollection(Seq(
+      HashPartitioning(Seq(l1, l2), 1),
+      HashPartitioning(Seq(r1, l2), 1),
+      HashPartitioning(Seq(r2, l2), 1)))
+    assert(bhj.outputPartitioning === expected)
+  }
+
+  test("BroadcastHashJoinExec output partitioning size should be limited with a config") {
+    val l1 = AttributeReference("l1", LongType)()
+    val l2 = AttributeReference("l2", LongType)()
+    val r1 = AttributeReference("r1", LongType)()
+    val r2 = AttributeReference("r2", LongType)()
+
+    val expected = Seq(
+      HashPartitioning(Seq(l1, l2), 1),
+      HashPartitioning(Seq(l1, r2), 1),
+      HashPartitioning(Seq(r1, l2), 1),
+      HashPartitioning(Seq(r1, r2), 1))
+
+    Seq(1, 2, 3, 4).foreach { limit =>
+      withSQLConf(
+        SQLConf.BROADCAST_HASH_JOIN_OUTPUT_PARTITIONING_EXPAND_LIMIT.key -> s"$limit") {
+        val bhj = BroadcastHashJoinExec(
+          leftKeys = Seq(l1, l2),
+          rightKeys = Seq(r1, r2),
+          Inner,
+          BuildRight,
+          None,
+          left = DummySparkPlan(outputPartitioning = HashPartitioning(Seq(l1, l2), 1)),
+          right = DummySparkPlan())
+        assert(bhj.outputPartitioning === PartitioningCollection(expected.take(limit)))
+      }
+    }
+  }
+
+  private def expressionsEqual(l: Seq[Expression], r: Seq[Expression]): Boolean = {
+    l.length == r.length && l.zip(r).forall { case (e1, e2) => e1.semanticEquals(e2) }
+  }
 }
 
 class BroadcastJoinSuite extends BroadcastJoinSuiteBase with DisableAdaptiveExecutionSuite
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/ExistenceJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/ExistenceJoinSuite.scala
index 5490246baceea..3588b9dda90d2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/ExistenceJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/ExistenceJoinSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.execution.joins
 
 import org.apache.spark.sql.{DataFrame, Row}
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight}
 import org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical.{Join, JoinHint}
@@ -81,12 +82,12 @@ class ExistenceJoinSuite extends SparkPlanTest with SharedSparkSession {
       joinType: JoinType,
       leftRows: => DataFrame,
       rightRows: => DataFrame,
-      condition: => Expression,
+      condition: => Option[Expression],
       expectedAnswer: Seq[Row]): Unit = {
 
     def extractJoinParts(): Option[ExtractEquiJoinKeys.ReturnType] = {
       val join = Join(leftRows.logicalPlan, rightRows.logicalPlan,
-        Inner, Some(condition), JoinHint.NONE)
+        Inner, condition, JoinHint.NONE)
       ExtractEquiJoinKeys.unapply(join)
     }
 
@@ -102,17 +103,17 @@ class ExistenceJoinSuite extends SparkPlanTest with SharedSparkSession {
       ProjectExec(output, FilterExec(condition, join))
     }
 
-    test(s"$testName using ShuffledHashJoin") {
+    testWithWholeStageCodegenOnAndOff(s"$testName using ShuffledHashJoin") { _ =>
       extractJoinParts().foreach { case (_, leftKeys, rightKeys, boundCondition, _, _, _) =>
         withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
           checkAnswer2(leftRows, rightRows, (left: SparkPlan, right: SparkPlan) =>
-            EnsureRequirements(left.sqlContext.sessionState.conf).apply(
+            EnsureRequirements.apply(
               ShuffledHashJoinExec(
                 leftKeys, rightKeys, joinType, BuildRight, boundCondition, left, right)),
             expectedAnswer,
             sortAnswers = true)
           checkAnswer2(leftRows, rightRows, (left: SparkPlan, right: SparkPlan) =>
-            EnsureRequirements(left.sqlContext.sessionState.conf).apply(
+            EnsureRequirements.apply(
               createLeftSemiPlusJoin(ShuffledHashJoinExec(
                 leftKeys, rightKeys, leftSemiPlus, BuildRight, boundCondition, left, right))),
             expectedAnswer,
@@ -125,13 +126,13 @@ class ExistenceJoinSuite extends SparkPlanTest with SharedSparkSession {
       extractJoinParts().foreach { case (_, leftKeys, rightKeys, boundCondition, _, _, _) =>
         withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
           checkAnswer2(leftRows, rightRows, (left: SparkPlan, right: SparkPlan) =>
-            EnsureRequirements(left.sqlContext.sessionState.conf).apply(
+            EnsureRequirements.apply(
               BroadcastHashJoinExec(
                 leftKeys, rightKeys, joinType, BuildRight, boundCondition, left, right)),
             expectedAnswer,
             sortAnswers = true)
           checkAnswer2(leftRows, rightRows, (left: SparkPlan, right: SparkPlan) =>
-            EnsureRequirements(left.sqlContext.sessionState.conf).apply(
+            EnsureRequirements.apply(
               createLeftSemiPlusJoin(BroadcastHashJoinExec(
                 leftKeys, rightKeys, leftSemiPlus, BuildRight, boundCondition, left, right))),
             expectedAnswer,
@@ -140,16 +141,16 @@ class ExistenceJoinSuite extends SparkPlanTest with SharedSparkSession {
       }
     }
 
-    test(s"$testName using SortMergeJoin") {
+    testWithWholeStageCodegenOnAndOff(s"$testName using SortMergeJoin") { _ =>
       extractJoinParts().foreach { case (_, leftKeys, rightKeys, boundCondition, _, _, _) =>
         withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
           checkAnswer2(leftRows, rightRows, (left: SparkPlan, right: SparkPlan) =>
-            EnsureRequirements(left.sqlContext.sessionState.conf).apply(
+            EnsureRequirements.apply(
               SortMergeJoinExec(leftKeys, rightKeys, joinType, boundCondition, left, right)),
             expectedAnswer,
             sortAnswers = true)
           checkAnswer2(leftRows, rightRows, (left: SparkPlan, right: SparkPlan) =>
-            EnsureRequirements(left.sqlContext.sessionState.conf).apply(
+            EnsureRequirements.apply(
               createLeftSemiPlusJoin(SortMergeJoinExec(
                 leftKeys, rightKeys, leftSemiPlus, boundCondition, left, right))),
             expectedAnswer,
@@ -161,42 +162,67 @@ class ExistenceJoinSuite extends SparkPlanTest with SharedSparkSession {
     test(s"$testName using BroadcastNestedLoopJoin build left") {
       withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
         checkAnswer2(leftRows, rightRows, (left: SparkPlan, right: SparkPlan) =>
-          EnsureRequirements(left.sqlContext.sessionState.conf).apply(
-            BroadcastNestedLoopJoinExec(left, right, BuildLeft, joinType, Some(condition))),
+          EnsureRequirements.apply(
+            BroadcastNestedLoopJoinExec(left, right, BuildLeft, joinType, condition)),
           expectedAnswer,
           sortAnswers = true)
         checkAnswer2(leftRows, rightRows, (left: SparkPlan, right: SparkPlan) =>
-          EnsureRequirements(left.sqlContext.sessionState.conf).apply(
+          EnsureRequirements.apply(
             createLeftSemiPlusJoin(BroadcastNestedLoopJoinExec(
-              left, right, BuildLeft, leftSemiPlus, Some(condition)))),
+              left, right, BuildLeft, leftSemiPlus, condition))),
           expectedAnswer,
           sortAnswers = true)
       }
     }
 
-    test(s"$testName using BroadcastNestedLoopJoin build right") {
+    testWithWholeStageCodegenOnAndOff(s"$testName using BroadcastNestedLoopJoin build right") { _ =>
       withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
         checkAnswer2(leftRows, rightRows, (left: SparkPlan, right: SparkPlan) =>
-          EnsureRequirements(left.sqlContext.sessionState.conf).apply(
-            BroadcastNestedLoopJoinExec(left, right, BuildRight, joinType, Some(condition))),
+          EnsureRequirements.apply(
+            BroadcastNestedLoopJoinExec(left, right, BuildRight, joinType, condition)),
           expectedAnswer,
           sortAnswers = true)
         checkAnswer2(leftRows, rightRows, (left: SparkPlan, right: SparkPlan) =>
-          EnsureRequirements(left.sqlContext.sessionState.conf).apply(
+          EnsureRequirements.apply(
             createLeftSemiPlusJoin(BroadcastNestedLoopJoinExec(
-              left, right, BuildRight, leftSemiPlus, Some(condition)))),
+              left, right, BuildRight, leftSemiPlus, condition))),
           expectedAnswer,
           sortAnswers = true)
       }
     }
   }
 
+  testExistenceJoin(
+    "test no condition with non-empty right side for left semi join",
+    LeftSemi,
+    left,
+    right,
+    None,
+    Seq(Row(1, 2.0), Row(1, 2.0), Row(2, 1.0), Row(2, 1.0), Row(3, 3.0), Row(null, null),
+      Row(null, 5.0), Row(6, null)))
+
+  testExistenceJoin(
+    "test no condition with empty right side for left semi join",
+    LeftSemi,
+    left,
+    spark.emptyDataFrame,
+    None,
+    Seq.empty)
+
   testExistenceJoin(
     "test single condition (equal) for left semi join",
     LeftSemi,
     left,
     right,
-    singleConditionEQ,
+    Some(singleConditionEQ),
+    Seq(Row(2, 1.0), Row(2, 1.0), Row(3, 3.0), Row(6, null)))
+
+  testExistenceJoin(
+    "test single unique condition (equal) for left semi join",
+    LeftSemi,
+    left,
+    right.select(right.col("c")).distinct(), /* Trigger BHJs and SHJs unique key code path! */
+    Some(singleConditionEQ),
     Seq(Row(2, 1.0), Row(2, 1.0), Row(3, 3.0), Row(6, null)))
 
   testExistenceJoin(
@@ -204,7 +230,7 @@ class ExistenceJoinSuite extends SparkPlanTest with SharedSparkSession {
     LeftSemi,
     left,
     right,
-    composedConditionEQ,
+    Some(composedConditionEQ),
     Seq(Row(2, 1.0), Row(2, 1.0)))
 
   testExistenceJoin(
@@ -212,47 +238,65 @@ class ExistenceJoinSuite extends SparkPlanTest with SharedSparkSession {
     LeftSemi,
     left,
     right,
-    composedConditionNEQ,
+    Some(composedConditionNEQ),
     Seq(Row(1, 2.0), Row(1, 2.0), Row(2, 1.0), Row(2, 1.0)))
 
   testExistenceJoin(
-    "test single condition (equal) for left Anti join",
+    "test no condition with non-empty right side for left anti join",
+    LeftAnti,
+    left,
+    right,
+    None,
+    Seq.empty)
+
+  testExistenceJoin(
+    "test no condition with empty right side for left anti join",
+    LeftAnti,
+    left,
+    spark.emptyDataFrame,
+    None,
+    Seq(Row(1, 2.0), Row(1, 2.0), Row(2, 1.0), Row(2, 1.0), Row(3, 3.0), Row(null, null),
+      Row(null, 5.0), Row(6, null)))
+
+  testExistenceJoin(
+    "test single condition (equal) for left anti join",
     LeftAnti,
     left,
     right,
-    singleConditionEQ,
+    Some(singleConditionEQ),
     Seq(Row(1, 2.0), Row(1, 2.0), Row(null, null), Row(null, 5.0)))
 
 
   testExistenceJoin(
-    "test single unique condition (equal) for left Anti join",
+    "test single unique condition (equal) for left anti join",
     LeftAnti,
     left,
-    right.select(right.col("c")).distinct(), /* Trigger BHJs unique key code path! */
-    singleConditionEQ,
+    right.select(right.col("c")).distinct(), /* Trigger BHJs and SHJs unique key code path! */
+    Some(singleConditionEQ),
     Seq(Row(1, 2.0), Row(1, 2.0), Row(null, null), Row(null, 5.0)))
 
   testExistenceJoin(
-    "test composed condition (equal & non-equal) test for anti join",
+    "test composed condition (equal & non-equal) test for left anti join",
     LeftAnti,
     left,
     right,
-    composedConditionEQ,
+    Some(composedConditionEQ),
     Seq(Row(1, 2.0), Row(1, 2.0), Row(3, 3.0), Row(6, null), Row(null, 5.0), Row(null, null)))
 
   testExistenceJoin(
-    "test composed condition (both non-equal) for anti join",
+    "test composed condition (both non-equal) for left anti join",
     LeftAnti,
     left,
     right,
-    composedConditionNEQ,
+    Some(composedConditionNEQ),
     Seq(Row(3, 3.0), Row(6, null), Row(null, 5.0), Row(null, null)))
 
   testExistenceJoin(
-    "test composed unique condition (both non-equal) for anti join",
+    "test composed unique condition (both non-equal) for left anti join",
     LeftAnti,
     left,
     rightUniqueKey,
-    (left.col("a") === rightUniqueKey.col("c") && left.col("b") < rightUniqueKey.col("d")).expr,
+    Some((left.col("a") === rightUniqueKey.col("c") && left.col("b") < rightUniqueKey.col("d"))
+      .expr),
     Seq(Row(1, 2.0), Row(1, 2.0), Row(3, 3.0), Row(null, null), Row(null, 5.0), Row(6, null)))
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala
index 3526aa254c280..84f62993eba2b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.execution.joins
 
 import java.io.{ByteArrayInputStream, ByteArrayOutputStream, ObjectInputStream, ObjectOutputStream}
 
+import scala.collection.mutable
 import scala.collection.mutable.ArrayBuffer
 import scala.util.Random
 
@@ -144,6 +145,7 @@ class HashedRelationSuite extends SharedSparkSession {
     }
 
     val longRelation2 = LongHashedRelation(rows.iterator ++ rows.iterator, key, 100, mm)
+        .asInstanceOf[LongHashedRelation]
     assert(!longRelation2.keyIsUnique)
     (0 until 100).foreach { i =>
       val rows = longRelation2.get(i).toArray
@@ -358,6 +360,45 @@ class HashedRelationSuite extends SharedSparkSession {
     assert(java.util.Arrays.equals(os.toByteArray, os2.toByteArray))
   }
 
+  test("SPARK-31511: Make BytesToBytesMap iterators thread-safe") {
+    val ser = sparkContext.env.serializer.newInstance()
+    val key = Seq(BoundReference(0, LongType, false))
+
+    val unsafeProj = UnsafeProjection.create(
+      Seq(BoundReference(0, LongType, false), BoundReference(1, IntegerType, true)))
+    val rows = (0 until 10000).map(i => unsafeProj(InternalRow(Int.int2long(i), i + 1)).copy())
+    val unsafeHashed = UnsafeHashedRelation(rows.iterator, key, 1, mm)
+
+    val os = new ByteArrayOutputStream()
+    val thread1 = new Thread {
+      override def run(): Unit = {
+        val out = new ObjectOutputStream(os)
+        unsafeHashed.asInstanceOf[UnsafeHashedRelation].writeExternal(out)
+        out.flush()
+      }
+    }
+
+    val thread2 = new Thread {
+      override def run(): Unit = {
+        val threadOut = new ObjectOutputStream(new ByteArrayOutputStream())
+        unsafeHashed.asInstanceOf[UnsafeHashedRelation].writeExternal(threadOut)
+        threadOut.flush()
+      }
+    }
+
+    thread1.start()
+    thread2.start()
+    thread1.join()
+    thread2.join()
+
+    val unsafeHashed2 = ser.deserialize[UnsafeHashedRelation](ser.serialize(unsafeHashed))
+    val os2 = new ByteArrayOutputStream()
+    val out2 = new ObjectOutputStream(os2)
+    unsafeHashed2.writeExternal(out2)
+    out2.flush()
+    assert(java.util.Arrays.equals(os.toByteArray, os2.toByteArray))
+  }
+
   // This test require 4G heap to run, should run it manually
   ignore("build HashedRelation that is larger than 1G") {
     val unsafeProj = UnsafeProjection.create(
@@ -579,4 +620,98 @@ class HashedRelationSuite extends SharedSparkSession {
       assert(proj(packedKeys).get(0, dt) == -i - 1)
     }
   }
+
+  test("EmptyHashedRelation override methods behavior test") {
+    val buildKey = Seq(BoundReference(0, LongType, false))
+    val hashed = HashedRelation(Seq.empty[InternalRow].toIterator, buildKey, 1, mm)
+    assert(hashed == EmptyHashedRelation)
+
+    val key = InternalRow(1L)
+    assert(hashed.get(0L) == null)
+    assert(hashed.get(key) == null)
+    assert(hashed.getValue(0L) == null)
+    assert(hashed.getValue(key) == null)
+
+    assert(hashed.keys().isEmpty)
+    assert(hashed.keyIsUnique)
+    assert(hashed.estimatedSize == 0)
+  }
+
+  test("SPARK-32399: test methods related to key index") {
+    val schema = StructType(StructField("a", IntegerType, true) :: Nil)
+    val toUnsafe = UnsafeProjection.create(schema)
+    val key = Seq(BoundReference(0, IntegerType, true))
+    val row = Seq(BoundReference(0, IntegerType, true), BoundReference(1, IntegerType, true))
+    val unsafeProj = UnsafeProjection.create(row)
+    var rows = (0 until 100).map(i => {
+      val k = if (i % 10 == 0) null else i % 10
+      unsafeProj(InternalRow(k, i)).copy()
+    })
+    rows = unsafeProj(InternalRow(-1, -1)).copy() +: rows
+    val unsafeRelation = UnsafeHashedRelation(rows.iterator, key, 10, mm, allowsNullKey = true)
+    val keyIndexToKeyMap = new mutable.HashMap[Int, String]
+    val keyIndexToValueMap = new mutable.HashMap[Int, Seq[Int]]
+
+    // test getWithKeyIndex()
+    (0 until 10).foreach(i => {
+      val key = if (i == 0) InternalRow(null) else InternalRow(i)
+      val valuesWithKeyIndex = unsafeRelation.getWithKeyIndex(toUnsafe(key)).map(
+        v => (v.getKeyIndex, v.getValue.getInt(1))).toArray
+      val keyIndex = valuesWithKeyIndex.head._1
+      val actualValues = valuesWithKeyIndex.map(_._2)
+      val expectedValues = (0 until 10).map(j => j * 10 + i)
+      if (i == 0) {
+        keyIndexToKeyMap(keyIndex) = "null"
+      } else {
+        keyIndexToKeyMap(keyIndex) = i.toString
+      }
+      keyIndexToValueMap(keyIndex) = actualValues
+      // key index is non-negative
+      assert(keyIndex >= 0)
+      // values are expected
+      assert(actualValues.sortWith(_ < _) === expectedValues)
+    })
+    // key index is unique per key
+    val numUniqueKeyIndex = (0 until 10).flatMap(i => {
+      val key = if (i == 0) InternalRow(null) else InternalRow(i)
+      val keyIndex = unsafeRelation.getWithKeyIndex(toUnsafe(key)).map(_.getKeyIndex).toSeq
+      keyIndex
+    }).distinct.size
+    assert(numUniqueKeyIndex == 10)
+    // NULL for non-existing key
+    assert(unsafeRelation.getWithKeyIndex(toUnsafe(InternalRow(100))) == null)
+
+    // test getValueWithKeyIndex()
+    val valuesWithKeyIndex = unsafeRelation.getValueWithKeyIndex(toUnsafe(InternalRow(-1)))
+    val keyIndex = valuesWithKeyIndex.getKeyIndex
+    keyIndexToKeyMap(keyIndex) = "-1"
+    keyIndexToValueMap(keyIndex) = Seq(-1)
+    // key index is non-negative
+    assert(valuesWithKeyIndex.getKeyIndex >= 0)
+    // value is expected
+    assert(valuesWithKeyIndex.getValue.getInt(1) == -1)
+    // NULL for non-existing key
+    assert(unsafeRelation.getValueWithKeyIndex(toUnsafe(InternalRow(100))) == null)
+
+    // test valuesWithKeyIndex()
+    val keyIndexToRowMap = unsafeRelation.valuesWithKeyIndex().map(
+      v => (v.getKeyIndex, v.getValue.copy())).toSeq.groupBy(_._1)
+    assert(keyIndexToRowMap.size == 11)
+    keyIndexToRowMap.foreach {
+      case (keyIndex, row) =>
+        val expectedKey = keyIndexToKeyMap(keyIndex)
+        val expectedValues = keyIndexToValueMap(keyIndex)
+        // key index returned from valuesWithKeyIndex()
+        // should be the same as returned from getWithKeyIndex()
+        if (expectedKey == "null") {
+          assert(row.head._2.isNullAt(0))
+        } else {
+          assert(row.head._2.getInt(0).toString == expectedKey)
+        }
+        // values returned from valuesWithKeyIndex()
+        // should have same value and order as returned from getWithKeyIndex()
+        val actualValues = row.map(_._2.getInt(1))
+        assert(actualValues === expectedValues)
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/InnerJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/InnerJoinSuite.scala
index 08898f80034e6..5262320134319 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/InnerJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/InnerJoinSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.execution.joins
 
 import org.apache.spark.sql.{DataFrame, Row}
 import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide}
 import org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys
 import org.apache.spark.sql.catalyst.plans.Inner
 import org.apache.spark.sql.catalyst.plans.logical.{Join, JoinHint}
@@ -100,7 +101,7 @@ class InnerJoinSuite extends SparkPlanTest with SharedSparkSession {
         boundCondition,
         leftPlan,
         rightPlan)
-      EnsureRequirements(spark.sessionState.conf).apply(broadcastJoin)
+      EnsureRequirements.apply(broadcastJoin)
     }
 
     def makeShuffledHashJoin(
@@ -114,7 +115,7 @@ class InnerJoinSuite extends SparkPlanTest with SharedSparkSession {
         side, None, leftPlan, rightPlan)
       val filteredJoin =
         boundCondition.map(FilterExec(_, shuffledHashJoin)).getOrElse(shuffledHashJoin)
-      EnsureRequirements(spark.sessionState.conf).apply(filteredJoin)
+      EnsureRequirements.apply(filteredJoin)
     }
 
     def makeSortMergeJoin(
@@ -125,7 +126,7 @@ class InnerJoinSuite extends SparkPlanTest with SharedSparkSession {
         rightPlan: SparkPlan) = {
       val sortMergeJoin = joins.SortMergeJoinExec(leftKeys, rightKeys, Inner, boundCondition,
         leftPlan, rightPlan)
-      EnsureRequirements(spark.sessionState.conf).apply(sortMergeJoin)
+      EnsureRequirements.apply(sortMergeJoin)
     }
 
     testWithWholeStageCodegenOnAndOff(s"$testName using BroadcastHashJoin (build=left)") { _ =>
@@ -133,7 +134,7 @@ class InnerJoinSuite extends SparkPlanTest with SharedSparkSession {
         withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
           checkAnswer2(leftRows, rightRows, (leftPlan: SparkPlan, rightPlan: SparkPlan) =>
             makeBroadcastHashJoin(
-              leftKeys, rightKeys, boundCondition, leftPlan, rightPlan, joins.BuildLeft),
+              leftKeys, rightKeys, boundCondition, leftPlan, rightPlan, BuildLeft),
             expectedAnswer.map(Row.fromTuple),
             sortAnswers = true)
         }
@@ -145,31 +146,31 @@ class InnerJoinSuite extends SparkPlanTest with SharedSparkSession {
         withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
           checkAnswer2(leftRows, rightRows, (leftPlan: SparkPlan, rightPlan: SparkPlan) =>
             makeBroadcastHashJoin(
-              leftKeys, rightKeys, boundCondition, leftPlan, rightPlan, joins.BuildRight),
+              leftKeys, rightKeys, boundCondition, leftPlan, rightPlan, BuildRight),
             expectedAnswer.map(Row.fromTuple),
             sortAnswers = true)
         }
       }
     }
 
-    test(s"$testName using ShuffledHashJoin (build=left)") {
+    testWithWholeStageCodegenOnAndOff(s"$testName using ShuffledHashJoin (build=left)") { _ =>
       extractJoinParts().foreach { case (_, leftKeys, rightKeys, boundCondition, _, _, _) =>
         withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
           checkAnswer2(leftRows, rightRows, (leftPlan: SparkPlan, rightPlan: SparkPlan) =>
             makeShuffledHashJoin(
-              leftKeys, rightKeys, boundCondition, leftPlan, rightPlan, joins.BuildLeft),
+              leftKeys, rightKeys, boundCondition, leftPlan, rightPlan, BuildLeft),
             expectedAnswer.map(Row.fromTuple),
             sortAnswers = true)
         }
       }
     }
 
-    test(s"$testName using ShuffledHashJoin (build=right)") {
+    testWithWholeStageCodegenOnAndOff(s"$testName using ShuffledHashJoin (build=right)") { _ =>
       extractJoinParts().foreach { case (_, leftKeys, rightKeys, boundCondition, _, _, _) =>
         withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
           checkAnswer2(leftRows, rightRows, (leftPlan: SparkPlan, rightPlan: SparkPlan) =>
             makeShuffledHashJoin(
-              leftKeys, rightKeys, boundCondition, leftPlan, rightPlan, joins.BuildRight),
+              leftKeys, rightKeys, boundCondition, leftPlan, rightPlan, BuildRight),
             expectedAnswer.map(Row.fromTuple),
             sortAnswers = true)
         }
@@ -197,7 +198,7 @@ class InnerJoinSuite extends SparkPlanTest with SharedSparkSession {
       }
     }
 
-    test(s"$testName using BroadcastNestedLoopJoin build left") {
+    testWithWholeStageCodegenOnAndOff(s"$testName using BroadcastNestedLoopJoin build left") { _ =>
       withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
         checkAnswer2(leftRows, rightRows, (left: SparkPlan, right: SparkPlan) =>
           BroadcastNestedLoopJoinExec(left, right, BuildLeft, Inner, Some(condition())),
@@ -206,7 +207,7 @@ class InnerJoinSuite extends SparkPlanTest with SharedSparkSession {
       }
     }
 
-    test(s"$testName using BroadcastNestedLoopJoin build right") {
+    testWithWholeStageCodegenOnAndOff(s"$testName using BroadcastNestedLoopJoin build right") { _ =>
       withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
         checkAnswer2(leftRows, rightRows, (left: SparkPlan, right: SparkPlan) =>
           BroadcastNestedLoopJoinExec(left, right, BuildRight, Inner, Some(condition())),
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala
index a5ade0d8d7508..744ee1ca1ad23 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.execution.joins
 
 import org.apache.spark.sql.{DataFrame, Row}
 import org.apache.spark.sql.catalyst.expressions.{And, Expression, LessThan}
+import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight}
 import org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical.{Join, JoinHint}
@@ -46,7 +47,7 @@ class OuterJoinSuite extends SparkPlanTest with SharedSparkSession {
     sparkContext.parallelize(Seq(
       Row(0, 0.0),
       Row(2, 3.0), // This row is duplicated to ensure that we will have multiple buffered matches
-      Row(2, -1.0),
+      Row(2, -1.0), // This row is duplicated to ensure that we will have multiple buffered matches
       Row(2, -1.0),
       Row(2, 3.0),
       Row(3, 2.0),
@@ -61,6 +62,32 @@ class OuterJoinSuite extends SparkPlanTest with SharedSparkSession {
       LessThan(left.col("b").expr, right.col("d").expr))
   }
 
+  private lazy val uniqueLeft = spark.createDataFrame(
+    sparkContext.parallelize(Seq(
+      Row(1, 2.0),
+      Row(2, 1.0),
+      Row(3, 3.0),
+      Row(5, 1.0),
+      Row(6, 6.0),
+      Row(null, null)
+    )), new StructType().add("a", IntegerType).add("b", DoubleType))
+
+  private lazy val uniqueRight = spark.createDataFrame(
+    sparkContext.parallelize(Seq(
+      Row(0, 0.0),
+      Row(2, 3.0),
+      Row(3, 2.0),
+      Row(4, 1.0),
+      Row(5, 3.0),
+      Row(7, 7.0),
+      Row(null, null)
+    )), new StructType().add("c", IntegerType).add("d", DoubleType))
+
+  private lazy val uniqueCondition = {
+    And((uniqueLeft.col("a") === uniqueRight.col("c")).expr,
+      LessThan(uniqueLeft.col("b").expr, uniqueRight.col("d").expr))
+  }
+
   // Note: the input dataframes and expression must be evaluated lazily because
   // the SQLContext should be used only within a test to keep SQL tests stable
   private def testOuterJoin(
@@ -77,18 +104,16 @@ class OuterJoinSuite extends SparkPlanTest with SharedSparkSession {
       ExtractEquiJoinKeys.unapply(join)
     }
 
-    if (joinType != FullOuter) {
-      test(s"$testName using ShuffledHashJoin") {
-        extractJoinParts().foreach { case (_, leftKeys, rightKeys, boundCondition, _, _, _) =>
-          withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
-            val buildSide = if (joinType == LeftOuter) BuildRight else BuildLeft
-            checkAnswer2(leftRows, rightRows, (left: SparkPlan, right: SparkPlan) =>
-              EnsureRequirements(spark.sessionState.conf).apply(
-                ShuffledHashJoinExec(
-                  leftKeys, rightKeys, joinType, buildSide, boundCondition, left, right)),
-              expectedAnswer.map(Row.fromTuple),
-              sortAnswers = true)
-          }
+    testWithWholeStageCodegenOnAndOff(s"$testName using ShuffledHashJoin") { _ =>
+      extractJoinParts().foreach { case (_, leftKeys, rightKeys, boundCondition, _, _, _) =>
+        withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
+          val buildSide = if (joinType == LeftOuter) BuildRight else BuildLeft
+          checkAnswer2(leftRows, rightRows, (left: SparkPlan, right: SparkPlan) =>
+            EnsureRequirements.apply(
+              ShuffledHashJoinExec(
+                leftKeys, rightKeys, joinType, buildSide, boundCondition, left, right)),
+            expectedAnswer.map(Row.fromTuple),
+            sortAnswers = true)
         }
       }
     }
@@ -112,11 +137,11 @@ class OuterJoinSuite extends SparkPlanTest with SharedSparkSession {
       }
     }
 
-    test(s"$testName using SortMergeJoin") {
+    testWithWholeStageCodegenOnAndOff(s"$testName using SortMergeJoin") { _ =>
       extractJoinParts().foreach { case (_, leftKeys, rightKeys, boundCondition, _, _, _) =>
         withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
           checkAnswer2(leftRows, rightRows, (left: SparkPlan, right: SparkPlan) =>
-            EnsureRequirements(spark.sessionState.conf).apply(
+            EnsureRequirements.apply(
               SortMergeJoinExec(leftKeys, rightKeys, joinType, boundCondition, left, right)),
             expectedAnswer.map(Row.fromTuple),
             sortAnswers = true)
@@ -124,7 +149,7 @@ class OuterJoinSuite extends SparkPlanTest with SharedSparkSession {
       }
     }
 
-    test(s"$testName using BroadcastNestedLoopJoin build left") {
+    testWithWholeStageCodegenOnAndOff(s"$testName using BroadcastNestedLoopJoin build left") { _ =>
       withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
         checkAnswer2(leftRows, rightRows, (left: SparkPlan, right: SparkPlan) =>
           BroadcastNestedLoopJoinExec(left, right, BuildLeft, joinType, Some(condition)),
@@ -133,7 +158,7 @@ class OuterJoinSuite extends SparkPlanTest with SharedSparkSession {
       }
     }
 
-    test(s"$testName using BroadcastNestedLoopJoin build right") {
+    testWithWholeStageCodegenOnAndOff(s"$testName using BroadcastNestedLoopJoin build right") { _ =>
       withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
         checkAnswer2(leftRows, rightRows, (left: SparkPlan, right: SparkPlan) =>
           BroadcastNestedLoopJoinExec(left, right, BuildRight, joinType, Some(condition)),
@@ -242,4 +267,39 @@ class OuterJoinSuite extends SparkPlanTest with SharedSparkSession {
     condition,
     Seq.empty
   )
+
+  // --- Join keys are unique ---------------------------------------------------------------------
+
+  testOuterJoin(
+    "left outer join with unique keys",
+    uniqueLeft,
+    uniqueRight,
+    LeftOuter,
+    uniqueCondition,
+    Seq(
+      (null, null, null, null),
+      (1, 2.0, null, null),
+      (2, 1.0, 2, 3.0),
+      (3, 3.0, null, null),
+      (5, 1.0, 5, 3.0),
+      (6, 6.0, null, null)
+    )
+  )
+
+  testOuterJoin(
+    "right outer join with unique keys",
+    uniqueLeft,
+    uniqueRight,
+    RightOuter,
+    uniqueCondition,
+    Seq(
+      (null, null, null, null),
+      (null, null, 0, 0.0),
+      (2, 1.0, 2, 3.0),
+      (null, null, 3, 2.0),
+      (null, null, 4, 1.0),
+      (5, 1.0, 5, 3.0),
+      (null, null, 7, 7.0)
+    )
+  )
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/CustomMetricsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/CustomMetricsSuite.scala
new file mode 100644
index 0000000000000..440b0dc08ecbd
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/CustomMetricsSuite.scala
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.metric
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.connector.metric.{CustomAvgMetric, CustomSumMetric}
+
+class CustomMetricsSuite extends SparkFunSuite {
+
+  test("Build/parse custom metric metric type") {
+    Seq(new TestCustomSumMetric, new TestCustomAvgMetric).foreach { customMetric =>
+      val metricType = CustomMetrics.buildV2CustomMetricTypeName(customMetric)
+
+      assert(metricType == CustomMetrics.V2_CUSTOM + "_" + customMetric.getClass.getCanonicalName)
+      assert(CustomMetrics.parseV2CustomMetricType(metricType).isDefined)
+      assert(CustomMetrics.parseV2CustomMetricType(metricType).get ==
+        customMetric.getClass.getCanonicalName)
+    }
+  }
+
+  test("Built-in CustomSumMetric") {
+    val metric = new TestCustomSumMetric
+
+    val metricValues1 = Array(0L, 1L, 5L, 5L, 7L, 10L)
+    assert(metric.aggregateTaskMetrics(metricValues1) == metricValues1.sum.toString)
+
+    val metricValues2 = Array.empty[Long]
+    assert(metric.aggregateTaskMetrics(metricValues2) == "0")
+  }
+
+  test("Built-in CustomAvgMetric") {
+    val metric = new TestCustomAvgMetric
+
+    val metricValues1 = Array(0L, 1L, 5L, 5L, 7L, 10L)
+    assert(metric.aggregateTaskMetrics(metricValues1) == "4.667")
+
+    val metricValues2 = Array.empty[Long]
+    assert(metric.aggregateTaskMetrics(metricValues2) == "0")
+  }
+}
+
+private[spark] class TestCustomSumMetric extends CustomSumMetric {
+  override def name(): String = "CustomSumMetric"
+  override def description(): String = "Sum up CustomMetric"
+}
+
+private[spark] class TestCustomAvgMetric extends CustomAvgMetric {
+  override def name(): String = "CustomAvgMetric"
+  override def description(): String = "Average CustomMetric"
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
index 08fb655bde467..1e7032ab07425 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
@@ -28,7 +28,9 @@ import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
 import org.apache.spark.sql.execution.{FilterExec, RangeExec, SparkPlan, WholeStageCodegenExec}
 import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecutionSuite
 import org.apache.spark.sql.execution.aggregate.HashAggregateExec
-import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
+import org.apache.spark.sql.execution.command.DataWritingCommandExec
+import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ShuffleExchangeExec}
+import org.apache.spark.sql.execution.joins.ShuffledHashJoinExec
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
@@ -166,7 +168,7 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils
     //           Exchange(nodeId = 5)
     //             LocalTableScan(nodeId = 6)
     Seq(true, false).foreach { enableWholeStage =>
-      val df = generateRandomBytesDF().repartition(1).groupBy('a).count()
+      val df = generateRandomBytesDF().repartition(2).groupBy('a).count()
       val nodeIds = if (enableWholeStage) {
         Set(4L, 1L)
       } else {
@@ -180,7 +182,7 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils
           assert(probes.toDouble > 1.0)
         } else {
           val mainValue = probes.split("\n").apply(1).stripPrefix("(").stripSuffix(")")
-          // Extract min, med, max from the string and strip off everthing else.
+          // Extract min, med, max from the string and strip off everything else.
           val index = mainValue.indexOf(" (", 0)
           mainValue.slice(0, index).split(", ").foreach {
             probe => assert(probe.toDouble > 1.0)
@@ -208,14 +210,41 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils
     // 2 partitions and each partition contains 2 keys
     val df2 = testData2.groupBy('a).agg(collect_set('a))
     testSparkPlanMetrics(df2, 1, Map(
-      2L -> (("ObjectHashAggregate", Map("number of output rows" -> 4L))),
+      2L -> (("ObjectHashAggregate", Map(
+        "number of output rows" -> 4L,
+        "number of tasks fall-backed to sort-based aggregation" -> 0L))),
       1L -> (("Exchange", Map(
         "shuffle records written" -> 4L,
         "records read" -> 4L,
         "local blocks read" -> 4L,
         "remote blocks read" -> 0L))),
-      0L -> (("ObjectHashAggregate", Map("number of output rows" -> 3L))))
+      0L -> (("ObjectHashAggregate", Map(
+        "number of output rows" -> 3L,
+        "number of tasks fall-backed to sort-based aggregation" -> 0L))))
     )
+
+    // 2 partitions and each partition contains 2 keys, with fallback to sort-based aggregation
+    withSQLConf(SQLConf.OBJECT_AGG_SORT_BASED_FALLBACK_THRESHOLD.key -> "1") {
+      val df3 = testData2.groupBy('a).agg(collect_set('a))
+      testSparkPlanMetrics(df3, 1, Map(
+        2L -> (("ObjectHashAggregate", Map(
+          "number of output rows" -> 4L,
+          "number of tasks fall-backed to sort-based aggregation" -> 2L))),
+        0L -> (("ObjectHashAggregate", Map(
+          "number of output rows" -> 3L,
+          "number of tasks fall-backed to sort-based aggregation" -> 1L))))
+      )
+      testSparkPlanMetricsWithPredicates(df3, 1, Map(
+        2L -> (("ObjectHashAggregate", Map(
+          "spill size" -> {
+            _.toString.matches(sizeMetricPattern)
+          }))),
+        0L -> (("ObjectHashAggregate", Map(
+          "spill size" -> {
+            _.toString.matches(sizeMetricPattern)
+          }))))
+      )
+    }
   }
 
   test("Sort metrics") {
@@ -280,11 +309,11 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils
       val rightJoinQuery = "SELECT * FROM testDataForJoin right JOIN testData2 ON " +
         "testData2.a = testDataForJoin.a"
 
-      Seq((leftJoinQuery, false), (leftJoinQuery, true), (rightJoinQuery, false),
-        (rightJoinQuery, true)).foreach { case (query, enableWholeStage) =>
+      Seq((leftJoinQuery, 0L, false), (leftJoinQuery, 1L, true), (rightJoinQuery, 0L, false),
+        (rightJoinQuery, 1L, true)).foreach { case (query, nodeId, enableWholeStage) =>
         val df = spark.sql(query)
         testSparkPlanMetrics(df, 1, Map(
-          0L -> (("SortMergeJoin", Map(
+          nodeId -> (("SortMergeJoin", Map(
             // It's 8 because we read 6 rows in the left and 2 row in the right one
             "number of output rows" -> 8L)))),
           enableWholeStage
@@ -310,8 +339,8 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils
 
   test("ShuffledHashJoin metrics") {
     withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "40",
-        SQLConf.SHUFFLE_PARTITIONS.key -> "2",
-        SQLConf.PREFER_SORTMERGEJOIN.key -> "false") {
+      SQLConf.SHUFFLE_PARTITIONS.key -> "2",
+      SQLConf.PREFER_SORTMERGEJOIN.key -> "false") {
       val df1 = Seq((1, "1"), (2, "2")).toDF("key", "value")
       val df2 = (1 to 10).map(i => (i, i.toString)).toSeq.toDF("key", "value")
       // Assume the execution plan is
@@ -325,30 +354,84 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils
       // +- LocalTableScan(nodeId = 7)
       Seq((1L, 2L, 5L, false), (2L, 3L, 7L, true)).foreach {
         case (nodeId1, nodeId2, nodeId3, enableWholeStage) =>
-        val df = df1.join(df2, "key")
+          val df = df1.join(df2, "key")
+          testSparkPlanMetrics(df, 1, Map(
+            nodeId1 -> (("ShuffledHashJoin", Map(
+              "number of output rows" -> 2L))),
+            nodeId2 -> (("Exchange", Map(
+              "shuffle records written" -> 2L,
+              "records read" -> 2L))),
+            nodeId3 -> (("Exchange", Map(
+              "shuffle records written" -> 10L,
+              "records read" -> 10L)))),
+            enableWholeStage
+          )
+      }
+    }
+  }
+
+  test("ShuffledHashJoin(left, outer) metrics") {
+    val leftDf = Seq((1, "1"), (2, "2")).toDF("key", "value")
+    val rightDf = (1 to 10).map(i => (i, i.toString)).toSeq.toDF("key2", "value")
+    Seq((0L, "right_outer", leftDf, rightDf, 10L, false),
+      (0L, "left_outer", rightDf, leftDf, 10L, false),
+      (1L, "right_outer", leftDf, rightDf, 10L, true),
+      (1L, "left_outer", rightDf, leftDf, 10L, true),
+      (2L, "left_anti", rightDf, leftDf, 8L, true),
+      (2L, "left_semi", rightDf, leftDf, 2L, true),
+      (1L, "left_anti", rightDf, leftDf, 8L, false),
+      (1L, "left_semi", rightDf, leftDf, 2L, false))
+      .foreach { case (nodeId, joinType, leftDf, rightDf, rows, enableWholeStage) =>
+        val df = leftDf.hint("shuffle_hash").join(
+          rightDf.hint("shuffle_hash"), $"key" === $"key2", joinType)
         testSparkPlanMetrics(df, 1, Map(
-          nodeId1 -> (("ShuffledHashJoin", Map(
-            "number of output rows" -> 2L))),
-          nodeId2 -> (("Exchange", Map(
-            "shuffle records written" -> 2L,
-            "records read" -> 2L))),
-          nodeId3 -> (("Exchange", Map(
-            "shuffle records written" -> 10L,
-            "records read" -> 10L)))),
+          nodeId -> (("ShuffledHashJoin", Map(
+            "number of output rows" -> rows)))),
           enableWholeStage
         )
       }
+  }
+
+  test("SPARK-32629: ShuffledHashJoin(full outer) metrics") {
+    val uniqueLeftDf = Seq(("1", "1"), ("11", "11")).toDF("key", "value")
+    val nonUniqueLeftDf = Seq(("1", "1"), ("1", "2"), ("11", "11")).toDF("key", "value")
+    val rightDf = (1 to 10).map(i => (i.toString, i.toString)).toDF("key2", "value")
+    Seq(
+      // Test unique key on build side
+      (uniqueLeftDf, rightDf, 11, 134228048, 10, 134221824),
+      // Test non-unique key on build side
+      (nonUniqueLeftDf, rightDf, 12, 134228552, 11, 134221824)
+    ).foreach { case (leftDf, rightDf, fojRows, fojBuildSize, rojRows, rojBuildSize) =>
+      val fojDf = leftDf.hint("shuffle_hash").join(
+        rightDf, $"key" === $"key2", "full_outer")
+      fojDf.collect()
+      val fojPlan = fojDf.queryExecution.executedPlan.collectFirst {
+        case s: ShuffledHashJoinExec => s
+      }
+      assert(fojPlan.isDefined, "The query plan should have shuffled hash join")
+      testMetricsInSparkPlanOperator(fojPlan.get,
+        Map("numOutputRows" -> fojRows, "buildDataSize" -> fojBuildSize))
+
+      // Test right outer join as well to verify build data size to be different
+      // from full outer join. This makes sure we take extra BitSet/OpenHashSet
+      // for full outer join into account.
+      val rojDf = leftDf.hint("shuffle_hash").join(
+        rightDf, $"key" === $"key2", "right_outer")
+      rojDf.collect()
+      val rojPlan = rojDf.queryExecution.executedPlan.collectFirst {
+        case s: ShuffledHashJoinExec => s
+      }
+      assert(rojPlan.isDefined, "The query plan should have shuffled hash join")
+      testMetricsInSparkPlanOperator(rojPlan.get,
+        Map("numOutputRows" -> rojRows, "buildDataSize" -> rojBuildSize))
     }
   }
 
   test("BroadcastHashJoin(outer) metrics") {
     val df1 = Seq((1, "a"), (1, "b"), (4, "c")).toDF("key", "value")
     val df2 = Seq((1, "a"), (1, "b"), (2, "c"), (3, "d")).toDF("key2", "value")
-    // Assume the execution plan is
-    // ... -> BroadcastHashJoin(nodeId = 0)
-    Seq(("left_outer", 0L, 5L, false), ("right_outer", 0L, 6L, false),
-      ("left_outer", 1L, 5L, true), ("right_outer", 1L, 6L, true)).foreach {
-      case (joinType, nodeId, numRows, enableWholeStage) =>
+    Seq(("left_outer", 0L, 5L, false), ("right_outer", 0L, 6L, false), ("left_outer", 1L, 5L, true),
+      ("right_outer", 1L, 6L, true)).foreach { case (joinType, nodeId, numRows, enableWholeStage) =>
       val df = df1.join(broadcast(df2), $"key" === $"key2", joinType)
       testSparkPlanMetrics(df, 2, Map(
         nodeId -> (("BroadcastHashJoin", Map(
@@ -365,12 +448,15 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils
       withTempView("testDataForJoin") {
         // Assume the execution plan is
         // ... -> BroadcastNestedLoopJoin(nodeId = 1) -> TungstenProject(nodeId = 0)
-        val query = "SELECT * FROM testData2 left JOIN testDataForJoin ON " +
+        val leftQuery = "SELECT * FROM testData2 LEFT JOIN testDataForJoin ON " +
+          "testData2.a * testDataForJoin.a != testData2.a + testDataForJoin.a"
+        val rightQuery = "SELECT * FROM testData2 RIGHT JOIN testDataForJoin ON " +
           "testData2.a * testDataForJoin.a != testData2.a + testDataForJoin.a"
-        Seq(false, true).foreach { enableWholeStage =>
+        Seq((leftQuery, 0L, false), (rightQuery, 0L, false), (leftQuery, 1L, true),
+          (rightQuery, 1L, true)).foreach { case (query, nodeId, enableWholeStage) =>
           val df = spark.sql(query)
           testSparkPlanMetrics(df, 2, Map(
-            0L -> (("BroadcastNestedLoopJoin", Map(
+            nodeId -> (("BroadcastNestedLoopJoin", Map(
               "number of output rows" -> 12L)))),
             enableWholeStage
           )
@@ -394,6 +480,19 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils
     }
   }
 
+  test("BroadcastLeftAntiJoinHash metrics") {
+    val df1 = Seq((1, "1"), (2, "2")).toDF("key", "value")
+    val df2 = Seq((1, "1"), (2, "2"), (3, "3"), (4, "4")).toDF("key2", "value")
+    Seq((1L, false), (2L, true)).foreach { case (nodeId, enableWholeStage) =>
+      val df = df2.join(broadcast(df1), $"key" === $"key2", "left_anti")
+      testSparkPlanMetrics(df, 2, Map(
+        nodeId -> (("BroadcastHashJoin", Map(
+          "number of output rows" -> 2L)))),
+        enableWholeStage
+      )
+    }
+  }
+
   test("CartesianProduct metrics") {
     withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "true") {
       val testDataForJoin = testData2.filter('a < 2) // TestData2(1, 1) :: TestData2(1, 2)
@@ -634,7 +733,7 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils
       sql("CREATE TEMPORARY VIEW inMemoryTable AS SELECT 1 AS c1")
       sql("CACHE TABLE inMemoryTable")
       testSparkPlanMetrics(spark.table("inMemoryTable"), 1,
-        Map(1L -> (("Scan In-memory table `inMemoryTable`", Map.empty)))
+        Map(1L -> (("Scan In-memory table inMemoryTable", Map.empty)))
       )
 
       sql("CREATE TEMPORARY VIEW ```a``b``` AS SELECT 2 AS c1")
@@ -651,16 +750,6 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils
   }
 
   test("SPARK-28332: SQLMetric merge should handle -1 properly") {
-    def checkSparkPlanMetrics(plan: SparkPlan, expected: Map[String, Long]): Unit = {
-      expected.foreach { case (metricName: String, metricValue: Long) =>
-        assert(plan.metrics.contains(metricName), s"The query plan should have metric $metricName")
-        val actualMetric = plan.metrics.get(metricName).get
-        assert(actualMetric.value == metricValue,
-          s"The query plan metric $metricName did not match, " +
-            s"expected:$metricValue, actual:${actualMetric.value}")
-      }
-    }
-
     val df = testData.join(testData2.filter('b === 0), $"key" === $"a", "left_outer")
     df.collect()
     val plan = df.queryExecution.executedPlan
@@ -671,7 +760,43 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils
 
     assert(exchanges.size == 2, "The query plan should have two shuffle exchanges")
 
-    checkSparkPlanMetrics(exchanges(0), Map("dataSize" -> 3200, "shuffleRecordsWritten" -> 100))
-    checkSparkPlanMetrics(exchanges(1), Map("dataSize" -> 0, "shuffleRecordsWritten" -> 0))
+    testMetricsInSparkPlanOperator(exchanges.head,
+      Map("dataSize" -> 3200, "shuffleRecordsWritten" -> 100))
+    testMetricsInSparkPlanOperator(exchanges(1), Map("dataSize" -> 0, "shuffleRecordsWritten" -> 0))
+  }
+
+  test("Add numRows to metric of BroadcastExchangeExec") {
+    withSQLConf(SQLConf.AUTO_SIZE_UPDATE_ENABLED.key -> "true") {
+      withTable("t1", "t2") {
+        spark.range(2).write.saveAsTable("t1")
+        spark.range(2).write.saveAsTable("t2")
+        val df = sql("SELECT t1.* FROM t1 JOIN t2 ON t1.id = t2.id")
+        df.collect()
+        val plan = df.queryExecution.executedPlan
+
+        val exchanges = plan.collect {
+          case s: BroadcastExchangeExec => s
+        }
+
+        assert(exchanges.size === 1)
+        testMetricsInSparkPlanOperator(exchanges.head, Map("numOutputRows" -> 2))
+      }
+    }
+  }
+
+  test("SPARK-34567: Add metrics for CTAS operator") {
+    withTable("t") {
+      val df = sql("CREATE TABLE t USING PARQUET AS SELECT 1 as a")
+      val dataWritingCommandExec =
+        df.queryExecution.executedPlan.asInstanceOf[DataWritingCommandExec]
+      dataWritingCommandExec.executeCollect()
+      val createTableAsSelect = dataWritingCommandExec.cmd
+      assert(createTableAsSelect.metrics.contains("numFiles"))
+      assert(createTableAsSelect.metrics("numFiles").value == 1)
+      assert(createTableAsSelect.metrics.contains("numOutputBytes"))
+      assert(createTableAsSelect.metrics("numOutputBytes").value > 0)
+      assert(createTableAsSelect.metrics.contains("numOutputRows"))
+      assert(createTableAsSelect.metrics("numOutputRows").value == 1)
+    }
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsTestUtils.scala
index 8bbf81efff316..81667d52e16ae 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsTestUtils.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsTestUtils.scala
@@ -25,7 +25,7 @@ import org.apache.spark.TestUtils
 import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd}
 import org.apache.spark.sql.DataFrame
 import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.execution.SparkPlanInfo
+import org.apache.spark.sql.execution.{SparkPlan, SparkPlanInfo}
 import org.apache.spark.sql.execution.ui.{SparkPlanGraph, SQLAppStatusStore}
 import org.apache.spark.sql.internal.SQLConf.WHOLESTAGE_CODEGEN_ENABLED
 import org.apache.spark.sql.test.SQLTestUtils
@@ -220,9 +220,9 @@ trait SQLMetricsTestUtils extends SQLTestUtils {
       (nodeName, nodeMetrics.mapValues(expectedMetricValue =>
         (actualMetricValue: Any) => {
           actualMetricValue.toString.matches(expectedMetricValue.toString)
-        }))
+        }).toMap)
     }
-    testSparkPlanMetricsWithPredicates(df, expectedNumOfJobs, expectedMetricsPredicates,
+    testSparkPlanMetricsWithPredicates(df, expectedNumOfJobs, expectedMetricsPredicates.toMap,
       enableWholeStage)
   }
 
@@ -254,6 +254,24 @@ trait SQLMetricsTestUtils extends SQLTestUtils {
       }
     }
   }
+
+  /**
+   * Verify if the metrics in `SparkPlan` operator are same as expected metrics.
+   *
+   * @param plan `SparkPlan` operator to check metrics
+   * @param expectedMetrics the expected metrics. The format is `metric name -> metric value`.
+   */
+  protected def testMetricsInSparkPlanOperator(
+      plan: SparkPlan,
+      expectedMetrics: Map[String, Long]): Unit = {
+    expectedMetrics.foreach { case (metricName: String, metricValue: Long) =>
+      assert(plan.metrics.contains(metricName), s"The query plan should have metric $metricName")
+      val actualMetric = plan.metrics(metricName)
+      assert(actualMetric.value == metricValue,
+        s"The query plan metric $metricName did not match, " +
+          s"expected:$metricValue, actual:${actualMetric.value}")
+    }
+  }
 }
 
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/python/BatchEvalPythonExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/python/BatchEvalPythonExecSuite.scala
index 5fe3d6a71167e..cb5e23e0534d0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/python/BatchEvalPythonExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/python/BatchEvalPythonExecSuite.scala
@@ -137,6 +137,13 @@ class MyDummyPythonUDF extends UserDefinedPythonFunction(
   pythonEvalType = PythonEvalType.SQL_BATCHED_UDF,
   udfDeterministic = true)
 
+class MyDummyNondeterministicPythonUDF extends UserDefinedPythonFunction(
+  name = "dummyNondeterministicUDF",
+  func = new DummyUDF,
+  dataType = BooleanType,
+  pythonEvalType = PythonEvalType.SQL_BATCHED_UDF,
+  udfDeterministic = false)
+
 class MyDummyGroupedAggPandasUDF extends UserDefinedPythonFunction(
   name = "dummyGroupedAggPandasUDF",
   func = new DummyUDF,
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala
index 87d541d2d22b0..325f4923bd6c6 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala
@@ -28,6 +28,7 @@ class ExtractPythonUDFsSuite extends SparkPlanTest with SharedSparkSession {
   import testImplicits._
 
   val batchedPythonUDF = new MyDummyPythonUDF
+  val batchedNondeterministicPythonUDF = new MyDummyNondeterministicPythonUDF
   val scalarPandasUDF = new MyDummyScalarPandasUDF
 
   private def collectBatchExec(plan: SparkPlan): Seq[BatchEvalPythonExec] = plan.collect {
@@ -166,5 +167,31 @@ class ExtractPythonUDFsSuite extends SparkPlanTest with SharedSparkSession {
     }
   }
 
+  test("SPARK-33303: Deterministic UDF calls are deduplicated") {
+    val df = Seq("Hello").toDF("a")
+
+    val df2 = df.withColumn("c", batchedPythonUDF(col("a"))).withColumn("d", col("c"))
+    val pythonEvalNodes2 = collectBatchExec(df2.queryExecution.executedPlan)
+    assert(pythonEvalNodes2.size == 1)
+    assert(pythonEvalNodes2.head.udfs.size == 1)
+
+    val df3 = df.withColumns(Seq("c", "d"),
+      Seq(batchedPythonUDF(col("a")), batchedPythonUDF(col("a"))))
+    val pythonEvalNodes3 = collectBatchExec(df3.queryExecution.executedPlan)
+    assert(pythonEvalNodes3.size == 1)
+    assert(pythonEvalNodes3.head.udfs.size == 1)
+
+    val df4 = df.withColumn("c", batchedNondeterministicPythonUDF(col("a")))
+      .withColumn("d", col("c"))
+    val pythonEvalNodes4 = collectBatchExec(df4.queryExecution.executedPlan)
+    assert(pythonEvalNodes4.size == 1)
+    assert(pythonEvalNodes4.head.udfs.size == 1)
+
+    val df5 = df.withColumns(Seq("c", "d"),
+      Seq(batchedNondeterministicPythonUDF(col("a")), batchedNondeterministicPythonUDF(col("a"))))
+    val pythonEvalNodes5 = collectBatchExec(df5.queryExecution.executedPlan)
+    assert(pythonEvalNodes5.size == 1)
+    assert(pythonEvalNodes5.head.udfs.size == 2)
+  }
 }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/python/RowQueueSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/python/RowQueueSuite.scala
index 06077c94b66fc..4314e0d0ee380 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/python/RowQueueSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/python/RowQueueSuite.scala
@@ -21,7 +21,7 @@ import java.io.File
 
 import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.internal.config._
-import org.apache.spark.memory.{TaskMemoryManager, TestMemoryManager}
+import org.apache.spark.memory.{MemoryMode, TaskMemoryManager, TestMemoryManager}
 import org.apache.spark.security.{CryptoStreamUtils, EncryptionFunSuite}
 import org.apache.spark.serializer.{JavaSerializer, SerializerManager}
 import org.apache.spark.sql.catalyst.expressions.UnsafeRow
@@ -94,48 +94,54 @@ class RowQueueSuite extends SparkFunSuite with EncryptionFunSuite {
     queue.close()
   }
 
-  encryptionTest("hybrid queue") { conf =>
-    val serManager = createSerializerManager(conf)
-    val mem = new TestMemoryManager(conf)
-    mem.limit(4<<10)
-    val taskM = new TaskMemoryManager(mem, 0)
-    val queue = HybridRowQueue(taskM, Utils.createTempDir().getCanonicalFile, 1, serManager)
-    val row = new UnsafeRow(1)
-    row.pointTo(new Array[Byte](16), 16)
-    val n = (4<<10) / 16 * 3
-    var i = 0
-    while (i < n) {
-      row.setLong(0, i)
-      assert(queue.add(row), "fail to add")
-      i += 1
-    }
-    assert(queue.numQueues() > 1, "should have more than one queue")
-    queue.spill(1<<20, null)
-    i = 0
-    while (i < n) {
-      val row = queue.remove()
-      assert(row != null, "fail to poll")
-      assert(row.getLong(0) == i, "does not match")
-      i += 1
-    }
+  Seq(true, false).foreach { isOffHeap =>
+    encryptionTest(s"hybrid queue (offHeap=$isOffHeap)") { conf =>
+      conf.set(MEMORY_OFFHEAP_ENABLED, isOffHeap)
+      if (isOffHeap) conf.set(MEMORY_OFFHEAP_SIZE, 1000L)
+      val serManager = createSerializerManager(conf)
+      val mem = new TestMemoryManager(conf)
+      mem.limit(4<<10)
+      val taskM = new TaskMemoryManager(mem, 0)
+      val queue = HybridRowQueue(taskM, Utils.createTempDir().getCanonicalFile, 1, serManager)
+      val mode = if (isOffHeap) MemoryMode.OFF_HEAP else MemoryMode.ON_HEAP
+      assert(queue.getMode === mode)
+      val row = new UnsafeRow(1)
+      row.pointTo(new Array[Byte](16), 16)
+      val n = (4<<10) / 16 * 3
+      var i = 0
+      while (i < n) {
+        row.setLong(0, i)
+        assert(queue.add(row), "fail to add")
+        i += 1
+      }
+      assert(queue.numQueues() > 1, "should have more than one queue")
+      queue.spill(1<<20, null)
+      i = 0
+      while (i < n) {
+        val row = queue.remove()
+        assert(row != null, "fail to poll")
+        assert(row.getLong(0) == i, "does not match")
+        i += 1
+      }
 
-    // fill again and spill
-    i = 0
-    while (i < n) {
-      row.setLong(0, i)
-      assert(queue.add(row), "fail to add")
-      i += 1
-    }
-    assert(queue.numQueues() > 1, "should have more than one queue")
-    queue.spill(1<<20, null)
-    assert(queue.numQueues() > 1, "should have more than one queue")
-    i = 0
-    while (i < n) {
-      val row = queue.remove()
-      assert(row != null, "fail to poll")
-      assert(row.getLong(0) == i, "does not match")
-      i += 1
+      // fill again and spill
+      i = 0
+      while (i < n) {
+        row.setLong(0, i)
+        assert(queue.add(row), "fail to add")
+        i += 1
+      }
+      assert(queue.numQueues() > 1, "should have more than one queue")
+      queue.spill(1<<20, null)
+      assert(queue.numQueues() > 1, "should have more than one queue")
+      i = 0
+      while (i < n) {
+        val row = queue.remove()
+        assert(row != null, "fail to poll")
+        assert(row.getLong(0) == i, "does not match")
+        i += 1
+      }
+      queue.close()
     }
-    queue.close()
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLogSuite.scala
index ead17d50b4e1e..c7655af98160d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLogSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLogSuite.scala
@@ -297,6 +297,4 @@ class FakeCompactibleFileStreamLog(
   override protected def defaultCompactInterval: Int = _defaultCompactInterval
 
   override protected val minBatchesToRetain: Int = _defaultMinBatchesToRetain
-
-  override def compactLogs(logs: Seq[String]): Seq[String] = logs
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLogSuite.scala
index f95daafdfe19b..d6707e7be71fc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLogSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLogSuite.scala
@@ -18,7 +18,14 @@
 package org.apache.spark.sql.execution.streaming
 
 import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
+import java.lang.{Long => JLong}
+import java.net.URI
 import java.nio.charset.StandardCharsets.UTF_8
+import java.util.concurrent.ConcurrentHashMap
+
+import scala.util.Random
+
+import org.apache.hadoop.fs.{FileSystem, FSDataInputStream, Path, RawLocalFileSystem}
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.internal.SQLConf
@@ -29,19 +36,10 @@ class FileStreamSinkLogSuite extends SparkFunSuite with SharedSparkSession {
   import CompactibleFileStreamLog._
   import FileStreamSinkLog._
 
-  test("compactLogs") {
+  test("shouldRetain") {
     withFileStreamSinkLog { sinkLog =>
-      val logs = Seq(
-        newFakeSinkFileStatus("/a/b/x", FileStreamSinkLog.ADD_ACTION),
-        newFakeSinkFileStatus("/a/b/y", FileStreamSinkLog.ADD_ACTION),
-        newFakeSinkFileStatus("/a/b/z", FileStreamSinkLog.ADD_ACTION))
-      assert(logs === sinkLog.compactLogs(logs))
-
-      val logs2 = Seq(
-        newFakeSinkFileStatus("/a/b/m", FileStreamSinkLog.ADD_ACTION),
-        newFakeSinkFileStatus("/a/b/n", FileStreamSinkLog.ADD_ACTION),
-        newFakeSinkFileStatus("/a/b/z", FileStreamSinkLog.DELETE_ACTION))
-      assert(logs.dropRight(1) ++ logs2.dropRight(1) === sinkLog.compactLogs(logs ++ logs2))
+      val log = newFakeSinkFileStatus("/a/b/x", FileStreamSinkLog.ADD_ACTION)
+      assert(sinkLog.shouldRetain(log, System.currentTimeMillis()))
     }
   }
 
@@ -56,14 +54,6 @@ class FileStreamSinkLogSuite extends SparkFunSuite with SharedSparkSession {
           blockReplication = 1,
           blockSize = 10000L,
           action = FileStreamSinkLog.ADD_ACTION),
-        SinkFileStatus(
-          path = "/a/b/y",
-          size = 200L,
-          isDir = false,
-          modificationTime = 2000L,
-          blockReplication = 2,
-          blockSize = 20000L,
-          action = FileStreamSinkLog.DELETE_ACTION),
         SinkFileStatus(
           path = "/a/b/z",
           size = 300L,
@@ -76,7 +66,6 @@ class FileStreamSinkLogSuite extends SparkFunSuite with SharedSparkSession {
       // scalastyle:off
       val expected = s"""v$VERSION
           |{"path":"/a/b/x","size":100,"isDir":false,"modificationTime":1000,"blockReplication":1,"blockSize":10000,"action":"add"}
-          |{"path":"/a/b/y","size":200,"isDir":false,"modificationTime":2000,"blockReplication":2,"blockSize":20000,"action":"delete"}
           |{"path":"/a/b/z","size":300,"isDir":false,"modificationTime":3000,"blockReplication":3,"blockSize":30000,"action":"add"}""".stripMargin
       // scalastyle:on
       val baos = new ByteArrayOutputStream()
@@ -93,7 +82,6 @@ class FileStreamSinkLogSuite extends SparkFunSuite with SharedSparkSession {
       // scalastyle:off
       val logs = s"""v$VERSION
           |{"path":"/a/b/x","size":100,"isDir":false,"modificationTime":1000,"blockReplication":1,"blockSize":10000,"action":"add"}
-          |{"path":"/a/b/y","size":200,"isDir":false,"modificationTime":2000,"blockReplication":2,"blockSize":20000,"action":"delete"}
           |{"path":"/a/b/z","size":300,"isDir":false,"modificationTime":3000,"blockReplication":3,"blockSize":30000,"action":"add"}""".stripMargin
       // scalastyle:on
 
@@ -106,14 +94,6 @@ class FileStreamSinkLogSuite extends SparkFunSuite with SharedSparkSession {
           blockReplication = 1,
           blockSize = 10000L,
           action = FileStreamSinkLog.ADD_ACTION),
-        SinkFileStatus(
-          path = "/a/b/y",
-          size = 200L,
-          isDir = false,
-          modificationTime = 2000L,
-          blockReplication = 2,
-          blockSize = 20000L,
-          action = FileStreamSinkLog.DELETE_ACTION),
         SinkFileStatus(
           path = "/a/b/z",
           size = 300L,
@@ -149,6 +129,17 @@ class FileStreamSinkLogSuite extends SparkFunSuite with SharedSparkSession {
     }
   }
 
+  private def listBatchFiles(fs: FileSystem, sinkLog: FileStreamSinkLog): Set[String] = {
+    fs.listStatus(sinkLog.metadataPath).map(_.getPath.getName).filter { fileName =>
+      try {
+        getBatchIdFromFileName(fileName)
+        true
+      } catch {
+        case _: NumberFormatException => false
+      }
+    }.toSet
+  }
+
   test("delete expired file") {
     // Set FILE_SINK_LOG_CLEANUP_DELAY to 0 so that we can detect the deleting behaviour
     // deterministically and one min batches to retain
@@ -158,18 +149,7 @@ class FileStreamSinkLogSuite extends SparkFunSuite with SharedSparkSession {
       SQLConf.MIN_BATCHES_TO_RETAIN.key -> "1") {
       withFileStreamSinkLog { sinkLog =>
         val fs = sinkLog.metadataPath.getFileSystem(spark.sessionState.newHadoopConf())
-
-        def listBatchFiles(): Set[String] = {
-          fs.listStatus(sinkLog.metadataPath).map(_.getPath.getName).filter { fileName =>
-            try {
-              getBatchIdFromFileName(fileName)
-              true
-            } catch {
-              case _: NumberFormatException => false
-            }
-          }.toSet
-        }
-
+        def listBatchFiles(): Set[String] = this.listBatchFiles(fs, sinkLog)
         sinkLog.add(0, Array(newFakeSinkFileStatus("/a/b/0", FileStreamSinkLog.ADD_ACTION)))
         assert(Set("0") === listBatchFiles())
         sinkLog.add(1, Array(newFakeSinkFileStatus("/a/b/1", FileStreamSinkLog.ADD_ACTION)))
@@ -193,18 +173,7 @@ class FileStreamSinkLogSuite extends SparkFunSuite with SharedSparkSession {
       SQLConf.MIN_BATCHES_TO_RETAIN.key -> "2") {
       withFileStreamSinkLog { sinkLog =>
         val fs = sinkLog.metadataPath.getFileSystem(spark.sessionState.newHadoopConf())
-
-        def listBatchFiles(): Set[String] = {
-          fs.listStatus(sinkLog.metadataPath).map(_.getPath.getName).filter { fileName =>
-            try {
-              getBatchIdFromFileName(fileName)
-              true
-            } catch {
-              case _: NumberFormatException => false
-            }
-          }.toSet
-        }
-
+        def listBatchFiles(): Set[String] = this.listBatchFiles(fs, sinkLog)
         sinkLog.add(0, Array(newFakeSinkFileStatus("/a/b/0", FileStreamSinkLog.ADD_ACTION)))
         assert(Set("0") === listBatchFiles())
         sinkLog.add(1, Array(newFakeSinkFileStatus("/a/b/1", FileStreamSinkLog.ADD_ACTION)))
@@ -225,9 +194,27 @@ class FileStreamSinkLogSuite extends SparkFunSuite with SharedSparkSession {
     }
   }
 
+  test("filter out outdated entries when compacting") {
+    val curTime = System.currentTimeMillis()
+    withFileStreamSinkLog(sinkLog => {
+      val logs = Seq(
+        newFakeSinkFileStatus("/a/b/x", FileStreamSinkLog.ADD_ACTION, curTime),
+        newFakeSinkFileStatus("/a/b/y", FileStreamSinkLog.ADD_ACTION, curTime),
+        newFakeSinkFileStatus("/a/b/z", FileStreamSinkLog.ADD_ACTION, curTime))
+      logs.foreach { log => assert(sinkLog.shouldRetain(log, curTime)) }
+
+      val logs2 = Seq(
+        newFakeSinkFileStatus("/a/b/m", FileStreamSinkLog.ADD_ACTION, curTime - 80000),
+        newFakeSinkFileStatus("/a/b/n", FileStreamSinkLog.ADD_ACTION, curTime - 120000))
+      logs2.foreach { log =>
+        assert(!sinkLog.shouldRetain(log, curTime))
+      }
+    }, Some(60000))
+  }
+
   test("read Spark 2.1.0 log format") {
     assert(readFromResource("file-sink-log-version-2.1.0") === Seq(
-      // SinkFileStatus("/a/b/0", 100, false, 100, 1, 100, FileStreamSinkLog.ADD_ACTION), -> deleted
+      SinkFileStatus("/a/b/0", 1, false, 1, 1, 100, FileStreamSinkLog.ADD_ACTION),
       SinkFileStatus("/a/b/1", 100, false, 100, 1, 100, FileStreamSinkLog.ADD_ACTION),
       SinkFileStatus("/a/b/2", 200, false, 200, 1, 100, FileStreamSinkLog.ADD_ACTION),
       SinkFileStatus("/a/b/3", 300, false, 300, 1, 100, FileStreamSinkLog.ADD_ACTION),
@@ -240,24 +227,68 @@ class FileStreamSinkLogSuite extends SparkFunSuite with SharedSparkSession {
     ))
   }
 
+  test("getLatestBatchId") {
+    withCountOpenLocalFileSystemAsLocalFileSystem {
+      val scheme = CountOpenLocalFileSystem.scheme
+      withSQLConf(SQLConf.FILE_SINK_LOG_COMPACT_INTERVAL.key -> "3") {
+        withTempDir { dir =>
+          val sinkLog = new FileStreamSinkLog(FileStreamSinkLog.VERSION, spark,
+            s"$scheme:///${dir.getCanonicalPath}")
+          for (batchId <- 0L to 2L) {
+            sinkLog.add(
+              batchId,
+              Array(newFakeSinkFileStatus("/a/b/" + batchId, FileStreamSinkLog.ADD_ACTION)))
+          }
+
+          def getCountForOpenOnMetadataFile(batchId: Long): Long = {
+            val path = sinkLog.batchIdToPath(batchId).toUri.getPath
+            CountOpenLocalFileSystem.pathToNumOpenCalled.getOrDefault(path, 0L)
+          }
+
+          CountOpenLocalFileSystem.resetCount()
+
+          assert(sinkLog.getLatestBatchId() === Some(2L))
+          // getLatestBatchId doesn't open the latest metadata log file
+          (0L to 2L).foreach { batchId =>
+            assert(getCountForOpenOnMetadataFile(batchId) === 0L)
+          }
+
+          assert(sinkLog.getLatest().map(_._1).getOrElse(-1L) === 2L)
+          (0L to 1L).foreach { batchId =>
+            assert(getCountForOpenOnMetadataFile(batchId) === 0L)
+          }
+          // getLatest opens the latest metadata log file, which explains the needs on
+          // having "getLatestBatchId".
+          assert(getCountForOpenOnMetadataFile(2L) === 1L)
+        }
+      }
+    }
+  }
+
   /**
-   * Create a fake SinkFileStatus using path and action. Most of tests don't care about other fields
-   * in SinkFileStatus.
+   * Create a fake SinkFileStatus using path and action, and optionally modification time.
+   * Most of tests don't care about other fields in SinkFileStatus.
    */
-  private def newFakeSinkFileStatus(path: String, action: String): SinkFileStatus = {
+  private def newFakeSinkFileStatus(
+      path: String,
+      action: String,
+      modificationTime: Long = Long.MaxValue): SinkFileStatus = {
     SinkFileStatus(
       path = path,
       size = 100L,
       isDir = false,
-      modificationTime = 100L,
+      modificationTime = modificationTime,
       blockReplication = 1,
       blockSize = 100L,
       action = action)
   }
 
-  private def withFileStreamSinkLog(f: FileStreamSinkLog => Unit): Unit = {
+  private def withFileStreamSinkLog(
+      f: FileStreamSinkLog => Unit,
+      ttl: Option[Long] = None): Unit = {
     withTempDir { file =>
-      val sinkLog = new FileStreamSinkLog(FileStreamSinkLog.VERSION, spark, file.getCanonicalPath)
+      val sinkLog = new FileStreamSinkLog(FileStreamSinkLog.VERSION, spark, file.getCanonicalPath,
+        ttl)
       f(sinkLog)
     }
   }
@@ -267,4 +298,41 @@ class FileStreamSinkLogSuite extends SparkFunSuite with SharedSparkSession {
     val log = new FileStreamSinkLog(FileStreamSinkLog.VERSION, spark, input.toString)
     log.allFiles()
   }
+
+  private def withCountOpenLocalFileSystemAsLocalFileSystem(body: => Unit): Unit = {
+    val optionKey = s"fs.${CountOpenLocalFileSystem.scheme}.impl"
+    val originClassForLocalFileSystem = spark.conf.getOption(optionKey)
+    try {
+      spark.conf.set(optionKey, classOf[CountOpenLocalFileSystem].getName)
+      body
+    } finally {
+      originClassForLocalFileSystem match {
+        case Some(fsClazz) => spark.conf.set(optionKey, fsClazz)
+        case _ => spark.conf.unset(optionKey)
+      }
+    }
+  }
+}
+
+class CountOpenLocalFileSystem extends RawLocalFileSystem {
+  import CountOpenLocalFileSystem._
+
+  override def getUri: URI = {
+    URI.create(s"$scheme:///")
+  }
+
+  override def open(f: Path, bufferSize: Int): FSDataInputStream = {
+    val path = f.toUri.getPath
+    pathToNumOpenCalled.compute(path, (_, v) => {
+      if (v == null) 1L else v + 1
+    })
+    super.open(f, bufferSize)
+  }
+}
+
+object CountOpenLocalFileSystem {
+  val scheme = s"FileStreamSinkLogSuite${math.abs(Random.nextInt)}fs"
+  val pathToNumOpenCalled = new ConcurrentHashMap[String, JLong]
+
+  def resetCount(): Unit = pathToNumOpenCalled.clear()
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLogSuite.scala
index 67dd88cbab63b..980d532dd4779 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLogSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLogSuite.scala
@@ -199,7 +199,7 @@ class HDFSMetadataLogSuite extends SharedSparkSession {
     intercept[IllegalStateException](verifyBatchIds(Seq(2, 3, 4), Some(1L), Some(5L)))
     intercept[IllegalStateException](verifyBatchIds(Seq(1, 2, 4, 5), Some(1L), Some(5L)))
 
-    // Related to SPARK-26629, this capatures the behavior for verifyBatchIds when startId > endId
+    // Related to SPARK-26629, this captures the behavior for verifyBatchIds when startId > endId
     intercept[IllegalStateException](verifyBatchIds(Seq(), Some(2L), Some(1L)))
     intercept[AssertionError](verifyBatchIds(Seq(2), Some(2L), Some(1L)))
     intercept[AssertionError](verifyBatchIds(Seq(1), Some(2L), Some(1L)))
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/MemorySinkSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/MemorySinkSuite.scala
index 3ead91fcf712a..014840d758c0c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/MemorySinkSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/MemorySinkSuite.scala
@@ -24,7 +24,7 @@ import org.scalatest.BeforeAndAfter
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.execution.streaming.sources._
-import org.apache.spark.sql.streaming.{OutputMode, StreamTest}
+import org.apache.spark.sql.streaming.StreamTest
 import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
 import org.apache.spark.util.Utils
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecutionSuite.scala
index c228740df07c8..a508f923ffa13 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecutionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecutionSuite.scala
@@ -19,8 +19,13 @@ package org.apache.spark.sql.execution.streaming
 
 import org.scalatest.BeforeAndAfter
 
-import org.apache.spark.sql.functions.{count, window}
+import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}
+import org.apache.spark.sql.catalyst.plans.logical.Range
+import org.apache.spark.sql.connector.read.streaming
+import org.apache.spark.sql.connector.read.streaming.SparkDataStream
+import org.apache.spark.sql.functions.{count, timestamp_seconds, window}
 import org.apache.spark.sql.streaming.StreamTest
+import org.apache.spark.sql.types.{LongType, StructType}
 
 class MicroBatchExecutionSuite extends StreamTest with BeforeAndAfter {
 
@@ -33,7 +38,7 @@ class MicroBatchExecutionSuite extends StreamTest with BeforeAndAfter {
   test("SPARK-24156: do not plan a no-data batch again after it has already been planned") {
     val inputData = MemoryStream[Int]
     val df = inputData.toDF()
-      .withColumn("eventTime", $"value".cast("timestamp"))
+      .withColumn("eventTime", timestamp_seconds($"value"))
       .withWatermark("eventTime", "10 seconds")
       .groupBy(window($"eventTime", "5 seconds") as 'window)
       .agg(count("*") as 'count)
@@ -68,4 +73,122 @@ class MicroBatchExecutionSuite extends StreamTest with BeforeAndAfter {
       CheckNewAnswer((25, 1), (30, 1))   // This should not throw the error reported in SPARK-24156
     )
   }
+
+  test("no-data-batch re-executed after restart should call V1 source.getBatch()") {
+    val testSource = ReExecutedBatchTestSource(spark)
+    val df = testSource.toDF()
+      .withColumn("eventTime", timestamp_seconds($"value"))
+      .withWatermark("eventTime", "10 seconds")
+      .groupBy(window($"eventTime", "5 seconds") as 'window)
+      .agg(count("*") as 'count)
+      .select($"window".getField("start").cast("long").as[Long])
+
+    /** Reset this test source so that it appears to be a new source requiring initialization */
+    def resetSource(): StreamAction = Execute("reset source") { _ =>
+      testSource.reset()  // Make it look like a new source that needs to be re-initialized
+      require(testSource.currentOffset === 0)
+      require(testSource.getBatchCallCount === 0)
+    }
+
+    /** Add data to this test source by incrementing its available offset */
+    def addData(numNewRows: Int): StreamAction = new AddData {
+      override def addData(query: Option[StreamExecution]): (SparkDataStream, streaming.Offset) = {
+        testSource.incrementAvailableOffset(numNewRows)
+        (testSource, testSource.getOffset.get)
+      }
+    }
+
+    testStream(df)(
+      addData(numNewRows = 10),   // generate values 1...10, sets watermark to 0
+      CheckAnswer(),
+      addData(numNewRows = 10),   // generate values 11...20, sets watermark to 10
+      ProcessAllAvailable(),      // let no-data-batch be executed
+      CheckAnswer(0, 5),          // start time of windows closed and outputted
+      Execute("verify source internal state before stop") { q =>
+        // Last batch should be a no-data batch
+        require(q.lastProgress.numInputRows === 0)
+        // Source should have expected internal state
+        require(testSource.currentOffset === 20)
+        // getBatch should be called only for 2 batches with data, not for no-data-batches
+        assert(testSource.getBatchCallCount === 2)
+      },
+      StopStream,
+
+      /* Verify that if the last no-data-batch was incomplete, getBatch() is called only once */
+      Execute("mark last batch as incomplete") { q =>
+        // Delete the last committed batch from the commit log to signify that the last batch
+        // (a no-data batch) did not complete and has to be re-executed on restart.
+        val commit = q.commitLog.getLatest().map(_._1).getOrElse(-1L)
+        q.commitLog.purgeAfter(commit - 1)
+      },
+      resetSource(),
+      StartStream(),
+      ProcessAllAvailable(),  // allow initialization and re-execution
+      Execute("verify source.getBatch() called after re-executed no-data-batch") { q =>
+        // After restart, getBatch() should be called once even for no-data batch
+        assert(testSource.getBatchCallCount === 1)
+        assert(testSource.currentOffset === 20)
+      },
+      addData(numNewRows = 10),   // generate values 21...30, sets watermark to 20
+      ProcessAllAvailable(),      // let no-data-batch be executed
+      CheckAnswer(0, 5, 10, 15),
+      StopStream,
+
+      /* Verify that if the last no-data-batch was complete, getBatch() is still called only once */
+      Execute("verify last batch was complete") { q =>
+        // Verify that the commit log records the last batch as completed
+        require(q.commitLog.getLatest().map(_._1).get === q.offsetLog.getLatest().map(_._1).get)
+      },
+      resetSource(),
+      StartStream(),
+      ProcessAllAvailable(),      // allow initialization to completed
+      Execute("verify source.getBatch() called even if no-data-batch was not re-executed") { q =>
+        // After restart, getBatch() should be called even for no-data batch, but only once
+        assert(testSource.getBatchCallCount === 1)
+        assert(testSource.currentOffset === 30)
+      },
+      addData(numNewRows = 10),   // generate values 31...40, sets watermark to 30
+      ProcessAllAvailable(),      // let no-data-batch be executed
+      CheckAnswer(0, 5, 10, 15, 20, 25)
+    )
+  }
+
+
+  case class ReExecutedBatchTestSource(spark: SparkSession) extends Source {
+    @volatile var currentOffset = 0L
+    @volatile var getBatchCallCount = 0
+
+    override def getOffset: Option[Offset] = {
+      if (currentOffset <= 0) None else Some(LongOffset(currentOffset))
+    }
+
+    override def getBatch(start: Option[Offset], end: Offset): DataFrame = {
+      getBatchCallCount = getBatchCallCount + 1
+      if (currentOffset == 0) currentOffset = getOffsetValue(end)
+      val plan = Range(
+        start.map(getOffsetValue).getOrElse(0L) + 1L, getOffsetValue(end) + 1L, 1, None,
+        isStreaming = true)
+      Dataset.ofRows(spark, plan)
+    }
+
+    def incrementAvailableOffset(numNewRows: Int): Unit = {
+      currentOffset = currentOffset + numNewRows
+    }
+
+    def reset(): Unit = {
+      currentOffset = 0L
+      getBatchCallCount = 0
+    }
+    def toDF(): DataFrame = Dataset.ofRows(spark, StreamingExecutionRelation(this, spark))
+    override def schema: StructType = new StructType().add("value", LongType)
+    override def stop(): Unit = {}
+    private def getOffsetValue(offset: Offset): Long = {
+      offset match {
+        case s: SerializedOffset => LongOffset(s).offset
+        case l: LongOffset => l.offset
+        case _ => throw new IllegalArgumentException("incorrect offset type: " + offset)
+      }
+    }
+  }
 }
+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/ContinuousShuffleSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/ContinuousShuffleSuite.scala
deleted file mode 100644
index 54ec4a8352c1b..0000000000000
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/ContinuousShuffleSuite.scala
+++ /dev/null
@@ -1,423 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.streaming.continuous.shuffle
-
-import java.util.UUID
-
-import scala.language.implicitConversions
-
-import org.apache.spark.{HashPartitioner, TaskContext, TaskContextImpl}
-import org.apache.spark.rpc.RpcEndpointRef
-import org.apache.spark.sql.catalyst.expressions.{GenericInternalRow, UnsafeProjection}
-import org.apache.spark.sql.streaming.StreamTest
-import org.apache.spark.sql.types.{DataType, IntegerType, StringType}
-import org.apache.spark.unsafe.types.UTF8String
-
-class ContinuousShuffleSuite extends StreamTest {
-  // In this unit test, we emulate that we're in the task thread where
-  // ContinuousShuffleReadRDD.compute() will be evaluated. This requires a task context
-  // thread local to be set.
-  var ctx: TaskContextImpl = _
-
-  override def beforeEach(): Unit = {
-    super.beforeEach()
-    ctx = TaskContext.empty()
-    TaskContext.setTaskContext(ctx)
-  }
-
-  override def afterEach(): Unit = {
-    ctx.markTaskCompleted(None)
-    TaskContext.unset()
-    ctx = null
-    super.afterEach()
-  }
-
-  private implicit def unsafeRow(value: Int) = {
-    UnsafeProjection.create(Array(IntegerType : DataType))(
-      new GenericInternalRow(Array(value: Any)))
-  }
-
-  private def unsafeRow(value: String) = {
-    UnsafeProjection.create(Array(StringType : DataType))(
-      new GenericInternalRow(Array(UTF8String.fromString(value): Any)))
-  }
-
-  private def send(endpoint: RpcEndpointRef, messages: RPCContinuousShuffleMessage*) = {
-    messages.foreach(endpoint.askSync[Unit](_))
-  }
-
-  private def readRDDEndpoint(rdd: ContinuousShuffleReadRDD) = {
-    rdd.partitions(0).asInstanceOf[ContinuousShuffleReadPartition].endpoint
-  }
-
-  private def readEpoch(rdd: ContinuousShuffleReadRDD) = {
-    rdd.compute(rdd.partitions(0), ctx).toSeq.map(_.getInt(0))
-  }
-
-  test("reader - one epoch") {
-    val rdd = new ContinuousShuffleReadRDD(sparkContext, numPartitions = 1)
-    val endpoint = rdd.partitions(0).asInstanceOf[ContinuousShuffleReadPartition].endpoint
-    send(
-      endpoint,
-      ReceiverRow(0, unsafeRow(111)),
-      ReceiverRow(0, unsafeRow(222)),
-      ReceiverRow(0, unsafeRow(333)),
-      ReceiverEpochMarker(0)
-    )
-
-    val iter = rdd.compute(rdd.partitions(0), ctx)
-    assert(iter.toSeq.map(_.getInt(0)) == Seq(111, 222, 333))
-  }
-
-  test("reader - multiple epochs") {
-    val rdd = new ContinuousShuffleReadRDD(sparkContext, numPartitions = 1)
-    val endpoint = rdd.partitions(0).asInstanceOf[ContinuousShuffleReadPartition].endpoint
-    send(
-      endpoint,
-      ReceiverRow(0, unsafeRow(111)),
-      ReceiverEpochMarker(0),
-      ReceiverRow(0, unsafeRow(222)),
-      ReceiverRow(0, unsafeRow(333)),
-      ReceiverEpochMarker(0)
-    )
-
-    val firstEpoch = rdd.compute(rdd.partitions(0), ctx)
-    assert(firstEpoch.toSeq.map(_.getInt(0)) == Seq(111))
-
-    val secondEpoch = rdd.compute(rdd.partitions(0), ctx)
-    assert(secondEpoch.toSeq.map(_.getInt(0)) == Seq(222, 333))
-  }
-
-  test("reader - empty epochs") {
-    val rdd = new ContinuousShuffleReadRDD(sparkContext, numPartitions = 1)
-    val endpoint = rdd.partitions(0).asInstanceOf[ContinuousShuffleReadPartition].endpoint
-
-    send(
-      endpoint,
-      ReceiverEpochMarker(0),
-      ReceiverEpochMarker(0),
-      ReceiverRow(0, unsafeRow(111)),
-      ReceiverEpochMarker(0),
-      ReceiverEpochMarker(0),
-      ReceiverEpochMarker(0)
-    )
-
-    assert(rdd.compute(rdd.partitions(0), ctx).isEmpty)
-    assert(rdd.compute(rdd.partitions(0), ctx).isEmpty)
-
-    val thirdEpoch = rdd.compute(rdd.partitions(0), ctx)
-    assert(thirdEpoch.toSeq.map(_.getInt(0)) == Seq(111))
-
-    assert(rdd.compute(rdd.partitions(0), ctx).isEmpty)
-    assert(rdd.compute(rdd.partitions(0), ctx).isEmpty)
-  }
-
-  test("reader - multiple partitions") {
-    val rdd = new ContinuousShuffleReadRDD(
-      sparkContext,
-      numPartitions = 5,
-      endpointNames = Seq.fill(5)(s"endpt-${UUID.randomUUID()}"))
-    // Send all data before processing to ensure there's no crossover.
-    for (p <- rdd.partitions) {
-      val part = p.asInstanceOf[ContinuousShuffleReadPartition]
-      // Send index for identification.
-      send(
-        part.endpoint,
-        ReceiverRow(0, unsafeRow(part.index)),
-        ReceiverEpochMarker(0)
-      )
-    }
-
-    for (p <- rdd.partitions) {
-      val part = p.asInstanceOf[ContinuousShuffleReadPartition]
-      val iter = rdd.compute(part, ctx)
-      assert(iter.next().getInt(0) == part.index)
-      assert(!iter.hasNext)
-    }
-  }
-
-  test("reader - blocks waiting for new rows") {
-    val rdd = new ContinuousShuffleReadRDD(
-      sparkContext, numPartitions = 1, epochIntervalMs = Long.MaxValue)
-    val epoch = rdd.compute(rdd.partitions(0), ctx)
-
-    val readRowThread = new Thread {
-      override def run(): Unit = {
-        try {
-          epoch.next().getInt(0)
-        } catch {
-          case _: InterruptedException => // do nothing - expected at test ending
-        }
-      }
-    }
-
-    try {
-      readRowThread.start()
-      eventually(timeout(streamingTimeout)) {
-        assert(readRowThread.getState == Thread.State.TIMED_WAITING)
-      }
-    } finally {
-      readRowThread.interrupt()
-      readRowThread.join()
-    }
-  }
-
-  test("reader - multiple writers") {
-    val rdd = new ContinuousShuffleReadRDD(sparkContext, numPartitions = 1, numShuffleWriters = 3)
-    val endpoint = rdd.partitions(0).asInstanceOf[ContinuousShuffleReadPartition].endpoint
-    send(
-      endpoint,
-      ReceiverRow(0, unsafeRow("writer0-row0")),
-      ReceiverRow(1, unsafeRow("writer1-row0")),
-      ReceiverRow(2, unsafeRow("writer2-row0")),
-      ReceiverEpochMarker(0),
-      ReceiverEpochMarker(1),
-      ReceiverEpochMarker(2)
-    )
-
-    val firstEpoch = rdd.compute(rdd.partitions(0), ctx)
-    assert(firstEpoch.toSeq.map(_.getUTF8String(0).toString).toSet ==
-      Set("writer0-row0", "writer1-row0", "writer2-row0"))
-  }
-
-  test("reader - epoch only ends when all writers send markers") {
-    val rdd = new ContinuousShuffleReadRDD(
-      sparkContext, numPartitions = 1, numShuffleWriters = 3, epochIntervalMs = Long.MaxValue)
-    val endpoint = rdd.partitions(0).asInstanceOf[ContinuousShuffleReadPartition].endpoint
-    send(
-      endpoint,
-      ReceiverRow(0, unsafeRow("writer0-row0")),
-      ReceiverRow(1, unsafeRow("writer1-row0")),
-      ReceiverRow(2, unsafeRow("writer2-row0")),
-      ReceiverEpochMarker(0),
-      ReceiverEpochMarker(2)
-    )
-
-    val epoch = rdd.compute(rdd.partitions(0), ctx)
-    val rows = (0 until 3).map(_ => epoch.next()).toSet
-    assert(rows.map(_.getUTF8String(0).toString) ==
-      Set("writer0-row0", "writer1-row0", "writer2-row0"))
-
-    // After checking the right rows, block until we get an epoch marker indicating there's no next.
-    // (Also fail the assertion if for some reason we get a row.)
-
-    val readEpochMarkerThread = new Thread {
-      override def run(): Unit = {
-        assert(!epoch.hasNext)
-      }
-    }
-
-    readEpochMarkerThread.start()
-    eventually(timeout(streamingTimeout)) {
-      assert(readEpochMarkerThread.getState == Thread.State.TIMED_WAITING)
-    }
-
-    // Send the last epoch marker - now the epoch should finish.
-    send(endpoint, ReceiverEpochMarker(1))
-    eventually(timeout(streamingTimeout)) {
-      !readEpochMarkerThread.isAlive
-    }
-
-    // Join to pick up assertion failures.
-    readEpochMarkerThread.join(streamingTimeout.toMillis)
-  }
-
-  test("reader - writer epochs non aligned") {
-    val rdd = new ContinuousShuffleReadRDD(sparkContext, numPartitions = 1, numShuffleWriters = 3)
-    val endpoint = rdd.partitions(0).asInstanceOf[ContinuousShuffleReadPartition].endpoint
-    // We send multiple epochs for 0, then multiple for 1, then multiple for 2. The receiver should
-    // collate them as though the markers were aligned in the first place.
-    send(
-      endpoint,
-      ReceiverRow(0, unsafeRow("writer0-row0")),
-      ReceiverEpochMarker(0),
-      ReceiverRow(0, unsafeRow("writer0-row1")),
-      ReceiverEpochMarker(0),
-      ReceiverEpochMarker(0),
-
-      ReceiverEpochMarker(1),
-      ReceiverRow(1, unsafeRow("writer1-row0")),
-      ReceiverEpochMarker(1),
-      ReceiverRow(1, unsafeRow("writer1-row1")),
-      ReceiverEpochMarker(1),
-
-      ReceiverEpochMarker(2),
-      ReceiverEpochMarker(2),
-      ReceiverRow(2, unsafeRow("writer2-row0")),
-      ReceiverEpochMarker(2)
-    )
-
-    val firstEpoch = rdd.compute(rdd.partitions(0), ctx).map(_.getUTF8String(0).toString).toSet
-    assert(firstEpoch == Set("writer0-row0"))
-
-    val secondEpoch = rdd.compute(rdd.partitions(0), ctx).map(_.getUTF8String(0).toString).toSet
-    assert(secondEpoch == Set("writer0-row1", "writer1-row0"))
-
-    val thirdEpoch = rdd.compute(rdd.partitions(0), ctx).map(_.getUTF8String(0).toString).toSet
-    assert(thirdEpoch == Set("writer1-row1", "writer2-row0"))
-  }
-
-  test("one epoch") {
-    val reader = new ContinuousShuffleReadRDD(sparkContext, numPartitions = 1)
-    val writer = new RPCContinuousShuffleWriter(
-      0, new HashPartitioner(1), Array(readRDDEndpoint(reader)))
-
-    writer.write(Iterator(1, 2, 3))
-
-    assert(readEpoch(reader) == Seq(1, 2, 3))
-  }
-
-  test("multiple epochs") {
-    val reader = new ContinuousShuffleReadRDD(sparkContext, numPartitions = 1)
-    val writer = new RPCContinuousShuffleWriter(
-      0, new HashPartitioner(1), Array(readRDDEndpoint(reader)))
-
-    writer.write(Iterator(1, 2, 3))
-    writer.write(Iterator(4, 5, 6))
-
-    assert(readEpoch(reader) == Seq(1, 2, 3))
-    assert(readEpoch(reader) == Seq(4, 5, 6))
-  }
-
-  test("empty epochs") {
-    val reader = new ContinuousShuffleReadRDD(sparkContext, numPartitions = 1)
-    val writer = new RPCContinuousShuffleWriter(
-      0, new HashPartitioner(1), Array(readRDDEndpoint(reader)))
-
-    writer.write(Iterator())
-    writer.write(Iterator(1, 2))
-    writer.write(Iterator())
-    writer.write(Iterator())
-    writer.write(Iterator(3, 4))
-    writer.write(Iterator())
-
-    assert(readEpoch(reader) == Seq())
-    assert(readEpoch(reader) == Seq(1, 2))
-    assert(readEpoch(reader) == Seq())
-    assert(readEpoch(reader) == Seq())
-    assert(readEpoch(reader) == Seq(3, 4))
-    assert(readEpoch(reader) == Seq())
-  }
-
-  test("blocks waiting for writer") {
-    val reader = new ContinuousShuffleReadRDD(sparkContext, numPartitions = 1)
-    val writer = new RPCContinuousShuffleWriter(
-      0, new HashPartitioner(1), Array(readRDDEndpoint(reader)))
-
-    val readerEpoch = reader.compute(reader.partitions(0), ctx)
-
-    val readRowThread = new Thread {
-      override def run(): Unit = {
-        assert(readerEpoch.toSeq.map(_.getInt(0)) == Seq(1))
-      }
-    }
-    readRowThread.start()
-
-    eventually(timeout(streamingTimeout)) {
-      assert(readRowThread.getState == Thread.State.TIMED_WAITING)
-    }
-
-    // Once we write the epoch the thread should stop waiting and succeed.
-    writer.write(Iterator(1))
-    readRowThread.join(streamingTimeout.toMillis)
-  }
-
-  test("multiple writer partitions") {
-    val numWriterPartitions = 3
-
-    val reader = new ContinuousShuffleReadRDD(
-      sparkContext, numPartitions = 1, numShuffleWriters = numWriterPartitions)
-    val writers = (0 until 3).map { idx =>
-      new RPCContinuousShuffleWriter(idx, new HashPartitioner(1), Array(readRDDEndpoint(reader)))
-    }
-
-    writers(0).write(Iterator(1, 4, 7))
-    writers(1).write(Iterator(2, 5))
-    writers(2).write(Iterator(3, 6))
-
-    writers(0).write(Iterator(4, 7, 10))
-    writers(1).write(Iterator(5, 8))
-    writers(2).write(Iterator(6, 9))
-
-    // Since there are multiple asynchronous writers, the original row sequencing is not guaranteed.
-    // The epochs should be deterministically preserved, however.
-    assert(readEpoch(reader).toSet == Seq(1, 2, 3, 4, 5, 6, 7).toSet)
-    assert(readEpoch(reader).toSet == Seq(4, 5, 6, 7, 8, 9, 10).toSet)
-  }
-
-  test("reader epoch only ends when all writer partitions write it") {
-    val numWriterPartitions = 3
-
-    val reader = new ContinuousShuffleReadRDD(
-      sparkContext, numPartitions = 1, numShuffleWriters = numWriterPartitions)
-    val writers = (0 until 3).map { idx =>
-      new RPCContinuousShuffleWriter(idx, new HashPartitioner(1), Array(readRDDEndpoint(reader)))
-    }
-
-    writers(1).write(Iterator())
-    writers(2).write(Iterator())
-
-    val readerEpoch = reader.compute(reader.partitions(0), ctx)
-
-    val readEpochMarkerThread = new Thread {
-      override def run(): Unit = {
-        assert(!readerEpoch.hasNext)
-      }
-    }
-
-    readEpochMarkerThread.start()
-    eventually(timeout(streamingTimeout)) {
-      assert(readEpochMarkerThread.getState == Thread.State.TIMED_WAITING)
-    }
-
-    writers(0).write(Iterator())
-    readEpochMarkerThread.join(streamingTimeout.toMillis)
-  }
-
-  test("receiver stopped with row last") {
-    val rdd = new ContinuousShuffleReadRDD(sparkContext, numPartitions = 1)
-    val endpoint = rdd.partitions(0).asInstanceOf[ContinuousShuffleReadPartition].endpoint
-    send(
-      endpoint,
-      ReceiverEpochMarker(0),
-      ReceiverRow(0, unsafeRow(111))
-    )
-
-    ctx.markTaskCompleted(None)
-    val receiver = rdd.partitions(0).asInstanceOf[ContinuousShuffleReadPartition].reader
-    eventually(timeout(streamingTimeout)) {
-      assert(receiver.asInstanceOf[RPCContinuousShuffleReader].stopped.get())
-    }
-  }
-
-  test("receiver stopped with marker last") {
-    val rdd = new ContinuousShuffleReadRDD(sparkContext, numPartitions = 1)
-    val endpoint = rdd.partitions(0).asInstanceOf[ContinuousShuffleReadPartition].endpoint
-    send(
-      endpoint,
-      ReceiverRow(0, unsafeRow(111)),
-      ReceiverEpochMarker(0)
-    )
-
-    ctx.markTaskCompleted(None)
-    val receiver = rdd.partitions(0).asInstanceOf[ContinuousShuffleReadPartition].reader
-    eventually(timeout(streamingTimeout)) {
-      assert(receiver.asInstanceOf[RPCContinuousShuffleReader].stopped.get())
-    }
-  }
-}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterSuite.scala
index 0c17320acade9..0fe339b93047a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterSuite.scala
@@ -26,7 +26,7 @@ import org.scalatest.BeforeAndAfter
 import org.apache.spark.SparkException
 import org.apache.spark.sql.ForeachWriter
 import org.apache.spark.sql.execution.streaming.MemoryStream
-import org.apache.spark.sql.functions.{count, window}
+import org.apache.spark.sql.functions.{count, timestamp_seconds, window}
 import org.apache.spark.sql.streaming.{OutputMode, StreamingQueryException, StreamTest}
 import org.apache.spark.sql.test.SharedSparkSession
 
@@ -163,7 +163,7 @@ class ForeachWriterSuite extends StreamTest with SharedSparkSession with BeforeA
     val inputData = MemoryStream[Int]
 
     val windowedAggregation = inputData.toDF()
-      .withColumn("eventTime", $"value".cast("timestamp"))
+      .withColumn("eventTime", timestamp_seconds($"value"))
       .withWatermark("eventTime", "10 seconds")
       .groupBy(window($"eventTime", "5 seconds") as 'window)
       .agg(count("*") as 'count)
@@ -197,7 +197,7 @@ class ForeachWriterSuite extends StreamTest with SharedSparkSession with BeforeA
     val inputData = MemoryStream[Int]
 
     val windowedAggregation = inputData.toDF()
-      .withColumn("eventTime", $"value".cast("timestamp"))
+      .withColumn("eventTime", timestamp_seconds($"value"))
       .withWatermark("eventTime", "10 seconds")
       .groupBy(window($"eventTime", "5 seconds") as 'window)
       .agg(count("*") as 'count)
@@ -333,6 +333,6 @@ class TestForeachWriter extends ForeachWriter[Int] {
 
   override def close(errorOrNull: Throwable): Unit = {
     events += ForeachWriterSuite.Close(error = Option(errorOrNull))
-    ForeachWriterSuite.addEvents(events)
+    ForeachWriterSuite.addEvents(events.toSeq)
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/FlatMapGroupsWithStateExecHelperSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/FlatMapGroupsWithStateExecHelperSuite.scala
index dec30fd01f7e2..ea6fd8ab312c9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/FlatMapGroupsWithStateExecHelperSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/FlatMapGroupsWithStateExecHelperSuite.scala
@@ -23,7 +23,6 @@ import org.apache.spark.sql.Encoder
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.catalyst.expressions.{GenericInternalRow, UnsafeProjection, UnsafeRow}
 import org.apache.spark.sql.execution.streaming.GroupStateImpl._
-import org.apache.spark.sql.streaming.FlatMapGroupsWithStateSuite._
 import org.apache.spark.sql.streaming.StreamTest
 import org.apache.spark.sql.types._
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateSchemaCompatibilityCheckerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateSchemaCompatibilityCheckerSuite.scala
new file mode 100644
index 0000000000000..4eb7603b316aa
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateSchemaCompatibilityCheckerSuite.scala
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.streaming.state
+
+import java.util.UUID
+
+import scala.util.Random
+
+import org.apache.hadoop.conf.Configuration
+
+import org.apache.spark.sql.execution.streaming.state.StateStoreTestsHelper.newDir
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types._
+
+class StateSchemaCompatibilityCheckerSuite extends SharedSparkSession {
+
+  private val hadoopConf: Configuration = new Configuration()
+  private val opId = Random.nextInt(100000)
+  private val partitionId = StateStore.PARTITION_ID_TO_CHECK_SCHEMA
+
+  private val structSchema = new StructType()
+    .add(StructField("nested1", IntegerType, nullable = true))
+    .add(StructField("nested2", StringType, nullable = true))
+
+  private val keySchema = new StructType()
+    .add(StructField("key1", IntegerType, nullable = true))
+    .add(StructField("key2", StringType, nullable = true))
+    .add(StructField("key3", structSchema, nullable = true))
+
+  private val valueSchema = new StructType()
+    .add(StructField("value1", IntegerType, nullable = true))
+    .add(StructField("value2", StringType, nullable = true))
+    .add(StructField("value3", structSchema, nullable = true))
+
+  test("adding field to key should fail") {
+    val fieldAddedKeySchema = keySchema.add(StructField("newKey", IntegerType))
+    verifyException(keySchema, valueSchema, fieldAddedKeySchema, valueSchema)
+  }
+
+  test("adding field to value should fail") {
+    val fieldAddedValueSchema = valueSchema.add(StructField("newValue", IntegerType))
+    verifyException(keySchema, valueSchema, keySchema, fieldAddedValueSchema)
+  }
+
+  test("adding nested field in key should fail") {
+    val fieldAddedNestedSchema = structSchema.add(StructField("newNested", IntegerType))
+    val newKeySchema = applyNewSchemaToNestedFieldInKey(fieldAddedNestedSchema)
+    verifyException(keySchema, valueSchema, newKeySchema, valueSchema)
+  }
+
+  test("adding nested field in value should fail") {
+    val fieldAddedNestedSchema = structSchema.add(StructField("newNested", IntegerType))
+    val newValueSchema = applyNewSchemaToNestedFieldInValue(fieldAddedNestedSchema)
+    verifyException(keySchema, valueSchema, keySchema, newValueSchema)
+  }
+
+  test("removing field from key should fail") {
+    val fieldRemovedKeySchema = StructType(keySchema.dropRight(1))
+    verifyException(keySchema, valueSchema, fieldRemovedKeySchema, valueSchema)
+  }
+
+  test("removing field from value should fail") {
+    val fieldRemovedValueSchema = StructType(valueSchema.drop(1))
+    verifyException(keySchema, valueSchema, keySchema, fieldRemovedValueSchema)
+  }
+
+  test("removing nested field from key should fail") {
+    val fieldRemovedNestedSchema = StructType(structSchema.dropRight(1))
+    val newKeySchema = applyNewSchemaToNestedFieldInKey(fieldRemovedNestedSchema)
+    verifyException(keySchema, valueSchema, newKeySchema, valueSchema)
+  }
+
+  test("removing nested field from value should fail") {
+    val fieldRemovedNestedSchema = StructType(structSchema.drop(1))
+    val newValueSchema = applyNewSchemaToNestedFieldInValue(fieldRemovedNestedSchema)
+    verifyException(keySchema, valueSchema, keySchema, newValueSchema)
+  }
+
+  test("changing the type of field in key should fail") {
+    val typeChangedKeySchema = StructType(keySchema.map(_.copy(dataType = TimestampType)))
+    verifyException(keySchema, valueSchema, typeChangedKeySchema, valueSchema)
+  }
+
+  test("changing the type of field in value should fail") {
+    val typeChangedValueSchema = StructType(valueSchema.map(_.copy(dataType = TimestampType)))
+    verifyException(keySchema, valueSchema, keySchema, typeChangedValueSchema)
+  }
+
+  test("changing the type of nested field in key should fail") {
+    val typeChangedNestedSchema = StructType(structSchema.map(_.copy(dataType = TimestampType)))
+    val newKeySchema = applyNewSchemaToNestedFieldInKey(typeChangedNestedSchema)
+    verifyException(keySchema, valueSchema, newKeySchema, valueSchema)
+  }
+
+  test("changing the type of nested field in value should fail") {
+    val typeChangedNestedSchema = StructType(structSchema.map(_.copy(dataType = TimestampType)))
+    val newValueSchema = applyNewSchemaToNestedFieldInValue(typeChangedNestedSchema)
+    verifyException(keySchema, valueSchema, keySchema, newValueSchema)
+  }
+
+  test("changing the nullability of nullable to non-nullable in key should fail") {
+    val nonNullChangedKeySchema = StructType(keySchema.map(_.copy(nullable = false)))
+    verifyException(keySchema, valueSchema, nonNullChangedKeySchema, valueSchema)
+  }
+
+  test("changing the nullability of nullable to non-nullable in value should fail") {
+    val nonNullChangedValueSchema = StructType(valueSchema.map(_.copy(nullable = false)))
+    verifyException(keySchema, valueSchema, keySchema, nonNullChangedValueSchema)
+  }
+
+  test("changing the nullability of nullable to nonnullable in nested field in key should fail") {
+    val typeChangedNestedSchema = StructType(structSchema.map(_.copy(nullable = false)))
+    val newKeySchema = applyNewSchemaToNestedFieldInKey(typeChangedNestedSchema)
+    verifyException(keySchema, valueSchema, newKeySchema, valueSchema)
+  }
+
+  test("changing the nullability of nullable to nonnullable in nested field in value should fail") {
+    val typeChangedNestedSchema = StructType(structSchema.map(_.copy(nullable = false)))
+    val newValueSchema = applyNewSchemaToNestedFieldInValue(typeChangedNestedSchema)
+    verifyException(keySchema, valueSchema, keySchema, newValueSchema)
+  }
+
+  test("changing the name of field in key should be allowed") {
+    val newName: StructField => StructField = f => f.copy(name = f.name + "_new")
+    val fieldNameChangedKeySchema = StructType(keySchema.map(newName))
+    verifySuccess(keySchema, valueSchema, fieldNameChangedKeySchema, valueSchema)
+  }
+
+  test("changing the name of field in value should be allowed") {
+    val newName: StructField => StructField = f => f.copy(name = f.name + "_new")
+    val fieldNameChangedValueSchema = StructType(valueSchema.map(newName))
+    verifySuccess(keySchema, valueSchema, keySchema, fieldNameChangedValueSchema)
+  }
+
+  test("changing the name of nested field in key should be allowed") {
+    val newName: StructField => StructField = f => f.copy(name = f.name + "_new")
+    val newNestedFieldsSchema = StructType(structSchema.map(newName))
+    val fieldNameChangedKeySchema = applyNewSchemaToNestedFieldInKey(newNestedFieldsSchema)
+    verifySuccess(keySchema, valueSchema, fieldNameChangedKeySchema, valueSchema)
+  }
+
+  test("changing the name of nested field in value should be allowed") {
+    val newName: StructField => StructField = f => f.copy(name = f.name + "_new")
+    val newNestedFieldsSchema = StructType(structSchema.map(newName))
+    val fieldNameChangedValueSchema = applyNewSchemaToNestedFieldInValue(newNestedFieldsSchema)
+    verifySuccess(keySchema, valueSchema, keySchema, fieldNameChangedValueSchema)
+  }
+
+  private def applyNewSchemaToNestedFieldInKey(newNestedSchema: StructType): StructType = {
+    applyNewSchemaToNestedField(keySchema, newNestedSchema, "key3")
+  }
+
+  private def applyNewSchemaToNestedFieldInValue(newNestedSchema: StructType): StructType = {
+    applyNewSchemaToNestedField(valueSchema, newNestedSchema, "value3")
+  }
+
+  private def applyNewSchemaToNestedField(
+      originSchema: StructType,
+      newNestedSchema: StructType,
+      fieldName: String): StructType = {
+    val newFields = originSchema.map { field =>
+      if (field.name == fieldName) {
+        field.copy(dataType = newNestedSchema)
+      } else {
+        field
+      }
+    }
+    StructType(newFields)
+  }
+
+  private def runSchemaChecker(
+      dir: String,
+      queryId: UUID,
+      newKeySchema: StructType,
+      newValueSchema: StructType): Unit = {
+    // in fact, Spark doesn't support online state schema change, so need to check
+    // schema only once for each running of JVM
+    val providerId = StateStoreProviderId(
+      StateStoreId(dir, opId, partitionId), queryId)
+
+    new StateSchemaCompatibilityChecker(providerId, hadoopConf)
+      .check(newKeySchema, newValueSchema)
+  }
+
+  private def verifyException(
+      oldKeySchema: StructType,
+      oldValueSchema: StructType,
+      newKeySchema: StructType,
+      newValueSchema: StructType): Unit = {
+    val dir = newDir()
+    val queryId = UUID.randomUUID()
+    runSchemaChecker(dir, queryId, oldKeySchema, oldValueSchema)
+
+    val e = intercept[StateSchemaNotCompatible] {
+      runSchemaChecker(dir, queryId, newKeySchema, newValueSchema)
+    }
+
+    e.getMessage.contains("Provided schema doesn't match to the schema for existing state!")
+    e.getMessage.contains(newKeySchema.json)
+    e.getMessage.contains(newValueSchema.json)
+    e.getMessage.contains(oldKeySchema.json)
+    e.getMessage.contains(oldValueSchema.json)
+  }
+
+  private def verifySuccess(
+      oldKeySchema: StructType,
+      oldValueSchema: StructType,
+      newKeySchema: StructType,
+      newValueSchema: StructType): Unit = {
+    val dir = newDir()
+    val queryId = UUID.randomUUID()
+    runSchemaChecker(dir, queryId, oldKeySchema, oldValueSchema)
+    runSchemaChecker(dir, queryId, newKeySchema, newValueSchema)
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreCompatibilitySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreCompatibilitySuite.scala
new file mode 100644
index 0000000000000..b189de8d2a21e
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreCompatibilitySuite.scala
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.streaming.state
+
+import java.io.File
+
+import org.apache.commons.io.FileUtils
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.io.CompressionCodec
+import org.apache.spark.sql.catalyst.plans.PlanTestBase
+import org.apache.spark.sql.catalyst.streaming.InternalOutputModes.Update
+import org.apache.spark.sql.execution.streaming.MemoryStream
+import org.apache.spark.sql.functions.count
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.streaming.StreamTest
+import org.apache.spark.util.Utils
+
+class StateStoreCompatibilitySuite extends StreamTest with StateStoreCodecsTest {
+   testWithAllCodec(
+      "SPARK-33263: Recovery from checkpoint before codec config introduced") {
+     val resourceUri = this.getClass.getResource(
+       "/structured-streaming/checkpoint-version-3.0.0-streaming-statestore-codec/").toURI
+     val checkpointDir = Utils.createTempDir().getCanonicalFile
+     FileUtils.copyDirectory(new File(resourceUri), checkpointDir)
+
+     import testImplicits._
+
+     val inputData = MemoryStream[Int]
+     val aggregated = inputData.toDF().groupBy("value").agg(count("*"))
+     inputData.addData(1, 2, 3)
+
+     /**
+      * Note: The checkpoint was generated using the following input in Spark version 3.0.0:
+      * AddData(inputData, 1, 2, 3)
+      */
+
+     testStream(aggregated, Update)(
+       StartStream(
+         checkpointLocation = checkpointDir.getAbsolutePath,
+         additionalConfs = Map(SQLConf.SHUFFLE_PARTITIONS.key -> "1")),
+       AddData(inputData, 1, 2),
+       CheckNewAnswer((1, 2), (2, 2))
+     )
+   }
+}
+
+trait StateStoreCodecsTest extends SparkFunSuite with PlanTestBase {
+  private val codecsInShortName =
+    CompressionCodec.ALL_COMPRESSION_CODECS.map { c => CompressionCodec.getShortName(c) }
+
+  protected def testWithAllCodec(name: String)(func: => Any): Unit = {
+    codecsInShortName.foreach { codecShortName =>
+      test(s"$name - with codec $codecShortName") {
+        withSQLConf(SQLConf.STATE_STORE_COMPRESSION_CODEC.key -> codecShortName) {
+          func
+        }
+      }
+    }
+
+    CompressionCodec.ALL_COMPRESSION_CODECS.foreach { codecShortName =>
+      test(s"$name - with codec $codecShortName") {
+        withSQLConf(SQLConf.STATE_STORE_COMPRESSION_CODEC.key -> codecShortName) {
+          func
+        }
+      }
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreCoordinatorSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreCoordinatorSuite.scala
index 7bca225dfdd8f..d039c72bb7d18 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreCoordinatorSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreCoordinatorSuite.scala
@@ -41,7 +41,7 @@ class StateStoreCoordinatorSuite extends SparkFunSuite with SharedSparkContext {
       assert(coordinatorRef.verifyIfInstanceActive(id, "exec1") === false)
       assert(coordinatorRef.getLocation(id) === None)
 
-      coordinatorRef.reportActiveInstance(id, "hostX", "exec1")
+      coordinatorRef.reportActiveInstance(id, "hostX", "exec1", Seq.empty)
       eventually(timeout(5.seconds)) {
         assert(coordinatorRef.verifyIfInstanceActive(id, "exec1"))
         assert(
@@ -49,7 +49,7 @@ class StateStoreCoordinatorSuite extends SparkFunSuite with SharedSparkContext {
             Some(ExecutorCacheTaskLocation("hostX", "exec1").toString))
       }
 
-      coordinatorRef.reportActiveInstance(id, "hostX", "exec2")
+      coordinatorRef.reportActiveInstance(id, "hostX", "exec2", Seq.empty)
 
       eventually(timeout(5.seconds)) {
         assert(coordinatorRef.verifyIfInstanceActive(id, "exec1") === false)
@@ -72,9 +72,9 @@ class StateStoreCoordinatorSuite extends SparkFunSuite with SharedSparkContext {
       val host = "hostX"
       val exec = "exec1"
 
-      coordinatorRef.reportActiveInstance(id1, host, exec)
-      coordinatorRef.reportActiveInstance(id2, host, exec)
-      coordinatorRef.reportActiveInstance(id3, host, exec)
+      coordinatorRef.reportActiveInstance(id1, host, exec, Seq.empty)
+      coordinatorRef.reportActiveInstance(id2, host, exec, Seq.empty)
+      coordinatorRef.reportActiveInstance(id3, host, exec, Seq.empty)
 
       eventually(timeout(5.seconds)) {
         assert(coordinatorRef.verifyIfInstanceActive(id1, exec))
@@ -106,7 +106,7 @@ class StateStoreCoordinatorSuite extends SparkFunSuite with SharedSparkContext {
 
       val id = StateStoreProviderId(StateStoreId("x", 0, 0), UUID.randomUUID)
 
-      coordRef1.reportActiveInstance(id, "hostX", "exec1")
+      coordRef1.reportActiveInstance(id, "hostX", "exec1", Seq.empty)
 
       eventually(timeout(5.seconds)) {
         assert(coordRef2.verifyIfInstanceActive(id, "exec1"))
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreRDDSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreRDDSuite.scala
index 015415a534ff5..378aa1dca139f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreRDDSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreRDDSuite.scala
@@ -159,8 +159,8 @@ class StateStoreRDDSuite extends SparkFunSuite with BeforeAndAfter with BeforeAn
         val coordinatorRef = sqlContext.streams.stateStoreCoordinator
         val storeProviderId1 = StateStoreProviderId(StateStoreId(path, opId, 0), queryRunId)
         val storeProviderId2 = StateStoreProviderId(StateStoreId(path, opId, 1), queryRunId)
-        coordinatorRef.reportActiveInstance(storeProviderId1, "host1", "exec1")
-        coordinatorRef.reportActiveInstance(storeProviderId2, "host2", "exec2")
+        coordinatorRef.reportActiveInstance(storeProviderId1, "host1", "exec1", Seq.empty)
+        coordinatorRef.reportActiveInstance(storeProviderId2, "host2", "exec2", Seq.empty)
 
         require(
           coordinatorRef.getLocation(storeProviderId1) ===
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreSuite.scala
index 488879938339d..bbc557be126be 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreSuite.scala
@@ -47,15 +47,11 @@ import org.apache.spark.unsafe.types.UTF8String
 import org.apache.spark.util.Utils
 
 class StateStoreSuite extends StateStoreSuiteBase[HDFSBackedStateStoreProvider]
-  with BeforeAndAfter with PrivateMethodTester {
-  type MapType = mutable.HashMap[UnsafeRow, UnsafeRow]
-  type ProviderMapType = java.util.concurrent.ConcurrentHashMap[UnsafeRow, UnsafeRow]
-
-  import StateStoreCoordinatorSuite._
+  with BeforeAndAfter {
   import StateStoreTestsHelper._
 
-  val keySchema = StructType(Seq(StructField("key", StringType, true)))
-  val valueSchema = StructType(Seq(StructField("value", IntegerType, true)))
+  override val keySchema = StructType(Seq(StructField("key", StringType, true)))
+  override val valueSchema = StructType(Seq(StructField("value", IntegerType, true)))
 
   before {
     StateStore.stop()
@@ -67,63 +63,21 @@ class StateStoreSuite extends StateStoreSuiteBase[HDFSBackedStateStoreProvider]
     require(!StateStore.isMaintenanceRunning)
   }
 
-  def updateVersionTo(
-      provider: StateStoreProvider,
-      currentVersion: Int,
-      targetVersion: Int): Int = {
-    var newCurrentVersion = currentVersion
-    for (i <- newCurrentVersion until targetVersion) {
-      newCurrentVersion = incrementVersion(provider, i)
-    }
-    require(newCurrentVersion === targetVersion)
-    newCurrentVersion
-  }
-
-  def incrementVersion(provider: StateStoreProvider, currentVersion: Int): Int = {
-    val store = provider.getStore(currentVersion)
-    put(store, "a", currentVersion + 1)
-    store.commit()
-    currentVersion + 1
-  }
-
-  def checkLoadedVersions(
-      loadedMaps: util.SortedMap[Long, ProviderMapType],
-      count: Int,
-      earliestKey: Long,
-      latestKey: Long): Unit = {
-    assert(loadedMaps.size() === count)
-    assert(loadedMaps.firstKey() === earliestKey)
-    assert(loadedMaps.lastKey() === latestKey)
-  }
-
-  def checkVersion(
-      loadedMaps: util.SortedMap[Long, ProviderMapType],
-      version: Long,
-      expectedData: Map[String, Int]): Unit = {
-
-    val originValueMap = loadedMaps.get(version).asScala.map { entry =>
-      rowToString(entry._1) -> rowToInt(entry._2)
-    }.toMap
-
-    assert(originValueMap === expectedData)
-  }
-
   test("retaining only two latest versions when MAX_BATCHES_TO_RETAIN_IN_MEMORY set to 2") {
-    val provider = newStoreProvider(opId = Random.nextInt, partition = 0,
-      numOfVersToRetainInMemory = 2)
+    val provider = newStoreProvider(minDeltasForSnapshot = 10, numOfVersToRetainInMemory = 2)
 
     var currentVersion = 0
 
     // commit the ver 1 : cache will have one element
     currentVersion = incrementVersion(provider, currentVersion)
-    assert(getData(provider) === Set("a" -> 1))
+    assert(getLatestData(provider) === Set("a" -> 1))
     var loadedMaps = provider.getLoadedMaps()
     checkLoadedVersions(loadedMaps, count = 1, earliestKey = 1, latestKey = 1)
     checkVersion(loadedMaps, 1, Map("a" -> 1))
 
     // commit the ver 2 : cache will have two elements
     currentVersion = incrementVersion(provider, currentVersion)
-    assert(getData(provider) === Set("a" -> 2))
+    assert(getLatestData(provider) === Set("a" -> 2))
     loadedMaps = provider.getLoadedMaps()
     checkLoadedVersions(loadedMaps, count = 2, earliestKey = 2, latestKey = 1)
     checkVersion(loadedMaps, 2, Map("a" -> 2))
@@ -132,7 +86,7 @@ class StateStoreSuite extends StateStoreSuiteBase[HDFSBackedStateStoreProvider]
     // commit the ver 3 : cache has already two elements and adding ver 3 incurs exceeding cache,
     // and ver 3 will be added but ver 1 will be evicted
     currentVersion = incrementVersion(provider, currentVersion)
-    assert(getData(provider) === Set("a" -> 3))
+    assert(getLatestData(provider) === Set("a" -> 3))
     loadedMaps = provider.getLoadedMaps()
     checkLoadedVersions(loadedMaps, count = 2, earliestKey = 3, latestKey = 2)
     checkVersion(loadedMaps, 3, Map("a" -> 3))
@@ -147,7 +101,7 @@ class StateStoreSuite extends StateStoreSuiteBase[HDFSBackedStateStoreProvider]
 
     // commit the ver 1 : cache will have one element
     currentVersion = incrementVersion(provider, currentVersion)
-    assert(getData(provider) === Set("a" -> 1))
+    assert(getLatestData(provider) === Set("a" -> 1))
     var loadedMaps = provider.getLoadedMaps()
     checkLoadedVersions(loadedMaps, count = 1, earliestKey = 1, latestKey = 1)
     checkVersion(loadedMaps, 1, Map("a" -> 1))
@@ -157,7 +111,7 @@ class StateStoreSuite extends StateStoreSuiteBase[HDFSBackedStateStoreProvider]
     // this fact ensures cache miss will occur when this partition succeeds commit
     // but there's a failure afterwards so have to reprocess previous batch
     currentVersion = incrementVersion(provider, currentVersion)
-    assert(getData(provider) === Set("a" -> 2))
+    assert(getLatestData(provider) === Set("a" -> 2))
     loadedMaps = provider.getLoadedMaps()
     checkLoadedVersions(loadedMaps, count = 1, earliestKey = 2, latestKey = 2)
     checkVersion(loadedMaps, 2, Map("a" -> 2))
@@ -173,7 +127,7 @@ class StateStoreSuite extends StateStoreSuiteBase[HDFSBackedStateStoreProvider]
     currentVersion += 1
 
     // make sure newly committed version is reflected to the cache (overwritten)
-    assert(getData(provider) === Set("a" -> -2))
+    assert(getLatestData(provider) === Set("a" -> -2))
     loadedMaps = provider.getLoadedMaps()
     checkLoadedVersions(loadedMaps, count = 1, earliestKey = 2, latestKey = 2)
     checkVersion(loadedMaps, 2, Map("a" -> -2))
@@ -187,61 +141,17 @@ class StateStoreSuite extends StateStoreSuiteBase[HDFSBackedStateStoreProvider]
 
     // commit the ver 1 : never cached
     currentVersion = incrementVersion(provider, currentVersion)
-    assert(getData(provider) === Set("a" -> 1))
+    assert(getLatestData(provider) === Set("a" -> 1))
     var loadedMaps = provider.getLoadedMaps()
     assert(loadedMaps.size() === 0)
 
     // commit the ver 2 : never cached
     currentVersion = incrementVersion(provider, currentVersion)
-    assert(getData(provider) === Set("a" -> 2))
+    assert(getLatestData(provider) === Set("a" -> 2))
     loadedMaps = provider.getLoadedMaps()
     assert(loadedMaps.size() === 0)
   }
 
-  test("snapshotting") {
-    val provider = newStoreProvider(opId = Random.nextInt, partition = 0, minDeltasForSnapshot = 5)
-
-    var currentVersion = 0
-
-    currentVersion = updateVersionTo(provider, currentVersion, 2)
-    require(getData(provider) === Set("a" -> 2))
-    provider.doMaintenance()               // should not generate snapshot files
-    assert(getData(provider) === Set("a" -> 2))
-
-    for (i <- 1 to currentVersion) {
-      assert(fileExists(provider, i, isSnapshot = false))  // all delta files present
-      assert(!fileExists(provider, i, isSnapshot = true))  // no snapshot files present
-    }
-
-    // After version 6, snapshotting should generate one snapshot file
-    currentVersion = updateVersionTo(provider, currentVersion, 6)
-    require(getData(provider) === Set("a" -> 6), "store not updated correctly")
-    provider.doMaintenance()       // should generate snapshot files
-
-    val snapshotVersion = (0 to 6).find(version => fileExists(provider, version, isSnapshot = true))
-    assert(snapshotVersion.nonEmpty, "snapshot file not generated")
-    deleteFilesEarlierThanVersion(provider, snapshotVersion.get)
-    assert(
-      getData(provider, snapshotVersion.get) === Set("a" -> snapshotVersion.get),
-      "snapshotting messed up the data of the snapshotted version")
-    assert(
-      getData(provider) === Set("a" -> 6),
-      "snapshotting messed up the data of the final version")
-
-    // After version 20, snapshotting should generate newer snapshot files
-    currentVersion = updateVersionTo(provider, currentVersion, 20)
-    require(getData(provider) === Set("a" -> 20), "store not updated correctly")
-    provider.doMaintenance()       // do snapshot
-
-    val latestSnapshotVersion = (0 to 20).filter(version =>
-      fileExists(provider, version, isSnapshot = true)).lastOption
-    assert(latestSnapshotVersion.nonEmpty, "no snapshot file found")
-    assert(latestSnapshotVersion.get > snapshotVersion.get, "newer snapshot not generated")
-
-    deleteFilesEarlierThanVersion(provider, latestSnapshotVersion.get)
-    assert(getData(provider) === Set("a" -> 20), "snapshotting messed up the data")
-  }
-
   test("cleaning") {
     val provider = newStoreProvider(opId = Random.nextInt, partition = 0, minDeltasForSnapshot = 5)
 
@@ -309,15 +219,6 @@ class StateStoreSuite extends StateStoreSuiteBase[HDFSBackedStateStoreProvider]
     }
   }
 
-  test("reports memory usage") {
-    val provider = newStoreProvider()
-    val store = provider.getStore(0)
-    val noDataMemoryUsed = store.metrics.memoryUsedBytes
-    put(store, "a", 1)
-    store.commit()
-    assert(store.metrics.memoryUsedBytes > noDataMemoryUsed)
-  }
-
   test("reports memory usage on current version") {
     def getSizeOfStateForCurrentVersion(metrics: StateStoreMetrics): Long = {
       val metricPair = metrics.customMetrics.find(_._1.name == "stateOnCurrentVersionSizeBytes")
@@ -334,156 +235,6 @@ class StateStoreSuite extends StateStoreSuiteBase[HDFSBackedStateStoreProvider]
     assert(getSizeOfStateForCurrentVersion(store.metrics) > noDataMemoryUsed)
   }
 
-  test("StateStore.get") {
-    quietly {
-      val dir = newDir()
-      val storeId = StateStoreProviderId(StateStoreId(dir, 0, 0), UUID.randomUUID)
-      val storeConf = StateStoreConf.empty
-      val hadoopConf = new Configuration()
-
-      // Verify that trying to get incorrect versions throw errors
-      intercept[IllegalArgumentException] {
-        StateStore.get(
-          storeId, keySchema, valueSchema, None, -1, storeConf, hadoopConf)
-      }
-      assert(!StateStore.isLoaded(storeId)) // version -1 should not attempt to load the store
-
-      intercept[IllegalStateException] {
-        StateStore.get(
-          storeId, keySchema, valueSchema, None, 1, storeConf, hadoopConf)
-      }
-
-      // Increase version of the store and try to get again
-      val store0 = StateStore.get(
-        storeId, keySchema, valueSchema, None, 0, storeConf, hadoopConf)
-      assert(store0.version === 0)
-      put(store0, "a", 1)
-      store0.commit()
-
-      val store1 = StateStore.get(
-        storeId, keySchema, valueSchema, None, 1, storeConf, hadoopConf)
-      assert(StateStore.isLoaded(storeId))
-      assert(store1.version === 1)
-      assert(rowsToSet(store1.iterator()) === Set("a" -> 1))
-
-      // Verify that you can also load older version
-      val store0reloaded = StateStore.get(
-        storeId, keySchema, valueSchema, None, 0, storeConf, hadoopConf)
-      assert(store0reloaded.version === 0)
-      assert(rowsToSet(store0reloaded.iterator()) === Set.empty)
-
-      // Verify that you can remove the store and still reload and use it
-      StateStore.unload(storeId)
-      assert(!StateStore.isLoaded(storeId))
-
-      val store1reloaded = StateStore.get(
-        storeId, keySchema, valueSchema, None, 1, storeConf, hadoopConf)
-      assert(StateStore.isLoaded(storeId))
-      assert(store1reloaded.version === 1)
-      put(store1reloaded, "a", 2)
-      assert(store1reloaded.commit() === 2)
-      assert(rowsToSet(store1reloaded.iterator()) === Set("a" -> 2))
-    }
-  }
-
-  test("maintenance") {
-    val conf = new SparkConf()
-      .setMaster("local")
-      .setAppName("test")
-      // Make maintenance thread do snapshots and cleanups very fast
-      .set(StateStore.MAINTENANCE_INTERVAL_CONFIG, "10ms")
-      // Make sure that when SparkContext stops, the StateStore maintenance thread 'quickly'
-      // fails to talk to the StateStoreCoordinator and unloads all the StateStores
-      .set(RPC_NUM_RETRIES, 1)
-    val opId = 0
-    val dir = newDir()
-    val storeProviderId = StateStoreProviderId(StateStoreId(dir, opId, 0), UUID.randomUUID)
-    val sqlConf = new SQLConf()
-    sqlConf.setConf(SQLConf.MIN_BATCHES_TO_RETAIN, 2)
-    val storeConf = StateStoreConf(sqlConf)
-    val hadoopConf = new Configuration()
-    val provider = newStoreProvider(storeProviderId.storeId)
-
-    var latestStoreVersion = 0
-
-    def generateStoreVersions(): Unit = {
-      for (i <- 1 to 20) {
-        val store = StateStore.get(storeProviderId, keySchema, valueSchema, None,
-          latestStoreVersion, storeConf, hadoopConf)
-        put(store, "a", i)
-        store.commit()
-        latestStoreVersion += 1
-      }
-    }
-
-    val timeoutDuration = 1.minute
-
-    quietly {
-      withSpark(new SparkContext(conf)) { sc =>
-        withCoordinatorRef(sc) { coordinatorRef =>
-          require(!StateStore.isMaintenanceRunning, "StateStore is unexpectedly running")
-
-          // Generate sufficient versions of store for snapshots
-          generateStoreVersions()
-
-          eventually(timeout(timeoutDuration)) {
-            // Store should have been reported to the coordinator
-            assert(coordinatorRef.getLocation(storeProviderId).nonEmpty,
-              "active instance was not reported")
-
-            // Background maintenance should clean up and generate snapshots
-            assert(StateStore.isMaintenanceRunning, "Maintenance task is not running")
-
-            // Some snapshots should have been generated
-            val snapshotVersions = (1 to latestStoreVersion).filter { version =>
-              fileExists(provider, version, isSnapshot = true)
-            }
-            assert(snapshotVersions.nonEmpty, "no snapshot file found")
-          }
-
-          // Generate more versions such that there is another snapshot and
-          // the earliest delta file will be cleaned up
-          generateStoreVersions()
-
-          // Earliest delta file should get cleaned up
-          eventually(timeout(timeoutDuration)) {
-            assert(!fileExists(provider, 1, isSnapshot = false), "earliest file not deleted")
-          }
-
-          // If driver decides to deactivate all stores related to a query run,
-          // then this instance should be unloaded
-          coordinatorRef.deactivateInstances(storeProviderId.queryRunId)
-          eventually(timeout(timeoutDuration)) {
-            assert(!StateStore.isLoaded(storeProviderId))
-          }
-
-          // Reload the store and verify
-          StateStore.get(storeProviderId, keySchema, valueSchema, indexOrdinal = None,
-            latestStoreVersion, storeConf, hadoopConf)
-          assert(StateStore.isLoaded(storeProviderId))
-
-          // If some other executor loads the store, then this instance should be unloaded
-          coordinatorRef.reportActiveInstance(storeProviderId, "other-host", "other-exec")
-          eventually(timeout(timeoutDuration)) {
-            assert(!StateStore.isLoaded(storeProviderId))
-          }
-
-          // Reload the store and verify
-          StateStore.get(storeProviderId, keySchema, valueSchema, indexOrdinal = None,
-            latestStoreVersion, storeConf, hadoopConf)
-          assert(StateStore.isLoaded(storeProviderId))
-        }
-      }
-
-      // Verify if instance is unloaded if SparkContext is stopped
-      eventually(timeout(timeoutDuration)) {
-        require(SparkEnv.get === null)
-        assert(!StateStore.isLoaded(storeProviderId))
-        assert(!StateStore.isMaintenanceRunning)
-      }
-    }
-  }
-
   testQuietly("SPARK-18342: commit fails when rename fails") {
     import RenameReturnsFalseFileSystem._
     val dir = scheme + "://" + newDir()
@@ -741,13 +492,21 @@ class StateStoreSuite extends StateStoreSuiteBase[HDFSBackedStateStoreProvider]
     newStoreProvider(storeId.operatorId, storeId.partitionId, dir = storeId.checkpointRootLocation)
   }
 
+  override def newStoreProvider(
+      minDeltasForSnapshot: Int,
+      numOfVersToRetainInMemory: Int): HDFSBackedStateStoreProvider = {
+    newStoreProvider(opId = Random.nextInt(), partition = 0,
+      minDeltasForSnapshot = minDeltasForSnapshot,
+      numOfVersToRetainInMemory = numOfVersToRetainInMemory)
+  }
+
   override def getLatestData(storeProvider: HDFSBackedStateStoreProvider): Set[(String, Int)] = {
-    getData(storeProvider)
+    getData(storeProvider, -1)
   }
 
   override def getData(
     provider: HDFSBackedStateStoreProvider,
-    version: Int = -1): Set[(String, Int)] = {
+    version: Int): Set[(String, Int)] = {
     val reloadedProvider = newStoreProvider(provider.stateStoreId)
     if (version < 0) {
       reloadedProvider.latestIterator().map(rowsToStringInt).toSet
@@ -756,6 +515,17 @@ class StateStoreSuite extends StateStoreSuiteBase[HDFSBackedStateStoreProvider]
     }
   }
 
+  override def getDefaultSQLConf(
+      minDeltasForSnapshot: Int,
+      numOfVersToRetainInMemory: Int): SQLConf = {
+    val sqlConf = new SQLConf()
+    sqlConf.setConf(SQLConf.STATE_STORE_MIN_DELTAS_FOR_SNAPSHOT, minDeltasForSnapshot)
+    sqlConf.setConf(SQLConf.MAX_BATCHES_TO_RETAIN_IN_MEMORY, numOfVersToRetainInMemory)
+    sqlConf.setConf(SQLConf.MIN_BATCHES_TO_RETAIN, 2)
+    sqlConf.setConf(SQLConf.STATE_STORE_COMPRESSION_CODEC, SQLConf.get.stateStoreCompressionCodec)
+    sqlConf
+  }
+
   def newStoreProvider(
       opId: Long,
       partition: Int,
@@ -763,10 +533,7 @@ class StateStoreSuite extends StateStoreSuiteBase[HDFSBackedStateStoreProvider]
       minDeltasForSnapshot: Int = SQLConf.STATE_STORE_MIN_DELTAS_FOR_SNAPSHOT.defaultValue.get,
       numOfVersToRetainInMemory: Int = SQLConf.MAX_BATCHES_TO_RETAIN_IN_MEMORY.defaultValue.get,
       hadoopConf: Configuration = new Configuration): HDFSBackedStateStoreProvider = {
-    val sqlConf = new SQLConf()
-    sqlConf.setConf(SQLConf.STATE_STORE_MIN_DELTAS_FOR_SNAPSHOT, minDeltasForSnapshot)
-    sqlConf.setConf(SQLConf.MAX_BATCHES_TO_RETAIN_IN_MEMORY, numOfVersToRetainInMemory)
-    sqlConf.setConf(SQLConf.MIN_BATCHES_TO_RETAIN, 2)
+    val sqlConf = getDefaultSQLConf(minDeltasForSnapshot, numOfVersToRetainInMemory)
     val provider = new HDFSBackedStateStoreProvider()
     provider.init(
       StateStoreId(dir, opId, partition),
@@ -778,27 +545,25 @@ class StateStoreSuite extends StateStoreSuiteBase[HDFSBackedStateStoreProvider]
     provider
   }
 
-  def fileExists(
-      provider: HDFSBackedStateStoreProvider,
-      version: Long,
-      isSnapshot: Boolean): Boolean = {
-    val method = PrivateMethod[Path](Symbol("baseDir"))
-    val basePath = provider invokePrivate method()
-    val fileName = if (isSnapshot) s"$version.snapshot" else s"$version.delta"
-    val filePath = new File(basePath.toString, fileName)
-    filePath.exists
+  def checkLoadedVersions(
+      loadedMaps: util.SortedMap[Long, ProviderMapType],
+      count: Int,
+      earliestKey: Long,
+      latestKey: Long): Unit = {
+    assert(loadedMaps.size() === count)
+    assert(loadedMaps.firstKey() === earliestKey)
+    assert(loadedMaps.lastKey() === latestKey)
   }
 
-  def deleteFilesEarlierThanVersion(provider: HDFSBackedStateStoreProvider, version: Long): Unit = {
-    val method = PrivateMethod[Path](Symbol("baseDir"))
-    val basePath = provider invokePrivate method()
-    for (version <- 0 until version.toInt) {
-      for (isSnapshot <- Seq(false, true)) {
-        val fileName = if (isSnapshot) s"$version.snapshot" else s"$version.delta"
-        val filePath = new File(basePath.toString, fileName)
-        if (filePath.exists) filePath.delete()
-      }
-    }
+  def checkVersion(
+      loadedMaps: util.SortedMap[Long, ProviderMapType],
+      version: Long,
+      expectedData: Map[String, Int]): Unit = {
+    val originValueMap = loadedMaps.get(version).asScala.map { entry =>
+      rowToString(entry._1) -> rowToInt(entry._2)
+    }.toMap
+
+    assert(originValueMap === expectedData)
   }
 
   def corruptFile(
@@ -815,10 +580,17 @@ class StateStoreSuite extends StateStoreSuiteBase[HDFSBackedStateStoreProvider]
 }
 
 abstract class StateStoreSuiteBase[ProviderClass <: StateStoreProvider]
-  extends SparkFunSuite {
+  extends StateStoreCodecsTest with PrivateMethodTester {
   import StateStoreTestsHelper._
+  import StateStoreCoordinatorSuite._
 
-  test("get, put, remove, commit, and all data iterator") {
+  type MapType = mutable.HashMap[UnsafeRow, UnsafeRow]
+  type ProviderMapType = java.util.concurrent.ConcurrentHashMap[UnsafeRow, UnsafeRow]
+
+  protected val keySchema: StructType
+  protected val valueSchema: StructType
+
+  testWithAllCodec("get, put, remove, commit, and all data iterator") {
     val provider = newStoreProvider()
 
     // Verify state before starting a new set of updates
@@ -833,7 +605,6 @@ abstract class StateStoreSuiteBase[ProviderClass <: StateStoreProvider]
     // Verify state after updating
     put(store, "a", 1)
     assert(get(store, "a") === Some(1))
-    assert(store.metrics.numKeys === 1)
 
     assert(store.iterator().nonEmpty)
     assert(getLatestData(provider).isEmpty)
@@ -841,9 +612,7 @@ abstract class StateStoreSuiteBase[ProviderClass <: StateStoreProvider]
     // Make updates, commit and then verify state
     put(store, "b", 2)
     put(store, "aa", 3)
-    assert(store.metrics.numKeys === 3)
     remove(store, _.startsWith("a"))
-    assert(store.metrics.numKeys === 1)
     assert(store.commit() === 1)
 
     assert(store.hasCommitted)
@@ -861,16 +630,38 @@ abstract class StateStoreSuiteBase[ProviderClass <: StateStoreProvider]
     // New updates to the reloaded store with new version, and does not change old version
     val reloadedProvider = newStoreProvider(store.id)
     val reloadedStore = reloadedProvider.getStore(1)
-    assert(reloadedStore.metrics.numKeys === 1)
     put(reloadedStore, "c", 4)
-    assert(reloadedStore.metrics.numKeys === 2)
     assert(reloadedStore.commit() === 2)
     assert(rowsToSet(reloadedStore.iterator()) === Set("b" -> 2, "c" -> 4))
     assert(getLatestData(provider) === Set("b" -> 2, "c" -> 4))
     assert(getData(provider, version = 1) === Set("b" -> 2))
   }
 
-  test("removing while iterating") {
+  testWithAllCodec("numKeys metrics") {
+    val provider = newStoreProvider()
+
+    // Verify state before starting a new set of updates
+    assert(getLatestData(provider).isEmpty)
+
+    val store = provider.getStore(0)
+    put(store, "a", 1)
+    put(store, "b", 2)
+    put(store, "c", 3)
+    put(store, "d", 4)
+    put(store, "e", 5)
+    assert(store.commit() === 1)
+    assert(store.metrics.numKeys === 5)
+    assert(rowsToSet(store.iterator()) === Set("a" -> 1, "b" -> 2, "c" -> 3, "d" -> 4, "e" -> 5))
+
+    val reloadedProvider = newStoreProvider(store.id)
+    val reloadedStore = reloadedProvider.getStore(1)
+    remove(reloadedStore, _ == "b")
+    assert(reloadedStore.commit() === 2)
+    assert(reloadedStore.metrics.numKeys === 4)
+    assert(rowsToSet(reloadedStore.iterator()) === Set("a" -> 1, "c" -> 3, "d" -> 4, "e" -> 5))
+  }
+
+  testWithAllCodec("removing while iterating") {
     val provider = newStoreProvider()
 
     // Verify state before starting a new set of updates
@@ -892,7 +683,7 @@ abstract class StateStoreSuiteBase[ProviderClass <: StateStoreProvider]
     assert(get(store, "b") === None)
   }
 
-  test("abort") {
+  testWithAllCodec("abort") {
     val provider = newStoreProvider()
     val store = provider.getStore(0)
     put(store, "a", 1)
@@ -905,7 +696,7 @@ abstract class StateStoreSuiteBase[ProviderClass <: StateStoreProvider]
     store1.abort()
   }
 
-  test("getStore with invalid versions") {
+  testWithAllCodec("getStore with invalid versions") {
     val provider = newStoreProvider()
 
     def checkInvalidVersion(version: Int): Unit = {
@@ -939,7 +730,7 @@ abstract class StateStoreSuiteBase[ProviderClass <: StateStoreProvider]
     checkInvalidVersion(3)
   }
 
-  test("two concurrent StateStores - one for read-only and one for read-write") {
+  testWithAllCodec("two concurrent StateStores - one for read-only and one for read-write") {
     // During Streaming Aggregation, we have two StateStores per task, one used as read-only in
     // `StateStoreRestoreExec`, and one read-write used in `StateStoreSaveExec`. `StateStore.abort`
     // will be called for these StateStores if they haven't committed their results. We need to
@@ -957,7 +748,7 @@ abstract class StateStoreSuiteBase[ProviderClass <: StateStoreProvider]
 
     // two state stores
     val provider1 = newStoreProvider(storeId)
-    val restoreStore = provider1.getStore(1)
+    val restoreStore = provider1.getReadStore(1)
     val saveStore = provider1.getStore(1)
 
     put(saveStore, key, get(restoreStore, key).get + 1)
@@ -970,12 +761,254 @@ abstract class StateStoreSuiteBase[ProviderClass <: StateStoreProvider]
     assert(rowsToSet(finalStore.iterator()) === Set(key -> 2))
   }
 
+  test("maintenance") {
+    val conf = new SparkConf()
+      .setMaster("local")
+      .setAppName("test")
+      // Make sure that when SparkContext stops, the StateStore maintenance thread 'quickly'
+      // fails to talk to the StateStoreCoordinator and unloads all the StateStores
+      .set(RPC_NUM_RETRIES, 1)
+    val opId = 0
+    val dir1 = newDir()
+    val storeProviderId1 = StateStoreProviderId(StateStoreId(dir1, opId, 0), UUID.randomUUID)
+    val dir2 = newDir()
+    val storeProviderId2 = StateStoreProviderId(StateStoreId(dir2, opId, 1), UUID.randomUUID)
+    val sqlConf = getDefaultSQLConf(SQLConf.STATE_STORE_MIN_DELTAS_FOR_SNAPSHOT.defaultValue.get,
+      SQLConf.MAX_BATCHES_TO_RETAIN_IN_MEMORY.defaultValue.get)
+    sqlConf.setConf(SQLConf.MIN_BATCHES_TO_RETAIN, 2)
+    // Make maintenance thread do snapshots and cleanups very fast
+    sqlConf.setConf(SQLConf.STREAMING_MAINTENANCE_INTERVAL, 10L)
+    val storeConf = StateStoreConf(sqlConf)
+    val hadoopConf = new Configuration()
+    val provider = newStoreProvider(storeProviderId1.storeId)
+
+    var latestStoreVersion = 0
+
+    def generateStoreVersions(): Unit = {
+      for (i <- 1 to 20) {
+        val store = StateStore.get(storeProviderId1, keySchema, valueSchema, None,
+          latestStoreVersion, storeConf, hadoopConf)
+        put(store, "a", i)
+        store.commit()
+        latestStoreVersion += 1
+      }
+    }
+
+    val timeoutDuration = 1.minute
+
+    quietly {
+      withSpark(new SparkContext(conf)) { sc =>
+        withCoordinatorRef(sc) { coordinatorRef =>
+          require(!StateStore.isMaintenanceRunning, "StateStore is unexpectedly running")
+
+          // Generate sufficient versions of store for snapshots
+          generateStoreVersions()
+
+          eventually(timeout(timeoutDuration)) {
+            // Store should have been reported to the coordinator
+            assert(coordinatorRef.getLocation(storeProviderId1).nonEmpty,
+              "active instance was not reported")
+
+            // Background maintenance should clean up and generate snapshots
+            assert(StateStore.isMaintenanceRunning, "Maintenance task is not running")
+
+            // Some snapshots should have been generated
+            val snapshotVersions = (1 to latestStoreVersion).filter { version =>
+              fileExists(provider, version, isSnapshot = true)
+            }
+            assert(snapshotVersions.nonEmpty, "no snapshot file found")
+          }
+
+          // Generate more versions such that there is another snapshot and
+          // the earliest delta file will be cleaned up
+          generateStoreVersions()
+
+          // Earliest delta file should get cleaned up
+          eventually(timeout(timeoutDuration)) {
+            assert(!fileExists(provider, 1, isSnapshot = false), "earliest file not deleted")
+          }
+
+          // If driver decides to deactivate all stores related to a query run,
+          // then this instance should be unloaded
+          coordinatorRef.deactivateInstances(storeProviderId1.queryRunId)
+          eventually(timeout(timeoutDuration)) {
+            assert(!StateStore.isLoaded(storeProviderId1))
+          }
+
+          // Reload the store and verify
+          StateStore.get(storeProviderId1, keySchema, valueSchema, indexOrdinal = None,
+            latestStoreVersion, storeConf, hadoopConf)
+          assert(StateStore.isLoaded(storeProviderId1))
+
+          // If some other executor loads the store, then this instance should be unloaded
+          coordinatorRef
+            .reportActiveInstance(storeProviderId1, "other-host", "other-exec", Seq.empty)
+          eventually(timeout(timeoutDuration)) {
+            assert(!StateStore.isLoaded(storeProviderId1))
+          }
+
+          // Reload the store and verify
+          StateStore.get(storeProviderId1, keySchema, valueSchema, indexOrdinal = None,
+            latestStoreVersion, storeConf, hadoopConf)
+          assert(StateStore.isLoaded(storeProviderId1))
+
+          // If some other executor loads the store, and when this executor loads other store,
+          // then this executor should unload inactive instances immediately.
+          coordinatorRef
+            .reportActiveInstance(storeProviderId1, "other-host", "other-exec", Seq.empty)
+          StateStore.get(storeProviderId2, keySchema, valueSchema, indexOrdinal = None,
+            0, storeConf, hadoopConf)
+          assert(!StateStore.isLoaded(storeProviderId1))
+          assert(StateStore.isLoaded(storeProviderId2))
+        }
+      }
+
+      // Verify if instance is unloaded if SparkContext is stopped
+      eventually(timeout(timeoutDuration)) {
+        require(SparkEnv.get === null)
+        assert(!StateStore.isLoaded(storeProviderId1))
+        assert(!StateStore.isLoaded(storeProviderId2))
+        assert(!StateStore.isMaintenanceRunning)
+      }
+    }
+  }
+
+  test("StateStore.get") {
+    quietly {
+      val dir = newDir()
+      val storeId = StateStoreProviderId(StateStoreId(dir, 0, 0), UUID.randomUUID)
+      val storeConf = getDefaultStoreConf
+      val hadoopConf = new Configuration()
+
+      // Verify that trying to get incorrect versions throw errors
+      intercept[IllegalArgumentException] {
+        StateStore.get(
+          storeId, keySchema, valueSchema, None, -1, storeConf, hadoopConf)
+      }
+      assert(!StateStore.isLoaded(storeId)) // version -1 should not attempt to load the store
+
+      intercept[IllegalStateException] {
+        StateStore.get(
+          storeId, keySchema, valueSchema, None, 1, storeConf, hadoopConf)
+      }
+
+      // Increase version of the store and try to get again
+      val store0 = StateStore.get(
+        storeId, keySchema, valueSchema, None, 0, storeConf, hadoopConf)
+      assert(store0.version === 0)
+      put(store0, "a", 1)
+      store0.commit()
+
+      val store1 = StateStore.get(
+        storeId, keySchema, valueSchema, None, 1, storeConf, hadoopConf)
+      assert(StateStore.isLoaded(storeId))
+      assert(store1.version === 1)
+      assert(rowsToSet(store1.iterator()) === Set("a" -> 1))
+
+      // Verify that you can also load older version
+      val store0reloaded = StateStore.get(
+        storeId, keySchema, valueSchema, None, 0, storeConf, hadoopConf)
+      assert(store0reloaded.version === 0)
+      assert(rowsToSet(store0reloaded.iterator()) === Set.empty)
+
+      // Verify that you can remove the store and still reload and use it
+      StateStore.unload(storeId)
+      assert(!StateStore.isLoaded(storeId))
+
+      val store1reloaded = StateStore.get(
+        storeId, keySchema, valueSchema, None, 1, storeConf, hadoopConf)
+      assert(StateStore.isLoaded(storeId))
+      assert(store1reloaded.version === 1)
+      put(store1reloaded, "a", 2)
+      assert(store1reloaded.commit() === 2)
+      assert(rowsToSet(store1reloaded.iterator()) === Set("a" -> 2))
+    }
+  }
+
+  test("snapshotting") {
+    val provider = newStoreProvider(minDeltasForSnapshot = 5, numOfVersToRetainInMemory = 2)
+
+    var currentVersion = 0
+
+    currentVersion = updateVersionTo(provider, currentVersion, 2)
+    require(getLatestData(provider) === Set("a" -> 2))
+    provider.doMaintenance()               // should not generate snapshot files
+    assert(getLatestData(provider) === Set("a" -> 2))
+
+    for (i <- 1 to currentVersion) {
+      assert(fileExists(provider, i, isSnapshot = false))  // all delta files present
+      assert(!fileExists(provider, i, isSnapshot = true))  // no snapshot files present
+    }
+
+    // After version 6, snapshotting should generate one snapshot file
+    currentVersion = updateVersionTo(provider, currentVersion, 6)
+    require(getLatestData(provider) === Set("a" -> 6), "store not updated correctly")
+    provider.doMaintenance()       // should generate snapshot files
+
+    val snapshotVersion = (0 to 6).find(version => fileExists(provider, version, isSnapshot = true))
+    assert(snapshotVersion.nonEmpty, "snapshot file not generated")
+    deleteFilesEarlierThanVersion(provider, snapshotVersion.get)
+    assert(
+      getData(provider, snapshotVersion.get) === Set("a" -> snapshotVersion.get),
+      "snapshotting messed up the data of the snapshotted version")
+    assert(
+      getLatestData(provider) === Set("a" -> 6),
+      "snapshotting messed up the data of the final version")
+
+    // After version 20, snapshotting should generate newer snapshot files
+    currentVersion = updateVersionTo(provider, currentVersion, 20)
+    require(getLatestData(provider) === Set("a" -> 20), "store not updated correctly")
+    provider.doMaintenance()       // do snapshot
+
+    val latestSnapshotVersion = (0 to 20).filter(version =>
+      fileExists(provider, version, isSnapshot = true)).lastOption
+    assert(latestSnapshotVersion.nonEmpty, "no snapshot file found")
+    assert(latestSnapshotVersion.get > snapshotVersion.get, "newer snapshot not generated")
+
+    deleteFilesEarlierThanVersion(provider, latestSnapshotVersion.get)
+    assert(getLatestData(provider) === Set("a" -> 20), "snapshotting messed up the data")
+  }
+
+  test("reports memory usage") {
+    val provider = newStoreProvider()
+    val store = provider.getStore(0)
+    val noDataMemoryUsed = store.metrics.memoryUsedBytes
+    put(store, "a", 1)
+    store.commit()
+    assert(store.metrics.memoryUsedBytes > noDataMemoryUsed)
+  }
+
+  test("SPARK-34270: StateStoreMetrics.combine should not override individual metrics") {
+    val customSumMetric = StateStoreCustomSumMetric("metric1", "custom metric 1")
+    val customSizeMetric = StateStoreCustomSizeMetric("metric2", "custom metric 2")
+    val customTimingMetric = StateStoreCustomTimingMetric("metric3", "custom metric 3")
+
+    val leftCustomMetrics: Map[StateStoreCustomMetric, Long] =
+      Map(customSumMetric -> 10L, customSizeMetric -> 5L, customTimingMetric -> 100L)
+    val leftMetrics = StateStoreMetrics(1, 10, leftCustomMetrics)
+
+    val rightCustomMetrics: Map[StateStoreCustomMetric, Long] =
+      Map(customSumMetric -> 20L, customSizeMetric -> 15L, customTimingMetric -> 300L)
+    val rightMetrics = StateStoreMetrics(3, 20, rightCustomMetrics)
+
+    val combinedMetrics = StateStoreMetrics.combine(Seq(leftMetrics, rightMetrics))
+    assert(combinedMetrics.numKeys == 4)
+    assert(combinedMetrics.memoryUsedBytes == 30)
+    assert(combinedMetrics.customMetrics.size == 3)
+    assert(combinedMetrics.customMetrics(customSumMetric) == 30L)
+    assert(combinedMetrics.customMetrics(customSizeMetric) == 20L)
+    assert(combinedMetrics.customMetrics(customTimingMetric) == 400L)
+  }
+
   /** Return a new provider with a random id */
   def newStoreProvider(): ProviderClass
 
   /** Return a new provider with the given id */
   def newStoreProvider(storeId: StateStoreId): ProviderClass
 
+  /** Return a new provider with minimum delta and version to retain in memory */
+  def newStoreProvider(minDeltasForSnapshot: Int, numOfVersToRetainInMemory: Int): ProviderClass
+
   /** Get the latest data referred to by the given provider but not using this provider */
   def getLatestData(storeProvider: ProviderClass): Set[(String, Int)]
 
@@ -992,6 +1025,54 @@ abstract class StateStoreSuiteBase[ProviderClass <: StateStoreProvider]
       }
     }
   }
+
+  /** Get the `SQLConf` by the given minimum delta and version to retain in memory */
+  def getDefaultSQLConf(minDeltasForSnapshot: Int, numOfVersToRetainInMemory: Int): SQLConf
+
+  /** Get the `StateStoreConf` used by the tests with default setting */
+  def getDefaultStoreConf(): StateStoreConf = StateStoreConf.empty
+
+  protected def fileExists(
+      provider: ProviderClass,
+      version: Long,
+      isSnapshot: Boolean): Boolean = {
+    val method = PrivateMethod[Path](Symbol("baseDir"))
+    val basePath = provider invokePrivate method()
+    val fileName = if (isSnapshot) s"$version.snapshot" else s"$version.delta"
+    val filePath = new File(basePath.toString, fileName)
+    filePath.exists
+  }
+
+  def updateVersionTo(
+      provider: StateStoreProvider,
+      currentVersion: Int,
+      targetVersion: Int): Int = {
+    var newCurrentVersion = currentVersion
+    for (i <- newCurrentVersion until targetVersion) {
+      newCurrentVersion = incrementVersion(provider, i)
+    }
+    require(newCurrentVersion === targetVersion)
+    newCurrentVersion
+  }
+
+  def incrementVersion(provider: StateStoreProvider, currentVersion: Int): Int = {
+    val store = provider.getStore(currentVersion)
+    put(store, "a", currentVersion + 1)
+    store.commit()
+    currentVersion + 1
+  }
+
+  def deleteFilesEarlierThanVersion(provider: ProviderClass, version: Long): Unit = {
+    val method = PrivateMethod[Path](Symbol("baseDir"))
+    val basePath = provider invokePrivate method()
+    for (version <- 0 until version.toInt) {
+      for (isSnapshot <- Seq(false, true)) {
+        val fileName = if (isSnapshot) s"$version.snapshot" else s"$version.delta"
+        val filePath = new File(basePath.toString, fileName)
+        if (filePath.exists) filePath.delete()
+      }
+    }
+  }
 }
 
 object StateStoreTestsHelper {
@@ -1033,7 +1114,7 @@ object StateStoreTestsHelper {
     store.put(stringToRow(key), intToRow(value))
   }
 
-  def get(store: StateStore, key: String): Option[Int] = {
+  def get(store: ReadStateStore, key: String): Option[Int] = {
     Option(store.get(stringToRow(key))).map(rowToInt)
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/MetricsAggregationBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/MetricsAggregationBenchmark.scala
index c09ff51ecaff2..533d98da240fc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/MetricsAggregationBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/MetricsAggregationBenchmark.scala
@@ -39,7 +39,8 @@ import org.apache.spark.util.kvstore.InMemoryStore
  * Benchmark for metrics aggregation in the SQL listener.
  * {{{
  *   To run this benchmark:
- *   1. without sbt: bin/spark-submit --class <this class> --jars <core test jar>
+ *   1. without sbt:
+ *      bin/spark-submit --class <this class> --jars <core test jar> <spark sql test jar>
  *   2. build/sbt "core/test:runMain <this class>"
  *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "core/test:runMain <this class>"
  *      Results will be written to "benchmarks/MetricsAggregationBenchmark-results.txt".
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala
index 4d5cd109b7c24..612b74a661d39 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala
@@ -37,13 +37,18 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
 import org.apache.spark.sql.catalyst.util.quietly
+import org.apache.spark.sql.connector.{RangeInputPartition, SimpleScanBuilder}
+import org.apache.spark.sql.connector.metric.{CustomMetric, CustomTaskMetric}
+import org.apache.spark.sql.connector.read.{InputPartition, PartitionReader, PartitionReaderFactory}
 import org.apache.spark.sql.execution.{LeafExecNode, QueryExecution, SparkPlanInfo, SQLExecution}
 import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecution
+import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
 import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics}
 import org.apache.spark.sql.functions.count
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.StaticSQLConf.UI_RETAINED_EXECUTIONS
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.StructType
 import org.apache.spark.status.ElementTrackingStore
 import org.apache.spark.util.{AccumulatorMetadata, JsonProtocol, LongAccumulator}
 import org.apache.spark.util.kvstore.InMemoryStore
@@ -209,21 +214,24 @@ class SQLAppStatusListenerSuite extends SharedSparkSession with JsonTestUtils
       (1L, 0, 0, createAccumulatorInfos(accumulatorUpdates))
     )))
 
-    checkAnswer(statusStore.executionMetrics(executionId), accumulatorUpdates.mapValues(_ * 2))
+    checkAnswer(statusStore.executionMetrics(executionId),
+      accumulatorUpdates.mapValues(_ * 2).toMap)
 
     // Driver accumulator updates don't belong to this execution should be filtered and no
     // exception will be thrown.
     listener.onOtherEvent(SparkListenerDriverAccumUpdates(0, Seq((999L, 2L))))
 
-    checkAnswer(statusStore.executionMetrics(executionId), accumulatorUpdates.mapValues(_ * 2))
+    checkAnswer(statusStore.executionMetrics(executionId),
+      accumulatorUpdates.mapValues(_ * 2).toMap)
 
     listener.onExecutorMetricsUpdate(SparkListenerExecutorMetricsUpdate("", Seq(
       // (task id, stage id, stage attempt, accum updates)
       (0L, 0, 0, createAccumulatorInfos(accumulatorUpdates)),
-      (1L, 0, 0, createAccumulatorInfos(accumulatorUpdates.mapValues(_ * 2)))
+      (1L, 0, 0, createAccumulatorInfos(accumulatorUpdates.mapValues(_ * 2).toMap))
     )))
 
-    checkAnswer(statusStore.executionMetrics(executionId), accumulatorUpdates.mapValues(_ * 3))
+    checkAnswer(statusStore.executionMetrics(executionId),
+      accumulatorUpdates.mapValues(_ * 3).toMap)
 
     // Retrying a stage should reset the metrics
     listener.onStageSubmitted(SparkListenerStageSubmitted(createStageInfo(0, 1)))
@@ -236,7 +244,8 @@ class SQLAppStatusListenerSuite extends SharedSparkSession with JsonTestUtils
       (1L, 0, 1, createAccumulatorInfos(accumulatorUpdates))
     )))
 
-    checkAnswer(statusStore.executionMetrics(executionId), accumulatorUpdates.mapValues(_ * 2))
+    checkAnswer(statusStore.executionMetrics(executionId),
+      accumulatorUpdates.mapValues(_ * 2).toMap)
 
     // Ignore the task end for the first attempt
     listener.onTaskEnd(SparkListenerTaskEnd(
@@ -244,11 +253,12 @@ class SQLAppStatusListenerSuite extends SharedSparkSession with JsonTestUtils
       stageAttemptId = 0,
       taskType = "",
       reason = null,
-      createTaskInfo(0, 0, accums = accumulatorUpdates.mapValues(_ * 100)),
+      createTaskInfo(0, 0, accums = accumulatorUpdates.mapValues(_ * 100).toMap),
       new ExecutorMetrics,
       null))
 
-    checkAnswer(statusStore.executionMetrics(executionId), accumulatorUpdates.mapValues(_ * 2))
+    checkAnswer(statusStore.executionMetrics(executionId),
+      accumulatorUpdates.mapValues(_ * 2).toMap)
 
     // Finish two tasks
     listener.onTaskEnd(SparkListenerTaskEnd(
@@ -256,7 +266,7 @@ class SQLAppStatusListenerSuite extends SharedSparkSession with JsonTestUtils
       stageAttemptId = 1,
       taskType = "",
       reason = null,
-      createTaskInfo(0, 0, accums = accumulatorUpdates.mapValues(_ * 2)),
+      createTaskInfo(0, 0, accums = accumulatorUpdates.mapValues(_ * 2).toMap),
       new ExecutorMetrics,
       null))
     listener.onTaskEnd(SparkListenerTaskEnd(
@@ -264,11 +274,12 @@ class SQLAppStatusListenerSuite extends SharedSparkSession with JsonTestUtils
       stageAttemptId = 1,
       taskType = "",
       reason = null,
-      createTaskInfo(1, 0, accums = accumulatorUpdates.mapValues(_ * 3)),
+      createTaskInfo(1, 0, accums = accumulatorUpdates.mapValues(_ * 3).toMap),
       new ExecutorMetrics,
       null))
 
-    checkAnswer(statusStore.executionMetrics(executionId), accumulatorUpdates.mapValues(_ * 5))
+    checkAnswer(statusStore.executionMetrics(executionId),
+      accumulatorUpdates.mapValues(_ * 5).toMap)
 
     // Summit a new stage
     listener.onStageSubmitted(SparkListenerStageSubmitted(createStageInfo(1, 0)))
@@ -281,7 +292,8 @@ class SQLAppStatusListenerSuite extends SharedSparkSession with JsonTestUtils
       (1L, 1, 0, createAccumulatorInfos(accumulatorUpdates))
     )))
 
-    checkAnswer(statusStore.executionMetrics(executionId), accumulatorUpdates.mapValues(_ * 7))
+    checkAnswer(statusStore.executionMetrics(executionId),
+      accumulatorUpdates.mapValues(_ * 7).toMap)
 
     // Finish two tasks
     listener.onTaskEnd(SparkListenerTaskEnd(
@@ -289,7 +301,7 @@ class SQLAppStatusListenerSuite extends SharedSparkSession with JsonTestUtils
       stageAttemptId = 0,
       taskType = "",
       reason = null,
-      createTaskInfo(0, 0, accums = accumulatorUpdates.mapValues(_ * 3)),
+      createTaskInfo(0, 0, accums = accumulatorUpdates.mapValues(_ * 3).toMap),
       new ExecutorMetrics,
       null))
     listener.onTaskEnd(SparkListenerTaskEnd(
@@ -297,11 +309,12 @@ class SQLAppStatusListenerSuite extends SharedSparkSession with JsonTestUtils
       stageAttemptId = 0,
       taskType = "",
       reason = null,
-      createTaskInfo(1, 0, accums = accumulatorUpdates.mapValues(_ * 3)),
+      createTaskInfo(1, 0, accums = accumulatorUpdates.mapValues(_ * 3).toMap),
       new ExecutorMetrics,
       null))
 
-    checkAnswer(statusStore.executionMetrics(executionId), accumulatorUpdates.mapValues(_ * 11))
+    checkAnswer(statusStore.executionMetrics(executionId),
+      accumulatorUpdates.mapValues(_ * 11).toMap)
 
     assertJobs(statusStore.execution(executionId), running = Seq(0))
 
@@ -315,7 +328,8 @@ class SQLAppStatusListenerSuite extends SharedSparkSession with JsonTestUtils
 
     assertJobs(statusStore.execution(executionId), completed = Seq(0))
 
-    checkAnswer(statusStore.executionMetrics(executionId), accumulatorUpdates.mapValues(_ * 11))
+    checkAnswer(statusStore.executionMetrics(executionId),
+      accumulatorUpdates.mapValues(_ * 11).toMap)
   }
 
   test("control a plan explain mode in listeners via SQLConf") {
@@ -670,6 +684,174 @@ class SQLAppStatusListenerSuite extends SharedSparkSession with JsonTestUtils
     val sparkPlanInfo = SparkPlanInfo.fromSparkPlan(df.queryExecution.executedPlan)
     assert(sparkPlanInfo.nodeName === "WholeStageCodegen (2)")
   }
+
+  test("SPARK-32615,SPARK-33016: SQLMetrics validation after sparkPlanInfo updated in AQE") {
+    val statusStore = createStatusStore()
+    val listener = statusStore.listener.get
+
+    val executionId = 0
+    val df = createTestDataFrame
+
+    // oldPlan SQLMetrics
+    // SQLPlanMetric(duration,0,timing)
+    // SQLPlanMetric(number of output rows,1,sum)
+    // SQLPlanMetric(number of output rows,2,sum)
+    val oldPlan = SparkPlanInfo.fromSparkPlan(df.queryExecution.executedPlan)
+    val oldAccumulatorIds =
+      SparkPlanGraph(oldPlan)
+        .allNodes.flatMap(_.metrics.map(_.accumulatorId))
+
+    listener.onOtherEvent(SparkListenerSQLExecutionStart(
+      executionId,
+      "test",
+      "test",
+      df.queryExecution.toString,
+      oldPlan,
+      System.currentTimeMillis()))
+
+    listener.onJobStart(SparkListenerJobStart(
+      jobId = 0,
+      time = System.currentTimeMillis(),
+      stageInfos = Seq(createStageInfo(0, 0)),
+      createProperties(executionId)))
+
+    listener.onStageSubmitted(SparkListenerStageSubmitted(createStageInfo(0, 0)))
+    listener.onTaskStart(SparkListenerTaskStart(0, 0, createTaskInfo(0, 0)))
+
+    assert(statusStore.executionMetrics(executionId).isEmpty)
+
+    // update old metrics with Id 1 & 2, since 0 is timing metrics,
+    // timing metrics has a complicated string presentation so we don't test it here.
+    val oldMetricsValueMap = oldAccumulatorIds.sorted.tail.map(id => (id, 100L)).toMap
+    listener.onExecutorMetricsUpdate(SparkListenerExecutorMetricsUpdate("", Seq(
+      (0L, 0, 0, createAccumulatorInfos(oldMetricsValueMap))
+    )))
+
+    assert(statusStore.executionMetrics(executionId).size == 2)
+    statusStore.executionMetrics(executionId).foreach { m =>
+      assert(m._2 == "100")
+    }
+
+    listener.onTaskEnd(SparkListenerTaskEnd(
+      stageId = 0,
+      stageAttemptId = 0,
+      taskType = "",
+      reason = null,
+      createTaskInfo(0, 0),
+      new ExecutorMetrics,
+      null))
+
+    listener.onStageCompleted(SparkListenerStageCompleted(createStageInfo(0, 0)))
+
+    listener.onJobEnd(SparkListenerJobEnd(
+      jobId = 0,
+      time = System.currentTimeMillis(),
+      JobSucceeded
+    ))
+
+    val df2 = createTestDataFrame.filter("_2 > 2")
+    // newPlan SQLMetrics
+    // SQLPlanMetric(duration,3,timing)
+    // SQLPlanMetric(number of output rows,4,sum)
+    // SQLPlanMetric(number of output rows,5,sum)
+    val newPlan = SparkPlanInfo.fromSparkPlan(df2.queryExecution.executedPlan)
+    val newAccumulatorIds =
+      SparkPlanGraph(newPlan)
+        .allNodes.flatMap(_.metrics.map(_.accumulatorId))
+
+    // Assume that AQE update sparkPlanInfo with newPlan
+    // ExecutionMetrics will be appended using newPlan's SQLMetrics
+    listener.onOtherEvent(SparkListenerSQLAdaptiveExecutionUpdate(
+      executionId,
+      "test",
+      newPlan))
+
+    listener.onJobStart(SparkListenerJobStart(
+      jobId = 1,
+      time = System.currentTimeMillis(),
+      stageInfos = Seq(createStageInfo(1, 0)),
+      createProperties(executionId)))
+
+    listener.onStageSubmitted(SparkListenerStageSubmitted(createStageInfo(1, 0)))
+    listener.onTaskStart(SparkListenerTaskStart(1, 0, createTaskInfo(0, 0)))
+
+    // historical metrics will be kept despite of the newPlan updated.
+    assert(statusStore.executionMetrics(executionId).size == 2)
+
+    // update new metrics with Id 4 & 5, since 3 is timing metrics,
+    // timing metrics has a complicated string presentation so we don't test it here.
+    val newMetricsValueMap = newAccumulatorIds.sorted.tail.map(id => (id, 500L)).toMap
+    listener.onExecutorMetricsUpdate(SparkListenerExecutorMetricsUpdate("", Seq(
+      (0L, 1, 0, createAccumulatorInfos(newMetricsValueMap))
+    )))
+
+    assert(statusStore.executionMetrics(executionId).size == 4)
+    statusStore.executionMetrics(executionId).foreach { m =>
+      assert(m._2 == "100" || m._2 == "500")
+    }
+
+    listener.onTaskEnd(SparkListenerTaskEnd(
+      stageId = 1,
+      stageAttemptId = 0,
+      taskType = "",
+      reason = null,
+      createTaskInfo(0, 0),
+      new ExecutorMetrics,
+      null))
+
+    listener.onStageCompleted(SparkListenerStageCompleted(createStageInfo(1, 0)))
+
+    listener.onJobEnd(SparkListenerJobEnd(
+      jobId = 1,
+      time = System.currentTimeMillis(),
+      JobSucceeded
+    ))
+
+    // aggregateMetrics should contains all metrics from job 0 and job 1
+    val aggregateMetrics = listener.liveExecutionMetrics(executionId)
+    if (aggregateMetrics.isDefined) {
+      assert(aggregateMetrics.get.keySet.size == 4)
+    }
+
+    listener.onOtherEvent(SparkListenerSQLExecutionEnd(
+      executionId, System.currentTimeMillis()))
+  }
+
+
+  test("SPARK-34338: Report metrics from Datasource v2 scan") {
+    val statusStore = spark.sharedState.statusStore
+    val oldCount = statusStore.executionsList().size
+
+    val schema = new StructType().add("i", "int").add("j", "int")
+    val physicalPlan = BatchScanExec(schema.toAttributes, new CustomMetricScanBuilder())
+    val dummyQueryExecution = new QueryExecution(spark, LocalRelation()) {
+      override lazy val sparkPlan = physicalPlan
+      override lazy val executedPlan = physicalPlan
+    }
+
+    SQLExecution.withNewExecutionId(dummyQueryExecution) {
+      physicalPlan.execute().collect()
+    }
+
+    // Wait until the new execution is started and being tracked.
+    while (statusStore.executionsCount() < oldCount) {
+      Thread.sleep(100)
+    }
+
+    // Wait for listener to finish computing the metrics for the execution.
+    while (statusStore.executionsList().isEmpty ||
+      statusStore.executionsList().last.metricValues == null) {
+      Thread.sleep(100)
+    }
+
+    val execId = statusStore.executionsList().last.executionId
+    val metrics = statusStore.executionMetrics(execId)
+    val expectedMetric = physicalPlan.metrics("custom_metric")
+    val expectedValue = "custom_metric: 12345, 12345"
+
+    assert(metrics.contains(expectedMetric.id))
+    assert(metrics(expectedMetric.id) === expectedValue)
+  }
 }
 
 
@@ -744,3 +926,50 @@ class SQLAppStatusListenerMemoryLeakSuite extends SparkFunSuite {
     }
   }
 }
+
+class SimpleCustomMetric extends CustomMetric {
+  override def name(): String = "custom_metric"
+  override def description(): String = "a simple custom metric"
+  override def aggregateTaskMetrics(taskMetrics: Array[Long]): String = {
+    s"custom_metric: ${taskMetrics.mkString(", ")}"
+  }
+}
+
+// The followings are for custom metrics of V2 data source.
+object CustomMetricReaderFactory extends PartitionReaderFactory {
+  override def createReader(partition: InputPartition): PartitionReader[InternalRow] = {
+    val RangeInputPartition(start, end) = partition
+    new PartitionReader[InternalRow] {
+      private var current = start - 1
+
+      override def next(): Boolean = {
+        current += 1
+        current < end
+      }
+
+      override def get(): InternalRow = InternalRow(current, -current)
+
+      override def close(): Unit = {}
+
+      override def currentMetricsValues(): Array[CustomTaskMetric] = {
+        val metric = new CustomTaskMetric {
+          override def name(): String = "custom_metric"
+          override def value(): Long = 12345
+        }
+        Array(metric)
+      }
+    }
+  }
+}
+
+class CustomMetricScanBuilder extends SimpleScanBuilder {
+  override def planInputPartitions(): Array[InputPartition] = {
+    Array(RangeInputPartition(0, 5), RangeInputPartition(5, 10))
+  }
+
+  override def supportedCustomMetrics(): Array[CustomMetric] = {
+    Array(new SimpleCustomMetric)
+  }
+
+  override def createReaderFactory(): PartitionReaderFactory = CustomMetricReaderFactory
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SparkPlanInfoSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SparkPlanInfoSuite.scala
new file mode 100644
index 0000000000000..dfc64a41d9f86
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SparkPlanInfoSuite.scala
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.ui
+
+import org.apache.spark.sql.execution.SparkPlanInfo
+import org.apache.spark.sql.test.SharedSparkSession
+
+class SparkPlanInfoSuite extends SharedSparkSession{
+
+  import testImplicits._
+
+  def validateSparkPlanInfo(sparkPlanInfo: SparkPlanInfo): Unit = {
+    sparkPlanInfo.nodeName match {
+      case "InMemoryTableScan" => assert(sparkPlanInfo.children.length == 1)
+      case _ => sparkPlanInfo.children.foreach(validateSparkPlanInfo)
+    }
+  }
+
+  test("SparkPlanInfo creation from SparkPlan with InMemoryTableScan node") {
+    val dfWithCache = Seq(
+      (1, 1),
+      (2, 2)
+    ).toDF().filter("_1 > 1").cache().repartition(10)
+
+    val planInfoResult = SparkPlanInfo.fromSparkPlan(dfWithCache.queryExecution.executedPlan)
+
+    validateSparkPlanInfo(planInfoResult)
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchBenchmark.scala
index 953b3a67d976f..f9ae611691a7f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchBenchmark.scala
@@ -32,7 +32,9 @@ import org.apache.spark.util.collection.BitSet
  * Benchmark to low level memory access using different ways to manage buffers.
  * To run this benchmark:
  * {{{
- *   1. without sbt: bin/spark-submit --class <this class> <spark sql test jar>
+ *   1. without sbt:
+ *      bin/spark-submit --class <this class>
+ *        --jars <spark core test jar> <spark sql test jar>
  *   2. build/sbt "sql/test:runMain <this class>"
  *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
  *      Results will be written to "benchmarks/ColumnarBatchBenchmark-results.txt".
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchSuite.scala
index 37d028d6a713f..bd69bab6f5da2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchSuite.scala
@@ -32,7 +32,7 @@ import org.apache.spark.memory.MemoryMode
 import org.apache.spark.sql.{RandomDataGenerator, Row}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
-import org.apache.spark.sql.catalyst.util.{ArrayBasedMapBuilder, DateTimeUtils, GenericArrayData}
+import org.apache.spark.sql.catalyst.util.{ArrayBasedMapBuilder, DateTimeUtils, GenericArrayData, MapData}
 import org.apache.spark.sql.execution.RowToColumnConverter
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.util.ArrowUtils
@@ -447,13 +447,6 @@ class ColumnarBatchSuite extends SparkFunSuite {
       Platform.putFloat(buffer, Platform.BYTE_ARRAY_OFFSET, 2.234f)
       Platform.putFloat(buffer, Platform.BYTE_ARRAY_OFFSET + 4, 1.123f)
 
-      if (ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) {
-        // Ensure array contains Little Endian floats
-        val bb = ByteBuffer.wrap(buffer).order(ByteOrder.LITTLE_ENDIAN)
-        Platform.putFloat(buffer, Platform.BYTE_ARRAY_OFFSET, bb.getFloat(0))
-        Platform.putFloat(buffer, Platform.BYTE_ARRAY_OFFSET + 4, bb.getFloat(4))
-      }
-
       column.putFloats(idx, 1, buffer, 4)
       column.putFloats(idx + 1, 1, buffer, 0)
       reference += 1.123f
@@ -491,6 +484,57 @@ class ColumnarBatchSuite extends SparkFunSuite {
       }
   }
 
+  testVector("[SPARK-31703] Float API - Little Endian", 1024, FloatType) {
+    column =>
+      val seed = System.currentTimeMillis()
+      val random = new Random(seed)
+      val reference = mutable.ArrayBuffer.empty[Float]
+
+      var idx = 0
+
+      val littleEndian = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN)
+      littleEndian.putFloat(0, 1.357f)
+      littleEndian.putFloat(4, 2.468f)
+      val arr = new Array[Byte](littleEndian.remaining)
+      littleEndian.get(arr)
+
+      column.putFloatsLittleEndian(idx, 1, arr, 4)
+      column.putFloatsLittleEndian(idx + 1, 1, arr, 0)
+      reference += 2.468f
+      reference += 1.357f
+      idx += 2
+
+      column.putFloatsLittleEndian(idx, 2, arr, 0)
+      reference += 1.357f
+      reference += 2.468f
+      idx += 2
+
+      while (idx < column.capacity) {
+        val single = random.nextBoolean()
+        if (single) {
+          val v = random.nextFloat()
+          column.putFloat(idx, v)
+          reference += v
+          idx += 1
+        } else {
+          val n = math.min(random.nextInt(column.capacity / 20), column.capacity - idx)
+          val v = random.nextFloat()
+          column.putFloats(idx, n, v)
+          var i = 0
+          while (i < n) {
+            reference += v
+            i += 1
+          }
+          idx += n
+        }
+      }
+
+      reference.zipWithIndex.foreach { v =>
+        assert(v._1 == column.getFloat(v._2),
+          "Seed = " + seed + " VectorType=" + column.getClass.getSimpleName)
+      }
+  }
+
   testVector("Double APIs", 1024, DoubleType) {
     column =>
       val seed = System.currentTimeMillis()
@@ -531,13 +575,6 @@ class ColumnarBatchSuite extends SparkFunSuite {
       Platform.putDouble(buffer, Platform.BYTE_ARRAY_OFFSET, 2.234)
       Platform.putDouble(buffer, Platform.BYTE_ARRAY_OFFSET + 8, 1.123)
 
-      if (ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) {
-        // Ensure array contains Little Endian doubles
-        val bb = ByteBuffer.wrap(buffer).order(ByteOrder.LITTLE_ENDIAN)
-        Platform.putDouble(buffer, Platform.BYTE_ARRAY_OFFSET, bb.getDouble(0))
-        Platform.putDouble(buffer, Platform.BYTE_ARRAY_OFFSET + 8, bb.getDouble(8))
-      }
-
       column.putDoubles(idx, 1, buffer, 8)
       column.putDoubles(idx + 1, 1, buffer, 0)
       reference += 1.123
@@ -575,6 +612,57 @@ class ColumnarBatchSuite extends SparkFunSuite {
       }
   }
 
+  testVector("[SPARK-31703] Double API - Little Endian", 1024, DoubleType) {
+    column =>
+      val seed = System.currentTimeMillis()
+      val random = new Random(seed)
+      val reference = mutable.ArrayBuffer.empty[Double]
+
+      var idx = 0
+
+      val littleEndian = ByteBuffer.allocate(16).order(ByteOrder.LITTLE_ENDIAN)
+      littleEndian.putDouble(0, 1.357)
+      littleEndian.putDouble(8, 2.468)
+      val arr = new Array[Byte](littleEndian.remaining)
+      littleEndian.get(arr)
+
+      column.putDoublesLittleEndian(idx, 1, arr, 8)
+      column.putDoublesLittleEndian(idx + 1, 1, arr, 0)
+      reference += 2.468
+      reference += 1.357
+      idx += 2
+
+      column.putDoublesLittleEndian(idx, 2, arr, 0)
+      reference += 1.357
+      reference += 2.468
+      idx += 2
+
+      while (idx < column.capacity) {
+        val single = random.nextBoolean()
+        if (single) {
+          val v = random.nextDouble()
+          column.putDouble(idx, v)
+          reference += v
+          idx += 1
+        } else {
+          val n = math.min(random.nextInt(column.capacity / 20), column.capacity - idx)
+          val v = random.nextDouble()
+          column.putDoubles(idx, n, v)
+          var i = 0
+          while (i < n) {
+            reference += v
+            i += 1
+          }
+          idx += n
+        }
+      }
+
+      reference.zipWithIndex.foreach { v =>
+        assert(v._1 == column.getDouble(v._2),
+          "Seed = " + seed + " VectorType=" + column.getClass.getSimpleName)
+      }
+  }
+
   testVector("String APIs", 7, StringType) {
     column =>
       val reference = mutable.ArrayBuffer.empty[String]
@@ -808,6 +896,16 @@ class ColumnarBatchSuite extends SparkFunSuite {
       assert(a4.asScala == Map())
       assert(a5.asScala == Map(3 -> 6, 4 -> 8, 5 -> 10))
 
+      def toScalaMap(mapData: MapData): Map[Int, Int] = {
+        val keys = mapData.keyArray().toSeq[Int](IntegerType)
+        val values = mapData.valueArray().toSeq[Int](IntegerType)
+        (keys zip values).toMap
+      }
+      assert(toScalaMap(column.getMap(0).copy()) === Map(0 -> 0))
+      assert(toScalaMap(column.getMap(1).copy()) === Map(1 -> 2, 2 -> 4))
+      assert(toScalaMap(column.getMap(3).copy()) === Map())
+      assert(toScalaMap(column.getMap(4).copy()) === Map(3 -> 6, 4 -> 8, 5 -> 10))
+
       column.close()
     }
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala
index e18514c6f93f9..438fd2351ab9f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala
@@ -20,11 +20,12 @@ package org.apache.spark.sql.expressions
 import scala.collection.parallel.immutable.ParVector
 
 import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.catalyst.FunctionIdentifier
-import org.apache.spark.sql.catalyst.expressions.ExpressionInfo
+import org.apache.spark.sql.catalyst.{FunctionIdentifier, InternalRow}
+import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.execution.HiveResult.hiveResultString
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.util.Utils
 
 class ExpressionInfoSuite extends SparkFunSuite with SharedSparkSession {
 
@@ -42,6 +43,10 @@ class ExpressionInfoSuite extends SparkFunSuite with SharedSparkSession {
   test("group info in ExpressionInfo") {
     val info = spark.sessionState.catalog.lookupFunctionInfo(FunctionIdentifier("sum"))
     assert(info.getGroup === "agg_funcs")
+    Seq("agg_funcs", "array_funcs", "binary_funcs", "bitwise_funcs", "collection_funcs",
+      "predicate_funcs", "conditional_funcs", "conversion_funcs", "csv_funcs", "datetime_funcs",
+      "generator_funcs", "hash_funcs", "json_funcs", "lambda_funcs", "map_funcs", "math_funcs",
+      "misc_funcs", "string_funcs", "struct_funcs", "window_funcs", "xml_funcs")
 
     Seq("agg_funcs", "array_funcs", "datetime_funcs", "json_funcs", "map_funcs", "window_funcs")
         .foreach { groupName =>
@@ -104,11 +109,38 @@ class ExpressionInfoSuite extends SparkFunSuite with SharedSparkSession {
     }
   }
 
+  test("SPARK-32870: Default expressions in FunctionRegistry should have their " +
+    "usage, examples, since, and group filled") {
+    val ignoreSet = Set(
+      // Explicitly inherits NonSQLExpression, and has no ExpressionDescription
+      "org.apache.spark.sql.catalyst.expressions.TimeWindow",
+      // Cast aliases do not need examples
+      "org.apache.spark.sql.catalyst.expressions.Cast")
+
+    spark.sessionState.functionRegistry.listFunction().foreach { funcId =>
+      val info = spark.sessionState.catalog.lookupFunctionInfo(funcId)
+      if (!ignoreSet.contains(info.getClassName)) {
+        withClue(s"Function '${info.getName}', Expression class '${info.getClassName}'") {
+          assert(info.getUsage.nonEmpty)
+          assert(info.getExamples.startsWith("\n    Examples:\n"))
+          assert(info.getExamples.endsWith("\n  "))
+          assert(info.getSince.matches("[0-9]+\\.[0-9]+\\.[0-9]+"))
+          assert(info.getGroup.nonEmpty)
+
+          if (info.getArguments.nonEmpty) {
+            assert(info.getArguments.startsWith("\n    Arguments:\n"))
+            assert(info.getArguments.endsWith("\n  "))
+          }
+        }
+      }
+    }
+  }
+
   test("check outputs of expression examples") {
     def unindentAndTrim(s: String): String = {
       s.replaceAll("\n\\s+", "\n").trim
     }
-    val beginSqlStmtRe = "  > ".r
+    val beginSqlStmtRe = "\n      > ".r
     val endSqlStmtRe = ";\n".r
     def checkExampleSyntax(example: String): Unit = {
       val beginStmtNum = beginSqlStmtRe.findAllIn(example).length
@@ -122,14 +154,24 @@ class ExpressionInfoSuite extends SparkFunSuite with SharedSparkSession {
       "org.apache.spark.sql.catalyst.expressions.UnixTimestamp",
       "org.apache.spark.sql.catalyst.expressions.CurrentDate",
       "org.apache.spark.sql.catalyst.expressions.CurrentTimestamp",
+      "org.apache.spark.sql.catalyst.expressions.CurrentTimeZone",
       "org.apache.spark.sql.catalyst.expressions.Now",
       // Random output without a seed
       "org.apache.spark.sql.catalyst.expressions.Rand",
       "org.apache.spark.sql.catalyst.expressions.Randn",
       "org.apache.spark.sql.catalyst.expressions.Shuffle",
       "org.apache.spark.sql.catalyst.expressions.Uuid",
+      // Other nondeterministic expressions
+      "org.apache.spark.sql.catalyst.expressions.MonotonicallyIncreasingID",
+      "org.apache.spark.sql.catalyst.expressions.SparkPartitionID",
+      "org.apache.spark.sql.catalyst.expressions.InputFileName",
+      "org.apache.spark.sql.catalyst.expressions.InputFileBlockStart",
+      "org.apache.spark.sql.catalyst.expressions.InputFileBlockLength",
       // The example calls methods that return unstable results.
-      "org.apache.spark.sql.catalyst.expressions.CallMethodViaReflection")
+      "org.apache.spark.sql.catalyst.expressions.CallMethodViaReflection",
+      "org.apache.spark.sql.catalyst.expressions.SparkVersion",
+      // Throws an error
+      "org.apache.spark.sql.catalyst.expressions.RaiseError")
 
     val parFuncs = new ParVector(spark.sessionState.functionRegistry.listFunction().toVector)
     parFuncs.foreach { funcId =>
@@ -156,4 +198,45 @@ class ExpressionInfoSuite extends SparkFunSuite with SharedSparkSession {
       }
     }
   }
+
+  test("Check whether SQL expressions should extend NullIntolerant") {
+    // Only check expressions extended from these expressions because these expressions are
+    // NullIntolerant by default.
+    val exprTypesToCheck = Seq(classOf[UnaryExpression], classOf[BinaryExpression],
+      classOf[TernaryExpression], classOf[QuaternaryExpression], classOf[SeptenaryExpression])
+
+    // Do not check these expressions, because these expressions override the eval method
+    val ignoreSet = Set(
+      // Extend NullIntolerant and avoid evaluating input1 if input2 is 0
+      classOf[IntegralDivide],
+      classOf[Divide],
+      classOf[Remainder],
+      classOf[Pmod],
+      // Throws an exception, even if input is null
+      classOf[RaiseError]
+    )
+
+    val candidateExprsToCheck = spark.sessionState.functionRegistry.listFunction()
+      .map(spark.sessionState.catalog.lookupFunctionInfo).map(_.getClassName)
+      .filterNot(c => ignoreSet.exists(_.getName.equals(c)))
+      .map(name => Utils.classForName(name))
+      .filterNot(classOf[NonSQLExpression].isAssignableFrom)
+
+    exprTypesToCheck.foreach { superClass =>
+      candidateExprsToCheck.filter(superClass.isAssignableFrom).foreach { clazz =>
+        val isEvalOverrode = clazz.getMethod("eval", classOf[InternalRow]) !=
+          superClass.getMethod("eval", classOf[InternalRow])
+        val isNullIntolerantMixedIn = classOf[NullIntolerant].isAssignableFrom(clazz)
+        if (isEvalOverrode && isNullIntolerantMixedIn) {
+          fail(s"${clazz.getName} should not extend ${classOf[NullIntolerant].getSimpleName}, " +
+            s"or add ${clazz.getName} in the ignoreSet of this test.")
+        } else if (!isEvalOverrode && !isNullIntolerantMixedIn) {
+          fail(s"${clazz.getName} should extend ${classOf[NullIntolerant].getSimpleName}.")
+        } else {
+          assert((!isEvalOverrode && isNullIntolerantMixedIn) ||
+            (isEvalOverrode && !isNullIntolerantMixedIn))
+        }
+      }
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala
index 4630a429830fb..edd96e8337918 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala
@@ -22,8 +22,9 @@ import java.io.File
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalog.{Column, Database, Function, Table}
 import org.apache.spark.sql.catalyst.{FunctionIdentifier, ScalaReflection, TableIdentifier}
+import org.apache.spark.sql.catalyst.analysis.AnalysisTest
 import org.apache.spark.sql.catalyst.catalog._
-import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionInfo}
+import org.apache.spark.sql.catalyst.expressions.Expression
 import org.apache.spark.sql.catalyst.plans.logical.Range
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.StructType
@@ -33,7 +34,7 @@ import org.apache.spark.storage.StorageLevel
 /**
  * Tests for the user-facing [[org.apache.spark.sql.catalog.Catalog]].
  */
-class CatalogSuite extends SharedSparkSession {
+class CatalogSuite extends SharedSparkSession with AnalysisTest {
   import testImplicits._
 
   private def sessionCatalog: SessionCatalog = spark.sessionState.catalog
@@ -58,7 +59,7 @@ class CatalogSuite extends SharedSparkSession {
   }
 
   private def createTempTable(name: String): Unit = {
-    sessionCatalog.createTempView(name, Range(1, 2, 3, 4), overrideIfExists = true)
+    createTempView(sessionCatalog, name, Range(1, 2, 3, 4), overrideIfExists = true)
   }
 
   private def dropTable(name: String, db: Option[String] = None): Unit = {
@@ -470,16 +471,20 @@ class CatalogSuite extends SharedSparkSession {
   }
 
   test("createTable with 'path' in options") {
+    val description = "this is a test table"
+
     withTable("t") {
       withTempDir { dir =>
         spark.catalog.createTable(
           tableName = "t",
           source = "json",
           schema = new StructType().add("i", "int"),
+          description = description,
           options = Map("path" -> dir.getAbsolutePath))
         val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t"))
         assert(table.tableType == CatalogTableType.EXTERNAL)
         assert(table.storage.locationUri.get == makeQualifiedPath(dir.getAbsolutePath))
+        assert(table.comment == Some(description))
 
         Seq((1)).toDF("i").write.insertInto("t")
         assert(dir.exists() && dir.listFiles().nonEmpty)
@@ -530,8 +535,8 @@ class CatalogSuite extends SharedSparkSession {
     dropTable("my_temp_table") // drop table in original session
     assert(spark.catalog.listTables().collect().map(_.name).toSet == Set())
     assert(forkedSession.catalog.listTables().collect().map(_.name).toSet == Set("my_temp_table"))
-    forkedSession.sessionState.catalog
-      .createTempView("fork_table", Range(1, 2, 3, 4), overrideIfExists = true)
+    createTempView(
+      forkedSession.sessionState.catalog, "fork_table", Range(1, 2, 3, 4), overrideIfExists = true)
     assert(spark.catalog.listTables().collect().map(_.name).toSet == Set())
   }
 
@@ -541,4 +546,11 @@ class CatalogSuite extends SharedSparkSession {
     assert(spark.table("my_temp_table").storageLevel == StorageLevel.DISK_ONLY)
   }
 
+  test("SPARK-34301: recover partitions of views is not supported") {
+    createTempTable("my_temp_table")
+    val errMsg = intercept[AnalysisException] {
+      spark.catalog.recoverPartitions("my_temp_table")
+    }.getMessage
+    assert(errMsg.contains("my_temp_table is a temp view. 'recoverPartitions()' expects a table"))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/ExecutorSideSQLConfSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/ExecutorSideSQLConfSuite.scala
index 567524ac75c2e..13b22dba1168b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/ExecutorSideSQLConfSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/ExecutorSideSQLConfSuite.scala
@@ -108,7 +108,7 @@ class ExecutorSideSQLConfSuite extends SparkFunSuite with SQLTestUtils {
           .queryExecution.executedPlan)
         assert(res.length == 2)
         assert(res.forall { case (_, code, _) =>
-          (code.contains("* Codegend pipeline") == flag) &&
+          (code.contains("* Codegened pipeline") == flag) &&
             (code.contains("// input[") == flag)
         })
       }
@@ -175,7 +175,7 @@ class ExecutorSideSQLConfSuite extends SparkFunSuite with SQLTestUtils {
         df.hint("broadcast")
       }
 
-      // set local propert and assert
+      // set local property and assert
       val df2 = generateBroadcastDataFrame(confKey, confValue1)
       spark.sparkContext.setLocalProperty(confKey, confValue1)
       val checks = df1.join(df2).collect()
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
index f1cd37f91d78a..90127557f09d5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
@@ -17,12 +17,13 @@
 
 package org.apache.spark.sql.internal
 
-import scala.language.reflectiveCalls
+import java.util.TimeZone
 
 import org.apache.hadoop.fs.Path
 import org.apache.log4j.Level
 
 import org.apache.spark.sql._
+import org.apache.spark.sql.catalyst.parser.ParseException
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.MIT
 import org.apache.spark.sql.internal.StaticSQLConf._
 import org.apache.spark.sql.test.{SharedSparkSession, TestSQLContext}
@@ -112,7 +113,22 @@ class SQLConfSuite extends QueryTest with SharedSparkSession {
       sql(s"set ${SQLConf.Deprecated.MAPRED_REDUCE_TASKS}=10")
       assert(spark.conf.get(SQLConf.SHUFFLE_PARTITIONS) === 10)
     } finally {
-      sql(s"set ${SQLConf.SHUFFLE_PARTITIONS}=$original")
+      sql(s"set ${SQLConf.SHUFFLE_PARTITIONS.key}=$original")
+    }
+  }
+
+  test(s"SPARK-35168: ${SQLConf.Deprecated.MAPRED_REDUCE_TASKS} should respect" +
+      s" ${SQLConf.SHUFFLE_PARTITIONS.key}") {
+    spark.sessionState.conf.clear()
+    try {
+      sql(s"SET ${SQLConf.ADAPTIVE_EXECUTION_ENABLED.key}=true")
+      sql(s"SET ${SQLConf.COALESCE_PARTITIONS_ENABLED.key}=true")
+      sql(s"SET ${SQLConf.COALESCE_PARTITIONS_INITIAL_PARTITION_NUM.key}=1")
+      sql(s"SET ${SQLConf.SHUFFLE_PARTITIONS.key}=2")
+      checkAnswer(sql(s"SET ${SQLConf.Deprecated.MAPRED_REDUCE_TASKS}"),
+        Row(SQLConf.SHUFFLE_PARTITIONS.key, "2"))
+    } finally {
+      spark.sessionState.conf.clear()
     }
   }
 
@@ -139,11 +155,14 @@ class SQLConfSuite extends QueryTest with SharedSparkSession {
       sql(s"set ${SQLConf.GROUP_BY_ORDINAL.key}=false")
       assert(spark.conf.get(SQLConf.GROUP_BY_ORDINAL) === false)
       assert(sql(s"set").where(s"key = '${SQLConf.GROUP_BY_ORDINAL.key}'").count() == 1)
+      assert(spark.conf.get(SQLConf.OPTIMIZER_EXCLUDED_RULES).isEmpty)
       sql(s"reset")
       assert(spark.conf.get(SQLConf.GROUP_BY_ORDINAL))
       assert(sql(s"set").where(s"key = '${SQLConf.GROUP_BY_ORDINAL.key}'").count() == 0)
+      assert(spark.conf.get(SQLConf.OPTIMIZER_EXCLUDED_RULES) ===
+        Some("org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation"))
     } finally {
-      sql(s"set ${SQLConf.GROUP_BY_ORDINAL}=$original")
+      sql(s"set ${SQLConf.GROUP_BY_ORDINAL.key}=$original")
     }
   }
 
@@ -159,7 +178,7 @@ class SQLConfSuite extends QueryTest with SharedSparkSession {
       assert(spark.conf.get(SQLConf.OPTIMIZER_MAX_ITERATIONS) === 100)
       assert(sql(s"set").where(s"key = '${SQLConf.OPTIMIZER_MAX_ITERATIONS.key}'").count() == 0)
     } finally {
-      sql(s"set ${SQLConf.OPTIMIZER_MAX_ITERATIONS}=$original")
+      sql(s"set ${SQLConf.OPTIMIZER_MAX_ITERATIONS.key}=$original")
     }
   }
 
@@ -178,6 +197,45 @@ class SQLConfSuite extends QueryTest with SharedSparkSession {
     }
   }
 
+  test("SPARK-32406: reset - single configuration") {
+    spark.sessionState.conf.clear()
+    // spark core conf w/o entry registered
+    val appId = spark.sparkContext.getConf.getAppId
+    sql("RESET spark.app.id")
+    assert(spark.conf.get("spark.app.id") === appId, "Should not change spark core ones")
+    // spark core conf w/ entry registered
+    val e1 = intercept[AnalysisException](sql("RESET spark.executor.cores"))
+    assert(e1.getMessage === "Cannot modify the value of a Spark config: spark.executor.cores")
+
+    // user defined settings
+    sql("SET spark.abc=xyz")
+    assert(spark.conf.get("spark.abc") === "xyz")
+    sql("RESET spark.abc")
+    intercept[NoSuchElementException](spark.conf.get("spark.abc"))
+    sql("RESET spark.abc") // ignore nonexistent keys
+
+    // runtime sql configs
+    val original = spark.conf.get(SQLConf.GROUP_BY_ORDINAL)
+    sql(s"SET ${SQLConf.GROUP_BY_ORDINAL.key}=false")
+    sql(s"RESET ${SQLConf.GROUP_BY_ORDINAL.key}")
+    assert(spark.conf.get(SQLConf.GROUP_BY_ORDINAL) === original)
+
+    // runtime sql configs with optional defaults
+    assert(spark.conf.get(SQLConf.OPTIMIZER_EXCLUDED_RULES).isEmpty)
+    sql(s"RESET ${SQLConf.OPTIMIZER_EXCLUDED_RULES.key}")
+    assert(spark.conf.get(SQLConf.OPTIMIZER_EXCLUDED_RULES) ===
+      Some("org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation"))
+    sql(s"SET ${SQLConf.PLAN_CHANGE_LOG_RULES.key}=abc")
+    sql(s"RESET ${SQLConf.PLAN_CHANGE_LOG_RULES.key}")
+    assert(spark.conf.get(SQLConf.PLAN_CHANGE_LOG_RULES).isEmpty)
+
+    // static sql configs
+    val e2 = intercept[AnalysisException](sql(s"RESET ${StaticSQLConf.WAREHOUSE_PATH.key}"))
+    assert(e2.getMessage ===
+      s"Cannot modify the value of a static config: ${StaticSQLConf.WAREHOUSE_PATH.key}")
+
+  }
+
   test("invalid conf value") {
     spark.sessionState.conf.clear()
     val e = intercept[IllegalArgumentException] {
@@ -239,23 +297,23 @@ class SQLConfSuite extends QueryTest with SharedSparkSession {
   }
 
   test("static SQL conf comes from SparkConf") {
-    val previousValue = sparkContext.conf.get(SCHEMA_STRING_LENGTH_THRESHOLD)
+    val previousValue = sparkContext.conf.get(GLOBAL_TEMP_DATABASE)
     try {
-      sparkContext.conf.set(SCHEMA_STRING_LENGTH_THRESHOLD, 2000)
+      sparkContext.conf.set(GLOBAL_TEMP_DATABASE, "a")
       val newSession = new SparkSession(sparkContext)
-      assert(newSession.conf.get(SCHEMA_STRING_LENGTH_THRESHOLD) == 2000)
+      assert(newSession.conf.get(GLOBAL_TEMP_DATABASE) == "a")
       checkAnswer(
-        newSession.sql(s"SET ${SCHEMA_STRING_LENGTH_THRESHOLD.key}"),
-        Row(SCHEMA_STRING_LENGTH_THRESHOLD.key, "2000"))
+        newSession.sql(s"SET ${GLOBAL_TEMP_DATABASE.key}"),
+        Row(GLOBAL_TEMP_DATABASE.key, "a"))
     } finally {
-      sparkContext.conf.set(SCHEMA_STRING_LENGTH_THRESHOLD, previousValue)
+      sparkContext.conf.set(GLOBAL_TEMP_DATABASE, previousValue)
     }
   }
 
   test("cannot set/unset static SQL conf") {
-    val e1 = intercept[AnalysisException](sql(s"SET ${SCHEMA_STRING_LENGTH_THRESHOLD.key}=10"))
+    val e1 = intercept[AnalysisException](sql(s"SET ${GLOBAL_TEMP_DATABASE.key}=10"))
     assert(e1.message.contains("Cannot modify the value of a static config"))
-    val e2 = intercept[AnalysisException](spark.conf.unset(SCHEMA_STRING_LENGTH_THRESHOLD.key))
+    val e2 = intercept[AnalysisException](spark.conf.unset(GLOBAL_TEMP_DATABASE.key))
     assert(e2.message.contains("Cannot modify the value of a static config"))
   }
 
@@ -371,16 +429,50 @@ class SQLConfSuite extends QueryTest with SharedSparkSession {
     spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, "America/Chicago")
     assert(sql(s"set ${SQLConf.SESSION_LOCAL_TIMEZONE.key}").head().getString(1) ===
       "America/Chicago")
+    spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, "GMT+8:00")
+    assert(sql(s"set ${SQLConf.SESSION_LOCAL_TIMEZONE.key}").head().getString(1) === "GMT+8:00")
 
     intercept[IllegalArgumentException] {
       spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, "pst")
     }
-    intercept[IllegalArgumentException] {
-      spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, "GMT+8:00")
-    }
     val e = intercept[IllegalArgumentException] {
       spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, "Asia/shanghai")
     }
     assert(e.getMessage === "Cannot resolve the given timezone with ZoneId.of(_, ZoneId.SHORT_IDS)")
   }
+
+  test("set time zone") {
+    TimeZone.getAvailableIDs().foreach { zid =>
+      sql(s"set time zone '$zid'")
+      assert(spark.conf.get(SQLConf.SESSION_LOCAL_TIMEZONE) === zid)
+    }
+    sql("set time zone local")
+    assert(spark.conf.get(SQLConf.SESSION_LOCAL_TIMEZONE) === TimeZone.getDefault.getID)
+
+    val e1 = intercept[IllegalArgumentException](sql("set time zone 'invalid'"))
+    assert(e1.getMessage === "Cannot resolve the given timezone with" +
+      " ZoneId.of(_, ZoneId.SHORT_IDS)")
+
+    (-18 to 18).map(v => (v, s"interval '$v' hours")).foreach { case (i, interval) =>
+      sql(s"set time zone $interval")
+      val zone = spark.conf.get(SQLConf.SESSION_LOCAL_TIMEZONE)
+      if (i == 0) {
+        assert(zone === "Z")
+      } else {
+        assert(zone === String.format("%+03d:00", new Integer(i)))
+      }
+    }
+    val e2 = intercept[ParseException](sql("set time zone interval 19 hours"))
+    assert(e2.getMessage contains "The interval value must be in the range of [-18, +18] hours")
+  }
+
+  test("SPARK-34454: configs from the legacy namespace should be internal") {
+    val nonInternalLegacyConfigs = spark.sessionState.conf.getAllDefinedConfs
+      .filter { case (key, _, _, _) => key.contains("spark.sql.legacy.") }
+    assert(nonInternalLegacyConfigs.isEmpty,
+      s"""
+         |Non internal legacy SQL configs:
+         |${nonInternalLegacyConfigs.map(_._1).mkString("\n")}
+         |""".stripMargin)
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/VariableSubstitutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/VariableSubstitutionSuite.scala
index d5a946aeaac31..d5da2553c7186 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/VariableSubstitutionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/VariableSubstitutionSuite.scala
@@ -18,12 +18,11 @@
 package org.apache.spark.sql.internal
 
 import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.plans.SQLHelper
 
-class VariableSubstitutionSuite extends SparkFunSuite {
+class VariableSubstitutionSuite extends SparkFunSuite with SQLHelper {
 
-  private lazy val conf = new SQLConf
-  private lazy val sub = new VariableSubstitution(conf)
+  private lazy val sub = new VariableSubstitution()
 
   test("system property") {
     System.setProperty("varSubSuite.var", "abcd")
@@ -35,26 +34,26 @@ class VariableSubstitutionSuite extends SparkFunSuite {
   }
 
   test("Spark configuration variable") {
-    conf.setConfString("some-random-string-abcd", "1234abcd")
-    assert(sub.substitute("${hiveconf:some-random-string-abcd}") == "1234abcd")
-    assert(sub.substitute("${sparkconf:some-random-string-abcd}") == "1234abcd")
-    assert(sub.substitute("${spark:some-random-string-abcd}") == "1234abcd")
-    assert(sub.substitute("${some-random-string-abcd}") == "1234abcd")
+    withSQLConf("some-random-string-abcd" -> "1234abcd") {
+      assert(sub.substitute("${hiveconf:some-random-string-abcd}") == "1234abcd")
+      assert(sub.substitute("${sparkconf:some-random-string-abcd}") == "1234abcd")
+      assert(sub.substitute("${spark:some-random-string-abcd}") == "1234abcd")
+      assert(sub.substitute("${some-random-string-abcd}") == "1234abcd")
+    }
   }
 
   test("multiple substitutes") {
     val q = "select ${bar} ${foo} ${doo} this is great"
-    conf.setConfString("bar", "1")
-    conf.setConfString("foo", "2")
-    conf.setConfString("doo", "3")
-    assert(sub.substitute(q) == "select 1 2 3 this is great")
+    withSQLConf("bar"-> "1", "foo"-> "2", "doo" -> "3") {
+      assert(sub.substitute(q) == "select 1 2 3 this is great")
+    }
   }
 
   test("test nested substitutes") {
     val q = "select ${bar} ${foo} this is great"
-    conf.setConfString("bar", "1")
-    conf.setConfString("foo", "${bar}")
-    assert(sub.substitute(q) == "select 1 1 this is great")
+    withSQLConf("bar"-> "1", "foo"-> "${bar}") {
+      assert(sub.substitute(q) == "select 1 1 this is great")
+    }
   }
 
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCNestedDataSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCNestedDataSourceSuite.scala
new file mode 100644
index 0000000000000..46bdb1918147c
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCNestedDataSourceSuite.scala
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc
+
+import org.apache.spark.sql.NestedDataSourceSuiteBase
+import org.apache.spark.sql.types.StructType
+import org.apache.spark.util.Utils
+
+class JDBCNestedDataSourceSuite extends NestedDataSourceSuiteBase {
+  override val nestedDataSources: Seq[String] = Seq("jdbc")
+  private val tempDir = Utils.createTempDir()
+  private val url = s"jdbc:h2:${tempDir.getCanonicalPath};user=testUser;password=testPass"
+  override val colType: String = "in the customSchema option value"
+
+  override def afterAll(): Unit = {
+    Utils.deleteRecursively(tempDir)
+    super.afterAll()
+  }
+
+  override def readOptions(schema: StructType): Map[String, String] = {
+    Map("url" -> url, "dbtable" -> "t1", "customSchema" -> schema.toDDL)
+  }
+
+  override def save(selectExpr: Seq[String], format: String, path: String): Unit = {
+    // We ignore `selectExpr` because:
+    //  1. H2 doesn't support nested columns
+    //  2. JDBC datasource checks duplicates before comparing of user's schema with
+    //     actual schema of `t1`.
+    spark
+      .range(1L)
+      .write.mode("overwrite")
+      .options(Map("url" -> url, "dbtable" -> "t1"))
+      .format(format)
+      .save()
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
index 7c723f083642d..5865ff245aa39 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
@@ -19,15 +19,21 @@ package org.apache.spark.sql.jdbc
 
 import java.math.BigDecimal
 import java.sql.{Date, DriverManager, SQLException, Timestamp}
-import java.util.{Calendar, GregorianCalendar, Properties}
+import java.time.{Instant, LocalDate}
+import java.util.{Calendar, GregorianCalendar, Properties, TimeZone}
+
+import scala.collection.JavaConverters._
 
 import org.h2.jdbc.JdbcSQLException
+import org.mockito.ArgumentMatchers._
+import org.mockito.Mockito._
 import org.scalatest.{BeforeAndAfter, PrivateMethodTester}
 
 import org.apache.spark.SparkException
 import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row}
-import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.{analysis, TableIdentifier}
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
+import org.apache.spark.sql.catalyst.plans.logical.ShowCreateTable
 import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateTimeTestUtils}
 import org.apache.spark.sql.execution.{DataSourceScanExec, ExtendedMode}
 import org.apache.spark.sql.execution.command.{ExplainCommand, ShowCreateTableCommand}
@@ -71,7 +77,10 @@ class JDBCSuite extends QueryTest
     }
   }
 
-  before {
+  val defaultMetadata = new MetadataBuilder().putLong("scale", 0).build()
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
     Utils.classForName("org.h2.Driver")
     // Extra properties that will be specified for our database. We need these to test
     // usage of parameters from OPTIONS clause in queries.
@@ -273,8 +282,9 @@ class JDBCSuite extends QueryTest
     // Untested: IDENTITY, OTHER, UUID, ARRAY, and GEOMETRY types.
   }
 
-  after {
+  override def afterAll(): Unit = {
     conn.close()
+    super.afterAll()
   }
 
   // Check whether the tables are fetched in the expected degree of parallelism
@@ -423,6 +433,36 @@ class JDBCSuite extends QueryTest
     assert(ids(2) === 3)
   }
 
+  test("SPARK-34843: columnPartition should generate the correct stride size" +
+    " and also realign the first partition for better distribution") {
+    val schema = StructType(Seq(
+      StructField("PartitionColumn", DateType)
+    ))
+
+    val numPartitions = 1000
+    val partitionConfig = Map(
+      "lowerBound" -> "1930-01-01",
+      "upperBound" -> "2020-12-31",
+      "numPartitions" -> numPartitions.toString,
+      "partitionColumn" -> "PartitionColumn"
+    )
+
+    val partitions = JDBCRelation.columnPartition(
+      schema,
+      analysis.caseInsensitiveResolution,
+      TimeZone.getDefault.toZoneId.toString,
+      new JDBCOptions(url, "table", partitionConfig)
+    )
+
+    val firstPredicate = partitions.head.asInstanceOf[JDBCPartition].whereClause
+    val lastPredicate = partitions(numPartitions - 1).asInstanceOf[JDBCPartition].whereClause
+
+    // 152 days (exclusive) to lower bound
+    assert(firstPredicate == """"PartitionColumn" < '1930-06-02' or "PartitionColumn" is null""")
+    // 152 days (inclusive) to upper bound
+    assert(lastPredicate == """"PartitionColumn" >= '2020-08-02'""")
+  }
+
   test("overflow of partition bound difference does not give negative stride") {
     val df = sql("SELECT * FROM partsoverflow")
     checkNumPartitions(df, expectedNumPartitions = 3)
@@ -621,9 +661,21 @@ class JDBCSuite extends QueryTest
     assert(cal.get(Calendar.HOUR) === 11)
     assert(cal.get(Calendar.MINUTE) === 22)
     assert(cal.get(Calendar.SECOND) === 33)
+    assert(cal.get(Calendar.MILLISECOND) === 543)
     assert(rows(0).getAs[java.sql.Timestamp](2).getNanos === 543543000)
   }
 
+  test("SPARK-34357: test TIME types") {
+    val rows = spark.read.jdbc(
+      urlWithUserAndPass, "TEST.TIMETYPES", new Properties()).collect()
+    val cachedRows = spark.read.jdbc(urlWithUserAndPass, "TEST.TIMETYPES", new Properties())
+      .cache().collect()
+    val expectedTimeAtEpoch = java.sql.Timestamp.valueOf("1970-01-01 12:34:56.0")
+    assert(rows(0).getAs[java.sql.Timestamp](0) === expectedTimeAtEpoch)
+    assert(rows(1).getAs[java.sql.Timestamp](0) === expectedTimeAtEpoch)
+    assert(cachedRows(0).getAs[java.sql.Timestamp](0) === expectedTimeAtEpoch)
+  }
+
   test("test DATE types") {
     val rows = spark.read.jdbc(
       urlWithUserAndPass, "TEST.TIMETYPES", new Properties()).collect()
@@ -694,7 +746,7 @@ class JDBCSuite extends QueryTest
   test("Remap types via JdbcDialects") {
     JdbcDialects.registerDialect(testH2Dialect)
     val df = spark.read.jdbc(urlWithUserAndPass, "TEST.PEOPLE", new Properties())
-    assert(df.schema.filter(_.dataType != org.apache.spark.sql.types.StringType).isEmpty)
+    assert(!df.schema.exists(_.dataType != org.apache.spark.sql.types.StringType))
     val rows = df.collect()
     assert(rows(0).get(0).isInstanceOf[String])
     assert(rows(0).get(1).isInstanceOf[String])
@@ -766,9 +818,14 @@ class JDBCSuite extends QueryTest
   }
 
   test("Dialect unregister") {
-    JdbcDialects.registerDialect(testH2Dialect)
-    JdbcDialects.unregisterDialect(testH2Dialect)
-    assert(JdbcDialects.get(urlWithUserAndPass) == NoopDialect)
+    JdbcDialects.unregisterDialect(H2Dialect)
+    try {
+      JdbcDialects.registerDialect(testH2Dialect)
+      JdbcDialects.unregisterDialect(testH2Dialect)
+      assert(JdbcDialects.get(urlWithUserAndPass) == NoopDialect)
+    } finally {
+      JdbcDialects.registerDialect(H2Dialect)
+    }
   }
 
   test("Aggregated dialects") {
@@ -849,6 +906,8 @@ class JDBCSuite extends QueryTest
     assert(Postgres.getCatalystType(java.sql.Types.OTHER, "jsonb", 1, null) === Some(StringType))
     assert(Postgres.getCatalystType(java.sql.Types.ARRAY, "_numeric", 0, md) ==
       Some(ArrayType(DecimalType.SYSTEM_DEFAULT)))
+    assert(Postgres.getCatalystType(java.sql.Types.ARRAY, "_bpchar", 64, md) ==
+      Some(ArrayType(StringType)))
     assert(Postgres.getJDBCType(FloatType).map(_.databaseTypeDefinition).get == "FLOAT4")
     assert(Postgres.getJDBCType(DoubleType).map(_.databaseTypeDefinition).get == "FLOAT8")
     assert(Postgres.getJDBCType(ByteType).map(_.databaseTypeDefinition).get == "SMALLINT")
@@ -994,6 +1053,19 @@ class JDBCSuite extends QueryTest
       === java.sql.Timestamp.valueOf("2002-02-20 11:22:33.543543"))
   }
 
+  test("SPARK-33867: Test DataFrame.where for LocalDate and Instant") {
+    // Test for SPARK-33867
+    val timestamp = Instant.parse("2001-02-20T11:22:33.543543Z")
+    val date = LocalDate.parse("1995-01-01")
+    withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
+      val jdbcDf = spark.read.jdbc(urlWithUserAndPass, "TEST.TIMETYPES", new Properties())
+      val rows = jdbcDf.where($"B" > date && $"C" > timestamp).collect()
+      assert(rows(0).getAs[LocalDate](1) === LocalDate.parse("1996-01-01"))
+      // 8 hour difference since saved time was America/Los_Angeles and Instant is GMT
+      assert(rows(0).getAs[Instant](2) === Instant.parse("2002-02-20T19:22:33.543543Z"))
+    }
+  }
+
   test("test credentials in the properties are not in plan output") {
     val df = sql("SELECT * FROM parts")
     val explain = ExplainCommand(df.queryExecution.logical, ExtendedMode)
@@ -1060,7 +1132,7 @@ class JDBCSuite extends QueryTest
            | password '$password')
          """.stripMargin)
 
-      val show = ShowCreateTableCommand(TableIdentifier(tableName))
+      val show = ShowCreateTableCommand(TableIdentifier(tableName), ShowCreateTable.getoutputAttrs)
       spark.sessionState.executePlan(show).executedPlan.executeCollect().foreach { r =>
         assert(!r.toString.contains(password))
         assert(r.toString.contains(dbTable))
@@ -1180,7 +1252,10 @@ class JDBCSuite extends QueryTest
 
   test("SPARK-16387: Reserved SQL words are not escaped by JDBC writer") {
     val df = spark.createDataset(Seq("a", "b", "c")).toDF("order")
-    val schema = JdbcUtils.schemaString(df, "jdbc:mysql://localhost:3306/temp")
+    val schema = JdbcUtils.schemaString(
+      df.schema,
+      df.sqlContext.conf.caseSensitiveAnalysis,
+      "jdbc:mysql://localhost:3306/temp")
     assert(schema.contains("`order` TEXT"))
   }
 
@@ -1214,8 +1289,8 @@ class JDBCSuite extends QueryTest
   }
 
   test("SPARK-16848: jdbc API throws an exception for user specified schema") {
-    val schema = StructType(Seq(
-      StructField("name", StringType, false), StructField("theid", IntegerType, false)))
+    val schema = StructType(Seq(StructField("name", StringType, false, defaultMetadata),
+      StructField("theid", IntegerType, false, defaultMetadata)))
     val parts = Array[String]("THEID < 2", "THEID >= 2")
     val e1 = intercept[AnalysisException] {
       spark.read.schema(schema).jdbc(urlWithUserAndPass, "TEST.PEOPLE", parts, new Properties())
@@ -1235,7 +1310,9 @@ class JDBCSuite extends QueryTest
     props.put("customSchema", customSchema)
     val df = spark.read.jdbc(urlWithUserAndPass, "TEST.PEOPLE", parts, props)
     assert(df.schema.size === 2)
-    assert(df.schema === CatalystSqlParser.parseTableSchema(customSchema))
+    val expectedSchema = new StructType(CatalystSqlParser.parseTableSchema(customSchema).map(
+      f => StructField(f.name, f.dataType, f.nullable, defaultMetadata)).toArray)
+    assert(df.schema === expectedSchema)
     assert(df.count() === 3)
   }
 
@@ -1251,7 +1328,9 @@ class JDBCSuite extends QueryTest
         """.stripMargin.replaceAll("\n", " "))
       val df = sql("select * from people_view")
       assert(df.schema.length === 2)
-      assert(df.schema === CatalystSqlParser.parseTableSchema(customSchema))
+      val expectedSchema = new StructType(CatalystSqlParser.parseTableSchema(customSchema)
+        .map(f => StructField(f.name, f.dataType, f.nullable, defaultMetadata)).toArray)
+      assert(df.schema === expectedSchema)
       assert(df.count() === 3)
     }
   }
@@ -1302,7 +1381,8 @@ class JDBCSuite extends QueryTest
     testJdbcOptions(new JDBCOptions(parameters))
     testJdbcOptions(new JDBCOptions(CaseInsensitiveMap(parameters)))
     // test add/remove key-value from the case-insensitive map
-    var modifiedParameters = CaseInsensitiveMap(Map.empty) ++ parameters
+    var modifiedParameters =
+      (CaseInsensitiveMap(Map.empty) ++ parameters).asInstanceOf[Map[String, String]]
     testJdbcOptions(new JDBCOptions(modifiedParameters))
     modifiedParameters -= "dbtable"
     assert(modifiedParameters.get("dbTAblE").isEmpty)
@@ -1365,8 +1445,8 @@ class JDBCSuite extends QueryTest
     }
 
   test("jdbc data source shouldn't have unnecessary metadata in its schema") {
-    val schema = StructType(Seq(
-      StructField("NAME", StringType, true), StructField("THEID", IntegerType, true)))
+    val schema = StructType(Seq(StructField("NAME", StringType, true, defaultMetadata),
+      StructField("THEID", IntegerType, true, defaultMetadata)))
 
     val df = spark.read.format("jdbc")
       .option("Url", urlWithUserAndPass)
@@ -1403,7 +1483,7 @@ class JDBCSuite extends QueryTest
   }
 
   test("SPARK-24327 verify and normalize a partition column based on a JDBC resolved schema") {
-    def testJdbcParitionColumn(partColName: String, expectedColumnName: String): Unit = {
+    def testJdbcPartitionColumn(partColName: String, expectedColumnName: String): Unit = {
       val df = spark.read.format("jdbc")
         .option("url", urlWithUserAndPass)
         .option("dbtable", "TEST.PARTITION")
@@ -1424,16 +1504,16 @@ class JDBCSuite extends QueryTest
       }
     }
 
-    testJdbcParitionColumn("THEID", "THEID")
-    testJdbcParitionColumn("\"THEID\"", "THEID")
+    testJdbcPartitionColumn("THEID", "THEID")
+    testJdbcPartitionColumn("\"THEID\"", "THEID")
     withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
-      testJdbcParitionColumn("ThEiD", "THEID")
+      testJdbcPartitionColumn("ThEiD", "THEID")
     }
-    testJdbcParitionColumn("THE ID", "THE ID")
+    testJdbcPartitionColumn("THE ID", "THE ID")
 
     def testIncorrectJdbcPartitionColumn(partColName: String): Unit = {
       val errMsg = intercept[AnalysisException] {
-        testJdbcParitionColumn(partColName, "THEID")
+        testJdbcPartitionColumn(partColName, "THEID")
       }.getMessage
       assert(errMsg.contains(s"User-defined partition column $partColName not found " +
         "in the JDBC relation:"))
@@ -1560,9 +1640,9 @@ class JDBCSuite extends QueryTest
       case LogicalRelation(JDBCRelation(_, parts, _), _, _, _) =>
         val whereClauses = parts.map(_.asInstanceOf[JDBCPartition].whereClause).toSet
         assert(whereClauses === Set(
-          """"D" < '2018-07-10' or "D" is null""",
-          """"D" >= '2018-07-10' AND "D" < '2018-07-14'""",
-          """"D" >= '2018-07-14'"""))
+          """"D" < '2018-07-11' or "D" is null""",
+          """"D" >= '2018-07-11' AND "D" < '2018-07-15'""",
+          """"D" >= '2018-07-15'"""))
     }
     checkAnswer(df1, expectedResult)
 
@@ -1715,4 +1795,47 @@ class JDBCSuite extends QueryTest
     jdbcDF = sqlContext.jdbc(urlWithUserAndPass, "TEST.PEOPLE", parts)
     checkAnswer(jdbcDF, Row("mary", 2) :: Nil)
   }
+
+  test("SPARK-32364: JDBCOption constructor") {
+    val extraOptions = CaseInsensitiveMap[String](Map("UrL" -> "url1", "dBTable" -> "table1"))
+    val connectionProperties = new Properties()
+    connectionProperties.put("url", "url2")
+    connectionProperties.put("dbtable", "table2")
+
+    // connection property should override the options in extraOptions
+    val params = extraOptions ++ connectionProperties.asScala
+    assert(params.size == 2)
+    assert(params.get("uRl").contains("url2"))
+    assert(params.get("DbtaBle").contains("table2"))
+
+    // JDBCOptions constructor parameter should overwrite the existing conf
+    val options = new JDBCOptions(url, "table3", params)
+    assert(options.asProperties.size == 2)
+    assert(options.asProperties.get("url") == url)
+    assert(options.asProperties.get("dbtable") == "table3")
+  }
+
+  test("SPARK-34379: Map JDBC RowID to StringType rather than LongType") {
+    val mockRsmd = mock(classOf[java.sql.ResultSetMetaData])
+    when(mockRsmd.getColumnCount).thenReturn(1)
+    when(mockRsmd.getColumnLabel(anyInt())).thenReturn("rowid")
+    when(mockRsmd.getColumnType(anyInt())).thenReturn(java.sql.Types.ROWID)
+    when(mockRsmd.getColumnTypeName(anyInt())).thenReturn("rowid")
+    when(mockRsmd.getPrecision(anyInt())).thenReturn(0)
+    when(mockRsmd.getScale(anyInt())).thenReturn(0)
+    when(mockRsmd.isSigned(anyInt())).thenReturn(false)
+    when(mockRsmd.isNullable(anyInt())).thenReturn(java.sql.ResultSetMetaData.columnNoNulls)
+
+    val mockRs = mock(classOf[java.sql.ResultSet])
+    when(mockRs.getMetaData).thenReturn(mockRsmd)
+
+    val mockDialect = mock(classOf[JdbcDialect])
+    when(mockDialect.getCatalystType(anyInt(), anyString(), anyInt(), any[MetadataBuilder]))
+      .thenReturn(None)
+
+    val schema = JdbcUtils.getSchema(mockRs, mockDialect)
+    val fields = schema.fields
+    assert(fields.length === 1)
+    assert(fields(0).dataType === StringType)
+   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala
new file mode 100644
index 0000000000000..a3a3f47280952
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc
+
+import java.sql.{Connection, DriverManager}
+import java.util.Properties
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.{QueryTest, Row}
+import org.apache.spark.sql.catalyst.analysis.CannotReplaceMissingTableException
+import org.apache.spark.sql.catalyst.plans.logical.Filter
+import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanRelation
+import org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTableCatalog
+import org.apache.spark.sql.functions.lit
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.util.Utils
+
+class JDBCV2Suite extends QueryTest with SharedSparkSession {
+  import testImplicits._
+
+  val tempDir = Utils.createTempDir()
+  val url = s"jdbc:h2:${tempDir.getCanonicalPath};user=testUser;password=testPass"
+  var conn: java.sql.Connection = null
+
+  override def sparkConf: SparkConf = super.sparkConf
+    .set("spark.sql.catalog.h2", classOf[JDBCTableCatalog].getName)
+    .set("spark.sql.catalog.h2.url", url)
+    .set("spark.sql.catalog.h2.driver", "org.h2.Driver")
+
+  private def withConnection[T](f: Connection => T): T = {
+    val conn = DriverManager.getConnection(url, new Properties())
+    try {
+      f(conn)
+    } finally {
+      conn.close()
+    }
+  }
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    Utils.classForName("org.h2.Driver")
+    withConnection { conn =>
+      conn.prepareStatement("CREATE SCHEMA \"test\"").executeUpdate()
+      conn.prepareStatement(
+        "CREATE TABLE \"test\".\"empty_table\" (name TEXT(32) NOT NULL, id INTEGER NOT NULL)")
+        .executeUpdate()
+      conn.prepareStatement(
+        "CREATE TABLE \"test\".\"people\" (name TEXT(32) NOT NULL, id INTEGER NOT NULL)")
+        .executeUpdate()
+      conn.prepareStatement("INSERT INTO \"test\".\"people\" VALUES ('fred', 1)").executeUpdate()
+      conn.prepareStatement("INSERT INTO \"test\".\"people\" VALUES ('mary', 2)").executeUpdate()
+    }
+  }
+
+  override def afterAll(): Unit = {
+    Utils.deleteRecursively(tempDir)
+    super.afterAll()
+  }
+
+  test("simple scan") {
+    checkAnswer(sql("SELECT * FROM h2.test.empty_table"), Seq())
+    checkAnswer(sql("SELECT * FROM h2.test.people"), Seq(Row("fred", 1), Row("mary", 2)))
+    checkAnswer(sql("SELECT name, id FROM h2.test.people"), Seq(Row("fred", 1), Row("mary", 2)))
+  }
+
+  test("scan with filter push-down") {
+    val df = spark.table("h2.test.people").filter($"id" > 1)
+    val filters = df.queryExecution.optimizedPlan.collect {
+      case f: Filter => f
+    }
+    assert(filters.isEmpty)
+    checkAnswer(df, Row("mary", 2))
+  }
+
+  test("scan with column pruning") {
+    val df = spark.table("h2.test.people").select("id")
+    val scan = df.queryExecution.optimizedPlan.collectFirst {
+      case s: DataSourceV2ScanRelation => s
+    }.get
+    assert(scan.schema.names.sameElements(Seq("ID")))
+    checkAnswer(df, Seq(Row(1), Row(2)))
+  }
+
+  test("scan with filter push-down and column pruning") {
+    val df = spark.table("h2.test.people").filter($"id" > 1).select("name")
+    val filters = df.queryExecution.optimizedPlan.collect {
+      case f: Filter => f
+    }
+    assert(filters.isEmpty)
+    val scan = df.queryExecution.optimizedPlan.collectFirst {
+      case s: DataSourceV2ScanRelation => s
+    }.get
+    assert(scan.schema.names.sameElements(Seq("NAME")))
+    checkAnswer(df, Row("mary"))
+  }
+
+  test("read/write with partition info") {
+    withTable("h2.test.abc") {
+      sql("CREATE TABLE h2.test.abc AS SELECT * FROM h2.test.people")
+      val df1 = Seq(("evan", 3), ("cathy", 4), ("alex", 5)).toDF("NAME", "ID")
+      val e = intercept[IllegalArgumentException] {
+        df1.write
+          .option("partitionColumn", "id")
+          .option("lowerBound", "0")
+          .option("upperBound", "3")
+          .option("numPartitions", "0")
+          .insertInto("h2.test.abc")
+      }.getMessage
+      assert(e.contains("Invalid value `0` for parameter `numPartitions` in table writing " +
+        "via JDBC. The minimum value is 1."))
+
+      df1.write
+        .option("partitionColumn", "id")
+        .option("lowerBound", "0")
+        .option("upperBound", "3")
+        .option("numPartitions", "3")
+        .insertInto("h2.test.abc")
+
+      val df2 = spark.read
+        .option("partitionColumn", "id")
+        .option("lowerBound", "0")
+        .option("upperBound", "3")
+        .option("numPartitions", "2")
+        .table("h2.test.abc")
+
+      assert(df2.rdd.getNumPartitions === 2)
+      assert(df2.count() === 5)
+    }
+  }
+
+  test("show tables") {
+    checkAnswer(sql("SHOW TABLES IN h2.test"),
+      Seq(Row("test", "people", false), Row("test", "empty_table", false)))
+  }
+
+  test("SQL API: create table as select") {
+    withTable("h2.test.abc") {
+      sql("CREATE TABLE h2.test.abc AS SELECT * FROM h2.test.people")
+      checkAnswer(sql("SELECT name, id FROM h2.test.abc"), Seq(Row("fred", 1), Row("mary", 2)))
+    }
+  }
+
+  test("DataFrameWriterV2: create table as select") {
+    withTable("h2.test.abc") {
+      spark.table("h2.test.people").writeTo("h2.test.abc").create()
+      checkAnswer(sql("SELECT name, id FROM h2.test.abc"), Seq(Row("fred", 1), Row("mary", 2)))
+    }
+  }
+
+  test("SQL API: replace table as select") {
+    withTable("h2.test.abc") {
+      intercept[CannotReplaceMissingTableException] {
+        sql("REPLACE TABLE h2.test.abc AS SELECT 1 as col")
+      }
+      sql("CREATE OR REPLACE TABLE h2.test.abc AS SELECT 1 as col")
+      checkAnswer(sql("SELECT col FROM h2.test.abc"), Row(1))
+      sql("REPLACE TABLE h2.test.abc AS SELECT * FROM h2.test.people")
+      checkAnswer(sql("SELECT name, id FROM h2.test.abc"), Seq(Row("fred", 1), Row("mary", 2)))
+    }
+  }
+
+  test("DataFrameWriterV2: replace table as select") {
+    withTable("h2.test.abc") {
+      intercept[CannotReplaceMissingTableException] {
+        sql("SELECT 1 AS col").writeTo("h2.test.abc").replace()
+      }
+      sql("SELECT 1 AS col").writeTo("h2.test.abc").createOrReplace()
+      checkAnswer(sql("SELECT col FROM h2.test.abc"), Row(1))
+      spark.table("h2.test.people").writeTo("h2.test.abc").replace()
+      checkAnswer(sql("SELECT name, id FROM h2.test.abc"), Seq(Row("fred", 1), Row("mary", 2)))
+    }
+  }
+
+  test("SQL API: insert and overwrite") {
+    withTable("h2.test.abc") {
+      sql("CREATE TABLE h2.test.abc AS SELECT * FROM h2.test.people")
+
+      sql("INSERT INTO h2.test.abc SELECT 'lucy', 3")
+      checkAnswer(
+        sql("SELECT name, id FROM h2.test.abc"),
+        Seq(Row("fred", 1), Row("mary", 2), Row("lucy", 3)))
+
+      sql("INSERT OVERWRITE h2.test.abc SELECT 'bob', 4")
+      checkAnswer(sql("SELECT name, id FROM h2.test.abc"), Row("bob", 4))
+    }
+  }
+
+  test("DataFrameWriterV2: insert and overwrite") {
+    withTable("h2.test.abc") {
+      sql("CREATE TABLE h2.test.abc AS SELECT * FROM h2.test.people")
+
+      // `DataFrameWriterV2` is by-name.
+      sql("SELECT 3 AS ID, 'lucy' AS NAME").writeTo("h2.test.abc").append()
+      checkAnswer(
+        sql("SELECT name, id FROM h2.test.abc"),
+        Seq(Row("fred", 1), Row("mary", 2), Row("lucy", 3)))
+
+      sql("SELECT 'bob' AS NAME, 4 AS ID").writeTo("h2.test.abc").overwrite(lit(true))
+      checkAnswer(sql("SELECT name, id FROM h2.test.abc"), Row("bob", 4))
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCWriteSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCWriteSuite.scala
index 8021ef1a17a18..efa2773bfd692 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCWriteSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCWriteSuite.scala
@@ -17,7 +17,8 @@
 
 package org.apache.spark.sql.jdbc
 
-import java.sql.DriverManager
+import java.sql.{Date, DriverManager, Timestamp}
+import java.time.{Instant, LocalDate}
 import java.util.Properties
 
 import scala.collection.JavaConverters.propertiesAsScalaMapConverter
@@ -81,6 +82,9 @@ class JDBCWriteSuite extends SharedSparkSession with BeforeAndAfter {
         |USING org.apache.spark.sql.jdbc
         |OPTIONS (url '$url1', dbtable 'TEST.PEOPLE1', user 'testUser', password 'testPass')
       """.stripMargin.replaceAll("\n", " "))
+
+    conn1.prepareStatement("create table test.timetypes (d DATE, t TIMESTAMP)").executeUpdate()
+    conn.commit()
   }
 
   after {
@@ -194,24 +198,29 @@ class JDBCWriteSuite extends SharedSparkSession with BeforeAndAfter {
   }
 
   test("Truncate") {
-    JdbcDialects.registerDialect(testH2Dialect)
-    val df = spark.createDataFrame(sparkContext.parallelize(arr2x2), schema2)
-    val df2 = spark.createDataFrame(sparkContext.parallelize(arr1x2), schema2)
-    val df3 = spark.createDataFrame(sparkContext.parallelize(arr2x3), schema3)
-
-    df.write.jdbc(url1, "TEST.TRUNCATETEST", properties)
-    df2.write.mode(SaveMode.Overwrite).option("truncate", true)
-      .jdbc(url1, "TEST.TRUNCATETEST", properties)
-    assert(1 === spark.read.jdbc(url1, "TEST.TRUNCATETEST", properties).count())
-    assert(2 === spark.read.jdbc(url1, "TEST.TRUNCATETEST", properties).collect()(0).length)
+    JdbcDialects.unregisterDialect(H2Dialect)
+    try {
+      JdbcDialects.registerDialect(testH2Dialect)
+      val df = spark.createDataFrame(sparkContext.parallelize(arr2x2), schema2)
+      val df2 = spark.createDataFrame(sparkContext.parallelize(arr1x2), schema2)
+      val df3 = spark.createDataFrame(sparkContext.parallelize(arr2x3), schema3)
 
-    val m = intercept[AnalysisException] {
-      df3.write.mode(SaveMode.Overwrite).option("truncate", true)
+      df.write.jdbc(url1, "TEST.TRUNCATETEST", properties)
+      df2.write.mode(SaveMode.Overwrite).option("truncate", true)
         .jdbc(url1, "TEST.TRUNCATETEST", properties)
-    }.getMessage
-    assert(m.contains("Column \"seq\" not found"))
-    assert(0 === spark.read.jdbc(url1, "TEST.TRUNCATETEST", properties).count())
-    JdbcDialects.unregisterDialect(testH2Dialect)
+      assert(1 === spark.read.jdbc(url1, "TEST.TRUNCATETEST", properties).count())
+      assert(2 === spark.read.jdbc(url1, "TEST.TRUNCATETEST", properties).collect()(0).length)
+
+      val m = intercept[AnalysisException] {
+        df3.write.mode(SaveMode.Overwrite).option("truncate", true)
+          .jdbc(url1, "TEST.TRUNCATETEST", properties)
+      }.getMessage
+      assert(m.contains("Column \"seq\" not found"))
+      assert(0 === spark.read.jdbc(url1, "TEST.TRUNCATETEST", properties).count())
+    } finally {
+      JdbcDialects.unregisterDialect(testH2Dialect)
+      JdbcDialects.registerDialect(H2Dialect)
+    }
   }
 
   test("createTableOptions") {
@@ -385,12 +394,15 @@ class JDBCWriteSuite extends SharedSparkSession with BeforeAndAfter {
         .foldLeft(new StructType())((schema, colType) => schema.add(colType._1, colType._2))
       val createTableColTypes =
         colTypes.map { case (col, dataType) => s"$col $dataType" }.mkString(", ")
-      val df = spark.createDataFrame(sparkContext.parallelize(Seq(Row.empty)), schema)
 
       val expectedSchemaStr =
         colTypes.map { case (col, dataType) => s""""$col" $dataType """ }.mkString(", ")
 
-      assert(JdbcUtils.schemaString(df, url1, Option(createTableColTypes)) == expectedSchemaStr)
+      assert(JdbcUtils.schemaString(
+        schema,
+        spark.sqlContext.conf.caseSensitiveAnalysis,
+        url1,
+        Option(createTableColTypes)) == expectedSchemaStr)
     }
 
     testCreateTableColDataTypes(Seq("boolean"))
@@ -598,4 +610,55 @@ class JDBCWriteSuite extends SharedSparkSession with BeforeAndAfter {
     sparkContext.removeSparkListener(listener)
     taskMetrics.sum
   }
+
+  test("SPARK-34144: write and read java.time LocalDate and Instant") {
+    withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
+      val schema = new StructType().add("d", DateType).add("t", TimestampType);
+      val values = Seq(Row.apply(LocalDate.parse("2020-01-01"),
+        Instant.parse("2020-02-02T12:13:14.56789Z")))
+      val df = spark.createDataFrame(sparkContext.makeRDD(values), schema)
+
+      df.write.jdbc(url, "TEST.TIMETYPES", new Properties())
+
+      val rows = spark.read.jdbc(url, "TEST.TIMETYPES", new Properties()).collect()
+      assert(1 === rows.length);
+      assert(rows(0).getAs[LocalDate](0) === LocalDate.parse("2020-01-01"))
+      assert(rows(0).getAs[Instant](1) === Instant.parse("2020-02-02T12:13:14.56789Z"))
+    }
+  }
+
+  test("SPARK-34144: write Date and Timestampt, read LocalDate and Instant") {
+    val schema = new StructType().add("d", DateType).add("t", TimestampType);
+    val values = Seq(Row.apply(Date.valueOf("2020-01-01"),
+      Timestamp.valueOf("2020-02-02 12:13:14.56789")))
+    val df = spark.createDataFrame(sparkContext.makeRDD(values), schema)
+
+    df.write.jdbc(url, "TEST.TIMETYPES", new Properties())
+
+    withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
+      val rows = spark.read.jdbc(url, "TEST.TIMETYPES", new Properties()).collect()
+      assert(1 === rows.length);
+      assert(rows(0).getAs[LocalDate](0) === LocalDate.parse("2020-01-01"))
+      // 8 hour difference since Timestamp was America/Los_Angeles and Instant is GMT
+      assert(rows(0).getAs[Instant](1) === Instant.parse("2020-02-02T20:13:14.56789Z"))
+    }
+  }
+
+  test("SPARK-34144: write LocalDate and Instant, read Date and Timestampt") {
+    withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
+      val schema = new StructType().add("d", DateType).add("t", TimestampType);
+      val values = Seq(Row.apply(LocalDate.parse("2020-01-01"),
+        Instant.parse("2020-02-02T12:13:14.56789Z")))
+      val df = spark.createDataFrame(sparkContext.makeRDD(values), schema)
+
+      df.write.jdbc(url, "TEST.TIMETYPES", new Properties())
+    }
+
+    val rows = spark.read.jdbc(url, "TEST.TIMETYPES", new Properties()).collect()
+    assert(1 === rows.length);
+    assert(rows(0).getAs[java.sql.Date](0) === java.sql.Date.valueOf("2020-01-01"))
+    // -8 hour difference since Instant was GMT and Timestamp is America/Los_Angeles
+    assert(rows(0).getAs[java.sql.Timestamp](1)
+      === java.sql.Timestamp.valueOf("2020-02-02 04:13:14.56789"))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
index e153c7168dbf2..d90c8732ea287 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
@@ -27,9 +27,8 @@ import org.apache.spark.sql.catalyst.catalog.BucketSpec
 import org.apache.spark.sql.catalyst.expressions
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning
-import org.apache.spark.sql.execution.{DataSourceScanExec, FileSourceScanExec, SortExec, SparkPlan}
-import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec
-import org.apache.spark.sql.execution.adaptive.AdaptiveTestUtils.assertExceptionMessage
+import org.apache.spark.sql.execution.{FileSourceScanExec, SortExec, SparkPlan}
+import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AdaptiveSparkPlanHelper, DisableAdaptiveExecution}
 import org.apache.spark.sql.execution.datasources.BucketingUtils
 import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
 import org.apache.spark.sql.execution.joins.SortMergeJoinExec
@@ -40,7 +39,8 @@ import org.apache.spark.sql.test.{SharedSparkSession, SQLTestUtils}
 import org.apache.spark.util.Utils
 import org.apache.spark.util.collection.BitSet
 
-class BucketedReadWithoutHiveSupportSuite extends BucketedReadSuite with SharedSparkSession {
+class BucketedReadWithoutHiveSupportSuite
+  extends BucketedReadSuite with SharedSparkSession {
   protected override def beforeAll(): Unit = {
     super.beforeAll()
     assert(spark.sparkContext.conf.get(CATALOG_IMPLEMENTATION) == "in-memory")
@@ -48,7 +48,7 @@ class BucketedReadWithoutHiveSupportSuite extends BucketedReadSuite with SharedS
 }
 
 
-abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
+abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with AdaptiveSparkPlanHelper {
   import testImplicits._
 
   protected override def beforeAll(): Unit = {
@@ -82,27 +82,29 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
         .bucketBy(8, "j", "k")
         .saveAsTable("bucketed_table")
 
-      val bucketValue = Random.nextInt(maxI)
-      val table = spark.table("bucketed_table").filter($"i" === bucketValue)
-      val query = table.queryExecution
-      val output = query.analyzed.output
-      val rdd = query.toRdd
-
-      assert(rdd.partitions.length == 8)
-
-      val attrs = table.select("j", "k").queryExecution.analyzed.output
-      val checkBucketId = rdd.mapPartitionsWithIndex((index, rows) => {
-        val getBucketId = UnsafeProjection.create(
-          HashPartitioning(attrs, 8).partitionIdExpression :: Nil,
-          output)
-        rows.map(row => getBucketId(row).getInt(0) -> index)
-      })
-      checkBucketId.collect().foreach(r => assert(r._1 == r._2))
+      withSQLConf(SQLConf.AUTO_BUCKETED_SCAN_ENABLED.key -> "false") {
+        val bucketValue = Random.nextInt(maxI)
+        val table = spark.table("bucketed_table").filter($"i" === bucketValue)
+        val query = table.queryExecution
+        val output = query.analyzed.output
+        val rdd = query.toRdd
+
+        assert(rdd.partitions.length == 8)
+
+        val attrs = table.select("j", "k").queryExecution.analyzed.output
+        val checkBucketId = rdd.mapPartitionsWithIndex((index, rows) => {
+          val getBucketId = UnsafeProjection.create(
+            HashPartitioning(attrs, 8).partitionIdExpression :: Nil,
+            output)
+          rows.map(row => getBucketId(row).getInt(0) -> index)
+        })
+        checkBucketId.collect().foreach(r => assert(r._1 == r._2))
+      }
     }
   }
 
   private def getFileScan(plan: SparkPlan): FileSourceScanExec = {
-    val fileScan = plan.collect { case f: FileSourceScanExec => f }
+    val fileScan = collect(plan) { case f: FileSourceScanExec => f }
     assert(fileScan.nonEmpty, plan)
     fileScan.head
   }
@@ -112,12 +114,15 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
   //   2) Verify the final result is the same as the expected one
   private def checkPrunedAnswers(
       bucketSpec: BucketSpec,
-      bucketValues: Seq[Integer],
+      bucketValues: Seq[Any],
       filterCondition: Column,
       originalDataFrame: DataFrame): Unit = {
-    // This test verifies parts of the plan. Disable whole stage codegen.
-    withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false") {
-      val bucketedDataFrame = spark.table("bucketed_table").select("i", "j", "k")
+    // This test verifies parts of the plan. Disable whole stage codegen,
+    // automatically bucketed scan, and filter push down for json data source.
+    withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false",
+      SQLConf.AUTO_BUCKETED_SCAN_ENABLED.key -> "false",
+      SQLConf.JSON_FILTER_PUSHDOWN_ENABLED.key -> "false") {
+      val bucketedDataFrame = spark.table("bucketed_table")
       val BucketSpec(numBuckets, bucketColumnNames, _) = bucketSpec
       // Limit: bucket pruning only works when the bucket column has one and only one column
       assert(bucketColumnNames.length == 1)
@@ -146,11 +151,41 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
         if (invalidBuckets.nonEmpty) {
           fail(s"Buckets ${invalidBuckets.mkString(",")} should have been pruned from:\n$plan")
         }
+
+        withSQLConf(SQLConf.AUTO_BUCKETED_SCAN_ENABLED.key -> "true") {
+          // Bucket pruning should still work without bucketed scan
+          val planWithoutBucketedScan = bucketedDataFrame.filter(filterCondition)
+            .queryExecution.executedPlan
+          val fileScan = getFileScan(planWithoutBucketedScan)
+          assert(!fileScan.bucketedScan, s"except no bucketed scan but found\n$fileScan")
+
+          val bucketColumnType = bucketedDataFrame.schema.apply(bucketColumnIndex).dataType
+          val rowsWithInvalidBuckets = fileScan.execute().filter(row => {
+            // Return rows should have been pruned
+            val bucketColumnValue = row.get(bucketColumnIndex, bucketColumnType)
+            val bucketId = BucketingUtils.getBucketIdFromValue(
+              bucketColumn, numBuckets, bucketColumnValue)
+            !matchedBuckets.get(bucketId)
+          }).collect()
+
+          if (rowsWithInvalidBuckets.nonEmpty) {
+            fail(s"Rows ${rowsWithInvalidBuckets.mkString(",")} should have been pruned from:\n" +
+              s"$planWithoutBucketedScan")
+          }
+        }
       }
 
+      val expectedDataFrame = originalDataFrame.filter(filterCondition).orderBy("i", "j", "k")
+        .select("i", "j", "k")
       checkAnswer(
-        bucketedDataFrame.filter(filterCondition).orderBy("i", "j", "k"),
-        originalDataFrame.filter(filterCondition).orderBy("i", "j", "k"))
+        bucketedDataFrame.filter(filterCondition).orderBy("i", "j", "k").select("i", "j", "k"),
+        expectedDataFrame)
+
+      withSQLConf(SQLConf.AUTO_BUCKETED_SCAN_ENABLED.key -> "true") {
+        checkAnswer(
+          bucketedDataFrame.filter(filterCondition).orderBy("i", "j", "k").select("i", "j", "k"),
+          expectedDataFrame)
+      }
     }
   }
 
@@ -158,7 +193,6 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
     withTable("bucketed_table") {
       val numBuckets = NumBucketsForPruningDF
       val bucketSpec = BucketSpec(numBuckets, Seq("j"), Nil)
-      // json does not support predicate push-down, and thus json is used here
       df.write
         .format("json")
         .partitionBy("i")
@@ -189,7 +223,7 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
 
       // Case 4: InSet
       val inSetExpr = expressions.InSet($"j".expr,
-        Set(bucketValue, bucketValue + 1, bucketValue + 2, bucketValue + 3).map(lit(_).expr))
+        Set(bucketValue, bucketValue + 1, bucketValue + 2, bucketValue + 3))
       checkPrunedAnswers(
         bucketSpec,
         bucketValues = Seq(bucketValue, bucketValue + 1, bucketValue + 2, bucketValue + 3),
@@ -244,6 +278,25 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
     }
   }
 
+  test("bucket pruning support IsNaN") {
+    withTable("bucketed_table") {
+      val numBuckets = NumBucketsForPruningNullDf
+      val bucketSpec = BucketSpec(numBuckets, Seq("j"), Nil)
+      val naNDF = nullDF.selectExpr("i", "cast(if(isnull(j), 'NaN', j) as double) as j", "k")
+      // json does not support predicate push-down, and thus json is used here
+      naNDF.write
+        .format("json")
+        .bucketBy(numBuckets, "j")
+        .saveAsTable("bucketed_table")
+
+      checkPrunedAnswers(
+        bucketSpec,
+        bucketValues = Double.NaN :: Nil,
+        filterCondition = $"j".isNaN,
+        naNDF)
+    }
+  }
+
   test("read partitioning bucketed tables having composite filters") {
     withTable("bucketed_table") {
       val numBuckets = NumBucketsForPruningDF
@@ -333,7 +386,8 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
       bucketSpec: Option[BucketSpec],
       numPartitions: Int = 10,
       expectedShuffle: Boolean = true,
-      expectedSort: Boolean = true)
+      expectedSort: Boolean = true,
+      expectedNumOutputPartitions: Option[Int] = None)
 
   /**
    * A helper method to test the bucket read functionality using join.  It will save `df1` and `df2`
@@ -346,10 +400,18 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
       bucketedTableTestSpecRight: BucketedTableTestSpec,
       joinType: String = "inner",
       joinCondition: (DataFrame, DataFrame) => Column): Unit = {
-    val BucketedTableTestSpec(bucketSpecLeft, numPartitionsLeft, shuffleLeft, sortLeft) =
-      bucketedTableTestSpecLeft
-    val BucketedTableTestSpec(bucketSpecRight, numPartitionsRight, shuffleRight, sortRight) =
-      bucketedTableTestSpecRight
+    val BucketedTableTestSpec(
+      bucketSpecLeft,
+      numPartitionsLeft,
+      shuffleLeft,
+      sortLeft,
+      numOutputPartitionsLeft) = bucketedTableTestSpecLeft
+    val BucketedTableTestSpec(
+      bucketSpecRight,
+      numPartitionsRight,
+      shuffleRight,
+      sortRight,
+      numOutputPartitionsRight) = bucketedTableTestSpecRight
 
     withTable("bucketed_table1", "bucketed_table2") {
       def withBucket(
@@ -414,6 +476,16 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
         assert(
           joinOperator.right.find(_.isInstanceOf[SortExec]).isDefined == sortRight,
           s"expected sort in the right child to be $sortRight but found\n${joinOperator.right}")
+
+        // check the output partitioning
+        if (numOutputPartitionsLeft.isDefined) {
+          assert(joinOperator.left.outputPartitioning.numPartitions ===
+            numOutputPartitionsLeft.get)
+        }
+        if (numOutputPartitionsRight.isDefined) {
+          assert(joinOperator.right.outputPartitioning.numPartitions ===
+            numOutputPartitionsRight.get)
+        }
       }
     }
   }
@@ -599,13 +671,14 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
     withTable("bucketed_table") {
       df1.write.format("parquet").bucketBy(8, "i", "j").saveAsTable("bucketed_table")
       val tbl = spark.table("bucketed_table")
-      val agged = tbl.groupBy("i", "j").agg(max("k"))
+      val aggregated = tbl.groupBy("i", "j").agg(max("k"))
 
       checkAnswer(
-        agged.sort("i", "j"),
+        aggregated.sort("i", "j"),
         df1.groupBy("i", "j").agg(max("k")).sort("i", "j"))
 
-      assert(agged.queryExecution.executedPlan.find(_.isInstanceOf[ShuffleExchangeExec]).isEmpty)
+      assert(
+        aggregated.queryExecution.executedPlan.find(_.isInstanceOf[ShuffleExchangeExec]).isEmpty)
     }
   }
 
@@ -639,13 +712,14 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
     withTable("bucketed_table") {
       df1.write.format("parquet").bucketBy(8, "i").saveAsTable("bucketed_table")
       val tbl = spark.table("bucketed_table")
-      val agged = tbl.groupBy("i", "j").agg(max("k"))
+      val aggregated = tbl.groupBy("i", "j").agg(max("k"))
 
       checkAnswer(
-        agged.sort("i", "j"),
+        aggregated.sort("i", "j"),
         df1.groupBy("i", "j").agg(max("k")).sort("i", "j"))
 
-      assert(agged.queryExecution.executedPlan.find(_.isInstanceOf[ShuffleExchangeExec]).isEmpty)
+      assert(
+        aggregated.queryExecution.executedPlan.find(_.isInstanceOf[ShuffleExchangeExec]).isEmpty)
     }
   }
 
@@ -766,12 +840,12 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
       Utils.deleteRecursively(tableDir)
       df1.write.parquet(tableDir.getAbsolutePath)
 
-      val agged = spark.table("bucketed_table").groupBy("i").count()
+      val aggregated = spark.table("bucketed_table").groupBy("i").count()
       val error = intercept[Exception] {
-        agged.count()
+        aggregated.count()
       }
 
-      assertExceptionMessage(error, "Invalid bucket file")
+      assert(error.toString contains "Invalid bucket file")
     }
   }
 
@@ -856,4 +930,145 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
       }
     }
   }
+
+  test("SPARK-32767 Bucket join should work if SHUFFLE_PARTITIONS larger than bucket number") {
+    withSQLConf(
+      SQLConf.SHUFFLE_PARTITIONS.key -> "9",
+      SQLConf.COALESCE_PARTITIONS_INITIAL_PARTITION_NUM.key -> "10")  {
+
+      val testSpec1 = BucketedTableTestSpec(
+        Some(BucketSpec(8, Seq("i", "j"), Seq("i", "j"))),
+        numPartitions = 1,
+        expectedShuffle = false,
+        expectedSort = false,
+        expectedNumOutputPartitions = Some(8))
+      val testSpec2 = BucketedTableTestSpec(
+        Some(BucketSpec(6, Seq("i", "j"), Seq("i", "j"))),
+        numPartitions = 1,
+        expectedShuffle = true,
+        expectedSort = true,
+        expectedNumOutputPartitions = Some(8))
+      Seq(false, true).foreach { enableAdaptive =>
+        withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> s"$enableAdaptive") {
+          Seq((testSpec1, testSpec2), (testSpec2, testSpec1)).foreach { specs =>
+            testBucketing(
+              bucketedTableTestSpecLeft = specs._1,
+              bucketedTableTestSpecRight = specs._2,
+              joinCondition = joinCondition(Seq("i", "j")))
+          }
+        }
+      }
+    }
+  }
+
+  test("bucket coalescing eliminates shuffle") {
+    withSQLConf(
+      SQLConf.COALESCE_BUCKETS_IN_JOIN_ENABLED.key -> "true",
+      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
+      // The side with bucketedTableTestSpec1 will be coalesced to have 4 output partitions.
+      // Currently, sort will be introduced for the side that is coalesced.
+      val testSpec1 = BucketedTableTestSpec(
+        Some(BucketSpec(8, Seq("i", "j"), Seq("i", "j"))),
+        numPartitions = 1,
+        expectedShuffle = false,
+        expectedSort = true,
+        expectedNumOutputPartitions = Some(4))
+      val testSpec2 = BucketedTableTestSpec(
+        Some(BucketSpec(4, Seq("i", "j"), Seq("i", "j"))),
+        numPartitions = 1,
+        expectedShuffle = false,
+        expectedSort = false,
+        expectedNumOutputPartitions = Some(4))
+
+      Seq((testSpec1, testSpec2), (testSpec2, testSpec1)).foreach { specs =>
+        testBucketing(
+          bucketedTableTestSpecLeft = specs._1,
+          bucketedTableTestSpecRight = specs._2,
+          joinCondition = joinCondition(Seq("i", "j")))
+      }
+    }
+  }
+
+  test("bucket coalescing is not satisfied") {
+    def run(testSpec1: BucketedTableTestSpec, testSpec2: BucketedTableTestSpec): Unit = {
+      Seq((testSpec1, testSpec2), (testSpec2, testSpec1)).foreach { specs =>
+        testBucketing(
+          bucketedTableTestSpecLeft = specs._1,
+          bucketedTableTestSpecRight = specs._2,
+          joinCondition = joinCondition(Seq("i", "j")))
+      }
+    }
+
+    withSQLConf(SQLConf.COALESCE_BUCKETS_IN_JOIN_ENABLED.key -> "false") {
+      // Coalescing buckets is disabled by a config.
+      run(
+        BucketedTableTestSpec(
+          Some(BucketSpec(8, Seq("i", "j"), Seq("i", "j"))), expectedShuffle = false),
+        BucketedTableTestSpec(
+          Some(BucketSpec(4, Seq("i", "j"), Seq("i", "j"))), expectedShuffle = true))
+    }
+
+    withSQLConf(
+      SQLConf.COALESCE_BUCKETS_IN_JOIN_ENABLED.key -> "true",
+      SQLConf.COALESCE_BUCKETS_IN_JOIN_MAX_BUCKET_RATIO.key -> "2") {
+      // Coalescing buckets is not applied because the ratio of the number of buckets (3)
+      // is greater than max allowed (2).
+      run(
+        BucketedTableTestSpec(
+          Some(BucketSpec(12, Seq("i", "j"), Seq("i", "j"))), expectedShuffle = false),
+        BucketedTableTestSpec(
+          Some(BucketSpec(4, Seq("i", "j"), Seq("i", "j"))), expectedShuffle = true))
+    }
+
+    withSQLConf(SQLConf.COALESCE_BUCKETS_IN_JOIN_ENABLED.key -> "true") {
+      run(
+        // Coalescing buckets is not applied because the bigger number of buckets (8) is not
+        // divisible by the smaller number of buckets (7).
+        BucketedTableTestSpec(
+          Some(BucketSpec(8, Seq("i", "j"), Seq("i", "j"))), expectedShuffle = false),
+        BucketedTableTestSpec(
+          Some(BucketSpec(7, Seq("i", "j"), Seq("i", "j"))), expectedShuffle = true))
+    }
+  }
+
+  test("bucket coalescing is applied when join expressions match with partitioning expressions",
+    DisableAdaptiveExecution("Expected shuffle num mismatched")) {
+    withTable("t1", "t2") {
+      df1.write.format("parquet").bucketBy(8, "i", "j").saveAsTable("t1")
+      df2.write.format("parquet").bucketBy(4, "i", "j").saveAsTable("t2")
+
+      withSQLConf(
+        SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "0",
+        SQLConf.COALESCE_BUCKETS_IN_JOIN_ENABLED.key -> "true") {
+        def verify(
+            query: String,
+            expectedNumShuffles: Int,
+            expectedCoalescedNumBuckets: Option[Int]): Unit = {
+          val plan = sql(query).queryExecution.executedPlan
+          val shuffles = plan.collect { case s: ShuffleExchangeExec => s }
+          assert(shuffles.length == expectedNumShuffles)
+
+          val scans = plan.collect {
+            case f: FileSourceScanExec if f.optionalNumCoalescedBuckets.isDefined => f
+          }
+          if (expectedCoalescedNumBuckets.isDefined) {
+            assert(scans.length == 1)
+            assert(scans.head.optionalNumCoalescedBuckets == expectedCoalescedNumBuckets)
+          } else {
+            assert(scans.isEmpty)
+          }
+        }
+
+        // Coalescing applied since join expressions match with the bucket columns.
+        verify("SELECT * FROM t1 JOIN t2 ON t1.i = t2.i AND t1.j = t2.j", 0, Some(4))
+        // Coalescing applied when columns are aliased.
+        verify(
+          "SELECT * FROM t1 JOIN (SELECT i AS x, j AS y FROM t2) ON t1.i = x AND t1.j = y",
+          0,
+          Some(4))
+        // Coalescing is not applied when join expressions do not match with bucket columns.
+        verify("SELECT * FROM t1 JOIN t2 ON t1.i = t2.i", 2, None)
+      }
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala
index a410f32d4af7e..0a5feda1bd533 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala
@@ -88,7 +88,7 @@ abstract class BucketedWriteSuite extends QueryTest with SQLTestUtils {
     val e = intercept[AnalysisException] {
       df.write.sortBy("j").saveAsTable("tt")
     }
-    assert(e.getMessage == "sortBy must be used together with bucketBy;")
+    assert(e.getMessage == "sortBy must be used together with bucketBy")
   }
 
   test("sorting by non-orderable column") {
@@ -102,7 +102,7 @@ abstract class BucketedWriteSuite extends QueryTest with SQLTestUtils {
     val e = intercept[AnalysisException] {
       df.write.bucketBy(2, "i").parquet("/tmp/path")
     }
-    assert(e.getMessage == "'save' does not support bucketBy right now;")
+    assert(e.getMessage == "'save' does not support bucketBy right now")
   }
 
   test("write bucketed and sorted data using save()") {
@@ -111,7 +111,7 @@ abstract class BucketedWriteSuite extends QueryTest with SQLTestUtils {
     val e = intercept[AnalysisException] {
       df.write.bucketBy(2, "i").sortBy("i").parquet("/tmp/path")
     }
-    assert(e.getMessage == "'save' does not support bucketBy and sortBy right now;")
+    assert(e.getMessage == "'save' does not support bucketBy and sortBy right now")
   }
 
   test("write bucketed data using insertInto()") {
@@ -120,7 +120,7 @@ abstract class BucketedWriteSuite extends QueryTest with SQLTestUtils {
     val e = intercept[AnalysisException] {
       df.write.bucketBy(2, "i").insertInto("tt")
     }
-    assert(e.getMessage == "'insertInto' does not support bucketBy right now;")
+    assert(e.getMessage == "'insertInto' does not support bucketBy right now")
   }
 
   test("write bucketed and sorted data using insertInto()") {
@@ -129,7 +129,7 @@ abstract class BucketedWriteSuite extends QueryTest with SQLTestUtils {
     val e = intercept[AnalysisException] {
       df.write.bucketBy(2, "i").sortBy("i").insertInto("tt")
     }
-    assert(e.getMessage == "'insertInto' does not support bucketBy and sortBy right now;")
+    assert(e.getMessage == "'insertInto' does not support bucketBy and sortBy right now")
   }
 
   private lazy val df = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala
index 983209051c8ae..9a7c7e0edc409 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala
@@ -22,7 +22,7 @@ import java.io.File
 import org.apache.spark.SparkException
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.catalog.BucketSpec
+import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTableType}
 import org.apache.spark.sql.catalyst.parser.ParseException
 import org.apache.spark.sql.internal.SQLConf.BUCKETING_MAX_BUCKETS
 import org.apache.spark.sql.test.SharedSparkSession
@@ -166,24 +166,21 @@ class CreateTableAsSelectSuite extends DataSourceTest with SharedSparkSession {
         )
       }.getMessage
       assert(error.contains("Operation not allowed") &&
-        error.contains("CREATE TEMPORARY TABLE ... USING ... AS query"))
+        error.contains("CREATE TEMPORARY TABLE"))
     }
   }
 
-  test("disallows CREATE EXTERNAL TABLE ... USING ... AS query") {
+  test("SPARK-33651: allow CREATE EXTERNAL TABLE ... USING ... if location is specified") {
     withTable("t") {
-      val error = intercept[ParseException] {
-        sql(
-          s"""
-             |CREATE EXTERNAL TABLE t USING PARQUET
-             |OPTIONS (PATH '${path.toURI}')
-             |AS SELECT 1 AS a, 2 AS b
-           """.stripMargin
-        )
-      }.getMessage
-
-      assert(error.contains("Operation not allowed") &&
-        error.contains("CREATE EXTERNAL TABLE ..."))
+      sql(
+        s"""
+           |CREATE EXTERNAL TABLE t USING PARQUET
+           |OPTIONS (PATH '${path.toURI}')
+           |AS SELECT 1 AS a, 2 AS b
+         """.stripMargin)
+      val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t"))
+      assert(table.tableType == CatalogTableType.EXTERNAL)
+      assert(table.location.toString == path.toURI.toString.stripSuffix("/"))
     }
   }
 
@@ -237,7 +234,7 @@ class CreateTableAsSelectSuite extends DataSourceTest with SharedSparkSession {
     }
   }
 
-  test("create table using as select - with overriden max number of buckets") {
+  test("create table using as select - with overridden max number of buckets") {
     def createTableSql(numBuckets: Int): String =
       s"""
          |CREATE TABLE t USING PARQUET
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/DataSourceAnalysisSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/DataSourceAnalysisSuite.scala
index a6c50904d395b..81ce979ef0b62 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/DataSourceAnalysisSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/DataSourceAnalysisSuite.scala
@@ -23,12 +23,13 @@ import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions.{Alias, AnsiCast, Attribute, Cast, Expression, Literal}
+import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.execution.datasources.DataSourceAnalysis
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy
 import org.apache.spark.sql.types.{DataType, IntegerType, StructType}
 
-class DataSourceAnalysisSuite extends SparkFunSuite with BeforeAndAfterAll {
+class DataSourceAnalysisSuite extends SparkFunSuite with BeforeAndAfterAll with SQLHelper {
 
   private var targetAttributes: Seq[Attribute] = _
   private var targetPartitionSchema: StructType = _
@@ -51,19 +52,26 @@ class DataSourceAnalysisSuite extends SparkFunSuite with BeforeAndAfterAll {
   }
 
   Seq(true, false).foreach { caseSensitive =>
-    val conf = new SQLConf().copy(SQLConf.CASE_SENSITIVE -> caseSensitive)
+    def testRule(testName: String, caseSensitive: Boolean)(func: => Unit): Unit = {
+      test(s"$testName (caseSensitive: $caseSensitive)") {
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
+          func
+        }
+      }
+    }
+
     def cast(e: Expression, dt: DataType): Expression = {
-      conf.storeAssignmentPolicy match {
+      SQLConf.get.storeAssignmentPolicy match {
         case StoreAssignmentPolicy.ANSI | StoreAssignmentPolicy.STRICT =>
-          AnsiCast(e, dt, Option(conf.sessionLocalTimeZone))
+          AnsiCast(e, dt, Option(SQLConf.get.sessionLocalTimeZone))
         case _ =>
-          Cast(e, dt, Option(conf.sessionLocalTimeZone))
+          Cast(e, dt, Option(SQLConf.get.sessionLocalTimeZone))
       }
     }
-    val rule = DataSourceAnalysis(conf)
-    test(
-      s"convertStaticPartitions only handle INSERT having at least static partitions " +
-        s"(caseSensitive: $caseSensitive)") {
+    val rule = DataSourceAnalysis
+    testRule(
+      "convertStaticPartitions only handle INSERT having at least static partitions",
+        caseSensitive) {
       intercept[AssertionError] {
         rule.convertStaticPartitions(
           sourceAttributes = Seq('e.int, 'f.int),
@@ -73,7 +81,7 @@ class DataSourceAnalysisSuite extends SparkFunSuite with BeforeAndAfterAll {
       }
     }
 
-    test(s"Missing columns (caseSensitive: $caseSensitive)") {
+    testRule("Missing columns", caseSensitive) {
       // Missing columns.
       intercept[AnalysisException] {
         rule.convertStaticPartitions(
@@ -84,7 +92,7 @@ class DataSourceAnalysisSuite extends SparkFunSuite with BeforeAndAfterAll {
       }
     }
 
-    test(s"Missing partitioning columns (caseSensitive: $caseSensitive)") {
+    testRule("Missing partitioning columns", caseSensitive) {
       // Missing partitioning columns.
       intercept[AnalysisException] {
         rule.convertStaticPartitions(
@@ -113,7 +121,7 @@ class DataSourceAnalysisSuite extends SparkFunSuite with BeforeAndAfterAll {
       }
     }
 
-    test(s"Wrong partitioning columns (caseSensitive: $caseSensitive)") {
+    testRule("Wrong partitioning columns", caseSensitive) {
       // Wrong partitioning columns.
       intercept[AnalysisException] {
         rule.convertStaticPartitions(
@@ -144,9 +152,7 @@ class DataSourceAnalysisSuite extends SparkFunSuite with BeforeAndAfterAll {
       }
     }
 
-    test(
-      s"Static partitions need to appear before dynamic partitions" +
-      s" (caseSensitive: $caseSensitive)") {
+    testRule("Static partitions need to appear before dynamic partitions", caseSensitive) {
       // Static partitions need to appear before dynamic partitions.
       intercept[AnalysisException] {
         rule.convertStaticPartitions(
@@ -157,7 +163,7 @@ class DataSourceAnalysisSuite extends SparkFunSuite with BeforeAndAfterAll {
       }
     }
 
-    test(s"All static partitions (caseSensitive: $caseSensitive)") {
+    testRule("All static partitions", caseSensitive) {
       if (!caseSensitive) {
         val nonPartitionedAttributes = Seq('e.int, 'f.int)
         val expected = nonPartitionedAttributes ++
@@ -195,7 +201,7 @@ class DataSourceAnalysisSuite extends SparkFunSuite with BeforeAndAfterAll {
       }
     }
 
-    test(s"Static partition and dynamic partition (caseSensitive: $caseSensitive)") {
+    testRule("Static partition and dynamic partition", caseSensitive) {
       val nonPartitionedAttributes = Seq('e.int, 'f.int)
       val dynamicPartitionAttributes = Seq('g.int)
       val expected =
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/DisableUnnecessaryBucketedScanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/DisableUnnecessaryBucketedScanSuite.scala
new file mode 100644
index 0000000000000..1a19824a31555
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/DisableUnnecessaryBucketedScanSuite.scala
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.sources
+
+import org.apache.spark.sql.QueryTest
+import org.apache.spark.sql.catalyst.expressions.AttributeReference
+import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning
+import org.apache.spark.sql.execution.FileSourceScanExec
+import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
+import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.StaticSQLConf.CATALOG_IMPLEMENTATION
+import org.apache.spark.sql.test.{SharedSparkSession, SQLTestUtils}
+
+class DisableUnnecessaryBucketedScanWithoutHiveSupportSuite
+  extends DisableUnnecessaryBucketedScanSuite
+  with SharedSparkSession
+  with DisableAdaptiveExecutionSuite {
+
+  protected override def beforeAll(): Unit = {
+    super.beforeAll()
+    assert(spark.sparkContext.conf.get(CATALOG_IMPLEMENTATION) == "in-memory")
+  }
+}
+
+class DisableUnnecessaryBucketedScanWithoutHiveSupportSuiteAE
+  extends DisableUnnecessaryBucketedScanSuite
+  with SharedSparkSession
+  with EnableAdaptiveExecutionSuite {
+
+  protected override def beforeAll(): Unit = {
+    super.beforeAll()
+    assert(spark.sparkContext.conf.get(CATALOG_IMPLEMENTATION) == "in-memory")
+  }
+}
+
+abstract class DisableUnnecessaryBucketedScanSuite
+  extends QueryTest
+  with SQLTestUtils
+  with AdaptiveSparkPlanHelper {
+
+  import testImplicits._
+
+  private lazy val df1 =
+    (0 until 50).map(i => (i % 5, i % 13, i.toString)).toDF("i", "j", "k").as("df1")
+  private lazy val df2 =
+    (0 until 50).map(i => (i % 7, i % 11, i.toString)).toDF("i", "j", "k").as("df2")
+
+  private def checkDisableBucketedScan(
+      query: String,
+      expectedNumScanWithAutoScanEnabled: Int,
+      expectedNumScanWithAutoScanDisabled: Int): Unit = {
+
+    def checkNumBucketedScan(query: String, expectedNumBucketedScan: Int): Unit = {
+      val plan = sql(query).queryExecution.executedPlan
+      val bucketedScan = collect(plan) { case s: FileSourceScanExec if s.bucketedScan => s }
+      assert(bucketedScan.length == expectedNumBucketedScan)
+    }
+
+    withSQLConf(SQLConf.AUTO_BUCKETED_SCAN_ENABLED.key -> "true") {
+      checkNumBucketedScan(query, expectedNumScanWithAutoScanEnabled)
+      val result = sql(query).collect()
+
+      withSQLConf(SQLConf.AUTO_BUCKETED_SCAN_ENABLED.key -> "false") {
+        checkNumBucketedScan(query, expectedNumScanWithAutoScanDisabled)
+        checkAnswer(sql(query), result)
+      }
+    }
+  }
+
+  test("SPARK-32859: disable unnecessary bucketed table scan - basic test") {
+    withTable("t1", "t2", "t3") {
+      df1.write.format("parquet").bucketBy(8, "i").saveAsTable("t1")
+      df2.write.format("parquet").bucketBy(8, "i").saveAsTable("t2")
+      df2.write.format("parquet").bucketBy(4, "i").saveAsTable("t3")
+
+      Seq(
+        // Read bucketed table
+        ("SELECT * FROM t1", 0, 1),
+        ("SELECT i FROM t1", 0, 1),
+        ("SELECT j FROM t1", 0, 0),
+        // Filter on bucketed column
+        ("SELECT * FROM t1 WHERE i = 1", 0, 1),
+        // Filter on non-bucketed column
+        ("SELECT * FROM t1 WHERE j = 1", 0, 1),
+        // Join with same buckets
+        ("SELECT /*+ broadcast(t1)*/ * FROM t1 JOIN t2 ON t1.i = t2.i", 0, 2),
+        ("SELECT /*+ shuffle_hash(t1)*/ * FROM t1 JOIN t2 ON t1.i = t2.i", 2, 2),
+        ("SELECT /*+ merge(t1)*/ * FROM t1 JOIN t2 ON t1.i = t2.i", 2, 2),
+        // Join with different buckets
+        ("SELECT /*+ broadcast(t1)*/ * FROM t1 JOIN t3 ON t1.i = t3.i", 0, 2),
+        ("SELECT /*+ shuffle_hash(t1)*/ * FROM t1 JOIN t3 ON t1.i = t3.i", 1, 2),
+        ("SELECT /*+ merge(t1)*/ * FROM t1 JOIN t3 ON t1.i = t3.i", 1, 2),
+        // Join on non-bucketed column
+        ("SELECT /*+ broadcast(t1)*/ * FROM t1 JOIN t2 ON t1.i = t2.j", 0, 2),
+        ("SELECT /*+ shuffle_hash(t1)*/ * FROM t1 JOIN t2 ON t1.i = t2.j", 1, 2),
+        ("SELECT /*+ merge(t1)*/ * FROM t1 JOIN t2 ON t1.i = t2.j", 1, 2),
+        ("SELECT /*+ broadcast(t1)*/ * FROM t1 JOIN t2 ON t1.j = t2.j", 0, 2),
+        ("SELECT /*+ shuffle_hash(t1)*/ * FROM t1 JOIN t2 ON t1.j = t2.j", 0, 2),
+        ("SELECT /*+ merge(t1)*/ * FROM t1 JOIN t2 ON t1.j = t2.j", 0, 2),
+        // Aggregate on bucketed column
+        ("SELECT SUM(i) FROM t1 GROUP BY i", 1, 1),
+        // Aggregate on non-bucketed column
+        ("SELECT SUM(i) FROM t1 GROUP BY j", 0, 1),
+        ("SELECT j, SUM(i), COUNT(j) FROM t1 GROUP BY j", 0, 1)
+      ).foreach { case (query, numScanWithAutoScanEnabled, numScanWithAutoScanDisabled) =>
+        checkDisableBucketedScan(query, numScanWithAutoScanEnabled, numScanWithAutoScanDisabled)
+      }
+    }
+  }
+
+  test("SPARK-32859: disable unnecessary bucketed table scan - multiple joins test") {
+    withTable("t1", "t2", "t3") {
+      df1.write.format("parquet").bucketBy(8, "i").saveAsTable("t1")
+      df2.write.format("parquet").bucketBy(8, "i").saveAsTable("t2")
+      df2.write.format("parquet").bucketBy(4, "i").saveAsTable("t3")
+
+      Seq(
+        // Multiple joins on bucketed columns
+        ("""
+         SELECT /*+ broadcast(t1, t3)*/ * FROM t1 JOIN t2 JOIN t3
+         ON t1.i = t2.i AND t2.i = t3.i
+         """.stripMargin, 0, 3),
+        ("""
+         SELECT /*+ broadcast(t1) merge(t3)*/ * FROM t1 JOIN t2 JOIN t3
+         ON t1.i = t2.i AND t2.i = t3.i
+         """.stripMargin, 2, 3),
+        ("""
+         SELECT /*+ merge(t1) broadcast(t3)*/ * FROM t1 JOIN t2 JOIN t3
+         ON t1.i = t2.i AND t2.i = t3.i
+         """.stripMargin, 2, 3),
+        ("""
+         SELECT /*+ merge(t1, t3)*/ * FROM t1 JOIN t2 JOIN t3
+         ON t1.i = t2.i AND t2.i = t3.i
+         """.stripMargin, 2, 3),
+        // Multiple joins on non-bucketed columns
+        ("""
+         SELECT /*+ broadcast(t1, t3)*/ * FROM t1 JOIN t2 JOIN t3
+         ON t1.i = t2.j AND t2.j = t3.i
+         """.stripMargin, 0, 3),
+        ("""
+         SELECT /*+ merge(t1, t3)*/ * FROM t1 JOIN t2 JOIN t3
+         ON t1.i = t2.j AND t2.j = t3.i
+         """.stripMargin, 1, 3),
+        ("""
+         SELECT /*+ merge(t1, t3)*/ * FROM t1 JOIN t2 JOIN t3
+         ON t1.j = t2.j AND t2.j = t3.j
+         """.stripMargin, 0, 3)
+      ).foreach { case (query, numScanWithAutoScanEnabled, numScanWithAutoScanDisabled) =>
+        checkDisableBucketedScan(query, numScanWithAutoScanEnabled, numScanWithAutoScanDisabled)
+      }
+    }
+  }
+
+  test("SPARK-32859: disable unnecessary bucketed table scan - multiple bucketed columns test") {
+    withTable("t1", "t2", "t3") {
+      df1.write.format("parquet").bucketBy(8, "i", "j").saveAsTable("t1")
+      df2.write.format("parquet").bucketBy(8, "i", "j").saveAsTable("t2")
+      df2.write.format("parquet").bucketBy(4, "i", "j").saveAsTable("t3")
+
+      Seq(
+        // Filter on bucketed columns
+        ("SELECT * FROM t1 WHERE i = 1", 0, 1),
+        ("SELECT * FROM t1 WHERE i = 1 AND j = 1", 0, 1),
+        // Join on bucketed columns
+        ("""
+         SELECT /*+ broadcast(t1)*/ * FROM t1 JOIN t2 ON t1.i = t2.i AND t1.j = t2.j
+         """.stripMargin, 0, 2),
+        ("""
+         SELECT /*+ merge(t1)*/ * FROM t1 JOIN t2 ON t1.i = t2.i AND t1.j = t2.j
+         """.stripMargin, 2, 2),
+        ("""
+         SELECT /*+ merge(t1)*/ * FROM t1 JOIN t3 ON t1.i = t3.i AND t1.j = t3.j
+         """.stripMargin, 1, 2),
+        ("SELECT /*+ merge(t1)*/ * FROM t1 JOIN t2 ON t1.i = t2.i", 0, 2),
+        // Aggregate on bucketed columns
+        ("SELECT i, j, COUNT(*) FROM t1 GROUP BY i, j", 1, 1),
+        ("SELECT i, COUNT(i) FROM t1 GROUP BY i", 0, 0),
+        ("SELECT i, COUNT(j) FROM t1 GROUP BY i", 0, 1)
+      ).foreach { case (query, numScanWithAutoScanEnabled, numScanWithAutoScanDisabled) =>
+        checkDisableBucketedScan(query, numScanWithAutoScanEnabled, numScanWithAutoScanDisabled)
+      }
+    }
+  }
+
+  test("SPARK-32859: disable unnecessary bucketed table scan - other operators test") {
+    withTable("t1", "t2", "t3") {
+      df1.write.format("parquet").bucketBy(8, "i").saveAsTable("t1")
+      df2.write.format("parquet").bucketBy(8, "i").saveAsTable("t2")
+      df1.write.format("parquet").saveAsTable("t3")
+
+      Seq(
+        // Operator with interesting partition not in sub-plan
+        ("""
+         SELECT t1.i FROM t1
+         UNION ALL
+         (SELECT t2.i FROM t2 GROUP BY t2.i)
+         """.stripMargin, 1, 2),
+        // Non-allowed operator in sub-plan
+        ("""
+         SELECT COUNT(*)
+         FROM (SELECT t1.i FROM t1 UNION ALL SELECT t2.i FROM t2)
+         GROUP BY i
+         """.stripMargin, 2, 2),
+        // Multiple [[Exchange]] in sub-plan
+        ("""
+         SELECT j, SUM(i), COUNT(*) FROM t1 GROUP BY j
+         DISTRIBUTE BY j
+         """.stripMargin, 0, 1),
+        ("""
+         SELECT j, COUNT(*)
+         FROM (SELECT i, j FROM t1 DISTRIBUTE BY i, j)
+         GROUP BY j
+         """.stripMargin, 0, 1),
+        // No bucketed table scan in plan
+        ("""
+         SELECT j, COUNT(*)
+         FROM (SELECT t1.j FROM t1 JOIN t3 ON t1.j = t3.j)
+         GROUP BY j
+         """.stripMargin, 0, 0)
+      ).foreach { case (query, numScanWithAutoScanEnabled, numScanWithAutoScanDisabled) =>
+        checkDisableBucketedScan(query, numScanWithAutoScanEnabled, numScanWithAutoScanDisabled)
+      }
+    }
+  }
+
+  test("SPARK-33075: not disable bucketed table scan for cached query") {
+    withTable("t1") {
+      withSQLConf(SQLConf.AUTO_BUCKETED_SCAN_ENABLED.key -> "true") {
+        df1.write.format("parquet").bucketBy(8, "i").saveAsTable("t1")
+        spark.catalog.cacheTable("t1")
+        assertCached(spark.table("t1"))
+
+        // Verify cached bucketed table scan not disabled
+        val partitioning = stripAQEPlan(spark.table("t1").queryExecution.executedPlan)
+          .outputPartitioning
+        assert(partitioning match {
+          case HashPartitioning(Seq(column: AttributeReference), 8) if column.name == "i" => true
+          case _ => false
+        })
+        val aggregateQueryPlan = sql("SELECT SUM(i) FROM t1 GROUP BY i").queryExecution.executedPlan
+        assert(find(aggregateQueryPlan)(_.isInstanceOf[ShuffleExchangeExec]).isEmpty)
+      }
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
index 87a4d061b8170..9de3b0e954848 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
@@ -623,12 +623,12 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
         var msg = intercept[AnalysisException] {
           sql("insert into t select 1L, 2")
         }.getMessage
-        assert(msg.contains("Cannot safely cast 'i': LongType to IntegerType"))
+        assert(msg.contains("Cannot safely cast 'i': bigint to int"))
 
         msg = intercept[AnalysisException] {
           sql("insert into t select 1, 2.0")
         }.getMessage
-        assert(msg.contains("Cannot safely cast 'd': DecimalType(2,1) to DoubleType"))
+        assert(msg.contains("Cannot safely cast 'd': decimal(2,1) to double"))
 
         msg = intercept[AnalysisException] {
           sql("insert into t select 1, 2.0D, 3")
@@ -660,18 +660,18 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
         var msg = intercept[AnalysisException] {
           sql("insert into t values('a', 'b')")
         }.getMessage
-        assert(msg.contains("Cannot safely cast 'i': StringType to IntegerType") &&
-          msg.contains("Cannot safely cast 'd': StringType to DoubleType"))
+        assert(msg.contains("Cannot safely cast 'i': string to int") &&
+          msg.contains("Cannot safely cast 'd': string to double"))
         msg = intercept[AnalysisException] {
           sql("insert into t values(now(), now())")
         }.getMessage
-        assert(msg.contains("Cannot safely cast 'i': TimestampType to IntegerType") &&
-          msg.contains("Cannot safely cast 'd': TimestampType to DoubleType"))
+        assert(msg.contains("Cannot safely cast 'i': timestamp to int") &&
+          msg.contains("Cannot safely cast 'd': timestamp to double"))
         msg = intercept[AnalysisException] {
           sql("insert into t values(true, false)")
         }.getMessage
-        assert(msg.contains("Cannot safely cast 'i': BooleanType to IntegerType") &&
-          msg.contains("Cannot safely cast 'd': BooleanType to DoubleType"))
+        assert(msg.contains("Cannot safely cast 'i': boolean to int") &&
+          msg.contains("Cannot safely cast 'd': boolean to double"))
       }
     }
   }
@@ -731,13 +731,13 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
         var msg = intercept[SparkException] {
           sql(s"insert into t values(${outOfRangeValue1}D)")
         }.getCause.getMessage
-        assert(msg.contains(s"Casting $outOfRangeValue1 to long causes overflow"))
+        assert(msg.contains(s"Casting $outOfRangeValue1 to bigint causes overflow"))
 
         val outOfRangeValue2 = Math.nextDown(Long.MinValue)
         msg = intercept[SparkException] {
           sql(s"insert into t values(${outOfRangeValue2}D)")
         }.getCause.getMessage
-        assert(msg.contains(s"Casting $outOfRangeValue2 to long causes overflow"))
+        assert(msg.contains(s"Casting $outOfRangeValue2 to bigint causes overflow"))
       }
     }
   }
@@ -756,23 +756,43 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
     }
   }
 
-  test("SPARK-30844: static partition should also follow StoreAssignmentPolicy") {
-    SQLConf.StoreAssignmentPolicy.values.foreach { policy =>
-      withSQLConf(
-        SQLConf.STORE_ASSIGNMENT_POLICY.key -> policy.toString) {
-        withTable("t") {
-          sql("create table t(a int, b string) using parquet partitioned by (a)")
-          policy match {
-            case SQLConf.StoreAssignmentPolicy.ANSI | SQLConf.StoreAssignmentPolicy.STRICT =>
-              val errorMsg = intercept[NumberFormatException] {
-                sql("insert into t partition(a='ansi') values('ansi')")
-              }.getMessage
-              assert(errorMsg.contains("invalid input syntax for type numeric: ansi"))
-            case SQLConf.StoreAssignmentPolicy.LEGACY =>
-              sql("insert into t partition(a='ansi') values('ansi')")
-              checkAnswer(sql("select * from t"), Row("ansi", null) :: Nil)
-          }
-        }
+  test("SPARK-33354: Throw exceptions on inserting invalid cast with ANSI casting policy") {
+    withSQLConf(
+      SQLConf.STORE_ASSIGNMENT_POLICY.key -> SQLConf.StoreAssignmentPolicy.ANSI.toString) {
+      withTable("t") {
+        sql("CREATE TABLE t(i int, t timestamp) USING parquet")
+        val msg = intercept[AnalysisException] {
+          sql("INSERT INTO t VALUES (TIMESTAMP('2010-09-02 14:10:10'), 1)")
+        }.getMessage
+        assert(msg.contains("Cannot safely cast 'i': timestamp to int"))
+        assert(msg.contains("Cannot safely cast 't': int to timestamp"))
+      }
+
+      withTable("t") {
+        sql("CREATE TABLE t(i int, d date) USING parquet")
+        val msg = intercept[AnalysisException] {
+          sql("INSERT INTO t VALUES (date('2010-09-02'), 1)")
+        }.getMessage
+        assert(msg.contains("Cannot safely cast 'i': date to int"))
+        assert(msg.contains("Cannot safely cast 'd': int to date"))
+      }
+
+      withTable("t") {
+        sql("CREATE TABLE t(b boolean, t timestamp) USING parquet")
+        val msg = intercept[AnalysisException] {
+          sql("INSERT INTO t VALUES (TIMESTAMP('2010-09-02 14:10:10'), true)")
+        }.getMessage
+        assert(msg.contains("Cannot safely cast 'b': timestamp to boolean"))
+        assert(msg.contains("Cannot safely cast 't': boolean to timestamp"))
+      }
+
+      withTable("t") {
+        sql("CREATE TABLE t(b boolean, d date) USING parquet")
+        val msg = intercept[AnalysisException] {
+          sql("INSERT INTO t VALUES (date('2010-09-02'), true)")
+        }.getMessage
+        assert(msg.contains("Cannot safely cast 'b': date to boolean"))
+        assert(msg.contains("Cannot safely cast 'd': boolean to date"))
       }
     }
   }
@@ -804,7 +824,7 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
         .add("s", StringType, false)
       val newTable = CatalogTable(
         identifier = TableIdentifier("test_table", None),
-        tableType = CatalogTableType.EXTERNAL,
+        tableType = CatalogTableType.MANAGED,
         storage = CatalogStorageFormat(
           locationUri = None,
           inputFormat = None,
@@ -826,21 +846,29 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
   }
 
   test("Stop task set if FileAlreadyExistsException was thrown") {
-    withSQLConf("fs.file.impl" -> classOf[FileExistingTestFileSystem].getName,
-        "fs.file.impl.disable.cache" -> "true") {
-      withTable("t") {
-        sql(
-          """
-            |CREATE TABLE t(i INT, part1 INT) USING PARQUET
-            |PARTITIONED BY (part1)
+    Seq(true, false).foreach { fastFail =>
+      withSQLConf("fs.file.impl" -> classOf[FileExistingTestFileSystem].getName,
+        "fs.file.impl.disable.cache" -> "true",
+        SQLConf.FASTFAIL_ON_FILEFORMAT_OUTPUT.key -> fastFail.toString) {
+        withTable("t") {
+          sql(
+            """
+              |CREATE TABLE t(i INT, part1 INT) USING PARQUET
+              |PARTITIONED BY (part1)
           """.stripMargin)
 
-        val df = Seq((1, 1)).toDF("i", "part1")
-        val err = intercept[SparkException] {
-          df.write.mode("overwrite").format("parquet").insertInto("t")
+          val df = Seq((1, 1)).toDF("i", "part1")
+          val err = intercept[SparkException] {
+            df.write.mode("overwrite").format("parquet").insertInto("t")
+          }
+
+          if (fastFail) {
+            assert(err.getCause.getMessage.contains("can not write to output file: " +
+              "org.apache.hadoop.fs.FileAlreadyExistsException"))
+          } else {
+            assert(err.getCause.getMessage.contains("Task failed while writing rows"))
+          }
         }
-        assert(err.getCause.getMessage.contains("can not write to output file: " +
-          "org.apache.hadoop.fs.FileAlreadyExistsException"))
       }
     }
   }
@@ -866,6 +894,62 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
     }.getMessage
     assert(message.contains("LOCAL is supported only with file: scheme"))
   }
+
+  test("SPARK-32508 " +
+    "Disallow empty part col values in partition spec before static partition writing") {
+    withTable("insertTable") {
+      sql(
+        """
+          |CREATE TABLE insertTable(i int, part1 string, part2 string) USING PARQUET
+          |PARTITIONED BY (part1, part2)
+            """.stripMargin)
+      val msg = "Partition spec is invalid"
+      assert(intercept[AnalysisException] {
+        sql("INSERT INTO TABLE insertTable PARTITION(part1=1, part2='') SELECT 1")
+      }.getMessage.contains(msg))
+      assert(intercept[AnalysisException] {
+        sql("INSERT INTO TABLE insertTable PARTITION(part1='', part2) SELECT 1 ,'' AS part2")
+      }.getMessage.contains(msg))
+
+      sql("INSERT INTO TABLE insertTable PARTITION(part1='1', part2='2') SELECT 1")
+      sql("INSERT INTO TABLE insertTable PARTITION(part1='1', part2) SELECT 1 ,'2' AS part2")
+      sql("INSERT INTO TABLE insertTable PARTITION(part1='1', part2) SELECT 1 ,'' AS part2")
+    }
+  }
+
+  test("SPARK-33294: Add query resolved check before analyze InsertIntoDir") {
+    withTempPath { path =>
+      val msg = intercept[AnalysisException] {
+        sql(
+          s"""
+            |INSERT OVERWRITE DIRECTORY '${path.getAbsolutePath}' USING PARQUET
+            |SELECT * FROM (
+            | SELECT c3 FROM (
+            |  SELECT c1, c2 from values(1,2) t(c1, c2)
+            |  )
+            |)
+          """.stripMargin)
+      }.getMessage
+      assert(msg.contains("cannot resolve 'c3' given input columns"))
+    }
+  }
+
+  test("SPARK-34926: PartitioningUtils.getPathFragment() should respect partition value is null") {
+    withTable("t1", "t2") {
+      sql("CREATE TABLE t1(id INT) USING PARQUET")
+      sql(
+        """
+          |CREATE TABLE t2 (c1 INT, part STRING)
+          |  USING parquet
+          |PARTITIONED BY (part)
+          |""".stripMargin)
+      sql(
+        """
+          |INSERT INTO TABLE t2 PARTITION (part = null)
+          |SELECT * FROM t1 where 1=0""".stripMargin)
+      checkAnswer(spark.table("t2"), Nil)
+    }
+  }
 }
 
 class FileExistingTestFileSystem extends RawLocalFileSystem {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala
index 1c4e2a967b0a2..b9266429f81a5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala
@@ -20,11 +20,12 @@ package org.apache.spark.sql.sources
 import java.io.File
 import java.sql.Timestamp
 
-import org.apache.hadoop.mapreduce.TaskAttemptContext
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.mapreduce.{JobContext, TaskAttemptContext}
 
 import org.apache.spark.TestUtils
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.{QueryTest, Row}
+import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
 import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol
@@ -156,4 +157,56 @@ class PartitionedWriteSuite extends QueryTest with SharedSparkSession {
       }
     }
   }
+
+  test("SPARK-31968: duplicate partition columns check") {
+    withTempPath { f =>
+      val e = intercept[AnalysisException](
+        Seq((3, 2)).toDF("a", "b").write.partitionBy("b", "b").csv(f.getAbsolutePath))
+      assert(e.getMessage.contains("Found duplicate column(s) b, b: `b`"))
+    }
+  }
+
+  test("SPARK-27194 SPARK-29302: Fix commit collision in dynamic partition overwrite mode") {
+    withSQLConf(SQLConf.PARTITION_OVERWRITE_MODE.key ->
+      SQLConf.PartitionOverwriteMode.DYNAMIC.toString,
+      SQLConf.FILE_COMMIT_PROTOCOL_CLASS.key ->
+        classOf[PartitionFileExistCommitProtocol].getName) {
+      withTempDir { d =>
+        withTable("t") {
+          sql(
+            s"""
+               | create table t(c1 int, p1 int) using parquet partitioned by (p1)
+               | location '${d.getAbsolutePath}'
+            """.stripMargin)
+
+          val df = Seq((1, 2)).toDF("c1", "p1")
+          df.write
+            .partitionBy("p1")
+            .mode("overwrite")
+            .saveAsTable("t")
+          checkAnswer(sql("select * from t"), df)
+        }
+      }
+    }
+  }
+}
+
+/**
+ * A file commit protocol with pre-created partition file. when try to overwrite partition dir
+ * in dynamic partition mode, FileAlreadyExist exception would raise without SPARK-27194
+ */
+private class PartitionFileExistCommitProtocol(
+    jobId: String,
+    path: String,
+    dynamicPartitionOverwrite: Boolean)
+  extends SQLHadoopMapReduceCommitProtocol(jobId, path, dynamicPartitionOverwrite) {
+  override def setupJob(jobContext: JobContext): Unit = {
+    super.setupJob(jobContext)
+    val stagingDir = new File(new Path(path).toUri.getPath, s".spark-staging-$jobId")
+    stagingDir.mkdirs()
+    val stagingPartDir = new File(stagingDir, "p1=2")
+    stagingPartDir.mkdirs()
+    val conflictTaskFile = new File(stagingPartDir, s"part-00000-$jobId.c000.snappy.parquet")
+    conflictTaskFile.createNewFile()
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/PathOptionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/PathOptionSuite.scala
index 9b26a5659df49..48d717daf00d4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/PathOptionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/PathOptionSuite.scala
@@ -19,8 +19,6 @@ package org.apache.spark.sql.sources
 
 import java.net.URI
 
-import org.apache.hadoop.fs.Path
-
 import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession, SQLContext}
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.catalog.CatalogUtils
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/ResolvedDataSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/ResolvedDataSourceSuite.scala
index 36fb418b09cb6..818a66eb436cc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/ResolvedDataSourceSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/ResolvedDataSourceSuite.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql.sources
 
+import java.time.ZoneId
+
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.execution.datasources.DataSource
@@ -27,7 +29,7 @@ class ResolvedDataSourceSuite extends SharedSparkSession {
     DataSource(
       sparkSession = spark,
       className = name,
-      options = Map(DateTimeUtils.TIMEZONE_OPTION -> DateTimeUtils.defaultTimeZone().getID)
+      options = Map(DateTimeUtils.TIMEZONE_OPTION -> ZoneId.systemDefault().getId)
     ).providingClass
 
   test("jdbc") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala
index 9a95bf770772e..de54b38627443 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala
@@ -22,6 +22,7 @@ import java.sql.{Date, Timestamp}
 
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql._
+import org.apache.spark.sql.catalyst.util.CharVarcharUtils
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types._
@@ -206,10 +207,6 @@ class TableScanSuite extends DataSourceTest with SharedSparkSession {
     (2 to 10).map(i => Row(i, i - 1)).toSeq)
 
   test("Schema and all fields") {
-    def hiveMetadata(dt: String): Metadata = {
-      new MetadataBuilder().putString(HIVE_TYPE_STRING, dt).build()
-    }
-
     val expectedSchema = StructType(
       StructField("string$%Field", StringType, true) ::
       StructField("binaryField", BinaryType, true) ::
@@ -224,8 +221,8 @@ class TableScanSuite extends DataSourceTest with SharedSparkSession {
       StructField("decimalField2", DecimalType(9, 2), true) ::
       StructField("dateField", DateType, true) ::
       StructField("timestampField", TimestampType, true) ::
-      StructField("varcharField", StringType, true, hiveMetadata("varchar(12)")) ::
-      StructField("charField", StringType, true, hiveMetadata("char(18)")) ::
+      StructField("varcharField", VarcharType(12), true) ::
+      StructField("charField", CharType(18), true) ::
       StructField("arrayFieldSimple", ArrayType(IntegerType), true) ::
       StructField("arrayFieldComplex",
         ArrayType(
@@ -248,7 +245,8 @@ class TableScanSuite extends DataSourceTest with SharedSparkSession {
       Nil
     )
 
-    assert(expectedSchema == spark.table("tableWithSchema").schema)
+    assert(CharVarcharUtils.replaceCharVarcharWithStringInSchema(expectedSchema) ==
+      spark.table("tableWithSchema").schema)
 
     withSQLConf(SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "false") {
         checkAnswer(
@@ -361,7 +359,7 @@ class TableScanSuite extends DataSourceTest with SharedSparkSession {
       val schemaNotMatch = intercept[Exception] {
         sql(
           s"""
-             |CREATE $tableType relationProvierWithSchema (i int)
+             |CREATE $tableType relationProviderWithSchema (i int)
              |USING org.apache.spark.sql.sources.SimpleScanSource
              |OPTIONS (
              |  From '1',
@@ -375,7 +373,7 @@ class TableScanSuite extends DataSourceTest with SharedSparkSession {
       val schemaNeeded = intercept[Exception] {
         sql(
           s"""
-             |CREATE $tableType schemaRelationProvierWithoutSchema
+             |CREATE $tableType schemaRelationProviderWithoutSchema
              |USING org.apache.spark.sql.sources.AllDataTypesScanSource
              |OPTIONS (
              |  From '1',
@@ -389,7 +387,7 @@ class TableScanSuite extends DataSourceTest with SharedSparkSession {
 
   test("read the data source tables that do not extend SchemaRelationProvider") {
     Seq("TEMPORARY VIEW", "TABLE").foreach { tableType =>
-      val tableName = "relationProvierWithSchema"
+      val tableName = "relationProviderWithSchema"
       withTable (tableName) {
         sql(
           s"""
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala
index 6486e1aee8649..67ab72a79145e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala
@@ -24,7 +24,9 @@ import java.util.{Calendar, Date, Locale}
 import java.util.concurrent.TimeUnit._
 
 import org.apache.commons.io.FileUtils
-import org.scalatest.{BeforeAndAfter, Matchers}
+import org.scalatest.BeforeAndAfter
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.{AnalysisException, Dataset}
@@ -32,7 +34,7 @@ import org.apache.spark.sql.catalyst.plans.logical.EventTimeWatermark
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.UTC
 import org.apache.spark.sql.execution.streaming._
 import org.apache.spark.sql.execution.streaming.sources.MemorySink
-import org.apache.spark.sql.functions.{count, window}
+import org.apache.spark.sql.functions.{count, timestamp_seconds, window}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.streaming.OutputMode._
 import org.apache.spark.util.Utils
@@ -129,7 +131,7 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche
     // No event time metrics when there is no watermarking
     val inputData1 = MemoryStream[Int]
     val aggWithoutWatermark = inputData1.toDF()
-      .withColumn("eventTime", $"value".cast("timestamp"))
+      .withColumn("eventTime", timestamp_seconds($"value"))
       .groupBy(window($"eventTime", "5 seconds") as 'window)
       .agg(count("*") as 'count)
       .select($"window".getField("start").cast("long").as[Long], $"count".as[Long])
@@ -146,7 +148,7 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche
     // All event time metrics where watermarking is set
     val inputData2 = MemoryStream[Int]
     val aggWithWatermark = inputData2.toDF()
-        .withColumn("eventTime", $"value".cast("timestamp"))
+        .withColumn("eventTime", timestamp_seconds($"value"))
         .withWatermark("eventTime", "10 seconds")
         .groupBy(window($"eventTime", "5 seconds") as 'window)
         .agg(count("*") as 'count)
@@ -169,7 +171,7 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche
     // All event time metrics where watermarking is set
     val inputData = MemoryStream[Int]
     val aggWithWatermark = inputData.toDF()
-        .withColumn("eventTime", $"value".cast("timestamp"))
+        .withColumn("eventTime", timestamp_seconds($"value"))
         .withWatermark("eventTime", "10 seconds")
         .groupBy(window($"eventTime", "5 seconds") as 'window)
         .agg(count("*") as 'count)
@@ -224,7 +226,7 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche
     // All event time metrics where watermarking is set
     val inputData = MemoryStream[Int]
     val aggWithWatermark = inputData.toDF()
-        .withColumn("eventTime", $"value".cast("timestamp"))
+        .withColumn("eventTime", timestamp_seconds($"value"))
         .withWatermark("eventTime", "10 seconds")
         .groupBy(window($"eventTime", "5 seconds") as 'window)
         .agg(count("*") as 'count)
@@ -286,7 +288,7 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche
     val inputData = MemoryStream[Int]
 
     val windowedAggregation = inputData.toDF()
-      .withColumn("eventTime", $"value".cast("timestamp"))
+      .withColumn("eventTime", timestamp_seconds($"value"))
       .withWatermark("eventTime", "10 seconds")
       .groupBy(window($"eventTime", "5 seconds") as 'window)
       .agg(count("*") as 'count)
@@ -298,9 +300,11 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche
       AddData(inputData, 25),   // Advance watermark to 15 seconds
       CheckNewAnswer((10, 5)),
       assertNumStateRows(2),
+      assertNumRowsDroppedByWatermark(0),
       AddData(inputData, 10),   // Should not emit anything as data less than watermark
       CheckNewAnswer(),
-      assertNumStateRows(2)
+      assertNumStateRows(2),
+      assertNumRowsDroppedByWatermark(1)
     )
   }
 
@@ -309,7 +313,7 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche
     spark.conf.set(SQLConf.SHUFFLE_PARTITIONS.key, "10")
 
     val windowedAggregation = inputData.toDF()
-      .withColumn("eventTime", $"value".cast("timestamp"))
+      .withColumn("eventTime", timestamp_seconds($"value"))
       .withWatermark("eventTime", "10 seconds")
       .groupBy(window($"eventTime", "5 seconds") as 'window)
       .agg(count("*") as 'count)
@@ -321,12 +325,15 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche
       AddData(inputData, 25),     // Advance watermark to 15 seconds
       CheckNewAnswer((25, 1)),
       assertNumStateRows(2),
+      assertNumRowsDroppedByWatermark(0),
       AddData(inputData, 10, 25), // Ignore 10 as its less than watermark
       CheckNewAnswer((25, 2)),
       assertNumStateRows(2),
+      assertNumRowsDroppedByWatermark(1),
       AddData(inputData, 10),     // Should not emit anything as data less than watermark
       CheckNewAnswer(),
-      assertNumStateRows(2)
+      assertNumStateRows(2),
+      assertNumRowsDroppedByWatermark(1)
     )
   }
 
@@ -336,7 +343,7 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche
 
     val input = MemoryStream[Long]
     val aggWithWatermark = input.toDF()
-      .withColumn("eventTime", $"value".cast("timestamp"))
+      .withColumn("eventTime", timestamp_seconds($"value"))
       .withWatermark("eventTime", "2 years 5 months")
       .groupBy(window($"eventTime", "5 seconds") as 'window)
       .agg(count("*") as 'count)
@@ -368,7 +375,7 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche
   test("recovery") {
     val inputData = MemoryStream[Int]
     val df = inputData.toDF()
-      .withColumn("eventTime", $"value".cast("timestamp"))
+      .withColumn("eventTime", timestamp_seconds($"value"))
       .withWatermark("eventTime", "10 seconds")
       .groupBy(window($"eventTime", "5 seconds") as 'window)
       .agg(count("*") as 'count)
@@ -403,14 +410,14 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche
     val first = MemoryStream[Int]
 
     val firstDf = first.toDF()
-      .withColumn("eventTime", $"value".cast("timestamp"))
+      .withColumn("eventTime", timestamp_seconds($"value"))
       .withWatermark("eventTime", "10 seconds")
       .select('value)
 
     val second = MemoryStream[Int]
 
     val secondDf = second.toDF()
-      .withColumn("eventTime", $"value".cast("timestamp"))
+      .withColumn("eventTime", timestamp_seconds($"value"))
       .withWatermark("eventTime", "5 seconds")
       .select('value)
 
@@ -480,7 +487,7 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche
     val inputData = MemoryStream[Int]
 
     val windowedAggregation = inputData.toDF()
-        .withColumn("eventTime", $"value".cast("timestamp"))
+        .withColumn("eventTime", timestamp_seconds($"value"))
         .withWatermark("eventTime", "10 seconds")
         .groupBy(window($"eventTime", "5 seconds") as 'window)
         .agg(count("*") as 'count)
@@ -505,7 +512,7 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche
     val inputData = MemoryStream[Int]
 
     val windowedAggregation = inputData.toDF()
-        .withColumn("eventTime", $"value".cast("timestamp"))
+        .withColumn("eventTime", timestamp_seconds($"value"))
         .withWatermark("eventTime", "10 seconds")
         .groupBy($"eventTime")
         .agg(count("*") as 'count)
@@ -544,8 +551,8 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche
 
   test("the new watermark should override the old one") {
     val df = MemoryStream[(Long, Long)].toDF()
-      .withColumn("first", $"_1".cast("timestamp"))
-      .withColumn("second", $"_2".cast("timestamp"))
+      .withColumn("first", timestamp_seconds($"_1"))
+      .withColumn("second", timestamp_seconds($"_2"))
       .withWatermark("first", "1 minute")
       .withWatermark("second", "2 minutes")
 
@@ -557,7 +564,7 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche
 
   test("EventTime watermark should be ignored in batch query.") {
     val df = testData
-      .withColumn("eventTime", $"key".cast("timestamp"))
+      .withColumn("eventTime", timestamp_seconds($"key"))
       .withWatermark("eventTime", "1 minute")
       .select("eventTime")
       .as[Long]
@@ -596,7 +603,7 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche
   test("SPARK-27340 Alias on TimeWindow expression cause watermark metadata lost") {
     val inputData = MemoryStream[Int]
     val aliasWindow = inputData.toDF()
-      .withColumn("eventTime", $"value".cast("timestamp"))
+      .withColumn("eventTime", timestamp_seconds($"value"))
       .withWatermark("eventTime", "10 seconds")
       .select(window($"eventTime", "5 seconds") as 'aliasWindow)
     // Check the eventTime metadata is kept in the top level alias.
@@ -626,7 +633,7 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche
     def testWithFlag(flag: Boolean): Unit = withClue(s"with $flagKey = $flag") {
       val inputData = MemoryStream[Int]
       val windowedAggregation = inputData.toDF()
-        .withColumn("eventTime", $"value".cast("timestamp"))
+        .withColumn("eventTime", timestamp_seconds($"value"))
         .withWatermark("eventTime", "10 seconds")
         .groupBy(window($"eventTime", "5 seconds") as 'window)
         .agg(count("*") as 'count)
@@ -762,10 +769,10 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche
       input1: MemoryStream[Int],
       input2: MemoryStream[Int]): Dataset[_] = {
     val df1 = input1.toDF
-      .withColumn("eventTime", $"value".cast("timestamp"))
+      .withColumn("eventTime", timestamp_seconds($"value"))
       .withWatermark("eventTime", "10 seconds")
     val df2 = input2.toDF
-      .withColumn("eventTime", $"value".cast("timestamp"))
+      .withColumn("eventTime", timestamp_seconds($"value"))
       .withWatermark("eventTime", "15 seconds")
     df1.union(df2).select($"eventTime".cast("int"))
   }
@@ -783,6 +790,20 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche
     true
   }
 
+  private def assertNumRowsDroppedByWatermark(
+      numRowsDroppedByWatermark: Long): AssertOnQuery = AssertOnQuery { q =>
+    q.processAllAvailable()
+    val progressWithData = q.recentProgress.filterNot { p =>
+      // filter out batches which are falling into one of types:
+      // 1) doesn't execute the batch run
+      // 2) empty input batch
+      p.inputRowsPerSecond == 0
+    }.lastOption.get
+    assert(progressWithData.stateOperators(0).numRowsDroppedByWatermark
+      === numRowsDroppedByWatermark)
+    true
+  }
+
   /** Assert event stats generated on that last batch with data in it */
   private def assertEventStats(body: ju.Map[String, String] => Unit): AssertOnQuery = {
     Execute("AssertEventStats") { q =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala
index 877965100f018..53ef832063230 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala
@@ -17,17 +17,18 @@
 
 package org.apache.spark.sql.streaming
 
-import java.io.File
+import java.io.{File, IOException}
 import java.nio.file.Files
 import java.util.Locale
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable.ArrayBuffer
 
-import org.apache.hadoop.fs.Path
+import org.apache.hadoop.fs.{FileStatus, Path, RawLocalFileSystem}
 import org.apache.hadoop.mapreduce.JobContext
 
 import org.apache.spark.SparkConf
+import org.apache.spark.deploy.SparkHadoopUtil
 import org.apache.spark.internal.io.FileCommitProtocol
 import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd}
 import org.apache.spark.sql.{AnalysisException, DataFrame}
@@ -210,7 +211,7 @@ abstract class FileStreamSinkSuite extends StreamTest {
     val inputData = MemoryStream[Long]
     val inputDF = inputData.toDF.toDF("time")
     val outputDf = inputDF
-      .selectExpr("CAST(time AS timestamp) AS timestamp")
+      .selectExpr("timestamp_seconds(time) AS timestamp")
       .withWatermark("timestamp", "10 seconds")
       .groupBy(window($"timestamp", "5 seconds"))
       .count()
@@ -555,10 +556,12 @@ abstract class FileStreamSinkSuite extends StreamTest {
             }
           }
 
-          val fs = new Path(outputDir.getCanonicalPath).getFileSystem(
-            spark.sessionState.newHadoopConf())
-          val sinkLog = new FileStreamSinkLog(FileStreamSinkLog.VERSION, spark,
-            outputDir.getCanonicalPath)
+          val outputDirPath = new Path(outputDir.getCanonicalPath)
+          val hadoopConf = spark.sessionState.newHadoopConf()
+          val fs = outputDirPath.getFileSystem(hadoopConf)
+          val logPath = FileStreamSink.getMetadataLogPath(fs, outputDirPath, conf)
+
+          val sinkLog = new FileStreamSinkLog(FileStreamSinkLog.VERSION, spark, logPath.toString)
 
           val allFiles = sinkLog.allFiles()
           // only files from non-empty partition should be logged
@@ -573,6 +576,30 @@ abstract class FileStreamSinkSuite extends StreamTest {
       }
     }
   }
+
+  test("formatCheck fail should not fail the query") {
+    withSQLConf(
+      "fs.file.impl" -> classOf[FailFormatCheckFileSystem].getName,
+      "fs.file.impl.disable.cache" -> "true") {
+      withTempDir { tempDir =>
+        val path = new File(tempDir, "text").getCanonicalPath
+        Seq("foo").toDF.write.format("text").save(path)
+        spark.read.format("text").load(path)
+      }
+    }
+  }
+
+  test("fail to check glob path should not fail the query") {
+    withSQLConf(
+      "fs.file.impl" -> classOf[FailFormatCheckFileSystem].getName,
+      "fs.file.impl.disable.cache" -> "true") {
+      withTempDir { tempDir =>
+        val path = new File(tempDir, "text").getCanonicalPath
+        Seq("foo").toDF.write.format("text").save(path)
+        spark.read.format("text").load(path + "/*")
+      }
+    }
+  }
 }
 
 object PendingCommitFilesTrackingManifestFileCommitProtocol {
@@ -683,3 +710,19 @@ class FileStreamSinkV2Suite extends FileStreamSinkSuite {
     // TODO: test partition pruning when file source V2 supports it.
   }
 }
+
+/**
+ * A special file system that fails when accessing metadata log directory or using a glob path to
+ * access.
+ */
+class FailFormatCheckFileSystem extends RawLocalFileSystem {
+  override def getFileStatus(f: Path): FileStatus = {
+    if (f.getName == FileStreamSink.metadataDir) {
+      throw new IOException("cannot access metadata log")
+    }
+    if (SparkHadoopUtil.get.isGlobPath(f)) {
+      throw new IOException("fail to access a glob path")
+    }
+    super.getFileStatus(f)
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala
index fa320333143ec..ff00c474e2ef0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala
@@ -19,6 +19,9 @@ package org.apache.spark.sql.streaming
 
 import java.io.File
 import java.net.URI
+import java.time.{LocalDateTime, ZoneOffset}
+import java.time.format.DateTimeFormatter
+import java.util.concurrent.atomic.AtomicLong
 
 import scala.collection.mutable
 import scala.util.Random
@@ -32,14 +35,13 @@ import org.scalatest.time.SpanSugar._
 import org.apache.spark.deploy.SparkHadoopUtil
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.connector.read.streaming.ReadLimit
 import org.apache.spark.sql.execution.streaming._
 import org.apache.spark.sql.execution.streaming.FileStreamSource.{FileEntry, SeenFilesMap, SourceFileArchiver}
 import org.apache.spark.sql.execution.streaming.sources.MemorySink
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.streaming.ExistsThrowsExceptionFileSystem._
 import org.apache.spark.sql.streaming.util.StreamManualClock
 import org.apache.spark.sql.test.SharedSparkSession
-import org.apache.spark.sql.types._
 import org.apache.spark.sql.types.{StructType, _}
 import org.apache.spark.util.Utils
 
@@ -411,7 +413,7 @@ class FileStreamSourceSuite extends FileStreamSourceTest {
           createFileStreamSourceAndGetSchema(
             format = Some("json"), path = Some(src.getCanonicalPath), schema = None)
         }
-        assert("Unable to infer schema for JSON. It must be specified manually.;" === e.getMessage)
+        assert("Unable to infer schema for JSON. It must be specified manually." === e.getMessage)
       }
     }
   }
@@ -532,6 +534,18 @@ class FileStreamSourceSuite extends FileStreamSourceTest {
     }
   }
 
+  test("SPARK-31935: Hadoop file system config should be effective in data source options") {
+    withTempDir { dir =>
+      val path = dir.getCanonicalPath
+      val defaultFs = "nonexistFS://nonexistFS"
+      val expectMessage = "No FileSystem for scheme nonexistFS"
+      val message = intercept[java.io.IOException] {
+        spark.readStream.option("fs.defaultFS", defaultFs).text(path)
+      }.getMessage
+      assert(message.filterNot(Set(':', '"').contains) == expectMessage)
+    }
+  }
+
   test("read from textfile") {
     withTempDirs { case (src, tmp) =>
       val textStream = spark.readStream.textFile(src.getCanonicalPath)
@@ -997,15 +1011,6 @@ class FileStreamSourceSuite extends FileStreamSourceTest {
   }
 
   test("when schema inference is turned on, should read partition data") {
-    def createFile(content: String, src: File, tmp: File): Unit = {
-      val tempFile = Utils.tempFileWith(new File(tmp, "text"))
-      val finalFile = new File(src, tempFile.getName)
-      require(!src.exists(), s"$src exists, dir: ${src.isDirectory}, file: ${src.isFile}")
-      require(src.mkdirs(), s"Cannot create $src")
-      require(src.isDirectory(), s"$src is not a directory")
-      require(stringToFile(tempFile, content).renameTo(finalFile))
-    }
-
     withSQLConf(SQLConf.STREAMING_SCHEMA_INFERENCE.key -> "true") {
       withTempDirs { case (dir, tmp) =>
         val partitionFooSubDir = new File(dir, "partition=foo")
@@ -1371,6 +1376,70 @@ class FileStreamSourceSuite extends FileStreamSourceTest {
     }
   }
 
+  test("restore from file stream source log") {
+    def createEntries(batchId: Long, count: Int): Array[FileEntry] = {
+      (1 to count).map { idx =>
+        FileEntry(s"path_${batchId}_$idx", 10000 * batchId + count, batchId)
+      }.toArray
+    }
+
+    withSQLConf(SQLConf.FILE_SOURCE_LOG_COMPACT_INTERVAL.key -> "5") {
+      def verifyBatchAvailabilityInCache(
+          fileEntryCache: java.util.LinkedHashMap[Long, Array[FileEntry]],
+          expectNotAvailable: Seq[Int],
+          expectAvailable: Seq[Int]): Unit = {
+        expectNotAvailable.foreach { batchId =>
+          assert(!fileEntryCache.containsKey(batchId.toLong))
+        }
+        expectAvailable.foreach { batchId =>
+          assert(fileEntryCache.containsKey(batchId.toLong))
+        }
+      }
+      withTempDir { chk =>
+        val _fileEntryCache = PrivateMethod[java.util.LinkedHashMap[Long, Array[FileEntry]]](
+          Symbol("fileEntryCache"))
+
+        val metadata = new FileStreamSourceLog(FileStreamSourceLog.VERSION, spark,
+          chk.getCanonicalPath)
+        val fileEntryCache = metadata invokePrivate _fileEntryCache()
+
+        (0 to 4).foreach { batchId =>
+          metadata.add(batchId, createEntries(batchId, 100))
+        }
+        val allFiles = metadata.allFiles()
+
+        // batch 4 is a compact batch which logs would be cached in fileEntryCache
+        verifyBatchAvailabilityInCache(fileEntryCache, Seq(0, 1, 2, 3), Seq(4))
+
+        val metadata2 = new FileStreamSourceLog(FileStreamSourceLog.VERSION, spark,
+          chk.getCanonicalPath)
+        val fileEntryCache2 = metadata2 invokePrivate _fileEntryCache()
+
+        // allFiles() doesn't restore the logs for the latest compact batch into file entry cache
+        assert(metadata2.allFiles() === allFiles)
+        verifyBatchAvailabilityInCache(fileEntryCache2, Seq(0, 1, 2, 3, 4), Seq.empty)
+
+        // restore() will restore the logs for the latest compact batch into file entry cache
+        assert(metadata2.restore() === allFiles)
+        verifyBatchAvailabilityInCache(fileEntryCache2, Seq(0, 1, 2, 3), Seq(4))
+
+        (5 to 5 + FileStreamSourceLog.PREV_NUM_BATCHES_TO_READ_IN_RESTORE).foreach { batchId =>
+          metadata2.add(batchId, createEntries(batchId, 100))
+        }
+
+        val metadata3 = new FileStreamSourceLog(FileStreamSourceLog.VERSION, spark,
+          chk.getCanonicalPath)
+        val fileEntryCache3 = metadata3 invokePrivate _fileEntryCache()
+
+        // restore() will not restore the logs for the latest compact batch into file entry cache
+        // if the latest batch is too far from latest compact batch, because it's unlikely Spark
+        // will request the batch for the start point.
+        assert(metadata3.restore() === metadata2.allFiles())
+        verifyBatchAvailabilityInCache(fileEntryCache3, Seq(0, 1, 2, 3, 4), Seq.empty)
+      }
+    }
+  }
+
   test("get arbitrary batch from FileStreamSource") {
     withTempDirs { case (src, tmp) =>
       withSQLConf(
@@ -1463,8 +1532,10 @@ class FileStreamSourceSuite extends FileStreamSourceTest {
 
   private def readOffsetFromResource(file: String): SerializedOffset = {
     import scala.io.Source
-    val str = Source.fromFile(getClass.getResource(s"/structured-streaming/$file").toURI).mkString
-    SerializedOffset(str.trim)
+    Utils.tryWithResource(
+      Source.fromFile(getClass.getResource(s"/structured-streaming/$file").toURI)) { source =>
+      SerializedOffset(source.mkString.trim)
+    }
   }
 
   private def runTwoBatchesAndVerifyResults(
@@ -1602,6 +1673,7 @@ class FileStreamSourceSuite extends FileStreamSourceTest {
   }
 
   test("do not recheck that files exist during getBatch") {
+    val scheme = ExistsThrowsExceptionFileSystem.scheme
     withTempDir { temp =>
       spark.conf.set(
         s"fs.$scheme.impl",
@@ -1876,9 +1948,9 @@ class FileStreamSourceSuite extends FileStreamSourceTest {
   test("SourceFileArchiver - fail when base archive path matches source pattern") {
     val fakeFileSystem = new FakeFileSystem("fake")
 
-    def assertThrowIllegalArgumentException(sourcePatttern: Path, baseArchivePath: Path): Unit = {
+    def assertThrowIllegalArgumentException(sourcePattern: Path, baseArchivePath: Path): Unit = {
       intercept[IllegalArgumentException] {
-        new SourceFileArchiver(fakeFileSystem, sourcePatttern, fakeFileSystem, baseArchivePath)
+        new SourceFileArchiver(fakeFileSystem, sourcePattern, fakeFileSystem, baseArchivePath)
       }
     }
 
@@ -1935,6 +2007,170 @@ class FileStreamSourceSuite extends FileStreamSourceTest {
     assert(expectedDir.exists())
     assert(expectedDir.list().exists(_.startsWith(filePrefix)))
   }
+
+  private def withCountListingLocalFileSystemAsLocalFileSystem(body: => Unit): Unit = {
+    val optionKey = s"fs.${CountListingLocalFileSystem.scheme}.impl"
+    val originClassForLocalFileSystem = spark.conf.getOption(optionKey)
+    try {
+      spark.conf.set(optionKey, classOf[CountListingLocalFileSystem].getName)
+      body
+    } finally {
+      originClassForLocalFileSystem match {
+        case Some(fsClazz) => spark.conf.set(optionKey, fsClazz)
+        case _ => spark.conf.unset(optionKey)
+      }
+    }
+  }
+
+  test("Caches and leverages unread files") {
+    withCountListingLocalFileSystemAsLocalFileSystem {
+      withThreeTempDirs { case (src, meta, tmp) =>
+        val options = Map("latestFirst" -> "false", "maxFilesPerTrigger" -> "10")
+        val scheme = CountListingLocalFileSystem.scheme
+        val source = new FileStreamSource(spark, s"$scheme:///${src.getCanonicalPath}/*/*", "text",
+          StructType(Nil), Seq.empty, meta.getCanonicalPath, options)
+        val _metadataLog = PrivateMethod[FileStreamSourceLog](Symbol("metadataLog"))
+        val metadataLog = source invokePrivate _metadataLog()
+
+        def verifyBatch(
+            offset: FileStreamSourceOffset,
+            expectedBatchId: Long,
+            inputFiles: Seq[File],
+            expectedListingCount: Int): Unit = {
+          val batchId = offset.logOffset
+          assert(batchId === expectedBatchId)
+
+          val files = metadataLog.get(batchId).getOrElse(Array.empty[FileEntry])
+          assert(files.forall(_.batchId == batchId))
+
+          val actualInputFiles = files.map { p => new Path(p.path).toUri.getPath }
+          val expectedInputFiles = inputFiles.slice(batchId.toInt * 10, batchId.toInt * 10 + 10)
+            .map(_.getCanonicalPath)
+          assert(actualInputFiles === expectedInputFiles)
+
+          assert(expectedListingCount === CountListingLocalFileSystem.pathToNumListStatusCalled
+            .get(src.getCanonicalPath).map(_.get()).getOrElse(0))
+        }
+
+        CountListingLocalFileSystem.resetCount()
+
+        // provide 41 files in src, with sequential "last modified" to guarantee ordering
+        val inputFiles = (0 to 40).map { idx =>
+          val f = createFile(idx.toString, new File(src, idx.toString), tmp)
+          f.setLastModified(idx * 10000)
+          f
+        }
+
+        // 4 batches will be available for 40 input files
+        (0 to 3).foreach { batchId =>
+          val offsetBatch = source.latestOffset(FileStreamSourceOffset(-1L), ReadLimit.maxFiles(10))
+            .asInstanceOf[FileStreamSourceOffset]
+          verifyBatch(offsetBatch, expectedBatchId = batchId, inputFiles, expectedListingCount = 1)
+        }
+
+        // batch 5 will trigger list operation though the batch 4 should have 1 unseen file:
+        // 1 is smaller than the threshold (refer FileStreamSource.DISCARD_UNSEEN_FILES_RATIO),
+        // hence unseen files for batch 4 will be discarded.
+        val offsetBatch = source.latestOffset(FileStreamSourceOffset(-1L), ReadLimit.maxFiles(10))
+          .asInstanceOf[FileStreamSourceOffset]
+        assert(4 === offsetBatch.logOffset)
+        assert(2 === CountListingLocalFileSystem.pathToNumListStatusCalled
+          .get(src.getCanonicalPath).map(_.get()).getOrElse(0))
+
+        val offsetBatch2 = source.latestOffset(FileStreamSourceOffset(-1L), ReadLimit.maxFiles(10))
+          .asInstanceOf[FileStreamSourceOffset]
+        // latestOffset returns the offset for previous batch which means no new batch is presented
+        assert(4 === offsetBatch2.logOffset)
+        // listing should be performed after the list of unread files are exhausted
+        assert(3 === CountListingLocalFileSystem.pathToNumListStatusCalled
+          .get(src.getCanonicalPath).map(_.get()).getOrElse(0))
+      }
+    }
+  }
+
+  test("Don't cache unread files when latestFirst is true") {
+    withCountListingLocalFileSystemAsLocalFileSystem {
+      withThreeTempDirs { case (src, meta, tmp) =>
+        val options = Map("latestFirst" -> "true", "maxFilesPerTrigger" -> "5")
+        val scheme = CountListingLocalFileSystem.scheme
+        val source = new FileStreamSource(spark, s"$scheme:///${src.getCanonicalPath}/*/*", "text",
+          StructType(Nil), Seq.empty, meta.getCanonicalPath, options)
+
+        CountListingLocalFileSystem.resetCount()
+
+        // provide 20 files in src, with sequential "last modified" to guarantee ordering
+        (0 to 19).map { idx =>
+          val f = createFile(idx.toString, new File(src, idx.toString), tmp)
+          f.setLastModified(idx * 10000)
+          f
+        }
+
+        source.latestOffset(FileStreamSourceOffset(-1L), ReadLimit.maxFiles(5))
+          .asInstanceOf[FileStreamSourceOffset]
+        assert(1 === CountListingLocalFileSystem.pathToNumListStatusCalled
+          .get(src.getCanonicalPath).map(_.get()).getOrElse(0))
+
+        // Even though the first batch doesn't read all available files, since latestFirst is true,
+        // file stream source will not leverage unread files - next batch will also trigger
+        // listing files
+        source.latestOffset(FileStreamSourceOffset(-1L), ReadLimit.maxFiles(5))
+          .asInstanceOf[FileStreamSourceOffset]
+        assert(2 === CountListingLocalFileSystem.pathToNumListStatusCalled
+          .get(src.getCanonicalPath).map(_.get()).getOrElse(0))
+      }
+    }
+  }
+
+  test("SPARK-31962: file stream source shouldn't allow modifiedBefore/modifiedAfter") {
+    def formatTime(time: LocalDateTime): String = {
+      time.format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss"))
+    }
+
+    def assertOptionIsNotSupported(options: Map[String, String], path: String): Unit = {
+      val schema = StructType(Seq(StructField("a", StringType)))
+      var dsReader = spark.readStream
+        .format("csv")
+        .option("timeZone", "UTC")
+        .schema(schema)
+
+      options.foreach { case (k, v) => dsReader = dsReader.option(k, v) }
+
+      val df = dsReader.load(path)
+
+      testStream(df)(
+        ExpectFailure[IllegalArgumentException](
+          t => assert(t.getMessage.contains("is not allowed in file stream source")),
+          isFatalError = false)
+      )
+    }
+
+    withTempDir { dir =>
+      // "modifiedBefore"
+      val futureTime = LocalDateTime.now(ZoneOffset.UTC).plusYears(1)
+      val formattedFutureTime = formatTime(futureTime)
+      assertOptionIsNotSupported(Map("modifiedBefore" -> formattedFutureTime), dir.getCanonicalPath)
+
+      // "modifiedAfter"
+      val prevTime = LocalDateTime.now(ZoneOffset.UTC).minusYears(1)
+      val formattedPrevTime = formatTime(prevTime)
+      assertOptionIsNotSupported(Map("modifiedAfter" -> formattedPrevTime), dir.getCanonicalPath)
+
+      // both
+      assertOptionIsNotSupported(
+        Map("modifiedBefore" -> formattedFutureTime, "modifiedAfter" -> formattedPrevTime),
+        dir.getCanonicalPath)
+    }
+  }
+
+  private def createFile(content: String, src: File, tmp: File): File = {
+    val tempFile = Utils.tempFileWith(new File(tmp, "text"))
+    val finalFile = new File(src, tempFile.getName)
+    require(!src.exists(), s"$src exists, dir: ${src.isDirectory}, file: ${src.isFile}")
+    require(src.mkdirs(), s"Cannot create $src")
+    require(src.isDirectory(), s"$src is not a directory")
+    require(stringToFile(tempFile, content).renameTo(finalFile))
+    finalFile
+  }
 }
 
 class FileStreamSourceStressTestSuite extends FileStreamSourceTest {
@@ -1961,6 +2197,8 @@ class FileStreamSourceStressTestSuite extends FileStreamSourceTest {
  * `DataSource.resolveRelation`.
  */
 class ExistsThrowsExceptionFileSystem extends RawLocalFileSystem {
+  import ExistsThrowsExceptionFileSystem._
+
   override def getUri: URI = {
     URI.create(s"$scheme:///")
   }
@@ -1980,3 +2218,24 @@ class ExistsThrowsExceptionFileSystem extends RawLocalFileSystem {
 object ExistsThrowsExceptionFileSystem {
   val scheme = s"FileStreamSourceSuite${math.abs(Random.nextInt)}fs"
 }
+
+class CountListingLocalFileSystem extends RawLocalFileSystem {
+  import CountListingLocalFileSystem._
+
+  override def getUri: URI = {
+    URI.create(s"$scheme:///")
+  }
+
+  override def listStatus(f: Path): Array[FileStatus] = {
+    val curVal = pathToNumListStatusCalled.getOrElseUpdate(f.toUri.getPath, new AtomicLong(0))
+    curVal.incrementAndGet()
+    super.listStatus(f)
+  }
+}
+
+object CountListingLocalFileSystem {
+  val scheme = s"CountListingLocalFileSystem${math.abs(Random.nextInt)}fs"
+  val pathToNumListStatusCalled = new mutable.HashMap[String, AtomicLong]
+
+  def resetCount(): Unit = pathToNumListStatusCalled.clear()
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala
index b04f8b0d4d174..788be539fe073 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala
@@ -21,12 +21,11 @@ import java.io.File
 import java.sql.Date
 
 import org.apache.commons.io.FileUtils
-import org.scalatest.BeforeAndAfterAll
 import org.scalatest.exceptions.TestFailedException
 
 import org.apache.spark.SparkException
 import org.apache.spark.api.java.function.FlatMapGroupsWithStateFunction
-import org.apache.spark.sql.Encoder
+import org.apache.spark.sql.{DataFrame, Encoder}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{GenericInternalRow, UnsafeProjection, UnsafeRow}
 import org.apache.spark.sql.catalyst.plans.logical.FlatMapGroupsWithState
@@ -34,7 +33,8 @@ import org.apache.spark.sql.catalyst.plans.physical.UnknownPartitioning
 import org.apache.spark.sql.catalyst.streaming.InternalOutputModes._
 import org.apache.spark.sql.execution.RDDScanExec
 import org.apache.spark.sql.execution.streaming._
-import org.apache.spark.sql.execution.streaming.state.{FlatMapGroupsWithStateExecHelper, MemoryStateStore, StateStore, StateStoreId, StateStoreMetrics, UnsafeRowPair}
+import org.apache.spark.sql.execution.streaming.state.{FlatMapGroupsWithStateExecHelper, MemoryStateStore, StateStore}
+import org.apache.spark.sql.functions.timestamp_seconds
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.streaming.util.StreamManualClock
 import org.apache.spark.sql.types.{DataType, IntegerType}
@@ -826,7 +826,7 @@ class FlatMapGroupsWithStateSuite extends StateStoreMetricsTest {
     val inputData = MemoryStream[(String, Int)]
     val result =
       inputData.toDS
-        .select($"_1".as("key"), $"_2".cast("timestamp").as("eventTime"))
+        .select($"_1".as("key"), timestamp_seconds($"_2").as("eventTime"))
         .withWatermark("eventTime", "10 seconds")
         .as[(String, Long)]
         .groupByKey(_._1)
@@ -901,7 +901,7 @@ class FlatMapGroupsWithStateSuite extends StateStoreMetricsTest {
     val inputData = MemoryStream[(String, Int)]
     val result =
       inputData.toDS
-        .select($"_1".as("key"), $"_2".cast("timestamp").as("eventTime"))
+        .select($"_1".as("key"), timestamp_seconds($"_2").as("eventTime"))
         .withWatermark("eventTime", "10 seconds")
         .as[(String, Long)]
         .groupByKey(_._1)
@@ -1019,6 +1019,56 @@ class FlatMapGroupsWithStateSuite extends StateStoreMetricsTest {
       spark.createDataset(Seq(("a", 2), ("b", 1))).toDF)
   }
 
+  testWithAllStateVersions("SPARK-29438: ensure UNION doesn't lead (flat)MapGroupsWithState" +
+    " to use shifted partition IDs") {
+    val stateFunc = (key: String, values: Iterator[String], state: GroupState[RunningCount]) => {
+      val count = state.getOption.map(_.count).getOrElse(0L) + values.size
+      state.update(RunningCount(count))
+      (key, count.toString)
+    }
+
+    def constructUnionDf(desiredPartitionsForInput1: Int)
+      : (MemoryStream[String], MemoryStream[String], DataFrame) = {
+      val input1 = MemoryStream[String](desiredPartitionsForInput1)
+      val input2 = MemoryStream[String]
+      val df1 = input1.toDF()
+        .select($"value", $"value")
+      val df2 = input2.toDS()
+        .groupByKey(x => x)
+        .mapGroupsWithState(stateFunc) // Types = State: MyState, Out: (Str, Str)
+        .toDF()
+
+      // Unioned DF would have columns as (String, String)
+      (input1, input2, df1.union(df2))
+    }
+
+    withTempDir { checkpointDir =>
+      val (input1, input2, unionDf) = constructUnionDf(2)
+      testStream(unionDf, Update)(
+        StartStream(checkpointLocation = checkpointDir.getAbsolutePath),
+        MultiAddData(input1, "input1-a")(input2, "input2-a"),
+        CheckNewAnswer(("input1-a", "input1-a"), ("input2-a", "1")),
+        StopStream
+      )
+
+      // We're restoring the query with different number of partitions in left side of UNION,
+      // which may lead right side of union to have mismatched partition IDs (e.g. if it relies on
+      // TaskContext.partitionId()). This test will verify (flat)MapGroupsWithState doesn't have
+      // such issue.
+
+      val (newInput1, newInput2, newUnionDf) = constructUnionDf(3)
+
+      newInput1.addData("input1-a")
+      newInput2.addData("input2-a")
+
+      testStream(newUnionDf, Update)(
+        StartStream(checkpointLocation = checkpointDir.getAbsolutePath),
+        MultiAddData(newInput1, "input1-a")(newInput2, "input2-a", "input2-b"),
+        CheckNewAnswer(("input1-a", "input1-a"), ("input2-a", "2"), ("input2-b", "1"))
+      )
+    }
+  }
+
   testQuietly("StateStore.abort on task failure handling") {
     val stateFunc = (key: String, values: Iterator[String], state: GroupState[RunningCount]) => {
       if (FlatMapGroupsWithStateSuite.failInTask) throw new Exception("expected failure")
@@ -1111,7 +1161,7 @@ class FlatMapGroupsWithStateSuite extends StateStoreMetricsTest {
       val inputData = MemoryStream[(String, Long)]
       val result =
         inputData.toDF().toDF("key", "time")
-          .selectExpr("key", "cast(time as timestamp) as timestamp")
+          .selectExpr("key", "timestamp_seconds(time) as timestamp")
           .withWatermark("timestamp", "10 second")
           .as[(String, Long)]
           .groupByKey(x => x._1)
@@ -1273,7 +1323,9 @@ class FlatMapGroupsWithStateSuite extends StateStoreMetricsTest {
   def testWithAllStateVersions(name: String)(func: => Unit): Unit = {
     for (version <- FlatMapGroupsWithStateExecHelper.supportedVersions) {
       test(s"$name - state format version $version") {
-        withSQLConf(SQLConf.FLATMAPGROUPSWITHSTATE_STATE_FORMAT_VERSION.key -> version.toString) {
+        withSQLConf(
+            SQLConf.FLATMAPGROUPSWITHSTATE_STATE_FORMAT_VERSION.key -> version.toString,
+            SQLConf.STATEFUL_OPERATOR_CHECK_CORRECTNESS_ENABLED.key -> "false") {
           func
         }
       }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StateStoreMetricsTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StateStoreMetricsTest.scala
index fb5d13d09fb0e..be83f0ee776da 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StateStoreMetricsTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StateStoreMetricsTest.scala
@@ -29,8 +29,12 @@ trait StateStoreMetricsTest extends StreamTest {
     lastCheckedRecentProgressIndex = -1
   }
 
-  def assertNumStateRows(total: Seq[Long], updated: Seq[Long]): AssertOnQuery =
-    AssertOnQuery(s"Check total state rows = $total, updated state rows = $updated") { q =>
+  def assertNumStateRows(
+      total: Seq[Long],
+      updated: Seq[Long],
+      droppedByWatermark: Seq[Long]): AssertOnQuery =
+    AssertOnQuery(s"Check total state rows = $total, updated state rows = $updated" +
+      s", rows dropped by watermark = $droppedByWatermark") { q =>
       // This assumes that the streaming query will not make any progress while the eventually
       // is being executed.
       eventually(timeout(streamingTimeout)) {
@@ -51,6 +55,9 @@ trait StateStoreMetricsTest extends StreamTest {
         val allNumUpdatedRowsSinceLastCheck =
           progressesSinceLastCheck.map(_.stateOperators.map(_.numRowsUpdated))
 
+        val allNumRowsDroppedByWatermarkSinceLastCheck =
+          progressesSinceLastCheck.map(_.stateOperators.map(_.numRowsDroppedByWatermark))
+
         lazy val debugString = "recent progresses:\n" +
           progressesSinceLastCheck.map(_.prettyJson).mkString("\n\n")
 
@@ -60,13 +67,27 @@ trait StateStoreMetricsTest extends StreamTest {
         val numUpdatedRows = arraySum(allNumUpdatedRowsSinceLastCheck, numStateOperators)
         assert(numUpdatedRows === updated, s"incorrect updates rows, $debugString")
 
+        val numRowsDroppedByWatermark = arraySum(allNumRowsDroppedByWatermarkSinceLastCheck,
+          numStateOperators)
+        assert(numRowsDroppedByWatermark === droppedByWatermark,
+          s"incorrect dropped rows by watermark, $debugString")
+
         lastCheckedRecentProgressIndex = recentProgress.length - 1
       }
       true
     }
 
-  def assertNumStateRows(total: Long, updated: Long): AssertOnQuery =
-    assertNumStateRows(Seq(total), Seq(updated))
+  def assertNumStateRows(total: Seq[Long], updated: Seq[Long]): AssertOnQuery = {
+    assert(total.length === updated.length)
+    assertNumStateRows(total, updated, droppedByWatermark = (0 until total.length).map(_ => 0L))
+  }
+
+  def assertNumStateRows(
+      total: Long,
+      updated: Long,
+      droppedByWatermark: Long = 0): AssertOnQuery = {
+    assertNumStateRows(Seq(total), Seq(updated), Seq(droppedByWatermark))
+  }
 
   def arraySum(arraySeq: Seq[Array[Long]], arrayLength: Int): Seq[Long] = {
     if (arraySeq.isEmpty) return Seq.fill(arrayLength)(0L)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
index bbd0220a74f88..fb6922aca19a1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.streaming
 
 import java.io.{File, InterruptedIOException, IOException, UncheckedIOException}
 import java.nio.channels.ClosedByInterruptException
+import java.time.ZoneId
 import java.util.concurrent.{CountDownLatch, ExecutionException, TimeUnit}
 
 import scala.concurrent.TimeoutException
@@ -33,9 +34,8 @@ import org.scalatest.time.SpanSugar._
 import org.apache.spark.{SparkConf, SparkContext, TaskContext, TestUtils}
 import org.apache.spark.scheduler.{SparkListener, SparkListenerJobStart}
 import org.apache.spark.sql._
-import org.apache.spark.sql.catalyst.plans.logical.Range
-import org.apache.spark.sql.catalyst.streaming.InternalOutputModes
-import org.apache.spark.sql.catalyst.util.DateTimeConstants.MICROS_PER_MILLIS
+import org.apache.spark.sql.catalyst.plans.logical.{Range, RepartitionByExpression}
+import org.apache.spark.sql.catalyst.streaming.{InternalOutputModes, StreamingRelationV2}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.execution.{LocalLimitExec, SimpleMode, SparkPlan}
 import org.apache.spark.sql.execution.command.ExplainCommand
@@ -46,7 +46,7 @@ import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.sources.StreamSourceProvider
 import org.apache.spark.sql.streaming.util.{BlockOnStopSourceProvider, StreamManualClock}
-import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
+import org.apache.spark.sql.types.{IntegerType, LongType, StructField, StructType}
 import org.apache.spark.util.Utils
 
 class StreamSuite extends StreamTest {
@@ -214,7 +214,9 @@ class StreamSuite extends StreamTest {
             .start(outputDir.getAbsolutePath)
           try {
             query.processAllAvailable()
-            val outputDf = spark.read.parquet(outputDir.getAbsolutePath).as[Long]
+            // Parquet write page-level CRC checksums will change the file size and
+            // affect the data order when reading these files. Please see PARQUET-1746 for details.
+            val outputDf = spark.read.parquet(outputDir.getAbsolutePath).sort('a).as[Long]
             checkDataset[Long](outputDf, (0L to 10L).toArray: _*)
           } finally {
             query.stop()
@@ -1064,13 +1066,13 @@ class StreamSuite extends StreamTest {
   }
 
   test("SPARK-30657: streaming limit should not apply on limits on state subplans") {
-    val streanData = MemoryStream[Int]
-    val streamingDF = streanData.toDF().toDF("value")
+    val streamData = MemoryStream[Int]
+    val streamingDF = streamData.toDF().toDF("value")
     val staticDF = spark.createDataset(Seq(1)).toDF("value").orderBy("value")
     testStream(streamingDF.join(staticDF.limit(1), "value"))(
-      AddData(streanData, 1, 2, 3),
+      AddData(streamData, 1, 2, 3),
       CheckAnswer(Row(1)),
-      AddData(streanData, 1, 3, 5),
+      AddData(streamData, 1, 3, 5),
       CheckAnswer(Row(1), Row(1)))
   }
 
@@ -1130,11 +1132,11 @@ class StreamSuite extends StreamTest {
     verifyLocalLimit(inputDF.dropDuplicates().repartition(1).limit(1), expectStreamingLimit = false)
 
     // Should be LocalLimitExec in the first place, not from optimization of StreamingLocalLimitExec
-    val staticDF = spark.range(1).toDF("value").limit(1)
+    val staticDF = spark.range(2).toDF("value").limit(1)
     verifyLocalLimit(inputDF.toDF("value").join(staticDF, "value"), expectStreamingLimit = false)
 
     verifyLocalLimit(
-      inputDF.groupBy().count().limit(1),
+      inputDF.groupBy("value").count().limit(1),
       expectStreamingLimit = false,
       outputMode = OutputMode.Complete())
   }
@@ -1219,7 +1221,8 @@ class StreamSuite extends StreamTest {
     }
 
     var lastTimestamp = System.currentTimeMillis()
-    val currentDate = DateTimeUtils.microsToDays(DateTimeUtils.millisToMicros(lastTimestamp))
+    val currentDate = DateTimeUtils.microsToDays(
+      DateTimeUtils.millisToMicros(lastTimestamp), ZoneId.systemDefault)
     testStream(df) (
       AddData(input, 1),
       CheckLastBatch { rows: Seq[Row] =>
@@ -1263,10 +1266,41 @@ class StreamSuite extends StreamTest {
       }
     }
   }
+
+  test("SPARK-34482: correct active SparkSession for logicalPlan") {
+    withSQLConf(
+      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+      SQLConf.COALESCE_PARTITIONS_INITIAL_PARTITION_NUM.key -> "10") {
+      val df = spark.readStream.format(classOf[FakeDefaultSource].getName).load()
+      var query: StreamExecution = null
+      try {
+        query =
+          df.repartition($"a")
+            .writeStream
+            .format("memory")
+            .queryName("memory")
+            .start()
+            .asInstanceOf[StreamingQueryWrapper]
+            .streamingQuery
+        query.awaitInitialization(streamingTimeout.toMillis)
+        val plan = query.logicalPlan
+        val numPartition = plan
+          .find { _.isInstanceOf[RepartitionByExpression] }
+          .map(_.asInstanceOf[RepartitionByExpression].numPartitions)
+        // Before the fix of SPARK-34482, the numPartition is the value of
+        // `COALESCE_PARTITIONS_INITIAL_PARTITION_NUM`.
+        assert(numPartition.get === spark.sqlContext.conf.getConf(SQLConf.SHUFFLE_PARTITIONS))
+      } finally {
+        if (query != null) {
+          query.stop()
+        }
+      }
+    }
+  }
 }
 
 abstract class FakeSource extends StreamSourceProvider {
-  private val fakeSchema = StructType(StructField("a", IntegerType) :: Nil)
+  private val fakeSchema = StructType(StructField("a", LongType) :: Nil)
 
   override def sourceSchema(
       spark: SQLContext,
@@ -1288,7 +1322,7 @@ class FakeDefaultSource extends FakeSource {
     new Source {
       private var offset = -1L
 
-      override def schema: StructType = StructType(StructField("a", IntegerType) :: Nil)
+      override def schema: StructType = StructType(StructField("a", LongType) :: Nil)
 
       override def getOffset: Option[Offset] = {
         if (offset >= 10) {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala
index 8d5439534b513..624b630401f47 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.streaming
 
 import scala.collection.mutable
 import scala.collection.mutable.ArrayBuffer
-import scala.language.experimental.macros
 import scala.reflect.ClassTag
 import scala.util.Random
 import scala.util.control.NonFatal
@@ -36,6 +35,7 @@ import org.apache.spark.sql.{Dataset, Encoder, QueryTest, Row}
 import org.apache.spark.sql.catalyst.encoders.{encoderFor, ExpressionEncoder, RowEncoder}
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.plans.physical.AllTuples
+import org.apache.spark.sql.catalyst.streaming.StreamingRelationV2
 import org.apache.spark.sql.catalyst.util._
 import org.apache.spark.sql.connector.read.streaming.{Offset => OffsetV2, SparkDataStream}
 import org.apache.spark.sql.execution.datasources.v2.StreamingDataSourceV2Relation
@@ -305,6 +305,14 @@ trait StreamTest extends QueryTest with SharedSparkSession with TimeLimits with
     def apply(func: StreamExecution => Any): AssertOnQuery = apply("Execute")(func)
   }
 
+  /** Call `StreamingQuery.processAllAvailable()` to wait. */
+  object ProcessAllAvailable {
+    def apply(): AssertOnQuery = AssertOnQuery { query =>
+      query.processAllAvailable()
+      true
+    }
+  }
+
   object AwaitEpoch {
     def apply(epoch: Long): AssertOnQuery =
       Execute {
@@ -873,7 +881,7 @@ trait StreamTest extends QueryTest with SharedSparkSession with TimeLimits with
     }
     if(!running) { actions += StartStream() }
     addCheck()
-    testStream(ds)(actions: _*)
+    testStream(ds)(actions.toSeq: _*)
   }
 
   object AwaitTerminationTester {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala
index 85e1b85b84d26..491b0d8b2c26c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala
@@ -20,14 +20,14 @@ package org.apache.spark.sql.streaming
 import java.io.File
 import java.util.{Locale, TimeZone}
 
-import scala.collection.mutable
+import scala.annotation.tailrec
 
 import org.apache.commons.io.FileUtils
 import org.scalatest.Assertions
 
 import org.apache.spark.{SparkEnv, SparkException}
 import org.apache.spark.rdd.BlockRDD
-import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset, SparkSession}
+import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset, Row, SparkSession}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.plans.logical.Aggregate
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
@@ -35,7 +35,7 @@ import org.apache.spark.sql.execution.{SparkPlan, UnaryExecNode}
 import org.apache.spark.sql.execution.exchange.Exchange
 import org.apache.spark.sql.execution.streaming._
 import org.apache.spark.sql.execution.streaming.sources.MemorySink
-import org.apache.spark.sql.execution.streaming.state.StreamingAggregationStateManager
+import org.apache.spark.sql.execution.streaming.state.{StateSchemaNotCompatible, StateStore, StreamingAggregationStateManager}
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.streaming.OutputMode._
@@ -188,7 +188,7 @@ class StreamingAggregationSuite extends StateStoreMetricsTest with Assertions {
   testWithAllStateVersions("state metrics - append mode") {
     val inputData = MemoryStream[Int]
     val aggWithWatermark = inputData.toDF()
-      .withColumn("eventTime", $"value".cast("timestamp"))
+      .withColumn("eventTime", timestamp_seconds($"value"))
       .withWatermark("eventTime", "10 seconds")
       .groupBy(window($"eventTime", "5 seconds") as 'window)
       .agg(count("*") as 'count)
@@ -337,6 +337,49 @@ class StreamingAggregationSuite extends StateStoreMetricsTest with Assertions {
     )
   }
 
+  testWithAllStateVersions("SPARK-29438: ensure UNION doesn't lead streaming aggregation to use" +
+    " shifted partition IDs") {
+    def constructUnionDf(desiredPartitionsForInput1: Int)
+      : (MemoryStream[Int], MemoryStream[Int], DataFrame) = {
+      val input1 = MemoryStream[Int](desiredPartitionsForInput1)
+      val input2 = MemoryStream[Int]
+      val df1 = input1.toDF()
+        .select($"value", $"value" + 1)
+      val df2 = input2.toDF()
+        .groupBy($"value")
+        .agg(count("*"))
+
+      // Unioned DF would have columns as (Int, Int)
+      (input1, input2, df1.union(df2))
+    }
+
+    withTempDir { checkpointDir =>
+      val (input1, input2, unionDf) = constructUnionDf(2)
+      testStream(unionDf, Update)(
+        StartStream(checkpointLocation = checkpointDir.getAbsolutePath),
+        MultiAddData(input1, 11, 12)(input2, 21, 22),
+        CheckNewAnswer(Row(11, 12), Row(12, 13), Row(21, 1), Row(22, 1)),
+        StopStream
+      )
+
+      // We're restoring the query with different number of partitions in left side of UNION,
+      // which may lead right side of union to have mismatched partition IDs (e.g. if it relies on
+      // TaskContext.partitionId()). This test will verify streaming aggregation doesn't have
+      // such issue.
+
+      val (newInput1, newInput2, newUnionDf) = constructUnionDf(3)
+
+      newInput1.addData(11, 12)
+      newInput2.addData(21, 22)
+
+      testStream(newUnionDf, Update)(
+        StartStream(checkpointLocation = checkpointDir.getAbsolutePath),
+        MultiAddData(newInput1, 13, 14)(newInput2, 22, 23),
+        CheckNewAnswer(Row(13, 14), Row(14, 15), Row(22, 2), Row(23, 1))
+      )
+    }
+  }
+
   testQuietlyWithAllStateVersions("midbatch failure") {
     val inputData = MemoryStream[Int]
     FailureSingleton.firstTime = true
@@ -712,6 +755,89 @@ class StreamingAggregationSuite extends StateStoreMetricsTest with Assertions {
     )
   }
 
+  testQuietlyWithAllStateVersions("changing schema of state when restarting query",
+    (SQLConf.STATE_STORE_FORMAT_VALIDATION_ENABLED.key, "false")) {
+    withTempDir { tempDir =>
+      val (inputData, aggregated) = prepareTestForChangingSchemaOfState(tempDir)
+
+      // if we don't have verification phase on state schema, modified query would throw NPE with
+      // stack trace which end users would not easily understand
+
+      testStream(aggregated, Update())(
+        StartStream(checkpointLocation = tempDir.getAbsolutePath),
+        AddData(inputData, 21),
+        ExpectFailure[SparkException] { e =>
+          val stateSchemaExc = findStateSchemaNotCompatible(e)
+          assert(stateSchemaExc.isDefined)
+          val msg = stateSchemaExc.get.getMessage
+          assert(msg.contains("Provided schema doesn't match to the schema for existing state"))
+          // other verifications are presented in StateStoreSuite
+        }
+      )
+    }
+  }
+
+  testQuietlyWithAllStateVersions("changing schema of state when restarting query -" +
+    " schema check off",
+    (SQLConf.STATE_SCHEMA_CHECK_ENABLED.key, "false"),
+    (SQLConf.STATE_STORE_FORMAT_VALIDATION_ENABLED.key, "false")) {
+    withTempDir { tempDir =>
+      val (inputData, aggregated) = prepareTestForChangingSchemaOfState(tempDir)
+
+      testStream(aggregated, Update())(
+        StartStream(checkpointLocation = tempDir.getAbsolutePath),
+        AddData(inputData, 21),
+        ExpectFailure[SparkException] { e =>
+          val stateSchemaExc = findStateSchemaNotCompatible(e)
+          // it would bring other error in runtime, but it shouldn't check schema in any way
+          assert(stateSchemaExc.isEmpty)
+        }
+      )
+    }
+  }
+
+  private def prepareTestForChangingSchemaOfState(
+      tempDir: File): (MemoryStream[Int], DataFrame) = {
+    val inputData = MemoryStream[Int]
+    val aggregated = inputData.toDF()
+      .selectExpr("value % 10 AS id", "value")
+      .groupBy($"id")
+      .agg(
+        sum("value").as("sum_value"),
+        avg("value").as("avg_value"),
+        max("value").as("max_value"))
+
+    testStream(aggregated, Update())(
+      StartStream(checkpointLocation = tempDir.getAbsolutePath),
+      AddData(inputData, 1, 11),
+      CheckLastBatch((1L, 12L, 6.0, 11)),
+      StopStream
+    )
+
+    StateStore.unloadAll()
+
+    val inputData2 = MemoryStream[Int]
+    val aggregated2 = inputData2.toDF()
+      .selectExpr("value % 10 AS id", "value")
+      .groupBy($"id")
+      .agg(
+        sum("value").as("sum_value"),
+        avg("value").as("avg_value"),
+        collect_list("value").as("values"))
+
+    inputData2.addData(1, 11)
+
+    (inputData2, aggregated2)
+  }
+
+  @tailrec
+  private def findStateSchemaNotCompatible(exc: Throwable): Option[StateSchemaNotCompatible] = {
+    exc match {
+      case e1: StateSchemaNotCompatible => Some(e1)
+      case e1 if e1.getCause != null => findStateSchemaNotCompatible(e1.getCause)
+      case _ => None
+    }
+  }
 
   /** Add blocks of data to the `BlockRDDBackedSource`. */
   case class AddBlockData(source: BlockRDDBackedSource, data: Seq[Int]*) extends AddData {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationSuite.scala
index f63778aef5a7f..ac9cd1a12d06f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationSuite.scala
@@ -17,12 +17,9 @@
 
 package org.apache.spark.sql.streaming
 
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.spark.sql.catalyst.plans.physical.{ClusteredDistribution, HashPartitioning, SinglePartition}
+import org.apache.spark.sql.DataFrame
 import org.apache.spark.sql.catalyst.streaming.InternalOutputModes._
-import org.apache.spark.sql.execution.streaming.{MemoryStream, StreamingDeduplicateExec}
-import org.apache.spark.sql.execution.streaming.state.StateStore
+import org.apache.spark.sql.execution.streaming.MemoryStream
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 
@@ -54,13 +51,13 @@ class StreamingDeduplicationSuite extends StateStoreMetricsTest {
     testStream(result, Append)(
       AddData(inputData, "a" -> 1),
       CheckLastBatch("a" -> 1),
-      assertNumStateRows(total = 1, updated = 1),
+      assertNumStateRows(total = 1, updated = 1, droppedByWatermark = 0),
       AddData(inputData, "a" -> 2), // Dropped
       CheckLastBatch(),
-      assertNumStateRows(total = 1, updated = 0),
+      assertNumStateRows(total = 1, updated = 0, droppedByWatermark = 0),
       AddData(inputData, "b" -> 1),
       CheckLastBatch("b" -> 1),
-      assertNumStateRows(total = 2, updated = 1)
+      assertNumStateRows(total = 2, updated = 1, droppedByWatermark = 0)
     )
   }
 
@@ -86,7 +83,7 @@ class StreamingDeduplicationSuite extends StateStoreMetricsTest {
   test("deduplicate with watermark") {
     val inputData = MemoryStream[Int]
     val result = inputData.toDS()
-      .withColumn("eventTime", $"value".cast("timestamp"))
+      .withColumn("eventTime", timestamp_seconds($"value"))
       .withWatermark("eventTime", "10 seconds")
       .dropDuplicates()
       .select($"eventTime".cast("long").as[Long])
@@ -102,7 +99,7 @@ class StreamingDeduplicationSuite extends StateStoreMetricsTest {
 
       AddData(inputData, 10), // Should not emit anything as data less than watermark
       CheckNewAnswer(),
-      assertNumStateRows(total = 1, updated = 0),
+      assertNumStateRows(total = 1, updated = 0, droppedByWatermark = 1),
 
       AddData(inputData, 45), // Advance watermark to 35 seconds, no-data-batch drops row 25
       CheckNewAnswer(45),
@@ -113,7 +110,7 @@ class StreamingDeduplicationSuite extends StateStoreMetricsTest {
   test("deduplicate with aggregate - append mode") {
     val inputData = MemoryStream[Int]
     val windowedaggregate = inputData.toDS()
-      .withColumn("eventTime", $"value".cast("timestamp"))
+      .withColumn("eventTime", timestamp_seconds($"value"))
       .withWatermark("eventTime", "10 seconds")
       .dropDuplicates()
       .withWatermark("eventTime", "10 seconds")
@@ -136,7 +133,8 @@ class StreamingDeduplicationSuite extends StateStoreMetricsTest {
 
       AddData(inputData, 10), // Should not emit anything as data less than watermark
       CheckLastBatch(),
-      assertNumStateRows(total = Seq(2L, 1L), updated = Seq(0L, 0L)),
+      assertNumStateRows(total = Seq(2L, 1L), updated = Seq(0L, 0L),
+        droppedByWatermark = Seq(0L, 1L)),
 
       AddData(inputData, 40), // Advance watermark to 30 seconds
       CheckLastBatch((15 -> 1), (25 -> 1)),
@@ -229,7 +227,7 @@ class StreamingDeduplicationSuite extends StateStoreMetricsTest {
   test("SPARK-19841: watermarkPredicate should filter based on keys") {
     val input = MemoryStream[(Int, Int)]
     val df = input.toDS.toDF("time", "id")
-      .withColumn("time", $"time".cast("timestamp"))
+      .withColumn("time", timestamp_seconds($"time"))
       .withWatermark("time", "1 second")
       .dropDuplicates("id", "time") // Change the column positions
       .select($"id")
@@ -248,7 +246,7 @@ class StreamingDeduplicationSuite extends StateStoreMetricsTest {
   test("SPARK-21546: dropDuplicates should ignore watermark when it's not a key") {
     val input = MemoryStream[(Int, Int)]
     val df = input.toDS.toDF("id", "time")
-      .withColumn("time", $"time".cast("timestamp"))
+      .withColumn("time", timestamp_seconds($"time"))
       .withWatermark("time", "1 second")
       .dropDuplicates("id")
       .select($"id", $"time".cast("long"))
@@ -264,7 +262,7 @@ class StreamingDeduplicationSuite extends StateStoreMetricsTest {
     def testWithFlag(flag: Boolean): Unit = withClue(s"with $flagKey = $flag") {
       val inputData = MemoryStream[Int]
       val result = inputData.toDS()
-        .withColumn("eventTime", $"value".cast("timestamp"))
+        .withColumn("eventTime", timestamp_seconds($"value"))
         .withWatermark("eventTime", "10 seconds")
         .dropDuplicates()
         .select($"eventTime".cast("long").as[Long])
@@ -281,11 +279,57 @@ class StreamingDeduplicationSuite extends StateStoreMetricsTest {
           if (flag) assertNumStateRows(total = 1, updated = 1)
           else assertNumStateRows(total = 7, updated = 1)
         },
-        AssertOnQuery(q => q.lastProgress.sink.numOutputRows == 0L)
+        AssertOnQuery { q =>
+          eventually(timeout(streamingTimeout)) {
+            q.lastProgress.sink.numOutputRows == 0L
+            true
+          }
+        }
       )
     }
 
     testWithFlag(true)
     testWithFlag(false)
   }
+
+  test("SPARK-29438: ensure UNION doesn't lead streaming deduplication to use" +
+    " shifted partition IDs") {
+    def constructUnionDf(desiredPartitionsForInput1: Int)
+      : (MemoryStream[Int], MemoryStream[Int], DataFrame) = {
+      val input1 = MemoryStream[Int](desiredPartitionsForInput1)
+      val input2 = MemoryStream[Int]
+      val df1 = input1.toDF().select($"value")
+      val df2 = input2.toDF().dropDuplicates("value")
+
+      // Unioned DF would have columns as (Int)
+      (input1, input2, df1.union(df2))
+    }
+
+    withTempDir { checkpointDir =>
+      val (input1, input2, unionDf) = constructUnionDf(2)
+      testStream(unionDf, Append)(
+        StartStream(checkpointLocation = checkpointDir.getAbsolutePath),
+        MultiAddData(input1, 11, 12)(input2, 21, 22),
+        CheckNewAnswer(11, 12, 21, 22),
+        StopStream
+      )
+
+      // We're restoring the query with different number of partitions in left side of UNION,
+      // which may lead right side of union to have mismatched partition IDs (e.g. if it relies on
+      // TaskContext.partitionId()). This test will verify streaming deduplication doesn't have
+      // such issue.
+
+      val (newInput1, newInput2, newUnionDf) = constructUnionDf(3)
+
+      newInput1.addData(11, 12)
+      newInput2.addData(21, 22)
+
+      testStream(newUnionDf, Append)(
+        StartStream(checkpointLocation = checkpointDir.getAbsolutePath),
+        MultiAddData(newInput1, 13, 14)(newInput2, 22, 23),
+        CheckNewAnswer(13, 14, 23)
+      )
+    }
+  }
+
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala
index 3f218c9cb7fd9..40131e822c5ce 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.streaming
 
 import java.io.File
+import java.sql.Timestamp
 import java.util.{Locale, UUID}
 
 import scala.util.Random
@@ -25,23 +26,17 @@ import scala.util.Random
 import org.apache.commons.io.FileUtils
 import org.scalatest.BeforeAndAfter
 
-import org.apache.spark.SparkContext
 import org.apache.spark.scheduler.ExecutorCacheTaskLocation
-import org.apache.spark.sql.{AnalysisException, DataFrame, Row, SparkSession}
-import org.apache.spark.sql.catalyst.analysis.StreamingJoinHelper
-import org.apache.spark.sql.catalyst.expressions.{AttributeReference, AttributeSet, Literal}
-import org.apache.spark.sql.catalyst.plans.logical.{EventTimeWatermark, Filter}
-import org.apache.spark.sql.catalyst.trees.TreeNode
-import org.apache.spark.sql.execution.{FileSourceScanExec, LogicalRDD}
-import org.apache.spark.sql.execution.datasources.LogicalRelation
+import org.apache.spark.sql.{DataFrame, Row, SparkSession}
 import org.apache.spark.sql.execution.streaming.{MemoryStream, StatefulOperatorStateInfo, StreamingSymmetricHashJoinExec, StreamingSymmetricHashJoinHelper}
 import org.apache.spark.sql.execution.streaming.state.{StateStore, StateStoreProviderId}
 import org.apache.spark.sql.functions._
-import org.apache.spark.sql.types._
 import org.apache.spark.util.Utils
 
+abstract class StreamingJoinSuite
+  extends StreamTest with StateStoreMetricsTest with BeforeAndAfter {
 
-class StreamingInnerJoinSuite extends StreamTest with StateStoreMetricsTest with BeforeAndAfter {
+  import testImplicits._
 
   before {
     SparkSession.setActiveSession(spark)  // set this before force initializing 'joinExec'
@@ -52,6 +47,162 @@ class StreamingInnerJoinSuite extends StreamTest with StateStoreMetricsTest with
     StateStore.stop()
   }
 
+  protected def setupStream(prefix: String, multiplier: Int): (MemoryStream[Int], DataFrame) = {
+    val input = MemoryStream[Int]
+    val df = input.toDF
+      .select(
+        'value as "key",
+        timestamp_seconds($"value")  as s"${prefix}Time",
+        ('value * multiplier) as s"${prefix}Value")
+      .withWatermark(s"${prefix}Time", "10 seconds")
+
+    (input, df)
+  }
+
+  protected def setupWindowedJoin(joinType: String)
+    : (MemoryStream[Int], MemoryStream[Int], DataFrame) = {
+
+    val (input1, df1) = setupStream("left", 2)
+    val (input2, df2) = setupStream("right", 3)
+    val windowed1 = df1.select('key, window('leftTime, "10 second"), 'leftValue)
+    val windowed2 = df2.select('key, window('rightTime, "10 second"), 'rightValue)
+    val joined = windowed1.join(windowed2, Seq("key", "window"), joinType)
+    val select = if (joinType == "left_semi") {
+      joined.select('key, $"window.end".cast("long"), 'leftValue)
+    } else {
+      joined.select('key, $"window.end".cast("long"), 'leftValue, 'rightValue)
+    }
+
+    (input1, input2, select)
+  }
+
+  protected def setupWindowedJoinWithLeftCondition(joinType: String)
+    : (MemoryStream[Int], MemoryStream[Int], DataFrame) = {
+
+    val (leftInput, df1) = setupStream("left", 2)
+    val (rightInput, df2) = setupStream("right", 3)
+    // Use different schemas to ensure the null row is being generated from the correct side.
+    val left = df1.select('key, window('leftTime, "10 second"), 'leftValue)
+    val right = df2.select('key, window('rightTime, "10 second"), 'rightValue.cast("string"))
+
+    val joined = left.join(
+      right,
+      left("key") === right("key")
+        && left("window") === right("window")
+        && 'leftValue > 4,
+      joinType)
+
+    val select = if (joinType == "left_semi") {
+      joined.select(left("key"), left("window.end").cast("long"), 'leftValue)
+    } else if (joinType == "left_outer") {
+      joined.select(left("key"), left("window.end").cast("long"), 'leftValue, 'rightValue)
+    } else if (joinType == "right_outer") {
+      joined.select(right("key"), right("window.end").cast("long"), 'leftValue, 'rightValue)
+    } else {
+      joined.select(left("key"), left("window.end").cast("long"), 'leftValue,
+        right("key"), right("window.end").cast("long"), 'rightValue)
+    }
+
+    (leftInput, rightInput, select)
+  }
+
+  protected def setupWindowedJoinWithRightCondition(joinType: String)
+    : (MemoryStream[Int], MemoryStream[Int], DataFrame) = {
+
+    val (leftInput, df1) = setupStream("left", 2)
+    val (rightInput, df2) = setupStream("right", 3)
+    // Use different schemas to ensure the null row is being generated from the correct side.
+    val left = df1.select('key, window('leftTime, "10 second"), 'leftValue)
+    val right = df2.select('key, window('rightTime, "10 second"), 'rightValue.cast("string"))
+
+    val joined = left.join(
+      right,
+      left("key") === right("key")
+        && left("window") === right("window")
+        && 'rightValue.cast("int") > 7,
+      joinType)
+
+    val select = if (joinType == "left_semi") {
+      joined.select(left("key"), left("window.end").cast("long"), 'leftValue)
+    } else if (joinType == "left_outer") {
+      joined.select(left("key"), left("window.end").cast("long"), 'leftValue, 'rightValue)
+    } else if (joinType == "right_outer") {
+      joined.select(right("key"), right("window.end").cast("long"), 'leftValue, 'rightValue)
+    } else {
+      joined.select(left("key"), left("window.end").cast("long"), 'leftValue,
+        right("key"), right("window.end").cast("long"), 'rightValue)
+    }
+
+    (leftInput, rightInput, select)
+  }
+
+  protected def setupJoinWithRangeCondition(joinType: String)
+    : (MemoryStream[(Int, Int)], MemoryStream[(Int, Int)], DataFrame) = {
+
+    val leftInput = MemoryStream[(Int, Int)]
+    val rightInput = MemoryStream[(Int, Int)]
+
+    val df1 = leftInput.toDF.toDF("leftKey", "time")
+      .select('leftKey, timestamp_seconds($"time") as "leftTime", ('leftKey * 2) as "leftValue")
+      .withWatermark("leftTime", "10 seconds")
+
+    val df2 = rightInput.toDF.toDF("rightKey", "time")
+      .select('rightKey, timestamp_seconds($"time") as "rightTime",
+        ('rightKey * 3) as "rightValue")
+      .withWatermark("rightTime", "10 seconds")
+
+    val joined =
+      df1.join(
+        df2,
+        expr("leftKey = rightKey AND " +
+          "leftTime BETWEEN rightTime - interval 5 seconds AND rightTime + interval 5 seconds"),
+        joinType)
+
+    val select = if (joinType == "left_semi") {
+      joined.select('leftKey, 'leftTime.cast("int"))
+    } else {
+      joined.select('leftKey, 'rightKey, 'leftTime.cast("int"), 'rightTime.cast("int"))
+    }
+
+    (leftInput, rightInput, select)
+  }
+
+  protected def setupSelfJoin(joinType: String)
+    : (MemoryStream[(Int, Long)], DataFrame) = {
+
+    val inputStream = MemoryStream[(Int, Long)]
+
+    val df = inputStream.toDS()
+      .select(col("_1").as("value"), timestamp_seconds($"_2").as("timestamp"))
+
+    val leftStream = df.select(col("value").as("leftId"), col("timestamp").as("leftTime"))
+
+    val rightStream = df
+      // Introduce misses for ease of debugging
+      .where(col("value") % 2 === 0)
+      .select(col("value").as("rightId"), col("timestamp").as("rightTime"))
+
+    val joined = leftStream
+      .withWatermark("leftTime", "5 seconds")
+      .join(
+        rightStream.withWatermark("rightTime", "5 seconds"),
+        expr("leftId = rightId AND rightTime >= leftTime AND " +
+          "rightTime <= leftTime + interval 5 seconds"),
+        joinType)
+
+    val select = if (joinType == "left_semi") {
+      joined.select(col("leftId"), col("leftTime").cast("int"))
+    } else {
+      joined.select(col("leftId"), col("leftTime").cast("int"),
+        col("rightId"), col("rightTime").cast("int"))
+    }
+
+    (inputStream, select)
+  }
+}
+
+class StreamingInnerJoinSuite extends StreamingJoinSuite {
+
   import testImplicits._
   test("stream stream inner join on non-time column") {
     val input1 = MemoryStream[Int]
@@ -87,11 +238,12 @@ class StreamingInnerJoinSuite extends StreamTest with StateStoreMetricsTest with
     val input2 = MemoryStream[Int]
 
     val df1 = input1.toDF
-      .select('value as "key", 'value.cast("timestamp") as "timestamp", ('value * 2) as "leftValue")
+      .select('value as "key", timestamp_seconds($"value") as "timestamp",
+        ('value * 2) as "leftValue")
       .select('key, window('timestamp, "10 second"), 'leftValue)
 
     val df2 = input2.toDF
-      .select('value as "key", 'value.cast("timestamp") as "timestamp",
+      .select('value as "key", timestamp_seconds($"value") as "timestamp",
         ('value * 3) as "rightValue")
       .select('key, window('timestamp, "10 second"), 'rightValue)
 
@@ -127,12 +279,13 @@ class StreamingInnerJoinSuite extends StreamTest with StateStoreMetricsTest with
     val input2 = MemoryStream[Int]
 
     val df1 = input1.toDF
-      .select('value as "key", 'value.cast("timestamp") as "timestamp", ('value * 2) as "leftValue")
+      .select('value as "key", timestamp_seconds($"value") as "timestamp",
+        ('value * 2) as "leftValue")
       .withWatermark("timestamp", "10 seconds")
       .select('key, window('timestamp, "10 second"), 'leftValue)
 
     val df2 = input2.toDF
-      .select('value as "key", 'value.cast("timestamp") as "timestamp",
+      .select('value as "key", timestamp_seconds($"value") as "timestamp",
         ('value * 3) as "rightValue")
       .select('key, window('timestamp, "10 second"), 'rightValue)
 
@@ -166,7 +319,7 @@ class StreamingInnerJoinSuite extends StreamTest with StateStoreMetricsTest with
 
       AddData(input1, 5),
       CheckNewAnswer(),                             // Same reason as above
-      assertNumStateRows(total = 2, updated = 0)
+      assertNumStateRows(total = 2, updated = 0, droppedByWatermark = 1)
     )
   }
 
@@ -177,11 +330,12 @@ class StreamingInnerJoinSuite extends StreamTest with StateStoreMetricsTest with
     val rightInput = MemoryStream[(Int, Int)]
 
     val df1 = leftInput.toDF.toDF("leftKey", "time")
-      .select('leftKey, 'time.cast("timestamp") as "leftTime", ('leftKey * 2) as "leftValue")
+      .select('leftKey, timestamp_seconds($"time") as "leftTime", ('leftKey * 2) as "leftValue")
       .withWatermark("leftTime", "10 seconds")
 
     val df2 = rightInput.toDF.toDF("rightKey", "time")
-      .select('rightKey, 'time.cast("timestamp") as "rightTime", ('rightKey * 3) as "rightValue")
+      .select('rightKey, timestamp_seconds($"time") as "rightTime",
+        ('rightKey * 3) as "rightValue")
       .withWatermark("rightTime", "10 seconds")
 
     val joined =
@@ -219,12 +373,12 @@ class StreamingInnerJoinSuite extends StreamTest with StateStoreMetricsTest with
       // (1, 28) ==> passed filter, matched with left (1, 3) and (1, 5), added to state
       AddData(rightInput, (1, 20), (1, 21), (1, 28)),
       CheckNewAnswer((1, 3, 21), (1, 5, 21), (1, 3, 28), (1, 5, 28)),
-      assertNumStateRows(total = 5, updated = 1),
+      assertNumStateRows(total = 5, updated = 1, droppedByWatermark = 1),
 
       // New data to left input with leftTime <= 20 should be filtered due to event time watermark
       AddData(leftInput, (1, 20), (1, 21)),
       CheckNewAnswer((1, 21, 28)),
-      assertNumStateRows(total = 6, updated = 1)
+      assertNumStateRows(total = 6, updated = 1, droppedByWatermark = 1)
     )
   }
 
@@ -235,11 +389,12 @@ class StreamingInnerJoinSuite extends StreamTest with StateStoreMetricsTest with
     val rightInput = MemoryStream[(Int, Int)]
 
     val df1 = leftInput.toDF.toDF("leftKey", "time")
-      .select('leftKey, 'time.cast("timestamp") as "leftTime", ('leftKey * 2) as "leftValue")
+      .select('leftKey, timestamp_seconds($"time") as "leftTime", ('leftKey * 2) as "leftValue")
       .withWatermark("leftTime", "20 seconds")
 
     val df2 = rightInput.toDF.toDF("rightKey", "time")
-      .select('rightKey, 'time.cast("timestamp") as "rightTime", ('rightKey * 3) as "rightValue")
+      .select('rightKey, timestamp_seconds($"time") as "rightTime",
+        ('rightKey * 3) as "rightValue")
       .withWatermark("rightTime", "30 seconds")
 
     val condition = expr(
@@ -293,7 +448,7 @@ class StreamingInnerJoinSuite extends StreamTest with StateStoreMetricsTest with
 
       AddData(leftInput, (1, 30), (1, 31)),     // 30 should not be processed or added to state
       CheckNewAnswer((1, 31, 26), (1, 31, 30), (1, 31, 31)),
-      assertNumStateRows(total = 11, updated = 1),  // only 31 added
+      assertNumStateRows(total = 11, updated = 1, droppedByWatermark = 1),  // only 31 added
 
       // Advance the watermark
       AddData(rightInput, (1, 80)),
@@ -307,7 +462,7 @@ class StreamingInnerJoinSuite extends StreamTest with StateStoreMetricsTest with
 
       AddData(rightInput, (1, 46), (1, 50)),     // 46 should not be processed or added to state
       CheckNewAnswer((1, 49, 50), (1, 50, 50)),
-      assertNumStateRows(total = 7, updated = 1)   // 50 added
+      assertNumStateRows(total = 7, updated = 1, droppedByWatermark = 1)   // 50 added
     )
   }
 
@@ -368,7 +523,7 @@ class StreamingInnerJoinSuite extends StreamTest with StateStoreMetricsTest with
       }.toMap
       partitionAndStoreNameToLocation.foreach { case ((partIndex, storeName), hostName) =>
         val providerId = StateStoreProviderId(stateInfo, partIndex, storeName)
-        coordinatorRef.reportActiveInstance(providerId, hostName, s"exec-$hostName")
+        coordinatorRef.reportActiveInstance(providerId, hostName, s"exec-$hostName", Seq.empty)
         require(
           coordinatorRef.getLocation(providerId) ===
             Some(ExecutorCacheTaskLocation(hostName, s"exec-$hostName").toString))
@@ -425,7 +580,7 @@ class StreamingInnerJoinSuite extends StreamTest with StateStoreMetricsTest with
   test("SPARK-26187 restore the stream-stream inner join query from Spark 2.4") {
     val inputStream = MemoryStream[(Int, Long)]
     val df = inputStream.toDS()
-      .select(col("_1").as("value"), col("_2").cast("timestamp").as("timestamp"))
+      .select(col("_1").as("value"), timestamp_seconds($"_2").as("timestamp"))
 
     val leftStream = df.select(col("value").as("leftId"), col("timestamp").as("leftTime"))
 
@@ -481,58 +636,13 @@ class StreamingInnerJoinSuite extends StreamTest with StateStoreMetricsTest with
 }
 
 
-class StreamingOuterJoinSuite extends StreamTest with StateStoreMetricsTest with BeforeAndAfter {
+class StreamingOuterJoinSuite extends StreamingJoinSuite {
 
   import testImplicits._
   import org.apache.spark.sql.functions._
 
-  before {
-    SparkSession.setActiveSession(spark) // set this before force initializing 'joinExec'
-    spark.streams.stateStoreCoordinator // initialize the lazy coordinator
-  }
-
-  after {
-    StateStore.stop()
-  }
-
-  private def setupStream(prefix: String, multiplier: Int): (MemoryStream[Int], DataFrame) = {
-    val input = MemoryStream[Int]
-    val df = input.toDF
-      .select(
-        'value as "key",
-        'value.cast("timestamp") as s"${prefix}Time",
-        ('value * multiplier) as s"${prefix}Value")
-      .withWatermark(s"${prefix}Time", "10 seconds")
-
-    return (input, df)
-  }
-
-  private def setupWindowedJoin(joinType: String):
-  (MemoryStream[Int], MemoryStream[Int], DataFrame) = {
-    val (input1, df1) = setupStream("left", 2)
-    val (input2, df2) = setupStream("right", 3)
-    val windowed1 = df1.select('key, window('leftTime, "10 second"), 'leftValue)
-    val windowed2 = df2.select('key, window('rightTime, "10 second"), 'rightValue)
-    val joined = windowed1.join(windowed2, Seq("key", "window"), joinType)
-      .select('key, $"window.end".cast("long"), 'leftValue, 'rightValue)
-
-    (input1, input2, joined)
-  }
-
   test("left outer early state exclusion on left") {
-    val (leftInput, df1) = setupStream("left", 2)
-    val (rightInput, df2) = setupStream("right", 3)
-    // Use different schemas to ensure the null row is being generated from the correct side.
-    val left = df1.select('key, window('leftTime, "10 second"), 'leftValue)
-    val right = df2.select('key, window('rightTime, "10 second"), 'rightValue.cast("string"))
-
-    val joined = left.join(
-        right,
-        left("key") === right("key")
-          && left("window") === right("window")
-          && 'leftValue > 4,
-        "left_outer")
-        .select(left("key"), left("window.end").cast("long"), 'leftValue, 'rightValue)
+    val (leftInput, rightInput, joined) = setupWindowedJoinWithLeftCondition("left_outer")
 
     testStream(joined)(
       MultiAddData(leftInput, 1, 2, 3)(rightInput, 3, 4, 5),
@@ -549,19 +659,7 @@ class StreamingOuterJoinSuite extends StreamTest with StateStoreMetricsTest with
   }
 
   test("left outer early state exclusion on right") {
-    val (leftInput, df1) = setupStream("left", 2)
-    val (rightInput, df2) = setupStream("right", 3)
-    // Use different schemas to ensure the null row is being generated from the correct side.
-    val left = df1.select('key, window('leftTime, "10 second"), 'leftValue)
-    val right = df2.select('key, window('rightTime, "10 second"), 'rightValue.cast("string"))
-
-    val joined = left.join(
-      right,
-      left("key") === right("key")
-        && left("window") === right("window")
-        && 'rightValue.cast("int") > 7,
-      "left_outer")
-      .select(left("key"), left("window.end").cast("long"), 'leftValue, 'rightValue)
+    val (leftInput, rightInput, joined) = setupWindowedJoinWithRightCondition("left_outer")
 
     testStream(joined)(
       MultiAddData(leftInput, 3, 4, 5)(rightInput, 1, 2, 3),
@@ -578,19 +676,7 @@ class StreamingOuterJoinSuite extends StreamTest with StateStoreMetricsTest with
   }
 
   test("right outer early state exclusion on left") {
-    val (leftInput, df1) = setupStream("left", 2)
-    val (rightInput, df2) = setupStream("right", 3)
-    // Use different schemas to ensure the null row is being generated from the correct side.
-    val left = df1.select('key, window('leftTime, "10 second"), 'leftValue)
-    val right = df2.select('key, window('rightTime, "10 second"), 'rightValue.cast("string"))
-
-    val joined = left.join(
-      right,
-      left("key") === right("key")
-        && left("window") === right("window")
-        && 'leftValue > 4,
-      "right_outer")
-      .select(right("key"), right("window.end").cast("long"), 'leftValue, 'rightValue)
+    val (leftInput, rightInput, joined) = setupWindowedJoinWithLeftCondition("right_outer")
 
     testStream(joined)(
       MultiAddData(leftInput, 1, 2, 3)(rightInput, 3, 4, 5),
@@ -607,19 +693,7 @@ class StreamingOuterJoinSuite extends StreamTest with StateStoreMetricsTest with
   }
 
   test("right outer early state exclusion on right") {
-    val (leftInput, df1) = setupStream("left", 2)
-    val (rightInput, df2) = setupStream("right", 3)
-    // Use different schemas to ensure the null row is being generated from the correct side.
-    val left = df1.select('key, window('leftTime, "10 second"), 'leftValue)
-    val right = df2.select('key, window('rightTime, "10 second"), 'rightValue.cast("string"))
-
-    val joined = left.join(
-      right,
-      left("key") === right("key")
-        && left("window") === right("window")
-        && 'rightValue.cast("int") > 7,
-      "right_outer")
-      .select(right("key"), right("window.end").cast("long"), 'leftValue, 'rightValue)
+    val (leftInput, rightInput, joined) = setupWindowedJoinWithRightCondition("right_outer")
 
     testStream(joined)(
       MultiAddData(leftInput, 3, 4, 5)(rightInput, 1, 2, 3),
@@ -676,26 +750,8 @@ class StreamingOuterJoinSuite extends StreamTest with StateStoreMetricsTest with
     ("right_outer", Row(null, 2, null, 5))
   ).foreach { case (joinType: String, outerResult) =>
     test(s"${joinType.replaceAllLiterally("_", " ")} with watermark range condition") {
-      import org.apache.spark.sql.functions._
-
-      val leftInput = MemoryStream[(Int, Int)]
-      val rightInput = MemoryStream[(Int, Int)]
-
-      val df1 = leftInput.toDF.toDF("leftKey", "time")
-        .select('leftKey, 'time.cast("timestamp") as "leftTime", ('leftKey * 2) as "leftValue")
-        .withWatermark("leftTime", "10 seconds")
-
-      val df2 = rightInput.toDF.toDF("rightKey", "time")
-        .select('rightKey, 'time.cast("timestamp") as "rightTime", ('rightKey * 3) as "rightValue")
-        .withWatermark("rightTime", "10 seconds")
-
-      val joined =
-        df1.join(
-          df2,
-          expr("leftKey = rightKey AND " +
-            "leftTime BETWEEN rightTime - interval 5 seconds AND rightTime + interval 5 seconds"),
-          joinType)
-          .select('leftKey, 'rightKey, 'leftTime.cast("int"), 'rightTime.cast("int"))
+      val (leftInput, rightInput, joined) = setupJoinWithRangeCondition(joinType)
+
       testStream(joined)(
         AddData(leftInput, (1, 5), (3, 5)),
         CheckAnswer(),
@@ -774,27 +830,7 @@ class StreamingOuterJoinSuite extends StreamTest with StateStoreMetricsTest with
   }
 
   test("SPARK-26187 self left outer join should not return outer nulls for already matched rows") {
-    val inputStream = MemoryStream[(Int, Long)]
-
-    val df = inputStream.toDS()
-      .select(col("_1").as("value"), col("_2").cast("timestamp").as("timestamp"))
-
-    val leftStream = df.select(col("value").as("leftId"), col("timestamp").as("leftTime"))
-
-    val rightStream = df
-      // Introduce misses for ease of debugging
-      .where(col("value") % 2 === 0)
-      .select(col("value").as("rightId"), col("timestamp").as("rightTime"))
-
-    val query = leftStream
-      .withWatermark("leftTime", "5 seconds")
-      .join(
-        rightStream.withWatermark("rightTime", "5 seconds"),
-        expr("leftId = rightId AND rightTime >= leftTime AND " +
-          "rightTime <= leftTime + interval 5 seconds"),
-        joinType = "leftOuter")
-      .select(col("leftId"), col("leftTime").cast("int"),
-        col("rightId"), col("rightTime").cast("int"))
+    val (inputStream, query) = setupSelfJoin("left_outer")
 
     testStream(query)(
       AddData(inputStream, (1, 1L), (2, 2L), (3, 3L), (4, 4L), (5, 5L)),
@@ -840,7 +876,7 @@ class StreamingOuterJoinSuite extends StreamTest with StateStoreMetricsTest with
     val inputStream = MemoryStream[(Int, Long)]
 
     val df = inputStream.toDS()
-      .select(col("_1").as("value"), col("_2").cast("timestamp").as("timestamp"))
+      .select(col("_1").as("value"), timestamp_seconds($"_2").as("timestamp"))
 
     // we're just flipping "left" and "right" from left outer join and apply right outer join
 
@@ -883,7 +919,7 @@ class StreamingOuterJoinSuite extends StreamTest with StateStoreMetricsTest with
   test("SPARK-26187 restore the stream-stream outer join query from Spark 2.4") {
     val inputStream = MemoryStream[(Int, Long)]
     val df = inputStream.toDS()
-      .select(col("_1").as("value"), col("_2").cast("timestamp").as("timestamp"))
+      .select(col("_1").as("value"), timestamp_seconds($"_2").as("timestamp"))
 
     val leftStream = df.select(col("value").as("leftId"), col("timestamp").as("leftTime"))
 
@@ -932,7 +968,7 @@ class StreamingOuterJoinSuite extends StreamTest with StateStoreMetricsTest with
       throw writer.exception.get
     }
     assert(e.getMessage.toLowerCase(Locale.ROOT)
-      .contains("the query is using stream-stream outer join with state format version 1"))
+      .contains("the query is using stream-stream leftouter join with state format version 1"))
   }
 
   test("SPARK-29438: ensure UNION doesn't lead stream-stream join to use shifted partition IDs") {
@@ -991,4 +1027,452 @@ class StreamingOuterJoinSuite extends StreamTest with StateStoreMetricsTest with
       )
     }
   }
+
+  test("SPARK-32148 stream-stream join regression on Spark 3.0.0") {
+    val input1 = MemoryStream[(Timestamp, String, String)]
+    val df1 = input1.toDF
+      .selectExpr("_1 as eventTime", "_2 as id", "_3 as comment")
+      .withWatermark(s"eventTime", "2 minutes")
+
+    val input2 = MemoryStream[(Timestamp, String, String)]
+    val df2 = input2.toDF
+      .selectExpr("_1 as eventTime", "_2 as id", "_3 as name")
+      .withWatermark(s"eventTime", "4 minutes")
+
+    val joined = df1.as("left")
+      .join(df2.as("right"),
+        expr("""
+               |left.id = right.id AND left.eventTime BETWEEN
+               |  right.eventTime - INTERVAL 30 seconds AND
+               |  right.eventTime + INTERVAL 30 seconds
+             """.stripMargin),
+        joinType = "leftOuter")
+
+    val inputDataForInput1 = Seq(
+      (Timestamp.valueOf("2020-01-01 00:00:00"), "abc", "has no join partner"),
+      (Timestamp.valueOf("2020-01-02 00:00:00"), "abc", "joined with A"),
+      (Timestamp.valueOf("2020-01-02 01:00:00"), "abc", "joined with B"))
+
+    val inputDataForInput2 = Seq(
+      (Timestamp.valueOf("2020-01-02 00:00:10"), "abc", "A"),
+      (Timestamp.valueOf("2020-01-02 00:59:59"), "abc", "B"),
+      (Timestamp.valueOf("2020-01-02 02:00:00"), "abc", "C"))
+
+    val expectedOutput = Seq(
+      (Timestamp.valueOf("2020-01-01 00:00:00"), "abc", "has no join partner", null, null, null),
+      (Timestamp.valueOf("2020-01-02 00:00:00"), "abc", "joined with A",
+        Timestamp.valueOf("2020-01-02 00:00:10"), "abc", "A"),
+      (Timestamp.valueOf("2020-01-02 01:00:00"), "abc", "joined with B",
+        Timestamp.valueOf("2020-01-02 00:59:59"), "abc", "B"))
+
+    testStream(joined)(
+      MultiAddData((input1, inputDataForInput1), (input2, inputDataForInput2)),
+      CheckNewAnswer(expectedOutput.head, expectedOutput.tail: _*)
+    )
+  }
+}
+
+class StreamingFullOuterJoinSuite extends StreamingJoinSuite {
+
+  test("windowed full outer join") {
+    val (leftInput, rightInput, joined) = setupWindowedJoin("full_outer")
+
+    testStream(joined)(
+      MultiAddData(leftInput, 1, 2, 3, 4, 5)(rightInput, 3, 4, 5, 6, 7),
+      CheckNewAnswer(Row(3, 10, 6, 9), Row(4, 10, 8, 12), Row(5, 10, 10, 15)),
+      // states
+      // left: 1, 2, 3, 4 ,5
+      // right: 3, 4, 5, 6, 7
+      assertNumStateRows(total = 10, updated = 10),
+      MultiAddData(leftInput, 21)(rightInput, 22),
+      // Watermark = 11, should remove rows having window=[0,10].
+      CheckNewAnswer(Row(1, 10, 2, null), Row(2, 10, 4, null), Row(6, 10, null, 18),
+        Row(7, 10, null, 21)),
+      // states
+      // left: 21
+      // right: 22
+      //
+      // states evicted
+      // left: 1, 2, 3, 4 ,5 (below watermark)
+      // right: 3, 4, 5, 6, 7 (below watermark)
+      assertNumStateRows(total = 2, updated = 2),
+      AddData(leftInput, 22),
+      CheckNewAnswer(Row(22, 30, 44, 66)),
+      // states
+      // left: 21, 22
+      // right: 22
+      assertNumStateRows(total = 3, updated = 1),
+      StopStream,
+      StartStream(),
+
+      AddData(leftInput, 1),
+      // Row not add as 1 < state key watermark = 12.
+      CheckNewAnswer(),
+      // states
+      // left: 21, 22
+      // right: 22
+      assertNumStateRows(total = 3, updated = 0, droppedByWatermark = 1),
+      AddData(rightInput, 5),
+      // Row not add as 5 < state key watermark = 12.
+      CheckNewAnswer(),
+      // states
+      // left: 21, 22
+      // right: 22
+      assertNumStateRows(total = 3, updated = 0, droppedByWatermark = 1)
+    )
+  }
+
+  test("full outer early state exclusion on left") {
+    val (leftInput, rightInput, joined) = setupWindowedJoinWithLeftCondition("full_outer")
+
+    testStream(joined)(
+      MultiAddData(leftInput, 1, 2, 3)(rightInput, 3, 4, 5),
+      // The left rows with leftValue <= 4 should generate their outer join rows now and
+      // not get added to the state.
+      CheckNewAnswer(Row(1, 10, 2, null, null, null), Row(2, 10, 4, null, null, null),
+        Row(3, 10, 6, 3, 10, "9")),
+      // states
+      // left: 3
+      // right: 3, 4, 5
+      assertNumStateRows(total = 4, updated = 4),
+      // Generate outer join result for all non-matched rows when the watermark advances.
+      MultiAddData(leftInput, 20)(rightInput, 21),
+      CheckNewAnswer(Row(null, null, null, 4, 10, "12"), Row(null, null, null, 5, 10, "15")),
+      // states
+      // left: 20
+      // right: 21
+      //
+      // states evicted
+      // left: 3 (below watermark)
+      // right: 3, 4, 5 (below watermark)
+      assertNumStateRows(total = 2, updated = 2),
+      AddData(rightInput, 20),
+      CheckNewAnswer(Row(20, 30, 40, 20, 30, "60")),
+      // states
+      // left: 20
+      // right: 21, 20
+      assertNumStateRows(total = 3, updated = 1)
+    )
+  }
+
+  test("full outer early state exclusion on right") {
+    val (leftInput, rightInput, joined) = setupWindowedJoinWithRightCondition("full_outer")
+
+    testStream(joined)(
+      MultiAddData(leftInput, 3, 4, 5)(rightInput, 1, 2, 3),
+      // The right rows with rightValue <= 7 should generate their outer join rows now,
+      // and never be added to the state.
+      // The right row with rightValue = 9 > 7, hence joined and added to state.
+      CheckNewAnswer(Row(null, null, null, 1, 10, "3"), Row(null, null, null, 2, 10, "6"),
+        Row(3, 10, 6, 3, 10, "9")),
+      // states
+      // left: 3, 4, 5
+      // right: 3
+      assertNumStateRows(total = 4, updated = 4),
+      // Generate outer join result for all non-matched rows when the watermark advances.
+      MultiAddData(leftInput, 20)(rightInput, 21),
+      CheckNewAnswer(Row(4, 10, 8, null, null, null), Row(5, 10, 10, null, null, null)),
+      // states
+      // left: 20
+      // right: 21
+      //
+      // states evicted
+      // left: 3, 4, 5 (below watermark)
+      // right: 3 (below watermark)
+      assertNumStateRows(total = 2, updated = 2),
+      AddData(rightInput, 20),
+      CheckNewAnswer(Row(20, 30, 40, 20, 30, "60")),
+      // states
+      // left: 20
+      // right: 21, 20
+      assertNumStateRows(total = 3, updated = 1)
+    )
+  }
+
+  test("full outer join with watermark range condition") {
+    val (leftInput, rightInput, joined) = setupJoinWithRangeCondition("full_outer")
+
+    testStream(joined)(
+      AddData(leftInput, (1, 5), (3, 5)),
+      CheckNewAnswer(),
+      // states
+      // left: (1, 5), (3, 5)
+      // right: nothing
+      assertNumStateRows(total = 2, updated = 2),
+      AddData(rightInput, (1, 10), (2, 5)),
+      // Match left row in the state.
+      CheckNewAnswer(Row(1, 1, 5, 10)),
+      // states
+      // left: (1, 5), (3, 5)
+      // right: (1, 10), (2, 5)
+      assertNumStateRows(total = 4, updated = 2),
+      AddData(rightInput, (1, 9)),
+      // Match left row in the state.
+      CheckNewAnswer(Row(1, 1, 5, 9)),
+      // states
+      // left: (1, 5), (3, 5)
+      // right: (1, 10), (2, 5), (1, 9)
+      assertNumStateRows(total = 5, updated = 1),
+      // Increase event time watermark to 20s by adding data with time = 30s on both inputs.
+      AddData(leftInput, (1, 7), (1, 30)),
+      CheckNewAnswer(Row(1, 1, 7, 9), Row(1, 1, 7, 10)),
+      // states
+      // left: (1, 5), (3, 5), (1, 7), (1, 30)
+      // right: (1, 10), (2, 5), (1, 9)
+      assertNumStateRows(total = 7, updated = 2),
+      // Watermark = 30 - 10 = 20, no matched row.
+      // Generate outer join result for all non-matched rows when the watermark advances.
+      AddData(rightInput, (0, 30)),
+      CheckNewAnswer(Row(3, null, 5, null), Row(null, 2, null, 5)),
+      // states
+      // left: (1, 30)
+      // right: (0, 30)
+      //
+      // states evicted
+      // left: (1, 5), (3, 5), (1, 5) (below watermark = 20)
+      // right: (1, 10), (2, 5), (1, 9) (below watermark = 20)
+      assertNumStateRows(total = 2, updated = 1)
+    )
+  }
+
+  test("self full outer join") {
+    val (inputStream, query) = setupSelfJoin("full_outer")
+
+    testStream(query)(
+      AddData(inputStream, (1, 1L), (2, 2L), (3, 3L), (4, 4L), (5, 5L)),
+      CheckNewAnswer(Row(2, 2L, 2, 2L), Row(4, 4L, 4, 4L)),
+      // batch 1 - global watermark = 0
+      // states
+      // left: (1, 1L), (2, 2L), (3, 3L), (4, 4L), (5, 5L)
+      // right: (2, 2L), (4, 4L)
+      assertNumStateRows(total = 7, updated = 7),
+      AddData(inputStream, (6, 6L), (7, 7L), (8, 8L), (9, 9L), (10, 10L)),
+      CheckNewAnswer(Row(6, 6L, 6, 6L), Row(8, 8L, 8, 8L), Row(10, 10L, 10, 10L)),
+      // batch 2 - global watermark = 5
+      // states
+      // left: (1, 1L), (2, 2L), (3, 3L), (4, 4L), (5, 5L), (6, 6L), (7, 7L), (8, 8L),
+      //       (9, 9L), (10, 10L)
+      // right: (6, 6L), (8, 8L), (10, 10L)
+      //
+      // states evicted
+      // left: nothing (it waits for 5 seconds more than watermark due to join condition)
+      // right: (2, 2L), (4, 4L)
+      assertNumStateRows(total = 13, updated = 8),
+      AddData(inputStream, (11, 11L), (12, 12L), (13, 13L), (14, 14L), (15, 15L)),
+      CheckNewAnswer(Row(12, 12L, 12, 12L), Row(14, 14L, 14, 14L), Row(1, 1L, null, null),
+        Row(3, 3L, null, null)),
+      // batch 3 - global watermark = 9
+      // states
+      // left: (4, 4L), (5, 5L), (6, 6L), (7, 7L), (8, 8L), (9, 9L), (10, 10L), (11, 11L),
+      //       (12, 12L), (13, 13L), (14, 14L), (15, 15L)
+      // right: (10, 10L), (12, 12L), (14, 14L)
+      //
+      // states evicted
+      // left: (1, 1L), (2, 2L), (3, 3L)
+      // right: (6, 6L), (8, 8L)
+      assertNumStateRows(total = 15, updated = 7)
+    )
+  }
+}
+
+class StreamingLeftSemiJoinSuite extends StreamingJoinSuite {
+
+  import testImplicits._
+
+  test("windowed left semi join") {
+    val (leftInput, rightInput, joined) = setupWindowedJoin("left_semi")
+
+    testStream(joined)(
+      MultiAddData(leftInput, 1, 2, 3, 4, 5)(rightInput, 3, 4, 5, 6, 7),
+      CheckNewAnswer(Row(3, 10, 6), Row(4, 10, 8), Row(5, 10, 10)),
+      // states
+      // left: 1, 2, 3, 4 ,5
+      // right: 3, 4, 5, 6, 7
+      assertNumStateRows(total = 10, updated = 10),
+      MultiAddData(leftInput, 21)(rightInput, 22),
+      // Watermark = 11, should remove rows having window=[0,10].
+      CheckNewAnswer(),
+      // states
+      // left: 21
+      // right: 22
+      //
+      // states evicted
+      // left: 1, 2, 3, 4 ,5 (below watermark)
+      // right: 3, 4, 5, 6, 7 (below watermark)
+      assertNumStateRows(total = 2, updated = 2),
+      AddData(leftInput, 22),
+      CheckNewAnswer(Row(22, 30, 44)),
+      // Unlike inner/outer joins, given left input row matches with right input row,
+      // we don't buffer the matched left input row to the state store.
+      //
+      // states
+      // left: 21
+      // right: 22
+      assertNumStateRows(total = 2, updated = 0),
+      StopStream,
+      StartStream(),
+
+      AddData(leftInput, 1),
+      // Row not add as 1 < state key watermark = 12.
+      CheckNewAnswer(),
+      // states
+      // left: 21
+      // right: 22
+      assertNumStateRows(total = 2, updated = 0, droppedByWatermark = 1),
+      AddData(rightInput, 5),
+      // Row not add as 5 < state key watermark = 12.
+      CheckNewAnswer(),
+      // states
+      // left: 21
+      // right: 22
+      assertNumStateRows(total = 2, updated = 0, droppedByWatermark = 1)
+    )
+  }
+
+  test("left semi early state exclusion on left") {
+    val (leftInput, rightInput, joined) = setupWindowedJoinWithLeftCondition("left_semi")
+
+    testStream(joined)(
+      MultiAddData(leftInput, 1, 2, 3)(rightInput, 3, 4, 5),
+      // The left rows with leftValue <= 4 should not generate their semi join rows and
+      // not get added to the state.
+      CheckNewAnswer(Row(3, 10, 6)),
+      // states
+      // left: 3
+      // right: 3, 4, 5
+      assertNumStateRows(total = 4, updated = 4),
+      // We shouldn't get more semi join rows when the watermark advances.
+      MultiAddData(leftInput, 20)(rightInput, 21),
+      CheckNewAnswer(),
+      // states
+      // left: 20
+      // right: 21
+      //
+      // states evicted
+      // left: 3 (below watermark)
+      // right: 3, 4, 5 (below watermark)
+      assertNumStateRows(total = 2, updated = 2),
+      AddData(rightInput, 20),
+      CheckNewAnswer((20, 30, 40)),
+      // states
+      // left: 20
+      // right: 21, 20
+      assertNumStateRows(total = 3, updated = 1)
+    )
+  }
+
+  test("left semi early state exclusion on right") {
+    val (leftInput, rightInput, joined) = setupWindowedJoinWithRightCondition("left_semi")
+
+    testStream(joined)(
+      MultiAddData(leftInput, 3, 4, 5)(rightInput, 1, 2, 3),
+      // The right rows with rightValue <= 7 should never be added to the state.
+      // The right row with rightValue = 9 > 7, hence joined and added to state.
+      CheckNewAnswer(Row(3, 10, 6)),
+      // states
+      // left: 3, 4, 5
+      // right: 3
+      assertNumStateRows(total = 4, updated = 4),
+      // We shouldn't get more semi join rows when the watermark advances.
+      MultiAddData(leftInput, 20)(rightInput, 21),
+      CheckNewAnswer(),
+      // states
+      // left: 20
+      // right: 21
+      //
+      // states evicted
+      // left: 3, 4, 5 (below watermark)
+      // right: 3 (below watermark)
+      assertNumStateRows(total = 2, updated = 2),
+      AddData(rightInput, 20),
+      CheckNewAnswer((20, 30, 40)),
+      // states
+      // left: 20
+      // right: 21, 20
+      assertNumStateRows(total = 3, updated = 1)
+    )
+  }
+
+  test("left semi join with watermark range condition") {
+    val (leftInput, rightInput, joined) = setupJoinWithRangeCondition("left_semi")
+
+    testStream(joined)(
+      AddData(leftInput, (1, 5), (3, 5)),
+      CheckNewAnswer(),
+      // states
+      // left: (1, 5), (3, 5)
+      // right: nothing
+      assertNumStateRows(total = 2, updated = 2),
+      AddData(rightInput, (1, 10), (2, 5)),
+      // Match left row in the state.
+      CheckNewAnswer((1, 5)),
+      // states
+      // left: (1, 5), (3, 5)
+      // right: (1, 10), (2, 5)
+      assertNumStateRows(total = 4, updated = 2),
+      AddData(rightInput, (1, 9)),
+      // No match as left row is already matched.
+      CheckNewAnswer(),
+      // states
+      // left: (1, 5), (3, 5)
+      // right: (1, 10), (2, 5), (1, 9)
+      assertNumStateRows(total = 5, updated = 1),
+      // Increase event time watermark to 20s by adding data with time = 30s on both inputs.
+      AddData(leftInput, (1, 7), (1, 30)),
+      CheckNewAnswer((1, 7)),
+      // states
+      // left: (1, 5), (3, 5), (1, 30)
+      // right: (1, 10), (2, 5), (1, 9)
+      assertNumStateRows(total = 6, updated = 1),
+      // Watermark = 30 - 10 = 20, no matched row.
+      AddData(rightInput, (0, 30)),
+      CheckNewAnswer(),
+      // states
+      // left: (1, 30)
+      // right: (0, 30)
+      //
+      // states evicted
+      // left: (1, 5), (3, 5) (below watermark = 20)
+      // right: (1, 10), (2, 5), (1, 9) (below watermark = 20)
+      assertNumStateRows(total = 2, updated = 1)
+    )
+  }
+
+  test("self left semi join") {
+    val (inputStream, query) = setupSelfJoin("left_semi")
+
+    testStream(query)(
+      AddData(inputStream, (1, 1L), (2, 2L), (3, 3L), (4, 4L), (5, 5L)),
+      CheckNewAnswer((2, 2), (4, 4)),
+      // batch 1 - global watermark = 0
+      // states
+      // left: (2, 2L), (4, 4L)
+      //       (left rows with value % 2 != 0 is filtered per [[PushPredicateThroughJoin]])
+      // right: (2, 2L), (4, 4L)
+      //       (right rows with value % 2 != 0 is filtered per [[PushPredicateThroughJoin]])
+      assertNumStateRows(total = 4, updated = 4),
+      AddData(inputStream, (6, 6L), (7, 7L), (8, 8L), (9, 9L), (10, 10L)),
+      CheckNewAnswer((6, 6), (8, 8), (10, 10)),
+      // batch 2 - global watermark = 5
+      // states
+      // left: (2, 2L), (4, 4L), (6, 6L), (8, 8L), (10, 10L)
+      // right: (6, 6L), (8, 8L), (10, 10L)
+      //
+      // states evicted
+      // left: nothing (it waits for 5 seconds more than watermark due to join condition)
+      // right: (2, 2L), (4, 4L)
+      assertNumStateRows(total = 8, updated = 6),
+      AddData(inputStream, (11, 11L), (12, 12L), (13, 13L), (14, 14L), (15, 15L)),
+      CheckNewAnswer((12, 12), (14, 14)),
+      // batch 3 - global watermark = 9
+      // states
+      // left: (4, 4L), (6, 6L), (8, 8L), (10, 10L), (12, 12L), (14, 14L)
+      // right: (10, 10L), (12, 12L), (14, 14L)
+      //
+      // states evicted
+      // left: (2, 2L)
+      // right: (6, 6L), (8, 8L)
+      assertNumStateRows(total = 9, updated = 4)
+    )
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala
index e585b8a885c9c..156528776d945 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala
@@ -49,7 +49,7 @@ class StreamingQueryListenerSuite extends StreamTest with BeforeAndAfter {
     assert(spark.streams.active.isEmpty)
     // Skip check default `StreamingQueryStatusListener` which is for streaming UI.
     assert(spark.streams.listListeners()
-      .filterNot(_.isInstanceOf[StreamingQueryStatusListener]).isEmpty)
+      .forall(_.isInstanceOf[StreamingQueryStatusListener]))
     // Make sure we don't leak any events to the next test
     spark.sparkContext.listenerBus.waitUntilEmpty()
   }
@@ -323,7 +323,7 @@ class StreamingQueryListenerSuite extends StreamTest with BeforeAndAfter {
         actions += AssertOnQuery { q =>
           q.recentProgress.size > 1 && q.recentProgress.size <= 11
         }
-        testStream(input.toDS)(actions: _*)
+        testStream(input.toDS)(actions.toSeq: _*)
         spark.sparkContext.listenerBus.waitUntilEmpty()
         // 11 is the max value of the possible numbers of events.
         assert(numProgressEvent > 1 && numProgressEvent <= 11)
@@ -389,7 +389,7 @@ class StreamingQueryListenerSuite extends StreamTest with BeforeAndAfter {
     // Structured Streaming in Spark 2.0.0. Because we renamed the classes,
     // SparkListenerApplicationEnd is the only valid event and it's the last event. We use it
     // to verify that we can skip broken jsons generated by Structured Streaming.
-    testReplayListenerBusWithBorkenEventJsons("query-event-logs-version-2.0.0.txt", 1)
+    testReplayListenerBusWithBrokenEventJsons("query-event-logs-version-2.0.0.txt", 1)
   }
 
   testQuietly("ReplayListenerBus should ignore broken event jsons generated in 2_0_1") {
@@ -397,14 +397,14 @@ class StreamingQueryListenerSuite extends StreamTest with BeforeAndAfter {
     // Structured Streaming in Spark 2.0.1. Because we renamed the classes,
     // SparkListenerApplicationEnd is the only valid event and it's the last event. We use it
     // to verify that we can skip broken jsons generated by Structured Streaming.
-    testReplayListenerBusWithBorkenEventJsons("query-event-logs-version-2.0.1.txt", 1)
+    testReplayListenerBusWithBrokenEventJsons("query-event-logs-version-2.0.1.txt", 1)
   }
 
   testQuietly("ReplayListenerBus should ignore broken event jsons generated in 2_0_2") {
     // query-event-logs-version-2.0.2.txt has all types of events generated by
     // Structured Streaming in Spark 2.0.2. SPARK-18516 refactored Structured Streaming query events
     // in 2.1.0. This test is to verify we are able to load events generated by Spark 2.0.2.
-    testReplayListenerBusWithBorkenEventJsons("query-event-logs-version-2.0.2.txt", 5)
+    testReplayListenerBusWithBrokenEventJsons("query-event-logs-version-2.0.2.txt", 5)
   }
 
   test("listener propagates observable metrics") {
@@ -433,9 +433,13 @@ class StreamingQueryListenerSuite extends StreamTest with BeforeAndAfter {
     }
 
     try {
+      val noDataProgressIntervalKey = SQLConf.STREAMING_NO_DATA_PROGRESS_EVENT_INTERVAL.key
       spark.streams.addListener(listener)
       testStream(df, OutputMode.Append)(
-        StartStream(Trigger.ProcessingTime(100), triggerClock = clock),
+        StartStream(
+          Trigger.ProcessingTime(100),
+          triggerClock = clock,
+          Map(noDataProgressIntervalKey -> "100")),
         // Batch 1
         AddData(inputData, 1, 2),
         AdvanceManualClock(100),
@@ -464,7 +468,49 @@ class StreamingQueryListenerSuite extends StreamTest with BeforeAndAfter {
     }
   }
 
-  private def testReplayListenerBusWithBorkenEventJsons(
+  test("SPARK-31593: remove unnecessary streaming query progress update") {
+    withSQLConf(SQLConf.STREAMING_NO_DATA_PROGRESS_EVENT_INTERVAL.key -> "100") {
+      @volatile var numProgressEvent = 0
+      val listener = new StreamingQueryListener {
+        override def onQueryStarted(event: QueryStartedEvent): Unit = {}
+        override def onQueryProgress(event: QueryProgressEvent): Unit = {
+          numProgressEvent += 1
+        }
+        override def onQueryTerminated(event: QueryTerminatedEvent): Unit = {}
+      }
+      spark.streams.addListener(listener)
+
+      def checkProgressEvent(count: Int): StreamAction = {
+        AssertOnQuery { _ =>
+          eventually(Timeout(streamingTimeout)) {
+            assert(numProgressEvent == count)
+          }
+          true
+        }
+      }
+
+      try {
+        val input = new MemoryStream[Int](0, sqlContext)
+        val clock = new StreamManualClock()
+        val result = input.toDF().select("value")
+        testStream(result)(
+          StartStream(trigger = Trigger.ProcessingTime(10), triggerClock = clock),
+          AddData(input, 10),
+          checkProgressEvent(1),
+          AdvanceManualClock(10),
+          checkProgressEvent(2),
+          AdvanceManualClock(90),
+          checkProgressEvent(2),
+          AdvanceManualClock(10),
+          checkProgressEvent(3)
+        )
+      } finally {
+        spark.streams.removeListener(listener)
+      }
+    }
+  }
+
+  private def testReplayListenerBusWithBrokenEventJsons(
       fileName: String,
       expectedEventSize: Int): Unit = {
     val input = getClass.getResourceAsStream(s"/structured-streaming/$fileName")
@@ -513,11 +559,11 @@ class StreamingQueryListenerSuite extends StreamTest with BeforeAndAfter {
     private val _progressEvents = new mutable.Queue[StreamingQueryProgress]
 
     def progressEvents: Seq[StreamingQueryProgress] = _progressEvents.synchronized {
-      _progressEvents.filter(_.numInputRows > 0)
+      _progressEvents.filter(_.numInputRows > 0).toSeq
     }
 
     def allProgressEvents: Seq[StreamingQueryProgress] = _progressEvents.synchronized {
-      _progressEvents.clone()
+      _progressEvents.clone().toSeq
     }
 
     def reset(): Unit = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryStatusAndProgressSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryStatusAndProgressSuite.scala
index 08b3644745f9a..c0aefb8120808 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryStatusAndProgressSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryStatusAndProgressSuite.scala
@@ -64,6 +64,7 @@ class StreamingQueryStatusAndProgressSuite extends StreamTest with Eventually {
         |    "numRowsTotal" : 0,
         |    "numRowsUpdated" : 1,
         |    "memoryUsedBytes" : 3,
+        |    "numRowsDroppedByWatermark" : 0,
         |    "customMetrics" : {
         |      "loadedMapCacheHitCount" : 1,
         |      "loadedMapCacheMissCount" : 0,
@@ -74,6 +75,7 @@ class StreamingQueryStatusAndProgressSuite extends StreamTest with Eventually {
         |    "description" : "source",
         |    "startOffset" : 123,
         |    "endOffset" : 456,
+        |    "latestOffset" : 789,
         |    "numInputRows" : 678,
         |    "inputRowsPerSecond" : 10.0
         |  } ],
@@ -113,12 +115,14 @@ class StreamingQueryStatusAndProgressSuite extends StreamTest with Eventually {
          |  "stateOperators" : [ {
          |    "numRowsTotal" : 0,
          |    "numRowsUpdated" : 1,
-         |    "memoryUsedBytes" : 2
+         |    "memoryUsedBytes" : 2,
+         |    "numRowsDroppedByWatermark" : 0
          |  } ],
          |  "sources" : [ {
          |    "description" : "source",
          |    "startOffset" : 123,
          |    "endOffset" : 456,
+         |    "latestOffset" : 789,
          |    "numInputRows" : 678
          |  } ],
          |  "sink" : {
@@ -314,23 +318,24 @@ object StreamingQueryStatusAndProgressSuite {
     timestamp = "2016-12-05T20:54:20.827Z",
     batchId = 2L,
     batchDuration = 0L,
-    durationMs = new java.util.HashMap(Map("total" -> 0L).mapValues(long2Long).asJava),
+    durationMs = new java.util.HashMap(Map("total" -> 0L).mapValues(long2Long).toMap.asJava),
     eventTime = new java.util.HashMap(Map(
       "max" -> "2016-12-05T20:54:20.827Z",
       "min" -> "2016-12-05T20:54:20.827Z",
       "avg" -> "2016-12-05T20:54:20.827Z",
       "watermark" -> "2016-12-05T20:54:20.827Z").asJava),
     stateOperators = Array(new StateOperatorProgress(
-      numRowsTotal = 0, numRowsUpdated = 1, memoryUsedBytes = 3,
+      numRowsTotal = 0, numRowsUpdated = 1, memoryUsedBytes = 3, numRowsDroppedByWatermark = 0,
       customMetrics = new java.util.HashMap(Map("stateOnCurrentVersionSizeBytes" -> 2L,
         "loadedMapCacheHitCount" -> 1L, "loadedMapCacheMissCount" -> 0L)
-        .mapValues(long2Long).asJava)
+        .mapValues(long2Long).toMap.asJava)
     )),
     sources = Array(
       new SourceProgress(
         description = "source",
         startOffset = "123",
         endOffset = "456",
+        latestOffset = "789",
         numInputRows = 678,
         inputRowsPerSecond = 10.0,
         processedRowsPerSecond = Double.PositiveInfinity  // should not be present in the json
@@ -349,16 +354,17 @@ object StreamingQueryStatusAndProgressSuite {
     timestamp = "2016-12-05T20:54:20.827Z",
     batchId = 2L,
     batchDuration = 0L,
-    durationMs = new java.util.HashMap(Map("total" -> 0L).mapValues(long2Long).asJava),
+    durationMs = new java.util.HashMap(Map("total" -> 0L).mapValues(long2Long).toMap.asJava),
     // empty maps should be handled correctly
     eventTime = new java.util.HashMap(Map.empty[String, String].asJava),
     stateOperators = Array(new StateOperatorProgress(
-      numRowsTotal = 0, numRowsUpdated = 1, memoryUsedBytes = 2)),
+      numRowsTotal = 0, numRowsUpdated = 1, memoryUsedBytes = 2, numRowsDroppedByWatermark = 0)),
     sources = Array(
       new SourceProgress(
         description = "source",
         startOffset = "123",
         endOffset = "456",
+        latestOffset = "789",
         numInputRows = 678,
         inputRowsPerSecond = Double.NaN, // should not be present in the json
         processedRowsPerSecond = Double.NegativeInfinity // should not be present in the json
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala
index 77f5c856ff0f4..9c2403dffbb1a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala
@@ -33,8 +33,11 @@ import org.scalatestplus.mockito.MockitoSugar
 
 import org.apache.spark.{SparkException, TestUtils}
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.{Column, DataFrame, Dataset, Row}
+import org.apache.spark.sql.{AnalysisException, Column, DataFrame, Dataset, Row}
+import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{Literal, Rand, Randn, Shuffle, Uuid}
+import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
+import org.apache.spark.sql.catalyst.streaming.InternalOutputModes.Complete
 import org.apache.spark.sql.connector.read.InputPartition
 import org.apache.spark.sql.connector.read.streaming.{Offset => OffsetV2}
 import org.apache.spark.sql.execution.exchange.ReusedExchangeExec
@@ -702,7 +705,7 @@ class StreamingQuerySuite extends StreamTest with BeforeAndAfter with Logging wi
     val q2 = startQuery(input(1).toDS.map { i =>
       // Emulate that `StreamingQuery` get captured with normal usage unintentionally.
       // It should not fail the query.
-      q1
+      val q = q1
       i
     }, "stream_serializable_test_2")
     val q3 = startQuery(input(2).toDS.map { i =>
@@ -1106,6 +1109,90 @@ class StreamingQuerySuite extends StreamTest with BeforeAndAfter with Logging wi
     }
   }
 
+  test("SPARK-32456: SQL union in streaming query of append mode without watermark") {
+    val inputData1 = MemoryStream[Int]
+    val inputData2 = MemoryStream[Int]
+    withTempView("s1", "s2") {
+      inputData1.toDF().createOrReplaceTempView("s1")
+      inputData2.toDF().createOrReplaceTempView("s2")
+      val unioned = spark.sql(
+        "select s1.value from s1 union select s2.value from s2")
+      checkExceptionMessage(unioned)
+    }
+  }
+
+  test("SPARK-32456: distinct in streaming query of append mode without watermark") {
+    val inputData = MemoryStream[Int]
+    withTempView("deduptest") {
+      inputData.toDF().toDF("value").createOrReplaceTempView("deduptest")
+      val distinct = spark.sql("select distinct value from deduptest")
+      checkExceptionMessage(distinct)
+    }
+  }
+
+  test("SPARK-32456: distinct in streaming query of complete mode") {
+    val inputData = MemoryStream[Int]
+    withTempView("deduptest") {
+      inputData.toDF().toDF("value").createOrReplaceTempView("deduptest")
+      val distinct = spark.sql("select distinct value from deduptest")
+
+      testStream(distinct, Complete)(
+        AddData(inputData, 1, 2, 3, 3, 4),
+        CheckAnswer(Row(1), Row(2), Row(3), Row(4))
+      )
+    }
+  }
+
+  testQuietly("limit on empty batch should not cause state store error") {
+    // The source only produces two batches, the first batch is empty and the second batch has data.
+    val source = new Source {
+      var batchId = 0
+      override def stop(): Unit = {}
+      override def getOffset: Option[Offset] = {
+        Some(LongOffset(batchId + 1))
+      }
+      override def getBatch(start: Option[Offset], end: Offset): DataFrame = {
+        if (batchId == 0) {
+          batchId += 1
+          Dataset.ofRows(spark, LocalRelation(schema.toAttributes, Nil, isStreaming = true))
+        } else {
+          Dataset.ofRows(spark,
+            LocalRelation(schema.toAttributes, InternalRow(10) :: Nil, isStreaming = true))
+        }
+      }
+      override def schema: StructType = MockSourceProvider.fakeSchema
+    }
+
+    MockSourceProvider.withMockSources(source) {
+      val df = spark.readStream
+        .format("org.apache.spark.sql.streaming.util.MockSourceProvider")
+        .load()
+        .limit(1)
+
+      testStream(df)(
+        StartStream(),
+        AssertOnQuery { q =>
+          q.processAllAvailable()
+          true
+        },
+        CheckAnswer(10))
+    }
+  }
+
+  private def checkExceptionMessage(df: DataFrame): Unit = {
+    withTempDir { outputDir =>
+      withTempDir { checkpointDir =>
+        val exception = intercept[AnalysisException](
+          df.writeStream
+            .option("checkpointLocation", checkpointDir.getCanonicalPath)
+            .start(outputDir.getCanonicalPath))
+        assert(exception.getMessage.contains(
+          "Append output mode not supported when there are streaming aggregations on streaming " +
+            "DataFrames/DataSets without watermark"))
+      }
+    }
+  }
+
   /** Create a streaming DF that only execute one batch in which it returns the given static DF */
   private def createSingleTriggerStreamingDF(triggerDF: DataFrame): DataFrame = {
     require(!triggerDF.isStreaming)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingStateStoreFormatCompatibilitySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingStateStoreFormatCompatibilitySuite.scala
new file mode 100644
index 0000000000000..1032d6c5b6ff2
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingStateStoreFormatCompatibilitySuite.scala
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.streaming
+
+import java.io.File
+
+import scala.annotation.tailrec
+
+import org.apache.commons.io.FileUtils
+
+import org.apache.spark.SparkException
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.streaming.InternalOutputModes.Complete
+import org.apache.spark.sql.execution.streaming.MemoryStream
+import org.apache.spark.sql.execution.streaming.state.{InvalidUnsafeRowException, StateSchemaNotCompatible}
+import org.apache.spark.sql.functions._
+import org.apache.spark.util.Utils
+
+/**
+ * An integrated test for streaming state store format compatibility.
+ * For each PR breaks this test, we need to pay attention to the underlying unsafe row format
+ * changing. All the checkpoint dirs were generated based on Spark version 2.4.5. If we accept the
+ * changes, it means the checkpoint for Structured Streaming will become non-reusable. Please add
+ * a new test for the issue, just like the test suite "SPARK-28067 changed the sum decimal unsafe
+ * row format".
+ */
+class StreamingStateStoreFormatCompatibilitySuite extends StreamTest {
+  import testImplicits._
+
+  private def prepareCheckpointDir(testName: String): File = {
+    val resourceUri = this.getClass.getResource("/structured-streaming/" +
+      s"checkpoint-version-2.4.5-for-compatibility-test-${testName}").toURI
+    val checkpointDir = Utils.createTempDir().getCanonicalFile
+    FileUtils.copyDirectory(new File(resourceUri), checkpointDir)
+    checkpointDir
+  }
+
+  test("common functions") {
+    val inputData = MemoryStream[Int]
+    val aggregated =
+      inputData.toDF().toDF("value")
+      .selectExpr(
+        "value",
+        "value % 5 AS id",
+        "CAST(value AS STRING) as str",
+        "CAST(value AS FLOAT) as f",
+        "CAST(value AS DOUBLE) as d",
+        "CAST(value AS DECIMAL) as dec",
+        "value % 3 AS mod",
+        "named_struct('key', CAST(value AS STRING), 'value', value) AS s")
+      .groupBy($"id")
+      .agg(
+        avg($"value").as("avg_v"),
+        avg($"f").as("avg_f"),
+        avg($"d").as("avg_d"),
+        avg($"dec").as("avg_dec"),
+        count($"value").as("cnt"),
+        first($"value").as("first_v"),
+        first($"s").as("first_s"),
+        last($"value").as("last_v"),
+        last($"s").as("last_s"),
+        min(struct("value", "str")).as("min_struct"),
+        max($"value").as("max_v"),
+        sum($"value").as("sum_v"),
+        sum($"f").as("sum_f"),
+        sum($"d").as("sum_d"),
+        // The test for sum decimal broke by SPARK-28067, use separated test for it
+        // sum($"dec").as("sum_dec"),
+        collect_list($"value").as("col_list"),
+        collect_set($"mod").as("col_set"))
+      .select("id", "avg_v", "avg_f", "avg_d", "avg_dec", "cnt", "first_v", "first_s.value",
+        "last_v", "last_s.value", "min_struct.value", "max_v", "sum_v", "sum_f", "sum_d",
+        "col_list", "col_set")
+
+    val checkpointDir = prepareCheckpointDir("common-functions")
+    inputData.addData(0 to 9: _*)
+
+    testStream(aggregated, Complete)(
+      StartStream(checkpointLocation = checkpointDir.getAbsolutePath),
+      /*
+        Note: The checkpoint was generated using the following input in Spark version 2.4.5
+        AddData(inputData, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9),
+        CheckAnswer(
+          Row(0, 2.5, 2.5F, 2.5, 2.5000, 2, 0, 0, 5, 5, 0, 5, 5, 5.0, 5.0, Seq(0, 5),
+            Seq(0, 2)),
+          Row(1, 3.5, 3.5F, 3.5, 3.5000, 2, 1, 1, 6, 6, 1, 6, 7, 7.0, 7.0, Seq(1, 6),
+            Seq(0, 1)),
+          Row(2, 4.5, 4.5F, 4.5, 4.5000, 2, 2, 2, 7, 7, 2, 7, 9, 9.0, 9.0, Seq(2, 7),
+            Seq(1, 2)),
+          Row(3, 5.5, 5.5F, 5.5, 5.5000, 2, 3, 3, 8, 8, 3, 8, 11, 11.0, 11.0, Seq(3, 8),
+            Seq(0, 2)),
+          Row(4, 6.5, 6.5F, 6.5, 6.5000, 2, 4, 4, 9, 9, 4, 9, 13, 13.0, 13.0, Seq(4, 9),
+            Seq(0, 1)))
+       */
+      AddData(inputData, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19),
+      CheckAnswer(
+        Row(0, 7.5, 7.5, 7.5, 7.5000, 4, 0, 0, 15, 15, 0, 15, 30, 30.0, 30.0,
+          Seq(0, 5, 10, 15), Seq(0, 1, 2)),
+        Row(1, 8.5, 8.5, 8.5, 8.5000, 4, 1, 1, 16, 16, 1, 16, 34, 34.0, 34.0,
+          Seq(1, 6, 11, 16), Seq(0, 1, 2)),
+        Row(2, 9.5, 9.5, 9.5, 9.5000, 4, 2, 2, 17, 17, 2, 17, 38, 38.0, 38.0,
+          Seq(2, 7, 12, 17), Seq(0, 1, 2)),
+        Row(3, 10.5, 10.5, 10.5, 10.5000, 4, 3, 3, 18, 18, 3, 18, 42, 42.0, 42.0,
+          Seq(3, 8, 13, 18), Seq(0, 1, 2)),
+        Row(4, 11.5, 11.5, 11.5, 11.5000, 4, 4, 4, 19, 19, 4, 19, 46, 46.0, 46.0,
+          Seq(4, 9, 14, 19), Seq(0, 1, 2)))
+    )
+  }
+
+  test("statistical functions") {
+    val inputData = MemoryStream[Long]
+    val aggregated =
+      inputData.toDF().toDF("value")
+        .selectExpr(
+          "value",
+          "value % 5 AS id",
+          "CAST(value AS STRING) as str",
+          "CAST(value AS FLOAT) as f",
+          "CAST(value AS DOUBLE) as d",
+          "CAST(value AS DECIMAL) as dec",
+          "value % 3 AS mod")
+        .groupBy($"id")
+        .agg(
+          kurtosis($"d").as("kts"),
+          skewness($"d").as("skew"),
+          approx_count_distinct($"mod").as("approx_cnt"),
+          approx_count_distinct($"f").as("approx_cnt_f"),
+          approx_count_distinct($"d").as("approx_cnt_d"),
+          approx_count_distinct($"dec").as("approx_cnt_dec"),
+          approx_count_distinct($"str").as("approx_cnt_str"),
+          stddev_pop($"d").as("stddev_pop"),
+          stddev_samp($"d").as("stddev_samp"),
+          var_pop($"d").as("var_pop"),
+          var_samp($"d").as("var_samp"),
+          covar_pop($"value", $"mod").as("covar_pop"),
+          covar_samp($"value", $"mod").as("covar_samp"),
+          corr($"value", $"mod").as("corr"))
+        .select("id", "kts", "skew", "approx_cnt", "approx_cnt_f", "approx_cnt_d",
+          "approx_cnt_dec", "approx_cnt_str", "stddev_pop", "stddev_samp", "var_pop", "var_samp",
+          "covar_pop", "covar_samp", "corr")
+
+    val checkpointDir = prepareCheckpointDir("statistical-functions")
+    inputData.addData(0L to 9L: _*)
+
+    testStream(aggregated, Complete)(
+      StartStream(checkpointLocation = checkpointDir.getAbsolutePath),
+      /*
+        Note: The checkpoint was generated using the following input in Spark version 2.4.5
+        AddData(inputData, 0L to 9L: _*),
+        CheckAnswer(
+          Row(0, -2.0, 0.0, 2, 2, 2, 2, 2, 2.5, 3.5355339059327378, 6.25, 12.5, 2.5, 5.0, 1.0),
+          Row(1, -2.0, 0.0, 2, 2, 2, 2, 2, 2.5, 3.5355339059327378, 6.25, 12.5, -1.25, -2.5, -1.0),
+          Row(2, -2.0, 0.0, 2, 2, 2, 2, 2, 2.5, 3.5355339059327378, 6.25, 12.5, -1.25, -2.5, -1.0),
+          Row(3, -2.0, 0.0, 2, 2, 2, 2, 2, 2.5, 3.5355339059327378, 6.25, 12.5, 2.5, 5.0, 1.0),
+          Row(4, -2.0, 0.0, 2, 2, 2, 2, 2, 2.5, 3.5355339059327378, 6.25, 12.5, -1.25, -2.5, -1.0))
+       */
+
+      AddData(inputData, 10L to 19L: _*),
+      CheckAnswer(
+        Row(0, -1.36, 0.0, 3, 4, 4, 4, 4, 5.5901699437494745, 6.454972243679028, 31.25,
+          41.666666666666664, -0.625, -0.8333333333333334, -0.13483997249264842),
+        Row(1, -1.36, 0.0, 3, 4, 4, 4, 4, 5.5901699437494745, 6.454972243679028, 31.25,
+          41.666666666666664, 1.25, 1.6666666666666667, 0.31622776601683794),
+        Row(2, -1.36, 0.0, 3, 4, 4, 4, 4, 5.5901699437494745, 6.454972243679028, 31.25,
+          41.666666666666664, -0.625, -0.8333333333333334, -0.13483997249264842),
+        Row(3, -1.36, 0.0, 3, 4, 4, 4, 4, 5.5901699437494745, 6.454972243679028, 31.25,
+          41.666666666666664, -0.625, -0.8333333333333334, -0.13483997249264842),
+        Row(4, -1.36, 0.0, 3, 4, 4, 4, 4, 5.5901699437494745, 6.454972243679028, 31.25,
+          41.666666666666664, 1.25, 1.6666666666666667, 0.31622776601683794))
+    )
+  }
+
+  test("deduplicate with all columns") {
+    val inputData = MemoryStream[Long]
+    val result = inputData.toDF().toDF("value")
+      .selectExpr(
+        "value",
+        "value + 10 AS key",
+        "CAST(value AS STRING) as topic",
+        "value + 100 AS partition",
+        "value + 5 AS offset")
+      .dropDuplicates()
+      .select("key", "value", "topic", "partition", "offset")
+
+    val checkpointDir = prepareCheckpointDir("deduplicate")
+    inputData.addData(0L, 1L, 2L, 3L, 4L)
+
+    testStream(result)(
+      StartStream(checkpointLocation = checkpointDir.getAbsolutePath),
+      /*
+        Note: The checkpoint was generated using the following input in Spark version 2.4.5
+        AddData(inputData, 0L, 1L, 2L, 3L, 4L),
+        CheckAnswer(
+          Row(10, 0, "0", 100, 5),
+          Row(11, 1, "1", 101, 6),
+          Row(12, 2, "2", 102, 7),
+          Row(13, 3, "3", 103, 8),
+          Row(14, 4, "4", 104, 9))
+       */
+      AddData(inputData, 3L, 4L, 5L, 6L),
+      CheckLastBatch(
+        Row(15, 5, "5", 105, 10),
+        Row(16, 6, "6", 106, 11))
+    )
+  }
+
+  test("SPARK-28067 changed the sum decimal unsafe row format") {
+    val inputData = MemoryStream[Int]
+    val aggregated =
+      inputData.toDF().toDF("value")
+        .selectExpr(
+          "value",
+          "value % 2 AS id",
+          "CAST(value AS DECIMAL) as dec")
+        .groupBy($"id")
+        .agg(sum($"dec").as("sum_dec"), collect_list($"value").as("col_list"))
+        .select("id", "sum_dec", "col_list")
+
+    val checkpointDir = prepareCheckpointDir("sum-decimal")
+    inputData.addData(0 to 9: _*)
+
+    testStream(aggregated, Complete)(
+      StartStream(checkpointLocation = checkpointDir.getAbsolutePath),
+      /*
+        Note: The checkpoint was generated using the following input in Spark version 2.4.5
+        AddData(inputData, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9),
+        CheckAnswer(Row(0, 20, Seq(0, 2, 4, 6, 8)), Row(1, 25, Seq(1, 3, 5, 7, 9)))
+       */
+      AddData(inputData, 10 to 19: _*),
+      ExpectFailure[SparkException] { e =>
+        assert(findStateSchemaException(e))
+      }
+    )
+  }
+
+  @tailrec
+  private def findStateSchemaException(exc: Throwable): Boolean = {
+    exc match {
+      case _: StateSchemaNotCompatible => true
+      case _: InvalidUnsafeRowException => true
+      case e1 if e1.getCause != null => findStateSchemaException(e1.getCause)
+      case _ => false
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousAggregationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousAggregationSuite.scala
deleted file mode 100644
index 3ec4750c59fc5..0000000000000
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousAggregationSuite.scala
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.streaming.continuous
-
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.execution.streaming.sources.ContinuousMemoryStream
-import org.apache.spark.sql.functions._
-import org.apache.spark.sql.internal.SQLConf.UNSUPPORTED_OPERATION_CHECK_ENABLED
-import org.apache.spark.sql.streaming.OutputMode
-
-class ContinuousAggregationSuite extends ContinuousSuiteBase {
-  import testImplicits._
-
-  test("not enabled") {
-    val ex = intercept[AnalysisException] {
-      val input = ContinuousMemoryStream.singlePartition[Int]
-      testStream(input.toDF().agg(max('value)), OutputMode.Complete)()
-    }
-
-    assert(ex.getMessage.contains(
-      "In continuous processing mode, coalesce(1) must be called before aggregate operation"))
-  }
-
-  test("basic") {
-    withSQLConf((UNSUPPORTED_OPERATION_CHECK_ENABLED.key, "false")) {
-      val input = ContinuousMemoryStream.singlePartition[Int]
-
-      testStream(input.toDF().agg(max('value)), OutputMode.Complete)(
-        AddData(input, 0, 1, 2),
-        CheckAnswer(2),
-        StopStream,
-        AddData(input, 3, 4, 5),
-        StartStream(),
-        CheckAnswer(5),
-        AddData(input, -1, -2, -3),
-        CheckAnswer(5))
-    }
-  }
-
-  test("multiple partitions with coalesce") {
-    val input = ContinuousMemoryStream[Int]
-
-    val df = input.toDF().coalesce(1).agg(max('value))
-
-    testStream(df, OutputMode.Complete)(
-      AddData(input, 0, 1, 2),
-      CheckAnswer(2),
-      StopStream,
-      AddData(input, 3, 4, 5),
-      StartStream(),
-      CheckAnswer(5),
-      AddData(input, -1, -2, -3),
-      CheckAnswer(5))
-  }
-
-  test("multiple partitions with coalesce - multiple transformations") {
-    val input = ContinuousMemoryStream[Int]
-
-    // We use a barrier to make sure predicates both before and after coalesce work
-    val df = input.toDF()
-      .select('value as 'copy, 'value)
-      .where('copy =!= 1)
-      .logicalPlan
-      .coalesce(1)
-      .where('copy =!= 2)
-      .agg(max('value))
-
-    testStream(df, OutputMode.Complete)(
-      AddData(input, 0, 1, 2),
-      CheckAnswer(0),
-      StopStream,
-      AddData(input, 3, 4, 5),
-      StartStream(),
-      CheckAnswer(5),
-      AddData(input, -1, -2, -3),
-      CheckAnswer(5))
-  }
-
-  test("multiple partitions with multiple coalesce") {
-    val input = ContinuousMemoryStream[Int]
-
-    val df = input.toDF()
-      .coalesce(1)
-      .logicalPlan
-      .coalesce(1)
-      .select('value as 'copy, 'value)
-      .agg(max('value))
-
-    testStream(df, OutputMode.Complete)(
-      AddData(input, 0, 1, 2),
-      CheckAnswer(2),
-      StopStream,
-      AddData(input, 3, 4, 5),
-      StartStream(),
-      CheckAnswer(5),
-      AddData(input, -1, -2, -3),
-      CheckAnswer(5))
-  }
-
-  test("repeated restart") {
-    withSQLConf((UNSUPPORTED_OPERATION_CHECK_ENABLED.key, "false")) {
-      val input = ContinuousMemoryStream.singlePartition[Int]
-
-      testStream(input.toDF().agg(max('value)), OutputMode.Complete)(
-        AddData(input, 0, 1, 2),
-        CheckAnswer(2),
-        StopStream,
-        StartStream(),
-        StopStream,
-        StartStream(),
-        StopStream,
-        StartStream(),
-        AddData(input, 0),
-        CheckAnswer(2),
-        AddData(input, 5),
-        CheckAnswer(5))
-    }
-  }
-}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousSuite.scala
index 0d17f2e0bc7fb..0e2fcfbd46356 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousSuite.scala
@@ -22,7 +22,6 @@ import java.sql.Timestamp
 import org.apache.spark.{SparkContext, SparkException}
 import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskStart}
 import org.apache.spark.sql._
-import org.apache.spark.sql.execution.datasources.v2.ContinuousScanExec
 import org.apache.spark.sql.execution.streaming._
 import org.apache.spark.sql.execution.streaming.continuous._
 import org.apache.spark.sql.execution.streaming.sources.ContinuousMemoryStream
@@ -55,27 +54,31 @@ class ContinuousSuiteBase extends StreamTest {
 
   protected def waitForRateSourceCommittedValue(
       query: ContinuousExecution,
-      desiredValue: Long,
+      partitionIdToDesiredValue: Map[Int, Long],
       maxWaitTimeMs: Long): Unit = {
-    def readHighestCommittedValue(c: ContinuousExecution): Option[Long] = {
+    def readCommittedValues(c: ContinuousExecution): Option[Map[Int, Long]] = {
       c.committedOffsets.lastOption.map { case (_, offset) =>
         offset match {
           case o: RateStreamOffset =>
-            o.partitionToValueAndRunTimeMs.map {
-              case (_, ValueRunTimeMsPair(value, _)) => value
-            }.max
+            o.partitionToValueAndRunTimeMs.mapValues(_.value).toMap
         }
       }
     }
 
+    def reachDesiredValues: Boolean = {
+      val committedValues = readCommittedValues(query).getOrElse(Map.empty)
+      partitionIdToDesiredValue.forall { case (key, value) =>
+        committedValues.contains(key) && committedValues(key) > value
+      }
+    }
+
     val maxWait = System.currentTimeMillis() + maxWaitTimeMs
-    while (System.currentTimeMillis() < maxWait &&
-      readHighestCommittedValue(query).getOrElse(Long.MinValue) < desiredValue) {
+    while (System.currentTimeMillis() < maxWait && !reachDesiredValues) {
       Thread.sleep(100)
     }
     if (System.currentTimeMillis() > maxWait) {
       logWarning(s"Couldn't reach desired value in $maxWaitTimeMs milliseconds!" +
-        s"Current highest committed value is ${readHighestCommittedValue(query)}")
+        s"Current committed values is ${readCommittedValues(query)}")
     }
   }
 
@@ -265,7 +268,7 @@ class ContinuousSuite extends ContinuousSuiteBase {
     val expected = Set(0, 1, 2, 3)
     val continuousExecution =
       query.asInstanceOf[StreamingQueryWrapper].streamingQuery.asInstanceOf[ContinuousExecution]
-    waitForRateSourceCommittedValue(continuousExecution, expected.max, 20 * 1000)
+    waitForRateSourceCommittedValue(continuousExecution, Map(0 -> 2, 1 -> 3), 20 * 1000)
     query.stop()
 
     val results = spark.read.table("noharness").collect()
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/sources/StreamingDataSourceV2Suite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/sources/StreamingDataSourceV2Suite.scala
index 05cf324f8d490..c360ec8e670bd 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/sources/StreamingDataSourceV2Suite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/sources/StreamingDataSourceV2Suite.scala
@@ -25,9 +25,10 @@ import scala.collection.JavaConverters._
 import org.apache.spark.sql.{DataFrame, SQLContext}
 import org.apache.spark.sql.connector.catalog.{SessionConfigSupport, SupportsRead, SupportsWrite, Table, TableCapability, TableProvider}
 import org.apache.spark.sql.connector.catalog.TableCapability._
+import org.apache.spark.sql.connector.expressions.Transform
 import org.apache.spark.sql.connector.read.{InputPartition, PartitionReaderFactory, Scan, ScanBuilder}
 import org.apache.spark.sql.connector.read.streaming.{ContinuousPartitionReaderFactory, ContinuousStream, MicroBatchStream, Offset, PartitionOffset}
-import org.apache.spark.sql.connector.write.{LogicalWriteInfo, PhysicalWriteInfo, WriteBuilder, WriterCommitMessage}
+import org.apache.spark.sql.connector.write.{LogicalWriteInfo, PhysicalWriteInfo, Write, WriteBuilder, WriterCommitMessage}
 import org.apache.spark.sql.connector.write.streaming.{StreamingDataWriterFactory, StreamingWrite}
 import org.apache.spark.sql.execution.datasources.DataSource
 import org.apache.spark.sql.execution.streaming.{ContinuousTrigger, RateStreamOffset, Sink, StreamingQueryWrapper}
@@ -67,8 +68,15 @@ class FakeScanBuilder extends ScanBuilder with Scan {
   override def toContinuousStream(checkpointLocation: String): ContinuousStream = new FakeDataStream
 }
 
-class FakeWriteBuilder extends WriteBuilder with StreamingWrite {
-  override def buildForStreaming(): StreamingWrite = this
+class FakeWriteBuilder extends WriteBuilder {
+  override def build(): Write = {
+    new Write {
+      override def toStreaming: StreamingWrite = new FakeStreamingWrite
+    }
+  }
+}
+
+class FakeStreamingWrite extends StreamingWrite {
   override def createStreamingWriterFactory(
       info: PhysicalWriteInfo): StreamingDataWriterFactory = {
     throw new IllegalStateException("fake sink - cannot actually write")
@@ -195,6 +203,30 @@ class FakeNoWrite extends DataSourceRegister with SimpleTableProvider {
   }
 }
 
+class FakeWriteSupportingExternalMetadata
+    extends DataSourceRegister
+    with TableProvider {
+  override def shortName(): String = "fake-write-supporting-external-metadata"
+
+  override def supportsExternalMetadata(): Boolean = true
+
+  override def inferSchema(options: CaseInsensitiveStringMap): StructType = {
+    throw new IllegalArgumentException(
+      "Data stream writer should not require inferring table schema the data source supports" +
+      " external Metadata.")
+  }
+
+  override def getTable(
+      tableSchema: StructType,
+      partitioning: Array[Transform],
+      properties: util.Map[String, String]): Table = {
+    new Table with FakeStreamingWriteTable {
+      override def name(): String = "fake"
+      override def schema(): StructType = tableSchema
+    }
+  }
+}
+
 case class FakeWriteV1FallbackException() extends Exception
 
 class FakeSink extends Sink {
@@ -265,7 +297,7 @@ class StreamingDataSourceV2Suite extends StreamTest {
     Trigger.Continuous(1000))
 
   private def testPositiveCase(readFormat: String, writeFormat: String, trigger: Trigger): Unit = {
-    testPositiveCaseWithQuery(readFormat, writeFormat, trigger)(() => _)
+    testPositiveCaseWithQuery(readFormat, writeFormat, trigger)(_ => ())
   }
 
   private def testPositiveCaseWithQuery(
@@ -314,6 +346,17 @@ class StreamingDataSourceV2Suite extends StreamTest {
     }
   }
 
+  test("SPARK-33369: Skip schema inference in DataStreamWriter.start() if table provider " +
+    "supports external metadata") {
+    testPositiveCaseWithQuery(
+      "fake-read-microbatch-continuous", "fake-write-supporting-external-metadata",
+      Trigger.Once()) { v2Query =>
+      val sink = v2Query.asInstanceOf[StreamingQueryWrapper].streamingQuery.sink
+      assert(sink.isInstanceOf[Table])
+      assert(sink.asInstanceOf[Table].schema() == StructType(Nil))
+    }
+  }
+
   test("disabled v2 write") {
     // Ensure the V2 path works normally and generates a V2 sink..
     testPositiveCaseWithQuery(
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
index f9fc540c2ab80..5e62db08345b2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
@@ -43,11 +43,13 @@ object LastOptions {
   var mockStreamSourceProvider = mock(classOf[StreamSourceProvider])
   var mockStreamSinkProvider = mock(classOf[StreamSinkProvider])
   var parameters: Map[String, String] = null
+  var sinkParameters: Map[String, String] = null
   var schema: Option[StructType] = null
   var partitionColumns: Seq[String] = Nil
 
   def clear(): Unit = {
     parameters = null
+    sinkParameters = null
     schema = null
     partitionColumns = null
     reset(mockStreamSourceProvider)
@@ -87,8 +89,6 @@ class DefaultSource extends StreamSourceProvider with StreamSinkProvider {
       override def getOffset: Option[Offset] = Some(new LongOffset(0))
 
       override def getBatch(start: Option[Offset], end: Offset): DataFrame = {
-        import spark.implicits._
-
         spark.internalCreateDataFrame(spark.sparkContext.emptyRDD, schema, isStreaming = true)
       }
 
@@ -101,7 +101,7 @@ class DefaultSource extends StreamSourceProvider with StreamSinkProvider {
       parameters: Map[String, String],
       partitionColumns: Seq[String],
       outputMode: OutputMode): Sink = {
-    LastOptions.parameters = parameters
+    LastOptions.sinkParameters = parameters
     LastOptions.partitionColumns = partitionColumns
     LastOptions.mockStreamSinkProvider.createSink(spark, parameters, partitionColumns, outputMode)
     (_: Long, _: DataFrame) => {}
@@ -109,6 +109,7 @@ class DefaultSource extends StreamSourceProvider with StreamSinkProvider {
 }
 
 class DataStreamReaderWriterSuite extends StreamTest with BeforeAndAfter {
+  import testImplicits._
 
   private def newMetadataDir =
     Utils.createTempDir(namePrefix = "streaming.metadata").getCanonicalPath
@@ -171,16 +172,48 @@ class DataStreamReaderWriterSuite extends StreamTest with BeforeAndAfter {
 
     df.writeStream
       .format("org.apache.spark.sql.streaming.test")
-      .option("opt1", "1")
-      .options(Map("opt2" -> "2"))
+      .option("opt1", "5")
+      .options(Map("opt2" -> "4"))
       .options(map)
       .option("checkpointLocation", newMetadataDir)
       .start()
       .stop()
 
-    assert(LastOptions.parameters("opt1") == "1")
-    assert(LastOptions.parameters("opt2") == "2")
-    assert(LastOptions.parameters("opt3") == "3")
+    assert(LastOptions.sinkParameters("opt1") == "5")
+    assert(LastOptions.sinkParameters("opt2") == "4")
+    assert(LastOptions.sinkParameters("opt3") == "3")
+    assert(LastOptions.sinkParameters.contains("checkpointLocation"))
+  }
+
+  test("SPARK-32832: later option should override earlier options for load()") {
+    spark.readStream
+      .format("org.apache.spark.sql.streaming.test")
+      .option("paTh", "1")
+      .option("PATH", "2")
+      .option("Path", "3")
+      .option("patH", "4")
+      .option("path", "5")
+      .load()
+    assert(LastOptions.parameters("path") == "5")
+  }
+
+  test("SPARK-32832: later option should override earlier options for start()") {
+    val ds = spark.readStream
+      .format("org.apache.spark.sql.streaming.test")
+      .load()
+    assert(LastOptions.parameters.isEmpty)
+
+    ds.writeStream
+      .format("org.apache.spark.sql.streaming.test")
+      .option("checkpointLocation", newMetadataDir)
+      .option("paTh", "1")
+      .option("PATH", "2")
+      .option("Path", "3")
+      .option("patH", "4")
+      .option("path", "5")
+      .start()
+      .stop()
+    assert(LastOptions.sinkParameters("path") == "5")
   }
 
   test("partitioning") {
@@ -397,8 +430,8 @@ class DataStreamReaderWriterSuite extends StreamTest with BeforeAndAfter {
       .format("org.apache.spark.sql.streaming.test")
       .load()
 
-    var w = df.writeStream
-    var e = intercept[IllegalArgumentException](w.foreach(null))
+    val w = df.writeStream
+    val e = intercept[IllegalArgumentException](w.foreach(null))
     Seq("foreach", "null").foreach { s =>
       assert(e.getMessage.toLowerCase(Locale.ROOT).contains(s.toLowerCase(Locale.ROOT)))
     }
@@ -414,8 +447,8 @@ class DataStreamReaderWriterSuite extends StreamTest with BeforeAndAfter {
       override def process(value: Row): Unit = {}
       override def close(errorOrNull: Throwable): Unit = {}
     }
-    var w = df.writeStream.partitionBy("value")
-    var e = intercept[AnalysisException](w.foreach(foreachWriter).start())
+    val w = df.writeStream.partitionBy("value")
+    val e = intercept[AnalysisException](w.foreach(foreachWriter).start())
     Seq("foreach", "partitioning").foreach { s =>
       assert(e.getMessage.toLowerCase(Locale.ROOT).contains(s.toLowerCase(Locale.ROOT)))
     }
@@ -435,7 +468,6 @@ class DataStreamReaderWriterSuite extends StreamTest with BeforeAndAfter {
   }
 
   private def testMemorySinkCheckpointRecovery(chkLoc: String, provideInWriter: Boolean): Unit = {
-    import testImplicits._
     val ms = new MemoryStream[Int](0, sqlContext)
     val df = ms.toDF().toDF("a")
     val tableName = "test"
@@ -703,4 +735,81 @@ class DataStreamReaderWriterSuite extends StreamTest with BeforeAndAfter {
       queries.foreach(_.stop())
     }
   }
+
+  test("SPARK-32516: 'path' cannot coexist with load()'s path parameter") {
+    def verifyLoadFails(f: => DataFrame): Unit = {
+      val e = intercept[AnalysisException](f)
+      assert(e.getMessage.contains(
+        "Either remove the path option, or call load() without the parameter"))
+    }
+
+    verifyLoadFails(spark.readStream.option("path", "tmp1").parquet("tmp2"))
+    verifyLoadFails(spark.readStream.option("path", "tmp1").parquet(""))
+    verifyLoadFails(spark.readStream.option("path", "tmp1").format("parquet").load("tmp2"))
+    verifyLoadFails(spark.readStream.option("path", "tmp1").format("parquet").load(""))
+
+    withClue("SPARK-32516: legacy behavior") {
+      withSQLConf(SQLConf.LEGACY_PATH_OPTION_BEHAVIOR.key -> "true") {
+        spark.readStream
+          .format("org.apache.spark.sql.streaming.test")
+          .option("path", "tmp1")
+          .load("tmp2")
+        // The legacy behavior overwrites the path option.
+        assert(LastOptions.parameters("path") == "tmp2")
+      }
+    }
+  }
+
+  test("SPARK-32516: 'path' cannot coexist with start()'s path parameter") {
+    val df = spark.readStream
+      .format("org.apache.spark.sql.streaming.test")
+      .load("tmp1")
+
+    def verifyStartFails(f: => StreamingQuery): Unit = {
+      val e = intercept[AnalysisException](f)
+      assert(e.getMessage.contains(
+        "Either remove the path option, or call start() without the parameter"))
+    }
+
+    verifyStartFails(
+      df.writeStream
+        .format("org.apache.spark.sql.streaming.test")
+        .option("path", "tmp2")
+        .start("tmp3"))
+    verifyStartFails(
+      df.writeStream
+        .format("org.apache.spark.sql.streaming.test")
+        .option("path", "tmp2")
+        .start(""))
+
+    withClue("SPARK-32516: legacy behavior") {
+      withTempDir { checkpointPath =>
+        withSQLConf(SQLConf.LEGACY_PATH_OPTION_BEHAVIOR.key -> "true",
+          SQLConf.CHECKPOINT_LOCATION.key -> checkpointPath.getAbsolutePath) {
+          df.writeStream
+            .format("org.apache.spark.sql.streaming.test")
+            .option("path", "tmp4")
+            .start("tmp5")
+            .stop()
+          // The legacy behavior overwrites the path option.
+          assert(LastOptions.sinkParameters("path") == "tmp5")
+        }
+      }
+    }
+  }
+
+  test("SPARK-32853: consecutive load/start calls should be allowed") {
+    val dfr = spark.readStream.format(classOf[DefaultSource].getName)
+    var df = dfr.load("1")
+    df = dfr.load("2")
+    withTempDir { checkpointPath =>
+      val dfw = df.writeStream
+        .option("checkpointLocation", checkpointPath.getCanonicalPath)
+        .format(classOf[DefaultSource].getName)
+      var query = dfw.start("1")
+      query.stop()
+      query = dfw.start("2")
+      query.stop()
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamTableAPISuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamTableAPISuite.scala
new file mode 100644
index 0000000000000..49e5218ea3352
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamTableAPISuite.scala
@@ -0,0 +1,463 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.streaming.test
+
+import java.io.File
+import java.util
+
+import scala.collection.JavaConverters._
+
+import org.scalatest.BeforeAndAfter
+
+import org.apache.spark.sql.{AnalysisException, Row}
+import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
+import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType}
+import org.apache.spark.sql.catalyst.streaming.StreamingRelationV2
+import org.apache.spark.sql.connector.{FakeV2Provider, InMemoryTableSessionCatalog}
+import org.apache.spark.sql.connector.catalog.{Identifier, InMemoryTableCatalog, SupportsRead, Table, TableCapability, V2TableWithV1Fallback}
+import org.apache.spark.sql.connector.expressions.Transform
+import org.apache.spark.sql.connector.read.ScanBuilder
+import org.apache.spark.sql.execution.streaming.{MemoryStream, MemoryStreamScanBuilder}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.streaming.StreamTest
+import org.apache.spark.sql.streaming.sources.FakeScanBuilder
+import org.apache.spark.sql.types.StructType
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
+import org.apache.spark.util.Utils
+
+class DataStreamTableAPISuite extends StreamTest with BeforeAndAfter {
+  import testImplicits._
+  import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+  before {
+    spark.conf.set("spark.sql.catalog.testcat", classOf[InMemoryTableCatalog].getName)
+    spark.conf.set("spark.sql.catalog.teststream", classOf[InMemoryStreamTableCatalog].getName)
+  }
+
+  after {
+    spark.sessionState.catalogManager.reset()
+    spark.sessionState.conf.clear()
+    sqlContext.streams.active.foreach(_.stop())
+  }
+
+  test("read: table API with file source") {
+    Seq("parquet", "").foreach { source =>
+      withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> source) {
+        withTempDir { tempDir =>
+          val tblName = "my_table"
+          val dir = tempDir.getAbsolutePath
+          withTable(tblName) {
+            spark.range(3).write.format("parquet").option("path", dir).saveAsTable(tblName)
+
+            testStream(spark.readStream.table(tblName))(
+              ProcessAllAvailable(),
+              CheckAnswer(Row(0), Row(1), Row(2))
+            )
+          }
+        }
+      }
+    }
+  }
+
+  test("read: read non-exist table") {
+    intercept[AnalysisException] {
+      spark.readStream.table("non_exist_table")
+    }.message.contains("Table not found")
+  }
+
+  test("read: stream table API with temp view") {
+    val tblName = "my_table"
+    val stream = MemoryStream[Int]
+    withTable(tblName) {
+      stream.toDF().createOrReplaceTempView(tblName)
+
+      testStream(spark.readStream.table(tblName)) (
+        AddData(stream, 1, 2, 3),
+        CheckLastBatch(1, 2, 3),
+        AddData(stream, 4, 5),
+        CheckLastBatch(4, 5)
+      )
+    }
+  }
+
+  test("read: stream table API with non-streaming temp view") {
+    val tblName = "my_table"
+    withTable(tblName) {
+      spark.range(3).createOrReplaceTempView(tblName)
+      intercept[AnalysisException] {
+        spark.readStream.table(tblName)
+      }.message.contains("is not a temp view of streaming logical plan")
+    }
+  }
+
+  test("read: read table without streaming capability support") {
+    val tableIdentifier = "testcat.table_name"
+
+    spark.sql(s"CREATE TABLE $tableIdentifier (id bigint, data string) USING foo")
+
+    intercept[AnalysisException] {
+      spark.readStream.table(tableIdentifier)
+    }.message.contains("does not support either micro-batch or continuous scan")
+  }
+
+  test("read: read table with custom catalog") {
+    val tblName = "teststream.table_name"
+    withTable(tblName) {
+      spark.sql(s"CREATE TABLE $tblName (data int) USING foo")
+      val stream = MemoryStream[Int]
+      val testCatalog = spark.sessionState.catalogManager.catalog("teststream").asTableCatalog
+      val table = testCatalog.loadTable(Identifier.of(Array(), "table_name"))
+      table.asInstanceOf[InMemoryStreamTable].setStream(stream)
+
+      testStream(spark.readStream.table(tblName)) (
+        AddData(stream, 1, 2, 3),
+        CheckLastBatch(1, 2, 3),
+        AddData(stream, 4, 5),
+        CheckLastBatch(4, 5)
+      )
+    }
+  }
+
+  test("read: read table with custom catalog & namespace") {
+    spark.sql("CREATE NAMESPACE teststream.ns")
+
+    val tblName = "teststream.ns.table_name"
+    withTable(tblName) {
+      spark.sql(s"CREATE TABLE $tblName (data int) USING foo")
+      val stream = MemoryStream[Int]
+      val testCatalog = spark.sessionState.catalogManager.catalog("teststream").asTableCatalog
+      val table = testCatalog.loadTable(Identifier.of(Array("ns"), "table_name"))
+      table.asInstanceOf[InMemoryStreamTable].setStream(stream)
+
+      testStream(spark.readStream.table(tblName)) (
+        AddData(stream, 1, 2, 3),
+        CheckLastBatch(1, 2, 3),
+        AddData(stream, 4, 5),
+        CheckLastBatch(4, 5)
+      )
+    }
+  }
+
+  test("read: fallback to V1 relation") {
+    val tblName = DataStreamTableAPISuite.V1FallbackTestTableName
+    spark.conf.set(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION.key,
+      classOf[InMemoryStreamTableCatalog].getName)
+    val v2Source = classOf[FakeV2Provider].getName
+    withTempDir { tempDir =>
+      withTable(tblName) {
+        spark.sql(s"CREATE TABLE $tblName (data int) USING $v2Source")
+
+        // Check the StreamingRelationV2 has been replaced by StreamingRelation
+        val plan = spark.readStream.option("path", tempDir.getCanonicalPath).table(tblName)
+          .queryExecution.analyzed.collectFirst {
+            case d: StreamingRelationV2 => d
+          }
+        assert(plan.isEmpty)
+      }
+    }
+  }
+
+  test("write: write to table with custom catalog & no namespace") {
+    val tableIdentifier = "testcat.table_name"
+
+    withTable(tableIdentifier) {
+      spark.sql(s"CREATE TABLE $tableIdentifier (id bigint, data string) USING foo")
+      checkAnswer(spark.table(tableIdentifier), Seq.empty)
+
+      runTestWithStreamAppend(tableIdentifier)
+    }
+  }
+
+  test("write: write to table with custom catalog & namespace") {
+    spark.sql("CREATE NAMESPACE testcat.ns")
+    val tableIdentifier = "testcat.ns.table_name"
+
+    withTable(tableIdentifier) {
+      spark.sql(s"CREATE TABLE $tableIdentifier (id bigint, data string) USING foo")
+      checkAnswer(spark.table(tableIdentifier), Seq.empty)
+
+      runTestWithStreamAppend(tableIdentifier)
+    }
+  }
+
+  test("write: write to table with default session catalog") {
+    val v2Source = classOf[FakeV2Provider].getName
+    spark.conf.set(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION.key,
+      classOf[InMemoryTableSessionCatalog].getName)
+
+    spark.sql("CREATE NAMESPACE ns")
+
+    val tableIdentifier = "ns.table_name"
+    withTable(tableIdentifier) {
+      spark.sql(s"CREATE TABLE $tableIdentifier (id bigint, data string) USING $v2Source")
+      checkAnswer(spark.table(tableIdentifier), Seq.empty)
+
+      runTestWithStreamAppend(tableIdentifier)
+    }
+  }
+
+  test("write: write to non-exist table with custom catalog") {
+    val tableIdentifier = "testcat.nonexistenttable"
+
+    withTable(tableIdentifier) {
+      runTestWithStreamAppend(tableIdentifier)
+    }
+  }
+
+  test("write: write to temporary view isn't allowed yet") {
+    val tableIdentifier = "testcat.table_name"
+    val tempViewIdentifier = "temp_view"
+
+    spark.sql(s"CREATE TABLE $tableIdentifier (id bigint, data string) USING foo")
+    checkAnswer(spark.table(tableIdentifier), Seq.empty)
+
+    spark.table(tableIdentifier).createOrReplaceTempView(tempViewIdentifier)
+
+    withTempDir { checkpointDir =>
+      val exc = intercept[AnalysisException] {
+        runStreamQueryAppendMode(tempViewIdentifier, checkpointDir, Seq.empty, Seq.empty)
+      }
+      assert(exc.getMessage.contains("doesn't support streaming write"))
+    }
+  }
+
+  test("write: write to view shouldn't be allowed") {
+    val tableIdentifier = "testcat.table_name"
+    val viewIdentifier = "table_view"
+
+    spark.sql(s"CREATE TABLE $tableIdentifier (id bigint, data string) USING foo")
+    checkAnswer(spark.table(tableIdentifier), Seq.empty)
+
+    spark.sql(s"CREATE VIEW $viewIdentifier AS SELECT id, data FROM $tableIdentifier")
+
+    withTempDir { checkpointDir =>
+      val exc = intercept[AnalysisException] {
+        runStreamQueryAppendMode(viewIdentifier, checkpointDir, Seq.empty, Seq.empty)
+      }
+      assert(exc.getMessage.contains(s"Streaming into views $viewIdentifier is not supported"))
+    }
+  }
+
+  test("write: write to an external table") {
+    withTempDir { dir =>
+      val tableName = "stream_test"
+      withTable(tableName) {
+        checkForStreamTable(Some(dir), tableName)
+      }
+    }
+  }
+
+  test("write: write to a managed table") {
+    val tableName = "stream_test"
+    withTable(tableName) {
+      checkForStreamTable(None, tableName)
+    }
+  }
+
+  test("write: write to an external table with existing path") {
+    withTempDir { dir =>
+      val tableName = "stream_test"
+      withTable(tableName) {
+        // The file written by batch will not be seen after the table was written by a streaming
+        // query. This is because we load files from the metadata log instead of listing them
+        // using HDFS API.
+        Seq(4, 5, 6).toDF("value").write.format("parquet")
+          .option("path", dir.getCanonicalPath).saveAsTable(tableName)
+
+        checkForStreamTable(Some(dir), tableName)
+      }
+    }
+  }
+
+  test("write: write to a managed table with existing path") {
+    val tableName = "stream_test"
+    withTable(tableName) {
+      // The file written by batch will not be seen after the table was written by a streaming
+      // query. This is because we load files from the metadata log instead of listing them
+      // using HDFS API.
+      Seq(4, 5, 6).toDF("value").write.format("parquet").saveAsTable(tableName)
+
+      checkForStreamTable(None, tableName)
+    }
+  }
+
+  test("write: write to an external path and create table") {
+    withTempDir { dir =>
+      val tableName = "stream_test"
+      withTable(tableName) {
+        // The file written by batch will not be seen after the table was written by a streaming
+        // query. This is because we load files from the metadata log instead of listing them
+        // using HDFS API.
+        Seq(4, 5, 6).toDF("value").write
+          .mode("append").format("parquet").save(dir.getCanonicalPath)
+
+        checkForStreamTable(Some(dir), tableName)
+      }
+    }
+  }
+
+  test("write: write to table with different format shouldn't be allowed") {
+    val tableName = "stream_test"
+
+    spark.sql(s"CREATE TABLE $tableName (id bigint, data string) USING json")
+    checkAnswer(spark.table(tableName), Seq.empty)
+
+    withTempDir { checkpointDir =>
+      val exc = intercept[AnalysisException] {
+        runStreamQueryAppendMode(tableName, checkpointDir, Seq.empty, Seq.empty)
+      }
+      assert(exc.getMessage.contains("The input source(parquet) is different from the table " +
+        s"$tableName's data source provider(json)"))
+    }
+  }
+
+  private def checkForStreamTable(dir: Option[File], tableName: String): Unit = {
+    val memory = MemoryStream[Int]
+    val dsw = memory.toDS().writeStream.format("parquet")
+    dir.foreach { output =>
+      dsw.option("path", output.getCanonicalPath)
+    }
+    val sq = dsw
+      .option("checkpointLocation", Utils.createTempDir().getCanonicalPath)
+      .toTable(tableName)
+    memory.addData(1, 2, 3)
+    sq.processAllAvailable()
+
+    checkDataset(
+      spark.table(tableName).as[Int],
+      1, 2, 3)
+    val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName))
+    val path = if (dir.nonEmpty) {
+      dir.get
+    } else {
+      new File(catalogTable.location)
+    }
+    checkDataset(
+      spark.read.format("parquet").load(path.getCanonicalPath).as[Int],
+      1, 2, 3)
+  }
+
+  private def runTestWithStreamAppend(tableIdentifier: String) = {
+    withTempDir { checkpointDir =>
+      val input1 = Seq((1L, "a"), (2L, "b"), (3L, "c"))
+      verifyStreamAppend(tableIdentifier, checkpointDir, Seq.empty, input1, input1)
+
+      val input2 = Seq((4L, "d"), (5L, "e"), (6L, "f"))
+      verifyStreamAppend(tableIdentifier, checkpointDir, Seq(input1), input2, input1 ++ input2)
+    }
+  }
+
+  private def runStreamQueryAppendMode(
+      tableIdentifier: String,
+      checkpointDir: File,
+      prevInputs: Seq[Seq[(Long, String)]],
+      newInputs: Seq[(Long, String)]): Unit = {
+    val inputData = MemoryStream[(Long, String)]
+    val inputDF = inputData.toDF().toDF("id", "data")
+
+    prevInputs.foreach { inputsPerBatch =>
+      inputData.addData(inputsPerBatch: _*)
+    }
+
+    val query = inputDF
+      .writeStream
+      .option("checkpointLocation", checkpointDir.getAbsolutePath)
+      .toTable(tableIdentifier)
+
+    inputData.addData(newInputs: _*)
+
+    query.processAllAvailable()
+    query.stop()
+  }
+
+  private def verifyStreamAppend(
+      tableIdentifier: String,
+      checkpointDir: File,
+      prevInputs: Seq[Seq[(Long, String)]],
+      newInputs: Seq[(Long, String)],
+      expectedOutputs: Seq[(Long, String)]): Unit = {
+    runStreamQueryAppendMode(tableIdentifier, checkpointDir, prevInputs, newInputs)
+    checkAnswer(
+      spark.table(tableIdentifier),
+      expectedOutputs.map { case (id, data) => Row(id, data) }
+    )
+  }
+}
+
+object DataStreamTableAPISuite {
+  val V1FallbackTestTableName = "fallbackV1Test"
+}
+
+class InMemoryStreamTable(override val name: String) extends Table with SupportsRead {
+  var stream: MemoryStream[Int] = _
+
+  def setStream(inputData: MemoryStream[Int]): Unit = stream = inputData
+
+  override def schema(): StructType = stream.fullSchema()
+
+  override def capabilities(): util.Set[TableCapability] = {
+    Set(TableCapability.MICRO_BATCH_READ, TableCapability.CONTINUOUS_READ).asJava
+  }
+
+  override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = {
+    new MemoryStreamScanBuilder(stream)
+  }
+}
+
+class NonStreamV2Table(override val name: String)
+    extends Table with SupportsRead with V2TableWithV1Fallback {
+  override def schema(): StructType = StructType(Nil)
+  override def capabilities(): util.Set[TableCapability] = Set(TableCapability.BATCH_READ).asJava
+  override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = new FakeScanBuilder
+
+  override def v1Table: CatalogTable = {
+    CatalogTable(
+      identifier =
+        TableIdentifier(DataStreamTableAPISuite.V1FallbackTestTableName, Some("default")),
+      tableType = CatalogTableType.MANAGED,
+      storage = CatalogStorageFormat.empty,
+      owner = null,
+      schema = schema(),
+      provider = Some("parquet"))
+  }
+}
+
+
+class InMemoryStreamTableCatalog extends InMemoryTableCatalog {
+  import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+  override def createTable(
+      ident: Identifier,
+      schema: StructType,
+      partitions: Array[Transform],
+      properties: util.Map[String, String]): Table = {
+    if (tables.containsKey(ident)) {
+      throw new TableAlreadyExistsException(ident)
+    }
+
+    val table = if (ident.name() == DataStreamTableAPISuite.V1FallbackTestTableName) {
+      new NonStreamV2Table(s"$name.${ident.quoted}")
+    } else {
+      new InMemoryStreamTable(s"$name.${ident.quoted}")
+    }
+    tables.put(ident, table)
+    namespaces.putIfAbsent(ident.namespace.toList, Map())
+    table
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryHistorySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryHistorySuite.scala
new file mode 100644
index 0000000000000..160535ea4d048
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryHistorySuite.scala
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.streaming.ui
+
+import java.util.Locale
+import javax.servlet.http.HttpServletRequest
+
+import org.mockito.Mockito.{mock, when}
+import org.scalatest.BeforeAndAfter
+
+import org.apache.spark.deploy.history.{Utils => HsUtils}
+import org.apache.spark.sql.execution.ui.StreamingQueryStatusStore
+import org.apache.spark.sql.test.SharedSparkSession
+
+class StreamingQueryHistorySuite extends SharedSparkSession with BeforeAndAfter {
+
+  test("support streaming query events") {
+    val logDir = Thread.currentThread().getContextClassLoader.getResource("spark-events").toString
+    HsUtils.withFsHistoryProvider(logDir) { provider =>
+      val appUi = provider.getAppUI("local-1596020211915", None).getOrElse {
+        assert(false, "Failed to load event log of local-1596020211915.")
+        null
+      }
+      assert(appUi.ui.appName == "StructuredKafkaWordCount")
+      assert(appUi.ui.store.store.count(classOf[StreamingQueryData]) == 1)
+      assert(appUi.ui.store.store.count(classOf[StreamingQueryProgressWrapper]) == 8)
+
+      val store = new StreamingQueryStatusStore(appUi.ui.store.store)
+      val tab = new StreamingQueryTab(store, appUi.ui)
+      val request = mock(classOf[HttpServletRequest])
+      var html = new StreamingQueryPage(tab).render(request)
+        .toString().toLowerCase(Locale.ROOT)
+      // 81.39: Avg Input /sec
+      assert(html.contains("81.39"))
+      // 157.05: Avg Process /sec
+      assert(html.contains("157.05"))
+
+      val id = "8d268dc2-bc9c-4be8-97a9-b135d2943028"
+      val runId = "e225d92f-2545-48f8-87a2-9c0309580f8a"
+      when(request.getParameter("id")).thenReturn(runId)
+      html = new StreamingQueryStatisticsPage(tab).render(request)
+        .toString().toLowerCase(Locale.ROOT)
+      assert(html.contains("<strong>8</strong> completed batches"))
+      assert(html.contains(id))
+      assert(html.contains(runId))
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPageSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPageSuite.scala
index 2a1e18ab66bb7..246fa1f7c9184 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPageSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPageSuite.scala
@@ -20,12 +20,16 @@ package org.apache.spark.sql.streaming.ui
 import java.util.{Locale, UUID}
 import javax.servlet.http.HttpServletRequest
 
+import scala.xml.Node
+
 import org.mockito.Mockito.{mock, when, RETURNS_SMART_NULLS}
 import org.scalatest.BeforeAndAfter
-import scala.xml.Node
 
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.execution.ui.StreamingQueryStatusStore
 import org.apache.spark.sql.streaming.StreamingQueryProgress
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.ui.SparkUI
 
 class StreamingQueryPageSuite extends SharedSparkSession with BeforeAndAfter {
 
@@ -33,31 +37,28 @@ class StreamingQueryPageSuite extends SharedSparkSession with BeforeAndAfter {
     val id = UUID.randomUUID()
     val request = mock(classOf[HttpServletRequest])
     val tab = mock(classOf[StreamingQueryTab], RETURNS_SMART_NULLS)
-    val statusListener = mock(classOf[StreamingQueryStatusListener], RETURNS_SMART_NULLS)
+    val store = mock(classOf[StreamingQueryStatusStore], RETURNS_SMART_NULLS)
     when(tab.appName).thenReturn("testing")
     when(tab.headerTabs).thenReturn(Seq.empty)
-    when(tab.statusListener).thenReturn(statusListener)
+    when(tab.store).thenReturn(store)
 
     val streamQuery = createStreamQueryUIData(id)
-    when(statusListener.allQueryStatus).thenReturn(Seq(streamQuery))
+    when(store.allQueryUIData).thenReturn(Seq(streamQuery))
     var html = renderStreamingQueryPage(request, tab)
       .toString().toLowerCase(Locale.ROOT)
     assert(html.contains("active streaming queries (1)"))
-    assert(html.contains("completed streaming queries (0)"))
 
-    when(streamQuery.isActive).thenReturn(false)
-    when(streamQuery.exception).thenReturn(None)
+    when(streamQuery.summary.isActive).thenReturn(false)
+    when(streamQuery.summary.exception).thenReturn(None)
     html = renderStreamingQueryPage(request, tab)
       .toString().toLowerCase(Locale.ROOT)
-    assert(html.contains("active streaming queries (0)"))
     assert(html.contains("completed streaming queries (1)"))
     assert(html.contains("finished"))
 
-    when(streamQuery.isActive).thenReturn(false)
-    when(streamQuery.exception).thenReturn(Option("exception in query"))
+    when(streamQuery.summary.isActive).thenReturn(false)
+    when(streamQuery.summary.exception).thenReturn(Option("exception in query"))
     html = renderStreamingQueryPage(request, tab)
       .toString().toLowerCase(Locale.ROOT)
-    assert(html.contains("active streaming queries (0)"))
     assert(html.contains("completed streaming queries (1)"))
     assert(html.contains("failed"))
     assert(html.contains("exception in query"))
@@ -67,14 +68,20 @@ class StreamingQueryPageSuite extends SharedSparkSession with BeforeAndAfter {
     val id = UUID.randomUUID()
     val request = mock(classOf[HttpServletRequest])
     val tab = mock(classOf[StreamingQueryTab], RETURNS_SMART_NULLS)
-    val statusListener = mock(classOf[StreamingQueryStatusListener], RETURNS_SMART_NULLS)
+    val store = mock(classOf[StreamingQueryStatusStore], RETURNS_SMART_NULLS)
     when(request.getParameter("id")).thenReturn(id.toString)
     when(tab.appName).thenReturn("testing")
     when(tab.headerTabs).thenReturn(Seq.empty)
-    when(tab.statusListener).thenReturn(statusListener)
+    when(tab.store).thenReturn(store)
+    val ui = mock(classOf[SparkUI])
+    when(request.getParameter("id")).thenReturn(id.toString)
+    when(tab.appName).thenReturn("testing")
+    when(tab.headerTabs).thenReturn(Seq.empty)
+    when(ui.conf).thenReturn(new SparkConf())
+    when(tab.parent).thenReturn(ui)
 
     val streamQuery = createStreamQueryUIData(id)
-    when(statusListener.allQueryStatus).thenReturn(Seq(streamQuery))
+    when(store.allQueryUIData).thenReturn(Seq(streamQuery))
     val html = renderStreamingQueryStatisticsPage(request, tab)
       .toString().toLowerCase(Locale.ROOT)
 
@@ -92,15 +99,18 @@ class StreamingQueryPageSuite extends SharedSparkSession with BeforeAndAfter {
     when(progress.batchId).thenReturn(2)
     when(progress.prettyJson).thenReturn("""{"a":1}""")
 
+    val summary = mock(classOf[StreamingQueryData], RETURNS_SMART_NULLS)
+    when(summary.isActive).thenReturn(true)
+    when(summary.name).thenReturn("query")
+    when(summary.id).thenReturn(id)
+    when(summary.runId).thenReturn(id)
+    when(summary.startTimestamp).thenReturn(1L)
+    when(summary.exception).thenReturn(None)
+
     val streamQuery = mock(classOf[StreamingQueryUIData], RETURNS_SMART_NULLS)
-    when(streamQuery.isActive).thenReturn(true)
-    when(streamQuery.name).thenReturn("query")
-    when(streamQuery.id).thenReturn(id)
-    when(streamQuery.runId).thenReturn(id)
-    when(streamQuery.startTimestamp).thenReturn(1L)
+    when(streamQuery.summary).thenReturn(summary)
     when(streamQuery.lastProgress).thenReturn(progress)
     when(streamQuery.recentProgress).thenReturn(Array(progress))
-    when(streamQuery.exception).thenReturn(None)
 
     streamQuery
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala
index 6aa440e5609c5..91c55d5598a6b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala
@@ -17,19 +17,28 @@
 
 package org.apache.spark.sql.streaming.ui
 
-import java.util.UUID
+import java.text.SimpleDateFormat
+import java.util.{Date, UUID}
 
 import org.mockito.Mockito.{mock, when, RETURNS_SMART_NULLS}
+import org.scalatest.time.SpanSugar._
 
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.getTimeZone
+import org.apache.spark.sql.execution.ui.StreamingQueryStatusStore
+import org.apache.spark.sql.internal.StaticSQLConf
 import org.apache.spark.sql.streaming.{StreamingQueryListener, StreamingQueryProgress, StreamTest}
 import org.apache.spark.sql.streaming
+import org.apache.spark.status.ElementTrackingStore
+import org.apache.spark.util.kvstore.InMemoryStore
 
 class StreamingQueryStatusListenerSuite extends StreamTest {
 
   test("onQueryStarted, onQueryProgress, onQueryTerminated") {
-    val listener = new StreamingQueryStatusListener(spark.sparkContext.conf)
+    val kvStore = new ElementTrackingStore(new InMemoryStore(), sparkConf)
+    val listener = new StreamingQueryStatusListener(spark.sparkContext.conf, kvStore)
+    val queryStore = new StreamingQueryStatusStore(kvStore)
 
-    // hanlde query started event
+    // handle query started event
     val id = UUID.randomUUID()
     val runId = UUID.randomUUID()
     val startEvent = new StreamingQueryListener.QueryStartedEvent(
@@ -37,8 +46,9 @@ class StreamingQueryStatusListenerSuite extends StreamTest {
     listener.onQueryStarted(startEvent)
 
     // result checking
-    assert(listener.activeQueryStatus.size() == 1)
-    assert(listener.activeQueryStatus.get(runId).name == "test")
+    assert(queryStore.allQueryUIData.count(_.summary.isActive) == 1)
+    assert(queryStore.allQueryUIData.filter(_.summary.isActive).exists(uiData =>
+      uiData.summary.runId == runId && uiData.summary.name.equals("test")))
 
     // handle query progress event
     val progress = mock(classOf[StreamingQueryProgress], RETURNS_SMART_NULLS)
@@ -53,28 +63,32 @@ class StreamingQueryStatusListenerSuite extends StreamTest {
     listener.onQueryProgress(processEvent)
 
     // result checking
-    val activeQuery = listener.activeQueryStatus.get(runId)
-    assert(activeQuery.isActive)
-    assert(activeQuery.recentProgress.length == 1)
-    assert(activeQuery.lastProgress.id == id)
-    assert(activeQuery.lastProgress.runId == runId)
-    assert(activeQuery.lastProgress.timestamp == "2001-10-01T01:00:00.100Z")
-    assert(activeQuery.lastProgress.inputRowsPerSecond == 10.0)
-    assert(activeQuery.lastProgress.processedRowsPerSecond == 12.0)
-    assert(activeQuery.lastProgress.batchId == 2)
-    assert(activeQuery.lastProgress.prettyJson == """{"a":1}""")
+    val activeQuery =
+      queryStore.allQueryUIData.filter(_.summary.isActive).find(_.summary.runId == runId)
+    assert(activeQuery.isDefined)
+    assert(activeQuery.get.summary.isActive)
+    assert(activeQuery.get.recentProgress.length == 1)
+    assert(activeQuery.get.lastProgress.id == id)
+    assert(activeQuery.get.lastProgress.runId == runId)
+    assert(activeQuery.get.lastProgress.timestamp == "2001-10-01T01:00:00.100Z")
+    assert(activeQuery.get.lastProgress.inputRowsPerSecond == 10.0)
+    assert(activeQuery.get.lastProgress.processedRowsPerSecond == 12.0)
+    assert(activeQuery.get.lastProgress.batchId == 2)
+    assert(activeQuery.get.lastProgress.prettyJson == """{"a":1}""")
 
     // handle terminate event
     val terminateEvent = new StreamingQueryListener.QueryTerminatedEvent(id, runId, None)
     listener.onQueryTerminated(terminateEvent)
 
-    assert(!listener.inactiveQueryStatus.head.isActive)
-    assert(listener.inactiveQueryStatus.head.runId == runId)
-    assert(listener.inactiveQueryStatus.head.id == id)
+    assert(!queryStore.allQueryUIData.filterNot(_.summary.isActive).head.summary.isActive)
+    assert(queryStore.allQueryUIData.filterNot(_.summary.isActive).head.summary.runId == runId)
+    assert(queryStore.allQueryUIData.filterNot(_.summary.isActive).head.summary.id == id)
   }
 
   test("same query start multiple times") {
-    val listener = new StreamingQueryStatusListener(spark.sparkContext.conf)
+    val kvStore = new ElementTrackingStore(new InMemoryStore(), sparkConf)
+    val listener = new StreamingQueryStatusListener(spark.sparkContext.conf, kvStore)
+    val queryStore = new StreamingQueryStatusStore(kvStore)
 
     // handle first time start
     val id = UUID.randomUUID()
@@ -94,11 +108,106 @@ class StreamingQueryStatusListenerSuite extends StreamTest {
     listener.onQueryStarted(startEvent1)
 
     // result checking
-    assert(listener.activeQueryStatus.size() == 1)
-    assert(listener.inactiveQueryStatus.length == 1)
-    assert(listener.activeQueryStatus.containsKey(runId1))
-    assert(listener.activeQueryStatus.get(runId1).id == id)
-    assert(listener.inactiveQueryStatus.head.runId == runId0)
-    assert(listener.inactiveQueryStatus.head.id == id)
+    assert(queryStore.allQueryUIData.count(_.summary.isActive) == 1)
+    assert(queryStore.allQueryUIData.filterNot(_.summary.isActive).length == 1)
+    assert(queryStore.allQueryUIData.filter(_.summary.isActive).exists(_.summary.runId == runId1))
+    assert(queryStore.allQueryUIData.filter(_.summary.isActive).exists(uiData =>
+      uiData.summary.runId == runId1 && uiData.summary.id == id))
+    assert(queryStore.allQueryUIData.filterNot(_.summary.isActive).head.summary.runId == runId0)
+    assert(queryStore.allQueryUIData.filterNot(_.summary.isActive).head.summary.id == id)
+  }
+
+  test("test small retained queries") {
+    val kvStore = new ElementTrackingStore(new InMemoryStore(), sparkConf)
+    val conf = spark.sparkContext.conf
+    conf.set(StaticSQLConf.STREAMING_UI_RETAINED_QUERIES.key, "2")
+    val listener = new StreamingQueryStatusListener(conf, kvStore)
+    val queryStore = new StreamingQueryStatusStore(kvStore)
+
+    def addNewQuery(): (UUID, UUID) = {
+      val format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'") // ISO8601
+      format.setTimeZone(getTimeZone("UTC"))
+      val id = UUID.randomUUID()
+      val runId = UUID.randomUUID()
+      val startEvent = new StreamingQueryListener.QueryStartedEvent(
+        id, runId, "test1", format.format(new Date(System.currentTimeMillis())))
+      listener.onQueryStarted(startEvent)
+      (id, runId)
+    }
+
+    def checkInactiveQueryStatus(numInactives: Int, targetInactives: Seq[UUID]): Unit = {
+      eventually(timeout(10.seconds)) {
+        val inactiveQueries = queryStore.allQueryUIData.filter(!_.summary.isActive)
+        assert(inactiveQueries.size == numInactives)
+        assert(inactiveQueries.map(_.summary.id).toSet == targetInactives.toSet)
+      }
+    }
+
+    val (id1, runId1) = addNewQuery()
+    val (id2, runId2) = addNewQuery()
+    val (id3, runId3) = addNewQuery()
+    assert(queryStore.allQueryUIData.count(!_.summary.isActive) == 0)
+
+    val terminateEvent1 = new StreamingQueryListener.QueryTerminatedEvent(id1, runId1, None)
+    listener.onQueryTerminated(terminateEvent1)
+    checkInactiveQueryStatus(1, Seq(id1))
+    val terminateEvent2 = new StreamingQueryListener.QueryTerminatedEvent(id2, runId2, None)
+    listener.onQueryTerminated(terminateEvent2)
+    checkInactiveQueryStatus(2, Seq(id1, id2))
+    val terminateEvent3 = new StreamingQueryListener.QueryTerminatedEvent(id3, runId3, None)
+    listener.onQueryTerminated(terminateEvent3)
+    checkInactiveQueryStatus(2, Seq(id2, id3))
+  }
+
+  test("test small retained progress") {
+    val kvStore = new ElementTrackingStore(new InMemoryStore(), sparkConf)
+    val conf = spark.sparkContext.conf
+    conf.set(StaticSQLConf.STREAMING_UI_RETAINED_PROGRESS_UPDATES.key, "5")
+    val listener = new StreamingQueryStatusListener(conf, kvStore)
+    val queryStore = new StreamingQueryStatusStore(kvStore)
+
+    val id = UUID.randomUUID()
+    val runId = UUID.randomUUID()
+    val startEvent = new StreamingQueryListener.QueryStartedEvent(
+      id, runId, "test", "2016-12-05T20:54:20.827Z")
+    listener.onQueryStarted(startEvent)
+
+    var batchId: Int = 0
+
+    def addQueryProgress(): Unit = {
+      val progress = mockProgressData(id, runId)
+      val processEvent = new streaming.StreamingQueryListener.QueryProgressEvent(progress)
+      listener.onQueryProgress(processEvent)
+    }
+
+    def mockProgressData(id: UUID, runId: UUID): StreamingQueryProgress = {
+      val format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'") // ISO8601
+      format.setTimeZone(getTimeZone("UTC"))
+
+      val progress = mock(classOf[StreamingQueryProgress], RETURNS_SMART_NULLS)
+      when(progress.id).thenReturn(id)
+      when(progress.runId).thenReturn(runId)
+      when(progress.timestamp).thenReturn(format.format(new Date(System.currentTimeMillis())))
+      when(progress.inputRowsPerSecond).thenReturn(10.0)
+      when(progress.processedRowsPerSecond).thenReturn(12.0)
+      when(progress.batchId).thenReturn(batchId)
+      when(progress.prettyJson).thenReturn("""{"a":1}""")
+
+      batchId += 1
+      progress
+    }
+
+    def checkQueryProcessData(targetNum: Int): Unit = {
+      eventually(timeout(10.seconds)) {
+        assert(queryStore.getQueryProgressData(runId).size == targetNum)
+      }
+    }
+
+    Array.tabulate(4) { _ => addQueryProgress() }
+    checkQueryProcessData(4)
+    addQueryProgress()
+    checkQueryProcessData(5)
+    addQueryProgress()
+    checkQueryProcessData(5)
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/UISeleniumSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/UISeleniumSuite.scala
index fdf4c6634d79f..db3d6529c9906 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/UISeleniumSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/UISeleniumSuite.scala
@@ -19,8 +19,10 @@ package org.apache.spark.sql.streaming.ui
 
 import org.openqa.selenium.WebDriver
 import org.openqa.selenium.htmlunit.HtmlUnitDriver
-import org.scalatest._
+import org.scalatest.BeforeAndAfterAll
 import org.scalatest.concurrent.Eventually._
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 import org.scalatest.time.SpanSugar._
 import org.scalatestplus.selenium.WebBrowser
 
@@ -29,7 +31,10 @@ import org.apache.spark.internal.config.UI.{UI_ENABLED, UI_PORT}
 import org.apache.spark.sql.LocalSparkSession.withSparkSession
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.util.quietly
-import org.apache.spark.sql.streaming.StreamingQueryException
+import org.apache.spark.sql.functions.{window => windowFn, _}
+import org.apache.spark.sql.internal.SQLConf.SHUFFLE_PARTITIONS
+import org.apache.spark.sql.internal.StaticSQLConf.ENABLED_STREAMING_UI_CUSTOM_METRIC_LIST
+import org.apache.spark.sql.streaming.{StreamingQueryException, Trigger}
 import org.apache.spark.ui.SparkUICssErrorHandler
 
 class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with BeforeAndAfterAll {
@@ -49,8 +54,10 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
     val conf = new SparkConf()
       .setMaster(master)
       .setAppName("ui-test")
+      .set(SHUFFLE_PARTITIONS, 5)
       .set(UI_ENABLED, true)
       .set(UI_PORT, 0)
+      .set(ENABLED_STREAMING_UI_CUSTOM_METRIC_LIST, Seq("stateOnCurrentVersionSizeBytes"))
     additionalConfs.foreach { case (k, v) => conf.set(k, v) }
     val spark = SparkSession.builder().master(master).config(conf).getOrCreate()
     assert(spark.sparkContext.ui.isDefined)
@@ -73,10 +80,17 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
           val h3Text = findAll(cssSelector("h3")).map(_.text).toSeq
           h3Text should not contain ("Streaming Query")
 
+          val input1 = spark.readStream.format("rate").load()
+          val input2 = spark.readStream.format("rate").load()
+          val input3 = spark.readStream.format("rate").load()
           val activeQuery =
-            spark.readStream.format("rate").load().writeStream.format("noop").start()
+            input1.selectExpr("timestamp", "mod(value, 100) as mod", "value")
+              .withWatermark("timestamp", "0 second")
+              .groupBy(windowFn($"timestamp", "10 seconds", "2 seconds"), $"mod")
+              .agg(avg("value").as("avg_value"))
+              .writeStream.format("noop").trigger(Trigger.ProcessingTime("5 seconds")).start()
           val completedQuery =
-            spark.readStream.format("rate").load().writeStream.format("noop").start()
+            input2.join(input3, "value").writeStream.format("noop").start()
           completedQuery.stop()
           val failedQuery = spark.readStream.format("rate").load().select("value").as[Long]
             .map(_ / 0).writeStream.format("noop").start()
@@ -91,21 +105,23 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
             goToUi(spark, "/StreamingQuery")
 
             findAll(cssSelector("h3")).map(_.text).toSeq should contain("Streaming Query")
-            findAll(cssSelector("""#activeQueries-table th""")).map(_.text).toSeq should be {
-              List("Name", "Status", "Id", "Run ID", "Start Time", "Duration", "Avg Input /sec",
-                "Avg Process /sec", "Lastest Batch")
+
+            val arrow = 0x25BE.toChar
+            findAll(cssSelector("""#active-table th""")).map(_.text).toList should be {
+              List("Name", "Status", "ID", "Run ID", s"Start Time $arrow", "Duration",
+                "Avg Input /sec", "Avg Process /sec", "Latest Batch")
             }
             val activeQueries =
-              findAll(cssSelector("""#activeQueries-table td""")).map(_.text).toSeq
+              findAll(cssSelector("""#active-table td""")).map(_.text).toSeq
             activeQueries should contain(activeQuery.id.toString)
             activeQueries should contain(activeQuery.runId.toString)
-            findAll(cssSelector("""#completedQueries-table th"""))
-              .map(_.text).toSeq should be {
-                List("Name", "Status", "Id", "Run ID", "Start Time", "Duration", "Avg Input /sec",
-                  "Avg Process /sec", "Lastest Batch", "Error")
+            findAll(cssSelector("""#completed-table th"""))
+              .map(_.text).toList should be {
+                List("Name", "Status", "ID", "Run ID", s"Start Time $arrow", "Duration",
+                  "Avg Input /sec", "Avg Process /sec", "Latest Batch", "Error")
               }
             val completedQueries =
-              findAll(cssSelector("""#completedQueries-table td""")).map(_.text).toSeq
+              findAll(cssSelector("""#completed-table td""")).map(_.text).toSeq
             completedQueries should contain(completedQuery.id.toString)
             completedQueries should contain(completedQuery.runId.toString)
             completedQueries should contain(failedQuery.id.toString)
@@ -113,7 +129,7 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
 
             // Check the query statistics page
             val activeQueryLink =
-              findAll(cssSelector("""#activeQueries-table a""")).flatMap(_.attribute("href")).next
+              findAll(cssSelector("""#active-table td a""")).flatMap(_.attribute("href")).next
             go to activeQueryLink
 
             findAll(cssSelector("h3"))
@@ -125,6 +141,20 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
             findAll(cssSelector("""#stat-table th""")).map(_.text).toSeq should be {
               List("", "Timelines", "Histograms")
             }
+            summaryText should contain ("Input Rate (?)")
+            summaryText should contain ("Process Rate (?)")
+            summaryText should contain ("Input Rows (?)")
+            summaryText should contain ("Batch Duration (?)")
+            summaryText should contain ("Operation Duration (?)")
+            summaryText should contain ("Global Watermark Gap (?)")
+            summaryText should contain ("Aggregated Number Of Total State Rows (?)")
+            summaryText should contain ("Aggregated Number Of Updated State Rows (?)")
+            summaryText should contain ("Aggregated State Memory Used In Bytes (?)")
+            summaryText should contain ("Aggregated Number Of Rows Dropped By Watermark (?)")
+            summaryText should contain ("Aggregated Custom Metric stateOnCurrentVersionSizeBytes" +
+              " (?)")
+            summaryText should not contain ("Aggregated Custom Metric loadedMapCacheHitCount (?)")
+            summaryText should not contain ("Aggregated Custom Metric loadedMapCacheMissCount (?)")
           }
         } finally {
           spark.streams.active.foreach(_.stop())
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/UIUtilsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/UIUtilsSuite.scala
index 46f2eadc05835..fbf4dcead4a9e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/UIUtilsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/UIUtilsSuite.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.streaming.ui
 
 import org.mockito.Mockito.{mock, when, RETURNS_SMART_NULLS}
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.streaming.StreamingQueryProgress
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
index 9747840ce4032..41d11568750cc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.test
 
 import java.io.File
-import java.util.Locale
+import java.util.{Locale, Random}
 import java.util.concurrent.ConcurrentLinkedQueue
 
 import scala.collection.JavaConverters._
@@ -40,7 +40,7 @@ import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.plans.logical.{AppendData, LogicalPlan, OverwriteByExpression}
 import org.apache.spark.sql.execution.QueryExecution
-import org.apache.spark.sql.execution.datasources.DataSourceUtils
+import org.apache.spark.sql.execution.datasources.{DataSourceUtils, HadoopFsRelation, LogicalRelation}
 import org.apache.spark.sql.execution.datasources.noop.NoopDataSource
 import org.apache.spark.sql.execution.datasources.parquet.SpecificParquetRecordReaderBase
 import org.apache.spark.sql.internal.SQLConf
@@ -224,6 +224,56 @@ class DataFrameReaderWriterSuite extends QueryTest with SharedSparkSession with
     assert(LastOptions.parameters("opt3") == "3")
   }
 
+  test("SPARK-32364: later option should override earlier options for load()") {
+    spark.read
+      .format("org.apache.spark.sql.test")
+      .option("paTh", "1")
+      .option("PATH", "2")
+      .option("Path", "3")
+      .option("patH", "4")
+      .option("path", "5")
+      .load()
+    assert(LastOptions.parameters("path") == "5")
+
+    withClue("SPARK-32516: legacy path option behavior") {
+      withSQLConf(SQLConf.LEGACY_PATH_OPTION_BEHAVIOR.key -> "true") {
+        spark.read
+          .format("org.apache.spark.sql.test")
+          .option("paTh", "1")
+          .option("PATH", "2")
+          .option("Path", "3")
+          .option("patH", "4")
+          .load("5")
+        assert(LastOptions.parameters("path") == "5")
+      }
+    }
+  }
+
+  test("SPARK-32364: later option should override earlier options for save()") {
+    Seq(1).toDF.write
+      .format("org.apache.spark.sql.test")
+      .option("paTh", "1")
+      .option("PATH", "2")
+      .option("Path", "3")
+      .option("patH", "4")
+      .option("path", "5")
+      .save()
+    assert(LastOptions.parameters("path") == "5")
+
+    withClue("SPARK-32516: legacy path option behavior") {
+      withSQLConf(SQLConf.LEGACY_PATH_OPTION_BEHAVIOR.key -> "true") {
+        Seq(1).toDF.write
+          .format("org.apache.spark.sql.test")
+          .option("paTh", "1")
+          .option("PATH", "2")
+          .option("Path", "3")
+          .option("patH", "4")
+          .save("5")
+        assert(LastOptions.parameters("path") == "5")
+      }
+    }
+  }
+
   test("pass partitionBy as options") {
     Seq(1).toDF.write
       .format("org.apache.spark.sql.test")
@@ -330,10 +380,10 @@ class DataFrameReaderWriterSuite extends QueryTest with SharedSparkSession with
       withTable("t") {
         sql("create table t(i int, d double) using parquet")
         // Calling `saveAsTable` to an existing table with append mode results in table insertion.
-        var msg = intercept[AnalysisException] {
+        val msg = intercept[AnalysisException] {
           Seq((1L, 2.0)).toDF("i", "d").write.mode("append").saveAsTable("t")
         }.getMessage
-        assert(msg.contains("Cannot safely cast 'i': LongType to IntegerType"))
+        assert(msg.contains("Cannot safely cast 'i': bigint to int"))
 
         // Insert into table successfully.
         Seq((1, 2.0)).toDF("i", "d").write.mode("append").saveAsTable("t")
@@ -354,14 +404,14 @@ class DataFrameReaderWriterSuite extends QueryTest with SharedSparkSession with
         var msg = intercept[AnalysisException] {
           Seq(("a", "b")).toDF("i", "d").write.mode("append").saveAsTable("t")
         }.getMessage
-        assert(msg.contains("Cannot safely cast 'i': StringType to IntegerType") &&
-          msg.contains("Cannot safely cast 'd': StringType to DoubleType"))
+        assert(msg.contains("Cannot safely cast 'i': string to int") &&
+          msg.contains("Cannot safely cast 'd': string to double"))
 
         msg = intercept[AnalysisException] {
           Seq((true, false)).toDF("i", "d").write.mode("append").saveAsTable("t")
         }.getMessage
-        assert(msg.contains("Cannot safely cast 'i': BooleanType to IntegerType") &&
-          msg.contains("Cannot safely cast 'd': BooleanType to DoubleType"))
+        assert(msg.contains("Cannot safely cast 'i': boolean to int") &&
+          msg.contains("Cannot safely cast 'd': boolean to double"))
       }
     }
   }
@@ -1083,4 +1133,126 @@ class DataFrameReaderWriterSuite extends QueryTest with SharedSparkSession with
       }
     }
   }
+
+  test("SPARK-32516: 'path' or 'paths' option cannot coexist with load()'s path parameters") {
+    def verifyLoadFails(f: => DataFrame): Unit = {
+      val e = intercept[AnalysisException](f)
+      assert(e.getMessage.contains(
+        "Either remove the path option if it's the same as the path parameter"))
+    }
+
+    val path = "/tmp"
+    verifyLoadFails(spark.read.option("path", path).parquet(path))
+    verifyLoadFails(spark.read.option("path", path).parquet(""))
+    verifyLoadFails(spark.read.option("path", path).format("parquet").load(path))
+    verifyLoadFails(spark.read.option("path", path).format("parquet").load(""))
+    verifyLoadFails(spark.read.option("paths", path).parquet(path))
+    verifyLoadFails(spark.read.option("paths", path).parquet(""))
+    verifyLoadFails(spark.read.option("paths", path).format("parquet").load(path))
+    verifyLoadFails(spark.read.option("paths", path).format("parquet").load(""))
+  }
+
+  test("SPARK-32516: legacy path option behavior in load()") {
+    withSQLConf(SQLConf.LEGACY_PATH_OPTION_BEHAVIOR.key -> "true") {
+      withTempDir { dir =>
+        val path = dir.getCanonicalPath
+        Seq(1).toDF.write.mode("overwrite").parquet(path)
+
+        // When there is one path parameter to load(), "path" option is overwritten.
+        checkAnswer(spark.read.format("parquet").option("path", path).load(path), Row(1))
+
+        // When there are multiple path parameters to load(), "path" option is added.
+        checkAnswer(
+          spark.read.format("parquet").option("path", path).load(path, path),
+          Seq(Row(1), Row(1), Row(1)))
+
+        // When built-in datasource functions are invoked (e.g, `csv`, `parquet`, etc.),
+        // the path option is always added regardless of the number of path parameters.
+        checkAnswer(spark.read.option("path", path).parquet(path), Seq(Row(1), Row(1)))
+        checkAnswer(
+          spark.read.option("path", path).parquet(path, path),
+          Seq(Row(1), Row(1), Row(1)))
+      }
+    }
+  }
+
+  test("SPARK-32516: 'path' option cannot coexist with save()'s path parameter") {
+    def verifyLoadFails(f: => Unit): Unit = {
+      val e = intercept[AnalysisException](f)
+      assert(e.getMessage.contains(
+        "Either remove the path option, or call save() without the parameter"))
+    }
+
+    val df = Seq(1).toDF
+    val path = "tmp"
+    verifyLoadFails(df.write.option("path", path).parquet(path))
+    verifyLoadFails(df.write.option("path", path).parquet(""))
+    verifyLoadFails(df.write.option("path", path).format("parquet").save(path))
+    verifyLoadFails(df.write.option("path", path).format("parquet").save(""))
+  }
+
+  test("SPARK-32853: consecutive load/save calls should be allowed") {
+    val dfr = spark.read.format(classOf[FakeSourceOne].getName)
+    dfr.load("1")
+    dfr.load("2")
+    val dfw = spark.range(10).write.format(classOf[DefaultSource].getName)
+    dfw.save("1")
+    dfw.save("2")
+  }
+
+  test("SPARK-32844: DataFrameReader.table take the specified options for V1 relation") {
+    withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "parquet") {
+      withTable("t") {
+        sql("CREATE TABLE t(i int, d double) USING parquet OPTIONS ('p1'='v1', 'p2'='v2')")
+
+        val msg = intercept[AnalysisException] {
+          spark.read.option("P1", "v3").table("t").count()
+        }.getMessage
+        assert(msg.contains("duplicated key"))
+
+        val df = spark.read.option("P2", "v2").option("p3", "v3").table("t")
+        val options = df.queryExecution.analyzed.collectFirst {
+          case r: LogicalRelation => r.relation.asInstanceOf[HadoopFsRelation].options
+        }.get
+        assert(options("p2") == "v2")
+        assert(options("p3") == "v3")
+      }
+    }
+  }
+
+  test("SPARK-26164: Allow concurrent writers for multiple partitions and buckets") {
+    withTable("t1", "t2") {
+      // Uses fixed seed to ensure reproducible test execution
+      val r = new Random(31)
+      val df = spark.range(200).map(_ => {
+        val n = r.nextInt()
+        (n, n.toString, n % 5)
+      }).toDF("k1", "k2", "part")
+      df.write.format("parquet").saveAsTable("t1")
+      spark.sql("CREATE TABLE t2(k1 int, k2 string, part int) USING parquet PARTITIONED " +
+        "BY (part) CLUSTERED BY (k1) INTO 3 BUCKETS")
+      val queryToInsertTable = "INSERT OVERWRITE TABLE t2 SELECT k1, k2, part FROM t1"
+
+      Seq(
+        // Single writer
+        0,
+        // Concurrent writers without fallback
+        200,
+        // concurrent writers with fallback
+        3
+      ).foreach { maxWriters =>
+        withSQLConf(SQLConf.MAX_CONCURRENT_OUTPUT_FILE_WRITERS.key -> maxWriters.toString) {
+          spark.sql(queryToInsertTable).collect()
+          checkAnswer(spark.table("t2").orderBy("k1"),
+            spark.table("t1").orderBy("k1"))
+
+          withSQLConf(SQLConf.MAX_RECORDS_PER_FILE.key -> "1") {
+            spark.sql(queryToInsertTable).collect()
+            checkAnswer(spark.table("t2").orderBy("k1"),
+              spark.table("t1").orderBy("k1"))
+          }
+        }
+      }
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/GenericFlatSpecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/GenericFlatSpecSuite.scala
index 2861b80190abe..dfa0348f1227c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/GenericFlatSpecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/GenericFlatSpecSuite.scala
@@ -17,15 +17,13 @@
 
 package org.apache.spark.sql.test
 
-import org.scalatest.FlatSpec
-
-import org.apache.spark.sql.Dataset
+import org.scalatest.flatspec.AnyFlatSpec
 
 /**
  * The purpose of this suite is to make sure that generic FlatSpec-based scala
  * tests work with a shared spark session
  */
-class GenericFlatSpecSuite extends FlatSpec with SharedSparkSessionBase {
+class GenericFlatSpecSuite extends AnyFlatSpec with SharedSparkSessionBase {
   import testImplicits._
 
   private def ds = Seq((1, 1), (2, 1), (3, 2), (4, 2), (5, 3), (6, 3), (7, 4), (8, 4)).toDS
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/GenericFunSpecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/GenericFunSpecSuite.scala
index efdaac3ae1f0c..d15e5c42732d1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/GenericFunSpecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/GenericFunSpecSuite.scala
@@ -17,15 +17,13 @@
 
 package org.apache.spark.sql.test
 
-import org.scalatest.FunSpec
-
-import org.apache.spark.sql.Dataset
+import org.scalatest.funspec.AnyFunSpec
 
 /**
  * The purpose of this suite is to make sure that generic FunSpec-based scala
  * tests work with a shared spark session
  */
-class GenericFunSpecSuite extends FunSpec with SharedSparkSessionBase {
+class GenericFunSpecSuite extends AnyFunSpec with SharedSparkSessionBase {
   import testImplicits._
 
   private def ds = Seq((1, 1), (2, 1), (3, 2), (4, 2), (5, 3), (6, 3), (7, 4), (8, 4)).toDS
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/GenericWordSpecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/GenericWordSpecSuite.scala
index 3a3540c1dbdbf..e693dc9c9b134 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/GenericWordSpecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/GenericWordSpecSuite.scala
@@ -17,15 +17,13 @@
 
 package org.apache.spark.sql.test
 
-import org.scalatest.WordSpec
-
-import org.apache.spark.sql.Dataset
+import org.scalatest.wordspec.AnyWordSpec
 
 /**
  * The purpose of this suite is to make sure that generic WordSpec-based scala
  * tests work with a shared spark session
  */
-class GenericWordSpecSuite extends WordSpec with SharedSparkSessionBase {
+class GenericWordSpecSuite extends AnyWordSpec with SharedSparkSessionBase {
   import testImplicits._
 
   private def ds = Seq((1, 1), (2, 1), (3, 2), (4, 2), (5, 3), (6, 3), (7, 4), (8, 4)).toDS
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestData.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestData.scala
index c51faaf10f5dd..a1fd4a0215b1f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestData.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestData.scala
@@ -169,10 +169,10 @@ private[sql] trait SQLTestData { self =>
     rdd
   }
 
-  protected lazy val calenderIntervalData: RDD[IntervalData] = {
+  protected lazy val calendarIntervalData: RDD[IntervalData] = {
     val rdd = spark.sparkContext.parallelize(
       IntervalData(new CalendarInterval(1, 1, 1)) :: Nil)
-    rdd.toDF().createOrReplaceTempView("calenderIntervalData")
+    rdd.toDF().createOrReplaceTempView("calendarIntervalData")
     rdd
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
index 7be15e9d87004..0e62be40607a1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
@@ -335,7 +335,7 @@ private[sql] trait SQLTestUtilsBase
   // Blocking uncache table for tests
   protected def uncacheTable(tableName: String): Unit = {
     val tableIdent = spark.sessionState.sqlParser.parseTableIdentifier(tableName)
-    val cascade = !spark.sessionState.catalog.isTemporaryTable(tableIdent)
+    val cascade = !spark.sessionState.catalog.isTempView(tableIdent)
     spark.sharedState.cacheManager.uncacheQuery(
       spark,
       spark.table(tableName).logicalPlan,
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala
index ee29b4b8fb32b..ed2e309fa075a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala
@@ -25,13 +25,14 @@ import org.scalatest.concurrent.Eventually
 import org.apache.spark.{DebugFilesystem, SparkConf}
 import org.apache.spark.internal.config.UNSAFE_EXCEPTION_ON_MEMORY_LEAK
 import org.apache.spark.sql.{SparkSession, SQLContext}
+import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode
 import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation
 import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
 
 trait SharedSparkSession extends SQLTestUtils with SharedSparkSessionBase {
 
   /**
-   * Suites extending [[SharedSparkSession]] are sharing resources (eg. SparkSession) in their
+   * Suites extending [[SharedSparkSession]] are sharing resources (e.g. SparkSession) in their
    * tests. That trait initializes the spark session in its [[beforeAll()]] implementation before
    * the automatic thread snapshot is performed, so the audit code could fail to report threads
    * leaked by that shared session.
@@ -67,6 +68,7 @@ trait SharedSparkSessionBase
       .set("spark.hadoop.fs.file.impl", classOf[DebugFilesystem].getName)
       .set(UNSAFE_EXCEPTION_ON_MEMORY_LEAK, true)
       .set(SQLConf.CODEGEN_FALLBACK.key, "false")
+      .set(SQLConf.CODEGEN_FACTORY_MODE.key, CodegenObjectFactoryMode.CODEGEN_ONLY.toString)
       // Disable ConvertToLocalRelation for better test coverage. Test cases built on
       // LocalRelation will exercise the optimization rules better by disabling it as
       // this rule may potentially block testing of other optimization rules such as
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/TestSQLContext.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/TestSQLContext.scala
index 17603deacdcdd..47a6f3617da63 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/TestSQLContext.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/TestSQLContext.scala
@@ -25,12 +25,12 @@ import org.apache.spark.sql.internal.{SessionState, SessionStateBuilder, SQLConf
  * A special `SparkSession` prepared for testing.
  */
 private[spark] class TestSparkSession(sc: SparkContext) extends SparkSession(sc) { self =>
-  def this(sparkConf: SparkConf) {
+  def this(sparkConf: SparkConf) = {
     this(new SparkContext("local[2]", "test-sql-context",
       sparkConf.set("spark.sql.testkey", "true")))
   }
 
-  def this() {
+  def this() = {
     this(new SparkConf)
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/util/DataFrameCallbackSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/util/DataFrameCallbackSuite.scala
index b17c93503804c..b3d29df1b29bc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/util/DataFrameCallbackSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/util/DataFrameCallbackSuite.scala
@@ -26,7 +26,7 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
 import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, InsertIntoStatement, LogicalPlan, Project}
 import org.apache.spark.sql.execution.{QueryExecution, QueryExecutionException, WholeStageCodegenExec}
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
-import org.apache.spark.sql.execution.command.RunnableCommand
+import org.apache.spark.sql.execution.command.LeafRunnableCommand
 import org.apache.spark.sql.execution.datasources.{CreateTable, InsertIntoHadoopFsRelationCommand}
 import org.apache.spark.sql.execution.datasources.json.JsonFileFormat
 import org.apache.spark.sql.test.SharedSparkSession
@@ -302,7 +302,7 @@ class DataFrameCallbackSuite extends QueryTest
 }
 
 /** A test command that throws `java.lang.Error` during execution. */
-case class ErrorTestCommand(foo: String) extends RunnableCommand {
+case class ErrorTestCommand(foo: String) extends LeafRunnableCommand {
 
   override val output: Seq[Attribute] = Seq(AttributeReference("foo", StringType)())
 
diff --git a/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceSuite.scala
index 43cca246cc47c..dbc33c47fed51 100644
--- a/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceSuite.scala
@@ -54,12 +54,11 @@ object SqlResourceSuite {
       SQLPlanMetric(NUMBER_OF_OUTPUT_ROWS, 4, ""),
       SQLPlanMetric(SIZE_OF_FILES_READ, 5, ""))))
 
+  val edges: Seq[SparkPlanGraphEdge] = Seq(SparkPlanGraphEdge(3, 2))
+
   val nodesWhenCodegenIsOff: Seq[SparkPlanGraphNode] =
     SparkPlanGraph(nodes, edges).allNodes.filterNot(_.name == WHOLE_STAGE_CODEGEN_1)
 
-  val edges: Seq[SparkPlanGraphEdge] =
-    Seq(SparkPlanGraphEdge(3, 2))
-
   val metrics: Seq[SQLPlanMetric] = {
     Seq(SQLPlanMetric(DURATION, 0, ""),
       SQLPlanMetric(NUMBER_OF_OUTPUT_ROWS, 1, ""),
diff --git a/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceWithActualMetricsSuite.scala b/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceWithActualMetricsSuite.scala
new file mode 100644
index 0000000000000..1510e8957f9ae
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceWithActualMetricsSuite.scala
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.status.api.v1.sql
+
+import java.net.URL
+import java.text.SimpleDateFormat
+
+import org.json4s.DefaultFormats
+import org.json4s.jackson.JsonMethods
+
+import org.apache.spark.SparkConf
+import org.apache.spark.deploy.history.HistoryServerSuite.getContentAndCode
+import org.apache.spark.sql.DataFrame
+import org.apache.spark.sql.catalyst.plans.SQLHelper
+import org.apache.spark.sql.execution.metric.SQLMetricsTestUtils
+import org.apache.spark.sql.internal.SQLConf.ADAPTIVE_EXECUTION_ENABLED
+import org.apache.spark.sql.test.SharedSparkSession
+
+case class Person(id: Int, name: String, age: Int)
+case class Salary(personId: Int, salary: Double)
+
+/**
+ * Sql Resource Public API Unit Tests running query and extracting the metrics.
+ */
+class SqlResourceWithActualMetricsSuite
+  extends SharedSparkSession with SQLMetricsTestUtils with SQLHelper {
+
+  import testImplicits._
+
+  // Exclude nodes which may not have the metrics
+  val excludedNodes = List("WholeStageCodegen", "Project", "SerializeFromObject")
+
+  implicit val formats = new DefaultFormats {
+    override def dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss")
+  }
+
+  override def sparkConf: SparkConf = {
+    super.sparkConf.set("spark.ui.enabled", "true")
+  }
+
+  test("Check Sql Rest Api Endpoints") {
+    // Materalize result DataFrame
+    withSQLConf(ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
+      val count = getDF().count()
+      assert(count == 2, s"Expected Query Count is 2 but received: $count")
+    }
+
+    // Spark apps launched by local-mode seems not having `attemptId` as default
+    // so UT is just added for existing endpoints.
+    val executionId = callSqlRestEndpointAndVerifyResult()
+    callSqlRestEndpointByExecutionIdAndVerifyResult(executionId)
+  }
+
+  private def callSqlRestEndpointAndVerifyResult(): Long = {
+    val url = new URL(spark.sparkContext.ui.get.webUrl
+      + s"/api/v1/applications/${spark.sparkContext.applicationId}/sql")
+    val jsonResult = verifyAndGetSqlRestResult(url)
+    val executionDatas = JsonMethods.parse(jsonResult).extract[Seq[ExecutionData]]
+    assert(executionDatas.size > 0,
+      s"Expected Query Result Size is higher than 0 but received: ${executionDatas.size}")
+    val executionData = executionDatas.head
+    verifySqlRestContent(executionData)
+    executionData.id
+  }
+
+  private def callSqlRestEndpointByExecutionIdAndVerifyResult(executionId: Long): Unit = {
+    val url = new URL(spark.sparkContext.ui.get.webUrl
+      + s"/api/v1/applications/${spark.sparkContext.applicationId}/sql/${executionId}")
+    val jsonResult = verifyAndGetSqlRestResult(url)
+    val executionData = JsonMethods.parse(jsonResult).extract[ExecutionData]
+    verifySqlRestContent(executionData)
+  }
+
+  private def verifySqlRestContent(executionData: ExecutionData): Unit = {
+    assert(executionData.status == "COMPLETED",
+      s"Expected status is COMPLETED but actual: ${executionData.status}")
+    assert(executionData.successJobIds.nonEmpty,
+      s"Expected successJobIds should not be empty")
+    assert(executionData.runningJobIds.isEmpty,
+      s"Expected runningJobIds should be empty but actual: ${executionData.runningJobIds}")
+    assert(executionData.failedJobIds.isEmpty,
+      s"Expected failedJobIds should be empty but actual: ${executionData.failedJobIds}")
+    assert(executionData.nodes.nonEmpty, "Expected nodes should not be empty}")
+    executionData.nodes.filterNot(node => excludedNodes.contains(node.nodeName)).foreach { node =>
+      assert(node.metrics.nonEmpty, "Expected metrics of nodes should not be empty")
+    }
+  }
+
+  private def verifyAndGetSqlRestResult(url: URL): String = {
+    val (code, resultOpt, error) = getContentAndCode(url)
+    assert(code == 200, s"Expected Http Response Code is 200 but received: $code for url: $url")
+    assert(resultOpt.nonEmpty, s"Rest result should not be empty for url: $url")
+    assert(error.isEmpty, s"Error message should be empty for url: $url")
+    resultOpt.get
+  }
+
+  private def getDF(): DataFrame = {
+    val person: DataFrame =
+      spark.sparkContext.parallelize(
+        Person(0, "mike", 30) ::
+          Person(1, "jim", 20) :: Nil).toDF()
+
+    val salary: DataFrame =
+      spark.sparkContext.parallelize(
+        Salary(0, 2000.0) ::
+          Salary(1, 1000.0) :: Nil).toDF()
+
+    val salaryDF = salary.withColumnRenamed("personId", "id")
+    val ds = person.join(salaryDF, "id")
+      .groupBy("name", "age", "salary").avg("age", "salary")
+      .filter(_.getAs[Int]("age") <= 30)
+      .sort()
+
+    ds.toDF
+  }
+
+}
diff --git a/sql/core/v1.2/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java b/sql/core/v1.2/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
deleted file mode 100644
index 6601bcb9018f4..0000000000000
--- a/sql/core/v1.2/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.orc;
-
-import java.math.BigDecimal;
-
-import org.apache.orc.storage.ql.exec.vector.*;
-
-import org.apache.spark.sql.catalyst.util.DateTimeUtils;
-import org.apache.spark.sql.catalyst.util.RebaseDateTime;
-import org.apache.spark.sql.types.DataType;
-import org.apache.spark.sql.types.DateType;
-import org.apache.spark.sql.types.Decimal;
-import org.apache.spark.sql.types.TimestampType;
-import org.apache.spark.sql.vectorized.ColumnarArray;
-import org.apache.spark.sql.vectorized.ColumnarMap;
-import org.apache.spark.unsafe.types.UTF8String;
-
-/**
- * A column vector class wrapping Hive's ColumnVector. Because Spark ColumnarBatch only accepts
- * Spark's vectorized.ColumnVector, this column vector is used to adapt Hive ColumnVector with
- * Spark ColumnarVector.
- */
-public class OrcColumnVector extends org.apache.spark.sql.vectorized.ColumnVector {
-  private ColumnVector baseData;
-  private LongColumnVector longData;
-  private DoubleColumnVector doubleData;
-  private BytesColumnVector bytesData;
-  private DecimalColumnVector decimalData;
-  private TimestampColumnVector timestampData;
-  private final boolean isTimestamp;
-  private final boolean isDate;
-
-  private int batchSize;
-
-  OrcColumnVector(DataType type, ColumnVector vector) {
-    super(type);
-
-    if (type instanceof TimestampType) {
-      isTimestamp = true;
-    } else {
-      isTimestamp = false;
-    }
-
-    if (type instanceof DateType) {
-      isDate = true;
-    } else {
-      isDate = false;
-    }
-
-    baseData = vector;
-    if (vector instanceof LongColumnVector) {
-      longData = (LongColumnVector) vector;
-    } else if (vector instanceof DoubleColumnVector) {
-      doubleData = (DoubleColumnVector) vector;
-    } else if (vector instanceof BytesColumnVector) {
-      bytesData = (BytesColumnVector) vector;
-    } else if (vector instanceof DecimalColumnVector) {
-      decimalData = (DecimalColumnVector) vector;
-    } else if (vector instanceof TimestampColumnVector) {
-      timestampData = (TimestampColumnVector) vector;
-    } else {
-      throw new UnsupportedOperationException();
-    }
-  }
-
-  public void setBatchSize(int batchSize) {
-    this.batchSize = batchSize;
-  }
-
-  @Override
-  public void close() {
-
-  }
-
-  @Override
-  public boolean hasNull() {
-    return !baseData.noNulls;
-  }
-
-  @Override
-  public int numNulls() {
-    if (baseData.isRepeating) {
-      if (baseData.isNull[0]) {
-        return batchSize;
-      } else {
-        return 0;
-      }
-    } else if (baseData.noNulls) {
-      return 0;
-    } else {
-      int count = 0;
-      for (int i = 0; i < batchSize; i++) {
-        if (baseData.isNull[i]) count++;
-      }
-      return count;
-    }
-  }
-
-  /* A helper method to get the row index in a column. */
-  private int getRowIndex(int rowId) {
-    return baseData.isRepeating ? 0 : rowId;
-  }
-
-  @Override
-  public boolean isNullAt(int rowId) {
-    return baseData.isNull[getRowIndex(rowId)];
-  }
-
-  @Override
-  public boolean getBoolean(int rowId) {
-    return longData.vector[getRowIndex(rowId)] == 1;
-  }
-
-  @Override
-  public byte getByte(int rowId) {
-    return (byte) longData.vector[getRowIndex(rowId)];
-  }
-
-  @Override
-  public short getShort(int rowId) {
-    return (short) longData.vector[getRowIndex(rowId)];
-  }
-
-  @Override
-  public int getInt(int rowId) {
-    int value = (int) longData.vector[getRowIndex(rowId)];
-    if (isDate) {
-      return RebaseDateTime.rebaseJulianToGregorianDays(value);
-    } else {
-      return value;
-    }
-  }
-
-  @Override
-  public long getLong(int rowId) {
-    int index = getRowIndex(rowId);
-    if (isTimestamp) {
-      return DateTimeUtils.fromJavaTimestamp(timestampData.asScratchTimestamp(index));
-    } else {
-      return longData.vector[index];
-    }
-  }
-
-  @Override
-  public float getFloat(int rowId) {
-    return (float) doubleData.vector[getRowIndex(rowId)];
-  }
-
-  @Override
-  public double getDouble(int rowId) {
-    return doubleData.vector[getRowIndex(rowId)];
-  }
-
-  @Override
-  public Decimal getDecimal(int rowId, int precision, int scale) {
-    if (isNullAt(rowId)) return null;
-    BigDecimal data = decimalData.vector[getRowIndex(rowId)].getHiveDecimal().bigDecimalValue();
-    return Decimal.apply(data, precision, scale);
-  }
-
-  @Override
-  public UTF8String getUTF8String(int rowId) {
-    if (isNullAt(rowId)) return null;
-    int index = getRowIndex(rowId);
-    BytesColumnVector col = bytesData;
-    return UTF8String.fromBytes(col.vector[index], col.start[index], col.length[index]);
-  }
-
-  @Override
-  public byte[] getBinary(int rowId) {
-    if (isNullAt(rowId)) return null;
-    int index = getRowIndex(rowId);
-    byte[] binary = new byte[bytesData.length[index]];
-    System.arraycopy(bytesData.vector[index], bytesData.start[index], binary, 0, binary.length);
-    return binary;
-  }
-
-  @Override
-  public ColumnarArray getArray(int rowId) {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public ColumnarMap getMap(int rowId) {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public org.apache.spark.sql.vectorized.ColumnVector getChild(int ordinal) {
-    throw new UnsupportedOperationException();
-  }
-}
diff --git a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/DaysWritable.scala b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/DaysWritable.scala
deleted file mode 100644
index 1dccf0ca1faef..0000000000000
--- a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/DaysWritable.scala
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.orc
-
-import java.io.{DataInput, DataOutput, IOException}
-import java.sql.Date
-
-import org.apache.hadoop.io.WritableUtils
-import org.apache.orc.storage.serde2.io.DateWritable
-
-import org.apache.spark.sql.catalyst.util.RebaseDateTime.{rebaseGregorianToJulianDays, rebaseJulianToGregorianDays}
-
-/**
- * The class accepts/returns days in Gregorian calendar and rebase them
- * via conversion to local date in Julian calendar for dates before 1582-10-15
- * in read/write for backward compatibility with Spark 2.4 and earlier versions.
- *
- * This is a clone of `org.apache.spark.sql.execution.datasources.DaysWritable`.
- * The class is cloned because Hive ORC v1.2 uses different `DateWritable`:
- *   - v1.2: `org.apache.orc.storage.serde2.io.DateWritable`
- *   - v2.3 and `HiveInspectors`: `org.apache.hadoop.hive.serde2.io.DateWritable`
- *
- * @param gregorianDays The number of days since the epoch 1970-01-01 in
- *                      Gregorian calendar.
- * @param julianDays The number of days since the epoch 1970-01-01 in
- *                   Julian calendar.
- */
-class DaysWritable(
-    var gregorianDays: Int,
-    var julianDays: Int)
-  extends DateWritable {
-
-  def this() = this(0, 0)
-  def this(gregorianDays: Int) =
-    this(gregorianDays, rebaseGregorianToJulianDays(gregorianDays))
-  def this(dateWritable: DateWritable) = {
-    this(
-      gregorianDays = dateWritable match {
-        case daysWritable: DaysWritable => daysWritable.gregorianDays
-        case dateWritable: DateWritable =>
-        rebaseJulianToGregorianDays(dateWritable.getDays)
-      },
-      julianDays = dateWritable.getDays)
-  }
-
-  override def getDays: Int = julianDays
-  override def get(): Date = new Date(DateWritable.daysToMillis(julianDays))
-
-  override def set(d: Int): Unit = {
-    gregorianDays = d
-    julianDays = rebaseGregorianToJulianDays(d)
-  }
-
-  @throws[IOException]
-  override def write(out: DataOutput): Unit = {
-    WritableUtils.writeVInt(out, julianDays)
-  }
-
-  @throws[IOException]
-  override def readFields(in: DataInput): Unit = {
-    julianDays = WritableUtils.readVInt(in)
-    gregorianDays = rebaseJulianToGregorianDays(julianDays)
-  }
-}
diff --git a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
deleted file mode 100644
index a01d5a44da714..0000000000000
--- a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
+++ /dev/null
@@ -1,271 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.orc
-
-import java.time.LocalDate
-
-import org.apache.orc.storage.common.`type`.HiveDecimal
-import org.apache.orc.storage.ql.io.sarg.{PredicateLeaf, SearchArgument}
-import org.apache.orc.storage.ql.io.sarg.SearchArgument.Builder
-import org.apache.orc.storage.ql.io.sarg.SearchArgumentFactory.newBuilder
-import org.apache.orc.storage.serde2.io.HiveDecimalWritable
-
-import org.apache.spark.SparkException
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.{localDateToDays, toJavaDate}
-import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.quoteIfNeeded
-import org.apache.spark.sql.sources.Filter
-import org.apache.spark.sql.types._
-
-/**
- * Helper object for building ORC `SearchArgument`s, which are used for ORC predicate push-down.
- *
- * Due to limitation of ORC `SearchArgument` builder, we had to implement separate checking and
- * conversion passes through the Filter to make sure we only convert predicates that are known
- * to be convertible.
- *
- * An ORC `SearchArgument` must be built in one pass using a single builder.  For example, you can't
- * build `a = 1` and `b = 2` first, and then combine them into `a = 1 AND b = 2`.  This is quite
- * different from the cases in Spark SQL or Parquet, where complex filters can be easily built using
- * existing simpler ones.
- *
- * The annoying part is that, `SearchArgument` builder methods like `startAnd()`, `startOr()`, and
- * `startNot()` mutate internal state of the builder instance.  This forces us to translate all
- * convertible filters with a single builder instance. However, if we try to translate a filter
- * before checking whether it can be converted or not, we may end up with a builder whose internal
- * state is inconsistent in the case of an inconvertible filter.
- *
- * For example, to convert an `And` filter with builder `b`, we call `b.startAnd()` first, and then
- * try to convert its children.  Say we convert `left` child successfully, but find that `right`
- * child is inconvertible.  Alas, `b.startAnd()` call can't be rolled back, and `b` is inconsistent
- * now.
- *
- * The workaround employed here is to trim the Spark filters before trying to convert them. This
- * way, we can only do the actual conversion on the part of the Filter that is known to be
- * convertible.
- *
- * P.S.: Hive seems to use `SearchArgument` together with `ExprNodeGenericFuncDesc` only.  Usage of
- * builder methods mentioned above can only be found in test code, where all tested filters are
- * known to be convertible.
- */
-private[sql] object OrcFilters extends OrcFiltersBase {
-
-  /**
-   * Create ORC filter as a SearchArgument instance.
-   */
-  def createFilter(schema: StructType, filters: Seq[Filter]): Option[SearchArgument] = {
-    val dataTypeMap = schema.map(f => quoteIfNeeded(f.name) -> f.dataType).toMap
-    // Combines all convertible filters using `And` to produce a single conjunction
-    // TODO (SPARK-25557): ORC doesn't support nested predicate pushdown, so they are removed.
-    val newFilters = filters.filter(!_.containsNestedColumn)
-    val conjunctionOptional = buildTree(convertibleFilters(schema, dataTypeMap, newFilters))
-    conjunctionOptional.map { conjunction =>
-      // Then tries to build a single ORC `SearchArgument` for the conjunction predicate.
-      // The input predicate is fully convertible. There should not be any empty result in the
-      // following recursive method call `buildSearchArgument`.
-      buildSearchArgument(dataTypeMap, conjunction, newBuilder).build()
-    }
-  }
-
-  def convertibleFilters(
-      schema: StructType,
-      dataTypeMap: Map[String, DataType],
-      filters: Seq[Filter]): Seq[Filter] = {
-    import org.apache.spark.sql.sources._
-
-    def convertibleFiltersHelper(
-        filter: Filter,
-        canPartialPushDown: Boolean): Option[Filter] = filter match {
-      // At here, it is not safe to just convert one side and remove the other side
-      // if we do not understand what the parent filters are.
-      //
-      // Here is an example used to explain the reason.
-      // Let's say we have NOT(a = 2 AND b in ('1')) and we do not understand how to
-      // convert b in ('1'). If we only convert a = 2, we will end up with a filter
-      // NOT(a = 2), which will generate wrong results.
-      //
-      // Pushing one side of AND down is only safe to do at the top level or in the child
-      // AND before hitting NOT or OR conditions, and in this case, the unsupported predicate
-      // can be safely removed.
-      case And(left, right) =>
-        val leftResultOptional = convertibleFiltersHelper(left, canPartialPushDown)
-        val rightResultOptional = convertibleFiltersHelper(right, canPartialPushDown)
-        (leftResultOptional, rightResultOptional) match {
-          case (Some(leftResult), Some(rightResult)) => Some(And(leftResult, rightResult))
-          case (Some(leftResult), None) if canPartialPushDown => Some(leftResult)
-          case (None, Some(rightResult)) if canPartialPushDown => Some(rightResult)
-          case _ => None
-        }
-
-      // The Or predicate is convertible when both of its children can be pushed down.
-      // That is to say, if one/both of the children can be partially pushed down, the Or
-      // predicate can be partially pushed down as well.
-      //
-      // Here is an example used to explain the reason.
-      // Let's say we have
-      // (a1 AND a2) OR (b1 AND b2),
-      // a1 and b1 is convertible, while a2 and b2 is not.
-      // The predicate can be converted as
-      // (a1 OR b1) AND (a1 OR b2) AND (a2 OR b1) AND (a2 OR b2)
-      // As per the logical in And predicate, we can push down (a1 OR b1).
-      case Or(left, right) =>
-        for {
-          lhs <- convertibleFiltersHelper(left, canPartialPushDown)
-          rhs <- convertibleFiltersHelper(right, canPartialPushDown)
-        } yield Or(lhs, rhs)
-      case Not(pred) =>
-        val childResultOptional = convertibleFiltersHelper(pred, canPartialPushDown = false)
-        childResultOptional.map(Not)
-      case other =>
-        for (_ <- buildLeafSearchArgument(dataTypeMap, other, newBuilder())) yield other
-    }
-    filters.flatMap { filter =>
-      convertibleFiltersHelper(filter, true)
-    }
-  }
-
-  /**
-   * Get PredicateLeafType which is corresponding to the given DataType.
-   */
-  private def getPredicateLeafType(dataType: DataType) = dataType match {
-    case BooleanType => PredicateLeaf.Type.BOOLEAN
-    case ByteType | ShortType | IntegerType | LongType => PredicateLeaf.Type.LONG
-    case FloatType | DoubleType => PredicateLeaf.Type.FLOAT
-    case StringType => PredicateLeaf.Type.STRING
-    case DateType => PredicateLeaf.Type.DATE
-    case TimestampType => PredicateLeaf.Type.TIMESTAMP
-    case _: DecimalType => PredicateLeaf.Type.DECIMAL
-    case _ => throw new UnsupportedOperationException(s"DataType: ${dataType.catalogString}")
-  }
-
-  /**
-   * Cast literal values for filters.
-   *
-   * We need to cast to long because ORC raises exceptions
-   * at 'checkLiteralType' of SearchArgumentImpl.java.
-   */
-  private def castLiteralValue(value: Any, dataType: DataType): Any = dataType match {
-    case ByteType | ShortType | IntegerType | LongType =>
-      value.asInstanceOf[Number].longValue
-    case FloatType | DoubleType =>
-      value.asInstanceOf[Number].doubleValue()
-    case _: DecimalType =>
-      new HiveDecimalWritable(HiveDecimal.create(value.asInstanceOf[java.math.BigDecimal]))
-    case _: DateType if value.isInstanceOf[LocalDate] =>
-      toJavaDate(localDateToDays(value.asInstanceOf[LocalDate]))
-    case _ => value
-  }
-
-  /**
-   * Build a SearchArgument and return the builder so far.
-   *
-   * @param dataTypeMap a map from the attribute name to its data type.
-   * @param expression the input predicates, which should be fully convertible to SearchArgument.
-   * @param builder the input SearchArgument.Builder.
-   * @return the builder so far.
-   */
-  private def buildSearchArgument(
-      dataTypeMap: Map[String, DataType],
-      expression: Filter,
-      builder: Builder): Builder = {
-    import org.apache.spark.sql.sources._
-
-    expression match {
-      case And(left, right) =>
-        val lhs = buildSearchArgument(dataTypeMap, left, builder.startAnd())
-        val rhs = buildSearchArgument(dataTypeMap, right, lhs)
-        rhs.end()
-
-      case Or(left, right) =>
-        val lhs = buildSearchArgument(dataTypeMap, left, builder.startOr())
-        val rhs = buildSearchArgument(dataTypeMap, right, lhs)
-        rhs.end()
-
-      case Not(child) =>
-        buildSearchArgument(dataTypeMap, child, builder.startNot()).end()
-
-      case other =>
-        buildLeafSearchArgument(dataTypeMap, other, builder).getOrElse {
-          throw new SparkException(
-            "The input filter of OrcFilters.buildSearchArgument should be fully convertible.")
-        }
-    }
-  }
-
-  /**
-   * Build a SearchArgument for a leaf predicate and return the builder so far.
-   *
-   * @param dataTypeMap a map from the attribute name to its data type.
-   * @param expression the input filter predicates.
-   * @param builder the input SearchArgument.Builder.
-   * @return the builder so far.
-   */
-  private def buildLeafSearchArgument(
-      dataTypeMap: Map[String, DataType],
-      expression: Filter,
-      builder: Builder): Option[Builder] = {
-    def getType(attribute: String): PredicateLeaf.Type =
-      getPredicateLeafType(dataTypeMap(attribute))
-
-    import org.apache.spark.sql.sources._
-
-    // NOTE: For all case branches dealing with leaf predicates below, the additional `startAnd()`
-    // call is mandatory. ORC `SearchArgument` builder requires that all leaf predicates must be
-    // wrapped by a "parent" predicate (`And`, `Or`, or `Not`).
-    // Since ORC 1.5.0 (ORC-323), we need to quote for column names with `.` characters
-    // in order to distinguish predicate pushdown for nested columns.
-    expression match {
-      case EqualTo(name, value) if isSearchableType(dataTypeMap(name)) =>
-        val castedValue = castLiteralValue(value, dataTypeMap(name))
-        Some(builder.startAnd().equals(name, getType(name), castedValue).end())
-
-      case EqualNullSafe(name, value) if isSearchableType(dataTypeMap(name)) =>
-        val castedValue = castLiteralValue(value, dataTypeMap(name))
-        Some(builder.startAnd().nullSafeEquals(name, getType(name), castedValue).end())
-
-      case LessThan(name, value) if isSearchableType(dataTypeMap(name)) =>
-        val castedValue = castLiteralValue(value, dataTypeMap(name))
-        Some(builder.startAnd().lessThan(name, getType(name), castedValue).end())
-
-      case LessThanOrEqual(name, value) if isSearchableType(dataTypeMap(name)) =>
-        val castedValue = castLiteralValue(value, dataTypeMap(name))
-        Some(builder.startAnd().lessThanEquals(name, getType(name), castedValue).end())
-
-      case GreaterThan(name, value) if isSearchableType(dataTypeMap(name)) =>
-        val castedValue = castLiteralValue(value, dataTypeMap(name))
-        Some(builder.startNot().lessThanEquals(name, getType(name), castedValue).end())
-
-      case GreaterThanOrEqual(name, value) if isSearchableType(dataTypeMap(name)) =>
-        val castedValue = castLiteralValue(value, dataTypeMap(name))
-        Some(builder.startNot().lessThan(name, getType(name), castedValue).end())
-
-      case IsNull(name) if isSearchableType(dataTypeMap(name)) =>
-        Some(builder.startAnd().isNull(name, getType(name)).end())
-
-      case IsNotNull(name) if isSearchableType(dataTypeMap(name)) =>
-        Some(builder.startNot().isNull(name, getType(name)).end())
-
-      case In(name, values) if isSearchableType(dataTypeMap(name)) =>
-        val castedValues = values.map(v => castLiteralValue(v, dataTypeMap(name)))
-        Some(builder.startAnd().in(name, getType(name),
-          castedValues.map(_.asInstanceOf[AnyRef]): _*).end())
-
-      case _ => None
-    }
-  }
-}
-
diff --git a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala
deleted file mode 100644
index 7fbc1cd205b13..0000000000000
--- a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.orc
-
-import org.apache.orc.storage.common.`type`.HiveDecimal
-import org.apache.orc.storage.ql.exec.vector.VectorizedRowBatch
-import org.apache.orc.storage.ql.io.sarg.{SearchArgument => OrcSearchArgument}
-import org.apache.orc.storage.ql.io.sarg.PredicateLeaf.{Operator => OrcOperator}
-import org.apache.orc.storage.serde2.io.{DateWritable, HiveDecimalWritable}
-
-import org.apache.spark.sql.catalyst.expressions.SpecializedGetters
-import org.apache.spark.sql.types.Decimal
-
-/**
- * Various utilities for ORC used to upgrade the built-in Hive.
- */
-private[sql] object OrcShimUtils {
-
-  class VectorizedRowBatchWrap(val batch: VectorizedRowBatch) {}
-
-  private[sql] type Operator = OrcOperator
-  private[sql] type SearchArgument = OrcSearchArgument
-
-  def getGregorianDays(value: Any): Int = {
-    new DaysWritable(value.asInstanceOf[DateWritable]).gregorianDays
-  }
-
-  def getDecimal(value: Any): Decimal = {
-    val decimal = value.asInstanceOf[HiveDecimalWritable].getHiveDecimal()
-    Decimal(decimal.bigDecimalValue, decimal.precision(), decimal.scale())
-  }
-
-  def getDateWritable(reuseObj: Boolean): (SpecializedGetters, Int) => DateWritable = {
-    if (reuseObj) {
-      val result = new DaysWritable()
-      (getter, ordinal) =>
-        result.set(getter.getInt(ordinal))
-        result
-    } else {
-      (getter: SpecializedGetters, ordinal: Int) =>
-        new DaysWritable(getter.getInt(ordinal))
-    }
-  }
-
-  def getHiveDecimalWritable(precision: Int, scale: Int):
-      (SpecializedGetters, Int) => HiveDecimalWritable = {
-    (getter, ordinal) =>
-      val d = getter.getDecimal(ordinal, precision, scale)
-      new HiveDecimalWritable(HiveDecimal.create(d.toJavaBigDecimal))
-  }
-}
diff --git a/sql/core/v1.2/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala b/sql/core/v1.2/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
deleted file mode 100644
index a1c325e7bb876..0000000000000
--- a/sql/core/v1.2/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
+++ /dev/null
@@ -1,461 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.orc
-
-import java.math.MathContext
-import java.nio.charset.StandardCharsets
-import java.sql.{Date, Timestamp}
-
-import scala.collection.JavaConverters._
-
-import org.apache.orc.storage.ql.io.sarg.{PredicateLeaf, SearchArgument}
-
-import org.apache.spark.SparkConf
-import org.apache.spark.sql.{AnalysisException, Column, DataFrame}
-import org.apache.spark.sql.catalyst.dsl.expressions._
-import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.planning.PhysicalOperation
-import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanRelation
-import org.apache.spark.sql.execution.datasources.v2.orc.OrcScan
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.test.SharedSparkSession
-import org.apache.spark.sql.types._
-
-/**
- * A test suite that tests Apache ORC filter API based filter pushdown optimization.
- * OrcFilterSuite and HiveOrcFilterSuite is logically duplicated to provide the same test coverage.
- * The difference are the packages containing 'Predicate' and 'SearchArgument' classes.
- * - OrcFilterSuite uses 'org.apache.orc.storage.ql.io.sarg' package.
- * - HiveOrcFilterSuite uses 'org.apache.hadoop.hive.ql.io.sarg' package.
- */
-class OrcFilterSuite extends OrcTest with SharedSparkSession {
-
-  override protected def sparkConf: SparkConf =
-    super
-      .sparkConf
-      .set(SQLConf.USE_V1_SOURCE_LIST, "")
-
-  protected def checkFilterPredicate(
-      df: DataFrame,
-      predicate: Predicate,
-      checker: (SearchArgument) => Unit): Unit = {
-    val output = predicate.collect { case a: Attribute => a }.distinct
-    val query = df
-      .select(output.map(e => Column(e)): _*)
-      .where(Column(predicate))
-
-    query.queryExecution.optimizedPlan match {
-      case PhysicalOperation(_, filters, DataSourceV2ScanRelation(_, o: OrcScan, _)) =>
-        assert(filters.nonEmpty, "No filter is analyzed from the given query")
-        assert(o.pushedFilters.nonEmpty, "No filter is pushed down")
-        val maybeFilter = OrcFilters.createFilter(query.schema, o.pushedFilters)
-        assert(maybeFilter.isDefined, s"Couldn't generate filter predicate for ${o.pushedFilters}")
-        checker(maybeFilter.get)
-
-      case _ =>
-        throw new AnalysisException("Can not match OrcTable in the query.")
-    }
-  }
-
-  protected def checkFilterPredicate
-      (predicate: Predicate, filterOperator: PredicateLeaf.Operator)
-      (implicit df: DataFrame): Unit = {
-    def checkComparisonOperator(filter: SearchArgument) = {
-      val operator = filter.getLeaves.asScala
-      assert(operator.map(_.getOperator).contains(filterOperator))
-    }
-    checkFilterPredicate(df, predicate, checkComparisonOperator)
-  }
-
-  protected def checkFilterPredicate
-      (predicate: Predicate, stringExpr: String)
-      (implicit df: DataFrame): Unit = {
-    def checkLogicalOperator(filter: SearchArgument) = {
-      assert(filter.toString == stringExpr)
-    }
-    checkFilterPredicate(df, predicate, checkLogicalOperator)
-  }
-
-  test("filter pushdown - integer") {
-    withOrcDataFrame((1 to 4).map(i => Tuple1(Option(i)))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === 1, PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> 1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < 2, PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > 3, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= 1, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= 4, PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal(1) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal(1) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(2) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(3) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(1) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(4) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-    }
-  }
-
-  test("filter pushdown - long") {
-    withOrcDataFrame((1 to 4).map(i => Tuple1(Option(i.toLong)))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === 1, PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> 1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < 2, PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > 3, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= 1, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= 4, PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal(1) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal(1) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(2) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(3) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(1) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(4) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-    }
-  }
-
-  test("filter pushdown - float") {
-    withOrcDataFrame((1 to 4).map(i => Tuple1(Option(i.toFloat)))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === 1, PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> 1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < 2, PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > 3, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= 1, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= 4, PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal(1) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal(1) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(2) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(3) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(1) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(4) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-    }
-  }
-
-  test("filter pushdown - double") {
-    withOrcDataFrame((1 to 4).map(i => Tuple1(Option(i.toDouble)))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === 1, PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> 1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < 2, PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > 3, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= 1, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= 4, PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal(1) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal(1) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(2) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(3) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(1) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(4) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-    }
-  }
-
-  test("filter pushdown - string") {
-    withOrcDataFrame((1 to 4).map(i => Tuple1(i.toString))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === "1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> "1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < "2", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > "3", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= "1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= "4", PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal("1") === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal("1") <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal("2") > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal("3") < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal("1") >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal("4") <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-    }
-  }
-
-  test("filter pushdown - boolean") {
-    withOrcDataFrame((true :: false :: Nil).map(b => Tuple1.apply(Option(b)))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === true, PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> true, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < true, PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > false, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= false, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= false, PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal(false) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal(false) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(false) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(true) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(true) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(true) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-    }
-  }
-
-  test("filter pushdown - decimal") {
-    withOrcDataFrame((1 to 4).map(i => Tuple1.apply(BigDecimal.valueOf(i)))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === BigDecimal.valueOf(1), PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> BigDecimal.valueOf(1), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < BigDecimal.valueOf(2), PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > BigDecimal.valueOf(3), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= BigDecimal.valueOf(1), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= BigDecimal.valueOf(4), PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(
-        Literal(BigDecimal.valueOf(1)) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(
-        Literal(BigDecimal.valueOf(1)) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(
-        Literal(BigDecimal.valueOf(2)) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(
-        Literal(BigDecimal.valueOf(3)) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(
-        Literal(BigDecimal.valueOf(1)) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(
-        Literal(BigDecimal.valueOf(4)) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-    }
-  }
-
-  test("filter pushdown - timestamp") {
-    val timeString = "2015-08-20 14:57:00"
-    val timestamps = (1 to 4).map { i =>
-      val milliseconds = Timestamp.valueOf(timeString).getTime + i * 3600
-      new Timestamp(milliseconds)
-    }
-    withOrcDataFrame(timestamps.map(Tuple1(_))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === timestamps(0), PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> timestamps(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < timestamps(1), PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > timestamps(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= timestamps(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= timestamps(3), PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal(timestamps(0)) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal(timestamps(0)) <=> $"_1",
-        PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(timestamps(1)) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(timestamps(2)) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(timestamps(0)) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(timestamps(3)) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-    }
-  }
-
-  test("filter pushdown - combinations with logical operators") {
-    withOrcDataFrame((1 to 4).map(i => Tuple1(Option(i)))) { implicit df =>
-      checkFilterPredicate(
-        $"_1".isNotNull,
-        "leaf-0 = (IS_NULL _1), expr = (not leaf-0)"
-      )
-      checkFilterPredicate(
-        $"_1" =!= 1,
-        "leaf-0 = (IS_NULL _1), leaf-1 = (EQUALS _1 1), expr = (and (not leaf-0) (not leaf-1))"
-      )
-      checkFilterPredicate(
-        !($"_1" < 4),
-        "leaf-0 = (IS_NULL _1), leaf-1 = (LESS_THAN _1 4), expr = (and (not leaf-0) (not leaf-1))"
-      )
-      checkFilterPredicate(
-        $"_1" < 2 || $"_1" > 3,
-        "leaf-0 = (LESS_THAN _1 2), leaf-1 = (LESS_THAN_EQUALS _1 3), " +
-          "expr = (or leaf-0 (not leaf-1))"
-      )
-      checkFilterPredicate(
-        $"_1" < 2 && $"_1" > 3,
-        "leaf-0 = (IS_NULL _1), leaf-1 = (LESS_THAN _1 2), leaf-2 = (LESS_THAN_EQUALS _1 3), " +
-          "expr = (and (not leaf-0) leaf-1 (not leaf-2))"
-      )
-    }
-  }
-
-  test("filter pushdown - date") {
-    val input = Seq("2017-08-18", "2017-08-19", "2017-08-20", "2017-08-21").map { day =>
-      Date.valueOf(day)
-    }
-    withOrcFile(input.map(Tuple1(_))) { path =>
-      Seq(false, true).foreach { java8Api =>
-        withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
-          readFile(path) { implicit df =>
-            val dates = input.map(Literal(_))
-            checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-            checkFilterPredicate($"_1" === dates(0), PredicateLeaf.Operator.EQUALS)
-            checkFilterPredicate($"_1" <=> dates(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-            checkFilterPredicate($"_1" < dates(1), PredicateLeaf.Operator.LESS_THAN)
-            checkFilterPredicate($"_1" > dates(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-            checkFilterPredicate($"_1" <= dates(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-            checkFilterPredicate($"_1" >= dates(3), PredicateLeaf.Operator.LESS_THAN)
-
-            checkFilterPredicate(dates(0) === $"_1", PredicateLeaf.Operator.EQUALS)
-            checkFilterPredicate(dates(0) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-            checkFilterPredicate(dates(1) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-            checkFilterPredicate(dates(2) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-            checkFilterPredicate(dates(0) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-            checkFilterPredicate(dates(3) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-          }
-        }
-      }
-    }
-  }
-
-  test("no filter pushdown - non-supported types") {
-    implicit class IntToBinary(int: Int) {
-      def b: Array[Byte] = int.toString.getBytes(StandardCharsets.UTF_8)
-    }
-    // ArrayType
-    withOrcDataFrame((1 to 4).map(i => Tuple1(Array(i)))) { implicit df =>
-      checkNoFilterPredicate($"_1".isNull, noneSupported = true)
-    }
-    // BinaryType
-    withOrcDataFrame((1 to 4).map(i => Tuple1(i.b))) { implicit df =>
-      checkNoFilterPredicate($"_1" <=> 1.b, noneSupported = true)
-    }
-    // MapType
-    withOrcDataFrame((1 to 4).map(i => Tuple1(Map(i -> i)))) { implicit df =>
-      checkNoFilterPredicate($"_1".isNotNull, noneSupported = true)
-    }
-  }
-
-  test("SPARK-12218 and SPARK-25699 Converting conjunctions into ORC SearchArguments") {
-    import org.apache.spark.sql.sources._
-    // The `LessThan` should be converted while the `StringContains` shouldn't
-    val schema = new StructType(
-      Array(
-        StructField("a", IntegerType, nullable = true),
-        StructField("b", StringType, nullable = true)))
-    assertResult("leaf-0 = (LESS_THAN a 10), expr = leaf-0") {
-      OrcFilters.createFilter(schema, Array(
-        LessThan("a", 10),
-        StringContains("b", "prefix")
-      )).get.toString
-    }
-
-    // The `LessThan` should be converted while the whole inner `And` shouldn't
-    assertResult("leaf-0 = (LESS_THAN a 10), expr = leaf-0") {
-      OrcFilters.createFilter(schema, Array(
-        LessThan("a", 10),
-        Not(And(
-          GreaterThan("a", 1),
-          StringContains("b", "prefix")
-        ))
-      )).get.toString
-    }
-
-    // Safely remove unsupported `StringContains` predicate and push down `LessThan`
-    assertResult("leaf-0 = (LESS_THAN a 10), expr = leaf-0") {
-      OrcFilters.createFilter(schema, Array(
-        And(
-          LessThan("a", 10),
-          StringContains("b", "prefix")
-        )
-      )).get.toString
-    }
-
-    // Safely remove unsupported `StringContains` predicate, push down `LessThan` and `GreaterThan`.
-    assertResult("leaf-0 = (LESS_THAN a 10), leaf-1 = (LESS_THAN_EQUALS a 1)," +
-      " expr = (and leaf-0 (not leaf-1))") {
-      OrcFilters.createFilter(schema, Array(
-        And(
-          And(
-            LessThan("a", 10),
-            StringContains("b", "prefix")
-          ),
-          GreaterThan("a", 1)
-        )
-      )).get.toString
-    }
-  }
-
-  test("SPARK-27699 Converting disjunctions into ORC SearchArguments") {
-    import org.apache.spark.sql.sources._
-    // The `LessThan` should be converted while the `StringContains` shouldn't
-    val schema = new StructType(
-      Array(
-        StructField("a", IntegerType, nullable = true),
-        StructField("b", StringType, nullable = true)))
-
-    // The predicate `StringContains` predicate is not able to be pushed down.
-    assertResult("leaf-0 = (LESS_THAN_EQUALS a 10), leaf-1 = (LESS_THAN a 1)," +
-      " expr = (or (not leaf-0) leaf-1)") {
-      OrcFilters.createFilter(schema, Array(
-        Or(
-          GreaterThan("a", 10),
-          And(
-            StringContains("b", "prefix"),
-            LessThan("a", 1)
-          )
-        )
-      )).get.toString
-    }
-
-    assertResult("leaf-0 = (LESS_THAN_EQUALS a 10), leaf-1 = (LESS_THAN a 1)," +
-      " expr = (or (not leaf-0) leaf-1)") {
-      OrcFilters.createFilter(schema, Array(
-        Or(
-          And(
-            GreaterThan("a", 10),
-            StringContains("b", "foobar")
-          ),
-          And(
-            StringContains("b", "prefix"),
-            LessThan("a", 1)
-          )
-        )
-      )).get.toString
-    }
-
-    assert(OrcFilters.createFilter(schema, Array(
-      Or(
-        StringContains("b", "foobar"),
-        And(
-          StringContains("b", "prefix"),
-          LessThan("a", 1)
-        )
-      )
-    )).isEmpty)
-  }
-
-  test("SPARK-27160: Fix casting of the DecimalType literal") {
-    import org.apache.spark.sql.sources._
-    val schema = StructType(Array(StructField("a", DecimalType(3, 2))))
-    assertResult("leaf-0 = (LESS_THAN a 3.14), expr = leaf-0") {
-      OrcFilters.createFilter(schema, Array(
-        LessThan(
-          "a",
-          new java.math.BigDecimal(3.14, MathContext.DECIMAL64).setScale(2)))
-      ).get.toString
-    }
-  }
-}
-
diff --git a/sql/core/v2.3/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java b/sql/core/v2.3/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
deleted file mode 100644
index 6e55fedfc4deb..0000000000000
--- a/sql/core/v2.3/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.orc;
-
-import java.math.BigDecimal;
-
-import org.apache.hadoop.hive.ql.exec.vector.*;
-
-import org.apache.spark.sql.catalyst.util.DateTimeUtils;
-import org.apache.spark.sql.catalyst.util.RebaseDateTime;
-import org.apache.spark.sql.types.DataType;
-import org.apache.spark.sql.types.DateType;
-import org.apache.spark.sql.types.Decimal;
-import org.apache.spark.sql.types.TimestampType;
-import org.apache.spark.sql.vectorized.ColumnarArray;
-import org.apache.spark.sql.vectorized.ColumnarMap;
-import org.apache.spark.unsafe.types.UTF8String;
-
-/**
- * A column vector class wrapping Hive's ColumnVector. Because Spark ColumnarBatch only accepts
- * Spark's vectorized.ColumnVector, this column vector is used to adapt Hive ColumnVector with
- * Spark ColumnarVector.
- */
-public class OrcColumnVector extends org.apache.spark.sql.vectorized.ColumnVector {
-  private ColumnVector baseData;
-  private LongColumnVector longData;
-  private DoubleColumnVector doubleData;
-  private BytesColumnVector bytesData;
-  private DecimalColumnVector decimalData;
-  private TimestampColumnVector timestampData;
-  private final boolean isTimestamp;
-  private final boolean isDate;
-
-  private int batchSize;
-
-  OrcColumnVector(DataType type, ColumnVector vector) {
-    super(type);
-
-    if (type instanceof TimestampType) {
-      isTimestamp = true;
-    } else {
-      isTimestamp = false;
-    }
-
-    if (type instanceof DateType) {
-      isDate = true;
-    } else {
-      isDate = false;
-    }
-
-    baseData = vector;
-    if (vector instanceof LongColumnVector) {
-      longData = (LongColumnVector) vector;
-    } else if (vector instanceof DoubleColumnVector) {
-      doubleData = (DoubleColumnVector) vector;
-    } else if (vector instanceof BytesColumnVector) {
-      bytesData = (BytesColumnVector) vector;
-    } else if (vector instanceof DecimalColumnVector) {
-      decimalData = (DecimalColumnVector) vector;
-    } else if (vector instanceof TimestampColumnVector) {
-      timestampData = (TimestampColumnVector) vector;
-    } else {
-      throw new UnsupportedOperationException();
-    }
-  }
-
-  public void setBatchSize(int batchSize) {
-    this.batchSize = batchSize;
-  }
-
-  @Override
-  public void close() {
-
-  }
-
-  @Override
-  public boolean hasNull() {
-    return !baseData.noNulls;
-  }
-
-  @Override
-  public int numNulls() {
-    if (baseData.isRepeating) {
-      if (baseData.isNull[0]) {
-        return batchSize;
-      } else {
-        return 0;
-      }
-    } else if (baseData.noNulls) {
-      return 0;
-    } else {
-      int count = 0;
-      for (int i = 0; i < batchSize; i++) {
-        if (baseData.isNull[i]) count++;
-      }
-      return count;
-    }
-  }
-
-  /* A helper method to get the row index in a column. */
-  private int getRowIndex(int rowId) {
-    return baseData.isRepeating ? 0 : rowId;
-  }
-
-  @Override
-  public boolean isNullAt(int rowId) {
-    return baseData.isNull[getRowIndex(rowId)];
-  }
-
-  @Override
-  public boolean getBoolean(int rowId) {
-    return longData.vector[getRowIndex(rowId)] == 1;
-  }
-
-  @Override
-  public byte getByte(int rowId) {
-    return (byte) longData.vector[getRowIndex(rowId)];
-  }
-
-  @Override
-  public short getShort(int rowId) {
-    return (short) longData.vector[getRowIndex(rowId)];
-  }
-
-  @Override
-  public int getInt(int rowId) {
-    int value = (int) longData.vector[getRowIndex(rowId)];
-    if (isDate) {
-      return RebaseDateTime.rebaseJulianToGregorianDays(value);
-    } else {
-      return value;
-    }
-  }
-
-  @Override
-  public long getLong(int rowId) {
-    int index = getRowIndex(rowId);
-    if (isTimestamp) {
-      return DateTimeUtils.fromJavaTimestamp(timestampData.asScratchTimestamp(index));
-    } else {
-      return longData.vector[index];
-    }
-  }
-
-  @Override
-  public float getFloat(int rowId) {
-    return (float) doubleData.vector[getRowIndex(rowId)];
-  }
-
-  @Override
-  public double getDouble(int rowId) {
-    return doubleData.vector[getRowIndex(rowId)];
-  }
-
-  @Override
-  public Decimal getDecimal(int rowId, int precision, int scale) {
-    if (isNullAt(rowId)) return null;
-    BigDecimal data = decimalData.vector[getRowIndex(rowId)].getHiveDecimal().bigDecimalValue();
-    return Decimal.apply(data, precision, scale);
-  }
-
-  @Override
-  public UTF8String getUTF8String(int rowId) {
-    if (isNullAt(rowId)) return null;
-    int index = getRowIndex(rowId);
-    BytesColumnVector col = bytesData;
-    return UTF8String.fromBytes(col.vector[index], col.start[index], col.length[index]);
-  }
-
-  @Override
-  public byte[] getBinary(int rowId) {
-    if (isNullAt(rowId)) return null;
-    int index = getRowIndex(rowId);
-    byte[] binary = new byte[bytesData.length[index]];
-    System.arraycopy(bytesData.vector[index], bytesData.start[index], binary, 0, binary.length);
-    return binary;
-  }
-
-  @Override
-  public ColumnarArray getArray(int rowId) {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public ColumnarMap getMap(int rowId) {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public org.apache.spark.sql.vectorized.ColumnVector getChild(int ordinal) {
-    throw new UnsupportedOperationException();
-  }
-}
diff --git a/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala b/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
deleted file mode 100644
index 445a52cece1c3..0000000000000
--- a/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.orc
-
-import java.time.LocalDate
-
-import org.apache.hadoop.hive.common.`type`.HiveDecimal
-import org.apache.hadoop.hive.ql.io.sarg.{PredicateLeaf, SearchArgument}
-import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.Builder
-import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory.newBuilder
-import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable
-
-import org.apache.spark.SparkException
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.{localDateToDays, toJavaDate}
-import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.quoteIfNeeded
-import org.apache.spark.sql.sources.Filter
-import org.apache.spark.sql.types._
-
-/**
- * Helper object for building ORC `SearchArgument`s, which are used for ORC predicate push-down.
- *
- * Due to limitation of ORC `SearchArgument` builder, we had to implement separate checking and
- * conversion passes through the Filter to make sure we only convert predicates that are known
- * to be convertible.
- *
- * An ORC `SearchArgument` must be built in one pass using a single builder.  For example, you can't
- * build `a = 1` and `b = 2` first, and then combine them into `a = 1 AND b = 2`.  This is quite
- * different from the cases in Spark SQL or Parquet, where complex filters can be easily built using
- * existing simpler ones.
- *
- * The annoying part is that, `SearchArgument` builder methods like `startAnd()`, `startOr()`, and
- * `startNot()` mutate internal state of the builder instance.  This forces us to translate all
- * convertible filters with a single builder instance. However, if we try to translate a filter
- * before checking whether it can be converted or not, we may end up with a builder whose internal
- * state is inconsistent in the case of an inconvertible filter.
- *
- * For example, to convert an `And` filter with builder `b`, we call `b.startAnd()` first, and then
- * try to convert its children.  Say we convert `left` child successfully, but find that `right`
- * child is inconvertible.  Alas, `b.startAnd()` call can't be rolled back, and `b` is inconsistent
- * now.
- *
- * The workaround employed here is to trim the Spark filters before trying to convert them. This
- * way, we can only do the actual conversion on the part of the Filter that is known to be
- * convertible.
- *
- * P.S.: Hive seems to use `SearchArgument` together with `ExprNodeGenericFuncDesc` only.  Usage of
- * builder methods mentioned above can only be found in test code, where all tested filters are
- * known to be convertible.
- */
-private[sql] object OrcFilters extends OrcFiltersBase {
-
-  /**
-   * Create ORC filter as a SearchArgument instance.
-   */
-  def createFilter(schema: StructType, filters: Seq[Filter]): Option[SearchArgument] = {
-    val dataTypeMap = schema.map(f => quoteIfNeeded(f.name) -> f.dataType).toMap
-    // Combines all convertible filters using `And` to produce a single conjunction
-    // TODO (SPARK-25557): ORC doesn't support nested predicate pushdown, so they are removed.
-    val newFilters = filters.filter(!_.containsNestedColumn)
-    val conjunctionOptional = buildTree(convertibleFilters(schema, dataTypeMap, newFilters))
-    conjunctionOptional.map { conjunction =>
-      // Then tries to build a single ORC `SearchArgument` for the conjunction predicate.
-      // The input predicate is fully convertible. There should not be any empty result in the
-      // following recursive method call `buildSearchArgument`.
-      buildSearchArgument(dataTypeMap, conjunction, newBuilder).build()
-    }
-  }
-
-  def convertibleFilters(
-      schema: StructType,
-      dataTypeMap: Map[String, DataType],
-      filters: Seq[Filter]): Seq[Filter] = {
-    import org.apache.spark.sql.sources._
-
-    def convertibleFiltersHelper(
-        filter: Filter,
-        canPartialPushDown: Boolean): Option[Filter] = filter match {
-      // At here, it is not safe to just convert one side and remove the other side
-      // if we do not understand what the parent filters are.
-      //
-      // Here is an example used to explain the reason.
-      // Let's say we have NOT(a = 2 AND b in ('1')) and we do not understand how to
-      // convert b in ('1'). If we only convert a = 2, we will end up with a filter
-      // NOT(a = 2), which will generate wrong results.
-      //
-      // Pushing one side of AND down is only safe to do at the top level or in the child
-      // AND before hitting NOT or OR conditions, and in this case, the unsupported predicate
-      // can be safely removed.
-      case And(left, right) =>
-        val leftResultOptional = convertibleFiltersHelper(left, canPartialPushDown)
-        val rightResultOptional = convertibleFiltersHelper(right, canPartialPushDown)
-        (leftResultOptional, rightResultOptional) match {
-          case (Some(leftResult), Some(rightResult)) => Some(And(leftResult, rightResult))
-          case (Some(leftResult), None) if canPartialPushDown => Some(leftResult)
-          case (None, Some(rightResult)) if canPartialPushDown => Some(rightResult)
-          case _ => None
-        }
-
-      // The Or predicate is convertible when both of its children can be pushed down.
-      // That is to say, if one/both of the children can be partially pushed down, the Or
-      // predicate can be partially pushed down as well.
-      //
-      // Here is an example used to explain the reason.
-      // Let's say we have
-      // (a1 AND a2) OR (b1 AND b2),
-      // a1 and b1 is convertible, while a2 and b2 is not.
-      // The predicate can be converted as
-      // (a1 OR b1) AND (a1 OR b2) AND (a2 OR b1) AND (a2 OR b2)
-      // As per the logical in And predicate, we can push down (a1 OR b1).
-      case Or(left, right) =>
-        for {
-          lhs <- convertibleFiltersHelper(left, canPartialPushDown)
-          rhs <- convertibleFiltersHelper(right, canPartialPushDown)
-        } yield Or(lhs, rhs)
-      case Not(pred) =>
-        val childResultOptional = convertibleFiltersHelper(pred, canPartialPushDown = false)
-        childResultOptional.map(Not)
-      case other =>
-        for (_ <- buildLeafSearchArgument(dataTypeMap, other, newBuilder())) yield other
-    }
-    filters.flatMap { filter =>
-      convertibleFiltersHelper(filter, true)
-    }
-  }
-
-  /**
-   * Get PredicateLeafType which is corresponding to the given DataType.
-   */
-  private def getPredicateLeafType(dataType: DataType) = dataType match {
-    case BooleanType => PredicateLeaf.Type.BOOLEAN
-    case ByteType | ShortType | IntegerType | LongType => PredicateLeaf.Type.LONG
-    case FloatType | DoubleType => PredicateLeaf.Type.FLOAT
-    case StringType => PredicateLeaf.Type.STRING
-    case DateType => PredicateLeaf.Type.DATE
-    case TimestampType => PredicateLeaf.Type.TIMESTAMP
-    case _: DecimalType => PredicateLeaf.Type.DECIMAL
-    case _ => throw new UnsupportedOperationException(s"DataType: ${dataType.catalogString}")
-  }
-
-  /**
-   * Cast literal values for filters.
-   *
-   * We need to cast to long because ORC raises exceptions
-   * at 'checkLiteralType' of SearchArgumentImpl.java.
-   */
-  private def castLiteralValue(value: Any, dataType: DataType): Any = dataType match {
-    case ByteType | ShortType | IntegerType | LongType =>
-      value.asInstanceOf[Number].longValue
-    case FloatType | DoubleType =>
-      value.asInstanceOf[Number].doubleValue()
-    case _: DecimalType =>
-      new HiveDecimalWritable(HiveDecimal.create(value.asInstanceOf[java.math.BigDecimal]))
-    case _: DateType if value.isInstanceOf[LocalDate] =>
-      toJavaDate(localDateToDays(value.asInstanceOf[LocalDate]))
-    case _ => value
-  }
-
-  /**
-   * Build a SearchArgument and return the builder so far.
-   *
-   * @param dataTypeMap a map from the attribute name to its data type.
-   * @param expression the input predicates, which should be fully convertible to SearchArgument.
-   * @param builder the input SearchArgument.Builder.
-   * @return the builder so far.
-   */
-  private def buildSearchArgument(
-      dataTypeMap: Map[String, DataType],
-      expression: Filter,
-      builder: Builder): Builder = {
-    import org.apache.spark.sql.sources._
-
-    expression match {
-      case And(left, right) =>
-        val lhs = buildSearchArgument(dataTypeMap, left, builder.startAnd())
-        val rhs = buildSearchArgument(dataTypeMap, right, lhs)
-        rhs.end()
-
-      case Or(left, right) =>
-        val lhs = buildSearchArgument(dataTypeMap, left, builder.startOr())
-        val rhs = buildSearchArgument(dataTypeMap, right, lhs)
-        rhs.end()
-
-      case Not(child) =>
-        buildSearchArgument(dataTypeMap, child, builder.startNot()).end()
-
-      case other =>
-        buildLeafSearchArgument(dataTypeMap, other, builder).getOrElse {
-          throw new SparkException(
-            "The input filter of OrcFilters.buildSearchArgument should be fully convertible.")
-        }
-    }
-  }
-
-  /**
-   * Build a SearchArgument for a leaf predicate and return the builder so far.
-   *
-   * @param dataTypeMap a map from the attribute name to its data type.
-   * @param expression the input filter predicates.
-   * @param builder the input SearchArgument.Builder.
-   * @return the builder so far.
-   */
-  private def buildLeafSearchArgument(
-      dataTypeMap: Map[String, DataType],
-      expression: Filter,
-      builder: Builder): Option[Builder] = {
-    def getType(attribute: String): PredicateLeaf.Type =
-      getPredicateLeafType(dataTypeMap(attribute))
-
-    import org.apache.spark.sql.sources._
-
-    // NOTE: For all case branches dealing with leaf predicates below, the additional `startAnd()`
-    // call is mandatory. ORC `SearchArgument` builder requires that all leaf predicates must be
-    // wrapped by a "parent" predicate (`And`, `Or`, or `Not`).
-    // Since ORC 1.5.0 (ORC-323), we need to quote for column names with `.` characters
-    // in order to distinguish predicate pushdown for nested columns.
-    expression match {
-      case EqualTo(name, value) if isSearchableType(dataTypeMap(name)) =>
-        val castedValue = castLiteralValue(value, dataTypeMap(name))
-        Some(builder.startAnd().equals(name, getType(name), castedValue).end())
-
-      case EqualNullSafe(name, value) if isSearchableType(dataTypeMap(name)) =>
-        val castedValue = castLiteralValue(value, dataTypeMap(name))
-        Some(builder.startAnd().nullSafeEquals(name, getType(name), castedValue).end())
-
-      case LessThan(name, value) if isSearchableType(dataTypeMap(name)) =>
-        val castedValue = castLiteralValue(value, dataTypeMap(name))
-        Some(builder.startAnd().lessThan(name, getType(name), castedValue).end())
-
-      case LessThanOrEqual(name, value) if isSearchableType(dataTypeMap(name)) =>
-        val castedValue = castLiteralValue(value, dataTypeMap(name))
-        Some(builder.startAnd().lessThanEquals(name, getType(name), castedValue).end())
-
-      case GreaterThan(name, value) if isSearchableType(dataTypeMap(name)) =>
-        val castedValue = castLiteralValue(value, dataTypeMap(name))
-        Some(builder.startNot().lessThanEquals(name, getType(name), castedValue).end())
-
-      case GreaterThanOrEqual(name, value) if isSearchableType(dataTypeMap(name)) =>
-        val castedValue = castLiteralValue(value, dataTypeMap(name))
-        Some(builder.startNot().lessThan(name, getType(name), castedValue).end())
-
-      case IsNull(name) if isSearchableType(dataTypeMap(name)) =>
-        Some(builder.startAnd().isNull(name, getType(name)).end())
-
-      case IsNotNull(name) if isSearchableType(dataTypeMap(name)) =>
-        Some(builder.startNot().isNull(name, getType(name)).end())
-
-      case In(name, values) if isSearchableType(dataTypeMap(name)) =>
-        val castedValues = values.map(v => castLiteralValue(v, dataTypeMap(name)))
-        Some(builder.startAnd().in(name, getType(name),
-          castedValues.map(_.asInstanceOf[AnyRef]): _*).end())
-
-      case _ => None
-    }
-  }
-}
diff --git a/sql/core/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala b/sql/core/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
deleted file mode 100644
index 815af05beb002..0000000000000
--- a/sql/core/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
+++ /dev/null
@@ -1,462 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.orc
-
-import java.math.MathContext
-import java.nio.charset.StandardCharsets
-import java.sql.{Date, Timestamp}
-
-import scala.collection.JavaConverters._
-
-import org.apache.hadoop.hive.ql.io.sarg.{PredicateLeaf, SearchArgument}
-
-import org.apache.spark.SparkConf
-import org.apache.spark.sql.{AnalysisException, Column, DataFrame}
-import org.apache.spark.sql.catalyst.dsl.expressions._
-import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.planning.PhysicalOperation
-import org.apache.spark.sql.execution.datasources.{DataSourceStrategy, HadoopFsRelation, LogicalRelation}
-import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Relation, DataSourceV2ScanRelation}
-import org.apache.spark.sql.execution.datasources.v2.orc.{OrcScan, OrcTable}
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.test.SharedSparkSession
-import org.apache.spark.sql.types._
-
-/**
- * A test suite that tests Apache ORC filter API based filter pushdown optimization.
- * OrcFilterSuite and HiveOrcFilterSuite is logically duplicated to provide the same test coverage.
- * The difference are the packages containing 'Predicate' and 'SearchArgument' classes.
- * - OrcFilterSuite uses 'org.apache.orc.storage.ql.io.sarg' package.
- * - HiveOrcFilterSuite uses 'org.apache.hadoop.hive.ql.io.sarg' package.
- */
-class OrcFilterSuite extends OrcTest with SharedSparkSession {
-
-  override protected def sparkConf: SparkConf =
-    super
-      .sparkConf
-      .set(SQLConf.USE_V1_SOURCE_LIST, "")
-
-  protected def checkFilterPredicate(
-      df: DataFrame,
-      predicate: Predicate,
-      checker: (SearchArgument) => Unit): Unit = {
-    val output = predicate.collect { case a: Attribute => a }.distinct
-    val query = df
-      .select(output.map(e => Column(e)): _*)
-      .where(Column(predicate))
-
-    query.queryExecution.optimizedPlan match {
-      case PhysicalOperation(_, filters, DataSourceV2ScanRelation(_, o: OrcScan, _)) =>
-        assert(filters.nonEmpty, "No filter is analyzed from the given query")
-        assert(o.pushedFilters.nonEmpty, "No filter is pushed down")
-        val maybeFilter = OrcFilters.createFilter(query.schema, o.pushedFilters)
-        assert(maybeFilter.isDefined, s"Couldn't generate filter predicate for ${o.pushedFilters}")
-        checker(maybeFilter.get)
-
-      case _ =>
-        throw new AnalysisException("Can not match OrcTable in the query.")
-    }
-  }
-
-  protected def checkFilterPredicate
-      (predicate: Predicate, filterOperator: PredicateLeaf.Operator)
-      (implicit df: DataFrame): Unit = {
-    def checkComparisonOperator(filter: SearchArgument) = {
-      val operator = filter.getLeaves.asScala
-      assert(operator.map(_.getOperator).contains(filterOperator))
-    }
-    checkFilterPredicate(df, predicate, checkComparisonOperator)
-  }
-
-  protected def checkFilterPredicate
-      (predicate: Predicate, stringExpr: String)
-      (implicit df: DataFrame): Unit = {
-    def checkLogicalOperator(filter: SearchArgument) = {
-      assert(filter.toString == stringExpr)
-    }
-    checkFilterPredicate(df, predicate, checkLogicalOperator)
-  }
-
-  test("filter pushdown - integer") {
-    withOrcDataFrame((1 to 4).map(i => Tuple1(Option(i)))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === 1, PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> 1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < 2, PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > 3, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= 1, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= 4, PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal(1) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal(1) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(2) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(3) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(1) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(4) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-    }
-  }
-
-  test("filter pushdown - long") {
-    withOrcDataFrame((1 to 4).map(i => Tuple1(Option(i.toLong)))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === 1, PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> 1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < 2, PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > 3, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= 1, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= 4, PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal(1) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal(1) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(2) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(3) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(1) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(4) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-    }
-  }
-
-  test("filter pushdown - float") {
-    withOrcDataFrame((1 to 4).map(i => Tuple1(Option(i.toFloat)))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === 1, PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> 1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < 2, PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > 3, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= 1, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= 4, PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal(1) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal(1) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(2) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(3) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(1) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(4) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-    }
-  }
-
-  test("filter pushdown - double") {
-    withOrcDataFrame((1 to 4).map(i => Tuple1(Option(i.toDouble)))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === 1, PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> 1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < 2, PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > 3, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= 1, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= 4, PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal(1) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal(1) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(2) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(3) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(1) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(4) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-    }
-  }
-
-  test("filter pushdown - string") {
-    withOrcDataFrame((1 to 4).map(i => Tuple1(i.toString))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === "1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> "1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < "2", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > "3", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= "1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= "4", PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal("1") === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal("1") <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal("2") > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal("3") < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal("1") >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal("4") <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-    }
-  }
-
-  test("filter pushdown - boolean") {
-    withOrcDataFrame((true :: false :: Nil).map(b => Tuple1.apply(Option(b)))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === true, PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> true, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < true, PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > false, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= false, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= false, PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal(false) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal(false) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(false) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(true) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(true) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(true) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-    }
-  }
-
-  test("filter pushdown - decimal") {
-    withOrcDataFrame((1 to 4).map(i => Tuple1.apply(BigDecimal.valueOf(i)))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === BigDecimal.valueOf(1), PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> BigDecimal.valueOf(1), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < BigDecimal.valueOf(2), PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > BigDecimal.valueOf(3), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= BigDecimal.valueOf(1), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= BigDecimal.valueOf(4), PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(
-        Literal(BigDecimal.valueOf(1)) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(
-        Literal(BigDecimal.valueOf(1)) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(
-        Literal(BigDecimal.valueOf(2)) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(
-        Literal(BigDecimal.valueOf(3)) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(
-        Literal(BigDecimal.valueOf(1)) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(
-        Literal(BigDecimal.valueOf(4)) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-    }
-  }
-
-  test("filter pushdown - timestamp") {
-    val timeString = "2015-08-20 14:57:00"
-    val timestamps = (1 to 4).map { i =>
-      val milliseconds = Timestamp.valueOf(timeString).getTime + i * 3600
-      new Timestamp(milliseconds)
-    }
-    withOrcDataFrame(timestamps.map(Tuple1(_))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === timestamps(0), PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> timestamps(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < timestamps(1), PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > timestamps(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= timestamps(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= timestamps(3), PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal(timestamps(0)) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(
-        Literal(timestamps(0)) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(timestamps(1)) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(timestamps(2)) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(timestamps(0)) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(timestamps(3)) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-    }
-  }
-
-  test("filter pushdown - combinations with logical operators") {
-    withOrcDataFrame((1 to 4).map(i => Tuple1(Option(i)))) { implicit df =>
-      checkFilterPredicate(
-        $"_1".isNotNull,
-        "leaf-0 = (IS_NULL _1), expr = (not leaf-0)"
-      )
-      checkFilterPredicate(
-        $"_1" =!= 1,
-        "leaf-0 = (IS_NULL _1), leaf-1 = (EQUALS _1 1), expr = (and (not leaf-0) (not leaf-1))"
-      )
-      checkFilterPredicate(
-        !($"_1" < 4),
-        "leaf-0 = (IS_NULL _1), leaf-1 = (LESS_THAN _1 4), expr = (and (not leaf-0) (not leaf-1))"
-      )
-      checkFilterPredicate(
-        $"_1" < 2 || $"_1" > 3,
-        "leaf-0 = (LESS_THAN _1 2), leaf-1 = (LESS_THAN_EQUALS _1 3), " +
-          "expr = (or leaf-0 (not leaf-1))"
-      )
-      checkFilterPredicate(
-        $"_1" < 2 && $"_1" > 3,
-        "leaf-0 = (IS_NULL _1), leaf-1 = (LESS_THAN _1 2), leaf-2 = (LESS_THAN_EQUALS _1 3), " +
-          "expr = (and (not leaf-0) leaf-1 (not leaf-2))"
-      )
-    }
-  }
-
-  test("filter pushdown - date") {
-    val input = Seq("2017-08-18", "2017-08-19", "2017-08-20", "2017-08-21").map { day =>
-      Date.valueOf(day)
-    }
-    withOrcFile(input.map(Tuple1(_))) { path =>
-      Seq(false, true).foreach { java8Api =>
-        withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
-          readFile(path) { implicit df =>
-            val dates = input.map(Literal(_))
-            checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-            checkFilterPredicate($"_1" === dates(0), PredicateLeaf.Operator.EQUALS)
-            checkFilterPredicate($"_1" <=> dates(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-            checkFilterPredicate($"_1" < dates(1), PredicateLeaf.Operator.LESS_THAN)
-            checkFilterPredicate($"_1" > dates(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-            checkFilterPredicate($"_1" <= dates(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-            checkFilterPredicate($"_1" >= dates(3), PredicateLeaf.Operator.LESS_THAN)
-
-            checkFilterPredicate(dates(0) === $"_1", PredicateLeaf.Operator.EQUALS)
-            checkFilterPredicate(dates(0) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-            checkFilterPredicate(dates(1) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-            checkFilterPredicate(dates(2) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-            checkFilterPredicate(dates(0) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-            checkFilterPredicate(dates(3) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-          }
-        }
-      }
-    }
-  }
-
-  test("no filter pushdown - non-supported types") {
-    implicit class IntToBinary(int: Int) {
-      def b: Array[Byte] = int.toString.getBytes(StandardCharsets.UTF_8)
-    }
-    // ArrayType
-    withOrcDataFrame((1 to 4).map(i => Tuple1(Array(i)))) { implicit df =>
-      checkNoFilterPredicate($"_1".isNull, noneSupported = true)
-    }
-    // BinaryType
-    withOrcDataFrame((1 to 4).map(i => Tuple1(i.b))) { implicit df =>
-      checkNoFilterPredicate($"_1" <=> 1.b, noneSupported = true)
-    }
-    // MapType
-    withOrcDataFrame((1 to 4).map(i => Tuple1(Map(i -> i)))) { implicit df =>
-      checkNoFilterPredicate($"_1".isNotNull, noneSupported = true)
-    }
-  }
-
-  test("SPARK-12218 and SPARK-25699 Converting conjunctions into ORC SearchArguments") {
-    import org.apache.spark.sql.sources._
-    // The `LessThan` should be converted while the `StringContains` shouldn't
-    val schema = new StructType(
-      Array(
-        StructField("a", IntegerType, nullable = true),
-        StructField("b", StringType, nullable = true)))
-    assertResult("leaf-0 = (LESS_THAN a 10), expr = leaf-0") {
-      OrcFilters.createFilter(schema, Array(
-        LessThan("a", 10),
-        StringContains("b", "prefix")
-      )).get.toString
-    }
-
-    // The `LessThan` should be converted while the whole inner `And` shouldn't
-    assertResult("leaf-0 = (LESS_THAN a 10), expr = leaf-0") {
-      OrcFilters.createFilter(schema, Array(
-        LessThan("a", 10),
-        Not(And(
-          GreaterThan("a", 1),
-          StringContains("b", "prefix")
-        ))
-      )).get.toString
-    }
-
-    // Safely remove unsupported `StringContains` predicate and push down `LessThan`
-    assertResult("leaf-0 = (LESS_THAN a 10), expr = leaf-0") {
-      OrcFilters.createFilter(schema, Array(
-        And(
-          LessThan("a", 10),
-          StringContains("b", "prefix")
-        )
-      )).get.toString
-    }
-
-    // Safely remove unsupported `StringContains` predicate, push down `LessThan` and `GreaterThan`.
-    assertResult("leaf-0 = (LESS_THAN a 10), leaf-1 = (LESS_THAN_EQUALS a 1)," +
-      " expr = (and leaf-0 (not leaf-1))") {
-      OrcFilters.createFilter(schema, Array(
-        And(
-          And(
-            LessThan("a", 10),
-            StringContains("b", "prefix")
-          ),
-          GreaterThan("a", 1)
-        )
-      )).get.toString
-    }
-  }
-
-  test("SPARK-27699 Converting disjunctions into ORC SearchArguments") {
-    import org.apache.spark.sql.sources._
-    // The `LessThan` should be converted while the `StringContains` shouldn't
-    val schema = new StructType(
-      Array(
-        StructField("a", IntegerType, nullable = true),
-        StructField("b", StringType, nullable = true)))
-
-    // The predicate `StringContains` predicate is not able to be pushed down.
-    assertResult("leaf-0 = (LESS_THAN_EQUALS a 10), leaf-1 = (LESS_THAN a 1)," +
-      " expr = (or (not leaf-0) leaf-1)") {
-      OrcFilters.createFilter(schema, Array(
-        Or(
-          GreaterThan("a", 10),
-          And(
-            StringContains("b", "prefix"),
-            LessThan("a", 1)
-          )
-        )
-      )).get.toString
-    }
-
-    assertResult("leaf-0 = (LESS_THAN_EQUALS a 10), leaf-1 = (LESS_THAN a 1)," +
-      " expr = (or (not leaf-0) leaf-1)") {
-      OrcFilters.createFilter(schema, Array(
-        Or(
-          And(
-            GreaterThan("a", 10),
-            StringContains("b", "foobar")
-          ),
-          And(
-            StringContains("b", "prefix"),
-            LessThan("a", 1)
-          )
-        )
-      )).get.toString
-    }
-
-    assert(OrcFilters.createFilter(schema, Array(
-      Or(
-        StringContains("b", "foobar"),
-        And(
-          StringContains("b", "prefix"),
-          LessThan("a", 1)
-        )
-      )
-    )).isEmpty)
-  }
-
-  test("SPARK-27160: Fix casting of the DecimalType literal") {
-    import org.apache.spark.sql.sources._
-    val schema = StructType(Array(StructField("a", DecimalType(3, 2))))
-    assertResult("leaf-0 = (LESS_THAN a 3.14), expr = leaf-0") {
-      OrcFilters.createFilter(schema, Array(
-        LessThan(
-          "a",
-          new java.math.BigDecimal(3.14, MathContext.DECIMAL64).setScale(2)))
-      ).get.toString
-    }
-  }
-}
-
diff --git a/sql/create-docs.sh b/sql/create-docs.sh
index 6614c714e90c7..8721df874ee73 100755
--- a/sql/create-docs.sh
+++ b/sql/create-docs.sh
@@ -27,14 +27,14 @@ set -e
 FWDIR="$(cd "`dirname "${BASH_SOURCE[0]}"`"; pwd)"
 SPARK_HOME="$(cd "`dirname "${BASH_SOURCE[0]}"`"/..; pwd)"
 
-if ! hash python 2>/dev/null; then
-  echo "Missing python in your path, skipping SQL documentation generation."
+if ! hash python3 2>/dev/null; then
+  echo "Missing python3 in your path, skipping SQL documentation generation."
   exit 0
 fi
 
 if ! hash mkdocs 2>/dev/null; then
   echo "Missing mkdocs in your path, trying to install mkdocs for SQL documentation generation."
-  pip install mkdocs
+  pip3 install mkdocs
 fi
 
 pushd "$FWDIR" > /dev/null
diff --git a/sql/gen-sql-api-docs.py b/sql/gen-sql-api-docs.py
index 61328997c1c58..17631a7352a02 100644
--- a/sql/gen-sql-api-docs.py
+++ b/sql/gen-sql-api-docs.py
@@ -24,6 +24,106 @@
 ExpressionInfo = namedtuple(
     "ExpressionInfo", "className name usage arguments examples note since deprecated")
 
+_virtual_operator_infos = [
+    ExpressionInfo(
+        className="",
+        name="!=",
+        usage="expr1 != expr2 - Returns true if `expr1` is not equal to `expr2`, " +
+              "or false otherwise.",
+        arguments="\n    Arguments:\n      " +
+                  """* expr1, expr2 - the two expressions must be same type or can be casted to
+                       a common type, and must be a type that can be used in equality comparison.
+                       Map type is not supported. For complex types such array/struct,
+                       the data types of fields must be orderable.""",
+        examples="\n    Examples:\n      " +
+                 "> SELECT 1 != 2;\n      " +
+                 " true\n      " +
+                 "> SELECT 1 != '2';\n      " +
+                 " true\n      " +
+                 "> SELECT true != NULL;\n      " +
+                 " NULL\n      " +
+                 "> SELECT NULL != NULL;\n      " +
+                 " NULL",
+        note="",
+        since="1.0.0",
+        deprecated=""),
+    ExpressionInfo(
+        className="",
+        name="<>",
+        usage="expr1 != expr2 - Returns true if `expr1` is not equal to `expr2`, " +
+              "or false otherwise.",
+        arguments="\n    Arguments:\n      " +
+                  """* expr1, expr2 - the two expressions must be same type or can be casted to
+                       a common type, and must be a type that can be used in equality comparison.
+                       Map type is not supported. For complex types such array/struct,
+                       the data types of fields must be orderable.""",
+        examples="\n    Examples:\n      " +
+                 "> SELECT 1 != 2;\n      " +
+                 " true\n      " +
+                 "> SELECT 1 != '2';\n      " +
+                 " true\n      " +
+                 "> SELECT true != NULL;\n      " +
+                 " NULL\n      " +
+                 "> SELECT NULL != NULL;\n      " +
+                 " NULL",
+        note="",
+        since="1.0.0",
+        deprecated=""),
+    ExpressionInfo(
+        className="",
+        name="between",
+        usage="expr1 [NOT] BETWEEN expr2 AND expr3 - " +
+              "evaluate if `expr1` is [not] in between `expr2` and `expr3`.",
+        arguments="",
+        examples="\n    Examples:\n      " +
+                 "> SELECT col1 FROM VALUES 1, 3, 5, 7 WHERE col1 BETWEEN 2 AND 5;\n      " +
+                 " 3\n      " +
+                 " 5",
+        note="",
+        since="1.0.0",
+        deprecated=""),
+    ExpressionInfo(
+        className="",
+        name="case",
+        usage="CASE expr1 WHEN expr2 THEN expr3 " +
+              "[WHEN expr4 THEN expr5]* [ELSE expr6] END - " +
+              "When `expr1` = `expr2`, returns `expr3`; " +
+              "when `expr1` = `expr4`, return `expr5`; else return `expr6`.",
+        arguments="\n    Arguments:\n      " +
+                  "* expr1 - the expression which is one operand of comparison.\n      " +
+                  "* expr2, expr4 - the expressions each of which is the other " +
+                  "  operand of comparison.\n      " +
+                  "* expr3, expr5, expr6 - the branch value expressions and else value expression" +
+                  "  should all be same type or coercible to a common type.",
+        examples="\n    Examples:\n      " +
+                 "> SELECT CASE col1 WHEN 1 THEN 'one' " +
+                 "WHEN 2 THEN 'two' ELSE '?' END FROM VALUES 1, 2, 3;\n      " +
+                 " one\n      " +
+                 " two\n      " +
+                 " ?\n      " +
+                 "> SELECT CASE col1 WHEN 1 THEN 'one' " +
+                 "WHEN 2 THEN 'two' END FROM VALUES 1, 2, 3;\n      " +
+                 " one\n      " +
+                 " two\n      " +
+                 " NULL",
+        note="",
+        since="1.0.1",
+        deprecated=""),
+    ExpressionInfo(
+        className="",
+        name="||",
+        usage="expr1 || expr2 - Returns the concatenation of `expr1` and `expr2`.",
+        arguments="",
+        examples="\n    Examples:\n      " +
+                 "> SELECT 'Spark' || 'SQL';\n      " +
+                 " SparkSQL\n      " +
+                 "> SELECT array(1, 2, 3) || array(4, 5) || array(6);\n      " +
+                 " [1,2,3,4,5,6]",
+        note="\n    || for arrays is available since 2.4.0.\n",
+        since="2.3.0",
+        deprecated="")
+]
+
 
 def _list_function_infos(jvm):
     """
@@ -32,7 +132,7 @@ def _list_function_infos(jvm):
     """
 
     jinfos = jvm.org.apache.spark.sql.api.python.PythonSQLUtils.listBuiltinFunctionInfos()
-    infos = []
+    infos = _virtual_operator_infos
     for jinfo in jinfos:
         name = jinfo.getName()
         usage = jinfo.getUsage()
@@ -195,6 +295,7 @@ def generate_sql_api_markdown(jvm, path):
     """
 
     with open(path, 'w') as mdfile:
+        mdfile.write("# Built-in Functions\n\n")
         for info in _list_function_infos(jvm):
             name = info.name
             usage = _make_pretty_usage(info.usage)
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index 1de2677d5ede5..7d61b92b4f6ef 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
@@ -77,6 +77,10 @@
       <groupId>${hive.group}</groupId>
       <artifactId>hive-beeline</artifactId>
     </dependency>
+    <dependency>
+      <groupId>${hive.group}</groupId>
+      <artifactId>hive-service-rpc</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.eclipse.jetty</groupId>
       <artifactId>jetty-server</artifactId>
@@ -95,7 +99,7 @@
     </dependency>
     <dependency>
       <groupId>org.seleniumhq.selenium</groupId>
-      <artifactId>selenium-htmlunit-driver</artifactId>
+      <artifactId>htmlunit-driver</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>
@@ -129,31 +133,13 @@
       <groupId>net.sf.jpam</groupId>
       <artifactId>jpam</artifactId>
     </dependency>
+    <dependency>
+      <groupId>commons-cli</groupId>
+      <artifactId>commons-cli</artifactId>
+    </dependency>
   </dependencies>
   <build>
     <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
     <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
-    <plugins>
-      <plugin>
-        <groupId>org.codehaus.mojo</groupId>
-        <artifactId>build-helper-maven-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>add-source</id>
-            <phase>generate-sources</phase>
-            <goals>
-              <goal>add-source</goal>
-            </goals>
-            <configuration>
-              <sources>
-                <source>v${hive.version.short}/src/gen/java</source>
-                <source>v${hive.version.short}/src/main/java</source>
-                <source>v${hive.version.short}/src/main/scala</source>
-              </sources>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-    </plugins>
   </build>
 </project>
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/AbstractService.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/AbstractService.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/AbstractService.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/AbstractService.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/CompositeService.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/CompositeService.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/CompositeService.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/CompositeService.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/CookieSigner.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/CookieSigner.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/CookieSigner.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/CookieSigner.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/ServiceOperations.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/ServiceOperations.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/ServiceOperations.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/ServiceOperations.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/ServiceUtils.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/ServiceUtils.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/ServiceUtils.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/ServiceUtils.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/auth/HiveAuthFactory.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/HiveAuthFactory.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/auth/HiveAuthFactory.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/HiveAuthFactory.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/auth/HttpAuthUtils.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/HttpAuthUtils.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/auth/HttpAuthUtils.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/HttpAuthUtils.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/auth/KerberosSaslHelper.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/KerberosSaslHelper.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/auth/KerberosSaslHelper.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/KerberosSaslHelper.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/auth/PlainSaslHelper.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/PlainSaslHelper.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/auth/PlainSaslHelper.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/PlainSaslHelper.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/auth/TSetIpAddressProcessor.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/TSetIpAddressProcessor.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/auth/TSetIpAddressProcessor.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/TSetIpAddressProcessor.java
diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/CLIService.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/CLIService.java
new file mode 100644
index 0000000000000..68f044c6a0f28
--- /dev/null
+++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/CLIService.java
@@ -0,0 +1,583 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.service.cli;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import javax.security.auth.login.LoginException;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.shims.Utils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hive.service.CompositeService;
+import org.apache.hive.service.ServiceException;
+import org.apache.hive.service.auth.HiveAuthFactory;
+import org.apache.hive.service.cli.operation.Operation;
+import org.apache.hive.service.cli.session.HiveSession;
+import org.apache.hive.service.cli.session.SessionManager;
+import org.apache.hive.service.rpc.thrift.TOperationHandle;
+import org.apache.hive.service.rpc.thrift.TProtocolVersion;
+import org.apache.hive.service.server.HiveServer2;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * CLIService.
+ *
+ */
+public class CLIService extends CompositeService implements ICLIService {
+
+  public static final TProtocolVersion SERVER_VERSION;
+
+  static {
+    TProtocolVersion[] protocols = TProtocolVersion.values();
+    SERVER_VERSION = protocols[protocols.length - 1];
+  }
+
+  private final Logger LOG = LoggerFactory.getLogger(CLIService.class.getName());
+
+  private HiveConf hiveConf;
+  private SessionManager sessionManager;
+  private UserGroupInformation serviceUGI;
+  private UserGroupInformation httpUGI;
+  // The HiveServer2 instance running this service
+  private final HiveServer2 hiveServer2;
+
+  public CLIService(HiveServer2 hiveServer2) {
+    super(CLIService.class.getSimpleName());
+    this.hiveServer2 = hiveServer2;
+  }
+
+  @Override
+  public synchronized void init(HiveConf hiveConf) {
+    this.hiveConf = hiveConf;
+    sessionManager = new SessionManager(hiveServer2);
+    addService(sessionManager);
+    //  If the hadoop cluster is secure, do a kerberos login for the service from the keytab
+    if (UserGroupInformation.isSecurityEnabled()) {
+      try {
+        HiveAuthFactory.loginFromKeytab(hiveConf);
+        this.serviceUGI = Utils.getUGI();
+      } catch (IOException e) {
+        throw new ServiceException("Unable to login to kerberos with given principal/keytab", e);
+      } catch (LoginException e) {
+        throw new ServiceException("Unable to login to kerberos with given principal/keytab", e);
+      }
+
+      // Also try creating a UGI object for the SPNego principal
+      String principal = hiveConf.getVar(ConfVars.HIVE_SERVER2_SPNEGO_PRINCIPAL);
+      String keyTabFile = hiveConf.getVar(ConfVars.HIVE_SERVER2_SPNEGO_KEYTAB);
+      if (principal.isEmpty() || keyTabFile.isEmpty()) {
+        LOG.info("SPNego httpUGI not created, spNegoPrincipal: " + principal +
+            ", ketabFile: " + keyTabFile);
+      } else {
+        try {
+          this.httpUGI = HiveAuthFactory.loginFromSpnegoKeytabAndReturnUGI(hiveConf);
+          LOG.info("SPNego httpUGI successfully created.");
+        } catch (IOException e) {
+          LOG.warn("SPNego httpUGI creation failed: ", e);
+        }
+      }
+    }
+    // creates connection to HMS and thus *must* occur after kerberos login above
+    try {
+      applyAuthorizationConfigPolicy(hiveConf);
+    } catch (Exception e) {
+      throw new RuntimeException("Error applying authorization policy on hive configuration: "
+          + e.getMessage(), e);
+    }
+    setupBlockedUdfs();
+    super.init(hiveConf);
+  }
+
+  private void applyAuthorizationConfigPolicy(HiveConf newHiveConf) throws HiveException,
+      MetaException {
+    // authorization setup using SessionState should be revisited eventually, as
+    // authorization and authentication are not session specific settings
+    SessionState ss = new SessionState(newHiveConf);
+    ss.setIsHiveServerQuery(true);
+    SessionState.start(ss);
+    ss.applyAuthorizationPolicy();
+  }
+
+  private void setupBlockedUdfs() {
+    FunctionRegistry.setupPermissionsForBuiltinUDFs(
+        hiveConf.getVar(ConfVars.HIVE_SERVER2_BUILTIN_UDF_WHITELIST),
+        hiveConf.getVar(ConfVars.HIVE_SERVER2_BUILTIN_UDF_BLACKLIST));
+  }
+
+  public UserGroupInformation getServiceUGI() {
+    return this.serviceUGI;
+  }
+
+  public UserGroupInformation getHttpUGI() {
+    return this.httpUGI;
+  }
+
+  @Override
+  public synchronized void start() {
+    super.start();
+    // Initialize and test a connection to the metastore
+    IMetaStoreClient metastoreClient = null;
+    try {
+      metastoreClient = new HiveMetaStoreClient(hiveConf);
+      metastoreClient.getDatabases("default");
+    } catch (Exception e) {
+      throw new ServiceException("Unable to connect to MetaStore!", e);
+    }
+    finally {
+      if (metastoreClient != null) {
+        metastoreClient.close();
+      }
+    }
+  }
+
+  @Override
+  public synchronized void stop() {
+    super.stop();
+  }
+
+  /**
+   * @deprecated  Use {@link #openSession(TProtocolVersion, String, String, String, Map)}
+   */
+  @Deprecated
+  public SessionHandle openSession(TProtocolVersion protocol, String username, String password,
+      Map<String, String> configuration) throws HiveSQLException {
+    SessionHandle sessionHandle = sessionManager.openSession(protocol, username, password, null, configuration, false, null);
+    LOG.debug(sessionHandle + ": openSession()");
+    return sessionHandle;
+  }
+
+  /**
+   * @deprecated  Use {@link #openSessionWithImpersonation(TProtocolVersion, String, String, String, Map, String)}
+   */
+  @Deprecated
+  public SessionHandle openSessionWithImpersonation(TProtocolVersion protocol, String username,
+      String password, Map<String, String> configuration, String delegationToken)
+          throws HiveSQLException {
+    SessionHandle sessionHandle = sessionManager.openSession(protocol, username, password, null, configuration,
+        true, delegationToken);
+    LOG.debug(sessionHandle + ": openSessionWithImpersonation()");
+    return sessionHandle;
+  }
+
+  public SessionHandle openSession(TProtocolVersion protocol, String username, String password, String ipAddress,
+      Map<String, String> configuration) throws HiveSQLException {
+    SessionHandle sessionHandle = sessionManager.openSession(protocol, username, password, ipAddress, configuration, false, null);
+    LOG.debug(sessionHandle + ": openSession()");
+    return sessionHandle;
+  }
+
+  public SessionHandle openSessionWithImpersonation(TProtocolVersion protocol, String username,
+      String password, String ipAddress, Map<String, String> configuration, String delegationToken)
+          throws HiveSQLException {
+    SessionHandle sessionHandle = sessionManager.openSession(protocol, username, password, ipAddress, configuration,
+        true, delegationToken);
+    LOG.debug(sessionHandle + ": openSession()");
+    return sessionHandle;
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#openSession(java.lang.String, java.lang.String, java.util.Map)
+   */
+  @Override
+  public SessionHandle openSession(String username, String password, Map<String, String> configuration)
+      throws HiveSQLException {
+    SessionHandle sessionHandle = sessionManager.openSession(SERVER_VERSION, username, password, null, configuration, false, null);
+    LOG.debug(sessionHandle + ": openSession()");
+    return sessionHandle;
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#openSession(java.lang.String, java.lang.String, java.util.Map)
+   */
+  @Override
+  public SessionHandle openSessionWithImpersonation(String username, String password, Map<String, String> configuration,
+      String delegationToken) throws HiveSQLException {
+    SessionHandle sessionHandle = sessionManager.openSession(SERVER_VERSION, username, password, null, configuration,
+        true, delegationToken);
+    LOG.debug(sessionHandle + ": openSession()");
+    return sessionHandle;
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#closeSession(org.apache.hive.service.cli.SessionHandle)
+   */
+  @Override
+  public void closeSession(SessionHandle sessionHandle)
+      throws HiveSQLException {
+    sessionManager.closeSession(sessionHandle);
+    LOG.debug(sessionHandle + ": closeSession()");
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#getInfo(org.apache.hive.service.cli.SessionHandle, java.util.List)
+   */
+  @Override
+  public GetInfoValue getInfo(SessionHandle sessionHandle, GetInfoType getInfoType)
+      throws HiveSQLException {
+    GetInfoValue infoValue = sessionManager.getSession(sessionHandle)
+        .getInfo(getInfoType);
+    LOG.debug(sessionHandle + ": getInfo()");
+    return infoValue;
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#executeStatement(org.apache.hive.service.cli.SessionHandle,
+   *  java.lang.String, java.util.Map)
+   */
+  @Override
+  public OperationHandle executeStatement(SessionHandle sessionHandle, String statement,
+      Map<String, String> confOverlay) throws HiveSQLException {
+    HiveSession session = sessionManager.getSession(sessionHandle);
+    // need to reset the monitor, as operation handle is not available down stream, Ideally the
+    // monitor should be associated with the operation handle.
+    session.getSessionState().updateProgressMonitor(null);
+    OperationHandle opHandle = session.executeStatement(statement, confOverlay);
+    LOG.debug(sessionHandle + ": executeStatement()");
+    return opHandle;
+  }
+
+  /**
+   * Execute statement on the server with a timeout. This is a blocking call.
+   */
+  @Override
+  public OperationHandle executeStatement(SessionHandle sessionHandle, String statement,
+        Map<String, String> confOverlay, long queryTimeout) throws HiveSQLException {
+    HiveSession session = sessionManager.getSession(sessionHandle);
+    // need to reset the monitor, as operation handle is not available down stream, Ideally the
+    // monitor should be associated with the operation handle.
+    session.getSessionState().updateProgressMonitor(null);
+    OperationHandle opHandle = session.executeStatement(statement, confOverlay, queryTimeout);
+    LOG.debug(sessionHandle + ": executeStatement()");
+    return opHandle;
+  }
+
+  /**
+   * Execute statement asynchronously on the server. This is a non-blocking call
+   */
+  @Override
+  public OperationHandle executeStatementAsync(SessionHandle sessionHandle, String statement,
+      Map<String, String> confOverlay) throws HiveSQLException {
+    HiveSession session = sessionManager.getSession(sessionHandle);
+    // need to reset the monitor, as operation handle is not available down stream, Ideally the
+    // monitor should be associated with the operation handle.
+    session.getSessionState().updateProgressMonitor(null);
+    OperationHandle opHandle = session.executeStatementAsync(statement, confOverlay);
+    LOG.debug(sessionHandle + ": executeStatementAsync()");
+    return opHandle;
+  }
+
+  /**
+   * Execute statement asynchronously on the server with a timeout. This is a non-blocking call
+   */
+  @Override
+  public OperationHandle executeStatementAsync(SessionHandle sessionHandle, String statement,
+      Map<String, String> confOverlay, long queryTimeout) throws HiveSQLException {
+    HiveSession session = sessionManager.getSession(sessionHandle);
+    // need to reset the monitor, as operation handle is not available down stream, Ideally the
+    // monitor should be associated with the operation handle.
+    session.getSessionState().updateProgressMonitor(null);
+    OperationHandle opHandle = session.executeStatementAsync(statement, confOverlay, queryTimeout);
+    LOG.debug(sessionHandle + ": executeStatementAsync()");
+    return opHandle;
+  }
+
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#getTypeInfo(org.apache.hive.service.cli.SessionHandle)
+   */
+  @Override
+  public OperationHandle getTypeInfo(SessionHandle sessionHandle)
+      throws HiveSQLException {
+    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
+        .getTypeInfo();
+    LOG.debug(sessionHandle + ": getTypeInfo()");
+    return opHandle;
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#getCatalogs(org.apache.hive.service.cli.SessionHandle)
+   */
+  @Override
+  public OperationHandle getCatalogs(SessionHandle sessionHandle)
+      throws HiveSQLException {
+    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
+        .getCatalogs();
+    LOG.debug(sessionHandle + ": getCatalogs()");
+    return opHandle;
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#getSchemas(org.apache.hive.service.cli.SessionHandle, java.lang.String, java.lang.String)
+   */
+  @Override
+  public OperationHandle getSchemas(SessionHandle sessionHandle,
+      String catalogName, String schemaName)
+          throws HiveSQLException {
+    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
+        .getSchemas(catalogName, schemaName);
+    LOG.debug(sessionHandle + ": getSchemas()");
+    return opHandle;
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#getTables(org.apache.hive.service.cli.SessionHandle, java.lang.String, java.lang.String, java.lang.String, java.util.List)
+   */
+  @Override
+  public OperationHandle getTables(SessionHandle sessionHandle,
+      String catalogName, String schemaName, String tableName, List<String> tableTypes)
+          throws HiveSQLException {
+    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
+        .getTables(catalogName, schemaName, tableName, tableTypes);
+    LOG.debug(sessionHandle + ": getTables()");
+    return opHandle;
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#getTableTypes(org.apache.hive.service.cli.SessionHandle)
+   */
+  @Override
+  public OperationHandle getTableTypes(SessionHandle sessionHandle)
+      throws HiveSQLException {
+    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
+        .getTableTypes();
+    LOG.debug(sessionHandle + ": getTableTypes()");
+    return opHandle;
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#getColumns(org.apache.hive.service.cli.SessionHandle)
+   */
+  @Override
+  public OperationHandle getColumns(SessionHandle sessionHandle,
+      String catalogName, String schemaName, String tableName, String columnName)
+          throws HiveSQLException {
+    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
+        .getColumns(catalogName, schemaName, tableName, columnName);
+    LOG.debug(sessionHandle + ": getColumns()");
+    return opHandle;
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#getFunctions(org.apache.hive.service.cli.SessionHandle)
+   */
+  @Override
+  public OperationHandle getFunctions(SessionHandle sessionHandle,
+      String catalogName, String schemaName, String functionName)
+          throws HiveSQLException {
+    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
+        .getFunctions(catalogName, schemaName, functionName);
+    LOG.debug(sessionHandle + ": getFunctions()");
+    return opHandle;
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#getPrimaryKeys(org.apache.hive.service.cli.SessionHandle)
+   */
+  @Override
+  public OperationHandle getPrimaryKeys(SessionHandle sessionHandle,
+      String catalog, String schema, String table) throws HiveSQLException {
+    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
+        .getPrimaryKeys(catalog, schema, table);
+    LOG.debug(sessionHandle + ": getPrimaryKeys()");
+    return opHandle;
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#getCrossReference(org.apache.hive.service.cli.SessionHandle)
+   */
+  @Override
+  public OperationHandle getCrossReference(SessionHandle sessionHandle,
+      String primaryCatalog, String primarySchema, String primaryTable, String foreignCatalog,
+      String foreignSchema, String foreignTable) throws HiveSQLException {
+    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
+        .getCrossReference(primaryCatalog, primarySchema, primaryTable,
+         foreignCatalog,
+         foreignSchema, foreignTable);
+    LOG.debug(sessionHandle + ": getCrossReference()");
+    return opHandle;
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#getOperationStatus(org.apache.hive.service.cli.OperationHandle)
+   */
+  @Override
+  public OperationStatus getOperationStatus(OperationHandle opHandle)
+      throws HiveSQLException {
+    Operation operation = sessionManager.getOperationManager().getOperation(opHandle);
+    /**
+     * If this is a background operation run asynchronously,
+     * we block for a configured duration, before we return
+     * (duration: HIVE_SERVER2_LONG_POLLING_TIMEOUT).
+     * However, if the background operation is complete, we return immediately.
+     */
+    if (operation.shouldRunAsync()) {
+      HiveConf conf = operation.getParentSession().getHiveConf();
+      long timeout = HiveConf.getTimeVar(conf,
+          HiveConf.ConfVars.HIVE_SERVER2_LONG_POLLING_TIMEOUT, TimeUnit.MILLISECONDS);
+      try {
+        operation.getBackgroundHandle().get(timeout, TimeUnit.MILLISECONDS);
+      } catch (TimeoutException e) {
+        // No Op, return to the caller since long polling timeout has expired
+        LOG.trace(opHandle + ": Long polling timed out");
+      } catch (CancellationException e) {
+        // The background operation thread was cancelled
+        LOG.trace(opHandle + ": The background operation was cancelled", e);
+      } catch (ExecutionException e) {
+        // The background operation thread was aborted
+        LOG.warn(opHandle + ": The background operation was aborted", e);
+      } catch (InterruptedException e) {
+        // No op, this thread was interrupted
+        // In this case, the call might return sooner than long polling timeout
+      }
+    }
+    OperationStatus opStatus = operation.getStatus();
+    LOG.debug(opHandle + ": getOperationStatus()");
+    return opStatus;
+  }
+
+  public HiveConf getSessionConf(SessionHandle sessionHandle) throws HiveSQLException {
+    return sessionManager.getSession(sessionHandle).getHiveConf();
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#cancelOperation(org.apache.hive.service.cli.OperationHandle)
+   */
+  @Override
+  public void cancelOperation(OperationHandle opHandle)
+      throws HiveSQLException {
+    sessionManager.getOperationManager().getOperation(opHandle)
+    .getParentSession().cancelOperation(opHandle);
+    LOG.debug(opHandle + ": cancelOperation()");
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#closeOperation(org.apache.hive.service.cli.OperationHandle)
+   */
+  @Override
+  public void closeOperation(OperationHandle opHandle)
+      throws HiveSQLException {
+    sessionManager.getOperationManager().getOperation(opHandle)
+    .getParentSession().closeOperation(opHandle);
+    LOG.debug(opHandle + ": closeOperation");
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#getResultSetMetadata(org.apache.hive.service.cli.OperationHandle)
+   */
+  @Override
+  public TableSchema getResultSetMetadata(OperationHandle opHandle)
+      throws HiveSQLException {
+    TableSchema tableSchema = sessionManager.getOperationManager()
+        .getOperation(opHandle).getParentSession().getResultSetMetadata(opHandle);
+    LOG.debug(opHandle + ": getResultSetMetadata()");
+    return tableSchema;
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#fetchResults(org.apache.hive.service.cli.OperationHandle)
+   */
+  @Override
+  public RowSet fetchResults(OperationHandle opHandle)
+      throws HiveSQLException {
+    return fetchResults(opHandle, Operation.DEFAULT_FETCH_ORIENTATION,
+        Operation.DEFAULT_FETCH_MAX_ROWS, FetchType.QUERY_OUTPUT);
+  }
+
+  @Override
+  public RowSet fetchResults(OperationHandle opHandle, FetchOrientation orientation,
+                             long maxRows, FetchType fetchType) throws HiveSQLException {
+    RowSet rowSet = sessionManager.getOperationManager().getOperation(opHandle)
+        .getParentSession().fetchResults(opHandle, orientation, maxRows, fetchType);
+    LOG.debug(opHandle + ": fetchResults()");
+    return rowSet;
+  }
+
+  // obtain delegation token for the give user from metastore
+  public synchronized String getDelegationTokenFromMetaStore(String owner)
+      throws HiveSQLException, UnsupportedOperationException, LoginException, IOException {
+    if (!hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL) ||
+        !hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS)) {
+      throw new UnsupportedOperationException(
+          "delegation token is can only be obtained for a secure remote metastore");
+    }
+
+    try {
+      Hive.closeCurrent();
+      return Hive.get(hiveConf).getDelegationToken(owner, owner);
+    } catch (HiveException e) {
+      if (e.getCause() instanceof UnsupportedOperationException) {
+        throw (UnsupportedOperationException)e.getCause();
+      } else {
+        throw new HiveSQLException("Error connect metastore to setup impersonation", e);
+      }
+    }
+  }
+
+  @Override
+  public String getDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
+      String owner, String renewer) throws HiveSQLException {
+    String delegationToken = sessionManager.getSession(sessionHandle)
+        .getDelegationToken(authFactory, owner, renewer);
+    LOG.info(sessionHandle  + ": getDelegationToken()");
+    return delegationToken;
+  }
+
+  @Override
+  public void cancelDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
+      String tokenStr) throws HiveSQLException {
+    sessionManager.getSession(sessionHandle).cancelDelegationToken(authFactory, tokenStr);
+    LOG.info(sessionHandle  + ": cancelDelegationToken()");
+  }
+
+  @Override
+  public void renewDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
+      String tokenStr) throws HiveSQLException {
+    sessionManager.getSession(sessionHandle).renewDelegationToken(authFactory, tokenStr);
+    LOG.info(sessionHandle  + ": renewDelegationToken()");
+  }
+
+  @Override
+  public String getQueryId(TOperationHandle opHandle) throws HiveSQLException {
+    Operation operation = sessionManager.getOperationManager().getOperation(
+        new OperationHandle(opHandle));
+    final String queryId = operation.getParentSession().getHiveConf().getVar(ConfVars.HIVEQUERYID);
+    LOG.debug(opHandle + ": getQueryId() " + queryId);
+    return queryId;
+  }
+
+  public SessionManager getSessionManager() {
+    return sessionManager;
+  }
+}
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/ColumnBasedSet.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/ColumnBasedSet.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/ColumnBasedSet.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/ColumnBasedSet.java
diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/ColumnDescriptor.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/ColumnDescriptor.java
new file mode 100644
index 0000000000000..b2ef1c7722ef8
--- /dev/null
+++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/ColumnDescriptor.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.service.cli;
+
+import org.apache.hadoop.hive.serde2.thrift.Type;
+import org.apache.hive.service.rpc.thrift.TColumnDesc;
+
+
+/**
+ * ColumnDescriptor.
+ *
+ */
+public class ColumnDescriptor {
+  private final String name;
+  private final String comment;
+  private final TypeDescriptor type;
+  // ordinal position of this column in the schema
+  private final int position;
+
+  public ColumnDescriptor(String name, String comment, TypeDescriptor type, int position) {
+    this.name = name;
+    this.comment = comment;
+    this.type = type;
+    this.position = position;
+  }
+
+  public ColumnDescriptor(TColumnDesc tColumnDesc) {
+    name = tColumnDesc.getColumnName();
+    comment = tColumnDesc.getComment();
+    type = new TypeDescriptor(tColumnDesc.getTypeDesc());
+    position = tColumnDesc.getPosition();
+  }
+
+  public static ColumnDescriptor newPrimitiveColumnDescriptor(String name, String comment, Type type, int position) {
+    // Current usage looks like it's only for metadata columns, but if that changes then
+    // this method may need to require a type qualifiers arguments.
+    return new ColumnDescriptor(name, comment, new TypeDescriptor(type), position);
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public String getComment() {
+    return comment;
+  }
+
+  public TypeDescriptor getTypeDescriptor() {
+    return type;
+  }
+
+  public int getOrdinalPosition() {
+    return position;
+  }
+
+  public TColumnDesc toTColumnDesc() {
+    TColumnDesc tColumnDesc = new TColumnDesc();
+    tColumnDesc.setColumnName(name);
+    tColumnDesc.setComment(comment);
+    tColumnDesc.setTypeDesc(type.toTTypeDesc());
+    tColumnDesc.setPosition(position);
+    return tColumnDesc;
+  }
+
+  public Type getType() {
+    return type.getType();
+  }
+
+  public boolean isPrimitive() {
+    return type.getType().isPrimitiveType();
+  }
+
+  public String getTypeName() {
+    return type.getTypeName();
+  }
+}
diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/ColumnValue.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/ColumnValue.java
new file mode 100644
index 0000000000000..44d9e8a296452
--- /dev/null
+++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/ColumnValue.java
@@ -0,0 +1,289 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.service.cli;
+
+import java.math.BigDecimal;
+import java.sql.Date;
+import java.sql.Timestamp;
+
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.serde2.thrift.Type;
+import org.apache.hive.service.rpc.thrift.TBoolValue;
+import org.apache.hive.service.rpc.thrift.TByteValue;
+import org.apache.hive.service.rpc.thrift.TColumnValue;
+import org.apache.hive.service.rpc.thrift.TDoubleValue;
+import org.apache.hive.service.rpc.thrift.TI16Value;
+import org.apache.hive.service.rpc.thrift.TI32Value;
+import org.apache.hive.service.rpc.thrift.TI64Value;
+import org.apache.hive.service.rpc.thrift.TStringValue;
+
+import org.apache.spark.unsafe.types.UTF8String;
+
+/**
+ * Protocols before HIVE_CLI_SERVICE_PROTOCOL_V6 (used by RowBasedSet)
+ *
+ */
+public class ColumnValue {
+
+  private static TColumnValue booleanValue(Boolean value) {
+    TBoolValue tBoolValue = new TBoolValue();
+    if (value != null) {
+      tBoolValue.setValue(value);
+    }
+    return TColumnValue.boolVal(tBoolValue);
+  }
+
+  private static TColumnValue byteValue(Byte value) {
+    TByteValue tByteValue = new TByteValue();
+    if (value != null) {
+      tByteValue.setValue(value);
+    }
+    return TColumnValue.byteVal(tByteValue);
+  }
+
+  private static TColumnValue shortValue(Short value) {
+    TI16Value tI16Value = new TI16Value();
+    if (value != null) {
+      tI16Value.setValue(value);
+    }
+    return TColumnValue.i16Val(tI16Value);
+  }
+
+  private static TColumnValue intValue(Integer value) {
+    TI32Value tI32Value = new TI32Value();
+    if (value != null) {
+      tI32Value.setValue(value);
+    }
+    return TColumnValue.i32Val(tI32Value);
+  }
+
+  private static TColumnValue longValue(Long value) {
+    TI64Value tI64Value = new TI64Value();
+    if (value != null) {
+      tI64Value.setValue(value);
+    }
+    return TColumnValue.i64Val(tI64Value);
+  }
+
+  private static TColumnValue floatValue(Float value) {
+    TDoubleValue tDoubleValue = new TDoubleValue();
+    if (value != null) {
+      tDoubleValue.setValue(value);
+    }
+    return TColumnValue.doubleVal(tDoubleValue);
+  }
+
+  private static TColumnValue doubleValue(Double value) {
+    TDoubleValue tDoubleValue = new TDoubleValue();
+    if (value != null) {
+      tDoubleValue.setValue(value);
+    }
+    return TColumnValue.doubleVal(tDoubleValue);
+  }
+
+  private static TColumnValue stringValue(String value) {
+    TStringValue tStringValue = new TStringValue();
+    if (value != null) {
+      tStringValue.setValue(value);
+    }
+    return TColumnValue.stringVal(tStringValue);
+  }
+
+  private static TColumnValue stringValue(HiveChar value) {
+    TStringValue tStringValue = new TStringValue();
+    if (value != null) {
+      tStringValue.setValue(value.toString());
+    }
+    return TColumnValue.stringVal(tStringValue);
+  }
+
+  private static TColumnValue stringValue(HiveVarchar value) {
+    TStringValue tStringValue = new TStringValue();
+    if (value != null) {
+      tStringValue.setValue(value.toString());
+    }
+    return TColumnValue.stringVal(tStringValue);
+  }
+
+  private static TColumnValue stringValue(HiveIntervalYearMonth value) {
+    TStringValue tStrValue = new TStringValue();
+    if (value != null) {
+      tStrValue.setValue(value.toString());
+    }
+    return TColumnValue.stringVal(tStrValue);
+  }
+
+  private static TColumnValue stringValue(HiveIntervalDayTime value) {
+    TStringValue tStrValue = new TStringValue();
+    if (value != null) {
+      tStrValue.setValue(value.toString());
+    }
+    return TColumnValue.stringVal(tStrValue);
+  }
+
+  public static TColumnValue toTColumnValue(TypeDescriptor typeDescriptor, Object value) {
+    Type type = typeDescriptor.getType();
+
+    switch (type) {
+    case BOOLEAN_TYPE:
+      return booleanValue((Boolean)value);
+    case TINYINT_TYPE:
+      return byteValue((Byte)value);
+    case SMALLINT_TYPE:
+      return shortValue((Short)value);
+    case INT_TYPE:
+      return intValue((Integer)value);
+    case BIGINT_TYPE:
+      return longValue((Long)value);
+    case FLOAT_TYPE:
+      return floatValue((Float)value);
+    case DOUBLE_TYPE:
+      return doubleValue((Double)value);
+    case STRING_TYPE:
+      return stringValue((String)value);
+    case CHAR_TYPE:
+      return stringValue((HiveChar)value);
+    case VARCHAR_TYPE:
+      return stringValue((HiveVarchar)value);
+    case DATE_TYPE:
+    case TIMESTAMP_TYPE:
+      // SPARK-31859, SPARK-31861: converted to string already in SparkExecuteStatementOperation
+      return stringValue((String)value);
+    case DECIMAL_TYPE:
+      String plainStr = value == null ? null : ((BigDecimal)value).toPlainString();
+      return stringValue(plainStr);
+    case BINARY_TYPE:
+      String strVal = value == null ? null : UTF8String.fromBytes((byte[])value).toString();
+      return stringValue(strVal);
+    case ARRAY_TYPE:
+    case MAP_TYPE:
+    case STRUCT_TYPE:
+    case UNION_TYPE:
+    case USER_DEFINED_TYPE:
+    case INTERVAL_YEAR_MONTH_TYPE:
+    case INTERVAL_DAY_TIME_TYPE:
+      return stringValue((String)value);
+    case NULL_TYPE:
+      return stringValue((String)value);
+    default:
+      return null;
+    }
+  }
+
+  private static Boolean getBooleanValue(TBoolValue tBoolValue) {
+    if (tBoolValue.isSetValue()) {
+      return tBoolValue.isValue();
+    }
+    return null;
+  }
+
+  private static Byte getByteValue(TByteValue tByteValue) {
+    if (tByteValue.isSetValue()) {
+      return tByteValue.getValue();
+    }
+    return null;
+  }
+
+  private static Short getShortValue(TI16Value tI16Value) {
+    if (tI16Value.isSetValue()) {
+      return tI16Value.getValue();
+    }
+    return null;
+  }
+
+  private static Integer getIntegerValue(TI32Value tI32Value) {
+    if (tI32Value.isSetValue()) {
+      return tI32Value.getValue();
+    }
+    return null;
+  }
+
+  private static Long getLongValue(TI64Value tI64Value) {
+    if (tI64Value.isSetValue()) {
+      return tI64Value.getValue();
+    }
+    return null;
+  }
+
+  private static Double getDoubleValue(TDoubleValue tDoubleValue) {
+    if (tDoubleValue.isSetValue()) {
+      return tDoubleValue.getValue();
+    }
+    return null;
+  }
+
+  private static String getStringValue(TStringValue tStringValue) {
+    if (tStringValue.isSetValue()) {
+      return tStringValue.getValue();
+    }
+    return null;
+  }
+
+  private static Date getDateValue(TStringValue tStringValue) {
+    if (tStringValue.isSetValue()) {
+      return Date.valueOf(tStringValue.getValue());
+    }
+    return null;
+  }
+
+  private static Timestamp getTimestampValue(TStringValue tStringValue) {
+    if (tStringValue.isSetValue()) {
+      return Timestamp.valueOf(tStringValue.getValue());
+    }
+    return null;
+  }
+
+  private static byte[] getBinaryValue(TStringValue tString) {
+    if (tString.isSetValue()) {
+      return tString.getValue().getBytes();
+    }
+    return null;
+  }
+
+  private static BigDecimal getBigDecimalValue(TStringValue tStringValue) {
+    if (tStringValue.isSetValue()) {
+      return new BigDecimal(tStringValue.getValue());
+    }
+    return null;
+  }
+
+  public static Object toColumnValue(TColumnValue value) {
+    TColumnValue._Fields field = value.getSetField();
+    switch (field) {
+      case BOOL_VAL:
+        return getBooleanValue(value.getBoolVal());
+      case BYTE_VAL:
+        return getByteValue(value.getByteVal());
+      case I16_VAL:
+        return getShortValue(value.getI16Val());
+      case I32_VAL:
+        return getIntegerValue(value.getI32Val());
+      case I64_VAL:
+        return getLongValue(value.getI64Val());
+      case DOUBLE_VAL:
+        return getDoubleValue(value.getDoubleVal());
+      case STRING_VAL:
+        return getStringValue(value.getStringVal());
+    }
+    throw new IllegalArgumentException("never");
+  }
+}
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/FetchOrientation.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/FetchOrientation.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/FetchOrientation.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/FetchOrientation.java
diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/GetInfoType.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/GetInfoType.java
new file mode 100644
index 0000000000000..575dff8f8f47b
--- /dev/null
+++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/GetInfoType.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.service.cli;
+
+import org.apache.hive.service.rpc.thrift.TGetInfoType;
+
+/**
+ * GetInfoType.
+ *
+ */
+public enum GetInfoType {
+  CLI_MAX_DRIVER_CONNECTIONS(TGetInfoType.CLI_MAX_DRIVER_CONNECTIONS),
+  CLI_MAX_CONCURRENT_ACTIVITIES(TGetInfoType.CLI_MAX_CONCURRENT_ACTIVITIES),
+  CLI_DATA_SOURCE_NAME(TGetInfoType.CLI_DATA_SOURCE_NAME),
+  CLI_FETCH_DIRECTION(TGetInfoType.CLI_FETCH_DIRECTION),
+  CLI_SERVER_NAME(TGetInfoType.CLI_SERVER_NAME),
+  CLI_SEARCH_PATTERN_ESCAPE(TGetInfoType.CLI_SEARCH_PATTERN_ESCAPE),
+  CLI_DBMS_NAME(TGetInfoType.CLI_DBMS_NAME),
+  CLI_DBMS_VER(TGetInfoType.CLI_DBMS_VER),
+  CLI_ACCESSIBLE_TABLES(TGetInfoType.CLI_ACCESSIBLE_TABLES),
+  CLI_ACCESSIBLE_PROCEDURES(TGetInfoType.CLI_ACCESSIBLE_PROCEDURES),
+  CLI_CURSOR_COMMIT_BEHAVIOR(TGetInfoType.CLI_CURSOR_COMMIT_BEHAVIOR),
+  CLI_DATA_SOURCE_READ_ONLY(TGetInfoType.CLI_DATA_SOURCE_READ_ONLY),
+  CLI_DEFAULT_TXN_ISOLATION(TGetInfoType.CLI_DEFAULT_TXN_ISOLATION),
+  CLI_IDENTIFIER_CASE(TGetInfoType.CLI_IDENTIFIER_CASE),
+  CLI_IDENTIFIER_QUOTE_CHAR(TGetInfoType.CLI_IDENTIFIER_QUOTE_CHAR),
+  CLI_MAX_COLUMN_NAME_LEN(TGetInfoType.CLI_MAX_COLUMN_NAME_LEN),
+  CLI_MAX_CURSOR_NAME_LEN(TGetInfoType.CLI_MAX_CURSOR_NAME_LEN),
+  CLI_MAX_SCHEMA_NAME_LEN(TGetInfoType.CLI_MAX_SCHEMA_NAME_LEN),
+  CLI_MAX_CATALOG_NAME_LEN(TGetInfoType.CLI_MAX_CATALOG_NAME_LEN),
+  CLI_MAX_TABLE_NAME_LEN(TGetInfoType.CLI_MAX_TABLE_NAME_LEN),
+  CLI_SCROLL_CONCURRENCY(TGetInfoType.CLI_SCROLL_CONCURRENCY),
+  CLI_TXN_CAPABLE(TGetInfoType.CLI_TXN_CAPABLE),
+  CLI_USER_NAME(TGetInfoType.CLI_USER_NAME),
+  CLI_TXN_ISOLATION_OPTION(TGetInfoType.CLI_TXN_ISOLATION_OPTION),
+  CLI_INTEGRITY(TGetInfoType.CLI_INTEGRITY),
+  CLI_GETDATA_EXTENSIONS(TGetInfoType.CLI_GETDATA_EXTENSIONS),
+  CLI_NULL_COLLATION(TGetInfoType.CLI_NULL_COLLATION),
+  CLI_ALTER_TABLE(TGetInfoType.CLI_ALTER_TABLE),
+  CLI_ORDER_BY_COLUMNS_IN_SELECT(TGetInfoType.CLI_ORDER_BY_COLUMNS_IN_SELECT),
+  CLI_SPECIAL_CHARACTERS(TGetInfoType.CLI_SPECIAL_CHARACTERS),
+  CLI_MAX_COLUMNS_IN_GROUP_BY(TGetInfoType.CLI_MAX_COLUMNS_IN_GROUP_BY),
+  CLI_MAX_COLUMNS_IN_INDEX(TGetInfoType.CLI_MAX_COLUMNS_IN_INDEX),
+  CLI_MAX_COLUMNS_IN_ORDER_BY(TGetInfoType.CLI_MAX_COLUMNS_IN_ORDER_BY),
+  CLI_MAX_COLUMNS_IN_SELECT(TGetInfoType.CLI_MAX_COLUMNS_IN_SELECT),
+  CLI_MAX_COLUMNS_IN_TABLE(TGetInfoType.CLI_MAX_COLUMNS_IN_TABLE),
+  CLI_MAX_INDEX_SIZE(TGetInfoType.CLI_MAX_INDEX_SIZE),
+  CLI_MAX_ROW_SIZE(TGetInfoType.CLI_MAX_ROW_SIZE),
+  CLI_MAX_STATEMENT_LEN(TGetInfoType.CLI_MAX_STATEMENT_LEN),
+  CLI_MAX_TABLES_IN_SELECT(TGetInfoType.CLI_MAX_TABLES_IN_SELECT),
+  CLI_MAX_USER_NAME_LEN(TGetInfoType.CLI_MAX_USER_NAME_LEN),
+  CLI_OJ_CAPABILITIES(TGetInfoType.CLI_OJ_CAPABILITIES),
+
+  CLI_XOPEN_CLI_YEAR(TGetInfoType.CLI_XOPEN_CLI_YEAR),
+  CLI_CURSOR_SENSITIVITY(TGetInfoType.CLI_CURSOR_SENSITIVITY),
+  CLI_DESCRIBE_PARAMETER(TGetInfoType.CLI_DESCRIBE_PARAMETER),
+  CLI_CATALOG_NAME(TGetInfoType.CLI_CATALOG_NAME),
+  CLI_COLLATION_SEQ(TGetInfoType.CLI_COLLATION_SEQ),
+  CLI_MAX_IDENTIFIER_LEN(TGetInfoType.CLI_MAX_IDENTIFIER_LEN),
+  CLI_ODBC_KEYWORDS(TGetInfoType.CLI_ODBC_KEYWORDS);
+
+  private final TGetInfoType tInfoType;
+
+  GetInfoType(TGetInfoType tInfoType) {
+    this.tInfoType = tInfoType;
+  }
+
+  public static GetInfoType getGetInfoType(TGetInfoType tGetInfoType) {
+    for (GetInfoType infoType : values()) {
+      if (tGetInfoType.equals(infoType.tInfoType)) {
+        return infoType;
+      }
+    }
+    throw new IllegalArgumentException("Unrecognized Thrift TGetInfoType value: " + tGetInfoType);
+  }
+
+  public TGetInfoType toTGetInfoType() {
+    return tInfoType;
+  }
+
+}
diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/GetInfoValue.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/GetInfoValue.java
new file mode 100644
index 0000000000000..bf3c6b27ea81d
--- /dev/null
+++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/GetInfoValue.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.service.cli;
+
+import org.apache.hive.service.rpc.thrift.TGetInfoValue;
+
+/**
+ * GetInfoValue.
+ *
+ */
+public class GetInfoValue {
+  private String stringValue = null;
+  private short shortValue;
+  private int intValue;
+  private long longValue;
+
+  public GetInfoValue(String stringValue) {
+    this.stringValue = stringValue;
+  }
+
+  public GetInfoValue(short shortValue) {
+    this.shortValue = shortValue;
+  }
+
+  public GetInfoValue(int intValue) {
+    this.intValue = intValue;
+  }
+
+  public GetInfoValue(long longValue) {
+    this.longValue = longValue;
+  }
+
+  public GetInfoValue(TGetInfoValue tGetInfoValue) {
+    switch (tGetInfoValue.getSetField()) {
+    case STRING_VALUE:
+      stringValue = tGetInfoValue.getStringValue();
+      break;
+    default:
+      throw new IllegalArgumentException("Unrecognized TGetInfoValue");
+    }
+  }
+
+  public TGetInfoValue toTGetInfoValue() {
+    TGetInfoValue tInfoValue = new TGetInfoValue();
+    if (stringValue != null) {
+      tInfoValue.setStringValue(stringValue);
+    }
+    return tInfoValue;
+  }
+
+  public String getStringValue() {
+    return stringValue;
+  }
+
+  public short getShortValue() {
+    return shortValue;
+  }
+
+  public int getIntValue() {
+    return intValue;
+  }
+
+  public long getLongValue() {
+    return longValue;
+  }
+}
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/Handle.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/Handle.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/Handle.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/Handle.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/HandleIdentifier.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/HandleIdentifier.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/HandleIdentifier.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/HandleIdentifier.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/HiveSQLException.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/HiveSQLException.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/HiveSQLException.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/HiveSQLException.java
diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/ICLIService.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/ICLIService.java
new file mode 100644
index 0000000000000..a87c6691ebac7
--- /dev/null
+++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/ICLIService.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive.service.cli;
+
+import java.util.List;
+import java.util.Map;
+
+
+
+
+import org.apache.hive.service.auth.HiveAuthFactory;
+import org.apache.hive.service.rpc.thrift.TOperationHandle;
+
+public interface ICLIService {
+
+  SessionHandle openSession(String username, String password,
+      Map<String, String> configuration)
+          throws HiveSQLException;
+
+  SessionHandle openSessionWithImpersonation(String username, String password,
+      Map<String, String> configuration, String delegationToken)
+          throws HiveSQLException;
+
+  void closeSession(SessionHandle sessionHandle)
+      throws HiveSQLException;
+
+  GetInfoValue getInfo(SessionHandle sessionHandle, GetInfoType infoType)
+      throws HiveSQLException;
+
+  OperationHandle executeStatement(SessionHandle sessionHandle, String statement,
+      Map<String, String> confOverlay) throws HiveSQLException;
+
+  OperationHandle executeStatement(SessionHandle sessionHandle, String statement,
+      Map<String, String> confOverlay, long queryTimeout) throws HiveSQLException;
+
+  OperationHandle executeStatementAsync(SessionHandle sessionHandle, String statement,
+      Map<String, String> confOverlay) throws HiveSQLException;
+  OperationHandle executeStatementAsync(SessionHandle sessionHandle, String statement,
+      Map<String, String> confOverlay, long queryTimeout) throws HiveSQLException;
+
+  OperationHandle getTypeInfo(SessionHandle sessionHandle)
+      throws HiveSQLException;
+
+  OperationHandle getCatalogs(SessionHandle sessionHandle)
+      throws HiveSQLException;
+
+  OperationHandle getSchemas(SessionHandle sessionHandle,
+      String catalogName, String schemaName)
+          throws HiveSQLException;
+
+  OperationHandle getTables(SessionHandle sessionHandle,
+      String catalogName, String schemaName, String tableName, List<String> tableTypes)
+          throws HiveSQLException;
+
+  OperationHandle getTableTypes(SessionHandle sessionHandle)
+      throws HiveSQLException;
+
+  OperationHandle getColumns(SessionHandle sessionHandle,
+      String catalogName, String schemaName, String tableName, String columnName)
+          throws HiveSQLException;
+
+  OperationHandle getFunctions(SessionHandle sessionHandle,
+      String catalogName, String schemaName, String functionName)
+          throws HiveSQLException;
+
+  OperationStatus getOperationStatus(OperationHandle opHandle)
+      throws HiveSQLException;
+
+  void cancelOperation(OperationHandle opHandle)
+      throws HiveSQLException;
+
+  void closeOperation(OperationHandle opHandle)
+      throws HiveSQLException;
+
+  TableSchema getResultSetMetadata(OperationHandle opHandle)
+      throws HiveSQLException;
+
+  RowSet fetchResults(OperationHandle opHandle)
+      throws HiveSQLException;
+
+  RowSet fetchResults(OperationHandle opHandle, FetchOrientation orientation,
+      long maxRows, FetchType fetchType) throws HiveSQLException;
+
+  String getDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
+      String owner, String renewer) throws HiveSQLException;
+
+  String getQueryId(TOperationHandle operationHandle) throws HiveSQLException;
+
+  void cancelDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
+      String tokenStr) throws HiveSQLException;
+
+  void renewDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
+      String tokenStr) throws HiveSQLException;
+
+  OperationHandle getPrimaryKeys(SessionHandle sessionHandle, String catalog,
+      String schema, String table) throws HiveSQLException;
+
+  OperationHandle getCrossReference(SessionHandle sessionHandle,
+      String primaryCatalog, String primarySchema, String primaryTable,
+      String foreignCatalog, String foreignSchema, String foreignTable) throws HiveSQLException;
+}
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/OperationHandle.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/OperationHandle.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/OperationHandle.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/OperationHandle.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/OperationState.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/OperationState.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/OperationState.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/OperationState.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/OperationType.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/OperationType.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/OperationType.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/OperationType.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/RowBasedSet.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/RowBasedSet.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/RowBasedSet.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/RowBasedSet.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/RowSet.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/RowSet.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/RowSet.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/RowSet.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/RowSetFactory.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/RowSetFactory.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/RowSetFactory.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/RowSetFactory.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/SessionHandle.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/SessionHandle.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/SessionHandle.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/SessionHandle.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/TableSchema.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/TableSchema.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/TableSchema.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/TableSchema.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/TypeDescriptor.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/TypeDescriptor.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/TypeDescriptor.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/TypeDescriptor.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/TypeQualifiers.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/TypeQualifiers.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/TypeQualifiers.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/TypeQualifiers.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/ClassicTableTypeMapping.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/ClassicTableTypeMapping.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/ClassicTableTypeMapping.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/ClassicTableTypeMapping.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/ExecuteStatementOperation.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/ExecuteStatementOperation.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/ExecuteStatementOperation.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/ExecuteStatementOperation.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/GetCatalogsOperation.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/GetCatalogsOperation.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/GetCatalogsOperation.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/GetCatalogsOperation.java
diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/GetColumnsOperation.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/GetColumnsOperation.java
new file mode 100644
index 0000000000000..59630672847e4
--- /dev/null
+++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/GetColumnsOperation.java
@@ -0,0 +1,251 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.service.cli.operation;
+
+import java.sql.DatabaseMetaData;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.PrimaryKeysRequest;
+import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.metadata.TableIterable;
+import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
+import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject;
+import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivilegeObjectType;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.serde2.thrift.Type;
+import org.apache.hive.service.cli.ColumnDescriptor;
+import org.apache.hive.service.cli.FetchOrientation;
+import org.apache.hive.service.cli.HiveSQLException;
+import org.apache.hive.service.cli.OperationState;
+import org.apache.hive.service.cli.OperationType;
+import org.apache.hive.service.cli.RowSet;
+import org.apache.hive.service.cli.RowSetFactory;
+import org.apache.hive.service.cli.TableSchema;
+import org.apache.hive.service.cli.session.HiveSession;
+
+/**
+ * GetColumnsOperation.
+ *
+ */
+public class GetColumnsOperation extends MetadataOperation {
+
+  protected static final TableSchema RESULT_SET_SCHEMA = new TableSchema()
+  .addPrimitiveColumn("TABLE_CAT", Type.STRING_TYPE,
+      "Catalog name. NULL if not applicable")
+  .addPrimitiveColumn("TABLE_SCHEM", Type.STRING_TYPE,
+      "Schema name")
+  .addPrimitiveColumn("TABLE_NAME", Type.STRING_TYPE,
+      "Table name")
+  .addPrimitiveColumn("COLUMN_NAME", Type.STRING_TYPE,
+      "Column name")
+  .addPrimitiveColumn("DATA_TYPE", Type.INT_TYPE,
+      "SQL type from java.sql.Types")
+  .addPrimitiveColumn("TYPE_NAME", Type.STRING_TYPE,
+      "Data source dependent type name, for a UDT the type name is fully qualified")
+  .addPrimitiveColumn("COLUMN_SIZE", Type.INT_TYPE,
+      "Column size. For char or date types this is the maximum number of characters,"
+      + " for numeric or decimal types this is precision.")
+  .addPrimitiveColumn("BUFFER_LENGTH", Type.TINYINT_TYPE,
+      "Unused")
+  .addPrimitiveColumn("DECIMAL_DIGITS", Type.INT_TYPE,
+      "The number of fractional digits")
+  .addPrimitiveColumn("NUM_PREC_RADIX", Type.INT_TYPE,
+      "Radix (typically either 10 or 2)")
+  .addPrimitiveColumn("NULLABLE", Type.INT_TYPE,
+      "Is NULL allowed")
+  .addPrimitiveColumn("REMARKS", Type.STRING_TYPE,
+      "Comment describing column (may be null)")
+  .addPrimitiveColumn("COLUMN_DEF", Type.STRING_TYPE,
+      "Default value (may be null)")
+  .addPrimitiveColumn("SQL_DATA_TYPE", Type.INT_TYPE,
+      "Unused")
+  .addPrimitiveColumn("SQL_DATETIME_SUB", Type.INT_TYPE,
+      "Unused")
+  .addPrimitiveColumn("CHAR_OCTET_LENGTH", Type.INT_TYPE,
+      "For char types the maximum number of bytes in the column")
+  .addPrimitiveColumn("ORDINAL_POSITION", Type.INT_TYPE,
+      "Index of column in table (starting at 1)")
+  .addPrimitiveColumn("IS_NULLABLE", Type.STRING_TYPE,
+      "\"NO\" means column definitely does not allow NULL values; "
+      + "\"YES\" means the column might allow NULL values. An empty "
+      + "string means nobody knows.")
+  .addPrimitiveColumn("SCOPE_CATALOG", Type.STRING_TYPE,
+      "Catalog of table that is the scope of a reference attribute "
+      + "(null if DATA_TYPE isn't REF)")
+  .addPrimitiveColumn("SCOPE_SCHEMA", Type.STRING_TYPE,
+      "Schema of table that is the scope of a reference attribute "
+      + "(null if the DATA_TYPE isn't REF)")
+  .addPrimitiveColumn("SCOPE_TABLE", Type.STRING_TYPE,
+      "Table name that this the scope of a reference attribute "
+      + "(null if the DATA_TYPE isn't REF)")
+  .addPrimitiveColumn("SOURCE_DATA_TYPE", Type.SMALLINT_TYPE,
+      "Source type of a distinct type or user-generated Ref type, "
+      + "SQL type from java.sql.Types (null if DATA_TYPE isn't DISTINCT or user-generated REF)")
+  .addPrimitiveColumn("IS_AUTO_INCREMENT", Type.STRING_TYPE,
+      "Indicates whether this column is auto incremented.");
+
+  private final String catalogName;
+  private final String schemaName;
+  private final String tableName;
+  private final String columnName;
+
+  protected final RowSet rowSet;
+
+  protected GetColumnsOperation(HiveSession parentSession, String catalogName, String schemaName,
+      String tableName, String columnName) {
+    super(parentSession, OperationType.GET_COLUMNS);
+    this.catalogName = catalogName;
+    this.schemaName = schemaName;
+    this.tableName = tableName;
+    this.columnName = columnName;
+    this.rowSet = RowSetFactory.create(RESULT_SET_SCHEMA, getProtocolVersion(), false);
+  }
+
+  @Override
+  public void runInternal() throws HiveSQLException {
+    setState(OperationState.RUNNING);
+    try {
+      IMetaStoreClient metastoreClient = getParentSession().getMetaStoreClient();
+      String schemaPattern = convertSchemaPattern(schemaName);
+      String tablePattern = convertIdentifierPattern(tableName, true);
+
+      Pattern columnPattern = null;
+      if (columnName != null) {
+        columnPattern = Pattern.compile(convertIdentifierPattern(columnName, false));
+      }
+
+      List<String> dbNames = metastoreClient.getDatabases(schemaPattern);
+      Collections.sort(dbNames);
+      Map<String, List<String>> db2Tabs = new HashMap<>();
+
+      for (String dbName : dbNames) {
+        List<String> tableNames = metastoreClient.getTables(dbName, tablePattern);
+        Collections.sort(tableNames);
+        db2Tabs.put(dbName, tableNames);
+      }
+
+      if (isAuthV2Enabled()) {
+        List<HivePrivilegeObject> privObjs = getPrivObjs(db2Tabs);
+        String cmdStr = "catalog : " + catalogName + ", schemaPattern : " + schemaName
+            + ", tablePattern : " + tableName;
+        authorizeMetaGets(HiveOperationType.GET_COLUMNS, privObjs, cmdStr);
+      }
+
+      int maxBatchSize = SessionState.get().getConf().getIntVar(ConfVars.METASTORE_BATCH_RETRIEVE_MAX);
+      for (Entry<String, List<String>> dbTabs : db2Tabs.entrySet()) {
+        String dbName = dbTabs.getKey();
+        List<String> tableNames = dbTabs.getValue();
+
+        for (Table table : new TableIterable(metastoreClient, dbName, tableNames, maxBatchSize)) {
+
+          TableSchema schema = new TableSchema(metastoreClient.getSchema(dbName,
+              table.getTableName()));
+          List<SQLPrimaryKey> primaryKeys = metastoreClient.getPrimaryKeys(new PrimaryKeysRequest(dbName, table.getTableName()));
+          Set<String> pkColNames = new HashSet<>();
+          for(SQLPrimaryKey key : primaryKeys) {
+            pkColNames.add(key.getColumn_name().toLowerCase());
+          }
+          for (ColumnDescriptor column : schema.getColumnDescriptors()) {
+            if (columnPattern != null && !columnPattern.matcher(column.getName()).matches()) {
+              continue;
+            }
+            Object[] rowData = new Object[] {
+                null,  // TABLE_CAT
+                table.getDbName(), // TABLE_SCHEM
+                table.getTableName(), // TABLE_NAME
+                column.getName(), // COLUMN_NAME
+                column.getType().toJavaSQLType(), // DATA_TYPE
+                column.getTypeName(), // TYPE_NAME
+                column.getTypeDescriptor().getColumnSize(), // COLUMN_SIZE
+                null, // BUFFER_LENGTH, unused
+                column.getTypeDescriptor().getDecimalDigits(), // DECIMAL_DIGITS
+                column.getType().getNumPrecRadix(), // NUM_PREC_RADIX
+                pkColNames.contains(column.getName().toLowerCase()) ? DatabaseMetaData.columnNoNulls
+                    : DatabaseMetaData.columnNullable, // NULLABLE
+                column.getComment(), // REMARKS
+                null, // COLUMN_DEF
+                null, // SQL_DATA_TYPE
+                null, // SQL_DATETIME_SUB
+                null, // CHAR_OCTET_LENGTH
+                column.getOrdinalPosition(), // ORDINAL_POSITION
+                pkColNames.contains(column.getName().toLowerCase()) ? "NO" : "YES", // IS_NULLABLE
+                null, // SCOPE_CATALOG
+                null, // SCOPE_SCHEMA
+                null, // SCOPE_TABLE
+                null, // SOURCE_DATA_TYPE
+                "NO", // IS_AUTO_INCREMENT
+            };
+            rowSet.addRow(rowData);
+          }
+        }
+      }
+      setState(OperationState.FINISHED);
+    } catch (Exception e) {
+      setState(OperationState.ERROR);
+      throw new HiveSQLException(e);
+    }
+
+  }
+
+
+  private List<HivePrivilegeObject> getPrivObjs(Map<String, List<String>> db2Tabs) {
+    List<HivePrivilegeObject> privObjs = new ArrayList<>();
+    for (Entry<String, List<String>> dbTabs : db2Tabs.entrySet()) {
+      for (String tabName : dbTabs.getValue()) {
+        privObjs.add(new HivePrivilegeObject(HivePrivilegeObjectType.TABLE_OR_VIEW, dbTabs.getKey(),
+            tabName));
+      }
+    }
+    return privObjs;
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.Operation#getResultSetSchema()
+   */
+  @Override
+  public TableSchema getResultSetSchema() throws HiveSQLException {
+    assertState(OperationState.FINISHED);
+    return RESULT_SET_SCHEMA;
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.Operation#getNextRowSet(org.apache.hive.service.cli.FetchOrientation, long)
+   */
+  @Override
+  public RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws HiveSQLException {
+    assertState(OperationState.FINISHED);
+    validateDefaultFetchOrientation(orientation);
+    if (orientation.equals(FetchOrientation.FETCH_FIRST)) {
+      rowSet.setStartOffset(0);
+    }
+    return rowSet.extractSubset((int)maxRows);
+  }
+
+}
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/GetCrossReferenceOperation.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/GetCrossReferenceOperation.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/GetCrossReferenceOperation.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/GetCrossReferenceOperation.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/GetFunctionsOperation.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/GetFunctionsOperation.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/GetFunctionsOperation.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/GetFunctionsOperation.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/GetPrimaryKeysOperation.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/GetPrimaryKeysOperation.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/GetPrimaryKeysOperation.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/GetPrimaryKeysOperation.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/GetSchemasOperation.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/GetSchemasOperation.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/GetSchemasOperation.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/GetSchemasOperation.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/GetTableTypesOperation.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/GetTableTypesOperation.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/GetTableTypesOperation.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/GetTableTypesOperation.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/GetTablesOperation.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/GetTablesOperation.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/GetTablesOperation.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/GetTablesOperation.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/GetTypeInfoOperation.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/GetTypeInfoOperation.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/GetTypeInfoOperation.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/GetTypeInfoOperation.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/HiveTableTypeMapping.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/HiveTableTypeMapping.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/HiveTableTypeMapping.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/HiveTableTypeMapping.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/MetadataOperation.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/MetadataOperation.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/MetadataOperation.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/MetadataOperation.java
diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/Operation.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/Operation.java
new file mode 100644
index 0000000000000..558c68f85c16b
--- /dev/null
+++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/Operation.java
@@ -0,0 +1,350 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive.service.cli.operation;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.QueryState;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
+import org.apache.hadoop.hive.ql.session.OperationLog;
+import org.apache.hive.service.cli.FetchOrientation;
+import org.apache.hive.service.cli.HiveSQLException;
+import org.apache.hive.service.cli.OperationHandle;
+import org.apache.hive.service.cli.OperationState;
+import org.apache.hive.service.cli.OperationStatus;
+import org.apache.hive.service.cli.OperationType;
+import org.apache.hive.service.cli.RowSet;
+import org.apache.hive.service.cli.TableSchema;
+import org.apache.hive.service.cli.session.HiveSession;
+import org.apache.hive.service.rpc.thrift.TProtocolVersion;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class Operation {
+  protected final HiveSession parentSession;
+  private OperationState state = OperationState.INITIALIZED;
+  private final OperationHandle opHandle;
+  private HiveConf configuration;
+  public static final Logger LOG = LoggerFactory.getLogger(Operation.class.getName());
+  public static final FetchOrientation DEFAULT_FETCH_ORIENTATION = FetchOrientation.FETCH_NEXT;
+  public static final long DEFAULT_FETCH_MAX_ROWS = 100;
+  protected boolean hasResultSet;
+  protected volatile HiveSQLException operationException;
+  protected final boolean runAsync;
+  protected volatile Future<?> backgroundHandle;
+  protected OperationLog operationLog;
+  protected boolean isOperationLogEnabled;
+  protected Map<String, String> confOverlay = new HashMap<String, String>();
+
+  private long operationTimeout;
+  private long lastAccessTime;
+
+  protected final QueryState queryState;
+
+  protected static final EnumSet<FetchOrientation> DEFAULT_FETCH_ORIENTATION_SET =
+      EnumSet.of(
+          FetchOrientation.FETCH_NEXT,
+          FetchOrientation.FETCH_FIRST,
+          FetchOrientation.FETCH_PRIOR);
+
+  protected Operation(HiveSession parentSession, OperationType opType) {
+    this(parentSession, null, opType);
+  }
+
+  protected Operation(HiveSession parentSession, Map<String, String> confOverlay,
+      OperationType opType) {
+    this(parentSession, confOverlay, opType, false);
+  }
+
+  protected Operation(HiveSession parentSession,
+      Map<String, String> confOverlay, OperationType opType, boolean runInBackground) {
+    this.parentSession = parentSession;
+    this.confOverlay = confOverlay;
+    this.runAsync = runInBackground;
+    this.opHandle = new OperationHandle(opType, parentSession.getProtocolVersion());
+    lastAccessTime = System.currentTimeMillis();
+    operationTimeout = HiveConf.getTimeVar(parentSession.getHiveConf(),
+        HiveConf.ConfVars.HIVE_SERVER2_IDLE_OPERATION_TIMEOUT, TimeUnit.MILLISECONDS);
+    queryState = new QueryState(parentSession.getHiveConf(), confOverlay, runInBackground);
+  }
+
+  public Future<?> getBackgroundHandle() {
+    return backgroundHandle;
+  }
+
+  protected void setBackgroundHandle(Future<?> backgroundHandle) {
+    this.backgroundHandle = backgroundHandle;
+  }
+
+  public boolean shouldRunAsync() {
+    return runAsync;
+  }
+
+  public void setConfiguration(HiveConf configuration) {
+    this.configuration = new HiveConf(configuration);
+  }
+
+  public HiveConf getConfiguration() {
+    return new HiveConf(configuration);
+  }
+
+  public HiveSession getParentSession() {
+    return parentSession;
+  }
+
+  public OperationHandle getHandle() {
+    return opHandle;
+  }
+
+  public TProtocolVersion getProtocolVersion() {
+    return opHandle.getProtocolVersion();
+  }
+
+  public OperationType getType() {
+    return opHandle.getOperationType();
+  }
+
+  public OperationStatus getStatus() {
+    return new OperationStatus(state, operationException);
+  }
+
+  public boolean hasResultSet() {
+    return hasResultSet;
+  }
+
+  protected void setHasResultSet(boolean hasResultSet) {
+    this.hasResultSet = hasResultSet;
+    opHandle.setHasResultSet(hasResultSet);
+  }
+
+  public OperationLog getOperationLog() {
+    return operationLog;
+  }
+
+  protected final OperationState setState(OperationState newState) throws HiveSQLException {
+    state.validateTransition(newState);
+    this.state = newState;
+    this.lastAccessTime = System.currentTimeMillis();
+    return this.state;
+  }
+
+  public boolean isTimedOut(long current) {
+    if (operationTimeout == 0) {
+      return false;
+    }
+    if (operationTimeout > 0) {
+      // check only when it's in terminal state
+      return state.isTerminal() && lastAccessTime + operationTimeout <= current;
+    }
+    return lastAccessTime + -operationTimeout <= current;
+  }
+
+  public long getLastAccessTime() {
+    return lastAccessTime;
+  }
+
+  public long getOperationTimeout() {
+    return operationTimeout;
+  }
+
+  public void setOperationTimeout(long operationTimeout) {
+    this.operationTimeout = operationTimeout;
+  }
+
+  protected void setOperationException(HiveSQLException operationException) {
+    this.operationException = operationException;
+  }
+
+  protected final void assertState(OperationState state) throws HiveSQLException {
+    if (this.state != state) {
+      throw new HiveSQLException("Expected state " + state + ", but found " + this.state);
+    }
+    this.lastAccessTime = System.currentTimeMillis();
+  }
+
+  public boolean isRunning() {
+    return OperationState.RUNNING.equals(state);
+  }
+
+  public boolean isFinished() {
+    return OperationState.FINISHED.equals(state);
+  }
+
+  public boolean isCanceled() {
+    return OperationState.CANCELED.equals(state);
+  }
+
+  public boolean isFailed() {
+    return OperationState.ERROR.equals(state);
+  }
+
+  protected void createOperationLog() {
+    if (parentSession.isOperationLogEnabled()) {
+      File operationLogFile = new File(parentSession.getOperationLogSessionDir(),
+          opHandle.getHandleIdentifier().toString());
+      isOperationLogEnabled = true;
+
+      // create log file
+      try {
+        if (operationLogFile.exists()) {
+          LOG.warn("The operation log file should not exist, but it is already there: " +
+              operationLogFile.getAbsolutePath());
+          operationLogFile.delete();
+        }
+        if (!operationLogFile.createNewFile()) {
+          // the log file already exists and cannot be deleted.
+          // If it can be read/written, keep its contents and use it.
+          if (!operationLogFile.canRead() || !operationLogFile.canWrite()) {
+            LOG.warn("The already existed operation log file cannot be recreated, " +
+                "and it cannot be read or written: " + operationLogFile.getAbsolutePath());
+            isOperationLogEnabled = false;
+            return;
+          }
+        }
+      } catch (Exception e) {
+        LOG.warn("Unable to create operation log file: " + operationLogFile.getAbsolutePath(), e);
+        isOperationLogEnabled = false;
+        return;
+      }
+
+      // create OperationLog object with above log file
+      try {
+        operationLog = new OperationLog(opHandle.toString(), operationLogFile, parentSession.getHiveConf());
+      } catch (FileNotFoundException e) {
+        LOG.warn("Unable to instantiate OperationLog object for operation: " +
+            opHandle, e);
+        isOperationLogEnabled = false;
+        return;
+      }
+
+      // register this operationLog to current thread
+      OperationLog.setCurrentOperationLog(operationLog);
+    }
+  }
+
+  protected void unregisterOperationLog() {
+    if (isOperationLogEnabled) {
+      OperationLog.removeCurrentOperationLog();
+    }
+  }
+
+  /**
+   * Invoked before runInternal().
+   * Set up some preconditions, or configurations.
+   */
+  protected void beforeRun() {
+    createOperationLog();
+  }
+
+  /**
+   * Invoked after runInternal(), even if an exception is thrown in runInternal().
+   * Clean up resources, which was set up in beforeRun().
+   */
+  protected void afterRun() {
+    unregisterOperationLog();
+  }
+
+  /**
+   * Implemented by subclass of Operation class to execute specific behaviors.
+   * @throws HiveSQLException
+   */
+  protected abstract void runInternal() throws HiveSQLException;
+
+  public void run() throws HiveSQLException {
+    beforeRun();
+    try {
+      runInternal();
+    } finally {
+      afterRun();
+    }
+  }
+
+  protected void cleanupOperationLog() {
+    if (isOperationLogEnabled) {
+      if (operationLog == null) {
+        LOG.error("Operation [ " + opHandle.getHandleIdentifier() + " ] "
+          + "logging is enabled, but its OperationLog object cannot be found.");
+      } else {
+        operationLog.close();
+      }
+    }
+  }
+
+  // TODO: make this abstract and implement in subclasses.
+  public void cancel() throws HiveSQLException {
+    setState(OperationState.CANCELED);
+    throw new UnsupportedOperationException("SQLOperation.cancel()");
+  }
+
+  public void close() throws HiveSQLException {
+    setState(OperationState.CLOSED);
+    cleanupOperationLog();
+  }
+
+  public abstract TableSchema getResultSetSchema() throws HiveSQLException;
+
+  public abstract RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws HiveSQLException;
+
+  public RowSet getNextRowSet() throws HiveSQLException {
+    return getNextRowSet(FetchOrientation.FETCH_NEXT, DEFAULT_FETCH_MAX_ROWS);
+  }
+
+  /**
+   * Verify if the given fetch orientation is part of the default orientation types.
+   * @param orientation
+   * @throws HiveSQLException
+   */
+  protected void validateDefaultFetchOrientation(FetchOrientation orientation)
+      throws HiveSQLException {
+    validateFetchOrientation(orientation, DEFAULT_FETCH_ORIENTATION_SET);
+  }
+
+  /**
+   * Verify if the given fetch orientation is part of the supported orientation types.
+   * @param orientation
+   * @param supportedOrientations
+   * @throws HiveSQLException
+   */
+  protected void validateFetchOrientation(FetchOrientation orientation,
+      EnumSet<FetchOrientation> supportedOrientations) throws HiveSQLException {
+    if (!supportedOrientations.contains(orientation)) {
+      throw new HiveSQLException("The fetch type " + orientation.toString() +
+          " is not supported for this resultset", "HY106");
+    }
+  }
+
+  protected HiveSQLException toSQLException(String prefix, CommandProcessorResponse response) {
+    HiveSQLException ex = new HiveSQLException(prefix + ": " + response.getErrorMessage(),
+        response.getSQLState(), response.getResponseCode());
+    if (response.getException() != null) {
+      ex.initCause(response.getException());
+    }
+    return ex;
+  }
+
+  protected Map<String, String> getConfOverlay() {
+    return confOverlay;
+  }
+}
diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/OperationManager.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/OperationManager.java
new file mode 100644
index 0000000000000..3df842d2b4af9
--- /dev/null
+++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/OperationManager.java
@@ -0,0 +1,311 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.service.cli.operation;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Schema;
+import org.apache.hadoop.hive.ql.session.OperationLog;
+import org.apache.hive.service.AbstractService;
+import org.apache.hive.service.cli.FetchOrientation;
+import org.apache.hive.service.cli.HiveSQLException;
+import org.apache.hive.service.cli.OperationHandle;
+import org.apache.hive.service.cli.OperationState;
+import org.apache.hive.service.cli.OperationStatus;
+import org.apache.hive.service.cli.RowSet;
+import org.apache.hive.service.cli.RowSetFactory;
+import org.apache.hive.service.cli.TableSchema;
+import org.apache.hive.service.cli.session.HiveSession;
+import org.apache.log4j.Appender;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * OperationManager.
+ *
+ */
+public class OperationManager extends AbstractService {
+  private final Logger LOG = LoggerFactory.getLogger(OperationManager.class.getName());
+
+  private final Map<OperationHandle, Operation> handleToOperation =
+      new HashMap<OperationHandle, Operation>();
+
+  public OperationManager() {
+    super(OperationManager.class.getSimpleName());
+  }
+
+  @Override
+  public synchronized void init(HiveConf hiveConf) {
+    if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_ENABLED)) {
+      initOperationLogCapture(hiveConf.getVar(
+        HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL));
+    } else {
+      LOG.debug("Operation level logging is turned off");
+    }
+    super.init(hiveConf);
+  }
+
+  @Override
+  public synchronized void start() {
+    super.start();
+    // TODO
+  }
+
+  @Override
+  public synchronized void stop() {
+    // TODO
+    super.stop();
+  }
+
+  private void initOperationLogCapture(String loggingMode) {
+    // Register another Appender (with the same layout) that talks to us.
+    Appender ap = new LogDivertAppender(this, OperationLog.getLoggingLevel(loggingMode));
+    org.apache.log4j.Logger.getRootLogger().addAppender(ap);
+  }
+
+  public ExecuteStatementOperation newExecuteStatementOperation(HiveSession parentSession,
+      String statement, Map<String, String> confOverlay, boolean runAsync)
+          throws HiveSQLException {
+    ExecuteStatementOperation executeStatementOperation = ExecuteStatementOperation
+        .newExecuteStatementOperation(parentSession, statement, confOverlay, runAsync, 0);
+    addOperation(executeStatementOperation);
+    return executeStatementOperation;
+  }
+
+  public ExecuteStatementOperation newExecuteStatementOperation(HiveSession parentSession,
+      String statement, Map<String, String> confOverlay, boolean runAsync, long queryTimeout)
+          throws HiveSQLException {
+    return newExecuteStatementOperation(parentSession, statement, confOverlay, runAsync,
+        queryTimeout);
+  }
+
+  public GetTypeInfoOperation newGetTypeInfoOperation(HiveSession parentSession) {
+    GetTypeInfoOperation operation = new GetTypeInfoOperation(parentSession);
+    addOperation(operation);
+    return operation;
+  }
+
+  public GetCatalogsOperation newGetCatalogsOperation(HiveSession parentSession) {
+    GetCatalogsOperation operation = new GetCatalogsOperation(parentSession);
+    addOperation(operation);
+    return operation;
+  }
+
+  public GetSchemasOperation newGetSchemasOperation(HiveSession parentSession,
+      String catalogName, String schemaName) {
+    GetSchemasOperation operation = new GetSchemasOperation(parentSession, catalogName, schemaName);
+    addOperation(operation);
+    return operation;
+  }
+
+  public MetadataOperation newGetTablesOperation(HiveSession parentSession,
+      String catalogName, String schemaName, String tableName,
+      List<String> tableTypes) {
+    MetadataOperation operation =
+        new GetTablesOperation(parentSession, catalogName, schemaName, tableName, tableTypes);
+    addOperation(operation);
+    return operation;
+  }
+
+  public GetTableTypesOperation newGetTableTypesOperation(HiveSession parentSession) {
+    GetTableTypesOperation operation = new GetTableTypesOperation(parentSession);
+    addOperation(operation);
+    return operation;
+  }
+
+  public GetColumnsOperation newGetColumnsOperation(HiveSession parentSession,
+      String catalogName, String schemaName, String tableName, String columnName) {
+    GetColumnsOperation operation = new GetColumnsOperation(parentSession,
+        catalogName, schemaName, tableName, columnName);
+    addOperation(operation);
+    return operation;
+  }
+
+  public GetFunctionsOperation newGetFunctionsOperation(HiveSession parentSession,
+      String catalogName, String schemaName, String functionName) {
+    GetFunctionsOperation operation = new GetFunctionsOperation(parentSession,
+        catalogName, schemaName, functionName);
+    addOperation(operation);
+    return operation;
+  }
+
+  public GetPrimaryKeysOperation newGetPrimaryKeysOperation(HiveSession parentSession,
+      String catalogName, String schemaName, String tableName) {
+    GetPrimaryKeysOperation operation = new GetPrimaryKeysOperation(parentSession,
+      catalogName, schemaName, tableName);
+    addOperation(operation);
+    return operation;
+  }
+
+  public GetCrossReferenceOperation newGetCrossReferenceOperation(
+      HiveSession session, String primaryCatalog, String primarySchema,
+      String primaryTable, String foreignCatalog, String foreignSchema,
+      String foreignTable) {
+   GetCrossReferenceOperation operation = new GetCrossReferenceOperation(session,
+     primaryCatalog, primarySchema, primaryTable, foreignCatalog, foreignSchema,
+     foreignTable);
+   addOperation(operation);
+   return operation;
+  }
+
+  public Operation getOperation(OperationHandle operationHandle) throws HiveSQLException {
+    Operation operation = getOperationInternal(operationHandle);
+    if (operation == null) {
+      throw new HiveSQLException("Invalid OperationHandle: " + operationHandle);
+    }
+    return operation;
+  }
+
+  private synchronized Operation getOperationInternal(OperationHandle operationHandle) {
+    return handleToOperation.get(operationHandle);
+  }
+
+  private synchronized Operation removeTimedOutOperation(OperationHandle operationHandle) {
+    Operation operation = handleToOperation.get(operationHandle);
+    if (operation != null && operation.isTimedOut(System.currentTimeMillis())) {
+      handleToOperation.remove(operationHandle);
+      return operation;
+    }
+    return null;
+  }
+
+  private synchronized void addOperation(Operation operation) {
+    handleToOperation.put(operation.getHandle(), operation);
+  }
+
+  private synchronized Operation removeOperation(OperationHandle opHandle) {
+    return handleToOperation.remove(opHandle);
+  }
+
+  public OperationStatus getOperationStatus(OperationHandle opHandle)
+      throws HiveSQLException {
+    return getOperation(opHandle).getStatus();
+  }
+
+  public void cancelOperation(OperationHandle opHandle) throws HiveSQLException {
+    Operation operation = getOperation(opHandle);
+    OperationState opState = operation.getStatus().getState();
+    if (opState == OperationState.CANCELED ||
+        opState == OperationState.TIMEDOUT ||
+        opState == OperationState.CLOSED ||
+        opState == OperationState.FINISHED ||
+        opState == OperationState.ERROR ||
+        opState == OperationState.UNKNOWN) {
+      // Cancel should be a no-op in either cases
+      LOG.debug(opHandle + ": Operation is already aborted in state - " + opState);
+    }
+    else {
+      LOG.debug(opHandle + ": Attempting to cancel from state - " + opState);
+      operation.cancel();
+    }
+  }
+
+  public void closeOperation(OperationHandle opHandle) throws HiveSQLException {
+    Operation operation = removeOperation(opHandle);
+    if (operation == null) {
+      throw new HiveSQLException("Operation does not exist!");
+    }
+    operation.close();
+  }
+
+  public TableSchema getOperationResultSetSchema(OperationHandle opHandle)
+      throws HiveSQLException {
+    return getOperation(opHandle).getResultSetSchema();
+  }
+
+  public RowSet getOperationNextRowSet(OperationHandle opHandle)
+      throws HiveSQLException {
+    return getOperation(opHandle).getNextRowSet();
+  }
+
+  public RowSet getOperationNextRowSet(OperationHandle opHandle,
+      FetchOrientation orientation, long maxRows)
+          throws HiveSQLException {
+    return getOperation(opHandle).getNextRowSet(orientation, maxRows);
+  }
+
+  public RowSet getOperationLogRowSet(OperationHandle opHandle,
+      FetchOrientation orientation, long maxRows)
+          throws HiveSQLException {
+    // get the OperationLog object from the operation
+    OperationLog operationLog = getOperation(opHandle).getOperationLog();
+    if (operationLog == null) {
+      throw new HiveSQLException("Couldn't find log associated with operation handle: " + opHandle);
+    }
+
+    // read logs
+    List<String> logs;
+    try {
+      logs = operationLog.readOperationLog(isFetchFirst(orientation), maxRows);
+    } catch (SQLException e) {
+      throw new HiveSQLException(e.getMessage(), e.getCause());
+    }
+
+
+    // convert logs to RowSet
+    TableSchema tableSchema = new TableSchema(getLogSchema());
+    RowSet rowSet = RowSetFactory.create(tableSchema,
+        getOperation(opHandle).getProtocolVersion(), false);
+    for (String log : logs) {
+      rowSet.addRow(new String[] {log});
+    }
+
+    return rowSet;
+  }
+
+  private boolean isFetchFirst(FetchOrientation fetchOrientation) {
+    //TODO: Since OperationLog is moved to package o.a.h.h.ql.session,
+    // we may add a Enum there and map FetchOrientation to it.
+    if (fetchOrientation.equals(FetchOrientation.FETCH_FIRST)) {
+      return true;
+    }
+    return false;
+  }
+
+  private Schema getLogSchema() {
+    Schema schema = new Schema();
+    FieldSchema fieldSchema = new FieldSchema();
+    fieldSchema.setName("operation_log");
+    fieldSchema.setType("string");
+    schema.addToFieldSchemas(fieldSchema);
+    return schema;
+  }
+
+  public OperationLog getOperationLogByThread() {
+    return OperationLog.getCurrentOperationLog();
+  }
+
+  public List<Operation> removeExpiredOperations(OperationHandle[] handles) {
+    List<Operation> removed = new ArrayList<Operation>();
+    for (OperationHandle handle : handles) {
+      Operation operation = removeTimedOutOperation(handle);
+      if (operation != null) {
+        LOG.warn("Operation " + handle + " is timed-out and will be closed");
+        removed.add(operation);
+      }
+    }
+    return removed;
+  }
+}
diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/SQLOperation.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/SQLOperation.java
new file mode 100644
index 0000000000000..894793152f409
--- /dev/null
+++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/SQLOperation.java
@@ -0,0 +1,457 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.service.cli.operation;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.security.PrivilegedExceptionAction;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.Future;
+import java.util.concurrent.RejectedExecutionException;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Schema;
+import org.apache.hadoop.hive.ql.CommandNeedRetryException;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.QueryState;
+import org.apache.hadoop.hive.ql.exec.ExplainTask;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.AbstractSerDe;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.SerDeUtils;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.shims.Utils;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hive.service.cli.FetchOrientation;
+import org.apache.hive.service.cli.HiveSQLException;
+import org.apache.hive.service.cli.OperationState;
+import org.apache.hive.service.cli.RowSet;
+import org.apache.hive.service.cli.RowSetFactory;
+import org.apache.hive.service.cli.TableSchema;
+import org.apache.hive.service.cli.session.HiveSession;
+import org.apache.hive.service.server.ThreadWithGarbageCleanup;
+
+/**
+ * SQLOperation.
+ *
+ */
+public class SQLOperation extends ExecuteStatementOperation {
+
+  private Driver driver = null;
+  private CommandProcessorResponse response;
+  private TableSchema resultSchema = null;
+  private Schema mResultSchema = null;
+  private AbstractSerDe serde = null;
+  private boolean fetchStarted = false;
+
+  public SQLOperation(HiveSession parentSession, String statement, Map<String, String> confOverlay,
+      boolean runInBackground, long queryTimeout) {
+    // TODO: call setRemoteUser in ExecuteStatementOperation or higher.
+    super(parentSession, statement, confOverlay, runInBackground);
+  }
+
+  /***
+   * Compile the query and extract metadata
+   * @param queryState
+   * @throws HiveSQLException
+   */
+  public void prepare(QueryState queryState) throws HiveSQLException {
+    setState(OperationState.RUNNING);
+
+    try {
+      driver = new Driver(queryState, getParentSession().getUserName());
+
+      // set the operation handle information in Driver, so that thrift API users
+      // can use the operation handle they receive, to lookup query information in
+      // Yarn ATS
+      String guid64 = Base64.encodeBase64URLSafeString(getHandle().getHandleIdentifier()
+          .toTHandleIdentifier().getGuid()).trim();
+      driver.setOperationId(guid64);
+
+      // In Hive server mode, we are not able to retry in the FetchTask
+      // case, when calling fetch queries since execute() has returned.
+      // For now, we disable the test attempts.
+      driver.setTryCount(Integer.MAX_VALUE);
+
+      response = driver.compileAndRespond(statement);
+      if (0 != response.getResponseCode()) {
+        throw toSQLException("Error while compiling statement", response);
+      }
+
+      mResultSchema = driver.getSchema();
+
+      // hasResultSet should be true only if the query has a FetchTask
+      // "explain" is an exception for now
+      if(driver.getPlan().getFetchTask() != null) {
+        //Schema has to be set
+        if (mResultSchema == null || !mResultSchema.isSetFieldSchemas()) {
+          throw new HiveSQLException("Error compiling query: Schema and FieldSchema " +
+              "should be set when query plan has a FetchTask");
+        }
+        resultSchema = new TableSchema(mResultSchema);
+        setHasResultSet(true);
+      } else {
+        setHasResultSet(false);
+      }
+      // Set hasResultSet true if the plan has ExplainTask
+      // TODO explain should use a FetchTask for reading
+      for (Task<? extends Serializable> task: driver.getPlan().getRootTasks()) {
+        if (task.getClass() == ExplainTask.class) {
+          resultSchema = new TableSchema(mResultSchema);
+          setHasResultSet(true);
+          break;
+        }
+      }
+    } catch (HiveSQLException e) {
+      setState(OperationState.ERROR);
+      throw e;
+    } catch (Exception e) {
+      setState(OperationState.ERROR);
+      throw new HiveSQLException("Error running query: " + e.toString(), e);
+    }
+  }
+
+  private void runQuery(HiveConf sqlOperationConf) throws HiveSQLException {
+    try {
+      // In Hive server mode, we are not able to retry in the FetchTask
+      // case, when calling fetch queries since execute() has returned.
+      // For now, we disable the test attempts.
+      driver.setTryCount(Integer.MAX_VALUE);
+      response = driver.run();
+      if (0 != response.getResponseCode()) {
+        throw toSQLException("Error while processing statement", response);
+      }
+    } catch (HiveSQLException e) {
+      // If the operation was cancelled by another thread or timed out,
+      // Driver#run will return a non-zero response code.
+      // We will simply return if the operation state is CANCELED or TIMEDOUT,
+      // otherwise throw an exception
+      if (getStatus().getState() == OperationState.CANCELED ||
+          getStatus().getState() == OperationState.TIMEDOUT) {
+        return;
+      }
+      else {
+        setState(OperationState.ERROR);
+        throw e;
+      }
+    } catch (Exception e) {
+      setState(OperationState.ERROR);
+      throw new HiveSQLException("Error running query: " + e.toString(), e);
+    }
+    setState(OperationState.FINISHED);
+  }
+
+  @Override
+  public void runInternal() throws HiveSQLException {
+    setState(OperationState.PENDING);
+    final HiveConf opConfig = getConfigForOperation();
+    prepare(queryState);
+    if (!shouldRunAsync()) {
+      runQuery(opConfig);
+    } else {
+      // We'll pass ThreadLocals in the background thread from the foreground (handler) thread
+      final SessionState parentSessionState = SessionState.get();
+      // ThreadLocal Hive object needs to be set in background thread.
+      // The metastore client in Hive is associated with right user.
+      final Hive parentHive = getSessionHive();
+      // Current UGI will get used by metastore when metsatore is in embedded mode
+      // So this needs to get passed to the new background thread
+      final UserGroupInformation currentUGI = getCurrentUGI(opConfig);
+      // Runnable impl to call runInternal asynchronously,
+      // from a different thread
+      Runnable backgroundOperation = new Runnable() {
+        @Override
+        public void run() {
+          PrivilegedExceptionAction<Object> doAsAction = new PrivilegedExceptionAction<Object>() {
+            @Override
+            public Object run() throws HiveSQLException {
+              Hive.set(parentHive);
+              SessionState.setCurrentSessionState(parentSessionState);
+              // Set current OperationLog in this async thread for keeping on saving query log.
+              registerCurrentOperationLog();
+              try {
+                runQuery(opConfig);
+              } catch (HiveSQLException e) {
+                setOperationException(e);
+                LOG.error("Error running hive query: ", e);
+              } finally {
+                unregisterOperationLog();
+              }
+              return null;
+            }
+          };
+
+          try {
+            currentUGI.doAs(doAsAction);
+          } catch (Exception e) {
+            setOperationException(new HiveSQLException(e));
+            LOG.error("Error running hive query as user : " + currentUGI.getShortUserName(), e);
+          }
+          finally {
+            /**
+             * We'll cache the ThreadLocal RawStore object for this background thread for an orderly cleanup
+             * when this thread is garbage collected later.
+             * @see org.apache.hive.service.server.ThreadWithGarbageCleanup#finalize()
+             */
+            if (ThreadWithGarbageCleanup.currentThread() instanceof ThreadWithGarbageCleanup) {
+              ThreadWithGarbageCleanup currentThread =
+                  (ThreadWithGarbageCleanup) ThreadWithGarbageCleanup.currentThread();
+              currentThread.cacheThreadLocalRawStore();
+            }
+          }
+        }
+      };
+      try {
+        // This submit blocks if no background threads are available to run this operation
+        Future<?> backgroundHandle =
+            getParentSession().getSessionManager().submitBackgroundOperation(backgroundOperation);
+        setBackgroundHandle(backgroundHandle);
+      } catch (RejectedExecutionException rejected) {
+        setState(OperationState.ERROR);
+        throw new HiveSQLException("The background threadpool cannot accept" +
+            " new task for execution, please retry the operation", rejected);
+      }
+    }
+  }
+
+  /**
+   * Returns the current UGI on the stack
+   * @param opConfig
+   * @return UserGroupInformation
+   * @throws HiveSQLException
+   */
+  private UserGroupInformation getCurrentUGI(HiveConf opConfig) throws HiveSQLException {
+    try {
+      return Utils.getUGI();
+    } catch (Exception e) {
+      throw new HiveSQLException("Unable to get current user", e);
+    }
+  }
+
+  /**
+   * Returns the ThreadLocal Hive for the current thread
+   * @return Hive
+   * @throws HiveSQLException
+   */
+  private Hive getSessionHive() throws HiveSQLException {
+    try {
+      return Hive.get();
+    } catch (HiveException e) {
+      throw new HiveSQLException("Failed to get ThreadLocal Hive object", e);
+    }
+  }
+
+  private void cleanup(OperationState state) throws HiveSQLException {
+    setState(state);
+    if (shouldRunAsync()) {
+      Future<?> backgroundHandle = getBackgroundHandle();
+      if (backgroundHandle != null) {
+        backgroundHandle.cancel(true);
+      }
+    }
+    if (driver != null) {
+      driver.close();
+      driver.destroy();
+    }
+    driver = null;
+
+    SessionState ss = SessionState.get();
+    if (ss.getTmpOutputFile() != null) {
+      ss.getTmpOutputFile().delete();
+    }
+  }
+
+  @Override
+  public void cancel() throws HiveSQLException {
+    cleanup(OperationState.CANCELED);
+  }
+
+  @Override
+  public void close() throws HiveSQLException {
+    cleanup(OperationState.CLOSED);
+    cleanupOperationLog();
+  }
+
+  @Override
+  public TableSchema getResultSetSchema() throws HiveSQLException {
+    assertState(OperationState.FINISHED);
+    if (resultSchema == null) {
+      resultSchema = new TableSchema(driver.getSchema());
+    }
+    return resultSchema;
+  }
+
+  private final transient List<Object> convey = new ArrayList<Object>();
+
+  @Override
+  public RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws HiveSQLException {
+    validateDefaultFetchOrientation(orientation);
+    assertState(OperationState.FINISHED);
+
+    RowSet rowSet = RowSetFactory.create(resultSchema, getProtocolVersion(), false);
+
+    try {
+      /* if client is requesting fetch-from-start and its not the first time reading from this operation
+       * then reset the fetch position to beginning
+       */
+      if (orientation.equals(FetchOrientation.FETCH_FIRST) && fetchStarted) {
+        driver.resetFetch();
+      }
+      fetchStarted = true;
+      driver.setMaxRows((int) maxRows);
+      if (driver.getResults(convey)) {
+        return decode(convey, rowSet);
+      }
+      return rowSet;
+    } catch (IOException e) {
+      throw new HiveSQLException(e);
+    } catch (CommandNeedRetryException e) {
+      throw new HiveSQLException(e);
+    } catch (Exception e) {
+      throw new HiveSQLException(e);
+    } finally {
+      convey.clear();
+    }
+  }
+
+  private RowSet decode(List<Object> rows, RowSet rowSet) throws Exception {
+    if (driver.isFetchingTable()) {
+      return prepareFromRow(rows, rowSet);
+    }
+    return decodeFromString(rows, rowSet);
+  }
+
+  // already encoded to thrift-able object in ThriftFormatter
+  private RowSet prepareFromRow(List<Object> rows, RowSet rowSet) throws Exception {
+    for (Object row : rows) {
+      rowSet.addRow((Object[]) row);
+    }
+    return rowSet;
+  }
+
+  private RowSet decodeFromString(List<Object> rows, RowSet rowSet)
+      throws SQLException, SerDeException {
+    getSerDe();
+    StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector();
+    List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs();
+
+    Object[] deserializedFields = new Object[fieldRefs.size()];
+    Object rowObj;
+    ObjectInspector fieldOI;
+
+    int protocol = getProtocolVersion().getValue();
+    for (Object rowString : rows) {
+      rowObj = serde.deserialize(new BytesWritable(((String)rowString).getBytes(UTF_8)));
+      for (int i = 0; i < fieldRefs.size(); i++) {
+        StructField fieldRef = fieldRefs.get(i);
+        fieldOI = fieldRef.getFieldObjectInspector();
+        Object fieldData = soi.getStructFieldData(rowObj, fieldRef);
+        deserializedFields[i] = SerDeUtils.toThriftPayload(fieldData, fieldOI, protocol);
+      }
+      rowSet.addRow(deserializedFields);
+    }
+    return rowSet;
+  }
+
+  private AbstractSerDe getSerDe() throws SQLException {
+    if (serde != null) {
+      return serde;
+    }
+    try {
+      List<FieldSchema> fieldSchemas = mResultSchema.getFieldSchemas();
+      StringBuilder namesSb = new StringBuilder();
+      StringBuilder typesSb = new StringBuilder();
+
+      if (fieldSchemas != null && !fieldSchemas.isEmpty()) {
+        for (int pos = 0; pos < fieldSchemas.size(); pos++) {
+          if (pos != 0) {
+            namesSb.append(",");
+            typesSb.append(",");
+          }
+          namesSb.append(fieldSchemas.get(pos).getName());
+          typesSb.append(fieldSchemas.get(pos).getType());
+        }
+      }
+      String names = namesSb.toString();
+      String types = typesSb.toString();
+
+      serde = new LazySimpleSerDe();
+      Properties props = new Properties();
+      if (names.length() > 0) {
+        LOG.debug("Column names: " + names);
+        props.setProperty(serdeConstants.LIST_COLUMNS, names);
+      }
+      if (types.length() > 0) {
+        LOG.debug("Column types: " + types);
+        props.setProperty(serdeConstants.LIST_COLUMN_TYPES, types);
+      }
+      SerDeUtils.initializeSerDe(serde, new HiveConf(), props, null);
+
+    } catch (Exception ex) {
+      ex.printStackTrace();
+      throw new SQLException("Could not create ResultSet: " + ex.getMessage(), ex);
+    }
+    return serde;
+  }
+
+  /**
+   * If there are query specific settings to overlay, then create a copy of config
+   * There are two cases we need to clone the session config that's being passed to hive driver
+   * 1. Async query -
+   *    If the client changes a config setting, that shouldn't reflect in the execution already underway
+   * 2. confOverlay -
+   *    The query specific settings should only be applied to the query config and not session
+   * @return new configuration
+   * @throws HiveSQLException
+   */
+  private HiveConf getConfigForOperation() throws HiveSQLException {
+    HiveConf sqlOperationConf = getParentSession().getHiveConf();
+    if (!getConfOverlay().isEmpty() || shouldRunAsync()) {
+      // clone the parent session config for this query
+      sqlOperationConf = new HiveConf(sqlOperationConf);
+
+      // apply overlay query specific settings, if any
+      for (Map.Entry<String, String> confEntry : getConfOverlay().entrySet()) {
+        try {
+          sqlOperationConf.verifyAndSet(confEntry.getKey(), confEntry.getValue());
+        } catch (IllegalArgumentException e) {
+          throw new HiveSQLException("Error applying statement specific settings", e);
+        }
+      }
+    }
+    return sqlOperationConf;
+  }
+}
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/TableTypeMapping.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/TableTypeMapping.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/TableTypeMapping.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/TableTypeMapping.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSession.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/HiveSession.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSession.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/HiveSession.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionBase.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/HiveSessionBase.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionBase.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/HiveSessionBase.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionHookContext.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/HiveSessionHookContext.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionHookContext.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/HiveSessionHookContext.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionHookContextImpl.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/HiveSessionHookContextImpl.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionHookContextImpl.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/HiveSessionHookContextImpl.java
diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
new file mode 100644
index 0000000000000..f47a4388f7bea
--- /dev/null
+++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
@@ -0,0 +1,901 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.service.cli.session;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hive.common.cli.HiveFileProcessor;
+import org.apache.hadoop.hive.common.cli.IHiveFileProcessor;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.conf.VariableSubstitution;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.history.HiveHistory;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.serde2.SerDeUtils;
+import org.apache.hadoop.hive.serde2.thrift.ThriftFormatter;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hive.common.util.HiveVersionInfo;
+import org.apache.hive.service.auth.HiveAuthFactory;
+import org.apache.hive.service.cli.FetchOrientation;
+import org.apache.hive.service.cli.FetchType;
+import org.apache.hive.service.cli.GetInfoType;
+import org.apache.hive.service.cli.GetInfoValue;
+import org.apache.hive.service.cli.HiveSQLException;
+import org.apache.hive.service.cli.OperationHandle;
+import org.apache.hive.service.cli.RowSet;
+import org.apache.hive.service.cli.SessionHandle;
+import org.apache.hive.service.cli.TableSchema;
+import org.apache.hive.service.cli.operation.ExecuteStatementOperation;
+import org.apache.hive.service.cli.operation.GetCatalogsOperation;
+import org.apache.hive.service.cli.operation.GetColumnsOperation;
+import org.apache.hive.service.cli.operation.GetCrossReferenceOperation;
+import org.apache.hive.service.cli.operation.GetFunctionsOperation;
+import org.apache.hive.service.cli.operation.GetPrimaryKeysOperation;
+import org.apache.hive.service.cli.operation.GetSchemasOperation;
+import org.apache.hive.service.cli.operation.GetTableTypesOperation;
+import org.apache.hive.service.cli.operation.GetTypeInfoOperation;
+import org.apache.hive.service.cli.operation.MetadataOperation;
+import org.apache.hive.service.cli.operation.Operation;
+import org.apache.hive.service.cli.operation.OperationManager;
+import org.apache.hive.service.rpc.thrift.TProtocolVersion;
+import org.apache.hive.service.server.ThreadWithGarbageCleanup;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.hive.conf.SystemVariables.ENV_PREFIX;
+import static org.apache.hadoop.hive.conf.SystemVariables.HIVECONF_PREFIX;
+import static org.apache.hadoop.hive.conf.SystemVariables.HIVEVAR_PREFIX;
+import static org.apache.hadoop.hive.conf.SystemVariables.METACONF_PREFIX;
+import static org.apache.hadoop.hive.conf.SystemVariables.SYSTEM_PREFIX;
+
+/**
+ * HiveSession
+ *
+ */
+public class HiveSessionImpl implements HiveSession {
+  private final SessionHandle sessionHandle;
+  private String username;
+  private final String password;
+  private HiveConf hiveConf;
+  private SessionState sessionState;
+  private String ipAddress;
+  private static final String FETCH_WORK_SERDE_CLASS =
+      "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe";
+  private static final Logger LOG = LoggerFactory.getLogger(HiveSessionImpl.class);
+  private SessionManager sessionManager;
+  private OperationManager operationManager;
+  private final Set<OperationHandle> opHandleSet = new HashSet<OperationHandle>();
+  private boolean isOperationLogEnabled;
+  private File sessionLogDir;
+  private volatile long lastAccessTime;
+  private volatile long lastIdleTime;
+
+  public HiveSessionImpl(TProtocolVersion protocol, String username, String password,
+      HiveConf serverhiveConf, String ipAddress) {
+    this.username = username;
+    this.password = password;
+    this.sessionHandle = new SessionHandle(protocol);
+    this.hiveConf = new HiveConf(serverhiveConf);
+    this.ipAddress = ipAddress;
+
+    try {
+      // In non-impersonation mode, map scheduler queue to current user
+      // if fair scheduler is configured.
+      if (! hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS) &&
+        hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE)) {
+        ShimLoader.getHadoopShims().refreshDefaultQueue(hiveConf, username);
+      }
+    } catch (IOException e) {
+      LOG.warn("Error setting scheduler queue: " + e, e);
+    }
+    // Set an explicit session name to control the download directory name
+    hiveConf.set(ConfVars.HIVESESSIONID.varname,
+        sessionHandle.getHandleIdentifier().toString());
+    // Use thrift transportable formatter
+    hiveConf.set(SerDeUtils.LIST_SINK_OUTPUT_FORMATTER, ThriftFormatter.class.getName());
+    hiveConf.setInt(SerDeUtils.LIST_SINK_OUTPUT_PROTOCOL, protocol.getValue());
+  }
+
+  @Override
+  /**
+   * Opens a new HiveServer2 session for the client connection.
+   * Creates a new SessionState object that will be associated with this HiveServer2 session.
+   * When the server executes multiple queries in the same session,
+   * this SessionState object is reused across multiple queries.
+   * Note that if doAs is true, this call goes through a proxy object,
+   * which wraps the method logic in a UserGroupInformation#doAs.
+   * That's why it is important to create SessionState here rather than in the constructor.
+   */
+  public void open(Map<String, String> sessionConfMap) throws HiveSQLException {
+    sessionState = new SessionState(hiveConf, username);
+    sessionState.setUserIpAddress(ipAddress);
+    sessionState.setIsHiveServerQuery(true);
+    SessionState.start(sessionState);
+    try {
+      sessionState.loadAuxJars();
+      sessionState.loadReloadableAuxJars();
+    } catch (IOException e) {
+      String msg = "Failed to load reloadable jar file path: " + e;
+      LOG.error(msg, e);
+      throw new HiveSQLException(msg, e);
+    }
+    // Process global init file: .hiverc
+    processGlobalInitFile();
+    if (sessionConfMap != null) {
+      configureSession(sessionConfMap);
+    }
+    lastAccessTime = System.currentTimeMillis();
+    lastIdleTime = lastAccessTime;
+  }
+
+  /**
+   * It is used for processing hiverc file from HiveServer2 side.
+   */
+  private class GlobalHivercFileProcessor extends HiveFileProcessor {
+    @Override
+    protected BufferedReader loadFile(String fileName) throws IOException {
+      FileInputStream initStream = null;
+      BufferedReader bufferedReader = null;
+      initStream = new FileInputStream(fileName);
+      bufferedReader = new BufferedReader(new InputStreamReader(initStream));
+      return bufferedReader;
+    }
+
+    @Override
+    protected int processCmd(String cmd) {
+      int rc = 0;
+      String cmd_trimmed = cmd.trim();
+      try {
+        executeStatementInternal(cmd_trimmed, null, false, 0);
+      } catch (HiveSQLException e) {
+        rc = -1;
+        LOG.warn("Failed to execute HQL command in global .hiverc file.", e);
+      }
+      return rc;
+    }
+  }
+
+  private void processGlobalInitFile() {
+    IHiveFileProcessor processor = new GlobalHivercFileProcessor();
+
+    try {
+      String hiverc = hiveConf.getVar(ConfVars.HIVE_SERVER2_GLOBAL_INIT_FILE_LOCATION);
+      if (hiverc != null) {
+        File hivercFile = new File(hiverc);
+        if (hivercFile.isDirectory()) {
+          hivercFile = new File(hivercFile, SessionManager.HIVERCFILE);
+        }
+        if (hivercFile.isFile()) {
+          LOG.info("Running global init file: " + hivercFile);
+          int rc = processor.processFile(hivercFile.getAbsolutePath());
+          if (rc != 0) {
+            LOG.error("Failed on initializing global .hiverc file");
+          }
+        } else {
+          LOG.debug("Global init file " + hivercFile + " does not exist");
+        }
+      }
+    } catch (IOException e) {
+      LOG.warn("Failed on initializing global .hiverc file", e);
+    }
+  }
+
+  private void configureSession(Map<String, String> sessionConfMap) throws HiveSQLException {
+    SessionState.setCurrentSessionState(sessionState);
+    for (Map.Entry<String, String> entry : sessionConfMap.entrySet()) {
+      String key = entry.getKey();
+      if (key.startsWith("set:")) {
+        try {
+          setVariable(key.substring(4), entry.getValue());
+        } catch (Exception e) {
+          throw new HiveSQLException(e);
+        }
+      } else if (key.startsWith("use:")) {
+        SessionState.get().setCurrentDatabase(entry.getValue());
+      } else {
+        hiveConf.verifyAndSet(key, entry.getValue());
+      }
+    }
+  }
+
+  // Copy from org.apache.hadoop.hive.ql.processors.SetProcessor, only change:
+  // setConf(varname, propName, varvalue, true) when varname.startsWith(HIVECONF_PREFIX)
+  public static int setVariable(String varname, String varvalue) throws Exception {
+    SessionState ss = SessionState.get();
+    VariableSubstitution substitution = new VariableSubstitution(() -> ss.getHiveVariables());
+    if (varvalue.contains("\n")){
+      ss.err.println("Warning: Value had a \\n character in it.");
+    }
+    varname = varname.trim();
+    if (varname.startsWith(ENV_PREFIX)){
+      ss.err.println("env:* variables can not be set.");
+      return 1;
+    } else if (varname.startsWith(SYSTEM_PREFIX)){
+      String propName = varname.substring(SYSTEM_PREFIX.length());
+      System.getProperties().setProperty(propName, substitution.substitute(ss.getConf(),varvalue));
+    } else if (varname.startsWith(HIVECONF_PREFIX)){
+      String propName = varname.substring(HIVECONF_PREFIX.length());
+      setConf(varname, propName, varvalue, true);
+    } else if (varname.startsWith(HIVEVAR_PREFIX)) {
+      String propName = varname.substring(HIVEVAR_PREFIX.length());
+      ss.getHiveVariables().put(propName, substitution.substitute(ss.getConf(),varvalue));
+    } else if (varname.startsWith(METACONF_PREFIX)) {
+      String propName = varname.substring(METACONF_PREFIX.length());
+      Hive hive = Hive.get(ss.getConf());
+      hive.setMetaConf(propName, substitution.substitute(ss.getConf(), varvalue));
+    } else {
+      setConf(varname, varname, varvalue, true);
+    }
+    return 0;
+  }
+
+  // returns non-null string for validation fail
+  private static void setConf(String varname, String key, String varvalue, boolean register)
+          throws IllegalArgumentException {
+    VariableSubstitution substitution =
+        new VariableSubstitution(() -> SessionState.get().getHiveVariables());
+    HiveConf conf = SessionState.get().getConf();
+    String value = substitution.substitute(conf, varvalue);
+    if (conf.getBoolVar(HiveConf.ConfVars.HIVECONFVALIDATION)) {
+      HiveConf.ConfVars confVars = HiveConf.getConfVars(key);
+      if (confVars != null) {
+        if (!confVars.isType(value)) {
+          StringBuilder message = new StringBuilder();
+          message.append("'SET ").append(varname).append('=').append(varvalue);
+          message.append("' FAILED because ").append(key).append(" expects ");
+          message.append(confVars.typeString()).append(" type value.");
+          throw new IllegalArgumentException(message.toString());
+        }
+        String fail = confVars.validate(value);
+        if (fail != null) {
+          StringBuilder message = new StringBuilder();
+          message.append("'SET ").append(varname).append('=').append(varvalue);
+          message.append("' FAILED in validation : ").append(fail).append('.');
+          throw new IllegalArgumentException(message.toString());
+        }
+      } else if (key.startsWith("hive.")) {
+        throw new IllegalArgumentException("hive configuration " + key + " does not exists.");
+      }
+    }
+    conf.verifyAndSet(key, value);
+    if (register) {
+      SessionState.get().getOverriddenConfigurations().put(key, value);
+    }
+  }
+
+  @Override
+  public void setOperationLogSessionDir(File operationLogRootDir) {
+    if (!operationLogRootDir.exists()) {
+      LOG.warn("The operation log root directory is removed, recreating: " +
+          operationLogRootDir.getAbsolutePath());
+      if (!operationLogRootDir.mkdirs()) {
+        LOG.warn("Unable to create operation log root directory: " +
+            operationLogRootDir.getAbsolutePath());
+      }
+    }
+    if (!operationLogRootDir.canWrite()) {
+      LOG.warn("The operation log root directory is not writable: " +
+          operationLogRootDir.getAbsolutePath());
+    }
+    sessionLogDir = new File(operationLogRootDir, sessionHandle.getHandleIdentifier().toString());
+    isOperationLogEnabled = true;
+    if (!sessionLogDir.exists()) {
+      if (!sessionLogDir.mkdir()) {
+        LOG.warn("Unable to create operation log session directory: " +
+            sessionLogDir.getAbsolutePath());
+        isOperationLogEnabled = false;
+      }
+    }
+    if (isOperationLogEnabled) {
+      LOG.info("Operation log session directory is created: " + sessionLogDir.getAbsolutePath());
+    }
+  }
+
+  @Override
+  public boolean isOperationLogEnabled() {
+    return isOperationLogEnabled;
+  }
+
+  @Override
+  public File getOperationLogSessionDir() {
+    return sessionLogDir;
+  }
+
+  @Override
+  public TProtocolVersion getProtocolVersion() {
+    return sessionHandle.getProtocolVersion();
+  }
+
+  @Override
+  public SessionManager getSessionManager() {
+    return sessionManager;
+  }
+
+  @Override
+  public void setSessionManager(SessionManager sessionManager) {
+    this.sessionManager = sessionManager;
+  }
+
+  private OperationManager getOperationManager() {
+    return operationManager;
+  }
+
+  @Override
+  public void setOperationManager(OperationManager operationManager) {
+    this.operationManager = operationManager;
+  }
+
+  protected synchronized void acquire(boolean userAccess) {
+    // Need to make sure that the this HiveServer2's session's SessionState is
+    // stored in the thread local for the handler thread.
+    SessionState.setCurrentSessionState(sessionState);
+    if (userAccess) {
+      lastAccessTime = System.currentTimeMillis();
+    }
+  }
+
+  /**
+   * 1. We'll remove the ThreadLocal SessionState as this thread might now serve
+   * other requests.
+   * 2. We'll cache the ThreadLocal RawStore object for this background thread for an orderly cleanup
+   * when this thread is garbage collected later.
+   * @see org.apache.hive.service.server.ThreadWithGarbageCleanup#finalize()
+   */
+  protected synchronized void release(boolean userAccess) {
+    SessionState.detachSession();
+    if (ThreadWithGarbageCleanup.currentThread() instanceof ThreadWithGarbageCleanup) {
+      ThreadWithGarbageCleanup currentThread =
+          (ThreadWithGarbageCleanup) ThreadWithGarbageCleanup.currentThread();
+      currentThread.cacheThreadLocalRawStore();
+    }
+    if (userAccess) {
+      lastAccessTime = System.currentTimeMillis();
+    }
+    if (opHandleSet.isEmpty()) {
+      lastIdleTime = System.currentTimeMillis();
+    } else {
+      lastIdleTime = 0;
+    }
+  }
+
+  @Override
+  public SessionHandle getSessionHandle() {
+    return sessionHandle;
+  }
+
+  @Override
+  public String getUsername() {
+    return username;
+  }
+
+  @Override
+  public String getPassword() {
+    return password;
+  }
+
+  @Override
+  public HiveConf getHiveConf() {
+    hiveConf.setVar(HiveConf.ConfVars.HIVEFETCHOUTPUTSERDE, FETCH_WORK_SERDE_CLASS);
+    return hiveConf;
+  }
+
+  @Override
+  public IMetaStoreClient getMetaStoreClient() throws HiveSQLException {
+    try {
+      return Hive.get(getHiveConf()).getMSC();
+    } catch (HiveException e) {
+      throw new HiveSQLException("Failed to get metastore connection", e);
+    } catch (MetaException e) {
+      throw new HiveSQLException("Failed to get metastore connection", e);
+    }
+  }
+
+  @Override
+  public GetInfoValue getInfo(GetInfoType getInfoType)
+      throws HiveSQLException {
+    acquire(true);
+    try {
+      switch (getInfoType) {
+      case CLI_SERVER_NAME:
+        return new GetInfoValue("Hive");
+      case CLI_DBMS_NAME:
+        return new GetInfoValue("Apache Hive");
+      case CLI_DBMS_VER:
+        return new GetInfoValue(HiveVersionInfo.getVersion());
+      case CLI_MAX_COLUMN_NAME_LEN:
+        return new GetInfoValue(128);
+      case CLI_MAX_SCHEMA_NAME_LEN:
+        return new GetInfoValue(128);
+      case CLI_MAX_TABLE_NAME_LEN:
+        return new GetInfoValue(128);
+      case CLI_TXN_CAPABLE:
+      default:
+        throw new HiveSQLException("Unrecognized GetInfoType value: " + getInfoType.toString());
+      }
+    } finally {
+      release(true);
+    }
+  }
+
+  @Override
+  public OperationHandle executeStatement(String statement, Map<String, String> confOverlay)
+      throws HiveSQLException {
+    return executeStatementInternal(statement, confOverlay, false, 0);
+  }
+
+  @Override
+  public OperationHandle executeStatement(String statement, Map<String, String> confOverlay,
+      long queryTimeout) throws HiveSQLException {
+    return executeStatementInternal(statement, confOverlay, false, queryTimeout);
+  }
+
+  @Override
+  public OperationHandle executeStatementAsync(String statement, Map<String, String> confOverlay)
+      throws HiveSQLException {
+    return executeStatementInternal(statement, confOverlay, true, 0);
+  }
+
+  @Override
+  public OperationHandle executeStatementAsync(String statement, Map<String, String> confOverlay,
+      long queryTimeout) throws HiveSQLException {
+    return executeStatementInternal(statement, confOverlay, true, queryTimeout);
+  }
+
+  private OperationHandle executeStatementInternal(String statement,
+      Map<String, String> confOverlay, boolean runAsync, long queryTimeout) throws HiveSQLException {
+    acquire(true);
+
+    OperationManager operationManager = getOperationManager();
+    ExecuteStatementOperation operation = operationManager
+        .newExecuteStatementOperation(getSession(), statement, confOverlay, runAsync, queryTimeout);
+    OperationHandle opHandle = operation.getHandle();
+    try {
+      operation.run();
+      opHandleSet.add(opHandle);
+      return opHandle;
+    } catch (HiveSQLException e) {
+      // Referring to SQLOperation.java, there is no chance that a HiveSQLException throws and the asyn
+      // background operation submits to thread pool successfully at the same time. So, Cleanup
+      // opHandle directly when got HiveSQLException
+      operationManager.closeOperation(opHandle);
+      throw e;
+    } finally {
+      release(true);
+    }
+  }
+
+  @Override
+  public OperationHandle getTypeInfo()
+      throws HiveSQLException {
+    acquire(true);
+
+    OperationManager operationManager = getOperationManager();
+    GetTypeInfoOperation operation = operationManager.newGetTypeInfoOperation(getSession());
+    OperationHandle opHandle = operation.getHandle();
+    try {
+      operation.run();
+      opHandleSet.add(opHandle);
+      return opHandle;
+    } catch (HiveSQLException e) {
+      operationManager.closeOperation(opHandle);
+      throw e;
+    } finally {
+      release(true);
+    }
+  }
+
+  @Override
+  public OperationHandle getCatalogs()
+      throws HiveSQLException {
+    acquire(true);
+
+    OperationManager operationManager = getOperationManager();
+    GetCatalogsOperation operation = operationManager.newGetCatalogsOperation(getSession());
+    OperationHandle opHandle = operation.getHandle();
+    try {
+      operation.run();
+      opHandleSet.add(opHandle);
+      return opHandle;
+    } catch (HiveSQLException e) {
+      operationManager.closeOperation(opHandle);
+      throw e;
+    } finally {
+      release(true);
+    }
+  }
+
+  @Override
+  public OperationHandle getSchemas(String catalogName, String schemaName)
+      throws HiveSQLException {
+    acquire(true);
+
+    OperationManager operationManager = getOperationManager();
+    GetSchemasOperation operation =
+        operationManager.newGetSchemasOperation(getSession(), catalogName, schemaName);
+    OperationHandle opHandle = operation.getHandle();
+    try {
+      operation.run();
+      opHandleSet.add(opHandle);
+      return opHandle;
+    } catch (HiveSQLException e) {
+      operationManager.closeOperation(opHandle);
+      throw e;
+    } finally {
+      release(true);
+    }
+  }
+
+  @Override
+  public OperationHandle getTables(String catalogName, String schemaName, String tableName,
+      List<String> tableTypes)
+          throws HiveSQLException {
+    acquire(true);
+
+    OperationManager operationManager = getOperationManager();
+    MetadataOperation operation =
+        operationManager.newGetTablesOperation(getSession(), catalogName, schemaName, tableName, tableTypes);
+    OperationHandle opHandle = operation.getHandle();
+    try {
+      operation.run();
+      opHandleSet.add(opHandle);
+      return opHandle;
+    } catch (HiveSQLException e) {
+      operationManager.closeOperation(opHandle);
+      throw e;
+    } finally {
+      release(true);
+    }
+  }
+
+  @Override
+  public OperationHandle getTableTypes()
+      throws HiveSQLException {
+    acquire(true);
+
+    OperationManager operationManager = getOperationManager();
+    GetTableTypesOperation operation = operationManager.newGetTableTypesOperation(getSession());
+    OperationHandle opHandle = operation.getHandle();
+    try {
+      operation.run();
+      opHandleSet.add(opHandle);
+      return opHandle;
+    } catch (HiveSQLException e) {
+      operationManager.closeOperation(opHandle);
+      throw e;
+    } finally {
+      release(true);
+    }
+  }
+
+  @Override
+  public OperationHandle getColumns(String catalogName, String schemaName,
+      String tableName, String columnName)  throws HiveSQLException {
+    acquire(true);
+    String addedJars = Utilities.getResourceFiles(hiveConf, SessionState.ResourceType.JAR);
+    if (StringUtils.isNotBlank(addedJars)) {
+       IMetaStoreClient metastoreClient = getSession().getMetaStoreClient();
+       metastoreClient.setHiveAddedJars(addedJars);
+    }
+    OperationManager operationManager = getOperationManager();
+    GetColumnsOperation operation = operationManager.newGetColumnsOperation(getSession(),
+        catalogName, schemaName, tableName, columnName);
+    OperationHandle opHandle = operation.getHandle();
+    try {
+      operation.run();
+      opHandleSet.add(opHandle);
+      return opHandle;
+    } catch (HiveSQLException e) {
+      operationManager.closeOperation(opHandle);
+      throw e;
+    } finally {
+      release(true);
+    }
+  }
+
+  @Override
+  public OperationHandle getFunctions(String catalogName, String schemaName, String functionName)
+      throws HiveSQLException {
+    acquire(true);
+
+    OperationManager operationManager = getOperationManager();
+    GetFunctionsOperation operation = operationManager
+        .newGetFunctionsOperation(getSession(), catalogName, schemaName, functionName);
+    OperationHandle opHandle = operation.getHandle();
+    try {
+      operation.run();
+      opHandleSet.add(opHandle);
+      return opHandle;
+    } catch (HiveSQLException e) {
+      operationManager.closeOperation(opHandle);
+      throw e;
+    } finally {
+      release(true);
+    }
+  }
+
+  @Override
+  public void close() throws HiveSQLException {
+    try {
+      acquire(true);
+      // Iterate through the opHandles and close their operations
+      for (OperationHandle opHandle : opHandleSet) {
+        try {
+          operationManager.closeOperation(opHandle);
+        } catch (Exception e) {
+          LOG.warn("Exception is thrown closing operation " + opHandle, e);
+        }
+      }
+      opHandleSet.clear();
+      // Cleanup session log directory.
+      cleanupSessionLogDir();
+      // Cleanup pipeout file.
+      cleanupPipeoutFile();
+      HiveHistory hiveHist = sessionState.getHiveHistory();
+      if (null != hiveHist) {
+        hiveHist.closeStream();
+      }
+      try {
+        sessionState.close();
+      } finally {
+        sessionState = null;
+      }
+    } catch (IOException ioe) {
+      throw new HiveSQLException("Failure to close", ioe);
+    } finally {
+      if (sessionState != null) {
+        try {
+          sessionState.close();
+        } catch (Throwable t) {
+          LOG.warn("Error closing session", t);
+        }
+        sessionState = null;
+      }
+      release(true);
+    }
+  }
+
+  private void cleanupPipeoutFile() {
+    String lScratchDir = hiveConf.getVar(ConfVars.LOCALSCRATCHDIR);
+    String sessionID = hiveConf.getVar(ConfVars.HIVESESSIONID);
+
+    File[] fileAry = new File(lScratchDir).listFiles(
+            (dir, name) -> name.startsWith(sessionID) && name.endsWith(".pipeout"));
+
+    if (fileAry == null) {
+      LOG.error("Unable to access pipeout files in " + lScratchDir);
+    } else {
+      for (File file : fileAry) {
+        try {
+          FileUtils.forceDelete(file);
+        } catch (Exception e) {
+          LOG.error("Failed to cleanup pipeout file: " + file, e);
+        }
+      }
+    }
+  }
+
+  private void cleanupSessionLogDir() {
+    if (isOperationLogEnabled) {
+      try {
+        FileUtils.forceDelete(sessionLogDir);
+      } catch (Exception e) {
+        LOG.error("Failed to cleanup session log dir: " + sessionHandle, e);
+      }
+    }
+  }
+
+  @Override
+  public SessionState getSessionState() {
+    return sessionState;
+  }
+
+  @Override
+  public String getUserName() {
+    return username;
+  }
+
+  @Override
+  public void setUserName(String userName) {
+    this.username = userName;
+  }
+
+  @Override
+  public long getLastAccessTime() {
+    return lastAccessTime;
+  }
+
+  @Override
+  public void closeExpiredOperations() {
+    OperationHandle[] handles = opHandleSet.toArray(new OperationHandle[opHandleSet.size()]);
+    if (handles.length > 0) {
+      List<Operation> operations = operationManager.removeExpiredOperations(handles);
+      if (!operations.isEmpty()) {
+        closeTimedOutOperations(operations);
+      }
+    }
+  }
+
+  @Override
+  public long getNoOperationTime() {
+    return lastIdleTime > 0 ? System.currentTimeMillis() - lastIdleTime : 0;
+  }
+
+  private void closeTimedOutOperations(List<Operation> operations) {
+    acquire(false);
+    try {
+      for (Operation operation : operations) {
+        opHandleSet.remove(operation.getHandle());
+        try {
+          operation.close();
+        } catch (Exception e) {
+          LOG.warn("Exception is thrown closing timed-out operation " + operation.getHandle(), e);
+        }
+      }
+    } finally {
+      release(false);
+    }
+  }
+
+  @Override
+  public void cancelOperation(OperationHandle opHandle) throws HiveSQLException {
+    acquire(true);
+    try {
+      sessionManager.getOperationManager().cancelOperation(opHandle);
+    } finally {
+      release(true);
+    }
+  }
+
+  @Override
+  public void closeOperation(OperationHandle opHandle) throws HiveSQLException {
+    acquire(true);
+    try {
+      operationManager.closeOperation(opHandle);
+      opHandleSet.remove(opHandle);
+    } finally {
+      release(true);
+    }
+  }
+
+  @Override
+  public TableSchema getResultSetMetadata(OperationHandle opHandle) throws HiveSQLException {
+    acquire(true);
+    try {
+      return sessionManager.getOperationManager().getOperationResultSetSchema(opHandle);
+    } finally {
+      release(true);
+    }
+  }
+
+  @Override
+  public RowSet fetchResults(OperationHandle opHandle, FetchOrientation orientation,
+      long maxRows, FetchType fetchType) throws HiveSQLException {
+    acquire(true);
+    try {
+      if (fetchType == FetchType.QUERY_OUTPUT) {
+        return operationManager.getOperationNextRowSet(opHandle, orientation, maxRows);
+      }
+      return operationManager.getOperationLogRowSet(opHandle, orientation, maxRows);
+    } finally {
+      release(true);
+    }
+  }
+
+  protected HiveSession getSession() {
+    return this;
+  }
+
+  @Override
+  public String getIpAddress() {
+    return ipAddress;
+  }
+
+  @Override
+  public void setIpAddress(String ipAddress) {
+    this.ipAddress = ipAddress;
+  }
+
+  @Override
+  public String getDelegationToken(HiveAuthFactory authFactory, String owner, String renewer)
+      throws HiveSQLException {
+    HiveAuthFactory.verifyProxyAccess(getUsername(), owner, getIpAddress(), getHiveConf());
+    return authFactory.getDelegationToken(owner, renewer, getIpAddress());
+  }
+
+  @Override
+  public void cancelDelegationToken(HiveAuthFactory authFactory, String tokenStr)
+      throws HiveSQLException {
+    HiveAuthFactory.verifyProxyAccess(getUsername(), getUserFromToken(authFactory, tokenStr),
+        getIpAddress(), getHiveConf());
+    authFactory.cancelDelegationToken(tokenStr);
+  }
+
+  @Override
+  public void renewDelegationToken(HiveAuthFactory authFactory, String tokenStr)
+      throws HiveSQLException {
+    HiveAuthFactory.verifyProxyAccess(getUsername(), getUserFromToken(authFactory, tokenStr),
+        getIpAddress(), getHiveConf());
+    authFactory.renewDelegationToken(tokenStr);
+  }
+
+  // extract the real user from the given token string
+  private String getUserFromToken(HiveAuthFactory authFactory, String tokenStr) throws HiveSQLException {
+    return authFactory.getUserFromToken(tokenStr);
+  }
+
+  @Override
+  public OperationHandle getPrimaryKeys(String catalog, String schema,
+      String table) throws HiveSQLException {
+    acquire(true);
+
+    OperationManager operationManager = getOperationManager();
+    GetPrimaryKeysOperation operation = operationManager
+        .newGetPrimaryKeysOperation(getSession(), catalog, schema, table);
+    OperationHandle opHandle = operation.getHandle();
+    try {
+      operation.run();
+      opHandleSet.add(opHandle);
+      return opHandle;
+    } catch (HiveSQLException e) {
+      operationManager.closeOperation(opHandle);
+      throw e;
+    } finally {
+      release(true);
+    }
+  }
+
+  @Override
+  public OperationHandle getCrossReference(String primaryCatalog,
+      String primarySchema, String primaryTable, String foreignCatalog,
+      String foreignSchema, String foreignTable) throws HiveSQLException {
+    acquire(true);
+
+    OperationManager operationManager = getOperationManager();
+    GetCrossReferenceOperation operation = operationManager
+      .newGetCrossReferenceOperation(getSession(), primaryCatalog,
+         primarySchema, primaryTable, foreignCatalog,
+         foreignSchema, foreignTable);
+    OperationHandle opHandle = operation.getHandle();
+    try {
+      operation.run();
+      opHandleSet.add(opHandle);
+      return opHandle;
+    } catch (HiveSQLException e) {
+      operationManager.closeOperation(opHandle);
+      throw e;
+    } finally {
+      release(true);
+    }
+  }
+}
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java
diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/SessionManager.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/SessionManager.java
new file mode 100644
index 0000000000000..5a381d170b4f9
--- /dev/null
+++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/SessionManager.java
@@ -0,0 +1,377 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.service.cli.session;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hive.service.CompositeService;
+import org.apache.hive.service.cli.HiveSQLException;
+import org.apache.hive.service.cli.SessionHandle;
+import org.apache.hive.service.cli.operation.OperationManager;
+import org.apache.hive.service.rpc.thrift.TProtocolVersion;
+import org.apache.hive.service.server.HiveServer2;
+import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * SessionManager.
+ *
+ */
+public class SessionManager extends CompositeService {
+
+  private static final Logger LOG = LoggerFactory.getLogger(SessionManager.class);
+  public static final String HIVERCFILE = ".hiverc";
+  private HiveConf hiveConf;
+  private final Map<SessionHandle, HiveSession> handleToSession =
+      new ConcurrentHashMap<SessionHandle, HiveSession>();
+  private final OperationManager operationManager = new OperationManager();
+  private ThreadPoolExecutor backgroundOperationPool;
+  private boolean isOperationLogEnabled;
+  private File operationLogRootDir;
+
+  private long checkInterval;
+  private long sessionTimeout;
+  private boolean checkOperation;
+
+  private volatile boolean shutdown;
+  // The HiveServer2 instance running this service
+  private final HiveServer2 hiveServer2;
+
+  public SessionManager(HiveServer2 hiveServer2) {
+    super(SessionManager.class.getSimpleName());
+    this.hiveServer2 = hiveServer2;
+  }
+
+  @Override
+  public synchronized void init(HiveConf hiveConf) {
+    this.hiveConf = hiveConf;
+    //Create operation log root directory, if operation logging is enabled
+    if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_ENABLED)) {
+      initOperationLogRootDir();
+    }
+    createBackgroundOperationPool();
+    addService(operationManager);
+    super.init(hiveConf);
+  }
+
+  private void createBackgroundOperationPool() {
+    int poolSize = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_ASYNC_EXEC_THREADS);
+    LOG.info("HiveServer2: Background operation thread pool size: " + poolSize);
+    int poolQueueSize = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_ASYNC_EXEC_WAIT_QUEUE_SIZE);
+    LOG.info("HiveServer2: Background operation thread wait queue size: " + poolQueueSize);
+    long keepAliveTime = HiveConf.getTimeVar(
+        hiveConf, ConfVars.HIVE_SERVER2_ASYNC_EXEC_KEEPALIVE_TIME, TimeUnit.SECONDS);
+    LOG.info(
+        "HiveServer2: Background operation thread keepalive time: " + keepAliveTime + " seconds");
+
+    // Create a thread pool with #poolSize threads
+    // Threads terminate when they are idle for more than the keepAliveTime
+    // A bounded blocking queue is used to queue incoming operations, if #operations > poolSize
+    String threadPoolName = "HiveServer2-Background-Pool";
+    backgroundOperationPool = new ThreadPoolExecutor(poolSize, poolSize,
+        keepAliveTime, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(poolQueueSize),
+        new ThreadFactoryWithGarbageCleanup(threadPoolName));
+    backgroundOperationPool.allowCoreThreadTimeOut(true);
+
+    checkInterval = HiveConf.getTimeVar(
+        hiveConf, ConfVars.HIVE_SERVER2_SESSION_CHECK_INTERVAL, TimeUnit.MILLISECONDS);
+    sessionTimeout = HiveConf.getTimeVar(
+        hiveConf, ConfVars.HIVE_SERVER2_IDLE_SESSION_TIMEOUT, TimeUnit.MILLISECONDS);
+    checkOperation = HiveConf.getBoolVar(hiveConf,
+        ConfVars.HIVE_SERVER2_IDLE_SESSION_CHECK_OPERATION);
+  }
+
+  private void initOperationLogRootDir() {
+    operationLogRootDir = new File(
+        hiveConf.getVar(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LOG_LOCATION));
+    isOperationLogEnabled = true;
+
+    if (operationLogRootDir.exists() && !operationLogRootDir.isDirectory()) {
+      LOG.warn("The operation log root directory exists, but it is not a directory: " +
+          operationLogRootDir.getAbsolutePath());
+      isOperationLogEnabled = false;
+    }
+
+    if (!operationLogRootDir.exists()) {
+      if (!operationLogRootDir.mkdirs()) {
+        LOG.warn("Unable to create operation log root directory: " +
+            operationLogRootDir.getAbsolutePath());
+        isOperationLogEnabled = false;
+      }
+    }
+
+    if (isOperationLogEnabled) {
+      LOG.info("Operation log root directory is created: " + operationLogRootDir.getAbsolutePath());
+      try {
+        FileUtils.forceDeleteOnExit(operationLogRootDir);
+      } catch (IOException e) {
+        LOG.warn("Failed to schedule cleanup HS2 operation logging root dir: " +
+            operationLogRootDir.getAbsolutePath(), e);
+      }
+    }
+  }
+
+  @Override
+  public synchronized void start() {
+    super.start();
+    if (checkInterval > 0) {
+      startTimeoutChecker();
+    }
+  }
+
+  private final Object timeoutCheckerLock = new Object();
+
+  private void startTimeoutChecker() {
+    final long interval = Math.max(checkInterval, 3000L);  // minimum 3 seconds
+    final Runnable timeoutChecker = new Runnable() {
+      @Override
+      public void run() {
+        sleepFor(interval);
+        while (!shutdown) {
+          long current = System.currentTimeMillis();
+          for (HiveSession session : new ArrayList<HiveSession>(handleToSession.values())) {
+            if (shutdown) {
+              break;
+            }
+            if (sessionTimeout > 0 && session.getLastAccessTime() + sessionTimeout <= current
+                && (!checkOperation || session.getNoOperationTime() > sessionTimeout)) {
+              SessionHandle handle = session.getSessionHandle();
+              LOG.warn("Session " + handle + " is Timed-out (last access : " +
+                  new Date(session.getLastAccessTime()) + ") and will be closed");
+              try {
+                closeSession(handle);
+              } catch (HiveSQLException e) {
+                LOG.warn("Exception is thrown closing session " + handle, e);
+              }
+            } else {
+              session.closeExpiredOperations();
+            }
+          }
+          sleepFor(interval);
+        }
+      }
+
+      private void sleepFor(long interval) {
+        synchronized (timeoutCheckerLock) {
+          try {
+            timeoutCheckerLock.wait(interval);
+          } catch (InterruptedException e) {
+            // Ignore, and break.
+          }
+        }
+      }
+    };
+    backgroundOperationPool.execute(timeoutChecker);
+  }
+
+  private void shutdownTimeoutChecker() {
+    shutdown = true;
+    synchronized (timeoutCheckerLock) {
+      timeoutCheckerLock.notify();
+    }
+  }
+
+  @Override
+  public synchronized void stop() {
+    super.stop();
+    shutdownTimeoutChecker();
+    if (backgroundOperationPool != null) {
+      backgroundOperationPool.shutdown();
+      long timeout = hiveConf.getTimeVar(
+          ConfVars.HIVE_SERVER2_ASYNC_EXEC_SHUTDOWN_TIMEOUT, TimeUnit.SECONDS);
+      try {
+        backgroundOperationPool.awaitTermination(timeout, TimeUnit.SECONDS);
+      } catch (InterruptedException e) {
+        LOG.warn("HIVE_SERVER2_ASYNC_EXEC_SHUTDOWN_TIMEOUT = " + timeout +
+            " seconds has been exceeded. RUNNING background operations will be shut down", e);
+      }
+      backgroundOperationPool = null;
+    }
+    cleanupLoggingRootDir();
+  }
+
+  private void cleanupLoggingRootDir() {
+    if (isOperationLogEnabled) {
+      try {
+        FileUtils.forceDelete(operationLogRootDir);
+      } catch (Exception e) {
+        LOG.warn("Failed to cleanup root dir of HS2 logging: " + operationLogRootDir
+            .getAbsolutePath(), e);
+      }
+    }
+  }
+
+  public SessionHandle openSession(TProtocolVersion protocol, String username, String password, String ipAddress,
+      Map<String, String> sessionConf) throws HiveSQLException {
+    return openSession(protocol, username, password, ipAddress, sessionConf, false, null);
+  }
+
+  /**
+   * Opens a new session and creates a session handle.
+   * The username passed to this method is the effective username.
+   * If withImpersonation is true (==doAs true) we wrap all the calls in HiveSession
+   * within a UGI.doAs, where UGI corresponds to the effective user.
+   *
+   * Please see {@code org.apache.hive.service.cli.thrift.ThriftCLIService.getUserName()} for
+   * more details.
+   *
+   * @param protocol
+   * @param username
+   * @param password
+   * @param ipAddress
+   * @param sessionConf
+   * @param withImpersonation
+   * @param delegationToken
+   * @return
+   * @throws HiveSQLException
+   */
+  public SessionHandle openSession(TProtocolVersion protocol, String username, String password, String ipAddress,
+      Map<String, String> sessionConf, boolean withImpersonation, String delegationToken)
+          throws HiveSQLException {
+    HiveSession session;
+    // If doAs is set to true for HiveServer2, we will create a proxy object for the session impl.
+    // Within the proxy object, we wrap the method call in a UserGroupInformation#doAs
+    if (withImpersonation) {
+      HiveSessionImplwithUGI sessionWithUGI = new HiveSessionImplwithUGI(protocol, username, password,
+          hiveConf, ipAddress, delegationToken);
+      session = HiveSessionProxy.getProxy(sessionWithUGI, sessionWithUGI.getSessionUgi());
+      sessionWithUGI.setProxySession(session);
+    } else {
+      session = new HiveSessionImpl(protocol, username, password, hiveConf, ipAddress);
+    }
+    session.setSessionManager(this);
+    session.setOperationManager(operationManager);
+    try {
+      session.open(sessionConf);
+    } catch (Exception e) {
+      try {
+        session.close();
+      } catch (Throwable t) {
+        LOG.warn("Error closing session", t);
+      }
+      session = null;
+      throw new HiveSQLException("Failed to open new session: " + e, e);
+    }
+    if (isOperationLogEnabled) {
+      session.setOperationLogSessionDir(operationLogRootDir);
+    }
+    handleToSession.put(session.getSessionHandle(), session);
+    return session.getSessionHandle();
+  }
+
+  public void closeSession(SessionHandle sessionHandle) throws HiveSQLException {
+    HiveSession session = handleToSession.remove(sessionHandle);
+    if (session == null) {
+      throw new HiveSQLException("Session does not exist!");
+    }
+    session.close();
+  }
+
+  public HiveSession getSession(SessionHandle sessionHandle) throws HiveSQLException {
+    HiveSession session = handleToSession.get(sessionHandle);
+    if (session == null) {
+      throw new HiveSQLException("Invalid SessionHandle: " + sessionHandle);
+    }
+    return session;
+  }
+
+  public OperationManager getOperationManager() {
+    return operationManager;
+  }
+
+  private static ThreadLocal<String> threadLocalIpAddress = new ThreadLocal<String>() {
+    @Override
+    protected synchronized String initialValue() {
+      return null;
+    }
+  };
+
+  public static void setIpAddress(String ipAddress) {
+    threadLocalIpAddress.set(ipAddress);
+  }
+
+  public static void clearIpAddress() {
+    threadLocalIpAddress.remove();
+  }
+
+  public static String getIpAddress() {
+    return threadLocalIpAddress.get();
+  }
+
+  private static ThreadLocal<String> threadLocalUserName = new ThreadLocal<String>(){
+    @Override
+    protected synchronized String initialValue() {
+      return null;
+    }
+  };
+
+  public static void setUserName(String userName) {
+    threadLocalUserName.set(userName);
+  }
+
+  public static void clearUserName() {
+    threadLocalUserName.remove();
+  }
+
+  public static String getUserName() {
+    return threadLocalUserName.get();
+  }
+
+  private static ThreadLocal<String> threadLocalProxyUserName = new ThreadLocal<String>(){
+    @Override
+    protected synchronized String initialValue() {
+      return null;
+    }
+  };
+
+  public static void setProxyUserName(String userName) {
+    LOG.debug("setting proxy user name based on query param to: " + userName);
+    threadLocalProxyUserName.set(userName);
+  }
+
+  public static String getProxyUserName() {
+    return threadLocalProxyUserName.get();
+  }
+
+  public static void clearProxyUserName() {
+    threadLocalProxyUserName.remove();
+  }
+
+  public Future<?> submitBackgroundOperation(Runnable r) {
+    return backgroundOperationPool.submit(r);
+  }
+
+  public int getOpenSessionCount() {
+    return handleToSession.size();
+  }
+}
+
diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
new file mode 100644
index 0000000000000..8085c8d12e6b2
--- /dev/null
+++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.service.cli.thrift;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.hive.common.auth.HiveAuthUtils;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hive.service.ServiceException;
+import org.apache.hive.service.auth.HiveAuthFactory;
+import org.apache.hive.service.cli.CLIService;
+import org.apache.hive.service.cli.HiveSQLException;
+import org.apache.hive.service.rpc.thrift.TGetQueryIdReq;
+import org.apache.hive.service.rpc.thrift.TGetQueryIdResp;
+import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup;
+import org.apache.thrift.TException;
+import org.apache.thrift.TProcessorFactory;
+import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.server.TServer;
+import org.apache.thrift.server.TThreadPoolServer;
+import org.apache.thrift.transport.TServerSocket;
+import org.apache.thrift.transport.TTransportFactory;
+
+
+public class ThriftBinaryCLIService extends ThriftCLIService {
+
+  protected TServer server;
+
+  public ThriftBinaryCLIService(CLIService cliService) {
+    super(cliService, ThriftBinaryCLIService.class.getSimpleName());
+  }
+
+  @Override
+  protected void initializeServer() {
+    try {
+      // Server thread pool
+      String threadPoolName = "HiveServer2-Handler-Pool";
+      ExecutorService executorService = new ThreadPoolExecutor(minWorkerThreads, maxWorkerThreads,
+          workerKeepAliveTime, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
+          new ThreadFactoryWithGarbageCleanup(threadPoolName));
+
+      // Thrift configs
+      hiveAuthFactory = new HiveAuthFactory(hiveConf);
+      TTransportFactory transportFactory = hiveAuthFactory.getAuthTransFactory();
+      TProcessorFactory processorFactory = hiveAuthFactory.getAuthProcFactory(this);
+      TServerSocket serverSocket = null;
+      List<String> sslVersionBlacklist = new ArrayList<String>();
+      for (String sslVersion : hiveConf.getVar(ConfVars.HIVE_SSL_PROTOCOL_BLACKLIST).split(",")) {
+        sslVersionBlacklist.add(sslVersion);
+      }
+      if (!hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_USE_SSL)) {
+        serverSocket = HiveAuthUtils.getServerSocket(hiveHost, portNum);
+      } else {
+        String keyStorePath = hiveConf.getVar(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH).trim();
+        if (keyStorePath.isEmpty()) {
+          throw new IllegalArgumentException(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname
+              + " Not configured for SSL connection");
+        }
+        String keyStorePassword = ShimLoader.getHadoopShims().getPassword(hiveConf,
+            HiveConf.ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname);
+        serverSocket = HiveAuthUtils.getServerSSLSocket(hiveHost, portNum, keyStorePath,
+            keyStorePassword, sslVersionBlacklist);
+      }
+
+      // In case HIVE_SERVER2_THRIFT_PORT or hive.server2.thrift.port is configured with 0 which
+      // represents any free port, we should set it to the actual one
+      portNum = serverSocket.getServerSocket().getLocalPort();
+
+      // Server args
+      int maxMessageSize = hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE);
+      int requestTimeout = (int) hiveConf.getTimeVar(
+          HiveConf.ConfVars.HIVE_SERVER2_THRIFT_LOGIN_TIMEOUT, TimeUnit.SECONDS);
+      int beBackoffSlotLength = (int) hiveConf.getTimeVar(
+          HiveConf.ConfVars.HIVE_SERVER2_THRIFT_LOGIN_BEBACKOFF_SLOT_LENGTH, TimeUnit.MILLISECONDS);
+      TThreadPoolServer.Args sargs = new TThreadPoolServer.Args(serverSocket)
+          .processorFactory(processorFactory).transportFactory(transportFactory)
+          .protocolFactory(new TBinaryProtocol.Factory())
+          .inputProtocolFactory(new TBinaryProtocol.Factory(true, true, maxMessageSize, maxMessageSize))
+          .requestTimeout(requestTimeout).requestTimeoutUnit(TimeUnit.SECONDS)
+          .beBackoffSlotLength(beBackoffSlotLength).beBackoffSlotLengthUnit(TimeUnit.MILLISECONDS)
+          .executorService(executorService);
+
+      // TCP Server
+      server = new TThreadPoolServer(sargs);
+      server.setServerEventHandler(serverEventHandler);
+      String msg = "Starting " + ThriftBinaryCLIService.class.getSimpleName() + " on port "
+          + portNum + " with " + minWorkerThreads + "..." + maxWorkerThreads + " worker threads";
+      LOG.info(msg);
+    } catch (Exception t) {
+      throw new ServiceException("Error initializing " + getName(), t);
+    }
+  }
+
+  @Override
+  protected void stopServer() {
+    server.stop();
+    server = null;
+    LOG.info("Thrift server has stopped");
+  }
+
+  @Override
+  public TGetQueryIdResp GetQueryId(TGetQueryIdReq req) throws TException {
+    try {
+      return new TGetQueryIdResp(cliService.getQueryId(req.getOperationHandle()));
+    } catch (HiveSQLException e) {
+      throw new TException(e);
+    }
+  }
+
+  @Override
+  public void run() {
+    try {
+      server.serve();
+    } catch (Throwable t) {
+      LOG.error(
+          "Error starting HiveServer2: could not start "
+              + ThriftBinaryCLIService.class.getSimpleName(), t);
+      System.exit(-1);
+    }
+  }
+
+}
diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
new file mode 100644
index 0000000000000..a65951c8aec2e
--- /dev/null
+++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
@@ -0,0 +1,756 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.service.cli.thrift;
+
+import javax.security.auth.login.LoginException;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hive.service.AbstractService;
+import org.apache.hive.service.ServiceException;
+import org.apache.hive.service.ServiceUtils;
+import org.apache.hive.service.auth.HiveAuthFactory;
+import org.apache.hive.service.auth.TSetIpAddressProcessor;
+import org.apache.hive.service.cli.*;
+import org.apache.hive.service.cli.session.SessionManager;
+import org.apache.hive.service.rpc.thrift.*;
+import org.apache.hive.service.server.HiveServer2;
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TProtocol;
+import org.apache.thrift.server.ServerContext;
+import org.apache.thrift.server.TServerEventHandler;
+import org.apache.thrift.transport.TTransport;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * ThriftCLIService.
+ *
+ */
+public abstract class ThriftCLIService extends AbstractService implements TCLIService.Iface, Runnable {
+
+  public static final Logger LOG = LoggerFactory.getLogger(ThriftCLIService.class.getName());
+
+  protected CLIService cliService;
+  private static final TStatus OK_STATUS = new TStatus(TStatusCode.SUCCESS_STATUS);
+  protected static HiveAuthFactory hiveAuthFactory;
+
+  protected int portNum;
+  protected InetAddress serverIPAddress;
+  protected String hiveHost;
+  private Thread serverThread = null;
+
+  private boolean isStarted = false;
+  protected boolean isEmbedded = false;
+
+  protected HiveConf hiveConf;
+
+  protected int minWorkerThreads;
+  protected int maxWorkerThreads;
+  protected long workerKeepAliveTime;
+
+  protected TServerEventHandler serverEventHandler;
+  protected ThreadLocal<ServerContext> currentServerContext;
+
+  static class ThriftCLIServerContext implements ServerContext {
+    private SessionHandle sessionHandle = null;
+
+    public void setSessionHandle(SessionHandle sessionHandle) {
+      this.sessionHandle = sessionHandle;
+    }
+
+    public SessionHandle getSessionHandle() {
+      return sessionHandle;
+    }
+  }
+
+  public ThriftCLIService(CLIService service, String serviceName) {
+    super(serviceName);
+    this.cliService = service;
+    currentServerContext = new ThreadLocal<ServerContext>();
+    serverEventHandler = new TServerEventHandler() {
+      @Override
+      public ServerContext createContext(
+          TProtocol input, TProtocol output) {
+        return new ThriftCLIServerContext();
+      }
+
+      @Override
+      public void deleteContext(ServerContext serverContext,
+          TProtocol input, TProtocol output) {
+        ThriftCLIServerContext context = (ThriftCLIServerContext)serverContext;
+        SessionHandle sessionHandle = context.getSessionHandle();
+        if (sessionHandle != null) {
+          LOG.info("Session disconnected without closing properly, close it now");
+          try {
+            cliService.closeSession(sessionHandle);
+          } catch (HiveSQLException e) {
+            LOG.warn("Failed to close session: " + e, e);
+          }
+        }
+      }
+
+      @Override
+      public void preServe() {
+      }
+
+      @Override
+      public void processContext(ServerContext serverContext,
+          TTransport input, TTransport output) {
+        currentServerContext.set(serverContext);
+      }
+    };
+  }
+
+  @Override
+  public synchronized void init(HiveConf hiveConf) {
+    this.hiveConf = hiveConf;
+    // Initialize common server configs needed in both binary & http modes
+    String portString;
+    hiveHost = System.getenv("HIVE_SERVER2_THRIFT_BIND_HOST");
+    if (hiveHost == null) {
+      hiveHost = hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST);
+    }
+    try {
+      if (hiveHost != null && !hiveHost.isEmpty()) {
+        serverIPAddress = InetAddress.getByName(hiveHost);
+      } else {
+        serverIPAddress = InetAddress.getLocalHost();
+      }
+    } catch (UnknownHostException e) {
+      throw new ServiceException(e);
+    }
+    // HTTP mode
+    if (HiveServer2.isHTTPTransportMode(hiveConf)) {
+      workerKeepAliveTime =
+          hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_WORKER_KEEPALIVE_TIME,
+              TimeUnit.SECONDS);
+      portString = System.getenv("HIVE_SERVER2_THRIFT_HTTP_PORT");
+      if (portString != null) {
+        portNum = Integer.valueOf(portString);
+      } else {
+        portNum = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT);
+      }
+    }
+    // Binary mode
+    else {
+      workerKeepAliveTime =
+          hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_THRIFT_WORKER_KEEPALIVE_TIME, TimeUnit.SECONDS);
+      portString = System.getenv("HIVE_SERVER2_THRIFT_PORT");
+      if (portString != null) {
+        portNum = Integer.valueOf(portString);
+      } else {
+        portNum = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT);
+      }
+    }
+    minWorkerThreads = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_MIN_WORKER_THREADS);
+    maxWorkerThreads = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_MAX_WORKER_THREADS);
+    super.init(hiveConf);
+  }
+
+  @Override
+  public synchronized void start() {
+    super.start();
+    if (!isStarted && !isEmbedded) {
+      initializeServer();
+      serverThread = new Thread(this);
+      serverThread.setName(getName());
+      serverThread.start();
+      isStarted = true;
+    }
+  }
+
+  protected abstract void stopServer();
+
+  @Override
+  public synchronized void stop() {
+    if (isStarted && !isEmbedded) {
+      if (serverThread != null) {
+        serverThread.interrupt();
+        serverThread = null;
+      }
+      stopServer();
+      isStarted = false;
+    }
+    super.stop();
+  }
+
+  public int getPortNumber() {
+    return portNum;
+  }
+
+  public InetAddress getServerIPAddress() {
+    return serverIPAddress;
+  }
+
+  @Override
+  public TGetDelegationTokenResp GetDelegationToken(TGetDelegationTokenReq req)
+      throws TException {
+    TGetDelegationTokenResp resp = new TGetDelegationTokenResp();
+    resp.setStatus(notSupportTokenErrorStatus());
+    return resp;
+  }
+
+  @Override
+  public TCancelDelegationTokenResp CancelDelegationToken(TCancelDelegationTokenReq req)
+      throws TException {
+    TCancelDelegationTokenResp resp = new TCancelDelegationTokenResp();
+    resp.setStatus(notSupportTokenErrorStatus());
+    return resp;
+  }
+
+  @Override
+  public TRenewDelegationTokenResp RenewDelegationToken(TRenewDelegationTokenReq req)
+      throws TException {
+    TRenewDelegationTokenResp resp = new TRenewDelegationTokenResp();
+    resp.setStatus(notSupportTokenErrorStatus());
+    return resp;
+  }
+
+  private TStatus notSupportTokenErrorStatus() {
+    TStatus errorStatus = new TStatus(TStatusCode.ERROR_STATUS);
+    errorStatus.setErrorMessage("Delegation token is not supported");
+    return errorStatus;
+  }
+
+  @Override
+  public TOpenSessionResp OpenSession(TOpenSessionReq req) throws TException {
+    LOG.info("Client protocol version: " + req.getClient_protocol());
+    TOpenSessionResp resp = new TOpenSessionResp();
+    try {
+      SessionHandle sessionHandle = getSessionHandle(req, resp);
+      resp.setSessionHandle(sessionHandle.toTSessionHandle());
+      // TODO: set real configuration map
+      resp.setConfiguration(new HashMap<String, String>());
+      resp.setStatus(OK_STATUS);
+      ThriftCLIServerContext context =
+        (ThriftCLIServerContext)currentServerContext.get();
+      if (context != null) {
+        context.setSessionHandle(sessionHandle);
+      }
+    } catch (Exception e) {
+      LOG.warn("Error opening session: ", e);
+      resp.setStatus(HiveSQLException.toTStatus(e));
+    }
+    return resp;
+  }
+
+  @Override
+  public TSetClientInfoResp SetClientInfo(TSetClientInfoReq req) throws TException {
+    // TODO: We don't do anything for now, just log this for debugging.
+    //       We may be able to make use of this later, e.g. for workload management.
+    if (req.isSetConfiguration()) {
+      StringBuilder sb = null;
+      for (Map.Entry<String, String> e : req.getConfiguration().entrySet()) {
+        if (sb == null) {
+          SessionHandle sh = new SessionHandle(req.getSessionHandle());
+          sb = new StringBuilder("Client information for ").append(sh).append(": ");
+        } else {
+          sb.append(", ");
+        }
+        sb.append(e.getKey()).append(" = ").append(e.getValue());
+      }
+      if (sb != null) {
+        LOG.info("{}", sb);
+      }
+    }
+    return new TSetClientInfoResp(OK_STATUS);
+  }
+
+  private String getIpAddress() {
+    String clientIpAddress;
+    // Http transport mode.
+    // We set the thread local ip address, in ThriftHttpServlet.
+    if (cliService.getHiveConf().getVar(
+        ConfVars.HIVE_SERVER2_TRANSPORT_MODE).equalsIgnoreCase("http")) {
+      clientIpAddress = SessionManager.getIpAddress();
+    }
+    else {
+      // Kerberos
+      if (isKerberosAuthMode()) {
+        clientIpAddress = hiveAuthFactory.getIpAddress();
+      }
+      // Except kerberos, NOSASL
+      else {
+        clientIpAddress = TSetIpAddressProcessor.getUserIpAddress();
+      }
+    }
+    LOG.debug("Client's IP Address: " + clientIpAddress);
+    return clientIpAddress;
+  }
+
+  /**
+   * Returns the effective username.
+   * 1. If hive.server2.allow.user.substitution = false: the username of the connecting user
+   * 2. If hive.server2.allow.user.substitution = true: the username of the end user,
+   * that the connecting user is trying to proxy for.
+   * This includes a check whether the connecting user is allowed to proxy for the end user.
+   * @param req
+   * @return
+   * @throws HiveSQLException
+   */
+  private String getUserName(TOpenSessionReq req) throws HiveSQLException {
+    String userName = null;
+    // Kerberos
+    if (isKerberosAuthMode()) {
+      userName = hiveAuthFactory.getRemoteUser();
+    }
+    // Except kerberos, NOSASL
+    if (userName == null) {
+      userName = TSetIpAddressProcessor.getUserName();
+    }
+    // Http transport mode.
+    // We set the thread local username, in ThriftHttpServlet.
+    if (cliService.getHiveConf().getVar(
+        ConfVars.HIVE_SERVER2_TRANSPORT_MODE).equalsIgnoreCase("http")) {
+      userName = SessionManager.getUserName();
+    }
+    if (userName == null) {
+      userName = req.getUsername();
+    }
+
+    userName = getShortName(userName);
+    String effectiveClientUser = getProxyUser(userName, req.getConfiguration(), getIpAddress());
+    LOG.debug("Client's username: " + effectiveClientUser);
+    return effectiveClientUser;
+  }
+
+  private String getShortName(String userName) {
+    String ret = null;
+    if (userName != null) {
+      int indexOfDomainMatch = ServiceUtils.indexOfDomainMatch(userName);
+      ret = (indexOfDomainMatch <= 0) ? userName :
+          userName.substring(0, indexOfDomainMatch);
+    }
+
+    return ret;
+  }
+
+  /**
+   * Create a session handle
+   * @param req
+   * @param res
+   * @return
+   * @throws HiveSQLException
+   * @throws LoginException
+   * @throws IOException
+   */
+  SessionHandle getSessionHandle(TOpenSessionReq req, TOpenSessionResp res)
+      throws HiveSQLException, LoginException, IOException {
+    String userName = getUserName(req);
+    String ipAddress = getIpAddress();
+    TProtocolVersion protocol = getMinVersion(CLIService.SERVER_VERSION,
+        req.getClient_protocol());
+    res.setServerProtocolVersion(protocol);
+    SessionHandle sessionHandle;
+    if (cliService.getHiveConf().getBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS) &&
+        (userName != null)) {
+      String delegationTokenStr = getDelegationToken(userName);
+      sessionHandle = cliService.openSessionWithImpersonation(protocol, userName,
+          req.getPassword(), ipAddress, req.getConfiguration(), delegationTokenStr);
+    } else {
+      sessionHandle = cliService.openSession(protocol, userName, req.getPassword(),
+          ipAddress, req.getConfiguration());
+    }
+    return sessionHandle;
+  }
+
+
+  private String getDelegationToken(String userName)
+      throws HiveSQLException, LoginException, IOException {
+    if (userName == null || !cliService.getHiveConf().getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION)
+        .equalsIgnoreCase(HiveAuthFactory.AuthTypes.KERBEROS.toString())) {
+      return null;
+    }
+    try {
+      return cliService.getDelegationTokenFromMetaStore(userName);
+    } catch (UnsupportedOperationException e) {
+      // The delegation token is not applicable in the given deployment mode
+    }
+    return null;
+  }
+
+  private TProtocolVersion getMinVersion(TProtocolVersion... versions) {
+    TProtocolVersion[] values = TProtocolVersion.values();
+    int current = values[values.length - 1].getValue();
+    for (TProtocolVersion version : versions) {
+      if (current > version.getValue()) {
+        current = version.getValue();
+      }
+    }
+    for (TProtocolVersion version : values) {
+      if (version.getValue() == current) {
+        return version;
+      }
+    }
+    throw new IllegalArgumentException("never");
+  }
+
+  @Override
+  public TCloseSessionResp CloseSession(TCloseSessionReq req) throws TException {
+    TCloseSessionResp resp = new TCloseSessionResp();
+    try {
+      SessionHandle sessionHandle = new SessionHandle(req.getSessionHandle());
+      cliService.closeSession(sessionHandle);
+      resp.setStatus(OK_STATUS);
+      ThriftCLIServerContext context =
+        (ThriftCLIServerContext)currentServerContext.get();
+      if (context != null) {
+        context.setSessionHandle(null);
+      }
+    } catch (Exception e) {
+      LOG.warn("Error closing session: ", e);
+      resp.setStatus(HiveSQLException.toTStatus(e));
+    }
+    return resp;
+  }
+
+  @Override
+  public TGetInfoResp GetInfo(TGetInfoReq req) throws TException {
+    TGetInfoResp resp = new TGetInfoResp();
+    try {
+      GetInfoValue getInfoValue =
+          cliService.getInfo(new SessionHandle(req.getSessionHandle()),
+              GetInfoType.getGetInfoType(req.getInfoType()));
+      resp.setInfoValue(getInfoValue.toTGetInfoValue());
+      resp.setStatus(OK_STATUS);
+    } catch (Exception e) {
+      LOG.warn("Error getting info: ", e);
+      resp.setStatus(HiveSQLException.toTStatus(e));
+    }
+    return resp;
+  }
+
+  @Override
+  public TExecuteStatementResp ExecuteStatement(TExecuteStatementReq req) throws TException {
+    TExecuteStatementResp resp = new TExecuteStatementResp();
+    try {
+      SessionHandle sessionHandle = new SessionHandle(req.getSessionHandle());
+      String statement = req.getStatement();
+      Map<String, String> confOverlay = req.getConfOverlay();
+      Boolean runAsync = req.isRunAsync();
+      long queryTimeout = req.getQueryTimeout();
+      OperationHandle operationHandle = runAsync ?
+          cliService.executeStatementAsync(sessionHandle, statement, confOverlay, queryTimeout)
+          : cliService.executeStatement(sessionHandle, statement, confOverlay, queryTimeout);
+          resp.setOperationHandle(operationHandle.toTOperationHandle());
+          resp.setStatus(OK_STATUS);
+    } catch (Exception e) {
+      LOG.warn("Error executing statement: ", e);
+      resp.setStatus(HiveSQLException.toTStatus(e));
+    }
+    return resp;
+  }
+
+  @Override
+  public TGetTypeInfoResp GetTypeInfo(TGetTypeInfoReq req) throws TException {
+    TGetTypeInfoResp resp = new TGetTypeInfoResp();
+    try {
+      OperationHandle operationHandle = cliService.getTypeInfo(new SessionHandle(req.getSessionHandle()));
+      resp.setOperationHandle(operationHandle.toTOperationHandle());
+      resp.setStatus(OK_STATUS);
+    } catch (Exception e) {
+      LOG.warn("Error getting type info: ", e);
+      resp.setStatus(HiveSQLException.toTStatus(e));
+    }
+    return resp;
+  }
+
+  @Override
+  public TGetCatalogsResp GetCatalogs(TGetCatalogsReq req) throws TException {
+    TGetCatalogsResp resp = new TGetCatalogsResp();
+    try {
+      OperationHandle opHandle = cliService.getCatalogs(new SessionHandle(req.getSessionHandle()));
+      resp.setOperationHandle(opHandle.toTOperationHandle());
+      resp.setStatus(OK_STATUS);
+    } catch (Exception e) {
+      LOG.warn("Error getting catalogs: ", e);
+      resp.setStatus(HiveSQLException.toTStatus(e));
+    }
+    return resp;
+  }
+
+  @Override
+  public TGetSchemasResp GetSchemas(TGetSchemasReq req) throws TException {
+    TGetSchemasResp resp = new TGetSchemasResp();
+    try {
+      OperationHandle opHandle = cliService.getSchemas(
+          new SessionHandle(req.getSessionHandle()), req.getCatalogName(), req.getSchemaName());
+      resp.setOperationHandle(opHandle.toTOperationHandle());
+      resp.setStatus(OK_STATUS);
+    } catch (Exception e) {
+      LOG.warn("Error getting schemas: ", e);
+      resp.setStatus(HiveSQLException.toTStatus(e));
+    }
+    return resp;
+  }
+
+  @Override
+  public TGetTablesResp GetTables(TGetTablesReq req) throws TException {
+    TGetTablesResp resp = new TGetTablesResp();
+    try {
+      OperationHandle opHandle = cliService
+          .getTables(new SessionHandle(req.getSessionHandle()), req.getCatalogName(),
+              req.getSchemaName(), req.getTableName(), req.getTableTypes());
+      resp.setOperationHandle(opHandle.toTOperationHandle());
+      resp.setStatus(OK_STATUS);
+    } catch (Exception e) {
+      LOG.warn("Error getting tables: ", e);
+      resp.setStatus(HiveSQLException.toTStatus(e));
+    }
+    return resp;
+  }
+
+  @Override
+  public TGetTableTypesResp GetTableTypes(TGetTableTypesReq req) throws TException {
+    TGetTableTypesResp resp = new TGetTableTypesResp();
+    try {
+      OperationHandle opHandle = cliService.getTableTypes(new SessionHandle(req.getSessionHandle()));
+      resp.setOperationHandle(opHandle.toTOperationHandle());
+      resp.setStatus(OK_STATUS);
+    } catch (Exception e) {
+      LOG.warn("Error getting table types: ", e);
+      resp.setStatus(HiveSQLException.toTStatus(e));
+    }
+    return resp;
+  }
+
+  @Override
+  public TGetColumnsResp GetColumns(TGetColumnsReq req) throws TException {
+    TGetColumnsResp resp = new TGetColumnsResp();
+    try {
+      OperationHandle opHandle = cliService.getColumns(
+          new SessionHandle(req.getSessionHandle()),
+          req.getCatalogName(),
+          req.getSchemaName(),
+          req.getTableName(),
+          req.getColumnName());
+      resp.setOperationHandle(opHandle.toTOperationHandle());
+      resp.setStatus(OK_STATUS);
+    } catch (Exception e) {
+      LOG.warn("Error getting columns: ", e);
+      resp.setStatus(HiveSQLException.toTStatus(e));
+    }
+    return resp;
+  }
+
+  @Override
+  public TGetFunctionsResp GetFunctions(TGetFunctionsReq req) throws TException {
+    TGetFunctionsResp resp = new TGetFunctionsResp();
+    try {
+      OperationHandle opHandle = cliService.getFunctions(
+          new SessionHandle(req.getSessionHandle()), req.getCatalogName(),
+          req.getSchemaName(), req.getFunctionName());
+      resp.setOperationHandle(opHandle.toTOperationHandle());
+      resp.setStatus(OK_STATUS);
+    } catch (Exception e) {
+      LOG.warn("Error getting functions: ", e);
+      resp.setStatus(HiveSQLException.toTStatus(e));
+    }
+    return resp;
+  }
+
+  @Override
+  public TGetOperationStatusResp GetOperationStatus(TGetOperationStatusReq req) throws TException {
+    TGetOperationStatusResp resp = new TGetOperationStatusResp();
+    try {
+      OperationStatus operationStatus = cliService.getOperationStatus(
+          new OperationHandle(req.getOperationHandle()));
+      resp.setOperationState(operationStatus.getState().toTOperationState());
+      HiveSQLException opException = operationStatus.getOperationException();
+      if (opException != null) {
+        resp.setSqlState(opException.getSQLState());
+        resp.setErrorCode(opException.getErrorCode());
+        resp.setErrorMessage(org.apache.hadoop.util.StringUtils
+            .stringifyException(opException));
+      }
+      resp.setStatus(OK_STATUS);
+    } catch (Exception e) {
+      LOG.warn("Error getting operation status: ", e);
+      resp.setStatus(HiveSQLException.toTStatus(e));
+    }
+    return resp;
+  }
+
+  @Override
+  public TCancelOperationResp CancelOperation(TCancelOperationReq req) throws TException {
+    TCancelOperationResp resp = new TCancelOperationResp();
+    try {
+      cliService.cancelOperation(new OperationHandle(req.getOperationHandle()));
+      resp.setStatus(OK_STATUS);
+    } catch (Exception e) {
+      LOG.warn("Error cancelling operation: ", e);
+      resp.setStatus(HiveSQLException.toTStatus(e));
+    }
+    return resp;
+  }
+
+  @Override
+  public TCloseOperationResp CloseOperation(TCloseOperationReq req) throws TException {
+    TCloseOperationResp resp = new TCloseOperationResp();
+    try {
+      cliService.closeOperation(new OperationHandle(req.getOperationHandle()));
+      resp.setStatus(OK_STATUS);
+    } catch (Exception e) {
+      LOG.warn("Error closing operation: ", e);
+      resp.setStatus(HiveSQLException.toTStatus(e));
+    }
+    return resp;
+  }
+
+  @Override
+  public TGetResultSetMetadataResp GetResultSetMetadata(TGetResultSetMetadataReq req)
+      throws TException {
+    TGetResultSetMetadataResp resp = new TGetResultSetMetadataResp();
+    try {
+      TableSchema schema = cliService.getResultSetMetadata(new OperationHandle(req.getOperationHandle()));
+      resp.setSchema(schema.toTTableSchema());
+      resp.setStatus(OK_STATUS);
+    } catch (Exception e) {
+      LOG.warn("Error getting result set metadata: ", e);
+      resp.setStatus(HiveSQLException.toTStatus(e));
+    }
+    return resp;
+  }
+
+  @Override
+  public TFetchResultsResp FetchResults(TFetchResultsReq req) throws TException {
+    TFetchResultsResp resp = new TFetchResultsResp();
+    try {
+      RowSet rowSet = cliService.fetchResults(
+          new OperationHandle(req.getOperationHandle()),
+          FetchOrientation.getFetchOrientation(req.getOrientation()),
+          req.getMaxRows(),
+          FetchType.getFetchType(req.getFetchType()));
+      resp.setResults(rowSet.toTRowSet());
+      resp.setHasMoreRows(false);
+      resp.setStatus(OK_STATUS);
+    } catch (Exception e) {
+      LOG.warn("Error fetching results: ", e);
+      resp.setStatus(HiveSQLException.toTStatus(e));
+    }
+    return resp;
+  }
+
+  @Override
+  public TGetPrimaryKeysResp GetPrimaryKeys(TGetPrimaryKeysReq req)
+      throws TException {
+    TGetPrimaryKeysResp resp = new TGetPrimaryKeysResp();
+    try {
+      OperationHandle opHandle = cliService.getPrimaryKeys(
+      new SessionHandle(req.getSessionHandle()), req.getCatalogName(),
+      req.getSchemaName(), req.getTableName());
+      resp.setOperationHandle(opHandle.toTOperationHandle());
+      resp.setStatus(OK_STATUS);
+    } catch (Exception e) {
+     LOG.warn("Error getting primary keys: ", e);
+     resp.setStatus(HiveSQLException.toTStatus(e));
+    }
+    return resp;
+  }
+
+  @Override
+  public TGetCrossReferenceResp GetCrossReference(TGetCrossReferenceReq req)
+      throws TException {
+    TGetCrossReferenceResp resp = new TGetCrossReferenceResp();
+    try {
+      OperationHandle opHandle = cliService.getCrossReference(
+        new SessionHandle(req.getSessionHandle()), req.getParentCatalogName(),
+          req.getParentSchemaName(), req.getParentTableName(),
+          req.getForeignCatalogName(), req.getForeignSchemaName(), req.getForeignTableName());
+          resp.setOperationHandle(opHandle.toTOperationHandle());
+          resp.setStatus(OK_STATUS);
+    } catch (Exception e) {
+      LOG.warn("Error getting cross reference: ", e);
+      resp.setStatus(HiveSQLException.toTStatus(e));
+    }
+    return resp;
+  }
+
+  protected abstract void initializeServer();
+
+  @Override
+  public TGetQueryIdResp GetQueryId(TGetQueryIdReq req) throws TException {
+    try {
+      return new TGetQueryIdResp(cliService.getQueryId(req.getOperationHandle()));
+    } catch (HiveSQLException e) {
+      throw new TException(e);
+    }
+  }
+
+  @Override
+  public abstract void run();
+
+  /**
+   * If the proxy user name is provided then check privileges to substitute the user.
+   * @param realUser
+   * @param sessionConf
+   * @param ipAddress
+   * @return
+   * @throws HiveSQLException
+   */
+  private String getProxyUser(String realUser, Map<String, String> sessionConf,
+      String ipAddress) throws HiveSQLException {
+    String proxyUser = null;
+    // Http transport mode.
+    // We set the thread local proxy username, in ThriftHttpServlet.
+    if (cliService.getHiveConf().getVar(
+        ConfVars.HIVE_SERVER2_TRANSPORT_MODE).equalsIgnoreCase("http")) {
+      proxyUser = SessionManager.getProxyUserName();
+      LOG.debug("Proxy user from query string: " + proxyUser);
+    }
+
+    if (proxyUser == null && sessionConf != null && sessionConf.containsKey(HiveAuthFactory.HS2_PROXY_USER)) {
+      String proxyUserFromThriftBody = sessionConf.get(HiveAuthFactory.HS2_PROXY_USER);
+      LOG.debug("Proxy user from thrift body: " + proxyUserFromThriftBody);
+      proxyUser = proxyUserFromThriftBody;
+    }
+
+    if (proxyUser == null) {
+      return realUser;
+    }
+
+    // check whether substitution is allowed
+    if (!hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ALLOW_USER_SUBSTITUTION)) {
+      throw new HiveSQLException("Proxy user substitution is not allowed");
+    }
+
+    // If there's no authentication, then directly substitute the user
+    if (HiveAuthFactory.AuthTypes.NONE.toString()
+        .equalsIgnoreCase(hiveConf.getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION))) {
+      return proxyUser;
+    }
+
+    // Verify proxy user privilege of the realUser for the proxyUser
+    HiveAuthFactory.verifyProxyAccess(realUser, proxyUser, ipAddress, hiveConf);
+    LOG.debug("Verified proxy user: " + proxyUser);
+    return proxyUser;
+  }
+
+  private boolean isKerberosAuthMode() {
+    return cliService.getHiveConf().getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION)
+        .equalsIgnoreCase(HiveAuthFactory.AuthTypes.KERBEROS.toString());
+  }
+}
diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIServiceClient.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIServiceClient.java
new file mode 100644
index 0000000000000..0e81e4446caac
--- /dev/null
+++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIServiceClient.java
@@ -0,0 +1,502 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.service.cli.thrift;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hive.service.auth.HiveAuthFactory;
+import org.apache.hive.service.cli.*;
+import org.apache.hive.service.rpc.thrift.*;
+import org.apache.thrift.TException;
+
+/**
+ * ThriftCLIServiceClient.
+ *
+ */
+public class ThriftCLIServiceClient extends CLIServiceClient {
+  private final TCLIService.Iface cliService;
+
+  public ThriftCLIServiceClient(TCLIService.Iface cliService) {
+    this.cliService = cliService;
+  }
+
+  public void checkStatus(TStatus status) throws HiveSQLException {
+    if (TStatusCode.ERROR_STATUS.equals(status.getStatusCode())) {
+      throw new HiveSQLException(status);
+    }
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#openSession(java.lang.String, java.lang.String, java.util.Map)
+   */
+  @Override
+  public SessionHandle openSession(String username, String password,
+      Map<String, String> configuration)
+          throws HiveSQLException {
+    try {
+      TOpenSessionReq req = new TOpenSessionReq();
+      req.setUsername(username);
+      req.setPassword(password);
+      req.setConfiguration(configuration);
+      TOpenSessionResp resp = cliService.OpenSession(req);
+      checkStatus(resp.getStatus());
+      return new SessionHandle(resp.getSessionHandle(), resp.getServerProtocolVersion());
+    } catch (HiveSQLException e) {
+      throw e;
+    } catch (Exception e) {
+      throw new HiveSQLException(e);
+    }
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#closeSession(org.apache.hive.service.cli.SessionHandle)
+   */
+  @Override
+  public SessionHandle openSessionWithImpersonation(String username, String password,
+      Map<String, String> configuration, String delegationToken) throws HiveSQLException {
+    throw new HiveSQLException("open with impersonation operation is not supported in the client");
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#closeSession(org.apache.hive.service.cli.SessionHandle)
+   */
+  @Override
+  public void closeSession(SessionHandle sessionHandle) throws HiveSQLException {
+    try {
+      TCloseSessionReq req = new TCloseSessionReq(sessionHandle.toTSessionHandle());
+      TCloseSessionResp resp = cliService.CloseSession(req);
+      checkStatus(resp.getStatus());
+    } catch (HiveSQLException e) {
+      throw e;
+    } catch (Exception e) {
+      throw new HiveSQLException(e);
+    }
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#getInfo(org.apache.hive.service.cli.SessionHandle, java.util.List)
+   */
+  @Override
+  public GetInfoValue getInfo(SessionHandle sessionHandle, GetInfoType infoType)
+      throws HiveSQLException {
+    try {
+      // FIXME extract the right info type
+      TGetInfoReq req = new TGetInfoReq(sessionHandle.toTSessionHandle(), infoType.toTGetInfoType());
+      TGetInfoResp resp = cliService.GetInfo(req);
+      checkStatus(resp.getStatus());
+      return new GetInfoValue(resp.getInfoValue());
+    } catch (HiveSQLException e) {
+      throw e;
+    } catch (Exception e) {
+      throw new HiveSQLException(e);
+    }
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#executeStatement(org.apache.hive.service.cli.SessionHandle, java.lang.String, java.util.Map)
+   */
+  @Override
+  public OperationHandle executeStatement(SessionHandle sessionHandle, String statement,
+      Map<String, String> confOverlay) throws HiveSQLException {
+    return executeStatementInternal(sessionHandle, statement, confOverlay, false, 0);
+  }
+
+  @Override
+  public OperationHandle executeStatement(SessionHandle sessionHandle, String statement,
+      Map<String, String> confOverlay, long queryTimeout) throws HiveSQLException {
+    return executeStatementInternal(sessionHandle, statement, confOverlay, false, queryTimeout);
+  }
+
+  @Override
+  public OperationHandle executeStatementAsync(SessionHandle sessionHandle, String statement,
+      Map<String, String> confOverlay) throws HiveSQLException {
+    return executeStatementInternal(sessionHandle, statement, confOverlay, true, 0);
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#executeStatementAsync(org.apache.hive.service.cli.SessionHandle, java.lang.String, java.util.Map)
+   */
+  @Override
+  public OperationHandle executeStatementAsync(SessionHandle sessionHandle, String statement,
+      Map<String, String> confOverlay, long queryTimeout) throws HiveSQLException {
+    return executeStatementInternal(sessionHandle, statement, confOverlay, true, queryTimeout);
+  }
+
+  private OperationHandle executeStatementInternal(SessionHandle sessionHandle, String statement,
+      Map<String, String> confOverlay, boolean isAsync, long queryTimeout) throws HiveSQLException {
+    try {
+      TExecuteStatementReq req =
+          new TExecuteStatementReq(sessionHandle.toTSessionHandle(), statement);
+      req.setConfOverlay(confOverlay);
+      req.setRunAsync(isAsync);
+      req.setQueryTimeout(queryTimeout);
+      TExecuteStatementResp resp = cliService.ExecuteStatement(req);
+      checkStatus(resp.getStatus());
+      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
+      return new OperationHandle(resp.getOperationHandle(), protocol);
+    } catch (HiveSQLException e) {
+      throw e;
+    } catch (Exception e) {
+      throw new HiveSQLException(e);
+    }
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#getTypeInfo(org.apache.hive.service.cli.SessionHandle)
+   */
+  @Override
+  public OperationHandle getTypeInfo(SessionHandle sessionHandle) throws HiveSQLException {
+    try {
+      TGetTypeInfoReq req = new TGetTypeInfoReq(sessionHandle.toTSessionHandle());
+      TGetTypeInfoResp resp = cliService.GetTypeInfo(req);
+      checkStatus(resp.getStatus());
+      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
+      return new OperationHandle(resp.getOperationHandle(), protocol);
+    } catch (HiveSQLException e) {
+      throw e;
+    } catch (Exception e) {
+      throw new HiveSQLException(e);
+    }
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#getCatalogs(org.apache.hive.service.cli.SessionHandle)
+   */
+  @Override
+  public OperationHandle getCatalogs(SessionHandle sessionHandle) throws HiveSQLException {
+    try {
+      TGetCatalogsReq req = new TGetCatalogsReq(sessionHandle.toTSessionHandle());
+      TGetCatalogsResp resp = cliService.GetCatalogs(req);
+      checkStatus(resp.getStatus());
+      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
+      return new OperationHandle(resp.getOperationHandle(), protocol);
+    } catch (HiveSQLException e) {
+      throw e;
+    } catch (Exception e) {
+      throw new HiveSQLException(e);
+    }
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#getSchemas(org.apache.hive.service.cli.SessionHandle, java.lang.String, java.lang.String)
+   */
+  @Override
+  public OperationHandle getSchemas(SessionHandle sessionHandle, String catalogName,
+      String schemaName)
+          throws HiveSQLException {
+    try {
+      TGetSchemasReq req = new TGetSchemasReq(sessionHandle.toTSessionHandle());
+      req.setCatalogName(catalogName);
+      req.setSchemaName(schemaName);
+      TGetSchemasResp resp = cliService.GetSchemas(req);
+      checkStatus(resp.getStatus());
+      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
+      return new OperationHandle(resp.getOperationHandle(), protocol);
+    } catch (HiveSQLException e) {
+      throw e;
+    } catch (Exception e) {
+      throw new HiveSQLException(e);
+    }
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#getTables(org.apache.hive.service.cli.SessionHandle, java.lang.String, java.lang.String, java.lang.String, java.util.List)
+   */
+  @Override
+  public OperationHandle getTables(SessionHandle sessionHandle, String catalogName,
+      String schemaName, String tableName, List<String> tableTypes)
+          throws HiveSQLException {
+    try {
+      TGetTablesReq req = new TGetTablesReq(sessionHandle.toTSessionHandle());
+      req.setTableName(tableName);
+      req.setTableTypes(tableTypes);
+      req.setSchemaName(schemaName);
+      TGetTablesResp resp = cliService.GetTables(req);
+      checkStatus(resp.getStatus());
+      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
+      return new OperationHandle(resp.getOperationHandle(), protocol);
+    } catch (HiveSQLException e) {
+      throw e;
+    } catch (Exception e) {
+      throw new HiveSQLException(e);
+    }
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#getTableTypes(org.apache.hive.service.cli.SessionHandle)
+   */
+  @Override
+  public OperationHandle getTableTypes(SessionHandle sessionHandle) throws HiveSQLException {
+    try {
+      TGetTableTypesReq req = new TGetTableTypesReq(sessionHandle.toTSessionHandle());
+      TGetTableTypesResp resp = cliService.GetTableTypes(req);
+      checkStatus(resp.getStatus());
+      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
+      return new OperationHandle(resp.getOperationHandle(), protocol);
+    } catch (HiveSQLException e) {
+      throw e;
+    } catch (Exception e) {
+      throw new HiveSQLException(e);
+    }
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#getColumns(org.apache.hive.service.cli.SessionHandle)
+   */
+  @Override
+  public OperationHandle getColumns(SessionHandle sessionHandle,
+      String catalogName, String schemaName, String tableName, String columnName)
+          throws HiveSQLException {
+    try {
+      TGetColumnsReq req = new TGetColumnsReq();
+      req.setSessionHandle(sessionHandle.toTSessionHandle());
+      req.setCatalogName(catalogName);
+      req.setSchemaName(schemaName);
+      req.setTableName(tableName);
+      req.setColumnName(columnName);
+      TGetColumnsResp resp = cliService.GetColumns(req);
+      checkStatus(resp.getStatus());
+      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
+      return new OperationHandle(resp.getOperationHandle(), protocol);
+    } catch (HiveSQLException e) {
+      throw e;
+    } catch (Exception e) {
+      throw new HiveSQLException(e);
+    }
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#getFunctions(org.apache.hive.service.cli.SessionHandle)
+   */
+  @Override
+  public OperationHandle getFunctions(SessionHandle sessionHandle,
+      String catalogName, String schemaName, String functionName) throws HiveSQLException {
+    try {
+      TGetFunctionsReq req = new TGetFunctionsReq(sessionHandle.toTSessionHandle(), functionName);
+      req.setCatalogName(catalogName);
+      req.setSchemaName(schemaName);
+      TGetFunctionsResp resp = cliService.GetFunctions(req);
+      checkStatus(resp.getStatus());
+      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
+      return new OperationHandle(resp.getOperationHandle(), protocol);
+    } catch (HiveSQLException e) {
+      throw e;
+    } catch (Exception e) {
+      throw new HiveSQLException(e);
+    }
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#getOperationStatus(org.apache.hive.service.cli.OperationHandle)
+   */
+  @Override
+  public OperationStatus getOperationStatus(OperationHandle opHandle) throws HiveSQLException {
+    try {
+      TGetOperationStatusReq req = new TGetOperationStatusReq(opHandle.toTOperationHandle());
+      TGetOperationStatusResp resp = cliService.GetOperationStatus(req);
+      // Checks the status of the RPC call, throws an exception in case of error
+      checkStatus(resp.getStatus());
+      OperationState opState = OperationState.getOperationState(resp.getOperationState());
+      HiveSQLException opException = null;
+      if (opState == OperationState.ERROR) {
+        opException = new HiveSQLException(resp.getErrorMessage(), resp.getSqlState(), resp.getErrorCode());
+      }
+      return new OperationStatus(opState, opException);
+    } catch (HiveSQLException e) {
+      throw e;
+    } catch (Exception e) {
+      throw new HiveSQLException(e);
+    }
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#cancelOperation(org.apache.hive.service.cli.OperationHandle)
+   */
+  @Override
+  public void cancelOperation(OperationHandle opHandle) throws HiveSQLException {
+    try {
+      TCancelOperationReq req = new TCancelOperationReq(opHandle.toTOperationHandle());
+      TCancelOperationResp resp = cliService.CancelOperation(req);
+      checkStatus(resp.getStatus());
+    } catch (HiveSQLException e) {
+      throw e;
+    } catch (Exception e) {
+      throw new HiveSQLException(e);
+    }
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#closeOperation(org.apache.hive.service.cli.OperationHandle)
+   */
+  @Override
+  public void closeOperation(OperationHandle opHandle)
+      throws HiveSQLException {
+    try {
+      TCloseOperationReq req  = new TCloseOperationReq(opHandle.toTOperationHandle());
+      TCloseOperationResp resp = cliService.CloseOperation(req);
+      checkStatus(resp.getStatus());
+    } catch (HiveSQLException e) {
+      throw e;
+    } catch (Exception e) {
+      throw new HiveSQLException(e);
+    }
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#getResultSetMetadata(org.apache.hive.service.cli.OperationHandle)
+   */
+  @Override
+  public TableSchema getResultSetMetadata(OperationHandle opHandle)
+      throws HiveSQLException {
+    try {
+      TGetResultSetMetadataReq req = new TGetResultSetMetadataReq(opHandle.toTOperationHandle());
+      TGetResultSetMetadataResp resp = cliService.GetResultSetMetadata(req);
+      checkStatus(resp.getStatus());
+      return new TableSchema(resp.getSchema());
+    } catch (HiveSQLException e) {
+      throw e;
+    } catch (Exception e) {
+      throw new HiveSQLException(e);
+    }
+  }
+
+  @Override
+  public RowSet fetchResults(OperationHandle opHandle, FetchOrientation orientation, long maxRows,
+      FetchType fetchType) throws HiveSQLException {
+    try {
+      TFetchResultsReq req = new TFetchResultsReq();
+      req.setOperationHandle(opHandle.toTOperationHandle());
+      req.setOrientation(orientation.toTFetchOrientation());
+      req.setMaxRows(maxRows);
+      req.setFetchType(fetchType.toTFetchType());
+      TFetchResultsResp resp = cliService.FetchResults(req);
+      checkStatus(resp.getStatus());
+      return RowSetFactory.create(resp.getResults(), opHandle.getProtocolVersion());
+    } catch (HiveSQLException e) {
+      throw e;
+    } catch (Exception e) {
+      throw new HiveSQLException(e);
+    }
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hive.service.cli.ICLIService#fetchResults(org.apache.hive.service.cli.OperationHandle)
+   */
+  @Override
+  public RowSet fetchResults(OperationHandle opHandle) throws HiveSQLException {
+    // TODO: set the correct default fetch size
+    return fetchResults(opHandle, FetchOrientation.FETCH_NEXT, 10000, FetchType.QUERY_OUTPUT);
+  }
+
+  @Override
+  public String getDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
+      String owner, String renewer) throws HiveSQLException {
+    TGetDelegationTokenReq req = new TGetDelegationTokenReq(
+        sessionHandle.toTSessionHandle(), owner, renewer);
+    try {
+      TGetDelegationTokenResp tokenResp = cliService.GetDelegationToken(req);
+      checkStatus(tokenResp.getStatus());
+      return tokenResp.getDelegationToken();
+    } catch (Exception e) {
+      throw new HiveSQLException(e);
+    }
+  }
+
+  @Override
+  public void cancelDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
+      String tokenStr) throws HiveSQLException {
+    TCancelDelegationTokenReq cancelReq = new TCancelDelegationTokenReq(
+          sessionHandle.toTSessionHandle(), tokenStr);
+    try {
+      TCancelDelegationTokenResp cancelResp =
+        cliService.CancelDelegationToken(cancelReq);
+      checkStatus(cancelResp.getStatus());
+      return;
+    } catch (TException e) {
+      throw new HiveSQLException(e);
+    }
+  }
+
+  @Override
+  public void renewDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
+      String tokenStr) throws HiveSQLException {
+    TRenewDelegationTokenReq cancelReq = new TRenewDelegationTokenReq(
+        sessionHandle.toTSessionHandle(), tokenStr);
+    try {
+      TRenewDelegationTokenResp renewResp =
+        cliService.RenewDelegationToken(cancelReq);
+      checkStatus(renewResp.getStatus());
+      return;
+    } catch (Exception e) {
+      throw new HiveSQLException(e);
+    }
+  }
+
+  @Override
+  public OperationHandle getPrimaryKeys(SessionHandle sessionHandle,
+      String catalog, String schema, String table) throws HiveSQLException {
+    try {
+      TGetPrimaryKeysReq req = new TGetPrimaryKeysReq(sessionHandle.toTSessionHandle());
+      req.setCatalogName(catalog);
+      req.setSchemaName(schema);
+      req.setTableName(table);
+      TGetPrimaryKeysResp resp = cliService.GetPrimaryKeys(req);
+      checkStatus(resp.getStatus());
+      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
+      return new OperationHandle(resp.getOperationHandle(), protocol);
+    } catch (HiveSQLException e) {
+      throw e;
+    } catch (Exception e) {
+      throw new HiveSQLException(e);
+    }
+  }
+
+  @Override
+  public OperationHandle getCrossReference(SessionHandle sessionHandle,
+      String primaryCatalog, String primarySchema, String primaryTable,
+      String foreignCatalog, String foreignSchema, String foreignTable) throws HiveSQLException {
+    try {
+      TGetCrossReferenceReq req = new TGetCrossReferenceReq(sessionHandle.toTSessionHandle());
+      req.setParentCatalogName(primaryCatalog);
+      req.setParentSchemaName(primarySchema);
+      req.setParentTableName(primaryTable);
+      req.setForeignCatalogName(foreignCatalog);
+      req.setForeignSchemaName(foreignSchema);
+      req.setForeignTableName(foreignTable);
+      TGetCrossReferenceResp resp = cliService.GetCrossReference(req);
+      checkStatus(resp.getStatus());
+      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
+      return new OperationHandle(resp.getOperationHandle(), protocol);
+    } catch (HiveSQLException e) {
+      throw e;
+    } catch (Exception e) {
+      throw new HiveSQLException(e);
+    }
+  }
+
+  @Override
+  public String getQueryId(TOperationHandle operationHandle) throws HiveSQLException {
+    try {
+      return cliService.GetQueryId(new TGetQueryIdReq(operationHandle)).getQueryId();
+    } catch (TException e) {
+      throw new HiveSQLException(e);
+    }
+  }
+}
diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
new file mode 100644
index 0000000000000..bc568be5d2ebb
--- /dev/null
+++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
@@ -0,0 +1,214 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.service.cli.thrift;
+
+import java.util.Arrays;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Shell;
+import org.apache.hive.service.ServiceException;
+import org.apache.hive.service.auth.HiveAuthFactory;
+import org.apache.hive.service.cli.CLIService;
+import org.apache.hive.service.rpc.thrift.TCLIService;
+import org.apache.hive.service.rpc.thrift.TCLIService.Iface;
+import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup;
+import org.apache.thrift.TProcessor;
+import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.protocol.TProtocolFactory;
+import org.apache.thrift.server.TServlet;
+import org.eclipse.jetty.server.AbstractConnectionFactory;
+import org.eclipse.jetty.server.ConnectionFactory;
+import org.eclipse.jetty.server.HttpConnectionFactory;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+import org.eclipse.jetty.util.thread.ExecutorThreadPool;
+import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler;
+
+
+public class ThriftHttpCLIService extends ThriftCLIService {
+
+  protected org.eclipse.jetty.server.Server httpServer;
+
+  public ThriftHttpCLIService(CLIService cliService) {
+    super(cliService, ThriftHttpCLIService.class.getSimpleName());
+  }
+
+  @Override
+  protected void initializeServer() {
+    try {
+      // Server thread pool
+      // Start with minWorkerThreads, expand till maxWorkerThreads and reject subsequent requests
+      String threadPoolName = "HiveServer2-HttpHandler-Pool";
+      ThreadPoolExecutor executorService = new ThreadPoolExecutor(minWorkerThreads, maxWorkerThreads,
+          workerKeepAliveTime, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
+          new ThreadFactoryWithGarbageCleanup(threadPoolName));
+      ExecutorThreadPool threadPool = new ExecutorThreadPool(executorService);
+
+      // HTTP Server
+      httpServer = new org.eclipse.jetty.server.Server(threadPool);
+
+      // Connector configs
+
+      ConnectionFactory[] connectionFactories;
+      boolean useSsl = hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_USE_SSL);
+      String schemeName = useSsl ? "https" : "http";
+      // Change connector if SSL is used
+      if (useSsl) {
+        String keyStorePath = hiveConf.getVar(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH).trim();
+        String keyStorePassword = ShimLoader.getHadoopShims().getPassword(hiveConf,
+            HiveConf.ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname);
+        if (keyStorePath.isEmpty()) {
+          throw new IllegalArgumentException(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname
+              + " Not configured for SSL connection");
+        }
+        SslContextFactory sslContextFactory = new SslContextFactory.Server();
+        String[] excludedProtocols = hiveConf.getVar(ConfVars.HIVE_SSL_PROTOCOL_BLACKLIST).split(",");
+        LOG.info("HTTP Server SSL: adding excluded protocols: " + Arrays.toString(excludedProtocols));
+        sslContextFactory.addExcludeProtocols(excludedProtocols);
+        LOG.info("HTTP Server SSL: SslContextFactory.getExcludeProtocols = " +
+          Arrays.toString(sslContextFactory.getExcludeProtocols()));
+        sslContextFactory.setKeyStorePath(keyStorePath);
+        sslContextFactory.setKeyStorePassword(keyStorePassword);
+        connectionFactories = AbstractConnectionFactory.getFactories(
+            sslContextFactory, new HttpConnectionFactory());
+      } else {
+        connectionFactories = new ConnectionFactory[] { new HttpConnectionFactory() };
+      }
+      ServerConnector connector = new ServerConnector(
+          httpServer,
+          null,
+          // Call this full constructor to set this, which forces daemon threads:
+          new ScheduledExecutorScheduler("HiveServer2-HttpHandler-JettyScheduler", true),
+          null,
+          -1,
+          -1,
+          connectionFactories);
+
+      connector.setPort(portNum);
+      // Linux:yes, Windows:no
+      connector.setReuseAddress(!Shell.WINDOWS);
+      int maxIdleTime = (int) hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_MAX_IDLE_TIME,
+          TimeUnit.MILLISECONDS);
+      connector.setIdleTimeout(maxIdleTime);
+
+      httpServer.addConnector(connector);
+
+      // Thrift configs
+      hiveAuthFactory = new HiveAuthFactory(hiveConf);
+      TProcessor processor = new TCLIService.Processor<Iface>(this);
+      TProtocolFactory protocolFactory = new TBinaryProtocol.Factory();
+      // Set during the init phase of HiveServer2 if auth mode is kerberos
+      // UGI for the hive/_HOST (kerberos) principal
+      UserGroupInformation serviceUGI = cliService.getServiceUGI();
+      // UGI for the http/_HOST (SPNego) principal
+      UserGroupInformation httpUGI = cliService.getHttpUGI();
+      String authType = hiveConf.getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION);
+      TServlet thriftHttpServlet = new ThriftHttpServlet(processor, protocolFactory, authType,
+          serviceUGI, httpUGI, hiveAuthFactory);
+
+      // Context handler
+      final ServletContextHandler context = new ServletContextHandler(
+          ServletContextHandler.SESSIONS);
+      context.setContextPath("/");
+      String httpPath = getHttpPath(hiveConf
+          .getVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH));
+      httpServer.setHandler(context);
+      context.addServlet(new ServletHolder(thriftHttpServlet), httpPath);
+
+      // TODO: check defaults: maxTimeout, keepalive, maxBodySize, bodyReceiveDuration, etc.
+      // Finally, start the server
+      httpServer.start();
+      // In case HIVE_SERVER2_THRIFT_HTTP_PORT or hive.server2.thrift.http.port is configured with
+      // 0 which represents any free port, we should set it to the actual one
+      portNum = connector.getLocalPort();
+      String msg = "Started " + ThriftHttpCLIService.class.getSimpleName() + " in " + schemeName
+          + " mode on port " + portNum + " path=" + httpPath + " with " + minWorkerThreads + "..."
+          + maxWorkerThreads + " worker threads";
+      LOG.info(msg);
+    } catch (Exception t) {
+      throw new ServiceException("Error initializing " + getName(), t);
+    }
+  }
+
+  @Override
+  protected void stopServer() {
+    if ((httpServer != null) && httpServer.isStarted()) {
+      try {
+        httpServer.stop();
+        httpServer = null;
+        LOG.info("Thrift HTTP server has been stopped");
+      } catch (Exception e) {
+        LOG.error("Error stopping HTTP server: ", e);
+      }
+    }
+  }
+
+  /**
+   * Configure Jetty to serve http requests. Example of a client connection URL:
+   * http://localhost:10000/servlets/thrifths2/ A gateway may cause actual target URL to differ,
+   * e.g. http://gateway:port/hive2/servlets/thrifths2/
+   */
+  @Override
+  public void run() {
+    try {
+      httpServer.join();
+    } catch (Throwable t) {
+      if (t instanceof InterruptedException) {
+        // This is likely a shutdown
+        LOG.info("Caught " + t.getClass().getSimpleName() + ". Shutting down thrift server.");
+      } else {
+        LOG.error("Error starting HiveServer2: could not start "
+            + ThriftHttpCLIService.class.getSimpleName(), t);
+        System.exit(-1);
+      }
+    }
+  }
+
+  /**
+   * The config parameter can be like "path", "/path", "/path/", "path/*", "/path1/path2/*" and so on.
+   * httpPath should end up as "/*", "/path/*" or "/path1/../pathN/*"
+   * @param httpPath
+   * @return
+   */
+  private String getHttpPath(String httpPath) {
+    if(httpPath == null || httpPath.equals("")) {
+      httpPath = "/*";
+    }
+    else {
+      if(!httpPath.startsWith("/")) {
+        httpPath = "/" + httpPath;
+      }
+      if(httpPath.endsWith("/")) {
+        httpPath = httpPath + "*";
+      }
+      if(!httpPath.endsWith("/*")) {
+        httpPath = httpPath + "/*";
+      }
+    }
+    return httpPath;
+  }
+}
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/server/HiveServer2.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/server/HiveServer2.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/server/HiveServer2.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/server/HiveServer2.java
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/server/ThreadWithGarbageCleanup.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/server/ThreadWithGarbageCleanup.java
similarity index 100%
rename from sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/server/ThreadWithGarbageCleanup.java
rename to sql/hive-thriftserver/src/main/java/org/apache/hive/service/server/ThreadWithGarbageCleanup.java
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/FetchIterator.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/FetchIterator.scala
new file mode 100644
index 0000000000000..b9db657952b56
--- /dev/null
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/FetchIterator.scala
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.thriftserver
+
+private[hive] sealed trait FetchIterator[A] extends Iterator[A] {
+  /**
+   * Begin a fetch block, forward from the current position.
+   * Resets the fetch start offset.
+   */
+  def fetchNext(): Unit
+
+  /**
+   * Begin a fetch block, moving the iterator back by offset from the start of the previous fetch
+   * block start.
+   * Resets the fetch start offset.
+   *
+   * @param offset the amount to move a fetch start position toward the prior direction.
+   */
+  def fetchPrior(offset: Long): Unit = fetchAbsolute(getFetchStart - offset)
+
+  /**
+   * Begin a fetch block, moving the iterator to the given position.
+   * Resets the fetch start offset.
+   *
+   * @param pos index to move a position of iterator.
+   */
+  def fetchAbsolute(pos: Long): Unit
+
+  def getFetchStart: Long
+
+  def getPosition: Long
+}
+
+private[hive] class ArrayFetchIterator[A](src: Array[A]) extends FetchIterator[A] {
+  private var fetchStart: Long = 0
+
+  private var position: Long = 0
+
+  override def fetchNext(): Unit = fetchStart = position
+
+  override def fetchAbsolute(pos: Long): Unit = {
+    position = (pos max 0) min src.length
+    fetchStart = position
+  }
+
+  override def getFetchStart: Long = fetchStart
+
+  override def getPosition: Long = position
+
+  override def hasNext: Boolean = position < src.length
+
+  override def next(): A = {
+    position += 1
+    src(position.toInt - 1)
+  }
+}
+
+private[hive] class IterableFetchIterator[A](iterable: Iterable[A]) extends FetchIterator[A] {
+  private var iter: Iterator[A] = iterable.iterator
+
+  private var fetchStart: Long = 0
+
+  private var position: Long = 0
+
+  override def fetchNext(): Unit = fetchStart = position
+
+  override def fetchAbsolute(pos: Long): Unit = {
+    val newPos = pos max 0
+    if (newPos < position) resetPosition()
+    while (position < newPos && hasNext) next()
+    fetchStart = position
+  }
+
+  override def getFetchStart: Long = fetchStart
+
+  override def getPosition: Long = position
+
+  override def hasNext: Boolean = iter.hasNext
+
+  override def next(): A = {
+    position += 1
+    iter.next()
+  }
+
+  private def resetPosition(): Unit = {
+    if (position != 0) {
+      iter = iterable.iterator
+      position = 0
+      fetchStart = 0
+    }
+  }
+}
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2.scala
index f9f2ceeed8a75..a1f2d62a0b72c 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2.scala
@@ -52,14 +52,17 @@ object HiveThriftServer2 extends Logging {
    */
   @DeveloperApi
   def startWithContext(sqlContext: SQLContext): HiveThriftServer2 = {
-    val server = new HiveThriftServer2(sqlContext)
-
     val executionHive = HiveUtils.newClientForExecution(
       sqlContext.sparkContext.conf,
       sqlContext.sessionState.newHadoopConf())
 
+    // Cleanup the scratch dir before starting
+    ServerUtils.cleanUpScratchDir(executionHive.conf)
+    val server = new HiveThriftServer2(sqlContext)
+
     server.init(executionHive.conf)
     server.start()
+    logInfo("HiveThriftServer2 started")
     createListenerAndUI(server, sqlContext.sparkContext)
     server
   }
@@ -97,18 +100,8 @@ object HiveThriftServer2 extends Logging {
       uiTab.foreach(_.detach())
     }
 
-    val executionHive = HiveUtils.newClientForExecution(
-      SparkSQLEnv.sqlContext.sparkContext.conf,
-      SparkSQLEnv.sqlContext.sessionState.newHadoopConf())
-
     try {
-      // Cleanup the scratch dir before starting
-      ServerUtils.cleanUpScratchDir(executionHive.conf)
-      val server = new HiveThriftServer2(SparkSQLEnv.sqlContext)
-      server.init(executionHive.conf)
-      server.start()
-      logInfo("HiveThriftServer2 started")
-      createListenerAndUI(server, SparkSQLEnv.sparkContext)
+      startWithContext(SparkSQLEnv.sqlContext)
       // If application was killed before HiveThriftServer2 start successfully then SparkSubmit
       // process can not exit, so check whether if SparkContext was stopped.
       if (SparkSQLEnv.sparkContext.stopped.get()) {
@@ -123,7 +116,7 @@ object HiveThriftServer2 extends Logging {
   }
 
   private[thriftserver] object ExecutionState extends Enumeration {
-    val STARTED, COMPILED, CANCELED, FAILED, FINISHED, CLOSED = Value
+    val STARTED, COMPILED, CANCELED, TIMEDOUT, FAILED, FINISHED, CLOSED = Value
     type ExecutionState = Value
   }
 }
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ReflectionUtils.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ReflectionUtils.scala
index a4024be67ac9c..8a136c11389d6 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ReflectionUtils.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ReflectionUtils.scala
@@ -53,6 +53,6 @@ private[hive] object ReflectionUtils {
     val (types, values) = args.unzip
     val method = clazz.getDeclaredMethod(methodName, types: _*)
     method.setAccessible(true)
-    method.invoke(obj, values.toSeq: _*)
+    method.invoke(obj, values: _*)
   }
 }
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
index d14d70f7d3d83..6196f941c858a 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
@@ -18,51 +18,61 @@
 package org.apache.spark.sql.hive.thriftserver
 
 import java.security.PrivilegedExceptionAction
-import java.sql.{Date, Timestamp}
-import java.util.{Arrays, Map => JMap, UUID}
-import java.util.concurrent.RejectedExecutionException
+import java.util.{Arrays, Map => JMap}
+import java.util.concurrent.{Executors, RejectedExecutionException, TimeUnit}
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable.ArrayBuffer
 import scala.util.control.NonFatal
 
-import org.apache.commons.lang3.exception.ExceptionUtils
 import org.apache.hadoop.hive.metastore.api.FieldSchema
 import org.apache.hadoop.hive.shims.Utils
 import org.apache.hive.service.cli._
 import org.apache.hive.service.cli.operation.ExecuteStatementOperation
 import org.apache.hive.service.cli.session.HiveSession
 
-import org.apache.spark.SparkContext
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.{DataFrame, Row => SparkRow, SQLContext}
-import org.apache.spark.sql.execution.HiveResult
-import org.apache.spark.sql.execution.command.SetCommand
+import org.apache.spark.sql.execution.HiveResult.{getTimeFormatters, toHiveString, TimeFormatters}
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.VariableSubstitution
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.CalendarInterval
 import org.apache.spark.util.{Utils => SparkUtils}
 
 private[hive] class SparkExecuteStatementOperation(
+    val sqlContext: SQLContext,
     parentSession: HiveSession,
     statement: String,
     confOverlay: JMap[String, String],
-    runInBackground: Boolean = true)
-    (sqlContext: SQLContext, sessionToActivePool: JMap[SessionHandle, String])
+    runInBackground: Boolean = true,
+    queryTimeout: Long)
   extends ExecuteStatementOperation(parentSession, statement, confOverlay, runInBackground)
+  with SparkOperation
   with Logging {
 
+  // If a timeout value `queryTimeout` is specified by users and it is smaller than
+  // a global timeout value, we use the user-specified value.
+  // This code follows the Hive timeout behaviour (See #29933 for details).
+  private val timeout = {
+    val globalTimeout = sqlContext.conf.getConf(SQLConf.THRIFTSERVER_QUERY_TIMEOUT)
+    if (globalTimeout > 0 && (queryTimeout <= 0 || globalTimeout < queryTimeout)) {
+      globalTimeout
+    } else {
+      queryTimeout
+    }
+  }
+
+  private val forceCancel = sqlContext.conf.getConf(SQLConf.THRIFTSERVER_FORCE_CANCEL)
+
+  private val substitutorStatement = SQLConf.withExistingConf(sqlContext.conf) {
+    new VariableSubstitution().substitute(statement)
+  }
+
   private var result: DataFrame = _
 
-  // We cache the returned rows to get iterators again in case the user wants to use FETCH_FIRST.
-  // This is only used when `spark.sql.thriftServer.incrementalCollect` is set to `false`.
-  // In case of `true`, this will be `None` and FETCH_FIRST will trigger re-execution.
-  private var resultList: Option[Array[SparkRow]] = _
-  private var previousFetchEndOffset: Long = 0
-  private var previousFetchStartOffset: Long = 0
-  private var iter: Iterator[SparkRow] = _
+  private var iter: FetchIterator[SparkRow] = _
   private var dataTypes: Array[DataType] = _
-  private var statementId: String = _
 
   private lazy val resultSchema: TableSchema = {
     if (result == null || result.schema.isEmpty) {
@@ -73,14 +83,11 @@ private[hive] class SparkExecuteStatementOperation(
     }
   }
 
-  override def close(): Unit = {
-    // RDDs will be cleaned automatically upon garbage collection.
-    logInfo(s"Close statement with $statementId")
-    cleanup(OperationState.CLOSED)
-    HiveThriftServer2.eventManager.onOperationClosed(statementId)
-  }
-
-  def addNonNullColumnValue(from: SparkRow, to: ArrayBuffer[Any], ordinal: Int): Unit = {
+  def addNonNullColumnValue(
+      from: SparkRow,
+      to: ArrayBuffer[Any],
+      ordinal: Int,
+      timeFormatters: TimeFormatters): Unit = {
     dataTypes(ordinal) match {
       case StringType =>
         to += from.getString(ordinal)
@@ -100,69 +107,56 @@ private[hive] class SparkExecuteStatementOperation(
         to += from.getByte(ordinal)
       case ShortType =>
         to += from.getShort(ordinal)
-      case DateType =>
-        to += from.getAs[Date](ordinal)
-      case TimestampType =>
-        to += from.getAs[Timestamp](ordinal)
       case BinaryType =>
         to += from.getAs[Array[Byte]](ordinal)
+      // SPARK-31859, SPARK-31861: Date and Timestamp need to be turned to String here to:
+      // - respect spark.sql.session.timeZone
+      // - work with spark.sql.datetime.java8API.enabled
+      // These types have always been sent over the wire as string, converted later.
+      case _: DateType | _: TimestampType =>
+        to += toHiveString((from.get(ordinal), dataTypes(ordinal)), false, timeFormatters)
       case CalendarIntervalType =>
-        to += HiveResult.toHiveString((from.getAs[CalendarInterval](ordinal), CalendarIntervalType))
-      case _: ArrayType | _: StructType | _: MapType | _: UserDefinedType[_] =>
-        val hiveString = HiveResult.toHiveString((from.get(ordinal), dataTypes(ordinal)))
-        to += hiveString
+        to += toHiveString(
+          (from.getAs[CalendarInterval](ordinal), CalendarIntervalType),
+          false,
+          timeFormatters)
+      case _: ArrayType | _: StructType | _: MapType | _: UserDefinedType[_] |
+          YearMonthIntervalType | DayTimeIntervalType =>
+        to += toHiveString((from.get(ordinal), dataTypes(ordinal)), false, timeFormatters)
+    }
+  }
+
+  def getNextRowSet(order: FetchOrientation, maxRowsL: Long): RowSet = withLocalProperties {
+    try {
+      sqlContext.sparkContext.setJobGroup(statementId, substitutorStatement, forceCancel)
+      getNextRowSetInternal(order, maxRowsL)
+    } finally {
+      sqlContext.sparkContext.clearJobGroup()
     }
   }
 
-  def getNextRowSet(order: FetchOrientation, maxRowsL: Long): RowSet = withSchedulerPool {
+  private def getNextRowSetInternal(
+      order: FetchOrientation,
+      maxRowsL: Long): RowSet = withLocalProperties {
     log.info(s"Received getNextRowSet request order=${order} and maxRowsL=${maxRowsL} " +
       s"with ${statementId}")
     validateDefaultFetchOrientation(order)
     assertState(OperationState.FINISHED)
     setHasResultSet(true)
-    val resultRowSet: RowSet =
-      ThriftserverShimUtils.resultRowSet(getResultSetSchema, getProtocolVersion)
-
-    // Reset iter when FETCH_FIRST or FETCH_PRIOR
-    if ((order.equals(FetchOrientation.FETCH_FIRST) ||
-        order.equals(FetchOrientation.FETCH_PRIOR)) && previousFetchEndOffset != 0) {
-      // Reset the iterator to the beginning of the query.
-      iter = if (sqlContext.getConf(SQLConf.THRIFTSERVER_INCREMENTAL_COLLECT.key).toBoolean) {
-        resultList = None
-        result.toLocalIterator.asScala
-      } else {
-        if (resultList.isEmpty) {
-          resultList = Some(result.collect())
-        }
-        resultList.get.iterator
-      }
-    }
+    val resultRowSet: RowSet = RowSetFactory.create(getResultSetSchema, getProtocolVersion, false)
 
-    var resultOffset = {
-      if (order.equals(FetchOrientation.FETCH_FIRST)) {
-        logInfo(s"FETCH_FIRST request with $statementId. Resetting to resultOffset=0")
-        0
-      } else if (order.equals(FetchOrientation.FETCH_PRIOR)) {
-        // TODO: FETCH_PRIOR should be handled more efficiently than rewinding to beginning and
-        // reiterating.
-        val targetOffset = math.max(previousFetchStartOffset - maxRowsL, 0)
-        logInfo(s"FETCH_PRIOR request with $statementId. Resetting to resultOffset=$targetOffset")
-        var off = 0
-        while (off < targetOffset && iter.hasNext) {
-          iter.next()
-          off += 1
-        }
-        off
-      } else { // FETCH_NEXT
-        previousFetchEndOffset
-      }
+    if (order.equals(FetchOrientation.FETCH_FIRST)) {
+      iter.fetchAbsolute(0)
+    } else if (order.equals(FetchOrientation.FETCH_PRIOR)) {
+      iter.fetchPrior(maxRowsL)
+    } else {
+      iter.fetchNext()
     }
-
-    resultRowSet.setStartOffset(resultOffset)
-    previousFetchStartOffset = resultOffset
+    resultRowSet.setStartOffset(iter.getPosition)
     if (!iter.hasNext) {
       resultRowSet
     } else {
+      val timeFormatters = getTimeFormatters
       // maxRowsL here typically maps to java.sql.Statement.getFetchSize, which is an int
       val maxRows = maxRowsL.toInt
       var curRow = 0
@@ -174,17 +168,15 @@ private[hive] class SparkExecuteStatementOperation(
           if (sparkRow.isNullAt(curCol)) {
             row += null
           } else {
-            addNonNullColumnValue(sparkRow, row, curCol)
+            addNonNullColumnValue(sparkRow, row, curCol, timeFormatters)
           }
           curCol += 1
         }
         resultRowSet.addRow(row.toArray.asInstanceOf[Array[Object]])
         curRow += 1
-        resultOffset += 1
       }
-      previousFetchEndOffset = resultOffset
       log.info(s"Returning result set with ${curRow} rows from offsets " +
-        s"[$previousFetchStartOffset, $previousFetchEndOffset) with $statementId")
+        s"[${iter.getFetchStart}, ${iter.getPosition}) with $statementId")
       resultRowSet
     }
   }
@@ -193,7 +185,6 @@ private[hive] class SparkExecuteStatementOperation(
 
   override def runInternal(): Unit = {
     setState(OperationState.PENDING)
-    statementId = UUID.randomUUID().toString
     logInfo(s"Submitting query '$statement' with $statementId")
     HiveThriftServer2.eventManager.onStatementStart(
       statementId,
@@ -203,6 +194,23 @@ private[hive] class SparkExecuteStatementOperation(
       parentSession.getUsername)
     setHasResultSet(true) // avoid no resultset for async run
 
+    if (timeout > 0) {
+      val timeoutExecutor = Executors.newSingleThreadScheduledExecutor()
+      timeoutExecutor.schedule(new Runnable {
+        override def run(): Unit = {
+          try {
+            timeoutCancel()
+          } catch {
+            case NonFatal(e) =>
+              setOperationException(new HiveSQLException(e))
+              logError(s"Error cancelling the query after timeout: $timeout seconds")
+          } finally {
+            timeoutExecutor.shutdown()
+          }
+        }
+      }, timeout, TimeUnit.SECONDS)
+    }
+
     if (!runInBackground) {
       execute()
     } else {
@@ -217,11 +225,11 @@ private[hive] class SparkExecuteStatementOperation(
             override def run(): Unit = {
               registerCurrentOperationLog()
               try {
-                execute()
+                withLocalProperties {
+                  execute()
+                }
               } catch {
-                case e: HiveSQLException =>
-                  setOperationException(e)
-                  log.error("Error running hive query: ", e)
+                case e: HiveSQLException => setOperationException(e)
               }
             }
           }
@@ -259,7 +267,7 @@ private[hive] class SparkExecuteStatementOperation(
     }
   }
 
-  private def execute(): Unit = withSchedulerPool {
+  private def execute(): Unit = {
     try {
       synchronized {
         if (getStatus.getState.isTerminal) {
@@ -279,26 +287,17 @@ private[hive] class SparkExecuteStatementOperation(
         parentSession.getSessionState.getConf.setClassLoader(executionHiveClassLoader)
       }
 
-      sqlContext.sparkContext.setJobGroup(statementId, statement)
+      sqlContext.sparkContext.setJobGroup(statementId, substitutorStatement, forceCancel)
       result = sqlContext.sql(statement)
       logDebug(result.queryExecution.toString())
-      result.queryExecution.logical match {
-        case SetCommand(Some((SQLConf.THRIFTSERVER_POOL.key, Some(value)))) =>
-          sessionToActivePool.put(parentSession.getSessionHandle, value)
-          logInfo(s"Setting ${SparkContext.SPARK_SCHEDULER_POOL}=$value for future statements " +
-            "in this session.")
-        case _ =>
-      }
       HiveThriftServer2.eventManager.onStatementParsed(statementId,
         result.queryExecution.toString())
-      iter = {
-        if (sqlContext.getConf(SQLConf.THRIFTSERVER_INCREMENTAL_COLLECT.key).toBoolean) {
-          resultList = None
-          result.toLocalIterator.asScala
-        } else {
-          resultList = Some(result.collect())
-          resultList.get.iterator
-        }
+      iter = if (sqlContext.getConf(SQLConf.THRIFTSERVER_INCREMENTAL_COLLECT.key).toBoolean) {
+        new IterableFetchIterator[SparkRow](new Iterable[SparkRow] {
+          override def iterator: Iterator[SparkRow] = result.toLocalIterator.asScala
+        })
+      } else {
+        new ArrayFetchIterator[SparkRow](result.collect())
       }
       dataTypes = result.schema.fields.map(_.dataType)
     } catch {
@@ -319,16 +318,11 @@ private[hive] class SparkExecuteStatementOperation(
         } else {
           logError(s"Error executing query with $statementId, currentState $currentState, ", e)
           setState(OperationState.ERROR)
+          HiveThriftServer2.eventManager.onStatementError(
+            statementId, e.getMessage, SparkUtils.exceptionString(e))
           e match {
-            case hiveException: HiveSQLException =>
-              HiveThriftServer2.eventManager.onStatementError(
-                statementId, hiveException.getMessage, SparkUtils.exceptionString(hiveException))
-              throw hiveException
-            case _ =>
-              val root = ExceptionUtils.getRootCause(e)
-              HiveThriftServer2.eventManager.onStatementError(
-                statementId, root.getMessage, SparkUtils.exceptionString(root))
-              throw new HiveSQLException("Error running query: " + root.toString, root)
+            case _: HiveSQLException => throw e
+            case _ => throw new HiveSQLException("Error running query: " + e.toString, e)
           }
         }
     } finally {
@@ -342,42 +336,40 @@ private[hive] class SparkExecuteStatementOperation(
     }
   }
 
+  def timeoutCancel(): Unit = {
+    synchronized {
+      if (!getStatus.getState.isTerminal) {
+        logInfo(s"Query with $statementId timed out after $timeout seconds")
+        setState(OperationState.TIMEDOUT)
+        cleanup()
+        HiveThriftServer2.eventManager.onStatementTimeout(statementId)
+      }
+    }
+  }
+
   override def cancel(): Unit = {
     synchronized {
       if (!getStatus.getState.isTerminal) {
         logInfo(s"Cancel query with $statementId")
-        cleanup(OperationState.CANCELED)
+        setState(OperationState.CANCELED)
+        cleanup()
         HiveThriftServer2.eventManager.onStatementCanceled(statementId)
       }
     }
   }
 
-  private def cleanup(state: OperationState): Unit = {
-    setState(state)
+  override protected def cleanup(): Unit = {
     if (runInBackground) {
       val backgroundHandle = getBackgroundHandle()
       if (backgroundHandle != null) {
         backgroundHandle.cancel(true)
       }
     }
+    // RDDs will be cleaned automatically upon garbage collection.
     if (statementId != null) {
       sqlContext.sparkContext.cancelJobGroup(statementId)
     }
   }
-
-  private def withSchedulerPool[T](body: => T): T = {
-    val pool = sessionToActivePool.get(parentSession.getSessionHandle)
-    if (pool != null) {
-      sqlContext.sparkContext.setLocalProperty(SparkContext.SPARK_SCHEDULER_POOL, pool)
-    }
-    try {
-      body
-    } finally {
-      if (pool != null) {
-        sqlContext.sparkContext.setLocalProperty(SparkContext.SPARK_SCHEDULER_POOL, null)
-      }
-    }
-  }
 }
 
 object SparkExecuteStatementOperation {
@@ -386,6 +378,8 @@ object SparkExecuteStatementOperation {
       val attrTypeString = field.dataType match {
         case NullType => "void"
         case CalendarIntervalType => StringType.catalogString
+        case YearMonthIntervalType => "interval_year_month"
+        case DayTimeIntervalType => "interval_day_time"
         case other => other.catalogString
       }
       new FieldSchema(field.name, attrTypeString, field.getComment.getOrElse(""))
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetCatalogsOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetCatalogsOperation.scala
index 2945cfd200e46..01ef78cde8956 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetCatalogsOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetCatalogsOperation.scala
@@ -17,17 +17,13 @@
 
 package org.apache.spark.sql.hive.thriftserver
 
-import java.util.UUID
-
-import org.apache.commons.lang3.exception.ExceptionUtils
 import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType
-import org.apache.hive.service.cli.{HiveSQLException, OperationState}
+import org.apache.hive.service.cli.OperationState
 import org.apache.hive.service.cli.operation.GetCatalogsOperation
 import org.apache.hive.service.cli.session.HiveSession
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.SQLContext
-import org.apache.spark.util.{Utils => SparkUtils}
 
 /**
  * Spark's own GetCatalogsOperation
@@ -36,19 +32,13 @@ import org.apache.spark.util.{Utils => SparkUtils}
  * @param parentSession a HiveSession from SessionManager
  */
 private[hive] class SparkGetCatalogsOperation(
-    sqlContext: SQLContext,
+    val sqlContext: SQLContext,
     parentSession: HiveSession)
-  extends GetCatalogsOperation(parentSession) with Logging {
-
-  private var statementId: String = _
-
-  override def close(): Unit = {
-    super.close()
-    HiveThriftServer2.eventManager.onOperationClosed(statementId)
-  }
+  extends GetCatalogsOperation(parentSession)
+  with SparkOperation
+  with Logging {
 
   override def runInternal(): Unit = {
-    statementId = UUID.randomUUID().toString
     val logMsg = "Listing catalogs"
     logInfo(s"$logMsg with $statementId")
     setState(OperationState.RUNNING)
@@ -68,22 +58,8 @@ private[hive] class SparkGetCatalogsOperation(
         authorizeMetaGets(HiveOperationType.GET_CATALOGS, null)
       }
       setState(OperationState.FINISHED)
-    } catch {
-      case e: Throwable =>
-        logError(s"Error executing get catalogs operation with $statementId", e)
-        setState(OperationState.ERROR)
-        e match {
-          case hiveException: HiveSQLException =>
-            HiveThriftServer2.eventManager.onStatementError(
-              statementId, hiveException.getMessage, SparkUtils.exceptionString(hiveException))
-            throw hiveException
-          case _ =>
-            val root = ExceptionUtils.getRootCause(e)
-            HiveThriftServer2.eventManager.onStatementError(
-              statementId, root.getMessage, SparkUtils.exceptionString(root))
-            throw new HiveSQLException("Error getting catalogs: " + root.toString, root)
-        }
-    }
+    } catch onError()
+
     HiveThriftServer2.eventManager.onStatementFinish(statementId)
   }
 }
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala
index ff7cbfeae13be..a354050d2fc47 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala
@@ -17,12 +17,10 @@
 
 package org.apache.spark.sql.hive.thriftserver
 
-import java.util.UUID
 import java.util.regex.Pattern
 
 import scala.collection.JavaConverters.seqAsJavaListConverter
 
-import org.apache.commons.lang3.exception.ExceptionUtils
 import org.apache.hadoop.hive.ql.security.authorization.plugin.{HiveOperationType, HivePrivilegeObject}
 import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivilegeObjectType
 import org.apache.hive.service.cli._
@@ -33,9 +31,7 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.sql.SQLContext
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.catalog.SessionCatalog
-import org.apache.spark.sql.hive.thriftserver.ThriftserverShimUtils.toJavaSQLType
-import org.apache.spark.sql.types.StructType
-import org.apache.spark.util.{Utils => SparkUtils}
+import org.apache.spark.sql.types._
 
 /**
  * Spark's own SparkGetColumnsOperation
@@ -48,26 +44,19 @@ import org.apache.spark.util.{Utils => SparkUtils}
  * @param columnName column name
  */
 private[hive] class SparkGetColumnsOperation(
-    sqlContext: SQLContext,
+    val sqlContext: SQLContext,
     parentSession: HiveSession,
     catalogName: String,
     schemaName: String,
     tableName: String,
     columnName: String)
   extends GetColumnsOperation(parentSession, catalogName, schemaName, tableName, columnName)
-    with Logging {
+  with SparkOperation
+  with Logging {
 
   val catalog: SessionCatalog = sqlContext.sessionState.catalog
 
-  private var statementId: String = _
-
-  override def close(): Unit = {
-    super.close()
-    HiveThriftServer2.eventManager.onOperationClosed(statementId)
-  }
-
   override def runInternal(): Unit = {
-    statementId = UUID.randomUUID().toString
     // Do not change cmdStr. It's used for Hive auditing and authorization.
     val cmdStr = s"catalog : $catalogName, schemaPattern : $schemaName, tablePattern : $tableName"
     val logMsg = s"Listing columns '$cmdStr, columnName : $columnName'"
@@ -116,44 +105,98 @@ private[hive] class SparkGetColumnsOperation(
       val databasePattern = Pattern.compile(CLIServiceUtils.patternToRegex(schemaName))
       if (databasePattern.matcher(globalTempViewDb).matches()) {
         catalog.globalTempViewManager.listViewNames(tablePattern).foreach { globalTempView =>
-          catalog.globalTempViewManager.get(globalTempView).foreach { plan =>
-            addToRowSet(columnPattern, globalTempViewDb, globalTempView, plan.schema)
+          catalog.getRawGlobalTempView(globalTempView).map(_.tableMeta.schema).foreach {
+            schema => addToRowSet(columnPattern, globalTempViewDb, globalTempView, schema)
           }
         }
       }
 
       // Temporary views
       catalog.listLocalTempViews(tablePattern).foreach { localTempView =>
-        catalog.getTempView(localTempView.table).foreach { plan =>
-          addToRowSet(columnPattern, null, localTempView.table, plan.schema)
+        catalog.getRawTempView(localTempView.table).map(_.tableMeta.schema).foreach {
+          schema => addToRowSet(columnPattern, null, localTempView.table, schema)
         }
       }
       setState(OperationState.FINISHED)
-    } catch {
-      case e: Throwable =>
-        logError(s"Error executing get columns operation with $statementId", e)
-        setState(OperationState.ERROR)
-        e match {
-          case hiveException: HiveSQLException =>
-            HiveThriftServer2.eventManager.onStatementError(
-              statementId, hiveException.getMessage, SparkUtils.exceptionString(hiveException))
-            throw hiveException
-          case _ =>
-            val root = ExceptionUtils.getRootCause(e)
-            HiveThriftServer2.eventManager.onStatementError(
-              statementId, root.getMessage, SparkUtils.exceptionString(root))
-            throw new HiveSQLException("Error getting columns: " + root.toString, root)
-        }
-    }
+    } catch onError()
+
     HiveThriftServer2.eventManager.onStatementFinish(statementId)
   }
 
+  /**
+   * For boolean, numeric and datetime types, it returns the default size of its catalyst type
+   * For struct type, when its elements are fixed-size, the summation of all element sizes will be
+   * returned.
+   * For array, map, string, and binaries, the column size is variable, return null as unknown.
+   */
+  private def getColumnSize(typ: DataType): Option[Int] = typ match {
+    case dt @ (BooleanType | _: NumericType | DateType | TimestampType |
+               CalendarIntervalType | NullType | YearMonthIntervalType | DayTimeIntervalType) =>
+      Some(dt.defaultSize)
+    case CharType(n) => Some(n)
+    case StructType(fields) =>
+      val sizeArr = fields.map(f => getColumnSize(f.dataType))
+      if (sizeArr.contains(None)) {
+        None
+      } else {
+        Some(sizeArr.map(_.get).sum)
+      }
+    case other => None
+  }
+
+  /**
+   * The number of fractional digits for this type.
+   * Null is returned for data types where this is not applicable.
+   * For boolean and integrals, the decimal digits is 0
+   * For floating types, we follow the IEEE Standard for Floating-Point Arithmetic (IEEE 754)
+   * For timestamp values, we support microseconds
+   * For decimals, it returns the scale
+   */
+  private def getDecimalDigits(typ: DataType) = typ match {
+    case BooleanType | _: IntegerType => Some(0)
+    case FloatType => Some(7)
+    case DoubleType => Some(15)
+    case d: DecimalType => Some(d.scale)
+    case TimestampType => Some(6)
+    case _ => None
+  }
+
+  private def getNumPrecRadix(typ: DataType): Option[Int] = typ match {
+    case _: NumericType => Some(10)
+    case _ => None
+  }
+
+  private def toJavaSQLType(typ: DataType): Integer = typ match {
+    case NullType => java.sql.Types.NULL
+    case BooleanType => java.sql.Types.BOOLEAN
+    case ByteType => java.sql.Types.TINYINT
+    case ShortType => java.sql.Types.SMALLINT
+    case IntegerType => java.sql.Types.INTEGER
+    case LongType => java.sql.Types.BIGINT
+    case FloatType => java.sql.Types.FLOAT
+    case DoubleType => java.sql.Types.DOUBLE
+    case _: DecimalType => java.sql.Types.DECIMAL
+    case StringType => java.sql.Types.VARCHAR
+    case VarcharType(_) => java.sql.Types.VARCHAR
+    case CharType(_) => java.sql.Types.CHAR
+    case BinaryType => java.sql.Types.BINARY
+    case DateType => java.sql.Types.DATE
+    case TimestampType => java.sql.Types.TIMESTAMP
+    case _: ArrayType => java.sql.Types.ARRAY
+    case _: MapType => java.sql.Types.JAVA_OBJECT
+    case _: StructType => java.sql.Types.STRUCT
+    // Hive's year-month and day-time intervals are mapping to java.sql.Types.OTHER
+    case _: CalendarIntervalType | YearMonthIntervalType | DayTimeIntervalType =>
+      java.sql.Types.OTHER
+    case _ => throw new IllegalArgumentException(s"Unrecognized type name: ${typ.sql}")
+  }
+
   private def addToRowSet(
       columnPattern: Pattern,
       dbName: String,
       tableName: String,
       schema: StructType): Unit = {
-    schema.foreach { column =>
+    schema.zipWithIndex.foreach { case (column, pos) =>
       if (columnPattern != null && !columnPattern.matcher(column.name).matches()) {
       } else {
         val rowData = Array[AnyRef](
@@ -161,19 +204,19 @@ private[hive] class SparkGetColumnsOperation(
           dbName, // TABLE_SCHEM
           tableName, // TABLE_NAME
           column.name, // COLUMN_NAME
-          toJavaSQLType(column.dataType.sql).asInstanceOf[AnyRef], // DATA_TYPE
+          toJavaSQLType(column.dataType), // DATA_TYPE
           column.dataType.sql, // TYPE_NAME
-          null, // COLUMN_SIZE
+          getColumnSize(column.dataType).map(_.asInstanceOf[AnyRef]).orNull, // COLUMN_SIZE
           null, // BUFFER_LENGTH, unused
-          null, // DECIMAL_DIGITS
-          null, // NUM_PREC_RADIX
+          getDecimalDigits(column.dataType).map(_.asInstanceOf[AnyRef]).orNull, // DECIMAL_DIGITS
+          getNumPrecRadix(column.dataType).map(_.asInstanceOf[AnyRef]).orNull, // NUM_PREC_RADIX
           (if (column.nullable) 1 else 0).asInstanceOf[AnyRef], // NULLABLE
           column.getComment().getOrElse(""), // REMARKS
           null, // COLUMN_DEF
           null, // SQL_DATA_TYPE
           null, // SQL_DATETIME_SUB
           null, // CHAR_OCTET_LENGTH
-          null, // ORDINAL_POSITION
+          pos.asInstanceOf[AnyRef], // ORDINAL_POSITION
           "YES", // IS_NULLABLE
           null, // SCOPE_CATALOG
           null, // SCOPE_SCHEMA
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetFunctionsOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetFunctionsOperation.scala
index d9c12b6ca9e64..352528e26e318 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetFunctionsOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetFunctionsOperation.scala
@@ -18,11 +18,9 @@
 package org.apache.spark.sql.hive.thriftserver
 
 import java.sql.DatabaseMetaData
-import java.util.UUID
 
 import scala.collection.JavaConverters.seqAsJavaListConverter
 
-import org.apache.commons.lang3.exception.ExceptionUtils
 import org.apache.hadoop.hive.ql.security.authorization.plugin.{HiveOperationType, HivePrivilegeObjectUtils}
 import org.apache.hive.service.cli._
 import org.apache.hive.service.cli.operation.GetFunctionsOperation
@@ -31,7 +29,6 @@ import org.apache.hive.service.cli.session.HiveSession
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.SQLContext
-import org.apache.spark.util.{Utils => SparkUtils}
 
 /**
  * Spark's own GetFunctionsOperation
@@ -43,22 +40,16 @@ import org.apache.spark.util.{Utils => SparkUtils}
  * @param functionName function name pattern
  */
 private[hive] class SparkGetFunctionsOperation(
-    sqlContext: SQLContext,
+    val sqlContext: SQLContext,
     parentSession: HiveSession,
     catalogName: String,
     schemaName: String,
     functionName: String)
-  extends GetFunctionsOperation(parentSession, catalogName, schemaName, functionName) with Logging {
-
-  private var statementId: String = _
-
-  override def close(): Unit = {
-    super.close()
-    HiveThriftServer2.eventManager.onOperationClosed(statementId)
-  }
+  extends GetFunctionsOperation(parentSession, catalogName, schemaName, functionName)
+  with SparkOperation
+  with Logging {
 
   override def runInternal(): Unit = {
-    statementId = UUID.randomUUID().toString
     // Do not change cmdStr. It's used for Hive auditing and authorization.
     val cmdStr = s"catalog : $catalogName, schemaPattern : $schemaName"
     val logMsg = s"Listing functions '$cmdStr, functionName : $functionName'"
@@ -97,29 +88,15 @@ private[hive] class SparkGetFunctionsOperation(
               DEFAULT_HIVE_CATALOG, // FUNCTION_CAT
               db, // FUNCTION_SCHEM
               funcIdentifier.funcName, // FUNCTION_NAME
-              info.getUsage, // REMARKS
+              s"Usage: ${info.getUsage}\nExtended Usage:${info.getExtended}", // REMARKS
               DatabaseMetaData.functionResultUnknown.asInstanceOf[AnyRef], // FUNCTION_TYPE
               info.getClassName) // SPECIFIC_NAME
             rowSet.addRow(rowData);
         }
       }
       setState(OperationState.FINISHED)
-    } catch {
-      case e: Throwable =>
-        logError(s"Error executing get functions operation with $statementId", e)
-        setState(OperationState.ERROR)
-        e match {
-          case hiveException: HiveSQLException =>
-            HiveThriftServer2.eventManager.onStatementError(
-              statementId, hiveException.getMessage, SparkUtils.exceptionString(hiveException))
-            throw hiveException
-          case _ =>
-            val root = ExceptionUtils.getRootCause(e)
-            HiveThriftServer2.eventManager.onStatementError(
-              statementId, root.getMessage, SparkUtils.exceptionString(root))
-            throw new HiveSQLException("Error getting functions: " + root.toString, root)
-        }
-    }
+    } catch onError()
+
     HiveThriftServer2.eventManager.onStatementFinish(statementId)
   }
 }
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetSchemasOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetSchemasOperation.scala
index db19880d1b99f..45cfa86ba9343 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetSchemasOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetSchemasOperation.scala
@@ -17,10 +17,8 @@
 
 package org.apache.spark.sql.hive.thriftserver
 
-import java.util.UUID
 import java.util.regex.Pattern
 
-import org.apache.commons.lang3.exception.ExceptionUtils
 import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType
 import org.apache.hive.service.cli._
 import org.apache.hive.service.cli.operation.GetSchemasOperation
@@ -29,7 +27,6 @@ import org.apache.hive.service.cli.session.HiveSession
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.SQLContext
-import org.apache.spark.util.{Utils => SparkUtils}
 
 /**
  * Spark's own GetSchemasOperation
@@ -40,21 +37,15 @@ import org.apache.spark.util.{Utils => SparkUtils}
  * @param schemaName database name, null or a concrete database name
  */
 private[hive] class SparkGetSchemasOperation(
-    sqlContext: SQLContext,
+    val sqlContext: SQLContext,
     parentSession: HiveSession,
     catalogName: String,
     schemaName: String)
-  extends GetSchemasOperation(parentSession, catalogName, schemaName) with Logging {
-
-  private var statementId: String = _
-
-  override def close(): Unit = {
-    super.close()
-    HiveThriftServer2.eventManager.onOperationClosed(statementId)
-  }
+  extends GetSchemasOperation(parentSession, catalogName, schemaName)
+  with SparkOperation
+  with Logging {
 
   override def runInternal(): Unit = {
-    statementId = UUID.randomUUID().toString
     // Do not change cmdStr. It's used for Hive auditing and authorization.
     val cmdStr = s"catalog : $catalogName, schemaPattern : $schemaName"
     val logMsg = s"Listing databases '$cmdStr'"
@@ -83,26 +74,13 @@ private[hive] class SparkGetSchemasOperation(
 
       val globalTempViewDb = sqlContext.sessionState.catalog.globalTempViewManager.database
       val databasePattern = Pattern.compile(CLIServiceUtils.patternToRegex(schemaName))
-      if (databasePattern.matcher(globalTempViewDb).matches()) {
+      if (schemaName == null || schemaName.isEmpty ||
+          databasePattern.matcher(globalTempViewDb).matches()) {
         rowSet.addRow(Array[AnyRef](globalTempViewDb, DEFAULT_HIVE_CATALOG))
       }
       setState(OperationState.FINISHED)
-    } catch {
-      case e: Throwable =>
-        logError(s"Error executing get schemas operation with $statementId", e)
-        setState(OperationState.ERROR)
-        e match {
-          case hiveException: HiveSQLException =>
-            HiveThriftServer2.eventManager.onStatementError(
-              statementId, hiveException.getMessage, SparkUtils.exceptionString(hiveException))
-            throw hiveException
-          case _ =>
-            val root = ExceptionUtils.getRootCause(e)
-            HiveThriftServer2.eventManager.onStatementError(
-              statementId, root.getMessage, SparkUtils.exceptionString(root))
-            throw new HiveSQLException("Error getting schemas: " + root.toString, root)
-        }
-    }
+    } catch onError()
+
     HiveThriftServer2.eventManager.onStatementFinish(statementId)
   }
 }
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTableTypesOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTableTypesOperation.scala
index b4093e58d3c07..9e31b8baad78e 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTableTypesOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTableTypesOperation.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.hive.thriftserver
 
 import java.util.UUID
 
-import org.apache.commons.lang3.exception.ExceptionUtils
 import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType
 import org.apache.hive.service.cli._
 import org.apache.hive.service.cli.operation.GetTableTypesOperation
@@ -28,7 +27,6 @@ import org.apache.hive.service.cli.session.HiveSession
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.SQLContext
 import org.apache.spark.sql.catalyst.catalog.CatalogTableType
-import org.apache.spark.util.{Utils => SparkUtils}
 
 /**
  * Spark's own GetTableTypesOperation
@@ -37,16 +35,11 @@ import org.apache.spark.util.{Utils => SparkUtils}
  * @param parentSession a HiveSession from SessionManager
  */
 private[hive] class SparkGetTableTypesOperation(
-    sqlContext: SQLContext,
+    val sqlContext: SQLContext,
     parentSession: HiveSession)
-  extends GetTableTypesOperation(parentSession) with SparkMetadataOperationUtils with Logging {
-
-  private var statementId: String = _
-
-  override def close(): Unit = {
-    super.close()
-    HiveThriftServer2.eventManager.onOperationClosed(statementId)
-  }
+  extends GetTableTypesOperation(parentSession)
+  with SparkOperation
+  with Logging {
 
   override def runInternal(): Unit = {
     statementId = UUID.randomUUID().toString
@@ -74,22 +67,8 @@ private[hive] class SparkGetTableTypesOperation(
         rowSet.addRow(Array[AnyRef](tableType))
       }
       setState(OperationState.FINISHED)
-    } catch {
-      case e: Throwable =>
-        logError(s"Error executing get table types operation with $statementId", e)
-        setState(OperationState.ERROR)
-        e match {
-          case hiveException: HiveSQLException =>
-            HiveThriftServer2.eventManager.onStatementError(
-              statementId, hiveException.getMessage, SparkUtils.exceptionString(hiveException))
-            throw hiveException
-          case _ =>
-            val root = ExceptionUtils.getRootCause(e)
-            HiveThriftServer2.eventManager.onStatementError(
-              statementId, root.getMessage, SparkUtils.exceptionString(root))
-            throw new HiveSQLException("Error getting table types: " + root.toString, root)
-        }
-    }
+    } catch onError()
+
     HiveThriftServer2.eventManager.onStatementFinish(statementId)
   }
 }
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTablesOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTablesOperation.scala
index 45c6d980aac47..bddf5eb82012f 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTablesOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTablesOperation.scala
@@ -17,14 +17,12 @@
 
 package org.apache.spark.sql.hive.thriftserver
 
-import java.util.{List => JList, UUID}
+import java.util.{List => JList}
 import java.util.regex.Pattern
 
 import scala.collection.JavaConverters._
 
-import org.apache.commons.lang3.exception.ExceptionUtils
-import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType
-import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObjectUtils
+import org.apache.hadoop.hive.ql.security.authorization.plugin.{HiveOperationType, HivePrivilegeObjectUtils}
 import org.apache.hive.service.cli._
 import org.apache.hive.service.cli.operation.GetTablesOperation
 import org.apache.hive.service.cli.session.HiveSession
@@ -32,8 +30,6 @@ import org.apache.hive.service.cli.session.HiveSession
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.SQLContext
 import org.apache.spark.sql.catalyst.catalog.CatalogTableType._
-import org.apache.spark.sql.hive.HiveUtils
-import org.apache.spark.util.{Utils => SparkUtils}
 
 /**
  * Spark's own GetTablesOperation
@@ -46,24 +42,17 @@ import org.apache.spark.util.{Utils => SparkUtils}
  * @param tableTypes list of allowed table types, e.g. "TABLE", "VIEW"
  */
 private[hive] class SparkGetTablesOperation(
-    sqlContext: SQLContext,
+    val sqlContext: SQLContext,
     parentSession: HiveSession,
     catalogName: String,
     schemaName: String,
     tableName: String,
     tableTypes: JList[String])
   extends GetTablesOperation(parentSession, catalogName, schemaName, tableName, tableTypes)
-    with SparkMetadataOperationUtils with Logging {
-
-  private var statementId: String = _
-
-  override def close(): Unit = {
-    super.close()
-    HiveThriftServer2.eventManager.onOperationClosed(statementId)
-  }
+  with SparkOperation
+  with Logging {
 
   override def runInternal(): Unit = {
-    statementId = UUID.randomUUID().toString
     // Do not change cmdStr. It's used for Hive auditing and authorization.
     val cmdStr = s"catalog : $catalogName, schemaPattern : $schemaName"
     val tableTypesStr = if (tableTypes == null) "null" else tableTypes.asScala.mkString(",")
@@ -118,22 +107,8 @@ private[hive] class SparkGetTablesOperation(
         }
       }
       setState(OperationState.FINISHED)
-    } catch {
-      case e: Throwable =>
-        logError(s"Error executing get tables operation with $statementId", e)
-        setState(OperationState.ERROR)
-        e match {
-          case hiveException: HiveSQLException =>
-            HiveThriftServer2.eventManager.onStatementError(
-              statementId, hiveException.getMessage, SparkUtils.exceptionString(hiveException))
-            throw hiveException
-          case _ =>
-            val root = ExceptionUtils.getRootCause(e)
-            HiveThriftServer2.eventManager.onStatementError(
-              statementId, root.getMessage, SparkUtils.exceptionString(root))
-            throw new HiveSQLException("Error getting tables: " + root.toString, root)
-        }
-    }
+    } catch onError()
+
     HiveThriftServer2.eventManager.onStatementFinish(statementId)
   }
 
@@ -149,10 +124,6 @@ private[hive] class SparkGetTablesOperation(
       tableType,
       comment.getOrElse(""))
     // Since HIVE-7575(Hive 2.0.0), adds 5 additional columns to the ResultSet of GetTables.
-    if (HiveUtils.isHive23) {
-      rowSet.addRow(rowData ++ Array(null, null, null, null, null))
-    } else {
-      rowSet.addRow(rowData)
-    }
+    rowSet.addRow(rowData ++ Array(null, null, null, null, null))
   }
 }
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTypeInfoOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTypeInfoOperation.scala
index dd5668a93f82d..cecb0dec72c80 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTypeInfoOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTypeInfoOperation.scala
@@ -19,15 +19,15 @@ package org.apache.spark.sql.hive.thriftserver
 
 import java.util.UUID
 
-import org.apache.commons.lang3.exception.ExceptionUtils
 import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType
-import org.apache.hive.service.cli.{HiveSQLException, OperationState}
+import org.apache.hadoop.hive.serde2.thrift.Type
+import org.apache.hadoop.hive.serde2.thrift.Type._
+import org.apache.hive.service.cli.OperationState
 import org.apache.hive.service.cli.operation.GetTypeInfoOperation
 import org.apache.hive.service.cli.session.HiveSession
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.SQLContext
-import org.apache.spark.util.{Utils => SparkUtils}
 
 /**
  * Spark's own GetTypeInfoOperation
@@ -36,16 +36,11 @@ import org.apache.spark.util.{Utils => SparkUtils}
  * @param parentSession a HiveSession from SessionManager
  */
 private[hive] class SparkGetTypeInfoOperation(
-    sqlContext: SQLContext,
+    val sqlContext: SQLContext,
     parentSession: HiveSession)
-  extends GetTypeInfoOperation(parentSession) with Logging {
-
-  private var statementId: String = _
-
-  override def close(): Unit = {
-    super.close()
-    HiveThriftServer2.eventManager.onOperationClosed(statementId)
-  }
+  extends GetTypeInfoOperation(parentSession)
+  with SparkOperation
+  with Logging {
 
   override def runInternal(): Unit = {
     statementId = UUID.randomUUID().toString
@@ -68,7 +63,7 @@ private[hive] class SparkGetTypeInfoOperation(
       parentSession.getUsername)
 
     try {
-      ThriftserverShimUtils.supportedType().foreach(typeInfo => {
+      SparkGetTypeInfoUtil.supportedType.foreach(typeInfo => {
         val rowData = Array[AnyRef](
           typeInfo.getName, // TYPE_NAME
           typeInfo.toJavaSQLType.asInstanceOf[AnyRef], // DATA_TYPE
@@ -92,22 +87,19 @@ private[hive] class SparkGetTypeInfoOperation(
         rowSet.addRow(rowData)
       })
       setState(OperationState.FINISHED)
-    } catch {
-      case e: Throwable =>
-        logError(s"Error executing get type info with $statementId", e)
-        setState(OperationState.ERROR)
-        e match {
-          case hiveException: HiveSQLException =>
-            HiveThriftServer2.eventManager.onStatementError(
-              statementId, hiveException.getMessage, SparkUtils.exceptionString(hiveException))
-            throw hiveException
-          case _ =>
-            val root = ExceptionUtils.getRootCause(e)
-            HiveThriftServer2.eventManager.onStatementError(
-              statementId, root.getMessage, SparkUtils.exceptionString(root))
-            throw new HiveSQLException("Error getting type info: " + root.toString, root)
-        }
-    }
+    } catch onError()
+
     HiveThriftServer2.eventManager.onStatementFinish(statementId)
   }
 }
+
+private[hive] object SparkGetTypeInfoUtil {
+  val supportedType: Seq[Type] = {
+    Seq(NULL_TYPE, BOOLEAN_TYPE, STRING_TYPE, BINARY_TYPE,
+      TINYINT_TYPE, SMALLINT_TYPE, INT_TYPE, BIGINT_TYPE,
+      FLOAT_TYPE, DOUBLE_TYPE, DECIMAL_TYPE,
+      DATE_TYPE, TIMESTAMP_TYPE,
+      ARRAY_TYPE, MAP_TYPE, STRUCT_TYPE, CHAR_TYPE, VARCHAR_TYPE,
+      INTERVAL_YEAR_MONTH_TYPE, INTERVAL_DAY_TIME_TYPE)
+  }
+}
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationUtils.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationUtils.scala
deleted file mode 100644
index f4c4b04bada2a..0000000000000
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationUtils.scala
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.hive.thriftserver
-
-import org.apache.spark.sql.catalyst.catalog.CatalogTableType
-import org.apache.spark.sql.catalyst.catalog.CatalogTableType.{EXTERNAL, MANAGED, VIEW}
-
-/**
- * Utils for metadata operations.
- */
-private[hive] trait SparkMetadataOperationUtils {
-
-  def tableTypeString(tableType: CatalogTableType): String = tableType match {
-    case EXTERNAL | MANAGED => "TABLE"
-    case VIEW => "VIEW"
-    case t =>
-      throw new IllegalArgumentException(s"Unknown table type is found: $t")
-  }
-}
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkOperation.scala
new file mode 100644
index 0000000000000..bbfc1b83379aa
--- /dev/null
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkOperation.scala
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.thriftserver
+
+import org.apache.hive.service.cli.{HiveSQLException, OperationState}
+import org.apache.hive.service.cli.operation.Operation
+
+import org.apache.spark.SparkContext
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.{SparkSession, SQLContext}
+import org.apache.spark.sql.catalyst.catalog.CatalogTableType
+import org.apache.spark.sql.catalyst.catalog.CatalogTableType.{EXTERNAL, MANAGED, VIEW}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.util.Utils
+
+/**
+ * Utils for Spark operations.
+ */
+private[hive] trait SparkOperation extends Operation with Logging {
+
+  protected def sqlContext: SQLContext
+
+  protected var statementId = getHandle().getHandleIdentifier().getPublicId().toString()
+
+  protected def cleanup(): Unit = () // noop by default
+
+  abstract override def run(): Unit = {
+    withLocalProperties {
+      super.run()
+    }
+  }
+
+  abstract override def close(): Unit = {
+    super.close()
+    cleanup()
+    logInfo(s"Close statement with $statementId")
+    HiveThriftServer2.eventManager.onOperationClosed(statementId)
+  }
+
+  // Set thread local properties for the execution of the operation.
+  // This method should be applied during the execution of the operation, by all the child threads.
+  // The original spark context local properties will be restored after the operation.
+  //
+  // It is used to:
+  // - set appropriate SparkSession
+  // - set scheduler pool for the operation
+  def withLocalProperties[T](f: => T): T = {
+    val originalProps = Utils.cloneProperties(sqlContext.sparkContext.getLocalProperties)
+    val originalSession = SparkSession.getActiveSession
+
+    try {
+      // Set active SparkSession
+      SparkSession.setActiveSession(sqlContext.sparkSession)
+
+      // Set scheduler pool
+      sqlContext.sparkSession.conf.getOption(SQLConf.THRIFTSERVER_POOL.key) match {
+        case Some(pool) =>
+          sqlContext.sparkContext.setLocalProperty(SparkContext.SPARK_SCHEDULER_POOL, pool)
+        case None =>
+      }
+
+      // run the body
+      f
+    } finally {
+      // reset local properties, will also reset SPARK_SCHEDULER_POOL
+      sqlContext.sparkContext.setLocalProperties(originalProps)
+
+      originalSession match {
+        case Some(session) => SparkSession.setActiveSession(session)
+        case None => SparkSession.clearActiveSession()
+      }
+    }
+  }
+
+  def tableTypeString(tableType: CatalogTableType): String = tableType match {
+    case EXTERNAL | MANAGED => "TABLE"
+    case VIEW => "VIEW"
+    case t =>
+      throw new IllegalArgumentException(s"Unknown table type is found: $t")
+  }
+
+  protected def onError(): PartialFunction[Throwable, Unit] = {
+    case e: Throwable =>
+      logError(s"Error operating $getType with $statementId", e)
+      super.setState(OperationState.ERROR)
+      HiveThriftServer2.eventManager.onStatementError(
+        statementId, e.getMessage, Utils.exceptionString(e))
+      e match {
+        case _: HiveSQLException => throw e
+        case _ => throw new HiveSQLException(s"Error operating $getType ${e.getMessage}", e)
+      }
+  }
+}
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
index c7848afd822d5..c3d3f29338b92 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
@@ -37,6 +37,7 @@ import org.apache.hadoop.hive.ql.session.SessionState
 import org.apache.hadoop.security.{Credentials, UserGroupInformation}
 import org.apache.log4j.Level
 import org.apache.thrift.transport.TSocket
+import org.slf4j.LoggerFactory
 import sun.misc.{Signal, SignalHandler}
 
 import org.apache.spark.SparkConf
@@ -59,6 +60,7 @@ private[hive] object SparkSQLCLIDriver extends Logging {
   private var transport: TSocket = _
   private final val SPARK_HADOOP_PROP_PREFIX = "spark.hadoop."
 
+  initializeLogIfNecessary(true)
   installSignalHandler()
 
   /**
@@ -131,7 +133,9 @@ private[hive] object SparkSQLCLIDriver extends Logging {
       UserGroupInformation.getCurrentUser.addCredentials(credentials)
     }
 
-    SharedState.loadHiveConfFile(sparkConf, conf)
+    val warehousePath = SharedState.resolveWarehousePath(sparkConf, conf)
+    val qualified = SharedState.qualifyWarehousePath(conf, warehousePath)
+    SharedState.setWarehousePathConf(sparkConf, conf, qualified)
     SessionState.start(sessionState)
 
     // Clean up after we exit
@@ -165,10 +169,10 @@ private[hive] object SparkSQLCLIDriver extends Logging {
     }
 
     // The class loader of CliSessionState's conf is current main thread's class loader
-    // used to load jars passed by --jars. One class loader used by AddJarCommand is
+    // used to load jars passed by --jars. One class loader used by AddJarsCommand is
     // sharedState.jarClassLoader which contain jar path passed by --jars in main thread.
     // We set CliSessionState's conf class loader to sharedState.jarClassLoader.
-    // Thus we can load all jars passed by --jars and AddJarCommand.
+    // Thus we can load all jars passed by --jars and AddJarsCommand.
     sessionState.getConf.setClassLoader(SparkSQLEnv.sqlContext.sharedState.jarClassLoader)
 
     // TODO work around for set the log output to console, because the HiveContext
@@ -197,6 +201,8 @@ private[hive] object SparkSQLCLIDriver extends Logging {
       SparkSQLEnv.sqlContext.setConf(k, v)
     }
 
+    cli.printMasterAndAppId
+
     if (sessionState.execString != null) {
       System.exit(cli.processLine(sessionState.execString))
     }
@@ -266,8 +272,6 @@ private[hive] object SparkSQLCLIDriver extends Logging {
     def continuedPromptWithDBSpaces: String = continuedPrompt + ReflectionUtils.invokeStatic(
       classOf[CliDriver], "spacesForString", classOf[String] -> currentDB)
 
-    cli.printMasterAndAppId
-
     var currentPrompt = promptWithCurrentDB
     var line = reader.readLine(currentPrompt + "> ")
 
@@ -306,7 +310,9 @@ private[hive] object SparkSQLCLIDriver extends Logging {
 private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
   private val sessionState = SessionState.get().asInstanceOf[CliSessionState]
 
-  private val console = ThriftserverShimUtils.getConsole
+  private val LOG = LoggerFactory.getLogger(classOf[SparkSQLCLIDriver])
+
+  private val console = new SessionState.LogHelper(LOG)
 
   private val isRemoteMode = {
     SparkSQLCLIDriver.isRemoteMode(sessionState)
@@ -400,7 +406,8 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
 
           val res = new JArrayList[String]()
 
-          if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CLI_PRINT_HEADER)) {
+          if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CLI_PRINT_HEADER) ||
+              SparkSQLEnv.sqlContext.conf.cliPrintHeader) {
             // Print the column names.
             Option(driver.getSchema.getFieldSchemas).foreach { fields =>
               out.println(fields.asScala.map(_.getName).mkString("\t"))
@@ -461,7 +468,7 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
       oldSignal = Signal.handle(interruptSignal, new SignalHandler() {
         private var interruptRequested: Boolean = false
 
-        override def handle(signal: Signal) {
+        override def handle(signal: Signal): Unit = {
           val initialRequest = !interruptRequested
           interruptRequested = true
 
@@ -499,7 +506,7 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
             val ignoreErrors = HiveConf.getBoolVar(conf, HiveConf.ConfVars.CLIIGNOREERRORS)
             if (ret != 0 && !ignoreErrors) {
               CommandProcessorFactory.clean(conf.asInstanceOf[HiveConf])
-              ret
+              return ret
             }
           }
         }
@@ -518,15 +525,32 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
   // Note: [SPARK-31595] if there is a `'` in a double quoted string, or a `"` in a single quoted
   // string, the origin implementation from Hive will not drop the trailing semicolon as expected,
   // hence we refined this function a little bit.
+  // Note: [SPARK-33100] Ignore a semicolon inside a bracketed comment in spark-sql.
   private def splitSemiColon(line: String): JList[String] = {
     var insideSingleQuote = false
     var insideDoubleQuote = false
-    var insideComment = false
+    var insideSimpleComment = false
+    var bracketedCommentLevel = 0
     var escape = false
     var beginIndex = 0
+    var leavingBracketedComment = false
+    var isStatement = false
     val ret = new JArrayList[String]
 
+    def insideBracketedComment: Boolean = bracketedCommentLevel > 0
+    def insideComment: Boolean = insideSimpleComment || insideBracketedComment
+    def statementInProgress(index: Int): Boolean = isStatement || (!insideComment &&
+      index > beginIndex && !s"${line.charAt(index)}".trim.isEmpty)
+
     for (index <- 0 until line.length) {
+      // Checks if we need to decrement a bracketed comment level; the last character '/' of
+      // bracketed comments is still inside the comment, so `insideBracketedComment` must keep true
+      // in the previous loop and we decrement the level here if needed.
+      if (leavingBracketedComment) {
+        bracketedCommentLevel -= 1
+        leavingBracketedComment = false
+      }
+
       if (line.charAt(index) == '\'' && !insideComment) {
         // take a look to see if it is escaped
         // See the comment above about SPARK-31595
@@ -549,21 +573,34 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
           // Sample query: select "quoted value --"
           //                                    ^^ avoids starting a comment if it's inside quotes.
         } else if (hasNext && line.charAt(index + 1) == '-') {
-          // ignore quotes and ;
-          insideComment = true
+          // ignore quotes and ; in simple comment
+          insideSimpleComment = true
         }
       } else if (line.charAt(index) == ';') {
         if (insideSingleQuote || insideDoubleQuote || insideComment) {
           // do not split
         } else {
-          // split, do not include ; itself
-          ret.add(line.substring(beginIndex, index))
+          if (isStatement) {
+            // split, do not include ; itself
+            ret.add(line.substring(beginIndex, index))
+          }
           beginIndex = index + 1
+          isStatement = false
         }
       } else if (line.charAt(index) == '\n') {
-        // with a new line the inline comment should end.
+        // with a new line the inline simple comment should end.
         if (!escape) {
-          insideComment = false
+          insideSimpleComment = false
+        }
+      } else if (line.charAt(index) == '/' && !insideSimpleComment) {
+        val hasNext = index + 1 < line.length
+        if (insideSingleQuote || insideDoubleQuote) {
+          // Ignores '/' in any case of quotes
+        } else if (insideBracketedComment && line.charAt(index - 1) == '*' ) {
+          // Decrements `bracketedCommentLevel` at the beginning of the next loop
+          leavingBracketedComment = true
+        } else if (hasNext && !insideBracketedComment && line.charAt(index + 1) == '*') {
+          bracketedCommentLevel += 1
         }
       }
       // set the escape
@@ -572,8 +609,12 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
       } else if (line.charAt(index) == '\\') {
         escape = true
       }
+
+      isStatement = statementInProgress(index)
+    }
+    if (isStatement) {
+      ret.add(line.substring(beginIndex))
     }
-    ret.add(line.substring(beginIndex))
     ret
   }
 }
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIService.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIService.scala
index 1644ecb2453be..e9420ad21bebd 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIService.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIService.scala
@@ -22,13 +22,13 @@ import java.util.{List => JList}
 import javax.security.auth.login.LoginException
 
 import scala.collection.JavaConverters._
+import scala.util.control.NonFatal
 
-import org.apache.commons.logging.Log
 import org.apache.hadoop.hive.conf.HiveConf
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars
 import org.apache.hadoop.hive.shims.Utils
 import org.apache.hadoop.security.{SecurityUtil, UserGroupInformation}
-import org.apache.hive.service.{AbstractService, Service, ServiceException}
+import org.apache.hive.service.{AbstractService, CompositeService, Service, ServiceException}
 import org.apache.hive.service.Service.STATE
 import org.apache.hive.service.auth.HiveAuthFactory
 import org.apache.hive.service.cli._
@@ -36,7 +36,6 @@ import org.apache.hive.service.server.HiveServer2
 import org.slf4j.Logger
 
 import org.apache.spark.sql.SQLContext
-import org.apache.spark.sql.hive.HiveUtils
 import org.apache.spark.sql.hive.thriftserver.ReflectionUtils._
 
 private[hive] class SparkSQLCLIService(hiveServer: HiveServer2, sqlContext: SQLContext)
@@ -94,17 +93,30 @@ private[hive] class SparkSQLCLIService(hiveServer: HiveServer2, sqlContext: SQLC
     initCompositeService(hiveConf)
   }
 
+  /**
+   * the super class [[CLIService#start]] starts a useless dummy metastore client, skip it and call
+   * the ancestor [[CompositeService#start]] directly.
+   */
+  override def start(): Unit = startCompositeService()
+
   override def getInfo(sessionHandle: SessionHandle, getInfoType: GetInfoType): GetInfoValue = {
     getInfoType match {
       case GetInfoType.CLI_SERVER_NAME => new GetInfoValue("Spark SQL")
       case GetInfoType.CLI_DBMS_NAME => new GetInfoValue("Spark SQL")
       case GetInfoType.CLI_DBMS_VER => new GetInfoValue(sqlContext.sparkContext.version)
+      case GetInfoType.CLI_ODBC_KEYWORDS => new GetInfoValue("Unimplemented")
       case _ => super.getInfo(sessionHandle, getInfoType)
     }
   }
 }
 
 private[thriftserver] trait ReflectedCompositeService { this: AbstractService =>
+
+  private val logInfo = (msg: String) => getAncestorField[Logger](this, 3, "LOG").info(msg)
+
+  private val logError = (msg: String, e: Throwable) =>
+    getAncestorField[Logger](this, 3, "LOG").error(msg, e)
+
   def initCompositeService(hiveConf: HiveConf): Unit = {
     // Emulating `CompositeService.init(hiveConf)`
     val serviceList = getAncestorField[JList[Service]](this, 2, "serviceList")
@@ -114,10 +126,30 @@ private[thriftserver] trait ReflectedCompositeService { this: AbstractService =>
     invoke(classOf[AbstractService], this, "ensureCurrentState", classOf[STATE] -> STATE.NOTINITED)
     setAncestorField(this, 3, "hiveConf", hiveConf)
     invoke(classOf[AbstractService], this, "changeState", classOf[STATE] -> STATE.INITED)
-    if (HiveUtils.isHive23) {
-      getAncestorField[Logger](this, 3, "LOG").info(s"Service: $getName is inited.")
-    } else {
-      getAncestorField[Log](this, 3, "LOG").info(s"Service: $getName is inited.")
+    logInfo(s"Service: $getName is inited.")
+  }
+
+  def startCompositeService(): Unit = {
+    // Emulating `CompositeService.start`
+    val serviceList = getAncestorField[JList[Service]](this, 2, "serviceList")
+    var serviceStartCount = 0
+    try {
+      serviceList.asScala.foreach { service =>
+        service.start()
+        serviceStartCount += 1
+      }
+      // Emulating `AbstractService.start`
+      val startTime = new java.lang.Long(System.currentTimeMillis())
+      setAncestorField(this, 3, "startTime", startTime)
+      invoke(classOf[AbstractService], this, "ensureCurrentState", classOf[STATE] -> STATE.INITED)
+      invoke(classOf[AbstractService], this, "changeState", classOf[STATE] -> STATE.STARTED)
+      logInfo(s"Service: $getName is started.")
+    } catch {
+      case NonFatal(e) =>
+      logError(s"Error starting services $getName", e)
+      invoke(classOf[CompositeService], this, "stop",
+        classOf[Int] -> new Integer(serviceStartCount))
+      throw new ServiceException("Failed to Start " + getName, e)
     }
   }
 }
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
index 12fba0eae6dce..8bc762fe99233 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
@@ -30,6 +30,7 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.sql.{AnalysisException, SQLContext}
 import org.apache.spark.sql.execution.{QueryExecution, SQLExecution}
 import org.apache.spark.sql.execution.HiveResult.hiveResultString
+import org.apache.spark.sql.internal.{SQLConf, VariableSubstitution}
 
 
 private[hive] class SparkSQLDriver(val context: SQLContext = SparkSQLEnv.sqlContext)
@@ -59,7 +60,10 @@ private[hive] class SparkSQLDriver(val context: SQLContext = SparkSQLEnv.sqlCont
   override def run(command: String): CommandProcessorResponse = {
     // TODO unify the error code
     try {
-      context.sparkContext.setJobDescription(command)
+      val substitutorCommand = SQLConf.withExistingConf(context.conf) {
+        new VariableSubstitution().substitute(command)
+      }
+      context.sparkContext.setJobDescription(substitutorCommand)
       val execution = context.sessionState.executePlan(context.sql(command).logicalPlan)
       hiveResponse = SQLExecution.withNewExecutionId(execution) {
         hiveResultString(execution.executedPlan)
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala
index 8944b93d9b697..291e426f148df 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala
@@ -24,9 +24,10 @@ import org.apache.spark.{SparkConf, SparkContext}
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.{SparkSession, SQLContext}
 import org.apache.spark.sql.hive.{HiveExternalCatalog, HiveUtils}
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.util.Utils
 
-/** A singleton object for the master program. The slaves should not access this. */
+/** A singleton object for the master program. The executors should not access this. */
 private[hive] object SparkSQLEnv extends Logging {
   logDebug("Initializing SparkSQLEnv")
 
@@ -45,8 +46,13 @@ private[hive] object SparkSQLEnv extends Logging {
 
       sparkConf
         .setAppName(maybeAppName.getOrElse(s"SparkSQL::${Utils.localHostName()}"))
+        .set(SQLConf.DATETIME_JAVA8API_ENABLED, true)
 
-      val sparkSession = SparkSession.builder.config(sparkConf).enableHiveSupport().getOrCreate()
+
+      val sparkSession = SparkSession.builder()
+        .config(sparkConf)
+        .config(HiveUtils.BUILTIN_HIVE_VERSION.key, HiveUtils.builtinHiveVersion)
+        .enableHiveSupport().getOrCreate()
       sparkContext = sparkSession.sparkContext
       sqlContext = sparkSession.sqlContext
 
@@ -60,7 +66,6 @@ private[hive] object SparkSQLEnv extends Logging {
       metadataHive.setOut(new PrintStream(System.out, true, UTF_8.name()))
       metadataHive.setInfo(new PrintStream(System.err, true, UTF_8.name()))
       metadataHive.setError(new PrintStream(System.err, true, UTF_8.name()))
-      sparkSession.conf.set(HiveUtils.FAKE_HIVE_VERSION.key, HiveUtils.builtinHiveVersion)
     }
   }
 
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLSessionManager.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLSessionManager.scala
index b3171897141c2..2312c8ed714c2 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLSessionManager.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLSessionManager.scala
@@ -17,24 +17,24 @@
 
 package org.apache.spark.sql.hive.thriftserver
 
-import java.util.concurrent.Executors
+import scala.util.control.NonFatal
 
-import org.apache.commons.logging.Log
 import org.apache.hadoop.hive.conf.HiveConf
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars
-import org.apache.hive.service.cli.SessionHandle
+import org.apache.hive.service.cli.{HiveSQLException, SessionHandle}
 import org.apache.hive.service.cli.session.SessionManager
+import org.apache.hive.service.rpc.thrift.TProtocolVersion
 import org.apache.hive.service.server.HiveServer2
 
+import org.apache.spark.internal.Logging
 import org.apache.spark.sql.SQLContext
-import org.apache.spark.sql.hive.HiveUtils
 import org.apache.spark.sql.hive.thriftserver.ReflectionUtils._
 import org.apache.spark.sql.hive.thriftserver.server.SparkSQLOperationManager
+import org.apache.spark.sql.internal.SQLConf
 
 
 private[hive] class SparkSQLSessionManager(hiveServer: HiveServer2, sqlContext: SQLContext)
   extends SessionManager(hiveServer)
-  with ReflectedCompositeService {
+  with ReflectedCompositeService with Logging {
 
   private lazy val sparkSqlOperationManager = new SparkSQLOperationManager()
 
@@ -44,7 +44,7 @@ private[hive] class SparkSQLSessionManager(hiveServer: HiveServer2, sqlContext:
   }
 
   override def openSession(
-      protocol: ThriftserverShimUtils.TProtocolVersion,
+      protocol: TProtocolVersion,
       username: String,
       passwd: String,
       ipAddress: String,
@@ -54,23 +54,34 @@ private[hive] class SparkSQLSessionManager(hiveServer: HiveServer2, sqlContext:
     val sessionHandle =
       super.openSession(protocol, username, passwd, ipAddress, sessionConf, withImpersonation,
           delegationToken)
-    val session = super.getSession(sessionHandle)
-    HiveThriftServer2.eventManager.onSessionCreated(
-      session.getIpAddress, sessionHandle.getSessionId.toString, session.getUsername)
-    val ctx = if (sqlContext.conf.hiveThriftServerSingleSession) {
-      sqlContext
-    } else {
-      sqlContext.newSession()
+    try {
+      val session = super.getSession(sessionHandle)
+      HiveThriftServer2.eventManager.onSessionCreated(
+        session.getIpAddress, sessionHandle.getSessionId.toString, session.getUsername)
+      val ctx = if (sqlContext.conf.hiveThriftServerSingleSession) {
+        sqlContext
+      } else {
+        sqlContext.newSession()
+      }
+      ctx.setConf(SQLConf.DATETIME_JAVA8API_ENABLED, true)
+      val hiveSessionState = session.getSessionState
+      setConfMap(ctx, hiveSessionState.getOverriddenConfigurations)
+      setConfMap(ctx, hiveSessionState.getHiveVariables)
+      if (sessionConf != null && sessionConf.containsKey("use:database")) {
+        ctx.sql(s"use ${sessionConf.get("use:database")}")
+      }
+      sparkSqlOperationManager.sessionToContexts.put(sessionHandle, ctx)
+      sessionHandle
+    } catch {
+      case NonFatal(e) =>
+        try {
+          closeSession(sessionHandle)
+        } catch {
+          case NonFatal(inner) =>
+            logWarning("Error closing session", inner)
+        }
+        throw new HiveSQLException("Failed to open new session: " + e, e)
     }
-    ctx.setConf(HiveUtils.FAKE_HIVE_VERSION.key, HiveUtils.builtinHiveVersion)
-    val hiveSessionState = session.getSessionState
-    setConfMap(ctx, hiveSessionState.getOverriddenConfigurations)
-    setConfMap(ctx, hiveSessionState.getHiveVariables)
-    if (sessionConf != null && sessionConf.containsKey("use:database")) {
-      ctx.sql(s"use ${sessionConf.get("use:database")}")
-    }
-    sparkSqlOperationManager.sessionToContexts.put(sessionHandle, ctx)
-    sessionHandle
   }
 
   override def closeSession(sessionHandle: SessionHandle): Unit = {
@@ -78,7 +89,6 @@ private[hive] class SparkSQLSessionManager(hiveServer: HiveServer2, sqlContext:
     val ctx = sparkSqlOperationManager.sessionToContexts.getOrDefault(sessionHandle, sqlContext)
     ctx.sparkSession.sessionState.catalog.getTempViewNames().foreach(ctx.uncacheTable)
     super.closeSession(sessionHandle)
-    sparkSqlOperationManager.sessionToActivePool.remove(sessionHandle)
     sparkSqlOperationManager.sessionToContexts.remove(sessionHandle)
   }
 
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/server/SparkSQLOperationManager.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/server/SparkSQLOperationManager.scala
index 3396560f43502..ba42eefed2a22 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/server/SparkSQLOperationManager.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/server/SparkSQLOperationManager.scala
@@ -38,21 +38,21 @@ private[thriftserver] class SparkSQLOperationManager()
   val handleToOperation = ReflectionUtils
     .getSuperField[JMap[OperationHandle, Operation]](this, "handleToOperation")
 
-  val sessionToActivePool = new ConcurrentHashMap[SessionHandle, String]()
   val sessionToContexts = new ConcurrentHashMap[SessionHandle, SQLContext]()
 
   override def newExecuteStatementOperation(
       parentSession: HiveSession,
       statement: String,
       confOverlay: JMap[String, String],
-      async: Boolean): ExecuteStatementOperation = synchronized {
+      async: Boolean,
+      queryTimeout: Long): ExecuteStatementOperation = synchronized {
     val sqlContext = sessionToContexts.get(parentSession.getSessionHandle)
     require(sqlContext != null, s"Session handle: ${parentSession.getSessionHandle} has not been" +
       s" initialized or had already closed.")
     val conf = sqlContext.sessionState.conf
     val runInBackground = async && conf.getConf(HiveUtils.HIVE_THRIFT_SERVER_ASYNC)
-    val operation = new SparkExecuteStatementOperation(parentSession, statement, confOverlay,
-      runInBackground)(sqlContext, sessionToActivePool)
+    val operation = new SparkExecuteStatementOperation(
+      sqlContext, parentSession, statement, confOverlay, runInBackground, queryTimeout)
     handleToOperation.put(operation.getHandle, operation)
     logDebug(s"Created Operation for $statement with session=$parentSession, " +
       s"runInBackground=$runInBackground")
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2AppStatusStore.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2AppStatusStore.scala
index 5cb78f6e64650..8bd8f29a4b9ec 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2AppStatusStore.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2AppStatusStore.scala
@@ -119,6 +119,7 @@ private[thriftserver] class ExecutionInfo(
   def isExecutionActive: Boolean = {
     !(state == ExecutionState.FAILED ||
       state == ExecutionState.CANCELED ||
+      state == ExecutionState.TIMEDOUT ||
       state == ExecutionState.CLOSED)
   }
 
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2EventManager.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2EventManager.scala
index fa04c67896a69..202fdf33c0dd9 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2EventManager.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2EventManager.scala
@@ -57,6 +57,10 @@ private[thriftserver] class HiveThriftServer2EventManager(sc: SparkContext) {
     postLiveListenerBus(SparkListenerThriftServerOperationCanceled(id, System.currentTimeMillis()))
   }
 
+  def onStatementTimeout(id: String): Unit = {
+    postLiveListenerBus(SparkListenerThriftServerOperationTimeout(id, System.currentTimeMillis()))
+  }
+
   def onStatementError(id: String, errorMsg: String, errorTrace: String): Unit = {
     postLiveListenerBus(SparkListenerThriftServerOperationError(id, errorMsg, errorTrace,
       System.currentTimeMillis()))
@@ -96,6 +100,9 @@ private[thriftserver] case class SparkListenerThriftServerOperationParsed(
 private[thriftserver] case class SparkListenerThriftServerOperationCanceled(
     id: String, finishTime: Long) extends SparkListenerEvent
 
+private[thriftserver] case class SparkListenerThriftServerOperationTimeout(
+    id: String, finishTime: Long) extends SparkListenerEvent
+
 private[thriftserver] case class SparkListenerThriftServerOperationError(
     id: String,
     errorMsg: String,
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2Listener.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2Listener.scala
index 6d0a506fa94dc..4cf672e3d9d9e 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2Listener.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2Listener.scala
@@ -25,6 +25,7 @@ import scala.collection.mutable.ArrayBuffer
 import org.apache.hive.service.server.HiveServer2
 
 import org.apache.spark.{SparkConf, SparkContext}
+import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.Status.LIVE_ENTITY_UPDATE_PERIOD
 import org.apache.spark.scheduler._
 import org.apache.spark.sql.hive.thriftserver.HiveThriftServer2.ExecutionState
@@ -38,7 +39,7 @@ private[thriftserver] class HiveThriftServer2Listener(
     kvstore: ElementTrackingStore,
     sparkConf: SparkConf,
     server: Option[HiveServer2],
-    live: Boolean = true) extends SparkListener {
+    live: Boolean = true) extends SparkListener with Logging {
 
   private val sessionList = new ConcurrentHashMap[String, LiveSessionData]()
   private val executionList = new ConcurrentHashMap[String, LiveExecutionData]()
@@ -118,6 +119,7 @@ private[thriftserver] class HiveThriftServer2Listener(
       case e: SparkListenerThriftServerOperationStart => onOperationStart(e)
       case e: SparkListenerThriftServerOperationParsed => onOperationParsed(e)
       case e: SparkListenerThriftServerOperationCanceled => onOperationCanceled(e)
+      case e: SparkListenerThriftServerOperationTimeout => onOperationTimeout(e)
       case e: SparkListenerThriftServerOperationError => onOperationError(e)
       case e: SparkListenerThriftServerOperationFinish => onOperationFinished(e)
       case e: SparkListenerThriftServerOperationClosed => onOperationClosed(e)
@@ -131,60 +133,92 @@ private[thriftserver] class HiveThriftServer2Listener(
     updateLiveStore(session)
   }
 
-  private def onSessionClosed(e: SparkListenerThriftServerSessionClosed): Unit = {
-    val session = sessionList.get(e.sessionId)
-    session.finishTimestamp = e.finishTime
-    updateStoreWithTriggerEnabled(session)
-    sessionList.remove(e.sessionId)
-  }
+  private def onSessionClosed(e: SparkListenerThriftServerSessionClosed): Unit =
+    Option(sessionList.get(e.sessionId)) match {
+      case Some(sessionData) =>
+        sessionData.finishTimestamp = e.finishTime
+        updateStoreWithTriggerEnabled(sessionData)
+        sessionList.remove(e.sessionId)
+      case None => logWarning(s"onSessionClosed called with unknown session id: ${e.sessionId}")
+    }
 
   private def onOperationStart(e: SparkListenerThriftServerOperationStart): Unit = {
-    val info = getOrCreateExecution(
+    val executionData = getOrCreateExecution(
       e.id,
       e.statement,
       e.sessionId,
       e.startTime,
       e.userName)
 
-    info.state = ExecutionState.STARTED
-    executionList.put(e.id, info)
-    sessionList.get(e.sessionId).totalExecution += 1
-    executionList.get(e.id).groupId = e.groupId
-    updateLiveStore(executionList.get(e.id))
-    updateLiveStore(sessionList.get(e.sessionId))
+    executionData.state = ExecutionState.STARTED
+    executionList.put(e.id, executionData)
+    executionData.groupId = e.groupId
+    updateLiveStore(executionData)
+
+    Option(sessionList.get(e.sessionId)) match {
+      case Some(sessionData) =>
+        sessionData.totalExecution += 1
+        updateLiveStore(sessionData)
+      case None => logWarning(s"onOperationStart called with unknown session id: ${e.sessionId}." +
+        s"Regardless, the operation has been registered.")
+    }
   }
 
-  private def onOperationParsed(e: SparkListenerThriftServerOperationParsed): Unit = {
-    executionList.get(e.id).executePlan = e.executionPlan
-    executionList.get(e.id).state = ExecutionState.COMPILED
-    updateLiveStore(executionList.get(e.id))
-  }
+  private def onOperationParsed(e: SparkListenerThriftServerOperationParsed): Unit =
+    Option(executionList.get(e.id)) match {
+      case Some(executionData) =>
+        executionData.executePlan = e.executionPlan
+        executionData.state = ExecutionState.COMPILED
+        updateLiveStore(executionData)
+      case None => logWarning(s"onOperationParsed called with unknown operation id: ${e.id}")
+    }
 
-  private def onOperationCanceled(e: SparkListenerThriftServerOperationCanceled): Unit = {
-    executionList.get(e.id).finishTimestamp = e.finishTime
-    executionList.get(e.id).state = ExecutionState.CANCELED
-    updateLiveStore(executionList.get(e.id))
-  }
+  private def onOperationCanceled(e: SparkListenerThriftServerOperationCanceled): Unit =
+    Option(executionList.get(e.id)) match {
+      case Some(executionData) =>
+        executionData.finishTimestamp = e.finishTime
+        executionData.state = ExecutionState.CANCELED
+        updateLiveStore(executionData)
+      case None => logWarning(s"onOperationCanceled called with unknown operation id: ${e.id}")
+    }
 
-  private def onOperationError(e: SparkListenerThriftServerOperationError): Unit = {
-    executionList.get(e.id).finishTimestamp = e.finishTime
-    executionList.get(e.id).detail = e.errorMsg
-    executionList.get(e.id).state = ExecutionState.FAILED
-    updateLiveStore(executionList.get(e.id))
-  }
+  private def onOperationTimeout(e: SparkListenerThriftServerOperationTimeout): Unit =
+    Option(executionList.get(e.id)) match {
+      case Some(executionData) =>
+        executionData.finishTimestamp = e.finishTime
+        executionData.state = ExecutionState.TIMEDOUT
+        updateLiveStore(executionData)
+      case None => logWarning(s"onOperationCanceled called with unknown operation id: ${e.id}")
+    }
 
-  private def onOperationFinished(e: SparkListenerThriftServerOperationFinish): Unit = {
-    executionList.get(e.id).finishTimestamp = e.finishTime
-    executionList.get(e.id).state = ExecutionState.FINISHED
-    updateLiveStore(executionList.get(e.id))
-  }
+  private def onOperationError(e: SparkListenerThriftServerOperationError): Unit =
+    Option(executionList.get(e.id)) match {
+      case Some(executionData) =>
+        executionData.finishTimestamp = e.finishTime
+        executionData.detail = e.errorMsg
+        executionData.state = ExecutionState.FAILED
+        updateLiveStore(executionData)
+      case None => logWarning(s"onOperationError called with unknown operation id: ${e.id}")
+    }
 
-  private def onOperationClosed(e: SparkListenerThriftServerOperationClosed): Unit = {
-    executionList.get(e.id).closeTimestamp = e.closeTime
-    executionList.get(e.id).state = ExecutionState.CLOSED
-    updateStoreWithTriggerEnabled(executionList.get(e.id))
-    executionList.remove(e.id)
-  }
+  private def onOperationFinished(e: SparkListenerThriftServerOperationFinish): Unit =
+    Option(executionList.get(e.id)) match {
+      case Some(executionData) =>
+        executionData.finishTimestamp = e.finishTime
+        executionData.state = ExecutionState.FINISHED
+        updateLiveStore(executionData)
+      case None => logWarning(s"onOperationFinished called with unknown operation id: ${e.id}")
+    }
+
+  private def onOperationClosed(e: SparkListenerThriftServerOperationClosed): Unit =
+    Option(executionList.get(e.id)) match {
+      case Some(executionData) =>
+        executionData.closeTimestamp = e.closeTime
+        executionData.state = ExecutionState.CLOSED
+        updateStoreWithTriggerEnabled(executionData)
+        executionList.remove(e.id)
+      case None => logWarning(s"onOperationClosed called with unknown operation id: ${e.id}")
+    }
 
   // Update both live and history stores. Trigger is enabled by default, hence
   // it will cleanup the entity which exceeds the threshold.
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala
index 7fb755c292b38..54a40e3990f09 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala
@@ -19,11 +19,9 @@ package org.apache.spark.sql.hive.thriftserver.ui
 
 import java.net.URLEncoder
 import java.nio.charset.StandardCharsets.UTF_8
-import java.util.Calendar
 import javax.servlet.http.HttpServletRequest
 
-import scala.collection.JavaConverters._
-import scala.xml.{Node, Unparsed}
+import scala.xml.Node
 
 import org.apache.commons.text.StringEscapeUtils
 
@@ -78,26 +76,8 @@ private[ui] class ThriftServerPage(parent: ThriftServerTab) extends WebUIPage(""
 
       val sqlTableTag = "sqlstat"
 
-      val parameterOtherTable = request.getParameterMap().asScala
-        .filterNot(_._1.startsWith(sqlTableTag))
-        .map { case (name, vals) =>
-          name + "=" + vals(0)
-        }
-
-      val parameterSqlTablePage = request.getParameter(s"$sqlTableTag.page")
-      val parameterSqlTableSortColumn = request.getParameter(s"$sqlTableTag.sort")
-      val parameterSqlTableSortDesc = request.getParameter(s"$sqlTableTag.desc")
-      val parameterSqlPageSize = request.getParameter(s"$sqlTableTag.pageSize")
-
-      val sqlTablePage = Option(parameterSqlTablePage).map(_.toInt).getOrElse(1)
-      val sqlTableSortColumn = Option(parameterSqlTableSortColumn).map { sortColumn =>
-        UIUtils.decodeURLParameter(sortColumn)
-      }.getOrElse("Start Time")
-      val sqlTableSortDesc = Option(parameterSqlTableSortDesc).map(_.toBoolean).getOrElse(
-        // New executions should be shown above old executions by default.
-        sqlTableSortColumn == "Start Time"
-      )
-      val sqlTablePageSize = Option(parameterSqlPageSize).map(_.toInt).getOrElse(100)
+      val sqlTablePage =
+        Option(request.getParameter(s"$sqlTableTag.page")).map(_.toInt).getOrElse(1)
 
       try {
         Some(new SqlStatsPagedTable(
@@ -106,12 +86,7 @@ private[ui] class ThriftServerPage(parent: ThriftServerTab) extends WebUIPage(""
           store.getExecutionList,
           "sqlserver",
           UIUtils.prependBaseUri(request, parent.basePath),
-          parameterOtherTable,
-          sqlTableTag,
-          pageSize = sqlTablePageSize,
-          sortColumn = sqlTableSortColumn,
-          desc = sqlTableSortDesc
-        ).table(sqlTablePage))
+          sqlTableTag).table(sqlTablePage))
       } catch {
         case e@(_: IllegalArgumentException | _: IndexOutOfBoundsException) =>
           Some(<div class="alert alert-error">
@@ -146,26 +121,8 @@ private[ui] class ThriftServerPage(parent: ThriftServerTab) extends WebUIPage(""
 
       val sessionTableTag = "sessionstat"
 
-      val parameterOtherTable = request.getParameterMap().asScala
-        .filterNot(_._1.startsWith(sessionTableTag))
-        .map { case (name, vals) =>
-          name + "=" + vals(0)
-        }
-
-      val parameterSessionTablePage = request.getParameter(s"$sessionTableTag.page")
-      val parameterSessionTableSortColumn = request.getParameter(s"$sessionTableTag.sort")
-      val parameterSessionTableSortDesc = request.getParameter(s"$sessionTableTag.desc")
-      val parameterSessionPageSize = request.getParameter(s"$sessionTableTag.pageSize")
-
-      val sessionTablePage = Option(parameterSessionTablePage).map(_.toInt).getOrElse(1)
-      val sessionTableSortColumn = Option(parameterSessionTableSortColumn).map { sortColumn =>
-        UIUtils.decodeURLParameter(sortColumn)
-      }.getOrElse("Start Time")
-      val sessionTableSortDesc = Option(parameterSessionTableSortDesc).map(_.toBoolean).getOrElse(
-        // New session should be shown above old session by default.
-        (sessionTableSortColumn == "Start Time")
-      )
-      val sessionTablePageSize = Option(parameterSessionPageSize).map(_.toInt).getOrElse(100)
+      val sessionTablePage =
+        Option(request.getParameter(s"$sessionTableTag.page")).map(_.toInt).getOrElse(1)
 
       try {
         Some(new SessionStatsPagedTable(
@@ -174,11 +131,7 @@ private[ui] class ThriftServerPage(parent: ThriftServerTab) extends WebUIPage(""
           store.getSessionList,
           "sqlserver",
           UIUtils.prependBaseUri(request, parent.basePath),
-          parameterOtherTable,
-          sessionTableTag,
-          pageSize = sessionTablePageSize,
-          sortColumn = sessionTableSortColumn,
-          desc = sessionTableSortDesc
+          sessionTableTag
         ).table(sessionTablePage))
       } catch {
         case e@(_: IllegalArgumentException | _: IndexOutOfBoundsException) =>
@@ -216,104 +169,59 @@ private[ui] class SqlStatsPagedTable(
     data: Seq[ExecutionInfo],
     subPath: String,
     basePath: String,
-    parameterOtherTable: Iterable[String],
-    sqlStatsTableTag: String,
-    pageSize: Int,
-    sortColumn: String,
-    desc: Boolean) extends PagedTable[SqlStatsTableRow] {
+    sqlStatsTableTag: String) extends PagedTable[SqlStatsTableRow] {
 
-  override val dataSource = new SqlStatsTableDataSource(data, pageSize, sortColumn, desc)
+  private val (sortColumn, desc, pageSize) =
+    getTableParameters(request, sqlStatsTableTag, "Start Time")
+
+  private val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
+
+  private val parameterPath =
+    s"$basePath/$subPath/?${getParameterOtherTable(request, sqlStatsTableTag)}"
 
-  private val parameterPath = s"$basePath/$subPath/?${parameterOtherTable.mkString("&")}"
+  override val dataSource = new SqlStatsTableDataSource(data, pageSize, sortColumn, desc)
 
   override def tableId: String = sqlStatsTableTag
 
   override def tableCssClass: String =
-    "table table-bordered table-sm table-striped " +
-      "table-head-clickable table-cell-width-limited"
+    "table table-bordered table-sm table-striped table-head-clickable table-cell-width-limited"
 
   override def pageLink(page: Int): String = {
-    val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
     parameterPath +
       s"&$pageNumberFormField=$page" +
       s"&$sqlStatsTableTag.sort=$encodedSortColumn" +
       s"&$sqlStatsTableTag.desc=$desc" +
-      s"&$pageSizeFormField=$pageSize"
+      s"&$pageSizeFormField=$pageSize" +
+      s"#$sqlStatsTableTag"
   }
 
   override def pageSizeFormField: String = s"$sqlStatsTableTag.pageSize"
 
   override def pageNumberFormField: String = s"$sqlStatsTableTag.page"
 
-  override def goButtonFormPath: String = {
-    val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
-    s"$parameterPath&$sqlStatsTableTag.sort=$encodedSortColumn&$sqlStatsTableTag.desc=$desc"
-  }
+  override def goButtonFormPath: String =
+    s"$parameterPath&$sqlStatsTableTag.sort=$encodedSortColumn" +
+      s"&$sqlStatsTableTag.desc=$desc#$sqlStatsTableTag"
 
   override def headers: Seq[Node] = {
-    val sqlTableHeaders = Seq("User", "JobID", "GroupID", "Start Time", "Finish Time",
-      "Close Time", "Execution Time", "Duration", "Statement", "State", "Detail")
-
-    val tooltips = Seq(None, None, None, None, Some(THRIFT_SERVER_FINISH_TIME),
-      Some(THRIFT_SERVER_CLOSE_TIME), Some(THRIFT_SERVER_EXECUTION),
-      Some(THRIFT_SERVER_DURATION), None, None, None)
-
-    assert(sqlTableHeaders.length == tooltips.length)
-
-    val headerRow: Seq[Node] = {
-      sqlTableHeaders.zip(tooltips).map { case (header, tooltip) =>
-        if (header == sortColumn) {
-          val headerLink = Unparsed(
-            parameterPath +
-              s"&$sqlStatsTableTag.sort=${URLEncoder.encode(header, UTF_8.name())}" +
-              s"&$sqlStatsTableTag.desc=${!desc}" +
-              s"&$sqlStatsTableTag.pageSize=$pageSize" +
-              s"#$sqlStatsTableTag")
-          val arrow = if (desc) "&#x25BE;" else "&#x25B4;" // UP or DOWN
-
-          if (tooltip.nonEmpty) {
-            <th>
-              <a href={headerLink}>
-                <span data-toggle="tooltip" title={tooltip.get}>
-                  {header}&nbsp;{Unparsed(arrow)}
-                </span>
-              </a>
-            </th>
-          } else {
-            <th>
-              <a href={headerLink}>
-                {header}&nbsp;{Unparsed(arrow)}
-              </a>
-            </th>
-          }
-        } else {
-          val headerLink = Unparsed(
-            parameterPath +
-              s"&$sqlStatsTableTag.sort=${URLEncoder.encode(header, UTF_8.name())}" +
-              s"&$sqlStatsTableTag.pageSize=$pageSize" +
-              s"#$sqlStatsTableTag")
-
-          if(tooltip.nonEmpty) {
-            <th>
-              <a href={headerLink}>
-                <span data-toggle="tooltip" title={tooltip.get}>
-                  {header}
-                </span>
-              </a>
-            </th>
-          } else {
-            <th>
-              <a href={headerLink}>
-                {header}
-              </a>
-            </th>
-          }
-        }
-      }
-    }
-    <thead>
-      {headerRow}
-    </thead>
+    val sqlTableHeadersAndTooltips: Seq[(String, Boolean, Option[String])] =
+      Seq(
+        ("User", true, None),
+        ("JobID", true, None),
+        ("GroupID", true, None),
+        ("Start Time", true, None),
+        ("Finish Time", true, Some(THRIFT_SERVER_FINISH_TIME)),
+        ("Close Time", true, Some(THRIFT_SERVER_CLOSE_TIME)),
+        ("Execution Time", true, Some(THRIFT_SERVER_EXECUTION)),
+        ("Duration", true, Some(THRIFT_SERVER_DURATION)),
+        ("Statement", true, None),
+        ("State", true, None),
+        ("Detail", true, None))
+
+    isSortColumnValid(sqlTableHeadersAndTooltips, sortColumn)
+
+    headerRow(sqlTableHeadersAndTooltips, desc, pageSize, sortColumn, parameterPath,
+      sqlStatsTableTag, sqlStatsTableTag)
   }
 
   override def row(sqlStatsTableRow: SqlStatsTableRow): Seq[Node] = {
@@ -391,101 +299,55 @@ private[ui] class SessionStatsPagedTable(
     data: Seq[SessionInfo],
     subPath: String,
     basePath: String,
-    parameterOtherTable: Iterable[String],
-    sessionStatsTableTag: String,
-    pageSize: Int,
-    sortColumn: String,
-    desc: Boolean) extends PagedTable[SessionInfo] {
+    sessionStatsTableTag: String) extends PagedTable[SessionInfo] {
 
-  override val dataSource = new SessionStatsTableDataSource(data, pageSize, sortColumn, desc)
+  private val (sortColumn, desc, pageSize) =
+    getTableParameters(request, sessionStatsTableTag, "Start Time")
+
+  private val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
 
-  private val parameterPath = s"$basePath/$subPath/?${parameterOtherTable.mkString("&")}"
+  private val parameterPath =
+    s"$basePath/$subPath/?${getParameterOtherTable(request, sessionStatsTableTag)}"
+
+  override val dataSource = new SessionStatsTableDataSource(data, pageSize, sortColumn, desc)
 
   override def tableId: String = sessionStatsTableTag
 
   override def tableCssClass: String =
-    "table table-bordered table-sm table-striped " +
-      "table-head-clickable table-cell-width-limited"
+    "table table-bordered table-sm table-striped table-head-clickable table-cell-width-limited"
 
   override def pageLink(page: Int): String = {
-    val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
     parameterPath +
       s"&$pageNumberFormField=$page" +
       s"&$sessionStatsTableTag.sort=$encodedSortColumn" +
       s"&$sessionStatsTableTag.desc=$desc" +
-      s"&$pageSizeFormField=$pageSize"
+      s"&$pageSizeFormField=$pageSize" +
+      s"#$sessionStatsTableTag"
   }
 
   override def pageSizeFormField: String = s"$sessionStatsTableTag.pageSize"
 
   override def pageNumberFormField: String = s"$sessionStatsTableTag.page"
 
-  override def goButtonFormPath: String = {
-    val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
-    s"$parameterPath&$sessionStatsTableTag.sort=$encodedSortColumn&$sessionStatsTableTag.desc=$desc"
-  }
+  override def goButtonFormPath: String =
+    s"$parameterPath&$sessionStatsTableTag.sort=$encodedSortColumn" +
+      s"&$sessionStatsTableTag.desc=$desc#$sessionStatsTableTag"
 
   override def headers: Seq[Node] = {
-    val sessionTableHeaders =
-      Seq("User", "IP", "Session ID", "Start Time", "Finish Time", "Duration", "Total Execute")
-
-    val tooltips = Seq(None, None, None, None, None, Some(THRIFT_SESSION_DURATION),
-      Some(THRIFT_SESSION_TOTAL_EXECUTE))
-    assert(sessionTableHeaders.length == tooltips.length)
-    val colWidthAttr = s"${100.toDouble / sessionTableHeaders.size}%"
-
-    val headerRow: Seq[Node] = {
-      sessionTableHeaders.zip(tooltips).map { case (header, tooltip) =>
-        if (header == sortColumn) {
-          val headerLink = Unparsed(
-            parameterPath +
-              s"&$sessionStatsTableTag.sort=${URLEncoder.encode(header, UTF_8.name())}" +
-              s"&$sessionStatsTableTag.desc=${!desc}" +
-              s"&$sessionStatsTableTag.pageSize=$pageSize" +
-              s"#$sessionStatsTableTag")
-          val arrow = if (desc) "&#x25BE;" else "&#x25B4;" // UP or DOWN
-            <th width={colWidthAttr}>
-              <a href={headerLink}>
-                {
-                  if (tooltip.nonEmpty) {
-                    <span data-toggle="tooltip" data-placement="top" title={tooltip.get}>
-                      {header}&nbsp;{Unparsed(arrow)}
-                    </span>
-                  } else {
-                    <span>
-                      {header}&nbsp;{Unparsed(arrow)}
-                    </span>
-                  }
-                }
-              </a>
-            </th>
-
-        } else {
-          val headerLink = Unparsed(
-            parameterPath +
-              s"&$sessionStatsTableTag.sort=${URLEncoder.encode(header, UTF_8.name())}" +
-              s"&$sessionStatsTableTag.pageSize=$pageSize" +
-              s"#$sessionStatsTableTag")
-
-            <th width={colWidthAttr}>
-              <a href={headerLink}>
-                {
-                  if (tooltip.nonEmpty) {
-                    <span data-toggle="tooltip" data-placement="top" title={tooltip.get}>
-                      {header}
-                    </span>
-                  } else {
-                    {header}
-                  }
-                }
-              </a>
-            </th>
-        }
-      }
-    }
-    <thead>
-      {headerRow}
-    </thead>
+    val sessionTableHeadersAndTooltips: Seq[(String, Boolean, Option[String])] =
+      Seq(
+        ("User", true, None),
+        ("IP", true, None),
+        ("Session ID", true, None),
+        ("Start Time", true, None),
+        ("Finish Time", true, None),
+        ("Duration", true, Some(THRIFT_SESSION_DURATION)),
+        ("Total Execute", true, Some(THRIFT_SESSION_TOTAL_EXECUTE)))
+
+    isSortColumnValid(sessionTableHeadersAndTooltips, sortColumn)
+
+    headerRow(sessionTableHeadersAndTooltips, desc, pageSize, sortColumn,
+      parameterPath, sessionStatsTableTag, sessionStatsTableTag)
   }
 
   override def row(session: SessionInfo): Seq[Node] = {
@@ -503,108 +365,94 @@ private[ui] class SessionStatsPagedTable(
   }
 }
 
-  private[ui] class SqlStatsTableRow(
+private[ui] class SqlStatsTableRow(
     val jobId: Seq[String],
     val duration: Long,
     val executionTime: Long,
     val executionInfo: ExecutionInfo,
     val detail: String)
 
-  private[ui] class SqlStatsTableDataSource(
+private[ui] class SqlStatsTableDataSource(
     info: Seq[ExecutionInfo],
     pageSize: Int,
     sortColumn: String,
     desc: Boolean) extends PagedDataSource[SqlStatsTableRow](pageSize) {
 
-    // Convert ExecutionInfo to SqlStatsTableRow which contains the final contents to show in
-    // the table so that we can avoid creating duplicate contents during sorting the data
-    private val data = info.map(sqlStatsTableRow).sorted(ordering(sortColumn, desc))
+  // Convert ExecutionInfo to SqlStatsTableRow which contains the final contents to show in
+  // the table so that we can avoid creating duplicate contents during sorting the data
+  private val data = info.map(sqlStatsTableRow).sorted(ordering(sortColumn, desc))
 
-    private var _slicedStartTime: Set[Long] = null
+  override def dataSize: Int = data.size
 
-    override def dataSize: Int = data.size
+  override def sliceData(from: Int, to: Int): Seq[SqlStatsTableRow] = data.slice(from, to)
 
-    override def sliceData(from: Int, to: Int): Seq[SqlStatsTableRow] = {
-      val r = data.slice(from, to)
-      _slicedStartTime = r.map(_.executionInfo.startTimestamp).toSet
-      r
-    }
+  private def sqlStatsTableRow(executionInfo: ExecutionInfo): SqlStatsTableRow = {
+    val duration = executionInfo.totalTime(executionInfo.closeTimestamp)
+    val executionTime = executionInfo.totalTime(executionInfo.finishTimestamp)
+    val detail = Option(executionInfo.detail).filter(!_.isEmpty)
+      .getOrElse(executionInfo.executePlan)
+    val jobId = executionInfo.jobId.toSeq.sorted
 
-    private def sqlStatsTableRow(executionInfo: ExecutionInfo): SqlStatsTableRow = {
-      val duration = executionInfo.totalTime(executionInfo.closeTimestamp)
-      val executionTime = executionInfo.totalTime(executionInfo.finishTimestamp)
-      val detail = Option(executionInfo.detail).filter(!_.isEmpty)
-        .getOrElse(executionInfo.executePlan)
-      val jobId = executionInfo.jobId.toSeq.sorted
-
-      new SqlStatsTableRow(jobId, duration, executionTime, executionInfo, detail)
+    new SqlStatsTableRow(jobId, duration, executionTime, executionInfo, detail)
+  }
 
+  /**
+   * Return Ordering according to sortColumn and desc.
+   */
+  private def ordering(sortColumn: String, desc: Boolean): Ordering[SqlStatsTableRow] = {
+    val ordering: Ordering[SqlStatsTableRow] = sortColumn match {
+      case "User" => Ordering.by(_.executionInfo.userName)
+      case "JobID" => Ordering by (_.jobId.headOption)
+      case "GroupID" => Ordering.by(_.executionInfo.groupId)
+      case "Start Time" => Ordering.by(_.executionInfo.startTimestamp)
+      case "Finish Time" => Ordering.by(_.executionInfo.finishTimestamp)
+      case "Close Time" => Ordering.by(_.executionInfo.closeTimestamp)
+      case "Execution Time" => Ordering.by(_.executionTime)
+      case "Duration" => Ordering.by(_.duration)
+      case "Statement" => Ordering.by(_.executionInfo.statement)
+      case "State" => Ordering.by(_.executionInfo.state)
+      case "Detail" => Ordering.by(_.detail)
+      case unknownColumn => throw new IllegalArgumentException(s"Unknown column: $unknownColumn")
     }
-
-    /**
-     * Return Ordering according to sortColumn and desc.
-     */
-    private def ordering(sortColumn: String, desc: Boolean): Ordering[SqlStatsTableRow] = {
-      val ordering: Ordering[SqlStatsTableRow] = sortColumn match {
-        case "User" => Ordering.by(_.executionInfo.userName)
-        case "JobID" => Ordering by (_.jobId.headOption)
-        case "GroupID" => Ordering.by(_.executionInfo.groupId)
-        case "Start Time" => Ordering.by(_.executionInfo.startTimestamp)
-        case "Finish Time" => Ordering.by(_.executionInfo.finishTimestamp)
-        case "Close Time" => Ordering.by(_.executionInfo.closeTimestamp)
-        case "Execution Time" => Ordering.by(_.executionTime)
-        case "Duration" => Ordering.by(_.duration)
-        case "Statement" => Ordering.by(_.executionInfo.statement)
-        case "State" => Ordering.by(_.executionInfo.state)
-        case "Detail" => Ordering.by(_.detail)
-        case unknownColumn => throw new IllegalArgumentException(s"Unknown column: $unknownColumn")
-      }
-      if (desc) {
-        ordering.reverse
-      } else {
-        ordering
-      }
+    if (desc) {
+      ordering.reverse
+    } else {
+      ordering
     }
-
   }
+}
 
-  private[ui] class SessionStatsTableDataSource(
+private[ui] class SessionStatsTableDataSource(
     info: Seq[SessionInfo],
     pageSize: Int,
     sortColumn: String,
     desc: Boolean) extends PagedDataSource[SessionInfo](pageSize) {
 
-    // Sorting SessionInfo data
-    private val data = info.sorted(ordering(sortColumn, desc))
-
-    private var _slicedStartTime: Set[Long] = null
-
-    override def dataSize: Int = data.size
-
-    override def sliceData(from: Int, to: Int): Seq[SessionInfo] = {
-      val r = data.slice(from, to)
-      _slicedStartTime = r.map(_.startTimestamp).toSet
-      r
+  // Sorting SessionInfo data
+  private val data = info.sorted(ordering(sortColumn, desc))
+
+  override def dataSize: Int = data.size
+
+  override def sliceData(from: Int, to: Int): Seq[SessionInfo] = data.slice(from, to)
+
+  /**
+   * Return Ordering according to sortColumn and desc.
+   */
+  private def ordering(sortColumn: String, desc: Boolean): Ordering[SessionInfo] = {
+    val ordering: Ordering[SessionInfo] = sortColumn match {
+      case "User" => Ordering.by(_.userName)
+      case "IP" => Ordering.by(_.ip)
+      case "Session ID" => Ordering.by(_.sessionId)
+      case "Start Time" => Ordering by (_.startTimestamp)
+      case "Finish Time" => Ordering.by(_.finishTimestamp)
+      case "Duration" => Ordering.by(_.totalTime)
+      case "Total Execute" => Ordering.by(_.totalExecution)
+      case unknownColumn => throw new IllegalArgumentException(s"Unknown column: $unknownColumn")
     }
-
-    /**
-     * Return Ordering according to sortColumn and desc.
-     */
-    private def ordering(sortColumn: String, desc: Boolean): Ordering[SessionInfo] = {
-      val ordering: Ordering[SessionInfo] = sortColumn match {
-        case "User" => Ordering.by(_.userName)
-        case "IP" => Ordering.by(_.ip)
-        case "Session ID" => Ordering.by(_.sessionId)
-        case "Start Time" => Ordering by (_.startTimestamp)
-        case "Finish Time" => Ordering.by(_.finishTimestamp)
-        case "Duration" => Ordering.by(_.totalTime)
-        case "Total Execute" => Ordering.by(_.totalExecution)
-        case unknownColumn => throw new IllegalArgumentException(s"Unknown column: $unknownColumn")
-      }
-      if (desc) {
-        ordering.reverse
-      } else {
-        ordering
-      }
+    if (desc) {
+      ordering.reverse
+    } else {
+      ordering
     }
   }
+}
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerSessionPage.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerSessionPage.scala
index 2d7adf552738c..87165cc8cac45 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerSessionPage.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerSessionPage.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.hive.thriftserver.ui
 
 import javax.servlet.http.HttpServletRequest
 
-import scala.collection.JavaConverters._
 import scala.xml.Node
 
 import org.apache.spark.internal.Logging
@@ -77,26 +76,8 @@ private[ui] class ThriftServerSessionPage(parent: ThriftServerTab)
 
       val sqlTableTag = "sqlsessionstat"
 
-      val parameterOtherTable = request.getParameterMap().asScala
-        .filterNot(_._1.startsWith(sqlTableTag))
-        .map { case (name, vals) =>
-          name + "=" + vals(0)
-        }
-
-      val parameterSqlTablePage = request.getParameter(s"$sqlTableTag.page")
-      val parameterSqlTableSortColumn = request.getParameter(s"$sqlTableTag.sort")
-      val parameterSqlTableSortDesc = request.getParameter(s"$sqlTableTag.desc")
-      val parameterSqlPageSize = request.getParameter(s"$sqlTableTag.pageSize")
-
-      val sqlTablePage = Option(parameterSqlTablePage).map(_.toInt).getOrElse(1)
-      val sqlTableSortColumn = Option(parameterSqlTableSortColumn).map { sortColumn =>
-        UIUtils.decodeURLParameter(sortColumn)
-      }.getOrElse("Start Time")
-      val sqlTableSortDesc = Option(parameterSqlTableSortDesc).map(_.toBoolean).getOrElse(
-        // New executions should be shown above old executions by default.
-        sqlTableSortColumn == "Start Time"
-      )
-      val sqlTablePageSize = Option(parameterSqlPageSize).map(_.toInt).getOrElse(100)
+      val sqlTablePage =
+        Option(request.getParameter(s"$sqlTableTag.page")).map(_.toInt).getOrElse(1)
 
       try {
         Some(new SqlStatsPagedTable(
@@ -105,11 +86,7 @@ private[ui] class ThriftServerSessionPage(parent: ThriftServerTab)
           executionList,
           "sqlserver/session",
           UIUtils.prependBaseUri(request, parent.basePath),
-          parameterOtherTable,
-          sqlTableTag,
-          pageSize = sqlTablePageSize,
-          sortColumn = sqlTableSortColumn,
-          desc = sqlTableSortDesc
+          sqlTableTag
         ).table(sqlTablePage))
       } catch {
         case e@(_: IllegalArgumentException | _: IndexOutOfBoundsException) =>
diff --git a/sql/hive-thriftserver/src/test/resources/log4j.properties b/sql/hive-thriftserver/src/test/resources/log4j.properties
new file mode 100644
index 0000000000000..ad799deea3678
--- /dev/null
+++ b/sql/hive-thriftserver/src/test/resources/log4j.properties
@@ -0,0 +1,75 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set everything to be logged to the file hive-thriftserver/target/unit-tests.log
+log4j.rootLogger=DEBUG, CA, FA
+
+#Console Appender
+log4j.appender.CA=org.apache.log4j.ConsoleAppender
+log4j.appender.CA.layout=org.apache.log4j.PatternLayout
+log4j.appender.CA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %p %c: %m%n
+log4j.appender.CA.Threshold = WARN
+
+# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805
+log4j.appender.CA.filter.1=org.apache.log4j.varia.StringMatchFilter
+log4j.appender.CA.filter.1.StringToMatch=Thrift error occurred during processing of message
+log4j.appender.CA.filter.1.AcceptOnMatch=false
+
+#File Appender
+log4j.appender.FA=org.apache.log4j.FileAppender
+log4j.appender.FA.append=false
+log4j.appender.FA.file=target/unit-tests.log
+log4j.appender.FA.layout=org.apache.log4j.PatternLayout
+log4j.appender.FA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %t %p %c{1}: %m%n
+
+log4j.appender.FA.filter.1=org.apache.log4j.varia.StringMatchFilter
+log4j.appender.FA.filter.1.StringToMatch=Thrift error occurred during processing of message
+log4j.appender.FA.filter.1.AcceptOnMatch=false
+
+# Set the logger level of File Appender to WARN
+log4j.appender.FA.Threshold = DEBUG
+
+# Some packages are noisy for no good reason.
+log4j.additivity.org.apache.hadoop.hive.serde2.lazy.LazyStruct=false
+log4j.logger.org.apache.hadoop.hive.serde2.lazy.LazyStruct=OFF
+
+log4j.additivity.org.apache.hadoop.hive.metastore.RetryingHMSHandler=false
+log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=OFF
+
+log4j.additivity.hive.log=false
+log4j.logger.hive.log=OFF
+
+log4j.additivity.parquet.hadoop.ParquetRecordReader=false
+log4j.logger.parquet.hadoop.ParquetRecordReader=OFF
+
+log4j.additivity.org.apache.parquet.hadoop.ParquetRecordReader=false
+log4j.logger.org.apache.parquet.hadoop.ParquetRecordReader=OFF
+
+log4j.additivity.org.apache.parquet.hadoop.ParquetOutputCommitter=false
+log4j.logger.org.apache.parquet.hadoop.ParquetOutputCommitter=OFF
+
+log4j.additivity.hive.ql.metadata.Hive=false
+log4j.logger.hive.ql.metadata.Hive=OFF
+
+log4j.additivity.org.apache.hadoop.hive.ql.io.RCFile=false
+log4j.logger.org.apache.hadoop.hive.ql.io.RCFile=ERROR
+
+# Parquet related logging
+log4j.logger.org.apache.parquet.CorruptStatistics=ERROR
+log4j.logger.parquet.CorruptStatistics=ERROR
+
+log4j.logger.org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation=OFF
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreLazyInitializationSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreLazyInitializationSuite.scala
index 277df548aefd0..951f92793732f 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreLazyInitializationSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreLazyInitializationSuite.scala
@@ -31,6 +31,7 @@ class HiveMetastoreLazyInitializationSuite extends SparkFunSuite {
       .config("spark.hadoop.hive.metastore.uris", "thrift://127.0.0.1:11111")
       .getOrCreate()
     val originalLevel = org.apache.log4j.Logger.getRootLogger().getLevel
+    val originalClassLoader = Thread.currentThread().getContextClassLoader
     try {
       // Avoid outputting a lot of expected warning logs
       spark.sparkContext.setLogLevel("error")
@@ -64,6 +65,7 @@ class HiveMetastoreLazyInitializationSuite extends SparkFunSuite {
         exceptionString.contains(msg)
       }
     } finally {
+      Thread.currentThread().setContextClassLoader(originalClassLoader)
       spark.sparkContext.setLogLevel(originalLevel.toString)
       spark.stop()
     }
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
index ea1a371151c36..7067b65a6232e 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
@@ -27,10 +27,11 @@ import scala.concurrent.Promise
 import scala.concurrent.duration._
 
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars
-import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+import org.scalatest.BeforeAndAfterAll
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.internal.Logging
+import org.apache.spark.sql.hive.HiveUtils._
 import org.apache.spark.sql.hive.test.HiveTestJars
 import org.apache.spark.sql.internal.StaticSQLConf
 import org.apache.spark.sql.test.ProcessTestUtils.ProcessOutputCapturer
@@ -98,10 +99,8 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
           Seq(answer)
         } else {
           // spark-sql echoes the submitted queries
-          val queryEcho = query.split("\n").toList match {
-            case firstLine :: tail =>
-              s"spark-sql> $firstLine" :: tail.map(l => s"         > $l")
-          }
+          val xs = query.split("\n").toList
+          val queryEcho = s"spark-sql> ${xs.head}" :: xs.tail.map(l => s"         > $l")
           // longer lines sometimes get split in the output,
           // match the first 60 characters of each query line
           queryEcho.map(_.take(60)) :+ answer
@@ -132,6 +131,7 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
     }
 
     var next = 0
+    val foundMasterAndApplicationIdMessage = Promise.apply[Unit]()
     val foundAllExpectedAnswers = Promise.apply[Unit]()
     val buffer = new ArrayBuffer[String]()
     val lock = new Object
@@ -143,6 +143,10 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
       log.info(newLine)
       buffer += newLine
 
+      if (line.startsWith("Spark master: ") && line.contains("Application Id: ")) {
+        foundMasterAndApplicationIdMessage.trySuccess(())
+      }
+
       // If we haven't found all expected answers and another expected answer comes up...
       if (next < expectedAnswers.size && line.contains(expectedAnswers(next))) {
         log.info(s"$source> found expected output line $next: '${expectedAnswers(next)}'")
@@ -172,7 +176,18 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
     new ProcessOutputCapturer(process.getErrorStream, captureOutput("stderr")).start()
 
     try {
-      ThreadUtils.awaitResult(foundAllExpectedAnswers.future, timeout)
+      val timeoutForQuery = if (!extraArgs.contains("-e")) {
+        // Wait for for cli driver to boot, up to two minutes
+        ThreadUtils.awaitResult(foundMasterAndApplicationIdMessage.future, 2.minutes)
+        log.info("Cli driver is booted. Waiting for expected answers.")
+        // Given timeout is applied after the cli driver is ready
+        timeout
+      } else {
+        // There's no boot message if -e option is provided, just extend timeout long enough
+        // so that the bootup duration is counted on the timeout
+        2.minutes + timeout
+      }
+      ThreadUtils.awaitResult(foundAllExpectedAnswers.future, timeoutForQuery)
       log.info("Found all expected output.")
     } catch { case cause: Throwable =>
       val message =
@@ -194,7 +209,7 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
     } finally {
       if (!process.waitFor(1, MINUTES)) {
         try {
-          fail("spark-sql did not exit gracefully.")
+          log.warn("spark-sql did not exit gracefully.")
         } finally {
           process.destroy()
         }
@@ -447,7 +462,7 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
     val jarFile = new File("../../sql/hive/src/test/resources/SPARK-21101-1.0.jar").getCanonicalPath
     val hiveContribJar = HiveTestJars.getHiveContribJar().getCanonicalPath
     runCliWithin(
-      1.minute,
+      2.minutes,
       Seq("--jars", s"$jarFile", "--conf",
         s"spark.hadoop.${ConfVars.HIVEAUXJARS}=$hiveContribJar"))(
       "CREATE TEMPORARY FUNCTION testjar AS" +
@@ -551,4 +566,49 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
       errorResponses = Seq("AnalysisException"))(
       ("", "Error in query: The second argument of 'date_sub' function needs to be an integer."))
   }
+
+  test("SPARK-30808: use Java 8 time API in Thrift SQL CLI by default") {
+    // If Java 8 time API is enabled via the SQL config `spark.sql.datetime.java8API.enabled`,
+    // the date formatter for `java.sql.LocalDate` must output negative years with sign.
+    runCliWithin(1.minute)("SELECT MAKE_DATE(-44, 3, 15);" -> "-0044-03-15")
+  }
+
+  test("SPARK-33100: Ignore a semicolon inside a bracketed comment in spark-sql") {
+    runCliWithin(4.minute)(
+      "/* SELECT 'test';*/ SELECT 'test';" -> "test",
+      ";;/* SELECT 'test';*/ SELECT 'test';" -> "test",
+      "/* SELECT 'test';*/;; SELECT 'test';" -> "test",
+      "SELECT 'test'; -- SELECT 'test';" -> "test",
+      "SELECT 'test'; /* SELECT 'test';*/;" -> "test",
+      "/*$meta chars{^\\;}*/ SELECT 'test';" -> "test",
+      "/*\nmulti-line\n*/ SELECT 'test';" -> "test",
+      "/*/* multi-level bracketed*/ SELECT 'test';" -> "test"
+    )
+  }
+
+  test("SPARK-33100: test sql statements with hint in bracketed comment") {
+    runCliWithin(2.minute)(
+      "CREATE TEMPORARY VIEW t1 AS SELECT * FROM VALUES(1, 2) AS t1(k, v);" -> "",
+      "CREATE TEMPORARY VIEW t2 AS SELECT * FROM VALUES(2, 1) AS t2(k, v);" -> "",
+      "EXPLAIN SELECT /*+ MERGEJOIN(t1) */ t1.* FROM t1 JOIN t2 ON t1.k = t2.v;" -> "SortMergeJoin",
+      "EXPLAIN SELECT /* + MERGEJOIN(t1) */ t1.* FROM t1 JOIN t2 ON t1.k = t2.v;"
+        -> "BroadcastHashJoin"
+    )
+  }
+
+  test("SPARK-35086: --verbose should be passed to Spark SQL CLI") {
+    runCliWithin(2.minute, Seq("--verbose"))(
+      "SELECT 'SPARK-35086' AS c1, '--verbose' AS c2;" ->
+        "SELECT 'SPARK-35086' AS c1, '--verbose' AS c2"
+    )
+  }
+
+  test("SPARK-35102: Make spark.sql.hive.version meaningful and not deprecated") {
+    runCliWithin(1.minute,
+      Seq("--conf", "spark.sql.hive.version=0.1"),
+      Seq(s"please use ${HIVE_METASTORE_VERSION.key}"))("" -> "")
+    runCliWithin(2.minute,
+      Seq("--conf", s"${BUILTIN_HIVE_VERSION.key}=$builtinHiveVersion"))(
+      s"set ${BUILTIN_HIVE_VERSION.key};" -> builtinHiveVersion, "SET -v;" -> builtinHiveVersion)
+  }
 }
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/DummyListeners.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/DummyListeners.scala
index 4564c2209a931..820859b65925b 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/DummyListeners.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/DummyListeners.scala
@@ -17,7 +17,7 @@
 
 /**
  * These classes in this package are intentionally placed to the outer package of spark,
- * because IsolatedClientLoader leverages Spark classloader for shared classess including
+ * because IsolatedClientLoader leverages Spark classloader for shared classes including
  * spark package, and the test should fail if Spark initializes these listeners with
  * IsolatedClientLoader.
  */
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/FetchIteratorSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/FetchIteratorSuite.scala
new file mode 100644
index 0000000000000..0fbdb8a9050c8
--- /dev/null
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/FetchIteratorSuite.scala
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.thriftserver
+
+import org.apache.spark.SparkFunSuite
+
+class FetchIteratorSuite extends SparkFunSuite {
+
+  private def getRows(fetchIter: FetchIterator[Int], maxRowCount: Int): Seq[Int] = {
+    for (_ <- 0 until maxRowCount if fetchIter.hasNext) yield fetchIter.next()
+  }
+
+  test("SPARK-33655: Test fetchNext and fetchPrior") {
+    val testData = 0 until 10
+
+    def iteratorTest(fetchIter: FetchIterator[Int]): Unit = {
+      fetchIter.fetchNext()
+      assert(fetchIter.getFetchStart == 0)
+      assert(fetchIter.getPosition == 0)
+      assertResult(0 until 2)(getRows(fetchIter, 2))
+      assert(fetchIter.getFetchStart == 0)
+      assert(fetchIter.getPosition == 2)
+
+      fetchIter.fetchNext()
+      assert(fetchIter.getFetchStart == 2)
+      assert(fetchIter.getPosition == 2)
+      assertResult(2 until 3)(getRows(fetchIter, 1))
+      assert(fetchIter.getFetchStart == 2)
+      assert(fetchIter.getPosition == 3)
+
+      fetchIter.fetchPrior(2)
+      assert(fetchIter.getFetchStart == 0)
+      assert(fetchIter.getPosition == 0)
+      assertResult(0 until 3)(getRows(fetchIter, 3))
+      assert(fetchIter.getFetchStart == 0)
+      assert(fetchIter.getPosition == 3)
+
+      fetchIter.fetchNext()
+      assert(fetchIter.getFetchStart == 3)
+      assert(fetchIter.getPosition == 3)
+      assertResult(3 until 8)(getRows(fetchIter, 5))
+      assert(fetchIter.getFetchStart == 3)
+      assert(fetchIter.getPosition == 8)
+
+      fetchIter.fetchPrior(2)
+      assert(fetchIter.getFetchStart == 1)
+      assert(fetchIter.getPosition == 1)
+      assertResult(1 until 4)(getRows(fetchIter, 3))
+      assert(fetchIter.getFetchStart == 1)
+      assert(fetchIter.getPosition == 4)
+
+      fetchIter.fetchNext()
+      assert(fetchIter.getFetchStart == 4)
+      assert(fetchIter.getPosition == 4)
+      assertResult(4 until 10)(getRows(fetchIter, 10))
+      assert(fetchIter.getFetchStart == 4)
+      assert(fetchIter.getPosition == 10)
+
+      fetchIter.fetchNext()
+      assert(fetchIter.getFetchStart == 10)
+      assert(fetchIter.getPosition == 10)
+      assertResult(Seq.empty[Int])(getRows(fetchIter, 10))
+      assert(fetchIter.getFetchStart == 10)
+      assert(fetchIter.getPosition == 10)
+
+      fetchIter.fetchPrior(20)
+      assert(fetchIter.getFetchStart == 0)
+      assert(fetchIter.getPosition == 0)
+      assertResult(0 until 3)(getRows(fetchIter, 3))
+      assert(fetchIter.getFetchStart == 0)
+      assert(fetchIter.getPosition == 3)
+    }
+    iteratorTest(new ArrayFetchIterator[Int](testData.toArray))
+    iteratorTest(new IterableFetchIterator[Int](testData))
+  }
+
+  test("SPARK-33655: Test fetchAbsolute") {
+    val testData = 0 until 10
+
+    def iteratorTest(fetchIter: FetchIterator[Int]): Unit = {
+      fetchIter.fetchNext()
+      assert(fetchIter.getFetchStart == 0)
+      assert(fetchIter.getPosition == 0)
+      assertResult(0 until 5)(getRows(fetchIter, 5))
+      assert(fetchIter.getFetchStart == 0)
+      assert(fetchIter.getPosition == 5)
+
+      fetchIter.fetchAbsolute(2)
+      assert(fetchIter.getFetchStart == 2)
+      assert(fetchIter.getPosition == 2)
+      assertResult(2 until 5)(getRows(fetchIter, 3))
+      assert(fetchIter.getFetchStart == 2)
+      assert(fetchIter.getPosition == 5)
+
+      fetchIter.fetchAbsolute(7)
+      assert(fetchIter.getFetchStart == 7)
+      assert(fetchIter.getPosition == 7)
+      assertResult(7 until 8)(getRows(fetchIter, 1))
+      assert(fetchIter.getFetchStart == 7)
+      assert(fetchIter.getPosition == 8)
+
+      fetchIter.fetchAbsolute(20)
+      assert(fetchIter.getFetchStart == 10)
+      assert(fetchIter.getPosition == 10)
+      assertResult(Seq.empty[Int])(getRows(fetchIter, 1))
+      assert(fetchIter.getFetchStart == 10)
+      assert(fetchIter.getPosition == 10)
+
+      fetchIter.fetchAbsolute(0)
+      assert(fetchIter.getFetchStart == 0)
+      assert(fetchIter.getPosition == 0)
+      assertResult(0 until 3)(getRows(fetchIter, 3))
+      assert(fetchIter.getFetchStart == 0)
+      assert(fetchIter.getPosition == 3)
+    }
+    iteratorTest(new ArrayFetchIterator[Int](testData.toArray))
+    iteratorTest(new IterableFetchIterator[Int](testData))
+  }
+}
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/GetCatalogsOperationMock.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/GetCatalogsOperationMock.scala
new file mode 100644
index 0000000000000..1bc9aaf672c3b
--- /dev/null
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/GetCatalogsOperationMock.scala
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.hive.thriftserver
+
+import java.nio.ByteBuffer
+import java.util.UUID
+
+import org.apache.hive.service.cli.OperationHandle
+import org.apache.hive.service.cli.operation.GetCatalogsOperation
+import org.apache.hive.service.cli.session.HiveSession
+import org.apache.hive.service.rpc.thrift.{THandleIdentifier, TOperationHandle, TOperationType}
+
+class GetCatalogsOperationMock(parentSession: HiveSession)
+  extends GetCatalogsOperation(parentSession) {
+
+  override def runInternal(): Unit = {}
+
+  override def getHandle: OperationHandle = {
+    val uuid: UUID = UUID.randomUUID()
+    val tHandleIdentifier: THandleIdentifier = new THandleIdentifier()
+    tHandleIdentifier.setGuid(getByteBufferFromUUID(uuid))
+    tHandleIdentifier.setSecret(getByteBufferFromUUID(uuid))
+    val tOperationHandle: TOperationHandle = new TOperationHandle()
+    tOperationHandle.setOperationId(tHandleIdentifier)
+    tOperationHandle.setOperationType(TOperationType.GET_TYPE_INFO)
+    tOperationHandle.setHasResultSetIsSet(false)
+    new OperationHandle(tOperationHandle)
+  }
+
+  private def getByteBufferFromUUID(uuid: UUID): Array[Byte] = {
+    val bb: ByteBuffer = ByteBuffer.wrap(new Array[Byte](16))
+    bb.putLong(uuid.getMostSignificantBits)
+    bb.putLong(uuid.getLeastSignificantBits)
+    bb.array
+  }
+}
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveSessionImplSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveSessionImplSuite.scala
new file mode 100644
index 0000000000000..7c42348f74453
--- /dev/null
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveSessionImplSuite.scala
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.hive.thriftserver
+
+import java.lang.reflect.InvocationTargetException
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+import org.apache.hadoop.hive.conf.HiveConf
+import org.apache.hive.service.cli.OperationHandle
+import org.apache.hive.service.cli.operation.{GetCatalogsOperation, Operation, OperationManager}
+import org.apache.hive.service.cli.session.{HiveSession, HiveSessionImpl, SessionManager}
+import org.apache.hive.service.rpc.thrift.TProtocolVersion
+
+import org.apache.spark.SparkFunSuite
+
+class HiveSessionImplSuite extends SparkFunSuite {
+  private var session: HiveSessionImpl = _
+  private var operationManager: OperationManagerMock = _
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+
+    val sessionManager = new SessionManager(null)
+    operationManager = new OperationManagerMock()
+
+    session = new HiveSessionImpl(
+      TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V1,
+      "",
+      "",
+      new HiveConf(),
+      ""
+    )
+    session.setSessionManager(sessionManager)
+    session.setOperationManager(operationManager)
+
+    session.open(Map.empty[String, String].asJava)
+  }
+
+  test("SPARK-31387 - session.close() closes all sessions regardless of thrown exceptions") {
+    val operationHandle1 = session.getCatalogs
+    val operationHandle2 = session.getCatalogs
+
+    session.close()
+
+    assert(operationManager.getCalledHandles.contains(operationHandle1))
+    assert(operationManager.getCalledHandles.contains(operationHandle2))
+  }
+}
+
+class OperationManagerMock extends OperationManager {
+  private val calledHandles: mutable.Set[OperationHandle] = new mutable.HashSet[OperationHandle]()
+
+  override def newGetCatalogsOperation(parentSession: HiveSession): GetCatalogsOperation = {
+    val operation = new GetCatalogsOperationMock(parentSession)
+    try {
+      val m = classOf[OperationManager].getDeclaredMethod("addOperation", classOf[Operation])
+      m.setAccessible(true)
+      m.invoke(this, operation)
+    } catch {
+      case e@(_: NoSuchMethodException | _: IllegalAccessException |
+              _: InvocationTargetException) =>
+        throw new RuntimeException(e)
+    }
+    operation
+  }
+
+  override def closeOperation(opHandle: OperationHandle): Unit = {
+    calledHandles.add(opHandle)
+    throw new RuntimeException
+  }
+
+  def getCalledHandles: mutable.Set[OperationHandle] = calledHandles
+}
+
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
index 0cec63460814c..792aea1dca84c 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
@@ -29,7 +29,7 @@ import scala.collection.mutable.ArrayBuffer
 import scala.concurrent.{ExecutionContext, Future, Promise}
 import scala.concurrent.duration._
 import scala.io.Source
-import scala.util.{Random, Try}
+import scala.util.Try
 
 import com.google.common.io.Files
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars
@@ -37,14 +37,17 @@ import org.apache.hive.jdbc.HiveDriver
 import org.apache.hive.service.auth.PlainSaslHelper
 import org.apache.hive.service.cli.{FetchOrientation, FetchType, GetInfoType, RowSet}
 import org.apache.hive.service.cli.thrift.ThriftCLIServiceClient
+import org.apache.hive.service.rpc.thrift.TCLIService.Client
 import org.apache.thrift.protocol.TBinaryProtocol
 import org.apache.thrift.transport.TSocket
 import org.scalatest.BeforeAndAfterAll
+import org.scalatest.concurrent.Eventually._
 
 import org.apache.spark.{SparkException, SparkFunSuite}
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.hive.HiveUtils
 import org.apache.spark.sql.hive.test.HiveTestJars
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.StaticSQLConf.HIVE_THRIFT_SERVER_SINGLESESSION
 import org.apache.spark.sql.test.ProcessTestUtils.ProcessOutputCapturer
 import org.apache.spark.util.{ThreadUtils, Utils}
@@ -58,7 +61,7 @@ object TestData {
   val smallKvWithNull = getTestDataFilePath("small_kv_with_null.txt")
 }
 
-class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
+class HiveThriftBinaryServerSuite extends HiveThriftServer2Test {
   override def mode: ServerMode.Value = ServerMode.binary
 
   private def withCLIServiceClient(f: ThriftCLIServiceClient => Unit): Unit = {
@@ -67,7 +70,7 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
     val user = System.getProperty("user.name")
     val transport = PlainSaslHelper.getPlainTransport(user, "anonymous", rawTransport)
     val protocol = new TBinaryProtocol(transport)
-    val client = new ThriftCLIServiceClient(new ThriftserverShimUtils.Client(protocol))
+    val client = new ThriftCLIServiceClient(new Client(protocol))
 
     transport.open()
     try f(client) finally transport.close()
@@ -284,7 +287,6 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
   }
 
   test("test multiple session") {
-    import org.apache.spark.sql.internal.SQLConf
     var defaultV1: String = null
     var defaultV2: String = null
     var data: ArrayBuffer[Int] = null
@@ -304,7 +306,7 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
         val plan = statement.executeQuery("explain select * from test_table")
         plan.next()
         plan.next()
-        assert(plan.getString(1).contains("Scan In-memory table `test_table`"))
+        assert(plan.getString(1).contains("Scan In-memory table test_table"))
 
         val rs1 = statement.executeQuery("SELECT key FROM test_table ORDER BY KEY DESC")
         val buf1 = new collection.mutable.ArrayBuffer[Int]()
@@ -387,10 +389,11 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
           statement.executeQuery("SELECT key FROM test_table ORDER BY KEY DESC")
         }
 
+        // The cached temporary table can be used indirectly if the query matches.
         val plan = statement.executeQuery("explain select key from test_map ORDER BY key DESC")
         plan.next()
         plan.next()
-        assert(plan.getString(1).contains("Scan In-memory table `test_table`"))
+        assert(plan.getString(1).contains("Scan In-memory table test_table"))
 
         val rs = statement.executeQuery("SELECT key FROM test_map ORDER BY KEY DESC")
         val buf = new collection.mutable.ArrayBuffer[Int]()
@@ -544,11 +547,7 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
         conf += resultSet.getString(1) -> resultSet.getString(2)
       }
 
-      if (HiveUtils.isHive23) {
-        assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("2.3.7"))
-      } else {
-        assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("1.2.1"))
-      }
+      assert(conf.get(HiveUtils.BUILTIN_HIVE_VERSION.key) === Some(HiveUtils.builtinHiveVersion))
     }
   }
 
@@ -561,11 +560,7 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
         conf += resultSet.getString(1) -> resultSet.getString(2)
       }
 
-      if (HiveUtils.isHive23) {
-        assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("2.3.7"))
-      } else {
-        assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("1.2.1"))
-      }
+      assert(conf.get(HiveUtils.BUILTIN_HIVE_VERSION.key) === Some(HiveUtils.builtinHiveVersion))
     }
   }
 
@@ -643,11 +638,7 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
       val sessionHandle = client.openSession(user, "")
       val sessionID = sessionHandle.getSessionId
 
-      if (HiveUtils.isHive23) {
-        assert(pipeoutFileList(sessionID).length == 2)
-      } else {
-        assert(pipeoutFileList(sessionID).length == 1)
-      }
+      assert(pipeoutFileList(sessionID).length == 2)
 
       client.closeSession(sessionHandle)
 
@@ -684,6 +675,55 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
       }
       assert(e.getMessage.contains("org.apache.spark.sql.catalyst.parser.ParseException"))
     }
+
+    withJdbcStatement() { statement =>
+      val rs = statement.executeQuery("SELECT interval '3-1' year to month;")
+      assert(rs.next())
+      assert(rs.getString(1) === "3-1")
+    }
+
+    withJdbcStatement() { statement =>
+      val rs = statement.executeQuery("SELECT interval '3 1:1:1' day to second;")
+      assert(rs.next())
+      assert(rs.getString(1) === "3 01:01:01.000000000")
+    }
+  }
+
+  test("Query Intervals in VIEWs through thrift server") {
+    val viewName1 = "view_interval_1"
+    val viewName2 = "view_interval_2"
+    val ddl1 =
+      s"""
+         |CREATE GLOBAL TEMP VIEW $viewName1
+         |AS SELECT
+         | INTERVAL 1 DAY AS a,
+         | INTERVAL '2-1' YEAR TO MONTH AS b,
+         | INTERVAL '3 1:1:1' DAY TO SECOND AS c
+       """.stripMargin
+    val ddl2 = s"CREATE TEMP VIEW $viewName2 as select * from global_temp.$viewName1"
+    withJdbcStatement(viewName1, viewName2) { statement =>
+      statement.executeQuery(ddl1)
+      statement.executeQuery(ddl2)
+      val rs = statement.executeQuery(
+        s"""
+           |SELECT v1.a AS a1, v2.a AS a2,
+           | v1.b AS b1, v2.b AS b2,
+           | v1.c AS c1, v2.c AS c2
+           |FROM global_temp.$viewName1 v1
+           |JOIN $viewName2 v2
+           |ON date_part('DAY', v1.a) = date_part('DAY', v2.a)
+           |  AND v1.b = v2.b
+           |  AND v1.c = v2.c
+           |""".stripMargin)
+      while (rs.next()) {
+        assert(rs.getString("a1") === "1 days")
+        assert(rs.getString("a2") === "1 days")
+        assert(rs.getString("b1") === "2-1")
+        assert(rs.getString("b2") === "2-1")
+        assert(rs.getString("c1") === "3 01:01:01.000000000")
+        assert(rs.getString("c2") === "3 01:01:01.000000000")
+      }
+    }
   }
 
   test("ThriftCLIService FetchResults FETCH_FIRST, FETCH_NEXT, FETCH_PRIOR") {
@@ -811,16 +851,132 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
       }
     }
   }
+
+  test("SPARK-31859 Thriftserver works with spark.sql.datetime.java8API.enabled=true") {
+    withJdbcStatement() { st =>
+      st.execute("set spark.sql.datetime.java8API.enabled=true")
+      val rs = st.executeQuery("select date '2020-05-28', timestamp '2020-05-28 00:00:00'")
+      rs.next()
+      assert(rs.getDate(1).toString() == "2020-05-28")
+      assert(rs.getTimestamp(2).toString() == "2020-05-28 00:00:00.0")
+    }
+  }
+
+  test("SPARK-31861 Thriftserver respects spark.sql.session.timeZone") {
+    withJdbcStatement() { st =>
+      st.execute("set spark.sql.session.timeZone=+03:15") // different than Thriftserver's JVM tz
+      val rs = st.executeQuery("select timestamp '2020-05-28 10:00:00'")
+      rs.next()
+      // The timestamp as string is the same as the literal
+      assert(rs.getString(1) == "2020-05-28 10:00:00.0")
+      // Parsing it to java.sql.Timestamp in the client will always result in a timestamp
+      // in client default JVM timezone. The string value of the Timestamp will match the literal,
+      // but if the JDBC application cares about the internal timezone and UTC offset of the
+      // Timestamp object, it should set spark.sql.session.timeZone to match its client JVM tz.
+      assert(rs.getTimestamp(1).toString() == "2020-05-28 10:00:00.0")
+    }
+  }
+
+  test("SPARK-31863 Session conf should persist between Thriftserver worker threads") {
+    val iter = 20
+    withJdbcStatement() { statement =>
+      // date 'now' is resolved during parsing, and relies on SQLConf.get to
+      // obtain the current set timezone. We exploit this to run this test.
+      // If the timezones are set correctly to 25 hours apart across threads,
+      // the dates should reflect this.
+
+      // iterate a few times for the odd chance the same thread is selected
+      for (_ <- 0 until iter) {
+        statement.execute("SET spark.sql.session.timeZone=GMT-12")
+        val firstResult = statement.executeQuery("SELECT date 'now'")
+        firstResult.next()
+        val beyondDateLineWest = firstResult.getDate(1)
+
+        statement.execute("SET spark.sql.session.timeZone=GMT+13")
+        val secondResult = statement.executeQuery("SELECT date 'now'")
+        secondResult.next()
+        val dateLineEast = secondResult.getDate(1)
+        assert(
+          dateLineEast after beyondDateLineWest,
+          "SQLConf changes should persist across execution threads")
+      }
+    }
+  }
+
+  test("SPARK-30808: use Java 8 time API and Proleptic Gregorian calendar by default") {
+    withJdbcStatement() { st =>
+      // Proleptic Gregorian calendar has no gap in the range 1582-10-04..1582-10-15
+      val date = "1582-10-10"
+      val rs = st.executeQuery(s"select date '$date'")
+      rs.next()
+      val expected = java.sql.Date.valueOf(date)
+      assert(rs.getDate(1) === expected)
+      assert(rs.getString(1) === expected.toString)
+    }
+  }
+
+  test("SPARK-26533: Support query auto timeout cancel on thriftserver - setQueryTimeout") {
+    withJdbcStatement() { statement =>
+      statement.setQueryTimeout(1)
+      val e = intercept[SQLException] {
+        statement.execute("select java_method('java.lang.Thread', 'sleep', 10000L)")
+      }.getMessage
+      assert(e.contains("Query timed out after"))
+
+      statement.setQueryTimeout(0)
+      val rs1 = statement.executeQuery(
+        "select 'test', java_method('java.lang.Thread', 'sleep', 3000L)")
+      rs1.next()
+      assert(rs1.getString(1) == "test")
+
+      statement.setQueryTimeout(-1)
+      val rs2 = statement.executeQuery(
+        "select 'test', java_method('java.lang.Thread', 'sleep', 3000L)")
+      rs2.next()
+      assert(rs2.getString(1) == "test")
+    }
+  }
+
+  test("SPARK-26533: Support query auto timeout cancel on thriftserver - SQLConf") {
+    withJdbcStatement() { statement =>
+      statement.execute(s"SET ${SQLConf.THRIFTSERVER_QUERY_TIMEOUT.key}=1")
+      val e1 = intercept[SQLException] {
+        statement.execute("select java_method('java.lang.Thread', 'sleep', 10000L)")
+      }.getMessage
+      assert(e1.contains("Query timed out after"))
+
+      statement.execute(s"SET ${SQLConf.THRIFTSERVER_QUERY_TIMEOUT.key}=0")
+      val rs = statement.executeQuery(
+        "select 'test', java_method('java.lang.Thread', 'sleep', 3000L)")
+      rs.next()
+      assert(rs.getString(1) == "test")
+
+      // Uses a smaller timeout value of a config value and an a user-specified one
+      statement.execute(s"SET ${SQLConf.THRIFTSERVER_QUERY_TIMEOUT.key}=1")
+      statement.setQueryTimeout(30)
+      val e2 = intercept[SQLException] {
+        statement.execute("select java_method('java.lang.Thread', 'sleep', 10000L)")
+      }.getMessage
+      assert(e2.contains("Query timed out after"))
+
+      statement.execute(s"SET ${SQLConf.THRIFTSERVER_QUERY_TIMEOUT.key}=30")
+      statement.setQueryTimeout(1)
+      val e3 = intercept[SQLException] {
+        statement.execute("select java_method('java.lang.Thread', 'sleep', 10000L)")
+      }.getMessage
+      assert(e3.contains("Query timed out after"))
+    }
+  }
 }
 
-class SingleSessionSuite extends HiveThriftJdbcTest {
+class SingleSessionSuite extends HiveThriftServer2TestBase {
   override def mode: ServerMode.Value = ServerMode.binary
 
   override protected def extraConf: Seq[String] =
     s"--conf ${HIVE_THRIFT_SERVER_SINGLESESSION.key}=true" :: Nil
 
   test("share the temporary functions across JDBC connections") {
-    withMultipleConnectionJdbcStatement()(
+    withMultipleConnectionJdbcStatement("test_udtf")(
       { statement =>
         val jarPath = "../hive/src/test/resources/TestUDTF.jar"
         val jarURL = s"file://${System.getProperty("user.dir")}/$jarPath"
@@ -924,7 +1080,7 @@ class SingleSessionSuite extends HiveThriftJdbcTest {
   }
 }
 
-class HiveThriftCleanUpScratchDirSuite extends HiveThriftJdbcTest{
+class HiveThriftCleanUpScratchDirSuite extends HiveThriftServer2TestBase {
   var tempScratchDir: File = _
 
   override protected def beforeAll(): Unit = {
@@ -957,7 +1113,7 @@ class HiveThriftCleanUpScratchDirSuite extends HiveThriftJdbcTest{
   }
 }
 
-class HiveThriftHttpServerSuite extends HiveThriftJdbcTest {
+class HiveThriftHttpServerSuite extends HiveThriftServer2Test {
   override def mode: ServerMode.Value = ServerMode.http
 
   test("JDBC query execution") {
@@ -1000,63 +1156,7 @@ object ServerMode extends Enumeration {
   val binary, http = Value
 }
 
-abstract class HiveThriftJdbcTest extends HiveThriftServer2Test {
-  Utils.classForName(classOf[HiveDriver].getCanonicalName)
-
-  private def jdbcUri = if (mode == ServerMode.http) {
-    s"""jdbc:hive2://localhost:$serverPort/
-       |default?
-       |hive.server2.transport.mode=http;
-       |hive.server2.thrift.http.path=cliservice;
-       |${hiveConfList}#${hiveVarList}
-     """.stripMargin.split("\n").mkString.trim
-  } else {
-    s"jdbc:hive2://localhost:$serverPort/?${hiveConfList}#${hiveVarList}"
-  }
-
-  def withMultipleConnectionJdbcStatement(tableNames: String*)(fs: (Statement => Unit)*): Unit = {
-    val user = System.getProperty("user.name")
-    val connections = fs.map { _ => DriverManager.getConnection(jdbcUri, user, "") }
-    val statements = connections.map(_.createStatement())
-
-    try {
-      statements.zip(fs).foreach { case (s, f) => f(s) }
-    } finally {
-      tableNames.foreach { name =>
-        // TODO: Need a better way to drop the view.
-        if (name.toUpperCase(Locale.ROOT).startsWith("VIEW")) {
-          statements(0).execute(s"DROP VIEW IF EXISTS $name")
-        } else {
-          statements(0).execute(s"DROP TABLE IF EXISTS $name")
-        }
-      }
-      statements.foreach(_.close())
-      connections.foreach(_.close())
-    }
-  }
-
-  def withDatabase(dbNames: String*)(fs: (Statement => Unit)*): Unit = {
-    val user = System.getProperty("user.name")
-    val connections = fs.map { _ => DriverManager.getConnection(jdbcUri, user, "") }
-    val statements = connections.map(_.createStatement())
-
-    try {
-      statements.zip(fs).foreach { case (s, f) => f(s) }
-    } finally {
-      dbNames.foreach { name =>
-        statements(0).execute(s"DROP DATABASE IF EXISTS $name")
-      }
-      statements.foreach(_.close())
-      connections.foreach(_.close())
-    }
-  }
-
-  def withJdbcStatement(tableNames: String*)(f: Statement => Unit): Unit = {
-    withMultipleConnectionJdbcStatement(tableNames: _*)(f)
-  }
-}
-
-abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAll with Logging {
+abstract class HiveThriftServer2TestBase extends SparkFunSuite with BeforeAndAfterAll with Logging {
   def mode: ServerMode.Value
 
   private val CLASS_NAME = HiveThriftServer2.getClass.getCanonicalName.stripSuffix("$")
@@ -1085,7 +1185,7 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl
 
   protected def extraConf: Seq[String] = Nil
 
-  protected def serverStartCommand(port: Int) = {
+  protected def serverStartCommand(): Seq[String] = {
     val portConf = if (mode == ServerMode.binary) {
       ConfVars.HIVE_SERVER2_THRIFT_PORT
     } else {
@@ -1098,7 +1198,7 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl
       val tempLog4jConf = Utils.createTempDir().getCanonicalPath
 
       Files.write(
-        """log4j.rootCategory=DEBUG, console
+        """log4j.rootCategory=INFO, console
           |log4j.appender.console=org.apache.log4j.ConsoleAppender
           |log4j.appender.console.target=System.err
           |log4j.appender.console.layout=org.apache.log4j.PatternLayout
@@ -1118,7 +1218,7 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl
        |  --hiveconf ${ConfVars.HIVE_SERVER2_TRANSPORT_MODE}=$mode
        |  --hiveconf ${ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LOG_LOCATION}=$operationLogPath
        |  --hiveconf ${ConfVars.LOCALSCRATCHDIR}=$lScratchDir
-       |  --hiveconf $portConf=$port
+       |  --hiveconf $portConf=0
        |  --driver-class-path $driverClassPath
        |  --driver-java-options -Dlog4j.debug
        |  --conf spark.ui.enabled=false
@@ -1140,7 +1240,7 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl
 
   val SERVER_STARTUP_TIMEOUT = 3.minutes
 
-  private def startThriftServer(port: Int, attempt: Int) = {
+  private def startThriftServer(attempt: Int) = {
     warehousePath = Utils.createTempDir()
     warehousePath.delete()
     metastorePath = Utils.createTempDir()
@@ -1152,18 +1252,16 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl
     logPath = null
     logTailingProcess = null
 
-    val command = serverStartCommand(port)
+    val command = serverStartCommand()
 
     diagnosisBuffer ++=
       s"""
          |### Attempt $attempt ###
          |HiveThriftServer2 command line: $command
-         |Listening port: $port
+         |Listening port: 0
          |System user: $user
        """.stripMargin.split("\n")
 
-    logInfo(s"Trying to start HiveThriftServer2: port=$port, mode=$mode, attempt=$attempt")
-
     logPath = {
       val lines = Utils.executeAndGetOutput(
         command = command,
@@ -1190,7 +1288,11 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl
 
     // Ensures that the following "tail" command won't fail.
     logPath.createNewFile()
-    val successLines = Seq(THRIFT_BINARY_SERVICE_LIVE, THRIFT_HTTP_SERVICE_LIVE)
+    val successLine = if (mode == ServerMode.http) {
+      THRIFT_HTTP_SERVICE_LIVE
+    } else {
+      THRIFT_BINARY_SERVICE_LIVE
+    }
 
     logTailingProcess = {
       val command = s"/usr/bin/env tail -n +0 -f ${logPath.getCanonicalPath}".split(" ")
@@ -1199,14 +1301,15 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl
       val captureOutput = (line: String) => diagnosisBuffer.synchronized {
         diagnosisBuffer += line
 
-        successLines.foreach { r =>
-          if (line.contains(r)) {
-            serverStarted.trySuccess(())
-          }
+        if (line.contains(successLine)) {
+          listeningPort = line.split(" on port ")(1).split(' ').head.toInt
+          logInfo(s"Started HiveThriftServer2: port=$listeningPort, mode=$mode, attempt=$attempt")
+          serverStarted.trySuccess(())
+          ()
         }
       }
 
-        val process = builder.start()
+      val process = builder.start()
 
       new ProcessOutputCapturer(process.getInputStream, captureOutput).start()
       new ProcessOutputCapturer(process.getErrorStream, captureOutput).start()
@@ -1257,16 +1360,18 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl
 
   override protected def beforeAll(): Unit = {
     super.beforeAll()
-    // Chooses a random port between 10000 and 19999
-    listeningPort = 10000 + Random.nextInt(10000)
     diagnosisBuffer.clear()
 
     // Retries up to 3 times with different port numbers if the server fails to start
-    (1 to 3).foldLeft(Try(startThriftServer(listeningPort, 0))) { case (started, attempt) =>
+    (1 to 3).foldLeft(Try(startThriftServer(0))) { case (started, attempt) =>
       started.orElse {
-        listeningPort += 1
         stopThriftServer()
-        Try(startThriftServer(listeningPort, attempt))
+        Try {
+          startThriftServer(attempt)
+          eventually(timeout(30.seconds), interval(1.seconds)) {
+            withJdbcStatement() { _.execute("SELECT 1") }
+          }
+        }
       }
     }.recover {
       case cause: Throwable =>
@@ -1285,4 +1390,91 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl
       super.afterAll()
     }
   }
+
+  Utils.classForName(classOf[HiveDriver].getCanonicalName)
+
+  protected def jdbcUri(database: String = "default"): String = if (mode == ServerMode.http) {
+    s"""jdbc:hive2://localhost:$serverPort/
+       |$database?
+       |hive.server2.transport.mode=http;
+       |hive.server2.thrift.http.path=cliservice;
+       |${hiveConfList}#${hiveVarList}
+     """.stripMargin.split("\n").mkString.trim
+  } else {
+    s"jdbc:hive2://localhost:$serverPort/$database?${hiveConfList}#${hiveVarList}"
+  }
+
+  private def tryCaptureSysLog(f: => Unit): Unit = {
+    try f catch {
+      case e: Exception =>
+        // Dump the HiveThriftServer2 log if error occurs, e.g. getConnection failure.
+        dumpLogs()
+        throw e
+    }
+  }
+
+  def withMultipleConnectionJdbcStatement(
+      tableNames: String*)(fs: (Statement => Unit)*): Unit = tryCaptureSysLog {
+    val user = System.getProperty("user.name")
+    val connections = fs.map { _ => DriverManager.getConnection(jdbcUri(), user, "") }
+    val statements = connections.map(_.createStatement())
+
+    try {
+      statements.zip(fs).foreach { case (s, f) => f(s) }
+    } finally {
+      tableNames.foreach { name =>
+        // TODO: Need a better way to drop the view.
+        if (name.toUpperCase(Locale.ROOT).startsWith("VIEW")) {
+          statements(0).execute(s"DROP VIEW IF EXISTS $name")
+        } else {
+          statements(0).execute(s"DROP TABLE IF EXISTS $name")
+        }
+      }
+      statements.foreach(_.close())
+      connections.foreach(_.close())
+    }
+  }
+
+  def withDatabase(dbNames: String*)(fs: (Statement => Unit)*): Unit = tryCaptureSysLog {
+    val user = System.getProperty("user.name")
+    val connections = fs.map { _ => DriverManager.getConnection(jdbcUri(), user, "") }
+    val statements = connections.map(_.createStatement())
+
+    try {
+      statements.zip(fs).foreach { case (s, f) => f(s) }
+    } finally {
+      dbNames.foreach { name =>
+        statements(0).execute(s"DROP DATABASE IF EXISTS $name")
+      }
+      statements.foreach(_.close())
+      connections.foreach(_.close())
+    }
+  }
+
+  def withJdbcStatement(tableNames: String*)(f: Statement => Unit): Unit = {
+    withMultipleConnectionJdbcStatement(tableNames: _*)(f)
+  }
+}
+
+/**
+ * Common tests for both binary and http mode thrift server
+ * TODO: SPARK-31914: Move common tests from subclasses to this trait
+ */
+abstract class HiveThriftServer2Test extends HiveThriftServer2TestBase {
+  test("SPARK-17819: Support default database in connection URIs") {
+    withDatabase("spark17819") { statement =>
+      statement.execute(s"CREATE DATABASE IF NOT EXISTS spark17819")
+      val jdbcStr = jdbcUri("spark17819")
+      val connection = DriverManager.getConnection(jdbcStr, user, "")
+      val statementN = connection.createStatement()
+      try {
+        val resultSet = statementN.executeQuery("select current_database()")
+        resultSet.next()
+        assert(resultSet.getString(1) === "spark17819")
+      } finally {
+        statementN.close()
+        connection.close()
+      }
+    }
+  }
 }
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/JdbcConnectionUriSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/JdbcConnectionUriSuite.scala
deleted file mode 100644
index fb8a7e273ae44..0000000000000
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/JdbcConnectionUriSuite.scala
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.hive.thriftserver
-
-import java.sql.DriverManager
-
-import org.apache.hive.jdbc.HiveDriver
-
-import org.apache.spark.util.Utils
-
-class JdbcConnectionUriSuite extends HiveThriftServer2Test {
-  Utils.classForName(classOf[HiveDriver].getCanonicalName)
-
-  override def mode: ServerMode.Value = ServerMode.binary
-
-  val JDBC_TEST_DATABASE = "jdbc_test_database"
-  val USER = System.getProperty("user.name")
-  val PASSWORD = ""
-
-  override protected def beforeAll(): Unit = {
-    super.beforeAll()
-
-    val jdbcUri = s"jdbc:hive2://localhost:$serverPort/"
-    val connection = DriverManager.getConnection(jdbcUri, USER, PASSWORD)
-    val statement = connection.createStatement()
-    statement.execute(s"CREATE DATABASE $JDBC_TEST_DATABASE")
-    connection.close()
-  }
-
-  override protected def afterAll(): Unit = {
-    try {
-      val jdbcUri = s"jdbc:hive2://localhost:$serverPort/"
-      val connection = DriverManager.getConnection(jdbcUri, USER, PASSWORD)
-      val statement = connection.createStatement()
-      statement.execute(s"DROP DATABASE $JDBC_TEST_DATABASE")
-      connection.close()
-    } finally {
-      super.afterAll()
-    }
-  }
-
-  test("SPARK-17819 Support default database in connection URIs") {
-    val jdbcUri = s"jdbc:hive2://localhost:$serverPort/$JDBC_TEST_DATABASE"
-    val connection = DriverManager.getConnection(jdbcUri, USER, PASSWORD)
-    val statement = connection.createStatement()
-    try {
-      val resultSet = statement.executeQuery("select current_database()")
-      resultSet.next()
-      assert(resultSet.getString(1) === JDBC_TEST_DATABASE)
-    } finally {
-      statement.close()
-      connection.close()
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala
index ce610098156f3..8f61268c838fe 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala
@@ -17,31 +17,47 @@
 
 package org.apache.spark.sql.hive.thriftserver
 
+import java.io.File
 import java.sql.{DriverManager, Statement}
+import java.util
 
+import scala.collection.JavaConverters._
 import scala.concurrent.duration._
-import scala.util.{Random, Try}
+import scala.util.Try
 
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars
+import org.apache.hadoop.hive.ql.metadata.Hive
+import org.apache.hadoop.hive.ql.session.SessionState
+import org.apache.hive.jdbc.HttpBasicAuthInterceptor
+import org.apache.hive.service.auth.PlainSaslHelper
+import org.apache.hive.service.cli.thrift.{ThriftCLIService, ThriftCLIServiceClient}
+import org.apache.hive.service.rpc.thrift.TCLIService.Client
+import org.apache.http.impl.client.HttpClientBuilder
+import org.apache.thrift.protocol.TBinaryProtocol
+import org.apache.thrift.transport.{THttpClient, TSocket}
 
-import org.apache.spark.sql.QueryTest
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.util.Utils
 
 trait SharedThriftServer extends SharedSparkSession {
 
   private var hiveServer2: HiveThriftServer2 = _
+  private var serverPort: Int = 0
+
+  protected val tempScratchDir: File = {
+    val dir = Utils.createTempDir()
+    dir.setWritable(true, false)
+    Utils.createTempDir(dir.getAbsolutePath)
+    dir
+  }
+
+  def mode: ServerMode.Value
 
   override def beforeAll(): Unit = {
     super.beforeAll()
-    // Chooses a random port between 10000 and 19999
-    var listeningPort = 10000 + Random.nextInt(10000)
-
     // Retries up to 3 times with different port numbers if the server fails to start
-    (1 to 3).foldLeft(Try(startThriftServer(listeningPort, 0))) { case (started, attempt) =>
-      started.orElse {
-        listeningPort += 1
-        Try(startThriftServer(listeningPort, attempt))
-      }
+    (1 to 3).foldLeft(Try(startThriftServer(0))) { case (started, attempt) =>
+      started.orElse(Try(startThriftServer(attempt)))
     }.recover {
       case cause: Throwable =>
         throw cause
@@ -51,18 +67,28 @@ trait SharedThriftServer extends SharedSparkSession {
 
   override def afterAll(): Unit = {
     try {
-      hiveServer2.stop()
+      if (hiveServer2 != null) {
+        hiveServer2.stop()
+      }
     } finally {
       super.afterAll()
+      SessionState.detachSession()
+      Hive.closeCurrent()
     }
   }
 
-  protected def withJdbcStatement(fs: (Statement => Unit)*): Unit = {
-    val user = System.getProperty("user.name")
+  protected def jdbcUri: String = if (mode == ServerMode.http) {
+    s"jdbc:hive2://localhost:$serverPort/default;transportMode=http;httpPath=cliservice"
+  } else {
+    s"jdbc:hive2://localhost:$serverPort/"
+  }
 
-    val serverPort = hiveServer2.getHiveConf.get(ConfVars.HIVE_SERVER2_THRIFT_PORT.varname)
+  protected def user: String = System.getProperty("user.name")
+
+  protected def withJdbcStatement(fs: (Statement => Unit)*): Unit = {
+    require(serverPort != 0, "Failed to bind an actual port for HiveThriftServer2")
     val connections =
-      fs.map { _ => DriverManager.getConnection(s"jdbc:hive2://localhost:$serverPort", user, "") }
+      fs.map { _ => DriverManager.getConnection(jdbcUri, user, "") }
     val statements = connections.map(_.createStatement())
 
     try {
@@ -73,16 +99,69 @@ trait SharedThriftServer extends SharedSparkSession {
     }
   }
 
-  private def startThriftServer(port: Int, attempt: Int): Unit = {
-    logInfo(s"Trying to start HiveThriftServer2: port=$port, attempt=$attempt")
+  protected def withCLIServiceClient(f: ThriftCLIServiceClient => Unit): Unit = {
+    require(serverPort != 0, "Failed to bind an actual port for HiveThriftServer2")
+    val transport = mode match {
+      case ServerMode.binary =>
+        val rawTransport = new TSocket("localhost", serverPort)
+        PlainSaslHelper.getPlainTransport(user, "anonymous", rawTransport)
+      case ServerMode.http =>
+        val interceptor = new HttpBasicAuthInterceptor(
+          user,
+          "anonymous",
+          null, null, true, new util.HashMap[String, String]())
+        new THttpClient(
+          s"http://localhost:$serverPort/cliservice",
+          HttpClientBuilder.create.addInterceptorFirst(interceptor).build())
+    }
+
+    val protocol = new TBinaryProtocol(transport)
+    val client = new ThriftCLIServiceClient(new Client(protocol))
+
+    transport.open()
+    try f(client) finally transport.close()
+  }
+
+  private def startThriftServer(attempt: Int): Unit = {
+    logInfo(s"Trying to start HiveThriftServer2: mode=$mode, attempt=$attempt")
     val sqlContext = spark.newSession().sqlContext
-    sqlContext.setConf(ConfVars.HIVE_SERVER2_THRIFT_PORT.varname, port.toString)
-    hiveServer2 = HiveThriftServer2.startWithContext(sqlContext)
+    // Set the HIVE_SERVER2_THRIFT_PORT and HIVE_SERVER2_THRIFT_HTTP_PORT to 0, so it could
+    // randomly pick any free port to use.
+    // It's much more robust than set a random port generated by ourselves ahead
+    sqlContext.setConf(ConfVars.HIVE_SERVER2_THRIFT_PORT.varname, "0")
+    sqlContext.setConf(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT.varname, "0")
+    sqlContext.setConf(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, mode.toString)
+    sqlContext.setConf(ConfVars.SCRATCHDIR.varname, tempScratchDir.getAbsolutePath)
+    sqlContext.setConf(ConfVars.HIVE_START_CLEANUP_SCRATCHDIR.varname, "true")
 
-    // Wait for thrift server to be ready to serve the query, via executing simple query
-    // till the query succeeds. See SPARK-30345 for more details.
-    eventually(timeout(30.seconds), interval(1.seconds)) {
-      withJdbcStatement { _.execute("SELECT 1") }
+    try {
+      hiveServer2 = HiveThriftServer2.startWithContext(sqlContext)
+      hiveServer2.getServices.asScala.foreach {
+        case t: ThriftCLIService =>
+          serverPort = t.getPortNumber
+          logInfo(s"Started HiveThriftServer2: mode=$mode, port=$serverPort, attempt=$attempt")
+        case _ =>
+      }
+
+      // the scratch dir will be recreated after the probe sql `SELECT 1` executed, so we
+      // check it here first.
+      assert(!tempScratchDir.exists())
+
+      // Wait for thrift server to be ready to serve the query, via executing simple query
+      // till the query succeeds. See SPARK-30345 for more details.
+      eventually(timeout(30.seconds), interval(1.seconds)) {
+        withJdbcStatement { _.execute("SELECT 1") }
+      }
+    } catch {
+      case e: Exception =>
+        logError("Error start hive server with Context ", e)
+        if (hiveServer2 != null) {
+          hiveServer2.stop()
+          hiveServer2 = null
+        }
+        SessionState.detachSession()
+        Hive.closeCurrent()
+        throw e
     }
   }
 }
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperationSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperationSuite.scala
index 13df3fabc4919..c8bb6d9ee0821 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperationSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperationSuite.scala
@@ -17,10 +17,26 @@
 
 package org.apache.spark.sql.hive.thriftserver
 
+import java.util
+import java.util.concurrent.Semaphore
+
+import scala.concurrent.duration._
+
+import org.apache.hadoop.hive.conf.HiveConf
+import org.apache.hive.service.cli.OperationState
+import org.apache.hive.service.cli.session.{HiveSession, HiveSessionImpl}
+import org.apache.hive.service.rpc.thrift.TProtocolVersion
+import org.mockito.Mockito.{doReturn, mock, spy, when, RETURNS_DEEP_STUBS}
+import org.mockito.invocation.InvocationOnMock
+
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.{DataFrame, SQLContext}
+import org.apache.spark.sql.hive.thriftserver.ui.HiveThriftServer2EventManager
+import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.{IntegerType, NullType, StringType, StructField, StructType}
 
-class SparkExecuteStatementOperationSuite extends SparkFunSuite {
+class SparkExecuteStatementOperationSuite extends SparkFunSuite with SharedSparkSession {
+
   test("SPARK-17112 `select null` via JDBC triggers IllegalArgumentException in ThriftServer") {
     val field1 = StructField("NULL", NullType)
     val field2 = StructField("(IF(true, NULL, NULL))", NullType)
@@ -42,4 +58,69 @@ class SparkExecuteStatementOperationSuite extends SparkFunSuite {
     assert(columns.get(1).getType().getName == "INT")
     assert(columns.get(1).getComment() == "")
   }
+
+  Seq(
+    (OperationState.CANCELED, (_: SparkExecuteStatementOperation).cancel()),
+    (OperationState.TIMEDOUT, (_: SparkExecuteStatementOperation).timeoutCancel()),
+    (OperationState.CLOSED, (_: SparkExecuteStatementOperation).close())
+  ).foreach { case (finalState, transition) =>
+    test("SPARK-32057 SparkExecuteStatementOperation should not transiently become ERROR " +
+      s"before being set to $finalState") {
+      val hiveSession = new HiveSessionImpl(TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V1,
+      "username", "password", new HiveConf, "ip address")
+      hiveSession.open(new util.HashMap)
+
+      HiveThriftServer2.eventManager = mock(classOf[HiveThriftServer2EventManager])
+
+      val spySqlContext = spy(sqlContext)
+
+      // When cancel() is called on the operation, cleanup causes an exception to be thrown inside
+      // of execute(). This should not cause the state to become ERROR. The exception here will be
+      // triggered in our custom cleanup().
+      val signal = new Semaphore(0)
+      val dataFrame = mock(classOf[DataFrame], RETURNS_DEEP_STUBS)
+      when(dataFrame.collect()).thenAnswer((_: InvocationOnMock) => {
+        signal.acquire()
+        throw new RuntimeException("Operation was cancelled by test cleanup.")
+      })
+      val statement = "stmt"
+      doReturn(dataFrame, Nil: _*).when(spySqlContext).sql(statement)
+
+      val executeStatementOperation = new MySparkExecuteStatementOperation(spySqlContext,
+        hiveSession, statement, signal, finalState)
+
+      val run = new Thread() {
+        override def run(): Unit = executeStatementOperation.runInternal()
+      }
+      assert(executeStatementOperation.getStatus.getState === OperationState.INITIALIZED)
+      run.start()
+      eventually(timeout(5.seconds)) {
+        assert(executeStatementOperation.getStatus.getState === OperationState.RUNNING)
+      }
+      transition(executeStatementOperation)
+      run.join()
+      assert(executeStatementOperation.getStatus.getState === finalState)
+    }
+  }
+
+  private class MySparkExecuteStatementOperation(
+      sqlContext: SQLContext,
+      hiveSession: HiveSession,
+      statement: String,
+      signal: Semaphore,
+      finalState: OperationState)
+    extends SparkExecuteStatementOperation(sqlContext, hiveSession, statement,
+      new util.HashMap, false, 0) {
+
+    override def cleanup(): Unit = {
+      super.cleanup()
+      signal.release()
+      // At this point, operation should already be in finalState (set by either close() or
+      // cancel()). We want to check if it stays in finalState after the exception thrown by
+      // releasing the semaphore propagates. We hence need to sleep for a short while.
+      Thread.sleep(1000)
+      // State should not be ERROR
+      assert(getStatus.getState === finalState)
+    }
+  }
 }
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala
index f7ee3e0a46cd1..bc2d413fdf1e9 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala
@@ -17,31 +17,56 @@
 
 package org.apache.spark.sql.hive.thriftserver
 
-import java.sql.{DatabaseMetaData, ResultSet}
+import java.sql.{DatabaseMetaData, ResultSet, SQLFeatureNotSupportedException}
 
-class SparkMetadataOperationSuite extends HiveThriftJdbcTest {
+import org.apache.hive.common.util.HiveVersionInfo
+import org.apache.hive.service.cli.HiveSQLException
+
+import org.apache.spark.SPARK_VERSION
+import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
+import org.apache.spark.sql.types._
+import org.apache.spark.util.VersionUtils
+
+class SparkMetadataOperationSuite extends HiveThriftServer2TestBase {
 
   override def mode: ServerMode.Value = ServerMode.binary
 
   test("Spark's own GetSchemasOperation(SparkGetSchemasOperation)") {
     def checkResult(rs: ResultSet, dbNames: Seq[String]): Unit = {
-      for (i <- dbNames.indices) {
-        assert(rs.next())
-        assert(rs.getString("TABLE_SCHEM") === dbNames(i))
+      val expected = dbNames.iterator
+      while(rs.next() || expected.hasNext) {
+        assert(rs.getString("TABLE_SCHEM") === expected.next)
+        assert(rs.getString("TABLE_CATALOG").isEmpty)
       }
       // Make sure there are no more elements
       assert(!rs.next())
+      assert(!expected.hasNext, "All expected schemas should be visited")
     }
 
-    withDatabase("db1", "db2") { statement =>
-      Seq("CREATE DATABASE db1", "CREATE DATABASE db2").foreach(statement.execute)
-
+    val dbs = Seq("db1", "db2", "db33", "db44")
+    val dbDflts = Seq("default", "global_temp")
+    withDatabase(dbs: _*) { statement =>
+      dbs.foreach( db => statement.execute(s"CREATE DATABASE IF NOT EXISTS $db"))
       val metaData = statement.getConnection.getMetaData
 
-      checkResult(metaData.getSchemas(null, "%"), Seq("db1", "db2", "default", "global_temp"))
+      Seq("", "%", null, ".*", "_*", "_%", ".%") foreach { pattern =>
+        checkResult(metaData.getSchemas(null, pattern), dbs ++ dbDflts)
+      }
+
+      Seq("db%", "db*") foreach { pattern =>
+        checkResult(metaData.getSchemas(null, pattern), dbs)
+      }
+
+      Seq("db_", "db.") foreach { pattern =>
+        checkResult(metaData.getSchemas(null, pattern), dbs.take(2))
+      }
+
       checkResult(metaData.getSchemas(null, "db1"), Seq("db1"))
       checkResult(metaData.getSchemas(null, "db_not_exist"), Seq.empty)
-      checkResult(metaData.getSchemas(null, "db*"), Seq("db1", "db2"))
+
+      val e = intercept[HiveSQLException](metaData.getSchemas(null, "*"))
+      assert(e.getCause.getMessage ===
+        "Error operating GET_SCHEMAS Dangling meta character '*' near index 0\n*\n^")
     }
   }
 
@@ -184,14 +209,16 @@ class SparkMetadataOperationSuite extends HiveThriftJdbcTest {
   }
 
   test("Spark's own GetFunctionsOperation(SparkGetFunctionsOperation)") {
-    def checkResult(rs: ResultSet, functionName: Seq[String]): Unit = {
-      for (i <- functionName.indices) {
+    def checkResult(rs: ResultSet, functionNames: Seq[String]): Unit = {
+      functionNames.foreach { func =>
+        val exprInfo = FunctionRegistry.expressions(func)._1
         assert(rs.next())
         assert(rs.getString("FUNCTION_SCHEM") === "default")
-        assert(rs.getString("FUNCTION_NAME") === functionName(i))
-        assert(rs.getString("REMARKS").startsWith(s"${functionName(i)}("))
+        assert(rs.getString("FUNCTION_NAME") === exprInfo.getName)
+        assert(rs.getString("REMARKS") ===
+          s"Usage: ${exprInfo.getUsage}\nExtended Usage:${exprInfo.getExtended}")
         assert(rs.getInt("FUNCTION_TYPE") === DatabaseMetaData.functionResultUnknown)
-        assert(rs.getString("SPECIFIC_NAME").startsWith("org.apache.spark.sql.catalyst"))
+        assert(rs.getString("SPECIFIC_NAME") === exprInfo.getClassName)
       }
       // Make sure there are no more elements
       assert(!rs.next())
@@ -207,20 +234,7 @@ class SparkMetadataOperationSuite extends HiveThriftJdbcTest {
       checkResult(metaData.getFunctions(null, "default", "overlay"), Seq("overlay"))
       checkResult(metaData.getFunctions(null, "default", "shift*"),
         Seq("shiftleft", "shiftright", "shiftrightunsigned"))
-    }
-
-    withJdbcStatement() { statement =>
-      val metaData = statement.getConnection.getMetaData
-      val rs = metaData.getFunctions(null, "default", "upPer")
-      assert(rs.next())
-      assert(rs.getString("FUNCTION_SCHEM") === "default")
-      assert(rs.getString("FUNCTION_NAME") === "upper")
-      assert(rs.getString("REMARKS") ===
-        "upper(str) - Returns `str` with all characters changed to uppercase.")
-      assert(rs.getInt("FUNCTION_TYPE") === DatabaseMetaData.functionResultUnknown)
-      assert(rs.getString("SPECIFIC_NAME") === "org.apache.spark.sql.catalyst.expressions.Upper")
-      // Make sure there are no more elements
-      assert(!rs.next())
+      checkResult(metaData.getFunctions(null, "default", "upPer"), Seq("upper"))
     }
   }
 
@@ -244,7 +258,404 @@ class SparkMetadataOperationSuite extends HiveThriftJdbcTest {
 
     withJdbcStatement() { statement =>
       val metaData = statement.getConnection.getMetaData
-      checkResult(metaData.getTypeInfo, ThriftserverShimUtils.supportedType().map(_.getName))
+      checkResult(metaData.getTypeInfo, SparkGetTypeInfoUtil.supportedType.map(_.getName))
+    }
+  }
+
+  test("check results from get columns operation from thrift server") {
+    val schemaName = "default"
+    val tableName = "spark_get_col_operation"
+    val schema = new StructType()
+      .add("c0", "boolean", nullable = false, "0")
+      .add("c1", "tinyint", nullable = true, "1")
+      .add("c2", "smallint", nullable = false, "2")
+      .add("c3", "int", nullable = true, "3")
+      .add("c4", "long", nullable = false, "4")
+      .add("c5", "float", nullable = true, "5")
+      .add("c6", "double", nullable = false, "6")
+      .add("c7", "decimal(38, 20)", nullable = true, "7")
+      .add("c8", "decimal(10, 2)", nullable = false, "8")
+      .add("c9", "string", nullable = true, "9")
+      .add("c10", "array<long>", nullable = false, "10")
+      .add("c11", "array<string>", nullable = true, "11")
+      .add("c12", "map<smallint, tinyint>", nullable = false, "12")
+      .add("c13", "date", nullable = true, "13")
+      .add("c14", "timestamp", nullable = false, "14")
+      .add("c15", "struct<X: bigint,Y: double>", nullable = true, "15")
+      .add("c16", "binary", nullable = false, "16")
+      .add("c17", "char(255)", nullable = true, "17")
+      .add("c18", "varchar(1024)", nullable = false, "18")
+
+    val ddl =
+      s"""
+         |CREATE TABLE $schemaName.$tableName (
+         |  ${schema.toDDL}
+         |)
+         |using parquet""".stripMargin
+
+    withJdbcStatement(tableName) { statement =>
+      statement.execute(ddl)
+
+      val databaseMetaData = statement.getConnection.getMetaData
+      val rowSet = databaseMetaData.getColumns("", schemaName, tableName, null)
+
+      import java.sql.Types._
+      val expectedJavaTypes = Seq(BOOLEAN, TINYINT, SMALLINT, INTEGER, BIGINT, FLOAT, DOUBLE,
+        DECIMAL, DECIMAL, VARCHAR, ARRAY, ARRAY, JAVA_OBJECT, DATE, TIMESTAMP, STRUCT, BINARY,
+        CHAR, VARCHAR)
+
+      var pos = 0
+
+      while (rowSet.next()) {
+        assert(rowSet.getString("TABLE_CAT") === null)
+        assert(rowSet.getString("TABLE_SCHEM") === schemaName)
+        assert(rowSet.getString("TABLE_NAME") === tableName)
+        assert(rowSet.getString("COLUMN_NAME") === schema(pos).name)
+        assert(rowSet.getInt("DATA_TYPE") === expectedJavaTypes(pos))
+        assert(rowSet.getString("TYPE_NAME") === schema(pos).dataType.sql)
+
+        val colSize = rowSet.getInt("COLUMN_SIZE")
+        schema(pos).dataType match {
+          case StringType | BinaryType | _: ArrayType | _: MapType | _: VarcharType =>
+            assert(colSize === 0)
+          case o => assert(colSize === o.defaultSize)
+        }
+
+        assert(rowSet.getInt("BUFFER_LENGTH") === 0) // not used
+        val decimalDigits = rowSet.getInt("DECIMAL_DIGITS")
+        schema(pos).dataType match {
+          case BooleanType | _: IntegerType => assert(decimalDigits === 0)
+          case d: DecimalType => assert(decimalDigits === d.scale)
+          case FloatType => assert(decimalDigits === 7)
+          case DoubleType => assert(decimalDigits === 15)
+          case TimestampType => assert(decimalDigits === 6)
+          case _ => assert(decimalDigits === 0) // nulls
+        }
+
+        val radix = rowSet.getInt("NUM_PREC_RADIX")
+        schema(pos).dataType match {
+          case _: NumericType => assert(radix === 10)
+          case _ => assert(radix === 0) // nulls
+        }
+
+        assert(rowSet.getInt("NULLABLE") === 1)
+        assert(rowSet.getString("REMARKS") === pos.toString)
+        assert(rowSet.getInt("ORDINAL_POSITION") === pos)
+        assert(rowSet.getString("IS_NULLABLE") === "YES")
+        assert(rowSet.getString("IS_AUTO_INCREMENT") === "NO")
+        pos += 1
+      }
+
+      assert(pos === 19, "all columns should have been verified")
+    }
+  }
+
+  test("get columns operation should handle interval column properly") {
+    val viewName = "view_interval"
+    val ddl = s"CREATE GLOBAL TEMP VIEW $viewName as select interval 1 day as i"
+
+    withJdbcStatement(viewName) { statement =>
+      statement.execute(ddl)
+      val data = statement.getConnection.getMetaData
+      val rowSet = data.getColumns("", "global_temp", viewName, null)
+      while (rowSet.next()) {
+        assert(rowSet.getString("TABLE_CAT") === null)
+        assert(rowSet.getString("TABLE_SCHEM") === "global_temp")
+        assert(rowSet.getString("TABLE_NAME") === viewName)
+        assert(rowSet.getString("COLUMN_NAME") === "i")
+        assert(rowSet.getInt("DATA_TYPE") === java.sql.Types.OTHER)
+        assert(rowSet.getString("TYPE_NAME").equalsIgnoreCase(CalendarIntervalType.sql))
+        assert(rowSet.getInt("COLUMN_SIZE") === CalendarIntervalType.defaultSize)
+        assert(rowSet.getInt("DECIMAL_DIGITS") === 0)
+        assert(rowSet.getInt("NUM_PREC_RADIX") === 0)
+        assert(rowSet.getInt("NULLABLE") === 0)
+        assert(rowSet.getString("REMARKS") === "")
+        assert(rowSet.getInt("ORDINAL_POSITION") === 0)
+        assert(rowSet.getString("IS_NULLABLE") === "YES")
+        assert(rowSet.getString("IS_AUTO_INCREMENT") === "NO")
+      }
+    }
+  }
+
+  test("SPARK-35085: Get columns operation should handle ANSI interval column properly") {
+    val viewName1 = "view_interval1"
+    val yearMonthDDL =
+      s"CREATE GLOBAL TEMP VIEW $viewName1 as select interval '1-1' year to month as i"
+
+    withJdbcStatement(viewName1) { statement =>
+      statement.execute(yearMonthDDL)
+      val data = statement.getConnection.getMetaData
+      val rowSet = data.getColumns("", "global_temp", viewName1, null)
+      while (rowSet.next()) {
+        assert(rowSet.getString("TABLE_CAT") === null)
+        assert(rowSet.getString("TABLE_SCHEM") === "global_temp")
+        assert(rowSet.getString("TABLE_NAME") === viewName1)
+        assert(rowSet.getString("COLUMN_NAME") === "i")
+        assert(rowSet.getInt("DATA_TYPE") === java.sql.Types.OTHER)
+        assert(rowSet.getString("TYPE_NAME").equalsIgnoreCase(YearMonthIntervalType.sql))
+        assert(rowSet.getInt("COLUMN_SIZE") === YearMonthIntervalType.defaultSize)
+        assert(rowSet.getInt("DECIMAL_DIGITS") === 0)
+        assert(rowSet.getInt("NUM_PREC_RADIX") === 0)
+        assert(rowSet.getInt("NULLABLE") === 0)
+        assert(rowSet.getString("REMARKS") === "")
+        assert(rowSet.getInt("ORDINAL_POSITION") === 0)
+        assert(rowSet.getString("IS_NULLABLE") === "YES")
+        assert(rowSet.getString("IS_AUTO_INCREMENT") === "NO")
+      }
+    }
+
+    val viewName2 = "view_interval2"
+    val dayTimeDDL =
+      s"CREATE GLOBAL TEMP VIEW $viewName2 as select interval '1 2:3:4.001' day to second as i"
+
+    withJdbcStatement(viewName2) { statement =>
+      statement.execute(dayTimeDDL)
+      val data = statement.getConnection.getMetaData
+      val rowSet = data.getColumns("", "global_temp", viewName2, null)
+      while (rowSet.next()) {
+        assert(rowSet.getString("TABLE_CAT") === null)
+        assert(rowSet.getString("TABLE_SCHEM") === "global_temp")
+        assert(rowSet.getString("TABLE_NAME") === viewName2)
+        assert(rowSet.getString("COLUMN_NAME") === "i")
+        assert(rowSet.getInt("DATA_TYPE") === java.sql.Types.OTHER)
+        assert(rowSet.getString("TYPE_NAME").equalsIgnoreCase(DayTimeIntervalType.sql))
+        assert(rowSet.getInt("COLUMN_SIZE") === DayTimeIntervalType.defaultSize)
+        assert(rowSet.getInt("DECIMAL_DIGITS") === 0)
+        assert(rowSet.getInt("NUM_PREC_RADIX") === 0)
+        assert(rowSet.getInt("NULLABLE") === 0)
+        assert(rowSet.getString("REMARKS") === "")
+        assert(rowSet.getInt("ORDINAL_POSITION") === 0)
+        assert(rowSet.getString("IS_NULLABLE") === "YES")
+        assert(rowSet.getString("IS_AUTO_INCREMENT") === "NO")
+      }
+    }
+  }
+
+  test("handling null in view for get columns operations") {
+    val viewName = "view_null"
+    val ddl = s"CREATE GLOBAL TEMP VIEW $viewName as select null as n"
+
+    withJdbcStatement(viewName) { statement =>
+      statement.execute(ddl)
+      val data = statement.getConnection.getMetaData
+      val rowSet = data.getColumns("", "global_temp", viewName, "n")
+      while (rowSet.next()) {
+        assert(rowSet.getString("TABLE_CAT") === null)
+        assert(rowSet.getString("TABLE_SCHEM") === "global_temp")
+        assert(rowSet.getString("TABLE_NAME") === viewName)
+        assert(rowSet.getString("COLUMN_NAME") === "n")
+        assert(rowSet.getInt("DATA_TYPE") === java.sql.Types.NULL)
+        assert(rowSet.getString("TYPE_NAME").equalsIgnoreCase(NullType.sql))
+        assert(rowSet.getInt("COLUMN_SIZE") === 1)
+        assert(rowSet.getInt("DECIMAL_DIGITS") === 0)
+        assert(rowSet.getInt("NUM_PREC_RADIX") === 0)
+        assert(rowSet.getInt("NULLABLE") === 1)
+        assert(rowSet.getString("REMARKS") === "")
+        assert(rowSet.getInt("ORDINAL_POSITION") === 0)
+        assert(rowSet.getString("IS_NULLABLE") === "YES")
+        assert(rowSet.getString("IS_AUTO_INCREMENT") === "NO")
+      }
+    }
+  }
+
+  test("Hive ThriftServer JDBC Database MetaData API Auditing - Method not supported") {
+    // These APIs belong to the upstream Apache Hive's hive-jdbc artifact where defines the hive
+    // behavior. Users can also use it to interact with Spark ThriftServer directly. Some behaviors
+    // are not fully consistent with Spark e.g. we support correlated subqueries but the hive-jdbc
+    // now fail directly at client side. There is nothing we can do but accept the current
+    // condition and highlight the difference and make it perspective in future changes both from
+    // upstream and inside Spark.
+    withJdbcStatement() { statement =>
+      val metaData = statement.getConnection.getMetaData
+      Seq(
+        () => metaData.allProceduresAreCallable,
+        () => metaData.getURL,
+        () => metaData.getUserName,
+        () => metaData.isReadOnly,
+        () => metaData.nullsAreSortedHigh,
+        () => metaData.nullsAreSortedLow,
+        () => metaData.nullsAreSortedAtStart,
+        () => metaData.nullsAreSortedAtEnd,
+        () => metaData.usesLocalFiles,
+        () => metaData.usesLocalFilePerTable,
+        () => metaData.supportsMixedCaseIdentifiers,
+        () => metaData.supportsMixedCaseQuotedIdentifiers,
+        () => metaData.storesUpperCaseIdentifiers,
+        () => metaData.storesUpperCaseQuotedIdentifiers,
+        () => metaData.storesLowerCaseIdentifiers,
+        () => metaData.storesLowerCaseQuotedIdentifiers,
+        () => metaData.storesMixedCaseIdentifiers,
+        () => metaData.storesMixedCaseQuotedIdentifiers,
+        () => metaData.getSQLKeywords,
+        () => metaData.nullPlusNonNullIsNull,
+        () => metaData.supportsConvert,
+        () => metaData.supportsTableCorrelationNames,
+        () => metaData.supportsDifferentTableCorrelationNames,
+        () => metaData.supportsExpressionsInOrderBy,
+        () => metaData.supportsOrderByUnrelated,
+        () => metaData.supportsGroupByUnrelated,
+        () => metaData.supportsGroupByBeyondSelect,
+        () => metaData.supportsLikeEscapeClause,
+        () => metaData.supportsMultipleTransactions,
+        () => metaData.supportsMinimumSQLGrammar,
+        () => metaData.supportsCoreSQLGrammar,
+        () => metaData.supportsExtendedSQLGrammar,
+        () => metaData.supportsANSI92EntryLevelSQL,
+        () => metaData.supportsANSI92IntermediateSQL,
+        () => metaData.supportsANSI92FullSQL,
+        () => metaData.supportsIntegrityEnhancementFacility,
+        () => metaData.isCatalogAtStart,
+        () => metaData.supportsSubqueriesInComparisons,
+        () => metaData.supportsSubqueriesInExists,
+        () => metaData.supportsSubqueriesInIns,
+        () => metaData.supportsSubqueriesInQuantifieds,
+        // Spark support this, see https://issues.apache.org/jira/browse/SPARK-18455
+        () => metaData.supportsCorrelatedSubqueries,
+        () => metaData.supportsOpenCursorsAcrossCommit,
+        () => metaData.supportsOpenCursorsAcrossRollback,
+        () => metaData.supportsOpenStatementsAcrossCommit,
+        () => metaData.supportsOpenStatementsAcrossRollback,
+        () => metaData.getMaxBinaryLiteralLength,
+        () => metaData.getMaxCharLiteralLength,
+        () => metaData.getMaxColumnsInGroupBy,
+        () => metaData.getMaxColumnsInIndex,
+        () => metaData.getMaxColumnsInOrderBy,
+        () => metaData.getMaxColumnsInSelect,
+        () => metaData.getMaxColumnsInTable,
+        () => metaData.getMaxConnections,
+        () => metaData.getMaxCursorNameLength,
+        () => metaData.getMaxIndexLength,
+        () => metaData.getMaxSchemaNameLength,
+        () => metaData.getMaxProcedureNameLength,
+        () => metaData.getMaxCatalogNameLength,
+        () => metaData.getMaxRowSize,
+        () => metaData.doesMaxRowSizeIncludeBlobs,
+        () => metaData.getMaxStatementLength,
+        () => metaData.getMaxStatements,
+        () => metaData.getMaxTableNameLength,
+        () => metaData.getMaxTablesInSelect,
+        () => metaData.getMaxUserNameLength,
+        () => metaData.supportsTransactionIsolationLevel(1),
+        () => metaData.supportsDataDefinitionAndDataManipulationTransactions,
+        () => metaData.supportsDataManipulationTransactionsOnly,
+        () => metaData.dataDefinitionCausesTransactionCommit,
+        () => metaData.dataDefinitionIgnoredInTransactions,
+        () => metaData.getColumnPrivileges("", "%", "%", "%"),
+        () => metaData.getTablePrivileges("", "%", "%"),
+        () => metaData.getBestRowIdentifier("", "%", "%", 0, true),
+        () => metaData.getVersionColumns("", "%", "%"),
+        () => metaData.getExportedKeys("", "default", ""),
+        () => metaData.supportsResultSetConcurrency(ResultSet.TYPE_FORWARD_ONLY, 2),
+        () => metaData.ownUpdatesAreVisible(ResultSet.TYPE_FORWARD_ONLY),
+        () => metaData.ownDeletesAreVisible(ResultSet.TYPE_FORWARD_ONLY),
+        () => metaData.ownInsertsAreVisible(ResultSet.TYPE_FORWARD_ONLY),
+        () => metaData.othersUpdatesAreVisible(ResultSet.TYPE_FORWARD_ONLY),
+        () => metaData.othersDeletesAreVisible(ResultSet.TYPE_FORWARD_ONLY),
+        () => metaData.othersInsertsAreVisible(ResultSet.TYPE_FORWARD_ONLY),
+        () => metaData.updatesAreDetected(ResultSet.TYPE_FORWARD_ONLY),
+        () => metaData.deletesAreDetected(ResultSet.TYPE_FORWARD_ONLY),
+        () => metaData.insertsAreDetected(ResultSet.TYPE_FORWARD_ONLY),
+        () => metaData.supportsNamedParameters,
+        () => metaData.supportsMultipleOpenResults,
+        () => metaData.supportsGetGeneratedKeys,
+        () => metaData.getSuperTypes("", "%", "%"),
+        () => metaData.getSuperTables("", "%", "%"),
+        () => metaData.getAttributes("", "%", "%", "%"),
+        () => metaData.getResultSetHoldability,
+        () => metaData.locatorsUpdateCopy,
+        () => metaData.supportsStatementPooling,
+        () => metaData.getRowIdLifetime,
+        () => metaData.supportsStoredFunctionsUsingCallSyntax,
+        () => metaData.autoCommitFailureClosesAllResultSets,
+        () => metaData.getClientInfoProperties,
+        () => metaData.getFunctionColumns("", "%", "%", "%"),
+        () => metaData.getPseudoColumns("", "%", "%", "%"),
+        () => metaData.generatedKeyAlwaysReturned).foreach { func =>
+        val e = intercept[SQLFeatureNotSupportedException](func())
+        assert(e.getMessage === "Method not supported")
+      }
+    }
+  }
+
+  test("Hive ThriftServer JDBC Database MetaData API Auditing - Method supported") {
+    // These APIs belong to the upstream Apache Hive's hive-jdbc artifact where defines the hive
+    // behavior. Users can also use it to interact with Spark ThriftServer directly. Some behaviors
+    // are not fully consistent with Spark e.g. we can work with multiple catalogs.
+    // There is nothing we can do but accept the current condition and highlight the difference
+    // and make it perspective in future changes both from upstream and inside Spark.
+    withJdbcStatement() { statement =>
+      val metaData = statement.getConnection.getMetaData
+      assert(metaData.allTablesAreSelectable)
+      assert(metaData.getDatabaseProductName === "Spark SQL")
+      assert(metaData.getDatabaseProductVersion === SPARK_VERSION)
+      assert(metaData.getDriverName === "Hive JDBC")
+      assert(metaData.getDriverVersion === HiveVersionInfo.getVersion)
+      assert(metaData.getDatabaseMajorVersion === VersionUtils.majorVersion(SPARK_VERSION))
+      assert(metaData.getDatabaseMinorVersion === VersionUtils.minorVersion(SPARK_VERSION))
+      assert(metaData.getIdentifierQuoteString === " ",
+        "This method returns a space \" \" if identifier quoting is not supported")
+      assert(metaData.getNumericFunctions === "")
+      assert(metaData.getStringFunctions === "")
+      assert(metaData.getSystemFunctions === "")
+      assert(metaData.getTimeDateFunctions === "")
+      assert(metaData.getSearchStringEscape === "\\")
+      assert(metaData.getExtraNameCharacters === "")
+      assert(metaData.supportsAlterTableWithAddColumn())
+      assert(!metaData.supportsAlterTableWithDropColumn())
+      assert(metaData.supportsColumnAliasing())
+      assert(metaData.supportsGroupBy)
+      assert(!metaData.supportsMultipleResultSets)
+      assert(!metaData.supportsNonNullableColumns)
+      assert(metaData.supportsOuterJoins)
+      assert(metaData.supportsFullOuterJoins)
+      assert(metaData.supportsLimitedOuterJoins)
+      assert(metaData.getSchemaTerm === "database")
+      assert(metaData.getProcedureTerm === "UDF")
+      assert(metaData.getCatalogTerm === "instance")
+      assert(metaData.getCatalogSeparator === ".")
+      assert(metaData.supportsSchemasInDataManipulation)
+      assert(!metaData.supportsSchemasInProcedureCalls)
+      assert(metaData.supportsSchemasInTableDefinitions)
+      assert(!metaData.supportsSchemasInIndexDefinitions)
+      assert(!metaData.supportsSchemasInPrivilegeDefinitions)
+      // This is actually supported, but hive jdbc package return false
+      assert(!metaData.supportsCatalogsInDataManipulation)
+      assert(!metaData.supportsCatalogsInProcedureCalls)
+      // This is actually supported, but hive jdbc package return false
+      assert(!metaData.supportsCatalogsInTableDefinitions)
+      assert(!metaData.supportsCatalogsInIndexDefinitions)
+      assert(!metaData.supportsCatalogsInPrivilegeDefinitions)
+      assert(!metaData.supportsPositionedDelete)
+      assert(!metaData.supportsPositionedUpdate)
+      assert(!metaData.supportsSelectForUpdate)
+      assert(!metaData.supportsStoredProcedures)
+      // This is actually supported, but hive jdbc package return false
+      assert(!metaData.supportsUnion)
+      assert(metaData.supportsUnionAll)
+      assert(metaData.getMaxColumnNameLength === 128)
+      assert(metaData.getDefaultTransactionIsolation === java.sql.Connection.TRANSACTION_NONE)
+      assert(!metaData.supportsTransactions)
+      assert(!metaData.getProcedureColumns("", "%", "%", "%").next())
+      assert(!metaData.getImportedKeys("", "default", "").next())
+
+      // TODO: SPARK-33219 Disable GetPrimaryKeys and GetCrossReference APIs explicitly
+      // for Spark ThriftServer
+      assert(!metaData.getPrimaryKeys("", "default", "").next())
+      assert(!metaData.getCrossReference("", "default", "src", "", "default", "src2").next())
+
+      assert(!metaData.getIndexInfo("", "default", "src", true, true).next())
+      assert(metaData.supportsResultSetType(ResultSet.TYPE_FORWARD_ONLY))
+      assert(metaData.supportsResultSetType(ResultSet.TYPE_SCROLL_INSENSITIVE))
+      assert(metaData.supportsResultSetType(ResultSet.TYPE_SCROLL_SENSITIVE))
+      assert(!metaData.supportsBatchUpdates)
+      assert(!metaData.getUDTs(",", "%", "%", null).next())
+      assert(!metaData.supportsSavepoints)
+      assert(!metaData.supportsResultSetHoldability(ResultSet.HOLD_CURSORS_OVER_COMMIT))
+      assert(metaData.getJDBCMajorVersion === 3)
+      assert(metaData.getJDBCMinorVersion === 0)
+      assert(metaData.getSQLStateType === DatabaseMetaData.sqlStateSQL)
+      assert(metaData.getMaxLogicalLobSize === 0)
+      assert(!metaData.supportsRefCursors)
     }
   }
 }
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnvSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnvSuite.scala
index f28faea2be868..f2bb337e4a826 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnvSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnvSuite.scala
@@ -42,7 +42,7 @@ class SparkSQLEnvSuite extends SparkFunSuite {
       QUERY_EXECUTION_LISTENERS.key -> classOf[DummyQueryExecutionListener].getCanonicalName,
       STREAMING_QUERY_LISTENERS.key -> classOf[DummyStreamingQueryListener].getCanonicalName,
       WAREHOUSE_PATH.key -> TestHiveContext.makeWarehouseDir().toURI.getPath,
-      // The issue occured from "maven" and list of custom jars, but providing list of custom
+      // The issue occurred from "maven" and list of custom jars, but providing list of custom
       // jars to initialize HiveClient isn't trivial, so just use "maven".
       HIVE_METASTORE_JARS.key -> "maven",
       HIVE_METASTORE_VERSION.key -> null,
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala
index fd45e7a48c0eb..8ca33d3e1b410 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala
@@ -20,34 +20,37 @@ package org.apache.spark.sql.hive.thriftserver
 import java.sql.{Date, Timestamp}
 import java.util.{List => JList, Properties}
 
+import org.apache.hadoop.hive.common.`type`.{HiveIntervalDayTime, HiveIntervalYearMonth}
 import org.apache.hive.jdbc.{HiveConnection, HiveQueryResultSet}
 import org.apache.hive.service.auth.PlainSaslHelper
 import org.apache.hive.service.cli.GetInfoType
+import org.apache.hive.service.rpc.thrift.{TExecuteStatementReq, TGetInfoReq, TGetTablesReq, TOpenSessionReq, TProtocolVersion}
+import org.apache.hive.service.rpc.thrift.TCLIService.Client
 import org.apache.thrift.protocol.TBinaryProtocol
 import org.apache.thrift.transport.TSocket
 
 import org.apache.spark.sql.catalyst.util.NumberConverter
 import org.apache.spark.unsafe.types.UTF8String
 
-class SparkThriftServerProtocolVersionsSuite extends HiveThriftJdbcTest {
+class SparkThriftServerProtocolVersionsSuite extends HiveThriftServer2TestBase {
 
   override def mode: ServerMode.Value = ServerMode.binary
 
   def testExecuteStatementWithProtocolVersion(
-      version: ThriftserverShimUtils.TProtocolVersion,
+      version: TProtocolVersion,
       sql: String)(f: HiveQueryResultSet => Unit): Unit = {
     val rawTransport = new TSocket("localhost", serverPort)
     val connection = new HiveConnection(s"jdbc:hive2://localhost:$serverPort", new Properties)
     val user = System.getProperty("user.name")
     val transport = PlainSaslHelper.getPlainTransport(user, "anonymous", rawTransport)
-    val client = new ThriftserverShimUtils.Client(new TBinaryProtocol(transport))
+    val client = new Client(new TBinaryProtocol(transport))
     transport.open()
     var rs: HiveQueryResultSet = null
     try {
-      val clientProtocol = new ThriftserverShimUtils.TOpenSessionReq(version)
+      val clientProtocol = new TOpenSessionReq(version)
       val openResp = client.OpenSession(clientProtocol)
       val sessHandle = openResp.getSessionHandle
-      val execReq = new ThriftserverShimUtils.TExecuteStatementReq(sessHandle, sql)
+      val execReq = new TExecuteStatementReq(sessHandle, sql)
       val execResp = client.ExecuteStatement(execReq)
       val stmtHandle = execResp.getOperationHandle
 
@@ -72,23 +75,21 @@ class SparkThriftServerProtocolVersionsSuite extends HiveThriftJdbcTest {
     }
   }
 
-  def testGetInfoWithProtocolVersion(version: ThriftserverShimUtils.TProtocolVersion): Unit = {
+  def testGetInfoWithProtocolVersion(version: TProtocolVersion): Unit = {
     val rawTransport = new TSocket("localhost", serverPort)
     val connection = new HiveConnection(s"jdbc:hive2://localhost:$serverPort", new Properties)
     val transport = PlainSaslHelper.getPlainTransport(user, "anonymous", rawTransport)
-    val client = new ThriftserverShimUtils.Client(new TBinaryProtocol(transport))
+    val client = new Client(new TBinaryProtocol(transport))
     transport.open()
     try {
-      val clientProtocol = new ThriftserverShimUtils.TOpenSessionReq(version)
+      val clientProtocol = new TOpenSessionReq(version)
       val openResp = client.OpenSession(clientProtocol)
       val sessHandle = openResp.getSessionHandle
 
-      val dbVersionReq =
-        new ThriftserverShimUtils.TGetInfoReq(sessHandle, GetInfoType.CLI_DBMS_VER.toTGetInfoType)
+      val dbVersionReq = new TGetInfoReq(sessHandle, GetInfoType.CLI_DBMS_VER.toTGetInfoType)
       val dbVersion = client.GetInfo(dbVersionReq).getInfoValue.getStringValue
 
-      val dbNameReq =
-        new ThriftserverShimUtils.TGetInfoReq(sessHandle, GetInfoType.CLI_DBMS_NAME.toTGetInfoType)
+      val dbNameReq = new TGetInfoReq(sessHandle, GetInfoType.CLI_DBMS_NAME.toTGetInfoType)
       val dbName = client.GetInfo(dbNameReq).getInfoValue.getStringValue
 
       assert(dbVersion === org.apache.spark.SPARK_VERSION)
@@ -101,21 +102,21 @@ class SparkThriftServerProtocolVersionsSuite extends HiveThriftJdbcTest {
   }
 
   def testGetTablesWithProtocolVersion(
-      version: ThriftserverShimUtils.TProtocolVersion,
+      version: TProtocolVersion,
       schema: String,
       tableNamePattern: String,
       tableTypes: JList[String])(f: HiveQueryResultSet => Unit): Unit = {
     val rawTransport = new TSocket("localhost", serverPort)
     val connection = new HiveConnection(s"jdbc:hive2://localhost:$serverPort", new Properties)
     val transport = PlainSaslHelper.getPlainTransport(user, "anonymous", rawTransport)
-    val client = new ThriftserverShimUtils.Client(new TBinaryProtocol(transport))
+    val client = new Client(new TBinaryProtocol(transport))
     transport.open()
     var rs: HiveQueryResultSet = null
     try {
-      val clientProtocol = new ThriftserverShimUtils.TOpenSessionReq(version)
+      val clientProtocol = new TOpenSessionReq(version)
       val openResp = client.OpenSession(clientProtocol)
       val sessHandle = openResp.getSessionHandle
-      val getTableReq = new ThriftserverShimUtils.TGetTablesReq(sessHandle)
+      val getTableReq = new TGetTablesReq(sessHandle)
       getTableReq.setSchemaName(schema)
       getTableReq.setTableName(tableNamePattern)
       getTableReq.setTableTypes(tableTypes)
@@ -143,11 +144,17 @@ class SparkThriftServerProtocolVersionsSuite extends HiveThriftJdbcTest {
     }
   }
 
-  ThriftserverShimUtils.testedProtocolVersions.foreach { version =>
+  TProtocolVersion.values().foreach { version =>
     test(s"$version get byte type") {
       testExecuteStatementWithProtocolVersion(version, "SELECT cast(1 as byte)") { rs =>
         assert(rs.next())
         assert(rs.getByte(1) === 1.toByte)
+        val metaData = rs.getMetaData
+        assert(metaData.getColumnName(1) === "CAST(1 AS TINYINT)")
+        assert(metaData.getColumnTypeName(1) === "tinyint")
+        assert(metaData.getColumnType(1) === java.sql.Types.TINYINT)
+        assert(metaData.getPrecision(1) === 3)
+        assert(metaData.getScale(1) === 0)
       }
     }
 
@@ -155,6 +162,12 @@ class SparkThriftServerProtocolVersionsSuite extends HiveThriftJdbcTest {
       testExecuteStatementWithProtocolVersion(version, "SELECT cast(1 as short)") { rs =>
         assert(rs.next())
         assert(rs.getShort(1) === 1.toShort)
+        val metaData = rs.getMetaData
+        assert(metaData.getColumnName(1) === "CAST(1 AS SMALLINT)")
+        assert(metaData.getColumnTypeName(1) === "smallint")
+        assert(metaData.getColumnType(1) === java.sql.Types.SMALLINT)
+        assert(metaData.getPrecision(1) === 5)
+        assert(metaData.getScale(1) === 0)
       }
     }
 
@@ -162,6 +175,12 @@ class SparkThriftServerProtocolVersionsSuite extends HiveThriftJdbcTest {
       testExecuteStatementWithProtocolVersion(version, "SELECT 1") { rs =>
         assert(rs.next())
         assert(rs.getInt(1) === 1)
+        val metaData = rs.getMetaData
+        assert(metaData.getColumnName(1) === "1")
+        assert(metaData.getColumnTypeName(1) === "int")
+        assert(metaData.getColumnType(1) === java.sql.Types.INTEGER)
+        assert(metaData.getPrecision(1) === 10)
+        assert(metaData.getScale(1) === 0)
       }
     }
 
@@ -169,6 +188,12 @@ class SparkThriftServerProtocolVersionsSuite extends HiveThriftJdbcTest {
       testExecuteStatementWithProtocolVersion(version, "SELECT cast(1 as bigint)") { rs =>
         assert(rs.next())
         assert(rs.getLong(1) === 1L)
+        val metaData = rs.getMetaData
+        assert(metaData.getColumnName(1) === "CAST(1 AS BIGINT)")
+        assert(metaData.getColumnTypeName(1) === "bigint")
+        assert(metaData.getColumnType(1) === java.sql.Types.BIGINT)
+        assert(metaData.getPrecision(1) === 19)
+        assert(metaData.getScale(1) === 0)
       }
     }
 
@@ -176,6 +201,12 @@ class SparkThriftServerProtocolVersionsSuite extends HiveThriftJdbcTest {
       testExecuteStatementWithProtocolVersion(version, "SELECT cast(1.2 as float)") { rs =>
         assert(rs.next())
         assert(rs.getFloat(1) === 1.2F)
+        val metaData = rs.getMetaData
+        assert(metaData.getColumnName(1) === "CAST(1.2 AS FLOAT)")
+        assert(metaData.getColumnTypeName(1) === "float")
+        assert(metaData.getColumnType(1) === java.sql.Types.FLOAT)
+        assert(metaData.getPrecision(1) === 7)
+        assert(metaData.getScale(1) === 7)
       }
     }
 
@@ -183,14 +214,30 @@ class SparkThriftServerProtocolVersionsSuite extends HiveThriftJdbcTest {
       testExecuteStatementWithProtocolVersion(version, "SELECT cast(1.2 as double)") { rs =>
         assert(rs.next())
         assert(rs.getDouble(1) === 1.2D)
+        val metaData = rs.getMetaData
+        assert(metaData.getColumnName(1) === "CAST(1.2 AS DOUBLE)")
+        assert(metaData.getColumnTypeName(1) === "double")
+        assert(metaData.getColumnType(1) === java.sql.Types.DOUBLE)
+        assert(metaData.getPrecision(1) === 15)
+        assert(metaData.getScale(1) === 15)
       }
     }
 
     test(s"$version get decimal type") {
       testExecuteStatementWithProtocolVersion(version,
-        "SELECT cast(1 as decimal(18, 2)) as c") { rs =>
+        "SELECT cast(1 as decimal(9, 1)) as col0, 1234.56BD as col1, 0.123 as col2") { rs =>
         assert(rs.next())
-        assert(rs.getBigDecimal(1) === new java.math.BigDecimal("1.00"))
+        assert(rs.getBigDecimal(1) === new java.math.BigDecimal("1.0"))
+        assert(rs.getBigDecimal("col1") === new java.math.BigDecimal("1234.56"))
+        assert(rs.getBigDecimal("col2") === new java.math.BigDecimal("0.123"))
+        val metaData = rs.getMetaData
+        (1 to 3) foreach { i =>
+          assert(metaData.getColumnName(i) === s"col${i - 1}")
+          assert(metaData.getColumnTypeName(i) === "decimal")
+          assert(metaData.getColumnType(i) === java.sql.Types.DECIMAL)
+          assert(metaData.getPrecision(i) == 12 - i * 3)
+          assert(metaData.getScale(i) == i)
+        }
       }
       testExecuteStatementWithProtocolVersion(version,
         "SELECT cast(null as decimal) ") { rs =>
@@ -203,6 +250,12 @@ class SparkThriftServerProtocolVersionsSuite extends HiveThriftJdbcTest {
       testExecuteStatementWithProtocolVersion(version, "SELECT 'str'") { rs =>
         assert(rs.next())
         assert(rs.getString(1) === "str")
+        val metaData = rs.getMetaData
+        assert(metaData.getColumnName(1) ==="str")
+        assert(metaData.getColumnTypeName(1) === "string")
+        assert(metaData.getColumnType(1) === java.sql.Types.VARCHAR)
+        assert(metaData.getPrecision(1) === Int.MaxValue)
+        assert(metaData.getScale(1) === 0)
       }
     }
 
@@ -211,6 +264,12 @@ class SparkThriftServerProtocolVersionsSuite extends HiveThriftJdbcTest {
         "SELECT cast('char-str' as char(10))") { rs =>
         assert(rs.next())
         assert(rs.getString(1) === "char-str")
+        val metaData = rs.getMetaData
+        assert(metaData.getColumnName(1) ==="CAST(char-str AS STRING)")
+        assert(metaData.getColumnTypeName(1) === "string")
+        assert(metaData.getColumnType(1) === java.sql.Types.VARCHAR)
+        assert(metaData.getPrecision(1) === Int.MaxValue)
+        assert(metaData.getScale(1) === 0)
       }
     }
 
@@ -219,6 +278,12 @@ class SparkThriftServerProtocolVersionsSuite extends HiveThriftJdbcTest {
         "SELECT cast('varchar-str' as varchar(10))") { rs =>
         assert(rs.next())
         assert(rs.getString(1) === "varchar-str")
+        val metaData = rs.getMetaData
+        assert(metaData.getColumnName(1) ==="CAST(varchar-str AS STRING)")
+        assert(metaData.getColumnTypeName(1) === "string")
+        assert(metaData.getColumnType(1) === java.sql.Types.VARCHAR)
+        assert(metaData.getPrecision(1) === Int.MaxValue)
+        assert(metaData.getScale(1) === 0)
       }
     }
 
@@ -226,6 +291,12 @@ class SparkThriftServerProtocolVersionsSuite extends HiveThriftJdbcTest {
       testExecuteStatementWithProtocolVersion(version, "SELECT cast('ABC' as binary)") { rs =>
         assert(rs.next())
         assert(rs.getString(1) === "ABC")
+        val metaData = rs.getMetaData
+        assert(metaData.getColumnName(1) === "CAST(ABC AS BINARY)")
+        assert(metaData.getColumnTypeName(1) === "binary")
+        assert(metaData.getColumnType(1) === java.sql.Types.BINARY)
+        assert(metaData.getPrecision(1) === Int.MaxValue)
+        assert(metaData.getScale(1) === 0)
       }
       testExecuteStatementWithProtocolVersion(version, "SELECT cast(49960 as binary)") { rs =>
         assert(rs.next())
@@ -241,6 +312,12 @@ class SparkThriftServerProtocolVersionsSuite extends HiveThriftJdbcTest {
       testExecuteStatementWithProtocolVersion(version, "SELECT true") { rs =>
         assert(rs.next())
         assert(rs.getBoolean(1) === true)
+        val metaData = rs.getMetaData
+        assert(metaData.getColumnName(1) === "true")
+        assert(metaData.getColumnTypeName(1) === "boolean")
+        assert(metaData.getColumnType(1) === java.sql.Types.BOOLEAN)
+        assert(metaData.getPrecision(1) === 1)
+        assert(metaData.getScale(1) === 0)
       }
     }
 
@@ -248,6 +325,12 @@ class SparkThriftServerProtocolVersionsSuite extends HiveThriftJdbcTest {
       testExecuteStatementWithProtocolVersion(version, "SELECT cast('2019-07-22' as date)") { rs =>
         assert(rs.next())
         assert(rs.getDate(1) === Date.valueOf("2019-07-22"))
+        val metaData = rs.getMetaData
+        assert(metaData.getColumnName(1) === "CAST(2019-07-22 AS DATE)")
+        assert(metaData.getColumnTypeName(1) === "date")
+        assert(metaData.getColumnType(1) === java.sql.Types.DATE)
+        assert(metaData.getPrecision(1) === 10)
+        assert(metaData.getScale(1) === 0)
       }
     }
 
@@ -256,6 +339,12 @@ class SparkThriftServerProtocolVersionsSuite extends HiveThriftJdbcTest {
         "SELECT cast('2019-07-22 18:14:00' as timestamp)") { rs =>
         assert(rs.next())
         assert(rs.getTimestamp(1) === Timestamp.valueOf("2019-07-22 18:14:00"))
+        val metaData = rs.getMetaData
+        assert(metaData.getColumnName(1) === "CAST(2019-07-22 18:14:00 AS TIMESTAMP)")
+        assert(metaData.getColumnTypeName(1) === "timestamp")
+        assert(metaData.getColumnType(1) === java.sql.Types.TIMESTAMP)
+        assert(metaData.getPrecision(1) === 29)
+        assert(metaData.getScale(1) === 9)
       }
     }
 
@@ -263,6 +352,12 @@ class SparkThriftServerProtocolVersionsSuite extends HiveThriftJdbcTest {
       testExecuteStatementWithProtocolVersion(version, "SELECT null") { rs =>
         assert(rs.next())
         assert(rs.getString(1) === null)
+        val metaData = rs.getMetaData
+        assert(metaData.getColumnName(1) === "NULL")
+        assert(metaData.getColumnTypeName(1) === "void")
+        assert(metaData.getColumnType(1) === java.sql.Types.NULL)
+        assert(metaData.getPrecision(1) === 0)
+        assert(metaData.getScale(1) === 0)
       }
     }
 
@@ -270,28 +365,67 @@ class SparkThriftServerProtocolVersionsSuite extends HiveThriftJdbcTest {
       testExecuteStatementWithProtocolVersion(version, "SELECT interval '1' year '2' day") { rs =>
         assert(rs.next())
         assert(rs.getString(1) === "1 years 2 days")
+        val metaData = rs.getMetaData
+        assert(metaData.getColumnName(1) === "INTERVAL '1 years 2 days'")
+        assert(metaData.getColumnTypeName(1) === "string")
+        assert(metaData.getColumnType(1) === java.sql.Types.VARCHAR)
+        assert(metaData.getPrecision(1) === Int.MaxValue)
+        assert(metaData.getScale(1) === 0)
       }
     }
 
     test(s"$version get array type") {
-      testExecuteStatementWithProtocolVersion(version, "SELECT array(1, 2)") { rs =>
+      testExecuteStatementWithProtocolVersion(
+        version, "SELECT array() AS col1, array(1, 2) AS col2") { rs =>
         assert(rs.next())
-        assert(rs.getString(1) === "[1,2]")
+        assert(rs.getString(2) === "[1,2]")
+        assert(rs.getObject("col1") === "[]")
+        assert(rs.getObject("col2") === "[1,2]")
+        val metaData = rs.getMetaData
+        (1 to 2) foreach { i =>
+          assert(metaData.getColumnName(i) === s"col$i")
+          assert(metaData.getColumnTypeName(i) === "array")
+          assert(metaData.getColumnType(i) === java.sql.Types.ARRAY)
+          assert(metaData.getPrecision(i) === Int.MaxValue)
+          assert(metaData.getScale(i) == 0)
+        }
       }
     }
 
     test(s"$version get map type") {
-      testExecuteStatementWithProtocolVersion(version, "SELECT map(1, 2)") { rs =>
+      testExecuteStatementWithProtocolVersion(version,
+        "SELECT map(), map(1, 2, 3, 4)") { rs =>
         assert(rs.next())
-        assert(rs.getString(1) === "{1:2}")
+        assert(rs.getObject(1) === "{}")
+        assert(rs.getObject(2) === "{1:2,3:4}")
+        assert(rs.getString(2) === "{1:2,3:4}")
+        val metaData = rs.getMetaData
+        (1 to 2) foreach { i =>
+          assert(metaData.getColumnName(i).startsWith("map("))
+          assert(metaData.getColumnTypeName(1) === "map")
+          assert(metaData.getColumnType(i) === java.sql.Types.JAVA_OBJECT)
+          assert(metaData.getPrecision(i) === Int.MaxValue)
+          assert(metaData.getScale(i) == 0)
+        }
       }
     }
 
     test(s"$version get struct type") {
       testExecuteStatementWithProtocolVersion(version,
-        "SELECT struct('alpha' AS A, 'beta' AS B)") { rs =>
+        "SELECT struct('alpha' AS A, 'beta' AS B) as col0," +
+          " struct('1', '2') AS col1, named_struct('a', 2, 'b', 4) AS col2") { rs =>
         assert(rs.next())
         assert(rs.getString(1) === """{"A":"alpha","B":"beta"}""")
+        assert(rs.getObject("col1") === """{"col1":"1","col2":"2"}""")
+        assert(rs.getObject("col2") === """{"a":2,"b":4}""")
+        val metaData = rs.getMetaData
+        (1 to 3) foreach { i =>
+          assert(metaData.getColumnName(i) === s"col${i - 1}")
+          assert(metaData.getColumnTypeName(1) === "struct")
+          assert(metaData.getColumnType(i) === java.sql.Types.STRUCT)
+          assert(metaData.getPrecision(i) === Int.MaxValue)
+          assert(metaData.getScale(i) == 0)
+        }
       }
     }
 
@@ -325,5 +459,29 @@ class SparkThriftServerProtocolVersionsSuite extends HiveThriftJdbcTest {
         }
       }
     }
+
+    test(s"SPARK-35017: $version get day-time interval type") {
+      testExecuteStatementWithProtocolVersion(
+        version, "SELECT INTERVAL '1 10:11:12' DAY TO SECOND AS dt") { rs =>
+        assert(rs.next())
+        assert(rs.getObject(1) === new HiveIntervalDayTime(1, 10, 11, 12, 0))
+        val metaData = rs.getMetaData
+        assert(metaData.getColumnName(1) === "dt")
+        assert(metaData.getColumnTypeName(1) === "interval_day_time")
+        assert(metaData.getColumnType(1) === java.sql.Types.OTHER)
+      }
+    }
+
+    test(s"SPARK-35018: $version get year-month interval type") {
+      testExecuteStatementWithProtocolVersion(
+        version, "SELECT INTERVAL '1-1' YEAR TO MONTH AS ym") { rs =>
+        assert(rs.next())
+        assert(rs.getObject(1) === new HiveIntervalYearMonth(1, 1))
+        val metaData = rs.getMetaData
+        assert(metaData.getColumnName(1) === "ym")
+        assert(metaData.getColumnTypeName(1) === "interval_year_month")
+        assert(metaData.getColumnType(1) === java.sql.Types.OTHER)
+      }
+    }
   }
 }
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
index 15cc3109da3f7..1d03c90cc812b 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
@@ -23,28 +23,41 @@ import java.util.{Locale, MissingFormatArgumentException}
 
 import scala.util.control.NonFatal
 
-import org.apache.commons.io.FileUtils
 import org.apache.commons.lang3.exception.ExceptionUtils
 
 import org.apache.spark.SparkException
 import org.apache.spark.sql.SQLQueryTestSuite
 import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
 import org.apache.spark.sql.catalyst.util.fileToString
-import org.apache.spark.sql.execution.HiveResult
+import org.apache.spark.sql.execution.HiveResult.{getTimeFormatters, toHiveString, TimeFormatters}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
+// scalastyle:off line.size.limit
 /**
  * Re-run all the tests in SQLQueryTestSuite via Thrift Server.
  *
+ * Each case is loaded from a file in "spark/sql/core/src/test/resources/sql-tests/inputs".
+ * Each case has a golden result file in "spark/sql/core/src/test/resources/sql-tests/results".
+ *
  * To run the entire test suite:
  * {{{
- *   build/sbt "hive-thriftserver/test-only *ThriftServerQueryTestSuite" -Phive-thriftserver
+ *   build/sbt -Phive-thriftserver "hive-thriftserver/testOnly *ThriftServerQueryTestSuite"
+ * }}}
+ *
+ * To run a single test file upon change:
+ * {{{
+ *   build/sbt -Phive-thriftserver "hive-thriftserver/testOnly *ThriftServerQueryTestSuite -- -z inline-table.sql"
  * }}}
  *
  * This test suite won't generate golden files. To re-generate golden files for entire suite, run:
  * {{{
- *   SPARK_GENERATE_GOLDEN_FILES=1 build/sbt "sql/test-only *SQLQueryTestSuite"
+ *   SPARK_GENERATE_GOLDEN_FILES=1 build/sbt "sql/testOnly *SQLQueryTestSuite"
+ * }}}
+ *
+ * To re-generate golden file for a single test, run:
+ * {{{
+ *   SPARK_GENERATE_GOLDEN_FILES=1 build/sbt "sql/testOnly *SQLQueryTestSuite -- -z describe.sql"
  * }}}
  *
  * TODO:
@@ -52,19 +65,18 @@ import org.apache.spark.sql.types._
  *   2. Support DESC command.
  *   3. Support SHOW command.
  */
+// scalastyle:on line.size.limit
 class ThriftServerQueryTestSuite extends SQLQueryTestSuite with SharedThriftServer {
 
+
+  override def mode: ServerMode.Value = ServerMode.binary
+
   override protected def testFile(fileName: String): String = {
-    val url = Thread.currentThread().getContextClassLoader.getResource(fileName)
-    // Copy to avoid URISyntaxException during accessing the resources in `sql/core`
-    val file = File.createTempFile("thriftserver-test", ".data")
-    file.deleteOnExit()
-    FileUtils.copyURLToFile(url, file)
-    file.getAbsolutePath
+    copyAndGetResourceFile(fileName, ".data").getAbsolutePath
   }
 
   /** List of test cases to ignore, in lower cases. */
-  override def blackList: Set[String] = super.blackList ++ Set(
+  override def ignoreList: Set[String] = super.ignoreList ++ Set(
     // Missing UDF
     "postgreSQL/boolean.sql",
     "postgreSQL/case.sql",
@@ -95,7 +107,10 @@ class ThriftServerQueryTestSuite extends SQLQueryTestSuite with SharedThriftServ
       }
 
       testCase match {
-        case _: PgSQLTest | _: AnsiTest =>
+        case _: PgSQLTest =>
+          statement.execute(s"SET ${SQLConf.ANSI_ENABLED.key} = true")
+          statement.execute(s"SET ${SQLConf.LEGACY_INTERVAL_ENABLED.key} = true")
+        case _: AnsiTest =>
           statement.execute(s"SET ${SQLConf.ANSI_ENABLED.key} = true")
         case _ =>
           statement.execute(s"SET ${SQLConf.ANSI_ENABLED.key} = false")
@@ -205,7 +220,7 @@ class ThriftServerQueryTestSuite extends SQLQueryTestSuite with SharedThriftServ
   }
 
   override def createScalaTestCase(testCase: TestCase): Unit = {
-    if (blackList.exists(t =>
+    if (ignoreList.exists(t =>
       testCase.name.toLowerCase(Locale.ROOT).contains(t.toLowerCase(Locale.ROOT)))) {
       // Create a test case to ignore this case.
       ignore(testCase.name) { /* Do nothing */ }
@@ -257,8 +272,9 @@ class ThriftServerQueryTestSuite extends SQLQueryTestSuite with SharedThriftServ
   private def getNormalizedResult(statement: Statement, sql: String): (String, Seq[String]) = {
     val rs = statement.executeQuery(sql)
     val cols = rs.getMetaData.getColumnCount
+    val timeFormatters = getTimeFormatters
     val buildStr = () => (for (i <- 1 to cols) yield {
-      getHiveResult(rs.getObject(i))
+      getHiveResult(rs.getObject(i), timeFormatters)
     }).mkString("\t")
 
     val answer = Iterator.continually(rs.next()).takeWhile(identity).map(_ => buildStr()).toSeq
@@ -280,18 +296,18 @@ class ThriftServerQueryTestSuite extends SQLQueryTestSuite with SharedThriftServ
       upperCase.startsWith("(")
   }
 
-  private def getHiveResult(obj: Object): String = {
+  private def getHiveResult(obj: Object, timeFormatters: TimeFormatters): String = {
     obj match {
       case null =>
-        HiveResult.toHiveString((null, StringType))
+        toHiveString((null, StringType), false, timeFormatters)
       case d: java.sql.Date =>
-        HiveResult.toHiveString((d, DateType))
+        toHiveString((d, DateType), false, timeFormatters)
       case t: Timestamp =>
-        HiveResult.toHiveString((t, TimestampType))
+        toHiveString((t, TimestampType), false, timeFormatters)
       case d: java.math.BigDecimal =>
-        HiveResult.toHiveString((d, DecimalType.fromDecimal(Decimal(d))))
+        toHiveString((d, DecimalType.fromDecimal(Decimal(d))), false, timeFormatters)
       case bin: Array[Byte] =>
-        HiveResult.toHiveString((bin, BinaryType))
+        toHiveString((bin, BinaryType), false, timeFormatters)
       case other =>
         other.toString
     }
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala
index 3e1fce78ae71c..3598f966b6259 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala
@@ -17,7 +17,20 @@
 
 package org.apache.spark.sql.hive.thriftserver
 
-class ThriftServerWithSparkContextSuite extends SharedThriftServer {
+import java.sql.SQLException
+import java.util.concurrent.atomic.AtomicBoolean
+
+import org.apache.hive.service.cli.HiveSQLException
+
+import org.apache.spark.TaskKilled
+import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd}
+import org.apache.spark.sql.internal.SQLConf
+
+trait ThriftServerWithSparkContextSuite extends SharedThriftServer {
+
+  test("the scratch dir will be deleted during server start but recreated with new operation") {
+    assert(tempScratchDir.exists())
+  }
 
   test("SPARK-29911: Uncache cached tables when session closed") {
     val cacheManager = spark.sharedState.cacheManager
@@ -41,4 +54,76 @@ class ThriftServerWithSparkContextSuite extends SharedThriftServer {
       assert(cacheManager.isEmpty)
     }
   }
+
+  test("Full stack traces as error message for jdbc or thrift client") {
+    val sql = "select date_sub(date'2011-11-11', '1.2')"
+    withCLIServiceClient { client =>
+      val sessionHandle = client.openSession(user, "")
+
+      val confOverlay = new java.util.HashMap[java.lang.String, java.lang.String]
+      val e = intercept[HiveSQLException] {
+        client.executeStatement(
+          sessionHandle,
+          sql,
+          confOverlay)
+      }
+
+      assert(e.getMessage
+        .contains("The second argument of 'date_sub' function needs to be an integer."))
+      assert(!e.getMessage.contains("" +
+        "java.lang.NumberFormatException: invalid input syntax for type numeric: 1.2"))
+    }
+
+    withJdbcStatement { statement =>
+      val e = intercept[SQLException] {
+        statement.executeQuery(sql)
+      }
+      assert(e.getMessage
+        .contains("The second argument of 'date_sub' function needs to be an integer."))
+      assert(e.getMessage.contains("" +
+        "java.lang.NumberFormatException: invalid input syntax for type numeric: 1.2"))
+    }
+  }
+
+  test("SPARK-33526: Add config to control if cancel invoke interrupt task on thriftserver") {
+    withJdbcStatement { statement =>
+      val forceCancel = new AtomicBoolean(false)
+      val listener = new SparkListener {
+        override def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit = {
+          assert(taskEnd.reason.isInstanceOf[TaskKilled])
+          if (forceCancel.get()) {
+            assert(System.currentTimeMillis() - taskEnd.taskInfo.launchTime < 1000)
+          } else {
+            // avoid accuracy, we check 2s instead of 3s.
+            assert(System.currentTimeMillis() - taskEnd.taskInfo.launchTime >= 2000)
+          }
+        }
+      }
+
+      spark.sparkContext.addSparkListener(listener)
+      try {
+        Seq(true, false).foreach { force =>
+          statement.setQueryTimeout(0)
+          statement.execute(s"SET ${SQLConf.THRIFTSERVER_FORCE_CANCEL.key}=$force")
+          statement.setQueryTimeout(1)
+          forceCancel.set(force)
+          val e = intercept[SQLException] {
+            statement.execute("select java_method('java.lang.Thread', 'sleep', 3000L)")
+          }.getMessage
+          assert(e.contains("Query timed out"))
+        }
+      } finally {
+        spark.sparkContext.removeSparkListener(listener)
+      }
+    }
+  }
+}
+
+
+class ThriftServerWithSparkContextInBinarySuite extends ThriftServerWithSparkContextSuite {
+  override def mode: ServerMode.Value = ServerMode.binary
+}
+
+class ThriftServerWithSparkContextInHttpSuite extends ThriftServerWithSparkContextSuite {
+  override def mode: ServerMode.Value = ServerMode.http
 }
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala
index 7f731f3d05e51..2d0edb8eb8d48 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala
@@ -22,15 +22,17 @@ import scala.util.Random
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars
 import org.openqa.selenium.WebDriver
 import org.openqa.selenium.htmlunit.HtmlUnitDriver
-import org.scalatest.{BeforeAndAfterAll, Matchers}
+import org.scalatest.BeforeAndAfterAll
 import org.scalatest.concurrent.Eventually._
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 import org.scalatest.time.SpanSugar._
 import org.scalatestplus.selenium.WebBrowser
 
 import org.apache.spark.ui.SparkUICssErrorHandler
 
 class UISeleniumSuite
-  extends HiveThriftJdbcTest
+  extends HiveThriftServer2TestBase
   with WebBrowser with Matchers with BeforeAndAfterAll {
 
   implicit var webDriver: WebDriver = _
@@ -55,7 +57,7 @@ class UISeleniumSuite
     }
   }
 
-  override protected def serverStartCommand(port: Int) = {
+  override protected def serverStartCommand(): Seq[String] = {
     val portConf = if (mode == ServerMode.binary) {
       ConfVars.HIVE_SERVER2_THRIFT_PORT
     } else {
@@ -69,7 +71,7 @@ class UISeleniumSuite
         |  --hiveconf ${ConfVars.METASTOREWAREHOUSE}=$warehousePath
         |  --hiveconf ${ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST}=localhost
         |  --hiveconf ${ConfVars.HIVE_SERVER2_TRANSPORT_MODE}=$mode
-        |  --hiveconf $portConf=$port
+        |  --hiveconf $portConf=0
         |  --driver-class-path ${sys.props("java.class.path")}
         |  --conf spark.ui.enabled=true
         |  --conf spark.ui.port=$uiPort
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2ListenerSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2ListenerSuite.scala
index 075032fa5d099..3f0538dd1c943 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2ListenerSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2ListenerSuite.scala
@@ -140,6 +140,24 @@ class HiveThriftServer2ListenerSuite extends SparkFunSuite with BeforeAndAfter {
     assert(listener.noLiveData())
   }
 
+  test("SPARK-31387 - listener update methods should not throw exception with unknown input") {
+    val (statusStore: HiveThriftServer2AppStatusStore, listener: HiveThriftServer2Listener) =
+      createAppStatusStore(true)
+
+    val unknownSession = "unknown_session"
+    val unknownOperation = "unknown_operation"
+    listener.onOtherEvent(SparkListenerThriftServerSessionClosed(unknownSession, 0))
+    listener.onOtherEvent(SparkListenerThriftServerOperationStart("id", unknownSession,
+      "stmt", "groupId", 0))
+    listener.onOtherEvent(SparkListenerThriftServerOperationParsed(unknownOperation, "query"))
+    listener.onOtherEvent(SparkListenerThriftServerOperationCanceled(unknownOperation, 0))
+    listener.onOtherEvent(SparkListenerThriftServerOperationTimeout(unknownOperation, 0))
+    listener.onOtherEvent(SparkListenerThriftServerOperationError(unknownOperation,
+      "msg", "trace", 0))
+    listener.onOtherEvent(SparkListenerThriftServerOperationFinish(unknownOperation, 0))
+    listener.onOtherEvent(SparkListenerThriftServerOperationClosed(unknownOperation, 0))
+  }
+
   private def createProperties: Properties = {
     val properties = new Properties()
     properties.setProperty(SparkContext.SPARK_JOB_GROUP_ID, "groupId")
diff --git a/sql/hive-thriftserver/v1.2/if/TCLIService.thrift b/sql/hive-thriftserver/v1.2/if/TCLIService.thrift
deleted file mode 100644
index 225e319737811..0000000000000
--- a/sql/hive-thriftserver/v1.2/if/TCLIService.thrift
+++ /dev/null
@@ -1,1173 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Coding Conventions for this file:
-//
-// Structs/Enums/Unions
-// * Struct, Enum, and Union names begin with a "T",
-//   and use a capital letter for each new word, with no underscores.
-// * All fields should be declared as either optional or required.
-//
-// Functions
-// * Function names start with a capital letter and have a capital letter for
-//   each new word, with no underscores.
-// * Each function should take exactly one parameter, named TFunctionNameReq,
-//   and should return either void or TFunctionNameResp. This convention allows
-//   incremental updates.
-//
-// Services
-// * Service names begin with the letter "T", use a capital letter for each
-//   new word (with no underscores), and end with the word "Service".
-
-namespace java org.apache.hive.service.cli.thrift
-namespace cpp apache.hive.service.cli.thrift
-
-// List of protocol versions. A new token should be
-// added to the end of this list every time a change is made.
-enum TProtocolVersion {
-  HIVE_CLI_SERVICE_PROTOCOL_V1,
-  
-  // V2 adds support for asynchronous execution
-  HIVE_CLI_SERVICE_PROTOCOL_V2
-
-  // V3 add varchar type, primitive type qualifiers
-  HIVE_CLI_SERVICE_PROTOCOL_V3
-
-  // V4 add decimal precision/scale, char type
-  HIVE_CLI_SERVICE_PROTOCOL_V4
-
-  // V5 adds error details when GetOperationStatus returns in error state
-  HIVE_CLI_SERVICE_PROTOCOL_V5
-
-  // V6 uses binary type for binary payload (was string) and uses columnar result set
-  HIVE_CLI_SERVICE_PROTOCOL_V6
-
-  // V7 adds support for delegation token based connection
-  HIVE_CLI_SERVICE_PROTOCOL_V7
-
-  // V8 adds support for interval types
-  HIVE_CLI_SERVICE_PROTOCOL_V8
-}
-
-enum TTypeId {
-  BOOLEAN_TYPE,
-  TINYINT_TYPE,
-  SMALLINT_TYPE,
-  INT_TYPE,
-  BIGINT_TYPE,
-  FLOAT_TYPE,
-  DOUBLE_TYPE,
-  STRING_TYPE,
-  TIMESTAMP_TYPE,
-  BINARY_TYPE,
-  ARRAY_TYPE,
-  MAP_TYPE,
-  STRUCT_TYPE,
-  UNION_TYPE,
-  USER_DEFINED_TYPE,
-  DECIMAL_TYPE,
-  NULL_TYPE,
-  DATE_TYPE,
-  VARCHAR_TYPE,
-  CHAR_TYPE,
-  INTERVAL_YEAR_MONTH_TYPE,
-  INTERVAL_DAY_TIME_TYPE
-}
-  
-const set<TTypeId> PRIMITIVE_TYPES = [
-  TTypeId.BOOLEAN_TYPE,
-  TTypeId.TINYINT_TYPE,
-  TTypeId.SMALLINT_TYPE,
-  TTypeId.INT_TYPE,
-  TTypeId.BIGINT_TYPE,
-  TTypeId.FLOAT_TYPE,
-  TTypeId.DOUBLE_TYPE,
-  TTypeId.STRING_TYPE,
-  TTypeId.TIMESTAMP_TYPE,
-  TTypeId.BINARY_TYPE,
-  TTypeId.DECIMAL_TYPE,
-  TTypeId.NULL_TYPE,
-  TTypeId.DATE_TYPE,
-  TTypeId.VARCHAR_TYPE,
-  TTypeId.CHAR_TYPE,
-  TTypeId.INTERVAL_YEAR_MONTH_TYPE,
-  TTypeId.INTERVAL_DAY_TIME_TYPE
-]
-
-const set<TTypeId> COMPLEX_TYPES = [
-  TTypeId.ARRAY_TYPE
-  TTypeId.MAP_TYPE
-  TTypeId.STRUCT_TYPE
-  TTypeId.UNION_TYPE
-  TTypeId.USER_DEFINED_TYPE
-]
-
-const set<TTypeId> COLLECTION_TYPES = [
-  TTypeId.ARRAY_TYPE
-  TTypeId.MAP_TYPE
-]
-
-const map<TTypeId,string> TYPE_NAMES = {
-  TTypeId.BOOLEAN_TYPE: "BOOLEAN",
-  TTypeId.TINYINT_TYPE: "TINYINT",
-  TTypeId.SMALLINT_TYPE: "SMALLINT",
-  TTypeId.INT_TYPE: "INT",
-  TTypeId.BIGINT_TYPE: "BIGINT",
-  TTypeId.FLOAT_TYPE: "FLOAT",
-  TTypeId.DOUBLE_TYPE: "DOUBLE",
-  TTypeId.STRING_TYPE: "STRING",
-  TTypeId.TIMESTAMP_TYPE: "TIMESTAMP",
-  TTypeId.BINARY_TYPE: "BINARY",
-  TTypeId.ARRAY_TYPE: "ARRAY",
-  TTypeId.MAP_TYPE: "MAP",
-  TTypeId.STRUCT_TYPE: "STRUCT",
-  TTypeId.UNION_TYPE: "UNIONTYPE",
-  TTypeId.DECIMAL_TYPE: "DECIMAL",
-  TTypeId.NULL_TYPE: "NULL"
-  TTypeId.DATE_TYPE: "DATE"
-  TTypeId.VARCHAR_TYPE: "VARCHAR"
-  TTypeId.CHAR_TYPE: "CHAR"
-  TTypeId.INTERVAL_YEAR_MONTH_TYPE: "INTERVAL_YEAR_MONTH"
-  TTypeId.INTERVAL_DAY_TIME_TYPE: "INTERVAL_DAY_TIME"
-}
-
-// Thrift does not support recursively defined types or forward declarations,
-// which makes it difficult to represent Hive's nested types.
-// To get around these limitations TTypeDesc employs a type list that maps
-// integer "pointers" to TTypeEntry objects. The following examples show
-// how different types are represented using this scheme:
-//
-// "INT":
-// TTypeDesc {
-//   types = [
-//     TTypeEntry.primitive_entry {
-//       type = INT_TYPE
-//     }
-//   ]
-// }
-//
-// "ARRAY<INT>":
-// TTypeDesc {
-//   types = [
-//     TTypeEntry.array_entry {
-//       object_type_ptr = 1
-//     },
-//     TTypeEntry.primitive_entry {
-//       type = INT_TYPE
-//     }
-//   ]
-// }
-//
-// "MAP<INT,STRING>":
-// TTypeDesc {
-//   types = [
-//     TTypeEntry.map_entry {
-//       key_type_ptr = 1
-//       value_type_ptr = 2
-//     },
-//     TTypeEntry.primitive_entry {
-//       type = INT_TYPE
-//     },
-//     TTypeEntry.primitive_entry {
-//       type = STRING_TYPE
-//     }
-//   ]
-// }
-
-typedef i32 TTypeEntryPtr
-
-// Valid TTypeQualifiers key names
-const string CHARACTER_MAXIMUM_LENGTH = "characterMaximumLength"
-
-// Type qualifier key name for decimal
-const string PRECISION = "precision"
-const string SCALE = "scale"
-
-union TTypeQualifierValue {
-  1: optional i32 i32Value
-  2: optional string stringValue
-}
-
-// Type qualifiers for primitive type.
-struct TTypeQualifiers {
-  1: required map <string, TTypeQualifierValue> qualifiers
-}
-
-// Type entry for a primitive type.
-struct TPrimitiveTypeEntry {
-  // The primitive type token. This must satisfy the condition
-  // that type is in the PRIMITIVE_TYPES set.
-  1: required TTypeId type
-  2: optional TTypeQualifiers typeQualifiers
-}
-
-// Type entry for an ARRAY type.
-struct TArrayTypeEntry {
-  1: required TTypeEntryPtr objectTypePtr
-}
-
-// Type entry for a MAP type.
-struct TMapTypeEntry {
-  1: required TTypeEntryPtr keyTypePtr
-  2: required TTypeEntryPtr valueTypePtr
-}
-
-// Type entry for a STRUCT type.
-struct TStructTypeEntry {
-  1: required map<string, TTypeEntryPtr> nameToTypePtr
-}
-
-// Type entry for a UNIONTYPE type.
-struct TUnionTypeEntry {
-  1: required map<string, TTypeEntryPtr> nameToTypePtr
-}
-
-struct TUserDefinedTypeEntry {
-  // The fully qualified name of the class implementing this type.
-  1: required string typeClassName
-}
-
-// We use a union here since Thrift does not support inheritance.
-union TTypeEntry {
-  1: TPrimitiveTypeEntry primitiveEntry
-  2: TArrayTypeEntry arrayEntry
-  3: TMapTypeEntry mapEntry
-  4: TStructTypeEntry structEntry
-  5: TUnionTypeEntry unionEntry
-  6: TUserDefinedTypeEntry userDefinedTypeEntry
-}
-
-// Type descriptor for columns.
-struct TTypeDesc {
-  // The "top" type is always the first element of the list.
-  // If the top type is an ARRAY, MAP, STRUCT, or UNIONTYPE
-  // type, then subsequent elements represent nested types.
-  1: required list<TTypeEntry> types
-}
-
-// A result set column descriptor.
-struct TColumnDesc {
-  // The name of the column
-  1: required string columnName
-
-  // The type descriptor for this column
-  2: required TTypeDesc typeDesc
-  
-  // The ordinal position of this column in the schema
-  3: required i32 position
-
-  4: optional string comment
-}
-
-// Metadata used to describe the schema (column names, types, comments)
-// of result sets.
-struct TTableSchema {
-  1: required list<TColumnDesc> columns
-}
-
-// A Boolean column value.
-struct TBoolValue {
-  // NULL if value is unset.
-  1: optional bool value
-}
-
-// A Byte column value.
-struct TByteValue {
-  // NULL if value is unset.
-  1: optional byte value
-}
-
-// A signed, 16 bit column value.
-struct TI16Value {
-  // NULL if value is unset
-  1: optional i16 value
-}
-
-// A signed, 32 bit column value
-struct TI32Value {
-  // NULL if value is unset
-  1: optional i32 value
-}
-
-// A signed 64 bit column value
-struct TI64Value {
-  // NULL if value is unset
-  1: optional i64 value
-}
-
-// A floating point 64 bit column value
-struct TDoubleValue {
-  // NULL if value is unset
-  1: optional double value
-}
-
-struct TStringValue {
-  // NULL if value is unset
-  1: optional string value
-}
-
-// A single column value in a result set.
-// Note that Hive's type system is richer than Thrift's,
-// so in some cases we have to map multiple Hive types
-// to the same Thrift type. On the client-side this is
-// disambiguated by looking at the Schema of the
-// result set.
-union TColumnValue {
-  1: TBoolValue   boolVal      // BOOLEAN
-  2: TByteValue   byteVal      // TINYINT
-  3: TI16Value    i16Val       // SMALLINT
-  4: TI32Value    i32Val       // INT
-  5: TI64Value    i64Val       // BIGINT, TIMESTAMP
-  6: TDoubleValue doubleVal    // FLOAT, DOUBLE
-  7: TStringValue stringVal    // STRING, LIST, MAP, STRUCT, UNIONTYPE, BINARY, DECIMAL, NULL, INTERVAL_YEAR_MONTH, INTERVAL_DAY_TIME
-}
-
-// Represents a row in a rowset.
-struct TRow {
-  1: required list<TColumnValue> colVals
-}
-
-struct TBoolColumn {
-  1: required list<bool> values
-  2: required binary nulls
-}
-
-struct TByteColumn {
-  1: required list<byte> values
-  2: required binary nulls
-}
-
-struct TI16Column {
-  1: required list<i16> values
-  2: required binary nulls
-}
-
-struct TI32Column {
-  1: required list<i32> values
-  2: required binary nulls
-}
-
-struct TI64Column {
-  1: required list<i64> values
-  2: required binary nulls
-}
-
-struct TDoubleColumn {
-  1: required list<double> values
-  2: required binary nulls
-}
-
-struct TStringColumn {
-  1: required list<string> values
-  2: required binary nulls
-}
-
-struct TBinaryColumn {
-  1: required list<binary> values
-  2: required binary nulls
-}
-
-// Note that Hive's type system is richer than Thrift's,
-// so in some cases we have to map multiple Hive types
-// to the same Thrift type. On the client-side this is
-// disambiguated by looking at the Schema of the
-// result set.
-union TColumn {
-  1: TBoolColumn   boolVal      // BOOLEAN
-  2: TByteColumn   byteVal      // TINYINT
-  3: TI16Column    i16Val       // SMALLINT
-  4: TI32Column    i32Val       // INT
-  5: TI64Column    i64Val       // BIGINT, TIMESTAMP
-  6: TDoubleColumn doubleVal    // FLOAT, DOUBLE
-  7: TStringColumn stringVal    // STRING, LIST, MAP, STRUCT, UNIONTYPE, DECIMAL, NULL
-  8: TBinaryColumn binaryVal    // BINARY
-}
-
-// Represents a rowset
-struct TRowSet {
-  // The starting row offset of this rowset.
-  1: required i64 startRowOffset
-  2: required list<TRow> rows
-  3: optional list<TColumn> columns
-}
-
-// The return status code contained in each response.
-enum TStatusCode {
-  SUCCESS_STATUS,
-  SUCCESS_WITH_INFO_STATUS,
-  STILL_EXECUTING_STATUS,
-  ERROR_STATUS,
-  INVALID_HANDLE_STATUS
-}
-
-// The return status of a remote request
-struct TStatus {
-  1: required TStatusCode statusCode
-
-  // If status is SUCCESS_WITH_INFO, info_msgs may be populated with
-  // additional diagnostic information.
-  2: optional list<string> infoMessages
-
-  // If status is ERROR, then the following fields may be set
-  3: optional string sqlState  // as defined in the ISO/IEF CLI specification
-  4: optional i32 errorCode    // internal error code
-  5: optional string errorMessage
-}
-
-// The state of an operation (i.e. a query or other
-// asynchronous operation that generates a result set)
-// on the server.
-enum TOperationState {
-  // The operation has been initialized
-  INITIALIZED_STATE,
-
-  // The operation is running. In this state the result
-  // set is not available.
-  RUNNING_STATE,
-
-  // The operation has completed. When an operation is in
-  // this state its result set may be fetched.
-  FINISHED_STATE,
-
-  // The operation was canceled by a client
-  CANCELED_STATE,
-
-  // The operation was closed by a client
-  CLOSED_STATE,
-
-  // The operation failed due to an error
-  ERROR_STATE,
-
-  // The operation is in an unrecognized state
-  UKNOWN_STATE,
-
-  // The operation is in an pending state
-  PENDING_STATE,
-}
-
-// A string identifier. This is interpreted literally.
-typedef string TIdentifier
-
-// A search pattern.
-//
-// Valid search pattern characters:
-// '_': Any single character.
-// '%': Any sequence of zero or more characters.
-// '\': Escape character used to include special characters,
-//      e.g. '_', '%', '\'. If a '\' precedes a non-special
-//      character it has no special meaning and is interpreted
-//      literally.
-typedef string TPattern
-
-
-// A search pattern or identifier. Used as input
-// parameter for many of the catalog functions.
-typedef string TPatternOrIdentifier
-
-struct THandleIdentifier {
-  // 16 byte globally unique identifier
-  // This is the public ID of the handle and
-  // can be used for reporting.
-  1: required binary guid,
-
-  // 16 byte secret generated by the server
-  // and used to verify that the handle is not
-  // being hijacked by another user.
-  2: required binary secret,
-}
-
-// Client-side handle to persistent
-// session information on the server-side.
-struct TSessionHandle {
-  1: required THandleIdentifier sessionId
-}
-
-// The subtype of an OperationHandle.
-enum TOperationType {
-  EXECUTE_STATEMENT,
-  GET_TYPE_INFO,
-  GET_CATALOGS,
-  GET_SCHEMAS,
-  GET_TABLES,
-  GET_TABLE_TYPES,
-  GET_COLUMNS,
-  GET_FUNCTIONS,
-  UNKNOWN,
-}
-
-// Client-side reference to a task running
-// asynchronously on the server.
-struct TOperationHandle {
-  1: required THandleIdentifier operationId
-  2: required TOperationType operationType
-
-  // If hasResultSet = TRUE, then this operation
-  // generates a result set that can be fetched.
-  // Note that the result set may be empty.
-  //
-  // If hasResultSet = FALSE, then this operation
-  // does not generate a result set, and calling
-  // GetResultSetMetadata or FetchResults against
-  // this OperationHandle will generate an error.
-  3: required bool hasResultSet
-
-  // For operations that don't generate result sets,
-  // modifiedRowCount is either:
-  //
-  // 1) The number of rows that were modified by
-  //    the DML operation (e.g. number of rows inserted,
-  //    number of rows deleted, etc).
-  //
-  // 2) 0 for operations that don't modify or add rows.
-  //
-  // 3) < 0 if the operation is capable of modifiying rows,
-  //    but Hive is unable to determine how many rows were
-  //    modified. For example, Hive's LOAD DATA command
-  //    doesn't generate row count information because
-  //    Hive doesn't inspect the data as it is loaded.
-  //
-  // modifiedRowCount is unset if the operation generates
-  // a result set.
-  4: optional double modifiedRowCount
-}
-
-
-// OpenSession()
-//
-// Open a session (connection) on the server against
-// which operations may be executed.
-struct TOpenSessionReq {
-  // The version of the HiveServer2 protocol that the client is using.
-  1: required TProtocolVersion client_protocol = TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V8
-
-  // Username and password for authentication.
-  // Depending on the authentication scheme being used,
-  // this information may instead be provided by a lower
-  // protocol layer, in which case these fields may be
-  // left unset.
-  2: optional string username
-  3: optional string password
-
-  // Configuration overlay which is applied when the session is
-  // first created.
-  4: optional map<string, string> configuration
-}
-
-struct TOpenSessionResp {
-  1: required TStatus status
-
-  // The protocol version that the server is using.
-  2: required TProtocolVersion serverProtocolVersion = TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V8
-
-  // Session Handle
-  3: optional TSessionHandle sessionHandle
-
-  // The configuration settings for this session.
-  4: optional map<string, string> configuration
-}
-
-
-// CloseSession()
-//
-// Closes the specified session and frees any resources
-// currently allocated to that session. Any open
-// operations in that session will be canceled.
-struct TCloseSessionReq {
-  1: required TSessionHandle sessionHandle
-}
-
-struct TCloseSessionResp {
-  1: required TStatus status
-}
-
-
-
-enum TGetInfoType {
-  CLI_MAX_DRIVER_CONNECTIONS =           0,
-  CLI_MAX_CONCURRENT_ACTIVITIES =        1,
-  CLI_DATA_SOURCE_NAME =                 2,
-  CLI_FETCH_DIRECTION =                  8,
-  CLI_SERVER_NAME =                      13,
-  CLI_SEARCH_PATTERN_ESCAPE =            14,
-  CLI_DBMS_NAME =                        17,
-  CLI_DBMS_VER =                         18,
-  CLI_ACCESSIBLE_TABLES =                19,
-  CLI_ACCESSIBLE_PROCEDURES =            20,
-  CLI_CURSOR_COMMIT_BEHAVIOR =           23,
-  CLI_DATA_SOURCE_READ_ONLY =            25,
-  CLI_DEFAULT_TXN_ISOLATION =            26,
-  CLI_IDENTIFIER_CASE =                  28,
-  CLI_IDENTIFIER_QUOTE_CHAR =            29,
-  CLI_MAX_COLUMN_NAME_LEN =              30,
-  CLI_MAX_CURSOR_NAME_LEN =              31,
-  CLI_MAX_SCHEMA_NAME_LEN =              32,
-  CLI_MAX_CATALOG_NAME_LEN =             34,
-  CLI_MAX_TABLE_NAME_LEN =               35,
-  CLI_SCROLL_CONCURRENCY =               43,
-  CLI_TXN_CAPABLE =                      46,
-  CLI_USER_NAME =                        47,
-  CLI_TXN_ISOLATION_OPTION =             72,
-  CLI_INTEGRITY =                        73,
-  CLI_GETDATA_EXTENSIONS =               81,
-  CLI_NULL_COLLATION =                   85,
-  CLI_ALTER_TABLE =                      86,
-  CLI_ORDER_BY_COLUMNS_IN_SELECT =       90,
-  CLI_SPECIAL_CHARACTERS =               94,
-  CLI_MAX_COLUMNS_IN_GROUP_BY =          97,
-  CLI_MAX_COLUMNS_IN_INDEX =             98,
-  CLI_MAX_COLUMNS_IN_ORDER_BY =          99,
-  CLI_MAX_COLUMNS_IN_SELECT =            100,
-  CLI_MAX_COLUMNS_IN_TABLE =             101,
-  CLI_MAX_INDEX_SIZE =                   102,
-  CLI_MAX_ROW_SIZE =                     104,
-  CLI_MAX_STATEMENT_LEN =                105,
-  CLI_MAX_TABLES_IN_SELECT =             106,
-  CLI_MAX_USER_NAME_LEN =                107,
-  CLI_OJ_CAPABILITIES =                  115,
-
-  CLI_XOPEN_CLI_YEAR =                   10000,
-  CLI_CURSOR_SENSITIVITY =               10001,
-  CLI_DESCRIBE_PARAMETER =               10002,
-  CLI_CATALOG_NAME =                     10003,
-  CLI_COLLATION_SEQ =                    10004,
-  CLI_MAX_IDENTIFIER_LEN =               10005,
-}
-
-union TGetInfoValue {
-  1: string stringValue
-  2: i16 smallIntValue
-  3: i32 integerBitmask
-  4: i32 integerFlag
-  5: i32 binaryValue
-  6: i64 lenValue
-}
-
-// GetInfo()
-//
-// This function is based on ODBC's CLIGetInfo() function.
-// The function returns general information about the data source
-// using the same keys as ODBC.
-struct TGetInfoReq {
-  // The session to run this request against
-  1: required TSessionHandle sessionHandle
-
-  2: required TGetInfoType infoType
-}
-
-struct TGetInfoResp {
-  1: required TStatus status
-
-  2: required TGetInfoValue infoValue
-}
-
-
-// ExecuteStatement()
-//
-// Execute a statement.
-// The returned OperationHandle can be used to check on the
-// status of the statement, and to fetch results once the
-// statement has finished executing.
-struct TExecuteStatementReq {
-  // The session to execute the statement against
-  1: required TSessionHandle sessionHandle
-
-  // The statement to be executed (DML, DDL, SET, etc)
-  2: required string statement
-
-  // Configuration properties that are overlayed on top of the
-  // the existing session configuration before this statement
-  // is executed. These properties apply to this statement
-  // only and will not affect the subsequent state of the Session.
-  3: optional map<string, string> confOverlay
-  
-  // Execute asynchronously when runAsync is true
-  4: optional bool runAsync = false
-}
-
-struct TExecuteStatementResp {
-  1: required TStatus status
-  2: optional TOperationHandle operationHandle
-}
-
-// GetTypeInfo()
-//
-// Get information about types supported by the HiveServer instance.
-// The information is returned as a result set which can be fetched
-// using the OperationHandle provided in the response.
-//
-// Refer to the documentation for ODBC's CLIGetTypeInfo function for
-// the format of the result set.
-struct TGetTypeInfoReq {
-  // The session to run this request against.
-  1: required TSessionHandle sessionHandle
-}
-
-struct TGetTypeInfoResp {
-  1: required TStatus status
-  2: optional TOperationHandle operationHandle
-}  
-
-
-// GetCatalogs()
-//
-// Returns the list of catalogs (databases) 
-// Results are ordered by TABLE_CATALOG 
-//
-// Resultset columns :
-// col1
-// name: TABLE_CAT
-// type: STRING
-// desc: Catalog name. NULL if not applicable.
-//
-struct TGetCatalogsReq {
-  // Session to run this request against
-  1: required TSessionHandle sessionHandle
-}
-
-struct TGetCatalogsResp {
-  1: required TStatus status
-  2: optional TOperationHandle operationHandle
-}
-
-
-// GetSchemas()
-//
-// Retrieves the schema names available in this database. 
-// The results are ordered by TABLE_CATALOG and TABLE_SCHEM.
-// col1
-// name: TABLE_SCHEM
-// type: STRING
-// desc: schema name
-// col2
-// name: TABLE_CATALOG
-// type: STRING
-// desc: catalog name
-struct TGetSchemasReq {
-  // Session to run this request against
-  1: required TSessionHandle sessionHandle
-
-  // Name of the catalog. Must not contain a search pattern.
-  2: optional TIdentifier catalogName
-
-  // schema name or pattern
-  3: optional TPatternOrIdentifier schemaName
-}
-
-struct TGetSchemasResp {
-  1: required TStatus status
-  2: optional TOperationHandle operationHandle
-}
-
-
-// GetTables()
-//
-// Returns a list of tables with catalog, schema, and table
-// type information. The information is returned as a result
-// set which can be fetched using the OperationHandle
-// provided in the response.
-// Results are ordered by TABLE_TYPE, TABLE_CAT, TABLE_SCHEM, and TABLE_NAME
-//
-// Result Set Columns:
-//
-// col1
-// name: TABLE_CAT
-// type: STRING
-// desc: Catalog name. NULL if not applicable.
-//
-// col2
-// name: TABLE_SCHEM
-// type: STRING
-// desc: Schema name.
-//
-// col3
-// name: TABLE_NAME
-// type: STRING
-// desc: Table name.
-//
-// col4
-// name: TABLE_TYPE
-// type: STRING
-// desc: The table type, e.g. "TABLE", "VIEW", etc.
-//
-// col5
-// name: REMARKS
-// type: STRING
-// desc: Comments about the table
-//
-struct TGetTablesReq {
-  // Session to run this request against
-  1: required TSessionHandle sessionHandle
-
-  // Name of the catalog or a search pattern.
-  2: optional TPatternOrIdentifier catalogName
-
-  // Name of the schema or a search pattern.
-  3: optional TPatternOrIdentifier schemaName
-
-  // Name of the table or a search pattern.
-  4: optional TPatternOrIdentifier tableName
-
-  // List of table types to match
-  // e.g. "TABLE", "VIEW", "SYSTEM TABLE", "GLOBAL TEMPORARY",
-  // "LOCAL TEMPORARY", "ALIAS", "SYNONYM", etc.
-  5: optional list<string> tableTypes
-}
-
-struct TGetTablesResp {
-  1: required TStatus status
-  2: optional TOperationHandle operationHandle
-}
-
-
-// GetTableTypes()
-//
-// Returns the table types available in this database. 
-// The results are ordered by table type. 
-// 
-// col1
-// name: TABLE_TYPE
-// type: STRING
-// desc: Table type name.
-struct TGetTableTypesReq {
-  // Session to run this request against
-  1: required TSessionHandle sessionHandle
-}
-
-struct TGetTableTypesResp {
-  1: required TStatus status
-  2: optional TOperationHandle operationHandle
-}
-
-
-// GetColumns()
-//
-// Returns a list of columns in the specified tables.
-// The information is returned as a result set which can be fetched
-// using the OperationHandle provided in the response.
-// Results are ordered by TABLE_CAT, TABLE_SCHEM, TABLE_NAME, 
-// and ORDINAL_POSITION. 
-//
-// Result Set Columns are the same as those for the ODBC CLIColumns
-// function.
-//
-struct TGetColumnsReq {
-  // Session to run this request against
-  1: required TSessionHandle sessionHandle
-
-  // Name of the catalog. Must not contain a search pattern.
-  2: optional TIdentifier catalogName
-
-  // Schema name or search pattern
-  3: optional TPatternOrIdentifier schemaName
-
-  // Table name or search pattern
-  4: optional TPatternOrIdentifier tableName
-
-  // Column name or search pattern
-  5: optional TPatternOrIdentifier columnName
-}
-
-struct TGetColumnsResp {
-  1: required TStatus status
-  2: optional TOperationHandle operationHandle
-}
-
-
-// GetFunctions()
-//
-// Returns a list of functions supported by the data source. The
-// behavior of this function matches
-// java.sql.DatabaseMetaData.getFunctions() both in terms of
-// inputs and outputs.
-//
-// Result Set Columns:
-//
-// col1
-// name: FUNCTION_CAT
-// type: STRING
-// desc: Function catalog (may be null)
-//
-// col2
-// name: FUNCTION_SCHEM
-// type: STRING
-// desc: Function schema (may be null)
-//
-// col3
-// name: FUNCTION_NAME
-// type: STRING
-// desc: Function name. This is the name used to invoke the function.
-//
-// col4
-// name: REMARKS
-// type: STRING
-// desc: Explanatory comment on the function.
-//
-// col5
-// name: FUNCTION_TYPE
-// type: SMALLINT
-// desc: Kind of function. One of:
-//       * functionResultUnknown - Cannot determine if a return value or a table
-//                                 will be returned.
-//       * functionNoTable       - Does not a return a table.
-//       * functionReturnsTable  - Returns a table.
-//
-// col6
-// name: SPECIFIC_NAME
-// type: STRING
-// desc: The name which uniquely identifies this function within its schema.
-//       In this case this is the fully qualified class name of the class
-//       that implements this function.
-//
-struct TGetFunctionsReq {
-  // Session to run this request against
-  1: required TSessionHandle sessionHandle
-
-  // A catalog name; must match the catalog name as it is stored in the
-  // database; "" retrieves those without a catalog; null means
-  // that the catalog name should not be used to narrow the search.
-  2: optional TIdentifier catalogName
-
-  // A schema name pattern; must match the schema name as it is stored
-  // in the database; "" retrieves those without a schema; null means
-  // that the schema name should not be used to narrow the search.
-  3: optional TPatternOrIdentifier schemaName
-
-  // A function name pattern; must match the function name as it is stored
-  // in the database.
-  4: required TPatternOrIdentifier functionName
-}
-
-struct TGetFunctionsResp {
-  1: required TStatus status
-  2: optional TOperationHandle operationHandle
-}
-  
-
-// GetOperationStatus()
-//
-// Get the status of an operation running on the server.
-struct TGetOperationStatusReq {
-  // Session to run this request against
-  1: required TOperationHandle operationHandle
-}
-
-struct TGetOperationStatusResp {
-  1: required TStatus status
-  2: optional TOperationState operationState
-
-  // If operationState is ERROR_STATE, then the following fields may be set
-  // sqlState as defined in the ISO/IEF CLI specification
-  3: optional string sqlState
-
-  // Internal error code
-  4: optional i32 errorCode
-
-  // Error message
-  5: optional string errorMessage
-}
-
-
-// CancelOperation()
-//
-// Cancels processing on the specified operation handle and
-// frees any resources which were allocated.
-struct TCancelOperationReq {
-  // Operation to cancel
-  1: required TOperationHandle operationHandle
-}
-
-struct TCancelOperationResp {
-  1: required TStatus status
-}
-
-
-// CloseOperation()
-//
-// Given an operation in the FINISHED, CANCELED,
-// or ERROR states, CloseOperation() will free
-// all of the resources which were allocated on
-// the server to service the operation.
-struct TCloseOperationReq {
-  1: required TOperationHandle operationHandle
-}
-
-struct TCloseOperationResp {
-  1: required TStatus status
-}
-
-
-// GetResultSetMetadata()
-//
-// Retrieves schema information for the specified operation
-struct TGetResultSetMetadataReq {
-  // Operation for which to fetch result set schema information
-  1: required TOperationHandle operationHandle
-}
-
-struct TGetResultSetMetadataResp {
-  1: required TStatus status
-  2: optional TTableSchema schema
-}
-
-
-enum TFetchOrientation {
-  // Get the next rowset. The fetch offset is ignored.
-  FETCH_NEXT,
-
-  // Get the previous rowset. The fetch offset is ignored.
-  FETCH_PRIOR,
-
-  // Return the rowset at the given fetch offset relative
-  // to the current rowset.
-  // NOT SUPPORTED
-  FETCH_RELATIVE,
-
-  // Return the rowset at the specified fetch offset.
-  // NOT SUPPORTED
-  FETCH_ABSOLUTE,
-
-  // Get the first rowset in the result set.
-  FETCH_FIRST,
-
-  // Get the last rowset in the result set.
-  // NOT SUPPORTED
-  FETCH_LAST
-}
-
-// FetchResults()
-//
-// Fetch rows from the server corresponding to
-// a particular OperationHandle.
-struct TFetchResultsReq {
-  // Operation from which to fetch results.
-  1: required TOperationHandle operationHandle
-
-  // The fetch orientation. This must be either
-  // FETCH_NEXT, FETCH_PRIOR or FETCH_FIRST. Defaults to FETCH_NEXT.
-  2: required TFetchOrientation orientation = TFetchOrientation.FETCH_NEXT
-  
-  // Max number of rows that should be returned in
-  // the rowset.
-  3: required i64 maxRows
-
-  // The type of a fetch results request. 0 represents Query output. 1 represents Log
-  4: optional i16 fetchType = 0
-}
-
-struct TFetchResultsResp {
-  1: required TStatus status
-
-  // TRUE if there are more rows left to fetch from the server.
-  2: optional bool hasMoreRows
-
-  // The rowset. This is optional so that we have the
-  // option in the future of adding alternate formats for
-  // representing result set data, e.g. delimited strings,
-  // binary encoded, etc.
-  3: optional TRowSet results
-}
-
-// GetDelegationToken()
-// Retrieve delegation token for the current user
-struct  TGetDelegationTokenReq {
-  // session handle
-  1: required TSessionHandle sessionHandle
-
-  // userid for the proxy user
-  2: required string owner
-
-  // designated renewer userid
-  3: required string renewer
-}
-
-struct TGetDelegationTokenResp {
-  // status of the request
-  1: required TStatus status
-
-  // delegation token string
-  2: optional string delegationToken
-}
-
-// CancelDelegationToken()
-// Cancel the given delegation token
-struct TCancelDelegationTokenReq {
-  // session handle
-  1: required TSessionHandle sessionHandle
-
-  // delegation token to cancel
-  2: required string delegationToken
-}
-
-struct TCancelDelegationTokenResp {
-  // status of the request
-  1: required TStatus status
-}
-
-// RenewDelegationToken()
-// Renew the given delegation token
-struct TRenewDelegationTokenReq {
-  // session handle
-  1: required TSessionHandle sessionHandle
-
-  // delegation token to renew
-  2: required string delegationToken
-}
-
-struct TRenewDelegationTokenResp {
-  // status of the request
-  1: required TStatus status
-}
-
-service TCLIService {
-
-  TOpenSessionResp OpenSession(1:TOpenSessionReq req);
-
-  TCloseSessionResp CloseSession(1:TCloseSessionReq req);
-
-  TGetInfoResp GetInfo(1:TGetInfoReq req);
-
-  TExecuteStatementResp ExecuteStatement(1:TExecuteStatementReq req);
-
-  TGetTypeInfoResp GetTypeInfo(1:TGetTypeInfoReq req);
-
-  TGetCatalogsResp GetCatalogs(1:TGetCatalogsReq req);
-
-  TGetSchemasResp GetSchemas(1:TGetSchemasReq req);
-
-  TGetTablesResp GetTables(1:TGetTablesReq req);
-
-  TGetTableTypesResp GetTableTypes(1:TGetTableTypesReq req);
-
-  TGetColumnsResp GetColumns(1:TGetColumnsReq req);
-
-  TGetFunctionsResp GetFunctions(1:TGetFunctionsReq req);
-
-  TGetOperationStatusResp GetOperationStatus(1:TGetOperationStatusReq req);
-  
-  TCancelOperationResp CancelOperation(1:TCancelOperationReq req);
-
-  TCloseOperationResp CloseOperation(1:TCloseOperationReq req);
-
-  TGetResultSetMetadataResp GetResultSetMetadata(1:TGetResultSetMetadataReq req);
-
-  TFetchResultsResp FetchResults(1:TFetchResultsReq req);
-
-  TGetDelegationTokenResp GetDelegationToken(1:TGetDelegationTokenReq req);
-
-  TCancelDelegationTokenResp CancelDelegationToken(1:TCancelDelegationTokenReq req);
-
-  TRenewDelegationTokenResp RenewDelegationToken(1:TRenewDelegationTokenReq req);
-}
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TArrayTypeEntry.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TArrayTypeEntry.java
deleted file mode 100644
index 6323d34eac734..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TArrayTypeEntry.java
+++ /dev/null
@@ -1,383 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TArrayTypeEntry implements org.apache.thrift.TBase<TArrayTypeEntry, TArrayTypeEntry._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TArrayTypeEntry");
-
-  private static final org.apache.thrift.protocol.TField OBJECT_TYPE_PTR_FIELD_DESC = new org.apache.thrift.protocol.TField("objectTypePtr", org.apache.thrift.protocol.TType.I32, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TArrayTypeEntryStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TArrayTypeEntryTupleSchemeFactory());
-  }
-
-  private int objectTypePtr; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    OBJECT_TYPE_PTR((short)1, "objectTypePtr");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // OBJECT_TYPE_PTR
-          return OBJECT_TYPE_PTR;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __OBJECTTYPEPTR_ISSET_ID = 0;
-  private byte __isset_bitfield = 0;
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.OBJECT_TYPE_PTR, new org.apache.thrift.meta_data.FieldMetaData("objectTypePtr", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32        , "TTypeEntryPtr")));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TArrayTypeEntry.class, metaDataMap);
-  }
-
-  public TArrayTypeEntry() {
-  }
-
-  public TArrayTypeEntry(
-    int objectTypePtr)
-  {
-    this();
-    this.objectTypePtr = objectTypePtr;
-    setObjectTypePtrIsSet(true);
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TArrayTypeEntry(TArrayTypeEntry other) {
-    __isset_bitfield = other.__isset_bitfield;
-    this.objectTypePtr = other.objectTypePtr;
-  }
-
-  public TArrayTypeEntry deepCopy() {
-    return new TArrayTypeEntry(this);
-  }
-
-  @Override
-  public void clear() {
-    setObjectTypePtrIsSet(false);
-    this.objectTypePtr = 0;
-  }
-
-  public int getObjectTypePtr() {
-    return this.objectTypePtr;
-  }
-
-  public void setObjectTypePtr(int objectTypePtr) {
-    this.objectTypePtr = objectTypePtr;
-    setObjectTypePtrIsSet(true);
-  }
-
-  public void unsetObjectTypePtr() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __OBJECTTYPEPTR_ISSET_ID);
-  }
-
-  /** Returns true if field objectTypePtr is set (has been assigned a value) and false otherwise */
-  public boolean isSetObjectTypePtr() {
-    return EncodingUtils.testBit(__isset_bitfield, __OBJECTTYPEPTR_ISSET_ID);
-  }
-
-  public void setObjectTypePtrIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __OBJECTTYPEPTR_ISSET_ID, value);
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case OBJECT_TYPE_PTR:
-      if (value == null) {
-        unsetObjectTypePtr();
-      } else {
-        setObjectTypePtr((Integer)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case OBJECT_TYPE_PTR:
-      return Integer.valueOf(getObjectTypePtr());
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case OBJECT_TYPE_PTR:
-      return isSetObjectTypePtr();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TArrayTypeEntry)
-      return this.equals((TArrayTypeEntry)that);
-    return false;
-  }
-
-  public boolean equals(TArrayTypeEntry that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_objectTypePtr = true;
-    boolean that_present_objectTypePtr = true;
-    if (this_present_objectTypePtr || that_present_objectTypePtr) {
-      if (!(this_present_objectTypePtr && that_present_objectTypePtr))
-        return false;
-      if (this.objectTypePtr != that.objectTypePtr)
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_objectTypePtr = true;
-    builder.append(present_objectTypePtr);
-    if (present_objectTypePtr)
-      builder.append(objectTypePtr);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TArrayTypeEntry other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TArrayTypeEntry typedOther = (TArrayTypeEntry)other;
-
-    lastComparison = Boolean.valueOf(isSetObjectTypePtr()).compareTo(typedOther.isSetObjectTypePtr());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetObjectTypePtr()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.objectTypePtr, typedOther.objectTypePtr);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TArrayTypeEntry(");
-    boolean first = true;
-
-    sb.append("objectTypePtr:");
-    sb.append(this.objectTypePtr);
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetObjectTypePtr()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'objectTypePtr' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TArrayTypeEntryStandardSchemeFactory implements SchemeFactory {
-    public TArrayTypeEntryStandardScheme getScheme() {
-      return new TArrayTypeEntryStandardScheme();
-    }
-  }
-
-  private static class TArrayTypeEntryStandardScheme extends StandardScheme<TArrayTypeEntry> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TArrayTypeEntry struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // OBJECT_TYPE_PTR
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.objectTypePtr = iprot.readI32();
-              struct.setObjectTypePtrIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TArrayTypeEntry struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      oprot.writeFieldBegin(OBJECT_TYPE_PTR_FIELD_DESC);
-      oprot.writeI32(struct.objectTypePtr);
-      oprot.writeFieldEnd();
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TArrayTypeEntryTupleSchemeFactory implements SchemeFactory {
-    public TArrayTypeEntryTupleScheme getScheme() {
-      return new TArrayTypeEntryTupleScheme();
-    }
-  }
-
-  private static class TArrayTypeEntryTupleScheme extends TupleScheme<TArrayTypeEntry> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TArrayTypeEntry struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      oprot.writeI32(struct.objectTypePtr);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TArrayTypeEntry struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.objectTypePtr = iprot.readI32();
-      struct.setObjectTypePtrIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TBinaryColumn.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TBinaryColumn.java
deleted file mode 100644
index 6b1b054d1acad..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TBinaryColumn.java
+++ /dev/null
@@ -1,550 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TBinaryColumn implements org.apache.thrift.TBase<TBinaryColumn, TBinaryColumn._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TBinaryColumn");
-
-  private static final org.apache.thrift.protocol.TField VALUES_FIELD_DESC = new org.apache.thrift.protocol.TField("values", org.apache.thrift.protocol.TType.LIST, (short)1);
-  private static final org.apache.thrift.protocol.TField NULLS_FIELD_DESC = new org.apache.thrift.protocol.TField("nulls", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TBinaryColumnStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TBinaryColumnTupleSchemeFactory());
-  }
-
-  private List<ByteBuffer> values; // required
-  private ByteBuffer nulls; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUES((short)1, "values"),
-    NULLS((short)2, "nulls");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUES
-          return VALUES;
-        case 2: // NULLS
-          return NULLS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUES, new org.apache.thrift.meta_data.FieldMetaData("values", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING            , true))));
-    tmpMap.put(_Fields.NULLS, new org.apache.thrift.meta_data.FieldMetaData("nulls", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TBinaryColumn.class, metaDataMap);
-  }
-
-  public TBinaryColumn() {
-  }
-
-  public TBinaryColumn(
-    List<ByteBuffer> values,
-    ByteBuffer nulls)
-  {
-    this();
-    this.values = values;
-    this.nulls = nulls;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TBinaryColumn(TBinaryColumn other) {
-    if (other.isSetValues()) {
-      List<ByteBuffer> __this__values = new ArrayList<ByteBuffer>();
-      for (ByteBuffer other_element : other.values) {
-        ByteBuffer temp_binary_element = org.apache.thrift.TBaseHelper.copyBinary(other_element);
-;
-        __this__values.add(temp_binary_element);
-      }
-      this.values = __this__values;
-    }
-    if (other.isSetNulls()) {
-      this.nulls = org.apache.thrift.TBaseHelper.copyBinary(other.nulls);
-;
-    }
-  }
-
-  public TBinaryColumn deepCopy() {
-    return new TBinaryColumn(this);
-  }
-
-  @Override
-  public void clear() {
-    this.values = null;
-    this.nulls = null;
-  }
-
-  public int getValuesSize() {
-    return (this.values == null) ? 0 : this.values.size();
-  }
-
-  public java.util.Iterator<ByteBuffer> getValuesIterator() {
-    return (this.values == null) ? null : this.values.iterator();
-  }
-
-  public void addToValues(ByteBuffer elem) {
-    if (this.values == null) {
-      this.values = new ArrayList<ByteBuffer>();
-    }
-    this.values.add(elem);
-  }
-
-  public List<ByteBuffer> getValues() {
-    return this.values;
-  }
-
-  public void setValues(List<ByteBuffer> values) {
-    this.values = values;
-  }
-
-  public void unsetValues() {
-    this.values = null;
-  }
-
-  /** Returns true if field values is set (has been assigned a value) and false otherwise */
-  public boolean isSetValues() {
-    return this.values != null;
-  }
-
-  public void setValuesIsSet(boolean value) {
-    if (!value) {
-      this.values = null;
-    }
-  }
-
-  public byte[] getNulls() {
-    setNulls(org.apache.thrift.TBaseHelper.rightSize(nulls));
-    return nulls == null ? null : nulls.array();
-  }
-
-  public ByteBuffer bufferForNulls() {
-    return nulls;
-  }
-
-  public void setNulls(byte[] nulls) {
-    setNulls(nulls == null ? (ByteBuffer)null : ByteBuffer.wrap(nulls));
-  }
-
-  public void setNulls(ByteBuffer nulls) {
-    this.nulls = nulls;
-  }
-
-  public void unsetNulls() {
-    this.nulls = null;
-  }
-
-  /** Returns true if field nulls is set (has been assigned a value) and false otherwise */
-  public boolean isSetNulls() {
-    return this.nulls != null;
-  }
-
-  public void setNullsIsSet(boolean value) {
-    if (!value) {
-      this.nulls = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUES:
-      if (value == null) {
-        unsetValues();
-      } else {
-        setValues((List<ByteBuffer>)value);
-      }
-      break;
-
-    case NULLS:
-      if (value == null) {
-        unsetNulls();
-      } else {
-        setNulls((ByteBuffer)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUES:
-      return getValues();
-
-    case NULLS:
-      return getNulls();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUES:
-      return isSetValues();
-    case NULLS:
-      return isSetNulls();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TBinaryColumn)
-      return this.equals((TBinaryColumn)that);
-    return false;
-  }
-
-  public boolean equals(TBinaryColumn that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_values = true && this.isSetValues();
-    boolean that_present_values = true && that.isSetValues();
-    if (this_present_values || that_present_values) {
-      if (!(this_present_values && that_present_values))
-        return false;
-      if (!this.values.equals(that.values))
-        return false;
-    }
-
-    boolean this_present_nulls = true && this.isSetNulls();
-    boolean that_present_nulls = true && that.isSetNulls();
-    if (this_present_nulls || that_present_nulls) {
-      if (!(this_present_nulls && that_present_nulls))
-        return false;
-      if (!this.nulls.equals(that.nulls))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_values = true && (isSetValues());
-    builder.append(present_values);
-    if (present_values)
-      builder.append(values);
-
-    boolean present_nulls = true && (isSetNulls());
-    builder.append(present_nulls);
-    if (present_nulls)
-      builder.append(nulls);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TBinaryColumn other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TBinaryColumn typedOther = (TBinaryColumn)other;
-
-    lastComparison = Boolean.valueOf(isSetValues()).compareTo(typedOther.isSetValues());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValues()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.values, typedOther.values);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetNulls()).compareTo(typedOther.isSetNulls());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetNulls()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.nulls, typedOther.nulls);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TBinaryColumn(");
-    boolean first = true;
-
-    sb.append("values:");
-    if (this.values == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.values);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("nulls:");
-    if (this.nulls == null) {
-      sb.append("null");
-    } else {
-      org.apache.thrift.TBaseHelper.toString(this.nulls, sb);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetValues()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'values' is unset! Struct:" + toString());
-    }
-
-    if (!isSetNulls()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'nulls' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TBinaryColumnStandardSchemeFactory implements SchemeFactory {
-    public TBinaryColumnStandardScheme getScheme() {
-      return new TBinaryColumnStandardScheme();
-    }
-  }
-
-  private static class TBinaryColumnStandardScheme extends StandardScheme<TBinaryColumn> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TBinaryColumn struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUES
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list110 = iprot.readListBegin();
-                struct.values = new ArrayList<ByteBuffer>(_list110.size);
-                for (int _i111 = 0; _i111 < _list110.size; ++_i111)
-                {
-                  ByteBuffer _elem112; // optional
-                  _elem112 = iprot.readBinary();
-                  struct.values.add(_elem112);
-                }
-                iprot.readListEnd();
-              }
-              struct.setValuesIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // NULLS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.nulls = iprot.readBinary();
-              struct.setNullsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TBinaryColumn struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.values != null) {
-        oprot.writeFieldBegin(VALUES_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.values.size()));
-          for (ByteBuffer _iter113 : struct.values)
-          {
-            oprot.writeBinary(_iter113);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      if (struct.nulls != null) {
-        oprot.writeFieldBegin(NULLS_FIELD_DESC);
-        oprot.writeBinary(struct.nulls);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TBinaryColumnTupleSchemeFactory implements SchemeFactory {
-    public TBinaryColumnTupleScheme getScheme() {
-      return new TBinaryColumnTupleScheme();
-    }
-  }
-
-  private static class TBinaryColumnTupleScheme extends TupleScheme<TBinaryColumn> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TBinaryColumn struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.values.size());
-        for (ByteBuffer _iter114 : struct.values)
-        {
-          oprot.writeBinary(_iter114);
-        }
-      }
-      oprot.writeBinary(struct.nulls);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TBinaryColumn struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TList _list115 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-        struct.values = new ArrayList<ByteBuffer>(_list115.size);
-        for (int _i116 = 0; _i116 < _list115.size; ++_i116)
-        {
-          ByteBuffer _elem117; // optional
-          _elem117 = iprot.readBinary();
-          struct.values.add(_elem117);
-        }
-      }
-      struct.setValuesIsSet(true);
-      struct.nulls = iprot.readBinary();
-      struct.setNullsIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TBoolColumn.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TBoolColumn.java
deleted file mode 100644
index efd571cfdfbbf..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TBoolColumn.java
+++ /dev/null
@@ -1,548 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TBoolColumn implements org.apache.thrift.TBase<TBoolColumn, TBoolColumn._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TBoolColumn");
-
-  private static final org.apache.thrift.protocol.TField VALUES_FIELD_DESC = new org.apache.thrift.protocol.TField("values", org.apache.thrift.protocol.TType.LIST, (short)1);
-  private static final org.apache.thrift.protocol.TField NULLS_FIELD_DESC = new org.apache.thrift.protocol.TField("nulls", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TBoolColumnStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TBoolColumnTupleSchemeFactory());
-  }
-
-  private List<Boolean> values; // required
-  private ByteBuffer nulls; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUES((short)1, "values"),
-    NULLS((short)2, "nulls");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUES
-          return VALUES;
-        case 2: // NULLS
-          return NULLS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUES, new org.apache.thrift.meta_data.FieldMetaData("values", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))));
-    tmpMap.put(_Fields.NULLS, new org.apache.thrift.meta_data.FieldMetaData("nulls", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TBoolColumn.class, metaDataMap);
-  }
-
-  public TBoolColumn() {
-  }
-
-  public TBoolColumn(
-    List<Boolean> values,
-    ByteBuffer nulls)
-  {
-    this();
-    this.values = values;
-    this.nulls = nulls;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TBoolColumn(TBoolColumn other) {
-    if (other.isSetValues()) {
-      List<Boolean> __this__values = new ArrayList<Boolean>();
-      for (Boolean other_element : other.values) {
-        __this__values.add(other_element);
-      }
-      this.values = __this__values;
-    }
-    if (other.isSetNulls()) {
-      this.nulls = org.apache.thrift.TBaseHelper.copyBinary(other.nulls);
-;
-    }
-  }
-
-  public TBoolColumn deepCopy() {
-    return new TBoolColumn(this);
-  }
-
-  @Override
-  public void clear() {
-    this.values = null;
-    this.nulls = null;
-  }
-
-  public int getValuesSize() {
-    return (this.values == null) ? 0 : this.values.size();
-  }
-
-  public java.util.Iterator<Boolean> getValuesIterator() {
-    return (this.values == null) ? null : this.values.iterator();
-  }
-
-  public void addToValues(boolean elem) {
-    if (this.values == null) {
-      this.values = new ArrayList<Boolean>();
-    }
-    this.values.add(elem);
-  }
-
-  public List<Boolean> getValues() {
-    return this.values;
-  }
-
-  public void setValues(List<Boolean> values) {
-    this.values = values;
-  }
-
-  public void unsetValues() {
-    this.values = null;
-  }
-
-  /** Returns true if field values is set (has been assigned a value) and false otherwise */
-  public boolean isSetValues() {
-    return this.values != null;
-  }
-
-  public void setValuesIsSet(boolean value) {
-    if (!value) {
-      this.values = null;
-    }
-  }
-
-  public byte[] getNulls() {
-    setNulls(org.apache.thrift.TBaseHelper.rightSize(nulls));
-    return nulls == null ? null : nulls.array();
-  }
-
-  public ByteBuffer bufferForNulls() {
-    return nulls;
-  }
-
-  public void setNulls(byte[] nulls) {
-    setNulls(nulls == null ? (ByteBuffer)null : ByteBuffer.wrap(nulls));
-  }
-
-  public void setNulls(ByteBuffer nulls) {
-    this.nulls = nulls;
-  }
-
-  public void unsetNulls() {
-    this.nulls = null;
-  }
-
-  /** Returns true if field nulls is set (has been assigned a value) and false otherwise */
-  public boolean isSetNulls() {
-    return this.nulls != null;
-  }
-
-  public void setNullsIsSet(boolean value) {
-    if (!value) {
-      this.nulls = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUES:
-      if (value == null) {
-        unsetValues();
-      } else {
-        setValues((List<Boolean>)value);
-      }
-      break;
-
-    case NULLS:
-      if (value == null) {
-        unsetNulls();
-      } else {
-        setNulls((ByteBuffer)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUES:
-      return getValues();
-
-    case NULLS:
-      return getNulls();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUES:
-      return isSetValues();
-    case NULLS:
-      return isSetNulls();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TBoolColumn)
-      return this.equals((TBoolColumn)that);
-    return false;
-  }
-
-  public boolean equals(TBoolColumn that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_values = true && this.isSetValues();
-    boolean that_present_values = true && that.isSetValues();
-    if (this_present_values || that_present_values) {
-      if (!(this_present_values && that_present_values))
-        return false;
-      if (!this.values.equals(that.values))
-        return false;
-    }
-
-    boolean this_present_nulls = true && this.isSetNulls();
-    boolean that_present_nulls = true && that.isSetNulls();
-    if (this_present_nulls || that_present_nulls) {
-      if (!(this_present_nulls && that_present_nulls))
-        return false;
-      if (!this.nulls.equals(that.nulls))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_values = true && (isSetValues());
-    builder.append(present_values);
-    if (present_values)
-      builder.append(values);
-
-    boolean present_nulls = true && (isSetNulls());
-    builder.append(present_nulls);
-    if (present_nulls)
-      builder.append(nulls);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TBoolColumn other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TBoolColumn typedOther = (TBoolColumn)other;
-
-    lastComparison = Boolean.valueOf(isSetValues()).compareTo(typedOther.isSetValues());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValues()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.values, typedOther.values);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetNulls()).compareTo(typedOther.isSetNulls());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetNulls()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.nulls, typedOther.nulls);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TBoolColumn(");
-    boolean first = true;
-
-    sb.append("values:");
-    if (this.values == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.values);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("nulls:");
-    if (this.nulls == null) {
-      sb.append("null");
-    } else {
-      org.apache.thrift.TBaseHelper.toString(this.nulls, sb);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetValues()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'values' is unset! Struct:" + toString());
-    }
-
-    if (!isSetNulls()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'nulls' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TBoolColumnStandardSchemeFactory implements SchemeFactory {
-    public TBoolColumnStandardScheme getScheme() {
-      return new TBoolColumnStandardScheme();
-    }
-  }
-
-  private static class TBoolColumnStandardScheme extends StandardScheme<TBoolColumn> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TBoolColumn struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUES
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list54 = iprot.readListBegin();
-                struct.values = new ArrayList<Boolean>(_list54.size);
-                for (int _i55 = 0; _i55 < _list54.size; ++_i55)
-                {
-                  boolean _elem56; // optional
-                  _elem56 = iprot.readBool();
-                  struct.values.add(_elem56);
-                }
-                iprot.readListEnd();
-              }
-              struct.setValuesIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // NULLS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.nulls = iprot.readBinary();
-              struct.setNullsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TBoolColumn struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.values != null) {
-        oprot.writeFieldBegin(VALUES_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.BOOL, struct.values.size()));
-          for (boolean _iter57 : struct.values)
-          {
-            oprot.writeBool(_iter57);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      if (struct.nulls != null) {
-        oprot.writeFieldBegin(NULLS_FIELD_DESC);
-        oprot.writeBinary(struct.nulls);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TBoolColumnTupleSchemeFactory implements SchemeFactory {
-    public TBoolColumnTupleScheme getScheme() {
-      return new TBoolColumnTupleScheme();
-    }
-  }
-
-  private static class TBoolColumnTupleScheme extends TupleScheme<TBoolColumn> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TBoolColumn struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.values.size());
-        for (boolean _iter58 : struct.values)
-        {
-          oprot.writeBool(_iter58);
-        }
-      }
-      oprot.writeBinary(struct.nulls);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TBoolColumn struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TList _list59 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.BOOL, iprot.readI32());
-        struct.values = new ArrayList<Boolean>(_list59.size);
-        for (int _i60 = 0; _i60 < _list59.size; ++_i60)
-        {
-          boolean _elem61; // optional
-          _elem61 = iprot.readBool();
-          struct.values.add(_elem61);
-        }
-      }
-      struct.setValuesIsSet(true);
-      struct.nulls = iprot.readBinary();
-      struct.setNullsIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TBoolValue.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TBoolValue.java
deleted file mode 100644
index c7495ee79e4b5..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TBoolValue.java
+++ /dev/null
@@ -1,386 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TBoolValue implements org.apache.thrift.TBase<TBoolValue, TBoolValue._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TBoolValue");
-
-  private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.BOOL, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TBoolValueStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TBoolValueTupleSchemeFactory());
-  }
-
-  private boolean value; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUE((short)1, "value");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUE
-          return VALUE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __VALUE_ISSET_ID = 0;
-  private byte __isset_bitfield = 0;
-  private _Fields optionals[] = {_Fields.VALUE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TBoolValue.class, metaDataMap);
-  }
-
-  public TBoolValue() {
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TBoolValue(TBoolValue other) {
-    __isset_bitfield = other.__isset_bitfield;
-    this.value = other.value;
-  }
-
-  public TBoolValue deepCopy() {
-    return new TBoolValue(this);
-  }
-
-  @Override
-  public void clear() {
-    setValueIsSet(false);
-    this.value = false;
-  }
-
-  public boolean isValue() {
-    return this.value;
-  }
-
-  public void setValue(boolean value) {
-    this.value = value;
-    setValueIsSet(true);
-  }
-
-  public void unsetValue() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __VALUE_ISSET_ID);
-  }
-
-  /** Returns true if field value is set (has been assigned a value) and false otherwise */
-  public boolean isSetValue() {
-    return EncodingUtils.testBit(__isset_bitfield, __VALUE_ISSET_ID);
-  }
-
-  public void setValueIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __VALUE_ISSET_ID, value);
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUE:
-      if (value == null) {
-        unsetValue();
-      } else {
-        setValue((Boolean)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUE:
-      return Boolean.valueOf(isValue());
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUE:
-      return isSetValue();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TBoolValue)
-      return this.equals((TBoolValue)that);
-    return false;
-  }
-
-  public boolean equals(TBoolValue that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_value = true && this.isSetValue();
-    boolean that_present_value = true && that.isSetValue();
-    if (this_present_value || that_present_value) {
-      if (!(this_present_value && that_present_value))
-        return false;
-      if (this.value != that.value)
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_value = true && (isSetValue());
-    builder.append(present_value);
-    if (present_value)
-      builder.append(value);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TBoolValue other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TBoolValue typedOther = (TBoolValue)other;
-
-    lastComparison = Boolean.valueOf(isSetValue()).compareTo(typedOther.isSetValue());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValue()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, typedOther.value);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TBoolValue(");
-    boolean first = true;
-
-    if (isSetValue()) {
-      sb.append("value:");
-      sb.append(this.value);
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TBoolValueStandardSchemeFactory implements SchemeFactory {
-    public TBoolValueStandardScheme getScheme() {
-      return new TBoolValueStandardScheme();
-    }
-  }
-
-  private static class TBoolValueStandardScheme extends StandardScheme<TBoolValue> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TBoolValue struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUE
-            if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
-              struct.value = iprot.readBool();
-              struct.setValueIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TBoolValue struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.isSetValue()) {
-        oprot.writeFieldBegin(VALUE_FIELD_DESC);
-        oprot.writeBool(struct.value);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TBoolValueTupleSchemeFactory implements SchemeFactory {
-    public TBoolValueTupleScheme getScheme() {
-      return new TBoolValueTupleScheme();
-    }
-  }
-
-  private static class TBoolValueTupleScheme extends TupleScheme<TBoolValue> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TBoolValue struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      BitSet optionals = new BitSet();
-      if (struct.isSetValue()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetValue()) {
-        oprot.writeBool(struct.value);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TBoolValue struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.value = iprot.readBool();
-        struct.setValueIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TByteColumn.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TByteColumn.java
deleted file mode 100644
index 169bfdeab3eea..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TByteColumn.java
+++ /dev/null
@@ -1,548 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TByteColumn implements org.apache.thrift.TBase<TByteColumn, TByteColumn._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TByteColumn");
-
-  private static final org.apache.thrift.protocol.TField VALUES_FIELD_DESC = new org.apache.thrift.protocol.TField("values", org.apache.thrift.protocol.TType.LIST, (short)1);
-  private static final org.apache.thrift.protocol.TField NULLS_FIELD_DESC = new org.apache.thrift.protocol.TField("nulls", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TByteColumnStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TByteColumnTupleSchemeFactory());
-  }
-
-  private List<Byte> values; // required
-  private ByteBuffer nulls; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUES((short)1, "values"),
-    NULLS((short)2, "nulls");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUES
-          return VALUES;
-        case 2: // NULLS
-          return NULLS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUES, new org.apache.thrift.meta_data.FieldMetaData("values", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BYTE))));
-    tmpMap.put(_Fields.NULLS, new org.apache.thrift.meta_data.FieldMetaData("nulls", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TByteColumn.class, metaDataMap);
-  }
-
-  public TByteColumn() {
-  }
-
-  public TByteColumn(
-    List<Byte> values,
-    ByteBuffer nulls)
-  {
-    this();
-    this.values = values;
-    this.nulls = nulls;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TByteColumn(TByteColumn other) {
-    if (other.isSetValues()) {
-      List<Byte> __this__values = new ArrayList<Byte>();
-      for (Byte other_element : other.values) {
-        __this__values.add(other_element);
-      }
-      this.values = __this__values;
-    }
-    if (other.isSetNulls()) {
-      this.nulls = org.apache.thrift.TBaseHelper.copyBinary(other.nulls);
-;
-    }
-  }
-
-  public TByteColumn deepCopy() {
-    return new TByteColumn(this);
-  }
-
-  @Override
-  public void clear() {
-    this.values = null;
-    this.nulls = null;
-  }
-
-  public int getValuesSize() {
-    return (this.values == null) ? 0 : this.values.size();
-  }
-
-  public java.util.Iterator<Byte> getValuesIterator() {
-    return (this.values == null) ? null : this.values.iterator();
-  }
-
-  public void addToValues(byte elem) {
-    if (this.values == null) {
-      this.values = new ArrayList<Byte>();
-    }
-    this.values.add(elem);
-  }
-
-  public List<Byte> getValues() {
-    return this.values;
-  }
-
-  public void setValues(List<Byte> values) {
-    this.values = values;
-  }
-
-  public void unsetValues() {
-    this.values = null;
-  }
-
-  /** Returns true if field values is set (has been assigned a value) and false otherwise */
-  public boolean isSetValues() {
-    return this.values != null;
-  }
-
-  public void setValuesIsSet(boolean value) {
-    if (!value) {
-      this.values = null;
-    }
-  }
-
-  public byte[] getNulls() {
-    setNulls(org.apache.thrift.TBaseHelper.rightSize(nulls));
-    return nulls == null ? null : nulls.array();
-  }
-
-  public ByteBuffer bufferForNulls() {
-    return nulls;
-  }
-
-  public void setNulls(byte[] nulls) {
-    setNulls(nulls == null ? (ByteBuffer)null : ByteBuffer.wrap(nulls));
-  }
-
-  public void setNulls(ByteBuffer nulls) {
-    this.nulls = nulls;
-  }
-
-  public void unsetNulls() {
-    this.nulls = null;
-  }
-
-  /** Returns true if field nulls is set (has been assigned a value) and false otherwise */
-  public boolean isSetNulls() {
-    return this.nulls != null;
-  }
-
-  public void setNullsIsSet(boolean value) {
-    if (!value) {
-      this.nulls = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUES:
-      if (value == null) {
-        unsetValues();
-      } else {
-        setValues((List<Byte>)value);
-      }
-      break;
-
-    case NULLS:
-      if (value == null) {
-        unsetNulls();
-      } else {
-        setNulls((ByteBuffer)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUES:
-      return getValues();
-
-    case NULLS:
-      return getNulls();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUES:
-      return isSetValues();
-    case NULLS:
-      return isSetNulls();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TByteColumn)
-      return this.equals((TByteColumn)that);
-    return false;
-  }
-
-  public boolean equals(TByteColumn that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_values = true && this.isSetValues();
-    boolean that_present_values = true && that.isSetValues();
-    if (this_present_values || that_present_values) {
-      if (!(this_present_values && that_present_values))
-        return false;
-      if (!this.values.equals(that.values))
-        return false;
-    }
-
-    boolean this_present_nulls = true && this.isSetNulls();
-    boolean that_present_nulls = true && that.isSetNulls();
-    if (this_present_nulls || that_present_nulls) {
-      if (!(this_present_nulls && that_present_nulls))
-        return false;
-      if (!this.nulls.equals(that.nulls))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_values = true && (isSetValues());
-    builder.append(present_values);
-    if (present_values)
-      builder.append(values);
-
-    boolean present_nulls = true && (isSetNulls());
-    builder.append(present_nulls);
-    if (present_nulls)
-      builder.append(nulls);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TByteColumn other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TByteColumn typedOther = (TByteColumn)other;
-
-    lastComparison = Boolean.valueOf(isSetValues()).compareTo(typedOther.isSetValues());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValues()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.values, typedOther.values);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetNulls()).compareTo(typedOther.isSetNulls());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetNulls()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.nulls, typedOther.nulls);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TByteColumn(");
-    boolean first = true;
-
-    sb.append("values:");
-    if (this.values == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.values);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("nulls:");
-    if (this.nulls == null) {
-      sb.append("null");
-    } else {
-      org.apache.thrift.TBaseHelper.toString(this.nulls, sb);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetValues()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'values' is unset! Struct:" + toString());
-    }
-
-    if (!isSetNulls()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'nulls' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TByteColumnStandardSchemeFactory implements SchemeFactory {
-    public TByteColumnStandardScheme getScheme() {
-      return new TByteColumnStandardScheme();
-    }
-  }
-
-  private static class TByteColumnStandardScheme extends StandardScheme<TByteColumn> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TByteColumn struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUES
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list62 = iprot.readListBegin();
-                struct.values = new ArrayList<Byte>(_list62.size);
-                for (int _i63 = 0; _i63 < _list62.size; ++_i63)
-                {
-                  byte _elem64; // optional
-                  _elem64 = iprot.readByte();
-                  struct.values.add(_elem64);
-                }
-                iprot.readListEnd();
-              }
-              struct.setValuesIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // NULLS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.nulls = iprot.readBinary();
-              struct.setNullsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TByteColumn struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.values != null) {
-        oprot.writeFieldBegin(VALUES_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.BYTE, struct.values.size()));
-          for (byte _iter65 : struct.values)
-          {
-            oprot.writeByte(_iter65);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      if (struct.nulls != null) {
-        oprot.writeFieldBegin(NULLS_FIELD_DESC);
-        oprot.writeBinary(struct.nulls);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TByteColumnTupleSchemeFactory implements SchemeFactory {
-    public TByteColumnTupleScheme getScheme() {
-      return new TByteColumnTupleScheme();
-    }
-  }
-
-  private static class TByteColumnTupleScheme extends TupleScheme<TByteColumn> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TByteColumn struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.values.size());
-        for (byte _iter66 : struct.values)
-        {
-          oprot.writeByte(_iter66);
-        }
-      }
-      oprot.writeBinary(struct.nulls);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TByteColumn struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TList _list67 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.BYTE, iprot.readI32());
-        struct.values = new ArrayList<Byte>(_list67.size);
-        for (int _i68 = 0; _i68 < _list67.size; ++_i68)
-        {
-          byte _elem69; // optional
-          _elem69 = iprot.readByte();
-          struct.values.add(_elem69);
-        }
-      }
-      struct.setValuesIsSet(true);
-      struct.nulls = iprot.readBinary();
-      struct.setNullsIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TByteValue.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TByteValue.java
deleted file mode 100644
index 23d9693759968..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TByteValue.java
+++ /dev/null
@@ -1,386 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TByteValue implements org.apache.thrift.TBase<TByteValue, TByteValue._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TByteValue");
-
-  private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.BYTE, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TByteValueStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TByteValueTupleSchemeFactory());
-  }
-
-  private byte value; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUE((short)1, "value");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUE
-          return VALUE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __VALUE_ISSET_ID = 0;
-  private byte __isset_bitfield = 0;
-  private _Fields optionals[] = {_Fields.VALUE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BYTE)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TByteValue.class, metaDataMap);
-  }
-
-  public TByteValue() {
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TByteValue(TByteValue other) {
-    __isset_bitfield = other.__isset_bitfield;
-    this.value = other.value;
-  }
-
-  public TByteValue deepCopy() {
-    return new TByteValue(this);
-  }
-
-  @Override
-  public void clear() {
-    setValueIsSet(false);
-    this.value = 0;
-  }
-
-  public byte getValue() {
-    return this.value;
-  }
-
-  public void setValue(byte value) {
-    this.value = value;
-    setValueIsSet(true);
-  }
-
-  public void unsetValue() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __VALUE_ISSET_ID);
-  }
-
-  /** Returns true if field value is set (has been assigned a value) and false otherwise */
-  public boolean isSetValue() {
-    return EncodingUtils.testBit(__isset_bitfield, __VALUE_ISSET_ID);
-  }
-
-  public void setValueIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __VALUE_ISSET_ID, value);
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUE:
-      if (value == null) {
-        unsetValue();
-      } else {
-        setValue((Byte)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUE:
-      return Byte.valueOf(getValue());
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUE:
-      return isSetValue();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TByteValue)
-      return this.equals((TByteValue)that);
-    return false;
-  }
-
-  public boolean equals(TByteValue that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_value = true && this.isSetValue();
-    boolean that_present_value = true && that.isSetValue();
-    if (this_present_value || that_present_value) {
-      if (!(this_present_value && that_present_value))
-        return false;
-      if (this.value != that.value)
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_value = true && (isSetValue());
-    builder.append(present_value);
-    if (present_value)
-      builder.append(value);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TByteValue other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TByteValue typedOther = (TByteValue)other;
-
-    lastComparison = Boolean.valueOf(isSetValue()).compareTo(typedOther.isSetValue());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValue()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, typedOther.value);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TByteValue(");
-    boolean first = true;
-
-    if (isSetValue()) {
-      sb.append("value:");
-      sb.append(this.value);
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TByteValueStandardSchemeFactory implements SchemeFactory {
-    public TByteValueStandardScheme getScheme() {
-      return new TByteValueStandardScheme();
-    }
-  }
-
-  private static class TByteValueStandardScheme extends StandardScheme<TByteValue> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TByteValue struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUE
-            if (schemeField.type == org.apache.thrift.protocol.TType.BYTE) {
-              struct.value = iprot.readByte();
-              struct.setValueIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TByteValue struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.isSetValue()) {
-        oprot.writeFieldBegin(VALUE_FIELD_DESC);
-        oprot.writeByte(struct.value);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TByteValueTupleSchemeFactory implements SchemeFactory {
-    public TByteValueTupleScheme getScheme() {
-      return new TByteValueTupleScheme();
-    }
-  }
-
-  private static class TByteValueTupleScheme extends TupleScheme<TByteValue> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TByteValue struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      BitSet optionals = new BitSet();
-      if (struct.isSetValue()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetValue()) {
-        oprot.writeByte(struct.value);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TByteValue struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.value = iprot.readByte();
-        struct.setValueIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCLIService.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCLIService.java
deleted file mode 100644
index 54851b8d51317..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCLIService.java
+++ /dev/null
@@ -1,15414 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TCLIService {
-
-  public interface Iface {
-
-    public TOpenSessionResp OpenSession(TOpenSessionReq req) throws org.apache.thrift.TException;
-
-    public TCloseSessionResp CloseSession(TCloseSessionReq req) throws org.apache.thrift.TException;
-
-    public TGetInfoResp GetInfo(TGetInfoReq req) throws org.apache.thrift.TException;
-
-    public TExecuteStatementResp ExecuteStatement(TExecuteStatementReq req) throws org.apache.thrift.TException;
-
-    public TGetTypeInfoResp GetTypeInfo(TGetTypeInfoReq req) throws org.apache.thrift.TException;
-
-    public TGetCatalogsResp GetCatalogs(TGetCatalogsReq req) throws org.apache.thrift.TException;
-
-    public TGetSchemasResp GetSchemas(TGetSchemasReq req) throws org.apache.thrift.TException;
-
-    public TGetTablesResp GetTables(TGetTablesReq req) throws org.apache.thrift.TException;
-
-    public TGetTableTypesResp GetTableTypes(TGetTableTypesReq req) throws org.apache.thrift.TException;
-
-    public TGetColumnsResp GetColumns(TGetColumnsReq req) throws org.apache.thrift.TException;
-
-    public TGetFunctionsResp GetFunctions(TGetFunctionsReq req) throws org.apache.thrift.TException;
-
-    public TGetOperationStatusResp GetOperationStatus(TGetOperationStatusReq req) throws org.apache.thrift.TException;
-
-    public TCancelOperationResp CancelOperation(TCancelOperationReq req) throws org.apache.thrift.TException;
-
-    public TCloseOperationResp CloseOperation(TCloseOperationReq req) throws org.apache.thrift.TException;
-
-    public TGetResultSetMetadataResp GetResultSetMetadata(TGetResultSetMetadataReq req) throws org.apache.thrift.TException;
-
-    public TFetchResultsResp FetchResults(TFetchResultsReq req) throws org.apache.thrift.TException;
-
-    public TGetDelegationTokenResp GetDelegationToken(TGetDelegationTokenReq req) throws org.apache.thrift.TException;
-
-    public TCancelDelegationTokenResp CancelDelegationToken(TCancelDelegationTokenReq req) throws org.apache.thrift.TException;
-
-    public TRenewDelegationTokenResp RenewDelegationToken(TRenewDelegationTokenReq req) throws org.apache.thrift.TException;
-
-  }
-
-  public interface AsyncIface {
-
-    public void OpenSession(TOpenSessionReq req, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.OpenSession_call> resultHandler) throws org.apache.thrift.TException;
-
-    public void CloseSession(TCloseSessionReq req, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.CloseSession_call> resultHandler) throws org.apache.thrift.TException;
-
-    public void GetInfo(TGetInfoReq req, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.GetInfo_call> resultHandler) throws org.apache.thrift.TException;
-
-    public void ExecuteStatement(TExecuteStatementReq req, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.ExecuteStatement_call> resultHandler) throws org.apache.thrift.TException;
-
-    public void GetTypeInfo(TGetTypeInfoReq req, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.GetTypeInfo_call> resultHandler) throws org.apache.thrift.TException;
-
-    public void GetCatalogs(TGetCatalogsReq req, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.GetCatalogs_call> resultHandler) throws org.apache.thrift.TException;
-
-    public void GetSchemas(TGetSchemasReq req, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.GetSchemas_call> resultHandler) throws org.apache.thrift.TException;
-
-    public void GetTables(TGetTablesReq req, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.GetTables_call> resultHandler) throws org.apache.thrift.TException;
-
-    public void GetTableTypes(TGetTableTypesReq req, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.GetTableTypes_call> resultHandler) throws org.apache.thrift.TException;
-
-    public void GetColumns(TGetColumnsReq req, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.GetColumns_call> resultHandler) throws org.apache.thrift.TException;
-
-    public void GetFunctions(TGetFunctionsReq req, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.GetFunctions_call> resultHandler) throws org.apache.thrift.TException;
-
-    public void GetOperationStatus(TGetOperationStatusReq req, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.GetOperationStatus_call> resultHandler) throws org.apache.thrift.TException;
-
-    public void CancelOperation(TCancelOperationReq req, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.CancelOperation_call> resultHandler) throws org.apache.thrift.TException;
-
-    public void CloseOperation(TCloseOperationReq req, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.CloseOperation_call> resultHandler) throws org.apache.thrift.TException;
-
-    public void GetResultSetMetadata(TGetResultSetMetadataReq req, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.GetResultSetMetadata_call> resultHandler) throws org.apache.thrift.TException;
-
-    public void FetchResults(TFetchResultsReq req, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.FetchResults_call> resultHandler) throws org.apache.thrift.TException;
-
-    public void GetDelegationToken(TGetDelegationTokenReq req, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.GetDelegationToken_call> resultHandler) throws org.apache.thrift.TException;
-
-    public void CancelDelegationToken(TCancelDelegationTokenReq req, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.CancelDelegationToken_call> resultHandler) throws org.apache.thrift.TException;
-
-    public void RenewDelegationToken(TRenewDelegationTokenReq req, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.RenewDelegationToken_call> resultHandler) throws org.apache.thrift.TException;
-
-  }
-
-  public static class Client extends org.apache.thrift.TServiceClient implements Iface {
-    public static class Factory implements org.apache.thrift.TServiceClientFactory<Client> {
-      public Factory() {}
-      public Client getClient(org.apache.thrift.protocol.TProtocol prot) {
-        return new Client(prot);
-      }
-      public Client getClient(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
-        return new Client(iprot, oprot);
-      }
-    }
-
-    public Client(org.apache.thrift.protocol.TProtocol prot)
-    {
-      super(prot, prot);
-    }
-
-    public Client(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
-      super(iprot, oprot);
-    }
-
-    public TOpenSessionResp OpenSession(TOpenSessionReq req) throws org.apache.thrift.TException
-    {
-      send_OpenSession(req);
-      return recv_OpenSession();
-    }
-
-    public void send_OpenSession(TOpenSessionReq req) throws org.apache.thrift.TException
-    {
-      OpenSession_args args = new OpenSession_args();
-      args.setReq(req);
-      sendBase("OpenSession", args);
-    }
-
-    public TOpenSessionResp recv_OpenSession() throws org.apache.thrift.TException
-    {
-      OpenSession_result result = new OpenSession_result();
-      receiveBase(result, "OpenSession");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "OpenSession failed: unknown result");
-    }
-
-    public TCloseSessionResp CloseSession(TCloseSessionReq req) throws org.apache.thrift.TException
-    {
-      send_CloseSession(req);
-      return recv_CloseSession();
-    }
-
-    public void send_CloseSession(TCloseSessionReq req) throws org.apache.thrift.TException
-    {
-      CloseSession_args args = new CloseSession_args();
-      args.setReq(req);
-      sendBase("CloseSession", args);
-    }
-
-    public TCloseSessionResp recv_CloseSession() throws org.apache.thrift.TException
-    {
-      CloseSession_result result = new CloseSession_result();
-      receiveBase(result, "CloseSession");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "CloseSession failed: unknown result");
-    }
-
-    public TGetInfoResp GetInfo(TGetInfoReq req) throws org.apache.thrift.TException
-    {
-      send_GetInfo(req);
-      return recv_GetInfo();
-    }
-
-    public void send_GetInfo(TGetInfoReq req) throws org.apache.thrift.TException
-    {
-      GetInfo_args args = new GetInfo_args();
-      args.setReq(req);
-      sendBase("GetInfo", args);
-    }
-
-    public TGetInfoResp recv_GetInfo() throws org.apache.thrift.TException
-    {
-      GetInfo_result result = new GetInfo_result();
-      receiveBase(result, "GetInfo");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "GetInfo failed: unknown result");
-    }
-
-    public TExecuteStatementResp ExecuteStatement(TExecuteStatementReq req) throws org.apache.thrift.TException
-    {
-      send_ExecuteStatement(req);
-      return recv_ExecuteStatement();
-    }
-
-    public void send_ExecuteStatement(TExecuteStatementReq req) throws org.apache.thrift.TException
-    {
-      ExecuteStatement_args args = new ExecuteStatement_args();
-      args.setReq(req);
-      sendBase("ExecuteStatement", args);
-    }
-
-    public TExecuteStatementResp recv_ExecuteStatement() throws org.apache.thrift.TException
-    {
-      ExecuteStatement_result result = new ExecuteStatement_result();
-      receiveBase(result, "ExecuteStatement");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "ExecuteStatement failed: unknown result");
-    }
-
-    public TGetTypeInfoResp GetTypeInfo(TGetTypeInfoReq req) throws org.apache.thrift.TException
-    {
-      send_GetTypeInfo(req);
-      return recv_GetTypeInfo();
-    }
-
-    public void send_GetTypeInfo(TGetTypeInfoReq req) throws org.apache.thrift.TException
-    {
-      GetTypeInfo_args args = new GetTypeInfo_args();
-      args.setReq(req);
-      sendBase("GetTypeInfo", args);
-    }
-
-    public TGetTypeInfoResp recv_GetTypeInfo() throws org.apache.thrift.TException
-    {
-      GetTypeInfo_result result = new GetTypeInfo_result();
-      receiveBase(result, "GetTypeInfo");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "GetTypeInfo failed: unknown result");
-    }
-
-    public TGetCatalogsResp GetCatalogs(TGetCatalogsReq req) throws org.apache.thrift.TException
-    {
-      send_GetCatalogs(req);
-      return recv_GetCatalogs();
-    }
-
-    public void send_GetCatalogs(TGetCatalogsReq req) throws org.apache.thrift.TException
-    {
-      GetCatalogs_args args = new GetCatalogs_args();
-      args.setReq(req);
-      sendBase("GetCatalogs", args);
-    }
-
-    public TGetCatalogsResp recv_GetCatalogs() throws org.apache.thrift.TException
-    {
-      GetCatalogs_result result = new GetCatalogs_result();
-      receiveBase(result, "GetCatalogs");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "GetCatalogs failed: unknown result");
-    }
-
-    public TGetSchemasResp GetSchemas(TGetSchemasReq req) throws org.apache.thrift.TException
-    {
-      send_GetSchemas(req);
-      return recv_GetSchemas();
-    }
-
-    public void send_GetSchemas(TGetSchemasReq req) throws org.apache.thrift.TException
-    {
-      GetSchemas_args args = new GetSchemas_args();
-      args.setReq(req);
-      sendBase("GetSchemas", args);
-    }
-
-    public TGetSchemasResp recv_GetSchemas() throws org.apache.thrift.TException
-    {
-      GetSchemas_result result = new GetSchemas_result();
-      receiveBase(result, "GetSchemas");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "GetSchemas failed: unknown result");
-    }
-
-    public TGetTablesResp GetTables(TGetTablesReq req) throws org.apache.thrift.TException
-    {
-      send_GetTables(req);
-      return recv_GetTables();
-    }
-
-    public void send_GetTables(TGetTablesReq req) throws org.apache.thrift.TException
-    {
-      GetTables_args args = new GetTables_args();
-      args.setReq(req);
-      sendBase("GetTables", args);
-    }
-
-    public TGetTablesResp recv_GetTables() throws org.apache.thrift.TException
-    {
-      GetTables_result result = new GetTables_result();
-      receiveBase(result, "GetTables");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "GetTables failed: unknown result");
-    }
-
-    public TGetTableTypesResp GetTableTypes(TGetTableTypesReq req) throws org.apache.thrift.TException
-    {
-      send_GetTableTypes(req);
-      return recv_GetTableTypes();
-    }
-
-    public void send_GetTableTypes(TGetTableTypesReq req) throws org.apache.thrift.TException
-    {
-      GetTableTypes_args args = new GetTableTypes_args();
-      args.setReq(req);
-      sendBase("GetTableTypes", args);
-    }
-
-    public TGetTableTypesResp recv_GetTableTypes() throws org.apache.thrift.TException
-    {
-      GetTableTypes_result result = new GetTableTypes_result();
-      receiveBase(result, "GetTableTypes");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "GetTableTypes failed: unknown result");
-    }
-
-    public TGetColumnsResp GetColumns(TGetColumnsReq req) throws org.apache.thrift.TException
-    {
-      send_GetColumns(req);
-      return recv_GetColumns();
-    }
-
-    public void send_GetColumns(TGetColumnsReq req) throws org.apache.thrift.TException
-    {
-      GetColumns_args args = new GetColumns_args();
-      args.setReq(req);
-      sendBase("GetColumns", args);
-    }
-
-    public TGetColumnsResp recv_GetColumns() throws org.apache.thrift.TException
-    {
-      GetColumns_result result = new GetColumns_result();
-      receiveBase(result, "GetColumns");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "GetColumns failed: unknown result");
-    }
-
-    public TGetFunctionsResp GetFunctions(TGetFunctionsReq req) throws org.apache.thrift.TException
-    {
-      send_GetFunctions(req);
-      return recv_GetFunctions();
-    }
-
-    public void send_GetFunctions(TGetFunctionsReq req) throws org.apache.thrift.TException
-    {
-      GetFunctions_args args = new GetFunctions_args();
-      args.setReq(req);
-      sendBase("GetFunctions", args);
-    }
-
-    public TGetFunctionsResp recv_GetFunctions() throws org.apache.thrift.TException
-    {
-      GetFunctions_result result = new GetFunctions_result();
-      receiveBase(result, "GetFunctions");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "GetFunctions failed: unknown result");
-    }
-
-    public TGetOperationStatusResp GetOperationStatus(TGetOperationStatusReq req) throws org.apache.thrift.TException
-    {
-      send_GetOperationStatus(req);
-      return recv_GetOperationStatus();
-    }
-
-    public void send_GetOperationStatus(TGetOperationStatusReq req) throws org.apache.thrift.TException
-    {
-      GetOperationStatus_args args = new GetOperationStatus_args();
-      args.setReq(req);
-      sendBase("GetOperationStatus", args);
-    }
-
-    public TGetOperationStatusResp recv_GetOperationStatus() throws org.apache.thrift.TException
-    {
-      GetOperationStatus_result result = new GetOperationStatus_result();
-      receiveBase(result, "GetOperationStatus");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "GetOperationStatus failed: unknown result");
-    }
-
-    public TCancelOperationResp CancelOperation(TCancelOperationReq req) throws org.apache.thrift.TException
-    {
-      send_CancelOperation(req);
-      return recv_CancelOperation();
-    }
-
-    public void send_CancelOperation(TCancelOperationReq req) throws org.apache.thrift.TException
-    {
-      CancelOperation_args args = new CancelOperation_args();
-      args.setReq(req);
-      sendBase("CancelOperation", args);
-    }
-
-    public TCancelOperationResp recv_CancelOperation() throws org.apache.thrift.TException
-    {
-      CancelOperation_result result = new CancelOperation_result();
-      receiveBase(result, "CancelOperation");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "CancelOperation failed: unknown result");
-    }
-
-    public TCloseOperationResp CloseOperation(TCloseOperationReq req) throws org.apache.thrift.TException
-    {
-      send_CloseOperation(req);
-      return recv_CloseOperation();
-    }
-
-    public void send_CloseOperation(TCloseOperationReq req) throws org.apache.thrift.TException
-    {
-      CloseOperation_args args = new CloseOperation_args();
-      args.setReq(req);
-      sendBase("CloseOperation", args);
-    }
-
-    public TCloseOperationResp recv_CloseOperation() throws org.apache.thrift.TException
-    {
-      CloseOperation_result result = new CloseOperation_result();
-      receiveBase(result, "CloseOperation");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "CloseOperation failed: unknown result");
-    }
-
-    public TGetResultSetMetadataResp GetResultSetMetadata(TGetResultSetMetadataReq req) throws org.apache.thrift.TException
-    {
-      send_GetResultSetMetadata(req);
-      return recv_GetResultSetMetadata();
-    }
-
-    public void send_GetResultSetMetadata(TGetResultSetMetadataReq req) throws org.apache.thrift.TException
-    {
-      GetResultSetMetadata_args args = new GetResultSetMetadata_args();
-      args.setReq(req);
-      sendBase("GetResultSetMetadata", args);
-    }
-
-    public TGetResultSetMetadataResp recv_GetResultSetMetadata() throws org.apache.thrift.TException
-    {
-      GetResultSetMetadata_result result = new GetResultSetMetadata_result();
-      receiveBase(result, "GetResultSetMetadata");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "GetResultSetMetadata failed: unknown result");
-    }
-
-    public TFetchResultsResp FetchResults(TFetchResultsReq req) throws org.apache.thrift.TException
-    {
-      send_FetchResults(req);
-      return recv_FetchResults();
-    }
-
-    public void send_FetchResults(TFetchResultsReq req) throws org.apache.thrift.TException
-    {
-      FetchResults_args args = new FetchResults_args();
-      args.setReq(req);
-      sendBase("FetchResults", args);
-    }
-
-    public TFetchResultsResp recv_FetchResults() throws org.apache.thrift.TException
-    {
-      FetchResults_result result = new FetchResults_result();
-      receiveBase(result, "FetchResults");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "FetchResults failed: unknown result");
-    }
-
-    public TGetDelegationTokenResp GetDelegationToken(TGetDelegationTokenReq req) throws org.apache.thrift.TException
-    {
-      send_GetDelegationToken(req);
-      return recv_GetDelegationToken();
-    }
-
-    public void send_GetDelegationToken(TGetDelegationTokenReq req) throws org.apache.thrift.TException
-    {
-      GetDelegationToken_args args = new GetDelegationToken_args();
-      args.setReq(req);
-      sendBase("GetDelegationToken", args);
-    }
-
-    public TGetDelegationTokenResp recv_GetDelegationToken() throws org.apache.thrift.TException
-    {
-      GetDelegationToken_result result = new GetDelegationToken_result();
-      receiveBase(result, "GetDelegationToken");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "GetDelegationToken failed: unknown result");
-    }
-
-    public TCancelDelegationTokenResp CancelDelegationToken(TCancelDelegationTokenReq req) throws org.apache.thrift.TException
-    {
-      send_CancelDelegationToken(req);
-      return recv_CancelDelegationToken();
-    }
-
-    public void send_CancelDelegationToken(TCancelDelegationTokenReq req) throws org.apache.thrift.TException
-    {
-      CancelDelegationToken_args args = new CancelDelegationToken_args();
-      args.setReq(req);
-      sendBase("CancelDelegationToken", args);
-    }
-
-    public TCancelDelegationTokenResp recv_CancelDelegationToken() throws org.apache.thrift.TException
-    {
-      CancelDelegationToken_result result = new CancelDelegationToken_result();
-      receiveBase(result, "CancelDelegationToken");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "CancelDelegationToken failed: unknown result");
-    }
-
-    public TRenewDelegationTokenResp RenewDelegationToken(TRenewDelegationTokenReq req) throws org.apache.thrift.TException
-    {
-      send_RenewDelegationToken(req);
-      return recv_RenewDelegationToken();
-    }
-
-    public void send_RenewDelegationToken(TRenewDelegationTokenReq req) throws org.apache.thrift.TException
-    {
-      RenewDelegationToken_args args = new RenewDelegationToken_args();
-      args.setReq(req);
-      sendBase("RenewDelegationToken", args);
-    }
-
-    public TRenewDelegationTokenResp recv_RenewDelegationToken() throws org.apache.thrift.TException
-    {
-      RenewDelegationToken_result result = new RenewDelegationToken_result();
-      receiveBase(result, "RenewDelegationToken");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "RenewDelegationToken failed: unknown result");
-    }
-
-  }
-  public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface {
-    public static class Factory implements org.apache.thrift.async.TAsyncClientFactory<AsyncClient> {
-      private org.apache.thrift.async.TAsyncClientManager clientManager;
-      private org.apache.thrift.protocol.TProtocolFactory protocolFactory;
-      public Factory(org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.protocol.TProtocolFactory protocolFactory) {
-        this.clientManager = clientManager;
-        this.protocolFactory = protocolFactory;
-      }
-      public AsyncClient getAsyncClient(org.apache.thrift.transport.TNonblockingTransport transport) {
-        return new AsyncClient(protocolFactory, clientManager, transport);
-      }
-    }
-
-    public AsyncClient(org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.transport.TNonblockingTransport transport) {
-      super(protocolFactory, clientManager, transport);
-    }
-
-    public void OpenSession(TOpenSessionReq req, org.apache.thrift.async.AsyncMethodCallback<OpenSession_call> resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      OpenSession_call method_call = new OpenSession_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class OpenSession_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TOpenSessionReq req;
-      public OpenSession_call(TOpenSessionReq req, org.apache.thrift.async.AsyncMethodCallback<OpenSession_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("OpenSession", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        OpenSession_args args = new OpenSession_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TOpenSessionResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_OpenSession();
-      }
-    }
-
-    public void CloseSession(TCloseSessionReq req, org.apache.thrift.async.AsyncMethodCallback<CloseSession_call> resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      CloseSession_call method_call = new CloseSession_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class CloseSession_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TCloseSessionReq req;
-      public CloseSession_call(TCloseSessionReq req, org.apache.thrift.async.AsyncMethodCallback<CloseSession_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("CloseSession", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        CloseSession_args args = new CloseSession_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TCloseSessionResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_CloseSession();
-      }
-    }
-
-    public void GetInfo(TGetInfoReq req, org.apache.thrift.async.AsyncMethodCallback<GetInfo_call> resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      GetInfo_call method_call = new GetInfo_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class GetInfo_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TGetInfoReq req;
-      public GetInfo_call(TGetInfoReq req, org.apache.thrift.async.AsyncMethodCallback<GetInfo_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("GetInfo", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        GetInfo_args args = new GetInfo_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TGetInfoResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_GetInfo();
-      }
-    }
-
-    public void ExecuteStatement(TExecuteStatementReq req, org.apache.thrift.async.AsyncMethodCallback<ExecuteStatement_call> resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      ExecuteStatement_call method_call = new ExecuteStatement_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class ExecuteStatement_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TExecuteStatementReq req;
-      public ExecuteStatement_call(TExecuteStatementReq req, org.apache.thrift.async.AsyncMethodCallback<ExecuteStatement_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("ExecuteStatement", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        ExecuteStatement_args args = new ExecuteStatement_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TExecuteStatementResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_ExecuteStatement();
-      }
-    }
-
-    public void GetTypeInfo(TGetTypeInfoReq req, org.apache.thrift.async.AsyncMethodCallback<GetTypeInfo_call> resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      GetTypeInfo_call method_call = new GetTypeInfo_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class GetTypeInfo_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TGetTypeInfoReq req;
-      public GetTypeInfo_call(TGetTypeInfoReq req, org.apache.thrift.async.AsyncMethodCallback<GetTypeInfo_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("GetTypeInfo", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        GetTypeInfo_args args = new GetTypeInfo_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TGetTypeInfoResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_GetTypeInfo();
-      }
-    }
-
-    public void GetCatalogs(TGetCatalogsReq req, org.apache.thrift.async.AsyncMethodCallback<GetCatalogs_call> resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      GetCatalogs_call method_call = new GetCatalogs_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class GetCatalogs_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TGetCatalogsReq req;
-      public GetCatalogs_call(TGetCatalogsReq req, org.apache.thrift.async.AsyncMethodCallback<GetCatalogs_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("GetCatalogs", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        GetCatalogs_args args = new GetCatalogs_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TGetCatalogsResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_GetCatalogs();
-      }
-    }
-
-    public void GetSchemas(TGetSchemasReq req, org.apache.thrift.async.AsyncMethodCallback<GetSchemas_call> resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      GetSchemas_call method_call = new GetSchemas_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class GetSchemas_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TGetSchemasReq req;
-      public GetSchemas_call(TGetSchemasReq req, org.apache.thrift.async.AsyncMethodCallback<GetSchemas_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("GetSchemas", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        GetSchemas_args args = new GetSchemas_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TGetSchemasResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_GetSchemas();
-      }
-    }
-
-    public void GetTables(TGetTablesReq req, org.apache.thrift.async.AsyncMethodCallback<GetTables_call> resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      GetTables_call method_call = new GetTables_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class GetTables_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TGetTablesReq req;
-      public GetTables_call(TGetTablesReq req, org.apache.thrift.async.AsyncMethodCallback<GetTables_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("GetTables", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        GetTables_args args = new GetTables_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TGetTablesResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_GetTables();
-      }
-    }
-
-    public void GetTableTypes(TGetTableTypesReq req, org.apache.thrift.async.AsyncMethodCallback<GetTableTypes_call> resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      GetTableTypes_call method_call = new GetTableTypes_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class GetTableTypes_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TGetTableTypesReq req;
-      public GetTableTypes_call(TGetTableTypesReq req, org.apache.thrift.async.AsyncMethodCallback<GetTableTypes_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("GetTableTypes", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        GetTableTypes_args args = new GetTableTypes_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TGetTableTypesResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_GetTableTypes();
-      }
-    }
-
-    public void GetColumns(TGetColumnsReq req, org.apache.thrift.async.AsyncMethodCallback<GetColumns_call> resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      GetColumns_call method_call = new GetColumns_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class GetColumns_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TGetColumnsReq req;
-      public GetColumns_call(TGetColumnsReq req, org.apache.thrift.async.AsyncMethodCallback<GetColumns_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("GetColumns", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        GetColumns_args args = new GetColumns_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TGetColumnsResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_GetColumns();
-      }
-    }
-
-    public void GetFunctions(TGetFunctionsReq req, org.apache.thrift.async.AsyncMethodCallback<GetFunctions_call> resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      GetFunctions_call method_call = new GetFunctions_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class GetFunctions_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TGetFunctionsReq req;
-      public GetFunctions_call(TGetFunctionsReq req, org.apache.thrift.async.AsyncMethodCallback<GetFunctions_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("GetFunctions", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        GetFunctions_args args = new GetFunctions_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TGetFunctionsResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_GetFunctions();
-      }
-    }
-
-    public void GetOperationStatus(TGetOperationStatusReq req, org.apache.thrift.async.AsyncMethodCallback<GetOperationStatus_call> resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      GetOperationStatus_call method_call = new GetOperationStatus_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class GetOperationStatus_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TGetOperationStatusReq req;
-      public GetOperationStatus_call(TGetOperationStatusReq req, org.apache.thrift.async.AsyncMethodCallback<GetOperationStatus_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("GetOperationStatus", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        GetOperationStatus_args args = new GetOperationStatus_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TGetOperationStatusResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_GetOperationStatus();
-      }
-    }
-
-    public void CancelOperation(TCancelOperationReq req, org.apache.thrift.async.AsyncMethodCallback<CancelOperation_call> resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      CancelOperation_call method_call = new CancelOperation_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class CancelOperation_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TCancelOperationReq req;
-      public CancelOperation_call(TCancelOperationReq req, org.apache.thrift.async.AsyncMethodCallback<CancelOperation_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("CancelOperation", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        CancelOperation_args args = new CancelOperation_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TCancelOperationResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_CancelOperation();
-      }
-    }
-
-    public void CloseOperation(TCloseOperationReq req, org.apache.thrift.async.AsyncMethodCallback<CloseOperation_call> resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      CloseOperation_call method_call = new CloseOperation_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class CloseOperation_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TCloseOperationReq req;
-      public CloseOperation_call(TCloseOperationReq req, org.apache.thrift.async.AsyncMethodCallback<CloseOperation_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("CloseOperation", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        CloseOperation_args args = new CloseOperation_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TCloseOperationResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_CloseOperation();
-      }
-    }
-
-    public void GetResultSetMetadata(TGetResultSetMetadataReq req, org.apache.thrift.async.AsyncMethodCallback<GetResultSetMetadata_call> resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      GetResultSetMetadata_call method_call = new GetResultSetMetadata_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class GetResultSetMetadata_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TGetResultSetMetadataReq req;
-      public GetResultSetMetadata_call(TGetResultSetMetadataReq req, org.apache.thrift.async.AsyncMethodCallback<GetResultSetMetadata_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("GetResultSetMetadata", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        GetResultSetMetadata_args args = new GetResultSetMetadata_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TGetResultSetMetadataResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_GetResultSetMetadata();
-      }
-    }
-
-    public void FetchResults(TFetchResultsReq req, org.apache.thrift.async.AsyncMethodCallback<FetchResults_call> resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      FetchResults_call method_call = new FetchResults_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class FetchResults_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TFetchResultsReq req;
-      public FetchResults_call(TFetchResultsReq req, org.apache.thrift.async.AsyncMethodCallback<FetchResults_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("FetchResults", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        FetchResults_args args = new FetchResults_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TFetchResultsResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_FetchResults();
-      }
-    }
-
-    public void GetDelegationToken(TGetDelegationTokenReq req, org.apache.thrift.async.AsyncMethodCallback<GetDelegationToken_call> resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      GetDelegationToken_call method_call = new GetDelegationToken_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class GetDelegationToken_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TGetDelegationTokenReq req;
-      public GetDelegationToken_call(TGetDelegationTokenReq req, org.apache.thrift.async.AsyncMethodCallback<GetDelegationToken_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("GetDelegationToken", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        GetDelegationToken_args args = new GetDelegationToken_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TGetDelegationTokenResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_GetDelegationToken();
-      }
-    }
-
-    public void CancelDelegationToken(TCancelDelegationTokenReq req, org.apache.thrift.async.AsyncMethodCallback<CancelDelegationToken_call> resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      CancelDelegationToken_call method_call = new CancelDelegationToken_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class CancelDelegationToken_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TCancelDelegationTokenReq req;
-      public CancelDelegationToken_call(TCancelDelegationTokenReq req, org.apache.thrift.async.AsyncMethodCallback<CancelDelegationToken_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("CancelDelegationToken", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        CancelDelegationToken_args args = new CancelDelegationToken_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TCancelDelegationTokenResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_CancelDelegationToken();
-      }
-    }
-
-    public void RenewDelegationToken(TRenewDelegationTokenReq req, org.apache.thrift.async.AsyncMethodCallback<RenewDelegationToken_call> resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      RenewDelegationToken_call method_call = new RenewDelegationToken_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class RenewDelegationToken_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TRenewDelegationTokenReq req;
-      public RenewDelegationToken_call(TRenewDelegationTokenReq req, org.apache.thrift.async.AsyncMethodCallback<RenewDelegationToken_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("RenewDelegationToken", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        RenewDelegationToken_args args = new RenewDelegationToken_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TRenewDelegationTokenResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_RenewDelegationToken();
-      }
-    }
-
-  }
-
-  public static class Processor<I extends Iface> extends org.apache.thrift.TBaseProcessor<I> implements org.apache.thrift.TProcessor {
-    private static final Logger LOGGER = LoggerFactory.getLogger(Processor.class.getName());
-    public Processor(I iface) {
-      super(iface, getProcessMap(new HashMap<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>>()));
-    }
-
-    protected Processor(I iface, Map<String,  org.apache.thrift.ProcessFunction<I, ? extends  org.apache.thrift.TBase>> processMap) {
-      super(iface, getProcessMap(processMap));
-    }
-
-    private static <I extends Iface> Map<String,  org.apache.thrift.ProcessFunction<I, ? extends  org.apache.thrift.TBase>> getProcessMap(Map<String,  org.apache.thrift.ProcessFunction<I, ? extends  org.apache.thrift.TBase>> processMap) {
-      processMap.put("OpenSession", new OpenSession());
-      processMap.put("CloseSession", new CloseSession());
-      processMap.put("GetInfo", new GetInfo());
-      processMap.put("ExecuteStatement", new ExecuteStatement());
-      processMap.put("GetTypeInfo", new GetTypeInfo());
-      processMap.put("GetCatalogs", new GetCatalogs());
-      processMap.put("GetSchemas", new GetSchemas());
-      processMap.put("GetTables", new GetTables());
-      processMap.put("GetTableTypes", new GetTableTypes());
-      processMap.put("GetColumns", new GetColumns());
-      processMap.put("GetFunctions", new GetFunctions());
-      processMap.put("GetOperationStatus", new GetOperationStatus());
-      processMap.put("CancelOperation", new CancelOperation());
-      processMap.put("CloseOperation", new CloseOperation());
-      processMap.put("GetResultSetMetadata", new GetResultSetMetadata());
-      processMap.put("FetchResults", new FetchResults());
-      processMap.put("GetDelegationToken", new GetDelegationToken());
-      processMap.put("CancelDelegationToken", new CancelDelegationToken());
-      processMap.put("RenewDelegationToken", new RenewDelegationToken());
-      return processMap;
-    }
-
-    public static class OpenSession<I extends Iface> extends org.apache.thrift.ProcessFunction<I, OpenSession_args> {
-      public OpenSession() {
-        super("OpenSession");
-      }
-
-      public OpenSession_args getEmptyArgsInstance() {
-        return new OpenSession_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public OpenSession_result getResult(I iface, OpenSession_args args) throws org.apache.thrift.TException {
-        OpenSession_result result = new OpenSession_result();
-        result.success = iface.OpenSession(args.req);
-        return result;
-      }
-    }
-
-    public static class CloseSession<I extends Iface> extends org.apache.thrift.ProcessFunction<I, CloseSession_args> {
-      public CloseSession() {
-        super("CloseSession");
-      }
-
-      public CloseSession_args getEmptyArgsInstance() {
-        return new CloseSession_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public CloseSession_result getResult(I iface, CloseSession_args args) throws org.apache.thrift.TException {
-        CloseSession_result result = new CloseSession_result();
-        result.success = iface.CloseSession(args.req);
-        return result;
-      }
-    }
-
-    public static class GetInfo<I extends Iface> extends org.apache.thrift.ProcessFunction<I, GetInfo_args> {
-      public GetInfo() {
-        super("GetInfo");
-      }
-
-      public GetInfo_args getEmptyArgsInstance() {
-        return new GetInfo_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public GetInfo_result getResult(I iface, GetInfo_args args) throws org.apache.thrift.TException {
-        GetInfo_result result = new GetInfo_result();
-        result.success = iface.GetInfo(args.req);
-        return result;
-      }
-    }
-
-    public static class ExecuteStatement<I extends Iface> extends org.apache.thrift.ProcessFunction<I, ExecuteStatement_args> {
-      public ExecuteStatement() {
-        super("ExecuteStatement");
-      }
-
-      public ExecuteStatement_args getEmptyArgsInstance() {
-        return new ExecuteStatement_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public ExecuteStatement_result getResult(I iface, ExecuteStatement_args args) throws org.apache.thrift.TException {
-        ExecuteStatement_result result = new ExecuteStatement_result();
-        result.success = iface.ExecuteStatement(args.req);
-        return result;
-      }
-    }
-
-    public static class GetTypeInfo<I extends Iface> extends org.apache.thrift.ProcessFunction<I, GetTypeInfo_args> {
-      public GetTypeInfo() {
-        super("GetTypeInfo");
-      }
-
-      public GetTypeInfo_args getEmptyArgsInstance() {
-        return new GetTypeInfo_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public GetTypeInfo_result getResult(I iface, GetTypeInfo_args args) throws org.apache.thrift.TException {
-        GetTypeInfo_result result = new GetTypeInfo_result();
-        result.success = iface.GetTypeInfo(args.req);
-        return result;
-      }
-    }
-
-    public static class GetCatalogs<I extends Iface> extends org.apache.thrift.ProcessFunction<I, GetCatalogs_args> {
-      public GetCatalogs() {
-        super("GetCatalogs");
-      }
-
-      public GetCatalogs_args getEmptyArgsInstance() {
-        return new GetCatalogs_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public GetCatalogs_result getResult(I iface, GetCatalogs_args args) throws org.apache.thrift.TException {
-        GetCatalogs_result result = new GetCatalogs_result();
-        result.success = iface.GetCatalogs(args.req);
-        return result;
-      }
-    }
-
-    public static class GetSchemas<I extends Iface> extends org.apache.thrift.ProcessFunction<I, GetSchemas_args> {
-      public GetSchemas() {
-        super("GetSchemas");
-      }
-
-      public GetSchemas_args getEmptyArgsInstance() {
-        return new GetSchemas_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public GetSchemas_result getResult(I iface, GetSchemas_args args) throws org.apache.thrift.TException {
-        GetSchemas_result result = new GetSchemas_result();
-        result.success = iface.GetSchemas(args.req);
-        return result;
-      }
-    }
-
-    public static class GetTables<I extends Iface> extends org.apache.thrift.ProcessFunction<I, GetTables_args> {
-      public GetTables() {
-        super("GetTables");
-      }
-
-      public GetTables_args getEmptyArgsInstance() {
-        return new GetTables_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public GetTables_result getResult(I iface, GetTables_args args) throws org.apache.thrift.TException {
-        GetTables_result result = new GetTables_result();
-        result.success = iface.GetTables(args.req);
-        return result;
-      }
-    }
-
-    public static class GetTableTypes<I extends Iface> extends org.apache.thrift.ProcessFunction<I, GetTableTypes_args> {
-      public GetTableTypes() {
-        super("GetTableTypes");
-      }
-
-      public GetTableTypes_args getEmptyArgsInstance() {
-        return new GetTableTypes_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public GetTableTypes_result getResult(I iface, GetTableTypes_args args) throws org.apache.thrift.TException {
-        GetTableTypes_result result = new GetTableTypes_result();
-        result.success = iface.GetTableTypes(args.req);
-        return result;
-      }
-    }
-
-    public static class GetColumns<I extends Iface> extends org.apache.thrift.ProcessFunction<I, GetColumns_args> {
-      public GetColumns() {
-        super("GetColumns");
-      }
-
-      public GetColumns_args getEmptyArgsInstance() {
-        return new GetColumns_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public GetColumns_result getResult(I iface, GetColumns_args args) throws org.apache.thrift.TException {
-        GetColumns_result result = new GetColumns_result();
-        result.success = iface.GetColumns(args.req);
-        return result;
-      }
-    }
-
-    public static class GetFunctions<I extends Iface> extends org.apache.thrift.ProcessFunction<I, GetFunctions_args> {
-      public GetFunctions() {
-        super("GetFunctions");
-      }
-
-      public GetFunctions_args getEmptyArgsInstance() {
-        return new GetFunctions_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public GetFunctions_result getResult(I iface, GetFunctions_args args) throws org.apache.thrift.TException {
-        GetFunctions_result result = new GetFunctions_result();
-        result.success = iface.GetFunctions(args.req);
-        return result;
-      }
-    }
-
-    public static class GetOperationStatus<I extends Iface> extends org.apache.thrift.ProcessFunction<I, GetOperationStatus_args> {
-      public GetOperationStatus() {
-        super("GetOperationStatus");
-      }
-
-      public GetOperationStatus_args getEmptyArgsInstance() {
-        return new GetOperationStatus_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public GetOperationStatus_result getResult(I iface, GetOperationStatus_args args) throws org.apache.thrift.TException {
-        GetOperationStatus_result result = new GetOperationStatus_result();
-        result.success = iface.GetOperationStatus(args.req);
-        return result;
-      }
-    }
-
-    public static class CancelOperation<I extends Iface> extends org.apache.thrift.ProcessFunction<I, CancelOperation_args> {
-      public CancelOperation() {
-        super("CancelOperation");
-      }
-
-      public CancelOperation_args getEmptyArgsInstance() {
-        return new CancelOperation_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public CancelOperation_result getResult(I iface, CancelOperation_args args) throws org.apache.thrift.TException {
-        CancelOperation_result result = new CancelOperation_result();
-        result.success = iface.CancelOperation(args.req);
-        return result;
-      }
-    }
-
-    public static class CloseOperation<I extends Iface> extends org.apache.thrift.ProcessFunction<I, CloseOperation_args> {
-      public CloseOperation() {
-        super("CloseOperation");
-      }
-
-      public CloseOperation_args getEmptyArgsInstance() {
-        return new CloseOperation_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public CloseOperation_result getResult(I iface, CloseOperation_args args) throws org.apache.thrift.TException {
-        CloseOperation_result result = new CloseOperation_result();
-        result.success = iface.CloseOperation(args.req);
-        return result;
-      }
-    }
-
-    public static class GetResultSetMetadata<I extends Iface> extends org.apache.thrift.ProcessFunction<I, GetResultSetMetadata_args> {
-      public GetResultSetMetadata() {
-        super("GetResultSetMetadata");
-      }
-
-      public GetResultSetMetadata_args getEmptyArgsInstance() {
-        return new GetResultSetMetadata_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public GetResultSetMetadata_result getResult(I iface, GetResultSetMetadata_args args) throws org.apache.thrift.TException {
-        GetResultSetMetadata_result result = new GetResultSetMetadata_result();
-        result.success = iface.GetResultSetMetadata(args.req);
-        return result;
-      }
-    }
-
-    public static class FetchResults<I extends Iface> extends org.apache.thrift.ProcessFunction<I, FetchResults_args> {
-      public FetchResults() {
-        super("FetchResults");
-      }
-
-      public FetchResults_args getEmptyArgsInstance() {
-        return new FetchResults_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public FetchResults_result getResult(I iface, FetchResults_args args) throws org.apache.thrift.TException {
-        FetchResults_result result = new FetchResults_result();
-        result.success = iface.FetchResults(args.req);
-        return result;
-      }
-    }
-
-    public static class GetDelegationToken<I extends Iface> extends org.apache.thrift.ProcessFunction<I, GetDelegationToken_args> {
-      public GetDelegationToken() {
-        super("GetDelegationToken");
-      }
-
-      public GetDelegationToken_args getEmptyArgsInstance() {
-        return new GetDelegationToken_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public GetDelegationToken_result getResult(I iface, GetDelegationToken_args args) throws org.apache.thrift.TException {
-        GetDelegationToken_result result = new GetDelegationToken_result();
-        result.success = iface.GetDelegationToken(args.req);
-        return result;
-      }
-    }
-
-    public static class CancelDelegationToken<I extends Iface> extends org.apache.thrift.ProcessFunction<I, CancelDelegationToken_args> {
-      public CancelDelegationToken() {
-        super("CancelDelegationToken");
-      }
-
-      public CancelDelegationToken_args getEmptyArgsInstance() {
-        return new CancelDelegationToken_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public CancelDelegationToken_result getResult(I iface, CancelDelegationToken_args args) throws org.apache.thrift.TException {
-        CancelDelegationToken_result result = new CancelDelegationToken_result();
-        result.success = iface.CancelDelegationToken(args.req);
-        return result;
-      }
-    }
-
-    public static class RenewDelegationToken<I extends Iface> extends org.apache.thrift.ProcessFunction<I, RenewDelegationToken_args> {
-      public RenewDelegationToken() {
-        super("RenewDelegationToken");
-      }
-
-      public RenewDelegationToken_args getEmptyArgsInstance() {
-        return new RenewDelegationToken_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public RenewDelegationToken_result getResult(I iface, RenewDelegationToken_args args) throws org.apache.thrift.TException {
-        RenewDelegationToken_result result = new RenewDelegationToken_result();
-        result.success = iface.RenewDelegationToken(args.req);
-        return result;
-      }
-    }
-
-  }
-
-  public static class OpenSession_args implements org.apache.thrift.TBase<OpenSession_args, OpenSession_args._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("OpenSession_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new OpenSession_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new OpenSession_argsTupleSchemeFactory());
-    }
-
-    private TOpenSessionReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOpenSessionReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(OpenSession_args.class, metaDataMap);
-    }
-
-    public OpenSession_args() {
-    }
-
-    public OpenSession_args(
-      TOpenSessionReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public OpenSession_args(OpenSession_args other) {
-      if (other.isSetReq()) {
-        this.req = new TOpenSessionReq(other.req);
-      }
-    }
-
-    public OpenSession_args deepCopy() {
-      return new OpenSession_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TOpenSessionReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TOpenSessionReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TOpenSessionReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof OpenSession_args)
-        return this.equals((OpenSession_args)that);
-      return false;
-    }
-
-    public boolean equals(OpenSession_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_req = true && (isSetReq());
-      builder.append(present_req);
-      if (present_req)
-        builder.append(req);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(OpenSession_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      OpenSession_args typedOther = (OpenSession_args)other;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(typedOther.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, typedOther.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("OpenSession_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class OpenSession_argsStandardSchemeFactory implements SchemeFactory {
-      public OpenSession_argsStandardScheme getScheme() {
-        return new OpenSession_argsStandardScheme();
-      }
-    }
-
-    private static class OpenSession_argsStandardScheme extends StandardScheme<OpenSession_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, OpenSession_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TOpenSessionReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, OpenSession_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class OpenSession_argsTupleSchemeFactory implements SchemeFactory {
-      public OpenSession_argsTupleScheme getScheme() {
-        return new OpenSession_argsTupleScheme();
-      }
-    }
-
-    private static class OpenSession_argsTupleScheme extends TupleScheme<OpenSession_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, OpenSession_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, OpenSession_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TOpenSessionReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class OpenSession_result implements org.apache.thrift.TBase<OpenSession_result, OpenSession_result._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("OpenSession_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new OpenSession_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new OpenSession_resultTupleSchemeFactory());
-    }
-
-    private TOpenSessionResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOpenSessionResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(OpenSession_result.class, metaDataMap);
-    }
-
-    public OpenSession_result() {
-    }
-
-    public OpenSession_result(
-      TOpenSessionResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public OpenSession_result(OpenSession_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TOpenSessionResp(other.success);
-      }
-    }
-
-    public OpenSession_result deepCopy() {
-      return new OpenSession_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TOpenSessionResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TOpenSessionResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TOpenSessionResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof OpenSession_result)
-        return this.equals((OpenSession_result)that);
-      return false;
-    }
-
-    public boolean equals(OpenSession_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_success = true && (isSetSuccess());
-      builder.append(present_success);
-      if (present_success)
-        builder.append(success);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(OpenSession_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      OpenSession_result typedOther = (OpenSession_result)other;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("OpenSession_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class OpenSession_resultStandardSchemeFactory implements SchemeFactory {
-      public OpenSession_resultStandardScheme getScheme() {
-        return new OpenSession_resultStandardScheme();
-      }
-    }
-
-    private static class OpenSession_resultStandardScheme extends StandardScheme<OpenSession_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, OpenSession_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TOpenSessionResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, OpenSession_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class OpenSession_resultTupleSchemeFactory implements SchemeFactory {
-      public OpenSession_resultTupleScheme getScheme() {
-        return new OpenSession_resultTupleScheme();
-      }
-    }
-
-    private static class OpenSession_resultTupleScheme extends TupleScheme<OpenSession_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, OpenSession_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, OpenSession_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TOpenSessionResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class CloseSession_args implements org.apache.thrift.TBase<CloseSession_args, CloseSession_args._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CloseSession_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new CloseSession_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new CloseSession_argsTupleSchemeFactory());
-    }
-
-    private TCloseSessionReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TCloseSessionReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CloseSession_args.class, metaDataMap);
-    }
-
-    public CloseSession_args() {
-    }
-
-    public CloseSession_args(
-      TCloseSessionReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public CloseSession_args(CloseSession_args other) {
-      if (other.isSetReq()) {
-        this.req = new TCloseSessionReq(other.req);
-      }
-    }
-
-    public CloseSession_args deepCopy() {
-      return new CloseSession_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TCloseSessionReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TCloseSessionReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TCloseSessionReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof CloseSession_args)
-        return this.equals((CloseSession_args)that);
-      return false;
-    }
-
-    public boolean equals(CloseSession_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_req = true && (isSetReq());
-      builder.append(present_req);
-      if (present_req)
-        builder.append(req);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(CloseSession_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      CloseSession_args typedOther = (CloseSession_args)other;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(typedOther.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, typedOther.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("CloseSession_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class CloseSession_argsStandardSchemeFactory implements SchemeFactory {
-      public CloseSession_argsStandardScheme getScheme() {
-        return new CloseSession_argsStandardScheme();
-      }
-    }
-
-    private static class CloseSession_argsStandardScheme extends StandardScheme<CloseSession_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, CloseSession_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TCloseSessionReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, CloseSession_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class CloseSession_argsTupleSchemeFactory implements SchemeFactory {
-      public CloseSession_argsTupleScheme getScheme() {
-        return new CloseSession_argsTupleScheme();
-      }
-    }
-
-    private static class CloseSession_argsTupleScheme extends TupleScheme<CloseSession_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, CloseSession_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, CloseSession_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TCloseSessionReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class CloseSession_result implements org.apache.thrift.TBase<CloseSession_result, CloseSession_result._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CloseSession_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new CloseSession_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new CloseSession_resultTupleSchemeFactory());
-    }
-
-    private TCloseSessionResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TCloseSessionResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CloseSession_result.class, metaDataMap);
-    }
-
-    public CloseSession_result() {
-    }
-
-    public CloseSession_result(
-      TCloseSessionResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public CloseSession_result(CloseSession_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TCloseSessionResp(other.success);
-      }
-    }
-
-    public CloseSession_result deepCopy() {
-      return new CloseSession_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TCloseSessionResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TCloseSessionResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TCloseSessionResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof CloseSession_result)
-        return this.equals((CloseSession_result)that);
-      return false;
-    }
-
-    public boolean equals(CloseSession_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_success = true && (isSetSuccess());
-      builder.append(present_success);
-      if (present_success)
-        builder.append(success);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(CloseSession_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      CloseSession_result typedOther = (CloseSession_result)other;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("CloseSession_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class CloseSession_resultStandardSchemeFactory implements SchemeFactory {
-      public CloseSession_resultStandardScheme getScheme() {
-        return new CloseSession_resultStandardScheme();
-      }
-    }
-
-    private static class CloseSession_resultStandardScheme extends StandardScheme<CloseSession_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, CloseSession_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TCloseSessionResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, CloseSession_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class CloseSession_resultTupleSchemeFactory implements SchemeFactory {
-      public CloseSession_resultTupleScheme getScheme() {
-        return new CloseSession_resultTupleScheme();
-      }
-    }
-
-    private static class CloseSession_resultTupleScheme extends TupleScheme<CloseSession_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, CloseSession_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, CloseSession_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TCloseSessionResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetInfo_args implements org.apache.thrift.TBase<GetInfo_args, GetInfo_args._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetInfo_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetInfo_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetInfo_argsTupleSchemeFactory());
-    }
-
-    private TGetInfoReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetInfoReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetInfo_args.class, metaDataMap);
-    }
-
-    public GetInfo_args() {
-    }
-
-    public GetInfo_args(
-      TGetInfoReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetInfo_args(GetInfo_args other) {
-      if (other.isSetReq()) {
-        this.req = new TGetInfoReq(other.req);
-      }
-    }
-
-    public GetInfo_args deepCopy() {
-      return new GetInfo_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TGetInfoReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TGetInfoReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TGetInfoReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetInfo_args)
-        return this.equals((GetInfo_args)that);
-      return false;
-    }
-
-    public boolean equals(GetInfo_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_req = true && (isSetReq());
-      builder.append(present_req);
-      if (present_req)
-        builder.append(req);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(GetInfo_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      GetInfo_args typedOther = (GetInfo_args)other;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(typedOther.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, typedOther.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetInfo_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetInfo_argsStandardSchemeFactory implements SchemeFactory {
-      public GetInfo_argsStandardScheme getScheme() {
-        return new GetInfo_argsStandardScheme();
-      }
-    }
-
-    private static class GetInfo_argsStandardScheme extends StandardScheme<GetInfo_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetInfo_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TGetInfoReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetInfo_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetInfo_argsTupleSchemeFactory implements SchemeFactory {
-      public GetInfo_argsTupleScheme getScheme() {
-        return new GetInfo_argsTupleScheme();
-      }
-    }
-
-    private static class GetInfo_argsTupleScheme extends TupleScheme<GetInfo_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetInfo_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetInfo_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TGetInfoReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetInfo_result implements org.apache.thrift.TBase<GetInfo_result, GetInfo_result._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetInfo_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetInfo_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetInfo_resultTupleSchemeFactory());
-    }
-
-    private TGetInfoResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetInfoResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetInfo_result.class, metaDataMap);
-    }
-
-    public GetInfo_result() {
-    }
-
-    public GetInfo_result(
-      TGetInfoResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetInfo_result(GetInfo_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TGetInfoResp(other.success);
-      }
-    }
-
-    public GetInfo_result deepCopy() {
-      return new GetInfo_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TGetInfoResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TGetInfoResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TGetInfoResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetInfo_result)
-        return this.equals((GetInfo_result)that);
-      return false;
-    }
-
-    public boolean equals(GetInfo_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_success = true && (isSetSuccess());
-      builder.append(present_success);
-      if (present_success)
-        builder.append(success);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(GetInfo_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      GetInfo_result typedOther = (GetInfo_result)other;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetInfo_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetInfo_resultStandardSchemeFactory implements SchemeFactory {
-      public GetInfo_resultStandardScheme getScheme() {
-        return new GetInfo_resultStandardScheme();
-      }
-    }
-
-    private static class GetInfo_resultStandardScheme extends StandardScheme<GetInfo_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetInfo_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TGetInfoResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetInfo_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetInfo_resultTupleSchemeFactory implements SchemeFactory {
-      public GetInfo_resultTupleScheme getScheme() {
-        return new GetInfo_resultTupleScheme();
-      }
-    }
-
-    private static class GetInfo_resultTupleScheme extends TupleScheme<GetInfo_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetInfo_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetInfo_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TGetInfoResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class ExecuteStatement_args implements org.apache.thrift.TBase<ExecuteStatement_args, ExecuteStatement_args._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("ExecuteStatement_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new ExecuteStatement_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new ExecuteStatement_argsTupleSchemeFactory());
-    }
-
-    private TExecuteStatementReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TExecuteStatementReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(ExecuteStatement_args.class, metaDataMap);
-    }
-
-    public ExecuteStatement_args() {
-    }
-
-    public ExecuteStatement_args(
-      TExecuteStatementReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public ExecuteStatement_args(ExecuteStatement_args other) {
-      if (other.isSetReq()) {
-        this.req = new TExecuteStatementReq(other.req);
-      }
-    }
-
-    public ExecuteStatement_args deepCopy() {
-      return new ExecuteStatement_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TExecuteStatementReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TExecuteStatementReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TExecuteStatementReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof ExecuteStatement_args)
-        return this.equals((ExecuteStatement_args)that);
-      return false;
-    }
-
-    public boolean equals(ExecuteStatement_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_req = true && (isSetReq());
-      builder.append(present_req);
-      if (present_req)
-        builder.append(req);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(ExecuteStatement_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      ExecuteStatement_args typedOther = (ExecuteStatement_args)other;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(typedOther.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, typedOther.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("ExecuteStatement_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class ExecuteStatement_argsStandardSchemeFactory implements SchemeFactory {
-      public ExecuteStatement_argsStandardScheme getScheme() {
-        return new ExecuteStatement_argsStandardScheme();
-      }
-    }
-
-    private static class ExecuteStatement_argsStandardScheme extends StandardScheme<ExecuteStatement_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, ExecuteStatement_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TExecuteStatementReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, ExecuteStatement_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class ExecuteStatement_argsTupleSchemeFactory implements SchemeFactory {
-      public ExecuteStatement_argsTupleScheme getScheme() {
-        return new ExecuteStatement_argsTupleScheme();
-      }
-    }
-
-    private static class ExecuteStatement_argsTupleScheme extends TupleScheme<ExecuteStatement_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, ExecuteStatement_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, ExecuteStatement_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TExecuteStatementReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class ExecuteStatement_result implements org.apache.thrift.TBase<ExecuteStatement_result, ExecuteStatement_result._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("ExecuteStatement_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new ExecuteStatement_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new ExecuteStatement_resultTupleSchemeFactory());
-    }
-
-    private TExecuteStatementResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TExecuteStatementResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(ExecuteStatement_result.class, metaDataMap);
-    }
-
-    public ExecuteStatement_result() {
-    }
-
-    public ExecuteStatement_result(
-      TExecuteStatementResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public ExecuteStatement_result(ExecuteStatement_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TExecuteStatementResp(other.success);
-      }
-    }
-
-    public ExecuteStatement_result deepCopy() {
-      return new ExecuteStatement_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TExecuteStatementResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TExecuteStatementResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TExecuteStatementResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof ExecuteStatement_result)
-        return this.equals((ExecuteStatement_result)that);
-      return false;
-    }
-
-    public boolean equals(ExecuteStatement_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_success = true && (isSetSuccess());
-      builder.append(present_success);
-      if (present_success)
-        builder.append(success);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(ExecuteStatement_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      ExecuteStatement_result typedOther = (ExecuteStatement_result)other;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("ExecuteStatement_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class ExecuteStatement_resultStandardSchemeFactory implements SchemeFactory {
-      public ExecuteStatement_resultStandardScheme getScheme() {
-        return new ExecuteStatement_resultStandardScheme();
-      }
-    }
-
-    private static class ExecuteStatement_resultStandardScheme extends StandardScheme<ExecuteStatement_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, ExecuteStatement_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TExecuteStatementResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, ExecuteStatement_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class ExecuteStatement_resultTupleSchemeFactory implements SchemeFactory {
-      public ExecuteStatement_resultTupleScheme getScheme() {
-        return new ExecuteStatement_resultTupleScheme();
-      }
-    }
-
-    private static class ExecuteStatement_resultTupleScheme extends TupleScheme<ExecuteStatement_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, ExecuteStatement_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, ExecuteStatement_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TExecuteStatementResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetTypeInfo_args implements org.apache.thrift.TBase<GetTypeInfo_args, GetTypeInfo_args._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetTypeInfo_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetTypeInfo_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetTypeInfo_argsTupleSchemeFactory());
-    }
-
-    private TGetTypeInfoReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetTypeInfoReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetTypeInfo_args.class, metaDataMap);
-    }
-
-    public GetTypeInfo_args() {
-    }
-
-    public GetTypeInfo_args(
-      TGetTypeInfoReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetTypeInfo_args(GetTypeInfo_args other) {
-      if (other.isSetReq()) {
-        this.req = new TGetTypeInfoReq(other.req);
-      }
-    }
-
-    public GetTypeInfo_args deepCopy() {
-      return new GetTypeInfo_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TGetTypeInfoReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TGetTypeInfoReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TGetTypeInfoReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetTypeInfo_args)
-        return this.equals((GetTypeInfo_args)that);
-      return false;
-    }
-
-    public boolean equals(GetTypeInfo_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_req = true && (isSetReq());
-      builder.append(present_req);
-      if (present_req)
-        builder.append(req);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(GetTypeInfo_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      GetTypeInfo_args typedOther = (GetTypeInfo_args)other;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(typedOther.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, typedOther.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetTypeInfo_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetTypeInfo_argsStandardSchemeFactory implements SchemeFactory {
-      public GetTypeInfo_argsStandardScheme getScheme() {
-        return new GetTypeInfo_argsStandardScheme();
-      }
-    }
-
-    private static class GetTypeInfo_argsStandardScheme extends StandardScheme<GetTypeInfo_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetTypeInfo_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TGetTypeInfoReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetTypeInfo_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetTypeInfo_argsTupleSchemeFactory implements SchemeFactory {
-      public GetTypeInfo_argsTupleScheme getScheme() {
-        return new GetTypeInfo_argsTupleScheme();
-      }
-    }
-
-    private static class GetTypeInfo_argsTupleScheme extends TupleScheme<GetTypeInfo_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetTypeInfo_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetTypeInfo_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TGetTypeInfoReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetTypeInfo_result implements org.apache.thrift.TBase<GetTypeInfo_result, GetTypeInfo_result._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetTypeInfo_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetTypeInfo_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetTypeInfo_resultTupleSchemeFactory());
-    }
-
-    private TGetTypeInfoResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetTypeInfoResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetTypeInfo_result.class, metaDataMap);
-    }
-
-    public GetTypeInfo_result() {
-    }
-
-    public GetTypeInfo_result(
-      TGetTypeInfoResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetTypeInfo_result(GetTypeInfo_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TGetTypeInfoResp(other.success);
-      }
-    }
-
-    public GetTypeInfo_result deepCopy() {
-      return new GetTypeInfo_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TGetTypeInfoResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TGetTypeInfoResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TGetTypeInfoResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetTypeInfo_result)
-        return this.equals((GetTypeInfo_result)that);
-      return false;
-    }
-
-    public boolean equals(GetTypeInfo_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_success = true && (isSetSuccess());
-      builder.append(present_success);
-      if (present_success)
-        builder.append(success);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(GetTypeInfo_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      GetTypeInfo_result typedOther = (GetTypeInfo_result)other;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetTypeInfo_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetTypeInfo_resultStandardSchemeFactory implements SchemeFactory {
-      public GetTypeInfo_resultStandardScheme getScheme() {
-        return new GetTypeInfo_resultStandardScheme();
-      }
-    }
-
-    private static class GetTypeInfo_resultStandardScheme extends StandardScheme<GetTypeInfo_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetTypeInfo_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TGetTypeInfoResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetTypeInfo_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetTypeInfo_resultTupleSchemeFactory implements SchemeFactory {
-      public GetTypeInfo_resultTupleScheme getScheme() {
-        return new GetTypeInfo_resultTupleScheme();
-      }
-    }
-
-    private static class GetTypeInfo_resultTupleScheme extends TupleScheme<GetTypeInfo_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetTypeInfo_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetTypeInfo_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TGetTypeInfoResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetCatalogs_args implements org.apache.thrift.TBase<GetCatalogs_args, GetCatalogs_args._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetCatalogs_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetCatalogs_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetCatalogs_argsTupleSchemeFactory());
-    }
-
-    private TGetCatalogsReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetCatalogsReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetCatalogs_args.class, metaDataMap);
-    }
-
-    public GetCatalogs_args() {
-    }
-
-    public GetCatalogs_args(
-      TGetCatalogsReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetCatalogs_args(GetCatalogs_args other) {
-      if (other.isSetReq()) {
-        this.req = new TGetCatalogsReq(other.req);
-      }
-    }
-
-    public GetCatalogs_args deepCopy() {
-      return new GetCatalogs_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TGetCatalogsReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TGetCatalogsReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TGetCatalogsReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetCatalogs_args)
-        return this.equals((GetCatalogs_args)that);
-      return false;
-    }
-
-    public boolean equals(GetCatalogs_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_req = true && (isSetReq());
-      builder.append(present_req);
-      if (present_req)
-        builder.append(req);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(GetCatalogs_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      GetCatalogs_args typedOther = (GetCatalogs_args)other;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(typedOther.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, typedOther.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetCatalogs_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetCatalogs_argsStandardSchemeFactory implements SchemeFactory {
-      public GetCatalogs_argsStandardScheme getScheme() {
-        return new GetCatalogs_argsStandardScheme();
-      }
-    }
-
-    private static class GetCatalogs_argsStandardScheme extends StandardScheme<GetCatalogs_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetCatalogs_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TGetCatalogsReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetCatalogs_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetCatalogs_argsTupleSchemeFactory implements SchemeFactory {
-      public GetCatalogs_argsTupleScheme getScheme() {
-        return new GetCatalogs_argsTupleScheme();
-      }
-    }
-
-    private static class GetCatalogs_argsTupleScheme extends TupleScheme<GetCatalogs_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetCatalogs_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetCatalogs_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TGetCatalogsReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetCatalogs_result implements org.apache.thrift.TBase<GetCatalogs_result, GetCatalogs_result._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetCatalogs_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetCatalogs_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetCatalogs_resultTupleSchemeFactory());
-    }
-
-    private TGetCatalogsResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetCatalogsResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetCatalogs_result.class, metaDataMap);
-    }
-
-    public GetCatalogs_result() {
-    }
-
-    public GetCatalogs_result(
-      TGetCatalogsResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetCatalogs_result(GetCatalogs_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TGetCatalogsResp(other.success);
-      }
-    }
-
-    public GetCatalogs_result deepCopy() {
-      return new GetCatalogs_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TGetCatalogsResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TGetCatalogsResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TGetCatalogsResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetCatalogs_result)
-        return this.equals((GetCatalogs_result)that);
-      return false;
-    }
-
-    public boolean equals(GetCatalogs_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_success = true && (isSetSuccess());
-      builder.append(present_success);
-      if (present_success)
-        builder.append(success);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(GetCatalogs_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      GetCatalogs_result typedOther = (GetCatalogs_result)other;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetCatalogs_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetCatalogs_resultStandardSchemeFactory implements SchemeFactory {
-      public GetCatalogs_resultStandardScheme getScheme() {
-        return new GetCatalogs_resultStandardScheme();
-      }
-    }
-
-    private static class GetCatalogs_resultStandardScheme extends StandardScheme<GetCatalogs_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetCatalogs_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TGetCatalogsResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetCatalogs_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetCatalogs_resultTupleSchemeFactory implements SchemeFactory {
-      public GetCatalogs_resultTupleScheme getScheme() {
-        return new GetCatalogs_resultTupleScheme();
-      }
-    }
-
-    private static class GetCatalogs_resultTupleScheme extends TupleScheme<GetCatalogs_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetCatalogs_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetCatalogs_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TGetCatalogsResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetSchemas_args implements org.apache.thrift.TBase<GetSchemas_args, GetSchemas_args._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetSchemas_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetSchemas_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetSchemas_argsTupleSchemeFactory());
-    }
-
-    private TGetSchemasReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetSchemasReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetSchemas_args.class, metaDataMap);
-    }
-
-    public GetSchemas_args() {
-    }
-
-    public GetSchemas_args(
-      TGetSchemasReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetSchemas_args(GetSchemas_args other) {
-      if (other.isSetReq()) {
-        this.req = new TGetSchemasReq(other.req);
-      }
-    }
-
-    public GetSchemas_args deepCopy() {
-      return new GetSchemas_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TGetSchemasReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TGetSchemasReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TGetSchemasReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetSchemas_args)
-        return this.equals((GetSchemas_args)that);
-      return false;
-    }
-
-    public boolean equals(GetSchemas_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_req = true && (isSetReq());
-      builder.append(present_req);
-      if (present_req)
-        builder.append(req);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(GetSchemas_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      GetSchemas_args typedOther = (GetSchemas_args)other;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(typedOther.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, typedOther.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetSchemas_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetSchemas_argsStandardSchemeFactory implements SchemeFactory {
-      public GetSchemas_argsStandardScheme getScheme() {
-        return new GetSchemas_argsStandardScheme();
-      }
-    }
-
-    private static class GetSchemas_argsStandardScheme extends StandardScheme<GetSchemas_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetSchemas_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TGetSchemasReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetSchemas_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetSchemas_argsTupleSchemeFactory implements SchemeFactory {
-      public GetSchemas_argsTupleScheme getScheme() {
-        return new GetSchemas_argsTupleScheme();
-      }
-    }
-
-    private static class GetSchemas_argsTupleScheme extends TupleScheme<GetSchemas_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetSchemas_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetSchemas_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TGetSchemasReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetSchemas_result implements org.apache.thrift.TBase<GetSchemas_result, GetSchemas_result._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetSchemas_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetSchemas_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetSchemas_resultTupleSchemeFactory());
-    }
-
-    private TGetSchemasResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetSchemasResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetSchemas_result.class, metaDataMap);
-    }
-
-    public GetSchemas_result() {
-    }
-
-    public GetSchemas_result(
-      TGetSchemasResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetSchemas_result(GetSchemas_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TGetSchemasResp(other.success);
-      }
-    }
-
-    public GetSchemas_result deepCopy() {
-      return new GetSchemas_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TGetSchemasResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TGetSchemasResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TGetSchemasResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetSchemas_result)
-        return this.equals((GetSchemas_result)that);
-      return false;
-    }
-
-    public boolean equals(GetSchemas_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_success = true && (isSetSuccess());
-      builder.append(present_success);
-      if (present_success)
-        builder.append(success);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(GetSchemas_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      GetSchemas_result typedOther = (GetSchemas_result)other;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetSchemas_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetSchemas_resultStandardSchemeFactory implements SchemeFactory {
-      public GetSchemas_resultStandardScheme getScheme() {
-        return new GetSchemas_resultStandardScheme();
-      }
-    }
-
-    private static class GetSchemas_resultStandardScheme extends StandardScheme<GetSchemas_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetSchemas_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TGetSchemasResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetSchemas_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetSchemas_resultTupleSchemeFactory implements SchemeFactory {
-      public GetSchemas_resultTupleScheme getScheme() {
-        return new GetSchemas_resultTupleScheme();
-      }
-    }
-
-    private static class GetSchemas_resultTupleScheme extends TupleScheme<GetSchemas_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetSchemas_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetSchemas_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TGetSchemasResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetTables_args implements org.apache.thrift.TBase<GetTables_args, GetTables_args._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetTables_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetTables_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetTables_argsTupleSchemeFactory());
-    }
-
-    private TGetTablesReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetTablesReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetTables_args.class, metaDataMap);
-    }
-
-    public GetTables_args() {
-    }
-
-    public GetTables_args(
-      TGetTablesReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetTables_args(GetTables_args other) {
-      if (other.isSetReq()) {
-        this.req = new TGetTablesReq(other.req);
-      }
-    }
-
-    public GetTables_args deepCopy() {
-      return new GetTables_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TGetTablesReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TGetTablesReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TGetTablesReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetTables_args)
-        return this.equals((GetTables_args)that);
-      return false;
-    }
-
-    public boolean equals(GetTables_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_req = true && (isSetReq());
-      builder.append(present_req);
-      if (present_req)
-        builder.append(req);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(GetTables_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      GetTables_args typedOther = (GetTables_args)other;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(typedOther.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, typedOther.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetTables_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetTables_argsStandardSchemeFactory implements SchemeFactory {
-      public GetTables_argsStandardScheme getScheme() {
-        return new GetTables_argsStandardScheme();
-      }
-    }
-
-    private static class GetTables_argsStandardScheme extends StandardScheme<GetTables_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetTables_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TGetTablesReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetTables_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetTables_argsTupleSchemeFactory implements SchemeFactory {
-      public GetTables_argsTupleScheme getScheme() {
-        return new GetTables_argsTupleScheme();
-      }
-    }
-
-    private static class GetTables_argsTupleScheme extends TupleScheme<GetTables_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetTables_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetTables_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TGetTablesReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetTables_result implements org.apache.thrift.TBase<GetTables_result, GetTables_result._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetTables_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetTables_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetTables_resultTupleSchemeFactory());
-    }
-
-    private TGetTablesResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetTablesResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetTables_result.class, metaDataMap);
-    }
-
-    public GetTables_result() {
-    }
-
-    public GetTables_result(
-      TGetTablesResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetTables_result(GetTables_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TGetTablesResp(other.success);
-      }
-    }
-
-    public GetTables_result deepCopy() {
-      return new GetTables_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TGetTablesResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TGetTablesResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TGetTablesResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetTables_result)
-        return this.equals((GetTables_result)that);
-      return false;
-    }
-
-    public boolean equals(GetTables_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_success = true && (isSetSuccess());
-      builder.append(present_success);
-      if (present_success)
-        builder.append(success);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(GetTables_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      GetTables_result typedOther = (GetTables_result)other;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetTables_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetTables_resultStandardSchemeFactory implements SchemeFactory {
-      public GetTables_resultStandardScheme getScheme() {
-        return new GetTables_resultStandardScheme();
-      }
-    }
-
-    private static class GetTables_resultStandardScheme extends StandardScheme<GetTables_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetTables_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TGetTablesResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetTables_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetTables_resultTupleSchemeFactory implements SchemeFactory {
-      public GetTables_resultTupleScheme getScheme() {
-        return new GetTables_resultTupleScheme();
-      }
-    }
-
-    private static class GetTables_resultTupleScheme extends TupleScheme<GetTables_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetTables_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetTables_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TGetTablesResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetTableTypes_args implements org.apache.thrift.TBase<GetTableTypes_args, GetTableTypes_args._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetTableTypes_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetTableTypes_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetTableTypes_argsTupleSchemeFactory());
-    }
-
-    private TGetTableTypesReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetTableTypesReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetTableTypes_args.class, metaDataMap);
-    }
-
-    public GetTableTypes_args() {
-    }
-
-    public GetTableTypes_args(
-      TGetTableTypesReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetTableTypes_args(GetTableTypes_args other) {
-      if (other.isSetReq()) {
-        this.req = new TGetTableTypesReq(other.req);
-      }
-    }
-
-    public GetTableTypes_args deepCopy() {
-      return new GetTableTypes_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TGetTableTypesReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TGetTableTypesReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TGetTableTypesReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetTableTypes_args)
-        return this.equals((GetTableTypes_args)that);
-      return false;
-    }
-
-    public boolean equals(GetTableTypes_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_req = true && (isSetReq());
-      builder.append(present_req);
-      if (present_req)
-        builder.append(req);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(GetTableTypes_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      GetTableTypes_args typedOther = (GetTableTypes_args)other;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(typedOther.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, typedOther.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetTableTypes_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetTableTypes_argsStandardSchemeFactory implements SchemeFactory {
-      public GetTableTypes_argsStandardScheme getScheme() {
-        return new GetTableTypes_argsStandardScheme();
-      }
-    }
-
-    private static class GetTableTypes_argsStandardScheme extends StandardScheme<GetTableTypes_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetTableTypes_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TGetTableTypesReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetTableTypes_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetTableTypes_argsTupleSchemeFactory implements SchemeFactory {
-      public GetTableTypes_argsTupleScheme getScheme() {
-        return new GetTableTypes_argsTupleScheme();
-      }
-    }
-
-    private static class GetTableTypes_argsTupleScheme extends TupleScheme<GetTableTypes_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetTableTypes_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetTableTypes_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TGetTableTypesReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetTableTypes_result implements org.apache.thrift.TBase<GetTableTypes_result, GetTableTypes_result._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetTableTypes_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetTableTypes_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetTableTypes_resultTupleSchemeFactory());
-    }
-
-    private TGetTableTypesResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetTableTypesResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetTableTypes_result.class, metaDataMap);
-    }
-
-    public GetTableTypes_result() {
-    }
-
-    public GetTableTypes_result(
-      TGetTableTypesResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetTableTypes_result(GetTableTypes_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TGetTableTypesResp(other.success);
-      }
-    }
-
-    public GetTableTypes_result deepCopy() {
-      return new GetTableTypes_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TGetTableTypesResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TGetTableTypesResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TGetTableTypesResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetTableTypes_result)
-        return this.equals((GetTableTypes_result)that);
-      return false;
-    }
-
-    public boolean equals(GetTableTypes_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_success = true && (isSetSuccess());
-      builder.append(present_success);
-      if (present_success)
-        builder.append(success);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(GetTableTypes_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      GetTableTypes_result typedOther = (GetTableTypes_result)other;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetTableTypes_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetTableTypes_resultStandardSchemeFactory implements SchemeFactory {
-      public GetTableTypes_resultStandardScheme getScheme() {
-        return new GetTableTypes_resultStandardScheme();
-      }
-    }
-
-    private static class GetTableTypes_resultStandardScheme extends StandardScheme<GetTableTypes_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetTableTypes_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TGetTableTypesResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetTableTypes_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetTableTypes_resultTupleSchemeFactory implements SchemeFactory {
-      public GetTableTypes_resultTupleScheme getScheme() {
-        return new GetTableTypes_resultTupleScheme();
-      }
-    }
-
-    private static class GetTableTypes_resultTupleScheme extends TupleScheme<GetTableTypes_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetTableTypes_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetTableTypes_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TGetTableTypesResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetColumns_args implements org.apache.thrift.TBase<GetColumns_args, GetColumns_args._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetColumns_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetColumns_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetColumns_argsTupleSchemeFactory());
-    }
-
-    private TGetColumnsReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetColumnsReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetColumns_args.class, metaDataMap);
-    }
-
-    public GetColumns_args() {
-    }
-
-    public GetColumns_args(
-      TGetColumnsReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetColumns_args(GetColumns_args other) {
-      if (other.isSetReq()) {
-        this.req = new TGetColumnsReq(other.req);
-      }
-    }
-
-    public GetColumns_args deepCopy() {
-      return new GetColumns_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TGetColumnsReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TGetColumnsReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TGetColumnsReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetColumns_args)
-        return this.equals((GetColumns_args)that);
-      return false;
-    }
-
-    public boolean equals(GetColumns_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_req = true && (isSetReq());
-      builder.append(present_req);
-      if (present_req)
-        builder.append(req);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(GetColumns_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      GetColumns_args typedOther = (GetColumns_args)other;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(typedOther.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, typedOther.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetColumns_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetColumns_argsStandardSchemeFactory implements SchemeFactory {
-      public GetColumns_argsStandardScheme getScheme() {
-        return new GetColumns_argsStandardScheme();
-      }
-    }
-
-    private static class GetColumns_argsStandardScheme extends StandardScheme<GetColumns_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetColumns_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TGetColumnsReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetColumns_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetColumns_argsTupleSchemeFactory implements SchemeFactory {
-      public GetColumns_argsTupleScheme getScheme() {
-        return new GetColumns_argsTupleScheme();
-      }
-    }
-
-    private static class GetColumns_argsTupleScheme extends TupleScheme<GetColumns_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetColumns_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetColumns_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TGetColumnsReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetColumns_result implements org.apache.thrift.TBase<GetColumns_result, GetColumns_result._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetColumns_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetColumns_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetColumns_resultTupleSchemeFactory());
-    }
-
-    private TGetColumnsResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetColumnsResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetColumns_result.class, metaDataMap);
-    }
-
-    public GetColumns_result() {
-    }
-
-    public GetColumns_result(
-      TGetColumnsResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetColumns_result(GetColumns_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TGetColumnsResp(other.success);
-      }
-    }
-
-    public GetColumns_result deepCopy() {
-      return new GetColumns_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TGetColumnsResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TGetColumnsResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TGetColumnsResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetColumns_result)
-        return this.equals((GetColumns_result)that);
-      return false;
-    }
-
-    public boolean equals(GetColumns_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_success = true && (isSetSuccess());
-      builder.append(present_success);
-      if (present_success)
-        builder.append(success);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(GetColumns_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      GetColumns_result typedOther = (GetColumns_result)other;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetColumns_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetColumns_resultStandardSchemeFactory implements SchemeFactory {
-      public GetColumns_resultStandardScheme getScheme() {
-        return new GetColumns_resultStandardScheme();
-      }
-    }
-
-    private static class GetColumns_resultStandardScheme extends StandardScheme<GetColumns_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetColumns_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TGetColumnsResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetColumns_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetColumns_resultTupleSchemeFactory implements SchemeFactory {
-      public GetColumns_resultTupleScheme getScheme() {
-        return new GetColumns_resultTupleScheme();
-      }
-    }
-
-    private static class GetColumns_resultTupleScheme extends TupleScheme<GetColumns_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetColumns_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetColumns_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TGetColumnsResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetFunctions_args implements org.apache.thrift.TBase<GetFunctions_args, GetFunctions_args._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetFunctions_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetFunctions_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetFunctions_argsTupleSchemeFactory());
-    }
-
-    private TGetFunctionsReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetFunctionsReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetFunctions_args.class, metaDataMap);
-    }
-
-    public GetFunctions_args() {
-    }
-
-    public GetFunctions_args(
-      TGetFunctionsReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetFunctions_args(GetFunctions_args other) {
-      if (other.isSetReq()) {
-        this.req = new TGetFunctionsReq(other.req);
-      }
-    }
-
-    public GetFunctions_args deepCopy() {
-      return new GetFunctions_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TGetFunctionsReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TGetFunctionsReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TGetFunctionsReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetFunctions_args)
-        return this.equals((GetFunctions_args)that);
-      return false;
-    }
-
-    public boolean equals(GetFunctions_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_req = true && (isSetReq());
-      builder.append(present_req);
-      if (present_req)
-        builder.append(req);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(GetFunctions_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      GetFunctions_args typedOther = (GetFunctions_args)other;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(typedOther.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, typedOther.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetFunctions_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetFunctions_argsStandardSchemeFactory implements SchemeFactory {
-      public GetFunctions_argsStandardScheme getScheme() {
-        return new GetFunctions_argsStandardScheme();
-      }
-    }
-
-    private static class GetFunctions_argsStandardScheme extends StandardScheme<GetFunctions_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetFunctions_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TGetFunctionsReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetFunctions_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetFunctions_argsTupleSchemeFactory implements SchemeFactory {
-      public GetFunctions_argsTupleScheme getScheme() {
-        return new GetFunctions_argsTupleScheme();
-      }
-    }
-
-    private static class GetFunctions_argsTupleScheme extends TupleScheme<GetFunctions_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetFunctions_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetFunctions_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TGetFunctionsReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetFunctions_result implements org.apache.thrift.TBase<GetFunctions_result, GetFunctions_result._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetFunctions_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetFunctions_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetFunctions_resultTupleSchemeFactory());
-    }
-
-    private TGetFunctionsResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetFunctionsResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetFunctions_result.class, metaDataMap);
-    }
-
-    public GetFunctions_result() {
-    }
-
-    public GetFunctions_result(
-      TGetFunctionsResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetFunctions_result(GetFunctions_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TGetFunctionsResp(other.success);
-      }
-    }
-
-    public GetFunctions_result deepCopy() {
-      return new GetFunctions_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TGetFunctionsResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TGetFunctionsResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TGetFunctionsResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetFunctions_result)
-        return this.equals((GetFunctions_result)that);
-      return false;
-    }
-
-    public boolean equals(GetFunctions_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_success = true && (isSetSuccess());
-      builder.append(present_success);
-      if (present_success)
-        builder.append(success);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(GetFunctions_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      GetFunctions_result typedOther = (GetFunctions_result)other;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetFunctions_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetFunctions_resultStandardSchemeFactory implements SchemeFactory {
-      public GetFunctions_resultStandardScheme getScheme() {
-        return new GetFunctions_resultStandardScheme();
-      }
-    }
-
-    private static class GetFunctions_resultStandardScheme extends StandardScheme<GetFunctions_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetFunctions_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TGetFunctionsResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetFunctions_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetFunctions_resultTupleSchemeFactory implements SchemeFactory {
-      public GetFunctions_resultTupleScheme getScheme() {
-        return new GetFunctions_resultTupleScheme();
-      }
-    }
-
-    private static class GetFunctions_resultTupleScheme extends TupleScheme<GetFunctions_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetFunctions_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetFunctions_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TGetFunctionsResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetOperationStatus_args implements org.apache.thrift.TBase<GetOperationStatus_args, GetOperationStatus_args._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetOperationStatus_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetOperationStatus_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetOperationStatus_argsTupleSchemeFactory());
-    }
-
-    private TGetOperationStatusReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetOperationStatusReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetOperationStatus_args.class, metaDataMap);
-    }
-
-    public GetOperationStatus_args() {
-    }
-
-    public GetOperationStatus_args(
-      TGetOperationStatusReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetOperationStatus_args(GetOperationStatus_args other) {
-      if (other.isSetReq()) {
-        this.req = new TGetOperationStatusReq(other.req);
-      }
-    }
-
-    public GetOperationStatus_args deepCopy() {
-      return new GetOperationStatus_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TGetOperationStatusReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TGetOperationStatusReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TGetOperationStatusReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetOperationStatus_args)
-        return this.equals((GetOperationStatus_args)that);
-      return false;
-    }
-
-    public boolean equals(GetOperationStatus_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_req = true && (isSetReq());
-      builder.append(present_req);
-      if (present_req)
-        builder.append(req);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(GetOperationStatus_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      GetOperationStatus_args typedOther = (GetOperationStatus_args)other;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(typedOther.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, typedOther.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetOperationStatus_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetOperationStatus_argsStandardSchemeFactory implements SchemeFactory {
-      public GetOperationStatus_argsStandardScheme getScheme() {
-        return new GetOperationStatus_argsStandardScheme();
-      }
-    }
-
-    private static class GetOperationStatus_argsStandardScheme extends StandardScheme<GetOperationStatus_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetOperationStatus_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TGetOperationStatusReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetOperationStatus_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetOperationStatus_argsTupleSchemeFactory implements SchemeFactory {
-      public GetOperationStatus_argsTupleScheme getScheme() {
-        return new GetOperationStatus_argsTupleScheme();
-      }
-    }
-
-    private static class GetOperationStatus_argsTupleScheme extends TupleScheme<GetOperationStatus_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetOperationStatus_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetOperationStatus_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TGetOperationStatusReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetOperationStatus_result implements org.apache.thrift.TBase<GetOperationStatus_result, GetOperationStatus_result._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetOperationStatus_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetOperationStatus_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetOperationStatus_resultTupleSchemeFactory());
-    }
-
-    private TGetOperationStatusResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetOperationStatusResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetOperationStatus_result.class, metaDataMap);
-    }
-
-    public GetOperationStatus_result() {
-    }
-
-    public GetOperationStatus_result(
-      TGetOperationStatusResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetOperationStatus_result(GetOperationStatus_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TGetOperationStatusResp(other.success);
-      }
-    }
-
-    public GetOperationStatus_result deepCopy() {
-      return new GetOperationStatus_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TGetOperationStatusResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TGetOperationStatusResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TGetOperationStatusResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetOperationStatus_result)
-        return this.equals((GetOperationStatus_result)that);
-      return false;
-    }
-
-    public boolean equals(GetOperationStatus_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_success = true && (isSetSuccess());
-      builder.append(present_success);
-      if (present_success)
-        builder.append(success);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(GetOperationStatus_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      GetOperationStatus_result typedOther = (GetOperationStatus_result)other;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetOperationStatus_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetOperationStatus_resultStandardSchemeFactory implements SchemeFactory {
-      public GetOperationStatus_resultStandardScheme getScheme() {
-        return new GetOperationStatus_resultStandardScheme();
-      }
-    }
-
-    private static class GetOperationStatus_resultStandardScheme extends StandardScheme<GetOperationStatus_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetOperationStatus_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TGetOperationStatusResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetOperationStatus_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetOperationStatus_resultTupleSchemeFactory implements SchemeFactory {
-      public GetOperationStatus_resultTupleScheme getScheme() {
-        return new GetOperationStatus_resultTupleScheme();
-      }
-    }
-
-    private static class GetOperationStatus_resultTupleScheme extends TupleScheme<GetOperationStatus_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetOperationStatus_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetOperationStatus_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TGetOperationStatusResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class CancelOperation_args implements org.apache.thrift.TBase<CancelOperation_args, CancelOperation_args._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CancelOperation_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new CancelOperation_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new CancelOperation_argsTupleSchemeFactory());
-    }
-
-    private TCancelOperationReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TCancelOperationReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CancelOperation_args.class, metaDataMap);
-    }
-
-    public CancelOperation_args() {
-    }
-
-    public CancelOperation_args(
-      TCancelOperationReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public CancelOperation_args(CancelOperation_args other) {
-      if (other.isSetReq()) {
-        this.req = new TCancelOperationReq(other.req);
-      }
-    }
-
-    public CancelOperation_args deepCopy() {
-      return new CancelOperation_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TCancelOperationReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TCancelOperationReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TCancelOperationReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof CancelOperation_args)
-        return this.equals((CancelOperation_args)that);
-      return false;
-    }
-
-    public boolean equals(CancelOperation_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_req = true && (isSetReq());
-      builder.append(present_req);
-      if (present_req)
-        builder.append(req);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(CancelOperation_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      CancelOperation_args typedOther = (CancelOperation_args)other;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(typedOther.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, typedOther.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("CancelOperation_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class CancelOperation_argsStandardSchemeFactory implements SchemeFactory {
-      public CancelOperation_argsStandardScheme getScheme() {
-        return new CancelOperation_argsStandardScheme();
-      }
-    }
-
-    private static class CancelOperation_argsStandardScheme extends StandardScheme<CancelOperation_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, CancelOperation_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TCancelOperationReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, CancelOperation_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class CancelOperation_argsTupleSchemeFactory implements SchemeFactory {
-      public CancelOperation_argsTupleScheme getScheme() {
-        return new CancelOperation_argsTupleScheme();
-      }
-    }
-
-    private static class CancelOperation_argsTupleScheme extends TupleScheme<CancelOperation_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, CancelOperation_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, CancelOperation_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TCancelOperationReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class CancelOperation_result implements org.apache.thrift.TBase<CancelOperation_result, CancelOperation_result._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CancelOperation_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new CancelOperation_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new CancelOperation_resultTupleSchemeFactory());
-    }
-
-    private TCancelOperationResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TCancelOperationResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CancelOperation_result.class, metaDataMap);
-    }
-
-    public CancelOperation_result() {
-    }
-
-    public CancelOperation_result(
-      TCancelOperationResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public CancelOperation_result(CancelOperation_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TCancelOperationResp(other.success);
-      }
-    }
-
-    public CancelOperation_result deepCopy() {
-      return new CancelOperation_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TCancelOperationResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TCancelOperationResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TCancelOperationResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof CancelOperation_result)
-        return this.equals((CancelOperation_result)that);
-      return false;
-    }
-
-    public boolean equals(CancelOperation_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_success = true && (isSetSuccess());
-      builder.append(present_success);
-      if (present_success)
-        builder.append(success);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(CancelOperation_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      CancelOperation_result typedOther = (CancelOperation_result)other;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("CancelOperation_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class CancelOperation_resultStandardSchemeFactory implements SchemeFactory {
-      public CancelOperation_resultStandardScheme getScheme() {
-        return new CancelOperation_resultStandardScheme();
-      }
-    }
-
-    private static class CancelOperation_resultStandardScheme extends StandardScheme<CancelOperation_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, CancelOperation_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TCancelOperationResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, CancelOperation_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class CancelOperation_resultTupleSchemeFactory implements SchemeFactory {
-      public CancelOperation_resultTupleScheme getScheme() {
-        return new CancelOperation_resultTupleScheme();
-      }
-    }
-
-    private static class CancelOperation_resultTupleScheme extends TupleScheme<CancelOperation_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, CancelOperation_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, CancelOperation_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TCancelOperationResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class CloseOperation_args implements org.apache.thrift.TBase<CloseOperation_args, CloseOperation_args._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CloseOperation_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new CloseOperation_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new CloseOperation_argsTupleSchemeFactory());
-    }
-
-    private TCloseOperationReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TCloseOperationReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CloseOperation_args.class, metaDataMap);
-    }
-
-    public CloseOperation_args() {
-    }
-
-    public CloseOperation_args(
-      TCloseOperationReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public CloseOperation_args(CloseOperation_args other) {
-      if (other.isSetReq()) {
-        this.req = new TCloseOperationReq(other.req);
-      }
-    }
-
-    public CloseOperation_args deepCopy() {
-      return new CloseOperation_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TCloseOperationReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TCloseOperationReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TCloseOperationReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof CloseOperation_args)
-        return this.equals((CloseOperation_args)that);
-      return false;
-    }
-
-    public boolean equals(CloseOperation_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_req = true && (isSetReq());
-      builder.append(present_req);
-      if (present_req)
-        builder.append(req);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(CloseOperation_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      CloseOperation_args typedOther = (CloseOperation_args)other;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(typedOther.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, typedOther.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("CloseOperation_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class CloseOperation_argsStandardSchemeFactory implements SchemeFactory {
-      public CloseOperation_argsStandardScheme getScheme() {
-        return new CloseOperation_argsStandardScheme();
-      }
-    }
-
-    private static class CloseOperation_argsStandardScheme extends StandardScheme<CloseOperation_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, CloseOperation_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TCloseOperationReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, CloseOperation_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class CloseOperation_argsTupleSchemeFactory implements SchemeFactory {
-      public CloseOperation_argsTupleScheme getScheme() {
-        return new CloseOperation_argsTupleScheme();
-      }
-    }
-
-    private static class CloseOperation_argsTupleScheme extends TupleScheme<CloseOperation_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, CloseOperation_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, CloseOperation_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TCloseOperationReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class CloseOperation_result implements org.apache.thrift.TBase<CloseOperation_result, CloseOperation_result._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CloseOperation_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new CloseOperation_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new CloseOperation_resultTupleSchemeFactory());
-    }
-
-    private TCloseOperationResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TCloseOperationResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CloseOperation_result.class, metaDataMap);
-    }
-
-    public CloseOperation_result() {
-    }
-
-    public CloseOperation_result(
-      TCloseOperationResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public CloseOperation_result(CloseOperation_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TCloseOperationResp(other.success);
-      }
-    }
-
-    public CloseOperation_result deepCopy() {
-      return new CloseOperation_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TCloseOperationResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TCloseOperationResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TCloseOperationResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof CloseOperation_result)
-        return this.equals((CloseOperation_result)that);
-      return false;
-    }
-
-    public boolean equals(CloseOperation_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_success = true && (isSetSuccess());
-      builder.append(present_success);
-      if (present_success)
-        builder.append(success);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(CloseOperation_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      CloseOperation_result typedOther = (CloseOperation_result)other;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("CloseOperation_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class CloseOperation_resultStandardSchemeFactory implements SchemeFactory {
-      public CloseOperation_resultStandardScheme getScheme() {
-        return new CloseOperation_resultStandardScheme();
-      }
-    }
-
-    private static class CloseOperation_resultStandardScheme extends StandardScheme<CloseOperation_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, CloseOperation_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TCloseOperationResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, CloseOperation_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class CloseOperation_resultTupleSchemeFactory implements SchemeFactory {
-      public CloseOperation_resultTupleScheme getScheme() {
-        return new CloseOperation_resultTupleScheme();
-      }
-    }
-
-    private static class CloseOperation_resultTupleScheme extends TupleScheme<CloseOperation_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, CloseOperation_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, CloseOperation_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TCloseOperationResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetResultSetMetadata_args implements org.apache.thrift.TBase<GetResultSetMetadata_args, GetResultSetMetadata_args._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetResultSetMetadata_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetResultSetMetadata_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetResultSetMetadata_argsTupleSchemeFactory());
-    }
-
-    private TGetResultSetMetadataReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetResultSetMetadataReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetResultSetMetadata_args.class, metaDataMap);
-    }
-
-    public GetResultSetMetadata_args() {
-    }
-
-    public GetResultSetMetadata_args(
-      TGetResultSetMetadataReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetResultSetMetadata_args(GetResultSetMetadata_args other) {
-      if (other.isSetReq()) {
-        this.req = new TGetResultSetMetadataReq(other.req);
-      }
-    }
-
-    public GetResultSetMetadata_args deepCopy() {
-      return new GetResultSetMetadata_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TGetResultSetMetadataReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TGetResultSetMetadataReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TGetResultSetMetadataReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetResultSetMetadata_args)
-        return this.equals((GetResultSetMetadata_args)that);
-      return false;
-    }
-
-    public boolean equals(GetResultSetMetadata_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_req = true && (isSetReq());
-      builder.append(present_req);
-      if (present_req)
-        builder.append(req);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(GetResultSetMetadata_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      GetResultSetMetadata_args typedOther = (GetResultSetMetadata_args)other;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(typedOther.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, typedOther.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetResultSetMetadata_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetResultSetMetadata_argsStandardSchemeFactory implements SchemeFactory {
-      public GetResultSetMetadata_argsStandardScheme getScheme() {
-        return new GetResultSetMetadata_argsStandardScheme();
-      }
-    }
-
-    private static class GetResultSetMetadata_argsStandardScheme extends StandardScheme<GetResultSetMetadata_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetResultSetMetadata_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TGetResultSetMetadataReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetResultSetMetadata_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetResultSetMetadata_argsTupleSchemeFactory implements SchemeFactory {
-      public GetResultSetMetadata_argsTupleScheme getScheme() {
-        return new GetResultSetMetadata_argsTupleScheme();
-      }
-    }
-
-    private static class GetResultSetMetadata_argsTupleScheme extends TupleScheme<GetResultSetMetadata_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetResultSetMetadata_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetResultSetMetadata_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TGetResultSetMetadataReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetResultSetMetadata_result implements org.apache.thrift.TBase<GetResultSetMetadata_result, GetResultSetMetadata_result._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetResultSetMetadata_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetResultSetMetadata_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetResultSetMetadata_resultTupleSchemeFactory());
-    }
-
-    private TGetResultSetMetadataResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetResultSetMetadataResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetResultSetMetadata_result.class, metaDataMap);
-    }
-
-    public GetResultSetMetadata_result() {
-    }
-
-    public GetResultSetMetadata_result(
-      TGetResultSetMetadataResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetResultSetMetadata_result(GetResultSetMetadata_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TGetResultSetMetadataResp(other.success);
-      }
-    }
-
-    public GetResultSetMetadata_result deepCopy() {
-      return new GetResultSetMetadata_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TGetResultSetMetadataResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TGetResultSetMetadataResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TGetResultSetMetadataResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetResultSetMetadata_result)
-        return this.equals((GetResultSetMetadata_result)that);
-      return false;
-    }
-
-    public boolean equals(GetResultSetMetadata_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_success = true && (isSetSuccess());
-      builder.append(present_success);
-      if (present_success)
-        builder.append(success);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(GetResultSetMetadata_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      GetResultSetMetadata_result typedOther = (GetResultSetMetadata_result)other;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetResultSetMetadata_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetResultSetMetadata_resultStandardSchemeFactory implements SchemeFactory {
-      public GetResultSetMetadata_resultStandardScheme getScheme() {
-        return new GetResultSetMetadata_resultStandardScheme();
-      }
-    }
-
-    private static class GetResultSetMetadata_resultStandardScheme extends StandardScheme<GetResultSetMetadata_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetResultSetMetadata_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TGetResultSetMetadataResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetResultSetMetadata_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetResultSetMetadata_resultTupleSchemeFactory implements SchemeFactory {
-      public GetResultSetMetadata_resultTupleScheme getScheme() {
-        return new GetResultSetMetadata_resultTupleScheme();
-      }
-    }
-
-    private static class GetResultSetMetadata_resultTupleScheme extends TupleScheme<GetResultSetMetadata_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetResultSetMetadata_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetResultSetMetadata_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TGetResultSetMetadataResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class FetchResults_args implements org.apache.thrift.TBase<FetchResults_args, FetchResults_args._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("FetchResults_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new FetchResults_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new FetchResults_argsTupleSchemeFactory());
-    }
-
-    private TFetchResultsReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TFetchResultsReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(FetchResults_args.class, metaDataMap);
-    }
-
-    public FetchResults_args() {
-    }
-
-    public FetchResults_args(
-      TFetchResultsReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public FetchResults_args(FetchResults_args other) {
-      if (other.isSetReq()) {
-        this.req = new TFetchResultsReq(other.req);
-      }
-    }
-
-    public FetchResults_args deepCopy() {
-      return new FetchResults_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TFetchResultsReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TFetchResultsReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TFetchResultsReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof FetchResults_args)
-        return this.equals((FetchResults_args)that);
-      return false;
-    }
-
-    public boolean equals(FetchResults_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_req = true && (isSetReq());
-      builder.append(present_req);
-      if (present_req)
-        builder.append(req);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(FetchResults_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      FetchResults_args typedOther = (FetchResults_args)other;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(typedOther.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, typedOther.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("FetchResults_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class FetchResults_argsStandardSchemeFactory implements SchemeFactory {
-      public FetchResults_argsStandardScheme getScheme() {
-        return new FetchResults_argsStandardScheme();
-      }
-    }
-
-    private static class FetchResults_argsStandardScheme extends StandardScheme<FetchResults_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, FetchResults_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TFetchResultsReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, FetchResults_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class FetchResults_argsTupleSchemeFactory implements SchemeFactory {
-      public FetchResults_argsTupleScheme getScheme() {
-        return new FetchResults_argsTupleScheme();
-      }
-    }
-
-    private static class FetchResults_argsTupleScheme extends TupleScheme<FetchResults_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, FetchResults_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, FetchResults_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TFetchResultsReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class FetchResults_result implements org.apache.thrift.TBase<FetchResults_result, FetchResults_result._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("FetchResults_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new FetchResults_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new FetchResults_resultTupleSchemeFactory());
-    }
-
-    private TFetchResultsResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TFetchResultsResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(FetchResults_result.class, metaDataMap);
-    }
-
-    public FetchResults_result() {
-    }
-
-    public FetchResults_result(
-      TFetchResultsResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public FetchResults_result(FetchResults_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TFetchResultsResp(other.success);
-      }
-    }
-
-    public FetchResults_result deepCopy() {
-      return new FetchResults_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TFetchResultsResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TFetchResultsResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TFetchResultsResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof FetchResults_result)
-        return this.equals((FetchResults_result)that);
-      return false;
-    }
-
-    public boolean equals(FetchResults_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_success = true && (isSetSuccess());
-      builder.append(present_success);
-      if (present_success)
-        builder.append(success);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(FetchResults_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      FetchResults_result typedOther = (FetchResults_result)other;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("FetchResults_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class FetchResults_resultStandardSchemeFactory implements SchemeFactory {
-      public FetchResults_resultStandardScheme getScheme() {
-        return new FetchResults_resultStandardScheme();
-      }
-    }
-
-    private static class FetchResults_resultStandardScheme extends StandardScheme<FetchResults_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, FetchResults_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TFetchResultsResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, FetchResults_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class FetchResults_resultTupleSchemeFactory implements SchemeFactory {
-      public FetchResults_resultTupleScheme getScheme() {
-        return new FetchResults_resultTupleScheme();
-      }
-    }
-
-    private static class FetchResults_resultTupleScheme extends TupleScheme<FetchResults_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, FetchResults_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, FetchResults_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TFetchResultsResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetDelegationToken_args implements org.apache.thrift.TBase<GetDelegationToken_args, GetDelegationToken_args._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetDelegationToken_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetDelegationToken_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetDelegationToken_argsTupleSchemeFactory());
-    }
-
-    private TGetDelegationTokenReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetDelegationTokenReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetDelegationToken_args.class, metaDataMap);
-    }
-
-    public GetDelegationToken_args() {
-    }
-
-    public GetDelegationToken_args(
-      TGetDelegationTokenReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetDelegationToken_args(GetDelegationToken_args other) {
-      if (other.isSetReq()) {
-        this.req = new TGetDelegationTokenReq(other.req);
-      }
-    }
-
-    public GetDelegationToken_args deepCopy() {
-      return new GetDelegationToken_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TGetDelegationTokenReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TGetDelegationTokenReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TGetDelegationTokenReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetDelegationToken_args)
-        return this.equals((GetDelegationToken_args)that);
-      return false;
-    }
-
-    public boolean equals(GetDelegationToken_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_req = true && (isSetReq());
-      builder.append(present_req);
-      if (present_req)
-        builder.append(req);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(GetDelegationToken_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      GetDelegationToken_args typedOther = (GetDelegationToken_args)other;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(typedOther.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, typedOther.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetDelegationToken_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetDelegationToken_argsStandardSchemeFactory implements SchemeFactory {
-      public GetDelegationToken_argsStandardScheme getScheme() {
-        return new GetDelegationToken_argsStandardScheme();
-      }
-    }
-
-    private static class GetDelegationToken_argsStandardScheme extends StandardScheme<GetDelegationToken_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetDelegationToken_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TGetDelegationTokenReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetDelegationToken_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetDelegationToken_argsTupleSchemeFactory implements SchemeFactory {
-      public GetDelegationToken_argsTupleScheme getScheme() {
-        return new GetDelegationToken_argsTupleScheme();
-      }
-    }
-
-    private static class GetDelegationToken_argsTupleScheme extends TupleScheme<GetDelegationToken_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetDelegationToken_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetDelegationToken_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TGetDelegationTokenReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetDelegationToken_result implements org.apache.thrift.TBase<GetDelegationToken_result, GetDelegationToken_result._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetDelegationToken_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetDelegationToken_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetDelegationToken_resultTupleSchemeFactory());
-    }
-
-    private TGetDelegationTokenResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetDelegationTokenResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetDelegationToken_result.class, metaDataMap);
-    }
-
-    public GetDelegationToken_result() {
-    }
-
-    public GetDelegationToken_result(
-      TGetDelegationTokenResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetDelegationToken_result(GetDelegationToken_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TGetDelegationTokenResp(other.success);
-      }
-    }
-
-    public GetDelegationToken_result deepCopy() {
-      return new GetDelegationToken_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TGetDelegationTokenResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TGetDelegationTokenResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TGetDelegationTokenResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetDelegationToken_result)
-        return this.equals((GetDelegationToken_result)that);
-      return false;
-    }
-
-    public boolean equals(GetDelegationToken_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_success = true && (isSetSuccess());
-      builder.append(present_success);
-      if (present_success)
-        builder.append(success);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(GetDelegationToken_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      GetDelegationToken_result typedOther = (GetDelegationToken_result)other;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetDelegationToken_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetDelegationToken_resultStandardSchemeFactory implements SchemeFactory {
-      public GetDelegationToken_resultStandardScheme getScheme() {
-        return new GetDelegationToken_resultStandardScheme();
-      }
-    }
-
-    private static class GetDelegationToken_resultStandardScheme extends StandardScheme<GetDelegationToken_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetDelegationToken_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TGetDelegationTokenResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetDelegationToken_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetDelegationToken_resultTupleSchemeFactory implements SchemeFactory {
-      public GetDelegationToken_resultTupleScheme getScheme() {
-        return new GetDelegationToken_resultTupleScheme();
-      }
-    }
-
-    private static class GetDelegationToken_resultTupleScheme extends TupleScheme<GetDelegationToken_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetDelegationToken_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetDelegationToken_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TGetDelegationTokenResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class CancelDelegationToken_args implements org.apache.thrift.TBase<CancelDelegationToken_args, CancelDelegationToken_args._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CancelDelegationToken_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new CancelDelegationToken_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new CancelDelegationToken_argsTupleSchemeFactory());
-    }
-
-    private TCancelDelegationTokenReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TCancelDelegationTokenReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CancelDelegationToken_args.class, metaDataMap);
-    }
-
-    public CancelDelegationToken_args() {
-    }
-
-    public CancelDelegationToken_args(
-      TCancelDelegationTokenReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public CancelDelegationToken_args(CancelDelegationToken_args other) {
-      if (other.isSetReq()) {
-        this.req = new TCancelDelegationTokenReq(other.req);
-      }
-    }
-
-    public CancelDelegationToken_args deepCopy() {
-      return new CancelDelegationToken_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TCancelDelegationTokenReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TCancelDelegationTokenReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TCancelDelegationTokenReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof CancelDelegationToken_args)
-        return this.equals((CancelDelegationToken_args)that);
-      return false;
-    }
-
-    public boolean equals(CancelDelegationToken_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_req = true && (isSetReq());
-      builder.append(present_req);
-      if (present_req)
-        builder.append(req);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(CancelDelegationToken_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      CancelDelegationToken_args typedOther = (CancelDelegationToken_args)other;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(typedOther.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, typedOther.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("CancelDelegationToken_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class CancelDelegationToken_argsStandardSchemeFactory implements SchemeFactory {
-      public CancelDelegationToken_argsStandardScheme getScheme() {
-        return new CancelDelegationToken_argsStandardScheme();
-      }
-    }
-
-    private static class CancelDelegationToken_argsStandardScheme extends StandardScheme<CancelDelegationToken_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, CancelDelegationToken_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TCancelDelegationTokenReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, CancelDelegationToken_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class CancelDelegationToken_argsTupleSchemeFactory implements SchemeFactory {
-      public CancelDelegationToken_argsTupleScheme getScheme() {
-        return new CancelDelegationToken_argsTupleScheme();
-      }
-    }
-
-    private static class CancelDelegationToken_argsTupleScheme extends TupleScheme<CancelDelegationToken_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, CancelDelegationToken_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, CancelDelegationToken_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TCancelDelegationTokenReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class CancelDelegationToken_result implements org.apache.thrift.TBase<CancelDelegationToken_result, CancelDelegationToken_result._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CancelDelegationToken_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new CancelDelegationToken_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new CancelDelegationToken_resultTupleSchemeFactory());
-    }
-
-    private TCancelDelegationTokenResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TCancelDelegationTokenResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CancelDelegationToken_result.class, metaDataMap);
-    }
-
-    public CancelDelegationToken_result() {
-    }
-
-    public CancelDelegationToken_result(
-      TCancelDelegationTokenResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public CancelDelegationToken_result(CancelDelegationToken_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TCancelDelegationTokenResp(other.success);
-      }
-    }
-
-    public CancelDelegationToken_result deepCopy() {
-      return new CancelDelegationToken_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TCancelDelegationTokenResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TCancelDelegationTokenResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TCancelDelegationTokenResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof CancelDelegationToken_result)
-        return this.equals((CancelDelegationToken_result)that);
-      return false;
-    }
-
-    public boolean equals(CancelDelegationToken_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_success = true && (isSetSuccess());
-      builder.append(present_success);
-      if (present_success)
-        builder.append(success);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(CancelDelegationToken_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      CancelDelegationToken_result typedOther = (CancelDelegationToken_result)other;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("CancelDelegationToken_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class CancelDelegationToken_resultStandardSchemeFactory implements SchemeFactory {
-      public CancelDelegationToken_resultStandardScheme getScheme() {
-        return new CancelDelegationToken_resultStandardScheme();
-      }
-    }
-
-    private static class CancelDelegationToken_resultStandardScheme extends StandardScheme<CancelDelegationToken_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, CancelDelegationToken_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TCancelDelegationTokenResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, CancelDelegationToken_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class CancelDelegationToken_resultTupleSchemeFactory implements SchemeFactory {
-      public CancelDelegationToken_resultTupleScheme getScheme() {
-        return new CancelDelegationToken_resultTupleScheme();
-      }
-    }
-
-    private static class CancelDelegationToken_resultTupleScheme extends TupleScheme<CancelDelegationToken_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, CancelDelegationToken_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, CancelDelegationToken_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TCancelDelegationTokenResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class RenewDelegationToken_args implements org.apache.thrift.TBase<RenewDelegationToken_args, RenewDelegationToken_args._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RenewDelegationToken_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new RenewDelegationToken_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new RenewDelegationToken_argsTupleSchemeFactory());
-    }
-
-    private TRenewDelegationTokenReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TRenewDelegationTokenReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(RenewDelegationToken_args.class, metaDataMap);
-    }
-
-    public RenewDelegationToken_args() {
-    }
-
-    public RenewDelegationToken_args(
-      TRenewDelegationTokenReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public RenewDelegationToken_args(RenewDelegationToken_args other) {
-      if (other.isSetReq()) {
-        this.req = new TRenewDelegationTokenReq(other.req);
-      }
-    }
-
-    public RenewDelegationToken_args deepCopy() {
-      return new RenewDelegationToken_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TRenewDelegationTokenReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TRenewDelegationTokenReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TRenewDelegationTokenReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof RenewDelegationToken_args)
-        return this.equals((RenewDelegationToken_args)that);
-      return false;
-    }
-
-    public boolean equals(RenewDelegationToken_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_req = true && (isSetReq());
-      builder.append(present_req);
-      if (present_req)
-        builder.append(req);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(RenewDelegationToken_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      RenewDelegationToken_args typedOther = (RenewDelegationToken_args)other;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(typedOther.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, typedOther.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("RenewDelegationToken_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class RenewDelegationToken_argsStandardSchemeFactory implements SchemeFactory {
-      public RenewDelegationToken_argsStandardScheme getScheme() {
-        return new RenewDelegationToken_argsStandardScheme();
-      }
-    }
-
-    private static class RenewDelegationToken_argsStandardScheme extends StandardScheme<RenewDelegationToken_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, RenewDelegationToken_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TRenewDelegationTokenReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, RenewDelegationToken_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class RenewDelegationToken_argsTupleSchemeFactory implements SchemeFactory {
-      public RenewDelegationToken_argsTupleScheme getScheme() {
-        return new RenewDelegationToken_argsTupleScheme();
-      }
-    }
-
-    private static class RenewDelegationToken_argsTupleScheme extends TupleScheme<RenewDelegationToken_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, RenewDelegationToken_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, RenewDelegationToken_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TRenewDelegationTokenReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class RenewDelegationToken_result implements org.apache.thrift.TBase<RenewDelegationToken_result, RenewDelegationToken_result._Fields>, java.io.Serializable, Cloneable   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RenewDelegationToken_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new RenewDelegationToken_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new RenewDelegationToken_resultTupleSchemeFactory());
-    }
-
-    private TRenewDelegationTokenResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TRenewDelegationTokenResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(RenewDelegationToken_result.class, metaDataMap);
-    }
-
-    public RenewDelegationToken_result() {
-    }
-
-    public RenewDelegationToken_result(
-      TRenewDelegationTokenResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public RenewDelegationToken_result(RenewDelegationToken_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TRenewDelegationTokenResp(other.success);
-      }
-    }
-
-    public RenewDelegationToken_result deepCopy() {
-      return new RenewDelegationToken_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TRenewDelegationTokenResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TRenewDelegationTokenResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TRenewDelegationTokenResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof RenewDelegationToken_result)
-        return this.equals((RenewDelegationToken_result)that);
-      return false;
-    }
-
-    public boolean equals(RenewDelegationToken_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      HashCodeBuilder builder = new HashCodeBuilder();
-
-      boolean present_success = true && (isSetSuccess());
-      builder.append(present_success);
-      if (present_success)
-        builder.append(success);
-
-      return builder.toHashCode();
-    }
-
-    public int compareTo(RenewDelegationToken_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-      RenewDelegationToken_result typedOther = (RenewDelegationToken_result)other;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("RenewDelegationToken_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class RenewDelegationToken_resultStandardSchemeFactory implements SchemeFactory {
-      public RenewDelegationToken_resultStandardScheme getScheme() {
-        return new RenewDelegationToken_resultStandardScheme();
-      }
-    }
-
-    private static class RenewDelegationToken_resultStandardScheme extends StandardScheme<RenewDelegationToken_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, RenewDelegationToken_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TRenewDelegationTokenResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, RenewDelegationToken_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class RenewDelegationToken_resultTupleSchemeFactory implements SchemeFactory {
-      public RenewDelegationToken_resultTupleScheme getScheme() {
-        return new RenewDelegationToken_resultTupleScheme();
-      }
-    }
-
-    private static class RenewDelegationToken_resultTupleScheme extends TupleScheme<RenewDelegationToken_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, RenewDelegationToken_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, RenewDelegationToken_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TRenewDelegationTokenResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-}
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCLIServiceConstants.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCLIServiceConstants.java
deleted file mode 100644
index 25a38b178428a..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCLIServiceConstants.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TCLIServiceConstants {
-
-  public static final Set<TTypeId> PRIMITIVE_TYPES = new HashSet<TTypeId>();
-  static {
-    PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.BOOLEAN_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.TINYINT_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.SMALLINT_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.INT_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.BIGINT_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.FLOAT_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.DOUBLE_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.STRING_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.TIMESTAMP_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.BINARY_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.DECIMAL_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.NULL_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.DATE_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.VARCHAR_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.CHAR_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.INTERVAL_YEAR_MONTH_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.INTERVAL_DAY_TIME_TYPE);
-  }
-
-  public static final Set<TTypeId> COMPLEX_TYPES = new HashSet<TTypeId>();
-  static {
-    COMPLEX_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.ARRAY_TYPE);
-    COMPLEX_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.MAP_TYPE);
-    COMPLEX_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.STRUCT_TYPE);
-    COMPLEX_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.UNION_TYPE);
-    COMPLEX_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.USER_DEFINED_TYPE);
-  }
-
-  public static final Set<TTypeId> COLLECTION_TYPES = new HashSet<TTypeId>();
-  static {
-    COLLECTION_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.ARRAY_TYPE);
-    COLLECTION_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.MAP_TYPE);
-  }
-
-  public static final Map<TTypeId,String> TYPE_NAMES = new HashMap<TTypeId,String>();
-  static {
-    TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.BOOLEAN_TYPE, "BOOLEAN");
-    TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.TINYINT_TYPE, "TINYINT");
-    TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.SMALLINT_TYPE, "SMALLINT");
-    TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.INT_TYPE, "INT");
-    TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.BIGINT_TYPE, "BIGINT");
-    TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.FLOAT_TYPE, "FLOAT");
-    TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.DOUBLE_TYPE, "DOUBLE");
-    TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.STRING_TYPE, "STRING");
-    TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.TIMESTAMP_TYPE, "TIMESTAMP");
-    TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.BINARY_TYPE, "BINARY");
-    TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.ARRAY_TYPE, "ARRAY");
-    TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.MAP_TYPE, "MAP");
-    TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.STRUCT_TYPE, "STRUCT");
-    TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.UNION_TYPE, "UNIONTYPE");
-    TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.DECIMAL_TYPE, "DECIMAL");
-    TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.NULL_TYPE, "NULL");
-    TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.DATE_TYPE, "DATE");
-    TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.VARCHAR_TYPE, "VARCHAR");
-    TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.CHAR_TYPE, "CHAR");
-    TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.INTERVAL_YEAR_MONTH_TYPE, "INTERVAL_YEAR_MONTH");
-    TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.INTERVAL_DAY_TIME_TYPE, "INTERVAL_DAY_TIME");
-  }
-
-  public static final String CHARACTER_MAXIMUM_LENGTH = "characterMaximumLength";
-
-  public static final String PRECISION = "precision";
-
-  public static final String SCALE = "scale";
-
-}
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCancelDelegationTokenReq.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCancelDelegationTokenReq.java
deleted file mode 100644
index e23fcdd77a1a4..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCancelDelegationTokenReq.java
+++ /dev/null
@@ -1,491 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TCancelDelegationTokenReq implements org.apache.thrift.TBase<TCancelDelegationTokenReq, TCancelDelegationTokenReq._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TCancelDelegationTokenReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField DELEGATION_TOKEN_FIELD_DESC = new org.apache.thrift.protocol.TField("delegationToken", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TCancelDelegationTokenReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TCancelDelegationTokenReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-  private String delegationToken; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle"),
-    DELEGATION_TOKEN((short)2, "delegationToken");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        case 2: // DELEGATION_TOKEN
-          return DELEGATION_TOKEN;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    tmpMap.put(_Fields.DELEGATION_TOKEN, new org.apache.thrift.meta_data.FieldMetaData("delegationToken", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TCancelDelegationTokenReq.class, metaDataMap);
-  }
-
-  public TCancelDelegationTokenReq() {
-  }
-
-  public TCancelDelegationTokenReq(
-    TSessionHandle sessionHandle,
-    String delegationToken)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-    this.delegationToken = delegationToken;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TCancelDelegationTokenReq(TCancelDelegationTokenReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-    if (other.isSetDelegationToken()) {
-      this.delegationToken = other.delegationToken;
-    }
-  }
-
-  public TCancelDelegationTokenReq deepCopy() {
-    return new TCancelDelegationTokenReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-    this.delegationToken = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public String getDelegationToken() {
-    return this.delegationToken;
-  }
-
-  public void setDelegationToken(String delegationToken) {
-    this.delegationToken = delegationToken;
-  }
-
-  public void unsetDelegationToken() {
-    this.delegationToken = null;
-  }
-
-  /** Returns true if field delegationToken is set (has been assigned a value) and false otherwise */
-  public boolean isSetDelegationToken() {
-    return this.delegationToken != null;
-  }
-
-  public void setDelegationTokenIsSet(boolean value) {
-    if (!value) {
-      this.delegationToken = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    case DELEGATION_TOKEN:
-      if (value == null) {
-        unsetDelegationToken();
-      } else {
-        setDelegationToken((String)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    case DELEGATION_TOKEN:
-      return getDelegationToken();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    case DELEGATION_TOKEN:
-      return isSetDelegationToken();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TCancelDelegationTokenReq)
-      return this.equals((TCancelDelegationTokenReq)that);
-    return false;
-  }
-
-  public boolean equals(TCancelDelegationTokenReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    boolean this_present_delegationToken = true && this.isSetDelegationToken();
-    boolean that_present_delegationToken = true && that.isSetDelegationToken();
-    if (this_present_delegationToken || that_present_delegationToken) {
-      if (!(this_present_delegationToken && that_present_delegationToken))
-        return false;
-      if (!this.delegationToken.equals(that.delegationToken))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    builder.append(present_sessionHandle);
-    if (present_sessionHandle)
-      builder.append(sessionHandle);
-
-    boolean present_delegationToken = true && (isSetDelegationToken());
-    builder.append(present_delegationToken);
-    if (present_delegationToken)
-      builder.append(delegationToken);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TCancelDelegationTokenReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TCancelDelegationTokenReq typedOther = (TCancelDelegationTokenReq)other;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(typedOther.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, typedOther.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetDelegationToken()).compareTo(typedOther.isSetDelegationToken());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetDelegationToken()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.delegationToken, typedOther.delegationToken);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TCancelDelegationTokenReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("delegationToken:");
-    if (this.delegationToken == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.delegationToken);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    if (!isSetDelegationToken()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'delegationToken' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TCancelDelegationTokenReqStandardSchemeFactory implements SchemeFactory {
-    public TCancelDelegationTokenReqStandardScheme getScheme() {
-      return new TCancelDelegationTokenReqStandardScheme();
-    }
-  }
-
-  private static class TCancelDelegationTokenReqStandardScheme extends StandardScheme<TCancelDelegationTokenReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TCancelDelegationTokenReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // DELEGATION_TOKEN
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.delegationToken = iprot.readString();
-              struct.setDelegationTokenIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TCancelDelegationTokenReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.delegationToken != null) {
-        oprot.writeFieldBegin(DELEGATION_TOKEN_FIELD_DESC);
-        oprot.writeString(struct.delegationToken);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TCancelDelegationTokenReqTupleSchemeFactory implements SchemeFactory {
-    public TCancelDelegationTokenReqTupleScheme getScheme() {
-      return new TCancelDelegationTokenReqTupleScheme();
-    }
-  }
-
-  private static class TCancelDelegationTokenReqTupleScheme extends TupleScheme<TCancelDelegationTokenReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TCancelDelegationTokenReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-      oprot.writeString(struct.delegationToken);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TCancelDelegationTokenReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-      struct.delegationToken = iprot.readString();
-      struct.setDelegationTokenIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCancelDelegationTokenResp.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCancelDelegationTokenResp.java
deleted file mode 100644
index 77c9ee77ec59b..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCancelDelegationTokenResp.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TCancelDelegationTokenResp implements org.apache.thrift.TBase<TCancelDelegationTokenResp, TCancelDelegationTokenResp._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TCancelDelegationTokenResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TCancelDelegationTokenRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TCancelDelegationTokenRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TCancelDelegationTokenResp.class, metaDataMap);
-  }
-
-  public TCancelDelegationTokenResp() {
-  }
-
-  public TCancelDelegationTokenResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TCancelDelegationTokenResp(TCancelDelegationTokenResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-  }
-
-  public TCancelDelegationTokenResp deepCopy() {
-    return new TCancelDelegationTokenResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TCancelDelegationTokenResp)
-      return this.equals((TCancelDelegationTokenResp)that);
-    return false;
-  }
-
-  public boolean equals(TCancelDelegationTokenResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_status = true && (isSetStatus());
-    builder.append(present_status);
-    if (present_status)
-      builder.append(status);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TCancelDelegationTokenResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TCancelDelegationTokenResp typedOther = (TCancelDelegationTokenResp)other;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(typedOther.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, typedOther.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TCancelDelegationTokenResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TCancelDelegationTokenRespStandardSchemeFactory implements SchemeFactory {
-    public TCancelDelegationTokenRespStandardScheme getScheme() {
-      return new TCancelDelegationTokenRespStandardScheme();
-    }
-  }
-
-  private static class TCancelDelegationTokenRespStandardScheme extends StandardScheme<TCancelDelegationTokenResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TCancelDelegationTokenResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TCancelDelegationTokenResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TCancelDelegationTokenRespTupleSchemeFactory implements SchemeFactory {
-    public TCancelDelegationTokenRespTupleScheme getScheme() {
-      return new TCancelDelegationTokenRespTupleScheme();
-    }
-  }
-
-  private static class TCancelDelegationTokenRespTupleScheme extends TupleScheme<TCancelDelegationTokenResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TCancelDelegationTokenResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TCancelDelegationTokenResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCancelOperationReq.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCancelOperationReq.java
deleted file mode 100644
index 45eac48ab12d3..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCancelOperationReq.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TCancelOperationReq implements org.apache.thrift.TBase<TCancelOperationReq, TCancelOperationReq._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TCancelOperationReq");
-
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TCancelOperationReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TCancelOperationReqTupleSchemeFactory());
-  }
-
-  private TOperationHandle operationHandle; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    OPERATION_HANDLE((short)1, "operationHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TCancelOperationReq.class, metaDataMap);
-  }
-
-  public TCancelOperationReq() {
-  }
-
-  public TCancelOperationReq(
-    TOperationHandle operationHandle)
-  {
-    this();
-    this.operationHandle = operationHandle;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TCancelOperationReq(TCancelOperationReq other) {
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-  }
-
-  public TCancelOperationReq deepCopy() {
-    return new TCancelOperationReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.operationHandle = null;
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TCancelOperationReq)
-      return this.equals((TCancelOperationReq)that);
-    return false;
-  }
-
-  public boolean equals(TCancelOperationReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    builder.append(present_operationHandle);
-    if (present_operationHandle)
-      builder.append(operationHandle);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TCancelOperationReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TCancelOperationReq typedOther = (TCancelOperationReq)other;
-
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(typedOther.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, typedOther.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TCancelOperationReq(");
-    boolean first = true;
-
-    sb.append("operationHandle:");
-    if (this.operationHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.operationHandle);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetOperationHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'operationHandle' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TCancelOperationReqStandardSchemeFactory implements SchemeFactory {
-    public TCancelOperationReqStandardScheme getScheme() {
-      return new TCancelOperationReqStandardScheme();
-    }
-  }
-
-  private static class TCancelOperationReqStandardScheme extends StandardScheme<TCancelOperationReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TCancelOperationReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TCancelOperationReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.operationHandle != null) {
-        oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-        struct.operationHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TCancelOperationReqTupleSchemeFactory implements SchemeFactory {
-    public TCancelOperationReqTupleScheme getScheme() {
-      return new TCancelOperationReqTupleScheme();
-    }
-  }
-
-  private static class TCancelOperationReqTupleScheme extends TupleScheme<TCancelOperationReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TCancelOperationReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.operationHandle.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TCancelOperationReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.operationHandle = new TOperationHandle();
-      struct.operationHandle.read(iprot);
-      struct.setOperationHandleIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCancelOperationResp.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCancelOperationResp.java
deleted file mode 100644
index 2a39414d601aa..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCancelOperationResp.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TCancelOperationResp implements org.apache.thrift.TBase<TCancelOperationResp, TCancelOperationResp._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TCancelOperationResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TCancelOperationRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TCancelOperationRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TCancelOperationResp.class, metaDataMap);
-  }
-
-  public TCancelOperationResp() {
-  }
-
-  public TCancelOperationResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TCancelOperationResp(TCancelOperationResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-  }
-
-  public TCancelOperationResp deepCopy() {
-    return new TCancelOperationResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TCancelOperationResp)
-      return this.equals((TCancelOperationResp)that);
-    return false;
-  }
-
-  public boolean equals(TCancelOperationResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_status = true && (isSetStatus());
-    builder.append(present_status);
-    if (present_status)
-      builder.append(status);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TCancelOperationResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TCancelOperationResp typedOther = (TCancelOperationResp)other;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(typedOther.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, typedOther.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TCancelOperationResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TCancelOperationRespStandardSchemeFactory implements SchemeFactory {
-    public TCancelOperationRespStandardScheme getScheme() {
-      return new TCancelOperationRespStandardScheme();
-    }
-  }
-
-  private static class TCancelOperationRespStandardScheme extends StandardScheme<TCancelOperationResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TCancelOperationResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TCancelOperationResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TCancelOperationRespTupleSchemeFactory implements SchemeFactory {
-    public TCancelOperationRespTupleScheme getScheme() {
-      return new TCancelOperationRespTupleScheme();
-    }
-  }
-
-  private static class TCancelOperationRespTupleScheme extends TupleScheme<TCancelOperationResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TCancelOperationResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TCancelOperationResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCloseOperationReq.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCloseOperationReq.java
deleted file mode 100644
index 0cbb7ccced073..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCloseOperationReq.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TCloseOperationReq implements org.apache.thrift.TBase<TCloseOperationReq, TCloseOperationReq._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TCloseOperationReq");
-
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TCloseOperationReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TCloseOperationReqTupleSchemeFactory());
-  }
-
-  private TOperationHandle operationHandle; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    OPERATION_HANDLE((short)1, "operationHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TCloseOperationReq.class, metaDataMap);
-  }
-
-  public TCloseOperationReq() {
-  }
-
-  public TCloseOperationReq(
-    TOperationHandle operationHandle)
-  {
-    this();
-    this.operationHandle = operationHandle;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TCloseOperationReq(TCloseOperationReq other) {
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-  }
-
-  public TCloseOperationReq deepCopy() {
-    return new TCloseOperationReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.operationHandle = null;
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TCloseOperationReq)
-      return this.equals((TCloseOperationReq)that);
-    return false;
-  }
-
-  public boolean equals(TCloseOperationReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    builder.append(present_operationHandle);
-    if (present_operationHandle)
-      builder.append(operationHandle);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TCloseOperationReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TCloseOperationReq typedOther = (TCloseOperationReq)other;
-
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(typedOther.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, typedOther.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TCloseOperationReq(");
-    boolean first = true;
-
-    sb.append("operationHandle:");
-    if (this.operationHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.operationHandle);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetOperationHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'operationHandle' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TCloseOperationReqStandardSchemeFactory implements SchemeFactory {
-    public TCloseOperationReqStandardScheme getScheme() {
-      return new TCloseOperationReqStandardScheme();
-    }
-  }
-
-  private static class TCloseOperationReqStandardScheme extends StandardScheme<TCloseOperationReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TCloseOperationReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TCloseOperationReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.operationHandle != null) {
-        oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-        struct.operationHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TCloseOperationReqTupleSchemeFactory implements SchemeFactory {
-    public TCloseOperationReqTupleScheme getScheme() {
-      return new TCloseOperationReqTupleScheme();
-    }
-  }
-
-  private static class TCloseOperationReqTupleScheme extends TupleScheme<TCloseOperationReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TCloseOperationReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.operationHandle.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TCloseOperationReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.operationHandle = new TOperationHandle();
-      struct.operationHandle.read(iprot);
-      struct.setOperationHandleIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCloseOperationResp.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCloseOperationResp.java
deleted file mode 100644
index 7334d67173d7b..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCloseOperationResp.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TCloseOperationResp implements org.apache.thrift.TBase<TCloseOperationResp, TCloseOperationResp._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TCloseOperationResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TCloseOperationRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TCloseOperationRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TCloseOperationResp.class, metaDataMap);
-  }
-
-  public TCloseOperationResp() {
-  }
-
-  public TCloseOperationResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TCloseOperationResp(TCloseOperationResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-  }
-
-  public TCloseOperationResp deepCopy() {
-    return new TCloseOperationResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TCloseOperationResp)
-      return this.equals((TCloseOperationResp)that);
-    return false;
-  }
-
-  public boolean equals(TCloseOperationResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_status = true && (isSetStatus());
-    builder.append(present_status);
-    if (present_status)
-      builder.append(status);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TCloseOperationResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TCloseOperationResp typedOther = (TCloseOperationResp)other;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(typedOther.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, typedOther.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TCloseOperationResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TCloseOperationRespStandardSchemeFactory implements SchemeFactory {
-    public TCloseOperationRespStandardScheme getScheme() {
-      return new TCloseOperationRespStandardScheme();
-    }
-  }
-
-  private static class TCloseOperationRespStandardScheme extends StandardScheme<TCloseOperationResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TCloseOperationResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TCloseOperationResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TCloseOperationRespTupleSchemeFactory implements SchemeFactory {
-    public TCloseOperationRespTupleScheme getScheme() {
-      return new TCloseOperationRespTupleScheme();
-    }
-  }
-
-  private static class TCloseOperationRespTupleScheme extends TupleScheme<TCloseOperationResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TCloseOperationResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TCloseOperationResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCloseSessionReq.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCloseSessionReq.java
deleted file mode 100644
index 027e8295436b0..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCloseSessionReq.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TCloseSessionReq implements org.apache.thrift.TBase<TCloseSessionReq, TCloseSessionReq._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TCloseSessionReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TCloseSessionReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TCloseSessionReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TCloseSessionReq.class, metaDataMap);
-  }
-
-  public TCloseSessionReq() {
-  }
-
-  public TCloseSessionReq(
-    TSessionHandle sessionHandle)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TCloseSessionReq(TCloseSessionReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-  }
-
-  public TCloseSessionReq deepCopy() {
-    return new TCloseSessionReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TCloseSessionReq)
-      return this.equals((TCloseSessionReq)that);
-    return false;
-  }
-
-  public boolean equals(TCloseSessionReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    builder.append(present_sessionHandle);
-    if (present_sessionHandle)
-      builder.append(sessionHandle);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TCloseSessionReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TCloseSessionReq typedOther = (TCloseSessionReq)other;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(typedOther.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, typedOther.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TCloseSessionReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TCloseSessionReqStandardSchemeFactory implements SchemeFactory {
-    public TCloseSessionReqStandardScheme getScheme() {
-      return new TCloseSessionReqStandardScheme();
-    }
-  }
-
-  private static class TCloseSessionReqStandardScheme extends StandardScheme<TCloseSessionReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TCloseSessionReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TCloseSessionReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TCloseSessionReqTupleSchemeFactory implements SchemeFactory {
-    public TCloseSessionReqTupleScheme getScheme() {
-      return new TCloseSessionReqTupleScheme();
-    }
-  }
-
-  private static class TCloseSessionReqTupleScheme extends TupleScheme<TCloseSessionReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TCloseSessionReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TCloseSessionReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCloseSessionResp.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCloseSessionResp.java
deleted file mode 100644
index 168c8fc775e33..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCloseSessionResp.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TCloseSessionResp implements org.apache.thrift.TBase<TCloseSessionResp, TCloseSessionResp._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TCloseSessionResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TCloseSessionRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TCloseSessionRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TCloseSessionResp.class, metaDataMap);
-  }
-
-  public TCloseSessionResp() {
-  }
-
-  public TCloseSessionResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TCloseSessionResp(TCloseSessionResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-  }
-
-  public TCloseSessionResp deepCopy() {
-    return new TCloseSessionResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TCloseSessionResp)
-      return this.equals((TCloseSessionResp)that);
-    return false;
-  }
-
-  public boolean equals(TCloseSessionResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_status = true && (isSetStatus());
-    builder.append(present_status);
-    if (present_status)
-      builder.append(status);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TCloseSessionResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TCloseSessionResp typedOther = (TCloseSessionResp)other;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(typedOther.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, typedOther.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TCloseSessionResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TCloseSessionRespStandardSchemeFactory implements SchemeFactory {
-    public TCloseSessionRespStandardScheme getScheme() {
-      return new TCloseSessionRespStandardScheme();
-    }
-  }
-
-  private static class TCloseSessionRespStandardScheme extends StandardScheme<TCloseSessionResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TCloseSessionResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TCloseSessionResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TCloseSessionRespTupleSchemeFactory implements SchemeFactory {
-    public TCloseSessionRespTupleScheme getScheme() {
-      return new TCloseSessionRespTupleScheme();
-    }
-  }
-
-  private static class TCloseSessionRespTupleScheme extends TupleScheme<TCloseSessionResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TCloseSessionResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TCloseSessionResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TColumn.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TColumn.java
deleted file mode 100644
index fc2171dc99e4c..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TColumn.java
+++ /dev/null
@@ -1,732 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TColumn extends org.apache.thrift.TUnion<TColumn, TColumn._Fields> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TColumn");
-  private static final org.apache.thrift.protocol.TField BOOL_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("boolVal", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField BYTE_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("byteVal", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-  private static final org.apache.thrift.protocol.TField I16_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("i16Val", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-  private static final org.apache.thrift.protocol.TField I32_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("i32Val", org.apache.thrift.protocol.TType.STRUCT, (short)4);
-  private static final org.apache.thrift.protocol.TField I64_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("i64Val", org.apache.thrift.protocol.TType.STRUCT, (short)5);
-  private static final org.apache.thrift.protocol.TField DOUBLE_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("doubleVal", org.apache.thrift.protocol.TType.STRUCT, (short)6);
-  private static final org.apache.thrift.protocol.TField STRING_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("stringVal", org.apache.thrift.protocol.TType.STRUCT, (short)7);
-  private static final org.apache.thrift.protocol.TField BINARY_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("binaryVal", org.apache.thrift.protocol.TType.STRUCT, (short)8);
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    BOOL_VAL((short)1, "boolVal"),
-    BYTE_VAL((short)2, "byteVal"),
-    I16_VAL((short)3, "i16Val"),
-    I32_VAL((short)4, "i32Val"),
-    I64_VAL((short)5, "i64Val"),
-    DOUBLE_VAL((short)6, "doubleVal"),
-    STRING_VAL((short)7, "stringVal"),
-    BINARY_VAL((short)8, "binaryVal");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // BOOL_VAL
-          return BOOL_VAL;
-        case 2: // BYTE_VAL
-          return BYTE_VAL;
-        case 3: // I16_VAL
-          return I16_VAL;
-        case 4: // I32_VAL
-          return I32_VAL;
-        case 5: // I64_VAL
-          return I64_VAL;
-        case 6: // DOUBLE_VAL
-          return DOUBLE_VAL;
-        case 7: // STRING_VAL
-          return STRING_VAL;
-        case 8: // BINARY_VAL
-          return BINARY_VAL;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.BOOL_VAL, new org.apache.thrift.meta_data.FieldMetaData("boolVal", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TBoolColumn.class)));
-    tmpMap.put(_Fields.BYTE_VAL, new org.apache.thrift.meta_data.FieldMetaData("byteVal", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TByteColumn.class)));
-    tmpMap.put(_Fields.I16_VAL, new org.apache.thrift.meta_data.FieldMetaData("i16Val", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TI16Column.class)));
-    tmpMap.put(_Fields.I32_VAL, new org.apache.thrift.meta_data.FieldMetaData("i32Val", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TI32Column.class)));
-    tmpMap.put(_Fields.I64_VAL, new org.apache.thrift.meta_data.FieldMetaData("i64Val", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TI64Column.class)));
-    tmpMap.put(_Fields.DOUBLE_VAL, new org.apache.thrift.meta_data.FieldMetaData("doubleVal", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TDoubleColumn.class)));
-    tmpMap.put(_Fields.STRING_VAL, new org.apache.thrift.meta_data.FieldMetaData("stringVal", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStringColumn.class)));
-    tmpMap.put(_Fields.BINARY_VAL, new org.apache.thrift.meta_data.FieldMetaData("binaryVal", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TBinaryColumn.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TColumn.class, metaDataMap);
-  }
-
-  public TColumn() {
-    super();
-  }
-
-  public TColumn(TColumn._Fields setField, Object value) {
-    super(setField, value);
-  }
-
-  public TColumn(TColumn other) {
-    super(other);
-  }
-  public TColumn deepCopy() {
-    return new TColumn(this);
-  }
-
-  public static TColumn boolVal(TBoolColumn value) {
-    TColumn x = new TColumn();
-    x.setBoolVal(value);
-    return x;
-  }
-
-  public static TColumn byteVal(TByteColumn value) {
-    TColumn x = new TColumn();
-    x.setByteVal(value);
-    return x;
-  }
-
-  public static TColumn i16Val(TI16Column value) {
-    TColumn x = new TColumn();
-    x.setI16Val(value);
-    return x;
-  }
-
-  public static TColumn i32Val(TI32Column value) {
-    TColumn x = new TColumn();
-    x.setI32Val(value);
-    return x;
-  }
-
-  public static TColumn i64Val(TI64Column value) {
-    TColumn x = new TColumn();
-    x.setI64Val(value);
-    return x;
-  }
-
-  public static TColumn doubleVal(TDoubleColumn value) {
-    TColumn x = new TColumn();
-    x.setDoubleVal(value);
-    return x;
-  }
-
-  public static TColumn stringVal(TStringColumn value) {
-    TColumn x = new TColumn();
-    x.setStringVal(value);
-    return x;
-  }
-
-  public static TColumn binaryVal(TBinaryColumn value) {
-    TColumn x = new TColumn();
-    x.setBinaryVal(value);
-    return x;
-  }
-
-
-  @Override
-  protected void checkType(_Fields setField, Object value) throws ClassCastException {
-    switch (setField) {
-      case BOOL_VAL:
-        if (value instanceof TBoolColumn) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TBoolColumn for field 'boolVal', but got " + value.getClass().getSimpleName());
-      case BYTE_VAL:
-        if (value instanceof TByteColumn) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TByteColumn for field 'byteVal', but got " + value.getClass().getSimpleName());
-      case I16_VAL:
-        if (value instanceof TI16Column) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TI16Column for field 'i16Val', but got " + value.getClass().getSimpleName());
-      case I32_VAL:
-        if (value instanceof TI32Column) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TI32Column for field 'i32Val', but got " + value.getClass().getSimpleName());
-      case I64_VAL:
-        if (value instanceof TI64Column) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TI64Column for field 'i64Val', but got " + value.getClass().getSimpleName());
-      case DOUBLE_VAL:
-        if (value instanceof TDoubleColumn) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TDoubleColumn for field 'doubleVal', but got " + value.getClass().getSimpleName());
-      case STRING_VAL:
-        if (value instanceof TStringColumn) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TStringColumn for field 'stringVal', but got " + value.getClass().getSimpleName());
-      case BINARY_VAL:
-        if (value instanceof TBinaryColumn) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TBinaryColumn for field 'binaryVal', but got " + value.getClass().getSimpleName());
-      default:
-        throw new IllegalArgumentException("Unknown field id " + setField);
-    }
-  }
-
-  @Override
-  protected Object standardSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TField field) throws org.apache.thrift.TException {
-    _Fields setField = _Fields.findByThriftId(field.id);
-    if (setField != null) {
-      switch (setField) {
-        case BOOL_VAL:
-          if (field.type == BOOL_VAL_FIELD_DESC.type) {
-            TBoolColumn boolVal;
-            boolVal = new TBoolColumn();
-            boolVal.read(iprot);
-            return boolVal;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case BYTE_VAL:
-          if (field.type == BYTE_VAL_FIELD_DESC.type) {
-            TByteColumn byteVal;
-            byteVal = new TByteColumn();
-            byteVal.read(iprot);
-            return byteVal;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case I16_VAL:
-          if (field.type == I16_VAL_FIELD_DESC.type) {
-            TI16Column i16Val;
-            i16Val = new TI16Column();
-            i16Val.read(iprot);
-            return i16Val;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case I32_VAL:
-          if (field.type == I32_VAL_FIELD_DESC.type) {
-            TI32Column i32Val;
-            i32Val = new TI32Column();
-            i32Val.read(iprot);
-            return i32Val;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case I64_VAL:
-          if (field.type == I64_VAL_FIELD_DESC.type) {
-            TI64Column i64Val;
-            i64Val = new TI64Column();
-            i64Val.read(iprot);
-            return i64Val;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case DOUBLE_VAL:
-          if (field.type == DOUBLE_VAL_FIELD_DESC.type) {
-            TDoubleColumn doubleVal;
-            doubleVal = new TDoubleColumn();
-            doubleVal.read(iprot);
-            return doubleVal;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case STRING_VAL:
-          if (field.type == STRING_VAL_FIELD_DESC.type) {
-            TStringColumn stringVal;
-            stringVal = new TStringColumn();
-            stringVal.read(iprot);
-            return stringVal;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case BINARY_VAL:
-          if (field.type == BINARY_VAL_FIELD_DESC.type) {
-            TBinaryColumn binaryVal;
-            binaryVal = new TBinaryColumn();
-            binaryVal.read(iprot);
-            return binaryVal;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        default:
-          throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!");
-      }
-    } else {
-      return null;
-    }
-  }
-
-  @Override
-  protected void standardSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    switch (setField_) {
-      case BOOL_VAL:
-        TBoolColumn boolVal = (TBoolColumn)value_;
-        boolVal.write(oprot);
-        return;
-      case BYTE_VAL:
-        TByteColumn byteVal = (TByteColumn)value_;
-        byteVal.write(oprot);
-        return;
-      case I16_VAL:
-        TI16Column i16Val = (TI16Column)value_;
-        i16Val.write(oprot);
-        return;
-      case I32_VAL:
-        TI32Column i32Val = (TI32Column)value_;
-        i32Val.write(oprot);
-        return;
-      case I64_VAL:
-        TI64Column i64Val = (TI64Column)value_;
-        i64Val.write(oprot);
-        return;
-      case DOUBLE_VAL:
-        TDoubleColumn doubleVal = (TDoubleColumn)value_;
-        doubleVal.write(oprot);
-        return;
-      case STRING_VAL:
-        TStringColumn stringVal = (TStringColumn)value_;
-        stringVal.write(oprot);
-        return;
-      case BINARY_VAL:
-        TBinaryColumn binaryVal = (TBinaryColumn)value_;
-        binaryVal.write(oprot);
-        return;
-      default:
-        throw new IllegalStateException("Cannot write union with unknown field " + setField_);
-    }
-  }
-
-  @Override
-  protected Object tupleSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, short fieldID) throws org.apache.thrift.TException {
-    _Fields setField = _Fields.findByThriftId(fieldID);
-    if (setField != null) {
-      switch (setField) {
-        case BOOL_VAL:
-          TBoolColumn boolVal;
-          boolVal = new TBoolColumn();
-          boolVal.read(iprot);
-          return boolVal;
-        case BYTE_VAL:
-          TByteColumn byteVal;
-          byteVal = new TByteColumn();
-          byteVal.read(iprot);
-          return byteVal;
-        case I16_VAL:
-          TI16Column i16Val;
-          i16Val = new TI16Column();
-          i16Val.read(iprot);
-          return i16Val;
-        case I32_VAL:
-          TI32Column i32Val;
-          i32Val = new TI32Column();
-          i32Val.read(iprot);
-          return i32Val;
-        case I64_VAL:
-          TI64Column i64Val;
-          i64Val = new TI64Column();
-          i64Val.read(iprot);
-          return i64Val;
-        case DOUBLE_VAL:
-          TDoubleColumn doubleVal;
-          doubleVal = new TDoubleColumn();
-          doubleVal.read(iprot);
-          return doubleVal;
-        case STRING_VAL:
-          TStringColumn stringVal;
-          stringVal = new TStringColumn();
-          stringVal.read(iprot);
-          return stringVal;
-        case BINARY_VAL:
-          TBinaryColumn binaryVal;
-          binaryVal = new TBinaryColumn();
-          binaryVal.read(iprot);
-          return binaryVal;
-        default:
-          throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!");
-      }
-    } else {
-      throw new TProtocolException("Couldn't find a field with field id " + fieldID);
-    }
-  }
-
-  @Override
-  protected void tupleSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    switch (setField_) {
-      case BOOL_VAL:
-        TBoolColumn boolVal = (TBoolColumn)value_;
-        boolVal.write(oprot);
-        return;
-      case BYTE_VAL:
-        TByteColumn byteVal = (TByteColumn)value_;
-        byteVal.write(oprot);
-        return;
-      case I16_VAL:
-        TI16Column i16Val = (TI16Column)value_;
-        i16Val.write(oprot);
-        return;
-      case I32_VAL:
-        TI32Column i32Val = (TI32Column)value_;
-        i32Val.write(oprot);
-        return;
-      case I64_VAL:
-        TI64Column i64Val = (TI64Column)value_;
-        i64Val.write(oprot);
-        return;
-      case DOUBLE_VAL:
-        TDoubleColumn doubleVal = (TDoubleColumn)value_;
-        doubleVal.write(oprot);
-        return;
-      case STRING_VAL:
-        TStringColumn stringVal = (TStringColumn)value_;
-        stringVal.write(oprot);
-        return;
-      case BINARY_VAL:
-        TBinaryColumn binaryVal = (TBinaryColumn)value_;
-        binaryVal.write(oprot);
-        return;
-      default:
-        throw new IllegalStateException("Cannot write union with unknown field " + setField_);
-    }
-  }
-
-  @Override
-  protected org.apache.thrift.protocol.TField getFieldDesc(_Fields setField) {
-    switch (setField) {
-      case BOOL_VAL:
-        return BOOL_VAL_FIELD_DESC;
-      case BYTE_VAL:
-        return BYTE_VAL_FIELD_DESC;
-      case I16_VAL:
-        return I16_VAL_FIELD_DESC;
-      case I32_VAL:
-        return I32_VAL_FIELD_DESC;
-      case I64_VAL:
-        return I64_VAL_FIELD_DESC;
-      case DOUBLE_VAL:
-        return DOUBLE_VAL_FIELD_DESC;
-      case STRING_VAL:
-        return STRING_VAL_FIELD_DESC;
-      case BINARY_VAL:
-        return BINARY_VAL_FIELD_DESC;
-      default:
-        throw new IllegalArgumentException("Unknown field id " + setField);
-    }
-  }
-
-  @Override
-  protected org.apache.thrift.protocol.TStruct getStructDesc() {
-    return STRUCT_DESC;
-  }
-
-  @Override
-  protected _Fields enumForId(short id) {
-    return _Fields.findByThriftIdOrThrow(id);
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-
-  public TBoolColumn getBoolVal() {
-    if (getSetField() == _Fields.BOOL_VAL) {
-      return (TBoolColumn)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'boolVal' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setBoolVal(TBoolColumn value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.BOOL_VAL;
-    value_ = value;
-  }
-
-  public TByteColumn getByteVal() {
-    if (getSetField() == _Fields.BYTE_VAL) {
-      return (TByteColumn)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'byteVal' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setByteVal(TByteColumn value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.BYTE_VAL;
-    value_ = value;
-  }
-
-  public TI16Column getI16Val() {
-    if (getSetField() == _Fields.I16_VAL) {
-      return (TI16Column)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'i16Val' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setI16Val(TI16Column value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.I16_VAL;
-    value_ = value;
-  }
-
-  public TI32Column getI32Val() {
-    if (getSetField() == _Fields.I32_VAL) {
-      return (TI32Column)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'i32Val' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setI32Val(TI32Column value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.I32_VAL;
-    value_ = value;
-  }
-
-  public TI64Column getI64Val() {
-    if (getSetField() == _Fields.I64_VAL) {
-      return (TI64Column)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'i64Val' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setI64Val(TI64Column value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.I64_VAL;
-    value_ = value;
-  }
-
-  public TDoubleColumn getDoubleVal() {
-    if (getSetField() == _Fields.DOUBLE_VAL) {
-      return (TDoubleColumn)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'doubleVal' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setDoubleVal(TDoubleColumn value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.DOUBLE_VAL;
-    value_ = value;
-  }
-
-  public TStringColumn getStringVal() {
-    if (getSetField() == _Fields.STRING_VAL) {
-      return (TStringColumn)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'stringVal' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setStringVal(TStringColumn value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.STRING_VAL;
-    value_ = value;
-  }
-
-  public TBinaryColumn getBinaryVal() {
-    if (getSetField() == _Fields.BINARY_VAL) {
-      return (TBinaryColumn)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'binaryVal' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setBinaryVal(TBinaryColumn value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.BINARY_VAL;
-    value_ = value;
-  }
-
-  public boolean isSetBoolVal() {
-    return setField_ == _Fields.BOOL_VAL;
-  }
-
-
-  public boolean isSetByteVal() {
-    return setField_ == _Fields.BYTE_VAL;
-  }
-
-
-  public boolean isSetI16Val() {
-    return setField_ == _Fields.I16_VAL;
-  }
-
-
-  public boolean isSetI32Val() {
-    return setField_ == _Fields.I32_VAL;
-  }
-
-
-  public boolean isSetI64Val() {
-    return setField_ == _Fields.I64_VAL;
-  }
-
-
-  public boolean isSetDoubleVal() {
-    return setField_ == _Fields.DOUBLE_VAL;
-  }
-
-
-  public boolean isSetStringVal() {
-    return setField_ == _Fields.STRING_VAL;
-  }
-
-
-  public boolean isSetBinaryVal() {
-    return setField_ == _Fields.BINARY_VAL;
-  }
-
-
-  public boolean equals(Object other) {
-    if (other instanceof TColumn) {
-      return equals((TColumn)other);
-    } else {
-      return false;
-    }
-  }
-
-  public boolean equals(TColumn other) {
-    return other != null && getSetField() == other.getSetField() && getFieldValue().equals(other.getFieldValue());
-  }
-
-  @Override
-  public int compareTo(TColumn other) {
-    int lastComparison = org.apache.thrift.TBaseHelper.compareTo(getSetField(), other.getSetField());
-    if (lastComparison == 0) {
-      return org.apache.thrift.TBaseHelper.compareTo(getFieldValue(), other.getFieldValue());
-    }
-    return lastComparison;
-  }
-
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder hcb = new HashCodeBuilder();
-    hcb.append(this.getClass().getName());
-    org.apache.thrift.TFieldIdEnum setField = getSetField();
-    if (setField != null) {
-      hcb.append(setField.getThriftFieldId());
-      Object value = getFieldValue();
-      if (value instanceof org.apache.thrift.TEnum) {
-        hcb.append(((org.apache.thrift.TEnum)getFieldValue()).getValue());
-      } else {
-        hcb.append(value);
-      }
-    }
-    return hcb.toHashCode();
-  }
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-
-}
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TColumnDesc.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TColumnDesc.java
deleted file mode 100644
index 247db6489457f..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TColumnDesc.java
+++ /dev/null
@@ -1,700 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TColumnDesc implements org.apache.thrift.TBase<TColumnDesc, TColumnDesc._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TColumnDesc");
-
-  private static final org.apache.thrift.protocol.TField COLUMN_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("columnName", org.apache.thrift.protocol.TType.STRING, (short)1);
-  private static final org.apache.thrift.protocol.TField TYPE_DESC_FIELD_DESC = new org.apache.thrift.protocol.TField("typeDesc", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-  private static final org.apache.thrift.protocol.TField POSITION_FIELD_DESC = new org.apache.thrift.protocol.TField("position", org.apache.thrift.protocol.TType.I32, (short)3);
-  private static final org.apache.thrift.protocol.TField COMMENT_FIELD_DESC = new org.apache.thrift.protocol.TField("comment", org.apache.thrift.protocol.TType.STRING, (short)4);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TColumnDescStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TColumnDescTupleSchemeFactory());
-  }
-
-  private String columnName; // required
-  private TTypeDesc typeDesc; // required
-  private int position; // required
-  private String comment; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    COLUMN_NAME((short)1, "columnName"),
-    TYPE_DESC((short)2, "typeDesc"),
-    POSITION((short)3, "position"),
-    COMMENT((short)4, "comment");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // COLUMN_NAME
-          return COLUMN_NAME;
-        case 2: // TYPE_DESC
-          return TYPE_DESC;
-        case 3: // POSITION
-          return POSITION;
-        case 4: // COMMENT
-          return COMMENT;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __POSITION_ISSET_ID = 0;
-  private byte __isset_bitfield = 0;
-  private _Fields optionals[] = {_Fields.COMMENT};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.COLUMN_NAME, new org.apache.thrift.meta_data.FieldMetaData("columnName", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    tmpMap.put(_Fields.TYPE_DESC, new org.apache.thrift.meta_data.FieldMetaData("typeDesc", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TTypeDesc.class)));
-    tmpMap.put(_Fields.POSITION, new org.apache.thrift.meta_data.FieldMetaData("position", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
-    tmpMap.put(_Fields.COMMENT, new org.apache.thrift.meta_data.FieldMetaData("comment", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TColumnDesc.class, metaDataMap);
-  }
-
-  public TColumnDesc() {
-  }
-
-  public TColumnDesc(
-    String columnName,
-    TTypeDesc typeDesc,
-    int position)
-  {
-    this();
-    this.columnName = columnName;
-    this.typeDesc = typeDesc;
-    this.position = position;
-    setPositionIsSet(true);
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TColumnDesc(TColumnDesc other) {
-    __isset_bitfield = other.__isset_bitfield;
-    if (other.isSetColumnName()) {
-      this.columnName = other.columnName;
-    }
-    if (other.isSetTypeDesc()) {
-      this.typeDesc = new TTypeDesc(other.typeDesc);
-    }
-    this.position = other.position;
-    if (other.isSetComment()) {
-      this.comment = other.comment;
-    }
-  }
-
-  public TColumnDesc deepCopy() {
-    return new TColumnDesc(this);
-  }
-
-  @Override
-  public void clear() {
-    this.columnName = null;
-    this.typeDesc = null;
-    setPositionIsSet(false);
-    this.position = 0;
-    this.comment = null;
-  }
-
-  public String getColumnName() {
-    return this.columnName;
-  }
-
-  public void setColumnName(String columnName) {
-    this.columnName = columnName;
-  }
-
-  public void unsetColumnName() {
-    this.columnName = null;
-  }
-
-  /** Returns true if field columnName is set (has been assigned a value) and false otherwise */
-  public boolean isSetColumnName() {
-    return this.columnName != null;
-  }
-
-  public void setColumnNameIsSet(boolean value) {
-    if (!value) {
-      this.columnName = null;
-    }
-  }
-
-  public TTypeDesc getTypeDesc() {
-    return this.typeDesc;
-  }
-
-  public void setTypeDesc(TTypeDesc typeDesc) {
-    this.typeDesc = typeDesc;
-  }
-
-  public void unsetTypeDesc() {
-    this.typeDesc = null;
-  }
-
-  /** Returns true if field typeDesc is set (has been assigned a value) and false otherwise */
-  public boolean isSetTypeDesc() {
-    return this.typeDesc != null;
-  }
-
-  public void setTypeDescIsSet(boolean value) {
-    if (!value) {
-      this.typeDesc = null;
-    }
-  }
-
-  public int getPosition() {
-    return this.position;
-  }
-
-  public void setPosition(int position) {
-    this.position = position;
-    setPositionIsSet(true);
-  }
-
-  public void unsetPosition() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __POSITION_ISSET_ID);
-  }
-
-  /** Returns true if field position is set (has been assigned a value) and false otherwise */
-  public boolean isSetPosition() {
-    return EncodingUtils.testBit(__isset_bitfield, __POSITION_ISSET_ID);
-  }
-
-  public void setPositionIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __POSITION_ISSET_ID, value);
-  }
-
-  public String getComment() {
-    return this.comment;
-  }
-
-  public void setComment(String comment) {
-    this.comment = comment;
-  }
-
-  public void unsetComment() {
-    this.comment = null;
-  }
-
-  /** Returns true if field comment is set (has been assigned a value) and false otherwise */
-  public boolean isSetComment() {
-    return this.comment != null;
-  }
-
-  public void setCommentIsSet(boolean value) {
-    if (!value) {
-      this.comment = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case COLUMN_NAME:
-      if (value == null) {
-        unsetColumnName();
-      } else {
-        setColumnName((String)value);
-      }
-      break;
-
-    case TYPE_DESC:
-      if (value == null) {
-        unsetTypeDesc();
-      } else {
-        setTypeDesc((TTypeDesc)value);
-      }
-      break;
-
-    case POSITION:
-      if (value == null) {
-        unsetPosition();
-      } else {
-        setPosition((Integer)value);
-      }
-      break;
-
-    case COMMENT:
-      if (value == null) {
-        unsetComment();
-      } else {
-        setComment((String)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case COLUMN_NAME:
-      return getColumnName();
-
-    case TYPE_DESC:
-      return getTypeDesc();
-
-    case POSITION:
-      return Integer.valueOf(getPosition());
-
-    case COMMENT:
-      return getComment();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case COLUMN_NAME:
-      return isSetColumnName();
-    case TYPE_DESC:
-      return isSetTypeDesc();
-    case POSITION:
-      return isSetPosition();
-    case COMMENT:
-      return isSetComment();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TColumnDesc)
-      return this.equals((TColumnDesc)that);
-    return false;
-  }
-
-  public boolean equals(TColumnDesc that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_columnName = true && this.isSetColumnName();
-    boolean that_present_columnName = true && that.isSetColumnName();
-    if (this_present_columnName || that_present_columnName) {
-      if (!(this_present_columnName && that_present_columnName))
-        return false;
-      if (!this.columnName.equals(that.columnName))
-        return false;
-    }
-
-    boolean this_present_typeDesc = true && this.isSetTypeDesc();
-    boolean that_present_typeDesc = true && that.isSetTypeDesc();
-    if (this_present_typeDesc || that_present_typeDesc) {
-      if (!(this_present_typeDesc && that_present_typeDesc))
-        return false;
-      if (!this.typeDesc.equals(that.typeDesc))
-        return false;
-    }
-
-    boolean this_present_position = true;
-    boolean that_present_position = true;
-    if (this_present_position || that_present_position) {
-      if (!(this_present_position && that_present_position))
-        return false;
-      if (this.position != that.position)
-        return false;
-    }
-
-    boolean this_present_comment = true && this.isSetComment();
-    boolean that_present_comment = true && that.isSetComment();
-    if (this_present_comment || that_present_comment) {
-      if (!(this_present_comment && that_present_comment))
-        return false;
-      if (!this.comment.equals(that.comment))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_columnName = true && (isSetColumnName());
-    builder.append(present_columnName);
-    if (present_columnName)
-      builder.append(columnName);
-
-    boolean present_typeDesc = true && (isSetTypeDesc());
-    builder.append(present_typeDesc);
-    if (present_typeDesc)
-      builder.append(typeDesc);
-
-    boolean present_position = true;
-    builder.append(present_position);
-    if (present_position)
-      builder.append(position);
-
-    boolean present_comment = true && (isSetComment());
-    builder.append(present_comment);
-    if (present_comment)
-      builder.append(comment);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TColumnDesc other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TColumnDesc typedOther = (TColumnDesc)other;
-
-    lastComparison = Boolean.valueOf(isSetColumnName()).compareTo(typedOther.isSetColumnName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetColumnName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columnName, typedOther.columnName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetTypeDesc()).compareTo(typedOther.isSetTypeDesc());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetTypeDesc()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.typeDesc, typedOther.typeDesc);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetPosition()).compareTo(typedOther.isSetPosition());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetPosition()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.position, typedOther.position);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetComment()).compareTo(typedOther.isSetComment());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetComment()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.comment, typedOther.comment);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TColumnDesc(");
-    boolean first = true;
-
-    sb.append("columnName:");
-    if (this.columnName == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.columnName);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("typeDesc:");
-    if (this.typeDesc == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.typeDesc);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("position:");
-    sb.append(this.position);
-    first = false;
-    if (isSetComment()) {
-      if (!first) sb.append(", ");
-      sb.append("comment:");
-      if (this.comment == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.comment);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetColumnName()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'columnName' is unset! Struct:" + toString());
-    }
-
-    if (!isSetTypeDesc()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'typeDesc' is unset! Struct:" + toString());
-    }
-
-    if (!isSetPosition()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'position' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (typeDesc != null) {
-      typeDesc.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TColumnDescStandardSchemeFactory implements SchemeFactory {
-    public TColumnDescStandardScheme getScheme() {
-      return new TColumnDescStandardScheme();
-    }
-  }
-
-  private static class TColumnDescStandardScheme extends StandardScheme<TColumnDesc> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TColumnDesc struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // COLUMN_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.columnName = iprot.readString();
-              struct.setColumnNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // TYPE_DESC
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.typeDesc = new TTypeDesc();
-              struct.typeDesc.read(iprot);
-              struct.setTypeDescIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // POSITION
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.position = iprot.readI32();
-              struct.setPositionIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // COMMENT
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.comment = iprot.readString();
-              struct.setCommentIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TColumnDesc struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.columnName != null) {
-        oprot.writeFieldBegin(COLUMN_NAME_FIELD_DESC);
-        oprot.writeString(struct.columnName);
-        oprot.writeFieldEnd();
-      }
-      if (struct.typeDesc != null) {
-        oprot.writeFieldBegin(TYPE_DESC_FIELD_DESC);
-        struct.typeDesc.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldBegin(POSITION_FIELD_DESC);
-      oprot.writeI32(struct.position);
-      oprot.writeFieldEnd();
-      if (struct.comment != null) {
-        if (struct.isSetComment()) {
-          oprot.writeFieldBegin(COMMENT_FIELD_DESC);
-          oprot.writeString(struct.comment);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TColumnDescTupleSchemeFactory implements SchemeFactory {
-    public TColumnDescTupleScheme getScheme() {
-      return new TColumnDescTupleScheme();
-    }
-  }
-
-  private static class TColumnDescTupleScheme extends TupleScheme<TColumnDesc> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TColumnDesc struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      oprot.writeString(struct.columnName);
-      struct.typeDesc.write(oprot);
-      oprot.writeI32(struct.position);
-      BitSet optionals = new BitSet();
-      if (struct.isSetComment()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetComment()) {
-        oprot.writeString(struct.comment);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TColumnDesc struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.columnName = iprot.readString();
-      struct.setColumnNameIsSet(true);
-      struct.typeDesc = new TTypeDesc();
-      struct.typeDesc.read(iprot);
-      struct.setTypeDescIsSet(true);
-      struct.position = iprot.readI32();
-      struct.setPositionIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.comment = iprot.readString();
-        struct.setCommentIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TColumnValue.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TColumnValue.java
deleted file mode 100644
index 8504c6d608d42..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TColumnValue.java
+++ /dev/null
@@ -1,671 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TColumnValue extends org.apache.thrift.TUnion<TColumnValue, TColumnValue._Fields> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TColumnValue");
-  private static final org.apache.thrift.protocol.TField BOOL_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("boolVal", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField BYTE_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("byteVal", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-  private static final org.apache.thrift.protocol.TField I16_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("i16Val", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-  private static final org.apache.thrift.protocol.TField I32_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("i32Val", org.apache.thrift.protocol.TType.STRUCT, (short)4);
-  private static final org.apache.thrift.protocol.TField I64_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("i64Val", org.apache.thrift.protocol.TType.STRUCT, (short)5);
-  private static final org.apache.thrift.protocol.TField DOUBLE_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("doubleVal", org.apache.thrift.protocol.TType.STRUCT, (short)6);
-  private static final org.apache.thrift.protocol.TField STRING_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("stringVal", org.apache.thrift.protocol.TType.STRUCT, (short)7);
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    BOOL_VAL((short)1, "boolVal"),
-    BYTE_VAL((short)2, "byteVal"),
-    I16_VAL((short)3, "i16Val"),
-    I32_VAL((short)4, "i32Val"),
-    I64_VAL((short)5, "i64Val"),
-    DOUBLE_VAL((short)6, "doubleVal"),
-    STRING_VAL((short)7, "stringVal");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // BOOL_VAL
-          return BOOL_VAL;
-        case 2: // BYTE_VAL
-          return BYTE_VAL;
-        case 3: // I16_VAL
-          return I16_VAL;
-        case 4: // I32_VAL
-          return I32_VAL;
-        case 5: // I64_VAL
-          return I64_VAL;
-        case 6: // DOUBLE_VAL
-          return DOUBLE_VAL;
-        case 7: // STRING_VAL
-          return STRING_VAL;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.BOOL_VAL, new org.apache.thrift.meta_data.FieldMetaData("boolVal", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TBoolValue.class)));
-    tmpMap.put(_Fields.BYTE_VAL, new org.apache.thrift.meta_data.FieldMetaData("byteVal", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TByteValue.class)));
-    tmpMap.put(_Fields.I16_VAL, new org.apache.thrift.meta_data.FieldMetaData("i16Val", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TI16Value.class)));
-    tmpMap.put(_Fields.I32_VAL, new org.apache.thrift.meta_data.FieldMetaData("i32Val", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TI32Value.class)));
-    tmpMap.put(_Fields.I64_VAL, new org.apache.thrift.meta_data.FieldMetaData("i64Val", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TI64Value.class)));
-    tmpMap.put(_Fields.DOUBLE_VAL, new org.apache.thrift.meta_data.FieldMetaData("doubleVal", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TDoubleValue.class)));
-    tmpMap.put(_Fields.STRING_VAL, new org.apache.thrift.meta_data.FieldMetaData("stringVal", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStringValue.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TColumnValue.class, metaDataMap);
-  }
-
-  public TColumnValue() {
-    super();
-  }
-
-  public TColumnValue(TColumnValue._Fields setField, Object value) {
-    super(setField, value);
-  }
-
-  public TColumnValue(TColumnValue other) {
-    super(other);
-  }
-  public TColumnValue deepCopy() {
-    return new TColumnValue(this);
-  }
-
-  public static TColumnValue boolVal(TBoolValue value) {
-    TColumnValue x = new TColumnValue();
-    x.setBoolVal(value);
-    return x;
-  }
-
-  public static TColumnValue byteVal(TByteValue value) {
-    TColumnValue x = new TColumnValue();
-    x.setByteVal(value);
-    return x;
-  }
-
-  public static TColumnValue i16Val(TI16Value value) {
-    TColumnValue x = new TColumnValue();
-    x.setI16Val(value);
-    return x;
-  }
-
-  public static TColumnValue i32Val(TI32Value value) {
-    TColumnValue x = new TColumnValue();
-    x.setI32Val(value);
-    return x;
-  }
-
-  public static TColumnValue i64Val(TI64Value value) {
-    TColumnValue x = new TColumnValue();
-    x.setI64Val(value);
-    return x;
-  }
-
-  public static TColumnValue doubleVal(TDoubleValue value) {
-    TColumnValue x = new TColumnValue();
-    x.setDoubleVal(value);
-    return x;
-  }
-
-  public static TColumnValue stringVal(TStringValue value) {
-    TColumnValue x = new TColumnValue();
-    x.setStringVal(value);
-    return x;
-  }
-
-
-  @Override
-  protected void checkType(_Fields setField, Object value) throws ClassCastException {
-    switch (setField) {
-      case BOOL_VAL:
-        if (value instanceof TBoolValue) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TBoolValue for field 'boolVal', but got " + value.getClass().getSimpleName());
-      case BYTE_VAL:
-        if (value instanceof TByteValue) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TByteValue for field 'byteVal', but got " + value.getClass().getSimpleName());
-      case I16_VAL:
-        if (value instanceof TI16Value) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TI16Value for field 'i16Val', but got " + value.getClass().getSimpleName());
-      case I32_VAL:
-        if (value instanceof TI32Value) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TI32Value for field 'i32Val', but got " + value.getClass().getSimpleName());
-      case I64_VAL:
-        if (value instanceof TI64Value) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TI64Value for field 'i64Val', but got " + value.getClass().getSimpleName());
-      case DOUBLE_VAL:
-        if (value instanceof TDoubleValue) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TDoubleValue for field 'doubleVal', but got " + value.getClass().getSimpleName());
-      case STRING_VAL:
-        if (value instanceof TStringValue) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TStringValue for field 'stringVal', but got " + value.getClass().getSimpleName());
-      default:
-        throw new IllegalArgumentException("Unknown field id " + setField);
-    }
-  }
-
-  @Override
-  protected Object standardSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TField field) throws org.apache.thrift.TException {
-    _Fields setField = _Fields.findByThriftId(field.id);
-    if (setField != null) {
-      switch (setField) {
-        case BOOL_VAL:
-          if (field.type == BOOL_VAL_FIELD_DESC.type) {
-            TBoolValue boolVal;
-            boolVal = new TBoolValue();
-            boolVal.read(iprot);
-            return boolVal;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case BYTE_VAL:
-          if (field.type == BYTE_VAL_FIELD_DESC.type) {
-            TByteValue byteVal;
-            byteVal = new TByteValue();
-            byteVal.read(iprot);
-            return byteVal;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case I16_VAL:
-          if (field.type == I16_VAL_FIELD_DESC.type) {
-            TI16Value i16Val;
-            i16Val = new TI16Value();
-            i16Val.read(iprot);
-            return i16Val;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case I32_VAL:
-          if (field.type == I32_VAL_FIELD_DESC.type) {
-            TI32Value i32Val;
-            i32Val = new TI32Value();
-            i32Val.read(iprot);
-            return i32Val;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case I64_VAL:
-          if (field.type == I64_VAL_FIELD_DESC.type) {
-            TI64Value i64Val;
-            i64Val = new TI64Value();
-            i64Val.read(iprot);
-            return i64Val;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case DOUBLE_VAL:
-          if (field.type == DOUBLE_VAL_FIELD_DESC.type) {
-            TDoubleValue doubleVal;
-            doubleVal = new TDoubleValue();
-            doubleVal.read(iprot);
-            return doubleVal;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case STRING_VAL:
-          if (field.type == STRING_VAL_FIELD_DESC.type) {
-            TStringValue stringVal;
-            stringVal = new TStringValue();
-            stringVal.read(iprot);
-            return stringVal;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        default:
-          throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!");
-      }
-    } else {
-      return null;
-    }
-  }
-
-  @Override
-  protected void standardSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    switch (setField_) {
-      case BOOL_VAL:
-        TBoolValue boolVal = (TBoolValue)value_;
-        boolVal.write(oprot);
-        return;
-      case BYTE_VAL:
-        TByteValue byteVal = (TByteValue)value_;
-        byteVal.write(oprot);
-        return;
-      case I16_VAL:
-        TI16Value i16Val = (TI16Value)value_;
-        i16Val.write(oprot);
-        return;
-      case I32_VAL:
-        TI32Value i32Val = (TI32Value)value_;
-        i32Val.write(oprot);
-        return;
-      case I64_VAL:
-        TI64Value i64Val = (TI64Value)value_;
-        i64Val.write(oprot);
-        return;
-      case DOUBLE_VAL:
-        TDoubleValue doubleVal = (TDoubleValue)value_;
-        doubleVal.write(oprot);
-        return;
-      case STRING_VAL:
-        TStringValue stringVal = (TStringValue)value_;
-        stringVal.write(oprot);
-        return;
-      default:
-        throw new IllegalStateException("Cannot write union with unknown field " + setField_);
-    }
-  }
-
-  @Override
-  protected Object tupleSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, short fieldID) throws org.apache.thrift.TException {
-    _Fields setField = _Fields.findByThriftId(fieldID);
-    if (setField != null) {
-      switch (setField) {
-        case BOOL_VAL:
-          TBoolValue boolVal;
-          boolVal = new TBoolValue();
-          boolVal.read(iprot);
-          return boolVal;
-        case BYTE_VAL:
-          TByteValue byteVal;
-          byteVal = new TByteValue();
-          byteVal.read(iprot);
-          return byteVal;
-        case I16_VAL:
-          TI16Value i16Val;
-          i16Val = new TI16Value();
-          i16Val.read(iprot);
-          return i16Val;
-        case I32_VAL:
-          TI32Value i32Val;
-          i32Val = new TI32Value();
-          i32Val.read(iprot);
-          return i32Val;
-        case I64_VAL:
-          TI64Value i64Val;
-          i64Val = new TI64Value();
-          i64Val.read(iprot);
-          return i64Val;
-        case DOUBLE_VAL:
-          TDoubleValue doubleVal;
-          doubleVal = new TDoubleValue();
-          doubleVal.read(iprot);
-          return doubleVal;
-        case STRING_VAL:
-          TStringValue stringVal;
-          stringVal = new TStringValue();
-          stringVal.read(iprot);
-          return stringVal;
-        default:
-          throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!");
-      }
-    } else {
-      throw new TProtocolException("Couldn't find a field with field id " + fieldID);
-    }
-  }
-
-  @Override
-  protected void tupleSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    switch (setField_) {
-      case BOOL_VAL:
-        TBoolValue boolVal = (TBoolValue)value_;
-        boolVal.write(oprot);
-        return;
-      case BYTE_VAL:
-        TByteValue byteVal = (TByteValue)value_;
-        byteVal.write(oprot);
-        return;
-      case I16_VAL:
-        TI16Value i16Val = (TI16Value)value_;
-        i16Val.write(oprot);
-        return;
-      case I32_VAL:
-        TI32Value i32Val = (TI32Value)value_;
-        i32Val.write(oprot);
-        return;
-      case I64_VAL:
-        TI64Value i64Val = (TI64Value)value_;
-        i64Val.write(oprot);
-        return;
-      case DOUBLE_VAL:
-        TDoubleValue doubleVal = (TDoubleValue)value_;
-        doubleVal.write(oprot);
-        return;
-      case STRING_VAL:
-        TStringValue stringVal = (TStringValue)value_;
-        stringVal.write(oprot);
-        return;
-      default:
-        throw new IllegalStateException("Cannot write union with unknown field " + setField_);
-    }
-  }
-
-  @Override
-  protected org.apache.thrift.protocol.TField getFieldDesc(_Fields setField) {
-    switch (setField) {
-      case BOOL_VAL:
-        return BOOL_VAL_FIELD_DESC;
-      case BYTE_VAL:
-        return BYTE_VAL_FIELD_DESC;
-      case I16_VAL:
-        return I16_VAL_FIELD_DESC;
-      case I32_VAL:
-        return I32_VAL_FIELD_DESC;
-      case I64_VAL:
-        return I64_VAL_FIELD_DESC;
-      case DOUBLE_VAL:
-        return DOUBLE_VAL_FIELD_DESC;
-      case STRING_VAL:
-        return STRING_VAL_FIELD_DESC;
-      default:
-        throw new IllegalArgumentException("Unknown field id " + setField);
-    }
-  }
-
-  @Override
-  protected org.apache.thrift.protocol.TStruct getStructDesc() {
-    return STRUCT_DESC;
-  }
-
-  @Override
-  protected _Fields enumForId(short id) {
-    return _Fields.findByThriftIdOrThrow(id);
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-
-  public TBoolValue getBoolVal() {
-    if (getSetField() == _Fields.BOOL_VAL) {
-      return (TBoolValue)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'boolVal' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setBoolVal(TBoolValue value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.BOOL_VAL;
-    value_ = value;
-  }
-
-  public TByteValue getByteVal() {
-    if (getSetField() == _Fields.BYTE_VAL) {
-      return (TByteValue)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'byteVal' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setByteVal(TByteValue value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.BYTE_VAL;
-    value_ = value;
-  }
-
-  public TI16Value getI16Val() {
-    if (getSetField() == _Fields.I16_VAL) {
-      return (TI16Value)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'i16Val' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setI16Val(TI16Value value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.I16_VAL;
-    value_ = value;
-  }
-
-  public TI32Value getI32Val() {
-    if (getSetField() == _Fields.I32_VAL) {
-      return (TI32Value)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'i32Val' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setI32Val(TI32Value value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.I32_VAL;
-    value_ = value;
-  }
-
-  public TI64Value getI64Val() {
-    if (getSetField() == _Fields.I64_VAL) {
-      return (TI64Value)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'i64Val' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setI64Val(TI64Value value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.I64_VAL;
-    value_ = value;
-  }
-
-  public TDoubleValue getDoubleVal() {
-    if (getSetField() == _Fields.DOUBLE_VAL) {
-      return (TDoubleValue)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'doubleVal' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setDoubleVal(TDoubleValue value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.DOUBLE_VAL;
-    value_ = value;
-  }
-
-  public TStringValue getStringVal() {
-    if (getSetField() == _Fields.STRING_VAL) {
-      return (TStringValue)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'stringVal' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setStringVal(TStringValue value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.STRING_VAL;
-    value_ = value;
-  }
-
-  public boolean isSetBoolVal() {
-    return setField_ == _Fields.BOOL_VAL;
-  }
-
-
-  public boolean isSetByteVal() {
-    return setField_ == _Fields.BYTE_VAL;
-  }
-
-
-  public boolean isSetI16Val() {
-    return setField_ == _Fields.I16_VAL;
-  }
-
-
-  public boolean isSetI32Val() {
-    return setField_ == _Fields.I32_VAL;
-  }
-
-
-  public boolean isSetI64Val() {
-    return setField_ == _Fields.I64_VAL;
-  }
-
-
-  public boolean isSetDoubleVal() {
-    return setField_ == _Fields.DOUBLE_VAL;
-  }
-
-
-  public boolean isSetStringVal() {
-    return setField_ == _Fields.STRING_VAL;
-  }
-
-
-  public boolean equals(Object other) {
-    if (other instanceof TColumnValue) {
-      return equals((TColumnValue)other);
-    } else {
-      return false;
-    }
-  }
-
-  public boolean equals(TColumnValue other) {
-    return other != null && getSetField() == other.getSetField() && getFieldValue().equals(other.getFieldValue());
-  }
-
-  @Override
-  public int compareTo(TColumnValue other) {
-    int lastComparison = org.apache.thrift.TBaseHelper.compareTo(getSetField(), other.getSetField());
-    if (lastComparison == 0) {
-      return org.apache.thrift.TBaseHelper.compareTo(getFieldValue(), other.getFieldValue());
-    }
-    return lastComparison;
-  }
-
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder hcb = new HashCodeBuilder();
-    hcb.append(this.getClass().getName());
-    org.apache.thrift.TFieldIdEnum setField = getSetField();
-    if (setField != null) {
-      hcb.append(setField.getThriftFieldId());
-      Object value = getFieldValue();
-      if (value instanceof org.apache.thrift.TEnum) {
-        hcb.append(((org.apache.thrift.TEnum)getFieldValue()).getValue());
-      } else {
-        hcb.append(value);
-      }
-    }
-    return hcb.toHashCode();
-  }
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-
-}
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TDoubleColumn.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TDoubleColumn.java
deleted file mode 100644
index 4fc54544c1bea..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TDoubleColumn.java
+++ /dev/null
@@ -1,548 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TDoubleColumn implements org.apache.thrift.TBase<TDoubleColumn, TDoubleColumn._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TDoubleColumn");
-
-  private static final org.apache.thrift.protocol.TField VALUES_FIELD_DESC = new org.apache.thrift.protocol.TField("values", org.apache.thrift.protocol.TType.LIST, (short)1);
-  private static final org.apache.thrift.protocol.TField NULLS_FIELD_DESC = new org.apache.thrift.protocol.TField("nulls", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TDoubleColumnStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TDoubleColumnTupleSchemeFactory());
-  }
-
-  private List<Double> values; // required
-  private ByteBuffer nulls; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUES((short)1, "values"),
-    NULLS((short)2, "nulls");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUES
-          return VALUES;
-        case 2: // NULLS
-          return NULLS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUES, new org.apache.thrift.meta_data.FieldMetaData("values", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE))));
-    tmpMap.put(_Fields.NULLS, new org.apache.thrift.meta_data.FieldMetaData("nulls", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TDoubleColumn.class, metaDataMap);
-  }
-
-  public TDoubleColumn() {
-  }
-
-  public TDoubleColumn(
-    List<Double> values,
-    ByteBuffer nulls)
-  {
-    this();
-    this.values = values;
-    this.nulls = nulls;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TDoubleColumn(TDoubleColumn other) {
-    if (other.isSetValues()) {
-      List<Double> __this__values = new ArrayList<Double>();
-      for (Double other_element : other.values) {
-        __this__values.add(other_element);
-      }
-      this.values = __this__values;
-    }
-    if (other.isSetNulls()) {
-      this.nulls = org.apache.thrift.TBaseHelper.copyBinary(other.nulls);
-;
-    }
-  }
-
-  public TDoubleColumn deepCopy() {
-    return new TDoubleColumn(this);
-  }
-
-  @Override
-  public void clear() {
-    this.values = null;
-    this.nulls = null;
-  }
-
-  public int getValuesSize() {
-    return (this.values == null) ? 0 : this.values.size();
-  }
-
-  public java.util.Iterator<Double> getValuesIterator() {
-    return (this.values == null) ? null : this.values.iterator();
-  }
-
-  public void addToValues(double elem) {
-    if (this.values == null) {
-      this.values = new ArrayList<Double>();
-    }
-    this.values.add(elem);
-  }
-
-  public List<Double> getValues() {
-    return this.values;
-  }
-
-  public void setValues(List<Double> values) {
-    this.values = values;
-  }
-
-  public void unsetValues() {
-    this.values = null;
-  }
-
-  /** Returns true if field values is set (has been assigned a value) and false otherwise */
-  public boolean isSetValues() {
-    return this.values != null;
-  }
-
-  public void setValuesIsSet(boolean value) {
-    if (!value) {
-      this.values = null;
-    }
-  }
-
-  public byte[] getNulls() {
-    setNulls(org.apache.thrift.TBaseHelper.rightSize(nulls));
-    return nulls == null ? null : nulls.array();
-  }
-
-  public ByteBuffer bufferForNulls() {
-    return nulls;
-  }
-
-  public void setNulls(byte[] nulls) {
-    setNulls(nulls == null ? (ByteBuffer)null : ByteBuffer.wrap(nulls));
-  }
-
-  public void setNulls(ByteBuffer nulls) {
-    this.nulls = nulls;
-  }
-
-  public void unsetNulls() {
-    this.nulls = null;
-  }
-
-  /** Returns true if field nulls is set (has been assigned a value) and false otherwise */
-  public boolean isSetNulls() {
-    return this.nulls != null;
-  }
-
-  public void setNullsIsSet(boolean value) {
-    if (!value) {
-      this.nulls = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUES:
-      if (value == null) {
-        unsetValues();
-      } else {
-        setValues((List<Double>)value);
-      }
-      break;
-
-    case NULLS:
-      if (value == null) {
-        unsetNulls();
-      } else {
-        setNulls((ByteBuffer)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUES:
-      return getValues();
-
-    case NULLS:
-      return getNulls();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUES:
-      return isSetValues();
-    case NULLS:
-      return isSetNulls();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TDoubleColumn)
-      return this.equals((TDoubleColumn)that);
-    return false;
-  }
-
-  public boolean equals(TDoubleColumn that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_values = true && this.isSetValues();
-    boolean that_present_values = true && that.isSetValues();
-    if (this_present_values || that_present_values) {
-      if (!(this_present_values && that_present_values))
-        return false;
-      if (!this.values.equals(that.values))
-        return false;
-    }
-
-    boolean this_present_nulls = true && this.isSetNulls();
-    boolean that_present_nulls = true && that.isSetNulls();
-    if (this_present_nulls || that_present_nulls) {
-      if (!(this_present_nulls && that_present_nulls))
-        return false;
-      if (!this.nulls.equals(that.nulls))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_values = true && (isSetValues());
-    builder.append(present_values);
-    if (present_values)
-      builder.append(values);
-
-    boolean present_nulls = true && (isSetNulls());
-    builder.append(present_nulls);
-    if (present_nulls)
-      builder.append(nulls);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TDoubleColumn other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TDoubleColumn typedOther = (TDoubleColumn)other;
-
-    lastComparison = Boolean.valueOf(isSetValues()).compareTo(typedOther.isSetValues());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValues()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.values, typedOther.values);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetNulls()).compareTo(typedOther.isSetNulls());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetNulls()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.nulls, typedOther.nulls);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TDoubleColumn(");
-    boolean first = true;
-
-    sb.append("values:");
-    if (this.values == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.values);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("nulls:");
-    if (this.nulls == null) {
-      sb.append("null");
-    } else {
-      org.apache.thrift.TBaseHelper.toString(this.nulls, sb);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetValues()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'values' is unset! Struct:" + toString());
-    }
-
-    if (!isSetNulls()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'nulls' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TDoubleColumnStandardSchemeFactory implements SchemeFactory {
-    public TDoubleColumnStandardScheme getScheme() {
-      return new TDoubleColumnStandardScheme();
-    }
-  }
-
-  private static class TDoubleColumnStandardScheme extends StandardScheme<TDoubleColumn> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TDoubleColumn struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUES
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list94 = iprot.readListBegin();
-                struct.values = new ArrayList<Double>(_list94.size);
-                for (int _i95 = 0; _i95 < _list94.size; ++_i95)
-                {
-                  double _elem96; // optional
-                  _elem96 = iprot.readDouble();
-                  struct.values.add(_elem96);
-                }
-                iprot.readListEnd();
-              }
-              struct.setValuesIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // NULLS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.nulls = iprot.readBinary();
-              struct.setNullsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TDoubleColumn struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.values != null) {
-        oprot.writeFieldBegin(VALUES_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.DOUBLE, struct.values.size()));
-          for (double _iter97 : struct.values)
-          {
-            oprot.writeDouble(_iter97);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      if (struct.nulls != null) {
-        oprot.writeFieldBegin(NULLS_FIELD_DESC);
-        oprot.writeBinary(struct.nulls);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TDoubleColumnTupleSchemeFactory implements SchemeFactory {
-    public TDoubleColumnTupleScheme getScheme() {
-      return new TDoubleColumnTupleScheme();
-    }
-  }
-
-  private static class TDoubleColumnTupleScheme extends TupleScheme<TDoubleColumn> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TDoubleColumn struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.values.size());
-        for (double _iter98 : struct.values)
-        {
-          oprot.writeDouble(_iter98);
-        }
-      }
-      oprot.writeBinary(struct.nulls);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TDoubleColumn struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TList _list99 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.DOUBLE, iprot.readI32());
-        struct.values = new ArrayList<Double>(_list99.size);
-        for (int _i100 = 0; _i100 < _list99.size; ++_i100)
-        {
-          double _elem101; // optional
-          _elem101 = iprot.readDouble();
-          struct.values.add(_elem101);
-        }
-      }
-      struct.setValuesIsSet(true);
-      struct.nulls = iprot.readBinary();
-      struct.setNullsIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TDoubleValue.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TDoubleValue.java
deleted file mode 100644
index d21573633ef51..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TDoubleValue.java
+++ /dev/null
@@ -1,386 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TDoubleValue implements org.apache.thrift.TBase<TDoubleValue, TDoubleValue._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TDoubleValue");
-
-  private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.DOUBLE, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TDoubleValueStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TDoubleValueTupleSchemeFactory());
-  }
-
-  private double value; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUE((short)1, "value");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUE
-          return VALUE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __VALUE_ISSET_ID = 0;
-  private byte __isset_bitfield = 0;
-  private _Fields optionals[] = {_Fields.VALUE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TDoubleValue.class, metaDataMap);
-  }
-
-  public TDoubleValue() {
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TDoubleValue(TDoubleValue other) {
-    __isset_bitfield = other.__isset_bitfield;
-    this.value = other.value;
-  }
-
-  public TDoubleValue deepCopy() {
-    return new TDoubleValue(this);
-  }
-
-  @Override
-  public void clear() {
-    setValueIsSet(false);
-    this.value = 0.0;
-  }
-
-  public double getValue() {
-    return this.value;
-  }
-
-  public void setValue(double value) {
-    this.value = value;
-    setValueIsSet(true);
-  }
-
-  public void unsetValue() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __VALUE_ISSET_ID);
-  }
-
-  /** Returns true if field value is set (has been assigned a value) and false otherwise */
-  public boolean isSetValue() {
-    return EncodingUtils.testBit(__isset_bitfield, __VALUE_ISSET_ID);
-  }
-
-  public void setValueIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __VALUE_ISSET_ID, value);
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUE:
-      if (value == null) {
-        unsetValue();
-      } else {
-        setValue((Double)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUE:
-      return Double.valueOf(getValue());
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUE:
-      return isSetValue();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TDoubleValue)
-      return this.equals((TDoubleValue)that);
-    return false;
-  }
-
-  public boolean equals(TDoubleValue that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_value = true && this.isSetValue();
-    boolean that_present_value = true && that.isSetValue();
-    if (this_present_value || that_present_value) {
-      if (!(this_present_value && that_present_value))
-        return false;
-      if (this.value != that.value)
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_value = true && (isSetValue());
-    builder.append(present_value);
-    if (present_value)
-      builder.append(value);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TDoubleValue other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TDoubleValue typedOther = (TDoubleValue)other;
-
-    lastComparison = Boolean.valueOf(isSetValue()).compareTo(typedOther.isSetValue());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValue()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, typedOther.value);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TDoubleValue(");
-    boolean first = true;
-
-    if (isSetValue()) {
-      sb.append("value:");
-      sb.append(this.value);
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TDoubleValueStandardSchemeFactory implements SchemeFactory {
-    public TDoubleValueStandardScheme getScheme() {
-      return new TDoubleValueStandardScheme();
-    }
-  }
-
-  private static class TDoubleValueStandardScheme extends StandardScheme<TDoubleValue> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TDoubleValue struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUE
-            if (schemeField.type == org.apache.thrift.protocol.TType.DOUBLE) {
-              struct.value = iprot.readDouble();
-              struct.setValueIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TDoubleValue struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.isSetValue()) {
-        oprot.writeFieldBegin(VALUE_FIELD_DESC);
-        oprot.writeDouble(struct.value);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TDoubleValueTupleSchemeFactory implements SchemeFactory {
-    public TDoubleValueTupleScheme getScheme() {
-      return new TDoubleValueTupleScheme();
-    }
-  }
-
-  private static class TDoubleValueTupleScheme extends TupleScheme<TDoubleValue> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TDoubleValue struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      BitSet optionals = new BitSet();
-      if (struct.isSetValue()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetValue()) {
-        oprot.writeDouble(struct.value);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TDoubleValue struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.value = iprot.readDouble();
-        struct.setValueIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TExecuteStatementReq.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TExecuteStatementReq.java
deleted file mode 100644
index 4f157ad5a6450..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TExecuteStatementReq.java
+++ /dev/null
@@ -1,769 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TExecuteStatementReq implements org.apache.thrift.TBase<TExecuteStatementReq, TExecuteStatementReq._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TExecuteStatementReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField STATEMENT_FIELD_DESC = new org.apache.thrift.protocol.TField("statement", org.apache.thrift.protocol.TType.STRING, (short)2);
-  private static final org.apache.thrift.protocol.TField CONF_OVERLAY_FIELD_DESC = new org.apache.thrift.protocol.TField("confOverlay", org.apache.thrift.protocol.TType.MAP, (short)3);
-  private static final org.apache.thrift.protocol.TField RUN_ASYNC_FIELD_DESC = new org.apache.thrift.protocol.TField("runAsync", org.apache.thrift.protocol.TType.BOOL, (short)4);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TExecuteStatementReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TExecuteStatementReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-  private String statement; // required
-  private Map<String,String> confOverlay; // optional
-  private boolean runAsync; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle"),
-    STATEMENT((short)2, "statement"),
-    CONF_OVERLAY((short)3, "confOverlay"),
-    RUN_ASYNC((short)4, "runAsync");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        case 2: // STATEMENT
-          return STATEMENT;
-        case 3: // CONF_OVERLAY
-          return CONF_OVERLAY;
-        case 4: // RUN_ASYNC
-          return RUN_ASYNC;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __RUNASYNC_ISSET_ID = 0;
-  private byte __isset_bitfield = 0;
-  private _Fields optionals[] = {_Fields.CONF_OVERLAY,_Fields.RUN_ASYNC};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    tmpMap.put(_Fields.STATEMENT, new org.apache.thrift.meta_data.FieldMetaData("statement", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    tmpMap.put(_Fields.CONF_OVERLAY, new org.apache.thrift.meta_data.FieldMetaData("confOverlay", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING), 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
-    tmpMap.put(_Fields.RUN_ASYNC, new org.apache.thrift.meta_data.FieldMetaData("runAsync", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TExecuteStatementReq.class, metaDataMap);
-  }
-
-  public TExecuteStatementReq() {
-    this.runAsync = false;
-
-  }
-
-  public TExecuteStatementReq(
-    TSessionHandle sessionHandle,
-    String statement)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-    this.statement = statement;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TExecuteStatementReq(TExecuteStatementReq other) {
-    __isset_bitfield = other.__isset_bitfield;
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-    if (other.isSetStatement()) {
-      this.statement = other.statement;
-    }
-    if (other.isSetConfOverlay()) {
-      Map<String,String> __this__confOverlay = new HashMap<String,String>();
-      for (Map.Entry<String, String> other_element : other.confOverlay.entrySet()) {
-
-        String other_element_key = other_element.getKey();
-        String other_element_value = other_element.getValue();
-
-        String __this__confOverlay_copy_key = other_element_key;
-
-        String __this__confOverlay_copy_value = other_element_value;
-
-        __this__confOverlay.put(__this__confOverlay_copy_key, __this__confOverlay_copy_value);
-      }
-      this.confOverlay = __this__confOverlay;
-    }
-    this.runAsync = other.runAsync;
-  }
-
-  public TExecuteStatementReq deepCopy() {
-    return new TExecuteStatementReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-    this.statement = null;
-    this.confOverlay = null;
-    this.runAsync = false;
-
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public String getStatement() {
-    return this.statement;
-  }
-
-  public void setStatement(String statement) {
-    this.statement = statement;
-  }
-
-  public void unsetStatement() {
-    this.statement = null;
-  }
-
-  /** Returns true if field statement is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatement() {
-    return this.statement != null;
-  }
-
-  public void setStatementIsSet(boolean value) {
-    if (!value) {
-      this.statement = null;
-    }
-  }
-
-  public int getConfOverlaySize() {
-    return (this.confOverlay == null) ? 0 : this.confOverlay.size();
-  }
-
-  public void putToConfOverlay(String key, String val) {
-    if (this.confOverlay == null) {
-      this.confOverlay = new HashMap<String,String>();
-    }
-    this.confOverlay.put(key, val);
-  }
-
-  public Map<String,String> getConfOverlay() {
-    return this.confOverlay;
-  }
-
-  public void setConfOverlay(Map<String,String> confOverlay) {
-    this.confOverlay = confOverlay;
-  }
-
-  public void unsetConfOverlay() {
-    this.confOverlay = null;
-  }
-
-  /** Returns true if field confOverlay is set (has been assigned a value) and false otherwise */
-  public boolean isSetConfOverlay() {
-    return this.confOverlay != null;
-  }
-
-  public void setConfOverlayIsSet(boolean value) {
-    if (!value) {
-      this.confOverlay = null;
-    }
-  }
-
-  public boolean isRunAsync() {
-    return this.runAsync;
-  }
-
-  public void setRunAsync(boolean runAsync) {
-    this.runAsync = runAsync;
-    setRunAsyncIsSet(true);
-  }
-
-  public void unsetRunAsync() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __RUNASYNC_ISSET_ID);
-  }
-
-  /** Returns true if field runAsync is set (has been assigned a value) and false otherwise */
-  public boolean isSetRunAsync() {
-    return EncodingUtils.testBit(__isset_bitfield, __RUNASYNC_ISSET_ID);
-  }
-
-  public void setRunAsyncIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __RUNASYNC_ISSET_ID, value);
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    case STATEMENT:
-      if (value == null) {
-        unsetStatement();
-      } else {
-        setStatement((String)value);
-      }
-      break;
-
-    case CONF_OVERLAY:
-      if (value == null) {
-        unsetConfOverlay();
-      } else {
-        setConfOverlay((Map<String,String>)value);
-      }
-      break;
-
-    case RUN_ASYNC:
-      if (value == null) {
-        unsetRunAsync();
-      } else {
-        setRunAsync((Boolean)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    case STATEMENT:
-      return getStatement();
-
-    case CONF_OVERLAY:
-      return getConfOverlay();
-
-    case RUN_ASYNC:
-      return Boolean.valueOf(isRunAsync());
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    case STATEMENT:
-      return isSetStatement();
-    case CONF_OVERLAY:
-      return isSetConfOverlay();
-    case RUN_ASYNC:
-      return isSetRunAsync();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TExecuteStatementReq)
-      return this.equals((TExecuteStatementReq)that);
-    return false;
-  }
-
-  public boolean equals(TExecuteStatementReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    boolean this_present_statement = true && this.isSetStatement();
-    boolean that_present_statement = true && that.isSetStatement();
-    if (this_present_statement || that_present_statement) {
-      if (!(this_present_statement && that_present_statement))
-        return false;
-      if (!this.statement.equals(that.statement))
-        return false;
-    }
-
-    boolean this_present_confOverlay = true && this.isSetConfOverlay();
-    boolean that_present_confOverlay = true && that.isSetConfOverlay();
-    if (this_present_confOverlay || that_present_confOverlay) {
-      if (!(this_present_confOverlay && that_present_confOverlay))
-        return false;
-      if (!this.confOverlay.equals(that.confOverlay))
-        return false;
-    }
-
-    boolean this_present_runAsync = true && this.isSetRunAsync();
-    boolean that_present_runAsync = true && that.isSetRunAsync();
-    if (this_present_runAsync || that_present_runAsync) {
-      if (!(this_present_runAsync && that_present_runAsync))
-        return false;
-      if (this.runAsync != that.runAsync)
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    builder.append(present_sessionHandle);
-    if (present_sessionHandle)
-      builder.append(sessionHandle);
-
-    boolean present_statement = true && (isSetStatement());
-    builder.append(present_statement);
-    if (present_statement)
-      builder.append(statement);
-
-    boolean present_confOverlay = true && (isSetConfOverlay());
-    builder.append(present_confOverlay);
-    if (present_confOverlay)
-      builder.append(confOverlay);
-
-    boolean present_runAsync = true && (isSetRunAsync());
-    builder.append(present_runAsync);
-    if (present_runAsync)
-      builder.append(runAsync);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TExecuteStatementReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TExecuteStatementReq typedOther = (TExecuteStatementReq)other;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(typedOther.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, typedOther.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetStatement()).compareTo(typedOther.isSetStatement());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatement()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.statement, typedOther.statement);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetConfOverlay()).compareTo(typedOther.isSetConfOverlay());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetConfOverlay()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.confOverlay, typedOther.confOverlay);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetRunAsync()).compareTo(typedOther.isSetRunAsync());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetRunAsync()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.runAsync, typedOther.runAsync);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TExecuteStatementReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("statement:");
-    if (this.statement == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.statement);
-    }
-    first = false;
-    if (isSetConfOverlay()) {
-      if (!first) sb.append(", ");
-      sb.append("confOverlay:");
-      if (this.confOverlay == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.confOverlay);
-      }
-      first = false;
-    }
-    if (isSetRunAsync()) {
-      if (!first) sb.append(", ");
-      sb.append("runAsync:");
-      sb.append(this.runAsync);
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    if (!isSetStatement()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'statement' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TExecuteStatementReqStandardSchemeFactory implements SchemeFactory {
-    public TExecuteStatementReqStandardScheme getScheme() {
-      return new TExecuteStatementReqStandardScheme();
-    }
-  }
-
-  private static class TExecuteStatementReqStandardScheme extends StandardScheme<TExecuteStatementReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TExecuteStatementReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // STATEMENT
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.statement = iprot.readString();
-              struct.setStatementIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // CONF_OVERLAY
-            if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
-              {
-                org.apache.thrift.protocol.TMap _map162 = iprot.readMapBegin();
-                struct.confOverlay = new HashMap<String,String>(2*_map162.size);
-                for (int _i163 = 0; _i163 < _map162.size; ++_i163)
-                {
-                  String _key164; // required
-                  String _val165; // required
-                  _key164 = iprot.readString();
-                  _val165 = iprot.readString();
-                  struct.confOverlay.put(_key164, _val165);
-                }
-                iprot.readMapEnd();
-              }
-              struct.setConfOverlayIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // RUN_ASYNC
-            if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
-              struct.runAsync = iprot.readBool();
-              struct.setRunAsyncIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TExecuteStatementReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.statement != null) {
-        oprot.writeFieldBegin(STATEMENT_FIELD_DESC);
-        oprot.writeString(struct.statement);
-        oprot.writeFieldEnd();
-      }
-      if (struct.confOverlay != null) {
-        if (struct.isSetConfOverlay()) {
-          oprot.writeFieldBegin(CONF_OVERLAY_FIELD_DESC);
-          {
-            oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.confOverlay.size()));
-            for (Map.Entry<String, String> _iter166 : struct.confOverlay.entrySet())
-            {
-              oprot.writeString(_iter166.getKey());
-              oprot.writeString(_iter166.getValue());
-            }
-            oprot.writeMapEnd();
-          }
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.isSetRunAsync()) {
-        oprot.writeFieldBegin(RUN_ASYNC_FIELD_DESC);
-        oprot.writeBool(struct.runAsync);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TExecuteStatementReqTupleSchemeFactory implements SchemeFactory {
-    public TExecuteStatementReqTupleScheme getScheme() {
-      return new TExecuteStatementReqTupleScheme();
-    }
-  }
-
-  private static class TExecuteStatementReqTupleScheme extends TupleScheme<TExecuteStatementReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TExecuteStatementReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-      oprot.writeString(struct.statement);
-      BitSet optionals = new BitSet();
-      if (struct.isSetConfOverlay()) {
-        optionals.set(0);
-      }
-      if (struct.isSetRunAsync()) {
-        optionals.set(1);
-      }
-      oprot.writeBitSet(optionals, 2);
-      if (struct.isSetConfOverlay()) {
-        {
-          oprot.writeI32(struct.confOverlay.size());
-          for (Map.Entry<String, String> _iter167 : struct.confOverlay.entrySet())
-          {
-            oprot.writeString(_iter167.getKey());
-            oprot.writeString(_iter167.getValue());
-          }
-        }
-      }
-      if (struct.isSetRunAsync()) {
-        oprot.writeBool(struct.runAsync);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TExecuteStatementReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-      struct.statement = iprot.readString();
-      struct.setStatementIsSet(true);
-      BitSet incoming = iprot.readBitSet(2);
-      if (incoming.get(0)) {
-        {
-          org.apache.thrift.protocol.TMap _map168 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-          struct.confOverlay = new HashMap<String,String>(2*_map168.size);
-          for (int _i169 = 0; _i169 < _map168.size; ++_i169)
-          {
-            String _key170; // required
-            String _val171; // required
-            _key170 = iprot.readString();
-            _val171 = iprot.readString();
-            struct.confOverlay.put(_key170, _val171);
-          }
-        }
-        struct.setConfOverlayIsSet(true);
-      }
-      if (incoming.get(1)) {
-        struct.runAsync = iprot.readBool();
-        struct.setRunAsyncIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TExecuteStatementResp.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TExecuteStatementResp.java
deleted file mode 100644
index fdde51e70f783..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TExecuteStatementResp.java
+++ /dev/null
@@ -1,505 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TExecuteStatementResp implements org.apache.thrift.TBase<TExecuteStatementResp, TExecuteStatementResp._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TExecuteStatementResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TExecuteStatementRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TExecuteStatementRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TOperationHandle operationHandle; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    OPERATION_HANDLE((short)2, "operationHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private _Fields optionals[] = {_Fields.OPERATION_HANDLE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TExecuteStatementResp.class, metaDataMap);
-  }
-
-  public TExecuteStatementResp() {
-  }
-
-  public TExecuteStatementResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TExecuteStatementResp(TExecuteStatementResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-  }
-
-  public TExecuteStatementResp deepCopy() {
-    return new TExecuteStatementResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.operationHandle = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TExecuteStatementResp)
-      return this.equals((TExecuteStatementResp)that);
-    return false;
-  }
-
-  public boolean equals(TExecuteStatementResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_status = true && (isSetStatus());
-    builder.append(present_status);
-    if (present_status)
-      builder.append(status);
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    builder.append(present_operationHandle);
-    if (present_operationHandle)
-      builder.append(operationHandle);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TExecuteStatementResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TExecuteStatementResp typedOther = (TExecuteStatementResp)other;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(typedOther.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, typedOther.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(typedOther.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, typedOther.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TExecuteStatementResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetOperationHandle()) {
-      if (!first) sb.append(", ");
-      sb.append("operationHandle:");
-      if (this.operationHandle == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.operationHandle);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TExecuteStatementRespStandardSchemeFactory implements SchemeFactory {
-    public TExecuteStatementRespStandardScheme getScheme() {
-      return new TExecuteStatementRespStandardScheme();
-    }
-  }
-
-  private static class TExecuteStatementRespStandardScheme extends StandardScheme<TExecuteStatementResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TExecuteStatementResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TExecuteStatementResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.operationHandle != null) {
-        if (struct.isSetOperationHandle()) {
-          oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-          struct.operationHandle.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TExecuteStatementRespTupleSchemeFactory implements SchemeFactory {
-    public TExecuteStatementRespTupleScheme getScheme() {
-      return new TExecuteStatementRespTupleScheme();
-    }
-  }
-
-  private static class TExecuteStatementRespTupleScheme extends TupleScheme<TExecuteStatementResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TExecuteStatementResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetOperationHandle()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetOperationHandle()) {
-        struct.operationHandle.write(oprot);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TExecuteStatementResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.operationHandle = new TOperationHandle();
-        struct.operationHandle.read(iprot);
-        struct.setOperationHandleIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TFetchOrientation.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TFetchOrientation.java
deleted file mode 100644
index b2a22effd91af..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TFetchOrientation.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.thrift.TEnum;
-
-public enum TFetchOrientation implements org.apache.thrift.TEnum {
-  FETCH_NEXT(0),
-  FETCH_PRIOR(1),
-  FETCH_RELATIVE(2),
-  FETCH_ABSOLUTE(3),
-  FETCH_FIRST(4),
-  FETCH_LAST(5);
-
-  private final int value;
-
-  private TFetchOrientation(int value) {
-    this.value = value;
-  }
-
-  /**
-   * Get the integer value of this enum value, as defined in the Thrift IDL.
-   */
-  public int getValue() {
-    return value;
-  }
-
-  /**
-   * Find a the enum type by its integer value, as defined in the Thrift IDL.
-   * @return null if the value is not found.
-   */
-  public static TFetchOrientation findByValue(int value) { 
-    switch (value) {
-      case 0:
-        return FETCH_NEXT;
-      case 1:
-        return FETCH_PRIOR;
-      case 2:
-        return FETCH_RELATIVE;
-      case 3:
-        return FETCH_ABSOLUTE;
-      case 4:
-        return FETCH_FIRST;
-      case 5:
-        return FETCH_LAST;
-      default:
-        return null;
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TFetchResultsReq.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TFetchResultsReq.java
deleted file mode 100644
index 068711fc44440..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TFetchResultsReq.java
+++ /dev/null
@@ -1,710 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TFetchResultsReq implements org.apache.thrift.TBase<TFetchResultsReq, TFetchResultsReq._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TFetchResultsReq");
-
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField ORIENTATION_FIELD_DESC = new org.apache.thrift.protocol.TField("orientation", org.apache.thrift.protocol.TType.I32, (short)2);
-  private static final org.apache.thrift.protocol.TField MAX_ROWS_FIELD_DESC = new org.apache.thrift.protocol.TField("maxRows", org.apache.thrift.protocol.TType.I64, (short)3);
-  private static final org.apache.thrift.protocol.TField FETCH_TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("fetchType", org.apache.thrift.protocol.TType.I16, (short)4);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TFetchResultsReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TFetchResultsReqTupleSchemeFactory());
-  }
-
-  private TOperationHandle operationHandle; // required
-  private TFetchOrientation orientation; // required
-  private long maxRows; // required
-  private short fetchType; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    OPERATION_HANDLE((short)1, "operationHandle"),
-    /**
-     * 
-     * @see TFetchOrientation
-     */
-    ORIENTATION((short)2, "orientation"),
-    MAX_ROWS((short)3, "maxRows"),
-    FETCH_TYPE((short)4, "fetchType");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        case 2: // ORIENTATION
-          return ORIENTATION;
-        case 3: // MAX_ROWS
-          return MAX_ROWS;
-        case 4: // FETCH_TYPE
-          return FETCH_TYPE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __MAXROWS_ISSET_ID = 0;
-  private static final int __FETCHTYPE_ISSET_ID = 1;
-  private byte __isset_bitfield = 0;
-  private _Fields optionals[] = {_Fields.FETCH_TYPE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    tmpMap.put(_Fields.ORIENTATION, new org.apache.thrift.meta_data.FieldMetaData("orientation", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TFetchOrientation.class)));
-    tmpMap.put(_Fields.MAX_ROWS, new org.apache.thrift.meta_data.FieldMetaData("maxRows", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
-    tmpMap.put(_Fields.FETCH_TYPE, new org.apache.thrift.meta_data.FieldMetaData("fetchType", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I16)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TFetchResultsReq.class, metaDataMap);
-  }
-
-  public TFetchResultsReq() {
-    this.orientation = org.apache.hive.service.cli.thrift.TFetchOrientation.FETCH_NEXT;
-
-    this.fetchType = (short)0;
-
-  }
-
-  public TFetchResultsReq(
-    TOperationHandle operationHandle,
-    TFetchOrientation orientation,
-    long maxRows)
-  {
-    this();
-    this.operationHandle = operationHandle;
-    this.orientation = orientation;
-    this.maxRows = maxRows;
-    setMaxRowsIsSet(true);
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TFetchResultsReq(TFetchResultsReq other) {
-    __isset_bitfield = other.__isset_bitfield;
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-    if (other.isSetOrientation()) {
-      this.orientation = other.orientation;
-    }
-    this.maxRows = other.maxRows;
-    this.fetchType = other.fetchType;
-  }
-
-  public TFetchResultsReq deepCopy() {
-    return new TFetchResultsReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.operationHandle = null;
-    this.orientation = org.apache.hive.service.cli.thrift.TFetchOrientation.FETCH_NEXT;
-
-    setMaxRowsIsSet(false);
-    this.maxRows = 0;
-    this.fetchType = (short)0;
-
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  /**
-   * 
-   * @see TFetchOrientation
-   */
-  public TFetchOrientation getOrientation() {
-    return this.orientation;
-  }
-
-  /**
-   * 
-   * @see TFetchOrientation
-   */
-  public void setOrientation(TFetchOrientation orientation) {
-    this.orientation = orientation;
-  }
-
-  public void unsetOrientation() {
-    this.orientation = null;
-  }
-
-  /** Returns true if field orientation is set (has been assigned a value) and false otherwise */
-  public boolean isSetOrientation() {
-    return this.orientation != null;
-  }
-
-  public void setOrientationIsSet(boolean value) {
-    if (!value) {
-      this.orientation = null;
-    }
-  }
-
-  public long getMaxRows() {
-    return this.maxRows;
-  }
-
-  public void setMaxRows(long maxRows) {
-    this.maxRows = maxRows;
-    setMaxRowsIsSet(true);
-  }
-
-  public void unsetMaxRows() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __MAXROWS_ISSET_ID);
-  }
-
-  /** Returns true if field maxRows is set (has been assigned a value) and false otherwise */
-  public boolean isSetMaxRows() {
-    return EncodingUtils.testBit(__isset_bitfield, __MAXROWS_ISSET_ID);
-  }
-
-  public void setMaxRowsIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MAXROWS_ISSET_ID, value);
-  }
-
-  public short getFetchType() {
-    return this.fetchType;
-  }
-
-  public void setFetchType(short fetchType) {
-    this.fetchType = fetchType;
-    setFetchTypeIsSet(true);
-  }
-
-  public void unsetFetchType() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __FETCHTYPE_ISSET_ID);
-  }
-
-  /** Returns true if field fetchType is set (has been assigned a value) and false otherwise */
-  public boolean isSetFetchType() {
-    return EncodingUtils.testBit(__isset_bitfield, __FETCHTYPE_ISSET_ID);
-  }
-
-  public void setFetchTypeIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __FETCHTYPE_ISSET_ID, value);
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    case ORIENTATION:
-      if (value == null) {
-        unsetOrientation();
-      } else {
-        setOrientation((TFetchOrientation)value);
-      }
-      break;
-
-    case MAX_ROWS:
-      if (value == null) {
-        unsetMaxRows();
-      } else {
-        setMaxRows((Long)value);
-      }
-      break;
-
-    case FETCH_TYPE:
-      if (value == null) {
-        unsetFetchType();
-      } else {
-        setFetchType((Short)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    case ORIENTATION:
-      return getOrientation();
-
-    case MAX_ROWS:
-      return Long.valueOf(getMaxRows());
-
-    case FETCH_TYPE:
-      return Short.valueOf(getFetchType());
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    case ORIENTATION:
-      return isSetOrientation();
-    case MAX_ROWS:
-      return isSetMaxRows();
-    case FETCH_TYPE:
-      return isSetFetchType();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TFetchResultsReq)
-      return this.equals((TFetchResultsReq)that);
-    return false;
-  }
-
-  public boolean equals(TFetchResultsReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    boolean this_present_orientation = true && this.isSetOrientation();
-    boolean that_present_orientation = true && that.isSetOrientation();
-    if (this_present_orientation || that_present_orientation) {
-      if (!(this_present_orientation && that_present_orientation))
-        return false;
-      if (!this.orientation.equals(that.orientation))
-        return false;
-    }
-
-    boolean this_present_maxRows = true;
-    boolean that_present_maxRows = true;
-    if (this_present_maxRows || that_present_maxRows) {
-      if (!(this_present_maxRows && that_present_maxRows))
-        return false;
-      if (this.maxRows != that.maxRows)
-        return false;
-    }
-
-    boolean this_present_fetchType = true && this.isSetFetchType();
-    boolean that_present_fetchType = true && that.isSetFetchType();
-    if (this_present_fetchType || that_present_fetchType) {
-      if (!(this_present_fetchType && that_present_fetchType))
-        return false;
-      if (this.fetchType != that.fetchType)
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    builder.append(present_operationHandle);
-    if (present_operationHandle)
-      builder.append(operationHandle);
-
-    boolean present_orientation = true && (isSetOrientation());
-    builder.append(present_orientation);
-    if (present_orientation)
-      builder.append(orientation.getValue());
-
-    boolean present_maxRows = true;
-    builder.append(present_maxRows);
-    if (present_maxRows)
-      builder.append(maxRows);
-
-    boolean present_fetchType = true && (isSetFetchType());
-    builder.append(present_fetchType);
-    if (present_fetchType)
-      builder.append(fetchType);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TFetchResultsReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TFetchResultsReq typedOther = (TFetchResultsReq)other;
-
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(typedOther.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, typedOther.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOrientation()).compareTo(typedOther.isSetOrientation());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOrientation()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.orientation, typedOther.orientation);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetMaxRows()).compareTo(typedOther.isSetMaxRows());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetMaxRows()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.maxRows, typedOther.maxRows);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetFetchType()).compareTo(typedOther.isSetFetchType());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetFetchType()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.fetchType, typedOther.fetchType);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TFetchResultsReq(");
-    boolean first = true;
-
-    sb.append("operationHandle:");
-    if (this.operationHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.operationHandle);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("orientation:");
-    if (this.orientation == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.orientation);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("maxRows:");
-    sb.append(this.maxRows);
-    first = false;
-    if (isSetFetchType()) {
-      if (!first) sb.append(", ");
-      sb.append("fetchType:");
-      sb.append(this.fetchType);
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetOperationHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'operationHandle' is unset! Struct:" + toString());
-    }
-
-    if (!isSetOrientation()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'orientation' is unset! Struct:" + toString());
-    }
-
-    if (!isSetMaxRows()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'maxRows' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TFetchResultsReqStandardSchemeFactory implements SchemeFactory {
-    public TFetchResultsReqStandardScheme getScheme() {
-      return new TFetchResultsReqStandardScheme();
-    }
-  }
-
-  private static class TFetchResultsReqStandardScheme extends StandardScheme<TFetchResultsReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TFetchResultsReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // ORIENTATION
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.orientation = TFetchOrientation.findByValue(iprot.readI32());
-              struct.setOrientationIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // MAX_ROWS
-            if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
-              struct.maxRows = iprot.readI64();
-              struct.setMaxRowsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // FETCH_TYPE
-            if (schemeField.type == org.apache.thrift.protocol.TType.I16) {
-              struct.fetchType = iprot.readI16();
-              struct.setFetchTypeIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TFetchResultsReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.operationHandle != null) {
-        oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-        struct.operationHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.orientation != null) {
-        oprot.writeFieldBegin(ORIENTATION_FIELD_DESC);
-        oprot.writeI32(struct.orientation.getValue());
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldBegin(MAX_ROWS_FIELD_DESC);
-      oprot.writeI64(struct.maxRows);
-      oprot.writeFieldEnd();
-      if (struct.isSetFetchType()) {
-        oprot.writeFieldBegin(FETCH_TYPE_FIELD_DESC);
-        oprot.writeI16(struct.fetchType);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TFetchResultsReqTupleSchemeFactory implements SchemeFactory {
-    public TFetchResultsReqTupleScheme getScheme() {
-      return new TFetchResultsReqTupleScheme();
-    }
-  }
-
-  private static class TFetchResultsReqTupleScheme extends TupleScheme<TFetchResultsReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TFetchResultsReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.operationHandle.write(oprot);
-      oprot.writeI32(struct.orientation.getValue());
-      oprot.writeI64(struct.maxRows);
-      BitSet optionals = new BitSet();
-      if (struct.isSetFetchType()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetFetchType()) {
-        oprot.writeI16(struct.fetchType);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TFetchResultsReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.operationHandle = new TOperationHandle();
-      struct.operationHandle.read(iprot);
-      struct.setOperationHandleIsSet(true);
-      struct.orientation = TFetchOrientation.findByValue(iprot.readI32());
-      struct.setOrientationIsSet(true);
-      struct.maxRows = iprot.readI64();
-      struct.setMaxRowsIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.fetchType = iprot.readI16();
-        struct.setFetchTypeIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TFetchResultsResp.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TFetchResultsResp.java
deleted file mode 100644
index 19991f1da3eb3..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TFetchResultsResp.java
+++ /dev/null
@@ -1,608 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TFetchResultsResp implements org.apache.thrift.TBase<TFetchResultsResp, TFetchResultsResp._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TFetchResultsResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField HAS_MORE_ROWS_FIELD_DESC = new org.apache.thrift.protocol.TField("hasMoreRows", org.apache.thrift.protocol.TType.BOOL, (short)2);
-  private static final org.apache.thrift.protocol.TField RESULTS_FIELD_DESC = new org.apache.thrift.protocol.TField("results", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TFetchResultsRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TFetchResultsRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private boolean hasMoreRows; // optional
-  private TRowSet results; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    HAS_MORE_ROWS((short)2, "hasMoreRows"),
-    RESULTS((short)3, "results");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // HAS_MORE_ROWS
-          return HAS_MORE_ROWS;
-        case 3: // RESULTS
-          return RESULTS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __HASMOREROWS_ISSET_ID = 0;
-  private byte __isset_bitfield = 0;
-  private _Fields optionals[] = {_Fields.HAS_MORE_ROWS,_Fields.RESULTS};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.HAS_MORE_ROWS, new org.apache.thrift.meta_data.FieldMetaData("hasMoreRows", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
-    tmpMap.put(_Fields.RESULTS, new org.apache.thrift.meta_data.FieldMetaData("results", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TRowSet.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TFetchResultsResp.class, metaDataMap);
-  }
-
-  public TFetchResultsResp() {
-  }
-
-  public TFetchResultsResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TFetchResultsResp(TFetchResultsResp other) {
-    __isset_bitfield = other.__isset_bitfield;
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    this.hasMoreRows = other.hasMoreRows;
-    if (other.isSetResults()) {
-      this.results = new TRowSet(other.results);
-    }
-  }
-
-  public TFetchResultsResp deepCopy() {
-    return new TFetchResultsResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    setHasMoreRowsIsSet(false);
-    this.hasMoreRows = false;
-    this.results = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public boolean isHasMoreRows() {
-    return this.hasMoreRows;
-  }
-
-  public void setHasMoreRows(boolean hasMoreRows) {
-    this.hasMoreRows = hasMoreRows;
-    setHasMoreRowsIsSet(true);
-  }
-
-  public void unsetHasMoreRows() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __HASMOREROWS_ISSET_ID);
-  }
-
-  /** Returns true if field hasMoreRows is set (has been assigned a value) and false otherwise */
-  public boolean isSetHasMoreRows() {
-    return EncodingUtils.testBit(__isset_bitfield, __HASMOREROWS_ISSET_ID);
-  }
-
-  public void setHasMoreRowsIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __HASMOREROWS_ISSET_ID, value);
-  }
-
-  public TRowSet getResults() {
-    return this.results;
-  }
-
-  public void setResults(TRowSet results) {
-    this.results = results;
-  }
-
-  public void unsetResults() {
-    this.results = null;
-  }
-
-  /** Returns true if field results is set (has been assigned a value) and false otherwise */
-  public boolean isSetResults() {
-    return this.results != null;
-  }
-
-  public void setResultsIsSet(boolean value) {
-    if (!value) {
-      this.results = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case HAS_MORE_ROWS:
-      if (value == null) {
-        unsetHasMoreRows();
-      } else {
-        setHasMoreRows((Boolean)value);
-      }
-      break;
-
-    case RESULTS:
-      if (value == null) {
-        unsetResults();
-      } else {
-        setResults((TRowSet)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case HAS_MORE_ROWS:
-      return Boolean.valueOf(isHasMoreRows());
-
-    case RESULTS:
-      return getResults();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case HAS_MORE_ROWS:
-      return isSetHasMoreRows();
-    case RESULTS:
-      return isSetResults();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TFetchResultsResp)
-      return this.equals((TFetchResultsResp)that);
-    return false;
-  }
-
-  public boolean equals(TFetchResultsResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_hasMoreRows = true && this.isSetHasMoreRows();
-    boolean that_present_hasMoreRows = true && that.isSetHasMoreRows();
-    if (this_present_hasMoreRows || that_present_hasMoreRows) {
-      if (!(this_present_hasMoreRows && that_present_hasMoreRows))
-        return false;
-      if (this.hasMoreRows != that.hasMoreRows)
-        return false;
-    }
-
-    boolean this_present_results = true && this.isSetResults();
-    boolean that_present_results = true && that.isSetResults();
-    if (this_present_results || that_present_results) {
-      if (!(this_present_results && that_present_results))
-        return false;
-      if (!this.results.equals(that.results))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_status = true && (isSetStatus());
-    builder.append(present_status);
-    if (present_status)
-      builder.append(status);
-
-    boolean present_hasMoreRows = true && (isSetHasMoreRows());
-    builder.append(present_hasMoreRows);
-    if (present_hasMoreRows)
-      builder.append(hasMoreRows);
-
-    boolean present_results = true && (isSetResults());
-    builder.append(present_results);
-    if (present_results)
-      builder.append(results);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TFetchResultsResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TFetchResultsResp typedOther = (TFetchResultsResp)other;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(typedOther.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, typedOther.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetHasMoreRows()).compareTo(typedOther.isSetHasMoreRows());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetHasMoreRows()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.hasMoreRows, typedOther.hasMoreRows);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetResults()).compareTo(typedOther.isSetResults());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetResults()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.results, typedOther.results);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TFetchResultsResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetHasMoreRows()) {
-      if (!first) sb.append(", ");
-      sb.append("hasMoreRows:");
-      sb.append(this.hasMoreRows);
-      first = false;
-    }
-    if (isSetResults()) {
-      if (!first) sb.append(", ");
-      sb.append("results:");
-      if (this.results == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.results);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-    if (results != null) {
-      results.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TFetchResultsRespStandardSchemeFactory implements SchemeFactory {
-    public TFetchResultsRespStandardScheme getScheme() {
-      return new TFetchResultsRespStandardScheme();
-    }
-  }
-
-  private static class TFetchResultsRespStandardScheme extends StandardScheme<TFetchResultsResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TFetchResultsResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // HAS_MORE_ROWS
-            if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
-              struct.hasMoreRows = iprot.readBool();
-              struct.setHasMoreRowsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // RESULTS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.results = new TRowSet();
-              struct.results.read(iprot);
-              struct.setResultsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TFetchResultsResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.isSetHasMoreRows()) {
-        oprot.writeFieldBegin(HAS_MORE_ROWS_FIELD_DESC);
-        oprot.writeBool(struct.hasMoreRows);
-        oprot.writeFieldEnd();
-      }
-      if (struct.results != null) {
-        if (struct.isSetResults()) {
-          oprot.writeFieldBegin(RESULTS_FIELD_DESC);
-          struct.results.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TFetchResultsRespTupleSchemeFactory implements SchemeFactory {
-    public TFetchResultsRespTupleScheme getScheme() {
-      return new TFetchResultsRespTupleScheme();
-    }
-  }
-
-  private static class TFetchResultsRespTupleScheme extends TupleScheme<TFetchResultsResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TFetchResultsResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetHasMoreRows()) {
-        optionals.set(0);
-      }
-      if (struct.isSetResults()) {
-        optionals.set(1);
-      }
-      oprot.writeBitSet(optionals, 2);
-      if (struct.isSetHasMoreRows()) {
-        oprot.writeBool(struct.hasMoreRows);
-      }
-      if (struct.isSetResults()) {
-        struct.results.write(oprot);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TFetchResultsResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(2);
-      if (incoming.get(0)) {
-        struct.hasMoreRows = iprot.readBool();
-        struct.setHasMoreRowsIsSet(true);
-      }
-      if (incoming.get(1)) {
-        struct.results = new TRowSet();
-        struct.results.read(iprot);
-        struct.setResultsIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetCatalogsReq.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetCatalogsReq.java
deleted file mode 100644
index cfd157f701b26..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetCatalogsReq.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TGetCatalogsReq implements org.apache.thrift.TBase<TGetCatalogsReq, TGetCatalogsReq._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetCatalogsReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetCatalogsReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetCatalogsReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetCatalogsReq.class, metaDataMap);
-  }
-
-  public TGetCatalogsReq() {
-  }
-
-  public TGetCatalogsReq(
-    TSessionHandle sessionHandle)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetCatalogsReq(TGetCatalogsReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-  }
-
-  public TGetCatalogsReq deepCopy() {
-    return new TGetCatalogsReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetCatalogsReq)
-      return this.equals((TGetCatalogsReq)that);
-    return false;
-  }
-
-  public boolean equals(TGetCatalogsReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    builder.append(present_sessionHandle);
-    if (present_sessionHandle)
-      builder.append(sessionHandle);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TGetCatalogsReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TGetCatalogsReq typedOther = (TGetCatalogsReq)other;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(typedOther.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, typedOther.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetCatalogsReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetCatalogsReqStandardSchemeFactory implements SchemeFactory {
-    public TGetCatalogsReqStandardScheme getScheme() {
-      return new TGetCatalogsReqStandardScheme();
-    }
-  }
-
-  private static class TGetCatalogsReqStandardScheme extends StandardScheme<TGetCatalogsReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetCatalogsReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetCatalogsReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetCatalogsReqTupleSchemeFactory implements SchemeFactory {
-    public TGetCatalogsReqTupleScheme getScheme() {
-      return new TGetCatalogsReqTupleScheme();
-    }
-  }
-
-  private static class TGetCatalogsReqTupleScheme extends TupleScheme<TGetCatalogsReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetCatalogsReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetCatalogsReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetCatalogsResp.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetCatalogsResp.java
deleted file mode 100644
index 1c5a35437d416..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetCatalogsResp.java
+++ /dev/null
@@ -1,505 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TGetCatalogsResp implements org.apache.thrift.TBase<TGetCatalogsResp, TGetCatalogsResp._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetCatalogsResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetCatalogsRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetCatalogsRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TOperationHandle operationHandle; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    OPERATION_HANDLE((short)2, "operationHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private _Fields optionals[] = {_Fields.OPERATION_HANDLE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetCatalogsResp.class, metaDataMap);
-  }
-
-  public TGetCatalogsResp() {
-  }
-
-  public TGetCatalogsResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetCatalogsResp(TGetCatalogsResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-  }
-
-  public TGetCatalogsResp deepCopy() {
-    return new TGetCatalogsResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.operationHandle = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetCatalogsResp)
-      return this.equals((TGetCatalogsResp)that);
-    return false;
-  }
-
-  public boolean equals(TGetCatalogsResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_status = true && (isSetStatus());
-    builder.append(present_status);
-    if (present_status)
-      builder.append(status);
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    builder.append(present_operationHandle);
-    if (present_operationHandle)
-      builder.append(operationHandle);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TGetCatalogsResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TGetCatalogsResp typedOther = (TGetCatalogsResp)other;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(typedOther.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, typedOther.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(typedOther.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, typedOther.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetCatalogsResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetOperationHandle()) {
-      if (!first) sb.append(", ");
-      sb.append("operationHandle:");
-      if (this.operationHandle == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.operationHandle);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetCatalogsRespStandardSchemeFactory implements SchemeFactory {
-    public TGetCatalogsRespStandardScheme getScheme() {
-      return new TGetCatalogsRespStandardScheme();
-    }
-  }
-
-  private static class TGetCatalogsRespStandardScheme extends StandardScheme<TGetCatalogsResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetCatalogsResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetCatalogsResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.operationHandle != null) {
-        if (struct.isSetOperationHandle()) {
-          oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-          struct.operationHandle.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetCatalogsRespTupleSchemeFactory implements SchemeFactory {
-    public TGetCatalogsRespTupleScheme getScheme() {
-      return new TGetCatalogsRespTupleScheme();
-    }
-  }
-
-  private static class TGetCatalogsRespTupleScheme extends TupleScheme<TGetCatalogsResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetCatalogsResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetOperationHandle()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetOperationHandle()) {
-        struct.operationHandle.write(oprot);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetCatalogsResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.operationHandle = new TOperationHandle();
-        struct.operationHandle.read(iprot);
-        struct.setOperationHandleIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetColumnsReq.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetColumnsReq.java
deleted file mode 100644
index a2c793bd95927..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetColumnsReq.java
+++ /dev/null
@@ -1,818 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TGetColumnsReq implements org.apache.thrift.TBase<TGetColumnsReq, TGetColumnsReq._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetColumnsReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField CATALOG_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("catalogName", org.apache.thrift.protocol.TType.STRING, (short)2);
-  private static final org.apache.thrift.protocol.TField SCHEMA_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("schemaName", org.apache.thrift.protocol.TType.STRING, (short)3);
-  private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)4);
-  private static final org.apache.thrift.protocol.TField COLUMN_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("columnName", org.apache.thrift.protocol.TType.STRING, (short)5);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetColumnsReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetColumnsReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-  private String catalogName; // optional
-  private String schemaName; // optional
-  private String tableName; // optional
-  private String columnName; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle"),
-    CATALOG_NAME((short)2, "catalogName"),
-    SCHEMA_NAME((short)3, "schemaName"),
-    TABLE_NAME((short)4, "tableName"),
-    COLUMN_NAME((short)5, "columnName");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        case 2: // CATALOG_NAME
-          return CATALOG_NAME;
-        case 3: // SCHEMA_NAME
-          return SCHEMA_NAME;
-        case 4: // TABLE_NAME
-          return TABLE_NAME;
-        case 5: // COLUMN_NAME
-          return COLUMN_NAME;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private _Fields optionals[] = {_Fields.CATALOG_NAME,_Fields.SCHEMA_NAME,_Fields.TABLE_NAME,_Fields.COLUMN_NAME};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    tmpMap.put(_Fields.CATALOG_NAME, new org.apache.thrift.meta_data.FieldMetaData("catalogName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TIdentifier")));
-    tmpMap.put(_Fields.SCHEMA_NAME, new org.apache.thrift.meta_data.FieldMetaData("schemaName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TPatternOrIdentifier")));
-    tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TPatternOrIdentifier")));
-    tmpMap.put(_Fields.COLUMN_NAME, new org.apache.thrift.meta_data.FieldMetaData("columnName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TPatternOrIdentifier")));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetColumnsReq.class, metaDataMap);
-  }
-
-  public TGetColumnsReq() {
-  }
-
-  public TGetColumnsReq(
-    TSessionHandle sessionHandle)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetColumnsReq(TGetColumnsReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-    if (other.isSetCatalogName()) {
-      this.catalogName = other.catalogName;
-    }
-    if (other.isSetSchemaName()) {
-      this.schemaName = other.schemaName;
-    }
-    if (other.isSetTableName()) {
-      this.tableName = other.tableName;
-    }
-    if (other.isSetColumnName()) {
-      this.columnName = other.columnName;
-    }
-  }
-
-  public TGetColumnsReq deepCopy() {
-    return new TGetColumnsReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-    this.catalogName = null;
-    this.schemaName = null;
-    this.tableName = null;
-    this.columnName = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public String getCatalogName() {
-    return this.catalogName;
-  }
-
-  public void setCatalogName(String catalogName) {
-    this.catalogName = catalogName;
-  }
-
-  public void unsetCatalogName() {
-    this.catalogName = null;
-  }
-
-  /** Returns true if field catalogName is set (has been assigned a value) and false otherwise */
-  public boolean isSetCatalogName() {
-    return this.catalogName != null;
-  }
-
-  public void setCatalogNameIsSet(boolean value) {
-    if (!value) {
-      this.catalogName = null;
-    }
-  }
-
-  public String getSchemaName() {
-    return this.schemaName;
-  }
-
-  public void setSchemaName(String schemaName) {
-    this.schemaName = schemaName;
-  }
-
-  public void unsetSchemaName() {
-    this.schemaName = null;
-  }
-
-  /** Returns true if field schemaName is set (has been assigned a value) and false otherwise */
-  public boolean isSetSchemaName() {
-    return this.schemaName != null;
-  }
-
-  public void setSchemaNameIsSet(boolean value) {
-    if (!value) {
-      this.schemaName = null;
-    }
-  }
-
-  public String getTableName() {
-    return this.tableName;
-  }
-
-  public void setTableName(String tableName) {
-    this.tableName = tableName;
-  }
-
-  public void unsetTableName() {
-    this.tableName = null;
-  }
-
-  /** Returns true if field tableName is set (has been assigned a value) and false otherwise */
-  public boolean isSetTableName() {
-    return this.tableName != null;
-  }
-
-  public void setTableNameIsSet(boolean value) {
-    if (!value) {
-      this.tableName = null;
-    }
-  }
-
-  public String getColumnName() {
-    return this.columnName;
-  }
-
-  public void setColumnName(String columnName) {
-    this.columnName = columnName;
-  }
-
-  public void unsetColumnName() {
-    this.columnName = null;
-  }
-
-  /** Returns true if field columnName is set (has been assigned a value) and false otherwise */
-  public boolean isSetColumnName() {
-    return this.columnName != null;
-  }
-
-  public void setColumnNameIsSet(boolean value) {
-    if (!value) {
-      this.columnName = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    case CATALOG_NAME:
-      if (value == null) {
-        unsetCatalogName();
-      } else {
-        setCatalogName((String)value);
-      }
-      break;
-
-    case SCHEMA_NAME:
-      if (value == null) {
-        unsetSchemaName();
-      } else {
-        setSchemaName((String)value);
-      }
-      break;
-
-    case TABLE_NAME:
-      if (value == null) {
-        unsetTableName();
-      } else {
-        setTableName((String)value);
-      }
-      break;
-
-    case COLUMN_NAME:
-      if (value == null) {
-        unsetColumnName();
-      } else {
-        setColumnName((String)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    case CATALOG_NAME:
-      return getCatalogName();
-
-    case SCHEMA_NAME:
-      return getSchemaName();
-
-    case TABLE_NAME:
-      return getTableName();
-
-    case COLUMN_NAME:
-      return getColumnName();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    case CATALOG_NAME:
-      return isSetCatalogName();
-    case SCHEMA_NAME:
-      return isSetSchemaName();
-    case TABLE_NAME:
-      return isSetTableName();
-    case COLUMN_NAME:
-      return isSetColumnName();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetColumnsReq)
-      return this.equals((TGetColumnsReq)that);
-    return false;
-  }
-
-  public boolean equals(TGetColumnsReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    boolean this_present_catalogName = true && this.isSetCatalogName();
-    boolean that_present_catalogName = true && that.isSetCatalogName();
-    if (this_present_catalogName || that_present_catalogName) {
-      if (!(this_present_catalogName && that_present_catalogName))
-        return false;
-      if (!this.catalogName.equals(that.catalogName))
-        return false;
-    }
-
-    boolean this_present_schemaName = true && this.isSetSchemaName();
-    boolean that_present_schemaName = true && that.isSetSchemaName();
-    if (this_present_schemaName || that_present_schemaName) {
-      if (!(this_present_schemaName && that_present_schemaName))
-        return false;
-      if (!this.schemaName.equals(that.schemaName))
-        return false;
-    }
-
-    boolean this_present_tableName = true && this.isSetTableName();
-    boolean that_present_tableName = true && that.isSetTableName();
-    if (this_present_tableName || that_present_tableName) {
-      if (!(this_present_tableName && that_present_tableName))
-        return false;
-      if (!this.tableName.equals(that.tableName))
-        return false;
-    }
-
-    boolean this_present_columnName = true && this.isSetColumnName();
-    boolean that_present_columnName = true && that.isSetColumnName();
-    if (this_present_columnName || that_present_columnName) {
-      if (!(this_present_columnName && that_present_columnName))
-        return false;
-      if (!this.columnName.equals(that.columnName))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    builder.append(present_sessionHandle);
-    if (present_sessionHandle)
-      builder.append(sessionHandle);
-
-    boolean present_catalogName = true && (isSetCatalogName());
-    builder.append(present_catalogName);
-    if (present_catalogName)
-      builder.append(catalogName);
-
-    boolean present_schemaName = true && (isSetSchemaName());
-    builder.append(present_schemaName);
-    if (present_schemaName)
-      builder.append(schemaName);
-
-    boolean present_tableName = true && (isSetTableName());
-    builder.append(present_tableName);
-    if (present_tableName)
-      builder.append(tableName);
-
-    boolean present_columnName = true && (isSetColumnName());
-    builder.append(present_columnName);
-    if (present_columnName)
-      builder.append(columnName);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TGetColumnsReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TGetColumnsReq typedOther = (TGetColumnsReq)other;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(typedOther.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, typedOther.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetCatalogName()).compareTo(typedOther.isSetCatalogName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetCatalogName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.catalogName, typedOther.catalogName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetSchemaName()).compareTo(typedOther.isSetSchemaName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSchemaName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.schemaName, typedOther.schemaName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetTableName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetColumnName()).compareTo(typedOther.isSetColumnName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetColumnName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columnName, typedOther.columnName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetColumnsReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    if (isSetCatalogName()) {
-      if (!first) sb.append(", ");
-      sb.append("catalogName:");
-      if (this.catalogName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.catalogName);
-      }
-      first = false;
-    }
-    if (isSetSchemaName()) {
-      if (!first) sb.append(", ");
-      sb.append("schemaName:");
-      if (this.schemaName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.schemaName);
-      }
-      first = false;
-    }
-    if (isSetTableName()) {
-      if (!first) sb.append(", ");
-      sb.append("tableName:");
-      if (this.tableName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.tableName);
-      }
-      first = false;
-    }
-    if (isSetColumnName()) {
-      if (!first) sb.append(", ");
-      sb.append("columnName:");
-      if (this.columnName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.columnName);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetColumnsReqStandardSchemeFactory implements SchemeFactory {
-    public TGetColumnsReqStandardScheme getScheme() {
-      return new TGetColumnsReqStandardScheme();
-    }
-  }
-
-  private static class TGetColumnsReqStandardScheme extends StandardScheme<TGetColumnsReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetColumnsReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // CATALOG_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.catalogName = iprot.readString();
-              struct.setCatalogNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // SCHEMA_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.schemaName = iprot.readString();
-              struct.setSchemaNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // TABLE_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.tableName = iprot.readString();
-              struct.setTableNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 5: // COLUMN_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.columnName = iprot.readString();
-              struct.setColumnNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetColumnsReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.catalogName != null) {
-        if (struct.isSetCatalogName()) {
-          oprot.writeFieldBegin(CATALOG_NAME_FIELD_DESC);
-          oprot.writeString(struct.catalogName);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.schemaName != null) {
-        if (struct.isSetSchemaName()) {
-          oprot.writeFieldBegin(SCHEMA_NAME_FIELD_DESC);
-          oprot.writeString(struct.schemaName);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.tableName != null) {
-        if (struct.isSetTableName()) {
-          oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC);
-          oprot.writeString(struct.tableName);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.columnName != null) {
-        if (struct.isSetColumnName()) {
-          oprot.writeFieldBegin(COLUMN_NAME_FIELD_DESC);
-          oprot.writeString(struct.columnName);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetColumnsReqTupleSchemeFactory implements SchemeFactory {
-    public TGetColumnsReqTupleScheme getScheme() {
-      return new TGetColumnsReqTupleScheme();
-    }
-  }
-
-  private static class TGetColumnsReqTupleScheme extends TupleScheme<TGetColumnsReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetColumnsReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetCatalogName()) {
-        optionals.set(0);
-      }
-      if (struct.isSetSchemaName()) {
-        optionals.set(1);
-      }
-      if (struct.isSetTableName()) {
-        optionals.set(2);
-      }
-      if (struct.isSetColumnName()) {
-        optionals.set(3);
-      }
-      oprot.writeBitSet(optionals, 4);
-      if (struct.isSetCatalogName()) {
-        oprot.writeString(struct.catalogName);
-      }
-      if (struct.isSetSchemaName()) {
-        oprot.writeString(struct.schemaName);
-      }
-      if (struct.isSetTableName()) {
-        oprot.writeString(struct.tableName);
-      }
-      if (struct.isSetColumnName()) {
-        oprot.writeString(struct.columnName);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetColumnsReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-      BitSet incoming = iprot.readBitSet(4);
-      if (incoming.get(0)) {
-        struct.catalogName = iprot.readString();
-        struct.setCatalogNameIsSet(true);
-      }
-      if (incoming.get(1)) {
-        struct.schemaName = iprot.readString();
-        struct.setSchemaNameIsSet(true);
-      }
-      if (incoming.get(2)) {
-        struct.tableName = iprot.readString();
-        struct.setTableNameIsSet(true);
-      }
-      if (incoming.get(3)) {
-        struct.columnName = iprot.readString();
-        struct.setColumnNameIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetColumnsResp.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetColumnsResp.java
deleted file mode 100644
index d6cf1be6d304b..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetColumnsResp.java
+++ /dev/null
@@ -1,505 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TGetColumnsResp implements org.apache.thrift.TBase<TGetColumnsResp, TGetColumnsResp._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetColumnsResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetColumnsRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetColumnsRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TOperationHandle operationHandle; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    OPERATION_HANDLE((short)2, "operationHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private _Fields optionals[] = {_Fields.OPERATION_HANDLE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetColumnsResp.class, metaDataMap);
-  }
-
-  public TGetColumnsResp() {
-  }
-
-  public TGetColumnsResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetColumnsResp(TGetColumnsResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-  }
-
-  public TGetColumnsResp deepCopy() {
-    return new TGetColumnsResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.operationHandle = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetColumnsResp)
-      return this.equals((TGetColumnsResp)that);
-    return false;
-  }
-
-  public boolean equals(TGetColumnsResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_status = true && (isSetStatus());
-    builder.append(present_status);
-    if (present_status)
-      builder.append(status);
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    builder.append(present_operationHandle);
-    if (present_operationHandle)
-      builder.append(operationHandle);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TGetColumnsResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TGetColumnsResp typedOther = (TGetColumnsResp)other;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(typedOther.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, typedOther.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(typedOther.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, typedOther.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetColumnsResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetOperationHandle()) {
-      if (!first) sb.append(", ");
-      sb.append("operationHandle:");
-      if (this.operationHandle == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.operationHandle);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetColumnsRespStandardSchemeFactory implements SchemeFactory {
-    public TGetColumnsRespStandardScheme getScheme() {
-      return new TGetColumnsRespStandardScheme();
-    }
-  }
-
-  private static class TGetColumnsRespStandardScheme extends StandardScheme<TGetColumnsResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetColumnsResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetColumnsResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.operationHandle != null) {
-        if (struct.isSetOperationHandle()) {
-          oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-          struct.operationHandle.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetColumnsRespTupleSchemeFactory implements SchemeFactory {
-    public TGetColumnsRespTupleScheme getScheme() {
-      return new TGetColumnsRespTupleScheme();
-    }
-  }
-
-  private static class TGetColumnsRespTupleScheme extends TupleScheme<TGetColumnsResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetColumnsResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetOperationHandle()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetOperationHandle()) {
-        struct.operationHandle.write(oprot);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetColumnsResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.operationHandle = new TOperationHandle();
-        struct.operationHandle.read(iprot);
-        struct.setOperationHandleIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetDelegationTokenReq.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetDelegationTokenReq.java
deleted file mode 100644
index 6c6bb00e43e43..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetDelegationTokenReq.java
+++ /dev/null
@@ -1,592 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TGetDelegationTokenReq implements org.apache.thrift.TBase<TGetDelegationTokenReq, TGetDelegationTokenReq._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetDelegationTokenReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField OWNER_FIELD_DESC = new org.apache.thrift.protocol.TField("owner", org.apache.thrift.protocol.TType.STRING, (short)2);
-  private static final org.apache.thrift.protocol.TField RENEWER_FIELD_DESC = new org.apache.thrift.protocol.TField("renewer", org.apache.thrift.protocol.TType.STRING, (short)3);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetDelegationTokenReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetDelegationTokenReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-  private String owner; // required
-  private String renewer; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle"),
-    OWNER((short)2, "owner"),
-    RENEWER((short)3, "renewer");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        case 2: // OWNER
-          return OWNER;
-        case 3: // RENEWER
-          return RENEWER;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    tmpMap.put(_Fields.OWNER, new org.apache.thrift.meta_data.FieldMetaData("owner", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    tmpMap.put(_Fields.RENEWER, new org.apache.thrift.meta_data.FieldMetaData("renewer", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetDelegationTokenReq.class, metaDataMap);
-  }
-
-  public TGetDelegationTokenReq() {
-  }
-
-  public TGetDelegationTokenReq(
-    TSessionHandle sessionHandle,
-    String owner,
-    String renewer)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-    this.owner = owner;
-    this.renewer = renewer;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetDelegationTokenReq(TGetDelegationTokenReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-    if (other.isSetOwner()) {
-      this.owner = other.owner;
-    }
-    if (other.isSetRenewer()) {
-      this.renewer = other.renewer;
-    }
-  }
-
-  public TGetDelegationTokenReq deepCopy() {
-    return new TGetDelegationTokenReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-    this.owner = null;
-    this.renewer = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public String getOwner() {
-    return this.owner;
-  }
-
-  public void setOwner(String owner) {
-    this.owner = owner;
-  }
-
-  public void unsetOwner() {
-    this.owner = null;
-  }
-
-  /** Returns true if field owner is set (has been assigned a value) and false otherwise */
-  public boolean isSetOwner() {
-    return this.owner != null;
-  }
-
-  public void setOwnerIsSet(boolean value) {
-    if (!value) {
-      this.owner = null;
-    }
-  }
-
-  public String getRenewer() {
-    return this.renewer;
-  }
-
-  public void setRenewer(String renewer) {
-    this.renewer = renewer;
-  }
-
-  public void unsetRenewer() {
-    this.renewer = null;
-  }
-
-  /** Returns true if field renewer is set (has been assigned a value) and false otherwise */
-  public boolean isSetRenewer() {
-    return this.renewer != null;
-  }
-
-  public void setRenewerIsSet(boolean value) {
-    if (!value) {
-      this.renewer = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    case OWNER:
-      if (value == null) {
-        unsetOwner();
-      } else {
-        setOwner((String)value);
-      }
-      break;
-
-    case RENEWER:
-      if (value == null) {
-        unsetRenewer();
-      } else {
-        setRenewer((String)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    case OWNER:
-      return getOwner();
-
-    case RENEWER:
-      return getRenewer();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    case OWNER:
-      return isSetOwner();
-    case RENEWER:
-      return isSetRenewer();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetDelegationTokenReq)
-      return this.equals((TGetDelegationTokenReq)that);
-    return false;
-  }
-
-  public boolean equals(TGetDelegationTokenReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    boolean this_present_owner = true && this.isSetOwner();
-    boolean that_present_owner = true && that.isSetOwner();
-    if (this_present_owner || that_present_owner) {
-      if (!(this_present_owner && that_present_owner))
-        return false;
-      if (!this.owner.equals(that.owner))
-        return false;
-    }
-
-    boolean this_present_renewer = true && this.isSetRenewer();
-    boolean that_present_renewer = true && that.isSetRenewer();
-    if (this_present_renewer || that_present_renewer) {
-      if (!(this_present_renewer && that_present_renewer))
-        return false;
-      if (!this.renewer.equals(that.renewer))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    builder.append(present_sessionHandle);
-    if (present_sessionHandle)
-      builder.append(sessionHandle);
-
-    boolean present_owner = true && (isSetOwner());
-    builder.append(present_owner);
-    if (present_owner)
-      builder.append(owner);
-
-    boolean present_renewer = true && (isSetRenewer());
-    builder.append(present_renewer);
-    if (present_renewer)
-      builder.append(renewer);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TGetDelegationTokenReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TGetDelegationTokenReq typedOther = (TGetDelegationTokenReq)other;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(typedOther.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, typedOther.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOwner()).compareTo(typedOther.isSetOwner());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOwner()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.owner, typedOther.owner);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetRenewer()).compareTo(typedOther.isSetRenewer());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetRenewer()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.renewer, typedOther.renewer);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetDelegationTokenReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("owner:");
-    if (this.owner == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.owner);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("renewer:");
-    if (this.renewer == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.renewer);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    if (!isSetOwner()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'owner' is unset! Struct:" + toString());
-    }
-
-    if (!isSetRenewer()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'renewer' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetDelegationTokenReqStandardSchemeFactory implements SchemeFactory {
-    public TGetDelegationTokenReqStandardScheme getScheme() {
-      return new TGetDelegationTokenReqStandardScheme();
-    }
-  }
-
-  private static class TGetDelegationTokenReqStandardScheme extends StandardScheme<TGetDelegationTokenReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetDelegationTokenReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // OWNER
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.owner = iprot.readString();
-              struct.setOwnerIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // RENEWER
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.renewer = iprot.readString();
-              struct.setRenewerIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetDelegationTokenReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.owner != null) {
-        oprot.writeFieldBegin(OWNER_FIELD_DESC);
-        oprot.writeString(struct.owner);
-        oprot.writeFieldEnd();
-      }
-      if (struct.renewer != null) {
-        oprot.writeFieldBegin(RENEWER_FIELD_DESC);
-        oprot.writeString(struct.renewer);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetDelegationTokenReqTupleSchemeFactory implements SchemeFactory {
-    public TGetDelegationTokenReqTupleScheme getScheme() {
-      return new TGetDelegationTokenReqTupleScheme();
-    }
-  }
-
-  private static class TGetDelegationTokenReqTupleScheme extends TupleScheme<TGetDelegationTokenReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetDelegationTokenReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-      oprot.writeString(struct.owner);
-      oprot.writeString(struct.renewer);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetDelegationTokenReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-      struct.owner = iprot.readString();
-      struct.setOwnerIsSet(true);
-      struct.renewer = iprot.readString();
-      struct.setRenewerIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetDelegationTokenResp.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetDelegationTokenResp.java
deleted file mode 100644
index d14c5e029a35d..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetDelegationTokenResp.java
+++ /dev/null
@@ -1,500 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TGetDelegationTokenResp implements org.apache.thrift.TBase<TGetDelegationTokenResp, TGetDelegationTokenResp._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetDelegationTokenResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField DELEGATION_TOKEN_FIELD_DESC = new org.apache.thrift.protocol.TField("delegationToken", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetDelegationTokenRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetDelegationTokenRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private String delegationToken; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    DELEGATION_TOKEN((short)2, "delegationToken");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // DELEGATION_TOKEN
-          return DELEGATION_TOKEN;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private _Fields optionals[] = {_Fields.DELEGATION_TOKEN};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.DELEGATION_TOKEN, new org.apache.thrift.meta_data.FieldMetaData("delegationToken", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetDelegationTokenResp.class, metaDataMap);
-  }
-
-  public TGetDelegationTokenResp() {
-  }
-
-  public TGetDelegationTokenResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetDelegationTokenResp(TGetDelegationTokenResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetDelegationToken()) {
-      this.delegationToken = other.delegationToken;
-    }
-  }
-
-  public TGetDelegationTokenResp deepCopy() {
-    return new TGetDelegationTokenResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.delegationToken = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public String getDelegationToken() {
-    return this.delegationToken;
-  }
-
-  public void setDelegationToken(String delegationToken) {
-    this.delegationToken = delegationToken;
-  }
-
-  public void unsetDelegationToken() {
-    this.delegationToken = null;
-  }
-
-  /** Returns true if field delegationToken is set (has been assigned a value) and false otherwise */
-  public boolean isSetDelegationToken() {
-    return this.delegationToken != null;
-  }
-
-  public void setDelegationTokenIsSet(boolean value) {
-    if (!value) {
-      this.delegationToken = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case DELEGATION_TOKEN:
-      if (value == null) {
-        unsetDelegationToken();
-      } else {
-        setDelegationToken((String)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case DELEGATION_TOKEN:
-      return getDelegationToken();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case DELEGATION_TOKEN:
-      return isSetDelegationToken();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetDelegationTokenResp)
-      return this.equals((TGetDelegationTokenResp)that);
-    return false;
-  }
-
-  public boolean equals(TGetDelegationTokenResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_delegationToken = true && this.isSetDelegationToken();
-    boolean that_present_delegationToken = true && that.isSetDelegationToken();
-    if (this_present_delegationToken || that_present_delegationToken) {
-      if (!(this_present_delegationToken && that_present_delegationToken))
-        return false;
-      if (!this.delegationToken.equals(that.delegationToken))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_status = true && (isSetStatus());
-    builder.append(present_status);
-    if (present_status)
-      builder.append(status);
-
-    boolean present_delegationToken = true && (isSetDelegationToken());
-    builder.append(present_delegationToken);
-    if (present_delegationToken)
-      builder.append(delegationToken);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TGetDelegationTokenResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TGetDelegationTokenResp typedOther = (TGetDelegationTokenResp)other;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(typedOther.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, typedOther.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetDelegationToken()).compareTo(typedOther.isSetDelegationToken());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetDelegationToken()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.delegationToken, typedOther.delegationToken);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetDelegationTokenResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetDelegationToken()) {
-      if (!first) sb.append(", ");
-      sb.append("delegationToken:");
-      if (this.delegationToken == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.delegationToken);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetDelegationTokenRespStandardSchemeFactory implements SchemeFactory {
-    public TGetDelegationTokenRespStandardScheme getScheme() {
-      return new TGetDelegationTokenRespStandardScheme();
-    }
-  }
-
-  private static class TGetDelegationTokenRespStandardScheme extends StandardScheme<TGetDelegationTokenResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetDelegationTokenResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // DELEGATION_TOKEN
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.delegationToken = iprot.readString();
-              struct.setDelegationTokenIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetDelegationTokenResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.delegationToken != null) {
-        if (struct.isSetDelegationToken()) {
-          oprot.writeFieldBegin(DELEGATION_TOKEN_FIELD_DESC);
-          oprot.writeString(struct.delegationToken);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetDelegationTokenRespTupleSchemeFactory implements SchemeFactory {
-    public TGetDelegationTokenRespTupleScheme getScheme() {
-      return new TGetDelegationTokenRespTupleScheme();
-    }
-  }
-
-  private static class TGetDelegationTokenRespTupleScheme extends TupleScheme<TGetDelegationTokenResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetDelegationTokenResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetDelegationToken()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetDelegationToken()) {
-        oprot.writeString(struct.delegationToken);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetDelegationTokenResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.delegationToken = iprot.readString();
-        struct.setDelegationTokenIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetFunctionsReq.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetFunctionsReq.java
deleted file mode 100644
index ff45ee0386cb9..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetFunctionsReq.java
+++ /dev/null
@@ -1,707 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TGetFunctionsReq implements org.apache.thrift.TBase<TGetFunctionsReq, TGetFunctionsReq._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetFunctionsReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField CATALOG_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("catalogName", org.apache.thrift.protocol.TType.STRING, (short)2);
-  private static final org.apache.thrift.protocol.TField SCHEMA_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("schemaName", org.apache.thrift.protocol.TType.STRING, (short)3);
-  private static final org.apache.thrift.protocol.TField FUNCTION_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("functionName", org.apache.thrift.protocol.TType.STRING, (short)4);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetFunctionsReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetFunctionsReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-  private String catalogName; // optional
-  private String schemaName; // optional
-  private String functionName; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle"),
-    CATALOG_NAME((short)2, "catalogName"),
-    SCHEMA_NAME((short)3, "schemaName"),
-    FUNCTION_NAME((short)4, "functionName");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        case 2: // CATALOG_NAME
-          return CATALOG_NAME;
-        case 3: // SCHEMA_NAME
-          return SCHEMA_NAME;
-        case 4: // FUNCTION_NAME
-          return FUNCTION_NAME;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private _Fields optionals[] = {_Fields.CATALOG_NAME,_Fields.SCHEMA_NAME};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    tmpMap.put(_Fields.CATALOG_NAME, new org.apache.thrift.meta_data.FieldMetaData("catalogName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TIdentifier")));
-    tmpMap.put(_Fields.SCHEMA_NAME, new org.apache.thrift.meta_data.FieldMetaData("schemaName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TPatternOrIdentifier")));
-    tmpMap.put(_Fields.FUNCTION_NAME, new org.apache.thrift.meta_data.FieldMetaData("functionName", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TPatternOrIdentifier")));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetFunctionsReq.class, metaDataMap);
-  }
-
-  public TGetFunctionsReq() {
-  }
-
-  public TGetFunctionsReq(
-    TSessionHandle sessionHandle,
-    String functionName)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-    this.functionName = functionName;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetFunctionsReq(TGetFunctionsReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-    if (other.isSetCatalogName()) {
-      this.catalogName = other.catalogName;
-    }
-    if (other.isSetSchemaName()) {
-      this.schemaName = other.schemaName;
-    }
-    if (other.isSetFunctionName()) {
-      this.functionName = other.functionName;
-    }
-  }
-
-  public TGetFunctionsReq deepCopy() {
-    return new TGetFunctionsReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-    this.catalogName = null;
-    this.schemaName = null;
-    this.functionName = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public String getCatalogName() {
-    return this.catalogName;
-  }
-
-  public void setCatalogName(String catalogName) {
-    this.catalogName = catalogName;
-  }
-
-  public void unsetCatalogName() {
-    this.catalogName = null;
-  }
-
-  /** Returns true if field catalogName is set (has been assigned a value) and false otherwise */
-  public boolean isSetCatalogName() {
-    return this.catalogName != null;
-  }
-
-  public void setCatalogNameIsSet(boolean value) {
-    if (!value) {
-      this.catalogName = null;
-    }
-  }
-
-  public String getSchemaName() {
-    return this.schemaName;
-  }
-
-  public void setSchemaName(String schemaName) {
-    this.schemaName = schemaName;
-  }
-
-  public void unsetSchemaName() {
-    this.schemaName = null;
-  }
-
-  /** Returns true if field schemaName is set (has been assigned a value) and false otherwise */
-  public boolean isSetSchemaName() {
-    return this.schemaName != null;
-  }
-
-  public void setSchemaNameIsSet(boolean value) {
-    if (!value) {
-      this.schemaName = null;
-    }
-  }
-
-  public String getFunctionName() {
-    return this.functionName;
-  }
-
-  public void setFunctionName(String functionName) {
-    this.functionName = functionName;
-  }
-
-  public void unsetFunctionName() {
-    this.functionName = null;
-  }
-
-  /** Returns true if field functionName is set (has been assigned a value) and false otherwise */
-  public boolean isSetFunctionName() {
-    return this.functionName != null;
-  }
-
-  public void setFunctionNameIsSet(boolean value) {
-    if (!value) {
-      this.functionName = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    case CATALOG_NAME:
-      if (value == null) {
-        unsetCatalogName();
-      } else {
-        setCatalogName((String)value);
-      }
-      break;
-
-    case SCHEMA_NAME:
-      if (value == null) {
-        unsetSchemaName();
-      } else {
-        setSchemaName((String)value);
-      }
-      break;
-
-    case FUNCTION_NAME:
-      if (value == null) {
-        unsetFunctionName();
-      } else {
-        setFunctionName((String)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    case CATALOG_NAME:
-      return getCatalogName();
-
-    case SCHEMA_NAME:
-      return getSchemaName();
-
-    case FUNCTION_NAME:
-      return getFunctionName();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    case CATALOG_NAME:
-      return isSetCatalogName();
-    case SCHEMA_NAME:
-      return isSetSchemaName();
-    case FUNCTION_NAME:
-      return isSetFunctionName();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetFunctionsReq)
-      return this.equals((TGetFunctionsReq)that);
-    return false;
-  }
-
-  public boolean equals(TGetFunctionsReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    boolean this_present_catalogName = true && this.isSetCatalogName();
-    boolean that_present_catalogName = true && that.isSetCatalogName();
-    if (this_present_catalogName || that_present_catalogName) {
-      if (!(this_present_catalogName && that_present_catalogName))
-        return false;
-      if (!this.catalogName.equals(that.catalogName))
-        return false;
-    }
-
-    boolean this_present_schemaName = true && this.isSetSchemaName();
-    boolean that_present_schemaName = true && that.isSetSchemaName();
-    if (this_present_schemaName || that_present_schemaName) {
-      if (!(this_present_schemaName && that_present_schemaName))
-        return false;
-      if (!this.schemaName.equals(that.schemaName))
-        return false;
-    }
-
-    boolean this_present_functionName = true && this.isSetFunctionName();
-    boolean that_present_functionName = true && that.isSetFunctionName();
-    if (this_present_functionName || that_present_functionName) {
-      if (!(this_present_functionName && that_present_functionName))
-        return false;
-      if (!this.functionName.equals(that.functionName))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    builder.append(present_sessionHandle);
-    if (present_sessionHandle)
-      builder.append(sessionHandle);
-
-    boolean present_catalogName = true && (isSetCatalogName());
-    builder.append(present_catalogName);
-    if (present_catalogName)
-      builder.append(catalogName);
-
-    boolean present_schemaName = true && (isSetSchemaName());
-    builder.append(present_schemaName);
-    if (present_schemaName)
-      builder.append(schemaName);
-
-    boolean present_functionName = true && (isSetFunctionName());
-    builder.append(present_functionName);
-    if (present_functionName)
-      builder.append(functionName);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TGetFunctionsReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TGetFunctionsReq typedOther = (TGetFunctionsReq)other;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(typedOther.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, typedOther.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetCatalogName()).compareTo(typedOther.isSetCatalogName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetCatalogName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.catalogName, typedOther.catalogName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetSchemaName()).compareTo(typedOther.isSetSchemaName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSchemaName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.schemaName, typedOther.schemaName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetFunctionName()).compareTo(typedOther.isSetFunctionName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetFunctionName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.functionName, typedOther.functionName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetFunctionsReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    if (isSetCatalogName()) {
-      if (!first) sb.append(", ");
-      sb.append("catalogName:");
-      if (this.catalogName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.catalogName);
-      }
-      first = false;
-    }
-    if (isSetSchemaName()) {
-      if (!first) sb.append(", ");
-      sb.append("schemaName:");
-      if (this.schemaName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.schemaName);
-      }
-      first = false;
-    }
-    if (!first) sb.append(", ");
-    sb.append("functionName:");
-    if (this.functionName == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.functionName);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    if (!isSetFunctionName()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'functionName' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetFunctionsReqStandardSchemeFactory implements SchemeFactory {
-    public TGetFunctionsReqStandardScheme getScheme() {
-      return new TGetFunctionsReqStandardScheme();
-    }
-  }
-
-  private static class TGetFunctionsReqStandardScheme extends StandardScheme<TGetFunctionsReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetFunctionsReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // CATALOG_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.catalogName = iprot.readString();
-              struct.setCatalogNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // SCHEMA_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.schemaName = iprot.readString();
-              struct.setSchemaNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // FUNCTION_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.functionName = iprot.readString();
-              struct.setFunctionNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetFunctionsReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.catalogName != null) {
-        if (struct.isSetCatalogName()) {
-          oprot.writeFieldBegin(CATALOG_NAME_FIELD_DESC);
-          oprot.writeString(struct.catalogName);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.schemaName != null) {
-        if (struct.isSetSchemaName()) {
-          oprot.writeFieldBegin(SCHEMA_NAME_FIELD_DESC);
-          oprot.writeString(struct.schemaName);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.functionName != null) {
-        oprot.writeFieldBegin(FUNCTION_NAME_FIELD_DESC);
-        oprot.writeString(struct.functionName);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetFunctionsReqTupleSchemeFactory implements SchemeFactory {
-    public TGetFunctionsReqTupleScheme getScheme() {
-      return new TGetFunctionsReqTupleScheme();
-    }
-  }
-
-  private static class TGetFunctionsReqTupleScheme extends TupleScheme<TGetFunctionsReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetFunctionsReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-      oprot.writeString(struct.functionName);
-      BitSet optionals = new BitSet();
-      if (struct.isSetCatalogName()) {
-        optionals.set(0);
-      }
-      if (struct.isSetSchemaName()) {
-        optionals.set(1);
-      }
-      oprot.writeBitSet(optionals, 2);
-      if (struct.isSetCatalogName()) {
-        oprot.writeString(struct.catalogName);
-      }
-      if (struct.isSetSchemaName()) {
-        oprot.writeString(struct.schemaName);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetFunctionsReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-      struct.functionName = iprot.readString();
-      struct.setFunctionNameIsSet(true);
-      BitSet incoming = iprot.readBitSet(2);
-      if (incoming.get(0)) {
-        struct.catalogName = iprot.readString();
-        struct.setCatalogNameIsSet(true);
-      }
-      if (incoming.get(1)) {
-        struct.schemaName = iprot.readString();
-        struct.setSchemaNameIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetFunctionsResp.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetFunctionsResp.java
deleted file mode 100644
index 3adafdacb54ef..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetFunctionsResp.java
+++ /dev/null
@@ -1,505 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TGetFunctionsResp implements org.apache.thrift.TBase<TGetFunctionsResp, TGetFunctionsResp._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetFunctionsResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetFunctionsRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetFunctionsRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TOperationHandle operationHandle; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    OPERATION_HANDLE((short)2, "operationHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private _Fields optionals[] = {_Fields.OPERATION_HANDLE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetFunctionsResp.class, metaDataMap);
-  }
-
-  public TGetFunctionsResp() {
-  }
-
-  public TGetFunctionsResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetFunctionsResp(TGetFunctionsResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-  }
-
-  public TGetFunctionsResp deepCopy() {
-    return new TGetFunctionsResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.operationHandle = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetFunctionsResp)
-      return this.equals((TGetFunctionsResp)that);
-    return false;
-  }
-
-  public boolean equals(TGetFunctionsResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_status = true && (isSetStatus());
-    builder.append(present_status);
-    if (present_status)
-      builder.append(status);
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    builder.append(present_operationHandle);
-    if (present_operationHandle)
-      builder.append(operationHandle);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TGetFunctionsResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TGetFunctionsResp typedOther = (TGetFunctionsResp)other;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(typedOther.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, typedOther.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(typedOther.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, typedOther.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetFunctionsResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetOperationHandle()) {
-      if (!first) sb.append(", ");
-      sb.append("operationHandle:");
-      if (this.operationHandle == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.operationHandle);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetFunctionsRespStandardSchemeFactory implements SchemeFactory {
-    public TGetFunctionsRespStandardScheme getScheme() {
-      return new TGetFunctionsRespStandardScheme();
-    }
-  }
-
-  private static class TGetFunctionsRespStandardScheme extends StandardScheme<TGetFunctionsResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetFunctionsResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetFunctionsResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.operationHandle != null) {
-        if (struct.isSetOperationHandle()) {
-          oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-          struct.operationHandle.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetFunctionsRespTupleSchemeFactory implements SchemeFactory {
-    public TGetFunctionsRespTupleScheme getScheme() {
-      return new TGetFunctionsRespTupleScheme();
-    }
-  }
-
-  private static class TGetFunctionsRespTupleScheme extends TupleScheme<TGetFunctionsResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetFunctionsResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetOperationHandle()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetOperationHandle()) {
-        struct.operationHandle.write(oprot);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetFunctionsResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.operationHandle = new TOperationHandle();
-        struct.operationHandle.read(iprot);
-        struct.setOperationHandleIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetInfoReq.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetInfoReq.java
deleted file mode 100644
index 0139bf04ec7db..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetInfoReq.java
+++ /dev/null
@@ -1,503 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TGetInfoReq implements org.apache.thrift.TBase<TGetInfoReq, TGetInfoReq._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetInfoReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField INFO_TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("infoType", org.apache.thrift.protocol.TType.I32, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetInfoReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetInfoReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-  private TGetInfoType infoType; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle"),
-    /**
-     * 
-     * @see TGetInfoType
-     */
-    INFO_TYPE((short)2, "infoType");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        case 2: // INFO_TYPE
-          return INFO_TYPE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    tmpMap.put(_Fields.INFO_TYPE, new org.apache.thrift.meta_data.FieldMetaData("infoType", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TGetInfoType.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetInfoReq.class, metaDataMap);
-  }
-
-  public TGetInfoReq() {
-  }
-
-  public TGetInfoReq(
-    TSessionHandle sessionHandle,
-    TGetInfoType infoType)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-    this.infoType = infoType;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetInfoReq(TGetInfoReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-    if (other.isSetInfoType()) {
-      this.infoType = other.infoType;
-    }
-  }
-
-  public TGetInfoReq deepCopy() {
-    return new TGetInfoReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-    this.infoType = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  /**
-   * 
-   * @see TGetInfoType
-   */
-  public TGetInfoType getInfoType() {
-    return this.infoType;
-  }
-
-  /**
-   * 
-   * @see TGetInfoType
-   */
-  public void setInfoType(TGetInfoType infoType) {
-    this.infoType = infoType;
-  }
-
-  public void unsetInfoType() {
-    this.infoType = null;
-  }
-
-  /** Returns true if field infoType is set (has been assigned a value) and false otherwise */
-  public boolean isSetInfoType() {
-    return this.infoType != null;
-  }
-
-  public void setInfoTypeIsSet(boolean value) {
-    if (!value) {
-      this.infoType = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    case INFO_TYPE:
-      if (value == null) {
-        unsetInfoType();
-      } else {
-        setInfoType((TGetInfoType)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    case INFO_TYPE:
-      return getInfoType();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    case INFO_TYPE:
-      return isSetInfoType();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetInfoReq)
-      return this.equals((TGetInfoReq)that);
-    return false;
-  }
-
-  public boolean equals(TGetInfoReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    boolean this_present_infoType = true && this.isSetInfoType();
-    boolean that_present_infoType = true && that.isSetInfoType();
-    if (this_present_infoType || that_present_infoType) {
-      if (!(this_present_infoType && that_present_infoType))
-        return false;
-      if (!this.infoType.equals(that.infoType))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    builder.append(present_sessionHandle);
-    if (present_sessionHandle)
-      builder.append(sessionHandle);
-
-    boolean present_infoType = true && (isSetInfoType());
-    builder.append(present_infoType);
-    if (present_infoType)
-      builder.append(infoType.getValue());
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TGetInfoReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TGetInfoReq typedOther = (TGetInfoReq)other;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(typedOther.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, typedOther.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetInfoType()).compareTo(typedOther.isSetInfoType());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetInfoType()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.infoType, typedOther.infoType);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetInfoReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("infoType:");
-    if (this.infoType == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.infoType);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    if (!isSetInfoType()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'infoType' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetInfoReqStandardSchemeFactory implements SchemeFactory {
-    public TGetInfoReqStandardScheme getScheme() {
-      return new TGetInfoReqStandardScheme();
-    }
-  }
-
-  private static class TGetInfoReqStandardScheme extends StandardScheme<TGetInfoReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetInfoReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // INFO_TYPE
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.infoType = TGetInfoType.findByValue(iprot.readI32());
-              struct.setInfoTypeIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetInfoReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.infoType != null) {
-        oprot.writeFieldBegin(INFO_TYPE_FIELD_DESC);
-        oprot.writeI32(struct.infoType.getValue());
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetInfoReqTupleSchemeFactory implements SchemeFactory {
-    public TGetInfoReqTupleScheme getScheme() {
-      return new TGetInfoReqTupleScheme();
-    }
-  }
-
-  private static class TGetInfoReqTupleScheme extends TupleScheme<TGetInfoReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetInfoReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-      oprot.writeI32(struct.infoType.getValue());
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetInfoReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-      struct.infoType = TGetInfoType.findByValue(iprot.readI32());
-      struct.setInfoTypeIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetInfoResp.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetInfoResp.java
deleted file mode 100644
index 2faaa9211b3ba..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetInfoResp.java
+++ /dev/null
@@ -1,493 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TGetInfoResp implements org.apache.thrift.TBase<TGetInfoResp, TGetInfoResp._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetInfoResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField INFO_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("infoValue", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetInfoRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetInfoRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TGetInfoValue infoValue; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    INFO_VALUE((short)2, "infoValue");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // INFO_VALUE
-          return INFO_VALUE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.INFO_VALUE, new org.apache.thrift.meta_data.FieldMetaData("infoValue", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetInfoValue.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetInfoResp.class, metaDataMap);
-  }
-
-  public TGetInfoResp() {
-  }
-
-  public TGetInfoResp(
-    TStatus status,
-    TGetInfoValue infoValue)
-  {
-    this();
-    this.status = status;
-    this.infoValue = infoValue;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetInfoResp(TGetInfoResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetInfoValue()) {
-      this.infoValue = new TGetInfoValue(other.infoValue);
-    }
-  }
-
-  public TGetInfoResp deepCopy() {
-    return new TGetInfoResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.infoValue = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public TGetInfoValue getInfoValue() {
-    return this.infoValue;
-  }
-
-  public void setInfoValue(TGetInfoValue infoValue) {
-    this.infoValue = infoValue;
-  }
-
-  public void unsetInfoValue() {
-    this.infoValue = null;
-  }
-
-  /** Returns true if field infoValue is set (has been assigned a value) and false otherwise */
-  public boolean isSetInfoValue() {
-    return this.infoValue != null;
-  }
-
-  public void setInfoValueIsSet(boolean value) {
-    if (!value) {
-      this.infoValue = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case INFO_VALUE:
-      if (value == null) {
-        unsetInfoValue();
-      } else {
-        setInfoValue((TGetInfoValue)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case INFO_VALUE:
-      return getInfoValue();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case INFO_VALUE:
-      return isSetInfoValue();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetInfoResp)
-      return this.equals((TGetInfoResp)that);
-    return false;
-  }
-
-  public boolean equals(TGetInfoResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_infoValue = true && this.isSetInfoValue();
-    boolean that_present_infoValue = true && that.isSetInfoValue();
-    if (this_present_infoValue || that_present_infoValue) {
-      if (!(this_present_infoValue && that_present_infoValue))
-        return false;
-      if (!this.infoValue.equals(that.infoValue))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_status = true && (isSetStatus());
-    builder.append(present_status);
-    if (present_status)
-      builder.append(status);
-
-    boolean present_infoValue = true && (isSetInfoValue());
-    builder.append(present_infoValue);
-    if (present_infoValue)
-      builder.append(infoValue);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TGetInfoResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TGetInfoResp typedOther = (TGetInfoResp)other;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(typedOther.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, typedOther.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetInfoValue()).compareTo(typedOther.isSetInfoValue());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetInfoValue()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.infoValue, typedOther.infoValue);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetInfoResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("infoValue:");
-    if (this.infoValue == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.infoValue);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    if (!isSetInfoValue()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'infoValue' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetInfoRespStandardSchemeFactory implements SchemeFactory {
-    public TGetInfoRespStandardScheme getScheme() {
-      return new TGetInfoRespStandardScheme();
-    }
-  }
-
-  private static class TGetInfoRespStandardScheme extends StandardScheme<TGetInfoResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetInfoResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // INFO_VALUE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.infoValue = new TGetInfoValue();
-              struct.infoValue.read(iprot);
-              struct.setInfoValueIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetInfoResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.infoValue != null) {
-        oprot.writeFieldBegin(INFO_VALUE_FIELD_DESC);
-        struct.infoValue.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetInfoRespTupleSchemeFactory implements SchemeFactory {
-    public TGetInfoRespTupleScheme getScheme() {
-      return new TGetInfoRespTupleScheme();
-    }
-  }
-
-  private static class TGetInfoRespTupleScheme extends TupleScheme<TGetInfoResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetInfoResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      struct.infoValue.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetInfoResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      struct.infoValue = new TGetInfoValue();
-      struct.infoValue.read(iprot);
-      struct.setInfoValueIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetInfoType.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetInfoType.java
deleted file mode 100644
index d9dd62414f001..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetInfoType.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.thrift.TEnum;
-
-public enum TGetInfoType implements org.apache.thrift.TEnum {
-  CLI_MAX_DRIVER_CONNECTIONS(0),
-  CLI_MAX_CONCURRENT_ACTIVITIES(1),
-  CLI_DATA_SOURCE_NAME(2),
-  CLI_FETCH_DIRECTION(8),
-  CLI_SERVER_NAME(13),
-  CLI_SEARCH_PATTERN_ESCAPE(14),
-  CLI_DBMS_NAME(17),
-  CLI_DBMS_VER(18),
-  CLI_ACCESSIBLE_TABLES(19),
-  CLI_ACCESSIBLE_PROCEDURES(20),
-  CLI_CURSOR_COMMIT_BEHAVIOR(23),
-  CLI_DATA_SOURCE_READ_ONLY(25),
-  CLI_DEFAULT_TXN_ISOLATION(26),
-  CLI_IDENTIFIER_CASE(28),
-  CLI_IDENTIFIER_QUOTE_CHAR(29),
-  CLI_MAX_COLUMN_NAME_LEN(30),
-  CLI_MAX_CURSOR_NAME_LEN(31),
-  CLI_MAX_SCHEMA_NAME_LEN(32),
-  CLI_MAX_CATALOG_NAME_LEN(34),
-  CLI_MAX_TABLE_NAME_LEN(35),
-  CLI_SCROLL_CONCURRENCY(43),
-  CLI_TXN_CAPABLE(46),
-  CLI_USER_NAME(47),
-  CLI_TXN_ISOLATION_OPTION(72),
-  CLI_INTEGRITY(73),
-  CLI_GETDATA_EXTENSIONS(81),
-  CLI_NULL_COLLATION(85),
-  CLI_ALTER_TABLE(86),
-  CLI_ORDER_BY_COLUMNS_IN_SELECT(90),
-  CLI_SPECIAL_CHARACTERS(94),
-  CLI_MAX_COLUMNS_IN_GROUP_BY(97),
-  CLI_MAX_COLUMNS_IN_INDEX(98),
-  CLI_MAX_COLUMNS_IN_ORDER_BY(99),
-  CLI_MAX_COLUMNS_IN_SELECT(100),
-  CLI_MAX_COLUMNS_IN_TABLE(101),
-  CLI_MAX_INDEX_SIZE(102),
-  CLI_MAX_ROW_SIZE(104),
-  CLI_MAX_STATEMENT_LEN(105),
-  CLI_MAX_TABLES_IN_SELECT(106),
-  CLI_MAX_USER_NAME_LEN(107),
-  CLI_OJ_CAPABILITIES(115),
-  CLI_XOPEN_CLI_YEAR(10000),
-  CLI_CURSOR_SENSITIVITY(10001),
-  CLI_DESCRIBE_PARAMETER(10002),
-  CLI_CATALOG_NAME(10003),
-  CLI_COLLATION_SEQ(10004),
-  CLI_MAX_IDENTIFIER_LEN(10005);
-
-  private final int value;
-
-  private TGetInfoType(int value) {
-    this.value = value;
-  }
-
-  /**
-   * Get the integer value of this enum value, as defined in the Thrift IDL.
-   */
-  public int getValue() {
-    return value;
-  }
-
-  /**
-   * Find a the enum type by its integer value, as defined in the Thrift IDL.
-   * @return null if the value is not found.
-   */
-  public static TGetInfoType findByValue(int value) { 
-    switch (value) {
-      case 0:
-        return CLI_MAX_DRIVER_CONNECTIONS;
-      case 1:
-        return CLI_MAX_CONCURRENT_ACTIVITIES;
-      case 2:
-        return CLI_DATA_SOURCE_NAME;
-      case 8:
-        return CLI_FETCH_DIRECTION;
-      case 13:
-        return CLI_SERVER_NAME;
-      case 14:
-        return CLI_SEARCH_PATTERN_ESCAPE;
-      case 17:
-        return CLI_DBMS_NAME;
-      case 18:
-        return CLI_DBMS_VER;
-      case 19:
-        return CLI_ACCESSIBLE_TABLES;
-      case 20:
-        return CLI_ACCESSIBLE_PROCEDURES;
-      case 23:
-        return CLI_CURSOR_COMMIT_BEHAVIOR;
-      case 25:
-        return CLI_DATA_SOURCE_READ_ONLY;
-      case 26:
-        return CLI_DEFAULT_TXN_ISOLATION;
-      case 28:
-        return CLI_IDENTIFIER_CASE;
-      case 29:
-        return CLI_IDENTIFIER_QUOTE_CHAR;
-      case 30:
-        return CLI_MAX_COLUMN_NAME_LEN;
-      case 31:
-        return CLI_MAX_CURSOR_NAME_LEN;
-      case 32:
-        return CLI_MAX_SCHEMA_NAME_LEN;
-      case 34:
-        return CLI_MAX_CATALOG_NAME_LEN;
-      case 35:
-        return CLI_MAX_TABLE_NAME_LEN;
-      case 43:
-        return CLI_SCROLL_CONCURRENCY;
-      case 46:
-        return CLI_TXN_CAPABLE;
-      case 47:
-        return CLI_USER_NAME;
-      case 72:
-        return CLI_TXN_ISOLATION_OPTION;
-      case 73:
-        return CLI_INTEGRITY;
-      case 81:
-        return CLI_GETDATA_EXTENSIONS;
-      case 85:
-        return CLI_NULL_COLLATION;
-      case 86:
-        return CLI_ALTER_TABLE;
-      case 90:
-        return CLI_ORDER_BY_COLUMNS_IN_SELECT;
-      case 94:
-        return CLI_SPECIAL_CHARACTERS;
-      case 97:
-        return CLI_MAX_COLUMNS_IN_GROUP_BY;
-      case 98:
-        return CLI_MAX_COLUMNS_IN_INDEX;
-      case 99:
-        return CLI_MAX_COLUMNS_IN_ORDER_BY;
-      case 100:
-        return CLI_MAX_COLUMNS_IN_SELECT;
-      case 101:
-        return CLI_MAX_COLUMNS_IN_TABLE;
-      case 102:
-        return CLI_MAX_INDEX_SIZE;
-      case 104:
-        return CLI_MAX_ROW_SIZE;
-      case 105:
-        return CLI_MAX_STATEMENT_LEN;
-      case 106:
-        return CLI_MAX_TABLES_IN_SELECT;
-      case 107:
-        return CLI_MAX_USER_NAME_LEN;
-      case 115:
-        return CLI_OJ_CAPABILITIES;
-      case 10000:
-        return CLI_XOPEN_CLI_YEAR;
-      case 10001:
-        return CLI_CURSOR_SENSITIVITY;
-      case 10002:
-        return CLI_DESCRIBE_PARAMETER;
-      case 10003:
-        return CLI_CATALOG_NAME;
-      case 10004:
-        return CLI_COLLATION_SEQ;
-      case 10005:
-        return CLI_MAX_IDENTIFIER_LEN;
-      default:
-        return null;
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetInfoValue.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetInfoValue.java
deleted file mode 100644
index fe2a211c46309..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetInfoValue.java
+++ /dev/null
@@ -1,593 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TGetInfoValue extends org.apache.thrift.TUnion<TGetInfoValue, TGetInfoValue._Fields> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetInfoValue");
-  private static final org.apache.thrift.protocol.TField STRING_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("stringValue", org.apache.thrift.protocol.TType.STRING, (short)1);
-  private static final org.apache.thrift.protocol.TField SMALL_INT_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("smallIntValue", org.apache.thrift.protocol.TType.I16, (short)2);
-  private static final org.apache.thrift.protocol.TField INTEGER_BITMASK_FIELD_DESC = new org.apache.thrift.protocol.TField("integerBitmask", org.apache.thrift.protocol.TType.I32, (short)3);
-  private static final org.apache.thrift.protocol.TField INTEGER_FLAG_FIELD_DESC = new org.apache.thrift.protocol.TField("integerFlag", org.apache.thrift.protocol.TType.I32, (short)4);
-  private static final org.apache.thrift.protocol.TField BINARY_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("binaryValue", org.apache.thrift.protocol.TType.I32, (short)5);
-  private static final org.apache.thrift.protocol.TField LEN_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("lenValue", org.apache.thrift.protocol.TType.I64, (short)6);
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STRING_VALUE((short)1, "stringValue"),
-    SMALL_INT_VALUE((short)2, "smallIntValue"),
-    INTEGER_BITMASK((short)3, "integerBitmask"),
-    INTEGER_FLAG((short)4, "integerFlag"),
-    BINARY_VALUE((short)5, "binaryValue"),
-    LEN_VALUE((short)6, "lenValue");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STRING_VALUE
-          return STRING_VALUE;
-        case 2: // SMALL_INT_VALUE
-          return SMALL_INT_VALUE;
-        case 3: // INTEGER_BITMASK
-          return INTEGER_BITMASK;
-        case 4: // INTEGER_FLAG
-          return INTEGER_FLAG;
-        case 5: // BINARY_VALUE
-          return BINARY_VALUE;
-        case 6: // LEN_VALUE
-          return LEN_VALUE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STRING_VALUE, new org.apache.thrift.meta_data.FieldMetaData("stringValue", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    tmpMap.put(_Fields.SMALL_INT_VALUE, new org.apache.thrift.meta_data.FieldMetaData("smallIntValue", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I16)));
-    tmpMap.put(_Fields.INTEGER_BITMASK, new org.apache.thrift.meta_data.FieldMetaData("integerBitmask", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
-    tmpMap.put(_Fields.INTEGER_FLAG, new org.apache.thrift.meta_data.FieldMetaData("integerFlag", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
-    tmpMap.put(_Fields.BINARY_VALUE, new org.apache.thrift.meta_data.FieldMetaData("binaryValue", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
-    tmpMap.put(_Fields.LEN_VALUE, new org.apache.thrift.meta_data.FieldMetaData("lenValue", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetInfoValue.class, metaDataMap);
-  }
-
-  public TGetInfoValue() {
-    super();
-  }
-
-  public TGetInfoValue(TGetInfoValue._Fields setField, Object value) {
-    super(setField, value);
-  }
-
-  public TGetInfoValue(TGetInfoValue other) {
-    super(other);
-  }
-  public TGetInfoValue deepCopy() {
-    return new TGetInfoValue(this);
-  }
-
-  public static TGetInfoValue stringValue(String value) {
-    TGetInfoValue x = new TGetInfoValue();
-    x.setStringValue(value);
-    return x;
-  }
-
-  public static TGetInfoValue smallIntValue(short value) {
-    TGetInfoValue x = new TGetInfoValue();
-    x.setSmallIntValue(value);
-    return x;
-  }
-
-  public static TGetInfoValue integerBitmask(int value) {
-    TGetInfoValue x = new TGetInfoValue();
-    x.setIntegerBitmask(value);
-    return x;
-  }
-
-  public static TGetInfoValue integerFlag(int value) {
-    TGetInfoValue x = new TGetInfoValue();
-    x.setIntegerFlag(value);
-    return x;
-  }
-
-  public static TGetInfoValue binaryValue(int value) {
-    TGetInfoValue x = new TGetInfoValue();
-    x.setBinaryValue(value);
-    return x;
-  }
-
-  public static TGetInfoValue lenValue(long value) {
-    TGetInfoValue x = new TGetInfoValue();
-    x.setLenValue(value);
-    return x;
-  }
-
-
-  @Override
-  protected void checkType(_Fields setField, Object value) throws ClassCastException {
-    switch (setField) {
-      case STRING_VALUE:
-        if (value instanceof String) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type String for field 'stringValue', but got " + value.getClass().getSimpleName());
-      case SMALL_INT_VALUE:
-        if (value instanceof Short) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type Short for field 'smallIntValue', but got " + value.getClass().getSimpleName());
-      case INTEGER_BITMASK:
-        if (value instanceof Integer) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type Integer for field 'integerBitmask', but got " + value.getClass().getSimpleName());
-      case INTEGER_FLAG:
-        if (value instanceof Integer) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type Integer for field 'integerFlag', but got " + value.getClass().getSimpleName());
-      case BINARY_VALUE:
-        if (value instanceof Integer) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type Integer for field 'binaryValue', but got " + value.getClass().getSimpleName());
-      case LEN_VALUE:
-        if (value instanceof Long) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type Long for field 'lenValue', but got " + value.getClass().getSimpleName());
-      default:
-        throw new IllegalArgumentException("Unknown field id " + setField);
-    }
-  }
-
-  @Override
-  protected Object standardSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TField field) throws org.apache.thrift.TException {
-    _Fields setField = _Fields.findByThriftId(field.id);
-    if (setField != null) {
-      switch (setField) {
-        case STRING_VALUE:
-          if (field.type == STRING_VALUE_FIELD_DESC.type) {
-            String stringValue;
-            stringValue = iprot.readString();
-            return stringValue;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case SMALL_INT_VALUE:
-          if (field.type == SMALL_INT_VALUE_FIELD_DESC.type) {
-            Short smallIntValue;
-            smallIntValue = iprot.readI16();
-            return smallIntValue;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case INTEGER_BITMASK:
-          if (field.type == INTEGER_BITMASK_FIELD_DESC.type) {
-            Integer integerBitmask;
-            integerBitmask = iprot.readI32();
-            return integerBitmask;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case INTEGER_FLAG:
-          if (field.type == INTEGER_FLAG_FIELD_DESC.type) {
-            Integer integerFlag;
-            integerFlag = iprot.readI32();
-            return integerFlag;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case BINARY_VALUE:
-          if (field.type == BINARY_VALUE_FIELD_DESC.type) {
-            Integer binaryValue;
-            binaryValue = iprot.readI32();
-            return binaryValue;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case LEN_VALUE:
-          if (field.type == LEN_VALUE_FIELD_DESC.type) {
-            Long lenValue;
-            lenValue = iprot.readI64();
-            return lenValue;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        default:
-          throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!");
-      }
-    } else {
-      return null;
-    }
-  }
-
-  @Override
-  protected void standardSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    switch (setField_) {
-      case STRING_VALUE:
-        String stringValue = (String)value_;
-        oprot.writeString(stringValue);
-        return;
-      case SMALL_INT_VALUE:
-        Short smallIntValue = (Short)value_;
-        oprot.writeI16(smallIntValue);
-        return;
-      case INTEGER_BITMASK:
-        Integer integerBitmask = (Integer)value_;
-        oprot.writeI32(integerBitmask);
-        return;
-      case INTEGER_FLAG:
-        Integer integerFlag = (Integer)value_;
-        oprot.writeI32(integerFlag);
-        return;
-      case BINARY_VALUE:
-        Integer binaryValue = (Integer)value_;
-        oprot.writeI32(binaryValue);
-        return;
-      case LEN_VALUE:
-        Long lenValue = (Long)value_;
-        oprot.writeI64(lenValue);
-        return;
-      default:
-        throw new IllegalStateException("Cannot write union with unknown field " + setField_);
-    }
-  }
-
-  @Override
-  protected Object tupleSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, short fieldID) throws org.apache.thrift.TException {
-    _Fields setField = _Fields.findByThriftId(fieldID);
-    if (setField != null) {
-      switch (setField) {
-        case STRING_VALUE:
-          String stringValue;
-          stringValue = iprot.readString();
-          return stringValue;
-        case SMALL_INT_VALUE:
-          Short smallIntValue;
-          smallIntValue = iprot.readI16();
-          return smallIntValue;
-        case INTEGER_BITMASK:
-          Integer integerBitmask;
-          integerBitmask = iprot.readI32();
-          return integerBitmask;
-        case INTEGER_FLAG:
-          Integer integerFlag;
-          integerFlag = iprot.readI32();
-          return integerFlag;
-        case BINARY_VALUE:
-          Integer binaryValue;
-          binaryValue = iprot.readI32();
-          return binaryValue;
-        case LEN_VALUE:
-          Long lenValue;
-          lenValue = iprot.readI64();
-          return lenValue;
-        default:
-          throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!");
-      }
-    } else {
-      throw new TProtocolException("Couldn't find a field with field id " + fieldID);
-    }
-  }
-
-  @Override
-  protected void tupleSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    switch (setField_) {
-      case STRING_VALUE:
-        String stringValue = (String)value_;
-        oprot.writeString(stringValue);
-        return;
-      case SMALL_INT_VALUE:
-        Short smallIntValue = (Short)value_;
-        oprot.writeI16(smallIntValue);
-        return;
-      case INTEGER_BITMASK:
-        Integer integerBitmask = (Integer)value_;
-        oprot.writeI32(integerBitmask);
-        return;
-      case INTEGER_FLAG:
-        Integer integerFlag = (Integer)value_;
-        oprot.writeI32(integerFlag);
-        return;
-      case BINARY_VALUE:
-        Integer binaryValue = (Integer)value_;
-        oprot.writeI32(binaryValue);
-        return;
-      case LEN_VALUE:
-        Long lenValue = (Long)value_;
-        oprot.writeI64(lenValue);
-        return;
-      default:
-        throw new IllegalStateException("Cannot write union with unknown field " + setField_);
-    }
-  }
-
-  @Override
-  protected org.apache.thrift.protocol.TField getFieldDesc(_Fields setField) {
-    switch (setField) {
-      case STRING_VALUE:
-        return STRING_VALUE_FIELD_DESC;
-      case SMALL_INT_VALUE:
-        return SMALL_INT_VALUE_FIELD_DESC;
-      case INTEGER_BITMASK:
-        return INTEGER_BITMASK_FIELD_DESC;
-      case INTEGER_FLAG:
-        return INTEGER_FLAG_FIELD_DESC;
-      case BINARY_VALUE:
-        return BINARY_VALUE_FIELD_DESC;
-      case LEN_VALUE:
-        return LEN_VALUE_FIELD_DESC;
-      default:
-        throw new IllegalArgumentException("Unknown field id " + setField);
-    }
-  }
-
-  @Override
-  protected org.apache.thrift.protocol.TStruct getStructDesc() {
-    return STRUCT_DESC;
-  }
-
-  @Override
-  protected _Fields enumForId(short id) {
-    return _Fields.findByThriftIdOrThrow(id);
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-
-  public String getStringValue() {
-    if (getSetField() == _Fields.STRING_VALUE) {
-      return (String)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'stringValue' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setStringValue(String value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.STRING_VALUE;
-    value_ = value;
-  }
-
-  public short getSmallIntValue() {
-    if (getSetField() == _Fields.SMALL_INT_VALUE) {
-      return (Short)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'smallIntValue' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setSmallIntValue(short value) {
-    setField_ = _Fields.SMALL_INT_VALUE;
-    value_ = value;
-  }
-
-  public int getIntegerBitmask() {
-    if (getSetField() == _Fields.INTEGER_BITMASK) {
-      return (Integer)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'integerBitmask' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setIntegerBitmask(int value) {
-    setField_ = _Fields.INTEGER_BITMASK;
-    value_ = value;
-  }
-
-  public int getIntegerFlag() {
-    if (getSetField() == _Fields.INTEGER_FLAG) {
-      return (Integer)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'integerFlag' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setIntegerFlag(int value) {
-    setField_ = _Fields.INTEGER_FLAG;
-    value_ = value;
-  }
-
-  public int getBinaryValue() {
-    if (getSetField() == _Fields.BINARY_VALUE) {
-      return (Integer)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'binaryValue' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setBinaryValue(int value) {
-    setField_ = _Fields.BINARY_VALUE;
-    value_ = value;
-  }
-
-  public long getLenValue() {
-    if (getSetField() == _Fields.LEN_VALUE) {
-      return (Long)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'lenValue' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setLenValue(long value) {
-    setField_ = _Fields.LEN_VALUE;
-    value_ = value;
-  }
-
-  public boolean isSetStringValue() {
-    return setField_ == _Fields.STRING_VALUE;
-  }
-
-
-  public boolean isSetSmallIntValue() {
-    return setField_ == _Fields.SMALL_INT_VALUE;
-  }
-
-
-  public boolean isSetIntegerBitmask() {
-    return setField_ == _Fields.INTEGER_BITMASK;
-  }
-
-
-  public boolean isSetIntegerFlag() {
-    return setField_ == _Fields.INTEGER_FLAG;
-  }
-
-
-  public boolean isSetBinaryValue() {
-    return setField_ == _Fields.BINARY_VALUE;
-  }
-
-
-  public boolean isSetLenValue() {
-    return setField_ == _Fields.LEN_VALUE;
-  }
-
-
-  public boolean equals(Object other) {
-    if (other instanceof TGetInfoValue) {
-      return equals((TGetInfoValue)other);
-    } else {
-      return false;
-    }
-  }
-
-  public boolean equals(TGetInfoValue other) {
-    return other != null && getSetField() == other.getSetField() && getFieldValue().equals(other.getFieldValue());
-  }
-
-  @Override
-  public int compareTo(TGetInfoValue other) {
-    int lastComparison = org.apache.thrift.TBaseHelper.compareTo(getSetField(), other.getSetField());
-    if (lastComparison == 0) {
-      return org.apache.thrift.TBaseHelper.compareTo(getFieldValue(), other.getFieldValue());
-    }
-    return lastComparison;
-  }
-
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder hcb = new HashCodeBuilder();
-    hcb.append(this.getClass().getName());
-    org.apache.thrift.TFieldIdEnum setField = getSetField();
-    if (setField != null) {
-      hcb.append(setField.getThriftFieldId());
-      Object value = getFieldValue();
-      if (value instanceof org.apache.thrift.TEnum) {
-        hcb.append(((org.apache.thrift.TEnum)getFieldValue()).getValue());
-      } else {
-        hcb.append(value);
-      }
-    }
-    return hcb.toHashCode();
-  }
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-
-}
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetOperationStatusReq.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetOperationStatusReq.java
deleted file mode 100644
index b88591ea1945b..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetOperationStatusReq.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TGetOperationStatusReq implements org.apache.thrift.TBase<TGetOperationStatusReq, TGetOperationStatusReq._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetOperationStatusReq");
-
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetOperationStatusReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetOperationStatusReqTupleSchemeFactory());
-  }
-
-  private TOperationHandle operationHandle; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    OPERATION_HANDLE((short)1, "operationHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetOperationStatusReq.class, metaDataMap);
-  }
-
-  public TGetOperationStatusReq() {
-  }
-
-  public TGetOperationStatusReq(
-    TOperationHandle operationHandle)
-  {
-    this();
-    this.operationHandle = operationHandle;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetOperationStatusReq(TGetOperationStatusReq other) {
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-  }
-
-  public TGetOperationStatusReq deepCopy() {
-    return new TGetOperationStatusReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.operationHandle = null;
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetOperationStatusReq)
-      return this.equals((TGetOperationStatusReq)that);
-    return false;
-  }
-
-  public boolean equals(TGetOperationStatusReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    builder.append(present_operationHandle);
-    if (present_operationHandle)
-      builder.append(operationHandle);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TGetOperationStatusReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TGetOperationStatusReq typedOther = (TGetOperationStatusReq)other;
-
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(typedOther.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, typedOther.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetOperationStatusReq(");
-    boolean first = true;
-
-    sb.append("operationHandle:");
-    if (this.operationHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.operationHandle);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetOperationHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'operationHandle' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetOperationStatusReqStandardSchemeFactory implements SchemeFactory {
-    public TGetOperationStatusReqStandardScheme getScheme() {
-      return new TGetOperationStatusReqStandardScheme();
-    }
-  }
-
-  private static class TGetOperationStatusReqStandardScheme extends StandardScheme<TGetOperationStatusReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetOperationStatusReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetOperationStatusReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.operationHandle != null) {
-        oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-        struct.operationHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetOperationStatusReqTupleSchemeFactory implements SchemeFactory {
-    public TGetOperationStatusReqTupleScheme getScheme() {
-      return new TGetOperationStatusReqTupleScheme();
-    }
-  }
-
-  private static class TGetOperationStatusReqTupleScheme extends TupleScheme<TGetOperationStatusReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetOperationStatusReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.operationHandle.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetOperationStatusReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.operationHandle = new TOperationHandle();
-      struct.operationHandle.read(iprot);
-      struct.setOperationHandleIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetOperationStatusResp.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetOperationStatusResp.java
deleted file mode 100644
index 94ba6bb1146de..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetOperationStatusResp.java
+++ /dev/null
@@ -1,827 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TGetOperationStatusResp implements org.apache.thrift.TBase<TGetOperationStatusResp, TGetOperationStatusResp._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetOperationStatusResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField OPERATION_STATE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationState", org.apache.thrift.protocol.TType.I32, (short)2);
-  private static final org.apache.thrift.protocol.TField SQL_STATE_FIELD_DESC = new org.apache.thrift.protocol.TField("sqlState", org.apache.thrift.protocol.TType.STRING, (short)3);
-  private static final org.apache.thrift.protocol.TField ERROR_CODE_FIELD_DESC = new org.apache.thrift.protocol.TField("errorCode", org.apache.thrift.protocol.TType.I32, (short)4);
-  private static final org.apache.thrift.protocol.TField ERROR_MESSAGE_FIELD_DESC = new org.apache.thrift.protocol.TField("errorMessage", org.apache.thrift.protocol.TType.STRING, (short)5);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetOperationStatusRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetOperationStatusRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TOperationState operationState; // optional
-  private String sqlState; // optional
-  private int errorCode; // optional
-  private String errorMessage; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    /**
-     * 
-     * @see TOperationState
-     */
-    OPERATION_STATE((short)2, "operationState"),
-    SQL_STATE((short)3, "sqlState"),
-    ERROR_CODE((short)4, "errorCode"),
-    ERROR_MESSAGE((short)5, "errorMessage");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // OPERATION_STATE
-          return OPERATION_STATE;
-        case 3: // SQL_STATE
-          return SQL_STATE;
-        case 4: // ERROR_CODE
-          return ERROR_CODE;
-        case 5: // ERROR_MESSAGE
-          return ERROR_MESSAGE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __ERRORCODE_ISSET_ID = 0;
-  private byte __isset_bitfield = 0;
-  private _Fields optionals[] = {_Fields.OPERATION_STATE,_Fields.SQL_STATE,_Fields.ERROR_CODE,_Fields.ERROR_MESSAGE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.OPERATION_STATE, new org.apache.thrift.meta_data.FieldMetaData("operationState", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TOperationState.class)));
-    tmpMap.put(_Fields.SQL_STATE, new org.apache.thrift.meta_data.FieldMetaData("sqlState", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    tmpMap.put(_Fields.ERROR_CODE, new org.apache.thrift.meta_data.FieldMetaData("errorCode", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
-    tmpMap.put(_Fields.ERROR_MESSAGE, new org.apache.thrift.meta_data.FieldMetaData("errorMessage", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetOperationStatusResp.class, metaDataMap);
-  }
-
-  public TGetOperationStatusResp() {
-  }
-
-  public TGetOperationStatusResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetOperationStatusResp(TGetOperationStatusResp other) {
-    __isset_bitfield = other.__isset_bitfield;
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetOperationState()) {
-      this.operationState = other.operationState;
-    }
-    if (other.isSetSqlState()) {
-      this.sqlState = other.sqlState;
-    }
-    this.errorCode = other.errorCode;
-    if (other.isSetErrorMessage()) {
-      this.errorMessage = other.errorMessage;
-    }
-  }
-
-  public TGetOperationStatusResp deepCopy() {
-    return new TGetOperationStatusResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.operationState = null;
-    this.sqlState = null;
-    setErrorCodeIsSet(false);
-    this.errorCode = 0;
-    this.errorMessage = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  /**
-   * 
-   * @see TOperationState
-   */
-  public TOperationState getOperationState() {
-    return this.operationState;
-  }
-
-  /**
-   * 
-   * @see TOperationState
-   */
-  public void setOperationState(TOperationState operationState) {
-    this.operationState = operationState;
-  }
-
-  public void unsetOperationState() {
-    this.operationState = null;
-  }
-
-  /** Returns true if field operationState is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationState() {
-    return this.operationState != null;
-  }
-
-  public void setOperationStateIsSet(boolean value) {
-    if (!value) {
-      this.operationState = null;
-    }
-  }
-
-  public String getSqlState() {
-    return this.sqlState;
-  }
-
-  public void setSqlState(String sqlState) {
-    this.sqlState = sqlState;
-  }
-
-  public void unsetSqlState() {
-    this.sqlState = null;
-  }
-
-  /** Returns true if field sqlState is set (has been assigned a value) and false otherwise */
-  public boolean isSetSqlState() {
-    return this.sqlState != null;
-  }
-
-  public void setSqlStateIsSet(boolean value) {
-    if (!value) {
-      this.sqlState = null;
-    }
-  }
-
-  public int getErrorCode() {
-    return this.errorCode;
-  }
-
-  public void setErrorCode(int errorCode) {
-    this.errorCode = errorCode;
-    setErrorCodeIsSet(true);
-  }
-
-  public void unsetErrorCode() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __ERRORCODE_ISSET_ID);
-  }
-
-  /** Returns true if field errorCode is set (has been assigned a value) and false otherwise */
-  public boolean isSetErrorCode() {
-    return EncodingUtils.testBit(__isset_bitfield, __ERRORCODE_ISSET_ID);
-  }
-
-  public void setErrorCodeIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __ERRORCODE_ISSET_ID, value);
-  }
-
-  public String getErrorMessage() {
-    return this.errorMessage;
-  }
-
-  public void setErrorMessage(String errorMessage) {
-    this.errorMessage = errorMessage;
-  }
-
-  public void unsetErrorMessage() {
-    this.errorMessage = null;
-  }
-
-  /** Returns true if field errorMessage is set (has been assigned a value) and false otherwise */
-  public boolean isSetErrorMessage() {
-    return this.errorMessage != null;
-  }
-
-  public void setErrorMessageIsSet(boolean value) {
-    if (!value) {
-      this.errorMessage = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case OPERATION_STATE:
-      if (value == null) {
-        unsetOperationState();
-      } else {
-        setOperationState((TOperationState)value);
-      }
-      break;
-
-    case SQL_STATE:
-      if (value == null) {
-        unsetSqlState();
-      } else {
-        setSqlState((String)value);
-      }
-      break;
-
-    case ERROR_CODE:
-      if (value == null) {
-        unsetErrorCode();
-      } else {
-        setErrorCode((Integer)value);
-      }
-      break;
-
-    case ERROR_MESSAGE:
-      if (value == null) {
-        unsetErrorMessage();
-      } else {
-        setErrorMessage((String)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case OPERATION_STATE:
-      return getOperationState();
-
-    case SQL_STATE:
-      return getSqlState();
-
-    case ERROR_CODE:
-      return Integer.valueOf(getErrorCode());
-
-    case ERROR_MESSAGE:
-      return getErrorMessage();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case OPERATION_STATE:
-      return isSetOperationState();
-    case SQL_STATE:
-      return isSetSqlState();
-    case ERROR_CODE:
-      return isSetErrorCode();
-    case ERROR_MESSAGE:
-      return isSetErrorMessage();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetOperationStatusResp)
-      return this.equals((TGetOperationStatusResp)that);
-    return false;
-  }
-
-  public boolean equals(TGetOperationStatusResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_operationState = true && this.isSetOperationState();
-    boolean that_present_operationState = true && that.isSetOperationState();
-    if (this_present_operationState || that_present_operationState) {
-      if (!(this_present_operationState && that_present_operationState))
-        return false;
-      if (!this.operationState.equals(that.operationState))
-        return false;
-    }
-
-    boolean this_present_sqlState = true && this.isSetSqlState();
-    boolean that_present_sqlState = true && that.isSetSqlState();
-    if (this_present_sqlState || that_present_sqlState) {
-      if (!(this_present_sqlState && that_present_sqlState))
-        return false;
-      if (!this.sqlState.equals(that.sqlState))
-        return false;
-    }
-
-    boolean this_present_errorCode = true && this.isSetErrorCode();
-    boolean that_present_errorCode = true && that.isSetErrorCode();
-    if (this_present_errorCode || that_present_errorCode) {
-      if (!(this_present_errorCode && that_present_errorCode))
-        return false;
-      if (this.errorCode != that.errorCode)
-        return false;
-    }
-
-    boolean this_present_errorMessage = true && this.isSetErrorMessage();
-    boolean that_present_errorMessage = true && that.isSetErrorMessage();
-    if (this_present_errorMessage || that_present_errorMessage) {
-      if (!(this_present_errorMessage && that_present_errorMessage))
-        return false;
-      if (!this.errorMessage.equals(that.errorMessage))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_status = true && (isSetStatus());
-    builder.append(present_status);
-    if (present_status)
-      builder.append(status);
-
-    boolean present_operationState = true && (isSetOperationState());
-    builder.append(present_operationState);
-    if (present_operationState)
-      builder.append(operationState.getValue());
-
-    boolean present_sqlState = true && (isSetSqlState());
-    builder.append(present_sqlState);
-    if (present_sqlState)
-      builder.append(sqlState);
-
-    boolean present_errorCode = true && (isSetErrorCode());
-    builder.append(present_errorCode);
-    if (present_errorCode)
-      builder.append(errorCode);
-
-    boolean present_errorMessage = true && (isSetErrorMessage());
-    builder.append(present_errorMessage);
-    if (present_errorMessage)
-      builder.append(errorMessage);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TGetOperationStatusResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TGetOperationStatusResp typedOther = (TGetOperationStatusResp)other;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(typedOther.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, typedOther.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOperationState()).compareTo(typedOther.isSetOperationState());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationState()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationState, typedOther.operationState);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetSqlState()).compareTo(typedOther.isSetSqlState());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSqlState()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sqlState, typedOther.sqlState);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetErrorCode()).compareTo(typedOther.isSetErrorCode());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetErrorCode()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.errorCode, typedOther.errorCode);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetErrorMessage()).compareTo(typedOther.isSetErrorMessage());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetErrorMessage()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.errorMessage, typedOther.errorMessage);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetOperationStatusResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetOperationState()) {
-      if (!first) sb.append(", ");
-      sb.append("operationState:");
-      if (this.operationState == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.operationState);
-      }
-      first = false;
-    }
-    if (isSetSqlState()) {
-      if (!first) sb.append(", ");
-      sb.append("sqlState:");
-      if (this.sqlState == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.sqlState);
-      }
-      first = false;
-    }
-    if (isSetErrorCode()) {
-      if (!first) sb.append(", ");
-      sb.append("errorCode:");
-      sb.append(this.errorCode);
-      first = false;
-    }
-    if (isSetErrorMessage()) {
-      if (!first) sb.append(", ");
-      sb.append("errorMessage:");
-      if (this.errorMessage == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.errorMessage);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetOperationStatusRespStandardSchemeFactory implements SchemeFactory {
-    public TGetOperationStatusRespStandardScheme getScheme() {
-      return new TGetOperationStatusRespStandardScheme();
-    }
-  }
-
-  private static class TGetOperationStatusRespStandardScheme extends StandardScheme<TGetOperationStatusResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetOperationStatusResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // OPERATION_STATE
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.operationState = TOperationState.findByValue(iprot.readI32());
-              struct.setOperationStateIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // SQL_STATE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.sqlState = iprot.readString();
-              struct.setSqlStateIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // ERROR_CODE
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.errorCode = iprot.readI32();
-              struct.setErrorCodeIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 5: // ERROR_MESSAGE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.errorMessage = iprot.readString();
-              struct.setErrorMessageIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetOperationStatusResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.operationState != null) {
-        if (struct.isSetOperationState()) {
-          oprot.writeFieldBegin(OPERATION_STATE_FIELD_DESC);
-          oprot.writeI32(struct.operationState.getValue());
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.sqlState != null) {
-        if (struct.isSetSqlState()) {
-          oprot.writeFieldBegin(SQL_STATE_FIELD_DESC);
-          oprot.writeString(struct.sqlState);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.isSetErrorCode()) {
-        oprot.writeFieldBegin(ERROR_CODE_FIELD_DESC);
-        oprot.writeI32(struct.errorCode);
-        oprot.writeFieldEnd();
-      }
-      if (struct.errorMessage != null) {
-        if (struct.isSetErrorMessage()) {
-          oprot.writeFieldBegin(ERROR_MESSAGE_FIELD_DESC);
-          oprot.writeString(struct.errorMessage);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetOperationStatusRespTupleSchemeFactory implements SchemeFactory {
-    public TGetOperationStatusRespTupleScheme getScheme() {
-      return new TGetOperationStatusRespTupleScheme();
-    }
-  }
-
-  private static class TGetOperationStatusRespTupleScheme extends TupleScheme<TGetOperationStatusResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetOperationStatusResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetOperationState()) {
-        optionals.set(0);
-      }
-      if (struct.isSetSqlState()) {
-        optionals.set(1);
-      }
-      if (struct.isSetErrorCode()) {
-        optionals.set(2);
-      }
-      if (struct.isSetErrorMessage()) {
-        optionals.set(3);
-      }
-      oprot.writeBitSet(optionals, 4);
-      if (struct.isSetOperationState()) {
-        oprot.writeI32(struct.operationState.getValue());
-      }
-      if (struct.isSetSqlState()) {
-        oprot.writeString(struct.sqlState);
-      }
-      if (struct.isSetErrorCode()) {
-        oprot.writeI32(struct.errorCode);
-      }
-      if (struct.isSetErrorMessage()) {
-        oprot.writeString(struct.errorMessage);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetOperationStatusResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(4);
-      if (incoming.get(0)) {
-        struct.operationState = TOperationState.findByValue(iprot.readI32());
-        struct.setOperationStateIsSet(true);
-      }
-      if (incoming.get(1)) {
-        struct.sqlState = iprot.readString();
-        struct.setSqlStateIsSet(true);
-      }
-      if (incoming.get(2)) {
-        struct.errorCode = iprot.readI32();
-        struct.setErrorCodeIsSet(true);
-      }
-      if (incoming.get(3)) {
-        struct.errorMessage = iprot.readString();
-        struct.setErrorMessageIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetResultSetMetadataReq.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetResultSetMetadataReq.java
deleted file mode 100644
index 3bf363c958468..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetResultSetMetadataReq.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TGetResultSetMetadataReq implements org.apache.thrift.TBase<TGetResultSetMetadataReq, TGetResultSetMetadataReq._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetResultSetMetadataReq");
-
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetResultSetMetadataReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetResultSetMetadataReqTupleSchemeFactory());
-  }
-
-  private TOperationHandle operationHandle; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    OPERATION_HANDLE((short)1, "operationHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetResultSetMetadataReq.class, metaDataMap);
-  }
-
-  public TGetResultSetMetadataReq() {
-  }
-
-  public TGetResultSetMetadataReq(
-    TOperationHandle operationHandle)
-  {
-    this();
-    this.operationHandle = operationHandle;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetResultSetMetadataReq(TGetResultSetMetadataReq other) {
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-  }
-
-  public TGetResultSetMetadataReq deepCopy() {
-    return new TGetResultSetMetadataReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.operationHandle = null;
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetResultSetMetadataReq)
-      return this.equals((TGetResultSetMetadataReq)that);
-    return false;
-  }
-
-  public boolean equals(TGetResultSetMetadataReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    builder.append(present_operationHandle);
-    if (present_operationHandle)
-      builder.append(operationHandle);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TGetResultSetMetadataReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TGetResultSetMetadataReq typedOther = (TGetResultSetMetadataReq)other;
-
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(typedOther.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, typedOther.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetResultSetMetadataReq(");
-    boolean first = true;
-
-    sb.append("operationHandle:");
-    if (this.operationHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.operationHandle);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetOperationHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'operationHandle' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetResultSetMetadataReqStandardSchemeFactory implements SchemeFactory {
-    public TGetResultSetMetadataReqStandardScheme getScheme() {
-      return new TGetResultSetMetadataReqStandardScheme();
-    }
-  }
-
-  private static class TGetResultSetMetadataReqStandardScheme extends StandardScheme<TGetResultSetMetadataReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetResultSetMetadataReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetResultSetMetadataReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.operationHandle != null) {
-        oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-        struct.operationHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetResultSetMetadataReqTupleSchemeFactory implements SchemeFactory {
-    public TGetResultSetMetadataReqTupleScheme getScheme() {
-      return new TGetResultSetMetadataReqTupleScheme();
-    }
-  }
-
-  private static class TGetResultSetMetadataReqTupleScheme extends TupleScheme<TGetResultSetMetadataReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetResultSetMetadataReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.operationHandle.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetResultSetMetadataReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.operationHandle = new TOperationHandle();
-      struct.operationHandle.read(iprot);
-      struct.setOperationHandleIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetResultSetMetadataResp.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetResultSetMetadataResp.java
deleted file mode 100644
index a9bef9f722c16..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetResultSetMetadataResp.java
+++ /dev/null
@@ -1,505 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TGetResultSetMetadataResp implements org.apache.thrift.TBase<TGetResultSetMetadataResp, TGetResultSetMetadataResp._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetResultSetMetadataResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField SCHEMA_FIELD_DESC = new org.apache.thrift.protocol.TField("schema", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetResultSetMetadataRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetResultSetMetadataRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TTableSchema schema; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    SCHEMA((short)2, "schema");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // SCHEMA
-          return SCHEMA;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private _Fields optionals[] = {_Fields.SCHEMA};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.SCHEMA, new org.apache.thrift.meta_data.FieldMetaData("schema", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TTableSchema.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetResultSetMetadataResp.class, metaDataMap);
-  }
-
-  public TGetResultSetMetadataResp() {
-  }
-
-  public TGetResultSetMetadataResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetResultSetMetadataResp(TGetResultSetMetadataResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetSchema()) {
-      this.schema = new TTableSchema(other.schema);
-    }
-  }
-
-  public TGetResultSetMetadataResp deepCopy() {
-    return new TGetResultSetMetadataResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.schema = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public TTableSchema getSchema() {
-    return this.schema;
-  }
-
-  public void setSchema(TTableSchema schema) {
-    this.schema = schema;
-  }
-
-  public void unsetSchema() {
-    this.schema = null;
-  }
-
-  /** Returns true if field schema is set (has been assigned a value) and false otherwise */
-  public boolean isSetSchema() {
-    return this.schema != null;
-  }
-
-  public void setSchemaIsSet(boolean value) {
-    if (!value) {
-      this.schema = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case SCHEMA:
-      if (value == null) {
-        unsetSchema();
-      } else {
-        setSchema((TTableSchema)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case SCHEMA:
-      return getSchema();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case SCHEMA:
-      return isSetSchema();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetResultSetMetadataResp)
-      return this.equals((TGetResultSetMetadataResp)that);
-    return false;
-  }
-
-  public boolean equals(TGetResultSetMetadataResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_schema = true && this.isSetSchema();
-    boolean that_present_schema = true && that.isSetSchema();
-    if (this_present_schema || that_present_schema) {
-      if (!(this_present_schema && that_present_schema))
-        return false;
-      if (!this.schema.equals(that.schema))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_status = true && (isSetStatus());
-    builder.append(present_status);
-    if (present_status)
-      builder.append(status);
-
-    boolean present_schema = true && (isSetSchema());
-    builder.append(present_schema);
-    if (present_schema)
-      builder.append(schema);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TGetResultSetMetadataResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TGetResultSetMetadataResp typedOther = (TGetResultSetMetadataResp)other;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(typedOther.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, typedOther.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetSchema()).compareTo(typedOther.isSetSchema());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSchema()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.schema, typedOther.schema);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetResultSetMetadataResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetSchema()) {
-      if (!first) sb.append(", ");
-      sb.append("schema:");
-      if (this.schema == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.schema);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-    if (schema != null) {
-      schema.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetResultSetMetadataRespStandardSchemeFactory implements SchemeFactory {
-    public TGetResultSetMetadataRespStandardScheme getScheme() {
-      return new TGetResultSetMetadataRespStandardScheme();
-    }
-  }
-
-  private static class TGetResultSetMetadataRespStandardScheme extends StandardScheme<TGetResultSetMetadataResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetResultSetMetadataResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // SCHEMA
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.schema = new TTableSchema();
-              struct.schema.read(iprot);
-              struct.setSchemaIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetResultSetMetadataResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.schema != null) {
-        if (struct.isSetSchema()) {
-          oprot.writeFieldBegin(SCHEMA_FIELD_DESC);
-          struct.schema.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetResultSetMetadataRespTupleSchemeFactory implements SchemeFactory {
-    public TGetResultSetMetadataRespTupleScheme getScheme() {
-      return new TGetResultSetMetadataRespTupleScheme();
-    }
-  }
-
-  private static class TGetResultSetMetadataRespTupleScheme extends TupleScheme<TGetResultSetMetadataResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetResultSetMetadataResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetSchema()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetSchema()) {
-        struct.schema.write(oprot);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetResultSetMetadataResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.schema = new TTableSchema();
-        struct.schema.read(iprot);
-        struct.setSchemaIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetSchemasReq.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetSchemasReq.java
deleted file mode 100644
index c2aadaa49a1e9..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetSchemasReq.java
+++ /dev/null
@@ -1,606 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TGetSchemasReq implements org.apache.thrift.TBase<TGetSchemasReq, TGetSchemasReq._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetSchemasReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField CATALOG_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("catalogName", org.apache.thrift.protocol.TType.STRING, (short)2);
-  private static final org.apache.thrift.protocol.TField SCHEMA_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("schemaName", org.apache.thrift.protocol.TType.STRING, (short)3);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetSchemasReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetSchemasReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-  private String catalogName; // optional
-  private String schemaName; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle"),
-    CATALOG_NAME((short)2, "catalogName"),
-    SCHEMA_NAME((short)3, "schemaName");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        case 2: // CATALOG_NAME
-          return CATALOG_NAME;
-        case 3: // SCHEMA_NAME
-          return SCHEMA_NAME;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private _Fields optionals[] = {_Fields.CATALOG_NAME,_Fields.SCHEMA_NAME};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    tmpMap.put(_Fields.CATALOG_NAME, new org.apache.thrift.meta_data.FieldMetaData("catalogName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TIdentifier")));
-    tmpMap.put(_Fields.SCHEMA_NAME, new org.apache.thrift.meta_data.FieldMetaData("schemaName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TPatternOrIdentifier")));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetSchemasReq.class, metaDataMap);
-  }
-
-  public TGetSchemasReq() {
-  }
-
-  public TGetSchemasReq(
-    TSessionHandle sessionHandle)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetSchemasReq(TGetSchemasReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-    if (other.isSetCatalogName()) {
-      this.catalogName = other.catalogName;
-    }
-    if (other.isSetSchemaName()) {
-      this.schemaName = other.schemaName;
-    }
-  }
-
-  public TGetSchemasReq deepCopy() {
-    return new TGetSchemasReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-    this.catalogName = null;
-    this.schemaName = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public String getCatalogName() {
-    return this.catalogName;
-  }
-
-  public void setCatalogName(String catalogName) {
-    this.catalogName = catalogName;
-  }
-
-  public void unsetCatalogName() {
-    this.catalogName = null;
-  }
-
-  /** Returns true if field catalogName is set (has been assigned a value) and false otherwise */
-  public boolean isSetCatalogName() {
-    return this.catalogName != null;
-  }
-
-  public void setCatalogNameIsSet(boolean value) {
-    if (!value) {
-      this.catalogName = null;
-    }
-  }
-
-  public String getSchemaName() {
-    return this.schemaName;
-  }
-
-  public void setSchemaName(String schemaName) {
-    this.schemaName = schemaName;
-  }
-
-  public void unsetSchemaName() {
-    this.schemaName = null;
-  }
-
-  /** Returns true if field schemaName is set (has been assigned a value) and false otherwise */
-  public boolean isSetSchemaName() {
-    return this.schemaName != null;
-  }
-
-  public void setSchemaNameIsSet(boolean value) {
-    if (!value) {
-      this.schemaName = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    case CATALOG_NAME:
-      if (value == null) {
-        unsetCatalogName();
-      } else {
-        setCatalogName((String)value);
-      }
-      break;
-
-    case SCHEMA_NAME:
-      if (value == null) {
-        unsetSchemaName();
-      } else {
-        setSchemaName((String)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    case CATALOG_NAME:
-      return getCatalogName();
-
-    case SCHEMA_NAME:
-      return getSchemaName();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    case CATALOG_NAME:
-      return isSetCatalogName();
-    case SCHEMA_NAME:
-      return isSetSchemaName();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetSchemasReq)
-      return this.equals((TGetSchemasReq)that);
-    return false;
-  }
-
-  public boolean equals(TGetSchemasReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    boolean this_present_catalogName = true && this.isSetCatalogName();
-    boolean that_present_catalogName = true && that.isSetCatalogName();
-    if (this_present_catalogName || that_present_catalogName) {
-      if (!(this_present_catalogName && that_present_catalogName))
-        return false;
-      if (!this.catalogName.equals(that.catalogName))
-        return false;
-    }
-
-    boolean this_present_schemaName = true && this.isSetSchemaName();
-    boolean that_present_schemaName = true && that.isSetSchemaName();
-    if (this_present_schemaName || that_present_schemaName) {
-      if (!(this_present_schemaName && that_present_schemaName))
-        return false;
-      if (!this.schemaName.equals(that.schemaName))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    builder.append(present_sessionHandle);
-    if (present_sessionHandle)
-      builder.append(sessionHandle);
-
-    boolean present_catalogName = true && (isSetCatalogName());
-    builder.append(present_catalogName);
-    if (present_catalogName)
-      builder.append(catalogName);
-
-    boolean present_schemaName = true && (isSetSchemaName());
-    builder.append(present_schemaName);
-    if (present_schemaName)
-      builder.append(schemaName);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TGetSchemasReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TGetSchemasReq typedOther = (TGetSchemasReq)other;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(typedOther.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, typedOther.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetCatalogName()).compareTo(typedOther.isSetCatalogName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetCatalogName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.catalogName, typedOther.catalogName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetSchemaName()).compareTo(typedOther.isSetSchemaName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSchemaName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.schemaName, typedOther.schemaName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetSchemasReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    if (isSetCatalogName()) {
-      if (!first) sb.append(", ");
-      sb.append("catalogName:");
-      if (this.catalogName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.catalogName);
-      }
-      first = false;
-    }
-    if (isSetSchemaName()) {
-      if (!first) sb.append(", ");
-      sb.append("schemaName:");
-      if (this.schemaName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.schemaName);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetSchemasReqStandardSchemeFactory implements SchemeFactory {
-    public TGetSchemasReqStandardScheme getScheme() {
-      return new TGetSchemasReqStandardScheme();
-    }
-  }
-
-  private static class TGetSchemasReqStandardScheme extends StandardScheme<TGetSchemasReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetSchemasReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // CATALOG_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.catalogName = iprot.readString();
-              struct.setCatalogNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // SCHEMA_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.schemaName = iprot.readString();
-              struct.setSchemaNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetSchemasReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.catalogName != null) {
-        if (struct.isSetCatalogName()) {
-          oprot.writeFieldBegin(CATALOG_NAME_FIELD_DESC);
-          oprot.writeString(struct.catalogName);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.schemaName != null) {
-        if (struct.isSetSchemaName()) {
-          oprot.writeFieldBegin(SCHEMA_NAME_FIELD_DESC);
-          oprot.writeString(struct.schemaName);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetSchemasReqTupleSchemeFactory implements SchemeFactory {
-    public TGetSchemasReqTupleScheme getScheme() {
-      return new TGetSchemasReqTupleScheme();
-    }
-  }
-
-  private static class TGetSchemasReqTupleScheme extends TupleScheme<TGetSchemasReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetSchemasReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetCatalogName()) {
-        optionals.set(0);
-      }
-      if (struct.isSetSchemaName()) {
-        optionals.set(1);
-      }
-      oprot.writeBitSet(optionals, 2);
-      if (struct.isSetCatalogName()) {
-        oprot.writeString(struct.catalogName);
-      }
-      if (struct.isSetSchemaName()) {
-        oprot.writeString(struct.schemaName);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetSchemasReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-      BitSet incoming = iprot.readBitSet(2);
-      if (incoming.get(0)) {
-        struct.catalogName = iprot.readString();
-        struct.setCatalogNameIsSet(true);
-      }
-      if (incoming.get(1)) {
-        struct.schemaName = iprot.readString();
-        struct.setSchemaNameIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetSchemasResp.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetSchemasResp.java
deleted file mode 100644
index ac1ea3e7cc7af..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetSchemasResp.java
+++ /dev/null
@@ -1,505 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TGetSchemasResp implements org.apache.thrift.TBase<TGetSchemasResp, TGetSchemasResp._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetSchemasResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetSchemasRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetSchemasRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TOperationHandle operationHandle; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    OPERATION_HANDLE((short)2, "operationHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private _Fields optionals[] = {_Fields.OPERATION_HANDLE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetSchemasResp.class, metaDataMap);
-  }
-
-  public TGetSchemasResp() {
-  }
-
-  public TGetSchemasResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetSchemasResp(TGetSchemasResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-  }
-
-  public TGetSchemasResp deepCopy() {
-    return new TGetSchemasResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.operationHandle = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetSchemasResp)
-      return this.equals((TGetSchemasResp)that);
-    return false;
-  }
-
-  public boolean equals(TGetSchemasResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_status = true && (isSetStatus());
-    builder.append(present_status);
-    if (present_status)
-      builder.append(status);
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    builder.append(present_operationHandle);
-    if (present_operationHandle)
-      builder.append(operationHandle);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TGetSchemasResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TGetSchemasResp typedOther = (TGetSchemasResp)other;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(typedOther.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, typedOther.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(typedOther.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, typedOther.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetSchemasResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetOperationHandle()) {
-      if (!first) sb.append(", ");
-      sb.append("operationHandle:");
-      if (this.operationHandle == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.operationHandle);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetSchemasRespStandardSchemeFactory implements SchemeFactory {
-    public TGetSchemasRespStandardScheme getScheme() {
-      return new TGetSchemasRespStandardScheme();
-    }
-  }
-
-  private static class TGetSchemasRespStandardScheme extends StandardScheme<TGetSchemasResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetSchemasResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetSchemasResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.operationHandle != null) {
-        if (struct.isSetOperationHandle()) {
-          oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-          struct.operationHandle.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetSchemasRespTupleSchemeFactory implements SchemeFactory {
-    public TGetSchemasRespTupleScheme getScheme() {
-      return new TGetSchemasRespTupleScheme();
-    }
-  }
-
-  private static class TGetSchemasRespTupleScheme extends TupleScheme<TGetSchemasResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetSchemasResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetOperationHandle()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetOperationHandle()) {
-        struct.operationHandle.write(oprot);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetSchemasResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.operationHandle = new TOperationHandle();
-        struct.operationHandle.read(iprot);
-        struct.setOperationHandleIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetTableTypesReq.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetTableTypesReq.java
deleted file mode 100644
index 6f2c713e0be6a..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetTableTypesReq.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TGetTableTypesReq implements org.apache.thrift.TBase<TGetTableTypesReq, TGetTableTypesReq._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetTableTypesReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetTableTypesReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetTableTypesReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetTableTypesReq.class, metaDataMap);
-  }
-
-  public TGetTableTypesReq() {
-  }
-
-  public TGetTableTypesReq(
-    TSessionHandle sessionHandle)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetTableTypesReq(TGetTableTypesReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-  }
-
-  public TGetTableTypesReq deepCopy() {
-    return new TGetTableTypesReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetTableTypesReq)
-      return this.equals((TGetTableTypesReq)that);
-    return false;
-  }
-
-  public boolean equals(TGetTableTypesReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    builder.append(present_sessionHandle);
-    if (present_sessionHandle)
-      builder.append(sessionHandle);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TGetTableTypesReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TGetTableTypesReq typedOther = (TGetTableTypesReq)other;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(typedOther.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, typedOther.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetTableTypesReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetTableTypesReqStandardSchemeFactory implements SchemeFactory {
-    public TGetTableTypesReqStandardScheme getScheme() {
-      return new TGetTableTypesReqStandardScheme();
-    }
-  }
-
-  private static class TGetTableTypesReqStandardScheme extends StandardScheme<TGetTableTypesReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetTableTypesReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetTableTypesReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetTableTypesReqTupleSchemeFactory implements SchemeFactory {
-    public TGetTableTypesReqTupleScheme getScheme() {
-      return new TGetTableTypesReqTupleScheme();
-    }
-  }
-
-  private static class TGetTableTypesReqTupleScheme extends TupleScheme<TGetTableTypesReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetTableTypesReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetTableTypesReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetTableTypesResp.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetTableTypesResp.java
deleted file mode 100644
index 6f33fbcf5dadc..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetTableTypesResp.java
+++ /dev/null
@@ -1,505 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TGetTableTypesResp implements org.apache.thrift.TBase<TGetTableTypesResp, TGetTableTypesResp._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetTableTypesResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetTableTypesRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetTableTypesRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TOperationHandle operationHandle; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    OPERATION_HANDLE((short)2, "operationHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private _Fields optionals[] = {_Fields.OPERATION_HANDLE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetTableTypesResp.class, metaDataMap);
-  }
-
-  public TGetTableTypesResp() {
-  }
-
-  public TGetTableTypesResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetTableTypesResp(TGetTableTypesResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-  }
-
-  public TGetTableTypesResp deepCopy() {
-    return new TGetTableTypesResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.operationHandle = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetTableTypesResp)
-      return this.equals((TGetTableTypesResp)that);
-    return false;
-  }
-
-  public boolean equals(TGetTableTypesResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_status = true && (isSetStatus());
-    builder.append(present_status);
-    if (present_status)
-      builder.append(status);
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    builder.append(present_operationHandle);
-    if (present_operationHandle)
-      builder.append(operationHandle);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TGetTableTypesResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TGetTableTypesResp typedOther = (TGetTableTypesResp)other;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(typedOther.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, typedOther.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(typedOther.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, typedOther.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetTableTypesResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetOperationHandle()) {
-      if (!first) sb.append(", ");
-      sb.append("operationHandle:");
-      if (this.operationHandle == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.operationHandle);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetTableTypesRespStandardSchemeFactory implements SchemeFactory {
-    public TGetTableTypesRespStandardScheme getScheme() {
-      return new TGetTableTypesRespStandardScheme();
-    }
-  }
-
-  private static class TGetTableTypesRespStandardScheme extends StandardScheme<TGetTableTypesResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetTableTypesResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetTableTypesResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.operationHandle != null) {
-        if (struct.isSetOperationHandle()) {
-          oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-          struct.operationHandle.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetTableTypesRespTupleSchemeFactory implements SchemeFactory {
-    public TGetTableTypesRespTupleScheme getScheme() {
-      return new TGetTableTypesRespTupleScheme();
-    }
-  }
-
-  private static class TGetTableTypesRespTupleScheme extends TupleScheme<TGetTableTypesResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetTableTypesResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetOperationHandle()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetOperationHandle()) {
-        struct.operationHandle.write(oprot);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetTableTypesResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.operationHandle = new TOperationHandle();
-        struct.operationHandle.read(iprot);
-        struct.setOperationHandleIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetTablesReq.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetTablesReq.java
deleted file mode 100644
index c973fcc24cb10..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetTablesReq.java
+++ /dev/null
@@ -1,870 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TGetTablesReq implements org.apache.thrift.TBase<TGetTablesReq, TGetTablesReq._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetTablesReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField CATALOG_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("catalogName", org.apache.thrift.protocol.TType.STRING, (short)2);
-  private static final org.apache.thrift.protocol.TField SCHEMA_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("schemaName", org.apache.thrift.protocol.TType.STRING, (short)3);
-  private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)4);
-  private static final org.apache.thrift.protocol.TField TABLE_TYPES_FIELD_DESC = new org.apache.thrift.protocol.TField("tableTypes", org.apache.thrift.protocol.TType.LIST, (short)5);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetTablesReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetTablesReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-  private String catalogName; // optional
-  private String schemaName; // optional
-  private String tableName; // optional
-  private List<String> tableTypes; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle"),
-    CATALOG_NAME((short)2, "catalogName"),
-    SCHEMA_NAME((short)3, "schemaName"),
-    TABLE_NAME((short)4, "tableName"),
-    TABLE_TYPES((short)5, "tableTypes");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        case 2: // CATALOG_NAME
-          return CATALOG_NAME;
-        case 3: // SCHEMA_NAME
-          return SCHEMA_NAME;
-        case 4: // TABLE_NAME
-          return TABLE_NAME;
-        case 5: // TABLE_TYPES
-          return TABLE_TYPES;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private _Fields optionals[] = {_Fields.CATALOG_NAME,_Fields.SCHEMA_NAME,_Fields.TABLE_NAME,_Fields.TABLE_TYPES};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    tmpMap.put(_Fields.CATALOG_NAME, new org.apache.thrift.meta_data.FieldMetaData("catalogName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TPatternOrIdentifier")));
-    tmpMap.put(_Fields.SCHEMA_NAME, new org.apache.thrift.meta_data.FieldMetaData("schemaName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TPatternOrIdentifier")));
-    tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TPatternOrIdentifier")));
-    tmpMap.put(_Fields.TABLE_TYPES, new org.apache.thrift.meta_data.FieldMetaData("tableTypes", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetTablesReq.class, metaDataMap);
-  }
-
-  public TGetTablesReq() {
-  }
-
-  public TGetTablesReq(
-    TSessionHandle sessionHandle)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetTablesReq(TGetTablesReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-    if (other.isSetCatalogName()) {
-      this.catalogName = other.catalogName;
-    }
-    if (other.isSetSchemaName()) {
-      this.schemaName = other.schemaName;
-    }
-    if (other.isSetTableName()) {
-      this.tableName = other.tableName;
-    }
-    if (other.isSetTableTypes()) {
-      List<String> __this__tableTypes = new ArrayList<String>();
-      for (String other_element : other.tableTypes) {
-        __this__tableTypes.add(other_element);
-      }
-      this.tableTypes = __this__tableTypes;
-    }
-  }
-
-  public TGetTablesReq deepCopy() {
-    return new TGetTablesReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-    this.catalogName = null;
-    this.schemaName = null;
-    this.tableName = null;
-    this.tableTypes = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public String getCatalogName() {
-    return this.catalogName;
-  }
-
-  public void setCatalogName(String catalogName) {
-    this.catalogName = catalogName;
-  }
-
-  public void unsetCatalogName() {
-    this.catalogName = null;
-  }
-
-  /** Returns true if field catalogName is set (has been assigned a value) and false otherwise */
-  public boolean isSetCatalogName() {
-    return this.catalogName != null;
-  }
-
-  public void setCatalogNameIsSet(boolean value) {
-    if (!value) {
-      this.catalogName = null;
-    }
-  }
-
-  public String getSchemaName() {
-    return this.schemaName;
-  }
-
-  public void setSchemaName(String schemaName) {
-    this.schemaName = schemaName;
-  }
-
-  public void unsetSchemaName() {
-    this.schemaName = null;
-  }
-
-  /** Returns true if field schemaName is set (has been assigned a value) and false otherwise */
-  public boolean isSetSchemaName() {
-    return this.schemaName != null;
-  }
-
-  public void setSchemaNameIsSet(boolean value) {
-    if (!value) {
-      this.schemaName = null;
-    }
-  }
-
-  public String getTableName() {
-    return this.tableName;
-  }
-
-  public void setTableName(String tableName) {
-    this.tableName = tableName;
-  }
-
-  public void unsetTableName() {
-    this.tableName = null;
-  }
-
-  /** Returns true if field tableName is set (has been assigned a value) and false otherwise */
-  public boolean isSetTableName() {
-    return this.tableName != null;
-  }
-
-  public void setTableNameIsSet(boolean value) {
-    if (!value) {
-      this.tableName = null;
-    }
-  }
-
-  public int getTableTypesSize() {
-    return (this.tableTypes == null) ? 0 : this.tableTypes.size();
-  }
-
-  public java.util.Iterator<String> getTableTypesIterator() {
-    return (this.tableTypes == null) ? null : this.tableTypes.iterator();
-  }
-
-  public void addToTableTypes(String elem) {
-    if (this.tableTypes == null) {
-      this.tableTypes = new ArrayList<String>();
-    }
-    this.tableTypes.add(elem);
-  }
-
-  public List<String> getTableTypes() {
-    return this.tableTypes;
-  }
-
-  public void setTableTypes(List<String> tableTypes) {
-    this.tableTypes = tableTypes;
-  }
-
-  public void unsetTableTypes() {
-    this.tableTypes = null;
-  }
-
-  /** Returns true if field tableTypes is set (has been assigned a value) and false otherwise */
-  public boolean isSetTableTypes() {
-    return this.tableTypes != null;
-  }
-
-  public void setTableTypesIsSet(boolean value) {
-    if (!value) {
-      this.tableTypes = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    case CATALOG_NAME:
-      if (value == null) {
-        unsetCatalogName();
-      } else {
-        setCatalogName((String)value);
-      }
-      break;
-
-    case SCHEMA_NAME:
-      if (value == null) {
-        unsetSchemaName();
-      } else {
-        setSchemaName((String)value);
-      }
-      break;
-
-    case TABLE_NAME:
-      if (value == null) {
-        unsetTableName();
-      } else {
-        setTableName((String)value);
-      }
-      break;
-
-    case TABLE_TYPES:
-      if (value == null) {
-        unsetTableTypes();
-      } else {
-        setTableTypes((List<String>)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    case CATALOG_NAME:
-      return getCatalogName();
-
-    case SCHEMA_NAME:
-      return getSchemaName();
-
-    case TABLE_NAME:
-      return getTableName();
-
-    case TABLE_TYPES:
-      return getTableTypes();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    case CATALOG_NAME:
-      return isSetCatalogName();
-    case SCHEMA_NAME:
-      return isSetSchemaName();
-    case TABLE_NAME:
-      return isSetTableName();
-    case TABLE_TYPES:
-      return isSetTableTypes();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetTablesReq)
-      return this.equals((TGetTablesReq)that);
-    return false;
-  }
-
-  public boolean equals(TGetTablesReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    boolean this_present_catalogName = true && this.isSetCatalogName();
-    boolean that_present_catalogName = true && that.isSetCatalogName();
-    if (this_present_catalogName || that_present_catalogName) {
-      if (!(this_present_catalogName && that_present_catalogName))
-        return false;
-      if (!this.catalogName.equals(that.catalogName))
-        return false;
-    }
-
-    boolean this_present_schemaName = true && this.isSetSchemaName();
-    boolean that_present_schemaName = true && that.isSetSchemaName();
-    if (this_present_schemaName || that_present_schemaName) {
-      if (!(this_present_schemaName && that_present_schemaName))
-        return false;
-      if (!this.schemaName.equals(that.schemaName))
-        return false;
-    }
-
-    boolean this_present_tableName = true && this.isSetTableName();
-    boolean that_present_tableName = true && that.isSetTableName();
-    if (this_present_tableName || that_present_tableName) {
-      if (!(this_present_tableName && that_present_tableName))
-        return false;
-      if (!this.tableName.equals(that.tableName))
-        return false;
-    }
-
-    boolean this_present_tableTypes = true && this.isSetTableTypes();
-    boolean that_present_tableTypes = true && that.isSetTableTypes();
-    if (this_present_tableTypes || that_present_tableTypes) {
-      if (!(this_present_tableTypes && that_present_tableTypes))
-        return false;
-      if (!this.tableTypes.equals(that.tableTypes))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    builder.append(present_sessionHandle);
-    if (present_sessionHandle)
-      builder.append(sessionHandle);
-
-    boolean present_catalogName = true && (isSetCatalogName());
-    builder.append(present_catalogName);
-    if (present_catalogName)
-      builder.append(catalogName);
-
-    boolean present_schemaName = true && (isSetSchemaName());
-    builder.append(present_schemaName);
-    if (present_schemaName)
-      builder.append(schemaName);
-
-    boolean present_tableName = true && (isSetTableName());
-    builder.append(present_tableName);
-    if (present_tableName)
-      builder.append(tableName);
-
-    boolean present_tableTypes = true && (isSetTableTypes());
-    builder.append(present_tableTypes);
-    if (present_tableTypes)
-      builder.append(tableTypes);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TGetTablesReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TGetTablesReq typedOther = (TGetTablesReq)other;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(typedOther.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, typedOther.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetCatalogName()).compareTo(typedOther.isSetCatalogName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetCatalogName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.catalogName, typedOther.catalogName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetSchemaName()).compareTo(typedOther.isSetSchemaName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSchemaName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.schemaName, typedOther.schemaName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetTableName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetTableTypes()).compareTo(typedOther.isSetTableTypes());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetTableTypes()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableTypes, typedOther.tableTypes);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetTablesReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    if (isSetCatalogName()) {
-      if (!first) sb.append(", ");
-      sb.append("catalogName:");
-      if (this.catalogName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.catalogName);
-      }
-      first = false;
-    }
-    if (isSetSchemaName()) {
-      if (!first) sb.append(", ");
-      sb.append("schemaName:");
-      if (this.schemaName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.schemaName);
-      }
-      first = false;
-    }
-    if (isSetTableName()) {
-      if (!first) sb.append(", ");
-      sb.append("tableName:");
-      if (this.tableName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.tableName);
-      }
-      first = false;
-    }
-    if (isSetTableTypes()) {
-      if (!first) sb.append(", ");
-      sb.append("tableTypes:");
-      if (this.tableTypes == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.tableTypes);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetTablesReqStandardSchemeFactory implements SchemeFactory {
-    public TGetTablesReqStandardScheme getScheme() {
-      return new TGetTablesReqStandardScheme();
-    }
-  }
-
-  private static class TGetTablesReqStandardScheme extends StandardScheme<TGetTablesReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetTablesReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // CATALOG_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.catalogName = iprot.readString();
-              struct.setCatalogNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // SCHEMA_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.schemaName = iprot.readString();
-              struct.setSchemaNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // TABLE_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.tableName = iprot.readString();
-              struct.setTableNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 5: // TABLE_TYPES
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list172 = iprot.readListBegin();
-                struct.tableTypes = new ArrayList<String>(_list172.size);
-                for (int _i173 = 0; _i173 < _list172.size; ++_i173)
-                {
-                  String _elem174; // optional
-                  _elem174 = iprot.readString();
-                  struct.tableTypes.add(_elem174);
-                }
-                iprot.readListEnd();
-              }
-              struct.setTableTypesIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetTablesReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.catalogName != null) {
-        if (struct.isSetCatalogName()) {
-          oprot.writeFieldBegin(CATALOG_NAME_FIELD_DESC);
-          oprot.writeString(struct.catalogName);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.schemaName != null) {
-        if (struct.isSetSchemaName()) {
-          oprot.writeFieldBegin(SCHEMA_NAME_FIELD_DESC);
-          oprot.writeString(struct.schemaName);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.tableName != null) {
-        if (struct.isSetTableName()) {
-          oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC);
-          oprot.writeString(struct.tableName);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.tableTypes != null) {
-        if (struct.isSetTableTypes()) {
-          oprot.writeFieldBegin(TABLE_TYPES_FIELD_DESC);
-          {
-            oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.tableTypes.size()));
-            for (String _iter175 : struct.tableTypes)
-            {
-              oprot.writeString(_iter175);
-            }
-            oprot.writeListEnd();
-          }
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetTablesReqTupleSchemeFactory implements SchemeFactory {
-    public TGetTablesReqTupleScheme getScheme() {
-      return new TGetTablesReqTupleScheme();
-    }
-  }
-
-  private static class TGetTablesReqTupleScheme extends TupleScheme<TGetTablesReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetTablesReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetCatalogName()) {
-        optionals.set(0);
-      }
-      if (struct.isSetSchemaName()) {
-        optionals.set(1);
-      }
-      if (struct.isSetTableName()) {
-        optionals.set(2);
-      }
-      if (struct.isSetTableTypes()) {
-        optionals.set(3);
-      }
-      oprot.writeBitSet(optionals, 4);
-      if (struct.isSetCatalogName()) {
-        oprot.writeString(struct.catalogName);
-      }
-      if (struct.isSetSchemaName()) {
-        oprot.writeString(struct.schemaName);
-      }
-      if (struct.isSetTableName()) {
-        oprot.writeString(struct.tableName);
-      }
-      if (struct.isSetTableTypes()) {
-        {
-          oprot.writeI32(struct.tableTypes.size());
-          for (String _iter176 : struct.tableTypes)
-          {
-            oprot.writeString(_iter176);
-          }
-        }
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetTablesReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-      BitSet incoming = iprot.readBitSet(4);
-      if (incoming.get(0)) {
-        struct.catalogName = iprot.readString();
-        struct.setCatalogNameIsSet(true);
-      }
-      if (incoming.get(1)) {
-        struct.schemaName = iprot.readString();
-        struct.setSchemaNameIsSet(true);
-      }
-      if (incoming.get(2)) {
-        struct.tableName = iprot.readString();
-        struct.setTableNameIsSet(true);
-      }
-      if (incoming.get(3)) {
-        {
-          org.apache.thrift.protocol.TList _list177 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-          struct.tableTypes = new ArrayList<String>(_list177.size);
-          for (int _i178 = 0; _i178 < _list177.size; ++_i178)
-          {
-            String _elem179; // optional
-            _elem179 = iprot.readString();
-            struct.tableTypes.add(_elem179);
-          }
-        }
-        struct.setTableTypesIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetTablesResp.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetTablesResp.java
deleted file mode 100644
index d526f4478a24e..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetTablesResp.java
+++ /dev/null
@@ -1,505 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TGetTablesResp implements org.apache.thrift.TBase<TGetTablesResp, TGetTablesResp._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetTablesResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetTablesRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetTablesRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TOperationHandle operationHandle; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    OPERATION_HANDLE((short)2, "operationHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private _Fields optionals[] = {_Fields.OPERATION_HANDLE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetTablesResp.class, metaDataMap);
-  }
-
-  public TGetTablesResp() {
-  }
-
-  public TGetTablesResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetTablesResp(TGetTablesResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-  }
-
-  public TGetTablesResp deepCopy() {
-    return new TGetTablesResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.operationHandle = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetTablesResp)
-      return this.equals((TGetTablesResp)that);
-    return false;
-  }
-
-  public boolean equals(TGetTablesResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_status = true && (isSetStatus());
-    builder.append(present_status);
-    if (present_status)
-      builder.append(status);
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    builder.append(present_operationHandle);
-    if (present_operationHandle)
-      builder.append(operationHandle);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TGetTablesResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TGetTablesResp typedOther = (TGetTablesResp)other;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(typedOther.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, typedOther.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(typedOther.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, typedOther.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetTablesResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetOperationHandle()) {
-      if (!first) sb.append(", ");
-      sb.append("operationHandle:");
-      if (this.operationHandle == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.operationHandle);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetTablesRespStandardSchemeFactory implements SchemeFactory {
-    public TGetTablesRespStandardScheme getScheme() {
-      return new TGetTablesRespStandardScheme();
-    }
-  }
-
-  private static class TGetTablesRespStandardScheme extends StandardScheme<TGetTablesResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetTablesResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetTablesResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.operationHandle != null) {
-        if (struct.isSetOperationHandle()) {
-          oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-          struct.operationHandle.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetTablesRespTupleSchemeFactory implements SchemeFactory {
-    public TGetTablesRespTupleScheme getScheme() {
-      return new TGetTablesRespTupleScheme();
-    }
-  }
-
-  private static class TGetTablesRespTupleScheme extends TupleScheme<TGetTablesResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetTablesResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetOperationHandle()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetOperationHandle()) {
-        struct.operationHandle.write(oprot);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetTablesResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.operationHandle = new TOperationHandle();
-        struct.operationHandle.read(iprot);
-        struct.setOperationHandleIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetTypeInfoReq.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetTypeInfoReq.java
deleted file mode 100644
index d40115e83ec45..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetTypeInfoReq.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TGetTypeInfoReq implements org.apache.thrift.TBase<TGetTypeInfoReq, TGetTypeInfoReq._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetTypeInfoReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetTypeInfoReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetTypeInfoReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetTypeInfoReq.class, metaDataMap);
-  }
-
-  public TGetTypeInfoReq() {
-  }
-
-  public TGetTypeInfoReq(
-    TSessionHandle sessionHandle)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetTypeInfoReq(TGetTypeInfoReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-  }
-
-  public TGetTypeInfoReq deepCopy() {
-    return new TGetTypeInfoReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetTypeInfoReq)
-      return this.equals((TGetTypeInfoReq)that);
-    return false;
-  }
-
-  public boolean equals(TGetTypeInfoReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    builder.append(present_sessionHandle);
-    if (present_sessionHandle)
-      builder.append(sessionHandle);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TGetTypeInfoReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TGetTypeInfoReq typedOther = (TGetTypeInfoReq)other;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(typedOther.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, typedOther.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetTypeInfoReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetTypeInfoReqStandardSchemeFactory implements SchemeFactory {
-    public TGetTypeInfoReqStandardScheme getScheme() {
-      return new TGetTypeInfoReqStandardScheme();
-    }
-  }
-
-  private static class TGetTypeInfoReqStandardScheme extends StandardScheme<TGetTypeInfoReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetTypeInfoReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetTypeInfoReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetTypeInfoReqTupleSchemeFactory implements SchemeFactory {
-    public TGetTypeInfoReqTupleScheme getScheme() {
-      return new TGetTypeInfoReqTupleScheme();
-    }
-  }
-
-  private static class TGetTypeInfoReqTupleScheme extends TupleScheme<TGetTypeInfoReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetTypeInfoReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetTypeInfoReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetTypeInfoResp.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetTypeInfoResp.java
deleted file mode 100644
index 59be1a33b55e2..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetTypeInfoResp.java
+++ /dev/null
@@ -1,505 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TGetTypeInfoResp implements org.apache.thrift.TBase<TGetTypeInfoResp, TGetTypeInfoResp._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetTypeInfoResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetTypeInfoRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetTypeInfoRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TOperationHandle operationHandle; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    OPERATION_HANDLE((short)2, "operationHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private _Fields optionals[] = {_Fields.OPERATION_HANDLE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetTypeInfoResp.class, metaDataMap);
-  }
-
-  public TGetTypeInfoResp() {
-  }
-
-  public TGetTypeInfoResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetTypeInfoResp(TGetTypeInfoResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-  }
-
-  public TGetTypeInfoResp deepCopy() {
-    return new TGetTypeInfoResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.operationHandle = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetTypeInfoResp)
-      return this.equals((TGetTypeInfoResp)that);
-    return false;
-  }
-
-  public boolean equals(TGetTypeInfoResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_status = true && (isSetStatus());
-    builder.append(present_status);
-    if (present_status)
-      builder.append(status);
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    builder.append(present_operationHandle);
-    if (present_operationHandle)
-      builder.append(operationHandle);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TGetTypeInfoResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TGetTypeInfoResp typedOther = (TGetTypeInfoResp)other;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(typedOther.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, typedOther.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(typedOther.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, typedOther.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetTypeInfoResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetOperationHandle()) {
-      if (!first) sb.append(", ");
-      sb.append("operationHandle:");
-      if (this.operationHandle == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.operationHandle);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetTypeInfoRespStandardSchemeFactory implements SchemeFactory {
-    public TGetTypeInfoRespStandardScheme getScheme() {
-      return new TGetTypeInfoRespStandardScheme();
-    }
-  }
-
-  private static class TGetTypeInfoRespStandardScheme extends StandardScheme<TGetTypeInfoResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetTypeInfoResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetTypeInfoResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.operationHandle != null) {
-        if (struct.isSetOperationHandle()) {
-          oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-          struct.operationHandle.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetTypeInfoRespTupleSchemeFactory implements SchemeFactory {
-    public TGetTypeInfoRespTupleScheme getScheme() {
-      return new TGetTypeInfoRespTupleScheme();
-    }
-  }
-
-  private static class TGetTypeInfoRespTupleScheme extends TupleScheme<TGetTypeInfoResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetTypeInfoResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetOperationHandle()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetOperationHandle()) {
-        struct.operationHandle.write(oprot);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetTypeInfoResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.operationHandle = new TOperationHandle();
-        struct.operationHandle.read(iprot);
-        struct.setOperationHandleIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/THandleIdentifier.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/THandleIdentifier.java
deleted file mode 100644
index 368273c341c7b..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/THandleIdentifier.java
+++ /dev/null
@@ -1,506 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class THandleIdentifier implements org.apache.thrift.TBase<THandleIdentifier, THandleIdentifier._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("THandleIdentifier");
-
-  private static final org.apache.thrift.protocol.TField GUID_FIELD_DESC = new org.apache.thrift.protocol.TField("guid", org.apache.thrift.protocol.TType.STRING, (short)1);
-  private static final org.apache.thrift.protocol.TField SECRET_FIELD_DESC = new org.apache.thrift.protocol.TField("secret", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new THandleIdentifierStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new THandleIdentifierTupleSchemeFactory());
-  }
-
-  private ByteBuffer guid; // required
-  private ByteBuffer secret; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    GUID((short)1, "guid"),
-    SECRET((short)2, "secret");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // GUID
-          return GUID;
-        case 2: // SECRET
-          return SECRET;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.GUID, new org.apache.thrift.meta_data.FieldMetaData("guid", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
-    tmpMap.put(_Fields.SECRET, new org.apache.thrift.meta_data.FieldMetaData("secret", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(THandleIdentifier.class, metaDataMap);
-  }
-
-  public THandleIdentifier() {
-  }
-
-  public THandleIdentifier(
-    ByteBuffer guid,
-    ByteBuffer secret)
-  {
-    this();
-    this.guid = guid;
-    this.secret = secret;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public THandleIdentifier(THandleIdentifier other) {
-    if (other.isSetGuid()) {
-      this.guid = org.apache.thrift.TBaseHelper.copyBinary(other.guid);
-;
-    }
-    if (other.isSetSecret()) {
-      this.secret = org.apache.thrift.TBaseHelper.copyBinary(other.secret);
-;
-    }
-  }
-
-  public THandleIdentifier deepCopy() {
-    return new THandleIdentifier(this);
-  }
-
-  @Override
-  public void clear() {
-    this.guid = null;
-    this.secret = null;
-  }
-
-  public byte[] getGuid() {
-    setGuid(org.apache.thrift.TBaseHelper.rightSize(guid));
-    return guid == null ? null : guid.array();
-  }
-
-  public ByteBuffer bufferForGuid() {
-    return guid;
-  }
-
-  public void setGuid(byte[] guid) {
-    setGuid(guid == null ? (ByteBuffer)null : ByteBuffer.wrap(guid));
-  }
-
-  public void setGuid(ByteBuffer guid) {
-    this.guid = guid;
-  }
-
-  public void unsetGuid() {
-    this.guid = null;
-  }
-
-  /** Returns true if field guid is set (has been assigned a value) and false otherwise */
-  public boolean isSetGuid() {
-    return this.guid != null;
-  }
-
-  public void setGuidIsSet(boolean value) {
-    if (!value) {
-      this.guid = null;
-    }
-  }
-
-  public byte[] getSecret() {
-    setSecret(org.apache.thrift.TBaseHelper.rightSize(secret));
-    return secret == null ? null : secret.array();
-  }
-
-  public ByteBuffer bufferForSecret() {
-    return secret;
-  }
-
-  public void setSecret(byte[] secret) {
-    setSecret(secret == null ? (ByteBuffer)null : ByteBuffer.wrap(secret));
-  }
-
-  public void setSecret(ByteBuffer secret) {
-    this.secret = secret;
-  }
-
-  public void unsetSecret() {
-    this.secret = null;
-  }
-
-  /** Returns true if field secret is set (has been assigned a value) and false otherwise */
-  public boolean isSetSecret() {
-    return this.secret != null;
-  }
-
-  public void setSecretIsSet(boolean value) {
-    if (!value) {
-      this.secret = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case GUID:
-      if (value == null) {
-        unsetGuid();
-      } else {
-        setGuid((ByteBuffer)value);
-      }
-      break;
-
-    case SECRET:
-      if (value == null) {
-        unsetSecret();
-      } else {
-        setSecret((ByteBuffer)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case GUID:
-      return getGuid();
-
-    case SECRET:
-      return getSecret();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case GUID:
-      return isSetGuid();
-    case SECRET:
-      return isSetSecret();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof THandleIdentifier)
-      return this.equals((THandleIdentifier)that);
-    return false;
-  }
-
-  public boolean equals(THandleIdentifier that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_guid = true && this.isSetGuid();
-    boolean that_present_guid = true && that.isSetGuid();
-    if (this_present_guid || that_present_guid) {
-      if (!(this_present_guid && that_present_guid))
-        return false;
-      if (!this.guid.equals(that.guid))
-        return false;
-    }
-
-    boolean this_present_secret = true && this.isSetSecret();
-    boolean that_present_secret = true && that.isSetSecret();
-    if (this_present_secret || that_present_secret) {
-      if (!(this_present_secret && that_present_secret))
-        return false;
-      if (!this.secret.equals(that.secret))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_guid = true && (isSetGuid());
-    builder.append(present_guid);
-    if (present_guid)
-      builder.append(guid);
-
-    boolean present_secret = true && (isSetSecret());
-    builder.append(present_secret);
-    if (present_secret)
-      builder.append(secret);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(THandleIdentifier other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    THandleIdentifier typedOther = (THandleIdentifier)other;
-
-    lastComparison = Boolean.valueOf(isSetGuid()).compareTo(typedOther.isSetGuid());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetGuid()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.guid, typedOther.guid);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetSecret()).compareTo(typedOther.isSetSecret());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSecret()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.secret, typedOther.secret);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("THandleIdentifier(");
-    boolean first = true;
-
-    sb.append("guid:");
-    if (this.guid == null) {
-      sb.append("null");
-    } else {
-      org.apache.thrift.TBaseHelper.toString(this.guid, sb);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("secret:");
-    if (this.secret == null) {
-      sb.append("null");
-    } else {
-      org.apache.thrift.TBaseHelper.toString(this.secret, sb);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetGuid()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'guid' is unset! Struct:" + toString());
-    }
-
-    if (!isSetSecret()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'secret' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class THandleIdentifierStandardSchemeFactory implements SchemeFactory {
-    public THandleIdentifierStandardScheme getScheme() {
-      return new THandleIdentifierStandardScheme();
-    }
-  }
-
-  private static class THandleIdentifierStandardScheme extends StandardScheme<THandleIdentifier> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, THandleIdentifier struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // GUID
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.guid = iprot.readBinary();
-              struct.setGuidIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // SECRET
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.secret = iprot.readBinary();
-              struct.setSecretIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, THandleIdentifier struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.guid != null) {
-        oprot.writeFieldBegin(GUID_FIELD_DESC);
-        oprot.writeBinary(struct.guid);
-        oprot.writeFieldEnd();
-      }
-      if (struct.secret != null) {
-        oprot.writeFieldBegin(SECRET_FIELD_DESC);
-        oprot.writeBinary(struct.secret);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class THandleIdentifierTupleSchemeFactory implements SchemeFactory {
-    public THandleIdentifierTupleScheme getScheme() {
-      return new THandleIdentifierTupleScheme();
-    }
-  }
-
-  private static class THandleIdentifierTupleScheme extends TupleScheme<THandleIdentifier> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, THandleIdentifier struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      oprot.writeBinary(struct.guid);
-      oprot.writeBinary(struct.secret);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, THandleIdentifier struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.guid = iprot.readBinary();
-      struct.setGuidIsSet(true);
-      struct.secret = iprot.readBinary();
-      struct.setSecretIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TI16Column.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TI16Column.java
deleted file mode 100644
index c83663072f877..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TI16Column.java
+++ /dev/null
@@ -1,548 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TI16Column implements org.apache.thrift.TBase<TI16Column, TI16Column._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TI16Column");
-
-  private static final org.apache.thrift.protocol.TField VALUES_FIELD_DESC = new org.apache.thrift.protocol.TField("values", org.apache.thrift.protocol.TType.LIST, (short)1);
-  private static final org.apache.thrift.protocol.TField NULLS_FIELD_DESC = new org.apache.thrift.protocol.TField("nulls", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TI16ColumnStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TI16ColumnTupleSchemeFactory());
-  }
-
-  private List<Short> values; // required
-  private ByteBuffer nulls; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUES((short)1, "values"),
-    NULLS((short)2, "nulls");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUES
-          return VALUES;
-        case 2: // NULLS
-          return NULLS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUES, new org.apache.thrift.meta_data.FieldMetaData("values", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I16))));
-    tmpMap.put(_Fields.NULLS, new org.apache.thrift.meta_data.FieldMetaData("nulls", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TI16Column.class, metaDataMap);
-  }
-
-  public TI16Column() {
-  }
-
-  public TI16Column(
-    List<Short> values,
-    ByteBuffer nulls)
-  {
-    this();
-    this.values = values;
-    this.nulls = nulls;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TI16Column(TI16Column other) {
-    if (other.isSetValues()) {
-      List<Short> __this__values = new ArrayList<Short>();
-      for (Short other_element : other.values) {
-        __this__values.add(other_element);
-      }
-      this.values = __this__values;
-    }
-    if (other.isSetNulls()) {
-      this.nulls = org.apache.thrift.TBaseHelper.copyBinary(other.nulls);
-;
-    }
-  }
-
-  public TI16Column deepCopy() {
-    return new TI16Column(this);
-  }
-
-  @Override
-  public void clear() {
-    this.values = null;
-    this.nulls = null;
-  }
-
-  public int getValuesSize() {
-    return (this.values == null) ? 0 : this.values.size();
-  }
-
-  public java.util.Iterator<Short> getValuesIterator() {
-    return (this.values == null) ? null : this.values.iterator();
-  }
-
-  public void addToValues(short elem) {
-    if (this.values == null) {
-      this.values = new ArrayList<Short>();
-    }
-    this.values.add(elem);
-  }
-
-  public List<Short> getValues() {
-    return this.values;
-  }
-
-  public void setValues(List<Short> values) {
-    this.values = values;
-  }
-
-  public void unsetValues() {
-    this.values = null;
-  }
-
-  /** Returns true if field values is set (has been assigned a value) and false otherwise */
-  public boolean isSetValues() {
-    return this.values != null;
-  }
-
-  public void setValuesIsSet(boolean value) {
-    if (!value) {
-      this.values = null;
-    }
-  }
-
-  public byte[] getNulls() {
-    setNulls(org.apache.thrift.TBaseHelper.rightSize(nulls));
-    return nulls == null ? null : nulls.array();
-  }
-
-  public ByteBuffer bufferForNulls() {
-    return nulls;
-  }
-
-  public void setNulls(byte[] nulls) {
-    setNulls(nulls == null ? (ByteBuffer)null : ByteBuffer.wrap(nulls));
-  }
-
-  public void setNulls(ByteBuffer nulls) {
-    this.nulls = nulls;
-  }
-
-  public void unsetNulls() {
-    this.nulls = null;
-  }
-
-  /** Returns true if field nulls is set (has been assigned a value) and false otherwise */
-  public boolean isSetNulls() {
-    return this.nulls != null;
-  }
-
-  public void setNullsIsSet(boolean value) {
-    if (!value) {
-      this.nulls = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUES:
-      if (value == null) {
-        unsetValues();
-      } else {
-        setValues((List<Short>)value);
-      }
-      break;
-
-    case NULLS:
-      if (value == null) {
-        unsetNulls();
-      } else {
-        setNulls((ByteBuffer)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUES:
-      return getValues();
-
-    case NULLS:
-      return getNulls();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUES:
-      return isSetValues();
-    case NULLS:
-      return isSetNulls();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TI16Column)
-      return this.equals((TI16Column)that);
-    return false;
-  }
-
-  public boolean equals(TI16Column that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_values = true && this.isSetValues();
-    boolean that_present_values = true && that.isSetValues();
-    if (this_present_values || that_present_values) {
-      if (!(this_present_values && that_present_values))
-        return false;
-      if (!this.values.equals(that.values))
-        return false;
-    }
-
-    boolean this_present_nulls = true && this.isSetNulls();
-    boolean that_present_nulls = true && that.isSetNulls();
-    if (this_present_nulls || that_present_nulls) {
-      if (!(this_present_nulls && that_present_nulls))
-        return false;
-      if (!this.nulls.equals(that.nulls))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_values = true && (isSetValues());
-    builder.append(present_values);
-    if (present_values)
-      builder.append(values);
-
-    boolean present_nulls = true && (isSetNulls());
-    builder.append(present_nulls);
-    if (present_nulls)
-      builder.append(nulls);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TI16Column other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TI16Column typedOther = (TI16Column)other;
-
-    lastComparison = Boolean.valueOf(isSetValues()).compareTo(typedOther.isSetValues());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValues()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.values, typedOther.values);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetNulls()).compareTo(typedOther.isSetNulls());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetNulls()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.nulls, typedOther.nulls);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TI16Column(");
-    boolean first = true;
-
-    sb.append("values:");
-    if (this.values == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.values);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("nulls:");
-    if (this.nulls == null) {
-      sb.append("null");
-    } else {
-      org.apache.thrift.TBaseHelper.toString(this.nulls, sb);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetValues()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'values' is unset! Struct:" + toString());
-    }
-
-    if (!isSetNulls()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'nulls' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TI16ColumnStandardSchemeFactory implements SchemeFactory {
-    public TI16ColumnStandardScheme getScheme() {
-      return new TI16ColumnStandardScheme();
-    }
-  }
-
-  private static class TI16ColumnStandardScheme extends StandardScheme<TI16Column> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TI16Column struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUES
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list70 = iprot.readListBegin();
-                struct.values = new ArrayList<Short>(_list70.size);
-                for (int _i71 = 0; _i71 < _list70.size; ++_i71)
-                {
-                  short _elem72; // optional
-                  _elem72 = iprot.readI16();
-                  struct.values.add(_elem72);
-                }
-                iprot.readListEnd();
-              }
-              struct.setValuesIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // NULLS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.nulls = iprot.readBinary();
-              struct.setNullsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TI16Column struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.values != null) {
-        oprot.writeFieldBegin(VALUES_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I16, struct.values.size()));
-          for (short _iter73 : struct.values)
-          {
-            oprot.writeI16(_iter73);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      if (struct.nulls != null) {
-        oprot.writeFieldBegin(NULLS_FIELD_DESC);
-        oprot.writeBinary(struct.nulls);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TI16ColumnTupleSchemeFactory implements SchemeFactory {
-    public TI16ColumnTupleScheme getScheme() {
-      return new TI16ColumnTupleScheme();
-    }
-  }
-
-  private static class TI16ColumnTupleScheme extends TupleScheme<TI16Column> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TI16Column struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.values.size());
-        for (short _iter74 : struct.values)
-        {
-          oprot.writeI16(_iter74);
-        }
-      }
-      oprot.writeBinary(struct.nulls);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TI16Column struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TList _list75 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I16, iprot.readI32());
-        struct.values = new ArrayList<Short>(_list75.size);
-        for (int _i76 = 0; _i76 < _list75.size; ++_i76)
-        {
-          short _elem77; // optional
-          _elem77 = iprot.readI16();
-          struct.values.add(_elem77);
-        }
-      }
-      struct.setValuesIsSet(true);
-      struct.nulls = iprot.readBinary();
-      struct.setNullsIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TI16Value.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TI16Value.java
deleted file mode 100644
index bb5ae9609de86..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TI16Value.java
+++ /dev/null
@@ -1,386 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TI16Value implements org.apache.thrift.TBase<TI16Value, TI16Value._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TI16Value");
-
-  private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.I16, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TI16ValueStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TI16ValueTupleSchemeFactory());
-  }
-
-  private short value; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUE((short)1, "value");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUE
-          return VALUE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __VALUE_ISSET_ID = 0;
-  private byte __isset_bitfield = 0;
-  private _Fields optionals[] = {_Fields.VALUE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I16)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TI16Value.class, metaDataMap);
-  }
-
-  public TI16Value() {
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TI16Value(TI16Value other) {
-    __isset_bitfield = other.__isset_bitfield;
-    this.value = other.value;
-  }
-
-  public TI16Value deepCopy() {
-    return new TI16Value(this);
-  }
-
-  @Override
-  public void clear() {
-    setValueIsSet(false);
-    this.value = 0;
-  }
-
-  public short getValue() {
-    return this.value;
-  }
-
-  public void setValue(short value) {
-    this.value = value;
-    setValueIsSet(true);
-  }
-
-  public void unsetValue() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __VALUE_ISSET_ID);
-  }
-
-  /** Returns true if field value is set (has been assigned a value) and false otherwise */
-  public boolean isSetValue() {
-    return EncodingUtils.testBit(__isset_bitfield, __VALUE_ISSET_ID);
-  }
-
-  public void setValueIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __VALUE_ISSET_ID, value);
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUE:
-      if (value == null) {
-        unsetValue();
-      } else {
-        setValue((Short)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUE:
-      return Short.valueOf(getValue());
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUE:
-      return isSetValue();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TI16Value)
-      return this.equals((TI16Value)that);
-    return false;
-  }
-
-  public boolean equals(TI16Value that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_value = true && this.isSetValue();
-    boolean that_present_value = true && that.isSetValue();
-    if (this_present_value || that_present_value) {
-      if (!(this_present_value && that_present_value))
-        return false;
-      if (this.value != that.value)
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_value = true && (isSetValue());
-    builder.append(present_value);
-    if (present_value)
-      builder.append(value);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TI16Value other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TI16Value typedOther = (TI16Value)other;
-
-    lastComparison = Boolean.valueOf(isSetValue()).compareTo(typedOther.isSetValue());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValue()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, typedOther.value);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TI16Value(");
-    boolean first = true;
-
-    if (isSetValue()) {
-      sb.append("value:");
-      sb.append(this.value);
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TI16ValueStandardSchemeFactory implements SchemeFactory {
-    public TI16ValueStandardScheme getScheme() {
-      return new TI16ValueStandardScheme();
-    }
-  }
-
-  private static class TI16ValueStandardScheme extends StandardScheme<TI16Value> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TI16Value struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUE
-            if (schemeField.type == org.apache.thrift.protocol.TType.I16) {
-              struct.value = iprot.readI16();
-              struct.setValueIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TI16Value struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.isSetValue()) {
-        oprot.writeFieldBegin(VALUE_FIELD_DESC);
-        oprot.writeI16(struct.value);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TI16ValueTupleSchemeFactory implements SchemeFactory {
-    public TI16ValueTupleScheme getScheme() {
-      return new TI16ValueTupleScheme();
-    }
-  }
-
-  private static class TI16ValueTupleScheme extends TupleScheme<TI16Value> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TI16Value struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      BitSet optionals = new BitSet();
-      if (struct.isSetValue()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetValue()) {
-        oprot.writeI16(struct.value);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TI16Value struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.value = iprot.readI16();
-        struct.setValueIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TI32Column.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TI32Column.java
deleted file mode 100644
index 6c6c5f35b7c8e..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TI32Column.java
+++ /dev/null
@@ -1,548 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TI32Column implements org.apache.thrift.TBase<TI32Column, TI32Column._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TI32Column");
-
-  private static final org.apache.thrift.protocol.TField VALUES_FIELD_DESC = new org.apache.thrift.protocol.TField("values", org.apache.thrift.protocol.TType.LIST, (short)1);
-  private static final org.apache.thrift.protocol.TField NULLS_FIELD_DESC = new org.apache.thrift.protocol.TField("nulls", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TI32ColumnStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TI32ColumnTupleSchemeFactory());
-  }
-
-  private List<Integer> values; // required
-  private ByteBuffer nulls; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUES((short)1, "values"),
-    NULLS((short)2, "nulls");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUES
-          return VALUES;
-        case 2: // NULLS
-          return NULLS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUES, new org.apache.thrift.meta_data.FieldMetaData("values", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))));
-    tmpMap.put(_Fields.NULLS, new org.apache.thrift.meta_data.FieldMetaData("nulls", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TI32Column.class, metaDataMap);
-  }
-
-  public TI32Column() {
-  }
-
-  public TI32Column(
-    List<Integer> values,
-    ByteBuffer nulls)
-  {
-    this();
-    this.values = values;
-    this.nulls = nulls;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TI32Column(TI32Column other) {
-    if (other.isSetValues()) {
-      List<Integer> __this__values = new ArrayList<Integer>();
-      for (Integer other_element : other.values) {
-        __this__values.add(other_element);
-      }
-      this.values = __this__values;
-    }
-    if (other.isSetNulls()) {
-      this.nulls = org.apache.thrift.TBaseHelper.copyBinary(other.nulls);
-;
-    }
-  }
-
-  public TI32Column deepCopy() {
-    return new TI32Column(this);
-  }
-
-  @Override
-  public void clear() {
-    this.values = null;
-    this.nulls = null;
-  }
-
-  public int getValuesSize() {
-    return (this.values == null) ? 0 : this.values.size();
-  }
-
-  public java.util.Iterator<Integer> getValuesIterator() {
-    return (this.values == null) ? null : this.values.iterator();
-  }
-
-  public void addToValues(int elem) {
-    if (this.values == null) {
-      this.values = new ArrayList<Integer>();
-    }
-    this.values.add(elem);
-  }
-
-  public List<Integer> getValues() {
-    return this.values;
-  }
-
-  public void setValues(List<Integer> values) {
-    this.values = values;
-  }
-
-  public void unsetValues() {
-    this.values = null;
-  }
-
-  /** Returns true if field values is set (has been assigned a value) and false otherwise */
-  public boolean isSetValues() {
-    return this.values != null;
-  }
-
-  public void setValuesIsSet(boolean value) {
-    if (!value) {
-      this.values = null;
-    }
-  }
-
-  public byte[] getNulls() {
-    setNulls(org.apache.thrift.TBaseHelper.rightSize(nulls));
-    return nulls == null ? null : nulls.array();
-  }
-
-  public ByteBuffer bufferForNulls() {
-    return nulls;
-  }
-
-  public void setNulls(byte[] nulls) {
-    setNulls(nulls == null ? (ByteBuffer)null : ByteBuffer.wrap(nulls));
-  }
-
-  public void setNulls(ByteBuffer nulls) {
-    this.nulls = nulls;
-  }
-
-  public void unsetNulls() {
-    this.nulls = null;
-  }
-
-  /** Returns true if field nulls is set (has been assigned a value) and false otherwise */
-  public boolean isSetNulls() {
-    return this.nulls != null;
-  }
-
-  public void setNullsIsSet(boolean value) {
-    if (!value) {
-      this.nulls = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUES:
-      if (value == null) {
-        unsetValues();
-      } else {
-        setValues((List<Integer>)value);
-      }
-      break;
-
-    case NULLS:
-      if (value == null) {
-        unsetNulls();
-      } else {
-        setNulls((ByteBuffer)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUES:
-      return getValues();
-
-    case NULLS:
-      return getNulls();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUES:
-      return isSetValues();
-    case NULLS:
-      return isSetNulls();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TI32Column)
-      return this.equals((TI32Column)that);
-    return false;
-  }
-
-  public boolean equals(TI32Column that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_values = true && this.isSetValues();
-    boolean that_present_values = true && that.isSetValues();
-    if (this_present_values || that_present_values) {
-      if (!(this_present_values && that_present_values))
-        return false;
-      if (!this.values.equals(that.values))
-        return false;
-    }
-
-    boolean this_present_nulls = true && this.isSetNulls();
-    boolean that_present_nulls = true && that.isSetNulls();
-    if (this_present_nulls || that_present_nulls) {
-      if (!(this_present_nulls && that_present_nulls))
-        return false;
-      if (!this.nulls.equals(that.nulls))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_values = true && (isSetValues());
-    builder.append(present_values);
-    if (present_values)
-      builder.append(values);
-
-    boolean present_nulls = true && (isSetNulls());
-    builder.append(present_nulls);
-    if (present_nulls)
-      builder.append(nulls);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TI32Column other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TI32Column typedOther = (TI32Column)other;
-
-    lastComparison = Boolean.valueOf(isSetValues()).compareTo(typedOther.isSetValues());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValues()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.values, typedOther.values);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetNulls()).compareTo(typedOther.isSetNulls());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetNulls()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.nulls, typedOther.nulls);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TI32Column(");
-    boolean first = true;
-
-    sb.append("values:");
-    if (this.values == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.values);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("nulls:");
-    if (this.nulls == null) {
-      sb.append("null");
-    } else {
-      org.apache.thrift.TBaseHelper.toString(this.nulls, sb);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetValues()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'values' is unset! Struct:" + toString());
-    }
-
-    if (!isSetNulls()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'nulls' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TI32ColumnStandardSchemeFactory implements SchemeFactory {
-    public TI32ColumnStandardScheme getScheme() {
-      return new TI32ColumnStandardScheme();
-    }
-  }
-
-  private static class TI32ColumnStandardScheme extends StandardScheme<TI32Column> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TI32Column struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUES
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list78 = iprot.readListBegin();
-                struct.values = new ArrayList<Integer>(_list78.size);
-                for (int _i79 = 0; _i79 < _list78.size; ++_i79)
-                {
-                  int _elem80; // optional
-                  _elem80 = iprot.readI32();
-                  struct.values.add(_elem80);
-                }
-                iprot.readListEnd();
-              }
-              struct.setValuesIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // NULLS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.nulls = iprot.readBinary();
-              struct.setNullsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TI32Column struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.values != null) {
-        oprot.writeFieldBegin(VALUES_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I32, struct.values.size()));
-          for (int _iter81 : struct.values)
-          {
-            oprot.writeI32(_iter81);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      if (struct.nulls != null) {
-        oprot.writeFieldBegin(NULLS_FIELD_DESC);
-        oprot.writeBinary(struct.nulls);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TI32ColumnTupleSchemeFactory implements SchemeFactory {
-    public TI32ColumnTupleScheme getScheme() {
-      return new TI32ColumnTupleScheme();
-    }
-  }
-
-  private static class TI32ColumnTupleScheme extends TupleScheme<TI32Column> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TI32Column struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.values.size());
-        for (int _iter82 : struct.values)
-        {
-          oprot.writeI32(_iter82);
-        }
-      }
-      oprot.writeBinary(struct.nulls);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TI32Column struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TList _list83 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I32, iprot.readI32());
-        struct.values = new ArrayList<Integer>(_list83.size);
-        for (int _i84 = 0; _i84 < _list83.size; ++_i84)
-        {
-          int _elem85; // optional
-          _elem85 = iprot.readI32();
-          struct.values.add(_elem85);
-        }
-      }
-      struct.setValuesIsSet(true);
-      struct.nulls = iprot.readBinary();
-      struct.setNullsIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TI32Value.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TI32Value.java
deleted file mode 100644
index 059408b96c8ce..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TI32Value.java
+++ /dev/null
@@ -1,386 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TI32Value implements org.apache.thrift.TBase<TI32Value, TI32Value._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TI32Value");
-
-  private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.I32, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TI32ValueStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TI32ValueTupleSchemeFactory());
-  }
-
-  private int value; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUE((short)1, "value");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUE
-          return VALUE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __VALUE_ISSET_ID = 0;
-  private byte __isset_bitfield = 0;
-  private _Fields optionals[] = {_Fields.VALUE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TI32Value.class, metaDataMap);
-  }
-
-  public TI32Value() {
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TI32Value(TI32Value other) {
-    __isset_bitfield = other.__isset_bitfield;
-    this.value = other.value;
-  }
-
-  public TI32Value deepCopy() {
-    return new TI32Value(this);
-  }
-
-  @Override
-  public void clear() {
-    setValueIsSet(false);
-    this.value = 0;
-  }
-
-  public int getValue() {
-    return this.value;
-  }
-
-  public void setValue(int value) {
-    this.value = value;
-    setValueIsSet(true);
-  }
-
-  public void unsetValue() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __VALUE_ISSET_ID);
-  }
-
-  /** Returns true if field value is set (has been assigned a value) and false otherwise */
-  public boolean isSetValue() {
-    return EncodingUtils.testBit(__isset_bitfield, __VALUE_ISSET_ID);
-  }
-
-  public void setValueIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __VALUE_ISSET_ID, value);
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUE:
-      if (value == null) {
-        unsetValue();
-      } else {
-        setValue((Integer)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUE:
-      return Integer.valueOf(getValue());
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUE:
-      return isSetValue();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TI32Value)
-      return this.equals((TI32Value)that);
-    return false;
-  }
-
-  public boolean equals(TI32Value that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_value = true && this.isSetValue();
-    boolean that_present_value = true && that.isSetValue();
-    if (this_present_value || that_present_value) {
-      if (!(this_present_value && that_present_value))
-        return false;
-      if (this.value != that.value)
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_value = true && (isSetValue());
-    builder.append(present_value);
-    if (present_value)
-      builder.append(value);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TI32Value other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TI32Value typedOther = (TI32Value)other;
-
-    lastComparison = Boolean.valueOf(isSetValue()).compareTo(typedOther.isSetValue());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValue()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, typedOther.value);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TI32Value(");
-    boolean first = true;
-
-    if (isSetValue()) {
-      sb.append("value:");
-      sb.append(this.value);
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TI32ValueStandardSchemeFactory implements SchemeFactory {
-    public TI32ValueStandardScheme getScheme() {
-      return new TI32ValueStandardScheme();
-    }
-  }
-
-  private static class TI32ValueStandardScheme extends StandardScheme<TI32Value> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TI32Value struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUE
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.value = iprot.readI32();
-              struct.setValueIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TI32Value struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.isSetValue()) {
-        oprot.writeFieldBegin(VALUE_FIELD_DESC);
-        oprot.writeI32(struct.value);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TI32ValueTupleSchemeFactory implements SchemeFactory {
-    public TI32ValueTupleScheme getScheme() {
-      return new TI32ValueTupleScheme();
-    }
-  }
-
-  private static class TI32ValueTupleScheme extends TupleScheme<TI32Value> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TI32Value struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      BitSet optionals = new BitSet();
-      if (struct.isSetValue()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetValue()) {
-        oprot.writeI32(struct.value);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TI32Value struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.value = iprot.readI32();
-        struct.setValueIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TI64Column.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TI64Column.java
deleted file mode 100644
index cc383ed089fa4..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TI64Column.java
+++ /dev/null
@@ -1,548 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TI64Column implements org.apache.thrift.TBase<TI64Column, TI64Column._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TI64Column");
-
-  private static final org.apache.thrift.protocol.TField VALUES_FIELD_DESC = new org.apache.thrift.protocol.TField("values", org.apache.thrift.protocol.TType.LIST, (short)1);
-  private static final org.apache.thrift.protocol.TField NULLS_FIELD_DESC = new org.apache.thrift.protocol.TField("nulls", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TI64ColumnStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TI64ColumnTupleSchemeFactory());
-  }
-
-  private List<Long> values; // required
-  private ByteBuffer nulls; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUES((short)1, "values"),
-    NULLS((short)2, "nulls");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUES
-          return VALUES;
-        case 2: // NULLS
-          return NULLS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUES, new org.apache.thrift.meta_data.FieldMetaData("values", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))));
-    tmpMap.put(_Fields.NULLS, new org.apache.thrift.meta_data.FieldMetaData("nulls", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TI64Column.class, metaDataMap);
-  }
-
-  public TI64Column() {
-  }
-
-  public TI64Column(
-    List<Long> values,
-    ByteBuffer nulls)
-  {
-    this();
-    this.values = values;
-    this.nulls = nulls;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TI64Column(TI64Column other) {
-    if (other.isSetValues()) {
-      List<Long> __this__values = new ArrayList<Long>();
-      for (Long other_element : other.values) {
-        __this__values.add(other_element);
-      }
-      this.values = __this__values;
-    }
-    if (other.isSetNulls()) {
-      this.nulls = org.apache.thrift.TBaseHelper.copyBinary(other.nulls);
-;
-    }
-  }
-
-  public TI64Column deepCopy() {
-    return new TI64Column(this);
-  }
-
-  @Override
-  public void clear() {
-    this.values = null;
-    this.nulls = null;
-  }
-
-  public int getValuesSize() {
-    return (this.values == null) ? 0 : this.values.size();
-  }
-
-  public java.util.Iterator<Long> getValuesIterator() {
-    return (this.values == null) ? null : this.values.iterator();
-  }
-
-  public void addToValues(long elem) {
-    if (this.values == null) {
-      this.values = new ArrayList<Long>();
-    }
-    this.values.add(elem);
-  }
-
-  public List<Long> getValues() {
-    return this.values;
-  }
-
-  public void setValues(List<Long> values) {
-    this.values = values;
-  }
-
-  public void unsetValues() {
-    this.values = null;
-  }
-
-  /** Returns true if field values is set (has been assigned a value) and false otherwise */
-  public boolean isSetValues() {
-    return this.values != null;
-  }
-
-  public void setValuesIsSet(boolean value) {
-    if (!value) {
-      this.values = null;
-    }
-  }
-
-  public byte[] getNulls() {
-    setNulls(org.apache.thrift.TBaseHelper.rightSize(nulls));
-    return nulls == null ? null : nulls.array();
-  }
-
-  public ByteBuffer bufferForNulls() {
-    return nulls;
-  }
-
-  public void setNulls(byte[] nulls) {
-    setNulls(nulls == null ? (ByteBuffer)null : ByteBuffer.wrap(nulls));
-  }
-
-  public void setNulls(ByteBuffer nulls) {
-    this.nulls = nulls;
-  }
-
-  public void unsetNulls() {
-    this.nulls = null;
-  }
-
-  /** Returns true if field nulls is set (has been assigned a value) and false otherwise */
-  public boolean isSetNulls() {
-    return this.nulls != null;
-  }
-
-  public void setNullsIsSet(boolean value) {
-    if (!value) {
-      this.nulls = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUES:
-      if (value == null) {
-        unsetValues();
-      } else {
-        setValues((List<Long>)value);
-      }
-      break;
-
-    case NULLS:
-      if (value == null) {
-        unsetNulls();
-      } else {
-        setNulls((ByteBuffer)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUES:
-      return getValues();
-
-    case NULLS:
-      return getNulls();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUES:
-      return isSetValues();
-    case NULLS:
-      return isSetNulls();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TI64Column)
-      return this.equals((TI64Column)that);
-    return false;
-  }
-
-  public boolean equals(TI64Column that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_values = true && this.isSetValues();
-    boolean that_present_values = true && that.isSetValues();
-    if (this_present_values || that_present_values) {
-      if (!(this_present_values && that_present_values))
-        return false;
-      if (!this.values.equals(that.values))
-        return false;
-    }
-
-    boolean this_present_nulls = true && this.isSetNulls();
-    boolean that_present_nulls = true && that.isSetNulls();
-    if (this_present_nulls || that_present_nulls) {
-      if (!(this_present_nulls && that_present_nulls))
-        return false;
-      if (!this.nulls.equals(that.nulls))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_values = true && (isSetValues());
-    builder.append(present_values);
-    if (present_values)
-      builder.append(values);
-
-    boolean present_nulls = true && (isSetNulls());
-    builder.append(present_nulls);
-    if (present_nulls)
-      builder.append(nulls);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TI64Column other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TI64Column typedOther = (TI64Column)other;
-
-    lastComparison = Boolean.valueOf(isSetValues()).compareTo(typedOther.isSetValues());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValues()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.values, typedOther.values);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetNulls()).compareTo(typedOther.isSetNulls());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetNulls()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.nulls, typedOther.nulls);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TI64Column(");
-    boolean first = true;
-
-    sb.append("values:");
-    if (this.values == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.values);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("nulls:");
-    if (this.nulls == null) {
-      sb.append("null");
-    } else {
-      org.apache.thrift.TBaseHelper.toString(this.nulls, sb);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetValues()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'values' is unset! Struct:" + toString());
-    }
-
-    if (!isSetNulls()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'nulls' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TI64ColumnStandardSchemeFactory implements SchemeFactory {
-    public TI64ColumnStandardScheme getScheme() {
-      return new TI64ColumnStandardScheme();
-    }
-  }
-
-  private static class TI64ColumnStandardScheme extends StandardScheme<TI64Column> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TI64Column struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUES
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list86 = iprot.readListBegin();
-                struct.values = new ArrayList<Long>(_list86.size);
-                for (int _i87 = 0; _i87 < _list86.size; ++_i87)
-                {
-                  long _elem88; // optional
-                  _elem88 = iprot.readI64();
-                  struct.values.add(_elem88);
-                }
-                iprot.readListEnd();
-              }
-              struct.setValuesIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // NULLS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.nulls = iprot.readBinary();
-              struct.setNullsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TI64Column struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.values != null) {
-        oprot.writeFieldBegin(VALUES_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I64, struct.values.size()));
-          for (long _iter89 : struct.values)
-          {
-            oprot.writeI64(_iter89);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      if (struct.nulls != null) {
-        oprot.writeFieldBegin(NULLS_FIELD_DESC);
-        oprot.writeBinary(struct.nulls);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TI64ColumnTupleSchemeFactory implements SchemeFactory {
-    public TI64ColumnTupleScheme getScheme() {
-      return new TI64ColumnTupleScheme();
-    }
-  }
-
-  private static class TI64ColumnTupleScheme extends TupleScheme<TI64Column> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TI64Column struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.values.size());
-        for (long _iter90 : struct.values)
-        {
-          oprot.writeI64(_iter90);
-        }
-      }
-      oprot.writeBinary(struct.nulls);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TI64Column struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TList _list91 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I64, iprot.readI32());
-        struct.values = new ArrayList<Long>(_list91.size);
-        for (int _i92 = 0; _i92 < _list91.size; ++_i92)
-        {
-          long _elem93; // optional
-          _elem93 = iprot.readI64();
-          struct.values.add(_elem93);
-        }
-      }
-      struct.setValuesIsSet(true);
-      struct.nulls = iprot.readBinary();
-      struct.setNullsIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TI64Value.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TI64Value.java
deleted file mode 100644
index 9a941cce0c077..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TI64Value.java
+++ /dev/null
@@ -1,386 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TI64Value implements org.apache.thrift.TBase<TI64Value, TI64Value._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TI64Value");
-
-  private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.I64, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TI64ValueStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TI64ValueTupleSchemeFactory());
-  }
-
-  private long value; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUE((short)1, "value");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUE
-          return VALUE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __VALUE_ISSET_ID = 0;
-  private byte __isset_bitfield = 0;
-  private _Fields optionals[] = {_Fields.VALUE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TI64Value.class, metaDataMap);
-  }
-
-  public TI64Value() {
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TI64Value(TI64Value other) {
-    __isset_bitfield = other.__isset_bitfield;
-    this.value = other.value;
-  }
-
-  public TI64Value deepCopy() {
-    return new TI64Value(this);
-  }
-
-  @Override
-  public void clear() {
-    setValueIsSet(false);
-    this.value = 0;
-  }
-
-  public long getValue() {
-    return this.value;
-  }
-
-  public void setValue(long value) {
-    this.value = value;
-    setValueIsSet(true);
-  }
-
-  public void unsetValue() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __VALUE_ISSET_ID);
-  }
-
-  /** Returns true if field value is set (has been assigned a value) and false otherwise */
-  public boolean isSetValue() {
-    return EncodingUtils.testBit(__isset_bitfield, __VALUE_ISSET_ID);
-  }
-
-  public void setValueIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __VALUE_ISSET_ID, value);
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUE:
-      if (value == null) {
-        unsetValue();
-      } else {
-        setValue((Long)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUE:
-      return Long.valueOf(getValue());
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUE:
-      return isSetValue();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TI64Value)
-      return this.equals((TI64Value)that);
-    return false;
-  }
-
-  public boolean equals(TI64Value that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_value = true && this.isSetValue();
-    boolean that_present_value = true && that.isSetValue();
-    if (this_present_value || that_present_value) {
-      if (!(this_present_value && that_present_value))
-        return false;
-      if (this.value != that.value)
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_value = true && (isSetValue());
-    builder.append(present_value);
-    if (present_value)
-      builder.append(value);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TI64Value other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TI64Value typedOther = (TI64Value)other;
-
-    lastComparison = Boolean.valueOf(isSetValue()).compareTo(typedOther.isSetValue());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValue()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, typedOther.value);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TI64Value(");
-    boolean first = true;
-
-    if (isSetValue()) {
-      sb.append("value:");
-      sb.append(this.value);
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TI64ValueStandardSchemeFactory implements SchemeFactory {
-    public TI64ValueStandardScheme getScheme() {
-      return new TI64ValueStandardScheme();
-    }
-  }
-
-  private static class TI64ValueStandardScheme extends StandardScheme<TI64Value> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TI64Value struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUE
-            if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
-              struct.value = iprot.readI64();
-              struct.setValueIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TI64Value struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.isSetValue()) {
-        oprot.writeFieldBegin(VALUE_FIELD_DESC);
-        oprot.writeI64(struct.value);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TI64ValueTupleSchemeFactory implements SchemeFactory {
-    public TI64ValueTupleScheme getScheme() {
-      return new TI64ValueTupleScheme();
-    }
-  }
-
-  private static class TI64ValueTupleScheme extends TupleScheme<TI64Value> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TI64Value struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      BitSet optionals = new BitSet();
-      if (struct.isSetValue()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetValue()) {
-        oprot.writeI64(struct.value);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TI64Value struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.value = iprot.readI64();
-        struct.setValueIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TMapTypeEntry.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TMapTypeEntry.java
deleted file mode 100644
index 425603cbdecbd..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TMapTypeEntry.java
+++ /dev/null
@@ -1,478 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TMapTypeEntry implements org.apache.thrift.TBase<TMapTypeEntry, TMapTypeEntry._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TMapTypeEntry");
-
-  private static final org.apache.thrift.protocol.TField KEY_TYPE_PTR_FIELD_DESC = new org.apache.thrift.protocol.TField("keyTypePtr", org.apache.thrift.protocol.TType.I32, (short)1);
-  private static final org.apache.thrift.protocol.TField VALUE_TYPE_PTR_FIELD_DESC = new org.apache.thrift.protocol.TField("valueTypePtr", org.apache.thrift.protocol.TType.I32, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TMapTypeEntryStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TMapTypeEntryTupleSchemeFactory());
-  }
-
-  private int keyTypePtr; // required
-  private int valueTypePtr; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    KEY_TYPE_PTR((short)1, "keyTypePtr"),
-    VALUE_TYPE_PTR((short)2, "valueTypePtr");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // KEY_TYPE_PTR
-          return KEY_TYPE_PTR;
-        case 2: // VALUE_TYPE_PTR
-          return VALUE_TYPE_PTR;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __KEYTYPEPTR_ISSET_ID = 0;
-  private static final int __VALUETYPEPTR_ISSET_ID = 1;
-  private byte __isset_bitfield = 0;
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.KEY_TYPE_PTR, new org.apache.thrift.meta_data.FieldMetaData("keyTypePtr", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32        , "TTypeEntryPtr")));
-    tmpMap.put(_Fields.VALUE_TYPE_PTR, new org.apache.thrift.meta_data.FieldMetaData("valueTypePtr", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32        , "TTypeEntryPtr")));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TMapTypeEntry.class, metaDataMap);
-  }
-
-  public TMapTypeEntry() {
-  }
-
-  public TMapTypeEntry(
-    int keyTypePtr,
-    int valueTypePtr)
-  {
-    this();
-    this.keyTypePtr = keyTypePtr;
-    setKeyTypePtrIsSet(true);
-    this.valueTypePtr = valueTypePtr;
-    setValueTypePtrIsSet(true);
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TMapTypeEntry(TMapTypeEntry other) {
-    __isset_bitfield = other.__isset_bitfield;
-    this.keyTypePtr = other.keyTypePtr;
-    this.valueTypePtr = other.valueTypePtr;
-  }
-
-  public TMapTypeEntry deepCopy() {
-    return new TMapTypeEntry(this);
-  }
-
-  @Override
-  public void clear() {
-    setKeyTypePtrIsSet(false);
-    this.keyTypePtr = 0;
-    setValueTypePtrIsSet(false);
-    this.valueTypePtr = 0;
-  }
-
-  public int getKeyTypePtr() {
-    return this.keyTypePtr;
-  }
-
-  public void setKeyTypePtr(int keyTypePtr) {
-    this.keyTypePtr = keyTypePtr;
-    setKeyTypePtrIsSet(true);
-  }
-
-  public void unsetKeyTypePtr() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __KEYTYPEPTR_ISSET_ID);
-  }
-
-  /** Returns true if field keyTypePtr is set (has been assigned a value) and false otherwise */
-  public boolean isSetKeyTypePtr() {
-    return EncodingUtils.testBit(__isset_bitfield, __KEYTYPEPTR_ISSET_ID);
-  }
-
-  public void setKeyTypePtrIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __KEYTYPEPTR_ISSET_ID, value);
-  }
-
-  public int getValueTypePtr() {
-    return this.valueTypePtr;
-  }
-
-  public void setValueTypePtr(int valueTypePtr) {
-    this.valueTypePtr = valueTypePtr;
-    setValueTypePtrIsSet(true);
-  }
-
-  public void unsetValueTypePtr() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __VALUETYPEPTR_ISSET_ID);
-  }
-
-  /** Returns true if field valueTypePtr is set (has been assigned a value) and false otherwise */
-  public boolean isSetValueTypePtr() {
-    return EncodingUtils.testBit(__isset_bitfield, __VALUETYPEPTR_ISSET_ID);
-  }
-
-  public void setValueTypePtrIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __VALUETYPEPTR_ISSET_ID, value);
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case KEY_TYPE_PTR:
-      if (value == null) {
-        unsetKeyTypePtr();
-      } else {
-        setKeyTypePtr((Integer)value);
-      }
-      break;
-
-    case VALUE_TYPE_PTR:
-      if (value == null) {
-        unsetValueTypePtr();
-      } else {
-        setValueTypePtr((Integer)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case KEY_TYPE_PTR:
-      return Integer.valueOf(getKeyTypePtr());
-
-    case VALUE_TYPE_PTR:
-      return Integer.valueOf(getValueTypePtr());
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case KEY_TYPE_PTR:
-      return isSetKeyTypePtr();
-    case VALUE_TYPE_PTR:
-      return isSetValueTypePtr();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TMapTypeEntry)
-      return this.equals((TMapTypeEntry)that);
-    return false;
-  }
-
-  public boolean equals(TMapTypeEntry that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_keyTypePtr = true;
-    boolean that_present_keyTypePtr = true;
-    if (this_present_keyTypePtr || that_present_keyTypePtr) {
-      if (!(this_present_keyTypePtr && that_present_keyTypePtr))
-        return false;
-      if (this.keyTypePtr != that.keyTypePtr)
-        return false;
-    }
-
-    boolean this_present_valueTypePtr = true;
-    boolean that_present_valueTypePtr = true;
-    if (this_present_valueTypePtr || that_present_valueTypePtr) {
-      if (!(this_present_valueTypePtr && that_present_valueTypePtr))
-        return false;
-      if (this.valueTypePtr != that.valueTypePtr)
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_keyTypePtr = true;
-    builder.append(present_keyTypePtr);
-    if (present_keyTypePtr)
-      builder.append(keyTypePtr);
-
-    boolean present_valueTypePtr = true;
-    builder.append(present_valueTypePtr);
-    if (present_valueTypePtr)
-      builder.append(valueTypePtr);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TMapTypeEntry other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TMapTypeEntry typedOther = (TMapTypeEntry)other;
-
-    lastComparison = Boolean.valueOf(isSetKeyTypePtr()).compareTo(typedOther.isSetKeyTypePtr());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetKeyTypePtr()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.keyTypePtr, typedOther.keyTypePtr);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetValueTypePtr()).compareTo(typedOther.isSetValueTypePtr());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValueTypePtr()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.valueTypePtr, typedOther.valueTypePtr);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TMapTypeEntry(");
-    boolean first = true;
-
-    sb.append("keyTypePtr:");
-    sb.append(this.keyTypePtr);
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("valueTypePtr:");
-    sb.append(this.valueTypePtr);
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetKeyTypePtr()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'keyTypePtr' is unset! Struct:" + toString());
-    }
-
-    if (!isSetValueTypePtr()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'valueTypePtr' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TMapTypeEntryStandardSchemeFactory implements SchemeFactory {
-    public TMapTypeEntryStandardScheme getScheme() {
-      return new TMapTypeEntryStandardScheme();
-    }
-  }
-
-  private static class TMapTypeEntryStandardScheme extends StandardScheme<TMapTypeEntry> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TMapTypeEntry struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // KEY_TYPE_PTR
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.keyTypePtr = iprot.readI32();
-              struct.setKeyTypePtrIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // VALUE_TYPE_PTR
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.valueTypePtr = iprot.readI32();
-              struct.setValueTypePtrIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TMapTypeEntry struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      oprot.writeFieldBegin(KEY_TYPE_PTR_FIELD_DESC);
-      oprot.writeI32(struct.keyTypePtr);
-      oprot.writeFieldEnd();
-      oprot.writeFieldBegin(VALUE_TYPE_PTR_FIELD_DESC);
-      oprot.writeI32(struct.valueTypePtr);
-      oprot.writeFieldEnd();
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TMapTypeEntryTupleSchemeFactory implements SchemeFactory {
-    public TMapTypeEntryTupleScheme getScheme() {
-      return new TMapTypeEntryTupleScheme();
-    }
-  }
-
-  private static class TMapTypeEntryTupleScheme extends TupleScheme<TMapTypeEntry> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TMapTypeEntry struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      oprot.writeI32(struct.keyTypePtr);
-      oprot.writeI32(struct.valueTypePtr);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TMapTypeEntry struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.keyTypePtr = iprot.readI32();
-      struct.setKeyTypePtrIsSet(true);
-      struct.valueTypePtr = iprot.readI32();
-      struct.setValueTypePtrIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TOpenSessionReq.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TOpenSessionReq.java
deleted file mode 100644
index c0481615b06d3..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TOpenSessionReq.java
+++ /dev/null
@@ -1,785 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TOpenSessionReq implements org.apache.thrift.TBase<TOpenSessionReq, TOpenSessionReq._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TOpenSessionReq");
-
-  private static final org.apache.thrift.protocol.TField CLIENT_PROTOCOL_FIELD_DESC = new org.apache.thrift.protocol.TField("client_protocol", org.apache.thrift.protocol.TType.I32, (short)1);
-  private static final org.apache.thrift.protocol.TField USERNAME_FIELD_DESC = new org.apache.thrift.protocol.TField("username", org.apache.thrift.protocol.TType.STRING, (short)2);
-  private static final org.apache.thrift.protocol.TField PASSWORD_FIELD_DESC = new org.apache.thrift.protocol.TField("password", org.apache.thrift.protocol.TType.STRING, (short)3);
-  private static final org.apache.thrift.protocol.TField CONFIGURATION_FIELD_DESC = new org.apache.thrift.protocol.TField("configuration", org.apache.thrift.protocol.TType.MAP, (short)4);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TOpenSessionReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TOpenSessionReqTupleSchemeFactory());
-  }
-
-  private TProtocolVersion client_protocol; // required
-  private String username; // optional
-  private String password; // optional
-  private Map<String,String> configuration; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    /**
-     * 
-     * @see TProtocolVersion
-     */
-    CLIENT_PROTOCOL((short)1, "client_protocol"),
-    USERNAME((short)2, "username"),
-    PASSWORD((short)3, "password"),
-    CONFIGURATION((short)4, "configuration");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // CLIENT_PROTOCOL
-          return CLIENT_PROTOCOL;
-        case 2: // USERNAME
-          return USERNAME;
-        case 3: // PASSWORD
-          return PASSWORD;
-        case 4: // CONFIGURATION
-          return CONFIGURATION;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private _Fields optionals[] = {_Fields.USERNAME,_Fields.PASSWORD,_Fields.CONFIGURATION};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.CLIENT_PROTOCOL, new org.apache.thrift.meta_data.FieldMetaData("client_protocol", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TProtocolVersion.class)));
-    tmpMap.put(_Fields.USERNAME, new org.apache.thrift.meta_data.FieldMetaData("username", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    tmpMap.put(_Fields.PASSWORD, new org.apache.thrift.meta_data.FieldMetaData("password", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    tmpMap.put(_Fields.CONFIGURATION, new org.apache.thrift.meta_data.FieldMetaData("configuration", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING), 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TOpenSessionReq.class, metaDataMap);
-  }
-
-  public TOpenSessionReq() {
-    this.client_protocol = org.apache.hive.service.cli.thrift.TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V8;
-
-  }
-
-  public TOpenSessionReq(
-    TProtocolVersion client_protocol)
-  {
-    this();
-    this.client_protocol = client_protocol;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TOpenSessionReq(TOpenSessionReq other) {
-    if (other.isSetClient_protocol()) {
-      this.client_protocol = other.client_protocol;
-    }
-    if (other.isSetUsername()) {
-      this.username = other.username;
-    }
-    if (other.isSetPassword()) {
-      this.password = other.password;
-    }
-    if (other.isSetConfiguration()) {
-      Map<String,String> __this__configuration = new HashMap<String,String>();
-      for (Map.Entry<String, String> other_element : other.configuration.entrySet()) {
-
-        String other_element_key = other_element.getKey();
-        String other_element_value = other_element.getValue();
-
-        String __this__configuration_copy_key = other_element_key;
-
-        String __this__configuration_copy_value = other_element_value;
-
-        __this__configuration.put(__this__configuration_copy_key, __this__configuration_copy_value);
-      }
-      this.configuration = __this__configuration;
-    }
-  }
-
-  public TOpenSessionReq deepCopy() {
-    return new TOpenSessionReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.client_protocol = org.apache.hive.service.cli.thrift.TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V8;
-
-    this.username = null;
-    this.password = null;
-    this.configuration = null;
-  }
-
-  /**
-   * 
-   * @see TProtocolVersion
-   */
-  public TProtocolVersion getClient_protocol() {
-    return this.client_protocol;
-  }
-
-  /**
-   * 
-   * @see TProtocolVersion
-   */
-  public void setClient_protocol(TProtocolVersion client_protocol) {
-    this.client_protocol = client_protocol;
-  }
-
-  public void unsetClient_protocol() {
-    this.client_protocol = null;
-  }
-
-  /** Returns true if field client_protocol is set (has been assigned a value) and false otherwise */
-  public boolean isSetClient_protocol() {
-    return this.client_protocol != null;
-  }
-
-  public void setClient_protocolIsSet(boolean value) {
-    if (!value) {
-      this.client_protocol = null;
-    }
-  }
-
-  public String getUsername() {
-    return this.username;
-  }
-
-  public void setUsername(String username) {
-    this.username = username;
-  }
-
-  public void unsetUsername() {
-    this.username = null;
-  }
-
-  /** Returns true if field username is set (has been assigned a value) and false otherwise */
-  public boolean isSetUsername() {
-    return this.username != null;
-  }
-
-  public void setUsernameIsSet(boolean value) {
-    if (!value) {
-      this.username = null;
-    }
-  }
-
-  public String getPassword() {
-    return this.password;
-  }
-
-  public void setPassword(String password) {
-    this.password = password;
-  }
-
-  public void unsetPassword() {
-    this.password = null;
-  }
-
-  /** Returns true if field password is set (has been assigned a value) and false otherwise */
-  public boolean isSetPassword() {
-    return this.password != null;
-  }
-
-  public void setPasswordIsSet(boolean value) {
-    if (!value) {
-      this.password = null;
-    }
-  }
-
-  public int getConfigurationSize() {
-    return (this.configuration == null) ? 0 : this.configuration.size();
-  }
-
-  public void putToConfiguration(String key, String val) {
-    if (this.configuration == null) {
-      this.configuration = new HashMap<String,String>();
-    }
-    this.configuration.put(key, val);
-  }
-
-  public Map<String,String> getConfiguration() {
-    return this.configuration;
-  }
-
-  public void setConfiguration(Map<String,String> configuration) {
-    this.configuration = configuration;
-  }
-
-  public void unsetConfiguration() {
-    this.configuration = null;
-  }
-
-  /** Returns true if field configuration is set (has been assigned a value) and false otherwise */
-  public boolean isSetConfiguration() {
-    return this.configuration != null;
-  }
-
-  public void setConfigurationIsSet(boolean value) {
-    if (!value) {
-      this.configuration = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case CLIENT_PROTOCOL:
-      if (value == null) {
-        unsetClient_protocol();
-      } else {
-        setClient_protocol((TProtocolVersion)value);
-      }
-      break;
-
-    case USERNAME:
-      if (value == null) {
-        unsetUsername();
-      } else {
-        setUsername((String)value);
-      }
-      break;
-
-    case PASSWORD:
-      if (value == null) {
-        unsetPassword();
-      } else {
-        setPassword((String)value);
-      }
-      break;
-
-    case CONFIGURATION:
-      if (value == null) {
-        unsetConfiguration();
-      } else {
-        setConfiguration((Map<String,String>)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case CLIENT_PROTOCOL:
-      return getClient_protocol();
-
-    case USERNAME:
-      return getUsername();
-
-    case PASSWORD:
-      return getPassword();
-
-    case CONFIGURATION:
-      return getConfiguration();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case CLIENT_PROTOCOL:
-      return isSetClient_protocol();
-    case USERNAME:
-      return isSetUsername();
-    case PASSWORD:
-      return isSetPassword();
-    case CONFIGURATION:
-      return isSetConfiguration();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TOpenSessionReq)
-      return this.equals((TOpenSessionReq)that);
-    return false;
-  }
-
-  public boolean equals(TOpenSessionReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_client_protocol = true && this.isSetClient_protocol();
-    boolean that_present_client_protocol = true && that.isSetClient_protocol();
-    if (this_present_client_protocol || that_present_client_protocol) {
-      if (!(this_present_client_protocol && that_present_client_protocol))
-        return false;
-      if (!this.client_protocol.equals(that.client_protocol))
-        return false;
-    }
-
-    boolean this_present_username = true && this.isSetUsername();
-    boolean that_present_username = true && that.isSetUsername();
-    if (this_present_username || that_present_username) {
-      if (!(this_present_username && that_present_username))
-        return false;
-      if (!this.username.equals(that.username))
-        return false;
-    }
-
-    boolean this_present_password = true && this.isSetPassword();
-    boolean that_present_password = true && that.isSetPassword();
-    if (this_present_password || that_present_password) {
-      if (!(this_present_password && that_present_password))
-        return false;
-      if (!this.password.equals(that.password))
-        return false;
-    }
-
-    boolean this_present_configuration = true && this.isSetConfiguration();
-    boolean that_present_configuration = true && that.isSetConfiguration();
-    if (this_present_configuration || that_present_configuration) {
-      if (!(this_present_configuration && that_present_configuration))
-        return false;
-      if (!this.configuration.equals(that.configuration))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_client_protocol = true && (isSetClient_protocol());
-    builder.append(present_client_protocol);
-    if (present_client_protocol)
-      builder.append(client_protocol.getValue());
-
-    boolean present_username = true && (isSetUsername());
-    builder.append(present_username);
-    if (present_username)
-      builder.append(username);
-
-    boolean present_password = true && (isSetPassword());
-    builder.append(present_password);
-    if (present_password)
-      builder.append(password);
-
-    boolean present_configuration = true && (isSetConfiguration());
-    builder.append(present_configuration);
-    if (present_configuration)
-      builder.append(configuration);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TOpenSessionReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TOpenSessionReq typedOther = (TOpenSessionReq)other;
-
-    lastComparison = Boolean.valueOf(isSetClient_protocol()).compareTo(typedOther.isSetClient_protocol());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetClient_protocol()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.client_protocol, typedOther.client_protocol);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetUsername()).compareTo(typedOther.isSetUsername());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetUsername()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.username, typedOther.username);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetPassword()).compareTo(typedOther.isSetPassword());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetPassword()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.password, typedOther.password);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetConfiguration()).compareTo(typedOther.isSetConfiguration());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetConfiguration()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.configuration, typedOther.configuration);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TOpenSessionReq(");
-    boolean first = true;
-
-    sb.append("client_protocol:");
-    if (this.client_protocol == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.client_protocol);
-    }
-    first = false;
-    if (isSetUsername()) {
-      if (!first) sb.append(", ");
-      sb.append("username:");
-      if (this.username == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.username);
-      }
-      first = false;
-    }
-    if (isSetPassword()) {
-      if (!first) sb.append(", ");
-      sb.append("password:");
-      if (this.password == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.password);
-      }
-      first = false;
-    }
-    if (isSetConfiguration()) {
-      if (!first) sb.append(", ");
-      sb.append("configuration:");
-      if (this.configuration == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.configuration);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetClient_protocol()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'client_protocol' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TOpenSessionReqStandardSchemeFactory implements SchemeFactory {
-    public TOpenSessionReqStandardScheme getScheme() {
-      return new TOpenSessionReqStandardScheme();
-    }
-  }
-
-  private static class TOpenSessionReqStandardScheme extends StandardScheme<TOpenSessionReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TOpenSessionReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // CLIENT_PROTOCOL
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.client_protocol = TProtocolVersion.findByValue(iprot.readI32());
-              struct.setClient_protocolIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // USERNAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.username = iprot.readString();
-              struct.setUsernameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // PASSWORD
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.password = iprot.readString();
-              struct.setPasswordIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // CONFIGURATION
-            if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
-              {
-                org.apache.thrift.protocol.TMap _map142 = iprot.readMapBegin();
-                struct.configuration = new HashMap<String,String>(2*_map142.size);
-                for (int _i143 = 0; _i143 < _map142.size; ++_i143)
-                {
-                  String _key144; // required
-                  String _val145; // required
-                  _key144 = iprot.readString();
-                  _val145 = iprot.readString();
-                  struct.configuration.put(_key144, _val145);
-                }
-                iprot.readMapEnd();
-              }
-              struct.setConfigurationIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TOpenSessionReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.client_protocol != null) {
-        oprot.writeFieldBegin(CLIENT_PROTOCOL_FIELD_DESC);
-        oprot.writeI32(struct.client_protocol.getValue());
-        oprot.writeFieldEnd();
-      }
-      if (struct.username != null) {
-        if (struct.isSetUsername()) {
-          oprot.writeFieldBegin(USERNAME_FIELD_DESC);
-          oprot.writeString(struct.username);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.password != null) {
-        if (struct.isSetPassword()) {
-          oprot.writeFieldBegin(PASSWORD_FIELD_DESC);
-          oprot.writeString(struct.password);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.configuration != null) {
-        if (struct.isSetConfiguration()) {
-          oprot.writeFieldBegin(CONFIGURATION_FIELD_DESC);
-          {
-            oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.configuration.size()));
-            for (Map.Entry<String, String> _iter146 : struct.configuration.entrySet())
-            {
-              oprot.writeString(_iter146.getKey());
-              oprot.writeString(_iter146.getValue());
-            }
-            oprot.writeMapEnd();
-          }
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TOpenSessionReqTupleSchemeFactory implements SchemeFactory {
-    public TOpenSessionReqTupleScheme getScheme() {
-      return new TOpenSessionReqTupleScheme();
-    }
-  }
-
-  private static class TOpenSessionReqTupleScheme extends TupleScheme<TOpenSessionReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TOpenSessionReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      oprot.writeI32(struct.client_protocol.getValue());
-      BitSet optionals = new BitSet();
-      if (struct.isSetUsername()) {
-        optionals.set(0);
-      }
-      if (struct.isSetPassword()) {
-        optionals.set(1);
-      }
-      if (struct.isSetConfiguration()) {
-        optionals.set(2);
-      }
-      oprot.writeBitSet(optionals, 3);
-      if (struct.isSetUsername()) {
-        oprot.writeString(struct.username);
-      }
-      if (struct.isSetPassword()) {
-        oprot.writeString(struct.password);
-      }
-      if (struct.isSetConfiguration()) {
-        {
-          oprot.writeI32(struct.configuration.size());
-          for (Map.Entry<String, String> _iter147 : struct.configuration.entrySet())
-          {
-            oprot.writeString(_iter147.getKey());
-            oprot.writeString(_iter147.getValue());
-          }
-        }
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TOpenSessionReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.client_protocol = TProtocolVersion.findByValue(iprot.readI32());
-      struct.setClient_protocolIsSet(true);
-      BitSet incoming = iprot.readBitSet(3);
-      if (incoming.get(0)) {
-        struct.username = iprot.readString();
-        struct.setUsernameIsSet(true);
-      }
-      if (incoming.get(1)) {
-        struct.password = iprot.readString();
-        struct.setPasswordIsSet(true);
-      }
-      if (incoming.get(2)) {
-        {
-          org.apache.thrift.protocol.TMap _map148 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-          struct.configuration = new HashMap<String,String>(2*_map148.size);
-          for (int _i149 = 0; _i149 < _map148.size; ++_i149)
-          {
-            String _key150; // required
-            String _val151; // required
-            _key150 = iprot.readString();
-            _val151 = iprot.readString();
-            struct.configuration.put(_key150, _val151);
-          }
-        }
-        struct.setConfigurationIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TOpenSessionResp.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TOpenSessionResp.java
deleted file mode 100644
index 351f78b2de20c..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TOpenSessionResp.java
+++ /dev/null
@@ -1,790 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TOpenSessionResp implements org.apache.thrift.TBase<TOpenSessionResp, TOpenSessionResp._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TOpenSessionResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField SERVER_PROTOCOL_VERSION_FIELD_DESC = new org.apache.thrift.protocol.TField("serverProtocolVersion", org.apache.thrift.protocol.TType.I32, (short)2);
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-  private static final org.apache.thrift.protocol.TField CONFIGURATION_FIELD_DESC = new org.apache.thrift.protocol.TField("configuration", org.apache.thrift.protocol.TType.MAP, (short)4);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TOpenSessionRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TOpenSessionRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TProtocolVersion serverProtocolVersion; // required
-  private TSessionHandle sessionHandle; // optional
-  private Map<String,String> configuration; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    /**
-     * 
-     * @see TProtocolVersion
-     */
-    SERVER_PROTOCOL_VERSION((short)2, "serverProtocolVersion"),
-    SESSION_HANDLE((short)3, "sessionHandle"),
-    CONFIGURATION((short)4, "configuration");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // SERVER_PROTOCOL_VERSION
-          return SERVER_PROTOCOL_VERSION;
-        case 3: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        case 4: // CONFIGURATION
-          return CONFIGURATION;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private _Fields optionals[] = {_Fields.SESSION_HANDLE,_Fields.CONFIGURATION};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.SERVER_PROTOCOL_VERSION, new org.apache.thrift.meta_data.FieldMetaData("serverProtocolVersion", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TProtocolVersion.class)));
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    tmpMap.put(_Fields.CONFIGURATION, new org.apache.thrift.meta_data.FieldMetaData("configuration", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING), 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TOpenSessionResp.class, metaDataMap);
-  }
-
-  public TOpenSessionResp() {
-    this.serverProtocolVersion = org.apache.hive.service.cli.thrift.TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V8;
-
-  }
-
-  public TOpenSessionResp(
-    TStatus status,
-    TProtocolVersion serverProtocolVersion)
-  {
-    this();
-    this.status = status;
-    this.serverProtocolVersion = serverProtocolVersion;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TOpenSessionResp(TOpenSessionResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetServerProtocolVersion()) {
-      this.serverProtocolVersion = other.serverProtocolVersion;
-    }
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-    if (other.isSetConfiguration()) {
-      Map<String,String> __this__configuration = new HashMap<String,String>();
-      for (Map.Entry<String, String> other_element : other.configuration.entrySet()) {
-
-        String other_element_key = other_element.getKey();
-        String other_element_value = other_element.getValue();
-
-        String __this__configuration_copy_key = other_element_key;
-
-        String __this__configuration_copy_value = other_element_value;
-
-        __this__configuration.put(__this__configuration_copy_key, __this__configuration_copy_value);
-      }
-      this.configuration = __this__configuration;
-    }
-  }
-
-  public TOpenSessionResp deepCopy() {
-    return new TOpenSessionResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.serverProtocolVersion = org.apache.hive.service.cli.thrift.TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V8;
-
-    this.sessionHandle = null;
-    this.configuration = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  /**
-   * 
-   * @see TProtocolVersion
-   */
-  public TProtocolVersion getServerProtocolVersion() {
-    return this.serverProtocolVersion;
-  }
-
-  /**
-   * 
-   * @see TProtocolVersion
-   */
-  public void setServerProtocolVersion(TProtocolVersion serverProtocolVersion) {
-    this.serverProtocolVersion = serverProtocolVersion;
-  }
-
-  public void unsetServerProtocolVersion() {
-    this.serverProtocolVersion = null;
-  }
-
-  /** Returns true if field serverProtocolVersion is set (has been assigned a value) and false otherwise */
-  public boolean isSetServerProtocolVersion() {
-    return this.serverProtocolVersion != null;
-  }
-
-  public void setServerProtocolVersionIsSet(boolean value) {
-    if (!value) {
-      this.serverProtocolVersion = null;
-    }
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public int getConfigurationSize() {
-    return (this.configuration == null) ? 0 : this.configuration.size();
-  }
-
-  public void putToConfiguration(String key, String val) {
-    if (this.configuration == null) {
-      this.configuration = new HashMap<String,String>();
-    }
-    this.configuration.put(key, val);
-  }
-
-  public Map<String,String> getConfiguration() {
-    return this.configuration;
-  }
-
-  public void setConfiguration(Map<String,String> configuration) {
-    this.configuration = configuration;
-  }
-
-  public void unsetConfiguration() {
-    this.configuration = null;
-  }
-
-  /** Returns true if field configuration is set (has been assigned a value) and false otherwise */
-  public boolean isSetConfiguration() {
-    return this.configuration != null;
-  }
-
-  public void setConfigurationIsSet(boolean value) {
-    if (!value) {
-      this.configuration = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case SERVER_PROTOCOL_VERSION:
-      if (value == null) {
-        unsetServerProtocolVersion();
-      } else {
-        setServerProtocolVersion((TProtocolVersion)value);
-      }
-      break;
-
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    case CONFIGURATION:
-      if (value == null) {
-        unsetConfiguration();
-      } else {
-        setConfiguration((Map<String,String>)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case SERVER_PROTOCOL_VERSION:
-      return getServerProtocolVersion();
-
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    case CONFIGURATION:
-      return getConfiguration();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case SERVER_PROTOCOL_VERSION:
-      return isSetServerProtocolVersion();
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    case CONFIGURATION:
-      return isSetConfiguration();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TOpenSessionResp)
-      return this.equals((TOpenSessionResp)that);
-    return false;
-  }
-
-  public boolean equals(TOpenSessionResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_serverProtocolVersion = true && this.isSetServerProtocolVersion();
-    boolean that_present_serverProtocolVersion = true && that.isSetServerProtocolVersion();
-    if (this_present_serverProtocolVersion || that_present_serverProtocolVersion) {
-      if (!(this_present_serverProtocolVersion && that_present_serverProtocolVersion))
-        return false;
-      if (!this.serverProtocolVersion.equals(that.serverProtocolVersion))
-        return false;
-    }
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    boolean this_present_configuration = true && this.isSetConfiguration();
-    boolean that_present_configuration = true && that.isSetConfiguration();
-    if (this_present_configuration || that_present_configuration) {
-      if (!(this_present_configuration && that_present_configuration))
-        return false;
-      if (!this.configuration.equals(that.configuration))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_status = true && (isSetStatus());
-    builder.append(present_status);
-    if (present_status)
-      builder.append(status);
-
-    boolean present_serverProtocolVersion = true && (isSetServerProtocolVersion());
-    builder.append(present_serverProtocolVersion);
-    if (present_serverProtocolVersion)
-      builder.append(serverProtocolVersion.getValue());
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    builder.append(present_sessionHandle);
-    if (present_sessionHandle)
-      builder.append(sessionHandle);
-
-    boolean present_configuration = true && (isSetConfiguration());
-    builder.append(present_configuration);
-    if (present_configuration)
-      builder.append(configuration);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TOpenSessionResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TOpenSessionResp typedOther = (TOpenSessionResp)other;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(typedOther.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, typedOther.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetServerProtocolVersion()).compareTo(typedOther.isSetServerProtocolVersion());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetServerProtocolVersion()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.serverProtocolVersion, typedOther.serverProtocolVersion);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(typedOther.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, typedOther.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetConfiguration()).compareTo(typedOther.isSetConfiguration());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetConfiguration()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.configuration, typedOther.configuration);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TOpenSessionResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("serverProtocolVersion:");
-    if (this.serverProtocolVersion == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.serverProtocolVersion);
-    }
-    first = false;
-    if (isSetSessionHandle()) {
-      if (!first) sb.append(", ");
-      sb.append("sessionHandle:");
-      if (this.sessionHandle == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.sessionHandle);
-      }
-      first = false;
-    }
-    if (isSetConfiguration()) {
-      if (!first) sb.append(", ");
-      sb.append("configuration:");
-      if (this.configuration == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.configuration);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    if (!isSetServerProtocolVersion()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'serverProtocolVersion' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TOpenSessionRespStandardSchemeFactory implements SchemeFactory {
-    public TOpenSessionRespStandardScheme getScheme() {
-      return new TOpenSessionRespStandardScheme();
-    }
-  }
-
-  private static class TOpenSessionRespStandardScheme extends StandardScheme<TOpenSessionResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TOpenSessionResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // SERVER_PROTOCOL_VERSION
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.serverProtocolVersion = TProtocolVersion.findByValue(iprot.readI32());
-              struct.setServerProtocolVersionIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // CONFIGURATION
-            if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
-              {
-                org.apache.thrift.protocol.TMap _map152 = iprot.readMapBegin();
-                struct.configuration = new HashMap<String,String>(2*_map152.size);
-                for (int _i153 = 0; _i153 < _map152.size; ++_i153)
-                {
-                  String _key154; // required
-                  String _val155; // required
-                  _key154 = iprot.readString();
-                  _val155 = iprot.readString();
-                  struct.configuration.put(_key154, _val155);
-                }
-                iprot.readMapEnd();
-              }
-              struct.setConfigurationIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TOpenSessionResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.serverProtocolVersion != null) {
-        oprot.writeFieldBegin(SERVER_PROTOCOL_VERSION_FIELD_DESC);
-        oprot.writeI32(struct.serverProtocolVersion.getValue());
-        oprot.writeFieldEnd();
-      }
-      if (struct.sessionHandle != null) {
-        if (struct.isSetSessionHandle()) {
-          oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-          struct.sessionHandle.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.configuration != null) {
-        if (struct.isSetConfiguration()) {
-          oprot.writeFieldBegin(CONFIGURATION_FIELD_DESC);
-          {
-            oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.configuration.size()));
-            for (Map.Entry<String, String> _iter156 : struct.configuration.entrySet())
-            {
-              oprot.writeString(_iter156.getKey());
-              oprot.writeString(_iter156.getValue());
-            }
-            oprot.writeMapEnd();
-          }
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TOpenSessionRespTupleSchemeFactory implements SchemeFactory {
-    public TOpenSessionRespTupleScheme getScheme() {
-      return new TOpenSessionRespTupleScheme();
-    }
-  }
-
-  private static class TOpenSessionRespTupleScheme extends TupleScheme<TOpenSessionResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TOpenSessionResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      oprot.writeI32(struct.serverProtocolVersion.getValue());
-      BitSet optionals = new BitSet();
-      if (struct.isSetSessionHandle()) {
-        optionals.set(0);
-      }
-      if (struct.isSetConfiguration()) {
-        optionals.set(1);
-      }
-      oprot.writeBitSet(optionals, 2);
-      if (struct.isSetSessionHandle()) {
-        struct.sessionHandle.write(oprot);
-      }
-      if (struct.isSetConfiguration()) {
-        {
-          oprot.writeI32(struct.configuration.size());
-          for (Map.Entry<String, String> _iter157 : struct.configuration.entrySet())
-          {
-            oprot.writeString(_iter157.getKey());
-            oprot.writeString(_iter157.getValue());
-          }
-        }
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TOpenSessionResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      struct.serverProtocolVersion = TProtocolVersion.findByValue(iprot.readI32());
-      struct.setServerProtocolVersionIsSet(true);
-      BitSet incoming = iprot.readBitSet(2);
-      if (incoming.get(0)) {
-        struct.sessionHandle = new TSessionHandle();
-        struct.sessionHandle.read(iprot);
-        struct.setSessionHandleIsSet(true);
-      }
-      if (incoming.get(1)) {
-        {
-          org.apache.thrift.protocol.TMap _map158 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-          struct.configuration = new HashMap<String,String>(2*_map158.size);
-          for (int _i159 = 0; _i159 < _map158.size; ++_i159)
-          {
-            String _key160; // required
-            String _val161; // required
-            _key160 = iprot.readString();
-            _val161 = iprot.readString();
-            struct.configuration.put(_key160, _val161);
-          }
-        }
-        struct.setConfigurationIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TOperationHandle.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TOperationHandle.java
deleted file mode 100644
index 8fbd8752eaca6..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TOperationHandle.java
+++ /dev/null
@@ -1,705 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TOperationHandle implements org.apache.thrift.TBase<TOperationHandle, TOperationHandle._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TOperationHandle");
-
-  private static final org.apache.thrift.protocol.TField OPERATION_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("operationId", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField OPERATION_TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationType", org.apache.thrift.protocol.TType.I32, (short)2);
-  private static final org.apache.thrift.protocol.TField HAS_RESULT_SET_FIELD_DESC = new org.apache.thrift.protocol.TField("hasResultSet", org.apache.thrift.protocol.TType.BOOL, (short)3);
-  private static final org.apache.thrift.protocol.TField MODIFIED_ROW_COUNT_FIELD_DESC = new org.apache.thrift.protocol.TField("modifiedRowCount", org.apache.thrift.protocol.TType.DOUBLE, (short)4);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TOperationHandleStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TOperationHandleTupleSchemeFactory());
-  }
-
-  private THandleIdentifier operationId; // required
-  private TOperationType operationType; // required
-  private boolean hasResultSet; // required
-  private double modifiedRowCount; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    OPERATION_ID((short)1, "operationId"),
-    /**
-     * 
-     * @see TOperationType
-     */
-    OPERATION_TYPE((short)2, "operationType"),
-    HAS_RESULT_SET((short)3, "hasResultSet"),
-    MODIFIED_ROW_COUNT((short)4, "modifiedRowCount");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // OPERATION_ID
-          return OPERATION_ID;
-        case 2: // OPERATION_TYPE
-          return OPERATION_TYPE;
-        case 3: // HAS_RESULT_SET
-          return HAS_RESULT_SET;
-        case 4: // MODIFIED_ROW_COUNT
-          return MODIFIED_ROW_COUNT;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __HASRESULTSET_ISSET_ID = 0;
-  private static final int __MODIFIEDROWCOUNT_ISSET_ID = 1;
-  private byte __isset_bitfield = 0;
-  private _Fields optionals[] = {_Fields.MODIFIED_ROW_COUNT};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.OPERATION_ID, new org.apache.thrift.meta_data.FieldMetaData("operationId", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, THandleIdentifier.class)));
-    tmpMap.put(_Fields.OPERATION_TYPE, new org.apache.thrift.meta_data.FieldMetaData("operationType", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TOperationType.class)));
-    tmpMap.put(_Fields.HAS_RESULT_SET, new org.apache.thrift.meta_data.FieldMetaData("hasResultSet", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
-    tmpMap.put(_Fields.MODIFIED_ROW_COUNT, new org.apache.thrift.meta_data.FieldMetaData("modifiedRowCount", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TOperationHandle.class, metaDataMap);
-  }
-
-  public TOperationHandle() {
-  }
-
-  public TOperationHandle(
-    THandleIdentifier operationId,
-    TOperationType operationType,
-    boolean hasResultSet)
-  {
-    this();
-    this.operationId = operationId;
-    this.operationType = operationType;
-    this.hasResultSet = hasResultSet;
-    setHasResultSetIsSet(true);
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TOperationHandle(TOperationHandle other) {
-    __isset_bitfield = other.__isset_bitfield;
-    if (other.isSetOperationId()) {
-      this.operationId = new THandleIdentifier(other.operationId);
-    }
-    if (other.isSetOperationType()) {
-      this.operationType = other.operationType;
-    }
-    this.hasResultSet = other.hasResultSet;
-    this.modifiedRowCount = other.modifiedRowCount;
-  }
-
-  public TOperationHandle deepCopy() {
-    return new TOperationHandle(this);
-  }
-
-  @Override
-  public void clear() {
-    this.operationId = null;
-    this.operationType = null;
-    setHasResultSetIsSet(false);
-    this.hasResultSet = false;
-    setModifiedRowCountIsSet(false);
-    this.modifiedRowCount = 0.0;
-  }
-
-  public THandleIdentifier getOperationId() {
-    return this.operationId;
-  }
-
-  public void setOperationId(THandleIdentifier operationId) {
-    this.operationId = operationId;
-  }
-
-  public void unsetOperationId() {
-    this.operationId = null;
-  }
-
-  /** Returns true if field operationId is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationId() {
-    return this.operationId != null;
-  }
-
-  public void setOperationIdIsSet(boolean value) {
-    if (!value) {
-      this.operationId = null;
-    }
-  }
-
-  /**
-   * 
-   * @see TOperationType
-   */
-  public TOperationType getOperationType() {
-    return this.operationType;
-  }
-
-  /**
-   * 
-   * @see TOperationType
-   */
-  public void setOperationType(TOperationType operationType) {
-    this.operationType = operationType;
-  }
-
-  public void unsetOperationType() {
-    this.operationType = null;
-  }
-
-  /** Returns true if field operationType is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationType() {
-    return this.operationType != null;
-  }
-
-  public void setOperationTypeIsSet(boolean value) {
-    if (!value) {
-      this.operationType = null;
-    }
-  }
-
-  public boolean isHasResultSet() {
-    return this.hasResultSet;
-  }
-
-  public void setHasResultSet(boolean hasResultSet) {
-    this.hasResultSet = hasResultSet;
-    setHasResultSetIsSet(true);
-  }
-
-  public void unsetHasResultSet() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __HASRESULTSET_ISSET_ID);
-  }
-
-  /** Returns true if field hasResultSet is set (has been assigned a value) and false otherwise */
-  public boolean isSetHasResultSet() {
-    return EncodingUtils.testBit(__isset_bitfield, __HASRESULTSET_ISSET_ID);
-  }
-
-  public void setHasResultSetIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __HASRESULTSET_ISSET_ID, value);
-  }
-
-  public double getModifiedRowCount() {
-    return this.modifiedRowCount;
-  }
-
-  public void setModifiedRowCount(double modifiedRowCount) {
-    this.modifiedRowCount = modifiedRowCount;
-    setModifiedRowCountIsSet(true);
-  }
-
-  public void unsetModifiedRowCount() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __MODIFIEDROWCOUNT_ISSET_ID);
-  }
-
-  /** Returns true if field modifiedRowCount is set (has been assigned a value) and false otherwise */
-  public boolean isSetModifiedRowCount() {
-    return EncodingUtils.testBit(__isset_bitfield, __MODIFIEDROWCOUNT_ISSET_ID);
-  }
-
-  public void setModifiedRowCountIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MODIFIEDROWCOUNT_ISSET_ID, value);
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case OPERATION_ID:
-      if (value == null) {
-        unsetOperationId();
-      } else {
-        setOperationId((THandleIdentifier)value);
-      }
-      break;
-
-    case OPERATION_TYPE:
-      if (value == null) {
-        unsetOperationType();
-      } else {
-        setOperationType((TOperationType)value);
-      }
-      break;
-
-    case HAS_RESULT_SET:
-      if (value == null) {
-        unsetHasResultSet();
-      } else {
-        setHasResultSet((Boolean)value);
-      }
-      break;
-
-    case MODIFIED_ROW_COUNT:
-      if (value == null) {
-        unsetModifiedRowCount();
-      } else {
-        setModifiedRowCount((Double)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case OPERATION_ID:
-      return getOperationId();
-
-    case OPERATION_TYPE:
-      return getOperationType();
-
-    case HAS_RESULT_SET:
-      return Boolean.valueOf(isHasResultSet());
-
-    case MODIFIED_ROW_COUNT:
-      return Double.valueOf(getModifiedRowCount());
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case OPERATION_ID:
-      return isSetOperationId();
-    case OPERATION_TYPE:
-      return isSetOperationType();
-    case HAS_RESULT_SET:
-      return isSetHasResultSet();
-    case MODIFIED_ROW_COUNT:
-      return isSetModifiedRowCount();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TOperationHandle)
-      return this.equals((TOperationHandle)that);
-    return false;
-  }
-
-  public boolean equals(TOperationHandle that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_operationId = true && this.isSetOperationId();
-    boolean that_present_operationId = true && that.isSetOperationId();
-    if (this_present_operationId || that_present_operationId) {
-      if (!(this_present_operationId && that_present_operationId))
-        return false;
-      if (!this.operationId.equals(that.operationId))
-        return false;
-    }
-
-    boolean this_present_operationType = true && this.isSetOperationType();
-    boolean that_present_operationType = true && that.isSetOperationType();
-    if (this_present_operationType || that_present_operationType) {
-      if (!(this_present_operationType && that_present_operationType))
-        return false;
-      if (!this.operationType.equals(that.operationType))
-        return false;
-    }
-
-    boolean this_present_hasResultSet = true;
-    boolean that_present_hasResultSet = true;
-    if (this_present_hasResultSet || that_present_hasResultSet) {
-      if (!(this_present_hasResultSet && that_present_hasResultSet))
-        return false;
-      if (this.hasResultSet != that.hasResultSet)
-        return false;
-    }
-
-    boolean this_present_modifiedRowCount = true && this.isSetModifiedRowCount();
-    boolean that_present_modifiedRowCount = true && that.isSetModifiedRowCount();
-    if (this_present_modifiedRowCount || that_present_modifiedRowCount) {
-      if (!(this_present_modifiedRowCount && that_present_modifiedRowCount))
-        return false;
-      if (this.modifiedRowCount != that.modifiedRowCount)
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_operationId = true && (isSetOperationId());
-    builder.append(present_operationId);
-    if (present_operationId)
-      builder.append(operationId);
-
-    boolean present_operationType = true && (isSetOperationType());
-    builder.append(present_operationType);
-    if (present_operationType)
-      builder.append(operationType.getValue());
-
-    boolean present_hasResultSet = true;
-    builder.append(present_hasResultSet);
-    if (present_hasResultSet)
-      builder.append(hasResultSet);
-
-    boolean present_modifiedRowCount = true && (isSetModifiedRowCount());
-    builder.append(present_modifiedRowCount);
-    if (present_modifiedRowCount)
-      builder.append(modifiedRowCount);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TOperationHandle other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TOperationHandle typedOther = (TOperationHandle)other;
-
-    lastComparison = Boolean.valueOf(isSetOperationId()).compareTo(typedOther.isSetOperationId());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationId()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationId, typedOther.operationId);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOperationType()).compareTo(typedOther.isSetOperationType());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationType()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationType, typedOther.operationType);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetHasResultSet()).compareTo(typedOther.isSetHasResultSet());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetHasResultSet()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.hasResultSet, typedOther.hasResultSet);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetModifiedRowCount()).compareTo(typedOther.isSetModifiedRowCount());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetModifiedRowCount()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.modifiedRowCount, typedOther.modifiedRowCount);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TOperationHandle(");
-    boolean first = true;
-
-    sb.append("operationId:");
-    if (this.operationId == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.operationId);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("operationType:");
-    if (this.operationType == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.operationType);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("hasResultSet:");
-    sb.append(this.hasResultSet);
-    first = false;
-    if (isSetModifiedRowCount()) {
-      if (!first) sb.append(", ");
-      sb.append("modifiedRowCount:");
-      sb.append(this.modifiedRowCount);
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetOperationId()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'operationId' is unset! Struct:" + toString());
-    }
-
-    if (!isSetOperationType()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'operationType' is unset! Struct:" + toString());
-    }
-
-    if (!isSetHasResultSet()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'hasResultSet' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (operationId != null) {
-      operationId.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TOperationHandleStandardSchemeFactory implements SchemeFactory {
-    public TOperationHandleStandardScheme getScheme() {
-      return new TOperationHandleStandardScheme();
-    }
-  }
-
-  private static class TOperationHandleStandardScheme extends StandardScheme<TOperationHandle> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TOperationHandle struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // OPERATION_ID
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationId = new THandleIdentifier();
-              struct.operationId.read(iprot);
-              struct.setOperationIdIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // OPERATION_TYPE
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.operationType = TOperationType.findByValue(iprot.readI32());
-              struct.setOperationTypeIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // HAS_RESULT_SET
-            if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
-              struct.hasResultSet = iprot.readBool();
-              struct.setHasResultSetIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // MODIFIED_ROW_COUNT
-            if (schemeField.type == org.apache.thrift.protocol.TType.DOUBLE) {
-              struct.modifiedRowCount = iprot.readDouble();
-              struct.setModifiedRowCountIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TOperationHandle struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.operationId != null) {
-        oprot.writeFieldBegin(OPERATION_ID_FIELD_DESC);
-        struct.operationId.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.operationType != null) {
-        oprot.writeFieldBegin(OPERATION_TYPE_FIELD_DESC);
-        oprot.writeI32(struct.operationType.getValue());
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldBegin(HAS_RESULT_SET_FIELD_DESC);
-      oprot.writeBool(struct.hasResultSet);
-      oprot.writeFieldEnd();
-      if (struct.isSetModifiedRowCount()) {
-        oprot.writeFieldBegin(MODIFIED_ROW_COUNT_FIELD_DESC);
-        oprot.writeDouble(struct.modifiedRowCount);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TOperationHandleTupleSchemeFactory implements SchemeFactory {
-    public TOperationHandleTupleScheme getScheme() {
-      return new TOperationHandleTupleScheme();
-    }
-  }
-
-  private static class TOperationHandleTupleScheme extends TupleScheme<TOperationHandle> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TOperationHandle struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.operationId.write(oprot);
-      oprot.writeI32(struct.operationType.getValue());
-      oprot.writeBool(struct.hasResultSet);
-      BitSet optionals = new BitSet();
-      if (struct.isSetModifiedRowCount()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetModifiedRowCount()) {
-        oprot.writeDouble(struct.modifiedRowCount);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TOperationHandle struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.operationId = new THandleIdentifier();
-      struct.operationId.read(iprot);
-      struct.setOperationIdIsSet(true);
-      struct.operationType = TOperationType.findByValue(iprot.readI32());
-      struct.setOperationTypeIsSet(true);
-      struct.hasResultSet = iprot.readBool();
-      struct.setHasResultSetIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.modifiedRowCount = iprot.readDouble();
-        struct.setModifiedRowCountIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TOperationState.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TOperationState.java
deleted file mode 100644
index 219866223a6b0..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TOperationState.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.thrift.TEnum;
-
-public enum TOperationState implements org.apache.thrift.TEnum {
-  INITIALIZED_STATE(0),
-  RUNNING_STATE(1),
-  FINISHED_STATE(2),
-  CANCELED_STATE(3),
-  CLOSED_STATE(4),
-  ERROR_STATE(5),
-  UKNOWN_STATE(6),
-  PENDING_STATE(7);
-
-  private final int value;
-
-  private TOperationState(int value) {
-    this.value = value;
-  }
-
-  /**
-   * Get the integer value of this enum value, as defined in the Thrift IDL.
-   */
-  public int getValue() {
-    return value;
-  }
-
-  /**
-   * Find a the enum type by its integer value, as defined in the Thrift IDL.
-   * @return null if the value is not found.
-   */
-  public static TOperationState findByValue(int value) { 
-    switch (value) {
-      case 0:
-        return INITIALIZED_STATE;
-      case 1:
-        return RUNNING_STATE;
-      case 2:
-        return FINISHED_STATE;
-      case 3:
-        return CANCELED_STATE;
-      case 4:
-        return CLOSED_STATE;
-      case 5:
-        return ERROR_STATE;
-      case 6:
-        return UKNOWN_STATE;
-      case 7:
-        return PENDING_STATE;
-      default:
-        return null;
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TOperationType.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TOperationType.java
deleted file mode 100644
index b6d4b2fab9f96..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TOperationType.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.thrift.TEnum;
-
-public enum TOperationType implements org.apache.thrift.TEnum {
-  EXECUTE_STATEMENT(0),
-  GET_TYPE_INFO(1),
-  GET_CATALOGS(2),
-  GET_SCHEMAS(3),
-  GET_TABLES(4),
-  GET_TABLE_TYPES(5),
-  GET_COLUMNS(6),
-  GET_FUNCTIONS(7),
-  UNKNOWN(8);
-
-  private final int value;
-
-  private TOperationType(int value) {
-    this.value = value;
-  }
-
-  /**
-   * Get the integer value of this enum value, as defined in the Thrift IDL.
-   */
-  public int getValue() {
-    return value;
-  }
-
-  /**
-   * Find a the enum type by its integer value, as defined in the Thrift IDL.
-   * @return null if the value is not found.
-   */
-  public static TOperationType findByValue(int value) { 
-    switch (value) {
-      case 0:
-        return EXECUTE_STATEMENT;
-      case 1:
-        return GET_TYPE_INFO;
-      case 2:
-        return GET_CATALOGS;
-      case 3:
-        return GET_SCHEMAS;
-      case 4:
-        return GET_TABLES;
-      case 5:
-        return GET_TABLE_TYPES;
-      case 6:
-        return GET_COLUMNS;
-      case 7:
-        return GET_FUNCTIONS;
-      case 8:
-        return UNKNOWN;
-      default:
-        return null;
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TPrimitiveTypeEntry.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TPrimitiveTypeEntry.java
deleted file mode 100644
index 9d2abf2b3b084..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TPrimitiveTypeEntry.java
+++ /dev/null
@@ -1,512 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TPrimitiveTypeEntry implements org.apache.thrift.TBase<TPrimitiveTypeEntry, TPrimitiveTypeEntry._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TPrimitiveTypeEntry");
-
-  private static final org.apache.thrift.protocol.TField TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("type", org.apache.thrift.protocol.TType.I32, (short)1);
-  private static final org.apache.thrift.protocol.TField TYPE_QUALIFIERS_FIELD_DESC = new org.apache.thrift.protocol.TField("typeQualifiers", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TPrimitiveTypeEntryStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TPrimitiveTypeEntryTupleSchemeFactory());
-  }
-
-  private TTypeId type; // required
-  private TTypeQualifiers typeQualifiers; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    /**
-     * 
-     * @see TTypeId
-     */
-    TYPE((short)1, "type"),
-    TYPE_QUALIFIERS((short)2, "typeQualifiers");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // TYPE
-          return TYPE;
-        case 2: // TYPE_QUALIFIERS
-          return TYPE_QUALIFIERS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private _Fields optionals[] = {_Fields.TYPE_QUALIFIERS};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.TYPE, new org.apache.thrift.meta_data.FieldMetaData("type", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TTypeId.class)));
-    tmpMap.put(_Fields.TYPE_QUALIFIERS, new org.apache.thrift.meta_data.FieldMetaData("typeQualifiers", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TTypeQualifiers.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TPrimitiveTypeEntry.class, metaDataMap);
-  }
-
-  public TPrimitiveTypeEntry() {
-  }
-
-  public TPrimitiveTypeEntry(
-    TTypeId type)
-  {
-    this();
-    this.type = type;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TPrimitiveTypeEntry(TPrimitiveTypeEntry other) {
-    if (other.isSetType()) {
-      this.type = other.type;
-    }
-    if (other.isSetTypeQualifiers()) {
-      this.typeQualifiers = new TTypeQualifiers(other.typeQualifiers);
-    }
-  }
-
-  public TPrimitiveTypeEntry deepCopy() {
-    return new TPrimitiveTypeEntry(this);
-  }
-
-  @Override
-  public void clear() {
-    this.type = null;
-    this.typeQualifiers = null;
-  }
-
-  /**
-   * 
-   * @see TTypeId
-   */
-  public TTypeId getType() {
-    return this.type;
-  }
-
-  /**
-   * 
-   * @see TTypeId
-   */
-  public void setType(TTypeId type) {
-    this.type = type;
-  }
-
-  public void unsetType() {
-    this.type = null;
-  }
-
-  /** Returns true if field type is set (has been assigned a value) and false otherwise */
-  public boolean isSetType() {
-    return this.type != null;
-  }
-
-  public void setTypeIsSet(boolean value) {
-    if (!value) {
-      this.type = null;
-    }
-  }
-
-  public TTypeQualifiers getTypeQualifiers() {
-    return this.typeQualifiers;
-  }
-
-  public void setTypeQualifiers(TTypeQualifiers typeQualifiers) {
-    this.typeQualifiers = typeQualifiers;
-  }
-
-  public void unsetTypeQualifiers() {
-    this.typeQualifiers = null;
-  }
-
-  /** Returns true if field typeQualifiers is set (has been assigned a value) and false otherwise */
-  public boolean isSetTypeQualifiers() {
-    return this.typeQualifiers != null;
-  }
-
-  public void setTypeQualifiersIsSet(boolean value) {
-    if (!value) {
-      this.typeQualifiers = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case TYPE:
-      if (value == null) {
-        unsetType();
-      } else {
-        setType((TTypeId)value);
-      }
-      break;
-
-    case TYPE_QUALIFIERS:
-      if (value == null) {
-        unsetTypeQualifiers();
-      } else {
-        setTypeQualifiers((TTypeQualifiers)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case TYPE:
-      return getType();
-
-    case TYPE_QUALIFIERS:
-      return getTypeQualifiers();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case TYPE:
-      return isSetType();
-    case TYPE_QUALIFIERS:
-      return isSetTypeQualifiers();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TPrimitiveTypeEntry)
-      return this.equals((TPrimitiveTypeEntry)that);
-    return false;
-  }
-
-  public boolean equals(TPrimitiveTypeEntry that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_type = true && this.isSetType();
-    boolean that_present_type = true && that.isSetType();
-    if (this_present_type || that_present_type) {
-      if (!(this_present_type && that_present_type))
-        return false;
-      if (!this.type.equals(that.type))
-        return false;
-    }
-
-    boolean this_present_typeQualifiers = true && this.isSetTypeQualifiers();
-    boolean that_present_typeQualifiers = true && that.isSetTypeQualifiers();
-    if (this_present_typeQualifiers || that_present_typeQualifiers) {
-      if (!(this_present_typeQualifiers && that_present_typeQualifiers))
-        return false;
-      if (!this.typeQualifiers.equals(that.typeQualifiers))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_type = true && (isSetType());
-    builder.append(present_type);
-    if (present_type)
-      builder.append(type.getValue());
-
-    boolean present_typeQualifiers = true && (isSetTypeQualifiers());
-    builder.append(present_typeQualifiers);
-    if (present_typeQualifiers)
-      builder.append(typeQualifiers);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TPrimitiveTypeEntry other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TPrimitiveTypeEntry typedOther = (TPrimitiveTypeEntry)other;
-
-    lastComparison = Boolean.valueOf(isSetType()).compareTo(typedOther.isSetType());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetType()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.type, typedOther.type);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetTypeQualifiers()).compareTo(typedOther.isSetTypeQualifiers());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetTypeQualifiers()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.typeQualifiers, typedOther.typeQualifiers);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TPrimitiveTypeEntry(");
-    boolean first = true;
-
-    sb.append("type:");
-    if (this.type == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.type);
-    }
-    first = false;
-    if (isSetTypeQualifiers()) {
-      if (!first) sb.append(", ");
-      sb.append("typeQualifiers:");
-      if (this.typeQualifiers == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.typeQualifiers);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetType()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'type' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (typeQualifiers != null) {
-      typeQualifiers.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TPrimitiveTypeEntryStandardSchemeFactory implements SchemeFactory {
-    public TPrimitiveTypeEntryStandardScheme getScheme() {
-      return new TPrimitiveTypeEntryStandardScheme();
-    }
-  }
-
-  private static class TPrimitiveTypeEntryStandardScheme extends StandardScheme<TPrimitiveTypeEntry> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TPrimitiveTypeEntry struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // TYPE
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.type = TTypeId.findByValue(iprot.readI32());
-              struct.setTypeIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // TYPE_QUALIFIERS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.typeQualifiers = new TTypeQualifiers();
-              struct.typeQualifiers.read(iprot);
-              struct.setTypeQualifiersIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TPrimitiveTypeEntry struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.type != null) {
-        oprot.writeFieldBegin(TYPE_FIELD_DESC);
-        oprot.writeI32(struct.type.getValue());
-        oprot.writeFieldEnd();
-      }
-      if (struct.typeQualifiers != null) {
-        if (struct.isSetTypeQualifiers()) {
-          oprot.writeFieldBegin(TYPE_QUALIFIERS_FIELD_DESC);
-          struct.typeQualifiers.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TPrimitiveTypeEntryTupleSchemeFactory implements SchemeFactory {
-    public TPrimitiveTypeEntryTupleScheme getScheme() {
-      return new TPrimitiveTypeEntryTupleScheme();
-    }
-  }
-
-  private static class TPrimitiveTypeEntryTupleScheme extends TupleScheme<TPrimitiveTypeEntry> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TPrimitiveTypeEntry struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      oprot.writeI32(struct.type.getValue());
-      BitSet optionals = new BitSet();
-      if (struct.isSetTypeQualifiers()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetTypeQualifiers()) {
-        struct.typeQualifiers.write(oprot);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TPrimitiveTypeEntry struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.type = TTypeId.findByValue(iprot.readI32());
-      struct.setTypeIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.typeQualifiers = new TTypeQualifiers();
-        struct.typeQualifiers.read(iprot);
-        struct.setTypeQualifiersIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TProtocolVersion.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TProtocolVersion.java
deleted file mode 100644
index a4279d29f662e..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TProtocolVersion.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.thrift.TEnum;
-
-public enum TProtocolVersion implements org.apache.thrift.TEnum {
-  HIVE_CLI_SERVICE_PROTOCOL_V1(0),
-  HIVE_CLI_SERVICE_PROTOCOL_V2(1),
-  HIVE_CLI_SERVICE_PROTOCOL_V3(2),
-  HIVE_CLI_SERVICE_PROTOCOL_V4(3),
-  HIVE_CLI_SERVICE_PROTOCOL_V5(4),
-  HIVE_CLI_SERVICE_PROTOCOL_V6(5),
-  HIVE_CLI_SERVICE_PROTOCOL_V7(6),
-  HIVE_CLI_SERVICE_PROTOCOL_V8(7);
-
-  private final int value;
-
-  private TProtocolVersion(int value) {
-    this.value = value;
-  }
-
-  /**
-   * Get the integer value of this enum value, as defined in the Thrift IDL.
-   */
-  public int getValue() {
-    return value;
-  }
-
-  /**
-   * Find a the enum type by its integer value, as defined in the Thrift IDL.
-   * @return null if the value is not found.
-   */
-  public static TProtocolVersion findByValue(int value) { 
-    switch (value) {
-      case 0:
-        return HIVE_CLI_SERVICE_PROTOCOL_V1;
-      case 1:
-        return HIVE_CLI_SERVICE_PROTOCOL_V2;
-      case 2:
-        return HIVE_CLI_SERVICE_PROTOCOL_V3;
-      case 3:
-        return HIVE_CLI_SERVICE_PROTOCOL_V4;
-      case 4:
-        return HIVE_CLI_SERVICE_PROTOCOL_V5;
-      case 5:
-        return HIVE_CLI_SERVICE_PROTOCOL_V6;
-      case 6:
-        return HIVE_CLI_SERVICE_PROTOCOL_V7;
-      case 7:
-        return HIVE_CLI_SERVICE_PROTOCOL_V8;
-      default:
-        return null;
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TRenewDelegationTokenReq.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TRenewDelegationTokenReq.java
deleted file mode 100644
index a3e39c8cdf321..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TRenewDelegationTokenReq.java
+++ /dev/null
@@ -1,491 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TRenewDelegationTokenReq implements org.apache.thrift.TBase<TRenewDelegationTokenReq, TRenewDelegationTokenReq._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TRenewDelegationTokenReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField DELEGATION_TOKEN_FIELD_DESC = new org.apache.thrift.protocol.TField("delegationToken", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TRenewDelegationTokenReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TRenewDelegationTokenReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-  private String delegationToken; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle"),
-    DELEGATION_TOKEN((short)2, "delegationToken");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        case 2: // DELEGATION_TOKEN
-          return DELEGATION_TOKEN;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    tmpMap.put(_Fields.DELEGATION_TOKEN, new org.apache.thrift.meta_data.FieldMetaData("delegationToken", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TRenewDelegationTokenReq.class, metaDataMap);
-  }
-
-  public TRenewDelegationTokenReq() {
-  }
-
-  public TRenewDelegationTokenReq(
-    TSessionHandle sessionHandle,
-    String delegationToken)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-    this.delegationToken = delegationToken;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TRenewDelegationTokenReq(TRenewDelegationTokenReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-    if (other.isSetDelegationToken()) {
-      this.delegationToken = other.delegationToken;
-    }
-  }
-
-  public TRenewDelegationTokenReq deepCopy() {
-    return new TRenewDelegationTokenReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-    this.delegationToken = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public String getDelegationToken() {
-    return this.delegationToken;
-  }
-
-  public void setDelegationToken(String delegationToken) {
-    this.delegationToken = delegationToken;
-  }
-
-  public void unsetDelegationToken() {
-    this.delegationToken = null;
-  }
-
-  /** Returns true if field delegationToken is set (has been assigned a value) and false otherwise */
-  public boolean isSetDelegationToken() {
-    return this.delegationToken != null;
-  }
-
-  public void setDelegationTokenIsSet(boolean value) {
-    if (!value) {
-      this.delegationToken = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    case DELEGATION_TOKEN:
-      if (value == null) {
-        unsetDelegationToken();
-      } else {
-        setDelegationToken((String)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    case DELEGATION_TOKEN:
-      return getDelegationToken();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    case DELEGATION_TOKEN:
-      return isSetDelegationToken();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TRenewDelegationTokenReq)
-      return this.equals((TRenewDelegationTokenReq)that);
-    return false;
-  }
-
-  public boolean equals(TRenewDelegationTokenReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    boolean this_present_delegationToken = true && this.isSetDelegationToken();
-    boolean that_present_delegationToken = true && that.isSetDelegationToken();
-    if (this_present_delegationToken || that_present_delegationToken) {
-      if (!(this_present_delegationToken && that_present_delegationToken))
-        return false;
-      if (!this.delegationToken.equals(that.delegationToken))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    builder.append(present_sessionHandle);
-    if (present_sessionHandle)
-      builder.append(sessionHandle);
-
-    boolean present_delegationToken = true && (isSetDelegationToken());
-    builder.append(present_delegationToken);
-    if (present_delegationToken)
-      builder.append(delegationToken);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TRenewDelegationTokenReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TRenewDelegationTokenReq typedOther = (TRenewDelegationTokenReq)other;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(typedOther.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, typedOther.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetDelegationToken()).compareTo(typedOther.isSetDelegationToken());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetDelegationToken()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.delegationToken, typedOther.delegationToken);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TRenewDelegationTokenReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("delegationToken:");
-    if (this.delegationToken == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.delegationToken);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    if (!isSetDelegationToken()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'delegationToken' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TRenewDelegationTokenReqStandardSchemeFactory implements SchemeFactory {
-    public TRenewDelegationTokenReqStandardScheme getScheme() {
-      return new TRenewDelegationTokenReqStandardScheme();
-    }
-  }
-
-  private static class TRenewDelegationTokenReqStandardScheme extends StandardScheme<TRenewDelegationTokenReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TRenewDelegationTokenReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // DELEGATION_TOKEN
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.delegationToken = iprot.readString();
-              struct.setDelegationTokenIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TRenewDelegationTokenReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.delegationToken != null) {
-        oprot.writeFieldBegin(DELEGATION_TOKEN_FIELD_DESC);
-        oprot.writeString(struct.delegationToken);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TRenewDelegationTokenReqTupleSchemeFactory implements SchemeFactory {
-    public TRenewDelegationTokenReqTupleScheme getScheme() {
-      return new TRenewDelegationTokenReqTupleScheme();
-    }
-  }
-
-  private static class TRenewDelegationTokenReqTupleScheme extends TupleScheme<TRenewDelegationTokenReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TRenewDelegationTokenReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-      oprot.writeString(struct.delegationToken);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TRenewDelegationTokenReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-      struct.delegationToken = iprot.readString();
-      struct.setDelegationTokenIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TRenewDelegationTokenResp.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TRenewDelegationTokenResp.java
deleted file mode 100644
index 5f3eb6c4d4b90..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TRenewDelegationTokenResp.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TRenewDelegationTokenResp implements org.apache.thrift.TBase<TRenewDelegationTokenResp, TRenewDelegationTokenResp._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TRenewDelegationTokenResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TRenewDelegationTokenRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TRenewDelegationTokenRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TRenewDelegationTokenResp.class, metaDataMap);
-  }
-
-  public TRenewDelegationTokenResp() {
-  }
-
-  public TRenewDelegationTokenResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TRenewDelegationTokenResp(TRenewDelegationTokenResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-  }
-
-  public TRenewDelegationTokenResp deepCopy() {
-    return new TRenewDelegationTokenResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TRenewDelegationTokenResp)
-      return this.equals((TRenewDelegationTokenResp)that);
-    return false;
-  }
-
-  public boolean equals(TRenewDelegationTokenResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_status = true && (isSetStatus());
-    builder.append(present_status);
-    if (present_status)
-      builder.append(status);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TRenewDelegationTokenResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TRenewDelegationTokenResp typedOther = (TRenewDelegationTokenResp)other;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(typedOther.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, typedOther.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TRenewDelegationTokenResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TRenewDelegationTokenRespStandardSchemeFactory implements SchemeFactory {
-    public TRenewDelegationTokenRespStandardScheme getScheme() {
-      return new TRenewDelegationTokenRespStandardScheme();
-    }
-  }
-
-  private static class TRenewDelegationTokenRespStandardScheme extends StandardScheme<TRenewDelegationTokenResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TRenewDelegationTokenResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TRenewDelegationTokenResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TRenewDelegationTokenRespTupleSchemeFactory implements SchemeFactory {
-    public TRenewDelegationTokenRespTupleScheme getScheme() {
-      return new TRenewDelegationTokenRespTupleScheme();
-    }
-  }
-
-  private static class TRenewDelegationTokenRespTupleScheme extends TupleScheme<TRenewDelegationTokenResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TRenewDelegationTokenResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TRenewDelegationTokenResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TRow.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TRow.java
deleted file mode 100644
index a44cfb08ff01a..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TRow.java
+++ /dev/null
@@ -1,439 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TRow implements org.apache.thrift.TBase<TRow, TRow._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TRow");
-
-  private static final org.apache.thrift.protocol.TField COL_VALS_FIELD_DESC = new org.apache.thrift.protocol.TField("colVals", org.apache.thrift.protocol.TType.LIST, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TRowStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TRowTupleSchemeFactory());
-  }
-
-  private List<TColumnValue> colVals; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    COL_VALS((short)1, "colVals");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // COL_VALS
-          return COL_VALS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.COL_VALS, new org.apache.thrift.meta_data.FieldMetaData("colVals", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TColumnValue.class))));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TRow.class, metaDataMap);
-  }
-
-  public TRow() {
-  }
-
-  public TRow(
-    List<TColumnValue> colVals)
-  {
-    this();
-    this.colVals = colVals;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TRow(TRow other) {
-    if (other.isSetColVals()) {
-      List<TColumnValue> __this__colVals = new ArrayList<TColumnValue>();
-      for (TColumnValue other_element : other.colVals) {
-        __this__colVals.add(new TColumnValue(other_element));
-      }
-      this.colVals = __this__colVals;
-    }
-  }
-
-  public TRow deepCopy() {
-    return new TRow(this);
-  }
-
-  @Override
-  public void clear() {
-    this.colVals = null;
-  }
-
-  public int getColValsSize() {
-    return (this.colVals == null) ? 0 : this.colVals.size();
-  }
-
-  public java.util.Iterator<TColumnValue> getColValsIterator() {
-    return (this.colVals == null) ? null : this.colVals.iterator();
-  }
-
-  public void addToColVals(TColumnValue elem) {
-    if (this.colVals == null) {
-      this.colVals = new ArrayList<TColumnValue>();
-    }
-    this.colVals.add(elem);
-  }
-
-  public List<TColumnValue> getColVals() {
-    return this.colVals;
-  }
-
-  public void setColVals(List<TColumnValue> colVals) {
-    this.colVals = colVals;
-  }
-
-  public void unsetColVals() {
-    this.colVals = null;
-  }
-
-  /** Returns true if field colVals is set (has been assigned a value) and false otherwise */
-  public boolean isSetColVals() {
-    return this.colVals != null;
-  }
-
-  public void setColValsIsSet(boolean value) {
-    if (!value) {
-      this.colVals = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case COL_VALS:
-      if (value == null) {
-        unsetColVals();
-      } else {
-        setColVals((List<TColumnValue>)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case COL_VALS:
-      return getColVals();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case COL_VALS:
-      return isSetColVals();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TRow)
-      return this.equals((TRow)that);
-    return false;
-  }
-
-  public boolean equals(TRow that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_colVals = true && this.isSetColVals();
-    boolean that_present_colVals = true && that.isSetColVals();
-    if (this_present_colVals || that_present_colVals) {
-      if (!(this_present_colVals && that_present_colVals))
-        return false;
-      if (!this.colVals.equals(that.colVals))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_colVals = true && (isSetColVals());
-    builder.append(present_colVals);
-    if (present_colVals)
-      builder.append(colVals);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TRow other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TRow typedOther = (TRow)other;
-
-    lastComparison = Boolean.valueOf(isSetColVals()).compareTo(typedOther.isSetColVals());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetColVals()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.colVals, typedOther.colVals);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TRow(");
-    boolean first = true;
-
-    sb.append("colVals:");
-    if (this.colVals == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.colVals);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetColVals()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'colVals' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TRowStandardSchemeFactory implements SchemeFactory {
-    public TRowStandardScheme getScheme() {
-      return new TRowStandardScheme();
-    }
-  }
-
-  private static class TRowStandardScheme extends StandardScheme<TRow> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TRow struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // COL_VALS
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list46 = iprot.readListBegin();
-                struct.colVals = new ArrayList<TColumnValue>(_list46.size);
-                for (int _i47 = 0; _i47 < _list46.size; ++_i47)
-                {
-                  TColumnValue _elem48; // optional
-                  _elem48 = new TColumnValue();
-                  _elem48.read(iprot);
-                  struct.colVals.add(_elem48);
-                }
-                iprot.readListEnd();
-              }
-              struct.setColValsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TRow struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.colVals != null) {
-        oprot.writeFieldBegin(COL_VALS_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.colVals.size()));
-          for (TColumnValue _iter49 : struct.colVals)
-          {
-            _iter49.write(oprot);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TRowTupleSchemeFactory implements SchemeFactory {
-    public TRowTupleScheme getScheme() {
-      return new TRowTupleScheme();
-    }
-  }
-
-  private static class TRowTupleScheme extends TupleScheme<TRow> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TRow struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.colVals.size());
-        for (TColumnValue _iter50 : struct.colVals)
-        {
-          _iter50.write(oprot);
-        }
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TRow struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TList _list51 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-        struct.colVals = new ArrayList<TColumnValue>(_list51.size);
-        for (int _i52 = 0; _i52 < _list51.size; ++_i52)
-        {
-          TColumnValue _elem53; // optional
-          _elem53 = new TColumnValue();
-          _elem53.read(iprot);
-          struct.colVals.add(_elem53);
-        }
-      }
-      struct.setColValsIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TRowSet.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TRowSet.java
deleted file mode 100644
index d16c8a4bb32da..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TRowSet.java
+++ /dev/null
@@ -1,702 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TRowSet implements org.apache.thrift.TBase<TRowSet, TRowSet._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TRowSet");
-
-  private static final org.apache.thrift.protocol.TField START_ROW_OFFSET_FIELD_DESC = new org.apache.thrift.protocol.TField("startRowOffset", org.apache.thrift.protocol.TType.I64, (short)1);
-  private static final org.apache.thrift.protocol.TField ROWS_FIELD_DESC = new org.apache.thrift.protocol.TField("rows", org.apache.thrift.protocol.TType.LIST, (short)2);
-  private static final org.apache.thrift.protocol.TField COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("columns", org.apache.thrift.protocol.TType.LIST, (short)3);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TRowSetStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TRowSetTupleSchemeFactory());
-  }
-
-  private long startRowOffset; // required
-  private List<TRow> rows; // required
-  private List<TColumn> columns; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    START_ROW_OFFSET((short)1, "startRowOffset"),
-    ROWS((short)2, "rows"),
-    COLUMNS((short)3, "columns");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // START_ROW_OFFSET
-          return START_ROW_OFFSET;
-        case 2: // ROWS
-          return ROWS;
-        case 3: // COLUMNS
-          return COLUMNS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __STARTROWOFFSET_ISSET_ID = 0;
-  private byte __isset_bitfield = 0;
-  private _Fields optionals[] = {_Fields.COLUMNS};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.START_ROW_OFFSET, new org.apache.thrift.meta_data.FieldMetaData("startRowOffset", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
-    tmpMap.put(_Fields.ROWS, new org.apache.thrift.meta_data.FieldMetaData("rows", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TRow.class))));
-    tmpMap.put(_Fields.COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("columns", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TColumn.class))));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TRowSet.class, metaDataMap);
-  }
-
-  public TRowSet() {
-  }
-
-  public TRowSet(
-    long startRowOffset,
-    List<TRow> rows)
-  {
-    this();
-    this.startRowOffset = startRowOffset;
-    setStartRowOffsetIsSet(true);
-    this.rows = rows;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TRowSet(TRowSet other) {
-    __isset_bitfield = other.__isset_bitfield;
-    this.startRowOffset = other.startRowOffset;
-    if (other.isSetRows()) {
-      List<TRow> __this__rows = new ArrayList<TRow>();
-      for (TRow other_element : other.rows) {
-        __this__rows.add(new TRow(other_element));
-      }
-      this.rows = __this__rows;
-    }
-    if (other.isSetColumns()) {
-      List<TColumn> __this__columns = new ArrayList<TColumn>();
-      for (TColumn other_element : other.columns) {
-        __this__columns.add(new TColumn(other_element));
-      }
-      this.columns = __this__columns;
-    }
-  }
-
-  public TRowSet deepCopy() {
-    return new TRowSet(this);
-  }
-
-  @Override
-  public void clear() {
-    setStartRowOffsetIsSet(false);
-    this.startRowOffset = 0;
-    this.rows = null;
-    this.columns = null;
-  }
-
-  public long getStartRowOffset() {
-    return this.startRowOffset;
-  }
-
-  public void setStartRowOffset(long startRowOffset) {
-    this.startRowOffset = startRowOffset;
-    setStartRowOffsetIsSet(true);
-  }
-
-  public void unsetStartRowOffset() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __STARTROWOFFSET_ISSET_ID);
-  }
-
-  /** Returns true if field startRowOffset is set (has been assigned a value) and false otherwise */
-  public boolean isSetStartRowOffset() {
-    return EncodingUtils.testBit(__isset_bitfield, __STARTROWOFFSET_ISSET_ID);
-  }
-
-  public void setStartRowOffsetIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __STARTROWOFFSET_ISSET_ID, value);
-  }
-
-  public int getRowsSize() {
-    return (this.rows == null) ? 0 : this.rows.size();
-  }
-
-  public java.util.Iterator<TRow> getRowsIterator() {
-    return (this.rows == null) ? null : this.rows.iterator();
-  }
-
-  public void addToRows(TRow elem) {
-    if (this.rows == null) {
-      this.rows = new ArrayList<TRow>();
-    }
-    this.rows.add(elem);
-  }
-
-  public List<TRow> getRows() {
-    return this.rows;
-  }
-
-  public void setRows(List<TRow> rows) {
-    this.rows = rows;
-  }
-
-  public void unsetRows() {
-    this.rows = null;
-  }
-
-  /** Returns true if field rows is set (has been assigned a value) and false otherwise */
-  public boolean isSetRows() {
-    return this.rows != null;
-  }
-
-  public void setRowsIsSet(boolean value) {
-    if (!value) {
-      this.rows = null;
-    }
-  }
-
-  public int getColumnsSize() {
-    return (this.columns == null) ? 0 : this.columns.size();
-  }
-
-  public java.util.Iterator<TColumn> getColumnsIterator() {
-    return (this.columns == null) ? null : this.columns.iterator();
-  }
-
-  public void addToColumns(TColumn elem) {
-    if (this.columns == null) {
-      this.columns = new ArrayList<TColumn>();
-    }
-    this.columns.add(elem);
-  }
-
-  public List<TColumn> getColumns() {
-    return this.columns;
-  }
-
-  public void setColumns(List<TColumn> columns) {
-    this.columns = columns;
-  }
-
-  public void unsetColumns() {
-    this.columns = null;
-  }
-
-  /** Returns true if field columns is set (has been assigned a value) and false otherwise */
-  public boolean isSetColumns() {
-    return this.columns != null;
-  }
-
-  public void setColumnsIsSet(boolean value) {
-    if (!value) {
-      this.columns = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case START_ROW_OFFSET:
-      if (value == null) {
-        unsetStartRowOffset();
-      } else {
-        setStartRowOffset((Long)value);
-      }
-      break;
-
-    case ROWS:
-      if (value == null) {
-        unsetRows();
-      } else {
-        setRows((List<TRow>)value);
-      }
-      break;
-
-    case COLUMNS:
-      if (value == null) {
-        unsetColumns();
-      } else {
-        setColumns((List<TColumn>)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case START_ROW_OFFSET:
-      return Long.valueOf(getStartRowOffset());
-
-    case ROWS:
-      return getRows();
-
-    case COLUMNS:
-      return getColumns();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case START_ROW_OFFSET:
-      return isSetStartRowOffset();
-    case ROWS:
-      return isSetRows();
-    case COLUMNS:
-      return isSetColumns();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TRowSet)
-      return this.equals((TRowSet)that);
-    return false;
-  }
-
-  public boolean equals(TRowSet that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_startRowOffset = true;
-    boolean that_present_startRowOffset = true;
-    if (this_present_startRowOffset || that_present_startRowOffset) {
-      if (!(this_present_startRowOffset && that_present_startRowOffset))
-        return false;
-      if (this.startRowOffset != that.startRowOffset)
-        return false;
-    }
-
-    boolean this_present_rows = true && this.isSetRows();
-    boolean that_present_rows = true && that.isSetRows();
-    if (this_present_rows || that_present_rows) {
-      if (!(this_present_rows && that_present_rows))
-        return false;
-      if (!this.rows.equals(that.rows))
-        return false;
-    }
-
-    boolean this_present_columns = true && this.isSetColumns();
-    boolean that_present_columns = true && that.isSetColumns();
-    if (this_present_columns || that_present_columns) {
-      if (!(this_present_columns && that_present_columns))
-        return false;
-      if (!this.columns.equals(that.columns))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_startRowOffset = true;
-    builder.append(present_startRowOffset);
-    if (present_startRowOffset)
-      builder.append(startRowOffset);
-
-    boolean present_rows = true && (isSetRows());
-    builder.append(present_rows);
-    if (present_rows)
-      builder.append(rows);
-
-    boolean present_columns = true && (isSetColumns());
-    builder.append(present_columns);
-    if (present_columns)
-      builder.append(columns);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TRowSet other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TRowSet typedOther = (TRowSet)other;
-
-    lastComparison = Boolean.valueOf(isSetStartRowOffset()).compareTo(typedOther.isSetStartRowOffset());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStartRowOffset()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.startRowOffset, typedOther.startRowOffset);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetRows()).compareTo(typedOther.isSetRows());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetRows()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.rows, typedOther.rows);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetColumns()).compareTo(typedOther.isSetColumns());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetColumns()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columns, typedOther.columns);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TRowSet(");
-    boolean first = true;
-
-    sb.append("startRowOffset:");
-    sb.append(this.startRowOffset);
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("rows:");
-    if (this.rows == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.rows);
-    }
-    first = false;
-    if (isSetColumns()) {
-      if (!first) sb.append(", ");
-      sb.append("columns:");
-      if (this.columns == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.columns);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStartRowOffset()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'startRowOffset' is unset! Struct:" + toString());
-    }
-
-    if (!isSetRows()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'rows' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TRowSetStandardSchemeFactory implements SchemeFactory {
-    public TRowSetStandardScheme getScheme() {
-      return new TRowSetStandardScheme();
-    }
-  }
-
-  private static class TRowSetStandardScheme extends StandardScheme<TRowSet> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TRowSet struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // START_ROW_OFFSET
-            if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
-              struct.startRowOffset = iprot.readI64();
-              struct.setStartRowOffsetIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // ROWS
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list118 = iprot.readListBegin();
-                struct.rows = new ArrayList<TRow>(_list118.size);
-                for (int _i119 = 0; _i119 < _list118.size; ++_i119)
-                {
-                  TRow _elem120; // optional
-                  _elem120 = new TRow();
-                  _elem120.read(iprot);
-                  struct.rows.add(_elem120);
-                }
-                iprot.readListEnd();
-              }
-              struct.setRowsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // COLUMNS
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list121 = iprot.readListBegin();
-                struct.columns = new ArrayList<TColumn>(_list121.size);
-                for (int _i122 = 0; _i122 < _list121.size; ++_i122)
-                {
-                  TColumn _elem123; // optional
-                  _elem123 = new TColumn();
-                  _elem123.read(iprot);
-                  struct.columns.add(_elem123);
-                }
-                iprot.readListEnd();
-              }
-              struct.setColumnsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TRowSet struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      oprot.writeFieldBegin(START_ROW_OFFSET_FIELD_DESC);
-      oprot.writeI64(struct.startRowOffset);
-      oprot.writeFieldEnd();
-      if (struct.rows != null) {
-        oprot.writeFieldBegin(ROWS_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.rows.size()));
-          for (TRow _iter124 : struct.rows)
-          {
-            _iter124.write(oprot);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      if (struct.columns != null) {
-        if (struct.isSetColumns()) {
-          oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
-          {
-            oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.columns.size()));
-            for (TColumn _iter125 : struct.columns)
-            {
-              _iter125.write(oprot);
-            }
-            oprot.writeListEnd();
-          }
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TRowSetTupleSchemeFactory implements SchemeFactory {
-    public TRowSetTupleScheme getScheme() {
-      return new TRowSetTupleScheme();
-    }
-  }
-
-  private static class TRowSetTupleScheme extends TupleScheme<TRowSet> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TRowSet struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      oprot.writeI64(struct.startRowOffset);
-      {
-        oprot.writeI32(struct.rows.size());
-        for (TRow _iter126 : struct.rows)
-        {
-          _iter126.write(oprot);
-        }
-      }
-      BitSet optionals = new BitSet();
-      if (struct.isSetColumns()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetColumns()) {
-        {
-          oprot.writeI32(struct.columns.size());
-          for (TColumn _iter127 : struct.columns)
-          {
-            _iter127.write(oprot);
-          }
-        }
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TRowSet struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.startRowOffset = iprot.readI64();
-      struct.setStartRowOffsetIsSet(true);
-      {
-        org.apache.thrift.protocol.TList _list128 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-        struct.rows = new ArrayList<TRow>(_list128.size);
-        for (int _i129 = 0; _i129 < _list128.size; ++_i129)
-        {
-          TRow _elem130; // optional
-          _elem130 = new TRow();
-          _elem130.read(iprot);
-          struct.rows.add(_elem130);
-        }
-      }
-      struct.setRowsIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        {
-          org.apache.thrift.protocol.TList _list131 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-          struct.columns = new ArrayList<TColumn>(_list131.size);
-          for (int _i132 = 0; _i132 < _list131.size; ++_i132)
-          {
-            TColumn _elem133; // optional
-            _elem133 = new TColumn();
-            _elem133.read(iprot);
-            struct.columns.add(_elem133);
-          }
-        }
-        struct.setColumnsIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TSessionHandle.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TSessionHandle.java
deleted file mode 100644
index 82c00dd68a98b..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TSessionHandle.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TSessionHandle implements org.apache.thrift.TBase<TSessionHandle, TSessionHandle._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TSessionHandle");
-
-  private static final org.apache.thrift.protocol.TField SESSION_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionId", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TSessionHandleStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TSessionHandleTupleSchemeFactory());
-  }
-
-  private THandleIdentifier sessionId; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_ID((short)1, "sessionId");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_ID
-          return SESSION_ID;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_ID, new org.apache.thrift.meta_data.FieldMetaData("sessionId", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, THandleIdentifier.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TSessionHandle.class, metaDataMap);
-  }
-
-  public TSessionHandle() {
-  }
-
-  public TSessionHandle(
-    THandleIdentifier sessionId)
-  {
-    this();
-    this.sessionId = sessionId;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TSessionHandle(TSessionHandle other) {
-    if (other.isSetSessionId()) {
-      this.sessionId = new THandleIdentifier(other.sessionId);
-    }
-  }
-
-  public TSessionHandle deepCopy() {
-    return new TSessionHandle(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionId = null;
-  }
-
-  public THandleIdentifier getSessionId() {
-    return this.sessionId;
-  }
-
-  public void setSessionId(THandleIdentifier sessionId) {
-    this.sessionId = sessionId;
-  }
-
-  public void unsetSessionId() {
-    this.sessionId = null;
-  }
-
-  /** Returns true if field sessionId is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionId() {
-    return this.sessionId != null;
-  }
-
-  public void setSessionIdIsSet(boolean value) {
-    if (!value) {
-      this.sessionId = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_ID:
-      if (value == null) {
-        unsetSessionId();
-      } else {
-        setSessionId((THandleIdentifier)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_ID:
-      return getSessionId();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_ID:
-      return isSetSessionId();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TSessionHandle)
-      return this.equals((TSessionHandle)that);
-    return false;
-  }
-
-  public boolean equals(TSessionHandle that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionId = true && this.isSetSessionId();
-    boolean that_present_sessionId = true && that.isSetSessionId();
-    if (this_present_sessionId || that_present_sessionId) {
-      if (!(this_present_sessionId && that_present_sessionId))
-        return false;
-      if (!this.sessionId.equals(that.sessionId))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_sessionId = true && (isSetSessionId());
-    builder.append(present_sessionId);
-    if (present_sessionId)
-      builder.append(sessionId);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TSessionHandle other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TSessionHandle typedOther = (TSessionHandle)other;
-
-    lastComparison = Boolean.valueOf(isSetSessionId()).compareTo(typedOther.isSetSessionId());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionId()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionId, typedOther.sessionId);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TSessionHandle(");
-    boolean first = true;
-
-    sb.append("sessionId:");
-    if (this.sessionId == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionId);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionId()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionId' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionId != null) {
-      sessionId.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TSessionHandleStandardSchemeFactory implements SchemeFactory {
-    public TSessionHandleStandardScheme getScheme() {
-      return new TSessionHandleStandardScheme();
-    }
-  }
-
-  private static class TSessionHandleStandardScheme extends StandardScheme<TSessionHandle> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TSessionHandle struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_ID
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionId = new THandleIdentifier();
-              struct.sessionId.read(iprot);
-              struct.setSessionIdIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TSessionHandle struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionId != null) {
-        oprot.writeFieldBegin(SESSION_ID_FIELD_DESC);
-        struct.sessionId.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TSessionHandleTupleSchemeFactory implements SchemeFactory {
-    public TSessionHandleTupleScheme getScheme() {
-      return new TSessionHandleTupleScheme();
-    }
-  }
-
-  private static class TSessionHandleTupleScheme extends TupleScheme<TSessionHandle> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TSessionHandle struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionId.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TSessionHandle struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionId = new THandleIdentifier();
-      struct.sessionId.read(iprot);
-      struct.setSessionIdIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TStatus.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TStatus.java
deleted file mode 100644
index 24a746e94965d..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TStatus.java
+++ /dev/null
@@ -1,874 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TStatus implements org.apache.thrift.TBase<TStatus, TStatus._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TStatus");
-
-  private static final org.apache.thrift.protocol.TField STATUS_CODE_FIELD_DESC = new org.apache.thrift.protocol.TField("statusCode", org.apache.thrift.protocol.TType.I32, (short)1);
-  private static final org.apache.thrift.protocol.TField INFO_MESSAGES_FIELD_DESC = new org.apache.thrift.protocol.TField("infoMessages", org.apache.thrift.protocol.TType.LIST, (short)2);
-  private static final org.apache.thrift.protocol.TField SQL_STATE_FIELD_DESC = new org.apache.thrift.protocol.TField("sqlState", org.apache.thrift.protocol.TType.STRING, (short)3);
-  private static final org.apache.thrift.protocol.TField ERROR_CODE_FIELD_DESC = new org.apache.thrift.protocol.TField("errorCode", org.apache.thrift.protocol.TType.I32, (short)4);
-  private static final org.apache.thrift.protocol.TField ERROR_MESSAGE_FIELD_DESC = new org.apache.thrift.protocol.TField("errorMessage", org.apache.thrift.protocol.TType.STRING, (short)5);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TStatusStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TStatusTupleSchemeFactory());
-  }
-
-  private TStatusCode statusCode; // required
-  private List<String> infoMessages; // optional
-  private String sqlState; // optional
-  private int errorCode; // optional
-  private String errorMessage; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    /**
-     * 
-     * @see TStatusCode
-     */
-    STATUS_CODE((short)1, "statusCode"),
-    INFO_MESSAGES((short)2, "infoMessages"),
-    SQL_STATE((short)3, "sqlState"),
-    ERROR_CODE((short)4, "errorCode"),
-    ERROR_MESSAGE((short)5, "errorMessage");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS_CODE
-          return STATUS_CODE;
-        case 2: // INFO_MESSAGES
-          return INFO_MESSAGES;
-        case 3: // SQL_STATE
-          return SQL_STATE;
-        case 4: // ERROR_CODE
-          return ERROR_CODE;
-        case 5: // ERROR_MESSAGE
-          return ERROR_MESSAGE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __ERRORCODE_ISSET_ID = 0;
-  private byte __isset_bitfield = 0;
-  private _Fields optionals[] = {_Fields.INFO_MESSAGES,_Fields.SQL_STATE,_Fields.ERROR_CODE,_Fields.ERROR_MESSAGE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS_CODE, new org.apache.thrift.meta_data.FieldMetaData("statusCode", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TStatusCode.class)));
-    tmpMap.put(_Fields.INFO_MESSAGES, new org.apache.thrift.meta_data.FieldMetaData("infoMessages", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
-    tmpMap.put(_Fields.SQL_STATE, new org.apache.thrift.meta_data.FieldMetaData("sqlState", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    tmpMap.put(_Fields.ERROR_CODE, new org.apache.thrift.meta_data.FieldMetaData("errorCode", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
-    tmpMap.put(_Fields.ERROR_MESSAGE, new org.apache.thrift.meta_data.FieldMetaData("errorMessage", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TStatus.class, metaDataMap);
-  }
-
-  public TStatus() {
-  }
-
-  public TStatus(
-    TStatusCode statusCode)
-  {
-    this();
-    this.statusCode = statusCode;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TStatus(TStatus other) {
-    __isset_bitfield = other.__isset_bitfield;
-    if (other.isSetStatusCode()) {
-      this.statusCode = other.statusCode;
-    }
-    if (other.isSetInfoMessages()) {
-      List<String> __this__infoMessages = new ArrayList<String>();
-      for (String other_element : other.infoMessages) {
-        __this__infoMessages.add(other_element);
-      }
-      this.infoMessages = __this__infoMessages;
-    }
-    if (other.isSetSqlState()) {
-      this.sqlState = other.sqlState;
-    }
-    this.errorCode = other.errorCode;
-    if (other.isSetErrorMessage()) {
-      this.errorMessage = other.errorMessage;
-    }
-  }
-
-  public TStatus deepCopy() {
-    return new TStatus(this);
-  }
-
-  @Override
-  public void clear() {
-    this.statusCode = null;
-    this.infoMessages = null;
-    this.sqlState = null;
-    setErrorCodeIsSet(false);
-    this.errorCode = 0;
-    this.errorMessage = null;
-  }
-
-  /**
-   * 
-   * @see TStatusCode
-   */
-  public TStatusCode getStatusCode() {
-    return this.statusCode;
-  }
-
-  /**
-   * 
-   * @see TStatusCode
-   */
-  public void setStatusCode(TStatusCode statusCode) {
-    this.statusCode = statusCode;
-  }
-
-  public void unsetStatusCode() {
-    this.statusCode = null;
-  }
-
-  /** Returns true if field statusCode is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatusCode() {
-    return this.statusCode != null;
-  }
-
-  public void setStatusCodeIsSet(boolean value) {
-    if (!value) {
-      this.statusCode = null;
-    }
-  }
-
-  public int getInfoMessagesSize() {
-    return (this.infoMessages == null) ? 0 : this.infoMessages.size();
-  }
-
-  public java.util.Iterator<String> getInfoMessagesIterator() {
-    return (this.infoMessages == null) ? null : this.infoMessages.iterator();
-  }
-
-  public void addToInfoMessages(String elem) {
-    if (this.infoMessages == null) {
-      this.infoMessages = new ArrayList<String>();
-    }
-    this.infoMessages.add(elem);
-  }
-
-  public List<String> getInfoMessages() {
-    return this.infoMessages;
-  }
-
-  public void setInfoMessages(List<String> infoMessages) {
-    this.infoMessages = infoMessages;
-  }
-
-  public void unsetInfoMessages() {
-    this.infoMessages = null;
-  }
-
-  /** Returns true if field infoMessages is set (has been assigned a value) and false otherwise */
-  public boolean isSetInfoMessages() {
-    return this.infoMessages != null;
-  }
-
-  public void setInfoMessagesIsSet(boolean value) {
-    if (!value) {
-      this.infoMessages = null;
-    }
-  }
-
-  public String getSqlState() {
-    return this.sqlState;
-  }
-
-  public void setSqlState(String sqlState) {
-    this.sqlState = sqlState;
-  }
-
-  public void unsetSqlState() {
-    this.sqlState = null;
-  }
-
-  /** Returns true if field sqlState is set (has been assigned a value) and false otherwise */
-  public boolean isSetSqlState() {
-    return this.sqlState != null;
-  }
-
-  public void setSqlStateIsSet(boolean value) {
-    if (!value) {
-      this.sqlState = null;
-    }
-  }
-
-  public int getErrorCode() {
-    return this.errorCode;
-  }
-
-  public void setErrorCode(int errorCode) {
-    this.errorCode = errorCode;
-    setErrorCodeIsSet(true);
-  }
-
-  public void unsetErrorCode() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __ERRORCODE_ISSET_ID);
-  }
-
-  /** Returns true if field errorCode is set (has been assigned a value) and false otherwise */
-  public boolean isSetErrorCode() {
-    return EncodingUtils.testBit(__isset_bitfield, __ERRORCODE_ISSET_ID);
-  }
-
-  public void setErrorCodeIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __ERRORCODE_ISSET_ID, value);
-  }
-
-  public String getErrorMessage() {
-    return this.errorMessage;
-  }
-
-  public void setErrorMessage(String errorMessage) {
-    this.errorMessage = errorMessage;
-  }
-
-  public void unsetErrorMessage() {
-    this.errorMessage = null;
-  }
-
-  /** Returns true if field errorMessage is set (has been assigned a value) and false otherwise */
-  public boolean isSetErrorMessage() {
-    return this.errorMessage != null;
-  }
-
-  public void setErrorMessageIsSet(boolean value) {
-    if (!value) {
-      this.errorMessage = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS_CODE:
-      if (value == null) {
-        unsetStatusCode();
-      } else {
-        setStatusCode((TStatusCode)value);
-      }
-      break;
-
-    case INFO_MESSAGES:
-      if (value == null) {
-        unsetInfoMessages();
-      } else {
-        setInfoMessages((List<String>)value);
-      }
-      break;
-
-    case SQL_STATE:
-      if (value == null) {
-        unsetSqlState();
-      } else {
-        setSqlState((String)value);
-      }
-      break;
-
-    case ERROR_CODE:
-      if (value == null) {
-        unsetErrorCode();
-      } else {
-        setErrorCode((Integer)value);
-      }
-      break;
-
-    case ERROR_MESSAGE:
-      if (value == null) {
-        unsetErrorMessage();
-      } else {
-        setErrorMessage((String)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS_CODE:
-      return getStatusCode();
-
-    case INFO_MESSAGES:
-      return getInfoMessages();
-
-    case SQL_STATE:
-      return getSqlState();
-
-    case ERROR_CODE:
-      return Integer.valueOf(getErrorCode());
-
-    case ERROR_MESSAGE:
-      return getErrorMessage();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS_CODE:
-      return isSetStatusCode();
-    case INFO_MESSAGES:
-      return isSetInfoMessages();
-    case SQL_STATE:
-      return isSetSqlState();
-    case ERROR_CODE:
-      return isSetErrorCode();
-    case ERROR_MESSAGE:
-      return isSetErrorMessage();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TStatus)
-      return this.equals((TStatus)that);
-    return false;
-  }
-
-  public boolean equals(TStatus that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_statusCode = true && this.isSetStatusCode();
-    boolean that_present_statusCode = true && that.isSetStatusCode();
-    if (this_present_statusCode || that_present_statusCode) {
-      if (!(this_present_statusCode && that_present_statusCode))
-        return false;
-      if (!this.statusCode.equals(that.statusCode))
-        return false;
-    }
-
-    boolean this_present_infoMessages = true && this.isSetInfoMessages();
-    boolean that_present_infoMessages = true && that.isSetInfoMessages();
-    if (this_present_infoMessages || that_present_infoMessages) {
-      if (!(this_present_infoMessages && that_present_infoMessages))
-        return false;
-      if (!this.infoMessages.equals(that.infoMessages))
-        return false;
-    }
-
-    boolean this_present_sqlState = true && this.isSetSqlState();
-    boolean that_present_sqlState = true && that.isSetSqlState();
-    if (this_present_sqlState || that_present_sqlState) {
-      if (!(this_present_sqlState && that_present_sqlState))
-        return false;
-      if (!this.sqlState.equals(that.sqlState))
-        return false;
-    }
-
-    boolean this_present_errorCode = true && this.isSetErrorCode();
-    boolean that_present_errorCode = true && that.isSetErrorCode();
-    if (this_present_errorCode || that_present_errorCode) {
-      if (!(this_present_errorCode && that_present_errorCode))
-        return false;
-      if (this.errorCode != that.errorCode)
-        return false;
-    }
-
-    boolean this_present_errorMessage = true && this.isSetErrorMessage();
-    boolean that_present_errorMessage = true && that.isSetErrorMessage();
-    if (this_present_errorMessage || that_present_errorMessage) {
-      if (!(this_present_errorMessage && that_present_errorMessage))
-        return false;
-      if (!this.errorMessage.equals(that.errorMessage))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_statusCode = true && (isSetStatusCode());
-    builder.append(present_statusCode);
-    if (present_statusCode)
-      builder.append(statusCode.getValue());
-
-    boolean present_infoMessages = true && (isSetInfoMessages());
-    builder.append(present_infoMessages);
-    if (present_infoMessages)
-      builder.append(infoMessages);
-
-    boolean present_sqlState = true && (isSetSqlState());
-    builder.append(present_sqlState);
-    if (present_sqlState)
-      builder.append(sqlState);
-
-    boolean present_errorCode = true && (isSetErrorCode());
-    builder.append(present_errorCode);
-    if (present_errorCode)
-      builder.append(errorCode);
-
-    boolean present_errorMessage = true && (isSetErrorMessage());
-    builder.append(present_errorMessage);
-    if (present_errorMessage)
-      builder.append(errorMessage);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TStatus other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TStatus typedOther = (TStatus)other;
-
-    lastComparison = Boolean.valueOf(isSetStatusCode()).compareTo(typedOther.isSetStatusCode());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatusCode()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.statusCode, typedOther.statusCode);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetInfoMessages()).compareTo(typedOther.isSetInfoMessages());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetInfoMessages()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.infoMessages, typedOther.infoMessages);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetSqlState()).compareTo(typedOther.isSetSqlState());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSqlState()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sqlState, typedOther.sqlState);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetErrorCode()).compareTo(typedOther.isSetErrorCode());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetErrorCode()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.errorCode, typedOther.errorCode);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetErrorMessage()).compareTo(typedOther.isSetErrorMessage());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetErrorMessage()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.errorMessage, typedOther.errorMessage);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TStatus(");
-    boolean first = true;
-
-    sb.append("statusCode:");
-    if (this.statusCode == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.statusCode);
-    }
-    first = false;
-    if (isSetInfoMessages()) {
-      if (!first) sb.append(", ");
-      sb.append("infoMessages:");
-      if (this.infoMessages == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.infoMessages);
-      }
-      first = false;
-    }
-    if (isSetSqlState()) {
-      if (!first) sb.append(", ");
-      sb.append("sqlState:");
-      if (this.sqlState == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.sqlState);
-      }
-      first = false;
-    }
-    if (isSetErrorCode()) {
-      if (!first) sb.append(", ");
-      sb.append("errorCode:");
-      sb.append(this.errorCode);
-      first = false;
-    }
-    if (isSetErrorMessage()) {
-      if (!first) sb.append(", ");
-      sb.append("errorMessage:");
-      if (this.errorMessage == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.errorMessage);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatusCode()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'statusCode' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TStatusStandardSchemeFactory implements SchemeFactory {
-    public TStatusStandardScheme getScheme() {
-      return new TStatusStandardScheme();
-    }
-  }
-
-  private static class TStatusStandardScheme extends StandardScheme<TStatus> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TStatus struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS_CODE
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.statusCode = TStatusCode.findByValue(iprot.readI32());
-              struct.setStatusCodeIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // INFO_MESSAGES
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list134 = iprot.readListBegin();
-                struct.infoMessages = new ArrayList<String>(_list134.size);
-                for (int _i135 = 0; _i135 < _list134.size; ++_i135)
-                {
-                  String _elem136; // optional
-                  _elem136 = iprot.readString();
-                  struct.infoMessages.add(_elem136);
-                }
-                iprot.readListEnd();
-              }
-              struct.setInfoMessagesIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // SQL_STATE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.sqlState = iprot.readString();
-              struct.setSqlStateIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // ERROR_CODE
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.errorCode = iprot.readI32();
-              struct.setErrorCodeIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 5: // ERROR_MESSAGE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.errorMessage = iprot.readString();
-              struct.setErrorMessageIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TStatus struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.statusCode != null) {
-        oprot.writeFieldBegin(STATUS_CODE_FIELD_DESC);
-        oprot.writeI32(struct.statusCode.getValue());
-        oprot.writeFieldEnd();
-      }
-      if (struct.infoMessages != null) {
-        if (struct.isSetInfoMessages()) {
-          oprot.writeFieldBegin(INFO_MESSAGES_FIELD_DESC);
-          {
-            oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.infoMessages.size()));
-            for (String _iter137 : struct.infoMessages)
-            {
-              oprot.writeString(_iter137);
-            }
-            oprot.writeListEnd();
-          }
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.sqlState != null) {
-        if (struct.isSetSqlState()) {
-          oprot.writeFieldBegin(SQL_STATE_FIELD_DESC);
-          oprot.writeString(struct.sqlState);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.isSetErrorCode()) {
-        oprot.writeFieldBegin(ERROR_CODE_FIELD_DESC);
-        oprot.writeI32(struct.errorCode);
-        oprot.writeFieldEnd();
-      }
-      if (struct.errorMessage != null) {
-        if (struct.isSetErrorMessage()) {
-          oprot.writeFieldBegin(ERROR_MESSAGE_FIELD_DESC);
-          oprot.writeString(struct.errorMessage);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TStatusTupleSchemeFactory implements SchemeFactory {
-    public TStatusTupleScheme getScheme() {
-      return new TStatusTupleScheme();
-    }
-  }
-
-  private static class TStatusTupleScheme extends TupleScheme<TStatus> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TStatus struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      oprot.writeI32(struct.statusCode.getValue());
-      BitSet optionals = new BitSet();
-      if (struct.isSetInfoMessages()) {
-        optionals.set(0);
-      }
-      if (struct.isSetSqlState()) {
-        optionals.set(1);
-      }
-      if (struct.isSetErrorCode()) {
-        optionals.set(2);
-      }
-      if (struct.isSetErrorMessage()) {
-        optionals.set(3);
-      }
-      oprot.writeBitSet(optionals, 4);
-      if (struct.isSetInfoMessages()) {
-        {
-          oprot.writeI32(struct.infoMessages.size());
-          for (String _iter138 : struct.infoMessages)
-          {
-            oprot.writeString(_iter138);
-          }
-        }
-      }
-      if (struct.isSetSqlState()) {
-        oprot.writeString(struct.sqlState);
-      }
-      if (struct.isSetErrorCode()) {
-        oprot.writeI32(struct.errorCode);
-      }
-      if (struct.isSetErrorMessage()) {
-        oprot.writeString(struct.errorMessage);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TStatus struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.statusCode = TStatusCode.findByValue(iprot.readI32());
-      struct.setStatusCodeIsSet(true);
-      BitSet incoming = iprot.readBitSet(4);
-      if (incoming.get(0)) {
-        {
-          org.apache.thrift.protocol.TList _list139 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-          struct.infoMessages = new ArrayList<String>(_list139.size);
-          for (int _i140 = 0; _i140 < _list139.size; ++_i140)
-          {
-            String _elem141; // optional
-            _elem141 = iprot.readString();
-            struct.infoMessages.add(_elem141);
-          }
-        }
-        struct.setInfoMessagesIsSet(true);
-      }
-      if (incoming.get(1)) {
-        struct.sqlState = iprot.readString();
-        struct.setSqlStateIsSet(true);
-      }
-      if (incoming.get(2)) {
-        struct.errorCode = iprot.readI32();
-        struct.setErrorCodeIsSet(true);
-      }
-      if (incoming.get(3)) {
-        struct.errorMessage = iprot.readString();
-        struct.setErrorMessageIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TStatusCode.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TStatusCode.java
deleted file mode 100644
index e7fde45fd131a..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TStatusCode.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.thrift.TEnum;
-
-public enum TStatusCode implements org.apache.thrift.TEnum {
-  SUCCESS_STATUS(0),
-  SUCCESS_WITH_INFO_STATUS(1),
-  STILL_EXECUTING_STATUS(2),
-  ERROR_STATUS(3),
-  INVALID_HANDLE_STATUS(4);
-
-  private final int value;
-
-  private TStatusCode(int value) {
-    this.value = value;
-  }
-
-  /**
-   * Get the integer value of this enum value, as defined in the Thrift IDL.
-   */
-  public int getValue() {
-    return value;
-  }
-
-  /**
-   * Find a the enum type by its integer value, as defined in the Thrift IDL.
-   * @return null if the value is not found.
-   */
-  public static TStatusCode findByValue(int value) { 
-    switch (value) {
-      case 0:
-        return SUCCESS_STATUS;
-      case 1:
-        return SUCCESS_WITH_INFO_STATUS;
-      case 2:
-        return STILL_EXECUTING_STATUS;
-      case 3:
-        return ERROR_STATUS;
-      case 4:
-        return INVALID_HANDLE_STATUS;
-      default:
-        return null;
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TStringColumn.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TStringColumn.java
deleted file mode 100644
index 3dae460c8621d..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TStringColumn.java
+++ /dev/null
@@ -1,548 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TStringColumn implements org.apache.thrift.TBase<TStringColumn, TStringColumn._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TStringColumn");
-
-  private static final org.apache.thrift.protocol.TField VALUES_FIELD_DESC = new org.apache.thrift.protocol.TField("values", org.apache.thrift.protocol.TType.LIST, (short)1);
-  private static final org.apache.thrift.protocol.TField NULLS_FIELD_DESC = new org.apache.thrift.protocol.TField("nulls", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TStringColumnStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TStringColumnTupleSchemeFactory());
-  }
-
-  private List<String> values; // required
-  private ByteBuffer nulls; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUES((short)1, "values"),
-    NULLS((short)2, "nulls");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUES
-          return VALUES;
-        case 2: // NULLS
-          return NULLS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUES, new org.apache.thrift.meta_data.FieldMetaData("values", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
-    tmpMap.put(_Fields.NULLS, new org.apache.thrift.meta_data.FieldMetaData("nulls", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TStringColumn.class, metaDataMap);
-  }
-
-  public TStringColumn() {
-  }
-
-  public TStringColumn(
-    List<String> values,
-    ByteBuffer nulls)
-  {
-    this();
-    this.values = values;
-    this.nulls = nulls;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TStringColumn(TStringColumn other) {
-    if (other.isSetValues()) {
-      List<String> __this__values = new ArrayList<String>();
-      for (String other_element : other.values) {
-        __this__values.add(other_element);
-      }
-      this.values = __this__values;
-    }
-    if (other.isSetNulls()) {
-      this.nulls = org.apache.thrift.TBaseHelper.copyBinary(other.nulls);
-;
-    }
-  }
-
-  public TStringColumn deepCopy() {
-    return new TStringColumn(this);
-  }
-
-  @Override
-  public void clear() {
-    this.values = null;
-    this.nulls = null;
-  }
-
-  public int getValuesSize() {
-    return (this.values == null) ? 0 : this.values.size();
-  }
-
-  public java.util.Iterator<String> getValuesIterator() {
-    return (this.values == null) ? null : this.values.iterator();
-  }
-
-  public void addToValues(String elem) {
-    if (this.values == null) {
-      this.values = new ArrayList<String>();
-    }
-    this.values.add(elem);
-  }
-
-  public List<String> getValues() {
-    return this.values;
-  }
-
-  public void setValues(List<String> values) {
-    this.values = values;
-  }
-
-  public void unsetValues() {
-    this.values = null;
-  }
-
-  /** Returns true if field values is set (has been assigned a value) and false otherwise */
-  public boolean isSetValues() {
-    return this.values != null;
-  }
-
-  public void setValuesIsSet(boolean value) {
-    if (!value) {
-      this.values = null;
-    }
-  }
-
-  public byte[] getNulls() {
-    setNulls(org.apache.thrift.TBaseHelper.rightSize(nulls));
-    return nulls == null ? null : nulls.array();
-  }
-
-  public ByteBuffer bufferForNulls() {
-    return nulls;
-  }
-
-  public void setNulls(byte[] nulls) {
-    setNulls(nulls == null ? (ByteBuffer)null : ByteBuffer.wrap(nulls));
-  }
-
-  public void setNulls(ByteBuffer nulls) {
-    this.nulls = nulls;
-  }
-
-  public void unsetNulls() {
-    this.nulls = null;
-  }
-
-  /** Returns true if field nulls is set (has been assigned a value) and false otherwise */
-  public boolean isSetNulls() {
-    return this.nulls != null;
-  }
-
-  public void setNullsIsSet(boolean value) {
-    if (!value) {
-      this.nulls = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUES:
-      if (value == null) {
-        unsetValues();
-      } else {
-        setValues((List<String>)value);
-      }
-      break;
-
-    case NULLS:
-      if (value == null) {
-        unsetNulls();
-      } else {
-        setNulls((ByteBuffer)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUES:
-      return getValues();
-
-    case NULLS:
-      return getNulls();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUES:
-      return isSetValues();
-    case NULLS:
-      return isSetNulls();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TStringColumn)
-      return this.equals((TStringColumn)that);
-    return false;
-  }
-
-  public boolean equals(TStringColumn that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_values = true && this.isSetValues();
-    boolean that_present_values = true && that.isSetValues();
-    if (this_present_values || that_present_values) {
-      if (!(this_present_values && that_present_values))
-        return false;
-      if (!this.values.equals(that.values))
-        return false;
-    }
-
-    boolean this_present_nulls = true && this.isSetNulls();
-    boolean that_present_nulls = true && that.isSetNulls();
-    if (this_present_nulls || that_present_nulls) {
-      if (!(this_present_nulls && that_present_nulls))
-        return false;
-      if (!this.nulls.equals(that.nulls))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_values = true && (isSetValues());
-    builder.append(present_values);
-    if (present_values)
-      builder.append(values);
-
-    boolean present_nulls = true && (isSetNulls());
-    builder.append(present_nulls);
-    if (present_nulls)
-      builder.append(nulls);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TStringColumn other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TStringColumn typedOther = (TStringColumn)other;
-
-    lastComparison = Boolean.valueOf(isSetValues()).compareTo(typedOther.isSetValues());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValues()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.values, typedOther.values);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetNulls()).compareTo(typedOther.isSetNulls());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetNulls()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.nulls, typedOther.nulls);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TStringColumn(");
-    boolean first = true;
-
-    sb.append("values:");
-    if (this.values == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.values);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("nulls:");
-    if (this.nulls == null) {
-      sb.append("null");
-    } else {
-      org.apache.thrift.TBaseHelper.toString(this.nulls, sb);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetValues()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'values' is unset! Struct:" + toString());
-    }
-
-    if (!isSetNulls()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'nulls' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TStringColumnStandardSchemeFactory implements SchemeFactory {
-    public TStringColumnStandardScheme getScheme() {
-      return new TStringColumnStandardScheme();
-    }
-  }
-
-  private static class TStringColumnStandardScheme extends StandardScheme<TStringColumn> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TStringColumn struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUES
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list102 = iprot.readListBegin();
-                struct.values = new ArrayList<String>(_list102.size);
-                for (int _i103 = 0; _i103 < _list102.size; ++_i103)
-                {
-                  String _elem104; // optional
-                  _elem104 = iprot.readString();
-                  struct.values.add(_elem104);
-                }
-                iprot.readListEnd();
-              }
-              struct.setValuesIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // NULLS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.nulls = iprot.readBinary();
-              struct.setNullsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TStringColumn struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.values != null) {
-        oprot.writeFieldBegin(VALUES_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.values.size()));
-          for (String _iter105 : struct.values)
-          {
-            oprot.writeString(_iter105);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      if (struct.nulls != null) {
-        oprot.writeFieldBegin(NULLS_FIELD_DESC);
-        oprot.writeBinary(struct.nulls);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TStringColumnTupleSchemeFactory implements SchemeFactory {
-    public TStringColumnTupleScheme getScheme() {
-      return new TStringColumnTupleScheme();
-    }
-  }
-
-  private static class TStringColumnTupleScheme extends TupleScheme<TStringColumn> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TStringColumn struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.values.size());
-        for (String _iter106 : struct.values)
-        {
-          oprot.writeString(_iter106);
-        }
-      }
-      oprot.writeBinary(struct.nulls);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TStringColumn struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TList _list107 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-        struct.values = new ArrayList<String>(_list107.size);
-        for (int _i108 = 0; _i108 < _list107.size; ++_i108)
-        {
-          String _elem109; // optional
-          _elem109 = iprot.readString();
-          struct.values.add(_elem109);
-        }
-      }
-      struct.setValuesIsSet(true);
-      struct.nulls = iprot.readBinary();
-      struct.setNullsIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TStringValue.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TStringValue.java
deleted file mode 100644
index af7a109775a8b..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TStringValue.java
+++ /dev/null
@@ -1,389 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TStringValue implements org.apache.thrift.TBase<TStringValue, TStringValue._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TStringValue");
-
-  private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.STRING, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TStringValueStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TStringValueTupleSchemeFactory());
-  }
-
-  private String value; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUE((short)1, "value");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUE
-          return VALUE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private _Fields optionals[] = {_Fields.VALUE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TStringValue.class, metaDataMap);
-  }
-
-  public TStringValue() {
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TStringValue(TStringValue other) {
-    if (other.isSetValue()) {
-      this.value = other.value;
-    }
-  }
-
-  public TStringValue deepCopy() {
-    return new TStringValue(this);
-  }
-
-  @Override
-  public void clear() {
-    this.value = null;
-  }
-
-  public String getValue() {
-    return this.value;
-  }
-
-  public void setValue(String value) {
-    this.value = value;
-  }
-
-  public void unsetValue() {
-    this.value = null;
-  }
-
-  /** Returns true if field value is set (has been assigned a value) and false otherwise */
-  public boolean isSetValue() {
-    return this.value != null;
-  }
-
-  public void setValueIsSet(boolean value) {
-    if (!value) {
-      this.value = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUE:
-      if (value == null) {
-        unsetValue();
-      } else {
-        setValue((String)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUE:
-      return getValue();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUE:
-      return isSetValue();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TStringValue)
-      return this.equals((TStringValue)that);
-    return false;
-  }
-
-  public boolean equals(TStringValue that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_value = true && this.isSetValue();
-    boolean that_present_value = true && that.isSetValue();
-    if (this_present_value || that_present_value) {
-      if (!(this_present_value && that_present_value))
-        return false;
-      if (!this.value.equals(that.value))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_value = true && (isSetValue());
-    builder.append(present_value);
-    if (present_value)
-      builder.append(value);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TStringValue other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TStringValue typedOther = (TStringValue)other;
-
-    lastComparison = Boolean.valueOf(isSetValue()).compareTo(typedOther.isSetValue());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValue()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, typedOther.value);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TStringValue(");
-    boolean first = true;
-
-    if (isSetValue()) {
-      sb.append("value:");
-      if (this.value == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.value);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TStringValueStandardSchemeFactory implements SchemeFactory {
-    public TStringValueStandardScheme getScheme() {
-      return new TStringValueStandardScheme();
-    }
-  }
-
-  private static class TStringValueStandardScheme extends StandardScheme<TStringValue> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TStringValue struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.value = iprot.readString();
-              struct.setValueIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TStringValue struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.value != null) {
-        if (struct.isSetValue()) {
-          oprot.writeFieldBegin(VALUE_FIELD_DESC);
-          oprot.writeString(struct.value);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TStringValueTupleSchemeFactory implements SchemeFactory {
-    public TStringValueTupleScheme getScheme() {
-      return new TStringValueTupleScheme();
-    }
-  }
-
-  private static class TStringValueTupleScheme extends TupleScheme<TStringValue> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TStringValue struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      BitSet optionals = new BitSet();
-      if (struct.isSetValue()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetValue()) {
-        oprot.writeString(struct.value);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TStringValue struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.value = iprot.readString();
-        struct.setValueIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TStructTypeEntry.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TStructTypeEntry.java
deleted file mode 100644
index 20f5fb6c29073..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TStructTypeEntry.java
+++ /dev/null
@@ -1,448 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TStructTypeEntry implements org.apache.thrift.TBase<TStructTypeEntry, TStructTypeEntry._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TStructTypeEntry");
-
-  private static final org.apache.thrift.protocol.TField NAME_TO_TYPE_PTR_FIELD_DESC = new org.apache.thrift.protocol.TField("nameToTypePtr", org.apache.thrift.protocol.TType.MAP, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TStructTypeEntryStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TStructTypeEntryTupleSchemeFactory());
-  }
-
-  private Map<String,Integer> nameToTypePtr; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    NAME_TO_TYPE_PTR((short)1, "nameToTypePtr");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // NAME_TO_TYPE_PTR
-          return NAME_TO_TYPE_PTR;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.NAME_TO_TYPE_PTR, new org.apache.thrift.meta_data.FieldMetaData("nameToTypePtr", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING), 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32            , "TTypeEntryPtr"))));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TStructTypeEntry.class, metaDataMap);
-  }
-
-  public TStructTypeEntry() {
-  }
-
-  public TStructTypeEntry(
-    Map<String,Integer> nameToTypePtr)
-  {
-    this();
-    this.nameToTypePtr = nameToTypePtr;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TStructTypeEntry(TStructTypeEntry other) {
-    if (other.isSetNameToTypePtr()) {
-      Map<String,Integer> __this__nameToTypePtr = new HashMap<String,Integer>();
-      for (Map.Entry<String, Integer> other_element : other.nameToTypePtr.entrySet()) {
-
-        String other_element_key = other_element.getKey();
-        Integer other_element_value = other_element.getValue();
-
-        String __this__nameToTypePtr_copy_key = other_element_key;
-
-        Integer __this__nameToTypePtr_copy_value = other_element_value;
-
-        __this__nameToTypePtr.put(__this__nameToTypePtr_copy_key, __this__nameToTypePtr_copy_value);
-      }
-      this.nameToTypePtr = __this__nameToTypePtr;
-    }
-  }
-
-  public TStructTypeEntry deepCopy() {
-    return new TStructTypeEntry(this);
-  }
-
-  @Override
-  public void clear() {
-    this.nameToTypePtr = null;
-  }
-
-  public int getNameToTypePtrSize() {
-    return (this.nameToTypePtr == null) ? 0 : this.nameToTypePtr.size();
-  }
-
-  public void putToNameToTypePtr(String key, int val) {
-    if (this.nameToTypePtr == null) {
-      this.nameToTypePtr = new HashMap<String,Integer>();
-    }
-    this.nameToTypePtr.put(key, val);
-  }
-
-  public Map<String,Integer> getNameToTypePtr() {
-    return this.nameToTypePtr;
-  }
-
-  public void setNameToTypePtr(Map<String,Integer> nameToTypePtr) {
-    this.nameToTypePtr = nameToTypePtr;
-  }
-
-  public void unsetNameToTypePtr() {
-    this.nameToTypePtr = null;
-  }
-
-  /** Returns true if field nameToTypePtr is set (has been assigned a value) and false otherwise */
-  public boolean isSetNameToTypePtr() {
-    return this.nameToTypePtr != null;
-  }
-
-  public void setNameToTypePtrIsSet(boolean value) {
-    if (!value) {
-      this.nameToTypePtr = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case NAME_TO_TYPE_PTR:
-      if (value == null) {
-        unsetNameToTypePtr();
-      } else {
-        setNameToTypePtr((Map<String,Integer>)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case NAME_TO_TYPE_PTR:
-      return getNameToTypePtr();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case NAME_TO_TYPE_PTR:
-      return isSetNameToTypePtr();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TStructTypeEntry)
-      return this.equals((TStructTypeEntry)that);
-    return false;
-  }
-
-  public boolean equals(TStructTypeEntry that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_nameToTypePtr = true && this.isSetNameToTypePtr();
-    boolean that_present_nameToTypePtr = true && that.isSetNameToTypePtr();
-    if (this_present_nameToTypePtr || that_present_nameToTypePtr) {
-      if (!(this_present_nameToTypePtr && that_present_nameToTypePtr))
-        return false;
-      if (!this.nameToTypePtr.equals(that.nameToTypePtr))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_nameToTypePtr = true && (isSetNameToTypePtr());
-    builder.append(present_nameToTypePtr);
-    if (present_nameToTypePtr)
-      builder.append(nameToTypePtr);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TStructTypeEntry other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TStructTypeEntry typedOther = (TStructTypeEntry)other;
-
-    lastComparison = Boolean.valueOf(isSetNameToTypePtr()).compareTo(typedOther.isSetNameToTypePtr());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetNameToTypePtr()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.nameToTypePtr, typedOther.nameToTypePtr);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TStructTypeEntry(");
-    boolean first = true;
-
-    sb.append("nameToTypePtr:");
-    if (this.nameToTypePtr == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.nameToTypePtr);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetNameToTypePtr()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'nameToTypePtr' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TStructTypeEntryStandardSchemeFactory implements SchemeFactory {
-    public TStructTypeEntryStandardScheme getScheme() {
-      return new TStructTypeEntryStandardScheme();
-    }
-  }
-
-  private static class TStructTypeEntryStandardScheme extends StandardScheme<TStructTypeEntry> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TStructTypeEntry struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // NAME_TO_TYPE_PTR
-            if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
-              {
-                org.apache.thrift.protocol.TMap _map10 = iprot.readMapBegin();
-                struct.nameToTypePtr = new HashMap<String,Integer>(2*_map10.size);
-                for (int _i11 = 0; _i11 < _map10.size; ++_i11)
-                {
-                  String _key12; // required
-                  int _val13; // required
-                  _key12 = iprot.readString();
-                  _val13 = iprot.readI32();
-                  struct.nameToTypePtr.put(_key12, _val13);
-                }
-                iprot.readMapEnd();
-              }
-              struct.setNameToTypePtrIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TStructTypeEntry struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.nameToTypePtr != null) {
-        oprot.writeFieldBegin(NAME_TO_TYPE_PTR_FIELD_DESC);
-        {
-          oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.I32, struct.nameToTypePtr.size()));
-          for (Map.Entry<String, Integer> _iter14 : struct.nameToTypePtr.entrySet())
-          {
-            oprot.writeString(_iter14.getKey());
-            oprot.writeI32(_iter14.getValue());
-          }
-          oprot.writeMapEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TStructTypeEntryTupleSchemeFactory implements SchemeFactory {
-    public TStructTypeEntryTupleScheme getScheme() {
-      return new TStructTypeEntryTupleScheme();
-    }
-  }
-
-  private static class TStructTypeEntryTupleScheme extends TupleScheme<TStructTypeEntry> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TStructTypeEntry struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.nameToTypePtr.size());
-        for (Map.Entry<String, Integer> _iter15 : struct.nameToTypePtr.entrySet())
-        {
-          oprot.writeString(_iter15.getKey());
-          oprot.writeI32(_iter15.getValue());
-        }
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TStructTypeEntry struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TMap _map16 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.I32, iprot.readI32());
-        struct.nameToTypePtr = new HashMap<String,Integer>(2*_map16.size);
-        for (int _i17 = 0; _i17 < _map16.size; ++_i17)
-        {
-          String _key18; // required
-          int _val19; // required
-          _key18 = iprot.readString();
-          _val19 = iprot.readI32();
-          struct.nameToTypePtr.put(_key18, _val19);
-        }
-      }
-      struct.setNameToTypePtrIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TTableSchema.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TTableSchema.java
deleted file mode 100644
index ff5e54db7c16c..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TTableSchema.java
+++ /dev/null
@@ -1,439 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TTableSchema implements org.apache.thrift.TBase<TTableSchema, TTableSchema._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TTableSchema");
-
-  private static final org.apache.thrift.protocol.TField COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("columns", org.apache.thrift.protocol.TType.LIST, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TTableSchemaStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TTableSchemaTupleSchemeFactory());
-  }
-
-  private List<TColumnDesc> columns; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    COLUMNS((short)1, "columns");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // COLUMNS
-          return COLUMNS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("columns", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TColumnDesc.class))));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TTableSchema.class, metaDataMap);
-  }
-
-  public TTableSchema() {
-  }
-
-  public TTableSchema(
-    List<TColumnDesc> columns)
-  {
-    this();
-    this.columns = columns;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TTableSchema(TTableSchema other) {
-    if (other.isSetColumns()) {
-      List<TColumnDesc> __this__columns = new ArrayList<TColumnDesc>();
-      for (TColumnDesc other_element : other.columns) {
-        __this__columns.add(new TColumnDesc(other_element));
-      }
-      this.columns = __this__columns;
-    }
-  }
-
-  public TTableSchema deepCopy() {
-    return new TTableSchema(this);
-  }
-
-  @Override
-  public void clear() {
-    this.columns = null;
-  }
-
-  public int getColumnsSize() {
-    return (this.columns == null) ? 0 : this.columns.size();
-  }
-
-  public java.util.Iterator<TColumnDesc> getColumnsIterator() {
-    return (this.columns == null) ? null : this.columns.iterator();
-  }
-
-  public void addToColumns(TColumnDesc elem) {
-    if (this.columns == null) {
-      this.columns = new ArrayList<TColumnDesc>();
-    }
-    this.columns.add(elem);
-  }
-
-  public List<TColumnDesc> getColumns() {
-    return this.columns;
-  }
-
-  public void setColumns(List<TColumnDesc> columns) {
-    this.columns = columns;
-  }
-
-  public void unsetColumns() {
-    this.columns = null;
-  }
-
-  /** Returns true if field columns is set (has been assigned a value) and false otherwise */
-  public boolean isSetColumns() {
-    return this.columns != null;
-  }
-
-  public void setColumnsIsSet(boolean value) {
-    if (!value) {
-      this.columns = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case COLUMNS:
-      if (value == null) {
-        unsetColumns();
-      } else {
-        setColumns((List<TColumnDesc>)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case COLUMNS:
-      return getColumns();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case COLUMNS:
-      return isSetColumns();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TTableSchema)
-      return this.equals((TTableSchema)that);
-    return false;
-  }
-
-  public boolean equals(TTableSchema that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_columns = true && this.isSetColumns();
-    boolean that_present_columns = true && that.isSetColumns();
-    if (this_present_columns || that_present_columns) {
-      if (!(this_present_columns && that_present_columns))
-        return false;
-      if (!this.columns.equals(that.columns))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_columns = true && (isSetColumns());
-    builder.append(present_columns);
-    if (present_columns)
-      builder.append(columns);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TTableSchema other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TTableSchema typedOther = (TTableSchema)other;
-
-    lastComparison = Boolean.valueOf(isSetColumns()).compareTo(typedOther.isSetColumns());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetColumns()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columns, typedOther.columns);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TTableSchema(");
-    boolean first = true;
-
-    sb.append("columns:");
-    if (this.columns == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.columns);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetColumns()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'columns' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TTableSchemaStandardSchemeFactory implements SchemeFactory {
-    public TTableSchemaStandardScheme getScheme() {
-      return new TTableSchemaStandardScheme();
-    }
-  }
-
-  private static class TTableSchemaStandardScheme extends StandardScheme<TTableSchema> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TTableSchema struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // COLUMNS
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list38 = iprot.readListBegin();
-                struct.columns = new ArrayList<TColumnDesc>(_list38.size);
-                for (int _i39 = 0; _i39 < _list38.size; ++_i39)
-                {
-                  TColumnDesc _elem40; // optional
-                  _elem40 = new TColumnDesc();
-                  _elem40.read(iprot);
-                  struct.columns.add(_elem40);
-                }
-                iprot.readListEnd();
-              }
-              struct.setColumnsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TTableSchema struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.columns != null) {
-        oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.columns.size()));
-          for (TColumnDesc _iter41 : struct.columns)
-          {
-            _iter41.write(oprot);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TTableSchemaTupleSchemeFactory implements SchemeFactory {
-    public TTableSchemaTupleScheme getScheme() {
-      return new TTableSchemaTupleScheme();
-    }
-  }
-
-  private static class TTableSchemaTupleScheme extends TupleScheme<TTableSchema> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TTableSchema struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.columns.size());
-        for (TColumnDesc _iter42 : struct.columns)
-        {
-          _iter42.write(oprot);
-        }
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TTableSchema struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TList _list43 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-        struct.columns = new ArrayList<TColumnDesc>(_list43.size);
-        for (int _i44 = 0; _i44 < _list43.size; ++_i44)
-        {
-          TColumnDesc _elem45; // optional
-          _elem45 = new TColumnDesc();
-          _elem45.read(iprot);
-          struct.columns.add(_elem45);
-        }
-      }
-      struct.setColumnsIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TTypeDesc.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TTypeDesc.java
deleted file mode 100644
index 251f86a914719..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TTypeDesc.java
+++ /dev/null
@@ -1,439 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TTypeDesc implements org.apache.thrift.TBase<TTypeDesc, TTypeDesc._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TTypeDesc");
-
-  private static final org.apache.thrift.protocol.TField TYPES_FIELD_DESC = new org.apache.thrift.protocol.TField("types", org.apache.thrift.protocol.TType.LIST, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TTypeDescStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TTypeDescTupleSchemeFactory());
-  }
-
-  private List<TTypeEntry> types; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    TYPES((short)1, "types");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // TYPES
-          return TYPES;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.TYPES, new org.apache.thrift.meta_data.FieldMetaData("types", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TTypeEntry.class))));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TTypeDesc.class, metaDataMap);
-  }
-
-  public TTypeDesc() {
-  }
-
-  public TTypeDesc(
-    List<TTypeEntry> types)
-  {
-    this();
-    this.types = types;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TTypeDesc(TTypeDesc other) {
-    if (other.isSetTypes()) {
-      List<TTypeEntry> __this__types = new ArrayList<TTypeEntry>();
-      for (TTypeEntry other_element : other.types) {
-        __this__types.add(new TTypeEntry(other_element));
-      }
-      this.types = __this__types;
-    }
-  }
-
-  public TTypeDesc deepCopy() {
-    return new TTypeDesc(this);
-  }
-
-  @Override
-  public void clear() {
-    this.types = null;
-  }
-
-  public int getTypesSize() {
-    return (this.types == null) ? 0 : this.types.size();
-  }
-
-  public java.util.Iterator<TTypeEntry> getTypesIterator() {
-    return (this.types == null) ? null : this.types.iterator();
-  }
-
-  public void addToTypes(TTypeEntry elem) {
-    if (this.types == null) {
-      this.types = new ArrayList<TTypeEntry>();
-    }
-    this.types.add(elem);
-  }
-
-  public List<TTypeEntry> getTypes() {
-    return this.types;
-  }
-
-  public void setTypes(List<TTypeEntry> types) {
-    this.types = types;
-  }
-
-  public void unsetTypes() {
-    this.types = null;
-  }
-
-  /** Returns true if field types is set (has been assigned a value) and false otherwise */
-  public boolean isSetTypes() {
-    return this.types != null;
-  }
-
-  public void setTypesIsSet(boolean value) {
-    if (!value) {
-      this.types = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case TYPES:
-      if (value == null) {
-        unsetTypes();
-      } else {
-        setTypes((List<TTypeEntry>)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case TYPES:
-      return getTypes();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case TYPES:
-      return isSetTypes();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TTypeDesc)
-      return this.equals((TTypeDesc)that);
-    return false;
-  }
-
-  public boolean equals(TTypeDesc that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_types = true && this.isSetTypes();
-    boolean that_present_types = true && that.isSetTypes();
-    if (this_present_types || that_present_types) {
-      if (!(this_present_types && that_present_types))
-        return false;
-      if (!this.types.equals(that.types))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_types = true && (isSetTypes());
-    builder.append(present_types);
-    if (present_types)
-      builder.append(types);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TTypeDesc other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TTypeDesc typedOther = (TTypeDesc)other;
-
-    lastComparison = Boolean.valueOf(isSetTypes()).compareTo(typedOther.isSetTypes());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetTypes()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.types, typedOther.types);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TTypeDesc(");
-    boolean first = true;
-
-    sb.append("types:");
-    if (this.types == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.types);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetTypes()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'types' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TTypeDescStandardSchemeFactory implements SchemeFactory {
-    public TTypeDescStandardScheme getScheme() {
-      return new TTypeDescStandardScheme();
-    }
-  }
-
-  private static class TTypeDescStandardScheme extends StandardScheme<TTypeDesc> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TTypeDesc struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // TYPES
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list30 = iprot.readListBegin();
-                struct.types = new ArrayList<TTypeEntry>(_list30.size);
-                for (int _i31 = 0; _i31 < _list30.size; ++_i31)
-                {
-                  TTypeEntry _elem32; // optional
-                  _elem32 = new TTypeEntry();
-                  _elem32.read(iprot);
-                  struct.types.add(_elem32);
-                }
-                iprot.readListEnd();
-              }
-              struct.setTypesIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TTypeDesc struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.types != null) {
-        oprot.writeFieldBegin(TYPES_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.types.size()));
-          for (TTypeEntry _iter33 : struct.types)
-          {
-            _iter33.write(oprot);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TTypeDescTupleSchemeFactory implements SchemeFactory {
-    public TTypeDescTupleScheme getScheme() {
-      return new TTypeDescTupleScheme();
-    }
-  }
-
-  private static class TTypeDescTupleScheme extends TupleScheme<TTypeDesc> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TTypeDesc struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.types.size());
-        for (TTypeEntry _iter34 : struct.types)
-        {
-          _iter34.write(oprot);
-        }
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TTypeDesc struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TList _list35 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-        struct.types = new ArrayList<TTypeEntry>(_list35.size);
-        for (int _i36 = 0; _i36 < _list35.size; ++_i36)
-        {
-          TTypeEntry _elem37; // optional
-          _elem37 = new TTypeEntry();
-          _elem37.read(iprot);
-          struct.types.add(_elem37);
-        }
-      }
-      struct.setTypesIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TTypeEntry.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TTypeEntry.java
deleted file mode 100644
index d0d70c1279572..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TTypeEntry.java
+++ /dev/null
@@ -1,610 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TTypeEntry extends org.apache.thrift.TUnion<TTypeEntry, TTypeEntry._Fields> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TTypeEntry");
-  private static final org.apache.thrift.protocol.TField PRIMITIVE_ENTRY_FIELD_DESC = new org.apache.thrift.protocol.TField("primitiveEntry", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField ARRAY_ENTRY_FIELD_DESC = new org.apache.thrift.protocol.TField("arrayEntry", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-  private static final org.apache.thrift.protocol.TField MAP_ENTRY_FIELD_DESC = new org.apache.thrift.protocol.TField("mapEntry", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-  private static final org.apache.thrift.protocol.TField STRUCT_ENTRY_FIELD_DESC = new org.apache.thrift.protocol.TField("structEntry", org.apache.thrift.protocol.TType.STRUCT, (short)4);
-  private static final org.apache.thrift.protocol.TField UNION_ENTRY_FIELD_DESC = new org.apache.thrift.protocol.TField("unionEntry", org.apache.thrift.protocol.TType.STRUCT, (short)5);
-  private static final org.apache.thrift.protocol.TField USER_DEFINED_TYPE_ENTRY_FIELD_DESC = new org.apache.thrift.protocol.TField("userDefinedTypeEntry", org.apache.thrift.protocol.TType.STRUCT, (short)6);
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    PRIMITIVE_ENTRY((short)1, "primitiveEntry"),
-    ARRAY_ENTRY((short)2, "arrayEntry"),
-    MAP_ENTRY((short)3, "mapEntry"),
-    STRUCT_ENTRY((short)4, "structEntry"),
-    UNION_ENTRY((short)5, "unionEntry"),
-    USER_DEFINED_TYPE_ENTRY((short)6, "userDefinedTypeEntry");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // PRIMITIVE_ENTRY
-          return PRIMITIVE_ENTRY;
-        case 2: // ARRAY_ENTRY
-          return ARRAY_ENTRY;
-        case 3: // MAP_ENTRY
-          return MAP_ENTRY;
-        case 4: // STRUCT_ENTRY
-          return STRUCT_ENTRY;
-        case 5: // UNION_ENTRY
-          return UNION_ENTRY;
-        case 6: // USER_DEFINED_TYPE_ENTRY
-          return USER_DEFINED_TYPE_ENTRY;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.PRIMITIVE_ENTRY, new org.apache.thrift.meta_data.FieldMetaData("primitiveEntry", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TPrimitiveTypeEntry.class)));
-    tmpMap.put(_Fields.ARRAY_ENTRY, new org.apache.thrift.meta_data.FieldMetaData("arrayEntry", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TArrayTypeEntry.class)));
-    tmpMap.put(_Fields.MAP_ENTRY, new org.apache.thrift.meta_data.FieldMetaData("mapEntry", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TMapTypeEntry.class)));
-    tmpMap.put(_Fields.STRUCT_ENTRY, new org.apache.thrift.meta_data.FieldMetaData("structEntry", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStructTypeEntry.class)));
-    tmpMap.put(_Fields.UNION_ENTRY, new org.apache.thrift.meta_data.FieldMetaData("unionEntry", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TUnionTypeEntry.class)));
-    tmpMap.put(_Fields.USER_DEFINED_TYPE_ENTRY, new org.apache.thrift.meta_data.FieldMetaData("userDefinedTypeEntry", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TUserDefinedTypeEntry.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TTypeEntry.class, metaDataMap);
-  }
-
-  public TTypeEntry() {
-    super();
-  }
-
-  public TTypeEntry(TTypeEntry._Fields setField, Object value) {
-    super(setField, value);
-  }
-
-  public TTypeEntry(TTypeEntry other) {
-    super(other);
-  }
-  public TTypeEntry deepCopy() {
-    return new TTypeEntry(this);
-  }
-
-  public static TTypeEntry primitiveEntry(TPrimitiveTypeEntry value) {
-    TTypeEntry x = new TTypeEntry();
-    x.setPrimitiveEntry(value);
-    return x;
-  }
-
-  public static TTypeEntry arrayEntry(TArrayTypeEntry value) {
-    TTypeEntry x = new TTypeEntry();
-    x.setArrayEntry(value);
-    return x;
-  }
-
-  public static TTypeEntry mapEntry(TMapTypeEntry value) {
-    TTypeEntry x = new TTypeEntry();
-    x.setMapEntry(value);
-    return x;
-  }
-
-  public static TTypeEntry structEntry(TStructTypeEntry value) {
-    TTypeEntry x = new TTypeEntry();
-    x.setStructEntry(value);
-    return x;
-  }
-
-  public static TTypeEntry unionEntry(TUnionTypeEntry value) {
-    TTypeEntry x = new TTypeEntry();
-    x.setUnionEntry(value);
-    return x;
-  }
-
-  public static TTypeEntry userDefinedTypeEntry(TUserDefinedTypeEntry value) {
-    TTypeEntry x = new TTypeEntry();
-    x.setUserDefinedTypeEntry(value);
-    return x;
-  }
-
-
-  @Override
-  protected void checkType(_Fields setField, Object value) throws ClassCastException {
-    switch (setField) {
-      case PRIMITIVE_ENTRY:
-        if (value instanceof TPrimitiveTypeEntry) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TPrimitiveTypeEntry for field 'primitiveEntry', but got " + value.getClass().getSimpleName());
-      case ARRAY_ENTRY:
-        if (value instanceof TArrayTypeEntry) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TArrayTypeEntry for field 'arrayEntry', but got " + value.getClass().getSimpleName());
-      case MAP_ENTRY:
-        if (value instanceof TMapTypeEntry) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TMapTypeEntry for field 'mapEntry', but got " + value.getClass().getSimpleName());
-      case STRUCT_ENTRY:
-        if (value instanceof TStructTypeEntry) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TStructTypeEntry for field 'structEntry', but got " + value.getClass().getSimpleName());
-      case UNION_ENTRY:
-        if (value instanceof TUnionTypeEntry) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TUnionTypeEntry for field 'unionEntry', but got " + value.getClass().getSimpleName());
-      case USER_DEFINED_TYPE_ENTRY:
-        if (value instanceof TUserDefinedTypeEntry) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TUserDefinedTypeEntry for field 'userDefinedTypeEntry', but got " + value.getClass().getSimpleName());
-      default:
-        throw new IllegalArgumentException("Unknown field id " + setField);
-    }
-  }
-
-  @Override
-  protected Object standardSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TField field) throws org.apache.thrift.TException {
-    _Fields setField = _Fields.findByThriftId(field.id);
-    if (setField != null) {
-      switch (setField) {
-        case PRIMITIVE_ENTRY:
-          if (field.type == PRIMITIVE_ENTRY_FIELD_DESC.type) {
-            TPrimitiveTypeEntry primitiveEntry;
-            primitiveEntry = new TPrimitiveTypeEntry();
-            primitiveEntry.read(iprot);
-            return primitiveEntry;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case ARRAY_ENTRY:
-          if (field.type == ARRAY_ENTRY_FIELD_DESC.type) {
-            TArrayTypeEntry arrayEntry;
-            arrayEntry = new TArrayTypeEntry();
-            arrayEntry.read(iprot);
-            return arrayEntry;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case MAP_ENTRY:
-          if (field.type == MAP_ENTRY_FIELD_DESC.type) {
-            TMapTypeEntry mapEntry;
-            mapEntry = new TMapTypeEntry();
-            mapEntry.read(iprot);
-            return mapEntry;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case STRUCT_ENTRY:
-          if (field.type == STRUCT_ENTRY_FIELD_DESC.type) {
-            TStructTypeEntry structEntry;
-            structEntry = new TStructTypeEntry();
-            structEntry.read(iprot);
-            return structEntry;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case UNION_ENTRY:
-          if (field.type == UNION_ENTRY_FIELD_DESC.type) {
-            TUnionTypeEntry unionEntry;
-            unionEntry = new TUnionTypeEntry();
-            unionEntry.read(iprot);
-            return unionEntry;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case USER_DEFINED_TYPE_ENTRY:
-          if (field.type == USER_DEFINED_TYPE_ENTRY_FIELD_DESC.type) {
-            TUserDefinedTypeEntry userDefinedTypeEntry;
-            userDefinedTypeEntry = new TUserDefinedTypeEntry();
-            userDefinedTypeEntry.read(iprot);
-            return userDefinedTypeEntry;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        default:
-          throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!");
-      }
-    } else {
-      return null;
-    }
-  }
-
-  @Override
-  protected void standardSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    switch (setField_) {
-      case PRIMITIVE_ENTRY:
-        TPrimitiveTypeEntry primitiveEntry = (TPrimitiveTypeEntry)value_;
-        primitiveEntry.write(oprot);
-        return;
-      case ARRAY_ENTRY:
-        TArrayTypeEntry arrayEntry = (TArrayTypeEntry)value_;
-        arrayEntry.write(oprot);
-        return;
-      case MAP_ENTRY:
-        TMapTypeEntry mapEntry = (TMapTypeEntry)value_;
-        mapEntry.write(oprot);
-        return;
-      case STRUCT_ENTRY:
-        TStructTypeEntry structEntry = (TStructTypeEntry)value_;
-        structEntry.write(oprot);
-        return;
-      case UNION_ENTRY:
-        TUnionTypeEntry unionEntry = (TUnionTypeEntry)value_;
-        unionEntry.write(oprot);
-        return;
-      case USER_DEFINED_TYPE_ENTRY:
-        TUserDefinedTypeEntry userDefinedTypeEntry = (TUserDefinedTypeEntry)value_;
-        userDefinedTypeEntry.write(oprot);
-        return;
-      default:
-        throw new IllegalStateException("Cannot write union with unknown field " + setField_);
-    }
-  }
-
-  @Override
-  protected Object tupleSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, short fieldID) throws org.apache.thrift.TException {
-    _Fields setField = _Fields.findByThriftId(fieldID);
-    if (setField != null) {
-      switch (setField) {
-        case PRIMITIVE_ENTRY:
-          TPrimitiveTypeEntry primitiveEntry;
-          primitiveEntry = new TPrimitiveTypeEntry();
-          primitiveEntry.read(iprot);
-          return primitiveEntry;
-        case ARRAY_ENTRY:
-          TArrayTypeEntry arrayEntry;
-          arrayEntry = new TArrayTypeEntry();
-          arrayEntry.read(iprot);
-          return arrayEntry;
-        case MAP_ENTRY:
-          TMapTypeEntry mapEntry;
-          mapEntry = new TMapTypeEntry();
-          mapEntry.read(iprot);
-          return mapEntry;
-        case STRUCT_ENTRY:
-          TStructTypeEntry structEntry;
-          structEntry = new TStructTypeEntry();
-          structEntry.read(iprot);
-          return structEntry;
-        case UNION_ENTRY:
-          TUnionTypeEntry unionEntry;
-          unionEntry = new TUnionTypeEntry();
-          unionEntry.read(iprot);
-          return unionEntry;
-        case USER_DEFINED_TYPE_ENTRY:
-          TUserDefinedTypeEntry userDefinedTypeEntry;
-          userDefinedTypeEntry = new TUserDefinedTypeEntry();
-          userDefinedTypeEntry.read(iprot);
-          return userDefinedTypeEntry;
-        default:
-          throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!");
-      }
-    } else {
-      throw new TProtocolException("Couldn't find a field with field id " + fieldID);
-    }
-  }
-
-  @Override
-  protected void tupleSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    switch (setField_) {
-      case PRIMITIVE_ENTRY:
-        TPrimitiveTypeEntry primitiveEntry = (TPrimitiveTypeEntry)value_;
-        primitiveEntry.write(oprot);
-        return;
-      case ARRAY_ENTRY:
-        TArrayTypeEntry arrayEntry = (TArrayTypeEntry)value_;
-        arrayEntry.write(oprot);
-        return;
-      case MAP_ENTRY:
-        TMapTypeEntry mapEntry = (TMapTypeEntry)value_;
-        mapEntry.write(oprot);
-        return;
-      case STRUCT_ENTRY:
-        TStructTypeEntry structEntry = (TStructTypeEntry)value_;
-        structEntry.write(oprot);
-        return;
-      case UNION_ENTRY:
-        TUnionTypeEntry unionEntry = (TUnionTypeEntry)value_;
-        unionEntry.write(oprot);
-        return;
-      case USER_DEFINED_TYPE_ENTRY:
-        TUserDefinedTypeEntry userDefinedTypeEntry = (TUserDefinedTypeEntry)value_;
-        userDefinedTypeEntry.write(oprot);
-        return;
-      default:
-        throw new IllegalStateException("Cannot write union with unknown field " + setField_);
-    }
-  }
-
-  @Override
-  protected org.apache.thrift.protocol.TField getFieldDesc(_Fields setField) {
-    switch (setField) {
-      case PRIMITIVE_ENTRY:
-        return PRIMITIVE_ENTRY_FIELD_DESC;
-      case ARRAY_ENTRY:
-        return ARRAY_ENTRY_FIELD_DESC;
-      case MAP_ENTRY:
-        return MAP_ENTRY_FIELD_DESC;
-      case STRUCT_ENTRY:
-        return STRUCT_ENTRY_FIELD_DESC;
-      case UNION_ENTRY:
-        return UNION_ENTRY_FIELD_DESC;
-      case USER_DEFINED_TYPE_ENTRY:
-        return USER_DEFINED_TYPE_ENTRY_FIELD_DESC;
-      default:
-        throw new IllegalArgumentException("Unknown field id " + setField);
-    }
-  }
-
-  @Override
-  protected org.apache.thrift.protocol.TStruct getStructDesc() {
-    return STRUCT_DESC;
-  }
-
-  @Override
-  protected _Fields enumForId(short id) {
-    return _Fields.findByThriftIdOrThrow(id);
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-
-  public TPrimitiveTypeEntry getPrimitiveEntry() {
-    if (getSetField() == _Fields.PRIMITIVE_ENTRY) {
-      return (TPrimitiveTypeEntry)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'primitiveEntry' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setPrimitiveEntry(TPrimitiveTypeEntry value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.PRIMITIVE_ENTRY;
-    value_ = value;
-  }
-
-  public TArrayTypeEntry getArrayEntry() {
-    if (getSetField() == _Fields.ARRAY_ENTRY) {
-      return (TArrayTypeEntry)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'arrayEntry' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setArrayEntry(TArrayTypeEntry value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.ARRAY_ENTRY;
-    value_ = value;
-  }
-
-  public TMapTypeEntry getMapEntry() {
-    if (getSetField() == _Fields.MAP_ENTRY) {
-      return (TMapTypeEntry)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'mapEntry' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setMapEntry(TMapTypeEntry value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.MAP_ENTRY;
-    value_ = value;
-  }
-
-  public TStructTypeEntry getStructEntry() {
-    if (getSetField() == _Fields.STRUCT_ENTRY) {
-      return (TStructTypeEntry)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'structEntry' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setStructEntry(TStructTypeEntry value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.STRUCT_ENTRY;
-    value_ = value;
-  }
-
-  public TUnionTypeEntry getUnionEntry() {
-    if (getSetField() == _Fields.UNION_ENTRY) {
-      return (TUnionTypeEntry)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'unionEntry' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setUnionEntry(TUnionTypeEntry value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.UNION_ENTRY;
-    value_ = value;
-  }
-
-  public TUserDefinedTypeEntry getUserDefinedTypeEntry() {
-    if (getSetField() == _Fields.USER_DEFINED_TYPE_ENTRY) {
-      return (TUserDefinedTypeEntry)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'userDefinedTypeEntry' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setUserDefinedTypeEntry(TUserDefinedTypeEntry value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.USER_DEFINED_TYPE_ENTRY;
-    value_ = value;
-  }
-
-  public boolean isSetPrimitiveEntry() {
-    return setField_ == _Fields.PRIMITIVE_ENTRY;
-  }
-
-
-  public boolean isSetArrayEntry() {
-    return setField_ == _Fields.ARRAY_ENTRY;
-  }
-
-
-  public boolean isSetMapEntry() {
-    return setField_ == _Fields.MAP_ENTRY;
-  }
-
-
-  public boolean isSetStructEntry() {
-    return setField_ == _Fields.STRUCT_ENTRY;
-  }
-
-
-  public boolean isSetUnionEntry() {
-    return setField_ == _Fields.UNION_ENTRY;
-  }
-
-
-  public boolean isSetUserDefinedTypeEntry() {
-    return setField_ == _Fields.USER_DEFINED_TYPE_ENTRY;
-  }
-
-
-  public boolean equals(Object other) {
-    if (other instanceof TTypeEntry) {
-      return equals((TTypeEntry)other);
-    } else {
-      return false;
-    }
-  }
-
-  public boolean equals(TTypeEntry other) {
-    return other != null && getSetField() == other.getSetField() && getFieldValue().equals(other.getFieldValue());
-  }
-
-  @Override
-  public int compareTo(TTypeEntry other) {
-    int lastComparison = org.apache.thrift.TBaseHelper.compareTo(getSetField(), other.getSetField());
-    if (lastComparison == 0) {
-      return org.apache.thrift.TBaseHelper.compareTo(getFieldValue(), other.getFieldValue());
-    }
-    return lastComparison;
-  }
-
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder hcb = new HashCodeBuilder();
-    hcb.append(this.getClass().getName());
-    org.apache.thrift.TFieldIdEnum setField = getSetField();
-    if (setField != null) {
-      hcb.append(setField.getThriftFieldId());
-      Object value = getFieldValue();
-      if (value instanceof org.apache.thrift.TEnum) {
-        hcb.append(((org.apache.thrift.TEnum)getFieldValue()).getValue());
-      } else {
-        hcb.append(value);
-      }
-    }
-    return hcb.toHashCode();
-  }
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-
-}
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TTypeId.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TTypeId.java
deleted file mode 100644
index 40f05894623c0..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TTypeId.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.thrift.TEnum;
-
-public enum TTypeId implements org.apache.thrift.TEnum {
-  BOOLEAN_TYPE(0),
-  TINYINT_TYPE(1),
-  SMALLINT_TYPE(2),
-  INT_TYPE(3),
-  BIGINT_TYPE(4),
-  FLOAT_TYPE(5),
-  DOUBLE_TYPE(6),
-  STRING_TYPE(7),
-  TIMESTAMP_TYPE(8),
-  BINARY_TYPE(9),
-  ARRAY_TYPE(10),
-  MAP_TYPE(11),
-  STRUCT_TYPE(12),
-  UNION_TYPE(13),
-  USER_DEFINED_TYPE(14),
-  DECIMAL_TYPE(15),
-  NULL_TYPE(16),
-  DATE_TYPE(17),
-  VARCHAR_TYPE(18),
-  CHAR_TYPE(19),
-  INTERVAL_YEAR_MONTH_TYPE(20),
-  INTERVAL_DAY_TIME_TYPE(21);
-
-  private final int value;
-
-  private TTypeId(int value) {
-    this.value = value;
-  }
-
-  /**
-   * Get the integer value of this enum value, as defined in the Thrift IDL.
-   */
-  public int getValue() {
-    return value;
-  }
-
-  /**
-   * Find a the enum type by its integer value, as defined in the Thrift IDL.
-   * @return null if the value is not found.
-   */
-  public static TTypeId findByValue(int value) { 
-    switch (value) {
-      case 0:
-        return BOOLEAN_TYPE;
-      case 1:
-        return TINYINT_TYPE;
-      case 2:
-        return SMALLINT_TYPE;
-      case 3:
-        return INT_TYPE;
-      case 4:
-        return BIGINT_TYPE;
-      case 5:
-        return FLOAT_TYPE;
-      case 6:
-        return DOUBLE_TYPE;
-      case 7:
-        return STRING_TYPE;
-      case 8:
-        return TIMESTAMP_TYPE;
-      case 9:
-        return BINARY_TYPE;
-      case 10:
-        return ARRAY_TYPE;
-      case 11:
-        return MAP_TYPE;
-      case 12:
-        return STRUCT_TYPE;
-      case 13:
-        return UNION_TYPE;
-      case 14:
-        return USER_DEFINED_TYPE;
-      case 15:
-        return DECIMAL_TYPE;
-      case 16:
-        return NULL_TYPE;
-      case 17:
-        return DATE_TYPE;
-      case 18:
-        return VARCHAR_TYPE;
-      case 19:
-        return CHAR_TYPE;
-      case 20:
-        return INTERVAL_YEAR_MONTH_TYPE;
-      case 21:
-        return INTERVAL_DAY_TIME_TYPE;
-      default:
-        return null;
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TTypeQualifierValue.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TTypeQualifierValue.java
deleted file mode 100644
index a3e3829372276..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TTypeQualifierValue.java
+++ /dev/null
@@ -1,361 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TTypeQualifierValue extends org.apache.thrift.TUnion<TTypeQualifierValue, TTypeQualifierValue._Fields> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TTypeQualifierValue");
-  private static final org.apache.thrift.protocol.TField I32_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("i32Value", org.apache.thrift.protocol.TType.I32, (short)1);
-  private static final org.apache.thrift.protocol.TField STRING_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("stringValue", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    I32_VALUE((short)1, "i32Value"),
-    STRING_VALUE((short)2, "stringValue");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // I32_VALUE
-          return I32_VALUE;
-        case 2: // STRING_VALUE
-          return STRING_VALUE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.I32_VALUE, new org.apache.thrift.meta_data.FieldMetaData("i32Value", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
-    tmpMap.put(_Fields.STRING_VALUE, new org.apache.thrift.meta_data.FieldMetaData("stringValue", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TTypeQualifierValue.class, metaDataMap);
-  }
-
-  public TTypeQualifierValue() {
-    super();
-  }
-
-  public TTypeQualifierValue(TTypeQualifierValue._Fields setField, Object value) {
-    super(setField, value);
-  }
-
-  public TTypeQualifierValue(TTypeQualifierValue other) {
-    super(other);
-  }
-  public TTypeQualifierValue deepCopy() {
-    return new TTypeQualifierValue(this);
-  }
-
-  public static TTypeQualifierValue i32Value(int value) {
-    TTypeQualifierValue x = new TTypeQualifierValue();
-    x.setI32Value(value);
-    return x;
-  }
-
-  public static TTypeQualifierValue stringValue(String value) {
-    TTypeQualifierValue x = new TTypeQualifierValue();
-    x.setStringValue(value);
-    return x;
-  }
-
-
-  @Override
-  protected void checkType(_Fields setField, Object value) throws ClassCastException {
-    switch (setField) {
-      case I32_VALUE:
-        if (value instanceof Integer) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type Integer for field 'i32Value', but got " + value.getClass().getSimpleName());
-      case STRING_VALUE:
-        if (value instanceof String) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type String for field 'stringValue', but got " + value.getClass().getSimpleName());
-      default:
-        throw new IllegalArgumentException("Unknown field id " + setField);
-    }
-  }
-
-  @Override
-  protected Object standardSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TField field) throws org.apache.thrift.TException {
-    _Fields setField = _Fields.findByThriftId(field.id);
-    if (setField != null) {
-      switch (setField) {
-        case I32_VALUE:
-          if (field.type == I32_VALUE_FIELD_DESC.type) {
-            Integer i32Value;
-            i32Value = iprot.readI32();
-            return i32Value;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case STRING_VALUE:
-          if (field.type == STRING_VALUE_FIELD_DESC.type) {
-            String stringValue;
-            stringValue = iprot.readString();
-            return stringValue;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        default:
-          throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!");
-      }
-    } else {
-      return null;
-    }
-  }
-
-  @Override
-  protected void standardSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    switch (setField_) {
-      case I32_VALUE:
-        Integer i32Value = (Integer)value_;
-        oprot.writeI32(i32Value);
-        return;
-      case STRING_VALUE:
-        String stringValue = (String)value_;
-        oprot.writeString(stringValue);
-        return;
-      default:
-        throw new IllegalStateException("Cannot write union with unknown field " + setField_);
-    }
-  }
-
-  @Override
-  protected Object tupleSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, short fieldID) throws org.apache.thrift.TException {
-    _Fields setField = _Fields.findByThriftId(fieldID);
-    if (setField != null) {
-      switch (setField) {
-        case I32_VALUE:
-          Integer i32Value;
-          i32Value = iprot.readI32();
-          return i32Value;
-        case STRING_VALUE:
-          String stringValue;
-          stringValue = iprot.readString();
-          return stringValue;
-        default:
-          throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!");
-      }
-    } else {
-      throw new TProtocolException("Couldn't find a field with field id " + fieldID);
-    }
-  }
-
-  @Override
-  protected void tupleSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    switch (setField_) {
-      case I32_VALUE:
-        Integer i32Value = (Integer)value_;
-        oprot.writeI32(i32Value);
-        return;
-      case STRING_VALUE:
-        String stringValue = (String)value_;
-        oprot.writeString(stringValue);
-        return;
-      default:
-        throw new IllegalStateException("Cannot write union with unknown field " + setField_);
-    }
-  }
-
-  @Override
-  protected org.apache.thrift.protocol.TField getFieldDesc(_Fields setField) {
-    switch (setField) {
-      case I32_VALUE:
-        return I32_VALUE_FIELD_DESC;
-      case STRING_VALUE:
-        return STRING_VALUE_FIELD_DESC;
-      default:
-        throw new IllegalArgumentException("Unknown field id " + setField);
-    }
-  }
-
-  @Override
-  protected org.apache.thrift.protocol.TStruct getStructDesc() {
-    return STRUCT_DESC;
-  }
-
-  @Override
-  protected _Fields enumForId(short id) {
-    return _Fields.findByThriftIdOrThrow(id);
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-
-  public int getI32Value() {
-    if (getSetField() == _Fields.I32_VALUE) {
-      return (Integer)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'i32Value' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setI32Value(int value) {
-    setField_ = _Fields.I32_VALUE;
-    value_ = value;
-  }
-
-  public String getStringValue() {
-    if (getSetField() == _Fields.STRING_VALUE) {
-      return (String)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'stringValue' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setStringValue(String value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.STRING_VALUE;
-    value_ = value;
-  }
-
-  public boolean isSetI32Value() {
-    return setField_ == _Fields.I32_VALUE;
-  }
-
-
-  public boolean isSetStringValue() {
-    return setField_ == _Fields.STRING_VALUE;
-  }
-
-
-  public boolean equals(Object other) {
-    if (other instanceof TTypeQualifierValue) {
-      return equals((TTypeQualifierValue)other);
-    } else {
-      return false;
-    }
-  }
-
-  public boolean equals(TTypeQualifierValue other) {
-    return other != null && getSetField() == other.getSetField() && getFieldValue().equals(other.getFieldValue());
-  }
-
-  @Override
-  public int compareTo(TTypeQualifierValue other) {
-    int lastComparison = org.apache.thrift.TBaseHelper.compareTo(getSetField(), other.getSetField());
-    if (lastComparison == 0) {
-      return org.apache.thrift.TBaseHelper.compareTo(getFieldValue(), other.getFieldValue());
-    }
-    return lastComparison;
-  }
-
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder hcb = new HashCodeBuilder();
-    hcb.append(this.getClass().getName());
-    org.apache.thrift.TFieldIdEnum setField = getSetField();
-    if (setField != null) {
-      hcb.append(setField.getThriftFieldId());
-      Object value = getFieldValue();
-      if (value instanceof org.apache.thrift.TEnum) {
-        hcb.append(((org.apache.thrift.TEnum)getFieldValue()).getValue());
-      } else {
-        hcb.append(value);
-      }
-    }
-    return hcb.toHashCode();
-  }
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-
-}
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TTypeQualifiers.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TTypeQualifiers.java
deleted file mode 100644
index 39355551d3722..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TTypeQualifiers.java
+++ /dev/null
@@ -1,450 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TTypeQualifiers implements org.apache.thrift.TBase<TTypeQualifiers, TTypeQualifiers._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TTypeQualifiers");
-
-  private static final org.apache.thrift.protocol.TField QUALIFIERS_FIELD_DESC = new org.apache.thrift.protocol.TField("qualifiers", org.apache.thrift.protocol.TType.MAP, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TTypeQualifiersStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TTypeQualifiersTupleSchemeFactory());
-  }
-
-  private Map<String,TTypeQualifierValue> qualifiers; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    QUALIFIERS((short)1, "qualifiers");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // QUALIFIERS
-          return QUALIFIERS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.QUALIFIERS, new org.apache.thrift.meta_data.FieldMetaData("qualifiers", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING), 
-            new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TTypeQualifierValue.class))));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TTypeQualifiers.class, metaDataMap);
-  }
-
-  public TTypeQualifiers() {
-  }
-
-  public TTypeQualifiers(
-    Map<String,TTypeQualifierValue> qualifiers)
-  {
-    this();
-    this.qualifiers = qualifiers;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TTypeQualifiers(TTypeQualifiers other) {
-    if (other.isSetQualifiers()) {
-      Map<String,TTypeQualifierValue> __this__qualifiers = new HashMap<String,TTypeQualifierValue>();
-      for (Map.Entry<String, TTypeQualifierValue> other_element : other.qualifiers.entrySet()) {
-
-        String other_element_key = other_element.getKey();
-        TTypeQualifierValue other_element_value = other_element.getValue();
-
-        String __this__qualifiers_copy_key = other_element_key;
-
-        TTypeQualifierValue __this__qualifiers_copy_value = new TTypeQualifierValue(other_element_value);
-
-        __this__qualifiers.put(__this__qualifiers_copy_key, __this__qualifiers_copy_value);
-      }
-      this.qualifiers = __this__qualifiers;
-    }
-  }
-
-  public TTypeQualifiers deepCopy() {
-    return new TTypeQualifiers(this);
-  }
-
-  @Override
-  public void clear() {
-    this.qualifiers = null;
-  }
-
-  public int getQualifiersSize() {
-    return (this.qualifiers == null) ? 0 : this.qualifiers.size();
-  }
-
-  public void putToQualifiers(String key, TTypeQualifierValue val) {
-    if (this.qualifiers == null) {
-      this.qualifiers = new HashMap<String,TTypeQualifierValue>();
-    }
-    this.qualifiers.put(key, val);
-  }
-
-  public Map<String,TTypeQualifierValue> getQualifiers() {
-    return this.qualifiers;
-  }
-
-  public void setQualifiers(Map<String,TTypeQualifierValue> qualifiers) {
-    this.qualifiers = qualifiers;
-  }
-
-  public void unsetQualifiers() {
-    this.qualifiers = null;
-  }
-
-  /** Returns true if field qualifiers is set (has been assigned a value) and false otherwise */
-  public boolean isSetQualifiers() {
-    return this.qualifiers != null;
-  }
-
-  public void setQualifiersIsSet(boolean value) {
-    if (!value) {
-      this.qualifiers = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case QUALIFIERS:
-      if (value == null) {
-        unsetQualifiers();
-      } else {
-        setQualifiers((Map<String,TTypeQualifierValue>)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case QUALIFIERS:
-      return getQualifiers();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case QUALIFIERS:
-      return isSetQualifiers();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TTypeQualifiers)
-      return this.equals((TTypeQualifiers)that);
-    return false;
-  }
-
-  public boolean equals(TTypeQualifiers that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_qualifiers = true && this.isSetQualifiers();
-    boolean that_present_qualifiers = true && that.isSetQualifiers();
-    if (this_present_qualifiers || that_present_qualifiers) {
-      if (!(this_present_qualifiers && that_present_qualifiers))
-        return false;
-      if (!this.qualifiers.equals(that.qualifiers))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_qualifiers = true && (isSetQualifiers());
-    builder.append(present_qualifiers);
-    if (present_qualifiers)
-      builder.append(qualifiers);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TTypeQualifiers other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TTypeQualifiers typedOther = (TTypeQualifiers)other;
-
-    lastComparison = Boolean.valueOf(isSetQualifiers()).compareTo(typedOther.isSetQualifiers());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetQualifiers()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.qualifiers, typedOther.qualifiers);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TTypeQualifiers(");
-    boolean first = true;
-
-    sb.append("qualifiers:");
-    if (this.qualifiers == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.qualifiers);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetQualifiers()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'qualifiers' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TTypeQualifiersStandardSchemeFactory implements SchemeFactory {
-    public TTypeQualifiersStandardScheme getScheme() {
-      return new TTypeQualifiersStandardScheme();
-    }
-  }
-
-  private static class TTypeQualifiersStandardScheme extends StandardScheme<TTypeQualifiers> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TTypeQualifiers struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // QUALIFIERS
-            if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
-              {
-                org.apache.thrift.protocol.TMap _map0 = iprot.readMapBegin();
-                struct.qualifiers = new HashMap<String,TTypeQualifierValue>(2*_map0.size);
-                for (int _i1 = 0; _i1 < _map0.size; ++_i1)
-                {
-                  String _key2; // required
-                  TTypeQualifierValue _val3; // required
-                  _key2 = iprot.readString();
-                  _val3 = new TTypeQualifierValue();
-                  _val3.read(iprot);
-                  struct.qualifiers.put(_key2, _val3);
-                }
-                iprot.readMapEnd();
-              }
-              struct.setQualifiersIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TTypeQualifiers struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.qualifiers != null) {
-        oprot.writeFieldBegin(QUALIFIERS_FIELD_DESC);
-        {
-          oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, struct.qualifiers.size()));
-          for (Map.Entry<String, TTypeQualifierValue> _iter4 : struct.qualifiers.entrySet())
-          {
-            oprot.writeString(_iter4.getKey());
-            _iter4.getValue().write(oprot);
-          }
-          oprot.writeMapEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TTypeQualifiersTupleSchemeFactory implements SchemeFactory {
-    public TTypeQualifiersTupleScheme getScheme() {
-      return new TTypeQualifiersTupleScheme();
-    }
-  }
-
-  private static class TTypeQualifiersTupleScheme extends TupleScheme<TTypeQualifiers> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TTypeQualifiers struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.qualifiers.size());
-        for (Map.Entry<String, TTypeQualifierValue> _iter5 : struct.qualifiers.entrySet())
-        {
-          oprot.writeString(_iter5.getKey());
-          _iter5.getValue().write(oprot);
-        }
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TTypeQualifiers struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TMap _map6 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-        struct.qualifiers = new HashMap<String,TTypeQualifierValue>(2*_map6.size);
-        for (int _i7 = 0; _i7 < _map6.size; ++_i7)
-        {
-          String _key8; // required
-          TTypeQualifierValue _val9; // required
-          _key8 = iprot.readString();
-          _val9 = new TTypeQualifierValue();
-          _val9.read(iprot);
-          struct.qualifiers.put(_key8, _val9);
-        }
-      }
-      struct.setQualifiersIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TUnionTypeEntry.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TUnionTypeEntry.java
deleted file mode 100644
index 73dd45d3dd01a..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TUnionTypeEntry.java
+++ /dev/null
@@ -1,448 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TUnionTypeEntry implements org.apache.thrift.TBase<TUnionTypeEntry, TUnionTypeEntry._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TUnionTypeEntry");
-
-  private static final org.apache.thrift.protocol.TField NAME_TO_TYPE_PTR_FIELD_DESC = new org.apache.thrift.protocol.TField("nameToTypePtr", org.apache.thrift.protocol.TType.MAP, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TUnionTypeEntryStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TUnionTypeEntryTupleSchemeFactory());
-  }
-
-  private Map<String,Integer> nameToTypePtr; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    NAME_TO_TYPE_PTR((short)1, "nameToTypePtr");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // NAME_TO_TYPE_PTR
-          return NAME_TO_TYPE_PTR;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.NAME_TO_TYPE_PTR, new org.apache.thrift.meta_data.FieldMetaData("nameToTypePtr", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING), 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32            , "TTypeEntryPtr"))));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TUnionTypeEntry.class, metaDataMap);
-  }
-
-  public TUnionTypeEntry() {
-  }
-
-  public TUnionTypeEntry(
-    Map<String,Integer> nameToTypePtr)
-  {
-    this();
-    this.nameToTypePtr = nameToTypePtr;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TUnionTypeEntry(TUnionTypeEntry other) {
-    if (other.isSetNameToTypePtr()) {
-      Map<String,Integer> __this__nameToTypePtr = new HashMap<String,Integer>();
-      for (Map.Entry<String, Integer> other_element : other.nameToTypePtr.entrySet()) {
-
-        String other_element_key = other_element.getKey();
-        Integer other_element_value = other_element.getValue();
-
-        String __this__nameToTypePtr_copy_key = other_element_key;
-
-        Integer __this__nameToTypePtr_copy_value = other_element_value;
-
-        __this__nameToTypePtr.put(__this__nameToTypePtr_copy_key, __this__nameToTypePtr_copy_value);
-      }
-      this.nameToTypePtr = __this__nameToTypePtr;
-    }
-  }
-
-  public TUnionTypeEntry deepCopy() {
-    return new TUnionTypeEntry(this);
-  }
-
-  @Override
-  public void clear() {
-    this.nameToTypePtr = null;
-  }
-
-  public int getNameToTypePtrSize() {
-    return (this.nameToTypePtr == null) ? 0 : this.nameToTypePtr.size();
-  }
-
-  public void putToNameToTypePtr(String key, int val) {
-    if (this.nameToTypePtr == null) {
-      this.nameToTypePtr = new HashMap<String,Integer>();
-    }
-    this.nameToTypePtr.put(key, val);
-  }
-
-  public Map<String,Integer> getNameToTypePtr() {
-    return this.nameToTypePtr;
-  }
-
-  public void setNameToTypePtr(Map<String,Integer> nameToTypePtr) {
-    this.nameToTypePtr = nameToTypePtr;
-  }
-
-  public void unsetNameToTypePtr() {
-    this.nameToTypePtr = null;
-  }
-
-  /** Returns true if field nameToTypePtr is set (has been assigned a value) and false otherwise */
-  public boolean isSetNameToTypePtr() {
-    return this.nameToTypePtr != null;
-  }
-
-  public void setNameToTypePtrIsSet(boolean value) {
-    if (!value) {
-      this.nameToTypePtr = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case NAME_TO_TYPE_PTR:
-      if (value == null) {
-        unsetNameToTypePtr();
-      } else {
-        setNameToTypePtr((Map<String,Integer>)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case NAME_TO_TYPE_PTR:
-      return getNameToTypePtr();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case NAME_TO_TYPE_PTR:
-      return isSetNameToTypePtr();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TUnionTypeEntry)
-      return this.equals((TUnionTypeEntry)that);
-    return false;
-  }
-
-  public boolean equals(TUnionTypeEntry that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_nameToTypePtr = true && this.isSetNameToTypePtr();
-    boolean that_present_nameToTypePtr = true && that.isSetNameToTypePtr();
-    if (this_present_nameToTypePtr || that_present_nameToTypePtr) {
-      if (!(this_present_nameToTypePtr && that_present_nameToTypePtr))
-        return false;
-      if (!this.nameToTypePtr.equals(that.nameToTypePtr))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_nameToTypePtr = true && (isSetNameToTypePtr());
-    builder.append(present_nameToTypePtr);
-    if (present_nameToTypePtr)
-      builder.append(nameToTypePtr);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TUnionTypeEntry other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TUnionTypeEntry typedOther = (TUnionTypeEntry)other;
-
-    lastComparison = Boolean.valueOf(isSetNameToTypePtr()).compareTo(typedOther.isSetNameToTypePtr());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetNameToTypePtr()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.nameToTypePtr, typedOther.nameToTypePtr);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TUnionTypeEntry(");
-    boolean first = true;
-
-    sb.append("nameToTypePtr:");
-    if (this.nameToTypePtr == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.nameToTypePtr);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetNameToTypePtr()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'nameToTypePtr' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TUnionTypeEntryStandardSchemeFactory implements SchemeFactory {
-    public TUnionTypeEntryStandardScheme getScheme() {
-      return new TUnionTypeEntryStandardScheme();
-    }
-  }
-
-  private static class TUnionTypeEntryStandardScheme extends StandardScheme<TUnionTypeEntry> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TUnionTypeEntry struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // NAME_TO_TYPE_PTR
-            if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
-              {
-                org.apache.thrift.protocol.TMap _map20 = iprot.readMapBegin();
-                struct.nameToTypePtr = new HashMap<String,Integer>(2*_map20.size);
-                for (int _i21 = 0; _i21 < _map20.size; ++_i21)
-                {
-                  String _key22; // required
-                  int _val23; // required
-                  _key22 = iprot.readString();
-                  _val23 = iprot.readI32();
-                  struct.nameToTypePtr.put(_key22, _val23);
-                }
-                iprot.readMapEnd();
-              }
-              struct.setNameToTypePtrIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TUnionTypeEntry struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.nameToTypePtr != null) {
-        oprot.writeFieldBegin(NAME_TO_TYPE_PTR_FIELD_DESC);
-        {
-          oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.I32, struct.nameToTypePtr.size()));
-          for (Map.Entry<String, Integer> _iter24 : struct.nameToTypePtr.entrySet())
-          {
-            oprot.writeString(_iter24.getKey());
-            oprot.writeI32(_iter24.getValue());
-          }
-          oprot.writeMapEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TUnionTypeEntryTupleSchemeFactory implements SchemeFactory {
-    public TUnionTypeEntryTupleScheme getScheme() {
-      return new TUnionTypeEntryTupleScheme();
-    }
-  }
-
-  private static class TUnionTypeEntryTupleScheme extends TupleScheme<TUnionTypeEntry> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TUnionTypeEntry struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.nameToTypePtr.size());
-        for (Map.Entry<String, Integer> _iter25 : struct.nameToTypePtr.entrySet())
-        {
-          oprot.writeString(_iter25.getKey());
-          oprot.writeI32(_iter25.getValue());
-        }
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TUnionTypeEntry struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TMap _map26 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.I32, iprot.readI32());
-        struct.nameToTypePtr = new HashMap<String,Integer>(2*_map26.size);
-        for (int _i27 = 0; _i27 < _map26.size; ++_i27)
-        {
-          String _key28; // required
-          int _val29; // required
-          _key28 = iprot.readString();
-          _val29 = iprot.readI32();
-          struct.nameToTypePtr.put(_key28, _val29);
-        }
-      }
-      struct.setNameToTypePtrIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TUserDefinedTypeEntry.java b/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TUserDefinedTypeEntry.java
deleted file mode 100644
index 3a111a2c8c2c6..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TUserDefinedTypeEntry.java
+++ /dev/null
@@ -1,385 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.cli.thrift;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TUserDefinedTypeEntry implements org.apache.thrift.TBase<TUserDefinedTypeEntry, TUserDefinedTypeEntry._Fields>, java.io.Serializable, Cloneable {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TUserDefinedTypeEntry");
-
-  private static final org.apache.thrift.protocol.TField TYPE_CLASS_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("typeClassName", org.apache.thrift.protocol.TType.STRING, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TUserDefinedTypeEntryStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TUserDefinedTypeEntryTupleSchemeFactory());
-  }
-
-  private String typeClassName; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    TYPE_CLASS_NAME((short)1, "typeClassName");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // TYPE_CLASS_NAME
-          return TYPE_CLASS_NAME;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.TYPE_CLASS_NAME, new org.apache.thrift.meta_data.FieldMetaData("typeClassName", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TUserDefinedTypeEntry.class, metaDataMap);
-  }
-
-  public TUserDefinedTypeEntry() {
-  }
-
-  public TUserDefinedTypeEntry(
-    String typeClassName)
-  {
-    this();
-    this.typeClassName = typeClassName;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TUserDefinedTypeEntry(TUserDefinedTypeEntry other) {
-    if (other.isSetTypeClassName()) {
-      this.typeClassName = other.typeClassName;
-    }
-  }
-
-  public TUserDefinedTypeEntry deepCopy() {
-    return new TUserDefinedTypeEntry(this);
-  }
-
-  @Override
-  public void clear() {
-    this.typeClassName = null;
-  }
-
-  public String getTypeClassName() {
-    return this.typeClassName;
-  }
-
-  public void setTypeClassName(String typeClassName) {
-    this.typeClassName = typeClassName;
-  }
-
-  public void unsetTypeClassName() {
-    this.typeClassName = null;
-  }
-
-  /** Returns true if field typeClassName is set (has been assigned a value) and false otherwise */
-  public boolean isSetTypeClassName() {
-    return this.typeClassName != null;
-  }
-
-  public void setTypeClassNameIsSet(boolean value) {
-    if (!value) {
-      this.typeClassName = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case TYPE_CLASS_NAME:
-      if (value == null) {
-        unsetTypeClassName();
-      } else {
-        setTypeClassName((String)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case TYPE_CLASS_NAME:
-      return getTypeClassName();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case TYPE_CLASS_NAME:
-      return isSetTypeClassName();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TUserDefinedTypeEntry)
-      return this.equals((TUserDefinedTypeEntry)that);
-    return false;
-  }
-
-  public boolean equals(TUserDefinedTypeEntry that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_typeClassName = true && this.isSetTypeClassName();
-    boolean that_present_typeClassName = true && that.isSetTypeClassName();
-    if (this_present_typeClassName || that_present_typeClassName) {
-      if (!(this_present_typeClassName && that_present_typeClassName))
-        return false;
-      if (!this.typeClassName.equals(that.typeClassName))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    HashCodeBuilder builder = new HashCodeBuilder();
-
-    boolean present_typeClassName = true && (isSetTypeClassName());
-    builder.append(present_typeClassName);
-    if (present_typeClassName)
-      builder.append(typeClassName);
-
-    return builder.toHashCode();
-  }
-
-  public int compareTo(TUserDefinedTypeEntry other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-    TUserDefinedTypeEntry typedOther = (TUserDefinedTypeEntry)other;
-
-    lastComparison = Boolean.valueOf(isSetTypeClassName()).compareTo(typedOther.isSetTypeClassName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetTypeClassName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.typeClassName, typedOther.typeClassName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TUserDefinedTypeEntry(");
-    boolean first = true;
-
-    sb.append("typeClassName:");
-    if (this.typeClassName == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.typeClassName);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetTypeClassName()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'typeClassName' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TUserDefinedTypeEntryStandardSchemeFactory implements SchemeFactory {
-    public TUserDefinedTypeEntryStandardScheme getScheme() {
-      return new TUserDefinedTypeEntryStandardScheme();
-    }
-  }
-
-  private static class TUserDefinedTypeEntryStandardScheme extends StandardScheme<TUserDefinedTypeEntry> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TUserDefinedTypeEntry struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // TYPE_CLASS_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.typeClassName = iprot.readString();
-              struct.setTypeClassNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TUserDefinedTypeEntry struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.typeClassName != null) {
-        oprot.writeFieldBegin(TYPE_CLASS_NAME_FIELD_DESC);
-        oprot.writeString(struct.typeClassName);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TUserDefinedTypeEntryTupleSchemeFactory implements SchemeFactory {
-    public TUserDefinedTypeEntryTupleScheme getScheme() {
-      return new TUserDefinedTypeEntryTupleScheme();
-    }
-  }
-
-  private static class TUserDefinedTypeEntryTupleScheme extends TupleScheme<TUserDefinedTypeEntry> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TUserDefinedTypeEntry struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      oprot.writeString(struct.typeClassName);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TUserDefinedTypeEntry struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.typeClassName = iprot.readString();
-      struct.setTypeClassNameIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/AbstractService.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/AbstractService.java
deleted file mode 100644
index 7e557aeccf5b0..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/AbstractService.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-
-/**
- * AbstractService.
- *
- */
-public abstract class AbstractService implements Service {
-
-  private static final Log LOG = LogFactory.getLog(AbstractService.class);
-
-  /**
-   * Service state: initially {@link STATE#NOTINITED}.
-   */
-  private Service.STATE state = STATE.NOTINITED;
-
-  /**
-   * Service name.
-   */
-  private final String name;
-  /**
-   * Service start time. Will be zero until the service is started.
-   */
-  private long startTime;
-
-  /**
-   * The configuration. Will be null until the service is initialized.
-   */
-  private HiveConf hiveConf;
-
-  /**
-   * List of state change listeners; it is final to ensure
-   * that it will never be null.
-   */
-  private final List<ServiceStateChangeListener> listeners =
-      new ArrayList<ServiceStateChangeListener>();
-
-  /**
-   * Construct the service.
-   *
-   * @param name
-   *          service name
-   */
-  public AbstractService(String name) {
-    this.name = name;
-  }
-
-  @Override
-  public synchronized Service.STATE getServiceState() {
-    return state;
-  }
-
-  /**
-   * {@inheritDoc}
-   *
-   * @throws IllegalStateException
-   *           if the current service state does not permit
-   *           this action
-   */
-  @Override
-  public synchronized void init(HiveConf hiveConf) {
-    ensureCurrentState(STATE.NOTINITED);
-    this.hiveConf = hiveConf;
-    changeState(STATE.INITED);
-    LOG.info("Service:" + getName() + " is inited.");
-  }
-
-  /**
-   * {@inheritDoc}
-   *
-   * @throws IllegalStateException
-   *           if the current service state does not permit
-   *           this action
-   */
-  @Override
-  public synchronized void start() {
-    startTime = System.currentTimeMillis();
-    ensureCurrentState(STATE.INITED);
-    changeState(STATE.STARTED);
-    LOG.info("Service:" + getName() + " is started.");
-  }
-
-  /**
-   * {@inheritDoc}
-   *
-   * @throws IllegalStateException
-   *           if the current service state does not permit
-   *           this action
-   */
-  @Override
-  public synchronized void stop() {
-    if (state == STATE.STOPPED ||
-        state == STATE.INITED ||
-        state == STATE.NOTINITED) {
-      // already stopped, or else it was never
-      // started (eg another service failing canceled startup)
-      return;
-    }
-    ensureCurrentState(STATE.STARTED);
-    changeState(STATE.STOPPED);
-    LOG.info("Service:" + getName() + " is stopped.");
-  }
-
-  @Override
-  public synchronized void register(ServiceStateChangeListener l) {
-    listeners.add(l);
-  }
-
-  @Override
-  public synchronized void unregister(ServiceStateChangeListener l) {
-    listeners.remove(l);
-  }
-
-  @Override
-  public String getName() {
-    return name;
-  }
-
-  @Override
-  public synchronized HiveConf getHiveConf() {
-    return hiveConf;
-  }
-
-  @Override
-  public long getStartTime() {
-    return startTime;
-  }
-
-  /**
-   * Verify that a service is in a given state.
-   *
-   * @param currentState
-   *          the desired state
-   * @throws IllegalStateException
-   *           if the service state is different from
-   *           the desired state
-   */
-  private void ensureCurrentState(Service.STATE currentState) {
-    ServiceOperations.ensureCurrentState(state, currentState);
-  }
-
-  /**
-   * Change to a new state and notify all listeners.
-   * This is a private method that is only invoked from synchronized methods,
-   * which avoid having to clone the listener list. It does imply that
-   * the state change listener methods should be short lived, as they
-   * will delay the state transition.
-   *
-   * @param newState
-   *          new service state
-   */
-  private void changeState(Service.STATE newState) {
-    state = newState;
-    // notify listeners
-    for (ServiceStateChangeListener l : listeners) {
-      l.stateChanged(this);
-    }
-  }
-
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/CompositeService.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/CompositeService.java
deleted file mode 100644
index 897911872b80f..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/CompositeService.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-
-/**
- * CompositeService.
- *
- */
-public class CompositeService extends AbstractService {
-
-  private static final Log LOG = LogFactory.getLog(CompositeService.class);
-
-  private final List<Service> serviceList = new ArrayList<Service>();
-
-  public CompositeService(String name) {
-    super(name);
-  }
-
-  public Collection<Service> getServices() {
-    return Collections.unmodifiableList(serviceList);
-  }
-
-  protected synchronized void addService(Service service) {
-    serviceList.add(service);
-  }
-
-  protected synchronized boolean removeService(Service service) {
-    return serviceList.remove(service);
-  }
-
-  @Override
-  public synchronized void init(HiveConf hiveConf) {
-    for (Service service : serviceList) {
-      service.init(hiveConf);
-    }
-    super.init(hiveConf);
-  }
-
-  @Override
-  public synchronized void start() {
-    int i = 0;
-    try {
-      for (int n = serviceList.size(); i < n; i++) {
-        Service service = serviceList.get(i);
-        service.start();
-      }
-      super.start();
-    } catch (Throwable e) {
-      LOG.error("Error starting services " + getName(), e);
-      // Note that the state of the failed service is still INITED and not
-      // STARTED. Even though the last service is not started completely, still
-      // call stop() on all services including failed service to make sure cleanup
-      // happens.
-      stop(i);
-      throw new ServiceException("Failed to Start " + getName(), e);
-    }
-
-  }
-
-  @Override
-  public synchronized void stop() {
-    if (this.getServiceState() == STATE.STOPPED) {
-      // The base composite-service is already stopped, don't do anything again.
-      return;
-    }
-    if (serviceList.size() > 0) {
-      stop(serviceList.size() - 1);
-    }
-    super.stop();
-  }
-
-  private synchronized void stop(int numOfServicesStarted) {
-    // stop in reserve order of start
-    for (int i = numOfServicesStarted; i >= 0; i--) {
-      Service service = serviceList.get(i);
-      try {
-        service.stop();
-      } catch (Throwable t) {
-        LOG.info("Error stopping " + service.getName(), t);
-      }
-    }
-  }
-
-  /**
-   * JVM Shutdown hook for CompositeService which will stop the given
-   * CompositeService gracefully in case of JVM shutdown.
-   */
-  public static class CompositeServiceShutdownHook implements Runnable {
-
-    private final CompositeService compositeService;
-
-    public CompositeServiceShutdownHook(CompositeService compositeService) {
-      this.compositeService = compositeService;
-    }
-
-    @Override
-    public void run() {
-      try {
-        // Stop the Composite Service
-        compositeService.stop();
-      } catch (Throwable t) {
-        LOG.info("Error stopping " + compositeService.getName(), t);
-      }
-    }
-  }
-
-
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/CookieSigner.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/CookieSigner.java
deleted file mode 100644
index f2a80c9d5ffbc..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/CookieSigner.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.Log;
-
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-
-/**
- * The cookie signer generates a signature based on SHA digest
- * and appends it to the cookie value generated at the
- * server side. It uses SHA digest algorithm to sign and verify signatures.
- */
-public class CookieSigner {
-  private static final String SIGNATURE = "&s=";
-  private static final String SHA_STRING = "SHA";
-  private byte[] secretBytes;
-  private static final Log LOG = LogFactory.getLog(CookieSigner.class);
-
-  /**
-   * Constructor
-   * @param secret Secret Bytes
-   */
-  public CookieSigner(byte[] secret) {
-    if (secret == null) {
-      throw new IllegalArgumentException(" NULL Secret Bytes");
-    }
-    this.secretBytes = secret.clone();
-  }
-
-  /**
-   * Sign the cookie given the string token as input.
-   * @param str Input token
-   * @return Signed token that can be used to create a cookie
-   */
-  public String signCookie(String str) {
-    if (str == null || str.isEmpty()) {
-      throw new IllegalArgumentException("NULL or empty string to sign");
-    }
-    String signature = getSignature(str);
-
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Signature generated for " + str + " is " + signature);
-    }
-    return str + SIGNATURE + signature;
-  }
-
-  /**
-   * Verify a signed string and extracts the original string.
-   * @param signedStr The already signed string
-   * @return Raw Value of the string without the signature
-   */
-  public String verifyAndExtract(String signedStr) {
-    int index = signedStr.lastIndexOf(SIGNATURE);
-    if (index == -1) {
-      throw new IllegalArgumentException("Invalid input sign: " + signedStr);
-    }
-    String originalSignature = signedStr.substring(index + SIGNATURE.length());
-    String rawValue = signedStr.substring(0, index);
-    String currentSignature = getSignature(rawValue);
-
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Signature generated for " + rawValue + " inside verify is " + currentSignature);
-    }
-    if (!MessageDigest.isEqual(originalSignature.getBytes(), currentSignature.getBytes())) {
-      throw new IllegalArgumentException("Invalid sign, original = " + originalSignature +
-        " current = " + currentSignature);
-    }
-    return rawValue;
-  }
-
-  /**
-   * Get the signature of the input string based on SHA digest algorithm.
-   * @param str Input token
-   * @return Signed String
-   */
-  private String getSignature(String str) {
-    try {
-      MessageDigest md = MessageDigest.getInstance(SHA_STRING);
-      md.update(str.getBytes());
-      md.update(secretBytes);
-      byte[] digest = md.digest();
-      return new Base64(0).encodeToString(digest);
-    } catch (NoSuchAlgorithmException ex) {
-      throw new RuntimeException("Invalid SHA digest String: " + SHA_STRING +
-        " " + ex.getMessage(), ex);
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/ServiceOperations.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/ServiceOperations.java
deleted file mode 100644
index f16863c1b41aa..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/ServiceOperations.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-
-/**
- * ServiceOperations.
- *
- */
-public final class ServiceOperations {
-  private static final Log LOG = LogFactory.getLog(ServiceOperations.class);
-
-  private ServiceOperations() {
-  }
-
-  /**
-   * Verify that a service is in a given state.
-   * @param state the actual state a service is in
-   * @param expectedState the desired state
-   * @throws IllegalStateException if the service state is different from
-   * the desired state
-   */
-  public static void ensureCurrentState(Service.STATE state,
-                                        Service.STATE expectedState) {
-    if (state != expectedState) {
-      throw new IllegalStateException("For this operation, the " +
-                                          "current service state must be "
-                                          + expectedState
-                                          + " instead of " + state);
-    }
-  }
-
-  /**
-   * Initialize a service.
-   *
-   * The service state is checked <i>before</i> the operation begins.
-   * This process is <i>not</i> thread safe.
-   * @param service a service that must be in the state
-   *   {@link Service.STATE#NOTINITED}
-   * @param configuration the configuration to initialize the service with
-   * @throws RuntimeException on a state change failure
-   * @throws IllegalStateException if the service is in the wrong state
-   */
-
-  public static void init(Service service, HiveConf configuration) {
-    Service.STATE state = service.getServiceState();
-    ensureCurrentState(state, Service.STATE.NOTINITED);
-    service.init(configuration);
-  }
-
-  /**
-   * Start a service.
-   *
-   * The service state is checked <i>before</i> the operation begins.
-   * This process is <i>not</i> thread safe.
-   * @param service a service that must be in the state
-   *   {@link Service.STATE#INITED}
-   * @throws RuntimeException on a state change failure
-   * @throws IllegalStateException if the service is in the wrong state
-   */
-
-  public static void start(Service service) {
-    Service.STATE state = service.getServiceState();
-    ensureCurrentState(state, Service.STATE.INITED);
-    service.start();
-  }
-
-  /**
-   * Initialize then start a service.
-   *
-   * The service state is checked <i>before</i> the operation begins.
-   * This process is <i>not</i> thread safe.
-   * @param service a service that must be in the state
-   *   {@link Service.STATE#NOTINITED}
-   * @param configuration the configuration to initialize the service with
-   * @throws RuntimeException on a state change failure
-   * @throws IllegalStateException if the service is in the wrong state
-   */
-  public static void deploy(Service service, HiveConf configuration) {
-    init(service, configuration);
-    start(service);
-  }
-
-  /**
-   * Stop a service.
-   *
-   * Do nothing if the service is null or not in a state in which it can be/needs to be stopped.
-   *
-   * The service state is checked <i>before</i> the operation begins.
-   * This process is <i>not</i> thread safe.
-   * @param service a service or null
-   */
-  public static void stop(Service service) {
-    if (service != null) {
-      Service.STATE state = service.getServiceState();
-      if (state == Service.STATE.STARTED) {
-        service.stop();
-      }
-    }
-  }
-
-  /**
-   * Stop a service; if it is null do nothing. Exceptions are caught and
-   * logged at warn level. (but not Throwables). This operation is intended to
-   * be used in cleanup operations
-   *
-   * @param service a service; may be null
-   * @return any exception that was caught; null if none was.
-   */
-  public static Exception stopQuietly(Service service) {
-    try {
-      stop(service);
-    } catch (Exception e) {
-      LOG.warn("When stopping the service " + service.getName()
-                   + " : " + e,
-               e);
-      return e;
-    }
-    return null;
-  }
-
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/ServiceUtils.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/ServiceUtils.java
deleted file mode 100644
index edb5eff9615bf..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/ServiceUtils.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.service;
-
-public class ServiceUtils {
-
-  /*
-   * Get the index separating the user name from domain name (the user's name up
-   * to the first '/' or '@').
-   *
-   * @param userName full user name.
-   * @return index of domain match or -1 if not found
-   */
-  public static int indexOfDomainMatch(String userName) {
-    if (userName == null) {
-      return -1;
-    }
-
-    int idx = userName.indexOf('/');
-    int idx2 = userName.indexOf('@');
-    int endIdx = Math.min(idx, idx2); // Use the earlier match.
-    // Unless at least one of '/' or '@' was not found, in
-    // which case, user the latter match.
-    if (endIdx == -1) {
-      endIdx = Math.max(idx, idx2);
-    }
-    return endIdx;
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/auth/HiveAuthFactory.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/auth/HiveAuthFactory.java
deleted file mode 100644
index 10000f12ab329..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/auth/HiveAuthFactory.java
+++ /dev/null
@@ -1,419 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.service.auth;
-
-import java.io.IOException;
-import java.lang.reflect.Field;
-import java.lang.reflect.Method;
-import java.net.InetSocketAddress;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-
-import javax.net.ssl.SSLServerSocket;
-import javax.security.auth.login.LoginException;
-import javax.security.sasl.Sasl;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.HiveMetaStore;
-import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.shims.HadoopShims.KerberosNameShim;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.hive.thrift.DBTokenStore;
-import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge;
-import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge.Server.ServerMode;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authorize.ProxyUsers;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.thrift.ThriftCLIService;
-import org.apache.thrift.TProcessorFactory;
-import org.apache.thrift.transport.TSSLTransportFactory;
-import org.apache.thrift.transport.TServerSocket;
-import org.apache.thrift.transport.TSocket;
-import org.apache.thrift.transport.TTransport;
-import org.apache.thrift.transport.TTransportException;
-import org.apache.thrift.transport.TTransportFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class helps in some aspects of authentication. It creates the proper Thrift classes for the
- * given configuration as well as helps with authenticating requests.
- */
-public class HiveAuthFactory {
-  private static final Logger LOG = LoggerFactory.getLogger(HiveAuthFactory.class);
-
-
-  public enum AuthTypes {
-    NOSASL("NOSASL"),
-    NONE("NONE"),
-    LDAP("LDAP"),
-    KERBEROS("KERBEROS"),
-    CUSTOM("CUSTOM"),
-    PAM("PAM");
-
-    private final String authType;
-
-    AuthTypes(String authType) {
-      this.authType = authType;
-    }
-
-    public String getAuthName() {
-      return authType;
-    }
-
-  }
-
-  private HadoopThriftAuthBridge.Server saslServer;
-  private String authTypeStr;
-  private final String transportMode;
-  private final HiveConf conf;
-
-  public static final String HS2_PROXY_USER = "hive.server2.proxy.user";
-  public static final String HS2_CLIENT_TOKEN = "hiveserver2ClientToken";
-
-  private static Field keytabFile = null;
-  private static Method getKeytab = null;
-  static {
-    Class<?> clz = UserGroupInformation.class;
-    try {
-      keytabFile = clz.getDeclaredField("keytabFile");
-      keytabFile.setAccessible(true);
-    } catch (NoSuchFieldException nfe) {
-      LOG.debug("Cannot find private field \"keytabFile\" in class: " +
-        UserGroupInformation.class.getCanonicalName(), nfe);
-      keytabFile = null;
-    }
-
-    try {
-      getKeytab = clz.getDeclaredMethod("getKeytab");
-      getKeytab.setAccessible(true);
-    } catch(NoSuchMethodException nme) {
-      LOG.debug("Cannot find private method \"getKeytab\" in class:" +
-        UserGroupInformation.class.getCanonicalName(), nme);
-      getKeytab = null;
-    }
-  }
-
-  public HiveAuthFactory(HiveConf conf) throws TTransportException, IOException {
-    this.conf = conf;
-    transportMode = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE);
-    authTypeStr = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION);
-
-    // In http mode we use NOSASL as the default auth type
-    if ("http".equalsIgnoreCase(transportMode)) {
-      if (authTypeStr == null) {
-        authTypeStr = AuthTypes.NOSASL.getAuthName();
-      }
-    } else {
-      if (authTypeStr == null) {
-        authTypeStr = AuthTypes.NONE.getAuthName();
-      }
-      if (authTypeStr.equalsIgnoreCase(AuthTypes.KERBEROS.getAuthName())) {
-        String principal = conf.getVar(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL);
-        String keytab = conf.getVar(ConfVars.HIVE_SERVER2_KERBEROS_KEYTAB);
-        if (needUgiLogin(UserGroupInformation.getCurrentUser(),
-          SecurityUtil.getServerPrincipal(principal, "0.0.0.0"), keytab)) {
-          saslServer = ShimLoader.getHadoopThriftAuthBridge().createServer(principal, keytab);
-        } else {
-          // Using the default constructor to avoid unnecessary UGI login.
-          saslServer = new HadoopThriftAuthBridge.Server();
-        }
-
-        // start delegation token manager
-        try {
-          // rawStore is only necessary for DBTokenStore
-          Object rawStore = null;
-          String tokenStoreClass = conf.getVar(HiveConf.ConfVars.METASTORE_CLUSTER_DELEGATION_TOKEN_STORE_CLS);
-
-          if (tokenStoreClass.equals(DBTokenStore.class.getName())) {
-            HMSHandler baseHandler = new HiveMetaStore.HMSHandler(
-                "new db based metaserver", conf, true);
-            rawStore = baseHandler.getMS();
-          }
-
-          saslServer.startDelegationTokenSecretManager(conf, rawStore, ServerMode.HIVESERVER2);
-        }
-        catch (MetaException|IOException e) {
-          throw new TTransportException("Failed to start token manager", e);
-        }
-      }
-    }
-  }
-
-  public Map<String, String> getSaslProperties() {
-    Map<String, String> saslProps = new HashMap<String, String>();
-    SaslQOP saslQOP = SaslQOP.fromString(conf.getVar(ConfVars.HIVE_SERVER2_THRIFT_SASL_QOP));
-    saslProps.put(Sasl.QOP, saslQOP.toString());
-    saslProps.put(Sasl.SERVER_AUTH, "true");
-    return saslProps;
-  }
-
-  public TTransportFactory getAuthTransFactory() throws LoginException {
-    TTransportFactory transportFactory;
-    if (authTypeStr.equalsIgnoreCase(AuthTypes.KERBEROS.getAuthName())) {
-      try {
-        transportFactory = saslServer.createTransportFactory(getSaslProperties());
-      } catch (TTransportException e) {
-        throw new LoginException(e.getMessage());
-      }
-    } else if (authTypeStr.equalsIgnoreCase(AuthTypes.NONE.getAuthName())) {
-      transportFactory = PlainSaslHelper.getPlainTransportFactory(authTypeStr);
-    } else if (authTypeStr.equalsIgnoreCase(AuthTypes.LDAP.getAuthName())) {
-      transportFactory = PlainSaslHelper.getPlainTransportFactory(authTypeStr);
-    } else if (authTypeStr.equalsIgnoreCase(AuthTypes.PAM.getAuthName())) {
-      transportFactory = PlainSaslHelper.getPlainTransportFactory(authTypeStr);
-    } else if (authTypeStr.equalsIgnoreCase(AuthTypes.NOSASL.getAuthName())) {
-      transportFactory = new TTransportFactory();
-    } else if (authTypeStr.equalsIgnoreCase(AuthTypes.CUSTOM.getAuthName())) {
-      transportFactory = PlainSaslHelper.getPlainTransportFactory(authTypeStr);
-    } else {
-      throw new LoginException("Unsupported authentication type " + authTypeStr);
-    }
-    return transportFactory;
-  }
-
-  /**
-   * Returns the thrift processor factory for HiveServer2 running in binary mode
-   * @param service
-   * @return
-   * @throws LoginException
-   */
-  public TProcessorFactory getAuthProcFactory(ThriftCLIService service) throws LoginException {
-    if (authTypeStr.equalsIgnoreCase(AuthTypes.KERBEROS.getAuthName())) {
-      return KerberosSaslHelper.getKerberosProcessorFactory(saslServer, service);
-    } else {
-      return PlainSaslHelper.getPlainProcessorFactory(service);
-    }
-  }
-
-  public String getRemoteUser() {
-    return saslServer == null ? null : saslServer.getRemoteUser();
-  }
-
-  public String getIpAddress() {
-    if (saslServer == null || saslServer.getRemoteAddress() == null) {
-      return null;
-    } else {
-      return saslServer.getRemoteAddress().getHostAddress();
-    }
-  }
-
-  // Perform kerberos login using the hadoop shim API if the configuration is available
-  public static void loginFromKeytab(HiveConf hiveConf) throws IOException {
-    String principal = hiveConf.getVar(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL);
-    String keyTabFile = hiveConf.getVar(ConfVars.HIVE_SERVER2_KERBEROS_KEYTAB);
-    if (principal.isEmpty() || keyTabFile.isEmpty()) {
-      throw new IOException("HiveServer2 Kerberos principal or keytab is not correctly configured");
-    } else {
-      UserGroupInformation.loginUserFromKeytab(SecurityUtil.getServerPrincipal(principal, "0.0.0.0"), keyTabFile);
-    }
-  }
-
-  // Perform SPNEGO login using the hadoop shim API if the configuration is available
-  public static UserGroupInformation loginFromSpnegoKeytabAndReturnUGI(HiveConf hiveConf)
-    throws IOException {
-    String principal = hiveConf.getVar(ConfVars.HIVE_SERVER2_SPNEGO_PRINCIPAL);
-    String keyTabFile = hiveConf.getVar(ConfVars.HIVE_SERVER2_SPNEGO_KEYTAB);
-    if (principal.isEmpty() || keyTabFile.isEmpty()) {
-      throw new IOException("HiveServer2 SPNEGO principal or keytab is not correctly configured");
-    } else {
-      return UserGroupInformation.loginUserFromKeytabAndReturnUGI(SecurityUtil.getServerPrincipal(principal, "0.0.0.0"), keyTabFile);
-    }
-  }
-
-  public static TTransport getSocketTransport(String host, int port, int loginTimeout) {
-    return new TSocket(host, port, loginTimeout);
-  }
-
-  public static TTransport getSSLSocket(String host, int port, int loginTimeout)
-    throws TTransportException {
-    return TSSLTransportFactory.getClientSocket(host, port, loginTimeout);
-  }
-
-  public static TTransport getSSLSocket(String host, int port, int loginTimeout,
-    String trustStorePath, String trustStorePassWord) throws TTransportException {
-    TSSLTransportFactory.TSSLTransportParameters params =
-      new TSSLTransportFactory.TSSLTransportParameters();
-    params.setTrustStore(trustStorePath, trustStorePassWord);
-    params.requireClientAuth(true);
-    return TSSLTransportFactory.getClientSocket(host, port, loginTimeout, params);
-  }
-
-  public static TServerSocket getServerSocket(String hiveHost, int portNum)
-    throws TTransportException {
-    InetSocketAddress serverAddress;
-    if (hiveHost == null || hiveHost.isEmpty()) {
-      // Wildcard bind
-      serverAddress = new InetSocketAddress(portNum);
-    } else {
-      serverAddress = new InetSocketAddress(hiveHost, portNum);
-    }
-    return new TServerSocket(serverAddress);
-  }
-
-  public static TServerSocket getServerSSLSocket(String hiveHost, int portNum, String keyStorePath,
-      String keyStorePassWord, List<String> sslVersionBlacklist) throws TTransportException,
-      UnknownHostException {
-    TSSLTransportFactory.TSSLTransportParameters params =
-        new TSSLTransportFactory.TSSLTransportParameters();
-    params.setKeyStore(keyStorePath, keyStorePassWord);
-    InetSocketAddress serverAddress;
-    if (hiveHost == null || hiveHost.isEmpty()) {
-      // Wildcard bind
-      serverAddress = new InetSocketAddress(portNum);
-    } else {
-      serverAddress = new InetSocketAddress(hiveHost, portNum);
-    }
-    TServerSocket thriftServerSocket =
-        TSSLTransportFactory.getServerSocket(portNum, 0, serverAddress.getAddress(), params);
-    if (thriftServerSocket.getServerSocket() instanceof SSLServerSocket) {
-      List<String> sslVersionBlacklistLocal = new ArrayList<String>();
-      for (String sslVersion : sslVersionBlacklist) {
-        sslVersionBlacklistLocal.add(sslVersion.trim().toLowerCase(Locale.ROOT));
-      }
-      SSLServerSocket sslServerSocket = (SSLServerSocket) thriftServerSocket.getServerSocket();
-      List<String> enabledProtocols = new ArrayList<String>();
-      for (String protocol : sslServerSocket.getEnabledProtocols()) {
-        if (sslVersionBlacklistLocal.contains(protocol.toLowerCase(Locale.ROOT))) {
-          LOG.debug("Disabling SSL Protocol: " + protocol);
-        } else {
-          enabledProtocols.add(protocol);
-        }
-      }
-      sslServerSocket.setEnabledProtocols(enabledProtocols.toArray(new String[0]));
-      LOG.info("SSL Server Socket Enabled Protocols: "
-          + Arrays.toString(sslServerSocket.getEnabledProtocols()));
-    }
-    return thriftServerSocket;
-  }
-
-  // retrieve delegation token for the given user
-  public String getDelegationToken(String owner, String renewer) throws HiveSQLException {
-    if (saslServer == null) {
-      throw new HiveSQLException(
-          "Delegation token only supported over kerberos authentication", "08S01");
-    }
-
-    try {
-      String tokenStr = saslServer.getDelegationTokenWithService(owner, renewer, HS2_CLIENT_TOKEN);
-      if (tokenStr == null || tokenStr.isEmpty()) {
-        throw new HiveSQLException(
-            "Received empty retrieving delegation token for user " + owner, "08S01");
-      }
-      return tokenStr;
-    } catch (IOException e) {
-      throw new HiveSQLException(
-          "Error retrieving delegation token for user " + owner, "08S01", e);
-    } catch (InterruptedException e) {
-      throw new HiveSQLException("delegation token retrieval interrupted", "08S01", e);
-    }
-  }
-
-  // cancel given delegation token
-  public void cancelDelegationToken(String delegationToken) throws HiveSQLException {
-    if (saslServer == null) {
-      throw new HiveSQLException(
-          "Delegation token only supported over kerberos authentication", "08S01");
-    }
-    try {
-      saslServer.cancelDelegationToken(delegationToken);
-    } catch (IOException e) {
-      throw new HiveSQLException(
-          "Error canceling delegation token " + delegationToken, "08S01", e);
-    }
-  }
-
-  public void renewDelegationToken(String delegationToken) throws HiveSQLException {
-    if (saslServer == null) {
-      throw new HiveSQLException(
-          "Delegation token only supported over kerberos authentication", "08S01");
-    }
-    try {
-      saslServer.renewDelegationToken(delegationToken);
-    } catch (IOException e) {
-      throw new HiveSQLException(
-          "Error renewing delegation token " + delegationToken, "08S01", e);
-    }
-  }
-
-  public String getUserFromToken(String delegationToken) throws HiveSQLException {
-    if (saslServer == null) {
-      throw new HiveSQLException(
-          "Delegation token only supported over kerberos authentication", "08S01");
-    }
-    try {
-      return saslServer.getUserFromToken(delegationToken);
-    } catch (IOException e) {
-      throw new HiveSQLException(
-          "Error extracting user from delegation token " + delegationToken, "08S01", e);
-    }
-  }
-
-  public static void verifyProxyAccess(String realUser, String proxyUser, String ipAddress,
-    HiveConf hiveConf) throws HiveSQLException {
-    try {
-      UserGroupInformation sessionUgi;
-      if (UserGroupInformation.isSecurityEnabled()) {
-        KerberosNameShim kerbName = ShimLoader.getHadoopShims().getKerberosNameShim(realUser);
-        sessionUgi = UserGroupInformation.createProxyUser(
-            kerbName.getServiceName(), UserGroupInformation.getLoginUser());
-      } else {
-        sessionUgi = UserGroupInformation.createRemoteUser(realUser);
-      }
-      if (!proxyUser.equalsIgnoreCase(realUser)) {
-        ProxyUsers.refreshSuperUserGroupsConfiguration(hiveConf);
-        ProxyUsers.authorize(UserGroupInformation.createProxyUser(proxyUser, sessionUgi),
-            ipAddress, hiveConf);
-      }
-    } catch (IOException e) {
-      throw new HiveSQLException(
-        "Failed to validate proxy privilege of " + realUser + " for " + proxyUser, "08S01", e);
-    }
-  }
-
-  public static boolean needUgiLogin(UserGroupInformation ugi, String principal, String keytab) {
-    return null == ugi || !ugi.hasKerberosCredentials() || !ugi.getUserName().equals(principal) ||
-      !Objects.equals(keytab, getKeytabFromUgi());
-  }
-
-  private static String getKeytabFromUgi() {
-    synchronized (UserGroupInformation.class) {
-      try {
-        if (keytabFile != null) {
-          return (String) keytabFile.get(null);
-        } else if (getKeytab != null) {
-          return (String) getKeytab.invoke(UserGroupInformation.getCurrentUser());
-        } else {
-          return null;
-        }
-      } catch (Exception e) {
-        LOG.debug("Fail to get keytabFile path via reflection", e);
-        return null;
-      }
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/auth/HttpAuthUtils.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/auth/HttpAuthUtils.java
deleted file mode 100644
index f7375ee707830..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/auth/HttpAuthUtils.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.auth;
-
-import java.security.AccessControlContext;
-import java.security.AccessController;
-import java.security.PrivilegedExceptionAction;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-import java.util.StringTokenizer;
-
-import javax.security.auth.Subject;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.http.protocol.BasicHttpContext;
-import org.apache.http.protocol.HttpContext;
-import org.ietf.jgss.GSSContext;
-import org.ietf.jgss.GSSManager;
-import org.ietf.jgss.GSSName;
-import org.ietf.jgss.Oid;
-
-/**
- * Utility functions for HTTP mode authentication.
- */
-public final class HttpAuthUtils {
-  public static final String WWW_AUTHENTICATE = "WWW-Authenticate";
-  public static final String AUTHORIZATION = "Authorization";
-  public static final String BASIC = "Basic";
-  public static final String NEGOTIATE = "Negotiate";
-  private static final Log LOG = LogFactory.getLog(HttpAuthUtils.class);
-  private static final String COOKIE_ATTR_SEPARATOR = "&";
-  private static final String COOKIE_CLIENT_USER_NAME = "cu";
-  private static final String COOKIE_CLIENT_RAND_NUMBER = "rn";
-  private static final String COOKIE_KEY_VALUE_SEPARATOR = "=";
-  private static final Set<String> COOKIE_ATTRIBUTES =
-    new HashSet<String>(Arrays.asList(COOKIE_CLIENT_USER_NAME, COOKIE_CLIENT_RAND_NUMBER));
-
-  /**
-   * @return Stringified Base64 encoded kerberosAuthHeader on success
-   * @throws Exception
-   */
-  public static String getKerberosServiceTicket(String principal, String host,
-      String serverHttpUrl, boolean assumeSubject) throws Exception {
-    String serverPrincipal =
-        ShimLoader.getHadoopThriftAuthBridge().getServerPrincipal(principal, host);
-    if (assumeSubject) {
-      // With this option, we're assuming that the external application,
-      // using the JDBC driver has done a JAAS kerberos login already
-      AccessControlContext context = AccessController.getContext();
-      Subject subject = Subject.getSubject(context);
-      if (subject == null) {
-        throw new Exception("The Subject is not set");
-      }
-      return Subject.doAs(subject, new HttpKerberosClientAction(serverPrincipal, serverHttpUrl));
-    } else {
-      // JAAS login from ticket cache to setup the client UserGroupInformation
-      UserGroupInformation clientUGI =
-          ShimLoader.getHadoopThriftAuthBridge().getCurrentUGIWithConf("kerberos");
-      return clientUGI.doAs(new HttpKerberosClientAction(serverPrincipal, serverHttpUrl));
-    }
-  }
-
-  /**
-   * Creates and returns a HS2 cookie token.
-   * @param clientUserName Client User name.
-   * @return An unsigned cookie token generated from input parameters.
-   * The final cookie generated is of the following format :
-   * {@code cu=<username>&rn=<randomNumber>&s=<cookieSignature>}
-   */
-  public static String createCookieToken(String clientUserName) {
-    StringBuffer sb = new StringBuffer();
-    sb.append(COOKIE_CLIENT_USER_NAME).append(COOKIE_KEY_VALUE_SEPARATOR).append(clientUserName)
-      .append(COOKIE_ATTR_SEPARATOR);
-    sb.append(COOKIE_CLIENT_RAND_NUMBER).append(COOKIE_KEY_VALUE_SEPARATOR)
-      .append((new Random(System.currentTimeMillis())).nextLong());
-    return sb.toString();
-  }
-
-  /**
-   * Parses a cookie token to retrieve client user name.
-   * @param tokenStr Token String.
-   * @return A valid user name if input is of valid format, else returns null.
-   */
-  public static String getUserNameFromCookieToken(String tokenStr) {
-    Map<String, String> map = splitCookieToken(tokenStr);
-
-    if (!map.keySet().equals(COOKIE_ATTRIBUTES)) {
-      LOG.error("Invalid token with missing attributes " + tokenStr);
-      return null;
-    }
-    return map.get(COOKIE_CLIENT_USER_NAME);
-  }
-
-  /**
-   * Splits the cookie token into attributes pairs.
-   * @param str input token.
-   * @return a map with the attribute pairs of the token if the input is valid.
-   * Else, returns null.
-   */
-  private static Map<String, String> splitCookieToken(String tokenStr) {
-    Map<String, String> map = new HashMap<String, String>();
-    StringTokenizer st = new StringTokenizer(tokenStr, COOKIE_ATTR_SEPARATOR);
-
-    while (st.hasMoreTokens()) {
-      String part = st.nextToken();
-      int separator = part.indexOf(COOKIE_KEY_VALUE_SEPARATOR);
-      if (separator == -1) {
-        LOG.error("Invalid token string " + tokenStr);
-        return null;
-      }
-      String key = part.substring(0, separator);
-      String value = part.substring(separator + 1);
-      map.put(key, value);
-    }
-    return map;
-  }
-
-
-  private HttpAuthUtils() {
-    throw new UnsupportedOperationException("Can't initialize class");
-  }
-
-  /**
-   * We'll create an instance of this class within a doAs block so that the client's TGT credentials
-   * can be read from the Subject
-   */
-  public static class HttpKerberosClientAction implements PrivilegedExceptionAction<String> {
-    public static final String HTTP_RESPONSE = "HTTP_RESPONSE";
-    public static final String SERVER_HTTP_URL = "SERVER_HTTP_URL";
-    private final String serverPrincipal;
-    private final String serverHttpUrl;
-    private final Base64 base64codec;
-    private final HttpContext httpContext;
-
-    public HttpKerberosClientAction(String serverPrincipal, String serverHttpUrl) {
-      this.serverPrincipal = serverPrincipal;
-      this.serverHttpUrl = serverHttpUrl;
-      base64codec = new Base64(0);
-      httpContext = new BasicHttpContext();
-      httpContext.setAttribute(SERVER_HTTP_URL, serverHttpUrl);
-    }
-
-    @Override
-    public String run() throws Exception {
-      // This Oid for Kerberos GSS-API mechanism.
-      Oid mechOid = new Oid("1.2.840.113554.1.2.2");
-      // Oid for kerberos principal name
-      Oid krb5PrincipalOid = new Oid("1.2.840.113554.1.2.2.1");
-      GSSManager manager = GSSManager.getInstance();
-      // GSS name for server
-      GSSName serverName = manager.createName(serverPrincipal, krb5PrincipalOid);
-      // Create a GSSContext for authentication with the service.
-      // We're passing client credentials as null since we want them to be read from the Subject.
-      GSSContext gssContext =
-          manager.createContext(serverName, mechOid, null, GSSContext.DEFAULT_LIFETIME);
-      gssContext.requestMutualAuth(false);
-      // Establish context
-      byte[] inToken = new byte[0];
-      byte[] outToken = gssContext.initSecContext(inToken, 0, inToken.length);
-      gssContext.dispose();
-      // Base64 encoded and stringified token for server
-      return new String(base64codec.encode(outToken));
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/auth/KerberosSaslHelper.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/auth/KerberosSaslHelper.java
deleted file mode 100644
index 52eb752f1e026..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/auth/KerberosSaslHelper.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.service.auth;
-
-import java.io.IOException;
-import java.util.Map;
-import javax.security.sasl.SaslException;
-
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge;
-import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge.Server;
-import org.apache.hive.service.cli.thrift.TCLIService;
-import org.apache.hive.service.cli.thrift.TCLIService.Iface;
-import org.apache.hive.service.cli.thrift.ThriftCLIService;
-import org.apache.thrift.TProcessor;
-import org.apache.thrift.TProcessorFactory;
-import org.apache.thrift.transport.TSaslClientTransport;
-import org.apache.thrift.transport.TTransport;
-
-public final class KerberosSaslHelper {
-
-  public static TProcessorFactory getKerberosProcessorFactory(Server saslServer,
-    ThriftCLIService service) {
-    return new CLIServiceProcessorFactory(saslServer, service);
-  }
-
-  public static TTransport getKerberosTransport(String principal, String host,
-    TTransport underlyingTransport, Map<String, String> saslProps, boolean assumeSubject)
-    throws SaslException {
-    try {
-      String[] names = principal.split("[/@]");
-      if (names.length != 3) {
-        throw new IllegalArgumentException("Kerberos principal should have 3 parts: " + principal);
-      }
-
-      if (assumeSubject) {
-        return createSubjectAssumedTransport(principal, underlyingTransport, saslProps);
-      } else {
-        HadoopThriftAuthBridge.Client authBridge =
-          ShimLoader.getHadoopThriftAuthBridge().createClientWithConf("kerberos");
-        return authBridge.createClientTransport(principal, host, "KERBEROS", null,
-                                                underlyingTransport, saslProps);
-      }
-    } catch (IOException e) {
-      throw new SaslException("Failed to open client transport", e);
-    }
-  }
-
-  public static TTransport createSubjectAssumedTransport(String principal,
-    TTransport underlyingTransport, Map<String, String> saslProps) throws IOException {
-    String[] names = principal.split("[/@]");
-    try {
-      TTransport saslTransport =
-        new TSaslClientTransport("GSSAPI", null, names[0], names[1], saslProps, null,
-          underlyingTransport);
-      return new TSubjectAssumingTransport(saslTransport);
-    } catch (SaslException se) {
-      throw new IOException("Could not instantiate SASL transport", se);
-    }
-  }
-
-  public static TTransport getTokenTransport(String tokenStr, String host,
-    TTransport underlyingTransport, Map<String, String> saslProps) throws SaslException {
-    HadoopThriftAuthBridge.Client authBridge =
-      ShimLoader.getHadoopThriftAuthBridge().createClientWithConf("kerberos");
-
-    try {
-      return authBridge.createClientTransport(null, host, "DIGEST", tokenStr, underlyingTransport,
-                                              saslProps);
-    } catch (IOException e) {
-      throw new SaslException("Failed to open client transport", e);
-    }
-  }
-
-  private KerberosSaslHelper() {
-    throw new UnsupportedOperationException("Can't initialize class");
-  }
-
-  private static class CLIServiceProcessorFactory extends TProcessorFactory {
-
-    private final ThriftCLIService service;
-    private final Server saslServer;
-
-    CLIServiceProcessorFactory(Server saslServer, ThriftCLIService service) {
-      super(null);
-      this.service = service;
-      this.saslServer = saslServer;
-    }
-
-    @Override
-    public TProcessor getProcessor(TTransport trans) {
-      TProcessor sqlProcessor = new TCLIService.Processor<Iface>(service);
-      return saslServer.wrapNonAssumingProcessor(sqlProcessor);
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/auth/PlainSaslHelper.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/auth/PlainSaslHelper.java
deleted file mode 100644
index afc144199f1e8..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/auth/PlainSaslHelper.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.service.auth;
-
-import java.io.IOException;
-import java.security.Security;
-import java.util.HashMap;
-import javax.security.auth.callback.Callback;
-import javax.security.auth.callback.CallbackHandler;
-import javax.security.auth.callback.NameCallback;
-import javax.security.auth.callback.PasswordCallback;
-import javax.security.auth.callback.UnsupportedCallbackException;
-import javax.security.auth.login.LoginException;
-import javax.security.sasl.AuthenticationException;
-import javax.security.sasl.AuthorizeCallback;
-import javax.security.sasl.SaslException;
-
-import org.apache.hive.service.auth.AuthenticationProviderFactory.AuthMethods;
-import org.apache.hive.service.auth.PlainSaslServer.SaslPlainProvider;
-import org.apache.hive.service.cli.thrift.TCLIService.Iface;
-import org.apache.hive.service.cli.thrift.ThriftCLIService;
-import org.apache.thrift.TProcessor;
-import org.apache.thrift.TProcessorFactory;
-import org.apache.thrift.transport.TSaslClientTransport;
-import org.apache.thrift.transport.TSaslServerTransport;
-import org.apache.thrift.transport.TTransport;
-import org.apache.thrift.transport.TTransportFactory;
-
-public final class PlainSaslHelper {
-
-  public static TProcessorFactory getPlainProcessorFactory(ThriftCLIService service) {
-    return new SQLPlainProcessorFactory(service);
-  }
-
-  // Register Plain SASL server provider
-  static {
-    Security.addProvider(new SaslPlainProvider());
-  }
-
-  public static TTransportFactory getPlainTransportFactory(String authTypeStr)
-    throws LoginException {
-    TSaslServerTransport.Factory saslFactory = new TSaslServerTransport.Factory();
-    try {
-      saslFactory.addServerDefinition("PLAIN", authTypeStr, null, new HashMap<String, String>(),
-        new PlainServerCallbackHandler(authTypeStr));
-    } catch (AuthenticationException e) {
-      throw new LoginException("Error setting callback handler" + e);
-    }
-    return saslFactory;
-  }
-
-  public static TTransport getPlainTransport(String username, String password,
-    TTransport underlyingTransport) throws SaslException {
-    return new TSaslClientTransport("PLAIN", null, null, null, new HashMap<String, String>(),
-      new PlainCallbackHandler(username, password), underlyingTransport);
-  }
-
-  private PlainSaslHelper() {
-    throw new UnsupportedOperationException("Can't initialize class");
-  }
-
-  private static final class PlainServerCallbackHandler implements CallbackHandler {
-
-    private final AuthMethods authMethod;
-
-    PlainServerCallbackHandler(String authMethodStr) throws AuthenticationException {
-      authMethod = AuthMethods.getValidAuthMethod(authMethodStr);
-    }
-
-    @Override
-    public void handle(Callback[] callbacks) throws IOException, UnsupportedCallbackException {
-      String username = null;
-      String password = null;
-      AuthorizeCallback ac = null;
-
-      for (Callback callback : callbacks) {
-        if (callback instanceof NameCallback) {
-          NameCallback nc = (NameCallback) callback;
-          username = nc.getName();
-        } else if (callback instanceof PasswordCallback) {
-          PasswordCallback pc = (PasswordCallback) callback;
-          password = new String(pc.getPassword());
-        } else if (callback instanceof AuthorizeCallback) {
-          ac = (AuthorizeCallback) callback;
-        } else {
-          throw new UnsupportedCallbackException(callback);
-        }
-      }
-      PasswdAuthenticationProvider provider =
-        AuthenticationProviderFactory.getAuthenticationProvider(authMethod);
-      provider.Authenticate(username, password);
-      if (ac != null) {
-        ac.setAuthorized(true);
-      }
-    }
-  }
-
-  public static class PlainCallbackHandler implements CallbackHandler {
-
-    private final String username;
-    private final String password;
-
-    public PlainCallbackHandler(String username, String password) {
-      this.username = username;
-      this.password = password;
-    }
-
-    @Override
-    public void handle(Callback[] callbacks) throws IOException, UnsupportedCallbackException {
-      for (Callback callback : callbacks) {
-        if (callback instanceof NameCallback) {
-          NameCallback nameCallback = (NameCallback) callback;
-          nameCallback.setName(username);
-        } else if (callback instanceof PasswordCallback) {
-          PasswordCallback passCallback = (PasswordCallback) callback;
-          passCallback.setPassword(password.toCharArray());
-        } else {
-          throw new UnsupportedCallbackException(callback);
-        }
-      }
-    }
-  }
-
-  private static final class SQLPlainProcessorFactory extends TProcessorFactory {
-
-    private final ThriftCLIService service;
-
-    SQLPlainProcessorFactory(ThriftCLIService service) {
-      super(null);
-      this.service = service;
-    }
-
-    @Override
-    public TProcessor getProcessor(TTransport trans) {
-      return new TSetIpAddressProcessor<Iface>(service);
-    }
-  }
-
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/auth/TSetIpAddressProcessor.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/auth/TSetIpAddressProcessor.java
deleted file mode 100644
index 9a61ad49942c8..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/auth/TSetIpAddressProcessor.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.auth;
-
-import org.apache.hive.service.cli.thrift.TCLIService;
-import org.apache.hive.service.cli.thrift.TCLIService.Iface;
-import org.apache.thrift.TException;
-import org.apache.thrift.protocol.TProtocol;
-import org.apache.thrift.transport.TSaslClientTransport;
-import org.apache.thrift.transport.TSaslServerTransport;
-import org.apache.thrift.transport.TSocket;
-import org.apache.thrift.transport.TTransport;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class is responsible for setting the ipAddress for operations executed via HiveServer2.
- *
- * - IP address is only set for operations that calls listeners with hookContext
- * - IP address is only set if the underlying transport mechanism is socket
- *
- * @see org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext
- */
-public class TSetIpAddressProcessor<I extends Iface> extends TCLIService.Processor<Iface> {
-
-  private static final Logger LOGGER = LoggerFactory.getLogger(TSetIpAddressProcessor.class.getName());
-
-  public TSetIpAddressProcessor(Iface iface) {
-    super(iface);
-  }
-
-  @Override
-  public boolean process(final TProtocol in, final TProtocol out) throws TException {
-    setIpAddress(in);
-    setUserName(in);
-    try {
-      return super.process(in, out);
-    } finally {
-      THREAD_LOCAL_USER_NAME.remove();
-      THREAD_LOCAL_IP_ADDRESS.remove();
-    }
-  }
-
-  private void setUserName(final TProtocol in) {
-    TTransport transport = in.getTransport();
-    if (transport instanceof TSaslServerTransport) {
-      String userName = ((TSaslServerTransport) transport).getSaslServer().getAuthorizationID();
-      THREAD_LOCAL_USER_NAME.set(userName);
-    }
-  }
-
-  protected void setIpAddress(final TProtocol in) {
-    TTransport transport = in.getTransport();
-    TSocket tSocket = getUnderlyingSocketFromTransport(transport);
-    if (tSocket == null) {
-      LOGGER.warn("Unknown Transport, cannot determine ipAddress");
-    } else {
-      THREAD_LOCAL_IP_ADDRESS.set(tSocket.getSocket().getInetAddress().getHostAddress());
-    }
-  }
-
-  private TSocket getUnderlyingSocketFromTransport(TTransport transport) {
-    while (transport != null) {
-      if (transport instanceof TSaslServerTransport) {
-        transport = ((TSaslServerTransport) transport).getUnderlyingTransport();
-      }
-      if (transport instanceof TSaslClientTransport) {
-        transport = ((TSaslClientTransport) transport).getUnderlyingTransport();
-      }
-      if (transport instanceof TSocket) {
-        return (TSocket) transport;
-      }
-    }
-    return null;
-  }
-
-  private static final ThreadLocal<String> THREAD_LOCAL_IP_ADDRESS = new ThreadLocal<String>() {
-    @Override
-    protected synchronized String initialValue() {
-      return null;
-    }
-  };
-
-  private static final ThreadLocal<String> THREAD_LOCAL_USER_NAME = new ThreadLocal<String>() {
-    @Override
-    protected synchronized String initialValue() {
-      return null;
-    }
-  };
-
-  public static String getUserIpAddress() {
-    return THREAD_LOCAL_IP_ADDRESS.get();
-  }
-
-  public static String getUserName() {
-    return THREAD_LOCAL_USER_NAME.get();
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/CLIService.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/CLIService.java
deleted file mode 100644
index 791ddcbd2c5b6..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/CLIService.java
+++ /dev/null
@@ -1,507 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CancellationException;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import javax.security.auth.login.LoginException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
-import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.shims.Utils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hive.service.CompositeService;
-import org.apache.hive.service.ServiceException;
-import org.apache.hive.service.auth.HiveAuthFactory;
-import org.apache.hive.service.cli.operation.Operation;
-import org.apache.hive.service.cli.session.SessionManager;
-import org.apache.hive.service.cli.thrift.TProtocolVersion;
-import org.apache.hive.service.server.HiveServer2;
-
-/**
- * CLIService.
- *
- */
-public class CLIService extends CompositeService implements ICLIService {
-
-  public static final TProtocolVersion SERVER_VERSION;
-
-  static {
-    TProtocolVersion[] protocols = TProtocolVersion.values();
-    SERVER_VERSION = protocols[protocols.length - 1];
-  }
-
-  private final Log LOG = LogFactory.getLog(CLIService.class.getName());
-
-  private HiveConf hiveConf;
-  private SessionManager sessionManager;
-  private UserGroupInformation serviceUGI;
-  private UserGroupInformation httpUGI;
-  // The HiveServer2 instance running this service
-  private final HiveServer2 hiveServer2;
-
-  public CLIService(HiveServer2 hiveServer2) {
-    super(CLIService.class.getSimpleName());
-    this.hiveServer2 = hiveServer2;
-  }
-
-  @Override
-  public synchronized void init(HiveConf hiveConf) {
-    this.hiveConf = hiveConf;
-    sessionManager = new SessionManager(hiveServer2);
-    addService(sessionManager);
-    //  If the hadoop cluster is secure, do a kerberos login for the service from the keytab
-    if (UserGroupInformation.isSecurityEnabled()) {
-      try {
-        HiveAuthFactory.loginFromKeytab(hiveConf);
-        this.serviceUGI = Utils.getUGI();
-      } catch (IOException e) {
-        throw new ServiceException("Unable to login to kerberos with given principal/keytab", e);
-      } catch (LoginException e) {
-        throw new ServiceException("Unable to login to kerberos with given principal/keytab", e);
-      }
-
-      // Also try creating a UGI object for the SPNego principal
-      String principal = hiveConf.getVar(ConfVars.HIVE_SERVER2_SPNEGO_PRINCIPAL);
-      String keyTabFile = hiveConf.getVar(ConfVars.HIVE_SERVER2_SPNEGO_KEYTAB);
-      if (principal.isEmpty() || keyTabFile.isEmpty()) {
-        LOG.info("SPNego httpUGI not created, spNegoPrincipal: " + principal +
-            ", ketabFile: " + keyTabFile);
-      } else {
-        try {
-          this.httpUGI = HiveAuthFactory.loginFromSpnegoKeytabAndReturnUGI(hiveConf);
-          LOG.info("SPNego httpUGI successfully created.");
-        } catch (IOException e) {
-          LOG.warn("SPNego httpUGI creation failed: ", e);
-        }
-      }
-    }
-    // creates connection to HMS and thus *must* occur after kerberos login above
-    try {
-      applyAuthorizationConfigPolicy(hiveConf);
-    } catch (Exception e) {
-      throw new RuntimeException("Error applying authorization policy on hive configuration: "
-          + e.getMessage(), e);
-    }
-    setupBlockedUdfs();
-    super.init(hiveConf);
-  }
-
-  private void applyAuthorizationConfigPolicy(HiveConf newHiveConf) throws HiveException,
-      MetaException {
-    // authorization setup using SessionState should be revisited eventually, as
-    // authorization and authentication are not session specific settings
-    SessionState ss = new SessionState(newHiveConf);
-    ss.setIsHiveServerQuery(true);
-    SessionState.start(ss);
-    ss.applyAuthorizationPolicy();
-  }
-
-  private void setupBlockedUdfs() {
-    FunctionRegistry.setupPermissionsForBuiltinUDFs(
-        hiveConf.getVar(ConfVars.HIVE_SERVER2_BUILTIN_UDF_WHITELIST),
-        hiveConf.getVar(ConfVars.HIVE_SERVER2_BUILTIN_UDF_BLACKLIST));
-  }
-
-  public UserGroupInformation getServiceUGI() {
-    return this.serviceUGI;
-  }
-
-  public UserGroupInformation getHttpUGI() {
-    return this.httpUGI;
-  }
-
-  @Override
-  public synchronized void start() {
-    super.start();
-    // Initialize and test a connection to the metastore
-    IMetaStoreClient metastoreClient = null;
-    try {
-      metastoreClient = new HiveMetaStoreClient(hiveConf);
-      metastoreClient.getDatabases("default");
-    } catch (Exception e) {
-      throw new ServiceException("Unable to connect to MetaStore!", e);
-    }
-    finally {
-      if (metastoreClient != null) {
-        metastoreClient.close();
-      }
-    }
-  }
-
-  @Override
-  public synchronized void stop() {
-    super.stop();
-  }
-
-  /**
-   * @deprecated  Use {@link #openSession(TProtocolVersion, String, String, String, Map)}
-   */
-  @Deprecated
-  public SessionHandle openSession(TProtocolVersion protocol, String username, String password,
-      Map<String, String> configuration) throws HiveSQLException {
-    SessionHandle sessionHandle = sessionManager.openSession(protocol, username, password, null, configuration, false, null);
-    LOG.debug(sessionHandle + ": openSession()");
-    return sessionHandle;
-  }
-
-  /**
-   * @deprecated  Use {@link #openSessionWithImpersonation(TProtocolVersion, String, String, String, Map, String)}
-   */
-  @Deprecated
-  public SessionHandle openSessionWithImpersonation(TProtocolVersion protocol, String username,
-      String password, Map<String, String> configuration, String delegationToken)
-          throws HiveSQLException {
-    SessionHandle sessionHandle = sessionManager.openSession(protocol, username, password, null, configuration,
-        true, delegationToken);
-    LOG.debug(sessionHandle + ": openSessionWithImpersonation()");
-    return sessionHandle;
-  }
-
-  public SessionHandle openSession(TProtocolVersion protocol, String username, String password, String ipAddress,
-      Map<String, String> configuration) throws HiveSQLException {
-    SessionHandle sessionHandle = sessionManager.openSession(protocol, username, password, ipAddress, configuration, false, null);
-    LOG.debug(sessionHandle + ": openSession()");
-    return sessionHandle;
-  }
-
-  public SessionHandle openSessionWithImpersonation(TProtocolVersion protocol, String username,
-      String password, String ipAddress, Map<String, String> configuration, String delegationToken)
-          throws HiveSQLException {
-    SessionHandle sessionHandle = sessionManager.openSession(protocol, username, password, ipAddress, configuration,
-        true, delegationToken);
-    LOG.debug(sessionHandle + ": openSession()");
-    return sessionHandle;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#openSession(java.lang.String, java.lang.String, java.util.Map)
-   */
-  @Override
-  public SessionHandle openSession(String username, String password, Map<String, String> configuration)
-      throws HiveSQLException {
-    SessionHandle sessionHandle = sessionManager.openSession(SERVER_VERSION, username, password, null, configuration, false, null);
-    LOG.debug(sessionHandle + ": openSession()");
-    return sessionHandle;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#openSession(java.lang.String, java.lang.String, java.util.Map)
-   */
-  @Override
-  public SessionHandle openSessionWithImpersonation(String username, String password, Map<String, String> configuration,
-      String delegationToken) throws HiveSQLException {
-    SessionHandle sessionHandle = sessionManager.openSession(SERVER_VERSION, username, password, null, configuration,
-        true, delegationToken);
-    LOG.debug(sessionHandle + ": openSession()");
-    return sessionHandle;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#closeSession(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public void closeSession(SessionHandle sessionHandle)
-      throws HiveSQLException {
-    sessionManager.closeSession(sessionHandle);
-    LOG.debug(sessionHandle + ": closeSession()");
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getInfo(org.apache.hive.service.cli.SessionHandle, java.util.List)
-   */
-  @Override
-  public GetInfoValue getInfo(SessionHandle sessionHandle, GetInfoType getInfoType)
-      throws HiveSQLException {
-    GetInfoValue infoValue = sessionManager.getSession(sessionHandle)
-        .getInfo(getInfoType);
-    LOG.debug(sessionHandle + ": getInfo()");
-    return infoValue;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#executeStatement(org.apache.hive.service.cli.SessionHandle,
-   *  java.lang.String, java.util.Map)
-   */
-  @Override
-  public OperationHandle executeStatement(SessionHandle sessionHandle, String statement,
-      Map<String, String> confOverlay)
-          throws HiveSQLException {
-    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
-        .executeStatement(statement, confOverlay);
-    LOG.debug(sessionHandle + ": executeStatement()");
-    return opHandle;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#executeStatementAsync(org.apache.hive.service.cli.SessionHandle,
-   *  java.lang.String, java.util.Map)
-   */
-  @Override
-  public OperationHandle executeStatementAsync(SessionHandle sessionHandle, String statement,
-      Map<String, String> confOverlay) throws HiveSQLException {
-    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
-        .executeStatementAsync(statement, confOverlay);
-    LOG.debug(sessionHandle + ": executeStatementAsync()");
-    return opHandle;
-  }
-
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getTypeInfo(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public OperationHandle getTypeInfo(SessionHandle sessionHandle)
-      throws HiveSQLException {
-    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
-        .getTypeInfo();
-    LOG.debug(sessionHandle + ": getTypeInfo()");
-    return opHandle;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getCatalogs(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public OperationHandle getCatalogs(SessionHandle sessionHandle)
-      throws HiveSQLException {
-    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
-        .getCatalogs();
-    LOG.debug(sessionHandle + ": getCatalogs()");
-    return opHandle;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getSchemas(org.apache.hive.service.cli.SessionHandle, java.lang.String, java.lang.String)
-   */
-  @Override
-  public OperationHandle getSchemas(SessionHandle sessionHandle,
-      String catalogName, String schemaName)
-          throws HiveSQLException {
-    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
-        .getSchemas(catalogName, schemaName);
-    LOG.debug(sessionHandle + ": getSchemas()");
-    return opHandle;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getTables(org.apache.hive.service.cli.SessionHandle, java.lang.String, java.lang.String, java.lang.String, java.util.List)
-   */
-  @Override
-  public OperationHandle getTables(SessionHandle sessionHandle,
-      String catalogName, String schemaName, String tableName, List<String> tableTypes)
-          throws HiveSQLException {
-    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
-        .getTables(catalogName, schemaName, tableName, tableTypes);
-    LOG.debug(sessionHandle + ": getTables()");
-    return opHandle;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getTableTypes(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public OperationHandle getTableTypes(SessionHandle sessionHandle)
-      throws HiveSQLException {
-    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
-        .getTableTypes();
-    LOG.debug(sessionHandle + ": getTableTypes()");
-    return opHandle;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getColumns(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public OperationHandle getColumns(SessionHandle sessionHandle,
-      String catalogName, String schemaName, String tableName, String columnName)
-          throws HiveSQLException {
-    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
-        .getColumns(catalogName, schemaName, tableName, columnName);
-    LOG.debug(sessionHandle + ": getColumns()");
-    return opHandle;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getFunctions(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public OperationHandle getFunctions(SessionHandle sessionHandle,
-      String catalogName, String schemaName, String functionName)
-          throws HiveSQLException {
-    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
-        .getFunctions(catalogName, schemaName, functionName);
-    LOG.debug(sessionHandle + ": getFunctions()");
-    return opHandle;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getOperationStatus(org.apache.hive.service.cli.OperationHandle)
-   */
-  @Override
-  public OperationStatus getOperationStatus(OperationHandle opHandle)
-      throws HiveSQLException {
-    Operation operation = sessionManager.getOperationManager().getOperation(opHandle);
-    /**
-     * If this is a background operation run asynchronously,
-     * we block for a configured duration, before we return
-     * (duration: HIVE_SERVER2_LONG_POLLING_TIMEOUT).
-     * However, if the background operation is complete, we return immediately.
-     */
-    if (operation.shouldRunAsync()) {
-      HiveConf conf = operation.getParentSession().getHiveConf();
-      long timeout = HiveConf.getTimeVar(conf,
-          HiveConf.ConfVars.HIVE_SERVER2_LONG_POLLING_TIMEOUT, TimeUnit.MILLISECONDS);
-      try {
-        operation.getBackgroundHandle().get(timeout, TimeUnit.MILLISECONDS);
-      } catch (TimeoutException e) {
-        // No Op, return to the caller since long polling timeout has expired
-        LOG.trace(opHandle + ": Long polling timed out");
-      } catch (CancellationException e) {
-        // The background operation thread was cancelled
-        LOG.trace(opHandle + ": The background operation was cancelled", e);
-      } catch (ExecutionException e) {
-        // The background operation thread was aborted
-        LOG.warn(opHandle + ": The background operation was aborted", e);
-      } catch (InterruptedException e) {
-        // No op, this thread was interrupted
-        // In this case, the call might return sooner than long polling timeout
-      }
-    }
-    OperationStatus opStatus = operation.getStatus();
-    LOG.debug(opHandle + ": getOperationStatus()");
-    return opStatus;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#cancelOperation(org.apache.hive.service.cli.OperationHandle)
-   */
-  @Override
-  public void cancelOperation(OperationHandle opHandle)
-      throws HiveSQLException {
-    sessionManager.getOperationManager().getOperation(opHandle)
-    .getParentSession().cancelOperation(opHandle);
-    LOG.debug(opHandle + ": cancelOperation()");
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#closeOperation(org.apache.hive.service.cli.OperationHandle)
-   */
-  @Override
-  public void closeOperation(OperationHandle opHandle)
-      throws HiveSQLException {
-    sessionManager.getOperationManager().getOperation(opHandle)
-    .getParentSession().closeOperation(opHandle);
-    LOG.debug(opHandle + ": closeOperation");
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getResultSetMetadata(org.apache.hive.service.cli.OperationHandle)
-   */
-  @Override
-  public TableSchema getResultSetMetadata(OperationHandle opHandle)
-      throws HiveSQLException {
-    TableSchema tableSchema = sessionManager.getOperationManager()
-        .getOperation(opHandle).getParentSession().getResultSetMetadata(opHandle);
-    LOG.debug(opHandle + ": getResultSetMetadata()");
-    return tableSchema;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#fetchResults(org.apache.hive.service.cli.OperationHandle)
-   */
-  @Override
-  public RowSet fetchResults(OperationHandle opHandle)
-      throws HiveSQLException {
-    return fetchResults(opHandle, Operation.DEFAULT_FETCH_ORIENTATION,
-        Operation.DEFAULT_FETCH_MAX_ROWS, FetchType.QUERY_OUTPUT);
-  }
-
-  @Override
-  public RowSet fetchResults(OperationHandle opHandle, FetchOrientation orientation,
-                             long maxRows, FetchType fetchType) throws HiveSQLException {
-    RowSet rowSet = sessionManager.getOperationManager().getOperation(opHandle)
-        .getParentSession().fetchResults(opHandle, orientation, maxRows, fetchType);
-    LOG.debug(opHandle + ": fetchResults()");
-    return rowSet;
-  }
-
-  // obtain delegation token for the give user from metastore
-  public synchronized String getDelegationTokenFromMetaStore(String owner)
-      throws HiveSQLException, UnsupportedOperationException, LoginException, IOException {
-    if (!hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL) ||
-        !hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS)) {
-      throw new UnsupportedOperationException(
-          "delegation token is can only be obtained for a secure remote metastore");
-    }
-
-    try {
-      Hive.closeCurrent();
-      return Hive.get(hiveConf).getDelegationToken(owner, owner);
-    } catch (HiveException e) {
-      if (e.getCause() instanceof UnsupportedOperationException) {
-        throw (UnsupportedOperationException)e.getCause();
-      } else {
-        throw new HiveSQLException("Error connect metastore to setup impersonation", e);
-      }
-    }
-  }
-
-  @Override
-  public String getDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
-      String owner, String renewer) throws HiveSQLException {
-    String delegationToken = sessionManager.getSession(sessionHandle)
-        .getDelegationToken(authFactory, owner, renewer);
-    LOG.info(sessionHandle  + ": getDelegationToken()");
-    return delegationToken;
-  }
-
-  @Override
-  public void cancelDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
-      String tokenStr) throws HiveSQLException {
-    sessionManager.getSession(sessionHandle).cancelDelegationToken(authFactory, tokenStr);
-    LOG.info(sessionHandle  + ": cancelDelegationToken()");
-  }
-
-  @Override
-  public void renewDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
-      String tokenStr) throws HiveSQLException {
-    sessionManager.getSession(sessionHandle).renewDelegationToken(authFactory, tokenStr);
-    LOG.info(sessionHandle  + ": renewDelegationToken()");
-  }
-
-  public SessionManager getSessionManager() {
-    return sessionManager;
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/Column.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/Column.java
deleted file mode 100644
index 26d0f718f383a..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/Column.java
+++ /dev/null
@@ -1,423 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import java.nio.ByteBuffer;
-import java.util.AbstractList;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.List;
-
-import com.google.common.primitives.Booleans;
-import com.google.common.primitives.Bytes;
-import com.google.common.primitives.Doubles;
-import com.google.common.primitives.Ints;
-import com.google.common.primitives.Longs;
-import com.google.common.primitives.Shorts;
-import org.apache.hive.service.cli.thrift.TBinaryColumn;
-import org.apache.hive.service.cli.thrift.TBoolColumn;
-import org.apache.hive.service.cli.thrift.TByteColumn;
-import org.apache.hive.service.cli.thrift.TColumn;
-import org.apache.hive.service.cli.thrift.TDoubleColumn;
-import org.apache.hive.service.cli.thrift.TI16Column;
-import org.apache.hive.service.cli.thrift.TI32Column;
-import org.apache.hive.service.cli.thrift.TI64Column;
-import org.apache.hive.service.cli.thrift.TStringColumn;
-
-/**
- * Column.
- */
-public class Column extends AbstractList {
-
-  private static final int DEFAULT_SIZE = 100;
-
-  private final Type type;
-
-  private BitSet nulls;
-
-  private int size;
-  private boolean[] boolVars;
-  private byte[] byteVars;
-  private short[] shortVars;
-  private int[] intVars;
-  private long[] longVars;
-  private double[] doubleVars;
-  private List<String> stringVars;
-  private List<ByteBuffer> binaryVars;
-
-  public Column(Type type, BitSet nulls, Object values) {
-    this.type = type;
-    this.nulls = nulls;
-    if (type == Type.BOOLEAN_TYPE) {
-      boolVars = (boolean[]) values;
-      size = boolVars.length;
-    } else if (type == Type.TINYINT_TYPE) {
-      byteVars = (byte[]) values;
-      size = byteVars.length;
-    } else if (type == Type.SMALLINT_TYPE) {
-      shortVars = (short[]) values;
-      size = shortVars.length;
-    } else if (type == Type.INT_TYPE) {
-      intVars = (int[]) values;
-      size = intVars.length;
-    } else if (type == Type.BIGINT_TYPE) {
-      longVars = (long[]) values;
-      size = longVars.length;
-    } else if (type == Type.DOUBLE_TYPE) {
-      doubleVars = (double[]) values;
-      size = doubleVars.length;
-    } else if (type == Type.BINARY_TYPE) {
-      binaryVars = (List<ByteBuffer>) values;
-      size = binaryVars.size();
-    } else if (type == Type.STRING_TYPE) {
-      stringVars = (List<String>) values;
-      size = stringVars.size();
-    } else {
-      throw new IllegalStateException("invalid union object");
-    }
-  }
-
-  public Column(Type type) {
-    nulls = new BitSet();
-    switch (type) {
-      case BOOLEAN_TYPE:
-        boolVars = new boolean[DEFAULT_SIZE];
-        break;
-      case TINYINT_TYPE:
-        byteVars = new byte[DEFAULT_SIZE];
-        break;
-      case SMALLINT_TYPE:
-        shortVars = new short[DEFAULT_SIZE];
-        break;
-      case INT_TYPE:
-        intVars = new int[DEFAULT_SIZE];
-        break;
-      case BIGINT_TYPE:
-        longVars = new long[DEFAULT_SIZE];
-        break;
-      case FLOAT_TYPE:
-      case DOUBLE_TYPE:
-        type = Type.DOUBLE_TYPE;
-        doubleVars = new double[DEFAULT_SIZE];
-        break;
-      case BINARY_TYPE:
-        binaryVars = new ArrayList<ByteBuffer>();
-        break;
-      default:
-        type = Type.STRING_TYPE;
-        stringVars = new ArrayList<String>();
-    }
-    this.type = type;
-  }
-
-  public Column(TColumn colValues) {
-    if (colValues.isSetBoolVal()) {
-      type = Type.BOOLEAN_TYPE;
-      nulls = toBitset(colValues.getBoolVal().getNulls());
-      boolVars = Booleans.toArray(colValues.getBoolVal().getValues());
-      size = boolVars.length;
-    } else if (colValues.isSetByteVal()) {
-      type = Type.TINYINT_TYPE;
-      nulls = toBitset(colValues.getByteVal().getNulls());
-      byteVars = Bytes.toArray(colValues.getByteVal().getValues());
-      size = byteVars.length;
-    } else if (colValues.isSetI16Val()) {
-      type = Type.SMALLINT_TYPE;
-      nulls = toBitset(colValues.getI16Val().getNulls());
-      shortVars = Shorts.toArray(colValues.getI16Val().getValues());
-      size = shortVars.length;
-    } else if (colValues.isSetI32Val()) {
-      type = Type.INT_TYPE;
-      nulls = toBitset(colValues.getI32Val().getNulls());
-      intVars = Ints.toArray(colValues.getI32Val().getValues());
-      size = intVars.length;
-    } else if (colValues.isSetI64Val()) {
-      type = Type.BIGINT_TYPE;
-      nulls = toBitset(colValues.getI64Val().getNulls());
-      longVars = Longs.toArray(colValues.getI64Val().getValues());
-      size = longVars.length;
-    } else if (colValues.isSetDoubleVal()) {
-      type = Type.DOUBLE_TYPE;
-      nulls = toBitset(colValues.getDoubleVal().getNulls());
-      doubleVars = Doubles.toArray(colValues.getDoubleVal().getValues());
-      size = doubleVars.length;
-    } else if (colValues.isSetBinaryVal()) {
-      type = Type.BINARY_TYPE;
-      nulls = toBitset(colValues.getBinaryVal().getNulls());
-      binaryVars = colValues.getBinaryVal().getValues();
-      size = binaryVars.size();
-    } else if (colValues.isSetStringVal()) {
-      type = Type.STRING_TYPE;
-      nulls = toBitset(colValues.getStringVal().getNulls());
-      stringVars = colValues.getStringVal().getValues();
-      size = stringVars.size();
-    } else {
-      throw new IllegalStateException("invalid union object");
-    }
-  }
-
-  public Column extractSubset(int start, int end) {
-    BitSet subNulls = nulls.get(start, end);
-    if (type == Type.BOOLEAN_TYPE) {
-      Column subset = new Column(type, subNulls, Arrays.copyOfRange(boolVars, start, end));
-      boolVars = Arrays.copyOfRange(boolVars, end, size);
-      nulls = nulls.get(start, size);
-      size = boolVars.length;
-      return subset;
-    }
-    if (type == Type.TINYINT_TYPE) {
-      Column subset = new Column(type, subNulls, Arrays.copyOfRange(byteVars, start, end));
-      byteVars = Arrays.copyOfRange(byteVars, end, size);
-      nulls = nulls.get(start, size);
-      size = byteVars.length;
-      return subset;
-    }
-    if (type == Type.SMALLINT_TYPE) {
-      Column subset = new Column(type, subNulls, Arrays.copyOfRange(shortVars, start, end));
-      shortVars = Arrays.copyOfRange(shortVars, end, size);
-      nulls = nulls.get(start, size);
-      size = shortVars.length;
-      return subset;
-    }
-    if (type == Type.INT_TYPE) {
-      Column subset = new Column(type, subNulls, Arrays.copyOfRange(intVars, start, end));
-      intVars = Arrays.copyOfRange(intVars, end, size);
-      nulls = nulls.get(start, size);
-      size = intVars.length;
-      return subset;
-    }
-    if (type == Type.BIGINT_TYPE) {
-      Column subset = new Column(type, subNulls, Arrays.copyOfRange(longVars, start, end));
-      longVars = Arrays.copyOfRange(longVars, end, size);
-      nulls = nulls.get(start, size);
-      size = longVars.length;
-      return subset;
-    }
-    if (type == Type.DOUBLE_TYPE) {
-      Column subset = new Column(type, subNulls, Arrays.copyOfRange(doubleVars, start, end));
-      doubleVars = Arrays.copyOfRange(doubleVars, end, size);
-      nulls = nulls.get(start, size);
-      size = doubleVars.length;
-      return subset;
-    }
-    if (type == Type.BINARY_TYPE) {
-      Column subset = new Column(type, subNulls, binaryVars.subList(start, end));
-      binaryVars = binaryVars.subList(end, binaryVars.size());
-      nulls = nulls.get(start, size);
-      size = binaryVars.size();
-      return subset;
-    }
-    if (type == Type.STRING_TYPE) {
-      Column subset = new Column(type, subNulls, stringVars.subList(start, end));
-      stringVars = stringVars.subList(end, stringVars.size());
-      nulls = nulls.get(start, size);
-      size = stringVars.size();
-      return subset;
-    }
-    throw new IllegalStateException("invalid union object");
-  }
-
-  private static final byte[] MASKS = new byte[] {
-      0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, (byte)0x80
-  };
-
-  private static BitSet toBitset(byte[] nulls) {
-    BitSet bitset = new BitSet();
-    int bits = nulls.length * 8;
-    for (int i = 0; i < bits; i++) {
-      bitset.set(i, (nulls[i / 8] & MASKS[i % 8]) != 0);
-    }
-    return bitset;
-  }
-
-  private static byte[] toBinary(BitSet bitset) {
-    byte[] nulls = new byte[1 + (bitset.length() / 8)];
-    for (int i = 0; i < bitset.length(); i++) {
-      nulls[i / 8] |= bitset.get(i) ? MASKS[i % 8] : 0;
-    }
-    return nulls;
-  }
-
-  public Type getType() {
-    return type;
-  }
-
-  @Override
-  public Object get(int index) {
-    if (nulls.get(index)) {
-      return null;
-    }
-    switch (type) {
-      case BOOLEAN_TYPE:
-        return boolVars[index];
-      case TINYINT_TYPE:
-        return byteVars[index];
-      case SMALLINT_TYPE:
-        return shortVars[index];
-      case INT_TYPE:
-        return intVars[index];
-      case BIGINT_TYPE:
-        return longVars[index];
-      case DOUBLE_TYPE:
-        return doubleVars[index];
-      case STRING_TYPE:
-        return stringVars.get(index);
-      case BINARY_TYPE:
-        return binaryVars.get(index).array();
-    }
-    return null;
-  }
-
-  @Override
-  public int size() {
-    return size;
-  }
-
-  public TColumn toTColumn() {
-    TColumn value = new TColumn();
-    ByteBuffer nullMasks = ByteBuffer.wrap(toBinary(nulls));
-    switch (type) {
-      case BOOLEAN_TYPE:
-        value.setBoolVal(new TBoolColumn(Booleans.asList(Arrays.copyOfRange(boolVars, 0, size)), nullMasks));
-        break;
-      case TINYINT_TYPE:
-        value.setByteVal(new TByteColumn(Bytes.asList(Arrays.copyOfRange(byteVars, 0, size)), nullMasks));
-        break;
-      case SMALLINT_TYPE:
-        value.setI16Val(new TI16Column(Shorts.asList(Arrays.copyOfRange(shortVars, 0, size)), nullMasks));
-        break;
-      case INT_TYPE:
-        value.setI32Val(new TI32Column(Ints.asList(Arrays.copyOfRange(intVars, 0, size)), nullMasks));
-        break;
-      case BIGINT_TYPE:
-        value.setI64Val(new TI64Column(Longs.asList(Arrays.copyOfRange(longVars, 0, size)), nullMasks));
-        break;
-      case DOUBLE_TYPE:
-        value.setDoubleVal(new TDoubleColumn(Doubles.asList(Arrays.copyOfRange(doubleVars, 0, size)), nullMasks));
-        break;
-      case STRING_TYPE:
-        value.setStringVal(new TStringColumn(stringVars, nullMasks));
-        break;
-      case BINARY_TYPE:
-        value.setBinaryVal(new TBinaryColumn(binaryVars, nullMasks));
-        break;
-    }
-    return value;
-  }
-
-  private static final ByteBuffer EMPTY_BINARY = ByteBuffer.allocate(0);
-  private static final String EMPTY_STRING = "";
-
-  public void addValue(Type type, Object field) {
-    switch (type) {
-      case BOOLEAN_TYPE:
-        nulls.set(size, field == null);
-        boolVars()[size] = field == null ? true : (Boolean)field;
-        break;
-      case TINYINT_TYPE:
-        nulls.set(size, field == null);
-        byteVars()[size] = field == null ? 0 : (Byte) field;
-        break;
-      case SMALLINT_TYPE:
-        nulls.set(size, field == null);
-        shortVars()[size] = field == null ? 0 : (Short)field;
-        break;
-      case INT_TYPE:
-        nulls.set(size, field == null);
-        intVars()[size] = field == null ? 0 : (Integer)field;
-        break;
-      case BIGINT_TYPE:
-        nulls.set(size, field == null);
-        longVars()[size] = field == null ? 0 : (Long)field;
-        break;
-      case FLOAT_TYPE:
-        nulls.set(size, field == null);
-        doubleVars()[size] = field == null ? 0 : Double.valueOf(field.toString());
-        break;
-      case DOUBLE_TYPE:
-        nulls.set(size, field == null);
-        doubleVars()[size] = field == null ? 0 : (Double)field;
-        break;
-      case BINARY_TYPE:
-        nulls.set(binaryVars.size(), field == null);
-        binaryVars.add(field == null ? EMPTY_BINARY : ByteBuffer.wrap((byte[])field));
-        break;
-      default:
-        nulls.set(stringVars.size(), field == null);
-        stringVars.add(field == null ? EMPTY_STRING : String.valueOf(field));
-        break;
-    }
-    size++;
-  }
-
-  private boolean[] boolVars() {
-    if (boolVars.length == size) {
-      boolean[] newVars = new boolean[size << 1];
-      System.arraycopy(boolVars, 0, newVars, 0, size);
-      return boolVars = newVars;
-    }
-    return boolVars;
-  }
-
-  private byte[] byteVars() {
-    if (byteVars.length == size) {
-      byte[] newVars = new byte[size << 1];
-      System.arraycopy(byteVars, 0, newVars, 0, size);
-      return byteVars = newVars;
-    }
-    return byteVars;
-  }
-
-  private short[] shortVars() {
-    if (shortVars.length == size) {
-      short[] newVars = new short[size << 1];
-      System.arraycopy(shortVars, 0, newVars, 0, size);
-      return shortVars = newVars;
-    }
-    return shortVars;
-  }
-
-  private int[] intVars() {
-    if (intVars.length == size) {
-      int[] newVars = new int[size << 1];
-      System.arraycopy(intVars, 0, newVars, 0, size);
-      return intVars = newVars;
-    }
-    return intVars;
-  }
-
-  private long[] longVars() {
-    if (longVars.length == size) {
-      long[] newVars = new long[size << 1];
-      System.arraycopy(longVars, 0, newVars, 0, size);
-      return longVars = newVars;
-    }
-    return longVars;
-  }
-
-  private double[] doubleVars() {
-    if (doubleVars.length == size) {
-      double[] newVars = new double[size << 1];
-      System.arraycopy(doubleVars, 0, newVars, 0, size);
-      return doubleVars = newVars;
-    }
-    return doubleVars;
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ColumnBasedSet.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ColumnBasedSet.java
deleted file mode 100644
index 47a582e2223e4..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ColumnBasedSet.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.hive.service.cli.thrift.TColumn;
-import org.apache.hive.service.cli.thrift.TRow;
-import org.apache.hive.service.cli.thrift.TRowSet;
-
-/**
- * ColumnBasedSet.
- */
-public class ColumnBasedSet implements RowSet {
-
-  private long startOffset;
-
-  private final Type[] types; // non-null only for writing (server-side)
-  private final List<Column> columns;
-
-  public ColumnBasedSet(TableSchema schema) {
-    types = schema.toTypes();
-    columns = new ArrayList<Column>();
-    for (ColumnDescriptor colDesc : schema.getColumnDescriptors()) {
-      columns.add(new Column(colDesc.getType()));
-    }
-  }
-
-  public ColumnBasedSet(TRowSet tRowSet) {
-    types = null;
-    columns = new ArrayList<Column>();
-    for (TColumn tvalue : tRowSet.getColumns()) {
-      columns.add(new Column(tvalue));
-    }
-    startOffset = tRowSet.getStartRowOffset();
-  }
-
-  private ColumnBasedSet(Type[] types, List<Column> columns, long startOffset) {
-    this.types = types;
-    this.columns = columns;
-    this.startOffset = startOffset;
-  }
-
-  @Override
-  public ColumnBasedSet addRow(Object[] fields) {
-    for (int i = 0; i < fields.length; i++) {
-      columns.get(i).addValue(types[i], fields[i]);
-    }
-    return this;
-  }
-
-  public List<Column> getColumns() {
-    return columns;
-  }
-
-  @Override
-  public int numColumns() {
-    return columns.size();
-  }
-
-  @Override
-  public int numRows() {
-    return columns.isEmpty() ? 0 : columns.get(0).size();
-  }
-
-  @Override
-  public ColumnBasedSet extractSubset(int maxRows) {
-    int numRows = Math.min(numRows(), maxRows);
-
-    List<Column> subset = new ArrayList<Column>();
-    for (int i = 0; i < columns.size(); i++) {
-      subset.add(columns.get(i).extractSubset(0, numRows));
-    }
-    ColumnBasedSet result = new ColumnBasedSet(types, subset, startOffset);
-    startOffset += numRows;
-    return result;
-  }
-
-  @Override
-  public long getStartOffset() {
-    return startOffset;
-  }
-
-  @Override
-  public void setStartOffset(long startOffset) {
-    this.startOffset = startOffset;
-  }
-
-  public TRowSet toTRowSet() {
-    TRowSet tRowSet = new TRowSet(startOffset, new ArrayList<TRow>());
-    for (int i = 0; i < columns.size(); i++) {
-      tRowSet.addToColumns(columns.get(i).toTColumn());
-    }
-    return tRowSet;
-  }
-
-  @Override
-  public Iterator<Object[]> iterator() {
-    return new Iterator<Object[]>() {
-
-      private int index;
-      private final Object[] convey = new Object[numColumns()];
-
-      @Override
-      public boolean hasNext() {
-        return index < numRows();
-      }
-
-      @Override
-      public Object[] next() {
-        for (int i = 0; i < columns.size(); i++) {
-          convey[i] = columns.get(i).get(index);
-        }
-        index++;
-        return convey;
-      }
-
-      @Override
-      public void remove() {
-        throw new UnsupportedOperationException("remove");
-      }
-    };
-  }
-
-  public Object[] fill(int index, Object[] convey) {
-    for (int i = 0; i < columns.size(); i++) {
-      convey[i] = columns.get(i).get(index);
-    }
-    return convey;
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ColumnDescriptor.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ColumnDescriptor.java
deleted file mode 100644
index f0bbf14693160..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ColumnDescriptor.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hive.service.cli.thrift.TColumnDesc;
-
-
-/**
- * ColumnDescriptor.
- *
- */
-public class ColumnDescriptor {
-  private final String name;
-  private final String comment;
-  private final TypeDescriptor type;
-  // ordinal position of this column in the schema
-  private final int position;
-
-  public ColumnDescriptor(String name, String comment, TypeDescriptor type, int position) {
-    this.name = name;
-    this.comment = comment;
-    this.type = type;
-    this.position = position;
-  }
-
-  public ColumnDescriptor(TColumnDesc tColumnDesc) {
-    name = tColumnDesc.getColumnName();
-    comment = tColumnDesc.getComment();
-    type = new TypeDescriptor(tColumnDesc.getTypeDesc());
-    position = tColumnDesc.getPosition();
-  }
-
-  public ColumnDescriptor(FieldSchema column, int position) {
-    name = column.getName();
-    comment = column.getComment();
-    type = new TypeDescriptor(column.getType());
-    this.position = position;
-  }
-
-  public static ColumnDescriptor newPrimitiveColumnDescriptor(String name, String comment, Type type, int position) {
-    // Current usage looks like it's only for metadata columns, but if that changes then
-    // this method may need to require a type qualifiers aruments.
-    return new ColumnDescriptor(name, comment, new TypeDescriptor(type), position);
-  }
-
-  public String getName() {
-    return name;
-  }
-
-  public String getComment() {
-    return comment;
-  }
-
-  public TypeDescriptor getTypeDescriptor() {
-    return type;
-  }
-
-  public int getOrdinalPosition() {
-    return position;
-  }
-
-  public TColumnDesc toTColumnDesc() {
-    TColumnDesc tColumnDesc = new TColumnDesc();
-    tColumnDesc.setColumnName(name);
-    tColumnDesc.setComment(comment);
-    tColumnDesc.setTypeDesc(type.toTTypeDesc());
-    tColumnDesc.setPosition(position);
-    return tColumnDesc;
-  }
-
-  public Type getType() {
-    return type.getType();
-  }
-
-  public boolean isPrimitive() {
-    return type.getType().isPrimitiveType();
-  }
-
-  public String getTypeName() {
-    return type.getTypeName();
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ColumnValue.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ColumnValue.java
deleted file mode 100644
index a770bea9c2aa6..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ColumnValue.java
+++ /dev/null
@@ -1,304 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import java.math.BigDecimal;
-import java.sql.Date;
-import java.sql.Timestamp;
-
-import org.apache.hadoop.hive.common.type.HiveChar;
-import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
-import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
-import org.apache.hadoop.hive.common.type.HiveVarchar;
-import org.apache.hive.service.cli.thrift.TBoolValue;
-import org.apache.hive.service.cli.thrift.TByteValue;
-import org.apache.hive.service.cli.thrift.TColumnValue;
-import org.apache.hive.service.cli.thrift.TDoubleValue;
-import org.apache.hive.service.cli.thrift.TI16Value;
-import org.apache.hive.service.cli.thrift.TI32Value;
-import org.apache.hive.service.cli.thrift.TI64Value;
-import org.apache.hive.service.cli.thrift.TStringValue;
-
-import org.apache.spark.unsafe.types.UTF8String;
-
-/**
- * Protocols before HIVE_CLI_SERVICE_PROTOCOL_V6 (used by RowBasedSet)
- *
- */
-public class ColumnValue {
-
-  private static TColumnValue booleanValue(Boolean value) {
-    TBoolValue tBoolValue = new TBoolValue();
-    if (value != null) {
-      tBoolValue.setValue(value);
-    }
-    return TColumnValue.boolVal(tBoolValue);
-  }
-
-  private static TColumnValue byteValue(Byte value) {
-    TByteValue tByteValue = new TByteValue();
-    if (value != null) {
-      tByteValue.setValue(value);
-    }
-    return TColumnValue.byteVal(tByteValue);
-  }
-
-  private static TColumnValue shortValue(Short value) {
-    TI16Value tI16Value = new TI16Value();
-    if (value != null) {
-      tI16Value.setValue(value);
-    }
-    return TColumnValue.i16Val(tI16Value);
-  }
-
-  private static TColumnValue intValue(Integer value) {
-    TI32Value tI32Value = new TI32Value();
-    if (value != null) {
-      tI32Value.setValue(value);
-    }
-    return TColumnValue.i32Val(tI32Value);
-  }
-
-  private static TColumnValue longValue(Long value) {
-    TI64Value tI64Value = new TI64Value();
-    if (value != null) {
-      tI64Value.setValue(value);
-    }
-    return TColumnValue.i64Val(tI64Value);
-  }
-
-  private static TColumnValue floatValue(Float value) {
-    TDoubleValue tDoubleValue = new TDoubleValue();
-    if (value != null) {
-      tDoubleValue.setValue(value);
-    }
-    return TColumnValue.doubleVal(tDoubleValue);
-  }
-
-  private static TColumnValue doubleValue(Double value) {
-    TDoubleValue tDoubleValue = new TDoubleValue();
-    if (value != null) {
-      tDoubleValue.setValue(value);
-    }
-    return TColumnValue.doubleVal(tDoubleValue);
-  }
-
-  private static TColumnValue stringValue(String value) {
-    TStringValue tStringValue = new TStringValue();
-    if (value != null) {
-      tStringValue.setValue(value);
-    }
-    return TColumnValue.stringVal(tStringValue);
-  }
-
-  private static TColumnValue stringValue(HiveChar value) {
-    TStringValue tStringValue = new TStringValue();
-    if (value != null) {
-      tStringValue.setValue(value.toString());
-    }
-    return TColumnValue.stringVal(tStringValue);
-  }
-
-  private static TColumnValue stringValue(HiveVarchar value) {
-    TStringValue tStringValue = new TStringValue();
-    if (value != null) {
-      tStringValue.setValue(value.toString());
-    }
-    return TColumnValue.stringVal(tStringValue);
-  }
-
-  private static TColumnValue dateValue(Date value) {
-    TStringValue tStringValue = new TStringValue();
-    if (value != null) {
-      tStringValue.setValue(value.toString());
-    }
-    return new TColumnValue(TColumnValue.stringVal(tStringValue));
-  }
-
-  private static TColumnValue timestampValue(Timestamp value) {
-    TStringValue tStringValue = new TStringValue();
-    if (value != null) {
-      tStringValue.setValue(value.toString());
-    }
-    return TColumnValue.stringVal(tStringValue);
-  }
-
-  private static TColumnValue stringValue(HiveIntervalYearMonth value) {
-    TStringValue tStrValue = new TStringValue();
-    if (value != null) {
-      tStrValue.setValue(value.toString());
-    }
-    return TColumnValue.stringVal(tStrValue);
-  }
-
-  private static TColumnValue stringValue(HiveIntervalDayTime value) {
-    TStringValue tStrValue = new TStringValue();
-    if (value != null) {
-      tStrValue.setValue(value.toString());
-    }
-    return TColumnValue.stringVal(tStrValue);
-  }
-
-  public static TColumnValue toTColumnValue(Type type, Object value) {
-    switch (type) {
-    case BOOLEAN_TYPE:
-      return booleanValue((Boolean)value);
-    case TINYINT_TYPE:
-      return byteValue((Byte)value);
-    case SMALLINT_TYPE:
-      return shortValue((Short)value);
-    case INT_TYPE:
-      return intValue((Integer)value);
-    case BIGINT_TYPE:
-      return longValue((Long)value);
-    case FLOAT_TYPE:
-      return floatValue((Float)value);
-    case DOUBLE_TYPE:
-      return doubleValue((Double)value);
-    case STRING_TYPE:
-      return stringValue((String)value);
-    case CHAR_TYPE:
-      return stringValue((HiveChar)value);
-    case VARCHAR_TYPE:
-      return stringValue((HiveVarchar)value);
-    case DATE_TYPE:
-      return dateValue((Date)value);
-    case TIMESTAMP_TYPE:
-      return timestampValue((Timestamp)value);
-    case INTERVAL_YEAR_MONTH_TYPE:
-      return stringValue((HiveIntervalYearMonth) value);
-    case INTERVAL_DAY_TIME_TYPE:
-      return stringValue((HiveIntervalDayTime) value);
-    case DECIMAL_TYPE:
-      String plainStr = value == null ? null : ((BigDecimal)value).toPlainString();
-      return stringValue(plainStr);
-    case BINARY_TYPE:
-      String strVal = value == null ? null : UTF8String.fromBytes((byte[])value).toString();
-      return stringValue(strVal);
-    case ARRAY_TYPE:
-    case MAP_TYPE:
-    case STRUCT_TYPE:
-    case UNION_TYPE:
-    case USER_DEFINED_TYPE:
-      return stringValue((String)value);
-    case NULL_TYPE:
-      return stringValue((String)value);
-    default:
-      return null;
-    }
-  }
-
-  private static Boolean getBooleanValue(TBoolValue tBoolValue) {
-    if (tBoolValue.isSetValue()) {
-      return tBoolValue.isValue();
-    }
-    return null;
-  }
-
-  private static Byte getByteValue(TByteValue tByteValue) {
-    if (tByteValue.isSetValue()) {
-      return tByteValue.getValue();
-    }
-    return null;
-  }
-
-  private static Short getShortValue(TI16Value tI16Value) {
-    if (tI16Value.isSetValue()) {
-      return tI16Value.getValue();
-    }
-    return null;
-  }
-
-  private static Integer getIntegerValue(TI32Value tI32Value) {
-    if (tI32Value.isSetValue()) {
-      return tI32Value.getValue();
-    }
-    return null;
-  }
-
-  private static Long getLongValue(TI64Value tI64Value) {
-    if (tI64Value.isSetValue()) {
-      return tI64Value.getValue();
-    }
-    return null;
-  }
-
-  private static Double getDoubleValue(TDoubleValue tDoubleValue) {
-    if (tDoubleValue.isSetValue()) {
-      return tDoubleValue.getValue();
-    }
-    return null;
-  }
-
-  private static String getStringValue(TStringValue tStringValue) {
-    if (tStringValue.isSetValue()) {
-      return tStringValue.getValue();
-    }
-    return null;
-  }
-
-  private static Date getDateValue(TStringValue tStringValue) {
-    if (tStringValue.isSetValue()) {
-      return Date.valueOf(tStringValue.getValue());
-    }
-    return null;
-  }
-
-  private static Timestamp getTimestampValue(TStringValue tStringValue) {
-    if (tStringValue.isSetValue()) {
-      return Timestamp.valueOf(tStringValue.getValue());
-    }
-    return null;
-  }
-
-  private static byte[] getBinaryValue(TStringValue tString) {
-    if (tString.isSetValue()) {
-      return tString.getValue().getBytes();
-    }
-    return null;
-  }
-
-  private static BigDecimal getBigDecimalValue(TStringValue tStringValue) {
-    if (tStringValue.isSetValue()) {
-      return new BigDecimal(tStringValue.getValue());
-    }
-    return null;
-  }
-
-  public static Object toColumnValue(TColumnValue value) {
-    TColumnValue._Fields field = value.getSetField();
-    switch (field) {
-      case BOOL_VAL:
-        return getBooleanValue(value.getBoolVal());
-      case BYTE_VAL:
-        return getByteValue(value.getByteVal());
-      case I16_VAL:
-        return getShortValue(value.getI16Val());
-      case I32_VAL:
-        return getIntegerValue(value.getI32Val());
-      case I64_VAL:
-        return getLongValue(value.getI64Val());
-      case DOUBLE_VAL:
-        return getDoubleValue(value.getDoubleVal());
-      case STRING_VAL:
-        return getStringValue(value.getStringVal());
-    }
-    throw new IllegalArgumentException("never");
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/EmbeddedCLIServiceClient.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/EmbeddedCLIServiceClient.java
deleted file mode 100644
index 9cad5be198c06..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/EmbeddedCLIServiceClient.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hive.service.auth.HiveAuthFactory;
-
-
-/**
- * EmbeddedCLIServiceClient.
- *
- */
-public class EmbeddedCLIServiceClient extends CLIServiceClient {
-  private final ICLIService cliService;
-
-  public EmbeddedCLIServiceClient(ICLIService cliService) {
-    this.cliService = cliService;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.CLIServiceClient#openSession(java.lang.String, java.lang.String, java.util.Map)
-   */
-  @Override
-  public SessionHandle openSession(String username, String password,
-      Map<String, String> configuration) throws HiveSQLException {
-    return cliService.openSession(username, password, configuration);
-  }
-
-  @Override
-  public SessionHandle openSessionWithImpersonation(String username, String password,
-      Map<String, String> configuration, String delegationToken) throws HiveSQLException {
-    throw new HiveSQLException("Impersonated session is not supported in the embedded mode");
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.CLIServiceClient#closeSession(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public void closeSession(SessionHandle sessionHandle) throws HiveSQLException {
-    cliService.closeSession(sessionHandle);
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.CLIServiceClient#getInfo(org.apache.hive.service.cli.SessionHandle, java.util.List)
-   */
-  @Override
-  public GetInfoValue getInfo(SessionHandle sessionHandle, GetInfoType getInfoType)
-      throws HiveSQLException {
-    return cliService.getInfo(sessionHandle, getInfoType);
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.CLIServiceClient#executeStatement(org.apache.hive.service.cli.SessionHandle,
-   *  java.lang.String, java.util.Map)
-   */
-  @Override
-  public OperationHandle executeStatement(SessionHandle sessionHandle, String statement,
-      Map<String, String> confOverlay) throws HiveSQLException {
-    return cliService.executeStatement(sessionHandle, statement, confOverlay);
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.CLIServiceClient#executeStatementAsync(org.apache.hive.service.cli.SessionHandle,
-   *  java.lang.String, java.util.Map)
-   */
-  @Override
-  public OperationHandle executeStatementAsync(SessionHandle sessionHandle, String statement,
-      Map<String, String> confOverlay) throws HiveSQLException {
-    return cliService.executeStatementAsync(sessionHandle, statement, confOverlay);
-  }
-
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.CLIServiceClient#getTypeInfo(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public OperationHandle getTypeInfo(SessionHandle sessionHandle) throws HiveSQLException {
-    return cliService.getTypeInfo(sessionHandle);
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.CLIServiceClient#getCatalogs(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public OperationHandle getCatalogs(SessionHandle sessionHandle) throws HiveSQLException {
-    return cliService.getCatalogs(sessionHandle);
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.CLIServiceClient#getSchemas(org.apache.hive.service.cli.SessionHandle, java.lang.String, java.lang.String)
-   */
-  @Override
-  public OperationHandle getSchemas(SessionHandle sessionHandle, String catalogName,
-      String schemaName) throws HiveSQLException {
-    return cliService.getSchemas(sessionHandle, catalogName, schemaName);
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.CLIServiceClient#getTables(org.apache.hive.service.cli.SessionHandle, java.lang.String, java.lang.String, java.lang.String, java.util.List)
-   */
-  @Override
-  public OperationHandle getTables(SessionHandle sessionHandle, String catalogName,
-      String schemaName, String tableName, List<String> tableTypes) throws HiveSQLException {
-    return cliService.getTables(sessionHandle, catalogName, schemaName, tableName, tableTypes);
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.CLIServiceClient#getTableTypes(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public OperationHandle getTableTypes(SessionHandle sessionHandle) throws HiveSQLException {
-    return cliService.getTableTypes(sessionHandle);
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.CLIServiceClient#getColumns(org.apache.hive.service.cli.SessionHandle, java.lang.String, java.lang.String, java.lang.String, java.lang.String)
-   */
-  @Override
-  public OperationHandle getColumns(SessionHandle sessionHandle, String catalogName,
-      String schemaName, String tableName, String columnName) throws HiveSQLException {
-    return cliService.getColumns(sessionHandle, catalogName, schemaName, tableName, columnName);
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.CLIServiceClient#getFunctions(org.apache.hive.service.cli.SessionHandle, java.lang.String)
-   */
-  @Override
-  public OperationHandle getFunctions(SessionHandle sessionHandle,
-      String catalogName, String schemaName, String functionName)
-          throws HiveSQLException {
-    return cliService.getFunctions(sessionHandle, catalogName, schemaName, functionName);
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.CLIServiceClient#getOperationStatus(org.apache.hive.service.cli.OperationHandle)
-   */
-  @Override
-  public OperationStatus getOperationStatus(OperationHandle opHandle) throws HiveSQLException {
-    return cliService.getOperationStatus(opHandle);
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.CLIServiceClient#cancelOperation(org.apache.hive.service.cli.OperationHandle)
-   */
-  @Override
-  public void cancelOperation(OperationHandle opHandle) throws HiveSQLException {
-    cliService.cancelOperation(opHandle);
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.CLIServiceClient#closeOperation(org.apache.hive.service.cli.OperationHandle)
-   */
-  @Override
-  public void closeOperation(OperationHandle opHandle) throws HiveSQLException {
-    cliService.closeOperation(opHandle);
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.CLIServiceClient#getResultSetMetadata(org.apache.hive.service.cli.OperationHandle)
-   */
-  @Override
-  public TableSchema getResultSetMetadata(OperationHandle opHandle) throws HiveSQLException {
-    return cliService.getResultSetMetadata(opHandle);
-  }
-
-  @Override
-  public RowSet fetchResults(OperationHandle opHandle, FetchOrientation orientation,
-      long maxRows,  FetchType fetchType) throws HiveSQLException {
-    return cliService.fetchResults(opHandle, orientation, maxRows, fetchType);
-  }
-
-
-  @Override
-  public String getDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
-         String owner, String renewer) throws HiveSQLException {
-    return cliService.getDelegationToken(sessionHandle, authFactory, owner, renewer);
-  }
-
-  @Override
-  public void cancelDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
-      String tokenStr) throws HiveSQLException {
-    cliService.cancelDelegationToken(sessionHandle, authFactory, tokenStr);
-  }
-
-  @Override
-  public void renewDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
-      String tokenStr) throws HiveSQLException {
-    cliService.renewDelegationToken(sessionHandle, authFactory, tokenStr);
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/FetchOrientation.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/FetchOrientation.java
deleted file mode 100644
index ffa6f2e1f3743..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/FetchOrientation.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import org.apache.hive.service.cli.thrift.TFetchOrientation;
-
-/**
- * FetchOrientation.
- *
- */
-public enum FetchOrientation {
-  FETCH_NEXT(TFetchOrientation.FETCH_NEXT),
-  FETCH_PRIOR(TFetchOrientation.FETCH_PRIOR),
-  FETCH_RELATIVE(TFetchOrientation.FETCH_RELATIVE),
-  FETCH_ABSOLUTE(TFetchOrientation.FETCH_ABSOLUTE),
-  FETCH_FIRST(TFetchOrientation.FETCH_FIRST),
-  FETCH_LAST(TFetchOrientation.FETCH_LAST);
-
-  private TFetchOrientation tFetchOrientation;
-
-  FetchOrientation(TFetchOrientation tFetchOrientation) {
-    this.tFetchOrientation = tFetchOrientation;
-  }
-
-  public static FetchOrientation getFetchOrientation(TFetchOrientation tFetchOrientation) {
-    for (FetchOrientation fetchOrientation : values()) {
-      if (tFetchOrientation.equals(fetchOrientation.toTFetchOrientation())) {
-        return fetchOrientation;
-      }
-    }
-    // TODO: Should this really default to FETCH_NEXT?
-    return FETCH_NEXT;
-  }
-
-  public TFetchOrientation toTFetchOrientation() {
-    return tFetchOrientation;
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/GetInfoType.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/GetInfoType.java
deleted file mode 100644
index 8dd33a88fdeb2..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/GetInfoType.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import org.apache.hive.service.cli.thrift.TGetInfoType;
-
-/**
- * GetInfoType.
- *
- */
-public enum GetInfoType {
-  CLI_MAX_DRIVER_CONNECTIONS(TGetInfoType.CLI_MAX_DRIVER_CONNECTIONS),
-  CLI_MAX_CONCURRENT_ACTIVITIES(TGetInfoType.CLI_MAX_CONCURRENT_ACTIVITIES),
-  CLI_DATA_SOURCE_NAME(TGetInfoType.CLI_DATA_SOURCE_NAME),
-  CLI_FETCH_DIRECTION(TGetInfoType.CLI_FETCH_DIRECTION),
-  CLI_SERVER_NAME(TGetInfoType.CLI_SERVER_NAME),
-  CLI_SEARCH_PATTERN_ESCAPE(TGetInfoType.CLI_SEARCH_PATTERN_ESCAPE),
-  CLI_DBMS_NAME(TGetInfoType.CLI_DBMS_NAME),
-  CLI_DBMS_VER(TGetInfoType.CLI_DBMS_VER),
-  CLI_ACCESSIBLE_TABLES(TGetInfoType.CLI_ACCESSIBLE_TABLES),
-  CLI_ACCESSIBLE_PROCEDURES(TGetInfoType.CLI_ACCESSIBLE_PROCEDURES),
-  CLI_CURSOR_COMMIT_BEHAVIOR(TGetInfoType.CLI_CURSOR_COMMIT_BEHAVIOR),
-  CLI_DATA_SOURCE_READ_ONLY(TGetInfoType.CLI_DATA_SOURCE_READ_ONLY),
-  CLI_DEFAULT_TXN_ISOLATION(TGetInfoType.CLI_DEFAULT_TXN_ISOLATION),
-  CLI_IDENTIFIER_CASE(TGetInfoType.CLI_IDENTIFIER_CASE),
-  CLI_IDENTIFIER_QUOTE_CHAR(TGetInfoType.CLI_IDENTIFIER_QUOTE_CHAR),
-  CLI_MAX_COLUMN_NAME_LEN(TGetInfoType.CLI_MAX_COLUMN_NAME_LEN),
-  CLI_MAX_CURSOR_NAME_LEN(TGetInfoType.CLI_MAX_CURSOR_NAME_LEN),
-  CLI_MAX_SCHEMA_NAME_LEN(TGetInfoType.CLI_MAX_SCHEMA_NAME_LEN),
-  CLI_MAX_CATALOG_NAME_LEN(TGetInfoType.CLI_MAX_CATALOG_NAME_LEN),
-  CLI_MAX_TABLE_NAME_LEN(TGetInfoType.CLI_MAX_TABLE_NAME_LEN),
-  CLI_SCROLL_CONCURRENCY(TGetInfoType.CLI_SCROLL_CONCURRENCY),
-  CLI_TXN_CAPABLE(TGetInfoType.CLI_TXN_CAPABLE),
-  CLI_USER_NAME(TGetInfoType.CLI_USER_NAME),
-  CLI_TXN_ISOLATION_OPTION(TGetInfoType.CLI_TXN_ISOLATION_OPTION),
-  CLI_INTEGRITY(TGetInfoType.CLI_INTEGRITY),
-  CLI_GETDATA_EXTENSIONS(TGetInfoType.CLI_GETDATA_EXTENSIONS),
-  CLI_NULL_COLLATION(TGetInfoType.CLI_NULL_COLLATION),
-  CLI_ALTER_TABLE(TGetInfoType.CLI_ALTER_TABLE),
-  CLI_ORDER_BY_COLUMNS_IN_SELECT(TGetInfoType.CLI_ORDER_BY_COLUMNS_IN_SELECT),
-  CLI_SPECIAL_CHARACTERS(TGetInfoType.CLI_SPECIAL_CHARACTERS),
-  CLI_MAX_COLUMNS_IN_GROUP_BY(TGetInfoType.CLI_MAX_COLUMNS_IN_GROUP_BY),
-  CLI_MAX_COLUMNS_IN_INDEX(TGetInfoType.CLI_MAX_COLUMNS_IN_INDEX),
-  CLI_MAX_COLUMNS_IN_ORDER_BY(TGetInfoType.CLI_MAX_COLUMNS_IN_ORDER_BY),
-  CLI_MAX_COLUMNS_IN_SELECT(TGetInfoType.CLI_MAX_COLUMNS_IN_SELECT),
-  CLI_MAX_COLUMNS_IN_TABLE(TGetInfoType.CLI_MAX_COLUMNS_IN_TABLE),
-  CLI_MAX_INDEX_SIZE(TGetInfoType.CLI_MAX_INDEX_SIZE),
-  CLI_MAX_ROW_SIZE(TGetInfoType.CLI_MAX_ROW_SIZE),
-  CLI_MAX_STATEMENT_LEN(TGetInfoType.CLI_MAX_STATEMENT_LEN),
-  CLI_MAX_TABLES_IN_SELECT(TGetInfoType.CLI_MAX_TABLES_IN_SELECT),
-  CLI_MAX_USER_NAME_LEN(TGetInfoType.CLI_MAX_USER_NAME_LEN),
-  CLI_OJ_CAPABILITIES(TGetInfoType.CLI_OJ_CAPABILITIES),
-
-  CLI_XOPEN_CLI_YEAR(TGetInfoType.CLI_XOPEN_CLI_YEAR),
-  CLI_CURSOR_SENSITIVITY(TGetInfoType.CLI_CURSOR_SENSITIVITY),
-  CLI_DESCRIBE_PARAMETER(TGetInfoType.CLI_DESCRIBE_PARAMETER),
-  CLI_CATALOG_NAME(TGetInfoType.CLI_CATALOG_NAME),
-  CLI_COLLATION_SEQ(TGetInfoType.CLI_COLLATION_SEQ),
-  CLI_MAX_IDENTIFIER_LEN(TGetInfoType.CLI_MAX_IDENTIFIER_LEN);
-
-  private final TGetInfoType tInfoType;
-
-  GetInfoType(TGetInfoType tInfoType) {
-    this.tInfoType = tInfoType;
-  }
-
-  public static GetInfoType getGetInfoType(TGetInfoType tGetInfoType) {
-    for (GetInfoType infoType : values()) {
-      if (tGetInfoType.equals(infoType.tInfoType)) {
-        return infoType;
-      }
-    }
-    throw new IllegalArgumentException("Unrecognized Thrift TGetInfoType value: " + tGetInfoType);
-  }
-
-  public TGetInfoType toTGetInfoType() {
-    return tInfoType;
-  }
-
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/GetInfoValue.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/GetInfoValue.java
deleted file mode 100644
index ba92ff4ab5c11..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/GetInfoValue.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import org.apache.hive.service.cli.thrift.TGetInfoValue;
-
-/**
- * GetInfoValue.
- *
- */
-public class GetInfoValue {
-  private String stringValue = null;
-  private short shortValue;
-  private int intValue;
-  private long longValue;
-
-  public GetInfoValue(String stringValue) {
-    this.stringValue = stringValue;
-  }
-
-  public GetInfoValue(short shortValue) {
-    this.shortValue = shortValue;
-  }
-
-  public GetInfoValue(int intValue) {
-    this.intValue = intValue;
-  }
-
-  public GetInfoValue(long longValue) {
-    this.longValue = longValue;
-  }
-
-  public GetInfoValue(TGetInfoValue tGetInfoValue) {
-    switch (tGetInfoValue.getSetField()) {
-    case STRING_VALUE:
-      stringValue = tGetInfoValue.getStringValue();
-      break;
-    default:
-      throw new IllegalArgumentException("Unreconigzed TGetInfoValue");
-    }
-  }
-
-  public TGetInfoValue toTGetInfoValue() {
-    TGetInfoValue tInfoValue = new TGetInfoValue();
-    if (stringValue != null) {
-      tInfoValue.setStringValue(stringValue);
-    }
-    return tInfoValue;
-  }
-
-  public String getStringValue() {
-    return stringValue;
-  }
-
-  public short getShortValue() {
-    return shortValue;
-  }
-
-  public int getIntValue() {
-    return intValue;
-  }
-
-  public long getLongValue() {
-    return longValue;
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/Handle.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/Handle.java
deleted file mode 100644
index cf3427ae20f3c..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/Handle.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.service.cli;
-
-import org.apache.hive.service.cli.thrift.THandleIdentifier;
-
-
-
-
-public abstract class Handle {
-
-  private final HandleIdentifier handleId;
-
-  public Handle() {
-    handleId = new HandleIdentifier();
-  }
-
-  public Handle(HandleIdentifier handleId) {
-    this.handleId = handleId;
-  }
-
-  public Handle(THandleIdentifier tHandleIdentifier) {
-    this.handleId = new HandleIdentifier(tHandleIdentifier);
-  }
-
-  public HandleIdentifier getHandleIdentifier() {
-    return handleId;
-  }
-
-  @Override
-  public int hashCode() {
-    final int prime = 31;
-    int result = 1;
-    result = prime * result + ((handleId == null) ? 0 : handleId.hashCode());
-    return result;
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    if (this == obj) {
-      return true;
-    }
-    if (obj == null) {
-      return false;
-    }
-    if (!(obj instanceof Handle)) {
-      return false;
-    }
-    Handle other = (Handle) obj;
-    if (handleId == null) {
-      if (other.handleId != null) {
-        return false;
-      }
-    } else if (!handleId.equals(other.handleId)) {
-      return false;
-    }
-    return true;
-  }
-
-  @Override
-  public abstract String toString();
-
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/HandleIdentifier.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/HandleIdentifier.java
deleted file mode 100644
index 4dc80da8dc500..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/HandleIdentifier.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import java.nio.ByteBuffer;
-import java.util.UUID;
-
-import org.apache.hive.service.cli.thrift.THandleIdentifier;
-
-/**
- * HandleIdentifier.
- *
- */
-public class HandleIdentifier {
-  private final UUID publicId;
-  private final UUID secretId;
-
-  public HandleIdentifier() {
-    publicId = UUID.randomUUID();
-    secretId = UUID.randomUUID();
-  }
-
-  public HandleIdentifier(UUID publicId, UUID secretId) {
-    this.publicId = publicId;
-    this.secretId = secretId;
-  }
-
-  public HandleIdentifier(THandleIdentifier tHandleId) {
-    ByteBuffer bb = ByteBuffer.wrap(tHandleId.getGuid());
-    this.publicId = new UUID(bb.getLong(), bb.getLong());
-    bb = ByteBuffer.wrap(tHandleId.getSecret());
-    this.secretId = new UUID(bb.getLong(), bb.getLong());
-  }
-
-  public UUID getPublicId() {
-    return publicId;
-  }
-
-  public UUID getSecretId() {
-    return secretId;
-  }
-
-  public THandleIdentifier toTHandleIdentifier() {
-    byte[] guid = new byte[16];
-    byte[] secret = new byte[16];
-    ByteBuffer guidBB = ByteBuffer.wrap(guid);
-    ByteBuffer secretBB = ByteBuffer.wrap(secret);
-    guidBB.putLong(publicId.getMostSignificantBits());
-    guidBB.putLong(publicId.getLeastSignificantBits());
-    secretBB.putLong(secretId.getMostSignificantBits());
-    secretBB.putLong(secretId.getLeastSignificantBits());
-    return new THandleIdentifier(ByteBuffer.wrap(guid), ByteBuffer.wrap(secret));
-  }
-
-  @Override
-  public int hashCode() {
-    final int prime = 31;
-    int result = 1;
-    result = prime * result + ((publicId == null) ? 0 : publicId.hashCode());
-    result = prime * result + ((secretId == null) ? 0 : secretId.hashCode());
-    return result;
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    if (this == obj) {
-      return true;
-    }
-    if (obj == null) {
-      return false;
-    }
-    if (!(obj instanceof HandleIdentifier)) {
-      return false;
-    }
-    HandleIdentifier other = (HandleIdentifier) obj;
-    if (publicId == null) {
-      if (other.publicId != null) {
-        return false;
-      }
-    } else if (!publicId.equals(other.publicId)) {
-      return false;
-    }
-    if (secretId == null) {
-      if (other.secretId != null) {
-        return false;
-      }
-    } else if (!secretId.equals(other.secretId)) {
-      return false;
-    }
-    return true;
-  }
-
-  @Override
-  public String toString() {
-    return publicId.toString();
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/HiveSQLException.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/HiveSQLException.java
deleted file mode 100644
index 86e57fbf31fe0..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/HiveSQLException.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hive.service.cli.thrift.TStatus;
-import org.apache.hive.service.cli.thrift.TStatusCode;
-
-/**
- * HiveSQLException.
- *
- */
-public class HiveSQLException extends SQLException {
-
-  /**
-   *
-   */
-  private static final long serialVersionUID = -6095254671958748094L;
-
-  /**
-   *
-   */
-  public HiveSQLException() {
-    super();
-  }
-
-  /**
-   * @param reason
-   */
-  public HiveSQLException(String reason) {
-    super(reason);
-  }
-
-  /**
-   * @param cause
-   */
-  public HiveSQLException(Throwable cause) {
-    super(cause);
-  }
-
-  /**
-   * @param reason
-   * @param sqlState
-   */
-  public HiveSQLException(String reason, String sqlState) {
-    super(reason, sqlState);
-  }
-
-  /**
-   * @param reason
-   * @param cause
-   */
-  public HiveSQLException(String reason, Throwable cause) {
-    super(reason, cause);
-  }
-
-  /**
-   * @param reason
-   * @param sqlState
-   * @param vendorCode
-   */
-  public HiveSQLException(String reason, String sqlState, int vendorCode) {
-    super(reason, sqlState, vendorCode);
-  }
-
-  /**
-   * @param reason
-   * @param sqlState
-   * @param cause
-   */
-  public HiveSQLException(String reason, String sqlState, Throwable cause) {
-    super(reason, sqlState, cause);
-  }
-
-  /**
-   * @param reason
-   * @param sqlState
-   * @param vendorCode
-   * @param cause
-   */
-  public HiveSQLException(String reason, String sqlState, int vendorCode, Throwable cause) {
-    super(reason, sqlState, vendorCode, cause);
-  }
-
-  public HiveSQLException(TStatus status) {
-    // TODO: set correct vendorCode field
-    super(status.getErrorMessage(), status.getSqlState(), status.getErrorCode());
-    if (status.getInfoMessages() != null) {
-      initCause(toCause(status.getInfoMessages()));
-    }
-  }
-
-  /**
-   * Converts current object to a {@link TStatus} object
-   * @return a {@link TStatus} object
-   */
-  public TStatus toTStatus() {
-    // TODO: convert sqlState, etc.
-    TStatus tStatus = new TStatus(TStatusCode.ERROR_STATUS);
-    tStatus.setSqlState(getSQLState());
-    tStatus.setErrorCode(getErrorCode());
-    tStatus.setErrorMessage(getMessage());
-    tStatus.setInfoMessages(toString(this));
-    return tStatus;
-  }
-
-  /**
-   * Converts the specified {@link Exception} object into a {@link TStatus} object
-   * @param e a {@link Exception} object
-   * @return a {@link TStatus} object
-   */
-  public static TStatus toTStatus(Exception e) {
-    if (e instanceof HiveSQLException) {
-      return ((HiveSQLException)e).toTStatus();
-    }
-    TStatus tStatus = new TStatus(TStatusCode.ERROR_STATUS);
-    tStatus.setErrorMessage(e.getMessage());
-    tStatus.setInfoMessages(toString(e));
-    return tStatus;
-  }
-
-  /**
-   * Converts a {@link Throwable} object into a flattened list of texts including its stack trace
-   * and the stack traces of the nested causes.
-   * @param ex  a {@link Throwable} object
-   * @return    a flattened list of texts including the {@link Throwable} object's stack trace
-   *            and the stack traces of the nested causes.
-   */
-  public static List<String> toString(Throwable ex) {
-    return toString(ex, null);
-  }
-
-  private static List<String> toString(Throwable cause, StackTraceElement[] parent) {
-    StackTraceElement[] trace = cause.getStackTrace();
-    int m = trace.length - 1;
-    if (parent != null) {
-      int n = parent.length - 1;
-      while (m >= 0 && n >= 0 && trace[m].equals(parent[n])) {
-        m--;
-        n--;
-      }
-    }
-    List<String> detail = enroll(cause, trace, m);
-    cause = cause.getCause();
-    if (cause != null) {
-      detail.addAll(toString(cause, trace));
-    }
-    return detail;
-  }
-
-  private static List<String> enroll(Throwable ex, StackTraceElement[] trace, int max) {
-    List<String> details = new ArrayList<String>();
-    StringBuilder builder = new StringBuilder();
-    builder.append('*').append(ex.getClass().getName()).append(':');
-    builder.append(ex.getMessage()).append(':');
-    builder.append(trace.length).append(':').append(max);
-    details.add(builder.toString());
-    for (int i = 0; i <= max; i++) {
-      builder.setLength(0);
-      builder.append(trace[i].getClassName()).append(':');
-      builder.append(trace[i].getMethodName()).append(':');
-      String fileName = trace[i].getFileName();
-      builder.append(fileName == null ? "" : fileName).append(':');
-      builder.append(trace[i].getLineNumber());
-      details.add(builder.toString());
-    }
-    return details;
-  }
-
-  /**
-   * Converts a flattened list of texts including the stack trace and the stack
-   * traces of the nested causes into a {@link Throwable} object.
-   * @param details a flattened list of texts including the stack trace and the stack
-   *                traces of the nested causes
-   * @return        a {@link Throwable} object
-   */
-  public static Throwable toCause(List<String> details) {
-    return toStackTrace(details, null, 0);
-  }
-
-  private static Throwable toStackTrace(List<String> details, StackTraceElement[] parent, int index) {
-    String detail = details.get(index++);
-    if (!detail.startsWith("*")) {
-      return null;  // should not be happened. ignore remaining
-    }
-    int i1 = detail.indexOf(':');
-    int i3 = detail.lastIndexOf(':');
-    int i2 = detail.substring(0, i3).lastIndexOf(':');
-    String exceptionClass = detail.substring(1, i1);
-    String exceptionMessage = detail.substring(i1 + 1, i2);
-    Throwable ex = newInstance(exceptionClass, exceptionMessage);
-
-    Integer length = Integer.valueOf(detail.substring(i2 + 1, i3));
-    Integer unique = Integer.valueOf(detail.substring(i3 + 1));
-
-    int i = 0;
-    StackTraceElement[] trace = new StackTraceElement[length];
-    for (; i <= unique; i++) {
-      detail = details.get(index++);
-      int j1 = detail.indexOf(':');
-      int j3 = detail.lastIndexOf(':');
-      int j2 = detail.substring(0, j3).lastIndexOf(':');
-      String className = detail.substring(0, j1);
-      String methodName = detail.substring(j1 + 1, j2);
-      String fileName = detail.substring(j2 + 1, j3);
-      if (fileName.isEmpty()) {
-        fileName = null;
-      }
-      int lineNumber = Integer.valueOf(detail.substring(j3 + 1));
-      trace[i] = new StackTraceElement(className, methodName, fileName, lineNumber);
-    }
-    int common = trace.length - i;
-    if (common > 0) {
-      System.arraycopy(parent, parent.length - common, trace, trace.length - common, common);
-    }
-    if (details.size() > index) {
-      ex.initCause(toStackTrace(details, trace, index));
-    }
-    ex.setStackTrace(trace);
-    return ex;
-  }
-
-  private static Throwable newInstance(String className, String message) {
-    try {
-      return (Throwable)Class.forName(className).getConstructor(String.class).newInstance(message);
-    } catch (Exception e) {
-      return new RuntimeException(className + ":" + message);
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ICLIService.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ICLIService.java
deleted file mode 100644
index c9cc1f4da56f1..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ICLIService.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.service.cli;
-
-import java.util.List;
-import java.util.Map;
-
-
-
-
-import org.apache.hive.service.auth.HiveAuthFactory;
-
-public interface ICLIService {
-
-  SessionHandle openSession(String username, String password,
-      Map<String, String> configuration)
-          throws HiveSQLException;
-
-  SessionHandle openSessionWithImpersonation(String username, String password,
-      Map<String, String> configuration, String delegationToken)
-          throws HiveSQLException;
-
-  void closeSession(SessionHandle sessionHandle)
-      throws HiveSQLException;
-
-  GetInfoValue getInfo(SessionHandle sessionHandle, GetInfoType infoType)
-      throws HiveSQLException;
-
-  OperationHandle executeStatement(SessionHandle sessionHandle, String statement,
-      Map<String, String> confOverlay)
-          throws HiveSQLException;
-
-  OperationHandle executeStatementAsync(SessionHandle sessionHandle,
-      String statement, Map<String, String> confOverlay)
-          throws HiveSQLException;
-
-  OperationHandle getTypeInfo(SessionHandle sessionHandle)
-      throws HiveSQLException;
-
-  OperationHandle getCatalogs(SessionHandle sessionHandle)
-      throws HiveSQLException;
-
-  OperationHandle getSchemas(SessionHandle sessionHandle,
-      String catalogName, String schemaName)
-          throws HiveSQLException;
-
-  OperationHandle getTables(SessionHandle sessionHandle,
-      String catalogName, String schemaName, String tableName, List<String> tableTypes)
-          throws HiveSQLException;
-
-  OperationHandle getTableTypes(SessionHandle sessionHandle)
-      throws HiveSQLException;
-
-  OperationHandle getColumns(SessionHandle sessionHandle,
-      String catalogName, String schemaName, String tableName, String columnName)
-          throws HiveSQLException;
-
-  OperationHandle getFunctions(SessionHandle sessionHandle,
-      String catalogName, String schemaName, String functionName)
-          throws HiveSQLException;
-
-  OperationStatus getOperationStatus(OperationHandle opHandle)
-      throws HiveSQLException;
-
-  void cancelOperation(OperationHandle opHandle)
-      throws HiveSQLException;
-
-  void closeOperation(OperationHandle opHandle)
-      throws HiveSQLException;
-
-  TableSchema getResultSetMetadata(OperationHandle opHandle)
-      throws HiveSQLException;
-
-  RowSet fetchResults(OperationHandle opHandle)
-      throws HiveSQLException;
-
-  RowSet fetchResults(OperationHandle opHandle, FetchOrientation orientation,
-      long maxRows, FetchType fetchType) throws HiveSQLException;
-
-  String getDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
-      String owner, String renewer) throws HiveSQLException;
-
-  void cancelDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
-      String tokenStr) throws HiveSQLException;
-
-  void renewDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
-      String tokenStr) throws HiveSQLException;
-
-
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/OperationHandle.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/OperationHandle.java
deleted file mode 100644
index 5426e28471239..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/OperationHandle.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.service.cli;
-
-import org.apache.hive.service.cli.thrift.TOperationHandle;
-import org.apache.hive.service.cli.thrift.TProtocolVersion;
-
-public class OperationHandle extends Handle {
-
-  private final OperationType opType;
-  private final TProtocolVersion protocol;
-  private boolean hasResultSet = false;
-
-  public OperationHandle(OperationType opType, TProtocolVersion protocol) {
-    super();
-    this.opType = opType;
-    this.protocol = protocol;
-  }
-
-  // dummy handle for ThriftCLIService
-  public OperationHandle(TOperationHandle tOperationHandle) {
-    this(tOperationHandle, TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V1);
-  }
-
-  public OperationHandle(TOperationHandle tOperationHandle, TProtocolVersion protocol) {
-    super(tOperationHandle.getOperationId());
-    this.opType = OperationType.getOperationType(tOperationHandle.getOperationType());
-    this.hasResultSet = tOperationHandle.isHasResultSet();
-    this.protocol = protocol;
-  }
-
-  public OperationType getOperationType() {
-    return opType;
-  }
-
-  public void setHasResultSet(boolean hasResultSet) {
-    this.hasResultSet = hasResultSet;
-  }
-
-  public boolean hasResultSet() {
-    return hasResultSet;
-  }
-
-  public TOperationHandle toTOperationHandle() {
-    TOperationHandle tOperationHandle = new TOperationHandle();
-    tOperationHandle.setOperationId(getHandleIdentifier().toTHandleIdentifier());
-    tOperationHandle.setOperationType(opType.toTOperationType());
-    tOperationHandle.setHasResultSet(hasResultSet);
-    return tOperationHandle;
-  }
-
-  public TProtocolVersion getProtocolVersion() {
-    return protocol;
-  }
-
-  @Override
-  public int hashCode() {
-    final int prime = 31;
-    int result = super.hashCode();
-    result = prime * result + ((opType == null) ? 0 : opType.hashCode());
-    return result;
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    if (this == obj) {
-      return true;
-    }
-    if (!super.equals(obj)) {
-      return false;
-    }
-    if (!(obj instanceof OperationHandle)) {
-      return false;
-    }
-    OperationHandle other = (OperationHandle) obj;
-    if (opType != other.opType) {
-      return false;
-    }
-    return true;
-  }
-
-  @Override
-  public String toString() {
-    return "OperationHandle [opType=" + opType + ", getHandleIdentifier()=" + getHandleIdentifier()
-        + "]";
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/OperationState.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/OperationState.java
deleted file mode 100644
index 1165180118413..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/OperationState.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import org.apache.hive.service.cli.thrift.TOperationState;
-
-/**
- * OperationState.
- *
- */
-public enum OperationState {
-  INITIALIZED(TOperationState.INITIALIZED_STATE, false),
-  RUNNING(TOperationState.RUNNING_STATE, false),
-  FINISHED(TOperationState.FINISHED_STATE, true),
-  CANCELED(TOperationState.CANCELED_STATE, true),
-  CLOSED(TOperationState.CLOSED_STATE, true),
-  ERROR(TOperationState.ERROR_STATE, true),
-  UNKNOWN(TOperationState.UKNOWN_STATE, false),
-  PENDING(TOperationState.PENDING_STATE, false);
-
-  private final TOperationState tOperationState;
-  private final boolean terminal;
-
-  OperationState(TOperationState tOperationState, boolean terminal) {
-    this.tOperationState = tOperationState;
-    this.terminal = terminal;
-  }
-
-  // must be sync with TOperationState in order
-  public static OperationState getOperationState(TOperationState tOperationState) {
-    return OperationState.values()[tOperationState.getValue()];
-  }
-
-  public static void validateTransition(OperationState oldState,
-      OperationState newState)
-          throws HiveSQLException {
-    switch (oldState) {
-    case INITIALIZED:
-      switch (newState) {
-      case PENDING:
-      case RUNNING:
-      case CANCELED:
-      case CLOSED:
-        return;
-      }
-      break;
-    case PENDING:
-      switch (newState) {
-      case RUNNING:
-      case FINISHED:
-      case CANCELED:
-      case ERROR:
-      case CLOSED:
-        return;
-      }
-      break;
-    case RUNNING:
-      switch (newState) {
-      case FINISHED:
-      case CANCELED:
-      case ERROR:
-      case CLOSED:
-        return;
-      }
-      break;
-    case FINISHED:
-    case CANCELED:
-    case ERROR:
-      if (OperationState.CLOSED.equals(newState)) {
-        return;
-      }
-      break;
-    default:
-      // fall-through
-    }
-    throw new HiveSQLException("Illegal Operation state transition " +
-        "from " + oldState + " to " + newState);
-  }
-
-  public void validateTransition(OperationState newState)
-      throws HiveSQLException {
-    validateTransition(this, newState);
-  }
-
-  public TOperationState toTOperationState() {
-    return tOperationState;
-  }
-
-  public boolean isTerminal() {
-    return terminal;
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/OperationType.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/OperationType.java
deleted file mode 100644
index 429d9a4c25688..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/OperationType.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import org.apache.hive.service.cli.thrift.TOperationType;
-
-/**
- * OperationType.
- *
- */
-public enum OperationType {
-
-  UNKNOWN_OPERATION(TOperationType.UNKNOWN),
-  EXECUTE_STATEMENT(TOperationType.EXECUTE_STATEMENT),
-  GET_TYPE_INFO(TOperationType.GET_TYPE_INFO),
-  GET_CATALOGS(TOperationType.GET_CATALOGS),
-  GET_SCHEMAS(TOperationType.GET_SCHEMAS),
-  GET_TABLES(TOperationType.GET_TABLES),
-  GET_TABLE_TYPES(TOperationType.GET_TABLE_TYPES),
-  GET_COLUMNS(TOperationType.GET_COLUMNS),
-  GET_FUNCTIONS(TOperationType.GET_FUNCTIONS);
-
-  private TOperationType tOperationType;
-
-  OperationType(TOperationType tOpType) {
-    this.tOperationType = tOpType;
-  }
-
-  public static OperationType getOperationType(TOperationType tOperationType) {
-    // TODO: replace this with a Map?
-    for (OperationType opType : values()) {
-      if (tOperationType.equals(opType.tOperationType)) {
-        return opType;
-      }
-    }
-    return OperationType.UNKNOWN_OPERATION;
-  }
-
-  public TOperationType toTOperationType() {
-    return tOperationType;
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/PatternOrIdentifier.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/PatternOrIdentifier.java
deleted file mode 100644
index 6e4d43fd5df63..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/PatternOrIdentifier.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-/**
- * PatternOrIdentifier.
- *
- */
-public class PatternOrIdentifier {
-
-  boolean isPattern = false;
-  String text;
-
-  public PatternOrIdentifier(String tpoi) {
-    text = tpoi;
-    isPattern = false;
-  }
-
-  public boolean isPattern() {
-    return isPattern;
-  }
-
-  public boolean isIdentifier() {
-    return !isPattern;
-  }
-
-  @Override
-  public String toString() {
-    return text;
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/RowBasedSet.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/RowBasedSet.java
deleted file mode 100644
index 7452137f077db..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/RowBasedSet.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.hive.service.cli.thrift.TColumnValue;
-import org.apache.hive.service.cli.thrift.TRow;
-import org.apache.hive.service.cli.thrift.TRowSet;
-
-/**
- * RowBasedSet
- */
-public class RowBasedSet implements RowSet {
-
-  private long startOffset;
-
-  private final Type[] types; // non-null only for writing (server-side)
-  private final RemovableList<TRow> rows;
-
-  public RowBasedSet(TableSchema schema) {
-    types = schema.toTypes();
-    rows = new RemovableList<TRow>();
-  }
-
-  public RowBasedSet(TRowSet tRowSet) {
-    types = null;
-    rows = new RemovableList<TRow>(tRowSet.getRows());
-    startOffset = tRowSet.getStartRowOffset();
-  }
-
-  private RowBasedSet(Type[] types, List<TRow> rows, long startOffset) {
-    this.types = types;
-    this.rows = new RemovableList<TRow>(rows);
-    this.startOffset = startOffset;
-  }
-
-  @Override
-  public RowBasedSet addRow(Object[] fields) {
-    TRow tRow = new TRow();
-    for (int i = 0; i < fields.length; i++) {
-      tRow.addToColVals(ColumnValue.toTColumnValue(types[i], fields[i]));
-    }
-    rows.add(tRow);
-    return this;
-  }
-
-  @Override
-  public int numColumns() {
-    return rows.isEmpty() ? 0 : rows.get(0).getColVals().size();
-  }
-
-  @Override
-  public int numRows() {
-    return rows.size();
-  }
-
-  public RowBasedSet extractSubset(int maxRows) {
-    int numRows = Math.min(numRows(), maxRows);
-    RowBasedSet result = new RowBasedSet(types, rows.subList(0, numRows), startOffset);
-    rows.removeRange(0, numRows);
-    startOffset += numRows;
-    return result;
-  }
-
-  public long getStartOffset() {
-    return startOffset;
-  }
-
-  public void setStartOffset(long startOffset) {
-    this.startOffset = startOffset;
-  }
-
-  public int getSize() {
-    return rows.size();
-  }
-
-  public TRowSet toTRowSet() {
-    TRowSet tRowSet = new TRowSet();
-    tRowSet.setStartRowOffset(startOffset);
-    tRowSet.setRows(new ArrayList<TRow>(rows));
-    return tRowSet;
-  }
-
-  @Override
-  public Iterator<Object[]> iterator() {
-    return new Iterator<Object[]>() {
-
-      final Iterator<TRow> iterator = rows.iterator();
-      final Object[] convey = new Object[numColumns()];
-
-      @Override
-      public boolean hasNext() {
-        return iterator.hasNext();
-      }
-
-      @Override
-      public Object[] next() {
-        TRow row = iterator.next();
-        List<TColumnValue> values = row.getColVals();
-        for (int i = 0; i < values.size(); i++) {
-          convey[i] = ColumnValue.toColumnValue(values.get(i));
-        }
-        return convey;
-      }
-
-      @Override
-      public void remove() {
-        throw new UnsupportedOperationException("remove");
-      }
-    };
-  }
-
-  private static class RemovableList<E> extends ArrayList<E> {
-    RemovableList() { super(); }
-    RemovableList(List<E> rows) { super(rows); }
-    @Override
-    public void removeRange(int fromIndex, int toIndex) {
-      super.removeRange(fromIndex, toIndex);
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/RowSet.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/RowSet.java
deleted file mode 100644
index ab0787e1d389e..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/RowSet.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import org.apache.hive.service.cli.thrift.TRowSet;
-
-public interface RowSet extends Iterable<Object[]> {
-
-  RowSet addRow(Object[] fields);
-
-  RowSet extractSubset(int maxRows);
-
-  int numColumns();
-
-  int numRows();
-
-  long getStartOffset();
-
-  void setStartOffset(long startOffset);
-
-  TRowSet toTRowSet();
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/RowSetFactory.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/RowSetFactory.java
deleted file mode 100644
index e8f68eaaf9063..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/RowSetFactory.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import org.apache.hive.service.cli.thrift.TProtocolVersion;
-import org.apache.hive.service.cli.thrift.TRowSet;
-
-import static org.apache.hive.service.cli.thrift.TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V6;
-
-public class RowSetFactory {
-
-  public static RowSet create(TableSchema schema, TProtocolVersion version) {
-    if (version.getValue() >= HIVE_CLI_SERVICE_PROTOCOL_V6.getValue()) {
-      return new ColumnBasedSet(schema);
-    }
-    return new RowBasedSet(schema);
-  }
-
-  public static RowSet create(TRowSet results, TProtocolVersion version) {
-    if (version.getValue() >= HIVE_CLI_SERVICE_PROTOCOL_V6.getValue()) {
-      return new ColumnBasedSet(results);
-    }
-    return new RowBasedSet(results);
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/SessionHandle.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/SessionHandle.java
deleted file mode 100644
index 52e0ad4834d8b..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/SessionHandle.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import java.util.UUID;
-
-import org.apache.hive.service.cli.thrift.TProtocolVersion;
-import org.apache.hive.service.cli.thrift.TSessionHandle;
-
-
-/**
- * SessionHandle.
- *
- */
-public class SessionHandle extends Handle {
-
-  private final TProtocolVersion protocol;
-
-  public SessionHandle(TProtocolVersion protocol) {
-    this.protocol = protocol;
-  }
-
-  // dummy handle for ThriftCLIService
-  public SessionHandle(TSessionHandle tSessionHandle) {
-    this(tSessionHandle, TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V1);
-  }
-
-  public SessionHandle(TSessionHandle tSessionHandle, TProtocolVersion protocol) {
-    super(tSessionHandle.getSessionId());
-    this.protocol = protocol;
-  }
-
-  public UUID getSessionId() {
-    return getHandleIdentifier().getPublicId();
-  }
-
-  public TSessionHandle toTSessionHandle() {
-    TSessionHandle tSessionHandle = new TSessionHandle();
-    tSessionHandle.setSessionId(getHandleIdentifier().toTHandleIdentifier());
-    return tSessionHandle;
-  }
-
-  public TProtocolVersion getProtocolVersion() {
-    return protocol;
-  }
-
-  @Override
-  public String toString() {
-    return "SessionHandle [" + getHandleIdentifier() + "]";
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/TableSchema.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/TableSchema.java
deleted file mode 100644
index ee019bc737101..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/TableSchema.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Schema;
-import org.apache.hive.service.cli.thrift.TColumnDesc;
-import org.apache.hive.service.cli.thrift.TTableSchema;
-
-/**
- * TableSchema.
- *
- */
-public class TableSchema {
-  private final List<ColumnDescriptor> columns = new ArrayList<ColumnDescriptor>();
-
-  public TableSchema() {
-  }
-
-  public TableSchema(int numColumns) {
-    // TODO: remove this constructor
-  }
-
-  public TableSchema(TTableSchema tTableSchema) {
-    for (TColumnDesc tColumnDesc : tTableSchema.getColumns()) {
-      columns.add(new ColumnDescriptor(tColumnDesc));
-    }
-  }
-
-  public TableSchema(List<FieldSchema> fieldSchemas) {
-    int pos = 1;
-    for (FieldSchema field : fieldSchemas) {
-      columns.add(new ColumnDescriptor(field, pos++));
-    }
-  }
-
-  public TableSchema(Schema schema) {
-    this(schema.getFieldSchemas());
-  }
-
-  public List<ColumnDescriptor> getColumnDescriptors() {
-    return new ArrayList<ColumnDescriptor>(columns);
-  }
-
-  public ColumnDescriptor getColumnDescriptorAt(int pos) {
-    return columns.get(pos);
-  }
-
-  public int getSize() {
-    return columns.size();
-  }
-
-  public void clear() {
-    columns.clear();
-  }
-
-
-  public TTableSchema toTTableSchema() {
-    TTableSchema tTableSchema = new TTableSchema();
-    for (ColumnDescriptor col : columns) {
-      tTableSchema.addToColumns(col.toTColumnDesc());
-    }
-    return tTableSchema;
-  }
-
-  public Type[] toTypes() {
-    Type[] types = new Type[columns.size()];
-    for (int i = 0; i < types.length; i++) {
-      types[i] = columns.get(i).getType();
-    }
-    return types;
-  }
-
-  public TableSchema addPrimitiveColumn(String columnName, Type columnType, String columnComment) {
-    columns.add(ColumnDescriptor.newPrimitiveColumnDescriptor(columnName, columnComment, columnType, columns.size() + 1));
-    return this;
-  }
-
-  public TableSchema addStringColumn(String columnName, String columnComment) {
-    columns.add(ColumnDescriptor.newPrimitiveColumnDescriptor(columnName, columnComment, Type.STRING_TYPE, columns.size() + 1));
-    return this;
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/Type.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/Type.java
deleted file mode 100644
index 7752ec03a29b7..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/Type.java
+++ /dev/null
@@ -1,349 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import java.sql.DatabaseMetaData;
-import java.util.Locale;
-
-import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hive.service.cli.thrift.TTypeId;
-
-/**
- * Type.
- *
- */
-public enum Type {
-  NULL_TYPE("VOID",
-      java.sql.Types.NULL,
-      TTypeId.NULL_TYPE),
-  BOOLEAN_TYPE("BOOLEAN",
-      java.sql.Types.BOOLEAN,
-      TTypeId.BOOLEAN_TYPE),
-  TINYINT_TYPE("TINYINT",
-      java.sql.Types.TINYINT,
-      TTypeId.TINYINT_TYPE),
-  SMALLINT_TYPE("SMALLINT",
-      java.sql.Types.SMALLINT,
-      TTypeId.SMALLINT_TYPE),
-  INT_TYPE("INT",
-      java.sql.Types.INTEGER,
-      TTypeId.INT_TYPE),
-  BIGINT_TYPE("BIGINT",
-      java.sql.Types.BIGINT,
-      TTypeId.BIGINT_TYPE),
-  FLOAT_TYPE("FLOAT",
-      java.sql.Types.FLOAT,
-      TTypeId.FLOAT_TYPE),
-  DOUBLE_TYPE("DOUBLE",
-      java.sql.Types.DOUBLE,
-      TTypeId.DOUBLE_TYPE),
-  STRING_TYPE("STRING",
-      java.sql.Types.VARCHAR,
-      TTypeId.STRING_TYPE),
-  CHAR_TYPE("CHAR",
-      java.sql.Types.CHAR,
-      TTypeId.CHAR_TYPE,
-      true, false, false),
-  VARCHAR_TYPE("VARCHAR",
-      java.sql.Types.VARCHAR,
-      TTypeId.VARCHAR_TYPE,
-      true, false, false),
-  DATE_TYPE("DATE",
-      java.sql.Types.DATE,
-      TTypeId.DATE_TYPE),
-  TIMESTAMP_TYPE("TIMESTAMP",
-      java.sql.Types.TIMESTAMP,
-      TTypeId.TIMESTAMP_TYPE),
-  INTERVAL_YEAR_MONTH_TYPE("INTERVAL_YEAR_MONTH",
-      java.sql.Types.OTHER,
-      TTypeId.INTERVAL_YEAR_MONTH_TYPE),
-  INTERVAL_DAY_TIME_TYPE("INTERVAL_DAY_TIME",
-      java.sql.Types.OTHER,
-      TTypeId.INTERVAL_DAY_TIME_TYPE),
-  BINARY_TYPE("BINARY",
-      java.sql.Types.BINARY,
-      TTypeId.BINARY_TYPE),
-  DECIMAL_TYPE("DECIMAL",
-      java.sql.Types.DECIMAL,
-      TTypeId.DECIMAL_TYPE,
-      true, false, false),
-  ARRAY_TYPE("ARRAY",
-      java.sql.Types.ARRAY,
-      TTypeId.ARRAY_TYPE,
-      true, true),
-  MAP_TYPE("MAP",
-      java.sql.Types.JAVA_OBJECT,
-      TTypeId.MAP_TYPE,
-      true, true),
-  STRUCT_TYPE("STRUCT",
-      java.sql.Types.STRUCT,
-      TTypeId.STRUCT_TYPE,
-      true, false),
-  UNION_TYPE("UNIONTYPE",
-      java.sql.Types.OTHER,
-      TTypeId.UNION_TYPE,
-      true, false),
-  USER_DEFINED_TYPE("USER_DEFINED",
-      java.sql.Types.OTHER,
-      TTypeId.USER_DEFINED_TYPE,
-      true, false);
-
-  private final String name;
-  private final TTypeId tType;
-  private final int javaSQLType;
-  private final boolean isQualified;
-  private final boolean isComplex;
-  private final boolean isCollection;
-
-  Type(String name, int javaSQLType, TTypeId tType, boolean isQualified, boolean isComplex, boolean isCollection) {
-    this.name = name;
-    this.javaSQLType = javaSQLType;
-    this.tType = tType;
-    this.isQualified = isQualified;
-    this.isComplex = isComplex;
-    this.isCollection = isCollection;
-  }
-
-  Type(String name, int javaSQLType, TTypeId tType, boolean isComplex, boolean isCollection) {
-    this(name, javaSQLType, tType, false, isComplex, isCollection);
-  }
-
-  Type(String name, int javaSqlType, TTypeId tType) {
-    this(name, javaSqlType, tType, false, false, false);
-  }
-
-  public boolean isPrimitiveType() {
-    return !isComplex;
-  }
-
-  public boolean isQualifiedType() {
-    return isQualified;
-  }
-
-  public boolean isComplexType() {
-    return isComplex;
-  }
-
-  public boolean isCollectionType() {
-    return isCollection;
-  }
-
-  public static Type getType(TTypeId tType) {
-    for (Type type : values()) {
-      if (tType.equals(type.tType)) {
-        return type;
-      }
-    }
-    throw new IllegalArgumentException("Unregonized Thrift TTypeId value: " + tType);
-  }
-
-  public static Type getType(String name) {
-    if (name == null) {
-      throw new IllegalArgumentException("Invalid type name: null");
-    }
-    for (Type type : values()) {
-      if (name.equalsIgnoreCase(type.name)) {
-        return type;
-      } else if (type.isQualifiedType() || type.isComplexType()) {
-        if (name.toUpperCase(Locale.ROOT).startsWith(type.name)) {
-            return type;
-        }
-      }
-    }
-    throw new IllegalArgumentException("Unrecognized type name: " + name);
-  }
-
-  /**
-   * Radix for this type (typically either 2 or 10)
-   * Null is returned for data types where this is not applicable.
-   */
-  public Integer getNumPrecRadix() {
-    if (this.isNumericType()) {
-      return 10;
-    }
-    return null;
-  }
-
-  /**
-   * Maximum precision for numeric types.
-   * Returns null for non-numeric types.
-   * @return
-   */
-  public Integer getMaxPrecision() {
-    switch (this) {
-    case TINYINT_TYPE:
-      return 3;
-    case SMALLINT_TYPE:
-      return 5;
-    case INT_TYPE:
-      return 10;
-    case BIGINT_TYPE:
-      return 19;
-    case FLOAT_TYPE:
-      return 7;
-    case DOUBLE_TYPE:
-      return 15;
-    case DECIMAL_TYPE:
-      return HiveDecimal.MAX_PRECISION;
-    default:
-      return null;
-    }
-  }
-
-  public boolean isNumericType() {
-    switch (this) {
-    case TINYINT_TYPE:
-    case SMALLINT_TYPE:
-    case INT_TYPE:
-    case BIGINT_TYPE:
-    case FLOAT_TYPE:
-    case DOUBLE_TYPE:
-    case DECIMAL_TYPE:
-      return true;
-    default:
-      return false;
-    }
-  }
-
-  /**
-   * Prefix used to quote a literal of this type (may be null)
-   */
-  public String getLiteralPrefix() {
-    return null;
-  }
-
-  /**
-   * Suffix used to quote a literal of this type (may be null)
-   * @return
-   */
-  public String getLiteralSuffix() {
-    return null;
-  }
-
-  /**
-   * Can you use NULL for this type?
-   * @return
-   * DatabaseMetaData.typeNoNulls - does not allow NULL values
-   * DatabaseMetaData.typeNullable - allows NULL values
-   * DatabaseMetaData.typeNullableUnknown - nullability unknown
-   */
-  public Short getNullable() {
-    // All Hive types are nullable
-    return DatabaseMetaData.typeNullable;
-  }
-
-  /**
-   * Is the type case sensitive?
-   * @return
-   */
-  public Boolean isCaseSensitive() {
-    switch (this) {
-    case STRING_TYPE:
-      return true;
-    default:
-      return false;
-    }
-  }
-
-  /**
-   * Parameters used in creating the type (may be null)
-   * @return
-   */
-  public String getCreateParams() {
-    return null;
-  }
-
-  /**
-   * Can you use WHERE based on this type?
-   * @return
-   * DatabaseMetaData.typePredNone - No support
-   * DatabaseMetaData.typePredChar - Only support with WHERE .. LIKE
-   * DatabaseMetaData.typePredBasic - Supported except for WHERE .. LIKE
-   * DatabaseMetaData.typeSearchable - Supported for all WHERE ..
-   */
-  public Short getSearchable() {
-    if (isPrimitiveType()) {
-      return DatabaseMetaData.typeSearchable;
-    }
-    return DatabaseMetaData.typePredNone;
-  }
-
-  /**
-   * Is this type unsigned?
-   * @return
-   */
-  public Boolean isUnsignedAttribute() {
-    if (isNumericType()) {
-      return false;
-    }
-    return true;
-  }
-
-  /**
-   * Can this type represent money?
-   * @return
-   */
-  public Boolean isFixedPrecScale() {
-    return false;
-  }
-
-  /**
-   * Can this type be used for an auto-increment value?
-   * @return
-   */
-  public Boolean isAutoIncrement() {
-    return false;
-  }
-
-  /**
-   * Localized version of type name (may be null).
-   * @return
-   */
-  public String getLocalizedName() {
-    return null;
-  }
-
-  /**
-   * Minimum scale supported for this type
-   * @return
-   */
-  public Short getMinimumScale() {
-    return 0;
-  }
-
-  /**
-   * Maximum scale supported for this type
-   * @return
-   */
-  public Short getMaximumScale() {
-    return 0;
-  }
-
-  public TTypeId toTType() {
-    return tType;
-  }
-
-  public int toJavaSQLType() {
-    return javaSQLType;
-  }
-
-  public String getName() {
-    return name;
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/TypeDescriptor.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/TypeDescriptor.java
deleted file mode 100644
index b80fd67884add..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/TypeDescriptor.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import java.util.List;
-
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.hive.service.cli.thrift.TPrimitiveTypeEntry;
-import org.apache.hive.service.cli.thrift.TTypeDesc;
-import org.apache.hive.service.cli.thrift.TTypeEntry;
-
-/**
- * TypeDescriptor.
- *
- */
-public class TypeDescriptor {
-
-  private final Type type;
-  private String typeName = null;
-  private TypeQualifiers typeQualifiers = null;
-
-  public TypeDescriptor(Type type) {
-    this.type = type;
-  }
-
-  public TypeDescriptor(TTypeDesc tTypeDesc) {
-    List<TTypeEntry> tTypeEntries = tTypeDesc.getTypes();
-    TPrimitiveTypeEntry top = tTypeEntries.get(0).getPrimitiveEntry();
-    this.type = Type.getType(top.getType());
-    if (top.isSetTypeQualifiers()) {
-      setTypeQualifiers(TypeQualifiers.fromTTypeQualifiers(top.getTypeQualifiers()));
-    }
-  }
-
-  public TypeDescriptor(String typeName) {
-    this.type = Type.getType(typeName);
-    if (this.type.isComplexType()) {
-      this.typeName = typeName;
-    } else if (this.type.isQualifiedType()) {
-      PrimitiveTypeInfo pti = TypeInfoFactory.getPrimitiveTypeInfo(typeName);
-      setTypeQualifiers(TypeQualifiers.fromTypeInfo(pti));
-    }
-  }
-
-  public Type getType() {
-    return type;
-  }
-
-  public TTypeDesc toTTypeDesc() {
-    TPrimitiveTypeEntry primitiveEntry = new TPrimitiveTypeEntry(type.toTType());
-    if (getTypeQualifiers() != null) {
-      primitiveEntry.setTypeQualifiers(getTypeQualifiers().toTTypeQualifiers());
-    }
-    TTypeEntry entry = TTypeEntry.primitiveEntry(primitiveEntry);
-
-    TTypeDesc desc = new TTypeDesc();
-    desc.addToTypes(entry);
-    return desc;
-  }
-
-  public String getTypeName() {
-    if (typeName != null) {
-      return typeName;
-    } else {
-      return type.getName();
-    }
-  }
-
-  public TypeQualifiers getTypeQualifiers() {
-    return typeQualifiers;
-  }
-
-  public void setTypeQualifiers(TypeQualifiers typeQualifiers) {
-    this.typeQualifiers = typeQualifiers;
-  }
-
-  /**
-   * The column size for this type.
-   * For numeric data this is the maximum precision.
-   * For character data this is the length in characters.
-   * For datetime types this is the length in characters of the String representation
-   * (assuming the maximum allowed precision of the fractional seconds component).
-   * For binary data this is the length in bytes.
-   * Null is returned for data types where the column size is not applicable.
-   */
-  public Integer getColumnSize() {
-    if (type.isNumericType()) {
-      return getPrecision();
-    }
-    switch (type) {
-    case STRING_TYPE:
-    case BINARY_TYPE:
-      return Integer.MAX_VALUE;
-    case CHAR_TYPE:
-    case VARCHAR_TYPE:
-      return typeQualifiers.getCharacterMaximumLength();
-    case DATE_TYPE:
-      return 10;
-    case TIMESTAMP_TYPE:
-      return 29;
-    default:
-      return null;
-    }
-  }
-
-  /**
-   * Maximum precision for numeric types.
-   * Returns null for non-numeric types.
-   * @return
-   */
-  public Integer getPrecision() {
-    if (this.type == Type.DECIMAL_TYPE) {
-      return typeQualifiers.getPrecision();
-    }
-    return this.type.getMaxPrecision();
-  }
-
-  /**
-   * The number of fractional digits for this type.
-   * Null is returned for data types where this is not applicable.
-   */
-  public Integer getDecimalDigits() {
-    switch (this.type) {
-    case BOOLEAN_TYPE:
-    case TINYINT_TYPE:
-    case SMALLINT_TYPE:
-    case INT_TYPE:
-    case BIGINT_TYPE:
-      return 0;
-    case FLOAT_TYPE:
-      return 7;
-    case DOUBLE_TYPE:
-      return 15;
-    case DECIMAL_TYPE:
-      return typeQualifiers.getScale();
-    case TIMESTAMP_TYPE:
-      return 9;
-    default:
-      return null;
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/TypeQualifiers.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/TypeQualifiers.java
deleted file mode 100644
index c6da52c15a2b5..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/TypeQualifiers.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
-import org.apache.hive.service.cli.thrift.TCLIServiceConstants;
-import org.apache.hive.service.cli.thrift.TTypeQualifierValue;
-import org.apache.hive.service.cli.thrift.TTypeQualifiers;
-
-/**
- * This class holds type qualifier information for a primitive type,
- * such as char/varchar length or decimal precision/scale.
- */
-public class TypeQualifiers {
-  private Integer characterMaximumLength;
-  private Integer precision;
-  private Integer scale;
-
-  public TypeQualifiers() {}
-
-  public Integer getCharacterMaximumLength() {
-    return characterMaximumLength;
-  }
-  public void setCharacterMaximumLength(int characterMaximumLength) {
-    this.characterMaximumLength = characterMaximumLength;
-  }
-
-  public TTypeQualifiers toTTypeQualifiers() {
-    TTypeQualifiers ret = null;
-
-    Map<String, TTypeQualifierValue> qMap = new HashMap<String, TTypeQualifierValue>();
-    if (getCharacterMaximumLength() != null) {
-      TTypeQualifierValue val = new TTypeQualifierValue();
-      val.setI32Value(getCharacterMaximumLength().intValue());
-      qMap.put(TCLIServiceConstants.CHARACTER_MAXIMUM_LENGTH, val);
-    }
-
-    if (precision != null) {
-      TTypeQualifierValue val = new TTypeQualifierValue();
-      val.setI32Value(precision.intValue());
-      qMap.put(TCLIServiceConstants.PRECISION, val);
-    }
-
-    if (scale != null) {
-      TTypeQualifierValue val = new TTypeQualifierValue();
-      val.setI32Value(scale.intValue());
-      qMap.put(TCLIServiceConstants.SCALE, val);
-    }
-
-    if (qMap.size() > 0) {
-      ret = new TTypeQualifiers(qMap);
-    }
-
-    return ret;
-  }
-
-  public static TypeQualifiers fromTTypeQualifiers(TTypeQualifiers ttq) {
-    TypeQualifiers ret = null;
-    if (ttq != null) {
-      ret = new TypeQualifiers();
-      Map<String, TTypeQualifierValue> tqMap = ttq.getQualifiers();
-
-      if (tqMap.containsKey(TCLIServiceConstants.CHARACTER_MAXIMUM_LENGTH)) {
-        ret.setCharacterMaximumLength(
-            tqMap.get(TCLIServiceConstants.CHARACTER_MAXIMUM_LENGTH).getI32Value());
-      }
-
-      if (tqMap.containsKey(TCLIServiceConstants.PRECISION)) {
-        ret.setPrecision(tqMap.get(TCLIServiceConstants.PRECISION).getI32Value());
-      }
-
-      if (tqMap.containsKey(TCLIServiceConstants.SCALE)) {
-        ret.setScale(tqMap.get(TCLIServiceConstants.SCALE).getI32Value());
-      }
-    }
-    return ret;
-  }
-
-  public static TypeQualifiers fromTypeInfo(PrimitiveTypeInfo pti) {
-    TypeQualifiers result = null;
-    if (pti instanceof VarcharTypeInfo) {
-      result = new TypeQualifiers();
-      result.setCharacterMaximumLength(((VarcharTypeInfo)pti).getLength());
-    }  else if (pti instanceof CharTypeInfo) {
-      result = new TypeQualifiers();
-      result.setCharacterMaximumLength(((CharTypeInfo)pti).getLength());
-    } else if (pti instanceof DecimalTypeInfo) {
-      result = new TypeQualifiers();
-      result.setPrecision(((DecimalTypeInfo)pti).precision());
-      result.setScale(((DecimalTypeInfo)pti).scale());
-    }
-    return result;
-  }
-
-  public Integer getPrecision() {
-    return precision;
-  }
-
-  public void setPrecision(Integer precision) {
-    this.precision = precision;
-  }
-
-  public Integer getScale() {
-    return scale;
-  }
-
-  public void setScale(Integer scale) {
-    this.scale = scale;
-  }
-
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/ClassicTableTypeMapping.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/ClassicTableTypeMapping.java
deleted file mode 100644
index af36057bdaeca..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/ClassicTableTypeMapping.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.operation;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.hadoop.hive.metastore.TableType;
-
-/**
- * ClassicTableTypeMapping.
- * Classic table type mapping :
- *  Managed Table to Table
- *  External Table to Table
- *  Virtual View to View
- */
-public class ClassicTableTypeMapping implements TableTypeMapping {
-
-  public enum ClassicTableTypes {
-    TABLE,
-    VIEW,
-  }
-
-  private final Map<String, String> hiveToClientMap = new HashMap<String, String>();
-  private final Map<String, String> clientToHiveMap = new HashMap<String, String>();
-
-  public ClassicTableTypeMapping() {
-    hiveToClientMap.put(TableType.MANAGED_TABLE.toString(),
-        ClassicTableTypes.TABLE.toString());
-    hiveToClientMap.put(TableType.EXTERNAL_TABLE.toString(),
-        ClassicTableTypes.TABLE.toString());
-    hiveToClientMap.put(TableType.VIRTUAL_VIEW.toString(),
-        ClassicTableTypes.VIEW.toString());
-
-    clientToHiveMap.put(ClassicTableTypes.TABLE.toString(),
-        TableType.MANAGED_TABLE.toString());
-    clientToHiveMap.put(ClassicTableTypes.VIEW.toString(),
-        TableType.VIRTUAL_VIEW.toString());
-  }
-
-  @Override
-  public String mapToHiveType(String clientTypeName) {
-    if (clientToHiveMap.containsKey(clientTypeName)) {
-      return clientToHiveMap.get(clientTypeName);
-    } else {
-      return clientTypeName;
-    }
-  }
-
-  @Override
-  public String mapToClientType(String hiveTypeName) {
-    if (hiveToClientMap.containsKey(hiveTypeName)) {
-      return hiveToClientMap.get(hiveTypeName);
-    } else {
-      return hiveTypeName;
-    }
-  }
-
-  @Override
-  public Set<String> getTableTypeNames() {
-    Set<String> typeNameSet = new HashSet<String>();
-    for (ClassicTableTypes typeNames : ClassicTableTypes.values()) {
-      typeNameSet.add(typeNames.toString());
-    }
-    return typeNameSet;
-  }
-
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/ExecuteStatementOperation.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/ExecuteStatementOperation.java
deleted file mode 100644
index 6740d3bb59dc3..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/ExecuteStatementOperation.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.service.cli.operation;
-
-import java.sql.SQLException;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hadoop.hive.ql.processors.CommandProcessor;
-import org.apache.hadoop.hive.ql.processors.CommandProcessorFactory;
-import org.apache.hadoop.hive.ql.session.OperationLog;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.OperationType;
-import org.apache.hive.service.cli.session.HiveSession;
-
-public abstract class ExecuteStatementOperation extends Operation {
-  protected String statement = null;
-  protected Map<String, String> confOverlay = new HashMap<String, String>();
-
-  public ExecuteStatementOperation(HiveSession parentSession, String statement,
-      Map<String, String> confOverlay, boolean runInBackground) {
-    super(parentSession, OperationType.EXECUTE_STATEMENT, runInBackground);
-    this.statement = statement;
-    setConfOverlay(confOverlay);
-  }
-
-  public String getStatement() {
-    return statement;
-  }
-
-  public static ExecuteStatementOperation newExecuteStatementOperation(
-      HiveSession parentSession, String statement, Map<String, String> confOverlay, boolean runAsync)
-          throws HiveSQLException {
-    String[] tokens = statement.trim().split("\\s+");
-    CommandProcessor processor = null;
-    try {
-      processor = CommandProcessorFactory.getForHiveCommand(tokens, parentSession.getHiveConf());
-    } catch (SQLException e) {
-      throw new HiveSQLException(e.getMessage(), e.getSQLState(), e);
-    }
-    if (processor == null) {
-      return new SQLOperation(parentSession, statement, confOverlay, runAsync);
-    }
-    return new HiveCommandOperation(parentSession, statement, processor, confOverlay);
-  }
-
-  protected Map<String, String> getConfOverlay() {
-    return confOverlay;
-  }
-
-  protected void setConfOverlay(Map<String, String> confOverlay) {
-    if (confOverlay != null) {
-      this.confOverlay = confOverlay;
-    }
-  }
-
-  protected void registerCurrentOperationLog() {
-    if (isOperationLogEnabled) {
-      if (operationLog == null) {
-        LOG.warn("Failed to get current OperationLog object of Operation: " +
-          getHandle().getHandleIdentifier());
-        isOperationLogEnabled = false;
-        return;
-      }
-      OperationLog.setCurrentOperationLog(operationLog);
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetCatalogsOperation.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetCatalogsOperation.java
deleted file mode 100644
index 581d975344060..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetCatalogsOperation.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.operation;
-
-import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
-import org.apache.hive.service.cli.FetchOrientation;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.OperationState;
-import org.apache.hive.service.cli.OperationType;
-import org.apache.hive.service.cli.RowSet;
-import org.apache.hive.service.cli.RowSetFactory;
-import org.apache.hive.service.cli.TableSchema;
-import org.apache.hive.service.cli.session.HiveSession;
-
-/**
- * GetCatalogsOperation.
- *
- */
-public class GetCatalogsOperation extends MetadataOperation {
-  private static final TableSchema RESULT_SET_SCHEMA = new TableSchema()
-  .addStringColumn("TABLE_CAT", "Catalog name. NULL if not applicable.");
-
-  protected final RowSet rowSet;
-
-  protected GetCatalogsOperation(HiveSession parentSession) {
-    super(parentSession, OperationType.GET_CATALOGS);
-    rowSet = RowSetFactory.create(RESULT_SET_SCHEMA, getProtocolVersion());
-  }
-
-  @Override
-  public void runInternal() throws HiveSQLException {
-    setState(OperationState.RUNNING);
-    try {
-      if (isAuthV2Enabled()) {
-        authorizeMetaGets(HiveOperationType.GET_CATALOGS, null);
-      }
-      setState(OperationState.FINISHED);
-    } catch (HiveSQLException e) {
-      setState(OperationState.ERROR);
-      throw e;
-    }
-
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.Operation#getResultSetSchema()
-   */
-  @Override
-  public TableSchema getResultSetSchema() throws HiveSQLException {
-    return RESULT_SET_SCHEMA;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.Operation#getNextRowSet(org.apache.hive.service.cli.FetchOrientation, long)
-   */
-  @Override
-  public RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws HiveSQLException {
-    assertState(OperationState.FINISHED);
-    validateDefaultFetchOrientation(orientation);
-    if (orientation.equals(FetchOrientation.FETCH_FIRST)) {
-      rowSet.setStartOffset(0);
-    }
-    return rowSet.extractSubset((int)maxRows);
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetColumnsOperation.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetColumnsOperation.java
deleted file mode 100644
index 96ba4890075ac..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetColumnsOperation.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.operation;
-
-import java.sql.DatabaseMetaData;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.regex.Pattern;
-
-import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
-import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject;
-import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivilegeObjectType;
-import org.apache.hive.service.cli.ColumnDescriptor;
-import org.apache.hive.service.cli.FetchOrientation;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.OperationState;
-import org.apache.hive.service.cli.OperationType;
-import org.apache.hive.service.cli.RowSet;
-import org.apache.hive.service.cli.RowSetFactory;
-import org.apache.hive.service.cli.TableSchema;
-import org.apache.hive.service.cli.Type;
-import org.apache.hive.service.cli.session.HiveSession;
-
-/**
- * GetColumnsOperation.
- *
- */
-public class GetColumnsOperation extends MetadataOperation {
-
-  protected static final TableSchema RESULT_SET_SCHEMA = new TableSchema()
-  .addPrimitiveColumn("TABLE_CAT", Type.STRING_TYPE,
-      "Catalog name. NULL if not applicable")
-  .addPrimitiveColumn("TABLE_SCHEM", Type.STRING_TYPE,
-      "Schema name")
-  .addPrimitiveColumn("TABLE_NAME", Type.STRING_TYPE,
-      "Table name")
-  .addPrimitiveColumn("COLUMN_NAME", Type.STRING_TYPE,
-      "Column name")
-  .addPrimitiveColumn("DATA_TYPE", Type.INT_TYPE,
-      "SQL type from java.sql.Types")
-  .addPrimitiveColumn("TYPE_NAME", Type.STRING_TYPE,
-      "Data source dependent type name, for a UDT the type name is fully qualified")
-  .addPrimitiveColumn("COLUMN_SIZE", Type.INT_TYPE,
-      "Column size. For char or date types this is the maximum number of characters,"
-      + " for numeric or decimal types this is precision.")
-  .addPrimitiveColumn("BUFFER_LENGTH", Type.TINYINT_TYPE,
-      "Unused")
-  .addPrimitiveColumn("DECIMAL_DIGITS", Type.INT_TYPE,
-      "The number of fractional digits")
-  .addPrimitiveColumn("NUM_PREC_RADIX", Type.INT_TYPE,
-      "Radix (typically either 10 or 2)")
-  .addPrimitiveColumn("NULLABLE", Type.INT_TYPE,
-      "Is NULL allowed")
-  .addPrimitiveColumn("REMARKS", Type.STRING_TYPE,
-      "Comment describing column (may be null)")
-  .addPrimitiveColumn("COLUMN_DEF", Type.STRING_TYPE,
-      "Default value (may be null)")
-  .addPrimitiveColumn("SQL_DATA_TYPE", Type.INT_TYPE,
-      "Unused")
-  .addPrimitiveColumn("SQL_DATETIME_SUB", Type.INT_TYPE,
-      "Unused")
-  .addPrimitiveColumn("CHAR_OCTET_LENGTH", Type.INT_TYPE,
-      "For char types the maximum number of bytes in the column")
-  .addPrimitiveColumn("ORDINAL_POSITION", Type.INT_TYPE,
-      "Index of column in table (starting at 1)")
-  .addPrimitiveColumn("IS_NULLABLE", Type.STRING_TYPE,
-      "\"NO\" means column definitely does not allow NULL values; "
-      + "\"YES\" means the column might allow NULL values. An empty "
-      + "string means nobody knows.")
-  .addPrimitiveColumn("SCOPE_CATALOG", Type.STRING_TYPE,
-      "Catalog of table that is the scope of a reference attribute "
-      + "(null if DATA_TYPE isn't REF)")
-  .addPrimitiveColumn("SCOPE_SCHEMA", Type.STRING_TYPE,
-      "Schema of table that is the scope of a reference attribute "
-      + "(null if the DATA_TYPE isn't REF)")
-  .addPrimitiveColumn("SCOPE_TABLE", Type.STRING_TYPE,
-      "Table name that this the scope of a reference attribure "
-      + "(null if the DATA_TYPE isn't REF)")
-  .addPrimitiveColumn("SOURCE_DATA_TYPE", Type.SMALLINT_TYPE,
-      "Source type of a distinct type or user-generated Ref type, "
-      + "SQL type from java.sql.Types (null if DATA_TYPE isn't DISTINCT or user-generated REF)")
-  .addPrimitiveColumn("IS_AUTO_INCREMENT", Type.STRING_TYPE,
-      "Indicates whether this column is auto incremented.");
-
-  private final String catalogName;
-  private final String schemaName;
-  private final String tableName;
-  private final String columnName;
-
-  protected final RowSet rowSet;
-
-  protected GetColumnsOperation(HiveSession parentSession, String catalogName, String schemaName,
-      String tableName, String columnName) {
-    super(parentSession, OperationType.GET_COLUMNS);
-    this.catalogName = catalogName;
-    this.schemaName = schemaName;
-    this.tableName = tableName;
-    this.columnName = columnName;
-    this.rowSet = RowSetFactory.create(RESULT_SET_SCHEMA, getProtocolVersion());
-  }
-
-  @Override
-  public void runInternal() throws HiveSQLException {
-    setState(OperationState.RUNNING);
-    try {
-      IMetaStoreClient metastoreClient = getParentSession().getMetaStoreClient();
-      String schemaPattern = convertSchemaPattern(schemaName);
-      String tablePattern = convertIdentifierPattern(tableName, true);
-
-      Pattern columnPattern = null;
-      if (columnName != null) {
-        columnPattern = Pattern.compile(convertIdentifierPattern(columnName, false));
-      }
-
-      List<String> dbNames = metastoreClient.getDatabases(schemaPattern);
-      Collections.sort(dbNames);
-      Map<String, List<String>> db2Tabs = new HashMap<>();
-
-      for (String dbName : dbNames) {
-        List<String> tableNames = metastoreClient.getTables(dbName, tablePattern);
-        Collections.sort(tableNames);
-        db2Tabs.put(dbName, tableNames);
-      }
-
-      if (isAuthV2Enabled()) {
-        List<HivePrivilegeObject> privObjs = getPrivObjs(db2Tabs);
-        String cmdStr = "catalog : " + catalogName + ", schemaPattern : " + schemaName
-            + ", tablePattern : " + tableName;
-        authorizeMetaGets(HiveOperationType.GET_COLUMNS, privObjs, cmdStr);
-      }
-
-      for (Entry<String, List<String>> dbTabs : db2Tabs.entrySet()) {
-        String dbName = dbTabs.getKey();
-        List<String> tableNames = dbTabs.getValue();
-        for (Table table : metastoreClient.getTableObjectsByName(dbName, tableNames)) {
-          TableSchema schema = new TableSchema(metastoreClient.getSchema(dbName, table.getTableName()));
-          for (ColumnDescriptor column : schema.getColumnDescriptors()) {
-            if (columnPattern != null && !columnPattern.matcher(column.getName()).matches()) {
-              continue;
-            }
-            Object[] rowData = new Object[] {
-                null,  // TABLE_CAT
-                table.getDbName(), // TABLE_SCHEM
-                table.getTableName(), // TABLE_NAME
-                column.getName(), // COLUMN_NAME
-                column.getType().toJavaSQLType(), // DATA_TYPE
-                column.getTypeName(), // TYPE_NAME
-                column.getTypeDescriptor().getColumnSize(), // COLUMN_SIZE
-                null, // BUFFER_LENGTH, unused
-                column.getTypeDescriptor().getDecimalDigits(), // DECIMAL_DIGITS
-                column.getType().getNumPrecRadix(), // NUM_PREC_RADIX
-                DatabaseMetaData.columnNullable, // NULLABLE
-                column.getComment(), // REMARKS
-                null, // COLUMN_DEF
-                null, // SQL_DATA_TYPE
-                null, // SQL_DATETIME_SUB
-                null, // CHAR_OCTET_LENGTH
-                column.getOrdinalPosition(), // ORDINAL_POSITION
-                "YES", // IS_NULLABLE
-                null, // SCOPE_CATALOG
-                null, // SCOPE_SCHEMA
-                null, // SCOPE_TABLE
-                null, // SOURCE_DATA_TYPE
-                "NO", // IS_AUTO_INCREMENT
-            };
-            rowSet.addRow(rowData);
-          }
-        }
-      }
-      setState(OperationState.FINISHED);
-    } catch (Exception e) {
-      setState(OperationState.ERROR);
-      throw new HiveSQLException(e);
-    }
-
-  }
-
-
-  private List<HivePrivilegeObject> getPrivObjs(Map<String, List<String>> db2Tabs) {
-    List<HivePrivilegeObject> privObjs = new ArrayList<>();
-    for (Entry<String, List<String>> dbTabs : db2Tabs.entrySet()) {
-      for (String tabName : dbTabs.getValue()) {
-        privObjs.add(new HivePrivilegeObject(HivePrivilegeObjectType.TABLE_OR_VIEW, dbTabs.getKey(),
-            tabName));
-      }
-    }
-    return privObjs;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.Operation#getResultSetSchema()
-   */
-  @Override
-  public TableSchema getResultSetSchema() throws HiveSQLException {
-    assertState(OperationState.FINISHED);
-    return RESULT_SET_SCHEMA;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.Operation#getNextRowSet(org.apache.hive.service.cli.FetchOrientation, long)
-   */
-  @Override
-  public RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws HiveSQLException {
-    assertState(OperationState.FINISHED);
-    validateDefaultFetchOrientation(orientation);
-    if (orientation.equals(FetchOrientation.FETCH_FIRST)) {
-      rowSet.setStartOffset(0);
-    }
-    return rowSet.extractSubset((int)maxRows);
-  }
-
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetFunctionsOperation.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetFunctionsOperation.java
deleted file mode 100644
index 5dec8bdbf45de..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetFunctionsOperation.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.operation;
-
-import java.sql.DatabaseMetaData;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hadoop.hive.ql.exec.FunctionInfo;
-import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
-import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
-import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject;
-import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObjectUtils;
-import org.apache.hive.service.cli.CLIServiceUtils;
-import org.apache.hive.service.cli.FetchOrientation;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.OperationState;
-import org.apache.hive.service.cli.OperationType;
-import org.apache.hive.service.cli.RowSet;
-import org.apache.hive.service.cli.RowSetFactory;
-import org.apache.hive.service.cli.TableSchema;
-import org.apache.hive.service.cli.Type;
-import org.apache.hive.service.cli.session.HiveSession;
-import org.apache.thrift.TException;
-
-/**
- * GetFunctionsOperation.
- *
- */
-public class GetFunctionsOperation extends MetadataOperation {
-  private static final TableSchema RESULT_SET_SCHEMA = new TableSchema()
-  .addPrimitiveColumn("FUNCTION_CAT", Type.STRING_TYPE,
-      "Function catalog (may be null)")
-  .addPrimitiveColumn("FUNCTION_SCHEM", Type.STRING_TYPE,
-      "Function schema (may be null)")
-  .addPrimitiveColumn("FUNCTION_NAME", Type.STRING_TYPE,
-      "Function name. This is the name used to invoke the function")
-  .addPrimitiveColumn("REMARKS", Type.STRING_TYPE,
-      "Explanatory comment on the function")
-  .addPrimitiveColumn("FUNCTION_TYPE", Type.INT_TYPE,
-      "Kind of function.")
-  .addPrimitiveColumn("SPECIFIC_NAME", Type.STRING_TYPE,
-      "The name which uniquely identifies this function within its schema");
-
-  private final String catalogName;
-  private final String schemaName;
-  private final String functionName;
-
-  protected final RowSet rowSet;
-
-  public GetFunctionsOperation(HiveSession parentSession,
-      String catalogName, String schemaName, String functionName) {
-    super(parentSession, OperationType.GET_FUNCTIONS);
-    this.catalogName = catalogName;
-    this.schemaName = schemaName;
-    this.functionName = functionName;
-    this.rowSet = RowSetFactory.create(RESULT_SET_SCHEMA, getProtocolVersion());
-  }
-
-  @Override
-  public void runInternal() throws HiveSQLException {
-    setState(OperationState.RUNNING);
-    if (isAuthV2Enabled()) {
-      // get databases for schema pattern
-      IMetaStoreClient metastoreClient = getParentSession().getMetaStoreClient();
-      String schemaPattern = convertSchemaPattern(schemaName);
-      List<String> matchingDbs;
-      try {
-        matchingDbs = metastoreClient.getDatabases(schemaPattern);
-      } catch (TException e) {
-        setState(OperationState.ERROR);
-        throw new HiveSQLException(e);
-      }
-      // authorize this call on the schema objects
-      List<HivePrivilegeObject> privObjs = HivePrivilegeObjectUtils
-          .getHivePrivDbObjects(matchingDbs);
-      String cmdStr = "catalog : " + catalogName + ", schemaPattern : " + schemaName;
-      authorizeMetaGets(HiveOperationType.GET_FUNCTIONS, privObjs, cmdStr);
-    }
-
-    try {
-      if ((null == catalogName || "".equals(catalogName))
-          && (null == schemaName || "".equals(schemaName))) {
-        Set<String> functionNames =  FunctionRegistry
-            .getFunctionNames(CLIServiceUtils.patternToRegex(functionName));
-        for (String functionName : functionNames) {
-          FunctionInfo functionInfo = FunctionRegistry.getFunctionInfo(functionName);
-          Object[] rowData = new Object[] {
-              null, // FUNCTION_CAT
-              null, // FUNCTION_SCHEM
-              functionInfo.getDisplayName(), // FUNCTION_NAME
-              "", // REMARKS
-              (functionInfo.isGenericUDTF() ?
-                  DatabaseMetaData.functionReturnsTable
-                  : DatabaseMetaData.functionNoTable), // FUNCTION_TYPE
-             functionInfo.getClass().getCanonicalName()
-          };
-          rowSet.addRow(rowData);
-        }
-      }
-      setState(OperationState.FINISHED);
-    } catch (Exception e) {
-      setState(OperationState.ERROR);
-      throw new HiveSQLException(e);
-    }
-  }
-
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.Operation#getResultSetSchema()
-   */
-  @Override
-  public TableSchema getResultSetSchema() throws HiveSQLException {
-    assertState(OperationState.FINISHED);
-    return RESULT_SET_SCHEMA;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.Operation#getNextRowSet(org.apache.hive.service.cli.FetchOrientation, long)
-   */
-  @Override
-  public RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws HiveSQLException {
-    assertState(OperationState.FINISHED);
-    validateDefaultFetchOrientation(orientation);
-    if (orientation.equals(FetchOrientation.FETCH_FIRST)) {
-      rowSet.setStartOffset(0);
-    }
-    return rowSet.extractSubset((int)maxRows);
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetSchemasOperation.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetSchemasOperation.java
deleted file mode 100644
index 3516bc2ba242c..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetSchemasOperation.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.operation;
-
-import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
-import org.apache.hive.service.cli.FetchOrientation;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.OperationState;
-import org.apache.hive.service.cli.OperationType;
-import org.apache.hive.service.cli.RowSet;
-import org.apache.hive.service.cli.RowSetFactory;
-import org.apache.hive.service.cli.TableSchema;
-import org.apache.hive.service.cli.session.HiveSession;
-
-/**
- * GetSchemasOperation.
- *
- */
-public class GetSchemasOperation extends MetadataOperation {
-  private final String catalogName;
-  private final String schemaName;
-
-  private static final TableSchema RESULT_SET_SCHEMA = new TableSchema()
-  .addStringColumn("TABLE_SCHEM", "Schema name.")
-  .addStringColumn("TABLE_CATALOG", "Catalog name.");
-
-  protected RowSet rowSet;
-
-  protected GetSchemasOperation(HiveSession parentSession,
-      String catalogName, String schemaName) {
-    super(parentSession, OperationType.GET_SCHEMAS);
-    this.catalogName = catalogName;
-    this.schemaName = schemaName;
-    this.rowSet = RowSetFactory.create(RESULT_SET_SCHEMA, getProtocolVersion());
-  }
-
-  @Override
-  public void runInternal() throws HiveSQLException {
-    setState(OperationState.RUNNING);
-    if (isAuthV2Enabled()) {
-      String cmdStr = "catalog : " + catalogName + ", schemaPattern : " + schemaName;
-      authorizeMetaGets(HiveOperationType.GET_SCHEMAS, null, cmdStr);
-    }
-    try {
-      IMetaStoreClient metastoreClient = getParentSession().getMetaStoreClient();
-      String schemaPattern = convertSchemaPattern(schemaName);
-      for (String dbName : metastoreClient.getDatabases(schemaPattern)) {
-        rowSet.addRow(new Object[] {dbName, DEFAULT_HIVE_CATALOG});
-      }
-      setState(OperationState.FINISHED);
-    } catch (Exception e) {
-      setState(OperationState.ERROR);
-      throw new HiveSQLException(e);
-    }
-  }
-
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.Operation#getResultSetSchema()
-   */
-  @Override
-  public TableSchema getResultSetSchema() throws HiveSQLException {
-    assertState(OperationState.FINISHED);
-    return RESULT_SET_SCHEMA;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.Operation#getNextRowSet(org.apache.hive.service.cli.FetchOrientation, long)
-   */
-  @Override
-  public RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws HiveSQLException {
-    assertState(OperationState.FINISHED);
-    validateDefaultFetchOrientation(orientation);
-    if (orientation.equals(FetchOrientation.FETCH_FIRST)) {
-      rowSet.setStartOffset(0);
-    }
-    return rowSet.extractSubset((int)maxRows);
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetTableTypesOperation.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetTableTypesOperation.java
deleted file mode 100644
index b372f55cedd1c..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetTableTypesOperation.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.operation;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.TableType;
-import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
-import org.apache.hive.service.cli.FetchOrientation;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.OperationState;
-import org.apache.hive.service.cli.OperationType;
-import org.apache.hive.service.cli.RowSet;
-import org.apache.hive.service.cli.RowSetFactory;
-import org.apache.hive.service.cli.TableSchema;
-import org.apache.hive.service.cli.session.HiveSession;
-
-/**
- * GetTableTypesOperation.
- *
- */
-public class GetTableTypesOperation extends MetadataOperation {
-
-  protected static TableSchema RESULT_SET_SCHEMA = new TableSchema()
-  .addStringColumn("TABLE_TYPE", "Table type name.");
-
-  protected final RowSet rowSet;
-  private final TableTypeMapping tableTypeMapping;
-
-  protected GetTableTypesOperation(HiveSession parentSession) {
-    super(parentSession, OperationType.GET_TABLE_TYPES);
-    String tableMappingStr = getParentSession().getHiveConf()
-      .getVar(HiveConf.ConfVars.HIVE_SERVER2_TABLE_TYPE_MAPPING);
-    tableTypeMapping =
-      TableTypeMappingFactory.getTableTypeMapping(tableMappingStr);
-    rowSet = RowSetFactory.create(RESULT_SET_SCHEMA, getProtocolVersion());
-  }
-
-  @Override
-  public void runInternal() throws HiveSQLException {
-    setState(OperationState.RUNNING);
-    if (isAuthV2Enabled()) {
-      authorizeMetaGets(HiveOperationType.GET_TABLETYPES, null);
-    }
-    try {
-      for (TableType type : TableType.values()) {
-        rowSet.addRow(new String[] {tableTypeMapping.mapToClientType(type.toString())});
-      }
-      setState(OperationState.FINISHED);
-    } catch (Exception e) {
-      setState(OperationState.ERROR);
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.Operation#getResultSetSchema()
-   */
-  @Override
-  public TableSchema getResultSetSchema() throws HiveSQLException {
-    assertState(OperationState.FINISHED);
-    return RESULT_SET_SCHEMA;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.Operation#getNextRowSet(org.apache.hive.service.cli.FetchOrientation, long)
-   */
-  @Override
-  public RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws HiveSQLException {
-    assertState(OperationState.FINISHED);
-    validateDefaultFetchOrientation(orientation);
-    if (orientation.equals(FetchOrientation.FETCH_FIRST)) {
-      rowSet.setStartOffset(0);
-    }
-    return rowSet.extractSubset((int)maxRows);
-  }
-
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetTablesOperation.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetTablesOperation.java
deleted file mode 100644
index 2af17a662a296..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetTablesOperation.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.operation;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
-import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject;
-import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObjectUtils;
-import org.apache.hive.service.cli.FetchOrientation;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.OperationState;
-import org.apache.hive.service.cli.OperationType;
-import org.apache.hive.service.cli.RowSet;
-import org.apache.hive.service.cli.RowSetFactory;
-import org.apache.hive.service.cli.TableSchema;
-import org.apache.hive.service.cli.session.HiveSession;
-
-/**
- * GetTablesOperation.
- *
- */
-public class GetTablesOperation extends MetadataOperation {
-
-  private final String catalogName;
-  private final String schemaName;
-  private final String tableName;
-  private final List<String> tableTypes = new ArrayList<String>();
-  protected final RowSet rowSet;
-  private final TableTypeMapping tableTypeMapping;
-
-
-  private static final TableSchema RESULT_SET_SCHEMA = new TableSchema()
-  .addStringColumn("TABLE_CAT", "Catalog name. NULL if not applicable.")
-  .addStringColumn("TABLE_SCHEM", "Schema name.")
-  .addStringColumn("TABLE_NAME", "Table name.")
-  .addStringColumn("TABLE_TYPE", "The table type, e.g. \"TABLE\", \"VIEW\", etc.")
-  .addStringColumn("REMARKS", "Comments about the table.");
-
-  protected GetTablesOperation(HiveSession parentSession,
-      String catalogName, String schemaName, String tableName,
-      List<String> tableTypes) {
-    super(parentSession, OperationType.GET_TABLES);
-    this.catalogName = catalogName;
-    this.schemaName = schemaName;
-    this.tableName = tableName;
-    String tableMappingStr = getParentSession().getHiveConf()
-        .getVar(HiveConf.ConfVars.HIVE_SERVER2_TABLE_TYPE_MAPPING);
-    tableTypeMapping =
-        TableTypeMappingFactory.getTableTypeMapping(tableMappingStr);
-    if (tableTypes != null) {
-      this.tableTypes.addAll(tableTypes);
-    }
-    this.rowSet = RowSetFactory.create(RESULT_SET_SCHEMA, getProtocolVersion());
-  }
-
-  @Override
-  public void runInternal() throws HiveSQLException {
-    setState(OperationState.RUNNING);
-    try {
-      IMetaStoreClient metastoreClient = getParentSession().getMetaStoreClient();
-      String schemaPattern = convertSchemaPattern(schemaName);
-      List<String> matchingDbs = metastoreClient.getDatabases(schemaPattern);
-      if(isAuthV2Enabled()){
-        List<HivePrivilegeObject> privObjs = HivePrivilegeObjectUtils.getHivePrivDbObjects(matchingDbs);
-        String cmdStr = "catalog : " + catalogName + ", schemaPattern : " + schemaName;
-        authorizeMetaGets(HiveOperationType.GET_TABLES, privObjs, cmdStr);
-      }
-
-      String tablePattern = convertIdentifierPattern(tableName, true);
-      for (String dbName : metastoreClient.getDatabases(schemaPattern)) {
-        List<String> tableNames = metastoreClient.getTables(dbName, tablePattern);
-        for (Table table : metastoreClient.getTableObjectsByName(dbName, tableNames)) {
-          Object[] rowData = new Object[] {
-              DEFAULT_HIVE_CATALOG,
-              table.getDbName(),
-              table.getTableName(),
-              tableTypeMapping.mapToClientType(table.getTableType()),
-              table.getParameters().get("comment")
-              };
-          if (tableTypes.isEmpty() || tableTypes.contains(
-                tableTypeMapping.mapToClientType(table.getTableType()))) {
-            rowSet.addRow(rowData);
-          }
-        }
-      }
-      setState(OperationState.FINISHED);
-    } catch (Exception e) {
-      setState(OperationState.ERROR);
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.Operation#getResultSetSchema()
-   */
-  @Override
-  public TableSchema getResultSetSchema() throws HiveSQLException {
-    assertState(OperationState.FINISHED);
-    return RESULT_SET_SCHEMA;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.Operation#getNextRowSet(org.apache.hive.service.cli.FetchOrientation, long)
-   */
-  @Override
-  public RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws HiveSQLException {
-    assertState(OperationState.FINISHED);
-    validateDefaultFetchOrientation(orientation);
-    if (orientation.equals(FetchOrientation.FETCH_FIRST)) {
-      rowSet.setStartOffset(0);
-    }
-    return rowSet.extractSubset((int)maxRows);
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetTypeInfoOperation.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetTypeInfoOperation.java
deleted file mode 100644
index 3e81f8afbd85f..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetTypeInfoOperation.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.operation;
-
-import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
-import org.apache.hive.service.cli.FetchOrientation;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.OperationState;
-import org.apache.hive.service.cli.OperationType;
-import org.apache.hive.service.cli.RowSet;
-import org.apache.hive.service.cli.RowSetFactory;
-import org.apache.hive.service.cli.TableSchema;
-import org.apache.hive.service.cli.Type;
-import org.apache.hive.service.cli.session.HiveSession;
-
-/**
- * GetTypeInfoOperation.
- *
- */
-public class GetTypeInfoOperation extends MetadataOperation {
-
-  private static final TableSchema RESULT_SET_SCHEMA = new TableSchema()
-  .addPrimitiveColumn("TYPE_NAME", Type.STRING_TYPE,
-      "Type name")
-  .addPrimitiveColumn("DATA_TYPE", Type.INT_TYPE,
-      "SQL data type from java.sql.Types")
-  .addPrimitiveColumn("PRECISION", Type.INT_TYPE,
-      "Maximum precision")
-  .addPrimitiveColumn("LITERAL_PREFIX", Type.STRING_TYPE,
-      "Prefix used to quote a literal (may be null)")
-  .addPrimitiveColumn("LITERAL_SUFFIX", Type.STRING_TYPE,
-      "Suffix used to quote a literal (may be null)")
-  .addPrimitiveColumn("CREATE_PARAMS", Type.STRING_TYPE,
-      "Parameters used in creating the type (may be null)")
-  .addPrimitiveColumn("NULLABLE", Type.SMALLINT_TYPE,
-      "Can you use NULL for this type")
-  .addPrimitiveColumn("CASE_SENSITIVE", Type.BOOLEAN_TYPE,
-      "Is it case sensitive")
-  .addPrimitiveColumn("SEARCHABLE", Type.SMALLINT_TYPE,
-      "Can you use \"WHERE\" based on this type")
-  .addPrimitiveColumn("UNSIGNED_ATTRIBUTE", Type.BOOLEAN_TYPE,
-      "Is it unsigned")
-  .addPrimitiveColumn("FIXED_PREC_SCALE", Type.BOOLEAN_TYPE,
-      "Can it be a money value")
-  .addPrimitiveColumn("AUTO_INCREMENT", Type.BOOLEAN_TYPE,
-      "Can it be used for an auto-increment value")
-  .addPrimitiveColumn("LOCAL_TYPE_NAME", Type.STRING_TYPE,
-      "Localized version of type name (may be null)")
-  .addPrimitiveColumn("MINIMUM_SCALE", Type.SMALLINT_TYPE,
-      "Minimum scale supported")
-  .addPrimitiveColumn("MAXIMUM_SCALE", Type.SMALLINT_TYPE,
-      "Maximum scale supported")
-  .addPrimitiveColumn("SQL_DATA_TYPE", Type.INT_TYPE,
-      "Unused")
-  .addPrimitiveColumn("SQL_DATETIME_SUB", Type.INT_TYPE,
-      "Unused")
-  .addPrimitiveColumn("NUM_PREC_RADIX", Type.INT_TYPE,
-      "Usually 2 or 10");
-
-  protected final RowSet rowSet;
-
-  protected GetTypeInfoOperation(HiveSession parentSession) {
-    super(parentSession, OperationType.GET_TYPE_INFO);
-    rowSet = RowSetFactory.create(RESULT_SET_SCHEMA, getProtocolVersion());
-  }
-
-  @Override
-  public void runInternal() throws HiveSQLException {
-    setState(OperationState.RUNNING);
-    if (isAuthV2Enabled()) {
-      authorizeMetaGets(HiveOperationType.GET_TYPEINFO, null);
-    }
-    try {
-      for (Type type : Type.values()) {
-        Object[] rowData = new Object[] {
-            type.getName(), // TYPE_NAME
-            type.toJavaSQLType(), // DATA_TYPE
-            type.getMaxPrecision(), // PRECISION
-            type.getLiteralPrefix(), // LITERAL_PREFIX
-            type.getLiteralSuffix(), // LITERAL_SUFFIX
-            type.getCreateParams(), // CREATE_PARAMS
-            type.getNullable(), // NULLABLE
-            type.isCaseSensitive(), // CASE_SENSITIVE
-            type.getSearchable(), // SEARCHABLE
-            type.isUnsignedAttribute(), // UNSIGNED_ATTRIBUTE
-            type.isFixedPrecScale(), // FIXED_PREC_SCALE
-            type.isAutoIncrement(), // AUTO_INCREMENT
-            type.getLocalizedName(), // LOCAL_TYPE_NAME
-            type.getMinimumScale(), // MINIMUM_SCALE
-            type.getMaximumScale(), // MAXIMUM_SCALE
-            null, // SQL_DATA_TYPE, unused
-            null, // SQL_DATETIME_SUB, unused
-            type.getNumPrecRadix() //NUM_PREC_RADIX
-        };
-        rowSet.addRow(rowData);
-      }
-      setState(OperationState.FINISHED);
-    } catch (Exception e) {
-      setState(OperationState.ERROR);
-      throw new HiveSQLException(e);
-    }
-  }
-
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.Operation#getResultSetSchema()
-   */
-  @Override
-  public TableSchema getResultSetSchema() throws HiveSQLException {
-    assertState(OperationState.FINISHED);
-    return RESULT_SET_SCHEMA;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.Operation#getNextRowSet(org.apache.hive.service.cli.FetchOrientation, long)
-   */
-  @Override
-  public RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws HiveSQLException {
-    assertState(OperationState.FINISHED);
-    validateDefaultFetchOrientation(orientation);
-    if (orientation.equals(FetchOrientation.FETCH_FIRST)) {
-      rowSet.setStartOffset(0);
-    }
-    return rowSet.extractSubset((int)maxRows);
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java
deleted file mode 100644
index 5b6e6ad042412..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.operation;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.io.UnsupportedEncodingException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import org.apache.hadoop.hive.metastore.api.Schema;
-import org.apache.hadoop.hive.ql.processors.CommandProcessor;
-import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hive.service.cli.FetchOrientation;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.OperationState;
-import org.apache.hive.service.cli.RowSet;
-import org.apache.hive.service.cli.RowSetFactory;
-import org.apache.hive.service.cli.TableSchema;
-import org.apache.hive.service.cli.session.HiveSession;
-
-/**
- * Executes a HiveCommand
- */
-public class HiveCommandOperation extends ExecuteStatementOperation {
-  private CommandProcessor commandProcessor;
-  private TableSchema resultSchema = null;
-
-  /**
-   * For processors other than Hive queries (Driver), they output to session.out (a temp file)
-   * first and the fetchOne/fetchN/fetchAll functions get the output from pipeIn.
-   */
-  private BufferedReader resultReader;
-
-
-  protected HiveCommandOperation(HiveSession parentSession, String statement,
-      CommandProcessor commandProcessor, Map<String, String> confOverlay) {
-    super(parentSession, statement, confOverlay, false);
-    this.commandProcessor = commandProcessor;
-    setupSessionIO(parentSession.getSessionState());
-  }
-
-  private void setupSessionIO(SessionState sessionState) {
-    try {
-      LOG.info("Putting temp output to file " + sessionState.getTmpOutputFile().toString());
-      sessionState.in = null; // hive server's session input stream is not used
-      // open a per-session file in auto-flush mode for writing temp results
-      sessionState.out = new PrintStream(new FileOutputStream(sessionState.getTmpOutputFile()), true, UTF_8.name());
-      // TODO: for hadoop jobs, progress is printed out to session.err,
-      // we should find a way to feed back job progress to client
-      sessionState.err = new PrintStream(System.err, true, UTF_8.name());
-    } catch (IOException e) {
-      LOG.error("Error in creating temp output file ", e);
-      try {
-        sessionState.in = null;
-        sessionState.out = new PrintStream(System.out, true, UTF_8.name());
-        sessionState.err = new PrintStream(System.err, true, UTF_8.name());
-      } catch (UnsupportedEncodingException ee) {
-        LOG.error("Error creating PrintStream", e);
-        ee.printStackTrace();
-        sessionState.out = null;
-        sessionState.err = null;
-      }
-    }
-  }
-
-
-  private void tearDownSessionIO() {
-    IOUtils.cleanup(LOG, parentSession.getSessionState().out);
-    IOUtils.cleanup(LOG, parentSession.getSessionState().err);
-  }
-
-  @Override
-  public void runInternal() throws HiveSQLException {
-    setState(OperationState.RUNNING);
-    try {
-      String command = getStatement().trim();
-      String[] tokens = statement.split("\\s");
-      String commandArgs = command.substring(tokens[0].length()).trim();
-
-      CommandProcessorResponse response = commandProcessor.run(commandArgs);
-      int returnCode = response.getResponseCode();
-      if (returnCode != 0) {
-        throw toSQLException("Error while processing statement", response);
-      }
-      Schema schema = response.getSchema();
-      if (schema != null) {
-        setHasResultSet(true);
-        resultSchema = new TableSchema(schema);
-      } else {
-        setHasResultSet(false);
-        resultSchema = new TableSchema();
-      }
-    } catch (HiveSQLException e) {
-      setState(OperationState.ERROR);
-      throw e;
-    } catch (Exception e) {
-      setState(OperationState.ERROR);
-      throw new HiveSQLException("Error running query: " + e.toString(), e);
-    }
-    setState(OperationState.FINISHED);
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.operation.Operation#close()
-   */
-  @Override
-  public void close() throws HiveSQLException {
-    setState(OperationState.CLOSED);
-    tearDownSessionIO();
-    cleanTmpFile();
-    cleanupOperationLog();
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.operation.Operation#getResultSetSchema()
-   */
-  @Override
-  public TableSchema getResultSetSchema() throws HiveSQLException {
-    return resultSchema;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.operation.Operation#getNextRowSet(org.apache.hive.service.cli.FetchOrientation, long)
-   */
-  @Override
-  public RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws HiveSQLException {
-    validateDefaultFetchOrientation(orientation);
-    if (orientation.equals(FetchOrientation.FETCH_FIRST)) {
-      resetResultReader();
-    }
-    List<String> rows = readResults((int) maxRows);
-    RowSet rowSet = RowSetFactory.create(resultSchema, getProtocolVersion());
-
-    for (String row : rows) {
-      rowSet.addRow(new String[] {row});
-    }
-    return rowSet;
-  }
-
-  /**
-   * Reads the temporary results for non-Hive (non-Driver) commands to the
-   * resulting List of strings.
-   * @param nLines number of lines read at once. If it is <= 0, then read all lines.
-   */
-  private List<String> readResults(int nLines) throws HiveSQLException {
-    if (resultReader == null) {
-      SessionState sessionState = getParentSession().getSessionState();
-      File tmp = sessionState.getTmpOutputFile();
-      try {
-        resultReader = new BufferedReader(new FileReader(tmp));
-      } catch (FileNotFoundException e) {
-        LOG.error("File " + tmp + " not found. ", e);
-        throw new HiveSQLException(e);
-      }
-    }
-    List<String> results = new ArrayList<String>();
-
-    for (int i = 0; i < nLines || nLines <= 0; ++i) {
-      try {
-        String line = resultReader.readLine();
-        if (line == null) {
-          // reached the end of the result file
-          break;
-        } else {
-          results.add(line);
-        }
-      } catch (IOException e) {
-        LOG.error("Reading temp results encountered an exception: ", e);
-        throw new HiveSQLException(e);
-      }
-    }
-    return results;
-  }
-
-  private void cleanTmpFile() {
-    resetResultReader();
-    SessionState sessionState = getParentSession().getSessionState();
-    File tmp = sessionState.getTmpOutputFile();
-    tmp.delete();
-  }
-
-  private void resetResultReader() {
-    if (resultReader != null) {
-      IOUtils.cleanup(LOG, resultReader);
-      resultReader = null;
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/HiveTableTypeMapping.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/HiveTableTypeMapping.java
deleted file mode 100644
index b530f217125b8..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/HiveTableTypeMapping.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.operation;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.hadoop.hive.metastore.TableType;
-
-/**
- * HiveTableTypeMapping.
- * Default table type mapping
- *
- */
-public class HiveTableTypeMapping implements TableTypeMapping {
-
-  @Override
-  public String mapToHiveType(String clientTypeName) {
-    return clientTypeName;
-  }
-
-  @Override
-  public String mapToClientType(String hiveTypeName) {
-    return hiveTypeName;
-  }
-
-  @Override
-  public Set<String> getTableTypeNames() {
-    Set<String> typeNameSet = new HashSet<String>();
-    for (TableType typeNames : TableType.values()) {
-      typeNameSet.add(typeNames.toString());
-    }
-    return typeNameSet;
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/MetadataOperation.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/MetadataOperation.java
deleted file mode 100644
index 6c819876a556d..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/MetadataOperation.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.operation;
-
-import java.util.List;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAccessControlException;
-import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzContext;
-import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzPluginException;
-import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
-import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.OperationState;
-import org.apache.hive.service.cli.OperationType;
-import org.apache.hive.service.cli.TableSchema;
-import org.apache.hive.service.cli.session.HiveSession;
-
-/**
- * MetadataOperation.
- *
- */
-public abstract class MetadataOperation extends Operation {
-
-  protected static final String DEFAULT_HIVE_CATALOG = "";
-  protected static TableSchema RESULT_SET_SCHEMA;
-  private static final char SEARCH_STRING_ESCAPE = '\\';
-
-  protected MetadataOperation(HiveSession parentSession, OperationType opType) {
-    super(parentSession, opType, false);
-    setHasResultSet(true);
-  }
-
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.Operation#close()
-   */
-  @Override
-  public void close() throws HiveSQLException {
-    setState(OperationState.CLOSED);
-    cleanupOperationLog();
-  }
-
-  /**
-   * Convert wildchars and escape sequence from JDBC format to datanucleous/regex
-   */
-  protected String convertIdentifierPattern(final String pattern, boolean datanucleusFormat) {
-    if (pattern == null) {
-      return convertPattern("%", true);
-    } else {
-      return convertPattern(pattern, datanucleusFormat);
-    }
-  }
-
-  /**
-   * Convert wildchars and escape sequence of schema pattern from JDBC format to datanucleous/regex
-   * The schema pattern treats empty string also as wildchar
-   */
-  protected String convertSchemaPattern(final String pattern) {
-    if ((pattern == null) || pattern.isEmpty()) {
-      return convertPattern("%", true);
-    } else {
-      return convertPattern(pattern, true);
-    }
-  }
-
-  /**
-   * Convert a pattern containing JDBC catalog search wildcards into
-   * Java regex patterns.
-   *
-   * @param pattern input which may contain '%' or '_' wildcard characters, or
-   * these characters escaped using {@link #getSearchStringEscape()}.
-   * @return replace %/_ with regex search characters, also handle escaped
-   * characters.
-   *
-   * The datanucleus module expects the wildchar as '*'. The columns search on the
-   * other hand is done locally inside the hive code and that requires the regex wildchar
-   * format '.*'  This is driven by the datanucleusFormat flag.
-   */
-  private String convertPattern(final String pattern, boolean datanucleusFormat) {
-    String wStr;
-    if (datanucleusFormat) {
-      wStr = "*";
-    } else {
-      wStr = ".*";
-    }
-    return pattern
-        .replaceAll("([^\\\\])%", "$1" + wStr).replaceAll("\\\\%", "%").replaceAll("^%", wStr)
-        .replaceAll("([^\\\\])_", "$1.").replaceAll("\\\\_", "_").replaceAll("^_", ".");
-  }
-
-  protected boolean isAuthV2Enabled(){
-    SessionState ss = SessionState.get();
-    return (ss.isAuthorizationModeV2() &&
-        HiveConf.getBoolVar(ss.getConf(), HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED));
-  }
-
-  protected void authorizeMetaGets(HiveOperationType opType, List<HivePrivilegeObject> inpObjs)
-      throws HiveSQLException {
-    authorizeMetaGets(opType, inpObjs, null);
-  }
-
-  protected void authorizeMetaGets(HiveOperationType opType, List<HivePrivilegeObject> inpObjs,
-      String cmdString) throws HiveSQLException {
-    SessionState ss = SessionState.get();
-    HiveAuthzContext.Builder ctxBuilder = new HiveAuthzContext.Builder();
-    ctxBuilder.setUserIpAddress(ss.getUserIpAddress());
-    ctxBuilder.setCommandString(cmdString);
-    try {
-      ss.getAuthorizerV2().checkPrivileges(opType, inpObjs, null,
-          ctxBuilder.build());
-    } catch (HiveAuthzPluginException | HiveAccessControlException e) {
-      throw new HiveSQLException(e.getMessage(), e);
-    }
-  }
-
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/Operation.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/Operation.java
deleted file mode 100644
index 51bb28748d9e2..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/Operation.java
+++ /dev/null
@@ -1,325 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.service.cli.operation;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.util.EnumSet;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
-import org.apache.hadoop.hive.ql.session.OperationLog;
-import org.apache.hive.service.cli.FetchOrientation;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.OperationHandle;
-import org.apache.hive.service.cli.OperationState;
-import org.apache.hive.service.cli.OperationStatus;
-import org.apache.hive.service.cli.OperationType;
-import org.apache.hive.service.cli.RowSet;
-import org.apache.hive.service.cli.TableSchema;
-import org.apache.hive.service.cli.session.HiveSession;
-import org.apache.hive.service.cli.thrift.TProtocolVersion;
-
-public abstract class Operation {
-  protected final HiveSession parentSession;
-  private OperationState state = OperationState.INITIALIZED;
-  private final OperationHandle opHandle;
-  private HiveConf configuration;
-  public static final Log LOG = LogFactory.getLog(Operation.class.getName());
-  public static final FetchOrientation DEFAULT_FETCH_ORIENTATION = FetchOrientation.FETCH_NEXT;
-  public static final long DEFAULT_FETCH_MAX_ROWS = 100;
-  protected boolean hasResultSet;
-  protected volatile HiveSQLException operationException;
-  protected final boolean runAsync;
-  protected volatile Future<?> backgroundHandle;
-  protected OperationLog operationLog;
-  protected boolean isOperationLogEnabled;
-
-  private long operationTimeout;
-  private long lastAccessTime;
-
-  protected static final EnumSet<FetchOrientation> DEFAULT_FETCH_ORIENTATION_SET =
-      EnumSet.of(
-          FetchOrientation.FETCH_NEXT,
-          FetchOrientation.FETCH_FIRST,
-          FetchOrientation.FETCH_PRIOR);
-
-  protected Operation(HiveSession parentSession, OperationType opType, boolean runInBackground) {
-    this.parentSession = parentSession;
-    this.runAsync = runInBackground;
-    this.opHandle = new OperationHandle(opType, parentSession.getProtocolVersion());
-    lastAccessTime = System.currentTimeMillis();
-    operationTimeout = HiveConf.getTimeVar(parentSession.getHiveConf(),
-        HiveConf.ConfVars.HIVE_SERVER2_IDLE_OPERATION_TIMEOUT, TimeUnit.MILLISECONDS);
-  }
-
-  public Future<?> getBackgroundHandle() {
-    return backgroundHandle;
-  }
-
-  protected void setBackgroundHandle(Future<?> backgroundHandle) {
-    this.backgroundHandle = backgroundHandle;
-  }
-
-  public boolean shouldRunAsync() {
-    return runAsync;
-  }
-
-  public void setConfiguration(HiveConf configuration) {
-    this.configuration = new HiveConf(configuration);
-  }
-
-  public HiveConf getConfiguration() {
-    return new HiveConf(configuration);
-  }
-
-  public HiveSession getParentSession() {
-    return parentSession;
-  }
-
-  public OperationHandle getHandle() {
-    return opHandle;
-  }
-
-  public TProtocolVersion getProtocolVersion() {
-    return opHandle.getProtocolVersion();
-  }
-
-  public OperationType getType() {
-    return opHandle.getOperationType();
-  }
-
-  public OperationStatus getStatus() {
-    return new OperationStatus(state, operationException);
-  }
-
-  public boolean hasResultSet() {
-    return hasResultSet;
-  }
-
-  protected void setHasResultSet(boolean hasResultSet) {
-    this.hasResultSet = hasResultSet;
-    opHandle.setHasResultSet(hasResultSet);
-  }
-
-  public OperationLog getOperationLog() {
-    return operationLog;
-  }
-
-  protected final OperationState setState(OperationState newState) throws HiveSQLException {
-    state.validateTransition(newState);
-    this.state = newState;
-    this.lastAccessTime = System.currentTimeMillis();
-    return this.state;
-  }
-
-  public boolean isTimedOut(long current) {
-    if (operationTimeout == 0) {
-      return false;
-    }
-    if (operationTimeout > 0) {
-      // check only when it's in terminal state
-      return state.isTerminal() && lastAccessTime + operationTimeout <= current;
-    }
-    return lastAccessTime + -operationTimeout <= current;
-  }
-
-  public long getLastAccessTime() {
-    return lastAccessTime;
-  }
-
-  public long getOperationTimeout() {
-    return operationTimeout;
-  }
-
-  public void setOperationTimeout(long operationTimeout) {
-    this.operationTimeout = operationTimeout;
-  }
-
-  protected void setOperationException(HiveSQLException operationException) {
-    this.operationException = operationException;
-  }
-
-  protected final void assertState(OperationState state) throws HiveSQLException {
-    if (this.state != state) {
-      throw new HiveSQLException("Expected state " + state + ", but found " + this.state);
-    }
-    this.lastAccessTime = System.currentTimeMillis();
-  }
-
-  public boolean isRunning() {
-    return OperationState.RUNNING.equals(state);
-  }
-
-  public boolean isFinished() {
-    return OperationState.FINISHED.equals(state);
-  }
-
-  public boolean isCanceled() {
-    return OperationState.CANCELED.equals(state);
-  }
-
-  public boolean isFailed() {
-    return OperationState.ERROR.equals(state);
-  }
-
-  protected void createOperationLog() {
-    if (parentSession.isOperationLogEnabled()) {
-      File operationLogFile = new File(parentSession.getOperationLogSessionDir(),
-          opHandle.getHandleIdentifier().toString());
-      isOperationLogEnabled = true;
-
-      // create log file
-      try {
-        if (operationLogFile.exists()) {
-          LOG.warn("The operation log file should not exist, but it is already there: " +
-              operationLogFile.getAbsolutePath());
-          operationLogFile.delete();
-        }
-        if (!operationLogFile.createNewFile()) {
-          // the log file already exists and cannot be deleted.
-          // If it can be read/written, keep its contents and use it.
-          if (!operationLogFile.canRead() || !operationLogFile.canWrite()) {
-            LOG.warn("The already existed operation log file cannot be recreated, " +
-                "and it cannot be read or written: " + operationLogFile.getAbsolutePath());
-            isOperationLogEnabled = false;
-            return;
-          }
-        }
-      } catch (Exception e) {
-        LOG.warn("Unable to create operation log file: " + operationLogFile.getAbsolutePath(), e);
-        isOperationLogEnabled = false;
-        return;
-      }
-
-      // create OperationLog object with above log file
-      try {
-        operationLog = new OperationLog(opHandle.toString(), operationLogFile, parentSession.getHiveConf());
-      } catch (FileNotFoundException e) {
-        LOG.warn("Unable to instantiate OperationLog object for operation: " +
-            opHandle, e);
-        isOperationLogEnabled = false;
-        return;
-      }
-
-      // register this operationLog to current thread
-      OperationLog.setCurrentOperationLog(operationLog);
-    }
-  }
-
-  protected void unregisterOperationLog() {
-    if (isOperationLogEnabled) {
-      OperationLog.removeCurrentOperationLog();
-    }
-  }
-
-  /**
-   * Invoked before runInternal().
-   * Set up some preconditions, or configurations.
-   */
-  protected void beforeRun() {
-    createOperationLog();
-  }
-
-  /**
-   * Invoked after runInternal(), even if an exception is thrown in runInternal().
-   * Clean up resources, which was set up in beforeRun().
-   */
-  protected void afterRun() {
-    unregisterOperationLog();
-  }
-
-  /**
-   * Implemented by subclass of Operation class to execute specific behaviors.
-   * @throws HiveSQLException
-   */
-  protected abstract void runInternal() throws HiveSQLException;
-
-  public void run() throws HiveSQLException {
-    beforeRun();
-    try {
-      runInternal();
-    } finally {
-      afterRun();
-    }
-  }
-
-  protected void cleanupOperationLog() {
-    if (isOperationLogEnabled) {
-      if (operationLog == null) {
-        LOG.error("Operation [ " + opHandle.getHandleIdentifier() + " ] "
-          + "logging is enabled, but its OperationLog object cannot be found.");
-      } else {
-        operationLog.close();
-      }
-    }
-  }
-
-  // TODO: make this abstract and implement in subclasses.
-  public void cancel() throws HiveSQLException {
-    setState(OperationState.CANCELED);
-    throw new UnsupportedOperationException("SQLOperation.cancel()");
-  }
-
-  public abstract void close() throws HiveSQLException;
-
-  public abstract TableSchema getResultSetSchema() throws HiveSQLException;
-
-  public abstract RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws HiveSQLException;
-
-  public RowSet getNextRowSet() throws HiveSQLException {
-    return getNextRowSet(FetchOrientation.FETCH_NEXT, DEFAULT_FETCH_MAX_ROWS);
-  }
-
-  /**
-   * Verify if the given fetch orientation is part of the default orientation types.
-   * @param orientation
-   * @throws HiveSQLException
-   */
-  protected void validateDefaultFetchOrientation(FetchOrientation orientation)
-      throws HiveSQLException {
-    validateFetchOrientation(orientation, DEFAULT_FETCH_ORIENTATION_SET);
-  }
-
-  /**
-   * Verify if the given fetch orientation is part of the supported orientation types.
-   * @param orientation
-   * @param supportedOrientations
-   * @throws HiveSQLException
-   */
-  protected void validateFetchOrientation(FetchOrientation orientation,
-      EnumSet<FetchOrientation> supportedOrientations) throws HiveSQLException {
-    if (!supportedOrientations.contains(orientation)) {
-      throw new HiveSQLException("The fetch type " + orientation.toString() +
-          " is not supported for this resultset", "HY106");
-    }
-  }
-
-  protected HiveSQLException toSQLException(String prefix, CommandProcessorResponse response) {
-    HiveSQLException ex = new HiveSQLException(prefix + ": " + response.getErrorMessage(),
-        response.getSQLState(), response.getResponseCode());
-    if (response.getException() != null) {
-      ex.initCause(response.getException());
-    }
-    return ex;
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/OperationManager.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/OperationManager.java
deleted file mode 100644
index 92c340a29c107..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/OperationManager.java
+++ /dev/null
@@ -1,284 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.operation;
-
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Schema;
-import org.apache.hadoop.hive.ql.session.OperationLog;
-import org.apache.hive.service.AbstractService;
-import org.apache.hive.service.cli.FetchOrientation;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.OperationHandle;
-import org.apache.hive.service.cli.OperationState;
-import org.apache.hive.service.cli.OperationStatus;
-import org.apache.hive.service.cli.RowSet;
-import org.apache.hive.service.cli.RowSetFactory;
-import org.apache.hive.service.cli.TableSchema;
-import org.apache.hive.service.cli.session.HiveSession;
-import org.apache.log4j.Appender;
-import org.apache.log4j.Logger;
-
-/**
- * OperationManager.
- *
- */
-public class OperationManager extends AbstractService {
-  private final Log LOG = LogFactory.getLog(OperationManager.class.getName());
-
-  private final Map<OperationHandle, Operation> handleToOperation =
-      new HashMap<OperationHandle, Operation>();
-
-  public OperationManager() {
-    super(OperationManager.class.getSimpleName());
-  }
-
-  @Override
-  public synchronized void init(HiveConf hiveConf) {
-    if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_ENABLED)) {
-      initOperationLogCapture(hiveConf.getVar(
-        HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL));
-    } else {
-      LOG.debug("Operation level logging is turned off");
-    }
-    super.init(hiveConf);
-  }
-
-  @Override
-  public synchronized void start() {
-    super.start();
-    // TODO
-  }
-
-  @Override
-  public synchronized void stop() {
-    // TODO
-    super.stop();
-  }
-
-  private void initOperationLogCapture(String loggingMode) {
-    // Register another Appender (with the same layout) that talks to us.
-    Appender ap = new LogDivertAppender(this, OperationLog.getLoggingLevel(loggingMode));
-    Logger.getRootLogger().addAppender(ap);
-  }
-
-  public ExecuteStatementOperation newExecuteStatementOperation(HiveSession parentSession,
-      String statement, Map<String, String> confOverlay, boolean runAsync)
-          throws HiveSQLException {
-    ExecuteStatementOperation executeStatementOperation = ExecuteStatementOperation
-        .newExecuteStatementOperation(parentSession, statement, confOverlay, runAsync);
-    addOperation(executeStatementOperation);
-    return executeStatementOperation;
-  }
-
-  public GetTypeInfoOperation newGetTypeInfoOperation(HiveSession parentSession) {
-    GetTypeInfoOperation operation = new GetTypeInfoOperation(parentSession);
-    addOperation(operation);
-    return operation;
-  }
-
-  public GetCatalogsOperation newGetCatalogsOperation(HiveSession parentSession) {
-    GetCatalogsOperation operation = new GetCatalogsOperation(parentSession);
-    addOperation(operation);
-    return operation;
-  }
-
-  public GetSchemasOperation newGetSchemasOperation(HiveSession parentSession,
-      String catalogName, String schemaName) {
-    GetSchemasOperation operation = new GetSchemasOperation(parentSession, catalogName, schemaName);
-    addOperation(operation);
-    return operation;
-  }
-
-  public MetadataOperation newGetTablesOperation(HiveSession parentSession,
-      String catalogName, String schemaName, String tableName,
-      List<String> tableTypes) {
-    MetadataOperation operation =
-        new GetTablesOperation(parentSession, catalogName, schemaName, tableName, tableTypes);
-    addOperation(operation);
-    return operation;
-  }
-
-  public GetTableTypesOperation newGetTableTypesOperation(HiveSession parentSession) {
-    GetTableTypesOperation operation = new GetTableTypesOperation(parentSession);
-    addOperation(operation);
-    return operation;
-  }
-
-  public GetColumnsOperation newGetColumnsOperation(HiveSession parentSession,
-      String catalogName, String schemaName, String tableName, String columnName) {
-    GetColumnsOperation operation = new GetColumnsOperation(parentSession,
-        catalogName, schemaName, tableName, columnName);
-    addOperation(operation);
-    return operation;
-  }
-
-  public GetFunctionsOperation newGetFunctionsOperation(HiveSession parentSession,
-      String catalogName, String schemaName, String functionName) {
-    GetFunctionsOperation operation = new GetFunctionsOperation(parentSession,
-        catalogName, schemaName, functionName);
-    addOperation(operation);
-    return operation;
-  }
-
-  public Operation getOperation(OperationHandle operationHandle) throws HiveSQLException {
-    Operation operation = getOperationInternal(operationHandle);
-    if (operation == null) {
-      throw new HiveSQLException("Invalid OperationHandle: " + operationHandle);
-    }
-    return operation;
-  }
-
-  private synchronized Operation getOperationInternal(OperationHandle operationHandle) {
-    return handleToOperation.get(operationHandle);
-  }
-
-  private synchronized Operation removeTimedOutOperation(OperationHandle operationHandle) {
-    Operation operation = handleToOperation.get(operationHandle);
-    if (operation != null && operation.isTimedOut(System.currentTimeMillis())) {
-      handleToOperation.remove(operationHandle);
-      return operation;
-    }
-    return null;
-  }
-
-  private synchronized void addOperation(Operation operation) {
-    handleToOperation.put(operation.getHandle(), operation);
-  }
-
-  private synchronized Operation removeOperation(OperationHandle opHandle) {
-    return handleToOperation.remove(opHandle);
-  }
-
-  public OperationStatus getOperationStatus(OperationHandle opHandle)
-      throws HiveSQLException {
-    return getOperation(opHandle).getStatus();
-  }
-
-  public void cancelOperation(OperationHandle opHandle) throws HiveSQLException {
-    Operation operation = getOperation(opHandle);
-    OperationState opState = operation.getStatus().getState();
-    if (opState == OperationState.CANCELED ||
-        opState == OperationState.CLOSED ||
-        opState == OperationState.FINISHED ||
-        opState == OperationState.ERROR ||
-        opState == OperationState.UNKNOWN) {
-      // Cancel should be a no-op in either cases
-      LOG.debug(opHandle + ": Operation is already aborted in state - " + opState);
-    }
-    else {
-      LOG.debug(opHandle + ": Attempting to cancel from state - " + opState);
-      operation.cancel();
-    }
-  }
-
-  public void closeOperation(OperationHandle opHandle) throws HiveSQLException {
-    Operation operation = removeOperation(opHandle);
-    if (operation == null) {
-      throw new HiveSQLException("Operation does not exist!");
-    }
-    operation.close();
-  }
-
-  public TableSchema getOperationResultSetSchema(OperationHandle opHandle)
-      throws HiveSQLException {
-    return getOperation(opHandle).getResultSetSchema();
-  }
-
-  public RowSet getOperationNextRowSet(OperationHandle opHandle)
-      throws HiveSQLException {
-    return getOperation(opHandle).getNextRowSet();
-  }
-
-  public RowSet getOperationNextRowSet(OperationHandle opHandle,
-      FetchOrientation orientation, long maxRows)
-          throws HiveSQLException {
-    return getOperation(opHandle).getNextRowSet(orientation, maxRows);
-  }
-
-  public RowSet getOperationLogRowSet(OperationHandle opHandle,
-      FetchOrientation orientation, long maxRows)
-          throws HiveSQLException {
-    // get the OperationLog object from the operation
-    OperationLog operationLog = getOperation(opHandle).getOperationLog();
-    if (operationLog == null) {
-      throw new HiveSQLException("Couldn't find log associated with operation handle: " + opHandle);
-    }
-
-    // read logs
-    List<String> logs;
-    try {
-      logs = operationLog.readOperationLog(isFetchFirst(orientation), maxRows);
-    } catch (SQLException e) {
-      throw new HiveSQLException(e.getMessage(), e.getCause());
-    }
-
-
-    // convert logs to RowSet
-    TableSchema tableSchema = new TableSchema(getLogSchema());
-    RowSet rowSet = RowSetFactory.create(tableSchema, getOperation(opHandle).getProtocolVersion());
-    for (String log : logs) {
-      rowSet.addRow(new String[] {log});
-    }
-
-    return rowSet;
-  }
-
-  private boolean isFetchFirst(FetchOrientation fetchOrientation) {
-    //TODO: Since OperationLog is moved to package o.a.h.h.ql.session,
-    // we may add a Enum there and map FetchOrientation to it.
-    if (fetchOrientation.equals(FetchOrientation.FETCH_FIRST)) {
-      return true;
-    }
-    return false;
-  }
-
-  private Schema getLogSchema() {
-    Schema schema = new Schema();
-    FieldSchema fieldSchema = new FieldSchema();
-    fieldSchema.setName("operation_log");
-    fieldSchema.setType("string");
-    schema.addToFieldSchemas(fieldSchema);
-    return schema;
-  }
-
-  public OperationLog getOperationLogByThread() {
-    return OperationLog.getCurrentOperationLog();
-  }
-
-  public List<Operation> removeExpiredOperations(OperationHandle[] handles) {
-    List<Operation> removed = new ArrayList<Operation>();
-    for (OperationHandle handle : handles) {
-      Operation operation = removeTimedOutOperation(handle);
-      if (operation != null) {
-        LOG.warn("Operation " + handle + " is timed-out and will be closed");
-        removed.add(operation);
-      }
-    }
-    return removed;
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/SQLOperation.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/SQLOperation.java
deleted file mode 100644
index c7726f1fac07a..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/SQLOperation.java
+++ /dev/null
@@ -1,456 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.operation;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.nio.charset.StandardCharsets;
-import java.security.PrivilegedExceptionAction;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.Future;
-import java.util.concurrent.RejectedExecutionException;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Schema;
-import org.apache.hadoop.hive.ql.CommandNeedRetryException;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.exec.ExplainTask;
-import org.apache.hadoop.hive.ql.exec.Task;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.parse.VariableSubstitution;
-import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.hive.serde2.SerDe;
-import org.apache.hadoop.hive.serde2.SerDeException;
-import org.apache.hadoop.hive.serde2.SerDeUtils;
-import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructField;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.shims.Utils;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hive.service.cli.FetchOrientation;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.OperationState;
-import org.apache.hive.service.cli.RowSet;
-import org.apache.hive.service.cli.RowSetFactory;
-import org.apache.hive.service.cli.TableSchema;
-import org.apache.hive.service.cli.session.HiveSession;
-import org.apache.hive.service.server.ThreadWithGarbageCleanup;
-
-/**
- * SQLOperation.
- *
- */
-public class SQLOperation extends ExecuteStatementOperation {
-
-  private Driver driver = null;
-  private CommandProcessorResponse response;
-  private TableSchema resultSchema = null;
-  private Schema mResultSchema = null;
-  private SerDe serde = null;
-  private boolean fetchStarted = false;
-
-  public SQLOperation(HiveSession parentSession, String statement, Map<String,
-      String> confOverlay, boolean runInBackground) {
-    // TODO: call setRemoteUser in ExecuteStatementOperation or higher.
-    super(parentSession, statement, confOverlay, runInBackground);
-  }
-
-  /***
-   * Compile the query and extract metadata
-   * @param sqlOperationConf
-   * @throws HiveSQLException
-   */
-  public void prepare(HiveConf sqlOperationConf) throws HiveSQLException {
-    setState(OperationState.RUNNING);
-
-    try {
-      driver = new Driver(sqlOperationConf, getParentSession().getUserName());
-
-      // set the operation handle information in Driver, so that thrift API users
-      // can use the operation handle they receive, to lookup query information in
-      // Yarn ATS
-      String guid64 = Base64.encodeBase64URLSafeString(getHandle().getHandleIdentifier()
-          .toTHandleIdentifier().getGuid()).trim();
-      driver.setOperationId(guid64);
-
-      // In Hive server mode, we are not able to retry in the FetchTask
-      // case, when calling fetch queries since execute() has returned.
-      // For now, we disable the test attempts.
-      driver.setTryCount(Integer.MAX_VALUE);
-
-      String subStatement = new VariableSubstitution().substitute(sqlOperationConf, statement);
-      response = driver.compileAndRespond(subStatement);
-      if (0 != response.getResponseCode()) {
-        throw toSQLException("Error while compiling statement", response);
-      }
-
-      mResultSchema = driver.getSchema();
-
-      // hasResultSet should be true only if the query has a FetchTask
-      // "explain" is an exception for now
-      if(driver.getPlan().getFetchTask() != null) {
-        //Schema has to be set
-        if (mResultSchema == null || !mResultSchema.isSetFieldSchemas()) {
-          throw new HiveSQLException("Error compiling query: Schema and FieldSchema " +
-              "should be set when query plan has a FetchTask");
-        }
-        resultSchema = new TableSchema(mResultSchema);
-        setHasResultSet(true);
-      } else {
-        setHasResultSet(false);
-      }
-      // Set hasResultSet true if the plan has ExplainTask
-      // TODO explain should use a FetchTask for reading
-      for (Task<? extends Serializable> task: driver.getPlan().getRootTasks()) {
-        if (task.getClass() == ExplainTask.class) {
-          resultSchema = new TableSchema(mResultSchema);
-          setHasResultSet(true);
-          break;
-        }
-      }
-    } catch (HiveSQLException e) {
-      setState(OperationState.ERROR);
-      throw e;
-    } catch (Exception e) {
-      setState(OperationState.ERROR);
-      throw new HiveSQLException("Error running query: " + e.toString(), e);
-    }
-  }
-
-  private void runQuery(HiveConf sqlOperationConf) throws HiveSQLException {
-    try {
-      // In Hive server mode, we are not able to retry in the FetchTask
-      // case, when calling fetch queries since execute() has returned.
-      // For now, we disable the test attempts.
-      driver.setTryCount(Integer.MAX_VALUE);
-      response = driver.run();
-      if (0 != response.getResponseCode()) {
-        throw toSQLException("Error while processing statement", response);
-      }
-    } catch (HiveSQLException e) {
-      // If the operation was cancelled by another thread,
-      // Driver#run will return a non-zero response code.
-      // We will simply return if the operation state is CANCELED,
-      // otherwise throw an exception
-      if (getStatus().getState() == OperationState.CANCELED) {
-        return;
-      }
-      else {
-        setState(OperationState.ERROR);
-        throw e;
-      }
-    } catch (Exception e) {
-      setState(OperationState.ERROR);
-      throw new HiveSQLException("Error running query: " + e.toString(), e);
-    }
-    setState(OperationState.FINISHED);
-  }
-
-  @Override
-  public void runInternal() throws HiveSQLException {
-    setState(OperationState.PENDING);
-    final HiveConf opConfig = getConfigForOperation();
-    prepare(opConfig);
-    if (!shouldRunAsync()) {
-      runQuery(opConfig);
-    } else {
-      // We'll pass ThreadLocals in the background thread from the foreground (handler) thread
-      final SessionState parentSessionState = SessionState.get();
-      // ThreadLocal Hive object needs to be set in background thread.
-      // The metastore client in Hive is associated with right user.
-      final Hive parentHive = getSessionHive();
-      // Current UGI will get used by metastore when metsatore is in embedded mode
-      // So this needs to get passed to the new background thread
-      final UserGroupInformation currentUGI = getCurrentUGI(opConfig);
-      // Runnable impl to call runInternal asynchronously,
-      // from a different thread
-      Runnable backgroundOperation = new Runnable() {
-        @Override
-        public void run() {
-          PrivilegedExceptionAction<Object> doAsAction = new PrivilegedExceptionAction<Object>() {
-            @Override
-            public Object run() throws HiveSQLException {
-              Hive.set(parentHive);
-              SessionState.setCurrentSessionState(parentSessionState);
-              // Set current OperationLog in this async thread for keeping on saving query log.
-              registerCurrentOperationLog();
-              try {
-                runQuery(opConfig);
-              } catch (HiveSQLException e) {
-                setOperationException(e);
-                LOG.error("Error running hive query: ", e);
-              } finally {
-                unregisterOperationLog();
-              }
-              return null;
-            }
-          };
-
-          try {
-            currentUGI.doAs(doAsAction);
-          } catch (Exception e) {
-            setOperationException(new HiveSQLException(e));
-            LOG.error("Error running hive query as user : " + currentUGI.getShortUserName(), e);
-          }
-          finally {
-            /**
-             * We'll cache the ThreadLocal RawStore object for this background thread for an orderly cleanup
-             * when this thread is garbage collected later.
-             * @see org.apache.hive.service.server.ThreadWithGarbageCleanup#finalize()
-             */
-            if (ThreadWithGarbageCleanup.currentThread() instanceof ThreadWithGarbageCleanup) {
-              ThreadWithGarbageCleanup currentThread =
-                  (ThreadWithGarbageCleanup) ThreadWithGarbageCleanup.currentThread();
-              currentThread.cacheThreadLocalRawStore();
-            }
-          }
-        }
-      };
-      try {
-        // This submit blocks if no background threads are available to run this operation
-        Future<?> backgroundHandle =
-            getParentSession().getSessionManager().submitBackgroundOperation(backgroundOperation);
-        setBackgroundHandle(backgroundHandle);
-      } catch (RejectedExecutionException rejected) {
-        setState(OperationState.ERROR);
-        throw new HiveSQLException("The background threadpool cannot accept" +
-            " new task for execution, please retry the operation", rejected);
-      }
-    }
-  }
-
-  /**
-   * Returns the current UGI on the stack
-   * @param opConfig
-   * @return UserGroupInformation
-   * @throws HiveSQLException
-   */
-  private UserGroupInformation getCurrentUGI(HiveConf opConfig) throws HiveSQLException {
-    try {
-      return Utils.getUGI();
-    } catch (Exception e) {
-      throw new HiveSQLException("Unable to get current user", e);
-    }
-  }
-
-  /**
-   * Returns the ThreadLocal Hive for the current thread
-   * @return Hive
-   * @throws HiveSQLException
-   */
-  private Hive getSessionHive() throws HiveSQLException {
-    try {
-      return Hive.get();
-    } catch (HiveException e) {
-      throw new HiveSQLException("Failed to get ThreadLocal Hive object", e);
-    }
-  }
-
-  private void cleanup(OperationState state) throws HiveSQLException {
-    setState(state);
-    if (shouldRunAsync()) {
-      Future<?> backgroundHandle = getBackgroundHandle();
-      if (backgroundHandle != null) {
-        backgroundHandle.cancel(true);
-      }
-    }
-    if (driver != null) {
-      driver.close();
-      driver.destroy();
-    }
-    driver = null;
-
-    SessionState ss = SessionState.get();
-    if (ss.getTmpOutputFile() != null) {
-      ss.getTmpOutputFile().delete();
-    }
-  }
-
-  @Override
-  public void cancel() throws HiveSQLException {
-    cleanup(OperationState.CANCELED);
-  }
-
-  @Override
-  public void close() throws HiveSQLException {
-    cleanup(OperationState.CLOSED);
-    cleanupOperationLog();
-  }
-
-  @Override
-  public TableSchema getResultSetSchema() throws HiveSQLException {
-    assertState(OperationState.FINISHED);
-    if (resultSchema == null) {
-      resultSchema = new TableSchema(driver.getSchema());
-    }
-    return resultSchema;
-  }
-
-  private final transient List<Object> convey = new ArrayList<Object>();
-
-  @Override
-  public RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws HiveSQLException {
-    validateDefaultFetchOrientation(orientation);
-    assertState(OperationState.FINISHED);
-
-    RowSet rowSet = RowSetFactory.create(resultSchema, getProtocolVersion());
-
-    try {
-      /* if client is requesting fetch-from-start and its not the first time reading from this operation
-       * then reset the fetch position to beginning
-       */
-      if (orientation.equals(FetchOrientation.FETCH_FIRST) && fetchStarted) {
-        driver.resetFetch();
-      }
-      fetchStarted = true;
-      driver.setMaxRows((int) maxRows);
-      if (driver.getResults(convey)) {
-        return decode(convey, rowSet);
-      }
-      return rowSet;
-    } catch (IOException e) {
-      throw new HiveSQLException(e);
-    } catch (CommandNeedRetryException e) {
-      throw new HiveSQLException(e);
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    } finally {
-      convey.clear();
-    }
-  }
-
-  private RowSet decode(List<Object> rows, RowSet rowSet) throws Exception {
-    if (driver.isFetchingTable()) {
-      return prepareFromRow(rows, rowSet);
-    }
-    return decodeFromString(rows, rowSet);
-  }
-
-  // already encoded to thrift-able object in ThriftFormatter
-  private RowSet prepareFromRow(List<Object> rows, RowSet rowSet) throws Exception {
-    for (Object row : rows) {
-      rowSet.addRow((Object[]) row);
-    }
-    return rowSet;
-  }
-
-  private RowSet decodeFromString(List<Object> rows, RowSet rowSet)
-      throws SQLException, SerDeException {
-    getSerDe();
-    StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector();
-    List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs();
-
-    Object[] deserializedFields = new Object[fieldRefs.size()];
-    Object rowObj;
-    ObjectInspector fieldOI;
-
-    int protocol = getProtocolVersion().getValue();
-    for (Object rowString : rows) {
-      rowObj = serde.deserialize(new BytesWritable(((String)rowString).getBytes(StandardCharsets.UTF_8)));
-      for (int i = 0; i < fieldRefs.size(); i++) {
-        StructField fieldRef = fieldRefs.get(i);
-        fieldOI = fieldRef.getFieldObjectInspector();
-        Object fieldData = soi.getStructFieldData(rowObj, fieldRef);
-        deserializedFields[i] = SerDeUtils.toThriftPayload(fieldData, fieldOI, protocol);
-      }
-      rowSet.addRow(deserializedFields);
-    }
-    return rowSet;
-  }
-
-  private SerDe getSerDe() throws SQLException {
-    if (serde != null) {
-      return serde;
-    }
-    try {
-      List<FieldSchema> fieldSchemas = mResultSchema.getFieldSchemas();
-      StringBuilder namesSb = new StringBuilder();
-      StringBuilder typesSb = new StringBuilder();
-
-      if (fieldSchemas != null && !fieldSchemas.isEmpty()) {
-        for (int pos = 0; pos < fieldSchemas.size(); pos++) {
-          if (pos != 0) {
-            namesSb.append(",");
-            typesSb.append(",");
-          }
-          namesSb.append(fieldSchemas.get(pos).getName());
-          typesSb.append(fieldSchemas.get(pos).getType());
-        }
-      }
-      String names = namesSb.toString();
-      String types = typesSb.toString();
-
-      serde = new LazySimpleSerDe();
-      Properties props = new Properties();
-      if (names.length() > 0) {
-        LOG.debug("Column names: " + names);
-        props.setProperty(serdeConstants.LIST_COLUMNS, names);
-      }
-      if (types.length() > 0) {
-        LOG.debug("Column types: " + types);
-        props.setProperty(serdeConstants.LIST_COLUMN_TYPES, types);
-      }
-      SerDeUtils.initializeSerDe(serde, new HiveConf(), props, null);
-
-    } catch (Exception ex) {
-      ex.printStackTrace();
-      throw new SQLException("Could not create ResultSet: " + ex.getMessage(), ex);
-    }
-    return serde;
-  }
-
-  /**
-   * If there are query specific settings to overlay, then create a copy of config
-   * There are two cases we need to clone the session config that's being passed to hive driver
-   * 1. Async query -
-   *    If the client changes a config setting, that shouldn't reflect in the execution already underway
-   * 2. confOverlay -
-   *    The query specific settings should only be applied to the query config and not session
-   * @return new configuration
-   * @throws HiveSQLException
-   */
-  private HiveConf getConfigForOperation() throws HiveSQLException {
-    HiveConf sqlOperationConf = getParentSession().getHiveConf();
-    if (!getConfOverlay().isEmpty() || shouldRunAsync()) {
-      // clone the parent session config for this query
-      sqlOperationConf = new HiveConf(sqlOperationConf);
-
-      // apply overlay query specific settings, if any
-      for (Map.Entry<String, String> confEntry : getConfOverlay().entrySet()) {
-        try {
-          sqlOperationConf.verifyAndSet(confEntry.getKey(), confEntry.getValue());
-        } catch (IllegalArgumentException e) {
-          throw new HiveSQLException("Error applying statement specific settings", e);
-        }
-      }
-    }
-    return sqlOperationConf;
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/TableTypeMapping.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/TableTypeMapping.java
deleted file mode 100644
index e59d19ea6be42..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/TableTypeMapping.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.operation;
-
-import java.util.Set;
-
-
-public interface TableTypeMapping {
-  /**
-   * Map client's table type name to hive's table type
-   * @param clientTypeName
-   * @return
-   */
-  String mapToHiveType(String clientTypeName);
-
-  /**
-   * Map hive's table type name to client's table type
-   * @param hiveTypeName
-   * @return
-   */
-  String mapToClientType(String hiveTypeName);
-
-  /**
-   * Get all the table types of this mapping
-   * @return
-   */
-  Set<String> getTableTypeNames();
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSession.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSession.java
deleted file mode 100644
index 65f9b298bf4f6..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSession.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.session;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hive.service.auth.HiveAuthFactory;
-import org.apache.hive.service.cli.*;
-
-public interface HiveSession extends HiveSessionBase {
-
-  void open(Map<String, String> sessionConfMap) throws Exception;
-
-  IMetaStoreClient getMetaStoreClient() throws HiveSQLException;
-
-  /**
-   * getInfo operation handler
-   * @param getInfoType
-   * @return
-   * @throws HiveSQLException
-   */
-  GetInfoValue getInfo(GetInfoType getInfoType) throws HiveSQLException;
-
-  /**
-   * execute operation handler
-   * @param statement
-   * @param confOverlay
-   * @return
-   * @throws HiveSQLException
-   */
-  OperationHandle executeStatement(String statement,
-      Map<String, String> confOverlay) throws HiveSQLException;
-
-  /**
-   * execute operation handler
-   * @param statement
-   * @param confOverlay
-   * @return
-   * @throws HiveSQLException
-   */
-  OperationHandle executeStatementAsync(String statement,
-      Map<String, String> confOverlay) throws HiveSQLException;
-
-  /**
-   * getTypeInfo operation handler
-   * @return
-   * @throws HiveSQLException
-   */
-  OperationHandle getTypeInfo() throws HiveSQLException;
-
-  /**
-   * getCatalogs operation handler
-   * @return
-   * @throws HiveSQLException
-   */
-  OperationHandle getCatalogs() throws HiveSQLException;
-
-  /**
-   * getSchemas operation handler
-   * @param catalogName
-   * @param schemaName
-   * @return
-   * @throws HiveSQLException
-   */
-  OperationHandle getSchemas(String catalogName, String schemaName)
-      throws HiveSQLException;
-
-  /**
-   * getTables operation handler
-   * @param catalogName
-   * @param schemaName
-   * @param tableName
-   * @param tableTypes
-   * @return
-   * @throws HiveSQLException
-   */
-  OperationHandle getTables(String catalogName, String schemaName,
-      String tableName, List<String> tableTypes) throws HiveSQLException;
-
-  /**
-   * getTableTypes operation handler
-   * @return
-   * @throws HiveSQLException
-   */
-  OperationHandle getTableTypes() throws HiveSQLException ;
-
-  /**
-   * getColumns operation handler
-   * @param catalogName
-   * @param schemaName
-   * @param tableName
-   * @param columnName
-   * @return
-   * @throws HiveSQLException
-   */
-  OperationHandle getColumns(String catalogName, String schemaName,
-      String tableName, String columnName)  throws HiveSQLException;
-
-  /**
-   * getFunctions operation handler
-   * @param catalogName
-   * @param schemaName
-   * @param functionName
-   * @return
-   * @throws HiveSQLException
-   */
-  OperationHandle getFunctions(String catalogName, String schemaName,
-      String functionName) throws HiveSQLException;
-
-  /**
-   * close the session
-   * @throws HiveSQLException
-   */
-  void close() throws HiveSQLException;
-
-  void cancelOperation(OperationHandle opHandle) throws HiveSQLException;
-
-  void closeOperation(OperationHandle opHandle) throws HiveSQLException;
-
-  TableSchema getResultSetMetadata(OperationHandle opHandle)
-      throws HiveSQLException;
-
-  RowSet fetchResults(OperationHandle opHandle, FetchOrientation orientation,
-      long maxRows, FetchType fetchType) throws HiveSQLException;
-
-  String getDelegationToken(HiveAuthFactory authFactory, String owner,
-      String renewer) throws HiveSQLException;
-
-  void cancelDelegationToken(HiveAuthFactory authFactory, String tokenStr)
-      throws HiveSQLException;
-
-  void renewDelegationToken(HiveAuthFactory authFactory, String tokenStr)
-      throws HiveSQLException;
-
-  void closeExpiredOperations();
-
-  long getNoOperationTime();
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionBase.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionBase.java
deleted file mode 100644
index b72c18b2b2135..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionBase.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.session;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hive.service.cli.SessionHandle;
-import org.apache.hive.service.cli.operation.OperationManager;
-import org.apache.hive.service.cli.thrift.TProtocolVersion;
-
-import java.io.File;
-
-/**
- * Methods that don't need to be executed under a doAs
- * context are here. Rest of them in HiveSession interface
- */
-public interface HiveSessionBase {
-
-  TProtocolVersion getProtocolVersion();
-
-  /**
-   * Set the session manager for the session
-   * @param sessionManager
-   */
-  void setSessionManager(SessionManager sessionManager);
-
-  /**
-   * Get the session manager for the session
-   */
-  SessionManager getSessionManager();
-
-  /**
-   * Set operation manager for the session
-   * @param operationManager
-   */
-  void setOperationManager(OperationManager operationManager);
-
-  /**
-   * Check whether operation logging is enabled and session dir is created successfully
-   */
-  boolean isOperationLogEnabled();
-
-  /**
-   * Get the session dir, which is the parent dir of operation logs
-   * @return a file representing the parent directory of operation logs
-   */
-  File getOperationLogSessionDir();
-
-  /**
-   * Set the session dir, which is the parent dir of operation logs
-   * @param operationLogRootDir the parent dir of the session dir
-   */
-  void setOperationLogSessionDir(File operationLogRootDir);
-
-  SessionHandle getSessionHandle();
-
-  String getUsername();
-
-  String getPassword();
-
-  HiveConf getHiveConf();
-
-  SessionState getSessionState();
-
-  String getUserName();
-
-  void setUserName(String userName);
-
-  String getIpAddress();
-
-  void setIpAddress(String ipAddress);
-
-  long getLastAccessTime();
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
deleted file mode 100644
index 745f385e87f78..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
+++ /dev/null
@@ -1,834 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.session;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.common.cli.HiveFileProcessor;
-import org.apache.hadoop.hive.common.cli.IHiveFileProcessor;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.ql.exec.FetchFormatter;
-import org.apache.hadoop.hive.ql.exec.ListSinkOperator;
-import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.history.HiveHistory;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.parse.VariableSubstitution;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hive.common.util.HiveVersionInfo;
-import org.apache.hive.service.auth.HiveAuthFactory;
-import org.apache.hive.service.cli.FetchOrientation;
-import org.apache.hive.service.cli.FetchType;
-import org.apache.hive.service.cli.GetInfoType;
-import org.apache.hive.service.cli.GetInfoValue;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.OperationHandle;
-import org.apache.hive.service.cli.RowSet;
-import org.apache.hive.service.cli.SessionHandle;
-import org.apache.hive.service.cli.TableSchema;
-import org.apache.hive.service.cli.operation.ExecuteStatementOperation;
-import org.apache.hive.service.cli.operation.GetCatalogsOperation;
-import org.apache.hive.service.cli.operation.GetColumnsOperation;
-import org.apache.hive.service.cli.operation.GetFunctionsOperation;
-import org.apache.hive.service.cli.operation.GetSchemasOperation;
-import org.apache.hive.service.cli.operation.GetTableTypesOperation;
-import org.apache.hive.service.cli.operation.GetTypeInfoOperation;
-import org.apache.hive.service.cli.operation.MetadataOperation;
-import org.apache.hive.service.cli.operation.Operation;
-import org.apache.hive.service.cli.operation.OperationManager;
-import org.apache.hive.service.cli.thrift.TProtocolVersion;
-import org.apache.hive.service.server.ThreadWithGarbageCleanup;
-
-import static org.apache.hadoop.hive.conf.SystemVariables.ENV_PREFIX;
-import static org.apache.hadoop.hive.conf.SystemVariables.HIVECONF_PREFIX;
-import static org.apache.hadoop.hive.conf.SystemVariables.HIVEVAR_PREFIX;
-import static org.apache.hadoop.hive.conf.SystemVariables.METACONF_PREFIX;
-import static org.apache.hadoop.hive.conf.SystemVariables.SYSTEM_PREFIX;
-
-/**
- * HiveSession
- *
- */
-public class HiveSessionImpl implements HiveSession {
-  private final SessionHandle sessionHandle;
-  private String username;
-  private final String password;
-  private HiveConf hiveConf;
-  private SessionState sessionState;
-  private String ipAddress;
-  private static final String FETCH_WORK_SERDE_CLASS =
-      "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe";
-  private static final Log LOG = LogFactory.getLog(HiveSessionImpl.class);
-  private SessionManager sessionManager;
-  private OperationManager operationManager;
-  private final Set<OperationHandle> opHandleSet = new HashSet<OperationHandle>();
-  private boolean isOperationLogEnabled;
-  private File sessionLogDir;
-  private volatile long lastAccessTime;
-  private volatile long lastIdleTime;
-
-  public HiveSessionImpl(TProtocolVersion protocol, String username, String password,
-      HiveConf serverhiveConf, String ipAddress) {
-    this.username = username;
-    this.password = password;
-    this.sessionHandle = new SessionHandle(protocol);
-    this.hiveConf = new HiveConf(serverhiveConf);
-    this.ipAddress = ipAddress;
-
-    try {
-      // In non-impersonation mode, map scheduler queue to current user
-      // if fair scheduler is configured.
-      if (! hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS) &&
-        hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE)) {
-        ShimLoader.getHadoopShims().refreshDefaultQueue(hiveConf, username);
-      }
-    } catch (IOException e) {
-      LOG.warn("Error setting scheduler queue: " + e, e);
-    }
-    // Set an explicit session name to control the download directory name
-    hiveConf.set(ConfVars.HIVESESSIONID.varname,
-        sessionHandle.getHandleIdentifier().toString());
-    // Use thrift transportable formatter
-    hiveConf.set(ListSinkOperator.OUTPUT_FORMATTER,
-        FetchFormatter.ThriftFormatter.class.getName());
-    hiveConf.setInt(ListSinkOperator.OUTPUT_PROTOCOL, protocol.getValue());
-  }
-
-  @Override
-  /**
-   * Opens a new HiveServer2 session for the client connection.
-   * Creates a new SessionState object that will be associated with this HiveServer2 session.
-   * When the server executes multiple queries in the same session,
-   * this SessionState object is reused across multiple queries.
-   * Note that if doAs is true, this call goes through a proxy object,
-   * which wraps the method logic in a UserGroupInformation#doAs.
-   * That's why it is important to create SessionState here rather than in the constructor.
-   */
-  public void open(Map<String, String> sessionConfMap) throws HiveSQLException {
-    sessionState = new SessionState(hiveConf, username);
-    sessionState.setUserIpAddress(ipAddress);
-    sessionState.setIsHiveServerQuery(true);
-    SessionState.start(sessionState);
-    try {
-      sessionState.reloadAuxJars();
-    } catch (IOException e) {
-      String msg = "Failed to load reloadable jar file path: " + e;
-      LOG.error(msg, e);
-      throw new HiveSQLException(msg, e);
-    }
-    // Process global init file: .hiverc
-    processGlobalInitFile();
-    if (sessionConfMap != null) {
-      configureSession(sessionConfMap);
-    }
-    lastAccessTime = System.currentTimeMillis();
-    lastIdleTime = lastAccessTime;
-  }
-
-  /**
-   * It is used for processing hiverc file from HiveServer2 side.
-   */
-  private class GlobalHivercFileProcessor extends HiveFileProcessor {
-    @Override
-    protected BufferedReader loadFile(String fileName) throws IOException {
-      FileInputStream initStream = null;
-      BufferedReader bufferedReader = null;
-      initStream = new FileInputStream(fileName);
-      bufferedReader = new BufferedReader(new InputStreamReader(initStream));
-      return bufferedReader;
-    }
-
-    @Override
-    protected int processCmd(String cmd) {
-      int rc = 0;
-      String cmd_trimed = cmd.trim();
-      try {
-        executeStatementInternal(cmd_trimed, null, false);
-      } catch (HiveSQLException e) {
-        rc = -1;
-        LOG.warn("Failed to execute HQL command in global .hiverc file.", e);
-      }
-      return rc;
-    }
-  }
-
-  private void processGlobalInitFile() {
-    IHiveFileProcessor processor = new GlobalHivercFileProcessor();
-
-    try {
-      String hiverc = hiveConf.getVar(ConfVars.HIVE_SERVER2_GLOBAL_INIT_FILE_LOCATION);
-      if (hiverc != null) {
-        File hivercFile = new File(hiverc);
-        if (hivercFile.isDirectory()) {
-          hivercFile = new File(hivercFile, SessionManager.HIVERCFILE);
-        }
-        if (hivercFile.isFile()) {
-          LOG.info("Running global init file: " + hivercFile);
-          int rc = processor.processFile(hivercFile.getAbsolutePath());
-          if (rc != 0) {
-            LOG.error("Failed on initializing global .hiverc file");
-          }
-        } else {
-          LOG.debug("Global init file " + hivercFile + " does not exist");
-        }
-      }
-    } catch (IOException e) {
-      LOG.warn("Failed on initializing global .hiverc file", e);
-    }
-  }
-
-  private void configureSession(Map<String, String> sessionConfMap) throws HiveSQLException {
-    SessionState.setCurrentSessionState(sessionState);
-    for (Map.Entry<String, String> entry : sessionConfMap.entrySet()) {
-      String key = entry.getKey();
-      if (key.startsWith("set:")) {
-        try {
-          setVariable(key.substring(4), entry.getValue());
-        } catch (Exception e) {
-          throw new HiveSQLException(e);
-        }
-      } else if (key.startsWith("use:")) {
-        SessionState.get().setCurrentDatabase(entry.getValue());
-      } else {
-        hiveConf.verifyAndSet(key, entry.getValue());
-      }
-    }
-  }
-
-  // Copy from org.apache.hadoop.hive.ql.processors.SetProcessor, only change:
-  // setConf(varname, propName, varvalue, true) when varname.startsWith(HIVECONF_PREFIX)
-  public static int setVariable(String varname, String varvalue) throws Exception {
-    SessionState ss = SessionState.get();
-    if (varvalue.contains("\n")){
-      ss.err.println("Warning: Value had a \\n character in it.");
-    }
-    varname = varname.trim();
-    if (varname.startsWith(ENV_PREFIX)){
-      ss.err.println("env:* variables can not be set.");
-      return 1;
-    } else if (varname.startsWith(SYSTEM_PREFIX)){
-      String propName = varname.substring(SYSTEM_PREFIX.length());
-      System.getProperties().setProperty(propName,
-              new VariableSubstitution().substitute(ss.getConf(),varvalue));
-    } else if (varname.startsWith(HIVECONF_PREFIX)){
-      String propName = varname.substring(HIVECONF_PREFIX.length());
-      setConf(varname, propName, varvalue, true);
-    } else if (varname.startsWith(HIVEVAR_PREFIX)) {
-      String propName = varname.substring(HIVEVAR_PREFIX.length());
-      ss.getHiveVariables().put(propName,
-              new VariableSubstitution().substitute(ss.getConf(),varvalue));
-    } else if (varname.startsWith(METACONF_PREFIX)) {
-      String propName = varname.substring(METACONF_PREFIX.length());
-      Hive hive = Hive.get(ss.getConf());
-      hive.setMetaConf(propName, new VariableSubstitution().substitute(ss.getConf(), varvalue));
-    } else {
-      setConf(varname, varname, varvalue, true);
-    }
-    return 0;
-  }
-
-  // returns non-null string for validation fail
-  private static void setConf(String varname, String key, String varvalue, boolean register)
-          throws IllegalArgumentException {
-    HiveConf conf = SessionState.get().getConf();
-    String value = new VariableSubstitution().substitute(conf, varvalue);
-    if (conf.getBoolVar(HiveConf.ConfVars.HIVECONFVALIDATION)) {
-      HiveConf.ConfVars confVars = HiveConf.getConfVars(key);
-      if (confVars != null) {
-        if (!confVars.isType(value)) {
-          StringBuilder message = new StringBuilder();
-          message.append("'SET ").append(varname).append('=').append(varvalue);
-          message.append("' FAILED because ").append(key).append(" expects ");
-          message.append(confVars.typeString()).append(" type value.");
-          throw new IllegalArgumentException(message.toString());
-        }
-        String fail = confVars.validate(value);
-        if (fail != null) {
-          StringBuilder message = new StringBuilder();
-          message.append("'SET ").append(varname).append('=').append(varvalue);
-          message.append("' FAILED in validation : ").append(fail).append('.');
-          throw new IllegalArgumentException(message.toString());
-        }
-      } else if (key.startsWith("hive.")) {
-        throw new IllegalArgumentException("hive configuration " + key + " does not exists.");
-      }
-    }
-    conf.verifyAndSet(key, value);
-    if (register) {
-      SessionState.get().getOverriddenConfigurations().put(key, value);
-    }
-  }
-
-  @Override
-  public void setOperationLogSessionDir(File operationLogRootDir) {
-    if (!operationLogRootDir.exists()) {
-      LOG.warn("The operation log root directory is removed, recreating: " +
-          operationLogRootDir.getAbsolutePath());
-      if (!operationLogRootDir.mkdirs()) {
-        LOG.warn("Unable to create operation log root directory: " +
-            operationLogRootDir.getAbsolutePath());
-      }
-    }
-    if (!operationLogRootDir.canWrite()) {
-      LOG.warn("The operation log root directory is not writable: " +
-          operationLogRootDir.getAbsolutePath());
-    }
-    sessionLogDir = new File(operationLogRootDir, sessionHandle.getHandleIdentifier().toString());
-    isOperationLogEnabled = true;
-    if (!sessionLogDir.exists()) {
-      if (!sessionLogDir.mkdir()) {
-        LOG.warn("Unable to create operation log session directory: " +
-            sessionLogDir.getAbsolutePath());
-        isOperationLogEnabled = false;
-      }
-    }
-    if (isOperationLogEnabled) {
-      LOG.info("Operation log session directory is created: " + sessionLogDir.getAbsolutePath());
-    }
-  }
-
-  @Override
-  public boolean isOperationLogEnabled() {
-    return isOperationLogEnabled;
-  }
-
-  @Override
-  public File getOperationLogSessionDir() {
-    return sessionLogDir;
-  }
-
-  @Override
-  public TProtocolVersion getProtocolVersion() {
-    return sessionHandle.getProtocolVersion();
-  }
-
-  @Override
-  public SessionManager getSessionManager() {
-    return sessionManager;
-  }
-
-  @Override
-  public void setSessionManager(SessionManager sessionManager) {
-    this.sessionManager = sessionManager;
-  }
-
-  private OperationManager getOperationManager() {
-    return operationManager;
-  }
-
-  @Override
-  public void setOperationManager(OperationManager operationManager) {
-    this.operationManager = operationManager;
-  }
-
-  protected synchronized void acquire(boolean userAccess) {
-    // Need to make sure that the this HiveServer2's session's SessionState is
-    // stored in the thread local for the handler thread.
-    SessionState.setCurrentSessionState(sessionState);
-    if (userAccess) {
-      lastAccessTime = System.currentTimeMillis();
-    }
-  }
-
-  /**
-   * 1. We'll remove the ThreadLocal SessionState as this thread might now serve
-   * other requests.
-   * 2. We'll cache the ThreadLocal RawStore object for this background thread for an orderly cleanup
-   * when this thread is garbage collected later.
-   * @see org.apache.hive.service.server.ThreadWithGarbageCleanup#finalize()
-   */
-  protected synchronized void release(boolean userAccess) {
-    SessionState.detachSession();
-    if (ThreadWithGarbageCleanup.currentThread() instanceof ThreadWithGarbageCleanup) {
-      ThreadWithGarbageCleanup currentThread =
-          (ThreadWithGarbageCleanup) ThreadWithGarbageCleanup.currentThread();
-      currentThread.cacheThreadLocalRawStore();
-    }
-    if (userAccess) {
-      lastAccessTime = System.currentTimeMillis();
-    }
-    if (opHandleSet.isEmpty()) {
-      lastIdleTime = System.currentTimeMillis();
-    } else {
-      lastIdleTime = 0;
-    }
-  }
-
-  @Override
-  public SessionHandle getSessionHandle() {
-    return sessionHandle;
-  }
-
-  @Override
-  public String getUsername() {
-    return username;
-  }
-
-  @Override
-  public String getPassword() {
-    return password;
-  }
-
-  @Override
-  public HiveConf getHiveConf() {
-    hiveConf.setVar(HiveConf.ConfVars.HIVEFETCHOUTPUTSERDE, FETCH_WORK_SERDE_CLASS);
-    return hiveConf;
-  }
-
-  @Override
-  public IMetaStoreClient getMetaStoreClient() throws HiveSQLException {
-    try {
-      return Hive.get(getHiveConf()).getMSC();
-    } catch (HiveException e) {
-      throw new HiveSQLException("Failed to get metastore connection", e);
-    } catch (MetaException e) {
-      throw new HiveSQLException("Failed to get metastore connection", e);
-    }
-  }
-
-  @Override
-  public GetInfoValue getInfo(GetInfoType getInfoType)
-      throws HiveSQLException {
-    acquire(true);
-    try {
-      switch (getInfoType) {
-      case CLI_SERVER_NAME:
-        return new GetInfoValue("Hive");
-      case CLI_DBMS_NAME:
-        return new GetInfoValue("Apache Hive");
-      case CLI_DBMS_VER:
-        return new GetInfoValue(HiveVersionInfo.getVersion());
-      case CLI_MAX_COLUMN_NAME_LEN:
-        return new GetInfoValue(128);
-      case CLI_MAX_SCHEMA_NAME_LEN:
-        return new GetInfoValue(128);
-      case CLI_MAX_TABLE_NAME_LEN:
-        return new GetInfoValue(128);
-      case CLI_TXN_CAPABLE:
-      default:
-        throw new HiveSQLException("Unrecognized GetInfoType value: " + getInfoType.toString());
-      }
-    } finally {
-      release(true);
-    }
-  }
-
-  @Override
-  public OperationHandle executeStatement(String statement, Map<String, String> confOverlay)
-      throws HiveSQLException {
-    return executeStatementInternal(statement, confOverlay, false);
-  }
-
-  @Override
-  public OperationHandle executeStatementAsync(String statement, Map<String, String> confOverlay)
-      throws HiveSQLException {
-    return executeStatementInternal(statement, confOverlay, true);
-  }
-
-  private OperationHandle executeStatementInternal(String statement, Map<String, String> confOverlay,
-      boolean runAsync)
-          throws HiveSQLException {
-    acquire(true);
-
-    OperationManager operationManager = getOperationManager();
-    ExecuteStatementOperation operation = operationManager
-        .newExecuteStatementOperation(getSession(), statement, confOverlay, runAsync);
-    OperationHandle opHandle = operation.getHandle();
-    try {
-      operation.run();
-      opHandleSet.add(opHandle);
-      return opHandle;
-    } catch (HiveSQLException e) {
-      // Referring to SQLOperation.java, there is no chance that a HiveSQLException throws and the asyn
-      // background operation submits to thread pool successfully at the same time. So, Cleanup
-      // opHandle directly when got HiveSQLException
-      operationManager.closeOperation(opHandle);
-      throw e;
-    } finally {
-      release(true);
-    }
-  }
-
-  @Override
-  public OperationHandle getTypeInfo()
-      throws HiveSQLException {
-    acquire(true);
-
-    OperationManager operationManager = getOperationManager();
-    GetTypeInfoOperation operation = operationManager.newGetTypeInfoOperation(getSession());
-    OperationHandle opHandle = operation.getHandle();
-    try {
-      operation.run();
-      opHandleSet.add(opHandle);
-      return opHandle;
-    } catch (HiveSQLException e) {
-      operationManager.closeOperation(opHandle);
-      throw e;
-    } finally {
-      release(true);
-    }
-  }
-
-  @Override
-  public OperationHandle getCatalogs()
-      throws HiveSQLException {
-    acquire(true);
-
-    OperationManager operationManager = getOperationManager();
-    GetCatalogsOperation operation = operationManager.newGetCatalogsOperation(getSession());
-    OperationHandle opHandle = operation.getHandle();
-    try {
-      operation.run();
-      opHandleSet.add(opHandle);
-      return opHandle;
-    } catch (HiveSQLException e) {
-      operationManager.closeOperation(opHandle);
-      throw e;
-    } finally {
-      release(true);
-    }
-  }
-
-  @Override
-  public OperationHandle getSchemas(String catalogName, String schemaName)
-      throws HiveSQLException {
-    acquire(true);
-
-    OperationManager operationManager = getOperationManager();
-    GetSchemasOperation operation =
-        operationManager.newGetSchemasOperation(getSession(), catalogName, schemaName);
-    OperationHandle opHandle = operation.getHandle();
-    try {
-      operation.run();
-      opHandleSet.add(opHandle);
-      return opHandle;
-    } catch (HiveSQLException e) {
-      operationManager.closeOperation(opHandle);
-      throw e;
-    } finally {
-      release(true);
-    }
-  }
-
-  @Override
-  public OperationHandle getTables(String catalogName, String schemaName, String tableName,
-      List<String> tableTypes)
-          throws HiveSQLException {
-    acquire(true);
-
-    OperationManager operationManager = getOperationManager();
-    MetadataOperation operation =
-        operationManager.newGetTablesOperation(getSession(), catalogName, schemaName, tableName, tableTypes);
-    OperationHandle opHandle = operation.getHandle();
-    try {
-      operation.run();
-      opHandleSet.add(opHandle);
-      return opHandle;
-    } catch (HiveSQLException e) {
-      operationManager.closeOperation(opHandle);
-      throw e;
-    } finally {
-      release(true);
-    }
-  }
-
-  @Override
-  public OperationHandle getTableTypes()
-      throws HiveSQLException {
-    acquire(true);
-
-    OperationManager operationManager = getOperationManager();
-    GetTableTypesOperation operation = operationManager.newGetTableTypesOperation(getSession());
-    OperationHandle opHandle = operation.getHandle();
-    try {
-      operation.run();
-      opHandleSet.add(opHandle);
-      return opHandle;
-    } catch (HiveSQLException e) {
-      operationManager.closeOperation(opHandle);
-      throw e;
-    } finally {
-      release(true);
-    }
-  }
-
-  @Override
-  public OperationHandle getColumns(String catalogName, String schemaName,
-      String tableName, String columnName)  throws HiveSQLException {
-    acquire(true);
-    String addedJars = Utilities.getResourceFiles(hiveConf, SessionState.ResourceType.JAR);
-    if (StringUtils.isNotBlank(addedJars)) {
-       IMetaStoreClient metastoreClient = getSession().getMetaStoreClient();
-       metastoreClient.setHiveAddedJars(addedJars);
-    }
-    OperationManager operationManager = getOperationManager();
-    GetColumnsOperation operation = operationManager.newGetColumnsOperation(getSession(),
-        catalogName, schemaName, tableName, columnName);
-    OperationHandle opHandle = operation.getHandle();
-    try {
-      operation.run();
-      opHandleSet.add(opHandle);
-      return opHandle;
-    } catch (HiveSQLException e) {
-      operationManager.closeOperation(opHandle);
-      throw e;
-    } finally {
-      release(true);
-    }
-  }
-
-  @Override
-  public OperationHandle getFunctions(String catalogName, String schemaName, String functionName)
-      throws HiveSQLException {
-    acquire(true);
-
-    OperationManager operationManager = getOperationManager();
-    GetFunctionsOperation operation = operationManager
-        .newGetFunctionsOperation(getSession(), catalogName, schemaName, functionName);
-    OperationHandle opHandle = operation.getHandle();
-    try {
-      operation.run();
-      opHandleSet.add(opHandle);
-      return opHandle;
-    } catch (HiveSQLException e) {
-      operationManager.closeOperation(opHandle);
-      throw e;
-    } finally {
-      release(true);
-    }
-  }
-
-  @Override
-  public void close() throws HiveSQLException {
-    try {
-      acquire(true);
-      // Iterate through the opHandles and close their operations
-      for (OperationHandle opHandle : opHandleSet) {
-        operationManager.closeOperation(opHandle);
-      }
-      opHandleSet.clear();
-      // Cleanup session log directory.
-      cleanupSessionLogDir();
-      // Cleanup pipeout file.
-      cleanupPipeoutFile();
-      HiveHistory hiveHist = sessionState.getHiveHistory();
-      if (null != hiveHist) {
-        hiveHist.closeStream();
-      }
-      try {
-        sessionState.close();
-      } finally {
-        sessionState = null;
-      }
-    } catch (IOException ioe) {
-      throw new HiveSQLException("Failure to close", ioe);
-    } finally {
-      if (sessionState != null) {
-        try {
-          sessionState.close();
-        } catch (Throwable t) {
-          LOG.warn("Error closing session", t);
-        }
-        sessionState = null;
-      }
-      release(true);
-    }
-  }
-
-  private void cleanupPipeoutFile() {
-    String lScratchDir = hiveConf.getVar(ConfVars.LOCALSCRATCHDIR);
-    String sessionID = hiveConf.getVar(ConfVars.HIVESESSIONID);
-
-    File[] fileAry = new File(lScratchDir).listFiles(
-            (dir, name) -> name.startsWith(sessionID) && name.endsWith(".pipeout"));
-
-    for (File file : fileAry) {
-      try {
-        FileUtils.forceDelete(file);
-      } catch (Exception e) {
-        LOG.error("Failed to cleanup pipeout file: " + file, e);
-      }
-    }
-  }
-
-  private void cleanupSessionLogDir() {
-    if (isOperationLogEnabled) {
-      try {
-        FileUtils.forceDelete(sessionLogDir);
-      } catch (Exception e) {
-        LOG.error("Failed to cleanup session log dir: " + sessionHandle, e);
-      }
-    }
-  }
-
-  @Override
-  public SessionState getSessionState() {
-    return sessionState;
-  }
-
-  @Override
-  public String getUserName() {
-    return username;
-  }
-
-  @Override
-  public void setUserName(String userName) {
-    this.username = userName;
-  }
-
-  @Override
-  public long getLastAccessTime() {
-    return lastAccessTime;
-  }
-
-  @Override
-  public void closeExpiredOperations() {
-    OperationHandle[] handles = opHandleSet.toArray(new OperationHandle[opHandleSet.size()]);
-    if (handles.length > 0) {
-      List<Operation> operations = operationManager.removeExpiredOperations(handles);
-      if (!operations.isEmpty()) {
-        closeTimedOutOperations(operations);
-      }
-    }
-  }
-
-  @Override
-  public long getNoOperationTime() {
-    return lastIdleTime > 0 ? System.currentTimeMillis() - lastIdleTime : 0;
-  }
-
-  private void closeTimedOutOperations(List<Operation> operations) {
-    acquire(false);
-    try {
-      for (Operation operation : operations) {
-        opHandleSet.remove(operation.getHandle());
-        try {
-          operation.close();
-        } catch (Exception e) {
-          LOG.warn("Exception is thrown closing timed-out operation " + operation.getHandle(), e);
-        }
-      }
-    } finally {
-      release(false);
-    }
-  }
-
-  @Override
-  public void cancelOperation(OperationHandle opHandle) throws HiveSQLException {
-    acquire(true);
-    try {
-      sessionManager.getOperationManager().cancelOperation(opHandle);
-    } finally {
-      release(true);
-    }
-  }
-
-  @Override
-  public void closeOperation(OperationHandle opHandle) throws HiveSQLException {
-    acquire(true);
-    try {
-      operationManager.closeOperation(opHandle);
-      opHandleSet.remove(opHandle);
-    } finally {
-      release(true);
-    }
-  }
-
-  @Override
-  public TableSchema getResultSetMetadata(OperationHandle opHandle) throws HiveSQLException {
-    acquire(true);
-    try {
-      return sessionManager.getOperationManager().getOperationResultSetSchema(opHandle);
-    } finally {
-      release(true);
-    }
-  }
-
-  @Override
-  public RowSet fetchResults(OperationHandle opHandle, FetchOrientation orientation,
-      long maxRows, FetchType fetchType) throws HiveSQLException {
-    acquire(true);
-    try {
-      if (fetchType == FetchType.QUERY_OUTPUT) {
-        return operationManager.getOperationNextRowSet(opHandle, orientation, maxRows);
-      }
-      return operationManager.getOperationLogRowSet(opHandle, orientation, maxRows);
-    } finally {
-      release(true);
-    }
-  }
-
-  protected HiveSession getSession() {
-    return this;
-  }
-
-  @Override
-  public String getIpAddress() {
-    return ipAddress;
-  }
-
-  @Override
-  public void setIpAddress(String ipAddress) {
-    this.ipAddress = ipAddress;
-  }
-
-  @Override
-  public String getDelegationToken(HiveAuthFactory authFactory, String owner, String renewer)
-      throws HiveSQLException {
-    HiveAuthFactory.verifyProxyAccess(getUsername(), owner, getIpAddress(), getHiveConf());
-    return authFactory.getDelegationToken(owner, renewer);
-  }
-
-  @Override
-  public void cancelDelegationToken(HiveAuthFactory authFactory, String tokenStr)
-      throws HiveSQLException {
-    HiveAuthFactory.verifyProxyAccess(getUsername(), getUserFromToken(authFactory, tokenStr),
-        getIpAddress(), getHiveConf());
-    authFactory.cancelDelegationToken(tokenStr);
-  }
-
-  @Override
-  public void renewDelegationToken(HiveAuthFactory authFactory, String tokenStr)
-      throws HiveSQLException {
-    HiveAuthFactory.verifyProxyAccess(getUsername(), getUserFromToken(authFactory, tokenStr),
-        getIpAddress(), getHiveConf());
-    authFactory.renewDelegationToken(tokenStr);
-  }
-
-  // extract the real user from the given token string
-  private String getUserFromToken(HiveAuthFactory authFactory, String tokenStr) throws HiveSQLException {
-    return authFactory.getUserFromToken(tokenStr);
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java
deleted file mode 100644
index 762dbb2faadec..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.session;
-
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.shims.Utils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hive.service.auth.HiveAuthFactory;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.thrift.TProtocolVersion;
-
-/**
- *
- * HiveSessionImplwithUGI.
- * HiveSession with connecting user's UGI and delegation token if required
- */
-public class HiveSessionImplwithUGI extends HiveSessionImpl {
-  public static final String HS2TOKEN = "HiveServer2ImpersonationToken";
-
-  private UserGroupInformation sessionUgi = null;
-  private String delegationTokenStr = null;
-  private Hive sessionHive = null;
-  private HiveSession proxySession = null;
-  static final Log LOG = LogFactory.getLog(HiveSessionImplwithUGI.class);
-
-  public HiveSessionImplwithUGI(TProtocolVersion protocol, String username, String password,
-      HiveConf hiveConf, String ipAddress, String delegationToken) throws HiveSQLException {
-    super(protocol, username, password, hiveConf, ipAddress);
-    setSessionUGI(username);
-    setDelegationToken(delegationToken);
-
-    // create a new metastore connection for this particular user session
-    Hive.set(null);
-    try {
-      sessionHive = Hive.get(getHiveConf());
-    } catch (HiveException e) {
-      throw new HiveSQLException("Failed to setup metastore connection", e);
-    }
-  }
-
-  // setup appropriate UGI for the session
-  public void setSessionUGI(String owner) throws HiveSQLException {
-    if (owner == null) {
-      throw new HiveSQLException("No username provided for impersonation");
-    }
-    if (UserGroupInformation.isSecurityEnabled()) {
-      try {
-        sessionUgi = UserGroupInformation.createProxyUser(
-            owner, UserGroupInformation.getLoginUser());
-      } catch (IOException e) {
-        throw new HiveSQLException("Couldn't setup proxy user", e);
-      }
-    } else {
-      sessionUgi = UserGroupInformation.createRemoteUser(owner);
-    }
-  }
-
-  public UserGroupInformation getSessionUgi() {
-    return this.sessionUgi;
-  }
-
-  public String getDelegationToken() {
-    return this.delegationTokenStr;
-  }
-
-  @Override
-  protected synchronized void acquire(boolean userAccess) {
-    super.acquire(userAccess);
-    // if we have a metastore connection with impersonation, then set it first
-    if (sessionHive != null) {
-      Hive.set(sessionHive);
-    }
-  }
-
-  /**
-   * Close the file systems for the session and remove it from the FileSystem cache.
-   * Cancel the session's delegation token and close the metastore connection
-   */
-  @Override
-  public void close() throws HiveSQLException {
-    try {
-      acquire(true);
-      cancelDelegationToken();
-    } finally {
-      try {
-        super.close();
-      } finally {
-        try {
-          FileSystem.closeAllForUGI(sessionUgi);
-        } catch (IOException ioe) {
-          throw new HiveSQLException("Could not clean up file-system handles for UGI: "
-              + sessionUgi, ioe);
-        }
-      }
-    }
-  }
-
-  /**
-   * Enable delegation token for the session
-   * save the token string and set the token.signature in hive conf. The metastore client uses
-   * this token.signature to determine where to use kerberos or delegation token
-   * @throws HiveException
-   * @throws IOException
-   */
-  private void setDelegationToken(String delegationTokenStr) throws HiveSQLException {
-    this.delegationTokenStr = delegationTokenStr;
-    if (delegationTokenStr != null) {
-      getHiveConf().set("hive.metastore.token.signature", HS2TOKEN);
-      try {
-        Utils.setTokenStr(sessionUgi, delegationTokenStr, HS2TOKEN);
-      } catch (IOException e) {
-        throw new HiveSQLException("Couldn't setup delegation token in the ugi", e);
-      }
-    }
-  }
-
-  // If the session has a delegation token obtained from the metastore, then cancel it
-  private void cancelDelegationToken() throws HiveSQLException {
-    if (delegationTokenStr != null) {
-      try {
-        Hive.get(getHiveConf()).cancelDelegationToken(delegationTokenStr);
-      } catch (HiveException e) {
-        throw new HiveSQLException("Couldn't cancel delegation token", e);
-      }
-      // close the metastore connection created with this delegation token
-      Hive.closeCurrent();
-    }
-  }
-
-  @Override
-  protected HiveSession getSession() {
-    assert proxySession != null;
-
-    return proxySession;
-  }
-
-  public void setProxySession(HiveSession proxySession) {
-    this.proxySession = proxySession;
-  }
-
-  @Override
-  public String getDelegationToken(HiveAuthFactory authFactory, String owner,
-      String renewer) throws HiveSQLException {
-    return authFactory.getDelegationToken(owner, renewer);
-  }
-
-  @Override
-  public void cancelDelegationToken(HiveAuthFactory authFactory, String tokenStr)
-      throws HiveSQLException {
-    authFactory.cancelDelegationToken(tokenStr);
-  }
-
-  @Override
-  public void renewDelegationToken(HiveAuthFactory authFactory, String tokenStr)
-      throws HiveSQLException {
-    authFactory.renewDelegationToken(tokenStr);
-  }
-
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/SessionManager.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/SessionManager.java
deleted file mode 100644
index 859f9c8b449e5..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/SessionManager.java
+++ /dev/null
@@ -1,361 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.session;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hive.service.CompositeService;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.SessionHandle;
-import org.apache.hive.service.cli.operation.OperationManager;
-import org.apache.hive.service.cli.thrift.TProtocolVersion;
-import org.apache.hive.service.server.HiveServer2;
-import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup;
-
-/**
- * SessionManager.
- *
- */
-public class SessionManager extends CompositeService {
-
-  private static final Log LOG = LogFactory.getLog(SessionManager.class);
-  public static final String HIVERCFILE = ".hiverc";
-  private HiveConf hiveConf;
-  private final Map<SessionHandle, HiveSession> handleToSession =
-      new ConcurrentHashMap<SessionHandle, HiveSession>();
-  private final OperationManager operationManager = new OperationManager();
-  private ThreadPoolExecutor backgroundOperationPool;
-  private boolean isOperationLogEnabled;
-  private File operationLogRootDir;
-
-  private long checkInterval;
-  private long sessionTimeout;
-  private boolean checkOperation;
-
-  private volatile boolean shutdown;
-  // The HiveServer2 instance running this service
-  private final HiveServer2 hiveServer2;
-
-  public SessionManager(HiveServer2 hiveServer2) {
-    super(SessionManager.class.getSimpleName());
-    this.hiveServer2 = hiveServer2;
-  }
-
-  @Override
-  public synchronized void init(HiveConf hiveConf) {
-    this.hiveConf = hiveConf;
-    //Create operation log root directory, if operation logging is enabled
-    if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_ENABLED)) {
-      initOperationLogRootDir();
-    }
-    createBackgroundOperationPool();
-    addService(operationManager);
-    super.init(hiveConf);
-  }
-
-  private void createBackgroundOperationPool() {
-    int poolSize = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_ASYNC_EXEC_THREADS);
-    LOG.info("HiveServer2: Background operation thread pool size: " + poolSize);
-    int poolQueueSize = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_ASYNC_EXEC_WAIT_QUEUE_SIZE);
-    LOG.info("HiveServer2: Background operation thread wait queue size: " + poolQueueSize);
-    long keepAliveTime = HiveConf.getTimeVar(
-        hiveConf, ConfVars.HIVE_SERVER2_ASYNC_EXEC_KEEPALIVE_TIME, TimeUnit.SECONDS);
-    LOG.info(
-        "HiveServer2: Background operation thread keepalive time: " + keepAliveTime + " seconds");
-
-    // Create a thread pool with #poolSize threads
-    // Threads terminate when they are idle for more than the keepAliveTime
-    // A bounded blocking queue is used to queue incoming operations, if #operations > poolSize
-    String threadPoolName = "HiveServer2-Background-Pool";
-    backgroundOperationPool = new ThreadPoolExecutor(poolSize, poolSize,
-        keepAliveTime, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(poolQueueSize),
-        new ThreadFactoryWithGarbageCleanup(threadPoolName));
-    backgroundOperationPool.allowCoreThreadTimeOut(true);
-
-    checkInterval = HiveConf.getTimeVar(
-        hiveConf, ConfVars.HIVE_SERVER2_SESSION_CHECK_INTERVAL, TimeUnit.MILLISECONDS);
-    sessionTimeout = HiveConf.getTimeVar(
-        hiveConf, ConfVars.HIVE_SERVER2_IDLE_SESSION_TIMEOUT, TimeUnit.MILLISECONDS);
-    checkOperation = HiveConf.getBoolVar(hiveConf,
-        ConfVars.HIVE_SERVER2_IDLE_SESSION_CHECK_OPERATION);
-  }
-
-  private void initOperationLogRootDir() {
-    operationLogRootDir = new File(
-        hiveConf.getVar(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LOG_LOCATION));
-    isOperationLogEnabled = true;
-
-    if (operationLogRootDir.exists() && !operationLogRootDir.isDirectory()) {
-      LOG.warn("The operation log root directory exists, but it is not a directory: " +
-          operationLogRootDir.getAbsolutePath());
-      isOperationLogEnabled = false;
-    }
-
-    if (!operationLogRootDir.exists()) {
-      if (!operationLogRootDir.mkdirs()) {
-        LOG.warn("Unable to create operation log root directory: " +
-            operationLogRootDir.getAbsolutePath());
-        isOperationLogEnabled = false;
-      }
-    }
-
-    if (isOperationLogEnabled) {
-      LOG.info("Operation log root directory is created: " + operationLogRootDir.getAbsolutePath());
-      try {
-        FileUtils.forceDeleteOnExit(operationLogRootDir);
-      } catch (IOException e) {
-        LOG.warn("Failed to schedule cleanup HS2 operation logging root dir: " +
-            operationLogRootDir.getAbsolutePath(), e);
-      }
-    }
-  }
-
-  @Override
-  public synchronized void start() {
-    super.start();
-    if (checkInterval > 0) {
-      startTimeoutChecker();
-    }
-  }
-
-  private void startTimeoutChecker() {
-    final long interval = Math.max(checkInterval, 3000L);  // minimum 3 seconds
-    Runnable timeoutChecker = new Runnable() {
-      @Override
-      public void run() {
-        for (sleepInterval(interval); !shutdown; sleepInterval(interval)) {
-          long current = System.currentTimeMillis();
-          for (HiveSession session : new ArrayList<HiveSession>(handleToSession.values())) {
-            if (sessionTimeout > 0 && session.getLastAccessTime() + sessionTimeout <= current
-                && (!checkOperation || session.getNoOperationTime() > sessionTimeout)) {
-              SessionHandle handle = session.getSessionHandle();
-              LOG.warn("Session " + handle + " is Timed-out (last access : " +
-                  new Date(session.getLastAccessTime()) + ") and will be closed");
-              try {
-                closeSession(handle);
-              } catch (HiveSQLException e) {
-                LOG.warn("Exception is thrown closing session " + handle, e);
-              }
-            } else {
-              session.closeExpiredOperations();
-            }
-          }
-        }
-      }
-
-      private void sleepInterval(long interval) {
-        try {
-          Thread.sleep(interval);
-        } catch (InterruptedException e) {
-          // ignore
-        }
-      }
-    };
-    backgroundOperationPool.execute(timeoutChecker);
-  }
-
-  @Override
-  public synchronized void stop() {
-    super.stop();
-    shutdown = true;
-    if (backgroundOperationPool != null) {
-      backgroundOperationPool.shutdown();
-      long timeout = hiveConf.getTimeVar(
-          ConfVars.HIVE_SERVER2_ASYNC_EXEC_SHUTDOWN_TIMEOUT, TimeUnit.SECONDS);
-      try {
-        backgroundOperationPool.awaitTermination(timeout, TimeUnit.SECONDS);
-      } catch (InterruptedException e) {
-        LOG.warn("HIVE_SERVER2_ASYNC_EXEC_SHUTDOWN_TIMEOUT = " + timeout +
-            " seconds has been exceeded. RUNNING background operations will be shut down", e);
-      }
-      backgroundOperationPool = null;
-    }
-    cleanupLoggingRootDir();
-  }
-
-  private void cleanupLoggingRootDir() {
-    if (isOperationLogEnabled) {
-      try {
-        FileUtils.forceDelete(operationLogRootDir);
-      } catch (Exception e) {
-        LOG.warn("Failed to cleanup root dir of HS2 logging: " + operationLogRootDir
-            .getAbsolutePath(), e);
-      }
-    }
-  }
-
-  public SessionHandle openSession(TProtocolVersion protocol, String username, String password, String ipAddress,
-      Map<String, String> sessionConf) throws HiveSQLException {
-    return openSession(protocol, username, password, ipAddress, sessionConf, false, null);
-  }
-
-  /**
-   * Opens a new session and creates a session handle.
-   * The username passed to this method is the effective username.
-   * If withImpersonation is true (==doAs true) we wrap all the calls in HiveSession
-   * within a UGI.doAs, where UGI corresponds to the effective user.
-   *
-   * Please see {@code org.apache.hive.service.cli.thrift.ThriftCLIService.getUserName()} for
-   * more details.
-   *
-   * @param protocol
-   * @param username
-   * @param password
-   * @param ipAddress
-   * @param sessionConf
-   * @param withImpersonation
-   * @param delegationToken
-   * @return
-   * @throws HiveSQLException
-   */
-  public SessionHandle openSession(TProtocolVersion protocol, String username, String password, String ipAddress,
-      Map<String, String> sessionConf, boolean withImpersonation, String delegationToken)
-          throws HiveSQLException {
-    HiveSession session;
-    // If doAs is set to true for HiveServer2, we will create a proxy object for the session impl.
-    // Within the proxy object, we wrap the method call in a UserGroupInformation#doAs
-    if (withImpersonation) {
-      HiveSessionImplwithUGI sessionWithUGI = new HiveSessionImplwithUGI(protocol, username, password,
-          hiveConf, ipAddress, delegationToken);
-      session = HiveSessionProxy.getProxy(sessionWithUGI, sessionWithUGI.getSessionUgi());
-      sessionWithUGI.setProxySession(session);
-    } else {
-      session = new HiveSessionImpl(protocol, username, password, hiveConf, ipAddress);
-    }
-    session.setSessionManager(this);
-    session.setOperationManager(operationManager);
-    try {
-      session.open(sessionConf);
-    } catch (Exception e) {
-      try {
-        session.close();
-      } catch (Throwable t) {
-        LOG.warn("Error closing session", t);
-      }
-      session = null;
-      throw new HiveSQLException("Failed to open new session: " + e, e);
-    }
-    if (isOperationLogEnabled) {
-      session.setOperationLogSessionDir(operationLogRootDir);
-    }
-    handleToSession.put(session.getSessionHandle(), session);
-    return session.getSessionHandle();
-  }
-
-  public void closeSession(SessionHandle sessionHandle) throws HiveSQLException {
-    HiveSession session = handleToSession.remove(sessionHandle);
-    if (session == null) {
-      throw new HiveSQLException("Session does not exist!");
-    }
-    session.close();
-  }
-
-  public HiveSession getSession(SessionHandle sessionHandle) throws HiveSQLException {
-    HiveSession session = handleToSession.get(sessionHandle);
-    if (session == null) {
-      throw new HiveSQLException("Invalid SessionHandle: " + sessionHandle);
-    }
-    return session;
-  }
-
-  public OperationManager getOperationManager() {
-    return operationManager;
-  }
-
-  private static ThreadLocal<String> threadLocalIpAddress = new ThreadLocal<String>() {
-    @Override
-    protected synchronized String initialValue() {
-      return null;
-    }
-  };
-
-  public static void setIpAddress(String ipAddress) {
-    threadLocalIpAddress.set(ipAddress);
-  }
-
-  public static void clearIpAddress() {
-    threadLocalIpAddress.remove();
-  }
-
-  public static String getIpAddress() {
-    return threadLocalIpAddress.get();
-  }
-
-  private static ThreadLocal<String> threadLocalUserName = new ThreadLocal<String>(){
-    @Override
-    protected synchronized String initialValue() {
-      return null;
-    }
-  };
-
-  public static void setUserName(String userName) {
-    threadLocalUserName.set(userName);
-  }
-
-  public static void clearUserName() {
-    threadLocalUserName.remove();
-  }
-
-  public static String getUserName() {
-    return threadLocalUserName.get();
-  }
-
-  private static ThreadLocal<String> threadLocalProxyUserName = new ThreadLocal<String>(){
-    @Override
-    protected synchronized String initialValue() {
-      return null;
-    }
-  };
-
-  public static void setProxyUserName(String userName) {
-    LOG.debug("setting proxy user name based on query param to: " + userName);
-    threadLocalProxyUserName.set(userName);
-  }
-
-  public static String getProxyUserName() {
-    return threadLocalProxyUserName.get();
-  }
-
-  public static void clearProxyUserName() {
-    threadLocalProxyUserName.remove();
-  }
-
-  public Future<?> submitBackgroundOperation(Runnable r) {
-    return backgroundOperationPool.submit(r);
-  }
-
-  public int getOpenSessionCount() {
-    return handleToSession.size();
-  }
-}
-
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
deleted file mode 100644
index 21b8bf7de75ce..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.thrift;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hive.service.auth.HiveAuthFactory;
-import org.apache.hive.service.cli.CLIService;
-import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup;
-import org.apache.thrift.TProcessorFactory;
-import org.apache.thrift.protocol.TBinaryProtocol;
-import org.apache.thrift.server.TThreadPoolServer;
-import org.apache.thrift.transport.TServerSocket;
-import org.apache.thrift.transport.TTransportFactory;
-
-
-public class ThriftBinaryCLIService extends ThriftCLIService {
-
-  public ThriftBinaryCLIService(CLIService cliService) {
-    super(cliService, ThriftBinaryCLIService.class.getSimpleName());
-  }
-
-  @Override
-  public void run() {
-    try {
-      // Server thread pool
-      String threadPoolName = "HiveServer2-Handler-Pool";
-      ExecutorService executorService = new ThreadPoolExecutor(minWorkerThreads, maxWorkerThreads,
-          workerKeepAliveTime, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
-          new ThreadFactoryWithGarbageCleanup(threadPoolName));
-
-      // Thrift configs
-      hiveAuthFactory = new HiveAuthFactory(hiveConf);
-      TTransportFactory transportFactory = hiveAuthFactory.getAuthTransFactory();
-      TProcessorFactory processorFactory = hiveAuthFactory.getAuthProcFactory(this);
-      TServerSocket serverSocket = null;
-      List<String> sslVersionBlacklist = new ArrayList<String>();
-      for (String sslVersion : hiveConf.getVar(ConfVars.HIVE_SSL_PROTOCOL_BLACKLIST).split(",")) {
-        sslVersionBlacklist.add(sslVersion);
-      }
-      if (!hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_USE_SSL)) {
-        serverSocket = HiveAuthFactory.getServerSocket(hiveHost, portNum);
-      } else {
-        String keyStorePath = hiveConf.getVar(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH).trim();
-        if (keyStorePath.isEmpty()) {
-          throw new IllegalArgumentException(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname
-              + " Not configured for SSL connection");
-        }
-        String keyStorePassword = ShimLoader.getHadoopShims().getPassword(hiveConf,
-            HiveConf.ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname);
-        serverSocket = HiveAuthFactory.getServerSSLSocket(hiveHost, portNum, keyStorePath,
-            keyStorePassword, sslVersionBlacklist);
-      }
-
-      // Server args
-      int maxMessageSize = hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE);
-      int requestTimeout = (int) hiveConf.getTimeVar(
-          HiveConf.ConfVars.HIVE_SERVER2_THRIFT_LOGIN_TIMEOUT, TimeUnit.SECONDS);
-      int beBackoffSlotLength = (int) hiveConf.getTimeVar(
-          HiveConf.ConfVars.HIVE_SERVER2_THRIFT_LOGIN_BEBACKOFF_SLOT_LENGTH, TimeUnit.MILLISECONDS);
-      TThreadPoolServer.Args sargs = new TThreadPoolServer.Args(serverSocket)
-          .processorFactory(processorFactory).transportFactory(transportFactory)
-          .protocolFactory(new TBinaryProtocol.Factory())
-          .inputProtocolFactory(new TBinaryProtocol.Factory(true, true, maxMessageSize, maxMessageSize))
-          .requestTimeout(requestTimeout).requestTimeoutUnit(TimeUnit.SECONDS)
-          .beBackoffSlotLength(beBackoffSlotLength).beBackoffSlotLengthUnit(TimeUnit.MILLISECONDS)
-          .executorService(executorService);
-
-      // TCP Server
-      server = new TThreadPoolServer(sargs);
-      server.setServerEventHandler(serverEventHandler);
-      String msg = "Starting " + ThriftBinaryCLIService.class.getSimpleName() + " on port "
-          + serverSocket.getServerSocket().getLocalPort() + " with " + minWorkerThreads + "..." + maxWorkerThreads + " worker threads";
-      LOG.info(msg);
-      server.serve();
-    } catch (Throwable t) {
-      LOG.fatal(
-          "Error starting HiveServer2: could not start "
-              + ThriftBinaryCLIService.class.getSimpleName(), t);
-      System.exit(-1);
-    }
-  }
-
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
deleted file mode 100644
index 8fce9d9383438..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
+++ /dev/null
@@ -1,689 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.thrift;
-
-import javax.security.auth.login.LoginException;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hive.service.AbstractService;
-import org.apache.hive.service.ServiceException;
-import org.apache.hive.service.ServiceUtils;
-import org.apache.hive.service.auth.HiveAuthFactory;
-import org.apache.hive.service.auth.TSetIpAddressProcessor;
-import org.apache.hive.service.cli.*;
-import org.apache.hive.service.cli.session.SessionManager;
-import org.apache.hive.service.server.HiveServer2;
-import org.apache.thrift.TException;
-import org.apache.thrift.protocol.TProtocol;
-import org.apache.thrift.server.ServerContext;
-import org.apache.thrift.server.TServer;
-import org.apache.thrift.server.TServerEventHandler;
-import org.apache.thrift.transport.TTransport;
-
-/**
- * ThriftCLIService.
- *
- */
-public abstract class ThriftCLIService extends AbstractService implements TCLIService.Iface, Runnable {
-
-  public static final Log LOG = LogFactory.getLog(ThriftCLIService.class.getName());
-
-  protected CLIService cliService;
-  private static final TStatus OK_STATUS = new TStatus(TStatusCode.SUCCESS_STATUS);
-  protected static HiveAuthFactory hiveAuthFactory;
-
-  protected int portNum;
-  protected InetAddress serverIPAddress;
-  protected String hiveHost;
-  protected TServer server;
-  protected org.eclipse.jetty.server.Server httpServer;
-
-  private boolean isStarted = false;
-  protected boolean isEmbedded = false;
-
-  protected HiveConf hiveConf;
-
-  protected int minWorkerThreads;
-  protected int maxWorkerThreads;
-  protected long workerKeepAliveTime;
-
-  protected TServerEventHandler serverEventHandler;
-  protected ThreadLocal<ServerContext> currentServerContext;
-
-  static class ThriftCLIServerContext implements ServerContext {
-    private SessionHandle sessionHandle = null;
-
-    public void setSessionHandle(SessionHandle sessionHandle) {
-      this.sessionHandle = sessionHandle;
-    }
-
-    public SessionHandle getSessionHandle() {
-      return sessionHandle;
-    }
-  }
-
-  public ThriftCLIService(CLIService service, String serviceName) {
-    super(serviceName);
-    this.cliService = service;
-    currentServerContext = new ThreadLocal<ServerContext>();
-    serverEventHandler = new TServerEventHandler() {
-      @Override
-      public ServerContext createContext(
-          TProtocol input, TProtocol output) {
-        return new ThriftCLIServerContext();
-      }
-
-      @Override
-      public void deleteContext(ServerContext serverContext,
-          TProtocol input, TProtocol output) {
-        ThriftCLIServerContext context = (ThriftCLIServerContext)serverContext;
-        SessionHandle sessionHandle = context.getSessionHandle();
-        if (sessionHandle != null) {
-          LOG.info("Session disconnected without closing properly, close it now");
-          try {
-            cliService.closeSession(sessionHandle);
-          } catch (HiveSQLException e) {
-            LOG.warn("Failed to close session: " + e, e);
-          }
-        }
-      }
-
-      @Override
-      public void preServe() {
-      }
-
-      @Override
-      public void processContext(ServerContext serverContext,
-          TTransport input, TTransport output) {
-        currentServerContext.set(serverContext);
-      }
-    };
-  }
-
-  @Override
-  public synchronized void init(HiveConf hiveConf) {
-    this.hiveConf = hiveConf;
-    // Initialize common server configs needed in both binary & http modes
-    String portString;
-    hiveHost = System.getenv("HIVE_SERVER2_THRIFT_BIND_HOST");
-    if (hiveHost == null) {
-      hiveHost = hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST);
-    }
-    try {
-      if (hiveHost != null && !hiveHost.isEmpty()) {
-        serverIPAddress = InetAddress.getByName(hiveHost);
-      } else {
-        serverIPAddress = InetAddress.getLocalHost();
-      }
-    } catch (UnknownHostException e) {
-      throw new ServiceException(e);
-    }
-    // HTTP mode
-    if (HiveServer2.isHTTPTransportMode(hiveConf)) {
-      workerKeepAliveTime =
-          hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_WORKER_KEEPALIVE_TIME,
-              TimeUnit.SECONDS);
-      portString = System.getenv("HIVE_SERVER2_THRIFT_HTTP_PORT");
-      if (portString != null) {
-        portNum = Integer.valueOf(portString);
-      } else {
-        portNum = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT);
-      }
-    }
-    // Binary mode
-    else {
-      workerKeepAliveTime =
-          hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_THRIFT_WORKER_KEEPALIVE_TIME, TimeUnit.SECONDS);
-      portString = System.getenv("HIVE_SERVER2_THRIFT_PORT");
-      if (portString != null) {
-        portNum = Integer.valueOf(portString);
-      } else {
-        portNum = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT);
-      }
-    }
-    minWorkerThreads = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_MIN_WORKER_THREADS);
-    maxWorkerThreads = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_MAX_WORKER_THREADS);
-    super.init(hiveConf);
-  }
-
-  @Override
-  public synchronized void start() {
-    super.start();
-    if (!isStarted && !isEmbedded) {
-      new Thread(this).start();
-      isStarted = true;
-    }
-  }
-
-  @Override
-  public synchronized void stop() {
-    if (isStarted && !isEmbedded) {
-      if(server != null) {
-        server.stop();
-        LOG.info("Thrift server has stopped");
-      }
-      if((httpServer != null) && httpServer.isStarted()) {
-        try {
-          httpServer.stop();
-          LOG.info("Http server has stopped");
-        } catch (Exception e) {
-          LOG.error("Error stopping Http server: ", e);
-        }
-      }
-      isStarted = false;
-    }
-    super.stop();
-  }
-
-  public int getPortNumber() {
-    return portNum;
-  }
-
-  public InetAddress getServerIPAddress() {
-    return serverIPAddress;
-  }
-
-  @Override
-  public TGetDelegationTokenResp GetDelegationToken(TGetDelegationTokenReq req)
-      throws TException {
-    TGetDelegationTokenResp resp = new TGetDelegationTokenResp();
-    resp.setStatus(notSupportTokenErrorStatus());
-    return resp;
-  }
-
-  @Override
-  public TCancelDelegationTokenResp CancelDelegationToken(TCancelDelegationTokenReq req)
-      throws TException {
-    TCancelDelegationTokenResp resp = new TCancelDelegationTokenResp();
-    resp.setStatus(notSupportTokenErrorStatus());
-    return resp;
-  }
-
-  @Override
-  public TRenewDelegationTokenResp RenewDelegationToken(TRenewDelegationTokenReq req)
-      throws TException {
-    TRenewDelegationTokenResp resp = new TRenewDelegationTokenResp();
-    resp.setStatus(notSupportTokenErrorStatus());
-    return resp;
-  }
-
-  private TStatus notSupportTokenErrorStatus() {
-    TStatus errorStatus = new TStatus(TStatusCode.ERROR_STATUS);
-    errorStatus.setErrorMessage("Delegation token is not supported");
-    return errorStatus;
-  }
-
-  @Override
-  public TOpenSessionResp OpenSession(TOpenSessionReq req) throws TException {
-    LOG.info("Client protocol version: " + req.getClient_protocol());
-    TOpenSessionResp resp = new TOpenSessionResp();
-    try {
-      SessionHandle sessionHandle = getSessionHandle(req, resp);
-      resp.setSessionHandle(sessionHandle.toTSessionHandle());
-      // TODO: set real configuration map
-      resp.setConfiguration(new HashMap<String, String>());
-      resp.setStatus(OK_STATUS);
-      ThriftCLIServerContext context =
-        (ThriftCLIServerContext)currentServerContext.get();
-      if (context != null) {
-        context.setSessionHandle(sessionHandle);
-      }
-    } catch (Exception e) {
-      LOG.warn("Error opening session: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  private String getIpAddress() {
-    String clientIpAddress;
-    // Http transport mode.
-    // We set the thread local ip address, in ThriftHttpServlet.
-    if (cliService.getHiveConf().getVar(
-        ConfVars.HIVE_SERVER2_TRANSPORT_MODE).equalsIgnoreCase("http")) {
-      clientIpAddress = SessionManager.getIpAddress();
-    }
-    else {
-      // Kerberos
-      if (isKerberosAuthMode()) {
-        clientIpAddress = hiveAuthFactory.getIpAddress();
-      }
-      // Except kerberos, NOSASL
-      else {
-        clientIpAddress = TSetIpAddressProcessor.getUserIpAddress();
-      }
-    }
-    LOG.debug("Client's IP Address: " + clientIpAddress);
-    return clientIpAddress;
-  }
-
-  /**
-   * Returns the effective username.
-   * 1. If hive.server2.allow.user.substitution = false: the username of the connecting user
-   * 2. If hive.server2.allow.user.substitution = true: the username of the end user,
-   * that the connecting user is trying to proxy for.
-   * This includes a check whether the connecting user is allowed to proxy for the end user.
-   * @param req
-   * @return
-   * @throws HiveSQLException
-   */
-  private String getUserName(TOpenSessionReq req) throws HiveSQLException {
-    String userName = null;
-    // Kerberos
-    if (isKerberosAuthMode()) {
-      userName = hiveAuthFactory.getRemoteUser();
-    }
-    // Except kerberos, NOSASL
-    if (userName == null) {
-      userName = TSetIpAddressProcessor.getUserName();
-    }
-    // Http transport mode.
-    // We set the thread local username, in ThriftHttpServlet.
-    if (cliService.getHiveConf().getVar(
-        ConfVars.HIVE_SERVER2_TRANSPORT_MODE).equalsIgnoreCase("http")) {
-      userName = SessionManager.getUserName();
-    }
-    if (userName == null) {
-      userName = req.getUsername();
-    }
-
-    userName = getShortName(userName);
-    String effectiveClientUser = getProxyUser(userName, req.getConfiguration(), getIpAddress());
-    LOG.debug("Client's username: " + effectiveClientUser);
-    return effectiveClientUser;
-  }
-
-  private String getShortName(String userName) {
-    String ret = null;
-    if (userName != null) {
-      int indexOfDomainMatch = ServiceUtils.indexOfDomainMatch(userName);
-      ret = (indexOfDomainMatch <= 0) ? userName :
-          userName.substring(0, indexOfDomainMatch);
-    }
-
-    return ret;
-  }
-
-  /**
-   * Create a session handle
-   * @param req
-   * @param res
-   * @return
-   * @throws HiveSQLException
-   * @throws LoginException
-   * @throws IOException
-   */
-  SessionHandle getSessionHandle(TOpenSessionReq req, TOpenSessionResp res)
-      throws HiveSQLException, LoginException, IOException {
-    String userName = getUserName(req);
-    String ipAddress = getIpAddress();
-    TProtocolVersion protocol = getMinVersion(CLIService.SERVER_VERSION,
-        req.getClient_protocol());
-    res.setServerProtocolVersion(protocol);
-    SessionHandle sessionHandle;
-    if (cliService.getHiveConf().getBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS) &&
-        (userName != null)) {
-      String delegationTokenStr = getDelegationToken(userName);
-      sessionHandle = cliService.openSessionWithImpersonation(protocol, userName,
-          req.getPassword(), ipAddress, req.getConfiguration(), delegationTokenStr);
-    } else {
-      sessionHandle = cliService.openSession(protocol, userName, req.getPassword(),
-          ipAddress, req.getConfiguration());
-    }
-    return sessionHandle;
-  }
-
-
-  private String getDelegationToken(String userName)
-      throws HiveSQLException, LoginException, IOException {
-    if (userName == null || !cliService.getHiveConf().getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION)
-        .equalsIgnoreCase(HiveAuthFactory.AuthTypes.KERBEROS.toString())) {
-      return null;
-    }
-    try {
-      return cliService.getDelegationTokenFromMetaStore(userName);
-    } catch (UnsupportedOperationException e) {
-      // The delegation token is not applicable in the given deployment mode
-    }
-    return null;
-  }
-
-  private TProtocolVersion getMinVersion(TProtocolVersion... versions) {
-    TProtocolVersion[] values = TProtocolVersion.values();
-    int current = values[values.length - 1].getValue();
-    for (TProtocolVersion version : versions) {
-      if (current > version.getValue()) {
-        current = version.getValue();
-      }
-    }
-    for (TProtocolVersion version : values) {
-      if (version.getValue() == current) {
-        return version;
-      }
-    }
-    throw new IllegalArgumentException("never");
-  }
-
-  @Override
-  public TCloseSessionResp CloseSession(TCloseSessionReq req) throws TException {
-    TCloseSessionResp resp = new TCloseSessionResp();
-    try {
-      SessionHandle sessionHandle = new SessionHandle(req.getSessionHandle());
-      cliService.closeSession(sessionHandle);
-      resp.setStatus(OK_STATUS);
-      ThriftCLIServerContext context =
-        (ThriftCLIServerContext)currentServerContext.get();
-      if (context != null) {
-        context.setSessionHandle(null);
-      }
-    } catch (Exception e) {
-      LOG.warn("Error closing session: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TGetInfoResp GetInfo(TGetInfoReq req) throws TException {
-    TGetInfoResp resp = new TGetInfoResp();
-    try {
-      GetInfoValue getInfoValue =
-          cliService.getInfo(new SessionHandle(req.getSessionHandle()),
-              GetInfoType.getGetInfoType(req.getInfoType()));
-      resp.setInfoValue(getInfoValue.toTGetInfoValue());
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error getting info: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TExecuteStatementResp ExecuteStatement(TExecuteStatementReq req) throws TException {
-    TExecuteStatementResp resp = new TExecuteStatementResp();
-    try {
-      SessionHandle sessionHandle = new SessionHandle(req.getSessionHandle());
-      String statement = req.getStatement();
-      Map<String, String> confOverlay = req.getConfOverlay();
-      Boolean runAsync = req.isRunAsync();
-      OperationHandle operationHandle = runAsync ?
-          cliService.executeStatementAsync(sessionHandle, statement, confOverlay)
-          : cliService.executeStatement(sessionHandle, statement, confOverlay);
-          resp.setOperationHandle(operationHandle.toTOperationHandle());
-          resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error executing statement: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TGetTypeInfoResp GetTypeInfo(TGetTypeInfoReq req) throws TException {
-    TGetTypeInfoResp resp = new TGetTypeInfoResp();
-    try {
-      OperationHandle operationHandle = cliService.getTypeInfo(new SessionHandle(req.getSessionHandle()));
-      resp.setOperationHandle(operationHandle.toTOperationHandle());
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error getting type info: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TGetCatalogsResp GetCatalogs(TGetCatalogsReq req) throws TException {
-    TGetCatalogsResp resp = new TGetCatalogsResp();
-    try {
-      OperationHandle opHandle = cliService.getCatalogs(new SessionHandle(req.getSessionHandle()));
-      resp.setOperationHandle(opHandle.toTOperationHandle());
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error getting catalogs: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TGetSchemasResp GetSchemas(TGetSchemasReq req) throws TException {
-    TGetSchemasResp resp = new TGetSchemasResp();
-    try {
-      OperationHandle opHandle = cliService.getSchemas(
-          new SessionHandle(req.getSessionHandle()), req.getCatalogName(), req.getSchemaName());
-      resp.setOperationHandle(opHandle.toTOperationHandle());
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error getting schemas: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TGetTablesResp GetTables(TGetTablesReq req) throws TException {
-    TGetTablesResp resp = new TGetTablesResp();
-    try {
-      OperationHandle opHandle = cliService
-          .getTables(new SessionHandle(req.getSessionHandle()), req.getCatalogName(),
-              req.getSchemaName(), req.getTableName(), req.getTableTypes());
-      resp.setOperationHandle(opHandle.toTOperationHandle());
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error getting tables: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TGetTableTypesResp GetTableTypes(TGetTableTypesReq req) throws TException {
-    TGetTableTypesResp resp = new TGetTableTypesResp();
-    try {
-      OperationHandle opHandle = cliService.getTableTypes(new SessionHandle(req.getSessionHandle()));
-      resp.setOperationHandle(opHandle.toTOperationHandle());
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error getting table types: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TGetColumnsResp GetColumns(TGetColumnsReq req) throws TException {
-    TGetColumnsResp resp = new TGetColumnsResp();
-    try {
-      OperationHandle opHandle = cliService.getColumns(
-          new SessionHandle(req.getSessionHandle()),
-          req.getCatalogName(),
-          req.getSchemaName(),
-          req.getTableName(),
-          req.getColumnName());
-      resp.setOperationHandle(opHandle.toTOperationHandle());
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error getting columns: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TGetFunctionsResp GetFunctions(TGetFunctionsReq req) throws TException {
-    TGetFunctionsResp resp = new TGetFunctionsResp();
-    try {
-      OperationHandle opHandle = cliService.getFunctions(
-          new SessionHandle(req.getSessionHandle()), req.getCatalogName(),
-          req.getSchemaName(), req.getFunctionName());
-      resp.setOperationHandle(opHandle.toTOperationHandle());
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error getting functions: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TGetOperationStatusResp GetOperationStatus(TGetOperationStatusReq req) throws TException {
-    TGetOperationStatusResp resp = new TGetOperationStatusResp();
-    try {
-      OperationStatus operationStatus = cliService.getOperationStatus(
-          new OperationHandle(req.getOperationHandle()));
-      resp.setOperationState(operationStatus.getState().toTOperationState());
-      HiveSQLException opException = operationStatus.getOperationException();
-      if (opException != null) {
-        resp.setSqlState(opException.getSQLState());
-        resp.setErrorCode(opException.getErrorCode());
-        resp.setErrorMessage(opException.getMessage());
-      }
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error getting operation status: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TCancelOperationResp CancelOperation(TCancelOperationReq req) throws TException {
-    TCancelOperationResp resp = new TCancelOperationResp();
-    try {
-      cliService.cancelOperation(new OperationHandle(req.getOperationHandle()));
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error cancelling operation: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TCloseOperationResp CloseOperation(TCloseOperationReq req) throws TException {
-    TCloseOperationResp resp = new TCloseOperationResp();
-    try {
-      cliService.closeOperation(new OperationHandle(req.getOperationHandle()));
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error closing operation: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TGetResultSetMetadataResp GetResultSetMetadata(TGetResultSetMetadataReq req)
-      throws TException {
-    TGetResultSetMetadataResp resp = new TGetResultSetMetadataResp();
-    try {
-      TableSchema schema = cliService.getResultSetMetadata(new OperationHandle(req.getOperationHandle()));
-      resp.setSchema(schema.toTTableSchema());
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error getting result set metadata: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TFetchResultsResp FetchResults(TFetchResultsReq req) throws TException {
-    TFetchResultsResp resp = new TFetchResultsResp();
-    try {
-      RowSet rowSet = cliService.fetchResults(
-          new OperationHandle(req.getOperationHandle()),
-          FetchOrientation.getFetchOrientation(req.getOrientation()),
-          req.getMaxRows(),
-          FetchType.getFetchType(req.getFetchType()));
-      resp.setResults(rowSet.toTRowSet());
-      resp.setHasMoreRows(false);
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error fetching results: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public abstract void run();
-
-  /**
-   * If the proxy user name is provided then check privileges to substitute the user.
-   * @param realUser
-   * @param sessionConf
-   * @param ipAddress
-   * @return
-   * @throws HiveSQLException
-   */
-  private String getProxyUser(String realUser, Map<String, String> sessionConf,
-      String ipAddress) throws HiveSQLException {
-    String proxyUser = null;
-    // Http transport mode.
-    // We set the thread local proxy username, in ThriftHttpServlet.
-    if (cliService.getHiveConf().getVar(
-        ConfVars.HIVE_SERVER2_TRANSPORT_MODE).equalsIgnoreCase("http")) {
-      proxyUser = SessionManager.getProxyUserName();
-      LOG.debug("Proxy user from query string: " + proxyUser);
-    }
-
-    if (proxyUser == null && sessionConf != null && sessionConf.containsKey(HiveAuthFactory.HS2_PROXY_USER)) {
-      String proxyUserFromThriftBody = sessionConf.get(HiveAuthFactory.HS2_PROXY_USER);
-      LOG.debug("Proxy user from thrift body: " + proxyUserFromThriftBody);
-      proxyUser = proxyUserFromThriftBody;
-    }
-
-    if (proxyUser == null) {
-      return realUser;
-    }
-
-    // check whether substitution is allowed
-    if (!hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ALLOW_USER_SUBSTITUTION)) {
-      throw new HiveSQLException("Proxy user substitution is not allowed");
-    }
-
-    // If there's no authentication, then directly substitute the user
-    if (HiveAuthFactory.AuthTypes.NONE.toString()
-        .equalsIgnoreCase(hiveConf.getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION))) {
-      return proxyUser;
-    }
-
-    // Verify proxy user privilege of the realUser for the proxyUser
-    HiveAuthFactory.verifyProxyAccess(realUser, proxyUser, ipAddress, hiveConf);
-    LOG.debug("Verified proxy user: " + proxyUser);
-    return proxyUser;
-  }
-
-  private boolean isKerberosAuthMode() {
-    return cliService.getHiveConf().getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION)
-        .equalsIgnoreCase(HiveAuthFactory.AuthTypes.KERBEROS.toString());
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIServiceClient.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIServiceClient.java
deleted file mode 100644
index 1af45398b895c..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIServiceClient.java
+++ /dev/null
@@ -1,440 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.thrift;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hive.service.auth.HiveAuthFactory;
-import org.apache.hive.service.cli.*;
-import org.apache.thrift.TException;
-
-/**
- * ThriftCLIServiceClient.
- *
- */
-public class ThriftCLIServiceClient extends CLIServiceClient {
-  private final TCLIService.Iface cliService;
-
-  public ThriftCLIServiceClient(TCLIService.Iface cliService) {
-    this.cliService = cliService;
-  }
-
-  public void checkStatus(TStatus status) throws HiveSQLException {
-    if (TStatusCode.ERROR_STATUS.equals(status.getStatusCode())) {
-      throw new HiveSQLException(status);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#openSession(java.lang.String, java.lang.String, java.util.Map)
-   */
-  @Override
-  public SessionHandle openSession(String username, String password,
-      Map<String, String> configuration)
-          throws HiveSQLException {
-    try {
-      TOpenSessionReq req = new TOpenSessionReq();
-      req.setUsername(username);
-      req.setPassword(password);
-      req.setConfiguration(configuration);
-      TOpenSessionResp resp = cliService.OpenSession(req);
-      checkStatus(resp.getStatus());
-      return new SessionHandle(resp.getSessionHandle(), resp.getServerProtocolVersion());
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#closeSession(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public SessionHandle openSessionWithImpersonation(String username, String password,
-      Map<String, String> configuration, String delegationToken) throws HiveSQLException {
-    throw new HiveSQLException("open with impersonation operation is not supported in the client");
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#closeSession(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public void closeSession(SessionHandle sessionHandle) throws HiveSQLException {
-    try {
-      TCloseSessionReq req = new TCloseSessionReq(sessionHandle.toTSessionHandle());
-      TCloseSessionResp resp = cliService.CloseSession(req);
-      checkStatus(resp.getStatus());
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getInfo(org.apache.hive.service.cli.SessionHandle, java.util.List)
-   */
-  @Override
-  public GetInfoValue getInfo(SessionHandle sessionHandle, GetInfoType infoType)
-      throws HiveSQLException {
-    try {
-      // FIXME extract the right info type
-      TGetInfoReq req = new TGetInfoReq(sessionHandle.toTSessionHandle(), infoType.toTGetInfoType());
-      TGetInfoResp resp = cliService.GetInfo(req);
-      checkStatus(resp.getStatus());
-      return new GetInfoValue(resp.getInfoValue());
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#executeStatement(org.apache.hive.service.cli.SessionHandle, java.lang.String, java.util.Map)
-   */
-  @Override
-  public OperationHandle executeStatement(SessionHandle sessionHandle, String statement,
-      Map<String, String> confOverlay)
-          throws HiveSQLException {
-    return executeStatementInternal(sessionHandle, statement, confOverlay, false);
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#executeStatementAsync(org.apache.hive.service.cli.SessionHandle, java.lang.String, java.util.Map)
-   */
-  @Override
-  public OperationHandle executeStatementAsync(SessionHandle sessionHandle, String statement,
-      Map<String, String> confOverlay)
-          throws HiveSQLException {
-    return executeStatementInternal(sessionHandle, statement, confOverlay, true);
-  }
-
-  private OperationHandle executeStatementInternal(SessionHandle sessionHandle, String statement,
-      Map<String, String> confOverlay, boolean isAsync)
-          throws HiveSQLException {
-    try {
-      TExecuteStatementReq req =
-          new TExecuteStatementReq(sessionHandle.toTSessionHandle(), statement);
-      req.setConfOverlay(confOverlay);
-      req.setRunAsync(isAsync);
-      TExecuteStatementResp resp = cliService.ExecuteStatement(req);
-      checkStatus(resp.getStatus());
-      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
-      return new OperationHandle(resp.getOperationHandle(), protocol);
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getTypeInfo(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public OperationHandle getTypeInfo(SessionHandle sessionHandle) throws HiveSQLException {
-    try {
-      TGetTypeInfoReq req = new TGetTypeInfoReq(sessionHandle.toTSessionHandle());
-      TGetTypeInfoResp resp = cliService.GetTypeInfo(req);
-      checkStatus(resp.getStatus());
-      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
-      return new OperationHandle(resp.getOperationHandle(), protocol);
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getCatalogs(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public OperationHandle getCatalogs(SessionHandle sessionHandle) throws HiveSQLException {
-    try {
-      TGetCatalogsReq req = new TGetCatalogsReq(sessionHandle.toTSessionHandle());
-      TGetCatalogsResp resp = cliService.GetCatalogs(req);
-      checkStatus(resp.getStatus());
-      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
-      return new OperationHandle(resp.getOperationHandle(), protocol);
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getSchemas(org.apache.hive.service.cli.SessionHandle, java.lang.String, java.lang.String)
-   */
-  @Override
-  public OperationHandle getSchemas(SessionHandle sessionHandle, String catalogName,
-      String schemaName)
-          throws HiveSQLException {
-    try {
-      TGetSchemasReq req = new TGetSchemasReq(sessionHandle.toTSessionHandle());
-      req.setCatalogName(catalogName);
-      req.setSchemaName(schemaName);
-      TGetSchemasResp resp = cliService.GetSchemas(req);
-      checkStatus(resp.getStatus());
-      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
-      return new OperationHandle(resp.getOperationHandle(), protocol);
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getTables(org.apache.hive.service.cli.SessionHandle, java.lang.String, java.lang.String, java.lang.String, java.util.List)
-   */
-  @Override
-  public OperationHandle getTables(SessionHandle sessionHandle, String catalogName,
-      String schemaName, String tableName, List<String> tableTypes)
-          throws HiveSQLException {
-    try {
-      TGetTablesReq req = new TGetTablesReq(sessionHandle.toTSessionHandle());
-      req.setTableName(tableName);
-      req.setTableTypes(tableTypes);
-      req.setSchemaName(schemaName);
-      TGetTablesResp resp = cliService.GetTables(req);
-      checkStatus(resp.getStatus());
-      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
-      return new OperationHandle(resp.getOperationHandle(), protocol);
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getTableTypes(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public OperationHandle getTableTypes(SessionHandle sessionHandle) throws HiveSQLException {
-    try {
-      TGetTableTypesReq req = new TGetTableTypesReq(sessionHandle.toTSessionHandle());
-      TGetTableTypesResp resp = cliService.GetTableTypes(req);
-      checkStatus(resp.getStatus());
-      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
-      return new OperationHandle(resp.getOperationHandle(), protocol);
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getColumns(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public OperationHandle getColumns(SessionHandle sessionHandle,
-      String catalogName, String schemaName, String tableName, String columnName)
-          throws HiveSQLException {
-    try {
-      TGetColumnsReq req = new TGetColumnsReq();
-      req.setSessionHandle(sessionHandle.toTSessionHandle());
-      req.setCatalogName(catalogName);
-      req.setSchemaName(schemaName);
-      req.setTableName(tableName);
-      req.setColumnName(columnName);
-      TGetColumnsResp resp = cliService.GetColumns(req);
-      checkStatus(resp.getStatus());
-      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
-      return new OperationHandle(resp.getOperationHandle(), protocol);
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getFunctions(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public OperationHandle getFunctions(SessionHandle sessionHandle,
-      String catalogName, String schemaName, String functionName) throws HiveSQLException {
-    try {
-      TGetFunctionsReq req = new TGetFunctionsReq(sessionHandle.toTSessionHandle(), functionName);
-      req.setCatalogName(catalogName);
-      req.setSchemaName(schemaName);
-      TGetFunctionsResp resp = cliService.GetFunctions(req);
-      checkStatus(resp.getStatus());
-      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
-      return new OperationHandle(resp.getOperationHandle(), protocol);
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getOperationStatus(org.apache.hive.service.cli.OperationHandle)
-   */
-  @Override
-  public OperationStatus getOperationStatus(OperationHandle opHandle) throws HiveSQLException {
-    try {
-      TGetOperationStatusReq req = new TGetOperationStatusReq(opHandle.toTOperationHandle());
-      TGetOperationStatusResp resp = cliService.GetOperationStatus(req);
-      // Checks the status of the RPC call, throws an exception in case of error
-      checkStatus(resp.getStatus());
-      OperationState opState = OperationState.getOperationState(resp.getOperationState());
-      HiveSQLException opException = null;
-      if (opState == OperationState.ERROR) {
-        opException = new HiveSQLException(resp.getErrorMessage(), resp.getSqlState(), resp.getErrorCode());
-      }
-      return new OperationStatus(opState, opException);
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#cancelOperation(org.apache.hive.service.cli.OperationHandle)
-   */
-  @Override
-  public void cancelOperation(OperationHandle opHandle) throws HiveSQLException {
-    try {
-      TCancelOperationReq req = new TCancelOperationReq(opHandle.toTOperationHandle());
-      TCancelOperationResp resp = cliService.CancelOperation(req);
-      checkStatus(resp.getStatus());
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#closeOperation(org.apache.hive.service.cli.OperationHandle)
-   */
-  @Override
-  public void closeOperation(OperationHandle opHandle)
-      throws HiveSQLException {
-    try {
-      TCloseOperationReq req  = new TCloseOperationReq(opHandle.toTOperationHandle());
-      TCloseOperationResp resp = cliService.CloseOperation(req);
-      checkStatus(resp.getStatus());
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getResultSetMetadata(org.apache.hive.service.cli.OperationHandle)
-   */
-  @Override
-  public TableSchema getResultSetMetadata(OperationHandle opHandle)
-      throws HiveSQLException {
-    try {
-      TGetResultSetMetadataReq req = new TGetResultSetMetadataReq(opHandle.toTOperationHandle());
-      TGetResultSetMetadataResp resp = cliService.GetResultSetMetadata(req);
-      checkStatus(resp.getStatus());
-      return new TableSchema(resp.getSchema());
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  @Override
-  public RowSet fetchResults(OperationHandle opHandle, FetchOrientation orientation, long maxRows,
-      FetchType fetchType) throws HiveSQLException {
-    try {
-      TFetchResultsReq req = new TFetchResultsReq();
-      req.setOperationHandle(opHandle.toTOperationHandle());
-      req.setOrientation(orientation.toTFetchOrientation());
-      req.setMaxRows(maxRows);
-      req.setFetchType(fetchType.toTFetchType());
-      TFetchResultsResp resp = cliService.FetchResults(req);
-      checkStatus(resp.getStatus());
-      return RowSetFactory.create(resp.getResults(), opHandle.getProtocolVersion());
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#fetchResults(org.apache.hive.service.cli.OperationHandle)
-   */
-  @Override
-  public RowSet fetchResults(OperationHandle opHandle) throws HiveSQLException {
-    // TODO: set the correct default fetch size
-    return fetchResults(opHandle, FetchOrientation.FETCH_NEXT, 10000, FetchType.QUERY_OUTPUT);
-  }
-
-  @Override
-  public String getDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
-      String owner, String renewer) throws HiveSQLException {
-    TGetDelegationTokenReq req = new TGetDelegationTokenReq(
-        sessionHandle.toTSessionHandle(), owner, renewer);
-    try {
-      TGetDelegationTokenResp tokenResp = cliService.GetDelegationToken(req);
-      checkStatus(tokenResp.getStatus());
-      return tokenResp.getDelegationToken();
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  @Override
-  public void cancelDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
-      String tokenStr) throws HiveSQLException {
-    TCancelDelegationTokenReq cancelReq = new TCancelDelegationTokenReq(
-          sessionHandle.toTSessionHandle(), tokenStr);
-    try {
-      TCancelDelegationTokenResp cancelResp =
-        cliService.CancelDelegationToken(cancelReq);
-      checkStatus(cancelResp.getStatus());
-      return;
-    } catch (TException e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  @Override
-  public void renewDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
-      String tokenStr) throws HiveSQLException {
-    TRenewDelegationTokenReq cancelReq = new TRenewDelegationTokenReq(
-        sessionHandle.toTSessionHandle(), tokenStr);
-    try {
-      TRenewDelegationTokenResp renewResp =
-        cliService.RenewDelegationToken(cancelReq);
-      checkStatus(renewResp.getStatus());
-      return;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
deleted file mode 100644
index 504e63dbc5e5e..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.thrift;
-
-import java.util.Arrays;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.Shell;
-import org.apache.hive.service.auth.HiveAuthFactory;
-import org.apache.hive.service.cli.CLIService;
-import org.apache.hive.service.cli.thrift.TCLIService.Iface;
-import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup;
-import org.apache.thrift.TProcessor;
-import org.apache.thrift.protocol.TBinaryProtocol;
-import org.apache.thrift.protocol.TProtocolFactory;
-import org.apache.thrift.server.TServlet;
-import org.eclipse.jetty.server.AbstractConnectionFactory;
-import org.eclipse.jetty.server.ConnectionFactory;
-import org.eclipse.jetty.server.HttpConnectionFactory;
-import org.eclipse.jetty.server.ServerConnector;
-import org.eclipse.jetty.servlet.ServletContextHandler;
-import org.eclipse.jetty.servlet.ServletHolder;
-import org.eclipse.jetty.util.ssl.SslContextFactory;
-import org.eclipse.jetty.util.thread.ExecutorThreadPool;
-import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler;
-
-
-public class ThriftHttpCLIService extends ThriftCLIService {
-
-  public ThriftHttpCLIService(CLIService cliService) {
-    super(cliService, ThriftHttpCLIService.class.getSimpleName());
-  }
-
-  /**
-   * Configure Jetty to serve http requests. Example of a client connection URL:
-   * http://localhost:10000/servlets/thrifths2/ A gateway may cause actual target URL to differ,
-   * e.g. http://gateway:port/hive2/servlets/thrifths2/
-   */
-  @Override
-  public void run() {
-    try {
-      // Server thread pool
-      // Start with minWorkerThreads, expand till maxWorkerThreads and reject subsequent requests
-      String threadPoolName = "HiveServer2-HttpHandler-Pool";
-      ThreadPoolExecutor executorService = new ThreadPoolExecutor(minWorkerThreads, maxWorkerThreads,
-          workerKeepAliveTime, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
-          new ThreadFactoryWithGarbageCleanup(threadPoolName));
-      ExecutorThreadPool threadPool = new ExecutorThreadPool(executorService);
-
-      // HTTP Server
-      httpServer = new org.eclipse.jetty.server.Server(threadPool);
-
-      // Connector configs
-
-      ConnectionFactory[] connectionFactories;
-      boolean useSsl = hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_USE_SSL);
-      String schemeName = useSsl ? "https" : "http";
-      // Change connector if SSL is used
-      if (useSsl) {
-        String keyStorePath = hiveConf.getVar(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH).trim();
-        String keyStorePassword = ShimLoader.getHadoopShims().getPassword(hiveConf,
-            HiveConf.ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname);
-        if (keyStorePath.isEmpty()) {
-          throw new IllegalArgumentException(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname
-              + " Not configured for SSL connection");
-        }
-        SslContextFactory sslContextFactory = new SslContextFactory.Server();
-        String[] excludedProtocols = hiveConf.getVar(ConfVars.HIVE_SSL_PROTOCOL_BLACKLIST).split(",");
-        LOG.info("HTTP Server SSL: adding excluded protocols: " + Arrays.toString(excludedProtocols));
-        sslContextFactory.addExcludeProtocols(excludedProtocols);
-        LOG.info("HTTP Server SSL: SslContextFactory.getExcludeProtocols = " +
-          Arrays.toString(sslContextFactory.getExcludeProtocols()));
-        sslContextFactory.setKeyStorePath(keyStorePath);
-        sslContextFactory.setKeyStorePassword(keyStorePassword);
-        connectionFactories = AbstractConnectionFactory.getFactories(
-            sslContextFactory, new HttpConnectionFactory());
-      } else {
-        connectionFactories = new ConnectionFactory[] { new HttpConnectionFactory() };
-      }
-      ServerConnector connector = new ServerConnector(
-          httpServer,
-          null,
-          // Call this full constructor to set this, which forces daemon threads:
-          new ScheduledExecutorScheduler("HiveServer2-HttpHandler-JettyScheduler", true),
-          null,
-          -1,
-          -1,
-          connectionFactories);
-
-      connector.setPort(portNum);
-      // Linux:yes, Windows:no
-      connector.setReuseAddress(!Shell.WINDOWS);
-      int maxIdleTime = (int) hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_MAX_IDLE_TIME,
-          TimeUnit.MILLISECONDS);
-      connector.setIdleTimeout(maxIdleTime);
-
-      httpServer.addConnector(connector);
-
-      // Thrift configs
-      hiveAuthFactory = new HiveAuthFactory(hiveConf);
-      TProcessor processor = new TCLIService.Processor<Iface>(this);
-      TProtocolFactory protocolFactory = new TBinaryProtocol.Factory();
-      // Set during the init phase of HiveServer2 if auth mode is kerberos
-      // UGI for the hive/_HOST (kerberos) principal
-      UserGroupInformation serviceUGI = cliService.getServiceUGI();
-      // UGI for the http/_HOST (SPNego) principal
-      UserGroupInformation httpUGI = cliService.getHttpUGI();
-      String authType = hiveConf.getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION);
-      TServlet thriftHttpServlet = new ThriftHttpServlet(processor, protocolFactory, authType,
-          serviceUGI, httpUGI);
-
-      // Context handler
-      final ServletContextHandler context = new ServletContextHandler(
-          ServletContextHandler.SESSIONS);
-      context.setContextPath("/");
-      String httpPath = getHttpPath(hiveConf
-          .getVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH));
-      httpServer.setHandler(context);
-      context.addServlet(new ServletHolder(thriftHttpServlet), httpPath);
-
-      // TODO: check defaults: maxTimeout, keepalive, maxBodySize, bodyRecieveDuration, etc.
-      // Finally, start the server
-      httpServer.start();
-      String msg = "Started " + ThriftHttpCLIService.class.getSimpleName() + " in " + schemeName
-          + " mode on port " + connector.getLocalPort()+ " path=" + httpPath + " with " + minWorkerThreads + "..."
-          + maxWorkerThreads + " worker threads";
-      LOG.info(msg);
-      httpServer.join();
-    } catch (Throwable t) {
-      LOG.fatal(
-          "Error starting HiveServer2: could not start "
-              + ThriftHttpCLIService.class.getSimpleName(), t);
-      System.exit(-1);
-    }
-  }
-
-  /**
-   * The config parameter can be like "path", "/path", "/path/", "path/*", "/path1/path2/*" and so on.
-   * httpPath should end up as "/*", "/path/*" or "/path1/../pathN/*"
-   * @param httpPath
-   * @return
-   */
-  private String getHttpPath(String httpPath) {
-    if(httpPath == null || httpPath.equals("")) {
-      httpPath = "/*";
-    }
-    else {
-      if(!httpPath.startsWith("/")) {
-        httpPath = "/" + httpPath;
-      }
-      if(httpPath.endsWith("/")) {
-        httpPath = httpPath + "*";
-      }
-      if(!httpPath.endsWith("/*")) {
-        httpPath = httpPath + "/*";
-      }
-    }
-    return httpPath;
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
deleted file mode 100644
index e15d2d0566d2b..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
+++ /dev/null
@@ -1,545 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.thrift;
-
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.security.PrivilegedExceptionAction;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.Cookie;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import javax.ws.rs.core.NewCookie;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.codec.binary.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.shims.HadoopShims.KerberosNameShim;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hive.service.auth.AuthenticationProviderFactory;
-import org.apache.hive.service.auth.AuthenticationProviderFactory.AuthMethods;
-import org.apache.hive.service.auth.HiveAuthFactory;
-import org.apache.hive.service.auth.HttpAuthUtils;
-import org.apache.hive.service.auth.HttpAuthenticationException;
-import org.apache.hive.service.auth.PasswdAuthenticationProvider;
-import org.apache.hive.service.cli.session.SessionManager;
-import org.apache.hive.service.CookieSigner;
-import org.apache.thrift.TProcessor;
-import org.apache.thrift.protocol.TProtocolFactory;
-import org.apache.thrift.server.TServlet;
-import org.ietf.jgss.GSSContext;
-import org.ietf.jgss.GSSCredential;
-import org.ietf.jgss.GSSException;
-import org.ietf.jgss.GSSManager;
-import org.ietf.jgss.GSSName;
-import org.ietf.jgss.Oid;
-
-/**
- *
- * ThriftHttpServlet
- *
- */
-public class ThriftHttpServlet extends TServlet {
-
-  private static final long serialVersionUID = 1L;
-  public static final Log LOG = LogFactory.getLog(ThriftHttpServlet.class.getName());
-  private final String authType;
-  private final UserGroupInformation serviceUGI;
-  private final UserGroupInformation httpUGI;
-  private HiveConf hiveConf = new HiveConf();
-
-  // Class members for cookie based authentication.
-  private CookieSigner signer;
-  public static final String AUTH_COOKIE = "hive.server2.auth";
-  private static final Random RAN = new Random();
-  private boolean isCookieAuthEnabled;
-  private String cookieDomain;
-  private String cookiePath;
-  private int cookieMaxAge;
-  private boolean isCookieSecure;
-  private boolean isHttpOnlyCookie;
-
-  public ThriftHttpServlet(TProcessor processor, TProtocolFactory protocolFactory,
-      String authType, UserGroupInformation serviceUGI, UserGroupInformation httpUGI) {
-    super(processor, protocolFactory);
-    this.authType = authType;
-    this.serviceUGI = serviceUGI;
-    this.httpUGI = httpUGI;
-    this.isCookieAuthEnabled = hiveConf.getBoolVar(
-      ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_AUTH_ENABLED);
-    // Initialize the cookie based authentication related variables.
-    if (isCookieAuthEnabled) {
-      // Generate the signer with secret.
-      String secret = Long.toString(RAN.nextLong());
-      LOG.debug("Using the random number as the secret for cookie generation " + secret);
-      this.signer = new CookieSigner(secret.getBytes());
-      this.cookieMaxAge = (int) hiveConf.getTimeVar(
-        ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_MAX_AGE, TimeUnit.SECONDS);
-      this.cookieDomain = hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_DOMAIN);
-      this.cookiePath = hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_PATH);
-      this.isCookieSecure = hiveConf.getBoolVar(
-        ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_IS_SECURE);
-      this.isHttpOnlyCookie = hiveConf.getBoolVar(
-        ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_IS_HTTPONLY);
-    }
-  }
-
-  @Override
-  protected void doPost(HttpServletRequest request, HttpServletResponse response)
-      throws ServletException, IOException {
-    String clientUserName = null;
-    String clientIpAddress;
-    boolean requireNewCookie = false;
-
-    try {
-      // If the cookie based authentication is already enabled, parse the
-      // request and validate the request cookies.
-      if (isCookieAuthEnabled) {
-        clientUserName = validateCookie(request);
-        requireNewCookie = (clientUserName == null);
-        if (requireNewCookie) {
-          LOG.info("Could not validate cookie sent, will try to generate a new cookie");
-        }
-      }
-      // If the cookie based authentication is not enabled or the request does
-      // not have a valid cookie, use the kerberos or password based authentication
-      // depending on the server setup.
-      if (clientUserName == null) {
-        // For a kerberos setup
-        if (isKerberosAuthMode(authType)) {
-          clientUserName = doKerberosAuth(request);
-        }
-        // For password based authentication
-        else {
-          clientUserName = doPasswdAuth(request, authType);
-        }
-      }
-      LOG.debug("Client username: " + clientUserName);
-
-      // Set the thread local username to be used for doAs if true
-      SessionManager.setUserName(clientUserName);
-
-      // find proxy user if any from query param
-      String doAsQueryParam = getDoAsQueryParam(request.getQueryString());
-      if (doAsQueryParam != null) {
-        SessionManager.setProxyUserName(doAsQueryParam);
-      }
-
-      clientIpAddress = request.getRemoteAddr();
-      LOG.debug("Client IP Address: " + clientIpAddress);
-      // Set the thread local ip address
-      SessionManager.setIpAddress(clientIpAddress);
-      // Generate new cookie and add it to the response
-      if (requireNewCookie &&
-          !authType.equalsIgnoreCase(HiveAuthFactory.AuthTypes.NOSASL.toString())) {
-        String cookieToken = HttpAuthUtils.createCookieToken(clientUserName);
-        Cookie hs2Cookie = createCookie(signer.signCookie(cookieToken));
-
-        if (isHttpOnlyCookie) {
-          response.setHeader("SET-COOKIE", getHttpOnlyCookieHeader(hs2Cookie));
-        } else {
-          response.addCookie(hs2Cookie);
-        }
-        LOG.info("Cookie added for clientUserName " + clientUserName);
-      }
-      super.doPost(request, response);
-    }
-    catch (HttpAuthenticationException e) {
-      LOG.error("Error: ", e);
-      // Send a 401 to the client
-      response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
-      if(isKerberosAuthMode(authType)) {
-        response.addHeader(HttpAuthUtils.WWW_AUTHENTICATE, HttpAuthUtils.NEGOTIATE);
-      }
-      response.getWriter().println("Authentication Error: " + e.getMessage());
-    }
-    finally {
-      // Clear the thread locals
-      SessionManager.clearUserName();
-      SessionManager.clearIpAddress();
-      SessionManager.clearProxyUserName();
-    }
-  }
-
-  /**
-   * Retrieves the client name from cookieString. If the cookie does not
-   * correspond to a valid client, the function returns null.
-   * @param cookies HTTP Request cookies.
-   * @return Client Username if cookieString has a HS2 Generated cookie that is currently valid.
-   * Else, returns null.
-   */
-  private String getClientNameFromCookie(Cookie[] cookies) {
-    // Current Cookie Name, Current Cookie Value
-    String currName, currValue;
-
-    // Following is the main loop which iterates through all the cookies send by the client.
-    // The HS2 generated cookies are of the format hive.server2.auth=<value>
-    // A cookie which is identified as a hiveserver2 generated cookie is validated
-    // by calling signer.verifyAndExtract(). If the validation passes, send the
-    // username for which the cookie is validated to the caller. If no client side
-    // cookie passes the validation, return null to the caller.
-    for (Cookie currCookie : cookies) {
-      // Get the cookie name
-      currName = currCookie.getName();
-      if (!currName.equals(AUTH_COOKIE)) {
-        // Not a HS2 generated cookie, continue.
-        continue;
-      }
-      // If we reached here, we have match for HS2 generated cookie
-      currValue = currCookie.getValue();
-      // Validate the value.
-      currValue = signer.verifyAndExtract(currValue);
-      // Retrieve the user name, do the final validation step.
-      if (currValue != null) {
-        String userName = HttpAuthUtils.getUserNameFromCookieToken(currValue);
-
-        if (userName == null) {
-          LOG.warn("Invalid cookie token " + currValue);
-          continue;
-        }
-        //We have found a valid cookie in the client request.
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Validated the cookie for user " + userName);
-        }
-        return userName;
-      }
-    }
-    // No valid HS2 generated cookies found, return null
-    return null;
-  }
-
-  /**
-   * Convert cookie array to human readable cookie string
-   * @param cookies Cookie Array
-   * @return String containing all the cookies separated by a newline character.
-   * Each cookie is of the format [key]=[value]
-   */
-  private String toCookieStr(Cookie[] cookies) {
-    String cookieStr = "";
-
-    for (Cookie c : cookies) {
-     cookieStr += c.getName() + "=" + c.getValue() + " ;\n";
-    }
-    return cookieStr;
-  }
-
-  /**
-   * Validate the request cookie. This function iterates over the request cookie headers
-   * and finds a cookie that represents a valid client/server session. If it finds one, it
-   * returns the client name associated with the session. Else, it returns null.
-   * @param request The HTTP Servlet Request send by the client
-   * @return Client Username if the request has valid HS2 cookie, else returns null
-   * @throws UnsupportedEncodingException
-   */
-  private String validateCookie(HttpServletRequest request) throws UnsupportedEncodingException {
-    // Find all the valid cookies associated with the request.
-    Cookie[] cookies = request.getCookies();
-
-    if (cookies == null) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("No valid cookies associated with the request " + request);
-      }
-      return null;
-    }
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Received cookies: " + toCookieStr(cookies));
-    }
-    return getClientNameFromCookie(cookies);
-  }
-
-  /**
-   * Generate a server side cookie given the cookie value as the input.
-   * @param str Input string token.
-   * @return The generated cookie.
-   * @throws UnsupportedEncodingException
-   */
-  private Cookie createCookie(String str) throws UnsupportedEncodingException {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Cookie name = " + AUTH_COOKIE + " value = " + str);
-    }
-    Cookie cookie = new Cookie(AUTH_COOKIE, str);
-
-    cookie.setMaxAge(cookieMaxAge);
-    if (cookieDomain != null) {
-      cookie.setDomain(cookieDomain);
-    }
-    if (cookiePath != null) {
-      cookie.setPath(cookiePath);
-    }
-    cookie.setSecure(isCookieSecure);
-    return cookie;
-  }
-
-  /**
-   * Generate httponly cookie from HS2 cookie
-   * @param cookie HS2 generated cookie
-   * @return The httponly cookie
-   */
-  private static String getHttpOnlyCookieHeader(Cookie cookie) {
-    NewCookie newCookie = new NewCookie(cookie.getName(), cookie.getValue(),
-      cookie.getPath(), cookie.getDomain(), cookie.getVersion(),
-      cookie.getComment(), cookie.getMaxAge(), cookie.getSecure());
-    return newCookie + "; HttpOnly";
-  }
-
-  /**
-   * Do the LDAP/PAM authentication
-   * @param request
-   * @param authType
-   * @throws HttpAuthenticationException
-   */
-  private String doPasswdAuth(HttpServletRequest request, String authType)
-      throws HttpAuthenticationException {
-    String userName = getUsername(request, authType);
-    // No-op when authType is NOSASL
-    if (!authType.equalsIgnoreCase(HiveAuthFactory.AuthTypes.NOSASL.toString())) {
-      try {
-        AuthMethods authMethod = AuthMethods.getValidAuthMethod(authType);
-        PasswdAuthenticationProvider provider =
-            AuthenticationProviderFactory.getAuthenticationProvider(authMethod);
-        provider.Authenticate(userName, getPassword(request, authType));
-
-      } catch (Exception e) {
-        throw new HttpAuthenticationException(e);
-      }
-    }
-    return userName;
-  }
-
-  /**
-   * Do the GSS-API kerberos authentication.
-   * We already have a logged in subject in the form of serviceUGI,
-   * which GSS-API will extract information from.
-   * In case of a SPNego request we use the httpUGI,
-   * for the authenticating service tickets.
-   * @param request
-   * @return
-   * @throws HttpAuthenticationException
-   */
-  private String doKerberosAuth(HttpServletRequest request)
-      throws HttpAuthenticationException {
-    // Try authenticating with the http/_HOST principal
-    if (httpUGI != null) {
-      try {
-        return httpUGI.doAs(new HttpKerberosServerAction(request, httpUGI));
-      } catch (Exception e) {
-        LOG.info("Failed to authenticate with http/_HOST kerberos principal, " +
-            "trying with hive/_HOST kerberos principal");
-      }
-    }
-    // Now try with hive/_HOST principal
-    try {
-      return serviceUGI.doAs(new HttpKerberosServerAction(request, serviceUGI));
-    } catch (Exception e) {
-      LOG.error("Failed to authenticate with hive/_HOST kerberos principal");
-      throw new HttpAuthenticationException(e);
-    }
-
-  }
-
-  class HttpKerberosServerAction implements PrivilegedExceptionAction<String> {
-    HttpServletRequest request;
-    UserGroupInformation serviceUGI;
-
-    HttpKerberosServerAction(HttpServletRequest request,
-        UserGroupInformation serviceUGI) {
-      this.request = request;
-      this.serviceUGI = serviceUGI;
-    }
-
-    @Override
-    public String run() throws HttpAuthenticationException {
-      // Get own Kerberos credentials for accepting connection
-      GSSManager manager = GSSManager.getInstance();
-      GSSContext gssContext = null;
-      String serverPrincipal = getPrincipalWithoutRealm(
-          serviceUGI.getUserName());
-      try {
-        // This Oid for Kerberos GSS-API mechanism.
-        Oid kerberosMechOid = new Oid("1.2.840.113554.1.2.2");
-        // Oid for SPNego GSS-API mechanism.
-        Oid spnegoMechOid = new Oid("1.3.6.1.5.5.2");
-        // Oid for kerberos principal name
-        Oid krb5PrincipalOid = new Oid("1.2.840.113554.1.2.2.1");
-
-        // GSS name for server
-        GSSName serverName = manager.createName(serverPrincipal, krb5PrincipalOid);
-
-        // GSS credentials for server
-        GSSCredential serverCreds = manager.createCredential(serverName,
-            GSSCredential.DEFAULT_LIFETIME,
-            new Oid[]{kerberosMechOid, spnegoMechOid},
-            GSSCredential.ACCEPT_ONLY);
-
-        // Create a GSS context
-        gssContext = manager.createContext(serverCreds);
-        // Get service ticket from the authorization header
-        String serviceTicketBase64 = getAuthHeader(request, authType);
-        byte[] inToken = Base64.decodeBase64(serviceTicketBase64.getBytes());
-        gssContext.acceptSecContext(inToken, 0, inToken.length);
-        // Authenticate or deny based on its context completion
-        if (!gssContext.isEstablished()) {
-          throw new HttpAuthenticationException("Kerberos authentication failed: " +
-              "unable to establish context with the service ticket " +
-              "provided by the client.");
-        }
-        else {
-          return getPrincipalWithoutRealmAndHost(gssContext.getSrcName().toString());
-        }
-      }
-      catch (GSSException e) {
-        throw new HttpAuthenticationException("Kerberos authentication failed: ", e);
-      }
-      finally {
-        if (gssContext != null) {
-          try {
-            gssContext.dispose();
-          } catch (GSSException e) {
-            // No-op
-          }
-        }
-      }
-    }
-
-    private String getPrincipalWithoutRealm(String fullPrincipal)
-        throws HttpAuthenticationException {
-      KerberosNameShim fullKerberosName;
-      try {
-        fullKerberosName = ShimLoader.getHadoopShims().getKerberosNameShim(fullPrincipal);
-      } catch (IOException e) {
-        throw new HttpAuthenticationException(e);
-      }
-      String serviceName = fullKerberosName.getServiceName();
-      String hostName = fullKerberosName.getHostName();
-      String principalWithoutRealm = serviceName;
-      if (hostName != null) {
-        principalWithoutRealm = serviceName + "/" + hostName;
-      }
-      return principalWithoutRealm;
-    }
-
-    private String getPrincipalWithoutRealmAndHost(String fullPrincipal)
-        throws HttpAuthenticationException {
-      KerberosNameShim fullKerberosName;
-      try {
-        fullKerberosName = ShimLoader.getHadoopShims().getKerberosNameShim(fullPrincipal);
-        return fullKerberosName.getShortName();
-      } catch (IOException e) {
-        throw new HttpAuthenticationException(e);
-      }
-    }
-  }
-
-  private String getUsername(HttpServletRequest request, String authType)
-      throws HttpAuthenticationException {
-    String[] creds = getAuthHeaderTokens(request, authType);
-    // Username must be present
-    if (creds[0] == null || creds[0].isEmpty()) {
-      throw new HttpAuthenticationException("Authorization header received " +
-          "from the client does not contain username.");
-    }
-    return creds[0];
-  }
-
-  private String getPassword(HttpServletRequest request, String authType)
-      throws HttpAuthenticationException {
-    String[] creds = getAuthHeaderTokens(request, authType);
-    // Password must be present
-    if (creds[1] == null || creds[1].isEmpty()) {
-      throw new HttpAuthenticationException("Authorization header received " +
-          "from the client does not contain username.");
-    }
-    return creds[1];
-  }
-
-  private String[] getAuthHeaderTokens(HttpServletRequest request,
-      String authType) throws HttpAuthenticationException {
-    String authHeaderBase64 = getAuthHeader(request, authType);
-    String authHeaderString = StringUtils.newStringUtf8(
-        Base64.decodeBase64(authHeaderBase64.getBytes()));
-    String[] creds = authHeaderString.split(":");
-    return creds;
-  }
-
-  /**
-   * Returns the base64 encoded auth header payload
-   * @param request
-   * @param authType
-   * @return
-   * @throws HttpAuthenticationException
-   */
-  private String getAuthHeader(HttpServletRequest request, String authType)
-      throws HttpAuthenticationException {
-    String authHeader = request.getHeader(HttpAuthUtils.AUTHORIZATION);
-    // Each http request must have an Authorization header
-    if (authHeader == null || authHeader.isEmpty()) {
-      throw new HttpAuthenticationException("Authorization header received " +
-          "from the client is empty.");
-    }
-
-    String authHeaderBase64String;
-    int beginIndex;
-    if (isKerberosAuthMode(authType)) {
-      beginIndex = (HttpAuthUtils.NEGOTIATE + " ").length();
-    }
-    else {
-      beginIndex = (HttpAuthUtils.BASIC + " ").length();
-    }
-    authHeaderBase64String = authHeader.substring(beginIndex);
-    // Authorization header must have a payload
-    if (authHeaderBase64String == null || authHeaderBase64String.isEmpty()) {
-      throw new HttpAuthenticationException("Authorization header received " +
-          "from the client does not contain any data.");
-    }
-    return authHeaderBase64String;
-  }
-
-  private boolean isKerberosAuthMode(String authType) {
-    return authType.equalsIgnoreCase(HiveAuthFactory.AuthTypes.KERBEROS.toString());
-  }
-
-  private static String getDoAsQueryParam(String queryString) {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("URL query string:" + queryString);
-    }
-    if (queryString == null) {
-      return null;
-    }
-    Map<String, String[]> params = javax.servlet.http.HttpUtils.parseQueryString( queryString );
-    Set<String> keySet = params.keySet();
-    for (String key: keySet) {
-      if (key.equalsIgnoreCase("doAs")) {
-        return params.get(key)[0];
-      }
-    }
-    return null;
-  }
-
-}
-
-
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/server/HiveServer2.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/server/HiveServer2.java
deleted file mode 100644
index 95233996cbbcb..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/server/HiveServer2.java
+++ /dev/null
@@ -1,277 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.server;
-
-import java.util.Properties;
-
-import scala.runtime.AbstractFunction0;
-import scala.runtime.BoxedUnit;
-
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hive.common.util.HiveStringUtils;
-import org.apache.hive.service.CompositeService;
-import org.apache.hive.service.cli.CLIService;
-import org.apache.hive.service.cli.thrift.ThriftBinaryCLIService;
-import org.apache.hive.service.cli.thrift.ThriftCLIService;
-import org.apache.hive.service.cli.thrift.ThriftHttpCLIService;
-
-import org.apache.spark.util.ShutdownHookManager;
-
-/**
- * HiveServer2.
- *
- */
-public class HiveServer2 extends CompositeService {
-  private static final Log LOG = LogFactory.getLog(HiveServer2.class);
-
-  private CLIService cliService;
-  private ThriftCLIService thriftCLIService;
-
-  public HiveServer2() {
-    super(HiveServer2.class.getSimpleName());
-    HiveConf.setLoadHiveServer2Config(true);
-  }
-
-  @Override
-  public synchronized void init(HiveConf hiveConf) {
-    cliService = new CLIService(this);
-    addService(cliService);
-    if (isHTTPTransportMode(hiveConf)) {
-      thriftCLIService = new ThriftHttpCLIService(cliService);
-    } else {
-      thriftCLIService = new ThriftBinaryCLIService(cliService);
-    }
-    addService(thriftCLIService);
-    super.init(hiveConf);
-
-    // Add a shutdown hook for catching SIGTERM & SIGINT
-    // this must be higher than the Hadoop Filesystem priority of 10,
-    // which the default priority is.
-    // The signature of the callback must match that of a scala () -> Unit
-    // function
-    ShutdownHookManager.addShutdownHook(
-        new AbstractFunction0<BoxedUnit>() {
-          public BoxedUnit apply() {
-            try {
-              LOG.info("Hive Server Shutdown hook invoked");
-              stop();
-            } catch (Throwable e) {
-              LOG.warn("Ignoring Exception while stopping Hive Server from shutdown hook",
-                  e);
-            }
-            return BoxedUnit.UNIT;
-          }
-        });
-  }
-
-  public static boolean isHTTPTransportMode(HiveConf hiveConf) {
-    String transportMode = System.getenv("HIVE_SERVER2_TRANSPORT_MODE");
-    if (transportMode == null) {
-      transportMode = hiveConf.getVar(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE);
-    }
-    if (transportMode != null && (transportMode.equalsIgnoreCase("http"))) {
-      return true;
-    }
-    return false;
-  }
-
-  @Override
-  public synchronized void start() {
-    super.start();
-  }
-
-  @Override
-  public synchronized void stop() {
-    LOG.info("Shutting down HiveServer2");
-    super.stop();
-  }
-
-  private static void startHiveServer2() throws Throwable {
-    long attempts = 0, maxAttempts = 1;
-    while (true) {
-      LOG.info("Starting HiveServer2");
-      HiveConf hiveConf = new HiveConf();
-      maxAttempts = hiveConf.getLongVar(HiveConf.ConfVars.HIVE_SERVER2_MAX_START_ATTEMPTS);
-      HiveServer2 server = null;
-      try {
-        server = new HiveServer2();
-        server.init(hiveConf);
-        server.start();
-        ShimLoader.getHadoopShims().startPauseMonitor(hiveConf);
-        break;
-      } catch (Throwable throwable) {
-        if (server != null) {
-          try {
-            server.stop();
-          } catch (Throwable t) {
-            LOG.info("Exception caught when calling stop of HiveServer2 before retrying start", t);
-          } finally {
-            server = null;
-          }
-        }
-        if (++attempts >= maxAttempts) {
-          throw new Error("Max start attempts " + maxAttempts + " exhausted", throwable);
-        } else {
-          LOG.warn("Error starting HiveServer2 on attempt " + attempts
-              + ", will retry in 60 seconds", throwable);
-          try {
-            Thread.sleep(60L * 1000L);
-          } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-          }
-        }
-      }
-    }
-  }
-
-  public static void main(String[] args) {
-    HiveConf.setLoadHiveServer2Config(true);
-    ServerOptionsProcessor oproc = new ServerOptionsProcessor("hiveserver2");
-    ServerOptionsProcessorResponse oprocResponse = oproc.parse(args);
-
-    HiveStringUtils.startupShutdownMessage(HiveServer2.class, args, LOG);
-
-    // Call the executor which will execute the appropriate command based on the parsed options
-    oprocResponse.getServerOptionsExecutor().execute();
-  }
-
-  /**
-   * ServerOptionsProcessor.
-   * Process arguments given to HiveServer2 (-hiveconf property=value)
-   * Set properties in System properties
-   * Create an appropriate response object,
-   * which has executor to execute the appropriate command based on the parsed options.
-   */
-  public static class ServerOptionsProcessor {
-    private final Options options = new Options();
-    private org.apache.commons.cli.CommandLine commandLine;
-    private final String serverName;
-    private final StringBuilder debugMessage = new StringBuilder();
-
-    @SuppressWarnings("static-access")
-    public ServerOptionsProcessor(String serverName) {
-      this.serverName = serverName;
-      // -hiveconf x=y
-      options.addOption(OptionBuilder
-          .withValueSeparator()
-          .hasArgs(2)
-          .withArgName("property=value")
-          .withLongOpt("hiveconf")
-          .withDescription("Use value for given property")
-          .create());
-      options.addOption(new Option("H", "help", false, "Print help information"));
-    }
-
-    public ServerOptionsProcessorResponse parse(String[] argv) {
-      try {
-        commandLine = new GnuParser().parse(options, argv);
-        // Process --hiveconf
-        // Get hiveconf param values and set the System property values
-        Properties confProps = commandLine.getOptionProperties("hiveconf");
-        for (String propKey : confProps.stringPropertyNames()) {
-          // save logging message for log4j output latter after log4j initialize properly
-          debugMessage.append("Setting " + propKey + "=" + confProps.getProperty(propKey) + ";\n");
-          System.setProperty(propKey, confProps.getProperty(propKey));
-        }
-
-        // Process --help
-        if (commandLine.hasOption('H')) {
-          return new ServerOptionsProcessorResponse(new HelpOptionExecutor(serverName, options));
-        }
-      } catch (ParseException e) {
-        // Error out & exit - we were not able to parse the args successfully
-        System.err.println("Error starting HiveServer2 with given arguments: ");
-        System.err.println(e.getMessage());
-        System.exit(-1);
-      }
-      // Default executor, when no option is specified
-      return new ServerOptionsProcessorResponse(new StartOptionExecutor());
-    }
-
-    StringBuilder getDebugMessage() {
-      return debugMessage;
-    }
-  }
-
-  /**
-   * The response sent back from {@link ServerOptionsProcessor#parse(String[])}
-   */
-  static class ServerOptionsProcessorResponse {
-    private final ServerOptionsExecutor serverOptionsExecutor;
-
-    ServerOptionsProcessorResponse(ServerOptionsExecutor serverOptionsExecutor) {
-      this.serverOptionsExecutor = serverOptionsExecutor;
-    }
-
-    ServerOptionsExecutor getServerOptionsExecutor() {
-      return serverOptionsExecutor;
-    }
-  }
-
-  /**
-   * The executor interface for running the appropriate HiveServer2 command based on parsed options
-   */
-  interface ServerOptionsExecutor {
-    void execute();
-  }
-
-  /**
-   * HelpOptionExecutor: executes the --help option by printing out the usage
-   */
-  static class HelpOptionExecutor implements ServerOptionsExecutor {
-    private final Options options;
-    private final String serverName;
-
-    HelpOptionExecutor(String serverName, Options options) {
-      this.options = options;
-      this.serverName = serverName;
-    }
-
-    @Override
-    public void execute() {
-      new HelpFormatter().printHelp(serverName, options);
-      System.exit(0);
-    }
-  }
-
-  /**
-   * StartOptionExecutor: starts HiveServer2.
-   * This is the default executor, when no option is specified.
-   */
-  static class StartOptionExecutor implements ServerOptionsExecutor {
-    @Override
-    public void execute() {
-      try {
-        startHiveServer2();
-      } catch (Throwable t) {
-        LOG.fatal("Error starting HiveServer2", t);
-        System.exit(-1);
-      }
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/server/ThreadWithGarbageCleanup.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/server/ThreadWithGarbageCleanup.java
deleted file mode 100644
index 8ee98103f7ef7..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/server/ThreadWithGarbageCleanup.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.hive.service.server;
-
-import java.util.Map;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.metastore.HiveMetaStore;
-import org.apache.hadoop.hive.metastore.RawStore;
-
-/**
- * A HiveServer2 thread used to construct new server threads.
- * In particular, this thread ensures an orderly cleanup,
- * when killed by its corresponding ExecutorService.
- */
-public class ThreadWithGarbageCleanup extends Thread {
-  private static final Log LOG = LogFactory.getLog(ThreadWithGarbageCleanup.class);
-
-  Map<Long, RawStore> threadRawStoreMap =
-      ThreadFactoryWithGarbageCleanup.getThreadRawStoreMap();
-
-  public ThreadWithGarbageCleanup(Runnable runnable) {
-    super(runnable);
-  }
-
-  /**
-   * Add any Thread specific garbage cleanup code here.
-   * Currently, it shuts down the RawStore object for this thread if it is not null.
-   */
-  @Override
-  public void finalize() throws Throwable {
-    cleanRawStore();
-    super.finalize();
-  }
-
-  private void cleanRawStore() {
-    Long threadId = this.getId();
-    RawStore threadLocalRawStore = threadRawStoreMap.get(threadId);
-    if (threadLocalRawStore != null) {
-      LOG.debug("RawStore: " + threadLocalRawStore + ", for the thread: " +
-          this.getName()  +  " will be closed now.");
-      threadLocalRawStore.shutdown();
-      threadRawStoreMap.remove(threadId);
-    }
-  }
-
-  /**
-   * Cache the ThreadLocal RawStore object. Called from the corresponding thread.
-   */
-  public void cacheThreadLocalRawStore() {
-    Long threadId = this.getId();
-    RawStore threadLocalRawStore = HiveMetaStore.HMSHandler.getRawStore();
-    if (threadLocalRawStore != null && !threadRawStoreMap.containsKey(threadId)) {
-      LOG.debug("Adding RawStore: " + threadLocalRawStore + ", for the thread: " +
-          this.getName() + " to threadRawStoreMap for future cleanup.");
-      threadRawStoreMap.put(threadId, threadLocalRawStore);
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala b/sql/hive-thriftserver/v1.2/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala
deleted file mode 100644
index fbfc698ecb4bf..0000000000000
--- a/sql/hive-thriftserver/v1.2/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.hive.thriftserver
-
-import org.apache.commons.logging.LogFactory
-import org.apache.hadoop.hive.ql.session.SessionState
-import org.apache.hive.service.cli.{RowSet, RowSetFactory, TableSchema, Type}
-import org.apache.hive.service.cli.Type._
-import org.apache.hive.service.cli.thrift.TProtocolVersion._
-
-/**
- * Various utilities for hive-thriftserver used to upgrade the built-in Hive.
- */
-private[thriftserver] object ThriftserverShimUtils {
-
-  private[thriftserver] type TProtocolVersion = org.apache.hive.service.cli.thrift.TProtocolVersion
-  private[thriftserver] type Client = org.apache.hive.service.cli.thrift.TCLIService.Client
-  private[thriftserver] type TOpenSessionReq = org.apache.hive.service.cli.thrift.TOpenSessionReq
-  private[thriftserver] type TGetSchemasReq = org.apache.hive.service.cli.thrift.TGetSchemasReq
-  private[thriftserver] type TGetTablesReq = org.apache.hive.service.cli.thrift.TGetTablesReq
-  private[thriftserver] type TGetColumnsReq = org.apache.hive.service.cli.thrift.TGetColumnsReq
-  private[thriftserver] type TGetInfoReq = org.apache.hive.service.cli.thrift.TGetInfoReq
-  private[thriftserver] type TExecuteStatementReq =
-    org.apache.hive.service.cli.thrift.TExecuteStatementReq
-
-  private[thriftserver] def getConsole: SessionState.LogHelper = {
-    val LOG = LogFactory.getLog(classOf[SparkSQLCLIDriver])
-    new SessionState.LogHelper(LOG)
-  }
-
-  private[thriftserver] def resultRowSet(
-      getResultSetSchema: TableSchema,
-      getProtocolVersion: TProtocolVersion): RowSet = {
-    RowSetFactory.create(getResultSetSchema, getProtocolVersion)
-  }
-
-  private[thriftserver] def toJavaSQLType(s: String): Int = Type.getType(s).toJavaSQLType
-
-  private[thriftserver] def supportedType(): Seq[Type] = {
-    Seq(NULL_TYPE, BOOLEAN_TYPE, STRING_TYPE, BINARY_TYPE,
-      TINYINT_TYPE, SMALLINT_TYPE, INT_TYPE, BIGINT_TYPE,
-      FLOAT_TYPE, DOUBLE_TYPE, DECIMAL_TYPE,
-      DATE_TYPE, TIMESTAMP_TYPE,
-      ARRAY_TYPE, MAP_TYPE, STRUCT_TYPE)
-  }
-
-  private[thriftserver] val testedProtocolVersions = Seq(
-    HIVE_CLI_SERVICE_PROTOCOL_V1,
-    HIVE_CLI_SERVICE_PROTOCOL_V2,
-    HIVE_CLI_SERVICE_PROTOCOL_V3,
-    HIVE_CLI_SERVICE_PROTOCOL_V4,
-    HIVE_CLI_SERVICE_PROTOCOL_V5,
-    HIVE_CLI_SERVICE_PROTOCOL_V6,
-    HIVE_CLI_SERVICE_PROTOCOL_V7,
-    HIVE_CLI_SERVICE_PROTOCOL_V8)
-}
diff --git a/sql/hive-thriftserver/v2.3/if/TCLIService.thrift b/sql/hive-thriftserver/v2.3/if/TCLIService.thrift
deleted file mode 100644
index 9026cd25df5b3..0000000000000
--- a/sql/hive-thriftserver/v2.3/if/TCLIService.thrift
+++ /dev/null
@@ -1,1269 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Coding Conventions for this file:
-//
-// Structs/Enums/Unions
-// * Struct, Enum, and Union names begin with a "T",
-//   and use a capital letter for each new word, with no underscores.
-// * All fields should be declared as either optional or required.
-//
-// Functions
-// * Function names start with a capital letter and have a capital letter for
-//   each new word, with no underscores.
-// * Each function should take exactly one parameter, named TFunctionNameReq,
-//   and should return either void or TFunctionNameResp. This convention allows
-//   incremental updates.
-//
-// Services
-// * Service names begin with the letter "T", use a capital letter for each
-//   new word (with no underscores), and end with the word "Service".
-
-namespace java org.apache.hive.service.rpc.thrift
-namespace cpp apache.hive.service.rpc.thrift
-
-// List of protocol versions. A new token should be
-// added to the end of this list every time a change is made.
-enum TProtocolVersion {
-  HIVE_CLI_SERVICE_PROTOCOL_V1,
-
-  // V2 adds support for asynchronous execution
-  HIVE_CLI_SERVICE_PROTOCOL_V2
-
-  // V3 add varchar type, primitive type qualifiers
-  HIVE_CLI_SERVICE_PROTOCOL_V3
-
-  // V4 add decimal precision/scale, char type
-  HIVE_CLI_SERVICE_PROTOCOL_V4
-
-  // V5 adds error details when GetOperationStatus returns in error state
-  HIVE_CLI_SERVICE_PROTOCOL_V5
-
-  // V6 uses binary type for binary payload (was string) and uses columnar result set
-  HIVE_CLI_SERVICE_PROTOCOL_V6
-
-  // V7 adds support for delegation token based connection
-  HIVE_CLI_SERVICE_PROTOCOL_V7
-
-  // V8 adds support for interval types
-  HIVE_CLI_SERVICE_PROTOCOL_V8
-
-  // V9 adds support for serializing ResultSets in SerDe
-  HIVE_CLI_SERVICE_PROTOCOL_V9
-
-  // V10 adds support for in place updates via GetOperationStatus
-  HIVE_CLI_SERVICE_PROTOCOL_V10
-}
-
-enum TTypeId {
-  BOOLEAN_TYPE,
-  TINYINT_TYPE,
-  SMALLINT_TYPE,
-  INT_TYPE,
-  BIGINT_TYPE,
-  FLOAT_TYPE,
-  DOUBLE_TYPE,
-  STRING_TYPE,
-  TIMESTAMP_TYPE,
-  BINARY_TYPE,
-  ARRAY_TYPE,
-  MAP_TYPE,
-  STRUCT_TYPE,
-  UNION_TYPE,
-  USER_DEFINED_TYPE,
-  DECIMAL_TYPE,
-  NULL_TYPE,
-  DATE_TYPE,
-  VARCHAR_TYPE,
-  CHAR_TYPE,
-  INTERVAL_YEAR_MONTH_TYPE,
-  INTERVAL_DAY_TIME_TYPE
-}
-
-const set<TTypeId> PRIMITIVE_TYPES = [
-  TTypeId.BOOLEAN_TYPE,
-  TTypeId.TINYINT_TYPE,
-  TTypeId.SMALLINT_TYPE,
-  TTypeId.INT_TYPE,
-  TTypeId.BIGINT_TYPE,
-  TTypeId.FLOAT_TYPE,
-  TTypeId.DOUBLE_TYPE,
-  TTypeId.STRING_TYPE,
-  TTypeId.TIMESTAMP_TYPE,
-  TTypeId.BINARY_TYPE,
-  TTypeId.DECIMAL_TYPE,
-  TTypeId.NULL_TYPE,
-  TTypeId.DATE_TYPE,
-  TTypeId.VARCHAR_TYPE,
-  TTypeId.CHAR_TYPE,
-  TTypeId.INTERVAL_YEAR_MONTH_TYPE,
-  TTypeId.INTERVAL_DAY_TIME_TYPE
-]
-
-const set<TTypeId> COMPLEX_TYPES = [
-  TTypeId.ARRAY_TYPE
-  TTypeId.MAP_TYPE
-  TTypeId.STRUCT_TYPE
-  TTypeId.UNION_TYPE
-  TTypeId.USER_DEFINED_TYPE
-]
-
-const set<TTypeId> COLLECTION_TYPES = [
-  TTypeId.ARRAY_TYPE
-  TTypeId.MAP_TYPE
-]
-
-const map<TTypeId,string> TYPE_NAMES = {
-  TTypeId.BOOLEAN_TYPE: "BOOLEAN",
-  TTypeId.TINYINT_TYPE: "TINYINT",
-  TTypeId.SMALLINT_TYPE: "SMALLINT",
-  TTypeId.INT_TYPE: "INT",
-  TTypeId.BIGINT_TYPE: "BIGINT",
-  TTypeId.FLOAT_TYPE: "FLOAT",
-  TTypeId.DOUBLE_TYPE: "DOUBLE",
-  TTypeId.STRING_TYPE: "STRING",
-  TTypeId.TIMESTAMP_TYPE: "TIMESTAMP",
-  TTypeId.BINARY_TYPE: "BINARY",
-  TTypeId.ARRAY_TYPE: "ARRAY",
-  TTypeId.MAP_TYPE: "MAP",
-  TTypeId.STRUCT_TYPE: "STRUCT",
-  TTypeId.UNION_TYPE: "UNIONTYPE",
-  TTypeId.DECIMAL_TYPE: "DECIMAL",
-  TTypeId.NULL_TYPE: "NULL"
-  TTypeId.DATE_TYPE: "DATE"
-  TTypeId.VARCHAR_TYPE: "VARCHAR"
-  TTypeId.CHAR_TYPE: "CHAR"
-  TTypeId.INTERVAL_YEAR_MONTH_TYPE: "INTERVAL_YEAR_MONTH"
-  TTypeId.INTERVAL_DAY_TIME_TYPE: "INTERVAL_DAY_TIME"
-}
-
-// Thrift does not support recursively defined types or forward declarations,
-// which makes it difficult to represent Hive's nested types.
-// To get around these limitations TTypeDesc employs a type list that maps
-// integer "pointers" to TTypeEntry objects. The following examples show
-// how different types are represented using this scheme:
-//
-// "INT":
-// TTypeDesc {
-//   types = [
-//     TTypeEntry.primitive_entry {
-//       type = INT_TYPE
-//     }
-//   ]
-// }
-//
-// "ARRAY<INT>":
-// TTypeDesc {
-//   types = [
-//     TTypeEntry.array_entry {
-//       object_type_ptr = 1
-//     },
-//     TTypeEntry.primitive_entry {
-//       type = INT_TYPE
-//     }
-//   ]
-// }
-//
-// "MAP<INT,STRING>":
-// TTypeDesc {
-//   types = [
-//     TTypeEntry.map_entry {
-//       key_type_ptr = 1
-//       value_type_ptr = 2
-//     },
-//     TTypeEntry.primitive_entry {
-//       type = INT_TYPE
-//     },
-//     TTypeEntry.primitive_entry {
-//       type = STRING_TYPE
-//     }
-//   ]
-// }
-
-typedef i32 TTypeEntryPtr
-
-// Valid TTypeQualifiers key names
-const string CHARACTER_MAXIMUM_LENGTH = "characterMaximumLength"
-
-// Type qualifier key name for decimal
-const string PRECISION = "precision"
-const string SCALE = "scale"
-
-union TTypeQualifierValue {
-  1: optional i32 i32Value
-  2: optional string stringValue
-}
-
-// Type qualifiers for primitive type.
-struct TTypeQualifiers {
-  1: required map <string, TTypeQualifierValue> qualifiers
-}
-
-// Type entry for a primitive type.
-struct TPrimitiveTypeEntry {
-  // The primitive type token. This must satisfy the condition
-  // that type is in the PRIMITIVE_TYPES set.
-  1: required TTypeId type
-  2: optional TTypeQualifiers typeQualifiers
-}
-
-// Type entry for an ARRAY type.
-struct TArrayTypeEntry {
-  1: required TTypeEntryPtr objectTypePtr
-}
-
-// Type entry for a MAP type.
-struct TMapTypeEntry {
-  1: required TTypeEntryPtr keyTypePtr
-  2: required TTypeEntryPtr valueTypePtr
-}
-
-// Type entry for a STRUCT type.
-struct TStructTypeEntry {
-  1: required map<string, TTypeEntryPtr> nameToTypePtr
-}
-
-// Type entry for a UNIONTYPE type.
-struct TUnionTypeEntry {
-  1: required map<string, TTypeEntryPtr> nameToTypePtr
-}
-
-struct TUserDefinedTypeEntry {
-  // The fully qualified name of the class implementing this type.
-  1: required string typeClassName
-}
-
-// We use a union here since Thrift does not support inheritance.
-union TTypeEntry {
-  1: TPrimitiveTypeEntry primitiveEntry
-  2: TArrayTypeEntry arrayEntry
-  3: TMapTypeEntry mapEntry
-  4: TStructTypeEntry structEntry
-  5: TUnionTypeEntry unionEntry
-  6: TUserDefinedTypeEntry userDefinedTypeEntry
-}
-
-// Type descriptor for columns.
-struct TTypeDesc {
-  // The "top" type is always the first element of the list.
-  // If the top type is an ARRAY, MAP, STRUCT, or UNIONTYPE
-  // type, then subsequent elements represent nested types.
-  1: required list<TTypeEntry> types
-}
-
-// A result set column descriptor.
-struct TColumnDesc {
-  // The name of the column
-  1: required string columnName
-
-  // The type descriptor for this column
-  2: required TTypeDesc typeDesc
-
-  // The ordinal position of this column in the schema
-  3: required i32 position
-
-  4: optional string comment
-}
-
-// Metadata used to describe the schema (column names, types, comments)
-// of result sets.
-struct TTableSchema {
-  1: required list<TColumnDesc> columns
-}
-
-// A Boolean column value.
-struct TBoolValue {
-  // NULL if value is unset.
-  1: optional bool value
-}
-
-// A Byte column value.
-struct TByteValue {
-  // NULL if value is unset.
-  1: optional byte value
-}
-
-// A signed, 16 bit column value.
-struct TI16Value {
-  // NULL if value is unset
-  1: optional i16 value
-}
-
-// A signed, 32 bit column value
-struct TI32Value {
-  // NULL if value is unset
-  1: optional i32 value
-}
-
-// A signed 64 bit column value
-struct TI64Value {
-  // NULL if value is unset
-  1: optional i64 value
-}
-
-// A floating point 64 bit column value
-struct TDoubleValue {
-  // NULL if value is unset
-  1: optional double value
-}
-
-struct TStringValue {
-  // NULL if value is unset
-  1: optional string value
-}
-
-// A single column value in a result set.
-// Note that Hive's type system is richer than Thrift's,
-// so in some cases we have to map multiple Hive types
-// to the same Thrift type. On the client-side this is
-// disambiguated by looking at the Schema of the
-// result set.
-union TColumnValue {
-  1: TBoolValue   boolVal      // BOOLEAN
-  2: TByteValue   byteVal      // TINYINT
-  3: TI16Value    i16Val       // SMALLINT
-  4: TI32Value    i32Val       // INT
-  5: TI64Value    i64Val       // BIGINT, TIMESTAMP
-  6: TDoubleValue doubleVal    // FLOAT, DOUBLE
-  7: TStringValue stringVal    // STRING, LIST, MAP, STRUCT, UNIONTYPE, BINARY, DECIMAL, NULL, INTERVAL_YEAR_MONTH, INTERVAL_DAY_TIME
-}
-
-// Represents a row in a rowset.
-struct TRow {
-  1: required list<TColumnValue> colVals
-}
-
-struct TBoolColumn {
-  1: required list<bool> values
-  2: required binary nulls
-}
-
-struct TByteColumn {
-  1: required list<byte> values
-  2: required binary nulls
-}
-
-struct TI16Column {
-  1: required list<i16> values
-  2: required binary nulls
-}
-
-struct TI32Column {
-  1: required list<i32> values
-  2: required binary nulls
-}
-
-struct TI64Column {
-  1: required list<i64> values
-  2: required binary nulls
-}
-
-struct TDoubleColumn {
-  1: required list<double> values
-  2: required binary nulls
-}
-
-struct TStringColumn {
-  1: required list<string> values
-  2: required binary nulls
-}
-
-struct TBinaryColumn {
-  1: required list<binary> values
-  2: required binary nulls
-}
-
-// Note that Hive's type system is richer than Thrift's,
-// so in some cases we have to map multiple Hive types
-// to the same Thrift type. On the client-side this is
-// disambiguated by looking at the Schema of the
-// result set.
-union TColumn {
-  1: TBoolColumn   boolVal      // BOOLEAN
-  2: TByteColumn   byteVal      // TINYINT
-  3: TI16Column    i16Val       // SMALLINT
-  4: TI32Column    i32Val       // INT
-  5: TI64Column    i64Val       // BIGINT, TIMESTAMP
-  6: TDoubleColumn doubleVal    // FLOAT, DOUBLE
-  7: TStringColumn stringVal    // STRING, LIST, MAP, STRUCT, UNIONTYPE, DECIMAL, NULL
-  8: TBinaryColumn binaryVal    // BINARY
-}
-
-// Represents a rowset
-struct TRowSet {
-  // The starting row offset of this rowset.
-  1: required i64 startRowOffset
-  2: required list<TRow> rows
-  3: optional list<TColumn> columns
-  4: optional binary binaryColumns
-  5: optional i32 columnCount
-}
-
-// The return status code contained in each response.
-enum TStatusCode {
-  SUCCESS_STATUS,
-  SUCCESS_WITH_INFO_STATUS,
-  STILL_EXECUTING_STATUS,
-  ERROR_STATUS,
-  INVALID_HANDLE_STATUS
-}
-
-// The return status of a remote request
-struct TStatus {
-  1: required TStatusCode statusCode
-
-  // If status is SUCCESS_WITH_INFO, info_msgs may be populated with
-  // additional diagnostic information.
-  2: optional list<string> infoMessages
-
-  // If status is ERROR, then the following fields may be set
-  3: optional string sqlState  // as defined in the ISO/IEF CLI specification
-  4: optional i32 errorCode    // internal error code
-  5: optional string errorMessage
-}
-
-// The state of an operation (i.e. a query or other
-// asynchronous operation that generates a result set)
-// on the server.
-enum TOperationState {
-  // The operation has been initialized
-  INITIALIZED_STATE,
-
-  // The operation is running. In this state the result
-  // set is not available.
-  RUNNING_STATE,
-
-  // The operation has completed. When an operation is in
-  // this state its result set may be fetched.
-  FINISHED_STATE,
-
-  // The operation was canceled by a client
-  CANCELED_STATE,
-
-  // The operation was closed by a client
-  CLOSED_STATE,
-
-  // The operation failed due to an error
-  ERROR_STATE,
-
-  // The operation is in an unrecognized state
-  UKNOWN_STATE,
-
-  // The operation is in an pending state
-  PENDING_STATE,
-
-  // The operation is in an timedout state
-  TIMEDOUT_STATE,
-}
-
-// A string identifier. This is interpreted literally.
-typedef string TIdentifier
-
-// A search pattern.
-//
-// Valid search pattern characters:
-// '_': Any single character.
-// '%': Any sequence of zero or more characters.
-// '\': Escape character used to include special characters,
-//      e.g. '_', '%', '\'. If a '\' precedes a non-special
-//      character it has no special meaning and is interpreted
-//      literally.
-typedef string TPattern
-
-
-// A search pattern or identifier. Used as input
-// parameter for many of the catalog functions.
-typedef string TPatternOrIdentifier
-
-struct THandleIdentifier {
-  // 16 byte globally unique identifier
-  // This is the public ID of the handle and
-  // can be used for reporting.
-  1: required binary guid,
-
-  // 16 byte secret generated by the server
-  // and used to verify that the handle is not
-  // being hijacked by another user.
-  2: required binary secret,
-}
-
-// Client-side handle to persistent
-// session information on the server-side.
-struct TSessionHandle {
-  1: required THandleIdentifier sessionId
-}
-
-// The subtype of an OperationHandle.
-enum TOperationType {
-  EXECUTE_STATEMENT,
-  GET_TYPE_INFO,
-  GET_CATALOGS,
-  GET_SCHEMAS,
-  GET_TABLES,
-  GET_TABLE_TYPES,
-  GET_COLUMNS,
-  GET_FUNCTIONS,
-  UNKNOWN,
-}
-
-// Client-side reference to a task running
-// asynchronously on the server.
-struct TOperationHandle {
-  1: required THandleIdentifier operationId
-  2: required TOperationType operationType
-
-  // If hasResultSet = TRUE, then this operation
-  // generates a result set that can be fetched.
-  // Note that the result set may be empty.
-  //
-  // If hasResultSet = FALSE, then this operation
-  // does not generate a result set, and calling
-  // GetResultSetMetadata or FetchResults against
-  // this OperationHandle will generate an error.
-  3: required bool hasResultSet
-
-  // For operations that don't generate result sets,
-  // modifiedRowCount is either:
-  //
-  // 1) The number of rows that were modified by
-  //    the DML operation (e.g. number of rows inserted,
-  //    number of rows deleted, etc).
-  //
-  // 2) 0 for operations that don't modify or add rows.
-  //
-  // 3) < 0 if the operation is capable of modifiying rows,
-  //    but Hive is unable to determine how many rows were
-  //    modified. For example, Hive's LOAD DATA command
-  //    doesn't generate row count information because
-  //    Hive doesn't inspect the data as it is loaded.
-  //
-  // modifiedRowCount is unset if the operation generates
-  // a result set.
-  4: optional double modifiedRowCount
-}
-
-
-// OpenSession()
-//
-// Open a session (connection) on the server against
-// which operations may be executed.
-struct TOpenSessionReq {
-  // The version of the HiveServer2 protocol that the client is using.
-  1: required TProtocolVersion client_protocol = TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V10
-
-  // Username and password for authentication.
-  // Depending on the authentication scheme being used,
-  // this information may instead be provided by a lower
-  // protocol layer, in which case these fields may be
-  // left unset.
-  2: optional string username
-  3: optional string password
-
-  // Configuration overlay which is applied when the session is
-  // first created.
-  4: optional map<string, string> configuration
-}
-
-struct TOpenSessionResp {
-  1: required TStatus status
-
-  // The protocol version that the server is using.
-  2: required TProtocolVersion serverProtocolVersion = TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V10
-
-  // Session Handle
-  3: optional TSessionHandle sessionHandle
-
-  // The configuration settings for this session.
-  4: optional map<string, string> configuration
-}
-
-
-// CloseSession()
-//
-// Closes the specified session and frees any resources
-// currently allocated to that session. Any open
-// operations in that session will be canceled.
-struct TCloseSessionReq {
-  1: required TSessionHandle sessionHandle
-}
-
-struct TCloseSessionResp {
-  1: required TStatus status
-}
-
-
-
-enum TGetInfoType {
-  CLI_MAX_DRIVER_CONNECTIONS =           0,
-  CLI_MAX_CONCURRENT_ACTIVITIES =        1,
-  CLI_DATA_SOURCE_NAME =                 2,
-  CLI_FETCH_DIRECTION =                  8,
-  CLI_SERVER_NAME =                      13,
-  CLI_SEARCH_PATTERN_ESCAPE =            14,
-  CLI_DBMS_NAME =                        17,
-  CLI_DBMS_VER =                         18,
-  CLI_ACCESSIBLE_TABLES =                19,
-  CLI_ACCESSIBLE_PROCEDURES =            20,
-  CLI_CURSOR_COMMIT_BEHAVIOR =           23,
-  CLI_DATA_SOURCE_READ_ONLY =            25,
-  CLI_DEFAULT_TXN_ISOLATION =            26,
-  CLI_IDENTIFIER_CASE =                  28,
-  CLI_IDENTIFIER_QUOTE_CHAR =            29,
-  CLI_MAX_COLUMN_NAME_LEN =              30,
-  CLI_MAX_CURSOR_NAME_LEN =              31,
-  CLI_MAX_SCHEMA_NAME_LEN =              32,
-  CLI_MAX_CATALOG_NAME_LEN =             34,
-  CLI_MAX_TABLE_NAME_LEN =               35,
-  CLI_SCROLL_CONCURRENCY =               43,
-  CLI_TXN_CAPABLE =                      46,
-  CLI_USER_NAME =                        47,
-  CLI_TXN_ISOLATION_OPTION =             72,
-  CLI_INTEGRITY =                        73,
-  CLI_GETDATA_EXTENSIONS =               81,
-  CLI_NULL_COLLATION =                   85,
-  CLI_ALTER_TABLE =                      86,
-  CLI_ORDER_BY_COLUMNS_IN_SELECT =       90,
-  CLI_SPECIAL_CHARACTERS =               94,
-  CLI_MAX_COLUMNS_IN_GROUP_BY =          97,
-  CLI_MAX_COLUMNS_IN_INDEX =             98,
-  CLI_MAX_COLUMNS_IN_ORDER_BY =          99,
-  CLI_MAX_COLUMNS_IN_SELECT =            100,
-  CLI_MAX_COLUMNS_IN_TABLE =             101,
-  CLI_MAX_INDEX_SIZE =                   102,
-  CLI_MAX_ROW_SIZE =                     104,
-  CLI_MAX_STATEMENT_LEN =                105,
-  CLI_MAX_TABLES_IN_SELECT =             106,
-  CLI_MAX_USER_NAME_LEN =                107,
-  CLI_OJ_CAPABILITIES =                  115,
-
-  CLI_XOPEN_CLI_YEAR =                   10000,
-  CLI_CURSOR_SENSITIVITY =               10001,
-  CLI_DESCRIBE_PARAMETER =               10002,
-  CLI_CATALOG_NAME =                     10003,
-  CLI_COLLATION_SEQ =                    10004,
-  CLI_MAX_IDENTIFIER_LEN =               10005,
-}
-
-union TGetInfoValue {
-  1: string stringValue
-  2: i16 smallIntValue
-  3: i32 integerBitmask
-  4: i32 integerFlag
-  5: i32 binaryValue
-  6: i64 lenValue
-}
-
-// GetInfo()
-//
-// This function is based on ODBC's CLIGetInfo() function.
-// The function returns general information about the data source
-// using the same keys as ODBC.
-struct TGetInfoReq {
-  // The sesssion to run this request against
-  1: required TSessionHandle sessionHandle
-
-  2: required TGetInfoType infoType
-}
-
-struct TGetInfoResp {
-  1: required TStatus status
-
-  2: required TGetInfoValue infoValue
-}
-
-
-// ExecuteStatement()
-//
-// Execute a statement.
-// The returned OperationHandle can be used to check on the
-// status of the statement, and to fetch results once the
-// statement has finished executing.
-struct TExecuteStatementReq {
-  // The session to execute the statement against
-  1: required TSessionHandle sessionHandle
-
-  // The statement to be executed (DML, DDL, SET, etc)
-  2: required string statement
-
-  // Configuration properties that are overlayed on top of the
-  // the existing session configuration before this statement
-  // is executed. These properties apply to this statement
-  // only and will not affect the subsequent state of the Session.
-  3: optional map<string, string> confOverlay
-
-  // Execute asynchronously when runAsync is true
-  4: optional bool runAsync = false
-
-  // The number of seconds after which the query will timeout on the server
-  5: optional i64 queryTimeout = 0
-}
-
-struct TExecuteStatementResp {
-  1: required TStatus status
-  2: optional TOperationHandle operationHandle
-}
-
-// GetTypeInfo()
-//
-// Get information about types supported by the HiveServer instance.
-// The information is returned as a result set which can be fetched
-// using the OperationHandle provided in the response.
-//
-// Refer to the documentation for ODBC's CLIGetTypeInfo function for
-// the format of the result set.
-struct TGetTypeInfoReq {
-  // The session to run this request against.
-  1: required TSessionHandle sessionHandle
-}
-
-struct TGetTypeInfoResp {
-  1: required TStatus status
-  2: optional TOperationHandle operationHandle
-}
-
-
-// GetCatalogs()
-//
-// Returns the list of catalogs (databases)
-// Results are ordered by TABLE_CATALOG
-//
-// Resultset columns :
-// col1
-// name: TABLE_CAT
-// type: STRING
-// desc: Catalog name. NULL if not applicable.
-//
-struct TGetCatalogsReq {
-  // Session to run this request against
-  1: required TSessionHandle sessionHandle
-}
-
-struct TGetCatalogsResp {
-  1: required TStatus status
-  2: optional TOperationHandle operationHandle
-}
-
-
-// GetSchemas()
-//
-// Retrieves the schema names available in this database. 
-// The results are ordered by TABLE_CATALOG and TABLE_SCHEM.
-// col1
-// name: TABLE_SCHEM
-// type: STRING
-// desc: schema name
-// col2
-// name: TABLE_CATALOG
-// type: STRING
-// desc: catalog name
-struct TGetSchemasReq {
-  // Session to run this request against
-  1: required TSessionHandle sessionHandle
-
-  // Name of the catalog. Must not contain a search pattern.
-  2: optional TIdentifier catalogName
-
-  // schema name or pattern
-  3: optional TPatternOrIdentifier schemaName
-}
-
-struct TGetSchemasResp {
-  1: required TStatus status
-  2: optional TOperationHandle operationHandle
-}
-
-
-// GetTables()
-//
-// Returns a list of tables with catalog, schema, and table
-// type information. The information is returned as a result
-// set which can be fetched using the OperationHandle
-// provided in the response.
-// Results are ordered by TABLE_TYPE, TABLE_CAT, TABLE_SCHEM, and TABLE_NAME
-//
-// Result Set Columns:
-//
-// col1
-// name: TABLE_CAT
-// type: STRING
-// desc: Catalog name. NULL if not applicable.
-//
-// col2
-// name: TABLE_SCHEM
-// type: STRING
-// desc: Schema name.
-//
-// col3
-// name: TABLE_NAME
-// type: STRING
-// desc: Table name.
-//
-// col4
-// name: TABLE_TYPE
-// type: STRING
-// desc: The table type, e.g. "TABLE", "VIEW", etc.
-//
-// col5
-// name: REMARKS
-// type: STRING
-// desc: Comments about the table
-//
-struct TGetTablesReq {
-  // Session to run this request against
-  1: required TSessionHandle sessionHandle
-
-  // Name of the catalog or a search pattern.
-  2: optional TPatternOrIdentifier catalogName
-
-  // Name of the schema or a search pattern.
-  3: optional TPatternOrIdentifier schemaName
-
-  // Name of the table or a search pattern.
-  4: optional TPatternOrIdentifier tableName
-
-  // List of table types to match
-  // e.g. "TABLE", "VIEW", "SYSTEM TABLE", "GLOBAL TEMPORARY",
-  // "LOCAL TEMPORARY", "ALIAS", "SYNONYM", etc.
-  5: optional list<string> tableTypes
-}
-
-struct TGetTablesResp {
-  1: required TStatus status
-  2: optional TOperationHandle operationHandle
-}
-
-
-// GetTableTypes()
-//
-// Returns the table types available in this database.
-// The results are ordered by table type.
-//
-// col1
-// name: TABLE_TYPE
-// type: STRING
-// desc: Table type name.
-struct TGetTableTypesReq {
-  // Session to run this request against
-  1: required TSessionHandle sessionHandle
-}
-
-struct TGetTableTypesResp {
-  1: required TStatus status
-  2: optional TOperationHandle operationHandle
-}
-
-
-// GetColumns()
-//
-// Returns a list of columns in the specified tables.
-// The information is returned as a result set which can be fetched
-// using the OperationHandle provided in the response.
-// Results are ordered by TABLE_CAT, TABLE_SCHEM, TABLE_NAME,
-// and ORDINAL_POSITION.
-//
-// Result Set Columns are the same as those for the ODBC CLIColumns
-// function.
-//
-struct TGetColumnsReq {
-  // Session to run this request against
-  1: required TSessionHandle sessionHandle
-
-  // Name of the catalog. Must not contain a search pattern.
-  2: optional TIdentifier catalogName
-
-  // Schema name or search pattern
-  3: optional TPatternOrIdentifier schemaName
-
-  // Table name or search pattern
-  4: optional TPatternOrIdentifier tableName
-
-  // Column name or search pattern
-  5: optional TPatternOrIdentifier columnName
-}
-
-struct TGetColumnsResp {
-  1: required TStatus status
-  2: optional TOperationHandle operationHandle
-}
-
-
-// GetFunctions()
-//
-// Returns a list of functions supported by the data source. The
-// behavior of this function matches
-// java.sql.DatabaseMetaData.getFunctions() both in terms of
-// inputs and outputs.
-//
-// Result Set Columns:
-//
-// col1
-// name: FUNCTION_CAT
-// type: STRING
-// desc: Function catalog (may be null)
-//
-// col2
-// name: FUNCTION_SCHEM
-// type: STRING
-// desc: Function schema (may be null)
-//
-// col3
-// name: FUNCTION_NAME
-// type: STRING
-// desc: Function name. This is the name used to invoke the function.
-//
-// col4
-// name: REMARKS
-// type: STRING
-// desc: Explanatory comment on the function.
-//
-// col5
-// name: FUNCTION_TYPE
-// type: SMALLINT
-// desc: Kind of function. One of:
-//       * functionResultUnknown - Cannot determine if a return value or a table
-//                                 will be returned.
-//       * functionNoTable       - Does not a return a table.
-//       * functionReturnsTable  - Returns a table.
-//
-// col6
-// name: SPECIFIC_NAME
-// type: STRING
-// desc: The name which uniquely identifies this function within its schema.
-//       In this case this is the fully qualified class name of the class
-//       that implements this function.
-//
-struct TGetFunctionsReq {
-  // Session to run this request against
-  1: required TSessionHandle sessionHandle
-
-  // A catalog name; must match the catalog name as it is stored in the
-  // database; "" retrieves those without a catalog; null means
-  // that the catalog name should not be used to narrow the search.
-  2: optional TIdentifier catalogName
-
-  // A schema name pattern; must match the schema name as it is stored
-  // in the database; "" retrieves those without a schema; null means
-  // that the schema name should not be used to narrow the search.
-  3: optional TPatternOrIdentifier schemaName
-
-  // A function name pattern; must match the function name as it is stored
-  // in the database.
-  4: required TPatternOrIdentifier functionName
-}
-
-struct TGetFunctionsResp {
-  1: required TStatus status
-  2: optional TOperationHandle operationHandle
-}
-
-struct TGetPrimaryKeysReq {
-  // Session to run this request against
-  1: required TSessionHandle sessionHandle
-
-  // Name of the catalog.
-  2: optional TIdentifier catalogName
-
-  // Name of the schema.
-  3: optional TIdentifier schemaName
-
-  // Name of the table.
-  4: optional TIdentifier tableName
-}
-
-struct TGetPrimaryKeysResp {
-  1: required TStatus status
-  2: optional TOperationHandle operationHandle
-}
-
-struct TGetCrossReferenceReq {
-  // Session to run this request against
-  1: required TSessionHandle sessionHandle
-
-  // Name of the parent catalog.
-  2: optional TIdentifier parentCatalogName
-
-  // Name of the parent schema.
-  3: optional TIdentifier parentSchemaName
-
-  // Name of the parent table.
-  4: optional TIdentifier parentTableName
-
-  // Name of the foreign catalog.
-  5: optional TIdentifier foreignCatalogName
-
-  // Name of the foreign schema.
-  6: optional TIdentifier foreignSchemaName
-
-  // Name of the foreign table.
-  7: optional TIdentifier foreignTableName
-}
-
-struct TGetCrossReferenceResp {
-  1: required TStatus status
-  2: optional TOperationHandle operationHandle
-}
-
-// GetOperationStatus()
-//
-// Get the status of an operation running on the server.
-struct TGetOperationStatusReq {
-  // Session to run this request against
-  1: required TOperationHandle operationHandle
-  // optional arguments to get progress information
-  2: optional bool getProgressUpdate
-}
-
-struct TGetOperationStatusResp {
-  1: required TStatus status
-  2: optional TOperationState operationState
-
-  // If operationState is ERROR_STATE, then the following fields may be set
-  // sqlState as defined in the ISO/IEF CLI specification
-  3: optional string sqlState
-
-  // Internal error code
-  4: optional i32 errorCode
-
-  // Error message
-  5: optional string errorMessage
-
-  // List of statuses of sub tasks
-  6: optional string taskStatus
-
-  // When was the operation started
-  7: optional i64 operationStarted
-
-  // When was the operation completed
-  8: optional i64 operationCompleted
-
-  // If the operation has the result
-  9: optional bool hasResultSet
-
-  10: optional TProgressUpdateResp progressUpdateResponse
-
-}
-
-
-// CancelOperation()
-//
-// Cancels processing on the specified operation handle and
-// frees any resources which were allocated.
-struct TCancelOperationReq {
-  // Operation to cancel
-  1: required TOperationHandle operationHandle
-}
-
-struct TCancelOperationResp {
-  1: required TStatus status
-}
-
-
-// CloseOperation()
-//
-// Given an operation in the FINISHED, CANCELED,
-// or ERROR states, CloseOperation() will free
-// all of the resources which were allocated on
-// the server to service the operation.
-struct TCloseOperationReq {
-  1: required TOperationHandle operationHandle
-}
-
-struct TCloseOperationResp {
-  1: required TStatus status
-}
-
-
-// GetResultSetMetadata()
-//
-// Retrieves schema information for the specified operation
-struct TGetResultSetMetadataReq {
-  // Operation for which to fetch result set schema information
-  1: required TOperationHandle operationHandle
-}
-
-struct TGetResultSetMetadataResp {
-  1: required TStatus status
-  2: optional TTableSchema schema
-}
-
-
-enum TFetchOrientation {
-  // Get the next rowset. The fetch offset is ignored.
-  FETCH_NEXT,
-
-  // Get the previous rowset. The fetch offset is ignored.
-  FETCH_PRIOR,
-
-  // Return the rowset at the given fetch offset relative
-  // to the curren rowset.
-  // NOT SUPPORTED
-  FETCH_RELATIVE,
-
-  // Return the rowset at the specified fetch offset.
-  // NOT SUPPORTED
-  FETCH_ABSOLUTE,
-
-  // Get the first rowset in the result set.
-  FETCH_FIRST,
-
-  // Get the last rowset in the result set.
-  // NOT SUPPORTED
-  FETCH_LAST
-}
-
-// FetchResults()
-//
-// Fetch rows from the server corresponding to
-// a particular OperationHandle.
-struct TFetchResultsReq {
-  // Operation from which to fetch results.
-  1: required TOperationHandle operationHandle
-
-  // The fetch orientation. This must be either
-  // FETCH_NEXT, FETCH_PRIOR or FETCH_FIRST. Defaults to FETCH_NEXT.
-  2: required TFetchOrientation orientation = TFetchOrientation.FETCH_NEXT
-
-  // Max number of rows that should be returned in
-  // the rowset.
-  3: required i64 maxRows
-
-  // The type of a fetch results request. 0 represents Query output. 1 represents Log
-  4: optional i16 fetchType = 0
-}
-
-struct TFetchResultsResp {
-  1: required TStatus status
-
-  // TRUE if there are more rows left to fetch from the server.
-  2: optional bool hasMoreRows
-
-  // The rowset. This is optional so that we have the
-  // option in the future of adding alternate formats for
-  // representing result set data, e.g. delimited strings,
-  // binary encoded, etc.
-  3: optional TRowSet results
-}
-
-// GetDelegationToken()
-// Retrieve delegation token for the current user
-struct  TGetDelegationTokenReq {
-  // session handle
-  1: required TSessionHandle sessionHandle
-
-  // userid for the proxy user
-  2: required string owner
-
-  // designated renewer userid
-  3: required string renewer
-}
-
-struct TGetDelegationTokenResp {
-  // status of the request
-  1: required TStatus status
-
-  // delegation token string
-  2: optional string delegationToken
-}
-
-// CancelDelegationToken()
-// Cancel the given delegation token
-struct TCancelDelegationTokenReq {
-  // session handle
-  1: required TSessionHandle sessionHandle
-
-  // delegation token to cancel
-  2: required string delegationToken
-}
-
-struct TCancelDelegationTokenResp {
-  // status of the request
-  1: required TStatus status
-}
-
-// RenewDelegationToken()
-// Renew the given delegation token
-struct TRenewDelegationTokenReq {
-  // session handle
-  1: required TSessionHandle sessionHandle
-
-  // delegation token to renew
-  2: required string delegationToken
-}
-
-struct TRenewDelegationTokenResp {
-  // status of the request
-  1: required TStatus status
-}
-
-enum TJobExecutionStatus {
-    IN_PROGRESS,
-    COMPLETE,
-    NOT_AVAILABLE
-}
-
-struct TProgressUpdateResp {
-  1: required list<string> headerNames
-  2: required list<list<string>> rows
-  3: required double progressedPercentage
-  4: required TJobExecutionStatus status
-  5: required string footerSummary
-  6: required i64 startTime
-}
-
-service TCLIService {
-
-  TOpenSessionResp OpenSession(1:TOpenSessionReq req);
-
-  TCloseSessionResp CloseSession(1:TCloseSessionReq req);
-
-  TGetInfoResp GetInfo(1:TGetInfoReq req);
-
-  TExecuteStatementResp ExecuteStatement(1:TExecuteStatementReq req);
-
-  TGetTypeInfoResp GetTypeInfo(1:TGetTypeInfoReq req);
-
-  TGetCatalogsResp GetCatalogs(1:TGetCatalogsReq req);
-
-  TGetSchemasResp GetSchemas(1:TGetSchemasReq req);
-
-  TGetTablesResp GetTables(1:TGetTablesReq req);
-
-  TGetTableTypesResp GetTableTypes(1:TGetTableTypesReq req);
-
-  TGetColumnsResp GetColumns(1:TGetColumnsReq req);
-
-  TGetFunctionsResp GetFunctions(1:TGetFunctionsReq req);
-
-  TGetPrimaryKeysResp GetPrimaryKeys(1:TGetPrimaryKeysReq req);
-
-  TGetCrossReferenceResp GetCrossReference(1:TGetCrossReferenceReq req);
-
-  TGetOperationStatusResp GetOperationStatus(1:TGetOperationStatusReq req);
-
-  TCancelOperationResp CancelOperation(1:TCancelOperationReq req);
-
-  TCloseOperationResp CloseOperation(1:TCloseOperationReq req);
-
-  TGetResultSetMetadataResp GetResultSetMetadata(1:TGetResultSetMetadataReq req);
-
-  TFetchResultsResp FetchResults(1:TFetchResultsReq req);
-
-  TGetDelegationTokenResp GetDelegationToken(1:TGetDelegationTokenReq req);
-
-  TCancelDelegationTokenResp CancelDelegationToken(1:TCancelDelegationTokenReq req);
-
-  TRenewDelegationTokenResp RenewDelegationToken(1:TRenewDelegationTokenReq req);
-}
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TArrayTypeEntry.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TArrayTypeEntry.java
deleted file mode 100644
index 358e322632144..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TArrayTypeEntry.java
+++ /dev/null
@@ -1,387 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TArrayTypeEntry implements org.apache.thrift.TBase<TArrayTypeEntry, TArrayTypeEntry._Fields>, java.io.Serializable, Cloneable, Comparable<TArrayTypeEntry> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TArrayTypeEntry");
-
-  private static final org.apache.thrift.protocol.TField OBJECT_TYPE_PTR_FIELD_DESC = new org.apache.thrift.protocol.TField("objectTypePtr", org.apache.thrift.protocol.TType.I32, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TArrayTypeEntryStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TArrayTypeEntryTupleSchemeFactory());
-  }
-
-  private int objectTypePtr; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    OBJECT_TYPE_PTR((short)1, "objectTypePtr");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // OBJECT_TYPE_PTR
-          return OBJECT_TYPE_PTR;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __OBJECTTYPEPTR_ISSET_ID = 0;
-  private byte __isset_bitfield = 0;
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.OBJECT_TYPE_PTR, new org.apache.thrift.meta_data.FieldMetaData("objectTypePtr", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32        , "TTypeEntryPtr")));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TArrayTypeEntry.class, metaDataMap);
-  }
-
-  public TArrayTypeEntry() {
-  }
-
-  public TArrayTypeEntry(
-    int objectTypePtr)
-  {
-    this();
-    this.objectTypePtr = objectTypePtr;
-    setObjectTypePtrIsSet(true);
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TArrayTypeEntry(TArrayTypeEntry other) {
-    __isset_bitfield = other.__isset_bitfield;
-    this.objectTypePtr = other.objectTypePtr;
-  }
-
-  public TArrayTypeEntry deepCopy() {
-    return new TArrayTypeEntry(this);
-  }
-
-  @Override
-  public void clear() {
-    setObjectTypePtrIsSet(false);
-    this.objectTypePtr = 0;
-  }
-
-  public int getObjectTypePtr() {
-    return this.objectTypePtr;
-  }
-
-  public void setObjectTypePtr(int objectTypePtr) {
-    this.objectTypePtr = objectTypePtr;
-    setObjectTypePtrIsSet(true);
-  }
-
-  public void unsetObjectTypePtr() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __OBJECTTYPEPTR_ISSET_ID);
-  }
-
-  /** Returns true if field objectTypePtr is set (has been assigned a value) and false otherwise */
-  public boolean isSetObjectTypePtr() {
-    return EncodingUtils.testBit(__isset_bitfield, __OBJECTTYPEPTR_ISSET_ID);
-  }
-
-  public void setObjectTypePtrIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __OBJECTTYPEPTR_ISSET_ID, value);
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case OBJECT_TYPE_PTR:
-      if (value == null) {
-        unsetObjectTypePtr();
-      } else {
-        setObjectTypePtr((Integer)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case OBJECT_TYPE_PTR:
-      return getObjectTypePtr();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case OBJECT_TYPE_PTR:
-      return isSetObjectTypePtr();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TArrayTypeEntry)
-      return this.equals((TArrayTypeEntry)that);
-    return false;
-  }
-
-  public boolean equals(TArrayTypeEntry that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_objectTypePtr = true;
-    boolean that_present_objectTypePtr = true;
-    if (this_present_objectTypePtr || that_present_objectTypePtr) {
-      if (!(this_present_objectTypePtr && that_present_objectTypePtr))
-        return false;
-      if (this.objectTypePtr != that.objectTypePtr)
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_objectTypePtr = true;
-    list.add(present_objectTypePtr);
-    if (present_objectTypePtr)
-      list.add(objectTypePtr);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TArrayTypeEntry other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetObjectTypePtr()).compareTo(other.isSetObjectTypePtr());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetObjectTypePtr()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.objectTypePtr, other.objectTypePtr);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TArrayTypeEntry(");
-    boolean first = true;
-
-    sb.append("objectTypePtr:");
-    sb.append(this.objectTypePtr);
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetObjectTypePtr()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'objectTypePtr' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TArrayTypeEntryStandardSchemeFactory implements SchemeFactory {
-    public TArrayTypeEntryStandardScheme getScheme() {
-      return new TArrayTypeEntryStandardScheme();
-    }
-  }
-
-  private static class TArrayTypeEntryStandardScheme extends StandardScheme<TArrayTypeEntry> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TArrayTypeEntry struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // OBJECT_TYPE_PTR
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.objectTypePtr = iprot.readI32();
-              struct.setObjectTypePtrIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TArrayTypeEntry struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      oprot.writeFieldBegin(OBJECT_TYPE_PTR_FIELD_DESC);
-      oprot.writeI32(struct.objectTypePtr);
-      oprot.writeFieldEnd();
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TArrayTypeEntryTupleSchemeFactory implements SchemeFactory {
-    public TArrayTypeEntryTupleScheme getScheme() {
-      return new TArrayTypeEntryTupleScheme();
-    }
-  }
-
-  private static class TArrayTypeEntryTupleScheme extends TupleScheme<TArrayTypeEntry> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TArrayTypeEntry struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      oprot.writeI32(struct.objectTypePtr);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TArrayTypeEntry struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.objectTypePtr = iprot.readI32();
-      struct.setObjectTypePtrIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TBinaryColumn.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TBinaryColumn.java
deleted file mode 100644
index a869cee007c0b..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TBinaryColumn.java
+++ /dev/null
@@ -1,548 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TBinaryColumn implements org.apache.thrift.TBase<TBinaryColumn, TBinaryColumn._Fields>, java.io.Serializable, Cloneable, Comparable<TBinaryColumn> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TBinaryColumn");
-
-  private static final org.apache.thrift.protocol.TField VALUES_FIELD_DESC = new org.apache.thrift.protocol.TField("values", org.apache.thrift.protocol.TType.LIST, (short)1);
-  private static final org.apache.thrift.protocol.TField NULLS_FIELD_DESC = new org.apache.thrift.protocol.TField("nulls", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TBinaryColumnStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TBinaryColumnTupleSchemeFactory());
-  }
-
-  private List<ByteBuffer> values; // required
-  private ByteBuffer nulls; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUES((short)1, "values"),
-    NULLS((short)2, "nulls");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUES
-          return VALUES;
-        case 2: // NULLS
-          return NULLS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUES, new org.apache.thrift.meta_data.FieldMetaData("values", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING            , true))));
-    tmpMap.put(_Fields.NULLS, new org.apache.thrift.meta_data.FieldMetaData("nulls", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TBinaryColumn.class, metaDataMap);
-  }
-
-  public TBinaryColumn() {
-  }
-
-  public TBinaryColumn(
-    List<ByteBuffer> values,
-    ByteBuffer nulls)
-  {
-    this();
-    this.values = values;
-    this.nulls = org.apache.thrift.TBaseHelper.copyBinary(nulls);
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TBinaryColumn(TBinaryColumn other) {
-    if (other.isSetValues()) {
-      List<ByteBuffer> __this__values = new ArrayList<ByteBuffer>(other.values);
-      this.values = __this__values;
-    }
-    if (other.isSetNulls()) {
-      this.nulls = org.apache.thrift.TBaseHelper.copyBinary(other.nulls);
-    }
-  }
-
-  public TBinaryColumn deepCopy() {
-    return new TBinaryColumn(this);
-  }
-
-  @Override
-  public void clear() {
-    this.values = null;
-    this.nulls = null;
-  }
-
-  public int getValuesSize() {
-    return (this.values == null) ? 0 : this.values.size();
-  }
-
-  public java.util.Iterator<ByteBuffer> getValuesIterator() {
-    return (this.values == null) ? null : this.values.iterator();
-  }
-
-  public void addToValues(ByteBuffer elem) {
-    if (this.values == null) {
-      this.values = new ArrayList<ByteBuffer>();
-    }
-    this.values.add(elem);
-  }
-
-  public List<ByteBuffer> getValues() {
-    return this.values;
-  }
-
-  public void setValues(List<ByteBuffer> values) {
-    this.values = values;
-  }
-
-  public void unsetValues() {
-    this.values = null;
-  }
-
-  /** Returns true if field values is set (has been assigned a value) and false otherwise */
-  public boolean isSetValues() {
-    return this.values != null;
-  }
-
-  public void setValuesIsSet(boolean value) {
-    if (!value) {
-      this.values = null;
-    }
-  }
-
-  public byte[] getNulls() {
-    setNulls(org.apache.thrift.TBaseHelper.rightSize(nulls));
-    return nulls == null ? null : nulls.array();
-  }
-
-  public ByteBuffer bufferForNulls() {
-    return org.apache.thrift.TBaseHelper.copyBinary(nulls);
-  }
-
-  public void setNulls(byte[] nulls) {
-    this.nulls = nulls == null ? (ByteBuffer)null : ByteBuffer.wrap(Arrays.copyOf(nulls, nulls.length));
-  }
-
-  public void setNulls(ByteBuffer nulls) {
-    this.nulls = org.apache.thrift.TBaseHelper.copyBinary(nulls);
-  }
-
-  public void unsetNulls() {
-    this.nulls = null;
-  }
-
-  /** Returns true if field nulls is set (has been assigned a value) and false otherwise */
-  public boolean isSetNulls() {
-    return this.nulls != null;
-  }
-
-  public void setNullsIsSet(boolean value) {
-    if (!value) {
-      this.nulls = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUES:
-      if (value == null) {
-        unsetValues();
-      } else {
-        setValues((List<ByteBuffer>)value);
-      }
-      break;
-
-    case NULLS:
-      if (value == null) {
-        unsetNulls();
-      } else {
-        setNulls((ByteBuffer)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUES:
-      return getValues();
-
-    case NULLS:
-      return getNulls();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUES:
-      return isSetValues();
-    case NULLS:
-      return isSetNulls();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TBinaryColumn)
-      return this.equals((TBinaryColumn)that);
-    return false;
-  }
-
-  public boolean equals(TBinaryColumn that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_values = true && this.isSetValues();
-    boolean that_present_values = true && that.isSetValues();
-    if (this_present_values || that_present_values) {
-      if (!(this_present_values && that_present_values))
-        return false;
-      if (!this.values.equals(that.values))
-        return false;
-    }
-
-    boolean this_present_nulls = true && this.isSetNulls();
-    boolean that_present_nulls = true && that.isSetNulls();
-    if (this_present_nulls || that_present_nulls) {
-      if (!(this_present_nulls && that_present_nulls))
-        return false;
-      if (!this.nulls.equals(that.nulls))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_values = true && (isSetValues());
-    list.add(present_values);
-    if (present_values)
-      list.add(values);
-
-    boolean present_nulls = true && (isSetNulls());
-    list.add(present_nulls);
-    if (present_nulls)
-      list.add(nulls);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TBinaryColumn other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetValues()).compareTo(other.isSetValues());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValues()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.values, other.values);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetNulls()).compareTo(other.isSetNulls());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetNulls()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.nulls, other.nulls);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TBinaryColumn(");
-    boolean first = true;
-
-    sb.append("values:");
-    if (this.values == null) {
-      sb.append("null");
-    } else {
-      org.apache.thrift.TBaseHelper.toString(this.values, sb);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("nulls:");
-    if (this.nulls == null) {
-      sb.append("null");
-    } else {
-      org.apache.thrift.TBaseHelper.toString(this.nulls, sb);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetValues()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'values' is unset! Struct:" + toString());
-    }
-
-    if (!isSetNulls()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'nulls' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TBinaryColumnStandardSchemeFactory implements SchemeFactory {
-    public TBinaryColumnStandardScheme getScheme() {
-      return new TBinaryColumnStandardScheme();
-    }
-  }
-
-  private static class TBinaryColumnStandardScheme extends StandardScheme<TBinaryColumn> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TBinaryColumn struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUES
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list110 = iprot.readListBegin();
-                struct.values = new ArrayList<ByteBuffer>(_list110.size);
-                ByteBuffer _elem111;
-                for (int _i112 = 0; _i112 < _list110.size; ++_i112)
-                {
-                  _elem111 = iprot.readBinary();
-                  struct.values.add(_elem111);
-                }
-                iprot.readListEnd();
-              }
-              struct.setValuesIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // NULLS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.nulls = iprot.readBinary();
-              struct.setNullsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TBinaryColumn struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.values != null) {
-        oprot.writeFieldBegin(VALUES_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.values.size()));
-          for (ByteBuffer _iter113 : struct.values)
-          {
-            oprot.writeBinary(_iter113);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      if (struct.nulls != null) {
-        oprot.writeFieldBegin(NULLS_FIELD_DESC);
-        oprot.writeBinary(struct.nulls);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TBinaryColumnTupleSchemeFactory implements SchemeFactory {
-    public TBinaryColumnTupleScheme getScheme() {
-      return new TBinaryColumnTupleScheme();
-    }
-  }
-
-  private static class TBinaryColumnTupleScheme extends TupleScheme<TBinaryColumn> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TBinaryColumn struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.values.size());
-        for (ByteBuffer _iter114 : struct.values)
-        {
-          oprot.writeBinary(_iter114);
-        }
-      }
-      oprot.writeBinary(struct.nulls);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TBinaryColumn struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TList _list115 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-        struct.values = new ArrayList<ByteBuffer>(_list115.size);
-        ByteBuffer _elem116;
-        for (int _i117 = 0; _i117 < _list115.size; ++_i117)
-        {
-          _elem116 = iprot.readBinary();
-          struct.values.add(_elem116);
-        }
-      }
-      struct.setValuesIsSet(true);
-      struct.nulls = iprot.readBinary();
-      struct.setNullsIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TBoolColumn.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TBoolColumn.java
deleted file mode 100644
index 9bb636672aa1e..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TBoolColumn.java
+++ /dev/null
@@ -1,548 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TBoolColumn implements org.apache.thrift.TBase<TBoolColumn, TBoolColumn._Fields>, java.io.Serializable, Cloneable, Comparable<TBoolColumn> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TBoolColumn");
-
-  private static final org.apache.thrift.protocol.TField VALUES_FIELD_DESC = new org.apache.thrift.protocol.TField("values", org.apache.thrift.protocol.TType.LIST, (short)1);
-  private static final org.apache.thrift.protocol.TField NULLS_FIELD_DESC = new org.apache.thrift.protocol.TField("nulls", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TBoolColumnStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TBoolColumnTupleSchemeFactory());
-  }
-
-  private List<Boolean> values; // required
-  private ByteBuffer nulls; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUES((short)1, "values"),
-    NULLS((short)2, "nulls");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUES
-          return VALUES;
-        case 2: // NULLS
-          return NULLS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUES, new org.apache.thrift.meta_data.FieldMetaData("values", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))));
-    tmpMap.put(_Fields.NULLS, new org.apache.thrift.meta_data.FieldMetaData("nulls", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TBoolColumn.class, metaDataMap);
-  }
-
-  public TBoolColumn() {
-  }
-
-  public TBoolColumn(
-    List<Boolean> values,
-    ByteBuffer nulls)
-  {
-    this();
-    this.values = values;
-    this.nulls = org.apache.thrift.TBaseHelper.copyBinary(nulls);
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TBoolColumn(TBoolColumn other) {
-    if (other.isSetValues()) {
-      List<Boolean> __this__values = new ArrayList<Boolean>(other.values);
-      this.values = __this__values;
-    }
-    if (other.isSetNulls()) {
-      this.nulls = org.apache.thrift.TBaseHelper.copyBinary(other.nulls);
-    }
-  }
-
-  public TBoolColumn deepCopy() {
-    return new TBoolColumn(this);
-  }
-
-  @Override
-  public void clear() {
-    this.values = null;
-    this.nulls = null;
-  }
-
-  public int getValuesSize() {
-    return (this.values == null) ? 0 : this.values.size();
-  }
-
-  public java.util.Iterator<Boolean> getValuesIterator() {
-    return (this.values == null) ? null : this.values.iterator();
-  }
-
-  public void addToValues(boolean elem) {
-    if (this.values == null) {
-      this.values = new ArrayList<Boolean>();
-    }
-    this.values.add(elem);
-  }
-
-  public List<Boolean> getValues() {
-    return this.values;
-  }
-
-  public void setValues(List<Boolean> values) {
-    this.values = values;
-  }
-
-  public void unsetValues() {
-    this.values = null;
-  }
-
-  /** Returns true if field values is set (has been assigned a value) and false otherwise */
-  public boolean isSetValues() {
-    return this.values != null;
-  }
-
-  public void setValuesIsSet(boolean value) {
-    if (!value) {
-      this.values = null;
-    }
-  }
-
-  public byte[] getNulls() {
-    setNulls(org.apache.thrift.TBaseHelper.rightSize(nulls));
-    return nulls == null ? null : nulls.array();
-  }
-
-  public ByteBuffer bufferForNulls() {
-    return org.apache.thrift.TBaseHelper.copyBinary(nulls);
-  }
-
-  public void setNulls(byte[] nulls) {
-    this.nulls = nulls == null ? (ByteBuffer)null : ByteBuffer.wrap(Arrays.copyOf(nulls, nulls.length));
-  }
-
-  public void setNulls(ByteBuffer nulls) {
-    this.nulls = org.apache.thrift.TBaseHelper.copyBinary(nulls);
-  }
-
-  public void unsetNulls() {
-    this.nulls = null;
-  }
-
-  /** Returns true if field nulls is set (has been assigned a value) and false otherwise */
-  public boolean isSetNulls() {
-    return this.nulls != null;
-  }
-
-  public void setNullsIsSet(boolean value) {
-    if (!value) {
-      this.nulls = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUES:
-      if (value == null) {
-        unsetValues();
-      } else {
-        setValues((List<Boolean>)value);
-      }
-      break;
-
-    case NULLS:
-      if (value == null) {
-        unsetNulls();
-      } else {
-        setNulls((ByteBuffer)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUES:
-      return getValues();
-
-    case NULLS:
-      return getNulls();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUES:
-      return isSetValues();
-    case NULLS:
-      return isSetNulls();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TBoolColumn)
-      return this.equals((TBoolColumn)that);
-    return false;
-  }
-
-  public boolean equals(TBoolColumn that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_values = true && this.isSetValues();
-    boolean that_present_values = true && that.isSetValues();
-    if (this_present_values || that_present_values) {
-      if (!(this_present_values && that_present_values))
-        return false;
-      if (!this.values.equals(that.values))
-        return false;
-    }
-
-    boolean this_present_nulls = true && this.isSetNulls();
-    boolean that_present_nulls = true && that.isSetNulls();
-    if (this_present_nulls || that_present_nulls) {
-      if (!(this_present_nulls && that_present_nulls))
-        return false;
-      if (!this.nulls.equals(that.nulls))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_values = true && (isSetValues());
-    list.add(present_values);
-    if (present_values)
-      list.add(values);
-
-    boolean present_nulls = true && (isSetNulls());
-    list.add(present_nulls);
-    if (present_nulls)
-      list.add(nulls);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TBoolColumn other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetValues()).compareTo(other.isSetValues());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValues()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.values, other.values);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetNulls()).compareTo(other.isSetNulls());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetNulls()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.nulls, other.nulls);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TBoolColumn(");
-    boolean first = true;
-
-    sb.append("values:");
-    if (this.values == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.values);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("nulls:");
-    if (this.nulls == null) {
-      sb.append("null");
-    } else {
-      org.apache.thrift.TBaseHelper.toString(this.nulls, sb);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetValues()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'values' is unset! Struct:" + toString());
-    }
-
-    if (!isSetNulls()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'nulls' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TBoolColumnStandardSchemeFactory implements SchemeFactory {
-    public TBoolColumnStandardScheme getScheme() {
-      return new TBoolColumnStandardScheme();
-    }
-  }
-
-  private static class TBoolColumnStandardScheme extends StandardScheme<TBoolColumn> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TBoolColumn struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUES
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list54 = iprot.readListBegin();
-                struct.values = new ArrayList<Boolean>(_list54.size);
-                boolean _elem55;
-                for (int _i56 = 0; _i56 < _list54.size; ++_i56)
-                {
-                  _elem55 = iprot.readBool();
-                  struct.values.add(_elem55);
-                }
-                iprot.readListEnd();
-              }
-              struct.setValuesIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // NULLS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.nulls = iprot.readBinary();
-              struct.setNullsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TBoolColumn struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.values != null) {
-        oprot.writeFieldBegin(VALUES_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.BOOL, struct.values.size()));
-          for (boolean _iter57 : struct.values)
-          {
-            oprot.writeBool(_iter57);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      if (struct.nulls != null) {
-        oprot.writeFieldBegin(NULLS_FIELD_DESC);
-        oprot.writeBinary(struct.nulls);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TBoolColumnTupleSchemeFactory implements SchemeFactory {
-    public TBoolColumnTupleScheme getScheme() {
-      return new TBoolColumnTupleScheme();
-    }
-  }
-
-  private static class TBoolColumnTupleScheme extends TupleScheme<TBoolColumn> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TBoolColumn struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.values.size());
-        for (boolean _iter58 : struct.values)
-        {
-          oprot.writeBool(_iter58);
-        }
-      }
-      oprot.writeBinary(struct.nulls);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TBoolColumn struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TList _list59 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.BOOL, iprot.readI32());
-        struct.values = new ArrayList<Boolean>(_list59.size);
-        boolean _elem60;
-        for (int _i61 = 0; _i61 < _list59.size; ++_i61)
-        {
-          _elem60 = iprot.readBool();
-          struct.values.add(_elem60);
-        }
-      }
-      struct.setValuesIsSet(true);
-      struct.nulls = iprot.readBinary();
-      struct.setNullsIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TBoolValue.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TBoolValue.java
deleted file mode 100644
index 87b3070a89b11..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TBoolValue.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TBoolValue implements org.apache.thrift.TBase<TBoolValue, TBoolValue._Fields>, java.io.Serializable, Cloneable, Comparable<TBoolValue> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TBoolValue");
-
-  private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.BOOL, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TBoolValueStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TBoolValueTupleSchemeFactory());
-  }
-
-  private boolean value; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUE((short)1, "value");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUE
-          return VALUE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __VALUE_ISSET_ID = 0;
-  private byte __isset_bitfield = 0;
-  private static final _Fields optionals[] = {_Fields.VALUE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TBoolValue.class, metaDataMap);
-  }
-
-  public TBoolValue() {
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TBoolValue(TBoolValue other) {
-    __isset_bitfield = other.__isset_bitfield;
-    this.value = other.value;
-  }
-
-  public TBoolValue deepCopy() {
-    return new TBoolValue(this);
-  }
-
-  @Override
-  public void clear() {
-    setValueIsSet(false);
-    this.value = false;
-  }
-
-  public boolean isValue() {
-    return this.value;
-  }
-
-  public void setValue(boolean value) {
-    this.value = value;
-    setValueIsSet(true);
-  }
-
-  public void unsetValue() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __VALUE_ISSET_ID);
-  }
-
-  /** Returns true if field value is set (has been assigned a value) and false otherwise */
-  public boolean isSetValue() {
-    return EncodingUtils.testBit(__isset_bitfield, __VALUE_ISSET_ID);
-  }
-
-  public void setValueIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __VALUE_ISSET_ID, value);
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUE:
-      if (value == null) {
-        unsetValue();
-      } else {
-        setValue((Boolean)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUE:
-      return isValue();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUE:
-      return isSetValue();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TBoolValue)
-      return this.equals((TBoolValue)that);
-    return false;
-  }
-
-  public boolean equals(TBoolValue that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_value = true && this.isSetValue();
-    boolean that_present_value = true && that.isSetValue();
-    if (this_present_value || that_present_value) {
-      if (!(this_present_value && that_present_value))
-        return false;
-      if (this.value != that.value)
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_value = true && (isSetValue());
-    list.add(present_value);
-    if (present_value)
-      list.add(value);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TBoolValue other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetValue()).compareTo(other.isSetValue());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValue()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, other.value);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TBoolValue(");
-    boolean first = true;
-
-    if (isSetValue()) {
-      sb.append("value:");
-      sb.append(this.value);
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TBoolValueStandardSchemeFactory implements SchemeFactory {
-    public TBoolValueStandardScheme getScheme() {
-      return new TBoolValueStandardScheme();
-    }
-  }
-
-  private static class TBoolValueStandardScheme extends StandardScheme<TBoolValue> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TBoolValue struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUE
-            if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
-              struct.value = iprot.readBool();
-              struct.setValueIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TBoolValue struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.isSetValue()) {
-        oprot.writeFieldBegin(VALUE_FIELD_DESC);
-        oprot.writeBool(struct.value);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TBoolValueTupleSchemeFactory implements SchemeFactory {
-    public TBoolValueTupleScheme getScheme() {
-      return new TBoolValueTupleScheme();
-    }
-  }
-
-  private static class TBoolValueTupleScheme extends TupleScheme<TBoolValue> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TBoolValue struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      BitSet optionals = new BitSet();
-      if (struct.isSetValue()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetValue()) {
-        oprot.writeBool(struct.value);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TBoolValue struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.value = iprot.readBool();
-        struct.setValueIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TByteColumn.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TByteColumn.java
deleted file mode 100644
index 68b3d3c31eb03..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TByteColumn.java
+++ /dev/null
@@ -1,548 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TByteColumn implements org.apache.thrift.TBase<TByteColumn, TByteColumn._Fields>, java.io.Serializable, Cloneable, Comparable<TByteColumn> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TByteColumn");
-
-  private static final org.apache.thrift.protocol.TField VALUES_FIELD_DESC = new org.apache.thrift.protocol.TField("values", org.apache.thrift.protocol.TType.LIST, (short)1);
-  private static final org.apache.thrift.protocol.TField NULLS_FIELD_DESC = new org.apache.thrift.protocol.TField("nulls", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TByteColumnStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TByteColumnTupleSchemeFactory());
-  }
-
-  private List<Byte> values; // required
-  private ByteBuffer nulls; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUES((short)1, "values"),
-    NULLS((short)2, "nulls");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUES
-          return VALUES;
-        case 2: // NULLS
-          return NULLS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUES, new org.apache.thrift.meta_data.FieldMetaData("values", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BYTE))));
-    tmpMap.put(_Fields.NULLS, new org.apache.thrift.meta_data.FieldMetaData("nulls", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TByteColumn.class, metaDataMap);
-  }
-
-  public TByteColumn() {
-  }
-
-  public TByteColumn(
-    List<Byte> values,
-    ByteBuffer nulls)
-  {
-    this();
-    this.values = values;
-    this.nulls = org.apache.thrift.TBaseHelper.copyBinary(nulls);
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TByteColumn(TByteColumn other) {
-    if (other.isSetValues()) {
-      List<Byte> __this__values = new ArrayList<Byte>(other.values);
-      this.values = __this__values;
-    }
-    if (other.isSetNulls()) {
-      this.nulls = org.apache.thrift.TBaseHelper.copyBinary(other.nulls);
-    }
-  }
-
-  public TByteColumn deepCopy() {
-    return new TByteColumn(this);
-  }
-
-  @Override
-  public void clear() {
-    this.values = null;
-    this.nulls = null;
-  }
-
-  public int getValuesSize() {
-    return (this.values == null) ? 0 : this.values.size();
-  }
-
-  public java.util.Iterator<Byte> getValuesIterator() {
-    return (this.values == null) ? null : this.values.iterator();
-  }
-
-  public void addToValues(byte elem) {
-    if (this.values == null) {
-      this.values = new ArrayList<Byte>();
-    }
-    this.values.add(elem);
-  }
-
-  public List<Byte> getValues() {
-    return this.values;
-  }
-
-  public void setValues(List<Byte> values) {
-    this.values = values;
-  }
-
-  public void unsetValues() {
-    this.values = null;
-  }
-
-  /** Returns true if field values is set (has been assigned a value) and false otherwise */
-  public boolean isSetValues() {
-    return this.values != null;
-  }
-
-  public void setValuesIsSet(boolean value) {
-    if (!value) {
-      this.values = null;
-    }
-  }
-
-  public byte[] getNulls() {
-    setNulls(org.apache.thrift.TBaseHelper.rightSize(nulls));
-    return nulls == null ? null : nulls.array();
-  }
-
-  public ByteBuffer bufferForNulls() {
-    return org.apache.thrift.TBaseHelper.copyBinary(nulls);
-  }
-
-  public void setNulls(byte[] nulls) {
-    this.nulls = nulls == null ? (ByteBuffer)null : ByteBuffer.wrap(Arrays.copyOf(nulls, nulls.length));
-  }
-
-  public void setNulls(ByteBuffer nulls) {
-    this.nulls = org.apache.thrift.TBaseHelper.copyBinary(nulls);
-  }
-
-  public void unsetNulls() {
-    this.nulls = null;
-  }
-
-  /** Returns true if field nulls is set (has been assigned a value) and false otherwise */
-  public boolean isSetNulls() {
-    return this.nulls != null;
-  }
-
-  public void setNullsIsSet(boolean value) {
-    if (!value) {
-      this.nulls = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUES:
-      if (value == null) {
-        unsetValues();
-      } else {
-        setValues((List<Byte>)value);
-      }
-      break;
-
-    case NULLS:
-      if (value == null) {
-        unsetNulls();
-      } else {
-        setNulls((ByteBuffer)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUES:
-      return getValues();
-
-    case NULLS:
-      return getNulls();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUES:
-      return isSetValues();
-    case NULLS:
-      return isSetNulls();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TByteColumn)
-      return this.equals((TByteColumn)that);
-    return false;
-  }
-
-  public boolean equals(TByteColumn that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_values = true && this.isSetValues();
-    boolean that_present_values = true && that.isSetValues();
-    if (this_present_values || that_present_values) {
-      if (!(this_present_values && that_present_values))
-        return false;
-      if (!this.values.equals(that.values))
-        return false;
-    }
-
-    boolean this_present_nulls = true && this.isSetNulls();
-    boolean that_present_nulls = true && that.isSetNulls();
-    if (this_present_nulls || that_present_nulls) {
-      if (!(this_present_nulls && that_present_nulls))
-        return false;
-      if (!this.nulls.equals(that.nulls))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_values = true && (isSetValues());
-    list.add(present_values);
-    if (present_values)
-      list.add(values);
-
-    boolean present_nulls = true && (isSetNulls());
-    list.add(present_nulls);
-    if (present_nulls)
-      list.add(nulls);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TByteColumn other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetValues()).compareTo(other.isSetValues());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValues()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.values, other.values);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetNulls()).compareTo(other.isSetNulls());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetNulls()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.nulls, other.nulls);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TByteColumn(");
-    boolean first = true;
-
-    sb.append("values:");
-    if (this.values == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.values);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("nulls:");
-    if (this.nulls == null) {
-      sb.append("null");
-    } else {
-      org.apache.thrift.TBaseHelper.toString(this.nulls, sb);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetValues()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'values' is unset! Struct:" + toString());
-    }
-
-    if (!isSetNulls()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'nulls' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TByteColumnStandardSchemeFactory implements SchemeFactory {
-    public TByteColumnStandardScheme getScheme() {
-      return new TByteColumnStandardScheme();
-    }
-  }
-
-  private static class TByteColumnStandardScheme extends StandardScheme<TByteColumn> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TByteColumn struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUES
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list62 = iprot.readListBegin();
-                struct.values = new ArrayList<Byte>(_list62.size);
-                byte _elem63;
-                for (int _i64 = 0; _i64 < _list62.size; ++_i64)
-                {
-                  _elem63 = iprot.readByte();
-                  struct.values.add(_elem63);
-                }
-                iprot.readListEnd();
-              }
-              struct.setValuesIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // NULLS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.nulls = iprot.readBinary();
-              struct.setNullsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TByteColumn struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.values != null) {
-        oprot.writeFieldBegin(VALUES_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.BYTE, struct.values.size()));
-          for (byte _iter65 : struct.values)
-          {
-            oprot.writeByte(_iter65);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      if (struct.nulls != null) {
-        oprot.writeFieldBegin(NULLS_FIELD_DESC);
-        oprot.writeBinary(struct.nulls);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TByteColumnTupleSchemeFactory implements SchemeFactory {
-    public TByteColumnTupleScheme getScheme() {
-      return new TByteColumnTupleScheme();
-    }
-  }
-
-  private static class TByteColumnTupleScheme extends TupleScheme<TByteColumn> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TByteColumn struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.values.size());
-        for (byte _iter66 : struct.values)
-        {
-          oprot.writeByte(_iter66);
-        }
-      }
-      oprot.writeBinary(struct.nulls);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TByteColumn struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TList _list67 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.BYTE, iprot.readI32());
-        struct.values = new ArrayList<Byte>(_list67.size);
-        byte _elem68;
-        for (int _i69 = 0; _i69 < _list67.size; ++_i69)
-        {
-          _elem68 = iprot.readByte();
-          struct.values.add(_elem68);
-        }
-      }
-      struct.setValuesIsSet(true);
-      struct.nulls = iprot.readBinary();
-      struct.setNullsIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TByteValue.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TByteValue.java
deleted file mode 100644
index a3d5951335fa7..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TByteValue.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TByteValue implements org.apache.thrift.TBase<TByteValue, TByteValue._Fields>, java.io.Serializable, Cloneable, Comparable<TByteValue> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TByteValue");
-
-  private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.BYTE, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TByteValueStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TByteValueTupleSchemeFactory());
-  }
-
-  private byte value; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUE((short)1, "value");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUE
-          return VALUE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __VALUE_ISSET_ID = 0;
-  private byte __isset_bitfield = 0;
-  private static final _Fields optionals[] = {_Fields.VALUE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BYTE)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TByteValue.class, metaDataMap);
-  }
-
-  public TByteValue() {
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TByteValue(TByteValue other) {
-    __isset_bitfield = other.__isset_bitfield;
-    this.value = other.value;
-  }
-
-  public TByteValue deepCopy() {
-    return new TByteValue(this);
-  }
-
-  @Override
-  public void clear() {
-    setValueIsSet(false);
-    this.value = 0;
-  }
-
-  public byte getValue() {
-    return this.value;
-  }
-
-  public void setValue(byte value) {
-    this.value = value;
-    setValueIsSet(true);
-  }
-
-  public void unsetValue() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __VALUE_ISSET_ID);
-  }
-
-  /** Returns true if field value is set (has been assigned a value) and false otherwise */
-  public boolean isSetValue() {
-    return EncodingUtils.testBit(__isset_bitfield, __VALUE_ISSET_ID);
-  }
-
-  public void setValueIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __VALUE_ISSET_ID, value);
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUE:
-      if (value == null) {
-        unsetValue();
-      } else {
-        setValue((Byte)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUE:
-      return getValue();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUE:
-      return isSetValue();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TByteValue)
-      return this.equals((TByteValue)that);
-    return false;
-  }
-
-  public boolean equals(TByteValue that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_value = true && this.isSetValue();
-    boolean that_present_value = true && that.isSetValue();
-    if (this_present_value || that_present_value) {
-      if (!(this_present_value && that_present_value))
-        return false;
-      if (this.value != that.value)
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_value = true && (isSetValue());
-    list.add(present_value);
-    if (present_value)
-      list.add(value);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TByteValue other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetValue()).compareTo(other.isSetValue());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValue()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, other.value);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TByteValue(");
-    boolean first = true;
-
-    if (isSetValue()) {
-      sb.append("value:");
-      sb.append(this.value);
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TByteValueStandardSchemeFactory implements SchemeFactory {
-    public TByteValueStandardScheme getScheme() {
-      return new TByteValueStandardScheme();
-    }
-  }
-
-  private static class TByteValueStandardScheme extends StandardScheme<TByteValue> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TByteValue struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUE
-            if (schemeField.type == org.apache.thrift.protocol.TType.BYTE) {
-              struct.value = iprot.readByte();
-              struct.setValueIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TByteValue struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.isSetValue()) {
-        oprot.writeFieldBegin(VALUE_FIELD_DESC);
-        oprot.writeByte(struct.value);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TByteValueTupleSchemeFactory implements SchemeFactory {
-    public TByteValueTupleScheme getScheme() {
-      return new TByteValueTupleScheme();
-    }
-  }
-
-  private static class TByteValueTupleScheme extends TupleScheme<TByteValue> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TByteValue struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      BitSet optionals = new BitSet();
-      if (struct.isSetValue()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetValue()) {
-        oprot.writeByte(struct.value);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TByteValue struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.value = iprot.readByte();
-        struct.setValueIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCLIService.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCLIService.java
deleted file mode 100644
index 6584c24a0142a..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCLIService.java
+++ /dev/null
@@ -1,18138 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TCLIService {
-
-  public interface Iface {
-
-    public TOpenSessionResp OpenSession(TOpenSessionReq req) throws org.apache.thrift.TException;
-
-    public TCloseSessionResp CloseSession(TCloseSessionReq req) throws org.apache.thrift.TException;
-
-    public TGetInfoResp GetInfo(TGetInfoReq req) throws org.apache.thrift.TException;
-
-    public TExecuteStatementResp ExecuteStatement(TExecuteStatementReq req) throws org.apache.thrift.TException;
-
-    public TGetTypeInfoResp GetTypeInfo(TGetTypeInfoReq req) throws org.apache.thrift.TException;
-
-    public TGetCatalogsResp GetCatalogs(TGetCatalogsReq req) throws org.apache.thrift.TException;
-
-    public TGetSchemasResp GetSchemas(TGetSchemasReq req) throws org.apache.thrift.TException;
-
-    public TGetTablesResp GetTables(TGetTablesReq req) throws org.apache.thrift.TException;
-
-    public TGetTableTypesResp GetTableTypes(TGetTableTypesReq req) throws org.apache.thrift.TException;
-
-    public TGetColumnsResp GetColumns(TGetColumnsReq req) throws org.apache.thrift.TException;
-
-    public TGetFunctionsResp GetFunctions(TGetFunctionsReq req) throws org.apache.thrift.TException;
-
-    public TGetPrimaryKeysResp GetPrimaryKeys(TGetPrimaryKeysReq req) throws org.apache.thrift.TException;
-
-    public TGetCrossReferenceResp GetCrossReference(TGetCrossReferenceReq req) throws org.apache.thrift.TException;
-
-    public TGetOperationStatusResp GetOperationStatus(TGetOperationStatusReq req) throws org.apache.thrift.TException;
-
-    public TCancelOperationResp CancelOperation(TCancelOperationReq req) throws org.apache.thrift.TException;
-
-    public TCloseOperationResp CloseOperation(TCloseOperationReq req) throws org.apache.thrift.TException;
-
-    public TGetResultSetMetadataResp GetResultSetMetadata(TGetResultSetMetadataReq req) throws org.apache.thrift.TException;
-
-    public TFetchResultsResp FetchResults(TFetchResultsReq req) throws org.apache.thrift.TException;
-
-    public TGetDelegationTokenResp GetDelegationToken(TGetDelegationTokenReq req) throws org.apache.thrift.TException;
-
-    public TCancelDelegationTokenResp CancelDelegationToken(TCancelDelegationTokenReq req) throws org.apache.thrift.TException;
-
-    public TRenewDelegationTokenResp RenewDelegationToken(TRenewDelegationTokenReq req) throws org.apache.thrift.TException;
-
-  }
-
-  public interface AsyncIface {
-
-    public void OpenSession(TOpenSessionReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
-    public void CloseSession(TCloseSessionReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
-    public void GetInfo(TGetInfoReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
-    public void ExecuteStatement(TExecuteStatementReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
-    public void GetTypeInfo(TGetTypeInfoReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
-    public void GetCatalogs(TGetCatalogsReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
-    public void GetSchemas(TGetSchemasReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
-    public void GetTables(TGetTablesReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
-    public void GetTableTypes(TGetTableTypesReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
-    public void GetColumns(TGetColumnsReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
-    public void GetFunctions(TGetFunctionsReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
-    public void GetPrimaryKeys(TGetPrimaryKeysReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
-    public void GetCrossReference(TGetCrossReferenceReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
-    public void GetOperationStatus(TGetOperationStatusReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
-    public void CancelOperation(TCancelOperationReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
-    public void CloseOperation(TCloseOperationReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
-    public void GetResultSetMetadata(TGetResultSetMetadataReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
-    public void FetchResults(TFetchResultsReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
-    public void GetDelegationToken(TGetDelegationTokenReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
-    public void CancelDelegationToken(TCancelDelegationTokenReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
-    public void RenewDelegationToken(TRenewDelegationTokenReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
-
-  }
-
-  public static class Client extends org.apache.thrift.TServiceClient implements Iface {
-    public static class Factory implements org.apache.thrift.TServiceClientFactory<Client> {
-      public Factory() {}
-      public Client getClient(org.apache.thrift.protocol.TProtocol prot) {
-        return new Client(prot);
-      }
-      public Client getClient(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
-        return new Client(iprot, oprot);
-      }
-    }
-
-    public Client(org.apache.thrift.protocol.TProtocol prot)
-    {
-      super(prot, prot);
-    }
-
-    public Client(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
-      super(iprot, oprot);
-    }
-
-    public TOpenSessionResp OpenSession(TOpenSessionReq req) throws org.apache.thrift.TException
-    {
-      send_OpenSession(req);
-      return recv_OpenSession();
-    }
-
-    public void send_OpenSession(TOpenSessionReq req) throws org.apache.thrift.TException
-    {
-      OpenSession_args args = new OpenSession_args();
-      args.setReq(req);
-      sendBase("OpenSession", args);
-    }
-
-    public TOpenSessionResp recv_OpenSession() throws org.apache.thrift.TException
-    {
-      OpenSession_result result = new OpenSession_result();
-      receiveBase(result, "OpenSession");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "OpenSession failed: unknown result");
-    }
-
-    public TCloseSessionResp CloseSession(TCloseSessionReq req) throws org.apache.thrift.TException
-    {
-      send_CloseSession(req);
-      return recv_CloseSession();
-    }
-
-    public void send_CloseSession(TCloseSessionReq req) throws org.apache.thrift.TException
-    {
-      CloseSession_args args = new CloseSession_args();
-      args.setReq(req);
-      sendBase("CloseSession", args);
-    }
-
-    public TCloseSessionResp recv_CloseSession() throws org.apache.thrift.TException
-    {
-      CloseSession_result result = new CloseSession_result();
-      receiveBase(result, "CloseSession");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "CloseSession failed: unknown result");
-    }
-
-    public TGetInfoResp GetInfo(TGetInfoReq req) throws org.apache.thrift.TException
-    {
-      send_GetInfo(req);
-      return recv_GetInfo();
-    }
-
-    public void send_GetInfo(TGetInfoReq req) throws org.apache.thrift.TException
-    {
-      GetInfo_args args = new GetInfo_args();
-      args.setReq(req);
-      sendBase("GetInfo", args);
-    }
-
-    public TGetInfoResp recv_GetInfo() throws org.apache.thrift.TException
-    {
-      GetInfo_result result = new GetInfo_result();
-      receiveBase(result, "GetInfo");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "GetInfo failed: unknown result");
-    }
-
-    public TExecuteStatementResp ExecuteStatement(TExecuteStatementReq req) throws org.apache.thrift.TException
-    {
-      send_ExecuteStatement(req);
-      return recv_ExecuteStatement();
-    }
-
-    public void send_ExecuteStatement(TExecuteStatementReq req) throws org.apache.thrift.TException
-    {
-      ExecuteStatement_args args = new ExecuteStatement_args();
-      args.setReq(req);
-      sendBase("ExecuteStatement", args);
-    }
-
-    public TExecuteStatementResp recv_ExecuteStatement() throws org.apache.thrift.TException
-    {
-      ExecuteStatement_result result = new ExecuteStatement_result();
-      receiveBase(result, "ExecuteStatement");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "ExecuteStatement failed: unknown result");
-    }
-
-    public TGetTypeInfoResp GetTypeInfo(TGetTypeInfoReq req) throws org.apache.thrift.TException
-    {
-      send_GetTypeInfo(req);
-      return recv_GetTypeInfo();
-    }
-
-    public void send_GetTypeInfo(TGetTypeInfoReq req) throws org.apache.thrift.TException
-    {
-      GetTypeInfo_args args = new GetTypeInfo_args();
-      args.setReq(req);
-      sendBase("GetTypeInfo", args);
-    }
-
-    public TGetTypeInfoResp recv_GetTypeInfo() throws org.apache.thrift.TException
-    {
-      GetTypeInfo_result result = new GetTypeInfo_result();
-      receiveBase(result, "GetTypeInfo");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "GetTypeInfo failed: unknown result");
-    }
-
-    public TGetCatalogsResp GetCatalogs(TGetCatalogsReq req) throws org.apache.thrift.TException
-    {
-      send_GetCatalogs(req);
-      return recv_GetCatalogs();
-    }
-
-    public void send_GetCatalogs(TGetCatalogsReq req) throws org.apache.thrift.TException
-    {
-      GetCatalogs_args args = new GetCatalogs_args();
-      args.setReq(req);
-      sendBase("GetCatalogs", args);
-    }
-
-    public TGetCatalogsResp recv_GetCatalogs() throws org.apache.thrift.TException
-    {
-      GetCatalogs_result result = new GetCatalogs_result();
-      receiveBase(result, "GetCatalogs");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "GetCatalogs failed: unknown result");
-    }
-
-    public TGetSchemasResp GetSchemas(TGetSchemasReq req) throws org.apache.thrift.TException
-    {
-      send_GetSchemas(req);
-      return recv_GetSchemas();
-    }
-
-    public void send_GetSchemas(TGetSchemasReq req) throws org.apache.thrift.TException
-    {
-      GetSchemas_args args = new GetSchemas_args();
-      args.setReq(req);
-      sendBase("GetSchemas", args);
-    }
-
-    public TGetSchemasResp recv_GetSchemas() throws org.apache.thrift.TException
-    {
-      GetSchemas_result result = new GetSchemas_result();
-      receiveBase(result, "GetSchemas");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "GetSchemas failed: unknown result");
-    }
-
-    public TGetTablesResp GetTables(TGetTablesReq req) throws org.apache.thrift.TException
-    {
-      send_GetTables(req);
-      return recv_GetTables();
-    }
-
-    public void send_GetTables(TGetTablesReq req) throws org.apache.thrift.TException
-    {
-      GetTables_args args = new GetTables_args();
-      args.setReq(req);
-      sendBase("GetTables", args);
-    }
-
-    public TGetTablesResp recv_GetTables() throws org.apache.thrift.TException
-    {
-      GetTables_result result = new GetTables_result();
-      receiveBase(result, "GetTables");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "GetTables failed: unknown result");
-    }
-
-    public TGetTableTypesResp GetTableTypes(TGetTableTypesReq req) throws org.apache.thrift.TException
-    {
-      send_GetTableTypes(req);
-      return recv_GetTableTypes();
-    }
-
-    public void send_GetTableTypes(TGetTableTypesReq req) throws org.apache.thrift.TException
-    {
-      GetTableTypes_args args = new GetTableTypes_args();
-      args.setReq(req);
-      sendBase("GetTableTypes", args);
-    }
-
-    public TGetTableTypesResp recv_GetTableTypes() throws org.apache.thrift.TException
-    {
-      GetTableTypes_result result = new GetTableTypes_result();
-      receiveBase(result, "GetTableTypes");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "GetTableTypes failed: unknown result");
-    }
-
-    public TGetColumnsResp GetColumns(TGetColumnsReq req) throws org.apache.thrift.TException
-    {
-      send_GetColumns(req);
-      return recv_GetColumns();
-    }
-
-    public void send_GetColumns(TGetColumnsReq req) throws org.apache.thrift.TException
-    {
-      GetColumns_args args = new GetColumns_args();
-      args.setReq(req);
-      sendBase("GetColumns", args);
-    }
-
-    public TGetColumnsResp recv_GetColumns() throws org.apache.thrift.TException
-    {
-      GetColumns_result result = new GetColumns_result();
-      receiveBase(result, "GetColumns");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "GetColumns failed: unknown result");
-    }
-
-    public TGetFunctionsResp GetFunctions(TGetFunctionsReq req) throws org.apache.thrift.TException
-    {
-      send_GetFunctions(req);
-      return recv_GetFunctions();
-    }
-
-    public void send_GetFunctions(TGetFunctionsReq req) throws org.apache.thrift.TException
-    {
-      GetFunctions_args args = new GetFunctions_args();
-      args.setReq(req);
-      sendBase("GetFunctions", args);
-    }
-
-    public TGetFunctionsResp recv_GetFunctions() throws org.apache.thrift.TException
-    {
-      GetFunctions_result result = new GetFunctions_result();
-      receiveBase(result, "GetFunctions");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "GetFunctions failed: unknown result");
-    }
-
-    public TGetPrimaryKeysResp GetPrimaryKeys(TGetPrimaryKeysReq req) throws org.apache.thrift.TException
-    {
-      send_GetPrimaryKeys(req);
-      return recv_GetPrimaryKeys();
-    }
-
-    public void send_GetPrimaryKeys(TGetPrimaryKeysReq req) throws org.apache.thrift.TException
-    {
-      GetPrimaryKeys_args args = new GetPrimaryKeys_args();
-      args.setReq(req);
-      sendBase("GetPrimaryKeys", args);
-    }
-
-    public TGetPrimaryKeysResp recv_GetPrimaryKeys() throws org.apache.thrift.TException
-    {
-      GetPrimaryKeys_result result = new GetPrimaryKeys_result();
-      receiveBase(result, "GetPrimaryKeys");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "GetPrimaryKeys failed: unknown result");
-    }
-
-    public TGetCrossReferenceResp GetCrossReference(TGetCrossReferenceReq req) throws org.apache.thrift.TException
-    {
-      send_GetCrossReference(req);
-      return recv_GetCrossReference();
-    }
-
-    public void send_GetCrossReference(TGetCrossReferenceReq req) throws org.apache.thrift.TException
-    {
-      GetCrossReference_args args = new GetCrossReference_args();
-      args.setReq(req);
-      sendBase("GetCrossReference", args);
-    }
-
-    public TGetCrossReferenceResp recv_GetCrossReference() throws org.apache.thrift.TException
-    {
-      GetCrossReference_result result = new GetCrossReference_result();
-      receiveBase(result, "GetCrossReference");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "GetCrossReference failed: unknown result");
-    }
-
-    public TGetOperationStatusResp GetOperationStatus(TGetOperationStatusReq req) throws org.apache.thrift.TException
-    {
-      send_GetOperationStatus(req);
-      return recv_GetOperationStatus();
-    }
-
-    public void send_GetOperationStatus(TGetOperationStatusReq req) throws org.apache.thrift.TException
-    {
-      GetOperationStatus_args args = new GetOperationStatus_args();
-      args.setReq(req);
-      sendBase("GetOperationStatus", args);
-    }
-
-    public TGetOperationStatusResp recv_GetOperationStatus() throws org.apache.thrift.TException
-    {
-      GetOperationStatus_result result = new GetOperationStatus_result();
-      receiveBase(result, "GetOperationStatus");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "GetOperationStatus failed: unknown result");
-    }
-
-    public TCancelOperationResp CancelOperation(TCancelOperationReq req) throws org.apache.thrift.TException
-    {
-      send_CancelOperation(req);
-      return recv_CancelOperation();
-    }
-
-    public void send_CancelOperation(TCancelOperationReq req) throws org.apache.thrift.TException
-    {
-      CancelOperation_args args = new CancelOperation_args();
-      args.setReq(req);
-      sendBase("CancelOperation", args);
-    }
-
-    public TCancelOperationResp recv_CancelOperation() throws org.apache.thrift.TException
-    {
-      CancelOperation_result result = new CancelOperation_result();
-      receiveBase(result, "CancelOperation");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "CancelOperation failed: unknown result");
-    }
-
-    public TCloseOperationResp CloseOperation(TCloseOperationReq req) throws org.apache.thrift.TException
-    {
-      send_CloseOperation(req);
-      return recv_CloseOperation();
-    }
-
-    public void send_CloseOperation(TCloseOperationReq req) throws org.apache.thrift.TException
-    {
-      CloseOperation_args args = new CloseOperation_args();
-      args.setReq(req);
-      sendBase("CloseOperation", args);
-    }
-
-    public TCloseOperationResp recv_CloseOperation() throws org.apache.thrift.TException
-    {
-      CloseOperation_result result = new CloseOperation_result();
-      receiveBase(result, "CloseOperation");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "CloseOperation failed: unknown result");
-    }
-
-    public TGetResultSetMetadataResp GetResultSetMetadata(TGetResultSetMetadataReq req) throws org.apache.thrift.TException
-    {
-      send_GetResultSetMetadata(req);
-      return recv_GetResultSetMetadata();
-    }
-
-    public void send_GetResultSetMetadata(TGetResultSetMetadataReq req) throws org.apache.thrift.TException
-    {
-      GetResultSetMetadata_args args = new GetResultSetMetadata_args();
-      args.setReq(req);
-      sendBase("GetResultSetMetadata", args);
-    }
-
-    public TGetResultSetMetadataResp recv_GetResultSetMetadata() throws org.apache.thrift.TException
-    {
-      GetResultSetMetadata_result result = new GetResultSetMetadata_result();
-      receiveBase(result, "GetResultSetMetadata");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "GetResultSetMetadata failed: unknown result");
-    }
-
-    public TFetchResultsResp FetchResults(TFetchResultsReq req) throws org.apache.thrift.TException
-    {
-      send_FetchResults(req);
-      return recv_FetchResults();
-    }
-
-    public void send_FetchResults(TFetchResultsReq req) throws org.apache.thrift.TException
-    {
-      FetchResults_args args = new FetchResults_args();
-      args.setReq(req);
-      sendBase("FetchResults", args);
-    }
-
-    public TFetchResultsResp recv_FetchResults() throws org.apache.thrift.TException
-    {
-      FetchResults_result result = new FetchResults_result();
-      receiveBase(result, "FetchResults");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "FetchResults failed: unknown result");
-    }
-
-    public TGetDelegationTokenResp GetDelegationToken(TGetDelegationTokenReq req) throws org.apache.thrift.TException
-    {
-      send_GetDelegationToken(req);
-      return recv_GetDelegationToken();
-    }
-
-    public void send_GetDelegationToken(TGetDelegationTokenReq req) throws org.apache.thrift.TException
-    {
-      GetDelegationToken_args args = new GetDelegationToken_args();
-      args.setReq(req);
-      sendBase("GetDelegationToken", args);
-    }
-
-    public TGetDelegationTokenResp recv_GetDelegationToken() throws org.apache.thrift.TException
-    {
-      GetDelegationToken_result result = new GetDelegationToken_result();
-      receiveBase(result, "GetDelegationToken");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "GetDelegationToken failed: unknown result");
-    }
-
-    public TCancelDelegationTokenResp CancelDelegationToken(TCancelDelegationTokenReq req) throws org.apache.thrift.TException
-    {
-      send_CancelDelegationToken(req);
-      return recv_CancelDelegationToken();
-    }
-
-    public void send_CancelDelegationToken(TCancelDelegationTokenReq req) throws org.apache.thrift.TException
-    {
-      CancelDelegationToken_args args = new CancelDelegationToken_args();
-      args.setReq(req);
-      sendBase("CancelDelegationToken", args);
-    }
-
-    public TCancelDelegationTokenResp recv_CancelDelegationToken() throws org.apache.thrift.TException
-    {
-      CancelDelegationToken_result result = new CancelDelegationToken_result();
-      receiveBase(result, "CancelDelegationToken");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "CancelDelegationToken failed: unknown result");
-    }
-
-    public TRenewDelegationTokenResp RenewDelegationToken(TRenewDelegationTokenReq req) throws org.apache.thrift.TException
-    {
-      send_RenewDelegationToken(req);
-      return recv_RenewDelegationToken();
-    }
-
-    public void send_RenewDelegationToken(TRenewDelegationTokenReq req) throws org.apache.thrift.TException
-    {
-      RenewDelegationToken_args args = new RenewDelegationToken_args();
-      args.setReq(req);
-      sendBase("RenewDelegationToken", args);
-    }
-
-    public TRenewDelegationTokenResp recv_RenewDelegationToken() throws org.apache.thrift.TException
-    {
-      RenewDelegationToken_result result = new RenewDelegationToken_result();
-      receiveBase(result, "RenewDelegationToken");
-      if (result.isSetSuccess()) {
-        return result.success;
-      }
-      throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "RenewDelegationToken failed: unknown result");
-    }
-
-  }
-  public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface {
-    public static class Factory implements org.apache.thrift.async.TAsyncClientFactory<AsyncClient> {
-      private org.apache.thrift.async.TAsyncClientManager clientManager;
-      private org.apache.thrift.protocol.TProtocolFactory protocolFactory;
-      public Factory(org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.protocol.TProtocolFactory protocolFactory) {
-        this.clientManager = clientManager;
-        this.protocolFactory = protocolFactory;
-      }
-      public AsyncClient getAsyncClient(org.apache.thrift.transport.TNonblockingTransport transport) {
-        return new AsyncClient(protocolFactory, clientManager, transport);
-      }
-    }
-
-    public AsyncClient(org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.transport.TNonblockingTransport transport) {
-      super(protocolFactory, clientManager, transport);
-    }
-
-    public void OpenSession(TOpenSessionReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      OpenSession_call method_call = new OpenSession_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class OpenSession_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TOpenSessionReq req;
-      public OpenSession_call(TOpenSessionReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("OpenSession", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        OpenSession_args args = new OpenSession_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TOpenSessionResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_OpenSession();
-      }
-    }
-
-    public void CloseSession(TCloseSessionReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      CloseSession_call method_call = new CloseSession_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class CloseSession_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TCloseSessionReq req;
-      public CloseSession_call(TCloseSessionReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("CloseSession", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        CloseSession_args args = new CloseSession_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TCloseSessionResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_CloseSession();
-      }
-    }
-
-    public void GetInfo(TGetInfoReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      GetInfo_call method_call = new GetInfo_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class GetInfo_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TGetInfoReq req;
-      public GetInfo_call(TGetInfoReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("GetInfo", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        GetInfo_args args = new GetInfo_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TGetInfoResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_GetInfo();
-      }
-    }
-
-    public void ExecuteStatement(TExecuteStatementReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      ExecuteStatement_call method_call = new ExecuteStatement_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class ExecuteStatement_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TExecuteStatementReq req;
-      public ExecuteStatement_call(TExecuteStatementReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("ExecuteStatement", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        ExecuteStatement_args args = new ExecuteStatement_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TExecuteStatementResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_ExecuteStatement();
-      }
-    }
-
-    public void GetTypeInfo(TGetTypeInfoReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      GetTypeInfo_call method_call = new GetTypeInfo_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class GetTypeInfo_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TGetTypeInfoReq req;
-      public GetTypeInfo_call(TGetTypeInfoReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("GetTypeInfo", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        GetTypeInfo_args args = new GetTypeInfo_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TGetTypeInfoResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_GetTypeInfo();
-      }
-    }
-
-    public void GetCatalogs(TGetCatalogsReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      GetCatalogs_call method_call = new GetCatalogs_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class GetCatalogs_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TGetCatalogsReq req;
-      public GetCatalogs_call(TGetCatalogsReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("GetCatalogs", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        GetCatalogs_args args = new GetCatalogs_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TGetCatalogsResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_GetCatalogs();
-      }
-    }
-
-    public void GetSchemas(TGetSchemasReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      GetSchemas_call method_call = new GetSchemas_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class GetSchemas_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TGetSchemasReq req;
-      public GetSchemas_call(TGetSchemasReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("GetSchemas", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        GetSchemas_args args = new GetSchemas_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TGetSchemasResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_GetSchemas();
-      }
-    }
-
-    public void GetTables(TGetTablesReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      GetTables_call method_call = new GetTables_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class GetTables_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TGetTablesReq req;
-      public GetTables_call(TGetTablesReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("GetTables", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        GetTables_args args = new GetTables_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TGetTablesResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_GetTables();
-      }
-    }
-
-    public void GetTableTypes(TGetTableTypesReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      GetTableTypes_call method_call = new GetTableTypes_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class GetTableTypes_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TGetTableTypesReq req;
-      public GetTableTypes_call(TGetTableTypesReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("GetTableTypes", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        GetTableTypes_args args = new GetTableTypes_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TGetTableTypesResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_GetTableTypes();
-      }
-    }
-
-    public void GetColumns(TGetColumnsReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      GetColumns_call method_call = new GetColumns_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class GetColumns_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TGetColumnsReq req;
-      public GetColumns_call(TGetColumnsReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("GetColumns", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        GetColumns_args args = new GetColumns_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TGetColumnsResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_GetColumns();
-      }
-    }
-
-    public void GetFunctions(TGetFunctionsReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      GetFunctions_call method_call = new GetFunctions_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class GetFunctions_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TGetFunctionsReq req;
-      public GetFunctions_call(TGetFunctionsReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("GetFunctions", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        GetFunctions_args args = new GetFunctions_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TGetFunctionsResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_GetFunctions();
-      }
-    }
-
-    public void GetPrimaryKeys(TGetPrimaryKeysReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      GetPrimaryKeys_call method_call = new GetPrimaryKeys_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class GetPrimaryKeys_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TGetPrimaryKeysReq req;
-      public GetPrimaryKeys_call(TGetPrimaryKeysReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("GetPrimaryKeys", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        GetPrimaryKeys_args args = new GetPrimaryKeys_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TGetPrimaryKeysResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_GetPrimaryKeys();
-      }
-    }
-
-    public void GetCrossReference(TGetCrossReferenceReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      GetCrossReference_call method_call = new GetCrossReference_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class GetCrossReference_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TGetCrossReferenceReq req;
-      public GetCrossReference_call(TGetCrossReferenceReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("GetCrossReference", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        GetCrossReference_args args = new GetCrossReference_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TGetCrossReferenceResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_GetCrossReference();
-      }
-    }
-
-    public void GetOperationStatus(TGetOperationStatusReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      GetOperationStatus_call method_call = new GetOperationStatus_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class GetOperationStatus_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TGetOperationStatusReq req;
-      public GetOperationStatus_call(TGetOperationStatusReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("GetOperationStatus", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        GetOperationStatus_args args = new GetOperationStatus_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TGetOperationStatusResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_GetOperationStatus();
-      }
-    }
-
-    public void CancelOperation(TCancelOperationReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      CancelOperation_call method_call = new CancelOperation_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class CancelOperation_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TCancelOperationReq req;
-      public CancelOperation_call(TCancelOperationReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("CancelOperation", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        CancelOperation_args args = new CancelOperation_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TCancelOperationResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_CancelOperation();
-      }
-    }
-
-    public void CloseOperation(TCloseOperationReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      CloseOperation_call method_call = new CloseOperation_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class CloseOperation_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TCloseOperationReq req;
-      public CloseOperation_call(TCloseOperationReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("CloseOperation", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        CloseOperation_args args = new CloseOperation_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TCloseOperationResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_CloseOperation();
-      }
-    }
-
-    public void GetResultSetMetadata(TGetResultSetMetadataReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      GetResultSetMetadata_call method_call = new GetResultSetMetadata_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class GetResultSetMetadata_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TGetResultSetMetadataReq req;
-      public GetResultSetMetadata_call(TGetResultSetMetadataReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("GetResultSetMetadata", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        GetResultSetMetadata_args args = new GetResultSetMetadata_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TGetResultSetMetadataResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_GetResultSetMetadata();
-      }
-    }
-
-    public void FetchResults(TFetchResultsReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      FetchResults_call method_call = new FetchResults_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class FetchResults_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TFetchResultsReq req;
-      public FetchResults_call(TFetchResultsReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("FetchResults", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        FetchResults_args args = new FetchResults_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TFetchResultsResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_FetchResults();
-      }
-    }
-
-    public void GetDelegationToken(TGetDelegationTokenReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      GetDelegationToken_call method_call = new GetDelegationToken_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class GetDelegationToken_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TGetDelegationTokenReq req;
-      public GetDelegationToken_call(TGetDelegationTokenReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("GetDelegationToken", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        GetDelegationToken_args args = new GetDelegationToken_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TGetDelegationTokenResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_GetDelegationToken();
-      }
-    }
-
-    public void CancelDelegationToken(TCancelDelegationTokenReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      CancelDelegationToken_call method_call = new CancelDelegationToken_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class CancelDelegationToken_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TCancelDelegationTokenReq req;
-      public CancelDelegationToken_call(TCancelDelegationTokenReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("CancelDelegationToken", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        CancelDelegationToken_args args = new CancelDelegationToken_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TCancelDelegationTokenResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_CancelDelegationToken();
-      }
-    }
-
-    public void RenewDelegationToken(TRenewDelegationTokenReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
-      checkReady();
-      RenewDelegationToken_call method_call = new RenewDelegationToken_call(req, resultHandler, this, ___protocolFactory, ___transport);
-      this.___currentMethod = method_call;
-      ___manager.call(method_call);
-    }
-
-    public static class RenewDelegationToken_call extends org.apache.thrift.async.TAsyncMethodCall {
-      private TRenewDelegationTokenReq req;
-      public RenewDelegationToken_call(TRenewDelegationTokenReq req, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
-        super(client, protocolFactory, transport, resultHandler, false);
-        this.req = req;
-      }
-
-      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
-        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("RenewDelegationToken", org.apache.thrift.protocol.TMessageType.CALL, 0));
-        RenewDelegationToken_args args = new RenewDelegationToken_args();
-        args.setReq(req);
-        args.write(prot);
-        prot.writeMessageEnd();
-      }
-
-      public TRenewDelegationTokenResp getResult() throws org.apache.thrift.TException {
-        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
-          throw new IllegalStateException("Method call not finished!");
-        }
-        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
-        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
-        return (new Client(prot)).recv_RenewDelegationToken();
-      }
-    }
-
-  }
-
-  public static class Processor<I extends Iface> extends org.apache.thrift.TBaseProcessor<I> implements org.apache.thrift.TProcessor {
-    private static final Logger LOGGER = LoggerFactory.getLogger(Processor.class.getName());
-    public Processor(I iface) {
-      super(iface, getProcessMap(new HashMap<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>>()));
-    }
-
-    protected Processor(I iface, Map<String,  org.apache.thrift.ProcessFunction<I, ? extends  org.apache.thrift.TBase>> processMap) {
-      super(iface, getProcessMap(processMap));
-    }
-
-    private static <I extends Iface> Map<String,  org.apache.thrift.ProcessFunction<I, ? extends  org.apache.thrift.TBase>> getProcessMap(Map<String,  org.apache.thrift.ProcessFunction<I, ? extends  org.apache.thrift.TBase>> processMap) {
-      processMap.put("OpenSession", new OpenSession());
-      processMap.put("CloseSession", new CloseSession());
-      processMap.put("GetInfo", new GetInfo());
-      processMap.put("ExecuteStatement", new ExecuteStatement());
-      processMap.put("GetTypeInfo", new GetTypeInfo());
-      processMap.put("GetCatalogs", new GetCatalogs());
-      processMap.put("GetSchemas", new GetSchemas());
-      processMap.put("GetTables", new GetTables());
-      processMap.put("GetTableTypes", new GetTableTypes());
-      processMap.put("GetColumns", new GetColumns());
-      processMap.put("GetFunctions", new GetFunctions());
-      processMap.put("GetPrimaryKeys", new GetPrimaryKeys());
-      processMap.put("GetCrossReference", new GetCrossReference());
-      processMap.put("GetOperationStatus", new GetOperationStatus());
-      processMap.put("CancelOperation", new CancelOperation());
-      processMap.put("CloseOperation", new CloseOperation());
-      processMap.put("GetResultSetMetadata", new GetResultSetMetadata());
-      processMap.put("FetchResults", new FetchResults());
-      processMap.put("GetDelegationToken", new GetDelegationToken());
-      processMap.put("CancelDelegationToken", new CancelDelegationToken());
-      processMap.put("RenewDelegationToken", new RenewDelegationToken());
-      return processMap;
-    }
-
-    public static class OpenSession<I extends Iface> extends org.apache.thrift.ProcessFunction<I, OpenSession_args> {
-      public OpenSession() {
-        super("OpenSession");
-      }
-
-      public OpenSession_args getEmptyArgsInstance() {
-        return new OpenSession_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public OpenSession_result getResult(I iface, OpenSession_args args) throws org.apache.thrift.TException {
-        OpenSession_result result = new OpenSession_result();
-        result.success = iface.OpenSession(args.req);
-        return result;
-      }
-    }
-
-    public static class CloseSession<I extends Iface> extends org.apache.thrift.ProcessFunction<I, CloseSession_args> {
-      public CloseSession() {
-        super("CloseSession");
-      }
-
-      public CloseSession_args getEmptyArgsInstance() {
-        return new CloseSession_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public CloseSession_result getResult(I iface, CloseSession_args args) throws org.apache.thrift.TException {
-        CloseSession_result result = new CloseSession_result();
-        result.success = iface.CloseSession(args.req);
-        return result;
-      }
-    }
-
-    public static class GetInfo<I extends Iface> extends org.apache.thrift.ProcessFunction<I, GetInfo_args> {
-      public GetInfo() {
-        super("GetInfo");
-      }
-
-      public GetInfo_args getEmptyArgsInstance() {
-        return new GetInfo_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public GetInfo_result getResult(I iface, GetInfo_args args) throws org.apache.thrift.TException {
-        GetInfo_result result = new GetInfo_result();
-        result.success = iface.GetInfo(args.req);
-        return result;
-      }
-    }
-
-    public static class ExecuteStatement<I extends Iface> extends org.apache.thrift.ProcessFunction<I, ExecuteStatement_args> {
-      public ExecuteStatement() {
-        super("ExecuteStatement");
-      }
-
-      public ExecuteStatement_args getEmptyArgsInstance() {
-        return new ExecuteStatement_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public ExecuteStatement_result getResult(I iface, ExecuteStatement_args args) throws org.apache.thrift.TException {
-        ExecuteStatement_result result = new ExecuteStatement_result();
-        result.success = iface.ExecuteStatement(args.req);
-        return result;
-      }
-    }
-
-    public static class GetTypeInfo<I extends Iface> extends org.apache.thrift.ProcessFunction<I, GetTypeInfo_args> {
-      public GetTypeInfo() {
-        super("GetTypeInfo");
-      }
-
-      public GetTypeInfo_args getEmptyArgsInstance() {
-        return new GetTypeInfo_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public GetTypeInfo_result getResult(I iface, GetTypeInfo_args args) throws org.apache.thrift.TException {
-        GetTypeInfo_result result = new GetTypeInfo_result();
-        result.success = iface.GetTypeInfo(args.req);
-        return result;
-      }
-    }
-
-    public static class GetCatalogs<I extends Iface> extends org.apache.thrift.ProcessFunction<I, GetCatalogs_args> {
-      public GetCatalogs() {
-        super("GetCatalogs");
-      }
-
-      public GetCatalogs_args getEmptyArgsInstance() {
-        return new GetCatalogs_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public GetCatalogs_result getResult(I iface, GetCatalogs_args args) throws org.apache.thrift.TException {
-        GetCatalogs_result result = new GetCatalogs_result();
-        result.success = iface.GetCatalogs(args.req);
-        return result;
-      }
-    }
-
-    public static class GetSchemas<I extends Iface> extends org.apache.thrift.ProcessFunction<I, GetSchemas_args> {
-      public GetSchemas() {
-        super("GetSchemas");
-      }
-
-      public GetSchemas_args getEmptyArgsInstance() {
-        return new GetSchemas_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public GetSchemas_result getResult(I iface, GetSchemas_args args) throws org.apache.thrift.TException {
-        GetSchemas_result result = new GetSchemas_result();
-        result.success = iface.GetSchemas(args.req);
-        return result;
-      }
-    }
-
-    public static class GetTables<I extends Iface> extends org.apache.thrift.ProcessFunction<I, GetTables_args> {
-      public GetTables() {
-        super("GetTables");
-      }
-
-      public GetTables_args getEmptyArgsInstance() {
-        return new GetTables_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public GetTables_result getResult(I iface, GetTables_args args) throws org.apache.thrift.TException {
-        GetTables_result result = new GetTables_result();
-        result.success = iface.GetTables(args.req);
-        return result;
-      }
-    }
-
-    public static class GetTableTypes<I extends Iface> extends org.apache.thrift.ProcessFunction<I, GetTableTypes_args> {
-      public GetTableTypes() {
-        super("GetTableTypes");
-      }
-
-      public GetTableTypes_args getEmptyArgsInstance() {
-        return new GetTableTypes_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public GetTableTypes_result getResult(I iface, GetTableTypes_args args) throws org.apache.thrift.TException {
-        GetTableTypes_result result = new GetTableTypes_result();
-        result.success = iface.GetTableTypes(args.req);
-        return result;
-      }
-    }
-
-    public static class GetColumns<I extends Iface> extends org.apache.thrift.ProcessFunction<I, GetColumns_args> {
-      public GetColumns() {
-        super("GetColumns");
-      }
-
-      public GetColumns_args getEmptyArgsInstance() {
-        return new GetColumns_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public GetColumns_result getResult(I iface, GetColumns_args args) throws org.apache.thrift.TException {
-        GetColumns_result result = new GetColumns_result();
-        result.success = iface.GetColumns(args.req);
-        return result;
-      }
-    }
-
-    public static class GetFunctions<I extends Iface> extends org.apache.thrift.ProcessFunction<I, GetFunctions_args> {
-      public GetFunctions() {
-        super("GetFunctions");
-      }
-
-      public GetFunctions_args getEmptyArgsInstance() {
-        return new GetFunctions_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public GetFunctions_result getResult(I iface, GetFunctions_args args) throws org.apache.thrift.TException {
-        GetFunctions_result result = new GetFunctions_result();
-        result.success = iface.GetFunctions(args.req);
-        return result;
-      }
-    }
-
-    public static class GetPrimaryKeys<I extends Iface> extends org.apache.thrift.ProcessFunction<I, GetPrimaryKeys_args> {
-      public GetPrimaryKeys() {
-        super("GetPrimaryKeys");
-      }
-
-      public GetPrimaryKeys_args getEmptyArgsInstance() {
-        return new GetPrimaryKeys_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public GetPrimaryKeys_result getResult(I iface, GetPrimaryKeys_args args) throws org.apache.thrift.TException {
-        GetPrimaryKeys_result result = new GetPrimaryKeys_result();
-        result.success = iface.GetPrimaryKeys(args.req);
-        return result;
-      }
-    }
-
-    public static class GetCrossReference<I extends Iface> extends org.apache.thrift.ProcessFunction<I, GetCrossReference_args> {
-      public GetCrossReference() {
-        super("GetCrossReference");
-      }
-
-      public GetCrossReference_args getEmptyArgsInstance() {
-        return new GetCrossReference_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public GetCrossReference_result getResult(I iface, GetCrossReference_args args) throws org.apache.thrift.TException {
-        GetCrossReference_result result = new GetCrossReference_result();
-        result.success = iface.GetCrossReference(args.req);
-        return result;
-      }
-    }
-
-    public static class GetOperationStatus<I extends Iface> extends org.apache.thrift.ProcessFunction<I, GetOperationStatus_args> {
-      public GetOperationStatus() {
-        super("GetOperationStatus");
-      }
-
-      public GetOperationStatus_args getEmptyArgsInstance() {
-        return new GetOperationStatus_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public GetOperationStatus_result getResult(I iface, GetOperationStatus_args args) throws org.apache.thrift.TException {
-        GetOperationStatus_result result = new GetOperationStatus_result();
-        result.success = iface.GetOperationStatus(args.req);
-        return result;
-      }
-    }
-
-    public static class CancelOperation<I extends Iface> extends org.apache.thrift.ProcessFunction<I, CancelOperation_args> {
-      public CancelOperation() {
-        super("CancelOperation");
-      }
-
-      public CancelOperation_args getEmptyArgsInstance() {
-        return new CancelOperation_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public CancelOperation_result getResult(I iface, CancelOperation_args args) throws org.apache.thrift.TException {
-        CancelOperation_result result = new CancelOperation_result();
-        result.success = iface.CancelOperation(args.req);
-        return result;
-      }
-    }
-
-    public static class CloseOperation<I extends Iface> extends org.apache.thrift.ProcessFunction<I, CloseOperation_args> {
-      public CloseOperation() {
-        super("CloseOperation");
-      }
-
-      public CloseOperation_args getEmptyArgsInstance() {
-        return new CloseOperation_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public CloseOperation_result getResult(I iface, CloseOperation_args args) throws org.apache.thrift.TException {
-        CloseOperation_result result = new CloseOperation_result();
-        result.success = iface.CloseOperation(args.req);
-        return result;
-      }
-    }
-
-    public static class GetResultSetMetadata<I extends Iface> extends org.apache.thrift.ProcessFunction<I, GetResultSetMetadata_args> {
-      public GetResultSetMetadata() {
-        super("GetResultSetMetadata");
-      }
-
-      public GetResultSetMetadata_args getEmptyArgsInstance() {
-        return new GetResultSetMetadata_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public GetResultSetMetadata_result getResult(I iface, GetResultSetMetadata_args args) throws org.apache.thrift.TException {
-        GetResultSetMetadata_result result = new GetResultSetMetadata_result();
-        result.success = iface.GetResultSetMetadata(args.req);
-        return result;
-      }
-    }
-
-    public static class FetchResults<I extends Iface> extends org.apache.thrift.ProcessFunction<I, FetchResults_args> {
-      public FetchResults() {
-        super("FetchResults");
-      }
-
-      public FetchResults_args getEmptyArgsInstance() {
-        return new FetchResults_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public FetchResults_result getResult(I iface, FetchResults_args args) throws org.apache.thrift.TException {
-        FetchResults_result result = new FetchResults_result();
-        result.success = iface.FetchResults(args.req);
-        return result;
-      }
-    }
-
-    public static class GetDelegationToken<I extends Iface> extends org.apache.thrift.ProcessFunction<I, GetDelegationToken_args> {
-      public GetDelegationToken() {
-        super("GetDelegationToken");
-      }
-
-      public GetDelegationToken_args getEmptyArgsInstance() {
-        return new GetDelegationToken_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public GetDelegationToken_result getResult(I iface, GetDelegationToken_args args) throws org.apache.thrift.TException {
-        GetDelegationToken_result result = new GetDelegationToken_result();
-        result.success = iface.GetDelegationToken(args.req);
-        return result;
-      }
-    }
-
-    public static class CancelDelegationToken<I extends Iface> extends org.apache.thrift.ProcessFunction<I, CancelDelegationToken_args> {
-      public CancelDelegationToken() {
-        super("CancelDelegationToken");
-      }
-
-      public CancelDelegationToken_args getEmptyArgsInstance() {
-        return new CancelDelegationToken_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public CancelDelegationToken_result getResult(I iface, CancelDelegationToken_args args) throws org.apache.thrift.TException {
-        CancelDelegationToken_result result = new CancelDelegationToken_result();
-        result.success = iface.CancelDelegationToken(args.req);
-        return result;
-      }
-    }
-
-    public static class RenewDelegationToken<I extends Iface> extends org.apache.thrift.ProcessFunction<I, RenewDelegationToken_args> {
-      public RenewDelegationToken() {
-        super("RenewDelegationToken");
-      }
-
-      public RenewDelegationToken_args getEmptyArgsInstance() {
-        return new RenewDelegationToken_args();
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public RenewDelegationToken_result getResult(I iface, RenewDelegationToken_args args) throws org.apache.thrift.TException {
-        RenewDelegationToken_result result = new RenewDelegationToken_result();
-        result.success = iface.RenewDelegationToken(args.req);
-        return result;
-      }
-    }
-
-  }
-
-  public static class AsyncProcessor<I extends AsyncIface> extends org.apache.thrift.TBaseAsyncProcessor<I> {
-    private static final Logger LOGGER = LoggerFactory.getLogger(AsyncProcessor.class.getName());
-    public AsyncProcessor(I iface) {
-      super(iface, getProcessMap(new HashMap<String, org.apache.thrift.AsyncProcessFunction<I, ? extends org.apache.thrift.TBase, ?>>()));
-    }
-
-    protected AsyncProcessor(I iface, Map<String,  org.apache.thrift.AsyncProcessFunction<I, ? extends  org.apache.thrift.TBase, ?>> processMap) {
-      super(iface, getProcessMap(processMap));
-    }
-
-    private static <I extends AsyncIface> Map<String,  org.apache.thrift.AsyncProcessFunction<I, ? extends  org.apache.thrift.TBase,?>> getProcessMap(Map<String,  org.apache.thrift.AsyncProcessFunction<I, ? extends  org.apache.thrift.TBase, ?>> processMap) {
-      processMap.put("OpenSession", new OpenSession());
-      processMap.put("CloseSession", new CloseSession());
-      processMap.put("GetInfo", new GetInfo());
-      processMap.put("ExecuteStatement", new ExecuteStatement());
-      processMap.put("GetTypeInfo", new GetTypeInfo());
-      processMap.put("GetCatalogs", new GetCatalogs());
-      processMap.put("GetSchemas", new GetSchemas());
-      processMap.put("GetTables", new GetTables());
-      processMap.put("GetTableTypes", new GetTableTypes());
-      processMap.put("GetColumns", new GetColumns());
-      processMap.put("GetFunctions", new GetFunctions());
-      processMap.put("GetPrimaryKeys", new GetPrimaryKeys());
-      processMap.put("GetCrossReference", new GetCrossReference());
-      processMap.put("GetOperationStatus", new GetOperationStatus());
-      processMap.put("CancelOperation", new CancelOperation());
-      processMap.put("CloseOperation", new CloseOperation());
-      processMap.put("GetResultSetMetadata", new GetResultSetMetadata());
-      processMap.put("FetchResults", new FetchResults());
-      processMap.put("GetDelegationToken", new GetDelegationToken());
-      processMap.put("CancelDelegationToken", new CancelDelegationToken());
-      processMap.put("RenewDelegationToken", new RenewDelegationToken());
-      return processMap;
-    }
-
-    public static class OpenSession<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, OpenSession_args, TOpenSessionResp> {
-      public OpenSession() {
-        super("OpenSession");
-      }
-
-      public OpenSession_args getEmptyArgsInstance() {
-        return new OpenSession_args();
-      }
-
-      public AsyncMethodCallback<TOpenSessionResp> getResultHandler(final AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) {
-        final org.apache.thrift.AsyncProcessFunction fcall = this;
-        return new AsyncMethodCallback<TOpenSessionResp>() { 
-          public void onComplete(TOpenSessionResp o) {
-            OpenSession_result result = new OpenSession_result();
-            result.success = o;
-            try {
-              fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
-              return;
-            } catch (Exception e) {
-              LOGGER.error("Exception writing to internal frame buffer", e);
-            }
-            fb.close();
-          }
-          public void onError(Exception e) {
-            byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
-            org.apache.thrift.TBase msg;
-            OpenSession_result result = new OpenSession_result();
-            {
-              msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
-              msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
-            }
-            try {
-              fcall.sendResponse(fb,msg,msgType,seqid);
-              return;
-            } catch (Exception ex) {
-              LOGGER.error("Exception writing to internal frame buffer", ex);
-            }
-            fb.close();
-          }
-        };
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public void start(I iface, OpenSession_args args, org.apache.thrift.async.AsyncMethodCallback<TOpenSessionResp> resultHandler) throws TException {
-        iface.OpenSession(args.req,resultHandler);
-      }
-    }
-
-    public static class CloseSession<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, CloseSession_args, TCloseSessionResp> {
-      public CloseSession() {
-        super("CloseSession");
-      }
-
-      public CloseSession_args getEmptyArgsInstance() {
-        return new CloseSession_args();
-      }
-
-      public AsyncMethodCallback<TCloseSessionResp> getResultHandler(final AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) {
-        final org.apache.thrift.AsyncProcessFunction fcall = this;
-        return new AsyncMethodCallback<TCloseSessionResp>() { 
-          public void onComplete(TCloseSessionResp o) {
-            CloseSession_result result = new CloseSession_result();
-            result.success = o;
-            try {
-              fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
-              return;
-            } catch (Exception e) {
-              LOGGER.error("Exception writing to internal frame buffer", e);
-            }
-            fb.close();
-          }
-          public void onError(Exception e) {
-            byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
-            org.apache.thrift.TBase msg;
-            CloseSession_result result = new CloseSession_result();
-            {
-              msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
-              msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
-            }
-            try {
-              fcall.sendResponse(fb,msg,msgType,seqid);
-              return;
-            } catch (Exception ex) {
-              LOGGER.error("Exception writing to internal frame buffer", ex);
-            }
-            fb.close();
-          }
-        };
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public void start(I iface, CloseSession_args args, org.apache.thrift.async.AsyncMethodCallback<TCloseSessionResp> resultHandler) throws TException {
-        iface.CloseSession(args.req,resultHandler);
-      }
-    }
-
-    public static class GetInfo<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, GetInfo_args, TGetInfoResp> {
-      public GetInfo() {
-        super("GetInfo");
-      }
-
-      public GetInfo_args getEmptyArgsInstance() {
-        return new GetInfo_args();
-      }
-
-      public AsyncMethodCallback<TGetInfoResp> getResultHandler(final AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) {
-        final org.apache.thrift.AsyncProcessFunction fcall = this;
-        return new AsyncMethodCallback<TGetInfoResp>() { 
-          public void onComplete(TGetInfoResp o) {
-            GetInfo_result result = new GetInfo_result();
-            result.success = o;
-            try {
-              fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
-              return;
-            } catch (Exception e) {
-              LOGGER.error("Exception writing to internal frame buffer", e);
-            }
-            fb.close();
-          }
-          public void onError(Exception e) {
-            byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
-            org.apache.thrift.TBase msg;
-            GetInfo_result result = new GetInfo_result();
-            {
-              msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
-              msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
-            }
-            try {
-              fcall.sendResponse(fb,msg,msgType,seqid);
-              return;
-            } catch (Exception ex) {
-              LOGGER.error("Exception writing to internal frame buffer", ex);
-            }
-            fb.close();
-          }
-        };
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public void start(I iface, GetInfo_args args, org.apache.thrift.async.AsyncMethodCallback<TGetInfoResp> resultHandler) throws TException {
-        iface.GetInfo(args.req,resultHandler);
-      }
-    }
-
-    public static class ExecuteStatement<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, ExecuteStatement_args, TExecuteStatementResp> {
-      public ExecuteStatement() {
-        super("ExecuteStatement");
-      }
-
-      public ExecuteStatement_args getEmptyArgsInstance() {
-        return new ExecuteStatement_args();
-      }
-
-      public AsyncMethodCallback<TExecuteStatementResp> getResultHandler(final AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) {
-        final org.apache.thrift.AsyncProcessFunction fcall = this;
-        return new AsyncMethodCallback<TExecuteStatementResp>() { 
-          public void onComplete(TExecuteStatementResp o) {
-            ExecuteStatement_result result = new ExecuteStatement_result();
-            result.success = o;
-            try {
-              fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
-              return;
-            } catch (Exception e) {
-              LOGGER.error("Exception writing to internal frame buffer", e);
-            }
-            fb.close();
-          }
-          public void onError(Exception e) {
-            byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
-            org.apache.thrift.TBase msg;
-            ExecuteStatement_result result = new ExecuteStatement_result();
-            {
-              msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
-              msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
-            }
-            try {
-              fcall.sendResponse(fb,msg,msgType,seqid);
-              return;
-            } catch (Exception ex) {
-              LOGGER.error("Exception writing to internal frame buffer", ex);
-            }
-            fb.close();
-          }
-        };
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public void start(I iface, ExecuteStatement_args args, org.apache.thrift.async.AsyncMethodCallback<TExecuteStatementResp> resultHandler) throws TException {
-        iface.ExecuteStatement(args.req,resultHandler);
-      }
-    }
-
-    public static class GetTypeInfo<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, GetTypeInfo_args, TGetTypeInfoResp> {
-      public GetTypeInfo() {
-        super("GetTypeInfo");
-      }
-
-      public GetTypeInfo_args getEmptyArgsInstance() {
-        return new GetTypeInfo_args();
-      }
-
-      public AsyncMethodCallback<TGetTypeInfoResp> getResultHandler(final AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) {
-        final org.apache.thrift.AsyncProcessFunction fcall = this;
-        return new AsyncMethodCallback<TGetTypeInfoResp>() { 
-          public void onComplete(TGetTypeInfoResp o) {
-            GetTypeInfo_result result = new GetTypeInfo_result();
-            result.success = o;
-            try {
-              fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
-              return;
-            } catch (Exception e) {
-              LOGGER.error("Exception writing to internal frame buffer", e);
-            }
-            fb.close();
-          }
-          public void onError(Exception e) {
-            byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
-            org.apache.thrift.TBase msg;
-            GetTypeInfo_result result = new GetTypeInfo_result();
-            {
-              msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
-              msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
-            }
-            try {
-              fcall.sendResponse(fb,msg,msgType,seqid);
-              return;
-            } catch (Exception ex) {
-              LOGGER.error("Exception writing to internal frame buffer", ex);
-            }
-            fb.close();
-          }
-        };
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public void start(I iface, GetTypeInfo_args args, org.apache.thrift.async.AsyncMethodCallback<TGetTypeInfoResp> resultHandler) throws TException {
-        iface.GetTypeInfo(args.req,resultHandler);
-      }
-    }
-
-    public static class GetCatalogs<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, GetCatalogs_args, TGetCatalogsResp> {
-      public GetCatalogs() {
-        super("GetCatalogs");
-      }
-
-      public GetCatalogs_args getEmptyArgsInstance() {
-        return new GetCatalogs_args();
-      }
-
-      public AsyncMethodCallback<TGetCatalogsResp> getResultHandler(final AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) {
-        final org.apache.thrift.AsyncProcessFunction fcall = this;
-        return new AsyncMethodCallback<TGetCatalogsResp>() { 
-          public void onComplete(TGetCatalogsResp o) {
-            GetCatalogs_result result = new GetCatalogs_result();
-            result.success = o;
-            try {
-              fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
-              return;
-            } catch (Exception e) {
-              LOGGER.error("Exception writing to internal frame buffer", e);
-            }
-            fb.close();
-          }
-          public void onError(Exception e) {
-            byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
-            org.apache.thrift.TBase msg;
-            GetCatalogs_result result = new GetCatalogs_result();
-            {
-              msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
-              msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
-            }
-            try {
-              fcall.sendResponse(fb,msg,msgType,seqid);
-              return;
-            } catch (Exception ex) {
-              LOGGER.error("Exception writing to internal frame buffer", ex);
-            }
-            fb.close();
-          }
-        };
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public void start(I iface, GetCatalogs_args args, org.apache.thrift.async.AsyncMethodCallback<TGetCatalogsResp> resultHandler) throws TException {
-        iface.GetCatalogs(args.req,resultHandler);
-      }
-    }
-
-    public static class GetSchemas<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, GetSchemas_args, TGetSchemasResp> {
-      public GetSchemas() {
-        super("GetSchemas");
-      }
-
-      public GetSchemas_args getEmptyArgsInstance() {
-        return new GetSchemas_args();
-      }
-
-      public AsyncMethodCallback<TGetSchemasResp> getResultHandler(final AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) {
-        final org.apache.thrift.AsyncProcessFunction fcall = this;
-        return new AsyncMethodCallback<TGetSchemasResp>() { 
-          public void onComplete(TGetSchemasResp o) {
-            GetSchemas_result result = new GetSchemas_result();
-            result.success = o;
-            try {
-              fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
-              return;
-            } catch (Exception e) {
-              LOGGER.error("Exception writing to internal frame buffer", e);
-            }
-            fb.close();
-          }
-          public void onError(Exception e) {
-            byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
-            org.apache.thrift.TBase msg;
-            GetSchemas_result result = new GetSchemas_result();
-            {
-              msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
-              msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
-            }
-            try {
-              fcall.sendResponse(fb,msg,msgType,seqid);
-              return;
-            } catch (Exception ex) {
-              LOGGER.error("Exception writing to internal frame buffer", ex);
-            }
-            fb.close();
-          }
-        };
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public void start(I iface, GetSchemas_args args, org.apache.thrift.async.AsyncMethodCallback<TGetSchemasResp> resultHandler) throws TException {
-        iface.GetSchemas(args.req,resultHandler);
-      }
-    }
-
-    public static class GetTables<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, GetTables_args, TGetTablesResp> {
-      public GetTables() {
-        super("GetTables");
-      }
-
-      public GetTables_args getEmptyArgsInstance() {
-        return new GetTables_args();
-      }
-
-      public AsyncMethodCallback<TGetTablesResp> getResultHandler(final AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) {
-        final org.apache.thrift.AsyncProcessFunction fcall = this;
-        return new AsyncMethodCallback<TGetTablesResp>() { 
-          public void onComplete(TGetTablesResp o) {
-            GetTables_result result = new GetTables_result();
-            result.success = o;
-            try {
-              fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
-              return;
-            } catch (Exception e) {
-              LOGGER.error("Exception writing to internal frame buffer", e);
-            }
-            fb.close();
-          }
-          public void onError(Exception e) {
-            byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
-            org.apache.thrift.TBase msg;
-            GetTables_result result = new GetTables_result();
-            {
-              msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
-              msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
-            }
-            try {
-              fcall.sendResponse(fb,msg,msgType,seqid);
-              return;
-            } catch (Exception ex) {
-              LOGGER.error("Exception writing to internal frame buffer", ex);
-            }
-            fb.close();
-          }
-        };
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public void start(I iface, GetTables_args args, org.apache.thrift.async.AsyncMethodCallback<TGetTablesResp> resultHandler) throws TException {
-        iface.GetTables(args.req,resultHandler);
-      }
-    }
-
-    public static class GetTableTypes<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, GetTableTypes_args, TGetTableTypesResp> {
-      public GetTableTypes() {
-        super("GetTableTypes");
-      }
-
-      public GetTableTypes_args getEmptyArgsInstance() {
-        return new GetTableTypes_args();
-      }
-
-      public AsyncMethodCallback<TGetTableTypesResp> getResultHandler(final AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) {
-        final org.apache.thrift.AsyncProcessFunction fcall = this;
-        return new AsyncMethodCallback<TGetTableTypesResp>() { 
-          public void onComplete(TGetTableTypesResp o) {
-            GetTableTypes_result result = new GetTableTypes_result();
-            result.success = o;
-            try {
-              fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
-              return;
-            } catch (Exception e) {
-              LOGGER.error("Exception writing to internal frame buffer", e);
-            }
-            fb.close();
-          }
-          public void onError(Exception e) {
-            byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
-            org.apache.thrift.TBase msg;
-            GetTableTypes_result result = new GetTableTypes_result();
-            {
-              msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
-              msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
-            }
-            try {
-              fcall.sendResponse(fb,msg,msgType,seqid);
-              return;
-            } catch (Exception ex) {
-              LOGGER.error("Exception writing to internal frame buffer", ex);
-            }
-            fb.close();
-          }
-        };
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public void start(I iface, GetTableTypes_args args, org.apache.thrift.async.AsyncMethodCallback<TGetTableTypesResp> resultHandler) throws TException {
-        iface.GetTableTypes(args.req,resultHandler);
-      }
-    }
-
-    public static class GetColumns<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, GetColumns_args, TGetColumnsResp> {
-      public GetColumns() {
-        super("GetColumns");
-      }
-
-      public GetColumns_args getEmptyArgsInstance() {
-        return new GetColumns_args();
-      }
-
-      public AsyncMethodCallback<TGetColumnsResp> getResultHandler(final AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) {
-        final org.apache.thrift.AsyncProcessFunction fcall = this;
-        return new AsyncMethodCallback<TGetColumnsResp>() { 
-          public void onComplete(TGetColumnsResp o) {
-            GetColumns_result result = new GetColumns_result();
-            result.success = o;
-            try {
-              fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
-              return;
-            } catch (Exception e) {
-              LOGGER.error("Exception writing to internal frame buffer", e);
-            }
-            fb.close();
-          }
-          public void onError(Exception e) {
-            byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
-            org.apache.thrift.TBase msg;
-            GetColumns_result result = new GetColumns_result();
-            {
-              msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
-              msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
-            }
-            try {
-              fcall.sendResponse(fb,msg,msgType,seqid);
-              return;
-            } catch (Exception ex) {
-              LOGGER.error("Exception writing to internal frame buffer", ex);
-            }
-            fb.close();
-          }
-        };
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public void start(I iface, GetColumns_args args, org.apache.thrift.async.AsyncMethodCallback<TGetColumnsResp> resultHandler) throws TException {
-        iface.GetColumns(args.req,resultHandler);
-      }
-    }
-
-    public static class GetFunctions<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, GetFunctions_args, TGetFunctionsResp> {
-      public GetFunctions() {
-        super("GetFunctions");
-      }
-
-      public GetFunctions_args getEmptyArgsInstance() {
-        return new GetFunctions_args();
-      }
-
-      public AsyncMethodCallback<TGetFunctionsResp> getResultHandler(final AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) {
-        final org.apache.thrift.AsyncProcessFunction fcall = this;
-        return new AsyncMethodCallback<TGetFunctionsResp>() { 
-          public void onComplete(TGetFunctionsResp o) {
-            GetFunctions_result result = new GetFunctions_result();
-            result.success = o;
-            try {
-              fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
-              return;
-            } catch (Exception e) {
-              LOGGER.error("Exception writing to internal frame buffer", e);
-            }
-            fb.close();
-          }
-          public void onError(Exception e) {
-            byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
-            org.apache.thrift.TBase msg;
-            GetFunctions_result result = new GetFunctions_result();
-            {
-              msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
-              msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
-            }
-            try {
-              fcall.sendResponse(fb,msg,msgType,seqid);
-              return;
-            } catch (Exception ex) {
-              LOGGER.error("Exception writing to internal frame buffer", ex);
-            }
-            fb.close();
-          }
-        };
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public void start(I iface, GetFunctions_args args, org.apache.thrift.async.AsyncMethodCallback<TGetFunctionsResp> resultHandler) throws TException {
-        iface.GetFunctions(args.req,resultHandler);
-      }
-    }
-
-    public static class GetPrimaryKeys<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, GetPrimaryKeys_args, TGetPrimaryKeysResp> {
-      public GetPrimaryKeys() {
-        super("GetPrimaryKeys");
-      }
-
-      public GetPrimaryKeys_args getEmptyArgsInstance() {
-        return new GetPrimaryKeys_args();
-      }
-
-      public AsyncMethodCallback<TGetPrimaryKeysResp> getResultHandler(final AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) {
-        final org.apache.thrift.AsyncProcessFunction fcall = this;
-        return new AsyncMethodCallback<TGetPrimaryKeysResp>() { 
-          public void onComplete(TGetPrimaryKeysResp o) {
-            GetPrimaryKeys_result result = new GetPrimaryKeys_result();
-            result.success = o;
-            try {
-              fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
-              return;
-            } catch (Exception e) {
-              LOGGER.error("Exception writing to internal frame buffer", e);
-            }
-            fb.close();
-          }
-          public void onError(Exception e) {
-            byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
-            org.apache.thrift.TBase msg;
-            GetPrimaryKeys_result result = new GetPrimaryKeys_result();
-            {
-              msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
-              msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
-            }
-            try {
-              fcall.sendResponse(fb,msg,msgType,seqid);
-              return;
-            } catch (Exception ex) {
-              LOGGER.error("Exception writing to internal frame buffer", ex);
-            }
-            fb.close();
-          }
-        };
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public void start(I iface, GetPrimaryKeys_args args, org.apache.thrift.async.AsyncMethodCallback<TGetPrimaryKeysResp> resultHandler) throws TException {
-        iface.GetPrimaryKeys(args.req,resultHandler);
-      }
-    }
-
-    public static class GetCrossReference<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, GetCrossReference_args, TGetCrossReferenceResp> {
-      public GetCrossReference() {
-        super("GetCrossReference");
-      }
-
-      public GetCrossReference_args getEmptyArgsInstance() {
-        return new GetCrossReference_args();
-      }
-
-      public AsyncMethodCallback<TGetCrossReferenceResp> getResultHandler(final AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) {
-        final org.apache.thrift.AsyncProcessFunction fcall = this;
-        return new AsyncMethodCallback<TGetCrossReferenceResp>() { 
-          public void onComplete(TGetCrossReferenceResp o) {
-            GetCrossReference_result result = new GetCrossReference_result();
-            result.success = o;
-            try {
-              fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
-              return;
-            } catch (Exception e) {
-              LOGGER.error("Exception writing to internal frame buffer", e);
-            }
-            fb.close();
-          }
-          public void onError(Exception e) {
-            byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
-            org.apache.thrift.TBase msg;
-            GetCrossReference_result result = new GetCrossReference_result();
-            {
-              msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
-              msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
-            }
-            try {
-              fcall.sendResponse(fb,msg,msgType,seqid);
-              return;
-            } catch (Exception ex) {
-              LOGGER.error("Exception writing to internal frame buffer", ex);
-            }
-            fb.close();
-          }
-        };
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public void start(I iface, GetCrossReference_args args, org.apache.thrift.async.AsyncMethodCallback<TGetCrossReferenceResp> resultHandler) throws TException {
-        iface.GetCrossReference(args.req,resultHandler);
-      }
-    }
-
-    public static class GetOperationStatus<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, GetOperationStatus_args, TGetOperationStatusResp> {
-      public GetOperationStatus() {
-        super("GetOperationStatus");
-      }
-
-      public GetOperationStatus_args getEmptyArgsInstance() {
-        return new GetOperationStatus_args();
-      }
-
-      public AsyncMethodCallback<TGetOperationStatusResp> getResultHandler(final AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) {
-        final org.apache.thrift.AsyncProcessFunction fcall = this;
-        return new AsyncMethodCallback<TGetOperationStatusResp>() { 
-          public void onComplete(TGetOperationStatusResp o) {
-            GetOperationStatus_result result = new GetOperationStatus_result();
-            result.success = o;
-            try {
-              fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
-              return;
-            } catch (Exception e) {
-              LOGGER.error("Exception writing to internal frame buffer", e);
-            }
-            fb.close();
-          }
-          public void onError(Exception e) {
-            byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
-            org.apache.thrift.TBase msg;
-            GetOperationStatus_result result = new GetOperationStatus_result();
-            {
-              msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
-              msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
-            }
-            try {
-              fcall.sendResponse(fb,msg,msgType,seqid);
-              return;
-            } catch (Exception ex) {
-              LOGGER.error("Exception writing to internal frame buffer", ex);
-            }
-            fb.close();
-          }
-        };
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public void start(I iface, GetOperationStatus_args args, org.apache.thrift.async.AsyncMethodCallback<TGetOperationStatusResp> resultHandler) throws TException {
-        iface.GetOperationStatus(args.req,resultHandler);
-      }
-    }
-
-    public static class CancelOperation<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, CancelOperation_args, TCancelOperationResp> {
-      public CancelOperation() {
-        super("CancelOperation");
-      }
-
-      public CancelOperation_args getEmptyArgsInstance() {
-        return new CancelOperation_args();
-      }
-
-      public AsyncMethodCallback<TCancelOperationResp> getResultHandler(final AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) {
-        final org.apache.thrift.AsyncProcessFunction fcall = this;
-        return new AsyncMethodCallback<TCancelOperationResp>() { 
-          public void onComplete(TCancelOperationResp o) {
-            CancelOperation_result result = new CancelOperation_result();
-            result.success = o;
-            try {
-              fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
-              return;
-            } catch (Exception e) {
-              LOGGER.error("Exception writing to internal frame buffer", e);
-            }
-            fb.close();
-          }
-          public void onError(Exception e) {
-            byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
-            org.apache.thrift.TBase msg;
-            CancelOperation_result result = new CancelOperation_result();
-            {
-              msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
-              msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
-            }
-            try {
-              fcall.sendResponse(fb,msg,msgType,seqid);
-              return;
-            } catch (Exception ex) {
-              LOGGER.error("Exception writing to internal frame buffer", ex);
-            }
-            fb.close();
-          }
-        };
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public void start(I iface, CancelOperation_args args, org.apache.thrift.async.AsyncMethodCallback<TCancelOperationResp> resultHandler) throws TException {
-        iface.CancelOperation(args.req,resultHandler);
-      }
-    }
-
-    public static class CloseOperation<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, CloseOperation_args, TCloseOperationResp> {
-      public CloseOperation() {
-        super("CloseOperation");
-      }
-
-      public CloseOperation_args getEmptyArgsInstance() {
-        return new CloseOperation_args();
-      }
-
-      public AsyncMethodCallback<TCloseOperationResp> getResultHandler(final AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) {
-        final org.apache.thrift.AsyncProcessFunction fcall = this;
-        return new AsyncMethodCallback<TCloseOperationResp>() { 
-          public void onComplete(TCloseOperationResp o) {
-            CloseOperation_result result = new CloseOperation_result();
-            result.success = o;
-            try {
-              fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
-              return;
-            } catch (Exception e) {
-              LOGGER.error("Exception writing to internal frame buffer", e);
-            }
-            fb.close();
-          }
-          public void onError(Exception e) {
-            byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
-            org.apache.thrift.TBase msg;
-            CloseOperation_result result = new CloseOperation_result();
-            {
-              msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
-              msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
-            }
-            try {
-              fcall.sendResponse(fb,msg,msgType,seqid);
-              return;
-            } catch (Exception ex) {
-              LOGGER.error("Exception writing to internal frame buffer", ex);
-            }
-            fb.close();
-          }
-        };
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public void start(I iface, CloseOperation_args args, org.apache.thrift.async.AsyncMethodCallback<TCloseOperationResp> resultHandler) throws TException {
-        iface.CloseOperation(args.req,resultHandler);
-      }
-    }
-
-    public static class GetResultSetMetadata<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, GetResultSetMetadata_args, TGetResultSetMetadataResp> {
-      public GetResultSetMetadata() {
-        super("GetResultSetMetadata");
-      }
-
-      public GetResultSetMetadata_args getEmptyArgsInstance() {
-        return new GetResultSetMetadata_args();
-      }
-
-      public AsyncMethodCallback<TGetResultSetMetadataResp> getResultHandler(final AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) {
-        final org.apache.thrift.AsyncProcessFunction fcall = this;
-        return new AsyncMethodCallback<TGetResultSetMetadataResp>() { 
-          public void onComplete(TGetResultSetMetadataResp o) {
-            GetResultSetMetadata_result result = new GetResultSetMetadata_result();
-            result.success = o;
-            try {
-              fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
-              return;
-            } catch (Exception e) {
-              LOGGER.error("Exception writing to internal frame buffer", e);
-            }
-            fb.close();
-          }
-          public void onError(Exception e) {
-            byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
-            org.apache.thrift.TBase msg;
-            GetResultSetMetadata_result result = new GetResultSetMetadata_result();
-            {
-              msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
-              msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
-            }
-            try {
-              fcall.sendResponse(fb,msg,msgType,seqid);
-              return;
-            } catch (Exception ex) {
-              LOGGER.error("Exception writing to internal frame buffer", ex);
-            }
-            fb.close();
-          }
-        };
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public void start(I iface, GetResultSetMetadata_args args, org.apache.thrift.async.AsyncMethodCallback<TGetResultSetMetadataResp> resultHandler) throws TException {
-        iface.GetResultSetMetadata(args.req,resultHandler);
-      }
-    }
-
-    public static class FetchResults<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, FetchResults_args, TFetchResultsResp> {
-      public FetchResults() {
-        super("FetchResults");
-      }
-
-      public FetchResults_args getEmptyArgsInstance() {
-        return new FetchResults_args();
-      }
-
-      public AsyncMethodCallback<TFetchResultsResp> getResultHandler(final AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) {
-        final org.apache.thrift.AsyncProcessFunction fcall = this;
-        return new AsyncMethodCallback<TFetchResultsResp>() { 
-          public void onComplete(TFetchResultsResp o) {
-            FetchResults_result result = new FetchResults_result();
-            result.success = o;
-            try {
-              fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
-              return;
-            } catch (Exception e) {
-              LOGGER.error("Exception writing to internal frame buffer", e);
-            }
-            fb.close();
-          }
-          public void onError(Exception e) {
-            byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
-            org.apache.thrift.TBase msg;
-            FetchResults_result result = new FetchResults_result();
-            {
-              msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
-              msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
-            }
-            try {
-              fcall.sendResponse(fb,msg,msgType,seqid);
-              return;
-            } catch (Exception ex) {
-              LOGGER.error("Exception writing to internal frame buffer", ex);
-            }
-            fb.close();
-          }
-        };
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public void start(I iface, FetchResults_args args, org.apache.thrift.async.AsyncMethodCallback<TFetchResultsResp> resultHandler) throws TException {
-        iface.FetchResults(args.req,resultHandler);
-      }
-    }
-
-    public static class GetDelegationToken<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, GetDelegationToken_args, TGetDelegationTokenResp> {
-      public GetDelegationToken() {
-        super("GetDelegationToken");
-      }
-
-      public GetDelegationToken_args getEmptyArgsInstance() {
-        return new GetDelegationToken_args();
-      }
-
-      public AsyncMethodCallback<TGetDelegationTokenResp> getResultHandler(final AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) {
-        final org.apache.thrift.AsyncProcessFunction fcall = this;
-        return new AsyncMethodCallback<TGetDelegationTokenResp>() { 
-          public void onComplete(TGetDelegationTokenResp o) {
-            GetDelegationToken_result result = new GetDelegationToken_result();
-            result.success = o;
-            try {
-              fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
-              return;
-            } catch (Exception e) {
-              LOGGER.error("Exception writing to internal frame buffer", e);
-            }
-            fb.close();
-          }
-          public void onError(Exception e) {
-            byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
-            org.apache.thrift.TBase msg;
-            GetDelegationToken_result result = new GetDelegationToken_result();
-            {
-              msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
-              msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
-            }
-            try {
-              fcall.sendResponse(fb,msg,msgType,seqid);
-              return;
-            } catch (Exception ex) {
-              LOGGER.error("Exception writing to internal frame buffer", ex);
-            }
-            fb.close();
-          }
-        };
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public void start(I iface, GetDelegationToken_args args, org.apache.thrift.async.AsyncMethodCallback<TGetDelegationTokenResp> resultHandler) throws TException {
-        iface.GetDelegationToken(args.req,resultHandler);
-      }
-    }
-
-    public static class CancelDelegationToken<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, CancelDelegationToken_args, TCancelDelegationTokenResp> {
-      public CancelDelegationToken() {
-        super("CancelDelegationToken");
-      }
-
-      public CancelDelegationToken_args getEmptyArgsInstance() {
-        return new CancelDelegationToken_args();
-      }
-
-      public AsyncMethodCallback<TCancelDelegationTokenResp> getResultHandler(final AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) {
-        final org.apache.thrift.AsyncProcessFunction fcall = this;
-        return new AsyncMethodCallback<TCancelDelegationTokenResp>() { 
-          public void onComplete(TCancelDelegationTokenResp o) {
-            CancelDelegationToken_result result = new CancelDelegationToken_result();
-            result.success = o;
-            try {
-              fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
-              return;
-            } catch (Exception e) {
-              LOGGER.error("Exception writing to internal frame buffer", e);
-            }
-            fb.close();
-          }
-          public void onError(Exception e) {
-            byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
-            org.apache.thrift.TBase msg;
-            CancelDelegationToken_result result = new CancelDelegationToken_result();
-            {
-              msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
-              msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
-            }
-            try {
-              fcall.sendResponse(fb,msg,msgType,seqid);
-              return;
-            } catch (Exception ex) {
-              LOGGER.error("Exception writing to internal frame buffer", ex);
-            }
-            fb.close();
-          }
-        };
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public void start(I iface, CancelDelegationToken_args args, org.apache.thrift.async.AsyncMethodCallback<TCancelDelegationTokenResp> resultHandler) throws TException {
-        iface.CancelDelegationToken(args.req,resultHandler);
-      }
-    }
-
-    public static class RenewDelegationToken<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, RenewDelegationToken_args, TRenewDelegationTokenResp> {
-      public RenewDelegationToken() {
-        super("RenewDelegationToken");
-      }
-
-      public RenewDelegationToken_args getEmptyArgsInstance() {
-        return new RenewDelegationToken_args();
-      }
-
-      public AsyncMethodCallback<TRenewDelegationTokenResp> getResultHandler(final AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) {
-        final org.apache.thrift.AsyncProcessFunction fcall = this;
-        return new AsyncMethodCallback<TRenewDelegationTokenResp>() { 
-          public void onComplete(TRenewDelegationTokenResp o) {
-            RenewDelegationToken_result result = new RenewDelegationToken_result();
-            result.success = o;
-            try {
-              fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
-              return;
-            } catch (Exception e) {
-              LOGGER.error("Exception writing to internal frame buffer", e);
-            }
-            fb.close();
-          }
-          public void onError(Exception e) {
-            byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
-            org.apache.thrift.TBase msg;
-            RenewDelegationToken_result result = new RenewDelegationToken_result();
-            {
-              msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
-              msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
-            }
-            try {
-              fcall.sendResponse(fb,msg,msgType,seqid);
-              return;
-            } catch (Exception ex) {
-              LOGGER.error("Exception writing to internal frame buffer", ex);
-            }
-            fb.close();
-          }
-        };
-      }
-
-      protected boolean isOneway() {
-        return false;
-      }
-
-      public void start(I iface, RenewDelegationToken_args args, org.apache.thrift.async.AsyncMethodCallback<TRenewDelegationTokenResp> resultHandler) throws TException {
-        iface.RenewDelegationToken(args.req,resultHandler);
-      }
-    }
-
-  }
-
-  public static class OpenSession_args implements org.apache.thrift.TBase<OpenSession_args, OpenSession_args._Fields>, java.io.Serializable, Cloneable, Comparable<OpenSession_args>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("OpenSession_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new OpenSession_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new OpenSession_argsTupleSchemeFactory());
-    }
-
-    private TOpenSessionReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOpenSessionReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(OpenSession_args.class, metaDataMap);
-    }
-
-    public OpenSession_args() {
-    }
-
-    public OpenSession_args(
-      TOpenSessionReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public OpenSession_args(OpenSession_args other) {
-      if (other.isSetReq()) {
-        this.req = new TOpenSessionReq(other.req);
-      }
-    }
-
-    public OpenSession_args deepCopy() {
-      return new OpenSession_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TOpenSessionReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TOpenSessionReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TOpenSessionReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof OpenSession_args)
-        return this.equals((OpenSession_args)that);
-      return false;
-    }
-
-    public boolean equals(OpenSession_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_req = true && (isSetReq());
-      list.add(present_req);
-      if (present_req)
-        list.add(req);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(OpenSession_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(other.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, other.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("OpenSession_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class OpenSession_argsStandardSchemeFactory implements SchemeFactory {
-      public OpenSession_argsStandardScheme getScheme() {
-        return new OpenSession_argsStandardScheme();
-      }
-    }
-
-    private static class OpenSession_argsStandardScheme extends StandardScheme<OpenSession_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, OpenSession_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TOpenSessionReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, OpenSession_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class OpenSession_argsTupleSchemeFactory implements SchemeFactory {
-      public OpenSession_argsTupleScheme getScheme() {
-        return new OpenSession_argsTupleScheme();
-      }
-    }
-
-    private static class OpenSession_argsTupleScheme extends TupleScheme<OpenSession_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, OpenSession_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, OpenSession_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TOpenSessionReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class OpenSession_result implements org.apache.thrift.TBase<OpenSession_result, OpenSession_result._Fields>, java.io.Serializable, Cloneable, Comparable<OpenSession_result>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("OpenSession_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new OpenSession_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new OpenSession_resultTupleSchemeFactory());
-    }
-
-    private TOpenSessionResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOpenSessionResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(OpenSession_result.class, metaDataMap);
-    }
-
-    public OpenSession_result() {
-    }
-
-    public OpenSession_result(
-      TOpenSessionResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public OpenSession_result(OpenSession_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TOpenSessionResp(other.success);
-      }
-    }
-
-    public OpenSession_result deepCopy() {
-      return new OpenSession_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TOpenSessionResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TOpenSessionResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TOpenSessionResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof OpenSession_result)
-        return this.equals((OpenSession_result)that);
-      return false;
-    }
-
-    public boolean equals(OpenSession_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_success = true && (isSetSuccess());
-      list.add(present_success);
-      if (present_success)
-        list.add(success);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(OpenSession_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("OpenSession_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class OpenSession_resultStandardSchemeFactory implements SchemeFactory {
-      public OpenSession_resultStandardScheme getScheme() {
-        return new OpenSession_resultStandardScheme();
-      }
-    }
-
-    private static class OpenSession_resultStandardScheme extends StandardScheme<OpenSession_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, OpenSession_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TOpenSessionResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, OpenSession_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class OpenSession_resultTupleSchemeFactory implements SchemeFactory {
-      public OpenSession_resultTupleScheme getScheme() {
-        return new OpenSession_resultTupleScheme();
-      }
-    }
-
-    private static class OpenSession_resultTupleScheme extends TupleScheme<OpenSession_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, OpenSession_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, OpenSession_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TOpenSessionResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class CloseSession_args implements org.apache.thrift.TBase<CloseSession_args, CloseSession_args._Fields>, java.io.Serializable, Cloneable, Comparable<CloseSession_args>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CloseSession_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new CloseSession_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new CloseSession_argsTupleSchemeFactory());
-    }
-
-    private TCloseSessionReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TCloseSessionReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CloseSession_args.class, metaDataMap);
-    }
-
-    public CloseSession_args() {
-    }
-
-    public CloseSession_args(
-      TCloseSessionReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public CloseSession_args(CloseSession_args other) {
-      if (other.isSetReq()) {
-        this.req = new TCloseSessionReq(other.req);
-      }
-    }
-
-    public CloseSession_args deepCopy() {
-      return new CloseSession_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TCloseSessionReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TCloseSessionReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TCloseSessionReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof CloseSession_args)
-        return this.equals((CloseSession_args)that);
-      return false;
-    }
-
-    public boolean equals(CloseSession_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_req = true && (isSetReq());
-      list.add(present_req);
-      if (present_req)
-        list.add(req);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(CloseSession_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(other.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, other.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("CloseSession_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class CloseSession_argsStandardSchemeFactory implements SchemeFactory {
-      public CloseSession_argsStandardScheme getScheme() {
-        return new CloseSession_argsStandardScheme();
-      }
-    }
-
-    private static class CloseSession_argsStandardScheme extends StandardScheme<CloseSession_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, CloseSession_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TCloseSessionReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, CloseSession_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class CloseSession_argsTupleSchemeFactory implements SchemeFactory {
-      public CloseSession_argsTupleScheme getScheme() {
-        return new CloseSession_argsTupleScheme();
-      }
-    }
-
-    private static class CloseSession_argsTupleScheme extends TupleScheme<CloseSession_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, CloseSession_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, CloseSession_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TCloseSessionReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class CloseSession_result implements org.apache.thrift.TBase<CloseSession_result, CloseSession_result._Fields>, java.io.Serializable, Cloneable, Comparable<CloseSession_result>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CloseSession_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new CloseSession_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new CloseSession_resultTupleSchemeFactory());
-    }
-
-    private TCloseSessionResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TCloseSessionResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CloseSession_result.class, metaDataMap);
-    }
-
-    public CloseSession_result() {
-    }
-
-    public CloseSession_result(
-      TCloseSessionResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public CloseSession_result(CloseSession_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TCloseSessionResp(other.success);
-      }
-    }
-
-    public CloseSession_result deepCopy() {
-      return new CloseSession_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TCloseSessionResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TCloseSessionResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TCloseSessionResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof CloseSession_result)
-        return this.equals((CloseSession_result)that);
-      return false;
-    }
-
-    public boolean equals(CloseSession_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_success = true && (isSetSuccess());
-      list.add(present_success);
-      if (present_success)
-        list.add(success);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(CloseSession_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("CloseSession_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class CloseSession_resultStandardSchemeFactory implements SchemeFactory {
-      public CloseSession_resultStandardScheme getScheme() {
-        return new CloseSession_resultStandardScheme();
-      }
-    }
-
-    private static class CloseSession_resultStandardScheme extends StandardScheme<CloseSession_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, CloseSession_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TCloseSessionResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, CloseSession_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class CloseSession_resultTupleSchemeFactory implements SchemeFactory {
-      public CloseSession_resultTupleScheme getScheme() {
-        return new CloseSession_resultTupleScheme();
-      }
-    }
-
-    private static class CloseSession_resultTupleScheme extends TupleScheme<CloseSession_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, CloseSession_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, CloseSession_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TCloseSessionResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetInfo_args implements org.apache.thrift.TBase<GetInfo_args, GetInfo_args._Fields>, java.io.Serializable, Cloneable, Comparable<GetInfo_args>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetInfo_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetInfo_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetInfo_argsTupleSchemeFactory());
-    }
-
-    private TGetInfoReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetInfoReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetInfo_args.class, metaDataMap);
-    }
-
-    public GetInfo_args() {
-    }
-
-    public GetInfo_args(
-      TGetInfoReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetInfo_args(GetInfo_args other) {
-      if (other.isSetReq()) {
-        this.req = new TGetInfoReq(other.req);
-      }
-    }
-
-    public GetInfo_args deepCopy() {
-      return new GetInfo_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TGetInfoReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TGetInfoReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TGetInfoReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetInfo_args)
-        return this.equals((GetInfo_args)that);
-      return false;
-    }
-
-    public boolean equals(GetInfo_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_req = true && (isSetReq());
-      list.add(present_req);
-      if (present_req)
-        list.add(req);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetInfo_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(other.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, other.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetInfo_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetInfo_argsStandardSchemeFactory implements SchemeFactory {
-      public GetInfo_argsStandardScheme getScheme() {
-        return new GetInfo_argsStandardScheme();
-      }
-    }
-
-    private static class GetInfo_argsStandardScheme extends StandardScheme<GetInfo_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetInfo_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TGetInfoReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetInfo_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetInfo_argsTupleSchemeFactory implements SchemeFactory {
-      public GetInfo_argsTupleScheme getScheme() {
-        return new GetInfo_argsTupleScheme();
-      }
-    }
-
-    private static class GetInfo_argsTupleScheme extends TupleScheme<GetInfo_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetInfo_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetInfo_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TGetInfoReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetInfo_result implements org.apache.thrift.TBase<GetInfo_result, GetInfo_result._Fields>, java.io.Serializable, Cloneable, Comparable<GetInfo_result>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetInfo_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetInfo_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetInfo_resultTupleSchemeFactory());
-    }
-
-    private TGetInfoResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetInfoResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetInfo_result.class, metaDataMap);
-    }
-
-    public GetInfo_result() {
-    }
-
-    public GetInfo_result(
-      TGetInfoResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetInfo_result(GetInfo_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TGetInfoResp(other.success);
-      }
-    }
-
-    public GetInfo_result deepCopy() {
-      return new GetInfo_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TGetInfoResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TGetInfoResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TGetInfoResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetInfo_result)
-        return this.equals((GetInfo_result)that);
-      return false;
-    }
-
-    public boolean equals(GetInfo_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_success = true && (isSetSuccess());
-      list.add(present_success);
-      if (present_success)
-        list.add(success);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetInfo_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetInfo_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetInfo_resultStandardSchemeFactory implements SchemeFactory {
-      public GetInfo_resultStandardScheme getScheme() {
-        return new GetInfo_resultStandardScheme();
-      }
-    }
-
-    private static class GetInfo_resultStandardScheme extends StandardScheme<GetInfo_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetInfo_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TGetInfoResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetInfo_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetInfo_resultTupleSchemeFactory implements SchemeFactory {
-      public GetInfo_resultTupleScheme getScheme() {
-        return new GetInfo_resultTupleScheme();
-      }
-    }
-
-    private static class GetInfo_resultTupleScheme extends TupleScheme<GetInfo_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetInfo_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetInfo_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TGetInfoResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class ExecuteStatement_args implements org.apache.thrift.TBase<ExecuteStatement_args, ExecuteStatement_args._Fields>, java.io.Serializable, Cloneable, Comparable<ExecuteStatement_args>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("ExecuteStatement_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new ExecuteStatement_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new ExecuteStatement_argsTupleSchemeFactory());
-    }
-
-    private TExecuteStatementReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TExecuteStatementReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(ExecuteStatement_args.class, metaDataMap);
-    }
-
-    public ExecuteStatement_args() {
-    }
-
-    public ExecuteStatement_args(
-      TExecuteStatementReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public ExecuteStatement_args(ExecuteStatement_args other) {
-      if (other.isSetReq()) {
-        this.req = new TExecuteStatementReq(other.req);
-      }
-    }
-
-    public ExecuteStatement_args deepCopy() {
-      return new ExecuteStatement_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TExecuteStatementReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TExecuteStatementReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TExecuteStatementReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof ExecuteStatement_args)
-        return this.equals((ExecuteStatement_args)that);
-      return false;
-    }
-
-    public boolean equals(ExecuteStatement_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_req = true && (isSetReq());
-      list.add(present_req);
-      if (present_req)
-        list.add(req);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(ExecuteStatement_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(other.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, other.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("ExecuteStatement_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class ExecuteStatement_argsStandardSchemeFactory implements SchemeFactory {
-      public ExecuteStatement_argsStandardScheme getScheme() {
-        return new ExecuteStatement_argsStandardScheme();
-      }
-    }
-
-    private static class ExecuteStatement_argsStandardScheme extends StandardScheme<ExecuteStatement_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, ExecuteStatement_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TExecuteStatementReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, ExecuteStatement_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class ExecuteStatement_argsTupleSchemeFactory implements SchemeFactory {
-      public ExecuteStatement_argsTupleScheme getScheme() {
-        return new ExecuteStatement_argsTupleScheme();
-      }
-    }
-
-    private static class ExecuteStatement_argsTupleScheme extends TupleScheme<ExecuteStatement_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, ExecuteStatement_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, ExecuteStatement_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TExecuteStatementReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class ExecuteStatement_result implements org.apache.thrift.TBase<ExecuteStatement_result, ExecuteStatement_result._Fields>, java.io.Serializable, Cloneable, Comparable<ExecuteStatement_result>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("ExecuteStatement_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new ExecuteStatement_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new ExecuteStatement_resultTupleSchemeFactory());
-    }
-
-    private TExecuteStatementResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TExecuteStatementResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(ExecuteStatement_result.class, metaDataMap);
-    }
-
-    public ExecuteStatement_result() {
-    }
-
-    public ExecuteStatement_result(
-      TExecuteStatementResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public ExecuteStatement_result(ExecuteStatement_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TExecuteStatementResp(other.success);
-      }
-    }
-
-    public ExecuteStatement_result deepCopy() {
-      return new ExecuteStatement_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TExecuteStatementResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TExecuteStatementResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TExecuteStatementResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof ExecuteStatement_result)
-        return this.equals((ExecuteStatement_result)that);
-      return false;
-    }
-
-    public boolean equals(ExecuteStatement_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_success = true && (isSetSuccess());
-      list.add(present_success);
-      if (present_success)
-        list.add(success);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(ExecuteStatement_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("ExecuteStatement_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class ExecuteStatement_resultStandardSchemeFactory implements SchemeFactory {
-      public ExecuteStatement_resultStandardScheme getScheme() {
-        return new ExecuteStatement_resultStandardScheme();
-      }
-    }
-
-    private static class ExecuteStatement_resultStandardScheme extends StandardScheme<ExecuteStatement_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, ExecuteStatement_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TExecuteStatementResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, ExecuteStatement_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class ExecuteStatement_resultTupleSchemeFactory implements SchemeFactory {
-      public ExecuteStatement_resultTupleScheme getScheme() {
-        return new ExecuteStatement_resultTupleScheme();
-      }
-    }
-
-    private static class ExecuteStatement_resultTupleScheme extends TupleScheme<ExecuteStatement_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, ExecuteStatement_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, ExecuteStatement_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TExecuteStatementResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetTypeInfo_args implements org.apache.thrift.TBase<GetTypeInfo_args, GetTypeInfo_args._Fields>, java.io.Serializable, Cloneable, Comparable<GetTypeInfo_args>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetTypeInfo_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetTypeInfo_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetTypeInfo_argsTupleSchemeFactory());
-    }
-
-    private TGetTypeInfoReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetTypeInfoReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetTypeInfo_args.class, metaDataMap);
-    }
-
-    public GetTypeInfo_args() {
-    }
-
-    public GetTypeInfo_args(
-      TGetTypeInfoReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetTypeInfo_args(GetTypeInfo_args other) {
-      if (other.isSetReq()) {
-        this.req = new TGetTypeInfoReq(other.req);
-      }
-    }
-
-    public GetTypeInfo_args deepCopy() {
-      return new GetTypeInfo_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TGetTypeInfoReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TGetTypeInfoReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TGetTypeInfoReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetTypeInfo_args)
-        return this.equals((GetTypeInfo_args)that);
-      return false;
-    }
-
-    public boolean equals(GetTypeInfo_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_req = true && (isSetReq());
-      list.add(present_req);
-      if (present_req)
-        list.add(req);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetTypeInfo_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(other.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, other.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetTypeInfo_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetTypeInfo_argsStandardSchemeFactory implements SchemeFactory {
-      public GetTypeInfo_argsStandardScheme getScheme() {
-        return new GetTypeInfo_argsStandardScheme();
-      }
-    }
-
-    private static class GetTypeInfo_argsStandardScheme extends StandardScheme<GetTypeInfo_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetTypeInfo_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TGetTypeInfoReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetTypeInfo_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetTypeInfo_argsTupleSchemeFactory implements SchemeFactory {
-      public GetTypeInfo_argsTupleScheme getScheme() {
-        return new GetTypeInfo_argsTupleScheme();
-      }
-    }
-
-    private static class GetTypeInfo_argsTupleScheme extends TupleScheme<GetTypeInfo_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetTypeInfo_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetTypeInfo_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TGetTypeInfoReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetTypeInfo_result implements org.apache.thrift.TBase<GetTypeInfo_result, GetTypeInfo_result._Fields>, java.io.Serializable, Cloneable, Comparable<GetTypeInfo_result>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetTypeInfo_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetTypeInfo_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetTypeInfo_resultTupleSchemeFactory());
-    }
-
-    private TGetTypeInfoResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetTypeInfoResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetTypeInfo_result.class, metaDataMap);
-    }
-
-    public GetTypeInfo_result() {
-    }
-
-    public GetTypeInfo_result(
-      TGetTypeInfoResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetTypeInfo_result(GetTypeInfo_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TGetTypeInfoResp(other.success);
-      }
-    }
-
-    public GetTypeInfo_result deepCopy() {
-      return new GetTypeInfo_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TGetTypeInfoResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TGetTypeInfoResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TGetTypeInfoResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetTypeInfo_result)
-        return this.equals((GetTypeInfo_result)that);
-      return false;
-    }
-
-    public boolean equals(GetTypeInfo_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_success = true && (isSetSuccess());
-      list.add(present_success);
-      if (present_success)
-        list.add(success);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetTypeInfo_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetTypeInfo_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetTypeInfo_resultStandardSchemeFactory implements SchemeFactory {
-      public GetTypeInfo_resultStandardScheme getScheme() {
-        return new GetTypeInfo_resultStandardScheme();
-      }
-    }
-
-    private static class GetTypeInfo_resultStandardScheme extends StandardScheme<GetTypeInfo_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetTypeInfo_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TGetTypeInfoResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetTypeInfo_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetTypeInfo_resultTupleSchemeFactory implements SchemeFactory {
-      public GetTypeInfo_resultTupleScheme getScheme() {
-        return new GetTypeInfo_resultTupleScheme();
-      }
-    }
-
-    private static class GetTypeInfo_resultTupleScheme extends TupleScheme<GetTypeInfo_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetTypeInfo_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetTypeInfo_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TGetTypeInfoResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetCatalogs_args implements org.apache.thrift.TBase<GetCatalogs_args, GetCatalogs_args._Fields>, java.io.Serializable, Cloneable, Comparable<GetCatalogs_args>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetCatalogs_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetCatalogs_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetCatalogs_argsTupleSchemeFactory());
-    }
-
-    private TGetCatalogsReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetCatalogsReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetCatalogs_args.class, metaDataMap);
-    }
-
-    public GetCatalogs_args() {
-    }
-
-    public GetCatalogs_args(
-      TGetCatalogsReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetCatalogs_args(GetCatalogs_args other) {
-      if (other.isSetReq()) {
-        this.req = new TGetCatalogsReq(other.req);
-      }
-    }
-
-    public GetCatalogs_args deepCopy() {
-      return new GetCatalogs_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TGetCatalogsReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TGetCatalogsReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TGetCatalogsReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetCatalogs_args)
-        return this.equals((GetCatalogs_args)that);
-      return false;
-    }
-
-    public boolean equals(GetCatalogs_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_req = true && (isSetReq());
-      list.add(present_req);
-      if (present_req)
-        list.add(req);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetCatalogs_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(other.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, other.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetCatalogs_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetCatalogs_argsStandardSchemeFactory implements SchemeFactory {
-      public GetCatalogs_argsStandardScheme getScheme() {
-        return new GetCatalogs_argsStandardScheme();
-      }
-    }
-
-    private static class GetCatalogs_argsStandardScheme extends StandardScheme<GetCatalogs_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetCatalogs_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TGetCatalogsReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetCatalogs_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetCatalogs_argsTupleSchemeFactory implements SchemeFactory {
-      public GetCatalogs_argsTupleScheme getScheme() {
-        return new GetCatalogs_argsTupleScheme();
-      }
-    }
-
-    private static class GetCatalogs_argsTupleScheme extends TupleScheme<GetCatalogs_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetCatalogs_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetCatalogs_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TGetCatalogsReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetCatalogs_result implements org.apache.thrift.TBase<GetCatalogs_result, GetCatalogs_result._Fields>, java.io.Serializable, Cloneable, Comparable<GetCatalogs_result>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetCatalogs_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetCatalogs_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetCatalogs_resultTupleSchemeFactory());
-    }
-
-    private TGetCatalogsResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetCatalogsResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetCatalogs_result.class, metaDataMap);
-    }
-
-    public GetCatalogs_result() {
-    }
-
-    public GetCatalogs_result(
-      TGetCatalogsResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetCatalogs_result(GetCatalogs_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TGetCatalogsResp(other.success);
-      }
-    }
-
-    public GetCatalogs_result deepCopy() {
-      return new GetCatalogs_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TGetCatalogsResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TGetCatalogsResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TGetCatalogsResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetCatalogs_result)
-        return this.equals((GetCatalogs_result)that);
-      return false;
-    }
-
-    public boolean equals(GetCatalogs_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_success = true && (isSetSuccess());
-      list.add(present_success);
-      if (present_success)
-        list.add(success);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetCatalogs_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetCatalogs_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetCatalogs_resultStandardSchemeFactory implements SchemeFactory {
-      public GetCatalogs_resultStandardScheme getScheme() {
-        return new GetCatalogs_resultStandardScheme();
-      }
-    }
-
-    private static class GetCatalogs_resultStandardScheme extends StandardScheme<GetCatalogs_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetCatalogs_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TGetCatalogsResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetCatalogs_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetCatalogs_resultTupleSchemeFactory implements SchemeFactory {
-      public GetCatalogs_resultTupleScheme getScheme() {
-        return new GetCatalogs_resultTupleScheme();
-      }
-    }
-
-    private static class GetCatalogs_resultTupleScheme extends TupleScheme<GetCatalogs_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetCatalogs_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetCatalogs_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TGetCatalogsResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetSchemas_args implements org.apache.thrift.TBase<GetSchemas_args, GetSchemas_args._Fields>, java.io.Serializable, Cloneable, Comparable<GetSchemas_args>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetSchemas_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetSchemas_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetSchemas_argsTupleSchemeFactory());
-    }
-
-    private TGetSchemasReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetSchemasReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetSchemas_args.class, metaDataMap);
-    }
-
-    public GetSchemas_args() {
-    }
-
-    public GetSchemas_args(
-      TGetSchemasReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetSchemas_args(GetSchemas_args other) {
-      if (other.isSetReq()) {
-        this.req = new TGetSchemasReq(other.req);
-      }
-    }
-
-    public GetSchemas_args deepCopy() {
-      return new GetSchemas_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TGetSchemasReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TGetSchemasReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TGetSchemasReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetSchemas_args)
-        return this.equals((GetSchemas_args)that);
-      return false;
-    }
-
-    public boolean equals(GetSchemas_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_req = true && (isSetReq());
-      list.add(present_req);
-      if (present_req)
-        list.add(req);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetSchemas_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(other.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, other.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetSchemas_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetSchemas_argsStandardSchemeFactory implements SchemeFactory {
-      public GetSchemas_argsStandardScheme getScheme() {
-        return new GetSchemas_argsStandardScheme();
-      }
-    }
-
-    private static class GetSchemas_argsStandardScheme extends StandardScheme<GetSchemas_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetSchemas_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TGetSchemasReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetSchemas_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetSchemas_argsTupleSchemeFactory implements SchemeFactory {
-      public GetSchemas_argsTupleScheme getScheme() {
-        return new GetSchemas_argsTupleScheme();
-      }
-    }
-
-    private static class GetSchemas_argsTupleScheme extends TupleScheme<GetSchemas_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetSchemas_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetSchemas_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TGetSchemasReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetSchemas_result implements org.apache.thrift.TBase<GetSchemas_result, GetSchemas_result._Fields>, java.io.Serializable, Cloneable, Comparable<GetSchemas_result>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetSchemas_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetSchemas_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetSchemas_resultTupleSchemeFactory());
-    }
-
-    private TGetSchemasResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetSchemasResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetSchemas_result.class, metaDataMap);
-    }
-
-    public GetSchemas_result() {
-    }
-
-    public GetSchemas_result(
-      TGetSchemasResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetSchemas_result(GetSchemas_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TGetSchemasResp(other.success);
-      }
-    }
-
-    public GetSchemas_result deepCopy() {
-      return new GetSchemas_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TGetSchemasResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TGetSchemasResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TGetSchemasResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetSchemas_result)
-        return this.equals((GetSchemas_result)that);
-      return false;
-    }
-
-    public boolean equals(GetSchemas_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_success = true && (isSetSuccess());
-      list.add(present_success);
-      if (present_success)
-        list.add(success);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetSchemas_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetSchemas_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetSchemas_resultStandardSchemeFactory implements SchemeFactory {
-      public GetSchemas_resultStandardScheme getScheme() {
-        return new GetSchemas_resultStandardScheme();
-      }
-    }
-
-    private static class GetSchemas_resultStandardScheme extends StandardScheme<GetSchemas_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetSchemas_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TGetSchemasResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetSchemas_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetSchemas_resultTupleSchemeFactory implements SchemeFactory {
-      public GetSchemas_resultTupleScheme getScheme() {
-        return new GetSchemas_resultTupleScheme();
-      }
-    }
-
-    private static class GetSchemas_resultTupleScheme extends TupleScheme<GetSchemas_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetSchemas_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetSchemas_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TGetSchemasResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetTables_args implements org.apache.thrift.TBase<GetTables_args, GetTables_args._Fields>, java.io.Serializable, Cloneable, Comparable<GetTables_args>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetTables_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetTables_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetTables_argsTupleSchemeFactory());
-    }
-
-    private TGetTablesReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetTablesReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetTables_args.class, metaDataMap);
-    }
-
-    public GetTables_args() {
-    }
-
-    public GetTables_args(
-      TGetTablesReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetTables_args(GetTables_args other) {
-      if (other.isSetReq()) {
-        this.req = new TGetTablesReq(other.req);
-      }
-    }
-
-    public GetTables_args deepCopy() {
-      return new GetTables_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TGetTablesReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TGetTablesReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TGetTablesReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetTables_args)
-        return this.equals((GetTables_args)that);
-      return false;
-    }
-
-    public boolean equals(GetTables_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_req = true && (isSetReq());
-      list.add(present_req);
-      if (present_req)
-        list.add(req);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetTables_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(other.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, other.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetTables_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetTables_argsStandardSchemeFactory implements SchemeFactory {
-      public GetTables_argsStandardScheme getScheme() {
-        return new GetTables_argsStandardScheme();
-      }
-    }
-
-    private static class GetTables_argsStandardScheme extends StandardScheme<GetTables_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetTables_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TGetTablesReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetTables_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetTables_argsTupleSchemeFactory implements SchemeFactory {
-      public GetTables_argsTupleScheme getScheme() {
-        return new GetTables_argsTupleScheme();
-      }
-    }
-
-    private static class GetTables_argsTupleScheme extends TupleScheme<GetTables_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetTables_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetTables_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TGetTablesReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetTables_result implements org.apache.thrift.TBase<GetTables_result, GetTables_result._Fields>, java.io.Serializable, Cloneable, Comparable<GetTables_result>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetTables_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetTables_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetTables_resultTupleSchemeFactory());
-    }
-
-    private TGetTablesResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetTablesResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetTables_result.class, metaDataMap);
-    }
-
-    public GetTables_result() {
-    }
-
-    public GetTables_result(
-      TGetTablesResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetTables_result(GetTables_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TGetTablesResp(other.success);
-      }
-    }
-
-    public GetTables_result deepCopy() {
-      return new GetTables_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TGetTablesResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TGetTablesResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TGetTablesResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetTables_result)
-        return this.equals((GetTables_result)that);
-      return false;
-    }
-
-    public boolean equals(GetTables_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_success = true && (isSetSuccess());
-      list.add(present_success);
-      if (present_success)
-        list.add(success);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetTables_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetTables_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetTables_resultStandardSchemeFactory implements SchemeFactory {
-      public GetTables_resultStandardScheme getScheme() {
-        return new GetTables_resultStandardScheme();
-      }
-    }
-
-    private static class GetTables_resultStandardScheme extends StandardScheme<GetTables_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetTables_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TGetTablesResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetTables_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetTables_resultTupleSchemeFactory implements SchemeFactory {
-      public GetTables_resultTupleScheme getScheme() {
-        return new GetTables_resultTupleScheme();
-      }
-    }
-
-    private static class GetTables_resultTupleScheme extends TupleScheme<GetTables_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetTables_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetTables_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TGetTablesResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetTableTypes_args implements org.apache.thrift.TBase<GetTableTypes_args, GetTableTypes_args._Fields>, java.io.Serializable, Cloneable, Comparable<GetTableTypes_args>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetTableTypes_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetTableTypes_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetTableTypes_argsTupleSchemeFactory());
-    }
-
-    private TGetTableTypesReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetTableTypesReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetTableTypes_args.class, metaDataMap);
-    }
-
-    public GetTableTypes_args() {
-    }
-
-    public GetTableTypes_args(
-      TGetTableTypesReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetTableTypes_args(GetTableTypes_args other) {
-      if (other.isSetReq()) {
-        this.req = new TGetTableTypesReq(other.req);
-      }
-    }
-
-    public GetTableTypes_args deepCopy() {
-      return new GetTableTypes_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TGetTableTypesReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TGetTableTypesReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TGetTableTypesReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetTableTypes_args)
-        return this.equals((GetTableTypes_args)that);
-      return false;
-    }
-
-    public boolean equals(GetTableTypes_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_req = true && (isSetReq());
-      list.add(present_req);
-      if (present_req)
-        list.add(req);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetTableTypes_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(other.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, other.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetTableTypes_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetTableTypes_argsStandardSchemeFactory implements SchemeFactory {
-      public GetTableTypes_argsStandardScheme getScheme() {
-        return new GetTableTypes_argsStandardScheme();
-      }
-    }
-
-    private static class GetTableTypes_argsStandardScheme extends StandardScheme<GetTableTypes_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetTableTypes_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TGetTableTypesReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetTableTypes_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetTableTypes_argsTupleSchemeFactory implements SchemeFactory {
-      public GetTableTypes_argsTupleScheme getScheme() {
-        return new GetTableTypes_argsTupleScheme();
-      }
-    }
-
-    private static class GetTableTypes_argsTupleScheme extends TupleScheme<GetTableTypes_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetTableTypes_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetTableTypes_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TGetTableTypesReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetTableTypes_result implements org.apache.thrift.TBase<GetTableTypes_result, GetTableTypes_result._Fields>, java.io.Serializable, Cloneable, Comparable<GetTableTypes_result>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetTableTypes_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetTableTypes_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetTableTypes_resultTupleSchemeFactory());
-    }
-
-    private TGetTableTypesResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetTableTypesResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetTableTypes_result.class, metaDataMap);
-    }
-
-    public GetTableTypes_result() {
-    }
-
-    public GetTableTypes_result(
-      TGetTableTypesResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetTableTypes_result(GetTableTypes_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TGetTableTypesResp(other.success);
-      }
-    }
-
-    public GetTableTypes_result deepCopy() {
-      return new GetTableTypes_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TGetTableTypesResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TGetTableTypesResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TGetTableTypesResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetTableTypes_result)
-        return this.equals((GetTableTypes_result)that);
-      return false;
-    }
-
-    public boolean equals(GetTableTypes_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_success = true && (isSetSuccess());
-      list.add(present_success);
-      if (present_success)
-        list.add(success);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetTableTypes_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetTableTypes_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetTableTypes_resultStandardSchemeFactory implements SchemeFactory {
-      public GetTableTypes_resultStandardScheme getScheme() {
-        return new GetTableTypes_resultStandardScheme();
-      }
-    }
-
-    private static class GetTableTypes_resultStandardScheme extends StandardScheme<GetTableTypes_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetTableTypes_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TGetTableTypesResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetTableTypes_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetTableTypes_resultTupleSchemeFactory implements SchemeFactory {
-      public GetTableTypes_resultTupleScheme getScheme() {
-        return new GetTableTypes_resultTupleScheme();
-      }
-    }
-
-    private static class GetTableTypes_resultTupleScheme extends TupleScheme<GetTableTypes_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetTableTypes_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetTableTypes_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TGetTableTypesResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetColumns_args implements org.apache.thrift.TBase<GetColumns_args, GetColumns_args._Fields>, java.io.Serializable, Cloneable, Comparable<GetColumns_args>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetColumns_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetColumns_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetColumns_argsTupleSchemeFactory());
-    }
-
-    private TGetColumnsReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetColumnsReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetColumns_args.class, metaDataMap);
-    }
-
-    public GetColumns_args() {
-    }
-
-    public GetColumns_args(
-      TGetColumnsReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetColumns_args(GetColumns_args other) {
-      if (other.isSetReq()) {
-        this.req = new TGetColumnsReq(other.req);
-      }
-    }
-
-    public GetColumns_args deepCopy() {
-      return new GetColumns_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TGetColumnsReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TGetColumnsReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TGetColumnsReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetColumns_args)
-        return this.equals((GetColumns_args)that);
-      return false;
-    }
-
-    public boolean equals(GetColumns_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_req = true && (isSetReq());
-      list.add(present_req);
-      if (present_req)
-        list.add(req);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetColumns_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(other.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, other.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetColumns_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetColumns_argsStandardSchemeFactory implements SchemeFactory {
-      public GetColumns_argsStandardScheme getScheme() {
-        return new GetColumns_argsStandardScheme();
-      }
-    }
-
-    private static class GetColumns_argsStandardScheme extends StandardScheme<GetColumns_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetColumns_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TGetColumnsReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetColumns_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetColumns_argsTupleSchemeFactory implements SchemeFactory {
-      public GetColumns_argsTupleScheme getScheme() {
-        return new GetColumns_argsTupleScheme();
-      }
-    }
-
-    private static class GetColumns_argsTupleScheme extends TupleScheme<GetColumns_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetColumns_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetColumns_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TGetColumnsReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetColumns_result implements org.apache.thrift.TBase<GetColumns_result, GetColumns_result._Fields>, java.io.Serializable, Cloneable, Comparable<GetColumns_result>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetColumns_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetColumns_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetColumns_resultTupleSchemeFactory());
-    }
-
-    private TGetColumnsResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetColumnsResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetColumns_result.class, metaDataMap);
-    }
-
-    public GetColumns_result() {
-    }
-
-    public GetColumns_result(
-      TGetColumnsResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetColumns_result(GetColumns_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TGetColumnsResp(other.success);
-      }
-    }
-
-    public GetColumns_result deepCopy() {
-      return new GetColumns_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TGetColumnsResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TGetColumnsResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TGetColumnsResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetColumns_result)
-        return this.equals((GetColumns_result)that);
-      return false;
-    }
-
-    public boolean equals(GetColumns_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_success = true && (isSetSuccess());
-      list.add(present_success);
-      if (present_success)
-        list.add(success);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetColumns_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetColumns_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetColumns_resultStandardSchemeFactory implements SchemeFactory {
-      public GetColumns_resultStandardScheme getScheme() {
-        return new GetColumns_resultStandardScheme();
-      }
-    }
-
-    private static class GetColumns_resultStandardScheme extends StandardScheme<GetColumns_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetColumns_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TGetColumnsResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetColumns_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetColumns_resultTupleSchemeFactory implements SchemeFactory {
-      public GetColumns_resultTupleScheme getScheme() {
-        return new GetColumns_resultTupleScheme();
-      }
-    }
-
-    private static class GetColumns_resultTupleScheme extends TupleScheme<GetColumns_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetColumns_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetColumns_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TGetColumnsResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetFunctions_args implements org.apache.thrift.TBase<GetFunctions_args, GetFunctions_args._Fields>, java.io.Serializable, Cloneable, Comparable<GetFunctions_args>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetFunctions_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetFunctions_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetFunctions_argsTupleSchemeFactory());
-    }
-
-    private TGetFunctionsReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetFunctionsReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetFunctions_args.class, metaDataMap);
-    }
-
-    public GetFunctions_args() {
-    }
-
-    public GetFunctions_args(
-      TGetFunctionsReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetFunctions_args(GetFunctions_args other) {
-      if (other.isSetReq()) {
-        this.req = new TGetFunctionsReq(other.req);
-      }
-    }
-
-    public GetFunctions_args deepCopy() {
-      return new GetFunctions_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TGetFunctionsReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TGetFunctionsReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TGetFunctionsReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetFunctions_args)
-        return this.equals((GetFunctions_args)that);
-      return false;
-    }
-
-    public boolean equals(GetFunctions_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_req = true && (isSetReq());
-      list.add(present_req);
-      if (present_req)
-        list.add(req);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetFunctions_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(other.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, other.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetFunctions_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetFunctions_argsStandardSchemeFactory implements SchemeFactory {
-      public GetFunctions_argsStandardScheme getScheme() {
-        return new GetFunctions_argsStandardScheme();
-      }
-    }
-
-    private static class GetFunctions_argsStandardScheme extends StandardScheme<GetFunctions_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetFunctions_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TGetFunctionsReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetFunctions_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetFunctions_argsTupleSchemeFactory implements SchemeFactory {
-      public GetFunctions_argsTupleScheme getScheme() {
-        return new GetFunctions_argsTupleScheme();
-      }
-    }
-
-    private static class GetFunctions_argsTupleScheme extends TupleScheme<GetFunctions_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetFunctions_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetFunctions_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TGetFunctionsReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetFunctions_result implements org.apache.thrift.TBase<GetFunctions_result, GetFunctions_result._Fields>, java.io.Serializable, Cloneable, Comparable<GetFunctions_result>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetFunctions_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetFunctions_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetFunctions_resultTupleSchemeFactory());
-    }
-
-    private TGetFunctionsResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetFunctionsResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetFunctions_result.class, metaDataMap);
-    }
-
-    public GetFunctions_result() {
-    }
-
-    public GetFunctions_result(
-      TGetFunctionsResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetFunctions_result(GetFunctions_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TGetFunctionsResp(other.success);
-      }
-    }
-
-    public GetFunctions_result deepCopy() {
-      return new GetFunctions_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TGetFunctionsResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TGetFunctionsResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TGetFunctionsResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetFunctions_result)
-        return this.equals((GetFunctions_result)that);
-      return false;
-    }
-
-    public boolean equals(GetFunctions_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_success = true && (isSetSuccess());
-      list.add(present_success);
-      if (present_success)
-        list.add(success);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetFunctions_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetFunctions_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetFunctions_resultStandardSchemeFactory implements SchemeFactory {
-      public GetFunctions_resultStandardScheme getScheme() {
-        return new GetFunctions_resultStandardScheme();
-      }
-    }
-
-    private static class GetFunctions_resultStandardScheme extends StandardScheme<GetFunctions_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetFunctions_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TGetFunctionsResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetFunctions_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetFunctions_resultTupleSchemeFactory implements SchemeFactory {
-      public GetFunctions_resultTupleScheme getScheme() {
-        return new GetFunctions_resultTupleScheme();
-      }
-    }
-
-    private static class GetFunctions_resultTupleScheme extends TupleScheme<GetFunctions_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetFunctions_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetFunctions_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TGetFunctionsResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetPrimaryKeys_args implements org.apache.thrift.TBase<GetPrimaryKeys_args, GetPrimaryKeys_args._Fields>, java.io.Serializable, Cloneable, Comparable<GetPrimaryKeys_args>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetPrimaryKeys_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetPrimaryKeys_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetPrimaryKeys_argsTupleSchemeFactory());
-    }
-
-    private TGetPrimaryKeysReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetPrimaryKeysReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetPrimaryKeys_args.class, metaDataMap);
-    }
-
-    public GetPrimaryKeys_args() {
-    }
-
-    public GetPrimaryKeys_args(
-      TGetPrimaryKeysReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetPrimaryKeys_args(GetPrimaryKeys_args other) {
-      if (other.isSetReq()) {
-        this.req = new TGetPrimaryKeysReq(other.req);
-      }
-    }
-
-    public GetPrimaryKeys_args deepCopy() {
-      return new GetPrimaryKeys_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TGetPrimaryKeysReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TGetPrimaryKeysReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TGetPrimaryKeysReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetPrimaryKeys_args)
-        return this.equals((GetPrimaryKeys_args)that);
-      return false;
-    }
-
-    public boolean equals(GetPrimaryKeys_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_req = true && (isSetReq());
-      list.add(present_req);
-      if (present_req)
-        list.add(req);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetPrimaryKeys_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(other.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, other.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetPrimaryKeys_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetPrimaryKeys_argsStandardSchemeFactory implements SchemeFactory {
-      public GetPrimaryKeys_argsStandardScheme getScheme() {
-        return new GetPrimaryKeys_argsStandardScheme();
-      }
-    }
-
-    private static class GetPrimaryKeys_argsStandardScheme extends StandardScheme<GetPrimaryKeys_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetPrimaryKeys_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TGetPrimaryKeysReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetPrimaryKeys_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetPrimaryKeys_argsTupleSchemeFactory implements SchemeFactory {
-      public GetPrimaryKeys_argsTupleScheme getScheme() {
-        return new GetPrimaryKeys_argsTupleScheme();
-      }
-    }
-
-    private static class GetPrimaryKeys_argsTupleScheme extends TupleScheme<GetPrimaryKeys_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetPrimaryKeys_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetPrimaryKeys_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TGetPrimaryKeysReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetPrimaryKeys_result implements org.apache.thrift.TBase<GetPrimaryKeys_result, GetPrimaryKeys_result._Fields>, java.io.Serializable, Cloneable, Comparable<GetPrimaryKeys_result>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetPrimaryKeys_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetPrimaryKeys_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetPrimaryKeys_resultTupleSchemeFactory());
-    }
-
-    private TGetPrimaryKeysResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetPrimaryKeysResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetPrimaryKeys_result.class, metaDataMap);
-    }
-
-    public GetPrimaryKeys_result() {
-    }
-
-    public GetPrimaryKeys_result(
-      TGetPrimaryKeysResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetPrimaryKeys_result(GetPrimaryKeys_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TGetPrimaryKeysResp(other.success);
-      }
-    }
-
-    public GetPrimaryKeys_result deepCopy() {
-      return new GetPrimaryKeys_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TGetPrimaryKeysResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TGetPrimaryKeysResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TGetPrimaryKeysResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetPrimaryKeys_result)
-        return this.equals((GetPrimaryKeys_result)that);
-      return false;
-    }
-
-    public boolean equals(GetPrimaryKeys_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_success = true && (isSetSuccess());
-      list.add(present_success);
-      if (present_success)
-        list.add(success);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetPrimaryKeys_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetPrimaryKeys_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetPrimaryKeys_resultStandardSchemeFactory implements SchemeFactory {
-      public GetPrimaryKeys_resultStandardScheme getScheme() {
-        return new GetPrimaryKeys_resultStandardScheme();
-      }
-    }
-
-    private static class GetPrimaryKeys_resultStandardScheme extends StandardScheme<GetPrimaryKeys_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetPrimaryKeys_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TGetPrimaryKeysResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetPrimaryKeys_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetPrimaryKeys_resultTupleSchemeFactory implements SchemeFactory {
-      public GetPrimaryKeys_resultTupleScheme getScheme() {
-        return new GetPrimaryKeys_resultTupleScheme();
-      }
-    }
-
-    private static class GetPrimaryKeys_resultTupleScheme extends TupleScheme<GetPrimaryKeys_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetPrimaryKeys_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetPrimaryKeys_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TGetPrimaryKeysResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetCrossReference_args implements org.apache.thrift.TBase<GetCrossReference_args, GetCrossReference_args._Fields>, java.io.Serializable, Cloneable, Comparable<GetCrossReference_args>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetCrossReference_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetCrossReference_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetCrossReference_argsTupleSchemeFactory());
-    }
-
-    private TGetCrossReferenceReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetCrossReferenceReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetCrossReference_args.class, metaDataMap);
-    }
-
-    public GetCrossReference_args() {
-    }
-
-    public GetCrossReference_args(
-      TGetCrossReferenceReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetCrossReference_args(GetCrossReference_args other) {
-      if (other.isSetReq()) {
-        this.req = new TGetCrossReferenceReq(other.req);
-      }
-    }
-
-    public GetCrossReference_args deepCopy() {
-      return new GetCrossReference_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TGetCrossReferenceReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TGetCrossReferenceReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TGetCrossReferenceReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetCrossReference_args)
-        return this.equals((GetCrossReference_args)that);
-      return false;
-    }
-
-    public boolean equals(GetCrossReference_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_req = true && (isSetReq());
-      list.add(present_req);
-      if (present_req)
-        list.add(req);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetCrossReference_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(other.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, other.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetCrossReference_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetCrossReference_argsStandardSchemeFactory implements SchemeFactory {
-      public GetCrossReference_argsStandardScheme getScheme() {
-        return new GetCrossReference_argsStandardScheme();
-      }
-    }
-
-    private static class GetCrossReference_argsStandardScheme extends StandardScheme<GetCrossReference_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetCrossReference_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TGetCrossReferenceReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetCrossReference_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetCrossReference_argsTupleSchemeFactory implements SchemeFactory {
-      public GetCrossReference_argsTupleScheme getScheme() {
-        return new GetCrossReference_argsTupleScheme();
-      }
-    }
-
-    private static class GetCrossReference_argsTupleScheme extends TupleScheme<GetCrossReference_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetCrossReference_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetCrossReference_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TGetCrossReferenceReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetCrossReference_result implements org.apache.thrift.TBase<GetCrossReference_result, GetCrossReference_result._Fields>, java.io.Serializable, Cloneable, Comparable<GetCrossReference_result>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetCrossReference_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetCrossReference_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetCrossReference_resultTupleSchemeFactory());
-    }
-
-    private TGetCrossReferenceResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetCrossReferenceResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetCrossReference_result.class, metaDataMap);
-    }
-
-    public GetCrossReference_result() {
-    }
-
-    public GetCrossReference_result(
-      TGetCrossReferenceResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetCrossReference_result(GetCrossReference_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TGetCrossReferenceResp(other.success);
-      }
-    }
-
-    public GetCrossReference_result deepCopy() {
-      return new GetCrossReference_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TGetCrossReferenceResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TGetCrossReferenceResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TGetCrossReferenceResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetCrossReference_result)
-        return this.equals((GetCrossReference_result)that);
-      return false;
-    }
-
-    public boolean equals(GetCrossReference_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_success = true && (isSetSuccess());
-      list.add(present_success);
-      if (present_success)
-        list.add(success);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetCrossReference_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetCrossReference_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetCrossReference_resultStandardSchemeFactory implements SchemeFactory {
-      public GetCrossReference_resultStandardScheme getScheme() {
-        return new GetCrossReference_resultStandardScheme();
-      }
-    }
-
-    private static class GetCrossReference_resultStandardScheme extends StandardScheme<GetCrossReference_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetCrossReference_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TGetCrossReferenceResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetCrossReference_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetCrossReference_resultTupleSchemeFactory implements SchemeFactory {
-      public GetCrossReference_resultTupleScheme getScheme() {
-        return new GetCrossReference_resultTupleScheme();
-      }
-    }
-
-    private static class GetCrossReference_resultTupleScheme extends TupleScheme<GetCrossReference_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetCrossReference_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetCrossReference_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TGetCrossReferenceResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetOperationStatus_args implements org.apache.thrift.TBase<GetOperationStatus_args, GetOperationStatus_args._Fields>, java.io.Serializable, Cloneable, Comparable<GetOperationStatus_args>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetOperationStatus_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetOperationStatus_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetOperationStatus_argsTupleSchemeFactory());
-    }
-
-    private TGetOperationStatusReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetOperationStatusReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetOperationStatus_args.class, metaDataMap);
-    }
-
-    public GetOperationStatus_args() {
-    }
-
-    public GetOperationStatus_args(
-      TGetOperationStatusReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetOperationStatus_args(GetOperationStatus_args other) {
-      if (other.isSetReq()) {
-        this.req = new TGetOperationStatusReq(other.req);
-      }
-    }
-
-    public GetOperationStatus_args deepCopy() {
-      return new GetOperationStatus_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TGetOperationStatusReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TGetOperationStatusReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TGetOperationStatusReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetOperationStatus_args)
-        return this.equals((GetOperationStatus_args)that);
-      return false;
-    }
-
-    public boolean equals(GetOperationStatus_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_req = true && (isSetReq());
-      list.add(present_req);
-      if (present_req)
-        list.add(req);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetOperationStatus_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(other.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, other.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetOperationStatus_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetOperationStatus_argsStandardSchemeFactory implements SchemeFactory {
-      public GetOperationStatus_argsStandardScheme getScheme() {
-        return new GetOperationStatus_argsStandardScheme();
-      }
-    }
-
-    private static class GetOperationStatus_argsStandardScheme extends StandardScheme<GetOperationStatus_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetOperationStatus_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TGetOperationStatusReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetOperationStatus_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetOperationStatus_argsTupleSchemeFactory implements SchemeFactory {
-      public GetOperationStatus_argsTupleScheme getScheme() {
-        return new GetOperationStatus_argsTupleScheme();
-      }
-    }
-
-    private static class GetOperationStatus_argsTupleScheme extends TupleScheme<GetOperationStatus_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetOperationStatus_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetOperationStatus_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TGetOperationStatusReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetOperationStatus_result implements org.apache.thrift.TBase<GetOperationStatus_result, GetOperationStatus_result._Fields>, java.io.Serializable, Cloneable, Comparable<GetOperationStatus_result>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetOperationStatus_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetOperationStatus_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetOperationStatus_resultTupleSchemeFactory());
-    }
-
-    private TGetOperationStatusResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetOperationStatusResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetOperationStatus_result.class, metaDataMap);
-    }
-
-    public GetOperationStatus_result() {
-    }
-
-    public GetOperationStatus_result(
-      TGetOperationStatusResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetOperationStatus_result(GetOperationStatus_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TGetOperationStatusResp(other.success);
-      }
-    }
-
-    public GetOperationStatus_result deepCopy() {
-      return new GetOperationStatus_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TGetOperationStatusResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TGetOperationStatusResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TGetOperationStatusResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetOperationStatus_result)
-        return this.equals((GetOperationStatus_result)that);
-      return false;
-    }
-
-    public boolean equals(GetOperationStatus_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_success = true && (isSetSuccess());
-      list.add(present_success);
-      if (present_success)
-        list.add(success);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetOperationStatus_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetOperationStatus_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetOperationStatus_resultStandardSchemeFactory implements SchemeFactory {
-      public GetOperationStatus_resultStandardScheme getScheme() {
-        return new GetOperationStatus_resultStandardScheme();
-      }
-    }
-
-    private static class GetOperationStatus_resultStandardScheme extends StandardScheme<GetOperationStatus_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetOperationStatus_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TGetOperationStatusResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetOperationStatus_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetOperationStatus_resultTupleSchemeFactory implements SchemeFactory {
-      public GetOperationStatus_resultTupleScheme getScheme() {
-        return new GetOperationStatus_resultTupleScheme();
-      }
-    }
-
-    private static class GetOperationStatus_resultTupleScheme extends TupleScheme<GetOperationStatus_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetOperationStatus_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetOperationStatus_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TGetOperationStatusResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class CancelOperation_args implements org.apache.thrift.TBase<CancelOperation_args, CancelOperation_args._Fields>, java.io.Serializable, Cloneable, Comparable<CancelOperation_args>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CancelOperation_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new CancelOperation_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new CancelOperation_argsTupleSchemeFactory());
-    }
-
-    private TCancelOperationReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TCancelOperationReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CancelOperation_args.class, metaDataMap);
-    }
-
-    public CancelOperation_args() {
-    }
-
-    public CancelOperation_args(
-      TCancelOperationReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public CancelOperation_args(CancelOperation_args other) {
-      if (other.isSetReq()) {
-        this.req = new TCancelOperationReq(other.req);
-      }
-    }
-
-    public CancelOperation_args deepCopy() {
-      return new CancelOperation_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TCancelOperationReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TCancelOperationReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TCancelOperationReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof CancelOperation_args)
-        return this.equals((CancelOperation_args)that);
-      return false;
-    }
-
-    public boolean equals(CancelOperation_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_req = true && (isSetReq());
-      list.add(present_req);
-      if (present_req)
-        list.add(req);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(CancelOperation_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(other.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, other.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("CancelOperation_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class CancelOperation_argsStandardSchemeFactory implements SchemeFactory {
-      public CancelOperation_argsStandardScheme getScheme() {
-        return new CancelOperation_argsStandardScheme();
-      }
-    }
-
-    private static class CancelOperation_argsStandardScheme extends StandardScheme<CancelOperation_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, CancelOperation_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TCancelOperationReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, CancelOperation_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class CancelOperation_argsTupleSchemeFactory implements SchemeFactory {
-      public CancelOperation_argsTupleScheme getScheme() {
-        return new CancelOperation_argsTupleScheme();
-      }
-    }
-
-    private static class CancelOperation_argsTupleScheme extends TupleScheme<CancelOperation_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, CancelOperation_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, CancelOperation_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TCancelOperationReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class CancelOperation_result implements org.apache.thrift.TBase<CancelOperation_result, CancelOperation_result._Fields>, java.io.Serializable, Cloneable, Comparable<CancelOperation_result>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CancelOperation_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new CancelOperation_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new CancelOperation_resultTupleSchemeFactory());
-    }
-
-    private TCancelOperationResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TCancelOperationResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CancelOperation_result.class, metaDataMap);
-    }
-
-    public CancelOperation_result() {
-    }
-
-    public CancelOperation_result(
-      TCancelOperationResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public CancelOperation_result(CancelOperation_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TCancelOperationResp(other.success);
-      }
-    }
-
-    public CancelOperation_result deepCopy() {
-      return new CancelOperation_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TCancelOperationResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TCancelOperationResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TCancelOperationResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof CancelOperation_result)
-        return this.equals((CancelOperation_result)that);
-      return false;
-    }
-
-    public boolean equals(CancelOperation_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_success = true && (isSetSuccess());
-      list.add(present_success);
-      if (present_success)
-        list.add(success);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(CancelOperation_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("CancelOperation_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class CancelOperation_resultStandardSchemeFactory implements SchemeFactory {
-      public CancelOperation_resultStandardScheme getScheme() {
-        return new CancelOperation_resultStandardScheme();
-      }
-    }
-
-    private static class CancelOperation_resultStandardScheme extends StandardScheme<CancelOperation_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, CancelOperation_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TCancelOperationResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, CancelOperation_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class CancelOperation_resultTupleSchemeFactory implements SchemeFactory {
-      public CancelOperation_resultTupleScheme getScheme() {
-        return new CancelOperation_resultTupleScheme();
-      }
-    }
-
-    private static class CancelOperation_resultTupleScheme extends TupleScheme<CancelOperation_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, CancelOperation_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, CancelOperation_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TCancelOperationResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class CloseOperation_args implements org.apache.thrift.TBase<CloseOperation_args, CloseOperation_args._Fields>, java.io.Serializable, Cloneable, Comparable<CloseOperation_args>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CloseOperation_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new CloseOperation_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new CloseOperation_argsTupleSchemeFactory());
-    }
-
-    private TCloseOperationReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TCloseOperationReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CloseOperation_args.class, metaDataMap);
-    }
-
-    public CloseOperation_args() {
-    }
-
-    public CloseOperation_args(
-      TCloseOperationReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public CloseOperation_args(CloseOperation_args other) {
-      if (other.isSetReq()) {
-        this.req = new TCloseOperationReq(other.req);
-      }
-    }
-
-    public CloseOperation_args deepCopy() {
-      return new CloseOperation_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TCloseOperationReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TCloseOperationReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TCloseOperationReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof CloseOperation_args)
-        return this.equals((CloseOperation_args)that);
-      return false;
-    }
-
-    public boolean equals(CloseOperation_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_req = true && (isSetReq());
-      list.add(present_req);
-      if (present_req)
-        list.add(req);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(CloseOperation_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(other.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, other.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("CloseOperation_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class CloseOperation_argsStandardSchemeFactory implements SchemeFactory {
-      public CloseOperation_argsStandardScheme getScheme() {
-        return new CloseOperation_argsStandardScheme();
-      }
-    }
-
-    private static class CloseOperation_argsStandardScheme extends StandardScheme<CloseOperation_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, CloseOperation_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TCloseOperationReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, CloseOperation_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class CloseOperation_argsTupleSchemeFactory implements SchemeFactory {
-      public CloseOperation_argsTupleScheme getScheme() {
-        return new CloseOperation_argsTupleScheme();
-      }
-    }
-
-    private static class CloseOperation_argsTupleScheme extends TupleScheme<CloseOperation_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, CloseOperation_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, CloseOperation_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TCloseOperationReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class CloseOperation_result implements org.apache.thrift.TBase<CloseOperation_result, CloseOperation_result._Fields>, java.io.Serializable, Cloneable, Comparable<CloseOperation_result>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CloseOperation_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new CloseOperation_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new CloseOperation_resultTupleSchemeFactory());
-    }
-
-    private TCloseOperationResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TCloseOperationResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CloseOperation_result.class, metaDataMap);
-    }
-
-    public CloseOperation_result() {
-    }
-
-    public CloseOperation_result(
-      TCloseOperationResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public CloseOperation_result(CloseOperation_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TCloseOperationResp(other.success);
-      }
-    }
-
-    public CloseOperation_result deepCopy() {
-      return new CloseOperation_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TCloseOperationResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TCloseOperationResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TCloseOperationResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof CloseOperation_result)
-        return this.equals((CloseOperation_result)that);
-      return false;
-    }
-
-    public boolean equals(CloseOperation_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_success = true && (isSetSuccess());
-      list.add(present_success);
-      if (present_success)
-        list.add(success);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(CloseOperation_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("CloseOperation_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class CloseOperation_resultStandardSchemeFactory implements SchemeFactory {
-      public CloseOperation_resultStandardScheme getScheme() {
-        return new CloseOperation_resultStandardScheme();
-      }
-    }
-
-    private static class CloseOperation_resultStandardScheme extends StandardScheme<CloseOperation_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, CloseOperation_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TCloseOperationResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, CloseOperation_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class CloseOperation_resultTupleSchemeFactory implements SchemeFactory {
-      public CloseOperation_resultTupleScheme getScheme() {
-        return new CloseOperation_resultTupleScheme();
-      }
-    }
-
-    private static class CloseOperation_resultTupleScheme extends TupleScheme<CloseOperation_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, CloseOperation_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, CloseOperation_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TCloseOperationResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetResultSetMetadata_args implements org.apache.thrift.TBase<GetResultSetMetadata_args, GetResultSetMetadata_args._Fields>, java.io.Serializable, Cloneable, Comparable<GetResultSetMetadata_args>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetResultSetMetadata_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetResultSetMetadata_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetResultSetMetadata_argsTupleSchemeFactory());
-    }
-
-    private TGetResultSetMetadataReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetResultSetMetadataReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetResultSetMetadata_args.class, metaDataMap);
-    }
-
-    public GetResultSetMetadata_args() {
-    }
-
-    public GetResultSetMetadata_args(
-      TGetResultSetMetadataReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetResultSetMetadata_args(GetResultSetMetadata_args other) {
-      if (other.isSetReq()) {
-        this.req = new TGetResultSetMetadataReq(other.req);
-      }
-    }
-
-    public GetResultSetMetadata_args deepCopy() {
-      return new GetResultSetMetadata_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TGetResultSetMetadataReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TGetResultSetMetadataReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TGetResultSetMetadataReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetResultSetMetadata_args)
-        return this.equals((GetResultSetMetadata_args)that);
-      return false;
-    }
-
-    public boolean equals(GetResultSetMetadata_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_req = true && (isSetReq());
-      list.add(present_req);
-      if (present_req)
-        list.add(req);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetResultSetMetadata_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(other.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, other.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetResultSetMetadata_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetResultSetMetadata_argsStandardSchemeFactory implements SchemeFactory {
-      public GetResultSetMetadata_argsStandardScheme getScheme() {
-        return new GetResultSetMetadata_argsStandardScheme();
-      }
-    }
-
-    private static class GetResultSetMetadata_argsStandardScheme extends StandardScheme<GetResultSetMetadata_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetResultSetMetadata_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TGetResultSetMetadataReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetResultSetMetadata_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetResultSetMetadata_argsTupleSchemeFactory implements SchemeFactory {
-      public GetResultSetMetadata_argsTupleScheme getScheme() {
-        return new GetResultSetMetadata_argsTupleScheme();
-      }
-    }
-
-    private static class GetResultSetMetadata_argsTupleScheme extends TupleScheme<GetResultSetMetadata_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetResultSetMetadata_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetResultSetMetadata_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TGetResultSetMetadataReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetResultSetMetadata_result implements org.apache.thrift.TBase<GetResultSetMetadata_result, GetResultSetMetadata_result._Fields>, java.io.Serializable, Cloneable, Comparable<GetResultSetMetadata_result>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetResultSetMetadata_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetResultSetMetadata_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetResultSetMetadata_resultTupleSchemeFactory());
-    }
-
-    private TGetResultSetMetadataResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetResultSetMetadataResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetResultSetMetadata_result.class, metaDataMap);
-    }
-
-    public GetResultSetMetadata_result() {
-    }
-
-    public GetResultSetMetadata_result(
-      TGetResultSetMetadataResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetResultSetMetadata_result(GetResultSetMetadata_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TGetResultSetMetadataResp(other.success);
-      }
-    }
-
-    public GetResultSetMetadata_result deepCopy() {
-      return new GetResultSetMetadata_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TGetResultSetMetadataResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TGetResultSetMetadataResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TGetResultSetMetadataResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetResultSetMetadata_result)
-        return this.equals((GetResultSetMetadata_result)that);
-      return false;
-    }
-
-    public boolean equals(GetResultSetMetadata_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_success = true && (isSetSuccess());
-      list.add(present_success);
-      if (present_success)
-        list.add(success);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetResultSetMetadata_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetResultSetMetadata_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetResultSetMetadata_resultStandardSchemeFactory implements SchemeFactory {
-      public GetResultSetMetadata_resultStandardScheme getScheme() {
-        return new GetResultSetMetadata_resultStandardScheme();
-      }
-    }
-
-    private static class GetResultSetMetadata_resultStandardScheme extends StandardScheme<GetResultSetMetadata_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetResultSetMetadata_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TGetResultSetMetadataResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetResultSetMetadata_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetResultSetMetadata_resultTupleSchemeFactory implements SchemeFactory {
-      public GetResultSetMetadata_resultTupleScheme getScheme() {
-        return new GetResultSetMetadata_resultTupleScheme();
-      }
-    }
-
-    private static class GetResultSetMetadata_resultTupleScheme extends TupleScheme<GetResultSetMetadata_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetResultSetMetadata_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetResultSetMetadata_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TGetResultSetMetadataResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class FetchResults_args implements org.apache.thrift.TBase<FetchResults_args, FetchResults_args._Fields>, java.io.Serializable, Cloneable, Comparable<FetchResults_args>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("FetchResults_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new FetchResults_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new FetchResults_argsTupleSchemeFactory());
-    }
-
-    private TFetchResultsReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TFetchResultsReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(FetchResults_args.class, metaDataMap);
-    }
-
-    public FetchResults_args() {
-    }
-
-    public FetchResults_args(
-      TFetchResultsReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public FetchResults_args(FetchResults_args other) {
-      if (other.isSetReq()) {
-        this.req = new TFetchResultsReq(other.req);
-      }
-    }
-
-    public FetchResults_args deepCopy() {
-      return new FetchResults_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TFetchResultsReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TFetchResultsReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TFetchResultsReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof FetchResults_args)
-        return this.equals((FetchResults_args)that);
-      return false;
-    }
-
-    public boolean equals(FetchResults_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_req = true && (isSetReq());
-      list.add(present_req);
-      if (present_req)
-        list.add(req);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(FetchResults_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(other.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, other.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("FetchResults_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class FetchResults_argsStandardSchemeFactory implements SchemeFactory {
-      public FetchResults_argsStandardScheme getScheme() {
-        return new FetchResults_argsStandardScheme();
-      }
-    }
-
-    private static class FetchResults_argsStandardScheme extends StandardScheme<FetchResults_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, FetchResults_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TFetchResultsReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, FetchResults_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class FetchResults_argsTupleSchemeFactory implements SchemeFactory {
-      public FetchResults_argsTupleScheme getScheme() {
-        return new FetchResults_argsTupleScheme();
-      }
-    }
-
-    private static class FetchResults_argsTupleScheme extends TupleScheme<FetchResults_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, FetchResults_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, FetchResults_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TFetchResultsReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class FetchResults_result implements org.apache.thrift.TBase<FetchResults_result, FetchResults_result._Fields>, java.io.Serializable, Cloneable, Comparable<FetchResults_result>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("FetchResults_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new FetchResults_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new FetchResults_resultTupleSchemeFactory());
-    }
-
-    private TFetchResultsResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TFetchResultsResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(FetchResults_result.class, metaDataMap);
-    }
-
-    public FetchResults_result() {
-    }
-
-    public FetchResults_result(
-      TFetchResultsResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public FetchResults_result(FetchResults_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TFetchResultsResp(other.success);
-      }
-    }
-
-    public FetchResults_result deepCopy() {
-      return new FetchResults_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TFetchResultsResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TFetchResultsResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TFetchResultsResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof FetchResults_result)
-        return this.equals((FetchResults_result)that);
-      return false;
-    }
-
-    public boolean equals(FetchResults_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_success = true && (isSetSuccess());
-      list.add(present_success);
-      if (present_success)
-        list.add(success);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(FetchResults_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("FetchResults_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class FetchResults_resultStandardSchemeFactory implements SchemeFactory {
-      public FetchResults_resultStandardScheme getScheme() {
-        return new FetchResults_resultStandardScheme();
-      }
-    }
-
-    private static class FetchResults_resultStandardScheme extends StandardScheme<FetchResults_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, FetchResults_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TFetchResultsResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, FetchResults_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class FetchResults_resultTupleSchemeFactory implements SchemeFactory {
-      public FetchResults_resultTupleScheme getScheme() {
-        return new FetchResults_resultTupleScheme();
-      }
-    }
-
-    private static class FetchResults_resultTupleScheme extends TupleScheme<FetchResults_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, FetchResults_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, FetchResults_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TFetchResultsResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetDelegationToken_args implements org.apache.thrift.TBase<GetDelegationToken_args, GetDelegationToken_args._Fields>, java.io.Serializable, Cloneable, Comparable<GetDelegationToken_args>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetDelegationToken_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetDelegationToken_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetDelegationToken_argsTupleSchemeFactory());
-    }
-
-    private TGetDelegationTokenReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetDelegationTokenReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetDelegationToken_args.class, metaDataMap);
-    }
-
-    public GetDelegationToken_args() {
-    }
-
-    public GetDelegationToken_args(
-      TGetDelegationTokenReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetDelegationToken_args(GetDelegationToken_args other) {
-      if (other.isSetReq()) {
-        this.req = new TGetDelegationTokenReq(other.req);
-      }
-    }
-
-    public GetDelegationToken_args deepCopy() {
-      return new GetDelegationToken_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TGetDelegationTokenReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TGetDelegationTokenReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TGetDelegationTokenReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetDelegationToken_args)
-        return this.equals((GetDelegationToken_args)that);
-      return false;
-    }
-
-    public boolean equals(GetDelegationToken_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_req = true && (isSetReq());
-      list.add(present_req);
-      if (present_req)
-        list.add(req);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetDelegationToken_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(other.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, other.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetDelegationToken_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetDelegationToken_argsStandardSchemeFactory implements SchemeFactory {
-      public GetDelegationToken_argsStandardScheme getScheme() {
-        return new GetDelegationToken_argsStandardScheme();
-      }
-    }
-
-    private static class GetDelegationToken_argsStandardScheme extends StandardScheme<GetDelegationToken_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetDelegationToken_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TGetDelegationTokenReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetDelegationToken_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetDelegationToken_argsTupleSchemeFactory implements SchemeFactory {
-      public GetDelegationToken_argsTupleScheme getScheme() {
-        return new GetDelegationToken_argsTupleScheme();
-      }
-    }
-
-    private static class GetDelegationToken_argsTupleScheme extends TupleScheme<GetDelegationToken_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetDelegationToken_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetDelegationToken_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TGetDelegationTokenReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class GetDelegationToken_result implements org.apache.thrift.TBase<GetDelegationToken_result, GetDelegationToken_result._Fields>, java.io.Serializable, Cloneable, Comparable<GetDelegationToken_result>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetDelegationToken_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new GetDelegationToken_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new GetDelegationToken_resultTupleSchemeFactory());
-    }
-
-    private TGetDelegationTokenResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetDelegationTokenResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetDelegationToken_result.class, metaDataMap);
-    }
-
-    public GetDelegationToken_result() {
-    }
-
-    public GetDelegationToken_result(
-      TGetDelegationTokenResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public GetDelegationToken_result(GetDelegationToken_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TGetDelegationTokenResp(other.success);
-      }
-    }
-
-    public GetDelegationToken_result deepCopy() {
-      return new GetDelegationToken_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TGetDelegationTokenResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TGetDelegationTokenResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TGetDelegationTokenResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof GetDelegationToken_result)
-        return this.equals((GetDelegationToken_result)that);
-      return false;
-    }
-
-    public boolean equals(GetDelegationToken_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_success = true && (isSetSuccess());
-      list.add(present_success);
-      if (present_success)
-        list.add(success);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(GetDelegationToken_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("GetDelegationToken_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class GetDelegationToken_resultStandardSchemeFactory implements SchemeFactory {
-      public GetDelegationToken_resultStandardScheme getScheme() {
-        return new GetDelegationToken_resultStandardScheme();
-      }
-    }
-
-    private static class GetDelegationToken_resultStandardScheme extends StandardScheme<GetDelegationToken_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, GetDelegationToken_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TGetDelegationTokenResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, GetDelegationToken_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class GetDelegationToken_resultTupleSchemeFactory implements SchemeFactory {
-      public GetDelegationToken_resultTupleScheme getScheme() {
-        return new GetDelegationToken_resultTupleScheme();
-      }
-    }
-
-    private static class GetDelegationToken_resultTupleScheme extends TupleScheme<GetDelegationToken_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, GetDelegationToken_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, GetDelegationToken_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TGetDelegationTokenResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class CancelDelegationToken_args implements org.apache.thrift.TBase<CancelDelegationToken_args, CancelDelegationToken_args._Fields>, java.io.Serializable, Cloneable, Comparable<CancelDelegationToken_args>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CancelDelegationToken_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new CancelDelegationToken_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new CancelDelegationToken_argsTupleSchemeFactory());
-    }
-
-    private TCancelDelegationTokenReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TCancelDelegationTokenReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CancelDelegationToken_args.class, metaDataMap);
-    }
-
-    public CancelDelegationToken_args() {
-    }
-
-    public CancelDelegationToken_args(
-      TCancelDelegationTokenReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public CancelDelegationToken_args(CancelDelegationToken_args other) {
-      if (other.isSetReq()) {
-        this.req = new TCancelDelegationTokenReq(other.req);
-      }
-    }
-
-    public CancelDelegationToken_args deepCopy() {
-      return new CancelDelegationToken_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TCancelDelegationTokenReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TCancelDelegationTokenReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TCancelDelegationTokenReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof CancelDelegationToken_args)
-        return this.equals((CancelDelegationToken_args)that);
-      return false;
-    }
-
-    public boolean equals(CancelDelegationToken_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_req = true && (isSetReq());
-      list.add(present_req);
-      if (present_req)
-        list.add(req);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(CancelDelegationToken_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(other.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, other.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("CancelDelegationToken_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class CancelDelegationToken_argsStandardSchemeFactory implements SchemeFactory {
-      public CancelDelegationToken_argsStandardScheme getScheme() {
-        return new CancelDelegationToken_argsStandardScheme();
-      }
-    }
-
-    private static class CancelDelegationToken_argsStandardScheme extends StandardScheme<CancelDelegationToken_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, CancelDelegationToken_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TCancelDelegationTokenReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, CancelDelegationToken_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class CancelDelegationToken_argsTupleSchemeFactory implements SchemeFactory {
-      public CancelDelegationToken_argsTupleScheme getScheme() {
-        return new CancelDelegationToken_argsTupleScheme();
-      }
-    }
-
-    private static class CancelDelegationToken_argsTupleScheme extends TupleScheme<CancelDelegationToken_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, CancelDelegationToken_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, CancelDelegationToken_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TCancelDelegationTokenReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class CancelDelegationToken_result implements org.apache.thrift.TBase<CancelDelegationToken_result, CancelDelegationToken_result._Fields>, java.io.Serializable, Cloneable, Comparable<CancelDelegationToken_result>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CancelDelegationToken_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new CancelDelegationToken_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new CancelDelegationToken_resultTupleSchemeFactory());
-    }
-
-    private TCancelDelegationTokenResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TCancelDelegationTokenResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CancelDelegationToken_result.class, metaDataMap);
-    }
-
-    public CancelDelegationToken_result() {
-    }
-
-    public CancelDelegationToken_result(
-      TCancelDelegationTokenResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public CancelDelegationToken_result(CancelDelegationToken_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TCancelDelegationTokenResp(other.success);
-      }
-    }
-
-    public CancelDelegationToken_result deepCopy() {
-      return new CancelDelegationToken_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TCancelDelegationTokenResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TCancelDelegationTokenResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TCancelDelegationTokenResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof CancelDelegationToken_result)
-        return this.equals((CancelDelegationToken_result)that);
-      return false;
-    }
-
-    public boolean equals(CancelDelegationToken_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_success = true && (isSetSuccess());
-      list.add(present_success);
-      if (present_success)
-        list.add(success);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(CancelDelegationToken_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("CancelDelegationToken_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class CancelDelegationToken_resultStandardSchemeFactory implements SchemeFactory {
-      public CancelDelegationToken_resultStandardScheme getScheme() {
-        return new CancelDelegationToken_resultStandardScheme();
-      }
-    }
-
-    private static class CancelDelegationToken_resultStandardScheme extends StandardScheme<CancelDelegationToken_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, CancelDelegationToken_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TCancelDelegationTokenResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, CancelDelegationToken_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class CancelDelegationToken_resultTupleSchemeFactory implements SchemeFactory {
-      public CancelDelegationToken_resultTupleScheme getScheme() {
-        return new CancelDelegationToken_resultTupleScheme();
-      }
-    }
-
-    private static class CancelDelegationToken_resultTupleScheme extends TupleScheme<CancelDelegationToken_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, CancelDelegationToken_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, CancelDelegationToken_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TCancelDelegationTokenResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class RenewDelegationToken_args implements org.apache.thrift.TBase<RenewDelegationToken_args, RenewDelegationToken_args._Fields>, java.io.Serializable, Cloneable, Comparable<RenewDelegationToken_args>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RenewDelegationToken_args");
-
-    private static final org.apache.thrift.protocol.TField REQ_FIELD_DESC = new org.apache.thrift.protocol.TField("req", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new RenewDelegationToken_argsStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new RenewDelegationToken_argsTupleSchemeFactory());
-    }
-
-    private TRenewDelegationTokenReq req; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      REQ((short)1, "req");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 1: // REQ
-            return REQ;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.REQ, new org.apache.thrift.meta_data.FieldMetaData("req", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TRenewDelegationTokenReq.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(RenewDelegationToken_args.class, metaDataMap);
-    }
-
-    public RenewDelegationToken_args() {
-    }
-
-    public RenewDelegationToken_args(
-      TRenewDelegationTokenReq req)
-    {
-      this();
-      this.req = req;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public RenewDelegationToken_args(RenewDelegationToken_args other) {
-      if (other.isSetReq()) {
-        this.req = new TRenewDelegationTokenReq(other.req);
-      }
-    }
-
-    public RenewDelegationToken_args deepCopy() {
-      return new RenewDelegationToken_args(this);
-    }
-
-    @Override
-    public void clear() {
-      this.req = null;
-    }
-
-    public TRenewDelegationTokenReq getReq() {
-      return this.req;
-    }
-
-    public void setReq(TRenewDelegationTokenReq req) {
-      this.req = req;
-    }
-
-    public void unsetReq() {
-      this.req = null;
-    }
-
-    /** Returns true if field req is set (has been assigned a value) and false otherwise */
-    public boolean isSetReq() {
-      return this.req != null;
-    }
-
-    public void setReqIsSet(boolean value) {
-      if (!value) {
-        this.req = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case REQ:
-        if (value == null) {
-          unsetReq();
-        } else {
-          setReq((TRenewDelegationTokenReq)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case REQ:
-        return getReq();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case REQ:
-        return isSetReq();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof RenewDelegationToken_args)
-        return this.equals((RenewDelegationToken_args)that);
-      return false;
-    }
-
-    public boolean equals(RenewDelegationToken_args that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_req = true && this.isSetReq();
-      boolean that_present_req = true && that.isSetReq();
-      if (this_present_req || that_present_req) {
-        if (!(this_present_req && that_present_req))
-          return false;
-        if (!this.req.equals(that.req))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_req = true && (isSetReq());
-      list.add(present_req);
-      if (present_req)
-        list.add(req);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(RenewDelegationToken_args other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetReq()).compareTo(other.isSetReq());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetReq()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.req, other.req);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("RenewDelegationToken_args(");
-      boolean first = true;
-
-      sb.append("req:");
-      if (this.req == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.req);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (req != null) {
-        req.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class RenewDelegationToken_argsStandardSchemeFactory implements SchemeFactory {
-      public RenewDelegationToken_argsStandardScheme getScheme() {
-        return new RenewDelegationToken_argsStandardScheme();
-      }
-    }
-
-    private static class RenewDelegationToken_argsStandardScheme extends StandardScheme<RenewDelegationToken_args> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, RenewDelegationToken_args struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 1: // REQ
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.req = new TRenewDelegationTokenReq();
-                struct.req.read(iprot);
-                struct.setReqIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, RenewDelegationToken_args struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.req != null) {
-          oprot.writeFieldBegin(REQ_FIELD_DESC);
-          struct.req.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class RenewDelegationToken_argsTupleSchemeFactory implements SchemeFactory {
-      public RenewDelegationToken_argsTupleScheme getScheme() {
-        return new RenewDelegationToken_argsTupleScheme();
-      }
-    }
-
-    private static class RenewDelegationToken_argsTupleScheme extends TupleScheme<RenewDelegationToken_args> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, RenewDelegationToken_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetReq()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetReq()) {
-          struct.req.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, RenewDelegationToken_args struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.req = new TRenewDelegationTokenReq();
-          struct.req.read(iprot);
-          struct.setReqIsSet(true);
-        }
-      }
-    }
-
-  }
-
-  public static class RenewDelegationToken_result implements org.apache.thrift.TBase<RenewDelegationToken_result, RenewDelegationToken_result._Fields>, java.io.Serializable, Cloneable, Comparable<RenewDelegationToken_result>   {
-    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RenewDelegationToken_result");
-
-    private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
-    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-    static {
-      schemes.put(StandardScheme.class, new RenewDelegationToken_resultStandardSchemeFactory());
-      schemes.put(TupleScheme.class, new RenewDelegationToken_resultTupleSchemeFactory());
-    }
-
-    private TRenewDelegationTokenResp success; // required
-
-    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-      SUCCESS((short)0, "success");
-
-      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-      static {
-        for (_Fields field : EnumSet.allOf(_Fields.class)) {
-          byName.put(field.getFieldName(), field);
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, or null if its not found.
-       */
-      public static _Fields findByThriftId(int fieldId) {
-        switch(fieldId) {
-          case 0: // SUCCESS
-            return SUCCESS;
-          default:
-            return null;
-        }
-      }
-
-      /**
-       * Find the _Fields constant that matches fieldId, throwing an exception
-       * if it is not found.
-       */
-      public static _Fields findByThriftIdOrThrow(int fieldId) {
-        _Fields fields = findByThriftId(fieldId);
-        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-        return fields;
-      }
-
-      /**
-       * Find the _Fields constant that matches name, or null if its not found.
-       */
-      public static _Fields findByName(String name) {
-        return byName.get(name);
-      }
-
-      private final short _thriftId;
-      private final String _fieldName;
-
-      _Fields(short thriftId, String fieldName) {
-        _thriftId = thriftId;
-        _fieldName = fieldName;
-      }
-
-      public short getThriftFieldId() {
-        return _thriftId;
-      }
-
-      public String getFieldName() {
-        return _fieldName;
-      }
-    }
-
-    // isset id assignments
-    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-    static {
-      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-      tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TRenewDelegationTokenResp.class)));
-      metaDataMap = Collections.unmodifiableMap(tmpMap);
-      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(RenewDelegationToken_result.class, metaDataMap);
-    }
-
-    public RenewDelegationToken_result() {
-    }
-
-    public RenewDelegationToken_result(
-      TRenewDelegationTokenResp success)
-    {
-      this();
-      this.success = success;
-    }
-
-    /**
-     * Performs a deep copy on <i>other</i>.
-     */
-    public RenewDelegationToken_result(RenewDelegationToken_result other) {
-      if (other.isSetSuccess()) {
-        this.success = new TRenewDelegationTokenResp(other.success);
-      }
-    }
-
-    public RenewDelegationToken_result deepCopy() {
-      return new RenewDelegationToken_result(this);
-    }
-
-    @Override
-    public void clear() {
-      this.success = null;
-    }
-
-    public TRenewDelegationTokenResp getSuccess() {
-      return this.success;
-    }
-
-    public void setSuccess(TRenewDelegationTokenResp success) {
-      this.success = success;
-    }
-
-    public void unsetSuccess() {
-      this.success = null;
-    }
-
-    /** Returns true if field success is set (has been assigned a value) and false otherwise */
-    public boolean isSetSuccess() {
-      return this.success != null;
-    }
-
-    public void setSuccessIsSet(boolean value) {
-      if (!value) {
-        this.success = null;
-      }
-    }
-
-    public void setFieldValue(_Fields field, Object value) {
-      switch (field) {
-      case SUCCESS:
-        if (value == null) {
-          unsetSuccess();
-        } else {
-          setSuccess((TRenewDelegationTokenResp)value);
-        }
-        break;
-
-      }
-    }
-
-    public Object getFieldValue(_Fields field) {
-      switch (field) {
-      case SUCCESS:
-        return getSuccess();
-
-      }
-      throw new IllegalStateException();
-    }
-
-    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-    public boolean isSet(_Fields field) {
-      if (field == null) {
-        throw new IllegalArgumentException();
-      }
-
-      switch (field) {
-      case SUCCESS:
-        return isSetSuccess();
-      }
-      throw new IllegalStateException();
-    }
-
-    @Override
-    public boolean equals(Object that) {
-      if (that == null)
-        return false;
-      if (that instanceof RenewDelegationToken_result)
-        return this.equals((RenewDelegationToken_result)that);
-      return false;
-    }
-
-    public boolean equals(RenewDelegationToken_result that) {
-      if (that == null)
-        return false;
-
-      boolean this_present_success = true && this.isSetSuccess();
-      boolean that_present_success = true && that.isSetSuccess();
-      if (this_present_success || that_present_success) {
-        if (!(this_present_success && that_present_success))
-          return false;
-        if (!this.success.equals(that.success))
-          return false;
-      }
-
-      return true;
-    }
-
-    @Override
-    public int hashCode() {
-      List<Object> list = new ArrayList<Object>();
-
-      boolean present_success = true && (isSetSuccess());
-      list.add(present_success);
-      if (present_success)
-        list.add(success);
-
-      return list.hashCode();
-    }
-
-    @Override
-    public int compareTo(RenewDelegationToken_result other) {
-      if (!getClass().equals(other.getClass())) {
-        return getClass().getName().compareTo(other.getClass().getName());
-      }
-
-      int lastComparison = 0;
-
-      lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-      if (isSetSuccess()) {
-        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
-        if (lastComparison != 0) {
-          return lastComparison;
-        }
-      }
-      return 0;
-    }
-
-    public _Fields fieldForId(int fieldId) {
-      return _Fields.findByThriftId(fieldId);
-    }
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-      schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-      schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-      }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder("RenewDelegationToken_result(");
-      boolean first = true;
-
-      sb.append("success:");
-      if (this.success == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.success);
-      }
-      first = false;
-      sb.append(")");
-      return sb.toString();
-    }
-
-    public void validate() throws org.apache.thrift.TException {
-      // check for required fields
-      // check for sub-struct validity
-      if (success != null) {
-        success.validate();
-      }
-    }
-
-    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-      try {
-        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-      try {
-        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-      } catch (org.apache.thrift.TException te) {
-        throw new java.io.IOException(te);
-      }
-    }
-
-    private static class RenewDelegationToken_resultStandardSchemeFactory implements SchemeFactory {
-      public RenewDelegationToken_resultStandardScheme getScheme() {
-        return new RenewDelegationToken_resultStandardScheme();
-      }
-    }
-
-    private static class RenewDelegationToken_resultStandardScheme extends StandardScheme<RenewDelegationToken_result> {
-
-      public void read(org.apache.thrift.protocol.TProtocol iprot, RenewDelegationToken_result struct) throws org.apache.thrift.TException {
-        org.apache.thrift.protocol.TField schemeField;
-        iprot.readStructBegin();
-        while (true)
-        {
-          schemeField = iprot.readFieldBegin();
-          if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-            break;
-          }
-          switch (schemeField.id) {
-            case 0: // SUCCESS
-              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-                struct.success = new TRenewDelegationTokenResp();
-                struct.success.read(iprot);
-                struct.setSuccessIsSet(true);
-              } else { 
-                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-              }
-              break;
-            default:
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-          }
-          iprot.readFieldEnd();
-        }
-        iprot.readStructEnd();
-        struct.validate();
-      }
-
-      public void write(org.apache.thrift.protocol.TProtocol oprot, RenewDelegationToken_result struct) throws org.apache.thrift.TException {
-        struct.validate();
-
-        oprot.writeStructBegin(STRUCT_DESC);
-        if (struct.success != null) {
-          oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
-          struct.success.write(oprot);
-          oprot.writeFieldEnd();
-        }
-        oprot.writeFieldStop();
-        oprot.writeStructEnd();
-      }
-
-    }
-
-    private static class RenewDelegationToken_resultTupleSchemeFactory implements SchemeFactory {
-      public RenewDelegationToken_resultTupleScheme getScheme() {
-        return new RenewDelegationToken_resultTupleScheme();
-      }
-    }
-
-    private static class RenewDelegationToken_resultTupleScheme extends TupleScheme<RenewDelegationToken_result> {
-
-      @Override
-      public void write(org.apache.thrift.protocol.TProtocol prot, RenewDelegationToken_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol oprot = (TTupleProtocol) prot;
-        BitSet optionals = new BitSet();
-        if (struct.isSetSuccess()) {
-          optionals.set(0);
-        }
-        oprot.writeBitSet(optionals, 1);
-        if (struct.isSetSuccess()) {
-          struct.success.write(oprot);
-        }
-      }
-
-      @Override
-      public void read(org.apache.thrift.protocol.TProtocol prot, RenewDelegationToken_result struct) throws org.apache.thrift.TException {
-        TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
-        if (incoming.get(0)) {
-          struct.success = new TRenewDelegationTokenResp();
-          struct.success.read(iprot);
-          struct.setSuccessIsSet(true);
-        }
-      }
-    }
-
-  }
-
-}
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCLIServiceConstants.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCLIServiceConstants.java
deleted file mode 100644
index 930bed731ed2a..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCLIServiceConstants.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-public class TCLIServiceConstants {
-
-  public static final Set<TTypeId> PRIMITIVE_TYPES = new HashSet<TTypeId>();
-  static {
-    PRIMITIVE_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.BOOLEAN_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.TINYINT_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.SMALLINT_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.INT_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.BIGINT_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.FLOAT_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.DOUBLE_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.STRING_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.TIMESTAMP_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.BINARY_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.DECIMAL_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.NULL_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.DATE_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.VARCHAR_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.CHAR_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.INTERVAL_YEAR_MONTH_TYPE);
-    PRIMITIVE_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.INTERVAL_DAY_TIME_TYPE);
-  }
-
-  public static final Set<TTypeId> COMPLEX_TYPES = new HashSet<TTypeId>();
-  static {
-    COMPLEX_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.ARRAY_TYPE);
-    COMPLEX_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.MAP_TYPE);
-    COMPLEX_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.STRUCT_TYPE);
-    COMPLEX_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.UNION_TYPE);
-    COMPLEX_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.USER_DEFINED_TYPE);
-  }
-
-  public static final Set<TTypeId> COLLECTION_TYPES = new HashSet<TTypeId>();
-  static {
-    COLLECTION_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.ARRAY_TYPE);
-    COLLECTION_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.MAP_TYPE);
-  }
-
-  public static final Map<TTypeId,String> TYPE_NAMES = new HashMap<TTypeId,String>();
-  static {
-    TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.BOOLEAN_TYPE, "BOOLEAN");
-    TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.TINYINT_TYPE, "TINYINT");
-    TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.SMALLINT_TYPE, "SMALLINT");
-    TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.INT_TYPE, "INT");
-    TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.BIGINT_TYPE, "BIGINT");
-    TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.FLOAT_TYPE, "FLOAT");
-    TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.DOUBLE_TYPE, "DOUBLE");
-    TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.STRING_TYPE, "STRING");
-    TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.TIMESTAMP_TYPE, "TIMESTAMP");
-    TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.BINARY_TYPE, "BINARY");
-    TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.ARRAY_TYPE, "ARRAY");
-    TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.MAP_TYPE, "MAP");
-    TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.STRUCT_TYPE, "STRUCT");
-    TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.UNION_TYPE, "UNIONTYPE");
-    TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.DECIMAL_TYPE, "DECIMAL");
-    TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.NULL_TYPE, "NULL");
-    TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.DATE_TYPE, "DATE");
-    TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.VARCHAR_TYPE, "VARCHAR");
-    TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.CHAR_TYPE, "CHAR");
-    TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.INTERVAL_YEAR_MONTH_TYPE, "INTERVAL_YEAR_MONTH");
-    TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.INTERVAL_DAY_TIME_TYPE, "INTERVAL_DAY_TIME");
-  }
-
-  public static final String CHARACTER_MAXIMUM_LENGTH = "characterMaximumLength";
-
-  public static final String PRECISION = "precision";
-
-  public static final String SCALE = "scale";
-
-}
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCancelDelegationTokenReq.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCancelDelegationTokenReq.java
deleted file mode 100644
index a7d4e7de1f60d..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCancelDelegationTokenReq.java
+++ /dev/null
@@ -1,495 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TCancelDelegationTokenReq implements org.apache.thrift.TBase<TCancelDelegationTokenReq, TCancelDelegationTokenReq._Fields>, java.io.Serializable, Cloneable, Comparable<TCancelDelegationTokenReq> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TCancelDelegationTokenReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField DELEGATION_TOKEN_FIELD_DESC = new org.apache.thrift.protocol.TField("delegationToken", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TCancelDelegationTokenReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TCancelDelegationTokenReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-  private String delegationToken; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle"),
-    DELEGATION_TOKEN((short)2, "delegationToken");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        case 2: // DELEGATION_TOKEN
-          return DELEGATION_TOKEN;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    tmpMap.put(_Fields.DELEGATION_TOKEN, new org.apache.thrift.meta_data.FieldMetaData("delegationToken", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TCancelDelegationTokenReq.class, metaDataMap);
-  }
-
-  public TCancelDelegationTokenReq() {
-  }
-
-  public TCancelDelegationTokenReq(
-    TSessionHandle sessionHandle,
-    String delegationToken)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-    this.delegationToken = delegationToken;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TCancelDelegationTokenReq(TCancelDelegationTokenReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-    if (other.isSetDelegationToken()) {
-      this.delegationToken = other.delegationToken;
-    }
-  }
-
-  public TCancelDelegationTokenReq deepCopy() {
-    return new TCancelDelegationTokenReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-    this.delegationToken = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public String getDelegationToken() {
-    return this.delegationToken;
-  }
-
-  public void setDelegationToken(String delegationToken) {
-    this.delegationToken = delegationToken;
-  }
-
-  public void unsetDelegationToken() {
-    this.delegationToken = null;
-  }
-
-  /** Returns true if field delegationToken is set (has been assigned a value) and false otherwise */
-  public boolean isSetDelegationToken() {
-    return this.delegationToken != null;
-  }
-
-  public void setDelegationTokenIsSet(boolean value) {
-    if (!value) {
-      this.delegationToken = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    case DELEGATION_TOKEN:
-      if (value == null) {
-        unsetDelegationToken();
-      } else {
-        setDelegationToken((String)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    case DELEGATION_TOKEN:
-      return getDelegationToken();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    case DELEGATION_TOKEN:
-      return isSetDelegationToken();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TCancelDelegationTokenReq)
-      return this.equals((TCancelDelegationTokenReq)that);
-    return false;
-  }
-
-  public boolean equals(TCancelDelegationTokenReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    boolean this_present_delegationToken = true && this.isSetDelegationToken();
-    boolean that_present_delegationToken = true && that.isSetDelegationToken();
-    if (this_present_delegationToken || that_present_delegationToken) {
-      if (!(this_present_delegationToken && that_present_delegationToken))
-        return false;
-      if (!this.delegationToken.equals(that.delegationToken))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    list.add(present_sessionHandle);
-    if (present_sessionHandle)
-      list.add(sessionHandle);
-
-    boolean present_delegationToken = true && (isSetDelegationToken());
-    list.add(present_delegationToken);
-    if (present_delegationToken)
-      list.add(delegationToken);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TCancelDelegationTokenReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(other.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, other.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetDelegationToken()).compareTo(other.isSetDelegationToken());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetDelegationToken()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.delegationToken, other.delegationToken);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TCancelDelegationTokenReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("delegationToken:");
-    if (this.delegationToken == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.delegationToken);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    if (!isSetDelegationToken()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'delegationToken' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TCancelDelegationTokenReqStandardSchemeFactory implements SchemeFactory {
-    public TCancelDelegationTokenReqStandardScheme getScheme() {
-      return new TCancelDelegationTokenReqStandardScheme();
-    }
-  }
-
-  private static class TCancelDelegationTokenReqStandardScheme extends StandardScheme<TCancelDelegationTokenReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TCancelDelegationTokenReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // DELEGATION_TOKEN
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.delegationToken = iprot.readString();
-              struct.setDelegationTokenIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TCancelDelegationTokenReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.delegationToken != null) {
-        oprot.writeFieldBegin(DELEGATION_TOKEN_FIELD_DESC);
-        oprot.writeString(struct.delegationToken);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TCancelDelegationTokenReqTupleSchemeFactory implements SchemeFactory {
-    public TCancelDelegationTokenReqTupleScheme getScheme() {
-      return new TCancelDelegationTokenReqTupleScheme();
-    }
-  }
-
-  private static class TCancelDelegationTokenReqTupleScheme extends TupleScheme<TCancelDelegationTokenReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TCancelDelegationTokenReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-      oprot.writeString(struct.delegationToken);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TCancelDelegationTokenReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-      struct.delegationToken = iprot.readString();
-      struct.setDelegationTokenIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCancelDelegationTokenResp.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCancelDelegationTokenResp.java
deleted file mode 100644
index 611e92ca2af30..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCancelDelegationTokenResp.java
+++ /dev/null
@@ -1,394 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TCancelDelegationTokenResp implements org.apache.thrift.TBase<TCancelDelegationTokenResp, TCancelDelegationTokenResp._Fields>, java.io.Serializable, Cloneable, Comparable<TCancelDelegationTokenResp> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TCancelDelegationTokenResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TCancelDelegationTokenRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TCancelDelegationTokenRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TCancelDelegationTokenResp.class, metaDataMap);
-  }
-
-  public TCancelDelegationTokenResp() {
-  }
-
-  public TCancelDelegationTokenResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TCancelDelegationTokenResp(TCancelDelegationTokenResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-  }
-
-  public TCancelDelegationTokenResp deepCopy() {
-    return new TCancelDelegationTokenResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TCancelDelegationTokenResp)
-      return this.equals((TCancelDelegationTokenResp)that);
-    return false;
-  }
-
-  public boolean equals(TCancelDelegationTokenResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_status = true && (isSetStatus());
-    list.add(present_status);
-    if (present_status)
-      list.add(status);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TCancelDelegationTokenResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(other.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, other.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TCancelDelegationTokenResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TCancelDelegationTokenRespStandardSchemeFactory implements SchemeFactory {
-    public TCancelDelegationTokenRespStandardScheme getScheme() {
-      return new TCancelDelegationTokenRespStandardScheme();
-    }
-  }
-
-  private static class TCancelDelegationTokenRespStandardScheme extends StandardScheme<TCancelDelegationTokenResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TCancelDelegationTokenResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TCancelDelegationTokenResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TCancelDelegationTokenRespTupleSchemeFactory implements SchemeFactory {
-    public TCancelDelegationTokenRespTupleScheme getScheme() {
-      return new TCancelDelegationTokenRespTupleScheme();
-    }
-  }
-
-  private static class TCancelDelegationTokenRespTupleScheme extends TupleScheme<TCancelDelegationTokenResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TCancelDelegationTokenResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TCancelDelegationTokenResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCancelOperationReq.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCancelOperationReq.java
deleted file mode 100644
index 4076c573fafb7..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCancelOperationReq.java
+++ /dev/null
@@ -1,394 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TCancelOperationReq implements org.apache.thrift.TBase<TCancelOperationReq, TCancelOperationReq._Fields>, java.io.Serializable, Cloneable, Comparable<TCancelOperationReq> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TCancelOperationReq");
-
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TCancelOperationReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TCancelOperationReqTupleSchemeFactory());
-  }
-
-  private TOperationHandle operationHandle; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    OPERATION_HANDLE((short)1, "operationHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TCancelOperationReq.class, metaDataMap);
-  }
-
-  public TCancelOperationReq() {
-  }
-
-  public TCancelOperationReq(
-    TOperationHandle operationHandle)
-  {
-    this();
-    this.operationHandle = operationHandle;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TCancelOperationReq(TCancelOperationReq other) {
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-  }
-
-  public TCancelOperationReq deepCopy() {
-    return new TCancelOperationReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.operationHandle = null;
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TCancelOperationReq)
-      return this.equals((TCancelOperationReq)that);
-    return false;
-  }
-
-  public boolean equals(TCancelOperationReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    list.add(present_operationHandle);
-    if (present_operationHandle)
-      list.add(operationHandle);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TCancelOperationReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(other.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, other.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TCancelOperationReq(");
-    boolean first = true;
-
-    sb.append("operationHandle:");
-    if (this.operationHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.operationHandle);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetOperationHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'operationHandle' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TCancelOperationReqStandardSchemeFactory implements SchemeFactory {
-    public TCancelOperationReqStandardScheme getScheme() {
-      return new TCancelOperationReqStandardScheme();
-    }
-  }
-
-  private static class TCancelOperationReqStandardScheme extends StandardScheme<TCancelOperationReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TCancelOperationReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TCancelOperationReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.operationHandle != null) {
-        oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-        struct.operationHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TCancelOperationReqTupleSchemeFactory implements SchemeFactory {
-    public TCancelOperationReqTupleScheme getScheme() {
-      return new TCancelOperationReqTupleScheme();
-    }
-  }
-
-  private static class TCancelOperationReqTupleScheme extends TupleScheme<TCancelOperationReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TCancelOperationReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.operationHandle.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TCancelOperationReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.operationHandle = new TOperationHandle();
-      struct.operationHandle.read(iprot);
-      struct.setOperationHandleIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCancelOperationResp.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCancelOperationResp.java
deleted file mode 100644
index 7bcc765c85daa..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCancelOperationResp.java
+++ /dev/null
@@ -1,394 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TCancelOperationResp implements org.apache.thrift.TBase<TCancelOperationResp, TCancelOperationResp._Fields>, java.io.Serializable, Cloneable, Comparable<TCancelOperationResp> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TCancelOperationResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TCancelOperationRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TCancelOperationRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TCancelOperationResp.class, metaDataMap);
-  }
-
-  public TCancelOperationResp() {
-  }
-
-  public TCancelOperationResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TCancelOperationResp(TCancelOperationResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-  }
-
-  public TCancelOperationResp deepCopy() {
-    return new TCancelOperationResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TCancelOperationResp)
-      return this.equals((TCancelOperationResp)that);
-    return false;
-  }
-
-  public boolean equals(TCancelOperationResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_status = true && (isSetStatus());
-    list.add(present_status);
-    if (present_status)
-      list.add(status);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TCancelOperationResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(other.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, other.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TCancelOperationResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TCancelOperationRespStandardSchemeFactory implements SchemeFactory {
-    public TCancelOperationRespStandardScheme getScheme() {
-      return new TCancelOperationRespStandardScheme();
-    }
-  }
-
-  private static class TCancelOperationRespStandardScheme extends StandardScheme<TCancelOperationResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TCancelOperationResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TCancelOperationResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TCancelOperationRespTupleSchemeFactory implements SchemeFactory {
-    public TCancelOperationRespTupleScheme getScheme() {
-      return new TCancelOperationRespTupleScheme();
-    }
-  }
-
-  private static class TCancelOperationRespTupleScheme extends TupleScheme<TCancelOperationResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TCancelOperationResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TCancelOperationResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCloseOperationReq.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCloseOperationReq.java
deleted file mode 100644
index 47a6b8329c05b..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCloseOperationReq.java
+++ /dev/null
@@ -1,394 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TCloseOperationReq implements org.apache.thrift.TBase<TCloseOperationReq, TCloseOperationReq._Fields>, java.io.Serializable, Cloneable, Comparable<TCloseOperationReq> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TCloseOperationReq");
-
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TCloseOperationReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TCloseOperationReqTupleSchemeFactory());
-  }
-
-  private TOperationHandle operationHandle; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    OPERATION_HANDLE((short)1, "operationHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TCloseOperationReq.class, metaDataMap);
-  }
-
-  public TCloseOperationReq() {
-  }
-
-  public TCloseOperationReq(
-    TOperationHandle operationHandle)
-  {
-    this();
-    this.operationHandle = operationHandle;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TCloseOperationReq(TCloseOperationReq other) {
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-  }
-
-  public TCloseOperationReq deepCopy() {
-    return new TCloseOperationReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.operationHandle = null;
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TCloseOperationReq)
-      return this.equals((TCloseOperationReq)that);
-    return false;
-  }
-
-  public boolean equals(TCloseOperationReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    list.add(present_operationHandle);
-    if (present_operationHandle)
-      list.add(operationHandle);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TCloseOperationReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(other.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, other.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TCloseOperationReq(");
-    boolean first = true;
-
-    sb.append("operationHandle:");
-    if (this.operationHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.operationHandle);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetOperationHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'operationHandle' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TCloseOperationReqStandardSchemeFactory implements SchemeFactory {
-    public TCloseOperationReqStandardScheme getScheme() {
-      return new TCloseOperationReqStandardScheme();
-    }
-  }
-
-  private static class TCloseOperationReqStandardScheme extends StandardScheme<TCloseOperationReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TCloseOperationReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TCloseOperationReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.operationHandle != null) {
-        oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-        struct.operationHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TCloseOperationReqTupleSchemeFactory implements SchemeFactory {
-    public TCloseOperationReqTupleScheme getScheme() {
-      return new TCloseOperationReqTupleScheme();
-    }
-  }
-
-  private static class TCloseOperationReqTupleScheme extends TupleScheme<TCloseOperationReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TCloseOperationReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.operationHandle.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TCloseOperationReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.operationHandle = new TOperationHandle();
-      struct.operationHandle.read(iprot);
-      struct.setOperationHandleIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCloseOperationResp.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCloseOperationResp.java
deleted file mode 100644
index 0860a2b1c5bac..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCloseOperationResp.java
+++ /dev/null
@@ -1,394 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TCloseOperationResp implements org.apache.thrift.TBase<TCloseOperationResp, TCloseOperationResp._Fields>, java.io.Serializable, Cloneable, Comparable<TCloseOperationResp> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TCloseOperationResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TCloseOperationRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TCloseOperationRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TCloseOperationResp.class, metaDataMap);
-  }
-
-  public TCloseOperationResp() {
-  }
-
-  public TCloseOperationResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TCloseOperationResp(TCloseOperationResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-  }
-
-  public TCloseOperationResp deepCopy() {
-    return new TCloseOperationResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TCloseOperationResp)
-      return this.equals((TCloseOperationResp)that);
-    return false;
-  }
-
-  public boolean equals(TCloseOperationResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_status = true && (isSetStatus());
-    list.add(present_status);
-    if (present_status)
-      list.add(status);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TCloseOperationResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(other.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, other.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TCloseOperationResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TCloseOperationRespStandardSchemeFactory implements SchemeFactory {
-    public TCloseOperationRespStandardScheme getScheme() {
-      return new TCloseOperationRespStandardScheme();
-    }
-  }
-
-  private static class TCloseOperationRespStandardScheme extends StandardScheme<TCloseOperationResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TCloseOperationResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TCloseOperationResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TCloseOperationRespTupleSchemeFactory implements SchemeFactory {
-    public TCloseOperationRespTupleScheme getScheme() {
-      return new TCloseOperationRespTupleScheme();
-    }
-  }
-
-  private static class TCloseOperationRespTupleScheme extends TupleScheme<TCloseOperationResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TCloseOperationResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TCloseOperationResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCloseSessionReq.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCloseSessionReq.java
deleted file mode 100644
index 43ee87f487a67..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCloseSessionReq.java
+++ /dev/null
@@ -1,394 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TCloseSessionReq implements org.apache.thrift.TBase<TCloseSessionReq, TCloseSessionReq._Fields>, java.io.Serializable, Cloneable, Comparable<TCloseSessionReq> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TCloseSessionReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TCloseSessionReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TCloseSessionReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TCloseSessionReq.class, metaDataMap);
-  }
-
-  public TCloseSessionReq() {
-  }
-
-  public TCloseSessionReq(
-    TSessionHandle sessionHandle)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TCloseSessionReq(TCloseSessionReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-  }
-
-  public TCloseSessionReq deepCopy() {
-    return new TCloseSessionReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TCloseSessionReq)
-      return this.equals((TCloseSessionReq)that);
-    return false;
-  }
-
-  public boolean equals(TCloseSessionReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    list.add(present_sessionHandle);
-    if (present_sessionHandle)
-      list.add(sessionHandle);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TCloseSessionReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(other.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, other.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TCloseSessionReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TCloseSessionReqStandardSchemeFactory implements SchemeFactory {
-    public TCloseSessionReqStandardScheme getScheme() {
-      return new TCloseSessionReqStandardScheme();
-    }
-  }
-
-  private static class TCloseSessionReqStandardScheme extends StandardScheme<TCloseSessionReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TCloseSessionReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TCloseSessionReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TCloseSessionReqTupleSchemeFactory implements SchemeFactory {
-    public TCloseSessionReqTupleScheme getScheme() {
-      return new TCloseSessionReqTupleScheme();
-    }
-  }
-
-  private static class TCloseSessionReqTupleScheme extends TupleScheme<TCloseSessionReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TCloseSessionReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TCloseSessionReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCloseSessionResp.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCloseSessionResp.java
deleted file mode 100644
index 38f82ac8d3cd2..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCloseSessionResp.java
+++ /dev/null
@@ -1,394 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TCloseSessionResp implements org.apache.thrift.TBase<TCloseSessionResp, TCloseSessionResp._Fields>, java.io.Serializable, Cloneable, Comparable<TCloseSessionResp> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TCloseSessionResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TCloseSessionRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TCloseSessionRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TCloseSessionResp.class, metaDataMap);
-  }
-
-  public TCloseSessionResp() {
-  }
-
-  public TCloseSessionResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TCloseSessionResp(TCloseSessionResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-  }
-
-  public TCloseSessionResp deepCopy() {
-    return new TCloseSessionResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TCloseSessionResp)
-      return this.equals((TCloseSessionResp)that);
-    return false;
-  }
-
-  public boolean equals(TCloseSessionResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_status = true && (isSetStatus());
-    list.add(present_status);
-    if (present_status)
-      list.add(status);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TCloseSessionResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(other.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, other.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TCloseSessionResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TCloseSessionRespStandardSchemeFactory implements SchemeFactory {
-    public TCloseSessionRespStandardScheme getScheme() {
-      return new TCloseSessionRespStandardScheme();
-    }
-  }
-
-  private static class TCloseSessionRespStandardScheme extends StandardScheme<TCloseSessionResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TCloseSessionResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TCloseSessionResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TCloseSessionRespTupleSchemeFactory implements SchemeFactory {
-    public TCloseSessionRespTupleScheme getScheme() {
-      return new TCloseSessionRespTupleScheme();
-    }
-  }
-
-  private static class TCloseSessionRespTupleScheme extends TupleScheme<TCloseSessionResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TCloseSessionResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TCloseSessionResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TColumn.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TColumn.java
deleted file mode 100644
index dd79482200961..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TColumn.java
+++ /dev/null
@@ -1,736 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-public class TColumn extends org.apache.thrift.TUnion<TColumn, TColumn._Fields> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TColumn");
-  private static final org.apache.thrift.protocol.TField BOOL_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("boolVal", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField BYTE_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("byteVal", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-  private static final org.apache.thrift.protocol.TField I16_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("i16Val", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-  private static final org.apache.thrift.protocol.TField I32_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("i32Val", org.apache.thrift.protocol.TType.STRUCT, (short)4);
-  private static final org.apache.thrift.protocol.TField I64_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("i64Val", org.apache.thrift.protocol.TType.STRUCT, (short)5);
-  private static final org.apache.thrift.protocol.TField DOUBLE_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("doubleVal", org.apache.thrift.protocol.TType.STRUCT, (short)6);
-  private static final org.apache.thrift.protocol.TField STRING_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("stringVal", org.apache.thrift.protocol.TType.STRUCT, (short)7);
-  private static final org.apache.thrift.protocol.TField BINARY_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("binaryVal", org.apache.thrift.protocol.TType.STRUCT, (short)8);
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    BOOL_VAL((short)1, "boolVal"),
-    BYTE_VAL((short)2, "byteVal"),
-    I16_VAL((short)3, "i16Val"),
-    I32_VAL((short)4, "i32Val"),
-    I64_VAL((short)5, "i64Val"),
-    DOUBLE_VAL((short)6, "doubleVal"),
-    STRING_VAL((short)7, "stringVal"),
-    BINARY_VAL((short)8, "binaryVal");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // BOOL_VAL
-          return BOOL_VAL;
-        case 2: // BYTE_VAL
-          return BYTE_VAL;
-        case 3: // I16_VAL
-          return I16_VAL;
-        case 4: // I32_VAL
-          return I32_VAL;
-        case 5: // I64_VAL
-          return I64_VAL;
-        case 6: // DOUBLE_VAL
-          return DOUBLE_VAL;
-        case 7: // STRING_VAL
-          return STRING_VAL;
-        case 8: // BINARY_VAL
-          return BINARY_VAL;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.BOOL_VAL, new org.apache.thrift.meta_data.FieldMetaData("boolVal", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TBoolColumn.class)));
-    tmpMap.put(_Fields.BYTE_VAL, new org.apache.thrift.meta_data.FieldMetaData("byteVal", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TByteColumn.class)));
-    tmpMap.put(_Fields.I16_VAL, new org.apache.thrift.meta_data.FieldMetaData("i16Val", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TI16Column.class)));
-    tmpMap.put(_Fields.I32_VAL, new org.apache.thrift.meta_data.FieldMetaData("i32Val", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TI32Column.class)));
-    tmpMap.put(_Fields.I64_VAL, new org.apache.thrift.meta_data.FieldMetaData("i64Val", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TI64Column.class)));
-    tmpMap.put(_Fields.DOUBLE_VAL, new org.apache.thrift.meta_data.FieldMetaData("doubleVal", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TDoubleColumn.class)));
-    tmpMap.put(_Fields.STRING_VAL, new org.apache.thrift.meta_data.FieldMetaData("stringVal", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStringColumn.class)));
-    tmpMap.put(_Fields.BINARY_VAL, new org.apache.thrift.meta_data.FieldMetaData("binaryVal", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TBinaryColumn.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TColumn.class, metaDataMap);
-  }
-
-  public TColumn() {
-    super();
-  }
-
-  public TColumn(TColumn._Fields setField, Object value) {
-    super(setField, value);
-  }
-
-  public TColumn(TColumn other) {
-    super(other);
-  }
-  public TColumn deepCopy() {
-    return new TColumn(this);
-  }
-
-  public static TColumn boolVal(TBoolColumn value) {
-    TColumn x = new TColumn();
-    x.setBoolVal(value);
-    return x;
-  }
-
-  public static TColumn byteVal(TByteColumn value) {
-    TColumn x = new TColumn();
-    x.setByteVal(value);
-    return x;
-  }
-
-  public static TColumn i16Val(TI16Column value) {
-    TColumn x = new TColumn();
-    x.setI16Val(value);
-    return x;
-  }
-
-  public static TColumn i32Val(TI32Column value) {
-    TColumn x = new TColumn();
-    x.setI32Val(value);
-    return x;
-  }
-
-  public static TColumn i64Val(TI64Column value) {
-    TColumn x = new TColumn();
-    x.setI64Val(value);
-    return x;
-  }
-
-  public static TColumn doubleVal(TDoubleColumn value) {
-    TColumn x = new TColumn();
-    x.setDoubleVal(value);
-    return x;
-  }
-
-  public static TColumn stringVal(TStringColumn value) {
-    TColumn x = new TColumn();
-    x.setStringVal(value);
-    return x;
-  }
-
-  public static TColumn binaryVal(TBinaryColumn value) {
-    TColumn x = new TColumn();
-    x.setBinaryVal(value);
-    return x;
-  }
-
-
-  @Override
-  protected void checkType(_Fields setField, Object value) throws ClassCastException {
-    switch (setField) {
-      case BOOL_VAL:
-        if (value instanceof TBoolColumn) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TBoolColumn for field 'boolVal', but got " + value.getClass().getSimpleName());
-      case BYTE_VAL:
-        if (value instanceof TByteColumn) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TByteColumn for field 'byteVal', but got " + value.getClass().getSimpleName());
-      case I16_VAL:
-        if (value instanceof TI16Column) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TI16Column for field 'i16Val', but got " + value.getClass().getSimpleName());
-      case I32_VAL:
-        if (value instanceof TI32Column) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TI32Column for field 'i32Val', but got " + value.getClass().getSimpleName());
-      case I64_VAL:
-        if (value instanceof TI64Column) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TI64Column for field 'i64Val', but got " + value.getClass().getSimpleName());
-      case DOUBLE_VAL:
-        if (value instanceof TDoubleColumn) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TDoubleColumn for field 'doubleVal', but got " + value.getClass().getSimpleName());
-      case STRING_VAL:
-        if (value instanceof TStringColumn) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TStringColumn for field 'stringVal', but got " + value.getClass().getSimpleName());
-      case BINARY_VAL:
-        if (value instanceof TBinaryColumn) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TBinaryColumn for field 'binaryVal', but got " + value.getClass().getSimpleName());
-      default:
-        throw new IllegalArgumentException("Unknown field id " + setField);
-    }
-  }
-
-  @Override
-  protected Object standardSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TField field) throws org.apache.thrift.TException {
-    _Fields setField = _Fields.findByThriftId(field.id);
-    if (setField != null) {
-      switch (setField) {
-        case BOOL_VAL:
-          if (field.type == BOOL_VAL_FIELD_DESC.type) {
-            TBoolColumn boolVal;
-            boolVal = new TBoolColumn();
-            boolVal.read(iprot);
-            return boolVal;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case BYTE_VAL:
-          if (field.type == BYTE_VAL_FIELD_DESC.type) {
-            TByteColumn byteVal;
-            byteVal = new TByteColumn();
-            byteVal.read(iprot);
-            return byteVal;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case I16_VAL:
-          if (field.type == I16_VAL_FIELD_DESC.type) {
-            TI16Column i16Val;
-            i16Val = new TI16Column();
-            i16Val.read(iprot);
-            return i16Val;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case I32_VAL:
-          if (field.type == I32_VAL_FIELD_DESC.type) {
-            TI32Column i32Val;
-            i32Val = new TI32Column();
-            i32Val.read(iprot);
-            return i32Val;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case I64_VAL:
-          if (field.type == I64_VAL_FIELD_DESC.type) {
-            TI64Column i64Val;
-            i64Val = new TI64Column();
-            i64Val.read(iprot);
-            return i64Val;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case DOUBLE_VAL:
-          if (field.type == DOUBLE_VAL_FIELD_DESC.type) {
-            TDoubleColumn doubleVal;
-            doubleVal = new TDoubleColumn();
-            doubleVal.read(iprot);
-            return doubleVal;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case STRING_VAL:
-          if (field.type == STRING_VAL_FIELD_DESC.type) {
-            TStringColumn stringVal;
-            stringVal = new TStringColumn();
-            stringVal.read(iprot);
-            return stringVal;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case BINARY_VAL:
-          if (field.type == BINARY_VAL_FIELD_DESC.type) {
-            TBinaryColumn binaryVal;
-            binaryVal = new TBinaryColumn();
-            binaryVal.read(iprot);
-            return binaryVal;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        default:
-          throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!");
-      }
-    } else {
-      org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-      return null;
-    }
-  }
-
-  @Override
-  protected void standardSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    switch (setField_) {
-      case BOOL_VAL:
-        TBoolColumn boolVal = (TBoolColumn)value_;
-        boolVal.write(oprot);
-        return;
-      case BYTE_VAL:
-        TByteColumn byteVal = (TByteColumn)value_;
-        byteVal.write(oprot);
-        return;
-      case I16_VAL:
-        TI16Column i16Val = (TI16Column)value_;
-        i16Val.write(oprot);
-        return;
-      case I32_VAL:
-        TI32Column i32Val = (TI32Column)value_;
-        i32Val.write(oprot);
-        return;
-      case I64_VAL:
-        TI64Column i64Val = (TI64Column)value_;
-        i64Val.write(oprot);
-        return;
-      case DOUBLE_VAL:
-        TDoubleColumn doubleVal = (TDoubleColumn)value_;
-        doubleVal.write(oprot);
-        return;
-      case STRING_VAL:
-        TStringColumn stringVal = (TStringColumn)value_;
-        stringVal.write(oprot);
-        return;
-      case BINARY_VAL:
-        TBinaryColumn binaryVal = (TBinaryColumn)value_;
-        binaryVal.write(oprot);
-        return;
-      default:
-        throw new IllegalStateException("Cannot write union with unknown field " + setField_);
-    }
-  }
-
-  @Override
-  protected Object tupleSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, short fieldID) throws org.apache.thrift.TException {
-    _Fields setField = _Fields.findByThriftId(fieldID);
-    if (setField != null) {
-      switch (setField) {
-        case BOOL_VAL:
-          TBoolColumn boolVal;
-          boolVal = new TBoolColumn();
-          boolVal.read(iprot);
-          return boolVal;
-        case BYTE_VAL:
-          TByteColumn byteVal;
-          byteVal = new TByteColumn();
-          byteVal.read(iprot);
-          return byteVal;
-        case I16_VAL:
-          TI16Column i16Val;
-          i16Val = new TI16Column();
-          i16Val.read(iprot);
-          return i16Val;
-        case I32_VAL:
-          TI32Column i32Val;
-          i32Val = new TI32Column();
-          i32Val.read(iprot);
-          return i32Val;
-        case I64_VAL:
-          TI64Column i64Val;
-          i64Val = new TI64Column();
-          i64Val.read(iprot);
-          return i64Val;
-        case DOUBLE_VAL:
-          TDoubleColumn doubleVal;
-          doubleVal = new TDoubleColumn();
-          doubleVal.read(iprot);
-          return doubleVal;
-        case STRING_VAL:
-          TStringColumn stringVal;
-          stringVal = new TStringColumn();
-          stringVal.read(iprot);
-          return stringVal;
-        case BINARY_VAL:
-          TBinaryColumn binaryVal;
-          binaryVal = new TBinaryColumn();
-          binaryVal.read(iprot);
-          return binaryVal;
-        default:
-          throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!");
-      }
-    } else {
-      throw new TProtocolException("Couldn't find a field with field id " + fieldID);
-    }
-  }
-
-  @Override
-  protected void tupleSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    switch (setField_) {
-      case BOOL_VAL:
-        TBoolColumn boolVal = (TBoolColumn)value_;
-        boolVal.write(oprot);
-        return;
-      case BYTE_VAL:
-        TByteColumn byteVal = (TByteColumn)value_;
-        byteVal.write(oprot);
-        return;
-      case I16_VAL:
-        TI16Column i16Val = (TI16Column)value_;
-        i16Val.write(oprot);
-        return;
-      case I32_VAL:
-        TI32Column i32Val = (TI32Column)value_;
-        i32Val.write(oprot);
-        return;
-      case I64_VAL:
-        TI64Column i64Val = (TI64Column)value_;
-        i64Val.write(oprot);
-        return;
-      case DOUBLE_VAL:
-        TDoubleColumn doubleVal = (TDoubleColumn)value_;
-        doubleVal.write(oprot);
-        return;
-      case STRING_VAL:
-        TStringColumn stringVal = (TStringColumn)value_;
-        stringVal.write(oprot);
-        return;
-      case BINARY_VAL:
-        TBinaryColumn binaryVal = (TBinaryColumn)value_;
-        binaryVal.write(oprot);
-        return;
-      default:
-        throw new IllegalStateException("Cannot write union with unknown field " + setField_);
-    }
-  }
-
-  @Override
-  protected org.apache.thrift.protocol.TField getFieldDesc(_Fields setField) {
-    switch (setField) {
-      case BOOL_VAL:
-        return BOOL_VAL_FIELD_DESC;
-      case BYTE_VAL:
-        return BYTE_VAL_FIELD_DESC;
-      case I16_VAL:
-        return I16_VAL_FIELD_DESC;
-      case I32_VAL:
-        return I32_VAL_FIELD_DESC;
-      case I64_VAL:
-        return I64_VAL_FIELD_DESC;
-      case DOUBLE_VAL:
-        return DOUBLE_VAL_FIELD_DESC;
-      case STRING_VAL:
-        return STRING_VAL_FIELD_DESC;
-      case BINARY_VAL:
-        return BINARY_VAL_FIELD_DESC;
-      default:
-        throw new IllegalArgumentException("Unknown field id " + setField);
-    }
-  }
-
-  @Override
-  protected org.apache.thrift.protocol.TStruct getStructDesc() {
-    return STRUCT_DESC;
-  }
-
-  @Override
-  protected _Fields enumForId(short id) {
-    return _Fields.findByThriftIdOrThrow(id);
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-
-  public TBoolColumn getBoolVal() {
-    if (getSetField() == _Fields.BOOL_VAL) {
-      return (TBoolColumn)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'boolVal' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setBoolVal(TBoolColumn value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.BOOL_VAL;
-    value_ = value;
-  }
-
-  public TByteColumn getByteVal() {
-    if (getSetField() == _Fields.BYTE_VAL) {
-      return (TByteColumn)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'byteVal' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setByteVal(TByteColumn value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.BYTE_VAL;
-    value_ = value;
-  }
-
-  public TI16Column getI16Val() {
-    if (getSetField() == _Fields.I16_VAL) {
-      return (TI16Column)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'i16Val' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setI16Val(TI16Column value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.I16_VAL;
-    value_ = value;
-  }
-
-  public TI32Column getI32Val() {
-    if (getSetField() == _Fields.I32_VAL) {
-      return (TI32Column)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'i32Val' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setI32Val(TI32Column value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.I32_VAL;
-    value_ = value;
-  }
-
-  public TI64Column getI64Val() {
-    if (getSetField() == _Fields.I64_VAL) {
-      return (TI64Column)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'i64Val' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setI64Val(TI64Column value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.I64_VAL;
-    value_ = value;
-  }
-
-  public TDoubleColumn getDoubleVal() {
-    if (getSetField() == _Fields.DOUBLE_VAL) {
-      return (TDoubleColumn)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'doubleVal' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setDoubleVal(TDoubleColumn value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.DOUBLE_VAL;
-    value_ = value;
-  }
-
-  public TStringColumn getStringVal() {
-    if (getSetField() == _Fields.STRING_VAL) {
-      return (TStringColumn)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'stringVal' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setStringVal(TStringColumn value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.STRING_VAL;
-    value_ = value;
-  }
-
-  public TBinaryColumn getBinaryVal() {
-    if (getSetField() == _Fields.BINARY_VAL) {
-      return (TBinaryColumn)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'binaryVal' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setBinaryVal(TBinaryColumn value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.BINARY_VAL;
-    value_ = value;
-  }
-
-  public boolean isSetBoolVal() {
-    return setField_ == _Fields.BOOL_VAL;
-  }
-
-
-  public boolean isSetByteVal() {
-    return setField_ == _Fields.BYTE_VAL;
-  }
-
-
-  public boolean isSetI16Val() {
-    return setField_ == _Fields.I16_VAL;
-  }
-
-
-  public boolean isSetI32Val() {
-    return setField_ == _Fields.I32_VAL;
-  }
-
-
-  public boolean isSetI64Val() {
-    return setField_ == _Fields.I64_VAL;
-  }
-
-
-  public boolean isSetDoubleVal() {
-    return setField_ == _Fields.DOUBLE_VAL;
-  }
-
-
-  public boolean isSetStringVal() {
-    return setField_ == _Fields.STRING_VAL;
-  }
-
-
-  public boolean isSetBinaryVal() {
-    return setField_ == _Fields.BINARY_VAL;
-  }
-
-
-  public boolean equals(Object other) {
-    if (other instanceof TColumn) {
-      return equals((TColumn)other);
-    } else {
-      return false;
-    }
-  }
-
-  public boolean equals(TColumn other) {
-    return other != null && getSetField() == other.getSetField() && getFieldValue().equals(other.getFieldValue());
-  }
-
-  @Override
-  public int compareTo(TColumn other) {
-    int lastComparison = org.apache.thrift.TBaseHelper.compareTo(getSetField(), other.getSetField());
-    if (lastComparison == 0) {
-      return org.apache.thrift.TBaseHelper.compareTo(getFieldValue(), other.getFieldValue());
-    }
-    return lastComparison;
-  }
-
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-    list.add(this.getClass().getName());
-    org.apache.thrift.TFieldIdEnum setField = getSetField();
-    if (setField != null) {
-      list.add(setField.getThriftFieldId());
-      Object value = getFieldValue();
-      if (value instanceof org.apache.thrift.TEnum) {
-        list.add(((org.apache.thrift.TEnum)getFieldValue()).getValue());
-      } else {
-        list.add(value);
-      }
-    }
-    return list.hashCode();
-  }
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-
-}
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TColumnDesc.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TColumnDesc.java
deleted file mode 100644
index 31472c8f54b94..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TColumnDesc.java
+++ /dev/null
@@ -1,704 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TColumnDesc implements org.apache.thrift.TBase<TColumnDesc, TColumnDesc._Fields>, java.io.Serializable, Cloneable, Comparable<TColumnDesc> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TColumnDesc");
-
-  private static final org.apache.thrift.protocol.TField COLUMN_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("columnName", org.apache.thrift.protocol.TType.STRING, (short)1);
-  private static final org.apache.thrift.protocol.TField TYPE_DESC_FIELD_DESC = new org.apache.thrift.protocol.TField("typeDesc", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-  private static final org.apache.thrift.protocol.TField POSITION_FIELD_DESC = new org.apache.thrift.protocol.TField("position", org.apache.thrift.protocol.TType.I32, (short)3);
-  private static final org.apache.thrift.protocol.TField COMMENT_FIELD_DESC = new org.apache.thrift.protocol.TField("comment", org.apache.thrift.protocol.TType.STRING, (short)4);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TColumnDescStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TColumnDescTupleSchemeFactory());
-  }
-
-  private String columnName; // required
-  private TTypeDesc typeDesc; // required
-  private int position; // required
-  private String comment; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    COLUMN_NAME((short)1, "columnName"),
-    TYPE_DESC((short)2, "typeDesc"),
-    POSITION((short)3, "position"),
-    COMMENT((short)4, "comment");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // COLUMN_NAME
-          return COLUMN_NAME;
-        case 2: // TYPE_DESC
-          return TYPE_DESC;
-        case 3: // POSITION
-          return POSITION;
-        case 4: // COMMENT
-          return COMMENT;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __POSITION_ISSET_ID = 0;
-  private byte __isset_bitfield = 0;
-  private static final _Fields optionals[] = {_Fields.COMMENT};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.COLUMN_NAME, new org.apache.thrift.meta_data.FieldMetaData("columnName", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    tmpMap.put(_Fields.TYPE_DESC, new org.apache.thrift.meta_data.FieldMetaData("typeDesc", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TTypeDesc.class)));
-    tmpMap.put(_Fields.POSITION, new org.apache.thrift.meta_data.FieldMetaData("position", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
-    tmpMap.put(_Fields.COMMENT, new org.apache.thrift.meta_data.FieldMetaData("comment", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TColumnDesc.class, metaDataMap);
-  }
-
-  public TColumnDesc() {
-  }
-
-  public TColumnDesc(
-    String columnName,
-    TTypeDesc typeDesc,
-    int position)
-  {
-    this();
-    this.columnName = columnName;
-    this.typeDesc = typeDesc;
-    this.position = position;
-    setPositionIsSet(true);
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TColumnDesc(TColumnDesc other) {
-    __isset_bitfield = other.__isset_bitfield;
-    if (other.isSetColumnName()) {
-      this.columnName = other.columnName;
-    }
-    if (other.isSetTypeDesc()) {
-      this.typeDesc = new TTypeDesc(other.typeDesc);
-    }
-    this.position = other.position;
-    if (other.isSetComment()) {
-      this.comment = other.comment;
-    }
-  }
-
-  public TColumnDesc deepCopy() {
-    return new TColumnDesc(this);
-  }
-
-  @Override
-  public void clear() {
-    this.columnName = null;
-    this.typeDesc = null;
-    setPositionIsSet(false);
-    this.position = 0;
-    this.comment = null;
-  }
-
-  public String getColumnName() {
-    return this.columnName;
-  }
-
-  public void setColumnName(String columnName) {
-    this.columnName = columnName;
-  }
-
-  public void unsetColumnName() {
-    this.columnName = null;
-  }
-
-  /** Returns true if field columnName is set (has been assigned a value) and false otherwise */
-  public boolean isSetColumnName() {
-    return this.columnName != null;
-  }
-
-  public void setColumnNameIsSet(boolean value) {
-    if (!value) {
-      this.columnName = null;
-    }
-  }
-
-  public TTypeDesc getTypeDesc() {
-    return this.typeDesc;
-  }
-
-  public void setTypeDesc(TTypeDesc typeDesc) {
-    this.typeDesc = typeDesc;
-  }
-
-  public void unsetTypeDesc() {
-    this.typeDesc = null;
-  }
-
-  /** Returns true if field typeDesc is set (has been assigned a value) and false otherwise */
-  public boolean isSetTypeDesc() {
-    return this.typeDesc != null;
-  }
-
-  public void setTypeDescIsSet(boolean value) {
-    if (!value) {
-      this.typeDesc = null;
-    }
-  }
-
-  public int getPosition() {
-    return this.position;
-  }
-
-  public void setPosition(int position) {
-    this.position = position;
-    setPositionIsSet(true);
-  }
-
-  public void unsetPosition() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __POSITION_ISSET_ID);
-  }
-
-  /** Returns true if field position is set (has been assigned a value) and false otherwise */
-  public boolean isSetPosition() {
-    return EncodingUtils.testBit(__isset_bitfield, __POSITION_ISSET_ID);
-  }
-
-  public void setPositionIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __POSITION_ISSET_ID, value);
-  }
-
-  public String getComment() {
-    return this.comment;
-  }
-
-  public void setComment(String comment) {
-    this.comment = comment;
-  }
-
-  public void unsetComment() {
-    this.comment = null;
-  }
-
-  /** Returns true if field comment is set (has been assigned a value) and false otherwise */
-  public boolean isSetComment() {
-    return this.comment != null;
-  }
-
-  public void setCommentIsSet(boolean value) {
-    if (!value) {
-      this.comment = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case COLUMN_NAME:
-      if (value == null) {
-        unsetColumnName();
-      } else {
-        setColumnName((String)value);
-      }
-      break;
-
-    case TYPE_DESC:
-      if (value == null) {
-        unsetTypeDesc();
-      } else {
-        setTypeDesc((TTypeDesc)value);
-      }
-      break;
-
-    case POSITION:
-      if (value == null) {
-        unsetPosition();
-      } else {
-        setPosition((Integer)value);
-      }
-      break;
-
-    case COMMENT:
-      if (value == null) {
-        unsetComment();
-      } else {
-        setComment((String)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case COLUMN_NAME:
-      return getColumnName();
-
-    case TYPE_DESC:
-      return getTypeDesc();
-
-    case POSITION:
-      return getPosition();
-
-    case COMMENT:
-      return getComment();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case COLUMN_NAME:
-      return isSetColumnName();
-    case TYPE_DESC:
-      return isSetTypeDesc();
-    case POSITION:
-      return isSetPosition();
-    case COMMENT:
-      return isSetComment();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TColumnDesc)
-      return this.equals((TColumnDesc)that);
-    return false;
-  }
-
-  public boolean equals(TColumnDesc that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_columnName = true && this.isSetColumnName();
-    boolean that_present_columnName = true && that.isSetColumnName();
-    if (this_present_columnName || that_present_columnName) {
-      if (!(this_present_columnName && that_present_columnName))
-        return false;
-      if (!this.columnName.equals(that.columnName))
-        return false;
-    }
-
-    boolean this_present_typeDesc = true && this.isSetTypeDesc();
-    boolean that_present_typeDesc = true && that.isSetTypeDesc();
-    if (this_present_typeDesc || that_present_typeDesc) {
-      if (!(this_present_typeDesc && that_present_typeDesc))
-        return false;
-      if (!this.typeDesc.equals(that.typeDesc))
-        return false;
-    }
-
-    boolean this_present_position = true;
-    boolean that_present_position = true;
-    if (this_present_position || that_present_position) {
-      if (!(this_present_position && that_present_position))
-        return false;
-      if (this.position != that.position)
-        return false;
-    }
-
-    boolean this_present_comment = true && this.isSetComment();
-    boolean that_present_comment = true && that.isSetComment();
-    if (this_present_comment || that_present_comment) {
-      if (!(this_present_comment && that_present_comment))
-        return false;
-      if (!this.comment.equals(that.comment))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_columnName = true && (isSetColumnName());
-    list.add(present_columnName);
-    if (present_columnName)
-      list.add(columnName);
-
-    boolean present_typeDesc = true && (isSetTypeDesc());
-    list.add(present_typeDesc);
-    if (present_typeDesc)
-      list.add(typeDesc);
-
-    boolean present_position = true;
-    list.add(present_position);
-    if (present_position)
-      list.add(position);
-
-    boolean present_comment = true && (isSetComment());
-    list.add(present_comment);
-    if (present_comment)
-      list.add(comment);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TColumnDesc other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetColumnName()).compareTo(other.isSetColumnName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetColumnName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columnName, other.columnName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetTypeDesc()).compareTo(other.isSetTypeDesc());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetTypeDesc()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.typeDesc, other.typeDesc);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetPosition()).compareTo(other.isSetPosition());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetPosition()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.position, other.position);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetComment()).compareTo(other.isSetComment());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetComment()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.comment, other.comment);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TColumnDesc(");
-    boolean first = true;
-
-    sb.append("columnName:");
-    if (this.columnName == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.columnName);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("typeDesc:");
-    if (this.typeDesc == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.typeDesc);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("position:");
-    sb.append(this.position);
-    first = false;
-    if (isSetComment()) {
-      if (!first) sb.append(", ");
-      sb.append("comment:");
-      if (this.comment == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.comment);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetColumnName()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'columnName' is unset! Struct:" + toString());
-    }
-
-    if (!isSetTypeDesc()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'typeDesc' is unset! Struct:" + toString());
-    }
-
-    if (!isSetPosition()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'position' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (typeDesc != null) {
-      typeDesc.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TColumnDescStandardSchemeFactory implements SchemeFactory {
-    public TColumnDescStandardScheme getScheme() {
-      return new TColumnDescStandardScheme();
-    }
-  }
-
-  private static class TColumnDescStandardScheme extends StandardScheme<TColumnDesc> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TColumnDesc struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // COLUMN_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.columnName = iprot.readString();
-              struct.setColumnNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // TYPE_DESC
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.typeDesc = new TTypeDesc();
-              struct.typeDesc.read(iprot);
-              struct.setTypeDescIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // POSITION
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.position = iprot.readI32();
-              struct.setPositionIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // COMMENT
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.comment = iprot.readString();
-              struct.setCommentIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TColumnDesc struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.columnName != null) {
-        oprot.writeFieldBegin(COLUMN_NAME_FIELD_DESC);
-        oprot.writeString(struct.columnName);
-        oprot.writeFieldEnd();
-      }
-      if (struct.typeDesc != null) {
-        oprot.writeFieldBegin(TYPE_DESC_FIELD_DESC);
-        struct.typeDesc.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldBegin(POSITION_FIELD_DESC);
-      oprot.writeI32(struct.position);
-      oprot.writeFieldEnd();
-      if (struct.comment != null) {
-        if (struct.isSetComment()) {
-          oprot.writeFieldBegin(COMMENT_FIELD_DESC);
-          oprot.writeString(struct.comment);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TColumnDescTupleSchemeFactory implements SchemeFactory {
-    public TColumnDescTupleScheme getScheme() {
-      return new TColumnDescTupleScheme();
-    }
-  }
-
-  private static class TColumnDescTupleScheme extends TupleScheme<TColumnDesc> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TColumnDesc struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      oprot.writeString(struct.columnName);
-      struct.typeDesc.write(oprot);
-      oprot.writeI32(struct.position);
-      BitSet optionals = new BitSet();
-      if (struct.isSetComment()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetComment()) {
-        oprot.writeString(struct.comment);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TColumnDesc struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.columnName = iprot.readString();
-      struct.setColumnNameIsSet(true);
-      struct.typeDesc = new TTypeDesc();
-      struct.typeDesc.read(iprot);
-      struct.setTypeDescIsSet(true);
-      struct.position = iprot.readI32();
-      struct.setPositionIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.comment = iprot.readString();
-        struct.setCommentIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TColumnValue.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TColumnValue.java
deleted file mode 100644
index d1cc8e919bc0c..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TColumnValue.java
+++ /dev/null
@@ -1,675 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-public class TColumnValue extends org.apache.thrift.TUnion<TColumnValue, TColumnValue._Fields> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TColumnValue");
-  private static final org.apache.thrift.protocol.TField BOOL_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("boolVal", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField BYTE_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("byteVal", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-  private static final org.apache.thrift.protocol.TField I16_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("i16Val", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-  private static final org.apache.thrift.protocol.TField I32_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("i32Val", org.apache.thrift.protocol.TType.STRUCT, (short)4);
-  private static final org.apache.thrift.protocol.TField I64_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("i64Val", org.apache.thrift.protocol.TType.STRUCT, (short)5);
-  private static final org.apache.thrift.protocol.TField DOUBLE_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("doubleVal", org.apache.thrift.protocol.TType.STRUCT, (short)6);
-  private static final org.apache.thrift.protocol.TField STRING_VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("stringVal", org.apache.thrift.protocol.TType.STRUCT, (short)7);
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    BOOL_VAL((short)1, "boolVal"),
-    BYTE_VAL((short)2, "byteVal"),
-    I16_VAL((short)3, "i16Val"),
-    I32_VAL((short)4, "i32Val"),
-    I64_VAL((short)5, "i64Val"),
-    DOUBLE_VAL((short)6, "doubleVal"),
-    STRING_VAL((short)7, "stringVal");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // BOOL_VAL
-          return BOOL_VAL;
-        case 2: // BYTE_VAL
-          return BYTE_VAL;
-        case 3: // I16_VAL
-          return I16_VAL;
-        case 4: // I32_VAL
-          return I32_VAL;
-        case 5: // I64_VAL
-          return I64_VAL;
-        case 6: // DOUBLE_VAL
-          return DOUBLE_VAL;
-        case 7: // STRING_VAL
-          return STRING_VAL;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.BOOL_VAL, new org.apache.thrift.meta_data.FieldMetaData("boolVal", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TBoolValue.class)));
-    tmpMap.put(_Fields.BYTE_VAL, new org.apache.thrift.meta_data.FieldMetaData("byteVal", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TByteValue.class)));
-    tmpMap.put(_Fields.I16_VAL, new org.apache.thrift.meta_data.FieldMetaData("i16Val", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TI16Value.class)));
-    tmpMap.put(_Fields.I32_VAL, new org.apache.thrift.meta_data.FieldMetaData("i32Val", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TI32Value.class)));
-    tmpMap.put(_Fields.I64_VAL, new org.apache.thrift.meta_data.FieldMetaData("i64Val", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TI64Value.class)));
-    tmpMap.put(_Fields.DOUBLE_VAL, new org.apache.thrift.meta_data.FieldMetaData("doubleVal", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TDoubleValue.class)));
-    tmpMap.put(_Fields.STRING_VAL, new org.apache.thrift.meta_data.FieldMetaData("stringVal", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStringValue.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TColumnValue.class, metaDataMap);
-  }
-
-  public TColumnValue() {
-    super();
-  }
-
-  public TColumnValue(TColumnValue._Fields setField, Object value) {
-    super(setField, value);
-  }
-
-  public TColumnValue(TColumnValue other) {
-    super(other);
-  }
-  public TColumnValue deepCopy() {
-    return new TColumnValue(this);
-  }
-
-  public static TColumnValue boolVal(TBoolValue value) {
-    TColumnValue x = new TColumnValue();
-    x.setBoolVal(value);
-    return x;
-  }
-
-  public static TColumnValue byteVal(TByteValue value) {
-    TColumnValue x = new TColumnValue();
-    x.setByteVal(value);
-    return x;
-  }
-
-  public static TColumnValue i16Val(TI16Value value) {
-    TColumnValue x = new TColumnValue();
-    x.setI16Val(value);
-    return x;
-  }
-
-  public static TColumnValue i32Val(TI32Value value) {
-    TColumnValue x = new TColumnValue();
-    x.setI32Val(value);
-    return x;
-  }
-
-  public static TColumnValue i64Val(TI64Value value) {
-    TColumnValue x = new TColumnValue();
-    x.setI64Val(value);
-    return x;
-  }
-
-  public static TColumnValue doubleVal(TDoubleValue value) {
-    TColumnValue x = new TColumnValue();
-    x.setDoubleVal(value);
-    return x;
-  }
-
-  public static TColumnValue stringVal(TStringValue value) {
-    TColumnValue x = new TColumnValue();
-    x.setStringVal(value);
-    return x;
-  }
-
-
-  @Override
-  protected void checkType(_Fields setField, Object value) throws ClassCastException {
-    switch (setField) {
-      case BOOL_VAL:
-        if (value instanceof TBoolValue) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TBoolValue for field 'boolVal', but got " + value.getClass().getSimpleName());
-      case BYTE_VAL:
-        if (value instanceof TByteValue) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TByteValue for field 'byteVal', but got " + value.getClass().getSimpleName());
-      case I16_VAL:
-        if (value instanceof TI16Value) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TI16Value for field 'i16Val', but got " + value.getClass().getSimpleName());
-      case I32_VAL:
-        if (value instanceof TI32Value) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TI32Value for field 'i32Val', but got " + value.getClass().getSimpleName());
-      case I64_VAL:
-        if (value instanceof TI64Value) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TI64Value for field 'i64Val', but got " + value.getClass().getSimpleName());
-      case DOUBLE_VAL:
-        if (value instanceof TDoubleValue) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TDoubleValue for field 'doubleVal', but got " + value.getClass().getSimpleName());
-      case STRING_VAL:
-        if (value instanceof TStringValue) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TStringValue for field 'stringVal', but got " + value.getClass().getSimpleName());
-      default:
-        throw new IllegalArgumentException("Unknown field id " + setField);
-    }
-  }
-
-  @Override
-  protected Object standardSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TField field) throws org.apache.thrift.TException {
-    _Fields setField = _Fields.findByThriftId(field.id);
-    if (setField != null) {
-      switch (setField) {
-        case BOOL_VAL:
-          if (field.type == BOOL_VAL_FIELD_DESC.type) {
-            TBoolValue boolVal;
-            boolVal = new TBoolValue();
-            boolVal.read(iprot);
-            return boolVal;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case BYTE_VAL:
-          if (field.type == BYTE_VAL_FIELD_DESC.type) {
-            TByteValue byteVal;
-            byteVal = new TByteValue();
-            byteVal.read(iprot);
-            return byteVal;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case I16_VAL:
-          if (field.type == I16_VAL_FIELD_DESC.type) {
-            TI16Value i16Val;
-            i16Val = new TI16Value();
-            i16Val.read(iprot);
-            return i16Val;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case I32_VAL:
-          if (field.type == I32_VAL_FIELD_DESC.type) {
-            TI32Value i32Val;
-            i32Val = new TI32Value();
-            i32Val.read(iprot);
-            return i32Val;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case I64_VAL:
-          if (field.type == I64_VAL_FIELD_DESC.type) {
-            TI64Value i64Val;
-            i64Val = new TI64Value();
-            i64Val.read(iprot);
-            return i64Val;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case DOUBLE_VAL:
-          if (field.type == DOUBLE_VAL_FIELD_DESC.type) {
-            TDoubleValue doubleVal;
-            doubleVal = new TDoubleValue();
-            doubleVal.read(iprot);
-            return doubleVal;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case STRING_VAL:
-          if (field.type == STRING_VAL_FIELD_DESC.type) {
-            TStringValue stringVal;
-            stringVal = new TStringValue();
-            stringVal.read(iprot);
-            return stringVal;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        default:
-          throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!");
-      }
-    } else {
-      org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-      return null;
-    }
-  }
-
-  @Override
-  protected void standardSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    switch (setField_) {
-      case BOOL_VAL:
-        TBoolValue boolVal = (TBoolValue)value_;
-        boolVal.write(oprot);
-        return;
-      case BYTE_VAL:
-        TByteValue byteVal = (TByteValue)value_;
-        byteVal.write(oprot);
-        return;
-      case I16_VAL:
-        TI16Value i16Val = (TI16Value)value_;
-        i16Val.write(oprot);
-        return;
-      case I32_VAL:
-        TI32Value i32Val = (TI32Value)value_;
-        i32Val.write(oprot);
-        return;
-      case I64_VAL:
-        TI64Value i64Val = (TI64Value)value_;
-        i64Val.write(oprot);
-        return;
-      case DOUBLE_VAL:
-        TDoubleValue doubleVal = (TDoubleValue)value_;
-        doubleVal.write(oprot);
-        return;
-      case STRING_VAL:
-        TStringValue stringVal = (TStringValue)value_;
-        stringVal.write(oprot);
-        return;
-      default:
-        throw new IllegalStateException("Cannot write union with unknown field " + setField_);
-    }
-  }
-
-  @Override
-  protected Object tupleSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, short fieldID) throws org.apache.thrift.TException {
-    _Fields setField = _Fields.findByThriftId(fieldID);
-    if (setField != null) {
-      switch (setField) {
-        case BOOL_VAL:
-          TBoolValue boolVal;
-          boolVal = new TBoolValue();
-          boolVal.read(iprot);
-          return boolVal;
-        case BYTE_VAL:
-          TByteValue byteVal;
-          byteVal = new TByteValue();
-          byteVal.read(iprot);
-          return byteVal;
-        case I16_VAL:
-          TI16Value i16Val;
-          i16Val = new TI16Value();
-          i16Val.read(iprot);
-          return i16Val;
-        case I32_VAL:
-          TI32Value i32Val;
-          i32Val = new TI32Value();
-          i32Val.read(iprot);
-          return i32Val;
-        case I64_VAL:
-          TI64Value i64Val;
-          i64Val = new TI64Value();
-          i64Val.read(iprot);
-          return i64Val;
-        case DOUBLE_VAL:
-          TDoubleValue doubleVal;
-          doubleVal = new TDoubleValue();
-          doubleVal.read(iprot);
-          return doubleVal;
-        case STRING_VAL:
-          TStringValue stringVal;
-          stringVal = new TStringValue();
-          stringVal.read(iprot);
-          return stringVal;
-        default:
-          throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!");
-      }
-    } else {
-      throw new TProtocolException("Couldn't find a field with field id " + fieldID);
-    }
-  }
-
-  @Override
-  protected void tupleSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    switch (setField_) {
-      case BOOL_VAL:
-        TBoolValue boolVal = (TBoolValue)value_;
-        boolVal.write(oprot);
-        return;
-      case BYTE_VAL:
-        TByteValue byteVal = (TByteValue)value_;
-        byteVal.write(oprot);
-        return;
-      case I16_VAL:
-        TI16Value i16Val = (TI16Value)value_;
-        i16Val.write(oprot);
-        return;
-      case I32_VAL:
-        TI32Value i32Val = (TI32Value)value_;
-        i32Val.write(oprot);
-        return;
-      case I64_VAL:
-        TI64Value i64Val = (TI64Value)value_;
-        i64Val.write(oprot);
-        return;
-      case DOUBLE_VAL:
-        TDoubleValue doubleVal = (TDoubleValue)value_;
-        doubleVal.write(oprot);
-        return;
-      case STRING_VAL:
-        TStringValue stringVal = (TStringValue)value_;
-        stringVal.write(oprot);
-        return;
-      default:
-        throw new IllegalStateException("Cannot write union with unknown field " + setField_);
-    }
-  }
-
-  @Override
-  protected org.apache.thrift.protocol.TField getFieldDesc(_Fields setField) {
-    switch (setField) {
-      case BOOL_VAL:
-        return BOOL_VAL_FIELD_DESC;
-      case BYTE_VAL:
-        return BYTE_VAL_FIELD_DESC;
-      case I16_VAL:
-        return I16_VAL_FIELD_DESC;
-      case I32_VAL:
-        return I32_VAL_FIELD_DESC;
-      case I64_VAL:
-        return I64_VAL_FIELD_DESC;
-      case DOUBLE_VAL:
-        return DOUBLE_VAL_FIELD_DESC;
-      case STRING_VAL:
-        return STRING_VAL_FIELD_DESC;
-      default:
-        throw new IllegalArgumentException("Unknown field id " + setField);
-    }
-  }
-
-  @Override
-  protected org.apache.thrift.protocol.TStruct getStructDesc() {
-    return STRUCT_DESC;
-  }
-
-  @Override
-  protected _Fields enumForId(short id) {
-    return _Fields.findByThriftIdOrThrow(id);
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-
-  public TBoolValue getBoolVal() {
-    if (getSetField() == _Fields.BOOL_VAL) {
-      return (TBoolValue)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'boolVal' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setBoolVal(TBoolValue value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.BOOL_VAL;
-    value_ = value;
-  }
-
-  public TByteValue getByteVal() {
-    if (getSetField() == _Fields.BYTE_VAL) {
-      return (TByteValue)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'byteVal' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setByteVal(TByteValue value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.BYTE_VAL;
-    value_ = value;
-  }
-
-  public TI16Value getI16Val() {
-    if (getSetField() == _Fields.I16_VAL) {
-      return (TI16Value)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'i16Val' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setI16Val(TI16Value value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.I16_VAL;
-    value_ = value;
-  }
-
-  public TI32Value getI32Val() {
-    if (getSetField() == _Fields.I32_VAL) {
-      return (TI32Value)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'i32Val' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setI32Val(TI32Value value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.I32_VAL;
-    value_ = value;
-  }
-
-  public TI64Value getI64Val() {
-    if (getSetField() == _Fields.I64_VAL) {
-      return (TI64Value)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'i64Val' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setI64Val(TI64Value value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.I64_VAL;
-    value_ = value;
-  }
-
-  public TDoubleValue getDoubleVal() {
-    if (getSetField() == _Fields.DOUBLE_VAL) {
-      return (TDoubleValue)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'doubleVal' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setDoubleVal(TDoubleValue value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.DOUBLE_VAL;
-    value_ = value;
-  }
-
-  public TStringValue getStringVal() {
-    if (getSetField() == _Fields.STRING_VAL) {
-      return (TStringValue)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'stringVal' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setStringVal(TStringValue value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.STRING_VAL;
-    value_ = value;
-  }
-
-  public boolean isSetBoolVal() {
-    return setField_ == _Fields.BOOL_VAL;
-  }
-
-
-  public boolean isSetByteVal() {
-    return setField_ == _Fields.BYTE_VAL;
-  }
-
-
-  public boolean isSetI16Val() {
-    return setField_ == _Fields.I16_VAL;
-  }
-
-
-  public boolean isSetI32Val() {
-    return setField_ == _Fields.I32_VAL;
-  }
-
-
-  public boolean isSetI64Val() {
-    return setField_ == _Fields.I64_VAL;
-  }
-
-
-  public boolean isSetDoubleVal() {
-    return setField_ == _Fields.DOUBLE_VAL;
-  }
-
-
-  public boolean isSetStringVal() {
-    return setField_ == _Fields.STRING_VAL;
-  }
-
-
-  public boolean equals(Object other) {
-    if (other instanceof TColumnValue) {
-      return equals((TColumnValue)other);
-    } else {
-      return false;
-    }
-  }
-
-  public boolean equals(TColumnValue other) {
-    return other != null && getSetField() == other.getSetField() && getFieldValue().equals(other.getFieldValue());
-  }
-
-  @Override
-  public int compareTo(TColumnValue other) {
-    int lastComparison = org.apache.thrift.TBaseHelper.compareTo(getSetField(), other.getSetField());
-    if (lastComparison == 0) {
-      return org.apache.thrift.TBaseHelper.compareTo(getFieldValue(), other.getFieldValue());
-    }
-    return lastComparison;
-  }
-
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-    list.add(this.getClass().getName());
-    org.apache.thrift.TFieldIdEnum setField = getSetField();
-    if (setField != null) {
-      list.add(setField.getThriftFieldId());
-      Object value = getFieldValue();
-      if (value instanceof org.apache.thrift.TEnum) {
-        list.add(((org.apache.thrift.TEnum)getFieldValue()).getValue());
-      } else {
-        list.add(value);
-      }
-    }
-    return list.hashCode();
-  }
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-
-}
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TDoubleColumn.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TDoubleColumn.java
deleted file mode 100644
index f93c9b4f0edc3..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TDoubleColumn.java
+++ /dev/null
@@ -1,548 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TDoubleColumn implements org.apache.thrift.TBase<TDoubleColumn, TDoubleColumn._Fields>, java.io.Serializable, Cloneable, Comparable<TDoubleColumn> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TDoubleColumn");
-
-  private static final org.apache.thrift.protocol.TField VALUES_FIELD_DESC = new org.apache.thrift.protocol.TField("values", org.apache.thrift.protocol.TType.LIST, (short)1);
-  private static final org.apache.thrift.protocol.TField NULLS_FIELD_DESC = new org.apache.thrift.protocol.TField("nulls", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TDoubleColumnStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TDoubleColumnTupleSchemeFactory());
-  }
-
-  private List<Double> values; // required
-  private ByteBuffer nulls; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUES((short)1, "values"),
-    NULLS((short)2, "nulls");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUES
-          return VALUES;
-        case 2: // NULLS
-          return NULLS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUES, new org.apache.thrift.meta_data.FieldMetaData("values", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE))));
-    tmpMap.put(_Fields.NULLS, new org.apache.thrift.meta_data.FieldMetaData("nulls", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TDoubleColumn.class, metaDataMap);
-  }
-
-  public TDoubleColumn() {
-  }
-
-  public TDoubleColumn(
-    List<Double> values,
-    ByteBuffer nulls)
-  {
-    this();
-    this.values = values;
-    this.nulls = org.apache.thrift.TBaseHelper.copyBinary(nulls);
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TDoubleColumn(TDoubleColumn other) {
-    if (other.isSetValues()) {
-      List<Double> __this__values = new ArrayList<Double>(other.values);
-      this.values = __this__values;
-    }
-    if (other.isSetNulls()) {
-      this.nulls = org.apache.thrift.TBaseHelper.copyBinary(other.nulls);
-    }
-  }
-
-  public TDoubleColumn deepCopy() {
-    return new TDoubleColumn(this);
-  }
-
-  @Override
-  public void clear() {
-    this.values = null;
-    this.nulls = null;
-  }
-
-  public int getValuesSize() {
-    return (this.values == null) ? 0 : this.values.size();
-  }
-
-  public java.util.Iterator<Double> getValuesIterator() {
-    return (this.values == null) ? null : this.values.iterator();
-  }
-
-  public void addToValues(double elem) {
-    if (this.values == null) {
-      this.values = new ArrayList<Double>();
-    }
-    this.values.add(elem);
-  }
-
-  public List<Double> getValues() {
-    return this.values;
-  }
-
-  public void setValues(List<Double> values) {
-    this.values = values;
-  }
-
-  public void unsetValues() {
-    this.values = null;
-  }
-
-  /** Returns true if field values is set (has been assigned a value) and false otherwise */
-  public boolean isSetValues() {
-    return this.values != null;
-  }
-
-  public void setValuesIsSet(boolean value) {
-    if (!value) {
-      this.values = null;
-    }
-  }
-
-  public byte[] getNulls() {
-    setNulls(org.apache.thrift.TBaseHelper.rightSize(nulls));
-    return nulls == null ? null : nulls.array();
-  }
-
-  public ByteBuffer bufferForNulls() {
-    return org.apache.thrift.TBaseHelper.copyBinary(nulls);
-  }
-
-  public void setNulls(byte[] nulls) {
-    this.nulls = nulls == null ? (ByteBuffer)null : ByteBuffer.wrap(Arrays.copyOf(nulls, nulls.length));
-  }
-
-  public void setNulls(ByteBuffer nulls) {
-    this.nulls = org.apache.thrift.TBaseHelper.copyBinary(nulls);
-  }
-
-  public void unsetNulls() {
-    this.nulls = null;
-  }
-
-  /** Returns true if field nulls is set (has been assigned a value) and false otherwise */
-  public boolean isSetNulls() {
-    return this.nulls != null;
-  }
-
-  public void setNullsIsSet(boolean value) {
-    if (!value) {
-      this.nulls = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUES:
-      if (value == null) {
-        unsetValues();
-      } else {
-        setValues((List<Double>)value);
-      }
-      break;
-
-    case NULLS:
-      if (value == null) {
-        unsetNulls();
-      } else {
-        setNulls((ByteBuffer)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUES:
-      return getValues();
-
-    case NULLS:
-      return getNulls();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUES:
-      return isSetValues();
-    case NULLS:
-      return isSetNulls();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TDoubleColumn)
-      return this.equals((TDoubleColumn)that);
-    return false;
-  }
-
-  public boolean equals(TDoubleColumn that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_values = true && this.isSetValues();
-    boolean that_present_values = true && that.isSetValues();
-    if (this_present_values || that_present_values) {
-      if (!(this_present_values && that_present_values))
-        return false;
-      if (!this.values.equals(that.values))
-        return false;
-    }
-
-    boolean this_present_nulls = true && this.isSetNulls();
-    boolean that_present_nulls = true && that.isSetNulls();
-    if (this_present_nulls || that_present_nulls) {
-      if (!(this_present_nulls && that_present_nulls))
-        return false;
-      if (!this.nulls.equals(that.nulls))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_values = true && (isSetValues());
-    list.add(present_values);
-    if (present_values)
-      list.add(values);
-
-    boolean present_nulls = true && (isSetNulls());
-    list.add(present_nulls);
-    if (present_nulls)
-      list.add(nulls);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TDoubleColumn other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetValues()).compareTo(other.isSetValues());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValues()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.values, other.values);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetNulls()).compareTo(other.isSetNulls());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetNulls()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.nulls, other.nulls);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TDoubleColumn(");
-    boolean first = true;
-
-    sb.append("values:");
-    if (this.values == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.values);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("nulls:");
-    if (this.nulls == null) {
-      sb.append("null");
-    } else {
-      org.apache.thrift.TBaseHelper.toString(this.nulls, sb);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetValues()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'values' is unset! Struct:" + toString());
-    }
-
-    if (!isSetNulls()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'nulls' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TDoubleColumnStandardSchemeFactory implements SchemeFactory {
-    public TDoubleColumnStandardScheme getScheme() {
-      return new TDoubleColumnStandardScheme();
-    }
-  }
-
-  private static class TDoubleColumnStandardScheme extends StandardScheme<TDoubleColumn> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TDoubleColumn struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUES
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list94 = iprot.readListBegin();
-                struct.values = new ArrayList<Double>(_list94.size);
-                double _elem95;
-                for (int _i96 = 0; _i96 < _list94.size; ++_i96)
-                {
-                  _elem95 = iprot.readDouble();
-                  struct.values.add(_elem95);
-                }
-                iprot.readListEnd();
-              }
-              struct.setValuesIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // NULLS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.nulls = iprot.readBinary();
-              struct.setNullsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TDoubleColumn struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.values != null) {
-        oprot.writeFieldBegin(VALUES_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.DOUBLE, struct.values.size()));
-          for (double _iter97 : struct.values)
-          {
-            oprot.writeDouble(_iter97);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      if (struct.nulls != null) {
-        oprot.writeFieldBegin(NULLS_FIELD_DESC);
-        oprot.writeBinary(struct.nulls);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TDoubleColumnTupleSchemeFactory implements SchemeFactory {
-    public TDoubleColumnTupleScheme getScheme() {
-      return new TDoubleColumnTupleScheme();
-    }
-  }
-
-  private static class TDoubleColumnTupleScheme extends TupleScheme<TDoubleColumn> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TDoubleColumn struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.values.size());
-        for (double _iter98 : struct.values)
-        {
-          oprot.writeDouble(_iter98);
-        }
-      }
-      oprot.writeBinary(struct.nulls);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TDoubleColumn struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TList _list99 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.DOUBLE, iprot.readI32());
-        struct.values = new ArrayList<Double>(_list99.size);
-        double _elem100;
-        for (int _i101 = 0; _i101 < _list99.size; ++_i101)
-        {
-          _elem100 = iprot.readDouble();
-          struct.values.add(_elem100);
-        }
-      }
-      struct.setValuesIsSet(true);
-      struct.nulls = iprot.readBinary();
-      struct.setNullsIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TDoubleValue.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TDoubleValue.java
deleted file mode 100644
index 5700355aad94d..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TDoubleValue.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TDoubleValue implements org.apache.thrift.TBase<TDoubleValue, TDoubleValue._Fields>, java.io.Serializable, Cloneable, Comparable<TDoubleValue> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TDoubleValue");
-
-  private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.DOUBLE, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TDoubleValueStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TDoubleValueTupleSchemeFactory());
-  }
-
-  private double value; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUE((short)1, "value");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUE
-          return VALUE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __VALUE_ISSET_ID = 0;
-  private byte __isset_bitfield = 0;
-  private static final _Fields optionals[] = {_Fields.VALUE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TDoubleValue.class, metaDataMap);
-  }
-
-  public TDoubleValue() {
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TDoubleValue(TDoubleValue other) {
-    __isset_bitfield = other.__isset_bitfield;
-    this.value = other.value;
-  }
-
-  public TDoubleValue deepCopy() {
-    return new TDoubleValue(this);
-  }
-
-  @Override
-  public void clear() {
-    setValueIsSet(false);
-    this.value = 0.0;
-  }
-
-  public double getValue() {
-    return this.value;
-  }
-
-  public void setValue(double value) {
-    this.value = value;
-    setValueIsSet(true);
-  }
-
-  public void unsetValue() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __VALUE_ISSET_ID);
-  }
-
-  /** Returns true if field value is set (has been assigned a value) and false otherwise */
-  public boolean isSetValue() {
-    return EncodingUtils.testBit(__isset_bitfield, __VALUE_ISSET_ID);
-  }
-
-  public void setValueIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __VALUE_ISSET_ID, value);
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUE:
-      if (value == null) {
-        unsetValue();
-      } else {
-        setValue((Double)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUE:
-      return getValue();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUE:
-      return isSetValue();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TDoubleValue)
-      return this.equals((TDoubleValue)that);
-    return false;
-  }
-
-  public boolean equals(TDoubleValue that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_value = true && this.isSetValue();
-    boolean that_present_value = true && that.isSetValue();
-    if (this_present_value || that_present_value) {
-      if (!(this_present_value && that_present_value))
-        return false;
-      if (this.value != that.value)
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_value = true && (isSetValue());
-    list.add(present_value);
-    if (present_value)
-      list.add(value);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TDoubleValue other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetValue()).compareTo(other.isSetValue());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValue()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, other.value);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TDoubleValue(");
-    boolean first = true;
-
-    if (isSetValue()) {
-      sb.append("value:");
-      sb.append(this.value);
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TDoubleValueStandardSchemeFactory implements SchemeFactory {
-    public TDoubleValueStandardScheme getScheme() {
-      return new TDoubleValueStandardScheme();
-    }
-  }
-
-  private static class TDoubleValueStandardScheme extends StandardScheme<TDoubleValue> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TDoubleValue struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUE
-            if (schemeField.type == org.apache.thrift.protocol.TType.DOUBLE) {
-              struct.value = iprot.readDouble();
-              struct.setValueIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TDoubleValue struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.isSetValue()) {
-        oprot.writeFieldBegin(VALUE_FIELD_DESC);
-        oprot.writeDouble(struct.value);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TDoubleValueTupleSchemeFactory implements SchemeFactory {
-    public TDoubleValueTupleScheme getScheme() {
-      return new TDoubleValueTupleScheme();
-    }
-  }
-
-  private static class TDoubleValueTupleScheme extends TupleScheme<TDoubleValue> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TDoubleValue struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      BitSet optionals = new BitSet();
-      if (struct.isSetValue()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetValue()) {
-        oprot.writeDouble(struct.value);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TDoubleValue struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.value = iprot.readDouble();
-        struct.setValueIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TExecuteStatementReq.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TExecuteStatementReq.java
deleted file mode 100644
index 1f73cec61af78..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TExecuteStatementReq.java
+++ /dev/null
@@ -1,863 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TExecuteStatementReq implements org.apache.thrift.TBase<TExecuteStatementReq, TExecuteStatementReq._Fields>, java.io.Serializable, Cloneable, Comparable<TExecuteStatementReq> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TExecuteStatementReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField STATEMENT_FIELD_DESC = new org.apache.thrift.protocol.TField("statement", org.apache.thrift.protocol.TType.STRING, (short)2);
-  private static final org.apache.thrift.protocol.TField CONF_OVERLAY_FIELD_DESC = new org.apache.thrift.protocol.TField("confOverlay", org.apache.thrift.protocol.TType.MAP, (short)3);
-  private static final org.apache.thrift.protocol.TField RUN_ASYNC_FIELD_DESC = new org.apache.thrift.protocol.TField("runAsync", org.apache.thrift.protocol.TType.BOOL, (short)4);
-  private static final org.apache.thrift.protocol.TField QUERY_TIMEOUT_FIELD_DESC = new org.apache.thrift.protocol.TField("queryTimeout", org.apache.thrift.protocol.TType.I64, (short)5);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TExecuteStatementReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TExecuteStatementReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-  private String statement; // required
-  private Map<String,String> confOverlay; // optional
-  private boolean runAsync; // optional
-  private long queryTimeout; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle"),
-    STATEMENT((short)2, "statement"),
-    CONF_OVERLAY((short)3, "confOverlay"),
-    RUN_ASYNC((short)4, "runAsync"),
-    QUERY_TIMEOUT((short)5, "queryTimeout");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        case 2: // STATEMENT
-          return STATEMENT;
-        case 3: // CONF_OVERLAY
-          return CONF_OVERLAY;
-        case 4: // RUN_ASYNC
-          return RUN_ASYNC;
-        case 5: // QUERY_TIMEOUT
-          return QUERY_TIMEOUT;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __RUNASYNC_ISSET_ID = 0;
-  private static final int __QUERYTIMEOUT_ISSET_ID = 1;
-  private byte __isset_bitfield = 0;
-  private static final _Fields optionals[] = {_Fields.CONF_OVERLAY,_Fields.RUN_ASYNC,_Fields.QUERY_TIMEOUT};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    tmpMap.put(_Fields.STATEMENT, new org.apache.thrift.meta_data.FieldMetaData("statement", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    tmpMap.put(_Fields.CONF_OVERLAY, new org.apache.thrift.meta_data.FieldMetaData("confOverlay", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING), 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
-    tmpMap.put(_Fields.RUN_ASYNC, new org.apache.thrift.meta_data.FieldMetaData("runAsync", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
-    tmpMap.put(_Fields.QUERY_TIMEOUT, new org.apache.thrift.meta_data.FieldMetaData("queryTimeout", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TExecuteStatementReq.class, metaDataMap);
-  }
-
-  public TExecuteStatementReq() {
-    this.runAsync = false;
-
-    this.queryTimeout = 0L;
-
-  }
-
-  public TExecuteStatementReq(
-    TSessionHandle sessionHandle,
-    String statement)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-    this.statement = statement;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TExecuteStatementReq(TExecuteStatementReq other) {
-    __isset_bitfield = other.__isset_bitfield;
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-    if (other.isSetStatement()) {
-      this.statement = other.statement;
-    }
-    if (other.isSetConfOverlay()) {
-      Map<String,String> __this__confOverlay = new HashMap<String,String>(other.confOverlay);
-      this.confOverlay = __this__confOverlay;
-    }
-    this.runAsync = other.runAsync;
-    this.queryTimeout = other.queryTimeout;
-  }
-
-  public TExecuteStatementReq deepCopy() {
-    return new TExecuteStatementReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-    this.statement = null;
-    this.confOverlay = null;
-    this.runAsync = false;
-
-    this.queryTimeout = 0L;
-
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public String getStatement() {
-    return this.statement;
-  }
-
-  public void setStatement(String statement) {
-    this.statement = statement;
-  }
-
-  public void unsetStatement() {
-    this.statement = null;
-  }
-
-  /** Returns true if field statement is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatement() {
-    return this.statement != null;
-  }
-
-  public void setStatementIsSet(boolean value) {
-    if (!value) {
-      this.statement = null;
-    }
-  }
-
-  public int getConfOverlaySize() {
-    return (this.confOverlay == null) ? 0 : this.confOverlay.size();
-  }
-
-  public void putToConfOverlay(String key, String val) {
-    if (this.confOverlay == null) {
-      this.confOverlay = new HashMap<String,String>();
-    }
-    this.confOverlay.put(key, val);
-  }
-
-  public Map<String,String> getConfOverlay() {
-    return this.confOverlay;
-  }
-
-  public void setConfOverlay(Map<String,String> confOverlay) {
-    this.confOverlay = confOverlay;
-  }
-
-  public void unsetConfOverlay() {
-    this.confOverlay = null;
-  }
-
-  /** Returns true if field confOverlay is set (has been assigned a value) and false otherwise */
-  public boolean isSetConfOverlay() {
-    return this.confOverlay != null;
-  }
-
-  public void setConfOverlayIsSet(boolean value) {
-    if (!value) {
-      this.confOverlay = null;
-    }
-  }
-
-  public boolean isRunAsync() {
-    return this.runAsync;
-  }
-
-  public void setRunAsync(boolean runAsync) {
-    this.runAsync = runAsync;
-    setRunAsyncIsSet(true);
-  }
-
-  public void unsetRunAsync() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __RUNASYNC_ISSET_ID);
-  }
-
-  /** Returns true if field runAsync is set (has been assigned a value) and false otherwise */
-  public boolean isSetRunAsync() {
-    return EncodingUtils.testBit(__isset_bitfield, __RUNASYNC_ISSET_ID);
-  }
-
-  public void setRunAsyncIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __RUNASYNC_ISSET_ID, value);
-  }
-
-  public long getQueryTimeout() {
-    return this.queryTimeout;
-  }
-
-  public void setQueryTimeout(long queryTimeout) {
-    this.queryTimeout = queryTimeout;
-    setQueryTimeoutIsSet(true);
-  }
-
-  public void unsetQueryTimeout() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __QUERYTIMEOUT_ISSET_ID);
-  }
-
-  /** Returns true if field queryTimeout is set (has been assigned a value) and false otherwise */
-  public boolean isSetQueryTimeout() {
-    return EncodingUtils.testBit(__isset_bitfield, __QUERYTIMEOUT_ISSET_ID);
-  }
-
-  public void setQueryTimeoutIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __QUERYTIMEOUT_ISSET_ID, value);
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    case STATEMENT:
-      if (value == null) {
-        unsetStatement();
-      } else {
-        setStatement((String)value);
-      }
-      break;
-
-    case CONF_OVERLAY:
-      if (value == null) {
-        unsetConfOverlay();
-      } else {
-        setConfOverlay((Map<String,String>)value);
-      }
-      break;
-
-    case RUN_ASYNC:
-      if (value == null) {
-        unsetRunAsync();
-      } else {
-        setRunAsync((Boolean)value);
-      }
-      break;
-
-    case QUERY_TIMEOUT:
-      if (value == null) {
-        unsetQueryTimeout();
-      } else {
-        setQueryTimeout((Long)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    case STATEMENT:
-      return getStatement();
-
-    case CONF_OVERLAY:
-      return getConfOverlay();
-
-    case RUN_ASYNC:
-      return isRunAsync();
-
-    case QUERY_TIMEOUT:
-      return getQueryTimeout();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    case STATEMENT:
-      return isSetStatement();
-    case CONF_OVERLAY:
-      return isSetConfOverlay();
-    case RUN_ASYNC:
-      return isSetRunAsync();
-    case QUERY_TIMEOUT:
-      return isSetQueryTimeout();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TExecuteStatementReq)
-      return this.equals((TExecuteStatementReq)that);
-    return false;
-  }
-
-  public boolean equals(TExecuteStatementReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    boolean this_present_statement = true && this.isSetStatement();
-    boolean that_present_statement = true && that.isSetStatement();
-    if (this_present_statement || that_present_statement) {
-      if (!(this_present_statement && that_present_statement))
-        return false;
-      if (!this.statement.equals(that.statement))
-        return false;
-    }
-
-    boolean this_present_confOverlay = true && this.isSetConfOverlay();
-    boolean that_present_confOverlay = true && that.isSetConfOverlay();
-    if (this_present_confOverlay || that_present_confOverlay) {
-      if (!(this_present_confOverlay && that_present_confOverlay))
-        return false;
-      if (!this.confOverlay.equals(that.confOverlay))
-        return false;
-    }
-
-    boolean this_present_runAsync = true && this.isSetRunAsync();
-    boolean that_present_runAsync = true && that.isSetRunAsync();
-    if (this_present_runAsync || that_present_runAsync) {
-      if (!(this_present_runAsync && that_present_runAsync))
-        return false;
-      if (this.runAsync != that.runAsync)
-        return false;
-    }
-
-    boolean this_present_queryTimeout = true && this.isSetQueryTimeout();
-    boolean that_present_queryTimeout = true && that.isSetQueryTimeout();
-    if (this_present_queryTimeout || that_present_queryTimeout) {
-      if (!(this_present_queryTimeout && that_present_queryTimeout))
-        return false;
-      if (this.queryTimeout != that.queryTimeout)
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    list.add(present_sessionHandle);
-    if (present_sessionHandle)
-      list.add(sessionHandle);
-
-    boolean present_statement = true && (isSetStatement());
-    list.add(present_statement);
-    if (present_statement)
-      list.add(statement);
-
-    boolean present_confOverlay = true && (isSetConfOverlay());
-    list.add(present_confOverlay);
-    if (present_confOverlay)
-      list.add(confOverlay);
-
-    boolean present_runAsync = true && (isSetRunAsync());
-    list.add(present_runAsync);
-    if (present_runAsync)
-      list.add(runAsync);
-
-    boolean present_queryTimeout = true && (isSetQueryTimeout());
-    list.add(present_queryTimeout);
-    if (present_queryTimeout)
-      list.add(queryTimeout);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TExecuteStatementReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(other.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, other.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetStatement()).compareTo(other.isSetStatement());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatement()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.statement, other.statement);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetConfOverlay()).compareTo(other.isSetConfOverlay());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetConfOverlay()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.confOverlay, other.confOverlay);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetRunAsync()).compareTo(other.isSetRunAsync());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetRunAsync()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.runAsync, other.runAsync);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetQueryTimeout()).compareTo(other.isSetQueryTimeout());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetQueryTimeout()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.queryTimeout, other.queryTimeout);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TExecuteStatementReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("statement:");
-    if (this.statement == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.statement);
-    }
-    first = false;
-    if (isSetConfOverlay()) {
-      if (!first) sb.append(", ");
-      sb.append("confOverlay:");
-      if (this.confOverlay == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.confOverlay);
-      }
-      first = false;
-    }
-    if (isSetRunAsync()) {
-      if (!first) sb.append(", ");
-      sb.append("runAsync:");
-      sb.append(this.runAsync);
-      first = false;
-    }
-    if (isSetQueryTimeout()) {
-      if (!first) sb.append(", ");
-      sb.append("queryTimeout:");
-      sb.append(this.queryTimeout);
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    if (!isSetStatement()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'statement' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TExecuteStatementReqStandardSchemeFactory implements SchemeFactory {
-    public TExecuteStatementReqStandardScheme getScheme() {
-      return new TExecuteStatementReqStandardScheme();
-    }
-  }
-
-  private static class TExecuteStatementReqStandardScheme extends StandardScheme<TExecuteStatementReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TExecuteStatementReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // STATEMENT
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.statement = iprot.readString();
-              struct.setStatementIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // CONF_OVERLAY
-            if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
-              {
-                org.apache.thrift.protocol.TMap _map162 = iprot.readMapBegin();
-                struct.confOverlay = new HashMap<String,String>(2*_map162.size);
-                String _key163;
-                String _val164;
-                for (int _i165 = 0; _i165 < _map162.size; ++_i165)
-                {
-                  _key163 = iprot.readString();
-                  _val164 = iprot.readString();
-                  struct.confOverlay.put(_key163, _val164);
-                }
-                iprot.readMapEnd();
-              }
-              struct.setConfOverlayIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // RUN_ASYNC
-            if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
-              struct.runAsync = iprot.readBool();
-              struct.setRunAsyncIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 5: // QUERY_TIMEOUT
-            if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
-              struct.queryTimeout = iprot.readI64();
-              struct.setQueryTimeoutIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TExecuteStatementReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.statement != null) {
-        oprot.writeFieldBegin(STATEMENT_FIELD_DESC);
-        oprot.writeString(struct.statement);
-        oprot.writeFieldEnd();
-      }
-      if (struct.confOverlay != null) {
-        if (struct.isSetConfOverlay()) {
-          oprot.writeFieldBegin(CONF_OVERLAY_FIELD_DESC);
-          {
-            oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.confOverlay.size()));
-            for (Map.Entry<String, String> _iter166 : struct.confOverlay.entrySet())
-            {
-              oprot.writeString(_iter166.getKey());
-              oprot.writeString(_iter166.getValue());
-            }
-            oprot.writeMapEnd();
-          }
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.isSetRunAsync()) {
-        oprot.writeFieldBegin(RUN_ASYNC_FIELD_DESC);
-        oprot.writeBool(struct.runAsync);
-        oprot.writeFieldEnd();
-      }
-      if (struct.isSetQueryTimeout()) {
-        oprot.writeFieldBegin(QUERY_TIMEOUT_FIELD_DESC);
-        oprot.writeI64(struct.queryTimeout);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TExecuteStatementReqTupleSchemeFactory implements SchemeFactory {
-    public TExecuteStatementReqTupleScheme getScheme() {
-      return new TExecuteStatementReqTupleScheme();
-    }
-  }
-
-  private static class TExecuteStatementReqTupleScheme extends TupleScheme<TExecuteStatementReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TExecuteStatementReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-      oprot.writeString(struct.statement);
-      BitSet optionals = new BitSet();
-      if (struct.isSetConfOverlay()) {
-        optionals.set(0);
-      }
-      if (struct.isSetRunAsync()) {
-        optionals.set(1);
-      }
-      if (struct.isSetQueryTimeout()) {
-        optionals.set(2);
-      }
-      oprot.writeBitSet(optionals, 3);
-      if (struct.isSetConfOverlay()) {
-        {
-          oprot.writeI32(struct.confOverlay.size());
-          for (Map.Entry<String, String> _iter167 : struct.confOverlay.entrySet())
-          {
-            oprot.writeString(_iter167.getKey());
-            oprot.writeString(_iter167.getValue());
-          }
-        }
-      }
-      if (struct.isSetRunAsync()) {
-        oprot.writeBool(struct.runAsync);
-      }
-      if (struct.isSetQueryTimeout()) {
-        oprot.writeI64(struct.queryTimeout);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TExecuteStatementReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-      struct.statement = iprot.readString();
-      struct.setStatementIsSet(true);
-      BitSet incoming = iprot.readBitSet(3);
-      if (incoming.get(0)) {
-        {
-          org.apache.thrift.protocol.TMap _map168 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-          struct.confOverlay = new HashMap<String,String>(2*_map168.size);
-          String _key169;
-          String _val170;
-          for (int _i171 = 0; _i171 < _map168.size; ++_i171)
-          {
-            _key169 = iprot.readString();
-            _val170 = iprot.readString();
-            struct.confOverlay.put(_key169, _val170);
-          }
-        }
-        struct.setConfOverlayIsSet(true);
-      }
-      if (incoming.get(1)) {
-        struct.runAsync = iprot.readBool();
-        struct.setRunAsyncIsSet(true);
-      }
-      if (incoming.get(2)) {
-        struct.queryTimeout = iprot.readI64();
-        struct.setQueryTimeoutIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TExecuteStatementResp.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TExecuteStatementResp.java
deleted file mode 100644
index 7101fa5bdb84c..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TExecuteStatementResp.java
+++ /dev/null
@@ -1,509 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TExecuteStatementResp implements org.apache.thrift.TBase<TExecuteStatementResp, TExecuteStatementResp._Fields>, java.io.Serializable, Cloneable, Comparable<TExecuteStatementResp> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TExecuteStatementResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TExecuteStatementRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TExecuteStatementRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TOperationHandle operationHandle; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    OPERATION_HANDLE((short)2, "operationHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final _Fields optionals[] = {_Fields.OPERATION_HANDLE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TExecuteStatementResp.class, metaDataMap);
-  }
-
-  public TExecuteStatementResp() {
-  }
-
-  public TExecuteStatementResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TExecuteStatementResp(TExecuteStatementResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-  }
-
-  public TExecuteStatementResp deepCopy() {
-    return new TExecuteStatementResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.operationHandle = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TExecuteStatementResp)
-      return this.equals((TExecuteStatementResp)that);
-    return false;
-  }
-
-  public boolean equals(TExecuteStatementResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_status = true && (isSetStatus());
-    list.add(present_status);
-    if (present_status)
-      list.add(status);
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    list.add(present_operationHandle);
-    if (present_operationHandle)
-      list.add(operationHandle);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TExecuteStatementResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(other.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, other.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(other.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, other.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TExecuteStatementResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetOperationHandle()) {
-      if (!first) sb.append(", ");
-      sb.append("operationHandle:");
-      if (this.operationHandle == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.operationHandle);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TExecuteStatementRespStandardSchemeFactory implements SchemeFactory {
-    public TExecuteStatementRespStandardScheme getScheme() {
-      return new TExecuteStatementRespStandardScheme();
-    }
-  }
-
-  private static class TExecuteStatementRespStandardScheme extends StandardScheme<TExecuteStatementResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TExecuteStatementResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TExecuteStatementResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.operationHandle != null) {
-        if (struct.isSetOperationHandle()) {
-          oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-          struct.operationHandle.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TExecuteStatementRespTupleSchemeFactory implements SchemeFactory {
-    public TExecuteStatementRespTupleScheme getScheme() {
-      return new TExecuteStatementRespTupleScheme();
-    }
-  }
-
-  private static class TExecuteStatementRespTupleScheme extends TupleScheme<TExecuteStatementResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TExecuteStatementResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetOperationHandle()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetOperationHandle()) {
-        struct.operationHandle.write(oprot);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TExecuteStatementResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.operationHandle = new TOperationHandle();
-        struct.operationHandle.read(iprot);
-        struct.setOperationHandleIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TFetchOrientation.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TFetchOrientation.java
deleted file mode 100644
index 159be45259434..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TFetchOrientation.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.thrift.TEnum;
-
-public enum TFetchOrientation implements org.apache.thrift.TEnum {
-  FETCH_NEXT(0),
-  FETCH_PRIOR(1),
-  FETCH_RELATIVE(2),
-  FETCH_ABSOLUTE(3),
-  FETCH_FIRST(4),
-  FETCH_LAST(5);
-
-  private final int value;
-
-  private TFetchOrientation(int value) {
-    this.value = value;
-  }
-
-  /**
-   * Get the integer value of this enum value, as defined in the Thrift IDL.
-   */
-  public int getValue() {
-    return value;
-  }
-
-  /**
-   * Find a the enum type by its integer value, as defined in the Thrift IDL.
-   * @return null if the value is not found.
-   */
-  public static TFetchOrientation findByValue(int value) { 
-    switch (value) {
-      case 0:
-        return FETCH_NEXT;
-      case 1:
-        return FETCH_PRIOR;
-      case 2:
-        return FETCH_RELATIVE;
-      case 3:
-        return FETCH_ABSOLUTE;
-      case 4:
-        return FETCH_FIRST;
-      case 5:
-        return FETCH_LAST;
-      default:
-        return null;
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TFetchResultsReq.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TFetchResultsReq.java
deleted file mode 100644
index 2c93339d0c68b..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TFetchResultsReq.java
+++ /dev/null
@@ -1,714 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TFetchResultsReq implements org.apache.thrift.TBase<TFetchResultsReq, TFetchResultsReq._Fields>, java.io.Serializable, Cloneable, Comparable<TFetchResultsReq> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TFetchResultsReq");
-
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField ORIENTATION_FIELD_DESC = new org.apache.thrift.protocol.TField("orientation", org.apache.thrift.protocol.TType.I32, (short)2);
-  private static final org.apache.thrift.protocol.TField MAX_ROWS_FIELD_DESC = new org.apache.thrift.protocol.TField("maxRows", org.apache.thrift.protocol.TType.I64, (short)3);
-  private static final org.apache.thrift.protocol.TField FETCH_TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("fetchType", org.apache.thrift.protocol.TType.I16, (short)4);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TFetchResultsReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TFetchResultsReqTupleSchemeFactory());
-  }
-
-  private TOperationHandle operationHandle; // required
-  private TFetchOrientation orientation; // required
-  private long maxRows; // required
-  private short fetchType; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    OPERATION_HANDLE((short)1, "operationHandle"),
-    /**
-     * 
-     * @see TFetchOrientation
-     */
-    ORIENTATION((short)2, "orientation"),
-    MAX_ROWS((short)3, "maxRows"),
-    FETCH_TYPE((short)4, "fetchType");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        case 2: // ORIENTATION
-          return ORIENTATION;
-        case 3: // MAX_ROWS
-          return MAX_ROWS;
-        case 4: // FETCH_TYPE
-          return FETCH_TYPE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __MAXROWS_ISSET_ID = 0;
-  private static final int __FETCHTYPE_ISSET_ID = 1;
-  private byte __isset_bitfield = 0;
-  private static final _Fields optionals[] = {_Fields.FETCH_TYPE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    tmpMap.put(_Fields.ORIENTATION, new org.apache.thrift.meta_data.FieldMetaData("orientation", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TFetchOrientation.class)));
-    tmpMap.put(_Fields.MAX_ROWS, new org.apache.thrift.meta_data.FieldMetaData("maxRows", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
-    tmpMap.put(_Fields.FETCH_TYPE, new org.apache.thrift.meta_data.FieldMetaData("fetchType", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I16)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TFetchResultsReq.class, metaDataMap);
-  }
-
-  public TFetchResultsReq() {
-    this.orientation = org.apache.hive.service.rpc.thrift.TFetchOrientation.FETCH_NEXT;
-
-    this.fetchType = (short)0;
-
-  }
-
-  public TFetchResultsReq(
-    TOperationHandle operationHandle,
-    TFetchOrientation orientation,
-    long maxRows)
-  {
-    this();
-    this.operationHandle = operationHandle;
-    this.orientation = orientation;
-    this.maxRows = maxRows;
-    setMaxRowsIsSet(true);
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TFetchResultsReq(TFetchResultsReq other) {
-    __isset_bitfield = other.__isset_bitfield;
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-    if (other.isSetOrientation()) {
-      this.orientation = other.orientation;
-    }
-    this.maxRows = other.maxRows;
-    this.fetchType = other.fetchType;
-  }
-
-  public TFetchResultsReq deepCopy() {
-    return new TFetchResultsReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.operationHandle = null;
-    this.orientation = org.apache.hive.service.rpc.thrift.TFetchOrientation.FETCH_NEXT;
-
-    setMaxRowsIsSet(false);
-    this.maxRows = 0;
-    this.fetchType = (short)0;
-
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  /**
-   * 
-   * @see TFetchOrientation
-   */
-  public TFetchOrientation getOrientation() {
-    return this.orientation;
-  }
-
-  /**
-   * 
-   * @see TFetchOrientation
-   */
-  public void setOrientation(TFetchOrientation orientation) {
-    this.orientation = orientation;
-  }
-
-  public void unsetOrientation() {
-    this.orientation = null;
-  }
-
-  /** Returns true if field orientation is set (has been assigned a value) and false otherwise */
-  public boolean isSetOrientation() {
-    return this.orientation != null;
-  }
-
-  public void setOrientationIsSet(boolean value) {
-    if (!value) {
-      this.orientation = null;
-    }
-  }
-
-  public long getMaxRows() {
-    return this.maxRows;
-  }
-
-  public void setMaxRows(long maxRows) {
-    this.maxRows = maxRows;
-    setMaxRowsIsSet(true);
-  }
-
-  public void unsetMaxRows() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __MAXROWS_ISSET_ID);
-  }
-
-  /** Returns true if field maxRows is set (has been assigned a value) and false otherwise */
-  public boolean isSetMaxRows() {
-    return EncodingUtils.testBit(__isset_bitfield, __MAXROWS_ISSET_ID);
-  }
-
-  public void setMaxRowsIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MAXROWS_ISSET_ID, value);
-  }
-
-  public short getFetchType() {
-    return this.fetchType;
-  }
-
-  public void setFetchType(short fetchType) {
-    this.fetchType = fetchType;
-    setFetchTypeIsSet(true);
-  }
-
-  public void unsetFetchType() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __FETCHTYPE_ISSET_ID);
-  }
-
-  /** Returns true if field fetchType is set (has been assigned a value) and false otherwise */
-  public boolean isSetFetchType() {
-    return EncodingUtils.testBit(__isset_bitfield, __FETCHTYPE_ISSET_ID);
-  }
-
-  public void setFetchTypeIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __FETCHTYPE_ISSET_ID, value);
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    case ORIENTATION:
-      if (value == null) {
-        unsetOrientation();
-      } else {
-        setOrientation((TFetchOrientation)value);
-      }
-      break;
-
-    case MAX_ROWS:
-      if (value == null) {
-        unsetMaxRows();
-      } else {
-        setMaxRows((Long)value);
-      }
-      break;
-
-    case FETCH_TYPE:
-      if (value == null) {
-        unsetFetchType();
-      } else {
-        setFetchType((Short)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    case ORIENTATION:
-      return getOrientation();
-
-    case MAX_ROWS:
-      return getMaxRows();
-
-    case FETCH_TYPE:
-      return getFetchType();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    case ORIENTATION:
-      return isSetOrientation();
-    case MAX_ROWS:
-      return isSetMaxRows();
-    case FETCH_TYPE:
-      return isSetFetchType();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TFetchResultsReq)
-      return this.equals((TFetchResultsReq)that);
-    return false;
-  }
-
-  public boolean equals(TFetchResultsReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    boolean this_present_orientation = true && this.isSetOrientation();
-    boolean that_present_orientation = true && that.isSetOrientation();
-    if (this_present_orientation || that_present_orientation) {
-      if (!(this_present_orientation && that_present_orientation))
-        return false;
-      if (!this.orientation.equals(that.orientation))
-        return false;
-    }
-
-    boolean this_present_maxRows = true;
-    boolean that_present_maxRows = true;
-    if (this_present_maxRows || that_present_maxRows) {
-      if (!(this_present_maxRows && that_present_maxRows))
-        return false;
-      if (this.maxRows != that.maxRows)
-        return false;
-    }
-
-    boolean this_present_fetchType = true && this.isSetFetchType();
-    boolean that_present_fetchType = true && that.isSetFetchType();
-    if (this_present_fetchType || that_present_fetchType) {
-      if (!(this_present_fetchType && that_present_fetchType))
-        return false;
-      if (this.fetchType != that.fetchType)
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    list.add(present_operationHandle);
-    if (present_operationHandle)
-      list.add(operationHandle);
-
-    boolean present_orientation = true && (isSetOrientation());
-    list.add(present_orientation);
-    if (present_orientation)
-      list.add(orientation.getValue());
-
-    boolean present_maxRows = true;
-    list.add(present_maxRows);
-    if (present_maxRows)
-      list.add(maxRows);
-
-    boolean present_fetchType = true && (isSetFetchType());
-    list.add(present_fetchType);
-    if (present_fetchType)
-      list.add(fetchType);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TFetchResultsReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(other.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, other.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOrientation()).compareTo(other.isSetOrientation());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOrientation()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.orientation, other.orientation);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetMaxRows()).compareTo(other.isSetMaxRows());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetMaxRows()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.maxRows, other.maxRows);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetFetchType()).compareTo(other.isSetFetchType());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetFetchType()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.fetchType, other.fetchType);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TFetchResultsReq(");
-    boolean first = true;
-
-    sb.append("operationHandle:");
-    if (this.operationHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.operationHandle);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("orientation:");
-    if (this.orientation == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.orientation);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("maxRows:");
-    sb.append(this.maxRows);
-    first = false;
-    if (isSetFetchType()) {
-      if (!first) sb.append(", ");
-      sb.append("fetchType:");
-      sb.append(this.fetchType);
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetOperationHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'operationHandle' is unset! Struct:" + toString());
-    }
-
-    if (!isSetOrientation()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'orientation' is unset! Struct:" + toString());
-    }
-
-    if (!isSetMaxRows()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'maxRows' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TFetchResultsReqStandardSchemeFactory implements SchemeFactory {
-    public TFetchResultsReqStandardScheme getScheme() {
-      return new TFetchResultsReqStandardScheme();
-    }
-  }
-
-  private static class TFetchResultsReqStandardScheme extends StandardScheme<TFetchResultsReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TFetchResultsReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // ORIENTATION
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.orientation = org.apache.hive.service.rpc.thrift.TFetchOrientation.findByValue(iprot.readI32());
-              struct.setOrientationIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // MAX_ROWS
-            if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
-              struct.maxRows = iprot.readI64();
-              struct.setMaxRowsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // FETCH_TYPE
-            if (schemeField.type == org.apache.thrift.protocol.TType.I16) {
-              struct.fetchType = iprot.readI16();
-              struct.setFetchTypeIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TFetchResultsReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.operationHandle != null) {
-        oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-        struct.operationHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.orientation != null) {
-        oprot.writeFieldBegin(ORIENTATION_FIELD_DESC);
-        oprot.writeI32(struct.orientation.getValue());
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldBegin(MAX_ROWS_FIELD_DESC);
-      oprot.writeI64(struct.maxRows);
-      oprot.writeFieldEnd();
-      if (struct.isSetFetchType()) {
-        oprot.writeFieldBegin(FETCH_TYPE_FIELD_DESC);
-        oprot.writeI16(struct.fetchType);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TFetchResultsReqTupleSchemeFactory implements SchemeFactory {
-    public TFetchResultsReqTupleScheme getScheme() {
-      return new TFetchResultsReqTupleScheme();
-    }
-  }
-
-  private static class TFetchResultsReqTupleScheme extends TupleScheme<TFetchResultsReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TFetchResultsReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.operationHandle.write(oprot);
-      oprot.writeI32(struct.orientation.getValue());
-      oprot.writeI64(struct.maxRows);
-      BitSet optionals = new BitSet();
-      if (struct.isSetFetchType()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetFetchType()) {
-        oprot.writeI16(struct.fetchType);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TFetchResultsReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.operationHandle = new TOperationHandle();
-      struct.operationHandle.read(iprot);
-      struct.setOperationHandleIsSet(true);
-      struct.orientation = org.apache.hive.service.rpc.thrift.TFetchOrientation.findByValue(iprot.readI32());
-      struct.setOrientationIsSet(true);
-      struct.maxRows = iprot.readI64();
-      struct.setMaxRowsIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.fetchType = iprot.readI16();
-        struct.setFetchTypeIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TFetchResultsResp.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TFetchResultsResp.java
deleted file mode 100644
index 8f86cee3ad468..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TFetchResultsResp.java
+++ /dev/null
@@ -1,612 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TFetchResultsResp implements org.apache.thrift.TBase<TFetchResultsResp, TFetchResultsResp._Fields>, java.io.Serializable, Cloneable, Comparable<TFetchResultsResp> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TFetchResultsResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField HAS_MORE_ROWS_FIELD_DESC = new org.apache.thrift.protocol.TField("hasMoreRows", org.apache.thrift.protocol.TType.BOOL, (short)2);
-  private static final org.apache.thrift.protocol.TField RESULTS_FIELD_DESC = new org.apache.thrift.protocol.TField("results", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TFetchResultsRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TFetchResultsRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private boolean hasMoreRows; // optional
-  private TRowSet results; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    HAS_MORE_ROWS((short)2, "hasMoreRows"),
-    RESULTS((short)3, "results");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // HAS_MORE_ROWS
-          return HAS_MORE_ROWS;
-        case 3: // RESULTS
-          return RESULTS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __HASMOREROWS_ISSET_ID = 0;
-  private byte __isset_bitfield = 0;
-  private static final _Fields optionals[] = {_Fields.HAS_MORE_ROWS,_Fields.RESULTS};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.HAS_MORE_ROWS, new org.apache.thrift.meta_data.FieldMetaData("hasMoreRows", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
-    tmpMap.put(_Fields.RESULTS, new org.apache.thrift.meta_data.FieldMetaData("results", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TRowSet.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TFetchResultsResp.class, metaDataMap);
-  }
-
-  public TFetchResultsResp() {
-  }
-
-  public TFetchResultsResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TFetchResultsResp(TFetchResultsResp other) {
-    __isset_bitfield = other.__isset_bitfield;
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    this.hasMoreRows = other.hasMoreRows;
-    if (other.isSetResults()) {
-      this.results = new TRowSet(other.results);
-    }
-  }
-
-  public TFetchResultsResp deepCopy() {
-    return new TFetchResultsResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    setHasMoreRowsIsSet(false);
-    this.hasMoreRows = false;
-    this.results = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public boolean isHasMoreRows() {
-    return this.hasMoreRows;
-  }
-
-  public void setHasMoreRows(boolean hasMoreRows) {
-    this.hasMoreRows = hasMoreRows;
-    setHasMoreRowsIsSet(true);
-  }
-
-  public void unsetHasMoreRows() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __HASMOREROWS_ISSET_ID);
-  }
-
-  /** Returns true if field hasMoreRows is set (has been assigned a value) and false otherwise */
-  public boolean isSetHasMoreRows() {
-    return EncodingUtils.testBit(__isset_bitfield, __HASMOREROWS_ISSET_ID);
-  }
-
-  public void setHasMoreRowsIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __HASMOREROWS_ISSET_ID, value);
-  }
-
-  public TRowSet getResults() {
-    return this.results;
-  }
-
-  public void setResults(TRowSet results) {
-    this.results = results;
-  }
-
-  public void unsetResults() {
-    this.results = null;
-  }
-
-  /** Returns true if field results is set (has been assigned a value) and false otherwise */
-  public boolean isSetResults() {
-    return this.results != null;
-  }
-
-  public void setResultsIsSet(boolean value) {
-    if (!value) {
-      this.results = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case HAS_MORE_ROWS:
-      if (value == null) {
-        unsetHasMoreRows();
-      } else {
-        setHasMoreRows((Boolean)value);
-      }
-      break;
-
-    case RESULTS:
-      if (value == null) {
-        unsetResults();
-      } else {
-        setResults((TRowSet)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case HAS_MORE_ROWS:
-      return isHasMoreRows();
-
-    case RESULTS:
-      return getResults();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case HAS_MORE_ROWS:
-      return isSetHasMoreRows();
-    case RESULTS:
-      return isSetResults();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TFetchResultsResp)
-      return this.equals((TFetchResultsResp)that);
-    return false;
-  }
-
-  public boolean equals(TFetchResultsResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_hasMoreRows = true && this.isSetHasMoreRows();
-    boolean that_present_hasMoreRows = true && that.isSetHasMoreRows();
-    if (this_present_hasMoreRows || that_present_hasMoreRows) {
-      if (!(this_present_hasMoreRows && that_present_hasMoreRows))
-        return false;
-      if (this.hasMoreRows != that.hasMoreRows)
-        return false;
-    }
-
-    boolean this_present_results = true && this.isSetResults();
-    boolean that_present_results = true && that.isSetResults();
-    if (this_present_results || that_present_results) {
-      if (!(this_present_results && that_present_results))
-        return false;
-      if (!this.results.equals(that.results))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_status = true && (isSetStatus());
-    list.add(present_status);
-    if (present_status)
-      list.add(status);
-
-    boolean present_hasMoreRows = true && (isSetHasMoreRows());
-    list.add(present_hasMoreRows);
-    if (present_hasMoreRows)
-      list.add(hasMoreRows);
-
-    boolean present_results = true && (isSetResults());
-    list.add(present_results);
-    if (present_results)
-      list.add(results);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TFetchResultsResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(other.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, other.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetHasMoreRows()).compareTo(other.isSetHasMoreRows());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetHasMoreRows()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.hasMoreRows, other.hasMoreRows);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetResults()).compareTo(other.isSetResults());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetResults()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.results, other.results);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TFetchResultsResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetHasMoreRows()) {
-      if (!first) sb.append(", ");
-      sb.append("hasMoreRows:");
-      sb.append(this.hasMoreRows);
-      first = false;
-    }
-    if (isSetResults()) {
-      if (!first) sb.append(", ");
-      sb.append("results:");
-      if (this.results == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.results);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-    if (results != null) {
-      results.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TFetchResultsRespStandardSchemeFactory implements SchemeFactory {
-    public TFetchResultsRespStandardScheme getScheme() {
-      return new TFetchResultsRespStandardScheme();
-    }
-  }
-
-  private static class TFetchResultsRespStandardScheme extends StandardScheme<TFetchResultsResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TFetchResultsResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // HAS_MORE_ROWS
-            if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
-              struct.hasMoreRows = iprot.readBool();
-              struct.setHasMoreRowsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // RESULTS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.results = new TRowSet();
-              struct.results.read(iprot);
-              struct.setResultsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TFetchResultsResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.isSetHasMoreRows()) {
-        oprot.writeFieldBegin(HAS_MORE_ROWS_FIELD_DESC);
-        oprot.writeBool(struct.hasMoreRows);
-        oprot.writeFieldEnd();
-      }
-      if (struct.results != null) {
-        if (struct.isSetResults()) {
-          oprot.writeFieldBegin(RESULTS_FIELD_DESC);
-          struct.results.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TFetchResultsRespTupleSchemeFactory implements SchemeFactory {
-    public TFetchResultsRespTupleScheme getScheme() {
-      return new TFetchResultsRespTupleScheme();
-    }
-  }
-
-  private static class TFetchResultsRespTupleScheme extends TupleScheme<TFetchResultsResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TFetchResultsResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetHasMoreRows()) {
-        optionals.set(0);
-      }
-      if (struct.isSetResults()) {
-        optionals.set(1);
-      }
-      oprot.writeBitSet(optionals, 2);
-      if (struct.isSetHasMoreRows()) {
-        oprot.writeBool(struct.hasMoreRows);
-      }
-      if (struct.isSetResults()) {
-        struct.results.write(oprot);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TFetchResultsResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(2);
-      if (incoming.get(0)) {
-        struct.hasMoreRows = iprot.readBool();
-        struct.setHasMoreRowsIsSet(true);
-      }
-      if (incoming.get(1)) {
-        struct.results = new TRowSet();
-        struct.results.read(iprot);
-        struct.setResultsIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetCatalogsReq.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetCatalogsReq.java
deleted file mode 100644
index b8a2ca6648069..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetCatalogsReq.java
+++ /dev/null
@@ -1,394 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetCatalogsReq implements org.apache.thrift.TBase<TGetCatalogsReq, TGetCatalogsReq._Fields>, java.io.Serializable, Cloneable, Comparable<TGetCatalogsReq> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetCatalogsReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetCatalogsReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetCatalogsReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetCatalogsReq.class, metaDataMap);
-  }
-
-  public TGetCatalogsReq() {
-  }
-
-  public TGetCatalogsReq(
-    TSessionHandle sessionHandle)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetCatalogsReq(TGetCatalogsReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-  }
-
-  public TGetCatalogsReq deepCopy() {
-    return new TGetCatalogsReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetCatalogsReq)
-      return this.equals((TGetCatalogsReq)that);
-    return false;
-  }
-
-  public boolean equals(TGetCatalogsReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    list.add(present_sessionHandle);
-    if (present_sessionHandle)
-      list.add(sessionHandle);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetCatalogsReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(other.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, other.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetCatalogsReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetCatalogsReqStandardSchemeFactory implements SchemeFactory {
-    public TGetCatalogsReqStandardScheme getScheme() {
-      return new TGetCatalogsReqStandardScheme();
-    }
-  }
-
-  private static class TGetCatalogsReqStandardScheme extends StandardScheme<TGetCatalogsReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetCatalogsReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetCatalogsReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetCatalogsReqTupleSchemeFactory implements SchemeFactory {
-    public TGetCatalogsReqTupleScheme getScheme() {
-      return new TGetCatalogsReqTupleScheme();
-    }
-  }
-
-  private static class TGetCatalogsReqTupleScheme extends TupleScheme<TGetCatalogsReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetCatalogsReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetCatalogsReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetCatalogsResp.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetCatalogsResp.java
deleted file mode 100644
index eeeac9a1f9292..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetCatalogsResp.java
+++ /dev/null
@@ -1,509 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetCatalogsResp implements org.apache.thrift.TBase<TGetCatalogsResp, TGetCatalogsResp._Fields>, java.io.Serializable, Cloneable, Comparable<TGetCatalogsResp> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetCatalogsResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetCatalogsRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetCatalogsRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TOperationHandle operationHandle; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    OPERATION_HANDLE((short)2, "operationHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final _Fields optionals[] = {_Fields.OPERATION_HANDLE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetCatalogsResp.class, metaDataMap);
-  }
-
-  public TGetCatalogsResp() {
-  }
-
-  public TGetCatalogsResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetCatalogsResp(TGetCatalogsResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-  }
-
-  public TGetCatalogsResp deepCopy() {
-    return new TGetCatalogsResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.operationHandle = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetCatalogsResp)
-      return this.equals((TGetCatalogsResp)that);
-    return false;
-  }
-
-  public boolean equals(TGetCatalogsResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_status = true && (isSetStatus());
-    list.add(present_status);
-    if (present_status)
-      list.add(status);
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    list.add(present_operationHandle);
-    if (present_operationHandle)
-      list.add(operationHandle);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetCatalogsResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(other.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, other.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(other.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, other.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetCatalogsResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetOperationHandle()) {
-      if (!first) sb.append(", ");
-      sb.append("operationHandle:");
-      if (this.operationHandle == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.operationHandle);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetCatalogsRespStandardSchemeFactory implements SchemeFactory {
-    public TGetCatalogsRespStandardScheme getScheme() {
-      return new TGetCatalogsRespStandardScheme();
-    }
-  }
-
-  private static class TGetCatalogsRespStandardScheme extends StandardScheme<TGetCatalogsResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetCatalogsResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetCatalogsResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.operationHandle != null) {
-        if (struct.isSetOperationHandle()) {
-          oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-          struct.operationHandle.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetCatalogsRespTupleSchemeFactory implements SchemeFactory {
-    public TGetCatalogsRespTupleScheme getScheme() {
-      return new TGetCatalogsRespTupleScheme();
-    }
-  }
-
-  private static class TGetCatalogsRespTupleScheme extends TupleScheme<TGetCatalogsResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetCatalogsResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetOperationHandle()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetOperationHandle()) {
-        struct.operationHandle.write(oprot);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetCatalogsResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.operationHandle = new TOperationHandle();
-        struct.operationHandle.read(iprot);
-        struct.setOperationHandleIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetColumnsReq.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetColumnsReq.java
deleted file mode 100644
index ba80279294957..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetColumnsReq.java
+++ /dev/null
@@ -1,822 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetColumnsReq implements org.apache.thrift.TBase<TGetColumnsReq, TGetColumnsReq._Fields>, java.io.Serializable, Cloneable, Comparable<TGetColumnsReq> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetColumnsReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField CATALOG_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("catalogName", org.apache.thrift.protocol.TType.STRING, (short)2);
-  private static final org.apache.thrift.protocol.TField SCHEMA_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("schemaName", org.apache.thrift.protocol.TType.STRING, (short)3);
-  private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)4);
-  private static final org.apache.thrift.protocol.TField COLUMN_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("columnName", org.apache.thrift.protocol.TType.STRING, (short)5);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetColumnsReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetColumnsReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-  private String catalogName; // optional
-  private String schemaName; // optional
-  private String tableName; // optional
-  private String columnName; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle"),
-    CATALOG_NAME((short)2, "catalogName"),
-    SCHEMA_NAME((short)3, "schemaName"),
-    TABLE_NAME((short)4, "tableName"),
-    COLUMN_NAME((short)5, "columnName");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        case 2: // CATALOG_NAME
-          return CATALOG_NAME;
-        case 3: // SCHEMA_NAME
-          return SCHEMA_NAME;
-        case 4: // TABLE_NAME
-          return TABLE_NAME;
-        case 5: // COLUMN_NAME
-          return COLUMN_NAME;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final _Fields optionals[] = {_Fields.CATALOG_NAME,_Fields.SCHEMA_NAME,_Fields.TABLE_NAME,_Fields.COLUMN_NAME};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    tmpMap.put(_Fields.CATALOG_NAME, new org.apache.thrift.meta_data.FieldMetaData("catalogName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TIdentifier")));
-    tmpMap.put(_Fields.SCHEMA_NAME, new org.apache.thrift.meta_data.FieldMetaData("schemaName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TPatternOrIdentifier")));
-    tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TPatternOrIdentifier")));
-    tmpMap.put(_Fields.COLUMN_NAME, new org.apache.thrift.meta_data.FieldMetaData("columnName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TPatternOrIdentifier")));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetColumnsReq.class, metaDataMap);
-  }
-
-  public TGetColumnsReq() {
-  }
-
-  public TGetColumnsReq(
-    TSessionHandle sessionHandle)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetColumnsReq(TGetColumnsReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-    if (other.isSetCatalogName()) {
-      this.catalogName = other.catalogName;
-    }
-    if (other.isSetSchemaName()) {
-      this.schemaName = other.schemaName;
-    }
-    if (other.isSetTableName()) {
-      this.tableName = other.tableName;
-    }
-    if (other.isSetColumnName()) {
-      this.columnName = other.columnName;
-    }
-  }
-
-  public TGetColumnsReq deepCopy() {
-    return new TGetColumnsReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-    this.catalogName = null;
-    this.schemaName = null;
-    this.tableName = null;
-    this.columnName = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public String getCatalogName() {
-    return this.catalogName;
-  }
-
-  public void setCatalogName(String catalogName) {
-    this.catalogName = catalogName;
-  }
-
-  public void unsetCatalogName() {
-    this.catalogName = null;
-  }
-
-  /** Returns true if field catalogName is set (has been assigned a value) and false otherwise */
-  public boolean isSetCatalogName() {
-    return this.catalogName != null;
-  }
-
-  public void setCatalogNameIsSet(boolean value) {
-    if (!value) {
-      this.catalogName = null;
-    }
-  }
-
-  public String getSchemaName() {
-    return this.schemaName;
-  }
-
-  public void setSchemaName(String schemaName) {
-    this.schemaName = schemaName;
-  }
-
-  public void unsetSchemaName() {
-    this.schemaName = null;
-  }
-
-  /** Returns true if field schemaName is set (has been assigned a value) and false otherwise */
-  public boolean isSetSchemaName() {
-    return this.schemaName != null;
-  }
-
-  public void setSchemaNameIsSet(boolean value) {
-    if (!value) {
-      this.schemaName = null;
-    }
-  }
-
-  public String getTableName() {
-    return this.tableName;
-  }
-
-  public void setTableName(String tableName) {
-    this.tableName = tableName;
-  }
-
-  public void unsetTableName() {
-    this.tableName = null;
-  }
-
-  /** Returns true if field tableName is set (has been assigned a value) and false otherwise */
-  public boolean isSetTableName() {
-    return this.tableName != null;
-  }
-
-  public void setTableNameIsSet(boolean value) {
-    if (!value) {
-      this.tableName = null;
-    }
-  }
-
-  public String getColumnName() {
-    return this.columnName;
-  }
-
-  public void setColumnName(String columnName) {
-    this.columnName = columnName;
-  }
-
-  public void unsetColumnName() {
-    this.columnName = null;
-  }
-
-  /** Returns true if field columnName is set (has been assigned a value) and false otherwise */
-  public boolean isSetColumnName() {
-    return this.columnName != null;
-  }
-
-  public void setColumnNameIsSet(boolean value) {
-    if (!value) {
-      this.columnName = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    case CATALOG_NAME:
-      if (value == null) {
-        unsetCatalogName();
-      } else {
-        setCatalogName((String)value);
-      }
-      break;
-
-    case SCHEMA_NAME:
-      if (value == null) {
-        unsetSchemaName();
-      } else {
-        setSchemaName((String)value);
-      }
-      break;
-
-    case TABLE_NAME:
-      if (value == null) {
-        unsetTableName();
-      } else {
-        setTableName((String)value);
-      }
-      break;
-
-    case COLUMN_NAME:
-      if (value == null) {
-        unsetColumnName();
-      } else {
-        setColumnName((String)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    case CATALOG_NAME:
-      return getCatalogName();
-
-    case SCHEMA_NAME:
-      return getSchemaName();
-
-    case TABLE_NAME:
-      return getTableName();
-
-    case COLUMN_NAME:
-      return getColumnName();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    case CATALOG_NAME:
-      return isSetCatalogName();
-    case SCHEMA_NAME:
-      return isSetSchemaName();
-    case TABLE_NAME:
-      return isSetTableName();
-    case COLUMN_NAME:
-      return isSetColumnName();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetColumnsReq)
-      return this.equals((TGetColumnsReq)that);
-    return false;
-  }
-
-  public boolean equals(TGetColumnsReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    boolean this_present_catalogName = true && this.isSetCatalogName();
-    boolean that_present_catalogName = true && that.isSetCatalogName();
-    if (this_present_catalogName || that_present_catalogName) {
-      if (!(this_present_catalogName && that_present_catalogName))
-        return false;
-      if (!this.catalogName.equals(that.catalogName))
-        return false;
-    }
-
-    boolean this_present_schemaName = true && this.isSetSchemaName();
-    boolean that_present_schemaName = true && that.isSetSchemaName();
-    if (this_present_schemaName || that_present_schemaName) {
-      if (!(this_present_schemaName && that_present_schemaName))
-        return false;
-      if (!this.schemaName.equals(that.schemaName))
-        return false;
-    }
-
-    boolean this_present_tableName = true && this.isSetTableName();
-    boolean that_present_tableName = true && that.isSetTableName();
-    if (this_present_tableName || that_present_tableName) {
-      if (!(this_present_tableName && that_present_tableName))
-        return false;
-      if (!this.tableName.equals(that.tableName))
-        return false;
-    }
-
-    boolean this_present_columnName = true && this.isSetColumnName();
-    boolean that_present_columnName = true && that.isSetColumnName();
-    if (this_present_columnName || that_present_columnName) {
-      if (!(this_present_columnName && that_present_columnName))
-        return false;
-      if (!this.columnName.equals(that.columnName))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    list.add(present_sessionHandle);
-    if (present_sessionHandle)
-      list.add(sessionHandle);
-
-    boolean present_catalogName = true && (isSetCatalogName());
-    list.add(present_catalogName);
-    if (present_catalogName)
-      list.add(catalogName);
-
-    boolean present_schemaName = true && (isSetSchemaName());
-    list.add(present_schemaName);
-    if (present_schemaName)
-      list.add(schemaName);
-
-    boolean present_tableName = true && (isSetTableName());
-    list.add(present_tableName);
-    if (present_tableName)
-      list.add(tableName);
-
-    boolean present_columnName = true && (isSetColumnName());
-    list.add(present_columnName);
-    if (present_columnName)
-      list.add(columnName);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetColumnsReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(other.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, other.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetCatalogName()).compareTo(other.isSetCatalogName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetCatalogName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.catalogName, other.catalogName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetSchemaName()).compareTo(other.isSetSchemaName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSchemaName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.schemaName, other.schemaName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetTableName()).compareTo(other.isSetTableName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetTableName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, other.tableName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetColumnName()).compareTo(other.isSetColumnName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetColumnName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columnName, other.columnName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetColumnsReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    if (isSetCatalogName()) {
-      if (!first) sb.append(", ");
-      sb.append("catalogName:");
-      if (this.catalogName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.catalogName);
-      }
-      first = false;
-    }
-    if (isSetSchemaName()) {
-      if (!first) sb.append(", ");
-      sb.append("schemaName:");
-      if (this.schemaName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.schemaName);
-      }
-      first = false;
-    }
-    if (isSetTableName()) {
-      if (!first) sb.append(", ");
-      sb.append("tableName:");
-      if (this.tableName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.tableName);
-      }
-      first = false;
-    }
-    if (isSetColumnName()) {
-      if (!first) sb.append(", ");
-      sb.append("columnName:");
-      if (this.columnName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.columnName);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetColumnsReqStandardSchemeFactory implements SchemeFactory {
-    public TGetColumnsReqStandardScheme getScheme() {
-      return new TGetColumnsReqStandardScheme();
-    }
-  }
-
-  private static class TGetColumnsReqStandardScheme extends StandardScheme<TGetColumnsReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetColumnsReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // CATALOG_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.catalogName = iprot.readString();
-              struct.setCatalogNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // SCHEMA_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.schemaName = iprot.readString();
-              struct.setSchemaNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // TABLE_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.tableName = iprot.readString();
-              struct.setTableNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 5: // COLUMN_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.columnName = iprot.readString();
-              struct.setColumnNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetColumnsReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.catalogName != null) {
-        if (struct.isSetCatalogName()) {
-          oprot.writeFieldBegin(CATALOG_NAME_FIELD_DESC);
-          oprot.writeString(struct.catalogName);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.schemaName != null) {
-        if (struct.isSetSchemaName()) {
-          oprot.writeFieldBegin(SCHEMA_NAME_FIELD_DESC);
-          oprot.writeString(struct.schemaName);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.tableName != null) {
-        if (struct.isSetTableName()) {
-          oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC);
-          oprot.writeString(struct.tableName);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.columnName != null) {
-        if (struct.isSetColumnName()) {
-          oprot.writeFieldBegin(COLUMN_NAME_FIELD_DESC);
-          oprot.writeString(struct.columnName);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetColumnsReqTupleSchemeFactory implements SchemeFactory {
-    public TGetColumnsReqTupleScheme getScheme() {
-      return new TGetColumnsReqTupleScheme();
-    }
-  }
-
-  private static class TGetColumnsReqTupleScheme extends TupleScheme<TGetColumnsReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetColumnsReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetCatalogName()) {
-        optionals.set(0);
-      }
-      if (struct.isSetSchemaName()) {
-        optionals.set(1);
-      }
-      if (struct.isSetTableName()) {
-        optionals.set(2);
-      }
-      if (struct.isSetColumnName()) {
-        optionals.set(3);
-      }
-      oprot.writeBitSet(optionals, 4);
-      if (struct.isSetCatalogName()) {
-        oprot.writeString(struct.catalogName);
-      }
-      if (struct.isSetSchemaName()) {
-        oprot.writeString(struct.schemaName);
-      }
-      if (struct.isSetTableName()) {
-        oprot.writeString(struct.tableName);
-      }
-      if (struct.isSetColumnName()) {
-        oprot.writeString(struct.columnName);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetColumnsReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-      BitSet incoming = iprot.readBitSet(4);
-      if (incoming.get(0)) {
-        struct.catalogName = iprot.readString();
-        struct.setCatalogNameIsSet(true);
-      }
-      if (incoming.get(1)) {
-        struct.schemaName = iprot.readString();
-        struct.setSchemaNameIsSet(true);
-      }
-      if (incoming.get(2)) {
-        struct.tableName = iprot.readString();
-        struct.setTableNameIsSet(true);
-      }
-      if (incoming.get(3)) {
-        struct.columnName = iprot.readString();
-        struct.setColumnNameIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetColumnsResp.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetColumnsResp.java
deleted file mode 100644
index c68aac9042fc1..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetColumnsResp.java
+++ /dev/null
@@ -1,509 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetColumnsResp implements org.apache.thrift.TBase<TGetColumnsResp, TGetColumnsResp._Fields>, java.io.Serializable, Cloneable, Comparable<TGetColumnsResp> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetColumnsResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetColumnsRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetColumnsRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TOperationHandle operationHandle; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    OPERATION_HANDLE((short)2, "operationHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final _Fields optionals[] = {_Fields.OPERATION_HANDLE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetColumnsResp.class, metaDataMap);
-  }
-
-  public TGetColumnsResp() {
-  }
-
-  public TGetColumnsResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetColumnsResp(TGetColumnsResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-  }
-
-  public TGetColumnsResp deepCopy() {
-    return new TGetColumnsResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.operationHandle = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetColumnsResp)
-      return this.equals((TGetColumnsResp)that);
-    return false;
-  }
-
-  public boolean equals(TGetColumnsResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_status = true && (isSetStatus());
-    list.add(present_status);
-    if (present_status)
-      list.add(status);
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    list.add(present_operationHandle);
-    if (present_operationHandle)
-      list.add(operationHandle);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetColumnsResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(other.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, other.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(other.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, other.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetColumnsResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetOperationHandle()) {
-      if (!first) sb.append(", ");
-      sb.append("operationHandle:");
-      if (this.operationHandle == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.operationHandle);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetColumnsRespStandardSchemeFactory implements SchemeFactory {
-    public TGetColumnsRespStandardScheme getScheme() {
-      return new TGetColumnsRespStandardScheme();
-    }
-  }
-
-  private static class TGetColumnsRespStandardScheme extends StandardScheme<TGetColumnsResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetColumnsResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetColumnsResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.operationHandle != null) {
-        if (struct.isSetOperationHandle()) {
-          oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-          struct.operationHandle.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetColumnsRespTupleSchemeFactory implements SchemeFactory {
-    public TGetColumnsRespTupleScheme getScheme() {
-      return new TGetColumnsRespTupleScheme();
-    }
-  }
-
-  private static class TGetColumnsRespTupleScheme extends TupleScheme<TGetColumnsResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetColumnsResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetOperationHandle()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetOperationHandle()) {
-        struct.operationHandle.write(oprot);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetColumnsResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.operationHandle = new TOperationHandle();
-        struct.operationHandle.read(iprot);
-        struct.setOperationHandleIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetCrossReferenceReq.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetCrossReferenceReq.java
deleted file mode 100644
index 972957063b297..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetCrossReferenceReq.java
+++ /dev/null
@@ -1,1034 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetCrossReferenceReq implements org.apache.thrift.TBase<TGetCrossReferenceReq, TGetCrossReferenceReq._Fields>, java.io.Serializable, Cloneable, Comparable<TGetCrossReferenceReq> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetCrossReferenceReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField PARENT_CATALOG_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("parentCatalogName", org.apache.thrift.protocol.TType.STRING, (short)2);
-  private static final org.apache.thrift.protocol.TField PARENT_SCHEMA_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("parentSchemaName", org.apache.thrift.protocol.TType.STRING, (short)3);
-  private static final org.apache.thrift.protocol.TField PARENT_TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("parentTableName", org.apache.thrift.protocol.TType.STRING, (short)4);
-  private static final org.apache.thrift.protocol.TField FOREIGN_CATALOG_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("foreignCatalogName", org.apache.thrift.protocol.TType.STRING, (short)5);
-  private static final org.apache.thrift.protocol.TField FOREIGN_SCHEMA_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("foreignSchemaName", org.apache.thrift.protocol.TType.STRING, (short)6);
-  private static final org.apache.thrift.protocol.TField FOREIGN_TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("foreignTableName", org.apache.thrift.protocol.TType.STRING, (short)7);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetCrossReferenceReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetCrossReferenceReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-  private String parentCatalogName; // optional
-  private String parentSchemaName; // optional
-  private String parentTableName; // optional
-  private String foreignCatalogName; // optional
-  private String foreignSchemaName; // optional
-  private String foreignTableName; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle"),
-    PARENT_CATALOG_NAME((short)2, "parentCatalogName"),
-    PARENT_SCHEMA_NAME((short)3, "parentSchemaName"),
-    PARENT_TABLE_NAME((short)4, "parentTableName"),
-    FOREIGN_CATALOG_NAME((short)5, "foreignCatalogName"),
-    FOREIGN_SCHEMA_NAME((short)6, "foreignSchemaName"),
-    FOREIGN_TABLE_NAME((short)7, "foreignTableName");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        case 2: // PARENT_CATALOG_NAME
-          return PARENT_CATALOG_NAME;
-        case 3: // PARENT_SCHEMA_NAME
-          return PARENT_SCHEMA_NAME;
-        case 4: // PARENT_TABLE_NAME
-          return PARENT_TABLE_NAME;
-        case 5: // FOREIGN_CATALOG_NAME
-          return FOREIGN_CATALOG_NAME;
-        case 6: // FOREIGN_SCHEMA_NAME
-          return FOREIGN_SCHEMA_NAME;
-        case 7: // FOREIGN_TABLE_NAME
-          return FOREIGN_TABLE_NAME;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final _Fields optionals[] = {_Fields.PARENT_CATALOG_NAME,_Fields.PARENT_SCHEMA_NAME,_Fields.PARENT_TABLE_NAME,_Fields.FOREIGN_CATALOG_NAME,_Fields.FOREIGN_SCHEMA_NAME,_Fields.FOREIGN_TABLE_NAME};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    tmpMap.put(_Fields.PARENT_CATALOG_NAME, new org.apache.thrift.meta_data.FieldMetaData("parentCatalogName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TIdentifier")));
-    tmpMap.put(_Fields.PARENT_SCHEMA_NAME, new org.apache.thrift.meta_data.FieldMetaData("parentSchemaName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TIdentifier")));
-    tmpMap.put(_Fields.PARENT_TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("parentTableName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TIdentifier")));
-    tmpMap.put(_Fields.FOREIGN_CATALOG_NAME, new org.apache.thrift.meta_data.FieldMetaData("foreignCatalogName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TIdentifier")));
-    tmpMap.put(_Fields.FOREIGN_SCHEMA_NAME, new org.apache.thrift.meta_data.FieldMetaData("foreignSchemaName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TIdentifier")));
-    tmpMap.put(_Fields.FOREIGN_TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("foreignTableName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TIdentifier")));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetCrossReferenceReq.class, metaDataMap);
-  }
-
-  public TGetCrossReferenceReq() {
-  }
-
-  public TGetCrossReferenceReq(
-    TSessionHandle sessionHandle)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetCrossReferenceReq(TGetCrossReferenceReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-    if (other.isSetParentCatalogName()) {
-      this.parentCatalogName = other.parentCatalogName;
-    }
-    if (other.isSetParentSchemaName()) {
-      this.parentSchemaName = other.parentSchemaName;
-    }
-    if (other.isSetParentTableName()) {
-      this.parentTableName = other.parentTableName;
-    }
-    if (other.isSetForeignCatalogName()) {
-      this.foreignCatalogName = other.foreignCatalogName;
-    }
-    if (other.isSetForeignSchemaName()) {
-      this.foreignSchemaName = other.foreignSchemaName;
-    }
-    if (other.isSetForeignTableName()) {
-      this.foreignTableName = other.foreignTableName;
-    }
-  }
-
-  public TGetCrossReferenceReq deepCopy() {
-    return new TGetCrossReferenceReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-    this.parentCatalogName = null;
-    this.parentSchemaName = null;
-    this.parentTableName = null;
-    this.foreignCatalogName = null;
-    this.foreignSchemaName = null;
-    this.foreignTableName = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public String getParentCatalogName() {
-    return this.parentCatalogName;
-  }
-
-  public void setParentCatalogName(String parentCatalogName) {
-    this.parentCatalogName = parentCatalogName;
-  }
-
-  public void unsetParentCatalogName() {
-    this.parentCatalogName = null;
-  }
-
-  /** Returns true if field parentCatalogName is set (has been assigned a value) and false otherwise */
-  public boolean isSetParentCatalogName() {
-    return this.parentCatalogName != null;
-  }
-
-  public void setParentCatalogNameIsSet(boolean value) {
-    if (!value) {
-      this.parentCatalogName = null;
-    }
-  }
-
-  public String getParentSchemaName() {
-    return this.parentSchemaName;
-  }
-
-  public void setParentSchemaName(String parentSchemaName) {
-    this.parentSchemaName = parentSchemaName;
-  }
-
-  public void unsetParentSchemaName() {
-    this.parentSchemaName = null;
-  }
-
-  /** Returns true if field parentSchemaName is set (has been assigned a value) and false otherwise */
-  public boolean isSetParentSchemaName() {
-    return this.parentSchemaName != null;
-  }
-
-  public void setParentSchemaNameIsSet(boolean value) {
-    if (!value) {
-      this.parentSchemaName = null;
-    }
-  }
-
-  public String getParentTableName() {
-    return this.parentTableName;
-  }
-
-  public void setParentTableName(String parentTableName) {
-    this.parentTableName = parentTableName;
-  }
-
-  public void unsetParentTableName() {
-    this.parentTableName = null;
-  }
-
-  /** Returns true if field parentTableName is set (has been assigned a value) and false otherwise */
-  public boolean isSetParentTableName() {
-    return this.parentTableName != null;
-  }
-
-  public void setParentTableNameIsSet(boolean value) {
-    if (!value) {
-      this.parentTableName = null;
-    }
-  }
-
-  public String getForeignCatalogName() {
-    return this.foreignCatalogName;
-  }
-
-  public void setForeignCatalogName(String foreignCatalogName) {
-    this.foreignCatalogName = foreignCatalogName;
-  }
-
-  public void unsetForeignCatalogName() {
-    this.foreignCatalogName = null;
-  }
-
-  /** Returns true if field foreignCatalogName is set (has been assigned a value) and false otherwise */
-  public boolean isSetForeignCatalogName() {
-    return this.foreignCatalogName != null;
-  }
-
-  public void setForeignCatalogNameIsSet(boolean value) {
-    if (!value) {
-      this.foreignCatalogName = null;
-    }
-  }
-
-  public String getForeignSchemaName() {
-    return this.foreignSchemaName;
-  }
-
-  public void setForeignSchemaName(String foreignSchemaName) {
-    this.foreignSchemaName = foreignSchemaName;
-  }
-
-  public void unsetForeignSchemaName() {
-    this.foreignSchemaName = null;
-  }
-
-  /** Returns true if field foreignSchemaName is set (has been assigned a value) and false otherwise */
-  public boolean isSetForeignSchemaName() {
-    return this.foreignSchemaName != null;
-  }
-
-  public void setForeignSchemaNameIsSet(boolean value) {
-    if (!value) {
-      this.foreignSchemaName = null;
-    }
-  }
-
-  public String getForeignTableName() {
-    return this.foreignTableName;
-  }
-
-  public void setForeignTableName(String foreignTableName) {
-    this.foreignTableName = foreignTableName;
-  }
-
-  public void unsetForeignTableName() {
-    this.foreignTableName = null;
-  }
-
-  /** Returns true if field foreignTableName is set (has been assigned a value) and false otherwise */
-  public boolean isSetForeignTableName() {
-    return this.foreignTableName != null;
-  }
-
-  public void setForeignTableNameIsSet(boolean value) {
-    if (!value) {
-      this.foreignTableName = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    case PARENT_CATALOG_NAME:
-      if (value == null) {
-        unsetParentCatalogName();
-      } else {
-        setParentCatalogName((String)value);
-      }
-      break;
-
-    case PARENT_SCHEMA_NAME:
-      if (value == null) {
-        unsetParentSchemaName();
-      } else {
-        setParentSchemaName((String)value);
-      }
-      break;
-
-    case PARENT_TABLE_NAME:
-      if (value == null) {
-        unsetParentTableName();
-      } else {
-        setParentTableName((String)value);
-      }
-      break;
-
-    case FOREIGN_CATALOG_NAME:
-      if (value == null) {
-        unsetForeignCatalogName();
-      } else {
-        setForeignCatalogName((String)value);
-      }
-      break;
-
-    case FOREIGN_SCHEMA_NAME:
-      if (value == null) {
-        unsetForeignSchemaName();
-      } else {
-        setForeignSchemaName((String)value);
-      }
-      break;
-
-    case FOREIGN_TABLE_NAME:
-      if (value == null) {
-        unsetForeignTableName();
-      } else {
-        setForeignTableName((String)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    case PARENT_CATALOG_NAME:
-      return getParentCatalogName();
-
-    case PARENT_SCHEMA_NAME:
-      return getParentSchemaName();
-
-    case PARENT_TABLE_NAME:
-      return getParentTableName();
-
-    case FOREIGN_CATALOG_NAME:
-      return getForeignCatalogName();
-
-    case FOREIGN_SCHEMA_NAME:
-      return getForeignSchemaName();
-
-    case FOREIGN_TABLE_NAME:
-      return getForeignTableName();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    case PARENT_CATALOG_NAME:
-      return isSetParentCatalogName();
-    case PARENT_SCHEMA_NAME:
-      return isSetParentSchemaName();
-    case PARENT_TABLE_NAME:
-      return isSetParentTableName();
-    case FOREIGN_CATALOG_NAME:
-      return isSetForeignCatalogName();
-    case FOREIGN_SCHEMA_NAME:
-      return isSetForeignSchemaName();
-    case FOREIGN_TABLE_NAME:
-      return isSetForeignTableName();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetCrossReferenceReq)
-      return this.equals((TGetCrossReferenceReq)that);
-    return false;
-  }
-
-  public boolean equals(TGetCrossReferenceReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    boolean this_present_parentCatalogName = true && this.isSetParentCatalogName();
-    boolean that_present_parentCatalogName = true && that.isSetParentCatalogName();
-    if (this_present_parentCatalogName || that_present_parentCatalogName) {
-      if (!(this_present_parentCatalogName && that_present_parentCatalogName))
-        return false;
-      if (!this.parentCatalogName.equals(that.parentCatalogName))
-        return false;
-    }
-
-    boolean this_present_parentSchemaName = true && this.isSetParentSchemaName();
-    boolean that_present_parentSchemaName = true && that.isSetParentSchemaName();
-    if (this_present_parentSchemaName || that_present_parentSchemaName) {
-      if (!(this_present_parentSchemaName && that_present_parentSchemaName))
-        return false;
-      if (!this.parentSchemaName.equals(that.parentSchemaName))
-        return false;
-    }
-
-    boolean this_present_parentTableName = true && this.isSetParentTableName();
-    boolean that_present_parentTableName = true && that.isSetParentTableName();
-    if (this_present_parentTableName || that_present_parentTableName) {
-      if (!(this_present_parentTableName && that_present_parentTableName))
-        return false;
-      if (!this.parentTableName.equals(that.parentTableName))
-        return false;
-    }
-
-    boolean this_present_foreignCatalogName = true && this.isSetForeignCatalogName();
-    boolean that_present_foreignCatalogName = true && that.isSetForeignCatalogName();
-    if (this_present_foreignCatalogName || that_present_foreignCatalogName) {
-      if (!(this_present_foreignCatalogName && that_present_foreignCatalogName))
-        return false;
-      if (!this.foreignCatalogName.equals(that.foreignCatalogName))
-        return false;
-    }
-
-    boolean this_present_foreignSchemaName = true && this.isSetForeignSchemaName();
-    boolean that_present_foreignSchemaName = true && that.isSetForeignSchemaName();
-    if (this_present_foreignSchemaName || that_present_foreignSchemaName) {
-      if (!(this_present_foreignSchemaName && that_present_foreignSchemaName))
-        return false;
-      if (!this.foreignSchemaName.equals(that.foreignSchemaName))
-        return false;
-    }
-
-    boolean this_present_foreignTableName = true && this.isSetForeignTableName();
-    boolean that_present_foreignTableName = true && that.isSetForeignTableName();
-    if (this_present_foreignTableName || that_present_foreignTableName) {
-      if (!(this_present_foreignTableName && that_present_foreignTableName))
-        return false;
-      if (!this.foreignTableName.equals(that.foreignTableName))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    list.add(present_sessionHandle);
-    if (present_sessionHandle)
-      list.add(sessionHandle);
-
-    boolean present_parentCatalogName = true && (isSetParentCatalogName());
-    list.add(present_parentCatalogName);
-    if (present_parentCatalogName)
-      list.add(parentCatalogName);
-
-    boolean present_parentSchemaName = true && (isSetParentSchemaName());
-    list.add(present_parentSchemaName);
-    if (present_parentSchemaName)
-      list.add(parentSchemaName);
-
-    boolean present_parentTableName = true && (isSetParentTableName());
-    list.add(present_parentTableName);
-    if (present_parentTableName)
-      list.add(parentTableName);
-
-    boolean present_foreignCatalogName = true && (isSetForeignCatalogName());
-    list.add(present_foreignCatalogName);
-    if (present_foreignCatalogName)
-      list.add(foreignCatalogName);
-
-    boolean present_foreignSchemaName = true && (isSetForeignSchemaName());
-    list.add(present_foreignSchemaName);
-    if (present_foreignSchemaName)
-      list.add(foreignSchemaName);
-
-    boolean present_foreignTableName = true && (isSetForeignTableName());
-    list.add(present_foreignTableName);
-    if (present_foreignTableName)
-      list.add(foreignTableName);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetCrossReferenceReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(other.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, other.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetParentCatalogName()).compareTo(other.isSetParentCatalogName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetParentCatalogName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.parentCatalogName, other.parentCatalogName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetParentSchemaName()).compareTo(other.isSetParentSchemaName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetParentSchemaName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.parentSchemaName, other.parentSchemaName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetParentTableName()).compareTo(other.isSetParentTableName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetParentTableName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.parentTableName, other.parentTableName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetForeignCatalogName()).compareTo(other.isSetForeignCatalogName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetForeignCatalogName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.foreignCatalogName, other.foreignCatalogName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetForeignSchemaName()).compareTo(other.isSetForeignSchemaName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetForeignSchemaName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.foreignSchemaName, other.foreignSchemaName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetForeignTableName()).compareTo(other.isSetForeignTableName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetForeignTableName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.foreignTableName, other.foreignTableName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetCrossReferenceReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    if (isSetParentCatalogName()) {
-      if (!first) sb.append(", ");
-      sb.append("parentCatalogName:");
-      if (this.parentCatalogName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.parentCatalogName);
-      }
-      first = false;
-    }
-    if (isSetParentSchemaName()) {
-      if (!first) sb.append(", ");
-      sb.append("parentSchemaName:");
-      if (this.parentSchemaName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.parentSchemaName);
-      }
-      first = false;
-    }
-    if (isSetParentTableName()) {
-      if (!first) sb.append(", ");
-      sb.append("parentTableName:");
-      if (this.parentTableName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.parentTableName);
-      }
-      first = false;
-    }
-    if (isSetForeignCatalogName()) {
-      if (!first) sb.append(", ");
-      sb.append("foreignCatalogName:");
-      if (this.foreignCatalogName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.foreignCatalogName);
-      }
-      first = false;
-    }
-    if (isSetForeignSchemaName()) {
-      if (!first) sb.append(", ");
-      sb.append("foreignSchemaName:");
-      if (this.foreignSchemaName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.foreignSchemaName);
-      }
-      first = false;
-    }
-    if (isSetForeignTableName()) {
-      if (!first) sb.append(", ");
-      sb.append("foreignTableName:");
-      if (this.foreignTableName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.foreignTableName);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetCrossReferenceReqStandardSchemeFactory implements SchemeFactory {
-    public TGetCrossReferenceReqStandardScheme getScheme() {
-      return new TGetCrossReferenceReqStandardScheme();
-    }
-  }
-
-  private static class TGetCrossReferenceReqStandardScheme extends StandardScheme<TGetCrossReferenceReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetCrossReferenceReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // PARENT_CATALOG_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.parentCatalogName = iprot.readString();
-              struct.setParentCatalogNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // PARENT_SCHEMA_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.parentSchemaName = iprot.readString();
-              struct.setParentSchemaNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // PARENT_TABLE_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.parentTableName = iprot.readString();
-              struct.setParentTableNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 5: // FOREIGN_CATALOG_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.foreignCatalogName = iprot.readString();
-              struct.setForeignCatalogNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 6: // FOREIGN_SCHEMA_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.foreignSchemaName = iprot.readString();
-              struct.setForeignSchemaNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 7: // FOREIGN_TABLE_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.foreignTableName = iprot.readString();
-              struct.setForeignTableNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetCrossReferenceReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.parentCatalogName != null) {
-        if (struct.isSetParentCatalogName()) {
-          oprot.writeFieldBegin(PARENT_CATALOG_NAME_FIELD_DESC);
-          oprot.writeString(struct.parentCatalogName);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.parentSchemaName != null) {
-        if (struct.isSetParentSchemaName()) {
-          oprot.writeFieldBegin(PARENT_SCHEMA_NAME_FIELD_DESC);
-          oprot.writeString(struct.parentSchemaName);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.parentTableName != null) {
-        if (struct.isSetParentTableName()) {
-          oprot.writeFieldBegin(PARENT_TABLE_NAME_FIELD_DESC);
-          oprot.writeString(struct.parentTableName);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.foreignCatalogName != null) {
-        if (struct.isSetForeignCatalogName()) {
-          oprot.writeFieldBegin(FOREIGN_CATALOG_NAME_FIELD_DESC);
-          oprot.writeString(struct.foreignCatalogName);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.foreignSchemaName != null) {
-        if (struct.isSetForeignSchemaName()) {
-          oprot.writeFieldBegin(FOREIGN_SCHEMA_NAME_FIELD_DESC);
-          oprot.writeString(struct.foreignSchemaName);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.foreignTableName != null) {
-        if (struct.isSetForeignTableName()) {
-          oprot.writeFieldBegin(FOREIGN_TABLE_NAME_FIELD_DESC);
-          oprot.writeString(struct.foreignTableName);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetCrossReferenceReqTupleSchemeFactory implements SchemeFactory {
-    public TGetCrossReferenceReqTupleScheme getScheme() {
-      return new TGetCrossReferenceReqTupleScheme();
-    }
-  }
-
-  private static class TGetCrossReferenceReqTupleScheme extends TupleScheme<TGetCrossReferenceReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetCrossReferenceReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetParentCatalogName()) {
-        optionals.set(0);
-      }
-      if (struct.isSetParentSchemaName()) {
-        optionals.set(1);
-      }
-      if (struct.isSetParentTableName()) {
-        optionals.set(2);
-      }
-      if (struct.isSetForeignCatalogName()) {
-        optionals.set(3);
-      }
-      if (struct.isSetForeignSchemaName()) {
-        optionals.set(4);
-      }
-      if (struct.isSetForeignTableName()) {
-        optionals.set(5);
-      }
-      oprot.writeBitSet(optionals, 6);
-      if (struct.isSetParentCatalogName()) {
-        oprot.writeString(struct.parentCatalogName);
-      }
-      if (struct.isSetParentSchemaName()) {
-        oprot.writeString(struct.parentSchemaName);
-      }
-      if (struct.isSetParentTableName()) {
-        oprot.writeString(struct.parentTableName);
-      }
-      if (struct.isSetForeignCatalogName()) {
-        oprot.writeString(struct.foreignCatalogName);
-      }
-      if (struct.isSetForeignSchemaName()) {
-        oprot.writeString(struct.foreignSchemaName);
-      }
-      if (struct.isSetForeignTableName()) {
-        oprot.writeString(struct.foreignTableName);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetCrossReferenceReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-      BitSet incoming = iprot.readBitSet(6);
-      if (incoming.get(0)) {
-        struct.parentCatalogName = iprot.readString();
-        struct.setParentCatalogNameIsSet(true);
-      }
-      if (incoming.get(1)) {
-        struct.parentSchemaName = iprot.readString();
-        struct.setParentSchemaNameIsSet(true);
-      }
-      if (incoming.get(2)) {
-        struct.parentTableName = iprot.readString();
-        struct.setParentTableNameIsSet(true);
-      }
-      if (incoming.get(3)) {
-        struct.foreignCatalogName = iprot.readString();
-        struct.setForeignCatalogNameIsSet(true);
-      }
-      if (incoming.get(4)) {
-        struct.foreignSchemaName = iprot.readString();
-        struct.setForeignSchemaNameIsSet(true);
-      }
-      if (incoming.get(5)) {
-        struct.foreignTableName = iprot.readString();
-        struct.setForeignTableNameIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetCrossReferenceResp.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetCrossReferenceResp.java
deleted file mode 100644
index 1bfe6d192df06..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetCrossReferenceResp.java
+++ /dev/null
@@ -1,509 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetCrossReferenceResp implements org.apache.thrift.TBase<TGetCrossReferenceResp, TGetCrossReferenceResp._Fields>, java.io.Serializable, Cloneable, Comparable<TGetCrossReferenceResp> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetCrossReferenceResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetCrossReferenceRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetCrossReferenceRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TOperationHandle operationHandle; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    OPERATION_HANDLE((short)2, "operationHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final _Fields optionals[] = {_Fields.OPERATION_HANDLE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetCrossReferenceResp.class, metaDataMap);
-  }
-
-  public TGetCrossReferenceResp() {
-  }
-
-  public TGetCrossReferenceResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetCrossReferenceResp(TGetCrossReferenceResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-  }
-
-  public TGetCrossReferenceResp deepCopy() {
-    return new TGetCrossReferenceResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.operationHandle = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetCrossReferenceResp)
-      return this.equals((TGetCrossReferenceResp)that);
-    return false;
-  }
-
-  public boolean equals(TGetCrossReferenceResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_status = true && (isSetStatus());
-    list.add(present_status);
-    if (present_status)
-      list.add(status);
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    list.add(present_operationHandle);
-    if (present_operationHandle)
-      list.add(operationHandle);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetCrossReferenceResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(other.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, other.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(other.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, other.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetCrossReferenceResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetOperationHandle()) {
-      if (!first) sb.append(", ");
-      sb.append("operationHandle:");
-      if (this.operationHandle == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.operationHandle);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetCrossReferenceRespStandardSchemeFactory implements SchemeFactory {
-    public TGetCrossReferenceRespStandardScheme getScheme() {
-      return new TGetCrossReferenceRespStandardScheme();
-    }
-  }
-
-  private static class TGetCrossReferenceRespStandardScheme extends StandardScheme<TGetCrossReferenceResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetCrossReferenceResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetCrossReferenceResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.operationHandle != null) {
-        if (struct.isSetOperationHandle()) {
-          oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-          struct.operationHandle.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetCrossReferenceRespTupleSchemeFactory implements SchemeFactory {
-    public TGetCrossReferenceRespTupleScheme getScheme() {
-      return new TGetCrossReferenceRespTupleScheme();
-    }
-  }
-
-  private static class TGetCrossReferenceRespTupleScheme extends TupleScheme<TGetCrossReferenceResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetCrossReferenceResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetOperationHandle()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetOperationHandle()) {
-        struct.operationHandle.write(oprot);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetCrossReferenceResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.operationHandle = new TOperationHandle();
-        struct.operationHandle.read(iprot);
-        struct.setOperationHandleIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetDelegationTokenReq.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetDelegationTokenReq.java
deleted file mode 100644
index e3e28c5860522..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetDelegationTokenReq.java
+++ /dev/null
@@ -1,596 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetDelegationTokenReq implements org.apache.thrift.TBase<TGetDelegationTokenReq, TGetDelegationTokenReq._Fields>, java.io.Serializable, Cloneable, Comparable<TGetDelegationTokenReq> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetDelegationTokenReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField OWNER_FIELD_DESC = new org.apache.thrift.protocol.TField("owner", org.apache.thrift.protocol.TType.STRING, (short)2);
-  private static final org.apache.thrift.protocol.TField RENEWER_FIELD_DESC = new org.apache.thrift.protocol.TField("renewer", org.apache.thrift.protocol.TType.STRING, (short)3);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetDelegationTokenReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetDelegationTokenReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-  private String owner; // required
-  private String renewer; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle"),
-    OWNER((short)2, "owner"),
-    RENEWER((short)3, "renewer");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        case 2: // OWNER
-          return OWNER;
-        case 3: // RENEWER
-          return RENEWER;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    tmpMap.put(_Fields.OWNER, new org.apache.thrift.meta_data.FieldMetaData("owner", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    tmpMap.put(_Fields.RENEWER, new org.apache.thrift.meta_data.FieldMetaData("renewer", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetDelegationTokenReq.class, metaDataMap);
-  }
-
-  public TGetDelegationTokenReq() {
-  }
-
-  public TGetDelegationTokenReq(
-    TSessionHandle sessionHandle,
-    String owner,
-    String renewer)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-    this.owner = owner;
-    this.renewer = renewer;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetDelegationTokenReq(TGetDelegationTokenReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-    if (other.isSetOwner()) {
-      this.owner = other.owner;
-    }
-    if (other.isSetRenewer()) {
-      this.renewer = other.renewer;
-    }
-  }
-
-  public TGetDelegationTokenReq deepCopy() {
-    return new TGetDelegationTokenReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-    this.owner = null;
-    this.renewer = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public String getOwner() {
-    return this.owner;
-  }
-
-  public void setOwner(String owner) {
-    this.owner = owner;
-  }
-
-  public void unsetOwner() {
-    this.owner = null;
-  }
-
-  /** Returns true if field owner is set (has been assigned a value) and false otherwise */
-  public boolean isSetOwner() {
-    return this.owner != null;
-  }
-
-  public void setOwnerIsSet(boolean value) {
-    if (!value) {
-      this.owner = null;
-    }
-  }
-
-  public String getRenewer() {
-    return this.renewer;
-  }
-
-  public void setRenewer(String renewer) {
-    this.renewer = renewer;
-  }
-
-  public void unsetRenewer() {
-    this.renewer = null;
-  }
-
-  /** Returns true if field renewer is set (has been assigned a value) and false otherwise */
-  public boolean isSetRenewer() {
-    return this.renewer != null;
-  }
-
-  public void setRenewerIsSet(boolean value) {
-    if (!value) {
-      this.renewer = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    case OWNER:
-      if (value == null) {
-        unsetOwner();
-      } else {
-        setOwner((String)value);
-      }
-      break;
-
-    case RENEWER:
-      if (value == null) {
-        unsetRenewer();
-      } else {
-        setRenewer((String)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    case OWNER:
-      return getOwner();
-
-    case RENEWER:
-      return getRenewer();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    case OWNER:
-      return isSetOwner();
-    case RENEWER:
-      return isSetRenewer();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetDelegationTokenReq)
-      return this.equals((TGetDelegationTokenReq)that);
-    return false;
-  }
-
-  public boolean equals(TGetDelegationTokenReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    boolean this_present_owner = true && this.isSetOwner();
-    boolean that_present_owner = true && that.isSetOwner();
-    if (this_present_owner || that_present_owner) {
-      if (!(this_present_owner && that_present_owner))
-        return false;
-      if (!this.owner.equals(that.owner))
-        return false;
-    }
-
-    boolean this_present_renewer = true && this.isSetRenewer();
-    boolean that_present_renewer = true && that.isSetRenewer();
-    if (this_present_renewer || that_present_renewer) {
-      if (!(this_present_renewer && that_present_renewer))
-        return false;
-      if (!this.renewer.equals(that.renewer))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    list.add(present_sessionHandle);
-    if (present_sessionHandle)
-      list.add(sessionHandle);
-
-    boolean present_owner = true && (isSetOwner());
-    list.add(present_owner);
-    if (present_owner)
-      list.add(owner);
-
-    boolean present_renewer = true && (isSetRenewer());
-    list.add(present_renewer);
-    if (present_renewer)
-      list.add(renewer);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetDelegationTokenReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(other.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, other.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOwner()).compareTo(other.isSetOwner());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOwner()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.owner, other.owner);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetRenewer()).compareTo(other.isSetRenewer());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetRenewer()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.renewer, other.renewer);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetDelegationTokenReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("owner:");
-    if (this.owner == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.owner);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("renewer:");
-    if (this.renewer == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.renewer);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    if (!isSetOwner()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'owner' is unset! Struct:" + toString());
-    }
-
-    if (!isSetRenewer()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'renewer' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetDelegationTokenReqStandardSchemeFactory implements SchemeFactory {
-    public TGetDelegationTokenReqStandardScheme getScheme() {
-      return new TGetDelegationTokenReqStandardScheme();
-    }
-  }
-
-  private static class TGetDelegationTokenReqStandardScheme extends StandardScheme<TGetDelegationTokenReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetDelegationTokenReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // OWNER
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.owner = iprot.readString();
-              struct.setOwnerIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // RENEWER
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.renewer = iprot.readString();
-              struct.setRenewerIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetDelegationTokenReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.owner != null) {
-        oprot.writeFieldBegin(OWNER_FIELD_DESC);
-        oprot.writeString(struct.owner);
-        oprot.writeFieldEnd();
-      }
-      if (struct.renewer != null) {
-        oprot.writeFieldBegin(RENEWER_FIELD_DESC);
-        oprot.writeString(struct.renewer);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetDelegationTokenReqTupleSchemeFactory implements SchemeFactory {
-    public TGetDelegationTokenReqTupleScheme getScheme() {
-      return new TGetDelegationTokenReqTupleScheme();
-    }
-  }
-
-  private static class TGetDelegationTokenReqTupleScheme extends TupleScheme<TGetDelegationTokenReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetDelegationTokenReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-      oprot.writeString(struct.owner);
-      oprot.writeString(struct.renewer);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetDelegationTokenReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-      struct.owner = iprot.readString();
-      struct.setOwnerIsSet(true);
-      struct.renewer = iprot.readString();
-      struct.setRenewerIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetDelegationTokenResp.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetDelegationTokenResp.java
deleted file mode 100644
index 6ef2acbbd9435..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetDelegationTokenResp.java
+++ /dev/null
@@ -1,504 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetDelegationTokenResp implements org.apache.thrift.TBase<TGetDelegationTokenResp, TGetDelegationTokenResp._Fields>, java.io.Serializable, Cloneable, Comparable<TGetDelegationTokenResp> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetDelegationTokenResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField DELEGATION_TOKEN_FIELD_DESC = new org.apache.thrift.protocol.TField("delegationToken", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetDelegationTokenRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetDelegationTokenRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private String delegationToken; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    DELEGATION_TOKEN((short)2, "delegationToken");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // DELEGATION_TOKEN
-          return DELEGATION_TOKEN;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final _Fields optionals[] = {_Fields.DELEGATION_TOKEN};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.DELEGATION_TOKEN, new org.apache.thrift.meta_data.FieldMetaData("delegationToken", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetDelegationTokenResp.class, metaDataMap);
-  }
-
-  public TGetDelegationTokenResp() {
-  }
-
-  public TGetDelegationTokenResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetDelegationTokenResp(TGetDelegationTokenResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetDelegationToken()) {
-      this.delegationToken = other.delegationToken;
-    }
-  }
-
-  public TGetDelegationTokenResp deepCopy() {
-    return new TGetDelegationTokenResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.delegationToken = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public String getDelegationToken() {
-    return this.delegationToken;
-  }
-
-  public void setDelegationToken(String delegationToken) {
-    this.delegationToken = delegationToken;
-  }
-
-  public void unsetDelegationToken() {
-    this.delegationToken = null;
-  }
-
-  /** Returns true if field delegationToken is set (has been assigned a value) and false otherwise */
-  public boolean isSetDelegationToken() {
-    return this.delegationToken != null;
-  }
-
-  public void setDelegationTokenIsSet(boolean value) {
-    if (!value) {
-      this.delegationToken = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case DELEGATION_TOKEN:
-      if (value == null) {
-        unsetDelegationToken();
-      } else {
-        setDelegationToken((String)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case DELEGATION_TOKEN:
-      return getDelegationToken();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case DELEGATION_TOKEN:
-      return isSetDelegationToken();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetDelegationTokenResp)
-      return this.equals((TGetDelegationTokenResp)that);
-    return false;
-  }
-
-  public boolean equals(TGetDelegationTokenResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_delegationToken = true && this.isSetDelegationToken();
-    boolean that_present_delegationToken = true && that.isSetDelegationToken();
-    if (this_present_delegationToken || that_present_delegationToken) {
-      if (!(this_present_delegationToken && that_present_delegationToken))
-        return false;
-      if (!this.delegationToken.equals(that.delegationToken))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_status = true && (isSetStatus());
-    list.add(present_status);
-    if (present_status)
-      list.add(status);
-
-    boolean present_delegationToken = true && (isSetDelegationToken());
-    list.add(present_delegationToken);
-    if (present_delegationToken)
-      list.add(delegationToken);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetDelegationTokenResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(other.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, other.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetDelegationToken()).compareTo(other.isSetDelegationToken());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetDelegationToken()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.delegationToken, other.delegationToken);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetDelegationTokenResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetDelegationToken()) {
-      if (!first) sb.append(", ");
-      sb.append("delegationToken:");
-      if (this.delegationToken == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.delegationToken);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetDelegationTokenRespStandardSchemeFactory implements SchemeFactory {
-    public TGetDelegationTokenRespStandardScheme getScheme() {
-      return new TGetDelegationTokenRespStandardScheme();
-    }
-  }
-
-  private static class TGetDelegationTokenRespStandardScheme extends StandardScheme<TGetDelegationTokenResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetDelegationTokenResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // DELEGATION_TOKEN
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.delegationToken = iprot.readString();
-              struct.setDelegationTokenIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetDelegationTokenResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.delegationToken != null) {
-        if (struct.isSetDelegationToken()) {
-          oprot.writeFieldBegin(DELEGATION_TOKEN_FIELD_DESC);
-          oprot.writeString(struct.delegationToken);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetDelegationTokenRespTupleSchemeFactory implements SchemeFactory {
-    public TGetDelegationTokenRespTupleScheme getScheme() {
-      return new TGetDelegationTokenRespTupleScheme();
-    }
-  }
-
-  private static class TGetDelegationTokenRespTupleScheme extends TupleScheme<TGetDelegationTokenResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetDelegationTokenResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetDelegationToken()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetDelegationToken()) {
-        oprot.writeString(struct.delegationToken);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetDelegationTokenResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.delegationToken = iprot.readString();
-        struct.setDelegationTokenIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetFunctionsReq.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetFunctionsReq.java
deleted file mode 100644
index ad4f8a5b031e8..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetFunctionsReq.java
+++ /dev/null
@@ -1,711 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetFunctionsReq implements org.apache.thrift.TBase<TGetFunctionsReq, TGetFunctionsReq._Fields>, java.io.Serializable, Cloneable, Comparable<TGetFunctionsReq> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetFunctionsReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField CATALOG_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("catalogName", org.apache.thrift.protocol.TType.STRING, (short)2);
-  private static final org.apache.thrift.protocol.TField SCHEMA_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("schemaName", org.apache.thrift.protocol.TType.STRING, (short)3);
-  private static final org.apache.thrift.protocol.TField FUNCTION_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("functionName", org.apache.thrift.protocol.TType.STRING, (short)4);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetFunctionsReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetFunctionsReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-  private String catalogName; // optional
-  private String schemaName; // optional
-  private String functionName; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle"),
-    CATALOG_NAME((short)2, "catalogName"),
-    SCHEMA_NAME((short)3, "schemaName"),
-    FUNCTION_NAME((short)4, "functionName");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        case 2: // CATALOG_NAME
-          return CATALOG_NAME;
-        case 3: // SCHEMA_NAME
-          return SCHEMA_NAME;
-        case 4: // FUNCTION_NAME
-          return FUNCTION_NAME;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final _Fields optionals[] = {_Fields.CATALOG_NAME,_Fields.SCHEMA_NAME};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    tmpMap.put(_Fields.CATALOG_NAME, new org.apache.thrift.meta_data.FieldMetaData("catalogName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TIdentifier")));
-    tmpMap.put(_Fields.SCHEMA_NAME, new org.apache.thrift.meta_data.FieldMetaData("schemaName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TPatternOrIdentifier")));
-    tmpMap.put(_Fields.FUNCTION_NAME, new org.apache.thrift.meta_data.FieldMetaData("functionName", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TPatternOrIdentifier")));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetFunctionsReq.class, metaDataMap);
-  }
-
-  public TGetFunctionsReq() {
-  }
-
-  public TGetFunctionsReq(
-    TSessionHandle sessionHandle,
-    String functionName)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-    this.functionName = functionName;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetFunctionsReq(TGetFunctionsReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-    if (other.isSetCatalogName()) {
-      this.catalogName = other.catalogName;
-    }
-    if (other.isSetSchemaName()) {
-      this.schemaName = other.schemaName;
-    }
-    if (other.isSetFunctionName()) {
-      this.functionName = other.functionName;
-    }
-  }
-
-  public TGetFunctionsReq deepCopy() {
-    return new TGetFunctionsReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-    this.catalogName = null;
-    this.schemaName = null;
-    this.functionName = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public String getCatalogName() {
-    return this.catalogName;
-  }
-
-  public void setCatalogName(String catalogName) {
-    this.catalogName = catalogName;
-  }
-
-  public void unsetCatalogName() {
-    this.catalogName = null;
-  }
-
-  /** Returns true if field catalogName is set (has been assigned a value) and false otherwise */
-  public boolean isSetCatalogName() {
-    return this.catalogName != null;
-  }
-
-  public void setCatalogNameIsSet(boolean value) {
-    if (!value) {
-      this.catalogName = null;
-    }
-  }
-
-  public String getSchemaName() {
-    return this.schemaName;
-  }
-
-  public void setSchemaName(String schemaName) {
-    this.schemaName = schemaName;
-  }
-
-  public void unsetSchemaName() {
-    this.schemaName = null;
-  }
-
-  /** Returns true if field schemaName is set (has been assigned a value) and false otherwise */
-  public boolean isSetSchemaName() {
-    return this.schemaName != null;
-  }
-
-  public void setSchemaNameIsSet(boolean value) {
-    if (!value) {
-      this.schemaName = null;
-    }
-  }
-
-  public String getFunctionName() {
-    return this.functionName;
-  }
-
-  public void setFunctionName(String functionName) {
-    this.functionName = functionName;
-  }
-
-  public void unsetFunctionName() {
-    this.functionName = null;
-  }
-
-  /** Returns true if field functionName is set (has been assigned a value) and false otherwise */
-  public boolean isSetFunctionName() {
-    return this.functionName != null;
-  }
-
-  public void setFunctionNameIsSet(boolean value) {
-    if (!value) {
-      this.functionName = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    case CATALOG_NAME:
-      if (value == null) {
-        unsetCatalogName();
-      } else {
-        setCatalogName((String)value);
-      }
-      break;
-
-    case SCHEMA_NAME:
-      if (value == null) {
-        unsetSchemaName();
-      } else {
-        setSchemaName((String)value);
-      }
-      break;
-
-    case FUNCTION_NAME:
-      if (value == null) {
-        unsetFunctionName();
-      } else {
-        setFunctionName((String)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    case CATALOG_NAME:
-      return getCatalogName();
-
-    case SCHEMA_NAME:
-      return getSchemaName();
-
-    case FUNCTION_NAME:
-      return getFunctionName();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    case CATALOG_NAME:
-      return isSetCatalogName();
-    case SCHEMA_NAME:
-      return isSetSchemaName();
-    case FUNCTION_NAME:
-      return isSetFunctionName();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetFunctionsReq)
-      return this.equals((TGetFunctionsReq)that);
-    return false;
-  }
-
-  public boolean equals(TGetFunctionsReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    boolean this_present_catalogName = true && this.isSetCatalogName();
-    boolean that_present_catalogName = true && that.isSetCatalogName();
-    if (this_present_catalogName || that_present_catalogName) {
-      if (!(this_present_catalogName && that_present_catalogName))
-        return false;
-      if (!this.catalogName.equals(that.catalogName))
-        return false;
-    }
-
-    boolean this_present_schemaName = true && this.isSetSchemaName();
-    boolean that_present_schemaName = true && that.isSetSchemaName();
-    if (this_present_schemaName || that_present_schemaName) {
-      if (!(this_present_schemaName && that_present_schemaName))
-        return false;
-      if (!this.schemaName.equals(that.schemaName))
-        return false;
-    }
-
-    boolean this_present_functionName = true && this.isSetFunctionName();
-    boolean that_present_functionName = true && that.isSetFunctionName();
-    if (this_present_functionName || that_present_functionName) {
-      if (!(this_present_functionName && that_present_functionName))
-        return false;
-      if (!this.functionName.equals(that.functionName))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    list.add(present_sessionHandle);
-    if (present_sessionHandle)
-      list.add(sessionHandle);
-
-    boolean present_catalogName = true && (isSetCatalogName());
-    list.add(present_catalogName);
-    if (present_catalogName)
-      list.add(catalogName);
-
-    boolean present_schemaName = true && (isSetSchemaName());
-    list.add(present_schemaName);
-    if (present_schemaName)
-      list.add(schemaName);
-
-    boolean present_functionName = true && (isSetFunctionName());
-    list.add(present_functionName);
-    if (present_functionName)
-      list.add(functionName);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetFunctionsReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(other.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, other.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetCatalogName()).compareTo(other.isSetCatalogName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetCatalogName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.catalogName, other.catalogName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetSchemaName()).compareTo(other.isSetSchemaName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSchemaName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.schemaName, other.schemaName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetFunctionName()).compareTo(other.isSetFunctionName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetFunctionName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.functionName, other.functionName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetFunctionsReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    if (isSetCatalogName()) {
-      if (!first) sb.append(", ");
-      sb.append("catalogName:");
-      if (this.catalogName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.catalogName);
-      }
-      first = false;
-    }
-    if (isSetSchemaName()) {
-      if (!first) sb.append(", ");
-      sb.append("schemaName:");
-      if (this.schemaName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.schemaName);
-      }
-      first = false;
-    }
-    if (!first) sb.append(", ");
-    sb.append("functionName:");
-    if (this.functionName == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.functionName);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    if (!isSetFunctionName()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'functionName' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetFunctionsReqStandardSchemeFactory implements SchemeFactory {
-    public TGetFunctionsReqStandardScheme getScheme() {
-      return new TGetFunctionsReqStandardScheme();
-    }
-  }
-
-  private static class TGetFunctionsReqStandardScheme extends StandardScheme<TGetFunctionsReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetFunctionsReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // CATALOG_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.catalogName = iprot.readString();
-              struct.setCatalogNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // SCHEMA_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.schemaName = iprot.readString();
-              struct.setSchemaNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // FUNCTION_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.functionName = iprot.readString();
-              struct.setFunctionNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetFunctionsReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.catalogName != null) {
-        if (struct.isSetCatalogName()) {
-          oprot.writeFieldBegin(CATALOG_NAME_FIELD_DESC);
-          oprot.writeString(struct.catalogName);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.schemaName != null) {
-        if (struct.isSetSchemaName()) {
-          oprot.writeFieldBegin(SCHEMA_NAME_FIELD_DESC);
-          oprot.writeString(struct.schemaName);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.functionName != null) {
-        oprot.writeFieldBegin(FUNCTION_NAME_FIELD_DESC);
-        oprot.writeString(struct.functionName);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetFunctionsReqTupleSchemeFactory implements SchemeFactory {
-    public TGetFunctionsReqTupleScheme getScheme() {
-      return new TGetFunctionsReqTupleScheme();
-    }
-  }
-
-  private static class TGetFunctionsReqTupleScheme extends TupleScheme<TGetFunctionsReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetFunctionsReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-      oprot.writeString(struct.functionName);
-      BitSet optionals = new BitSet();
-      if (struct.isSetCatalogName()) {
-        optionals.set(0);
-      }
-      if (struct.isSetSchemaName()) {
-        optionals.set(1);
-      }
-      oprot.writeBitSet(optionals, 2);
-      if (struct.isSetCatalogName()) {
-        oprot.writeString(struct.catalogName);
-      }
-      if (struct.isSetSchemaName()) {
-        oprot.writeString(struct.schemaName);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetFunctionsReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-      struct.functionName = iprot.readString();
-      struct.setFunctionNameIsSet(true);
-      BitSet incoming = iprot.readBitSet(2);
-      if (incoming.get(0)) {
-        struct.catalogName = iprot.readString();
-        struct.setCatalogNameIsSet(true);
-      }
-      if (incoming.get(1)) {
-        struct.schemaName = iprot.readString();
-        struct.setSchemaNameIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetFunctionsResp.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetFunctionsResp.java
deleted file mode 100644
index ead37fb91cc2f..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetFunctionsResp.java
+++ /dev/null
@@ -1,509 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetFunctionsResp implements org.apache.thrift.TBase<TGetFunctionsResp, TGetFunctionsResp._Fields>, java.io.Serializable, Cloneable, Comparable<TGetFunctionsResp> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetFunctionsResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetFunctionsRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetFunctionsRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TOperationHandle operationHandle; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    OPERATION_HANDLE((short)2, "operationHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final _Fields optionals[] = {_Fields.OPERATION_HANDLE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetFunctionsResp.class, metaDataMap);
-  }
-
-  public TGetFunctionsResp() {
-  }
-
-  public TGetFunctionsResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetFunctionsResp(TGetFunctionsResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-  }
-
-  public TGetFunctionsResp deepCopy() {
-    return new TGetFunctionsResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.operationHandle = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetFunctionsResp)
-      return this.equals((TGetFunctionsResp)that);
-    return false;
-  }
-
-  public boolean equals(TGetFunctionsResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_status = true && (isSetStatus());
-    list.add(present_status);
-    if (present_status)
-      list.add(status);
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    list.add(present_operationHandle);
-    if (present_operationHandle)
-      list.add(operationHandle);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetFunctionsResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(other.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, other.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(other.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, other.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetFunctionsResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetOperationHandle()) {
-      if (!first) sb.append(", ");
-      sb.append("operationHandle:");
-      if (this.operationHandle == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.operationHandle);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetFunctionsRespStandardSchemeFactory implements SchemeFactory {
-    public TGetFunctionsRespStandardScheme getScheme() {
-      return new TGetFunctionsRespStandardScheme();
-    }
-  }
-
-  private static class TGetFunctionsRespStandardScheme extends StandardScheme<TGetFunctionsResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetFunctionsResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetFunctionsResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.operationHandle != null) {
-        if (struct.isSetOperationHandle()) {
-          oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-          struct.operationHandle.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetFunctionsRespTupleSchemeFactory implements SchemeFactory {
-    public TGetFunctionsRespTupleScheme getScheme() {
-      return new TGetFunctionsRespTupleScheme();
-    }
-  }
-
-  private static class TGetFunctionsRespTupleScheme extends TupleScheme<TGetFunctionsResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetFunctionsResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetOperationHandle()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetOperationHandle()) {
-        struct.operationHandle.write(oprot);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetFunctionsResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.operationHandle = new TOperationHandle();
-        struct.operationHandle.read(iprot);
-        struct.setOperationHandleIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetInfoReq.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetInfoReq.java
deleted file mode 100644
index b319b70e5eba5..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetInfoReq.java
+++ /dev/null
@@ -1,507 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetInfoReq implements org.apache.thrift.TBase<TGetInfoReq, TGetInfoReq._Fields>, java.io.Serializable, Cloneable, Comparable<TGetInfoReq> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetInfoReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField INFO_TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("infoType", org.apache.thrift.protocol.TType.I32, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetInfoReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetInfoReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-  private TGetInfoType infoType; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle"),
-    /**
-     * 
-     * @see TGetInfoType
-     */
-    INFO_TYPE((short)2, "infoType");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        case 2: // INFO_TYPE
-          return INFO_TYPE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    tmpMap.put(_Fields.INFO_TYPE, new org.apache.thrift.meta_data.FieldMetaData("infoType", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TGetInfoType.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetInfoReq.class, metaDataMap);
-  }
-
-  public TGetInfoReq() {
-  }
-
-  public TGetInfoReq(
-    TSessionHandle sessionHandle,
-    TGetInfoType infoType)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-    this.infoType = infoType;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetInfoReq(TGetInfoReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-    if (other.isSetInfoType()) {
-      this.infoType = other.infoType;
-    }
-  }
-
-  public TGetInfoReq deepCopy() {
-    return new TGetInfoReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-    this.infoType = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  /**
-   * 
-   * @see TGetInfoType
-   */
-  public TGetInfoType getInfoType() {
-    return this.infoType;
-  }
-
-  /**
-   * 
-   * @see TGetInfoType
-   */
-  public void setInfoType(TGetInfoType infoType) {
-    this.infoType = infoType;
-  }
-
-  public void unsetInfoType() {
-    this.infoType = null;
-  }
-
-  /** Returns true if field infoType is set (has been assigned a value) and false otherwise */
-  public boolean isSetInfoType() {
-    return this.infoType != null;
-  }
-
-  public void setInfoTypeIsSet(boolean value) {
-    if (!value) {
-      this.infoType = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    case INFO_TYPE:
-      if (value == null) {
-        unsetInfoType();
-      } else {
-        setInfoType((TGetInfoType)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    case INFO_TYPE:
-      return getInfoType();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    case INFO_TYPE:
-      return isSetInfoType();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetInfoReq)
-      return this.equals((TGetInfoReq)that);
-    return false;
-  }
-
-  public boolean equals(TGetInfoReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    boolean this_present_infoType = true && this.isSetInfoType();
-    boolean that_present_infoType = true && that.isSetInfoType();
-    if (this_present_infoType || that_present_infoType) {
-      if (!(this_present_infoType && that_present_infoType))
-        return false;
-      if (!this.infoType.equals(that.infoType))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    list.add(present_sessionHandle);
-    if (present_sessionHandle)
-      list.add(sessionHandle);
-
-    boolean present_infoType = true && (isSetInfoType());
-    list.add(present_infoType);
-    if (present_infoType)
-      list.add(infoType.getValue());
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetInfoReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(other.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, other.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetInfoType()).compareTo(other.isSetInfoType());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetInfoType()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.infoType, other.infoType);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetInfoReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("infoType:");
-    if (this.infoType == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.infoType);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    if (!isSetInfoType()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'infoType' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetInfoReqStandardSchemeFactory implements SchemeFactory {
-    public TGetInfoReqStandardScheme getScheme() {
-      return new TGetInfoReqStandardScheme();
-    }
-  }
-
-  private static class TGetInfoReqStandardScheme extends StandardScheme<TGetInfoReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetInfoReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // INFO_TYPE
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.infoType = org.apache.hive.service.rpc.thrift.TGetInfoType.findByValue(iprot.readI32());
-              struct.setInfoTypeIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetInfoReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.infoType != null) {
-        oprot.writeFieldBegin(INFO_TYPE_FIELD_DESC);
-        oprot.writeI32(struct.infoType.getValue());
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetInfoReqTupleSchemeFactory implements SchemeFactory {
-    public TGetInfoReqTupleScheme getScheme() {
-      return new TGetInfoReqTupleScheme();
-    }
-  }
-
-  private static class TGetInfoReqTupleScheme extends TupleScheme<TGetInfoReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetInfoReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-      oprot.writeI32(struct.infoType.getValue());
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetInfoReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-      struct.infoType = org.apache.hive.service.rpc.thrift.TGetInfoType.findByValue(iprot.readI32());
-      struct.setInfoTypeIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetInfoResp.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetInfoResp.java
deleted file mode 100644
index 9be810b024987..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetInfoResp.java
+++ /dev/null
@@ -1,497 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetInfoResp implements org.apache.thrift.TBase<TGetInfoResp, TGetInfoResp._Fields>, java.io.Serializable, Cloneable, Comparable<TGetInfoResp> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetInfoResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField INFO_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("infoValue", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetInfoRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetInfoRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TGetInfoValue infoValue; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    INFO_VALUE((short)2, "infoValue");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // INFO_VALUE
-          return INFO_VALUE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.INFO_VALUE, new org.apache.thrift.meta_data.FieldMetaData("infoValue", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGetInfoValue.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetInfoResp.class, metaDataMap);
-  }
-
-  public TGetInfoResp() {
-  }
-
-  public TGetInfoResp(
-    TStatus status,
-    TGetInfoValue infoValue)
-  {
-    this();
-    this.status = status;
-    this.infoValue = infoValue;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetInfoResp(TGetInfoResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetInfoValue()) {
-      this.infoValue = new TGetInfoValue(other.infoValue);
-    }
-  }
-
-  public TGetInfoResp deepCopy() {
-    return new TGetInfoResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.infoValue = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public TGetInfoValue getInfoValue() {
-    return this.infoValue;
-  }
-
-  public void setInfoValue(TGetInfoValue infoValue) {
-    this.infoValue = infoValue;
-  }
-
-  public void unsetInfoValue() {
-    this.infoValue = null;
-  }
-
-  /** Returns true if field infoValue is set (has been assigned a value) and false otherwise */
-  public boolean isSetInfoValue() {
-    return this.infoValue != null;
-  }
-
-  public void setInfoValueIsSet(boolean value) {
-    if (!value) {
-      this.infoValue = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case INFO_VALUE:
-      if (value == null) {
-        unsetInfoValue();
-      } else {
-        setInfoValue((TGetInfoValue)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case INFO_VALUE:
-      return getInfoValue();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case INFO_VALUE:
-      return isSetInfoValue();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetInfoResp)
-      return this.equals((TGetInfoResp)that);
-    return false;
-  }
-
-  public boolean equals(TGetInfoResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_infoValue = true && this.isSetInfoValue();
-    boolean that_present_infoValue = true && that.isSetInfoValue();
-    if (this_present_infoValue || that_present_infoValue) {
-      if (!(this_present_infoValue && that_present_infoValue))
-        return false;
-      if (!this.infoValue.equals(that.infoValue))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_status = true && (isSetStatus());
-    list.add(present_status);
-    if (present_status)
-      list.add(status);
-
-    boolean present_infoValue = true && (isSetInfoValue());
-    list.add(present_infoValue);
-    if (present_infoValue)
-      list.add(infoValue);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetInfoResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(other.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, other.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetInfoValue()).compareTo(other.isSetInfoValue());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetInfoValue()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.infoValue, other.infoValue);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetInfoResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("infoValue:");
-    if (this.infoValue == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.infoValue);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    if (!isSetInfoValue()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'infoValue' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetInfoRespStandardSchemeFactory implements SchemeFactory {
-    public TGetInfoRespStandardScheme getScheme() {
-      return new TGetInfoRespStandardScheme();
-    }
-  }
-
-  private static class TGetInfoRespStandardScheme extends StandardScheme<TGetInfoResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetInfoResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // INFO_VALUE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.infoValue = new TGetInfoValue();
-              struct.infoValue.read(iprot);
-              struct.setInfoValueIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetInfoResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.infoValue != null) {
-        oprot.writeFieldBegin(INFO_VALUE_FIELD_DESC);
-        struct.infoValue.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetInfoRespTupleSchemeFactory implements SchemeFactory {
-    public TGetInfoRespTupleScheme getScheme() {
-      return new TGetInfoRespTupleScheme();
-    }
-  }
-
-  private static class TGetInfoRespTupleScheme extends TupleScheme<TGetInfoResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetInfoResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      struct.infoValue.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetInfoResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      struct.infoValue = new TGetInfoValue();
-      struct.infoValue.read(iprot);
-      struct.setInfoValueIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetInfoType.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetInfoType.java
deleted file mode 100644
index 5b219b62656d7..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetInfoType.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.thrift.TEnum;
-
-public enum TGetInfoType implements org.apache.thrift.TEnum {
-  CLI_MAX_DRIVER_CONNECTIONS(0),
-  CLI_MAX_CONCURRENT_ACTIVITIES(1),
-  CLI_DATA_SOURCE_NAME(2),
-  CLI_FETCH_DIRECTION(8),
-  CLI_SERVER_NAME(13),
-  CLI_SEARCH_PATTERN_ESCAPE(14),
-  CLI_DBMS_NAME(17),
-  CLI_DBMS_VER(18),
-  CLI_ACCESSIBLE_TABLES(19),
-  CLI_ACCESSIBLE_PROCEDURES(20),
-  CLI_CURSOR_COMMIT_BEHAVIOR(23),
-  CLI_DATA_SOURCE_READ_ONLY(25),
-  CLI_DEFAULT_TXN_ISOLATION(26),
-  CLI_IDENTIFIER_CASE(28),
-  CLI_IDENTIFIER_QUOTE_CHAR(29),
-  CLI_MAX_COLUMN_NAME_LEN(30),
-  CLI_MAX_CURSOR_NAME_LEN(31),
-  CLI_MAX_SCHEMA_NAME_LEN(32),
-  CLI_MAX_CATALOG_NAME_LEN(34),
-  CLI_MAX_TABLE_NAME_LEN(35),
-  CLI_SCROLL_CONCURRENCY(43),
-  CLI_TXN_CAPABLE(46),
-  CLI_USER_NAME(47),
-  CLI_TXN_ISOLATION_OPTION(72),
-  CLI_INTEGRITY(73),
-  CLI_GETDATA_EXTENSIONS(81),
-  CLI_NULL_COLLATION(85),
-  CLI_ALTER_TABLE(86),
-  CLI_ORDER_BY_COLUMNS_IN_SELECT(90),
-  CLI_SPECIAL_CHARACTERS(94),
-  CLI_MAX_COLUMNS_IN_GROUP_BY(97),
-  CLI_MAX_COLUMNS_IN_INDEX(98),
-  CLI_MAX_COLUMNS_IN_ORDER_BY(99),
-  CLI_MAX_COLUMNS_IN_SELECT(100),
-  CLI_MAX_COLUMNS_IN_TABLE(101),
-  CLI_MAX_INDEX_SIZE(102),
-  CLI_MAX_ROW_SIZE(104),
-  CLI_MAX_STATEMENT_LEN(105),
-  CLI_MAX_TABLES_IN_SELECT(106),
-  CLI_MAX_USER_NAME_LEN(107),
-  CLI_OJ_CAPABILITIES(115),
-  CLI_XOPEN_CLI_YEAR(10000),
-  CLI_CURSOR_SENSITIVITY(10001),
-  CLI_DESCRIBE_PARAMETER(10002),
-  CLI_CATALOG_NAME(10003),
-  CLI_COLLATION_SEQ(10004),
-  CLI_MAX_IDENTIFIER_LEN(10005);
-
-  private final int value;
-
-  private TGetInfoType(int value) {
-    this.value = value;
-  }
-
-  /**
-   * Get the integer value of this enum value, as defined in the Thrift IDL.
-   */
-  public int getValue() {
-    return value;
-  }
-
-  /**
-   * Find a the enum type by its integer value, as defined in the Thrift IDL.
-   * @return null if the value is not found.
-   */
-  public static TGetInfoType findByValue(int value) { 
-    switch (value) {
-      case 0:
-        return CLI_MAX_DRIVER_CONNECTIONS;
-      case 1:
-        return CLI_MAX_CONCURRENT_ACTIVITIES;
-      case 2:
-        return CLI_DATA_SOURCE_NAME;
-      case 8:
-        return CLI_FETCH_DIRECTION;
-      case 13:
-        return CLI_SERVER_NAME;
-      case 14:
-        return CLI_SEARCH_PATTERN_ESCAPE;
-      case 17:
-        return CLI_DBMS_NAME;
-      case 18:
-        return CLI_DBMS_VER;
-      case 19:
-        return CLI_ACCESSIBLE_TABLES;
-      case 20:
-        return CLI_ACCESSIBLE_PROCEDURES;
-      case 23:
-        return CLI_CURSOR_COMMIT_BEHAVIOR;
-      case 25:
-        return CLI_DATA_SOURCE_READ_ONLY;
-      case 26:
-        return CLI_DEFAULT_TXN_ISOLATION;
-      case 28:
-        return CLI_IDENTIFIER_CASE;
-      case 29:
-        return CLI_IDENTIFIER_QUOTE_CHAR;
-      case 30:
-        return CLI_MAX_COLUMN_NAME_LEN;
-      case 31:
-        return CLI_MAX_CURSOR_NAME_LEN;
-      case 32:
-        return CLI_MAX_SCHEMA_NAME_LEN;
-      case 34:
-        return CLI_MAX_CATALOG_NAME_LEN;
-      case 35:
-        return CLI_MAX_TABLE_NAME_LEN;
-      case 43:
-        return CLI_SCROLL_CONCURRENCY;
-      case 46:
-        return CLI_TXN_CAPABLE;
-      case 47:
-        return CLI_USER_NAME;
-      case 72:
-        return CLI_TXN_ISOLATION_OPTION;
-      case 73:
-        return CLI_INTEGRITY;
-      case 81:
-        return CLI_GETDATA_EXTENSIONS;
-      case 85:
-        return CLI_NULL_COLLATION;
-      case 86:
-        return CLI_ALTER_TABLE;
-      case 90:
-        return CLI_ORDER_BY_COLUMNS_IN_SELECT;
-      case 94:
-        return CLI_SPECIAL_CHARACTERS;
-      case 97:
-        return CLI_MAX_COLUMNS_IN_GROUP_BY;
-      case 98:
-        return CLI_MAX_COLUMNS_IN_INDEX;
-      case 99:
-        return CLI_MAX_COLUMNS_IN_ORDER_BY;
-      case 100:
-        return CLI_MAX_COLUMNS_IN_SELECT;
-      case 101:
-        return CLI_MAX_COLUMNS_IN_TABLE;
-      case 102:
-        return CLI_MAX_INDEX_SIZE;
-      case 104:
-        return CLI_MAX_ROW_SIZE;
-      case 105:
-        return CLI_MAX_STATEMENT_LEN;
-      case 106:
-        return CLI_MAX_TABLES_IN_SELECT;
-      case 107:
-        return CLI_MAX_USER_NAME_LEN;
-      case 115:
-        return CLI_OJ_CAPABILITIES;
-      case 10000:
-        return CLI_XOPEN_CLI_YEAR;
-      case 10001:
-        return CLI_CURSOR_SENSITIVITY;
-      case 10002:
-        return CLI_DESCRIBE_PARAMETER;
-      case 10003:
-        return CLI_CATALOG_NAME;
-      case 10004:
-        return CLI_COLLATION_SEQ;
-      case 10005:
-        return CLI_MAX_IDENTIFIER_LEN;
-      default:
-        return null;
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetInfoValue.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetInfoValue.java
deleted file mode 100644
index 8e3045a58e5ac..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetInfoValue.java
+++ /dev/null
@@ -1,597 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-public class TGetInfoValue extends org.apache.thrift.TUnion<TGetInfoValue, TGetInfoValue._Fields> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetInfoValue");
-  private static final org.apache.thrift.protocol.TField STRING_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("stringValue", org.apache.thrift.protocol.TType.STRING, (short)1);
-  private static final org.apache.thrift.protocol.TField SMALL_INT_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("smallIntValue", org.apache.thrift.protocol.TType.I16, (short)2);
-  private static final org.apache.thrift.protocol.TField INTEGER_BITMASK_FIELD_DESC = new org.apache.thrift.protocol.TField("integerBitmask", org.apache.thrift.protocol.TType.I32, (short)3);
-  private static final org.apache.thrift.protocol.TField INTEGER_FLAG_FIELD_DESC = new org.apache.thrift.protocol.TField("integerFlag", org.apache.thrift.protocol.TType.I32, (short)4);
-  private static final org.apache.thrift.protocol.TField BINARY_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("binaryValue", org.apache.thrift.protocol.TType.I32, (short)5);
-  private static final org.apache.thrift.protocol.TField LEN_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("lenValue", org.apache.thrift.protocol.TType.I64, (short)6);
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STRING_VALUE((short)1, "stringValue"),
-    SMALL_INT_VALUE((short)2, "smallIntValue"),
-    INTEGER_BITMASK((short)3, "integerBitmask"),
-    INTEGER_FLAG((short)4, "integerFlag"),
-    BINARY_VALUE((short)5, "binaryValue"),
-    LEN_VALUE((short)6, "lenValue");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STRING_VALUE
-          return STRING_VALUE;
-        case 2: // SMALL_INT_VALUE
-          return SMALL_INT_VALUE;
-        case 3: // INTEGER_BITMASK
-          return INTEGER_BITMASK;
-        case 4: // INTEGER_FLAG
-          return INTEGER_FLAG;
-        case 5: // BINARY_VALUE
-          return BINARY_VALUE;
-        case 6: // LEN_VALUE
-          return LEN_VALUE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STRING_VALUE, new org.apache.thrift.meta_data.FieldMetaData("stringValue", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    tmpMap.put(_Fields.SMALL_INT_VALUE, new org.apache.thrift.meta_data.FieldMetaData("smallIntValue", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I16)));
-    tmpMap.put(_Fields.INTEGER_BITMASK, new org.apache.thrift.meta_data.FieldMetaData("integerBitmask", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
-    tmpMap.put(_Fields.INTEGER_FLAG, new org.apache.thrift.meta_data.FieldMetaData("integerFlag", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
-    tmpMap.put(_Fields.BINARY_VALUE, new org.apache.thrift.meta_data.FieldMetaData("binaryValue", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
-    tmpMap.put(_Fields.LEN_VALUE, new org.apache.thrift.meta_data.FieldMetaData("lenValue", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetInfoValue.class, metaDataMap);
-  }
-
-  public TGetInfoValue() {
-    super();
-  }
-
-  public TGetInfoValue(TGetInfoValue._Fields setField, Object value) {
-    super(setField, value);
-  }
-
-  public TGetInfoValue(TGetInfoValue other) {
-    super(other);
-  }
-  public TGetInfoValue deepCopy() {
-    return new TGetInfoValue(this);
-  }
-
-  public static TGetInfoValue stringValue(String value) {
-    TGetInfoValue x = new TGetInfoValue();
-    x.setStringValue(value);
-    return x;
-  }
-
-  public static TGetInfoValue smallIntValue(short value) {
-    TGetInfoValue x = new TGetInfoValue();
-    x.setSmallIntValue(value);
-    return x;
-  }
-
-  public static TGetInfoValue integerBitmask(int value) {
-    TGetInfoValue x = new TGetInfoValue();
-    x.setIntegerBitmask(value);
-    return x;
-  }
-
-  public static TGetInfoValue integerFlag(int value) {
-    TGetInfoValue x = new TGetInfoValue();
-    x.setIntegerFlag(value);
-    return x;
-  }
-
-  public static TGetInfoValue binaryValue(int value) {
-    TGetInfoValue x = new TGetInfoValue();
-    x.setBinaryValue(value);
-    return x;
-  }
-
-  public static TGetInfoValue lenValue(long value) {
-    TGetInfoValue x = new TGetInfoValue();
-    x.setLenValue(value);
-    return x;
-  }
-
-
-  @Override
-  protected void checkType(_Fields setField, Object value) throws ClassCastException {
-    switch (setField) {
-      case STRING_VALUE:
-        if (value instanceof String) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type String for field 'stringValue', but got " + value.getClass().getSimpleName());
-      case SMALL_INT_VALUE:
-        if (value instanceof Short) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type Short for field 'smallIntValue', but got " + value.getClass().getSimpleName());
-      case INTEGER_BITMASK:
-        if (value instanceof Integer) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type Integer for field 'integerBitmask', but got " + value.getClass().getSimpleName());
-      case INTEGER_FLAG:
-        if (value instanceof Integer) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type Integer for field 'integerFlag', but got " + value.getClass().getSimpleName());
-      case BINARY_VALUE:
-        if (value instanceof Integer) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type Integer for field 'binaryValue', but got " + value.getClass().getSimpleName());
-      case LEN_VALUE:
-        if (value instanceof Long) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type Long for field 'lenValue', but got " + value.getClass().getSimpleName());
-      default:
-        throw new IllegalArgumentException("Unknown field id " + setField);
-    }
-  }
-
-  @Override
-  protected Object standardSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TField field) throws org.apache.thrift.TException {
-    _Fields setField = _Fields.findByThriftId(field.id);
-    if (setField != null) {
-      switch (setField) {
-        case STRING_VALUE:
-          if (field.type == STRING_VALUE_FIELD_DESC.type) {
-            String stringValue;
-            stringValue = iprot.readString();
-            return stringValue;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case SMALL_INT_VALUE:
-          if (field.type == SMALL_INT_VALUE_FIELD_DESC.type) {
-            Short smallIntValue;
-            smallIntValue = iprot.readI16();
-            return smallIntValue;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case INTEGER_BITMASK:
-          if (field.type == INTEGER_BITMASK_FIELD_DESC.type) {
-            Integer integerBitmask;
-            integerBitmask = iprot.readI32();
-            return integerBitmask;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case INTEGER_FLAG:
-          if (field.type == INTEGER_FLAG_FIELD_DESC.type) {
-            Integer integerFlag;
-            integerFlag = iprot.readI32();
-            return integerFlag;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case BINARY_VALUE:
-          if (field.type == BINARY_VALUE_FIELD_DESC.type) {
-            Integer binaryValue;
-            binaryValue = iprot.readI32();
-            return binaryValue;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case LEN_VALUE:
-          if (field.type == LEN_VALUE_FIELD_DESC.type) {
-            Long lenValue;
-            lenValue = iprot.readI64();
-            return lenValue;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        default:
-          throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!");
-      }
-    } else {
-      org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-      return null;
-    }
-  }
-
-  @Override
-  protected void standardSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    switch (setField_) {
-      case STRING_VALUE:
-        String stringValue = (String)value_;
-        oprot.writeString(stringValue);
-        return;
-      case SMALL_INT_VALUE:
-        Short smallIntValue = (Short)value_;
-        oprot.writeI16(smallIntValue);
-        return;
-      case INTEGER_BITMASK:
-        Integer integerBitmask = (Integer)value_;
-        oprot.writeI32(integerBitmask);
-        return;
-      case INTEGER_FLAG:
-        Integer integerFlag = (Integer)value_;
-        oprot.writeI32(integerFlag);
-        return;
-      case BINARY_VALUE:
-        Integer binaryValue = (Integer)value_;
-        oprot.writeI32(binaryValue);
-        return;
-      case LEN_VALUE:
-        Long lenValue = (Long)value_;
-        oprot.writeI64(lenValue);
-        return;
-      default:
-        throw new IllegalStateException("Cannot write union with unknown field " + setField_);
-    }
-  }
-
-  @Override
-  protected Object tupleSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, short fieldID) throws org.apache.thrift.TException {
-    _Fields setField = _Fields.findByThriftId(fieldID);
-    if (setField != null) {
-      switch (setField) {
-        case STRING_VALUE:
-          String stringValue;
-          stringValue = iprot.readString();
-          return stringValue;
-        case SMALL_INT_VALUE:
-          Short smallIntValue;
-          smallIntValue = iprot.readI16();
-          return smallIntValue;
-        case INTEGER_BITMASK:
-          Integer integerBitmask;
-          integerBitmask = iprot.readI32();
-          return integerBitmask;
-        case INTEGER_FLAG:
-          Integer integerFlag;
-          integerFlag = iprot.readI32();
-          return integerFlag;
-        case BINARY_VALUE:
-          Integer binaryValue;
-          binaryValue = iprot.readI32();
-          return binaryValue;
-        case LEN_VALUE:
-          Long lenValue;
-          lenValue = iprot.readI64();
-          return lenValue;
-        default:
-          throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!");
-      }
-    } else {
-      throw new TProtocolException("Couldn't find a field with field id " + fieldID);
-    }
-  }
-
-  @Override
-  protected void tupleSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    switch (setField_) {
-      case STRING_VALUE:
-        String stringValue = (String)value_;
-        oprot.writeString(stringValue);
-        return;
-      case SMALL_INT_VALUE:
-        Short smallIntValue = (Short)value_;
-        oprot.writeI16(smallIntValue);
-        return;
-      case INTEGER_BITMASK:
-        Integer integerBitmask = (Integer)value_;
-        oprot.writeI32(integerBitmask);
-        return;
-      case INTEGER_FLAG:
-        Integer integerFlag = (Integer)value_;
-        oprot.writeI32(integerFlag);
-        return;
-      case BINARY_VALUE:
-        Integer binaryValue = (Integer)value_;
-        oprot.writeI32(binaryValue);
-        return;
-      case LEN_VALUE:
-        Long lenValue = (Long)value_;
-        oprot.writeI64(lenValue);
-        return;
-      default:
-        throw new IllegalStateException("Cannot write union with unknown field " + setField_);
-    }
-  }
-
-  @Override
-  protected org.apache.thrift.protocol.TField getFieldDesc(_Fields setField) {
-    switch (setField) {
-      case STRING_VALUE:
-        return STRING_VALUE_FIELD_DESC;
-      case SMALL_INT_VALUE:
-        return SMALL_INT_VALUE_FIELD_DESC;
-      case INTEGER_BITMASK:
-        return INTEGER_BITMASK_FIELD_DESC;
-      case INTEGER_FLAG:
-        return INTEGER_FLAG_FIELD_DESC;
-      case BINARY_VALUE:
-        return BINARY_VALUE_FIELD_DESC;
-      case LEN_VALUE:
-        return LEN_VALUE_FIELD_DESC;
-      default:
-        throw new IllegalArgumentException("Unknown field id " + setField);
-    }
-  }
-
-  @Override
-  protected org.apache.thrift.protocol.TStruct getStructDesc() {
-    return STRUCT_DESC;
-  }
-
-  @Override
-  protected _Fields enumForId(short id) {
-    return _Fields.findByThriftIdOrThrow(id);
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-
-  public String getStringValue() {
-    if (getSetField() == _Fields.STRING_VALUE) {
-      return (String)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'stringValue' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setStringValue(String value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.STRING_VALUE;
-    value_ = value;
-  }
-
-  public short getSmallIntValue() {
-    if (getSetField() == _Fields.SMALL_INT_VALUE) {
-      return (Short)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'smallIntValue' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setSmallIntValue(short value) {
-    setField_ = _Fields.SMALL_INT_VALUE;
-    value_ = value;
-  }
-
-  public int getIntegerBitmask() {
-    if (getSetField() == _Fields.INTEGER_BITMASK) {
-      return (Integer)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'integerBitmask' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setIntegerBitmask(int value) {
-    setField_ = _Fields.INTEGER_BITMASK;
-    value_ = value;
-  }
-
-  public int getIntegerFlag() {
-    if (getSetField() == _Fields.INTEGER_FLAG) {
-      return (Integer)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'integerFlag' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setIntegerFlag(int value) {
-    setField_ = _Fields.INTEGER_FLAG;
-    value_ = value;
-  }
-
-  public int getBinaryValue() {
-    if (getSetField() == _Fields.BINARY_VALUE) {
-      return (Integer)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'binaryValue' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setBinaryValue(int value) {
-    setField_ = _Fields.BINARY_VALUE;
-    value_ = value;
-  }
-
-  public long getLenValue() {
-    if (getSetField() == _Fields.LEN_VALUE) {
-      return (Long)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'lenValue' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setLenValue(long value) {
-    setField_ = _Fields.LEN_VALUE;
-    value_ = value;
-  }
-
-  public boolean isSetStringValue() {
-    return setField_ == _Fields.STRING_VALUE;
-  }
-
-
-  public boolean isSetSmallIntValue() {
-    return setField_ == _Fields.SMALL_INT_VALUE;
-  }
-
-
-  public boolean isSetIntegerBitmask() {
-    return setField_ == _Fields.INTEGER_BITMASK;
-  }
-
-
-  public boolean isSetIntegerFlag() {
-    return setField_ == _Fields.INTEGER_FLAG;
-  }
-
-
-  public boolean isSetBinaryValue() {
-    return setField_ == _Fields.BINARY_VALUE;
-  }
-
-
-  public boolean isSetLenValue() {
-    return setField_ == _Fields.LEN_VALUE;
-  }
-
-
-  public boolean equals(Object other) {
-    if (other instanceof TGetInfoValue) {
-      return equals((TGetInfoValue)other);
-    } else {
-      return false;
-    }
-  }
-
-  public boolean equals(TGetInfoValue other) {
-    return other != null && getSetField() == other.getSetField() && getFieldValue().equals(other.getFieldValue());
-  }
-
-  @Override
-  public int compareTo(TGetInfoValue other) {
-    int lastComparison = org.apache.thrift.TBaseHelper.compareTo(getSetField(), other.getSetField());
-    if (lastComparison == 0) {
-      return org.apache.thrift.TBaseHelper.compareTo(getFieldValue(), other.getFieldValue());
-    }
-    return lastComparison;
-  }
-
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-    list.add(this.getClass().getName());
-    org.apache.thrift.TFieldIdEnum setField = getSetField();
-    if (setField != null) {
-      list.add(setField.getThriftFieldId());
-      Object value = getFieldValue();
-      if (value instanceof org.apache.thrift.TEnum) {
-        list.add(((org.apache.thrift.TEnum)getFieldValue()).getValue());
-      } else {
-        list.add(value);
-      }
-    }
-    return list.hashCode();
-  }
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-
-}
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetOperationStatusReq.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetOperationStatusReq.java
deleted file mode 100644
index af31ce2b22819..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetOperationStatusReq.java
+++ /dev/null
@@ -1,501 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetOperationStatusReq implements org.apache.thrift.TBase<TGetOperationStatusReq, TGetOperationStatusReq._Fields>, java.io.Serializable, Cloneable, Comparable<TGetOperationStatusReq> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetOperationStatusReq");
-
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField GET_PROGRESS_UPDATE_FIELD_DESC = new org.apache.thrift.protocol.TField("getProgressUpdate", org.apache.thrift.protocol.TType.BOOL, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetOperationStatusReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetOperationStatusReqTupleSchemeFactory());
-  }
-
-  private TOperationHandle operationHandle; // required
-  private boolean getProgressUpdate; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    OPERATION_HANDLE((short)1, "operationHandle"),
-    GET_PROGRESS_UPDATE((short)2, "getProgressUpdate");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        case 2: // GET_PROGRESS_UPDATE
-          return GET_PROGRESS_UPDATE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __GETPROGRESSUPDATE_ISSET_ID = 0;
-  private byte __isset_bitfield = 0;
-  private static final _Fields optionals[] = {_Fields.GET_PROGRESS_UPDATE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    tmpMap.put(_Fields.GET_PROGRESS_UPDATE, new org.apache.thrift.meta_data.FieldMetaData("getProgressUpdate", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetOperationStatusReq.class, metaDataMap);
-  }
-
-  public TGetOperationStatusReq() {
-  }
-
-  public TGetOperationStatusReq(
-    TOperationHandle operationHandle)
-  {
-    this();
-    this.operationHandle = operationHandle;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetOperationStatusReq(TGetOperationStatusReq other) {
-    __isset_bitfield = other.__isset_bitfield;
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-    this.getProgressUpdate = other.getProgressUpdate;
-  }
-
-  public TGetOperationStatusReq deepCopy() {
-    return new TGetOperationStatusReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.operationHandle = null;
-    setGetProgressUpdateIsSet(false);
-    this.getProgressUpdate = false;
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public boolean isGetProgressUpdate() {
-    return this.getProgressUpdate;
-  }
-
-  public void setGetProgressUpdate(boolean getProgressUpdate) {
-    this.getProgressUpdate = getProgressUpdate;
-    setGetProgressUpdateIsSet(true);
-  }
-
-  public void unsetGetProgressUpdate() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __GETPROGRESSUPDATE_ISSET_ID);
-  }
-
-  /** Returns true if field getProgressUpdate is set (has been assigned a value) and false otherwise */
-  public boolean isSetGetProgressUpdate() {
-    return EncodingUtils.testBit(__isset_bitfield, __GETPROGRESSUPDATE_ISSET_ID);
-  }
-
-  public void setGetProgressUpdateIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __GETPROGRESSUPDATE_ISSET_ID, value);
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    case GET_PROGRESS_UPDATE:
-      if (value == null) {
-        unsetGetProgressUpdate();
-      } else {
-        setGetProgressUpdate((Boolean)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    case GET_PROGRESS_UPDATE:
-      return isGetProgressUpdate();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    case GET_PROGRESS_UPDATE:
-      return isSetGetProgressUpdate();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetOperationStatusReq)
-      return this.equals((TGetOperationStatusReq)that);
-    return false;
-  }
-
-  public boolean equals(TGetOperationStatusReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    boolean this_present_getProgressUpdate = true && this.isSetGetProgressUpdate();
-    boolean that_present_getProgressUpdate = true && that.isSetGetProgressUpdate();
-    if (this_present_getProgressUpdate || that_present_getProgressUpdate) {
-      if (!(this_present_getProgressUpdate && that_present_getProgressUpdate))
-        return false;
-      if (this.getProgressUpdate != that.getProgressUpdate)
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    list.add(present_operationHandle);
-    if (present_operationHandle)
-      list.add(operationHandle);
-
-    boolean present_getProgressUpdate = true && (isSetGetProgressUpdate());
-    list.add(present_getProgressUpdate);
-    if (present_getProgressUpdate)
-      list.add(getProgressUpdate);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetOperationStatusReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(other.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, other.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetGetProgressUpdate()).compareTo(other.isSetGetProgressUpdate());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetGetProgressUpdate()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.getProgressUpdate, other.getProgressUpdate);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetOperationStatusReq(");
-    boolean first = true;
-
-    sb.append("operationHandle:");
-    if (this.operationHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.operationHandle);
-    }
-    first = false;
-    if (isSetGetProgressUpdate()) {
-      if (!first) sb.append(", ");
-      sb.append("getProgressUpdate:");
-      sb.append(this.getProgressUpdate);
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetOperationHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'operationHandle' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetOperationStatusReqStandardSchemeFactory implements SchemeFactory {
-    public TGetOperationStatusReqStandardScheme getScheme() {
-      return new TGetOperationStatusReqStandardScheme();
-    }
-  }
-
-  private static class TGetOperationStatusReqStandardScheme extends StandardScheme<TGetOperationStatusReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetOperationStatusReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // GET_PROGRESS_UPDATE
-            if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
-              struct.getProgressUpdate = iprot.readBool();
-              struct.setGetProgressUpdateIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetOperationStatusReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.operationHandle != null) {
-        oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-        struct.operationHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.isSetGetProgressUpdate()) {
-        oprot.writeFieldBegin(GET_PROGRESS_UPDATE_FIELD_DESC);
-        oprot.writeBool(struct.getProgressUpdate);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetOperationStatusReqTupleSchemeFactory implements SchemeFactory {
-    public TGetOperationStatusReqTupleScheme getScheme() {
-      return new TGetOperationStatusReqTupleScheme();
-    }
-  }
-
-  private static class TGetOperationStatusReqTupleScheme extends TupleScheme<TGetOperationStatusReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetOperationStatusReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.operationHandle.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetGetProgressUpdate()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetGetProgressUpdate()) {
-        oprot.writeBool(struct.getProgressUpdate);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetOperationStatusReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.operationHandle = new TOperationHandle();
-      struct.operationHandle.read(iprot);
-      struct.setOperationHandleIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.getProgressUpdate = iprot.readBool();
-        struct.setGetProgressUpdateIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetOperationStatusResp.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetOperationStatusResp.java
deleted file mode 100644
index dbfbb44aa6986..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetOperationStatusResp.java
+++ /dev/null
@@ -1,1342 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetOperationStatusResp implements org.apache.thrift.TBase<TGetOperationStatusResp, TGetOperationStatusResp._Fields>, java.io.Serializable, Cloneable, Comparable<TGetOperationStatusResp> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetOperationStatusResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField OPERATION_STATE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationState", org.apache.thrift.protocol.TType.I32, (short)2);
-  private static final org.apache.thrift.protocol.TField SQL_STATE_FIELD_DESC = new org.apache.thrift.protocol.TField("sqlState", org.apache.thrift.protocol.TType.STRING, (short)3);
-  private static final org.apache.thrift.protocol.TField ERROR_CODE_FIELD_DESC = new org.apache.thrift.protocol.TField("errorCode", org.apache.thrift.protocol.TType.I32, (short)4);
-  private static final org.apache.thrift.protocol.TField ERROR_MESSAGE_FIELD_DESC = new org.apache.thrift.protocol.TField("errorMessage", org.apache.thrift.protocol.TType.STRING, (short)5);
-  private static final org.apache.thrift.protocol.TField TASK_STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("taskStatus", org.apache.thrift.protocol.TType.STRING, (short)6);
-  private static final org.apache.thrift.protocol.TField OPERATION_STARTED_FIELD_DESC = new org.apache.thrift.protocol.TField("operationStarted", org.apache.thrift.protocol.TType.I64, (short)7);
-  private static final org.apache.thrift.protocol.TField OPERATION_COMPLETED_FIELD_DESC = new org.apache.thrift.protocol.TField("operationCompleted", org.apache.thrift.protocol.TType.I64, (short)8);
-  private static final org.apache.thrift.protocol.TField HAS_RESULT_SET_FIELD_DESC = new org.apache.thrift.protocol.TField("hasResultSet", org.apache.thrift.protocol.TType.BOOL, (short)9);
-  private static final org.apache.thrift.protocol.TField PROGRESS_UPDATE_RESPONSE_FIELD_DESC = new org.apache.thrift.protocol.TField("progressUpdateResponse", org.apache.thrift.protocol.TType.STRUCT, (short)10);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetOperationStatusRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetOperationStatusRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TOperationState operationState; // optional
-  private String sqlState; // optional
-  private int errorCode; // optional
-  private String errorMessage; // optional
-  private String taskStatus; // optional
-  private long operationStarted; // optional
-  private long operationCompleted; // optional
-  private boolean hasResultSet; // optional
-  private TProgressUpdateResp progressUpdateResponse; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    /**
-     * 
-     * @see TOperationState
-     */
-    OPERATION_STATE((short)2, "operationState"),
-    SQL_STATE((short)3, "sqlState"),
-    ERROR_CODE((short)4, "errorCode"),
-    ERROR_MESSAGE((short)5, "errorMessage"),
-    TASK_STATUS((short)6, "taskStatus"),
-    OPERATION_STARTED((short)7, "operationStarted"),
-    OPERATION_COMPLETED((short)8, "operationCompleted"),
-    HAS_RESULT_SET((short)9, "hasResultSet"),
-    PROGRESS_UPDATE_RESPONSE((short)10, "progressUpdateResponse");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // OPERATION_STATE
-          return OPERATION_STATE;
-        case 3: // SQL_STATE
-          return SQL_STATE;
-        case 4: // ERROR_CODE
-          return ERROR_CODE;
-        case 5: // ERROR_MESSAGE
-          return ERROR_MESSAGE;
-        case 6: // TASK_STATUS
-          return TASK_STATUS;
-        case 7: // OPERATION_STARTED
-          return OPERATION_STARTED;
-        case 8: // OPERATION_COMPLETED
-          return OPERATION_COMPLETED;
-        case 9: // HAS_RESULT_SET
-          return HAS_RESULT_SET;
-        case 10: // PROGRESS_UPDATE_RESPONSE
-          return PROGRESS_UPDATE_RESPONSE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __ERRORCODE_ISSET_ID = 0;
-  private static final int __OPERATIONSTARTED_ISSET_ID = 1;
-  private static final int __OPERATIONCOMPLETED_ISSET_ID = 2;
-  private static final int __HASRESULTSET_ISSET_ID = 3;
-  private byte __isset_bitfield = 0;
-  private static final _Fields optionals[] = {_Fields.OPERATION_STATE,_Fields.SQL_STATE,_Fields.ERROR_CODE,_Fields.ERROR_MESSAGE,_Fields.TASK_STATUS,_Fields.OPERATION_STARTED,_Fields.OPERATION_COMPLETED,_Fields.HAS_RESULT_SET,_Fields.PROGRESS_UPDATE_RESPONSE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.OPERATION_STATE, new org.apache.thrift.meta_data.FieldMetaData("operationState", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TOperationState.class)));
-    tmpMap.put(_Fields.SQL_STATE, new org.apache.thrift.meta_data.FieldMetaData("sqlState", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    tmpMap.put(_Fields.ERROR_CODE, new org.apache.thrift.meta_data.FieldMetaData("errorCode", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
-    tmpMap.put(_Fields.ERROR_MESSAGE, new org.apache.thrift.meta_data.FieldMetaData("errorMessage", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    tmpMap.put(_Fields.TASK_STATUS, new org.apache.thrift.meta_data.FieldMetaData("taskStatus", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    tmpMap.put(_Fields.OPERATION_STARTED, new org.apache.thrift.meta_data.FieldMetaData("operationStarted", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
-    tmpMap.put(_Fields.OPERATION_COMPLETED, new org.apache.thrift.meta_data.FieldMetaData("operationCompleted", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
-    tmpMap.put(_Fields.HAS_RESULT_SET, new org.apache.thrift.meta_data.FieldMetaData("hasResultSet", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
-    tmpMap.put(_Fields.PROGRESS_UPDATE_RESPONSE, new org.apache.thrift.meta_data.FieldMetaData("progressUpdateResponse", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT        , "TProgressUpdateResp")));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetOperationStatusResp.class, metaDataMap);
-  }
-
-  public TGetOperationStatusResp() {
-  }
-
-  public TGetOperationStatusResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetOperationStatusResp(TGetOperationStatusResp other) {
-    __isset_bitfield = other.__isset_bitfield;
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetOperationState()) {
-      this.operationState = other.operationState;
-    }
-    if (other.isSetSqlState()) {
-      this.sqlState = other.sqlState;
-    }
-    this.errorCode = other.errorCode;
-    if (other.isSetErrorMessage()) {
-      this.errorMessage = other.errorMessage;
-    }
-    if (other.isSetTaskStatus()) {
-      this.taskStatus = other.taskStatus;
-    }
-    this.operationStarted = other.operationStarted;
-    this.operationCompleted = other.operationCompleted;
-    this.hasResultSet = other.hasResultSet;
-    if (other.isSetProgressUpdateResponse()) {
-      this.progressUpdateResponse = other.progressUpdateResponse;
-    }
-  }
-
-  public TGetOperationStatusResp deepCopy() {
-    return new TGetOperationStatusResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.operationState = null;
-    this.sqlState = null;
-    setErrorCodeIsSet(false);
-    this.errorCode = 0;
-    this.errorMessage = null;
-    this.taskStatus = null;
-    setOperationStartedIsSet(false);
-    this.operationStarted = 0;
-    setOperationCompletedIsSet(false);
-    this.operationCompleted = 0;
-    setHasResultSetIsSet(false);
-    this.hasResultSet = false;
-    this.progressUpdateResponse = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  /**
-   * 
-   * @see TOperationState
-   */
-  public TOperationState getOperationState() {
-    return this.operationState;
-  }
-
-  /**
-   * 
-   * @see TOperationState
-   */
-  public void setOperationState(TOperationState operationState) {
-    this.operationState = operationState;
-  }
-
-  public void unsetOperationState() {
-    this.operationState = null;
-  }
-
-  /** Returns true if field operationState is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationState() {
-    return this.operationState != null;
-  }
-
-  public void setOperationStateIsSet(boolean value) {
-    if (!value) {
-      this.operationState = null;
-    }
-  }
-
-  public String getSqlState() {
-    return this.sqlState;
-  }
-
-  public void setSqlState(String sqlState) {
-    this.sqlState = sqlState;
-  }
-
-  public void unsetSqlState() {
-    this.sqlState = null;
-  }
-
-  /** Returns true if field sqlState is set (has been assigned a value) and false otherwise */
-  public boolean isSetSqlState() {
-    return this.sqlState != null;
-  }
-
-  public void setSqlStateIsSet(boolean value) {
-    if (!value) {
-      this.sqlState = null;
-    }
-  }
-
-  public int getErrorCode() {
-    return this.errorCode;
-  }
-
-  public void setErrorCode(int errorCode) {
-    this.errorCode = errorCode;
-    setErrorCodeIsSet(true);
-  }
-
-  public void unsetErrorCode() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __ERRORCODE_ISSET_ID);
-  }
-
-  /** Returns true if field errorCode is set (has been assigned a value) and false otherwise */
-  public boolean isSetErrorCode() {
-    return EncodingUtils.testBit(__isset_bitfield, __ERRORCODE_ISSET_ID);
-  }
-
-  public void setErrorCodeIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __ERRORCODE_ISSET_ID, value);
-  }
-
-  public String getErrorMessage() {
-    return this.errorMessage;
-  }
-
-  public void setErrorMessage(String errorMessage) {
-    this.errorMessage = errorMessage;
-  }
-
-  public void unsetErrorMessage() {
-    this.errorMessage = null;
-  }
-
-  /** Returns true if field errorMessage is set (has been assigned a value) and false otherwise */
-  public boolean isSetErrorMessage() {
-    return this.errorMessage != null;
-  }
-
-  public void setErrorMessageIsSet(boolean value) {
-    if (!value) {
-      this.errorMessage = null;
-    }
-  }
-
-  public String getTaskStatus() {
-    return this.taskStatus;
-  }
-
-  public void setTaskStatus(String taskStatus) {
-    this.taskStatus = taskStatus;
-  }
-
-  public void unsetTaskStatus() {
-    this.taskStatus = null;
-  }
-
-  /** Returns true if field taskStatus is set (has been assigned a value) and false otherwise */
-  public boolean isSetTaskStatus() {
-    return this.taskStatus != null;
-  }
-
-  public void setTaskStatusIsSet(boolean value) {
-    if (!value) {
-      this.taskStatus = null;
-    }
-  }
-
-  public long getOperationStarted() {
-    return this.operationStarted;
-  }
-
-  public void setOperationStarted(long operationStarted) {
-    this.operationStarted = operationStarted;
-    setOperationStartedIsSet(true);
-  }
-
-  public void unsetOperationStarted() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __OPERATIONSTARTED_ISSET_ID);
-  }
-
-  /** Returns true if field operationStarted is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationStarted() {
-    return EncodingUtils.testBit(__isset_bitfield, __OPERATIONSTARTED_ISSET_ID);
-  }
-
-  public void setOperationStartedIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __OPERATIONSTARTED_ISSET_ID, value);
-  }
-
-  public long getOperationCompleted() {
-    return this.operationCompleted;
-  }
-
-  public void setOperationCompleted(long operationCompleted) {
-    this.operationCompleted = operationCompleted;
-    setOperationCompletedIsSet(true);
-  }
-
-  public void unsetOperationCompleted() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __OPERATIONCOMPLETED_ISSET_ID);
-  }
-
-  /** Returns true if field operationCompleted is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationCompleted() {
-    return EncodingUtils.testBit(__isset_bitfield, __OPERATIONCOMPLETED_ISSET_ID);
-  }
-
-  public void setOperationCompletedIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __OPERATIONCOMPLETED_ISSET_ID, value);
-  }
-
-  public boolean isHasResultSet() {
-    return this.hasResultSet;
-  }
-
-  public void setHasResultSet(boolean hasResultSet) {
-    this.hasResultSet = hasResultSet;
-    setHasResultSetIsSet(true);
-  }
-
-  public void unsetHasResultSet() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __HASRESULTSET_ISSET_ID);
-  }
-
-  /** Returns true if field hasResultSet is set (has been assigned a value) and false otherwise */
-  public boolean isSetHasResultSet() {
-    return EncodingUtils.testBit(__isset_bitfield, __HASRESULTSET_ISSET_ID);
-  }
-
-  public void setHasResultSetIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __HASRESULTSET_ISSET_ID, value);
-  }
-
-  public TProgressUpdateResp getProgressUpdateResponse() {
-    return this.progressUpdateResponse;
-  }
-
-  public void setProgressUpdateResponse(TProgressUpdateResp progressUpdateResponse) {
-    this.progressUpdateResponse = progressUpdateResponse;
-  }
-
-  public void unsetProgressUpdateResponse() {
-    this.progressUpdateResponse = null;
-  }
-
-  /** Returns true if field progressUpdateResponse is set (has been assigned a value) and false otherwise */
-  public boolean isSetProgressUpdateResponse() {
-    return this.progressUpdateResponse != null;
-  }
-
-  public void setProgressUpdateResponseIsSet(boolean value) {
-    if (!value) {
-      this.progressUpdateResponse = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case OPERATION_STATE:
-      if (value == null) {
-        unsetOperationState();
-      } else {
-        setOperationState((TOperationState)value);
-      }
-      break;
-
-    case SQL_STATE:
-      if (value == null) {
-        unsetSqlState();
-      } else {
-        setSqlState((String)value);
-      }
-      break;
-
-    case ERROR_CODE:
-      if (value == null) {
-        unsetErrorCode();
-      } else {
-        setErrorCode((Integer)value);
-      }
-      break;
-
-    case ERROR_MESSAGE:
-      if (value == null) {
-        unsetErrorMessage();
-      } else {
-        setErrorMessage((String)value);
-      }
-      break;
-
-    case TASK_STATUS:
-      if (value == null) {
-        unsetTaskStatus();
-      } else {
-        setTaskStatus((String)value);
-      }
-      break;
-
-    case OPERATION_STARTED:
-      if (value == null) {
-        unsetOperationStarted();
-      } else {
-        setOperationStarted((Long)value);
-      }
-      break;
-
-    case OPERATION_COMPLETED:
-      if (value == null) {
-        unsetOperationCompleted();
-      } else {
-        setOperationCompleted((Long)value);
-      }
-      break;
-
-    case HAS_RESULT_SET:
-      if (value == null) {
-        unsetHasResultSet();
-      } else {
-        setHasResultSet((Boolean)value);
-      }
-      break;
-
-    case PROGRESS_UPDATE_RESPONSE:
-      if (value == null) {
-        unsetProgressUpdateResponse();
-      } else {
-        setProgressUpdateResponse((TProgressUpdateResp)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case OPERATION_STATE:
-      return getOperationState();
-
-    case SQL_STATE:
-      return getSqlState();
-
-    case ERROR_CODE:
-      return getErrorCode();
-
-    case ERROR_MESSAGE:
-      return getErrorMessage();
-
-    case TASK_STATUS:
-      return getTaskStatus();
-
-    case OPERATION_STARTED:
-      return getOperationStarted();
-
-    case OPERATION_COMPLETED:
-      return getOperationCompleted();
-
-    case HAS_RESULT_SET:
-      return isHasResultSet();
-
-    case PROGRESS_UPDATE_RESPONSE:
-      return getProgressUpdateResponse();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case OPERATION_STATE:
-      return isSetOperationState();
-    case SQL_STATE:
-      return isSetSqlState();
-    case ERROR_CODE:
-      return isSetErrorCode();
-    case ERROR_MESSAGE:
-      return isSetErrorMessage();
-    case TASK_STATUS:
-      return isSetTaskStatus();
-    case OPERATION_STARTED:
-      return isSetOperationStarted();
-    case OPERATION_COMPLETED:
-      return isSetOperationCompleted();
-    case HAS_RESULT_SET:
-      return isSetHasResultSet();
-    case PROGRESS_UPDATE_RESPONSE:
-      return isSetProgressUpdateResponse();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetOperationStatusResp)
-      return this.equals((TGetOperationStatusResp)that);
-    return false;
-  }
-
-  public boolean equals(TGetOperationStatusResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_operationState = true && this.isSetOperationState();
-    boolean that_present_operationState = true && that.isSetOperationState();
-    if (this_present_operationState || that_present_operationState) {
-      if (!(this_present_operationState && that_present_operationState))
-        return false;
-      if (!this.operationState.equals(that.operationState))
-        return false;
-    }
-
-    boolean this_present_sqlState = true && this.isSetSqlState();
-    boolean that_present_sqlState = true && that.isSetSqlState();
-    if (this_present_sqlState || that_present_sqlState) {
-      if (!(this_present_sqlState && that_present_sqlState))
-        return false;
-      if (!this.sqlState.equals(that.sqlState))
-        return false;
-    }
-
-    boolean this_present_errorCode = true && this.isSetErrorCode();
-    boolean that_present_errorCode = true && that.isSetErrorCode();
-    if (this_present_errorCode || that_present_errorCode) {
-      if (!(this_present_errorCode && that_present_errorCode))
-        return false;
-      if (this.errorCode != that.errorCode)
-        return false;
-    }
-
-    boolean this_present_errorMessage = true && this.isSetErrorMessage();
-    boolean that_present_errorMessage = true && that.isSetErrorMessage();
-    if (this_present_errorMessage || that_present_errorMessage) {
-      if (!(this_present_errorMessage && that_present_errorMessage))
-        return false;
-      if (!this.errorMessage.equals(that.errorMessage))
-        return false;
-    }
-
-    boolean this_present_taskStatus = true && this.isSetTaskStatus();
-    boolean that_present_taskStatus = true && that.isSetTaskStatus();
-    if (this_present_taskStatus || that_present_taskStatus) {
-      if (!(this_present_taskStatus && that_present_taskStatus))
-        return false;
-      if (!this.taskStatus.equals(that.taskStatus))
-        return false;
-    }
-
-    boolean this_present_operationStarted = true && this.isSetOperationStarted();
-    boolean that_present_operationStarted = true && that.isSetOperationStarted();
-    if (this_present_operationStarted || that_present_operationStarted) {
-      if (!(this_present_operationStarted && that_present_operationStarted))
-        return false;
-      if (this.operationStarted != that.operationStarted)
-        return false;
-    }
-
-    boolean this_present_operationCompleted = true && this.isSetOperationCompleted();
-    boolean that_present_operationCompleted = true && that.isSetOperationCompleted();
-    if (this_present_operationCompleted || that_present_operationCompleted) {
-      if (!(this_present_operationCompleted && that_present_operationCompleted))
-        return false;
-      if (this.operationCompleted != that.operationCompleted)
-        return false;
-    }
-
-    boolean this_present_hasResultSet = true && this.isSetHasResultSet();
-    boolean that_present_hasResultSet = true && that.isSetHasResultSet();
-    if (this_present_hasResultSet || that_present_hasResultSet) {
-      if (!(this_present_hasResultSet && that_present_hasResultSet))
-        return false;
-      if (this.hasResultSet != that.hasResultSet)
-        return false;
-    }
-
-    boolean this_present_progressUpdateResponse = true && this.isSetProgressUpdateResponse();
-    boolean that_present_progressUpdateResponse = true && that.isSetProgressUpdateResponse();
-    if (this_present_progressUpdateResponse || that_present_progressUpdateResponse) {
-      if (!(this_present_progressUpdateResponse && that_present_progressUpdateResponse))
-        return false;
-      if (!this.progressUpdateResponse.equals(that.progressUpdateResponse))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_status = true && (isSetStatus());
-    list.add(present_status);
-    if (present_status)
-      list.add(status);
-
-    boolean present_operationState = true && (isSetOperationState());
-    list.add(present_operationState);
-    if (present_operationState)
-      list.add(operationState.getValue());
-
-    boolean present_sqlState = true && (isSetSqlState());
-    list.add(present_sqlState);
-    if (present_sqlState)
-      list.add(sqlState);
-
-    boolean present_errorCode = true && (isSetErrorCode());
-    list.add(present_errorCode);
-    if (present_errorCode)
-      list.add(errorCode);
-
-    boolean present_errorMessage = true && (isSetErrorMessage());
-    list.add(present_errorMessage);
-    if (present_errorMessage)
-      list.add(errorMessage);
-
-    boolean present_taskStatus = true && (isSetTaskStatus());
-    list.add(present_taskStatus);
-    if (present_taskStatus)
-      list.add(taskStatus);
-
-    boolean present_operationStarted = true && (isSetOperationStarted());
-    list.add(present_operationStarted);
-    if (present_operationStarted)
-      list.add(operationStarted);
-
-    boolean present_operationCompleted = true && (isSetOperationCompleted());
-    list.add(present_operationCompleted);
-    if (present_operationCompleted)
-      list.add(operationCompleted);
-
-    boolean present_hasResultSet = true && (isSetHasResultSet());
-    list.add(present_hasResultSet);
-    if (present_hasResultSet)
-      list.add(hasResultSet);
-
-    boolean present_progressUpdateResponse = true && (isSetProgressUpdateResponse());
-    list.add(present_progressUpdateResponse);
-    if (present_progressUpdateResponse)
-      list.add(progressUpdateResponse);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetOperationStatusResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(other.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, other.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOperationState()).compareTo(other.isSetOperationState());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationState()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationState, other.operationState);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetSqlState()).compareTo(other.isSetSqlState());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSqlState()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sqlState, other.sqlState);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetErrorCode()).compareTo(other.isSetErrorCode());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetErrorCode()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.errorCode, other.errorCode);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetErrorMessage()).compareTo(other.isSetErrorMessage());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetErrorMessage()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.errorMessage, other.errorMessage);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetTaskStatus()).compareTo(other.isSetTaskStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetTaskStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.taskStatus, other.taskStatus);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOperationStarted()).compareTo(other.isSetOperationStarted());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationStarted()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationStarted, other.operationStarted);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOperationCompleted()).compareTo(other.isSetOperationCompleted());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationCompleted()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationCompleted, other.operationCompleted);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetHasResultSet()).compareTo(other.isSetHasResultSet());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetHasResultSet()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.hasResultSet, other.hasResultSet);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetProgressUpdateResponse()).compareTo(other.isSetProgressUpdateResponse());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetProgressUpdateResponse()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.progressUpdateResponse, other.progressUpdateResponse);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetOperationStatusResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetOperationState()) {
-      if (!first) sb.append(", ");
-      sb.append("operationState:");
-      if (this.operationState == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.operationState);
-      }
-      first = false;
-    }
-    if (isSetSqlState()) {
-      if (!first) sb.append(", ");
-      sb.append("sqlState:");
-      if (this.sqlState == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.sqlState);
-      }
-      first = false;
-    }
-    if (isSetErrorCode()) {
-      if (!first) sb.append(", ");
-      sb.append("errorCode:");
-      sb.append(this.errorCode);
-      first = false;
-    }
-    if (isSetErrorMessage()) {
-      if (!first) sb.append(", ");
-      sb.append("errorMessage:");
-      if (this.errorMessage == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.errorMessage);
-      }
-      first = false;
-    }
-    if (isSetTaskStatus()) {
-      if (!first) sb.append(", ");
-      sb.append("taskStatus:");
-      if (this.taskStatus == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.taskStatus);
-      }
-      first = false;
-    }
-    if (isSetOperationStarted()) {
-      if (!first) sb.append(", ");
-      sb.append("operationStarted:");
-      sb.append(this.operationStarted);
-      first = false;
-    }
-    if (isSetOperationCompleted()) {
-      if (!first) sb.append(", ");
-      sb.append("operationCompleted:");
-      sb.append(this.operationCompleted);
-      first = false;
-    }
-    if (isSetHasResultSet()) {
-      if (!first) sb.append(", ");
-      sb.append("hasResultSet:");
-      sb.append(this.hasResultSet);
-      first = false;
-    }
-    if (isSetProgressUpdateResponse()) {
-      if (!first) sb.append(", ");
-      sb.append("progressUpdateResponse:");
-      if (this.progressUpdateResponse == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.progressUpdateResponse);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetOperationStatusRespStandardSchemeFactory implements SchemeFactory {
-    public TGetOperationStatusRespStandardScheme getScheme() {
-      return new TGetOperationStatusRespStandardScheme();
-    }
-  }
-
-  private static class TGetOperationStatusRespStandardScheme extends StandardScheme<TGetOperationStatusResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetOperationStatusResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // OPERATION_STATE
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.operationState = org.apache.hive.service.rpc.thrift.TOperationState.findByValue(iprot.readI32());
-              struct.setOperationStateIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // SQL_STATE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.sqlState = iprot.readString();
-              struct.setSqlStateIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // ERROR_CODE
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.errorCode = iprot.readI32();
-              struct.setErrorCodeIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 5: // ERROR_MESSAGE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.errorMessage = iprot.readString();
-              struct.setErrorMessageIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 6: // TASK_STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.taskStatus = iprot.readString();
-              struct.setTaskStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 7: // OPERATION_STARTED
-            if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
-              struct.operationStarted = iprot.readI64();
-              struct.setOperationStartedIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 8: // OPERATION_COMPLETED
-            if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
-              struct.operationCompleted = iprot.readI64();
-              struct.setOperationCompletedIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 9: // HAS_RESULT_SET
-            if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
-              struct.hasResultSet = iprot.readBool();
-              struct.setHasResultSetIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 10: // PROGRESS_UPDATE_RESPONSE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.progressUpdateResponse = new TProgressUpdateResp();
-              struct.progressUpdateResponse.read(iprot);
-              struct.setProgressUpdateResponseIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetOperationStatusResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.operationState != null) {
-        if (struct.isSetOperationState()) {
-          oprot.writeFieldBegin(OPERATION_STATE_FIELD_DESC);
-          oprot.writeI32(struct.operationState.getValue());
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.sqlState != null) {
-        if (struct.isSetSqlState()) {
-          oprot.writeFieldBegin(SQL_STATE_FIELD_DESC);
-          oprot.writeString(struct.sqlState);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.isSetErrorCode()) {
-        oprot.writeFieldBegin(ERROR_CODE_FIELD_DESC);
-        oprot.writeI32(struct.errorCode);
-        oprot.writeFieldEnd();
-      }
-      if (struct.errorMessage != null) {
-        if (struct.isSetErrorMessage()) {
-          oprot.writeFieldBegin(ERROR_MESSAGE_FIELD_DESC);
-          oprot.writeString(struct.errorMessage);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.taskStatus != null) {
-        if (struct.isSetTaskStatus()) {
-          oprot.writeFieldBegin(TASK_STATUS_FIELD_DESC);
-          oprot.writeString(struct.taskStatus);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.isSetOperationStarted()) {
-        oprot.writeFieldBegin(OPERATION_STARTED_FIELD_DESC);
-        oprot.writeI64(struct.operationStarted);
-        oprot.writeFieldEnd();
-      }
-      if (struct.isSetOperationCompleted()) {
-        oprot.writeFieldBegin(OPERATION_COMPLETED_FIELD_DESC);
-        oprot.writeI64(struct.operationCompleted);
-        oprot.writeFieldEnd();
-      }
-      if (struct.isSetHasResultSet()) {
-        oprot.writeFieldBegin(HAS_RESULT_SET_FIELD_DESC);
-        oprot.writeBool(struct.hasResultSet);
-        oprot.writeFieldEnd();
-      }
-      if (struct.progressUpdateResponse != null) {
-        if (struct.isSetProgressUpdateResponse()) {
-          oprot.writeFieldBegin(PROGRESS_UPDATE_RESPONSE_FIELD_DESC);
-          struct.progressUpdateResponse.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetOperationStatusRespTupleSchemeFactory implements SchemeFactory {
-    public TGetOperationStatusRespTupleScheme getScheme() {
-      return new TGetOperationStatusRespTupleScheme();
-    }
-  }
-
-  private static class TGetOperationStatusRespTupleScheme extends TupleScheme<TGetOperationStatusResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetOperationStatusResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetOperationState()) {
-        optionals.set(0);
-      }
-      if (struct.isSetSqlState()) {
-        optionals.set(1);
-      }
-      if (struct.isSetErrorCode()) {
-        optionals.set(2);
-      }
-      if (struct.isSetErrorMessage()) {
-        optionals.set(3);
-      }
-      if (struct.isSetTaskStatus()) {
-        optionals.set(4);
-      }
-      if (struct.isSetOperationStarted()) {
-        optionals.set(5);
-      }
-      if (struct.isSetOperationCompleted()) {
-        optionals.set(6);
-      }
-      if (struct.isSetHasResultSet()) {
-        optionals.set(7);
-      }
-      if (struct.isSetProgressUpdateResponse()) {
-        optionals.set(8);
-      }
-      oprot.writeBitSet(optionals, 9);
-      if (struct.isSetOperationState()) {
-        oprot.writeI32(struct.operationState.getValue());
-      }
-      if (struct.isSetSqlState()) {
-        oprot.writeString(struct.sqlState);
-      }
-      if (struct.isSetErrorCode()) {
-        oprot.writeI32(struct.errorCode);
-      }
-      if (struct.isSetErrorMessage()) {
-        oprot.writeString(struct.errorMessage);
-      }
-      if (struct.isSetTaskStatus()) {
-        oprot.writeString(struct.taskStatus);
-      }
-      if (struct.isSetOperationStarted()) {
-        oprot.writeI64(struct.operationStarted);
-      }
-      if (struct.isSetOperationCompleted()) {
-        oprot.writeI64(struct.operationCompleted);
-      }
-      if (struct.isSetHasResultSet()) {
-        oprot.writeBool(struct.hasResultSet);
-      }
-      if (struct.isSetProgressUpdateResponse()) {
-        struct.progressUpdateResponse.write(oprot);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetOperationStatusResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(9);
-      if (incoming.get(0)) {
-        struct.operationState = org.apache.hive.service.rpc.thrift.TOperationState.findByValue(iprot.readI32());
-        struct.setOperationStateIsSet(true);
-      }
-      if (incoming.get(1)) {
-        struct.sqlState = iprot.readString();
-        struct.setSqlStateIsSet(true);
-      }
-      if (incoming.get(2)) {
-        struct.errorCode = iprot.readI32();
-        struct.setErrorCodeIsSet(true);
-      }
-      if (incoming.get(3)) {
-        struct.errorMessage = iprot.readString();
-        struct.setErrorMessageIsSet(true);
-      }
-      if (incoming.get(4)) {
-        struct.taskStatus = iprot.readString();
-        struct.setTaskStatusIsSet(true);
-      }
-      if (incoming.get(5)) {
-        struct.operationStarted = iprot.readI64();
-        struct.setOperationStartedIsSet(true);
-      }
-      if (incoming.get(6)) {
-        struct.operationCompleted = iprot.readI64();
-        struct.setOperationCompletedIsSet(true);
-      }
-      if (incoming.get(7)) {
-        struct.hasResultSet = iprot.readBool();
-        struct.setHasResultSetIsSet(true);
-      }
-      if (incoming.get(8)) {
-        struct.progressUpdateResponse = new TProgressUpdateResp();
-        struct.progressUpdateResponse.read(iprot);
-        struct.setProgressUpdateResponseIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetPrimaryKeysReq.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetPrimaryKeysReq.java
deleted file mode 100644
index 1bec9b51c72d8..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetPrimaryKeysReq.java
+++ /dev/null
@@ -1,716 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetPrimaryKeysReq implements org.apache.thrift.TBase<TGetPrimaryKeysReq, TGetPrimaryKeysReq._Fields>, java.io.Serializable, Cloneable, Comparable<TGetPrimaryKeysReq> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetPrimaryKeysReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField CATALOG_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("catalogName", org.apache.thrift.protocol.TType.STRING, (short)2);
-  private static final org.apache.thrift.protocol.TField SCHEMA_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("schemaName", org.apache.thrift.protocol.TType.STRING, (short)3);
-  private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)4);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetPrimaryKeysReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetPrimaryKeysReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-  private String catalogName; // optional
-  private String schemaName; // optional
-  private String tableName; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle"),
-    CATALOG_NAME((short)2, "catalogName"),
-    SCHEMA_NAME((short)3, "schemaName"),
-    TABLE_NAME((short)4, "tableName");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        case 2: // CATALOG_NAME
-          return CATALOG_NAME;
-        case 3: // SCHEMA_NAME
-          return SCHEMA_NAME;
-        case 4: // TABLE_NAME
-          return TABLE_NAME;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final _Fields optionals[] = {_Fields.CATALOG_NAME,_Fields.SCHEMA_NAME,_Fields.TABLE_NAME};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    tmpMap.put(_Fields.CATALOG_NAME, new org.apache.thrift.meta_data.FieldMetaData("catalogName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TIdentifier")));
-    tmpMap.put(_Fields.SCHEMA_NAME, new org.apache.thrift.meta_data.FieldMetaData("schemaName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TIdentifier")));
-    tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TIdentifier")));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetPrimaryKeysReq.class, metaDataMap);
-  }
-
-  public TGetPrimaryKeysReq() {
-  }
-
-  public TGetPrimaryKeysReq(
-    TSessionHandle sessionHandle)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetPrimaryKeysReq(TGetPrimaryKeysReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-    if (other.isSetCatalogName()) {
-      this.catalogName = other.catalogName;
-    }
-    if (other.isSetSchemaName()) {
-      this.schemaName = other.schemaName;
-    }
-    if (other.isSetTableName()) {
-      this.tableName = other.tableName;
-    }
-  }
-
-  public TGetPrimaryKeysReq deepCopy() {
-    return new TGetPrimaryKeysReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-    this.catalogName = null;
-    this.schemaName = null;
-    this.tableName = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public String getCatalogName() {
-    return this.catalogName;
-  }
-
-  public void setCatalogName(String catalogName) {
-    this.catalogName = catalogName;
-  }
-
-  public void unsetCatalogName() {
-    this.catalogName = null;
-  }
-
-  /** Returns true if field catalogName is set (has been assigned a value) and false otherwise */
-  public boolean isSetCatalogName() {
-    return this.catalogName != null;
-  }
-
-  public void setCatalogNameIsSet(boolean value) {
-    if (!value) {
-      this.catalogName = null;
-    }
-  }
-
-  public String getSchemaName() {
-    return this.schemaName;
-  }
-
-  public void setSchemaName(String schemaName) {
-    this.schemaName = schemaName;
-  }
-
-  public void unsetSchemaName() {
-    this.schemaName = null;
-  }
-
-  /** Returns true if field schemaName is set (has been assigned a value) and false otherwise */
-  public boolean isSetSchemaName() {
-    return this.schemaName != null;
-  }
-
-  public void setSchemaNameIsSet(boolean value) {
-    if (!value) {
-      this.schemaName = null;
-    }
-  }
-
-  public String getTableName() {
-    return this.tableName;
-  }
-
-  public void setTableName(String tableName) {
-    this.tableName = tableName;
-  }
-
-  public void unsetTableName() {
-    this.tableName = null;
-  }
-
-  /** Returns true if field tableName is set (has been assigned a value) and false otherwise */
-  public boolean isSetTableName() {
-    return this.tableName != null;
-  }
-
-  public void setTableNameIsSet(boolean value) {
-    if (!value) {
-      this.tableName = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    case CATALOG_NAME:
-      if (value == null) {
-        unsetCatalogName();
-      } else {
-        setCatalogName((String)value);
-      }
-      break;
-
-    case SCHEMA_NAME:
-      if (value == null) {
-        unsetSchemaName();
-      } else {
-        setSchemaName((String)value);
-      }
-      break;
-
-    case TABLE_NAME:
-      if (value == null) {
-        unsetTableName();
-      } else {
-        setTableName((String)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    case CATALOG_NAME:
-      return getCatalogName();
-
-    case SCHEMA_NAME:
-      return getSchemaName();
-
-    case TABLE_NAME:
-      return getTableName();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    case CATALOG_NAME:
-      return isSetCatalogName();
-    case SCHEMA_NAME:
-      return isSetSchemaName();
-    case TABLE_NAME:
-      return isSetTableName();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetPrimaryKeysReq)
-      return this.equals((TGetPrimaryKeysReq)that);
-    return false;
-  }
-
-  public boolean equals(TGetPrimaryKeysReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    boolean this_present_catalogName = true && this.isSetCatalogName();
-    boolean that_present_catalogName = true && that.isSetCatalogName();
-    if (this_present_catalogName || that_present_catalogName) {
-      if (!(this_present_catalogName && that_present_catalogName))
-        return false;
-      if (!this.catalogName.equals(that.catalogName))
-        return false;
-    }
-
-    boolean this_present_schemaName = true && this.isSetSchemaName();
-    boolean that_present_schemaName = true && that.isSetSchemaName();
-    if (this_present_schemaName || that_present_schemaName) {
-      if (!(this_present_schemaName && that_present_schemaName))
-        return false;
-      if (!this.schemaName.equals(that.schemaName))
-        return false;
-    }
-
-    boolean this_present_tableName = true && this.isSetTableName();
-    boolean that_present_tableName = true && that.isSetTableName();
-    if (this_present_tableName || that_present_tableName) {
-      if (!(this_present_tableName && that_present_tableName))
-        return false;
-      if (!this.tableName.equals(that.tableName))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    list.add(present_sessionHandle);
-    if (present_sessionHandle)
-      list.add(sessionHandle);
-
-    boolean present_catalogName = true && (isSetCatalogName());
-    list.add(present_catalogName);
-    if (present_catalogName)
-      list.add(catalogName);
-
-    boolean present_schemaName = true && (isSetSchemaName());
-    list.add(present_schemaName);
-    if (present_schemaName)
-      list.add(schemaName);
-
-    boolean present_tableName = true && (isSetTableName());
-    list.add(present_tableName);
-    if (present_tableName)
-      list.add(tableName);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetPrimaryKeysReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(other.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, other.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetCatalogName()).compareTo(other.isSetCatalogName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetCatalogName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.catalogName, other.catalogName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetSchemaName()).compareTo(other.isSetSchemaName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSchemaName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.schemaName, other.schemaName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetTableName()).compareTo(other.isSetTableName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetTableName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, other.tableName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetPrimaryKeysReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    if (isSetCatalogName()) {
-      if (!first) sb.append(", ");
-      sb.append("catalogName:");
-      if (this.catalogName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.catalogName);
-      }
-      first = false;
-    }
-    if (isSetSchemaName()) {
-      if (!first) sb.append(", ");
-      sb.append("schemaName:");
-      if (this.schemaName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.schemaName);
-      }
-      first = false;
-    }
-    if (isSetTableName()) {
-      if (!first) sb.append(", ");
-      sb.append("tableName:");
-      if (this.tableName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.tableName);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetPrimaryKeysReqStandardSchemeFactory implements SchemeFactory {
-    public TGetPrimaryKeysReqStandardScheme getScheme() {
-      return new TGetPrimaryKeysReqStandardScheme();
-    }
-  }
-
-  private static class TGetPrimaryKeysReqStandardScheme extends StandardScheme<TGetPrimaryKeysReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetPrimaryKeysReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // CATALOG_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.catalogName = iprot.readString();
-              struct.setCatalogNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // SCHEMA_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.schemaName = iprot.readString();
-              struct.setSchemaNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // TABLE_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.tableName = iprot.readString();
-              struct.setTableNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetPrimaryKeysReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.catalogName != null) {
-        if (struct.isSetCatalogName()) {
-          oprot.writeFieldBegin(CATALOG_NAME_FIELD_DESC);
-          oprot.writeString(struct.catalogName);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.schemaName != null) {
-        if (struct.isSetSchemaName()) {
-          oprot.writeFieldBegin(SCHEMA_NAME_FIELD_DESC);
-          oprot.writeString(struct.schemaName);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.tableName != null) {
-        if (struct.isSetTableName()) {
-          oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC);
-          oprot.writeString(struct.tableName);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetPrimaryKeysReqTupleSchemeFactory implements SchemeFactory {
-    public TGetPrimaryKeysReqTupleScheme getScheme() {
-      return new TGetPrimaryKeysReqTupleScheme();
-    }
-  }
-
-  private static class TGetPrimaryKeysReqTupleScheme extends TupleScheme<TGetPrimaryKeysReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetPrimaryKeysReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetCatalogName()) {
-        optionals.set(0);
-      }
-      if (struct.isSetSchemaName()) {
-        optionals.set(1);
-      }
-      if (struct.isSetTableName()) {
-        optionals.set(2);
-      }
-      oprot.writeBitSet(optionals, 3);
-      if (struct.isSetCatalogName()) {
-        oprot.writeString(struct.catalogName);
-      }
-      if (struct.isSetSchemaName()) {
-        oprot.writeString(struct.schemaName);
-      }
-      if (struct.isSetTableName()) {
-        oprot.writeString(struct.tableName);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetPrimaryKeysReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-      BitSet incoming = iprot.readBitSet(3);
-      if (incoming.get(0)) {
-        struct.catalogName = iprot.readString();
-        struct.setCatalogNameIsSet(true);
-      }
-      if (incoming.get(1)) {
-        struct.schemaName = iprot.readString();
-        struct.setSchemaNameIsSet(true);
-      }
-      if (incoming.get(2)) {
-        struct.tableName = iprot.readString();
-        struct.setTableNameIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetPrimaryKeysResp.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetPrimaryKeysResp.java
deleted file mode 100644
index 72d9507fe1031..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetPrimaryKeysResp.java
+++ /dev/null
@@ -1,509 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetPrimaryKeysResp implements org.apache.thrift.TBase<TGetPrimaryKeysResp, TGetPrimaryKeysResp._Fields>, java.io.Serializable, Cloneable, Comparable<TGetPrimaryKeysResp> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetPrimaryKeysResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetPrimaryKeysRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetPrimaryKeysRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TOperationHandle operationHandle; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    OPERATION_HANDLE((short)2, "operationHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final _Fields optionals[] = {_Fields.OPERATION_HANDLE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetPrimaryKeysResp.class, metaDataMap);
-  }
-
-  public TGetPrimaryKeysResp() {
-  }
-
-  public TGetPrimaryKeysResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetPrimaryKeysResp(TGetPrimaryKeysResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-  }
-
-  public TGetPrimaryKeysResp deepCopy() {
-    return new TGetPrimaryKeysResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.operationHandle = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetPrimaryKeysResp)
-      return this.equals((TGetPrimaryKeysResp)that);
-    return false;
-  }
-
-  public boolean equals(TGetPrimaryKeysResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_status = true && (isSetStatus());
-    list.add(present_status);
-    if (present_status)
-      list.add(status);
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    list.add(present_operationHandle);
-    if (present_operationHandle)
-      list.add(operationHandle);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetPrimaryKeysResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(other.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, other.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(other.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, other.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetPrimaryKeysResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetOperationHandle()) {
-      if (!first) sb.append(", ");
-      sb.append("operationHandle:");
-      if (this.operationHandle == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.operationHandle);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetPrimaryKeysRespStandardSchemeFactory implements SchemeFactory {
-    public TGetPrimaryKeysRespStandardScheme getScheme() {
-      return new TGetPrimaryKeysRespStandardScheme();
-    }
-  }
-
-  private static class TGetPrimaryKeysRespStandardScheme extends StandardScheme<TGetPrimaryKeysResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetPrimaryKeysResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetPrimaryKeysResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.operationHandle != null) {
-        if (struct.isSetOperationHandle()) {
-          oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-          struct.operationHandle.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetPrimaryKeysRespTupleSchemeFactory implements SchemeFactory {
-    public TGetPrimaryKeysRespTupleScheme getScheme() {
-      return new TGetPrimaryKeysRespTupleScheme();
-    }
-  }
-
-  private static class TGetPrimaryKeysRespTupleScheme extends TupleScheme<TGetPrimaryKeysResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetPrimaryKeysResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetOperationHandle()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetOperationHandle()) {
-        struct.operationHandle.write(oprot);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetPrimaryKeysResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.operationHandle = new TOperationHandle();
-        struct.operationHandle.read(iprot);
-        struct.setOperationHandleIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetResultSetMetadataReq.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetResultSetMetadataReq.java
deleted file mode 100644
index b94d827de264d..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetResultSetMetadataReq.java
+++ /dev/null
@@ -1,394 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetResultSetMetadataReq implements org.apache.thrift.TBase<TGetResultSetMetadataReq, TGetResultSetMetadataReq._Fields>, java.io.Serializable, Cloneable, Comparable<TGetResultSetMetadataReq> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetResultSetMetadataReq");
-
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetResultSetMetadataReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetResultSetMetadataReqTupleSchemeFactory());
-  }
-
-  private TOperationHandle operationHandle; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    OPERATION_HANDLE((short)1, "operationHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetResultSetMetadataReq.class, metaDataMap);
-  }
-
-  public TGetResultSetMetadataReq() {
-  }
-
-  public TGetResultSetMetadataReq(
-    TOperationHandle operationHandle)
-  {
-    this();
-    this.operationHandle = operationHandle;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetResultSetMetadataReq(TGetResultSetMetadataReq other) {
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-  }
-
-  public TGetResultSetMetadataReq deepCopy() {
-    return new TGetResultSetMetadataReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.operationHandle = null;
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetResultSetMetadataReq)
-      return this.equals((TGetResultSetMetadataReq)that);
-    return false;
-  }
-
-  public boolean equals(TGetResultSetMetadataReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    list.add(present_operationHandle);
-    if (present_operationHandle)
-      list.add(operationHandle);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetResultSetMetadataReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(other.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, other.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetResultSetMetadataReq(");
-    boolean first = true;
-
-    sb.append("operationHandle:");
-    if (this.operationHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.operationHandle);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetOperationHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'operationHandle' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetResultSetMetadataReqStandardSchemeFactory implements SchemeFactory {
-    public TGetResultSetMetadataReqStandardScheme getScheme() {
-      return new TGetResultSetMetadataReqStandardScheme();
-    }
-  }
-
-  private static class TGetResultSetMetadataReqStandardScheme extends StandardScheme<TGetResultSetMetadataReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetResultSetMetadataReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetResultSetMetadataReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.operationHandle != null) {
-        oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-        struct.operationHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetResultSetMetadataReqTupleSchemeFactory implements SchemeFactory {
-    public TGetResultSetMetadataReqTupleScheme getScheme() {
-      return new TGetResultSetMetadataReqTupleScheme();
-    }
-  }
-
-  private static class TGetResultSetMetadataReqTupleScheme extends TupleScheme<TGetResultSetMetadataReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetResultSetMetadataReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.operationHandle.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetResultSetMetadataReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.operationHandle = new TOperationHandle();
-      struct.operationHandle.read(iprot);
-      struct.setOperationHandleIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetResultSetMetadataResp.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetResultSetMetadataResp.java
deleted file mode 100644
index ae2021ebd5a10..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetResultSetMetadataResp.java
+++ /dev/null
@@ -1,509 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetResultSetMetadataResp implements org.apache.thrift.TBase<TGetResultSetMetadataResp, TGetResultSetMetadataResp._Fields>, java.io.Serializable, Cloneable, Comparable<TGetResultSetMetadataResp> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetResultSetMetadataResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField SCHEMA_FIELD_DESC = new org.apache.thrift.protocol.TField("schema", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetResultSetMetadataRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetResultSetMetadataRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TTableSchema schema; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    SCHEMA((short)2, "schema");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // SCHEMA
-          return SCHEMA;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final _Fields optionals[] = {_Fields.SCHEMA};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.SCHEMA, new org.apache.thrift.meta_data.FieldMetaData("schema", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TTableSchema.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetResultSetMetadataResp.class, metaDataMap);
-  }
-
-  public TGetResultSetMetadataResp() {
-  }
-
-  public TGetResultSetMetadataResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetResultSetMetadataResp(TGetResultSetMetadataResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetSchema()) {
-      this.schema = new TTableSchema(other.schema);
-    }
-  }
-
-  public TGetResultSetMetadataResp deepCopy() {
-    return new TGetResultSetMetadataResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.schema = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public TTableSchema getSchema() {
-    return this.schema;
-  }
-
-  public void setSchema(TTableSchema schema) {
-    this.schema = schema;
-  }
-
-  public void unsetSchema() {
-    this.schema = null;
-  }
-
-  /** Returns true if field schema is set (has been assigned a value) and false otherwise */
-  public boolean isSetSchema() {
-    return this.schema != null;
-  }
-
-  public void setSchemaIsSet(boolean value) {
-    if (!value) {
-      this.schema = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case SCHEMA:
-      if (value == null) {
-        unsetSchema();
-      } else {
-        setSchema((TTableSchema)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case SCHEMA:
-      return getSchema();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case SCHEMA:
-      return isSetSchema();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetResultSetMetadataResp)
-      return this.equals((TGetResultSetMetadataResp)that);
-    return false;
-  }
-
-  public boolean equals(TGetResultSetMetadataResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_schema = true && this.isSetSchema();
-    boolean that_present_schema = true && that.isSetSchema();
-    if (this_present_schema || that_present_schema) {
-      if (!(this_present_schema && that_present_schema))
-        return false;
-      if (!this.schema.equals(that.schema))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_status = true && (isSetStatus());
-    list.add(present_status);
-    if (present_status)
-      list.add(status);
-
-    boolean present_schema = true && (isSetSchema());
-    list.add(present_schema);
-    if (present_schema)
-      list.add(schema);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetResultSetMetadataResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(other.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, other.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetSchema()).compareTo(other.isSetSchema());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSchema()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.schema, other.schema);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetResultSetMetadataResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetSchema()) {
-      if (!first) sb.append(", ");
-      sb.append("schema:");
-      if (this.schema == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.schema);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-    if (schema != null) {
-      schema.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetResultSetMetadataRespStandardSchemeFactory implements SchemeFactory {
-    public TGetResultSetMetadataRespStandardScheme getScheme() {
-      return new TGetResultSetMetadataRespStandardScheme();
-    }
-  }
-
-  private static class TGetResultSetMetadataRespStandardScheme extends StandardScheme<TGetResultSetMetadataResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetResultSetMetadataResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // SCHEMA
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.schema = new TTableSchema();
-              struct.schema.read(iprot);
-              struct.setSchemaIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetResultSetMetadataResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.schema != null) {
-        if (struct.isSetSchema()) {
-          oprot.writeFieldBegin(SCHEMA_FIELD_DESC);
-          struct.schema.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetResultSetMetadataRespTupleSchemeFactory implements SchemeFactory {
-    public TGetResultSetMetadataRespTupleScheme getScheme() {
-      return new TGetResultSetMetadataRespTupleScheme();
-    }
-  }
-
-  private static class TGetResultSetMetadataRespTupleScheme extends TupleScheme<TGetResultSetMetadataResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetResultSetMetadataResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetSchema()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetSchema()) {
-        struct.schema.write(oprot);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetResultSetMetadataResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.schema = new TTableSchema();
-        struct.schema.read(iprot);
-        struct.setSchemaIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetSchemasReq.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetSchemasReq.java
deleted file mode 100644
index 17eed87ae096f..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetSchemasReq.java
+++ /dev/null
@@ -1,610 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetSchemasReq implements org.apache.thrift.TBase<TGetSchemasReq, TGetSchemasReq._Fields>, java.io.Serializable, Cloneable, Comparable<TGetSchemasReq> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetSchemasReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField CATALOG_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("catalogName", org.apache.thrift.protocol.TType.STRING, (short)2);
-  private static final org.apache.thrift.protocol.TField SCHEMA_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("schemaName", org.apache.thrift.protocol.TType.STRING, (short)3);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetSchemasReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetSchemasReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-  private String catalogName; // optional
-  private String schemaName; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle"),
-    CATALOG_NAME((short)2, "catalogName"),
-    SCHEMA_NAME((short)3, "schemaName");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        case 2: // CATALOG_NAME
-          return CATALOG_NAME;
-        case 3: // SCHEMA_NAME
-          return SCHEMA_NAME;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final _Fields optionals[] = {_Fields.CATALOG_NAME,_Fields.SCHEMA_NAME};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    tmpMap.put(_Fields.CATALOG_NAME, new org.apache.thrift.meta_data.FieldMetaData("catalogName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TIdentifier")));
-    tmpMap.put(_Fields.SCHEMA_NAME, new org.apache.thrift.meta_data.FieldMetaData("schemaName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TPatternOrIdentifier")));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetSchemasReq.class, metaDataMap);
-  }
-
-  public TGetSchemasReq() {
-  }
-
-  public TGetSchemasReq(
-    TSessionHandle sessionHandle)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetSchemasReq(TGetSchemasReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-    if (other.isSetCatalogName()) {
-      this.catalogName = other.catalogName;
-    }
-    if (other.isSetSchemaName()) {
-      this.schemaName = other.schemaName;
-    }
-  }
-
-  public TGetSchemasReq deepCopy() {
-    return new TGetSchemasReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-    this.catalogName = null;
-    this.schemaName = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public String getCatalogName() {
-    return this.catalogName;
-  }
-
-  public void setCatalogName(String catalogName) {
-    this.catalogName = catalogName;
-  }
-
-  public void unsetCatalogName() {
-    this.catalogName = null;
-  }
-
-  /** Returns true if field catalogName is set (has been assigned a value) and false otherwise */
-  public boolean isSetCatalogName() {
-    return this.catalogName != null;
-  }
-
-  public void setCatalogNameIsSet(boolean value) {
-    if (!value) {
-      this.catalogName = null;
-    }
-  }
-
-  public String getSchemaName() {
-    return this.schemaName;
-  }
-
-  public void setSchemaName(String schemaName) {
-    this.schemaName = schemaName;
-  }
-
-  public void unsetSchemaName() {
-    this.schemaName = null;
-  }
-
-  /** Returns true if field schemaName is set (has been assigned a value) and false otherwise */
-  public boolean isSetSchemaName() {
-    return this.schemaName != null;
-  }
-
-  public void setSchemaNameIsSet(boolean value) {
-    if (!value) {
-      this.schemaName = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    case CATALOG_NAME:
-      if (value == null) {
-        unsetCatalogName();
-      } else {
-        setCatalogName((String)value);
-      }
-      break;
-
-    case SCHEMA_NAME:
-      if (value == null) {
-        unsetSchemaName();
-      } else {
-        setSchemaName((String)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    case CATALOG_NAME:
-      return getCatalogName();
-
-    case SCHEMA_NAME:
-      return getSchemaName();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    case CATALOG_NAME:
-      return isSetCatalogName();
-    case SCHEMA_NAME:
-      return isSetSchemaName();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetSchemasReq)
-      return this.equals((TGetSchemasReq)that);
-    return false;
-  }
-
-  public boolean equals(TGetSchemasReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    boolean this_present_catalogName = true && this.isSetCatalogName();
-    boolean that_present_catalogName = true && that.isSetCatalogName();
-    if (this_present_catalogName || that_present_catalogName) {
-      if (!(this_present_catalogName && that_present_catalogName))
-        return false;
-      if (!this.catalogName.equals(that.catalogName))
-        return false;
-    }
-
-    boolean this_present_schemaName = true && this.isSetSchemaName();
-    boolean that_present_schemaName = true && that.isSetSchemaName();
-    if (this_present_schemaName || that_present_schemaName) {
-      if (!(this_present_schemaName && that_present_schemaName))
-        return false;
-      if (!this.schemaName.equals(that.schemaName))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    list.add(present_sessionHandle);
-    if (present_sessionHandle)
-      list.add(sessionHandle);
-
-    boolean present_catalogName = true && (isSetCatalogName());
-    list.add(present_catalogName);
-    if (present_catalogName)
-      list.add(catalogName);
-
-    boolean present_schemaName = true && (isSetSchemaName());
-    list.add(present_schemaName);
-    if (present_schemaName)
-      list.add(schemaName);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetSchemasReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(other.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, other.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetCatalogName()).compareTo(other.isSetCatalogName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetCatalogName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.catalogName, other.catalogName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetSchemaName()).compareTo(other.isSetSchemaName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSchemaName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.schemaName, other.schemaName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetSchemasReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    if (isSetCatalogName()) {
-      if (!first) sb.append(", ");
-      sb.append("catalogName:");
-      if (this.catalogName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.catalogName);
-      }
-      first = false;
-    }
-    if (isSetSchemaName()) {
-      if (!first) sb.append(", ");
-      sb.append("schemaName:");
-      if (this.schemaName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.schemaName);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetSchemasReqStandardSchemeFactory implements SchemeFactory {
-    public TGetSchemasReqStandardScheme getScheme() {
-      return new TGetSchemasReqStandardScheme();
-    }
-  }
-
-  private static class TGetSchemasReqStandardScheme extends StandardScheme<TGetSchemasReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetSchemasReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // CATALOG_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.catalogName = iprot.readString();
-              struct.setCatalogNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // SCHEMA_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.schemaName = iprot.readString();
-              struct.setSchemaNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetSchemasReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.catalogName != null) {
-        if (struct.isSetCatalogName()) {
-          oprot.writeFieldBegin(CATALOG_NAME_FIELD_DESC);
-          oprot.writeString(struct.catalogName);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.schemaName != null) {
-        if (struct.isSetSchemaName()) {
-          oprot.writeFieldBegin(SCHEMA_NAME_FIELD_DESC);
-          oprot.writeString(struct.schemaName);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetSchemasReqTupleSchemeFactory implements SchemeFactory {
-    public TGetSchemasReqTupleScheme getScheme() {
-      return new TGetSchemasReqTupleScheme();
-    }
-  }
-
-  private static class TGetSchemasReqTupleScheme extends TupleScheme<TGetSchemasReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetSchemasReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetCatalogName()) {
-        optionals.set(0);
-      }
-      if (struct.isSetSchemaName()) {
-        optionals.set(1);
-      }
-      oprot.writeBitSet(optionals, 2);
-      if (struct.isSetCatalogName()) {
-        oprot.writeString(struct.catalogName);
-      }
-      if (struct.isSetSchemaName()) {
-        oprot.writeString(struct.schemaName);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetSchemasReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-      BitSet incoming = iprot.readBitSet(2);
-      if (incoming.get(0)) {
-        struct.catalogName = iprot.readString();
-        struct.setCatalogNameIsSet(true);
-      }
-      if (incoming.get(1)) {
-        struct.schemaName = iprot.readString();
-        struct.setSchemaNameIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetSchemasResp.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetSchemasResp.java
deleted file mode 100644
index e5317f7ff5046..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetSchemasResp.java
+++ /dev/null
@@ -1,509 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetSchemasResp implements org.apache.thrift.TBase<TGetSchemasResp, TGetSchemasResp._Fields>, java.io.Serializable, Cloneable, Comparable<TGetSchemasResp> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetSchemasResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetSchemasRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetSchemasRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TOperationHandle operationHandle; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    OPERATION_HANDLE((short)2, "operationHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final _Fields optionals[] = {_Fields.OPERATION_HANDLE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetSchemasResp.class, metaDataMap);
-  }
-
-  public TGetSchemasResp() {
-  }
-
-  public TGetSchemasResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetSchemasResp(TGetSchemasResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-  }
-
-  public TGetSchemasResp deepCopy() {
-    return new TGetSchemasResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.operationHandle = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetSchemasResp)
-      return this.equals((TGetSchemasResp)that);
-    return false;
-  }
-
-  public boolean equals(TGetSchemasResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_status = true && (isSetStatus());
-    list.add(present_status);
-    if (present_status)
-      list.add(status);
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    list.add(present_operationHandle);
-    if (present_operationHandle)
-      list.add(operationHandle);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetSchemasResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(other.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, other.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(other.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, other.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetSchemasResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetOperationHandle()) {
-      if (!first) sb.append(", ");
-      sb.append("operationHandle:");
-      if (this.operationHandle == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.operationHandle);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetSchemasRespStandardSchemeFactory implements SchemeFactory {
-    public TGetSchemasRespStandardScheme getScheme() {
-      return new TGetSchemasRespStandardScheme();
-    }
-  }
-
-  private static class TGetSchemasRespStandardScheme extends StandardScheme<TGetSchemasResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetSchemasResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetSchemasResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.operationHandle != null) {
-        if (struct.isSetOperationHandle()) {
-          oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-          struct.operationHandle.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetSchemasRespTupleSchemeFactory implements SchemeFactory {
-    public TGetSchemasRespTupleScheme getScheme() {
-      return new TGetSchemasRespTupleScheme();
-    }
-  }
-
-  private static class TGetSchemasRespTupleScheme extends TupleScheme<TGetSchemasResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetSchemasResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetOperationHandle()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetOperationHandle()) {
-        struct.operationHandle.write(oprot);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetSchemasResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.operationHandle = new TOperationHandle();
-        struct.operationHandle.read(iprot);
-        struct.setOperationHandleIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetTableTypesReq.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetTableTypesReq.java
deleted file mode 100644
index c027748a336e6..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetTableTypesReq.java
+++ /dev/null
@@ -1,394 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetTableTypesReq implements org.apache.thrift.TBase<TGetTableTypesReq, TGetTableTypesReq._Fields>, java.io.Serializable, Cloneable, Comparable<TGetTableTypesReq> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetTableTypesReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetTableTypesReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetTableTypesReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetTableTypesReq.class, metaDataMap);
-  }
-
-  public TGetTableTypesReq() {
-  }
-
-  public TGetTableTypesReq(
-    TSessionHandle sessionHandle)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetTableTypesReq(TGetTableTypesReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-  }
-
-  public TGetTableTypesReq deepCopy() {
-    return new TGetTableTypesReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetTableTypesReq)
-      return this.equals((TGetTableTypesReq)that);
-    return false;
-  }
-
-  public boolean equals(TGetTableTypesReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    list.add(present_sessionHandle);
-    if (present_sessionHandle)
-      list.add(sessionHandle);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetTableTypesReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(other.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, other.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetTableTypesReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetTableTypesReqStandardSchemeFactory implements SchemeFactory {
-    public TGetTableTypesReqStandardScheme getScheme() {
-      return new TGetTableTypesReqStandardScheme();
-    }
-  }
-
-  private static class TGetTableTypesReqStandardScheme extends StandardScheme<TGetTableTypesReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetTableTypesReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetTableTypesReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetTableTypesReqTupleSchemeFactory implements SchemeFactory {
-    public TGetTableTypesReqTupleScheme getScheme() {
-      return new TGetTableTypesReqTupleScheme();
-    }
-  }
-
-  private static class TGetTableTypesReqTupleScheme extends TupleScheme<TGetTableTypesReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetTableTypesReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetTableTypesReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetTableTypesResp.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetTableTypesResp.java
deleted file mode 100644
index c6ce0d4368fdd..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetTableTypesResp.java
+++ /dev/null
@@ -1,509 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetTableTypesResp implements org.apache.thrift.TBase<TGetTableTypesResp, TGetTableTypesResp._Fields>, java.io.Serializable, Cloneable, Comparable<TGetTableTypesResp> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetTableTypesResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetTableTypesRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetTableTypesRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TOperationHandle operationHandle; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    OPERATION_HANDLE((short)2, "operationHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final _Fields optionals[] = {_Fields.OPERATION_HANDLE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetTableTypesResp.class, metaDataMap);
-  }
-
-  public TGetTableTypesResp() {
-  }
-
-  public TGetTableTypesResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetTableTypesResp(TGetTableTypesResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-  }
-
-  public TGetTableTypesResp deepCopy() {
-    return new TGetTableTypesResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.operationHandle = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetTableTypesResp)
-      return this.equals((TGetTableTypesResp)that);
-    return false;
-  }
-
-  public boolean equals(TGetTableTypesResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_status = true && (isSetStatus());
-    list.add(present_status);
-    if (present_status)
-      list.add(status);
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    list.add(present_operationHandle);
-    if (present_operationHandle)
-      list.add(operationHandle);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetTableTypesResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(other.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, other.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(other.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, other.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetTableTypesResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetOperationHandle()) {
-      if (!first) sb.append(", ");
-      sb.append("operationHandle:");
-      if (this.operationHandle == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.operationHandle);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetTableTypesRespStandardSchemeFactory implements SchemeFactory {
-    public TGetTableTypesRespStandardScheme getScheme() {
-      return new TGetTableTypesRespStandardScheme();
-    }
-  }
-
-  private static class TGetTableTypesRespStandardScheme extends StandardScheme<TGetTableTypesResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetTableTypesResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetTableTypesResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.operationHandle != null) {
-        if (struct.isSetOperationHandle()) {
-          oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-          struct.operationHandle.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetTableTypesRespTupleSchemeFactory implements SchemeFactory {
-    public TGetTableTypesRespTupleScheme getScheme() {
-      return new TGetTableTypesRespTupleScheme();
-    }
-  }
-
-  private static class TGetTableTypesRespTupleScheme extends TupleScheme<TGetTableTypesResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetTableTypesResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetOperationHandle()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetOperationHandle()) {
-        struct.operationHandle.write(oprot);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetTableTypesResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.operationHandle = new TOperationHandle();
-        struct.operationHandle.read(iprot);
-        struct.setOperationHandleIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetTablesReq.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetTablesReq.java
deleted file mode 100644
index 1aa3f946727b6..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetTablesReq.java
+++ /dev/null
@@ -1,871 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetTablesReq implements org.apache.thrift.TBase<TGetTablesReq, TGetTablesReq._Fields>, java.io.Serializable, Cloneable, Comparable<TGetTablesReq> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetTablesReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField CATALOG_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("catalogName", org.apache.thrift.protocol.TType.STRING, (short)2);
-  private static final org.apache.thrift.protocol.TField SCHEMA_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("schemaName", org.apache.thrift.protocol.TType.STRING, (short)3);
-  private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)4);
-  private static final org.apache.thrift.protocol.TField TABLE_TYPES_FIELD_DESC = new org.apache.thrift.protocol.TField("tableTypes", org.apache.thrift.protocol.TType.LIST, (short)5);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetTablesReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetTablesReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-  private String catalogName; // optional
-  private String schemaName; // optional
-  private String tableName; // optional
-  private List<String> tableTypes; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle"),
-    CATALOG_NAME((short)2, "catalogName"),
-    SCHEMA_NAME((short)3, "schemaName"),
-    TABLE_NAME((short)4, "tableName"),
-    TABLE_TYPES((short)5, "tableTypes");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        case 2: // CATALOG_NAME
-          return CATALOG_NAME;
-        case 3: // SCHEMA_NAME
-          return SCHEMA_NAME;
-        case 4: // TABLE_NAME
-          return TABLE_NAME;
-        case 5: // TABLE_TYPES
-          return TABLE_TYPES;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final _Fields optionals[] = {_Fields.CATALOG_NAME,_Fields.SCHEMA_NAME,_Fields.TABLE_NAME,_Fields.TABLE_TYPES};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    tmpMap.put(_Fields.CATALOG_NAME, new org.apache.thrift.meta_data.FieldMetaData("catalogName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TPatternOrIdentifier")));
-    tmpMap.put(_Fields.SCHEMA_NAME, new org.apache.thrift.meta_data.FieldMetaData("schemaName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TPatternOrIdentifier")));
-    tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , "TPatternOrIdentifier")));
-    tmpMap.put(_Fields.TABLE_TYPES, new org.apache.thrift.meta_data.FieldMetaData("tableTypes", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetTablesReq.class, metaDataMap);
-  }
-
-  public TGetTablesReq() {
-  }
-
-  public TGetTablesReq(
-    TSessionHandle sessionHandle)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetTablesReq(TGetTablesReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-    if (other.isSetCatalogName()) {
-      this.catalogName = other.catalogName;
-    }
-    if (other.isSetSchemaName()) {
-      this.schemaName = other.schemaName;
-    }
-    if (other.isSetTableName()) {
-      this.tableName = other.tableName;
-    }
-    if (other.isSetTableTypes()) {
-      List<String> __this__tableTypes = new ArrayList<String>(other.tableTypes);
-      this.tableTypes = __this__tableTypes;
-    }
-  }
-
-  public TGetTablesReq deepCopy() {
-    return new TGetTablesReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-    this.catalogName = null;
-    this.schemaName = null;
-    this.tableName = null;
-    this.tableTypes = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public String getCatalogName() {
-    return this.catalogName;
-  }
-
-  public void setCatalogName(String catalogName) {
-    this.catalogName = catalogName;
-  }
-
-  public void unsetCatalogName() {
-    this.catalogName = null;
-  }
-
-  /** Returns true if field catalogName is set (has been assigned a value) and false otherwise */
-  public boolean isSetCatalogName() {
-    return this.catalogName != null;
-  }
-
-  public void setCatalogNameIsSet(boolean value) {
-    if (!value) {
-      this.catalogName = null;
-    }
-  }
-
-  public String getSchemaName() {
-    return this.schemaName;
-  }
-
-  public void setSchemaName(String schemaName) {
-    this.schemaName = schemaName;
-  }
-
-  public void unsetSchemaName() {
-    this.schemaName = null;
-  }
-
-  /** Returns true if field schemaName is set (has been assigned a value) and false otherwise */
-  public boolean isSetSchemaName() {
-    return this.schemaName != null;
-  }
-
-  public void setSchemaNameIsSet(boolean value) {
-    if (!value) {
-      this.schemaName = null;
-    }
-  }
-
-  public String getTableName() {
-    return this.tableName;
-  }
-
-  public void setTableName(String tableName) {
-    this.tableName = tableName;
-  }
-
-  public void unsetTableName() {
-    this.tableName = null;
-  }
-
-  /** Returns true if field tableName is set (has been assigned a value) and false otherwise */
-  public boolean isSetTableName() {
-    return this.tableName != null;
-  }
-
-  public void setTableNameIsSet(boolean value) {
-    if (!value) {
-      this.tableName = null;
-    }
-  }
-
-  public int getTableTypesSize() {
-    return (this.tableTypes == null) ? 0 : this.tableTypes.size();
-  }
-
-  public java.util.Iterator<String> getTableTypesIterator() {
-    return (this.tableTypes == null) ? null : this.tableTypes.iterator();
-  }
-
-  public void addToTableTypes(String elem) {
-    if (this.tableTypes == null) {
-      this.tableTypes = new ArrayList<String>();
-    }
-    this.tableTypes.add(elem);
-  }
-
-  public List<String> getTableTypes() {
-    return this.tableTypes;
-  }
-
-  public void setTableTypes(List<String> tableTypes) {
-    this.tableTypes = tableTypes;
-  }
-
-  public void unsetTableTypes() {
-    this.tableTypes = null;
-  }
-
-  /** Returns true if field tableTypes is set (has been assigned a value) and false otherwise */
-  public boolean isSetTableTypes() {
-    return this.tableTypes != null;
-  }
-
-  public void setTableTypesIsSet(boolean value) {
-    if (!value) {
-      this.tableTypes = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    case CATALOG_NAME:
-      if (value == null) {
-        unsetCatalogName();
-      } else {
-        setCatalogName((String)value);
-      }
-      break;
-
-    case SCHEMA_NAME:
-      if (value == null) {
-        unsetSchemaName();
-      } else {
-        setSchemaName((String)value);
-      }
-      break;
-
-    case TABLE_NAME:
-      if (value == null) {
-        unsetTableName();
-      } else {
-        setTableName((String)value);
-      }
-      break;
-
-    case TABLE_TYPES:
-      if (value == null) {
-        unsetTableTypes();
-      } else {
-        setTableTypes((List<String>)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    case CATALOG_NAME:
-      return getCatalogName();
-
-    case SCHEMA_NAME:
-      return getSchemaName();
-
-    case TABLE_NAME:
-      return getTableName();
-
-    case TABLE_TYPES:
-      return getTableTypes();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    case CATALOG_NAME:
-      return isSetCatalogName();
-    case SCHEMA_NAME:
-      return isSetSchemaName();
-    case TABLE_NAME:
-      return isSetTableName();
-    case TABLE_TYPES:
-      return isSetTableTypes();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetTablesReq)
-      return this.equals((TGetTablesReq)that);
-    return false;
-  }
-
-  public boolean equals(TGetTablesReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    boolean this_present_catalogName = true && this.isSetCatalogName();
-    boolean that_present_catalogName = true && that.isSetCatalogName();
-    if (this_present_catalogName || that_present_catalogName) {
-      if (!(this_present_catalogName && that_present_catalogName))
-        return false;
-      if (!this.catalogName.equals(that.catalogName))
-        return false;
-    }
-
-    boolean this_present_schemaName = true && this.isSetSchemaName();
-    boolean that_present_schemaName = true && that.isSetSchemaName();
-    if (this_present_schemaName || that_present_schemaName) {
-      if (!(this_present_schemaName && that_present_schemaName))
-        return false;
-      if (!this.schemaName.equals(that.schemaName))
-        return false;
-    }
-
-    boolean this_present_tableName = true && this.isSetTableName();
-    boolean that_present_tableName = true && that.isSetTableName();
-    if (this_present_tableName || that_present_tableName) {
-      if (!(this_present_tableName && that_present_tableName))
-        return false;
-      if (!this.tableName.equals(that.tableName))
-        return false;
-    }
-
-    boolean this_present_tableTypes = true && this.isSetTableTypes();
-    boolean that_present_tableTypes = true && that.isSetTableTypes();
-    if (this_present_tableTypes || that_present_tableTypes) {
-      if (!(this_present_tableTypes && that_present_tableTypes))
-        return false;
-      if (!this.tableTypes.equals(that.tableTypes))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    list.add(present_sessionHandle);
-    if (present_sessionHandle)
-      list.add(sessionHandle);
-
-    boolean present_catalogName = true && (isSetCatalogName());
-    list.add(present_catalogName);
-    if (present_catalogName)
-      list.add(catalogName);
-
-    boolean present_schemaName = true && (isSetSchemaName());
-    list.add(present_schemaName);
-    if (present_schemaName)
-      list.add(schemaName);
-
-    boolean present_tableName = true && (isSetTableName());
-    list.add(present_tableName);
-    if (present_tableName)
-      list.add(tableName);
-
-    boolean present_tableTypes = true && (isSetTableTypes());
-    list.add(present_tableTypes);
-    if (present_tableTypes)
-      list.add(tableTypes);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetTablesReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(other.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, other.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetCatalogName()).compareTo(other.isSetCatalogName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetCatalogName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.catalogName, other.catalogName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetSchemaName()).compareTo(other.isSetSchemaName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSchemaName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.schemaName, other.schemaName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetTableName()).compareTo(other.isSetTableName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetTableName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, other.tableName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetTableTypes()).compareTo(other.isSetTableTypes());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetTableTypes()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableTypes, other.tableTypes);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetTablesReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    if (isSetCatalogName()) {
-      if (!first) sb.append(", ");
-      sb.append("catalogName:");
-      if (this.catalogName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.catalogName);
-      }
-      first = false;
-    }
-    if (isSetSchemaName()) {
-      if (!first) sb.append(", ");
-      sb.append("schemaName:");
-      if (this.schemaName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.schemaName);
-      }
-      first = false;
-    }
-    if (isSetTableName()) {
-      if (!first) sb.append(", ");
-      sb.append("tableName:");
-      if (this.tableName == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.tableName);
-      }
-      first = false;
-    }
-    if (isSetTableTypes()) {
-      if (!first) sb.append(", ");
-      sb.append("tableTypes:");
-      if (this.tableTypes == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.tableTypes);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetTablesReqStandardSchemeFactory implements SchemeFactory {
-    public TGetTablesReqStandardScheme getScheme() {
-      return new TGetTablesReqStandardScheme();
-    }
-  }
-
-  private static class TGetTablesReqStandardScheme extends StandardScheme<TGetTablesReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetTablesReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // CATALOG_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.catalogName = iprot.readString();
-              struct.setCatalogNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // SCHEMA_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.schemaName = iprot.readString();
-              struct.setSchemaNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // TABLE_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.tableName = iprot.readString();
-              struct.setTableNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 5: // TABLE_TYPES
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list172 = iprot.readListBegin();
-                struct.tableTypes = new ArrayList<String>(_list172.size);
-                String _elem173;
-                for (int _i174 = 0; _i174 < _list172.size; ++_i174)
-                {
-                  _elem173 = iprot.readString();
-                  struct.tableTypes.add(_elem173);
-                }
-                iprot.readListEnd();
-              }
-              struct.setTableTypesIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetTablesReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.catalogName != null) {
-        if (struct.isSetCatalogName()) {
-          oprot.writeFieldBegin(CATALOG_NAME_FIELD_DESC);
-          oprot.writeString(struct.catalogName);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.schemaName != null) {
-        if (struct.isSetSchemaName()) {
-          oprot.writeFieldBegin(SCHEMA_NAME_FIELD_DESC);
-          oprot.writeString(struct.schemaName);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.tableName != null) {
-        if (struct.isSetTableName()) {
-          oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC);
-          oprot.writeString(struct.tableName);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.tableTypes != null) {
-        if (struct.isSetTableTypes()) {
-          oprot.writeFieldBegin(TABLE_TYPES_FIELD_DESC);
-          {
-            oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.tableTypes.size()));
-            for (String _iter175 : struct.tableTypes)
-            {
-              oprot.writeString(_iter175);
-            }
-            oprot.writeListEnd();
-          }
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetTablesReqTupleSchemeFactory implements SchemeFactory {
-    public TGetTablesReqTupleScheme getScheme() {
-      return new TGetTablesReqTupleScheme();
-    }
-  }
-
-  private static class TGetTablesReqTupleScheme extends TupleScheme<TGetTablesReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetTablesReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetCatalogName()) {
-        optionals.set(0);
-      }
-      if (struct.isSetSchemaName()) {
-        optionals.set(1);
-      }
-      if (struct.isSetTableName()) {
-        optionals.set(2);
-      }
-      if (struct.isSetTableTypes()) {
-        optionals.set(3);
-      }
-      oprot.writeBitSet(optionals, 4);
-      if (struct.isSetCatalogName()) {
-        oprot.writeString(struct.catalogName);
-      }
-      if (struct.isSetSchemaName()) {
-        oprot.writeString(struct.schemaName);
-      }
-      if (struct.isSetTableName()) {
-        oprot.writeString(struct.tableName);
-      }
-      if (struct.isSetTableTypes()) {
-        {
-          oprot.writeI32(struct.tableTypes.size());
-          for (String _iter176 : struct.tableTypes)
-          {
-            oprot.writeString(_iter176);
-          }
-        }
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetTablesReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-      BitSet incoming = iprot.readBitSet(4);
-      if (incoming.get(0)) {
-        struct.catalogName = iprot.readString();
-        struct.setCatalogNameIsSet(true);
-      }
-      if (incoming.get(1)) {
-        struct.schemaName = iprot.readString();
-        struct.setSchemaNameIsSet(true);
-      }
-      if (incoming.get(2)) {
-        struct.tableName = iprot.readString();
-        struct.setTableNameIsSet(true);
-      }
-      if (incoming.get(3)) {
-        {
-          org.apache.thrift.protocol.TList _list177 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-          struct.tableTypes = new ArrayList<String>(_list177.size);
-          String _elem178;
-          for (int _i179 = 0; _i179 < _list177.size; ++_i179)
-          {
-            _elem178 = iprot.readString();
-            struct.tableTypes.add(_elem178);
-          }
-        }
-        struct.setTableTypesIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetTablesResp.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetTablesResp.java
deleted file mode 100644
index 0b7c3825d35a5..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetTablesResp.java
+++ /dev/null
@@ -1,509 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetTablesResp implements org.apache.thrift.TBase<TGetTablesResp, TGetTablesResp._Fields>, java.io.Serializable, Cloneable, Comparable<TGetTablesResp> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetTablesResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetTablesRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetTablesRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TOperationHandle operationHandle; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    OPERATION_HANDLE((short)2, "operationHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final _Fields optionals[] = {_Fields.OPERATION_HANDLE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetTablesResp.class, metaDataMap);
-  }
-
-  public TGetTablesResp() {
-  }
-
-  public TGetTablesResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetTablesResp(TGetTablesResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-  }
-
-  public TGetTablesResp deepCopy() {
-    return new TGetTablesResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.operationHandle = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetTablesResp)
-      return this.equals((TGetTablesResp)that);
-    return false;
-  }
-
-  public boolean equals(TGetTablesResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_status = true && (isSetStatus());
-    list.add(present_status);
-    if (present_status)
-      list.add(status);
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    list.add(present_operationHandle);
-    if (present_operationHandle)
-      list.add(operationHandle);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetTablesResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(other.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, other.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(other.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, other.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetTablesResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetOperationHandle()) {
-      if (!first) sb.append(", ");
-      sb.append("operationHandle:");
-      if (this.operationHandle == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.operationHandle);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetTablesRespStandardSchemeFactory implements SchemeFactory {
-    public TGetTablesRespStandardScheme getScheme() {
-      return new TGetTablesRespStandardScheme();
-    }
-  }
-
-  private static class TGetTablesRespStandardScheme extends StandardScheme<TGetTablesResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetTablesResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetTablesResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.operationHandle != null) {
-        if (struct.isSetOperationHandle()) {
-          oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-          struct.operationHandle.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetTablesRespTupleSchemeFactory implements SchemeFactory {
-    public TGetTablesRespTupleScheme getScheme() {
-      return new TGetTablesRespTupleScheme();
-    }
-  }
-
-  private static class TGetTablesRespTupleScheme extends TupleScheme<TGetTablesResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetTablesResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetOperationHandle()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetOperationHandle()) {
-        struct.operationHandle.write(oprot);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetTablesResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.operationHandle = new TOperationHandle();
-        struct.operationHandle.read(iprot);
-        struct.setOperationHandleIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetTypeInfoReq.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetTypeInfoReq.java
deleted file mode 100644
index 2e0ec60e4bc3d..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetTypeInfoReq.java
+++ /dev/null
@@ -1,394 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetTypeInfoReq implements org.apache.thrift.TBase<TGetTypeInfoReq, TGetTypeInfoReq._Fields>, java.io.Serializable, Cloneable, Comparable<TGetTypeInfoReq> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetTypeInfoReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetTypeInfoReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetTypeInfoReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetTypeInfoReq.class, metaDataMap);
-  }
-
-  public TGetTypeInfoReq() {
-  }
-
-  public TGetTypeInfoReq(
-    TSessionHandle sessionHandle)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetTypeInfoReq(TGetTypeInfoReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-  }
-
-  public TGetTypeInfoReq deepCopy() {
-    return new TGetTypeInfoReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetTypeInfoReq)
-      return this.equals((TGetTypeInfoReq)that);
-    return false;
-  }
-
-  public boolean equals(TGetTypeInfoReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    list.add(present_sessionHandle);
-    if (present_sessionHandle)
-      list.add(sessionHandle);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetTypeInfoReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(other.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, other.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetTypeInfoReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetTypeInfoReqStandardSchemeFactory implements SchemeFactory {
-    public TGetTypeInfoReqStandardScheme getScheme() {
-      return new TGetTypeInfoReqStandardScheme();
-    }
-  }
-
-  private static class TGetTypeInfoReqStandardScheme extends StandardScheme<TGetTypeInfoReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetTypeInfoReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetTypeInfoReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetTypeInfoReqTupleSchemeFactory implements SchemeFactory {
-    public TGetTypeInfoReqTupleScheme getScheme() {
-      return new TGetTypeInfoReqTupleScheme();
-    }
-  }
-
-  private static class TGetTypeInfoReqTupleScheme extends TupleScheme<TGetTypeInfoReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetTypeInfoReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetTypeInfoReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetTypeInfoResp.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetTypeInfoResp.java
deleted file mode 100644
index cc2910ef29feb..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetTypeInfoResp.java
+++ /dev/null
@@ -1,509 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TGetTypeInfoResp implements org.apache.thrift.TBase<TGetTypeInfoResp, TGetTypeInfoResp._Fields>, java.io.Serializable, Cloneable, Comparable<TGetTypeInfoResp> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGetTypeInfoResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField OPERATION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationHandle", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TGetTypeInfoRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TGetTypeInfoRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TOperationHandle operationHandle; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    OPERATION_HANDLE((short)2, "operationHandle");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // OPERATION_HANDLE
-          return OPERATION_HANDLE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final _Fields optionals[] = {_Fields.OPERATION_HANDLE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.OPERATION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("operationHandle", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TOperationHandle.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetTypeInfoResp.class, metaDataMap);
-  }
-
-  public TGetTypeInfoResp() {
-  }
-
-  public TGetTypeInfoResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TGetTypeInfoResp(TGetTypeInfoResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetOperationHandle()) {
-      this.operationHandle = new TOperationHandle(other.operationHandle);
-    }
-  }
-
-  public TGetTypeInfoResp deepCopy() {
-    return new TGetTypeInfoResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.operationHandle = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public TOperationHandle getOperationHandle() {
-    return this.operationHandle;
-  }
-
-  public void setOperationHandle(TOperationHandle operationHandle) {
-    this.operationHandle = operationHandle;
-  }
-
-  public void unsetOperationHandle() {
-    this.operationHandle = null;
-  }
-
-  /** Returns true if field operationHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationHandle() {
-    return this.operationHandle != null;
-  }
-
-  public void setOperationHandleIsSet(boolean value) {
-    if (!value) {
-      this.operationHandle = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case OPERATION_HANDLE:
-      if (value == null) {
-        unsetOperationHandle();
-      } else {
-        setOperationHandle((TOperationHandle)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case OPERATION_HANDLE:
-      return getOperationHandle();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case OPERATION_HANDLE:
-      return isSetOperationHandle();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TGetTypeInfoResp)
-      return this.equals((TGetTypeInfoResp)that);
-    return false;
-  }
-
-  public boolean equals(TGetTypeInfoResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_operationHandle = true && this.isSetOperationHandle();
-    boolean that_present_operationHandle = true && that.isSetOperationHandle();
-    if (this_present_operationHandle || that_present_operationHandle) {
-      if (!(this_present_operationHandle && that_present_operationHandle))
-        return false;
-      if (!this.operationHandle.equals(that.operationHandle))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_status = true && (isSetStatus());
-    list.add(present_status);
-    if (present_status)
-      list.add(status);
-
-    boolean present_operationHandle = true && (isSetOperationHandle());
-    list.add(present_operationHandle);
-    if (present_operationHandle)
-      list.add(operationHandle);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TGetTypeInfoResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(other.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, other.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOperationHandle()).compareTo(other.isSetOperationHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationHandle, other.operationHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TGetTypeInfoResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (isSetOperationHandle()) {
-      if (!first) sb.append(", ");
-      sb.append("operationHandle:");
-      if (this.operationHandle == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.operationHandle);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-    if (operationHandle != null) {
-      operationHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TGetTypeInfoRespStandardSchemeFactory implements SchemeFactory {
-    public TGetTypeInfoRespStandardScheme getScheme() {
-      return new TGetTypeInfoRespStandardScheme();
-    }
-  }
-
-  private static class TGetTypeInfoRespStandardScheme extends StandardScheme<TGetTypeInfoResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TGetTypeInfoResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // OPERATION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationHandle = new TOperationHandle();
-              struct.operationHandle.read(iprot);
-              struct.setOperationHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TGetTypeInfoResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.operationHandle != null) {
-        if (struct.isSetOperationHandle()) {
-          oprot.writeFieldBegin(OPERATION_HANDLE_FIELD_DESC);
-          struct.operationHandle.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TGetTypeInfoRespTupleSchemeFactory implements SchemeFactory {
-    public TGetTypeInfoRespTupleScheme getScheme() {
-      return new TGetTypeInfoRespTupleScheme();
-    }
-  }
-
-  private static class TGetTypeInfoRespTupleScheme extends TupleScheme<TGetTypeInfoResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TGetTypeInfoResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      BitSet optionals = new BitSet();
-      if (struct.isSetOperationHandle()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetOperationHandle()) {
-        struct.operationHandle.write(oprot);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TGetTypeInfoResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.operationHandle = new TOperationHandle();
-        struct.operationHandle.read(iprot);
-        struct.setOperationHandleIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/THandleIdentifier.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/THandleIdentifier.java
deleted file mode 100644
index a3879d830000b..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/THandleIdentifier.java
+++ /dev/null
@@ -1,508 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class THandleIdentifier implements org.apache.thrift.TBase<THandleIdentifier, THandleIdentifier._Fields>, java.io.Serializable, Cloneable, Comparable<THandleIdentifier> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("THandleIdentifier");
-
-  private static final org.apache.thrift.protocol.TField GUID_FIELD_DESC = new org.apache.thrift.protocol.TField("guid", org.apache.thrift.protocol.TType.STRING, (short)1);
-  private static final org.apache.thrift.protocol.TField SECRET_FIELD_DESC = new org.apache.thrift.protocol.TField("secret", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new THandleIdentifierStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new THandleIdentifierTupleSchemeFactory());
-  }
-
-  private ByteBuffer guid; // required
-  private ByteBuffer secret; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    GUID((short)1, "guid"),
-    SECRET((short)2, "secret");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // GUID
-          return GUID;
-        case 2: // SECRET
-          return SECRET;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.GUID, new org.apache.thrift.meta_data.FieldMetaData("guid", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
-    tmpMap.put(_Fields.SECRET, new org.apache.thrift.meta_data.FieldMetaData("secret", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(THandleIdentifier.class, metaDataMap);
-  }
-
-  public THandleIdentifier() {
-  }
-
-  public THandleIdentifier(
-    ByteBuffer guid,
-    ByteBuffer secret)
-  {
-    this();
-    this.guid = org.apache.thrift.TBaseHelper.copyBinary(guid);
-    this.secret = org.apache.thrift.TBaseHelper.copyBinary(secret);
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public THandleIdentifier(THandleIdentifier other) {
-    if (other.isSetGuid()) {
-      this.guid = org.apache.thrift.TBaseHelper.copyBinary(other.guid);
-    }
-    if (other.isSetSecret()) {
-      this.secret = org.apache.thrift.TBaseHelper.copyBinary(other.secret);
-    }
-  }
-
-  public THandleIdentifier deepCopy() {
-    return new THandleIdentifier(this);
-  }
-
-  @Override
-  public void clear() {
-    this.guid = null;
-    this.secret = null;
-  }
-
-  public byte[] getGuid() {
-    setGuid(org.apache.thrift.TBaseHelper.rightSize(guid));
-    return guid == null ? null : guid.array();
-  }
-
-  public ByteBuffer bufferForGuid() {
-    return org.apache.thrift.TBaseHelper.copyBinary(guid);
-  }
-
-  public void setGuid(byte[] guid) {
-    this.guid = guid == null ? (ByteBuffer)null : ByteBuffer.wrap(Arrays.copyOf(guid, guid.length));
-  }
-
-  public void setGuid(ByteBuffer guid) {
-    this.guid = org.apache.thrift.TBaseHelper.copyBinary(guid);
-  }
-
-  public void unsetGuid() {
-    this.guid = null;
-  }
-
-  /** Returns true if field guid is set (has been assigned a value) and false otherwise */
-  public boolean isSetGuid() {
-    return this.guid != null;
-  }
-
-  public void setGuidIsSet(boolean value) {
-    if (!value) {
-      this.guid = null;
-    }
-  }
-
-  public byte[] getSecret() {
-    setSecret(org.apache.thrift.TBaseHelper.rightSize(secret));
-    return secret == null ? null : secret.array();
-  }
-
-  public ByteBuffer bufferForSecret() {
-    return org.apache.thrift.TBaseHelper.copyBinary(secret);
-  }
-
-  public void setSecret(byte[] secret) {
-    this.secret = secret == null ? (ByteBuffer)null : ByteBuffer.wrap(Arrays.copyOf(secret, secret.length));
-  }
-
-  public void setSecret(ByteBuffer secret) {
-    this.secret = org.apache.thrift.TBaseHelper.copyBinary(secret);
-  }
-
-  public void unsetSecret() {
-    this.secret = null;
-  }
-
-  /** Returns true if field secret is set (has been assigned a value) and false otherwise */
-  public boolean isSetSecret() {
-    return this.secret != null;
-  }
-
-  public void setSecretIsSet(boolean value) {
-    if (!value) {
-      this.secret = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case GUID:
-      if (value == null) {
-        unsetGuid();
-      } else {
-        setGuid((ByteBuffer)value);
-      }
-      break;
-
-    case SECRET:
-      if (value == null) {
-        unsetSecret();
-      } else {
-        setSecret((ByteBuffer)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case GUID:
-      return getGuid();
-
-    case SECRET:
-      return getSecret();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case GUID:
-      return isSetGuid();
-    case SECRET:
-      return isSetSecret();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof THandleIdentifier)
-      return this.equals((THandleIdentifier)that);
-    return false;
-  }
-
-  public boolean equals(THandleIdentifier that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_guid = true && this.isSetGuid();
-    boolean that_present_guid = true && that.isSetGuid();
-    if (this_present_guid || that_present_guid) {
-      if (!(this_present_guid && that_present_guid))
-        return false;
-      if (!this.guid.equals(that.guid))
-        return false;
-    }
-
-    boolean this_present_secret = true && this.isSetSecret();
-    boolean that_present_secret = true && that.isSetSecret();
-    if (this_present_secret || that_present_secret) {
-      if (!(this_present_secret && that_present_secret))
-        return false;
-      if (!this.secret.equals(that.secret))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_guid = true && (isSetGuid());
-    list.add(present_guid);
-    if (present_guid)
-      list.add(guid);
-
-    boolean present_secret = true && (isSetSecret());
-    list.add(present_secret);
-    if (present_secret)
-      list.add(secret);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(THandleIdentifier other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetGuid()).compareTo(other.isSetGuid());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetGuid()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.guid, other.guid);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetSecret()).compareTo(other.isSetSecret());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSecret()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.secret, other.secret);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("THandleIdentifier(");
-    boolean first = true;
-
-    sb.append("guid:");
-    if (this.guid == null) {
-      sb.append("null");
-    } else {
-      org.apache.thrift.TBaseHelper.toString(this.guid, sb);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("secret:");
-    if (this.secret == null) {
-      sb.append("null");
-    } else {
-      org.apache.thrift.TBaseHelper.toString(this.secret, sb);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetGuid()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'guid' is unset! Struct:" + toString());
-    }
-
-    if (!isSetSecret()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'secret' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class THandleIdentifierStandardSchemeFactory implements SchemeFactory {
-    public THandleIdentifierStandardScheme getScheme() {
-      return new THandleIdentifierStandardScheme();
-    }
-  }
-
-  private static class THandleIdentifierStandardScheme extends StandardScheme<THandleIdentifier> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, THandleIdentifier struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // GUID
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.guid = iprot.readBinary();
-              struct.setGuidIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // SECRET
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.secret = iprot.readBinary();
-              struct.setSecretIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, THandleIdentifier struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.guid != null) {
-        oprot.writeFieldBegin(GUID_FIELD_DESC);
-        oprot.writeBinary(struct.guid);
-        oprot.writeFieldEnd();
-      }
-      if (struct.secret != null) {
-        oprot.writeFieldBegin(SECRET_FIELD_DESC);
-        oprot.writeBinary(struct.secret);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class THandleIdentifierTupleSchemeFactory implements SchemeFactory {
-    public THandleIdentifierTupleScheme getScheme() {
-      return new THandleIdentifierTupleScheme();
-    }
-  }
-
-  private static class THandleIdentifierTupleScheme extends TupleScheme<THandleIdentifier> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, THandleIdentifier struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      oprot.writeBinary(struct.guid);
-      oprot.writeBinary(struct.secret);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, THandleIdentifier struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.guid = iprot.readBinary();
-      struct.setGuidIsSet(true);
-      struct.secret = iprot.readBinary();
-      struct.setSecretIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TI16Column.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TI16Column.java
deleted file mode 100644
index 3c44b602b4ff7..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TI16Column.java
+++ /dev/null
@@ -1,548 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TI16Column implements org.apache.thrift.TBase<TI16Column, TI16Column._Fields>, java.io.Serializable, Cloneable, Comparable<TI16Column> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TI16Column");
-
-  private static final org.apache.thrift.protocol.TField VALUES_FIELD_DESC = new org.apache.thrift.protocol.TField("values", org.apache.thrift.protocol.TType.LIST, (short)1);
-  private static final org.apache.thrift.protocol.TField NULLS_FIELD_DESC = new org.apache.thrift.protocol.TField("nulls", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TI16ColumnStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TI16ColumnTupleSchemeFactory());
-  }
-
-  private List<Short> values; // required
-  private ByteBuffer nulls; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUES((short)1, "values"),
-    NULLS((short)2, "nulls");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUES
-          return VALUES;
-        case 2: // NULLS
-          return NULLS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUES, new org.apache.thrift.meta_data.FieldMetaData("values", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I16))));
-    tmpMap.put(_Fields.NULLS, new org.apache.thrift.meta_data.FieldMetaData("nulls", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TI16Column.class, metaDataMap);
-  }
-
-  public TI16Column() {
-  }
-
-  public TI16Column(
-    List<Short> values,
-    ByteBuffer nulls)
-  {
-    this();
-    this.values = values;
-    this.nulls = org.apache.thrift.TBaseHelper.copyBinary(nulls);
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TI16Column(TI16Column other) {
-    if (other.isSetValues()) {
-      List<Short> __this__values = new ArrayList<Short>(other.values);
-      this.values = __this__values;
-    }
-    if (other.isSetNulls()) {
-      this.nulls = org.apache.thrift.TBaseHelper.copyBinary(other.nulls);
-    }
-  }
-
-  public TI16Column deepCopy() {
-    return new TI16Column(this);
-  }
-
-  @Override
-  public void clear() {
-    this.values = null;
-    this.nulls = null;
-  }
-
-  public int getValuesSize() {
-    return (this.values == null) ? 0 : this.values.size();
-  }
-
-  public java.util.Iterator<Short> getValuesIterator() {
-    return (this.values == null) ? null : this.values.iterator();
-  }
-
-  public void addToValues(short elem) {
-    if (this.values == null) {
-      this.values = new ArrayList<Short>();
-    }
-    this.values.add(elem);
-  }
-
-  public List<Short> getValues() {
-    return this.values;
-  }
-
-  public void setValues(List<Short> values) {
-    this.values = values;
-  }
-
-  public void unsetValues() {
-    this.values = null;
-  }
-
-  /** Returns true if field values is set (has been assigned a value) and false otherwise */
-  public boolean isSetValues() {
-    return this.values != null;
-  }
-
-  public void setValuesIsSet(boolean value) {
-    if (!value) {
-      this.values = null;
-    }
-  }
-
-  public byte[] getNulls() {
-    setNulls(org.apache.thrift.TBaseHelper.rightSize(nulls));
-    return nulls == null ? null : nulls.array();
-  }
-
-  public ByteBuffer bufferForNulls() {
-    return org.apache.thrift.TBaseHelper.copyBinary(nulls);
-  }
-
-  public void setNulls(byte[] nulls) {
-    this.nulls = nulls == null ? (ByteBuffer)null : ByteBuffer.wrap(Arrays.copyOf(nulls, nulls.length));
-  }
-
-  public void setNulls(ByteBuffer nulls) {
-    this.nulls = org.apache.thrift.TBaseHelper.copyBinary(nulls);
-  }
-
-  public void unsetNulls() {
-    this.nulls = null;
-  }
-
-  /** Returns true if field nulls is set (has been assigned a value) and false otherwise */
-  public boolean isSetNulls() {
-    return this.nulls != null;
-  }
-
-  public void setNullsIsSet(boolean value) {
-    if (!value) {
-      this.nulls = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUES:
-      if (value == null) {
-        unsetValues();
-      } else {
-        setValues((List<Short>)value);
-      }
-      break;
-
-    case NULLS:
-      if (value == null) {
-        unsetNulls();
-      } else {
-        setNulls((ByteBuffer)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUES:
-      return getValues();
-
-    case NULLS:
-      return getNulls();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUES:
-      return isSetValues();
-    case NULLS:
-      return isSetNulls();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TI16Column)
-      return this.equals((TI16Column)that);
-    return false;
-  }
-
-  public boolean equals(TI16Column that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_values = true && this.isSetValues();
-    boolean that_present_values = true && that.isSetValues();
-    if (this_present_values || that_present_values) {
-      if (!(this_present_values && that_present_values))
-        return false;
-      if (!this.values.equals(that.values))
-        return false;
-    }
-
-    boolean this_present_nulls = true && this.isSetNulls();
-    boolean that_present_nulls = true && that.isSetNulls();
-    if (this_present_nulls || that_present_nulls) {
-      if (!(this_present_nulls && that_present_nulls))
-        return false;
-      if (!this.nulls.equals(that.nulls))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_values = true && (isSetValues());
-    list.add(present_values);
-    if (present_values)
-      list.add(values);
-
-    boolean present_nulls = true && (isSetNulls());
-    list.add(present_nulls);
-    if (present_nulls)
-      list.add(nulls);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TI16Column other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetValues()).compareTo(other.isSetValues());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValues()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.values, other.values);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetNulls()).compareTo(other.isSetNulls());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetNulls()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.nulls, other.nulls);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TI16Column(");
-    boolean first = true;
-
-    sb.append("values:");
-    if (this.values == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.values);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("nulls:");
-    if (this.nulls == null) {
-      sb.append("null");
-    } else {
-      org.apache.thrift.TBaseHelper.toString(this.nulls, sb);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetValues()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'values' is unset! Struct:" + toString());
-    }
-
-    if (!isSetNulls()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'nulls' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TI16ColumnStandardSchemeFactory implements SchemeFactory {
-    public TI16ColumnStandardScheme getScheme() {
-      return new TI16ColumnStandardScheme();
-    }
-  }
-
-  private static class TI16ColumnStandardScheme extends StandardScheme<TI16Column> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TI16Column struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUES
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list70 = iprot.readListBegin();
-                struct.values = new ArrayList<Short>(_list70.size);
-                short _elem71;
-                for (int _i72 = 0; _i72 < _list70.size; ++_i72)
-                {
-                  _elem71 = iprot.readI16();
-                  struct.values.add(_elem71);
-                }
-                iprot.readListEnd();
-              }
-              struct.setValuesIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // NULLS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.nulls = iprot.readBinary();
-              struct.setNullsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TI16Column struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.values != null) {
-        oprot.writeFieldBegin(VALUES_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I16, struct.values.size()));
-          for (short _iter73 : struct.values)
-          {
-            oprot.writeI16(_iter73);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      if (struct.nulls != null) {
-        oprot.writeFieldBegin(NULLS_FIELD_DESC);
-        oprot.writeBinary(struct.nulls);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TI16ColumnTupleSchemeFactory implements SchemeFactory {
-    public TI16ColumnTupleScheme getScheme() {
-      return new TI16ColumnTupleScheme();
-    }
-  }
-
-  private static class TI16ColumnTupleScheme extends TupleScheme<TI16Column> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TI16Column struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.values.size());
-        for (short _iter74 : struct.values)
-        {
-          oprot.writeI16(_iter74);
-        }
-      }
-      oprot.writeBinary(struct.nulls);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TI16Column struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TList _list75 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I16, iprot.readI32());
-        struct.values = new ArrayList<Short>(_list75.size);
-        short _elem76;
-        for (int _i77 = 0; _i77 < _list75.size; ++_i77)
-        {
-          _elem76 = iprot.readI16();
-          struct.values.add(_elem76);
-        }
-      }
-      struct.setValuesIsSet(true);
-      struct.nulls = iprot.readBinary();
-      struct.setNullsIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TI16Value.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TI16Value.java
deleted file mode 100644
index 29fb4cb85201d..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TI16Value.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TI16Value implements org.apache.thrift.TBase<TI16Value, TI16Value._Fields>, java.io.Serializable, Cloneable, Comparable<TI16Value> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TI16Value");
-
-  private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.I16, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TI16ValueStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TI16ValueTupleSchemeFactory());
-  }
-
-  private short value; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUE((short)1, "value");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUE
-          return VALUE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __VALUE_ISSET_ID = 0;
-  private byte __isset_bitfield = 0;
-  private static final _Fields optionals[] = {_Fields.VALUE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I16)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TI16Value.class, metaDataMap);
-  }
-
-  public TI16Value() {
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TI16Value(TI16Value other) {
-    __isset_bitfield = other.__isset_bitfield;
-    this.value = other.value;
-  }
-
-  public TI16Value deepCopy() {
-    return new TI16Value(this);
-  }
-
-  @Override
-  public void clear() {
-    setValueIsSet(false);
-    this.value = 0;
-  }
-
-  public short getValue() {
-    return this.value;
-  }
-
-  public void setValue(short value) {
-    this.value = value;
-    setValueIsSet(true);
-  }
-
-  public void unsetValue() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __VALUE_ISSET_ID);
-  }
-
-  /** Returns true if field value is set (has been assigned a value) and false otherwise */
-  public boolean isSetValue() {
-    return EncodingUtils.testBit(__isset_bitfield, __VALUE_ISSET_ID);
-  }
-
-  public void setValueIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __VALUE_ISSET_ID, value);
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUE:
-      if (value == null) {
-        unsetValue();
-      } else {
-        setValue((Short)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUE:
-      return getValue();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUE:
-      return isSetValue();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TI16Value)
-      return this.equals((TI16Value)that);
-    return false;
-  }
-
-  public boolean equals(TI16Value that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_value = true && this.isSetValue();
-    boolean that_present_value = true && that.isSetValue();
-    if (this_present_value || that_present_value) {
-      if (!(this_present_value && that_present_value))
-        return false;
-      if (this.value != that.value)
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_value = true && (isSetValue());
-    list.add(present_value);
-    if (present_value)
-      list.add(value);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TI16Value other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetValue()).compareTo(other.isSetValue());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValue()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, other.value);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TI16Value(");
-    boolean first = true;
-
-    if (isSetValue()) {
-      sb.append("value:");
-      sb.append(this.value);
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TI16ValueStandardSchemeFactory implements SchemeFactory {
-    public TI16ValueStandardScheme getScheme() {
-      return new TI16ValueStandardScheme();
-    }
-  }
-
-  private static class TI16ValueStandardScheme extends StandardScheme<TI16Value> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TI16Value struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUE
-            if (schemeField.type == org.apache.thrift.protocol.TType.I16) {
-              struct.value = iprot.readI16();
-              struct.setValueIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TI16Value struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.isSetValue()) {
-        oprot.writeFieldBegin(VALUE_FIELD_DESC);
-        oprot.writeI16(struct.value);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TI16ValueTupleSchemeFactory implements SchemeFactory {
-    public TI16ValueTupleScheme getScheme() {
-      return new TI16ValueTupleScheme();
-    }
-  }
-
-  private static class TI16ValueTupleScheme extends TupleScheme<TI16Value> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TI16Value struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      BitSet optionals = new BitSet();
-      if (struct.isSetValue()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetValue()) {
-        oprot.writeI16(struct.value);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TI16Value struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.value = iprot.readI16();
-        struct.setValueIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TI32Column.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TI32Column.java
deleted file mode 100644
index 9834f1ce8f01b..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TI32Column.java
+++ /dev/null
@@ -1,548 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TI32Column implements org.apache.thrift.TBase<TI32Column, TI32Column._Fields>, java.io.Serializable, Cloneable, Comparable<TI32Column> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TI32Column");
-
-  private static final org.apache.thrift.protocol.TField VALUES_FIELD_DESC = new org.apache.thrift.protocol.TField("values", org.apache.thrift.protocol.TType.LIST, (short)1);
-  private static final org.apache.thrift.protocol.TField NULLS_FIELD_DESC = new org.apache.thrift.protocol.TField("nulls", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TI32ColumnStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TI32ColumnTupleSchemeFactory());
-  }
-
-  private List<Integer> values; // required
-  private ByteBuffer nulls; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUES((short)1, "values"),
-    NULLS((short)2, "nulls");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUES
-          return VALUES;
-        case 2: // NULLS
-          return NULLS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUES, new org.apache.thrift.meta_data.FieldMetaData("values", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))));
-    tmpMap.put(_Fields.NULLS, new org.apache.thrift.meta_data.FieldMetaData("nulls", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TI32Column.class, metaDataMap);
-  }
-
-  public TI32Column() {
-  }
-
-  public TI32Column(
-    List<Integer> values,
-    ByteBuffer nulls)
-  {
-    this();
-    this.values = values;
-    this.nulls = org.apache.thrift.TBaseHelper.copyBinary(nulls);
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TI32Column(TI32Column other) {
-    if (other.isSetValues()) {
-      List<Integer> __this__values = new ArrayList<Integer>(other.values);
-      this.values = __this__values;
-    }
-    if (other.isSetNulls()) {
-      this.nulls = org.apache.thrift.TBaseHelper.copyBinary(other.nulls);
-    }
-  }
-
-  public TI32Column deepCopy() {
-    return new TI32Column(this);
-  }
-
-  @Override
-  public void clear() {
-    this.values = null;
-    this.nulls = null;
-  }
-
-  public int getValuesSize() {
-    return (this.values == null) ? 0 : this.values.size();
-  }
-
-  public java.util.Iterator<Integer> getValuesIterator() {
-    return (this.values == null) ? null : this.values.iterator();
-  }
-
-  public void addToValues(int elem) {
-    if (this.values == null) {
-      this.values = new ArrayList<Integer>();
-    }
-    this.values.add(elem);
-  }
-
-  public List<Integer> getValues() {
-    return this.values;
-  }
-
-  public void setValues(List<Integer> values) {
-    this.values = values;
-  }
-
-  public void unsetValues() {
-    this.values = null;
-  }
-
-  /** Returns true if field values is set (has been assigned a value) and false otherwise */
-  public boolean isSetValues() {
-    return this.values != null;
-  }
-
-  public void setValuesIsSet(boolean value) {
-    if (!value) {
-      this.values = null;
-    }
-  }
-
-  public byte[] getNulls() {
-    setNulls(org.apache.thrift.TBaseHelper.rightSize(nulls));
-    return nulls == null ? null : nulls.array();
-  }
-
-  public ByteBuffer bufferForNulls() {
-    return org.apache.thrift.TBaseHelper.copyBinary(nulls);
-  }
-
-  public void setNulls(byte[] nulls) {
-    this.nulls = nulls == null ? (ByteBuffer)null : ByteBuffer.wrap(Arrays.copyOf(nulls, nulls.length));
-  }
-
-  public void setNulls(ByteBuffer nulls) {
-    this.nulls = org.apache.thrift.TBaseHelper.copyBinary(nulls);
-  }
-
-  public void unsetNulls() {
-    this.nulls = null;
-  }
-
-  /** Returns true if field nulls is set (has been assigned a value) and false otherwise */
-  public boolean isSetNulls() {
-    return this.nulls != null;
-  }
-
-  public void setNullsIsSet(boolean value) {
-    if (!value) {
-      this.nulls = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUES:
-      if (value == null) {
-        unsetValues();
-      } else {
-        setValues((List<Integer>)value);
-      }
-      break;
-
-    case NULLS:
-      if (value == null) {
-        unsetNulls();
-      } else {
-        setNulls((ByteBuffer)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUES:
-      return getValues();
-
-    case NULLS:
-      return getNulls();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUES:
-      return isSetValues();
-    case NULLS:
-      return isSetNulls();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TI32Column)
-      return this.equals((TI32Column)that);
-    return false;
-  }
-
-  public boolean equals(TI32Column that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_values = true && this.isSetValues();
-    boolean that_present_values = true && that.isSetValues();
-    if (this_present_values || that_present_values) {
-      if (!(this_present_values && that_present_values))
-        return false;
-      if (!this.values.equals(that.values))
-        return false;
-    }
-
-    boolean this_present_nulls = true && this.isSetNulls();
-    boolean that_present_nulls = true && that.isSetNulls();
-    if (this_present_nulls || that_present_nulls) {
-      if (!(this_present_nulls && that_present_nulls))
-        return false;
-      if (!this.nulls.equals(that.nulls))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_values = true && (isSetValues());
-    list.add(present_values);
-    if (present_values)
-      list.add(values);
-
-    boolean present_nulls = true && (isSetNulls());
-    list.add(present_nulls);
-    if (present_nulls)
-      list.add(nulls);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TI32Column other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetValues()).compareTo(other.isSetValues());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValues()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.values, other.values);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetNulls()).compareTo(other.isSetNulls());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetNulls()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.nulls, other.nulls);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TI32Column(");
-    boolean first = true;
-
-    sb.append("values:");
-    if (this.values == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.values);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("nulls:");
-    if (this.nulls == null) {
-      sb.append("null");
-    } else {
-      org.apache.thrift.TBaseHelper.toString(this.nulls, sb);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetValues()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'values' is unset! Struct:" + toString());
-    }
-
-    if (!isSetNulls()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'nulls' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TI32ColumnStandardSchemeFactory implements SchemeFactory {
-    public TI32ColumnStandardScheme getScheme() {
-      return new TI32ColumnStandardScheme();
-    }
-  }
-
-  private static class TI32ColumnStandardScheme extends StandardScheme<TI32Column> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TI32Column struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUES
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list78 = iprot.readListBegin();
-                struct.values = new ArrayList<Integer>(_list78.size);
-                int _elem79;
-                for (int _i80 = 0; _i80 < _list78.size; ++_i80)
-                {
-                  _elem79 = iprot.readI32();
-                  struct.values.add(_elem79);
-                }
-                iprot.readListEnd();
-              }
-              struct.setValuesIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // NULLS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.nulls = iprot.readBinary();
-              struct.setNullsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TI32Column struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.values != null) {
-        oprot.writeFieldBegin(VALUES_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I32, struct.values.size()));
-          for (int _iter81 : struct.values)
-          {
-            oprot.writeI32(_iter81);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      if (struct.nulls != null) {
-        oprot.writeFieldBegin(NULLS_FIELD_DESC);
-        oprot.writeBinary(struct.nulls);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TI32ColumnTupleSchemeFactory implements SchemeFactory {
-    public TI32ColumnTupleScheme getScheme() {
-      return new TI32ColumnTupleScheme();
-    }
-  }
-
-  private static class TI32ColumnTupleScheme extends TupleScheme<TI32Column> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TI32Column struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.values.size());
-        for (int _iter82 : struct.values)
-        {
-          oprot.writeI32(_iter82);
-        }
-      }
-      oprot.writeBinary(struct.nulls);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TI32Column struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TList _list83 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I32, iprot.readI32());
-        struct.values = new ArrayList<Integer>(_list83.size);
-        int _elem84;
-        for (int _i85 = 0; _i85 < _list83.size; ++_i85)
-        {
-          _elem84 = iprot.readI32();
-          struct.values.add(_elem84);
-        }
-      }
-      struct.setValuesIsSet(true);
-      struct.nulls = iprot.readBinary();
-      struct.setNullsIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TI32Value.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TI32Value.java
deleted file mode 100644
index 8a69632b2d76e..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TI32Value.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TI32Value implements org.apache.thrift.TBase<TI32Value, TI32Value._Fields>, java.io.Serializable, Cloneable, Comparable<TI32Value> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TI32Value");
-
-  private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.I32, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TI32ValueStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TI32ValueTupleSchemeFactory());
-  }
-
-  private int value; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUE((short)1, "value");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUE
-          return VALUE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __VALUE_ISSET_ID = 0;
-  private byte __isset_bitfield = 0;
-  private static final _Fields optionals[] = {_Fields.VALUE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TI32Value.class, metaDataMap);
-  }
-
-  public TI32Value() {
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TI32Value(TI32Value other) {
-    __isset_bitfield = other.__isset_bitfield;
-    this.value = other.value;
-  }
-
-  public TI32Value deepCopy() {
-    return new TI32Value(this);
-  }
-
-  @Override
-  public void clear() {
-    setValueIsSet(false);
-    this.value = 0;
-  }
-
-  public int getValue() {
-    return this.value;
-  }
-
-  public void setValue(int value) {
-    this.value = value;
-    setValueIsSet(true);
-  }
-
-  public void unsetValue() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __VALUE_ISSET_ID);
-  }
-
-  /** Returns true if field value is set (has been assigned a value) and false otherwise */
-  public boolean isSetValue() {
-    return EncodingUtils.testBit(__isset_bitfield, __VALUE_ISSET_ID);
-  }
-
-  public void setValueIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __VALUE_ISSET_ID, value);
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUE:
-      if (value == null) {
-        unsetValue();
-      } else {
-        setValue((Integer)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUE:
-      return getValue();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUE:
-      return isSetValue();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TI32Value)
-      return this.equals((TI32Value)that);
-    return false;
-  }
-
-  public boolean equals(TI32Value that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_value = true && this.isSetValue();
-    boolean that_present_value = true && that.isSetValue();
-    if (this_present_value || that_present_value) {
-      if (!(this_present_value && that_present_value))
-        return false;
-      if (this.value != that.value)
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_value = true && (isSetValue());
-    list.add(present_value);
-    if (present_value)
-      list.add(value);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TI32Value other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetValue()).compareTo(other.isSetValue());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValue()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, other.value);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TI32Value(");
-    boolean first = true;
-
-    if (isSetValue()) {
-      sb.append("value:");
-      sb.append(this.value);
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TI32ValueStandardSchemeFactory implements SchemeFactory {
-    public TI32ValueStandardScheme getScheme() {
-      return new TI32ValueStandardScheme();
-    }
-  }
-
-  private static class TI32ValueStandardScheme extends StandardScheme<TI32Value> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TI32Value struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUE
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.value = iprot.readI32();
-              struct.setValueIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TI32Value struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.isSetValue()) {
-        oprot.writeFieldBegin(VALUE_FIELD_DESC);
-        oprot.writeI32(struct.value);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TI32ValueTupleSchemeFactory implements SchemeFactory {
-    public TI32ValueTupleScheme getScheme() {
-      return new TI32ValueTupleScheme();
-    }
-  }
-
-  private static class TI32ValueTupleScheme extends TupleScheme<TI32Value> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TI32Value struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      BitSet optionals = new BitSet();
-      if (struct.isSetValue()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetValue()) {
-        oprot.writeI32(struct.value);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TI32Value struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.value = iprot.readI32();
-        struct.setValueIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TI64Column.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TI64Column.java
deleted file mode 100644
index cd5ef2d7a9ed9..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TI64Column.java
+++ /dev/null
@@ -1,548 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TI64Column implements org.apache.thrift.TBase<TI64Column, TI64Column._Fields>, java.io.Serializable, Cloneable, Comparable<TI64Column> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TI64Column");
-
-  private static final org.apache.thrift.protocol.TField VALUES_FIELD_DESC = new org.apache.thrift.protocol.TField("values", org.apache.thrift.protocol.TType.LIST, (short)1);
-  private static final org.apache.thrift.protocol.TField NULLS_FIELD_DESC = new org.apache.thrift.protocol.TField("nulls", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TI64ColumnStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TI64ColumnTupleSchemeFactory());
-  }
-
-  private List<Long> values; // required
-  private ByteBuffer nulls; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUES((short)1, "values"),
-    NULLS((short)2, "nulls");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUES
-          return VALUES;
-        case 2: // NULLS
-          return NULLS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUES, new org.apache.thrift.meta_data.FieldMetaData("values", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))));
-    tmpMap.put(_Fields.NULLS, new org.apache.thrift.meta_data.FieldMetaData("nulls", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TI64Column.class, metaDataMap);
-  }
-
-  public TI64Column() {
-  }
-
-  public TI64Column(
-    List<Long> values,
-    ByteBuffer nulls)
-  {
-    this();
-    this.values = values;
-    this.nulls = org.apache.thrift.TBaseHelper.copyBinary(nulls);
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TI64Column(TI64Column other) {
-    if (other.isSetValues()) {
-      List<Long> __this__values = new ArrayList<Long>(other.values);
-      this.values = __this__values;
-    }
-    if (other.isSetNulls()) {
-      this.nulls = org.apache.thrift.TBaseHelper.copyBinary(other.nulls);
-    }
-  }
-
-  public TI64Column deepCopy() {
-    return new TI64Column(this);
-  }
-
-  @Override
-  public void clear() {
-    this.values = null;
-    this.nulls = null;
-  }
-
-  public int getValuesSize() {
-    return (this.values == null) ? 0 : this.values.size();
-  }
-
-  public java.util.Iterator<Long> getValuesIterator() {
-    return (this.values == null) ? null : this.values.iterator();
-  }
-
-  public void addToValues(long elem) {
-    if (this.values == null) {
-      this.values = new ArrayList<Long>();
-    }
-    this.values.add(elem);
-  }
-
-  public List<Long> getValues() {
-    return this.values;
-  }
-
-  public void setValues(List<Long> values) {
-    this.values = values;
-  }
-
-  public void unsetValues() {
-    this.values = null;
-  }
-
-  /** Returns true if field values is set (has been assigned a value) and false otherwise */
-  public boolean isSetValues() {
-    return this.values != null;
-  }
-
-  public void setValuesIsSet(boolean value) {
-    if (!value) {
-      this.values = null;
-    }
-  }
-
-  public byte[] getNulls() {
-    setNulls(org.apache.thrift.TBaseHelper.rightSize(nulls));
-    return nulls == null ? null : nulls.array();
-  }
-
-  public ByteBuffer bufferForNulls() {
-    return org.apache.thrift.TBaseHelper.copyBinary(nulls);
-  }
-
-  public void setNulls(byte[] nulls) {
-    this.nulls = nulls == null ? (ByteBuffer)null : ByteBuffer.wrap(Arrays.copyOf(nulls, nulls.length));
-  }
-
-  public void setNulls(ByteBuffer nulls) {
-    this.nulls = org.apache.thrift.TBaseHelper.copyBinary(nulls);
-  }
-
-  public void unsetNulls() {
-    this.nulls = null;
-  }
-
-  /** Returns true if field nulls is set (has been assigned a value) and false otherwise */
-  public boolean isSetNulls() {
-    return this.nulls != null;
-  }
-
-  public void setNullsIsSet(boolean value) {
-    if (!value) {
-      this.nulls = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUES:
-      if (value == null) {
-        unsetValues();
-      } else {
-        setValues((List<Long>)value);
-      }
-      break;
-
-    case NULLS:
-      if (value == null) {
-        unsetNulls();
-      } else {
-        setNulls((ByteBuffer)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUES:
-      return getValues();
-
-    case NULLS:
-      return getNulls();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUES:
-      return isSetValues();
-    case NULLS:
-      return isSetNulls();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TI64Column)
-      return this.equals((TI64Column)that);
-    return false;
-  }
-
-  public boolean equals(TI64Column that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_values = true && this.isSetValues();
-    boolean that_present_values = true && that.isSetValues();
-    if (this_present_values || that_present_values) {
-      if (!(this_present_values && that_present_values))
-        return false;
-      if (!this.values.equals(that.values))
-        return false;
-    }
-
-    boolean this_present_nulls = true && this.isSetNulls();
-    boolean that_present_nulls = true && that.isSetNulls();
-    if (this_present_nulls || that_present_nulls) {
-      if (!(this_present_nulls && that_present_nulls))
-        return false;
-      if (!this.nulls.equals(that.nulls))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_values = true && (isSetValues());
-    list.add(present_values);
-    if (present_values)
-      list.add(values);
-
-    boolean present_nulls = true && (isSetNulls());
-    list.add(present_nulls);
-    if (present_nulls)
-      list.add(nulls);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TI64Column other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetValues()).compareTo(other.isSetValues());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValues()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.values, other.values);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetNulls()).compareTo(other.isSetNulls());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetNulls()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.nulls, other.nulls);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TI64Column(");
-    boolean first = true;
-
-    sb.append("values:");
-    if (this.values == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.values);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("nulls:");
-    if (this.nulls == null) {
-      sb.append("null");
-    } else {
-      org.apache.thrift.TBaseHelper.toString(this.nulls, sb);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetValues()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'values' is unset! Struct:" + toString());
-    }
-
-    if (!isSetNulls()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'nulls' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TI64ColumnStandardSchemeFactory implements SchemeFactory {
-    public TI64ColumnStandardScheme getScheme() {
-      return new TI64ColumnStandardScheme();
-    }
-  }
-
-  private static class TI64ColumnStandardScheme extends StandardScheme<TI64Column> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TI64Column struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUES
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list86 = iprot.readListBegin();
-                struct.values = new ArrayList<Long>(_list86.size);
-                long _elem87;
-                for (int _i88 = 0; _i88 < _list86.size; ++_i88)
-                {
-                  _elem87 = iprot.readI64();
-                  struct.values.add(_elem87);
-                }
-                iprot.readListEnd();
-              }
-              struct.setValuesIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // NULLS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.nulls = iprot.readBinary();
-              struct.setNullsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TI64Column struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.values != null) {
-        oprot.writeFieldBegin(VALUES_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I64, struct.values.size()));
-          for (long _iter89 : struct.values)
-          {
-            oprot.writeI64(_iter89);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      if (struct.nulls != null) {
-        oprot.writeFieldBegin(NULLS_FIELD_DESC);
-        oprot.writeBinary(struct.nulls);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TI64ColumnTupleSchemeFactory implements SchemeFactory {
-    public TI64ColumnTupleScheme getScheme() {
-      return new TI64ColumnTupleScheme();
-    }
-  }
-
-  private static class TI64ColumnTupleScheme extends TupleScheme<TI64Column> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TI64Column struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.values.size());
-        for (long _iter90 : struct.values)
-        {
-          oprot.writeI64(_iter90);
-        }
-      }
-      oprot.writeBinary(struct.nulls);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TI64Column struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TList _list91 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I64, iprot.readI32());
-        struct.values = new ArrayList<Long>(_list91.size);
-        long _elem92;
-        for (int _i93 = 0; _i93 < _list91.size; ++_i93)
-        {
-          _elem92 = iprot.readI64();
-          struct.values.add(_elem92);
-        }
-      }
-      struct.setValuesIsSet(true);
-      struct.nulls = iprot.readBinary();
-      struct.setNullsIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TI64Value.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TI64Value.java
deleted file mode 100644
index 393c0bd28610d..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TI64Value.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TI64Value implements org.apache.thrift.TBase<TI64Value, TI64Value._Fields>, java.io.Serializable, Cloneable, Comparable<TI64Value> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TI64Value");
-
-  private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.I64, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TI64ValueStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TI64ValueTupleSchemeFactory());
-  }
-
-  private long value; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUE((short)1, "value");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUE
-          return VALUE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __VALUE_ISSET_ID = 0;
-  private byte __isset_bitfield = 0;
-  private static final _Fields optionals[] = {_Fields.VALUE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TI64Value.class, metaDataMap);
-  }
-
-  public TI64Value() {
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TI64Value(TI64Value other) {
-    __isset_bitfield = other.__isset_bitfield;
-    this.value = other.value;
-  }
-
-  public TI64Value deepCopy() {
-    return new TI64Value(this);
-  }
-
-  @Override
-  public void clear() {
-    setValueIsSet(false);
-    this.value = 0;
-  }
-
-  public long getValue() {
-    return this.value;
-  }
-
-  public void setValue(long value) {
-    this.value = value;
-    setValueIsSet(true);
-  }
-
-  public void unsetValue() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __VALUE_ISSET_ID);
-  }
-
-  /** Returns true if field value is set (has been assigned a value) and false otherwise */
-  public boolean isSetValue() {
-    return EncodingUtils.testBit(__isset_bitfield, __VALUE_ISSET_ID);
-  }
-
-  public void setValueIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __VALUE_ISSET_ID, value);
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUE:
-      if (value == null) {
-        unsetValue();
-      } else {
-        setValue((Long)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUE:
-      return getValue();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUE:
-      return isSetValue();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TI64Value)
-      return this.equals((TI64Value)that);
-    return false;
-  }
-
-  public boolean equals(TI64Value that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_value = true && this.isSetValue();
-    boolean that_present_value = true && that.isSetValue();
-    if (this_present_value || that_present_value) {
-      if (!(this_present_value && that_present_value))
-        return false;
-      if (this.value != that.value)
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_value = true && (isSetValue());
-    list.add(present_value);
-    if (present_value)
-      list.add(value);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TI64Value other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetValue()).compareTo(other.isSetValue());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValue()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, other.value);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TI64Value(");
-    boolean first = true;
-
-    if (isSetValue()) {
-      sb.append("value:");
-      sb.append(this.value);
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TI64ValueStandardSchemeFactory implements SchemeFactory {
-    public TI64ValueStandardScheme getScheme() {
-      return new TI64ValueStandardScheme();
-    }
-  }
-
-  private static class TI64ValueStandardScheme extends StandardScheme<TI64Value> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TI64Value struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUE
-            if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
-              struct.value = iprot.readI64();
-              struct.setValueIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TI64Value struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.isSetValue()) {
-        oprot.writeFieldBegin(VALUE_FIELD_DESC);
-        oprot.writeI64(struct.value);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TI64ValueTupleSchemeFactory implements SchemeFactory {
-    public TI64ValueTupleScheme getScheme() {
-      return new TI64ValueTupleScheme();
-    }
-  }
-
-  private static class TI64ValueTupleScheme extends TupleScheme<TI64Value> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TI64Value struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      BitSet optionals = new BitSet();
-      if (struct.isSetValue()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetValue()) {
-        oprot.writeI64(struct.value);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TI64Value struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.value = iprot.readI64();
-        struct.setValueIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TJobExecutionStatus.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TJobExecutionStatus.java
deleted file mode 100644
index b39f208c1b878..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TJobExecutionStatus.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.thrift.TEnum;
-
-public enum TJobExecutionStatus implements org.apache.thrift.TEnum {
-  IN_PROGRESS(0),
-  COMPLETE(1),
-  NOT_AVAILABLE(2);
-
-  private final int value;
-
-  private TJobExecutionStatus(int value) {
-    this.value = value;
-  }
-
-  /**
-   * Get the integer value of this enum value, as defined in the Thrift IDL.
-   */
-  public int getValue() {
-    return value;
-  }
-
-  /**
-   * Find a the enum type by its integer value, as defined in the Thrift IDL.
-   * @return null if the value is not found.
-   */
-  public static TJobExecutionStatus findByValue(int value) { 
-    switch (value) {
-      case 0:
-        return IN_PROGRESS;
-      case 1:
-        return COMPLETE;
-      case 2:
-        return NOT_AVAILABLE;
-      default:
-        return null;
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TMapTypeEntry.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TMapTypeEntry.java
deleted file mode 100644
index 7ebc15c9432be..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TMapTypeEntry.java
+++ /dev/null
@@ -1,482 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TMapTypeEntry implements org.apache.thrift.TBase<TMapTypeEntry, TMapTypeEntry._Fields>, java.io.Serializable, Cloneable, Comparable<TMapTypeEntry> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TMapTypeEntry");
-
-  private static final org.apache.thrift.protocol.TField KEY_TYPE_PTR_FIELD_DESC = new org.apache.thrift.protocol.TField("keyTypePtr", org.apache.thrift.protocol.TType.I32, (short)1);
-  private static final org.apache.thrift.protocol.TField VALUE_TYPE_PTR_FIELD_DESC = new org.apache.thrift.protocol.TField("valueTypePtr", org.apache.thrift.protocol.TType.I32, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TMapTypeEntryStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TMapTypeEntryTupleSchemeFactory());
-  }
-
-  private int keyTypePtr; // required
-  private int valueTypePtr; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    KEY_TYPE_PTR((short)1, "keyTypePtr"),
-    VALUE_TYPE_PTR((short)2, "valueTypePtr");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // KEY_TYPE_PTR
-          return KEY_TYPE_PTR;
-        case 2: // VALUE_TYPE_PTR
-          return VALUE_TYPE_PTR;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __KEYTYPEPTR_ISSET_ID = 0;
-  private static final int __VALUETYPEPTR_ISSET_ID = 1;
-  private byte __isset_bitfield = 0;
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.KEY_TYPE_PTR, new org.apache.thrift.meta_data.FieldMetaData("keyTypePtr", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32        , "TTypeEntryPtr")));
-    tmpMap.put(_Fields.VALUE_TYPE_PTR, new org.apache.thrift.meta_data.FieldMetaData("valueTypePtr", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32        , "TTypeEntryPtr")));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TMapTypeEntry.class, metaDataMap);
-  }
-
-  public TMapTypeEntry() {
-  }
-
-  public TMapTypeEntry(
-    int keyTypePtr,
-    int valueTypePtr)
-  {
-    this();
-    this.keyTypePtr = keyTypePtr;
-    setKeyTypePtrIsSet(true);
-    this.valueTypePtr = valueTypePtr;
-    setValueTypePtrIsSet(true);
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TMapTypeEntry(TMapTypeEntry other) {
-    __isset_bitfield = other.__isset_bitfield;
-    this.keyTypePtr = other.keyTypePtr;
-    this.valueTypePtr = other.valueTypePtr;
-  }
-
-  public TMapTypeEntry deepCopy() {
-    return new TMapTypeEntry(this);
-  }
-
-  @Override
-  public void clear() {
-    setKeyTypePtrIsSet(false);
-    this.keyTypePtr = 0;
-    setValueTypePtrIsSet(false);
-    this.valueTypePtr = 0;
-  }
-
-  public int getKeyTypePtr() {
-    return this.keyTypePtr;
-  }
-
-  public void setKeyTypePtr(int keyTypePtr) {
-    this.keyTypePtr = keyTypePtr;
-    setKeyTypePtrIsSet(true);
-  }
-
-  public void unsetKeyTypePtr() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __KEYTYPEPTR_ISSET_ID);
-  }
-
-  /** Returns true if field keyTypePtr is set (has been assigned a value) and false otherwise */
-  public boolean isSetKeyTypePtr() {
-    return EncodingUtils.testBit(__isset_bitfield, __KEYTYPEPTR_ISSET_ID);
-  }
-
-  public void setKeyTypePtrIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __KEYTYPEPTR_ISSET_ID, value);
-  }
-
-  public int getValueTypePtr() {
-    return this.valueTypePtr;
-  }
-
-  public void setValueTypePtr(int valueTypePtr) {
-    this.valueTypePtr = valueTypePtr;
-    setValueTypePtrIsSet(true);
-  }
-
-  public void unsetValueTypePtr() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __VALUETYPEPTR_ISSET_ID);
-  }
-
-  /** Returns true if field valueTypePtr is set (has been assigned a value) and false otherwise */
-  public boolean isSetValueTypePtr() {
-    return EncodingUtils.testBit(__isset_bitfield, __VALUETYPEPTR_ISSET_ID);
-  }
-
-  public void setValueTypePtrIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __VALUETYPEPTR_ISSET_ID, value);
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case KEY_TYPE_PTR:
-      if (value == null) {
-        unsetKeyTypePtr();
-      } else {
-        setKeyTypePtr((Integer)value);
-      }
-      break;
-
-    case VALUE_TYPE_PTR:
-      if (value == null) {
-        unsetValueTypePtr();
-      } else {
-        setValueTypePtr((Integer)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case KEY_TYPE_PTR:
-      return getKeyTypePtr();
-
-    case VALUE_TYPE_PTR:
-      return getValueTypePtr();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case KEY_TYPE_PTR:
-      return isSetKeyTypePtr();
-    case VALUE_TYPE_PTR:
-      return isSetValueTypePtr();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TMapTypeEntry)
-      return this.equals((TMapTypeEntry)that);
-    return false;
-  }
-
-  public boolean equals(TMapTypeEntry that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_keyTypePtr = true;
-    boolean that_present_keyTypePtr = true;
-    if (this_present_keyTypePtr || that_present_keyTypePtr) {
-      if (!(this_present_keyTypePtr && that_present_keyTypePtr))
-        return false;
-      if (this.keyTypePtr != that.keyTypePtr)
-        return false;
-    }
-
-    boolean this_present_valueTypePtr = true;
-    boolean that_present_valueTypePtr = true;
-    if (this_present_valueTypePtr || that_present_valueTypePtr) {
-      if (!(this_present_valueTypePtr && that_present_valueTypePtr))
-        return false;
-      if (this.valueTypePtr != that.valueTypePtr)
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_keyTypePtr = true;
-    list.add(present_keyTypePtr);
-    if (present_keyTypePtr)
-      list.add(keyTypePtr);
-
-    boolean present_valueTypePtr = true;
-    list.add(present_valueTypePtr);
-    if (present_valueTypePtr)
-      list.add(valueTypePtr);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TMapTypeEntry other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetKeyTypePtr()).compareTo(other.isSetKeyTypePtr());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetKeyTypePtr()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.keyTypePtr, other.keyTypePtr);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetValueTypePtr()).compareTo(other.isSetValueTypePtr());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValueTypePtr()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.valueTypePtr, other.valueTypePtr);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TMapTypeEntry(");
-    boolean first = true;
-
-    sb.append("keyTypePtr:");
-    sb.append(this.keyTypePtr);
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("valueTypePtr:");
-    sb.append(this.valueTypePtr);
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetKeyTypePtr()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'keyTypePtr' is unset! Struct:" + toString());
-    }
-
-    if (!isSetValueTypePtr()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'valueTypePtr' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TMapTypeEntryStandardSchemeFactory implements SchemeFactory {
-    public TMapTypeEntryStandardScheme getScheme() {
-      return new TMapTypeEntryStandardScheme();
-    }
-  }
-
-  private static class TMapTypeEntryStandardScheme extends StandardScheme<TMapTypeEntry> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TMapTypeEntry struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // KEY_TYPE_PTR
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.keyTypePtr = iprot.readI32();
-              struct.setKeyTypePtrIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // VALUE_TYPE_PTR
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.valueTypePtr = iprot.readI32();
-              struct.setValueTypePtrIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TMapTypeEntry struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      oprot.writeFieldBegin(KEY_TYPE_PTR_FIELD_DESC);
-      oprot.writeI32(struct.keyTypePtr);
-      oprot.writeFieldEnd();
-      oprot.writeFieldBegin(VALUE_TYPE_PTR_FIELD_DESC);
-      oprot.writeI32(struct.valueTypePtr);
-      oprot.writeFieldEnd();
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TMapTypeEntryTupleSchemeFactory implements SchemeFactory {
-    public TMapTypeEntryTupleScheme getScheme() {
-      return new TMapTypeEntryTupleScheme();
-    }
-  }
-
-  private static class TMapTypeEntryTupleScheme extends TupleScheme<TMapTypeEntry> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TMapTypeEntry struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      oprot.writeI32(struct.keyTypePtr);
-      oprot.writeI32(struct.valueTypePtr);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TMapTypeEntry struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.keyTypePtr = iprot.readI32();
-      struct.setKeyTypePtrIsSet(true);
-      struct.valueTypePtr = iprot.readI32();
-      struct.setValueTypePtrIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TOpenSessionReq.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TOpenSessionReq.java
deleted file mode 100644
index e47abbb862cf1..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TOpenSessionReq.java
+++ /dev/null
@@ -1,778 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TOpenSessionReq implements org.apache.thrift.TBase<TOpenSessionReq, TOpenSessionReq._Fields>, java.io.Serializable, Cloneable, Comparable<TOpenSessionReq> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TOpenSessionReq");
-
-  private static final org.apache.thrift.protocol.TField CLIENT_PROTOCOL_FIELD_DESC = new org.apache.thrift.protocol.TField("client_protocol", org.apache.thrift.protocol.TType.I32, (short)1);
-  private static final org.apache.thrift.protocol.TField USERNAME_FIELD_DESC = new org.apache.thrift.protocol.TField("username", org.apache.thrift.protocol.TType.STRING, (short)2);
-  private static final org.apache.thrift.protocol.TField PASSWORD_FIELD_DESC = new org.apache.thrift.protocol.TField("password", org.apache.thrift.protocol.TType.STRING, (short)3);
-  private static final org.apache.thrift.protocol.TField CONFIGURATION_FIELD_DESC = new org.apache.thrift.protocol.TField("configuration", org.apache.thrift.protocol.TType.MAP, (short)4);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TOpenSessionReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TOpenSessionReqTupleSchemeFactory());
-  }
-
-  private TProtocolVersion client_protocol; // required
-  private String username; // optional
-  private String password; // optional
-  private Map<String,String> configuration; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    /**
-     * 
-     * @see TProtocolVersion
-     */
-    CLIENT_PROTOCOL((short)1, "client_protocol"),
-    USERNAME((short)2, "username"),
-    PASSWORD((short)3, "password"),
-    CONFIGURATION((short)4, "configuration");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // CLIENT_PROTOCOL
-          return CLIENT_PROTOCOL;
-        case 2: // USERNAME
-          return USERNAME;
-        case 3: // PASSWORD
-          return PASSWORD;
-        case 4: // CONFIGURATION
-          return CONFIGURATION;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final _Fields optionals[] = {_Fields.USERNAME,_Fields.PASSWORD,_Fields.CONFIGURATION};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.CLIENT_PROTOCOL, new org.apache.thrift.meta_data.FieldMetaData("client_protocol", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TProtocolVersion.class)));
-    tmpMap.put(_Fields.USERNAME, new org.apache.thrift.meta_data.FieldMetaData("username", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    tmpMap.put(_Fields.PASSWORD, new org.apache.thrift.meta_data.FieldMetaData("password", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    tmpMap.put(_Fields.CONFIGURATION, new org.apache.thrift.meta_data.FieldMetaData("configuration", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING), 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TOpenSessionReq.class, metaDataMap);
-  }
-
-  public TOpenSessionReq() {
-    this.client_protocol = org.apache.hive.service.rpc.thrift.TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V10;
-
-  }
-
-  public TOpenSessionReq(
-    TProtocolVersion client_protocol)
-  {
-    this();
-    this.client_protocol = client_protocol;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TOpenSessionReq(TOpenSessionReq other) {
-    if (other.isSetClient_protocol()) {
-      this.client_protocol = other.client_protocol;
-    }
-    if (other.isSetUsername()) {
-      this.username = other.username;
-    }
-    if (other.isSetPassword()) {
-      this.password = other.password;
-    }
-    if (other.isSetConfiguration()) {
-      Map<String,String> __this__configuration = new HashMap<String,String>(other.configuration);
-      this.configuration = __this__configuration;
-    }
-  }
-
-  public TOpenSessionReq deepCopy() {
-    return new TOpenSessionReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.client_protocol = org.apache.hive.service.rpc.thrift.TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V10;
-
-    this.username = null;
-    this.password = null;
-    this.configuration = null;
-  }
-
-  /**
-   * 
-   * @see TProtocolVersion
-   */
-  public TProtocolVersion getClient_protocol() {
-    return this.client_protocol;
-  }
-
-  /**
-   * 
-   * @see TProtocolVersion
-   */
-  public void setClient_protocol(TProtocolVersion client_protocol) {
-    this.client_protocol = client_protocol;
-  }
-
-  public void unsetClient_protocol() {
-    this.client_protocol = null;
-  }
-
-  /** Returns true if field client_protocol is set (has been assigned a value) and false otherwise */
-  public boolean isSetClient_protocol() {
-    return this.client_protocol != null;
-  }
-
-  public void setClient_protocolIsSet(boolean value) {
-    if (!value) {
-      this.client_protocol = null;
-    }
-  }
-
-  public String getUsername() {
-    return this.username;
-  }
-
-  public void setUsername(String username) {
-    this.username = username;
-  }
-
-  public void unsetUsername() {
-    this.username = null;
-  }
-
-  /** Returns true if field username is set (has been assigned a value) and false otherwise */
-  public boolean isSetUsername() {
-    return this.username != null;
-  }
-
-  public void setUsernameIsSet(boolean value) {
-    if (!value) {
-      this.username = null;
-    }
-  }
-
-  public String getPassword() {
-    return this.password;
-  }
-
-  public void setPassword(String password) {
-    this.password = password;
-  }
-
-  public void unsetPassword() {
-    this.password = null;
-  }
-
-  /** Returns true if field password is set (has been assigned a value) and false otherwise */
-  public boolean isSetPassword() {
-    return this.password != null;
-  }
-
-  public void setPasswordIsSet(boolean value) {
-    if (!value) {
-      this.password = null;
-    }
-  }
-
-  public int getConfigurationSize() {
-    return (this.configuration == null) ? 0 : this.configuration.size();
-  }
-
-  public void putToConfiguration(String key, String val) {
-    if (this.configuration == null) {
-      this.configuration = new HashMap<String,String>();
-    }
-    this.configuration.put(key, val);
-  }
-
-  public Map<String,String> getConfiguration() {
-    return this.configuration;
-  }
-
-  public void setConfiguration(Map<String,String> configuration) {
-    this.configuration = configuration;
-  }
-
-  public void unsetConfiguration() {
-    this.configuration = null;
-  }
-
-  /** Returns true if field configuration is set (has been assigned a value) and false otherwise */
-  public boolean isSetConfiguration() {
-    return this.configuration != null;
-  }
-
-  public void setConfigurationIsSet(boolean value) {
-    if (!value) {
-      this.configuration = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case CLIENT_PROTOCOL:
-      if (value == null) {
-        unsetClient_protocol();
-      } else {
-        setClient_protocol((TProtocolVersion)value);
-      }
-      break;
-
-    case USERNAME:
-      if (value == null) {
-        unsetUsername();
-      } else {
-        setUsername((String)value);
-      }
-      break;
-
-    case PASSWORD:
-      if (value == null) {
-        unsetPassword();
-      } else {
-        setPassword((String)value);
-      }
-      break;
-
-    case CONFIGURATION:
-      if (value == null) {
-        unsetConfiguration();
-      } else {
-        setConfiguration((Map<String,String>)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case CLIENT_PROTOCOL:
-      return getClient_protocol();
-
-    case USERNAME:
-      return getUsername();
-
-    case PASSWORD:
-      return getPassword();
-
-    case CONFIGURATION:
-      return getConfiguration();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case CLIENT_PROTOCOL:
-      return isSetClient_protocol();
-    case USERNAME:
-      return isSetUsername();
-    case PASSWORD:
-      return isSetPassword();
-    case CONFIGURATION:
-      return isSetConfiguration();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TOpenSessionReq)
-      return this.equals((TOpenSessionReq)that);
-    return false;
-  }
-
-  public boolean equals(TOpenSessionReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_client_protocol = true && this.isSetClient_protocol();
-    boolean that_present_client_protocol = true && that.isSetClient_protocol();
-    if (this_present_client_protocol || that_present_client_protocol) {
-      if (!(this_present_client_protocol && that_present_client_protocol))
-        return false;
-      if (!this.client_protocol.equals(that.client_protocol))
-        return false;
-    }
-
-    boolean this_present_username = true && this.isSetUsername();
-    boolean that_present_username = true && that.isSetUsername();
-    if (this_present_username || that_present_username) {
-      if (!(this_present_username && that_present_username))
-        return false;
-      if (!this.username.equals(that.username))
-        return false;
-    }
-
-    boolean this_present_password = true && this.isSetPassword();
-    boolean that_present_password = true && that.isSetPassword();
-    if (this_present_password || that_present_password) {
-      if (!(this_present_password && that_present_password))
-        return false;
-      if (!this.password.equals(that.password))
-        return false;
-    }
-
-    boolean this_present_configuration = true && this.isSetConfiguration();
-    boolean that_present_configuration = true && that.isSetConfiguration();
-    if (this_present_configuration || that_present_configuration) {
-      if (!(this_present_configuration && that_present_configuration))
-        return false;
-      if (!this.configuration.equals(that.configuration))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_client_protocol = true && (isSetClient_protocol());
-    list.add(present_client_protocol);
-    if (present_client_protocol)
-      list.add(client_protocol.getValue());
-
-    boolean present_username = true && (isSetUsername());
-    list.add(present_username);
-    if (present_username)
-      list.add(username);
-
-    boolean present_password = true && (isSetPassword());
-    list.add(present_password);
-    if (present_password)
-      list.add(password);
-
-    boolean present_configuration = true && (isSetConfiguration());
-    list.add(present_configuration);
-    if (present_configuration)
-      list.add(configuration);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TOpenSessionReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetClient_protocol()).compareTo(other.isSetClient_protocol());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetClient_protocol()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.client_protocol, other.client_protocol);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetUsername()).compareTo(other.isSetUsername());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetUsername()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.username, other.username);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetPassword()).compareTo(other.isSetPassword());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetPassword()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.password, other.password);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetConfiguration()).compareTo(other.isSetConfiguration());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetConfiguration()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.configuration, other.configuration);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TOpenSessionReq(");
-    boolean first = true;
-
-    sb.append("client_protocol:");
-    if (this.client_protocol == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.client_protocol);
-    }
-    first = false;
-    if (isSetUsername()) {
-      if (!first) sb.append(", ");
-      sb.append("username:");
-      if (this.username == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.username);
-      }
-      first = false;
-    }
-    if (isSetPassword()) {
-      if (!first) sb.append(", ");
-      sb.append("password:");
-      if (this.password == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.password);
-      }
-      first = false;
-    }
-    if (isSetConfiguration()) {
-      if (!first) sb.append(", ");
-      sb.append("configuration:");
-      if (this.configuration == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.configuration);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetClient_protocol()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'client_protocol' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TOpenSessionReqStandardSchemeFactory implements SchemeFactory {
-    public TOpenSessionReqStandardScheme getScheme() {
-      return new TOpenSessionReqStandardScheme();
-    }
-  }
-
-  private static class TOpenSessionReqStandardScheme extends StandardScheme<TOpenSessionReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TOpenSessionReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // CLIENT_PROTOCOL
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.client_protocol = org.apache.hive.service.rpc.thrift.TProtocolVersion.findByValue(iprot.readI32());
-              struct.setClient_protocolIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // USERNAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.username = iprot.readString();
-              struct.setUsernameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // PASSWORD
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.password = iprot.readString();
-              struct.setPasswordIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // CONFIGURATION
-            if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
-              {
-                org.apache.thrift.protocol.TMap _map142 = iprot.readMapBegin();
-                struct.configuration = new HashMap<String,String>(2*_map142.size);
-                String _key143;
-                String _val144;
-                for (int _i145 = 0; _i145 < _map142.size; ++_i145)
-                {
-                  _key143 = iprot.readString();
-                  _val144 = iprot.readString();
-                  struct.configuration.put(_key143, _val144);
-                }
-                iprot.readMapEnd();
-              }
-              struct.setConfigurationIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TOpenSessionReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.client_protocol != null) {
-        oprot.writeFieldBegin(CLIENT_PROTOCOL_FIELD_DESC);
-        oprot.writeI32(struct.client_protocol.getValue());
-        oprot.writeFieldEnd();
-      }
-      if (struct.username != null) {
-        if (struct.isSetUsername()) {
-          oprot.writeFieldBegin(USERNAME_FIELD_DESC);
-          oprot.writeString(struct.username);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.password != null) {
-        if (struct.isSetPassword()) {
-          oprot.writeFieldBegin(PASSWORD_FIELD_DESC);
-          oprot.writeString(struct.password);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.configuration != null) {
-        if (struct.isSetConfiguration()) {
-          oprot.writeFieldBegin(CONFIGURATION_FIELD_DESC);
-          {
-            oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.configuration.size()));
-            for (Map.Entry<String, String> _iter146 : struct.configuration.entrySet())
-            {
-              oprot.writeString(_iter146.getKey());
-              oprot.writeString(_iter146.getValue());
-            }
-            oprot.writeMapEnd();
-          }
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TOpenSessionReqTupleSchemeFactory implements SchemeFactory {
-    public TOpenSessionReqTupleScheme getScheme() {
-      return new TOpenSessionReqTupleScheme();
-    }
-  }
-
-  private static class TOpenSessionReqTupleScheme extends TupleScheme<TOpenSessionReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TOpenSessionReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      oprot.writeI32(struct.client_protocol.getValue());
-      BitSet optionals = new BitSet();
-      if (struct.isSetUsername()) {
-        optionals.set(0);
-      }
-      if (struct.isSetPassword()) {
-        optionals.set(1);
-      }
-      if (struct.isSetConfiguration()) {
-        optionals.set(2);
-      }
-      oprot.writeBitSet(optionals, 3);
-      if (struct.isSetUsername()) {
-        oprot.writeString(struct.username);
-      }
-      if (struct.isSetPassword()) {
-        oprot.writeString(struct.password);
-      }
-      if (struct.isSetConfiguration()) {
-        {
-          oprot.writeI32(struct.configuration.size());
-          for (Map.Entry<String, String> _iter147 : struct.configuration.entrySet())
-          {
-            oprot.writeString(_iter147.getKey());
-            oprot.writeString(_iter147.getValue());
-          }
-        }
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TOpenSessionReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.client_protocol = org.apache.hive.service.rpc.thrift.TProtocolVersion.findByValue(iprot.readI32());
-      struct.setClient_protocolIsSet(true);
-      BitSet incoming = iprot.readBitSet(3);
-      if (incoming.get(0)) {
-        struct.username = iprot.readString();
-        struct.setUsernameIsSet(true);
-      }
-      if (incoming.get(1)) {
-        struct.password = iprot.readString();
-        struct.setPasswordIsSet(true);
-      }
-      if (incoming.get(2)) {
-        {
-          org.apache.thrift.protocol.TMap _map148 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-          struct.configuration = new HashMap<String,String>(2*_map148.size);
-          String _key149;
-          String _val150;
-          for (int _i151 = 0; _i151 < _map148.size; ++_i151)
-          {
-            _key149 = iprot.readString();
-            _val150 = iprot.readString();
-            struct.configuration.put(_key149, _val150);
-          }
-        }
-        struct.setConfigurationIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TOpenSessionResp.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TOpenSessionResp.java
deleted file mode 100644
index ee1c87bfd76fa..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TOpenSessionResp.java
+++ /dev/null
@@ -1,783 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TOpenSessionResp implements org.apache.thrift.TBase<TOpenSessionResp, TOpenSessionResp._Fields>, java.io.Serializable, Cloneable, Comparable<TOpenSessionResp> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TOpenSessionResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField SERVER_PROTOCOL_VERSION_FIELD_DESC = new org.apache.thrift.protocol.TField("serverProtocolVersion", org.apache.thrift.protocol.TType.I32, (short)2);
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-  private static final org.apache.thrift.protocol.TField CONFIGURATION_FIELD_DESC = new org.apache.thrift.protocol.TField("configuration", org.apache.thrift.protocol.TType.MAP, (short)4);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TOpenSessionRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TOpenSessionRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-  private TProtocolVersion serverProtocolVersion; // required
-  private TSessionHandle sessionHandle; // optional
-  private Map<String,String> configuration; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status"),
-    /**
-     * 
-     * @see TProtocolVersion
-     */
-    SERVER_PROTOCOL_VERSION((short)2, "serverProtocolVersion"),
-    SESSION_HANDLE((short)3, "sessionHandle"),
-    CONFIGURATION((short)4, "configuration");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        case 2: // SERVER_PROTOCOL_VERSION
-          return SERVER_PROTOCOL_VERSION;
-        case 3: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        case 4: // CONFIGURATION
-          return CONFIGURATION;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final _Fields optionals[] = {_Fields.SESSION_HANDLE,_Fields.CONFIGURATION};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    tmpMap.put(_Fields.SERVER_PROTOCOL_VERSION, new org.apache.thrift.meta_data.FieldMetaData("serverProtocolVersion", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TProtocolVersion.class)));
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    tmpMap.put(_Fields.CONFIGURATION, new org.apache.thrift.meta_data.FieldMetaData("configuration", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING), 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TOpenSessionResp.class, metaDataMap);
-  }
-
-  public TOpenSessionResp() {
-    this.serverProtocolVersion = org.apache.hive.service.rpc.thrift.TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V10;
-
-  }
-
-  public TOpenSessionResp(
-    TStatus status,
-    TProtocolVersion serverProtocolVersion)
-  {
-    this();
-    this.status = status;
-    this.serverProtocolVersion = serverProtocolVersion;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TOpenSessionResp(TOpenSessionResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-    if (other.isSetServerProtocolVersion()) {
-      this.serverProtocolVersion = other.serverProtocolVersion;
-    }
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-    if (other.isSetConfiguration()) {
-      Map<String,String> __this__configuration = new HashMap<String,String>(other.configuration);
-      this.configuration = __this__configuration;
-    }
-  }
-
-  public TOpenSessionResp deepCopy() {
-    return new TOpenSessionResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-    this.serverProtocolVersion = org.apache.hive.service.rpc.thrift.TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V10;
-
-    this.sessionHandle = null;
-    this.configuration = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  /**
-   * 
-   * @see TProtocolVersion
-   */
-  public TProtocolVersion getServerProtocolVersion() {
-    return this.serverProtocolVersion;
-  }
-
-  /**
-   * 
-   * @see TProtocolVersion
-   */
-  public void setServerProtocolVersion(TProtocolVersion serverProtocolVersion) {
-    this.serverProtocolVersion = serverProtocolVersion;
-  }
-
-  public void unsetServerProtocolVersion() {
-    this.serverProtocolVersion = null;
-  }
-
-  /** Returns true if field serverProtocolVersion is set (has been assigned a value) and false otherwise */
-  public boolean isSetServerProtocolVersion() {
-    return this.serverProtocolVersion != null;
-  }
-
-  public void setServerProtocolVersionIsSet(boolean value) {
-    if (!value) {
-      this.serverProtocolVersion = null;
-    }
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public int getConfigurationSize() {
-    return (this.configuration == null) ? 0 : this.configuration.size();
-  }
-
-  public void putToConfiguration(String key, String val) {
-    if (this.configuration == null) {
-      this.configuration = new HashMap<String,String>();
-    }
-    this.configuration.put(key, val);
-  }
-
-  public Map<String,String> getConfiguration() {
-    return this.configuration;
-  }
-
-  public void setConfiguration(Map<String,String> configuration) {
-    this.configuration = configuration;
-  }
-
-  public void unsetConfiguration() {
-    this.configuration = null;
-  }
-
-  /** Returns true if field configuration is set (has been assigned a value) and false otherwise */
-  public boolean isSetConfiguration() {
-    return this.configuration != null;
-  }
-
-  public void setConfigurationIsSet(boolean value) {
-    if (!value) {
-      this.configuration = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    case SERVER_PROTOCOL_VERSION:
-      if (value == null) {
-        unsetServerProtocolVersion();
-      } else {
-        setServerProtocolVersion((TProtocolVersion)value);
-      }
-      break;
-
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    case CONFIGURATION:
-      if (value == null) {
-        unsetConfiguration();
-      } else {
-        setConfiguration((Map<String,String>)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    case SERVER_PROTOCOL_VERSION:
-      return getServerProtocolVersion();
-
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    case CONFIGURATION:
-      return getConfiguration();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    case SERVER_PROTOCOL_VERSION:
-      return isSetServerProtocolVersion();
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    case CONFIGURATION:
-      return isSetConfiguration();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TOpenSessionResp)
-      return this.equals((TOpenSessionResp)that);
-    return false;
-  }
-
-  public boolean equals(TOpenSessionResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_serverProtocolVersion = true && this.isSetServerProtocolVersion();
-    boolean that_present_serverProtocolVersion = true && that.isSetServerProtocolVersion();
-    if (this_present_serverProtocolVersion || that_present_serverProtocolVersion) {
-      if (!(this_present_serverProtocolVersion && that_present_serverProtocolVersion))
-        return false;
-      if (!this.serverProtocolVersion.equals(that.serverProtocolVersion))
-        return false;
-    }
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    boolean this_present_configuration = true && this.isSetConfiguration();
-    boolean that_present_configuration = true && that.isSetConfiguration();
-    if (this_present_configuration || that_present_configuration) {
-      if (!(this_present_configuration && that_present_configuration))
-        return false;
-      if (!this.configuration.equals(that.configuration))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_status = true && (isSetStatus());
-    list.add(present_status);
-    if (present_status)
-      list.add(status);
-
-    boolean present_serverProtocolVersion = true && (isSetServerProtocolVersion());
-    list.add(present_serverProtocolVersion);
-    if (present_serverProtocolVersion)
-      list.add(serverProtocolVersion.getValue());
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    list.add(present_sessionHandle);
-    if (present_sessionHandle)
-      list.add(sessionHandle);
-
-    boolean present_configuration = true && (isSetConfiguration());
-    list.add(present_configuration);
-    if (present_configuration)
-      list.add(configuration);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TOpenSessionResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(other.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, other.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetServerProtocolVersion()).compareTo(other.isSetServerProtocolVersion());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetServerProtocolVersion()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.serverProtocolVersion, other.serverProtocolVersion);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(other.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, other.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetConfiguration()).compareTo(other.isSetConfiguration());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetConfiguration()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.configuration, other.configuration);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TOpenSessionResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("serverProtocolVersion:");
-    if (this.serverProtocolVersion == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.serverProtocolVersion);
-    }
-    first = false;
-    if (isSetSessionHandle()) {
-      if (!first) sb.append(", ");
-      sb.append("sessionHandle:");
-      if (this.sessionHandle == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.sessionHandle);
-      }
-      first = false;
-    }
-    if (isSetConfiguration()) {
-      if (!first) sb.append(", ");
-      sb.append("configuration:");
-      if (this.configuration == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.configuration);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    if (!isSetServerProtocolVersion()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'serverProtocolVersion' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TOpenSessionRespStandardSchemeFactory implements SchemeFactory {
-    public TOpenSessionRespStandardScheme getScheme() {
-      return new TOpenSessionRespStandardScheme();
-    }
-  }
-
-  private static class TOpenSessionRespStandardScheme extends StandardScheme<TOpenSessionResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TOpenSessionResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // SERVER_PROTOCOL_VERSION
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.serverProtocolVersion = org.apache.hive.service.rpc.thrift.TProtocolVersion.findByValue(iprot.readI32());
-              struct.setServerProtocolVersionIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // CONFIGURATION
-            if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
-              {
-                org.apache.thrift.protocol.TMap _map152 = iprot.readMapBegin();
-                struct.configuration = new HashMap<String,String>(2*_map152.size);
-                String _key153;
-                String _val154;
-                for (int _i155 = 0; _i155 < _map152.size; ++_i155)
-                {
-                  _key153 = iprot.readString();
-                  _val154 = iprot.readString();
-                  struct.configuration.put(_key153, _val154);
-                }
-                iprot.readMapEnd();
-              }
-              struct.setConfigurationIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TOpenSessionResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.serverProtocolVersion != null) {
-        oprot.writeFieldBegin(SERVER_PROTOCOL_VERSION_FIELD_DESC);
-        oprot.writeI32(struct.serverProtocolVersion.getValue());
-        oprot.writeFieldEnd();
-      }
-      if (struct.sessionHandle != null) {
-        if (struct.isSetSessionHandle()) {
-          oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-          struct.sessionHandle.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.configuration != null) {
-        if (struct.isSetConfiguration()) {
-          oprot.writeFieldBegin(CONFIGURATION_FIELD_DESC);
-          {
-            oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.configuration.size()));
-            for (Map.Entry<String, String> _iter156 : struct.configuration.entrySet())
-            {
-              oprot.writeString(_iter156.getKey());
-              oprot.writeString(_iter156.getValue());
-            }
-            oprot.writeMapEnd();
-          }
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TOpenSessionRespTupleSchemeFactory implements SchemeFactory {
-    public TOpenSessionRespTupleScheme getScheme() {
-      return new TOpenSessionRespTupleScheme();
-    }
-  }
-
-  private static class TOpenSessionRespTupleScheme extends TupleScheme<TOpenSessionResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TOpenSessionResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-      oprot.writeI32(struct.serverProtocolVersion.getValue());
-      BitSet optionals = new BitSet();
-      if (struct.isSetSessionHandle()) {
-        optionals.set(0);
-      }
-      if (struct.isSetConfiguration()) {
-        optionals.set(1);
-      }
-      oprot.writeBitSet(optionals, 2);
-      if (struct.isSetSessionHandle()) {
-        struct.sessionHandle.write(oprot);
-      }
-      if (struct.isSetConfiguration()) {
-        {
-          oprot.writeI32(struct.configuration.size());
-          for (Map.Entry<String, String> _iter157 : struct.configuration.entrySet())
-          {
-            oprot.writeString(_iter157.getKey());
-            oprot.writeString(_iter157.getValue());
-          }
-        }
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TOpenSessionResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-      struct.serverProtocolVersion = org.apache.hive.service.rpc.thrift.TProtocolVersion.findByValue(iprot.readI32());
-      struct.setServerProtocolVersionIsSet(true);
-      BitSet incoming = iprot.readBitSet(2);
-      if (incoming.get(0)) {
-        struct.sessionHandle = new TSessionHandle();
-        struct.sessionHandle.read(iprot);
-        struct.setSessionHandleIsSet(true);
-      }
-      if (incoming.get(1)) {
-        {
-          org.apache.thrift.protocol.TMap _map158 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-          struct.configuration = new HashMap<String,String>(2*_map158.size);
-          String _key159;
-          String _val160;
-          for (int _i161 = 0; _i161 < _map158.size; ++_i161)
-          {
-            _key159 = iprot.readString();
-            _val160 = iprot.readString();
-            struct.configuration.put(_key159, _val160);
-          }
-        }
-        struct.setConfigurationIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TOperationHandle.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TOperationHandle.java
deleted file mode 100644
index 9eaf2be3ed5ea..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TOperationHandle.java
+++ /dev/null
@@ -1,709 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TOperationHandle implements org.apache.thrift.TBase<TOperationHandle, TOperationHandle._Fields>, java.io.Serializable, Cloneable, Comparable<TOperationHandle> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TOperationHandle");
-
-  private static final org.apache.thrift.protocol.TField OPERATION_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("operationId", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField OPERATION_TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("operationType", org.apache.thrift.protocol.TType.I32, (short)2);
-  private static final org.apache.thrift.protocol.TField HAS_RESULT_SET_FIELD_DESC = new org.apache.thrift.protocol.TField("hasResultSet", org.apache.thrift.protocol.TType.BOOL, (short)3);
-  private static final org.apache.thrift.protocol.TField MODIFIED_ROW_COUNT_FIELD_DESC = new org.apache.thrift.protocol.TField("modifiedRowCount", org.apache.thrift.protocol.TType.DOUBLE, (short)4);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TOperationHandleStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TOperationHandleTupleSchemeFactory());
-  }
-
-  private THandleIdentifier operationId; // required
-  private TOperationType operationType; // required
-  private boolean hasResultSet; // required
-  private double modifiedRowCount; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    OPERATION_ID((short)1, "operationId"),
-    /**
-     * 
-     * @see TOperationType
-     */
-    OPERATION_TYPE((short)2, "operationType"),
-    HAS_RESULT_SET((short)3, "hasResultSet"),
-    MODIFIED_ROW_COUNT((short)4, "modifiedRowCount");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // OPERATION_ID
-          return OPERATION_ID;
-        case 2: // OPERATION_TYPE
-          return OPERATION_TYPE;
-        case 3: // HAS_RESULT_SET
-          return HAS_RESULT_SET;
-        case 4: // MODIFIED_ROW_COUNT
-          return MODIFIED_ROW_COUNT;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __HASRESULTSET_ISSET_ID = 0;
-  private static final int __MODIFIEDROWCOUNT_ISSET_ID = 1;
-  private byte __isset_bitfield = 0;
-  private static final _Fields optionals[] = {_Fields.MODIFIED_ROW_COUNT};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.OPERATION_ID, new org.apache.thrift.meta_data.FieldMetaData("operationId", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, THandleIdentifier.class)));
-    tmpMap.put(_Fields.OPERATION_TYPE, new org.apache.thrift.meta_data.FieldMetaData("operationType", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TOperationType.class)));
-    tmpMap.put(_Fields.HAS_RESULT_SET, new org.apache.thrift.meta_data.FieldMetaData("hasResultSet", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
-    tmpMap.put(_Fields.MODIFIED_ROW_COUNT, new org.apache.thrift.meta_data.FieldMetaData("modifiedRowCount", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TOperationHandle.class, metaDataMap);
-  }
-
-  public TOperationHandle() {
-  }
-
-  public TOperationHandle(
-    THandleIdentifier operationId,
-    TOperationType operationType,
-    boolean hasResultSet)
-  {
-    this();
-    this.operationId = operationId;
-    this.operationType = operationType;
-    this.hasResultSet = hasResultSet;
-    setHasResultSetIsSet(true);
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TOperationHandle(TOperationHandle other) {
-    __isset_bitfield = other.__isset_bitfield;
-    if (other.isSetOperationId()) {
-      this.operationId = new THandleIdentifier(other.operationId);
-    }
-    if (other.isSetOperationType()) {
-      this.operationType = other.operationType;
-    }
-    this.hasResultSet = other.hasResultSet;
-    this.modifiedRowCount = other.modifiedRowCount;
-  }
-
-  public TOperationHandle deepCopy() {
-    return new TOperationHandle(this);
-  }
-
-  @Override
-  public void clear() {
-    this.operationId = null;
-    this.operationType = null;
-    setHasResultSetIsSet(false);
-    this.hasResultSet = false;
-    setModifiedRowCountIsSet(false);
-    this.modifiedRowCount = 0.0;
-  }
-
-  public THandleIdentifier getOperationId() {
-    return this.operationId;
-  }
-
-  public void setOperationId(THandleIdentifier operationId) {
-    this.operationId = operationId;
-  }
-
-  public void unsetOperationId() {
-    this.operationId = null;
-  }
-
-  /** Returns true if field operationId is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationId() {
-    return this.operationId != null;
-  }
-
-  public void setOperationIdIsSet(boolean value) {
-    if (!value) {
-      this.operationId = null;
-    }
-  }
-
-  /**
-   * 
-   * @see TOperationType
-   */
-  public TOperationType getOperationType() {
-    return this.operationType;
-  }
-
-  /**
-   * 
-   * @see TOperationType
-   */
-  public void setOperationType(TOperationType operationType) {
-    this.operationType = operationType;
-  }
-
-  public void unsetOperationType() {
-    this.operationType = null;
-  }
-
-  /** Returns true if field operationType is set (has been assigned a value) and false otherwise */
-  public boolean isSetOperationType() {
-    return this.operationType != null;
-  }
-
-  public void setOperationTypeIsSet(boolean value) {
-    if (!value) {
-      this.operationType = null;
-    }
-  }
-
-  public boolean isHasResultSet() {
-    return this.hasResultSet;
-  }
-
-  public void setHasResultSet(boolean hasResultSet) {
-    this.hasResultSet = hasResultSet;
-    setHasResultSetIsSet(true);
-  }
-
-  public void unsetHasResultSet() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __HASRESULTSET_ISSET_ID);
-  }
-
-  /** Returns true if field hasResultSet is set (has been assigned a value) and false otherwise */
-  public boolean isSetHasResultSet() {
-    return EncodingUtils.testBit(__isset_bitfield, __HASRESULTSET_ISSET_ID);
-  }
-
-  public void setHasResultSetIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __HASRESULTSET_ISSET_ID, value);
-  }
-
-  public double getModifiedRowCount() {
-    return this.modifiedRowCount;
-  }
-
-  public void setModifiedRowCount(double modifiedRowCount) {
-    this.modifiedRowCount = modifiedRowCount;
-    setModifiedRowCountIsSet(true);
-  }
-
-  public void unsetModifiedRowCount() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __MODIFIEDROWCOUNT_ISSET_ID);
-  }
-
-  /** Returns true if field modifiedRowCount is set (has been assigned a value) and false otherwise */
-  public boolean isSetModifiedRowCount() {
-    return EncodingUtils.testBit(__isset_bitfield, __MODIFIEDROWCOUNT_ISSET_ID);
-  }
-
-  public void setModifiedRowCountIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MODIFIEDROWCOUNT_ISSET_ID, value);
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case OPERATION_ID:
-      if (value == null) {
-        unsetOperationId();
-      } else {
-        setOperationId((THandleIdentifier)value);
-      }
-      break;
-
-    case OPERATION_TYPE:
-      if (value == null) {
-        unsetOperationType();
-      } else {
-        setOperationType((TOperationType)value);
-      }
-      break;
-
-    case HAS_RESULT_SET:
-      if (value == null) {
-        unsetHasResultSet();
-      } else {
-        setHasResultSet((Boolean)value);
-      }
-      break;
-
-    case MODIFIED_ROW_COUNT:
-      if (value == null) {
-        unsetModifiedRowCount();
-      } else {
-        setModifiedRowCount((Double)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case OPERATION_ID:
-      return getOperationId();
-
-    case OPERATION_TYPE:
-      return getOperationType();
-
-    case HAS_RESULT_SET:
-      return isHasResultSet();
-
-    case MODIFIED_ROW_COUNT:
-      return getModifiedRowCount();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case OPERATION_ID:
-      return isSetOperationId();
-    case OPERATION_TYPE:
-      return isSetOperationType();
-    case HAS_RESULT_SET:
-      return isSetHasResultSet();
-    case MODIFIED_ROW_COUNT:
-      return isSetModifiedRowCount();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TOperationHandle)
-      return this.equals((TOperationHandle)that);
-    return false;
-  }
-
-  public boolean equals(TOperationHandle that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_operationId = true && this.isSetOperationId();
-    boolean that_present_operationId = true && that.isSetOperationId();
-    if (this_present_operationId || that_present_operationId) {
-      if (!(this_present_operationId && that_present_operationId))
-        return false;
-      if (!this.operationId.equals(that.operationId))
-        return false;
-    }
-
-    boolean this_present_operationType = true && this.isSetOperationType();
-    boolean that_present_operationType = true && that.isSetOperationType();
-    if (this_present_operationType || that_present_operationType) {
-      if (!(this_present_operationType && that_present_operationType))
-        return false;
-      if (!this.operationType.equals(that.operationType))
-        return false;
-    }
-
-    boolean this_present_hasResultSet = true;
-    boolean that_present_hasResultSet = true;
-    if (this_present_hasResultSet || that_present_hasResultSet) {
-      if (!(this_present_hasResultSet && that_present_hasResultSet))
-        return false;
-      if (this.hasResultSet != that.hasResultSet)
-        return false;
-    }
-
-    boolean this_present_modifiedRowCount = true && this.isSetModifiedRowCount();
-    boolean that_present_modifiedRowCount = true && that.isSetModifiedRowCount();
-    if (this_present_modifiedRowCount || that_present_modifiedRowCount) {
-      if (!(this_present_modifiedRowCount && that_present_modifiedRowCount))
-        return false;
-      if (this.modifiedRowCount != that.modifiedRowCount)
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_operationId = true && (isSetOperationId());
-    list.add(present_operationId);
-    if (present_operationId)
-      list.add(operationId);
-
-    boolean present_operationType = true && (isSetOperationType());
-    list.add(present_operationType);
-    if (present_operationType)
-      list.add(operationType.getValue());
-
-    boolean present_hasResultSet = true;
-    list.add(present_hasResultSet);
-    if (present_hasResultSet)
-      list.add(hasResultSet);
-
-    boolean present_modifiedRowCount = true && (isSetModifiedRowCount());
-    list.add(present_modifiedRowCount);
-    if (present_modifiedRowCount)
-      list.add(modifiedRowCount);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TOperationHandle other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetOperationId()).compareTo(other.isSetOperationId());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationId()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationId, other.operationId);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetOperationType()).compareTo(other.isSetOperationType());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetOperationType()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.operationType, other.operationType);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetHasResultSet()).compareTo(other.isSetHasResultSet());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetHasResultSet()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.hasResultSet, other.hasResultSet);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetModifiedRowCount()).compareTo(other.isSetModifiedRowCount());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetModifiedRowCount()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.modifiedRowCount, other.modifiedRowCount);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TOperationHandle(");
-    boolean first = true;
-
-    sb.append("operationId:");
-    if (this.operationId == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.operationId);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("operationType:");
-    if (this.operationType == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.operationType);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("hasResultSet:");
-    sb.append(this.hasResultSet);
-    first = false;
-    if (isSetModifiedRowCount()) {
-      if (!first) sb.append(", ");
-      sb.append("modifiedRowCount:");
-      sb.append(this.modifiedRowCount);
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetOperationId()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'operationId' is unset! Struct:" + toString());
-    }
-
-    if (!isSetOperationType()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'operationType' is unset! Struct:" + toString());
-    }
-
-    if (!isSetHasResultSet()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'hasResultSet' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (operationId != null) {
-      operationId.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TOperationHandleStandardSchemeFactory implements SchemeFactory {
-    public TOperationHandleStandardScheme getScheme() {
-      return new TOperationHandleStandardScheme();
-    }
-  }
-
-  private static class TOperationHandleStandardScheme extends StandardScheme<TOperationHandle> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TOperationHandle struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // OPERATION_ID
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.operationId = new THandleIdentifier();
-              struct.operationId.read(iprot);
-              struct.setOperationIdIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // OPERATION_TYPE
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.operationType = org.apache.hive.service.rpc.thrift.TOperationType.findByValue(iprot.readI32());
-              struct.setOperationTypeIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // HAS_RESULT_SET
-            if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
-              struct.hasResultSet = iprot.readBool();
-              struct.setHasResultSetIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // MODIFIED_ROW_COUNT
-            if (schemeField.type == org.apache.thrift.protocol.TType.DOUBLE) {
-              struct.modifiedRowCount = iprot.readDouble();
-              struct.setModifiedRowCountIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TOperationHandle struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.operationId != null) {
-        oprot.writeFieldBegin(OPERATION_ID_FIELD_DESC);
-        struct.operationId.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.operationType != null) {
-        oprot.writeFieldBegin(OPERATION_TYPE_FIELD_DESC);
-        oprot.writeI32(struct.operationType.getValue());
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldBegin(HAS_RESULT_SET_FIELD_DESC);
-      oprot.writeBool(struct.hasResultSet);
-      oprot.writeFieldEnd();
-      if (struct.isSetModifiedRowCount()) {
-        oprot.writeFieldBegin(MODIFIED_ROW_COUNT_FIELD_DESC);
-        oprot.writeDouble(struct.modifiedRowCount);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TOperationHandleTupleSchemeFactory implements SchemeFactory {
-    public TOperationHandleTupleScheme getScheme() {
-      return new TOperationHandleTupleScheme();
-    }
-  }
-
-  private static class TOperationHandleTupleScheme extends TupleScheme<TOperationHandle> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TOperationHandle struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.operationId.write(oprot);
-      oprot.writeI32(struct.operationType.getValue());
-      oprot.writeBool(struct.hasResultSet);
-      BitSet optionals = new BitSet();
-      if (struct.isSetModifiedRowCount()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetModifiedRowCount()) {
-        oprot.writeDouble(struct.modifiedRowCount);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TOperationHandle struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.operationId = new THandleIdentifier();
-      struct.operationId.read(iprot);
-      struct.setOperationIdIsSet(true);
-      struct.operationType = org.apache.hive.service.rpc.thrift.TOperationType.findByValue(iprot.readI32());
-      struct.setOperationTypeIsSet(true);
-      struct.hasResultSet = iprot.readBool();
-      struct.setHasResultSetIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.modifiedRowCount = iprot.readDouble();
-        struct.setModifiedRowCountIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TOperationState.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TOperationState.java
deleted file mode 100644
index 4390b4b887583..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TOperationState.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.thrift.TEnum;
-
-public enum TOperationState implements org.apache.thrift.TEnum {
-  INITIALIZED_STATE(0),
-  RUNNING_STATE(1),
-  FINISHED_STATE(2),
-  CANCELED_STATE(3),
-  CLOSED_STATE(4),
-  ERROR_STATE(5),
-  UKNOWN_STATE(6),
-  PENDING_STATE(7),
-  TIMEDOUT_STATE(8);
-
-  private final int value;
-
-  private TOperationState(int value) {
-    this.value = value;
-  }
-
-  /**
-   * Get the integer value of this enum value, as defined in the Thrift IDL.
-   */
-  public int getValue() {
-    return value;
-  }
-
-  /**
-   * Find a the enum type by its integer value, as defined in the Thrift IDL.
-   * @return null if the value is not found.
-   */
-  public static TOperationState findByValue(int value) { 
-    switch (value) {
-      case 0:
-        return INITIALIZED_STATE;
-      case 1:
-        return RUNNING_STATE;
-      case 2:
-        return FINISHED_STATE;
-      case 3:
-        return CANCELED_STATE;
-      case 4:
-        return CLOSED_STATE;
-      case 5:
-        return ERROR_STATE;
-      case 6:
-        return UKNOWN_STATE;
-      case 7:
-        return PENDING_STATE;
-      case 8:
-        return TIMEDOUT_STATE;
-      default:
-        return null;
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TOperationType.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TOperationType.java
deleted file mode 100644
index 08002ad1dc8e8..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TOperationType.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.thrift.TEnum;
-
-public enum TOperationType implements org.apache.thrift.TEnum {
-  EXECUTE_STATEMENT(0),
-  GET_TYPE_INFO(1),
-  GET_CATALOGS(2),
-  GET_SCHEMAS(3),
-  GET_TABLES(4),
-  GET_TABLE_TYPES(5),
-  GET_COLUMNS(6),
-  GET_FUNCTIONS(7),
-  UNKNOWN(8);
-
-  private final int value;
-
-  private TOperationType(int value) {
-    this.value = value;
-  }
-
-  /**
-   * Get the integer value of this enum value, as defined in the Thrift IDL.
-   */
-  public int getValue() {
-    return value;
-  }
-
-  /**
-   * Find a the enum type by its integer value, as defined in the Thrift IDL.
-   * @return null if the value is not found.
-   */
-  public static TOperationType findByValue(int value) { 
-    switch (value) {
-      case 0:
-        return EXECUTE_STATEMENT;
-      case 1:
-        return GET_TYPE_INFO;
-      case 2:
-        return GET_CATALOGS;
-      case 3:
-        return GET_SCHEMAS;
-      case 4:
-        return GET_TABLES;
-      case 5:
-        return GET_TABLE_TYPES;
-      case 6:
-        return GET_COLUMNS;
-      case 7:
-        return GET_FUNCTIONS;
-      case 8:
-        return UNKNOWN;
-      default:
-        return null;
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TPrimitiveTypeEntry.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TPrimitiveTypeEntry.java
deleted file mode 100644
index 910c90967f614..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TPrimitiveTypeEntry.java
+++ /dev/null
@@ -1,516 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TPrimitiveTypeEntry implements org.apache.thrift.TBase<TPrimitiveTypeEntry, TPrimitiveTypeEntry._Fields>, java.io.Serializable, Cloneable, Comparable<TPrimitiveTypeEntry> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TPrimitiveTypeEntry");
-
-  private static final org.apache.thrift.protocol.TField TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("type", org.apache.thrift.protocol.TType.I32, (short)1);
-  private static final org.apache.thrift.protocol.TField TYPE_QUALIFIERS_FIELD_DESC = new org.apache.thrift.protocol.TField("typeQualifiers", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TPrimitiveTypeEntryStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TPrimitiveTypeEntryTupleSchemeFactory());
-  }
-
-  private TTypeId type; // required
-  private TTypeQualifiers typeQualifiers; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    /**
-     * 
-     * @see TTypeId
-     */
-    TYPE((short)1, "type"),
-    TYPE_QUALIFIERS((short)2, "typeQualifiers");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // TYPE
-          return TYPE;
-        case 2: // TYPE_QUALIFIERS
-          return TYPE_QUALIFIERS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final _Fields optionals[] = {_Fields.TYPE_QUALIFIERS};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.TYPE, new org.apache.thrift.meta_data.FieldMetaData("type", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TTypeId.class)));
-    tmpMap.put(_Fields.TYPE_QUALIFIERS, new org.apache.thrift.meta_data.FieldMetaData("typeQualifiers", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TTypeQualifiers.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TPrimitiveTypeEntry.class, metaDataMap);
-  }
-
-  public TPrimitiveTypeEntry() {
-  }
-
-  public TPrimitiveTypeEntry(
-    TTypeId type)
-  {
-    this();
-    this.type = type;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TPrimitiveTypeEntry(TPrimitiveTypeEntry other) {
-    if (other.isSetType()) {
-      this.type = other.type;
-    }
-    if (other.isSetTypeQualifiers()) {
-      this.typeQualifiers = new TTypeQualifiers(other.typeQualifiers);
-    }
-  }
-
-  public TPrimitiveTypeEntry deepCopy() {
-    return new TPrimitiveTypeEntry(this);
-  }
-
-  @Override
-  public void clear() {
-    this.type = null;
-    this.typeQualifiers = null;
-  }
-
-  /**
-   * 
-   * @see TTypeId
-   */
-  public TTypeId getType() {
-    return this.type;
-  }
-
-  /**
-   * 
-   * @see TTypeId
-   */
-  public void setType(TTypeId type) {
-    this.type = type;
-  }
-
-  public void unsetType() {
-    this.type = null;
-  }
-
-  /** Returns true if field type is set (has been assigned a value) and false otherwise */
-  public boolean isSetType() {
-    return this.type != null;
-  }
-
-  public void setTypeIsSet(boolean value) {
-    if (!value) {
-      this.type = null;
-    }
-  }
-
-  public TTypeQualifiers getTypeQualifiers() {
-    return this.typeQualifiers;
-  }
-
-  public void setTypeQualifiers(TTypeQualifiers typeQualifiers) {
-    this.typeQualifiers = typeQualifiers;
-  }
-
-  public void unsetTypeQualifiers() {
-    this.typeQualifiers = null;
-  }
-
-  /** Returns true if field typeQualifiers is set (has been assigned a value) and false otherwise */
-  public boolean isSetTypeQualifiers() {
-    return this.typeQualifiers != null;
-  }
-
-  public void setTypeQualifiersIsSet(boolean value) {
-    if (!value) {
-      this.typeQualifiers = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case TYPE:
-      if (value == null) {
-        unsetType();
-      } else {
-        setType((TTypeId)value);
-      }
-      break;
-
-    case TYPE_QUALIFIERS:
-      if (value == null) {
-        unsetTypeQualifiers();
-      } else {
-        setTypeQualifiers((TTypeQualifiers)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case TYPE:
-      return getType();
-
-    case TYPE_QUALIFIERS:
-      return getTypeQualifiers();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case TYPE:
-      return isSetType();
-    case TYPE_QUALIFIERS:
-      return isSetTypeQualifiers();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TPrimitiveTypeEntry)
-      return this.equals((TPrimitiveTypeEntry)that);
-    return false;
-  }
-
-  public boolean equals(TPrimitiveTypeEntry that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_type = true && this.isSetType();
-    boolean that_present_type = true && that.isSetType();
-    if (this_present_type || that_present_type) {
-      if (!(this_present_type && that_present_type))
-        return false;
-      if (!this.type.equals(that.type))
-        return false;
-    }
-
-    boolean this_present_typeQualifiers = true && this.isSetTypeQualifiers();
-    boolean that_present_typeQualifiers = true && that.isSetTypeQualifiers();
-    if (this_present_typeQualifiers || that_present_typeQualifiers) {
-      if (!(this_present_typeQualifiers && that_present_typeQualifiers))
-        return false;
-      if (!this.typeQualifiers.equals(that.typeQualifiers))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_type = true && (isSetType());
-    list.add(present_type);
-    if (present_type)
-      list.add(type.getValue());
-
-    boolean present_typeQualifiers = true && (isSetTypeQualifiers());
-    list.add(present_typeQualifiers);
-    if (present_typeQualifiers)
-      list.add(typeQualifiers);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TPrimitiveTypeEntry other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetType()).compareTo(other.isSetType());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetType()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.type, other.type);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetTypeQualifiers()).compareTo(other.isSetTypeQualifiers());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetTypeQualifiers()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.typeQualifiers, other.typeQualifiers);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TPrimitiveTypeEntry(");
-    boolean first = true;
-
-    sb.append("type:");
-    if (this.type == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.type);
-    }
-    first = false;
-    if (isSetTypeQualifiers()) {
-      if (!first) sb.append(", ");
-      sb.append("typeQualifiers:");
-      if (this.typeQualifiers == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.typeQualifiers);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetType()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'type' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (typeQualifiers != null) {
-      typeQualifiers.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TPrimitiveTypeEntryStandardSchemeFactory implements SchemeFactory {
-    public TPrimitiveTypeEntryStandardScheme getScheme() {
-      return new TPrimitiveTypeEntryStandardScheme();
-    }
-  }
-
-  private static class TPrimitiveTypeEntryStandardScheme extends StandardScheme<TPrimitiveTypeEntry> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TPrimitiveTypeEntry struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // TYPE
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.type = org.apache.hive.service.rpc.thrift.TTypeId.findByValue(iprot.readI32());
-              struct.setTypeIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // TYPE_QUALIFIERS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.typeQualifiers = new TTypeQualifiers();
-              struct.typeQualifiers.read(iprot);
-              struct.setTypeQualifiersIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TPrimitiveTypeEntry struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.type != null) {
-        oprot.writeFieldBegin(TYPE_FIELD_DESC);
-        oprot.writeI32(struct.type.getValue());
-        oprot.writeFieldEnd();
-      }
-      if (struct.typeQualifiers != null) {
-        if (struct.isSetTypeQualifiers()) {
-          oprot.writeFieldBegin(TYPE_QUALIFIERS_FIELD_DESC);
-          struct.typeQualifiers.write(oprot);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TPrimitiveTypeEntryTupleSchemeFactory implements SchemeFactory {
-    public TPrimitiveTypeEntryTupleScheme getScheme() {
-      return new TPrimitiveTypeEntryTupleScheme();
-    }
-  }
-
-  private static class TPrimitiveTypeEntryTupleScheme extends TupleScheme<TPrimitiveTypeEntry> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TPrimitiveTypeEntry struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      oprot.writeI32(struct.type.getValue());
-      BitSet optionals = new BitSet();
-      if (struct.isSetTypeQualifiers()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetTypeQualifiers()) {
-        struct.typeQualifiers.write(oprot);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TPrimitiveTypeEntry struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.type = org.apache.hive.service.rpc.thrift.TTypeId.findByValue(iprot.readI32());
-      struct.setTypeIsSet(true);
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.typeQualifiers = new TTypeQualifiers();
-        struct.typeQualifiers.read(iprot);
-        struct.setTypeQualifiersIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TProgressUpdateResp.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TProgressUpdateResp.java
deleted file mode 100644
index ecc413aad4cdc..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TProgressUpdateResp.java
+++ /dev/null
@@ -1,1033 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TProgressUpdateResp implements org.apache.thrift.TBase<TProgressUpdateResp, TProgressUpdateResp._Fields>, java.io.Serializable, Cloneable, Comparable<TProgressUpdateResp> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TProgressUpdateResp");
-
-  private static final org.apache.thrift.protocol.TField HEADER_NAMES_FIELD_DESC = new org.apache.thrift.protocol.TField("headerNames", org.apache.thrift.protocol.TType.LIST, (short)1);
-  private static final org.apache.thrift.protocol.TField ROWS_FIELD_DESC = new org.apache.thrift.protocol.TField("rows", org.apache.thrift.protocol.TType.LIST, (short)2);
-  private static final org.apache.thrift.protocol.TField PROGRESSED_PERCENTAGE_FIELD_DESC = new org.apache.thrift.protocol.TField("progressedPercentage", org.apache.thrift.protocol.TType.DOUBLE, (short)3);
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.I32, (short)4);
-  private static final org.apache.thrift.protocol.TField FOOTER_SUMMARY_FIELD_DESC = new org.apache.thrift.protocol.TField("footerSummary", org.apache.thrift.protocol.TType.STRING, (short)5);
-  private static final org.apache.thrift.protocol.TField START_TIME_FIELD_DESC = new org.apache.thrift.protocol.TField("startTime", org.apache.thrift.protocol.TType.I64, (short)6);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TProgressUpdateRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TProgressUpdateRespTupleSchemeFactory());
-  }
-
-  private List<String> headerNames; // required
-  private List<List<String>> rows; // required
-  private double progressedPercentage; // required
-  private TJobExecutionStatus status; // required
-  private String footerSummary; // required
-  private long startTime; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    HEADER_NAMES((short)1, "headerNames"),
-    ROWS((short)2, "rows"),
-    PROGRESSED_PERCENTAGE((short)3, "progressedPercentage"),
-    /**
-     * 
-     * @see TJobExecutionStatus
-     */
-    STATUS((short)4, "status"),
-    FOOTER_SUMMARY((short)5, "footerSummary"),
-    START_TIME((short)6, "startTime");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // HEADER_NAMES
-          return HEADER_NAMES;
-        case 2: // ROWS
-          return ROWS;
-        case 3: // PROGRESSED_PERCENTAGE
-          return PROGRESSED_PERCENTAGE;
-        case 4: // STATUS
-          return STATUS;
-        case 5: // FOOTER_SUMMARY
-          return FOOTER_SUMMARY;
-        case 6: // START_TIME
-          return START_TIME;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __PROGRESSEDPERCENTAGE_ISSET_ID = 0;
-  private static final int __STARTTIME_ISSET_ID = 1;
-  private byte __isset_bitfield = 0;
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.HEADER_NAMES, new org.apache.thrift.meta_data.FieldMetaData("headerNames", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
-    tmpMap.put(_Fields.ROWS, new org.apache.thrift.meta_data.FieldMetaData("rows", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-                new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))));
-    tmpMap.put(_Fields.PROGRESSED_PERCENTAGE, new org.apache.thrift.meta_data.FieldMetaData("progressedPercentage", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE)));
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TJobExecutionStatus.class)));
-    tmpMap.put(_Fields.FOOTER_SUMMARY, new org.apache.thrift.meta_data.FieldMetaData("footerSummary", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    tmpMap.put(_Fields.START_TIME, new org.apache.thrift.meta_data.FieldMetaData("startTime", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TProgressUpdateResp.class, metaDataMap);
-  }
-
-  public TProgressUpdateResp() {
-  }
-
-  public TProgressUpdateResp(
-    List<String> headerNames,
-    List<List<String>> rows,
-    double progressedPercentage,
-    TJobExecutionStatus status,
-    String footerSummary,
-    long startTime)
-  {
-    this();
-    this.headerNames = headerNames;
-    this.rows = rows;
-    this.progressedPercentage = progressedPercentage;
-    setProgressedPercentageIsSet(true);
-    this.status = status;
-    this.footerSummary = footerSummary;
-    this.startTime = startTime;
-    setStartTimeIsSet(true);
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TProgressUpdateResp(TProgressUpdateResp other) {
-    __isset_bitfield = other.__isset_bitfield;
-    if (other.isSetHeaderNames()) {
-      List<String> __this__headerNames = new ArrayList<String>(other.headerNames);
-      this.headerNames = __this__headerNames;
-    }
-    if (other.isSetRows()) {
-      List<List<String>> __this__rows = new ArrayList<List<String>>(other.rows.size());
-      for (List<String> other_element : other.rows) {
-        List<String> __this__rows_copy = new ArrayList<String>(other_element);
-        __this__rows.add(__this__rows_copy);
-      }
-      this.rows = __this__rows;
-    }
-    this.progressedPercentage = other.progressedPercentage;
-    if (other.isSetStatus()) {
-      this.status = other.status;
-    }
-    if (other.isSetFooterSummary()) {
-      this.footerSummary = other.footerSummary;
-    }
-    this.startTime = other.startTime;
-  }
-
-  public TProgressUpdateResp deepCopy() {
-    return new TProgressUpdateResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.headerNames = null;
-    this.rows = null;
-    setProgressedPercentageIsSet(false);
-    this.progressedPercentage = 0.0;
-    this.status = null;
-    this.footerSummary = null;
-    setStartTimeIsSet(false);
-    this.startTime = 0;
-  }
-
-  public int getHeaderNamesSize() {
-    return (this.headerNames == null) ? 0 : this.headerNames.size();
-  }
-
-  public java.util.Iterator<String> getHeaderNamesIterator() {
-    return (this.headerNames == null) ? null : this.headerNames.iterator();
-  }
-
-  public void addToHeaderNames(String elem) {
-    if (this.headerNames == null) {
-      this.headerNames = new ArrayList<String>();
-    }
-    this.headerNames.add(elem);
-  }
-
-  public List<String> getHeaderNames() {
-    return this.headerNames;
-  }
-
-  public void setHeaderNames(List<String> headerNames) {
-    this.headerNames = headerNames;
-  }
-
-  public void unsetHeaderNames() {
-    this.headerNames = null;
-  }
-
-  /** Returns true if field headerNames is set (has been assigned a value) and false otherwise */
-  public boolean isSetHeaderNames() {
-    return this.headerNames != null;
-  }
-
-  public void setHeaderNamesIsSet(boolean value) {
-    if (!value) {
-      this.headerNames = null;
-    }
-  }
-
-  public int getRowsSize() {
-    return (this.rows == null) ? 0 : this.rows.size();
-  }
-
-  public java.util.Iterator<List<String>> getRowsIterator() {
-    return (this.rows == null) ? null : this.rows.iterator();
-  }
-
-  public void addToRows(List<String> elem) {
-    if (this.rows == null) {
-      this.rows = new ArrayList<List<String>>();
-    }
-    this.rows.add(elem);
-  }
-
-  public List<List<String>> getRows() {
-    return this.rows;
-  }
-
-  public void setRows(List<List<String>> rows) {
-    this.rows = rows;
-  }
-
-  public void unsetRows() {
-    this.rows = null;
-  }
-
-  /** Returns true if field rows is set (has been assigned a value) and false otherwise */
-  public boolean isSetRows() {
-    return this.rows != null;
-  }
-
-  public void setRowsIsSet(boolean value) {
-    if (!value) {
-      this.rows = null;
-    }
-  }
-
-  public double getProgressedPercentage() {
-    return this.progressedPercentage;
-  }
-
-  public void setProgressedPercentage(double progressedPercentage) {
-    this.progressedPercentage = progressedPercentage;
-    setProgressedPercentageIsSet(true);
-  }
-
-  public void unsetProgressedPercentage() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __PROGRESSEDPERCENTAGE_ISSET_ID);
-  }
-
-  /** Returns true if field progressedPercentage is set (has been assigned a value) and false otherwise */
-  public boolean isSetProgressedPercentage() {
-    return EncodingUtils.testBit(__isset_bitfield, __PROGRESSEDPERCENTAGE_ISSET_ID);
-  }
-
-  public void setProgressedPercentageIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __PROGRESSEDPERCENTAGE_ISSET_ID, value);
-  }
-
-  /**
-   * 
-   * @see TJobExecutionStatus
-   */
-  public TJobExecutionStatus getStatus() {
-    return this.status;
-  }
-
-  /**
-   * 
-   * @see TJobExecutionStatus
-   */
-  public void setStatus(TJobExecutionStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public String getFooterSummary() {
-    return this.footerSummary;
-  }
-
-  public void setFooterSummary(String footerSummary) {
-    this.footerSummary = footerSummary;
-  }
-
-  public void unsetFooterSummary() {
-    this.footerSummary = null;
-  }
-
-  /** Returns true if field footerSummary is set (has been assigned a value) and false otherwise */
-  public boolean isSetFooterSummary() {
-    return this.footerSummary != null;
-  }
-
-  public void setFooterSummaryIsSet(boolean value) {
-    if (!value) {
-      this.footerSummary = null;
-    }
-  }
-
-  public long getStartTime() {
-    return this.startTime;
-  }
-
-  public void setStartTime(long startTime) {
-    this.startTime = startTime;
-    setStartTimeIsSet(true);
-  }
-
-  public void unsetStartTime() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __STARTTIME_ISSET_ID);
-  }
-
-  /** Returns true if field startTime is set (has been assigned a value) and false otherwise */
-  public boolean isSetStartTime() {
-    return EncodingUtils.testBit(__isset_bitfield, __STARTTIME_ISSET_ID);
-  }
-
-  public void setStartTimeIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __STARTTIME_ISSET_ID, value);
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case HEADER_NAMES:
-      if (value == null) {
-        unsetHeaderNames();
-      } else {
-        setHeaderNames((List<String>)value);
-      }
-      break;
-
-    case ROWS:
-      if (value == null) {
-        unsetRows();
-      } else {
-        setRows((List<List<String>>)value);
-      }
-      break;
-
-    case PROGRESSED_PERCENTAGE:
-      if (value == null) {
-        unsetProgressedPercentage();
-      } else {
-        setProgressedPercentage((Double)value);
-      }
-      break;
-
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TJobExecutionStatus)value);
-      }
-      break;
-
-    case FOOTER_SUMMARY:
-      if (value == null) {
-        unsetFooterSummary();
-      } else {
-        setFooterSummary((String)value);
-      }
-      break;
-
-    case START_TIME:
-      if (value == null) {
-        unsetStartTime();
-      } else {
-        setStartTime((Long)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case HEADER_NAMES:
-      return getHeaderNames();
-
-    case ROWS:
-      return getRows();
-
-    case PROGRESSED_PERCENTAGE:
-      return getProgressedPercentage();
-
-    case STATUS:
-      return getStatus();
-
-    case FOOTER_SUMMARY:
-      return getFooterSummary();
-
-    case START_TIME:
-      return getStartTime();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case HEADER_NAMES:
-      return isSetHeaderNames();
-    case ROWS:
-      return isSetRows();
-    case PROGRESSED_PERCENTAGE:
-      return isSetProgressedPercentage();
-    case STATUS:
-      return isSetStatus();
-    case FOOTER_SUMMARY:
-      return isSetFooterSummary();
-    case START_TIME:
-      return isSetStartTime();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TProgressUpdateResp)
-      return this.equals((TProgressUpdateResp)that);
-    return false;
-  }
-
-  public boolean equals(TProgressUpdateResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_headerNames = true && this.isSetHeaderNames();
-    boolean that_present_headerNames = true && that.isSetHeaderNames();
-    if (this_present_headerNames || that_present_headerNames) {
-      if (!(this_present_headerNames && that_present_headerNames))
-        return false;
-      if (!this.headerNames.equals(that.headerNames))
-        return false;
-    }
-
-    boolean this_present_rows = true && this.isSetRows();
-    boolean that_present_rows = true && that.isSetRows();
-    if (this_present_rows || that_present_rows) {
-      if (!(this_present_rows && that_present_rows))
-        return false;
-      if (!this.rows.equals(that.rows))
-        return false;
-    }
-
-    boolean this_present_progressedPercentage = true;
-    boolean that_present_progressedPercentage = true;
-    if (this_present_progressedPercentage || that_present_progressedPercentage) {
-      if (!(this_present_progressedPercentage && that_present_progressedPercentage))
-        return false;
-      if (this.progressedPercentage != that.progressedPercentage)
-        return false;
-    }
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    boolean this_present_footerSummary = true && this.isSetFooterSummary();
-    boolean that_present_footerSummary = true && that.isSetFooterSummary();
-    if (this_present_footerSummary || that_present_footerSummary) {
-      if (!(this_present_footerSummary && that_present_footerSummary))
-        return false;
-      if (!this.footerSummary.equals(that.footerSummary))
-        return false;
-    }
-
-    boolean this_present_startTime = true;
-    boolean that_present_startTime = true;
-    if (this_present_startTime || that_present_startTime) {
-      if (!(this_present_startTime && that_present_startTime))
-        return false;
-      if (this.startTime != that.startTime)
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_headerNames = true && (isSetHeaderNames());
-    list.add(present_headerNames);
-    if (present_headerNames)
-      list.add(headerNames);
-
-    boolean present_rows = true && (isSetRows());
-    list.add(present_rows);
-    if (present_rows)
-      list.add(rows);
-
-    boolean present_progressedPercentage = true;
-    list.add(present_progressedPercentage);
-    if (present_progressedPercentage)
-      list.add(progressedPercentage);
-
-    boolean present_status = true && (isSetStatus());
-    list.add(present_status);
-    if (present_status)
-      list.add(status.getValue());
-
-    boolean present_footerSummary = true && (isSetFooterSummary());
-    list.add(present_footerSummary);
-    if (present_footerSummary)
-      list.add(footerSummary);
-
-    boolean present_startTime = true;
-    list.add(present_startTime);
-    if (present_startTime)
-      list.add(startTime);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TProgressUpdateResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetHeaderNames()).compareTo(other.isSetHeaderNames());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetHeaderNames()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.headerNames, other.headerNames);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetRows()).compareTo(other.isSetRows());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetRows()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.rows, other.rows);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetProgressedPercentage()).compareTo(other.isSetProgressedPercentage());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetProgressedPercentage()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.progressedPercentage, other.progressedPercentage);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(other.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, other.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetFooterSummary()).compareTo(other.isSetFooterSummary());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetFooterSummary()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.footerSummary, other.footerSummary);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetStartTime()).compareTo(other.isSetStartTime());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStartTime()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.startTime, other.startTime);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TProgressUpdateResp(");
-    boolean first = true;
-
-    sb.append("headerNames:");
-    if (this.headerNames == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.headerNames);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("rows:");
-    if (this.rows == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.rows);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("progressedPercentage:");
-    sb.append(this.progressedPercentage);
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("footerSummary:");
-    if (this.footerSummary == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.footerSummary);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("startTime:");
-    sb.append(this.startTime);
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetHeaderNames()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'headerNames' is unset! Struct:" + toString());
-    }
-
-    if (!isSetRows()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'rows' is unset! Struct:" + toString());
-    }
-
-    if (!isSetProgressedPercentage()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'progressedPercentage' is unset! Struct:" + toString());
-    }
-
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    if (!isSetFooterSummary()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'footerSummary' is unset! Struct:" + toString());
-    }
-
-    if (!isSetStartTime()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'startTime' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TProgressUpdateRespStandardSchemeFactory implements SchemeFactory {
-    public TProgressUpdateRespStandardScheme getScheme() {
-      return new TProgressUpdateRespStandardScheme();
-    }
-  }
-
-  private static class TProgressUpdateRespStandardScheme extends StandardScheme<TProgressUpdateResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TProgressUpdateResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // HEADER_NAMES
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list180 = iprot.readListBegin();
-                struct.headerNames = new ArrayList<String>(_list180.size);
-                String _elem181;
-                for (int _i182 = 0; _i182 < _list180.size; ++_i182)
-                {
-                  _elem181 = iprot.readString();
-                  struct.headerNames.add(_elem181);
-                }
-                iprot.readListEnd();
-              }
-              struct.setHeaderNamesIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // ROWS
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list183 = iprot.readListBegin();
-                struct.rows = new ArrayList<List<String>>(_list183.size);
-                List<String> _elem184;
-                for (int _i185 = 0; _i185 < _list183.size; ++_i185)
-                {
-                  {
-                    org.apache.thrift.protocol.TList _list186 = iprot.readListBegin();
-                    _elem184 = new ArrayList<String>(_list186.size);
-                    String _elem187;
-                    for (int _i188 = 0; _i188 < _list186.size; ++_i188)
-                    {
-                      _elem187 = iprot.readString();
-                      _elem184.add(_elem187);
-                    }
-                    iprot.readListEnd();
-                  }
-                  struct.rows.add(_elem184);
-                }
-                iprot.readListEnd();
-              }
-              struct.setRowsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // PROGRESSED_PERCENTAGE
-            if (schemeField.type == org.apache.thrift.protocol.TType.DOUBLE) {
-              struct.progressedPercentage = iprot.readDouble();
-              struct.setProgressedPercentageIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.status = org.apache.hive.service.rpc.thrift.TJobExecutionStatus.findByValue(iprot.readI32());
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 5: // FOOTER_SUMMARY
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.footerSummary = iprot.readString();
-              struct.setFooterSummaryIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 6: // START_TIME
-            if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
-              struct.startTime = iprot.readI64();
-              struct.setStartTimeIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TProgressUpdateResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.headerNames != null) {
-        oprot.writeFieldBegin(HEADER_NAMES_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.headerNames.size()));
-          for (String _iter189 : struct.headerNames)
-          {
-            oprot.writeString(_iter189);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      if (struct.rows != null) {
-        oprot.writeFieldBegin(ROWS_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.LIST, struct.rows.size()));
-          for (List<String> _iter190 : struct.rows)
-          {
-            {
-              oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, _iter190.size()));
-              for (String _iter191 : _iter190)
-              {
-                oprot.writeString(_iter191);
-              }
-              oprot.writeListEnd();
-            }
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldBegin(PROGRESSED_PERCENTAGE_FIELD_DESC);
-      oprot.writeDouble(struct.progressedPercentage);
-      oprot.writeFieldEnd();
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        oprot.writeI32(struct.status.getValue());
-        oprot.writeFieldEnd();
-      }
-      if (struct.footerSummary != null) {
-        oprot.writeFieldBegin(FOOTER_SUMMARY_FIELD_DESC);
-        oprot.writeString(struct.footerSummary);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldBegin(START_TIME_FIELD_DESC);
-      oprot.writeI64(struct.startTime);
-      oprot.writeFieldEnd();
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TProgressUpdateRespTupleSchemeFactory implements SchemeFactory {
-    public TProgressUpdateRespTupleScheme getScheme() {
-      return new TProgressUpdateRespTupleScheme();
-    }
-  }
-
-  private static class TProgressUpdateRespTupleScheme extends TupleScheme<TProgressUpdateResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TProgressUpdateResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.headerNames.size());
-        for (String _iter192 : struct.headerNames)
-        {
-          oprot.writeString(_iter192);
-        }
-      }
-      {
-        oprot.writeI32(struct.rows.size());
-        for (List<String> _iter193 : struct.rows)
-        {
-          {
-            oprot.writeI32(_iter193.size());
-            for (String _iter194 : _iter193)
-            {
-              oprot.writeString(_iter194);
-            }
-          }
-        }
-      }
-      oprot.writeDouble(struct.progressedPercentage);
-      oprot.writeI32(struct.status.getValue());
-      oprot.writeString(struct.footerSummary);
-      oprot.writeI64(struct.startTime);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TProgressUpdateResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TList _list195 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-        struct.headerNames = new ArrayList<String>(_list195.size);
-        String _elem196;
-        for (int _i197 = 0; _i197 < _list195.size; ++_i197)
-        {
-          _elem196 = iprot.readString();
-          struct.headerNames.add(_elem196);
-        }
-      }
-      struct.setHeaderNamesIsSet(true);
-      {
-        org.apache.thrift.protocol.TList _list198 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.LIST, iprot.readI32());
-        struct.rows = new ArrayList<List<String>>(_list198.size);
-        List<String> _elem199;
-        for (int _i200 = 0; _i200 < _list198.size; ++_i200)
-        {
-          {
-            org.apache.thrift.protocol.TList _list201 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-            _elem199 = new ArrayList<String>(_list201.size);
-            String _elem202;
-            for (int _i203 = 0; _i203 < _list201.size; ++_i203)
-            {
-              _elem202 = iprot.readString();
-              _elem199.add(_elem202);
-            }
-          }
-          struct.rows.add(_elem199);
-        }
-      }
-      struct.setRowsIsSet(true);
-      struct.progressedPercentage = iprot.readDouble();
-      struct.setProgressedPercentageIsSet(true);
-      struct.status = org.apache.hive.service.rpc.thrift.TJobExecutionStatus.findByValue(iprot.readI32());
-      struct.setStatusIsSet(true);
-      struct.footerSummary = iprot.readString();
-      struct.setFooterSummaryIsSet(true);
-      struct.startTime = iprot.readI64();
-      struct.setStartTimeIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TProtocolVersion.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TProtocolVersion.java
deleted file mode 100644
index 18a782513c500..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TProtocolVersion.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.thrift.TEnum;
-
-public enum TProtocolVersion implements org.apache.thrift.TEnum {
-  HIVE_CLI_SERVICE_PROTOCOL_V1(0),
-  HIVE_CLI_SERVICE_PROTOCOL_V2(1),
-  HIVE_CLI_SERVICE_PROTOCOL_V3(2),
-  HIVE_CLI_SERVICE_PROTOCOL_V4(3),
-  HIVE_CLI_SERVICE_PROTOCOL_V5(4),
-  HIVE_CLI_SERVICE_PROTOCOL_V6(5),
-  HIVE_CLI_SERVICE_PROTOCOL_V7(6),
-  HIVE_CLI_SERVICE_PROTOCOL_V8(7),
-  HIVE_CLI_SERVICE_PROTOCOL_V9(8),
-  HIVE_CLI_SERVICE_PROTOCOL_V10(9);
-
-  private final int value;
-
-  private TProtocolVersion(int value) {
-    this.value = value;
-  }
-
-  /**
-   * Get the integer value of this enum value, as defined in the Thrift IDL.
-   */
-  public int getValue() {
-    return value;
-  }
-
-  /**
-   * Find a the enum type by its integer value, as defined in the Thrift IDL.
-   * @return null if the value is not found.
-   */
-  public static TProtocolVersion findByValue(int value) { 
-    switch (value) {
-      case 0:
-        return HIVE_CLI_SERVICE_PROTOCOL_V1;
-      case 1:
-        return HIVE_CLI_SERVICE_PROTOCOL_V2;
-      case 2:
-        return HIVE_CLI_SERVICE_PROTOCOL_V3;
-      case 3:
-        return HIVE_CLI_SERVICE_PROTOCOL_V4;
-      case 4:
-        return HIVE_CLI_SERVICE_PROTOCOL_V5;
-      case 5:
-        return HIVE_CLI_SERVICE_PROTOCOL_V6;
-      case 6:
-        return HIVE_CLI_SERVICE_PROTOCOL_V7;
-      case 7:
-        return HIVE_CLI_SERVICE_PROTOCOL_V8;
-      case 8:
-        return HIVE_CLI_SERVICE_PROTOCOL_V9;
-      case 9:
-        return HIVE_CLI_SERVICE_PROTOCOL_V10;
-      default:
-        return null;
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TRenewDelegationTokenReq.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TRenewDelegationTokenReq.java
deleted file mode 100644
index 8957ebc8d2fff..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TRenewDelegationTokenReq.java
+++ /dev/null
@@ -1,495 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TRenewDelegationTokenReq implements org.apache.thrift.TBase<TRenewDelegationTokenReq, TRenewDelegationTokenReq._Fields>, java.io.Serializable, Cloneable, Comparable<TRenewDelegationTokenReq> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TRenewDelegationTokenReq");
-
-  private static final org.apache.thrift.protocol.TField SESSION_HANDLE_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionHandle", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField DELEGATION_TOKEN_FIELD_DESC = new org.apache.thrift.protocol.TField("delegationToken", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TRenewDelegationTokenReqStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TRenewDelegationTokenReqTupleSchemeFactory());
-  }
-
-  private TSessionHandle sessionHandle; // required
-  private String delegationToken; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_HANDLE((short)1, "sessionHandle"),
-    DELEGATION_TOKEN((short)2, "delegationToken");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_HANDLE
-          return SESSION_HANDLE;
-        case 2: // DELEGATION_TOKEN
-          return DELEGATION_TOKEN;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_HANDLE, new org.apache.thrift.meta_data.FieldMetaData("sessionHandle", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TSessionHandle.class)));
-    tmpMap.put(_Fields.DELEGATION_TOKEN, new org.apache.thrift.meta_data.FieldMetaData("delegationToken", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TRenewDelegationTokenReq.class, metaDataMap);
-  }
-
-  public TRenewDelegationTokenReq() {
-  }
-
-  public TRenewDelegationTokenReq(
-    TSessionHandle sessionHandle,
-    String delegationToken)
-  {
-    this();
-    this.sessionHandle = sessionHandle;
-    this.delegationToken = delegationToken;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TRenewDelegationTokenReq(TRenewDelegationTokenReq other) {
-    if (other.isSetSessionHandle()) {
-      this.sessionHandle = new TSessionHandle(other.sessionHandle);
-    }
-    if (other.isSetDelegationToken()) {
-      this.delegationToken = other.delegationToken;
-    }
-  }
-
-  public TRenewDelegationTokenReq deepCopy() {
-    return new TRenewDelegationTokenReq(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionHandle = null;
-    this.delegationToken = null;
-  }
-
-  public TSessionHandle getSessionHandle() {
-    return this.sessionHandle;
-  }
-
-  public void setSessionHandle(TSessionHandle sessionHandle) {
-    this.sessionHandle = sessionHandle;
-  }
-
-  public void unsetSessionHandle() {
-    this.sessionHandle = null;
-  }
-
-  /** Returns true if field sessionHandle is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionHandle() {
-    return this.sessionHandle != null;
-  }
-
-  public void setSessionHandleIsSet(boolean value) {
-    if (!value) {
-      this.sessionHandle = null;
-    }
-  }
-
-  public String getDelegationToken() {
-    return this.delegationToken;
-  }
-
-  public void setDelegationToken(String delegationToken) {
-    this.delegationToken = delegationToken;
-  }
-
-  public void unsetDelegationToken() {
-    this.delegationToken = null;
-  }
-
-  /** Returns true if field delegationToken is set (has been assigned a value) and false otherwise */
-  public boolean isSetDelegationToken() {
-    return this.delegationToken != null;
-  }
-
-  public void setDelegationTokenIsSet(boolean value) {
-    if (!value) {
-      this.delegationToken = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_HANDLE:
-      if (value == null) {
-        unsetSessionHandle();
-      } else {
-        setSessionHandle((TSessionHandle)value);
-      }
-      break;
-
-    case DELEGATION_TOKEN:
-      if (value == null) {
-        unsetDelegationToken();
-      } else {
-        setDelegationToken((String)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_HANDLE:
-      return getSessionHandle();
-
-    case DELEGATION_TOKEN:
-      return getDelegationToken();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_HANDLE:
-      return isSetSessionHandle();
-    case DELEGATION_TOKEN:
-      return isSetDelegationToken();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TRenewDelegationTokenReq)
-      return this.equals((TRenewDelegationTokenReq)that);
-    return false;
-  }
-
-  public boolean equals(TRenewDelegationTokenReq that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionHandle = true && this.isSetSessionHandle();
-    boolean that_present_sessionHandle = true && that.isSetSessionHandle();
-    if (this_present_sessionHandle || that_present_sessionHandle) {
-      if (!(this_present_sessionHandle && that_present_sessionHandle))
-        return false;
-      if (!this.sessionHandle.equals(that.sessionHandle))
-        return false;
-    }
-
-    boolean this_present_delegationToken = true && this.isSetDelegationToken();
-    boolean that_present_delegationToken = true && that.isSetDelegationToken();
-    if (this_present_delegationToken || that_present_delegationToken) {
-      if (!(this_present_delegationToken && that_present_delegationToken))
-        return false;
-      if (!this.delegationToken.equals(that.delegationToken))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_sessionHandle = true && (isSetSessionHandle());
-    list.add(present_sessionHandle);
-    if (present_sessionHandle)
-      list.add(sessionHandle);
-
-    boolean present_delegationToken = true && (isSetDelegationToken());
-    list.add(present_delegationToken);
-    if (present_delegationToken)
-      list.add(delegationToken);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TRenewDelegationTokenReq other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetSessionHandle()).compareTo(other.isSetSessionHandle());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionHandle()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionHandle, other.sessionHandle);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetDelegationToken()).compareTo(other.isSetDelegationToken());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetDelegationToken()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.delegationToken, other.delegationToken);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TRenewDelegationTokenReq(");
-    boolean first = true;
-
-    sb.append("sessionHandle:");
-    if (this.sessionHandle == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionHandle);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("delegationToken:");
-    if (this.delegationToken == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.delegationToken);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionHandle()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionHandle' is unset! Struct:" + toString());
-    }
-
-    if (!isSetDelegationToken()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'delegationToken' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionHandle != null) {
-      sessionHandle.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TRenewDelegationTokenReqStandardSchemeFactory implements SchemeFactory {
-    public TRenewDelegationTokenReqStandardScheme getScheme() {
-      return new TRenewDelegationTokenReqStandardScheme();
-    }
-  }
-
-  private static class TRenewDelegationTokenReqStandardScheme extends StandardScheme<TRenewDelegationTokenReq> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TRenewDelegationTokenReq struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_HANDLE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionHandle = new TSessionHandle();
-              struct.sessionHandle.read(iprot);
-              struct.setSessionHandleIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // DELEGATION_TOKEN
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.delegationToken = iprot.readString();
-              struct.setDelegationTokenIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TRenewDelegationTokenReq struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionHandle != null) {
-        oprot.writeFieldBegin(SESSION_HANDLE_FIELD_DESC);
-        struct.sessionHandle.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      if (struct.delegationToken != null) {
-        oprot.writeFieldBegin(DELEGATION_TOKEN_FIELD_DESC);
-        oprot.writeString(struct.delegationToken);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TRenewDelegationTokenReqTupleSchemeFactory implements SchemeFactory {
-    public TRenewDelegationTokenReqTupleScheme getScheme() {
-      return new TRenewDelegationTokenReqTupleScheme();
-    }
-  }
-
-  private static class TRenewDelegationTokenReqTupleScheme extends TupleScheme<TRenewDelegationTokenReq> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TRenewDelegationTokenReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionHandle.write(oprot);
-      oprot.writeString(struct.delegationToken);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TRenewDelegationTokenReq struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionHandle = new TSessionHandle();
-      struct.sessionHandle.read(iprot);
-      struct.setSessionHandleIsSet(true);
-      struct.delegationToken = iprot.readString();
-      struct.setDelegationTokenIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TRenewDelegationTokenResp.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TRenewDelegationTokenResp.java
deleted file mode 100644
index 6f5004ccc38e4..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TRenewDelegationTokenResp.java
+++ /dev/null
@@ -1,394 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TRenewDelegationTokenResp implements org.apache.thrift.TBase<TRenewDelegationTokenResp, TRenewDelegationTokenResp._Fields>, java.io.Serializable, Cloneable, Comparable<TRenewDelegationTokenResp> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TRenewDelegationTokenResp");
-
-  private static final org.apache.thrift.protocol.TField STATUS_FIELD_DESC = new org.apache.thrift.protocol.TField("status", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TRenewDelegationTokenRespStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TRenewDelegationTokenRespTupleSchemeFactory());
-  }
-
-  private TStatus status; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    STATUS((short)1, "status");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS
-          return STATUS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS, new org.apache.thrift.meta_data.FieldMetaData("status", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStatus.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TRenewDelegationTokenResp.class, metaDataMap);
-  }
-
-  public TRenewDelegationTokenResp() {
-  }
-
-  public TRenewDelegationTokenResp(
-    TStatus status)
-  {
-    this();
-    this.status = status;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TRenewDelegationTokenResp(TRenewDelegationTokenResp other) {
-    if (other.isSetStatus()) {
-      this.status = new TStatus(other.status);
-    }
-  }
-
-  public TRenewDelegationTokenResp deepCopy() {
-    return new TRenewDelegationTokenResp(this);
-  }
-
-  @Override
-  public void clear() {
-    this.status = null;
-  }
-
-  public TStatus getStatus() {
-    return this.status;
-  }
-
-  public void setStatus(TStatus status) {
-    this.status = status;
-  }
-
-  public void unsetStatus() {
-    this.status = null;
-  }
-
-  /** Returns true if field status is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatus() {
-    return this.status != null;
-  }
-
-  public void setStatusIsSet(boolean value) {
-    if (!value) {
-      this.status = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS:
-      if (value == null) {
-        unsetStatus();
-      } else {
-        setStatus((TStatus)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS:
-      return getStatus();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS:
-      return isSetStatus();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TRenewDelegationTokenResp)
-      return this.equals((TRenewDelegationTokenResp)that);
-    return false;
-  }
-
-  public boolean equals(TRenewDelegationTokenResp that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_status = true && this.isSetStatus();
-    boolean that_present_status = true && that.isSetStatus();
-    if (this_present_status || that_present_status) {
-      if (!(this_present_status && that_present_status))
-        return false;
-      if (!this.status.equals(that.status))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_status = true && (isSetStatus());
-    list.add(present_status);
-    if (present_status)
-      list.add(status);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TRenewDelegationTokenResp other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetStatus()).compareTo(other.isSetStatus());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatus()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.status, other.status);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TRenewDelegationTokenResp(");
-    boolean first = true;
-
-    sb.append("status:");
-    if (this.status == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.status);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatus()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'status' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (status != null) {
-      status.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TRenewDelegationTokenRespStandardSchemeFactory implements SchemeFactory {
-    public TRenewDelegationTokenRespStandardScheme getScheme() {
-      return new TRenewDelegationTokenRespStandardScheme();
-    }
-  }
-
-  private static class TRenewDelegationTokenRespStandardScheme extends StandardScheme<TRenewDelegationTokenResp> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TRenewDelegationTokenResp struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.status = new TStatus();
-              struct.status.read(iprot);
-              struct.setStatusIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TRenewDelegationTokenResp struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.status != null) {
-        oprot.writeFieldBegin(STATUS_FIELD_DESC);
-        struct.status.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TRenewDelegationTokenRespTupleSchemeFactory implements SchemeFactory {
-    public TRenewDelegationTokenRespTupleScheme getScheme() {
-      return new TRenewDelegationTokenRespTupleScheme();
-    }
-  }
-
-  private static class TRenewDelegationTokenRespTupleScheme extends TupleScheme<TRenewDelegationTokenResp> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TRenewDelegationTokenResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.status.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TRenewDelegationTokenResp struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.status = new TStatus();
-      struct.status.read(iprot);
-      struct.setStatusIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TRow.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TRow.java
deleted file mode 100644
index e95299df97c3a..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TRow.java
+++ /dev/null
@@ -1,443 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TRow implements org.apache.thrift.TBase<TRow, TRow._Fields>, java.io.Serializable, Cloneable, Comparable<TRow> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TRow");
-
-  private static final org.apache.thrift.protocol.TField COL_VALS_FIELD_DESC = new org.apache.thrift.protocol.TField("colVals", org.apache.thrift.protocol.TType.LIST, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TRowStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TRowTupleSchemeFactory());
-  }
-
-  private List<TColumnValue> colVals; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    COL_VALS((short)1, "colVals");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // COL_VALS
-          return COL_VALS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.COL_VALS, new org.apache.thrift.meta_data.FieldMetaData("colVals", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TColumnValue.class))));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TRow.class, metaDataMap);
-  }
-
-  public TRow() {
-  }
-
-  public TRow(
-    List<TColumnValue> colVals)
-  {
-    this();
-    this.colVals = colVals;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TRow(TRow other) {
-    if (other.isSetColVals()) {
-      List<TColumnValue> __this__colVals = new ArrayList<TColumnValue>(other.colVals.size());
-      for (TColumnValue other_element : other.colVals) {
-        __this__colVals.add(new TColumnValue(other_element));
-      }
-      this.colVals = __this__colVals;
-    }
-  }
-
-  public TRow deepCopy() {
-    return new TRow(this);
-  }
-
-  @Override
-  public void clear() {
-    this.colVals = null;
-  }
-
-  public int getColValsSize() {
-    return (this.colVals == null) ? 0 : this.colVals.size();
-  }
-
-  public java.util.Iterator<TColumnValue> getColValsIterator() {
-    return (this.colVals == null) ? null : this.colVals.iterator();
-  }
-
-  public void addToColVals(TColumnValue elem) {
-    if (this.colVals == null) {
-      this.colVals = new ArrayList<TColumnValue>();
-    }
-    this.colVals.add(elem);
-  }
-
-  public List<TColumnValue> getColVals() {
-    return this.colVals;
-  }
-
-  public void setColVals(List<TColumnValue> colVals) {
-    this.colVals = colVals;
-  }
-
-  public void unsetColVals() {
-    this.colVals = null;
-  }
-
-  /** Returns true if field colVals is set (has been assigned a value) and false otherwise */
-  public boolean isSetColVals() {
-    return this.colVals != null;
-  }
-
-  public void setColValsIsSet(boolean value) {
-    if (!value) {
-      this.colVals = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case COL_VALS:
-      if (value == null) {
-        unsetColVals();
-      } else {
-        setColVals((List<TColumnValue>)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case COL_VALS:
-      return getColVals();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case COL_VALS:
-      return isSetColVals();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TRow)
-      return this.equals((TRow)that);
-    return false;
-  }
-
-  public boolean equals(TRow that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_colVals = true && this.isSetColVals();
-    boolean that_present_colVals = true && that.isSetColVals();
-    if (this_present_colVals || that_present_colVals) {
-      if (!(this_present_colVals && that_present_colVals))
-        return false;
-      if (!this.colVals.equals(that.colVals))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_colVals = true && (isSetColVals());
-    list.add(present_colVals);
-    if (present_colVals)
-      list.add(colVals);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TRow other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetColVals()).compareTo(other.isSetColVals());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetColVals()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.colVals, other.colVals);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TRow(");
-    boolean first = true;
-
-    sb.append("colVals:");
-    if (this.colVals == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.colVals);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetColVals()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'colVals' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TRowStandardSchemeFactory implements SchemeFactory {
-    public TRowStandardScheme getScheme() {
-      return new TRowStandardScheme();
-    }
-  }
-
-  private static class TRowStandardScheme extends StandardScheme<TRow> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TRow struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // COL_VALS
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list46 = iprot.readListBegin();
-                struct.colVals = new ArrayList<TColumnValue>(_list46.size);
-                TColumnValue _elem47;
-                for (int _i48 = 0; _i48 < _list46.size; ++_i48)
-                {
-                  _elem47 = new TColumnValue();
-                  _elem47.read(iprot);
-                  struct.colVals.add(_elem47);
-                }
-                iprot.readListEnd();
-              }
-              struct.setColValsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TRow struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.colVals != null) {
-        oprot.writeFieldBegin(COL_VALS_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.colVals.size()));
-          for (TColumnValue _iter49 : struct.colVals)
-          {
-            _iter49.write(oprot);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TRowTupleSchemeFactory implements SchemeFactory {
-    public TRowTupleScheme getScheme() {
-      return new TRowTupleScheme();
-    }
-  }
-
-  private static class TRowTupleScheme extends TupleScheme<TRow> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TRow struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.colVals.size());
-        for (TColumnValue _iter50 : struct.colVals)
-        {
-          _iter50.write(oprot);
-        }
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TRow struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TList _list51 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-        struct.colVals = new ArrayList<TColumnValue>(_list51.size);
-        TColumnValue _elem52;
-        for (int _i53 = 0; _i53 < _list51.size; ++_i53)
-        {
-          _elem52 = new TColumnValue();
-          _elem52.read(iprot);
-          struct.colVals.add(_elem52);
-        }
-      }
-      struct.setColValsIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TRowSet.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TRowSet.java
deleted file mode 100644
index da3d9d3ca8820..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TRowSet.java
+++ /dev/null
@@ -1,920 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TRowSet implements org.apache.thrift.TBase<TRowSet, TRowSet._Fields>, java.io.Serializable, Cloneable, Comparable<TRowSet> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TRowSet");
-
-  private static final org.apache.thrift.protocol.TField START_ROW_OFFSET_FIELD_DESC = new org.apache.thrift.protocol.TField("startRowOffset", org.apache.thrift.protocol.TType.I64, (short)1);
-  private static final org.apache.thrift.protocol.TField ROWS_FIELD_DESC = new org.apache.thrift.protocol.TField("rows", org.apache.thrift.protocol.TType.LIST, (short)2);
-  private static final org.apache.thrift.protocol.TField COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("columns", org.apache.thrift.protocol.TType.LIST, (short)3);
-  private static final org.apache.thrift.protocol.TField BINARY_COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("binaryColumns", org.apache.thrift.protocol.TType.STRING, (short)4);
-  private static final org.apache.thrift.protocol.TField COLUMN_COUNT_FIELD_DESC = new org.apache.thrift.protocol.TField("columnCount", org.apache.thrift.protocol.TType.I32, (short)5);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TRowSetStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TRowSetTupleSchemeFactory());
-  }
-
-  private long startRowOffset; // required
-  private List<TRow> rows; // required
-  private List<TColumn> columns; // optional
-  private ByteBuffer binaryColumns; // optional
-  private int columnCount; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    START_ROW_OFFSET((short)1, "startRowOffset"),
-    ROWS((short)2, "rows"),
-    COLUMNS((short)3, "columns"),
-    BINARY_COLUMNS((short)4, "binaryColumns"),
-    COLUMN_COUNT((short)5, "columnCount");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // START_ROW_OFFSET
-          return START_ROW_OFFSET;
-        case 2: // ROWS
-          return ROWS;
-        case 3: // COLUMNS
-          return COLUMNS;
-        case 4: // BINARY_COLUMNS
-          return BINARY_COLUMNS;
-        case 5: // COLUMN_COUNT
-          return COLUMN_COUNT;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __STARTROWOFFSET_ISSET_ID = 0;
-  private static final int __COLUMNCOUNT_ISSET_ID = 1;
-  private byte __isset_bitfield = 0;
-  private static final _Fields optionals[] = {_Fields.COLUMNS,_Fields.BINARY_COLUMNS,_Fields.COLUMN_COUNT};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.START_ROW_OFFSET, new org.apache.thrift.meta_data.FieldMetaData("startRowOffset", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
-    tmpMap.put(_Fields.ROWS, new org.apache.thrift.meta_data.FieldMetaData("rows", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TRow.class))));
-    tmpMap.put(_Fields.COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("columns", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TColumn.class))));
-    tmpMap.put(_Fields.BINARY_COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("binaryColumns", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
-    tmpMap.put(_Fields.COLUMN_COUNT, new org.apache.thrift.meta_data.FieldMetaData("columnCount", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TRowSet.class, metaDataMap);
-  }
-
-  public TRowSet() {
-  }
-
-  public TRowSet(
-    long startRowOffset,
-    List<TRow> rows)
-  {
-    this();
-    this.startRowOffset = startRowOffset;
-    setStartRowOffsetIsSet(true);
-    this.rows = rows;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TRowSet(TRowSet other) {
-    __isset_bitfield = other.__isset_bitfield;
-    this.startRowOffset = other.startRowOffset;
-    if (other.isSetRows()) {
-      List<TRow> __this__rows = new ArrayList<TRow>(other.rows.size());
-      for (TRow other_element : other.rows) {
-        __this__rows.add(new TRow(other_element));
-      }
-      this.rows = __this__rows;
-    }
-    if (other.isSetColumns()) {
-      List<TColumn> __this__columns = new ArrayList<TColumn>(other.columns.size());
-      for (TColumn other_element : other.columns) {
-        __this__columns.add(new TColumn(other_element));
-      }
-      this.columns = __this__columns;
-    }
-    if (other.isSetBinaryColumns()) {
-      this.binaryColumns = org.apache.thrift.TBaseHelper.copyBinary(other.binaryColumns);
-    }
-    this.columnCount = other.columnCount;
-  }
-
-  public TRowSet deepCopy() {
-    return new TRowSet(this);
-  }
-
-  @Override
-  public void clear() {
-    setStartRowOffsetIsSet(false);
-    this.startRowOffset = 0;
-    this.rows = null;
-    this.columns = null;
-    this.binaryColumns = null;
-    setColumnCountIsSet(false);
-    this.columnCount = 0;
-  }
-
-  public long getStartRowOffset() {
-    return this.startRowOffset;
-  }
-
-  public void setStartRowOffset(long startRowOffset) {
-    this.startRowOffset = startRowOffset;
-    setStartRowOffsetIsSet(true);
-  }
-
-  public void unsetStartRowOffset() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __STARTROWOFFSET_ISSET_ID);
-  }
-
-  /** Returns true if field startRowOffset is set (has been assigned a value) and false otherwise */
-  public boolean isSetStartRowOffset() {
-    return EncodingUtils.testBit(__isset_bitfield, __STARTROWOFFSET_ISSET_ID);
-  }
-
-  public void setStartRowOffsetIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __STARTROWOFFSET_ISSET_ID, value);
-  }
-
-  public int getRowsSize() {
-    return (this.rows == null) ? 0 : this.rows.size();
-  }
-
-  public java.util.Iterator<TRow> getRowsIterator() {
-    return (this.rows == null) ? null : this.rows.iterator();
-  }
-
-  public void addToRows(TRow elem) {
-    if (this.rows == null) {
-      this.rows = new ArrayList<TRow>();
-    }
-    this.rows.add(elem);
-  }
-
-  public List<TRow> getRows() {
-    return this.rows;
-  }
-
-  public void setRows(List<TRow> rows) {
-    this.rows = rows;
-  }
-
-  public void unsetRows() {
-    this.rows = null;
-  }
-
-  /** Returns true if field rows is set (has been assigned a value) and false otherwise */
-  public boolean isSetRows() {
-    return this.rows != null;
-  }
-
-  public void setRowsIsSet(boolean value) {
-    if (!value) {
-      this.rows = null;
-    }
-  }
-
-  public int getColumnsSize() {
-    return (this.columns == null) ? 0 : this.columns.size();
-  }
-
-  public java.util.Iterator<TColumn> getColumnsIterator() {
-    return (this.columns == null) ? null : this.columns.iterator();
-  }
-
-  public void addToColumns(TColumn elem) {
-    if (this.columns == null) {
-      this.columns = new ArrayList<TColumn>();
-    }
-    this.columns.add(elem);
-  }
-
-  public List<TColumn> getColumns() {
-    return this.columns;
-  }
-
-  public void setColumns(List<TColumn> columns) {
-    this.columns = columns;
-  }
-
-  public void unsetColumns() {
-    this.columns = null;
-  }
-
-  /** Returns true if field columns is set (has been assigned a value) and false otherwise */
-  public boolean isSetColumns() {
-    return this.columns != null;
-  }
-
-  public void setColumnsIsSet(boolean value) {
-    if (!value) {
-      this.columns = null;
-    }
-  }
-
-  public byte[] getBinaryColumns() {
-    setBinaryColumns(org.apache.thrift.TBaseHelper.rightSize(binaryColumns));
-    return binaryColumns == null ? null : binaryColumns.array();
-  }
-
-  public ByteBuffer bufferForBinaryColumns() {
-    return org.apache.thrift.TBaseHelper.copyBinary(binaryColumns);
-  }
-
-  public void setBinaryColumns(byte[] binaryColumns) {
-    this.binaryColumns = binaryColumns == null ? (ByteBuffer)null : ByteBuffer.wrap(Arrays.copyOf(binaryColumns, binaryColumns.length));
-  }
-
-  public void setBinaryColumns(ByteBuffer binaryColumns) {
-    this.binaryColumns = org.apache.thrift.TBaseHelper.copyBinary(binaryColumns);
-  }
-
-  public void unsetBinaryColumns() {
-    this.binaryColumns = null;
-  }
-
-  /** Returns true if field binaryColumns is set (has been assigned a value) and false otherwise */
-  public boolean isSetBinaryColumns() {
-    return this.binaryColumns != null;
-  }
-
-  public void setBinaryColumnsIsSet(boolean value) {
-    if (!value) {
-      this.binaryColumns = null;
-    }
-  }
-
-  public int getColumnCount() {
-    return this.columnCount;
-  }
-
-  public void setColumnCount(int columnCount) {
-    this.columnCount = columnCount;
-    setColumnCountIsSet(true);
-  }
-
-  public void unsetColumnCount() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __COLUMNCOUNT_ISSET_ID);
-  }
-
-  /** Returns true if field columnCount is set (has been assigned a value) and false otherwise */
-  public boolean isSetColumnCount() {
-    return EncodingUtils.testBit(__isset_bitfield, __COLUMNCOUNT_ISSET_ID);
-  }
-
-  public void setColumnCountIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __COLUMNCOUNT_ISSET_ID, value);
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case START_ROW_OFFSET:
-      if (value == null) {
-        unsetStartRowOffset();
-      } else {
-        setStartRowOffset((Long)value);
-      }
-      break;
-
-    case ROWS:
-      if (value == null) {
-        unsetRows();
-      } else {
-        setRows((List<TRow>)value);
-      }
-      break;
-
-    case COLUMNS:
-      if (value == null) {
-        unsetColumns();
-      } else {
-        setColumns((List<TColumn>)value);
-      }
-      break;
-
-    case BINARY_COLUMNS:
-      if (value == null) {
-        unsetBinaryColumns();
-      } else {
-        setBinaryColumns((ByteBuffer)value);
-      }
-      break;
-
-    case COLUMN_COUNT:
-      if (value == null) {
-        unsetColumnCount();
-      } else {
-        setColumnCount((Integer)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case START_ROW_OFFSET:
-      return getStartRowOffset();
-
-    case ROWS:
-      return getRows();
-
-    case COLUMNS:
-      return getColumns();
-
-    case BINARY_COLUMNS:
-      return getBinaryColumns();
-
-    case COLUMN_COUNT:
-      return getColumnCount();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case START_ROW_OFFSET:
-      return isSetStartRowOffset();
-    case ROWS:
-      return isSetRows();
-    case COLUMNS:
-      return isSetColumns();
-    case BINARY_COLUMNS:
-      return isSetBinaryColumns();
-    case COLUMN_COUNT:
-      return isSetColumnCount();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TRowSet)
-      return this.equals((TRowSet)that);
-    return false;
-  }
-
-  public boolean equals(TRowSet that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_startRowOffset = true;
-    boolean that_present_startRowOffset = true;
-    if (this_present_startRowOffset || that_present_startRowOffset) {
-      if (!(this_present_startRowOffset && that_present_startRowOffset))
-        return false;
-      if (this.startRowOffset != that.startRowOffset)
-        return false;
-    }
-
-    boolean this_present_rows = true && this.isSetRows();
-    boolean that_present_rows = true && that.isSetRows();
-    if (this_present_rows || that_present_rows) {
-      if (!(this_present_rows && that_present_rows))
-        return false;
-      if (!this.rows.equals(that.rows))
-        return false;
-    }
-
-    boolean this_present_columns = true && this.isSetColumns();
-    boolean that_present_columns = true && that.isSetColumns();
-    if (this_present_columns || that_present_columns) {
-      if (!(this_present_columns && that_present_columns))
-        return false;
-      if (!this.columns.equals(that.columns))
-        return false;
-    }
-
-    boolean this_present_binaryColumns = true && this.isSetBinaryColumns();
-    boolean that_present_binaryColumns = true && that.isSetBinaryColumns();
-    if (this_present_binaryColumns || that_present_binaryColumns) {
-      if (!(this_present_binaryColumns && that_present_binaryColumns))
-        return false;
-      if (!this.binaryColumns.equals(that.binaryColumns))
-        return false;
-    }
-
-    boolean this_present_columnCount = true && this.isSetColumnCount();
-    boolean that_present_columnCount = true && that.isSetColumnCount();
-    if (this_present_columnCount || that_present_columnCount) {
-      if (!(this_present_columnCount && that_present_columnCount))
-        return false;
-      if (this.columnCount != that.columnCount)
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_startRowOffset = true;
-    list.add(present_startRowOffset);
-    if (present_startRowOffset)
-      list.add(startRowOffset);
-
-    boolean present_rows = true && (isSetRows());
-    list.add(present_rows);
-    if (present_rows)
-      list.add(rows);
-
-    boolean present_columns = true && (isSetColumns());
-    list.add(present_columns);
-    if (present_columns)
-      list.add(columns);
-
-    boolean present_binaryColumns = true && (isSetBinaryColumns());
-    list.add(present_binaryColumns);
-    if (present_binaryColumns)
-      list.add(binaryColumns);
-
-    boolean present_columnCount = true && (isSetColumnCount());
-    list.add(present_columnCount);
-    if (present_columnCount)
-      list.add(columnCount);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TRowSet other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetStartRowOffset()).compareTo(other.isSetStartRowOffset());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStartRowOffset()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.startRowOffset, other.startRowOffset);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetRows()).compareTo(other.isSetRows());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetRows()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.rows, other.rows);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetColumns()).compareTo(other.isSetColumns());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetColumns()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columns, other.columns);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetBinaryColumns()).compareTo(other.isSetBinaryColumns());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetBinaryColumns()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.binaryColumns, other.binaryColumns);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetColumnCount()).compareTo(other.isSetColumnCount());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetColumnCount()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columnCount, other.columnCount);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TRowSet(");
-    boolean first = true;
-
-    sb.append("startRowOffset:");
-    sb.append(this.startRowOffset);
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("rows:");
-    if (this.rows == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.rows);
-    }
-    first = false;
-    if (isSetColumns()) {
-      if (!first) sb.append(", ");
-      sb.append("columns:");
-      if (this.columns == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.columns);
-      }
-      first = false;
-    }
-    if (isSetBinaryColumns()) {
-      if (!first) sb.append(", ");
-      sb.append("binaryColumns:");
-      if (this.binaryColumns == null) {
-        sb.append("null");
-      } else {
-        org.apache.thrift.TBaseHelper.toString(this.binaryColumns, sb);
-      }
-      first = false;
-    }
-    if (isSetColumnCount()) {
-      if (!first) sb.append(", ");
-      sb.append("columnCount:");
-      sb.append(this.columnCount);
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStartRowOffset()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'startRowOffset' is unset! Struct:" + toString());
-    }
-
-    if (!isSetRows()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'rows' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TRowSetStandardSchemeFactory implements SchemeFactory {
-    public TRowSetStandardScheme getScheme() {
-      return new TRowSetStandardScheme();
-    }
-  }
-
-  private static class TRowSetStandardScheme extends StandardScheme<TRowSet> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TRowSet struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // START_ROW_OFFSET
-            if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
-              struct.startRowOffset = iprot.readI64();
-              struct.setStartRowOffsetIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // ROWS
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list118 = iprot.readListBegin();
-                struct.rows = new ArrayList<TRow>(_list118.size);
-                TRow _elem119;
-                for (int _i120 = 0; _i120 < _list118.size; ++_i120)
-                {
-                  _elem119 = new TRow();
-                  _elem119.read(iprot);
-                  struct.rows.add(_elem119);
-                }
-                iprot.readListEnd();
-              }
-              struct.setRowsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // COLUMNS
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list121 = iprot.readListBegin();
-                struct.columns = new ArrayList<TColumn>(_list121.size);
-                TColumn _elem122;
-                for (int _i123 = 0; _i123 < _list121.size; ++_i123)
-                {
-                  _elem122 = new TColumn();
-                  _elem122.read(iprot);
-                  struct.columns.add(_elem122);
-                }
-                iprot.readListEnd();
-              }
-              struct.setColumnsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // BINARY_COLUMNS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.binaryColumns = iprot.readBinary();
-              struct.setBinaryColumnsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 5: // COLUMN_COUNT
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.columnCount = iprot.readI32();
-              struct.setColumnCountIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TRowSet struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      oprot.writeFieldBegin(START_ROW_OFFSET_FIELD_DESC);
-      oprot.writeI64(struct.startRowOffset);
-      oprot.writeFieldEnd();
-      if (struct.rows != null) {
-        oprot.writeFieldBegin(ROWS_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.rows.size()));
-          for (TRow _iter124 : struct.rows)
-          {
-            _iter124.write(oprot);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      if (struct.columns != null) {
-        if (struct.isSetColumns()) {
-          oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
-          {
-            oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.columns.size()));
-            for (TColumn _iter125 : struct.columns)
-            {
-              _iter125.write(oprot);
-            }
-            oprot.writeListEnd();
-          }
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.binaryColumns != null) {
-        if (struct.isSetBinaryColumns()) {
-          oprot.writeFieldBegin(BINARY_COLUMNS_FIELD_DESC);
-          oprot.writeBinary(struct.binaryColumns);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.isSetColumnCount()) {
-        oprot.writeFieldBegin(COLUMN_COUNT_FIELD_DESC);
-        oprot.writeI32(struct.columnCount);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TRowSetTupleSchemeFactory implements SchemeFactory {
-    public TRowSetTupleScheme getScheme() {
-      return new TRowSetTupleScheme();
-    }
-  }
-
-  private static class TRowSetTupleScheme extends TupleScheme<TRowSet> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TRowSet struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      oprot.writeI64(struct.startRowOffset);
-      {
-        oprot.writeI32(struct.rows.size());
-        for (TRow _iter126 : struct.rows)
-        {
-          _iter126.write(oprot);
-        }
-      }
-      BitSet optionals = new BitSet();
-      if (struct.isSetColumns()) {
-        optionals.set(0);
-      }
-      if (struct.isSetBinaryColumns()) {
-        optionals.set(1);
-      }
-      if (struct.isSetColumnCount()) {
-        optionals.set(2);
-      }
-      oprot.writeBitSet(optionals, 3);
-      if (struct.isSetColumns()) {
-        {
-          oprot.writeI32(struct.columns.size());
-          for (TColumn _iter127 : struct.columns)
-          {
-            _iter127.write(oprot);
-          }
-        }
-      }
-      if (struct.isSetBinaryColumns()) {
-        oprot.writeBinary(struct.binaryColumns);
-      }
-      if (struct.isSetColumnCount()) {
-        oprot.writeI32(struct.columnCount);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TRowSet struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.startRowOffset = iprot.readI64();
-      struct.setStartRowOffsetIsSet(true);
-      {
-        org.apache.thrift.protocol.TList _list128 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-        struct.rows = new ArrayList<TRow>(_list128.size);
-        TRow _elem129;
-        for (int _i130 = 0; _i130 < _list128.size; ++_i130)
-        {
-          _elem129 = new TRow();
-          _elem129.read(iprot);
-          struct.rows.add(_elem129);
-        }
-      }
-      struct.setRowsIsSet(true);
-      BitSet incoming = iprot.readBitSet(3);
-      if (incoming.get(0)) {
-        {
-          org.apache.thrift.protocol.TList _list131 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-          struct.columns = new ArrayList<TColumn>(_list131.size);
-          TColumn _elem132;
-          for (int _i133 = 0; _i133 < _list131.size; ++_i133)
-          {
-            _elem132 = new TColumn();
-            _elem132.read(iprot);
-            struct.columns.add(_elem132);
-          }
-        }
-        struct.setColumnsIsSet(true);
-      }
-      if (incoming.get(1)) {
-        struct.binaryColumns = iprot.readBinary();
-        struct.setBinaryColumnsIsSet(true);
-      }
-      if (incoming.get(2)) {
-        struct.columnCount = iprot.readI32();
-        struct.setColumnCountIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TSessionHandle.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TSessionHandle.java
deleted file mode 100644
index b5cb6e7b15aa6..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TSessionHandle.java
+++ /dev/null
@@ -1,394 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TSessionHandle implements org.apache.thrift.TBase<TSessionHandle, TSessionHandle._Fields>, java.io.Serializable, Cloneable, Comparable<TSessionHandle> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TSessionHandle");
-
-  private static final org.apache.thrift.protocol.TField SESSION_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("sessionId", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TSessionHandleStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TSessionHandleTupleSchemeFactory());
-  }
-
-  private THandleIdentifier sessionId; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    SESSION_ID((short)1, "sessionId");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // SESSION_ID
-          return SESSION_ID;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.SESSION_ID, new org.apache.thrift.meta_data.FieldMetaData("sessionId", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, THandleIdentifier.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TSessionHandle.class, metaDataMap);
-  }
-
-  public TSessionHandle() {
-  }
-
-  public TSessionHandle(
-    THandleIdentifier sessionId)
-  {
-    this();
-    this.sessionId = sessionId;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TSessionHandle(TSessionHandle other) {
-    if (other.isSetSessionId()) {
-      this.sessionId = new THandleIdentifier(other.sessionId);
-    }
-  }
-
-  public TSessionHandle deepCopy() {
-    return new TSessionHandle(this);
-  }
-
-  @Override
-  public void clear() {
-    this.sessionId = null;
-  }
-
-  public THandleIdentifier getSessionId() {
-    return this.sessionId;
-  }
-
-  public void setSessionId(THandleIdentifier sessionId) {
-    this.sessionId = sessionId;
-  }
-
-  public void unsetSessionId() {
-    this.sessionId = null;
-  }
-
-  /** Returns true if field sessionId is set (has been assigned a value) and false otherwise */
-  public boolean isSetSessionId() {
-    return this.sessionId != null;
-  }
-
-  public void setSessionIdIsSet(boolean value) {
-    if (!value) {
-      this.sessionId = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case SESSION_ID:
-      if (value == null) {
-        unsetSessionId();
-      } else {
-        setSessionId((THandleIdentifier)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case SESSION_ID:
-      return getSessionId();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case SESSION_ID:
-      return isSetSessionId();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TSessionHandle)
-      return this.equals((TSessionHandle)that);
-    return false;
-  }
-
-  public boolean equals(TSessionHandle that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_sessionId = true && this.isSetSessionId();
-    boolean that_present_sessionId = true && that.isSetSessionId();
-    if (this_present_sessionId || that_present_sessionId) {
-      if (!(this_present_sessionId && that_present_sessionId))
-        return false;
-      if (!this.sessionId.equals(that.sessionId))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_sessionId = true && (isSetSessionId());
-    list.add(present_sessionId);
-    if (present_sessionId)
-      list.add(sessionId);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TSessionHandle other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetSessionId()).compareTo(other.isSetSessionId());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSessionId()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sessionId, other.sessionId);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TSessionHandle(");
-    boolean first = true;
-
-    sb.append("sessionId:");
-    if (this.sessionId == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.sessionId);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetSessionId()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'sessionId' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-    if (sessionId != null) {
-      sessionId.validate();
-    }
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TSessionHandleStandardSchemeFactory implements SchemeFactory {
-    public TSessionHandleStandardScheme getScheme() {
-      return new TSessionHandleStandardScheme();
-    }
-  }
-
-  private static class TSessionHandleStandardScheme extends StandardScheme<TSessionHandle> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TSessionHandle struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // SESSION_ID
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
-              struct.sessionId = new THandleIdentifier();
-              struct.sessionId.read(iprot);
-              struct.setSessionIdIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TSessionHandle struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.sessionId != null) {
-        oprot.writeFieldBegin(SESSION_ID_FIELD_DESC);
-        struct.sessionId.write(oprot);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TSessionHandleTupleSchemeFactory implements SchemeFactory {
-    public TSessionHandleTupleScheme getScheme() {
-      return new TSessionHandleTupleScheme();
-    }
-  }
-
-  private static class TSessionHandleTupleScheme extends TupleScheme<TSessionHandle> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TSessionHandle struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      struct.sessionId.write(oprot);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TSessionHandle struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.sessionId = new THandleIdentifier();
-      struct.sessionId.read(iprot);
-      struct.setSessionIdIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TStatus.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TStatus.java
deleted file mode 100644
index 50f4531b0a209..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TStatus.java
+++ /dev/null
@@ -1,875 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TStatus implements org.apache.thrift.TBase<TStatus, TStatus._Fields>, java.io.Serializable, Cloneable, Comparable<TStatus> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TStatus");
-
-  private static final org.apache.thrift.protocol.TField STATUS_CODE_FIELD_DESC = new org.apache.thrift.protocol.TField("statusCode", org.apache.thrift.protocol.TType.I32, (short)1);
-  private static final org.apache.thrift.protocol.TField INFO_MESSAGES_FIELD_DESC = new org.apache.thrift.protocol.TField("infoMessages", org.apache.thrift.protocol.TType.LIST, (short)2);
-  private static final org.apache.thrift.protocol.TField SQL_STATE_FIELD_DESC = new org.apache.thrift.protocol.TField("sqlState", org.apache.thrift.protocol.TType.STRING, (short)3);
-  private static final org.apache.thrift.protocol.TField ERROR_CODE_FIELD_DESC = new org.apache.thrift.protocol.TField("errorCode", org.apache.thrift.protocol.TType.I32, (short)4);
-  private static final org.apache.thrift.protocol.TField ERROR_MESSAGE_FIELD_DESC = new org.apache.thrift.protocol.TField("errorMessage", org.apache.thrift.protocol.TType.STRING, (short)5);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TStatusStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TStatusTupleSchemeFactory());
-  }
-
-  private TStatusCode statusCode; // required
-  private List<String> infoMessages; // optional
-  private String sqlState; // optional
-  private int errorCode; // optional
-  private String errorMessage; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    /**
-     * 
-     * @see TStatusCode
-     */
-    STATUS_CODE((short)1, "statusCode"),
-    INFO_MESSAGES((short)2, "infoMessages"),
-    SQL_STATE((short)3, "sqlState"),
-    ERROR_CODE((short)4, "errorCode"),
-    ERROR_MESSAGE((short)5, "errorMessage");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // STATUS_CODE
-          return STATUS_CODE;
-        case 2: // INFO_MESSAGES
-          return INFO_MESSAGES;
-        case 3: // SQL_STATE
-          return SQL_STATE;
-        case 4: // ERROR_CODE
-          return ERROR_CODE;
-        case 5: // ERROR_MESSAGE
-          return ERROR_MESSAGE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final int __ERRORCODE_ISSET_ID = 0;
-  private byte __isset_bitfield = 0;
-  private static final _Fields optionals[] = {_Fields.INFO_MESSAGES,_Fields.SQL_STATE,_Fields.ERROR_CODE,_Fields.ERROR_MESSAGE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.STATUS_CODE, new org.apache.thrift.meta_data.FieldMetaData("statusCode", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TStatusCode.class)));
-    tmpMap.put(_Fields.INFO_MESSAGES, new org.apache.thrift.meta_data.FieldMetaData("infoMessages", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
-    tmpMap.put(_Fields.SQL_STATE, new org.apache.thrift.meta_data.FieldMetaData("sqlState", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    tmpMap.put(_Fields.ERROR_CODE, new org.apache.thrift.meta_data.FieldMetaData("errorCode", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
-    tmpMap.put(_Fields.ERROR_MESSAGE, new org.apache.thrift.meta_data.FieldMetaData("errorMessage", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TStatus.class, metaDataMap);
-  }
-
-  public TStatus() {
-  }
-
-  public TStatus(
-    TStatusCode statusCode)
-  {
-    this();
-    this.statusCode = statusCode;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TStatus(TStatus other) {
-    __isset_bitfield = other.__isset_bitfield;
-    if (other.isSetStatusCode()) {
-      this.statusCode = other.statusCode;
-    }
-    if (other.isSetInfoMessages()) {
-      List<String> __this__infoMessages = new ArrayList<String>(other.infoMessages);
-      this.infoMessages = __this__infoMessages;
-    }
-    if (other.isSetSqlState()) {
-      this.sqlState = other.sqlState;
-    }
-    this.errorCode = other.errorCode;
-    if (other.isSetErrorMessage()) {
-      this.errorMessage = other.errorMessage;
-    }
-  }
-
-  public TStatus deepCopy() {
-    return new TStatus(this);
-  }
-
-  @Override
-  public void clear() {
-    this.statusCode = null;
-    this.infoMessages = null;
-    this.sqlState = null;
-    setErrorCodeIsSet(false);
-    this.errorCode = 0;
-    this.errorMessage = null;
-  }
-
-  /**
-   * 
-   * @see TStatusCode
-   */
-  public TStatusCode getStatusCode() {
-    return this.statusCode;
-  }
-
-  /**
-   * 
-   * @see TStatusCode
-   */
-  public void setStatusCode(TStatusCode statusCode) {
-    this.statusCode = statusCode;
-  }
-
-  public void unsetStatusCode() {
-    this.statusCode = null;
-  }
-
-  /** Returns true if field statusCode is set (has been assigned a value) and false otherwise */
-  public boolean isSetStatusCode() {
-    return this.statusCode != null;
-  }
-
-  public void setStatusCodeIsSet(boolean value) {
-    if (!value) {
-      this.statusCode = null;
-    }
-  }
-
-  public int getInfoMessagesSize() {
-    return (this.infoMessages == null) ? 0 : this.infoMessages.size();
-  }
-
-  public java.util.Iterator<String> getInfoMessagesIterator() {
-    return (this.infoMessages == null) ? null : this.infoMessages.iterator();
-  }
-
-  public void addToInfoMessages(String elem) {
-    if (this.infoMessages == null) {
-      this.infoMessages = new ArrayList<String>();
-    }
-    this.infoMessages.add(elem);
-  }
-
-  public List<String> getInfoMessages() {
-    return this.infoMessages;
-  }
-
-  public void setInfoMessages(List<String> infoMessages) {
-    this.infoMessages = infoMessages;
-  }
-
-  public void unsetInfoMessages() {
-    this.infoMessages = null;
-  }
-
-  /** Returns true if field infoMessages is set (has been assigned a value) and false otherwise */
-  public boolean isSetInfoMessages() {
-    return this.infoMessages != null;
-  }
-
-  public void setInfoMessagesIsSet(boolean value) {
-    if (!value) {
-      this.infoMessages = null;
-    }
-  }
-
-  public String getSqlState() {
-    return this.sqlState;
-  }
-
-  public void setSqlState(String sqlState) {
-    this.sqlState = sqlState;
-  }
-
-  public void unsetSqlState() {
-    this.sqlState = null;
-  }
-
-  /** Returns true if field sqlState is set (has been assigned a value) and false otherwise */
-  public boolean isSetSqlState() {
-    return this.sqlState != null;
-  }
-
-  public void setSqlStateIsSet(boolean value) {
-    if (!value) {
-      this.sqlState = null;
-    }
-  }
-
-  public int getErrorCode() {
-    return this.errorCode;
-  }
-
-  public void setErrorCode(int errorCode) {
-    this.errorCode = errorCode;
-    setErrorCodeIsSet(true);
-  }
-
-  public void unsetErrorCode() {
-    __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __ERRORCODE_ISSET_ID);
-  }
-
-  /** Returns true if field errorCode is set (has been assigned a value) and false otherwise */
-  public boolean isSetErrorCode() {
-    return EncodingUtils.testBit(__isset_bitfield, __ERRORCODE_ISSET_ID);
-  }
-
-  public void setErrorCodeIsSet(boolean value) {
-    __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __ERRORCODE_ISSET_ID, value);
-  }
-
-  public String getErrorMessage() {
-    return this.errorMessage;
-  }
-
-  public void setErrorMessage(String errorMessage) {
-    this.errorMessage = errorMessage;
-  }
-
-  public void unsetErrorMessage() {
-    this.errorMessage = null;
-  }
-
-  /** Returns true if field errorMessage is set (has been assigned a value) and false otherwise */
-  public boolean isSetErrorMessage() {
-    return this.errorMessage != null;
-  }
-
-  public void setErrorMessageIsSet(boolean value) {
-    if (!value) {
-      this.errorMessage = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case STATUS_CODE:
-      if (value == null) {
-        unsetStatusCode();
-      } else {
-        setStatusCode((TStatusCode)value);
-      }
-      break;
-
-    case INFO_MESSAGES:
-      if (value == null) {
-        unsetInfoMessages();
-      } else {
-        setInfoMessages((List<String>)value);
-      }
-      break;
-
-    case SQL_STATE:
-      if (value == null) {
-        unsetSqlState();
-      } else {
-        setSqlState((String)value);
-      }
-      break;
-
-    case ERROR_CODE:
-      if (value == null) {
-        unsetErrorCode();
-      } else {
-        setErrorCode((Integer)value);
-      }
-      break;
-
-    case ERROR_MESSAGE:
-      if (value == null) {
-        unsetErrorMessage();
-      } else {
-        setErrorMessage((String)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case STATUS_CODE:
-      return getStatusCode();
-
-    case INFO_MESSAGES:
-      return getInfoMessages();
-
-    case SQL_STATE:
-      return getSqlState();
-
-    case ERROR_CODE:
-      return getErrorCode();
-
-    case ERROR_MESSAGE:
-      return getErrorMessage();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case STATUS_CODE:
-      return isSetStatusCode();
-    case INFO_MESSAGES:
-      return isSetInfoMessages();
-    case SQL_STATE:
-      return isSetSqlState();
-    case ERROR_CODE:
-      return isSetErrorCode();
-    case ERROR_MESSAGE:
-      return isSetErrorMessage();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TStatus)
-      return this.equals((TStatus)that);
-    return false;
-  }
-
-  public boolean equals(TStatus that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_statusCode = true && this.isSetStatusCode();
-    boolean that_present_statusCode = true && that.isSetStatusCode();
-    if (this_present_statusCode || that_present_statusCode) {
-      if (!(this_present_statusCode && that_present_statusCode))
-        return false;
-      if (!this.statusCode.equals(that.statusCode))
-        return false;
-    }
-
-    boolean this_present_infoMessages = true && this.isSetInfoMessages();
-    boolean that_present_infoMessages = true && that.isSetInfoMessages();
-    if (this_present_infoMessages || that_present_infoMessages) {
-      if (!(this_present_infoMessages && that_present_infoMessages))
-        return false;
-      if (!this.infoMessages.equals(that.infoMessages))
-        return false;
-    }
-
-    boolean this_present_sqlState = true && this.isSetSqlState();
-    boolean that_present_sqlState = true && that.isSetSqlState();
-    if (this_present_sqlState || that_present_sqlState) {
-      if (!(this_present_sqlState && that_present_sqlState))
-        return false;
-      if (!this.sqlState.equals(that.sqlState))
-        return false;
-    }
-
-    boolean this_present_errorCode = true && this.isSetErrorCode();
-    boolean that_present_errorCode = true && that.isSetErrorCode();
-    if (this_present_errorCode || that_present_errorCode) {
-      if (!(this_present_errorCode && that_present_errorCode))
-        return false;
-      if (this.errorCode != that.errorCode)
-        return false;
-    }
-
-    boolean this_present_errorMessage = true && this.isSetErrorMessage();
-    boolean that_present_errorMessage = true && that.isSetErrorMessage();
-    if (this_present_errorMessage || that_present_errorMessage) {
-      if (!(this_present_errorMessage && that_present_errorMessage))
-        return false;
-      if (!this.errorMessage.equals(that.errorMessage))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_statusCode = true && (isSetStatusCode());
-    list.add(present_statusCode);
-    if (present_statusCode)
-      list.add(statusCode.getValue());
-
-    boolean present_infoMessages = true && (isSetInfoMessages());
-    list.add(present_infoMessages);
-    if (present_infoMessages)
-      list.add(infoMessages);
-
-    boolean present_sqlState = true && (isSetSqlState());
-    list.add(present_sqlState);
-    if (present_sqlState)
-      list.add(sqlState);
-
-    boolean present_errorCode = true && (isSetErrorCode());
-    list.add(present_errorCode);
-    if (present_errorCode)
-      list.add(errorCode);
-
-    boolean present_errorMessage = true && (isSetErrorMessage());
-    list.add(present_errorMessage);
-    if (present_errorMessage)
-      list.add(errorMessage);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TStatus other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetStatusCode()).compareTo(other.isSetStatusCode());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetStatusCode()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.statusCode, other.statusCode);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetInfoMessages()).compareTo(other.isSetInfoMessages());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetInfoMessages()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.infoMessages, other.infoMessages);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetSqlState()).compareTo(other.isSetSqlState());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSqlState()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sqlState, other.sqlState);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetErrorCode()).compareTo(other.isSetErrorCode());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetErrorCode()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.errorCode, other.errorCode);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetErrorMessage()).compareTo(other.isSetErrorMessage());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetErrorMessage()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.errorMessage, other.errorMessage);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TStatus(");
-    boolean first = true;
-
-    sb.append("statusCode:");
-    if (this.statusCode == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.statusCode);
-    }
-    first = false;
-    if (isSetInfoMessages()) {
-      if (!first) sb.append(", ");
-      sb.append("infoMessages:");
-      if (this.infoMessages == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.infoMessages);
-      }
-      first = false;
-    }
-    if (isSetSqlState()) {
-      if (!first) sb.append(", ");
-      sb.append("sqlState:");
-      if (this.sqlState == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.sqlState);
-      }
-      first = false;
-    }
-    if (isSetErrorCode()) {
-      if (!first) sb.append(", ");
-      sb.append("errorCode:");
-      sb.append(this.errorCode);
-      first = false;
-    }
-    if (isSetErrorMessage()) {
-      if (!first) sb.append(", ");
-      sb.append("errorMessage:");
-      if (this.errorMessage == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.errorMessage);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetStatusCode()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'statusCode' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
-      __isset_bitfield = 0;
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TStatusStandardSchemeFactory implements SchemeFactory {
-    public TStatusStandardScheme getScheme() {
-      return new TStatusStandardScheme();
-    }
-  }
-
-  private static class TStatusStandardScheme extends StandardScheme<TStatus> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TStatus struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // STATUS_CODE
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.statusCode = org.apache.hive.service.rpc.thrift.TStatusCode.findByValue(iprot.readI32());
-              struct.setStatusCodeIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // INFO_MESSAGES
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list134 = iprot.readListBegin();
-                struct.infoMessages = new ArrayList<String>(_list134.size);
-                String _elem135;
-                for (int _i136 = 0; _i136 < _list134.size; ++_i136)
-                {
-                  _elem135 = iprot.readString();
-                  struct.infoMessages.add(_elem135);
-                }
-                iprot.readListEnd();
-              }
-              struct.setInfoMessagesIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 3: // SQL_STATE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.sqlState = iprot.readString();
-              struct.setSqlStateIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 4: // ERROR_CODE
-            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
-              struct.errorCode = iprot.readI32();
-              struct.setErrorCodeIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 5: // ERROR_MESSAGE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.errorMessage = iprot.readString();
-              struct.setErrorMessageIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TStatus struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.statusCode != null) {
-        oprot.writeFieldBegin(STATUS_CODE_FIELD_DESC);
-        oprot.writeI32(struct.statusCode.getValue());
-        oprot.writeFieldEnd();
-      }
-      if (struct.infoMessages != null) {
-        if (struct.isSetInfoMessages()) {
-          oprot.writeFieldBegin(INFO_MESSAGES_FIELD_DESC);
-          {
-            oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.infoMessages.size()));
-            for (String _iter137 : struct.infoMessages)
-            {
-              oprot.writeString(_iter137);
-            }
-            oprot.writeListEnd();
-          }
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.sqlState != null) {
-        if (struct.isSetSqlState()) {
-          oprot.writeFieldBegin(SQL_STATE_FIELD_DESC);
-          oprot.writeString(struct.sqlState);
-          oprot.writeFieldEnd();
-        }
-      }
-      if (struct.isSetErrorCode()) {
-        oprot.writeFieldBegin(ERROR_CODE_FIELD_DESC);
-        oprot.writeI32(struct.errorCode);
-        oprot.writeFieldEnd();
-      }
-      if (struct.errorMessage != null) {
-        if (struct.isSetErrorMessage()) {
-          oprot.writeFieldBegin(ERROR_MESSAGE_FIELD_DESC);
-          oprot.writeString(struct.errorMessage);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TStatusTupleSchemeFactory implements SchemeFactory {
-    public TStatusTupleScheme getScheme() {
-      return new TStatusTupleScheme();
-    }
-  }
-
-  private static class TStatusTupleScheme extends TupleScheme<TStatus> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TStatus struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      oprot.writeI32(struct.statusCode.getValue());
-      BitSet optionals = new BitSet();
-      if (struct.isSetInfoMessages()) {
-        optionals.set(0);
-      }
-      if (struct.isSetSqlState()) {
-        optionals.set(1);
-      }
-      if (struct.isSetErrorCode()) {
-        optionals.set(2);
-      }
-      if (struct.isSetErrorMessage()) {
-        optionals.set(3);
-      }
-      oprot.writeBitSet(optionals, 4);
-      if (struct.isSetInfoMessages()) {
-        {
-          oprot.writeI32(struct.infoMessages.size());
-          for (String _iter138 : struct.infoMessages)
-          {
-            oprot.writeString(_iter138);
-          }
-        }
-      }
-      if (struct.isSetSqlState()) {
-        oprot.writeString(struct.sqlState);
-      }
-      if (struct.isSetErrorCode()) {
-        oprot.writeI32(struct.errorCode);
-      }
-      if (struct.isSetErrorMessage()) {
-        oprot.writeString(struct.errorMessage);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TStatus struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.statusCode = org.apache.hive.service.rpc.thrift.TStatusCode.findByValue(iprot.readI32());
-      struct.setStatusCodeIsSet(true);
-      BitSet incoming = iprot.readBitSet(4);
-      if (incoming.get(0)) {
-        {
-          org.apache.thrift.protocol.TList _list139 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-          struct.infoMessages = new ArrayList<String>(_list139.size);
-          String _elem140;
-          for (int _i141 = 0; _i141 < _list139.size; ++_i141)
-          {
-            _elem140 = iprot.readString();
-            struct.infoMessages.add(_elem140);
-          }
-        }
-        struct.setInfoMessagesIsSet(true);
-      }
-      if (incoming.get(1)) {
-        struct.sqlState = iprot.readString();
-        struct.setSqlStateIsSet(true);
-      }
-      if (incoming.get(2)) {
-        struct.errorCode = iprot.readI32();
-        struct.setErrorCodeIsSet(true);
-      }
-      if (incoming.get(3)) {
-        struct.errorMessage = iprot.readString();
-        struct.setErrorMessageIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TStatusCode.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TStatusCode.java
deleted file mode 100644
index fbf14184fa9a8..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TStatusCode.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.thrift.TEnum;
-
-public enum TStatusCode implements org.apache.thrift.TEnum {
-  SUCCESS_STATUS(0),
-  SUCCESS_WITH_INFO_STATUS(1),
-  STILL_EXECUTING_STATUS(2),
-  ERROR_STATUS(3),
-  INVALID_HANDLE_STATUS(4);
-
-  private final int value;
-
-  private TStatusCode(int value) {
-    this.value = value;
-  }
-
-  /**
-   * Get the integer value of this enum value, as defined in the Thrift IDL.
-   */
-  public int getValue() {
-    return value;
-  }
-
-  /**
-   * Find a the enum type by its integer value, as defined in the Thrift IDL.
-   * @return null if the value is not found.
-   */
-  public static TStatusCode findByValue(int value) { 
-    switch (value) {
-      case 0:
-        return SUCCESS_STATUS;
-      case 1:
-        return SUCCESS_WITH_INFO_STATUS;
-      case 2:
-        return STILL_EXECUTING_STATUS;
-      case 3:
-        return ERROR_STATUS;
-      case 4:
-        return INVALID_HANDLE_STATUS;
-      default:
-        return null;
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TStringColumn.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TStringColumn.java
deleted file mode 100644
index c83a1fd0de3c2..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TStringColumn.java
+++ /dev/null
@@ -1,548 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TStringColumn implements org.apache.thrift.TBase<TStringColumn, TStringColumn._Fields>, java.io.Serializable, Cloneable, Comparable<TStringColumn> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TStringColumn");
-
-  private static final org.apache.thrift.protocol.TField VALUES_FIELD_DESC = new org.apache.thrift.protocol.TField("values", org.apache.thrift.protocol.TType.LIST, (short)1);
-  private static final org.apache.thrift.protocol.TField NULLS_FIELD_DESC = new org.apache.thrift.protocol.TField("nulls", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TStringColumnStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TStringColumnTupleSchemeFactory());
-  }
-
-  private List<String> values; // required
-  private ByteBuffer nulls; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUES((short)1, "values"),
-    NULLS((short)2, "nulls");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUES
-          return VALUES;
-        case 2: // NULLS
-          return NULLS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUES, new org.apache.thrift.meta_data.FieldMetaData("values", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
-    tmpMap.put(_Fields.NULLS, new org.apache.thrift.meta_data.FieldMetaData("nulls", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING        , true)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TStringColumn.class, metaDataMap);
-  }
-
-  public TStringColumn() {
-  }
-
-  public TStringColumn(
-    List<String> values,
-    ByteBuffer nulls)
-  {
-    this();
-    this.values = values;
-    this.nulls = org.apache.thrift.TBaseHelper.copyBinary(nulls);
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TStringColumn(TStringColumn other) {
-    if (other.isSetValues()) {
-      List<String> __this__values = new ArrayList<String>(other.values);
-      this.values = __this__values;
-    }
-    if (other.isSetNulls()) {
-      this.nulls = org.apache.thrift.TBaseHelper.copyBinary(other.nulls);
-    }
-  }
-
-  public TStringColumn deepCopy() {
-    return new TStringColumn(this);
-  }
-
-  @Override
-  public void clear() {
-    this.values = null;
-    this.nulls = null;
-  }
-
-  public int getValuesSize() {
-    return (this.values == null) ? 0 : this.values.size();
-  }
-
-  public java.util.Iterator<String> getValuesIterator() {
-    return (this.values == null) ? null : this.values.iterator();
-  }
-
-  public void addToValues(String elem) {
-    if (this.values == null) {
-      this.values = new ArrayList<String>();
-    }
-    this.values.add(elem);
-  }
-
-  public List<String> getValues() {
-    return this.values;
-  }
-
-  public void setValues(List<String> values) {
-    this.values = values;
-  }
-
-  public void unsetValues() {
-    this.values = null;
-  }
-
-  /** Returns true if field values is set (has been assigned a value) and false otherwise */
-  public boolean isSetValues() {
-    return this.values != null;
-  }
-
-  public void setValuesIsSet(boolean value) {
-    if (!value) {
-      this.values = null;
-    }
-  }
-
-  public byte[] getNulls() {
-    setNulls(org.apache.thrift.TBaseHelper.rightSize(nulls));
-    return nulls == null ? null : nulls.array();
-  }
-
-  public ByteBuffer bufferForNulls() {
-    return org.apache.thrift.TBaseHelper.copyBinary(nulls);
-  }
-
-  public void setNulls(byte[] nulls) {
-    this.nulls = nulls == null ? (ByteBuffer)null : ByteBuffer.wrap(Arrays.copyOf(nulls, nulls.length));
-  }
-
-  public void setNulls(ByteBuffer nulls) {
-    this.nulls = org.apache.thrift.TBaseHelper.copyBinary(nulls);
-  }
-
-  public void unsetNulls() {
-    this.nulls = null;
-  }
-
-  /** Returns true if field nulls is set (has been assigned a value) and false otherwise */
-  public boolean isSetNulls() {
-    return this.nulls != null;
-  }
-
-  public void setNullsIsSet(boolean value) {
-    if (!value) {
-      this.nulls = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUES:
-      if (value == null) {
-        unsetValues();
-      } else {
-        setValues((List<String>)value);
-      }
-      break;
-
-    case NULLS:
-      if (value == null) {
-        unsetNulls();
-      } else {
-        setNulls((ByteBuffer)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUES:
-      return getValues();
-
-    case NULLS:
-      return getNulls();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUES:
-      return isSetValues();
-    case NULLS:
-      return isSetNulls();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TStringColumn)
-      return this.equals((TStringColumn)that);
-    return false;
-  }
-
-  public boolean equals(TStringColumn that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_values = true && this.isSetValues();
-    boolean that_present_values = true && that.isSetValues();
-    if (this_present_values || that_present_values) {
-      if (!(this_present_values && that_present_values))
-        return false;
-      if (!this.values.equals(that.values))
-        return false;
-    }
-
-    boolean this_present_nulls = true && this.isSetNulls();
-    boolean that_present_nulls = true && that.isSetNulls();
-    if (this_present_nulls || that_present_nulls) {
-      if (!(this_present_nulls && that_present_nulls))
-        return false;
-      if (!this.nulls.equals(that.nulls))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_values = true && (isSetValues());
-    list.add(present_values);
-    if (present_values)
-      list.add(values);
-
-    boolean present_nulls = true && (isSetNulls());
-    list.add(present_nulls);
-    if (present_nulls)
-      list.add(nulls);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TStringColumn other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetValues()).compareTo(other.isSetValues());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValues()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.values, other.values);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetNulls()).compareTo(other.isSetNulls());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetNulls()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.nulls, other.nulls);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TStringColumn(");
-    boolean first = true;
-
-    sb.append("values:");
-    if (this.values == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.values);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("nulls:");
-    if (this.nulls == null) {
-      sb.append("null");
-    } else {
-      org.apache.thrift.TBaseHelper.toString(this.nulls, sb);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetValues()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'values' is unset! Struct:" + toString());
-    }
-
-    if (!isSetNulls()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'nulls' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TStringColumnStandardSchemeFactory implements SchemeFactory {
-    public TStringColumnStandardScheme getScheme() {
-      return new TStringColumnStandardScheme();
-    }
-  }
-
-  private static class TStringColumnStandardScheme extends StandardScheme<TStringColumn> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TStringColumn struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUES
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list102 = iprot.readListBegin();
-                struct.values = new ArrayList<String>(_list102.size);
-                String _elem103;
-                for (int _i104 = 0; _i104 < _list102.size; ++_i104)
-                {
-                  _elem103 = iprot.readString();
-                  struct.values.add(_elem103);
-                }
-                iprot.readListEnd();
-              }
-              struct.setValuesIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 2: // NULLS
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.nulls = iprot.readBinary();
-              struct.setNullsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TStringColumn struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.values != null) {
-        oprot.writeFieldBegin(VALUES_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.values.size()));
-          for (String _iter105 : struct.values)
-          {
-            oprot.writeString(_iter105);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      if (struct.nulls != null) {
-        oprot.writeFieldBegin(NULLS_FIELD_DESC);
-        oprot.writeBinary(struct.nulls);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TStringColumnTupleSchemeFactory implements SchemeFactory {
-    public TStringColumnTupleScheme getScheme() {
-      return new TStringColumnTupleScheme();
-    }
-  }
-
-  private static class TStringColumnTupleScheme extends TupleScheme<TStringColumn> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TStringColumn struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.values.size());
-        for (String _iter106 : struct.values)
-        {
-          oprot.writeString(_iter106);
-        }
-      }
-      oprot.writeBinary(struct.nulls);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TStringColumn struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TList _list107 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-        struct.values = new ArrayList<String>(_list107.size);
-        String _elem108;
-        for (int _i109 = 0; _i109 < _list107.size; ++_i109)
-        {
-          _elem108 = iprot.readString();
-          struct.values.add(_elem108);
-        }
-      }
-      struct.setValuesIsSet(true);
-      struct.nulls = iprot.readBinary();
-      struct.setNullsIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TStringValue.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TStringValue.java
deleted file mode 100644
index 13874e5516632..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TStringValue.java
+++ /dev/null
@@ -1,393 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TStringValue implements org.apache.thrift.TBase<TStringValue, TStringValue._Fields>, java.io.Serializable, Cloneable, Comparable<TStringValue> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TStringValue");
-
-  private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.STRING, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TStringValueStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TStringValueTupleSchemeFactory());
-  }
-
-  private String value; // optional
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    VALUE((short)1, "value");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // VALUE
-          return VALUE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  private static final _Fields optionals[] = {_Fields.VALUE};
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TStringValue.class, metaDataMap);
-  }
-
-  public TStringValue() {
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TStringValue(TStringValue other) {
-    if (other.isSetValue()) {
-      this.value = other.value;
-    }
-  }
-
-  public TStringValue deepCopy() {
-    return new TStringValue(this);
-  }
-
-  @Override
-  public void clear() {
-    this.value = null;
-  }
-
-  public String getValue() {
-    return this.value;
-  }
-
-  public void setValue(String value) {
-    this.value = value;
-  }
-
-  public void unsetValue() {
-    this.value = null;
-  }
-
-  /** Returns true if field value is set (has been assigned a value) and false otherwise */
-  public boolean isSetValue() {
-    return this.value != null;
-  }
-
-  public void setValueIsSet(boolean value) {
-    if (!value) {
-      this.value = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case VALUE:
-      if (value == null) {
-        unsetValue();
-      } else {
-        setValue((String)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case VALUE:
-      return getValue();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case VALUE:
-      return isSetValue();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TStringValue)
-      return this.equals((TStringValue)that);
-    return false;
-  }
-
-  public boolean equals(TStringValue that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_value = true && this.isSetValue();
-    boolean that_present_value = true && that.isSetValue();
-    if (this_present_value || that_present_value) {
-      if (!(this_present_value && that_present_value))
-        return false;
-      if (!this.value.equals(that.value))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_value = true && (isSetValue());
-    list.add(present_value);
-    if (present_value)
-      list.add(value);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TStringValue other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetValue()).compareTo(other.isSetValue());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetValue()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, other.value);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TStringValue(");
-    boolean first = true;
-
-    if (isSetValue()) {
-      sb.append("value:");
-      if (this.value == null) {
-        sb.append("null");
-      } else {
-        sb.append(this.value);
-      }
-      first = false;
-    }
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TStringValueStandardSchemeFactory implements SchemeFactory {
-    public TStringValueStandardScheme getScheme() {
-      return new TStringValueStandardScheme();
-    }
-  }
-
-  private static class TStringValueStandardScheme extends StandardScheme<TStringValue> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TStringValue struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // VALUE
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.value = iprot.readString();
-              struct.setValueIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TStringValue struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.value != null) {
-        if (struct.isSetValue()) {
-          oprot.writeFieldBegin(VALUE_FIELD_DESC);
-          oprot.writeString(struct.value);
-          oprot.writeFieldEnd();
-        }
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TStringValueTupleSchemeFactory implements SchemeFactory {
-    public TStringValueTupleScheme getScheme() {
-      return new TStringValueTupleScheme();
-    }
-  }
-
-  private static class TStringValueTupleScheme extends TupleScheme<TStringValue> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TStringValue struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      BitSet optionals = new BitSet();
-      if (struct.isSetValue()) {
-        optionals.set(0);
-      }
-      oprot.writeBitSet(optionals, 1);
-      if (struct.isSetValue()) {
-        oprot.writeString(struct.value);
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TStringValue struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      BitSet incoming = iprot.readBitSet(1);
-      if (incoming.get(0)) {
-        struct.value = iprot.readString();
-        struct.setValueIsSet(true);
-      }
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TStructTypeEntry.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TStructTypeEntry.java
deleted file mode 100644
index 6c2c4f5dd2ddf..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TStructTypeEntry.java
+++ /dev/null
@@ -1,452 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TStructTypeEntry implements org.apache.thrift.TBase<TStructTypeEntry, TStructTypeEntry._Fields>, java.io.Serializable, Cloneable, Comparable<TStructTypeEntry> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TStructTypeEntry");
-
-  private static final org.apache.thrift.protocol.TField NAME_TO_TYPE_PTR_FIELD_DESC = new org.apache.thrift.protocol.TField("nameToTypePtr", org.apache.thrift.protocol.TType.MAP, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TStructTypeEntryStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TStructTypeEntryTupleSchemeFactory());
-  }
-
-  private Map<String,Integer> nameToTypePtr; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    NAME_TO_TYPE_PTR((short)1, "nameToTypePtr");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // NAME_TO_TYPE_PTR
-          return NAME_TO_TYPE_PTR;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.NAME_TO_TYPE_PTR, new org.apache.thrift.meta_data.FieldMetaData("nameToTypePtr", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING), 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32            , "TTypeEntryPtr"))));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TStructTypeEntry.class, metaDataMap);
-  }
-
-  public TStructTypeEntry() {
-  }
-
-  public TStructTypeEntry(
-    Map<String,Integer> nameToTypePtr)
-  {
-    this();
-    this.nameToTypePtr = nameToTypePtr;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TStructTypeEntry(TStructTypeEntry other) {
-    if (other.isSetNameToTypePtr()) {
-      Map<String,Integer> __this__nameToTypePtr = new HashMap<String,Integer>(other.nameToTypePtr.size());
-      for (Map.Entry<String, Integer> other_element : other.nameToTypePtr.entrySet()) {
-
-        String other_element_key = other_element.getKey();
-        Integer other_element_value = other_element.getValue();
-
-        String __this__nameToTypePtr_copy_key = other_element_key;
-
-        Integer __this__nameToTypePtr_copy_value = other_element_value;
-
-        __this__nameToTypePtr.put(__this__nameToTypePtr_copy_key, __this__nameToTypePtr_copy_value);
-      }
-      this.nameToTypePtr = __this__nameToTypePtr;
-    }
-  }
-
-  public TStructTypeEntry deepCopy() {
-    return new TStructTypeEntry(this);
-  }
-
-  @Override
-  public void clear() {
-    this.nameToTypePtr = null;
-  }
-
-  public int getNameToTypePtrSize() {
-    return (this.nameToTypePtr == null) ? 0 : this.nameToTypePtr.size();
-  }
-
-  public void putToNameToTypePtr(String key, int val) {
-    if (this.nameToTypePtr == null) {
-      this.nameToTypePtr = new HashMap<String,Integer>();
-    }
-    this.nameToTypePtr.put(key, val);
-  }
-
-  public Map<String,Integer> getNameToTypePtr() {
-    return this.nameToTypePtr;
-  }
-
-  public void setNameToTypePtr(Map<String,Integer> nameToTypePtr) {
-    this.nameToTypePtr = nameToTypePtr;
-  }
-
-  public void unsetNameToTypePtr() {
-    this.nameToTypePtr = null;
-  }
-
-  /** Returns true if field nameToTypePtr is set (has been assigned a value) and false otherwise */
-  public boolean isSetNameToTypePtr() {
-    return this.nameToTypePtr != null;
-  }
-
-  public void setNameToTypePtrIsSet(boolean value) {
-    if (!value) {
-      this.nameToTypePtr = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case NAME_TO_TYPE_PTR:
-      if (value == null) {
-        unsetNameToTypePtr();
-      } else {
-        setNameToTypePtr((Map<String,Integer>)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case NAME_TO_TYPE_PTR:
-      return getNameToTypePtr();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case NAME_TO_TYPE_PTR:
-      return isSetNameToTypePtr();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TStructTypeEntry)
-      return this.equals((TStructTypeEntry)that);
-    return false;
-  }
-
-  public boolean equals(TStructTypeEntry that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_nameToTypePtr = true && this.isSetNameToTypePtr();
-    boolean that_present_nameToTypePtr = true && that.isSetNameToTypePtr();
-    if (this_present_nameToTypePtr || that_present_nameToTypePtr) {
-      if (!(this_present_nameToTypePtr && that_present_nameToTypePtr))
-        return false;
-      if (!this.nameToTypePtr.equals(that.nameToTypePtr))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_nameToTypePtr = true && (isSetNameToTypePtr());
-    list.add(present_nameToTypePtr);
-    if (present_nameToTypePtr)
-      list.add(nameToTypePtr);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TStructTypeEntry other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetNameToTypePtr()).compareTo(other.isSetNameToTypePtr());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetNameToTypePtr()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.nameToTypePtr, other.nameToTypePtr);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TStructTypeEntry(");
-    boolean first = true;
-
-    sb.append("nameToTypePtr:");
-    if (this.nameToTypePtr == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.nameToTypePtr);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetNameToTypePtr()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'nameToTypePtr' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TStructTypeEntryStandardSchemeFactory implements SchemeFactory {
-    public TStructTypeEntryStandardScheme getScheme() {
-      return new TStructTypeEntryStandardScheme();
-    }
-  }
-
-  private static class TStructTypeEntryStandardScheme extends StandardScheme<TStructTypeEntry> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TStructTypeEntry struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // NAME_TO_TYPE_PTR
-            if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
-              {
-                org.apache.thrift.protocol.TMap _map10 = iprot.readMapBegin();
-                struct.nameToTypePtr = new HashMap<String,Integer>(2*_map10.size);
-                String _key11;
-                int _val12;
-                for (int _i13 = 0; _i13 < _map10.size; ++_i13)
-                {
-                  _key11 = iprot.readString();
-                  _val12 = iprot.readI32();
-                  struct.nameToTypePtr.put(_key11, _val12);
-                }
-                iprot.readMapEnd();
-              }
-              struct.setNameToTypePtrIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TStructTypeEntry struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.nameToTypePtr != null) {
-        oprot.writeFieldBegin(NAME_TO_TYPE_PTR_FIELD_DESC);
-        {
-          oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.I32, struct.nameToTypePtr.size()));
-          for (Map.Entry<String, Integer> _iter14 : struct.nameToTypePtr.entrySet())
-          {
-            oprot.writeString(_iter14.getKey());
-            oprot.writeI32(_iter14.getValue());
-          }
-          oprot.writeMapEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TStructTypeEntryTupleSchemeFactory implements SchemeFactory {
-    public TStructTypeEntryTupleScheme getScheme() {
-      return new TStructTypeEntryTupleScheme();
-    }
-  }
-
-  private static class TStructTypeEntryTupleScheme extends TupleScheme<TStructTypeEntry> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TStructTypeEntry struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.nameToTypePtr.size());
-        for (Map.Entry<String, Integer> _iter15 : struct.nameToTypePtr.entrySet())
-        {
-          oprot.writeString(_iter15.getKey());
-          oprot.writeI32(_iter15.getValue());
-        }
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TStructTypeEntry struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TMap _map16 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.I32, iprot.readI32());
-        struct.nameToTypePtr = new HashMap<String,Integer>(2*_map16.size);
-        String _key17;
-        int _val18;
-        for (int _i19 = 0; _i19 < _map16.size; ++_i19)
-        {
-          _key17 = iprot.readString();
-          _val18 = iprot.readI32();
-          struct.nameToTypePtr.put(_key17, _val18);
-        }
-      }
-      struct.setNameToTypePtrIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TTableSchema.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TTableSchema.java
deleted file mode 100644
index 007b1603546ac..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TTableSchema.java
+++ /dev/null
@@ -1,443 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TTableSchema implements org.apache.thrift.TBase<TTableSchema, TTableSchema._Fields>, java.io.Serializable, Cloneable, Comparable<TTableSchema> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TTableSchema");
-
-  private static final org.apache.thrift.protocol.TField COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("columns", org.apache.thrift.protocol.TType.LIST, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TTableSchemaStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TTableSchemaTupleSchemeFactory());
-  }
-
-  private List<TColumnDesc> columns; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    COLUMNS((short)1, "columns");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // COLUMNS
-          return COLUMNS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("columns", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TColumnDesc.class))));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TTableSchema.class, metaDataMap);
-  }
-
-  public TTableSchema() {
-  }
-
-  public TTableSchema(
-    List<TColumnDesc> columns)
-  {
-    this();
-    this.columns = columns;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TTableSchema(TTableSchema other) {
-    if (other.isSetColumns()) {
-      List<TColumnDesc> __this__columns = new ArrayList<TColumnDesc>(other.columns.size());
-      for (TColumnDesc other_element : other.columns) {
-        __this__columns.add(new TColumnDesc(other_element));
-      }
-      this.columns = __this__columns;
-    }
-  }
-
-  public TTableSchema deepCopy() {
-    return new TTableSchema(this);
-  }
-
-  @Override
-  public void clear() {
-    this.columns = null;
-  }
-
-  public int getColumnsSize() {
-    return (this.columns == null) ? 0 : this.columns.size();
-  }
-
-  public java.util.Iterator<TColumnDesc> getColumnsIterator() {
-    return (this.columns == null) ? null : this.columns.iterator();
-  }
-
-  public void addToColumns(TColumnDesc elem) {
-    if (this.columns == null) {
-      this.columns = new ArrayList<TColumnDesc>();
-    }
-    this.columns.add(elem);
-  }
-
-  public List<TColumnDesc> getColumns() {
-    return this.columns;
-  }
-
-  public void setColumns(List<TColumnDesc> columns) {
-    this.columns = columns;
-  }
-
-  public void unsetColumns() {
-    this.columns = null;
-  }
-
-  /** Returns true if field columns is set (has been assigned a value) and false otherwise */
-  public boolean isSetColumns() {
-    return this.columns != null;
-  }
-
-  public void setColumnsIsSet(boolean value) {
-    if (!value) {
-      this.columns = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case COLUMNS:
-      if (value == null) {
-        unsetColumns();
-      } else {
-        setColumns((List<TColumnDesc>)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case COLUMNS:
-      return getColumns();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case COLUMNS:
-      return isSetColumns();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TTableSchema)
-      return this.equals((TTableSchema)that);
-    return false;
-  }
-
-  public boolean equals(TTableSchema that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_columns = true && this.isSetColumns();
-    boolean that_present_columns = true && that.isSetColumns();
-    if (this_present_columns || that_present_columns) {
-      if (!(this_present_columns && that_present_columns))
-        return false;
-      if (!this.columns.equals(that.columns))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_columns = true && (isSetColumns());
-    list.add(present_columns);
-    if (present_columns)
-      list.add(columns);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TTableSchema other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetColumns()).compareTo(other.isSetColumns());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetColumns()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columns, other.columns);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TTableSchema(");
-    boolean first = true;
-
-    sb.append("columns:");
-    if (this.columns == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.columns);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetColumns()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'columns' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TTableSchemaStandardSchemeFactory implements SchemeFactory {
-    public TTableSchemaStandardScheme getScheme() {
-      return new TTableSchemaStandardScheme();
-    }
-  }
-
-  private static class TTableSchemaStandardScheme extends StandardScheme<TTableSchema> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TTableSchema struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // COLUMNS
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list38 = iprot.readListBegin();
-                struct.columns = new ArrayList<TColumnDesc>(_list38.size);
-                TColumnDesc _elem39;
-                for (int _i40 = 0; _i40 < _list38.size; ++_i40)
-                {
-                  _elem39 = new TColumnDesc();
-                  _elem39.read(iprot);
-                  struct.columns.add(_elem39);
-                }
-                iprot.readListEnd();
-              }
-              struct.setColumnsIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TTableSchema struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.columns != null) {
-        oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.columns.size()));
-          for (TColumnDesc _iter41 : struct.columns)
-          {
-            _iter41.write(oprot);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TTableSchemaTupleSchemeFactory implements SchemeFactory {
-    public TTableSchemaTupleScheme getScheme() {
-      return new TTableSchemaTupleScheme();
-    }
-  }
-
-  private static class TTableSchemaTupleScheme extends TupleScheme<TTableSchema> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TTableSchema struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.columns.size());
-        for (TColumnDesc _iter42 : struct.columns)
-        {
-          _iter42.write(oprot);
-        }
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TTableSchema struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TList _list43 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-        struct.columns = new ArrayList<TColumnDesc>(_list43.size);
-        TColumnDesc _elem44;
-        for (int _i45 = 0; _i45 < _list43.size; ++_i45)
-        {
-          _elem44 = new TColumnDesc();
-          _elem44.read(iprot);
-          struct.columns.add(_elem44);
-        }
-      }
-      struct.setColumnsIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TTypeDesc.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TTypeDesc.java
deleted file mode 100644
index 055a14d06a2d6..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TTypeDesc.java
+++ /dev/null
@@ -1,443 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TTypeDesc implements org.apache.thrift.TBase<TTypeDesc, TTypeDesc._Fields>, java.io.Serializable, Cloneable, Comparable<TTypeDesc> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TTypeDesc");
-
-  private static final org.apache.thrift.protocol.TField TYPES_FIELD_DESC = new org.apache.thrift.protocol.TField("types", org.apache.thrift.protocol.TType.LIST, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TTypeDescStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TTypeDescTupleSchemeFactory());
-  }
-
-  private List<TTypeEntry> types; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    TYPES((short)1, "types");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // TYPES
-          return TYPES;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.TYPES, new org.apache.thrift.meta_data.FieldMetaData("types", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
-            new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TTypeEntry.class))));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TTypeDesc.class, metaDataMap);
-  }
-
-  public TTypeDesc() {
-  }
-
-  public TTypeDesc(
-    List<TTypeEntry> types)
-  {
-    this();
-    this.types = types;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TTypeDesc(TTypeDesc other) {
-    if (other.isSetTypes()) {
-      List<TTypeEntry> __this__types = new ArrayList<TTypeEntry>(other.types.size());
-      for (TTypeEntry other_element : other.types) {
-        __this__types.add(new TTypeEntry(other_element));
-      }
-      this.types = __this__types;
-    }
-  }
-
-  public TTypeDesc deepCopy() {
-    return new TTypeDesc(this);
-  }
-
-  @Override
-  public void clear() {
-    this.types = null;
-  }
-
-  public int getTypesSize() {
-    return (this.types == null) ? 0 : this.types.size();
-  }
-
-  public java.util.Iterator<TTypeEntry> getTypesIterator() {
-    return (this.types == null) ? null : this.types.iterator();
-  }
-
-  public void addToTypes(TTypeEntry elem) {
-    if (this.types == null) {
-      this.types = new ArrayList<TTypeEntry>();
-    }
-    this.types.add(elem);
-  }
-
-  public List<TTypeEntry> getTypes() {
-    return this.types;
-  }
-
-  public void setTypes(List<TTypeEntry> types) {
-    this.types = types;
-  }
-
-  public void unsetTypes() {
-    this.types = null;
-  }
-
-  /** Returns true if field types is set (has been assigned a value) and false otherwise */
-  public boolean isSetTypes() {
-    return this.types != null;
-  }
-
-  public void setTypesIsSet(boolean value) {
-    if (!value) {
-      this.types = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case TYPES:
-      if (value == null) {
-        unsetTypes();
-      } else {
-        setTypes((List<TTypeEntry>)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case TYPES:
-      return getTypes();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case TYPES:
-      return isSetTypes();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TTypeDesc)
-      return this.equals((TTypeDesc)that);
-    return false;
-  }
-
-  public boolean equals(TTypeDesc that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_types = true && this.isSetTypes();
-    boolean that_present_types = true && that.isSetTypes();
-    if (this_present_types || that_present_types) {
-      if (!(this_present_types && that_present_types))
-        return false;
-      if (!this.types.equals(that.types))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_types = true && (isSetTypes());
-    list.add(present_types);
-    if (present_types)
-      list.add(types);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TTypeDesc other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetTypes()).compareTo(other.isSetTypes());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetTypes()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.types, other.types);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TTypeDesc(");
-    boolean first = true;
-
-    sb.append("types:");
-    if (this.types == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.types);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetTypes()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'types' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TTypeDescStandardSchemeFactory implements SchemeFactory {
-    public TTypeDescStandardScheme getScheme() {
-      return new TTypeDescStandardScheme();
-    }
-  }
-
-  private static class TTypeDescStandardScheme extends StandardScheme<TTypeDesc> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TTypeDesc struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // TYPES
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list30 = iprot.readListBegin();
-                struct.types = new ArrayList<TTypeEntry>(_list30.size);
-                TTypeEntry _elem31;
-                for (int _i32 = 0; _i32 < _list30.size; ++_i32)
-                {
-                  _elem31 = new TTypeEntry();
-                  _elem31.read(iprot);
-                  struct.types.add(_elem31);
-                }
-                iprot.readListEnd();
-              }
-              struct.setTypesIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TTypeDesc struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.types != null) {
-        oprot.writeFieldBegin(TYPES_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.types.size()));
-          for (TTypeEntry _iter33 : struct.types)
-          {
-            _iter33.write(oprot);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TTypeDescTupleSchemeFactory implements SchemeFactory {
-    public TTypeDescTupleScheme getScheme() {
-      return new TTypeDescTupleScheme();
-    }
-  }
-
-  private static class TTypeDescTupleScheme extends TupleScheme<TTypeDesc> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TTypeDesc struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.types.size());
-        for (TTypeEntry _iter34 : struct.types)
-        {
-          _iter34.write(oprot);
-        }
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TTypeDesc struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TList _list35 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-        struct.types = new ArrayList<TTypeEntry>(_list35.size);
-        TTypeEntry _elem36;
-        for (int _i37 = 0; _i37 < _list35.size; ++_i37)
-        {
-          _elem36 = new TTypeEntry();
-          _elem36.read(iprot);
-          struct.types.add(_elem36);
-        }
-      }
-      struct.setTypesIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TTypeEntry.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TTypeEntry.java
deleted file mode 100644
index b609151b8fbee..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TTypeEntry.java
+++ /dev/null
@@ -1,614 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-public class TTypeEntry extends org.apache.thrift.TUnion<TTypeEntry, TTypeEntry._Fields> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TTypeEntry");
-  private static final org.apache.thrift.protocol.TField PRIMITIVE_ENTRY_FIELD_DESC = new org.apache.thrift.protocol.TField("primitiveEntry", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-  private static final org.apache.thrift.protocol.TField ARRAY_ENTRY_FIELD_DESC = new org.apache.thrift.protocol.TField("arrayEntry", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-  private static final org.apache.thrift.protocol.TField MAP_ENTRY_FIELD_DESC = new org.apache.thrift.protocol.TField("mapEntry", org.apache.thrift.protocol.TType.STRUCT, (short)3);
-  private static final org.apache.thrift.protocol.TField STRUCT_ENTRY_FIELD_DESC = new org.apache.thrift.protocol.TField("structEntry", org.apache.thrift.protocol.TType.STRUCT, (short)4);
-  private static final org.apache.thrift.protocol.TField UNION_ENTRY_FIELD_DESC = new org.apache.thrift.protocol.TField("unionEntry", org.apache.thrift.protocol.TType.STRUCT, (short)5);
-  private static final org.apache.thrift.protocol.TField USER_DEFINED_TYPE_ENTRY_FIELD_DESC = new org.apache.thrift.protocol.TField("userDefinedTypeEntry", org.apache.thrift.protocol.TType.STRUCT, (short)6);
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    PRIMITIVE_ENTRY((short)1, "primitiveEntry"),
-    ARRAY_ENTRY((short)2, "arrayEntry"),
-    MAP_ENTRY((short)3, "mapEntry"),
-    STRUCT_ENTRY((short)4, "structEntry"),
-    UNION_ENTRY((short)5, "unionEntry"),
-    USER_DEFINED_TYPE_ENTRY((short)6, "userDefinedTypeEntry");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // PRIMITIVE_ENTRY
-          return PRIMITIVE_ENTRY;
-        case 2: // ARRAY_ENTRY
-          return ARRAY_ENTRY;
-        case 3: // MAP_ENTRY
-          return MAP_ENTRY;
-        case 4: // STRUCT_ENTRY
-          return STRUCT_ENTRY;
-        case 5: // UNION_ENTRY
-          return UNION_ENTRY;
-        case 6: // USER_DEFINED_TYPE_ENTRY
-          return USER_DEFINED_TYPE_ENTRY;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.PRIMITIVE_ENTRY, new org.apache.thrift.meta_data.FieldMetaData("primitiveEntry", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TPrimitiveTypeEntry.class)));
-    tmpMap.put(_Fields.ARRAY_ENTRY, new org.apache.thrift.meta_data.FieldMetaData("arrayEntry", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TArrayTypeEntry.class)));
-    tmpMap.put(_Fields.MAP_ENTRY, new org.apache.thrift.meta_data.FieldMetaData("mapEntry", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TMapTypeEntry.class)));
-    tmpMap.put(_Fields.STRUCT_ENTRY, new org.apache.thrift.meta_data.FieldMetaData("structEntry", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TStructTypeEntry.class)));
-    tmpMap.put(_Fields.UNION_ENTRY, new org.apache.thrift.meta_data.FieldMetaData("unionEntry", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TUnionTypeEntry.class)));
-    tmpMap.put(_Fields.USER_DEFINED_TYPE_ENTRY, new org.apache.thrift.meta_data.FieldMetaData("userDefinedTypeEntry", org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TUserDefinedTypeEntry.class)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TTypeEntry.class, metaDataMap);
-  }
-
-  public TTypeEntry() {
-    super();
-  }
-
-  public TTypeEntry(TTypeEntry._Fields setField, Object value) {
-    super(setField, value);
-  }
-
-  public TTypeEntry(TTypeEntry other) {
-    super(other);
-  }
-  public TTypeEntry deepCopy() {
-    return new TTypeEntry(this);
-  }
-
-  public static TTypeEntry primitiveEntry(TPrimitiveTypeEntry value) {
-    TTypeEntry x = new TTypeEntry();
-    x.setPrimitiveEntry(value);
-    return x;
-  }
-
-  public static TTypeEntry arrayEntry(TArrayTypeEntry value) {
-    TTypeEntry x = new TTypeEntry();
-    x.setArrayEntry(value);
-    return x;
-  }
-
-  public static TTypeEntry mapEntry(TMapTypeEntry value) {
-    TTypeEntry x = new TTypeEntry();
-    x.setMapEntry(value);
-    return x;
-  }
-
-  public static TTypeEntry structEntry(TStructTypeEntry value) {
-    TTypeEntry x = new TTypeEntry();
-    x.setStructEntry(value);
-    return x;
-  }
-
-  public static TTypeEntry unionEntry(TUnionTypeEntry value) {
-    TTypeEntry x = new TTypeEntry();
-    x.setUnionEntry(value);
-    return x;
-  }
-
-  public static TTypeEntry userDefinedTypeEntry(TUserDefinedTypeEntry value) {
-    TTypeEntry x = new TTypeEntry();
-    x.setUserDefinedTypeEntry(value);
-    return x;
-  }
-
-
-  @Override
-  protected void checkType(_Fields setField, Object value) throws ClassCastException {
-    switch (setField) {
-      case PRIMITIVE_ENTRY:
-        if (value instanceof TPrimitiveTypeEntry) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TPrimitiveTypeEntry for field 'primitiveEntry', but got " + value.getClass().getSimpleName());
-      case ARRAY_ENTRY:
-        if (value instanceof TArrayTypeEntry) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TArrayTypeEntry for field 'arrayEntry', but got " + value.getClass().getSimpleName());
-      case MAP_ENTRY:
-        if (value instanceof TMapTypeEntry) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TMapTypeEntry for field 'mapEntry', but got " + value.getClass().getSimpleName());
-      case STRUCT_ENTRY:
-        if (value instanceof TStructTypeEntry) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TStructTypeEntry for field 'structEntry', but got " + value.getClass().getSimpleName());
-      case UNION_ENTRY:
-        if (value instanceof TUnionTypeEntry) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TUnionTypeEntry for field 'unionEntry', but got " + value.getClass().getSimpleName());
-      case USER_DEFINED_TYPE_ENTRY:
-        if (value instanceof TUserDefinedTypeEntry) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type TUserDefinedTypeEntry for field 'userDefinedTypeEntry', but got " + value.getClass().getSimpleName());
-      default:
-        throw new IllegalArgumentException("Unknown field id " + setField);
-    }
-  }
-
-  @Override
-  protected Object standardSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TField field) throws org.apache.thrift.TException {
-    _Fields setField = _Fields.findByThriftId(field.id);
-    if (setField != null) {
-      switch (setField) {
-        case PRIMITIVE_ENTRY:
-          if (field.type == PRIMITIVE_ENTRY_FIELD_DESC.type) {
-            TPrimitiveTypeEntry primitiveEntry;
-            primitiveEntry = new TPrimitiveTypeEntry();
-            primitiveEntry.read(iprot);
-            return primitiveEntry;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case ARRAY_ENTRY:
-          if (field.type == ARRAY_ENTRY_FIELD_DESC.type) {
-            TArrayTypeEntry arrayEntry;
-            arrayEntry = new TArrayTypeEntry();
-            arrayEntry.read(iprot);
-            return arrayEntry;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case MAP_ENTRY:
-          if (field.type == MAP_ENTRY_FIELD_DESC.type) {
-            TMapTypeEntry mapEntry;
-            mapEntry = new TMapTypeEntry();
-            mapEntry.read(iprot);
-            return mapEntry;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case STRUCT_ENTRY:
-          if (field.type == STRUCT_ENTRY_FIELD_DESC.type) {
-            TStructTypeEntry structEntry;
-            structEntry = new TStructTypeEntry();
-            structEntry.read(iprot);
-            return structEntry;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case UNION_ENTRY:
-          if (field.type == UNION_ENTRY_FIELD_DESC.type) {
-            TUnionTypeEntry unionEntry;
-            unionEntry = new TUnionTypeEntry();
-            unionEntry.read(iprot);
-            return unionEntry;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case USER_DEFINED_TYPE_ENTRY:
-          if (field.type == USER_DEFINED_TYPE_ENTRY_FIELD_DESC.type) {
-            TUserDefinedTypeEntry userDefinedTypeEntry;
-            userDefinedTypeEntry = new TUserDefinedTypeEntry();
-            userDefinedTypeEntry.read(iprot);
-            return userDefinedTypeEntry;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        default:
-          throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!");
-      }
-    } else {
-      org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-      return null;
-    }
-  }
-
-  @Override
-  protected void standardSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    switch (setField_) {
-      case PRIMITIVE_ENTRY:
-        TPrimitiveTypeEntry primitiveEntry = (TPrimitiveTypeEntry)value_;
-        primitiveEntry.write(oprot);
-        return;
-      case ARRAY_ENTRY:
-        TArrayTypeEntry arrayEntry = (TArrayTypeEntry)value_;
-        arrayEntry.write(oprot);
-        return;
-      case MAP_ENTRY:
-        TMapTypeEntry mapEntry = (TMapTypeEntry)value_;
-        mapEntry.write(oprot);
-        return;
-      case STRUCT_ENTRY:
-        TStructTypeEntry structEntry = (TStructTypeEntry)value_;
-        structEntry.write(oprot);
-        return;
-      case UNION_ENTRY:
-        TUnionTypeEntry unionEntry = (TUnionTypeEntry)value_;
-        unionEntry.write(oprot);
-        return;
-      case USER_DEFINED_TYPE_ENTRY:
-        TUserDefinedTypeEntry userDefinedTypeEntry = (TUserDefinedTypeEntry)value_;
-        userDefinedTypeEntry.write(oprot);
-        return;
-      default:
-        throw new IllegalStateException("Cannot write union with unknown field " + setField_);
-    }
-  }
-
-  @Override
-  protected Object tupleSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, short fieldID) throws org.apache.thrift.TException {
-    _Fields setField = _Fields.findByThriftId(fieldID);
-    if (setField != null) {
-      switch (setField) {
-        case PRIMITIVE_ENTRY:
-          TPrimitiveTypeEntry primitiveEntry;
-          primitiveEntry = new TPrimitiveTypeEntry();
-          primitiveEntry.read(iprot);
-          return primitiveEntry;
-        case ARRAY_ENTRY:
-          TArrayTypeEntry arrayEntry;
-          arrayEntry = new TArrayTypeEntry();
-          arrayEntry.read(iprot);
-          return arrayEntry;
-        case MAP_ENTRY:
-          TMapTypeEntry mapEntry;
-          mapEntry = new TMapTypeEntry();
-          mapEntry.read(iprot);
-          return mapEntry;
-        case STRUCT_ENTRY:
-          TStructTypeEntry structEntry;
-          structEntry = new TStructTypeEntry();
-          structEntry.read(iprot);
-          return structEntry;
-        case UNION_ENTRY:
-          TUnionTypeEntry unionEntry;
-          unionEntry = new TUnionTypeEntry();
-          unionEntry.read(iprot);
-          return unionEntry;
-        case USER_DEFINED_TYPE_ENTRY:
-          TUserDefinedTypeEntry userDefinedTypeEntry;
-          userDefinedTypeEntry = new TUserDefinedTypeEntry();
-          userDefinedTypeEntry.read(iprot);
-          return userDefinedTypeEntry;
-        default:
-          throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!");
-      }
-    } else {
-      throw new TProtocolException("Couldn't find a field with field id " + fieldID);
-    }
-  }
-
-  @Override
-  protected void tupleSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    switch (setField_) {
-      case PRIMITIVE_ENTRY:
-        TPrimitiveTypeEntry primitiveEntry = (TPrimitiveTypeEntry)value_;
-        primitiveEntry.write(oprot);
-        return;
-      case ARRAY_ENTRY:
-        TArrayTypeEntry arrayEntry = (TArrayTypeEntry)value_;
-        arrayEntry.write(oprot);
-        return;
-      case MAP_ENTRY:
-        TMapTypeEntry mapEntry = (TMapTypeEntry)value_;
-        mapEntry.write(oprot);
-        return;
-      case STRUCT_ENTRY:
-        TStructTypeEntry structEntry = (TStructTypeEntry)value_;
-        structEntry.write(oprot);
-        return;
-      case UNION_ENTRY:
-        TUnionTypeEntry unionEntry = (TUnionTypeEntry)value_;
-        unionEntry.write(oprot);
-        return;
-      case USER_DEFINED_TYPE_ENTRY:
-        TUserDefinedTypeEntry userDefinedTypeEntry = (TUserDefinedTypeEntry)value_;
-        userDefinedTypeEntry.write(oprot);
-        return;
-      default:
-        throw new IllegalStateException("Cannot write union with unknown field " + setField_);
-    }
-  }
-
-  @Override
-  protected org.apache.thrift.protocol.TField getFieldDesc(_Fields setField) {
-    switch (setField) {
-      case PRIMITIVE_ENTRY:
-        return PRIMITIVE_ENTRY_FIELD_DESC;
-      case ARRAY_ENTRY:
-        return ARRAY_ENTRY_FIELD_DESC;
-      case MAP_ENTRY:
-        return MAP_ENTRY_FIELD_DESC;
-      case STRUCT_ENTRY:
-        return STRUCT_ENTRY_FIELD_DESC;
-      case UNION_ENTRY:
-        return UNION_ENTRY_FIELD_DESC;
-      case USER_DEFINED_TYPE_ENTRY:
-        return USER_DEFINED_TYPE_ENTRY_FIELD_DESC;
-      default:
-        throw new IllegalArgumentException("Unknown field id " + setField);
-    }
-  }
-
-  @Override
-  protected org.apache.thrift.protocol.TStruct getStructDesc() {
-    return STRUCT_DESC;
-  }
-
-  @Override
-  protected _Fields enumForId(short id) {
-    return _Fields.findByThriftIdOrThrow(id);
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-
-  public TPrimitiveTypeEntry getPrimitiveEntry() {
-    if (getSetField() == _Fields.PRIMITIVE_ENTRY) {
-      return (TPrimitiveTypeEntry)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'primitiveEntry' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setPrimitiveEntry(TPrimitiveTypeEntry value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.PRIMITIVE_ENTRY;
-    value_ = value;
-  }
-
-  public TArrayTypeEntry getArrayEntry() {
-    if (getSetField() == _Fields.ARRAY_ENTRY) {
-      return (TArrayTypeEntry)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'arrayEntry' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setArrayEntry(TArrayTypeEntry value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.ARRAY_ENTRY;
-    value_ = value;
-  }
-
-  public TMapTypeEntry getMapEntry() {
-    if (getSetField() == _Fields.MAP_ENTRY) {
-      return (TMapTypeEntry)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'mapEntry' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setMapEntry(TMapTypeEntry value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.MAP_ENTRY;
-    value_ = value;
-  }
-
-  public TStructTypeEntry getStructEntry() {
-    if (getSetField() == _Fields.STRUCT_ENTRY) {
-      return (TStructTypeEntry)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'structEntry' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setStructEntry(TStructTypeEntry value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.STRUCT_ENTRY;
-    value_ = value;
-  }
-
-  public TUnionTypeEntry getUnionEntry() {
-    if (getSetField() == _Fields.UNION_ENTRY) {
-      return (TUnionTypeEntry)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'unionEntry' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setUnionEntry(TUnionTypeEntry value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.UNION_ENTRY;
-    value_ = value;
-  }
-
-  public TUserDefinedTypeEntry getUserDefinedTypeEntry() {
-    if (getSetField() == _Fields.USER_DEFINED_TYPE_ENTRY) {
-      return (TUserDefinedTypeEntry)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'userDefinedTypeEntry' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setUserDefinedTypeEntry(TUserDefinedTypeEntry value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.USER_DEFINED_TYPE_ENTRY;
-    value_ = value;
-  }
-
-  public boolean isSetPrimitiveEntry() {
-    return setField_ == _Fields.PRIMITIVE_ENTRY;
-  }
-
-
-  public boolean isSetArrayEntry() {
-    return setField_ == _Fields.ARRAY_ENTRY;
-  }
-
-
-  public boolean isSetMapEntry() {
-    return setField_ == _Fields.MAP_ENTRY;
-  }
-
-
-  public boolean isSetStructEntry() {
-    return setField_ == _Fields.STRUCT_ENTRY;
-  }
-
-
-  public boolean isSetUnionEntry() {
-    return setField_ == _Fields.UNION_ENTRY;
-  }
-
-
-  public boolean isSetUserDefinedTypeEntry() {
-    return setField_ == _Fields.USER_DEFINED_TYPE_ENTRY;
-  }
-
-
-  public boolean equals(Object other) {
-    if (other instanceof TTypeEntry) {
-      return equals((TTypeEntry)other);
-    } else {
-      return false;
-    }
-  }
-
-  public boolean equals(TTypeEntry other) {
-    return other != null && getSetField() == other.getSetField() && getFieldValue().equals(other.getFieldValue());
-  }
-
-  @Override
-  public int compareTo(TTypeEntry other) {
-    int lastComparison = org.apache.thrift.TBaseHelper.compareTo(getSetField(), other.getSetField());
-    if (lastComparison == 0) {
-      return org.apache.thrift.TBaseHelper.compareTo(getFieldValue(), other.getFieldValue());
-    }
-    return lastComparison;
-  }
-
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-    list.add(this.getClass().getName());
-    org.apache.thrift.TFieldIdEnum setField = getSetField();
-    if (setField != null) {
-      list.add(setField.getThriftFieldId());
-      Object value = getFieldValue();
-      if (value instanceof org.apache.thrift.TEnum) {
-        list.add(((org.apache.thrift.TEnum)getFieldValue()).getValue());
-      } else {
-        list.add(value);
-      }
-    }
-    return list.hashCode();
-  }
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-
-}
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TTypeId.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TTypeId.java
deleted file mode 100644
index a3735ebf3ec07..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TTypeId.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.thrift.TEnum;
-
-public enum TTypeId implements org.apache.thrift.TEnum {
-  BOOLEAN_TYPE(0),
-  TINYINT_TYPE(1),
-  SMALLINT_TYPE(2),
-  INT_TYPE(3),
-  BIGINT_TYPE(4),
-  FLOAT_TYPE(5),
-  DOUBLE_TYPE(6),
-  STRING_TYPE(7),
-  TIMESTAMP_TYPE(8),
-  BINARY_TYPE(9),
-  ARRAY_TYPE(10),
-  MAP_TYPE(11),
-  STRUCT_TYPE(12),
-  UNION_TYPE(13),
-  USER_DEFINED_TYPE(14),
-  DECIMAL_TYPE(15),
-  NULL_TYPE(16),
-  DATE_TYPE(17),
-  VARCHAR_TYPE(18),
-  CHAR_TYPE(19),
-  INTERVAL_YEAR_MONTH_TYPE(20),
-  INTERVAL_DAY_TIME_TYPE(21);
-
-  private final int value;
-
-  private TTypeId(int value) {
-    this.value = value;
-  }
-
-  /**
-   * Get the integer value of this enum value, as defined in the Thrift IDL.
-   */
-  public int getValue() {
-    return value;
-  }
-
-  /**
-   * Find a the enum type by its integer value, as defined in the Thrift IDL.
-   * @return null if the value is not found.
-   */
-  public static TTypeId findByValue(int value) { 
-    switch (value) {
-      case 0:
-        return BOOLEAN_TYPE;
-      case 1:
-        return TINYINT_TYPE;
-      case 2:
-        return SMALLINT_TYPE;
-      case 3:
-        return INT_TYPE;
-      case 4:
-        return BIGINT_TYPE;
-      case 5:
-        return FLOAT_TYPE;
-      case 6:
-        return DOUBLE_TYPE;
-      case 7:
-        return STRING_TYPE;
-      case 8:
-        return TIMESTAMP_TYPE;
-      case 9:
-        return BINARY_TYPE;
-      case 10:
-        return ARRAY_TYPE;
-      case 11:
-        return MAP_TYPE;
-      case 12:
-        return STRUCT_TYPE;
-      case 13:
-        return UNION_TYPE;
-      case 14:
-        return USER_DEFINED_TYPE;
-      case 15:
-        return DECIMAL_TYPE;
-      case 16:
-        return NULL_TYPE;
-      case 17:
-        return DATE_TYPE;
-      case 18:
-        return VARCHAR_TYPE;
-      case 19:
-        return CHAR_TYPE;
-      case 20:
-        return INTERVAL_YEAR_MONTH_TYPE;
-      case 21:
-        return INTERVAL_DAY_TIME_TYPE;
-      default:
-        return null;
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TTypeQualifierValue.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TTypeQualifierValue.java
deleted file mode 100644
index 1720c0e9a72c2..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TTypeQualifierValue.java
+++ /dev/null
@@ -1,365 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-public class TTypeQualifierValue extends org.apache.thrift.TUnion<TTypeQualifierValue, TTypeQualifierValue._Fields> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TTypeQualifierValue");
-  private static final org.apache.thrift.protocol.TField I32_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("i32Value", org.apache.thrift.protocol.TType.I32, (short)1);
-  private static final org.apache.thrift.protocol.TField STRING_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("stringValue", org.apache.thrift.protocol.TType.STRING, (short)2);
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    I32_VALUE((short)1, "i32Value"),
-    STRING_VALUE((short)2, "stringValue");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // I32_VALUE
-          return I32_VALUE;
-        case 2: // STRING_VALUE
-          return STRING_VALUE;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.I32_VALUE, new org.apache.thrift.meta_data.FieldMetaData("i32Value", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
-    tmpMap.put(_Fields.STRING_VALUE, new org.apache.thrift.meta_data.FieldMetaData("stringValue", org.apache.thrift.TFieldRequirementType.OPTIONAL, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TTypeQualifierValue.class, metaDataMap);
-  }
-
-  public TTypeQualifierValue() {
-    super();
-  }
-
-  public TTypeQualifierValue(TTypeQualifierValue._Fields setField, Object value) {
-    super(setField, value);
-  }
-
-  public TTypeQualifierValue(TTypeQualifierValue other) {
-    super(other);
-  }
-  public TTypeQualifierValue deepCopy() {
-    return new TTypeQualifierValue(this);
-  }
-
-  public static TTypeQualifierValue i32Value(int value) {
-    TTypeQualifierValue x = new TTypeQualifierValue();
-    x.setI32Value(value);
-    return x;
-  }
-
-  public static TTypeQualifierValue stringValue(String value) {
-    TTypeQualifierValue x = new TTypeQualifierValue();
-    x.setStringValue(value);
-    return x;
-  }
-
-
-  @Override
-  protected void checkType(_Fields setField, Object value) throws ClassCastException {
-    switch (setField) {
-      case I32_VALUE:
-        if (value instanceof Integer) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type Integer for field 'i32Value', but got " + value.getClass().getSimpleName());
-      case STRING_VALUE:
-        if (value instanceof String) {
-          break;
-        }
-        throw new ClassCastException("Was expecting value of type String for field 'stringValue', but got " + value.getClass().getSimpleName());
-      default:
-        throw new IllegalArgumentException("Unknown field id " + setField);
-    }
-  }
-
-  @Override
-  protected Object standardSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TField field) throws org.apache.thrift.TException {
-    _Fields setField = _Fields.findByThriftId(field.id);
-    if (setField != null) {
-      switch (setField) {
-        case I32_VALUE:
-          if (field.type == I32_VALUE_FIELD_DESC.type) {
-            Integer i32Value;
-            i32Value = iprot.readI32();
-            return i32Value;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        case STRING_VALUE:
-          if (field.type == STRING_VALUE_FIELD_DESC.type) {
-            String stringValue;
-            stringValue = iprot.readString();
-            return stringValue;
-          } else {
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-            return null;
-          }
-        default:
-          throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!");
-      }
-    } else {
-      org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
-      return null;
-    }
-  }
-
-  @Override
-  protected void standardSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    switch (setField_) {
-      case I32_VALUE:
-        Integer i32Value = (Integer)value_;
-        oprot.writeI32(i32Value);
-        return;
-      case STRING_VALUE:
-        String stringValue = (String)value_;
-        oprot.writeString(stringValue);
-        return;
-      default:
-        throw new IllegalStateException("Cannot write union with unknown field " + setField_);
-    }
-  }
-
-  @Override
-  protected Object tupleSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, short fieldID) throws org.apache.thrift.TException {
-    _Fields setField = _Fields.findByThriftId(fieldID);
-    if (setField != null) {
-      switch (setField) {
-        case I32_VALUE:
-          Integer i32Value;
-          i32Value = iprot.readI32();
-          return i32Value;
-        case STRING_VALUE:
-          String stringValue;
-          stringValue = iprot.readString();
-          return stringValue;
-        default:
-          throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!");
-      }
-    } else {
-      throw new TProtocolException("Couldn't find a field with field id " + fieldID);
-    }
-  }
-
-  @Override
-  protected void tupleSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    switch (setField_) {
-      case I32_VALUE:
-        Integer i32Value = (Integer)value_;
-        oprot.writeI32(i32Value);
-        return;
-      case STRING_VALUE:
-        String stringValue = (String)value_;
-        oprot.writeString(stringValue);
-        return;
-      default:
-        throw new IllegalStateException("Cannot write union with unknown field " + setField_);
-    }
-  }
-
-  @Override
-  protected org.apache.thrift.protocol.TField getFieldDesc(_Fields setField) {
-    switch (setField) {
-      case I32_VALUE:
-        return I32_VALUE_FIELD_DESC;
-      case STRING_VALUE:
-        return STRING_VALUE_FIELD_DESC;
-      default:
-        throw new IllegalArgumentException("Unknown field id " + setField);
-    }
-  }
-
-  @Override
-  protected org.apache.thrift.protocol.TStruct getStructDesc() {
-    return STRUCT_DESC;
-  }
-
-  @Override
-  protected _Fields enumForId(short id) {
-    return _Fields.findByThriftIdOrThrow(id);
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-
-  public int getI32Value() {
-    if (getSetField() == _Fields.I32_VALUE) {
-      return (Integer)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'i32Value' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setI32Value(int value) {
-    setField_ = _Fields.I32_VALUE;
-    value_ = value;
-  }
-
-  public String getStringValue() {
-    if (getSetField() == _Fields.STRING_VALUE) {
-      return (String)getFieldValue();
-    } else {
-      throw new RuntimeException("Cannot get field 'stringValue' because union is currently set to " + getFieldDesc(getSetField()).name);
-    }
-  }
-
-  public void setStringValue(String value) {
-    if (value == null) throw new NullPointerException();
-    setField_ = _Fields.STRING_VALUE;
-    value_ = value;
-  }
-
-  public boolean isSetI32Value() {
-    return setField_ == _Fields.I32_VALUE;
-  }
-
-
-  public boolean isSetStringValue() {
-    return setField_ == _Fields.STRING_VALUE;
-  }
-
-
-  public boolean equals(Object other) {
-    if (other instanceof TTypeQualifierValue) {
-      return equals((TTypeQualifierValue)other);
-    } else {
-      return false;
-    }
-  }
-
-  public boolean equals(TTypeQualifierValue other) {
-    return other != null && getSetField() == other.getSetField() && getFieldValue().equals(other.getFieldValue());
-  }
-
-  @Override
-  public int compareTo(TTypeQualifierValue other) {
-    int lastComparison = org.apache.thrift.TBaseHelper.compareTo(getSetField(), other.getSetField());
-    if (lastComparison == 0) {
-      return org.apache.thrift.TBaseHelper.compareTo(getFieldValue(), other.getFieldValue());
-    }
-    return lastComparison;
-  }
-
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-    list.add(this.getClass().getName());
-    org.apache.thrift.TFieldIdEnum setField = getSetField();
-    if (setField != null) {
-      list.add(setField.getThriftFieldId());
-      Object value = getFieldValue();
-      if (value instanceof org.apache.thrift.TEnum) {
-        list.add(((org.apache.thrift.TEnum)getFieldValue()).getValue());
-      } else {
-        list.add(value);
-      }
-    }
-    return list.hashCode();
-  }
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-
-}
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TTypeQualifiers.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TTypeQualifiers.java
deleted file mode 100644
index f46d2ceb79caa..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TTypeQualifiers.java
+++ /dev/null
@@ -1,454 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TTypeQualifiers implements org.apache.thrift.TBase<TTypeQualifiers, TTypeQualifiers._Fields>, java.io.Serializable, Cloneable, Comparable<TTypeQualifiers> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TTypeQualifiers");
-
-  private static final org.apache.thrift.protocol.TField QUALIFIERS_FIELD_DESC = new org.apache.thrift.protocol.TField("qualifiers", org.apache.thrift.protocol.TType.MAP, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TTypeQualifiersStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TTypeQualifiersTupleSchemeFactory());
-  }
-
-  private Map<String,TTypeQualifierValue> qualifiers; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    QUALIFIERS((short)1, "qualifiers");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // QUALIFIERS
-          return QUALIFIERS;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.QUALIFIERS, new org.apache.thrift.meta_data.FieldMetaData("qualifiers", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING), 
-            new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TTypeQualifierValue.class))));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TTypeQualifiers.class, metaDataMap);
-  }
-
-  public TTypeQualifiers() {
-  }
-
-  public TTypeQualifiers(
-    Map<String,TTypeQualifierValue> qualifiers)
-  {
-    this();
-    this.qualifiers = qualifiers;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TTypeQualifiers(TTypeQualifiers other) {
-    if (other.isSetQualifiers()) {
-      Map<String,TTypeQualifierValue> __this__qualifiers = new HashMap<String,TTypeQualifierValue>(other.qualifiers.size());
-      for (Map.Entry<String, TTypeQualifierValue> other_element : other.qualifiers.entrySet()) {
-
-        String other_element_key = other_element.getKey();
-        TTypeQualifierValue other_element_value = other_element.getValue();
-
-        String __this__qualifiers_copy_key = other_element_key;
-
-        TTypeQualifierValue __this__qualifiers_copy_value = new TTypeQualifierValue(other_element_value);
-
-        __this__qualifiers.put(__this__qualifiers_copy_key, __this__qualifiers_copy_value);
-      }
-      this.qualifiers = __this__qualifiers;
-    }
-  }
-
-  public TTypeQualifiers deepCopy() {
-    return new TTypeQualifiers(this);
-  }
-
-  @Override
-  public void clear() {
-    this.qualifiers = null;
-  }
-
-  public int getQualifiersSize() {
-    return (this.qualifiers == null) ? 0 : this.qualifiers.size();
-  }
-
-  public void putToQualifiers(String key, TTypeQualifierValue val) {
-    if (this.qualifiers == null) {
-      this.qualifiers = new HashMap<String,TTypeQualifierValue>();
-    }
-    this.qualifiers.put(key, val);
-  }
-
-  public Map<String,TTypeQualifierValue> getQualifiers() {
-    return this.qualifiers;
-  }
-
-  public void setQualifiers(Map<String,TTypeQualifierValue> qualifiers) {
-    this.qualifiers = qualifiers;
-  }
-
-  public void unsetQualifiers() {
-    this.qualifiers = null;
-  }
-
-  /** Returns true if field qualifiers is set (has been assigned a value) and false otherwise */
-  public boolean isSetQualifiers() {
-    return this.qualifiers != null;
-  }
-
-  public void setQualifiersIsSet(boolean value) {
-    if (!value) {
-      this.qualifiers = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case QUALIFIERS:
-      if (value == null) {
-        unsetQualifiers();
-      } else {
-        setQualifiers((Map<String,TTypeQualifierValue>)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case QUALIFIERS:
-      return getQualifiers();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case QUALIFIERS:
-      return isSetQualifiers();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TTypeQualifiers)
-      return this.equals((TTypeQualifiers)that);
-    return false;
-  }
-
-  public boolean equals(TTypeQualifiers that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_qualifiers = true && this.isSetQualifiers();
-    boolean that_present_qualifiers = true && that.isSetQualifiers();
-    if (this_present_qualifiers || that_present_qualifiers) {
-      if (!(this_present_qualifiers && that_present_qualifiers))
-        return false;
-      if (!this.qualifiers.equals(that.qualifiers))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_qualifiers = true && (isSetQualifiers());
-    list.add(present_qualifiers);
-    if (present_qualifiers)
-      list.add(qualifiers);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TTypeQualifiers other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetQualifiers()).compareTo(other.isSetQualifiers());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetQualifiers()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.qualifiers, other.qualifiers);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TTypeQualifiers(");
-    boolean first = true;
-
-    sb.append("qualifiers:");
-    if (this.qualifiers == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.qualifiers);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetQualifiers()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'qualifiers' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TTypeQualifiersStandardSchemeFactory implements SchemeFactory {
-    public TTypeQualifiersStandardScheme getScheme() {
-      return new TTypeQualifiersStandardScheme();
-    }
-  }
-
-  private static class TTypeQualifiersStandardScheme extends StandardScheme<TTypeQualifiers> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TTypeQualifiers struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // QUALIFIERS
-            if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
-              {
-                org.apache.thrift.protocol.TMap _map0 = iprot.readMapBegin();
-                struct.qualifiers = new HashMap<String,TTypeQualifierValue>(2*_map0.size);
-                String _key1;
-                TTypeQualifierValue _val2;
-                for (int _i3 = 0; _i3 < _map0.size; ++_i3)
-                {
-                  _key1 = iprot.readString();
-                  _val2 = new TTypeQualifierValue();
-                  _val2.read(iprot);
-                  struct.qualifiers.put(_key1, _val2);
-                }
-                iprot.readMapEnd();
-              }
-              struct.setQualifiersIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TTypeQualifiers struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.qualifiers != null) {
-        oprot.writeFieldBegin(QUALIFIERS_FIELD_DESC);
-        {
-          oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, struct.qualifiers.size()));
-          for (Map.Entry<String, TTypeQualifierValue> _iter4 : struct.qualifiers.entrySet())
-          {
-            oprot.writeString(_iter4.getKey());
-            _iter4.getValue().write(oprot);
-          }
-          oprot.writeMapEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TTypeQualifiersTupleSchemeFactory implements SchemeFactory {
-    public TTypeQualifiersTupleScheme getScheme() {
-      return new TTypeQualifiersTupleScheme();
-    }
-  }
-
-  private static class TTypeQualifiersTupleScheme extends TupleScheme<TTypeQualifiers> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TTypeQualifiers struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.qualifiers.size());
-        for (Map.Entry<String, TTypeQualifierValue> _iter5 : struct.qualifiers.entrySet())
-        {
-          oprot.writeString(_iter5.getKey());
-          _iter5.getValue().write(oprot);
-        }
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TTypeQualifiers struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TMap _map6 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
-        struct.qualifiers = new HashMap<String,TTypeQualifierValue>(2*_map6.size);
-        String _key7;
-        TTypeQualifierValue _val8;
-        for (int _i9 = 0; _i9 < _map6.size; ++_i9)
-        {
-          _key7 = iprot.readString();
-          _val8 = new TTypeQualifierValue();
-          _val8.read(iprot);
-          struct.qualifiers.put(_key7, _val8);
-        }
-      }
-      struct.setQualifiersIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TUnionTypeEntry.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TUnionTypeEntry.java
deleted file mode 100644
index d53f74cb8eff1..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TUnionTypeEntry.java
+++ /dev/null
@@ -1,452 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TUnionTypeEntry implements org.apache.thrift.TBase<TUnionTypeEntry, TUnionTypeEntry._Fields>, java.io.Serializable, Cloneable, Comparable<TUnionTypeEntry> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TUnionTypeEntry");
-
-  private static final org.apache.thrift.protocol.TField NAME_TO_TYPE_PTR_FIELD_DESC = new org.apache.thrift.protocol.TField("nameToTypePtr", org.apache.thrift.protocol.TType.MAP, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TUnionTypeEntryStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TUnionTypeEntryTupleSchemeFactory());
-  }
-
-  private Map<String,Integer> nameToTypePtr; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    NAME_TO_TYPE_PTR((short)1, "nameToTypePtr");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // NAME_TO_TYPE_PTR
-          return NAME_TO_TYPE_PTR;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.NAME_TO_TYPE_PTR, new org.apache.thrift.meta_data.FieldMetaData("nameToTypePtr", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING), 
-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32            , "TTypeEntryPtr"))));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TUnionTypeEntry.class, metaDataMap);
-  }
-
-  public TUnionTypeEntry() {
-  }
-
-  public TUnionTypeEntry(
-    Map<String,Integer> nameToTypePtr)
-  {
-    this();
-    this.nameToTypePtr = nameToTypePtr;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TUnionTypeEntry(TUnionTypeEntry other) {
-    if (other.isSetNameToTypePtr()) {
-      Map<String,Integer> __this__nameToTypePtr = new HashMap<String,Integer>(other.nameToTypePtr.size());
-      for (Map.Entry<String, Integer> other_element : other.nameToTypePtr.entrySet()) {
-
-        String other_element_key = other_element.getKey();
-        Integer other_element_value = other_element.getValue();
-
-        String __this__nameToTypePtr_copy_key = other_element_key;
-
-        Integer __this__nameToTypePtr_copy_value = other_element_value;
-
-        __this__nameToTypePtr.put(__this__nameToTypePtr_copy_key, __this__nameToTypePtr_copy_value);
-      }
-      this.nameToTypePtr = __this__nameToTypePtr;
-    }
-  }
-
-  public TUnionTypeEntry deepCopy() {
-    return new TUnionTypeEntry(this);
-  }
-
-  @Override
-  public void clear() {
-    this.nameToTypePtr = null;
-  }
-
-  public int getNameToTypePtrSize() {
-    return (this.nameToTypePtr == null) ? 0 : this.nameToTypePtr.size();
-  }
-
-  public void putToNameToTypePtr(String key, int val) {
-    if (this.nameToTypePtr == null) {
-      this.nameToTypePtr = new HashMap<String,Integer>();
-    }
-    this.nameToTypePtr.put(key, val);
-  }
-
-  public Map<String,Integer> getNameToTypePtr() {
-    return this.nameToTypePtr;
-  }
-
-  public void setNameToTypePtr(Map<String,Integer> nameToTypePtr) {
-    this.nameToTypePtr = nameToTypePtr;
-  }
-
-  public void unsetNameToTypePtr() {
-    this.nameToTypePtr = null;
-  }
-
-  /** Returns true if field nameToTypePtr is set (has been assigned a value) and false otherwise */
-  public boolean isSetNameToTypePtr() {
-    return this.nameToTypePtr != null;
-  }
-
-  public void setNameToTypePtrIsSet(boolean value) {
-    if (!value) {
-      this.nameToTypePtr = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case NAME_TO_TYPE_PTR:
-      if (value == null) {
-        unsetNameToTypePtr();
-      } else {
-        setNameToTypePtr((Map<String,Integer>)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case NAME_TO_TYPE_PTR:
-      return getNameToTypePtr();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case NAME_TO_TYPE_PTR:
-      return isSetNameToTypePtr();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TUnionTypeEntry)
-      return this.equals((TUnionTypeEntry)that);
-    return false;
-  }
-
-  public boolean equals(TUnionTypeEntry that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_nameToTypePtr = true && this.isSetNameToTypePtr();
-    boolean that_present_nameToTypePtr = true && that.isSetNameToTypePtr();
-    if (this_present_nameToTypePtr || that_present_nameToTypePtr) {
-      if (!(this_present_nameToTypePtr && that_present_nameToTypePtr))
-        return false;
-      if (!this.nameToTypePtr.equals(that.nameToTypePtr))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_nameToTypePtr = true && (isSetNameToTypePtr());
-    list.add(present_nameToTypePtr);
-    if (present_nameToTypePtr)
-      list.add(nameToTypePtr);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TUnionTypeEntry other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetNameToTypePtr()).compareTo(other.isSetNameToTypePtr());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetNameToTypePtr()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.nameToTypePtr, other.nameToTypePtr);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TUnionTypeEntry(");
-    boolean first = true;
-
-    sb.append("nameToTypePtr:");
-    if (this.nameToTypePtr == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.nameToTypePtr);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetNameToTypePtr()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'nameToTypePtr' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TUnionTypeEntryStandardSchemeFactory implements SchemeFactory {
-    public TUnionTypeEntryStandardScheme getScheme() {
-      return new TUnionTypeEntryStandardScheme();
-    }
-  }
-
-  private static class TUnionTypeEntryStandardScheme extends StandardScheme<TUnionTypeEntry> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TUnionTypeEntry struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // NAME_TO_TYPE_PTR
-            if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
-              {
-                org.apache.thrift.protocol.TMap _map20 = iprot.readMapBegin();
-                struct.nameToTypePtr = new HashMap<String,Integer>(2*_map20.size);
-                String _key21;
-                int _val22;
-                for (int _i23 = 0; _i23 < _map20.size; ++_i23)
-                {
-                  _key21 = iprot.readString();
-                  _val22 = iprot.readI32();
-                  struct.nameToTypePtr.put(_key21, _val22);
-                }
-                iprot.readMapEnd();
-              }
-              struct.setNameToTypePtrIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TUnionTypeEntry struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.nameToTypePtr != null) {
-        oprot.writeFieldBegin(NAME_TO_TYPE_PTR_FIELD_DESC);
-        {
-          oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.I32, struct.nameToTypePtr.size()));
-          for (Map.Entry<String, Integer> _iter24 : struct.nameToTypePtr.entrySet())
-          {
-            oprot.writeString(_iter24.getKey());
-            oprot.writeI32(_iter24.getValue());
-          }
-          oprot.writeMapEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TUnionTypeEntryTupleSchemeFactory implements SchemeFactory {
-    public TUnionTypeEntryTupleScheme getScheme() {
-      return new TUnionTypeEntryTupleScheme();
-    }
-  }
-
-  private static class TUnionTypeEntryTupleScheme extends TupleScheme<TUnionTypeEntry> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TUnionTypeEntry struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      {
-        oprot.writeI32(struct.nameToTypePtr.size());
-        for (Map.Entry<String, Integer> _iter25 : struct.nameToTypePtr.entrySet())
-        {
-          oprot.writeString(_iter25.getKey());
-          oprot.writeI32(_iter25.getValue());
-        }
-      }
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TUnionTypeEntry struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      {
-        org.apache.thrift.protocol.TMap _map26 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.I32, iprot.readI32());
-        struct.nameToTypePtr = new HashMap<String,Integer>(2*_map26.size);
-        String _key27;
-        int _val28;
-        for (int _i29 = 0; _i29 < _map26.size; ++_i29)
-        {
-          _key27 = iprot.readString();
-          _val28 = iprot.readI32();
-          struct.nameToTypePtr.put(_key27, _val28);
-        }
-      }
-      struct.setNameToTypePtrIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TUserDefinedTypeEntry.java b/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TUserDefinedTypeEntry.java
deleted file mode 100644
index b80c4dd5c6302..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TUserDefinedTypeEntry.java
+++ /dev/null
@@ -1,389 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hive.service.rpc.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-public class TUserDefinedTypeEntry implements org.apache.thrift.TBase<TUserDefinedTypeEntry, TUserDefinedTypeEntry._Fields>, java.io.Serializable, Cloneable, Comparable<TUserDefinedTypeEntry> {
-  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TUserDefinedTypeEntry");
-
-  private static final org.apache.thrift.protocol.TField TYPE_CLASS_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("typeClassName", org.apache.thrift.protocol.TType.STRING, (short)1);
-
-  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
-  static {
-    schemes.put(StandardScheme.class, new TUserDefinedTypeEntryStandardSchemeFactory());
-    schemes.put(TupleScheme.class, new TUserDefinedTypeEntryTupleSchemeFactory());
-  }
-
-  private String typeClassName; // required
-
-  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
-  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-    TYPE_CLASS_NAME((short)1, "typeClassName");
-
-    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
-    static {
-      for (_Fields field : EnumSet.allOf(_Fields.class)) {
-        byName.put(field.getFieldName(), field);
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, or null if its not found.
-     */
-    public static _Fields findByThriftId(int fieldId) {
-      switch(fieldId) {
-        case 1: // TYPE_CLASS_NAME
-          return TYPE_CLASS_NAME;
-        default:
-          return null;
-      }
-    }
-
-    /**
-     * Find the _Fields constant that matches fieldId, throwing an exception
-     * if it is not found.
-     */
-    public static _Fields findByThriftIdOrThrow(int fieldId) {
-      _Fields fields = findByThriftId(fieldId);
-      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
-      return fields;
-    }
-
-    /**
-     * Find the _Fields constant that matches name, or null if its not found.
-     */
-    public static _Fields findByName(String name) {
-      return byName.get(name);
-    }
-
-    private final short _thriftId;
-    private final String _fieldName;
-
-    _Fields(short thriftId, String fieldName) {
-      _thriftId = thriftId;
-      _fieldName = fieldName;
-    }
-
-    public short getThriftFieldId() {
-      return _thriftId;
-    }
-
-    public String getFieldName() {
-      return _fieldName;
-    }
-  }
-
-  // isset id assignments
-  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
-  static {
-    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-    tmpMap.put(_Fields.TYPE_CLASS_NAME, new org.apache.thrift.meta_data.FieldMetaData("typeClassName", org.apache.thrift.TFieldRequirementType.REQUIRED, 
-        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-    metaDataMap = Collections.unmodifiableMap(tmpMap);
-    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TUserDefinedTypeEntry.class, metaDataMap);
-  }
-
-  public TUserDefinedTypeEntry() {
-  }
-
-  public TUserDefinedTypeEntry(
-    String typeClassName)
-  {
-    this();
-    this.typeClassName = typeClassName;
-  }
-
-  /**
-   * Performs a deep copy on <i>other</i>.
-   */
-  public TUserDefinedTypeEntry(TUserDefinedTypeEntry other) {
-    if (other.isSetTypeClassName()) {
-      this.typeClassName = other.typeClassName;
-    }
-  }
-
-  public TUserDefinedTypeEntry deepCopy() {
-    return new TUserDefinedTypeEntry(this);
-  }
-
-  @Override
-  public void clear() {
-    this.typeClassName = null;
-  }
-
-  public String getTypeClassName() {
-    return this.typeClassName;
-  }
-
-  public void setTypeClassName(String typeClassName) {
-    this.typeClassName = typeClassName;
-  }
-
-  public void unsetTypeClassName() {
-    this.typeClassName = null;
-  }
-
-  /** Returns true if field typeClassName is set (has been assigned a value) and false otherwise */
-  public boolean isSetTypeClassName() {
-    return this.typeClassName != null;
-  }
-
-  public void setTypeClassNameIsSet(boolean value) {
-    if (!value) {
-      this.typeClassName = null;
-    }
-  }
-
-  public void setFieldValue(_Fields field, Object value) {
-    switch (field) {
-    case TYPE_CLASS_NAME:
-      if (value == null) {
-        unsetTypeClassName();
-      } else {
-        setTypeClassName((String)value);
-      }
-      break;
-
-    }
-  }
-
-  public Object getFieldValue(_Fields field) {
-    switch (field) {
-    case TYPE_CLASS_NAME:
-      return getTypeClassName();
-
-    }
-    throw new IllegalStateException();
-  }
-
-  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
-  public boolean isSet(_Fields field) {
-    if (field == null) {
-      throw new IllegalArgumentException();
-    }
-
-    switch (field) {
-    case TYPE_CLASS_NAME:
-      return isSetTypeClassName();
-    }
-    throw new IllegalStateException();
-  }
-
-  @Override
-  public boolean equals(Object that) {
-    if (that == null)
-      return false;
-    if (that instanceof TUserDefinedTypeEntry)
-      return this.equals((TUserDefinedTypeEntry)that);
-    return false;
-  }
-
-  public boolean equals(TUserDefinedTypeEntry that) {
-    if (that == null)
-      return false;
-
-    boolean this_present_typeClassName = true && this.isSetTypeClassName();
-    boolean that_present_typeClassName = true && that.isSetTypeClassName();
-    if (this_present_typeClassName || that_present_typeClassName) {
-      if (!(this_present_typeClassName && that_present_typeClassName))
-        return false;
-      if (!this.typeClassName.equals(that.typeClassName))
-        return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    List<Object> list = new ArrayList<Object>();
-
-    boolean present_typeClassName = true && (isSetTypeClassName());
-    list.add(present_typeClassName);
-    if (present_typeClassName)
-      list.add(typeClassName);
-
-    return list.hashCode();
-  }
-
-  @Override
-  public int compareTo(TUserDefinedTypeEntry other) {
-    if (!getClass().equals(other.getClass())) {
-      return getClass().getName().compareTo(other.getClass().getName());
-    }
-
-    int lastComparison = 0;
-
-    lastComparison = Boolean.valueOf(isSetTypeClassName()).compareTo(other.isSetTypeClassName());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetTypeClassName()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.typeClassName, other.typeClassName);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    return 0;
-  }
-
-  public _Fields fieldForId(int fieldId) {
-    return _Fields.findByThriftId(fieldId);
-  }
-
-  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
-    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
-  }
-
-  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
-    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("TUserDefinedTypeEntry(");
-    boolean first = true;
-
-    sb.append("typeClassName:");
-    if (this.typeClassName == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.typeClassName);
-    }
-    first = false;
-    sb.append(")");
-    return sb.toString();
-  }
-
-  public void validate() throws org.apache.thrift.TException {
-    // check for required fields
-    if (!isSetTypeClassName()) {
-      throw new org.apache.thrift.protocol.TProtocolException("Required field 'typeClassName' is unset! Struct:" + toString());
-    }
-
-    // check for sub-struct validity
-  }
-
-  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
-    try {
-      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
-    try {
-      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
-    } catch (org.apache.thrift.TException te) {
-      throw new java.io.IOException(te);
-    }
-  }
-
-  private static class TUserDefinedTypeEntryStandardSchemeFactory implements SchemeFactory {
-    public TUserDefinedTypeEntryStandardScheme getScheme() {
-      return new TUserDefinedTypeEntryStandardScheme();
-    }
-  }
-
-  private static class TUserDefinedTypeEntryStandardScheme extends StandardScheme<TUserDefinedTypeEntry> {
-
-    public void read(org.apache.thrift.protocol.TProtocol iprot, TUserDefinedTypeEntry struct) throws org.apache.thrift.TException {
-      org.apache.thrift.protocol.TField schemeField;
-      iprot.readStructBegin();
-      while (true)
-      {
-        schemeField = iprot.readFieldBegin();
-        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
-          break;
-        }
-        switch (schemeField.id) {
-          case 1: // TYPE_CLASS_NAME
-            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-              struct.typeClassName = iprot.readString();
-              struct.setTypeClassNameIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          default:
-            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-        }
-        iprot.readFieldEnd();
-      }
-      iprot.readStructEnd();
-      struct.validate();
-    }
-
-    public void write(org.apache.thrift.protocol.TProtocol oprot, TUserDefinedTypeEntry struct) throws org.apache.thrift.TException {
-      struct.validate();
-
-      oprot.writeStructBegin(STRUCT_DESC);
-      if (struct.typeClassName != null) {
-        oprot.writeFieldBegin(TYPE_CLASS_NAME_FIELD_DESC);
-        oprot.writeString(struct.typeClassName);
-        oprot.writeFieldEnd();
-      }
-      oprot.writeFieldStop();
-      oprot.writeStructEnd();
-    }
-
-  }
-
-  private static class TUserDefinedTypeEntryTupleSchemeFactory implements SchemeFactory {
-    public TUserDefinedTypeEntryTupleScheme getScheme() {
-      return new TUserDefinedTypeEntryTupleScheme();
-    }
-  }
-
-  private static class TUserDefinedTypeEntryTupleScheme extends TupleScheme<TUserDefinedTypeEntry> {
-
-    @Override
-    public void write(org.apache.thrift.protocol.TProtocol prot, TUserDefinedTypeEntry struct) throws org.apache.thrift.TException {
-      TTupleProtocol oprot = (TTupleProtocol) prot;
-      oprot.writeString(struct.typeClassName);
-    }
-
-    @Override
-    public void read(org.apache.thrift.protocol.TProtocol prot, TUserDefinedTypeEntry struct) throws org.apache.thrift.TException {
-      TTupleProtocol iprot = (TTupleProtocol) prot;
-      struct.typeClassName = iprot.readString();
-      struct.setTypeClassNameIsSet(true);
-    }
-  }
-
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/CLIService.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/CLIService.java
deleted file mode 100644
index bdc1e6251e560..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/CLIService.java
+++ /dev/null
@@ -1,573 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CancellationException;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import javax.security.auth.login.LoginException;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
-import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.shims.Utils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hive.service.CompositeService;
-import org.apache.hive.service.ServiceException;
-import org.apache.hive.service.auth.HiveAuthFactory;
-import org.apache.hive.service.cli.operation.Operation;
-import org.apache.hive.service.cli.session.HiveSession;
-import org.apache.hive.service.cli.session.SessionManager;
-import org.apache.hive.service.rpc.thrift.TProtocolVersion;
-import org.apache.hive.service.server.HiveServer2;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * CLIService.
- *
- */
-public class CLIService extends CompositeService implements ICLIService {
-
-  public static final TProtocolVersion SERVER_VERSION;
-
-  static {
-    TProtocolVersion[] protocols = TProtocolVersion.values();
-    SERVER_VERSION = protocols[protocols.length - 1];
-  }
-
-  private final Logger LOG = LoggerFactory.getLogger(CLIService.class.getName());
-
-  private HiveConf hiveConf;
-  private SessionManager sessionManager;
-  private UserGroupInformation serviceUGI;
-  private UserGroupInformation httpUGI;
-  // The HiveServer2 instance running this service
-  private final HiveServer2 hiveServer2;
-
-  public CLIService(HiveServer2 hiveServer2) {
-    super(CLIService.class.getSimpleName());
-    this.hiveServer2 = hiveServer2;
-  }
-
-  @Override
-  public synchronized void init(HiveConf hiveConf) {
-    this.hiveConf = hiveConf;
-    sessionManager = new SessionManager(hiveServer2);
-    addService(sessionManager);
-    //  If the hadoop cluster is secure, do a kerberos login for the service from the keytab
-    if (UserGroupInformation.isSecurityEnabled()) {
-      try {
-        HiveAuthFactory.loginFromKeytab(hiveConf);
-        this.serviceUGI = Utils.getUGI();
-      } catch (IOException e) {
-        throw new ServiceException("Unable to login to kerberos with given principal/keytab", e);
-      } catch (LoginException e) {
-        throw new ServiceException("Unable to login to kerberos with given principal/keytab", e);
-      }
-
-      // Also try creating a UGI object for the SPNego principal
-      String principal = hiveConf.getVar(ConfVars.HIVE_SERVER2_SPNEGO_PRINCIPAL);
-      String keyTabFile = hiveConf.getVar(ConfVars.HIVE_SERVER2_SPNEGO_KEYTAB);
-      if (principal.isEmpty() || keyTabFile.isEmpty()) {
-        LOG.info("SPNego httpUGI not created, spNegoPrincipal: " + principal +
-            ", ketabFile: " + keyTabFile);
-      } else {
-        try {
-          this.httpUGI = HiveAuthFactory.loginFromSpnegoKeytabAndReturnUGI(hiveConf);
-          LOG.info("SPNego httpUGI successfully created.");
-        } catch (IOException e) {
-          LOG.warn("SPNego httpUGI creation failed: ", e);
-        }
-      }
-    }
-    // creates connection to HMS and thus *must* occur after kerberos login above
-    try {
-      applyAuthorizationConfigPolicy(hiveConf);
-    } catch (Exception e) {
-      throw new RuntimeException("Error applying authorization policy on hive configuration: "
-          + e.getMessage(), e);
-    }
-    setupBlockedUdfs();
-    super.init(hiveConf);
-  }
-
-  private void applyAuthorizationConfigPolicy(HiveConf newHiveConf) throws HiveException,
-      MetaException {
-    // authorization setup using SessionState should be revisited eventually, as
-    // authorization and authentication are not session specific settings
-    SessionState ss = new SessionState(newHiveConf);
-    ss.setIsHiveServerQuery(true);
-    SessionState.start(ss);
-    ss.applyAuthorizationPolicy();
-  }
-
-  private void setupBlockedUdfs() {
-    FunctionRegistry.setupPermissionsForBuiltinUDFs(
-        hiveConf.getVar(ConfVars.HIVE_SERVER2_BUILTIN_UDF_WHITELIST),
-        hiveConf.getVar(ConfVars.HIVE_SERVER2_BUILTIN_UDF_BLACKLIST));
-  }
-
-  public UserGroupInformation getServiceUGI() {
-    return this.serviceUGI;
-  }
-
-  public UserGroupInformation getHttpUGI() {
-    return this.httpUGI;
-  }
-
-  @Override
-  public synchronized void start() {
-    super.start();
-    // Initialize and test a connection to the metastore
-    IMetaStoreClient metastoreClient = null;
-    try {
-      metastoreClient = new HiveMetaStoreClient(hiveConf);
-      metastoreClient.getDatabases("default");
-    } catch (Exception e) {
-      throw new ServiceException("Unable to connect to MetaStore!", e);
-    }
-    finally {
-      if (metastoreClient != null) {
-        metastoreClient.close();
-      }
-    }
-  }
-
-  @Override
-  public synchronized void stop() {
-    super.stop();
-  }
-
-  /**
-   * @deprecated  Use {@link #openSession(TProtocolVersion, String, String, String, Map)}
-   */
-  @Deprecated
-  public SessionHandle openSession(TProtocolVersion protocol, String username, String password,
-      Map<String, String> configuration) throws HiveSQLException {
-    SessionHandle sessionHandle = sessionManager.openSession(protocol, username, password, null, configuration, false, null);
-    LOG.debug(sessionHandle + ": openSession()");
-    return sessionHandle;
-  }
-
-  /**
-   * @deprecated  Use {@link #openSessionWithImpersonation(TProtocolVersion, String, String, String, Map, String)}
-   */
-  @Deprecated
-  public SessionHandle openSessionWithImpersonation(TProtocolVersion protocol, String username,
-      String password, Map<String, String> configuration, String delegationToken)
-          throws HiveSQLException {
-    SessionHandle sessionHandle = sessionManager.openSession(protocol, username, password, null, configuration,
-        true, delegationToken);
-    LOG.debug(sessionHandle + ": openSessionWithImpersonation()");
-    return sessionHandle;
-  }
-
-  public SessionHandle openSession(TProtocolVersion protocol, String username, String password, String ipAddress,
-      Map<String, String> configuration) throws HiveSQLException {
-    SessionHandle sessionHandle = sessionManager.openSession(protocol, username, password, ipAddress, configuration, false, null);
-    LOG.debug(sessionHandle + ": openSession()");
-    return sessionHandle;
-  }
-
-  public SessionHandle openSessionWithImpersonation(TProtocolVersion protocol, String username,
-      String password, String ipAddress, Map<String, String> configuration, String delegationToken)
-          throws HiveSQLException {
-    SessionHandle sessionHandle = sessionManager.openSession(protocol, username, password, ipAddress, configuration,
-        true, delegationToken);
-    LOG.debug(sessionHandle + ": openSession()");
-    return sessionHandle;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#openSession(java.lang.String, java.lang.String, java.util.Map)
-   */
-  @Override
-  public SessionHandle openSession(String username, String password, Map<String, String> configuration)
-      throws HiveSQLException {
-    SessionHandle sessionHandle = sessionManager.openSession(SERVER_VERSION, username, password, null, configuration, false, null);
-    LOG.debug(sessionHandle + ": openSession()");
-    return sessionHandle;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#openSession(java.lang.String, java.lang.String, java.util.Map)
-   */
-  @Override
-  public SessionHandle openSessionWithImpersonation(String username, String password, Map<String, String> configuration,
-      String delegationToken) throws HiveSQLException {
-    SessionHandle sessionHandle = sessionManager.openSession(SERVER_VERSION, username, password, null, configuration,
-        true, delegationToken);
-    LOG.debug(sessionHandle + ": openSession()");
-    return sessionHandle;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#closeSession(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public void closeSession(SessionHandle sessionHandle)
-      throws HiveSQLException {
-    sessionManager.closeSession(sessionHandle);
-    LOG.debug(sessionHandle + ": closeSession()");
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getInfo(org.apache.hive.service.cli.SessionHandle, java.util.List)
-   */
-  @Override
-  public GetInfoValue getInfo(SessionHandle sessionHandle, GetInfoType getInfoType)
-      throws HiveSQLException {
-    GetInfoValue infoValue = sessionManager.getSession(sessionHandle)
-        .getInfo(getInfoType);
-    LOG.debug(sessionHandle + ": getInfo()");
-    return infoValue;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#executeStatement(org.apache.hive.service.cli.SessionHandle,
-   *  java.lang.String, java.util.Map)
-   */
-  @Override
-  public OperationHandle executeStatement(SessionHandle sessionHandle, String statement,
-      Map<String, String> confOverlay) throws HiveSQLException {
-    HiveSession session = sessionManager.getSession(sessionHandle);
-    // need to reset the monitor, as operation handle is not available down stream, Ideally the
-    // monitor should be associated with the operation handle.
-    session.getSessionState().updateProgressMonitor(null);
-    OperationHandle opHandle = session.executeStatement(statement, confOverlay);
-    LOG.debug(sessionHandle + ": executeStatement()");
-    return opHandle;
-  }
-
-  /**
-   * Execute statement on the server with a timeout. This is a blocking call.
-   */
-  @Override
-  public OperationHandle executeStatement(SessionHandle sessionHandle, String statement,
-        Map<String, String> confOverlay, long queryTimeout) throws HiveSQLException {
-    HiveSession session = sessionManager.getSession(sessionHandle);
-    // need to reset the monitor, as operation handle is not available down stream, Ideally the
-    // monitor should be associated with the operation handle.
-    session.getSessionState().updateProgressMonitor(null);
-    OperationHandle opHandle = session.executeStatement(statement, confOverlay, queryTimeout);
-    LOG.debug(sessionHandle + ": executeStatement()");
-    return opHandle;
-  }
-
-  /**
-   * Execute statement asynchronously on the server. This is a non-blocking call
-   */
-  @Override
-  public OperationHandle executeStatementAsync(SessionHandle sessionHandle, String statement,
-      Map<String, String> confOverlay) throws HiveSQLException {
-    HiveSession session = sessionManager.getSession(sessionHandle);
-    // need to reset the monitor, as operation handle is not available down stream, Ideally the
-    // monitor should be associated with the operation handle.
-    session.getSessionState().updateProgressMonitor(null);
-    OperationHandle opHandle = session.executeStatementAsync(statement, confOverlay);
-    LOG.debug(sessionHandle + ": executeStatementAsync()");
-    return opHandle;
-  }
-
-  /**
-   * Execute statement asynchronously on the server with a timeout. This is a non-blocking call
-   */
-  @Override
-  public OperationHandle executeStatementAsync(SessionHandle sessionHandle, String statement,
-      Map<String, String> confOverlay, long queryTimeout) throws HiveSQLException {
-    HiveSession session = sessionManager.getSession(sessionHandle);
-    // need to reset the monitor, as operation handle is not available down stream, Ideally the
-    // monitor should be associated with the operation handle.
-    session.getSessionState().updateProgressMonitor(null);
-    OperationHandle opHandle = session.executeStatementAsync(statement, confOverlay, queryTimeout);
-    LOG.debug(sessionHandle + ": executeStatementAsync()");
-    return opHandle;
-  }
-
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getTypeInfo(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public OperationHandle getTypeInfo(SessionHandle sessionHandle)
-      throws HiveSQLException {
-    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
-        .getTypeInfo();
-    LOG.debug(sessionHandle + ": getTypeInfo()");
-    return opHandle;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getCatalogs(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public OperationHandle getCatalogs(SessionHandle sessionHandle)
-      throws HiveSQLException {
-    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
-        .getCatalogs();
-    LOG.debug(sessionHandle + ": getCatalogs()");
-    return opHandle;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getSchemas(org.apache.hive.service.cli.SessionHandle, java.lang.String, java.lang.String)
-   */
-  @Override
-  public OperationHandle getSchemas(SessionHandle sessionHandle,
-      String catalogName, String schemaName)
-          throws HiveSQLException {
-    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
-        .getSchemas(catalogName, schemaName);
-    LOG.debug(sessionHandle + ": getSchemas()");
-    return opHandle;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getTables(org.apache.hive.service.cli.SessionHandle, java.lang.String, java.lang.String, java.lang.String, java.util.List)
-   */
-  @Override
-  public OperationHandle getTables(SessionHandle sessionHandle,
-      String catalogName, String schemaName, String tableName, List<String> tableTypes)
-          throws HiveSQLException {
-    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
-        .getTables(catalogName, schemaName, tableName, tableTypes);
-    LOG.debug(sessionHandle + ": getTables()");
-    return opHandle;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getTableTypes(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public OperationHandle getTableTypes(SessionHandle sessionHandle)
-      throws HiveSQLException {
-    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
-        .getTableTypes();
-    LOG.debug(sessionHandle + ": getTableTypes()");
-    return opHandle;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getColumns(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public OperationHandle getColumns(SessionHandle sessionHandle,
-      String catalogName, String schemaName, String tableName, String columnName)
-          throws HiveSQLException {
-    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
-        .getColumns(catalogName, schemaName, tableName, columnName);
-    LOG.debug(sessionHandle + ": getColumns()");
-    return opHandle;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getFunctions(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public OperationHandle getFunctions(SessionHandle sessionHandle,
-      String catalogName, String schemaName, String functionName)
-          throws HiveSQLException {
-    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
-        .getFunctions(catalogName, schemaName, functionName);
-    LOG.debug(sessionHandle + ": getFunctions()");
-    return opHandle;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getPrimaryKeys(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public OperationHandle getPrimaryKeys(SessionHandle sessionHandle,
-      String catalog, String schema, String table) throws HiveSQLException {
-    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
-        .getPrimaryKeys(catalog, schema, table);
-    LOG.debug(sessionHandle + ": getPrimaryKeys()");
-    return opHandle;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getCrossReference(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public OperationHandle getCrossReference(SessionHandle sessionHandle,
-      String primaryCatalog, String primarySchema, String primaryTable, String foreignCatalog,
-      String foreignSchema, String foreignTable) throws HiveSQLException {
-    OperationHandle opHandle = sessionManager.getSession(sessionHandle)
-        .getCrossReference(primaryCatalog, primarySchema, primaryTable,
-         foreignCatalog,
-         foreignSchema, foreignTable);
-    LOG.debug(sessionHandle + ": getCrossReference()");
-    return opHandle;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getOperationStatus(org.apache.hive.service.cli.OperationHandle)
-   */
-  @Override
-  public OperationStatus getOperationStatus(OperationHandle opHandle)
-      throws HiveSQLException {
-    Operation operation = sessionManager.getOperationManager().getOperation(opHandle);
-    /**
-     * If this is a background operation run asynchronously,
-     * we block for a configured duration, before we return
-     * (duration: HIVE_SERVER2_LONG_POLLING_TIMEOUT).
-     * However, if the background operation is complete, we return immediately.
-     */
-    if (operation.shouldRunAsync()) {
-      HiveConf conf = operation.getParentSession().getHiveConf();
-      long timeout = HiveConf.getTimeVar(conf,
-          HiveConf.ConfVars.HIVE_SERVER2_LONG_POLLING_TIMEOUT, TimeUnit.MILLISECONDS);
-      try {
-        operation.getBackgroundHandle().get(timeout, TimeUnit.MILLISECONDS);
-      } catch (TimeoutException e) {
-        // No Op, return to the caller since long polling timeout has expired
-        LOG.trace(opHandle + ": Long polling timed out");
-      } catch (CancellationException e) {
-        // The background operation thread was cancelled
-        LOG.trace(opHandle + ": The background operation was cancelled", e);
-      } catch (ExecutionException e) {
-        // The background operation thread was aborted
-        LOG.warn(opHandle + ": The background operation was aborted", e);
-      } catch (InterruptedException e) {
-        // No op, this thread was interrupted
-        // In this case, the call might return sooner than long polling timeout
-      }
-    }
-    OperationStatus opStatus = operation.getStatus();
-    LOG.debug(opHandle + ": getOperationStatus()");
-    return opStatus;
-  }
-
-  public HiveConf getSessionConf(SessionHandle sessionHandle) throws HiveSQLException {
-    return sessionManager.getSession(sessionHandle).getHiveConf();
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#cancelOperation(org.apache.hive.service.cli.OperationHandle)
-   */
-  @Override
-  public void cancelOperation(OperationHandle opHandle)
-      throws HiveSQLException {
-    sessionManager.getOperationManager().getOperation(opHandle)
-    .getParentSession().cancelOperation(opHandle);
-    LOG.debug(opHandle + ": cancelOperation()");
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#closeOperation(org.apache.hive.service.cli.OperationHandle)
-   */
-  @Override
-  public void closeOperation(OperationHandle opHandle)
-      throws HiveSQLException {
-    sessionManager.getOperationManager().getOperation(opHandle)
-    .getParentSession().closeOperation(opHandle);
-    LOG.debug(opHandle + ": closeOperation");
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getResultSetMetadata(org.apache.hive.service.cli.OperationHandle)
-   */
-  @Override
-  public TableSchema getResultSetMetadata(OperationHandle opHandle)
-      throws HiveSQLException {
-    TableSchema tableSchema = sessionManager.getOperationManager()
-        .getOperation(opHandle).getParentSession().getResultSetMetadata(opHandle);
-    LOG.debug(opHandle + ": getResultSetMetadata()");
-    return tableSchema;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#fetchResults(org.apache.hive.service.cli.OperationHandle)
-   */
-  @Override
-  public RowSet fetchResults(OperationHandle opHandle)
-      throws HiveSQLException {
-    return fetchResults(opHandle, Operation.DEFAULT_FETCH_ORIENTATION,
-        Operation.DEFAULT_FETCH_MAX_ROWS, FetchType.QUERY_OUTPUT);
-  }
-
-  @Override
-  public RowSet fetchResults(OperationHandle opHandle, FetchOrientation orientation,
-                             long maxRows, FetchType fetchType) throws HiveSQLException {
-    RowSet rowSet = sessionManager.getOperationManager().getOperation(opHandle)
-        .getParentSession().fetchResults(opHandle, orientation, maxRows, fetchType);
-    LOG.debug(opHandle + ": fetchResults()");
-    return rowSet;
-  }
-
-  // obtain delegation token for the give user from metastore
-  public synchronized String getDelegationTokenFromMetaStore(String owner)
-      throws HiveSQLException, UnsupportedOperationException, LoginException, IOException {
-    if (!hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL) ||
-        !hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS)) {
-      throw new UnsupportedOperationException(
-          "delegation token is can only be obtained for a secure remote metastore");
-    }
-
-    try {
-      Hive.closeCurrent();
-      return Hive.get(hiveConf).getDelegationToken(owner, owner);
-    } catch (HiveException e) {
-      if (e.getCause() instanceof UnsupportedOperationException) {
-        throw (UnsupportedOperationException)e.getCause();
-      } else {
-        throw new HiveSQLException("Error connect metastore to setup impersonation", e);
-      }
-    }
-  }
-
-  @Override
-  public String getDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
-      String owner, String renewer) throws HiveSQLException {
-    String delegationToken = sessionManager.getSession(sessionHandle)
-        .getDelegationToken(authFactory, owner, renewer);
-    LOG.info(sessionHandle  + ": getDelegationToken()");
-    return delegationToken;
-  }
-
-  @Override
-  public void cancelDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
-      String tokenStr) throws HiveSQLException {
-    sessionManager.getSession(sessionHandle).cancelDelegationToken(authFactory, tokenStr);
-    LOG.info(sessionHandle  + ": cancelDelegationToken()");
-  }
-
-  @Override
-  public void renewDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
-      String tokenStr) throws HiveSQLException {
-    sessionManager.getSession(sessionHandle).renewDelegationToken(authFactory, tokenStr);
-    LOG.info(sessionHandle  + ": renewDelegationToken()");
-  }
-
-  public SessionManager getSessionManager() {
-    return sessionManager;
-  }
-}
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/ColumnDescriptor.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/ColumnDescriptor.java
deleted file mode 100644
index d8e61a87e7f62..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/ColumnDescriptor.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import org.apache.hadoop.hive.serde2.thrift.Type;
-import org.apache.hive.service.rpc.thrift.TColumnDesc;
-
-
-/**
- * ColumnDescriptor.
- *
- */
-public class ColumnDescriptor {
-  private final String name;
-  private final String comment;
-  private final TypeDescriptor type;
-  // ordinal position of this column in the schema
-  private final int position;
-
-  public ColumnDescriptor(String name, String comment, TypeDescriptor type, int position) {
-    this.name = name;
-    this.comment = comment;
-    this.type = type;
-    this.position = position;
-  }
-
-  public ColumnDescriptor(TColumnDesc tColumnDesc) {
-    name = tColumnDesc.getColumnName();
-    comment = tColumnDesc.getComment();
-    type = new TypeDescriptor(tColumnDesc.getTypeDesc());
-    position = tColumnDesc.getPosition();
-  }
-
-  public static ColumnDescriptor newPrimitiveColumnDescriptor(String name, String comment, Type type, int position) {
-    // Current usage looks like it's only for metadata columns, but if that changes then
-    // this method may need to require a type qualifiers aruments.
-    return new ColumnDescriptor(name, comment, new TypeDescriptor(type), position);
-  }
-
-  public String getName() {
-    return name;
-  }
-
-  public String getComment() {
-    return comment;
-  }
-
-  public TypeDescriptor getTypeDescriptor() {
-    return type;
-  }
-
-  public int getOrdinalPosition() {
-    return position;
-  }
-
-  public TColumnDesc toTColumnDesc() {
-    TColumnDesc tColumnDesc = new TColumnDesc();
-    tColumnDesc.setColumnName(name);
-    tColumnDesc.setComment(comment);
-    tColumnDesc.setTypeDesc(type.toTTypeDesc());
-    tColumnDesc.setPosition(position);
-    return tColumnDesc;
-  }
-
-  public Type getType() {
-    return type.getType();
-  }
-
-  public boolean isPrimitive() {
-    return type.getType().isPrimitiveType();
-  }
-
-  public String getTypeName() {
-    return type.getTypeName();
-  }
-}
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/ColumnValue.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/ColumnValue.java
deleted file mode 100644
index 53f0465a056d8..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/ColumnValue.java
+++ /dev/null
@@ -1,307 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import java.math.BigDecimal;
-import java.sql.Date;
-import java.sql.Timestamp;
-
-import org.apache.hadoop.hive.common.type.HiveChar;
-import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
-import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
-import org.apache.hadoop.hive.common.type.HiveVarchar;
-import org.apache.hadoop.hive.serde2.thrift.Type;
-import org.apache.hive.service.rpc.thrift.TBoolValue;
-import org.apache.hive.service.rpc.thrift.TByteValue;
-import org.apache.hive.service.rpc.thrift.TColumnValue;
-import org.apache.hive.service.rpc.thrift.TDoubleValue;
-import org.apache.hive.service.rpc.thrift.TI16Value;
-import org.apache.hive.service.rpc.thrift.TI32Value;
-import org.apache.hive.service.rpc.thrift.TI64Value;
-import org.apache.hive.service.rpc.thrift.TStringValue;
-
-import org.apache.spark.unsafe.types.UTF8String;
-
-/**
- * Protocols before HIVE_CLI_SERVICE_PROTOCOL_V6 (used by RowBasedSet)
- *
- */
-public class ColumnValue {
-
-  private static TColumnValue booleanValue(Boolean value) {
-    TBoolValue tBoolValue = new TBoolValue();
-    if (value != null) {
-      tBoolValue.setValue(value);
-    }
-    return TColumnValue.boolVal(tBoolValue);
-  }
-
-  private static TColumnValue byteValue(Byte value) {
-    TByteValue tByteValue = new TByteValue();
-    if (value != null) {
-      tByteValue.setValue(value);
-    }
-    return TColumnValue.byteVal(tByteValue);
-  }
-
-  private static TColumnValue shortValue(Short value) {
-    TI16Value tI16Value = new TI16Value();
-    if (value != null) {
-      tI16Value.setValue(value);
-    }
-    return TColumnValue.i16Val(tI16Value);
-  }
-
-  private static TColumnValue intValue(Integer value) {
-    TI32Value tI32Value = new TI32Value();
-    if (value != null) {
-      tI32Value.setValue(value);
-    }
-    return TColumnValue.i32Val(tI32Value);
-  }
-
-  private static TColumnValue longValue(Long value) {
-    TI64Value tI64Value = new TI64Value();
-    if (value != null) {
-      tI64Value.setValue(value);
-    }
-    return TColumnValue.i64Val(tI64Value);
-  }
-
-  private static TColumnValue floatValue(Float value) {
-    TDoubleValue tDoubleValue = new TDoubleValue();
-    if (value != null) {
-      tDoubleValue.setValue(value);
-    }
-    return TColumnValue.doubleVal(tDoubleValue);
-  }
-
-  private static TColumnValue doubleValue(Double value) {
-    TDoubleValue tDoubleValue = new TDoubleValue();
-    if (value != null) {
-      tDoubleValue.setValue(value);
-    }
-    return TColumnValue.doubleVal(tDoubleValue);
-  }
-
-  private static TColumnValue stringValue(String value) {
-    TStringValue tStringValue = new TStringValue();
-    if (value != null) {
-      tStringValue.setValue(value);
-    }
-    return TColumnValue.stringVal(tStringValue);
-  }
-
-  private static TColumnValue stringValue(HiveChar value) {
-    TStringValue tStringValue = new TStringValue();
-    if (value != null) {
-      tStringValue.setValue(value.toString());
-    }
-    return TColumnValue.stringVal(tStringValue);
-  }
-
-  private static TColumnValue stringValue(HiveVarchar value) {
-    TStringValue tStringValue = new TStringValue();
-    if (value != null) {
-      tStringValue.setValue(value.toString());
-    }
-    return TColumnValue.stringVal(tStringValue);
-  }
-
-  private static TColumnValue dateValue(Date value) {
-    TStringValue tStringValue = new TStringValue();
-    if (value != null) {
-      tStringValue.setValue(value.toString());
-    }
-    return new TColumnValue(TColumnValue.stringVal(tStringValue));
-  }
-
-  private static TColumnValue timestampValue(Timestamp value) {
-    TStringValue tStringValue = new TStringValue();
-    if (value != null) {
-      tStringValue.setValue(value.toString());
-    }
-    return TColumnValue.stringVal(tStringValue);
-  }
-
-  private static TColumnValue stringValue(HiveIntervalYearMonth value) {
-    TStringValue tStrValue = new TStringValue();
-    if (value != null) {
-      tStrValue.setValue(value.toString());
-    }
-    return TColumnValue.stringVal(tStrValue);
-  }
-
-  private static TColumnValue stringValue(HiveIntervalDayTime value) {
-    TStringValue tStrValue = new TStringValue();
-    if (value != null) {
-      tStrValue.setValue(value.toString());
-    }
-    return TColumnValue.stringVal(tStrValue);
-  }
-
-  public static TColumnValue toTColumnValue(TypeDescriptor typeDescriptor, Object value) {
-    Type type = typeDescriptor.getType();
-
-    switch (type) {
-    case BOOLEAN_TYPE:
-      return booleanValue((Boolean)value);
-    case TINYINT_TYPE:
-      return byteValue((Byte)value);
-    case SMALLINT_TYPE:
-      return shortValue((Short)value);
-    case INT_TYPE:
-      return intValue((Integer)value);
-    case BIGINT_TYPE:
-      return longValue((Long)value);
-    case FLOAT_TYPE:
-      return floatValue((Float)value);
-    case DOUBLE_TYPE:
-      return doubleValue((Double)value);
-    case STRING_TYPE:
-      return stringValue((String)value);
-    case CHAR_TYPE:
-      return stringValue((HiveChar)value);
-    case VARCHAR_TYPE:
-      return stringValue((HiveVarchar)value);
-    case DATE_TYPE:
-      return dateValue((Date)value);
-    case TIMESTAMP_TYPE:
-      return timestampValue((Timestamp)value);
-    case INTERVAL_YEAR_MONTH_TYPE:
-      return stringValue((HiveIntervalYearMonth) value);
-    case INTERVAL_DAY_TIME_TYPE:
-      return stringValue((HiveIntervalDayTime) value);
-    case DECIMAL_TYPE:
-      String plainStr = value == null ? null : ((BigDecimal)value).toPlainString();
-      return stringValue(plainStr);
-    case BINARY_TYPE:
-      String strVal = value == null ? null : UTF8String.fromBytes((byte[])value).toString();
-      return stringValue(strVal);
-    case ARRAY_TYPE:
-    case MAP_TYPE:
-    case STRUCT_TYPE:
-    case UNION_TYPE:
-    case USER_DEFINED_TYPE:
-      return stringValue((String)value);
-    case NULL_TYPE:
-      return stringValue((String)value);
-    default:
-      return null;
-    }
-  }
-
-  private static Boolean getBooleanValue(TBoolValue tBoolValue) {
-    if (tBoolValue.isSetValue()) {
-      return tBoolValue.isValue();
-    }
-    return null;
-  }
-
-  private static Byte getByteValue(TByteValue tByteValue) {
-    if (tByteValue.isSetValue()) {
-      return tByteValue.getValue();
-    }
-    return null;
-  }
-
-  private static Short getShortValue(TI16Value tI16Value) {
-    if (tI16Value.isSetValue()) {
-      return tI16Value.getValue();
-    }
-    return null;
-  }
-
-  private static Integer getIntegerValue(TI32Value tI32Value) {
-    if (tI32Value.isSetValue()) {
-      return tI32Value.getValue();
-    }
-    return null;
-  }
-
-  private static Long getLongValue(TI64Value tI64Value) {
-    if (tI64Value.isSetValue()) {
-      return tI64Value.getValue();
-    }
-    return null;
-  }
-
-  private static Double getDoubleValue(TDoubleValue tDoubleValue) {
-    if (tDoubleValue.isSetValue()) {
-      return tDoubleValue.getValue();
-    }
-    return null;
-  }
-
-  private static String getStringValue(TStringValue tStringValue) {
-    if (tStringValue.isSetValue()) {
-      return tStringValue.getValue();
-    }
-    return null;
-  }
-
-  private static Date getDateValue(TStringValue tStringValue) {
-    if (tStringValue.isSetValue()) {
-      return Date.valueOf(tStringValue.getValue());
-    }
-    return null;
-  }
-
-  private static Timestamp getTimestampValue(TStringValue tStringValue) {
-    if (tStringValue.isSetValue()) {
-      return Timestamp.valueOf(tStringValue.getValue());
-    }
-    return null;
-  }
-
-  private static byte[] getBinaryValue(TStringValue tString) {
-    if (tString.isSetValue()) {
-      return tString.getValue().getBytes();
-    }
-    return null;
-  }
-
-  private static BigDecimal getBigDecimalValue(TStringValue tStringValue) {
-    if (tStringValue.isSetValue()) {
-      return new BigDecimal(tStringValue.getValue());
-    }
-    return null;
-  }
-
-  public static Object toColumnValue(TColumnValue value) {
-    TColumnValue._Fields field = value.getSetField();
-    switch (field) {
-      case BOOL_VAL:
-        return getBooleanValue(value.getBoolVal());
-      case BYTE_VAL:
-        return getByteValue(value.getByteVal());
-      case I16_VAL:
-        return getShortValue(value.getI16Val());
-      case I32_VAL:
-        return getIntegerValue(value.getI32Val());
-      case I64_VAL:
-        return getLongValue(value.getI64Val());
-      case DOUBLE_VAL:
-        return getDoubleValue(value.getDoubleVal());
-      case STRING_VAL:
-        return getStringValue(value.getStringVal());
-    }
-    throw new IllegalArgumentException("never");
-  }
-}
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/GetInfoType.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/GetInfoType.java
deleted file mode 100644
index a64d262a8f301..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/GetInfoType.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import org.apache.hive.service.rpc.thrift.TGetInfoType;
-
-/**
- * GetInfoType.
- *
- */
-public enum GetInfoType {
-  CLI_MAX_DRIVER_CONNECTIONS(TGetInfoType.CLI_MAX_DRIVER_CONNECTIONS),
-  CLI_MAX_CONCURRENT_ACTIVITIES(TGetInfoType.CLI_MAX_CONCURRENT_ACTIVITIES),
-  CLI_DATA_SOURCE_NAME(TGetInfoType.CLI_DATA_SOURCE_NAME),
-  CLI_FETCH_DIRECTION(TGetInfoType.CLI_FETCH_DIRECTION),
-  CLI_SERVER_NAME(TGetInfoType.CLI_SERVER_NAME),
-  CLI_SEARCH_PATTERN_ESCAPE(TGetInfoType.CLI_SEARCH_PATTERN_ESCAPE),
-  CLI_DBMS_NAME(TGetInfoType.CLI_DBMS_NAME),
-  CLI_DBMS_VER(TGetInfoType.CLI_DBMS_VER),
-  CLI_ACCESSIBLE_TABLES(TGetInfoType.CLI_ACCESSIBLE_TABLES),
-  CLI_ACCESSIBLE_PROCEDURES(TGetInfoType.CLI_ACCESSIBLE_PROCEDURES),
-  CLI_CURSOR_COMMIT_BEHAVIOR(TGetInfoType.CLI_CURSOR_COMMIT_BEHAVIOR),
-  CLI_DATA_SOURCE_READ_ONLY(TGetInfoType.CLI_DATA_SOURCE_READ_ONLY),
-  CLI_DEFAULT_TXN_ISOLATION(TGetInfoType.CLI_DEFAULT_TXN_ISOLATION),
-  CLI_IDENTIFIER_CASE(TGetInfoType.CLI_IDENTIFIER_CASE),
-  CLI_IDENTIFIER_QUOTE_CHAR(TGetInfoType.CLI_IDENTIFIER_QUOTE_CHAR),
-  CLI_MAX_COLUMN_NAME_LEN(TGetInfoType.CLI_MAX_COLUMN_NAME_LEN),
-  CLI_MAX_CURSOR_NAME_LEN(TGetInfoType.CLI_MAX_CURSOR_NAME_LEN),
-  CLI_MAX_SCHEMA_NAME_LEN(TGetInfoType.CLI_MAX_SCHEMA_NAME_LEN),
-  CLI_MAX_CATALOG_NAME_LEN(TGetInfoType.CLI_MAX_CATALOG_NAME_LEN),
-  CLI_MAX_TABLE_NAME_LEN(TGetInfoType.CLI_MAX_TABLE_NAME_LEN),
-  CLI_SCROLL_CONCURRENCY(TGetInfoType.CLI_SCROLL_CONCURRENCY),
-  CLI_TXN_CAPABLE(TGetInfoType.CLI_TXN_CAPABLE),
-  CLI_USER_NAME(TGetInfoType.CLI_USER_NAME),
-  CLI_TXN_ISOLATION_OPTION(TGetInfoType.CLI_TXN_ISOLATION_OPTION),
-  CLI_INTEGRITY(TGetInfoType.CLI_INTEGRITY),
-  CLI_GETDATA_EXTENSIONS(TGetInfoType.CLI_GETDATA_EXTENSIONS),
-  CLI_NULL_COLLATION(TGetInfoType.CLI_NULL_COLLATION),
-  CLI_ALTER_TABLE(TGetInfoType.CLI_ALTER_TABLE),
-  CLI_ORDER_BY_COLUMNS_IN_SELECT(TGetInfoType.CLI_ORDER_BY_COLUMNS_IN_SELECT),
-  CLI_SPECIAL_CHARACTERS(TGetInfoType.CLI_SPECIAL_CHARACTERS),
-  CLI_MAX_COLUMNS_IN_GROUP_BY(TGetInfoType.CLI_MAX_COLUMNS_IN_GROUP_BY),
-  CLI_MAX_COLUMNS_IN_INDEX(TGetInfoType.CLI_MAX_COLUMNS_IN_INDEX),
-  CLI_MAX_COLUMNS_IN_ORDER_BY(TGetInfoType.CLI_MAX_COLUMNS_IN_ORDER_BY),
-  CLI_MAX_COLUMNS_IN_SELECT(TGetInfoType.CLI_MAX_COLUMNS_IN_SELECT),
-  CLI_MAX_COLUMNS_IN_TABLE(TGetInfoType.CLI_MAX_COLUMNS_IN_TABLE),
-  CLI_MAX_INDEX_SIZE(TGetInfoType.CLI_MAX_INDEX_SIZE),
-  CLI_MAX_ROW_SIZE(TGetInfoType.CLI_MAX_ROW_SIZE),
-  CLI_MAX_STATEMENT_LEN(TGetInfoType.CLI_MAX_STATEMENT_LEN),
-  CLI_MAX_TABLES_IN_SELECT(TGetInfoType.CLI_MAX_TABLES_IN_SELECT),
-  CLI_MAX_USER_NAME_LEN(TGetInfoType.CLI_MAX_USER_NAME_LEN),
-  CLI_OJ_CAPABILITIES(TGetInfoType.CLI_OJ_CAPABILITIES),
-
-  CLI_XOPEN_CLI_YEAR(TGetInfoType.CLI_XOPEN_CLI_YEAR),
-  CLI_CURSOR_SENSITIVITY(TGetInfoType.CLI_CURSOR_SENSITIVITY),
-  CLI_DESCRIBE_PARAMETER(TGetInfoType.CLI_DESCRIBE_PARAMETER),
-  CLI_CATALOG_NAME(TGetInfoType.CLI_CATALOG_NAME),
-  CLI_COLLATION_SEQ(TGetInfoType.CLI_COLLATION_SEQ),
-  CLI_MAX_IDENTIFIER_LEN(TGetInfoType.CLI_MAX_IDENTIFIER_LEN);
-
-  private final TGetInfoType tInfoType;
-
-  GetInfoType(TGetInfoType tInfoType) {
-    this.tInfoType = tInfoType;
-  }
-
-  public static GetInfoType getGetInfoType(TGetInfoType tGetInfoType) {
-    for (GetInfoType infoType : values()) {
-      if (tGetInfoType.equals(infoType.tInfoType)) {
-        return infoType;
-      }
-    }
-    throw new IllegalArgumentException("Unrecognized Thrift TGetInfoType value: " + tGetInfoType);
-  }
-
-  public TGetInfoType toTGetInfoType() {
-    return tInfoType;
-  }
-
-}
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/GetInfoValue.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/GetInfoValue.java
deleted file mode 100644
index 2b2359cc13c0f..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/GetInfoValue.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli;
-
-import org.apache.hive.service.rpc.thrift.TGetInfoValue;
-
-/**
- * GetInfoValue.
- *
- */
-public class GetInfoValue {
-  private String stringValue = null;
-  private short shortValue;
-  private int intValue;
-  private long longValue;
-
-  public GetInfoValue(String stringValue) {
-    this.stringValue = stringValue;
-  }
-
-  public GetInfoValue(short shortValue) {
-    this.shortValue = shortValue;
-  }
-
-  public GetInfoValue(int intValue) {
-    this.intValue = intValue;
-  }
-
-  public GetInfoValue(long longValue) {
-    this.longValue = longValue;
-  }
-
-  public GetInfoValue(TGetInfoValue tGetInfoValue) {
-    switch (tGetInfoValue.getSetField()) {
-    case STRING_VALUE:
-      stringValue = tGetInfoValue.getStringValue();
-      break;
-    default:
-      throw new IllegalArgumentException("Unreconigzed TGetInfoValue");
-    }
-  }
-
-  public TGetInfoValue toTGetInfoValue() {
-    TGetInfoValue tInfoValue = new TGetInfoValue();
-    if (stringValue != null) {
-      tInfoValue.setStringValue(stringValue);
-    }
-    return tInfoValue;
-  }
-
-  public String getStringValue() {
-    return stringValue;
-  }
-
-  public short getShortValue() {
-    return shortValue;
-  }
-
-  public int getIntValue() {
-    return intValue;
-  }
-
-  public long getLongValue() {
-    return longValue;
-  }
-}
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/ICLIService.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/ICLIService.java
deleted file mode 100644
index 3200909477821..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/ICLIService.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.service.cli;
-
-import java.util.List;
-import java.util.Map;
-
-
-
-
-import org.apache.hive.service.auth.HiveAuthFactory;
-
-public interface ICLIService {
-
-  SessionHandle openSession(String username, String password,
-      Map<String, String> configuration)
-          throws HiveSQLException;
-
-  SessionHandle openSessionWithImpersonation(String username, String password,
-      Map<String, String> configuration, String delegationToken)
-          throws HiveSQLException;
-
-  void closeSession(SessionHandle sessionHandle)
-      throws HiveSQLException;
-
-  GetInfoValue getInfo(SessionHandle sessionHandle, GetInfoType infoType)
-      throws HiveSQLException;
-
-  OperationHandle executeStatement(SessionHandle sessionHandle, String statement,
-      Map<String, String> confOverlay) throws HiveSQLException;
-
-  OperationHandle executeStatement(SessionHandle sessionHandle, String statement,
-      Map<String, String> confOverlay, long queryTimeout) throws HiveSQLException;
-
-  OperationHandle executeStatementAsync(SessionHandle sessionHandle, String statement,
-      Map<String, String> confOverlay) throws HiveSQLException;
-  OperationHandle executeStatementAsync(SessionHandle sessionHandle, String statement,
-      Map<String, String> confOverlay, long queryTimeout) throws HiveSQLException;
-
-  OperationHandle getTypeInfo(SessionHandle sessionHandle)
-      throws HiveSQLException;
-
-  OperationHandle getCatalogs(SessionHandle sessionHandle)
-      throws HiveSQLException;
-
-  OperationHandle getSchemas(SessionHandle sessionHandle,
-      String catalogName, String schemaName)
-          throws HiveSQLException;
-
-  OperationHandle getTables(SessionHandle sessionHandle,
-      String catalogName, String schemaName, String tableName, List<String> tableTypes)
-          throws HiveSQLException;
-
-  OperationHandle getTableTypes(SessionHandle sessionHandle)
-      throws HiveSQLException;
-
-  OperationHandle getColumns(SessionHandle sessionHandle,
-      String catalogName, String schemaName, String tableName, String columnName)
-          throws HiveSQLException;
-
-  OperationHandle getFunctions(SessionHandle sessionHandle,
-      String catalogName, String schemaName, String functionName)
-          throws HiveSQLException;
-
-  OperationStatus getOperationStatus(OperationHandle opHandle)
-      throws HiveSQLException;
-
-  void cancelOperation(OperationHandle opHandle)
-      throws HiveSQLException;
-
-  void closeOperation(OperationHandle opHandle)
-      throws HiveSQLException;
-
-  TableSchema getResultSetMetadata(OperationHandle opHandle)
-      throws HiveSQLException;
-
-  RowSet fetchResults(OperationHandle opHandle)
-      throws HiveSQLException;
-
-  RowSet fetchResults(OperationHandle opHandle, FetchOrientation orientation,
-      long maxRows, FetchType fetchType) throws HiveSQLException;
-
-  String getDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
-      String owner, String renewer) throws HiveSQLException;
-
-  void cancelDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
-      String tokenStr) throws HiveSQLException;
-
-  void renewDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
-      String tokenStr) throws HiveSQLException;
-
-  OperationHandle getPrimaryKeys(SessionHandle sessionHandle, String catalog,
-      String schema, String table) throws HiveSQLException;
-
-  OperationHandle getCrossReference(SessionHandle sessionHandle,
-      String primaryCatalog, String primarySchema, String primaryTable,
-      String foreignCatalog, String foreignSchema, String foreignTable) throws HiveSQLException;
-}
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/GetColumnsOperation.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/GetColumnsOperation.java
deleted file mode 100644
index c25c742d392b3..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/GetColumnsOperation.java
+++ /dev/null
@@ -1,251 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.operation;
-
-import java.sql.DatabaseMetaData;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.regex.Pattern;
-
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hadoop.hive.metastore.api.PrimaryKeysRequest;
-import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.ql.metadata.TableIterable;
-import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
-import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject;
-import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivilegeObjectType;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.serde2.thrift.Type;
-import org.apache.hive.service.cli.ColumnDescriptor;
-import org.apache.hive.service.cli.FetchOrientation;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.OperationState;
-import org.apache.hive.service.cli.OperationType;
-import org.apache.hive.service.cli.RowSet;
-import org.apache.hive.service.cli.RowSetFactory;
-import org.apache.hive.service.cli.TableSchema;
-import org.apache.hive.service.cli.session.HiveSession;
-
-/**
- * GetColumnsOperation.
- *
- */
-public class GetColumnsOperation extends MetadataOperation {
-
-  protected static final TableSchema RESULT_SET_SCHEMA = new TableSchema()
-  .addPrimitiveColumn("TABLE_CAT", Type.STRING_TYPE,
-      "Catalog name. NULL if not applicable")
-  .addPrimitiveColumn("TABLE_SCHEM", Type.STRING_TYPE,
-      "Schema name")
-  .addPrimitiveColumn("TABLE_NAME", Type.STRING_TYPE,
-      "Table name")
-  .addPrimitiveColumn("COLUMN_NAME", Type.STRING_TYPE,
-      "Column name")
-  .addPrimitiveColumn("DATA_TYPE", Type.INT_TYPE,
-      "SQL type from java.sql.Types")
-  .addPrimitiveColumn("TYPE_NAME", Type.STRING_TYPE,
-      "Data source dependent type name, for a UDT the type name is fully qualified")
-  .addPrimitiveColumn("COLUMN_SIZE", Type.INT_TYPE,
-      "Column size. For char or date types this is the maximum number of characters,"
-      + " for numeric or decimal types this is precision.")
-  .addPrimitiveColumn("BUFFER_LENGTH", Type.TINYINT_TYPE,
-      "Unused")
-  .addPrimitiveColumn("DECIMAL_DIGITS", Type.INT_TYPE,
-      "The number of fractional digits")
-  .addPrimitiveColumn("NUM_PREC_RADIX", Type.INT_TYPE,
-      "Radix (typically either 10 or 2)")
-  .addPrimitiveColumn("NULLABLE", Type.INT_TYPE,
-      "Is NULL allowed")
-  .addPrimitiveColumn("REMARKS", Type.STRING_TYPE,
-      "Comment describing column (may be null)")
-  .addPrimitiveColumn("COLUMN_DEF", Type.STRING_TYPE,
-      "Default value (may be null)")
-  .addPrimitiveColumn("SQL_DATA_TYPE", Type.INT_TYPE,
-      "Unused")
-  .addPrimitiveColumn("SQL_DATETIME_SUB", Type.INT_TYPE,
-      "Unused")
-  .addPrimitiveColumn("CHAR_OCTET_LENGTH", Type.INT_TYPE,
-      "For char types the maximum number of bytes in the column")
-  .addPrimitiveColumn("ORDINAL_POSITION", Type.INT_TYPE,
-      "Index of column in table (starting at 1)")
-  .addPrimitiveColumn("IS_NULLABLE", Type.STRING_TYPE,
-      "\"NO\" means column definitely does not allow NULL values; "
-      + "\"YES\" means the column might allow NULL values. An empty "
-      + "string means nobody knows.")
-  .addPrimitiveColumn("SCOPE_CATALOG", Type.STRING_TYPE,
-      "Catalog of table that is the scope of a reference attribute "
-      + "(null if DATA_TYPE isn't REF)")
-  .addPrimitiveColumn("SCOPE_SCHEMA", Type.STRING_TYPE,
-      "Schema of table that is the scope of a reference attribute "
-      + "(null if the DATA_TYPE isn't REF)")
-  .addPrimitiveColumn("SCOPE_TABLE", Type.STRING_TYPE,
-      "Table name that this the scope of a reference attribure "
-      + "(null if the DATA_TYPE isn't REF)")
-  .addPrimitiveColumn("SOURCE_DATA_TYPE", Type.SMALLINT_TYPE,
-      "Source type of a distinct type or user-generated Ref type, "
-      + "SQL type from java.sql.Types (null if DATA_TYPE isn't DISTINCT or user-generated REF)")
-  .addPrimitiveColumn("IS_AUTO_INCREMENT", Type.STRING_TYPE,
-      "Indicates whether this column is auto incremented.");
-
-  private final String catalogName;
-  private final String schemaName;
-  private final String tableName;
-  private final String columnName;
-
-  protected final RowSet rowSet;
-
-  protected GetColumnsOperation(HiveSession parentSession, String catalogName, String schemaName,
-      String tableName, String columnName) {
-    super(parentSession, OperationType.GET_COLUMNS);
-    this.catalogName = catalogName;
-    this.schemaName = schemaName;
-    this.tableName = tableName;
-    this.columnName = columnName;
-    this.rowSet = RowSetFactory.create(RESULT_SET_SCHEMA, getProtocolVersion(), false);
-  }
-
-  @Override
-  public void runInternal() throws HiveSQLException {
-    setState(OperationState.RUNNING);
-    try {
-      IMetaStoreClient metastoreClient = getParentSession().getMetaStoreClient();
-      String schemaPattern = convertSchemaPattern(schemaName);
-      String tablePattern = convertIdentifierPattern(tableName, true);
-
-      Pattern columnPattern = null;
-      if (columnName != null) {
-        columnPattern = Pattern.compile(convertIdentifierPattern(columnName, false));
-      }
-
-      List<String> dbNames = metastoreClient.getDatabases(schemaPattern);
-      Collections.sort(dbNames);
-      Map<String, List<String>> db2Tabs = new HashMap<>();
-
-      for (String dbName : dbNames) {
-        List<String> tableNames = metastoreClient.getTables(dbName, tablePattern);
-        Collections.sort(tableNames);
-        db2Tabs.put(dbName, tableNames);
-      }
-
-      if (isAuthV2Enabled()) {
-        List<HivePrivilegeObject> privObjs = getPrivObjs(db2Tabs);
-        String cmdStr = "catalog : " + catalogName + ", schemaPattern : " + schemaName
-            + ", tablePattern : " + tableName;
-        authorizeMetaGets(HiveOperationType.GET_COLUMNS, privObjs, cmdStr);
-      }
-
-      int maxBatchSize = SessionState.get().getConf().getIntVar(ConfVars.METASTORE_BATCH_RETRIEVE_MAX);
-      for (Entry<String, List<String>> dbTabs : db2Tabs.entrySet()) {
-        String dbName = dbTabs.getKey();
-        List<String> tableNames = dbTabs.getValue();
-
-        for (Table table : new TableIterable(metastoreClient, dbName, tableNames, maxBatchSize)) {
-
-          TableSchema schema = new TableSchema(metastoreClient.getSchema(dbName,
-              table.getTableName()));
-          List<SQLPrimaryKey> primaryKeys = metastoreClient.getPrimaryKeys(new PrimaryKeysRequest(dbName, table.getTableName()));
-          Set<String> pkColNames = new HashSet<>();
-          for(SQLPrimaryKey key : primaryKeys) {
-            pkColNames.add(key.getColumn_name().toLowerCase());
-          }
-          for (ColumnDescriptor column : schema.getColumnDescriptors()) {
-            if (columnPattern != null && !columnPattern.matcher(column.getName()).matches()) {
-              continue;
-            }
-            Object[] rowData = new Object[] {
-                null,  // TABLE_CAT
-                table.getDbName(), // TABLE_SCHEM
-                table.getTableName(), // TABLE_NAME
-                column.getName(), // COLUMN_NAME
-                column.getType().toJavaSQLType(), // DATA_TYPE
-                column.getTypeName(), // TYPE_NAME
-                column.getTypeDescriptor().getColumnSize(), // COLUMN_SIZE
-                null, // BUFFER_LENGTH, unused
-                column.getTypeDescriptor().getDecimalDigits(), // DECIMAL_DIGITS
-                column.getType().getNumPrecRadix(), // NUM_PREC_RADIX
-                pkColNames.contains(column.getName().toLowerCase()) ? DatabaseMetaData.columnNoNulls
-                    : DatabaseMetaData.columnNullable, // NULLABLE
-                column.getComment(), // REMARKS
-                null, // COLUMN_DEF
-                null, // SQL_DATA_TYPE
-                null, // SQL_DATETIME_SUB
-                null, // CHAR_OCTET_LENGTH
-                column.getOrdinalPosition(), // ORDINAL_POSITION
-                pkColNames.contains(column.getName().toLowerCase()) ? "NO" : "YES", // IS_NULLABLE
-                null, // SCOPE_CATALOG
-                null, // SCOPE_SCHEMA
-                null, // SCOPE_TABLE
-                null, // SOURCE_DATA_TYPE
-                "NO", // IS_AUTO_INCREMENT
-            };
-            rowSet.addRow(rowData);
-          }
-        }
-      }
-      setState(OperationState.FINISHED);
-    } catch (Exception e) {
-      setState(OperationState.ERROR);
-      throw new HiveSQLException(e);
-    }
-
-  }
-
-
-  private List<HivePrivilegeObject> getPrivObjs(Map<String, List<String>> db2Tabs) {
-    List<HivePrivilegeObject> privObjs = new ArrayList<>();
-    for (Entry<String, List<String>> dbTabs : db2Tabs.entrySet()) {
-      for (String tabName : dbTabs.getValue()) {
-        privObjs.add(new HivePrivilegeObject(HivePrivilegeObjectType.TABLE_OR_VIEW, dbTabs.getKey(),
-            tabName));
-      }
-    }
-    return privObjs;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.Operation#getResultSetSchema()
-   */
-  @Override
-  public TableSchema getResultSetSchema() throws HiveSQLException {
-    assertState(OperationState.FINISHED);
-    return RESULT_SET_SCHEMA;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.Operation#getNextRowSet(org.apache.hive.service.cli.FetchOrientation, long)
-   */
-  @Override
-  public RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws HiveSQLException {
-    assertState(OperationState.FINISHED);
-    validateDefaultFetchOrientation(orientation);
-    if (orientation.equals(FetchOrientation.FETCH_FIRST)) {
-      rowSet.setStartOffset(0);
-    }
-    return rowSet.extractSubset((int)maxRows);
-  }
-
-}
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/Operation.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/Operation.java
deleted file mode 100644
index f26c715add987..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/Operation.java
+++ /dev/null
@@ -1,347 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.service.cli.operation;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.QueryState;
-import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
-import org.apache.hadoop.hive.ql.session.OperationLog;
-import org.apache.hive.service.cli.FetchOrientation;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.OperationHandle;
-import org.apache.hive.service.cli.OperationState;
-import org.apache.hive.service.cli.OperationStatus;
-import org.apache.hive.service.cli.OperationType;
-import org.apache.hive.service.cli.RowSet;
-import org.apache.hive.service.cli.TableSchema;
-import org.apache.hive.service.cli.session.HiveSession;
-import org.apache.hive.service.rpc.thrift.TProtocolVersion;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public abstract class Operation {
-  protected final HiveSession parentSession;
-  private OperationState state = OperationState.INITIALIZED;
-  private final OperationHandle opHandle;
-  private HiveConf configuration;
-  public static final Logger LOG = LoggerFactory.getLogger(Operation.class.getName());
-  public static final FetchOrientation DEFAULT_FETCH_ORIENTATION = FetchOrientation.FETCH_NEXT;
-  public static final long DEFAULT_FETCH_MAX_ROWS = 100;
-  protected boolean hasResultSet;
-  protected volatile HiveSQLException operationException;
-  protected final boolean runAsync;
-  protected volatile Future<?> backgroundHandle;
-  protected OperationLog operationLog;
-  protected boolean isOperationLogEnabled;
-  protected Map<String, String> confOverlay = new HashMap<String, String>();
-
-  private long operationTimeout;
-  private long lastAccessTime;
-
-  protected final QueryState queryState;
-
-  protected static final EnumSet<FetchOrientation> DEFAULT_FETCH_ORIENTATION_SET =
-      EnumSet.of(
-          FetchOrientation.FETCH_NEXT,
-          FetchOrientation.FETCH_FIRST,
-          FetchOrientation.FETCH_PRIOR);
-
-  protected Operation(HiveSession parentSession, OperationType opType) {
-    this(parentSession, null, opType);
-  }
-
-  protected Operation(HiveSession parentSession, Map<String, String> confOverlay,
-      OperationType opType) {
-    this(parentSession, confOverlay, opType, false);
-  }
-
-  protected Operation(HiveSession parentSession,
-      Map<String, String> confOverlay, OperationType opType, boolean runInBackground) {
-    this.parentSession = parentSession;
-    this.confOverlay = confOverlay;
-    this.runAsync = runInBackground;
-    this.opHandle = new OperationHandle(opType, parentSession.getProtocolVersion());
-    lastAccessTime = System.currentTimeMillis();
-    operationTimeout = HiveConf.getTimeVar(parentSession.getHiveConf(),
-        HiveConf.ConfVars.HIVE_SERVER2_IDLE_OPERATION_TIMEOUT, TimeUnit.MILLISECONDS);
-    queryState = new QueryState(parentSession.getHiveConf(), confOverlay, runInBackground);
-  }
-
-  public Future<?> getBackgroundHandle() {
-    return backgroundHandle;
-  }
-
-  protected void setBackgroundHandle(Future<?> backgroundHandle) {
-    this.backgroundHandle = backgroundHandle;
-  }
-
-  public boolean shouldRunAsync() {
-    return runAsync;
-  }
-
-  public void setConfiguration(HiveConf configuration) {
-    this.configuration = new HiveConf(configuration);
-  }
-
-  public HiveConf getConfiguration() {
-    return new HiveConf(configuration);
-  }
-
-  public HiveSession getParentSession() {
-    return parentSession;
-  }
-
-  public OperationHandle getHandle() {
-    return opHandle;
-  }
-
-  public TProtocolVersion getProtocolVersion() {
-    return opHandle.getProtocolVersion();
-  }
-
-  public OperationType getType() {
-    return opHandle.getOperationType();
-  }
-
-  public OperationStatus getStatus() {
-    return new OperationStatus(state, operationException);
-  }
-
-  public boolean hasResultSet() {
-    return hasResultSet;
-  }
-
-  protected void setHasResultSet(boolean hasResultSet) {
-    this.hasResultSet = hasResultSet;
-    opHandle.setHasResultSet(hasResultSet);
-  }
-
-  public OperationLog getOperationLog() {
-    return operationLog;
-  }
-
-  protected final OperationState setState(OperationState newState) throws HiveSQLException {
-    state.validateTransition(newState);
-    this.state = newState;
-    this.lastAccessTime = System.currentTimeMillis();
-    return this.state;
-  }
-
-  public boolean isTimedOut(long current) {
-    if (operationTimeout == 0) {
-      return false;
-    }
-    if (operationTimeout > 0) {
-      // check only when it's in terminal state
-      return state.isTerminal() && lastAccessTime + operationTimeout <= current;
-    }
-    return lastAccessTime + -operationTimeout <= current;
-  }
-
-  public long getLastAccessTime() {
-    return lastAccessTime;
-  }
-
-  public long getOperationTimeout() {
-    return operationTimeout;
-  }
-
-  public void setOperationTimeout(long operationTimeout) {
-    this.operationTimeout = operationTimeout;
-  }
-
-  protected void setOperationException(HiveSQLException operationException) {
-    this.operationException = operationException;
-  }
-
-  protected final void assertState(OperationState state) throws HiveSQLException {
-    if (this.state != state) {
-      throw new HiveSQLException("Expected state " + state + ", but found " + this.state);
-    }
-    this.lastAccessTime = System.currentTimeMillis();
-  }
-
-  public boolean isRunning() {
-    return OperationState.RUNNING.equals(state);
-  }
-
-  public boolean isFinished() {
-    return OperationState.FINISHED.equals(state);
-  }
-
-  public boolean isCanceled() {
-    return OperationState.CANCELED.equals(state);
-  }
-
-  public boolean isFailed() {
-    return OperationState.ERROR.equals(state);
-  }
-
-  protected void createOperationLog() {
-    if (parentSession.isOperationLogEnabled()) {
-      File operationLogFile = new File(parentSession.getOperationLogSessionDir(),
-          opHandle.getHandleIdentifier().toString());
-      isOperationLogEnabled = true;
-
-      // create log file
-      try {
-        if (operationLogFile.exists()) {
-          LOG.warn("The operation log file should not exist, but it is already there: " +
-              operationLogFile.getAbsolutePath());
-          operationLogFile.delete();
-        }
-        if (!operationLogFile.createNewFile()) {
-          // the log file already exists and cannot be deleted.
-          // If it can be read/written, keep its contents and use it.
-          if (!operationLogFile.canRead() || !operationLogFile.canWrite()) {
-            LOG.warn("The already existed operation log file cannot be recreated, " +
-                "and it cannot be read or written: " + operationLogFile.getAbsolutePath());
-            isOperationLogEnabled = false;
-            return;
-          }
-        }
-      } catch (Exception e) {
-        LOG.warn("Unable to create operation log file: " + operationLogFile.getAbsolutePath(), e);
-        isOperationLogEnabled = false;
-        return;
-      }
-
-      // create OperationLog object with above log file
-      try {
-        operationLog = new OperationLog(opHandle.toString(), operationLogFile, parentSession.getHiveConf());
-      } catch (FileNotFoundException e) {
-        LOG.warn("Unable to instantiate OperationLog object for operation: " +
-            opHandle, e);
-        isOperationLogEnabled = false;
-        return;
-      }
-
-      // register this operationLog to current thread
-      OperationLog.setCurrentOperationLog(operationLog);
-    }
-  }
-
-  protected void unregisterOperationLog() {
-    if (isOperationLogEnabled) {
-      OperationLog.removeCurrentOperationLog();
-    }
-  }
-
-  /**
-   * Invoked before runInternal().
-   * Set up some preconditions, or configurations.
-   */
-  protected void beforeRun() {
-    createOperationLog();
-  }
-
-  /**
-   * Invoked after runInternal(), even if an exception is thrown in runInternal().
-   * Clean up resources, which was set up in beforeRun().
-   */
-  protected void afterRun() {
-    unregisterOperationLog();
-  }
-
-  /**
-   * Implemented by subclass of Operation class to execute specific behaviors.
-   * @throws HiveSQLException
-   */
-  protected abstract void runInternal() throws HiveSQLException;
-
-  public void run() throws HiveSQLException {
-    beforeRun();
-    try {
-      runInternal();
-    } finally {
-      afterRun();
-    }
-  }
-
-  protected void cleanupOperationLog() {
-    if (isOperationLogEnabled) {
-      if (operationLog == null) {
-        LOG.error("Operation [ " + opHandle.getHandleIdentifier() + " ] "
-          + "logging is enabled, but its OperationLog object cannot be found.");
-      } else {
-        operationLog.close();
-      }
-    }
-  }
-
-  // TODO: make this abstract and implement in subclasses.
-  public void cancel() throws HiveSQLException {
-    setState(OperationState.CANCELED);
-    throw new UnsupportedOperationException("SQLOperation.cancel()");
-  }
-
-  public abstract void close() throws HiveSQLException;
-
-  public abstract TableSchema getResultSetSchema() throws HiveSQLException;
-
-  public abstract RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws HiveSQLException;
-
-  public RowSet getNextRowSet() throws HiveSQLException {
-    return getNextRowSet(FetchOrientation.FETCH_NEXT, DEFAULT_FETCH_MAX_ROWS);
-  }
-
-  /**
-   * Verify if the given fetch orientation is part of the default orientation types.
-   * @param orientation
-   * @throws HiveSQLException
-   */
-  protected void validateDefaultFetchOrientation(FetchOrientation orientation)
-      throws HiveSQLException {
-    validateFetchOrientation(orientation, DEFAULT_FETCH_ORIENTATION_SET);
-  }
-
-  /**
-   * Verify if the given fetch orientation is part of the supported orientation types.
-   * @param orientation
-   * @param supportedOrientations
-   * @throws HiveSQLException
-   */
-  protected void validateFetchOrientation(FetchOrientation orientation,
-      EnumSet<FetchOrientation> supportedOrientations) throws HiveSQLException {
-    if (!supportedOrientations.contains(orientation)) {
-      throw new HiveSQLException("The fetch type " + orientation.toString() +
-          " is not supported for this resultset", "HY106");
-    }
-  }
-
-  protected HiveSQLException toSQLException(String prefix, CommandProcessorResponse response) {
-    HiveSQLException ex = new HiveSQLException(prefix + ": " + response.getErrorMessage(),
-        response.getSQLState(), response.getResponseCode());
-    if (response.getException() != null) {
-      ex.initCause(response.getException());
-    }
-    return ex;
-  }
-
-  protected Map<String, String> getConfOverlay() {
-    return confOverlay;
-  }
-}
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/OperationManager.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/OperationManager.java
deleted file mode 100644
index 75edc5763ce44..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/OperationManager.java
+++ /dev/null
@@ -1,309 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.operation;
-
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Schema;
-import org.apache.hadoop.hive.ql.session.OperationLog;
-import org.apache.hive.service.AbstractService;
-import org.apache.hive.service.cli.FetchOrientation;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.OperationHandle;
-import org.apache.hive.service.cli.OperationState;
-import org.apache.hive.service.cli.OperationStatus;
-import org.apache.hive.service.cli.RowSet;
-import org.apache.hive.service.cli.RowSetFactory;
-import org.apache.hive.service.cli.TableSchema;
-import org.apache.hive.service.cli.session.HiveSession;
-import org.apache.log4j.Appender;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * OperationManager.
- *
- */
-public class OperationManager extends AbstractService {
-  private final Logger LOG = LoggerFactory.getLogger(OperationManager.class.getName());
-
-  private final Map<OperationHandle, Operation> handleToOperation =
-      new HashMap<OperationHandle, Operation>();
-
-  public OperationManager() {
-    super(OperationManager.class.getSimpleName());
-  }
-
-  @Override
-  public synchronized void init(HiveConf hiveConf) {
-    if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_ENABLED)) {
-      initOperationLogCapture(hiveConf.getVar(
-        HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL));
-    } else {
-      LOG.debug("Operation level logging is turned off");
-    }
-    super.init(hiveConf);
-  }
-
-  @Override
-  public synchronized void start() {
-    super.start();
-    // TODO
-  }
-
-  @Override
-  public synchronized void stop() {
-    // TODO
-    super.stop();
-  }
-
-  private void initOperationLogCapture(String loggingMode) {
-    // Register another Appender (with the same layout) that talks to us.
-    Appender ap = new LogDivertAppender(this, OperationLog.getLoggingLevel(loggingMode));
-    org.apache.log4j.Logger.getRootLogger().addAppender(ap);
-  }
-
-  public ExecuteStatementOperation newExecuteStatementOperation(HiveSession parentSession,
-      String statement, Map<String, String> confOverlay, boolean runAsync)
-          throws HiveSQLException {
-    ExecuteStatementOperation executeStatementOperation = ExecuteStatementOperation
-        .newExecuteStatementOperation(parentSession, statement, confOverlay, runAsync, 0);
-    addOperation(executeStatementOperation);
-    return executeStatementOperation;
-  }
-
-  public ExecuteStatementOperation newExecuteStatementOperation(HiveSession parentSession,
-      String statement, Map<String, String> confOverlay, boolean runAsync, long queryTimeout)
-          throws HiveSQLException {
-    return newExecuteStatementOperation(parentSession, statement, confOverlay, runAsync);
-  }
-
-  public GetTypeInfoOperation newGetTypeInfoOperation(HiveSession parentSession) {
-    GetTypeInfoOperation operation = new GetTypeInfoOperation(parentSession);
-    addOperation(operation);
-    return operation;
-  }
-
-  public GetCatalogsOperation newGetCatalogsOperation(HiveSession parentSession) {
-    GetCatalogsOperation operation = new GetCatalogsOperation(parentSession);
-    addOperation(operation);
-    return operation;
-  }
-
-  public GetSchemasOperation newGetSchemasOperation(HiveSession parentSession,
-      String catalogName, String schemaName) {
-    GetSchemasOperation operation = new GetSchemasOperation(parentSession, catalogName, schemaName);
-    addOperation(operation);
-    return operation;
-  }
-
-  public MetadataOperation newGetTablesOperation(HiveSession parentSession,
-      String catalogName, String schemaName, String tableName,
-      List<String> tableTypes) {
-    MetadataOperation operation =
-        new GetTablesOperation(parentSession, catalogName, schemaName, tableName, tableTypes);
-    addOperation(operation);
-    return operation;
-  }
-
-  public GetTableTypesOperation newGetTableTypesOperation(HiveSession parentSession) {
-    GetTableTypesOperation operation = new GetTableTypesOperation(parentSession);
-    addOperation(operation);
-    return operation;
-  }
-
-  public GetColumnsOperation newGetColumnsOperation(HiveSession parentSession,
-      String catalogName, String schemaName, String tableName, String columnName) {
-    GetColumnsOperation operation = new GetColumnsOperation(parentSession,
-        catalogName, schemaName, tableName, columnName);
-    addOperation(operation);
-    return operation;
-  }
-
-  public GetFunctionsOperation newGetFunctionsOperation(HiveSession parentSession,
-      String catalogName, String schemaName, String functionName) {
-    GetFunctionsOperation operation = new GetFunctionsOperation(parentSession,
-        catalogName, schemaName, functionName);
-    addOperation(operation);
-    return operation;
-  }
-
-  public GetPrimaryKeysOperation newGetPrimaryKeysOperation(HiveSession parentSession,
-      String catalogName, String schemaName, String tableName) {
-    GetPrimaryKeysOperation operation = new GetPrimaryKeysOperation(parentSession,
-      catalogName, schemaName, tableName);
-    addOperation(operation);
-    return operation;
-  }
-
-  public GetCrossReferenceOperation newGetCrossReferenceOperation(
-      HiveSession session, String primaryCatalog, String primarySchema,
-      String primaryTable, String foreignCatalog, String foreignSchema,
-      String foreignTable) {
-   GetCrossReferenceOperation operation = new GetCrossReferenceOperation(session,
-     primaryCatalog, primarySchema, primaryTable, foreignCatalog, foreignSchema,
-     foreignTable);
-   addOperation(operation);
-   return operation;
-  }
-
-  public Operation getOperation(OperationHandle operationHandle) throws HiveSQLException {
-    Operation operation = getOperationInternal(operationHandle);
-    if (operation == null) {
-      throw new HiveSQLException("Invalid OperationHandle: " + operationHandle);
-    }
-    return operation;
-  }
-
-  private synchronized Operation getOperationInternal(OperationHandle operationHandle) {
-    return handleToOperation.get(operationHandle);
-  }
-
-  private synchronized Operation removeTimedOutOperation(OperationHandle operationHandle) {
-    Operation operation = handleToOperation.get(operationHandle);
-    if (operation != null && operation.isTimedOut(System.currentTimeMillis())) {
-      handleToOperation.remove(operationHandle);
-      return operation;
-    }
-    return null;
-  }
-
-  private synchronized void addOperation(Operation operation) {
-    handleToOperation.put(operation.getHandle(), operation);
-  }
-
-  private synchronized Operation removeOperation(OperationHandle opHandle) {
-    return handleToOperation.remove(opHandle);
-  }
-
-  public OperationStatus getOperationStatus(OperationHandle opHandle)
-      throws HiveSQLException {
-    return getOperation(opHandle).getStatus();
-  }
-
-  public void cancelOperation(OperationHandle opHandle) throws HiveSQLException {
-    Operation operation = getOperation(opHandle);
-    OperationState opState = operation.getStatus().getState();
-    if (opState == OperationState.CANCELED ||
-        opState == OperationState.CLOSED ||
-        opState == OperationState.FINISHED ||
-        opState == OperationState.ERROR ||
-        opState == OperationState.UNKNOWN) {
-      // Cancel should be a no-op in either cases
-      LOG.debug(opHandle + ": Operation is already aborted in state - " + opState);
-    }
-    else {
-      LOG.debug(opHandle + ": Attempting to cancel from state - " + opState);
-      operation.cancel();
-    }
-  }
-
-  public void closeOperation(OperationHandle opHandle) throws HiveSQLException {
-    Operation operation = removeOperation(opHandle);
-    if (operation == null) {
-      throw new HiveSQLException("Operation does not exist!");
-    }
-    operation.close();
-  }
-
-  public TableSchema getOperationResultSetSchema(OperationHandle opHandle)
-      throws HiveSQLException {
-    return getOperation(opHandle).getResultSetSchema();
-  }
-
-  public RowSet getOperationNextRowSet(OperationHandle opHandle)
-      throws HiveSQLException {
-    return getOperation(opHandle).getNextRowSet();
-  }
-
-  public RowSet getOperationNextRowSet(OperationHandle opHandle,
-      FetchOrientation orientation, long maxRows)
-          throws HiveSQLException {
-    return getOperation(opHandle).getNextRowSet(orientation, maxRows);
-  }
-
-  public RowSet getOperationLogRowSet(OperationHandle opHandle,
-      FetchOrientation orientation, long maxRows)
-          throws HiveSQLException {
-    // get the OperationLog object from the operation
-    OperationLog operationLog = getOperation(opHandle).getOperationLog();
-    if (operationLog == null) {
-      throw new HiveSQLException("Couldn't find log associated with operation handle: " + opHandle);
-    }
-
-    // read logs
-    List<String> logs;
-    try {
-      logs = operationLog.readOperationLog(isFetchFirst(orientation), maxRows);
-    } catch (SQLException e) {
-      throw new HiveSQLException(e.getMessage(), e.getCause());
-    }
-
-
-    // convert logs to RowSet
-    TableSchema tableSchema = new TableSchema(getLogSchema());
-    RowSet rowSet = RowSetFactory.create(tableSchema,
-        getOperation(opHandle).getProtocolVersion(), false);
-    for (String log : logs) {
-      rowSet.addRow(new String[] {log});
-    }
-
-    return rowSet;
-  }
-
-  private boolean isFetchFirst(FetchOrientation fetchOrientation) {
-    //TODO: Since OperationLog is moved to package o.a.h.h.ql.session,
-    // we may add a Enum there and map FetchOrientation to it.
-    if (fetchOrientation.equals(FetchOrientation.FETCH_FIRST)) {
-      return true;
-    }
-    return false;
-  }
-
-  private Schema getLogSchema() {
-    Schema schema = new Schema();
-    FieldSchema fieldSchema = new FieldSchema();
-    fieldSchema.setName("operation_log");
-    fieldSchema.setType("string");
-    schema.addToFieldSchemas(fieldSchema);
-    return schema;
-  }
-
-  public OperationLog getOperationLogByThread() {
-    return OperationLog.getCurrentOperationLog();
-  }
-
-  public List<Operation> removeExpiredOperations(OperationHandle[] handles) {
-    List<Operation> removed = new ArrayList<Operation>();
-    for (OperationHandle handle : handles) {
-      Operation operation = removeTimedOutOperation(handle);
-      if (operation != null) {
-        LOG.warn("Operation " + handle + " is timed-out and will be closed");
-        removed.add(operation);
-      }
-    }
-    return removed;
-  }
-}
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/SQLOperation.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/SQLOperation.java
deleted file mode 100644
index e2ac1ea78c1ab..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/SQLOperation.java
+++ /dev/null
@@ -1,456 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.operation;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.security.PrivilegedExceptionAction;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.Future;
-import java.util.concurrent.RejectedExecutionException;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Schema;
-import org.apache.hadoop.hive.ql.CommandNeedRetryException;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.QueryState;
-import org.apache.hadoop.hive.ql.exec.ExplainTask;
-import org.apache.hadoop.hive.ql.exec.Task;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.hive.serde2.AbstractSerDe;
-import org.apache.hadoop.hive.serde2.SerDeException;
-import org.apache.hadoop.hive.serde2.SerDeUtils;
-import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructField;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.shims.Utils;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hive.service.cli.FetchOrientation;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.OperationState;
-import org.apache.hive.service.cli.RowSet;
-import org.apache.hive.service.cli.RowSetFactory;
-import org.apache.hive.service.cli.TableSchema;
-import org.apache.hive.service.cli.session.HiveSession;
-import org.apache.hive.service.server.ThreadWithGarbageCleanup;
-
-/**
- * SQLOperation.
- *
- */
-public class SQLOperation extends ExecuteStatementOperation {
-
-  private Driver driver = null;
-  private CommandProcessorResponse response;
-  private TableSchema resultSchema = null;
-  private Schema mResultSchema = null;
-  private AbstractSerDe serde = null;
-  private boolean fetchStarted = false;
-
-  public SQLOperation(HiveSession parentSession, String statement, Map<String, String> confOverlay,
-      boolean runInBackground, long queryTimeout) {
-    // TODO: call setRemoteUser in ExecuteStatementOperation or higher.
-    super(parentSession, statement, confOverlay, runInBackground);
-  }
-
-  /***
-   * Compile the query and extract metadata
-   * @param queryState
-   * @throws HiveSQLException
-   */
-  public void prepare(QueryState queryState) throws HiveSQLException {
-    setState(OperationState.RUNNING);
-
-    try {
-      driver = new Driver(queryState, getParentSession().getUserName());
-
-      // set the operation handle information in Driver, so that thrift API users
-      // can use the operation handle they receive, to lookup query information in
-      // Yarn ATS
-      String guid64 = Base64.encodeBase64URLSafeString(getHandle().getHandleIdentifier()
-          .toTHandleIdentifier().getGuid()).trim();
-      driver.setOperationId(guid64);
-
-      // In Hive server mode, we are not able to retry in the FetchTask
-      // case, when calling fetch queries since execute() has returned.
-      // For now, we disable the test attempts.
-      driver.setTryCount(Integer.MAX_VALUE);
-
-      response = driver.compileAndRespond(statement);
-      if (0 != response.getResponseCode()) {
-        throw toSQLException("Error while compiling statement", response);
-      }
-
-      mResultSchema = driver.getSchema();
-
-      // hasResultSet should be true only if the query has a FetchTask
-      // "explain" is an exception for now
-      if(driver.getPlan().getFetchTask() != null) {
-        //Schema has to be set
-        if (mResultSchema == null || !mResultSchema.isSetFieldSchemas()) {
-          throw new HiveSQLException("Error compiling query: Schema and FieldSchema " +
-              "should be set when query plan has a FetchTask");
-        }
-        resultSchema = new TableSchema(mResultSchema);
-        setHasResultSet(true);
-      } else {
-        setHasResultSet(false);
-      }
-      // Set hasResultSet true if the plan has ExplainTask
-      // TODO explain should use a FetchTask for reading
-      for (Task<? extends Serializable> task: driver.getPlan().getRootTasks()) {
-        if (task.getClass() == ExplainTask.class) {
-          resultSchema = new TableSchema(mResultSchema);
-          setHasResultSet(true);
-          break;
-        }
-      }
-    } catch (HiveSQLException e) {
-      setState(OperationState.ERROR);
-      throw e;
-    } catch (Exception e) {
-      setState(OperationState.ERROR);
-      throw new HiveSQLException("Error running query: " + e.toString(), e);
-    }
-  }
-
-  private void runQuery(HiveConf sqlOperationConf) throws HiveSQLException {
-    try {
-      // In Hive server mode, we are not able to retry in the FetchTask
-      // case, when calling fetch queries since execute() has returned.
-      // For now, we disable the test attempts.
-      driver.setTryCount(Integer.MAX_VALUE);
-      response = driver.run();
-      if (0 != response.getResponseCode()) {
-        throw toSQLException("Error while processing statement", response);
-      }
-    } catch (HiveSQLException e) {
-      // If the operation was cancelled by another thread,
-      // Driver#run will return a non-zero response code.
-      // We will simply return if the operation state is CANCELED,
-      // otherwise throw an exception
-      if (getStatus().getState() == OperationState.CANCELED) {
-        return;
-      }
-      else {
-        setState(OperationState.ERROR);
-        throw e;
-      }
-    } catch (Exception e) {
-      setState(OperationState.ERROR);
-      throw new HiveSQLException("Error running query: " + e.toString(), e);
-    }
-    setState(OperationState.FINISHED);
-  }
-
-  @Override
-  public void runInternal() throws HiveSQLException {
-    setState(OperationState.PENDING);
-    final HiveConf opConfig = getConfigForOperation();
-    prepare(queryState);
-    if (!shouldRunAsync()) {
-      runQuery(opConfig);
-    } else {
-      // We'll pass ThreadLocals in the background thread from the foreground (handler) thread
-      final SessionState parentSessionState = SessionState.get();
-      // ThreadLocal Hive object needs to be set in background thread.
-      // The metastore client in Hive is associated with right user.
-      final Hive parentHive = getSessionHive();
-      // Current UGI will get used by metastore when metsatore is in embedded mode
-      // So this needs to get passed to the new background thread
-      final UserGroupInformation currentUGI = getCurrentUGI(opConfig);
-      // Runnable impl to call runInternal asynchronously,
-      // from a different thread
-      Runnable backgroundOperation = new Runnable() {
-        @Override
-        public void run() {
-          PrivilegedExceptionAction<Object> doAsAction = new PrivilegedExceptionAction<Object>() {
-            @Override
-            public Object run() throws HiveSQLException {
-              Hive.set(parentHive);
-              SessionState.setCurrentSessionState(parentSessionState);
-              // Set current OperationLog in this async thread for keeping on saving query log.
-              registerCurrentOperationLog();
-              try {
-                runQuery(opConfig);
-              } catch (HiveSQLException e) {
-                setOperationException(e);
-                LOG.error("Error running hive query: ", e);
-              } finally {
-                unregisterOperationLog();
-              }
-              return null;
-            }
-          };
-
-          try {
-            currentUGI.doAs(doAsAction);
-          } catch (Exception e) {
-            setOperationException(new HiveSQLException(e));
-            LOG.error("Error running hive query as user : " + currentUGI.getShortUserName(), e);
-          }
-          finally {
-            /**
-             * We'll cache the ThreadLocal RawStore object for this background thread for an orderly cleanup
-             * when this thread is garbage collected later.
-             * @see org.apache.hive.service.server.ThreadWithGarbageCleanup#finalize()
-             */
-            if (ThreadWithGarbageCleanup.currentThread() instanceof ThreadWithGarbageCleanup) {
-              ThreadWithGarbageCleanup currentThread =
-                  (ThreadWithGarbageCleanup) ThreadWithGarbageCleanup.currentThread();
-              currentThread.cacheThreadLocalRawStore();
-            }
-          }
-        }
-      };
-      try {
-        // This submit blocks if no background threads are available to run this operation
-        Future<?> backgroundHandle =
-            getParentSession().getSessionManager().submitBackgroundOperation(backgroundOperation);
-        setBackgroundHandle(backgroundHandle);
-      } catch (RejectedExecutionException rejected) {
-        setState(OperationState.ERROR);
-        throw new HiveSQLException("The background threadpool cannot accept" +
-            " new task for execution, please retry the operation", rejected);
-      }
-    }
-  }
-
-  /**
-   * Returns the current UGI on the stack
-   * @param opConfig
-   * @return UserGroupInformation
-   * @throws HiveSQLException
-   */
-  private UserGroupInformation getCurrentUGI(HiveConf opConfig) throws HiveSQLException {
-    try {
-      return Utils.getUGI();
-    } catch (Exception e) {
-      throw new HiveSQLException("Unable to get current user", e);
-    }
-  }
-
-  /**
-   * Returns the ThreadLocal Hive for the current thread
-   * @return Hive
-   * @throws HiveSQLException
-   */
-  private Hive getSessionHive() throws HiveSQLException {
-    try {
-      return Hive.get();
-    } catch (HiveException e) {
-      throw new HiveSQLException("Failed to get ThreadLocal Hive object", e);
-    }
-  }
-
-  private void cleanup(OperationState state) throws HiveSQLException {
-    setState(state);
-    if (shouldRunAsync()) {
-      Future<?> backgroundHandle = getBackgroundHandle();
-      if (backgroundHandle != null) {
-        backgroundHandle.cancel(true);
-      }
-    }
-    if (driver != null) {
-      driver.close();
-      driver.destroy();
-    }
-    driver = null;
-
-    SessionState ss = SessionState.get();
-    if (ss.getTmpOutputFile() != null) {
-      ss.getTmpOutputFile().delete();
-    }
-  }
-
-  @Override
-  public void cancel() throws HiveSQLException {
-    cleanup(OperationState.CANCELED);
-  }
-
-  @Override
-  public void close() throws HiveSQLException {
-    cleanup(OperationState.CLOSED);
-    cleanupOperationLog();
-  }
-
-  @Override
-  public TableSchema getResultSetSchema() throws HiveSQLException {
-    assertState(OperationState.FINISHED);
-    if (resultSchema == null) {
-      resultSchema = new TableSchema(driver.getSchema());
-    }
-    return resultSchema;
-  }
-
-  private final transient List<Object> convey = new ArrayList<Object>();
-
-  @Override
-  public RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws HiveSQLException {
-    validateDefaultFetchOrientation(orientation);
-    assertState(OperationState.FINISHED);
-
-    RowSet rowSet = RowSetFactory.create(resultSchema, getProtocolVersion(), false);
-
-    try {
-      /* if client is requesting fetch-from-start and its not the first time reading from this operation
-       * then reset the fetch position to beginning
-       */
-      if (orientation.equals(FetchOrientation.FETCH_FIRST) && fetchStarted) {
-        driver.resetFetch();
-      }
-      fetchStarted = true;
-      driver.setMaxRows((int) maxRows);
-      if (driver.getResults(convey)) {
-        return decode(convey, rowSet);
-      }
-      return rowSet;
-    } catch (IOException e) {
-      throw new HiveSQLException(e);
-    } catch (CommandNeedRetryException e) {
-      throw new HiveSQLException(e);
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    } finally {
-      convey.clear();
-    }
-  }
-
-  private RowSet decode(List<Object> rows, RowSet rowSet) throws Exception {
-    if (driver.isFetchingTable()) {
-      return prepareFromRow(rows, rowSet);
-    }
-    return decodeFromString(rows, rowSet);
-  }
-
-  // already encoded to thrift-able object in ThriftFormatter
-  private RowSet prepareFromRow(List<Object> rows, RowSet rowSet) throws Exception {
-    for (Object row : rows) {
-      rowSet.addRow((Object[]) row);
-    }
-    return rowSet;
-  }
-
-  private RowSet decodeFromString(List<Object> rows, RowSet rowSet)
-      throws SQLException, SerDeException {
-    getSerDe();
-    StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector();
-    List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs();
-
-    Object[] deserializedFields = new Object[fieldRefs.size()];
-    Object rowObj;
-    ObjectInspector fieldOI;
-
-    int protocol = getProtocolVersion().getValue();
-    for (Object rowString : rows) {
-      rowObj = serde.deserialize(new BytesWritable(((String)rowString).getBytes(UTF_8)));
-      for (int i = 0; i < fieldRefs.size(); i++) {
-        StructField fieldRef = fieldRefs.get(i);
-        fieldOI = fieldRef.getFieldObjectInspector();
-        Object fieldData = soi.getStructFieldData(rowObj, fieldRef);
-        deserializedFields[i] = SerDeUtils.toThriftPayload(fieldData, fieldOI, protocol);
-      }
-      rowSet.addRow(deserializedFields);
-    }
-    return rowSet;
-  }
-
-  private AbstractSerDe getSerDe() throws SQLException {
-    if (serde != null) {
-      return serde;
-    }
-    try {
-      List<FieldSchema> fieldSchemas = mResultSchema.getFieldSchemas();
-      StringBuilder namesSb = new StringBuilder();
-      StringBuilder typesSb = new StringBuilder();
-
-      if (fieldSchemas != null && !fieldSchemas.isEmpty()) {
-        for (int pos = 0; pos < fieldSchemas.size(); pos++) {
-          if (pos != 0) {
-            namesSb.append(",");
-            typesSb.append(",");
-          }
-          namesSb.append(fieldSchemas.get(pos).getName());
-          typesSb.append(fieldSchemas.get(pos).getType());
-        }
-      }
-      String names = namesSb.toString();
-      String types = typesSb.toString();
-
-      serde = new LazySimpleSerDe();
-      Properties props = new Properties();
-      if (names.length() > 0) {
-        LOG.debug("Column names: " + names);
-        props.setProperty(serdeConstants.LIST_COLUMNS, names);
-      }
-      if (types.length() > 0) {
-        LOG.debug("Column types: " + types);
-        props.setProperty(serdeConstants.LIST_COLUMN_TYPES, types);
-      }
-      SerDeUtils.initializeSerDe(serde, new HiveConf(), props, null);
-
-    } catch (Exception ex) {
-      ex.printStackTrace();
-      throw new SQLException("Could not create ResultSet: " + ex.getMessage(), ex);
-    }
-    return serde;
-  }
-
-  /**
-   * If there are query specific settings to overlay, then create a copy of config
-   * There are two cases we need to clone the session config that's being passed to hive driver
-   * 1. Async query -
-   *    If the client changes a config setting, that shouldn't reflect in the execution already underway
-   * 2. confOverlay -
-   *    The query specific settings should only be applied to the query config and not session
-   * @return new configuration
-   * @throws HiveSQLException
-   */
-  private HiveConf getConfigForOperation() throws HiveSQLException {
-    HiveConf sqlOperationConf = getParentSession().getHiveConf();
-    if (!getConfOverlay().isEmpty() || shouldRunAsync()) {
-      // clone the parent session config for this query
-      sqlOperationConf = new HiveConf(sqlOperationConf);
-
-      // apply overlay query specific settings, if any
-      for (Map.Entry<String, String> confEntry : getConfOverlay().entrySet()) {
-        try {
-          sqlOperationConf.verifyAndSet(confEntry.getKey(), confEntry.getValue());
-        } catch (IllegalArgumentException e) {
-          throw new HiveSQLException("Error applying statement specific settings", e);
-        }
-      }
-    }
-    return sqlOperationConf;
-  }
-}
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
deleted file mode 100644
index 14e9c4704c977..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
+++ /dev/null
@@ -1,893 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.session;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.hive.common.cli.HiveFileProcessor;
-import org.apache.hadoop.hive.common.cli.IHiveFileProcessor;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.conf.VariableSubstitution;
-import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.history.HiveHistory;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.serde2.SerDeUtils;
-import org.apache.hadoop.hive.serde2.thrift.ThriftFormatter;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hive.common.util.HiveVersionInfo;
-import org.apache.hive.service.auth.HiveAuthFactory;
-import org.apache.hive.service.cli.FetchOrientation;
-import org.apache.hive.service.cli.FetchType;
-import org.apache.hive.service.cli.GetInfoType;
-import org.apache.hive.service.cli.GetInfoValue;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.OperationHandle;
-import org.apache.hive.service.cli.RowSet;
-import org.apache.hive.service.cli.SessionHandle;
-import org.apache.hive.service.cli.TableSchema;
-import org.apache.hive.service.cli.operation.ExecuteStatementOperation;
-import org.apache.hive.service.cli.operation.GetCatalogsOperation;
-import org.apache.hive.service.cli.operation.GetColumnsOperation;
-import org.apache.hive.service.cli.operation.GetCrossReferenceOperation;
-import org.apache.hive.service.cli.operation.GetFunctionsOperation;
-import org.apache.hive.service.cli.operation.GetPrimaryKeysOperation;
-import org.apache.hive.service.cli.operation.GetSchemasOperation;
-import org.apache.hive.service.cli.operation.GetTableTypesOperation;
-import org.apache.hive.service.cli.operation.GetTypeInfoOperation;
-import org.apache.hive.service.cli.operation.MetadataOperation;
-import org.apache.hive.service.cli.operation.Operation;
-import org.apache.hive.service.cli.operation.OperationManager;
-import org.apache.hive.service.rpc.thrift.TProtocolVersion;
-import org.apache.hive.service.server.ThreadWithGarbageCleanup;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.hadoop.hive.conf.SystemVariables.ENV_PREFIX;
-import static org.apache.hadoop.hive.conf.SystemVariables.HIVECONF_PREFIX;
-import static org.apache.hadoop.hive.conf.SystemVariables.HIVEVAR_PREFIX;
-import static org.apache.hadoop.hive.conf.SystemVariables.METACONF_PREFIX;
-import static org.apache.hadoop.hive.conf.SystemVariables.SYSTEM_PREFIX;
-
-/**
- * HiveSession
- *
- */
-public class HiveSessionImpl implements HiveSession {
-  private final SessionHandle sessionHandle;
-  private String username;
-  private final String password;
-  private HiveConf hiveConf;
-  private SessionState sessionState;
-  private String ipAddress;
-  private static final String FETCH_WORK_SERDE_CLASS =
-      "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe";
-  private static final Logger LOG = LoggerFactory.getLogger(HiveSessionImpl.class);
-  private SessionManager sessionManager;
-  private OperationManager operationManager;
-  private final Set<OperationHandle> opHandleSet = new HashSet<OperationHandle>();
-  private boolean isOperationLogEnabled;
-  private File sessionLogDir;
-  private volatile long lastAccessTime;
-  private volatile long lastIdleTime;
-
-  public HiveSessionImpl(TProtocolVersion protocol, String username, String password,
-      HiveConf serverhiveConf, String ipAddress) {
-    this.username = username;
-    this.password = password;
-    this.sessionHandle = new SessionHandle(protocol);
-    this.hiveConf = new HiveConf(serverhiveConf);
-    this.ipAddress = ipAddress;
-
-    try {
-      // In non-impersonation mode, map scheduler queue to current user
-      // if fair scheduler is configured.
-      if (! hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS) &&
-        hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE)) {
-        ShimLoader.getHadoopShims().refreshDefaultQueue(hiveConf, username);
-      }
-    } catch (IOException e) {
-      LOG.warn("Error setting scheduler queue: " + e, e);
-    }
-    // Set an explicit session name to control the download directory name
-    hiveConf.set(ConfVars.HIVESESSIONID.varname,
-        sessionHandle.getHandleIdentifier().toString());
-    // Use thrift transportable formatter
-    hiveConf.set(SerDeUtils.LIST_SINK_OUTPUT_FORMATTER, ThriftFormatter.class.getName());
-    hiveConf.setInt(SerDeUtils.LIST_SINK_OUTPUT_PROTOCOL, protocol.getValue());
-  }
-
-  @Override
-  /**
-   * Opens a new HiveServer2 session for the client connection.
-   * Creates a new SessionState object that will be associated with this HiveServer2 session.
-   * When the server executes multiple queries in the same session,
-   * this SessionState object is reused across multiple queries.
-   * Note that if doAs is true, this call goes through a proxy object,
-   * which wraps the method logic in a UserGroupInformation#doAs.
-   * That's why it is important to create SessionState here rather than in the constructor.
-   */
-  public void open(Map<String, String> sessionConfMap) throws HiveSQLException {
-    sessionState = new SessionState(hiveConf, username);
-    sessionState.setUserIpAddress(ipAddress);
-    sessionState.setIsHiveServerQuery(true);
-    SessionState.start(sessionState);
-    try {
-      sessionState.loadAuxJars();
-      sessionState.loadReloadableAuxJars();
-    } catch (IOException e) {
-      String msg = "Failed to load reloadable jar file path: " + e;
-      LOG.error(msg, e);
-      throw new HiveSQLException(msg, e);
-    }
-    // Process global init file: .hiverc
-    processGlobalInitFile();
-    if (sessionConfMap != null) {
-      configureSession(sessionConfMap);
-    }
-    lastAccessTime = System.currentTimeMillis();
-    lastIdleTime = lastAccessTime;
-  }
-
-  /**
-   * It is used for processing hiverc file from HiveServer2 side.
-   */
-  private class GlobalHivercFileProcessor extends HiveFileProcessor {
-    @Override
-    protected BufferedReader loadFile(String fileName) throws IOException {
-      FileInputStream initStream = null;
-      BufferedReader bufferedReader = null;
-      initStream = new FileInputStream(fileName);
-      bufferedReader = new BufferedReader(new InputStreamReader(initStream));
-      return bufferedReader;
-    }
-
-    @Override
-    protected int processCmd(String cmd) {
-      int rc = 0;
-      String cmd_trimed = cmd.trim();
-      try {
-        executeStatementInternal(cmd_trimed, null, false, 0);
-      } catch (HiveSQLException e) {
-        rc = -1;
-        LOG.warn("Failed to execute HQL command in global .hiverc file.", e);
-      }
-      return rc;
-    }
-  }
-
-  private void processGlobalInitFile() {
-    IHiveFileProcessor processor = new GlobalHivercFileProcessor();
-
-    try {
-      String hiverc = hiveConf.getVar(ConfVars.HIVE_SERVER2_GLOBAL_INIT_FILE_LOCATION);
-      if (hiverc != null) {
-        File hivercFile = new File(hiverc);
-        if (hivercFile.isDirectory()) {
-          hivercFile = new File(hivercFile, SessionManager.HIVERCFILE);
-        }
-        if (hivercFile.isFile()) {
-          LOG.info("Running global init file: " + hivercFile);
-          int rc = processor.processFile(hivercFile.getAbsolutePath());
-          if (rc != 0) {
-            LOG.error("Failed on initializing global .hiverc file");
-          }
-        } else {
-          LOG.debug("Global init file " + hivercFile + " does not exist");
-        }
-      }
-    } catch (IOException e) {
-      LOG.warn("Failed on initializing global .hiverc file", e);
-    }
-  }
-
-  private void configureSession(Map<String, String> sessionConfMap) throws HiveSQLException {
-    SessionState.setCurrentSessionState(sessionState);
-    for (Map.Entry<String, String> entry : sessionConfMap.entrySet()) {
-      String key = entry.getKey();
-      if (key.startsWith("set:")) {
-        try {
-          setVariable(key.substring(4), entry.getValue());
-        } catch (Exception e) {
-          throw new HiveSQLException(e);
-        }
-      } else if (key.startsWith("use:")) {
-        SessionState.get().setCurrentDatabase(entry.getValue());
-      } else {
-        hiveConf.verifyAndSet(key, entry.getValue());
-      }
-    }
-  }
-
-  // Copy from org.apache.hadoop.hive.ql.processors.SetProcessor, only change:
-  // setConf(varname, propName, varvalue, true) when varname.startsWith(HIVECONF_PREFIX)
-  public static int setVariable(String varname, String varvalue) throws Exception {
-    SessionState ss = SessionState.get();
-    VariableSubstitution substitution = new VariableSubstitution(() -> ss.getHiveVariables());
-    if (varvalue.contains("\n")){
-      ss.err.println("Warning: Value had a \\n character in it.");
-    }
-    varname = varname.trim();
-    if (varname.startsWith(ENV_PREFIX)){
-      ss.err.println("env:* variables can not be set.");
-      return 1;
-    } else if (varname.startsWith(SYSTEM_PREFIX)){
-      String propName = varname.substring(SYSTEM_PREFIX.length());
-      System.getProperties().setProperty(propName, substitution.substitute(ss.getConf(),varvalue));
-    } else if (varname.startsWith(HIVECONF_PREFIX)){
-      String propName = varname.substring(HIVECONF_PREFIX.length());
-      setConf(varname, propName, varvalue, true);
-    } else if (varname.startsWith(HIVEVAR_PREFIX)) {
-      String propName = varname.substring(HIVEVAR_PREFIX.length());
-      ss.getHiveVariables().put(propName, substitution.substitute(ss.getConf(),varvalue));
-    } else if (varname.startsWith(METACONF_PREFIX)) {
-      String propName = varname.substring(METACONF_PREFIX.length());
-      Hive hive = Hive.get(ss.getConf());
-      hive.setMetaConf(propName, substitution.substitute(ss.getConf(), varvalue));
-    } else {
-      setConf(varname, varname, varvalue, true);
-    }
-    return 0;
-  }
-
-  // returns non-null string for validation fail
-  private static void setConf(String varname, String key, String varvalue, boolean register)
-          throws IllegalArgumentException {
-    VariableSubstitution substitution =
-        new VariableSubstitution(() -> SessionState.get().getHiveVariables());
-    HiveConf conf = SessionState.get().getConf();
-    String value = substitution.substitute(conf, varvalue);
-    if (conf.getBoolVar(HiveConf.ConfVars.HIVECONFVALIDATION)) {
-      HiveConf.ConfVars confVars = HiveConf.getConfVars(key);
-      if (confVars != null) {
-        if (!confVars.isType(value)) {
-          StringBuilder message = new StringBuilder();
-          message.append("'SET ").append(varname).append('=').append(varvalue);
-          message.append("' FAILED because ").append(key).append(" expects ");
-          message.append(confVars.typeString()).append(" type value.");
-          throw new IllegalArgumentException(message.toString());
-        }
-        String fail = confVars.validate(value);
-        if (fail != null) {
-          StringBuilder message = new StringBuilder();
-          message.append("'SET ").append(varname).append('=').append(varvalue);
-          message.append("' FAILED in validation : ").append(fail).append('.');
-          throw new IllegalArgumentException(message.toString());
-        }
-      } else if (key.startsWith("hive.")) {
-        throw new IllegalArgumentException("hive configuration " + key + " does not exists.");
-      }
-    }
-    conf.verifyAndSet(key, value);
-    if (register) {
-      SessionState.get().getOverriddenConfigurations().put(key, value);
-    }
-  }
-
-  @Override
-  public void setOperationLogSessionDir(File operationLogRootDir) {
-    if (!operationLogRootDir.exists()) {
-      LOG.warn("The operation log root directory is removed, recreating: " +
-          operationLogRootDir.getAbsolutePath());
-      if (!operationLogRootDir.mkdirs()) {
-        LOG.warn("Unable to create operation log root directory: " +
-            operationLogRootDir.getAbsolutePath());
-      }
-    }
-    if (!operationLogRootDir.canWrite()) {
-      LOG.warn("The operation log root directory is not writable: " +
-          operationLogRootDir.getAbsolutePath());
-    }
-    sessionLogDir = new File(operationLogRootDir, sessionHandle.getHandleIdentifier().toString());
-    isOperationLogEnabled = true;
-    if (!sessionLogDir.exists()) {
-      if (!sessionLogDir.mkdir()) {
-        LOG.warn("Unable to create operation log session directory: " +
-            sessionLogDir.getAbsolutePath());
-        isOperationLogEnabled = false;
-      }
-    }
-    if (isOperationLogEnabled) {
-      LOG.info("Operation log session directory is created: " + sessionLogDir.getAbsolutePath());
-    }
-  }
-
-  @Override
-  public boolean isOperationLogEnabled() {
-    return isOperationLogEnabled;
-  }
-
-  @Override
-  public File getOperationLogSessionDir() {
-    return sessionLogDir;
-  }
-
-  @Override
-  public TProtocolVersion getProtocolVersion() {
-    return sessionHandle.getProtocolVersion();
-  }
-
-  @Override
-  public SessionManager getSessionManager() {
-    return sessionManager;
-  }
-
-  @Override
-  public void setSessionManager(SessionManager sessionManager) {
-    this.sessionManager = sessionManager;
-  }
-
-  private OperationManager getOperationManager() {
-    return operationManager;
-  }
-
-  @Override
-  public void setOperationManager(OperationManager operationManager) {
-    this.operationManager = operationManager;
-  }
-
-  protected synchronized void acquire(boolean userAccess) {
-    // Need to make sure that the this HiveServer2's session's SessionState is
-    // stored in the thread local for the handler thread.
-    SessionState.setCurrentSessionState(sessionState);
-    if (userAccess) {
-      lastAccessTime = System.currentTimeMillis();
-    }
-  }
-
-  /**
-   * 1. We'll remove the ThreadLocal SessionState as this thread might now serve
-   * other requests.
-   * 2. We'll cache the ThreadLocal RawStore object for this background thread for an orderly cleanup
-   * when this thread is garbage collected later.
-   * @see org.apache.hive.service.server.ThreadWithGarbageCleanup#finalize()
-   */
-  protected synchronized void release(boolean userAccess) {
-    SessionState.detachSession();
-    if (ThreadWithGarbageCleanup.currentThread() instanceof ThreadWithGarbageCleanup) {
-      ThreadWithGarbageCleanup currentThread =
-          (ThreadWithGarbageCleanup) ThreadWithGarbageCleanup.currentThread();
-      currentThread.cacheThreadLocalRawStore();
-    }
-    if (userAccess) {
-      lastAccessTime = System.currentTimeMillis();
-    }
-    if (opHandleSet.isEmpty()) {
-      lastIdleTime = System.currentTimeMillis();
-    } else {
-      lastIdleTime = 0;
-    }
-  }
-
-  @Override
-  public SessionHandle getSessionHandle() {
-    return sessionHandle;
-  }
-
-  @Override
-  public String getUsername() {
-    return username;
-  }
-
-  @Override
-  public String getPassword() {
-    return password;
-  }
-
-  @Override
-  public HiveConf getHiveConf() {
-    hiveConf.setVar(HiveConf.ConfVars.HIVEFETCHOUTPUTSERDE, FETCH_WORK_SERDE_CLASS);
-    return hiveConf;
-  }
-
-  @Override
-  public IMetaStoreClient getMetaStoreClient() throws HiveSQLException {
-    try {
-      return Hive.get(getHiveConf()).getMSC();
-    } catch (HiveException e) {
-      throw new HiveSQLException("Failed to get metastore connection", e);
-    } catch (MetaException e) {
-      throw new HiveSQLException("Failed to get metastore connection", e);
-    }
-  }
-
-  @Override
-  public GetInfoValue getInfo(GetInfoType getInfoType)
-      throws HiveSQLException {
-    acquire(true);
-    try {
-      switch (getInfoType) {
-      case CLI_SERVER_NAME:
-        return new GetInfoValue("Hive");
-      case CLI_DBMS_NAME:
-        return new GetInfoValue("Apache Hive");
-      case CLI_DBMS_VER:
-        return new GetInfoValue(HiveVersionInfo.getVersion());
-      case CLI_MAX_COLUMN_NAME_LEN:
-        return new GetInfoValue(128);
-      case CLI_MAX_SCHEMA_NAME_LEN:
-        return new GetInfoValue(128);
-      case CLI_MAX_TABLE_NAME_LEN:
-        return new GetInfoValue(128);
-      case CLI_TXN_CAPABLE:
-      default:
-        throw new HiveSQLException("Unrecognized GetInfoType value: " + getInfoType.toString());
-      }
-    } finally {
-      release(true);
-    }
-  }
-
-  @Override
-  public OperationHandle executeStatement(String statement, Map<String, String> confOverlay)
-      throws HiveSQLException {
-    return executeStatementInternal(statement, confOverlay, false, 0);
-  }
-
-  @Override
-  public OperationHandle executeStatement(String statement, Map<String, String> confOverlay,
-      long queryTimeout) throws HiveSQLException {
-    return executeStatementInternal(statement, confOverlay, false, queryTimeout);
-  }
-
-  @Override
-  public OperationHandle executeStatementAsync(String statement, Map<String, String> confOverlay)
-      throws HiveSQLException {
-    return executeStatementInternal(statement, confOverlay, true, 0);
-  }
-
-  @Override
-  public OperationHandle executeStatementAsync(String statement, Map<String, String> confOverlay,
-      long queryTimeout) throws HiveSQLException {
-    return executeStatementInternal(statement, confOverlay, true, queryTimeout);
-  }
-
-  private OperationHandle executeStatementInternal(String statement,
-      Map<String, String> confOverlay, boolean runAsync, long queryTimeout) throws HiveSQLException {
-    acquire(true);
-
-    OperationManager operationManager = getOperationManager();
-    ExecuteStatementOperation operation = operationManager
-        .newExecuteStatementOperation(getSession(), statement, confOverlay, runAsync, queryTimeout);
-    OperationHandle opHandle = operation.getHandle();
-    try {
-      operation.run();
-      opHandleSet.add(opHandle);
-      return opHandle;
-    } catch (HiveSQLException e) {
-      // Referring to SQLOperation.java, there is no chance that a HiveSQLException throws and the asyn
-      // background operation submits to thread pool successfully at the same time. So, Cleanup
-      // opHandle directly when got HiveSQLException
-      operationManager.closeOperation(opHandle);
-      throw e;
-    } finally {
-      release(true);
-    }
-  }
-
-  @Override
-  public OperationHandle getTypeInfo()
-      throws HiveSQLException {
-    acquire(true);
-
-    OperationManager operationManager = getOperationManager();
-    GetTypeInfoOperation operation = operationManager.newGetTypeInfoOperation(getSession());
-    OperationHandle opHandle = operation.getHandle();
-    try {
-      operation.run();
-      opHandleSet.add(opHandle);
-      return opHandle;
-    } catch (HiveSQLException e) {
-      operationManager.closeOperation(opHandle);
-      throw e;
-    } finally {
-      release(true);
-    }
-  }
-
-  @Override
-  public OperationHandle getCatalogs()
-      throws HiveSQLException {
-    acquire(true);
-
-    OperationManager operationManager = getOperationManager();
-    GetCatalogsOperation operation = operationManager.newGetCatalogsOperation(getSession());
-    OperationHandle opHandle = operation.getHandle();
-    try {
-      operation.run();
-      opHandleSet.add(opHandle);
-      return opHandle;
-    } catch (HiveSQLException e) {
-      operationManager.closeOperation(opHandle);
-      throw e;
-    } finally {
-      release(true);
-    }
-  }
-
-  @Override
-  public OperationHandle getSchemas(String catalogName, String schemaName)
-      throws HiveSQLException {
-    acquire(true);
-
-    OperationManager operationManager = getOperationManager();
-    GetSchemasOperation operation =
-        operationManager.newGetSchemasOperation(getSession(), catalogName, schemaName);
-    OperationHandle opHandle = operation.getHandle();
-    try {
-      operation.run();
-      opHandleSet.add(opHandle);
-      return opHandle;
-    } catch (HiveSQLException e) {
-      operationManager.closeOperation(opHandle);
-      throw e;
-    } finally {
-      release(true);
-    }
-  }
-
-  @Override
-  public OperationHandle getTables(String catalogName, String schemaName, String tableName,
-      List<String> tableTypes)
-          throws HiveSQLException {
-    acquire(true);
-
-    OperationManager operationManager = getOperationManager();
-    MetadataOperation operation =
-        operationManager.newGetTablesOperation(getSession(), catalogName, schemaName, tableName, tableTypes);
-    OperationHandle opHandle = operation.getHandle();
-    try {
-      operation.run();
-      opHandleSet.add(opHandle);
-      return opHandle;
-    } catch (HiveSQLException e) {
-      operationManager.closeOperation(opHandle);
-      throw e;
-    } finally {
-      release(true);
-    }
-  }
-
-  @Override
-  public OperationHandle getTableTypes()
-      throws HiveSQLException {
-    acquire(true);
-
-    OperationManager operationManager = getOperationManager();
-    GetTableTypesOperation operation = operationManager.newGetTableTypesOperation(getSession());
-    OperationHandle opHandle = operation.getHandle();
-    try {
-      operation.run();
-      opHandleSet.add(opHandle);
-      return opHandle;
-    } catch (HiveSQLException e) {
-      operationManager.closeOperation(opHandle);
-      throw e;
-    } finally {
-      release(true);
-    }
-  }
-
-  @Override
-  public OperationHandle getColumns(String catalogName, String schemaName,
-      String tableName, String columnName)  throws HiveSQLException {
-    acquire(true);
-    String addedJars = Utilities.getResourceFiles(hiveConf, SessionState.ResourceType.JAR);
-    if (StringUtils.isNotBlank(addedJars)) {
-       IMetaStoreClient metastoreClient = getSession().getMetaStoreClient();
-       metastoreClient.setHiveAddedJars(addedJars);
-    }
-    OperationManager operationManager = getOperationManager();
-    GetColumnsOperation operation = operationManager.newGetColumnsOperation(getSession(),
-        catalogName, schemaName, tableName, columnName);
-    OperationHandle opHandle = operation.getHandle();
-    try {
-      operation.run();
-      opHandleSet.add(opHandle);
-      return opHandle;
-    } catch (HiveSQLException e) {
-      operationManager.closeOperation(opHandle);
-      throw e;
-    } finally {
-      release(true);
-    }
-  }
-
-  @Override
-  public OperationHandle getFunctions(String catalogName, String schemaName, String functionName)
-      throws HiveSQLException {
-    acquire(true);
-
-    OperationManager operationManager = getOperationManager();
-    GetFunctionsOperation operation = operationManager
-        .newGetFunctionsOperation(getSession(), catalogName, schemaName, functionName);
-    OperationHandle opHandle = operation.getHandle();
-    try {
-      operation.run();
-      opHandleSet.add(opHandle);
-      return opHandle;
-    } catch (HiveSQLException e) {
-      operationManager.closeOperation(opHandle);
-      throw e;
-    } finally {
-      release(true);
-    }
-  }
-
-  @Override
-  public void close() throws HiveSQLException {
-    try {
-      acquire(true);
-      // Iterate through the opHandles and close their operations
-      for (OperationHandle opHandle : opHandleSet) {
-        operationManager.closeOperation(opHandle);
-      }
-      opHandleSet.clear();
-      // Cleanup session log directory.
-      cleanupSessionLogDir();
-      // Cleanup pipeout file.
-      cleanupPipeoutFile();
-      HiveHistory hiveHist = sessionState.getHiveHistory();
-      if (null != hiveHist) {
-        hiveHist.closeStream();
-      }
-      try {
-        sessionState.close();
-      } finally {
-        sessionState = null;
-      }
-    } catch (IOException ioe) {
-      throw new HiveSQLException("Failure to close", ioe);
-    } finally {
-      if (sessionState != null) {
-        try {
-          sessionState.close();
-        } catch (Throwable t) {
-          LOG.warn("Error closing session", t);
-        }
-        sessionState = null;
-      }
-      release(true);
-    }
-  }
-
-  private void cleanupPipeoutFile() {
-    String lScratchDir = hiveConf.getVar(ConfVars.LOCALSCRATCHDIR);
-    String sessionID = hiveConf.getVar(ConfVars.HIVESESSIONID);
-
-    File[] fileAry = new File(lScratchDir).listFiles(
-            (dir, name) -> name.startsWith(sessionID) && name.endsWith(".pipeout"));
-
-    for (File file : fileAry) {
-      try {
-        FileUtils.forceDelete(file);
-      } catch (Exception e) {
-        LOG.error("Failed to cleanup pipeout file: " + file, e);
-      }
-    }
-  }
-
-  private void cleanupSessionLogDir() {
-    if (isOperationLogEnabled) {
-      try {
-        FileUtils.forceDelete(sessionLogDir);
-      } catch (Exception e) {
-        LOG.error("Failed to cleanup session log dir: " + sessionHandle, e);
-      }
-    }
-  }
-
-  @Override
-  public SessionState getSessionState() {
-    return sessionState;
-  }
-
-  @Override
-  public String getUserName() {
-    return username;
-  }
-
-  @Override
-  public void setUserName(String userName) {
-    this.username = userName;
-  }
-
-  @Override
-  public long getLastAccessTime() {
-    return lastAccessTime;
-  }
-
-  @Override
-  public void closeExpiredOperations() {
-    OperationHandle[] handles = opHandleSet.toArray(new OperationHandle[opHandleSet.size()]);
-    if (handles.length > 0) {
-      List<Operation> operations = operationManager.removeExpiredOperations(handles);
-      if (!operations.isEmpty()) {
-        closeTimedOutOperations(operations);
-      }
-    }
-  }
-
-  @Override
-  public long getNoOperationTime() {
-    return lastIdleTime > 0 ? System.currentTimeMillis() - lastIdleTime : 0;
-  }
-
-  private void closeTimedOutOperations(List<Operation> operations) {
-    acquire(false);
-    try {
-      for (Operation operation : operations) {
-        opHandleSet.remove(operation.getHandle());
-        try {
-          operation.close();
-        } catch (Exception e) {
-          LOG.warn("Exception is thrown closing timed-out operation " + operation.getHandle(), e);
-        }
-      }
-    } finally {
-      release(false);
-    }
-  }
-
-  @Override
-  public void cancelOperation(OperationHandle opHandle) throws HiveSQLException {
-    acquire(true);
-    try {
-      sessionManager.getOperationManager().cancelOperation(opHandle);
-    } finally {
-      release(true);
-    }
-  }
-
-  @Override
-  public void closeOperation(OperationHandle opHandle) throws HiveSQLException {
-    acquire(true);
-    try {
-      operationManager.closeOperation(opHandle);
-      opHandleSet.remove(opHandle);
-    } finally {
-      release(true);
-    }
-  }
-
-  @Override
-  public TableSchema getResultSetMetadata(OperationHandle opHandle) throws HiveSQLException {
-    acquire(true);
-    try {
-      return sessionManager.getOperationManager().getOperationResultSetSchema(opHandle);
-    } finally {
-      release(true);
-    }
-  }
-
-  @Override
-  public RowSet fetchResults(OperationHandle opHandle, FetchOrientation orientation,
-      long maxRows, FetchType fetchType) throws HiveSQLException {
-    acquire(true);
-    try {
-      if (fetchType == FetchType.QUERY_OUTPUT) {
-        return operationManager.getOperationNextRowSet(opHandle, orientation, maxRows);
-      }
-      return operationManager.getOperationLogRowSet(opHandle, orientation, maxRows);
-    } finally {
-      release(true);
-    }
-  }
-
-  protected HiveSession getSession() {
-    return this;
-  }
-
-  @Override
-  public String getIpAddress() {
-    return ipAddress;
-  }
-
-  @Override
-  public void setIpAddress(String ipAddress) {
-    this.ipAddress = ipAddress;
-  }
-
-  @Override
-  public String getDelegationToken(HiveAuthFactory authFactory, String owner, String renewer)
-      throws HiveSQLException {
-    HiveAuthFactory.verifyProxyAccess(getUsername(), owner, getIpAddress(), getHiveConf());
-    return authFactory.getDelegationToken(owner, renewer, getIpAddress());
-  }
-
-  @Override
-  public void cancelDelegationToken(HiveAuthFactory authFactory, String tokenStr)
-      throws HiveSQLException {
-    HiveAuthFactory.verifyProxyAccess(getUsername(), getUserFromToken(authFactory, tokenStr),
-        getIpAddress(), getHiveConf());
-    authFactory.cancelDelegationToken(tokenStr);
-  }
-
-  @Override
-  public void renewDelegationToken(HiveAuthFactory authFactory, String tokenStr)
-      throws HiveSQLException {
-    HiveAuthFactory.verifyProxyAccess(getUsername(), getUserFromToken(authFactory, tokenStr),
-        getIpAddress(), getHiveConf());
-    authFactory.renewDelegationToken(tokenStr);
-  }
-
-  // extract the real user from the given token string
-  private String getUserFromToken(HiveAuthFactory authFactory, String tokenStr) throws HiveSQLException {
-    return authFactory.getUserFromToken(tokenStr);
-  }
-
-  @Override
-  public OperationHandle getPrimaryKeys(String catalog, String schema,
-      String table) throws HiveSQLException {
-    acquire(true);
-
-    OperationManager operationManager = getOperationManager();
-    GetPrimaryKeysOperation operation = operationManager
-        .newGetPrimaryKeysOperation(getSession(), catalog, schema, table);
-    OperationHandle opHandle = operation.getHandle();
-    try {
-      operation.run();
-      opHandleSet.add(opHandle);
-      return opHandle;
-    } catch (HiveSQLException e) {
-      operationManager.closeOperation(opHandle);
-      throw e;
-    } finally {
-      release(true);
-    }
-  }
-
-  @Override
-  public OperationHandle getCrossReference(String primaryCatalog,
-      String primarySchema, String primaryTable, String foreignCatalog,
-      String foreignSchema, String foreignTable) throws HiveSQLException {
-    acquire(true);
-
-    OperationManager operationManager = getOperationManager();
-    GetCrossReferenceOperation operation = operationManager
-      .newGetCrossReferenceOperation(getSession(), primaryCatalog,
-         primarySchema, primaryTable, foreignCatalog,
-         foreignSchema, foreignTable);
-    OperationHandle opHandle = operation.getHandle();
-    try {
-      operation.run();
-      opHandleSet.add(opHandle);
-      return opHandle;
-    } catch (HiveSQLException e) {
-      operationManager.closeOperation(opHandle);
-      throw e;
-    } finally {
-      release(true);
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/SessionManager.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/SessionManager.java
deleted file mode 100644
index 49221b13bb892..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/SessionManager.java
+++ /dev/null
@@ -1,361 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.session;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hive.service.CompositeService;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.SessionHandle;
-import org.apache.hive.service.cli.operation.OperationManager;
-import org.apache.hive.service.rpc.thrift.TProtocolVersion;
-import org.apache.hive.service.server.HiveServer2;
-import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * SessionManager.
- *
- */
-public class SessionManager extends CompositeService {
-
-  private static final Logger LOG = LoggerFactory.getLogger(SessionManager.class);
-  public static final String HIVERCFILE = ".hiverc";
-  private HiveConf hiveConf;
-  private final Map<SessionHandle, HiveSession> handleToSession =
-      new ConcurrentHashMap<SessionHandle, HiveSession>();
-  private final OperationManager operationManager = new OperationManager();
-  private ThreadPoolExecutor backgroundOperationPool;
-  private boolean isOperationLogEnabled;
-  private File operationLogRootDir;
-
-  private long checkInterval;
-  private long sessionTimeout;
-  private boolean checkOperation;
-
-  private volatile boolean shutdown;
-  // The HiveServer2 instance running this service
-  private final HiveServer2 hiveServer2;
-
-  public SessionManager(HiveServer2 hiveServer2) {
-    super(SessionManager.class.getSimpleName());
-    this.hiveServer2 = hiveServer2;
-  }
-
-  @Override
-  public synchronized void init(HiveConf hiveConf) {
-    this.hiveConf = hiveConf;
-    //Create operation log root directory, if operation logging is enabled
-    if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_ENABLED)) {
-      initOperationLogRootDir();
-    }
-    createBackgroundOperationPool();
-    addService(operationManager);
-    super.init(hiveConf);
-  }
-
-  private void createBackgroundOperationPool() {
-    int poolSize = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_ASYNC_EXEC_THREADS);
-    LOG.info("HiveServer2: Background operation thread pool size: " + poolSize);
-    int poolQueueSize = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_ASYNC_EXEC_WAIT_QUEUE_SIZE);
-    LOG.info("HiveServer2: Background operation thread wait queue size: " + poolQueueSize);
-    long keepAliveTime = HiveConf.getTimeVar(
-        hiveConf, ConfVars.HIVE_SERVER2_ASYNC_EXEC_KEEPALIVE_TIME, TimeUnit.SECONDS);
-    LOG.info(
-        "HiveServer2: Background operation thread keepalive time: " + keepAliveTime + " seconds");
-
-    // Create a thread pool with #poolSize threads
-    // Threads terminate when they are idle for more than the keepAliveTime
-    // A bounded blocking queue is used to queue incoming operations, if #operations > poolSize
-    String threadPoolName = "HiveServer2-Background-Pool";
-    backgroundOperationPool = new ThreadPoolExecutor(poolSize, poolSize,
-        keepAliveTime, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(poolQueueSize),
-        new ThreadFactoryWithGarbageCleanup(threadPoolName));
-    backgroundOperationPool.allowCoreThreadTimeOut(true);
-
-    checkInterval = HiveConf.getTimeVar(
-        hiveConf, ConfVars.HIVE_SERVER2_SESSION_CHECK_INTERVAL, TimeUnit.MILLISECONDS);
-    sessionTimeout = HiveConf.getTimeVar(
-        hiveConf, ConfVars.HIVE_SERVER2_IDLE_SESSION_TIMEOUT, TimeUnit.MILLISECONDS);
-    checkOperation = HiveConf.getBoolVar(hiveConf,
-        ConfVars.HIVE_SERVER2_IDLE_SESSION_CHECK_OPERATION);
-  }
-
-  private void initOperationLogRootDir() {
-    operationLogRootDir = new File(
-        hiveConf.getVar(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LOG_LOCATION));
-    isOperationLogEnabled = true;
-
-    if (operationLogRootDir.exists() && !operationLogRootDir.isDirectory()) {
-      LOG.warn("The operation log root directory exists, but it is not a directory: " +
-          operationLogRootDir.getAbsolutePath());
-      isOperationLogEnabled = false;
-    }
-
-    if (!operationLogRootDir.exists()) {
-      if (!operationLogRootDir.mkdirs()) {
-        LOG.warn("Unable to create operation log root directory: " +
-            operationLogRootDir.getAbsolutePath());
-        isOperationLogEnabled = false;
-      }
-    }
-
-    if (isOperationLogEnabled) {
-      LOG.info("Operation log root directory is created: " + operationLogRootDir.getAbsolutePath());
-      try {
-        FileUtils.forceDeleteOnExit(operationLogRootDir);
-      } catch (IOException e) {
-        LOG.warn("Failed to schedule cleanup HS2 operation logging root dir: " +
-            operationLogRootDir.getAbsolutePath(), e);
-      }
-    }
-  }
-
-  @Override
-  public synchronized void start() {
-    super.start();
-    if (checkInterval > 0) {
-      startTimeoutChecker();
-    }
-  }
-
-  private void startTimeoutChecker() {
-    final long interval = Math.max(checkInterval, 3000L);  // minimum 3 seconds
-    Runnable timeoutChecker = new Runnable() {
-      @Override
-      public void run() {
-        for (sleepInterval(interval); !shutdown; sleepInterval(interval)) {
-          long current = System.currentTimeMillis();
-          for (HiveSession session : new ArrayList<HiveSession>(handleToSession.values())) {
-            if (sessionTimeout > 0 && session.getLastAccessTime() + sessionTimeout <= current
-                && (!checkOperation || session.getNoOperationTime() > sessionTimeout)) {
-              SessionHandle handle = session.getSessionHandle();
-              LOG.warn("Session " + handle + " is Timed-out (last access : " +
-                  new Date(session.getLastAccessTime()) + ") and will be closed");
-              try {
-                closeSession(handle);
-              } catch (HiveSQLException e) {
-                LOG.warn("Exception is thrown closing session " + handle, e);
-              }
-            } else {
-              session.closeExpiredOperations();
-            }
-          }
-        }
-      }
-
-      private void sleepInterval(long interval) {
-        try {
-          Thread.sleep(interval);
-        } catch (InterruptedException e) {
-          // ignore
-        }
-      }
-    };
-    backgroundOperationPool.execute(timeoutChecker);
-  }
-
-  @Override
-  public synchronized void stop() {
-    super.stop();
-    shutdown = true;
-    if (backgroundOperationPool != null) {
-      backgroundOperationPool.shutdown();
-      long timeout = hiveConf.getTimeVar(
-          ConfVars.HIVE_SERVER2_ASYNC_EXEC_SHUTDOWN_TIMEOUT, TimeUnit.SECONDS);
-      try {
-        backgroundOperationPool.awaitTermination(timeout, TimeUnit.SECONDS);
-      } catch (InterruptedException e) {
-        LOG.warn("HIVE_SERVER2_ASYNC_EXEC_SHUTDOWN_TIMEOUT = " + timeout +
-            " seconds has been exceeded. RUNNING background operations will be shut down", e);
-      }
-      backgroundOperationPool = null;
-    }
-    cleanupLoggingRootDir();
-  }
-
-  private void cleanupLoggingRootDir() {
-    if (isOperationLogEnabled) {
-      try {
-        FileUtils.forceDelete(operationLogRootDir);
-      } catch (Exception e) {
-        LOG.warn("Failed to cleanup root dir of HS2 logging: " + operationLogRootDir
-            .getAbsolutePath(), e);
-      }
-    }
-  }
-
-  public SessionHandle openSession(TProtocolVersion protocol, String username, String password, String ipAddress,
-      Map<String, String> sessionConf) throws HiveSQLException {
-    return openSession(protocol, username, password, ipAddress, sessionConf, false, null);
-  }
-
-  /**
-   * Opens a new session and creates a session handle.
-   * The username passed to this method is the effective username.
-   * If withImpersonation is true (==doAs true) we wrap all the calls in HiveSession
-   * within a UGI.doAs, where UGI corresponds to the effective user.
-   *
-   * Please see {@code org.apache.hive.service.cli.thrift.ThriftCLIService.getUserName()} for
-   * more details.
-   *
-   * @param protocol
-   * @param username
-   * @param password
-   * @param ipAddress
-   * @param sessionConf
-   * @param withImpersonation
-   * @param delegationToken
-   * @return
-   * @throws HiveSQLException
-   */
-  public SessionHandle openSession(TProtocolVersion protocol, String username, String password, String ipAddress,
-      Map<String, String> sessionConf, boolean withImpersonation, String delegationToken)
-          throws HiveSQLException {
-    HiveSession session;
-    // If doAs is set to true for HiveServer2, we will create a proxy object for the session impl.
-    // Within the proxy object, we wrap the method call in a UserGroupInformation#doAs
-    if (withImpersonation) {
-      HiveSessionImplwithUGI sessionWithUGI = new HiveSessionImplwithUGI(protocol, username, password,
-          hiveConf, ipAddress, delegationToken);
-      session = HiveSessionProxy.getProxy(sessionWithUGI, sessionWithUGI.getSessionUgi());
-      sessionWithUGI.setProxySession(session);
-    } else {
-      session = new HiveSessionImpl(protocol, username, password, hiveConf, ipAddress);
-    }
-    session.setSessionManager(this);
-    session.setOperationManager(operationManager);
-    try {
-      session.open(sessionConf);
-    } catch (Exception e) {
-      try {
-        session.close();
-      } catch (Throwable t) {
-        LOG.warn("Error closing session", t);
-      }
-      session = null;
-      throw new HiveSQLException("Failed to open new session: " + e, e);
-    }
-    if (isOperationLogEnabled) {
-      session.setOperationLogSessionDir(operationLogRootDir);
-    }
-    handleToSession.put(session.getSessionHandle(), session);
-    return session.getSessionHandle();
-  }
-
-  public void closeSession(SessionHandle sessionHandle) throws HiveSQLException {
-    HiveSession session = handleToSession.remove(sessionHandle);
-    if (session == null) {
-      throw new HiveSQLException("Session does not exist!");
-    }
-    session.close();
-  }
-
-  public HiveSession getSession(SessionHandle sessionHandle) throws HiveSQLException {
-    HiveSession session = handleToSession.get(sessionHandle);
-    if (session == null) {
-      throw new HiveSQLException("Invalid SessionHandle: " + sessionHandle);
-    }
-    return session;
-  }
-
-  public OperationManager getOperationManager() {
-    return operationManager;
-  }
-
-  private static ThreadLocal<String> threadLocalIpAddress = new ThreadLocal<String>() {
-    @Override
-    protected synchronized String initialValue() {
-      return null;
-    }
-  };
-
-  public static void setIpAddress(String ipAddress) {
-    threadLocalIpAddress.set(ipAddress);
-  }
-
-  public static void clearIpAddress() {
-    threadLocalIpAddress.remove();
-  }
-
-  public static String getIpAddress() {
-    return threadLocalIpAddress.get();
-  }
-
-  private static ThreadLocal<String> threadLocalUserName = new ThreadLocal<String>(){
-    @Override
-    protected synchronized String initialValue() {
-      return null;
-    }
-  };
-
-  public static void setUserName(String userName) {
-    threadLocalUserName.set(userName);
-  }
-
-  public static void clearUserName() {
-    threadLocalUserName.remove();
-  }
-
-  public static String getUserName() {
-    return threadLocalUserName.get();
-  }
-
-  private static ThreadLocal<String> threadLocalProxyUserName = new ThreadLocal<String>(){
-    @Override
-    protected synchronized String initialValue() {
-      return null;
-    }
-  };
-
-  public static void setProxyUserName(String userName) {
-    LOG.debug("setting proxy user name based on query param to: " + userName);
-    threadLocalProxyUserName.set(userName);
-  }
-
-  public static String getProxyUserName() {
-    return threadLocalProxyUserName.get();
-  }
-
-  public static void clearProxyUserName() {
-    threadLocalProxyUserName.remove();
-  }
-
-  public Future<?> submitBackgroundOperation(Runnable r) {
-    return backgroundOperationPool.submit(r);
-  }
-
-  public int getOpenSessionCount() {
-    return handleToSession.size();
-  }
-}
-
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
deleted file mode 100644
index fc19c65daaf54..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.thrift;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.hive.common.auth.HiveAuthUtils;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hive.service.auth.HiveAuthFactory;
-import org.apache.hive.service.cli.CLIService;
-import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup;
-import org.apache.thrift.TProcessorFactory;
-import org.apache.thrift.protocol.TBinaryProtocol;
-import org.apache.thrift.server.TThreadPoolServer;
-import org.apache.thrift.transport.TServerSocket;
-import org.apache.thrift.transport.TTransportFactory;
-
-
-public class ThriftBinaryCLIService extends ThriftCLIService {
-
-  public ThriftBinaryCLIService(CLIService cliService) {
-    super(cliService, ThriftBinaryCLIService.class.getSimpleName());
-  }
-
-  @Override
-  public void run() {
-    try {
-      // Server thread pool
-      String threadPoolName = "HiveServer2-Handler-Pool";
-      ExecutorService executorService = new ThreadPoolExecutor(minWorkerThreads, maxWorkerThreads,
-          workerKeepAliveTime, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
-          new ThreadFactoryWithGarbageCleanup(threadPoolName));
-
-      // Thrift configs
-      hiveAuthFactory = new HiveAuthFactory(hiveConf);
-      TTransportFactory transportFactory = hiveAuthFactory.getAuthTransFactory();
-      TProcessorFactory processorFactory = hiveAuthFactory.getAuthProcFactory(this);
-      TServerSocket serverSocket = null;
-      List<String> sslVersionBlacklist = new ArrayList<String>();
-      for (String sslVersion : hiveConf.getVar(ConfVars.HIVE_SSL_PROTOCOL_BLACKLIST).split(",")) {
-        sslVersionBlacklist.add(sslVersion);
-      }
-      if (!hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_USE_SSL)) {
-        serverSocket = HiveAuthUtils.getServerSocket(hiveHost, portNum);
-      } else {
-        String keyStorePath = hiveConf.getVar(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH).trim();
-        if (keyStorePath.isEmpty()) {
-          throw new IllegalArgumentException(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname
-              + " Not configured for SSL connection");
-        }
-        String keyStorePassword = ShimLoader.getHadoopShims().getPassword(hiveConf,
-            HiveConf.ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname);
-        serverSocket = HiveAuthUtils.getServerSSLSocket(hiveHost, portNum, keyStorePath,
-            keyStorePassword, sslVersionBlacklist);
-      }
-
-      // Server args
-      int maxMessageSize = hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE);
-      int requestTimeout = (int) hiveConf.getTimeVar(
-          HiveConf.ConfVars.HIVE_SERVER2_THRIFT_LOGIN_TIMEOUT, TimeUnit.SECONDS);
-      int beBackoffSlotLength = (int) hiveConf.getTimeVar(
-          HiveConf.ConfVars.HIVE_SERVER2_THRIFT_LOGIN_BEBACKOFF_SLOT_LENGTH, TimeUnit.MILLISECONDS);
-      TThreadPoolServer.Args sargs = new TThreadPoolServer.Args(serverSocket)
-          .processorFactory(processorFactory).transportFactory(transportFactory)
-          .protocolFactory(new TBinaryProtocol.Factory())
-          .inputProtocolFactory(new TBinaryProtocol.Factory(true, true, maxMessageSize, maxMessageSize))
-          .requestTimeout(requestTimeout).requestTimeoutUnit(TimeUnit.SECONDS)
-          .beBackoffSlotLength(beBackoffSlotLength).beBackoffSlotLengthUnit(TimeUnit.MILLISECONDS)
-          .executorService(executorService);
-
-      // TCP Server
-      server = new TThreadPoolServer(sargs);
-      server.setServerEventHandler(serverEventHandler);
-      String msg = "Starting " + ThriftBinaryCLIService.class.getSimpleName() + " on port "
-          + portNum + " with " + minWorkerThreads + "..." + maxWorkerThreads + " worker threads";
-      LOG.info(msg);
-      server.serve();
-    } catch (Throwable t) {
-      LOG.error(
-          "Error starting HiveServer2: could not start "
-              + ThriftBinaryCLIService.class.getSimpleName(), t);
-      System.exit(-1);
-    }
-  }
-
-}
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
deleted file mode 100644
index d41c3b493bb47..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
+++ /dev/null
@@ -1,726 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.thrift;
-
-import javax.security.auth.login.LoginException;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hive.service.AbstractService;
-import org.apache.hive.service.ServiceException;
-import org.apache.hive.service.ServiceUtils;
-import org.apache.hive.service.auth.HiveAuthFactory;
-import org.apache.hive.service.auth.TSetIpAddressProcessor;
-import org.apache.hive.service.cli.*;
-import org.apache.hive.service.cli.session.SessionManager;
-import org.apache.hive.service.rpc.thrift.*;
-import org.apache.hive.service.server.HiveServer2;
-import org.apache.thrift.TException;
-import org.apache.thrift.protocol.TProtocol;
-import org.apache.thrift.server.ServerContext;
-import org.apache.thrift.server.TServer;
-import org.apache.thrift.server.TServerEventHandler;
-import org.apache.thrift.transport.TTransport;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * ThriftCLIService.
- *
- */
-public abstract class ThriftCLIService extends AbstractService implements TCLIService.Iface, Runnable {
-
-  public static final Logger LOG = LoggerFactory.getLogger(ThriftCLIService.class.getName());
-
-  protected CLIService cliService;
-  private static final TStatus OK_STATUS = new TStatus(TStatusCode.SUCCESS_STATUS);
-  protected static HiveAuthFactory hiveAuthFactory;
-
-  protected int portNum;
-  protected InetAddress serverIPAddress;
-  protected String hiveHost;
-  protected TServer server;
-  protected org.eclipse.jetty.server.Server httpServer;
-
-  private boolean isStarted = false;
-  protected boolean isEmbedded = false;
-
-  protected HiveConf hiveConf;
-
-  protected int minWorkerThreads;
-  protected int maxWorkerThreads;
-  protected long workerKeepAliveTime;
-
-  protected TServerEventHandler serverEventHandler;
-  protected ThreadLocal<ServerContext> currentServerContext;
-
-  static class ThriftCLIServerContext implements ServerContext {
-    private SessionHandle sessionHandle = null;
-
-    public void setSessionHandle(SessionHandle sessionHandle) {
-      this.sessionHandle = sessionHandle;
-    }
-
-    public SessionHandle getSessionHandle() {
-      return sessionHandle;
-    }
-  }
-
-  public ThriftCLIService(CLIService service, String serviceName) {
-    super(serviceName);
-    this.cliService = service;
-    currentServerContext = new ThreadLocal<ServerContext>();
-    serverEventHandler = new TServerEventHandler() {
-      @Override
-      public ServerContext createContext(
-          TProtocol input, TProtocol output) {
-        return new ThriftCLIServerContext();
-      }
-
-      @Override
-      public void deleteContext(ServerContext serverContext,
-          TProtocol input, TProtocol output) {
-        ThriftCLIServerContext context = (ThriftCLIServerContext)serverContext;
-        SessionHandle sessionHandle = context.getSessionHandle();
-        if (sessionHandle != null) {
-          LOG.info("Session disconnected without closing properly, close it now");
-          try {
-            cliService.closeSession(sessionHandle);
-          } catch (HiveSQLException e) {
-            LOG.warn("Failed to close session: " + e, e);
-          }
-        }
-      }
-
-      @Override
-      public void preServe() {
-      }
-
-      @Override
-      public void processContext(ServerContext serverContext,
-          TTransport input, TTransport output) {
-        currentServerContext.set(serverContext);
-      }
-    };
-  }
-
-  @Override
-  public synchronized void init(HiveConf hiveConf) {
-    this.hiveConf = hiveConf;
-    // Initialize common server configs needed in both binary & http modes
-    String portString;
-    hiveHost = System.getenv("HIVE_SERVER2_THRIFT_BIND_HOST");
-    if (hiveHost == null) {
-      hiveHost = hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST);
-    }
-    try {
-      if (hiveHost != null && !hiveHost.isEmpty()) {
-        serverIPAddress = InetAddress.getByName(hiveHost);
-      } else {
-        serverIPAddress = InetAddress.getLocalHost();
-      }
-    } catch (UnknownHostException e) {
-      throw new ServiceException(e);
-    }
-    // HTTP mode
-    if (HiveServer2.isHTTPTransportMode(hiveConf)) {
-      workerKeepAliveTime =
-          hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_WORKER_KEEPALIVE_TIME,
-              TimeUnit.SECONDS);
-      portString = System.getenv("HIVE_SERVER2_THRIFT_HTTP_PORT");
-      if (portString != null) {
-        portNum = Integer.valueOf(portString);
-      } else {
-        portNum = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT);
-      }
-    }
-    // Binary mode
-    else {
-      workerKeepAliveTime =
-          hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_THRIFT_WORKER_KEEPALIVE_TIME, TimeUnit.SECONDS);
-      portString = System.getenv("HIVE_SERVER2_THRIFT_PORT");
-      if (portString != null) {
-        portNum = Integer.valueOf(portString);
-      } else {
-        portNum = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT);
-      }
-    }
-    minWorkerThreads = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_MIN_WORKER_THREADS);
-    maxWorkerThreads = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_MAX_WORKER_THREADS);
-    super.init(hiveConf);
-  }
-
-  @Override
-  public synchronized void start() {
-    super.start();
-    if (!isStarted && !isEmbedded) {
-      new Thread(this).start();
-      isStarted = true;
-    }
-  }
-
-  @Override
-  public synchronized void stop() {
-    if (isStarted && !isEmbedded) {
-      if(server != null) {
-        server.stop();
-        LOG.info("Thrift server has stopped");
-      }
-      if((httpServer != null) && httpServer.isStarted()) {
-        try {
-          httpServer.stop();
-          LOG.info("Http server has stopped");
-        } catch (Exception e) {
-          LOG.error("Error stopping Http server: ", e);
-        }
-      }
-      isStarted = false;
-    }
-    super.stop();
-  }
-
-  public int getPortNumber() {
-    return portNum;
-  }
-
-  public InetAddress getServerIPAddress() {
-    return serverIPAddress;
-  }
-
-  @Override
-  public TGetDelegationTokenResp GetDelegationToken(TGetDelegationTokenReq req)
-      throws TException {
-    TGetDelegationTokenResp resp = new TGetDelegationTokenResp();
-    resp.setStatus(notSupportTokenErrorStatus());
-    return resp;
-  }
-
-  @Override
-  public TCancelDelegationTokenResp CancelDelegationToken(TCancelDelegationTokenReq req)
-      throws TException {
-    TCancelDelegationTokenResp resp = new TCancelDelegationTokenResp();
-    resp.setStatus(notSupportTokenErrorStatus());
-    return resp;
-  }
-
-  @Override
-  public TRenewDelegationTokenResp RenewDelegationToken(TRenewDelegationTokenReq req)
-      throws TException {
-    TRenewDelegationTokenResp resp = new TRenewDelegationTokenResp();
-    resp.setStatus(notSupportTokenErrorStatus());
-    return resp;
-  }
-
-  private TStatus notSupportTokenErrorStatus() {
-    TStatus errorStatus = new TStatus(TStatusCode.ERROR_STATUS);
-    errorStatus.setErrorMessage("Delegation token is not supported");
-    return errorStatus;
-  }
-
-  @Override
-  public TOpenSessionResp OpenSession(TOpenSessionReq req) throws TException {
-    LOG.info("Client protocol version: " + req.getClient_protocol());
-    TOpenSessionResp resp = new TOpenSessionResp();
-    try {
-      SessionHandle sessionHandle = getSessionHandle(req, resp);
-      resp.setSessionHandle(sessionHandle.toTSessionHandle());
-      // TODO: set real configuration map
-      resp.setConfiguration(new HashMap<String, String>());
-      resp.setStatus(OK_STATUS);
-      ThriftCLIServerContext context =
-        (ThriftCLIServerContext)currentServerContext.get();
-      if (context != null) {
-        context.setSessionHandle(sessionHandle);
-      }
-    } catch (Exception e) {
-      LOG.warn("Error opening session: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  private String getIpAddress() {
-    String clientIpAddress;
-    // Http transport mode.
-    // We set the thread local ip address, in ThriftHttpServlet.
-    if (cliService.getHiveConf().getVar(
-        ConfVars.HIVE_SERVER2_TRANSPORT_MODE).equalsIgnoreCase("http")) {
-      clientIpAddress = SessionManager.getIpAddress();
-    }
-    else {
-      // Kerberos
-      if (isKerberosAuthMode()) {
-        clientIpAddress = hiveAuthFactory.getIpAddress();
-      }
-      // Except kerberos, NOSASL
-      else {
-        clientIpAddress = TSetIpAddressProcessor.getUserIpAddress();
-      }
-    }
-    LOG.debug("Client's IP Address: " + clientIpAddress);
-    return clientIpAddress;
-  }
-
-  /**
-   * Returns the effective username.
-   * 1. If hive.server2.allow.user.substitution = false: the username of the connecting user
-   * 2. If hive.server2.allow.user.substitution = true: the username of the end user,
-   * that the connecting user is trying to proxy for.
-   * This includes a check whether the connecting user is allowed to proxy for the end user.
-   * @param req
-   * @return
-   * @throws HiveSQLException
-   */
-  private String getUserName(TOpenSessionReq req) throws HiveSQLException {
-    String userName = null;
-    // Kerberos
-    if (isKerberosAuthMode()) {
-      userName = hiveAuthFactory.getRemoteUser();
-    }
-    // Except kerberos, NOSASL
-    if (userName == null) {
-      userName = TSetIpAddressProcessor.getUserName();
-    }
-    // Http transport mode.
-    // We set the thread local username, in ThriftHttpServlet.
-    if (cliService.getHiveConf().getVar(
-        ConfVars.HIVE_SERVER2_TRANSPORT_MODE).equalsIgnoreCase("http")) {
-      userName = SessionManager.getUserName();
-    }
-    if (userName == null) {
-      userName = req.getUsername();
-    }
-
-    userName = getShortName(userName);
-    String effectiveClientUser = getProxyUser(userName, req.getConfiguration(), getIpAddress());
-    LOG.debug("Client's username: " + effectiveClientUser);
-    return effectiveClientUser;
-  }
-
-  private String getShortName(String userName) {
-    String ret = null;
-    if (userName != null) {
-      int indexOfDomainMatch = ServiceUtils.indexOfDomainMatch(userName);
-      ret = (indexOfDomainMatch <= 0) ? userName :
-          userName.substring(0, indexOfDomainMatch);
-    }
-
-    return ret;
-  }
-
-  /**
-   * Create a session handle
-   * @param req
-   * @param res
-   * @return
-   * @throws HiveSQLException
-   * @throws LoginException
-   * @throws IOException
-   */
-  SessionHandle getSessionHandle(TOpenSessionReq req, TOpenSessionResp res)
-      throws HiveSQLException, LoginException, IOException {
-    String userName = getUserName(req);
-    String ipAddress = getIpAddress();
-    TProtocolVersion protocol = getMinVersion(CLIService.SERVER_VERSION,
-        req.getClient_protocol());
-    res.setServerProtocolVersion(protocol);
-    SessionHandle sessionHandle;
-    if (cliService.getHiveConf().getBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS) &&
-        (userName != null)) {
-      String delegationTokenStr = getDelegationToken(userName);
-      sessionHandle = cliService.openSessionWithImpersonation(protocol, userName,
-          req.getPassword(), ipAddress, req.getConfiguration(), delegationTokenStr);
-    } else {
-      sessionHandle = cliService.openSession(protocol, userName, req.getPassword(),
-          ipAddress, req.getConfiguration());
-    }
-    return sessionHandle;
-  }
-
-
-  private String getDelegationToken(String userName)
-      throws HiveSQLException, LoginException, IOException {
-    if (userName == null || !cliService.getHiveConf().getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION)
-        .equalsIgnoreCase(HiveAuthFactory.AuthTypes.KERBEROS.toString())) {
-      return null;
-    }
-    try {
-      return cliService.getDelegationTokenFromMetaStore(userName);
-    } catch (UnsupportedOperationException e) {
-      // The delegation token is not applicable in the given deployment mode
-    }
-    return null;
-  }
-
-  private TProtocolVersion getMinVersion(TProtocolVersion... versions) {
-    TProtocolVersion[] values = TProtocolVersion.values();
-    int current = values[values.length - 1].getValue();
-    for (TProtocolVersion version : versions) {
-      if (current > version.getValue()) {
-        current = version.getValue();
-      }
-    }
-    for (TProtocolVersion version : values) {
-      if (version.getValue() == current) {
-        return version;
-      }
-    }
-    throw new IllegalArgumentException("never");
-  }
-
-  @Override
-  public TCloseSessionResp CloseSession(TCloseSessionReq req) throws TException {
-    TCloseSessionResp resp = new TCloseSessionResp();
-    try {
-      SessionHandle sessionHandle = new SessionHandle(req.getSessionHandle());
-      cliService.closeSession(sessionHandle);
-      resp.setStatus(OK_STATUS);
-      ThriftCLIServerContext context =
-        (ThriftCLIServerContext)currentServerContext.get();
-      if (context != null) {
-        context.setSessionHandle(null);
-      }
-    } catch (Exception e) {
-      LOG.warn("Error closing session: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TGetInfoResp GetInfo(TGetInfoReq req) throws TException {
-    TGetInfoResp resp = new TGetInfoResp();
-    try {
-      GetInfoValue getInfoValue =
-          cliService.getInfo(new SessionHandle(req.getSessionHandle()),
-              GetInfoType.getGetInfoType(req.getInfoType()));
-      resp.setInfoValue(getInfoValue.toTGetInfoValue());
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error getting info: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TExecuteStatementResp ExecuteStatement(TExecuteStatementReq req) throws TException {
-    TExecuteStatementResp resp = new TExecuteStatementResp();
-    try {
-      SessionHandle sessionHandle = new SessionHandle(req.getSessionHandle());
-      String statement = req.getStatement();
-      Map<String, String> confOverlay = req.getConfOverlay();
-      Boolean runAsync = req.isRunAsync();
-      long queryTimeout = req.getQueryTimeout();
-      OperationHandle operationHandle = runAsync ?
-          cliService.executeStatementAsync(sessionHandle, statement, confOverlay, queryTimeout)
-          : cliService.executeStatement(sessionHandle, statement, confOverlay, queryTimeout);
-          resp.setOperationHandle(operationHandle.toTOperationHandle());
-          resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error executing statement: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TGetTypeInfoResp GetTypeInfo(TGetTypeInfoReq req) throws TException {
-    TGetTypeInfoResp resp = new TGetTypeInfoResp();
-    try {
-      OperationHandle operationHandle = cliService.getTypeInfo(new SessionHandle(req.getSessionHandle()));
-      resp.setOperationHandle(operationHandle.toTOperationHandle());
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error getting type info: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TGetCatalogsResp GetCatalogs(TGetCatalogsReq req) throws TException {
-    TGetCatalogsResp resp = new TGetCatalogsResp();
-    try {
-      OperationHandle opHandle = cliService.getCatalogs(new SessionHandle(req.getSessionHandle()));
-      resp.setOperationHandle(opHandle.toTOperationHandle());
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error getting catalogs: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TGetSchemasResp GetSchemas(TGetSchemasReq req) throws TException {
-    TGetSchemasResp resp = new TGetSchemasResp();
-    try {
-      OperationHandle opHandle = cliService.getSchemas(
-          new SessionHandle(req.getSessionHandle()), req.getCatalogName(), req.getSchemaName());
-      resp.setOperationHandle(opHandle.toTOperationHandle());
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error getting schemas: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TGetTablesResp GetTables(TGetTablesReq req) throws TException {
-    TGetTablesResp resp = new TGetTablesResp();
-    try {
-      OperationHandle opHandle = cliService
-          .getTables(new SessionHandle(req.getSessionHandle()), req.getCatalogName(),
-              req.getSchemaName(), req.getTableName(), req.getTableTypes());
-      resp.setOperationHandle(opHandle.toTOperationHandle());
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error getting tables: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TGetTableTypesResp GetTableTypes(TGetTableTypesReq req) throws TException {
-    TGetTableTypesResp resp = new TGetTableTypesResp();
-    try {
-      OperationHandle opHandle = cliService.getTableTypes(new SessionHandle(req.getSessionHandle()));
-      resp.setOperationHandle(opHandle.toTOperationHandle());
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error getting table types: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TGetColumnsResp GetColumns(TGetColumnsReq req) throws TException {
-    TGetColumnsResp resp = new TGetColumnsResp();
-    try {
-      OperationHandle opHandle = cliService.getColumns(
-          new SessionHandle(req.getSessionHandle()),
-          req.getCatalogName(),
-          req.getSchemaName(),
-          req.getTableName(),
-          req.getColumnName());
-      resp.setOperationHandle(opHandle.toTOperationHandle());
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error getting columns: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TGetFunctionsResp GetFunctions(TGetFunctionsReq req) throws TException {
-    TGetFunctionsResp resp = new TGetFunctionsResp();
-    try {
-      OperationHandle opHandle = cliService.getFunctions(
-          new SessionHandle(req.getSessionHandle()), req.getCatalogName(),
-          req.getSchemaName(), req.getFunctionName());
-      resp.setOperationHandle(opHandle.toTOperationHandle());
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error getting functions: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TGetOperationStatusResp GetOperationStatus(TGetOperationStatusReq req) throws TException {
-    TGetOperationStatusResp resp = new TGetOperationStatusResp();
-    try {
-      OperationStatus operationStatus = cliService.getOperationStatus(
-          new OperationHandle(req.getOperationHandle()));
-      resp.setOperationState(operationStatus.getState().toTOperationState());
-      HiveSQLException opException = operationStatus.getOperationException();
-      if (opException != null) {
-        resp.setSqlState(opException.getSQLState());
-        resp.setErrorCode(opException.getErrorCode());
-        resp.setErrorMessage(opException.getMessage());
-      }
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error getting operation status: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TCancelOperationResp CancelOperation(TCancelOperationReq req) throws TException {
-    TCancelOperationResp resp = new TCancelOperationResp();
-    try {
-      cliService.cancelOperation(new OperationHandle(req.getOperationHandle()));
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error cancelling operation: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TCloseOperationResp CloseOperation(TCloseOperationReq req) throws TException {
-    TCloseOperationResp resp = new TCloseOperationResp();
-    try {
-      cliService.closeOperation(new OperationHandle(req.getOperationHandle()));
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error closing operation: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TGetResultSetMetadataResp GetResultSetMetadata(TGetResultSetMetadataReq req)
-      throws TException {
-    TGetResultSetMetadataResp resp = new TGetResultSetMetadataResp();
-    try {
-      TableSchema schema = cliService.getResultSetMetadata(new OperationHandle(req.getOperationHandle()));
-      resp.setSchema(schema.toTTableSchema());
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error getting result set metadata: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TFetchResultsResp FetchResults(TFetchResultsReq req) throws TException {
-    TFetchResultsResp resp = new TFetchResultsResp();
-    try {
-      RowSet rowSet = cliService.fetchResults(
-          new OperationHandle(req.getOperationHandle()),
-          FetchOrientation.getFetchOrientation(req.getOrientation()),
-          req.getMaxRows(),
-          FetchType.getFetchType(req.getFetchType()));
-      resp.setResults(rowSet.toTRowSet());
-      resp.setHasMoreRows(false);
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error fetching results: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TGetPrimaryKeysResp GetPrimaryKeys(TGetPrimaryKeysReq req)
-      throws TException {
-    TGetPrimaryKeysResp resp = new TGetPrimaryKeysResp();
-    try {
-      OperationHandle opHandle = cliService.getPrimaryKeys(
-      new SessionHandle(req.getSessionHandle()), req.getCatalogName(),
-      req.getSchemaName(), req.getTableName());
-      resp.setOperationHandle(opHandle.toTOperationHandle());
-      resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-     LOG.warn("Error getting functions: ", e);
-     resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public TGetCrossReferenceResp GetCrossReference(TGetCrossReferenceReq req)
-      throws TException {
-    TGetCrossReferenceResp resp = new TGetCrossReferenceResp();
-    try {
-      OperationHandle opHandle = cliService.getCrossReference(
-        new SessionHandle(req.getSessionHandle()), req.getParentCatalogName(),
-          req.getParentSchemaName(), req.getParentTableName(),
-          req.getForeignCatalogName(), req.getForeignSchemaName(), req.getForeignTableName());
-          resp.setOperationHandle(opHandle.toTOperationHandle());
-          resp.setStatus(OK_STATUS);
-    } catch (Exception e) {
-      LOG.warn("Error getting functions: ", e);
-      resp.setStatus(HiveSQLException.toTStatus(e));
-    }
-    return resp;
-  }
-
-  @Override
-  public abstract void run();
-
-  /**
-   * If the proxy user name is provided then check privileges to substitute the user.
-   * @param realUser
-   * @param sessionConf
-   * @param ipAddress
-   * @return
-   * @throws HiveSQLException
-   */
-  private String getProxyUser(String realUser, Map<String, String> sessionConf,
-      String ipAddress) throws HiveSQLException {
-    String proxyUser = null;
-    // Http transport mode.
-    // We set the thread local proxy username, in ThriftHttpServlet.
-    if (cliService.getHiveConf().getVar(
-        ConfVars.HIVE_SERVER2_TRANSPORT_MODE).equalsIgnoreCase("http")) {
-      proxyUser = SessionManager.getProxyUserName();
-      LOG.debug("Proxy user from query string: " + proxyUser);
-    }
-
-    if (proxyUser == null && sessionConf != null && sessionConf.containsKey(HiveAuthFactory.HS2_PROXY_USER)) {
-      String proxyUserFromThriftBody = sessionConf.get(HiveAuthFactory.HS2_PROXY_USER);
-      LOG.debug("Proxy user from thrift body: " + proxyUserFromThriftBody);
-      proxyUser = proxyUserFromThriftBody;
-    }
-
-    if (proxyUser == null) {
-      return realUser;
-    }
-
-    // check whether substitution is allowed
-    if (!hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ALLOW_USER_SUBSTITUTION)) {
-      throw new HiveSQLException("Proxy user substitution is not allowed");
-    }
-
-    // If there's no authentication, then directly substitute the user
-    if (HiveAuthFactory.AuthTypes.NONE.toString()
-        .equalsIgnoreCase(hiveConf.getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION))) {
-      return proxyUser;
-    }
-
-    // Verify proxy user privilege of the realUser for the proxyUser
-    HiveAuthFactory.verifyProxyAccess(realUser, proxyUser, ipAddress, hiveConf);
-    LOG.debug("Verified proxy user: " + proxyUser);
-    return proxyUser;
-  }
-
-  private boolean isKerberosAuthMode() {
-    return cliService.getHiveConf().getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION)
-        .equalsIgnoreCase(HiveAuthFactory.AuthTypes.KERBEROS.toString());
-  }
-}
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIServiceClient.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIServiceClient.java
deleted file mode 100644
index b13ddf72f77e7..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIServiceClient.java
+++ /dev/null
@@ -1,493 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.thrift;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hive.service.auth.HiveAuthFactory;
-import org.apache.hive.service.cli.*;
-import org.apache.hive.service.rpc.thrift.*;
-import org.apache.thrift.TException;
-
-/**
- * ThriftCLIServiceClient.
- *
- */
-public class ThriftCLIServiceClient extends CLIServiceClient {
-  private final TCLIService.Iface cliService;
-
-  public ThriftCLIServiceClient(TCLIService.Iface cliService) {
-    this.cliService = cliService;
-  }
-
-  public void checkStatus(TStatus status) throws HiveSQLException {
-    if (TStatusCode.ERROR_STATUS.equals(status.getStatusCode())) {
-      throw new HiveSQLException(status);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#openSession(java.lang.String, java.lang.String, java.util.Map)
-   */
-  @Override
-  public SessionHandle openSession(String username, String password,
-      Map<String, String> configuration)
-          throws HiveSQLException {
-    try {
-      TOpenSessionReq req = new TOpenSessionReq();
-      req.setUsername(username);
-      req.setPassword(password);
-      req.setConfiguration(configuration);
-      TOpenSessionResp resp = cliService.OpenSession(req);
-      checkStatus(resp.getStatus());
-      return new SessionHandle(resp.getSessionHandle(), resp.getServerProtocolVersion());
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#closeSession(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public SessionHandle openSessionWithImpersonation(String username, String password,
-      Map<String, String> configuration, String delegationToken) throws HiveSQLException {
-    throw new HiveSQLException("open with impersonation operation is not supported in the client");
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#closeSession(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public void closeSession(SessionHandle sessionHandle) throws HiveSQLException {
-    try {
-      TCloseSessionReq req = new TCloseSessionReq(sessionHandle.toTSessionHandle());
-      TCloseSessionResp resp = cliService.CloseSession(req);
-      checkStatus(resp.getStatus());
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getInfo(org.apache.hive.service.cli.SessionHandle, java.util.List)
-   */
-  @Override
-  public GetInfoValue getInfo(SessionHandle sessionHandle, GetInfoType infoType)
-      throws HiveSQLException {
-    try {
-      // FIXME extract the right info type
-      TGetInfoReq req = new TGetInfoReq(sessionHandle.toTSessionHandle(), infoType.toTGetInfoType());
-      TGetInfoResp resp = cliService.GetInfo(req);
-      checkStatus(resp.getStatus());
-      return new GetInfoValue(resp.getInfoValue());
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#executeStatement(org.apache.hive.service.cli.SessionHandle, java.lang.String, java.util.Map)
-   */
-  @Override
-  public OperationHandle executeStatement(SessionHandle sessionHandle, String statement,
-      Map<String, String> confOverlay) throws HiveSQLException {
-    return executeStatementInternal(sessionHandle, statement, confOverlay, false, 0);
-  }
-
-  @Override
-  public OperationHandle executeStatement(SessionHandle sessionHandle, String statement,
-      Map<String, String> confOverlay, long queryTimeout) throws HiveSQLException {
-    return executeStatementInternal(sessionHandle, statement, confOverlay, false, queryTimeout);
-  }
-
-  @Override
-  public OperationHandle executeStatementAsync(SessionHandle sessionHandle, String statement,
-      Map<String, String> confOverlay) throws HiveSQLException {
-    return executeStatementInternal(sessionHandle, statement, confOverlay, true, 0);
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#executeStatementAsync(org.apache.hive.service.cli.SessionHandle, java.lang.String, java.util.Map)
-   */
-  @Override
-  public OperationHandle executeStatementAsync(SessionHandle sessionHandle, String statement,
-      Map<String, String> confOverlay, long queryTimeout) throws HiveSQLException {
-    return executeStatementInternal(sessionHandle, statement, confOverlay, true, queryTimeout);
-  }
-
-  private OperationHandle executeStatementInternal(SessionHandle sessionHandle, String statement,
-      Map<String, String> confOverlay, boolean isAsync, long queryTimeout) throws HiveSQLException {
-    try {
-      TExecuteStatementReq req =
-          new TExecuteStatementReq(sessionHandle.toTSessionHandle(), statement);
-      req.setConfOverlay(confOverlay);
-      req.setRunAsync(isAsync);
-      req.setQueryTimeout(queryTimeout);
-      TExecuteStatementResp resp = cliService.ExecuteStatement(req);
-      checkStatus(resp.getStatus());
-      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
-      return new OperationHandle(resp.getOperationHandle(), protocol);
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getTypeInfo(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public OperationHandle getTypeInfo(SessionHandle sessionHandle) throws HiveSQLException {
-    try {
-      TGetTypeInfoReq req = new TGetTypeInfoReq(sessionHandle.toTSessionHandle());
-      TGetTypeInfoResp resp = cliService.GetTypeInfo(req);
-      checkStatus(resp.getStatus());
-      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
-      return new OperationHandle(resp.getOperationHandle(), protocol);
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getCatalogs(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public OperationHandle getCatalogs(SessionHandle sessionHandle) throws HiveSQLException {
-    try {
-      TGetCatalogsReq req = new TGetCatalogsReq(sessionHandle.toTSessionHandle());
-      TGetCatalogsResp resp = cliService.GetCatalogs(req);
-      checkStatus(resp.getStatus());
-      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
-      return new OperationHandle(resp.getOperationHandle(), protocol);
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getSchemas(org.apache.hive.service.cli.SessionHandle, java.lang.String, java.lang.String)
-   */
-  @Override
-  public OperationHandle getSchemas(SessionHandle sessionHandle, String catalogName,
-      String schemaName)
-          throws HiveSQLException {
-    try {
-      TGetSchemasReq req = new TGetSchemasReq(sessionHandle.toTSessionHandle());
-      req.setCatalogName(catalogName);
-      req.setSchemaName(schemaName);
-      TGetSchemasResp resp = cliService.GetSchemas(req);
-      checkStatus(resp.getStatus());
-      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
-      return new OperationHandle(resp.getOperationHandle(), protocol);
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getTables(org.apache.hive.service.cli.SessionHandle, java.lang.String, java.lang.String, java.lang.String, java.util.List)
-   */
-  @Override
-  public OperationHandle getTables(SessionHandle sessionHandle, String catalogName,
-      String schemaName, String tableName, List<String> tableTypes)
-          throws HiveSQLException {
-    try {
-      TGetTablesReq req = new TGetTablesReq(sessionHandle.toTSessionHandle());
-      req.setTableName(tableName);
-      req.setTableTypes(tableTypes);
-      req.setSchemaName(schemaName);
-      TGetTablesResp resp = cliService.GetTables(req);
-      checkStatus(resp.getStatus());
-      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
-      return new OperationHandle(resp.getOperationHandle(), protocol);
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getTableTypes(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public OperationHandle getTableTypes(SessionHandle sessionHandle) throws HiveSQLException {
-    try {
-      TGetTableTypesReq req = new TGetTableTypesReq(sessionHandle.toTSessionHandle());
-      TGetTableTypesResp resp = cliService.GetTableTypes(req);
-      checkStatus(resp.getStatus());
-      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
-      return new OperationHandle(resp.getOperationHandle(), protocol);
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getColumns(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public OperationHandle getColumns(SessionHandle sessionHandle,
-      String catalogName, String schemaName, String tableName, String columnName)
-          throws HiveSQLException {
-    try {
-      TGetColumnsReq req = new TGetColumnsReq();
-      req.setSessionHandle(sessionHandle.toTSessionHandle());
-      req.setCatalogName(catalogName);
-      req.setSchemaName(schemaName);
-      req.setTableName(tableName);
-      req.setColumnName(columnName);
-      TGetColumnsResp resp = cliService.GetColumns(req);
-      checkStatus(resp.getStatus());
-      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
-      return new OperationHandle(resp.getOperationHandle(), protocol);
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getFunctions(org.apache.hive.service.cli.SessionHandle)
-   */
-  @Override
-  public OperationHandle getFunctions(SessionHandle sessionHandle,
-      String catalogName, String schemaName, String functionName) throws HiveSQLException {
-    try {
-      TGetFunctionsReq req = new TGetFunctionsReq(sessionHandle.toTSessionHandle(), functionName);
-      req.setCatalogName(catalogName);
-      req.setSchemaName(schemaName);
-      TGetFunctionsResp resp = cliService.GetFunctions(req);
-      checkStatus(resp.getStatus());
-      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
-      return new OperationHandle(resp.getOperationHandle(), protocol);
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getOperationStatus(org.apache.hive.service.cli.OperationHandle)
-   */
-  @Override
-  public OperationStatus getOperationStatus(OperationHandle opHandle) throws HiveSQLException {
-    try {
-      TGetOperationStatusReq req = new TGetOperationStatusReq(opHandle.toTOperationHandle());
-      TGetOperationStatusResp resp = cliService.GetOperationStatus(req);
-      // Checks the status of the RPC call, throws an exception in case of error
-      checkStatus(resp.getStatus());
-      OperationState opState = OperationState.getOperationState(resp.getOperationState());
-      HiveSQLException opException = null;
-      if (opState == OperationState.ERROR) {
-        opException = new HiveSQLException(resp.getErrorMessage(), resp.getSqlState(), resp.getErrorCode());
-      }
-      return new OperationStatus(opState, opException);
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#cancelOperation(org.apache.hive.service.cli.OperationHandle)
-   */
-  @Override
-  public void cancelOperation(OperationHandle opHandle) throws HiveSQLException {
-    try {
-      TCancelOperationReq req = new TCancelOperationReq(opHandle.toTOperationHandle());
-      TCancelOperationResp resp = cliService.CancelOperation(req);
-      checkStatus(resp.getStatus());
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#closeOperation(org.apache.hive.service.cli.OperationHandle)
-   */
-  @Override
-  public void closeOperation(OperationHandle opHandle)
-      throws HiveSQLException {
-    try {
-      TCloseOperationReq req  = new TCloseOperationReq(opHandle.toTOperationHandle());
-      TCloseOperationResp resp = cliService.CloseOperation(req);
-      checkStatus(resp.getStatus());
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#getResultSetMetadata(org.apache.hive.service.cli.OperationHandle)
-   */
-  @Override
-  public TableSchema getResultSetMetadata(OperationHandle opHandle)
-      throws HiveSQLException {
-    try {
-      TGetResultSetMetadataReq req = new TGetResultSetMetadataReq(opHandle.toTOperationHandle());
-      TGetResultSetMetadataResp resp = cliService.GetResultSetMetadata(req);
-      checkStatus(resp.getStatus());
-      return new TableSchema(resp.getSchema());
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  @Override
-  public RowSet fetchResults(OperationHandle opHandle, FetchOrientation orientation, long maxRows,
-      FetchType fetchType) throws HiveSQLException {
-    try {
-      TFetchResultsReq req = new TFetchResultsReq();
-      req.setOperationHandle(opHandle.toTOperationHandle());
-      req.setOrientation(orientation.toTFetchOrientation());
-      req.setMaxRows(maxRows);
-      req.setFetchType(fetchType.toTFetchType());
-      TFetchResultsResp resp = cliService.FetchResults(req);
-      checkStatus(resp.getStatus());
-      return RowSetFactory.create(resp.getResults(), opHandle.getProtocolVersion());
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hive.service.cli.ICLIService#fetchResults(org.apache.hive.service.cli.OperationHandle)
-   */
-  @Override
-  public RowSet fetchResults(OperationHandle opHandle) throws HiveSQLException {
-    // TODO: set the correct default fetch size
-    return fetchResults(opHandle, FetchOrientation.FETCH_NEXT, 10000, FetchType.QUERY_OUTPUT);
-  }
-
-  @Override
-  public String getDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
-      String owner, String renewer) throws HiveSQLException {
-    TGetDelegationTokenReq req = new TGetDelegationTokenReq(
-        sessionHandle.toTSessionHandle(), owner, renewer);
-    try {
-      TGetDelegationTokenResp tokenResp = cliService.GetDelegationToken(req);
-      checkStatus(tokenResp.getStatus());
-      return tokenResp.getDelegationToken();
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  @Override
-  public void cancelDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
-      String tokenStr) throws HiveSQLException {
-    TCancelDelegationTokenReq cancelReq = new TCancelDelegationTokenReq(
-          sessionHandle.toTSessionHandle(), tokenStr);
-    try {
-      TCancelDelegationTokenResp cancelResp =
-        cliService.CancelDelegationToken(cancelReq);
-      checkStatus(cancelResp.getStatus());
-      return;
-    } catch (TException e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  @Override
-  public void renewDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory,
-      String tokenStr) throws HiveSQLException {
-    TRenewDelegationTokenReq cancelReq = new TRenewDelegationTokenReq(
-        sessionHandle.toTSessionHandle(), tokenStr);
-    try {
-      TRenewDelegationTokenResp renewResp =
-        cliService.RenewDelegationToken(cancelReq);
-      checkStatus(renewResp.getStatus());
-      return;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  @Override
-  public OperationHandle getPrimaryKeys(SessionHandle sessionHandle,
-      String catalog, String schema, String table) throws HiveSQLException {
-    try {
-      TGetPrimaryKeysReq req = new TGetPrimaryKeysReq(sessionHandle.toTSessionHandle());
-      req.setCatalogName(catalog);
-      req.setSchemaName(schema);
-      req.setTableName(table);
-      TGetPrimaryKeysResp resp = cliService.GetPrimaryKeys(req);
-      checkStatus(resp.getStatus());
-      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
-      return new OperationHandle(resp.getOperationHandle(), protocol);
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-
-  @Override
-  public OperationHandle getCrossReference(SessionHandle sessionHandle,
-      String primaryCatalog, String primarySchema, String primaryTable,
-      String foreignCatalog, String foreignSchema, String foreignTable) throws HiveSQLException {
-    try {
-      TGetCrossReferenceReq req = new TGetCrossReferenceReq(sessionHandle.toTSessionHandle());
-      req.setParentCatalogName(primaryCatalog);
-      req.setParentSchemaName(primarySchema);
-      req.setParentTableName(primaryTable);
-      req.setForeignCatalogName(foreignCatalog);
-      req.setForeignSchemaName(foreignSchema);
-      req.setForeignTableName(foreignTable);
-      TGetCrossReferenceResp resp = cliService.GetCrossReference(req);
-      checkStatus(resp.getStatus());
-      TProtocolVersion protocol = sessionHandle.getProtocolVersion();
-      return new OperationHandle(resp.getOperationHandle(), protocol);
-    } catch (HiveSQLException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new HiveSQLException(e);
-    }
-  }
-}
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
deleted file mode 100644
index 08626e7eb146d..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.service.cli.thrift;
-
-import java.util.Arrays;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.Shell;
-import org.apache.hive.service.auth.HiveAuthFactory;
-import org.apache.hive.service.cli.CLIService;
-import org.apache.hive.service.rpc.thrift.TCLIService;
-import org.apache.hive.service.rpc.thrift.TCLIService.Iface;
-import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup;
-import org.apache.thrift.TProcessor;
-import org.apache.thrift.protocol.TBinaryProtocol;
-import org.apache.thrift.protocol.TProtocolFactory;
-import org.apache.thrift.server.TServlet;
-import org.eclipse.jetty.server.AbstractConnectionFactory;
-import org.eclipse.jetty.server.ConnectionFactory;
-import org.eclipse.jetty.server.HttpConnectionFactory;
-import org.eclipse.jetty.server.ServerConnector;
-import org.eclipse.jetty.servlet.ServletContextHandler;
-import org.eclipse.jetty.servlet.ServletHolder;
-import org.eclipse.jetty.util.ssl.SslContextFactory;
-import org.eclipse.jetty.util.thread.ExecutorThreadPool;
-import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler;
-
-
-public class ThriftHttpCLIService extends ThriftCLIService {
-
-  public ThriftHttpCLIService(CLIService cliService) {
-    super(cliService, ThriftHttpCLIService.class.getSimpleName());
-  }
-
-  /**
-   * Configure Jetty to serve http requests. Example of a client connection URL:
-   * http://localhost:10000/servlets/thrifths2/ A gateway may cause actual target URL to differ,
-   * e.g. http://gateway:port/hive2/servlets/thrifths2/
-   */
-  @Override
-  public void run() {
-    try {
-      // Server thread pool
-      // Start with minWorkerThreads, expand till maxWorkerThreads and reject subsequent requests
-      String threadPoolName = "HiveServer2-HttpHandler-Pool";
-      ThreadPoolExecutor executorService = new ThreadPoolExecutor(minWorkerThreads, maxWorkerThreads,
-          workerKeepAliveTime, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
-          new ThreadFactoryWithGarbageCleanup(threadPoolName));
-      ExecutorThreadPool threadPool = new ExecutorThreadPool(executorService);
-
-      // HTTP Server
-      httpServer = new org.eclipse.jetty.server.Server(threadPool);
-
-      // Connector configs
-
-      ConnectionFactory[] connectionFactories;
-      boolean useSsl = hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_USE_SSL);
-      String schemeName = useSsl ? "https" : "http";
-      // Change connector if SSL is used
-      if (useSsl) {
-        String keyStorePath = hiveConf.getVar(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH).trim();
-        String keyStorePassword = ShimLoader.getHadoopShims().getPassword(hiveConf,
-            HiveConf.ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname);
-        if (keyStorePath.isEmpty()) {
-          throw new IllegalArgumentException(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname
-              + " Not configured for SSL connection");
-        }
-        SslContextFactory sslContextFactory = new SslContextFactory.Server();
-        String[] excludedProtocols = hiveConf.getVar(ConfVars.HIVE_SSL_PROTOCOL_BLACKLIST).split(",");
-        LOG.info("HTTP Server SSL: adding excluded protocols: " + Arrays.toString(excludedProtocols));
-        sslContextFactory.addExcludeProtocols(excludedProtocols);
-        LOG.info("HTTP Server SSL: SslContextFactory.getExcludeProtocols = " +
-          Arrays.toString(sslContextFactory.getExcludeProtocols()));
-        sslContextFactory.setKeyStorePath(keyStorePath);
-        sslContextFactory.setKeyStorePassword(keyStorePassword);
-        connectionFactories = AbstractConnectionFactory.getFactories(
-            sslContextFactory, new HttpConnectionFactory());
-      } else {
-        connectionFactories = new ConnectionFactory[] { new HttpConnectionFactory() };
-      }
-      ServerConnector connector = new ServerConnector(
-          httpServer,
-          null,
-          // Call this full constructor to set this, which forces daemon threads:
-          new ScheduledExecutorScheduler("HiveServer2-HttpHandler-JettyScheduler", true),
-          null,
-          -1,
-          -1,
-          connectionFactories);
-
-      connector.setPort(portNum);
-      // Linux:yes, Windows:no
-      connector.setReuseAddress(!Shell.WINDOWS);
-      int maxIdleTime = (int) hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_MAX_IDLE_TIME,
-          TimeUnit.MILLISECONDS);
-      connector.setIdleTimeout(maxIdleTime);
-
-      httpServer.addConnector(connector);
-
-      // Thrift configs
-      hiveAuthFactory = new HiveAuthFactory(hiveConf);
-      TProcessor processor = new TCLIService.Processor<Iface>(this);
-      TProtocolFactory protocolFactory = new TBinaryProtocol.Factory();
-      // Set during the init phase of HiveServer2 if auth mode is kerberos
-      // UGI for the hive/_HOST (kerberos) principal
-      UserGroupInformation serviceUGI = cliService.getServiceUGI();
-      // UGI for the http/_HOST (SPNego) principal
-      UserGroupInformation httpUGI = cliService.getHttpUGI();
-      String authType = hiveConf.getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION);
-      TServlet thriftHttpServlet = new ThriftHttpServlet(processor, protocolFactory, authType,
-          serviceUGI, httpUGI, hiveAuthFactory);
-
-      // Context handler
-      final ServletContextHandler context = new ServletContextHandler(
-          ServletContextHandler.SESSIONS);
-      context.setContextPath("/");
-      String httpPath = getHttpPath(hiveConf
-          .getVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH));
-      httpServer.setHandler(context);
-      context.addServlet(new ServletHolder(thriftHttpServlet), httpPath);
-
-      // TODO: check defaults: maxTimeout, keepalive, maxBodySize, bodyRecieveDuration, etc.
-      // Finally, start the server
-      httpServer.start();
-      String msg = "Started " + ThriftHttpCLIService.class.getSimpleName() + " in " + schemeName
-          + " mode on port " + portNum + " path=" + httpPath + " with " + minWorkerThreads + "..."
-          + maxWorkerThreads + " worker threads";
-      LOG.info(msg);
-      httpServer.join();
-    } catch (Throwable t) {
-      LOG.error(
-          "Error starting HiveServer2: could not start "
-              + ThriftHttpCLIService.class.getSimpleName(), t);
-      System.exit(-1);
-    }
-  }
-
-  /**
-   * The config parameter can be like "path", "/path", "/path/", "path/*", "/path1/path2/*" and so on.
-   * httpPath should end up as "/*", "/path/*" or "/path1/../pathN/*"
-   * @param httpPath
-   * @return
-   */
-  private String getHttpPath(String httpPath) {
-    if(httpPath == null || httpPath.equals("")) {
-      httpPath = "/*";
-    }
-    else {
-      if(!httpPath.startsWith("/")) {
-        httpPath = "/" + httpPath;
-      }
-      if(httpPath.endsWith("/")) {
-        httpPath = httpPath + "*";
-      }
-      if(!httpPath.endsWith("/*")) {
-        httpPath = httpPath + "/*";
-      }
-    }
-    return httpPath;
-  }
-}
diff --git a/sql/hive-thriftserver/v2.3/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala b/sql/hive-thriftserver/v2.3/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala
deleted file mode 100644
index 850382fe2bfd7..0000000000000
--- a/sql/hive-thriftserver/v2.3/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.hive.thriftserver
-
-import org.apache.hadoop.hive.ql.session.SessionState
-import org.apache.hadoop.hive.serde2.thrift.Type
-import org.apache.hadoop.hive.serde2.thrift.Type._
-import org.apache.hive.service.cli.{RowSet, RowSetFactory, TableSchema}
-import org.apache.hive.service.rpc.thrift.TProtocolVersion._
-import org.slf4j.LoggerFactory
-
-/**
- * Various utilities for hive-thriftserver used to upgrade the built-in Hive.
- */
-private[thriftserver] object ThriftserverShimUtils {
-
-  private[thriftserver] type TProtocolVersion = org.apache.hive.service.rpc.thrift.TProtocolVersion
-  private[thriftserver] type Client = org.apache.hive.service.rpc.thrift.TCLIService.Client
-  private[thriftserver] type TOpenSessionReq = org.apache.hive.service.rpc.thrift.TOpenSessionReq
-  private[thriftserver] type TGetSchemasReq = org.apache.hive.service.rpc.thrift.TGetSchemasReq
-  private[thriftserver] type TGetTablesReq = org.apache.hive.service.rpc.thrift.TGetTablesReq
-  private[thriftserver] type TGetColumnsReq = org.apache.hive.service.rpc.thrift.TGetColumnsReq
-  private[thriftserver] type TGetInfoReq = org.apache.hive.service.rpc.thrift.TGetInfoReq
-  private[thriftserver] type TExecuteStatementReq =
-    org.apache.hive.service.rpc.thrift.TExecuteStatementReq
-
-  private[thriftserver] def getConsole: SessionState.LogHelper = {
-    val LOG = LoggerFactory.getLogger(classOf[SparkSQLCLIDriver])
-    new SessionState.LogHelper(LOG)
-  }
-
-  private[thriftserver] def resultRowSet(
-      getResultSetSchema: TableSchema,
-      getProtocolVersion: TProtocolVersion): RowSet = {
-    RowSetFactory.create(getResultSetSchema, getProtocolVersion, false)
-  }
-
-  private[thriftserver] def toJavaSQLType(s: String): Int = Type.getType(s).toJavaSQLType
-
-  private[thriftserver] def supportedType(): Seq[Type] = {
-    Seq(NULL_TYPE, BOOLEAN_TYPE, STRING_TYPE, BINARY_TYPE,
-      TINYINT_TYPE, SMALLINT_TYPE, INT_TYPE, BIGINT_TYPE,
-      FLOAT_TYPE, DOUBLE_TYPE, DECIMAL_TYPE,
-      DATE_TYPE, TIMESTAMP_TYPE,
-      ARRAY_TYPE, MAP_TYPE, STRUCT_TYPE)
-  }
-
-  private[thriftserver] val testedProtocolVersions = Seq(
-    HIVE_CLI_SERVICE_PROTOCOL_V1,
-    HIVE_CLI_SERVICE_PROTOCOL_V2,
-    HIVE_CLI_SERVICE_PROTOCOL_V3,
-    HIVE_CLI_SERVICE_PROTOCOL_V4,
-    HIVE_CLI_SERVICE_PROTOCOL_V5,
-    HIVE_CLI_SERVICE_PROTOCOL_V6,
-    HIVE_CLI_SERVICE_PROTOCOL_V7,
-    HIVE_CLI_SERVICE_PROTOCOL_V8,
-    HIVE_CLI_SERVICE_PROTOCOL_V9,
-    HIVE_CLI_SERVICE_PROTOCOL_V10)
-}
diff --git a/sql/hive/benchmarks/InsertIntoHiveTableBenchmark-hive2.3-results.txt b/sql/hive/benchmarks/InsertIntoHiveTableBenchmark-hive2.3-results.txt
new file mode 100644
index 0000000000000..080beb1a7e04c
--- /dev/null
+++ b/sql/hive/benchmarks/InsertIntoHiveTableBenchmark-hive2.3-results.txt
@@ -0,0 +1,11 @@
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
+insert hive table benchmark:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+INSERT INTO DYNAMIC                                3706           3988         399          0.0      361896.5       1.0X
+INSERT INTO HYBRID                                  546            570          22          0.0       53358.1       6.8X
+INSERT INTO STATIC                                  174            184           6          0.1       16982.0      21.3X
+INSERT OVERWRITE DYNAMIC                           3263           3290          39          0.0      318624.1       1.1X
+INSERT OVERWRITE HYBRID                             493            536          42          0.0       48105.7       7.5X
+INSERT OVERWRITE STATIC                             187            196          10          0.1       18240.2      19.8X
+
diff --git a/sql/hive/benchmarks/InsertIntoHiveTableBenchmark-jdk11-hive2.3-results.txt b/sql/hive/benchmarks/InsertIntoHiveTableBenchmark-jdk11-hive2.3-results.txt
new file mode 100644
index 0000000000000..df4abe92db528
--- /dev/null
+++ b/sql/hive/benchmarks/InsertIntoHiveTableBenchmark-jdk11-hive2.3-results.txt
@@ -0,0 +1,11 @@
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
+insert hive table benchmark:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+INSERT INTO DYNAMIC                                4281           4685         572          0.0      418070.8       1.0X
+INSERT INTO HYBRID                                  776            834          56          0.0       75780.6       5.5X
+INSERT INTO STATIC                                  196            215          14          0.1       19143.1      21.8X
+INSERT OVERWRITE DYNAMIC                           4597           4775         253          0.0      448878.6       0.9X
+INSERT OVERWRITE HYBRID                             722            737          22          0.0       70517.6       5.9X
+INSERT OVERWRITE STATIC                             193            211          14          0.1       18830.4      22.2X
+
diff --git a/sql/hive/benchmarks/ObjectHashAggregateExecBenchmark-jdk11-results.txt b/sql/hive/benchmarks/ObjectHashAggregateExecBenchmark-jdk11-results.txt
index 4a8058766319f..90b7b9bede66d 100644
--- a/sql/hive/benchmarks/ObjectHashAggregateExecBenchmark-jdk11-results.txt
+++ b/sql/hive/benchmarks/ObjectHashAggregateExecBenchmark-jdk11-results.txt
@@ -2,44 +2,44 @@
 Hive UDAF vs Spark AF
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 hive udaf vs spark af:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hive udaf w/o group by                             6492           7169         388          0.0       99066.1       1.0X
-spark af w/o group by                                58             88          24          1.1         890.2     111.3X
-hive udaf w/ group by                              4864           4888          33          0.0       74221.0       1.3X
-spark af w/ group by w/o fallback                    60             67           7          1.1         912.9     108.5X
-spark af w/ group by w/ fallback                    154            164          27          0.4        2348.2      42.2X
+hive udaf w/o group by                             7504           7577          56          0.0      114496.1       1.0X
+spark af w/o group by                                37             44           7          1.8         560.2     204.4X
+hive udaf w/ group by                              5867           6075         194          0.0       89527.2       1.3X
+spark af w/ group by w/o fallback                    40             46           6          1.6         608.7     188.1X
+spark af w/ group by w/ fallback                     50             55           5          1.3         764.4     149.8X
 
 
 ================================================================================================
 ObjectHashAggregateExec vs SortAggregateExec - typed_count
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 object agg v.s. sort agg:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-sort agg w/ group by                              51728          51728           0          2.0         493.3       1.0X
-object agg w/ group by w/o fallback               10174          10218          34         10.3          97.0       5.1X
-object agg w/ group by w/ fallback                29341          29537         277          3.6         279.8       1.8X
-sort agg w/o group by                              7541           7577          28         13.9          71.9       6.9X
-object agg w/o group by w/o fallback               5574           5620          38         18.8          53.2       9.3X
+sort agg w/ group by                              41856          42278         597          2.5         399.2       1.0X
+object agg w/ group by w/o fallback               12479          12754         320          8.4         119.0       3.4X
+object agg w/ group by w/ fallback                25981          26058         109          4.0         247.8       1.6X
+sort agg w/o group by                              7351           7473         116         14.3          70.1       5.7X
+object agg w/o group by w/o fallback               7095           7406         490         14.8          67.7       5.9X
 
 
 ================================================================================================
 ObjectHashAggregateExec vs SortAggregateExec - percentile_approx
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 object agg v.s. sort agg:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-sort agg w/ group by                                900            925          14          2.3         429.0       1.0X
-object agg w/ group by w/o fallback                 597            633          14          3.5         284.6       1.5X
-object agg w/ group by w/ fallback                  905            923          10          2.3         431.6       1.0X
-sort agg w/o group by                               611            631          10          3.4         291.4       1.5X
-object agg w/o group by w/o fallback                559            576          11          3.8         266.5       1.6X
+sort agg w/ group by                                952           1047          92          2.2         453.9       1.0X
+object agg w/ group by w/o fallback                 829            957          90          2.5         395.2       1.1X
+object agg w/ group by w/ fallback                  972           1107         128          2.2         463.5       1.0X
+sort agg w/o group by                               732            858          98          2.9         349.1       1.3X
+object agg w/o group by w/o fallback                770            897          92          2.7         367.4       1.2X
 
 
diff --git a/sql/hive/benchmarks/ObjectHashAggregateExecBenchmark-results.txt b/sql/hive/benchmarks/ObjectHashAggregateExecBenchmark-results.txt
index 8c58a5a5fdf0b..60486269993a6 100644
--- a/sql/hive/benchmarks/ObjectHashAggregateExecBenchmark-results.txt
+++ b/sql/hive/benchmarks/ObjectHashAggregateExecBenchmark-results.txt
@@ -2,44 +2,44 @@
 Hive UDAF vs Spark AF
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 hive udaf vs spark af:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hive udaf w/o group by                             7014           7206         120          0.0      107031.0       1.0X
-spark af w/o group by                                47             59          11          1.4         716.9     149.3X
-hive udaf w/ group by                              4811           4831          28          0.0       73409.1       1.5X
-spark af w/ group by w/o fallback                    50             56           7          1.3         762.9     140.3X
-spark af w/ group by w/ fallback                    126            130           8          0.5        1916.6      55.8X
+hive udaf w/o group by                             4805           4881          55          0.0       73323.1       1.0X
+spark af w/o group by                                34             39           6          2.0         511.6     143.3X
+hive udaf w/ group by                              3770           3884         202          0.0       57526.2       1.3X
+spark af w/ group by w/o fallback                    34             39           5          1.9         525.7     139.5X
+spark af w/ group by w/ fallback                     39             44           5          1.7         596.5     122.9X
 
 
 ================================================================================================
 ObjectHashAggregateExec vs SortAggregateExec - typed_count
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 object agg v.s. sort agg:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-sort agg w/ group by                              42969          43306         476          2.4         409.8       1.0X
-object agg w/ group by w/o fallback                9744           9844         145         10.8          92.9       4.4X
-object agg w/ group by w/ fallback                26814          26960         206          3.9         255.7       1.6X
-sort agg w/o group by                              6278           6330          57         16.7          59.9       6.8X
-object agg w/o group by w/o fallback               5433           5478          60         19.3          51.8       7.9X
+sort agg w/ group by                              30645          31295         919          3.4         292.3       1.0X
+object agg w/ group by w/o fallback                7025           7425         425         14.9          67.0       4.4X
+object agg w/ group by w/ fallback                17222          18169        1251          6.1         164.2       1.8X
+sort agg w/o group by                              4755           4827         143         22.1          45.3       6.4X
+object agg w/o group by w/o fallback               4183           4495         269         25.1          39.9       7.3X
 
 
 ================================================================================================
 ObjectHashAggregateExec vs SortAggregateExec - percentile_approx
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
 object agg v.s. sort agg:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-sort agg w/ group by                                756            773           9          2.8         360.3       1.0X
-object agg w/ group by w/o fallback                 548            560           7          3.8         261.3       1.4X
-object agg w/ group by w/ fallback                  759            773           7          2.8         362.0       1.0X
-sort agg w/o group by                               471            483          13          4.4         224.8       1.6X
-object agg w/o group by w/o fallback                471            482          12          4.5         224.7       1.6X
+sort agg w/ group by                                625            673          29          3.4         297.9       1.0X
+object agg w/ group by w/o fallback                 509            549          24          4.1         242.9       1.2X
+object agg w/ group by w/ fallback                  600            652          33          3.5         286.2       1.0X
+sort agg w/o group by                               461            499          20          4.6         219.6       1.4X
+object agg w/o group by w/o fallback                470            510          22          4.5         224.0       1.3X
 
 
diff --git a/sql/hive/benchmarks/OrcReadBenchmark-jdk11-results.txt b/sql/hive/benchmarks/OrcReadBenchmark-jdk11-results.txt
index d516d3369ad05..114a9c2b4e896 100644
--- a/sql/hive/benchmarks/OrcReadBenchmark-jdk11-results.txt
+++ b/sql/hive/benchmarks/OrcReadBenchmark-jdk11-results.txt
@@ -2,155 +2,155 @@
 SQL Single Numeric Column Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 SQL Single TINYINT Column Scan:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                      1750           1872         173          9.0         111.2       1.0X
-Native ORC Vectorized                               433            499          68         36.3          27.5       4.0X
-Hive built-in ORC                                  2540           2575          49          6.2         161.5       0.7X
+Native ORC MR                                       873            886          12         18.0          55.5       1.0X
+Native ORC Vectorized                               114            162          42        138.2           7.2       7.7X
+Hive built-in ORC                                  1854           1924          99          8.5         117.9       0.5X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 SQL Single SMALLINT Column Scan:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                      1979           2001          31          7.9         125.8       1.0X
-Native ORC Vectorized                               261            303          42         60.3          16.6       7.6X
-Hive built-in ORC                                  2559           2583          34          6.1         162.7       0.8X
+Native ORC MR                                      1136           1330         275         13.8          72.2       1.0X
+Native ORC Vectorized                               191            288          85         82.1          12.2       5.9X
+Hive built-in ORC                                  2056           2068          17          7.7         130.7       0.6X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 SQL Single INT Column Scan:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                      2094           2158          91          7.5         133.2       1.0X
-Native ORC Vectorized                               309            361          41         50.8          19.7       6.8X
-Hive built-in ORC                                  2649           2744         135          5.9         168.4       0.8X
+Native ORC MR                                      1389           1457          97         11.3          88.3       1.0X
+Native ORC Vectorized                               394            408          11         40.0          25.0       3.5X
+Hive built-in ORC                                  2089           2105          23          7.5         132.8       0.7X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 SQL Single BIGINT Column Scan:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                      2256           2271          22          7.0         143.4       1.0X
-Native ORC Vectorized                               511            518          11         30.8          32.5       4.4X
-Hive built-in ORC                                  2867           2880          19          5.5         182.3       0.8X
+Native ORC MR                                      1361           1372          15         11.6          86.6       1.0X
+Native ORC Vectorized                               468            487          15         33.6          29.7       2.9X
+Hive built-in ORC                                  2305           2323          25          6.8         146.6       0.6X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 SQL Single FLOAT Column Scan:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                      2270           2325          78          6.9         144.3       1.0X
-Native ORC Vectorized                               502            508           5         31.3          31.9       4.5X
-Hive built-in ORC                                  2862           2880          24          5.5         182.0       0.8X
+Native ORC MR                                      1278           1285          10         12.3          81.2       1.0X
+Native ORC Vectorized                               379            428          49         41.6          24.1       3.4X
+Hive built-in ORC                                  2138           2143           6          7.4         135.9       0.6X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 SQL Single DOUBLE Column Scan:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                      2376           2426          71          6.6         151.0       1.0X
-Native ORC Vectorized                               609            616           8         25.8          38.7       3.9X
-Hive built-in ORC                                  2979           2991          17          5.3         189.4       0.8X
+Native ORC MR                                      1327           1341          20         11.9          84.3       1.0X
+Native ORC Vectorized                               443            466          21         35.5          28.1       3.0X
+Hive built-in ORC                                  2476           2538          88          6.4         157.4       0.5X
 
 
 ================================================================================================
 Int and String Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Int and String Scan:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                      4112           4232         170          2.6         392.1       1.0X
-Native ORC Vectorized                              2199           2223          35          4.8         209.7       1.9X
-Hive built-in ORC                                  5150           5238         123          2.0         491.2       0.8X
+Native ORC MR                                      2987           3079         130          3.5         284.9       1.0X
+Native ORC Vectorized                              1570           1582          16          6.7         149.8       1.9X
+Hive built-in ORC                                  3680           3689          13          2.8         350.9       0.8X
 
 
 ================================================================================================
 Partitioned Table Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Partitioned Table:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Data column - Native ORC MR                        2398           2435          53          6.6         152.4       1.0X
-Data column - Native ORC Vectorized                 458            482          26         34.3          29.1       5.2X
-Data column - Hive built-in ORC                    3126           3171          64          5.0         198.8       0.8X
-Partition column - Native ORC MR                   1639           1680          58          9.6         104.2       1.5X
-Partition column - Native ORC Vectorized            105            119          11        149.6           6.7      22.8X
-Partition column - Hive built-in ORC               2223           2229           8          7.1         141.4       1.1X
-Both columns - Native ORC MR                       2588           2608          28          6.1         164.5       0.9X
-Both columns - Native ORC Vectorized                489            522          49         32.2          31.1       4.9X
-Both columns - Hive built-in ORC                   3258           3292          48          4.8         207.1       0.7X
+Data column - Native ORC MR                        2264           2285          29          6.9         144.0       1.0X
+Data column - Native ORC Vectorized                 513            573          41         30.7          32.6       4.4X
+Data column - Hive built-in ORC                    3257           3303          65          4.8         207.1       0.7X
+Partition column - Native ORC MR                   1482           1500          24         10.6          94.2       1.5X
+Partition column - Native ORC Vectorized             48             77          29        326.2           3.1      47.0X
+Partition column - Hive built-in ORC               2104           2105           1          7.5         133.8       1.1X
+Both columns - Native ORC MR                       2414           2422          11          6.5         153.5       0.9X
+Both columns - Native ORC Vectorized                527            566          46         29.8          33.5       4.3X
+Both columns - Hive built-in ORC                   3101           3123          31          5.1         197.2       0.7X
 
 
 ================================================================================================
 Repeated String Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Repeated String:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                      1991           2028          52          5.3         189.9       1.0X
-Native ORC Vectorized                               392            398           8         26.7          37.4       5.1X
-Hive built-in ORC                                  2810           2816           8          3.7         268.0       0.7X
+Native ORC MR                                      1440           1458          26          7.3         137.3       1.0X
+Native ORC Vectorized                               253            314          97         41.4          24.1       5.7X
+Hive built-in ORC                                  1907           1915          11          5.5         181.9       0.8X
 
 
 ================================================================================================
 String with Nulls Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 String with Nulls Scan (0.0%):            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                      3638           3647          13          2.9         346.9       1.0X
-Native ORC Vectorized                              1171           1181          14          9.0         111.7       3.1X
-Hive built-in ORC                                  4847           4871          34          2.2         462.2       0.8X
+Native ORC MR                                      2394           2484         128          4.4         228.3       1.0X
+Native ORC Vectorized                               811            819          10         12.9          77.3       3.0X
+Hive built-in ORC                                  3317           3325          12          3.2         316.3       0.7X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 String with Nulls Scan (50.0%):           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                      3280           3283           5          3.2         312.8       1.0X
-Native ORC Vectorized                              1199           1206          10          8.7         114.4       2.7X
-Hive built-in ORC                                  4263           4273          14          2.5         406.5       0.8X
+Native ORC MR                                      1920           1987          94          5.5         183.1       1.0X
+Native ORC Vectorized                               937            989          46         11.2          89.3       2.0X
+Hive built-in ORC                                  2839           3060         313          3.7         270.7       0.7X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 String with Nulls Scan (95.0%):           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                      1935           1950          21          5.4         184.6       1.0X
-Native ORC Vectorized                               451            459          10         23.2          43.1       4.3X
-Hive built-in ORC                                  2542           2552          14          4.1         242.4       0.8X
+Native ORC MR                                      1050           1060          14         10.0         100.1       1.0X
+Native ORC Vectorized                               283            357          88         37.1          27.0       3.7X
+Hive built-in ORC                                  1745           1752          10          6.0         166.4       0.6X
 
 
 ================================================================================================
 Single Column Scan From Wide Columns
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Single Column Scan from 100 columns:      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                       270            292          23          3.9         257.2       1.0X
-Native ORC Vectorized                               143            155          12          7.3         136.2       1.9X
-Hive built-in ORC                                  1593           1627          48          0.7        1519.1       0.2X
+Native ORC MR                                       125            155          38          8.4         119.5       1.0X
+Native ORC Vectorized                                87            121          35         12.1          82.7       1.4X
+Hive built-in ORC                                  1261           1401         198          0.8        1202.9       0.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Single Column Scan from 200 columns:      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                       369            386          17          2.8         351.5       1.0X
-Native ORC Vectorized                               218            231          15          4.8         208.3       1.7X
-Hive built-in ORC                                  3092           3101          12          0.3        2949.1       0.1X
+Native ORC MR                                       151            197          47          7.0         143.8       1.0X
+Native ORC Vectorized                                90            121          36         11.7          85.7       1.7X
+Hive built-in ORC                                  2285           2293          12          0.5        2178.9       0.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 11.0.10+9-LTS on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz
 Single Column Scan from 300 columns:      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                       498            531          33          2.1         475.0       1.0X
-Native ORC Vectorized                               360            376          18          2.9         342.9       1.4X
-Hive built-in ORC                                  4786           4786           1          0.2        4564.1       0.1X
+Native ORC MR                                       172            232          55          6.1         163.7       1.0X
+Native ORC Vectorized                               111            154          37          9.4         106.2       1.5X
+Hive built-in ORC                                  3211           3218          10          0.3        3062.5       0.1X
 
 
diff --git a/sql/hive/benchmarks/OrcReadBenchmark-results.txt b/sql/hive/benchmarks/OrcReadBenchmark-results.txt
index c7d6c976192b2..c8f47da641dcf 100644
--- a/sql/hive/benchmarks/OrcReadBenchmark-results.txt
+++ b/sql/hive/benchmarks/OrcReadBenchmark-results.txt
@@ -2,155 +2,155 @@
 SQL Single Numeric Column Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 SQL Single TINYINT Column Scan:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                      1844           1851          10          8.5         117.2       1.0X
-Native ORC Vectorized                               284            312          36         55.5          18.0       6.5X
-Hive built-in ORC                                  2380           2380           1          6.6         151.3       0.8X
+Native ORC MR                                       897            922          25         17.5          57.0       1.0X
+Native ORC Vectorized                               117            128          13        134.6           7.4       7.7X
+Hive built-in ORC                                  1328           1332           6         11.8          84.4       0.7X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 SQL Single SMALLINT Column Scan:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                      1999           2031          45          7.9         127.1       1.0X
-Native ORC Vectorized                               252            264          15         62.5          16.0       7.9X
-Hive built-in ORC                                  2483           2509          37          6.3         157.9       0.8X
+Native ORC MR                                      1003           1010          10         15.7          63.8       1.0X
+Native ORC Vectorized                               148            168          34        106.0           9.4       6.8X
+Hive built-in ORC                                  1348           1364          23         11.7          85.7       0.7X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 SQL Single INT Column Scan:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                      2134           2135           2          7.4         135.7       1.0X
-Native ORC Vectorized                               329            351          34         47.8          20.9       6.5X
-Hive built-in ORC                                  2672           2716          61          5.9         169.9       0.8X
+Native ORC MR                                      1043           1047           6         15.1          66.3       1.0X
+Native ORC Vectorized                               168            174          10         93.8          10.7       6.2X
+Hive built-in ORC                                  1398           1440          60         11.3          88.9       0.7X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 SQL Single BIGINT Column Scan:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                      2172           2247         105          7.2         138.1       1.0X
-Native ORC Vectorized                               407            427          23         38.7          25.9       5.3X
-Hive built-in ORC                                  2806           2822          22          5.6         178.4       0.8X
+Native ORC MR                                      1114           1182          96         14.1          70.8       1.0X
+Native ORC Vectorized                               249            259          17         63.3          15.8       4.5X
+Hive built-in ORC                                  1533           1682         210         10.3          97.5       0.7X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 SQL Single FLOAT Column Scan:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                      2187           2200          19          7.2         139.0       1.0X
-Native ORC Vectorized                               451            457           5         34.9          28.7       4.8X
-Hive built-in ORC                                  2886           2938          73          5.4         183.5       0.8X
+Native ORC MR                                      1120           1127          10         14.0          71.2       1.0X
+Native ORC Vectorized                               228            232           4         69.1          14.5       4.9X
+Hive built-in ORC                                  1474           1481           9         10.7          93.7       0.8X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 SQL Single DOUBLE Column Scan:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                      2313           2319           9          6.8         147.1       1.0X
-Native ORC Vectorized                               554            562           7         28.4          35.2       4.2X
-Hive built-in ORC                                  2927           2933           8          5.4         186.1       0.8X
+Native ORC MR                                      1177           1206          42         13.4          74.8       1.0X
+Native ORC Vectorized                               270            295          47         58.3          17.2       4.4X
+Hive built-in ORC                                  1500           1504           6         10.5          95.4       0.8X
 
 
 ================================================================================================
 Int and String Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Int and String Scan:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                      4162           4294         186          2.5         397.0       1.0X
-Native ORC Vectorized                              2236           2258          32          4.7         213.2       1.9X
-Hive built-in ORC                                  5054           5135         114          2.1         482.0       0.8X
+Native ORC MR                                      2174           2184          14          4.8         207.3       1.0X
+Native ORC Vectorized                              1179           1195          22          8.9         112.4       1.8X
+Hive built-in ORC                                  2639           2669          43          4.0         251.7       0.8X
 
 
 ================================================================================================
 Partitioned Table Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Partitioned Table:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Data column - Native ORC MR                        2436           2447          16          6.5         154.8       1.0X
-Data column - Native ORC Vectorized                 421            443          35         37.4          26.8       5.8X
-Data column - Hive built-in ORC                    3007           3026          27          5.2         191.2       0.8X
-Partition column - Native ORC MR                   1603           1630          39          9.8         101.9       1.5X
-Partition column - Native ORC Vectorized             84             96          15        186.7           5.4      28.9X
-Partition column - Hive built-in ORC               2174           2187          18          7.2         138.2       1.1X
-Both columns - Native ORC MR                       2609           2645          51          6.0         165.9       0.9X
-Both columns - Native ORC Vectorized                460            470           9         34.2          29.3       5.3X
-Both columns - Hive built-in ORC                   3094           3099           8          5.1         196.7       0.8X
+Data column - Native ORC MR                        1502           1510          11         10.5          95.5       1.0X
+Data column - Native ORC Vectorized                 256            292          50         61.4          16.3       5.9X
+Data column - Hive built-in ORC                    1783           1793          15          8.8         113.3       0.8X
+Partition column - Native ORC MR                    968            979          11         16.2          61.6       1.6X
+Partition column - Native ORC Vectorized             46             52          16        340.2           2.9      32.5X
+Partition column - Hive built-in ORC               1370           1375           6         11.5          87.1       1.1X
+Both columns - Native ORC MR                       1516           1525          13         10.4          96.4       1.0X
+Both columns - Native ORC Vectorized                288            298          10         54.6          18.3       5.2X
+Both columns - Hive built-in ORC                   1796           1827          44          8.8         114.2       0.8X
 
 
 ================================================================================================
 Repeated String Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Repeated String:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                      2036           2046          13          5.1         194.2       1.0X
-Native ORC Vectorized                               366            386          18         28.6          34.9       5.6X
-Hive built-in ORC                                  2683           2686           4          3.9         255.9       0.8X
+Native ORC MR                                      1035           1057          32         10.1          98.7       1.0X
+Native ORC Vectorized                               197            200           2         53.3          18.8       5.3X
+Hive built-in ORC                                  1435           1453          25          7.3         136.9       0.7X
 
 
 ================================================================================================
 String with Nulls Scan
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 String with Nulls Scan (0.0%):            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                      3614           3643          40          2.9         344.7       1.0X
-Native ORC Vectorized                              1072           1087          22          9.8         102.2       3.4X
-Hive built-in ORC                                  4625           4636          15          2.3         441.1       0.8X
+Native ORC MR                                      1905           1927          31          5.5         181.7       1.0X
+Native ORC Vectorized                               563            645          80         18.6          53.7       3.4X
+Hive built-in ORC                                  2469           2477          11          4.2         235.5       0.8X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 String with Nulls Scan (50.0%):           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                      3347           3376          42          3.1         319.2       1.0X
-Native ORC Vectorized                              1220           1225           7          8.6         116.3       2.7X
-Hive built-in ORC                                  4168           4184          23          2.5         397.5       0.8X
+Native ORC MR                                      1733           1743          14          6.1         165.3       1.0X
+Native ORC Vectorized                               636            667          39         16.5          60.6       2.7X
+Hive built-in ORC                                  2201           2267          93          4.8         209.9       0.8X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 String with Nulls Scan (95.0%):           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                      1851           1862          16          5.7         176.5       1.0X
-Native ORC Vectorized                               466            471           7         22.5          44.4       4.0X
-Hive built-in ORC                                  2523           2529           8          4.2         240.6       0.7X
+Native ORC MR                                       966           1052         134         10.9          92.1       1.0X
+Native ORC Vectorized                               241            247           6         43.4          23.0       4.0X
+Hive built-in ORC                                  1285           1287           3          8.2         122.5       0.8X
 
 
 ================================================================================================
 Single Column Scan From Wide Columns
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Single Column Scan from 100 columns:      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                       250            264          15          4.2         238.1       1.0X
-Native ORC Vectorized                               121            138          24          8.7         115.5       2.1X
-Hive built-in ORC                                  1761           1792          43          0.6        1679.3       0.1X
+Native ORC MR                                       124            134          17          8.5         118.3       1.0X
+Native ORC Vectorized                                64             71          12         16.5          60.8       1.9X
+Hive built-in ORC                                  1037           1043           9          1.0         988.9       0.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Single Column Scan from 200 columns:      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                       319            341          17          3.3         304.5       1.0X
-Native ORC Vectorized                               188            222          50          5.6         178.8       1.7X
-Hive built-in ORC                                  3492           3508          24          0.3        3329.8       0.1X
+Native ORC MR                                       143            154          16          7.3         136.6       1.0X
+Native ORC Vectorized                                85             92          13         12.4          80.7       1.7X
+Hive built-in ORC                                  1987           1997          15          0.5        1894.5       0.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+OpenJDK 64-Bit Server VM 1.8.0_282-b08 on Linux 5.4.0-1043-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
 Single Column Scan from 300 columns:      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Native ORC MR                                       443            456          12          2.4         422.9       1.0X
-Native ORC Vectorized                               306            321          23          3.4         292.0       1.4X
-Hive built-in ORC                                  5295           5312          24          0.2        5049.9       0.1X
+Native ORC MR                                       167            190          23          6.3         159.3       1.0X
+Native ORC Vectorized                               105            117          14         10.0          99.7       1.6X
+Hive built-in ORC                                  2976           2985          13          0.4        2837.8       0.1X
 
 
diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
index db1f6fbd97d90..c263932c2f535 100644
--- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
+++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
@@ -22,14 +22,15 @@ import java.io.File
 import org.scalatest.BeforeAndAfter
 
 import org.apache.spark.sql.catalyst.rules.RuleExecutor
-import org.apache.spark.sql.hive.HiveUtils
 import org.apache.spark.sql.hive.test.TestHive
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy
+import org.apache.spark.tags.SlowHiveTest
 
 /**
  * Runs the test cases that are included in the hive distribution.
  */
+@SlowHiveTest
 class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
   // TODO: bundle in jar files... get from classpath
   private lazy val hiveQueryDir = TestHive.getHiveFile(
@@ -39,6 +40,8 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
   private val originalInMemoryPartitionPruning = TestHive.conf.inMemoryPartitionPruning
   private val originalCrossJoinEnabled = TestHive.conf.crossJoinEnabled
   private val originalSessionLocalTimeZone = TestHive.conf.sessionLocalTimeZone
+  private val originalCreateHiveTable =
+    TestHive.conf.getConf(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT)
 
   def testCases: Seq[(String, File)] = {
     hiveQueryDir.listFiles.map(f => f.getName.stripSuffix(".q") -> f)
@@ -53,11 +56,12 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
     TestHive.setConf(SQLConf.IN_MEMORY_PARTITION_PRUNING, true)
     // Ensures that cross joins are enabled so that we can test them
     TestHive.setConf(SQLConf.CROSS_JOINS_ENABLED, true)
-    // Ensures that the table insertion behaivor is consistent with Hive
+    // Ensures that the table insertion behavior is consistent with Hive
     TestHive.setConf(SQLConf.STORE_ASSIGNMENT_POLICY, StoreAssignmentPolicy.LEGACY.toString)
     // Fix session local timezone to America/Los_Angeles for those timezone sensitive tests
     // (timestamp_*)
     TestHive.setConf(SQLConf.SESSION_LOCAL_TIMEZONE, "America/Los_Angeles")
+    TestHive.setConf(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT, true)
     RuleExecutor.resetMetrics()
   }
 
@@ -68,6 +72,7 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
       TestHive.setConf(SQLConf.IN_MEMORY_PARTITION_PRUNING, originalInMemoryPartitionPruning)
       TestHive.setConf(SQLConf.CROSS_JOINS_ENABLED, originalCrossJoinEnabled)
       TestHive.setConf(SQLConf.SESSION_LOCAL_TIMEZONE, originalSessionLocalTimeZone)
+      TestHive.setConf(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT, originalCreateHiveTable)
 
       // For debugging dump some statistics about how much time was spent in various optimizer rules
       logWarning(RuleExecutor.dumpTimeSpent())
@@ -77,7 +82,7 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
   }
 
   /** A list of tests deemed out of scope currently and thus completely disregarded. */
-  override def blackList: Seq[String] = Seq(
+  override def excludeList: Seq[String] = Seq(
     // These tests use hooks that are not on the classpath and thus break all subsequent execution.
     "hook_order",
     "hook_context_cs",
@@ -300,7 +305,7 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
     // Unsupported underscore syntax.
     "inputddl5",
 
-    // Thift is broken...
+    // Thrift is broken...
     "inputddl8",
 
     // Hive changed ordering of ddl:
@@ -491,7 +496,7 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
     "drop_partitions_filter2",
     "drop_partitions_filter3",
 
-    // The following failes due to truncate table
+    // The following fails due to truncate table
     "truncate_table",
 
     // We do not support DFS command.
@@ -508,7 +513,7 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
     // This test uses CREATE EXTERNAL TABLE without specifying LOCATION
     "alter2",
 
-    // [SPARK-16248][SQL] Whitelist the list of Hive fallback functions
+    // [SPARK-16248][SQL] Include the list of Hive fallback functions
     "udf_field",
     "udf_reflect2",
     "udf_xpath",
@@ -596,7 +601,7 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
     "udf_radians"
   )
 
-  private def commonWhiteList = Seq(
+  private def commonIncludeList = Seq(
     "add_part_exist",
     "add_part_multiple",
     "add_partition_no_whitelist",
@@ -711,7 +716,7 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
     "groupby_multi_insert_common_distinct",
     "groupby_multi_single_reducer2",
     "groupby_multi_single_reducer3",
-    "groupby_mutli_insert_common_distinct",
+    "groupby_multi_insert_common_distinct",
     "groupby_neg_float",
     "groupby_ppd",
     "groupby_ppr",
@@ -953,8 +958,8 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
     "subq2",
     "subquery_exists",
     "subquery_exists_having",
-    "subquery_notexists",
-    "subquery_notexists_having",
+    "subquery_nonexistent",
+    "subquery_nonexistent_having",
     "subquery_in_having",
     "tablename_with_select",
     "timestamp_comparison",
@@ -1134,14 +1139,11 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
   )
 
   /**
-   * The set of tests that are believed to be working in catalyst. Tests not on whiteList or
-   * blacklist are implicitly marked as ignored.
+   * The set of tests that are believed to be working in catalyst. Tests not on includeList or
+   * excludeList are implicitly marked as ignored.
    */
-  override def whiteList: Seq[String] = if (HiveUtils.isHive23) {
-    commonWhiteList ++ Seq(
+  override def includeList: Seq[String] =
+    commonIncludeList ++ Seq(
       "decimal_1_1"
     )
-  } else {
-    commonWhiteList
-  }
 }
diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala
index 2c0970c85449f..1b801ad69564c 100644
--- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala
+++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala
@@ -763,7 +763,7 @@ class HiveWindowFunctionQueryFileSuite
     }
   }
 
-  override def blackList: Seq[String] = Seq(
+  override def excludeList: Seq[String] = Seq(
     // Partitioned table functions are not supported.
     "ptf*",
     // tests of windowing.q are in HiveWindowFunctionQueryBaseSuite
@@ -791,12 +791,12 @@ class HiveWindowFunctionQueryFileSuite
     "windowing_adjust_rowcontainer_sz"
   )
 
-  override def whiteList: Seq[String] = Seq(
+  override def includeList: Seq[String] = Seq(
     "windowing_udaf2"
   )
 
-  // Only run those query tests in the realWhileList (do not try other ignored query files).
+  // Only run those query tests in the realIncludeList (do not try other ignored query files).
   override def testCases: Seq[(String, File)] = super.testCases.filter {
-    case (name, _) => realWhiteList.contains(name)
+    case (name, _) => realIncludeList.contains(name)
   }
 }
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index 474c6066ed040..729d3f414287e 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
@@ -35,11 +35,6 @@
   </properties>
 
   <dependencies>
-    <!-- Added for Hive Parquet SerDe -->
-    <dependency>
-      <groupId>${hive.parquet.group}</groupId>
-      <artifactId>parquet-hadoop-bundle</artifactId>
-    </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-core_${scala.binary.version}</artifactId>
@@ -77,6 +72,13 @@
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.parquet</groupId>
+      <artifactId>parquet-hadoop</artifactId>
+      <version>${parquet.version}</version>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
 <!--
     <dependency>
       <groupId>com.google.guava</groupId>
@@ -127,12 +129,9 @@
       <groupId>org.apache.avro</groupId>
       <artifactId>avro</artifactId>
     </dependency>
-    <!-- use the build matching the hadoop api of avro-mapred (i.e. no classifier for hadoop 1 API,
-    hadoop2 classifier for hadoop 2 API. avro-mapred is a dependency of org.spark-project.hive:hive-serde -->
     <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro-mapred</artifactId>
-      <classifier>${avro.mapred.classifier}</classifier>
     </dependency>
     <dependency>
       <groupId>commons-httpclient</groupId>
@@ -167,6 +166,11 @@
       <groupId>org.datanucleus</groupId>
       <artifactId>datanucleus-core</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>${hadoop-client-runtime.artifact}</artifactId>
+      <scope>${hadoop.deps.scope}</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.thrift</groupId>
       <artifactId>libthrift</artifactId>
@@ -217,6 +221,15 @@
         </plugins>
       </build>
     </profile>
+    <profile>
+      <id>scala-2.13</id>
+      <dependencies>
+        <dependency>
+          <groupId>org.scala-lang.modules</groupId>
+          <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
+        </dependency>
+      </dependencies>
+    </profile>
   </profiles>
 
   <build>
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
index 02a5117f005e8..2d72443a8b661 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
@@ -34,7 +34,7 @@ class HiveContext private[hive](_sparkSession: SparkSession)
   self =>
 
   def this(sc: SparkContext) = {
-    this(SparkSession.builder().sparkContext(HiveUtils.withHiveExternalCatalog(sc)).getOrCreate())
+    this(SparkSession.builder().enableHiveSupport().sparkContext(sc).getOrCreate())
   }
 
   def this(sc: JavaSparkContext) = this(sc.sc)
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
index 2faf42028f3a2..7f50d522ff42c 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
@@ -40,8 +40,7 @@ import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils._
 import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
-import org.apache.spark.sql.connector.catalog.TableCatalog
+import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateTimeUtils}
 import org.apache.spark.sql.execution.command.DDLUtils
 import org.apache.spark.sql.execution.datasources.{PartitioningUtils, SourceOptions}
 import org.apache.spark.sql.hive.client.HiveClient
@@ -61,6 +60,10 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
   import HiveExternalCatalog._
   import CatalogTableType._
 
+  // SPARK-32256: Make sure `VersionInfo` is initialized before touching the isolated classloader.
+  // This is to ensure Hive can get the Hadoop version when using the isolated classloader.
+  org.apache.hadoop.util.VersionInfo.getVersion()
+
   /**
    * A Hive client used to interact with the metastore.
    */
@@ -426,18 +429,8 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
     val properties = new mutable.HashMap[String, String]
 
     properties.put(CREATED_SPARK_VERSION, table.createVersion)
-
-    // Serialized JSON schema string may be too long to be stored into a single metastore table
-    // property. In this case, we split the JSON string and store each part as a separate table
-    // property.
-    val threshold = conf.get(SCHEMA_STRING_LENGTH_THRESHOLD)
-    val schemaJsonString = schema.json
-    // Split the JSON string.
-    val parts = schemaJsonString.grouped(threshold).toSeq
-    properties.put(DATASOURCE_SCHEMA_NUMPARTS, parts.size.toString)
-    parts.zipWithIndex.foreach { case (part, index) =>
-      properties.put(s"$DATASOURCE_SCHEMA_PART_PREFIX$index", part)
-    }
+    CatalogTable.splitLargeTableProp(
+      DATASOURCE_SCHEMA, schema.json, properties.put, conf.get(SCHEMA_STRING_LENGTH_THRESHOLD))
 
     if (partitionColumns.nonEmpty) {
       properties.put(DATASOURCE_SCHEMA_NUMPARTCOLS, partitionColumns.length.toString)
@@ -741,8 +734,8 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       case None if table.tableType == VIEW =>
         // If this is a view created by Spark 2.2 or higher versions, we should restore its schema
         // from table properties.
-        if (table.properties.contains(DATASOURCE_SCHEMA_NUMPARTS)) {
-          table = table.copy(schema = getSchemaFromTableProperties(table))
+        CatalogTable.readLargeTableProp(table.properties, DATASOURCE_SCHEMA).foreach { schemaJson =>
+          table = table.copy(schema = DataType.fromJson(schemaJson).asInstanceOf[StructType])
         }
 
       // No provider in table properties, which means this is a Hive serde table.
@@ -792,8 +785,9 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
 
     // If this is a Hive serde table created by Spark 2.1 or higher versions, we should restore its
     // schema from table properties.
-    if (table.properties.contains(DATASOURCE_SCHEMA_NUMPARTS)) {
-      val schemaFromTableProps = getSchemaFromTableProperties(table)
+    val schemaJson = CatalogTable.readLargeTableProp(table.properties, DATASOURCE_SCHEMA)
+    if (schemaJson.isDefined) {
+      val schemaFromTableProps = DataType.fromJson(schemaJson.get).asInstanceOf[StructType]
       val partColumnNames = getPartitionColumnsFromTableProperties(table)
       val reorderedSchema = reorderSchema(schema = schemaFromTableProps, partColumnNames)
 
@@ -829,11 +823,12 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       updateLocationInStorageProps(table, newPath = None).copy(
         locationUri = tableLocation.map(CatalogUtils.stringToURI(_)))
     }
-    val storageWithoutHiveGeneratedProperties = storageWithLocation.copy(
-      properties = storageWithLocation.properties.filterKeys(!HIVE_GENERATED_STORAGE_PROPERTIES(_)))
+    val storageWithoutHiveGeneratedProperties = storageWithLocation.copy(properties =
+      storageWithLocation.properties.filterKeys(!HIVE_GENERATED_STORAGE_PROPERTIES(_)).toMap)
     val partitionProvider = table.properties.get(TABLE_PARTITION_PROVIDER)
 
-    val schemaFromTableProps = getSchemaFromTableProperties(table)
+    val schemaFromTableProps = CatalogTable.readLargeTableProp(table.properties, DATASOURCE_SCHEMA)
+      .map(json => DataType.fromJson(json).asInstanceOf[StructType]).getOrElse(new StructType())
     val partColumnNames = getPartitionColumnsFromTableProperties(table)
     val reorderedSchema = reorderSchema(schema = schemaFromTableProps, partColumnNames)
 
@@ -844,7 +839,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       partitionColumnNames = partColumnNames,
       bucketSpec = getBucketSpecFromTableProperties(table),
       tracksPartitionsInCatalog = partitionProvider == Some(TABLE_PARTITION_PROVIDER_CATALOG),
-      properties = table.properties.filterKeys(!HIVE_GENERATED_TABLE_PROPERTIES(_)))
+      properties = table.properties.filterKeys(!HIVE_GENERATED_TABLE_PROPERTIES(_)).toMap)
   }
 
   override def tableExists(db: String, table: String): Boolean = withClient {
@@ -947,9 +942,10 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
   // Hive metastore is not case preserving and the partition columns are always lower cased. We need
   // to lower case the column names in partition specification before calling partition related Hive
   // APIs, to match this behaviour.
-  private def lowerCasePartitionSpec(spec: TablePartitionSpec): TablePartitionSpec = {
+  private def toMetaStorePartitionSpec(spec: TablePartitionSpec): TablePartitionSpec = {
     // scalastyle:off caselocale
-    spec.map { case (k, v) => k.toLowerCase -> v }
+    val lowNames = spec.map { case (k, v) => k.toLowerCase -> v }
+    ExternalCatalogUtils.convertNullPartitionValues(lowNames)
     // scalastyle:on caselocale
   }
 
@@ -998,8 +994,9 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       }
       p.copy(storage = p.storage.copy(locationUri = Some(partitionPath.toUri)))
     }
-    val lowerCasedParts = partsWithLocation.map(p => p.copy(spec = lowerCasePartitionSpec(p.spec)))
-    client.createPartitions(db, table, lowerCasedParts, ignoreIfExists)
+    val metaStoreParts = partsWithLocation
+      .map(p => p.copy(spec = toMetaStorePartitionSpec(p.spec)))
+    client.createPartitions(db, table, metaStoreParts, ignoreIfExists)
   }
 
   override def dropPartitions(
@@ -1011,7 +1008,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       retainData: Boolean): Unit = withClient {
     requireTableExists(db, table)
     client.dropPartitions(
-      db, table, parts.map(lowerCasePartitionSpec), ignoreIfNotExists, purge, retainData)
+      db, table, parts.map(toMetaStorePartitionSpec), ignoreIfNotExists, purge, retainData)
   }
 
   override def renamePartitions(
@@ -1020,7 +1017,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       specs: Seq[TablePartitionSpec],
       newSpecs: Seq[TablePartitionSpec]): Unit = withClient {
     client.renamePartitions(
-      db, table, specs.map(lowerCasePartitionSpec), newSpecs.map(lowerCasePartitionSpec))
+      db, table, specs.map(toMetaStorePartitionSpec), newSpecs.map(toMetaStorePartitionSpec))
 
     val tableMeta = getTable(db, table)
     val partitionColumnNames = tableMeta.partitionColumnNames
@@ -1036,7 +1033,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       val fs = tablePath.getFileSystem(hadoopConf)
       val newParts = newSpecs.map { spec =>
         val rightPath = renamePartitionDirectory(fs, tablePath, partitionColumnNames, spec)
-        val partition = client.getPartition(db, table, lowerCasePartitionSpec(spec))
+        val partition = client.getPartition(db, table, toMetaStorePartitionSpec(spec))
         partition.copy(storage = partition.storage.copy(locationUri = Some(rightPath.toUri)))
       }
       alterPartitions(db, table, newParts)
@@ -1121,7 +1118,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       val colStats = new mutable.HashMap[String, CatalogColumnStat]
       val colStatsProps = properties.filterKeys(_.startsWith(STATISTICS_COL_STATS_PREFIX)).map {
         case (k, v) => k.drop(STATISTICS_COL_STATS_PREFIX.length) -> v
-      }
+      }.toMap
 
       // Find all the column names by matching the KEY_VERSION properties for them.
       colStatsProps.keys.filter {
@@ -1146,12 +1143,9 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       db: String,
       table: String,
       newParts: Seq[CatalogTablePartition]): Unit = withClient {
-    val lowerCasedParts = newParts.map(p => p.copy(spec = lowerCasePartitionSpec(p.spec)))
-
-    val rawTable = getRawTable(db, table)
-
+    val metaStoreParts = newParts.map(p => p.copy(spec = toMetaStorePartitionSpec(p.spec)))
     // convert partition statistics to properties so that we can persist them through hive api
-    val withStatsProps = lowerCasedParts.map { p =>
+    val withStatsProps = metaStoreParts.map { p =>
       if (p.stats.isDefined) {
         val statsProperties = statsToProperties(p.stats.get)
         p.copy(parameters = p.parameters ++ statsProperties)
@@ -1167,7 +1161,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       db: String,
       table: String,
       spec: TablePartitionSpec): CatalogTablePartition = withClient {
-    val part = client.getPartition(db, table, lowerCasePartitionSpec(spec))
+    val part = client.getPartition(db, table, toMetaStorePartitionSpec(spec))
     restorePartitionMetadata(part, getTable(db, table))
   }
 
@@ -1205,7 +1199,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       db: String,
       table: String,
       spec: TablePartitionSpec): Option[CatalogTablePartition] = withClient {
-    client.getPartitionOption(db, table, lowerCasePartitionSpec(spec)).map { part =>
+    client.getPartitionOption(db, table, toMetaStorePartitionSpec(spec)).map { part =>
       restorePartitionMetadata(part, getTable(db, table))
     }
   }
@@ -1220,7 +1214,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
     val catalogTable = getTable(db, table)
     val partColNameMap = buildLowerCasePartColNameMap(catalogTable).mapValues(escapePathName)
     val clientPartitionNames =
-      client.getPartitionNames(catalogTable, partialSpec.map(lowerCasePartitionSpec))
+      client.getPartitionNames(catalogTable, partialSpec.map(toMetaStorePartitionSpec))
     clientPartitionNames.map { partitionPath =>
       val partSpec = PartitioningUtils.parsePathFragmentAsSeq(partitionPath)
       partSpec.map { case (partName, partValue) =>
@@ -1239,11 +1233,12 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       table: String,
       partialSpec: Option[TablePartitionSpec] = None): Seq[CatalogTablePartition] = withClient {
     val partColNameMap = buildLowerCasePartColNameMap(getTable(db, table))
-    val res = client.getPartitions(db, table, partialSpec.map(lowerCasePartitionSpec)).map { part =>
-      part.copy(spec = restorePartitionSpec(part.spec, partColNameMap))
+    val metaStoreSpec = partialSpec.map(toMetaStorePartitionSpec)
+    val res = client.getPartitions(db, table, metaStoreSpec)
+      .map { part => part.copy(spec = restorePartitionSpec(part.spec, partColNameMap))
     }
 
-    partialSpec match {
+    metaStoreSpec match {
       // This might be a bug of Hive: When the partition value inside the partial partition spec
       // contains dot, and we ask Hive to list partitions w.r.t. the partial partition spec, Hive
       // treats dot as matching any single character and may return more partitions than we
@@ -1261,11 +1256,13 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       defaultTimeZoneId: String): Seq[CatalogTablePartition] = withClient {
     val rawTable = getRawTable(db, table)
     val catalogTable = restoreTableMetadata(rawTable)
+    val timeZoneId = CaseInsensitiveMap(catalogTable.storage.properties).getOrElse(
+      DateTimeUtils.TIMEZONE_OPTION, defaultTimeZoneId)
 
     val partColNameMap = buildLowerCasePartColNameMap(catalogTable)
 
     val clientPrunedPartitions =
-      client.getPartitionsByFilter(rawTable, predicates).map { part =>
+      client.getPartitionsByFilter(rawTable, predicates, timeZoneId).map { part =>
         part.copy(spec = restorePartitionSpec(part.spec, partColNameMap))
       }
     prunePartitionsByFilter(catalogTable, clientPrunedPartitions, predicates, defaultTimeZoneId)
@@ -1335,7 +1332,6 @@ object HiveExternalCatalog {
   val DATASOURCE_PROVIDER = DATASOURCE_PREFIX + "provider"
   val DATASOURCE_SCHEMA = DATASOURCE_PREFIX + "schema"
   val DATASOURCE_SCHEMA_PREFIX = DATASOURCE_SCHEMA + "."
-  val DATASOURCE_SCHEMA_NUMPARTS = DATASOURCE_SCHEMA_PREFIX + "numParts"
   val DATASOURCE_SCHEMA_NUMPARTCOLS = DATASOURCE_SCHEMA_PREFIX + "numPartCols"
   val DATASOURCE_SCHEMA_NUMSORTCOLS = DATASOURCE_SCHEMA_PREFIX + "numSortCols"
   val DATASOURCE_SCHEMA_NUMBUCKETS = DATASOURCE_SCHEMA_PREFIX + "numBuckets"
@@ -1368,40 +1364,6 @@ object HiveExternalCatalog {
   val EMPTY_DATA_SCHEMA = new StructType()
     .add("col", "array<string>", nullable = true, comment = "from deserializer")
 
-  // A persisted data source table always store its schema in the catalog.
-  private def getSchemaFromTableProperties(metadata: CatalogTable): StructType = {
-    val errorMessage = "Could not read schema from the hive metastore because it is corrupted."
-    val props = metadata.properties
-    val schema = props.get(DATASOURCE_SCHEMA)
-    if (schema.isDefined) {
-      // Originally, we used `spark.sql.sources.schema` to store the schema of a data source table.
-      // After SPARK-6024, we removed this flag.
-      // Although we are not using `spark.sql.sources.schema` any more, we need to still support.
-      DataType.fromJson(schema.get).asInstanceOf[StructType]
-    } else if (props.filterKeys(_.startsWith(DATASOURCE_SCHEMA_PREFIX)).isEmpty) {
-      // If there is no schema information in table properties, it means the schema of this table
-      // was empty when saving into metastore, which is possible in older version(prior to 2.1) of
-      // Spark. We should respect it.
-      new StructType()
-    } else {
-      val numSchemaParts = props.get(DATASOURCE_SCHEMA_NUMPARTS)
-      if (numSchemaParts.isDefined) {
-        val parts = (0 until numSchemaParts.get.toInt).map { index =>
-          val part = metadata.properties.get(s"$DATASOURCE_SCHEMA_PART_PREFIX$index").orNull
-          if (part == null) {
-            throw new AnalysisException(errorMessage +
-              s" (missing part $index of the schema, ${numSchemaParts.get} parts are expected).")
-          }
-          part
-        }
-        // Stick all parts back to a single schema string.
-        DataType.fromJson(parts.mkString).asInstanceOf[StructType]
-      } else {
-        throw new AnalysisException(errorMessage)
-      }
-    }
-  }
-
   private def getColumnNamesByType(
       props: Map[String, String],
       colType: String,
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
index 16e9014340244..37a1fc0bae655 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
@@ -18,11 +18,12 @@
 package org.apache.spark.sql.hive
 
 import java.lang.reflect.{ParameterizedType, Type, WildcardType}
+import java.time.Duration
 
 import scala.collection.JavaConverters._
 
 import org.apache.hadoop.{io => hadoopIo}
-import org.apache.hadoop.hive.common.`type`.{HiveChar, HiveDecimal, HiveVarchar}
+import org.apache.hadoop.hive.common.`type`.{HiveChar, HiveDecimal, HiveIntervalDayTime, HiveIntervalYearMonth, HiveVarchar}
 import org.apache.hadoop.hive.serde2.{io => hiveIo}
 import org.apache.hadoop.hive.serde2.objectinspector.{StructField => HiveStructField, _}
 import org.apache.hadoop.hive.serde2.objectinspector.primitive._
@@ -346,6 +347,17 @@ private[hive] trait HiveInspectors {
         withNullSafe(o => getTimestampWritable(o))
       case _: TimestampObjectInspector =>
         withNullSafe(o => DateTimeUtils.toJavaTimestamp(o.asInstanceOf[Long]))
+      case _: HiveIntervalDayTimeObjectInspector  if x.preferWritable() =>
+        withNullSafe(o => getHiveIntervalDayTimeWritable(o))
+      case _: HiveIntervalDayTimeObjectInspector =>
+        withNullSafe(o => {
+          val duration = IntervalUtils.microsToDuration(o.asInstanceOf[Long])
+          new HiveIntervalDayTime(duration.getSeconds, duration.getNano)
+        })
+      case _: HiveIntervalYearMonthObjectInspector if x.preferWritable() =>
+        withNullSafe(o => getHiveIntervalYearMonthWritable(o))
+      case _: HiveIntervalYearMonthObjectInspector =>
+        withNullSafe(o => new HiveIntervalYearMonth(o.asInstanceOf[Int]))
       case _: VoidObjectInspector =>
         (_: Any) => null // always be null for void object inspector
     }
@@ -512,6 +524,13 @@ private[hive] trait HiveInspectors {
         _ => constant
       case poi: VoidObjectInspector =>
         _ => null // always be null for void object inspector
+      case dt: WritableConstantHiveIntervalDayTimeObjectInspector =>
+        val constant = dt.getWritableConstantValue.asInstanceOf[HiveIntervalDayTime]
+        _ => IntervalUtils.durationToMicros(
+          Duration.ofSeconds(constant.getTotalSeconds).plusNanos(constant.getNanos.toLong))
+      case ym: WritableConstantHiveIntervalYearMonthObjectInspector =>
+        val constant = ym.getWritableConstantValue.asInstanceOf[HiveIntervalYearMonth]
+        _ => constant.getTotalMonths
       case pi: PrimitiveObjectInspector => pi match {
         // We think HiveVarchar/HiveChar is also a String
         case hvoi: HiveVarcharObjectInspector if hvoi.preferWritable() =>
@@ -647,6 +666,42 @@ private[hive] trait HiveInspectors {
               null
             }
           }
+        case dt: HiveIntervalDayTimeObjectInspector if dt.preferWritable() =>
+          data: Any => {
+            if (data != null) {
+              val dayTime = dt.getPrimitiveWritableObject(data).getHiveIntervalDayTime
+              IntervalUtils.durationToMicros(
+                Duration.ofSeconds(dayTime.getTotalSeconds).plusNanos(dayTime.getNanos.toLong))
+            } else {
+              null
+            }
+          }
+        case dt: HiveIntervalDayTimeObjectInspector =>
+          data: Any => {
+            if (data != null) {
+              val dayTime = dt.getPrimitiveJavaObject(data)
+              IntervalUtils.durationToMicros(
+                Duration.ofSeconds(dayTime.getTotalSeconds).plusNanos(dayTime.getNanos.toLong))
+            } else {
+              null
+            }
+          }
+        case ym: HiveIntervalYearMonthObjectInspector if ym.preferWritable() =>
+          data: Any => {
+            if (data != null) {
+              ym.getPrimitiveWritableObject(data).getHiveIntervalYearMonth.getTotalMonths
+            } else {
+              null
+            }
+          }
+        case ym: HiveIntervalYearMonthObjectInspector =>
+          data: Any => {
+            if (data != null) {
+              ym.getPrimitiveJavaObject(data).getTotalMonths
+            } else {
+              null
+            }
+          }
         case _ =>
           data: Any => {
             if (data != null) {
@@ -694,7 +749,7 @@ private[hive] trait HiveInspectors {
         }
         data: Any => {
           if (data != null) {
-            InternalRow.fromSeq(unwrappers.map(_(data)))
+            InternalRow.fromSeq(unwrappers.map(_(data)).toSeq)
           } else {
             null
           }
@@ -785,6 +840,10 @@ private[hive] trait HiveInspectors {
     case BinaryType => PrimitiveObjectInspectorFactory.javaByteArrayObjectInspector
     case DateType => PrimitiveObjectInspectorFactory.javaDateObjectInspector
     case TimestampType => PrimitiveObjectInspectorFactory.javaTimestampObjectInspector
+    case DayTimeIntervalType =>
+      PrimitiveObjectInspectorFactory.javaHiveIntervalDayTimeObjectInspector
+    case YearMonthIntervalType =>
+      PrimitiveObjectInspectorFactory.javaHiveIntervalYearMonthObjectInspector
     // TODO decimal precision?
     case DecimalType() => PrimitiveObjectInspectorFactory.javaHiveDecimalObjectInspector
     case StructType(fields) =>
@@ -830,6 +889,10 @@ private[hive] trait HiveInspectors {
       getDecimalWritableConstantObjectInspector(value)
     case Literal(_, NullType) =>
       getPrimitiveNullWritableConstantObjectInspector
+    case Literal(_, DayTimeIntervalType) =>
+      getHiveIntervalDayTimeWritableConstantObjectInspector
+    case Literal(_, YearMonthIntervalType) =>
+      getHiveIntervalYearMonthWritableConstantObjectInspector
     case Literal(value, ArrayType(dt, _)) =>
       val listObjectInspector = toInspector(dt)
       if (value == null) {
@@ -872,7 +935,7 @@ private[hive] trait HiveInspectors {
       StructType(s.getAllStructFieldRefs.asScala.map(f =>
         types.StructField(
           f.getFieldName, inspectorToDataType(f.getFieldObjectInspector), nullable = true)
-      ))
+      ).toSeq)
     case l: ListObjectInspector => ArrayType(inspectorToDataType(l.getListElementObjectInspector))
     case m: MapObjectInspector =>
       MapType(
@@ -906,6 +969,10 @@ private[hive] trait HiveInspectors {
     case _: JavaDateObjectInspector => DateType
     case _: WritableTimestampObjectInspector => TimestampType
     case _: JavaTimestampObjectInspector => TimestampType
+    case _: WritableHiveIntervalDayTimeObjectInspector => DayTimeIntervalType
+    case _: JavaHiveIntervalDayTimeObjectInspector => DayTimeIntervalType
+    case _: WritableHiveIntervalYearMonthObjectInspector => YearMonthIntervalType
+    case _: JavaHiveIntervalYearMonthObjectInspector => YearMonthIntervalType
     case _: WritableVoidObjectInspector => NullType
     case _: JavaVoidObjectInspector => NullType
   }
@@ -967,6 +1034,14 @@ private[hive] trait HiveInspectors {
     PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
       TypeInfoFactory.voidTypeInfo, null)
 
+  private def getHiveIntervalDayTimeWritableConstantObjectInspector: ObjectInspector =
+    PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
+      TypeInfoFactory.intervalDayTimeTypeInfo, null)
+
+  private def getHiveIntervalYearMonthWritableConstantObjectInspector: ObjectInspector =
+    PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
+      TypeInfoFactory.intervalYearMonthTypeInfo, null)
+
   private def getStringWritable(value: Any): hadoopIo.Text =
     if (value == null) null else new hadoopIo.Text(value.asInstanceOf[UTF8String].getBytes)
 
@@ -1024,6 +1099,22 @@ private[hive] trait HiveInspectors {
       new hiveIo.TimestampWritable(DateTimeUtils.toJavaTimestamp(value.asInstanceOf[Long]))
     }
 
+  private def getHiveIntervalDayTimeWritable(value: Any): hiveIo.HiveIntervalDayTimeWritable =
+    if (value == null) {
+      null
+    } else {
+      val duration = IntervalUtils.microsToDuration(value.asInstanceOf[Long])
+      new hiveIo.HiveIntervalDayTimeWritable(
+        new HiveIntervalDayTime(duration.getSeconds, duration.getNano))
+    }
+
+  private def getHiveIntervalYearMonthWritable(value: Any): hiveIo.HiveIntervalYearMonthWritable =
+    if (value == null) {
+      null
+    } else {
+      new hiveIo.HiveIntervalYearMonthWritable(new HiveIntervalYearMonth(value.asInstanceOf[Int]))
+    }
+
   private def getDecimalWritable(value: Any): hiveIo.HiveDecimalWritable =
     if (value == null) {
       null
@@ -1039,6 +1130,7 @@ private[hive] trait HiveInspectors {
 
     private def decimalTypeInfo(decimalType: DecimalType): TypeInfo = decimalType match {
       case DecimalType.Fixed(precision, scale) => new DecimalTypeInfo(precision, scale)
+      case dt => throw new AnalysisException(s"${dt.catalogString} is not supported.")
     }
 
     def toTypeInfo: TypeInfo = dt match {
@@ -1046,8 +1138,8 @@ private[hive] trait HiveInspectors {
         getListTypeInfo(elemType.toTypeInfo)
       case StructType(fields) =>
         getStructTypeInfo(
-          java.util.Arrays.asList(fields.map(_.name) : _*),
-          java.util.Arrays.asList(fields.map(_.dataType.toTypeInfo) : _*))
+          java.util.Arrays.asList(fields.map(_.name): _*),
+          java.util.Arrays.asList(fields.map(_.dataType.toTypeInfo): _*))
       case MapType(keyType, valueType, _) =>
         getMapTypeInfo(keyType.toTypeInfo, valueType.toTypeInfo)
       case BinaryType => binaryTypeInfo
@@ -1063,6 +1155,11 @@ private[hive] trait HiveInspectors {
       case DateType => dateTypeInfo
       case TimestampType => timestampTypeInfo
       case NullType => voidTypeInfo
+      case DayTimeIntervalType => intervalDayTimeTypeInfo
+      case YearMonthIntervalType => intervalYearMonthTypeInfo
+      case dt =>
+        throw new AnalysisException(
+          s"${dt.catalogString} cannot be converted to Hive TypeInfo")
     }
   }
 }
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
index 2981e391c0439..e02589e5cad00 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
@@ -131,12 +131,12 @@ private[hive] class HiveMetastoreCatalog(sparkSession: SparkSession) extends Log
     // Consider table and storage properties. For properties existing in both sides, storage
     // properties will supersede table properties.
     if (serde.contains("parquet")) {
-      val options = relation.tableMeta.properties.filterKeys(isParquetProperty) ++
+      val options = relation.tableMeta.properties.filterKeys(isParquetProperty).toMap ++
         relation.tableMeta.storage.properties + (ParquetOptions.MERGE_SCHEMA ->
         SQLConf.get.getConf(HiveUtils.CONVERT_METASTORE_PARQUET_WITH_SCHEMA_MERGING).toString)
         convertToLogicalRelation(relation, options, classOf[ParquetFileFormat], "parquet")
     } else {
-      val options = relation.tableMeta.properties.filterKeys(isOrcProperty) ++
+      val options = relation.tableMeta.properties.filterKeys(isOrcProperty).toMap ++
         relation.tableMeta.storage.properties
       if (SQLConf.get.getConf(SQLConf.ORC_IMPLEMENTATION) == "native") {
         convertToLogicalRelation(
@@ -332,7 +332,7 @@ private[hive] object HiveMetastoreCatalog {
       metastoreSchema: StructType,
       inferredSchema: StructType): StructType = try {
     // scalastyle:off caselocale
-    // Find any nullable fields in mestastore schema that are missing from the inferred schema.
+    // Find any nullable fields in metastore schema that are missing from the inferred schema.
     val metastoreFields = metastoreSchema.map(f => f.name.toLowerCase -> f).toMap
     val missingNullables = metastoreFields
       .filterKeys(!inferredSchema.map(_.name.toLowerCase).contains(_))
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionCatalog.scala
index bc7760c982aab..7cbaa8a0b6477 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionCatalog.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionCatalog.scala
@@ -29,12 +29,11 @@ import org.apache.hadoop.hive.ql.udf.generic.{AbstractGenericUDAFResolver, Gener
 
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.FunctionIdentifier
-import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
-import org.apache.spark.sql.catalyst.catalog.{CatalogFunction, ExternalCatalog, FunctionResourceLoader, GlobalTempViewManager, SessionCatalog}
+import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TableFunctionRegistry}
+import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.expressions.{Cast, Expression}
 import org.apache.spark.sql.catalyst.parser.ParserInterface
 import org.apache.spark.sql.hive.HiveShim.HiveFunctionWrapper
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{DecimalType, DoubleType}
 import org.apache.spark.util.Utils
 
@@ -44,7 +43,7 @@ private[sql] class HiveSessionCatalog(
     globalTempViewManagerBuilder: () => GlobalTempViewManager,
     val metastoreCatalog: HiveMetastoreCatalog,
     functionRegistry: FunctionRegistry,
-    conf: SQLConf,
+    tableFunctionRegistry: TableFunctionRegistry,
     hadoopConf: Configuration,
     parser: ParserInterface,
     functionResourceLoader: FunctionResourceLoader)
@@ -52,11 +51,54 @@ private[sql] class HiveSessionCatalog(
       externalCatalogBuilder,
       globalTempViewManagerBuilder,
       functionRegistry,
-      conf,
+      tableFunctionRegistry,
       hadoopConf,
       parser,
       functionResourceLoader) {
 
+  private def makeHiveFunctionExpression(
+      name: String,
+      clazz: Class[_],
+      input: Seq[Expression]): Expression = {
+    var udfExpr: Option[Expression] = None
+    try {
+      // When we instantiate hive UDF wrapper class, we may throw exception if the input
+      // expressions don't satisfy the hive UDF, such as type mismatch, input number
+      // mismatch, etc. Here we catch the exception and throw AnalysisException instead.
+      if (classOf[UDF].isAssignableFrom(clazz)) {
+        udfExpr = Some(HiveSimpleUDF(name, new HiveFunctionWrapper(clazz.getName), input))
+        udfExpr.get.dataType // Force it to check input data types.
+      } else if (classOf[GenericUDF].isAssignableFrom(clazz)) {
+        udfExpr = Some(HiveGenericUDF(name, new HiveFunctionWrapper(clazz.getName), input))
+        udfExpr.get.dataType // Force it to check input data types.
+      } else if (classOf[AbstractGenericUDAFResolver].isAssignableFrom(clazz)) {
+        udfExpr = Some(HiveUDAFFunction(name, new HiveFunctionWrapper(clazz.getName), input))
+        udfExpr.get.dataType // Force it to check input data types.
+      } else if (classOf[UDAF].isAssignableFrom(clazz)) {
+        udfExpr = Some(HiveUDAFFunction(
+          name,
+          new HiveFunctionWrapper(clazz.getName),
+          input,
+          isUDAFBridgeRequired = true))
+        udfExpr.get.dataType // Force it to check input data types.
+      } else if (classOf[GenericUDTF].isAssignableFrom(clazz)) {
+        udfExpr = Some(HiveGenericUDTF(name, new HiveFunctionWrapper(clazz.getName), input))
+        // Force it to check data types.
+        udfExpr.get.asInstanceOf[HiveGenericUDTF].elementSchema
+      }
+    } catch {
+      case NonFatal(e) =>
+        val errorMsg = s"No handler for UDF/UDAF/UDTF '${clazz.getCanonicalName}': $e"
+        val analysisException = new AnalysisException(errorMsg)
+        analysisException.setStackTrace(e.getStackTrace)
+        throw analysisException
+    }
+    udfExpr.getOrElse {
+      throw new InvalidUDFClassException(
+        s"No handler for UDF/UDAF/UDTF '${clazz.getCanonicalName}'")
+    }
+  }
+
   /**
    * Constructs a [[Expression]] based on the provided class that represents a function.
    *
@@ -69,49 +111,14 @@ private[sql] class HiveSessionCatalog(
     // Current thread context classloader may not be the one loaded the class. Need to switch
     // context classloader to initialize instance properly.
     Utils.withContextClassLoader(clazz.getClassLoader) {
-      Try(super.makeFunctionExpression(name, clazz, input)).getOrElse {
-        var udfExpr: Option[Expression] = None
-        try {
-          // When we instantiate hive UDF wrapper class, we may throw exception if the input
-          // expressions don't satisfy the hive UDF, such as type mismatch, input number
-          // mismatch, etc. Here we catch the exception and throw AnalysisException instead.
-          if (classOf[UDF].isAssignableFrom(clazz)) {
-            udfExpr = Some(HiveSimpleUDF(name, new HiveFunctionWrapper(clazz.getName), input))
-            udfExpr.get.dataType // Force it to check input data types.
-          } else if (classOf[GenericUDF].isAssignableFrom(clazz)) {
-            udfExpr = Some(HiveGenericUDF(name, new HiveFunctionWrapper(clazz.getName), input))
-            udfExpr.get.dataType // Force it to check input data types.
-          } else if (classOf[AbstractGenericUDAFResolver].isAssignableFrom(clazz)) {
-            udfExpr = Some(HiveUDAFFunction(name, new HiveFunctionWrapper(clazz.getName), input))
-            udfExpr.get.dataType // Force it to check input data types.
-          } else if (classOf[UDAF].isAssignableFrom(clazz)) {
-            udfExpr = Some(HiveUDAFFunction(
-              name,
-              new HiveFunctionWrapper(clazz.getName),
-              input,
-              isUDAFBridgeRequired = true))
-            udfExpr.get.dataType // Force it to check input data types.
-          } else if (classOf[GenericUDTF].isAssignableFrom(clazz)) {
-            udfExpr = Some(HiveGenericUDTF(name, new HiveFunctionWrapper(clazz.getName), input))
-            udfExpr.get.asInstanceOf[HiveGenericUDTF].elementSchema // Force it to check data types.
-          }
-        } catch {
-          case NonFatal(e) =>
-            val noHandlerMsg = s"No handler for UDF/UDAF/UDTF '${clazz.getCanonicalName}': $e"
-            val errorMsg =
-              if (classOf[GenericUDTF].isAssignableFrom(clazz)) {
-                s"$noHandlerMsg\nPlease make sure your function overrides " +
-                  "`public StructObjectInspector initialize(ObjectInspector[] args)`."
-              } else {
-                noHandlerMsg
-              }
-            val analysisException = new AnalysisException(errorMsg)
-            analysisException.setStackTrace(e.getStackTrace)
-            throw analysisException
-        }
-        udfExpr.getOrElse {
-          throw new AnalysisException(s"No handler for UDF/UDAF/UDTF '${clazz.getCanonicalName}'")
-        }
+      try {
+        super.makeFunctionExpression(name, clazz, input)
+      } catch {
+        // If `super.makeFunctionExpression` throw `InvalidUDFClassException`, we construct
+        // Hive UDF/UDAF/UDTF with function definition. Otherwise, we just throw it earlier.
+        case _: InvalidUDFClassException =>
+          makeHiveFunctionExpression(name, clazz, input)
+        case NonFatal(e) => throw e
       }
     }
   }
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
index 64726755237a6..7bf9b283de99c 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
@@ -17,27 +17,29 @@
 
 package org.apache.spark.sql.hive
 
-import org.apache.spark.annotation.Unstable
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.analysis.{Analyzer, ResolveSessionCatalog}
 import org.apache.spark.sql.catalyst.catalog.ExternalCatalogWithListener
-import org.apache.spark.sql.catalyst.optimizer.Optimizer
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.execution.{SparkOptimizer, SparkPlanner}
+import org.apache.spark.sql.execution.SparkPlanner
+import org.apache.spark.sql.execution.aggregate.ResolveEncodersInScalaAgg
 import org.apache.spark.sql.execution.analysis.DetectAmbiguousSelfJoin
 import org.apache.spark.sql.execution.command.CommandCheck
 import org.apache.spark.sql.execution.datasources._
 import org.apache.spark.sql.execution.datasources.v2.TableCapabilityCheck
+import org.apache.spark.sql.execution.streaming.ResolveWriteToStream
 import org.apache.spark.sql.hive.client.HiveClient
 import org.apache.spark.sql.hive.execution.PruneHiveTablePartitions
 import org.apache.spark.sql.internal.{BaseSessionStateBuilder, SessionResourceLoader, SessionState}
+import org.apache.spark.util.Utils
 
 /**
  * Builder that produces a Hive-aware `SessionState`.
  */
-@Unstable
-class HiveSessionStateBuilder(session: SparkSession, parentState: Option[SessionState] = None)
+class HiveSessionStateBuilder(
+    session: SparkSession,
+    parentState: Option[SessionState])
   extends BaseSessionStateBuilder(session, parentState) {
 
   private def externalCatalog: ExternalCatalogWithListener = session.sharedState.externalCatalog
@@ -59,7 +61,7 @@ class HiveSessionStateBuilder(session: SparkSession, parentState: Option[Session
       () => session.sharedState.globalTempViewManager,
       new HiveMetastoreCatalog(session),
       functionRegistry,
-      conf,
+      tableFunctionRegistry,
       SessionState.newHadoopConf(session.sparkContext.hadoopConfiguration, conf),
       sqlParser,
       resourceLoader)
@@ -70,23 +72,24 @@ class HiveSessionStateBuilder(session: SparkSession, parentState: Option[Session
   /**
    * A logical query plan `Analyzer` with rules specific to Hive.
    */
-  override protected def analyzer: Analyzer = new Analyzer(catalogManager, conf) {
+  override protected def analyzer: Analyzer = new Analyzer(catalogManager) {
     override val extendedResolutionRules: Seq[Rule[LogicalPlan]] =
       new ResolveHiveSerdeTable(session) +:
         new FindDataSourceTable(session) +:
         new ResolveSQLOnFile(session) +:
         new FallBackFileSourceV2(session) +:
-        new ResolveSessionCatalog(
-          catalogManager, conf, catalog.isTempView, catalog.isTempFunction) +:
+        ResolveEncodersInScalaAgg +:
+        new ResolveSessionCatalog(catalogManager) +:
+        ResolveWriteToStream +:
         customResolutionRules
 
     override val postHocResolutionRules: Seq[Rule[LogicalPlan]] =
-      new DetectAmbiguousSelfJoin(conf) +:
+      DetectAmbiguousSelfJoin +:
         new DetermineTableStats(session) +:
-        RelationConversions(conf, catalog) +:
+        RelationConversions(catalog) +:
         PreprocessTableCreation(session) +:
-        PreprocessTableInsertion(conf) +:
-        DataSourceAnalysis(conf) +:
+        PreprocessTableInsertion +:
+        DataSourceAnalysis +:
         HiveAnalysis +:
         customPostHocResolutionRules
 
@@ -94,7 +97,7 @@ class HiveSessionStateBuilder(session: SparkSession, parentState: Option[Session
       PreWriteCheck +:
         PreReadCheck +:
         TableCapabilityCheck +:
-        CommandCheck(conf) +:
+        CommandCheck +:
         customCheckRules
   }
 
@@ -105,11 +108,12 @@ class HiveSessionStateBuilder(session: SparkSession, parentState: Option[Session
    * Planner that takes into account Hive-specific strategies.
    */
   override protected def planner: SparkPlanner = {
-    new SparkPlanner(session, conf, experimentalMethods) with HiveStrategies {
+    new SparkPlanner(session, experimentalMethods) with HiveStrategies {
       override val sparkSession: SparkSession = session
 
       override def extraPlanningStrategies: Seq[Strategy] =
-        super.extraPlanningStrategies ++ customPlanningStrategies ++ Seq(HiveTableScans, Scripts)
+        super.extraPlanningStrategies ++ customPlanningStrategies ++
+          Seq(HiveTableScans, HiveScripts)
     }
   }
 
@@ -122,7 +126,10 @@ class HiveSessionResourceLoader(
   extends SessionResourceLoader(session) {
   private lazy val client = clientBuilder()
   override def addJar(path: String): Unit = {
-    client.addJar(path)
-    super.addJar(path)
+    val uri = Utils.resolveURI(path)
+    resolveJars(uri).foreach { p =>
+      client.addJar(p)
+      super.addJar(p)
+    }
   }
 }
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala
index 04a6a8f8aa9a5..3a53a2a8dadd8 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala
@@ -17,18 +17,16 @@
 
 package org.apache.spark.sql.hive
 
-import java.io.{InputStream, OutputStream}
-import java.lang.reflect.Method
 import java.rmi.server.UID
 
 import scala.collection.JavaConverters._
 import scala.language.implicitConversions
-import scala.reflect.ClassTag
 
 import com.google.common.base.Objects
 import org.apache.avro.Schema
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.Path
+import org.apache.hadoop.hive.ql.exec.SerializationUtilities
 import org.apache.hadoop.hive.ql.exec.UDF
 import org.apache.hadoop.hive.ql.plan.{FileSinkDesc, TableDesc}
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFMacro
@@ -148,60 +146,12 @@ private[hive] object HiveShim {
       case _ => false
     }
 
-    private lazy val serUtilClass =
-      Utils.classForName("org.apache.hadoop.hive.ql.exec.SerializationUtilities")
-    private lazy val utilClass = Utils.classForName("org.apache.hadoop.hive.ql.exec.Utilities")
-    private val deserializeMethodName = "deserializeObjectByKryo"
-    private val serializeMethodName = "serializeObjectByKryo"
-
-    private def findMethod(klass: Class[_], name: String, args: Class[_]*): Method = {
-      val method = klass.getDeclaredMethod(name, args: _*)
-      method.setAccessible(true)
-      method
-    }
-
     def deserializePlan[UDFType](is: java.io.InputStream, clazz: Class[_]): UDFType = {
-      if (HiveUtils.isHive23) {
-        val borrowKryo = serUtilClass.getMethod("borrowKryo")
-        val kryo = borrowKryo.invoke(serUtilClass)
-        val deserializeObjectByKryo = findMethod(serUtilClass, deserializeMethodName,
-          kryo.getClass.getSuperclass, classOf[InputStream], classOf[Class[_]])
-        try {
-          deserializeObjectByKryo.invoke(null, kryo, is, clazz).asInstanceOf[UDFType]
-        } finally {
-          serUtilClass.getMethod("releaseKryo", kryo.getClass.getSuperclass).invoke(null, kryo)
-        }
-      } else {
-        val runtimeSerializationKryo = utilClass.getField("runtimeSerializationKryo")
-        val threadLocalValue = runtimeSerializationKryo.get(utilClass)
-        val getMethod = threadLocalValue.getClass.getMethod("get")
-        val kryo = getMethod.invoke(threadLocalValue)
-        val deserializeObjectByKryo = findMethod(utilClass, deserializeMethodName,
-          kryo.getClass, classOf[InputStream], classOf[Class[_]])
-        deserializeObjectByKryo.invoke(null, kryo, is, clazz).asInstanceOf[UDFType]
-      }
+      SerializationUtilities.deserializePlan(is, clazz).asInstanceOf[UDFType]
     }
 
     def serializePlan(function: AnyRef, out: java.io.OutputStream): Unit = {
-      if (HiveUtils.isHive23) {
-        val borrowKryo = serUtilClass.getMethod("borrowKryo")
-        val kryo = borrowKryo.invoke(serUtilClass)
-        val serializeObjectByKryo = findMethod(serUtilClass, serializeMethodName,
-          kryo.getClass.getSuperclass, classOf[Object], classOf[OutputStream])
-        try {
-          serializeObjectByKryo.invoke(null, kryo, function, out)
-        } finally {
-          serUtilClass.getMethod("releaseKryo", kryo.getClass.getSuperclass).invoke(null, kryo)
-        }
-      } else {
-        val runtimeSerializationKryo = utilClass.getField("runtimeSerializationKryo")
-        val threadLocalValue = runtimeSerializationKryo.get(utilClass)
-        val getMethod = threadLocalValue.getClass.getMethod("get")
-        val kryo = getMethod.invoke(threadLocalValue)
-        val serializeObjectByKryo = findMethod(utilClass, serializeMethodName,
-          kryo.getClass, classOf[Object], classOf[OutputStream])
-        serializeObjectByKryo.invoke(null, kryo, function, out)
-      }
+      SerializationUtilities.serializePlan(function, out)
     }
 
     def writeExternal(out: java.io.ObjectOutput): Unit = {
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala
index b9c98f4ea15e9..2bca463217f78 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala
@@ -28,11 +28,13 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.planning._
 import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoDir, InsertIntoStatement, LogicalPlan, ScriptTransformation, Statistics}
 import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.connector.catalog.CatalogV2Util.assertNoNullTypeInSchema
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.execution.command.{CreateTableCommand, DDLUtils}
-import org.apache.spark.sql.execution.datasources.CreateTable
+import org.apache.spark.sql.execution.datasources.{CreateTable, DataSourceStrategy}
 import org.apache.spark.sql.hive.execution._
-import org.apache.spark.sql.internal.{HiveSerDe, SQLConf}
+import org.apache.spark.sql.hive.execution.HiveScriptTransformationExec
+import org.apache.spark.sql.internal.HiveSerDe
 
 
 /**
@@ -48,7 +50,7 @@ class ResolveHiveSerdeTable(session: SparkSession) extends Rule[LogicalPlan] {
         throw new AnalysisException("Creating bucketed Hive serde table is not supported yet.")
       }
 
-      val defaultStorage = HiveSerDe.getDefaultStorage(session.sessionState.conf)
+      val defaultStorage = HiveSerDe.getDefaultStorage(conf)
       val options = new HiveOptions(table.storage.properties)
 
       val fileStorage = if (options.fileFormat.isDefined) {
@@ -115,7 +117,6 @@ class DetermineTableStats(session: SparkSession) extends Rule[LogicalPlan] {
   private def hiveTableWithStats(relation: HiveTableRelation): HiveTableRelation = {
     val table = relation.tableMeta
     val partitionCols = relation.partitionCols
-    val conf = session.sessionState.conf
     // For partitioned tables, the partition directory may be outside of the table directory.
     // Which is expensive to get table size. Please see how we implemented it in the AnalyzeTable.
     val sizeInBytes = if (conf.fallBackToHdfsForStatsEnabled && partitionCols.isEmpty) {
@@ -144,7 +145,7 @@ class DetermineTableStats(session: SparkSession) extends Rule[LogicalPlan] {
 
     // handles InsertIntoStatement specially as the table in InsertIntoStatement is not added in its
     // children, hence not matched directly by previous HiveTableRelation case.
-    case i @ InsertIntoStatement(relation: HiveTableRelation, _, _, _, _)
+    case i @ InsertIntoStatement(relation: HiveTableRelation, _, _, _, _, _)
       if DDLUtils.isHiveTable(relation.tableMeta) && relation.tableMeta.stats.isEmpty =>
       i.copy(table = hiveTableWithStats(relation))
   }
@@ -158,7 +159,8 @@ class DetermineTableStats(session: SparkSession) extends Rule[LogicalPlan] {
  */
 object HiveAnalysis extends Rule[LogicalPlan] {
   override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
-    case InsertIntoStatement(r: HiveTableRelation, partSpec, query, overwrite, ifPartitionNotExists)
+    case InsertIntoStatement(
+        r: HiveTableRelation, partSpec, _, query, overwrite, ifPartitionNotExists)
         if DDLUtils.isHiveTable(r.tableMeta) =>
       InsertIntoHiveTable(r.tableMeta, partSpec, query, overwrite,
         ifPartitionNotExists, query.output.map(_.name))
@@ -189,7 +191,6 @@ object HiveAnalysis extends Rule[LogicalPlan] {
  * `PreprocessTableCreation`, `PreprocessTableInsertion`, `DataSourceAnalysis` and `HiveAnalysis`.
  */
 case class RelationConversions(
-    conf: SQLConf,
     sessionCatalog: HiveSessionCatalog) extends Rule[LogicalPlan] {
   private def isConvertible(relation: HiveTableRelation): Boolean = {
     isConvertible(relation.tableMeta)
@@ -197,8 +198,8 @@ case class RelationConversions(
 
   private def isConvertible(tableMeta: CatalogTable): Boolean = {
     val serde = tableMeta.storage.serde.getOrElse("").toLowerCase(Locale.ROOT)
-    serde.contains("parquet") && SQLConf.get.getConf(HiveUtils.CONVERT_METASTORE_PARQUET) ||
-      serde.contains("orc") && SQLConf.get.getConf(HiveUtils.CONVERT_METASTORE_ORC)
+    serde.contains("parquet") && conf.getConf(HiveUtils.CONVERT_METASTORE_PARQUET) ||
+      serde.contains("orc") && conf.getConf(HiveUtils.CONVERT_METASTORE_ORC)
   }
 
   private val metastoreCatalog = sessionCatalog.metastoreCatalog
@@ -207,11 +208,11 @@ case class RelationConversions(
     plan resolveOperators {
       // Write path
       case InsertIntoStatement(
-           r: HiveTableRelation, partition, query, overwrite, ifPartitionNotExists)
+          r: HiveTableRelation, partition, cols, query, overwrite, ifPartitionNotExists)
           if query.resolved && DDLUtils.isHiveTable(r.tableMeta) &&
-            (!r.isPartitioned || SQLConf.get.getConf(HiveUtils.CONVERT_INSERTING_PARTITIONED_TABLE))
+            (!r.isPartitioned || conf.getConf(HiveUtils.CONVERT_INSERTING_PARTITIONED_TABLE))
             && isConvertible(r) =>
-        InsertIntoStatement(metastoreCatalog.convert(r), partition,
+        InsertIntoStatement(metastoreCatalog.convert(r), partition, cols,
           query, overwrite, ifPartitionNotExists)
 
       // Read path
@@ -221,10 +222,13 @@ case class RelationConversions(
 
       // CTAS
       case CreateTable(tableDesc, mode, Some(query))
-          if DDLUtils.isHiveTable(tableDesc) && tableDesc.partitionColumnNames.isEmpty &&
-            isConvertible(tableDesc) && SQLConf.get.getConf(HiveUtils.CONVERT_METASTORE_CTAS) =>
+          if query.resolved && DDLUtils.isHiveTable(tableDesc) &&
+            tableDesc.partitionColumnNames.isEmpty && isConvertible(tableDesc) &&
+            conf.getConf(HiveUtils.CONVERT_METASTORE_CTAS) =>
         // validation is required to be done here before relation conversion.
         DDLUtils.checkDataColNames(tableDesc.copy(schema = query.schema))
+        // This is for CREATE TABLE .. STORED AS PARQUET/ORC AS SELECT null
+        assertNoNullTypeInSchema(query.schema)
         OptimizedCreateHiveTableAsSelectCommand(
           tableDesc, query, query.output.map(_.name), mode)
     }
@@ -237,11 +241,11 @@ private[hive] trait HiveStrategies {
 
   val sparkSession: SparkSession
 
-  object Scripts extends Strategy {
+  object HiveScripts extends Strategy {
     def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match {
-      case ScriptTransformation(input, script, output, child, ioschema) =>
-        val hiveIoSchema = HiveScriptIOSchema(ioschema)
-        ScriptTransformationExec(input, script, output, planLater(child), hiveIoSchema) :: Nil
+      case ScriptTransformation(script, output, child, ioschema) =>
+        val hiveIoSchema = ScriptTransformationIOSchema(ioschema)
+        HiveScriptTransformationExec(script, output, planLater(child), hiveIoSchema) :: Nil
       case _ => Nil
     }
   }
@@ -252,20 +256,21 @@ private[hive] trait HiveStrategies {
    */
   object HiveTableScans extends Strategy {
     def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match {
-      case ScanOperation(projectList, predicates, relation: HiveTableRelation) =>
+      case ScanOperation(projectList, filters, relation: HiveTableRelation) =>
         // Filter out all predicates that only deal with partition keys, these are given to the
         // hive table scan operator to be used for partition pruning.
         val partitionKeyIds = AttributeSet(relation.partitionCols)
-        val (pruningPredicates, otherPredicates) = predicates.partition { predicate =>
-          !predicate.references.isEmpty &&
-          predicate.references.subsetOf(partitionKeyIds)
-        }
+        val normalizedFilters = DataSourceStrategy.normalizeExprs(
+          filters.filter(_.deterministic), relation.output)
+
+        val partitionKeyFilters = DataSourceStrategy.getPushedDownFilters(relation.partitionCols,
+          normalizedFilters)
 
         pruneFilterProject(
           projectList,
-          otherPredicates,
+          filters.filter(f => f.references.isEmpty || !f.references.subsetOf(partitionKeyIds)),
           identity[Seq[Expression]],
-          HiveTableScanExec(_, relation, pruningPredicates)(sparkSession)) :: Nil
+          HiveTableScanExec(_, relation, partitionKeyFilters.toSeq)(sparkSession)) :: Nil
       case _ =>
         Nil
     }
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
index 04caf57efdc74..8c2f02cfa1185 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
@@ -24,7 +24,7 @@ import java.util.concurrent.TimeUnit
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable.HashMap
-import scala.language.implicitConversions
+import scala.util.Try
 
 import org.apache.commons.lang3.{JavaVersion, SystemUtils}
 import org.apache.hadoop.conf.Configuration
@@ -34,53 +34,54 @@ import org.apache.hadoop.hive.ql.session.SessionState
 import org.apache.hadoop.util.VersionInfo
 import org.apache.hive.common.util.HiveVersionInfo
 
-import org.apache.spark.{SparkConf, SparkContext}
+import org.apache.spark.SparkConf
 import org.apache.spark.deploy.SparkHadoopUtil
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.catalog.CatalogTable
 import org.apache.spark.sql.execution.command.DDLUtils
+import org.apache.spark.sql.execution.datasources.DataSource
 import org.apache.spark.sql.hive.client._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf._
-import org.apache.spark.sql.internal.StaticSQLConf.{CATALOG_IMPLEMENTATION, WAREHOUSE_PATH}
+import org.apache.spark.sql.internal.StaticSQLConf.WAREHOUSE_PATH
 import org.apache.spark.sql.types._
 import org.apache.spark.util.{ChildFirstURLClassLoader, Utils}
 
 
 private[spark] object HiveUtils extends Logging {
 
-  def withHiveExternalCatalog(sc: SparkContext): SparkContext = {
-    sc.conf.set(CATALOG_IMPLEMENTATION.key, "hive")
-    sc
-  }
+  /** The version of hive used internally by Spark SQL. */
+  val builtinHiveVersion: String = HiveVersionInfo.getVersion
 
-  private val hiveVersion = HiveVersionInfo.getVersion
-  val isHive23: Boolean = hiveVersion.startsWith("2.3")
+  val BUILTIN_HIVE_VERSION = buildStaticConf("spark.sql.hive.version")
+    .doc("The compiled, a.k.a, builtin Hive version of the Spark distribution bundled with." +
+        " Note that, this a read-only conf and only used to report the built-in hive version." +
+        " If you want a different metastore client for Spark to call, please refer to" +
+        " spark.sql.hive.metastore.version.")
+    .version("1.1.1")
+    .stringConf
+    .checkValue(_ == builtinHiveVersion,
+      "The builtin Hive version is read-only, please use spark.sql.hive.metastore.version")
+    .createWithDefault(builtinHiveVersion)
 
-  /** The version of hive used internally by Spark SQL. */
-  val builtinHiveVersion: String = if (isHive23) hiveVersion else "1.2.1"
+  private def isCompatibleHiveVersion(hiveVersionStr: String): Boolean = {
+    Try { IsolatedClientLoader.hiveVersion(hiveVersionStr) }.isSuccess
+  }
 
   val HIVE_METASTORE_VERSION = buildStaticConf("spark.sql.hive.metastore.version")
     .doc("Version of the Hive metastore. Available options are " +
-        "<code>0.12.0</code> through <code>2.3.7</code> and " +
+        "<code>0.12.0</code> through <code>2.3.8</code> and " +
         "<code>3.0.0</code> through <code>3.1.2</code>.")
     .version("1.4.0")
     .stringConf
+    .checkValue(isCompatibleHiveVersion, "Unsupported Hive Metastore version")
     .createWithDefault(builtinHiveVersion)
 
-  // A fake config which is only here for backward compatibility reasons. This config has no effect
-  // to Spark, just for reporting the builtin Hive version of Spark to existing applications that
-  // already rely on this config.
-  val FAKE_HIVE_VERSION = buildConf("spark.sql.hive.version")
-    .doc(s"deprecated, please use ${HIVE_METASTORE_VERSION.key} to get the Hive version in Spark.")
-    .version("1.1.1")
-    .fallbackConf(HIVE_METASTORE_VERSION)
-
   val HIVE_METASTORE_JARS = buildStaticConf("spark.sql.hive.metastore.jars")
     .doc(s"""
       | Location of the jars that should be used to instantiate the HiveMetastoreClient.
-      | This property can be one of three options: "
+      | This property can be one of four options:
       | 1. "builtin"
       |   Use Hive ${builtinHiveVersion}, which is bundled with the Spark assembly when
       |   <code>-Phive</code> is enabled. When this option is chosen,
@@ -88,12 +89,35 @@ private[spark] object HiveUtils extends Logging {
       |   <code>${builtinHiveVersion}</code> or not defined.
       | 2. "maven"
       |   Use Hive jars of specified version downloaded from Maven repositories.
-      | 3. A classpath in the standard format for both Hive and Hadoop.
+      | 3. "path"
+      |   Use Hive jars configured by `spark.sql.hive.metastore.jars.path`
+      |   in comma separated format. Support both local or remote paths.The provided jars
+      |   should be the same version as ${HIVE_METASTORE_VERSION}.
+      | 4. A classpath in the standard format for both Hive and Hadoop. The provided jars
+      |   should be the same version as ${HIVE_METASTORE_VERSION}.
       """.stripMargin)
     .version("1.4.0")
     .stringConf
     .createWithDefault("builtin")
 
+  val HIVE_METASTORE_JARS_PATH = buildStaticConf("spark.sql.hive.metastore.jars.path")
+    .doc(s"""
+      | Comma-separated paths of the jars that used to instantiate the HiveMetastoreClient.
+      | This configuration is useful only when `{$HIVE_METASTORE_JARS.key}` is set as `path`.
+      | The paths can be any of the following format:
+      | 1. file://path/to/jar/foo.jar
+      | 2. hdfs://nameservice/path/to/jar/foo.jar
+      | 3. /path/to/jar/ (path without URI scheme follow conf `fs.defaultFS`'s URI schema)
+      | 4. [http/https/ftp]://path/to/jar/foo.jar
+      | Note that 1, 2, and 3 support wildcard. For example:
+      | 1. file://path/to/jar/*,file://path2/to/jar/*/*.jar
+      | 2. hdfs://nameservice/path/to/jar/*,hdfs://nameservice2/path/to/jar/*/*.jar
+      """.stripMargin)
+    .version("3.1.0")
+    .stringConf
+    .toSequence
+    .createWithDefault(Nil)
+
   val CONVERT_METASTORE_PARQUET = buildConf("spark.sql.hive.convertMetastoreParquet")
     .doc("When set to true, the built-in Parquet reader and writer are used to process " +
       "parquet tables created by using the HiveQL syntax, instead of Hive serde.")
@@ -178,6 +202,7 @@ private[spark] object HiveUtils extends Logging {
    * The location of the jars that should be used to instantiate the HiveMetastoreClient.  This
    * property can be one of three options:
    *  - a classpath in the standard format for both hive and hadoop.
+   *  - path - attempt to discover the jars with paths configured by `HIVE_METASTORE_JARS_PATH`.
    *  - builtin - attempt to discover the jars that were used to load Spark SQL and use those. This
    *              option is only valid when using the execution version of Hive.
    *  - maven - download the correct version of hive on demand from maven.
@@ -186,6 +211,13 @@ private[spark] object HiveUtils extends Logging {
     conf.getConf(HIVE_METASTORE_JARS)
   }
 
+  /**
+   * Hive jars paths, only work when `HIVE_METASTORE_JARS` is `path`.
+   */
+  private def hiveMetastoreJarsPath(conf: SQLConf): Seq[String] = {
+    conf.getConf(HIVE_METASTORE_JARS_PATH)
+  }
+
   /**
    * A comma separated list of class prefixes that should be loaded using the classloader that
    * is shared between Spark SQL and a specific version of Hive. An example of classes that should
@@ -336,6 +368,20 @@ private[spark] object HiveUtils extends Logging {
     val hiveMetastoreBarrierPrefixes = HiveUtils.hiveMetastoreBarrierPrefixes(sqlConf)
     val metaVersion = IsolatedClientLoader.hiveVersion(hiveMetastoreVersion)
 
+    def addLocalHiveJars(file: File): Seq[URL] = {
+      if (file.getName == "*") {
+        val files = file.getParentFile.listFiles()
+        if (files == null) {
+          logWarning(s"Hive jar path '${file.getPath}' does not exist.")
+          Nil
+        } else {
+          files.filter(_.getName.toLowerCase(Locale.ROOT).endsWith(".jar")).map(_.toURI.toURL).toSeq
+        }
+      } else {
+        file.toURI.toURL :: Nil
+      }
+    }
+
     val isolatedLoader = if (hiveMetastoreJars == "builtin") {
       if (builtinHiveVersion != hiveMetastoreVersion) {
         throw new IllegalArgumentException(
@@ -396,24 +442,43 @@ private[spark] object HiveUtils extends Logging {
         config = configurations,
         barrierPrefixes = hiveMetastoreBarrierPrefixes,
         sharedPrefixes = hiveMetastoreSharedPrefixes)
+    } else if (hiveMetastoreJars == "path") {
+      // Convert to files and expand any directories.
+      val jars =
+        HiveUtils.hiveMetastoreJarsPath(sqlConf)
+          .flatMap {
+            case path if path.contains("\\") && Utils.isWindows =>
+              addLocalHiveJars(new File(path))
+            case path =>
+              DataSource.checkAndGlobPathIfNecessary(
+                pathStrings = Seq(path),
+                hadoopConf = hadoopConf,
+                checkEmptyGlobPath = true,
+                checkFilesExist = false,
+                enableGlobbing = true
+              ).map(_.toUri.toURL)
+          }
+
+      logInfo(
+        s"Initializing HiveMetastoreConnection version $hiveMetastoreVersion " +
+          s"using path: ${jars.mkString(";")}")
+      new IsolatedClientLoader(
+        version = metaVersion,
+        sparkConf = conf,
+        hadoopConf = hadoopConf,
+        execJars = jars,
+        config = configurations,
+        isolationOn = true,
+        barrierPrefixes = hiveMetastoreBarrierPrefixes,
+        sharedPrefixes = hiveMetastoreSharedPrefixes)
     } else {
       // Convert to files and expand any directories.
       val jars =
         hiveMetastoreJars
           .split(File.pathSeparator)
-          .flatMap {
-          case path if new File(path).getName == "*" =>
-            val files = new File(path).getParentFile.listFiles()
-            if (files == null) {
-              logWarning(s"Hive jar path '$path' does not exist.")
-              Nil
-            } else {
-              files.filter(_.getName.toLowerCase(Locale.ROOT).endsWith(".jar"))
-            }
-          case path =>
-            new File(path) :: Nil
-        }
-          .map(_.toURI.toURL)
+          .flatMap { path =>
+            addLocalHiveJars(new File(path))
+          }
 
       logInfo(
         s"Initializing HiveMetastoreConnection version $hiveMetastoreVersion " +
@@ -505,7 +570,7 @@ private[spark] object HiveUtils extends Logging {
       // partition columns are part of the schema
       val partCols = hiveTable.getPartCols.asScala.map(HiveClientImpl.fromHiveColumn)
       val dataCols = hiveTable.getCols.asScala.map(HiveClientImpl.fromHiveColumn)
-      table.copy(schema = StructType(dataCols ++ partCols))
+      table.copy(schema = StructType((dataCols ++ partCols).toSeq))
     }
   }
 }
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
index 4d18eb6289418..96949a33a137a 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.ql.exec.Utilities
 import org.apache.hadoop.hive.ql.metadata.{Partition => HivePartition, Table => HiveTable}
 import org.apache.hadoop.hive.ql.plan.TableDesc
 import org.apache.hadoop.hive.serde2.Deserializer
+import org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils.AvroTableProperties
 import org.apache.hadoop.hive.serde2.objectinspector.{ObjectInspectorConverters, StructObjectInspector}
 import org.apache.hadoop.hive.serde2.objectinspector.primitive._
 import org.apache.hadoop.io.Writable
@@ -39,7 +40,7 @@ import org.apache.spark.deploy.SparkHadoopUtil
 import org.apache.spark.internal.Logging
 import org.apache.spark.rdd.{EmptyRDD, HadoopRDD, NewHadoopRDD, RDD, UnionRDD}
 import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.{InternalRow, SQLConfHelper}
 import org.apache.spark.sql.catalyst.analysis.CastSupport
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
@@ -68,7 +69,7 @@ class HadoopTableReader(
     @transient private val tableDesc: TableDesc,
     @transient private val sparkSession: SparkSession,
     hadoopConf: Configuration)
-  extends TableReader with CastSupport with Logging {
+  extends TableReader with CastSupport with SQLConfHelper with Logging {
 
   // Hadoop honors "mapreduce.job.maps" as hint,
   // but will ignore when mapreduce.jobtracker.address is "local".
@@ -239,6 +240,8 @@ class HadoopTableReader(
       fillPartitionKeys(partValues, mutableRow)
 
       val tableProperties = tableDesc.getProperties
+      val avroSchemaProperties = Seq(AvroTableProperties.SCHEMA_LITERAL,
+        AvroTableProperties.SCHEMA_URL).map(_.getPropName())
 
       // Create local references so that the outer object isn't serialized.
       val localTableDesc = tableDesc
@@ -248,10 +251,15 @@ class HadoopTableReader(
         // SPARK-13709: For SerDes like AvroSerDe, some essential information (e.g. Avro schema
         // information) may be defined in table properties. Here we should merge table properties
         // and partition properties before initializing the deserializer. Note that partition
-        // properties take a higher priority here. For example, a partition may have a different
-        // SerDe as the one defined in table properties.
+        // properties take a higher priority here except for the Avro table properties
+        // to support schema evolution: in that case the properties given at table level will
+        // be used (for details please check SPARK-26836).
+        // For example, a partition may have a different SerDe as the one defined in table
+        // properties.
         val props = new Properties(tableProperties)
-        partProps.asScala.foreach {
+        partProps.asScala.filterNot { case (k, _) =>
+          avroSchemaProperties.contains(k) && tableProperties.containsKey(k)
+        }.foreach {
           case (key, value) => props.setProperty(key, value)
         }
         DeserializerLock.synchronized {
@@ -308,7 +316,7 @@ class HadoopTableReader(
 
   /**
    * Creates a HadoopRDD based on the broadcasted HiveConf and other job properties that will be
-   * applied locally on each slave.
+   * applied locally on each executor.
    */
   private def createOldHadoopRDD(tableDesc: TableDesc, path: String): RDD[Writable] = {
     val initializeJobConfFunc = HadoopTableReader.initializeLocalJobConfFunc(path, tableDesc) _
@@ -330,7 +338,7 @@ class HadoopTableReader(
 
   /**
    * Creates a NewHadoopRDD based on the broadcasted HiveConf and other job properties that will be
-   * applied locally on each slave.
+   * applied locally on each executor.
    */
   private def createNewHadoopRDD(tableDesc: TableDesc, path: String): RDD[Writable] = {
     val newJobConf = new JobConf(hadoopConf)
@@ -486,13 +494,19 @@ private[hive] object HadoopTableReader extends HiveInspectors with Logging {
       var i = 0
       val length = fieldRefs.length
       while (i < length) {
-        val fieldValue = soi.getStructFieldData(raw, fieldRefs(i))
-        if (fieldValue == null) {
-          mutableRow.setNullAt(fieldOrdinals(i))
-        } else {
-          unwrappers(i)(fieldValue, mutableRow, fieldOrdinals(i))
+        try {
+          val fieldValue = soi.getStructFieldData(raw, fieldRefs(i))
+          if (fieldValue == null) {
+            mutableRow.setNullAt(fieldOrdinals(i))
+          } else {
+            unwrappers(i)(fieldValue, mutableRow, fieldOrdinals(i))
+          }
+          i += 1
+        } catch {
+          case ex: Throwable =>
+            logError(s"Exception thrown in field <${fieldRefs(i).getFieldName}>")
+            throw ex
         }
-        i += 1
       }
 
       mutableRow: InternalRow
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala
index 3ea80eaf6f714..48f3837740933 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala
@@ -233,7 +233,8 @@ private[hive] trait HiveClient {
   /** Returns partitions filtered by predicates for the given table. */
   def getPartitionsByFilter(
       catalogTable: CatalogTable,
-      predicates: Seq[Expression]): Seq[CatalogTablePartition]
+      predicates: Seq[Expression],
+      timeZoneId: String): Seq[CatalogTablePartition]
 
   /** Loads a static partition into an existing table. */
   def loadPartition(
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
index 6ad5e9d3c9080..e9728b8c0461d 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
@@ -17,8 +17,9 @@
 
 package org.apache.spark.sql.hive.client
 
-import java.io.{File, PrintStream}
+import java.io.PrintStream
 import java.lang.{Iterable => JIterable}
+import java.lang.reflect.InvocationTargetException
 import java.nio.charset.StandardCharsets.UTF_8
 import java.util.{Locale, Map => JMap}
 import java.util.concurrent.TimeUnit._
@@ -48,19 +49,19 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.metrics.source.HiveCatalogMetrics
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.{NoSuchDatabaseException, NoSuchPartitionException}
+import org.apache.spark.sql.catalyst.analysis.{NoSuchDatabaseException, NoSuchPartitionException, NoSuchPartitionsException, PartitionAlreadyExistsException, PartitionsAlreadyExistException}
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
 import org.apache.spark.sql.catalyst.expressions.Expression
 import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, ParseException}
+import org.apache.spark.sql.catalyst.util.CharVarcharUtils
 import org.apache.spark.sql.connector.catalog.SupportsNamespaces._
 import org.apache.spark.sql.execution.QueryExecutionException
 import org.apache.spark.sql.hive.HiveExternalCatalog
-import org.apache.spark.sql.hive.HiveExternalCatalog.{DATASOURCE_SCHEMA, DATASOURCE_SCHEMA_NUMPARTS, DATASOURCE_SCHEMA_PART_PREFIX}
-import org.apache.spark.sql.hive.HiveUtils
+import org.apache.spark.sql.hive.HiveExternalCatalog.DATASOURCE_SCHEMA
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
-import org.apache.spark.util.{CircularBuffer, Utils}
+import org.apache.spark.util.{CircularBuffer, ShutdownHookManager, Utils}
 
 /**
  * A class that wraps the HiveClient and converts its responses to externally visible classes.
@@ -154,6 +155,25 @@ private[hive] class HiveClientImpl(
     }
   }
 
+  private def closeState(): Unit = withHiveState {
+    // These temp files are registered in o.a.h.u.ShutdownHookManager too during state start.
+    // The state.close() will delete them if they are not null and try remove them from the
+    // o.a.h.u.ShutdownHookManager which causes undesirable IllegalStateException.
+    // We delete them ahead with a high priority hook here and set them to null to bypass the
+    // deletion in state.close().
+    if (state.getTmpOutputFile != null) {
+      state.getTmpOutputFile.delete()
+      state.setTmpOutputFile(null)
+    }
+    if (state.getTmpErrOutputFile != null) {
+      state.getTmpErrOutputFile.delete()
+      state.setTmpErrOutputFile(null)
+    }
+    state.close()
+  }
+
+  ShutdownHookManager.addShutdownHook(() => closeState())
+
   // Log the default warehouse location.
   logInfo(
     s"Warehouse location for Hive client " +
@@ -166,12 +186,10 @@ private[hive] class HiveClientImpl(
       Hive.set(clientLoader.cachedHive.asInstanceOf[Hive])
     }
     // Hive 2.3 will set UDFClassLoader to hiveConf when initializing SessionState
-    // since HIVE-11878, and ADDJarCommand will add jars to clientLoader.classLoader.
-    // For this reason we cannot load the jars added by ADDJarCommand because of class loader
+    // since HIVE-11878, and ADDJarsCommand will add jars to clientLoader.classLoader.
+    // For this reason we cannot load the jars added by ADDJarsCommand because of class loader
     // got changed. We reset it to clientLoader.ClassLoader here.
-    if (HiveUtils.isHive23) {
-      state.getConf.setClassLoader(clientLoader.classLoader)
-    }
+    state.getConf.setClassLoader(clientLoader.classLoader)
     SessionState.start(state)
     state.out = new PrintStream(outputBuffer, true, UTF_8.name())
     state.err = new PrintStream(outputBuffer, true, UTF_8.name())
@@ -179,9 +197,7 @@ private[hive] class HiveClientImpl(
   }
 
   /** Returns the configuration for the current session. */
-  def conf: HiveConf = if (!HiveUtils.isHive23) {
-    state.getConf
-  } else {
+  def conf: HiveConf = {
     val hiveConf = state.getConf
     // Hive changed the default of datanucleus.schema.autoCreateAll from true to false
     // and hive.metastore.schema.verification from false to true since Hive 2.0.
@@ -276,15 +292,17 @@ private[hive] class HiveClientImpl(
   def withHiveState[A](f: => A): A = retryLocked {
     val original = Thread.currentThread().getContextClassLoader
     val originalConfLoader = state.getConf.getClassLoader
-    // The classloader in clientLoader could be changed after addJar, always use the latest
-    // classloader. We explicitly set the context class loader since "conf.setClassLoader" does
+    // We explicitly set the context class loader since "conf.setClassLoader" does
     // not do that, and the Hive client libraries may need to load classes defined by the client's
-    // class loader.
+    // class loader. See SPARK-19804 for more details.
     Thread.currentThread().setContextClassLoader(clientLoader.classLoader)
     state.getConf.setClassLoader(clientLoader.classLoader)
     // Set the thread local metastore client to the client associated with this HiveClientImpl.
     Hive.set(client)
     // Replace conf in the thread local Hive with current conf
+    // with the side-effect of Hive.get(conf) to avoid using out-of-date HiveConf.
+    // See discussion in https://github.com/apache/spark/pull/16826/files#r104606859
+    // for more details.
     Hive.get(conf)
     // setCurrentSessionState will use the classLoader associated
     // with the HiveConf in `state` to override the context class loader of the current
@@ -293,11 +311,9 @@ private[hive] class HiveClientImpl(
     val ret = try {
       f
     } catch {
-      case e: NoClassDefFoundError
-        if HiveUtils.isHive23 && e.getMessage.contains("org/apache/hadoop/hive/serde2/SerDe") =>
+      case e: NoClassDefFoundError if e.getMessage.contains("apache/hadoop/hive/serde2/SerDe") =>
         throw new ClassNotFoundException("The SerDe interface removed since Hive 2.3(HIVE-15167)." +
-          " Please migrate your custom SerDes to Hive 2.3 or build your own Spark with" +
-          " hive-1.2 profile. See HIVE-15167 for more details.", e)
+          " Please migrate your custom SerDes to Hive 2.3. See HIVE-15167 for more details.", e)
     } finally {
       state.getConf.setClassLoader(originalConfLoader)
       Thread.currentThread().setContextClassLoader(original)
@@ -374,14 +390,14 @@ private[hive] class HiveClientImpl(
 
   override def getDatabase(dbName: String): CatalogDatabase = withHiveState {
     Option(client.getDatabase(dbName)).map { d =>
-      val paras = Option(d.getParameters).map(_.asScala.toMap).getOrElse(Map()) ++
+      val params = Option(d.getParameters).map(_.asScala.toMap).getOrElse(Map()) ++
         Map(PROP_OWNER -> shim.getDatabaseOwnerName(d))
 
       CatalogDatabase(
         name = d.getName,
         description = Option(d.getDescription).getOrElse(""),
         locationUri = CatalogUtils.stringToURI(d.getLocationUri),
-        properties = paras)
+        properties = params)
     }.getOrElse(throw new NoSuchDatabaseException(dbName))
   }
 
@@ -390,7 +406,7 @@ private[hive] class HiveClientImpl(
   }
 
   override def listDatabases(pattern: String): Seq[String] = withHiveState {
-    client.getDatabasesByPattern(pattern).asScala
+    client.getDatabasesByPattern(pattern).asScala.toSeq
   }
 
   private def getRawTableOption(dbName: String, tableName: String): Option[HiveTable] = {
@@ -400,7 +416,7 @@ private[hive] class HiveClientImpl(
   private def getRawTablesByName(dbName: String, tableNames: Seq[String]): Seq[HiveTable] = {
     try {
       msClient.getTableObjectsByName(dbName, tableNames.asJava).asScala
-        .map(extraFixesForNonView).map(new HiveTable(_))
+        .map(extraFixesForNonView).map(new HiveTable(_)).toSeq
     } catch {
       case ex: Exception =>
         throw new HiveException(s"Unable to fetch tables of db $dbName", ex);
@@ -434,7 +450,7 @@ private[hive] class HiveClientImpl(
         throw new SparkException(
           s"${ex.getMessage}, db: ${h.getDbName}, table: ${h.getTableName}", ex)
     }
-    val schema = StructType(cols ++ partCols)
+    val schema = StructType((cols ++ partCols).toSeq)
 
     val bucketSpec = if (h.getNumBuckets > 0) {
       val sortColumnOrders = h.getSortCols.asScala
@@ -450,7 +466,7 @@ private[hive] class HiveClientImpl(
       } else {
         Seq.empty
       }
-      Option(BucketSpec(h.getNumBuckets, h.getBucketCols.asScala, sortColumnNames))
+      Option(BucketSpec(h.getNumBuckets, h.getBucketCols.asScala.toSeq, sortColumnNames.toSeq))
     } else {
       None
     }
@@ -502,7 +518,7 @@ private[hive] class HiveClientImpl(
           throw new AnalysisException(s"Hive $tableTypeStr is not supported.")
       },
       schema = schema,
-      partitionColumnNames = partCols.map(_.name),
+      partitionColumnNames = partCols.map(_.name).toSeq,
       // If the table is written by Spark, we will put bucketing information in table properties,
       // and will always overwrite the bucket spec in hive metastore by the bucketing information
       // in table properties. This means, if we have bucket spec in both hive metastore and
@@ -539,7 +555,7 @@ private[hive] class HiveClientImpl(
       // that created by older versions of Spark.
       viewOriginalText = Option(h.getViewOriginalText),
       viewText = Option(h.getViewExpandedText),
-      unsupportedFeatures = unsupportedFeatures,
+      unsupportedFeatures = unsupportedFeatures.toSeq,
       ignoredProperties = ignoredProperties.toMap)
   }
 
@@ -586,9 +602,7 @@ private[hive] class HiveClientImpl(
     val it = oldTable.getParameters.entrySet.iterator
     while (it.hasNext) {
       val entry = it.next()
-      val isSchemaProp = entry.getKey.startsWith(DATASOURCE_SCHEMA_PART_PREFIX) ||
-        entry.getKey == DATASOURCE_SCHEMA || entry.getKey == DATASOURCE_SCHEMA_NUMPARTS
-      if (isSchemaProp) {
+      if (CatalogTable.isLargeTableProp(DATASOURCE_SCHEMA, entry.getKey)) {
         it.remove()
       }
     }
@@ -605,7 +619,17 @@ private[hive] class HiveClientImpl(
       table: String,
       parts: Seq[CatalogTablePartition],
       ignoreIfExists: Boolean): Unit = withHiveState {
-    shim.createPartitions(client, db, table, parts, ignoreIfExists)
+    def replaceExistException(e: Throwable): Unit = e match {
+      case _: HiveException if e.getCause.isInstanceOf[AlreadyExistsException] =>
+        throw new PartitionsAlreadyExistException(db, table, parts.map(_.spec))
+      case _ => throw e
+    }
+    try {
+      shim.createPartitions(client, db, table, parts, ignoreIfExists)
+    } catch {
+      case e: InvocationTargetException => replaceExistException(e.getCause)
+      case e: Throwable => replaceExistException(e)
+    }
   }
 
   override def dropPartitions(
@@ -626,9 +650,7 @@ private[hive] class HiveClientImpl(
         // (b='1', c='1') and (b='1', c='2'), a partial spec of (b='1') will match both.
         val parts = client.getPartitions(hiveTable, s.asJava).asScala
         if (parts.isEmpty && !ignoreIfNotExists) {
-          throw new AnalysisException(
-            s"No partition is dropped. One partition spec '$s' does not exist in table '$table' " +
-            s"database '$db'")
+          throw new NoSuchPartitionsException(db, table, Seq(s))
         }
         parts.map(_.getValues)
       }.distinct
@@ -638,7 +660,7 @@ private[hive] class HiveClientImpl(
         shim.dropPartition(client, db, table, partition, !retainData, purge)
       } catch {
         case e: Exception =>
-          val remainingParts = matchingParts.toBuffer -- droppedParts
+          val remainingParts = matchingParts.toBuffer --= droppedParts
           logError(
             s"""
                |======================
@@ -665,6 +687,9 @@ private[hive] class HiveClientImpl(
     val catalogTable = getTable(db, table)
     val hiveTable = toHiveTable(catalogTable, Some(userName))
     specs.zip(newSpecs).foreach { case (oldSpec, newSpec) =>
+      if (client.getPartition(hiveTable, newSpec.asJava, false) != null) {
+        throw new PartitionAlreadyExistsException(db, table, newSpec)
+      }
       val hivePart = getPartitionOption(catalogTable, oldSpec)
         .map { p => toHivePartition(p.copy(spec = newSpec), hiveTable) }
         .getOrElse { throw new NoSuchPartitionException(db, table, oldSpec) }
@@ -708,7 +733,7 @@ private[hive] class HiveClientImpl(
           assert(s.values.forall(_.nonEmpty), s"partition spec '$s' is invalid")
           client.getPartitionNames(table.database, table.identifier.table, s.asJava, -1)
       }
-    hivePartitionNames.asScala.sorted
+    hivePartitionNames.asScala.sorted.toSeq
   }
 
   override def getPartitionOption(
@@ -735,39 +760,43 @@ private[hive] class HiveClientImpl(
     }
     val parts = client.getPartitions(hiveTable, partSpec.asJava).asScala.map(fromHivePartition)
     HiveCatalogMetrics.incrementFetchedPartitions(parts.length)
-    parts
+    parts.toSeq
   }
 
   override def getPartitionsByFilter(
       table: CatalogTable,
-      predicates: Seq[Expression]): Seq[CatalogTablePartition] = withHiveState {
+      predicates: Seq[Expression],
+      timeZoneId: String): Seq[CatalogTablePartition] = withHiveState {
     val hiveTable = toHiveTable(table, Some(userName))
-    val parts = shim.getPartitionsByFilter(client, hiveTable, predicates).map(fromHivePartition)
+    val parts = shim.getPartitionsByFilter(client, hiveTable, predicates, timeZoneId)
+      .map(fromHivePartition)
     HiveCatalogMetrics.incrementFetchedPartitions(parts.length)
     parts
   }
 
   override def listTables(dbName: String): Seq[String] = withHiveState {
-    client.getAllTables(dbName).asScala
+    client.getAllTables(dbName).asScala.toSeq
   }
 
   override def listTables(dbName: String, pattern: String): Seq[String] = withHiveState {
-    client.getTablesByPattern(dbName, pattern).asScala
+    client.getTablesByPattern(dbName, pattern).asScala.toSeq
   }
 
   override def listTablesByType(
       dbName: String,
       pattern: String,
       tableType: CatalogTableType): Seq[String] = withHiveState {
+    val hiveTableType = toHiveTableType(tableType)
     try {
       // Try with Hive API getTablesByType first, it's supported from Hive 2.3+.
-      shim.getTablesByType(client, dbName, pattern, toHiveTableType(tableType))
+      shim.getTablesByType(client, dbName, pattern, hiveTableType)
     } catch {
       case _: UnsupportedOperationException =>
         // Fallback to filter logic if getTablesByType not supported.
         val tableNames = client.getTablesByPattern(dbName, pattern).asScala
-        val tables = getTablesByName(dbName, tableNames).filter(_.tableType == tableType)
-        tables.map(_.identifier.table)
+        getRawTablesByName(dbName, tableNames.toSeq)
+          .filter(_.getTableType == hiveTableType)
+          .map(_.getTableName)
     }
   }
 
@@ -927,16 +956,8 @@ private[hive] class HiveClientImpl(
   }
 
   def addJar(path: String): Unit = {
-    val uri = new Path(path).toUri
-    val jarURL = if (uri.getScheme == null) {
-      // `path` is a local file path without a URL scheme
-      new File(path).toURI.toURL
-    } else {
-      // `path` is a URL with a scheme
-      uri.toURL
-    }
-    clientLoader.addJar(jarURL)
-    runSqlHive(s"ADD JAR $path")
+    val jarURI = Utils.resolveURI(path)
+    clientLoader.addJar(jarURI.toURL)
   }
 
   def newSession(): HiveClientImpl = {
@@ -981,11 +1002,11 @@ private[hive] class HiveClientImpl(
 private[hive] object HiveClientImpl extends Logging {
   /** Converts the native StructField to Hive's FieldSchema. */
   def toHiveColumn(c: StructField): FieldSchema = {
-    val typeString = if (c.metadata.contains(HIVE_TYPE_STRING)) {
-      c.metadata.getString(HIVE_TYPE_STRING)
-    } else {
-      c.dataType.catalogString
-    }
+    // For Hive Serde, we still need to to restore the raw type for char and varchar type.
+    // When reading data in parquet, orc, or avro file format with string type for char,
+    // the tailing spaces may lost if we are not going to pad it.
+    val typeString = CharVarcharUtils.getRawTypeString(c.metadata)
+      .getOrElse(HiveVoidType.replaceVoidType(c.dataType).catalogString)
     new FieldSchema(c.name, typeString, c.getComment().orNull)
   }
 
@@ -1003,17 +1024,10 @@ private[hive] object HiveClientImpl extends Logging {
   /** Builds the native StructField from Hive's FieldSchema. */
   def fromHiveColumn(hc: FieldSchema): StructField = {
     val columnType = getSparkSQLDataType(hc)
-    val metadata = if (hc.getType != columnType.catalogString) {
-      new MetadataBuilder().putString(HIVE_TYPE_STRING, hc.getType).build()
-    } else {
-      Metadata.empty
-    }
-
     val field = StructField(
       name = hc.getName,
       dataType = columnType,
-      nullable = true,
-      metadata = metadata)
+      nullable = true)
     Option(hc.getComment).map(field.withComment).getOrElse(field)
   }
 
@@ -1271,3 +1285,22 @@ private[hive] object HiveClientImpl extends Logging {
     hiveConf
   }
 }
+
+private[hive] case object HiveVoidType extends DataType {
+  override def defaultSize: Int = 1
+  override def asNullable: DataType = HiveVoidType
+  override def simpleString: String = "void"
+
+  def replaceVoidType(dt: DataType): DataType = dt match {
+    case ArrayType(et, nullable) =>
+      ArrayType(replaceVoidType(et), nullable)
+    case MapType(kt, vt, nullable) =>
+      MapType(replaceVoidType(kt), replaceVoidType(vt), nullable)
+    case StructType(fields) =>
+      StructType(fields.map { field =>
+        field.copy(dataType = replaceVoidType(field.dataType))
+      })
+    case _: NullType => HiveVoidType
+    case _ => dt
+  }
+}
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
index 2b806609426a1..2f7fe96013dea 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
@@ -41,13 +41,13 @@ import org.apache.hadoop.hive.serde.serdeConstants
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.FunctionIdentifier
 import org.apache.spark.sql.catalyst.analysis.NoSuchPermanentFunctionException
 import org.apache.spark.sql.catalyst.catalog.{CatalogFunction, CatalogTablePartition, CatalogUtils, FunctionResource, FunctionResourceType}
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.util.{DateFormatter, DateTimeUtils, TypeUtils}
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{AtomicType, IntegralType, StringType}
+import org.apache.spark.sql.types.{AtomicType, DateType, IntegralType, StringType}
 import org.apache.spark.unsafe.types.UTF8String
 import org.apache.spark.util.Utils
 
@@ -79,7 +79,11 @@ private[client] sealed abstract class Shim {
 
   def getAllPartitions(hive: Hive, table: Table): Seq[Partition]
 
-  def getPartitionsByFilter(hive: Hive, table: Table, predicates: Seq[Expression]): Seq[Partition]
+  def getPartitionsByFilter(
+      hive: Hive,
+      table: Table,
+      predicates: Seq[Expression],
+      timeZoneId: String): Seq[Partition]
 
   def getCommandProcessor(token: String, conf: HiveConf): CommandProcessor
 
@@ -349,7 +353,8 @@ private[client] class Shim_v0_12 extends Shim with Logging {
   override def getPartitionsByFilter(
       hive: Hive,
       table: Table,
-      predicates: Seq[Expression]): Seq[Partition] = {
+      predicates: Seq[Expression],
+      timeZoneId: String): Seq[Partition] = {
     // getPartitionsByFilter() doesn't support binary comparison ops in Hive 0.12.
     // See HIVE-4888.
     logDebug("Hive 0.12 doesn't support predicate pushdown to metastore. " +
@@ -363,7 +368,7 @@ private[client] class Shim_v0_12 extends Shim with Logging {
   override def getDriverResults(driver: Driver): Seq[String] = {
     val res = new JArrayList[String]()
     getDriverResultsMethod.invoke(driver, res)
-    res.asScala
+    res.asScala.toSeq
   }
 
   override def getMetastoreClientConnectRetryDelayMillis(conf: HiveConf): Long = {
@@ -600,7 +605,7 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
       }
       FunctionResource(FunctionResourceType.fromString(resourceType), uri.getUri())
     }
-    CatalogFunction(name, hf.getClassName, resources)
+    CatalogFunction(name, hf.getClassName, resources.toSeq)
   }
 
   override def getFunctionOption(hive: Hive, db: String, name: String): Option[CatalogFunction] = {
@@ -623,7 +628,7 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
   }
 
   override def listFunctions(hive: Hive, db: String, pattern: String): Seq[String] = {
-    hive.getFunctions(db, pattern).asScala
+    hive.getFunctions(db, pattern).asScala.toSeq
   }
 
   /**
@@ -632,7 +637,9 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
    *
    * Unsupported predicates are skipped.
    */
-  def convertFilters(table: Table, filters: Seq[Expression]): String = {
+  def convertFilters(table: Table, filters: Seq[Expression], timeZoneId: String): String = {
+    lazy val dateFormatter = DateFormatter(DateTimeUtils.getZoneId(timeZoneId))
+
     /**
      * An extractor that matches all binary comparison operators except null-safe equality.
      *
@@ -650,6 +657,8 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
         case Literal(null, _) => None // `null`s can be cast as other types; we want to avoid NPEs.
         case Literal(value, _: IntegralType) => Some(value.toString)
         case Literal(value, _: StringType) => Some(quoteStringLiteral(value.toString))
+        case Literal(value, _: DateType) =>
+          Some(dateFormatter.format(value.asInstanceOf[Int]))
         case _ => None
       }
     }
@@ -691,7 +700,22 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
       }
 
       def unapply(values: Set[Any]): Option[Seq[String]] = {
-        val extractables = values.toSeq.map(valueToLiteralString.lift)
+        val extractables = values.filter(_ != null).toSeq.map(valueToLiteralString.lift)
+        if (extractables.nonEmpty && extractables.forall(_.isDefined)) {
+          Some(extractables.map(_.get))
+        } else {
+          None
+        }
+      }
+    }
+
+    object ExtractableDateValues {
+      private lazy val valueToLiteralString: PartialFunction[Any, String] = {
+        case value: Int => dateFormatter.format(value)
+      }
+
+      def unapply(values: Set[Any]): Option[Seq[String]] = {
+        val extractables = values.filter(_ != null).toSeq.map(valueToLiteralString.lift)
         if (extractables.nonEmpty && extractables.forall(_.isDefined)) {
           Some(extractables.map(_.get))
         } else {
@@ -708,9 +732,11 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
         .map(col => col.getName).toSet
 
       def unapply(attr: Attribute): Option[String] = {
-        if (varcharKeys.contains(attr.name)) {
+        val resolver = SQLConf.get.resolver
+        if (varcharKeys.exists(c => resolver(c, attr.name))) {
           None
-        } else if (attr.dataType.isInstanceOf[IntegralType] || attr.dataType == StringType) {
+        } else if (attr.dataType.isInstanceOf[IntegralType] || attr.dataType == StringType ||
+            attr.dataType == DateType) {
           Some(attr.name)
         } else {
           None
@@ -722,13 +748,23 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
       values.map(value => s"$name = $value").mkString("(", " or ", ")")
     }
 
+    def convertNotInToAnd(name: String, values: Seq[String]): String = {
+      values.map(value => s"$name != $value").mkString("(", " and ", ")")
+    }
+
+    def hasNullLiteral(list: Seq[Expression]): Boolean = list.exists {
+      case Literal(null, _) => true
+      case _ => false
+    }
+
     val useAdvanced = SQLConf.get.advancedPartitionPredicatePushdownEnabled
+    val inSetThreshold = SQLConf.get.metastorePartitionPruningInSetThreshold
 
     object ExtractAttribute {
       def unapply(expr: Expression): Option[Attribute] = {
         expr match {
           case attr: Attribute => Some(attr)
-          case Cast(child @ AtomicType(), dt: AtomicType, _)
+          case Cast(child @ IntegralType(), dt: IntegralType, _)
               if Cast.canUpCast(child.dataType.asInstanceOf[AtomicType], dt) => unapply(child)
           case _ => None
         }
@@ -736,14 +772,44 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
     }
 
     def convert(expr: Expression): Option[String] = expr match {
+      case Not(InSet(_, values)) if values.size > inSetThreshold =>
+        None
+
+      case Not(In(_, list)) if hasNullLiteral(list) => None
+      case Not(InSet(_, list)) if list.contains(null) => None
+
       case In(ExtractAttribute(SupportedAttribute(name)), ExtractableLiterals(values))
           if useAdvanced =>
         Some(convertInToOr(name, values))
 
+      case Not(In(ExtractAttribute(SupportedAttribute(name)), ExtractableLiterals(values)))
+        if useAdvanced =>
+        Some(convertNotInToAnd(name, values))
+
+      case InSet(child, values) if useAdvanced && values.size > inSetThreshold =>
+        val dataType = child.dataType
+        // Skip null here is safe, more details could see at ExtractableLiterals.
+        val sortedValues = values.filter(_ != null).toSeq
+          .sorted(TypeUtils.getInterpretedOrdering(dataType))
+        convert(And(GreaterThanOrEqual(child, Literal(sortedValues.head, dataType)),
+          LessThanOrEqual(child, Literal(sortedValues.last, dataType))))
+
+      case InSet(child @ ExtractAttribute(SupportedAttribute(name)), ExtractableDateValues(values))
+          if useAdvanced && child.dataType == DateType =>
+        Some(convertInToOr(name, values))
+
+      case Not(InSet(child @ ExtractAttribute(SupportedAttribute(name)),
+        ExtractableDateValues(values))) if useAdvanced && child.dataType == DateType =>
+        Some(convertNotInToAnd(name, values))
+
       case InSet(ExtractAttribute(SupportedAttribute(name)), ExtractableValues(values))
           if useAdvanced =>
         Some(convertInToOr(name, values))
 
+      case Not(InSet(ExtractAttribute(SupportedAttribute(name)), ExtractableValues(values)))
+        if useAdvanced =>
+        Some(convertNotInToAnd(name, values))
+
       case op @ SpecialBinaryComparison(
           ExtractAttribute(SupportedAttribute(name)), ExtractableLiteral(value)) =>
         Some(s"$name ${op.symbol} $value")
@@ -752,6 +818,15 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
           ExtractableLiteral(value), ExtractAttribute(SupportedAttribute(name))) =>
         Some(s"$value ${op.symbol} $name")
 
+      case Contains(ExtractAttribute(SupportedAttribute(name)), ExtractableLiteral(value)) =>
+        Some(s"$name like " + (("\".*" + value.drop(1)).dropRight(1) + ".*\""))
+
+      case StartsWith(ExtractAttribute(SupportedAttribute(name)), ExtractableLiteral(value)) =>
+        Some(s"$name like " + (value.dropRight(1) + ".*\""))
+
+      case EndsWith(ExtractAttribute(SupportedAttribute(name)), ExtractableLiteral(value)) =>
+        Some(s"$name like " + ("\".*" + value.drop(1)))
+
       case And(expr1, expr2) if useAdvanced =>
         val converted = convert(expr1) ++ convert(expr2)
         if (converted.isEmpty) {
@@ -766,6 +841,14 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
           right <- convert(expr2)
         } yield s"($left or $right)"
 
+      case Not(EqualTo(
+          ExtractAttribute(SupportedAttribute(name)), ExtractableLiteral(value))) if useAdvanced =>
+        Some(s"$name != $value")
+
+      case Not(EqualTo(
+          ExtractableLiteral(value), ExtractAttribute(SupportedAttribute(name)))) if useAdvanced =>
+        Some(s"$value != $name")
+
       case _ => None
     }
 
@@ -786,11 +869,12 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
   override def getPartitionsByFilter(
       hive: Hive,
       table: Table,
-      predicates: Seq[Expression]): Seq[Partition] = {
+      predicates: Seq[Expression],
+      timeZoneId: String): Seq[Partition] = {
 
     // Hive getPartitionsByFilter() takes a string that represents partition
     // predicates like "str_key=\"value\" and int_key=1 ..."
-    val filter = convertFilters(table, predicates)
+    val filter = convertFilters(table, predicates, timeZoneId)
 
     val partitions =
       if (filter.isEmpty) {
@@ -842,7 +926,7 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
         case s: String => s
         case a: Array[Object] => a(0).asInstanceOf[String]
       }
-    }
+    }.toSeq
   }
 
   override def getDatabaseOwnerName(db: Database): String = {
@@ -1251,7 +1335,7 @@ private[client] class Shim_v2_3 extends Shim_v2_1 {
       pattern: String,
       tableType: TableType): Seq[String] = {
     getTablesByTypeMethod.invoke(hive, dbName, pattern, tableType)
-      .asInstanceOf[JList[String]].asScala
+      .asInstanceOf[JList[String]].asScala.toSeq
   }
 }
 
@@ -1326,10 +1410,7 @@ private[client] class Shim_v3_0 extends Shim_v2_3 {
       inheritTableSpecs: Boolean,
       isSkewedStoreAsSubdir: Boolean,
       isSrcLocal: Boolean): Unit = {
-    val session = SparkSession.getActiveSession
-    assert(session.nonEmpty)
-    val database = session.get.sessionState.catalog.getCurrentDatabase
-    val table = hive.getTable(database, tableName)
+    val table = hive.getTable(tableName)
     val loadFileType = if (replace) {
       clazzLoadFileType.getEnumConstants.find(_.toString.equalsIgnoreCase("REPLACE_ALL"))
     } else {
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
index 42a0ec0253b85..6171d32e53881 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
@@ -37,7 +37,7 @@ import org.apache.spark.sql.catalyst.util.quietly
 import org.apache.spark.sql.hive.HiveUtils
 import org.apache.spark.sql.internal.NonClosableMutableURLClassLoader
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.util.{MutableURLClassLoader, Utils}
+import org.apache.spark.util.{MutableURLClassLoader, Utils, VersionUtils}
 
 /** Factory for `IsolatedClientLoader` with specific versions of hive. */
 private[hive] object IsolatedClientLoader extends Logging {
@@ -52,12 +52,9 @@ private[hive] object IsolatedClientLoader extends Logging {
       config: Map[String, String] = Map.empty,
       ivyPath: Option[String] = None,
       sharedPrefixes: Seq[String] = Seq.empty,
-      barrierPrefixes: Seq[String] = Seq.empty,
-      sharesHadoopClasses: Boolean = true): IsolatedClientLoader = synchronized {
+      barrierPrefixes: Seq[String] = Seq.empty): IsolatedClientLoader = synchronized {
     val resolvedVersion = hiveVersion(hiveMetastoreVersion)
-    // We will first try to share Hadoop classes. If we cannot resolve the Hadoop artifact
-    // with the given version, we will use Hadoop 2.7 and then will not share Hadoop classes.
-    var _sharesHadoopClasses = sharesHadoopClasses
+    // We will use Hadoop 2.7 if we cannot resolve the Hadoop artifact.
     val files = if (resolvedVersions.contains((resolvedVersion, hadoopVersion))) {
       resolvedVersions((resolvedVersion, hadoopVersion))
     } else {
@@ -72,10 +69,8 @@ private[hive] object IsolatedClientLoader extends Logging {
             val fallbackVersion = "2.7.4"
             logWarning(s"Failed to resolve Hadoop artifacts for the version $hadoopVersion. We " +
               s"will change the hadoop version from $hadoopVersion to $fallbackVersion and try " +
-              "again. Hadoop classes will not be shared between Spark and Hive metastore client. " +
-              "It is recommended to set jars used by Hive metastore client through " +
+              "again. It is recommended to set jars used by Hive metastore client through " +
               "spark.sql.hive.metastore.jars in the production environment.")
-            _sharesHadoopClasses = false
             (downloadVersion(
               resolvedVersion, fallbackVersion, ivyPath, remoteRepos), fallbackVersion)
         }
@@ -89,28 +84,36 @@ private[hive] object IsolatedClientLoader extends Logging {
       execJars = files,
       hadoopConf = hadoopConf,
       config = config,
-      sharesHadoopClasses = _sharesHadoopClasses,
       sharedPrefixes = sharedPrefixes,
       barrierPrefixes = barrierPrefixes)
   }
 
-  def hiveVersion(version: String): HiveVersion = version match {
-    case "12" | "0.12" | "0.12.0" => hive.v12
-    case "13" | "0.13" | "0.13.0" | "0.13.1" => hive.v13
-    case "14" | "0.14" | "0.14.0" => hive.v14
-    case "1.0" | "1.0.0" | "1.0.1" => hive.v1_0
-    case "1.1" | "1.1.0" | "1.1.1" => hive.v1_1
-    case "1.2" | "1.2.0" | "1.2.1" | "1.2.2" => hive.v1_2
-    case "2.0" | "2.0.0" | "2.0.1" => hive.v2_0
-    case "2.1" | "2.1.0" | "2.1.1" => hive.v2_1
-    case "2.2" | "2.2.0" => hive.v2_2
-    case "2.3" | "2.3.0" | "2.3.1" | "2.3.2" | "2.3.3" | "2.3.4" | "2.3.5" | "2.3.6" | "2.3.7" =>
-      hive.v2_3
-    case "3.0" | "3.0.0" => hive.v3_0
-    case "3.1" | "3.1.0" | "3.1.1" | "3.1.2" => hive.v3_1
-    case version =>
+  def hiveVersion(version: String): HiveVersion = {
+    VersionUtils.majorMinorPatchVersion(version).flatMap {
+      case (12, _, _) | (0, 12, _) => Some(hive.v12)
+      case (13, _, _) | (0, 13, _) => Some(hive.v13)
+      case (14, _, _) | (0, 14, _) => Some(hive.v14)
+      case (1, 0, _) => Some(hive.v1_0)
+      case (1, 1, _) => Some(hive.v1_1)
+      case (1, 2, _) => Some(hive.v1_2)
+      case (2, 0, _) => Some(hive.v2_0)
+      case (2, 1, _) => Some(hive.v2_1)
+      case (2, 2, _) => Some(hive.v2_2)
+      case (2, 3, _) => Some(hive.v2_3)
+      case (3, 0, _) => Some(hive.v3_0)
+      case (3, 1, _) => Some(hive.v3_1)
+      case _ => None
+    }.getOrElse {
       throw new UnsupportedOperationException(s"Unsupported Hive Metastore version ($version). " +
         s"Please set ${HiveUtils.HIVE_METASTORE_VERSION.key} with a valid version.")
+    }
+  }
+
+  def supportsHadoopShadedClient(hadoopVersion: String): Boolean = {
+    VersionUtils.majorMinorPatchVersion(hadoopVersion).exists {
+      case (3, 2, v) if v >= 2 => true
+      case _ => false
+    }
   }
 
   private def downloadVersion(
@@ -118,21 +121,27 @@ private[hive] object IsolatedClientLoader extends Logging {
       hadoopVersion: String,
       ivyPath: Option[String],
       remoteRepos: String): Seq[URL] = {
+    val hadoopJarNames = if (supportsHadoopShadedClient(hadoopVersion)) {
+      Seq(s"org.apache.hadoop:hadoop-client-api:$hadoopVersion",
+        s"org.apache.hadoop:hadoop-client-runtime:$hadoopVersion")
+    } else {
+      Seq(s"org.apache.hadoop:hadoop-client:$hadoopVersion")
+    }
     val hiveArtifacts = version.extraDeps ++
       Seq("hive-metastore", "hive-exec", "hive-common", "hive-serde")
         .map(a => s"org.apache.hive:$a:${version.fullVersion}") ++
-      Seq("com.google.guava:guava:14.0.1",
-        s"org.apache.hadoop:hadoop-client:$hadoopVersion")
+      Seq("com.google.guava:guava:14.0.1") ++ hadoopJarNames
 
-    val classpath = quietly {
+    val classpaths = quietly {
       SparkSubmitUtils.resolveMavenCoordinates(
         hiveArtifacts.mkString(","),
         SparkSubmitUtils.buildIvySettings(
           Some(remoteRepos),
           ivyPath),
+        transitive = true,
         exclusions = version.exclusions)
     }
-    val allFiles = classpath.split(",").map(new File(_)).toSet
+    val allFiles = classpaths.map(new File(_)).toSet
 
     // TODO: Remove copy logic.
     val tempDir = Utils.createTempDir(namePrefix = s"hive-${version}")
@@ -164,7 +173,6 @@ private[hive] object IsolatedClientLoader extends Logging {
  * @param config   A set of options that will be added to the HiveConf of the constructed client.
  * @param isolationOn When true, custom versions of barrier classes will be constructed.  Must be
  *                    true unless loading the version of hive that is on Spark's classloader.
- * @param sharesHadoopClasses When true, we will share Hadoop classes between Spark and
  * @param baseClassLoader The spark classloader that is used to load shared classes.
  */
 private[hive] class IsolatedClientLoader(
@@ -174,7 +182,6 @@ private[hive] class IsolatedClientLoader(
     val execJars: Seq[URL] = Seq.empty,
     val config: Map[String, String] = Map.empty,
     val isolationOn: Boolean = true,
-    val sharesHadoopClasses: Boolean = true,
     val baseClassLoader: ClassLoader = Thread.currentThread().getContextClassLoader,
     val sharedPrefixes: Seq[String] = Seq.empty,
     val barrierPrefixes: Seq[String] = Seq.empty)
@@ -191,7 +198,7 @@ private[hive] class IsolatedClientLoader(
     name.startsWith("org.apache.log4j") || // log4j1.x
     name.startsWith("org.apache.logging.log4j") || // log4j2
     name.startsWith("org.apache.spark.") ||
-    (sharesHadoopClasses && isHadoopClass) ||
+    isHadoopClass ||
     name.startsWith("scala.") ||
     (name.startsWith("com.google") && !name.startsWith("com.google.cloud")) ||
     name.startsWith("java.") ||
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala
index 8526d86454604..bb78944fb1613 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala
@@ -22,7 +22,20 @@ package object client {
   private[hive] sealed abstract class HiveVersion(
       val fullVersion: String,
       val extraDeps: Seq[String] = Nil,
-      val exclusions: Seq[String] = Nil)
+      val exclusions: Seq[String] = Nil) extends Ordered[HiveVersion] {
+    override def compare(that: HiveVersion): Int = {
+      val thisVersionParts = fullVersion.split('.').map(_.toInt)
+      val thatVersionParts = that.fullVersion.split('.').map(_.toInt)
+      assert(thisVersionParts.length == thatVersionParts.length)
+      thisVersionParts.zip(thatVersionParts).foreach { case (l, r) =>
+        val candidate = l - r
+        if (candidate != 0) {
+          return candidate
+        }
+      }
+      0
+    }
+  }
 
   // scalastyle:off
   private[hive] object hive {
@@ -87,11 +100,13 @@ package object client {
         "org.apache.curator:*",
         "org.pentaho:pentaho-aggdesigner-algorithm"))
 
-    // Since HIVE-14496, Hive materialized view need calcite-core.
+    // Since HIVE-23980, calcite-core included in Hive package jar.
     // For spark, only VersionsSuite currently creates a hive materialized view for testing.
-    case object v2_3 extends HiveVersion("2.3.7",
-      exclusions = Seq("org.apache.calcite:calcite-druid",
+    case object v2_3 extends HiveVersion("2.3.8",
+      exclusions = Seq("org.apache.calcite:calcite-core",
+        "org.apache.calcite:calcite-druid",
         "org.apache.calcite.avatica:avatica",
+        "com.fasterxml.jackson.core:*",
         "org.apache.curator:*",
         "org.pentaho:pentaho-aggdesigner-algorithm"))
 
@@ -101,7 +116,6 @@ package object client {
       extraDeps = Seq("org.apache.logging.log4j:log4j-api:2.10.0",
         "org.apache.derby:derby:10.14.1.0"),
       exclusions = Seq("org.apache.calcite:calcite-druid",
-        "org.apache.calcite.avatica:avatica",
         "org.apache.curator:*",
         "org.pentaho:pentaho-aggdesigner-algorithm"))
 
@@ -111,7 +125,6 @@ package object client {
       extraDeps = Seq("org.apache.logging.log4j:log4j-api:2.10.0",
         "org.apache.derby:derby:10.14.1.0"),
       exclusions = Seq("org.apache.calcite:calcite-druid",
-        "org.apache.calcite.avatica:avatica",
         "org.apache.curator:*",
         "org.pentaho:pentaho-aggdesigner-algorithm"))
 
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/CreateHiveTableAsSelectCommand.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/CreateHiveTableAsSelectCommand.scala
index 9f79997e2979a..fe5d74f889dbb 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/CreateHiveTableAsSelectCommand.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/CreateHiveTableAsSelectCommand.scala
@@ -22,6 +22,7 @@ import scala.util.control.NonFatal
 import org.apache.spark.sql.{AnalysisException, Row, SaveMode, SparkSession}
 import org.apache.spark.sql.catalyst.catalog.{CatalogTable, SessionCatalog}
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.util.CharVarcharUtils
 import org.apache.spark.sql.execution.SparkPlan
 import org.apache.spark.sql.execution.command.{DataWritingCommand, DDLUtils}
 import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, InsertIntoHadoopFsRelationCommand, LogicalRelation}
@@ -54,19 +55,22 @@ trait CreateHiveTableAsSelectBase extends DataWritingCommand {
 
       val command = getWritingCommand(catalog, tableDesc, tableExists = true)
       command.run(sparkSession, child)
+      DataWritingCommand.propogateMetrics(sparkSession.sparkContext, command, metrics)
     } else {
       // TODO ideally, we should get the output data ready first and then
       // add the relation into catalog, just in case of failure occurs while data
       // processing.
+      val tableSchema = CharVarcharUtils.getRawSchema(outputColumns.toStructType)
       assert(tableDesc.schema.isEmpty)
       catalog.createTable(
-        tableDesc.copy(schema = outputColumns.toStructType), ignoreIfExists = false)
+        tableDesc.copy(schema = tableSchema), ignoreIfExists = false)
 
       try {
         // Read back the metadata of the table which was created just now.
         val createdTableMeta = catalog.getTableMetadata(tableDesc.identifier)
         val command = getWritingCommand(catalog, createdTableMeta, tableExists = false)
         command.run(sparkSession, child)
+        DataWritingCommand.propogateMetrics(sparkSession.sparkContext, command, metrics)
       } catch {
         case NonFatal(e) =>
           // drop the created table.
@@ -126,6 +130,9 @@ case class CreateHiveTableAsSelectCommand(
 
   override def writingCommandClassName: String =
     Utils.getSimpleName(classOf[InsertIntoHiveTable])
+
+  override protected def withNewChildInternal(
+    newChild: LogicalPlan): CreateHiveTableAsSelectCommand = copy(query = newChild)
 }
 
 /**
@@ -173,4 +180,7 @@ case class OptimizedCreateHiveTableAsSelectCommand(
 
   override def writingCommandClassName: String =
     Utils.getSimpleName(classOf[InsertIntoHadoopFsRelationCommand])
+
+  override protected def withNewChildInternal(
+    newChild: LogicalPlan): OptimizedCreateHiveTableAsSelectCommand = copy(query = newChild)
 }
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveFileFormat.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveFileFormat.scala
index c51c521cacba0..d4ec590f79f5e 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveFileFormat.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveFileFormat.scala
@@ -108,7 +108,7 @@ class HiveFileFormat(fileSinkConf: FileSinkDesc)
 }
 
 class HiveOutputWriter(
-    path: String,
+    val path: String,
     fileSinkConf: FileSinkDesc,
     jobConf: JobConf,
     dataSchema: StructType) extends OutputWriter with HiveInspectors {
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveOptions.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveOptions.scala
index 802ddafdbee4d..7b51618772edc 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveOptions.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveOptions.scala
@@ -87,7 +87,7 @@ class HiveOptions(@transient private val parameters: CaseInsensitiveMap[String])
 
   def serdeProperties: Map[String, String] = parameters.filterKeys {
     k => !lowerCasedOptionNames.contains(k.toLowerCase(Locale.ROOT))
-  }.map { case (k, v) => delimiterOptions.getOrElse(k, k) -> v }
+  }.map { case (k, v) => delimiterOptions.getOrElse(k, k) -> v }.toMap
 }
 
 object HiveOptions {
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala
new file mode 100644
index 0000000000000..219b1a27f70a2
--- /dev/null
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala
@@ -0,0 +1,316 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.execution
+
+import java.io._
+import java.util.Properties
+
+import scala.collection.JavaConverters._
+import scala.util.control.NonFatal
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.hive.ql.exec.{RecordReader, RecordWriter}
+import org.apache.hadoop.hive.serde.serdeConstants
+import org.apache.hadoop.hive.serde2.AbstractSerDe
+import org.apache.hadoop.hive.serde2.objectinspector._
+import org.apache.hadoop.io.Writable
+
+import org.apache.spark.TaskContext
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.execution._
+import org.apache.spark.sql.hive.HiveInspectors
+import org.apache.spark.sql.hive.HiveShim._
+import org.apache.spark.sql.types.DataType
+import org.apache.spark.util.{CircularBuffer, Utils}
+
+/**
+ * Transforms the input by forking and running the specified script.
+ *
+ * @param script the command that should be executed.
+ * @param output the attributes that are produced by the script.
+ * @param child logical plan whose output is transformed.
+ * @param ioschema the class set that defines how to handle input/output data.
+ */
+private[hive] case class HiveScriptTransformationExec(
+    script: String,
+    output: Seq[Attribute],
+    child: SparkPlan,
+    ioschema: ScriptTransformationIOSchema)
+  extends BaseScriptTransformationExec {
+  import HiveScriptIOSchema._
+
+  private def createOutputIteratorWithSerde(
+      writerThread: BaseScriptTransformationWriterThread,
+      inputStream: InputStream,
+      proc: Process,
+      stderrBuffer: CircularBuffer,
+      outputSerde: AbstractSerDe,
+      outputSoi: StructObjectInspector,
+      hadoopConf: Configuration): Iterator[InternalRow] = {
+    new Iterator[InternalRow] with HiveInspectors {
+      var curLine: String = null
+      val scriptOutputStream = new DataInputStream(inputStream)
+
+      val scriptOutputReader =
+        recordReader(ioschema, scriptOutputStream, hadoopConf).orNull
+
+      var scriptOutputWritable: Writable = null
+      val reusedWritableObject = outputSerde.getSerializedClass.getConstructor().newInstance()
+      val mutableRow = new SpecificInternalRow(output.map(_.dataType))
+
+      @transient
+      lazy val unwrappers = outputSoi.getAllStructFieldRefs.asScala.map(unwrapperFor)
+
+      override def hasNext: Boolean = {
+        try {
+          if (scriptOutputWritable == null) {
+            scriptOutputWritable = reusedWritableObject
+
+            if (scriptOutputReader != null) {
+              if (scriptOutputReader.next(scriptOutputWritable) <= 0) {
+                checkFailureAndPropagate(writerThread, null, proc, stderrBuffer)
+                return false
+              }
+            } else {
+              try {
+                scriptOutputWritable.readFields(scriptOutputStream)
+              } catch {
+                case _: EOFException =>
+                  // This means that the stdout of `proc` (i.e. TRANSFORM process) has exhausted.
+                  // Ideally the proc should *not* be alive at this point but
+                  // there can be a lag between EOF being written out and the process
+                  // being terminated. So explicitly waiting for the process to be done.
+                  checkFailureAndPropagate(writerThread, null, proc, stderrBuffer)
+                  return false
+              }
+            }
+          }
+
+          true
+        } catch {
+          case NonFatal(e) =>
+            // If this exception is due to abrupt / unclean termination of `proc`,
+            // then detect it and propagate a better exception message for end users
+            checkFailureAndPropagate(writerThread, e, proc, stderrBuffer)
+
+            throw e
+        }
+      }
+
+      override def next(): InternalRow = {
+        if (!hasNext) {
+          throw new NoSuchElementException
+        }
+        val raw = outputSerde.deserialize(scriptOutputWritable)
+        scriptOutputWritable = null
+        val dataList = outputSoi.getStructFieldsDataAsList(raw)
+        var i = 0
+        while (i < dataList.size()) {
+          if (dataList.get(i) == null) {
+            mutableRow.setNullAt(i)
+          } else {
+            unwrappers(i)(dataList.get(i), mutableRow, i)
+          }
+          i += 1
+        }
+        mutableRow
+      }
+    }
+  }
+
+  override def processIterator(
+      inputIterator: Iterator[InternalRow],
+      hadoopConf: Configuration): Iterator[InternalRow] = {
+
+    val (outputStream, proc, inputStream, stderrBuffer) = initProc
+
+    val (inputSerde, inputSoi) = initInputSerDe(ioschema, child.output).getOrElse((null, null))
+
+    // For HiveScriptTransformationExec, if inputSerde == null, but outputSerde != null
+    // We will use StringBuffer to pass data, in this case, we should cast data as string too.
+    val finalInput = if (inputSerde == null) {
+      inputExpressionsWithoutSerde
+    } else {
+      child.output
+    }
+
+    val outputProjection = new InterpretedProjection(finalInput, child.output)
+
+    // This new thread will consume the ScriptTransformation's input rows and write them to the
+    // external process. That process's output will be read by this current thread.
+    val writerThread = HiveScriptTransformationWriterThread(
+      inputIterator.map(outputProjection),
+      finalInput.map(_.dataType),
+      inputSerde,
+      inputSoi,
+      ioschema,
+      outputStream,
+      proc,
+      stderrBuffer,
+      TaskContext.get(),
+      hadoopConf
+    )
+
+    val (outputSerde, outputSoi) = {
+      initOutputSerDe(ioschema, output).getOrElse((null, null))
+    }
+
+    val outputIterator = if (outputSerde == null) {
+      createOutputIteratorWithoutSerde(writerThread, inputStream, proc, stderrBuffer)
+    } else {
+      createOutputIteratorWithSerde(
+        writerThread, inputStream, proc, stderrBuffer, outputSerde, outputSoi, hadoopConf)
+    }
+
+    writerThread.start()
+
+    outputIterator
+  }
+
+  override protected def withNewChildInternal(newChild: SparkPlan): HiveScriptTransformationExec =
+    copy(child = newChild)
+}
+
+private[hive] case class HiveScriptTransformationWriterThread(
+    iter: Iterator[InternalRow],
+    inputSchema: Seq[DataType],
+    inputSerde: AbstractSerDe,
+    inputSoi: StructObjectInspector,
+    ioSchema: ScriptTransformationIOSchema,
+    outputStream: OutputStream,
+    proc: Process,
+    stderrBuffer: CircularBuffer,
+    taskContext: TaskContext,
+    conf: Configuration)
+  extends BaseScriptTransformationWriterThread with HiveInspectors {
+  import HiveScriptIOSchema._
+
+  override def processRows(): Unit = {
+    val dataOutputStream = new DataOutputStream(outputStream)
+    val scriptInputWriter = recordWriter(ioSchema, dataOutputStream, conf).orNull
+
+    if (inputSerde == null) {
+      processRowsWithoutSerde()
+    } else {
+      // Convert Spark InternalRows to hive data via `HiveInspectors.wrapperFor`.
+      val hiveData = new Array[Any](inputSchema.length)
+      val fieldOIs = inputSoi.getAllStructFieldRefs.asScala.map(_.getFieldObjectInspector).toArray
+      val wrappers = fieldOIs.zip(inputSchema).map { case (f, dt) => wrapperFor(f, dt) }
+
+      iter.foreach { row =>
+        var i = 0
+        while (i < fieldOIs.length) {
+          hiveData(i) = if (row.isNullAt(i)) null else wrappers(i)(row.get(i, inputSchema(i)))
+          i += 1
+        }
+
+        val writable = inputSerde.serialize(hiveData, inputSoi)
+        if (scriptInputWriter != null) {
+          scriptInputWriter.write(writable)
+        } else {
+          prepareWritable(writable, ioSchema.outputSerdeProps).write(dataOutputStream)
+        }
+      }
+    }
+  }
+}
+
+object HiveScriptIOSchema extends HiveInspectors {
+
+  def initInputSerDe(
+      ioschema: ScriptTransformationIOSchema,
+      input: Seq[Expression]): Option[(AbstractSerDe, StructObjectInspector)] = {
+    ioschema.inputSerdeClass.map { serdeClass =>
+      val (columns, columnTypes) = parseAttrs(input)
+      val serde = initSerDe(serdeClass, columns, columnTypes, ioschema.inputSerdeProps)
+      val fieldObjectInspectors = columnTypes.map(toInspector)
+      val objectInspector = ObjectInspectorFactory
+        .getStandardStructObjectInspector(columns.asJava, fieldObjectInspectors.asJava)
+      (serde, objectInspector)
+    }
+  }
+
+  def initOutputSerDe(
+      ioschema: ScriptTransformationIOSchema,
+      output: Seq[Attribute]): Option[(AbstractSerDe, StructObjectInspector)] = {
+    ioschema.outputSerdeClass.map { serdeClass =>
+      val (columns, columnTypes) = parseAttrs(output)
+      val serde = initSerDe(serdeClass, columns, columnTypes, ioschema.outputSerdeProps)
+      val structObjectInspector = serde.getObjectInspector().asInstanceOf[StructObjectInspector]
+      (serde, structObjectInspector)
+    }
+  }
+
+  private def parseAttrs(attrs: Seq[Expression]): (Seq[String], Seq[DataType]) = {
+    val columns = attrs.zipWithIndex.map(e => s"${e._1.prettyName}_${e._2}")
+    val columnTypes = attrs.map(_.dataType)
+    (columns, columnTypes)
+  }
+
+  def initSerDe(
+      serdeClassName: String,
+      columns: Seq[String],
+      columnTypes: Seq[DataType],
+      serdeProps: Seq[(String, String)]): AbstractSerDe = {
+
+    val serde = Utils.classForName[AbstractSerDe](serdeClassName).getConstructor().
+      newInstance()
+
+    val columnTypesNames = columnTypes.map(_.toTypeInfo.getTypeName()).mkString(",")
+
+    var propsMap = serdeProps.toMap + (serdeConstants.LIST_COLUMNS -> columns.mkString(","))
+    propsMap = propsMap + (serdeConstants.LIST_COLUMN_TYPES -> columnTypesNames)
+
+    val properties = new Properties()
+    // Can not use properties.putAll(propsMap.asJava) in scala-2.12
+    // See https://github.com/scala/bug/issues/10418
+    propsMap.foreach { case (k, v) => properties.put(k, v) }
+    serde.initialize(null, properties)
+
+    serde
+  }
+
+  def recordReader(
+      ioschema: ScriptTransformationIOSchema,
+      inputStream: InputStream,
+      conf: Configuration): Option[RecordReader] = {
+    ioschema.recordReaderClass.map { klass =>
+      val instance = Utils.classForName[RecordReader](klass).getConstructor().
+        newInstance()
+      val props = new Properties()
+      // Can not use props.putAll(outputSerdeProps.toMap.asJava) in scala-2.12
+      // See https://github.com/scala/bug/issues/10418
+      ioschema.outputSerdeProps.toMap.foreach { case (k, v) => props.put(k, v) }
+      instance.initialize(inputStream, conf, props)
+      instance
+    }
+  }
+
+  def recordWriter(
+      ioschema: ScriptTransformationIOSchema,
+      outputStream: OutputStream,
+      conf: Configuration): Option[RecordWriter] = {
+    ioschema.recordWriterClass.map { klass =>
+      val instance = Utils.classForName[RecordWriter](klass).getConstructor().
+        newInstance()
+      instance.initialize(outputStream, conf)
+      instance
+    }
+  }
+}
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScanExec.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScanExec.scala
index 4dccacef337e9..41820b0135f4a 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScanExec.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScanExec.scala
@@ -156,7 +156,7 @@ case class HiveTableScanExec(
 
         // Only partitioned values are needed here, since the predicate has already been bound to
         // partition key attribute references.
-        val row = InternalRow.fromSeq(castedValues)
+        val row = InternalRow.fromSeq(castedValues.toSeq)
         shouldKeep.eval(row).asInstanceOf[Boolean]
       }
     }
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveDirCommand.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveDirCommand.scala
index b66c302a7d7ea..09aa1e8eea1f8 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveDirCommand.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveDirCommand.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable}
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.execution.SparkPlan
+import org.apache.spark.sql.execution.command.DDLUtils
 import org.apache.spark.sql.hive.client.HiveClientImpl
 import org.apache.spark.sql.util.SchemaUtils
 
@@ -63,12 +64,16 @@ case class InsertIntoHiveDirCommand(
       s"when inserting into ${storage.locationUri.get}",
       sparkSession.sessionState.conf.caseSensitiveAnalysis)
 
-    val hiveTable = HiveClientImpl.toHiveTable(CatalogTable(
+    val table = CatalogTable(
       identifier = TableIdentifier(storage.locationUri.get.toString, Some("default")),
+      provider = Some(DDLUtils.HIVE_PROVIDER),
       tableType = org.apache.spark.sql.catalyst.catalog.CatalogTableType.VIEW,
       storage = storage,
       schema = outputColumns.toStructType
-    ))
+    )
+    DDLUtils.checkDataColNames(table)
+
+    val hiveTable = HiveClientImpl.toHiveTable(table)
     hiveTable.getMetadata.put(serdeConstants.SERIALIZATION_LIB,
       storage.serde.getOrElse(classOf[LazySimpleSerDe].getName))
 
@@ -132,5 +137,8 @@ case class InsertIntoHiveDirCommand(
 
     Seq.empty[Row]
   }
+
+  override protected def withNewChildInternal(
+    newChild: LogicalPlan): InsertIntoHiveDirCommand = copy(query = newChild)
 }
 
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
index 19f439598142e..fcd11e67587cf 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
@@ -26,13 +26,16 @@ import org.apache.hadoop.hive.ql.plan.TableDesc
 
 import org.apache.spark.SparkException
 import org.apache.spark.sql.{AnalysisException, Row, SparkSession}
-import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType, ExternalCatalog, ExternalCatalogUtils}
+import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
 import org.apache.spark.sql.execution.SparkPlan
 import org.apache.spark.sql.execution.command.CommandUtils
+import org.apache.spark.sql.hive.HiveExternalCatalog
 import org.apache.spark.sql.hive.HiveShim.{ShimFileSinkDesc => FileSinkDesc}
 import org.apache.spark.sql.hive.client.HiveClientImpl
+import org.apache.spark.sql.hive.client.hive._
 
 
 /**
@@ -130,6 +133,7 @@ case class InsertIntoHiveTable(
     val numDynamicPartitions = partition.values.count(_.isEmpty)
     val numStaticPartitions = partition.values.count(_.nonEmpty)
     val partitionSpec = partition.map {
+      case (key, Some(null)) => key -> ExternalCatalogUtils.DEFAULT_PARTITION_NAME
       case (key, Some(value)) => key -> value
       case (key, None) => key -> ""
     }
@@ -169,7 +173,7 @@ case class InsertIntoHiveTable(
     table.bucketSpec match {
       case Some(bucketSpec) =>
         // Writes to bucketed hive tables are allowed only if user does not care about maintaining
-        // table's bucketing ie. both "hive.enforce.bucketing" and "hive.enforce.sorting" are
+        // table's bucketing i.e. both "hive.enforce.bucketing" and "hive.enforce.sorting" are
         // set to false
         val enforceBucketingConfig = "hive.enforce.bucketing"
         val enforceSortingConfig = "hive.enforce.sorting"
@@ -223,9 +227,13 @@ case class InsertIntoHiveTable(
                 ExternalCatalogUtils.unescapePathName(splitPart(1))
             }.toMap
 
+            val caseInsensitiveDpMap = CaseInsensitiveMap(dpMap)
+
             val updatedPartitionSpec = partition.map {
+              case (key, Some(null)) => key -> ExternalCatalogUtils.DEFAULT_PARTITION_NAME
               case (key, Some(value)) => key -> value
-              case (key, None) if dpMap.contains(key) => key -> dpMap(key)
+              case (key, None) if caseInsensitiveDpMap.contains(key) =>
+                key -> caseInsensitiveDpMap(key)
               case (key, _) =>
                 throw new SparkException(s"Dynamic partition key $key is not among " +
                   "written partition paths.")
@@ -239,7 +247,7 @@ case class InsertIntoHiveTable(
             if (fs.exists(partitionPath)) {
               if (!fs.delete(partitionPath, true)) {
                 throw new RuntimeException(
-                  "Cannot remove partition directory '" + partitionPath.toString)
+                  s"Cannot remove partition directory '$partitionPath'")
               }
             }
           }
@@ -285,13 +293,27 @@ case class InsertIntoHiveTable(
           // Newer Hive largely improves insert overwrite performance. As Spark uses older Hive
           // version and we may not want to catch up new Hive version every time. We delete the
           // Hive partition first and then load data file into the Hive partition.
-          if (partitionPath.nonEmpty && overwrite) {
+          val hiveVersion = externalCatalog.asInstanceOf[ExternalCatalogWithListener]
+            .unwrapped.asInstanceOf[HiveExternalCatalog]
+            .client
+            .version
+          // SPARK-31684:
+          // For Hive 2.0.0 and onwards, as https://issues.apache.org/jira/browse/HIVE-11940
+          // has been fixed, and there is no performance issue anymore. We should leave the
+          // overwrite logic to hive to avoid failure in `FileSystem#checkPath` when the table
+          // and partition locations do not belong to the same `FileSystem`
+          // TODO(SPARK-31675): For Hive 2.2.0 and earlier, if the table and partition locations
+          // do not belong together, we will still get the same error thrown by hive encryption
+          // check. see https://issues.apache.org/jira/browse/HIVE-14380.
+          // So we still disable for Hive overwrite for Hive 1.x for better performance because
+          // the partition and table are on the same cluster in most cases.
+          if (partitionPath.nonEmpty && overwrite && hiveVersion < v2_0) {
             partitionPath.foreach { path =>
               val fs = path.getFileSystem(hadoopConf)
               if (fs.exists(path)) {
                 if (!fs.delete(path, true)) {
                   throw new RuntimeException(
-                    "Cannot remove partition directory '" + path.toString)
+                    s"Cannot remove partition directory '$path'")
                 }
                 // Don't let Hive do overwrite operation since it is slower.
                 doHiveOverwrite = false
@@ -321,4 +343,7 @@ case class InsertIntoHiveTable(
         isSrcLocal = false)
     }
   }
+
+  override protected def withNewChildInternal(newChild: LogicalPlan): InsertIntoHiveTable =
+    copy(query = newChild)
 }
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/PruneHiveTablePartitions.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/PruneHiveTablePartitions.scala
index da6e4c52cf3a7..a831e8b8983cc 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/PruneHiveTablePartitions.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/PruneHiveTablePartitions.scala
@@ -21,13 +21,13 @@ import org.apache.hadoop.hive.common.StatsSetupConst
 
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.analysis.CastSupport
-import org.apache.spark.sql.catalyst.catalog.{CatalogStatistics, CatalogTable, CatalogTablePartition, ExternalCatalogUtils, HiveTableRelation}
-import org.apache.spark.sql.catalyst.expressions.{And, AttributeSet, Expression, ExpressionSet, SubqueryExpression}
+import org.apache.spark.sql.catalyst.catalog._
+import org.apache.spark.sql.catalyst.expressions.{And, AttributeSet, Expression, ExpressionSet, PredicateHelper, SubqueryExpression}
 import org.apache.spark.sql.catalyst.planning.PhysicalOperation
 import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project}
+import org.apache.spark.sql.catalyst.plans.logical.statsEstimation.FilterEstimation
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution.datasources.DataSourceStrategy
-import org.apache.spark.sql.internal.SQLConf
 
 /**
  * Prune hive table partitions using partition filters on [[HiveTableRelation]]. The pruned
@@ -35,15 +35,13 @@ import org.apache.spark.sql.internal.SQLConf
  * the hive table relation will be updated based on pruned partitions.
  *
  * This rule is executed in optimization phase, so the statistics can be updated before physical
- * planning, which is useful for some spark strategy, eg.
+ * planning, which is useful for some spark strategy, e.g.
  * [[org.apache.spark.sql.execution.SparkStrategies.JoinSelection]].
  *
  * TODO: merge this with PruneFileSourcePartitions after we completely make hive as a data source.
  */
 private[sql] class PruneHiveTablePartitions(session: SparkSession)
-  extends Rule[LogicalPlan] with CastSupport {
-
-  override val conf: SQLConf = session.sessionState.conf
+  extends Rule[LogicalPlan] with CastSupport with PredicateHelper {
 
   /**
    * Extract the partition filters from the filters on the table.
@@ -54,9 +52,8 @@ private[sql] class PruneHiveTablePartitions(session: SparkSession)
     val normalizedFilters = DataSourceStrategy.normalizeExprs(
       filters.filter(f => f.deterministic && !SubqueryExpression.hasSubquery(f)), relation.output)
     val partitionColumnSet = AttributeSet(relation.partitionCols)
-    ExpressionSet(normalizedFilters.filter { f =>
-      !f.references.isEmpty && f.references.subsetOf(partitionColumnSet)
-    })
+    ExpressionSet(
+      normalizedFilters.flatMap(extractPredicatesWithinOutputSet(_, partitionColumnSet)))
   }
 
   /**
@@ -79,8 +76,9 @@ private[sql] class PruneHiveTablePartitions(session: SparkSession)
    * Update the statistics of the table.
    */
   private def updateTableMeta(
-      tableMeta: CatalogTable,
-      prunedPartitions: Seq[CatalogTablePartition]): CatalogTable = {
+      relation: HiveTableRelation,
+      prunedPartitions: Seq[CatalogTablePartition],
+      partitionKeyFilters: ExpressionSet): CatalogTable = {
     val sizeOfPartitions = prunedPartitions.map { partition =>
       val rawDataSize = partition.parameters.get(StatsSetupConst.RAW_DATA_SIZE).map(_.toLong)
       val totalSize = partition.parameters.get(StatsSetupConst.TOTAL_SIZE).map(_.toLong)
@@ -93,10 +91,18 @@ private[sql] class PruneHiveTablePartitions(session: SparkSession)
       }
     }
     if (sizeOfPartitions.forall(_ > 0)) {
-      val sizeInBytes = sizeOfPartitions.sum
-      tableMeta.copy(stats = Some(CatalogStatistics(sizeInBytes = BigInt(sizeInBytes))))
+      val filteredStats =
+        FilterEstimation(Filter(partitionKeyFilters.reduce(And), relation)).estimate
+      val colStats = filteredStats.map(_.attributeStats.map { case (attr, colStat) =>
+        (attr.name, colStat.toCatalogColumnStat(attr.name, attr.dataType))
+      })
+      relation.tableMeta.copy(
+        stats = Some(CatalogStatistics(
+          sizeInBytes = BigInt(sizeOfPartitions.sum),
+          rowCount = filteredStats.flatMap(_.rowCount),
+          colStats = colStats.getOrElse(Map.empty))))
     } else {
-      tableMeta
+      relation.tableMeta
     }
   }
 
@@ -106,7 +112,7 @@ private[sql] class PruneHiveTablePartitions(session: SparkSession)
       val partitionKeyFilters = getPartitionKeyFilters(filters, relation)
       if (partitionKeyFilters.nonEmpty) {
         val newPartitions = prunePartitions(relation, partitionKeyFilters)
-        val newTableMeta = updateTableMeta(relation.tableMeta, newPartitions)
+        val newTableMeta = updateTableMeta(relation, newPartitions, partitionKeyFilters)
         val newRelation = relation.copy(
           tableMeta = newTableMeta, prunedPartitions = Some(newPartitions))
         // Keep partition filters so that they are visible in physical planning
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/SaveAsHiveFile.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/SaveAsHiveFile.scala
index 4be3cd45454c6..c712a4a2b7c23 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/SaveAsHiveFile.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/SaveAsHiveFile.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.hive.execution
 
-import java.io.{File, IOException}
+import java.io.IOException
 import java.net.URI
 import java.text.SimpleDateFormat
 import java.util.{Date, Locale, Random}
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformationExec.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformationExec.scala
deleted file mode 100644
index c7183fd7385a6..0000000000000
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformationExec.scala
+++ /dev/null
@@ -1,468 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.hive.execution
-
-import java.io._
-import java.nio.charset.StandardCharsets
-import java.util.Properties
-import java.util.concurrent.TimeUnit
-import javax.annotation.Nullable
-
-import scala.collection.JavaConverters._
-import scala.util.control.NonFatal
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.hive.ql.exec.{RecordReader, RecordWriter}
-import org.apache.hadoop.hive.serde.serdeConstants
-import org.apache.hadoop.hive.serde2.AbstractSerDe
-import org.apache.hadoop.hive.serde2.objectinspector._
-import org.apache.hadoop.io.Writable
-
-import org.apache.spark.{SparkException, TaskContext}
-import org.apache.spark.internal.Logging
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
-import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.plans.logical.ScriptInputOutputSchema
-import org.apache.spark.sql.catalyst.plans.physical.Partitioning
-import org.apache.spark.sql.execution._
-import org.apache.spark.sql.hive.HiveInspectors
-import org.apache.spark.sql.hive.HiveShim._
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.DataType
-import org.apache.spark.util.{CircularBuffer, RedirectThread, SerializableConfiguration, Utils}
-
-/**
- * Transforms the input by forking and running the specified script.
- *
- * @param input the set of expression that should be passed to the script.
- * @param script the command that should be executed.
- * @param output the attributes that are produced by the script.
- */
-case class ScriptTransformationExec(
-    input: Seq[Expression],
-    script: String,
-    output: Seq[Attribute],
-    child: SparkPlan,
-    ioschema: HiveScriptIOSchema)
-  extends UnaryExecNode {
-
-  override def producedAttributes: AttributeSet = outputSet -- inputSet
-
-  override def outputPartitioning: Partitioning = child.outputPartitioning
-
-  protected override def doExecute(): RDD[InternalRow] = {
-    def processIterator(inputIterator: Iterator[InternalRow], hadoopConf: Configuration)
-      : Iterator[InternalRow] = {
-      val cmd = List("/bin/bash", "-c", script)
-      val builder = new ProcessBuilder(cmd.asJava)
-
-      val proc = builder.start()
-      val inputStream = proc.getInputStream
-      val outputStream = proc.getOutputStream
-      val errorStream = proc.getErrorStream
-
-      // In order to avoid deadlocks, we need to consume the error output of the child process.
-      // To avoid issues caused by large error output, we use a circular buffer to limit the amount
-      // of error output that we retain. See SPARK-7862 for more discussion of the deadlock / hang
-      // that motivates this.
-      val stderrBuffer = new CircularBuffer(2048)
-      new RedirectThread(
-        errorStream,
-        stderrBuffer,
-        "Thread-ScriptTransformation-STDERR-Consumer").start()
-
-      val outputProjection = new InterpretedProjection(input, child.output)
-
-      // This nullability is a performance optimization in order to avoid an Option.foreach() call
-      // inside of a loop
-      @Nullable val (inputSerde, inputSoi) = ioschema.initInputSerDe(input).getOrElse((null, null))
-
-      // This new thread will consume the ScriptTransformation's input rows and write them to the
-      // external process. That process's output will be read by this current thread.
-      val writerThread = new ScriptTransformationWriterThread(
-        inputIterator.map(outputProjection),
-        input.map(_.dataType),
-        inputSerde,
-        inputSoi,
-        ioschema,
-        outputStream,
-        proc,
-        stderrBuffer,
-        TaskContext.get(),
-        hadoopConf
-      )
-
-      // This nullability is a performance optimization in order to avoid an Option.foreach() call
-      // inside of a loop
-      @Nullable val (outputSerde, outputSoi) = {
-        ioschema.initOutputSerDe(output).getOrElse((null, null))
-      }
-
-      val reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))
-      val outputIterator: Iterator[InternalRow] = new Iterator[InternalRow] with HiveInspectors {
-        var curLine: String = null
-        val scriptOutputStream = new DataInputStream(inputStream)
-
-        @Nullable val scriptOutputReader =
-          ioschema.recordReader(scriptOutputStream, hadoopConf).orNull
-
-        var scriptOutputWritable: Writable = null
-        val reusedWritableObject: Writable = if (null != outputSerde) {
-          outputSerde.getSerializedClass().getConstructor().newInstance()
-        } else {
-          null
-        }
-        val mutableRow = new SpecificInternalRow(output.map(_.dataType))
-
-        @transient
-        lazy val unwrappers = outputSoi.getAllStructFieldRefs.asScala.map(unwrapperFor)
-
-        private def checkFailureAndPropagate(cause: Throwable = null): Unit = {
-          if (writerThread.exception.isDefined) {
-            throw writerThread.exception.get
-          }
-
-          // There can be a lag between reader read EOF and the process termination.
-          // If the script fails to startup, this kind of error may be missed.
-          // So explicitly waiting for the process termination.
-          val timeout = conf.getConf(SQLConf.SCRIPT_TRANSFORMATION_EXIT_TIMEOUT)
-          val exitRes = proc.waitFor(timeout, TimeUnit.SECONDS)
-          if (!exitRes) {
-            log.warn(s"Transformation script process exits timeout in $timeout seconds")
-          }
-
-          if (!proc.isAlive) {
-            val exitCode = proc.exitValue()
-            if (exitCode != 0) {
-              logError(stderrBuffer.toString) // log the stderr circular buffer
-              throw new SparkException(s"Subprocess exited with status $exitCode. " +
-                s"Error: ${stderrBuffer.toString}", cause)
-            }
-          }
-        }
-
-        override def hasNext: Boolean = {
-          try {
-            if (outputSerde == null) {
-              if (curLine == null) {
-                curLine = reader.readLine()
-                if (curLine == null) {
-                  checkFailureAndPropagate()
-                  return false
-                }
-              }
-            } else if (scriptOutputWritable == null) {
-              scriptOutputWritable = reusedWritableObject
-
-              if (scriptOutputReader != null) {
-                if (scriptOutputReader.next(scriptOutputWritable) <= 0) {
-                  checkFailureAndPropagate()
-                  return false
-                }
-              } else {
-                try {
-                  scriptOutputWritable.readFields(scriptOutputStream)
-                } catch {
-                  case _: EOFException =>
-                    // This means that the stdout of `proc` (ie. TRANSFORM process) has exhausted.
-                    // Ideally the proc should *not* be alive at this point but
-                    // there can be a lag between EOF being written out and the process
-                    // being terminated. So explicitly waiting for the process to be done.
-                    checkFailureAndPropagate()
-                    return false
-                }
-              }
-            }
-
-            true
-          } catch {
-            case NonFatal(e) =>
-              // If this exception is due to abrupt / unclean termination of `proc`,
-              // then detect it and propagate a better exception message for end users
-              checkFailureAndPropagate(e)
-
-              throw e
-          }
-        }
-
-        override def next(): InternalRow = {
-          if (!hasNext) {
-            throw new NoSuchElementException
-          }
-          if (outputSerde == null) {
-            val prevLine = curLine
-            curLine = reader.readLine()
-            if (!ioschema.schemaLess) {
-              new GenericInternalRow(
-                prevLine.split(ioschema.outputRowFormatMap("TOK_TABLEROWFORMATFIELD"))
-                  .map(CatalystTypeConverters.convertToCatalyst))
-            } else {
-              new GenericInternalRow(
-                prevLine.split(ioschema.outputRowFormatMap("TOK_TABLEROWFORMATFIELD"), 2)
-                  .map(CatalystTypeConverters.convertToCatalyst))
-            }
-          } else {
-            val raw = outputSerde.deserialize(scriptOutputWritable)
-            scriptOutputWritable = null
-            val dataList = outputSoi.getStructFieldsDataAsList(raw)
-            var i = 0
-            while (i < dataList.size()) {
-              if (dataList.get(i) == null) {
-                mutableRow.setNullAt(i)
-              } else {
-                unwrappers(i)(dataList.get(i), mutableRow, i)
-              }
-              i += 1
-            }
-            mutableRow
-          }
-        }
-      }
-
-      writerThread.start()
-
-      outputIterator
-    }
-
-    val broadcastedHadoopConf =
-      new SerializableConfiguration(sqlContext.sessionState.newHadoopConf())
-
-    child.execute().mapPartitions { iter =>
-      if (iter.hasNext) {
-        val proj = UnsafeProjection.create(schema)
-        processIterator(iter, broadcastedHadoopConf.value).map(proj)
-      } else {
-        // If the input iterator has no rows then do not launch the external script.
-        Iterator.empty
-      }
-    }
-  }
-}
-
-private class ScriptTransformationWriterThread(
-    iter: Iterator[InternalRow],
-    inputSchema: Seq[DataType],
-    @Nullable inputSerde: AbstractSerDe,
-    @Nullable inputSoi: StructObjectInspector,
-    ioschema: HiveScriptIOSchema,
-    outputStream: OutputStream,
-    proc: Process,
-    stderrBuffer: CircularBuffer,
-    taskContext: TaskContext,
-    conf: Configuration
-  ) extends Thread("Thread-ScriptTransformation-Feed") with HiveInspectors with Logging {
-
-  setDaemon(true)
-
-  @volatile private var _exception: Throwable = null
-
-  /** Contains the exception thrown while writing the parent iterator to the external process. */
-  def exception: Option[Throwable] = Option(_exception)
-
-  override def run(): Unit = Utils.logUncaughtExceptions {
-    TaskContext.setTaskContext(taskContext)
-
-    val dataOutputStream = new DataOutputStream(outputStream)
-    @Nullable val scriptInputWriter = ioschema.recordWriter(dataOutputStream, conf).orNull
-
-    // We can't use Utils.tryWithSafeFinally here because we also need a `catch` block, so
-    // let's use a variable to record whether the `finally` block was hit due to an exception
-    var threwException: Boolean = true
-    val len = inputSchema.length
-    try {
-      if (inputSerde == null) {
-        iter.foreach { row =>
-          val data = if (len == 0) {
-            ioschema.inputRowFormatMap("TOK_TABLEROWFORMATLINES")
-          } else {
-            val sb = new StringBuilder
-            sb.append(row.get(0, inputSchema(0)))
-            var i = 1
-            while (i < len) {
-              sb.append(ioschema.inputRowFormatMap("TOK_TABLEROWFORMATFIELD"))
-              sb.append(row.get(i, inputSchema(i)))
-              i += 1
-            }
-            sb.append(ioschema.inputRowFormatMap("TOK_TABLEROWFORMATLINES"))
-            sb.toString()
-          }
-          outputStream.write(data.getBytes(StandardCharsets.UTF_8))
-        }
-      } else {
-        // Convert Spark InternalRows to hive data via `HiveInspectors.wrapperFor`.
-        val hiveData = new Array[Any](inputSchema.length)
-        val fieldOIs = inputSoi.getAllStructFieldRefs.asScala.map(_.getFieldObjectInspector).toArray
-        val wrappers = fieldOIs.zip(inputSchema).map { case (f, dt) => wrapperFor(f, dt) }
-
-        iter.foreach { row =>
-          var i = 0
-          while (i < fieldOIs.length) {
-            hiveData(i) = if (row.isNullAt(i)) null else wrappers(i)(row.get(i, inputSchema(i)))
-            i += 1
-          }
-
-          val writable = inputSerde.serialize(hiveData, inputSoi)
-          if (scriptInputWriter != null) {
-            scriptInputWriter.write(writable)
-          } else {
-            prepareWritable(writable, ioschema.outputSerdeProps).write(dataOutputStream)
-          }
-        }
-      }
-      threwException = false
-    } catch {
-      // SPARK-25158 Exception should not be thrown again, otherwise it will be captured by
-      // SparkUncaughtExceptionHandler, then Executor will exit because of this Uncaught Exception,
-      // so pass the exception to `ScriptTransformationExec` is enough.
-      case t: Throwable =>
-        // An error occurred while writing input, so kill the child process. According to the
-        // Javadoc this call will not throw an exception:
-        _exception = t
-        proc.destroy()
-        logError("Thread-ScriptTransformation-Feed exit cause by: ", t)
-    } finally {
-      try {
-        Utils.tryLogNonFatalError(outputStream.close())
-        if (proc.waitFor() != 0) {
-          logError(stderrBuffer.toString) // log the stderr circular buffer
-        }
-      } catch {
-        case NonFatal(exceptionFromFinallyBlock) =>
-          if (!threwException) {
-            throw exceptionFromFinallyBlock
-          } else {
-            log.error("Exception in finally block", exceptionFromFinallyBlock)
-          }
-      }
-    }
-  }
-}
-
-object HiveScriptIOSchema {
-  def apply(input: ScriptInputOutputSchema): HiveScriptIOSchema = {
-    HiveScriptIOSchema(
-      input.inputRowFormat,
-      input.outputRowFormat,
-      input.inputSerdeClass,
-      input.outputSerdeClass,
-      input.inputSerdeProps,
-      input.outputSerdeProps,
-      input.recordReaderClass,
-      input.recordWriterClass,
-      input.schemaLess)
-  }
-}
-
-/**
- * The wrapper class of Hive input and output schema properties
- */
-case class HiveScriptIOSchema (
-    inputRowFormat: Seq[(String, String)],
-    outputRowFormat: Seq[(String, String)],
-    inputSerdeClass: Option[String],
-    outputSerdeClass: Option[String],
-    inputSerdeProps: Seq[(String, String)],
-    outputSerdeProps: Seq[(String, String)],
-    recordReaderClass: Option[String],
-    recordWriterClass: Option[String],
-    schemaLess: Boolean)
-  extends HiveInspectors {
-
-  private val defaultFormat = Map(
-    ("TOK_TABLEROWFORMATFIELD", "\t"),
-    ("TOK_TABLEROWFORMATLINES", "\n")
-  )
-
-  val inputRowFormatMap = inputRowFormat.toMap.withDefault((k) => defaultFormat(k))
-  val outputRowFormatMap = outputRowFormat.toMap.withDefault((k) => defaultFormat(k))
-
-
-  def initInputSerDe(input: Seq[Expression]): Option[(AbstractSerDe, StructObjectInspector)] = {
-    inputSerdeClass.map { serdeClass =>
-      val (columns, columnTypes) = parseAttrs(input)
-      val serde = initSerDe(serdeClass, columns, columnTypes, inputSerdeProps)
-      val fieldObjectInspectors = columnTypes.map(toInspector)
-      val objectInspector = ObjectInspectorFactory
-        .getStandardStructObjectInspector(columns.asJava, fieldObjectInspectors.asJava)
-      (serde, objectInspector)
-    }
-  }
-
-  def initOutputSerDe(output: Seq[Attribute]): Option[(AbstractSerDe, StructObjectInspector)] = {
-    outputSerdeClass.map { serdeClass =>
-      val (columns, columnTypes) = parseAttrs(output)
-      val serde = initSerDe(serdeClass, columns, columnTypes, outputSerdeProps)
-      val structObjectInspector = serde.getObjectInspector().asInstanceOf[StructObjectInspector]
-      (serde, structObjectInspector)
-    }
-  }
-
-  private def parseAttrs(attrs: Seq[Expression]): (Seq[String], Seq[DataType]) = {
-    val columns = attrs.zipWithIndex.map(e => s"${e._1.prettyName}_${e._2}")
-    val columnTypes = attrs.map(_.dataType)
-    (columns, columnTypes)
-  }
-
-  private def initSerDe(
-      serdeClassName: String,
-      columns: Seq[String],
-      columnTypes: Seq[DataType],
-      serdeProps: Seq[(String, String)]): AbstractSerDe = {
-
-    val serde = Utils.classForName[AbstractSerDe](serdeClassName).getConstructor().
-      newInstance()
-
-    val columnTypesNames = columnTypes.map(_.toTypeInfo.getTypeName()).mkString(",")
-
-    var propsMap = serdeProps.toMap + (serdeConstants.LIST_COLUMNS -> columns.mkString(","))
-    propsMap = propsMap + (serdeConstants.LIST_COLUMN_TYPES -> columnTypesNames)
-
-    val properties = new Properties()
-    // Can not use properties.putAll(propsMap.asJava) in scala-2.12
-    // See https://github.com/scala/bug/issues/10418
-    propsMap.foreach { case (k, v) => properties.put(k, v) }
-    serde.initialize(null, properties)
-
-    serde
-  }
-
-  def recordReader(
-      inputStream: InputStream,
-      conf: Configuration): Option[RecordReader] = {
-    recordReaderClass.map { klass =>
-      val instance = Utils.classForName[RecordReader](klass).getConstructor().
-        newInstance()
-      val props = new Properties()
-      // Can not use props.putAll(outputSerdeProps.toMap.asJava) in scala-2.12
-      // See https://github.com/scala/bug/issues/10418
-      outputSerdeProps.toMap.foreach { case (k, v) => props.put(k, v) }
-      instance.initialize(inputStream, conf, props)
-      instance
-    }
-  }
-
-  def recordWriter(outputStream: OutputStream, conf: Configuration): Option[RecordWriter] = {
-    recordWriterClass.map { klass =>
-      val instance = Utils.classForName[RecordWriter](klass).getConstructor().
-        newInstance()
-      instance.initialize(outputStream, conf)
-      instance
-    }
-  }
-}
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
index 05d608a2016a5..7c3d1617bfaeb 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
@@ -17,7 +17,6 @@
 
 package org.apache.spark.sql.hive
 
-import java.lang.{Boolean => JBoolean}
 import java.nio.ByteBuffer
 
 import scala.collection.JavaConverters._
@@ -39,8 +38,12 @@ import org.apache.spark.sql.catalyst.expressions.aggregate._
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
 import org.apache.spark.sql.hive.HiveShim._
 import org.apache.spark.sql.types._
-import org.apache.spark.util.Utils
 
+/**
+ * Here we cannot extends `ImplicitTypeCasts` to compatible with UDF input data type, the reason is:
+ * we use children data type to reflect UDF method first and will get exception if it fails so that
+ * we can never go into `ImplicitTypeCasts`.
+ */
 private[hive] case class HiveSimpleUDF(
     name: String, funcWrapper: HiveFunctionWrapper, children: Seq[Expression])
   extends Expression
@@ -107,6 +110,9 @@ private[hive] case class HiveSimpleUDF(
   override def prettyName: String = name
 
   override def sql: String = s"$name(${children.map(_.sql).mkString(", ")})"
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
+    copy(children = newChildren)
 }
 
 // Adapter from Catalyst ExpressionResult to Hive DeferredObject
@@ -183,6 +189,9 @@ private[hive] case class HiveGenericUDF(
   override def toString: String = {
     s"$nodeName#${funcWrapper.functionClassName}(${children.mkString(",")})"
   }
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
+    copy(children = newChildren)
 }
 
 /**
@@ -210,10 +219,14 @@ private[hive] case class HiveGenericUDTF(
   }
 
   @transient
-  protected lazy val inputInspectors = children.map(toInspector)
+  protected lazy val inputInspector = {
+    val inspectors = children.map(toInspector)
+    val fields = inspectors.indices.map(index => s"_col$index").asJava
+    ObjectInspectorFactory.getStandardStructObjectInspector(fields, inspectors.asJava)
+  }
 
   @transient
-  protected lazy val outputInspector = function.initialize(inputInspectors.toArray)
+  protected lazy val outputInspector = function.initialize(inputInspector)
 
   @transient
   protected lazy val udtInput = new Array[AnyRef](children.length)
@@ -224,7 +237,7 @@ private[hive] case class HiveGenericUDTF(
   override lazy val elementSchema = StructType(outputInspector.getAllStructFieldRefs.asScala.map {
     field => StructField(field.getFieldName, inspectorToDataType(field.getFieldObjectInspector),
       nullable = true)
-  })
+  }.toSeq)
 
   @transient
   private lazy val inputDataTypes: Array[DataType] = children.map(_.dataType).toArray
@@ -257,7 +270,7 @@ private[hive] case class HiveGenericUDTF(
     def collectRows(): Seq[InternalRow] = {
       val toCollect = collected
       collected = new ArrayBuffer[InternalRow]
-      toCollect
+      toCollect.toSeq
     }
   }
 
@@ -272,6 +285,9 @@ private[hive] case class HiveGenericUDTF(
   }
 
   override def prettyName: String = name
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
+    copy(children = newChildren)
 }
 
 /**
@@ -344,20 +360,8 @@ private[hive] case class HiveUDAFFunction(
       funcWrapper.createFunction[AbstractGenericUDAFResolver]()
     }
 
-    val clazz = Utils.classForName(classOf[SimpleGenericUDAFParameterInfo].getName)
-    if (HiveUtils.isHive23) {
-      val ctor = clazz.getDeclaredConstructor(
-        classOf[Array[ObjectInspector]], JBoolean.TYPE, JBoolean.TYPE, JBoolean.TYPE)
-      val args = Array[AnyRef](inputInspectors, JBoolean.FALSE, JBoolean.FALSE, JBoolean.FALSE)
-      val parameterInfo = ctor.newInstance(args: _*).asInstanceOf[SimpleGenericUDAFParameterInfo]
-      resolver.getEvaluator(parameterInfo)
-    } else {
-      val ctor = clazz.getDeclaredConstructor(
-        classOf[Array[ObjectInspector]], JBoolean.TYPE, JBoolean.TYPE)
-      val args = Array[AnyRef](inputInspectors, JBoolean.FALSE, JBoolean.FALSE)
-      val parameterInfo = ctor.newInstance(args: _*).asInstanceOf[SimpleGenericUDAFParameterInfo]
-      resolver.getEvaluator(parameterInfo)
-    }
+    val parameterInfo = new SimpleGenericUDAFParameterInfo(inputInspectors, false, false, false)
+    resolver.getEvaluator(parameterInfo)
   }
 
   private case class HiveEvaluator(
@@ -533,6 +537,9 @@ private[hive] case class HiveUDAFFunction(
       buffer
     }
   }
+
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
+    copy(children = newChildren)
 }
 
 case class HiveUDAFBuffer(buf: AggregationBuffer, canDoMerge: Boolean)
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFileFormat.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFileFormat.scala
index 7f2eb14956dc1..d2ac06ad0a16a 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFileFormat.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFileFormat.scala
@@ -32,12 +32,14 @@ import org.apache.hadoop.fs.{FileStatus, Path}
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars
 import org.apache.hadoop.hive.ql.io.orc._
 import org.apache.hadoop.hive.ql.io.sarg.SearchArgument
+import org.apache.hadoop.hive.serde2.objectinspector
 import org.apache.hadoop.hive.serde2.objectinspector.{SettableStructObjectInspector, StructObjectInspector}
 import org.apache.hadoop.hive.serde2.typeinfo.{StructTypeInfo, TypeInfoUtils}
 import org.apache.hadoop.io.{NullWritable, Writable}
 import org.apache.hadoop.mapred.{JobConf, OutputFormat => MapRedOutputFormat, RecordWriter, Reporter}
 import org.apache.hadoop.mapreduce._
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat
+import org.apache.orc.OrcConf
 import org.apache.orc.OrcConf.COMPRESS
 
 import org.apache.spark.{SPARK_VERSION_SHORT, TaskContext}
@@ -47,7 +49,7 @@ import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.execution.datasources._
-import org.apache.spark.sql.execution.datasources.orc.OrcOptions
+import org.apache.spark.sql.execution.datasources.orc.{OrcFilters, OrcOptions}
 import org.apache.spark.sql.hive.{HiveInspectors, HiveShim}
 import org.apache.spark.sql.sources.{Filter, _}
 import org.apache.spark.sql.types._
@@ -70,14 +72,12 @@ class OrcFileFormat extends FileFormat with DataSourceRegister with Serializable
     val orcOptions = new OrcOptions(options, sparkSession.sessionState.conf)
     if (orcOptions.mergeSchema) {
       SchemaMergeUtils.mergeSchemasInParallel(
-        sparkSession,
-        files,
-        OrcFileOperator.readOrcSchemasInParallel)
+        sparkSession, options, files, OrcFileOperator.readOrcSchemasInParallel)
     } else {
       val ignoreCorruptFiles = sparkSession.sessionState.conf.ignoreCorruptFiles
       OrcFileOperator.readSchema(
         files.map(_.getPath.toString),
-        Some(sparkSession.sessionState.newHadoopConf()),
+        Some(sparkSession.sessionState.newHadoopConfWithOptions(options)),
         ignoreCorruptFiles
       )
     }
@@ -141,7 +141,7 @@ class OrcFileFormat extends FileFormat with DataSourceRegister with Serializable
 
     if (sparkSession.sessionState.conf.orcFilterPushDown) {
       // Sets pushed predicates
-      OrcFilters.createFilter(requiredSchema, filters.toArray).foreach { f =>
+      OrcFilters.createFilter(requiredSchema, filters).foreach { f =>
         hadoopConf.set(OrcFileFormat.SARG_PUSHDOWN, toKryo(f))
         hadoopConf.setBoolean(ConfVars.HIVEOPTINDEXFILTER.varname, true)
       }
@@ -271,7 +271,7 @@ private[orc] class OrcSerializer(dataSchema: StructType, conf: Configuration)
 }
 
 private[orc] class OrcOutputWriter(
-    path: String,
+    val path: String,
     dataSchema: StructType,
     context: TaskAttemptContext)
   extends OutputWriter {
@@ -298,7 +298,7 @@ private[orc] class OrcOutputWriter(
 
   override def close(): Unit = {
     if (recordWriterInstantiated) {
-      // Hive 1.2.1 ORC initializes its private `writer` field at the first write.
+      // Hive ORC initializes its private `writer` field at the first write.
       OrcFileFormat.addSparkVersionMetadata(recordWriter)
       recordWriter.close(Reporter.NULL)
     }
@@ -325,13 +325,19 @@ private[orc] object OrcFileFormat extends HiveInspectors with Logging {
     val deserializer = new OrcSerde
     val mutableRow = new SpecificInternalRow(requiredSchema.map(_.dataType))
     val unsafeProjection = UnsafeProjection.create(requiredSchema)
+    val forcePositionalEvolution = OrcConf.FORCE_POSITIONAL_EVOLUTION.getBoolean(conf)
 
     def unwrap(oi: StructObjectInspector): Iterator[InternalRow] = {
       val (fieldRefs, fieldOrdinals) = requiredSchema.zipWithIndex.map {
         case (field, ordinal) =>
-          var ref = oi.getStructFieldRef(field.name)
-          if (ref == null) {
-            ref = oi.getStructFieldRef("_col" + dataSchema.fieldIndex(field.name))
+          var ref: objectinspector.StructField = null
+          if (forcePositionalEvolution) {
+            ref = oi.getAllStructFieldRefs.get(dataSchema.fieldIndex(field.name))
+          } else {
+            ref = oi.getStructFieldRef(field.name)
+            if (ref == null) {
+              ref = oi.getStructFieldRef("_col" + dataSchema.fieldIndex(field.name))
+            }
           }
           ref -> ordinal
       }.unzip
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala
deleted file mode 100644
index f9c514567c639..0000000000000
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala
+++ /dev/null
@@ -1,262 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.hive.orc
-
-import java.lang.reflect.Method
-
-import org.apache.hadoop.hive.ql.io.sarg.SearchArgument
-import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.Builder
-import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory.newBuilder
-
-import org.apache.spark.SparkException
-import org.apache.spark.internal.Logging
-import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.quoteIfNeeded
-import org.apache.spark.sql.execution.datasources.orc.{OrcFilters => DatasourceOrcFilters}
-import org.apache.spark.sql.execution.datasources.orc.OrcFilters.buildTree
-import org.apache.spark.sql.hive.HiveUtils
-import org.apache.spark.sql.sources._
-import org.apache.spark.sql.types._
-
-/**
- * Helper object for building ORC `SearchArgument`s, which are used for ORC predicate push-down.
- *
- * Due to limitation of ORC `SearchArgument` builder, we had to end up with a pretty weird double-
- * checking pattern when converting `And`/`Or`/`Not` filters.
- *
- * An ORC `SearchArgument` must be built in one pass using a single builder.  For example, you can't
- * build `a = 1` and `b = 2` first, and then combine them into `a = 1 AND b = 2`.  This is quite
- * different from the cases in Spark SQL or Parquet, where complex filters can be easily built using
- * existing simpler ones.
- *
- * The annoying part is that, `SearchArgument` builder methods like `startAnd()`, `startOr()`, and
- * `startNot()` mutate internal state of the builder instance.  This forces us to translate all
- * convertible filters with a single builder instance. However, before actually converting a filter,
- * we've no idea whether it can be recognized by ORC or not. Thus, when an inconvertible filter is
- * found, we may already end up with a builder whose internal state is inconsistent.
- *
- * For example, to convert an `And` filter with builder `b`, we call `b.startAnd()` first, and then
- * try to convert its children.  Say we convert `left` child successfully, but find that `right`
- * child is inconvertible.  Alas, `b.startAnd()` call can't be rolled back, and `b` is inconsistent
- * now.
- *
- * The workaround employed here is that, for `And`/`Or`/`Not`, we first try to convert their
- * children with brand new builders, and only do the actual conversion with the right builder
- * instance when the children are proven to be convertible.
- *
- * P.S.: Hive seems to use `SearchArgument` together with `ExprNodeGenericFuncDesc` only.  Usage of
- * builder methods mentioned above can only be found in test code, where all tested filters are
- * known to be convertible.
- */
-private[orc] object OrcFilters extends Logging {
-
-  private def findMethod(klass: Class[_], name: String, args: Class[_]*): Method = {
-    val method = klass.getMethod(name, args: _*)
-    method.setAccessible(true)
-    method
-  }
-
-  def createFilter(schema: StructType, filters: Array[Filter]): Option[SearchArgument] = {
-    if (HiveUtils.isHive23) {
-      DatasourceOrcFilters.createFilter(schema, filters).asInstanceOf[Option[SearchArgument]]
-    } else {
-      val dataTypeMap = schema.map(f => quoteIfNeeded(f.name) -> f.dataType).toMap
-      // TODO (SPARK-25557): ORC doesn't support nested predicate pushdown, so they are removed.
-      val newFilters = filters.filter(!_.containsNestedColumn)
-      // Combines all convertible filters using `And` to produce a single conjunction
-      val conjunctionOptional = buildTree(convertibleFilters(schema, dataTypeMap, newFilters))
-      conjunctionOptional.map { conjunction =>
-        // Then tries to build a single ORC `SearchArgument` for the conjunction predicate.
-        // The input predicate is fully convertible. There should not be any empty result in the
-        // following recursive method call `buildSearchArgument`.
-        buildSearchArgument(dataTypeMap, conjunction, newBuilder).build()
-      }
-    }
-  }
-
-  def convertibleFilters(
-      schema: StructType,
-      dataTypeMap: Map[String, DataType],
-      filters: Seq[Filter]): Seq[Filter] = {
-    import org.apache.spark.sql.sources._
-
-    def convertibleFiltersHelper(
-        filter: Filter,
-        canPartialPushDown: Boolean): Option[Filter] = filter match {
-      // At here, it is not safe to just convert one side and remove the other side
-      // if we do not understand what the parent filters are.
-      //
-      // Here is an example used to explain the reason.
-      // Let's say we have NOT(a = 2 AND b in ('1')) and we do not understand how to
-      // convert b in ('1'). If we only convert a = 2, we will end up with a filter
-      // NOT(a = 2), which will generate wrong results.
-      //
-      // Pushing one side of AND down is only safe to do at the top level or in the child
-      // AND before hitting NOT or OR conditions, and in this case, the unsupported predicate
-      // can be safely removed.
-      case And(left, right) =>
-        val leftResultOptional = convertibleFiltersHelper(left, canPartialPushDown)
-        val rightResultOptional = convertibleFiltersHelper(right, canPartialPushDown)
-        (leftResultOptional, rightResultOptional) match {
-          case (Some(leftResult), Some(rightResult)) => Some(And(leftResult, rightResult))
-          case (Some(leftResult), None) if canPartialPushDown => Some(leftResult)
-          case (None, Some(rightResult)) if canPartialPushDown => Some(rightResult)
-          case _ => None
-        }
-
-      // The Or predicate is convertible when both of its children can be pushed down.
-      // That is to say, if one/both of the children can be partially pushed down, the Or
-      // predicate can be partially pushed down as well.
-      //
-      // Here is an example used to explain the reason.
-      // Let's say we have
-      // (a1 AND a2) OR (b1 AND b2),
-      // a1 and b1 is convertible, while a2 and b2 is not.
-      // The predicate can be converted as
-      // (a1 OR b1) AND (a1 OR b2) AND (a2 OR b1) AND (a2 OR b2)
-      // As per the logical in And predicate, we can push down (a1 OR b1).
-      case Or(left, right) =>
-        for {
-          lhs <- convertibleFiltersHelper(left, canPartialPushDown)
-          rhs <- convertibleFiltersHelper(right, canPartialPushDown)
-        } yield Or(lhs, rhs)
-      case Not(pred) =>
-        val childResultOptional = convertibleFiltersHelper(pred, canPartialPushDown = false)
-        childResultOptional.map(Not)
-      case other =>
-        for (_ <- buildLeafSearchArgument(dataTypeMap, other, newBuilder())) yield other
-    }
-    filters.flatMap { filter =>
-      convertibleFiltersHelper(filter, true)
-    }
-  }
-
-  /**
-   * Build a SearchArgument and return the builder so far.
-   *
-   * @param dataTypeMap a map from the attribute name to its data type.
-   * @param expression the input predicates, which should be fully convertible to SearchArgument.
-   * @param builder the input SearchArgument.Builder.
-   * @return the builder so far.
-   */
-  private def buildSearchArgument(
-      dataTypeMap: Map[String, DataType],
-      expression: Filter,
-      builder: Builder): Builder = {
-    expression match {
-      case And(left, right) =>
-        val lhs = buildSearchArgument(dataTypeMap, left, builder.startAnd())
-        val rhs = buildSearchArgument(dataTypeMap, right, lhs)
-        rhs.end()
-
-      case Or(left, right) =>
-        val lhs = buildSearchArgument(dataTypeMap, left, builder.startOr())
-        val rhs = buildSearchArgument(dataTypeMap, right, lhs)
-        rhs.end()
-
-      case Not(child) =>
-        buildSearchArgument(dataTypeMap, child, builder.startNot()).end()
-
-      case other =>
-        buildLeafSearchArgument(dataTypeMap, other, builder).getOrElse {
-          throw new SparkException(
-            "The input filter of OrcFilters.buildSearchArgument should be fully convertible.")
-        }
-    }
-  }
-
-  /**
-   * Build a SearchArgument for a leaf predicate and return the builder so far.
-   *
-   * @param dataTypeMap a map from the attribute name to its data type.
-   * @param expression the input filter predicates.
-   * @param builder the input SearchArgument.Builder.
-   * @return the builder so far.
-   */
-  private def buildLeafSearchArgument(
-      dataTypeMap: Map[String, DataType],
-      expression: Filter,
-      builder: Builder): Option[Builder] = {
-    def isSearchableType(dataType: DataType): Boolean = dataType match {
-      // Only the values in the Spark types below can be recognized by
-      // the `SearchArgumentImpl.BuilderImpl.boxLiteral()` method.
-      case ByteType | ShortType | FloatType | DoubleType => true
-      case IntegerType | LongType | StringType | BooleanType => true
-      case TimestampType | _: DecimalType => true
-      case _ => false
-    }
-
-    import org.apache.spark.sql.sources._
-
-    // NOTE: For all case branches dealing with leaf predicates below, the additional `startAnd()`
-    // call is mandatory. ORC `SearchArgument` builder requires that all leaf predicates must be
-    // wrapped by a "parent" predicate (`And`, `Or`, or `Not`).
-    expression match {
-      // NOTE: For all case branches dealing with leaf predicates below, the additional `startAnd()`
-      // call is mandatory.  ORC `SearchArgument` builder requires that all leaf predicates must be
-      // wrapped by a "parent" predicate (`And`, `Or`, or `Not`).
-
-      case EqualTo(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val bd = builder.startAnd()
-        val method = findMethod(bd.getClass, "equals", classOf[String], classOf[Object])
-        Some(method.invoke(bd, attribute, value.asInstanceOf[AnyRef]).asInstanceOf[Builder].end())
-
-      case EqualNullSafe(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val bd = builder.startAnd()
-        val method = findMethod(bd.getClass, "nullSafeEquals", classOf[String], classOf[Object])
-        Some(method.invoke(bd, attribute, value.asInstanceOf[AnyRef]).asInstanceOf[Builder].end())
-
-      case LessThan(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val bd = builder.startAnd()
-        val method = findMethod(bd.getClass, "lessThan", classOf[String], classOf[Object])
-        Some(method.invoke(bd, attribute, value.asInstanceOf[AnyRef]).asInstanceOf[Builder].end())
-
-      case LessThanOrEqual(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val bd = builder.startAnd()
-        val method = findMethod(bd.getClass, "lessThanEquals", classOf[String], classOf[Object])
-        Some(method.invoke(bd, attribute, value.asInstanceOf[AnyRef]).asInstanceOf[Builder].end())
-
-      case GreaterThan(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val bd = builder.startNot()
-        val method = findMethod(bd.getClass, "lessThanEquals", classOf[String], classOf[Object])
-        Some(method.invoke(bd, attribute, value.asInstanceOf[AnyRef]).asInstanceOf[Builder].end())
-
-      case GreaterThanOrEqual(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val bd = builder.startNot()
-        val method = findMethod(bd.getClass, "lessThan", classOf[String], classOf[Object])
-        Some(method.invoke(bd, attribute, value.asInstanceOf[AnyRef]).asInstanceOf[Builder].end())
-
-      case IsNull(attribute) if isSearchableType(dataTypeMap(attribute)) =>
-        val bd = builder.startAnd()
-        val method = findMethod(bd.getClass, "isNull", classOf[String])
-        Some(method.invoke(bd, attribute).asInstanceOf[Builder].end())
-
-      case IsNotNull(attribute) if isSearchableType(dataTypeMap(attribute)) =>
-        val bd = builder.startNot()
-        val method = findMethod(bd.getClass, "isNull", classOf[String])
-        Some(method.invoke(bd, attribute).asInstanceOf[Builder].end())
-
-      case In(attribute, values) if isSearchableType(dataTypeMap(attribute)) =>
-        val bd = builder.startAnd()
-        val method = findMethod(bd.getClass, "in", classOf[String], classOf[Array[Object]])
-        Some(method.invoke(bd, attribute, values.map(_.asInstanceOf[AnyRef]))
-          .asInstanceOf[Builder].end())
-
-      case _ => None
-    }
-  }
-}
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/package.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/package.scala
index 14276c9b583f2..f89a8a55d8f7e 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/package.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/package.scala
@@ -23,5 +23,8 @@ package org.apache.spark.sql
  *  - Using HiveQL to express queries.
  *  - Reading metadata from the Hive Metastore using HiveSerDes.
  *  - Hive UDFs, UDAs, UDTs
+ *
+ * Note that this is a private package. All classes in this package are considered an internal API
+ * to Spark and are subject to change between minor releases.
  */
 package object hive
diff --git a/sql/hive/src/test/java/org/apache/spark/sql/hive/JavaDataFrameSuite.java b/sql/hive/src/test/java/org/apache/spark/sql/hive/JavaDataFrameSuite.java
index 2b532389bafb6..268a31dc4a5cd 100644
--- a/sql/hive/src/test/java/org/apache/spark/sql/hive/JavaDataFrameSuite.java
+++ b/sql/hive/src/test/java/org/apache/spark/sql/hive/JavaDataFrameSuite.java
@@ -85,7 +85,7 @@ public void testUDAF() {
           udaf.distinct(col("value")),
           udaf.apply(col("value")),
           registeredUDAF.apply(col("value")),
-          callUDF("mydoublesum", col("value")));
+          callUDF("mydoublesum", col("value")));  // test deprecated one
 
     List<Row> expectedResult = new ArrayList<>();
     expectedResult.add(RowFactory.create(4950.0, 9900.0, 9900.0, 9900.0));
diff --git a/sql/hive/src/test/resources/data/scripts/cat.py b/sql/hive/src/test/resources/data/scripts/cat.py
index aea0362f899fa..420d9f832a184 100644
--- a/sql/hive/src/test/resources/data/scripts/cat.py
+++ b/sql/hive/src/test/resources/data/scripts/cat.py
@@ -16,7 +16,6 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-from __future__ import print_function
 import sys
 import os
 
diff --git a/sql/hive/src/test/resources/data/scripts/dumpdata_script.py b/sql/hive/src/test/resources/data/scripts/dumpdata_script.py
index 5b360208d36f6..f724fdc85b177 100644
--- a/sql/hive/src/test/resources/data/scripts/dumpdata_script.py
+++ b/sql/hive/src/test/resources/data/scripts/dumpdata_script.py
@@ -18,12 +18,9 @@
 #
 import sys
 
-if sys.version_info[0] >= 3:
-    xrange = range
-
-for i in xrange(50):
-    for j in xrange(5):
-        for k in xrange(20022):
+for i in range(50):
+    for j in range(5):
+        for k in range(20022):
             print(20000 * i + k)
 
 for line in sys.stdin:
diff --git a/sql/hive/src/test/resources/data/scripts/test_transform.py b/sql/hive/src/test/resources/data/scripts/test_transform.py
index ac6d11d8b919c..dedb370f6c90e 100755
--- a/sql/hive/src/test/resources/data/scripts/test_transform.py
+++ b/sql/hive/src/test/resources/data/scripts/test_transform.py
@@ -1,3 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
 import sys
 
 delim = sys.argv[1]
diff --git a/sql/hive/src/test/resources/golden/constant null testing-0-237a6af90a857da1efcbe98f6bbbf9d6 b/sql/hive/src/test/resources/golden/constant null testing-0-6a01a94ef1b0d29152c88cd3083fd70b
similarity index 100%
rename from sql/hive/src/test/resources/golden/constant null testing-0-237a6af90a857da1efcbe98f6bbbf9d6
rename to sql/hive/src/test/resources/golden/constant null testing-0-6a01a94ef1b0d29152c88cd3083fd70b
diff --git a/sql/hive/src/test/resources/golden/timestamp cast #2-0-732ed232ac592c5e7f7c913a88874fd2 b/sql/hive/src/test/resources/golden/timestamp cast #2-0-732ed232ac592c5e7f7c913a88874fd2
deleted file mode 100644
index 5625e59da8873..0000000000000
--- a/sql/hive/src/test/resources/golden/timestamp cast #2-0-732ed232ac592c5e7f7c913a88874fd2	
+++ /dev/null
@@ -1 +0,0 @@
-1.2
diff --git a/sql/hive/src/test/resources/golden/timestamp cast #3-0-9f6fcc8c1a29c793c2238bad91453e9f b/sql/hive/src/test/resources/golden/timestamp cast #3-0-9f6fcc8c1a29c793c2238bad91453e9f
new file mode 100644
index 0000000000000..f99e724db6af8
--- /dev/null
+++ b/sql/hive/src/test/resources/golden/timestamp cast #3-0-9f6fcc8c1a29c793c2238bad91453e9f	
@@ -0,0 +1,2 @@
+1.2
+
diff --git a/sql/hive/src/test/resources/golden/timestamp cast #4-0-732ed232ac592c5e7f7c913a88874fd2 b/sql/hive/src/test/resources/golden/timestamp cast #4-0-732ed232ac592c5e7f7c913a88874fd2
deleted file mode 100644
index 5625e59da8873..0000000000000
--- a/sql/hive/src/test/resources/golden/timestamp cast #4-0-732ed232ac592c5e7f7c913a88874fd2	
+++ /dev/null
@@ -1 +0,0 @@
-1.2
diff --git a/sql/hive/src/test/resources/golden/timestamp cast #4-0-e9286317470d42e9f8122bc98a2c1ce1 b/sql/hive/src/test/resources/golden/timestamp cast #4-0-e9286317470d42e9f8122bc98a2c1ce1
new file mode 100644
index 0000000000000..decdb1d30e6a6
--- /dev/null
+++ b/sql/hive/src/test/resources/golden/timestamp cast #4-0-e9286317470d42e9f8122bc98a2c1ce1	
@@ -0,0 +1,2 @@
+-1.2
+
diff --git a/sql/hive/src/test/resources/golden/timestamp cast #6-0-6d2da5cfada03605834e38bc4075bc79 b/sql/hive/src/test/resources/golden/timestamp cast #6-0-6d2da5cfada03605834e38bc4075bc79
deleted file mode 100644
index 1d94c8a014fb4..0000000000000
--- a/sql/hive/src/test/resources/golden/timestamp cast #6-0-6d2da5cfada03605834e38bc4075bc79	
+++ /dev/null
@@ -1 +0,0 @@
--1.2
diff --git a/sql/hive/src/test/resources/golden/timestamp cast #8-0-6d2da5cfada03605834e38bc4075bc79 b/sql/hive/src/test/resources/golden/timestamp cast #8-0-6d2da5cfada03605834e38bc4075bc79
deleted file mode 100644
index 1d94c8a014fb4..0000000000000
--- a/sql/hive/src/test/resources/golden/timestamp cast #8-0-6d2da5cfada03605834e38bc4075bc79	
+++ /dev/null
@@ -1 +0,0 @@
--1.2
diff --git a/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/add_partition_no_includelist.q b/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/add_partition_no_includelist.q
new file mode 100644
index 0000000000000..17677122a1bca
--- /dev/null
+++ b/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/add_partition_no_includelist.q
@@ -0,0 +1,7 @@
+SET hive.metastore.partition.name.whitelist.pattern=;
+-- Test with no partition name include-list pattern
+
+CREATE TABLE part_noincludelist_test (key STRING, value STRING) PARTITIONED BY (ds STRING);
+SHOW PARTITIONS part_noincludelist_test;
+
+ALTER TABLE part_noincludelist_test ADD PARTITION (ds='1,2,3,4');
diff --git a/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/add_partition_no_whitelist.q b/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/add_partition_no_whitelist.q
deleted file mode 100644
index f51c53c2ff627..0000000000000
--- a/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/add_partition_no_whitelist.q
+++ /dev/null
@@ -1,7 +0,0 @@
-SET hive.metastore.partition.name.whitelist.pattern=;
--- Test with no partition name whitelist pattern
-
-CREATE TABLE part_nowhitelist_test (key STRING, value STRING) PARTITIONED BY (ds STRING);
-SHOW PARTITIONS part_nowhitelist_test;
-
-ALTER TABLE part_nowhitelist_test ADD PARTITION (ds='1,2,3,4');
diff --git a/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/add_partition_with_includelist.q b/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/add_partition_with_includelist.q
new file mode 100644
index 0000000000000..7e7f30dc37305
--- /dev/null
+++ b/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/add_partition_with_includelist.q
@@ -0,0 +1,7 @@
+SET hive.metastore.partition.name.whitelist.pattern=[A-Za-z]*;
+-- This pattern matches only letters.
+
+CREATE TABLE part_includelist_test (key STRING, value STRING) PARTITIONED BY (ds STRING);
+SHOW PARTITIONS part_includelist_test;
+
+ALTER TABLE part_includelist_test ADD PARTITION (ds='Part');
diff --git a/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/add_partition_with_whitelist.q b/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/add_partition_with_whitelist.q
deleted file mode 100644
index 009c7610ef917..0000000000000
--- a/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/add_partition_with_whitelist.q
+++ /dev/null
@@ -1,9 +0,0 @@
-SET hive.metastore.partition.name.whitelist.pattern=[A-Za-z]*;
--- This pattern matches only letters.
-
-CREATE TABLE part_whitelist_test (key STRING, value STRING) PARTITIONED BY (ds STRING);
-SHOW PARTITIONS part_whitelist_test;
-
-ALTER TABLE part_whitelist_test ADD PARTITION (ds='Part');
-
-      
diff --git a/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/alter_partition_with_includelist.q b/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/alter_partition_with_includelist.q
new file mode 100644
index 0000000000000..fcef12cbaac4e
--- /dev/null
+++ b/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/alter_partition_with_includelist.q
@@ -0,0 +1,9 @@
+SET hive.metastore.partition.name.whitelist.pattern=[A-Za-z]*;
+-- This pattern matches only letters.
+
+CREATE TABLE part_includelist_test (key STRING, value STRING) PARTITIONED BY (ds STRING);
+SHOW PARTITIONS part_includelist_test;
+
+ALTER TABLE part_includelist_test ADD PARTITION (ds='Part');
+
+ALTER TABLE part_includelist_test PARTITION (ds='Part') rename to partition (ds='Apart');
diff --git a/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/alter_partition_with_whitelist.q b/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/alter_partition_with_whitelist.q
deleted file mode 100644
index 301362a881456..0000000000000
--- a/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/alter_partition_with_whitelist.q
+++ /dev/null
@@ -1,9 +0,0 @@
-SET hive.metastore.partition.name.whitelist.pattern=[A-Za-z]*;
--- This pattern matches only letters.
-
-CREATE TABLE part_whitelist_test (key STRING, value STRING) PARTITIONED BY (ds STRING);
-SHOW PARTITIONS part_whitelist_test;
-
-ALTER TABLE part_whitelist_test ADD PARTITION (ds='Part');
-
-ALTER TABLE part_whitelist_test PARTITION (ds='Part') rename to partition (ds='Apart');     
diff --git a/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/auto_sortmerge_join_13.q b/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/auto_sortmerge_join_13.q
index 28bbc2d8f1a3e..df5334c785f6a 100644
--- a/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/auto_sortmerge_join_13.q
+++ b/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/auto_sortmerge_join_13.q
@@ -19,7 +19,7 @@ set hive.optimize.bucketmapjoin = true;
 set hive.optimize.bucketmapjoin.sortedmerge = true;
 set hive.auto.convert.join=true;
 
--- A SMB join followed by a mutli-insert
+-- A SMB join followed by a multi-insert
 explain 
 from (
   SELECT a.key key1, a.value value1, b.key key2, b.value value2 
@@ -41,7 +41,7 @@ select * from dest2 order by k1, k2;
 set hive.auto.convert.join.noconditionaltask=true;
 set hive.auto.convert.join.noconditionaltask.size=200;
 
--- A SMB join followed by a mutli-insert
+-- A SMB join followed by a multi-insert
 explain 
 from (
   SELECT a.key key1, a.value value1, b.key key2, b.value value2 
@@ -61,7 +61,7 @@ select * from dest1 order by k1, k2;
 select * from dest2 order by k1, k2;
 
 set hive.auto.convert.sortmerge.join.to.mapjoin=true;
--- A SMB join followed by a mutli-insert
+-- A SMB join followed by a multi-insert
 explain 
 from (
   SELECT a.key key1, a.value value1, b.key key2, b.value value2 
diff --git a/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_3.q b/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_3.q
index 91e97de62c82f..843ba4a3dbacd 100644
--- a/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_3.q
+++ b/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_3.q
@@ -18,7 +18,7 @@ FROM src
 INSERT OVERWRITE TABLE test_table1 PARTITION (ds = '1') SELECT *;
 
 -- Insert data into the bucketed table by selecting from another bucketed table
--- The bucketing positions dont match - although the actual bucketing do.
+-- The bucketing positions don't match - although the actual bucketing do.
 -- This should be a map-only operation
 EXPLAIN
 INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1')
@@ -37,7 +37,7 @@ CREATE TABLE test_table3 (key INT, value STRING) PARTITIONED BY (ds STRING)
 CLUSTERED BY (value) SORTED BY (value) INTO 2 BUCKETS;
 
 -- Insert data into the bucketed table by selecting from another bucketed table
--- The bucketing positions dont match - this should be a map-reduce operation
+-- The bucketing positions don't match - this should be a map-reduce operation
 EXPLAIN
 INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1')
 SELECT x.key, x.value from 
diff --git a/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/smb_mapjoin_20.q b/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/smb_mapjoin_20.q
index f70e7d5c86237..4c56cad2411fc 100644
--- a/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/smb_mapjoin_20.q
+++ b/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/smb_mapjoin_20.q
@@ -32,7 +32,7 @@ CREATE TABLE test_table3 (key STRING, value1 int, value2 string) PARTITIONED BY
 CLUSTERED BY (value1) SORTED BY (value1) INTO 2 BUCKETS;
 
 -- Insert data into the bucketed table by selecting from another bucketed table
--- This should be a map-only operation, although the bucketing positions dont match
+-- This should be a map-only operation, although the bucketing positions don't match
 EXPLAIN
 INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1')
 SELECT a.value, a.key, a.value FROM test_table1 a WHERE a.ds = '1';
diff --git a/sql/hive/src/test/resources/regression-test-SPARK-8489/test-2.13.jar b/sql/hive/src/test/resources/regression-test-SPARK-8489/test-2.13.jar
new file mode 100644
index 0000000000000..0d10f7ff03b35
Binary files /dev/null and b/sql/hive/src/test/resources/regression-test-SPARK-8489/test-2.13.jar differ
diff --git a/sql/hive/src/test/resources/schemaWithOneField.avsc b/sql/hive/src/test/resources/schemaWithOneField.avsc
new file mode 100644
index 0000000000000..e6e2431707f6e
--- /dev/null
+++ b/sql/hive/src/test/resources/schemaWithOneField.avsc
@@ -0,0 +1,12 @@
+{
+  "namespace": "test",
+  "name": "some_schema",
+  "type": "record",
+  "fields": [
+    {
+      "name": "col2",
+      "type": "string"
+    }
+  ]
+}
+
diff --git a/sql/hive/src/test/resources/schemaWithTwoFields.avsc b/sql/hive/src/test/resources/schemaWithTwoFields.avsc
new file mode 100644
index 0000000000000..3d1d24cfa2e96
--- /dev/null
+++ b/sql/hive/src/test/resources/schemaWithTwoFields.avsc
@@ -0,0 +1,16 @@
+{
+  "namespace": "test",
+  "name": "some_schema",
+  "type": "record",
+  "fields": [
+    {
+      "name": "col1",
+      "type": "string",
+      "default": "col1_default"
+    },
+    {
+      "name": "col2",
+      "type": "string"
+    }
+  ]
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/HiveCharVarcharTestSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/HiveCharVarcharTestSuite.scala
new file mode 100644
index 0000000000000..bb7918c881c7e
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/HiveCharVarcharTestSuite.scala
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import org.apache.spark.sql.execution.command.CharVarcharDDLTestBase
+import org.apache.spark.sql.hive.test.TestHiveSingleton
+
+class HiveCharVarcharTestSuite extends CharVarcharTestSuite with TestHiveSingleton {
+
+  // The default Hive serde doesn't support nested null values.
+  override def format: String = "hive OPTIONS(fileFormat='parquet')"
+
+  private var originalPartitionMode = ""
+
+  override protected def beforeAll(): Unit = {
+    super.beforeAll()
+    originalPartitionMode = spark.conf.get("hive.exec.dynamic.partition.mode", "")
+    spark.conf.set("hive.exec.dynamic.partition.mode", "nonstrict")
+  }
+
+  override protected def afterAll(): Unit = {
+    if (originalPartitionMode == "") {
+      spark.conf.unset("hive.exec.dynamic.partition.mode")
+    } else {
+      spark.conf.set("hive.exec.dynamic.partition.mode", originalPartitionMode)
+    }
+    super.afterAll()
+  }
+
+  test("SPARK-33892: SHOW CREATE TABLE AS SERDE w/ char/varchar") {
+    withTable("t") {
+      sql(s"CREATE TABLE t(v VARCHAR(3), c CHAR(5)) USING $format")
+      val rest = sql("SHOW CREATE TABLE t AS SERDE").head().getString(0)
+      assert(rest.contains("VARCHAR(3)"))
+      assert(rest.contains("CHAR(5)"))
+    }
+  }
+}
+
+class HiveCharVarcharDDLTestSuite extends CharVarcharDDLTestBase with TestHiveSingleton {
+
+  // The default Hive serde doesn't support nested null values.
+  override def format: String = "hive OPTIONS(fileFormat='parquet')"
+
+  private var originalPartitionMode = ""
+
+  override protected def beforeAll(): Unit = {
+    super.beforeAll()
+    originalPartitionMode = spark.conf.get("hive.exec.dynamic.partition.mode", "")
+    spark.conf.set("hive.exec.dynamic.partition.mode", "nonstrict")
+  }
+
+  override protected def afterAll(): Unit = {
+    if (originalPartitionMode == "") {
+      spark.conf.unset("hive.exec.dynamic.partition.mode")
+    } else {
+      spark.conf.set("hive.exec.dynamic.partition.mode", originalPartitionMode)
+    }
+    super.afterAll()
+  }
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/execution/benchmark/InsertIntoHiveTableBenchmark.scala b/sql/hive/src/test/scala/org/apache/spark/sql/execution/benchmark/InsertIntoHiveTableBenchmark.scala
new file mode 100644
index 0000000000000..1df91a1174ba4
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/execution/benchmark/InsertIntoHiveTableBenchmark.scala
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.benchmark
+
+import org.apache.spark.benchmark.Benchmark
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.hive.test.TestHive
+
+/**
+ * Benchmark to measure hive table write performance.
+ * To run this benchmark:
+ * {{{
+ *   1. without sbt: bin/spark-submit --class <this class>
+ *        --jars <spark catalyst test jar>,<spark core test jar>,<spark sql test jar>
+ *        <spark hive test jar>
+ *   2. build/sbt "hive/test:runMain <this class>"
+ *   3. generate result:
+ *   SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "hive/test:runMain <this class>"
+ *      Results will be written to "benchmarks/InsertIntoHiveTableBenchmark-hive2.3-results.txt".
+ * }}}
+ */
+object InsertIntoHiveTableBenchmark extends SqlBasedBenchmark {
+
+  override def getSparkSession: SparkSession = TestHive.sparkSession
+
+  val tempView = "temp"
+  val numRows = 1024 * 10
+  val sql = spark.sql _
+
+  // scalastyle:off hadoopconfiguration
+  private val hadoopConf = spark.sparkContext.hadoopConfiguration
+  // scalastyle:on hadoopconfiguration
+  hadoopConf.set("hive.exec.dynamic.partition", "true")
+  hadoopConf.set("hive.exec.dynamic.partition.mode", "nonstrict")
+  hadoopConf.set("hive.exec.max.dynamic.partitions", numRows.toString)
+
+  def withTable(tableNames: String*)(f: => Unit): Unit = {
+    tableNames.foreach { name =>
+      sql(s"CREATE TABLE $name(a INT) STORED AS TEXTFILE PARTITIONED BY (b INT, c INT)")
+    }
+    try f finally {
+      tableNames.foreach { name =>
+        spark.sql(s"DROP TABLE IF EXISTS $name")
+      }
+    }
+  }
+
+  def insertOverwriteDynamic(table: String, benchmark: Benchmark): Unit = {
+    benchmark.addCase("INSERT OVERWRITE DYNAMIC") { _ =>
+      sql(s"INSERT OVERWRITE TABLE $table SELECT CAST(id AS INT) AS a," +
+        s" CAST(id % 10 AS INT) AS b, CAST(id % 100 AS INT) AS c FROM $tempView")
+    }
+  }
+
+  def insertOverwriteHybrid(table: String, benchmark: Benchmark): Unit = {
+    benchmark.addCase("INSERT OVERWRITE HYBRID") { _ =>
+      sql(s"INSERT OVERWRITE TABLE $table partition(b=1, c) SELECT CAST(id AS INT) AS a," +
+        s" CAST(id % 10 AS INT) AS c FROM $tempView")
+    }
+  }
+
+  def insertOverwriteStatic(table: String, benchmark: Benchmark): Unit = {
+    benchmark.addCase("INSERT OVERWRITE STATIC") { _ =>
+      sql(s"INSERT OVERWRITE TABLE $table partition(b=1, c=10) SELECT CAST(id AS INT) AS a" +
+        s" FROM $tempView")
+    }
+  }
+
+  def insertIntoDynamic(table: String, benchmark: Benchmark): Unit = {
+    benchmark.addCase("INSERT INTO DYNAMIC") { _ =>
+      sql(s"INSERT INTO TABLE $table SELECT CAST(id AS INT) AS a," +
+        s" CAST(id % 10 AS INT) AS b, CAST(id % 100 AS INT) AS c FROM $tempView")
+    }
+  }
+
+  def insertIntoHybrid(table: String, benchmark: Benchmark): Unit = {
+    benchmark.addCase("INSERT INTO HYBRID") { _ =>
+      sql(s"INSERT INTO TABLE $table partition(b=1, c) SELECT CAST(id AS INT) AS a," +
+        s" CAST(id % 10 AS INT) AS c FROM $tempView")
+    }
+  }
+
+  def insertIntoStatic(table: String, benchmark: Benchmark): Unit = {
+    benchmark.addCase("INSERT INTO STATIC") { _ =>
+      sql(s"INSERT INTO TABLE $table partition(b=1, c=10) SELECT CAST(id AS INT) AS a" +
+        s" FROM $tempView")
+    }
+  }
+
+  override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
+    spark.range(numRows).createOrReplaceTempView(tempView)
+
+    try {
+      val t1 = "t1"
+      val t2 = "t2"
+      val t3 = "t3"
+      val t4 = "t4"
+      val t5 = "t5"
+      val t6 = "t6"
+
+      val benchmark = new Benchmark(s"insert hive table benchmark", numRows, output = output)
+
+      withTable(t1, t2, t3, t4, t5, t6) {
+
+        insertIntoDynamic(t1, benchmark)
+        insertIntoHybrid(t2, benchmark)
+        insertIntoStatic(t3, benchmark)
+
+        insertOverwriteDynamic(t4, benchmark)
+        insertOverwriteHybrid(t5, benchmark)
+        insertOverwriteStatic(t6, benchmark)
+
+        benchmark.run()
+      }
+    } finally {
+      spark.catalog.dropTempView(tempView)
+    }
+  }
+
+  override def suffix: String = "-hive2.3"
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/execution/benchmark/ObjectHashAggregateExecBenchmark.scala b/sql/hive/src/test/scala/org/apache/spark/sql/execution/benchmark/ObjectHashAggregateExecBenchmark.scala
index 3f806ad24ca10..46a60efc5b569 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/execution/benchmark/ObjectHashAggregateExecBenchmark.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/execution/benchmark/ObjectHashAggregateExecBenchmark.scala
@@ -35,8 +35,7 @@ import org.apache.spark.sql.types.LongType
  * To run this benchmark:
  * {{{
  *   1. without sbt: bin/spark-submit --class <this class>
- *        --jars <spark catalyst test jar>,<spark core test jar>,<spark hive jar>
- *        --packages org.spark-project.hive:hive-exec:1.2.1.spark2
+ *        --jars <spark catalyst test jar>,<spark core test jar>,<spark sql test jar>
  *        <spark hive test jar>
  *   2. build/sbt "hive/test:runMain <this class>"
  *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "hive/test:runMain <this class>"
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
index fc793534641df..a8598db1003d8 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.hive
 
 import java.io.File
 
-import org.apache.spark.sql.{AnalysisException, Dataset, QueryTest, SaveMode}
+import org.apache.spark.sql.{AnalysisException, Dataset, QueryTest, Row, SaveMode}
 import org.apache.spark.sql.catalyst.parser.ParseException
 import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
 import org.apache.spark.sql.execution.datasources.{CatalogFileIndex, HadoopFsRelation, LogicalRelation}
@@ -113,7 +113,7 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
     e = intercept[AnalysisException] {
       sql("UNCACHE TABLE nonexistentTable")
     }.getMessage
-    assert(e.contains(s"$expectedErrorMsg default.nonexistentTable"))
+    assert(e.contains("Table or view not found: nonexistentTable"))
     sql("UNCACHE TABLE IF EXISTS nonexistentTable")
   }
 
@@ -364,14 +364,14 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
           // Cache the table 'cachedTable' in temp db with qualified table name,
           // and then check whether the table is cached with expected name
           sql(s"CACHE TABLE $db.cachedTable OPTIONS('storageLevel' 'MEMORY_ONLY')")
-          assertCached(sql(s"SELECT * FROM $db.cachedTable"), s"`$db`.`cachedTable`", MEMORY_ONLY)
+          assertCached(sql(s"SELECT * FROM $db.cachedTable"), s"$db.cachedTable", MEMORY_ONLY)
           assert(spark.catalog.isCached(s"$db.cachedTable"),
             s"Table '$db.cachedTable' should be cached.")
 
           // Refresh the table 'cachedTable' in temp db with qualified table name, and then check
           // whether the table is still cached with the same name and storage level.
           sql(s"REFRESH TABLE $db.cachedTable")
-          assertCached(sql(s"select * from $db.cachedTable"), s"`$db`.`cachedTable`", MEMORY_ONLY)
+          assertCached(sql(s"select * from $db.cachedTable"), s"$db.cachedTable", MEMORY_ONLY)
           assert(spark.catalog.isCached(s"$db.cachedTable"),
             s"Table '$db.cachedTable' should be cached after refreshing with its qualified name.")
 
@@ -382,7 +382,7 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
           // 'cachedTable', instead of '$db.cachedTable'
           activateDatabase(db) {
             sql("REFRESH TABLE cachedTable")
-            assertCached(sql("SELECT * FROM cachedTable"), s"`$db`.`cachedTable`", MEMORY_ONLY)
+            assertCached(sql("SELECT * FROM cachedTable"), s"$db.cachedTable", MEMORY_ONLY)
             assert(spark.catalog.isCached("cachedTable"),
               s"Table '$db.cachedTable' should be cached after refreshing with its " +
                 "unqualified name.")
@@ -403,13 +403,13 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
           // Cache the table 'cachedTable' in default db without qualified table name , and then
           // check whether the table is cached with expected name.
           sql("CACHE TABLE cachedTable OPTIONS('storageLevel' 'DISK_ONLY')")
-          assertCached(sql("SELECT * FROM cachedTable"), "`default`.`cachedTable`", DISK_ONLY)
+          assertCached(sql("SELECT * FROM cachedTable"), "cachedTable", DISK_ONLY)
           assert(spark.catalog.isCached("cachedTable"), "Table 'cachedTable' should be cached.")
 
           // Refresh the table 'cachedTable' in default db with unqualified table name, and then
           // check whether the table is still cached with the same name.
           sql("REFRESH TABLE cachedTable")
-          assertCached(sql("SELECT * FROM cachedTable"), "`default`.`cachedTable`", DISK_ONLY)
+          assertCached(sql("SELECT * FROM cachedTable"), "cachedTable", DISK_ONLY)
           assert(spark.catalog.isCached("cachedTable"),
             "Table 'cachedTable' should be cached after refreshing with its unqualified name.")
 
@@ -421,7 +421,7 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
           activateDatabase(db) {
             sql("REFRESH TABLE default.cachedTable")
             assertCached(
-              sql("SELECT * FROM default.cachedTable"), "`default`.`cachedTable`", DISK_ONLY)
+              sql("SELECT * FROM default.cachedTable"), "cachedTable", DISK_ONLY)
             assert(spark.catalog.isCached("default.cachedTable"),
               "Table 'cachedTable' should be cached after refreshing with its qualified name.")
           }
@@ -429,4 +429,99 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
       }
     }
   }
+
+  test("SPARK-33963: do not use table stats while looking in table cache") {
+    val t = "table_on_test"
+    withTable(t) {
+      sql(s"CREATE TABLE $t (col int)")
+      assert(!spark.catalog.isCached(t))
+      sql(s"CACHE TABLE $t")
+      assert(spark.catalog.isCached(t))
+    }
+  }
+
+  test("SPARK-33965: cache table in spark_catalog") {
+    withNamespace("spark_catalog.ns") {
+      sql("CREATE NAMESPACE spark_catalog.ns")
+      val t = "spark_catalog.ns.tbl"
+      withTable(t) {
+        sql(s"CREATE TABLE $t (col int)")
+        assert(!spark.catalog.isCached(t))
+        sql(s"CACHE TABLE $t")
+        assert(spark.catalog.isCached(t))
+      }
+    }
+  }
+
+  test("SPARK-34076: should be able to drop temp view with cached tables") {
+    val t = "cachedTable"
+    val v = "tempView"
+    withTable(t) {
+      withTempView(v) {
+        sql(s"CREATE TEMPORARY VIEW $v AS SELECT key FROM src LIMIT 10")
+        sql(s"CREATE TABLE $t AS SELECT * FROM src")
+        sql(s"CACHE TABLE $t")
+      }
+    }
+  }
+
+  test("SPARK-34076: should be able to drop global temp view with cached tables") {
+    val t = "cachedTable"
+    val v = "globalTempView"
+    withTable(t) {
+      withGlobalTempView(v) {
+        sql(s"CREATE GLOBAL TEMPORARY VIEW $v AS SELECT key FROM src LIMIT 10")
+        sql(s"CREATE TABLE $t AS SELECT * FROM src")
+        sql(s"CACHE TABLE $t")
+      }
+    }
+  }
+
+  private def getPartitionLocation(t: String, partition: String): String = {
+    val information = sql(s"SHOW TABLE EXTENDED LIKE '$t' PARTITION ($partition)")
+      .select("information")
+      .first().getString(0)
+    information
+      .split("\\r?\\n")
+      .filter(_.startsWith("Location:"))
+      .head
+      .replace("Location: file:", "")
+  }
+
+  test("SPARK-34213: LOAD DATA refreshes cached table") {
+    withTable("src_tbl") {
+      withTable("dst_tbl") {
+        sql("CREATE TABLE src_tbl (c0 int, part int) USING hive PARTITIONED BY (part)")
+        sql("INSERT INTO src_tbl PARTITION (part=0) SELECT 0")
+        sql("CREATE TABLE dst_tbl (c0 int, part int) USING hive PARTITIONED BY (part)")
+        sql("INSERT INTO dst_tbl PARTITION (part=1) SELECT 1")
+        sql("CACHE TABLE dst_tbl")
+        assert(spark.catalog.isCached("dst_tbl"))
+        checkAnswer(sql("SELECT * FROM dst_tbl"), Row(1, 1))
+        val location = getPartitionLocation("src_tbl", "part=0")
+        sql(s"LOAD DATA LOCAL INPATH '$location' INTO TABLE dst_tbl PARTITION (part=0)")
+        assert(spark.catalog.isCached("dst_tbl"))
+        checkAnswer(sql("SELECT * FROM dst_tbl"), Seq(Row(0, 0), Row(1, 1)))
+      }
+    }
+  }
+
+  test("SPARK-34262: ALTER TABLE .. SET LOCATION refreshes cached table") {
+    withTable("src_tbl") {
+      withTable("dst_tbl") {
+        sql("CREATE TABLE src_tbl (c0 int, part int) USING hive PARTITIONED BY (part)")
+        sql("INSERT INTO src_tbl PARTITION (part=0) SELECT 0")
+        sql("CREATE TABLE dst_tbl (c0 int, part int) USING hive PARTITIONED BY (part)")
+        sql("ALTER TABLE dst_tbl ADD PARTITION (part=0)")
+        sql("INSERT INTO dst_tbl PARTITION (part=1) SELECT 1")
+        sql("CACHE TABLE dst_tbl")
+        assert(spark.catalog.isCached("dst_tbl"))
+        checkAnswer(sql("SELECT * FROM dst_tbl"), Row(1, 1))
+        val location = getPartitionLocation("src_tbl", "part=0")
+        sql(s"ALTER TABLE dst_tbl PARTITION (part=0) SET LOCATION '$location'")
+        assert(spark.catalog.isCached("dst_tbl"))
+        checkAnswer(sql("SELECT * FROM dst_tbl"), Seq(Row(0, 0), Row(1, 1)))
+      }
+    }
+  }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/ClasspathDependenciesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/ClasspathDependenciesSuite.scala
index a696d6aaff27b..c136c4c9790fd 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/ClasspathDependenciesSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/ClasspathDependenciesSuite.scala
@@ -57,20 +57,12 @@ class ClasspathDependenciesSuite extends SparkFunSuite {
     }
   }
 
-  test("shaded Protobuf") {
-    if (HiveUtils.isHive23) {
-      assertLoads("com.google.protobuf.ServiceException")
-    } else {
-      assertLoads("org.apache.hive.com.google.protobuf.ServiceException")
-    }
+  test("protobuf") {
+    assertLoads("com.google.protobuf.ServiceException")
   }
 
-  test("shaded Kryo") {
-    if (HiveUtils.isHive23) {
-      assertLoads("com.esotericsoftware.kryo.Kryo")
-    } else {
-      assertLoads("org.apache.hive.com.esotericsoftware.kryo.Kryo")
-    }
+  test("kryo") {
+    assertLoads("com.esotericsoftware.kryo.Kryo")
   }
 
   test("hive-common") {
@@ -89,12 +81,7 @@ class ClasspathDependenciesSuite extends SparkFunSuite {
   }
 
   test("parquet-hadoop-bundle") {
-    if (HiveUtils.isHive23) {
-      assertLoads("org.apache.parquet.hadoop.ParquetOutputFormat")
-      assertLoads("org.apache.parquet.hadoop.ParquetInputFormat")
-    } else {
-      assertLoads("parquet.hadoop.ParquetOutputFormat")
-      assertLoads("parquet.hadoop.ParquetInputFormat")
-    }
+    assertLoads("org.apache.parquet.hadoop.ParquetOutputFormat")
+    assertLoads("org.apache.parquet.hadoop.ParquetInputFormat")
   }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala
index 473a93bf129df..e413e0ee73cb9 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.sql.hive
 
-import java.net.URI
-
 import org.apache.hadoop.conf.Configuration
 
 import org.apache.spark.SparkConf
@@ -181,41 +179,25 @@ class HiveExternalCatalogSuite extends ExternalCatalogSuite {
       "INSERT overwrite directory \"fs://localhost/tmp\" select 1 as a"))
   }
 
-  test("SPARK-31061: alterTable should be able to change table provider") {
+  test("SPARK-31061: alterTable should be able to change table provider/hive") {
     val catalog = newBasicCatalog()
-    val parquetTable = CatalogTable(
-      identifier = TableIdentifier("parq_tbl", Some("db1")),
-      tableType = CatalogTableType.MANAGED,
-      storage = storageFormat.copy(locationUri = Some(new URI("file:/some/path"))),
-      schema = new StructType().add("col1", "int").add("col2", "string"),
-      provider = Some("parquet"))
-    catalog.createTable(parquetTable, ignoreIfExists = false)
-
-    val rawTable = externalCatalog.getTable("db1", "parq_tbl")
-    assert(rawTable.provider === Some("parquet"))
-
-    val fooTable = parquetTable.copy(provider = Some("foo"))
-    catalog.alterTable(fooTable)
-    val alteredTable = externalCatalog.getTable("db1", "parq_tbl")
-    assert(alteredTable.provider === Some("foo"))
-  }
-
-  test("SPARK-31061: alterTable should be able to change table provider from hive") {
-    val catalog = newBasicCatalog()
-    val hiveTable = CatalogTable(
-      identifier = TableIdentifier("parq_tbl", Some("db1")),
-      tableType = CatalogTableType.MANAGED,
-      storage = storageFormat,
-      schema = new StructType().add("col1", "int").add("col2", "string"),
-      provider = Some("hive"))
-    catalog.createTable(hiveTable, ignoreIfExists = false)
-
-    val rawTable = externalCatalog.getTable("db1", "parq_tbl")
-    assert(rawTable.provider === Some("hive"))
-
-    val fooTable = rawTable.copy(provider = Some("foo"))
-    catalog.alterTable(fooTable)
-    val alteredTable = externalCatalog.getTable("db1", "parq_tbl")
-    assert(alteredTable.provider === Some("foo"))
+    Seq("parquet", "hive").foreach( provider => {
+      val tableDDL = CatalogTable(
+        identifier = TableIdentifier("parq_tbl", Some("db1")),
+        tableType = CatalogTableType.MANAGED,
+        storage = storageFormat,
+        schema = new StructType().add("col1", "int"),
+        provider = Some(provider))
+      catalog.dropTable("db1", "parq_tbl", true, true)
+      catalog.createTable(tableDDL, ignoreIfExists = false)
+
+      val rawTable = externalCatalog.getTable("db1", "parq_tbl")
+      assert(rawTable.provider === Some(provider))
+
+      val fooTable = rawTable.copy(provider = Some("foo"))
+      catalog.alterTable(fooTable)
+      val alteredTable = externalCatalog.getTable("db1", "parq_tbl")
+      assert(alteredTable.provider === Some("foo"))
+    })
   }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
index 8be3d26bfc93a..60cca97ef384e 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
@@ -27,7 +27,7 @@ import scala.util.control.NonFatal
 import org.apache.commons.lang3.{JavaVersion, SystemUtils}
 import org.apache.hadoop.conf.Configuration
 
-import org.apache.spark.{SecurityManager, SparkConf, TestUtils}
+import org.apache.spark.{SparkConf, TestUtils}
 import org.apache.spark.internal.config.MASTER_REST_SERVER_ENABLED
 import org.apache.spark.internal.config.UI.UI_ENABLED
 import org.apache.spark.sql.{QueryTest, Row, SparkSession}
@@ -35,32 +35,44 @@ import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.catalog.CatalogTableType
 import org.apache.spark.sql.internal.StaticSQLConf.WAREHOUSE_PATH
 import org.apache.spark.sql.test.SQLTestUtils
-import org.apache.spark.tags.ExtendedHiveTest
+import org.apache.spark.tags.{ExtendedHiveTest, SlowHiveTest}
 import org.apache.spark.util.Utils
 
 /**
  * Test HiveExternalCatalog backward compatibility.
  *
  * Note that, this test suite will automatically download spark binary packages of different
- * versions to a local directory `/tmp/spark-test`. If there is already a spark folder with
- * expected version under this local directory, e.g. `/tmp/spark-test/spark-2.0.3`, we will skip the
- * downloading for this spark version.
+ * versions to a local directory. If the `spark.test.cache-dir` system property is defined, this
+ * directory will be used. If there is already a spark folder with expected version under this
+ * local directory, e.g. `/{cache-dir}/spark-2.0.3`, downloading for this spark version will be
+ * skipped. If the system property is not present, a temporary directory will be used and cleaned
+ * up after the test.
  */
+@SlowHiveTest
 @ExtendedHiveTest
 class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils {
-  private val isTestAtLeastJava9 = SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_9)
+  import HiveExternalCatalogVersionsSuite._
   private val wareHousePath = Utils.createTempDir(namePrefix = "warehouse")
   private val tmpDataDir = Utils.createTempDir(namePrefix = "test-data")
-  // For local test, you can set `sparkTestingDir` to a static value like `/tmp/test-spark`, to
+  // For local test, you can set `spark.test.cache-dir` to a static value like `/tmp/test-spark`, to
   // avoid downloading Spark of different versions in each run.
-  private val sparkTestingDir = new File("/tmp/test-spark")
+  private val sparkTestingDir = Option(System.getProperty(SPARK_TEST_CACHE_DIR_SYSTEM_PROPERTY))
+      .map(new File(_)).getOrElse(Utils.createTempDir(namePrefix = "test-spark"))
   private val unusedJar = TestUtils.createJarWithClasses(Seq.empty)
+  val hiveVersion = if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_9)) {
+    HiveUtils.builtinHiveVersion
+  } else {
+    "1.2.1"
+  }
 
   override def afterAll(): Unit = {
     try {
       Utils.deleteRecursively(wareHousePath)
       Utils.deleteRecursively(tmpDataDir)
-      Utils.deleteRecursively(sparkTestingDir)
+      // Only delete sparkTestingDir if it wasn't defined to a static location by the system prop
+      if (Option(System.getProperty(SPARK_TEST_CACHE_DIR_SYSTEM_PROPERTY)).isEmpty) {
+        Utils.deleteRecursively(sparkTestingDir)
+      }
     } finally {
       super.afterAll()
     }
@@ -81,7 +93,11 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils {
       mirrors.distinct :+ "https://archive.apache.org/dist" :+ PROCESS_TABLES.releaseMirror
     logInfo(s"Trying to download Spark $version from $sites")
     for (site <- sites) {
-      val filename = s"spark-$version-bin-hadoop2.7.tgz"
+      val filename = if (version.startsWith("3")) {
+        s"spark-$version-bin-hadoop3.2.tgz"
+      } else {
+        s"spark-$version-bin-hadoop2.7.tgz"
+      }
       val url = s"$site/spark/spark-$version/$filename"
       logInfo(s"Downloading Spark $version from $url")
       try {
@@ -118,7 +134,6 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils {
     // if the caller passes the name of an existing file, we want doFetchFile to write over it with
     // the contents from the specified url.
     conf.set("spark.files.overwrite", "true")
-    val securityManager = new SecurityManager(conf)
     val hadoopConf = new Configuration
 
     val outDir = new File(targetDir)
@@ -127,7 +142,7 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils {
     }
 
     // propagate exceptions up to the caller of getFileFromUrl
-    Utils.doFetchFile(urlString, outDir, filename, conf, securityManager, hadoopConf)
+    Utils.doFetchFile(urlString, outDir, filename, conf, hadoopConf)
   }
 
   private def getStringFromUrl(urlString: String): String = {
@@ -141,7 +156,9 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils {
     new String(Files.readAllBytes(contentPath), StandardCharsets.UTF_8)
   }
 
-  private def prepare(): Unit = {
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+
     val tempPyFile = File.createTempFile("test", ".py")
     // scalastyle:off line.size.limit
     Files.write(tempPyFile.toPath,
@@ -177,7 +194,7 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils {
     // scalastyle:on line.size.limit
 
     if (PROCESS_TABLES.testingVersions.isEmpty) {
-      fail("Fail to get the lates Spark versions to test.")
+      logError("Fail to get the latest Spark versions to test.")
     }
 
     PROCESS_TABLES.testingVersions.zipWithIndex.foreach { case (version, index) =>
@@ -191,7 +208,7 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils {
         "--master", "local[2]",
         "--conf", s"${UI_ENABLED.key}=false",
         "--conf", s"${MASTER_REST_SERVER_ENABLED.key}=false",
-        "--conf", s"${HiveUtils.HIVE_METASTORE_VERSION.key}=1.2.1",
+        "--conf", s"${HiveUtils.HIVE_METASTORE_VERSION.key}=$hiveVersion",
         "--conf", s"${HiveUtils.HIVE_METASTORE_JARS.key}=maven",
         "--conf", s"${WAREHOUSE_PATH.key}=${wareHousePath.getCanonicalPath}",
         "--conf", s"spark.sql.test.version.index=$index",
@@ -203,47 +220,43 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils {
     tempPyFile.delete()
   }
 
-  override def beforeAll(): Unit = {
-    super.beforeAll()
-    if (!isTestAtLeastJava9) {
-      prepare()
-    }
-  }
-
   test("backward compatibility") {
-    // TODO SPARK-28704 Test backward compatibility on JDK9+ once we have a version supports JDK9+
-    assume(!isTestAtLeastJava9)
     val args = Seq(
       "--class", PROCESS_TABLES.getClass.getName.stripSuffix("$"),
       "--name", "HiveExternalCatalog backward compatibility test",
       "--master", "local[2]",
       "--conf", s"${UI_ENABLED.key}=false",
       "--conf", s"${MASTER_REST_SERVER_ENABLED.key}=false",
-      "--conf", s"${HiveUtils.HIVE_METASTORE_VERSION.key}=1.2.1",
+      "--conf", s"${HiveUtils.HIVE_METASTORE_VERSION.key}=$hiveVersion",
       "--conf", s"${HiveUtils.HIVE_METASTORE_JARS.key}=maven",
       "--conf", s"${WAREHOUSE_PATH.key}=${wareHousePath.getCanonicalPath}",
       "--driver-java-options", s"-Dderby.system.home=${wareHousePath.getCanonicalPath}",
       unusedJar.toString)
-    runSparkSubmit(args)
+    if (PROCESS_TABLES.testingVersions.nonEmpty) runSparkSubmit(args)
   }
 }
 
 object PROCESS_TABLES extends QueryTest with SQLTestUtils {
-  val releaseMirror = "https://dist.apache.org/repos/dist/release"
+  val releaseMirror = sys.env.getOrElse("SPARK_RELEASE_MIRROR",
+    "https://dist.apache.org/repos/dist/release")
   // Tests the latest version of every release line.
   val testingVersions: Seq[String] = {
     import scala.io.Source
-    try {
-      Source.fromURL(s"${releaseMirror}/spark").mkString
+    val versions: Seq[String] = try Utils.tryWithResource(
+      Source.fromURL(s"$releaseMirror/spark")) { source =>
+      source.mkString
         .split("\n")
-        .filter(_.contains("""<li><a href="spark-"""))
+        .filter(_.contains("""<a href="spark-"""))
         .filterNot(_.contains("preview"))
         .map("""<a href="spark-(\d.\d.\d)/">""".r.findFirstMatchIn(_).get.group(1))
         .filter(_ < org.apache.spark.SPARK_VERSION)
     } catch {
-      // do not throw exception during object initialization.
-      case NonFatal(_) => Seq("2.3.4", "2.4.5") // A temporary fallback to use a specific version
+      // Do not throw exception during object initialization.
+      case NonFatal(_) => Nil
     }
+    versions
+      .filter(v => v.startsWith("3") || !TestUtils.isPythonVersionAtLeast38())
+      .filter(v => v.startsWith("3") || !SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_9))
   }
 
   protected var spark: SparkSession = _
@@ -305,3 +318,8 @@ object PROCESS_TABLES extends QueryTest with SQLTestUtils {
     }
   }
 }
+
+object HiveExternalCatalogVersionsSuite {
+  private val SPARK_TEST_CACHE_DIR_SYSTEM_PROPERTY = "spark.test.cache-dir"
+}
+
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetadataCacheSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetadataCacheSuite.scala
index 743cdbd6457d7..db8ebcd45f3eb 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetadataCacheSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetadataCacheSuite.scala
@@ -21,7 +21,6 @@ import org.apache.hadoop.fs.Path
 
 import org.apache.spark.SparkException
 import org.apache.spark.sql.QueryTest
-import org.apache.spark.sql.execution.adaptive.AdaptiveTestUtils.assertExceptionMessage
 import org.apache.spark.sql.hive.test.TestHiveSingleton
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SQLTestUtils
@@ -100,7 +99,7 @@ class HiveMetadataCacheSuite extends QueryTest with SQLTestUtils with TestHiveSi
             val e = intercept[SparkException] {
               sql("select * from test").count()
             }
-            assertExceptionMessage(e, "FileNotFoundException")
+            assert(e.getMessage.contains("FileNotFoundException"))
 
             // Test refreshing the cache.
             spark.catalog.refreshTable("test")
@@ -115,7 +114,7 @@ class HiveMetadataCacheSuite extends QueryTest with SQLTestUtils with TestHiveSi
             val e2 = intercept[SparkException] {
               sql("select * from test").count()
             }
-            assertExceptionMessage(e2, "FileNotFoundException")
+            assert(e.getMessage.contains("FileNotFoundException"))
             spark.catalog.refreshByPath(dir.getAbsolutePath)
             assert(sql("select * from test").count() == 3)
           }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
index 95e99c653d6f6..1a6f6843d3911 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
@@ -113,24 +113,19 @@ class HiveMetastoreCatalogSuite extends TestHiveSingleton with SQLTestUtils {
         .add("c9", "date")
         .add("c10", "timestamp")
         .add("c11", "string")
-        .add("c12", "string", true,
-          new MetadataBuilder().putString(HIVE_TYPE_STRING, "char(10)").build())
-        .add("c13", "string", true,
-          new MetadataBuilder().putString(HIVE_TYPE_STRING, "varchar(10)").build())
+        .add("c12", CharType(10), true)
+        .add("c13", VarcharType(10), true)
         .add("c14", "binary")
         .add("c15", "decimal")
         .add("c16", "decimal(10)")
         .add("c17", "decimal(10,2)")
         .add("c18", "array<string>")
         .add("c19", "array<int>")
-        .add("c20", "array<string>", true,
-          new MetadataBuilder().putString(HIVE_TYPE_STRING, "array<char(10)>").build())
+        .add("c20", ArrayType(CharType(10)), true)
         .add("c21", "map<int,int>")
-        .add("c22", "map<int,string>", true,
-          new MetadataBuilder().putString(HIVE_TYPE_STRING, "map<int,char(10)>").build())
+        .add("c22", MapType(IntegerType, CharType(10)), true)
         .add("c23", "struct<a:int,b:int>")
-        .add("c24", "struct<c:string,d:int>", true,
-          new MetadataBuilder().putString(HIVE_TYPE_STRING, "struct<c:varchar(10),d:int>").build())
+        .add("c24", new StructType().add("c", VarcharType(10)).add("d", "int"), true)
       assert(schema == expectedSchema)
     }
   }
@@ -206,13 +201,8 @@ class DataSourceWithHiveMetastoreCatalogSuite
         assert(columns.map(_.dataType) === Seq(DecimalType(10, 3), StringType))
 
         checkAnswer(table("t"), testDF)
-        if (HiveUtils.isHive23) {
-          assert(sparkSession.metadataHive.runSqlHive("SELECT * FROM t") ===
-            Seq("1.100\t1", "2.100\t2"))
-        } else {
-          assert(sparkSession.metadataHive.runSqlHive("SELECT * FROM t") ===
-            Seq("1.1\t1", "2.1\t2"))
-        }
+        assert(sparkSession.metadataHive.runSqlHive("SELECT * FROM t") ===
+          Seq("1.100\t1", "2.100\t2"))
       }
     }
 
@@ -244,13 +234,8 @@ class DataSourceWithHiveMetastoreCatalogSuite
           assert(columns.map(_.dataType) === Seq(DecimalType(10, 3), StringType))
 
           checkAnswer(table("t"), testDF)
-          if (HiveUtils.isHive23) {
-            assert(sparkSession.metadataHive.runSqlHive("SELECT * FROM t") ===
-              Seq("1.100\t1", "2.100\t2"))
-          } else {
-            assert(sparkSession.metadataHive.runSqlHive("SELECT * FROM t") ===
-              Seq("1.1\t1", "2.1\t2"))
-          }
+          assert(sparkSession.metadataHive.runSqlHive("SELECT * FROM t") ===
+            Seq("1.100\t1", "2.100\t2"))
         }
       }
     }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetMetastoreSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetMetastoreSuite.scala
index 0bdaa0c23c537..035175430d927 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetMetastoreSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetMetastoreSuite.scala
@@ -473,7 +473,7 @@ class HiveParquetMetastoreSuite extends ParquetPartitioningTest {
     checkCached(tableIdentifier)
     // For insert into non-partitioned table, we will do the conversion,
     // so the converted test_insert_parquet should be cached.
-    sessionState.refreshTable("test_insert_parquet")
+    spark.catalog.refreshTable("test_insert_parquet")
     assert(getCachedDataSourceTable(tableIdentifier) === null)
     sql(
       """
@@ -486,7 +486,7 @@ class HiveParquetMetastoreSuite extends ParquetPartitioningTest {
       sql("select * from test_insert_parquet"),
       sql("select a, b from jt").collect())
     // Invalidate the cache.
-    sessionState.refreshTable("test_insert_parquet")
+    spark.catalog.refreshTable("test_insert_parquet")
     assert(getCachedDataSourceTable(tableIdentifier) === null)
 
     // Create a partitioned table.
@@ -536,7 +536,7 @@ class HiveParquetMetastoreSuite extends ParquetPartitioningTest {
           |select b, '2015-04-02', a FROM jt
         """.stripMargin).collect())
 
-    sessionState.refreshTable("test_parquet_partitioned_cache_test")
+    spark.catalog.refreshTable("test_parquet_partitioned_cache_test")
     assert(getCachedDataSourceTable(tableIdentifier) === null)
 
     dropTables("test_insert_parquet", "test_parquet_partitioned_cache_test")
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSourceSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSourceSuite.scala
index b557fe73f1154..b3ea54a7bc931 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSourceSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSourceSuite.scala
@@ -25,7 +25,6 @@ import org.apache.spark.sql.catalyst.catalog.HiveTableRelation
 import org.apache.spark.sql.execution.datasources.LogicalRelation
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
-import org.apache.spark.util.Utils
 
 /**
  * A suite of tests for the Parquet support through the data sources API.
@@ -230,6 +229,12 @@ class HiveParquetSourceSuite extends ParquetPartitioningTest {
         withTempPath { path =>
           withTable("parq_tbl1", "parq_tbl2", "parq_tbl3",
             "tbl1", "tbl2", "tbl3", "tbl4", "tbl5", "tbl6") {
+
+            def checkErrorMsg(path: String): String = {
+              s"Path: ${path} is a directory, which is not supported by the record reader " +
+                s"when `mapreduce.input.fileinputformat.input.dir.recursive` is false."
+            }
+
             val parquetTblStatement1 =
               s"""
                  |CREATE EXTERNAL TABLE parq_tbl1(
@@ -287,7 +292,7 @@ class HiveParquetSourceSuite extends ParquetPartitioningTest {
               val msg = intercept[IOException] {
                 sql("SELECT * FROM tbl1").show()
               }.getMessage
-              assert(msg.contains("Not a file:"))
+              assert(msg.contains(checkErrorMsg(s"$path/l1")))
             }
 
             val l1DirStatement =
@@ -305,7 +310,7 @@ class HiveParquetSourceSuite extends ParquetPartitioningTest {
               val msg = intercept[IOException] {
                 sql("SELECT * FROM tbl2").show()
               }.getMessage
-              assert(msg.contains("Not a file:"))
+              assert(msg.contains(checkErrorMsg(s"$path/l1/l2")))
             }
 
             val l2DirStatement =
@@ -323,7 +328,7 @@ class HiveParquetSourceSuite extends ParquetPartitioningTest {
               val msg = intercept[IOException] {
                 sql("SELECT * FROM tbl3").show()
               }.getMessage
-              assert(msg.contains("Not a file:"))
+              assert(msg.contains(checkErrorMsg(s"$path/l1/l2/l3")))
             }
 
             val wildcardTopDirStatement =
@@ -341,7 +346,7 @@ class HiveParquetSourceSuite extends ParquetPartitioningTest {
               val msg = intercept[IOException] {
                 sql("SELECT * FROM tbl4").show()
               }.getMessage
-              assert(msg.contains("Not a file:"))
+              assert(msg.contains(checkErrorMsg(s"$path/l1/l2")))
             }
 
             val wildcardL1DirStatement =
@@ -359,7 +364,7 @@ class HiveParquetSourceSuite extends ParquetPartitioningTest {
               val msg = intercept[IOException] {
                 sql("SELECT * FROM tbl5").show()
               }.getMessage
-              assert(msg.contains("Not a file:"))
+              assert(msg.contains(checkErrorMsg(s"$path/l1/l2/l3")))
             }
 
             val wildcardL2DirStatement =
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala
index 470c6a342b4dd..8940ab4664562 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.hive
 
-import org.apache.spark.sql.{QueryTest, Row}
+import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
 import org.apache.spark.sql.execution.datasources.parquet.ParquetTest
 import org.apache.spark.sql.hive.test.TestHiveSingleton
 import org.apache.spark.sql.internal.SQLConf
@@ -106,4 +106,21 @@ class HiveParquetSuite extends QueryTest with ParquetTest with TestHiveSingleton
       }
     }
   }
+
+  test("SPARK-33323: Add query resolved check before convert hive relation") {
+    withTable("t") {
+      val msg = intercept[AnalysisException] {
+        sql(
+          s"""
+             |CREATE TABLE t STORED AS PARQUET AS
+             |SELECT * FROM (
+             | SELECT c3 FROM (
+             |  SELECT c1, c2 from values(1,2) t(c1, c2)
+             |  )
+             |)
+          """.stripMargin)
+      }.getMessage
+      assert(msg.contains("cannot resolve 'c3' given input columns"))
+    }
+  }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSQLInsertTestSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSQLInsertTestSuite.scala
new file mode 100644
index 0000000000000..0b1d511f08511
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSQLInsertTestSuite.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import org.apache.spark.sql.SQLInsertTestSuite
+import org.apache.spark.sql.hive.test.TestHiveSingleton
+
+class HiveSQLInsertTestSuite extends SQLInsertTestSuite with TestHiveSingleton {
+
+  private val originalPartitionMode = spark.conf.getOption("hive.exec.dynamic.partition.mode")
+
+  override protected def beforeAll(): Unit = {
+    super.beforeAll()
+    spark.conf.set("hive.exec.dynamic.partition.mode", "nonstrict")
+  }
+
+  override protected def afterAll(): Unit = {
+    originalPartitionMode
+      .map(v => spark.conf.set("hive.exec.dynamic.partition.mode", v))
+      .getOrElse(spark.conf.unset("hive.exec.dynamic.partition.mode"))
+    super.afterAll()
+  }
+
+  override def format: String = "hive OPTIONS(fileFormat='parquet')"
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSchemaInferenceSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSchemaInferenceSuite.scala
index 590ef949ffbd7..ebf02287489a7 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSchemaInferenceSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSchemaInferenceSuite.scala
@@ -118,18 +118,19 @@ class HiveSchemaInferenceSuite
         properties = Map.empty),
       true)
 
-    // Add partition records (if specified)
-    if (!partitionCols.isEmpty) {
-      spark.catalog.recoverPartitions(TEST_TABLE_NAME)
-    }
-
     // Check that the table returned by HiveExternalCatalog has schemaPreservesCase set to false
     // and that the raw table returned by the Hive client doesn't have any Spark SQL properties
     // set (table needs to be obtained from client since HiveExternalCatalog filters these
     // properties out).
     assert(!externalCatalog.getTable(DATABASE, TEST_TABLE_NAME).schemaPreservesCase)
     val rawTable = client.getTable(DATABASE, TEST_TABLE_NAME)
-    assert(rawTable.properties.filterKeys(_.startsWith(DATASOURCE_SCHEMA_PREFIX)) == Map.empty)
+    assert(rawTable.properties.filterKeys(_.startsWith(DATASOURCE_SCHEMA_PREFIX)).isEmpty)
+
+    // Add partition records (if specified)
+    if (!partitionCols.isEmpty) {
+      spark.catalog.recoverPartitions(TEST_TABLE_NAME)
+    }
+
     schema
   }
 
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSharedStateSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSharedStateSuite.scala
index 78535b094b83d..e2e96bfc3a89f 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSharedStateSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSharedStateSuite.scala
@@ -17,38 +17,55 @@
 
 package org.apache.spark.sql.hive
 
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.hive.common.FileUtils
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars
 
 import org.apache.spark.{SparkConf, SparkContext, SparkFunSuite}
-import org.apache.spark.sql.internal.SharedState
+import org.apache.spark.internal.config.UI
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.internal.StaticSQLConf
 import org.apache.spark.sql.internal.StaticSQLConf._
 import org.apache.spark.util.Utils
 
 class HiveSharedStateSuite extends SparkFunSuite {
 
+  override def beforeEach(): Unit = {
+    SparkSession.clearActiveSession()
+    SparkSession.clearDefaultSession()
+    super.beforeEach()
+  }
+
   test("initial configs should be passed to SharedState but not SparkContext") {
     val conf = new SparkConf().setMaster("local").setAppName("SharedState Test")
     val sc = SparkContext.getOrCreate(conf)
+    val warehousePath = Utils.createTempDir().toString
     val invalidPath = "invalid/path"
     val metastorePath = Utils.createTempDir()
     val tmpDb = "tmp_db"
 
     // The initial configs used to generate SharedState, none of these should affect the global
-    // shared SparkContext's configurations. Especially, all these configs are passed to the cloned
-    // confs inside SharedState except metastore warehouse dir.
+    // shared SparkContext's configurations, except spark.sql.warehouse.dir.
+    // Especially, all these configs are passed to the cloned confs inside SharedState for sharing
+    // cross sessions.
     val initialConfigs = Map("spark.foo" -> "bar",
-      WAREHOUSE_PATH.key -> invalidPath,
-      ConfVars.METASTOREWAREHOUSE.varname -> invalidPath,
+      WAREHOUSE_PATH.key -> warehousePath,
+      ConfVars.METASTOREWAREHOUSE.varname -> warehousePath,
       CATALOG_IMPLEMENTATION.key -> "hive",
       ConfVars.METASTORECONNECTURLKEY.varname ->
         s"jdbc:derby:;databaseName=$metastorePath/metastore_db;create=true",
       GLOBAL_TEMP_DATABASE.key -> tmpDb)
 
-    val state = new SharedState(sc, initialConfigs)
-    assert(sc.conf.get(WAREHOUSE_PATH.key) !== invalidPath,
-      "warehouse conf in session options can't affect application wide spark conf")
-    assert(sc.hadoopConfiguration.get(ConfVars.METASTOREWAREHOUSE.varname) !== invalidPath,
-      "warehouse conf in session options can't affect application wide hadoop conf")
+    val builder = SparkSession.builder()
+    initialConfigs.foreach { case (k, v) => builder.config(k, v) }
+    val ss = builder.getOrCreate()
+    val state = ss.sharedState
+    val qualifiedWHPath =
+      FileUtils.makeQualified(new Path(warehousePath), sc.hadoopConfiguration).toString
+    assert(sc.conf.get(WAREHOUSE_PATH.key) === qualifiedWHPath,
+      "initial warehouse conf in session options can affect application wide spark conf")
+    assert(sc.hadoopConfiguration.get(ConfVars.METASTOREWAREHOUSE.varname) === qualifiedWHPath,
+      "initial warehouse conf in session options can affect application wide hadoop conf")
 
     assert(!state.sparkContext.conf.contains("spark.foo"),
       "static spark conf should not be affected by session")
@@ -57,9 +74,39 @@ class HiveSharedStateSuite extends SparkFunSuite {
     val client = state.externalCatalog.unwrapped.asInstanceOf[HiveExternalCatalog].client
     assert(client.getConf("spark.foo", "") === "bar",
       "session level conf should be passed to catalog")
-    assert(client.getConf(ConfVars.METASTOREWAREHOUSE.varname, invalidPath) !== invalidPath,
-      "session level conf should be passed to catalog except warehouse dir")
+    assert(client.getConf(ConfVars.METASTOREWAREHOUSE.varname, "") === qualifiedWHPath,
+      "session level conf should be passed to catalog")
 
     assert(state.globalTempViewManager.database === tmpDb)
+
+   val ss2 =
+     builder.config("spark.foo", "bar2222").config(WAREHOUSE_PATH.key, invalidPath).getOrCreate()
+
+    assert(!ss2.sparkContext.conf.get(WAREHOUSE_PATH.key).contains(invalidPath),
+      "warehouse conf in session options can't affect application wide spark conf")
+    assert(ss2.sparkContext.hadoopConfiguration.get(ConfVars.METASTOREWAREHOUSE.varname) !==
+      invalidPath, "warehouse conf in session options can't affect application wide hadoop conf")
+    assert(ss.conf.get("spark.foo") === "bar2222", "session level conf should be passed to catalog")
+    assert(!ss.conf.get(WAREHOUSE_PATH).contains(invalidPath),
+      "session level conf should be passed to catalog")
+  }
+
+  test("SPARK-34568: When SparkContext's conf not enable hive, " +
+    "we should respect `enableHiveSupport()` when build SparkSession too") {
+    val conf = new SparkConf().setMaster("local").setAppName("SPARK-34568")
+      .set(UI.UI_ENABLED, false)
+    val sc = SparkContext.getOrCreate(conf)
+    val catalog = sc.conf.get(StaticSQLConf.CATALOG_IMPLEMENTATION)
+    try {
+      sc.conf.set(StaticSQLConf.CATALOG_IMPLEMENTATION, "in-memory")
+      val sparkSession = SparkSession.builder().enableHiveSupport().sparkContext(sc).getOrCreate()
+      assert(
+        sparkSession.sparkContext.conf.get(StaticSQLConf.CATALOG_IMPLEMENTATION) === "in-memory")
+      assert(sparkSession.sharedState.conf.get(StaticSQLConf.CATALOG_IMPLEMENTATION) === "hive")
+      assert(sparkSession.sessionState.catalog.getClass
+        .getCanonicalName.contains("HiveSessionCatalog"))
+    } finally {
+      sc.conf.set(StaticSQLConf.CATALOG_IMPLEMENTATION, catalog)
+    }
   }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShimSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShimSuite.scala
index 14d07cdf8db08..89131a79e59de 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShimSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShimSuite.scala
@@ -16,9 +16,6 @@
  */
 package org.apache.spark.sql.hive
 
-import scala.collection.JavaConverters._
-import scala.language.implicitConversions
-
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.hive.serde2.ColumnProjectionUtils
 
@@ -35,18 +32,10 @@ class HiveShimSuite extends SparkFunSuite {
 
     // test when READ_COLUMN_NAMES_CONF_STR is empty
     HiveShim.appendReadColumns(conf, ids, names)
-    if (HiveUtils.isHive23) {
-      assert(names === ColumnProjectionUtils.getReadColumnNames(conf))
-    } else {
-      assert(names.asJava === ColumnProjectionUtils.getReadColumnNames(conf))
-    }
+    assert(names === ColumnProjectionUtils.getReadColumnNames(conf))
 
     // test when READ_COLUMN_NAMES_CONF_STR is non-empty
     HiveShim.appendReadColumns(conf, moreIds, moreNames)
-    if (HiveUtils.isHive23) {
-      assert((names ++ moreNames) === ColumnProjectionUtils.getReadColumnNames(conf))
-    } else {
-      assert((names ++ moreNames).asJava === ColumnProjectionUtils.getReadColumnNames(conf))
-    }
+    assert((names ++ moreNames) === ColumnProjectionUtils.getReadColumnNames(conf))
   }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala
index cfcf70c0e79f0..2fb67c793dc6a 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala
@@ -25,6 +25,22 @@ import org.apache.spark.sql.internal.{HiveSerDe, SQLConf}
 
 class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSingleton {
 
+  private var origCreateHiveTableConfig = false
+
+  protected override def beforeAll(): Unit = {
+    super.beforeAll()
+    origCreateHiveTableConfig =
+      spark.conf.get(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT)
+    spark.conf.set(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT.key, true)
+  }
+
+  protected override def afterAll(): Unit = {
+    spark.conf.set(
+      SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT.key,
+      origCreateHiveTableConfig)
+    super.afterAll()
+  }
+
   test("view") {
     Seq(true, false).foreach { serde =>
       withView("v1") {
@@ -279,7 +295,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet
       table.copy(
         createTime = 0L,
         lastAccessTime = 0L,
-        properties = table.properties.filterKeys(!nondeterministicProps.contains(_)),
+        properties = table.properties.filterKeys(!nondeterministicProps.contains(_)).toMap,
         stats = None,
         ignoredProperties = Map.empty,
         storage = table.storage.copy(properties = Map.empty),
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
index 8b97489e2d818..d56d7f39e39e3 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
@@ -23,8 +23,10 @@ import scala.util.Properties
 
 import org.apache.commons.lang3.{JavaVersion, SystemUtils}
 import org.apache.hadoop.fs.Path
-import org.scalatest.{BeforeAndAfterEach, Matchers}
+import org.apache.hadoop.hive.common.FileUtils
 import org.scalatest.Assertions._
+import org.scalatest.BeforeAndAfterEach
+import org.scalatest.matchers.must.Matchers
 
 import org.apache.spark._
 import org.apache.spark.internal.Logging
@@ -35,15 +37,16 @@ import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.execution.command.DDLUtils
 import org.apache.spark.sql.expressions.Window
 import org.apache.spark.sql.hive.test.{HiveTestJars, TestHiveContext}
-import org.apache.spark.sql.internal.SQLConf.SHUFFLE_PARTITIONS
+import org.apache.spark.sql.internal.SQLConf.{LEGACY_TIME_PARSER_POLICY, SHUFFLE_PARTITIONS}
 import org.apache.spark.sql.internal.StaticSQLConf.WAREHOUSE_PATH
 import org.apache.spark.sql.types.{DecimalType, StructType}
-import org.apache.spark.tags.ExtendedHiveTest
+import org.apache.spark.tags.{ExtendedHiveTest, SlowHiveTest}
 import org.apache.spark.util.{ResetSystemProperties, Utils}
 
 /**
  * This suite tests spark-submit with applications using HiveContext.
  */
+@SlowHiveTest
 @ExtendedHiveTest
 class HiveSparkSubmitSuite
   extends SparkSubmitTestUtils
@@ -151,7 +154,7 @@ class HiveSparkSubmitSuite
     // For more detail, see sql/hive/src/test/resources/regression-test-SPARK-8489/*scala.
     // TODO: revisit for Scala 2.13 support
     val version = Properties.versionNumberString match {
-      case v if v.startsWith("2.12") => v.substring(0, 4)
+      case v if v.startsWith("2.12") || v.startsWith("2.13") => v.substring(0, 4)
       case x => throw new Exception(s"Unsupported Scala Version: $x")
     }
     val jarDir = getTestResourcePath("regression-test-SPARK-8489")
@@ -335,6 +338,30 @@ class HiveSparkSubmitSuite
       unusedJar.toString)
     runSparkSubmit(argsForShowTables)
   }
+
+  test("SPARK-34772: RebaseDateTime loadRebaseRecords should use Spark classloader " +
+    "instead of context") {
+    assume(!SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_9))
+    val unusedJar = TestUtils.createJarWithClasses(Seq.empty)
+
+    // We need to specify the metastore database location in case of conflict with other hive
+    // versions.
+    withTempDir { file =>
+      file.delete()
+      val metastore = s"jdbc:derby:;databaseName=${file.getAbsolutePath};create=true"
+
+      val args = Seq(
+        "--class", SPARK_34772.getClass.getName.stripSuffix("$"),
+        "--name", "SPARK-34772",
+        "--master", "local-cluster[2,1,1024]",
+        "--conf", s"${LEGACY_TIME_PARSER_POLICY.key}=LEGACY",
+        "--conf", s"${HiveUtils.HIVE_METASTORE_VERSION.key}=1.2.1",
+        "--conf", s"${HiveUtils.HIVE_METASTORE_JARS.key}=maven",
+        "--conf", s"spark.hadoop.javax.jdo.option.ConnectionURL=$metastore",
+        unusedJar.toString)
+      runSparkSubmit(args)
+    }
+  }
 }
 
 object SetMetastoreURLTest extends Logging {
@@ -406,10 +433,11 @@ object SetWarehouseLocationTest extends Logging {
 
     }
 
-    if (sparkSession.conf.get(WAREHOUSE_PATH.key) != expectedWarehouseLocation) {
+    val qualifiedWHPath = FileUtils.makeQualified(
+      new Path(expectedWarehouseLocation), sparkSession.sparkContext.hadoopConfiguration).toString
+    if (sparkSession.conf.get(WAREHOUSE_PATH.key) != qualifiedWHPath) {
       throw new Exception(
-        s"${WAREHOUSE_PATH.key} is not set to the expected warehouse location " +
-        s"$expectedWarehouseLocation.")
+        s"${WAREHOUSE_PATH.key} is not set to the expected warehouse location $qualifiedWHPath.")
     }
 
     val catalog = sparkSession.sessionState.catalog
@@ -422,7 +450,7 @@ object SetWarehouseLocationTest extends Logging {
       val tableMetadata =
         catalog.getTableMetadata(TableIdentifier("testLocation", Some("default")))
       val expectedLocation =
-        CatalogUtils.stringToURI(s"file:${expectedWarehouseLocation.toString}/testlocation")
+        CatalogUtils.stringToURI(s"$qualifiedWHPath/testlocation")
       val actualLocation = tableMetadata.location
       if (actualLocation != expectedLocation) {
         throw new Exception(
@@ -438,7 +466,7 @@ object SetWarehouseLocationTest extends Logging {
       val tableMetadata =
         catalog.getTableMetadata(TableIdentifier("testLocation", Some("testLocationDB")))
       val expectedLocation = CatalogUtils.stringToURI(
-        s"file:${expectedWarehouseLocation.toString}/testlocationdb.db/testlocation")
+        s"$qualifiedWHPath/testlocationdb.db/testlocation")
       val actualLocation = tableMetadata.location
       if (actualLocation != expectedLocation) {
         throw new Exception(
@@ -709,7 +737,7 @@ object SPARK_9757 extends QueryTest {
         val df =
           hiveContext
             .range(10)
-            .select(callUDF("struct", ($"id" + 0.2) cast DecimalType(10, 3)) as "dec_struct")
+            .select(call_udf("struct", ($"id" + 0.2) cast DecimalType(10, 3)) as "dec_struct")
         df.write.option("path", dir.getCanonicalPath).mode("overwrite").saveAsTable("t")
         checkAnswer(hiveContext.table("t"), df)
       }
@@ -768,8 +796,6 @@ object SPARK_14244 extends QueryTest {
     val hiveContext = new TestHiveContext(sparkContext)
     spark = hiveContext.sparkSession
 
-    import hiveContext.implicits._
-
     try {
       val window = Window.orderBy("id")
       val df = spark.range(2).select(cume_dist().over(window).as("cdist")).orderBy("cdist")
@@ -845,3 +871,18 @@ object SPARK_18989_DESC_TABLE {
     }
   }
 }
+
+object SPARK_34772 {
+  def main(args: Array[String]): Unit = {
+    val spark = SparkSession.builder()
+      .config(UI_ENABLED.key, "false")
+      .enableHiveSupport()
+      .getOrCreate()
+    try {
+      spark.sql("CREATE TABLE t (c int) PARTITIONED BY (p date)")
+      spark.sql("SELECT * FROM t WHERE p='2021-01-01'").collect()
+    } finally {
+      spark.sql("DROP TABLE IF EXISTS t")
+    }
+  }
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUserDefinedTypeSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUserDefinedTypeSuite.scala
index ca1af73b038a7..d0af8dc7ae49f 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUserDefinedTypeSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUserDefinedTypeSuite.scala
@@ -18,7 +18,6 @@
 package org.apache.spark.sql.hive
 
 import scala.collection.JavaConverters._
-import scala.util.Random
 
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDF
 import org.apache.hadoop.hive.serde2.objectinspector.{ObjectInspector, StandardListObjectInspector}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala
index 4ad97eaa2b1c8..d8e1e01292820 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala
@@ -23,9 +23,8 @@ import org.apache.hadoop.hive.conf.HiveConf.ConfVars
 import org.apache.spark.SparkConf
 import org.apache.spark.deploy.SparkHadoopUtil
 import org.apache.spark.sql.QueryTest
-import org.apache.spark.sql.execution.HiveResult
 import org.apache.spark.sql.hive.test.TestHiveSingleton
-import org.apache.spark.sql.test.{ExamplePoint, ExamplePointUDT, SQLTestUtils}
+import org.apache.spark.sql.test.SQLTestUtils
 import org.apache.spark.util.ChildFirstURLClassLoader
 
 class HiveUtilsSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala
index 421dcb499bd6a..b715f484fa02a 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala
@@ -35,7 +35,7 @@ import org.apache.spark.util.Utils
 
 case class TestData(key: Int, value: String)
 
-case class ThreeCloumntable(key: Int, value: String, key1: String)
+case class ThreeColumnTable(key: Int, value: String, key1: String)
 
 class InsertSuite extends QueryTest with TestHiveSingleton with BeforeAndAfter
     with SQLTestUtils  with PrivateMethodTester  {
@@ -277,7 +277,8 @@ class InsertSuite extends QueryTest with TestHiveSingleton with BeforeAndAfter
   test("Test partition mode = strict") {
     withSQLConf(("hive.exec.dynamic.partition.mode", "strict")) {
       withTable("partitioned") {
-        sql("CREATE TABLE partitioned (id bigint, data string) PARTITIONED BY (part string)")
+        sql("CREATE TABLE partitioned (id bigint, data string) USING hive " +
+          "PARTITIONED BY (part string)")
         val data = (1 to 10).map(i => (i, s"data-$i", if ((i % 2) == 0) "even" else "odd"))
           .toDF("id", "data", "part")
 
@@ -763,7 +764,7 @@ class InsertSuite extends QueryTest with TestHiveSingleton with BeforeAndAfter
       val path = dir.toURI.getPath
 
       val e = intercept[AnalysisException] {
-        sql(s"INSERT OVERWRITE LOCAL DIRECTORY '${path}' TABLE notexists")
+        sql(s"INSERT OVERWRITE LOCAL DIRECTORY '${path}' TABLE nonexistent")
       }.getMessage
       assert(e.contains("Table or view not found"))
     }
@@ -847,4 +848,26 @@ class InsertSuite extends QueryTest with TestHiveSingleton with BeforeAndAfter
       }
     }
   }
+
+  test("SPARK-32508 " +
+    "Disallow empty part col values in partition spec before static partition writing") {
+    withTable("t1") {
+      spark.sql(
+        """
+          |CREATE TABLE t1 (c1 int)
+          |PARTITIONED BY (d string)
+          """.stripMargin)
+
+      val e = intercept[AnalysisException] {
+        spark.sql(
+          """
+            |INSERT OVERWRITE TABLE t1 PARTITION(d='')
+            |SELECT 1
+          """.stripMargin)
+      }.getMessage
+
+      assert(!e.contains("get partition: Value for key d is null or empty"))
+      assert(e.contains("Partition spec is invalid"))
+    }
+  }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/ListTablesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/ListTablesSuite.scala
index 32db22e704b3e..49cd0885e722a 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/ListTablesSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/ListTablesSuite.scala
@@ -22,9 +22,13 @@ import org.scalatest.BeforeAndAfterAll
 import org.apache.spark.sql.QueryTest
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.analysis.AnalysisTest
 import org.apache.spark.sql.hive.test.TestHiveSingleton
 
-class ListTablesSuite extends QueryTest with TestHiveSingleton with BeforeAndAfterAll {
+class ListTablesSuite extends QueryTest
+  with AnalysisTest
+  with TestHiveSingleton
+  with BeforeAndAfterAll {
   import hiveContext._
   import hiveContext.implicits._
 
@@ -33,8 +37,8 @@ class ListTablesSuite extends QueryTest with TestHiveSingleton with BeforeAndAft
   override def beforeAll(): Unit = {
     super.beforeAll()
     // The catalog in HiveContext is a case insensitive one.
-    sessionState.catalog.createTempView(
-      "ListTablesSuiteTable", df.logicalPlan, overrideIfExists = true)
+    createTempView(
+      sessionState.catalog, "ListTablesSuiteTable", df.logicalPlan, overrideIfExists = true)
     sql("CREATE TABLE HiveListTablesSuiteTable (key int, value string)")
     sql("CREATE DATABASE IF NOT EXISTS ListTablesSuiteDB")
     sql("CREATE TABLE ListTablesSuiteDB.HiveInDBListTablesSuiteTable (key int, value string)")
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
index 41a26344f7c21..ba441922d3600 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
@@ -256,13 +256,13 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
           sql("SELECT `c_!@(3)` FROM expectedJsonTable").collect().toSeq)
 
         // Discard the cached relation.
-        sessionState.refreshTable("jsonTable")
+        spark.catalog.refreshTable("jsonTable")
 
         checkAnswer(
           sql("SELECT * FROM jsonTable"),
           sql("SELECT `c_!@(3)` FROM expectedJsonTable").collect().toSeq)
 
-        sessionState.refreshTable("jsonTable")
+        spark.catalog.refreshTable("jsonTable")
         val expectedSchema = StructType(StructField("c_!@(3)", IntegerType, true) :: Nil)
 
         assert(expectedSchema === table("jsonTable").schema)
@@ -350,7 +350,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
            """.stripMargin)
 
         // Discard the cached relation.
-        sessionState.refreshTable("ctasJsonTable")
+        spark.catalog.refreshTable("ctasJsonTable")
 
         // Schema should not be changed.
         assert(table("ctasJsonTable").schema === table("jsonTable").schema)
@@ -438,7 +438,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
         sql("SELECT * FROM savedJsonTable tmp where tmp.a > 5"),
         (6 to 10).map(i => Row(i, s"str$i")))
 
-      sessionState.refreshTable("savedJsonTable")
+      spark.catalog.refreshTable("savedJsonTable")
 
       checkAnswer(
         sql("SELECT * FROM savedJsonTable where savedJsonTable.a < 5"),
@@ -711,14 +711,14 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
           identifier = TableIdentifier("wide_schema"),
           tableType = CatalogTableType.EXTERNAL,
           storage = CatalogStorageFormat.empty.copy(
-            properties = Map("path" -> tempDir.getCanonicalPath)
+            locationUri = Some(tempDir.toURI)
           ),
           schema = schema,
           provider = Some("json")
         )
         spark.sessionState.catalog.createTable(tableDesc, ignoreIfExists = false)
 
-        sessionState.refreshTable("wide_schema")
+        spark.catalog.refreshTable("wide_schema")
 
         val actualSchema = table("wide_schema").schema
         assert(schema === actualSchema)
@@ -751,7 +751,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
 
       hiveClient.createTable(hiveTable, ignoreIfExists = false)
 
-      sessionState.refreshTable(tableName)
+      spark.catalog.refreshTable(tableName)
       val actualSchema = table(tableName).schema
       assert(schema === actualSchema)
     }
@@ -763,7 +763,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
 
     withTable(tableName) {
       df.write.format("parquet").partitionBy("d", "b").saveAsTable(tableName)
-      sessionState.refreshTable(tableName)
+      spark.catalog.refreshTable(tableName)
       val metastoreTable = hiveClient.getTable("default", tableName)
       val expectedPartitionColumns = StructType(df.schema("d") :: df.schema("b") :: Nil)
 
@@ -798,7 +798,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
         .bucketBy(8, "d", "b")
         .sortBy("c")
         .saveAsTable(tableName)
-      sessionState.refreshTable(tableName)
+      spark.catalog.refreshTable(tableName)
       val metastoreTable = hiveClient.getTable("default", tableName)
       val expectedBucketByColumns = StructType(df.schema("d") :: df.schema("b") :: Nil)
       val expectedSortByColumns = StructType(df.schema("c") :: Nil)
@@ -1076,7 +1076,8 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
         identifier = TableIdentifier("skip_hive_metadata", Some("default")),
         tableType = CatalogTableType.EXTERNAL,
         storage = CatalogStorageFormat.empty.copy(
-          properties = Map("path" -> tempPath.getCanonicalPath, "skipHiveMetadata" -> "true")
+          locationUri = Some(tempPath.toURI),
+          properties = Map("skipHiveMetadata" -> "true")
         ),
         schema = schema,
         provider = Some("parquet")
@@ -1337,7 +1338,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
       val e = intercept[AnalysisException] {
         sharedState.externalCatalog.getTable("default", "t")
       }.getMessage
-      assert(e.contains(s"Could not read schema from the hive metastore because it is corrupted"))
+      assert(e.contains("Cannot read table property 'spark.sql.sources.schema' as it's corrupted"))
 
       withDebugMode {
         val tableMeta = sharedState.externalCatalog.getTable("default", "t")
@@ -1354,7 +1355,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
       val newSession = sparkSession.newSession()
       newSession.sql("CREATE TABLE abc(i int) USING json")
       val tableMeta = newSession.sessionState.catalog.getTableMetadata(TableIdentifier("abc"))
-      assert(tableMeta.properties(DATASOURCE_SCHEMA_NUMPARTS).toInt == 1)
+      assert(tableMeta.properties.contains(DATASOURCE_SCHEMA))
       assert(tableMeta.properties(DATASOURCE_PROVIDER) == "json")
     }
   }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/ParquetEncryptionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/ParquetEncryptionSuite.scala
new file mode 100644
index 0000000000000..184ccadcb2612
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/ParquetEncryptionSuite.scala
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import java.io.File
+import java.io.RandomAccessFile
+import java.nio.charset.StandardCharsets
+import java.util.Base64
+
+import org.apache.spark.sql.QueryTest
+import org.apache.spark.sql.hive.test.TestHiveSingleton
+
+/**
+ * A test suite that tests parquet modular encryption usage.
+ */
+class ParquetEncryptionSuite extends QueryTest with TestHiveSingleton {
+  import spark.implicits._
+
+  private val encoder = Base64.getEncoder
+  private val footerKey =
+    encoder.encodeToString("0123456789012345".getBytes(StandardCharsets.UTF_8))
+  private val key1 = encoder.encodeToString("1234567890123450".getBytes(StandardCharsets.UTF_8))
+  private val key2 = encoder.encodeToString("1234567890123451".getBytes(StandardCharsets.UTF_8))
+
+  test("SPARK-34990: Write and read an encrypted parquet") {
+    withTempDir { dir =>
+      withSQLConf(
+        "parquet.crypto.factory.class" ->
+          "org.apache.parquet.crypto.keytools.PropertiesDrivenCryptoFactory",
+        "parquet.encryption.kms.client.class" ->
+          "org.apache.parquet.crypto.keytools.mocks.InMemoryKMS",
+        "parquet.encryption.key.list" ->
+          s"footerKey: ${footerKey}, key1: ${key1}, key2: ${key2}") {
+
+        val inputDF = Seq((1, 22, 333)).toDF("a", "b", "c")
+        val parquetDir = new File(dir, "parquet").getCanonicalPath
+        inputDF.write
+          .option("parquet.encryption.column.keys", "key1: a, b; key2: c")
+          .option("parquet.encryption.footer.key", "footerKey")
+          .parquet(parquetDir)
+
+        verifyParquetEncrypted(parquetDir)
+
+        val parquetDF = spark.read.parquet(parquetDir)
+        assert(parquetDF.inputFiles.nonEmpty)
+        val readDataset = parquetDF.select("a", "b", "c")
+        checkAnswer(readDataset, inputDF)
+      }
+    }
+  }
+
+  /**
+   * Verify that the directory contains an encrypted parquet in
+   * encrypted footer mode by means of checking for all the parquet part files
+   * in the parquet directory that their magic string is PARE, as defined in the spec:
+   * https://github.com/apache/parquet-format/blob/master/Encryption.md#54-encrypted-footer-mode
+   */
+  private def verifyParquetEncrypted(parquetDir: String): Unit = {
+    val parquetPartitionFiles = getListOfParquetFiles(new File(parquetDir))
+    assert(parquetPartitionFiles.size >= 1)
+    parquetPartitionFiles.foreach { parquetFile =>
+      val magicString = "PARE"
+      val magicStringLength = magicString.length()
+      val byteArray = new Array[Byte](magicStringLength)
+      val randomAccessFile = new RandomAccessFile(parquetFile, "r")
+      try {
+        randomAccessFile.read(byteArray, 0, magicStringLength)
+      } finally {
+        randomAccessFile.close()
+      }
+      val stringRead = new String(byteArray, StandardCharsets.UTF_8)
+      assert(magicString == stringRead)
+    }
+  }
+
+  private def getListOfParquetFiles(dir: File): List[File] = {
+    dir.listFiles.filter(_.isFile).toList.filter { file =>
+      file.getName.endsWith("parquet")
+    }
+  }
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionProviderCompatibilitySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionProviderCompatibilitySuite.scala
index 80afc9d8f44bc..e1b0637963b75 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionProviderCompatibilitySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionProviderCompatibilitySuite.scala
@@ -53,7 +53,8 @@ class PartitionProviderCompatibilitySuite
       s"ALTER TABLE $tableName PARTITION (partCol=1) SET LOCATION '/foo'",
       s"ALTER TABLE $tableName DROP PARTITION (partCol=1)",
       s"DESCRIBE $tableName PARTITION (partCol=1)",
-      s"SHOW PARTITIONS $tableName")
+      s"SHOW PARTITIONS $tableName",
+      s"SHOW TABLE EXTENDED LIKE '$tableName' PARTITION (partCol=1)")
 
     withSQLConf(SQLConf.HIVE_MANAGE_FILESOURCE_PARTITIONS.key -> "true") {
       for (cmd <- unsupportedCommands) {
@@ -124,10 +125,15 @@ class PartitionProviderCompatibilitySuite
         }
         // disabled
         withSQLConf(SQLConf.HIVE_MANAGE_FILESOURCE_PARTITIONS.key -> "false") {
-          val e = intercept[AnalysisException] {
-            spark.sql(s"show partitions test")
+          Seq(
+            "SHOW PARTITIONS test",
+            "SHOW TABLE EXTENDED LIKE 'test' PARTITION (partCol=1)"
+          ).foreach { showPartitions =>
+            val e = intercept[AnalysisException] {
+              spark.sql(showPartitions)
+            }
+            assert(e.getMessage.contains("filesource partition management is disabled"))
           }
-          assert(e.getMessage.contains("filesource partition management is disabled"))
           spark.sql("refresh table test")
           assert(spark.sql("select * from test").count() == 5)
         }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionedTablePerfStatsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionedTablePerfStatsSuite.scala
index 3af163af0968c..de3b1ffccf00c 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionedTablePerfStatsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionedTablePerfStatsSuite.scala
@@ -300,7 +300,7 @@ class PartitionedTablePerfStatsSuite
 
           HiveCatalogMetrics.reset()
           assert(spark.sql("show partitions test").count() == 100)
-          assert(HiveCatalogMetrics.METRIC_HIVE_CLIENT_CALLS.getCount() < 10)
+          assert(HiveCatalogMetrics.METRIC_HIVE_CLIENT_CALLS.getCount() <= 10)
         }
       }
     }
@@ -323,7 +323,7 @@ class PartitionedTablePerfStatsSuite
 
           HiveCatalogMetrics.reset()
           assert(spark.sql("show partitions test").count() == 100)
-          assert(HiveCatalogMetrics.METRIC_HIVE_CLIENT_CALLS.getCount() < 10)
+          assert(HiveCatalogMetrics.METRIC_HIVE_CLIENT_CALLS.getCount() <= 10)
         }
       }
     }
@@ -390,8 +390,8 @@ class PartitionedTablePerfStatsSuite
     withSQLConf(SQLConf.HIVE_MANAGE_FILESOURCE_PARTITIONS.key -> "false") {
       withTable("test") {
         withTempDir { dir =>
-          HiveCatalogMetrics.reset()
           setupPartitionedHiveTable("test", dir, 50)
+          HiveCatalogMetrics.reset()
           // select the table in multi-threads
           val executorPool = Executors.newFixedThreadPool(10)
           (1 to 10).map(threadId => {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/QueryPartitionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/QueryPartitionSuite.scala
index 1e396553c9c52..cec6ec1ee1275 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/QueryPartitionSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/QueryPartitionSuite.scala
@@ -17,12 +17,8 @@
 
 package org.apache.spark.sql.hive
 
-import java.io.File
 import java.sql.Timestamp
 
-import com.google.common.io.Files
-import org.apache.hadoop.fs.FileSystem
-
 import org.apache.spark.internal.config._
 import org.apache.spark.sql._
 import org.apache.spark.sql.hive.test.TestHiveSingleton
@@ -42,7 +38,7 @@ class QueryPartitionSuite extends QueryTest with SQLTestUtils with TestHiveSingl
           testData.createOrReplaceTempView("testData")
 
           // create the table for test
-          sql(s"CREATE TABLE table_with_partition(key int,value string) " +
+          sql(s"CREATE TABLE table_with_partition(key int,value string) USING hive " +
               s"PARTITIONED by (ds string) location '${tmpDir.toURI}' ")
           sql("INSERT OVERWRITE TABLE table_with_partition  partition (ds='1') " +
               "SELECT key,value FROM testData")
@@ -85,7 +81,8 @@ class QueryPartitionSuite extends QueryTest with SQLTestUtils with TestHiveSingl
 
   test("SPARK-21739: Cast expression should initialize timezoneId") {
     withTable("table_with_timestamp_partition") {
-      sql("CREATE TABLE table_with_timestamp_partition(value int) PARTITIONED BY (ts TIMESTAMP)")
+      sql("CREATE TABLE table_with_timestamp_partition(value int) USING hive " +
+        "PARTITIONED BY (ts TIMESTAMP)")
       sql("INSERT OVERWRITE TABLE table_with_timestamp_partition " +
         "PARTITION (ts = '2010-01-01 00:00:00.000') VALUES (1)")
 
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
index c1eab63ec073f..8afdd50b0c96b 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
@@ -31,7 +31,7 @@ import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.analysis.NoSuchPartitionException
 import org.apache.spark.sql.catalyst.catalog.{CatalogColumnStat, CatalogStatistics, HiveTableRelation}
-import org.apache.spark.sql.catalyst.plans.logical.{ColumnStat, HistogramBin, HistogramSerializer}
+import org.apache.spark.sql.catalyst.plans.logical.HistogramBin
 import org.apache.spark.sql.catalyst.util.{DateTimeUtils, StringUtils}
 import org.apache.spark.sql.execution.command.{AnalyzeColumnCommand, CommandUtils, DDLUtils}
 import org.apache.spark.sql.execution.datasources.LogicalRelation
@@ -101,14 +101,9 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
             .asInstanceOf[HiveTableRelation]
 
           val properties = relation.tableMeta.ignoredProperties
-          if (HiveUtils.isHive23) {
-            // Since HIVE-6727, Hive fixes table-level stats for external tables are incorrect.
-            assert(properties("totalSize").toLong == 6)
-            assert(properties.get("rawDataSize").isEmpty)
-          } else {
-            assert(properties("totalSize").toLong <= 0, "external table totalSize must be <= 0")
-            assert(properties("rawDataSize").toLong <= 0, "external table rawDataSize must be <= 0")
-          }
+          // Since HIVE-6727, Hive fixes table-level stats for external tables are incorrect.
+          assert(properties("totalSize").toLong == 6)
+          assert(properties.get("rawDataSize").isEmpty)
 
           val sizeInBytes = relation.stats.sizeInBytes
           assert(sizeInBytes === BigInt(file1.length() + file2.length()))
@@ -170,7 +165,8 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
     // Partitioned table
     val partTable = "part_table"
     withTable(partTable) {
-      sql(s"CREATE TABLE $partTable (key STRING, value STRING) PARTITIONED BY (ds STRING)")
+      sql(s"CREATE TABLE $partTable (key STRING, value STRING) USING hive " +
+        "PARTITIONED BY (ds STRING)")
       sql(s"INSERT INTO TABLE $partTable PARTITION (ds='2010-01-01') SELECT * FROM src")
       sql(s"INSERT INTO TABLE $partTable PARTITION (ds='2010-01-02') SELECT * FROM src")
       sql(s"INSERT INTO TABLE $partTable PARTITION (ds='2010-01-03') SELECT * FROM src")
@@ -196,7 +192,8 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
       SQLConf.PARALLEL_FILE_LISTING_IN_STATS_COMPUTATION.key -> "True") {
       val checkSizeTable = "checkSizeTable"
       withTable(checkSizeTable) {
-          sql(s"CREATE TABLE $checkSizeTable (key STRING, value STRING) PARTITIONED BY (ds STRING)")
+          sql(s"CREATE TABLE $checkSizeTable (key STRING, value STRING) USING hive " +
+            "PARTITIONED BY (ds STRING)")
           sql(s"INSERT INTO TABLE $checkSizeTable PARTITION (ds='2010-01-01') SELECT * FROM src")
           sql(s"INSERT INTO TABLE $checkSizeTable PARTITION (ds='2010-01-02') SELECT * FROM src")
           sql(s"INSERT INTO TABLE $checkSizeTable PARTITION (ds='2010-01-03') SELECT * FROM src")
@@ -279,7 +276,8 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
   test("SPARK-22745 - read Hive's statistics for partition") {
     val tableName = "hive_stats_part_table"
     withTable(tableName) {
-      sql(s"CREATE TABLE $tableName (key STRING, value STRING) PARTITIONED BY (ds STRING)")
+      sql(s"CREATE TABLE $tableName (key STRING, value STRING) USING hive " +
+        "PARTITIONED BY (ds STRING)")
       sql(s"INSERT INTO TABLE $tableName PARTITION (ds='2017-01-01') SELECT * FROM src")
       var partition = spark.sessionState.catalog
         .getPartition(TableIdentifier(tableName), Map("ds" -> "2017-01-01"))
@@ -301,7 +299,8 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
     val tableName = "analyzeTable_part"
     withTable(tableName) {
       withTempPath { path =>
-        sql(s"CREATE TABLE $tableName (key STRING, value STRING) PARTITIONED BY (ds STRING)")
+        sql(s"CREATE TABLE $tableName (key STRING, value STRING) USING hive " +
+          "PARTITIONED BY (ds STRING)")
 
         val partitionDates = List("2010-01-01", "2010-01-02", "2010-01-03")
         partitionDates.foreach { ds =>
@@ -326,6 +325,7 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
           sql(
             s"""
                |CREATE TABLE $sourceTableName (key STRING, value STRING)
+               |USING hive
                |PARTITIONED BY (ds STRING)
                |LOCATION '${path.toURI}'
              """.stripMargin)
@@ -343,6 +343,7 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
           sql(
             s"""
                |CREATE TABLE $tableName (key STRING, value STRING)
+               |USING hive
                |PARTITIONED BY (ds STRING)
                |LOCATION '${path.toURI}'
              """.stripMargin)
@@ -376,7 +377,8 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
     }
 
     withTable(tableName) {
-      sql(s"CREATE TABLE $tableName (key STRING, value STRING) PARTITIONED BY (ds STRING)")
+      sql(s"CREATE TABLE $tableName (key STRING, value STRING) USING hive " +
+        "PARTITIONED BY (ds STRING)")
 
       createPartition("2010-01-01", "SELECT '1', 'A' from src")
       createPartition("2010-01-02", "SELECT '1', 'A' from src UNION ALL SELECT '1', 'A' from src")
@@ -429,7 +431,8 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
     }
 
     withTable(tableName) {
-      sql(s"CREATE TABLE $tableName (key STRING, value STRING) PARTITIONED BY (ds STRING, hr INT)")
+      sql(s"CREATE TABLE $tableName (key STRING, value STRING) USING hive " +
+        "PARTITIONED BY (ds STRING, hr INT)")
 
       createPartition("2010-01-01", 10, "SELECT '1', 'A' from src")
       createPartition("2010-01-01", 11, "SELECT '1', 'A' from src")
@@ -477,7 +480,8 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
     }
 
     withTable(tableName) {
-      sql(s"CREATE TABLE $tableName (key STRING, value STRING) PARTITIONED BY (ds STRING, hr INT)")
+      sql(s"CREATE TABLE $tableName (key STRING, value STRING) USING hive " +
+        "PARTITIONED BY (ds STRING, hr INT)")
 
       createPartition("2010-01-01", 10, "SELECT '1', 'A' from src")
       createPartition("2010-01-01", 11, "SELECT '1', 'A' from src")
@@ -731,7 +735,7 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
     }
   }
 
-  test("analyze column command paramaters validation") {
+  test("analyze column command parameters validation") {
     val e1 = intercept[IllegalArgumentException] {
       AnalyzeColumnCommand(TableIdentifier("test"), Option(Seq("c1")), true).run(spark)
     }
@@ -811,32 +815,6 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
     }
   }
 
-  test("alter table rename after analyze table") {
-    Seq(true, false).foreach { analyzedBySpark =>
-      val oldName = "tab1"
-      val newName = "tab2"
-      withTable(oldName, newName) {
-        createNonPartitionedTable(oldName, analyzedByHive = true, analyzedBySpark = analyzedBySpark)
-        val fetchedStats1 = checkTableStats(
-          oldName, hasSizeInBytes = true, expectedRowCounts = Some(500))
-        sql(s"ALTER TABLE $oldName RENAME TO $newName")
-        val fetchedStats2 = checkTableStats(
-          newName, hasSizeInBytes = true, expectedRowCounts = Some(500))
-        assert(fetchedStats1 == fetchedStats2)
-
-        // ALTER TABLE RENAME does not affect the contents of Hive specific statistics
-        val describeResult = hiveClient.runSqlHive(s"DESCRIBE FORMATTED $newName")
-
-        val rawDataSize = extractStatsPropValues(describeResult, "rawDataSize")
-        val numRows = extractStatsPropValues(describeResult, "numRows")
-        val totalSize = extractStatsPropValues(describeResult, "totalSize")
-        assert(rawDataSize.isDefined && rawDataSize.get > 0, "rawDataSize is lost")
-        assert(numRows.isDefined && numRows.get == 500, "numRows is lost")
-        assert(totalSize.isDefined && totalSize.get > 0, "totalSize is lost")
-      }
-    }
-  }
-
   test("alter table should not have the side effect to store statistics in Spark side") {
     val table = "alter_table_side_effect"
     withTable(table) {
@@ -872,25 +850,10 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
         assert(totalSize.isDefined && totalSize.get > 0, "totalSize is lost")
 
         val numRows = extractStatsPropValues(describeResult, "numRows")
-        if (HiveUtils.isHive23) {
-          // Since HIVE-15653(Hive 2.3.0), Hive fixs some ALTER TABLE commands drop table stats.
-          assert(numRows.isDefined && numRows.get == 500)
-          val rawDataSize = extractStatsPropValues(describeResult, "rawDataSize")
-          assert(rawDataSize.isDefined && rawDataSize.get == 5312)
-          checkTableStats(tabName, hasSizeInBytes = true, expectedRowCounts = Some(500))
-        } else {
-          // ALTER TABLE SET/UNSET TBLPROPERTIES invalidates some Hive specific statistics, but not
-          // Spark specific statistics. This is triggered by the Hive alterTable API.
-          assert(numRows.isDefined && numRows.get == -1, "numRows is lost")
-          val rawDataSize = extractStatsPropValues(describeResult, "rawDataSize")
-          assert(rawDataSize.isDefined && rawDataSize.get == -1, "rawDataSize is lost")
-
-          if (analyzedBySpark) {
-            checkTableStats(tabName, hasSizeInBytes = true, expectedRowCounts = Some(500))
-          } else {
-            checkTableStats(tabName, hasSizeInBytes = true, expectedRowCounts = None)
-          }
-        }
+        assert(numRows.isDefined && numRows.get == 500)
+        val rawDataSize = extractStatsPropValues(describeResult, "rawDataSize")
+        assert(rawDataSize.isDefined && rawDataSize.get == 5312)
+        checkTableStats(tabName, hasSizeInBytes = true, expectedRowCounts = Some(500))
       }
     }
   }
@@ -911,7 +874,7 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
    */
   private def getStatsProperties(tableName: String): Map[String, String] = {
     val hTable = hiveClient.getTable(spark.sessionState.catalog.getCurrentDatabase, tableName)
-    hTable.properties.filterKeys(_.startsWith(STATISTICS_PREFIX))
+    hTable.properties.filterKeys(_.startsWith(STATISTICS_PREFIX)).toMap
   }
 
   test("change stats after insert command for hive table") {
@@ -981,7 +944,8 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
     Seq(false, true).foreach { autoUpdate =>
       withSQLConf(SQLConf.AUTO_SIZE_UPDATE_ENABLED.key -> autoUpdate.toString) {
         withTable(table) {
-          sql(s"CREATE TABLE $table (i INT, j STRING) PARTITIONED BY (ds STRING, hr STRING)")
+          sql(s"CREATE TABLE $table (i INT, j STRING) USING hive " +
+            "PARTITIONED BY (ds STRING, hr STRING)")
           // table has two partitions initially
           for (ds <- Seq("2008-04-08"); hr <- Seq("11", "12")) {
             sql(s"INSERT OVERWRITE TABLE $table PARTITION (ds='$ds',hr='$hr') SELECT 1, 'a'")
@@ -993,12 +957,16 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
           assert(fetched1.get.colStats.size == 2)
 
           withTempPaths(numPaths = 2) { case Seq(dir1, dir2) =>
-            val file1 = new File(dir1 + "/data")
+            val partDir1 = new File(new File(dir1, "ds=2008-04-09"), "hr=11")
+            val file1 = new File(partDir1, "data")
+            file1.getParentFile.mkdirs()
             Utils.tryWithResource(new PrintWriter(file1)) { writer =>
               writer.write("1,a")
             }
 
-            val file2 = new File(dir2 + "/data")
+            val partDir2 = new File(new File(dir2, "ds=2008-04-09"), "hr=12")
+            val file2 = new File(partDir2, "data")
+            file2.getParentFile.mkdirs()
             Utils.tryWithResource(new PrintWriter(file2)) { writer =>
               writer.write("1,a")
             }
@@ -1007,8 +975,8 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
             sql(
               s"""
                  |ALTER TABLE $table ADD
-                 |PARTITION (ds='2008-04-09', hr='11') LOCATION '${dir1.toURI.toString}'
-                 |PARTITION (ds='2008-04-09', hr='12') LOCATION '${dir2.toURI.toString}'
+                 |PARTITION (ds='2008-04-09', hr='11') LOCATION '${partDir1.toURI.toString}'
+                 |PARTITION (ds='2008-04-09', hr='12') LOCATION '${partDir1.toURI.toString}'
             """.stripMargin)
             if (autoUpdate) {
               val fetched2 = checkTableStats(table, hasSizeInBytes = true, expectedRowCounts = None)
@@ -1054,6 +1022,7 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
       sql(
         s"""
            |CREATE TABLE $managedTable (key INT, value STRING)
+           |USING hive
            |PARTITIONED BY (ds STRING, hr STRING)
          """.stripMargin)
 
@@ -1128,7 +1097,7 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
     def checkColStatsProps(expected: Map[String, String]): Unit = {
       sql(s"ANALYZE TABLE $tableName COMPUTE STATISTICS FOR COLUMNS " + stats.keys.mkString(", "))
       val table = hiveClient.getTable("default", tableName)
-      val props = table.properties.filterKeys(_.startsWith("spark.sql.statistics.colStats"))
+      val props = table.properties.filterKeys(_.startsWith("spark.sql.statistics.colStats")).toMap
       assert(props == expected)
     }
 
@@ -1533,26 +1502,27 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
         Seq(tbl, ext_tbl).foreach { tblName =>
           sql(s"INSERT INTO $tblName VALUES (1, 'a', '2019-12-13')")
 
+          val expectedSize = 657
           // analyze table
           sql(s"ANALYZE TABLE $tblName COMPUTE STATISTICS NOSCAN")
           var tableStats = getTableStats(tblName)
-          assert(tableStats.sizeInBytes == 601)
+          assert(tableStats.sizeInBytes == expectedSize)
           assert(tableStats.rowCount.isEmpty)
 
           sql(s"ANALYZE TABLE $tblName COMPUTE STATISTICS")
           tableStats = getTableStats(tblName)
-          assert(tableStats.sizeInBytes == 601)
+          assert(tableStats.sizeInBytes == expectedSize)
           assert(tableStats.rowCount.get == 1)
 
           // analyze a single partition
           sql(s"ANALYZE TABLE $tblName PARTITION (ds='2019-12-13') COMPUTE STATISTICS NOSCAN")
           var partStats = getPartitionStats(tblName, Map("ds" -> "2019-12-13"))
-          assert(partStats.sizeInBytes == 601)
+          assert(partStats.sizeInBytes == expectedSize)
           assert(partStats.rowCount.isEmpty)
 
           sql(s"ANALYZE TABLE $tblName PARTITION (ds='2019-12-13') COMPUTE STATISTICS")
           partStats = getPartitionStats(tblName, Map("ds" -> "2019-12-13"))
-          assert(partStats.sizeInBytes == 601)
+          assert(partStats.sizeInBytes == expectedSize)
           assert(partStats.rowCount.get == 1)
         }
       }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/FiltersSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/FiltersSuite.scala
index 2a4efd0cce6e0..fcdc97325d010 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/FiltersSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/FiltersSuite.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.hive.client
 
+import java.sql.Date
 import java.util.Collections
 
 import org.apache.hadoop.hive.metastore.api.FieldSchema
@@ -29,6 +30,7 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.UTF8String
 
 /**
  * A set of tests for the filter conversion logic used when pushing partition pruning into the
@@ -63,6 +65,28 @@ class FiltersSuite extends SparkFunSuite with Logging with PlanTest {
     (Literal(1) === a("intcol", IntegerType)) :: (Literal("a") === a("strcol", IntegerType)) :: Nil,
     "1 = intcol and \"a\" = strcol")
 
+  filterTest("date filter",
+    (a("datecol", DateType) === Literal(Date.valueOf("2019-01-01"))) :: Nil,
+    "datecol = 2019-01-01")
+
+  filterTest("date filter with IN predicate",
+    (a("datecol", DateType) in
+      (Literal(Date.valueOf("2019-01-01")), Literal(Date.valueOf("2019-01-07")))) :: Nil,
+    "(datecol = 2019-01-01 or datecol = 2019-01-07)")
+
+  filterTest("date and string filter",
+    (Literal(Date.valueOf("2019-01-01")) === a("datecol", DateType)) ::
+      (Literal("a") === a("strcol", IntegerType)) :: Nil,
+    "2019-01-01 = datecol and \"a\" = strcol")
+
+  filterTest("date filter with null",
+    (a("datecol", DateType) ===  Literal(null)) :: Nil,
+    "")
+
+  filterTest("string filter with InSet predicate",
+    InSet(a("strcol", StringType), Set("1", "2").map(s => UTF8String.fromString(s))) :: Nil,
+    "(strcol = \"1\" or strcol = \"2\")")
+
   filterTest("skip varchar",
     (Literal("") === a("varchar", StringType)) :: Nil,
     "")
@@ -76,6 +100,55 @@ class FiltersSuite extends SparkFunSuite with Logging with PlanTest {
     (a("intcol", IntegerType) in (Literal(1), Literal(null))) :: Nil,
     "(intcol = 1)")
 
+  filterTest("NOT: int and string filters",
+    (a("intcol", IntegerType) =!= Literal(1)) :: (Literal("a") =!= a("strcol", IntegerType)) :: Nil,
+    """intcol != 1 and "a" != strcol""")
+
+  filterTest("NOT: date filter",
+    (a("datecol", DateType) =!= Literal(Date.valueOf("2019-01-01"))) :: Nil,
+    "datecol != 2019-01-01")
+
+  filterTest("not-in, string filter",
+    (Not(In(a("strcol", StringType), Seq(Literal("a"), Literal("b"))))) :: Nil,
+    """(strcol != "a" and strcol != "b")""")
+
+  filterTest("not-in, string filter with null",
+    (Not(In(a("strcol", StringType), Seq(Literal("a"), Literal("b"), Literal(null))))) :: Nil,
+    "")
+
+  filterTest("not-in, date filter",
+    (Not(In(a("datecol", DateType),
+      Seq(Literal(Date.valueOf("2021-01-01")), Literal(Date.valueOf("2021-01-02")))))) :: Nil,
+    """(datecol != 2021-01-01 and datecol != 2021-01-02)""")
+
+  filterTest("not-in, date filter with null",
+    (Not(In(a("datecol", DateType),
+      Seq(Literal(Date.valueOf("2021-01-01")), Literal(Date.valueOf("2021-01-02")),
+        Literal(null))))) :: Nil,
+    "")
+
+  filterTest("not-inset, string filter",
+    (Not(InSet(a("strcol", StringType), Set(Literal("a").eval(), Literal("b").eval())))) :: Nil,
+    """(strcol != "a" and strcol != "b")""")
+
+  filterTest("not-inset, string filter with null",
+    (Not(InSet(a("strcol", StringType),
+      Set(Literal("a").eval(), Literal("b").eval(), Literal(null).eval())))) :: Nil,
+    "")
+
+  filterTest("not-inset, date filter",
+    (Not(InSet(a("datecol", DateType),
+      Set(Literal(Date.valueOf("2020-01-01")).eval(),
+        Literal(Date.valueOf("2020-01-02")).eval())))) :: Nil,
+    """(datecol != 2020-01-01 and datecol != 2020-01-02)""")
+
+  filterTest("not-inset, date filter with null",
+    (Not(InSet(a("datecol", DateType),
+      Set(Literal(Date.valueOf("2020-01-01")).eval(),
+        Literal(Date.valueOf("2020-01-02")).eval(),
+        Literal(null).eval())))) :: Nil,
+    "")
+
   // Applying the predicate `x IN (NULL)` should return an empty set, but since this optimization
   // will be applied by Catalyst, this filter converter does not need to account for this.
   filterTest("SPARK-24879 IN predicates with only NULLs will not cause a NPE",
@@ -89,7 +162,7 @@ class FiltersSuite extends SparkFunSuite with Logging with PlanTest {
   private def filterTest(name: String, filters: Seq[Expression], result: String) = {
     test(name) {
       withSQLConf(SQLConf.ADVANCED_PARTITION_PREDICATE_PUSHDOWN.key -> "true") {
-        val converted = shim.convertFilters(testTable, filters)
+        val converted = shim.convertFilters(testTable, filters, conf.sessionLocalTimeZone)
         if (converted != result) {
           fail(s"Expected ${filters.mkString(",")} to convert to '$result' but got '$converted'")
         }
@@ -104,7 +177,7 @@ class FiltersSuite extends SparkFunSuite with Logging with PlanTest {
         val filters =
           (Literal(1) === a("intcol", IntegerType) ||
             Literal(2) === a("intcol", IntegerType)) :: Nil
-        val converted = shim.convertFilters(testTable, filters)
+        val converted = shim.convertFilters(testTable, filters, conf.sessionLocalTimeZone)
         if (enabled) {
           assert(converted == "(1 = intcol or 2 = intcol)")
         } else {
@@ -114,5 +187,69 @@ class FiltersSuite extends SparkFunSuite with Logging with PlanTest {
     }
   }
 
+  test("SPARK-33416: Avoid Hive metastore stack overflow when InSet predicate have many values") {
+    def checkConverted(inSet: InSet, result: String): Unit = {
+      assert(shim.convertFilters(testTable, inSet :: Nil, conf.sessionLocalTimeZone) == result)
+    }
+
+    withSQLConf(SQLConf.HIVE_METASTORE_PARTITION_PRUNING_INSET_THRESHOLD.key -> "15") {
+      checkConverted(
+        InSet(a("intcol", IntegerType),
+          Range(1, 20).map(s => Literal(s).eval(EmptyRow)).toSet),
+        "(intcol >= 1 and intcol <= 19)")
+
+      checkConverted(
+        InSet(a("stringcol", StringType),
+          Range(1, 20).map(s => Literal(s.toString).eval(EmptyRow)).toSet),
+        "(stringcol >= \"1\" and stringcol <= \"9\")")
+
+      checkConverted(
+        InSet(a("intcol", IntegerType).cast(LongType),
+          Range(1, 20).map(s => Literal(s.toLong).eval(EmptyRow)).toSet),
+        "(intcol >= 1 and intcol <= 19)")
+
+      checkConverted(
+        InSet(a("doublecol", DoubleType),
+          Range(1, 20).map(s => Literal(s.toDouble).eval(EmptyRow)).toSet),
+        "")
+
+      checkConverted(
+        InSet(a("datecol", DateType),
+          Range(1, 20).map(d => Literal(d, DateType).eval(EmptyRow)).toSet),
+        "(datecol >= 1970-01-02 and datecol <= 1970-01-20)")
+    }
+  }
+
+  test("SPARK-34515: Fix NPE if InSet contains null value during getPartitionsByFilter") {
+    withSQLConf(SQLConf.HIVE_METASTORE_PARTITION_PRUNING_INSET_THRESHOLD.key -> "2") {
+      val filter = InSet(a("p", IntegerType), Set(null, 1, 2))
+      val converted = shim.convertFilters(testTable, Seq(filter), conf.sessionLocalTimeZone)
+      assert(converted == "(p >= 1 and p <= 2)")
+    }
+  }
+
+  test("Don't push not inset if it's values exceeds the threshold") {
+    withSQLConf(SQLConf.HIVE_METASTORE_PARTITION_PRUNING_INSET_THRESHOLD.key -> "2") {
+      val filter = Not(InSet(a("p", IntegerType), Set(1, 2, 3)))
+      val converted = shim.convertFilters(testTable, Seq(filter), conf.sessionLocalTimeZone)
+      assert(converted.isEmpty)
+    }
+  }
+
+  test("SPARK-34538: Skip InSet null value during push filter to Hive metastore") {
+    withSQLConf(SQLConf.HIVE_METASTORE_PARTITION_PRUNING_INSET_THRESHOLD.key -> "3") {
+      val intFilter = InSet(a("p", IntegerType), Set(null, 1, 2))
+      val intConverted = shim.convertFilters(testTable, Seq(intFilter), conf.sessionLocalTimeZone)
+      assert(intConverted == "(p = 1 or p = 2)")
+    }
+
+    withSQLConf(SQLConf.HIVE_METASTORE_PARTITION_PRUNING_INSET_THRESHOLD.key -> "3") {
+      val dateFilter = InSet(a("p", DateType), Set(null,
+        Literal(Date.valueOf("2020-01-01")).eval(), Literal(Date.valueOf("2021-01-01")).eval()))
+      val dateConverted = shim.convertFilters(testTable, Seq(dateFilter), conf.sessionLocalTimeZone)
+      assert(dateConverted == "(p = 2020-01-01 or p = 2021-01-01)")
+    }
+  }
+
   private def a(name: String, dataType: DataType) = AttributeReference(name, dataType)()
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HadoopVersionInfoSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HadoopVersionInfoSuite.scala
new file mode 100644
index 0000000000000..6ada46412bf9f
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HadoopVersionInfoSuite.scala
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.client
+
+import java.io.File
+import java.net.URLClassLoader
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.util.VersionInfo
+
+import org.apache.spark.{SparkConf, SparkFunSuite}
+import org.apache.spark.sql.hive.{HiveExternalCatalog, HiveUtils}
+import org.apache.spark.util.Utils
+
+/**
+ * This test suite requires a clean JVM because it's testing the initialization of static codes in
+ * `org.apache.hadoop.util.VersionInfo`.
+ */
+class HadoopVersionInfoSuite extends SparkFunSuite {
+  override protected val enableAutoThreadAudit = false
+
+  test("SPARK-32256: Hadoop VersionInfo should be preloaded") {
+    val ivyPath =
+      Utils.createTempDir(namePrefix = s"${classOf[HadoopVersionInfoSuite].getSimpleName}-ivy")
+    try {
+      val hadoopConf = new Configuration()
+      hadoopConf.set("test", "success")
+      hadoopConf.set("datanucleus.schema.autoCreateAll", "true")
+      hadoopConf.set("hive.metastore.schema.verification", "false")
+
+      // Download jars for Hive 2.0
+      val client = IsolatedClientLoader.forVersion(
+        hiveMetastoreVersion = "2.0",
+        hadoopVersion = "2.7.4",
+        sparkConf = new SparkConf(),
+        hadoopConf = hadoopConf,
+        config = HiveClientBuilder.buildConf(Map.empty),
+        ivyPath = Some(ivyPath.getCanonicalPath))
+      val jars = client.classLoader.getParent.asInstanceOf[URLClassLoader].getURLs
+        .map(u => new File(u.toURI))
+        // Drop all Hadoop jars to use the existing Hadoop jars on the classpath
+        .filter(!_.getName.startsWith("org.apache.hadoop_hadoop-"))
+
+      val sparkConf = new SparkConf()
+      sparkConf.set(HiveUtils.HIVE_METASTORE_VERSION, "2.0")
+      sparkConf.set(
+        HiveUtils.HIVE_METASTORE_JARS,
+        jars.map(_.getCanonicalPath).mkString(File.pathSeparator))
+      HiveClientBuilder.buildConf(Map.empty).foreach { case (k, v) =>
+        hadoopConf.set(k, v)
+      }
+      new HiveExternalCatalog(sparkConf, hadoopConf).client.getState
+    } finally {
+      Utils.deleteRecursively(ivyPath)
+    }
+  }
+
+  test("SPARK-32212: test supportHadoopShadedClient()") {
+    Seq("3.2.2", "3.2.3", "3.2.2.1", "3.2.2-XYZ", "3.2.2.4-SNAPSHOT").foreach { version =>
+      assert(IsolatedClientLoader.supportsHadoopShadedClient(version), s"version $version")
+    }
+
+    // negative cases
+    Seq("3.1.3", "3.2", "3.2.1", "4").foreach { version =>
+      assert(!IsolatedClientLoader.supportsHadoopShadedClient(version), s"version $version")
+    }
+  }
+
+  test("SPARK-32212: built-in Hadoop version should support shaded client if it is not hadoop 2") {
+    val hadoopVersion = VersionInfo.getVersion
+    if (!hadoopVersion.startsWith("2")) {
+      assert(IsolatedClientLoader.supportsHadoopShadedClient(hadoopVersion))
+    }
+  }
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientBuilder.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientBuilder.scala
index ab73f668c6ca6..f40b4f00d9fd0 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientBuilder.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientBuilder.scala
@@ -33,7 +33,7 @@ private[client] object HiveClientBuilder {
       Some(new File(sys.props("java.io.tmpdir"), "hive-ivy-cache").getAbsolutePath))
   }
 
-  private def buildConf(extraConf: Map[String, String]) = {
+  private[client] def buildConf(extraConf: Map[String, String]): Map[String, String] = {
     lazy val warehousePath = Utils.createTempDir()
     lazy val metastorePath = Utils.createTempDir()
     metastorePath.delete()
@@ -46,15 +46,13 @@ private[client] object HiveClientBuilder {
   def buildClient(
       version: String,
       hadoopConf: Configuration,
-      extraConf: Map[String, String] = Map.empty,
-      sharesHadoopClasses: Boolean = true): HiveClient = {
+      extraConf: Map[String, String] = Map.empty): HiveClient = {
     IsolatedClientLoader.forVersion(
       hiveMetastoreVersion = version,
       hadoopVersion = VersionInfo.getVersion,
       sparkConf = new SparkConf(),
       hadoopConf = hadoopConf,
       config = buildConf(extraConf),
-      ivyPath = ivyPath,
-      sharesHadoopClasses = sharesHadoopClasses).createClient()
+      ivyPath = ivyPath).createClient()
   }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientUserNameSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientUserNameSuite.scala
index 77956f4fe69da..b94d517e89e30 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientUserNameSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientUserNameSuite.scala
@@ -21,7 +21,6 @@ import java.security.PrivilegedExceptionAction
 
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.security.UserGroupInformation
-import org.scalatest.{BeforeAndAfterAll, PrivateMethodTester}
 
 import org.apache.spark.util.Utils
 
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HivePartitionFilteringSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HivePartitionFilteringSuite.scala
index 2d615f6fdc261..16e1a415977a1 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HivePartitionFilteringSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HivePartitionFilteringSuite.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql.hive.client
 
+import java.sql.Date
+
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.hive.conf.HiveConf
 import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -26,9 +28,11 @@ import org.scalatest.BeforeAndAfterAll
 
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.catalog._
+import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils.DEFAULT_PARTITION_NAME
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.types.{BooleanType, IntegerType, LongType, StructType}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types.{BooleanType, DateType, IntegerType, LongType, StringType, StructType}
 import org.apache.spark.util.Utils
 
 class HivePartitionFilteringSuite(version: String)
@@ -36,47 +40,55 @@ class HivePartitionFilteringSuite(version: String)
 
   private val tryDirectSqlKey = HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL.varname
 
-  private val testPartitionCount = 3 * 5 * 4
+  // Support default partition in metastoredirectsql since HIVE-11898(Hive 2.0.0).
+  private val defaultPartition = if (version >= "2.0") Some(DEFAULT_PARTITION_NAME) else None
+
+  private val dsValue = 20170101 to 20170103
+  private val hValue = 0 to 4
+  private val chunkValue = Seq("aa", "ab", "ba", "bb")
+  private val dateValue = Seq("2019-01-01", "2019-01-02", "2019-01-03") ++ defaultPartition
+  private val dateStrValue = Seq("2020-01-01", "2020-01-02", "2020-01-03")
+  private val testPartitionCount =
+    dsValue.size * hValue.size * chunkValue.size * dateValue.size * dateStrValue.size
+
+  private val storageFormat = CatalogStorageFormat(
+    locationUri = None,
+    inputFormat = Some(classOf[TextInputFormat].getName),
+    outputFormat = Some(classOf[HiveIgnoreKeyTextOutputFormat[_, _]].getName),
+    serde = Some(classOf[LazySimpleSerDe].getName()),
+    compressed = false,
+    properties = Map.empty
+  )
 
   private def init(tryDirectSql: Boolean): HiveClient = {
-    val storageFormat = CatalogStorageFormat(
-      locationUri = None,
-      inputFormat = None,
-      outputFormat = None,
-      serde = None,
-      compressed = false,
-      properties = Map.empty)
-
     val hadoopConf = new Configuration()
     hadoopConf.setBoolean(tryDirectSqlKey, tryDirectSql)
     hadoopConf.set("hive.metastore.warehouse.dir", Utils.createTempDir().toURI().toString())
     val client = buildClient(hadoopConf)
     val tableSchema =
       new StructType().add("value", "int").add("ds", "int").add("h", "int").add("chunk", "string")
+        .add("d", "date").add("datestr", "string")
     val table = CatalogTable(
       identifier = TableIdentifier("test", Some("default")),
       tableType = CatalogTableType.MANAGED,
       schema = tableSchema,
-      partitionColumnNames = Seq("ds", "h", "chunk"),
-      storage = CatalogStorageFormat(
-        locationUri = None,
-        inputFormat = Some(classOf[TextInputFormat].getName),
-        outputFormat = Some(classOf[HiveIgnoreKeyTextOutputFormat[_, _]].getName),
-        serde = Some(classOf[LazySimpleSerDe].getName()),
-        compressed = false,
-        properties = Map.empty
-      ))
+      partitionColumnNames = Seq("ds", "h", "chunk", "d", "datestr"),
+      storage = storageFormat)
     client.createTable(table, ignoreIfExists = false)
 
     val partitions =
       for {
-        ds <- 20170101 to 20170103
-        h <- 0 to 4
-        chunk <- Seq("aa", "ab", "ba", "bb")
+        ds <- dsValue
+        h <- hValue
+        chunk <- chunkValue
+        date <- dateValue
+        dateStr <- dateStrValue
       } yield CatalogTablePartition(Map(
         "ds" -> ds.toString,
         "h" -> h.toString,
-        "chunk" -> chunk
+        "chunk" -> chunk,
+        "d" -> date,
+        "datestr" -> dateStr
       ), storageFormat)
     assert(partitions.size == testPartitionCount)
 
@@ -102,7 +114,7 @@ class HivePartitionFilteringSuite(version: String)
   test(s"getPartitionsByFilter returns all partitions when $tryDirectSqlKey=false") {
     val client = init(false)
     val filteredPartitions = client.getPartitionsByFilter(client.getTable("default", "test"),
-      Seq(attr("ds") === 20170101))
+      Seq(attr("ds") === 20170101), SQLConf.get.sessionLocalTimeZone)
 
     assert(filteredPartitions.size == testPartitionCount)
   }
@@ -111,17 +123,21 @@ class HivePartitionFilteringSuite(version: String)
     // Should return all partitions where <=> is not supported
     testMetastorePartitionFiltering(
       attr("ds") <=> 20170101,
-      20170101 to 20170103,
-      0 to 4,
-      "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+      dsValue,
+      hValue,
+      chunkValue,
+      dateValue,
+      dateStrValue)
   }
 
   test("getPartitionsByFilter: ds=20170101") {
     testMetastorePartitionFiltering(
       attr("ds") === 20170101,
       20170101 to 20170101,
-      0 to 4,
-      "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+      hValue,
+      chunkValue,
+      dateValue,
+      dateStrValue)
   }
 
   test("getPartitionsByFilter: ds=(20170101 + 1) and h=0") {
@@ -129,41 +145,51 @@ class HivePartitionFilteringSuite(version: String)
     // comparisons to non-literal values
     testMetastorePartitionFiltering(
       attr("ds") === (Literal(20170101) + 1) && attr("h") === 0,
-      20170101 to 20170103,
+      dsValue,
       0 to 0,
-      "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+      chunkValue,
+      dateValue,
+      dateStrValue)
   }
 
   test("getPartitionsByFilter: chunk='aa'") {
     testMetastorePartitionFiltering(
       attr("chunk") === "aa",
-      20170101 to 20170103,
-      0 to 4,
-      "aa" :: Nil)
+      dsValue,
+      hValue,
+      "aa" :: Nil,
+      dateValue,
+      dateStrValue)
   }
 
   test("getPartitionsByFilter: cast(chunk as int)=1 (not a valid partition predicate)") {
     testMetastorePartitionFiltering(
       attr("chunk").cast(IntegerType) === 1,
-      20170101 to 20170103,
-      0 to 4,
-      "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+      dsValue,
+      hValue,
+      chunkValue,
+      dateValue,
+      dateStrValue)
   }
 
   test("getPartitionsByFilter: cast(chunk as boolean)=true (not a valid partition predicate)") {
     testMetastorePartitionFiltering(
       attr("chunk").cast(BooleanType) === true,
-      20170101 to 20170103,
-      0 to 4,
-      "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+      dsValue,
+      hValue,
+      chunkValue,
+      dateValue,
+      dateStrValue)
   }
 
   test("getPartitionsByFilter: 20170101=ds") {
     testMetastorePartitionFiltering(
       Literal(20170101) === attr("ds"),
       20170101 to 20170101,
-      0 to 4,
-      "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+      hValue,
+      chunkValue,
+      dateValue,
+      dateStrValue)
   }
 
   test("getPartitionsByFilter: ds=20170101 and h=2") {
@@ -171,7 +197,9 @@ class HivePartitionFilteringSuite(version: String)
       attr("ds") === 20170101 && attr("h") === 2,
       20170101 to 20170101,
       2 to 2,
-      "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+      chunkValue,
+      dateValue,
+      dateStrValue)
   }
 
   test("getPartitionsByFilter: cast(ds as long)=20170101L and h=2") {
@@ -179,39 +207,49 @@ class HivePartitionFilteringSuite(version: String)
       attr("ds").cast(LongType) === 20170101L && attr("h") === 2,
       20170101 to 20170101,
       2 to 2,
-      "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+      chunkValue,
+      dateValue,
+      dateStrValue)
   }
 
   test("getPartitionsByFilter: ds=20170101 or ds=20170102") {
     testMetastorePartitionFiltering(
       attr("ds") === 20170101 || attr("ds") === 20170102,
       20170101 to 20170102,
-      0 to 4,
-      "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+      hValue,
+      chunkValue,
+      dateValue,
+      dateStrValue)
   }
 
   test("getPartitionsByFilter: ds in (20170102, 20170103) (using IN expression)") {
     testMetastorePartitionFiltering(
       attr("ds").in(20170102, 20170103),
       20170102 to 20170103,
-      0 to 4,
-      "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+      hValue,
+      chunkValue,
+      dateValue,
+      dateStrValue)
   }
 
   test("getPartitionsByFilter: cast(ds as long) in (20170102L, 20170103L) (using IN expression)") {
     testMetastorePartitionFiltering(
       attr("ds").cast(LongType).in(20170102L, 20170103L),
       20170102 to 20170103,
-      0 to 4,
-      "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+      hValue,
+      chunkValue,
+      dateValue,
+      dateStrValue)
   }
 
   test("getPartitionsByFilter: ds in (20170102, 20170103) (using INSET expression)") {
     testMetastorePartitionFiltering(
       attr("ds").in(20170102, 20170103),
       20170102 to 20170103,
-      0 to 4,
-      "aa" :: "ab" :: "ba" :: "bb" :: Nil, {
+      hValue,
+      chunkValue,
+      dateValue,
+      dateStrValue, {
         case expr @ In(v, list) if expr.inSetConvertible =>
           InSet(v, list.map(_.eval(EmptyRow)).toSet)
       })
@@ -222,8 +260,10 @@ class HivePartitionFilteringSuite(version: String)
     testMetastorePartitionFiltering(
       attr("ds").cast(LongType).in(20170102L, 20170103L),
       20170102 to 20170103,
-      0 to 4,
-      "aa" :: "ab" :: "ba" :: "bb" :: Nil, {
+      hValue,
+      chunkValue,
+      dateValue,
+      dateStrValue, {
         case expr @ In(v, list) if expr.inSetConvertible =>
           InSet(v, list.map(_.eval(EmptyRow)).toSet)
       })
@@ -232,58 +272,307 @@ class HivePartitionFilteringSuite(version: String)
   test("getPartitionsByFilter: chunk in ('ab', 'ba') (using IN expression)") {
     testMetastorePartitionFiltering(
       attr("chunk").in("ab", "ba"),
-      20170101 to 20170103,
-      0 to 4,
-      "ab" :: "ba" :: Nil)
+      dsValue,
+      hValue,
+      "ab" :: "ba" :: Nil,
+      dateValue,
+      dateStrValue)
   }
 
   test("getPartitionsByFilter: chunk in ('ab', 'ba') (using INSET expression)") {
     testMetastorePartitionFiltering(
       attr("chunk").in("ab", "ba"),
-      20170101 to 20170103,
-      0 to 4,
-      "ab" :: "ba" :: Nil, {
+      dsValue,
+      hValue,
+      "ab" :: "ba" :: Nil,
+      dateValue,
+      dateStrValue, {
         case expr @ In(v, list) if expr.inSetConvertible =>
           InSet(v, list.map(_.eval(EmptyRow)).toSet)
       })
   }
 
   test("getPartitionsByFilter: (ds=20170101 and h>=2) or (ds=20170102 and h<2)") {
-    val day1 = (20170101 to 20170101, 2 to 4, Seq("aa", "ab", "ba", "bb"))
-    val day2 = (20170102 to 20170102, 0 to 1, Seq("aa", "ab", "ba", "bb"))
+    val day1 = (20170101 to 20170101, 2 to 4, chunkValue, dateValue, dateStrValue)
+    val day2 = (20170102 to 20170102, 0 to 1, chunkValue, dateValue, dateStrValue)
     testMetastorePartitionFiltering((attr("ds") === 20170101 && attr("h") >= 2) ||
         (attr("ds") === 20170102 && attr("h") < 2), day1 :: day2 :: Nil)
   }
 
   test("getPartitionsByFilter: (ds=20170101 and h>=2) or (ds=20170102 and h<(1+1))") {
-    val day1 = (20170101 to 20170101, 2 to 4, Seq("aa", "ab", "ba", "bb"))
+    val day1 = (20170101 to 20170101, 2 to 4, chunkValue, dateValue, dateStrValue)
     // Day 2 should include all hours because we can't build a filter for h<(7+1)
-    val day2 = (20170102 to 20170102, 0 to 4, Seq("aa", "ab", "ba", "bb"))
+    val day2 = (20170102 to 20170102, 0 to 4, chunkValue, dateValue, dateStrValue)
     testMetastorePartitionFiltering((attr("ds") === 20170101 && attr("h") >= 2) ||
         (attr("ds") === 20170102 && attr("h") < (Literal(1) + 1)), day1 :: day2 :: Nil)
   }
 
   test("getPartitionsByFilter: " +
       "chunk in ('ab', 'ba') and ((ds=20170101 and h>=2) or (ds=20170102 and h<2))") {
-    val day1 = (20170101 to 20170101, 2 to 4, Seq("ab", "ba"))
-    val day2 = (20170102 to 20170102, 0 to 1, Seq("ab", "ba"))
+    val day1 = (20170101 to 20170101, 2 to 4, Seq("ab", "ba"), dateValue, dateStrValue)
+    val day2 = (20170102 to 20170102, 0 to 1, Seq("ab", "ba"), dateValue, dateStrValue)
     testMetastorePartitionFiltering(attr("chunk").in("ab", "ba") &&
         ((attr("ds") === 20170101 && attr("h") >= 2) || (attr("ds") === 20170102 && attr("h") < 2)),
       day1 :: day2 :: Nil)
   }
 
-  test("create client with sharesHadoopClasses = false") {
-    buildClient(new Configuration(), sharesHadoopClasses = false)
+  test("getPartitionsByFilter: chunk contains bb") {
+    testMetastorePartitionFiltering(
+      attr("chunk").contains("bb"),
+      dsValue,
+      hValue,
+      Seq("bb"),
+      dateValue,
+      dateStrValue)
+  }
+
+  test("getPartitionsByFilter: chunk startsWith b") {
+    testMetastorePartitionFiltering(
+      attr("chunk").startsWith("b"),
+      dsValue,
+      hValue,
+      Seq("ba", "bb"),
+      dateValue,
+      dateStrValue)
+  }
+
+  test("getPartitionsByFilter: chunk endsWith b") {
+    testMetastorePartitionFiltering(
+      attr("chunk").endsWith("b"),
+      dsValue,
+      hValue,
+      Seq("ab", "bb"),
+      dateValue,
+      dateStrValue)
+  }
+
+  test("getPartitionsByFilter: chunk in ('ab', 'ba') and ((cast(ds as string)>'20170102')") {
+    testMetastorePartitionFiltering(
+      attr("chunk").in("ab", "ba") && (attr("ds").cast(StringType) > "20170102"),
+      dsValue,
+      hValue,
+      Seq("ab", "ba"),
+      dateValue,
+      dateStrValue)
+  }
+
+  test("getPartitionsByFilter: ds<>20170101") {
+    testMetastorePartitionFiltering(
+      attr("ds") =!= 20170101,
+      20170102 to 20170103,
+      hValue,
+      chunkValue,
+      dateValue,
+      dateStrValue)
+  }
+
+  test("getPartitionsByFilter: h<>0 and chunk<>ab and d<>2019-01-01") {
+    testMetastorePartitionFiltering(
+      attr("h") =!= 0 && attr("chunk") =!= "ab" && attr("d") =!= Date.valueOf("2019-01-01"),
+      dsValue,
+      1 to 4,
+      Seq("aa", "ba", "bb"),
+      Seq("2019-01-02", "2019-01-03"),
+      dateStrValue)
+  }
+
+  test("getPartitionsByFilter: d=2019-01-01") {
+    testMetastorePartitionFiltering(
+      attr("d") === Date.valueOf("2019-01-01"),
+      dsValue,
+      hValue,
+      chunkValue,
+      Seq("2019-01-01"),
+      dateStrValue)
+  }
+
+  test("getPartitionsByFilter: d>2019-01-02") {
+    testMetastorePartitionFiltering(
+      attr("d") > Date.valueOf("2019-01-02"),
+      dsValue,
+      hValue,
+      chunkValue,
+      Seq("2019-01-03"),
+      dateStrValue)
+  }
+
+  test("getPartitionsByFilter: In(d, 2019-01-01, 2019-01-02)") {
+    testMetastorePartitionFiltering(
+      In(attr("d"),
+        Seq("2019-01-01", "2019-01-02").map(d => Literal(Date.valueOf(d)))),
+      dsValue,
+      hValue,
+      chunkValue,
+      Seq("2019-01-01", "2019-01-02"),
+      dateStrValue)
+  }
+
+  test("getPartitionsByFilter: InSet(d, 2019-01-01, 2019-01-02)") {
+    testMetastorePartitionFiltering(
+      InSet(attr("d"),
+        Set("2019-01-01", "2019-01-02").map(d => Literal(Date.valueOf(d)).eval(EmptyRow))),
+      dsValue,
+      hValue,
+      chunkValue,
+      Seq("2019-01-01", "2019-01-02"),
+      dateStrValue)
+  }
+
+  test("getPartitionsByFilter: not in/inset string type") {
+    def check(condition: Expression, result: Seq[String]): Unit = {
+      testMetastorePartitionFiltering(
+        condition,
+        dsValue,
+        hValue,
+        result,
+        dateValue,
+        dateStrValue
+      )
+    }
+
+    check(
+      Not(In(attr("chunk"), Seq(Literal("aa"), Literal("ab")))),
+      Seq("ba", "bb")
+    )
+    check(
+      Not(In(attr("chunk"), Seq(Literal("aa"), Literal("ab"), Literal(null)))),
+      chunkValue
+    )
+
+    check(
+      Not(InSet(attr("chunk"), Set(Literal("aa").eval(), Literal("ab").eval()))),
+      Seq("ba", "bb")
+    )
+    check(
+      Not(InSet(attr("chunk"), Set("aa", "ab", null))),
+      chunkValue
+    )
+  }
+
+  test("getPartitionsByFilter: not in/inset date type") {
+    def check(condition: Expression, result: Seq[String]): Unit = {
+      testMetastorePartitionFiltering(
+        condition,
+        dsValue,
+        hValue,
+        chunkValue,
+        result,
+        dateStrValue
+      )
+    }
+
+    check(
+      Not(In(attr("d"),
+        Seq(Literal(Date.valueOf("2019-01-01")),
+          Literal(Date.valueOf("2019-01-02"))))),
+      Seq("2019-01-03")
+    )
+    check(
+      Not(In(attr("d"),
+        Seq(Literal(Date.valueOf("2019-01-01")),
+          Literal(Date.valueOf("2019-01-02")), Literal(null)))),
+      dateValue
+    )
+
+    check(
+      Not(InSet(attr("d"),
+        Set(Literal(Date.valueOf("2019-01-01")).eval(),
+          Literal(Date.valueOf("2019-01-02")).eval()))),
+      Seq("2019-01-03")
+    )
+    check(
+      Not(InSet(attr("d"),
+        Set(Literal(Date.valueOf("2019-01-01")).eval(),
+          Literal(Date.valueOf("2019-01-02")).eval(), null))),
+      dateValue
+    )
+  }
+
+  test("getPartitionsByFilter: cast(datestr as date)= 2020-01-01") {
+    testMetastorePartitionFiltering(
+      attr("datestr").cast(DateType) === Date.valueOf("2020-01-01"),
+      dsValue,
+      hValue,
+      chunkValue,
+      dateValue,
+      dateStrValue)
+  }
+
+  test("getPartitionsByFilter: IS NULL / IS NOT NULL") {
+    // returns all partitions
+    Seq(attr("d").isNull, attr("d").isNotNull).foreach { filterExpr =>
+      testMetastorePartitionFiltering(
+        filterExpr,
+        dsValue,
+        hValue,
+        chunkValue,
+        dateValue,
+        dateStrValue)
+    }
+  }
+
+  test("getPartitionsByFilter: IS NULL / IS NOT NULL with other filter") {
+    Seq(attr("d").isNull, attr("d").isNotNull).foreach { filterExpr =>
+      testMetastorePartitionFiltering(
+        filterExpr && attr("d") === Date.valueOf("2019-01-01"),
+        dsValue,
+        hValue,
+        chunkValue,
+        Seq("2019-01-01"),
+        dateStrValue)
+    }
+  }
+
+  test("getPartitionsByFilter: d =!= 2019-01-01") {
+    testMetastorePartitionFiltering(
+      attr("d") =!= Date.valueOf("2019-01-01"),
+      dsValue,
+      hValue,
+      chunkValue,
+      Seq("2019-01-02", "2019-01-03"),
+      dateStrValue)
+  }
+
+  test("getPartitionsByFilter: d =!= 2019-01-01 || IS NULL") {
+    testMetastorePartitionFiltering(
+      attr("d") =!= Date.valueOf("2019-01-01") || attr("d").isNull,
+      dsValue,
+      hValue,
+      chunkValue,
+      dateValue,
+      dateStrValue)
+  }
+
+  test("getPartitionsByFilter: d <=> 2019-01-01") {
+    testMetastorePartitionFiltering(
+      attr("d") <=> Date.valueOf("2019-01-01"),
+      dsValue,
+      hValue,
+      chunkValue,
+      dateValue,
+      dateStrValue)
+  }
+
+  test("getPartitionsByFilter: d <=> null") {
+    testMetastorePartitionFiltering(
+      attr("d") <=> Literal(null, DateType),
+      dsValue,
+      hValue,
+      chunkValue,
+      dateValue,
+      dateStrValue)
   }
 
   private def testMetastorePartitionFiltering(
       filterExpr: Expression,
       expectedDs: Seq[Int],
       expectedH: Seq[Int],
-      expectedChunks: Seq[String]): Unit = {
+      expectedChunks: Seq[String],
+      expectedD: Seq[String],
+      expectedDatestr: Seq[String]): Unit = {
     testMetastorePartitionFiltering(
       filterExpr,
-      (expectedDs, expectedH, expectedChunks) :: Nil,
+      (expectedDs, expectedH, expectedChunks, expectedD, expectedDatestr) :: Nil,
       identity)
   }
 
@@ -292,43 +581,51 @@ class HivePartitionFilteringSuite(version: String)
       expectedDs: Seq[Int],
       expectedH: Seq[Int],
       expectedChunks: Seq[String],
+      expectedD: Seq[String],
+      expectedDatestr: Seq[String],
       transform: Expression => Expression): Unit = {
     testMetastorePartitionFiltering(
       filterExpr,
-      (expectedDs, expectedH, expectedChunks) :: Nil,
+      (expectedDs, expectedH, expectedChunks, expectedD, expectedDatestr) :: Nil,
       transform)
   }
 
   private def testMetastorePartitionFiltering(
       filterExpr: Expression,
-      expectedPartitionCubes: Seq[(Seq[Int], Seq[Int], Seq[String])]): Unit = {
+      expectedPartitionCubes:
+        Seq[(Seq[Int], Seq[Int], Seq[String], Seq[String], Seq[String])]): Unit = {
     testMetastorePartitionFiltering(filterExpr, expectedPartitionCubes, identity)
   }
 
   private def testMetastorePartitionFiltering(
       filterExpr: Expression,
-      expectedPartitionCubes: Seq[(Seq[Int], Seq[Int], Seq[String])],
+      expectedPartitionCubes: Seq[(Seq[Int], Seq[Int], Seq[String], Seq[String], Seq[String])],
       transform: Expression => Expression): Unit = {
     val filteredPartitions = client.getPartitionsByFilter(client.getTable("default", "test"),
       Seq(
         transform(filterExpr)
-      ))
+      ), SQLConf.get.sessionLocalTimeZone)
 
     val expectedPartitionCount = expectedPartitionCubes.map {
-      case (expectedDs, expectedH, expectedChunks) =>
-        expectedDs.size * expectedH.size * expectedChunks.size
+      case (expectedDs, expectedH, expectedChunks, expectedD, expectedDatestr) =>
+        expectedDs.size * expectedH.size * expectedChunks.size *
+          expectedD.size * expectedDatestr.size
     }.sum
 
     val expectedPartitions = expectedPartitionCubes.map {
-      case (expectedDs, expectedH, expectedChunks) =>
+      case (expectedDs, expectedH, expectedChunks, expectedD, expectedDatestr) =>
         for {
           ds <- expectedDs
           h <- expectedH
           chunk <- expectedChunks
+          d <- expectedD
+          datestr <- expectedDatestr
         } yield Set(
           "ds" -> ds.toString,
           "h" -> h.toString,
-          "chunk" -> chunk
+          "chunk" -> chunk,
+          "d" -> d,
+          "datestr" -> datestr
         )
     }.reduce(_ ++ _)
 
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveVersionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveVersionSuite.scala
index dd58c302e0197..02e9b7fb151fd 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveVersionSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveVersionSuite.scala
@@ -28,9 +28,7 @@ private[client] abstract class HiveVersionSuite(version: String) extends SparkFu
   override protected val enableAutoThreadAudit = false
   protected var client: HiveClient = null
 
-  protected def buildClient(
-      hadoopConf: Configuration,
-      sharesHadoopClasses: Boolean = true): HiveClient = {
+  protected def buildClient(hadoopConf: Configuration): HiveClient = {
     // Hive changed the default of datanucleus.schema.autoCreateAll from true to false and
     // hive.metastore.schema.verification from false to true since 2.0
     // For details, see the JIRA HIVE-6113 and HIVE-12463
@@ -46,8 +44,7 @@ private[client] abstract class HiveVersionSuite(version: String) extends SparkFu
     HiveClientBuilder.buildClient(
       version,
       hadoopConf,
-      HiveUtils.formatTimeVarsForHiveClient(hadoopConf),
-      sharesHadoopClasses = sharesHadoopClasses)
+      HiveUtils.formatTimeVarsForHiveClient(hadoopConf))
   }
 
   override def suiteName: String = s"${super.suiteName}($version)"
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
index d1dd13623650d..b5500eaf47158 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
@@ -33,7 +33,7 @@ import org.apache.spark.SparkFunSuite
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.{AnalysisException, Row}
 import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
-import org.apache.spark.sql.catalyst.analysis.{NoSuchDatabaseException, NoSuchPermanentFunctionException}
+import org.apache.spark.sql.catalyst.analysis.{NoSuchDatabaseException, NoSuchPermanentFunctionException, PartitionsAlreadyExistException}
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.expressions.{AttributeReference, EqualTo, Literal}
 import org.apache.spark.sql.catalyst.util.quietly
@@ -41,7 +41,7 @@ import org.apache.spark.sql.hive.{HiveExternalCatalog, HiveUtils}
 import org.apache.spark.sql.hive.test.TestHiveVersion
 import org.apache.spark.sql.types.IntegerType
 import org.apache.spark.sql.types.StructType
-import org.apache.spark.tags.ExtendedHiveTest
+import org.apache.spark.tags.{ExtendedHiveTest, SlowHiveTest}
 import org.apache.spark.util.{MutableURLClassLoader, Utils}
 
 /**
@@ -51,6 +51,7 @@ import org.apache.spark.util.{MutableURLClassLoader, Utils}
  * is not fully tested.
  */
 // TODO: Refactor this to `HiveClientSuite` and make it a subclass of `HiveVersionSuite`
+@SlowHiveTest
 @ExtendedHiveTest
 class VersionsSuite extends SparkFunSuite with Logging {
 
@@ -487,7 +488,8 @@ class VersionsSuite extends SparkFunSuite with Logging {
     test(s"$version: getPartitionsByFilter") {
       // Only one partition [1, 1] for key2 == 1
       val result = client.getPartitionsByFilter(client.getTable("default", "src_part"),
-        Seq(EqualTo(AttributeReference("key2", IntegerType)(), Literal(1))))
+        Seq(EqualTo(AttributeReference("key2", IntegerType)(), Literal(1))),
+        versionSpark.conf.sessionLocalTimeZone)
 
       // Hive 0.12 doesn't support getPartitionsByFilter, it ignores the filter condition.
       if (version != "0.12") {
@@ -592,6 +594,27 @@ class VersionsSuite extends SparkFunSuite with Logging {
       assert(client.getPartitionOption("default", "src_part", spec).isEmpty)
     }
 
+    test(s"$version: createPartitions if already exists") {
+      val partitions = Seq(CatalogTablePartition(
+        Map("key1" -> "101", "key2" -> "102"),
+        storageFormat))
+      try {
+        client.createPartitions("default", "src_part", partitions, ignoreIfExists = false)
+        val errMsg = intercept[PartitionsAlreadyExistException] {
+          client.createPartitions("default", "src_part", partitions, ignoreIfExists = false)
+        }.getMessage
+        assert(errMsg.contains("partitions already exists"))
+      } finally {
+        client.dropPartitions(
+          "default",
+          "src_part",
+          partitions.map(_.spec),
+          ignoreIfNotExists = true,
+          purge = false,
+          retainData = false)
+      }
+    }
+
     ///////////////////////////////////////////////////////////////////////////
     // Function related API
     ///////////////////////////////////////////////////////////////////////////
@@ -796,6 +819,7 @@ class VersionsSuite extends SparkFunSuite with Logging {
         versionSpark.sql(
           """
             |CREATE TABLE tbl(c1 string)
+            |USING hive
             |PARTITIONED BY (ds STRING)
           """.stripMargin)
         versionSpark.sql("INSERT OVERWRITE TABLE tbl partition (ds='2') SELECT '1'")
@@ -982,7 +1006,7 @@ class VersionsSuite extends SparkFunSuite with Logging {
            """.stripMargin
           )
 
-          val errorMsg = "Cannot safely cast 'f0': DecimalType(2,1) to BinaryType"
+          val errorMsg = "Cannot safely cast 'f0': decimal(2,1) to binary"
 
           if (isPartitioned) {
             val insertStmt = s"INSERT OVERWRITE TABLE $tableName partition (ds='a') SELECT 1.3"
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
index fac981267f4d7..70dcfb05c2ba9 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
@@ -31,6 +31,7 @@ import org.apache.spark.sql.hive.test.TestHiveSingleton
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SQLTestUtils
 import org.apache.spark.sql.types._
+import org.apache.spark.tags.SlowHiveTest
 import org.apache.spark.unsafe.UnsafeAlignedOffset
 
 
@@ -780,7 +781,7 @@ abstract class AggregationQuerySuite extends QueryTest with SQLTestUtils with Te
     assert(math.abs(corr6 + 1.0) < 1e-12)
 
     // Test for udaf_corr in HiveCompatibilitySuite
-    // udaf_corr has been blacklisted due to numerical errors
+    // udaf_corr has been excluded due to numerical errors
     // We test it here:
     // SELECT corr(b, c) FROM covar_tab WHERE a < 1; => NULL
     // SELECT corr(b, c) FROM covar_tab WHERE a < 3; => NULL
@@ -824,7 +825,7 @@ abstract class AggregationQuerySuite extends QueryTest with SQLTestUtils with Te
           """
             |SELECT corr(b, c) FROM covar_tab WHERE a = 3
           """.stripMargin),
-        Row(Double.NaN) :: Nil)
+        Row(null) :: Nil)
 
       checkAnswer(
         spark.sql(
@@ -833,10 +834,10 @@ abstract class AggregationQuerySuite extends QueryTest with SQLTestUtils with Te
           """.stripMargin),
         Row(1, null) ::
         Row(2, null) ::
-        Row(3, Double.NaN) ::
-        Row(4, Double.NaN) ::
-        Row(5, Double.NaN) ::
-        Row(6, Double.NaN) :: Nil)
+        Row(3, null) ::
+        Row(4, null) ::
+        Row(5, null) ::
+        Row(6, null) :: Nil)
 
       val corr7 = spark.sql("SELECT corr(b, c) FROM covar_tab").collect()(0).getDouble(0)
       assert(math.abs(corr7 - 0.6633880657639323) < 1e-12)
@@ -868,7 +869,7 @@ abstract class AggregationQuerySuite extends QueryTest with SQLTestUtils with Te
 
     // one row test
     val df3 = Seq.tabulate(1)(x => (1 * x, x * x * x - 2)).toDF("a", "b")
-    checkAnswer(df3.groupBy().agg(covar_samp("a", "b")), Row(Double.NaN))
+    checkAnswer(df3.groupBy().agg(covar_samp("a", "b")), Row(null))
     checkAnswer(df3.groupBy().agg(covar_pop("a", "b")), Row(0.0))
   }
 
@@ -1054,6 +1055,7 @@ abstract class AggregationQuerySuite extends QueryTest with SQLTestUtils with Te
 class HashAggregationQuerySuite extends AggregationQuerySuite
 
 
+@SlowHiveTest
 class HashAggregationQueryWithControlledFallbackSuite extends AggregationQuerySuite {
 
   override protected def checkAnswer(actual: => DataFrame, expectedAnswer: Seq[Row]): Unit = {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala
index dcec8bf5c0cc6..8834006888576 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala
@@ -22,9 +22,8 @@ import java.io.File
 import com.google.common.io.Files
 import org.apache.hadoop.fs.{FileContext, FsConstants, Path}
 
-import org.apache.spark.sql.{AnalysisException, QueryTest, Row, SaveMode}
+import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
 import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
 import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType}
 import org.apache.spark.sql.execution.command.LoadDataCommand
 import org.apache.spark.sql.hive.test.TestHiveSingleton
@@ -33,7 +32,6 @@ import org.apache.spark.sql.test.SQLTestUtils
 import org.apache.spark.sql.types.StructType
 
 class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
-  import testImplicits._
 
   protected override def beforeAll(): Unit = {
     super.beforeAll()
@@ -58,27 +56,11 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
         |STORED AS PARQUET
         |TBLPROPERTIES('prop1Key'="prop1Val", '`prop2Key`'="prop2Val")
       """.stripMargin)
-    sql("CREATE TABLE parquet_tab3(col1 int, `col 2` int) USING hive")
     sql("CREATE TABLE parquet_tab4 (price int, qty int) partitioned by (year int, month int)")
     sql("INSERT INTO parquet_tab4 PARTITION(year = 2015, month = 1) SELECT 1, 1")
     sql("INSERT INTO parquet_tab4 PARTITION(year = 2015, month = 2) SELECT 2, 2")
     sql("INSERT INTO parquet_tab4 PARTITION(year = 2016, month = 2) SELECT 3, 3")
     sql("INSERT INTO parquet_tab4 PARTITION(year = 2016, month = 3) SELECT 3, 3")
-    sql(
-      """
-        |CREATE TABLE parquet_tab5 (price int, qty int)
-        |PARTITIONED BY (year int, month int, hour int, minute int, sec int, extra int)
-      """.stripMargin)
-    sql(
-      """
-        |INSERT INTO parquet_tab5
-        |PARTITION(year = 2016, month = 3, hour = 10, minute = 10, sec = 10, extra = 1) SELECT 3, 3
-      """.stripMargin)
-    sql(
-      """
-        |INSERT INTO parquet_tab5
-        |PARTITION(year = 2016, month = 4, hour = 10, minute = 10, sec = 10, extra = 1) SELECT 3, 3
-      """.stripMargin)
     sql("CREATE VIEW parquet_view1 as select * from parquet_tab4")
   }
 
@@ -86,37 +68,13 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
     try {
       sql("DROP TABLE IF EXISTS parquet_tab1")
       sql("DROP TABLE IF EXISTS parquet_tab2")
-      sql("DROP TABLE IF EXISTS parquet_tab3")
       sql("DROP VIEW IF EXISTS parquet_view1")
       sql("DROP TABLE IF EXISTS parquet_tab4")
-      sql("DROP TABLE IF EXISTS parquet_tab5")
     } finally {
       super.afterAll()
     }
   }
 
-  test("show tables") {
-    withTable("show1a", "show2b") {
-      sql("CREATE TABLE show1a(c1 int)")
-      sql("CREATE TABLE show2b(c2 int)")
-      checkAnswer(
-        sql("SHOW TABLES IN default 'show1*'"),
-        Row("default", "show1a", false) :: Nil)
-      checkAnswer(
-        sql("SHOW TABLES IN default 'show1*|show2*'"),
-        Row("default", "show1a", false) ::
-          Row("default", "show2b", false) :: Nil)
-      checkAnswer(
-        sql("SHOW TABLES 'show1*|show2*'"),
-        Row("default", "show1a", false) ::
-          Row("default", "show2b", false) :: Nil)
-      assert(
-        sql("SHOW TABLES").count() >= 2)
-      assert(
-        sql("SHOW TABLES IN default").count() >= 2)
-    }
-  }
-
   test("show views") {
     withView("show1a", "show2b", "global_temp.temp1", "temp2") {
       sql("CREATE VIEW show1a AS SELECT 1 AS id")
@@ -171,7 +129,7 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
 
     checkAnswer(
       sql(s"SHOW TBLPROPERTIES parquet_tab1('my_key1')"),
-      Row("v1") :: Nil
+      Row("my_key1", "v1") :: Nil
     )
   }
 
@@ -184,13 +142,23 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
     // When key is not found, a row containing the error is returned.
     checkAnswer(
       sql("SHOW TBLPROPERTIES parquet_tab1('invalid.prop.key')"),
-      Row("Table default.parquet_tab1 does not have property: invalid.prop.key") :: Nil
+      Row("invalid.prop.key",
+        "Table default.parquet_tab1 does not have property: invalid.prop.key") :: Nil
     )
   }
 
-  test("show tblproperties for hive table") {
-    checkAnswer(sql("SHOW TBLPROPERTIES parquet_tab2('prop1Key')"), Row("prop1Val"))
-    checkAnswer(sql("SHOW TBLPROPERTIES parquet_tab2('`prop2Key`')"), Row("prop2Val"))
+  test("SPARK-34240 Unify output of SHOW TBLPROPERTIES and pass output attributes properly") {
+    checkAnswer(sql("SHOW TBLPROPERTIES parquet_tab2").filter("key != 'transient_lastDdlTime'"),
+      Row("prop1Key", "prop1Val") :: Row("`prop2Key`", "prop2Val") :: Nil)
+    checkAnswer(sql("SHOW TBLPROPERTIES parquet_tab2('prop1Key')"), Row("prop1Key", "prop1Val"))
+    checkAnswer(sql("SHOW TBLPROPERTIES parquet_tab2('`prop2Key`')"), Row("`prop2Key`", "prop2Val"))
+    withSQLConf(SQLConf.LEGACY_KEEP_COMMAND_OUTPUT_SCHEMA.key -> "true") {
+      checkAnswer(sql("SHOW TBLPROPERTIES parquet_tab2").filter("key != 'transient_lastDdlTime'"),
+        Row("prop1Key", "prop1Val") :: Row("`prop2Key`", "prop2Val") :: Nil)
+      checkAnswer(sql("SHOW TBLPROPERTIES parquet_tab2('prop1Key')"), Row("prop1Val"))
+      checkAnswer(sql("SHOW TBLPROPERTIES parquet_tab2('`prop2Key`')"),
+        Row("prop2Val"))
+    }
   }
 
   Seq(true, false).foreach { local =>
@@ -349,154 +317,6 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
     }
   }
 
-  test("Truncate Table") {
-    withTable("non_part_table", "part_table") {
-      sql(
-        """
-          |CREATE TABLE non_part_table (employeeID INT, employeeName STRING)
-          |ROW FORMAT DELIMITED
-          |FIELDS TERMINATED BY '|'
-          |LINES TERMINATED BY '\n'
-        """.stripMargin)
-
-      val testData = hiveContext.getHiveFile("data/files/employee.dat").toURI
-
-      sql(s"""LOAD DATA LOCAL INPATH "$testData" INTO TABLE non_part_table""")
-      checkAnswer(
-        sql("SELECT * FROM non_part_table WHERE employeeID = 16"),
-        Row(16, "john") :: Nil)
-
-      val testResults = sql("SELECT * FROM non_part_table").collect()
-
-      sql("TRUNCATE TABLE non_part_table")
-      checkAnswer(sql("SELECT * FROM non_part_table"), Seq.empty[Row])
-
-      sql(
-        """
-          |CREATE TABLE part_table (employeeID INT, employeeName STRING)
-          |PARTITIONED BY (c STRING, d STRING)
-          |ROW FORMAT DELIMITED
-          |FIELDS TERMINATED BY '|'
-          |LINES TERMINATED BY '\n'
-        """.stripMargin)
-
-      sql(s"""LOAD DATA LOCAL INPATH "$testData" INTO TABLE part_table PARTITION(c="1", d="1")""")
-      checkAnswer(
-        sql("SELECT employeeID, employeeName FROM part_table WHERE c = '1' AND d = '1'"),
-        testResults)
-
-      sql(s"""LOAD DATA LOCAL INPATH "$testData" INTO TABLE part_table PARTITION(c="1", d="2")""")
-      checkAnswer(
-        sql("SELECT employeeID, employeeName FROM part_table WHERE c = '1' AND d = '2'"),
-        testResults)
-
-      sql(s"""LOAD DATA LOCAL INPATH "$testData" INTO TABLE part_table PARTITION(c="2", d="2")""")
-      checkAnswer(
-        sql("SELECT employeeID, employeeName FROM part_table WHERE c = '2' AND d = '2'"),
-        testResults)
-
-      sql("TRUNCATE TABLE part_table PARTITION(c='1', d='1')")
-      checkAnswer(
-        sql("SELECT employeeID, employeeName FROM part_table WHERE c = '1' AND d = '1'"),
-        Seq.empty[Row])
-      checkAnswer(
-        sql("SELECT employeeID, employeeName FROM part_table WHERE c = '1' AND d = '2'"),
-        testResults)
-
-      sql("TRUNCATE TABLE part_table PARTITION(c='1')")
-      checkAnswer(
-        sql("SELECT employeeID, employeeName FROM part_table WHERE c = '1'"),
-        Seq.empty[Row])
-
-      sql("TRUNCATE TABLE part_table")
-      checkAnswer(
-        sql("SELECT employeeID, employeeName FROM part_table"),
-        Seq.empty[Row])
-    }
-  }
-
-
-  test("show partitions - show everything") {
-    checkAnswer(
-      sql("show partitions parquet_tab4"),
-      Row("year=2015/month=1") ::
-        Row("year=2015/month=2") ::
-        Row("year=2016/month=2") ::
-        Row("year=2016/month=3") :: Nil)
-
-    checkAnswer(
-      sql("show partitions default.parquet_tab4"),
-      Row("year=2015/month=1") ::
-        Row("year=2015/month=2") ::
-        Row("year=2016/month=2") ::
-        Row("year=2016/month=3") :: Nil)
-  }
-
-  test("show partitions - show everything more than 5 part keys") {
-    checkAnswer(
-      sql("show partitions parquet_tab5"),
-      Row("year=2016/month=3/hour=10/minute=10/sec=10/extra=1") ::
-        Row("year=2016/month=4/hour=10/minute=10/sec=10/extra=1") :: Nil)
-  }
-
-  test("show partitions - filter") {
-    checkAnswer(
-      sql("show partitions default.parquet_tab4 PARTITION(year=2015)"),
-      Row("year=2015/month=1") ::
-        Row("year=2015/month=2") :: Nil)
-
-    checkAnswer(
-      sql("show partitions default.parquet_tab4 PARTITION(year=2015, month=1)"),
-      Row("year=2015/month=1") :: Nil)
-
-    checkAnswer(
-      sql("show partitions default.parquet_tab4 PARTITION(month=2)"),
-      Row("year=2015/month=2") ::
-        Row("year=2016/month=2") :: Nil)
-  }
-
-  test("show partitions - empty row") {
-    withTempView("parquet_temp") {
-      sql(
-        """
-         |CREATE TEMPORARY VIEW parquet_temp (c1 INT, c2 STRING)
-         |USING org.apache.spark.sql.parquet.DefaultSource
-        """.stripMargin)
-      // An empty sequence of row is returned for session temporary table.
-      intercept[NoSuchTableException] {
-        sql("SHOW PARTITIONS parquet_temp")
-      }
-
-      val message1 = intercept[AnalysisException] {
-        sql("SHOW PARTITIONS parquet_tab3")
-      }.getMessage
-      assert(message1.contains("not allowed on a table that is not partitioned"))
-
-      val message2 = intercept[AnalysisException] {
-        sql("SHOW PARTITIONS parquet_tab4 PARTITION(abcd=2015, xyz=1)")
-      }.getMessage
-      assert(message2.contains("Non-partitioning column(s) [abcd, xyz] are specified"))
-
-      val message3 = intercept[AnalysisException] {
-        sql("SHOW PARTITIONS parquet_view1")
-      }.getMessage
-      assert(message3.contains("is not allowed on a view"))
-    }
-  }
-
-  test("show partitions - datasource") {
-    withTable("part_datasrc") {
-      val df = (1 to 3).map(i => (i, s"val_$i", i * 2)).toDF("a", "b", "c")
-      df.write
-        .partitionBy("a")
-        .format("parquet")
-        .mode(SaveMode.Overwrite)
-        .saveAsTable("part_datasrc")
-
-      assert(sql("SHOW PARTITIONS part_datasrc").count() == 3)
-    }
-  }
-
   test("SPARK-25918: LOAD DATA LOCAL INPATH should handle a relative path") {
     val localFS = FileContext.getLocalFSFileContext()
     val workingDir = localFS.getWorkingDirectory
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
index 2e4c01830432f..67b51f9347827 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
@@ -171,7 +171,7 @@ abstract class HiveComparisonTest extends SparkFunSuite with BeforeAndAfterAll {
       // and does not return it as a query answer.
       case _: SetCommand => Seq("0")
       case _: ExplainCommand => answer
-      case _: DescribeCommandBase | ShowColumnsCommand(_, _) =>
+      case _: DescribeCommandBase | ShowColumnsCommand(_, _, _) =>
         // Filter out non-deterministic lines and lines which do not have actual results but
         // can introduce problems because of the way Hive formats these lines.
         // Then, remove empty lines. Do not sort the results.
@@ -371,11 +371,11 @@ abstract class HiveComparisonTest extends SparkFunSuite with BeforeAndAfterAll {
 
             // We will ignore the ExplainCommand, ShowFunctions, DescribeFunction
             if ((!hiveQuery.logical.isInstanceOf[ExplainCommand]) &&
-                (!hiveQuery.logical.isInstanceOf[ShowFunctionsStatement]) &&
-                (!hiveQuery.logical.isInstanceOf[DescribeFunctionStatement]) &&
+                (!hiveQuery.logical.isInstanceOf[ShowFunctions]) &&
+                (!hiveQuery.logical.isInstanceOf[DescribeFunction]) &&
                 (!hiveQuery.logical.isInstanceOf[DescribeCommandBase]) &&
                 (!hiveQuery.logical.isInstanceOf[DescribeRelation]) &&
-                (!hiveQuery.logical.isInstanceOf[DescribeColumnStatement]) &&
+                (!hiveQuery.logical.isInstanceOf[DescribeColumn]) &&
                 preparedHive != catalyst) {
 
               val hivePrintOut = s"== HIVE - ${preparedHive.size} row(s) ==" +: preparedHive
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
index e8548fd62ddc1..9a39f1872be09 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
@@ -19,34 +19,38 @@ package org.apache.spark.sql.hive.execution
 
 import java.io.File
 import java.net.URI
+import java.util.Locale
 
 import org.apache.hadoop.fs.Path
 import org.apache.parquet.format.converter.ParquetMetadataConverter.NO_FILTER
-import org.apache.parquet.hadoop.ParquetFileReader
 import org.scalatest.BeforeAndAfterEach
 
 import org.apache.spark.SparkException
 import org.apache.spark.sql.{AnalysisException, QueryTest, Row, SaveMode}
 import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionException, TableAlreadyExistsException}
+import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.parser.ParseException
+import org.apache.spark.sql.connector.FakeV2Provider
 import org.apache.spark.sql.connector.catalog.CatalogManager
 import org.apache.spark.sql.connector.catalog.SupportsNamespaces.PROP_OWNER
 import org.apache.spark.sql.execution.command.{DDLSuite, DDLUtils}
+import org.apache.spark.sql.execution.datasources.parquet.ParquetFooterReader
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.hive.HiveExternalCatalog
 import org.apache.spark.sql.hive.HiveUtils.{CONVERT_METASTORE_ORC, CONVERT_METASTORE_PARQUET}
 import org.apache.spark.sql.hive.orc.OrcFileOperator
-import org.apache.spark.sql.hive.test.TestHiveSingleton
+import org.apache.spark.sql.hive.test.{TestHive, TestHiveSingleton, TestHiveSparkSession}
 import org.apache.spark.sql.internal.{HiveSerDe, SQLConf}
 import org.apache.spark.sql.internal.SQLConf.ORC_IMPLEMENTATION
 import org.apache.spark.sql.internal.StaticSQLConf.CATALOG_IMPLEMENTATION
 import org.apache.spark.sql.test.SQLTestUtils
 import org.apache.spark.sql.types._
+import org.apache.spark.tags.SlowHiveTest
 import org.apache.spark.util.Utils
 
 // TODO(gatorsmile): combine HiveCatalogedDDLSuite and HiveDDLSuite
+@SlowHiveTest
 class HiveCatalogedDDLSuite extends DDLSuite with TestHiveSingleton with BeforeAndAfterEach {
   override def afterEach(): Unit = {
     try {
@@ -125,7 +129,7 @@ class HiveCatalogedDDLSuite extends DDLSuite with TestHiveSingleton with BeforeA
       createTime = 0L,
       lastAccessTime = 0L,
       owner = "",
-      properties = table.properties.filterKeys(!nondeterministicProps.contains(_)),
+      properties = table.properties.filterKeys(!nondeterministicProps.contains(_)).toMap,
       // View texts are checked separately
       viewText = None
     )
@@ -155,22 +159,6 @@ class HiveCatalogedDDLSuite extends DDLSuite with TestHiveSingleton with BeforeA
     testChangeColumn(isDatasourceTable = false)
   }
 
-  test("alter table: rename partition") {
-    testRenamePartitions(isDatasourceTable = false)
-  }
-
-  test("alter table: drop partition") {
-    testDropPartitions(isDatasourceTable = false)
-  }
-
-  test("alter table: add partition") {
-    testAddPartitions(isDatasourceTable = false)
-  }
-
-  test("drop table") {
-    testDropTable(isDatasourceTable = false)
-  }
-
   test("alter datasource table add columns - orc") {
     testAddColumn("orc")
   }
@@ -381,11 +369,11 @@ class HiveCatalogedDDLSuite extends DDLSuite with TestHiveSingleton with BeforeA
       val db = "spark_29425_1"
       sql(s"CREATE DATABASE $db")
       assert(sql(s"DESCRIBE DATABASE EXTENDED $db")
-        .where("database_description_item='Owner'")
+        .where("info_name='Owner'")
         .collect().head.getString(1) === Utils.getCurrentUserName())
       sql(s"ALTER DATABASE $db SET DBPROPERTIES('abc'='xyz')")
       assert(sql(s"DESCRIBE DATABASE EXTENDED $db")
-        .where("database_description_item='Owner'")
+        .where("info_name='Owner'")
         .collect().head.getString(1) === Utils.getCurrentUserName())
     } finally {
       catalog.reset()
@@ -404,6 +392,7 @@ class HiveCatalogedDDLSuite extends DDLSuite with TestHiveSingleton with BeforeA
   }
 }
 
+@SlowHiveTest
 class HiveDDLSuite
   extends QueryTest with SQLTestUtils with TestHiveSingleton with BeforeAndAfterEach {
   import testImplicits._
@@ -454,15 +443,15 @@ class HiveDDLSuite
       withTable("tab1", "tab2") {
         (("a", "b") :: Nil).toDF().write.json(tempDir.getCanonicalPath)
 
-        var e = intercept[AnalysisException] { sql("CREATE TABLE tab1 USING hive") }.getMessage
-        assert(e.contains("Unable to infer the schema. The schema specification is required to " +
-          "create the table `default`.`tab1`"))
+        assertAnalysisError(
+          "CREATE TABLE tab1 USING hive",
+          "Unable to infer the schema. The schema specification is required to " +
+            "create the table `default`.`tab1`")
 
-        e = intercept[AnalysisException] {
-          sql(s"CREATE TABLE tab2 USING hive location '${tempDir.getCanonicalPath}'")
-        }.getMessage
-        assert(e.contains("Unable to infer the schema. The schema specification is required to " +
-          "create the table `default`.`tab2`"))
+        assertAnalysisError(
+          s"CREATE TABLE tab2 USING hive location '${tempDir.getCanonicalPath}'",
+          "Unable to infer the schema. The schema specification is required to " +
+            "create the table `default`.`tab2`")
       }
     }
   }
@@ -584,26 +573,23 @@ class HiveDDLSuite
   }
 
   test("create table: partition column names exist in table definition") {
-    val e = intercept[AnalysisException] {
-      sql("CREATE TABLE tbl(a int) PARTITIONED BY (a string)")
-    }
-    assert(e.message == "Found duplicate column(s) in the table definition of `default`.`tbl`: `a`")
+    assertAnalysisError(
+      "CREATE TABLE tbl(a int) PARTITIONED BY (a string)",
+      "Found duplicate column(s) in the table definition of `default`.`tbl`: `a`")
   }
 
   test("create partitioned table without specifying data type for the partition columns") {
-    val e = intercept[AnalysisException] {
-      sql("CREATE TABLE tbl(a int) PARTITIONED BY (b) STORED AS parquet")
-    }
-    assert(e.message.contains("Must specify a data type for each partition column while creating " +
-      "Hive partitioned table."))
+    assertAnalysisError(
+      "CREATE TABLE tbl(a int) PARTITIONED BY (b) STORED AS parquet",
+      "partition column b is not defined in table")
   }
 
   test("add/drop partition with location - managed table") {
     val tab = "tab_with_partitions"
     withTempDir { tmpDir =>
       val basePath = new File(tmpDir.getCanonicalPath)
-      val part1Path = new File(basePath + "/part1")
-      val part2Path = new File(basePath + "/part2")
+      val part1Path = new File(new File(basePath, "part10"), "part11")
+      val part2Path = new File(new File(basePath, "part20"), "part21")
       val dirSet = part1Path :: part2Path :: Nil
 
       // Before data insertion, all the directory are empty
@@ -647,11 +633,10 @@ class HiveDDLSuite
   test("SPARK-19129: drop partition with a empty string will drop the whole table") {
     val df = spark.createDataFrame(Seq((0, "a"), (1, "b"))).toDF("partCol1", "name")
     df.write.mode("overwrite").partitionBy("partCol1").saveAsTable("partitionedTable")
-    val e = intercept[AnalysisException] {
-      spark.sql("alter table partitionedTable drop partition(partCol1='')")
-    }.getMessage
-    assert(e.contains("Partition spec is invalid. The spec ([partCol1=]) contains an empty " +
-      "partition column value"))
+    assertAnalysisError(
+      "alter table partitionedTable drop partition(partCol1='')",
+      "Partition spec is invalid. The spec ([partCol1=]) contains an empty " +
+        "partition column value")
   }
 
   test("add/drop partitions - external table") {
@@ -696,11 +681,10 @@ class HiveDDLSuite
         // After data insertion, all the directory are not empty
         assert(dirSet.forall(dir => dir.listFiles.nonEmpty))
 
-        val message = intercept[AnalysisException] {
-          sql(s"ALTER TABLE $externalTab DROP PARTITION (ds='2008-04-09', unknownCol='12')")
-        }
-        assert(message.getMessage.contains("unknownCol is not a valid partition column in table " +
-          "`default`.`exttable_with_partitions`"))
+        assertAnalysisError(
+          s"ALTER TABLE $externalTab DROP PARTITION (ds='2008-04-09', unknownCol='12')",
+          "unknownCol is not a valid partition column in table " +
+            "`default`.`exttable_with_partitions`")
 
         sql(
           s"""
@@ -802,15 +786,18 @@ class HiveDDLSuite
         sql(s"ALTER VIEW $viewName UNSET TBLPROPERTIES ('p')")
         checkProperties(Map())
 
-        val message = intercept[AnalysisException] {
-          sql(s"ALTER VIEW $viewName UNSET TBLPROPERTIES ('p')")
-        }.getMessage
-        assert(message.contains(
-          "Attempted to unset non-existent property 'p' in table '`default`.`view1`'"))
+        assertAnalysisError(
+          s"ALTER VIEW $viewName UNSET TBLPROPERTIES ('p')",
+          "Attempted to unset non-existent property 'p' in table '`default`.`view1`'")
       }
     }
   }
 
+  private def assertAnalysisError(sqlText: String, message: String): Unit = {
+    val e = intercept[AnalysisException](sql(sqlText))
+    assert(e.message.contains(message))
+  }
+
   private def assertErrorForAlterTableOnView(sqlText: String): Unit = {
     val message = intercept[AnalysisException](sql(sqlText)).getMessage
     assert(message.contains("Cannot alter a view with ALTER TABLE. Please use ALTER VIEW instead"))
@@ -821,13 +808,26 @@ class HiveDDLSuite
     assert(message.contains("Cannot alter a table with ALTER VIEW. Please use ALTER TABLE instead"))
   }
 
+  private def assertErrorForAlterTableOnView(
+      sqlText: String, viewName: String, cmdName: String): Unit = {
+    assertAnalysisError(
+      sqlText,
+      s"$viewName is a view. '$cmdName' expects a table. Please use ALTER VIEW instead.")
+  }
+
+  private def assertErrorForAlterViewOnTable(
+      sqlText: String, tableName: String, cmdName: String): Unit = {
+    assertAnalysisError(
+      sqlText,
+      s"$tableName is a table. '$cmdName' expects a view. Please use ALTER TABLE instead.")
+  }
+
   test("create table - SET TBLPROPERTIES EXTERNAL to TRUE") {
     val tabName = "tab1"
     withTable(tabName) {
-      val message = intercept[AnalysisException] {
-        sql(s"CREATE TABLE $tabName (height INT, length INT) TBLPROPERTIES('EXTERNAL'='TRUE')")
-      }.getMessage
-      assert(message.contains("Cannot set or change the preserved property key: 'EXTERNAL'"))
+      assertAnalysisError(
+        s"CREATE TABLE $tabName (height INT, length INT) TBLPROPERTIES('EXTERNAL'='TRUE')",
+        "Cannot set or change the preserved property key: 'EXTERNAL'")
     }
   }
 
@@ -838,10 +838,9 @@ class HiveDDLSuite
       sql(s"CREATE TABLE $tabName (height INT, length INT)")
       assert(
         catalog.getTableMetadata(TableIdentifier(tabName)).tableType == CatalogTableType.MANAGED)
-      val message = intercept[AnalysisException] {
-        sql(s"ALTER TABLE $tabName SET TBLPROPERTIES ('EXTERNAL' = 'TRUE')")
-      }.getMessage
-      assert(message.contains("Cannot set or change the preserved property key: 'EXTERNAL'"))
+      assertAnalysisError(
+        s"ALTER TABLE $tabName SET TBLPROPERTIES ('EXTERNAL' = 'TRUE')",
+        "Cannot set or change the preserved property key: 'EXTERNAL'")
       // The table type is not changed to external
       assert(
         catalog.getTableMetadata(TableIdentifier(tabName)).tableType == CatalogTableType.MANAGED)
@@ -871,32 +870,65 @@ class HiveDDLSuite
 
         assertErrorForAlterTableOnView(s"ALTER TABLE $oldViewName RENAME TO $newViewName")
 
-        assertErrorForAlterViewOnTable(s"ALTER VIEW $tabName SET TBLPROPERTIES ('p' = 'an')")
+        assertErrorForAlterViewOnTable(
+          s"ALTER VIEW $tabName SET TBLPROPERTIES ('p' = 'an')",
+          tabName,
+          "ALTER VIEW ... SET TBLPROPERTIES")
 
-        assertErrorForAlterTableOnView(s"ALTER TABLE $oldViewName SET TBLPROPERTIES ('p' = 'an')")
+        assertErrorForAlterTableOnView(
+          s"ALTER TABLE $oldViewName SET TBLPROPERTIES ('p' = 'an')",
+          oldViewName,
+          "ALTER TABLE ... SET TBLPROPERTIES")
 
-        assertErrorForAlterViewOnTable(s"ALTER VIEW $tabName UNSET TBLPROPERTIES ('p')")
+        assertErrorForAlterViewOnTable(
+          s"ALTER VIEW $tabName UNSET TBLPROPERTIES ('p')",
+          tabName,
+          "ALTER VIEW ... UNSET TBLPROPERTIES")
 
-        assertErrorForAlterTableOnView(s"ALTER TABLE $oldViewName UNSET TBLPROPERTIES ('p')")
+        assertErrorForAlterTableOnView(
+          s"ALTER TABLE $oldViewName UNSET TBLPROPERTIES ('p')",
+          oldViewName,
+          "ALTER TABLE ... UNSET TBLPROPERTIES")
 
-        assertErrorForAlterTableOnView(s"ALTER TABLE $oldViewName SET LOCATION '/path/to/home'")
+        assertErrorForAlterTableOnView(
+          s"ALTER TABLE $oldViewName SET LOCATION '/path/to/home'",
+          oldViewName,
+          "ALTER TABLE ... SET LOCATION ...")
 
-        assertErrorForAlterTableOnView(s"ALTER TABLE $oldViewName SET SERDE 'whatever'")
+        assertErrorForAlterTableOnView(
+          s"ALTER TABLE $oldViewName SET SERDE 'whatever'",
+          oldViewName,
+          "ALTER TABLE ... SET [SERDE|SERDEPROPERTIES]")
 
-        assertErrorForAlterTableOnView(s"ALTER TABLE $oldViewName SET SERDEPROPERTIES ('x' = 'y')")
+        assertErrorForAlterTableOnView(
+          s"ALTER TABLE $oldViewName SET SERDEPROPERTIES ('x' = 'y')",
+          oldViewName,
+          "ALTER TABLE ... SET [SERDE|SERDEPROPERTIES]")
 
         assertErrorForAlterTableOnView(
-          s"ALTER TABLE $oldViewName PARTITION (a=1, b=2) SET SERDEPROPERTIES ('x' = 'y')")
+          s"ALTER TABLE $oldViewName PARTITION (a=1, b=2) SET SERDEPROPERTIES ('x' = 'y')",
+          oldViewName,
+          "ALTER TABLE ... SET [SERDE|SERDEPROPERTIES]")
 
         assertErrorForAlterTableOnView(
-          s"ALTER TABLE $oldViewName ADD IF NOT EXISTS PARTITION (a='4', b='8')")
+          s"ALTER TABLE $oldViewName RECOVER PARTITIONS",
+          oldViewName,
+          "ALTER TABLE ... RECOVER PARTITIONS")
 
-        assertErrorForAlterTableOnView(s"ALTER TABLE $oldViewName DROP IF EXISTS PARTITION (a='2')")
+        assertErrorForAlterTableOnView(
+          s"ALTER TABLE $oldViewName PARTITION (a='1') RENAME TO PARTITION (a='100')",
+          oldViewName,
+          "ALTER TABLE ... RENAME TO PARTITION")
 
-        assertErrorForAlterTableOnView(s"ALTER TABLE $oldViewName RECOVER PARTITIONS")
+        assertErrorForAlterTableOnView(
+          s"ALTER TABLE $oldViewName ADD IF NOT EXISTS PARTITION (a='4', b='8')",
+          oldViewName,
+          "ALTER TABLE ... ADD PARTITION ...")
 
         assertErrorForAlterTableOnView(
-          s"ALTER TABLE $oldViewName PARTITION (a='1') RENAME TO PARTITION (a='100')")
+          s"ALTER TABLE $oldViewName DROP IF EXISTS PARTITION (a='2')",
+          oldViewName,
+          "ALTER TABLE ... DROP PARTITION ...")
 
         assert(catalog.tableExists(TableIdentifier(tabName)))
         assert(catalog.tableExists(TableIdentifier(oldViewName)))
@@ -973,7 +1005,7 @@ class HiveDDLSuite
   }
 
   test("alter table partition - storage information") {
-    sql("CREATE TABLE boxes (height INT, length INT) PARTITIONED BY (width INT)")
+    sql("CREATE TABLE boxes (height INT, length INT) STORED AS textfile PARTITIONED BY (width INT)")
     sql("INSERT OVERWRITE TABLE boxes PARTITION (width=4) SELECT 4, 4")
     val catalog = spark.sessionState.catalog
     val expectedSerde = "com.sparkbricks.serde.ColumnarSerDe"
@@ -990,7 +1022,7 @@ class HiveDDLSuite
       |""".stripMargin)
     val newPart = catalog.getPartition(TableIdentifier("boxes"), Map("width" -> "4"))
     assert(newPart.storage.serde == Some(expectedSerde))
-    assert(newPart.storage.properties.filterKeys(expectedSerdeProps.contains) ==
+    assert(newPart.storage.properties.filterKeys(expectedSerdeProps.contains).toMap ==
       expectedSerdeProps)
   }
 
@@ -1035,10 +1067,9 @@ class HiveDDLSuite
   test("drop table using drop view") {
     withTable("tab1") {
       sql("CREATE TABLE tab1(c1 int)")
-      val message = intercept[AnalysisException] {
-        sql("DROP VIEW tab1")
-      }.getMessage
-      assert(message.contains("Cannot drop a table with DROP VIEW. Please use DROP TABLE instead"))
+      assertAnalysisError(
+        "DROP VIEW tab1",
+        "tab1 is a table. 'DROP VIEW' expects a view. Please use DROP TABLE instead.")
     }
   }
 
@@ -1047,10 +1078,9 @@ class HiveDDLSuite
       spark.range(10).write.saveAsTable("tab1")
       withView("view1") {
         sql("CREATE VIEW view1 AS SELECT * FROM tab1")
-        val message = intercept[AnalysisException] {
-          sql("DROP TABLE view1")
-        }.getMessage
-        assert(message.contains("Cannot drop a view with DROP TABLE. Please use DROP VIEW instead"))
+        assertAnalysisError(
+          "DROP TABLE view1",
+          "Cannot drop a view with DROP TABLE. Please use DROP VIEW instead")
       }
     }
   }
@@ -1188,7 +1218,7 @@ class HiveDDLSuite
       expectedDBUri,
       Map.empty))
     // the database directory was created
-    assert(fs.exists(dbPath) && fs.isDirectory(dbPath))
+    assert(fs.exists(dbPath) && fs.getFileStatus(dbPath).isDirectory)
     sql(s"USE $dbName")
 
     val tabName = "tab1"
@@ -1204,10 +1234,9 @@ class HiveDDLSuite
     sql(s"USE default")
     val sqlDropDatabase = s"DROP DATABASE $dbName ${if (cascade) "CASCADE" else "RESTRICT"}"
     if (tableExists && !cascade) {
-      val message = intercept[AnalysisException] {
-        sql(sqlDropDatabase)
-      }.getMessage
-      assert(message.contains(s"Database $dbName is not empty. One or more tables exist."))
+      assertAnalysisError(
+        sqlDropDatabase,
+        s"Database $dbName is not empty. One or more tables exist.")
       // the database directory was not removed
       assert(fs.exists(new Path(expectedDBLocation)))
     } else {
@@ -1236,17 +1265,15 @@ class HiveDDLSuite
   test("drop default database") {
     Seq("true", "false").foreach { caseSensitive =>
       withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive) {
-        var message = intercept[AnalysisException] {
-          sql("DROP DATABASE default")
-        }.getMessage
-        assert(message.contains("Can not drop default database"))
+        assertAnalysisError(
+          "DROP DATABASE default",
+          "Can not drop default database")
 
         // SQLConf.CASE_SENSITIVE does not affect the result
         // because the Hive metastore is not case sensitive.
-        message = intercept[AnalysisException] {
-          sql("DROP DATABASE DeFault")
-        }.getMessage
-        assert(message.contains("Can not drop default database"))
+        assertAnalysisError(
+          "DROP DATABASE DeFault",
+          "Can not drop default database")
       }
     }
   }
@@ -1636,10 +1663,9 @@ class HiveDDLSuite
         }
 
         // When tableExists is not invoked, we still can get an AnalysisException
-        val e = intercept[AnalysisException] {
-          sql(s"DESCRIBE $indexTabName")
-        }.getMessage
-        assert(e.contains("Hive index table is not supported."))
+        assertAnalysisError(
+          s"DESCRIBE $indexTabName",
+          "Hive index table is not supported.")
       } finally {
         client.runSqlHive(s"DROP INDEX IF EXISTS $indexName ON $tabName")
       }
@@ -1709,84 +1735,18 @@ class HiveDDLSuite
       sql("CREATE TABLE tbl(a INT) STORED AS parquet")
 
       Seq(DATASOURCE_PREFIX, STATISTICS_PREFIX).foreach { forbiddenPrefix =>
-        val e = intercept[AnalysisException] {
-          sql(s"ALTER TABLE tbl SET TBLPROPERTIES ('${forbiddenPrefix}foo' = 'loser')")
-        }
-        assert(e.getMessage.contains(forbiddenPrefix + "foo"))
-
-        val e2 = intercept[AnalysisException] {
-          sql(s"ALTER TABLE tbl UNSET TBLPROPERTIES ('${forbiddenPrefix}foo')")
-        }
-        assert(e2.getMessage.contains(forbiddenPrefix + "foo"))
-
-        val e3 = intercept[AnalysisException] {
-          sql(s"CREATE TABLE tbl2 (a INT) TBLPROPERTIES ('${forbiddenPrefix}foo'='anything')")
-        }
-        assert(e3.getMessage.contains(forbiddenPrefix + "foo"))
-      }
-    }
-  }
-
-  test("truncate table - datasource table") {
-    import testImplicits._
-
-    val data = (1 to 10).map { i => (i, i) }.toDF("width", "length")
-    // Test both a Hive compatible and incompatible code path.
-    Seq("json", "parquet").foreach { format =>
-      withTable("rectangles") {
-        data.write.format(format).saveAsTable("rectangles")
-        assert(spark.table("rectangles").collect().nonEmpty,
-          "bad test; table was empty to begin with")
-
-        sql("TRUNCATE TABLE rectangles")
-        assert(spark.table("rectangles").collect().isEmpty)
+        assertAnalysisError(
+          s"ALTER TABLE tbl SET TBLPROPERTIES ('${forbiddenPrefix}foo' = 'loser')",
+          s"${forbiddenPrefix}foo")
 
-        // not supported since the table is not partitioned
-        val e = intercept[AnalysisException] {
-          sql("TRUNCATE TABLE rectangles PARTITION (width=1)")
-        }
-        assert(e.message.contains("Operation not allowed"))
-      }
-    }
-  }
-
-  test("truncate partitioned table - datasource table") {
-    import testImplicits._
-
-    val data = (1 to 10).map { i => (i % 3, i % 5, i) }.toDF("width", "length", "height")
-
-    withTable("partTable") {
-      data.write.partitionBy("width", "length").saveAsTable("partTable")
-      // supported since partitions are stored in the metastore
-      sql("TRUNCATE TABLE partTable PARTITION (width=1, length=1)")
-      assert(spark.table("partTable").filter($"width" === 1).collect().nonEmpty)
-      assert(spark.table("partTable").filter($"width" === 1 && $"length" === 1).collect().isEmpty)
-    }
-
-    withTable("partTable") {
-      data.write.partitionBy("width", "length").saveAsTable("partTable")
-      // support partial partition spec
-      sql("TRUNCATE TABLE partTable PARTITION (width=1)")
-      assert(spark.table("partTable").collect().nonEmpty)
-      assert(spark.table("partTable").filter($"width" === 1).collect().isEmpty)
-    }
-
-    withTable("partTable") {
-      data.write.partitionBy("width", "length").saveAsTable("partTable")
-      // do nothing if no partition is matched for the given partial partition spec
-      sql("TRUNCATE TABLE partTable PARTITION (width=100)")
-      assert(spark.table("partTable").count() == data.count())
+        assertAnalysisError(
+          s"ALTER TABLE tbl UNSET TBLPROPERTIES ('${forbiddenPrefix}foo')",
+          s"${forbiddenPrefix}foo")
 
-      // throw exception if no partition is matched for the given non-partial partition spec.
-      intercept[NoSuchPartitionException] {
-        sql("TRUNCATE TABLE partTable PARTITION (width=100, length=100)")
+        assertAnalysisError(
+          s"CREATE TABLE tbl2 (a INT) TBLPROPERTIES ('${forbiddenPrefix}foo'='anything')",
+          s"${forbiddenPrefix}foo")
       }
-
-      // throw exception if the column in partition spec is not a partition column.
-      val e = intercept[AnalysisException] {
-        sql("TRUNCATE TABLE partTable PARTITION (unknown=1)")
-      }
-      assert(e.message.contains("unknown is not a valid partition column"))
     }
   }
 
@@ -1893,6 +1853,138 @@ class HiveDDLSuite
     }
   }
 
+  test("SPARK-34370: support Avro schema evolution (add column with avro.schema.url)") {
+    checkAvroSchemaEvolutionAddColumn(
+      s"'avro.schema.url'='${TestHive.getHiveFile("schemaWithOneField.avsc").toURI}'",
+      s"'avro.schema.url'='${TestHive.getHiveFile("schemaWithTwoFields.avsc").toURI}'")
+  }
+
+  test("SPARK-26836: support Avro schema evolution (add column with avro.schema.literal)") {
+    val originalSchema =
+      """
+        |{
+        |  "namespace": "test",
+        |  "name": "some_schema",
+        |  "type": "record",
+        |  "fields": [
+        |    {
+        |      "name": "col2",
+        |      "type": "string"
+        |    }
+        |  ]
+        |}
+      """.stripMargin
+    val evolvedSchema =
+      """
+        |{
+        |  "namespace": "test",
+        |  "name": "some_schema",
+        |  "type": "record",
+        |  "fields": [
+        |    {
+        |      "name": "col1",
+        |      "type": "string",
+        |      "default": "col1_default"
+        |    },
+        |    {
+        |      "name": "col2",
+        |      "type": "string"
+        |    }
+        |  ]
+        |}
+      """.stripMargin
+    checkAvroSchemaEvolutionAddColumn(
+      s"'avro.schema.literal'='$originalSchema'",
+      s"'avro.schema.literal'='$evolvedSchema'")
+  }
+
+  private def checkAvroSchemaEvolutionAddColumn(
+    originalSerdeProperties: String,
+    evolvedSerdeProperties: String) = {
+    withTable("t") {
+      sql(
+        s"""
+          |CREATE TABLE t PARTITIONED BY (ds string)
+          |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
+          |WITH SERDEPROPERTIES ($originalSerdeProperties)
+          |STORED AS
+          |INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
+          |OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
+        """.stripMargin)
+      sql("INSERT INTO t partition (ds='1981-01-07') VALUES ('col2_value')")
+      sql(s"ALTER TABLE t SET SERDEPROPERTIES ($evolvedSerdeProperties)")
+      sql("INSERT INTO t partition (ds='1983-04-27') VALUES ('col1_value', 'col2_value')")
+      checkAnswer(spark.table("t"), Row("col1_default", "col2_value", "1981-01-07")
+        :: Row("col1_value", "col2_value", "1983-04-27") :: Nil)
+    }
+  }
+
+  test("SPARK-34370: support Avro schema evolution (remove column with avro.schema.url)") {
+    checkAvroSchemaEvolutionRemoveColumn(
+      s"'avro.schema.url'='${TestHive.getHiveFile("schemaWithTwoFields.avsc").toURI}'",
+      s"'avro.schema.url'='${TestHive.getHiveFile("schemaWithOneField.avsc").toURI}'")
+  }
+
+  test("SPARK-26836: support Avro schema evolution (remove column with avro.schema.literal)") {
+    val originalSchema =
+      """
+        |{
+        |  "namespace": "test",
+        |  "name": "some_schema",
+        |  "type": "record",
+        |  "fields": [
+        |    {
+        |      "name": "col1",
+        |      "type": "string",
+        |      "default": "col1_default"
+        |    },
+        |    {
+        |      "name": "col2",
+        |      "type": "string"
+        |    }
+        |  ]
+        |}
+      """.stripMargin
+    val evolvedSchema =
+      """
+        |{
+        |  "namespace": "test",
+        |  "name": "some_schema",
+        |  "type": "record",
+        |  "fields": [
+        |    {
+        |      "name": "col2",
+        |      "type": "string"
+        |    }
+        |  ]
+        |}
+      """.stripMargin
+    checkAvroSchemaEvolutionRemoveColumn(
+      s"'avro.schema.literal'='$originalSchema'",
+      s"'avro.schema.literal'='$evolvedSchema'")
+  }
+
+  private def checkAvroSchemaEvolutionRemoveColumn(
+    originalSerdeProperties: String,
+    evolvedSerdeProperties: String) = {
+    withTable("t") {
+      sql(
+        s"""
+          |CREATE TABLE t PARTITIONED BY (ds string)
+          |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
+          |WITH SERDEPROPERTIES ($originalSerdeProperties)
+          |STORED AS
+          |INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
+          |OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
+        """.stripMargin)
+      sql("INSERT INTO t partition (ds='1983-04-27') VALUES ('col1_value', 'col2_value')")
+      sql(s"ALTER TABLE t SET SERDEPROPERTIES ($evolvedSerdeProperties)")
+      sql("INSERT INTO t partition (ds='1981-01-07') VALUES ('col2_value')")
+      checkAnswer(spark.table("t"), Row("col2_value", "1981-01-07")
+        :: Row("col2_value", "1983-04-27") :: Nil)
+    }
+  }
+
   test("append data to hive serde table") {
     withTable("t", "t1") {
       Seq(1 -> "a").toDF("i", "j")
@@ -2144,10 +2236,9 @@ class HiveDDLSuite
             assert(loc.listFiles().length >= 1)
             checkAnswer(spark.table("t"), Row("1") :: Nil)
           } else {
-            val e = intercept[AnalysisException] {
-              spark.sql("INSERT INTO TABLE t SELECT 1")
-            }.getMessage
-            assert(e.contains("java.net.URISyntaxException: Relative path in absolute URI: a:b"))
+            assertAnalysisError(
+              "INSERT INTO TABLE t SELECT 1",
+              "java.net.URISyntaxException: Relative path in absolute URI: a:b")
           }
         }
 
@@ -2186,15 +2277,13 @@ class HiveDDLSuite
                 Row("1", "2") :: Row("1", "2017-03-03 12:13%3A14") :: Nil)
             }
           } else {
-            val e = intercept[AnalysisException] {
-              spark.sql("INSERT INTO TABLE t1 PARTITION(b=2) SELECT 1")
-            }.getMessage
-            assert(e.contains("java.net.URISyntaxException: Relative path in absolute URI: a:b"))
-
-            val e1 = intercept[AnalysisException] {
-              spark.sql("INSERT INTO TABLE t1 PARTITION(b='2017-03-03 12:13%3A14') SELECT 1")
-            }.getMessage
-            assert(e1.contains("java.net.URISyntaxException: Relative path in absolute URI: a:b"))
+            assertAnalysisError(
+              "INSERT INTO TABLE t1 PARTITION(b=2) SELECT 1",
+              "java.net.URISyntaxException: Relative path in absolute URI: a:b")
+
+            assertAnalysisError(
+              "INSERT INTO TABLE t1 PARTITION(b='2017-03-03 12:13%3A14') SELECT 1",
+              "java.net.URISyntaxException: Relative path in absolute URI: a:b")
           }
         }
       }
@@ -2241,8 +2330,8 @@ class HiveDDLSuite
         )
 
         sql("ALTER TABLE tab ADD COLUMNS (c5 char(10))")
-        assert(spark.table("tab").schema.find(_.name == "c5")
-          .get.metadata.getString("HIVE_TYPE_STRING") == "char(10)")
+        assert(spark.sharedState.externalCatalog.getTable("default", "tab")
+          .schema.find(_.name == "c5").get.dataType == CharType(10))
       }
     }
   }
@@ -2279,36 +2368,143 @@ class HiveDDLSuite
           sql("CREATE TABLE tab (c1 int) PARTITIONED BY (c2 int) STORED AS PARQUET")
           if (!caseSensitive) {
             // duplicating partitioning column name
-            val e1 = intercept[AnalysisException] {
-              sql("ALTER TABLE tab ADD COLUMNS (C2 string)")
-            }.getMessage
-            assert(e1.contains("Found duplicate column(s)"))
+            assertAnalysisError(
+              "ALTER TABLE tab ADD COLUMNS (C2 string)",
+              "Found duplicate column(s)")
 
             // duplicating data column name
-            val e2 = intercept[AnalysisException] {
-              sql("ALTER TABLE tab ADD COLUMNS (C1 string)")
-            }.getMessage
-            assert(e2.contains("Found duplicate column(s)"))
+            assertAnalysisError(
+              "ALTER TABLE tab ADD COLUMNS (C1 string)",
+              "Found duplicate column(s)")
           } else {
             // hive catalog will still complains that c1 is duplicate column name because hive
             // identifiers are case insensitive.
-            val e1 = intercept[AnalysisException] {
-              sql("ALTER TABLE tab ADD COLUMNS (C2 string)")
-            }.getMessage
-            assert(e1.contains("HiveException"))
+            assertAnalysisError(
+              "ALTER TABLE tab ADD COLUMNS (C2 string)",
+              "HiveException")
 
             // hive catalog will still complains that c1 is duplicate column name because hive
             // identifiers are case insensitive.
-            val e2 = intercept[AnalysisException] {
-              sql("ALTER TABLE tab ADD COLUMNS (C1 string)")
-            }.getMessage
-            assert(e2.contains("HiveException"))
+            assertAnalysisError(
+              "ALTER TABLE tab ADD COLUMNS (C1 string)",
+              "HiveException")
           }
         }
       }
     }
   }
 
+  test("SPARK-20680: do not support for null column datatype") {
+    withTable("t") {
+      withView("tabNullType") {
+        hiveClient.runSqlHive("CREATE TABLE t (t1 int)")
+        hiveClient.runSqlHive("INSERT INTO t VALUES (3)")
+        hiveClient.runSqlHive("CREATE VIEW tabNullType AS SELECT NULL AS col FROM t")
+        checkAnswer(spark.table("tabNullType"), Row(null))
+        // No exception shows
+        val desc = spark.sql("DESC tabNullType").collect().toSeq
+        assert(desc.contains(Row("col", NullType.simpleString, null)))
+      }
+    }
+
+    // Forbid CTAS with null type
+    withTable("t1", "t2", "t3") {
+      assertAnalysisError(
+        "CREATE TABLE t1 USING PARQUET AS SELECT null as null_col",
+        "Cannot create tables with null type")
+
+      assertAnalysisError(
+        "CREATE TABLE t2 AS SELECT null as null_col",
+        "Cannot create tables with null type")
+
+      assertAnalysisError(
+        "CREATE TABLE t3 STORED AS PARQUET AS SELECT null as null_col",
+        "Cannot create tables with null type")
+    }
+
+    // Forbid Replace table AS SELECT with null type
+    withTable("t") {
+      val v2Source = classOf[FakeV2Provider].getName
+      assertAnalysisError(
+        s"CREATE OR REPLACE TABLE t USING $v2Source AS SELECT null as null_col",
+        "Cannot create tables with null type")
+    }
+
+    // Forbid creating table with VOID type in Spark
+    withTable("t1", "t2", "t3", "t4") {
+      assertAnalysisError(
+        "CREATE TABLE t1 (v VOID) USING PARQUET",
+        "Cannot create tables with null type")
+      assertAnalysisError(
+        "CREATE TABLE t2 (v VOID) USING hive",
+        "Cannot create tables with null type")
+      assertAnalysisError(
+        "CREATE TABLE t3 (v VOID)",
+        "Cannot create tables with null type")
+      assertAnalysisError(
+        "CREATE TABLE t4 (v VOID) STORED AS PARQUET",
+        "Cannot create tables with null type")
+    }
+
+    // Forbid Replace table with VOID type
+    withTable("t") {
+      val v2Source = classOf[FakeV2Provider].getName
+      assertAnalysisError(
+        s"CREATE OR REPLACE TABLE t (v VOID) USING $v2Source",
+        "Cannot create tables with null type")
+    }
+
+    // Make sure spark.catalog.createTable with null type will fail
+    val schema1 = new StructType().add("c", NullType)
+    assertHiveTableNullType(schema1)
+    assertDSTableNullType(schema1)
+
+    val schema2 = new StructType()
+      .add("c", StructType(Seq(StructField.apply("c1", NullType))))
+    assertHiveTableNullType(schema2)
+    assertDSTableNullType(schema2)
+
+    val schema3 = new StructType().add("c", ArrayType(NullType))
+    assertHiveTableNullType(schema3)
+    assertDSTableNullType(schema3)
+
+    val schema4 = new StructType()
+      .add("c", MapType(StringType, NullType))
+    assertHiveTableNullType(schema4)
+    assertDSTableNullType(schema4)
+
+    val schema5 = new StructType()
+      .add("c", MapType(NullType, StringType))
+    assertHiveTableNullType(schema5)
+    assertDSTableNullType(schema5)
+  }
+
+  private def assertHiveTableNullType(schema: StructType): Unit = {
+    withTable("t") {
+      val e = intercept[AnalysisException] {
+        spark.catalog.createTable(
+          tableName = "t",
+          source = "hive",
+          schema = schema,
+          options = Map("fileFormat" -> "parquet"))
+      }.getMessage
+      assert(e.contains("Cannot create tables with null type"))
+    }
+  }
+
+  private def assertDSTableNullType(schema: StructType): Unit = {
+    withTable("t") {
+      val e = intercept[AnalysisException] {
+        spark.catalog.createTable(
+          tableName = "t",
+          source = "json",
+          schema = schema,
+          options = Map.empty[String, String])
+      }.getMessage
+      assert(e.contains("Cannot create tables with null type"))
+    }
+  }
+
   test("SPARK-21216: join with a streaming DataFrame") {
     import org.apache.spark.sql.execution.streaming.MemoryStream
     import testImplicits._
@@ -2370,7 +2566,7 @@ class HiveDDLSuite
         OrcFileOperator.getFileReader(maybeFile.get.toPath.toString).get.getCompression.name
 
       case "parquet" =>
-        val footer = ParquetFileReader.readFooter(
+        val footer = ParquetFooterReader.readFooter(
           sparkContext.hadoopConfiguration, new Path(maybeFile.get.getPath), NO_FILTER)
         footer.getBlocks.get(0).getColumns.get(0).getCodec.toString
     }
@@ -2489,9 +2685,9 @@ class HiveDDLSuite
   test("load command for non local invalid path validation") {
     withTable("tbl") {
       sql("CREATE TABLE tbl(i INT, j STRING) USING hive")
-      val e = intercept[AnalysisException](
-        sql("load data inpath '/doesnotexist.csv' into table tbl"))
-      assert(e.message.contains("LOAD DATA input path does not exist"))
+      assertAnalysisError(
+        "load data inpath '/doesnotexist.csv' into table tbl",
+        "LOAD DATA input path does not exist")
     }
   }
 
@@ -2570,8 +2766,7 @@ class HiveDDLSuite
            |AS SELECT 1 as a, "a" as b
                  """.stripMargin)
     }.getMessage
-    assert(err1.contains("Schema may not be specified in a Create Table As Select " +
-      "(CTAS) statement"))
+    assert(err1.contains("Schema may not be specified in a Create Table As Select"))
 
     val err2 = intercept[ParseException] {
       spark.sql(
@@ -2582,8 +2777,7 @@ class HiveDDLSuite
            |AS SELECT 1 as a, "a" as b
                  """.stripMargin)
     }.getMessage
-    assert(err2.contains("Create Partitioned Table As Select cannot specify data type for " +
-      "the partition columns of the target table"))
+    assert(err2.contains("Partition column types may not be specified in Create Table As Select"))
   }
 
   test("Hive CTAS with dynamic partition") {
@@ -2640,69 +2834,44 @@ class HiveDDLSuite
 
   test("Create Table LIKE with row format") {
     val catalog = spark.sessionState.catalog
-    withTable("sourceHiveTable", "sourceDsTable", "targetHiveTable1", "targetHiveTable2") {
+    withTable("sourceHiveTable", "sourceDsTable") {
       sql("CREATE TABLE sourceHiveTable(a INT, b INT) STORED AS PARQUET")
       sql("CREATE TABLE sourceDsTable(a INT, b INT) USING PARQUET")
 
       // row format doesn't work in create targetDsTable
-      var e = intercept[AnalysisException] {
-        spark.sql(
-          """
-            |CREATE TABLE targetDsTable LIKE sourceHiveTable USING PARQUET
-            |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
-            """.stripMargin)
-      }.getMessage
-      assert(e.contains("'ROW FORMAT' must be used with 'STORED AS'"))
+      assertAnalysisError(
+        """
+          |CREATE TABLE targetDsTable LIKE sourceHiveTable USING PARQUET
+          |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+        """.stripMargin,
+        "Operation not allowed: CREATE TABLE LIKE ... USING ... ROW FORMAT SERDE")
 
       // row format doesn't work with provider hive
-      e = intercept[AnalysisException] {
-        spark.sql(
-          """
-            |CREATE TABLE targetHiveTable LIKE sourceHiveTable USING hive
-            |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
-            |WITH SERDEPROPERTIES ('test' = 'test')
-          """.stripMargin)
-      }.getMessage
-      assert(e.contains("'ROW FORMAT' must be used with 'STORED AS'"))
+      assertAnalysisError(
+        """
+          |CREATE TABLE targetHiveTable LIKE sourceHiveTable USING hive
+          |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+          |WITH SERDEPROPERTIES ('test' = 'test')
+        """.stripMargin,
+        "Operation not allowed: CREATE TABLE LIKE ... USING ... ROW FORMAT SERDE")
 
       // row format doesn't work without 'STORED AS'
-      e = intercept[AnalysisException] {
-        spark.sql(
-          """
-            |CREATE TABLE targetDsTable LIKE sourceDsTable
-            |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
-            |WITH SERDEPROPERTIES ('test' = 'test')
-          """.stripMargin)
-      }.getMessage
-      assert(e.contains("'ROW FORMAT' must be used with 'STORED AS'"))
-
-      // row format works with STORED AS hive format (from hive table)
-      spark.sql(
+      assertAnalysisError(
         """
-          |CREATE TABLE targetHiveTable1 LIKE sourceHiveTable STORED AS PARQUET
+          |CREATE TABLE targetDsTable LIKE sourceDsTable
           |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
           |WITH SERDEPROPERTIES ('test' = 'test')
-          """.stripMargin)
-      var table = catalog.getTableMetadata(TableIdentifier("targetHiveTable1"))
-      assert(table.provider === Some("hive"))
-      assert(table.storage.inputFormat ===
-        Some("org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat"))
-      assert(table.storage.serde === Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"))
-      assert(table.storage.properties("test") == "test")
+        """.stripMargin,
+        "'ROW FORMAT' must be used with 'STORED AS'")
 
-      // row format works with STORED AS hive format (from datasource table)
-      spark.sql(
+      // 'INPUTFORMAT' and 'OUTPUTFORMAT' conflict with 'USING'
+      assertAnalysisError(
         """
-          |CREATE TABLE targetHiveTable2 LIKE sourceDsTable STORED AS PARQUET
+          |CREATE TABLE targetDsTable LIKE sourceDsTable USING format
+          |STORED AS INPUTFORMAT 'inFormat' OUTPUTFORMAT 'outFormat'
           |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
-          |WITH SERDEPROPERTIES ('test' = 'test')
-          """.stripMargin)
-      table = catalog.getTableMetadata(TableIdentifier("targetHiveTable2"))
-      assert(table.provider === Some("hive"))
-      assert(table.storage.inputFormat ===
-        Some("org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat"))
-      assert(table.storage.serde === Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"))
-      assert(table.storage.properties("test") == "test")
+        """.stripMargin,
+        "Operation not allowed: CREATE TABLE LIKE ... USING ... STORED AS")
     }
   }
 
@@ -2720,4 +2889,144 @@ class HiveDDLSuite
       checkAnswer(sql("SHOW PARTITIONS ta_part"), Row("ts=10") :: Nil)
     }
   }
+
+  test("SPARK-31904: Fix case sensitive problem of char and varchar partition columns") {
+    withTable("t1", "t2") {
+      sql("CREATE TABLE t1(a STRING, B VARCHAR(10), C CHAR(10)) STORED AS parquet")
+      sql("CREATE TABLE t2 USING parquet PARTITIONED BY (b, c) AS SELECT * FROM t1")
+      // make sure there is no exception
+      assert(sql("SELECT * FROM t2 WHERE b = 'A'").collect().isEmpty)
+      assert(sql("SELECT * FROM t2 WHERE c = 'A'").collect().isEmpty)
+    }
+  }
+
+  test("SPARK-33546: CREATE TABLE LIKE should validate row format & file format") {
+    val catalog = spark.sessionState.catalog
+    withTable("sourceHiveTable", "sourceDsTable") {
+      sql("CREATE TABLE sourceHiveTable(a INT, b INT) STORED AS PARQUET")
+      sql("CREATE TABLE sourceDsTable(a INT, b INT) USING PARQUET")
+
+      // ROW FORMAT SERDE ... STORED AS [SEQUENCEFILE | RCFILE | TEXTFILE]
+      val allowSerdeFileFormats = Seq("TEXTFILE", "SEQUENCEFILE", "RCFILE")
+      Seq("sourceHiveTable", "sourceDsTable").foreach { sourceTable =>
+        allowSerdeFileFormats.foreach { format =>
+          withTable("targetTable") {
+            spark.sql(
+              s"""
+                 |CREATE TABLE targetTable LIKE $sourceTable
+                 |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+                 |STORED AS $format
+             """.stripMargin)
+
+            val expectedSerde = HiveSerDe.sourceToSerDe(format)
+            val table = catalog.getTableMetadata(TableIdentifier("targetTable", Some("default")))
+            assert(table.provider === Some("hive"))
+            assert(table.storage.inputFormat === Some(expectedSerde.get.inputFormat.get))
+            assert(table.storage.outputFormat === Some(expectedSerde.get.outputFormat.get))
+            assert(table.storage.serde ===
+              Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"))
+          }
+        }
+
+        // negative case
+        hiveFormats.filterNot(allowSerdeFileFormats.contains(_)).foreach { format =>
+          withTable("targetTable") {
+            assertAnalysisError(
+              s"""
+                 |CREATE TABLE targetTable LIKE $sourceTable
+                 |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+                 |STORED AS $format
+              """.stripMargin,
+              s"ROW FORMAT SERDE is incompatible with format '${format.toLowerCase(Locale.ROOT)}'")
+          }
+        }
+      }
+
+      // ROW FORMAT DELIMITED ... STORED AS TEXTFILE
+      Seq("sourceHiveTable", "sourceDsTable").foreach { sourceTable =>
+        withTable("targetTable") {
+          spark.sql(
+            s"""
+               |CREATE TABLE targetTable LIKE $sourceTable
+               |ROW FORMAT DELIMITED
+               |STORED AS TEXTFILE
+             """.stripMargin)
+
+          val expectedSerde = HiveSerDe.sourceToSerDe("TEXTFILE")
+          val table = catalog.getTableMetadata(TableIdentifier("targetTable", Some("default")))
+          assert(table.provider === Some("hive"))
+          assert(table.storage.inputFormat === Some(expectedSerde.get.inputFormat.get))
+          assert(table.storage.outputFormat === Some(expectedSerde.get.outputFormat.get))
+          assert(table.storage.serde === Some(expectedSerde.get.serde.get))
+
+          // negative case
+          assertAnalysisError(
+            s"""
+               |CREATE TABLE targetTable LIKE $sourceTable
+               |ROW FORMAT DELIMITED
+               |STORED AS PARQUET
+            """.stripMargin,
+            "ROW FORMAT DELIMITED is only compatible with 'textfile'")
+        }
+      }
+
+      // ROW FORMAT ... STORED AS INPUTFORMAT ... OUTPUTFORMAT ...
+      hiveFormats.foreach { tableType =>
+        val expectedSerde = HiveSerDe.sourceToSerDe(tableType)
+        Seq("sourceHiveTable", "sourceDsTable").foreach { sourceTable =>
+          withTable("targetTable") {
+            spark.sql(
+              s"""
+                 |CREATE TABLE targetTable LIKE $sourceTable
+                 |ROW FORMAT SERDE '${expectedSerde.get.serde.get}'
+                 |STORED AS INPUTFORMAT '${expectedSerde.get.inputFormat.get}'
+                 |OUTPUTFORMAT '${expectedSerde.get.outputFormat.get}'
+               """.stripMargin)
+
+            val table = catalog.getTableMetadata(TableIdentifier("targetTable", Some("default")))
+            assert(table.provider === Some("hive"))
+            assert(table.storage.inputFormat === Some(expectedSerde.get.inputFormat.get))
+            assert(table.storage.outputFormat === Some(expectedSerde.get.outputFormat.get))
+            assert(table.storage.serde === Some(expectedSerde.get.serde.get))
+          }
+        }
+      }
+    }
+  }
+
+  test("SPARK-33844: Insert overwrite directory should check schema too") {
+    withView("v") {
+      spark.range(1).createTempView("v")
+      withTempPath { path =>
+        val e = intercept[AnalysisException] {
+          spark.sql(s"INSERT OVERWRITE LOCAL DIRECTORY '${path.getCanonicalPath}' " +
+            s"STORED AS PARQUET SELECT ID, if(1=1, 1, 0), abs(id), '^-' FROM v")
+        }.getMessage
+        assert(e.contains("Attribute name \"(IF((1 = 1), 1, 0))\" contains" +
+          " invalid character(s) among \" ,;{}()\\n\\t=\". Please use alias to rename it."))
+      }
+    }
+  }
+
+  test("SPARK-34261: Avoid side effect if create exists temporary function") {
+    withUserDefinedFunction("f1" -> true) {
+      sql("CREATE TEMPORARY FUNCTION f1 AS 'org.apache.hadoop.hive.ql.udf.UDFUUID'")
+
+      val jarName = "TestUDTF.jar"
+      val jar = spark.asInstanceOf[TestHiveSparkSession].getHiveFile(jarName).toURI.toString
+      spark.sparkContext.addedJars.keys.find(_.contains(jarName))
+        .foreach(spark.sparkContext.addedJars.remove)
+      assert(!spark.sparkContext.listJars().exists(_.contains(jarName)))
+      val msg = intercept[AnalysisException] {
+        sql("CREATE TEMPORARY FUNCTION f1 AS " +
+          s"'org.apache.hadoop.hive.ql.udf.UDFUUID' USING JAR '$jar'")
+      }.getMessage
+      assert(msg.contains("Function f1 already exists"))
+      assert(!spark.sparkContext.listJars().exists(_.contains(jarName)))
+
+      sql("CREATE OR REPLACE TEMPORARY FUNCTION f1 AS " +
+        s"'org.apache.hadoop.hive.ql.udf.UDFUUID' USING JAR '$jar'")
+      assert(spark.sparkContext.listJars().exists(_.contains(jarName)))
+    }
+  }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQueryFileTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQueryFileTest.scala
index bb4ce6d3aa3f1..192fff2b98879 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQueryFileTest.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQueryFileTest.scala
@@ -25,18 +25,18 @@ import org.apache.spark.sql.catalyst.util._
  * A framework for running the query tests that are listed as a set of text files.
  *
  * TestSuites that derive from this class must provide a map of testCaseName to testCaseFiles
- * that should be included. Additionally, there is support for whitelisting and blacklisting
+ * that should be included. Additionally, there is support for including and excluding
  * tests as development progresses.
  */
 abstract class HiveQueryFileTest extends HiveComparisonTest {
   /** A list of tests deemed out of scope and thus completely disregarded */
-  def blackList: Seq[String] = Nil
+  def excludeList: Seq[String] = Nil
 
   /**
-   * The set of tests that are believed to be working in catalyst. Tests not in whiteList
-   * blacklist are implicitly marked as ignored.
+   * The set of tests that are believed to be working in catalyst. Tests not in includeList or
+   * excludeList are implicitly marked as ignored.
    */
-  def whiteList: Seq[String] = ".*" :: Nil
+  def includeList: Seq[String] = ".*" :: Nil
 
   def testCases: Seq[(String, File)]
 
@@ -45,25 +45,34 @@ abstract class HiveQueryFileTest extends HiveComparisonTest {
     runOnlyDirectories.nonEmpty ||
     skipDirectories.nonEmpty
 
-  val whiteListProperty: String = "spark.hive.whitelist"
-  // Allow the whiteList to be overridden by a system property
-  val realWhiteList: Seq[String] =
-    Option(System.getProperty(whiteListProperty)).map(_.split(",").toSeq).getOrElse(whiteList)
+  val deprecatedIncludeListProperty: String = "spark.hive.whitelist"
+  val includeListProperty: String = "spark.hive.includelist"
+  if (System.getProperty(deprecatedIncludeListProperty) != null) {
+    logWarning(s"System property `$deprecatedIncludeListProperty` is deprecated; please update " +
+        s"to use new property: $includeListProperty")
+  }
+  // Allow the includeList to be overridden by a system property
+  val realIncludeList: Seq[String] =
+    Option(System.getProperty(includeListProperty))
+        .orElse(Option(System.getProperty(deprecatedIncludeListProperty)))
+        .map(_.split(",").toSeq)
+        .getOrElse(includeList)
 
   // Go through all the test cases and add them to scala test.
   testCases.sorted.foreach {
     case (testCaseName, testCaseFile) =>
-      if (blackList.map(_.r.pattern.matcher(testCaseName).matches()).reduceLeft(_||_)) {
-        logDebug(s"Blacklisted test skipped $testCaseName")
-      } else if (realWhiteList.map(_.r.pattern.matcher(testCaseName).matches()).reduceLeft(_||_) ||
+      if (excludeList.map(_.r.pattern.matcher(testCaseName).matches()).reduceLeft(_||_)) {
+        logDebug(s"Excluded test skipped $testCaseName")
+      } else if (
+        realIncludeList.map(_.r.pattern.matcher(testCaseName).matches()).reduceLeft(_||_) ||
         runAll) {
         // Build a test case and submit it to scala test framework...
         val queriesString = fileToString(testCaseFile)
         createQueryTest(testCaseName, queriesString, reset = true, tryWithoutResettingFirst = true)
       } else {
-        // Only output warnings for the built in whitelist as this clutters the output when the user
-        // trying to execute a single test from the commandline.
-        if (System.getProperty(whiteListProperty) == null && !runAll) {
+        // Only output warnings for the built in includeList as this clutters the output when the
+        // user is trying to execute a single test from the commandline.
+        if (System.getProperty(includeListProperty) == null && !runAll) {
           ignore(testCaseName) {}
         }
       }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
index 63b985fbe4d32..b08368f0fecb2 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.hive.execution
 
 import java.io.File
 import java.net.URI
+import java.nio.file.Files
 import java.sql.Timestamp
 import java.util.Locale
 
@@ -34,11 +35,12 @@ import org.apache.spark.sql.catalyst.expressions.Cast
 import org.apache.spark.sql.catalyst.parser.ParseException
 import org.apache.spark.sql.catalyst.plans.logical.Project
 import org.apache.spark.sql.execution.joins.BroadcastNestedLoopJoinExec
-import org.apache.spark.sql.hive._
+import org.apache.spark.sql.hive.HiveUtils.{builtinHiveVersion => hiveVersion}
 import org.apache.spark.sql.hive.test.{HiveTestJars, TestHive}
 import org.apache.spark.sql.hive.test.TestHive._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SQLTestUtils
+import org.apache.spark.tags.SlowHiveTest
 
 case class TestData(a: Int, b: String)
 
@@ -46,6 +48,7 @@ case class TestData(a: Int, b: String)
  * A set of test cases expressed in Hive QL that are not covered by the tests
  * included in the hive distribution.
  */
+@SlowHiveTest
 class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAndAfter {
   import org.apache.spark.sql.hive.test.TestHive.implicits._
 
@@ -201,14 +204,17 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
       |IF(TRUE, CAST(NULL AS BINARY), CAST("1" AS BINARY)) AS COL18,
       |IF(FALSE, CAST(NULL AS DATE), CAST("1970-01-01" AS DATE)) AS COL19,
       |IF(TRUE, CAST(NULL AS DATE), CAST("1970-01-01" AS DATE)) AS COL20,
-      |IF(TRUE, CAST(NULL AS TIMESTAMP), CAST(1 AS TIMESTAMP)) AS COL21,
+      |IF(TRUE, CAST(NULL AS TIMESTAMP), CAST('1969-12-31 16:00:01' AS TIMESTAMP)) AS COL21,
       |IF(FALSE, CAST(NULL AS DECIMAL), CAST(1 AS DECIMAL)) AS COL22,
       |IF(TRUE, CAST(NULL AS DECIMAL), CAST(1 AS DECIMAL)) AS COL23
       |FROM src LIMIT 1""".stripMargin)
 
   test("constant null testing timestamp") {
-    val r1 = sql("SELECT IF(FALSE, CAST(NULL AS TIMESTAMP), CAST(1 AS TIMESTAMP)) AS COL20")
-      .collect().head
+    val r1 = sql(
+      """
+        |SELECT IF(FALSE, CAST(NULL AS TIMESTAMP),
+        |CAST('1969-12-31 16:00:01' AS TIMESTAMP)) AS COL20
+      """.stripMargin).collect().head
     assert(new Timestamp(1000) == r1.getTimestamp(0))
   }
 
@@ -552,37 +558,31 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
 
   // Jdk version leads to different query output for double, so not use createQueryTest here
   test("timestamp cast #1") {
-    val res = sql("SELECT CAST(CAST(1 AS TIMESTAMP) AS DOUBLE) FROM src LIMIT 1").collect().head
+    val res = sql("SELECT CAST(TIMESTAMP_SECONDS(1) AS DOUBLE) FROM src LIMIT 1").collect().head
     assert(1 == res.getDouble(0))
   }
 
-  createQueryTest("timestamp cast #2",
-    "SELECT CAST(CAST(1.2 AS TIMESTAMP) AS DOUBLE) FROM src LIMIT 1")
-
-  test("timestamp cast #3") {
-    val res = sql("SELECT CAST(CAST(1200 AS TIMESTAMP) AS INT) FROM src LIMIT 1").collect().head
-    assert(1200 == res.getInt(0))
+  test("timestamp cast #2") {
+    val res = sql("SELECT CAST(TIMESTAMP_SECONDS(-1) AS DOUBLE) FROM src LIMIT 1").collect().head
+    assert(-1 == res.get(0))
   }
 
+  createQueryTest("timestamp cast #3",
+    "SELECT CAST(TIMESTAMP_SECONDS(1.2) AS DOUBLE) FROM src LIMIT 1")
+
   createQueryTest("timestamp cast #4",
-    "SELECT CAST(CAST(1.2 AS TIMESTAMP) AS DOUBLE) FROM src LIMIT 1")
+    "SELECT CAST(TIMESTAMP_SECONDS(-1.2) AS DOUBLE) FROM src LIMIT 1")
 
   test("timestamp cast #5") {
-    val res = sql("SELECT CAST(CAST(-1 AS TIMESTAMP) AS DOUBLE) FROM src LIMIT 1").collect().head
-    assert(-1 == res.get(0))
+    val res = sql("SELECT CAST(TIMESTAMP_SECONDS(1200) AS INT) FROM src LIMIT 1").collect().head
+    assert(1200 == res.getInt(0))
   }
 
-  createQueryTest("timestamp cast #6",
-    "SELECT CAST(CAST(-1.2 AS TIMESTAMP) AS DOUBLE) FROM src LIMIT 1")
-
-  test("timestamp cast #7") {
-    val res = sql("SELECT CAST(CAST(-1200 AS TIMESTAMP) AS INT) FROM src LIMIT 1").collect().head
+  test("timestamp cast #6") {
+    val res = sql("SELECT CAST(TIMESTAMP_SECONDS(-1200) AS INT) FROM src LIMIT 1").collect().head
     assert(-1200 == res.getInt(0))
   }
 
-  createQueryTest("timestamp cast #8",
-    "SELECT CAST(CAST(-1.2 AS TIMESTAMP) AS DOUBLE) FROM src LIMIT 1")
-
   createQueryTest("select null from table",
     "SELECT null FROM src LIMIT 1")
 
@@ -830,6 +830,18 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
     assert(sql(s"list jar $testJar").count() == 1)
   }
 
+  test("SPARK-34955: ADD JAR should treat paths which contains white spaces") {
+    withTempDir { dir =>
+      val file = File.createTempFile("someprefix1", "somesuffix1", dir)
+      Files.write(file.toPath, "test_file1".getBytes)
+      val jarFile = new File(dir, "test file.jar")
+      TestUtils.createJar(Seq(file), jarFile)
+      sql(s"ADD JAR '${jarFile.getAbsolutePath}'")
+      assert(sql("LIST JARS").
+        filter(_.getString(0).contains(s"${jarFile.getName}".replace(" ", "%20"))).count() > 0)
+    }
+  }
+
   test("CREATE TEMPORARY FUNCTION") {
     val funcJar = TestHive.getHiveFile("TestUDTF.jar")
     val jarURL = funcJar.toURI.toURL
@@ -858,6 +870,273 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
     assert(sql(s"list file $testFile").count() == 1)
   }
 
+  test("ADD ARCHIVE/LIST ARCHIVES commands") {
+    withTempDir { dir =>
+      val file1 = File.createTempFile("someprefix1", "somesuffix1", dir)
+      val file2 = File.createTempFile("someprefix2", "somesuffix2", dir)
+
+      Files.write(file1.toPath, "file1".getBytes)
+      Files.write(file2.toPath, "file2".getBytes)
+
+      val zipFile = new File(dir, "test.zip")
+      val jarFile = new File(dir, "test.jar")
+      TestUtils.createJar(Seq(file1), zipFile)
+      TestUtils.createJar(Seq(file2), jarFile)
+
+      sql(s"ADD ARCHIVE ${zipFile.getAbsolutePath}#foo")
+      sql(s"ADD ARCHIVE ${jarFile.getAbsolutePath}#bar")
+
+      val checkAddArchive =
+        sparkContext.parallelize(
+          Seq(
+            "foo",
+            s"foo/${file1.getName}",
+            "nonexistence",
+            "bar",
+            s"bar/${file2.getName}"), 1).map { name =>
+          val file = new File(SparkFiles.get(name))
+          val contents =
+            if (file.isFile) {
+              Some(String.join("", new String(Files.readAllBytes(file.toPath))))
+            } else {
+              None
+            }
+          (name, file.canRead, contents)
+        }.collect()
+
+      assert(checkAddArchive(0) === ("foo", true, None))
+      assert(checkAddArchive(1) === (s"foo/${file1.getName}", true, Some("file1")))
+      assert(checkAddArchive(2) === ("nonexistence", false, None))
+      assert(checkAddArchive(3) === ("bar", true, None))
+      assert(checkAddArchive(4) === (s"bar/${file2.getName}", true, Some("file2")))
+      assert(sql("list archives").
+        filter(_.getString(0).contains(s"${zipFile.getAbsolutePath}")).count() > 0)
+      assert(sql("list archive").
+        filter(_.getString(0).contains(s"${jarFile.getAbsolutePath}")).count() > 0)
+      assert(sql(s"list archive ${zipFile.getAbsolutePath}").count() === 1)
+      assert(sql(s"list archives ${zipFile.getAbsolutePath} nonexistence").count() === 1)
+      assert(sql(s"list archives ${zipFile.getAbsolutePath} " +
+        s"${jarFile.getAbsolutePath}").count === 2)
+    }
+  }
+
+  test("ADD ARCHIVE/List ARCHIVES commands - unsupported archive formats") {
+    withTempDir { dir =>
+      val file1 = File.createTempFile("someprefix1", "somesuffix1", dir)
+      val file2 = File.createTempFile("someprefix2", "somesuffix2", dir)
+
+      Files.write(file1.toPath, "file1".getBytes)
+      Files.write(file2.toPath, "file2".getBytes)
+
+      // Emulate unsupported archive formats with .bz2 and .xz suffix.
+      val bz2File = new File(dir, "test.bz2")
+      val xzFile = new File(dir, "test.xz")
+      TestUtils.createJar(Seq(file1), bz2File)
+      TestUtils.createJar(Seq(file2), xzFile)
+
+      sql(s"ADD ARCHIVE ${bz2File.getAbsolutePath}#foo")
+      sql(s"ADD ARCHIVE ${xzFile.getAbsolutePath}#bar")
+
+      val checkAddArchive =
+        sparkContext.parallelize(
+          Seq(
+            "foo",
+            "bar"), 1).map { name =>
+          val file = new File(SparkFiles.get(name))
+          val contents =
+            if (file.isFile) {
+              Some(Files.readAllBytes(file.toPath).toSeq)
+            } else {
+              None
+            }
+          (name, file.canRead, contents)
+        }.collect()
+
+      assert(checkAddArchive(0) === ("foo", true, Some(Files.readAllBytes(bz2File.toPath).toSeq)))
+      assert(checkAddArchive(1) === ("bar", true, Some(Files.readAllBytes(xzFile.toPath).toSeq)))
+      assert(sql("list archives").
+        filter(_.getString(0).contains(s"${bz2File.getAbsolutePath}")).count() > 0)
+      assert(sql("list archive").
+        filter(_.getString(0).contains(s"${xzFile.getAbsolutePath}")).count() > 0)
+      assert(sql(s"list archive ${bz2File.getAbsolutePath}").count() === 1)
+      assert(sql(s"list archives ${bz2File.getAbsolutePath} " +
+        s"${xzFile.getAbsolutePath}").count === 2)
+    }
+  }
+
+  test("SPARK-35105: ADD FILES command with multiple files") {
+    withTempDir { dir =>
+      val file1 = File.createTempFile("someprefix1", "somesuffix1", dir)
+      val file2 = File.createTempFile("someprefix2", "somesuffix 2", dir)
+      val file3 = File.createTempFile("someprefix3", "somesuffix 3", dir)
+      val file4 = File.createTempFile("someprefix4", "somesuffix4", dir)
+
+      Files.write(file1.toPath, "file1".getBytes)
+      Files.write(file2.toPath, "file2".getBytes)
+      Files.write(file3.toPath, "file3".getBytes)
+      Files.write(file4.toPath, "file3".getBytes)
+
+      sql(s"ADD FILE ${file1.getAbsolutePath} '${file2.getAbsoluteFile}'")
+      sql(s"""ADD FILES "${file3.getAbsolutePath}" ${file4.getAbsoluteFile}""")
+      val listFiles = sql(s"LIST FILES ${file1.getAbsolutePath} " +
+        s"'${file2.getAbsolutePath}' '${file3.getAbsolutePath}' ${file4.getAbsolutePath}")
+      assert(listFiles.count === 4)
+      assert(listFiles.filter(_.getString(0).contains(file1.getName)).count() === 1)
+      assert(listFiles.filter(
+        _.getString(0).contains(file2.getName.replace(" ", "%20"))).count() === 1)
+      assert(listFiles.filter(
+        _.getString(0).contains(file3.getName.replace(" ", "%20"))).count() === 1)
+      assert(listFiles.filter(_.getString(0).contains(file4.getName)).count() === 1)
+    }
+  }
+
+  test("SPARK-35105: ADD JARS command with multiple files") {
+    withTempDir { dir =>
+      val file1 = new File(dir, "test1.txt")
+      val file2 = new File(dir, "test2.txt")
+      val file3 = new File(dir, "test3.txt")
+      val file4 = new File(dir, "test4.txt")
+
+      Files.write(file1.toPath, "file1".getBytes)
+      Files.write(file2.toPath, "file2".getBytes)
+      Files.write(file3.toPath, "file3".getBytes)
+      Files.write(file4.toPath, "file4".getBytes)
+
+      val jarFile1 = File.createTempFile("someprefix1", "somesuffix 1", dir)
+      val jarFile2 = File.createTempFile("someprefix2", "somesuffix2", dir)
+      val jarFile3 = File.createTempFile("someprefix3", "somesuffix3", dir)
+      val jarFile4 = File.createTempFile("someprefix4", "somesuffix 4", dir)
+
+      TestUtils.createJar(Seq(file1), jarFile1)
+      TestUtils.createJar(Seq(file2), jarFile2)
+      TestUtils.createJar(Seq(file3), jarFile3)
+      TestUtils.createJar(Seq(file4), jarFile4)
+
+      sql(s"""ADD JAR "${jarFile1.getAbsolutePath}" ${jarFile2.getAbsoluteFile}""")
+      sql(s"ADD JARS ${jarFile3.getAbsolutePath} '${jarFile4.getAbsoluteFile}'")
+      val listFiles = sql(s"LIST JARS '${jarFile1.getAbsolutePath}' " +
+        s"${jarFile2.getAbsolutePath} ${jarFile3.getAbsolutePath} '${jarFile4.getAbsoluteFile}'")
+      assert(listFiles.count === 4)
+      assert(listFiles.filter(
+        _.getString(0).contains(jarFile1.getName.replace(" ", "%20"))).count() === 1)
+      assert(listFiles.filter(_.getString(0).contains(jarFile2.getName)).count() === 1)
+      assert(listFiles.filter(_.getString(0).contains(jarFile3.getName)).count() === 1)
+      assert(listFiles.filter(
+        _.getString(0).contains(jarFile4.getName.replace(" ", "%20"))).count() === 1)
+    }
+  }
+
+  test("SPARK-35105: ADD ARCHIVES command with multiple files") {
+    withTempDir { dir =>
+      val file1 = new File(dir, "test1.txt")
+      val file2 = new File(dir, "test2.txt")
+      val file3 = new File(dir, "test3.txt")
+      val file4 = new File(dir, "test4.txt")
+
+      Files.write(file1.toPath, "file1".getBytes)
+      Files.write(file2.toPath, "file2".getBytes)
+      Files.write(file3.toPath, "file3".getBytes)
+      Files.write(file4.toPath, "file4".getBytes)
+
+      val jarFile1 = File.createTempFile("someprefix1", "somesuffix1", dir)
+      val jarFile2 = File.createTempFile("someprefix2", "somesuffix 2", dir)
+      val jarFile3 = File.createTempFile("someprefix3", "somesuffix3", dir)
+      val jarFile4 = File.createTempFile("someprefix4", "somesuffix 4", dir)
+
+      TestUtils.createJar(Seq(file1), jarFile1)
+      TestUtils.createJar(Seq(file2), jarFile2)
+      TestUtils.createJar(Seq(file3), jarFile3)
+      TestUtils.createJar(Seq(file4), jarFile4)
+
+      sql(s"""ADD ARCHIVE ${jarFile1.getAbsolutePath} "${jarFile2.getAbsoluteFile}"""")
+      sql(s"ADD ARCHIVES ${jarFile3.getAbsolutePath} '${jarFile4.getAbsoluteFile}'")
+      val listFiles = sql(s"LIST ARCHIVES ${jarFile1.getAbsolutePath} " +
+        s"'${jarFile2.getAbsolutePath}' ${jarFile3.getAbsolutePath} '${jarFile4.getAbsolutePath}'")
+      assert(listFiles.count === 4)
+      assert(listFiles.filter(_.getString(0).contains(jarFile1.getName)).count() === 1)
+      assert(listFiles.filter(
+        _.getString(0).contains(jarFile2.getName.replace(" ", "%20"))).count() === 1)
+      assert(listFiles.filter(_.getString(0).contains(jarFile3.getName)).count() === 1)
+      assert(listFiles.filter(
+        _.getString(0).contains(jarFile4.getName.replace(" ", "%20"))).count() === 1)
+    }
+  }
+
+  test("SPARK-34977: LIST FILES/JARS/ARCHIVES should handle multiple quoted path arguments") {
+    withTempDir { dir =>
+      val file1 = File.createTempFile("someprefix1", "somesuffix1", dir)
+      val file2 = File.createTempFile("someprefix2", "somesuffix2", dir)
+      val file3 = File.createTempFile("someprefix3", "somesuffix 3", dir)
+
+      Files.write(file1.toPath, "file1".getBytes)
+      Files.write(file2.toPath, "file2".getBytes)
+      Files.write(file3.toPath, "file3".getBytes)
+
+      sql(s"ADD FILE ${file1.getAbsolutePath}")
+      sql(s"ADD FILE ${file2.getAbsolutePath}")
+      sql(s"ADD FILE '${file3.getAbsolutePath}'")
+      val listFiles = sql("LIST FILES " +
+        s"""'${file1.getAbsolutePath}' ${file2.getAbsolutePath} "${file3.getAbsolutePath}"""")
+
+      assert(listFiles.count === 3)
+      assert(listFiles.filter(_.getString(0).contains(file1.getName)).count() === 1)
+      assert(listFiles.filter(_.getString(0).contains(file2.getName)).count() === 1)
+      assert(listFiles.filter(
+        _.getString(0).contains(file3.getName.replace(" ", "%20"))).count() === 1)
+
+      val file4 = File.createTempFile("someprefix4", "somesuffix4", dir)
+      val file5 = File.createTempFile("someprefix5", "somesuffix5", dir)
+      val file6 = File.createTempFile("someprefix6", "somesuffix6", dir)
+      Files.write(file4.toPath, "file4".getBytes)
+      Files.write(file5.toPath, "file5".getBytes)
+      Files.write(file6.toPath, "file6".getBytes)
+
+      val jarFile1 = new File(dir, "test1.jar")
+      val jarFile2 = new File(dir, "test2.jar")
+      val jarFile3 = new File(dir, "test 3.jar")
+      TestUtils.createJar(Seq(file4), jarFile1)
+      TestUtils.createJar(Seq(file5), jarFile2)
+      TestUtils.createJar(Seq(file6), jarFile3)
+
+      sql(s"ADD ARCHIVE ${jarFile1.getAbsolutePath}")
+      sql(s"ADD ARCHIVE ${jarFile2.getAbsolutePath}#foo")
+      sql(s"ADD ARCHIVE '${jarFile3.getAbsolutePath}'")
+      val listArchives = sql(s"LIST ARCHIVES '${jarFile1.getAbsolutePath}' " +
+        s"""${jarFile2.getAbsolutePath} "${jarFile3.getAbsolutePath}"""")
+
+      assert(listArchives.count === 3)
+      assert(listArchives.filter(_.getString(0).contains(jarFile1.getName)).count() === 1)
+      assert(listArchives.filter(_.getString(0).contains(jarFile2.getName)).count() === 1)
+      assert(listArchives.filter(
+        _.getString(0).contains(jarFile3.getName.replace(" ", "%20"))).count() === 1)
+
+      val file7 = File.createTempFile("someprefix7", "somesuffix7", dir)
+      val file8 = File.createTempFile("someprefix8", "somesuffix8", dir)
+      val file9 = File.createTempFile("someprefix9", "somesuffix9", dir)
+      Files.write(file4.toPath, "file7".getBytes)
+      Files.write(file5.toPath, "file8".getBytes)
+      Files.write(file6.toPath, "file9".getBytes)
+
+      val jarFile4 = new File(dir, "test4.jar")
+      val jarFile5 = new File(dir, "test5.jar")
+      val jarFile6 = new File(dir, "test 6.jar")
+      TestUtils.createJar(Seq(file7), jarFile4)
+      TestUtils.createJar(Seq(file8), jarFile5)
+      TestUtils.createJar(Seq(file9), jarFile6)
+
+      sql(s"ADD JAR ${jarFile4.getAbsolutePath}")
+      sql(s"ADD JAR ${jarFile5.getAbsolutePath}")
+      sql(s"ADD JAR '${jarFile6.getAbsolutePath}'")
+      val listJars = sql(s"LIST JARS '${jarFile4.getAbsolutePath}' " +
+        s"""${jarFile5.getAbsolutePath} "${jarFile6.getAbsolutePath}"""")
+      assert(listJars.count === 3)
+      assert(listJars.filter(_.getString(0).contains(jarFile4.getName)).count() === 1)
+      assert(listJars.filter(_.getString(0).contains(jarFile5.getName)).count() === 1)
+      assert(listJars.filter(
+        _.getString(0).contains(jarFile6.getName.replace(" ", "%20"))).count() === 1)
+    }
+  }
+
   createQueryTest("dynamic_partition",
     """
       |DROP TABLE IF EXISTS dynamic_part_table;
@@ -1221,6 +1500,28 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
       }
     }
   }
+
+  test("SPARK-33084: Add jar support Ivy URI in SQL") {
+    val testData = TestHive.getHiveFile("data/files/sample.json").toURI
+    withTable("t") {
+      // hive-catalog-core has some transitive dependencies which dont exist on maven central
+      // and hence cannot be found in the test environment or are non-jar (.pom) which cause
+      // failures in tests. Use transitive=false as it should be good enough to test the Ivy
+      // support in Hive ADD JAR
+      sql(s"ADD JAR ivy://org.apache.hive.hcatalog:hive-hcatalog-core:$hiveVersion" +
+        "?transitive=false")
+      sql(
+        """CREATE TABLE t(a string, b string)
+          |ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'""".stripMargin)
+      sql(s"""LOAD DATA LOCAL INPATH "$testData" INTO TABLE t""")
+      sql("SELECT * FROM src JOIN t on src.key = t.a")
+      assert(sql("LIST JARS").filter(_.getString(0).contains(
+        s"org.apache.hive.hcatalog_hive-hcatalog-core-$hiveVersion.jar")).count() > 0)
+      assert(sql("LIST JAR").
+        filter(_.getString(0).contains(
+          s"org.apache.hive.hcatalog_hive-hcatalog-core-$hiveVersion.jar")).count() > 0)
+    }
+  }
 }
 
 // for SPARK-2180 test
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSQLViewSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSQLViewSuite.scala
index fa43ff14fd796..8aae7a1545b1a 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSQLViewSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSQLViewSuite.scala
@@ -17,12 +17,12 @@
 
 package org.apache.spark.sql.hive.execution
 
-import org.apache.spark.sql.{AnalysisException, Row, SaveMode, SparkSession}
+import org.apache.spark.sql.{AnalysisException, Row}
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType}
 import org.apache.spark.sql.execution.SQLViewSuite
-import org.apache.spark.sql.hive.test.{TestHive, TestHiveSingleton}
-import org.apache.spark.sql.types.StructType
+import org.apache.spark.sql.hive.test.TestHiveSingleton
+import org.apache.spark.sql.types.{NullType, StructType}
 
 /**
  * A test suite for Hive view related functionality.
@@ -137,4 +137,24 @@ class HiveSQLViewSuite extends SQLViewSuite with TestHiveSingleton {
       }
     }
   }
+
+  test("SPARK-20680: Add HiveVoidType to compatible with Hive void type") {
+    withView("v1") {
+      sql("create view v1 as select null as c")
+      val df = sql("select * from v1")
+      assert(df.schema.fields.head.dataType == NullType)
+      checkAnswer(
+        df,
+        Row(null)
+      )
+
+      sql("alter view v1 as select null as c1, 1 as c2")
+      val df2 = sql("select * from v1")
+      assert(df2.schema.fields.head.dataType == NullType)
+      checkAnswer(
+        df2,
+        Row(null, 1)
+      )
+    }
+  }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
new file mode 100644
index 0000000000000..71a745b180494
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
@@ -0,0 +1,574 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.execution
+
+import java.sql.Timestamp
+import java.time.{Duration, Period}
+import java.time.temporal.ChronoUnit
+
+import org.apache.hadoop.hive.serde2.`lazy`.LazySimpleSerDe
+import org.scalatest.exceptions.TestFailedException
+
+import org.apache.spark.{SparkException, TestUtils}
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
+import org.apache.spark.sql.catalyst.util.DateTimeConstants
+import org.apache.spark.sql.execution._
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql.hive.test.TestHiveSingleton
+import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.CalendarInterval
+
+class HiveScriptTransformationSuite extends BaseScriptTransformationSuite with TestHiveSingleton {
+  import testImplicits._
+
+  import ScriptTransformationIOSchema._
+
+  override protected def defaultSerDe(): String = "hive-serde"
+
+  override def createScriptTransformationExec(
+      script: String,
+      output: Seq[Attribute],
+      child: SparkPlan,
+      ioschema: ScriptTransformationIOSchema): BaseScriptTransformationExec = {
+    HiveScriptTransformationExec(
+      script = script,
+      output = output,
+      child = child,
+      ioschema = ioschema
+    )
+  }
+
+  private val hiveIOSchema: ScriptTransformationIOSchema = {
+    defaultIOSchema.copy(
+      inputSerdeClass = Some(classOf[LazySimpleSerDe].getCanonicalName),
+      outputSerdeClass = Some(classOf[LazySimpleSerDe].getCanonicalName)
+    )
+  }
+
+  test("cat with LazySimpleSerDe") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+
+    val rowsDf = Seq("a", "b", "c").map(Tuple1.apply).toDF("a")
+    checkAnswer(
+      rowsDf,
+      (child: SparkPlan) => createScriptTransformationExec(
+        script = "cat",
+        output = Seq(AttributeReference("a", StringType)()),
+        child = child,
+        ioschema = hiveIOSchema
+      ),
+      rowsDf.collect())
+    assert(uncaughtExceptionHandler.exception.isEmpty)
+  }
+
+  test("script transformation should not swallow errors from upstream operators (hive serde)") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+
+    val rowsDf = Seq("a", "b", "c").map(Tuple1.apply).toDF("a")
+    val e = intercept[TestFailedException] {
+      checkAnswer(
+        rowsDf,
+        (child: SparkPlan) => createScriptTransformationExec(
+          script = "cat",
+          output = Seq(AttributeReference("a", StringType)()),
+          child = ExceptionInjectingOperator(child),
+          ioschema = hiveIOSchema
+        ),
+        rowsDf.collect())
+    }
+    assert(e.getMessage().contains("intentional exception"))
+    // Before SPARK-25158, uncaughtExceptionHandler will catch IllegalArgumentException
+    assert(uncaughtExceptionHandler.exception.isEmpty)
+  }
+
+  test("SPARK-14400 script transformation should fail for bad script command (hive serde)") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+
+    val rowsDf = Seq("a", "b", "c").map(Tuple1.apply).toDF("a")
+
+    val e = intercept[SparkException] {
+      val plan =
+        createScriptTransformationExec(
+          script = "some_non_existent_command",
+          output = Seq(AttributeReference("a", StringType)()),
+          child = rowsDf.queryExecution.sparkPlan,
+          ioschema = hiveIOSchema)
+      SparkPlanTest.executePlan(plan, hiveContext)
+    }
+    assert(e.getMessage.contains("Subprocess exited with status"))
+    assert(uncaughtExceptionHandler.exception.isEmpty)
+  }
+
+  test("SPARK-24339 verify the result after pruning the unused columns (hive serde)") {
+    val rowsDf = Seq(
+      ("Bob", 16, 176),
+      ("Alice", 32, 164),
+      ("David", 60, 192),
+      ("Amy", 24, 180)
+    ).toDF("name", "age", "height")
+
+    checkAnswer(
+      rowsDf,
+      (child: SparkPlan) => createScriptTransformationExec(
+        script = "cat",
+        output = Seq(AttributeReference("name", StringType)()),
+        child = child,
+        ioschema = hiveIOSchema
+      ),
+      rowsDf.select("name").collect())
+    assert(uncaughtExceptionHandler.exception.isEmpty)
+  }
+
+  test("SPARK-30973: TRANSFORM should wait for the termination of the script (hive serde)") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+
+    val rowsDf = Seq("a", "b", "c").map(Tuple1.apply).toDF("a")
+    val e = intercept[SparkException] {
+      val plan =
+        createScriptTransformationExec(
+          script = "some_non_existent_command",
+          output = Seq(AttributeReference("a", StringType)()),
+          child = rowsDf.queryExecution.sparkPlan,
+          ioschema = hiveIOSchema)
+      SparkPlanTest.executePlan(plan, hiveContext)
+    }
+    assert(e.getMessage.contains("Subprocess exited with status"))
+    assert(uncaughtExceptionHandler.exception.isEmpty)
+  }
+
+  test("SPARK-32388: TRANSFORM should handle schema less correctly (hive serde)") {
+    withTempView("v") {
+      val df = Seq(
+        (1, "1", 1.0, BigDecimal(1.0), new Timestamp(1)),
+        (2, "2", 2.0, BigDecimal(2.0), new Timestamp(2)),
+        (3, "3", 3.0, BigDecimal(3.0), new Timestamp(3))
+      ).toDF("a", "b", "c", "d", "e") // Note column d's data type is Decimal(38, 18)
+      df.createTempView("v")
+
+      // In hive default serde mode, if we don't define output schema,
+      // when output column size > 2 and don't specify serde,
+      // it will choose take rest columns in second column as output schema
+      // (key: String, value: String)
+      checkAnswer(
+        sql(
+          s"""
+             |SELECT TRANSFORM(a, b, c, d, e)
+             |  USING 'cat'
+             |FROM v
+        """.stripMargin),
+        identity,
+        df.select(
+          'a.cast("string").as("key"),
+          concat_ws("\t",
+            'b.cast("string"),
+            'c.cast("string"),
+            'd.cast("string"),
+            'e.cast("string")).as("value")).collect())
+
+      // In hive default serde mode, if we don't define output schema,
+      // when output column size > 2 and just specify serde,
+      // it will choose take rest columns in second column as output schema
+      // (key: String, value: String)
+      checkAnswer(
+        sql(
+          s"""
+             |SELECT TRANSFORM(a, b, c, d, e)
+             |  ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+             |  WITH SERDEPROPERTIES (
+             |    'field.delim' = '\t'
+             |  )
+             |  USING 'cat'
+             |  ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+             |  WITH SERDEPROPERTIES (
+             |    'field.delim' = '\t'
+             |  )
+             |FROM v
+        """.stripMargin),
+        identity,
+        df.select(
+          'a.cast("string").as("key"),
+          'b.cast("string").as("value")).collect())
+
+
+      // In hive default serde mode, if we don't define output schema,
+      // when output column size > 2 and specify serde with
+      // 'serialization.last.column.takes.rest=true',
+      // it will choose take rest columns in second column as output schema
+      // (key: String, value: String)
+      checkAnswer(
+        sql(
+          s"""
+             |SELECT TRANSFORM(a, b, c, d, e)
+             |  ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+             |  WITH SERDEPROPERTIES (
+             |    'field.delim' = '\t',
+             |    'serialization.last.column.takes.rest' = 'true'
+             |  )
+             |  USING 'cat'
+             |  ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+             |  WITH SERDEPROPERTIES (
+             |    'field.delim' = '\t',
+             |    'serialization.last.column.takes.rest' = 'true'
+             |  )
+             |FROM v
+        """.stripMargin),
+        identity,
+        df.select(
+          'a.cast("string").as("key"),
+          concat_ws("\t",
+            'b.cast("string"),
+            'c.cast("string"),
+            'd.cast("string"),
+            'e.cast("string")).as("value")).collect())
+
+      // In hive default serde mode, if we don't define output schema,
+      // when output column size > 2 and specify serde
+      // with 'serialization.last.column.takes.rest=false',
+      // it will choose first two column as output schema (key: String, value: String)
+      checkAnswer(
+        sql(
+          s"""
+             |SELECT TRANSFORM(a, b, c, d, e)
+             |  ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+             |  WITH SERDEPROPERTIES (
+             |    'field.delim' = '\t',
+             |    'serialization.last.column.takes.rest' = 'false'
+             |  )
+             |  USING 'cat'
+             |  ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+             |  WITH SERDEPROPERTIES (
+             |    'field.delim' = '\t',
+             |    'serialization.last.column.takes.rest' = 'false'
+             |  )
+             |FROM v
+        """.stripMargin),
+        identity,
+        df.select(
+          'a.cast("string").as("key"),
+          'b.cast("string").as("value")).collect())
+
+      // In hive default serde mode, if we don't define output schema,
+      // when output column size = 2 and specify serde, it will these two column as
+      // output schema (key: String, value: String)
+      checkAnswer(
+        sql(
+          s"""
+             |SELECT TRANSFORM(a, b)
+             |  ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+             |  WITH SERDEPROPERTIES (
+             |    'field.delim' = '\t',
+             |    'serialization.last.column.takes.rest' = 'true'
+             |  )
+             |  USING 'cat'
+             |  ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+             |  WITH SERDEPROPERTIES (
+             |    'field.delim' = '\t',
+             |    'serialization.last.column.takes.rest' = 'true'
+             |  )
+             |FROM v
+        """.stripMargin),
+        identity,
+        df.select(
+          'a.cast("string").as("key"),
+          'b.cast("string").as("value")).collect())
+
+      // In hive default serde mode, if we don't define output schema,
+      // when output column size < 2 and specify serde, it will return null for deficiency
+      // output schema (key: String, value: String)
+      checkAnswer(
+        sql(
+          s"""
+             |SELECT TRANSFORM(a)
+             |  ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+             |  WITH SERDEPROPERTIES (
+             |    'field.delim' = '\t',
+             |    'serialization.last.column.takes.rest' = 'true'
+             |  )
+             |  USING 'cat'
+             |  ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+             |  WITH SERDEPROPERTIES (
+             |    'field.delim' = '\t',
+             |    'serialization.last.column.takes.rest' = 'true'
+             |  )
+             |FROM v
+        """.stripMargin),
+        identity,
+        df.select(
+          'a.cast("string").as("key"),
+          lit(null)).collect())
+    }
+  }
+
+  testBasicInputDataTypesWith(hiveIOSchema, "hive serde")
+
+  test("SPARK-32400: TRANSFORM supports complex data types type (hive serde)") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+    withTempView("v") {
+      val df = Seq(
+        (1, "1", Array(0, 1, 2), Map("a" -> 1)),
+        (2, "2", Array(3, 4, 5), Map("b" -> 2))
+      ).toDF("a", "b", "c", "d")
+        .select('a, 'b, 'c, 'd, struct('a, 'b).as("e"))
+      df.createTempView("v")
+
+      // Hive serde support ArrayType/MapType/StructType as input and output data type
+      checkAnswer(
+        df.select('c, 'd, 'e),
+        (child: SparkPlan) => createScriptTransformationExec(
+          script = "cat",
+          output = Seq(
+            AttributeReference("c", ArrayType(IntegerType))(),
+            AttributeReference("d", MapType(StringType, IntegerType))(),
+            AttributeReference("e", StructType(
+              Seq(
+                StructField("col1", IntegerType, false),
+                StructField("col2", StringType, true))))()),
+          child = child,
+          ioschema = hiveIOSchema
+        ),
+        df.select('c, 'd, 'e).collect())
+    }
+  }
+
+  test("SPARK-32400: TRANSFORM supports complex data types end to end (hive serde)") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+    withTempView("v") {
+      val df = Seq(
+        (1, "1", Array(0, 1, 2), Map("a" -> 1)),
+        (2, "2", Array(3, 4, 5), Map("b" -> 2))
+      ).toDF("a", "b", "c", "d")
+        .select('a, 'b, 'c, 'd, struct('a, 'b).as("e"))
+      df.createTempView("v")
+
+      // Hive serde support ArrayType/MapType/StructType as input and output data type
+      val query = sql(
+        """
+          |SELECT TRANSFORM (c, d, e)
+          |USING 'cat' AS (c array<int>, d map<string, int>, e struct<col1:int, col2:string>)
+          |FROM v
+        """.stripMargin)
+      checkAnswer(query, identity, df.select('c, 'd, 'e).collect())
+    }
+  }
+
+  test("SPARK-32400: TRANSFORM doesn't support CalendarIntervalType/UserDefinedType (hive serde)") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+    withTempView("v") {
+      val df = Seq(
+        (1, new CalendarInterval(7, 1, 1000), new TestUDT.MyDenseVector(Array(1, 2, 3))),
+        (1, new CalendarInterval(7, 1, 1000), new TestUDT.MyDenseVector(Array(1, 2, 3)))
+      ).toDF("a", "b", "c")
+      df.createTempView("v")
+
+      val e1 = intercept[SparkException] {
+        val plan = createScriptTransformationExec(
+          script = "cat",
+          output = Seq(
+            AttributeReference("a", IntegerType)(),
+            AttributeReference("b", CalendarIntervalType)()),
+          child = df.select('a, 'b).queryExecution.sparkPlan,
+          ioschema = hiveIOSchema)
+        SparkPlanTest.executePlan(plan, hiveContext)
+      }.getMessage
+      assert(e1.contains("interval cannot be converted to Hive TypeInfo"))
+
+      val e2 = intercept[SparkException] {
+        val plan = createScriptTransformationExec(
+          script = "cat",
+          output = Seq(
+            AttributeReference("a", IntegerType)(),
+            AttributeReference("c", new TestUDT.MyDenseVectorUDT)()),
+          child = df.select('a, 'c).queryExecution.sparkPlan,
+          ioschema = hiveIOSchema)
+        SparkPlanTest.executePlan(plan, hiveContext)
+      }.getMessage
+      assert(e2.contains("array<double> cannot be converted to Hive TypeInfo"))
+    }
+  }
+
+  test("SPARK-32400: TRANSFORM doesn't support" +
+    " CalendarIntervalType/UserDefinedType end to end (hive serde)") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+    withTempView("v") {
+      val df = Seq(
+        (1, new CalendarInterval(7, 1, 1000), new TestUDT.MyDenseVector(Array(1, 2, 3))),
+        (1, new CalendarInterval(7, 1, 1000), new TestUDT.MyDenseVector(Array(1, 2, 3)))
+      ).toDF("a", "b", "c")
+      df.createTempView("v")
+
+      val e1 = intercept[SparkException] {
+        sql(
+          """
+            |SELECT TRANSFORM(a, b) USING 'cat' AS (a, b)
+            |FROM v
+          """.stripMargin).collect()
+      }.getMessage
+      assert(e1.contains("interval cannot be converted to Hive TypeInfo"))
+
+      val e2 = intercept[SparkException] {
+        sql(
+          """
+            |SELECT TRANSFORM(a, c) USING 'cat' AS (a, c)
+            |FROM v
+          """.stripMargin).collect()
+      }.getMessage
+      assert(e2.contains("array<double> cannot be converted to Hive TypeInfo"))
+    }
+  }
+
+  test("SPARK-32685: When use specified serde, filed.delim's default value is '\t'") {
+    val query1 = sql(
+      """
+        |SELECT split(value, "\t") FROM (
+        |SELECT TRANSFORM(a, b, c)
+        |USING 'cat'
+        |FROM (SELECT 1 AS a, 2 AS b, 3 AS c) t
+        |) temp;
+      """.stripMargin)
+    checkAnswer(query1, identity, Row(Seq("2", "3")) :: Nil)
+
+    val query2 = sql(
+      """
+        |SELECT split(value, "\t") FROM (
+        |SELECT TRANSFORM(a, b, c)
+        |  ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+        |USING 'cat'
+        |  ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+        |  WITH SERDEPROPERTIES (
+        |   'serialization.last.column.takes.rest' = 'true'
+        |  )
+        |FROM (SELECT 1 AS a, 2 AS b, 3 AS c) t
+        |) temp;
+      """.stripMargin)
+    checkAnswer(query2, identity, Row(Seq("2", "3")) :: Nil)
+
+    val query3 = sql(
+      """
+        |SELECT split(value, "&") FROM (
+        |SELECT TRANSFORM(a, b, c)
+        |  ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+        |  WITH SERDEPROPERTIES (
+        |   'field.delim' = '&'
+        |  )
+        |USING 'cat'
+        |  ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+        |  WITH SERDEPROPERTIES (
+        |   'serialization.last.column.takes.rest' = 'true',
+        |   'field.delim' = '&'
+        |  )
+        |FROM (SELECT 1 AS a, 2 AS b, 3 AS c) t
+        |) temp;
+      """.stripMargin)
+    checkAnswer(query3, identity, Row(Seq("2", "3")) :: Nil)
+
+    val query4 = sql(
+      """
+        |SELECT split(value, "&") FROM (
+        |SELECT TRANSFORM(a, b, c)
+        |  ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+        |USING 'cat'
+        |  ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+        |  WITH SERDEPROPERTIES (
+        |   'serialization.last.column.takes.rest' = 'true',
+        |   'field.delim' = '&'
+        |  )
+        |FROM (SELECT 1 AS a, 2 AS b, 3 AS c) t
+        |) temp;
+      """.stripMargin)
+    checkAnswer(query4, identity, Row(null) :: Nil)
+  }
+
+  test("SPARK-32684: Script transform hive serde mode null format is same with hive as '\\N'") {
+    val query1 = sql(
+      """
+        |SELECT TRANSFORM(null, null, null)
+        |USING 'cat'
+        |FROM (SELECT 1 AS a) t
+      """.stripMargin)
+    checkAnswer(query1, identity, Row(null, "\\N\t\\N") :: Nil)
+
+    val query2 = sql(
+      """
+        |SELECT TRANSFORM(null, null, null)
+        |  ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+        |  WITH SERDEPROPERTIES (
+        |   'field.delim' = ','
+        |  )
+        |USING 'cat' AS (a)
+        |  ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+        |  WITH SERDEPROPERTIES (
+        |   'field.delim' = '&'
+        |  )
+        |FROM (SELECT 1 AS a) t
+      """.stripMargin)
+    checkAnswer(query2, identity, Row("\\N,\\N,\\N") :: Nil)
+
+  }
+
+  test("SPARK-34879: HiveInspectors supports DayTimeIntervalType and YearMonthIntervalType") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+    withTempView("v") {
+      val df = Seq(
+        (Duration.ofDays(1),
+          Duration.ofSeconds(100).plusNanos(123456),
+          Duration.of(Long.MaxValue, ChronoUnit.MICROS),
+          Period.ofMonths(10)),
+        (Duration.ofDays(1),
+          Duration.ofSeconds(100).plusNanos(1123456789),
+          Duration.ofSeconds(Long.MaxValue / DateTimeConstants.MICROS_PER_SECOND),
+          Period.ofMonths(10))
+      ).toDF("a", "b", "c", "d")
+      df.createTempView("v")
+
+      // Hive serde supports DayTimeIntervalType/YearMonthIntervalType as input and output data type
+      checkAnswer(
+        df,
+        (child: SparkPlan) => createScriptTransformationExec(
+          script = "cat",
+          output = Seq(
+            AttributeReference("a", DayTimeIntervalType)(),
+            AttributeReference("b", DayTimeIntervalType)(),
+            AttributeReference("c", DayTimeIntervalType)(),
+            AttributeReference("d", YearMonthIntervalType)()),
+          child = child,
+          ioschema = hiveIOSchema),
+        df.select($"a", $"b", $"c", $"d").collect())
+    }
+  }
+
+  test("SPARK-34879: HiveInspectors throw overflow when" +
+    " HiveIntervalDayTime overflow then DayTimeIntervalType") {
+    withTempView("v") {
+      val df = Seq(("579025220 15:30:06.000001000")).toDF("a")
+      df.createTempView("v")
+
+      val e = intercept[Exception] {
+        checkAnswer(
+          df,
+          (child: SparkPlan) => createScriptTransformationExec(
+            script = "cat",
+            output = Seq(AttributeReference("a", DayTimeIntervalType)()),
+            child = child,
+            ioschema = hiveIOSchema),
+          df.select($"a").collect())
+      }.getMessage
+      assert(e.contains("java.lang.ArithmeticException: long overflow"))
+    }
+  }
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeReadWriteSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeReadWriteSuite.scala
index f8ba7bf2c1a62..aae49f70ca93f 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeReadWriteSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeReadWriteSuite.scala
@@ -135,11 +135,12 @@ class HiveSerDeReadWriteSuite extends QueryTest with SQLTestUtils with TestHiveS
     }
     // MAP<primitive_type, data_type>
     withTable("hive_serde") {
-      hiveClient.runSqlHive(s"CREATE TABLE hive_serde (c1 MAP <INT, STRING>) STORED AS $fileFormat")
-      hiveClient.runSqlHive("INSERT INTO TABLE hive_serde SELECT MAP(1, 'a') FROM (SELECT 1) t")
-      checkAnswer(spark.table("hive_serde"), Row(Map(1 -> "a")))
-      spark.sql("INSERT INTO TABLE hive_serde SELECT MAP(2, 'b')")
-      checkAnswer(spark.table("hive_serde"), Seq(Row(Map(1 -> "a")), Row(Map(2 -> "b"))))
+      hiveClient.runSqlHive(
+        s"CREATE TABLE hive_serde (c1 MAP <STRING, STRING>) STORED AS $fileFormat")
+      hiveClient.runSqlHive("INSERT INTO TABLE hive_serde SELECT MAP('1', 'a') FROM (SELECT 1) t")
+      checkAnswer(spark.table("hive_serde"), Row(Map("1" -> "a")))
+      spark.sql("INSERT INTO TABLE hive_serde SELECT MAP('2', 'b')")
+      checkAnswer(spark.table("hive_serde"), Seq(Row(Map("1" -> "a")), Row(Map("2" -> "b"))))
     }
 
     // STRUCT<col_name : data_type [COMMENT col_comment], ...>
@@ -154,7 +155,7 @@ class HiveSerDeReadWriteSuite extends QueryTest with SQLTestUtils with TestHiveS
     }
   }
 
-  Seq("PARQUET", "ORC").foreach { fileFormat =>
+  Seq("SEQUENCEFILE", "TEXTFILE", "RCFILE", "ORC", "PARQUET", "AVRO").foreach { fileFormat =>
     test(s"Read/Write Hive $fileFormat serde table") {
       // Numeric Types
       checkNumericTypes(fileFormat, "TINYINT", 2)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala
index 24b1e3405379c..d7129bcb37e69 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala
@@ -21,11 +21,10 @@ import java.net.URI
 
 import org.scalatest.BeforeAndAfterAll
 
-import org.apache.spark.sql.{AnalysisException, SaveMode, SparkSession}
+import org.apache.spark.sql.{AnalysisException, SparkSession}
 import org.apache.spark.sql.catalyst.catalog.CatalogTable
 import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.execution.command.{CreateTableCommand, DDLUtils}
-import org.apache.spark.sql.execution.datasources.CreateTable
 import org.apache.spark.sql.execution.metric.InputOutputMetricsHelper
 import org.apache.spark.sql.hive.test.TestHive
 import org.apache.spark.sql.internal.{HiveSerDe, SQLConf}
@@ -71,8 +70,8 @@ class HiveSerDeSuite extends HiveComparisonTest with PlanTest with BeforeAndAfte
   }
 
   private def extractTableDesc(sql: String): (CatalogTable, Boolean) = {
-    TestHive.sessionState.sqlParser.parsePlan(sql).collect {
-      case CreateTable(tableDesc, mode, _) => (tableDesc, mode == SaveMode.Ignore)
+    TestHive.sessionState.analyzer.execute(TestHive.sessionState.sqlParser.parsePlan(sql)).collect {
+      case CreateTableCommand(tableDesc, ifNotExists) => (tableDesc, ifNotExists)
     }.head
   }
 
@@ -89,7 +88,7 @@ class HiveSerDeSuite extends HiveComparisonTest with PlanTest with BeforeAndAfte
   test("Test the default fileformat for Hive-serde tables") {
     withSQLConf("hive.default.fileformat" -> "orc") {
       val (desc, exists) = extractTableDesc(
-        "CREATE TABLE IF NOT EXISTS fileformat_test (id int)")
+        "CREATE TABLE IF NOT EXISTS fileformat_test (id int) USING hive")
       assert(exists)
       assert(desc.storage.inputFormat == Some("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"))
       assert(desc.storage.outputFormat == Some("org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat"))
@@ -97,7 +96,8 @@ class HiveSerDeSuite extends HiveComparisonTest with PlanTest with BeforeAndAfte
     }
 
     withSQLConf("hive.default.fileformat" -> "parquet") {
-      val (desc, exists) = extractTableDesc("CREATE TABLE IF NOT EXISTS fileformat_test (id int)")
+      val (desc, exists) = extractTableDesc(
+        "CREATE TABLE IF NOT EXISTS fileformat_test (id int) USING hive")
       assert(exists)
       val input = desc.storage.inputFormat
       val output = desc.storage.outputFormat
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala
index 67d7ed0841abb..5b43f82f253ea 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala
@@ -17,7 +17,10 @@
 
 package org.apache.spark.sql.hive.execution
 
+import java.io.{File, IOException}
+
 import org.apache.spark.sql.Row
+import org.apache.spark.sql.functions.col
 import org.apache.spark.sql.hive.test.{TestHive, TestHiveSingleton}
 import org.apache.spark.sql.hive.test.TestHive._
 import org.apache.spark.sql.hive.test.TestHive.implicits._
@@ -110,6 +113,7 @@ class HiveTableScanSuite extends HiveComparisonTest with SQLTestUtils with TestH
         sql(
           s"""
              |CREATE TABLE $table(id string)
+             |USING hive
              |PARTITIONED BY (p1 string,p2 string,p3 string,p4 string,p5 string)
            """.stripMargin)
         sql(
@@ -154,6 +158,7 @@ class HiveTableScanSuite extends HiveComparisonTest with SQLTestUtils with TestH
         sql(
           s"""
              |CREATE TABLE $table(id string)
+             |USING hive
              |PARTITIONED BY (p1 string,p2 string,p3 string,p4 string,p5 string)
            """.stripMargin)
         sql(
@@ -179,6 +184,7 @@ class HiveTableScanSuite extends HiveComparisonTest with SQLTestUtils with TestH
       sql(
         s"""
            |CREATE TABLE $table (id int)
+           |USING hive
            |PARTITIONED BY (a int, b int)
          """.stripMargin)
       val scan1 = getHiveTableScanExec(s"SELECT * FROM $table WHERE a = 1 AND b = 2")
@@ -187,6 +193,86 @@ class HiveTableScanSuite extends HiveComparisonTest with SQLTestUtils with TestH
     }
   }
 
+  test("SPARK-32867: When explain, HiveTableRelation show limited message") {
+    withSQLConf("hive.exec.dynamic.partition.mode" -> "nonstrict") {
+      withTable("df") {
+        spark.range(30)
+          .select(col("id"), col("id").as("k"))
+          .write
+          .partitionBy("k")
+          .format("hive")
+          .mode("overwrite")
+          .saveAsTable("df")
+
+        val scan1 = getHiveTableScanExec("SELECT * FROM df WHERE df.k < 3")
+        assert(scan1.simpleString(100).replaceAll("#\\d+L", "") ==
+          "Scan hive default.df [id, k]," +
+            " HiveTableRelation [" +
+            "`default`.`df`," +
+            " org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe," +
+            " Data Cols: [id]," +
+            " Partition Cols: [k]," +
+            " Pruned Partitions: [(k=0), (k=1), (k=2)]" +
+            "]," +
+            " [isnotnull(k), (k < 3)]")
+
+        val scan2 = getHiveTableScanExec("SELECT * FROM df WHERE df.k < 30")
+        assert(scan2.simpleString(100).replaceAll("#\\d+L", "") ==
+          "Scan hive default.df [id, k]," +
+            " HiveTableRelation [" +
+            "`default`.`df`," +
+            " org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe," +
+            " Data Cols: [id]," +
+            " Partition Cols: [k]," +
+            " Pruned Partitions: [(k=0), (k=1), (k=10), (k=11), (k=12), (k=13), (k=14), (k=15)," +
+            " (k=16), (k=17), (k=18), (k=19), (k..." +
+            "]," +
+            " [isnotnull(k), (k < 30)]")
+
+        sql(
+          """
+            |ALTER TABLE df PARTITION (k=10) SET SERDE
+            |'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe';
+          """.stripMargin)
+
+        val scan3 = getHiveTableScanExec("SELECT * FROM df WHERE df.k < 30")
+        assert(scan3.simpleString(100).replaceAll("#\\d+L", "") ==
+          "Scan hive default.df [id, k]," +
+            " HiveTableRelation [" +
+            "`default`.`df`," +
+            " org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe," +
+            " Data Cols: [id]," +
+            " Partition Cols: [k]," +
+            " Pruned Partitions: [(k=0), (k=1)," +
+            " (k=10, org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe)," +
+            " (k=11), (k=12), (k=1..." +
+            "]," +
+            " [isnotnull(k), (k < 30)]")
+      }
+    }
+  }
+
+  test("SPARK-32069: Improve error message on reading unexpected directory") {
+    withTable("t") {
+      withTempDir { f =>
+        sql(s"CREATE TABLE t(i LONG) USING hive LOCATION '${f.getAbsolutePath}'")
+        sql("INSERT INTO t VALUES(1)")
+        val dir = new File(f.getCanonicalPath + "/data")
+        dir.mkdir()
+        sql("set mapreduce.input.fileinputformat.input.dir.recursive=true")
+        assert(sql("select * from t").collect().head.getLong(0) == 1)
+        sql("set mapreduce.input.fileinputformat.input.dir.recursive=false")
+        val e = intercept[IOException] {
+          sql("SELECT * FROM t").collect()
+        }
+        assert(e.getMessage.contains(s"Path: ${dir.getAbsoluteFile} is a directory, " +
+          s"which is not supported by the record reader " +
+          s"when `mapreduce.input.fileinputformat.input.dir.recursive` is false."))
+        dir.delete()
+      }
+    }
+  }
+
   private def getHiveTableScanExec(query: String): HiveTableScanExec = {
     sql(query).queryExecution.sparkPlan.collectFirst {
       case p: HiveTableScanExec => p
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala
index 9e33a8ee4cc5c..ed44dcd8d7a29 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala
@@ -161,6 +161,20 @@ class HiveUDAFSuite extends QueryTest
       checkAnswer(sql("select histogram_numeric(a,2) from abc where a=3"), Row(null))
     }
   }
+
+  test("SPARK-32243: Spark UDAF Invalid arguments number error should throw earlier") {
+    // func need two arguments
+    val functionName = "longProductSum"
+    val functionClass = "org.apache.spark.sql.hive.execution.LongProductSum"
+    withUserDefinedFunction(functionName -> true) {
+      sql(s"CREATE TEMPORARY FUNCTION $functionName AS '$functionClass'")
+      val e = intercept[AnalysisException] {
+        sql(s"SELECT $functionName(100)")
+      }.getMessage
+      assert(e.contains(
+        s"Invalid number of arguments for function $functionName. Expected: 2; Found: 1;"))
+    }
+  }
 }
 
 /**
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
index 7bca2af379934..f98828760b410 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
@@ -20,6 +20,8 @@ package org.apache.spark.sql.hive.execution
 import java.io.{DataInput, DataOutput, File, PrintWriter}
 import java.util.{ArrayList, Arrays, Properties}
 
+import scala.collection.JavaConverters._
+
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.hive.ql.exec.UDF
 import org.apache.hadoop.hive.ql.udf.{UDAFPercentile, UDFType}
@@ -30,6 +32,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.{ObjectInspector, ObjectIns
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory
 import org.apache.hadoop.io.{LongWritable, Writable}
 
+import org.apache.spark.{SparkFiles, TestUtils}
 import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
 import org.apache.spark.sql.catalyst.plans.logical.Project
 import org.apache.spark.sql.execution.command.FunctionsCommand
@@ -434,8 +437,8 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton with SQLTestUtils {
     withTempView("tab1") {
       Seq(Tuple1(1451400761)).toDF("test_date").createOrReplaceTempView("tab1")
       sql(s"CREATE TEMPORARY FUNCTION testUDFToDate AS '${classOf[GenericUDFToDate].getName}'")
-      val count = sql("select testUDFToDate(cast(test_date as timestamp))" +
-        " from tab1 group by testUDFToDate(cast(test_date as timestamp))").count()
+      val count = sql("select testUDFToDate(timestamp_seconds(test_date))" +
+        " from tab1 group by testUDFToDate(timestamp_seconds(test_date))").count()
       sql("DROP TEMPORARY FUNCTION IF EXISTS testUDFToDate")
       assert(count == 1)
     }
@@ -658,6 +661,65 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton with SQLTestUtils {
     }
   }
 
+  test("SPARK-32877: add test for Hive UDF complex decimal type") {
+    withUserDefinedFunction("testArraySum" -> false) {
+      sql(s"CREATE FUNCTION testArraySum AS '${classOf[ArraySumUDF].getName}'")
+      checkAnswer(
+        sql("SELECT testArraySum(array(1, 1.1, 1.2))"),
+        Seq(Row(3.3)))
+
+      val msg = intercept[AnalysisException] {
+        sql("SELECT testArraySum(1)")
+      }.getMessage
+      assert(msg.contains(s"No handler for UDF/UDAF/UDTF '${classOf[ArraySumUDF].getName}'"))
+
+      val msg2 = intercept[AnalysisException] {
+        sql("SELECT testArraySum(1, 2)")
+      }.getMessage
+      assert(msg2.contains(s"No handler for UDF/UDAF/UDTF '${classOf[ArraySumUDF].getName}'"))
+    }
+  }
+
+  test("SPARK-35236: CREATE FUNCTION should take an archive in USING clause") {
+    withTempDir { dir =>
+      withUserDefinedFunction("testListFiles1" -> false) {
+        val text1 = File.createTempFile("test1_", ".txt", dir)
+        val json1 = File.createTempFile("test1_", ".json", dir)
+        val zipFile1 = File.createTempFile("test1_", ".zip", dir)
+        TestUtils.createJar(Seq(text1, json1), zipFile1)
+
+        sql(s"CREATE FUNCTION testListFiles1 AS '${classOf[ListFiles].getName}' " +
+          s"USING ARCHIVE '${zipFile1.getAbsolutePath}'")
+        val df1 = sql(s"SELECT testListFiles1('${SparkFiles.get(zipFile1.getName)}')")
+        val fileList1 =
+          df1.collect().map(_.getList[String](0)).head.asScala.filter(_ != "META-INF")
+
+        assert(fileList1.length === 2)
+        assert(fileList1.contains(text1.getName))
+        assert(fileList1.contains(json1.getName))
+      }
+
+      // Test for file#alias style archive registration.
+      withUserDefinedFunction("testListFiles2" -> false) {
+        val text2 = File.createTempFile("test2_", ".txt", dir)
+        val json2 = File.createTempFile("test2_", ".json", dir)
+        val csv2 = File.createTempFile("test2", ".csv", dir)
+        val zipFile2 = File.createTempFile("test2_", ".zip", dir)
+        TestUtils.createJar(Seq(text2, json2, csv2), zipFile2)
+
+        sql(s"CREATE FUNCTION testListFiles2 AS '${classOf[ListFiles].getName}' " +
+          s"USING ARCHIVE '${zipFile2.getAbsolutePath}#foo'")
+        val df2 = sql(s"SELECT testListFiles2('${SparkFiles.get("foo")}')")
+        val fileList2 =
+          df2.collect().map(_.getList[String](0)).head.asScala.filter(_ != "META-INF")
+
+        assert(fileList2.length === 3)
+        assert(fileList2.contains(text2.getName))
+        assert(fileList2.contains(json2.getName))
+        assert(fileList2.contains(csv2.getName))
+      }
+    }
+  }
 }
 
 class TestPair(x: Int, y: Int) extends Writable with Serializable {
@@ -741,3 +803,21 @@ class StatelessUDF extends UDF {
     result
   }
 }
+
+class ArraySumUDF extends UDF {
+  def evaluate(values: java.util.List[java.lang.Double]): java.lang.Double = {
+    var r = 0d
+    for (v <- values.asScala) {
+      r += v
+    }
+    r
+  }
+}
+
+class ListFiles extends UDF {
+  import java.util.{ArrayList, Arrays, List => JList}
+  def evaluate(path: String): JList[String] = {
+    val fileArray = new File(path).list()
+    if (fileArray != null) Arrays.asList(fileArray: _*) else new ArrayList[String]()
+  }
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/Hive_2_1_DDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/Hive_2_1_DDLSuite.scala
index b20ef035594da..6f37e39a532d6 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/Hive_2_1_DDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/Hive_2_1_DDLSuite.scala
@@ -27,13 +27,14 @@ import org.apache.spark.sql.hive.{HiveExternalCatalog, HiveUtils}
 import org.apache.spark.sql.hive.test.TestHiveSingleton
 import org.apache.spark.sql.internal.StaticSQLConf._
 import org.apache.spark.sql.types._
-import org.apache.spark.tags.ExtendedHiveTest
+import org.apache.spark.tags.{ExtendedHiveTest, SlowHiveTest}
 import org.apache.spark.util.Utils
 
 /**
  * A separate set of DDL tests that uses Hive 2.1 libraries, which behave a little differently
  * from the built-in ones.
  */
+@SlowHiveTest
 @ExtendedHiveTest
 class Hive_2_1_DDLSuite extends SparkFunSuite with TestHiveSingleton with BeforeAndAfterEach
   with BeforeAndAfterAll {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ObjectHashAggregateSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ObjectHashAggregateSuite.scala
index 327e4104d59a8..2bca25ed48d97 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ObjectHashAggregateSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ObjectHashAggregateSuite.scala
@@ -20,11 +20,9 @@ package org.apache.spark.sql.hive.execution
 import scala.util.Random
 
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFMax
-import org.scalatest.Matchers._
+import org.scalatest.matchers.must.Matchers._
 
 import org.apache.spark.sql._
-import org.apache.spark.sql.catalyst.FunctionIdentifier
-import org.apache.spark.sql.catalyst.analysis.UnresolvedFunction
 import org.apache.spark.sql.catalyst.expressions.{ExpressionEvalHelper, Literal}
 import org.apache.spark.sql.catalyst.expressions.aggregate.ApproximatePercentile
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
@@ -221,7 +219,7 @@ class ObjectHashAggregateSuite
     val withPartialSafe = max($"c2")
 
     // A Spark SQL native distinct aggregate function
-    val withDistinct = countDistinct($"c3")
+    val withDistinct = count_distinct($"c3")
 
     val allAggs = Seq(
       "typed" -> typed,
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala
index c9c36992906a8..ab37645b1c90c 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala
@@ -17,24 +17,24 @@
 
 package org.apache.spark.sql.hive.execution
 
-import org.scalatest.Matchers._
+import org.scalatest.matchers.should.Matchers._
 
-import org.apache.spark.sql.QueryTest
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project}
 import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.execution.{FileSourceScanExec, SparkPlan}
 import org.apache.spark.sql.execution.datasources.{CatalogFileIndex, HadoopFsRelation, LogicalRelation, PruneFileSourcePartitions}
 import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
 import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec
 import org.apache.spark.sql.functions.broadcast
-import org.apache.spark.sql.hive.test.TestHiveSingleton
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.test.SQLTestUtils
 import org.apache.spark.sql.types.StructType
 
-class PruneFileSourcePartitionsSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
+class PruneFileSourcePartitionsSuite extends PrunePartitionSuiteBase {
+
+  override def format: String = "parquet"
 
   object Optimize extends RuleExecutor[LogicalPlan] {
     val batches = Batch("PruneFileSourcePartitions", Once, PruneFileSourcePartitions) :: Nil
@@ -108,4 +108,10 @@ class PruneFileSourcePartitionsSuite extends QueryTest with SQLTestUtils with Te
       }
     }
   }
+
+  override def getScanExecPartitionSize(plan: SparkPlan): Long = {
+    plan.collectFirst {
+      case p: FileSourceScanExec => p
+    }.get.selectedPartitions.length
+  }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneHiveTablePartitionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneHiveTablePartitionsSuite.scala
index e41709841a736..677b250960960 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneHiveTablePartitionsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneHiveTablePartitionsSuite.scala
@@ -17,14 +17,16 @@
 
 package org.apache.spark.sql.hive.execution
 
-import org.apache.spark.sql.QueryTest
 import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.{ColumnStat, LogicalPlan}
 import org.apache.spark.sql.catalyst.rules.RuleExecutor
-import org.apache.spark.sql.hive.test.TestHiveSingleton
-import org.apache.spark.sql.test.SQLTestUtils
+import org.apache.spark.sql.execution.SparkPlan
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types.LongType
 
-class PruneHiveTablePartitionsSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
+class PruneHiveTablePartitionsSuite extends PrunePartitionSuiteBase {
+
+  override def format(): String = "hive"
 
   object Optimize extends RuleExecutor[LogicalPlan] {
     val batches =
@@ -32,7 +34,7 @@ class PruneHiveTablePartitionsSuite extends QueryTest with SQLTestUtils with Tes
         EliminateSubqueryAliases, new PruneHiveTablePartitions(spark)) :: Nil
   }
 
-  test("SPARK-15616 statistics pruned after going throuhg PruneHiveTablePartitions") {
+  test("SPARK-15616: statistics pruned after going through PruneHiveTablePartitions") {
     withTable("test", "temp") {
       sql(
         s"""
@@ -54,4 +56,84 @@ class PruneHiveTablePartitionsSuite extends QueryTest with SQLTestUtils with Tes
         Optimize.execute(analyzed2).stats.sizeInBytes)
     }
   }
+
+  test("Avoid generating too many predicates in partition pruning") {
+    withTempView("temp") {
+      withTable("t") {
+        sql(
+          s"""
+             |CREATE TABLE t(i INT, p0 INT, p1 INT)
+             |USING $format
+             |PARTITIONED BY (p0, p1)""".stripMargin)
+
+        spark.range(0, 10, 1).selectExpr("id as col")
+          .createOrReplaceTempView("temp")
+
+        for (part <- (0 to 25)) {
+          sql(
+            s"""
+               |INSERT OVERWRITE TABLE t PARTITION (p0='$part', p1='$part')
+               |SELECT col FROM temp""".stripMargin)
+        }
+        val scale = 20
+        val predicate = (1 to scale).map(i => s"(p0 = '$i' AND p1 = '$i')").mkString(" OR ")
+        val expectedStr = {
+          // left
+          "(((((((p0 = 1) && (p1 = 1)) || ((p0 = 2) && (p1 = 2))) ||" +
+          " ((p0 = 3) && (p1 = 3))) || (((p0 = 4) && (p1 = 4)) ||" +
+          " ((p0 = 5) && (p1 = 5)))) || (((((p0 = 6) && (p1 = 6)) ||" +
+          " ((p0 = 7) && (p1 = 7))) || ((p0 = 8) && (p1 = 8))) ||" +
+          " (((p0 = 9) && (p1 = 9)) || ((p0 = 10) && (p1 = 10))))) ||" +
+          // right
+          " ((((((p0 = 11) && (p1 = 11)) || ((p0 = 12) && (p1 = 12))) ||" +
+          " ((p0 = 13) && (p1 = 13))) || (((p0 = 14) && (p1 = 14)) ||" +
+          " ((p0 = 15) && (p1 = 15)))) || (((((p0 = 16) && (p1 = 16)) ||" +
+          " ((p0 = 17) && (p1 = 17))) || ((p0 = 18) && (p1 = 18))) ||" +
+          " (((p0 = 19) && (p1 = 19)) || ((p0 = 20) && (p1 = 20))))))"
+        }
+        assertPrunedPartitions(s"SELECT * FROM t WHERE $predicate", scale,
+          expectedStr)
+      }
+    }
+  }
+
+  test("SPARK-34119: Keep necessary stats after PruneHiveTablePartitions") {
+    withTable("SPARK_34119") {
+      withSQLConf(
+        SQLConf.CBO_ENABLED.key -> "true",
+        "hive.exec.dynamic.partition.mode" -> "nonstrict") {
+        sql(s"CREATE TABLE SPARK_34119 PARTITIONED BY (p) STORED AS textfile AS " +
+          "(SELECT id, CAST(id % 5 AS STRING) AS p FROM range(20))")
+        sql(s"ANALYZE TABLE SPARK_34119 COMPUTE STATISTICS FOR ALL COLUMNS")
+
+        checkOptimizedPlanStats(sql(s"SELECT id FROM SPARK_34119"),
+          320L,
+          Some(20),
+          Seq(ColumnStat(
+            distinctCount = Some(20),
+            min = Some(0),
+            max = Some(19),
+            nullCount = Some(0),
+            avgLen = Some(LongType.defaultSize),
+            maxLen = Some(LongType.defaultSize))))
+
+        checkOptimizedPlanStats(sql("SELECT id FROM SPARK_34119 WHERE p = '2'"),
+          64L,
+          Some(4),
+          Seq(ColumnStat(
+            distinctCount = Some(4),
+            min = Some(0),
+            max = Some(19),
+            nullCount = Some(0),
+            avgLen = Some(LongType.defaultSize),
+            maxLen = Some(LongType.defaultSize))))
+      }
+    }
+  }
+
+  override def getScanExecPartitionSize(plan: SparkPlan): Long = {
+    plan.collectFirst {
+      case p: HiveTableScanExec => p
+    }.get.prunedPartitions.size
+  }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PrunePartitionSuiteBase.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PrunePartitionSuiteBase.scala
new file mode 100644
index 0000000000000..3b13cee2d1fda
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PrunePartitionSuiteBase.scala
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.execution
+
+import org.apache.spark.sql.StatisticsCollectionTestBase
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BinaryOperator, Expression, IsNotNull, Literal}
+import org.apache.spark.sql.execution.{FileSourceScanExec, SparkPlan}
+import org.apache.spark.sql.hive.test.TestHiveSingleton
+import org.apache.spark.sql.internal.SQLConf.ADAPTIVE_EXECUTION_ENABLED
+
+abstract class PrunePartitionSuiteBase extends StatisticsCollectionTestBase with TestHiveSingleton {
+
+  protected def format: String
+
+  test("SPARK-28169: Convert scan predicate condition to CNF") {
+    withSQLConf(ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
+      withTempView("temp") {
+        withTable("t") {
+          sql(
+            s"""
+               |CREATE TABLE t(i INT, p STRING)
+               |USING $format
+               |PARTITIONED BY (p)""".stripMargin)
+
+          spark.range(0, 1000, 1).selectExpr("id as col")
+            .createOrReplaceTempView("temp")
+
+          for (part <- Seq(1, 2, 3, 4)) {
+            sql(
+              s"""
+                 |INSERT OVERWRITE TABLE t PARTITION (p='$part')
+                 |SELECT col FROM temp""".stripMargin)
+          }
+
+          assertPrunedPartitions(
+            "SELECT * FROM t WHERE p = '1' OR (p = '2' AND i = 1)", 2,
+            "((p = '1') || (p = '2'))")
+          assertPrunedPartitions(
+            "SELECT * FROM t WHERE (p = '1' AND i = 2) OR (i = 1 OR p = '2')", 4,
+            "")
+          assertPrunedPartitions(
+            "SELECT * FROM t WHERE (p = '1' AND i = 2) OR (p = '3' AND i = 3 )", 2,
+            "((p = '1') || (p = '3'))")
+          assertPrunedPartitions(
+            "SELECT * FROM t WHERE (p = '1' AND i = 2) OR (p = '2' OR p = '3')", 3,
+            "((p = '1') || ((p = '2') || (p = '3')))")
+          assertPrunedPartitions(
+            "SELECT * FROM t", 4,
+            "")
+          assertPrunedPartitions(
+            "SELECT * FROM t WHERE p = '1' AND i = 2", 1,
+            "(p = '1')")
+          assertPrunedPartitions(
+            """
+              |SELECT i, COUNT(1) FROM (
+              |SELECT * FROM t WHERE  p = '1' OR (p = '2' AND i = 1)
+              |) tmp GROUP BY i
+            """.stripMargin, 2, "((p = '1') || (p = '2'))")
+        }
+      }
+    }
+  }
+
+  private def getCleanStringRepresentation(exp: Expression): String = exp match {
+    case attr: AttributeReference =>
+      attr.sql.replaceAll("spark_catalog.default.t.", "")
+    case l: Literal =>
+      l.sql
+    case e: BinaryOperator =>
+      s"(${getCleanStringRepresentation(e.left)} ${e.symbol} " +
+        s"${getCleanStringRepresentation(e.right)})"
+  }
+
+  protected def assertPrunedPartitions(
+      query: String,
+      expectedPartitionCount: Long,
+      expectedPushedDownFilters: String): Unit = {
+    val qe = sql(query).queryExecution
+    val plan = qe.sparkPlan
+    assert(getScanExecPartitionSize(plan) == expectedPartitionCount)
+
+    val pushedDownPartitionFilters = qe.executedPlan.collectFirst {
+      case scan: FileSourceScanExec => scan.partitionFilters
+      case scan: HiveTableScanExec => scan.partitionPruningPred
+    }.map(exps => exps.filterNot(e => e.isInstanceOf[IsNotNull]))
+    val pushedFilters = pushedDownPartitionFilters.map(filters => {
+      filters.foldLeft("")((currentStr, exp) => {
+        if (currentStr == "") {
+          s"${getCleanStringRepresentation(exp)}"
+        } else {
+          s"$currentStr AND ${getCleanStringRepresentation(exp)}"
+        }
+      })
+    })
+
+    assert(pushedFilters == Some(expectedPushedDownFilters))
+  }
+
+  protected def getScanExecPartitionSize(plan: SparkPlan): Long
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLMetricsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLMetricsSuite.scala
index 4d6dafd598a2e..a2de43d737704 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLMetricsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLMetricsSuite.scala
@@ -18,7 +18,9 @@
 package org.apache.spark.sql.hive.execution
 
 import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecutionSuite
+import org.apache.spark.sql.execution.command.DataWritingCommandExec
 import org.apache.spark.sql.execution.metric.SQLMetricsTestUtils
+import org.apache.spark.sql.hive.HiveUtils
 import org.apache.spark.sql.hive.test.TestHiveSingleton
 
 // Disable AQE because metric info is different with AQE on/off
@@ -34,4 +36,29 @@ class SQLMetricsSuite extends SQLMetricsTestUtils with TestHiveSingleton
       testMetricsDynamicPartition("hive", "hive", "t1")
     }
   }
+
+  test("SPARK-34567: Add metrics for CTAS operator") {
+    Seq(false, true).foreach { canOptimized =>
+      withSQLConf(HiveUtils.CONVERT_METASTORE_CTAS.key -> canOptimized.toString) {
+        withTable("t") {
+          val df = sql(s"CREATE TABLE t STORED AS PARQUET AS SELECT 1 as a")
+          val dataWritingCommandExec =
+            df.queryExecution.executedPlan.asInstanceOf[DataWritingCommandExec]
+          dataWritingCommandExec.executeCollect()
+          val createTableAsSelect = dataWritingCommandExec.cmd
+          if (canOptimized) {
+            assert(createTableAsSelect.isInstanceOf[OptimizedCreateHiveTableAsSelectCommand])
+          } else {
+            assert(createTableAsSelect.isInstanceOf[CreateHiveTableAsSelectCommand])
+          }
+          assert(createTableAsSelect.metrics.contains("numFiles"))
+          assert(createTableAsSelect.metrics("numFiles").value == 1)
+          assert(createTableAsSelect.metrics.contains("numOutputBytes"))
+          assert(createTableAsSelect.metrics("numOutputBytes").value > 0)
+          assert(createTableAsSelect.metrics.contains("numOutputRows"))
+          assert(createTableAsSelect.metrics("numOutputRows").value == 1)
+        }
+      }
+    }
+  }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index 79c6ade2807d3..f3aad782cebc6 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -33,6 +33,7 @@ import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, Functio
 import org.apache.spark.sql.catalyst.catalog.{CatalogTableType, CatalogUtils, HiveTableRelation}
 import org.apache.spark.sql.catalyst.parser.ParseException
 import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, SubqueryAlias}
+import org.apache.spark.sql.execution.TestUncaughtExceptionHandler
 import org.apache.spark.sql.execution.adaptive.{DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
 import org.apache.spark.sql.execution.command.{FunctionsCommand, LoadDataCommand}
 import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation}
@@ -43,7 +44,7 @@ import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.StaticSQLConf.GLOBAL_TEMP_DATABASE
 import org.apache.spark.sql.test.SQLTestUtils
 import org.apache.spark.sql.types._
-import org.apache.spark.util.Utils
+import org.apache.spark.tags.SlowHiveTest
 
 case class Nested1(f1: Nested2)
 case class Nested2(f2: Nested3)
@@ -91,7 +92,8 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
   test("script") {
     withTempView("script_table") {
       assume(TestUtils.testCommandAvailable("/bin/bash"))
-      assume(TestUtils.testCommandAvailable("echo | sed"))
+      assume(TestUtils.testCommandAvailable("echo"))
+      assume(TestUtils.testCommandAvailable("sed"))
       val scriptFilePath = getTestResourcePath("test_script.sh")
       val df = Seq(("x1", "y1", "z1"), ("x2", "y2", "z2")).toDF("c1", "c2", "c3")
       df.createOrReplaceTempView("script_table")
@@ -226,7 +228,7 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
       checkAnswer(sql(s"SHOW functions $db.temp_abs"), Row("temp_abs"))
       checkAnswer(sql(s"SHOW functions `$db`.`temp_abs`"), Row("temp_abs"))
       checkAnswer(sql(s"SHOW functions `$db`.`temp_abs`"), Row("temp_abs"))
-      checkAnswer(sql("SHOW functions `a function doens't exist`"), Nil)
+      checkAnswer(sql("SHOW functions `a function doesn't exist`"), Nil)
       checkAnswer(sql("SHOW functions `temp_weekofyea*`"), Row("temp_weekofyear"))
 
       // this probably will failed if we add more function with `sha` prefixing.
@@ -710,8 +712,7 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
               |AS SELECT key, value FROM mytable1
             """.stripMargin)
         }.getMessage
-        assert(e.contains("Create Partitioned Table As Select cannot specify data type for " +
-          "the partition columns of the target table"))
+        assert(e.contains("Partition column types may not be specified in Create Table As Select"))
       }
     }
   }
@@ -767,14 +768,14 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
           sql("SELECT * FROM nested").collect().toSeq)
 
         intercept[AnalysisException] {
-          sql("CREATE TABLE test_ctas_1234 AS SELECT * from notexists").collect()
+          sql("CREATE TABLE test_ctas_1234 AS SELECT * from nonexistent").collect()
         }
       }
     }
   }
 
   test("test CTAS") {
-    withTable("test_ctas_1234") {
+    withTable("test_ctas_123") {
       sql("CREATE TABLE test_ctas_123 AS SELECT key, value FROM src")
       checkAnswer(
         sql("SELECT key, value FROM test_ctas_123 ORDER BY key"),
@@ -1172,7 +1173,7 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
 
   test("SPARK-6785: HiveQuerySuite - Date comparison test 2") {
     checkAnswer(
-      sql("SELECT CAST(CAST(0 AS timestamp) AS date) > CAST(0 AS timestamp) FROM src LIMIT 1"),
+      sql("SELECT CAST(timestamp_seconds(0) AS date) > timestamp_seconds(0) FROM src LIMIT 1"),
       Row(false))
   }
 
@@ -1182,10 +1183,10 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
       sql(
         """
           | SELECT
-          | CAST(CAST(0 AS timestamp) AS date),
-          | CAST(CAST(CAST(0 AS timestamp) AS date) AS string),
-          | CAST(0 AS timestamp),
-          | CAST(CAST(0 AS timestamp) AS string),
+          | CAST(timestamp_seconds(0) AS date),
+          | CAST(CAST(timestamp_seconds(0) AS date) AS string),
+          | timestamp_seconds(0),
+          | CAST(timestamp_seconds(0) AS string),
           | CAST(CAST(CAST('1970-01-01 23:00:00' AS timestamp) AS date) AS timestamp)
           | FROM src LIMIT 1
         """.stripMargin),
@@ -1738,12 +1739,12 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
            |SELECT 'blarr'
          """.stripMargin)
 
-      // project list is the same order of paritioning columns in table definition
+      // project list is the same order of partitioning columns in table definition
       checkAnswer(
         sql(s"SELECT p1, p2, p3, p4, p5, c1 FROM $table"),
         Row("a", "b", "c", "d", "e", "blarr") :: Nil)
 
-      // project list does not have the same order of paritioning columns in table definition
+      // project list does not have the same order of partitioning columns in table definition
       checkAnswer(
         sql(s"SELECT p2, p3, p4, p1, p5, c1 FROM $table"),
         Row("b", "c", "d", "a", "e", "blarr") :: Nil)
@@ -1968,7 +1969,7 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
       for (i <- 1 to 3) {
         Files.write(s"$i", new File(dirPath, s"part-r-0000$i"), StandardCharsets.UTF_8)
       }
-      withTable("load_t_folder_wildcard") {
+      withTable("load_t") {
         sql("CREATE TABLE load_t (a STRING) USING hive")
         sql(s"LOAD DATA LOCAL INPATH '${
           path.substring(0, path.length - 1)
@@ -2025,6 +2026,7 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
           sql(
             """
               |CREATE TABLE part_table (c STRING)
+              |STORED AS textfile
               |PARTITIONED BY (d STRING)
             """.stripMargin)
           sql(s"LOAD DATA LOCAL INPATH '$path/part-r-000011' " +
@@ -2158,32 +2160,6 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
     }
   }
 
-  test("SPARK-21101 UDTF should override initialize(ObjectInspector[] args)") {
-    withUserDefinedFunction("udtf_stack1" -> true, "udtf_stack2" -> true) {
-      sql(
-        s"""
-           |CREATE TEMPORARY FUNCTION udtf_stack1
-           |AS 'org.apache.spark.sql.hive.execution.UDTFStack'
-           |USING JAR '${hiveContext.getHiveFile("SPARK-21101-1.0.jar").toURI}'
-        """.stripMargin)
-      val cnt =
-        sql("SELECT udtf_stack1(2, 'A', 10, date '2015-01-01', 'B', 20, date '2016-01-01')").count()
-      assert(cnt === 2)
-
-      sql(
-        s"""
-           |CREATE TEMPORARY FUNCTION udtf_stack2
-           |AS 'org.apache.spark.sql.hive.execution.UDTFStack2'
-           |USING JAR '${hiveContext.getHiveFile("SPARK-21101-1.0.jar").toURI}'
-        """.stripMargin)
-      val e = intercept[org.apache.spark.sql.AnalysisException] {
-        sql("SELECT udtf_stack2(2, 'A', 10, date '2015-01-01', 'B', 20, date '2016-01-01')")
-      }
-      assert(
-        e.getMessage.contains("public StructObjectInspector initialize(ObjectInspector[] args)"))
-    }
-  }
-
   test("SPARK-21721: Clear FileSystem deleterOnExit cache if path is successfully removed") {
     val table = "test21721"
     withTable(table) {
@@ -2203,39 +2179,63 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
     }
   }
 
-  test("SPARK-21912 ORC/Parquet table should not create invalid column names") {
+  test("SPARK-21912 Parquet table should not create invalid column names") {
     Seq(" ", ",", ";", "{", "}", "(", ")", "\n", "\t", "=").foreach { name =>
-      Seq("ORC", "PARQUET").foreach { source =>
-        withTable("t21912") {
-          val m = intercept[AnalysisException] {
-            sql(s"CREATE TABLE t21912(`col$name` INT) USING $source")
-          }.getMessage
-          assert(m.contains(s"contains invalid character(s)"))
+      val source = "PARQUET"
+      withTable("t21912") {
+        val m = intercept[AnalysisException] {
+          sql(s"CREATE TABLE t21912(`col$name` INT) USING $source")
+        }.getMessage
+        assert(m.contains(s"contains invalid character(s)"))
 
-          val m1 = intercept[AnalysisException] {
-            sql(s"CREATE TABLE t21912 STORED AS $source AS SELECT 1 `col$name`")
-          }.getMessage
-          assert(m1.contains(s"contains invalid character(s)"))
+        val m1 = intercept[AnalysisException] {
+          sql(s"CREATE TABLE t21912 STORED AS $source AS SELECT 1 `col$name`")
+        }.getMessage
+        assert(m1.contains(s"contains invalid character(s)"))
+
+        val m2 = intercept[AnalysisException] {
+          sql(s"CREATE TABLE t21912 USING $source AS SELECT 1 `col$name`")
+        }.getMessage
+        assert(m2.contains(s"contains invalid character(s)"))
 
-          val m2 = intercept[AnalysisException] {
-            sql(s"CREATE TABLE t21912 USING $source AS SELECT 1 `col$name`")
+        withSQLConf(HiveUtils.CONVERT_METASTORE_PARQUET.key -> "false") {
+          val m3 = intercept[AnalysisException] {
+            sql(s"CREATE TABLE t21912(`col$name` INT) USING hive OPTIONS (fileFormat '$source')")
           }.getMessage
-          assert(m2.contains(s"contains invalid character(s)"))
+          assert(m3.contains(s"contains invalid character(s)"))
+        }
 
-          withSQLConf(HiveUtils.CONVERT_METASTORE_PARQUET.key -> "false") {
-            val m3 = intercept[AnalysisException] {
-              sql(s"CREATE TABLE t21912(`col$name` INT) USING hive OPTIONS (fileFormat '$source')")
-            }.getMessage
-            assert(m3.contains(s"contains invalid character(s)"))
-          }
+        sql(s"CREATE TABLE t21912(`col` INT) USING $source")
+        val m4 = intercept[AnalysisException] {
+          sql(s"ALTER TABLE t21912 ADD COLUMNS(`col$name` INT)")
+        }.getMessage
+        assert(m4.contains(s"contains invalid character(s)"))
+      }
+    }
+  }
 
-          sql(s"CREATE TABLE t21912(`col` INT) USING $source")
-          val m4 = intercept[AnalysisException] {
-            sql(s"ALTER TABLE t21912 ADD COLUMNS(`col$name` INT)")
-          }.getMessage
-          assert(m4.contains(s"contains invalid character(s)"))
+  test("SPARK-32889: ORC table column name supports special characters") {
+    // " " "," is not allowed.
+    Seq("$", ";", "{", "}", "(", ")", "\n", "\t", "=").foreach { name =>
+      val source = "ORC"
+      Seq(s"CREATE TABLE t32889(`$name` INT) USING $source",
+          s"CREATE TABLE t32889 STORED AS $source AS SELECT 1 `$name`",
+          s"CREATE TABLE t32889 USING $source AS SELECT 1 `$name`",
+          s"CREATE TABLE t32889(`$name` INT) USING hive OPTIONS (fileFormat '$source')")
+      .foreach { command =>
+        withTable("t32889") {
+          sql(command)
+          assertResult(name)(
+            sessionState.catalog.getTableMetadata(TableIdentifier("t32889")).schema.fields(0).name)
         }
       }
+
+      withTable("t32889") {
+        sql(s"CREATE TABLE t32889(`col` INT) USING $source")
+        sql(s"ALTER TABLE t32889 ADD COLUMNS(`$name` INT)")
+        assertResult(name)(
+          sessionState.catalog.getTableMetadata(TableIdentifier("t32889")).schema.fields(1).name)
+      }
     }
   }
 
@@ -2544,8 +2544,47 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
       assert(e.getMessage.contains("Cannot modify the value of a static config"))
     }
   }
+
+  test("SPARK-29295: dynamic partition map parsed from partition path should be case insensitive") {
+    withTable("t") {
+      withSQLConf("hive.exec.dynamic.partition" -> "true",
+        "hive.exec.dynamic.partition.mode" -> "nonstrict") {
+        withTempDir { loc =>
+          sql(s"CREATE TABLE t(c1 INT) PARTITIONED BY(P1 STRING) LOCATION '${loc.getAbsolutePath}'")
+          sql("INSERT OVERWRITE TABLE t PARTITION(P1) VALUES(1, 'caseSensitive')")
+          checkAnswer(sql("select * from t"), Row(1, "caseSensitive"))
+        }
+      }
+    }
+  }
+
+  test("SPARK-32668: HiveGenericUDTF initialize UDTF should use StructObjectInspector method") {
+    withUserDefinedFunction("udtf_stack1" -> true, "udtf_stack2" -> true) {
+      sql(
+        s"""
+           |CREATE TEMPORARY FUNCTION udtf_stack1
+           |AS 'org.apache.spark.sql.hive.execution.UDTFStack'
+           |USING JAR '${hiveContext.getHiveFile("SPARK-21101-1.0.jar").toURI}'
+        """.stripMargin)
+      sql(
+        s"""
+           |CREATE TEMPORARY FUNCTION udtf_stack2
+           |AS 'org.apache.spark.sql.hive.execution.UDTFStack2'
+           |USING JAR '${hiveContext.getHiveFile("SPARK-21101-1.0.jar").toURI}'
+        """.stripMargin)
+
+      Seq("udtf_stack1", "udtf_stack2").foreach { udf =>
+        checkAnswer(
+          sql(s"SELECT $udf(2, 'A', 10, date '2015-01-01', 'B', 20, date '2016-01-01')"),
+          Seq(Row("A", 10, Date.valueOf("2015-01-01")),
+            Row("B", 20, Date.valueOf("2016-01-01"))))
+      }
+    }
+  }
 }
 
+@SlowHiveTest
 class SQLQuerySuite extends SQLQuerySuiteBase with DisableAdaptiveExecutionSuite
+@SlowHiveTest
 class SQLQuerySuiteAE extends SQLQuerySuiteBase with EnableAdaptiveExecutionSuite
 
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala
deleted file mode 100644
index b97eb869a9e54..0000000000000
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala
+++ /dev/null
@@ -1,280 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.hive.execution
-
-import java.sql.Timestamp
-
-import org.apache.hadoop.hive.serde2.`lazy`.LazySimpleSerDe
-import org.scalatest.Assertions._
-import org.scalatest.BeforeAndAfterEach
-import org.scalatest.exceptions.TestFailedException
-
-import org.apache.spark.{SparkException, TaskContext, TestUtils}
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.Column
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
-import org.apache.spark.sql.catalyst.plans.physical.Partitioning
-import org.apache.spark.sql.execution.{SparkPlan, SparkPlanTest, UnaryExecNode}
-import org.apache.spark.sql.hive.HiveUtils
-import org.apache.spark.sql.hive.test.TestHiveSingleton
-import org.apache.spark.sql.test.SQLTestUtils
-import org.apache.spark.sql.types.StringType
-
-class ScriptTransformationSuite extends SparkPlanTest with SQLTestUtils with TestHiveSingleton
-  with BeforeAndAfterEach {
-  import spark.implicits._
-
-  private val noSerdeIOSchema = HiveScriptIOSchema(
-    inputRowFormat = Seq.empty,
-    outputRowFormat = Seq.empty,
-    inputSerdeClass = None,
-    outputSerdeClass = None,
-    inputSerdeProps = Seq.empty,
-    outputSerdeProps = Seq.empty,
-    recordReaderClass = None,
-    recordWriterClass = None,
-    schemaLess = false
-  )
-
-  private val serdeIOSchema = noSerdeIOSchema.copy(
-    inputSerdeClass = Some(classOf[LazySimpleSerDe].getCanonicalName),
-    outputSerdeClass = Some(classOf[LazySimpleSerDe].getCanonicalName)
-  )
-
-  private var defaultUncaughtExceptionHandler: Thread.UncaughtExceptionHandler = _
-
-  private val uncaughtExceptionHandler = new TestUncaughtExceptionHandler
-
-  protected override def beforeAll(): Unit = {
-    super.beforeAll()
-    defaultUncaughtExceptionHandler = Thread.getDefaultUncaughtExceptionHandler
-    Thread.setDefaultUncaughtExceptionHandler(uncaughtExceptionHandler)
-  }
-
-  protected override def afterAll(): Unit = {
-    super.afterAll()
-    Thread.setDefaultUncaughtExceptionHandler(defaultUncaughtExceptionHandler)
-  }
-
-  override protected def afterEach(): Unit = {
-    super.afterEach()
-    uncaughtExceptionHandler.cleanStatus()
-  }
-
-  test("cat without SerDe") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-
-    val rowsDf = Seq("a", "b", "c").map(Tuple1.apply).toDF("a")
-    checkAnswer(
-      rowsDf,
-      (child: SparkPlan) => new ScriptTransformationExec(
-        input = Seq(rowsDf.col("a").expr),
-        script = "cat",
-        output = Seq(AttributeReference("a", StringType)()),
-        child = child,
-        ioschema = noSerdeIOSchema
-      ),
-      rowsDf.collect())
-    assert(uncaughtExceptionHandler.exception.isEmpty)
-  }
-
-  test("cat with LazySimpleSerDe") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-
-    val rowsDf = Seq("a", "b", "c").map(Tuple1.apply).toDF("a")
-    checkAnswer(
-      rowsDf,
-      (child: SparkPlan) => new ScriptTransformationExec(
-        input = Seq(rowsDf.col("a").expr),
-        script = "cat",
-        output = Seq(AttributeReference("a", StringType)()),
-        child = child,
-        ioschema = serdeIOSchema
-      ),
-      rowsDf.collect())
-    assert(uncaughtExceptionHandler.exception.isEmpty)
-  }
-
-  test("script transformation should not swallow errors from upstream operators (no serde)") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-
-    val rowsDf = Seq("a", "b", "c").map(Tuple1.apply).toDF("a")
-    val e = intercept[TestFailedException] {
-      checkAnswer(
-        rowsDf,
-        (child: SparkPlan) => new ScriptTransformationExec(
-          input = Seq(rowsDf.col("a").expr),
-          script = "cat",
-          output = Seq(AttributeReference("a", StringType)()),
-          child = ExceptionInjectingOperator(child),
-          ioschema = noSerdeIOSchema
-        ),
-        rowsDf.collect())
-    }
-    assert(e.getMessage().contains("intentional exception"))
-    // Before SPARK-25158, uncaughtExceptionHandler will catch IllegalArgumentException
-    assert(uncaughtExceptionHandler.exception.isEmpty)
-  }
-
-  test("script transformation should not swallow errors from upstream operators (with serde)") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-
-    val rowsDf = Seq("a", "b", "c").map(Tuple1.apply).toDF("a")
-    val e = intercept[TestFailedException] {
-      checkAnswer(
-        rowsDf,
-        (child: SparkPlan) => new ScriptTransformationExec(
-          input = Seq(rowsDf.col("a").expr),
-          script = "cat",
-          output = Seq(AttributeReference("a", StringType)()),
-          child = ExceptionInjectingOperator(child),
-          ioschema = serdeIOSchema
-        ),
-        rowsDf.collect())
-    }
-    assert(e.getMessage().contains("intentional exception"))
-    // Before SPARK-25158, uncaughtExceptionHandler will catch IllegalArgumentException
-    assert(uncaughtExceptionHandler.exception.isEmpty)
-  }
-
-  test("SPARK-14400 script transformation should fail for bad script command") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-
-    val rowsDf = Seq("a", "b", "c").map(Tuple1.apply).toDF("a")
-
-    val e = intercept[SparkException] {
-      val plan =
-        new ScriptTransformationExec(
-          input = Seq(rowsDf.col("a").expr),
-          script = "some_non_existent_command",
-          output = Seq(AttributeReference("a", StringType)()),
-          child = rowsDf.queryExecution.sparkPlan,
-          ioschema = serdeIOSchema)
-      SparkPlanTest.executePlan(plan, hiveContext)
-    }
-    assert(e.getMessage.contains("Subprocess exited with status"))
-    assert(uncaughtExceptionHandler.exception.isEmpty)
-  }
-
-  test("SPARK-24339 verify the result after pruning the unused columns") {
-    val rowsDf = Seq(
-      ("Bob", 16, 176),
-      ("Alice", 32, 164),
-      ("David", 60, 192),
-      ("Amy", 24, 180)).toDF("name", "age", "height")
-
-    checkAnswer(
-      rowsDf,
-      (child: SparkPlan) => new ScriptTransformationExec(
-        input = Seq(rowsDf.col("name").expr),
-        script = "cat",
-        output = Seq(AttributeReference("name", StringType)()),
-        child = child,
-        ioschema = serdeIOSchema
-      ),
-      rowsDf.select("name").collect())
-    assert(uncaughtExceptionHandler.exception.isEmpty)
-  }
-
-  test("SPARK-25990: TRANSFORM should handle different data types correctly") {
-    assume(TestUtils.testCommandAvailable("python"))
-    val scriptFilePath = getTestResourcePath("test_script.py")
-
-    withTempView("v") {
-      val df = Seq(
-        (1, "1", 1.0, BigDecimal(1.0), new Timestamp(1)),
-        (2, "2", 2.0, BigDecimal(2.0), new Timestamp(2)),
-        (3, "3", 3.0, BigDecimal(3.0), new Timestamp(3))
-      ).toDF("a", "b", "c", "d", "e") // Note column d's data type is Decimal(38, 18)
-      df.createTempView("v")
-
-      val query = sql(
-        s"""
-          |SELECT
-          |TRANSFORM(a, b, c, d, e)
-          |USING 'python $scriptFilePath' AS (a, b, c, d, e)
-          |FROM v
-        """.stripMargin)
-
-      // In Hive 1.2, the string representation of a decimal omits trailing zeroes.
-      // But in Hive 2.3, it is always padded to 18 digits with trailing zeroes if necessary.
-      val decimalToString: Column => Column = if (HiveUtils.isHive23) {
-        c => c.cast("string")
-      } else {
-        c => c.cast("decimal(1, 0)").cast("string")
-      }
-      checkAnswer(query, identity, df.select(
-        'a.cast("string"),
-        'b.cast("string"),
-        'c.cast("string"),
-        decimalToString('d),
-        'e.cast("string")).collect())
-    }
-  }
-
-  test("SPARK-30973: TRANSFORM should wait for the termination of the script (no serde)") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-
-    val rowsDf = Seq("a", "b", "c").map(Tuple1.apply).toDF("a")
-    val e = intercept[SparkException] {
-      val plan =
-        new ScriptTransformationExec(
-          input = Seq(rowsDf.col("a").expr),
-          script = "some_non_existent_command",
-          output = Seq(AttributeReference("a", StringType)()),
-          child = rowsDf.queryExecution.sparkPlan,
-          ioschema = noSerdeIOSchema)
-      SparkPlanTest.executePlan(plan, hiveContext)
-    }
-    assert(e.getMessage.contains("Subprocess exited with status"))
-    assert(uncaughtExceptionHandler.exception.isEmpty)
-  }
-
-  test("SPARK-30973: TRANSFORM should wait for the termination of the script (with serde)") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-
-    val rowsDf = Seq("a", "b", "c").map(Tuple1.apply).toDF("a")
-    val e = intercept[SparkException] {
-      val plan =
-        new ScriptTransformationExec(
-          input = Seq(rowsDf.col("a").expr),
-          script = "some_non_existent_command",
-          output = Seq(AttributeReference("a", StringType)()),
-          child = rowsDf.queryExecution.sparkPlan,
-          ioschema = serdeIOSchema)
-      SparkPlanTest.executePlan(plan, hiveContext)
-    }
-    assert(e.getMessage.contains("Subprocess exited with status"))
-    assert(uncaughtExceptionHandler.exception.isEmpty)
-  }
-}
-
-private case class ExceptionInjectingOperator(child: SparkPlan) extends UnaryExecNode {
-  override protected def doExecute(): RDD[InternalRow] = {
-    child.execute().map { x =>
-      assert(TaskContext.get() != null) // Make sure that TaskContext is defined.
-      Thread.sleep(1000) // This sleep gives the external process time to start.
-      throw new IllegalArgumentException("intentional exception")
-    }
-  }
-
-  override def output: Seq[Attribute] = child.output
-
-  override def outputPartitioning: Partitioning = child.outputPartitioning
-}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/TestingTypedCount.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/TestingTypedCount.scala
index 31b24301767af..ee233fbd7238f 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/TestingTypedCount.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/TestingTypedCount.scala
@@ -22,6 +22,7 @@ import java.io.{ByteArrayInputStream, ByteArrayOutputStream, DataInputStream, Da
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate.{ImperativeAggregate, TypedImperativeAggregate}
+import org.apache.spark.sql.catalyst.trees.UnaryLike
 import org.apache.spark.sql.hive.execution.TestingTypedCount.State
 import org.apache.spark.sql.types._
 
@@ -32,12 +33,11 @@ case class TestingTypedCount(
     child: Expression,
     mutableAggBufferOffset: Int = 0,
     inputAggBufferOffset: Int = 0)
-  extends TypedImperativeAggregate[TestingTypedCount.State] {
+  extends TypedImperativeAggregate[TestingTypedCount.State]
+  with UnaryLike[Expression] {
 
   def this(child: Expression) = this(child, 0, 0)
 
-  override def children: Seq[Expression] = child :: Nil
-
   override def dataType: DataType = LongType
 
   override def nullable: Boolean = false
@@ -78,6 +78,9 @@ case class TestingTypedCount(
     copy(inputAggBufferOffset = newInputAggBufferOffset)
 
   override val prettyName: String = "typed_count"
+
+  override protected def withNewChildInternal(newChild: Expression): TestingTypedCount =
+    copy(child = newChild)
 }
 
 object TestingTypedCount {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/UDAQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/UDAQuerySuite.scala
index e6856a58b0ea9..50f13efccc915 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/UDAQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/UDAQuerySuite.scala
@@ -17,23 +17,15 @@
 
 package org.apache.spark.sql.hive.execution
 
-import java.lang.{Double => jlDouble, Integer => jlInt, Long => jlLong}
-
-import scala.collection.JavaConverters._
-import scala.util.Random
-
-import test.org.apache.spark.sql.MyDoubleAvg
-import test.org.apache.spark.sql.MyDoubleSum
+import java.lang.{Double => jlDouble, Long => jlLong}
 
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
-import org.apache.spark.sql.catalyst.expressions.UnsafeRow
-import org.apache.spark.sql.expressions.{Aggregator}
+import org.apache.spark.sql.expressions.Aggregator
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.hive.test.TestHiveSingleton
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SQLTestUtils
 import org.apache.spark.sql.types._
 
@@ -119,6 +111,27 @@ object CountSerDeAgg extends Aggregator[Int, CountSerDeSQL, CountSerDeSQL] {
   def outputEncoder: Encoder[CountSerDeSQL] = ExpressionEncoder[CountSerDeSQL]()
 }
 
+object ArrayDataAgg extends Aggregator[Array[Double], Array[Double], Array[Double]] {
+  def zero: Array[Double] = Array(0.0, 0.0, 0.0)
+  def reduce(s: Array[Double], array: Array[Double]): Array[Double] = {
+    require(s.length == array.length)
+    for ( j <- 0 until s.length ) {
+      s(j) += array(j)
+    }
+    s
+  }
+  def merge(s1: Array[Double], s2: Array[Double]): Array[Double] = {
+    require(s1.length == s2.length)
+    for ( j <- 0 until s1.length ) {
+      s1(j) += s2(j)
+    }
+    s1
+  }
+  def finish(s: Array[Double]): Array[Double] = s
+  def bufferEncoder: Encoder[Array[Double]] = ExpressionEncoder[Array[Double]]
+  def outputEncoder: Encoder[Array[Double]] = ExpressionEncoder[Array[Double]]
+}
+
 abstract class UDAQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
   import testImplicits._
 
@@ -156,20 +169,11 @@ abstract class UDAQuerySuite extends QueryTest with SQLTestUtils with TestHiveSi
       (3, null, null)).toDF("key", "value1", "value2")
     data2.write.saveAsTable("agg2")
 
-    val data3 = Seq[(Seq[Integer], Integer, Integer)](
-      (Seq[Integer](1, 1), 10, -10),
-      (Seq[Integer](null), -60, 60),
-      (Seq[Integer](1, 1), 30, -30),
-      (Seq[Integer](1), 30, 30),
-      (Seq[Integer](2), 1, 1),
-      (null, -10, 10),
-      (Seq[Integer](2, 3), -1, null),
-      (Seq[Integer](2, 3), 1, 1),
-      (Seq[Integer](2, 3, 4), null, 1),
-      (Seq[Integer](null), 100, -10),
-      (Seq[Integer](3), null, 3),
-      (null, null, null),
-      (Seq[Integer](3), null, null)).toDF("key", "value1", "value2")
+    val data3 = Seq[(Seq[Double], Int)](
+      (Seq(1.0, 2.0, 3.0), 0),
+      (Seq(4.0, 5.0, 6.0), 0),
+      (Seq(7.0, 8.0, 9.0), 0)
+    ).toDF("data", "dummy")
     data3.write.saveAsTable("agg3")
 
     val data4 = Seq[Boolean](true, false, true).toDF("boolvalues")
@@ -184,6 +188,7 @@ abstract class UDAQuerySuite extends QueryTest with SQLTestUtils with TestHiveSi
     spark.udf.register("mydoublesum", udaf(MyDoubleSumAgg))
     spark.udf.register("mydoubleavg", udaf(MyDoubleAvgAgg))
     spark.udf.register("longProductSum", udaf(LongProductSumAgg))
+    spark.udf.register("arraysum", udaf(ArrayDataAgg))
   }
 
   override def afterAll(): Unit = {
@@ -354,6 +359,12 @@ abstract class UDAQuerySuite extends QueryTest with SQLTestUtils with TestHiveSi
         Row(3, 0, null, 1, 3, 0, 0, 0, null, 1, 3, 0, 2, 2) :: Nil)
   }
 
+  test("SPARK-32159: array encoders should be resolved in analyzer") {
+    checkAnswer(
+      spark.sql("SELECT arraysum(data) FROM agg3"),
+      Row(Seq(12.0, 15.0, 18.0)) :: Nil)
+  }
+
   test("verify aggregator ser/de behavior") {
     val data = sparkContext.parallelize((1 to 100).toSeq, 3).toDF("value1")
     val agg = udaf(CountSerDeAgg)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/WindowQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/WindowQuerySuite.scala
index 15712a18ce751..6bf7bd6cbb90e 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/WindowQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/WindowQuerySuite.scala
@@ -62,7 +62,6 @@ class WindowQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleto
     // Moved because:
     // - Spark uses a different default stddev (sample instead of pop)
     // - Tiny numerical differences in stddev results.
-    // - Different StdDev behavior when n=1 (NaN instead of 0)
     checkAnswer(sql(s"""
       |select  p_mfgr,p_name, p_size,
       |rank() over(distribute by p_mfgr sort by p_name) as r,
@@ -88,22 +87,22 @@ class WindowQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleto
         Row("Manufacturer#1", "almond antique salmon chartreuse burlywood", 6, 4, 3, 0.6666666666666666, 0.6, 2, 4, 11.0, 15.448840301675292, 2, 6, 2),
         Row("Manufacturer#1", "almond aquamarine burnished black steel", 28, 5, 4, 0.8333333333333334, 0.8, 3, 5, 14.4, 15.388307249337076, 2, 28, 34),
         Row("Manufacturer#1", "almond aquamarine pink moccasin thistle", 42, 6, 5, 1.0, 1.0, 3, 6, 19.0, 17.787636155487327, 2, 42, 6),
-        Row("Manufacturer#2", "almond antique violet chocolate turquoise", 14, 1, 1, 0.2, 0.0, 1, 1, 14.0, Double.NaN, 4, 14, 14),
+        Row("Manufacturer#2", "almond antique violet chocolate turquoise", 14, 1, 1, 0.2, 0.0, 1, 1, 14.0, null, 4, 14, 14),
         Row("Manufacturer#2", "almond antique violet turquoise frosted", 40, 2, 2, 0.4, 0.25, 1, 2, 27.0, 18.384776310850235, 4, 40, 14),
         Row("Manufacturer#2", "almond aquamarine midnight light salmon", 2, 3, 3, 0.6, 0.5, 2, 3, 18.666666666666668, 19.42506971244462, 4, 2, 14),
         Row("Manufacturer#2", "almond aquamarine rose maroon antique", 25, 4, 4, 0.8, 0.75, 2, 4, 20.25, 16.17353805861084, 4, 25, 40),
         Row("Manufacturer#2", "almond aquamarine sandy cyan gainsboro", 18, 5, 5, 1.0, 1.0, 3, 5, 19.8, 14.042791745233567, 4, 18, 2),
-        Row("Manufacturer#3", "almond antique chartreuse khaki white", 17, 1, 1, 0.2, 0.0, 1, 1, 17.0,Double.NaN, 2, 17, 17),
+        Row("Manufacturer#3", "almond antique chartreuse khaki white", 17, 1, 1, 0.2, 0.0, 1, 1, 17.0, null, 2, 17, 17),
         Row("Manufacturer#3", "almond antique forest lavender goldenrod", 14, 2, 2, 0.4, 0.25, 1, 2, 15.5, 2.1213203435596424, 2, 14, 17),
         Row("Manufacturer#3", "almond antique metallic orange dim", 19, 3, 3, 0.6, 0.5, 2, 3, 16.666666666666668, 2.516611478423583, 2, 19, 17),
         Row("Manufacturer#3", "almond antique misty red olive", 1, 4, 4, 0.8, 0.75, 2, 4, 12.75, 8.098353742170895, 2, 1, 14),
         Row("Manufacturer#3", "almond antique olive coral navajo", 45, 5, 5, 1.0, 1.0, 3, 5, 19.2, 16.037456157383566, 2, 45, 19),
-        Row("Manufacturer#4", "almond antique gainsboro frosted violet", 10, 1, 1, 0.2, 0.0, 1, 1, 10.0, Double.NaN, 0, 10, 10),
+        Row("Manufacturer#4", "almond antique gainsboro frosted violet", 10, 1, 1, 0.2, 0.0, 1, 1, 10.0, null, 0, 10, 10),
         Row("Manufacturer#4", "almond antique violet mint lemon", 39, 2, 2, 0.4, 0.25, 1, 2, 24.5, 20.506096654409877, 0, 39, 10),
         Row("Manufacturer#4", "almond aquamarine floral ivory bisque", 27, 3, 3, 0.6, 0.5, 2, 3, 25.333333333333332, 14.571661996262929, 0, 27, 10),
         Row("Manufacturer#4", "almond aquamarine yellow dodger mint", 7, 4, 4, 0.8, 0.75, 2, 4, 20.75, 15.01943185787443, 0, 7, 39),
         Row("Manufacturer#4", "almond azure aquamarine papaya violet", 12, 5, 5, 1.0, 1.0, 3, 5, 19.0, 13.583077707206124, 0, 12, 27),
-        Row("Manufacturer#5", "almond antique blue firebrick mint", 31, 1, 1, 0.2, 0.0, 1, 1, 31.0, Double.NaN, 1, 31, 31),
+        Row("Manufacturer#5", "almond antique blue firebrick mint", 31, 1, 1, 0.2, 0.0, 1, 1, 31.0, null, 1, 31, 31),
         Row("Manufacturer#5", "almond antique medium spring khaki", 6, 2, 2, 0.4, 0.25, 1, 2, 18.5, 17.67766952966369, 1, 6, 31),
         Row("Manufacturer#5", "almond antique sky peru orange", 2, 3, 3, 0.6, 0.5, 2, 3, 13.0, 15.716233645501712, 1, 2, 31),
         Row("Manufacturer#5", "almond aquamarine dodger light gainsboro", 46, 4, 4, 0.8, 0.75, 2, 4, 21.25, 20.902551678363736, 1, 46, 6),
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/AlterTableAddPartitionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/AlterTableAddPartitionSuite.scala
new file mode 100644
index 0000000000000..24f62abe66095
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/AlterTableAddPartitionSuite.scala
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.execution.command
+
+import org.apache.spark.sql.execution.command.v1
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * The class contains tests for the `ALTER TABLE .. ADD PARTITION` command to check
+ * V1 Hive external table catalog.
+ */
+class AlterTableAddPartitionSuite
+  extends v1.AlterTableAddPartitionSuiteBase
+  with CommandSuiteBase {
+
+  test("hive client calls") {
+    Seq(false, true).foreach { statsOn =>
+      withSQLConf(SQLConf.AUTO_SIZE_UPDATE_ENABLED.key -> statsOn.toString) {
+        withNamespaceAndTable("ns", "tbl") { t =>
+          sql(s"CREATE TABLE $t (id int, part int) $defaultUsing PARTITIONED BY (part)")
+          sql(s"INSERT INTO $t PARTITION (part=0) SELECT 0")
+          assert(!statsOn || getTableSize(t) > 0)
+
+          checkHiveClientCalls(expected = 17) {
+            sql(s"ALTER TABLE $t ADD PARTITION (part=1)")
+          }
+          sql(s"CACHE TABLE $t")
+          checkHiveClientCalls(expected = 17) {
+            sql(s"ALTER TABLE $t ADD PARTITION (part=2)")
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/AlterTableDropPartitionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/AlterTableDropPartitionSuite.scala
new file mode 100644
index 0000000000000..eeebfd32a2838
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/AlterTableDropPartitionSuite.scala
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.execution.command
+
+import org.apache.spark.sql.execution.command.v1
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * The class contains tests for the `ALTER TABLE .. DROP PARTITION` command to check
+ * V1 Hive external table catalog.
+ */
+class AlterTableDropPartitionSuite
+  extends v1.AlterTableDropPartitionSuiteBase
+  with CommandSuiteBase {
+
+  test("hive client calls") {
+    Seq(false, true).foreach { statsOn =>
+      withSQLConf(SQLConf.AUTO_SIZE_UPDATE_ENABLED.key -> statsOn.toString) {
+        withNamespaceAndTable("ns", "tbl") { t =>
+          sql(s"CREATE TABLE $t (id int, part int) $defaultUsing PARTITIONED BY (part)")
+          sql(s"INSERT INTO $t PARTITION (part=0) SELECT 0")
+          sql(s"INSERT INTO $t PARTITION (part=1) SELECT 1")
+          sql(s"ALTER TABLE $t ADD PARTITION (part=2)") // empty partition
+          checkHiveClientCalls(expected = if (statsOn) 24 else 16) {
+            sql(s"ALTER TABLE $t DROP PARTITION (part=2)")
+          }
+          checkHiveClientCalls(expected = if (statsOn) 29 else 16) {
+            sql(s"ALTER TABLE $t DROP PARTITION (part=0)")
+          }
+          sql(s"CACHE TABLE $t")
+          checkHiveClientCalls(expected = if (statsOn) 29 else 16) {
+            sql(s"ALTER TABLE $t DROP PARTITION (part=1)")
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/AlterTableRecoverPartitionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/AlterTableRecoverPartitionsSuite.scala
new file mode 100644
index 0000000000000..a6badb98a8645
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/AlterTableRecoverPartitionsSuite.scala
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.execution.command
+
+import org.apache.spark.sql.execution.command.v1
+
+/**
+ * The class contains tests for the `ALTER TABLE .. RECOVER PARTITIONS` command to check
+ * V1 Hive external table catalog.
+ */
+class AlterTableRecoverPartitionsSuite
+  extends v1.AlterTableRecoverPartitionsSuiteBase
+  with CommandSuiteBase
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/AlterTableRenamePartitionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/AlterTableRenamePartitionSuite.scala
new file mode 100644
index 0000000000000..c1f4d111a97f9
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/AlterTableRenamePartitionSuite.scala
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.execution.command
+
+import org.apache.spark.sql.execution.command.v1
+
+/**
+ * The class contains tests for the `ALTER TABLE .. RENAME PARTITION` command to check
+ * V1 Hive external table catalog.
+ */
+class AlterTableRenamePartitionSuite
+  extends v1.AlterTableRenamePartitionSuiteBase
+  with CommandSuiteBase {
+
+  test("hive client calls") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (id int, part int) $defaultUsing PARTITIONED BY (part)")
+      sql(s"INSERT INTO $t PARTITION (part=0) SELECT 0")
+
+      checkHiveClientCalls(expected = 18) {
+        sql(s"ALTER TABLE $t PARTITION (part=0) RENAME TO PARTITION (part=1)")
+      }
+      sql(s"CACHE TABLE $t")
+      checkHiveClientCalls(expected = 18) {
+        sql(s"ALTER TABLE $t PARTITION (part=1) RENAME TO PARTITION (part=2)")
+      }
+    }
+
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (id int, PART int) $defaultUsing PARTITIONED BY (PART)")
+      sql(s"INSERT INTO $t PARTITION (PART=0) SELECT 0")
+      checkHiveClientCalls(expected = 23) {
+        sql(s"ALTER TABLE $t PARTITION (PART=0) RENAME TO PARTITION (PART=1)")
+      }
+    }
+  }
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/AlterTableRenameSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/AlterTableRenameSuite.scala
new file mode 100644
index 0000000000000..c282cb13d5515
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/AlterTableRenameSuite.scala
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.execution.command
+
+import org.apache.spark.sql.execution.command.v1
+
+/**
+ * The class contains tests for the `ALTER TABLE .. RENAME` command to check
+ * V1 Hive external table catalog.
+ */
+class AlterTableRenameSuite extends v1.AlterTableRenameSuiteBase with CommandSuiteBase
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/CommandSuiteBase.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/CommandSuiteBase.scala
new file mode 100644
index 0000000000000..0709b12cdfc11
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/CommandSuiteBase.scala
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.execution.command
+
+import org.apache.spark.metrics.source.HiveCatalogMetrics
+import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
+import org.apache.spark.sql.connector.catalog.CatalogManager
+import org.apache.spark.sql.hive.test.TestHiveSingleton
+
+/**
+ * The trait contains settings and utility functions. It can be mixed to the test suites for
+ * datasource v1 Hive external catalog. This trait complements the common trait
+ * `org.apache.spark.sql.execution.command.DDLCommandTestUtils` with utility functions and
+ * settings for all unified datasource V1 and V2 test suites.
+ */
+trait CommandSuiteBase extends TestHiveSingleton {
+  def version: String = "Hive V1" // The prefix is added to test names
+  def catalog: String = CatalogManager.SESSION_CATALOG_NAME
+  def defaultUsing: String = "USING HIVE" // The clause is used in creating tables under testing
+
+  def checkLocation(
+      t: String,
+      spec: TablePartitionSpec,
+      expected: String): Unit = {
+    val tablePath = t.split('.')
+    val tableName = tablePath.last
+    val ns = tablePath.init.mkString(".")
+    val partSpec = spec.map { case (key, value) => s"$key = $value"}.mkString(", ")
+    val information =
+      spark.sql(s"SHOW TABLE EXTENDED IN $ns LIKE '$tableName' PARTITION($partSpec)")
+        .select("information")
+        .first().getString(0)
+    val location = information.split("\\r?\\n").filter(_.startsWith("Location:")).head
+    assert(location.endsWith(expected))
+  }
+
+  def checkHiveClientCalls[T](expected: Int)(f: => T): Unit = {
+    HiveCatalogMetrics.reset()
+    assert(HiveCatalogMetrics.METRIC_HIVE_CLIENT_CALLS.getCount === 0)
+    f
+    assert(HiveCatalogMetrics.METRIC_HIVE_CLIENT_CALLS.getCount === expected)
+  }
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/DropTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/DropTableSuite.scala
new file mode 100644
index 0000000000000..0ca6184c9469e
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/DropTableSuite.scala
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.execution.command
+
+import org.apache.spark.sql.execution.command.v1
+
+/**
+ * The class contains tests for the `DROP TABLE` command to check V1 Hive external table catalog.
+ */
+class DropTableSuite extends v1.DropTableSuiteBase with CommandSuiteBase {
+  test("hive client calls") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (id int) $defaultUsing")
+      checkHiveClientCalls(expected = 15) {
+        sql(s"DROP TABLE $t")
+      }
+    }
+  }
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/MsckRepairTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/MsckRepairTableSuite.scala
new file mode 100644
index 0000000000000..fc40aa2b82f93
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/MsckRepairTableSuite.scala
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.execution.command
+
+import org.apache.spark.sql.execution.command.v1
+
+/**
+ * The class contains tests for the `MSCK REPAIR TABLE` command to check
+ * V1 Hive external table catalog.
+ */
+class MsckRepairTableSuite extends v1.MsckRepairTableSuiteBase with CommandSuiteBase
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowNamespacesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowNamespacesSuite.scala
new file mode 100644
index 0000000000000..7aae000144ef0
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowNamespacesSuite.scala
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.execution.command
+
+import org.apache.spark.sql.execution.command.v1
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * The class contains tests for the `SHOW NAMESPACES` and `SHOW DATABASES` commands to check
+ * V1 Hive external table catalog.
+ */
+class ShowNamespacesSuite extends v1.ShowNamespacesSuiteBase with CommandSuiteBase {
+  test("case sensitivity") {
+    Seq(true, false).foreach { caseSensitive =>
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
+        withNamespace(s"$catalog.AAA", s"$catalog.bbb") {
+          sql(s"CREATE NAMESPACE $catalog.AAA")
+          sql(s"CREATE NAMESPACE $catalog.bbb")
+          runShowNamespacesSql(
+            s"SHOW NAMESPACES IN $catalog",
+            Seq("aaa", "bbb") ++ builtinTopNamespaces)
+          runShowNamespacesSql(s"SHOW NAMESPACES IN $catalog LIKE 'AAA'", Seq("aaa"))
+          runShowNamespacesSql(s"SHOW NAMESPACES IN $catalog LIKE 'aaa'", Seq("aaa"))
+        }
+      }
+    }
+  }
+
+  test("hive client calls") {
+    withNamespace(s"$catalog.ns1", s"$catalog.ns2") {
+      sql(s"CREATE NAMESPACE $catalog.ns1")
+      sql(s"CREATE NAMESPACE $catalog.ns2")
+
+      checkHiveClientCalls(expected = 1) {
+        sql(s"SHOW NAMESPACES IN $catalog")
+      }
+    }
+  }
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowPartitionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowPartitionsSuite.scala
new file mode 100644
index 0000000000000..c3d9790e1f6e5
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowPartitionsSuite.scala
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.execution.command
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.execution.command.v1
+
+/**
+ * The class contains tests for the `SHOW PARTITIONS` command to check
+ * V1 Hive external table catalog.
+ */
+class ShowPartitionsSuite extends v1.ShowPartitionsSuiteBase with CommandSuiteBase {
+  test("SPARK-33904: null and empty string as partition values") {
+    withSQLConf("hive.exec.dynamic.partition.mode" -> "nonstrict") {
+      withNamespaceAndTable("ns", "tbl") { t =>
+        createNullPartTable(t, "hive")
+        runShowPartitionsSql(
+          s"SHOW PARTITIONS $t",
+          Row("part=__HIVE_DEFAULT_PARTITION__") :: Nil)
+        checkAnswer(spark.table(t),
+          Row(0, "__HIVE_DEFAULT_PARTITION__") ::
+          Row(1, "__HIVE_DEFAULT_PARTITION__") :: Nil)
+      }
+    }
+  }
+
+  test("hive client calls") {
+    withNamespaceAndTable("ns", "dateTable") { t =>
+      createDateTable(t)
+      checkHiveClientCalls(expected = 10) {
+        sql(s"SHOW PARTITIONS $t")
+      }
+    }
+  }
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowTablesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowTablesSuite.scala
new file mode 100644
index 0000000000000..b6db9a3e74315
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowTablesSuite.scala
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.execution.command
+
+import org.apache.spark.sql.execution.command.v1
+
+/**
+ * The class contains tests for the `SHOW TABLES` command to check V1 Hive external table catalog.
+ */
+class ShowTablesSuite extends v1.ShowTablesSuiteBase with CommandSuiteBase {
+  test("hive client calls") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      sql(s"CREATE TABLE $t (id int) $defaultUsing")
+      checkHiveClientCalls(expected = 3) {
+        sql(s"SHOW TABLES IN $catalog.ns")
+      }
+    }
+  }
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/TruncateTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/TruncateTableSuite.scala
new file mode 100644
index 0000000000000..37d20eff8211d
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/TruncateTableSuite.scala
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.execution.command
+
+import org.apache.spark.sql.execution.command.v1
+
+/**
+ * The class contains tests for the `TRUNCATE TABLE` command to check
+ * V1 Hive external table catalog.
+ */
+class TruncateTableSuite extends v1.TruncateTableSuiteBase with CommandSuiteBase
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcFilterSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcFilterSuite.scala
deleted file mode 100644
index 5fc41067f661d..0000000000000
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcFilterSuite.scala
+++ /dev/null
@@ -1,484 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.hive.orc
-
-import java.nio.charset.StandardCharsets
-import java.sql.{Date, Timestamp}
-
-import scala.collection.JavaConverters._
-
-import org.apache.hadoop.hive.ql.io.sarg.{PredicateLeaf, SearchArgument}
-
-import org.apache.spark.sql.{Column, DataFrame}
-import org.apache.spark.sql.catalyst.dsl.expressions._
-import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.planning.PhysicalOperation
-import org.apache.spark.sql.execution.datasources.{DataSourceStrategy, HadoopFsRelation, LogicalRelation}
-import org.apache.spark.sql.execution.datasources.orc.OrcTest
-import org.apache.spark.sql.hive.HiveUtils
-import org.apache.spark.sql.hive.test.TestHiveSingleton
-import org.apache.spark.sql.types._
-
-/**
- * A test suite that tests Hive ORC filter API based filter pushdown optimization.
- */
-class HiveOrcFilterSuite extends OrcTest with TestHiveSingleton {
-
-  override val orcImp: String = "hive"
-
-  private def checkFilterPredicate(
-      df: DataFrame,
-      predicate: Predicate,
-      checker: (SearchArgument) => Unit): Unit = {
-    val output = predicate.collect { case a: Attribute => a }.distinct
-    val query = df
-      .select(output.map(e => Column(e)): _*)
-      .where(Column(predicate))
-
-    var maybeRelation: Option[HadoopFsRelation] = None
-    val maybeAnalyzedPredicate = query.queryExecution.optimizedPlan.collect {
-      case PhysicalOperation(_, filters, LogicalRelation(orcRelation: HadoopFsRelation, _, _, _)) =>
-        maybeRelation = Some(orcRelation)
-        filters
-    }.flatten.reduceLeftOption(_ && _)
-    assert(maybeAnalyzedPredicate.isDefined, "No filter is analyzed from the given query")
-
-    val (_, selectedFilters, _) =
-      DataSourceStrategy.selectFilters(maybeRelation.get, maybeAnalyzedPredicate.toSeq)
-    assert(selectedFilters.nonEmpty, "No filter is pushed down")
-
-    val maybeFilter = OrcFilters.createFilter(query.schema, selectedFilters.toArray)
-    assert(maybeFilter.isDefined, s"Couldn't generate filter predicate for $selectedFilters")
-    checker(maybeFilter.get)
-  }
-
-  private def checkFilterPredicate
-      (predicate: Predicate, filterOperator: PredicateLeaf.Operator)
-      (implicit df: DataFrame): Unit = {
-    def checkComparisonOperator(filter: SearchArgument) = {
-      val operator = filter.getLeaves.asScala
-      assert(operator.map(_.getOperator).contains(filterOperator))
-    }
-    checkFilterPredicate(df, predicate, checkComparisonOperator)
-  }
-
-  private def checkFilterPredicateWithDiffHiveVersion
-      (predicate: Predicate, stringExpr: String)
-      (implicit df: DataFrame): Unit = {
-    def checkLogicalOperator(filter: SearchArgument) = {
-      if (HiveUtils.isHive23) {
-        assert(filter.toString == stringExpr.replace("\n", ", "))
-      } else {
-        assert(filter.toString == stringExpr)
-      }
-    }
-    checkFilterPredicate(df, predicate, checkLogicalOperator)
-  }
-
-  private def assertResultWithDiffHiveVersion(expected : String)(c : scala.Any) = {
-    if (HiveUtils.isHive23) {
-      assertResult(expected.replace("\n", ", "))(c)
-    } else {
-      assertResult(expected)(c)
-    }
-  }
-
-  private def checkNoFilterPredicate
-      (predicate: Predicate)
-      (implicit df: DataFrame): Unit = {
-    val output = predicate.collect { case a: Attribute => a }.distinct
-    val query = df
-      .select(output.map(e => Column(e)): _*)
-      .where(Column(predicate))
-
-    var maybeRelation: Option[HadoopFsRelation] = None
-    val maybeAnalyzedPredicate = query.queryExecution.optimizedPlan.collect {
-      case PhysicalOperation(_, filters, LogicalRelation(orcRelation: HadoopFsRelation, _, _, _)) =>
-        maybeRelation = Some(orcRelation)
-        filters
-    }.flatten.reduceLeftOption(_ && _)
-    assert(maybeAnalyzedPredicate.isDefined, "No filter is analyzed from the given query")
-
-    val (_, selectedFilters, _) =
-      DataSourceStrategy.selectFilters(maybeRelation.get, maybeAnalyzedPredicate.toSeq)
-    assert(selectedFilters.nonEmpty, "No filter is pushed down")
-
-    val maybeFilter = OrcFilters.createFilter(query.schema, selectedFilters.toArray)
-    assert(maybeFilter.isEmpty, s"Could generate filter predicate for $selectedFilters")
-  }
-
-  test("filter pushdown - integer") {
-    withOrcDataFrame((1 to 4).map(i => Tuple1(Option(i)))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === 1, PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> 1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < 2, PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > 3, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= 1, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= 4, PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal(1) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal(1) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(2) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(3) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(1) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(4) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-    }
-  }
-
-  test("filter pushdown - long") {
-    withOrcDataFrame((1 to 4).map(i => Tuple1(Option(i.toLong)))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === 1, PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> 1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < 2, PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > 3, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= 1, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= 4, PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal(1) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal(1) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(2) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(3) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(1) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(4) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-    }
-  }
-
-  test("filter pushdown - float") {
-    withOrcDataFrame((1 to 4).map(i => Tuple1(Option(i.toFloat)))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === 1, PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> 1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < 2, PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > 3, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= 1, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= 4, PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal(1) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal(1) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(2) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(3) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(1) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(4) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-    }
-  }
-
-  test("filter pushdown - double") {
-    withOrcDataFrame((1 to 4).map(i => Tuple1(Option(i.toDouble)))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === 1, PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> 1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < 2, PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > 3, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= 1, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= 4, PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal(1) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal(1) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(2) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(3) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(1) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(4) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-    }
-  }
-
-  test("filter pushdown - string") {
-    withOrcDataFrame((1 to 4).map(i => Tuple1(i.toString))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === "1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> "1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < "2", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > "3", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= "1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= "4", PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal("1") === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal("1") <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal("2") > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal("3") < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal("1") >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal("4") <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-    }
-  }
-
-  test("filter pushdown - boolean") {
-    withOrcDataFrame((true :: false :: Nil).map(b => Tuple1.apply(Option(b)))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === true, PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> true, PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < true, PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > false, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= false, PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= false, PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal(false) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal(false) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(false) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(true) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(true) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(true) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-    }
-  }
-
-  test("filter pushdown - decimal") {
-    withOrcDataFrame((1 to 4).map(i => Tuple1.apply(BigDecimal.valueOf(i)))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === BigDecimal.valueOf(1), PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> BigDecimal.valueOf(1), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < BigDecimal.valueOf(2), PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > BigDecimal.valueOf(3), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= BigDecimal.valueOf(1), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= BigDecimal.valueOf(4), PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(
-        Literal(BigDecimal.valueOf(1)) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(
-        Literal(BigDecimal.valueOf(1)) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(
-        Literal(BigDecimal.valueOf(2)) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(
-        Literal(BigDecimal.valueOf(3)) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(
-        Literal(BigDecimal.valueOf(1)) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(
-        Literal(BigDecimal.valueOf(4)) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-    }
-  }
-
-  test("filter pushdown - timestamp") {
-    val timeString = "2015-08-20 14:57:00"
-    val timestamps = (1 to 4).map { i =>
-      val milliseconds = Timestamp.valueOf(timeString).getTime + i * 3600
-      new Timestamp(milliseconds)
-    }
-    withOrcDataFrame(timestamps.map(Tuple1(_))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === timestamps(0), PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> timestamps(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < timestamps(1), PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > timestamps(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= timestamps(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= timestamps(3), PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal(timestamps(0)) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal(timestamps(0)) <=> $"_1",
-        PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(timestamps(1)) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(timestamps(2)) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(timestamps(0)) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(timestamps(3)) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
-    }
-  }
-
-  test("filter pushdown - combinations with logical operators") {
-    withOrcDataFrame((1 to 4).map(i => Tuple1(Option(i)))) { implicit df =>
-      // Because `ExpressionTree` is not accessible at Hive 1.2.x, this should be checked
-      // in string form in order to check filter creation including logical operators
-      // such as `and`, `or` or `not`. So, this function uses `SearchArgument.toString()`
-      // to produce string expression and then compare it to given string expression below.
-      // This might have to be changed after Hive version is upgraded.
-      checkFilterPredicateWithDiffHiveVersion(
-        $"_1".isNotNull,
-        """leaf-0 = (IS_NULL _1)
-          |expr = (not leaf-0)""".stripMargin.trim
-      )
-      checkFilterPredicateWithDiffHiveVersion(
-        $"_1" =!= 1,
-        """leaf-0 = (IS_NULL _1)
-          |leaf-1 = (EQUALS _1 1)
-          |expr = (and (not leaf-0) (not leaf-1))""".stripMargin.trim
-      )
-      checkFilterPredicateWithDiffHiveVersion(
-        !($"_1" < 4),
-        """leaf-0 = (IS_NULL _1)
-          |leaf-1 = (LESS_THAN _1 4)
-          |expr = (and (not leaf-0) (not leaf-1))""".stripMargin.trim
-      )
-      checkFilterPredicateWithDiffHiveVersion(
-        $"_1" < 2 || $"_1" > 3,
-        """leaf-0 = (LESS_THAN _1 2)
-          |leaf-1 = (LESS_THAN_EQUALS _1 3)
-          |expr = (or leaf-0 (not leaf-1))""".stripMargin.trim
-      )
-      checkFilterPredicateWithDiffHiveVersion(
-        $"_1" < 2 && $"_1" > 3,
-        """leaf-0 = (IS_NULL _1)
-          |leaf-1 = (LESS_THAN _1 2)
-          |leaf-2 = (LESS_THAN_EQUALS _1 3)
-          |expr = (and (not leaf-0) leaf-1 (not leaf-2))""".stripMargin.trim
-      )
-    }
-  }
-
-  test("no filter pushdown - non-supported types") {
-    implicit class IntToBinary(int: Int) {
-      def b: Array[Byte] = int.toString.getBytes(StandardCharsets.UTF_8)
-    }
-    // ArrayType
-    withOrcDataFrame((1 to 4).map(i => Tuple1(Array(i)))) { implicit df =>
-      checkNoFilterPredicate($"_1".isNull)
-    }
-    // BinaryType
-    withOrcDataFrame((1 to 4).map(i => Tuple1(i.b))) { implicit df =>
-      checkNoFilterPredicate($"_1" <=> 1.b)
-    }
-    // DateType
-    if (!HiveUtils.isHive23) {
-      val stringDate = "2015-01-01"
-      withOrcDataFrame(Seq(Tuple1(Date.valueOf(stringDate)))) { implicit df =>
-        checkNoFilterPredicate($"_1" === Date.valueOf(stringDate))
-      }
-    }
-    // MapType
-    withOrcDataFrame((1 to 4).map(i => Tuple1(Map(i -> i)))) { implicit df =>
-      checkNoFilterPredicate($"_1".isNotNull)
-    }
-  }
-
-  test("SPARK-12218 and SPARK-25699 Converting conjunctions into ORC SearchArguments") {
-    import org.apache.spark.sql.sources._
-    // The `LessThan` should be converted while the `StringContains` shouldn't
-    val schema = new StructType(
-      Array(
-        StructField("a", IntegerType, nullable = true),
-        StructField("b", StringType, nullable = true)))
-    assertResultWithDiffHiveVersion(
-      """leaf-0 = (LESS_THAN a 10)
-        |expr = leaf-0
-      """.stripMargin.trim
-    ) {
-      OrcFilters.createFilter(schema, Array(
-        LessThan("a", 10),
-        StringContains("b", "prefix")
-      )).get.toString
-    }
-
-    // The `LessThan` should be converted while the whole inner `And` shouldn't
-    assertResultWithDiffHiveVersion(
-      """leaf-0 = (LESS_THAN a 10)
-        |expr = leaf-0
-      """.stripMargin.trim
-    ) {
-      OrcFilters.createFilter(schema, Array(
-        LessThan("a", 10),
-        Not(And(
-          GreaterThan("a", 1),
-          StringContains("b", "prefix")
-        ))
-      )).get.toString
-    }
-
-    // Safely remove unsupported `StringContains` predicate and push down `LessThan`
-    assertResultWithDiffHiveVersion(
-      """leaf-0 = (LESS_THAN a 10)
-        |expr = leaf-0
-      """.stripMargin.trim
-    ) {
-      OrcFilters.createFilter(schema, Array(
-        And(
-          LessThan("a", 10),
-          StringContains("b", "prefix")
-        )
-      )).get.toString
-    }
-
-    // Safely remove unsupported `StringContains` predicate, push down `LessThan` and `GreaterThan`.
-    assertResultWithDiffHiveVersion(
-      """leaf-0 = (LESS_THAN a 10)
-        |leaf-1 = (LESS_THAN_EQUALS a 1)
-        |expr = (and leaf-0 (not leaf-1))
-      """.stripMargin.trim
-    ) {
-      OrcFilters.createFilter(schema, Array(
-        And(
-          And(
-            LessThan("a", 10),
-            StringContains("b", "prefix")
-          ),
-          GreaterThan("a", 1)
-        )
-      )).get.toString
-    }
-  }
-
-  test("SPARK-27699 Converting disjunctions into ORC SearchArguments") {
-    import org.apache.spark.sql.sources._
-    // The `LessThan` should be converted while the `StringContains` shouldn't
-    val schema = new StructType(
-      Array(
-        StructField("a", IntegerType, nullable = true),
-        StructField("b", StringType, nullable = true)))
-
-    // The predicate `StringContains` predicate is not able to be pushed down.
-    assertResultWithDiffHiveVersion("leaf-0 = (LESS_THAN_EQUALS a 10)\nleaf-1 = (LESS_THAN a 1)\n" +
-      "expr = (or (not leaf-0) leaf-1)") {
-      OrcFilters.createFilter(schema, Array(
-        Or(
-          GreaterThan("a", 10),
-          And(
-            StringContains("b", "prefix"),
-            LessThan("a", 1)
-          )
-        )
-      )).get.toString
-    }
-
-    assertResultWithDiffHiveVersion("leaf-0 = (LESS_THAN_EQUALS a 10)\nleaf-1 = (LESS_THAN a 1)\n" +
-      "expr = (or (not leaf-0) leaf-1)") {
-      OrcFilters.createFilter(schema, Array(
-        Or(
-          And(
-            GreaterThan("a", 10),
-            StringContains("b", "foobar")
-          ),
-          And(
-            StringContains("b", "prefix"),
-            LessThan("a", 1)
-          )
-        )
-      )).get.toString
-    }
-
-    assert(OrcFilters.createFilter(schema, Array(
-      Or(
-        StringContains("b", "foobar"),
-        And(
-          StringContains("b", "prefix"),
-          LessThan("a", 1)
-        )
-      )
-    )).isEmpty)
-  }
-}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcQuerySuite.scala
index 990d9425fb7fc..280ffb05fc1dd 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcQuerySuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.hive.orc
 import java.io.File
 
 import com.google.common.io.Files
+import org.apache.orc.OrcConf
 
 import org.apache.spark.sql.{AnalysisException, Row}
 import org.apache.spark.sql.catalyst.TableIdentifier
@@ -168,9 +169,6 @@ class HiveOrcQuerySuite extends OrcQueryTest with TestHiveSingleton {
     }
   }
 
-  // Since Hive 1.2.1 library code path still has this problem, users may hit this
-  // when spark.sql.hive.convertMetastoreOrc=false. However, after SPARK-22279,
-  // Apache Spark with the default configuration doesn't hit this bug.
   test("SPARK-22267 Spark SQL incorrectly reads ORC files when column order is different") {
     Seq("native", "hive").foreach { orcImpl =>
       withSQLConf(SQLConf.ORC_IMPLEMENTATION.key -> orcImpl) {
@@ -179,10 +177,12 @@ class HiveOrcQuerySuite extends OrcQueryTest with TestHiveSingleton {
           Seq(1 -> 2).toDF("c1", "c2").write.orc(path)
           checkAnswer(spark.read.orc(path), Row(1, 2))
 
-          withSQLConf(HiveUtils.CONVERT_METASTORE_ORC.key -> "true") { // default since 2.3.0
-            withTable("t") {
-              sql(s"CREATE EXTERNAL TABLE t(c2 INT, c1 INT) STORED AS ORC LOCATION '$path'")
-              checkAnswer(spark.table("t"), Row(2, 1))
+          Seq(true, false).foreach { convertMetastoreOrc =>
+            withSQLConf(HiveUtils.CONVERT_METASTORE_ORC.key -> convertMetastoreOrc.toString) {
+              withTable("t") {
+                sql(s"CREATE EXTERNAL TABLE t(c2 INT, c1 INT) STORED AS ORC LOCATION '$path'")
+                checkAnswer(spark.table("t"), Row(2, 1))
+              }
             }
           }
         }
@@ -190,9 +190,6 @@ class HiveOrcQuerySuite extends OrcQueryTest with TestHiveSingleton {
     }
   }
 
-  // Since Hive 1.2.1 library code path still has this problem, users may hit this
-  // when spark.sql.hive.convertMetastoreOrc=false. However, after SPARK-22279,
-  // Apache Spark with the default configuration doesn't hit this bug.
   test("SPARK-19809 NullPointerException on zero-size ORC file") {
     Seq("native", "hive").foreach { orcImpl =>
       withSQLConf(SQLConf.ORC_IMPLEMENTATION.key -> orcImpl) {
@@ -201,8 +198,10 @@ class HiveOrcQuerySuite extends OrcQueryTest with TestHiveSingleton {
             sql(s"CREATE TABLE spark_19809(a int) STORED AS ORC LOCATION '$dir'")
             Files.touch(new File(s"${dir.getCanonicalPath}", "zero.orc"))
 
-            withSQLConf(HiveUtils.CONVERT_METASTORE_ORC.key -> "true") { // default since 2.3.0
-              checkAnswer(spark.table("spark_19809"), Seq.empty)
+            Seq(true, false).foreach { convertMetastoreOrc =>
+              withSQLConf(HiveUtils.CONVERT_METASTORE_ORC.key -> convertMetastoreOrc.toString) {
+                checkAnswer(spark.table("spark_19809"), Seq.empty)
+              }
             }
           }
         }
@@ -224,7 +223,6 @@ class HiveOrcQuerySuite extends OrcQueryTest with TestHiveSingleton {
   }
 
   test("SPARK-26437 Can not query decimal type when value is 0") {
-    assume(HiveUtils.isHive23, "bad test: This bug fixed by HIVE-13083(Hive 2.0.1)")
     withSQLConf(HiveUtils.CONVERT_METASTORE_ORC.key -> "false") {
       withTable("spark_26437") {
         sql("CREATE TABLE spark_26437 STORED AS ORCFILE AS SELECT 0.00 AS c1")
@@ -257,7 +255,6 @@ class HiveOrcQuerySuite extends OrcQueryTest with TestHiveSingleton {
         withSQLConf(HiveUtils.CONVERT_METASTORE_ORC.key -> "true",
           HiveUtils.CONVERT_INSERTING_PARTITIONED_TABLE.key -> conversion) {
           withTable("dummy_orc_partitioned") {
-            spark.sessionState.refreshTable("dummy_orc_partitioned")
             spark.sql(
               s"""
                  |CREATE TABLE dummy_orc_partitioned(key INT, value STRING)
@@ -288,4 +285,102 @@ class HiveOrcQuerySuite extends OrcQueryTest with TestHiveSingleton {
       }
     }
   }
+
+  test("SPARK-32234 read ORC table with column names all starting with '_col'") {
+    Seq("native", "hive").foreach { orcImpl =>
+      Seq("false", "true").foreach { vectorized =>
+        withSQLConf(
+          SQLConf.ORC_IMPLEMENTATION.key -> orcImpl,
+          SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vectorized) {
+          withTable("test_hive_orc_impl") {
+            spark.sql(
+              s"""
+                 | CREATE TABLE test_hive_orc_impl
+                 | (_col1 INT, _col2 STRING, _col3 INT)
+                 | STORED AS ORC
+               """.stripMargin)
+            spark.sql(
+              s"""
+                 | INSERT INTO
+                 | test_hive_orc_impl
+                 | VALUES(9, '12', 2020)
+               """.stripMargin)
+
+            val df = spark.sql("SELECT _col2 FROM test_hive_orc_impl")
+            checkAnswer(df, Row("12"))
+          }
+        }
+      }
+    }
+  }
+
+  test("SPARK-32864: Support ORC forced positional evolution") {
+    Seq("native", "hive").foreach { orcImpl =>
+      Seq(true, false).foreach { forcePositionalEvolution =>
+        Seq(true, false).foreach { convertMetastore =>
+          withSQLConf(SQLConf.ORC_IMPLEMENTATION.key -> orcImpl,
+            OrcConf.FORCE_POSITIONAL_EVOLUTION.getAttribute -> forcePositionalEvolution.toString,
+            HiveUtils.CONVERT_METASTORE_ORC.key -> convertMetastore.toString) {
+            withTempPath { f =>
+              val path = f.getCanonicalPath
+              Seq[(Integer, Integer)]((1, 2), (3, 4), (5, 6), (null, null))
+                .toDF("c1", "c2").write.orc(path)
+              val correctAnswer = Seq(Row(1, 2), Row(3, 4), Row(5, 6), Row(null, null))
+              checkAnswer(spark.read.orc(path), correctAnswer)
+
+              withTable("t") {
+                sql(s"CREATE EXTERNAL TABLE t(c3 INT, c2 INT) STORED AS ORC LOCATION '$path'")
+
+                val expected = if (forcePositionalEvolution) {
+                  correctAnswer
+                } else {
+                  Seq(Row(null, 2), Row(null, 4), Row(null, 6), Row(null, null))
+                }
+
+                checkAnswer(spark.table("t"), expected)
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  test("SPARK-32864: Support ORC forced positional evolution with partitioned table") {
+    Seq("native", "hive").foreach { orcImpl =>
+      Seq(true, false).foreach { forcePositionalEvolution =>
+        Seq(true, false).foreach { convertMetastore =>
+          withSQLConf(SQLConf.ORC_IMPLEMENTATION.key -> orcImpl,
+            OrcConf.FORCE_POSITIONAL_EVOLUTION.getAttribute -> forcePositionalEvolution.toString,
+            HiveUtils.CONVERT_METASTORE_ORC.key -> convertMetastore.toString) {
+            withTempPath { f =>
+              val path = f.getCanonicalPath
+              Seq[(Integer, Integer, Integer)]((1, 2, 1), (3, 4, 2), (5, 6, 3), (null, null, 4))
+                .toDF("c1", "c2", "p").write.partitionBy("p").orc(path)
+              val correctAnswer = Seq(Row(1, 2, 1), Row(3, 4, 2), Row(5, 6, 3), Row(null, null, 4))
+              checkAnswer(spark.read.orc(path), correctAnswer)
+
+              withTable("t") {
+                sql(
+                  s"""
+                     |CREATE EXTERNAL TABLE t(c3 INT, c2 INT)
+                     |PARTITIONED BY (p int)
+                     |STORED AS ORC
+                     |LOCATION '$path'
+                     |""".stripMargin)
+                sql("MSCK REPAIR TABLE t")
+                val expected = if (forcePositionalEvolution) {
+                  correctAnswer
+                } else {
+                  Seq(Row(null, 2, 1), Row(null, 4, 2), Row(null, 6, 3), Row(null, null, 4))
+                }
+
+                checkAnswer(spark.table("t"), expected)
+              }
+            }
+          }
+        }
+      }
+    }
+  }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
index 91fd8a47339fc..e94e0b39c859c 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
@@ -149,12 +149,7 @@ class HiveOrcSourceSuite extends OrcSuite with TestHiveSingleton {
   test("Check BloomFilter creation") {
     Seq(true, false).foreach { convertMetastore =>
       withSQLConf(HiveUtils.CONVERT_METASTORE_ORC.key -> s"$convertMetastore") {
-        if (HiveUtils.isHive23) {
-          testBloomFilterCreation(org.apache.orc.OrcProto.Stream.Kind.BLOOM_FILTER_UTF8)
-        } else {
-          // Before ORC-101
-          testBloomFilterCreation(org.apache.orc.OrcProto.Stream.Kind.BLOOM_FILTER)
-        }
+        testBloomFilterCreation(org.apache.orc.OrcProto.Stream.Kind.BLOOM_FILTER_UTF8)
       }
     }
   }
@@ -162,7 +157,7 @@ class HiveOrcSourceSuite extends OrcSuite with TestHiveSingleton {
   test("Enforce direct encoding column-wise selectively") {
     Seq(true, false).foreach { convertMetastore =>
       withSQLConf(HiveUtils.CONVERT_METASTORE_ORC.key -> s"$convertMetastore") {
-        testSelectiveDictionaryEncoding(isSelective = false, isHive23 = HiveUtils.isHive23)
+        testSelectiveDictionaryEncoding(isSelective = false, isHiveOrc = true)
       }
     }
   }
@@ -322,7 +317,6 @@ class HiveOrcSourceSuite extends OrcSuite with TestHiveSingleton {
   }
 
   test("SPARK-31580: Read a file written before ORC-569") {
-    assume(HiveUtils.isHive23) // Hive 1.2 doesn't use Apache ORC
     // Test ORC file came from ORC-621
     val df = readResourceOrcFile("test-data/TestStringDictionary.testRowIndex.orc")
     assert(df.where("str < 'row 001000'").count() === 1000)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcReadBenchmark.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcReadBenchmark.scala
index a26412c5163ec..6acb03e919e68 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcReadBenchmark.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcReadBenchmark.scala
@@ -33,8 +33,7 @@ import org.apache.spark.sql.types._
  * {{{
  *   To run this benchmark:
  *   1. without sbt: bin/spark-submit --class <this class>
- *        --jars <catalyst test jar>,<core test jar>,<sql jar>,<hive-exec jar>,<spark-hive jar>
- *       <spark-hive test jar>
+ *        --jars <catalyst test jar>,<core test jar>,<spark sql test jar> <spark-hive test jar>
  *   2. build/sbt "hive/test:runMain <this class>"
  *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "hive/test:runMain <this class>"
  *      Results will be written to "benchmarks/OrcReadBenchmark-results.txt".
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/security/HiveHadoopDelegationTokenManagerSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/security/HiveHadoopDelegationTokenManagerSuite.scala
index 97eab4f3f4f77..f8f555197daef 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/security/HiveHadoopDelegationTokenManagerSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/security/HiveHadoopDelegationTokenManagerSuite.scala
@@ -52,8 +52,8 @@ class HiveHadoopDelegationTokenManagerSuite extends SparkFunSuite {
           throw new ClassNotFoundException(name)
         }
 
-        val prefixBlacklist = Seq("java", "scala", "com.sun.", "sun.")
-        if (prefixBlacklist.exists(name.startsWith(_))) {
+        val prefixExcludeList = Seq("java", "scala", "com.sun.", "sun.")
+        if (prefixExcludeList.exists(name.startsWith(_))) {
           return currentLoader.loadClass(name)
         }
 
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
index a3e2444cae887..ec73d7f71887f 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
@@ -23,7 +23,6 @@ import java.util.{Set => JavaSet}
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable
-import scala.language.implicitConversions
 
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.Path
@@ -38,11 +37,12 @@ import org.apache.spark.internal.config.UI._
 import org.apache.spark.sql.{DataFrame, Dataset, SparkSession, SQLContext}
 import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
 import org.apache.spark.sql.catalyst.catalog.ExternalCatalogWithListener
+import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode
 import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation
 import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, OneRowRelation}
+import org.apache.spark.sql.connector.catalog.CatalogManager
 import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
 import org.apache.spark.sql.execution.{QueryExecution, SQLExecution}
-import org.apache.spark.sql.execution.command.CacheTableCommand
 import org.apache.spark.sql.hive._
 import org.apache.spark.sql.hive.client.HiveClient
 import org.apache.spark.sql.internal.{SessionState, SharedState, SQLConf, WithTestConf}
@@ -58,6 +58,7 @@ object TestHive
       new SparkConf()
         .set("spark.sql.test", "")
         .set(SQLConf.CODEGEN_FALLBACK.key, "false")
+        .set(SQLConf.CODEGEN_FACTORY_MODE.key, CodegenObjectFactoryMode.CODEGEN_ONLY.toString)
         .set(HiveUtils.HIVE_METASTORE_BARRIER_PREFIXES.key,
           "org.apache.spark.sql.hive.execution.PairSerDe")
         .set(WAREHOUSE_PATH.key, TestHiveContext.makeWarehouseDir().toURI.getPath)
@@ -128,12 +129,12 @@ class TestHiveContext(
    * If loadTestTables is false, no test tables are loaded. Note that this flag can only be true
    * when running in the JVM, i.e. it needs to be false when calling from Python.
    */
-  def this(sc: SparkContext, loadTestTables: Boolean = true) {
-    this(new TestHiveSparkSession(HiveUtils.withHiveExternalCatalog(sc), loadTestTables))
+  def this(sc: SparkContext, loadTestTables: Boolean = true) = {
+    this(new TestHiveSparkSession(TestHiveContext.withHiveExternalCatalog(sc), loadTestTables))
   }
 
-  def this(sc: SparkContext, hiveClient: HiveClient) {
-    this(new TestHiveSparkSession(HiveUtils.withHiveExternalCatalog(sc),
+  def this(sc: SparkContext, hiveClient: HiveClient) = {
+    this(new TestHiveSparkSession(TestHiveContext.withHiveExternalCatalog(sc),
       hiveClient,
       loadTestTables = false))
   }
@@ -176,7 +177,7 @@ private[hive] class TestHiveSparkSession(
     private val loadTestTables: Boolean)
   extends SparkSession(sc) with Logging { self =>
 
-  def this(sc: SparkContext, loadTestTables: Boolean) {
+  def this(sc: SparkContext, loadTestTables: Boolean) = {
     this(
       sc,
       existingSharedState = None,
@@ -184,7 +185,7 @@ private[hive] class TestHiveSparkSession(
       loadTestTables)
   }
 
-  def this(sc: SparkContext, hiveClient: HiveClient, loadTestTables: Boolean) {
+  def this(sc: SparkContext, hiveClient: HiveClient, loadTestTables: Boolean) = {
     this(
       sc,
       existingSharedState = Some(new TestHiveSharedState(sc, Some(hiveClient))),
@@ -326,20 +327,22 @@ private[hive] class TestHiveSparkSession(
   }
 
   if (loadTestTables) {
+    def createTableSQL(tblName: String): String = {
+      s"CREATE TABLE $tblName (key INT, value STRING) STORED AS textfile"
+    }
     // The test tables that are defined in the Hive QTestUtil.
     // /itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
     // https://github.com/apache/hive/blob/branch-0.13/data/scripts/q_test_init.sql
     @transient
     val hiveQTestUtilTables: Seq[TestTable] = Seq(
       TestTable("src",
-        "CREATE TABLE src (key INT, value STRING) STORED AS TEXTFILE".cmd,
+        createTableSQL("src").cmd,
         s"LOAD DATA LOCAL INPATH '${quoteHiveFile("data/files/kv1.txt")}' INTO TABLE src".cmd),
       TestTable("src1",
-        "CREATE TABLE src1 (key INT, value STRING) STORED AS TEXTFILE".cmd,
+        createTableSQL("src1").cmd,
         s"LOAD DATA LOCAL INPATH '${quoteHiveFile("data/files/kv3.txt")}' INTO TABLE src1".cmd),
       TestTable("srcpart", () => {
-        "CREATE TABLE srcpart (key INT, value STRING) PARTITIONED BY (ds STRING, hr STRING)"
-          .cmd.apply()
+        s"${createTableSQL("srcpart")} PARTITIONED BY (ds STRING, hr STRING)".cmd.apply()
         for (ds <- Seq("2008-04-08", "2008-04-09"); hr <- Seq("11", "12")) {
           s"""
              |LOAD DATA LOCAL INPATH '${quoteHiveFile("data/files/kv1.txt")}'
@@ -348,8 +351,7 @@ private[hive] class TestHiveSparkSession(
         }
       }),
       TestTable("srcpart1", () => {
-        "CREATE TABLE srcpart1 (key INT, value STRING) PARTITIONED BY (ds STRING, hr INT)"
-          .cmd.apply()
+        s"${createTableSQL("srcpart1")} PARTITIONED BY (ds STRING, hr INT)".cmd.apply()
         for (ds <- Seq("2008-04-08", "2008-04-09"); hr <- 11 to 12) {
           s"""
              |LOAD DATA LOCAL INPATH '${quoteHiveFile("data/files/kv1.txt")}'
@@ -494,7 +496,10 @@ private[hive] class TestHiveSparkSession(
   def getLoadedTables: collection.mutable.HashSet[String] = sharedState.loadedTables
 
   def loadTestTable(name: String): Unit = {
-    if (!sharedState.loadedTables.contains(name)) {
+    // LOAD DATA does not work on temporary views. Since temporary views are resolved first,
+    // skip loading if there exists a temporary view with the given name.
+    if (sessionState.catalog.getTempView(name).isEmpty &&
+        !sharedState.loadedTables.contains(name)) {
       // Marks the table as loaded first to prevent infinite mutually recursive table loading.
       sharedState.loadedTables += name
       logDebug(s"Loading test table $name")
@@ -582,24 +587,22 @@ private[hive] class TestHiveQueryExecution(
     logicalPlan: LogicalPlan)
   extends QueryExecution(sparkSession, logicalPlan) with Logging {
 
-  def this(sparkSession: TestHiveSparkSession, sql: String) {
+  def this(sparkSession: TestHiveSparkSession, sql: String) = {
     this(sparkSession, sparkSession.sessionState.sqlParser.parsePlan(sql))
   }
 
-  def this(sql: String) {
+  def this(sql: String) = {
     this(TestHive.sparkSession, sql)
   }
 
   override lazy val analyzed: LogicalPlan = sparkSession.withActive {
-    val describedTables = logical match {
-      case CacheTableCommand(tbl, _, _, _) => tbl :: Nil
-      case _ => Nil
-    }
-
     // Make sure any test tables referenced are loaded.
-    val referencedTables =
-      describedTables ++
-        logical.collect { case UnresolvedRelation(ident) => ident.asTableIdentifier }
+    val referencedTables = logical.collect {
+      case UnresolvedRelation(ident, _, _) =>
+        if (ident.length > 1 && ident.head.equalsIgnoreCase(CatalogManager.SESSION_CATALOG_NAME)) {
+          ident.tail.asTableIdentifier
+        } else ident.asTableIdentifier
+    }
     val resolver = sparkSession.sessionState.conf.resolver
     val referencedTestTables = referencedTables.flatMap { tbl =>
       val testTableOpt = sparkSession.testTables.keys.find(resolver(_, tbl.table))
@@ -644,6 +647,10 @@ private[hive] object TestHiveContext {
     scratchDir
   }
 
+  private def withHiveExternalCatalog(sc: SparkContext): SparkContext = {
+    sc.conf.set(CATALOG_IMPLEMENTATION.key, "hive")
+    sc
+  }
 }
 
 private[sql] class TestHiveSessionStateBuilder(
@@ -677,7 +684,7 @@ private[hive] object HiveTestJars {
     val fileName = urlString.split("/").last
     val targetFile = new File(hiveTestJarsDir, fileName)
     if (!targetFile.exists()) {
-      Utils.doFetchFile(urlString, hiveTestJarsDir, fileName, new SparkConf, null, null)
+      Utils.doFetchFile(urlString, hiveTestJarsDir, fileName, new SparkConf, null)
     }
     targetFile
   }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/sources/DisableUnnecessaryBucketedScanWithHiveSupportSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/sources/DisableUnnecessaryBucketedScanWithHiveSupportSuite.scala
new file mode 100644
index 0000000000000..30eb93cb5c3e8
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/sources/DisableUnnecessaryBucketedScanWithHiveSupportSuite.scala
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.sources
+
+import org.apache.spark.sql.hive.test.TestHiveSingleton
+import org.apache.spark.sql.internal.StaticSQLConf.CATALOG_IMPLEMENTATION
+
+class DisableUnnecessaryBucketedScanWithHiveSupportSuite
+  extends DisableUnnecessaryBucketedScanSuite
+  with TestHiveSingleton {
+
+  protected override def beforeAll(): Unit = {
+    super.beforeAll()
+    assert(spark.sparkContext.conf.get(CATALOG_IMPLEMENTATION) == "hive")
+  }
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
index cbea74103343e..2637a1d18e345 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.sources
 
 import java.io.File
+import java.util.Locale
 
 import scala.util.Random
 
@@ -154,12 +155,23 @@ abstract class HadoopFsRelationTest extends QueryTest with SQLTestUtils with Tes
             java8ApiConfValues.foreach { java8Api =>
               withSQLConf(
                 SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString,
-                SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString,
-                SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString) {
+                SQLConf.PARQUET_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString,
+                SQLConf.PARQUET_INT96_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString,
+                SQLConf.AVRO_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString) {
                 val dataGenerator = RandomDataGenerator.forType(
                   dataType = dataType,
                   nullable = true,
-                  new Random(seed)
+                  new Random(seed),
+                  // TODO(SPARK-34440): Allow saving/loading datetime in ORC w/o rebasing
+                  // The ORC datasource always performs datetime rebasing that can lead to
+                  // shifting of the original dates/timestamps. For instance, 1582-10-06 is valid
+                  // date in the Proleptic Gregorian calendar but it does not exist in the Julian
+                  // calendar. The ORC datasource shifts the date to the next valid date 1582-10-15
+                  // during rebasing of this date to the Julian calendar. Since the test compares
+                  // the original date before saving and the date loaded back from the ORC files,
+                  // we set `validJulianDatetime` to `true` to generate only Proleptic Gregorian
+                  // dates that exist in the Julian calendar and will be not changed during rebase.
+                  validJulianDatetime = dataSourceName.toLowerCase(Locale.ROOT).contains("orc")
                 ).getOrElse {
                   fail(s"Failed to create data generator for schema $dataType")
                 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/sources/SimpleTextRelation.scala b/sql/hive/src/test/scala/org/apache/spark/sql/sources/SimpleTextRelation.scala
index d1b97b2852fbc..debe1ab734cc9 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/sources/SimpleTextRelation.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/sources/SimpleTextRelation.scala
@@ -117,7 +117,7 @@ class SimpleTextSource extends TextBasedFileFormat with DataSourceRegister {
   }
 }
 
-class SimpleTextOutputWriter(path: String, dataSchema: StructType, context: TaskAttemptContext)
+class SimpleTextOutputWriter(val path: String, dataSchema: StructType, context: TaskAttemptContext)
   extends OutputWriter {
 
   private val writer = CodecStreams.createOutputStreamWriter(context, new Path(path))
diff --git a/streaming/pom.xml b/streaming/pom.xml
index ea351d449481a..bd8d352092e73 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
@@ -109,7 +109,7 @@
     </dependency>
     <dependency>
       <groupId>org.seleniumhq.selenium</groupId>
-      <artifactId>selenium-htmlunit-driver</artifactId>
+      <artifactId>htmlunit-driver</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>
diff --git a/streaming/src/main/scala/org/apache/spark/status/api/v1/streaming/ApiStreamingRootResource.scala b/streaming/src/main/scala/org/apache/spark/status/api/v1/streaming/ApiStreamingRootResource.scala
index a2571b910f615..99d59e4a1447a 100644
--- a/streaming/src/main/scala/org/apache/spark/status/api/v1/streaming/ApiStreamingRootResource.scala
+++ b/streaming/src/main/scala/org/apache/spark/status/api/v1/streaming/ApiStreamingRootResource.scala
@@ -23,9 +23,7 @@ import javax.ws.rs.core.MediaType
 
 import org.apache.spark.status.api.v1.NotFoundException
 import org.apache.spark.streaming.Time
-import org.apache.spark.streaming.ui.StreamingJobProgressListener
 import org.apache.spark.streaming.ui.StreamingJobProgressListener._
-import org.apache.spark.ui.SparkUI
 
 @Produces(Array(MediaType.APPLICATION_JSON))
 private[v1] class ApiStreamingRootResource extends BaseStreamingAppResource {
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/DStreamGraph.scala b/streaming/src/main/scala/org/apache/spark/streaming/DStreamGraph.scala
index 683db21d3f0e1..43aaa7e1eeaec 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/DStreamGraph.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/DStreamGraph.scala
@@ -19,7 +19,7 @@ package org.apache.spark.streaming
 
 import java.io.{IOException, ObjectInputStream, ObjectOutputStream}
 
-import scala.collection.mutable.ArrayBuffer
+import scala.collection.mutable
 import scala.collection.parallel.immutable.ParVector
 
 import org.apache.spark.internal.Logging
@@ -29,8 +29,8 @@ import org.apache.spark.util.Utils
 
 final private[streaming] class DStreamGraph extends Serializable with Logging {
 
-  private val inputStreams = new ArrayBuffer[InputDStream[_]]()
-  private val outputStreams = new ArrayBuffer[DStream[_]]()
+  private var inputStreams = mutable.ArraySeq.empty[InputDStream[_]]
+  private var outputStreams = mutable.ArraySeq.empty[DStream[_]]
 
   @volatile private var inputStreamNameAndID: Seq[(String, Int)] = Nil
 
@@ -91,14 +91,14 @@ final private[streaming] class DStreamGraph extends Serializable with Logging {
   def addInputStream(inputStream: InputDStream[_]): Unit = {
     this.synchronized {
       inputStream.setGraph(this)
-      inputStreams += inputStream
+      inputStreams = inputStreams :+ inputStream
     }
   }
 
   def addOutputStream(outputStream: DStream[_]): Unit = {
     this.synchronized {
       outputStream.setGraph(this)
-      outputStreams += outputStream
+      outputStreams = outputStreams :+ outputStream
     }
   }
 
@@ -123,7 +123,7 @@ final private[streaming] class DStreamGraph extends Serializable with Logging {
         val jobOption = outputStream.generateJob(time)
         jobOption.foreach(_.setCallSite(outputStream.creationSite))
         jobOption
-      }
+      }.toSeq
     }
     logDebug("Generated " + jobs.length + " jobs for time " + time)
     jobs
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/State.scala b/streaming/src/main/scala/org/apache/spark/streaming/State.scala
index 734c6ef42696e..c4cd1a9dc336b 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/State.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/State.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.streaming
 
-import scala.language.implicitConversions
-
 import org.apache.spark.annotation.Experimental
 
 /**
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaStreamingContext.scala b/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaStreamingContext.scala
index 2d53a1b4c78b6..af3f5a060f54b 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaStreamingContext.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaStreamingContext.scala
@@ -67,7 +67,7 @@ class JavaStreamingContext(val ssc: StreamingContext) extends Closeable {
    * @param master Name of the Spark Master
    * @param appName Name to be used when registering with the scheduler
    * @param batchDuration The time interval at which streaming data will be divided into batches
-   * @param sparkHome The SPARK_HOME directory on the slave nodes
+   * @param sparkHome The SPARK_HOME directory on the worker nodes
    * @param jarFile JAR file containing job code, to ship to cluster. This can be a path on the
    *                local file system or an HDFS, HTTP, HTTPS, or FTP URL.
    */
@@ -84,7 +84,7 @@ class JavaStreamingContext(val ssc: StreamingContext) extends Closeable {
    * @param master Name of the Spark Master
    * @param appName Name to be used when registering with the scheduler
    * @param batchDuration The time interval at which streaming data will be divided into batches
-   * @param sparkHome The SPARK_HOME directory on the slave nodes
+   * @param sparkHome The SPARK_HOME directory on the worker nodes
    * @param jars Collection of JARs to send to the cluster. These can be paths on the local file
    *             system or HDFS, HTTP, HTTPS, or FTP URLs.
    */
@@ -101,7 +101,7 @@ class JavaStreamingContext(val ssc: StreamingContext) extends Closeable {
    * @param master Name of the Spark Master
    * @param appName Name to be used when registering with the scheduler
    * @param batchDuration The time interval at which streaming data will be divided into batches
-   * @param sparkHome The SPARK_HOME directory on the slave nodes
+   * @param sparkHome The SPARK_HOME directory on the worker nodes
    * @param jars Collection of JARs to send to the cluster. These can be paths on the local file
    *             system or HDFS, HTTP, HTTPS, or FTP URLs.
    * @param environment Environment variables to set on worker nodes
@@ -366,7 +366,7 @@ class JavaStreamingContext(val ssc: StreamingContext) extends Closeable {
     implicit val cm: ClassTag[T] =
       implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[T]]
     val sQueue = new scala.collection.mutable.Queue[RDD[T]]
-    sQueue.enqueue(queue.asScala.map(_.rdd).toSeq: _*)
+    sQueue ++= queue.asScala.map(_.rdd)
     ssc.queueStream(sQueue)
   }
 
@@ -390,7 +390,7 @@ class JavaStreamingContext(val ssc: StreamingContext) extends Closeable {
     implicit val cm: ClassTag[T] =
       implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[T]]
     val sQueue = new scala.collection.mutable.Queue[RDD[T]]
-    sQueue.enqueue(queue.asScala.map(_.rdd).toSeq: _*)
+    sQueue ++= queue.asScala.map(_.rdd)
     ssc.queueStream(sQueue, oneAtATime)
   }
 
@@ -415,7 +415,7 @@ class JavaStreamingContext(val ssc: StreamingContext) extends Closeable {
     implicit val cm: ClassTag[T] =
       implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[T]]
     val sQueue = new scala.collection.mutable.Queue[RDD[T]]
-    sQueue.enqueue(queue.asScala.map(_.rdd).toSeq: _*)
+    sQueue ++= queue.asScala.map(_.rdd)
     ssc.queueStream(sQueue, oneAtATime, defaultRDD.rdd)
   }
 
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaStreamingListenerWrapper.scala b/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaStreamingListenerWrapper.scala
index ee8370d262609..7555e2f57fccb 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaStreamingListenerWrapper.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaStreamingListenerWrapper.scala
@@ -65,7 +65,7 @@ private[streaming] class JavaStreamingListenerWrapper(javaStreamingListener: Jav
   private def toJavaBatchInfo(batchInfo: BatchInfo): JavaBatchInfo = {
     JavaBatchInfo(
       batchInfo.batchTime,
-      batchInfo.streamIdToInputInfo.mapValues(toJavaStreamInputInfo(_)).asJava,
+      batchInfo.streamIdToInputInfo.mapValues(toJavaStreamInputInfo).toMap.asJava,
       batchInfo.submissionTime,
       batchInfo.processingStartTime.getOrElse(-1),
       batchInfo.processingEndTime.getOrElse(-1),
@@ -73,7 +73,7 @@ private[streaming] class JavaStreamingListenerWrapper(javaStreamingListener: Jav
       batchInfo.processingDelay.getOrElse(-1),
       batchInfo.totalDelay.getOrElse(-1),
       batchInfo.numRecords,
-      batchInfo.outputOperationInfos.mapValues(toJavaOutputOperationInfo(_)).asJava
+      batchInfo.outputOperationInfos.mapValues(toJavaOutputOperationInfo).toMap.asJava
     )
   }
 
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/api/python/PythonDStream.scala b/streaming/src/main/scala/org/apache/spark/streaming/api/python/PythonDStream.scala
index 570663c6f6ad3..7a8e3f1d2ccf4 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/api/python/PythonDStream.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/api/python/PythonDStream.scala
@@ -163,7 +163,7 @@ private[python] object PythonTransformFunctionSerializer {
 private[streaming] object PythonDStream {
 
   /**
-   * can not access PythonTransformFunctionSerializer.register() via Py4j
+   * cannot access PythonTransformFunctionSerializer.register() via Py4j
    * Py4JError: PythonTransformFunctionSerializerregister does not exist in the JVM
    */
   def registerSerializer(ser: PythonTransformFunctionSerializer): Unit = {
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/dstream/DStream.scala b/streaming/src/main/scala/org/apache/spark/streaming/dstream/DStream.scala
index e037f26088347..ca4f3670d5ad7 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/dstream/DStream.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/dstream/DStream.scala
@@ -960,7 +960,7 @@ object DStream {
   /** Get the creation site of a DStream from the stack trace of when the DStream is created. */
   private[streaming] def getCreationSite(): CallSite = {
     /** Filtering function that excludes non-user classes for a streaming application */
-    def streamingExclustionFunction(className: String): Boolean = {
+    def streamingExclusionFunction(className: String): Boolean = {
       def doesMatch(r: Regex): Boolean = r.findFirstIn(className).isDefined
       val isSparkClass = doesMatch(SPARK_CLASS_REGEX)
       val isSparkExampleClass = doesMatch(SPARK_EXAMPLES_CLASS_REGEX)
@@ -972,6 +972,6 @@ object DStream {
       // non-Spark and non-Scala class, as the rest would streaming application classes.
       (isSparkClass || isScalaClass) && !isSparkExampleClass && !isSparkStreamingTestClass
     }
-    org.apache.spark.util.Utils.getCallSite(streamingExclustionFunction)
+    org.apache.spark.util.Utils.getCallSite(streamingExclusionFunction)
   }
 }
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/dstream/UnionDStream.scala b/streaming/src/main/scala/org/apache/spark/streaming/dstream/UnionDStream.scala
index d46c0a01e05d9..2f4536ec6f0c2 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/dstream/UnionDStream.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/dstream/UnionDStream.scala
@@ -45,7 +45,7 @@ class UnionDStream[T: ClassTag](parents: Array[DStream[T]])
         s" time $validTime")
     }
     if (rdds.nonEmpty) {
-      Some(ssc.sc.union(rdds))
+      Some(ssc.sc.union(rdds.toSeq))
     } else {
       None
     }
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/rdd/MapWithStateRDD.scala b/streaming/src/main/scala/org/apache/spark/streaming/rdd/MapWithStateRDD.scala
index 8da5a5f8193cf..662312b7b0db8 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/rdd/MapWithStateRDD.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/rdd/MapWithStateRDD.scala
@@ -75,7 +75,7 @@ private[streaming] object MapWithStateRDDRecord {
       }
     }
 
-    MapWithStateRDDRecord(newStateMap, mappedData)
+    MapWithStateRDDRecord(newStateMap, mappedData.toSeq)
   }
 }
 
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/rdd/WriteAheadLogBackedBlockRDD.scala b/streaming/src/main/scala/org/apache/spark/streaming/rdd/WriteAheadLogBackedBlockRDD.scala
index f677c492d561f..6494e512713f8 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/rdd/WriteAheadLogBackedBlockRDD.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/rdd/WriteAheadLogBackedBlockRDD.scala
@@ -96,7 +96,7 @@ class WriteAheadLogBackedBlockRDD[T: ClassTag](
   @transient private val hadoopConfig = sc.hadoopConfiguration
   private val broadcastedHadoopConf = new SerializableConfiguration(hadoopConfig)
 
-  override def isValid(): Boolean = true
+  override def isValid: Boolean = true
 
   override def getPartitions: Array[Partition] = {
     assertValid()
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/receiver/ReceivedBlockHandler.scala b/streaming/src/main/scala/org/apache/spark/streaming/receiver/ReceivedBlockHandler.scala
index 12ed8015117e5..7a561ecb4990f 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/receiver/ReceivedBlockHandler.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/receiver/ReceivedBlockHandler.scala
@@ -234,7 +234,7 @@ private[streaming] class CountingIterator[T](iterator: Iterator[T]) extends Iter
 
    private def isFullyConsumed: Boolean = !iterator.hasNext
 
-   def hasNext(): Boolean = iterator.hasNext
+   def hasNext: Boolean = iterator.hasNext
 
    def count(): Option[Long] = {
      if (isFullyConsumed) Some(_count) else None
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ExecutorAllocationManager.scala b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ExecutorAllocationManager.scala
index 58bd56c591d04..1037950a4424f 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ExecutorAllocationManager.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ExecutorAllocationManager.scala
@@ -22,8 +22,10 @@ import scala.util.Random
 
 import org.apache.spark.{ExecutorAllocationClient, SparkConf}
 import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config.DECOMMISSION_ENABLED
 import org.apache.spark.internal.config.Streaming._
 import org.apache.spark.resource.ResourceProfile
+import org.apache.spark.scheduler.ExecutorDecommissionInfo
 import org.apache.spark.streaming.util.RecurringTimer
 import org.apache.spark.util.{Clock, Utils}
 
@@ -133,7 +135,13 @@ private[streaming] class ExecutorAllocationManager(
       logDebug(s"Removable executors (${removableExecIds.size}): ${removableExecIds}")
       if (removableExecIds.nonEmpty) {
         val execIdToRemove = removableExecIds(Random.nextInt(removableExecIds.size))
-        client.killExecutor(execIdToRemove)
+        if (conf.get(DECOMMISSION_ENABLED)) {
+          client.decommissionExecutor(execIdToRemove,
+            ExecutorDecommissionInfo("spark scale down", None),
+            adjustTargetNumExecutors = true)
+        } else {
+          client.killExecutor(execIdToRemove)
+        }
         logInfo(s"Requested to kill executor $execIdToRemove")
       } else {
         logInfo(s"No non-receiver executors to kill")
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceivedBlockTracker.scala b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceivedBlockTracker.scala
index 6c71b18b46213..4ac1c62822e7a 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceivedBlockTracker.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceivedBlockTracker.scala
@@ -21,7 +21,6 @@ import java.nio.ByteBuffer
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable
-import scala.language.implicitConversions
 import scala.util.control.NonFatal
 
 import org.apache.hadoop.conf.Configuration
@@ -116,7 +115,9 @@ private[streaming] class ReceivedBlockTracker(
       // a few thousand elements.  So we explicitly allocate a collection for serialization which
       // we know doesn't have this issue.  (See SPARK-26734).
       val streamIdToBlocks = streamIds.map { streamId =>
-        (streamId, mutable.ArrayBuffer(getReceivedBlockQueue(streamId).clone(): _*))
+        val blocks = mutable.ArrayBuffer[ReceivedBlockInfo]()
+        blocks ++= getReceivedBlockQueue(streamId).clone()
+        (streamId, blocks.toSeq)
       }.toMap
       val allocatedBlocks = AllocatedBlocks(streamIdToBlocks)
       if (writeToLog(BatchAllocationEvent(batchTime, allocatedBlocks))) {
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceiverSchedulingPolicy.scala b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceiverSchedulingPolicy.scala
index 4105171a3db24..0e4a64f6c0e4b 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceiverSchedulingPolicy.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceiverSchedulingPolicy.scala
@@ -128,14 +128,14 @@ private[streaming] class ReceiverSchedulingPolicy {
     }
 
     // Assign idle executors to receivers that have less executors
-    val idleExecutors = numReceiversOnExecutor.filter(_._2 == 0).map(_._1)
+    val idleExecutors = numReceiversOnExecutor.filter(_._2 == 0).keys
     for (executor <- idleExecutors) {
       // Assign an idle executor to the receiver that has least candidate executors.
       val leastScheduledExecutors = scheduledLocations.minBy(_.size)
       leastScheduledExecutors += executor
     }
 
-    receivers.map(_.streamId).zip(scheduledLocations).toMap
+    receivers.map(_.streamId).zip(scheduledLocations.map(_.toSeq)).toMap
   }
 
   /**
@@ -183,7 +183,7 @@ private[streaming] class ReceiverSchedulingPolicy {
 
     val executorWeights: Map[ExecutorCacheTaskLocation, Double] = {
       receiverTrackingInfoMap.values.flatMap(convertReceiverTrackingInfoToExecutorWeights)
-        .groupBy(_._1).mapValues(_.map(_._2).sum) // Sum weights for each executor
+        .groupBy(_._1).mapValues(_.map(_._2).sum).toMap // Sum weights for each executor
     }
 
     val idleExecutors = executors.toSet -- executorWeights.keys
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceiverTracker.scala b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceiverTracker.scala
index 13cf5cc0e71ea..342a0a43b5068 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceiverTracker.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceiverTracker.scala
@@ -248,7 +248,7 @@ class ReceiverTracker(ssc: StreamingContext, skipReceiverLaunch: Boolean = false
         _.runningExecutor.map {
           _.executorId
         }
-      }
+      }.toMap
     } else {
       Map.empty
     }
@@ -415,7 +415,7 @@ class ReceiverTracker(ssc: StreamingContext, skipReceiverLaunch: Boolean = false
   }
 
   /**
-   * Run the dummy Spark job to ensure that all slaves have registered. This avoids all the
+   * Run the dummy Spark job to ensure that all executors have registered. This avoids all the
    * receivers to be scheduled on the same node.
    *
    * TODO Should poll the executor number and wait for executors according to
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/AllBatchesTable.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/AllBatchesTable.scala
index 1e443f656734c..c0eec0e0b0a85 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/ui/AllBatchesTable.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/AllBatchesTable.scala
@@ -17,30 +17,41 @@
 
 package org.apache.spark.streaming.ui
 
-import scala.xml.Node
-
-import org.apache.spark.ui.{UIUtils => SparkUIUtils}
+import java.net.URLEncoder
+import java.nio.charset.StandardCharsets.UTF_8
+import javax.servlet.http.HttpServletRequest
 
-private[ui] abstract class BatchTableBase(tableId: String, batchInterval: Long) {
+import scala.xml.Node
 
-  protected def columns: Seq[Node] = {
-    <th>Batch Time</th>
-      <th>Records</th>
-      <th>Scheduling Delay
-        {SparkUIUtils.tooltip("Time taken by Streaming scheduler to submit jobs of a batch", "top")}
-      </th>
-      <th>Processing Time
-        {SparkUIUtils.tooltip("Time taken to process all jobs of a batch", "top")}</th>
-  }
+import org.apache.spark.ui.{PagedDataSource, PagedTable, UIUtils => SparkUIUtils}
+
+private[ui] class StreamingPagedTable(
+    request: HttpServletRequest,
+    tableTag: String,
+    batches: Seq[BatchUIData],
+    basePath: String,
+    subPath: String,
+    batchInterval: Long) extends PagedTable[BatchUIData] {
+
+  private val(sortColumn, desc, pageSize) = getTableParameters(request, tableTag, "Batch Time")
+  private val parameterPath = s"$basePath/$subPath/?${getParameterOtherTable(request, tableTag)}"
+  private val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
+
+  private val firstFailureReason: Option[String] =
+    if (!tableTag.equals("waitingBatches")) {
+      getFirstFailureReason(batches)
+    } else {
+      None
+    }
 
   /**
    * Return the first failure reason if finding in the batches.
    */
-  protected def getFirstFailureReason(batches: Seq[BatchUIData]): Option[String] = {
+  private def getFirstFailureReason(batches: Seq[BatchUIData]): Option[String] = {
     batches.flatMap(_.outputOperations.flatMap(_._2.failureReason)).headOption
   }
 
-  protected def getFirstFailureTableCell(batch: BatchUIData): Seq[Node] = {
+  private def getFirstFailureTableCell(batch: BatchUIData): Seq[Node] = {
     val firstFailureReason = batch.outputOperations.flatMap(_._2.failureReason).headOption
     firstFailureReason.map { failureReason =>
       val failureReasonForUI = UIUtils.createOutputOperationFailureForUI(failureReason)
@@ -49,147 +60,154 @@ private[ui] abstract class BatchTableBase(tableId: String, batchInterval: Long)
     }.getOrElse(<td>-</td>)
   }
 
-  protected def baseRow(batch: BatchUIData): Seq[Node] = {
-    val batchTime = batch.batchTime.milliseconds
-    val formattedBatchTime = SparkUIUtils.formatBatchTime(batchTime, batchInterval)
-    val numRecords = batch.numRecords
-    val schedulingDelay = batch.schedulingDelay
-    val formattedSchedulingDelay = schedulingDelay.map(SparkUIUtils.formatDuration).getOrElse("-")
-    val processingTime = batch.processingDelay
-    val formattedProcessingTime = processingTime.map(SparkUIUtils.formatDuration).getOrElse("-")
-    val batchTimeId = s"batch-$batchTime"
-
-    <td id={batchTimeId} sorttable_customkey={batchTime.toString}
-        isFailed={batch.isFailed.toString}>
-      <a href={s"batch?id=$batchTime"}>
-        {formattedBatchTime}
-      </a>
-    </td>
-      <td sorttable_customkey={numRecords.toString}>{numRecords.toString} records</td>
-      <td sorttable_customkey={schedulingDelay.getOrElse(Long.MaxValue).toString}>
-        {formattedSchedulingDelay}
-      </td>
-      <td sorttable_customkey={processingTime.getOrElse(Long.MaxValue).toString}>
-        {formattedProcessingTime}
-      </td>
-  }
-
-  private def batchTable: Seq[Node] = {
-    <table id={tableId} class="table table-bordered table-striped table-sm sortable">
-      <thead>
-        {columns}
-      </thead>
-      <tbody>
-        {renderRows}
-      </tbody>
-    </table>
-  }
-
-  def toNodeSeq: Seq[Node] = {
-    batchTable
-  }
-
-  protected def createOutputOperationProgressBar(batch: BatchUIData): Seq[Node] = {
+  private def createOutputOperationProgressBar(batch: BatchUIData): Seq[Node] = {
     <td class="progress-cell">
       {
-      SparkUIUtils.makeProgressBar(
-        started = batch.numActiveOutputOp,
-        completed = batch.numCompletedOutputOp,
-        failed = batch.numFailedOutputOp,
-        skipped = 0,
-        reasonToNumKilled = Map.empty,
-        total = batch.outputOperations.size)
+        SparkUIUtils.makeProgressBar(
+          started = batch.numActiveOutputOp,
+          completed = batch.numCompletedOutputOp,
+          failed = batch.numFailedOutputOp,
+          skipped = 0,
+          reasonToNumKilled = Map.empty,
+          total = batch.outputOperations.size)
       }
     </td>
   }
 
-  /**
-   * Return HTML for all rows of this table.
-   */
-  protected def renderRows: Seq[Node]
-}
+  override def tableId: String = s"$tableTag-table"
 
-private[ui] class ActiveBatchTable(
-    runningBatches: Seq[BatchUIData],
-    waitingBatches: Seq[BatchUIData],
-    batchInterval: Long) extends BatchTableBase("active-batches-table", batchInterval) {
+  override def tableCssClass: String =
+    "table table-bordered table-sm table-striped table-head-clickable table-cell-width-limited"
 
-  private val firstFailureReason = getFirstFailureReason(runningBatches)
+  override def pageSizeFormField: String = s"$tableTag.pageSize"
 
-  override protected def columns: Seq[Node] = super.columns ++ {
-    <th>Output Ops: Succeeded/Total</th>
-      <th>Status</th> ++ {
-      if (firstFailureReason.nonEmpty) {
-        <th>Error</th>
-      } else {
-        Nil
-      }
-    }
-  }
+  override def pageNumberFormField: String = s"$tableTag.page"
 
-  override protected def renderRows: Seq[Node] = {
-    // The "batchTime"s of "waitingBatches" must be greater than "runningBatches"'s, so display
-    // waiting batches before running batches
-    waitingBatches.flatMap(batch => <tr>{waitingBatchRow(batch)}</tr>) ++
-      runningBatches.flatMap(batch => <tr>{runningBatchRow(batch)}</tr>)
+  override def pageLink(page: Int): String = {
+    parameterPath +
+    s"&$tableTag.sort=$encodedSortColumn" +
+    s"&$tableTag.desc=$desc" +
+    s"&$pageNumberFormField=$page" +
+    s"&$pageSizeFormField=$pageSize" +
+    s"#$tableTag"
   }
 
-  private def runningBatchRow(batch: BatchUIData): Seq[Node] = {
-    baseRow(batch) ++ createOutputOperationProgressBar(batch) ++ <td>processing</td> ++ {
-      if (firstFailureReason.nonEmpty) {
-        getFirstFailureTableCell(batch)
-      } else {
-        Nil
+  override def goButtonFormPath: String =
+    s"$parameterPath&$tableTag.sort=$encodedSortColumn&$tableTag.desc=$desc#$tableTag"
+
+  override def dataSource: PagedDataSource[BatchUIData] =
+    new StreamingDataSource(batches, pageSize, sortColumn, desc)
+
+  override def headers: Seq[Node] = {
+    // headers, sortable and tooltips
+    val headersAndCssClasses: Seq[(String, Boolean, Option[String])] = {
+      Seq(
+        ("Batch Time", true, None),
+        ("Records", true, None),
+        ("Scheduling Delay", true, Some("Time taken by Streaming scheduler to submit jobs " +
+          "of a batch")),
+        ("Processing Time", true, Some("Time taken to process all jobs of a batch"))) ++ {
+        if (tableTag.equals("completedBatches")) {
+          Seq(
+            ("Total Delay", true, Some("Total time taken to handle a batch")),
+            ("Output Ops: Succeeded/Total", false, None))
+        } else {
+          Seq(
+            ("Output Ops: Succeeded/Total", false, None),
+            ("Status", false, None))
+        }
+      } ++ {
+        if (firstFailureReason.nonEmpty) {
+          Seq(("Error", false, None))
+        } else {
+          Nil
+        }
       }
     }
+    // check if sort column is a valid sortable column
+    isSortColumnValid(headersAndCssClasses, sortColumn)
+
+    headerRow(headersAndCssClasses, desc, pageSize, sortColumn, parameterPath, tableTag, tableTag)
   }
 
-  private def waitingBatchRow(batch: BatchUIData): Seq[Node] = {
-    baseRow(batch) ++ createOutputOperationProgressBar(batch) ++ <td>queued</td>++ {
-      if (firstFailureReason.nonEmpty) {
-        // Waiting batches have not run yet, so must have no failure reasons.
-        <td>-</td>
-      } else {
-        Nil
+  override def row(batch: BatchUIData): Seq[Node] = {
+    val batchTime = batch.batchTime.milliseconds
+    val formattedBatchTime = SparkUIUtils.formatBatchTime(batchTime, batchInterval)
+    val numRecords = batch.numRecords
+    val schedulingDelay = batch.schedulingDelay
+    val formattedSchedulingDelay = schedulingDelay.map(SparkUIUtils.formatDuration).getOrElse("-")
+    val processingTime = batch.processingDelay
+    val formattedProcessingTime = processingTime.map(SparkUIUtils.formatDuration).getOrElse("-")
+    val batchTimeId = s"batch-$batchTime"
+    val totalDelay = batch.totalDelay
+    val formattedTotalDelay = totalDelay.map(SparkUIUtils.formatDuration).getOrElse("-")
+
+    <tr>
+      <td id={batchTimeId} isFailed={batch.isFailed.toString}>
+        <a href={s"batch?id=$batchTime"}>
+          {formattedBatchTime}
+        </a>
+      </td>
+      <td> {numRecords.toString} records </td>
+      <td> {formattedSchedulingDelay} </td>
+      <td> {formattedProcessingTime} </td>
+      {
+        if (tableTag.equals("completedBatches")) {
+          <td> {formattedTotalDelay} </td> ++
+          createOutputOperationProgressBar(batch) ++ {
+            if (firstFailureReason.nonEmpty) {
+              getFirstFailureTableCell(batch)
+            } else {
+              Nil
+            }
+          }
+        } else if (tableTag.equals("runningBatches")) {
+          createOutputOperationProgressBar(batch) ++
+          <td> processing </td>  ++ {
+            if (firstFailureReason.nonEmpty) {
+              getFirstFailureTableCell(batch)
+            } else {
+              Nil
+            }
+          }
+        } else {
+          createOutputOperationProgressBar(batch) ++
+          <td> queued </td> ++ {
+            if (firstFailureReason.nonEmpty) {
+              // Waiting batches have not run yet, so must have no failure reasons.
+              <td>-</td>
+            } else {
+              Nil
+            }
+          }
+        }
       }
-    }
+    </tr>
   }
 }
 
-private[ui] class CompletedBatchTable(batches: Seq[BatchUIData], batchInterval: Long)
-  extends BatchTableBase("completed-batches-table", batchInterval) {
+private[ui] class StreamingDataSource(info: Seq[BatchUIData], pageSize: Int, sortColumn: String,
+    desc: Boolean) extends PagedDataSource[BatchUIData](pageSize) {
 
-  private val firstFailureReason = getFirstFailureReason(batches)
+  private val data = info.sorted(ordering(sortColumn, desc))
 
-  override protected def columns: Seq[Node] = super.columns ++ {
-    <th>Total Delay {SparkUIUtils.tooltip("Total time taken to handle a batch", "top")}</th>
-      <th>Output Ops: Succeeded/Total</th> ++ {
-      if (firstFailureReason.nonEmpty) {
-        <th>Error</th>
-      } else {
-        Nil
-      }
-    }
-  }
+  override protected def dataSize: Int = data.size
 
-  override protected def renderRows: Seq[Node] = {
-    batches.flatMap(batch => <tr>{completedBatchRow(batch)}</tr>)
-  }
+  override protected def sliceData(from: Int, to: Int): Seq[BatchUIData] = data.slice(from, to)
 
-  private def completedBatchRow(batch: BatchUIData): Seq[Node] = {
-    val totalDelay = batch.totalDelay
-    val formattedTotalDelay = totalDelay.map(SparkUIUtils.formatDuration).getOrElse("-")
-
-    baseRow(batch) ++ {
-      <td sorttable_customkey={totalDelay.getOrElse(Long.MaxValue).toString}>
-        {formattedTotalDelay}
-      </td>
-    } ++ createOutputOperationProgressBar(batch)++ {
-      if (firstFailureReason.nonEmpty) {
-        getFirstFailureTableCell(batch)
-      } else {
-        Nil
-      }
+  private def ordering(column: String, desc: Boolean): Ordering[BatchUIData] = {
+    val ordering: Ordering[BatchUIData] = column match {
+      case "Batch Time" => Ordering.by(_.batchTime.milliseconds)
+      case "Records" => Ordering.by(_.numRecords)
+      case "Scheduling Delay" => Ordering.by(_.schedulingDelay.getOrElse(Long.MaxValue))
+      case "Processing Time" => Ordering.by(_.processingDelay.getOrElse(Long.MaxValue))
+      case "Total Delay" => Ordering.by(_.totalDelay.getOrElse(Long.MaxValue))
+      case unknownColumn => throw new IllegalArgumentException(s"Unknown Column: $unknownColumn")
+    }
+    if (desc) {
+      ordering.reverse
+    } else {
+      ordering
     }
   }
 }
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala
index 3bdf009dbce66..42d0e50a068ec 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala
@@ -20,10 +20,12 @@ package org.apache.spark.streaming.ui
 import java.util.concurrent.TimeUnit
 import javax.servlet.http.HttpServletRequest
 
+import scala.collection.mutable
 import scala.xml.{Node, Unparsed}
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.ui.{GraphUIData, JsCollector, UIUtils => SparkUIUtils, WebUIPage}
+import org.apache.spark.util.Utils
 
 /**
  * A helper class for "scheduling delay", "processing time" and "total delay" to generate data that
@@ -86,7 +88,7 @@ private[ui] class StreamingPage(parent: StreamingTab)
       onClickTimelineFunc ++ basicInfo ++
       listener.synchronized {
         generateStatTable() ++
-          generateBatchListTables()
+          generateBatchListTables(request)
       }
     SparkUIUtils.headerSparkPage(request, "Streaming Statistics", content, parent)
   }
@@ -432,50 +434,97 @@ private[ui] class StreamingPage(parent: StreamingTab)
     </tr>
   }
 
-  private def generateBatchListTables(): Seq[Node] = {
+  private def streamingTable(request: HttpServletRequest, batches: Seq[BatchUIData],
+      tableTag: String): Seq[Node] = {
+    val interval: Long = listener.batchDuration
+    val streamingPage = Option(request.getParameter(s"$tableTag.page")).map(_.toInt).getOrElse(1)
+
+    try {
+      new StreamingPagedTable(
+        request,
+        tableTag,
+        batches,
+        SparkUIUtils.prependBaseUri(request, parent.basePath),
+        "streaming",
+        interval
+      ).table(streamingPage)
+    } catch {
+      case e @ (_: IllegalArgumentException | _: IndexOutOfBoundsException) =>
+        <div class="alert alert-error">
+          <p>Error while rendering streaming table:</p>
+          <pre>
+            {Utils.exceptionString(e)}
+          </pre>
+        </div>
+    }
+  }
+
+  private def generateBatchListTables(request: HttpServletRequest): Seq[Node] = {
     val runningBatches = listener.runningBatches.sortBy(_.batchTime.milliseconds).reverse
     val waitingBatches = listener.waitingBatches.sortBy(_.batchTime.milliseconds).reverse
     val completedBatches = listener.retainedCompletedBatches.
       sortBy(_.batchTime.milliseconds).reverse
 
-    val activeBatchesContent = {
-      <div class="row">
-        <div class="col-12">
-          <span id="activeBatches" class="collapse-aggregated-activeBatches collapse-table"
-                onClick="collapseTable('collapse-aggregated-activeBatches',
-                'aggregated-activeBatches')">
-            <h4>
-              <span class="collapse-table-arrow arrow-open"></span>
-              <a>Active Batches ({runningBatches.size + waitingBatches.size})</a>
-            </h4>
-          </span>
-          <div class="aggregated-activeBatches collapsible-table">
-            {new ActiveBatchTable(runningBatches, waitingBatches, listener.batchDuration).toNodeSeq}
+    val content = mutable.ListBuffer[Node]()
+
+    if (runningBatches.nonEmpty) {
+      content ++=
+        <div class="row">
+          <div class="col-12">
+            <span id="runningBatches" class="collapse-aggregated-runningBatches collapse-table"
+                  onClick="collapseTable('collapse-aggregated-runningBatches',
+                  'aggregated-runningBatches')">
+              <h4>
+                <span class="collapse-table-arrow arrow-open"></span>
+                <a>Running Batches ({runningBatches.size})</a>
+              </h4>
+            </span>
+            <div class="aggregated-runningBatches collapsible-table">
+              { streamingTable(request, runningBatches, "runningBatches") }
+            </div>
           </div>
         </div>
-      </div>
     }
 
-    val completedBatchesContent = {
-      <div class="row">
-        <div class="col-12">
-          <span id="completedBatches" class="collapse-aggregated-completedBatches collapse-table"
-                onClick="collapseTable('collapse-aggregated-completedBatches',
-                'aggregated-completedBatches')">
-            <h4>
-              <span class="collapse-table-arrow arrow-open"></span>
-              <a>Completed Batches (last {completedBatches.size}
-                out of {listener.numTotalCompletedBatches})</a>
-            </h4>
-          </span>
-          <div class="aggregated-completedBatches collapsible-table">
-            {new CompletedBatchTable(completedBatches, listener.batchDuration).toNodeSeq}
+    if (waitingBatches.nonEmpty) {
+      content ++=
+        <div class="row">
+          <div class="col-12">
+            <span id="waitingBatches" class="collapse-aggregated-waitingBatches collapse-table"
+                  onClick="collapseTable('collapse-aggregated-waitingBatches',
+                  'aggregated-waitingBatches')">
+              <h4>
+                <span class="collapse-table-arrow arrow-open"></span>
+                <a>Waiting Batches ({waitingBatches.size})</a>
+              </h4>
+            </span>
+            <div class="aggregated-waitingBatches collapsible-table">
+              { streamingTable(request, waitingBatches, "waitingBatches") }
+            </div>
           </div>
         </div>
-      </div>
     }
 
-    activeBatchesContent ++ completedBatchesContent
+    if (completedBatches.nonEmpty) {
+      content ++=
+        <div class="row">
+          <div class="col-12">
+            <span id="completedBatches" class="collapse-aggregated-completedBatches collapse-table"
+                  onClick="collapseTable('collapse-aggregated-completedBatches',
+                  'aggregated-completedBatches')">
+              <h4>
+                <span class="collapse-table-arrow arrow-open"></span>
+                <a>Completed Batches (last {completedBatches.size}
+                  out of {listener.numTotalCompletedBatches})</a>
+              </h4>
+            </span>
+            <div class="aggregated-completedBatches collapsible-table">
+              { streamingTable(request, completedBatches, "completedBatches") }
+            </div>
+          </div>
+        </div>
+    }
+    content
   }
 }
 
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/util/BatchedWriteAheadLog.scala b/streaming/src/main/scala/org/apache/spark/streaming/util/BatchedWriteAheadLog.scala
index 31e4c6b59a64a..d0a3517af70b9 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/util/BatchedWriteAheadLog.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/util/BatchedWriteAheadLog.scala
@@ -170,7 +170,7 @@ private[util] class BatchedWriteAheadLog(val wrappedLog: WriteAheadLog, conf: Sp
         // We take the latest record for the timestamp. Please refer to the class Javadoc for
         // detailed explanation
         val time = sortedByTime.last.time
-        segment = wrappedLog.write(aggregate(sortedByTime), time)
+        segment = wrappedLog.write(aggregate(sortedByTime.toSeq), time)
       }
       buffer.foreach(_.promise.success(segment))
     } catch {
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/util/FileBasedWriteAheadLog.scala b/streaming/src/main/scala/org/apache/spark/streaming/util/FileBasedWriteAheadLog.scala
index d33f83c819086..d1f9dfb791355 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/util/FileBasedWriteAheadLog.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/util/FileBasedWriteAheadLog.scala
@@ -146,7 +146,7 @@ private[streaming] class FileBasedWriteAheadLog(
     } else {
       // For performance gains, it makes sense to parallelize the recovery if
       // closeFileAfterWrite = true
-      seqToParIterator(executionContext, logFilesToRead, readFile).asJava
+      seqToParIterator(executionContext, logFilesToRead.toSeq, readFile).asJava
     }
   }
 
@@ -277,10 +277,10 @@ private[streaming] object FileBasedWriteAheadLog {
   }
 
   def getCallerName(): Option[String] = {
-    val blacklist = Seq("WriteAheadLog", "Logging", "java.lang", "scala.")
+    val ignoreList = Seq("WriteAheadLog", "Logging", "java.lang", "scala.")
     Thread.currentThread.getStackTrace()
       .map(_.getClassName)
-      .find { c => !blacklist.exists(c.contains) }
+      .find { c => !ignoreList.exists(c.contains) }
       .flatMap(_.split("\\.").lastOption)
       .flatMap(_.split("\\$\\$").headOption)
   }
@@ -293,7 +293,7 @@ private[streaming] object FileBasedWriteAheadLog {
           val startTime = startTimeStr.toLong
           val stopTime = stopTimeStr.toLong
           Some(LogInfo(startTime, stopTime, file.toString))
-        case None =>
+        case None | Some(_) =>
           None
       }
     }.sortBy { _.startTime }
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/util/HdfsUtils.scala b/streaming/src/main/scala/org/apache/spark/streaming/util/HdfsUtils.scala
index 146577214de17..ef040681adf37 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/util/HdfsUtils.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/util/HdfsUtils.scala
@@ -39,7 +39,7 @@ private[streaming] object HdfsUtils {
           throw new IllegalStateException("File exists and there is no append support!")
         }
       } else {
-        // we dont' want to use hdfs erasure coding, as that lacks support for append and hflush
+        // we don't want to use hdfs erasure coding, as that lacks support for append and hflush
         SparkHadoopUtil.createFile(dfs, dfsPath, false)
       }
     }
@@ -58,7 +58,7 @@ private[streaming] object HdfsUtils {
         // If we are really unlucky, the file may be deleted as we're opening the stream.
         // This can happen as clean up is performed by daemon threads that may be left over from
         // previous runs.
-        if (!dfs.isFile(dfsPath)) null else throw e
+        if (!dfs.getFileStatus(dfsPath).isFile) null else throw e
     }
   }
 
@@ -92,6 +92,10 @@ private[streaming] object HdfsUtils {
   def checkFileExists(path: String, conf: Configuration): Boolean = {
     val hdpPath = new Path(path)
     val fs = getFileSystemForPath(hdpPath, conf)
-    fs.isFile(hdpPath)
+    try {
+      fs.getFileStatus(hdpPath).isFile
+    } catch {
+      case _: FileNotFoundException => false
+    }
   }
 }
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/util/RawTextHelper.scala b/streaming/src/main/scala/org/apache/spark/streaming/util/RawTextHelper.scala
index 9cdfdb8374322..e207dab7de068 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/util/RawTextHelper.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/util/RawTextHelper.scala
@@ -93,7 +93,7 @@ object RawTextHelper {
   }
 
   /**
-   * Warms up the SparkContext in master and slave by running tasks to force JIT kick in
+   * Warms up the SparkContext in master and executor by running tasks to force JIT kick in
    * before real workload starts.
    */
   def warmUp(sc: SparkContext): Unit = {
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/util/RawTextSender.scala b/streaming/src/main/scala/org/apache/spark/streaming/util/RawTextSender.scala
index 5d4fcf8bd1596..a63d50f1334fe 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/util/RawTextSender.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/util/RawTextSender.scala
@@ -26,7 +26,7 @@ import scala.io.Source
 import org.apache.spark.SparkConf
 import org.apache.spark.internal.Logging
 import org.apache.spark.serializer.KryoSerializer
-import org.apache.spark.util.IntParam
+import org.apache.spark.util.{IntParam, Utils}
 
 /**
  * A helper program that sends blocks of Kryo-serialized text strings out on a socket at a
@@ -45,7 +45,7 @@ object RawTextSender extends Logging {
     val Array(IntParam(port), file, IntParam(blockSize), IntParam(bytesPerSec)) = args
 
     // Repeat the input data multiple times to fill in a buffer
-    val lines = Source.fromFile(file).getLines().toArray
+    val lines = Utils.tryWithResource(Source.fromFile(file))(_.getLines().toArray)
     val bufferStream = new ByteArrayOutputStream(blockSize + 1000)
     val ser = new KryoSerializer(new SparkConf()).newInstance()
     val serStream = ser.serializeStream(bufferStream)
diff --git a/streaming/src/test/java/test/org/apache/spark/streaming/JavaAPISuite.java b/streaming/src/test/java/test/org/apache/spark/streaming/JavaAPISuite.java
index c7cde5674f547..8a57b0c58b228 100644
--- a/streaming/src/test/java/test/org/apache/spark/streaming/JavaAPISuite.java
+++ b/streaming/src/test/java/test/org/apache/spark/streaming/JavaAPISuite.java
@@ -1595,7 +1595,7 @@ public void testContextGetOrCreate() throws InterruptedException {
   /* TEST DISABLED: Pending a discussion about checkpoint() semantics with TD
   @SuppressWarnings("unchecked")
   @Test
-  public void testCheckpointofIndividualStream() throws InterruptedException {
+  public void testCheckpointOfIndividualStream() throws InterruptedException {
     List<List<String>> inputData = Arrays.asList(
         Arrays.asList("this", "is"),
         Arrays.asList("a", "test"),
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/DStreamScopeSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/DStreamScopeSuite.scala
index 36036fcd44b04..541a6e2d48b51 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/DStreamScopeSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/DStreamScopeSuite.scala
@@ -190,7 +190,7 @@ class DStreamScopeSuite
     assertDefined(foreachBaseScope)
     assert(foreachBaseScope.get.name === "foreachRDD")
 
-    val rddScopes = generatedRDDs.map { _.scope }
+    val rddScopes = generatedRDDs.map { _.scope }.toSeq
     assertDefined(rddScopes: _*)
     rddScopes.zipWithIndex.foreach { case (rddScope, idx) =>
       assert(rddScope.get.name === "reduceByKey")
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/InputStreamsSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/InputStreamsSuite.scala
index 53ef840864bce..03182ae64db3d 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/InputStreamsSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/InputStreamsSuite.scala
@@ -264,7 +264,7 @@ class InputStreamsSuite extends TestSuiteBase with BeforeAndAfter {
 
         def write(path: Path, text: String): Unit = {
           val out = fs.create(path, true)
-          IOUtils.write(text, out)
+          IOUtils.write(text, out, StandardCharsets.UTF_8)
           out.close()
         }
 
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/JavaTestUtils.scala b/streaming/src/test/scala/org/apache/spark/streaming/JavaTestUtils.scala
index 0c4a64ccc513f..42a5aaba5178f 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/JavaTestUtils.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/JavaTestUtils.scala
@@ -36,7 +36,7 @@ trait JavaTestBase extends TestSuiteBase {
       ssc: JavaStreamingContext,
       data: JList[JList[T]],
       numPartitions: Int): JavaDStream[T] = {
-    val seqData = data.asScala.map(_.asScala)
+    val seqData = data.asScala.map(_.asScala.toSeq).toSeq
 
     implicit val cm: ClassTag[T] =
       implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[T]]
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/LocalStreamingContext.scala b/streaming/src/test/scala/org/apache/spark/streaming/LocalStreamingContext.scala
index 5bf24a9705dc9..99ad8ede78ed4 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/LocalStreamingContext.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/LocalStreamingContext.scala
@@ -30,7 +30,7 @@ import org.apache.spark.internal.Logging
 trait LocalStreamingContext extends BeforeAndAfterEach { self: Suite =>
 
   @transient var ssc: StreamingContext = _
-  @transient var stopSparkContext: Boolean = true
+  @transient val stopSparkContext: Boolean = true
 
   override def afterEach(): Unit = {
     try {
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/MapWithStateSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/MapWithStateSuite.scala
index b2b8d2f41fc80..3ffaa62bd75ac 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/MapWithStateSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/MapWithStateSuite.scala
@@ -541,12 +541,12 @@ class MapWithStateSuite extends SparkFunSuite with LocalStreamingContext
     // Setup the stream computation
     val ssc = new StreamingContext(sc, Seconds(1))
     val inputStream = new TestInputStream(ssc, input, numPartitions = 2)
-    val trackeStateStream = inputStream.map(x => (x, 1)).mapWithState(mapWithStateSpec)
+    val trackedStateStream = inputStream.map(x => (x, 1)).mapWithState(mapWithStateSpec)
     val collectedOutputs = new ConcurrentLinkedQueue[Seq[T]]
-    val outputStream = new TestOutputStream(trackeStateStream, collectedOutputs)
+    val outputStream = new TestOutputStream(trackedStateStream, collectedOutputs)
     val collectedStateSnapshots = new ConcurrentLinkedQueue[Seq[(K, S)]]
     val stateSnapshotStream = new TestOutputStream(
-      trackeStateStream.stateSnapshots(), collectedStateSnapshots)
+      trackedStateStream.stateSnapshots(), collectedStateSnapshots)
     outputStream.register()
     stateSnapshotStream.register()
 
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/MasterFailureTest.scala b/streaming/src/test/scala/org/apache/spark/streaming/MasterFailureTest.scala
index d0a5ababc7cac..4fc9d13cddadc 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/MasterFailureTest.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/MasterFailureTest.scala
@@ -284,7 +284,7 @@ object MasterFailureTest extends Logging {
         })
       }
     }
-    mergedOutput
+    mergedOutput.toSeq
   }
 
   /**
@@ -332,7 +332,7 @@ class KillingThread(ssc: StreamingContext, maxKillWaitTime: Long) extends Thread
   override def run(): Unit = {
     try {
       // If it is the first killing, then allow the first checkpoint to be created
-      var minKillWaitTime = if (MasterFailureTest.killCount == 0) 5000 else 2000
+      val minKillWaitTime = if (MasterFailureTest.killCount == 0) 5000 else 2000
       val killWaitTime = minKillWaitTime + math.abs(Random.nextLong % maxKillWaitTime)
       logInfo("Kill wait time = " + killWaitTime)
       Thread.sleep(killWaitTime)
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/ReceivedBlockHandlerSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/ReceivedBlockHandlerSuite.scala
index 0976494b6d094..425e39c5980a1 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/ReceivedBlockHandlerSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/ReceivedBlockHandlerSuite.scala
@@ -26,8 +26,10 @@ import scala.concurrent.duration._
 import scala.reflect.ClassTag
 
 import org.apache.hadoop.conf.Configuration
-import org.scalatest.{BeforeAndAfter, Matchers}
+import org.scalatest.BeforeAndAfter
 import org.scalatest.concurrent.Eventually._
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark._
 import org.apache.spark.broadcast.BroadcastManager
@@ -69,11 +71,11 @@ abstract class BaseReceivedBlockHandlerSuite(enableEncryption: Boolean)
   val hadoopConf = new Configuration()
   val streamId = 1
   val securityMgr = new SecurityManager(conf, encryptionKey)
-  val broadcastManager = new BroadcastManager(true, conf, securityMgr)
+  val broadcastManager = new BroadcastManager(true, conf)
   val mapOutputTracker = new MapOutputTrackerMaster(conf, broadcastManager, true)
   val shuffleManager = new SortShuffleManager(conf)
   val serializer = new KryoSerializer(conf)
-  var serializerManager = new SerializerManager(serializer, conf, encryptionKey)
+  val serializerManager = new SerializerManager(serializer, conf, encryptionKey)
   val manualClock = new ManualClock
   val blockManagerSize = 10000000
   val blockManagerBuffer = new ArrayBuffer[BlockManager]()
@@ -91,7 +93,7 @@ abstract class BaseReceivedBlockHandlerSuite(enableEncryption: Boolean)
     val blockManagerInfo = new mutable.HashMap[BlockManagerId, BlockManagerInfo]()
     blockManagerMaster = new BlockManagerMaster(rpcEnv.setupEndpoint("blockmanager",
       new BlockManagerMasterEndpoint(rpcEnv, true, conf,
-        new LiveListenerBus(conf), None, blockManagerInfo)),
+        new LiveListenerBus(conf), None, blockManagerInfo, mapOutputTracker)),
       rpcEnv.setupEndpoint("blockmanagerHeartbeat",
       new BlockManagerMasterHeartbeatEndpoint(rpcEnv, true, blockManagerInfo)), conf, true)
 
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/ReceivedBlockTrackerSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/ReceivedBlockTrackerSuite.scala
index 368411cc2214b..420f7f20ce186 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/ReceivedBlockTrackerSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/ReceivedBlockTrackerSuite.scala
@@ -28,8 +28,10 @@ import scala.util.Random
 import org.apache.hadoop.conf.Configuration
 import org.mockito.ArgumentMatchers.any
 import org.mockito.Mockito.{doThrow, reset, spy}
-import org.scalatest.{BeforeAndAfter, Matchers}
+import org.scalatest.BeforeAndAfter
 import org.scalatest.concurrent.Eventually._
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.internal.Logging
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/ReceiverInputDStreamSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/ReceiverInputDStreamSuite.scala
index 6b332206e8f6d..9d4b67bccecaf 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/ReceiverInputDStreamSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/ReceiverInputDStreamSuite.scala
@@ -22,7 +22,6 @@ import scala.util.Random
 import org.apache.spark.{SparkConf, SparkEnv}
 import org.apache.spark.rdd.BlockRDD
 import org.apache.spark.storage.{StorageLevel, StreamBlockId}
-import org.apache.spark.streaming.StreamingConf.RECEIVER_WAL_ENABLE_CONF_KEY
 import org.apache.spark.streaming.dstream.ReceiverInputDStream
 import org.apache.spark.streaming.rdd.WriteAheadLogBackedBlockRDD
 import org.apache.spark.streaming.receiver.{BlockManagerBasedStoreResult, Receiver, WriteAheadLogBasedStoreResult}
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/StreamingContextSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/StreamingContextSuite.scala
index 4eff464dcdafb..29eb1db638627 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/StreamingContextSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/StreamingContextSuite.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.streaming
 
 import java.io.{File, NotSerializableException}
+import java.nio.charset.StandardCharsets
 import java.util.Locale
 import java.util.concurrent.{CountDownLatch, TimeUnit}
 import java.util.concurrent.atomic.AtomicInteger
@@ -293,8 +294,7 @@ class StreamingContextSuite
     }
   }
 
-  // TODO (SPARK-31728): re-enable it
-  ignore("stop gracefully") {
+  test("stop gracefully") {
     val conf = new SparkConf().setMaster(master).setAppName(appName)
     conf.set("spark.dummyTimeConfig", "3600s")
     val sc = new SparkContext(conf)
@@ -910,7 +910,7 @@ class StreamingContextSuite
   def createCorruptedCheckpoint(): String = {
     val checkpointDirectory = Utils.createTempDir().getAbsolutePath()
     val fakeCheckpointFile = Checkpoint.checkpointFile(checkpointDirectory, Time(1000))
-    FileUtils.write(new File(fakeCheckpointFile.toString()), "blablabla")
+    FileUtils.write(new File(fakeCheckpointFile.toString()), "blablabla", StandardCharsets.UTF_8)
     assert(Checkpoint.getCheckpointFiles(checkpointDirectory).nonEmpty)
     checkpointDirectory
   }
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/StreamingListenerSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/StreamingListenerSuite.scala
index 679c58dbae92b..b54d60aa29c4f 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/StreamingListenerSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/StreamingListenerSuite.scala
@@ -25,8 +25,9 @@ import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
 
 import org.mockito.Mockito.{mock, reset, verifyNoMoreInteractions}
-import org.scalatest.Matchers
 import org.scalatest.concurrent.Eventually._
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 import org.scalatest.time.SpanSugar._
 
 import org.apache.spark.SparkException
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/TestSuiteBase.scala b/streaming/src/test/scala/org/apache/spark/streaming/TestSuiteBase.scala
index 55c2950261a07..7ce4343acbdac 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/TestSuiteBase.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/TestSuiteBase.scala
@@ -17,11 +17,10 @@
 
 package org.apache.spark.streaming
 
-import java.io.{File, IOException, ObjectInputStream}
+import java.io.{IOException, ObjectInputStream}
 import java.util.concurrent.{ConcurrentLinkedQueue, TimeUnit}
 
 import scala.collection.JavaConverters._
-import scala.language.implicitConversions
 import scala.reflect.ClassTag
 
 import org.scalatest.BeforeAndAfterEach
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala
index bdc9e9ee2aed1..f797101992573 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala
@@ -21,8 +21,10 @@ import scala.collection.mutable.Queue
 
 import org.openqa.selenium.WebDriver
 import org.openqa.selenium.htmlunit.HtmlUnitDriver
-import org.scalatest._
+import org.scalatest.BeforeAndAfterAll
 import org.scalatest.concurrent.Eventually._
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 import org.scalatest.time.SpanSugar._
 import org.scalatestplus.selenium.WebBrowser
 
@@ -63,7 +65,7 @@ class UISeleniumSuite
       .setMaster("local")
       .setAppName("test")
       .set(UI_ENABLED, true)
-    val ssc = new StreamingContext(conf, Seconds(1))
+    val ssc = new StreamingContext(conf, Milliseconds(100))
     assert(ssc.sc.ui.isDefined, "Spark UI is not started!")
     ssc
   }
@@ -104,7 +106,7 @@ class UISeleniumSuite
         find(cssSelector( """ul li a[href*="streaming"]""")) should not be (None)
       }
 
-      eventually(timeout(10.seconds), interval(50.milliseconds)) {
+      eventually(timeout(10.seconds), interval(500.milliseconds)) {
         // check whether streaming page exists
         go to (sparkUI.webUrl.stripSuffix("/") + "/streaming")
         val h3Text = findAll(cssSelector("h3")).map(_.text).toSeq
@@ -125,24 +127,37 @@ class UISeleniumSuite
 
         // Check batch tables
         val h4Text = findAll(cssSelector("h4")).map(_.text).toSeq
-        h4Text.exists(_.matches("Active Batches \\(\\d+\\)")) should be (true)
         h4Text.exists(_.matches("Completed Batches \\(last \\d+ out of \\d+\\)")) should be (true)
 
-        findAll(cssSelector("""#active-batches-table th""")).map(_.text).toSeq should be {
-          List("Batch Time", "Records", "Scheduling Delay (?)", "Processing Time (?)",
-            "Output Ops: Succeeded/Total", "Status")
-        }
-        findAll(cssSelector("""#completed-batches-table th""")).map(_.text).toSeq should be {
-          List("Batch Time", "Records", "Scheduling Delay (?)", "Processing Time (?)",
-            "Total Delay (?)", "Output Ops: Succeeded/Total")
+        val arrow = 0x25BE.toChar
+        findAll(cssSelector("""#completedBatches-table th""")).map(_.text).toList should be {
+          List(s"Batch Time $arrow", "Records", "Scheduling Delay", "Processing Time",
+            "Total Delay", "Output Ops: Succeeded/Total")
         }
 
-        val batchLinks =
-          findAll(cssSelector("""#completed-batches-table a""")).flatMap(_.attribute("href")).toSeq
+        val pageSize = 1
+        val pagedTablePath = "/streaming/?completedBatches.sort=Batch+Time" +
+          "&completedBatches.desc=true&completedBatches.page=1" +
+          s"&completedBatches.pageSize=$pageSize#completedBatches"
+
+        go to (sparkUI.webUrl.stripSuffix("/") + pagedTablePath)
+        val completedTableRows = findAll(cssSelector("""#completedBatches-table tr"""))
+          .map(_.text).toList
+        // header row + pagesize
+        completedTableRows.length should be (1 + pageSize)
+
+        val sortedBatchTimePath = "/streaming/?&completedBatches.sort=Batch+Time" +
+          s"&completedBatches.desc=false&completedBatches.pageSize=$pageSize#completedBatches"
+
+        // sort batches in ascending order of batch time
+        go to (sparkUI.webUrl.stripSuffix("/") + sortedBatchTimePath)
+
+        val batchLinks = findAll(cssSelector("""#completedBatches-table td a"""))
+          .flatMap(_.attribute("href")).toSeq
         batchLinks.size should be >= 1
 
         // Check a normal batch page
-        go to (batchLinks.last) // Last should be the first batch, so it will have some jobs
+        go to (batchLinks.head) // Head is the first batch, so it will have some jobs
         val summaryText = findAll(cssSelector("li strong")).map(_.text).toSeq
         summaryText should contain ("Batch Duration:")
         summaryText should contain ("Input data size:")
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/rdd/MapWithStateRDDSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/rdd/MapWithStateRDDSuite.scala
index 58ce3a93251a9..f06b1feb8c0cd 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/rdd/MapWithStateRDDSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/rdd/MapWithStateRDDSuite.scala
@@ -320,7 +320,7 @@ class MapWithStateRDDSuite extends SparkFunSuite with RDDCheckpointTester with B
       makeStateRDDWithLongLineageDataRDD, reliableCheckpoint = true, rddCollectFunc _)
 
     /** Generate MapWithStateRDD with parent state RDD having a long lineage */
-    def makeStateRDDWithLongLineageParenttateRDD(
+    def makeStateRDDWithLongLineageParentStateRDD(
         longLineageRDD: RDD[Int]): MapWithStateRDD[Int, Int, Int, Int] = {
 
       // Create a MapWithStateRDD that has a long lineage using the data RDD with a long lineage
@@ -337,9 +337,9 @@ class MapWithStateRDDSuite extends SparkFunSuite with RDDCheckpointTester with B
     }
 
     testRDD(
-      makeStateRDDWithLongLineageParenttateRDD, reliableCheckpoint = true, rddCollectFunc _)
+      makeStateRDDWithLongLineageParentStateRDD, reliableCheckpoint = true, rddCollectFunc _)
     testRDDPartitions(
-      makeStateRDDWithLongLineageParenttateRDD, reliableCheckpoint = true, rddCollectFunc _)
+      makeStateRDDWithLongLineageParentStateRDD, reliableCheckpoint = true, rddCollectFunc _)
   }
 
   test("checkpointing empty state RDD") {
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/receiver/BlockGeneratorSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/receiver/BlockGeneratorSuite.scala
index 4c0dd0d258c53..31456b0b95b18 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/receiver/BlockGeneratorSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/receiver/BlockGeneratorSuite.scala
@@ -23,9 +23,9 @@ import scala.collection.JavaConverters._
 import scala.collection.mutable
 
 import org.scalatest.BeforeAndAfter
-import org.scalatest.Matchers._
 import org.scalatest.concurrent.{Signaler, ThreadSignaler, TimeLimits}
 import org.scalatest.concurrent.Eventually._
+import org.scalatest.matchers.should.Matchers._
 import org.scalatest.time.SpanSugar._
 
 import org.apache.spark.{SparkConf, SparkException, SparkFunSuite}
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/scheduler/ExecutorAllocationManagerSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/scheduler/ExecutorAllocationManagerSuite.scala
index 65efa10bfcf92..c2b039244d01f 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/scheduler/ExecutorAllocationManagerSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/scheduler/ExecutorAllocationManagerSuite.scala
@@ -17,17 +17,18 @@
 
 package org.apache.spark.streaming.scheduler
 
-import org.mockito.ArgumentMatchers.{eq => meq}
+import org.mockito.ArgumentMatchers.{any, eq => meq}
 import org.mockito.Mockito.{never, reset, times, verify, when}
-import org.scalatest.{BeforeAndAfterEach, PrivateMethodTester}
+import org.scalatest.PrivateMethodTester
 import org.scalatest.concurrent.Eventually.{eventually, timeout}
 import org.scalatest.time.SpanSugar._
 import org.scalatestplus.mockito.MockitoSugar
 
 import org.apache.spark.{ExecutorAllocationClient, SparkConf}
-import org.apache.spark.internal.config.{DYN_ALLOCATION_ENABLED, DYN_ALLOCATION_TESTING}
+import org.apache.spark.internal.config.{DECOMMISSION_ENABLED, DYN_ALLOCATION_ENABLED, DYN_ALLOCATION_TESTING}
 import org.apache.spark.internal.config.Streaming._
 import org.apache.spark.resource.ResourceProfile
+import org.apache.spark.scheduler.ExecutorDecommissionInfo
 import org.apache.spark.streaming.{DummyInputDStream, Seconds, StreamingContext, TestSuiteBase}
 import org.apache.spark.util.{ManualClock, Utils}
 
@@ -44,11 +45,22 @@ class ExecutorAllocationManagerSuite extends TestSuiteBase
   }
 
   test("basic functionality") {
+    basicTest(decommissioning = false)
+  }
+
+  test("basic decommissioning") {
+    basicTest(decommissioning = true)
+  }
+
+  def basicTest(decommissioning: Boolean): Unit = {
     // Test that adding batch processing time info to allocation manager
     // causes executors to be requested and killed accordingly
+    conf.set(DECOMMISSION_ENABLED, decommissioning)
 
     // There is 1 receiver, and exec 1 has been allocated to it
-    withAllocationManager(numReceivers = 1) { case (receiverTracker, allocationManager) =>
+    withAllocationManager(numReceivers = 1, conf = conf) {
+      case (receiverTracker, allocationManager) =>
+
       when(receiverTracker.allocatedExecutors).thenReturn(Map(1 -> Some("1")))
 
       /** Add data point for batch processing time and verify executor allocation */
@@ -83,53 +95,67 @@ class ExecutorAllocationManagerSuite extends TestSuiteBase
             Map.empty)}
       }
 
-      /** Verify that a particular executor was killed */
-      def verifyKilledExec(expectedKilledExec: Option[String]): Unit = {
-        if (expectedKilledExec.nonEmpty) {
-          verify(allocationClient, times(1)).killExecutor(meq(expectedKilledExec.get))
+      /** Verify that a particular executor was scaled down. */
+      def verifyScaledDownExec(expectedExec: Option[String]): Unit = {
+        if (expectedExec.nonEmpty) {
+          val decomInfo = ExecutorDecommissionInfo("spark scale down", None)
+          if (decommissioning) {
+            verify(allocationClient, times(1)).decommissionExecutor(
+              meq(expectedExec.get), meq(decomInfo), meq(true), any())
+            verify(allocationClient, never).killExecutor(meq(expectedExec.get))
+          } else {
+            verify(allocationClient, times(1)).killExecutor(meq(expectedExec.get))
+            verify(allocationClient, never).decommissionExecutor(
+              meq(expectedExec.get), meq(decomInfo), meq(true), any())
+          }
         } else {
-          verify(allocationClient, never).killExecutor(null)
+          if (decommissioning) {
+            verify(allocationClient, never).decommissionExecutor(null, null, false)
+            verify(allocationClient, never).decommissionExecutor(null, null, true)
+          } else {
+            verify(allocationClient, never).killExecutor(null)
+          }
         }
       }
 
       // Batch proc time = batch interval, should increase allocation by 1
       addBatchProcTimeAndVerifyAllocation(batchDurationMillis) {
         verifyTotalRequestedExecs(Some(3)) // one already allocated, increase allocation by 1
-        verifyKilledExec(None)
+        verifyScaledDownExec(None)
       }
 
       // Batch proc time = batch interval * 2, should increase allocation by 2
       addBatchProcTimeAndVerifyAllocation(batchDurationMillis * 2) {
         verifyTotalRequestedExecs(Some(4))
-        verifyKilledExec(None)
+        verifyScaledDownExec(None)
       }
 
       // Batch proc time slightly more than the scale up ratio, should increase allocation by 1
       addBatchProcTimeAndVerifyAllocation(
         batchDurationMillis * STREAMING_DYN_ALLOCATION_SCALING_UP_RATIO.defaultValue.get + 1) {
         verifyTotalRequestedExecs(Some(3))
-        verifyKilledExec(None)
+        verifyScaledDownExec(None)
       }
 
       // Batch proc time slightly less than the scale up ratio, should not change allocation
       addBatchProcTimeAndVerifyAllocation(
         batchDurationMillis * STREAMING_DYN_ALLOCATION_SCALING_UP_RATIO.defaultValue.get - 1) {
         verifyTotalRequestedExecs(None)
-        verifyKilledExec(None)
+        verifyScaledDownExec(None)
       }
 
       // Batch proc time slightly more than the scale down ratio, should not change allocation
       addBatchProcTimeAndVerifyAllocation(
         batchDurationMillis * STREAMING_DYN_ALLOCATION_SCALING_DOWN_RATIO.defaultValue.get + 1) {
         verifyTotalRequestedExecs(None)
-        verifyKilledExec(None)
+        verifyScaledDownExec(None)
       }
 
       // Batch proc time slightly more than the scale down ratio, should not change allocation
       addBatchProcTimeAndVerifyAllocation(
         batchDurationMillis * STREAMING_DYN_ALLOCATION_SCALING_DOWN_RATIO.defaultValue.get - 1) {
         verifyTotalRequestedExecs(None)
-        verifyKilledExec(Some("2"))
+        verifyScaledDownExec(Some("2"))
       }
     }
   }
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/scheduler/ReceiverTrackerSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/scheduler/ReceiverTrackerSuite.scala
index 1a0154600bf3c..5a8941de3fafd 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/scheduler/ReceiverTrackerSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/scheduler/ReceiverTrackerSuite.scala
@@ -203,7 +203,7 @@ private[streaming] object RateTestReceiver {
  */
 class StoppableReceiver extends Receiver[Int](StorageLevel.MEMORY_ONLY) {
 
-  var receivingThreadOption: Option[Thread] = None
+  val receivingThreadOption: Option[Thread] = None
 
   def onStart(): Unit = {
     val thread = new Thread() {
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/scheduler/rate/PIDRateEstimatorSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/scheduler/rate/PIDRateEstimatorSuite.scala
index 1a0460cd669af..bad5cd2564988 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/scheduler/rate/PIDRateEstimatorSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/scheduler/rate/PIDRateEstimatorSuite.scala
@@ -20,7 +20,8 @@ package org.apache.spark.streaming.scheduler.rate
 import scala.util.Random
 
 import org.scalatest.Inspectors.forAll
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.streaming.Seconds
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/ui/StreamingJobProgressListenerSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/ui/StreamingJobProgressListenerSuite.scala
index 10f92f9386173..01177f1cca70a 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/ui/StreamingJobProgressListenerSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/ui/StreamingJobProgressListenerSuite.scala
@@ -19,7 +19,8 @@ package org.apache.spark.streaming.ui
 
 import java.util.Properties
 
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.scheduler.SparkListenerJobStart
 import org.apache.spark.streaming.{LocalStreamingContext, _}
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/ui/UIUtilsSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/ui/UIUtilsSuite.scala
index 576083723f8bd..7f96bbfd7b19a 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/ui/UIUtilsSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/ui/UIUtilsSuite.scala
@@ -20,12 +20,13 @@ package org.apache.spark.streaming.ui
 import java.util.TimeZone
 import java.util.concurrent.TimeUnit
 
-import org.scalatest.Matchers
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.ui.{UIUtils => SparkUIUtils}
 
-class UIUtilsSuite extends SparkFunSuite with Matchers{
+class UIUtilsSuite extends SparkFunSuite with Matchers {
 
   test("shortTimeUnitString") {
     assert("ns" === UIUtils.shortTimeUnitString(TimeUnit.NANOSECONDS))
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/util/WriteAheadLogSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/util/WriteAheadLogSuite.scala
index bb60d6fa7bf78..60e04403937a2 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/util/WriteAheadLogSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/util/WriteAheadLogSuite.scala
@@ -612,7 +612,7 @@ object WriteAheadLogSuite {
       }
     }
     writer.close()
-    segments
+    segments.toSeq
   }
 
   /**
@@ -685,7 +685,7 @@ object WriteAheadLogSuite {
     } finally {
       reader.close()
     }
-    buffer
+    buffer.toSeq
   }
 
   /** Read all the data from a log file using reader class and return the list of byte buffers. */
diff --git a/tools/pom.xml b/tools/pom.xml
index 6e806413ef261..8fe8ab358d60c 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.1.0-SNAPSHOT</version>
+    <version>3.2.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala b/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala
index f9bc499961ad7..a6fee8616df11 100644
--- a/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala
+++ b/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala
@@ -24,6 +24,7 @@ import scala.reflect.runtime.universe.runtimeMirror
 import scala.util.Try
 
 import org.clapper.classutil.ClassFinder
+import org.objectweb.asm.Opcodes
 
 /**
  * A tool for generating classes to be excluded during binary checking with MIMA. It is expected
@@ -146,7 +147,7 @@ object GenerateMIMAIgnore {
    * and subpackages both from directories and jars present on the classpath.
    */
   private def getClasses(packageName: String): Set[String] = {
-    val finder = ClassFinder()
+    val finder = ClassFinder(maybeOverrideAsmVersion = Some(Opcodes.ASM7))
     finder
       .getClasses
       .map(_.name)